0 files changed, 0 insertions, 0 deletions
diff --git a/xlators/Makefile.am b/xlators/Makefile.am
index 2a03f905b6d..ef20cbb64fa 100644
--- a/xlators/Makefile.am
+++ b/xlators/Makefile.am
@@ -1,4 +1,13 @@
-SUBDIRS = cluster storage protocol performance debug features encryption mount nfs mgmt system \
-          playground meta
+if BUILD_GNFS
+  GNFS_DIR = nfs
+endif
+
+DIST_SUBDIRS = cluster storage protocol performance debug features \
+          mount nfs mgmt system playground meta
+
+SUBDIRS = cluster storage protocol performance debug features \
+          mount ${GNFS_DIR} mgmt system playground meta
+
+EXTRA_DIST = xlator.sym
 
 CLEANFILES =
diff --git a/xlators/cluster/Makefile.am b/xlators/cluster/Makefile.am
index 903fbb39f12..8e067d5ab58 100644
--- a/xlators/cluster/Makefile.am
+++ b/xlators/cluster/Makefile.am
@@ -1,3 +1,3 @@
-SUBDIRS = stripe afr dht ec
+SUBDIRS = afr dht ec
 
 CLEANFILES = 
diff --git a/xlators/cluster/afr/src/Makefile.am b/xlators/cluster/afr/src/Makefile.am
index ac66bf3bb11..610819b28fc 100644
--- a/xlators/cluster/afr/src/Makefile.am
+++ b/xlators/cluster/afr/src/Makefile.am
@@ -1,4 +1,4 @@
-xlator_LTLIBRARIES = afr.la pump.la
+xlator_LTLIBRARIES = afr.la
 xlatordir = $(libdir)/glusterfs/$(PACKAGE_VERSION)/xlator/cluster
 
 afr_common_source = afr-dir-read.c afr-dir-write.c afr-inode-read.c \
@@ -10,22 +10,19 @@ AFR_SELFHEAL_SOURCES = afr-self-heal-common.c afr-self-heal-data.c \
 	afr-self-heal-entry.c afr-self-heal-metadata.c afr-self-heald.c \
 	afr-self-heal-name.c
 
-afr_la_LDFLAGS = -module -avoid-version
+afr_la_LDFLAGS = -module $(GF_XLATOR_DEFAULT_LDFLAGS)
 afr_la_SOURCES = $(afr_common_source) $(AFR_SELFHEAL_SOURCES) afr.c
 afr_la_LIBADD = $(top_builddir)/libglusterfs/src/libglusterfs.la
 
-pump_la_LDFLAGS = -module -avoid-version
-pump_la_SOURCES =  $(afr_common_source) $(AFR_SELFHEAL_SOURCES) pump.c
-pump_la_LIBADD = $(top_builddir)/libglusterfs/src/libglusterfs.la
-
 noinst_HEADERS = afr.h afr-transaction.h afr-inode-write.h afr-inode-read.h \
 	afr-dir-read.h afr-dir-write.h afr-self-heal.h afr-mem-types.h \
-	afr-common.c afr-self-heald.h pump.h \
+	afr-common.c afr-self-heald.h \
 	$(top_builddir)/xlators/lib/src/libxlator.h afr-messages.h
 
 AM_CPPFLAGS = $(GF_CPPFLAGS) \
 	-I$(top_srcdir)/libglusterfs/src -I$(top_srcdir)/xlators/lib/src \
-	-I$(top_srcdir)/rpc/rpc-lib/src
+	-I$(top_srcdir)/rpc/rpc-lib/src \
+	-I$(top_srcdir)/rpc/xdr/src -I$(top_builddir)/rpc/xdr/src
 
 AM_CFLAGS = -Wall $(GF_CFLAGS)
 
diff --git a/xlators/cluster/afr/src/afr-common.c b/xlators/cluster/afr/src/afr-common.c
index 5654e3ad03d..032ab5c8001 100644
--- a/xlators/cluster/afr/src/afr-common.c
+++ b/xlators/cluster/afr/src/afr-common.c
@@ -15,29 +15,20 @@
 #include <stdlib.h>
 #include <signal.h>
 
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
-
-#include "glusterfs.h"
+#include <glusterfs/glusterfs.h>
 #include "afr.h"
-#include "dict.h"
-#include "xlator.h"
-#include "hashfn.h"
-#include "logging.h"
-#include "stack.h"
-#include "list.h"
-#include "call-stub.h"
-#include "defaults.h"
-#include "common-utils.h"
-#include "compat-errno.h"
-#include "compat.h"
-#include "byte-order.h"
-#include "statedump.h"
-#include "inode.h"
-
-#include "fd.h"
+#include <glusterfs/dict.h>
+#include <glusterfs/hashfn.h>
+#include <glusterfs/list.h>
+#include <glusterfs/call-stub.h>
+#include <glusterfs/defaults.h>
+#include <glusterfs/common-utils.h>
+#include <glusterfs/compat-errno.h>
+#include <glusterfs/compat.h>
+#include <glusterfs/byte-order.h>
+#include <glusterfs/statedump.h>
+#include <glusterfs/events.h>
+#include <glusterfs/upcall-utils.h>
 
 #include "afr-inode-read.h"
 #include "afr-inode-write.h"
@@ -48,56 +39,799 @@
 #include "afr-self-heald.h"
 #include "afr-messages.h"
 
-call_frame_t *
-afr_copy_frame (call_frame_t *base)
+int32_t
+afr_quorum_errno(afr_private_t *priv)
+{
+    return ENOTCONN;
+}
+
+gf_boolean_t
+afr_is_private_directory(afr_private_t *priv, uuid_t pargfid, const char *name,
+                         pid_t pid)
 {
-	afr_local_t *local = NULL;
-	call_frame_t *frame = NULL;
-	int op_errno = 0;
+    if (!__is_root_gfid(pargfid)) {
+        return _gf_false;
+    }
+
+    if (strcmp(name, GF_REPLICATE_TRASH_DIR) == 0) {
+        /*For backward compatibility /.landfill is private*/
+        return _gf_true;
+    }
+
+    if (pid == GF_CLIENT_PID_GSYNCD) {
+        /*geo-rep needs to create/sync private directory on slave because
+         * it appears in changelog*/
+        return _gf_false;
+    }
 
-	frame = copy_frame (base);
-	if (!frame)
-		return NULL;
-	local = AFR_FRAME_INIT (frame, op_errno);
-	if (!local) {
-		AFR_STACK_DESTROY (frame);
-		return NULL;
-	}
+    if (pid == GF_CLIENT_PID_GLFS_HEAL || pid == GF_CLIENT_PID_SELF_HEALD) {
+        if (strcmp(name, priv->anon_inode_name) == 0) {
+            /* anonymous-inode dir is private*/
+            return _gf_true;
+        }
+    } else {
+        if (strncmp(name, AFR_ANON_DIR_PREFIX, strlen(AFR_ANON_DIR_PREFIX)) ==
+            0) {
+            /* anonymous-inode dir prefix is private for geo-rep to work*/
+            return _gf_true;
+        }
+    }
 
-	return frame;
+    return _gf_false;
 }
 
-int
-__afr_inode_ctx_get (xlator_t *this, inode_t *inode, afr_inode_ctx_t **ctx)
+void
+afr_fill_success_replies(afr_local_t *local, afr_private_t *priv,
+                         unsigned char *replies)
 {
-        uint64_t                ctx_int = 0;
-        int                     ret     = -1;
-        afr_inode_ctx_t        *tmp_ctx = NULL;
+    int i = 0;
 
-        ret = __inode_ctx_get (inode, this, &ctx_int);
-        if (ret) {
-                tmp_ctx = GF_CALLOC (1, sizeof (afr_inode_ctx_t),
-                                     gf_afr_mt_inode_ctx_t);
-                if (!tmp_ctx)
-                        goto out;
-
-                ctx_int = (long) tmp_ctx;
-                ret = __inode_ctx_set (inode, this, &ctx_int);
-                if (ret) {
-                        GF_FREE (tmp_ctx);
-                        goto out;
-                }
-                tmp_ctx->spb_choice = -1;
-                tmp_ctx->read_subvol = 0;
+    for (i = 0; i < priv->child_count; i++) {
+        if (local->replies[i].valid && local->replies[i].op_ret == 0) {
+            replies[i] = 1;
         } else {
-                tmp_ctx = (afr_inode_ctx_t *) ctx_int;
+            replies[i] = 0;
         }
+    }
+}
 
-        *ctx = tmp_ctx;
-        ret = 0;
+int
+afr_fav_child_reset_sink_xattrs(void *opaque);
+
+int
+afr_fav_child_reset_sink_xattrs_cbk(int ret, call_frame_t *frame, void *opaque);
+
+static void
+afr_discover_done(call_frame_t *frame, xlator_t *this);
+
+int
+afr_dom_lock_acquire_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                         int op_ret, int op_errno, dict_t *xdata)
+{
+    afr_local_t *local = frame->local;
+    afr_private_t *priv = this->private;
+    int i = (long)cookie;
+
+    local->cont.lk.dom_lock_op_ret[i] = op_ret;
+    local->cont.lk.dom_lock_op_errno[i] = op_errno;
+    if (op_ret < 0) {
+        gf_msg(this->name, GF_LOG_ERROR, op_errno, AFR_MSG_LK_HEAL_DOM,
+               "%s: Failed to acquire %s on %s",
+               uuid_utoa(local->fd->inode->gfid), AFR_LK_HEAL_DOM,
+               priv->children[i]->name);
+    } else {
+        local->cont.lk.dom_locked_nodes[i] = 1;
+    }
+
+    syncbarrier_wake(&local->barrier);
+
+    return 0;
+}
+
+int
+afr_dom_lock_acquire(call_frame_t *frame)
+{
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
+    struct gf_flock flock = {
+        0,
+    };
+    int i = 0;
+
+    priv = frame->this->private;
+    local = frame->local;
+    local->cont.lk.dom_locked_nodes = GF_CALLOC(
+        priv->child_count, sizeof(*local->cont.lk.locked_nodes),
+        gf_afr_mt_char);
+    if (!local->cont.lk.dom_locked_nodes) {
+        return -ENOMEM;
+    }
+    local->cont.lk.dom_lock_op_ret = GF_CALLOC(
+        priv->child_count, sizeof(*local->cont.lk.dom_lock_op_ret),
+        gf_afr_mt_int32_t);
+    if (!local->cont.lk.dom_lock_op_ret) {
+        return -ENOMEM; /* CALLOC'd members are freed in afr_local_cleanup. */
+    }
+    local->cont.lk.dom_lock_op_errno = GF_CALLOC(
+        priv->child_count, sizeof(*local->cont.lk.dom_lock_op_errno),
+        gf_afr_mt_int32_t);
+    if (!local->cont.lk.dom_lock_op_errno) {
+        return -ENOMEM; /* CALLOC'd members are freed in afr_local_cleanup. */
+    }
+    flock.l_type = F_WRLCK;
+
+    AFR_ONALL(frame, afr_dom_lock_acquire_cbk, finodelk, AFR_LK_HEAL_DOM,
+              local->fd, F_SETLK, &flock, NULL);
+
+    if (!afr_has_quorum(local->cont.lk.dom_locked_nodes, frame->this, NULL))
+        goto blocking_lock;
+
+    /*If any of the bricks returned EAGAIN, we still need blocking locks.*/
+    if (AFR_COUNT(local->cont.lk.dom_locked_nodes, priv->child_count) !=
+        priv->child_count) {
+        for (i = 0; i < priv->child_count; i++) {
+            if (local->cont.lk.dom_lock_op_ret[i] == -1 &&
+                local->cont.lk.dom_lock_op_errno[i] == EAGAIN)
+                goto blocking_lock;
+        }
+    }
+
+    return 0;
+
+blocking_lock:
+    afr_dom_lock_release(frame);
+    AFR_ONALL(frame, afr_dom_lock_acquire_cbk, finodelk, AFR_LK_HEAL_DOM,
+              local->fd, F_SETLKW, &flock, NULL);
+    if (!afr_has_quorum(local->cont.lk.dom_locked_nodes, frame->this, NULL)) {
+        afr_dom_lock_release(frame);
+        return -afr_quorum_errno(priv);
+    }
+
+    return 0;
+}
+
+int
+afr_dom_lock_release_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                         int op_ret, int op_errno, dict_t *xdata)
+{
+    afr_local_t *local = frame->local;
+    afr_private_t *priv = this->private;
+    int i = (long)cookie;
+
+    if (op_ret < 0) {
+        gf_msg(this->name, GF_LOG_ERROR, op_errno, AFR_MSG_LK_HEAL_DOM,
+               "%s: Failed to release %s on %s", local->loc.path,
+               AFR_LK_HEAL_DOM, priv->children[i]->name);
+    }
+    local->cont.lk.dom_locked_nodes[i] = 0;
+
+    syncbarrier_wake(&local->barrier);
+
+    return 0;
+}
+
+void
+afr_dom_lock_release(call_frame_t *frame)
+{
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
+    unsigned char *locked_on = NULL;
+    struct gf_flock flock = {
+        0,
+    };
+
+    local = frame->local;
+    priv = frame->this->private;
+    locked_on = local->cont.lk.dom_locked_nodes;
+    if (AFR_COUNT(locked_on, priv->child_count) == 0)
+        return;
+    flock.l_type = F_UNLCK;
+
+    AFR_ONLIST(locked_on, frame, afr_dom_lock_release_cbk, finodelk,
+               AFR_LK_HEAL_DOM, local->fd, F_SETLK, &flock, NULL);
+
+    return;
+}
+
+static void
+afr_lk_heal_info_cleanup(afr_lk_heal_info_t *info)
+{
+    if (!info)
+        return;
+    if (info->xdata_req)
+        dict_unref(info->xdata_req);
+    if (info->fd)
+        fd_unref(info->fd);
+    GF_FREE(info->locked_nodes);
+    GF_FREE(info->child_up_event_gen);
+    GF_FREE(info->child_down_event_gen);
+    GF_FREE(info);
+}
+
+static int
+afr_add_lock_to_saved_locks(call_frame_t *frame, xlator_t *this)
+{
+    afr_private_t *priv = this->private;
+    afr_local_t *local = frame->local;
+    afr_lk_heal_info_t *info = NULL;
+    afr_fd_ctx_t *fd_ctx = NULL;
+    int ret = -ENOMEM;
+
+    info = GF_CALLOC(sizeof(*info), 1, gf_afr_mt_lk_heal_info_t);
+    if (!info) {
+        goto cleanup;
+    }
+    INIT_LIST_HEAD(&info->pos);
+    info->fd = fd_ref(local->fd);
+    info->cmd = local->cont.lk.cmd;
+    info->pid = frame->root->pid;
+    info->flock = local->cont.lk.user_flock;
+    info->xdata_req = dict_copy_with_ref(local->xdata_req, NULL);
+    if (!info->xdata_req) {
+        goto cleanup;
+    }
+    info->lk_owner = frame->root->lk_owner;
+    info->locked_nodes = GF_MALLOC(
+        sizeof(*info->locked_nodes) * priv->child_count, gf_afr_mt_char);
+    if (!info->locked_nodes) {
+        goto cleanup;
+    }
+    memcpy(info->locked_nodes, local->cont.lk.locked_nodes,
+           sizeof(*info->locked_nodes) * priv->child_count);
+    info->child_up_event_gen = GF_CALLOC(sizeof(*info->child_up_event_gen),
+                                         priv->child_count, gf_afr_mt_int32_t);
+    if (!info->child_up_event_gen) {
+        goto cleanup;
+    }
+    info->child_down_event_gen = GF_CALLOC(sizeof(*info->child_down_event_gen),
+                                           priv->child_count,
+                                           gf_afr_mt_int32_t);
+    if (!info->child_down_event_gen) {
+        goto cleanup;
+    }
+
+    LOCK(&local->fd->lock);
+    {
+        fd_ctx = __afr_fd_ctx_get(local->fd, this);
+        if (fd_ctx)
+            fd_ctx->lk_heal_info = info;
+    }
+    UNLOCK(&local->fd->lock);
+    if (!fd_ctx) {
+        goto cleanup;
+    }
+
+    LOCK(&priv->lock);
+    {
+        list_add_tail(&info->pos, &priv->saved_locks);
+    }
+    UNLOCK(&priv->lock);
+
+    return 0;
+cleanup:
+    gf_msg(this->name, GF_LOG_ERROR, -ret, AFR_MSG_LK_HEAL_DOM,
+           "%s: Failed to add lock to healq",
+           uuid_utoa(local->fd->inode->gfid));
+    if (info) {
+        afr_lk_heal_info_cleanup(info);
+        if (fd_ctx) {
+            LOCK(&local->fd->lock);
+            {
+                fd_ctx->lk_heal_info = NULL;
+            }
+            UNLOCK(&local->fd->lock);
+        }
+    }
+    return ret;
+}
+
+static int
+afr_remove_lock_from_saved_locks(afr_local_t *local, xlator_t *this)
+{
+    afr_private_t *priv = this->private;
+    struct gf_flock flock = local->cont.lk.user_flock;
+    afr_lk_heal_info_t *info = NULL;
+    afr_fd_ctx_t *fd_ctx = NULL;
+    int ret = -EINVAL;
+
+    fd_ctx = afr_fd_ctx_get(local->fd, this);
+    if (!fd_ctx || !fd_ctx->lk_heal_info) {
+        goto out;
+    }
+
+    info = fd_ctx->lk_heal_info;
+    if ((info->flock.l_start != flock.l_start) ||
+        (info->flock.l_whence != flock.l_whence) ||
+        (info->flock.l_len != flock.l_len)) {
+        /*TODO: Compare lkowners too.*/
+        goto out;
+    }
+
+    LOCK(&priv->lock);
+    {
+        list_del(&fd_ctx->lk_heal_info->pos);
+    }
+    UNLOCK(&priv->lock);
+
+    afr_lk_heal_info_cleanup(info);
+    fd_ctx->lk_heal_info = NULL;
+    ret = 0;
 out:
-        return ret;
+    if (ret)
+        gf_msg(this->name, GF_LOG_ERROR, -ret, AFR_MSG_LK_HEAL_DOM,
+               "%s: Failed to remove lock from healq",
+               uuid_utoa(local->fd->inode->gfid));
+    return ret;
+}
+
+int
+afr_lock_heal_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                  int32_t op_ret, int32_t op_errno, struct gf_flock *lock,
+                  dict_t *xdata)
+{
+    afr_local_t *local = frame->local;
+    int i = (long)cookie;
+
+    local->replies[i].valid = 1;
+    local->replies[i].op_ret = op_ret;
+    local->replies[i].op_errno = op_errno;
+    if (op_ret != 0) {
+        gf_msg(this->name, GF_LOG_ERROR, op_errno, AFR_MSG_LK_HEAL_DOM,
+               "Failed to heal lock on child %d for %s", i,
+               uuid_utoa(local->fd->inode->gfid));
+    }
+    syncbarrier_wake(&local->barrier);
+    return 0;
+}
+
+int
+afr_getlk_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int32_t op_ret,
+              int32_t op_errno, struct gf_flock *lock, dict_t *xdata)
+{
+    afr_local_t *local = frame->local;
+    int i = (long)cookie;
+
+    local->replies[i].valid = 1;
+    local->replies[i].op_ret = op_ret;
+    local->replies[i].op_errno = op_errno;
+    if (op_ret != 0) {
+        gf_msg(this->name, GF_LOG_ERROR, op_errno, AFR_MSG_LK_HEAL_DOM,
+               "Failed getlk for %s", uuid_utoa(local->fd->inode->gfid));
+    } else {
+        local->cont.lk.getlk_rsp[i] = *lock;
+    }
+
+    syncbarrier_wake(&local->barrier);
+    return 0;
+}
+
+static gf_boolean_t
+afr_does_lk_owner_match(call_frame_t *frame, afr_private_t *priv,
+                        afr_lk_heal_info_t *info)
+{
+    int i = 0;
+    afr_local_t *local = frame->local;
+    struct gf_flock flock = {
+        0,
+    };
+    gf_boolean_t ret = _gf_true;
+    char *wind_on = alloca0(priv->child_count);
+    unsigned char *success_replies = alloca0(priv->child_count);
+    local->cont.lk.getlk_rsp = GF_CALLOC(sizeof(*local->cont.lk.getlk_rsp),
+                                         priv->child_count, gf_afr_mt_gf_lock);
+
+    flock = info->flock;
+    for (i = 0; i < priv->child_count; i++) {
+        if (info->locked_nodes[i])
+            wind_on[i] = 1;
+    }
+
+    AFR_ONLIST(wind_on, frame, afr_getlk_cbk, lk, info->fd, F_GETLK, &flock,
+               info->xdata_req);
+
+    afr_fill_success_replies(local, priv, success_replies);
+    if (AFR_COUNT(success_replies, priv->child_count) == 0) {
+        ret = _gf_false;
+        goto out;
+    }
+
+    for (i = 0; i < priv->child_count; i++) {
+        if (!local->replies[i].valid || local->replies[i].op_ret != 0)
+            continue;
+        if (local->cont.lk.getlk_rsp[i].l_type == F_UNLCK)
+            continue;
+        /*TODO: Do we really need to compare lkowner if F_UNLCK is true?*/
+        if (!is_same_lkowner(&local->cont.lk.getlk_rsp[i].l_owner,
+                             &info->lk_owner)) {
+            ret = _gf_false;
+            break;
+        }
+    }
+out:
+    afr_local_replies_wipe(local, priv);
+    GF_FREE(local->cont.lk.getlk_rsp);
+    local->cont.lk.getlk_rsp = NULL;
+    return ret;
+}
+
+static void
+afr_mark_fd_bad(fd_t *fd, xlator_t *this)
+{
+    afr_fd_ctx_t *fd_ctx = NULL;
+
+    if (!fd)
+        return;
+    LOCK(&fd->lock);
+    {
+        fd_ctx = __afr_fd_ctx_get(fd, this);
+        if (fd_ctx) {
+            fd_ctx->is_fd_bad = _gf_true;
+            fd_ctx->lk_heal_info = NULL;
+        }
+    }
+    UNLOCK(&fd->lock);
+}
+
+static void
+afr_add_lock_to_lkhealq(afr_private_t *priv, afr_lk_heal_info_t *info)
+{
+    LOCK(&priv->lock);
+    {
+        list_del(&info->pos);
+        list_add_tail(&info->pos, &priv->lk_healq);
+    }
+    UNLOCK(&priv->lock);
 }
+
+static void
+afr_lock_heal_do(call_frame_t *frame, afr_private_t *priv,
+                 afr_lk_heal_info_t *info)
+{
+    int i = 0;
+    int op_errno = 0;
+    int32_t *current_event_gen = NULL;
+    afr_local_t *local = frame->local;
+    xlator_t *this = frame->this;
+    char *wind_on = alloca0(priv->child_count);
+    gf_boolean_t retry = _gf_true;
+
+    frame->root->pid = info->pid;
+    lk_owner_copy(&frame->root->lk_owner, &info->lk_owner);
+
+    op_errno = -afr_dom_lock_acquire(frame);
+    if ((op_errno != 0)) {
+        goto release;
+    }
+
+    if (!afr_does_lk_owner_match(frame, priv, info)) {
+        gf_msg(this->name, GF_LOG_WARNING, 0, AFR_MSG_LK_HEAL_DOM,
+               "Ignoring lock heal for %s since lk-onwers mismatch. "
+               "Lock possibly pre-empted by another client.",
+               uuid_utoa(info->fd->inode->gfid));
+        goto release;
+    }
+
+    for (i = 0; i < priv->child_count; i++) {
+        if (info->locked_nodes[i])
+            continue;
+        wind_on[i] = 1;
+    }
+
+    current_event_gen = alloca(priv->child_count);
+    memcpy(current_event_gen, info->child_up_event_gen,
+           priv->child_count * sizeof *current_event_gen);
+    AFR_ONLIST(wind_on, frame, afr_lock_heal_cbk, lk, info->fd, info->cmd,
+               &info->flock, info->xdata_req);
+
+    LOCK(&priv->lock);
+    {
+        for (i = 0; i < priv->child_count; i++) {
+            if (!wind_on[i])
+                continue;
+            if ((!local->replies[i].valid) || (local->replies[i].op_ret != 0)) {
+                continue;
+            }
+
+            if ((current_event_gen[i] == info->child_up_event_gen[i]) &&
+                (current_event_gen[i] > info->child_down_event_gen[i])) {
+                info->locked_nodes[i] = 1;
+                retry = _gf_false;
+                list_del_init(&info->pos);
+                list_add_tail(&info->pos, &priv->saved_locks);
+            } else {
+                /*We received subsequent child up/down events while heal was in
+                 * progress; don't mark child as healed. Attempt again on the
+                 * new child up*/
+                gf_msg(this->name, GF_LOG_ERROR, 0, AFR_MSG_LK_HEAL_DOM,
+                       "Event gen mismatch: skipped healing lock on child %d "
+                       "for %s.",
+                       i, uuid_utoa(info->fd->inode->gfid));
+            }
+        }
+    }
+    UNLOCK(&priv->lock);
+
+release:
+    afr_dom_lock_release(frame);
+    if (retry)
+        afr_add_lock_to_lkhealq(priv, info);
+    return;
+}
+
+static int
+afr_lock_heal_done(int ret, call_frame_t *frame, void *opaque)
+{
+    STACK_DESTROY(frame->root);
+    return 0;
+}
+
+static int
+afr_lock_heal(void *opaque)
+{
+    call_frame_t *frame = (call_frame_t *)opaque;
+    call_frame_t *iter_frame = NULL;
+    xlator_t *this = frame->this;
+    afr_private_t *priv = this->private;
+    afr_lk_heal_info_t *info = NULL;
+    afr_lk_heal_info_t *tmp = NULL;
+    struct list_head healq = {
+        0,
+    };
+    int ret = 0;
+
+    iter_frame = afr_copy_frame(frame);
+    if (!iter_frame) {
+        return ENOMEM;
+    }
+
+    INIT_LIST_HEAD(&healq);
+    LOCK(&priv->lock);
+    {
+        list_splice_init(&priv->lk_healq, &healq);
+    }
+    UNLOCK(&priv->lock);
+
+    list_for_each_entry_safe(info, tmp, &healq, pos)
+    {
+        GF_ASSERT((AFR_COUNT(info->locked_nodes, priv->child_count) <
+                   priv->child_count));
+        ((afr_local_t *)(iter_frame->local))->fd = fd_ref(info->fd);
+        afr_lock_heal_do(iter_frame, priv, info);
+        AFR_STACK_RESET(iter_frame);
+        if (iter_frame->local == NULL) {
+            ret = ENOTCONN;
+            gf_msg(frame->this->name, GF_LOG_ERROR, ENOTCONN,
+                   AFR_MSG_LK_HEAL_DOM,
+                   "Aborting processing of lk_healq."
+                   "Healing will be reattempted on next child up for locks "
+                   "that are still in quorum.");
+            LOCK(&priv->lock);
+            {
+                list_add_tail(&healq, &priv->lk_healq);
+            }
+            UNLOCK(&priv->lock);
+            break;
+        }
+    }
+
+    AFR_STACK_DESTROY(iter_frame);
+    return ret;
+}
+
+static int
+__afr_lock_heal_synctask(xlator_t *this, afr_private_t *priv, int child)
+{
+    int ret = 0;
+    call_frame_t *frame = NULL;
+    afr_lk_heal_info_t *info = NULL;
+    afr_lk_heal_info_t *tmp = NULL;
+
+    if (priv->shd.iamshd)
+        return 0;
+
+    list_for_each_entry_safe(info, tmp, &priv->saved_locks, pos)
+    {
+        info->child_up_event_gen[child] = priv->event_generation;
+        list_del_init(&info->pos);
+        list_add_tail(&info->pos, &priv->lk_healq);
+    }
+
+    frame = create_frame(this, this->ctx->pool);
+    if (!frame)
+        return -1;
+
+    ret = synctask_new(this->ctx->env, afr_lock_heal, afr_lock_heal_done, frame,
+                       frame);
+    if (ret)
+        gf_msg(this->name, GF_LOG_ERROR, ENOMEM, AFR_MSG_LK_HEAL_DOM,
+               "Failed to launch lock heal synctask");
+
+    return ret;
+}
+
+static int
+__afr_mark_pending_lk_heal(xlator_t *this, afr_private_t *priv, int child)
+{
+    afr_lk_heal_info_t *info = NULL;
+    afr_lk_heal_info_t *tmp = NULL;
+
+    if (priv->shd.iamshd)
+        return 0;
+    list_for_each_entry_safe(info, tmp, &priv->saved_locks, pos)
+    {
+        info->child_down_event_gen[child] = priv->event_generation;
+        if (info->locked_nodes[child] == 1)
+            info->locked_nodes[child] = 0;
+        if (!afr_has_quorum(info->locked_nodes, this, NULL)) {
+            /* Since the lock was lost on quorum no. of nodes, we should
+             * not attempt to heal it anymore. Some other client could have
+             * acquired the lock, modified data and released it and this
+             * client wouldn't know about it if we heal it.*/
+            afr_mark_fd_bad(info->fd, this);
+            list_del(&info->pos);
+            afr_lk_heal_info_cleanup(info);
+            /* We're not winding an unlock on the node where the lock is still
+             * present because when fencing logic switches over to the new
+             * client (since we marked the fd bad), it should preempt any
+             * existing lock. */
+        }
+    }
+    return 0;
+}
+
+gf_boolean_t
+afr_is_consistent_io_possible(afr_local_t *local, afr_private_t *priv,
+                              int32_t *op_errno)
+{
+    if (priv->consistent_io && local->call_count != priv->child_count) {
+        gf_msg(THIS->name, GF_LOG_INFO, 0, AFR_MSG_SUBVOLS_DOWN,
+               "All subvolumes are not up");
+        if (op_errno)
+            *op_errno = ENOTCONN;
+        return _gf_false;
+    }
+    return _gf_true;
+}
+
+gf_boolean_t
+afr_is_lock_mode_mandatory(dict_t *xdata)
+{
+    int ret = 0;
+    uint32_t lk_mode = GF_LK_ADVISORY;
+
+    ret = dict_get_uint32(xdata, GF_LOCK_MODE, &lk_mode);
+    if (!ret && lk_mode == GF_LK_MANDATORY)
+        return _gf_true;
+
+    return _gf_false;
+}
+
+call_frame_t *
+afr_copy_frame(call_frame_t *base)
+{
+    afr_local_t *local = NULL;
+    call_frame_t *frame = NULL;
+    int op_errno = 0;
+
+    frame = copy_frame(base);
+    if (!frame)
+        return NULL;
+    local = AFR_FRAME_INIT(frame, op_errno);
+    if (!local) {
+        AFR_STACK_DESTROY(frame);
+        return NULL;
+    }
+
+    return frame;
+}
+
+/* Check if an entry or inode could be undergoing a transaction. */
+gf_boolean_t
+afr_is_possibly_under_txn(afr_transaction_type type, afr_local_t *local,
+                          xlator_t *this)
+{
+    int i = 0;
+    int tmp = 0;
+    afr_private_t *priv = NULL;
+    GF_UNUSED char *key = NULL;
+    int keylen = 0;
+
+    priv = this->private;
+
+    if (type == AFR_ENTRY_TRANSACTION) {
+        key = GLUSTERFS_PARENT_ENTRYLK;
+        keylen = SLEN(GLUSTERFS_PARENT_ENTRYLK);
+    } else if (type == AFR_DATA_TRANSACTION) {
+        /*FIXME: Use GLUSTERFS_INODELK_DOM_COUNT etc. once
+         * pl_inodelk_xattr_fill supports separate keys for different
+         * domains.*/
+        key = GLUSTERFS_INODELK_COUNT;
+        keylen = SLEN(GLUSTERFS_INODELK_COUNT);
+    }
+    for (i = 0; i < priv->child_count; i++) {
+        if (!local->replies[i].xdata)
+            continue;
+        if (dict_get_int32n(local->replies[i].xdata, key, keylen, &tmp) == 0)
+            if (tmp)
+                return _gf_true;
+    }
+
+    return _gf_false;
+}
+
+static void
+afr_inode_ctx_destroy(afr_inode_ctx_t *ctx)
+{
+    int i = 0;
+
+    if (!ctx)
+        return;
+
+    for (i = 0; i < AFR_NUM_CHANGE_LOGS; i++) {
+        GF_FREE(ctx->pre_op_done[i]);
+    }
+
+    GF_FREE(ctx);
+}
+
+int
+__afr_inode_ctx_get(xlator_t *this, inode_t *inode, afr_inode_ctx_t **ctx)
+{
+    uint64_t ctx_int = 0;
+    int ret = -1;
+    int i = -1;
+    int num_locks = -1;
+    afr_inode_ctx_t *ictx = NULL;
+    afr_lock_t *lock = NULL;
+    afr_private_t *priv = this->private;
+
+    ret = __inode_ctx_get(inode, this, &ctx_int);
+    if (ret == 0) {
+        *ctx = (afr_inode_ctx_t *)(uintptr_t)ctx_int;
+        return 0;
+    }
+
+    ictx = GF_CALLOC(1, sizeof(afr_inode_ctx_t), gf_afr_mt_inode_ctx_t);
+    if (!ictx)
+        goto out;
+
+    for (i = 0; i < AFR_NUM_CHANGE_LOGS; i++) {
+        ictx->pre_op_done[i] = GF_CALLOC(sizeof *ictx->pre_op_done[i],
+                                         priv->child_count, gf_afr_mt_int32_t);
+        if (!ictx->pre_op_done[i]) {
+            ret = -ENOMEM;
+            goto out;
+        }
+    }
+
+    num_locks = sizeof(ictx->lock) / sizeof(afr_lock_t);
+    for (i = 0; i < num_locks; i++) {
+        lock = &ictx->lock[i];
+        INIT_LIST_HEAD(&lock->post_op);
+        INIT_LIST_HEAD(&lock->frozen);
+        INIT_LIST_HEAD(&lock->waiting);
+        INIT_LIST_HEAD(&lock->owners);
+    }
+
+    ctx_int = (uint64_t)(uintptr_t)ictx;
+    ret = __inode_ctx_set(inode, this, &ctx_int);
+    if (ret) {
+        goto out;
+    }
+
+    ictx->spb_choice = -1;
+    ictx->read_subvol = 0;
+    ictx->write_subvol = 0;
+    ictx->lock_count = 0;
+    ret = 0;
+    *ctx = ictx;
+out:
+    if (ret) {
+        afr_inode_ctx_destroy(ictx);
+    }
+    return ret;
+}
+
 /*
  * INODE CTX 64-bit VALUE FORMAT FOR SMALL (<= 16) SUBVOL COUNTS:
  *
@@ -129,1349 +863,2258 @@ out:
  */
 
 int
-__afr_inode_read_subvol_get_small (inode_t *inode, xlator_t *this,
-				   unsigned char *data, unsigned char *metadata,
-				   int *event_p)
+__afr_set_in_flight_sb_status(xlator_t *this, afr_local_t *local,
+                              inode_t *inode)
 {
-	afr_private_t *priv = NULL;
-	int ret = -1;
-	uint16_t datamap = 0;
-	uint16_t metadatamap = 0;
-	uint32_t event = 0;
-	uint64_t val = 0;
-	int i = 0;
-        afr_inode_ctx_t *ctx = NULL;
-
-	priv = this->private;
+    int i = 0;
+    int txn_type = 0;
+    int count = 0;
+    int index = -1;
+    uint16_t datamap_old = 0;
+    uint16_t metadatamap_old = 0;
+    uint16_t datamap = 0;
+    uint16_t metadatamap = 0;
+    uint16_t tmp_map = 0;
+    uint16_t mask = 0;
+    uint32_t event = 0;
+    uint64_t val = 0;
+    afr_private_t *priv = NULL;
+
+    priv = this->private;
+    txn_type = local->transaction.type;
+
+    if (txn_type == AFR_DATA_TRANSACTION)
+        val = local->inode_ctx->write_subvol;
+    else
+        val = local->inode_ctx->read_subvol;
+
+    metadatamap_old = metadatamap = (val & 0x000000000000ffff);
+    datamap_old = datamap = (val & 0x00000000ffff0000) >> 16;
+    event = (val & 0xffffffff00000000) >> 32;
+
+    if (txn_type == AFR_DATA_TRANSACTION)
+        tmp_map = datamap;
+    else if (txn_type == AFR_METADATA_TRANSACTION)
+        tmp_map = metadatamap;
+
+    count = gf_bits_count(tmp_map);
+
+    for (i = 0; i < priv->child_count; i++) {
+        if (!local->transaction.failed_subvols[i])
+            continue;
+
+        mask = 1 << i;
+        if (txn_type == AFR_METADATA_TRANSACTION)
+            metadatamap &= ~mask;
+        else if (txn_type == AFR_DATA_TRANSACTION)
+            datamap &= ~mask;
+    }
+
+    switch (txn_type) {
+        case AFR_METADATA_TRANSACTION:
+            if ((metadatamap_old != 0) && (metadatamap == 0) && (count == 1)) {
+                index = gf_bits_index(tmp_map);
+                local->transaction.in_flight_sb_errno = local->replies[index]
+                                                            .op_errno;
+                local->transaction.in_flight_sb = _gf_true;
+                metadatamap |= (1 << index);
+            }
+            if (metadatamap_old != metadatamap) {
+                __afr_inode_need_refresh_set(inode, this);
+            }
+            break;
+
+        case AFR_DATA_TRANSACTION:
+            if ((datamap_old != 0) && (datamap == 0) && (count == 1)) {
+                index = gf_bits_index(tmp_map);
+                local->transaction.in_flight_sb_errno = local->replies[index]
+                                                            .op_errno;
+                local->transaction.in_flight_sb = _gf_true;
+                datamap |= (1 << index);
+            }
+            if (datamap_old != datamap)
+                __afr_inode_need_refresh_set(inode, this);
+            break;
 
-	ret = __afr_inode_ctx_get (this, inode, &ctx);
-	if (ret < 0)
-		return ret;
-
-        val = ctx->read_subvol;
+        default:
+            break;
+    }
 
-	metadatamap = (val & 0x000000000000ffff);
-	datamap =     (val & 0x00000000ffff0000) >> 16;
-	event =       (val & 0xffffffff00000000) >> 32;
+    val = ((uint64_t)metadatamap) | (((uint64_t)datamap) << 16) |
+          (((uint64_t)event) << 32);
 
-	for (i = 0; i < priv->child_count; i++) {
-		if (metadata)
-			metadata[i] = (metadatamap >> i) & 1;
-		if (data)
-			data[i] = (datamap >> i) & 1;
-	}
+    if (txn_type == AFR_DATA_TRANSACTION)
+        local->inode_ctx->write_subvol = val;
+    local->inode_ctx->read_subvol = val;
 
-	if (event_p)
-		*event_p = event;
-	return ret;
+    return 0;
 }
 
+gf_boolean_t
+afr_is_symmetric_error(call_frame_t *frame, xlator_t *this)
+{
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
+    int op_errno = 0;
+    int i_errno = 0;
+    gf_boolean_t matching_errors = _gf_true;
+    int i = 0;
+
+    priv = this->private;
+    local = frame->local;
+
+    for (i = 0; i < priv->child_count; i++) {
+        if (!local->replies[i].valid)
+            continue;
+        if (local->replies[i].op_ret != -1) {
+            /* Operation succeeded on at least one subvol,
+               so it is not a failed-everywhere situation.
+            */
+            matching_errors = _gf_false;
+            break;
+        }
+        i_errno = local->replies[i].op_errno;
+
+        if (i_errno == ENOTCONN) {
+            /* ENOTCONN is not a symmetric error. We do not
+               know if the operation was performed on the
+               backend or not.
+            */
+            matching_errors = _gf_false;
+            break;
+        }
+
+        if (!op_errno) {
+            op_errno = i_errno;
+        } else if (op_errno != i_errno) {
+            /* Mismatching op_errno's */
+            matching_errors = _gf_false;
+            break;
+        }
+    }
+
+    return matching_errors;
+}
 
 int
-__afr_inode_read_subvol_set_small (inode_t *inode, xlator_t *this,
-				   unsigned char *data, unsigned char *metadata,
-				   int event)
+afr_set_in_flight_sb_status(xlator_t *this, call_frame_t *frame, inode_t *inode)
 {
-	afr_private_t *priv = NULL;
-	uint16_t datamap = 0;
-	uint16_t metadatamap = 0;
-	uint64_t val = 0;
-	int i = 0;
-        int ret = -1;
-        afr_inode_ctx_t *ctx = NULL;
+    int ret = -1;
+    afr_private_t *priv = NULL;
+    afr_local_t *local = NULL;
 
-	priv = this->private;
+    priv = this->private;
+    local = frame->local;
 
-        ret = __afr_inode_ctx_get (this, inode, &ctx);
-        if (ret)
-                goto out;
+    /* If this transaction saw no failures, then exit. */
+    if (AFR_COUNT(local->transaction.failed_subvols, priv->child_count) == 0)
+        return 0;
 
-	for (i = 0; i < priv->child_count; i++) {
-		if (data[i])
-			datamap |= (1 << i);
-		if (metadata[i])
-			metadatamap |= (1 << i);
-	}
+    if (afr_is_symmetric_error(frame, this))
+        return 0;
 
-	val = ((uint64_t) metadatamap) |
-		(((uint64_t) datamap) << 16) |
-		(((uint64_t) event) << 32);
+    LOCK(&inode->lock);
+    {
+        ret = __afr_set_in_flight_sb_status(this, local, inode);
+    }
+    UNLOCK(&inode->lock);
 
-        ctx->read_subvol = val;
+    return ret;
+}
 
-        ret = 0;
-out:
+int
+__afr_inode_read_subvol_get_small(inode_t *inode, xlator_t *this,
+                                  unsigned char *data, unsigned char *metadata,
+                                  int *event_p)
+{
+    afr_private_t *priv = NULL;
+    int ret = -1;
+    uint16_t datamap = 0;
+    uint16_t metadatamap = 0;
+    uint32_t event = 0;
+    uint64_t val = 0;
+    int i = 0;
+    afr_inode_ctx_t *ctx = NULL;
+
+    priv = this->private;
+
+    ret = __afr_inode_ctx_get(this, inode, &ctx);
+    if (ret < 0)
         return ret;
+
+    val = ctx->read_subvol;
+
+    metadatamap = (val & 0x000000000000ffff);
+    datamap = (val & 0x00000000ffff0000) >> 16;
+    event = (val & 0xffffffff00000000) >> 32;
+
+    for (i = 0; i < priv->child_count; i++) {
+        if (metadata)
+            metadata[i] = (metadatamap >> i) & 1;
+        if (data)
+            data[i] = (datamap >> i) & 1;
+    }
+
+    if (event_p)
+        *event_p = event;
+    return ret;
 }
 
 int
-__afr_inode_read_subvol_reset_small (inode_t *inode, xlator_t *this)
+__afr_inode_read_subvol_set_small(inode_t *inode, xlator_t *this,
+                                  unsigned char *data, unsigned char *metadata,
+                                  int event)
 {
-	int ret = -1;
-	uint16_t datamap = 0;
-	uint16_t metadatamap = 0;
-	uint32_t event = 0;
-	uint64_t val = 0;
-        afr_inode_ctx_t *ctx = NULL;
+    afr_private_t *priv = NULL;
+    uint16_t datamap = 0;
+    uint16_t metadatamap = 0;
+    uint64_t val = 0;
+    int i = 0;
+    int ret = -1;
+    afr_inode_ctx_t *ctx = NULL;
 
-	ret = __afr_inode_ctx_get (this, inode, &ctx);
-        if (ret)
-                return ret;
+    priv = this->private;
 
-        val = ctx->read_subvol;
+    ret = __afr_inode_ctx_get(this, inode, &ctx);
+    if (ret)
+        goto out;
 
-	metadatamap = (val & 0x000000000000ffff) >> 0;
-	datamap =     (val & 0x00000000ffff0000) >> 16;
-	event = 0;
+    for (i = 0; i < priv->child_count; i++) {
+        if (data[i])
+            datamap |= (1 << i);
+        if (metadata[i])
+            metadatamap |= (1 << i);
+    }
 
-	val = ((uint64_t) metadatamap) |
-		(((uint64_t) datamap) << 16) |
-		(((uint64_t) event) << 32);
+    val = ((uint64_t)metadatamap) | (((uint64_t)datamap) << 16) |
+          (((uint64_t)event) << 32);
 
-        ctx->read_subvol = val;
+    ctx->read_subvol = val;
 
-        return ret;
+    ret = 0;
+out:
+    return ret;
 }
 
-
 int
-__afr_inode_read_subvol_get (inode_t *inode, xlator_t *this,
-			     unsigned char *data, unsigned char *metadata,
-			     int *event_p)
+__afr_inode_read_subvol_get(inode_t *inode, xlator_t *this, unsigned char *data,
+                            unsigned char *metadata, int *event_p)
 {
-	afr_private_t *priv = NULL;
-	int ret = -1;
+    afr_private_t *priv = NULL;
+    int ret = -1;
 
-	priv = this->private;
+    priv = this->private;
 
-	if (priv->child_count <= 16)
-		ret = __afr_inode_read_subvol_get_small (inode, this, data,
-							 metadata, event_p);
-	else
-		/* TBD: allocate structure with array and read from it */
-		ret = -1;
+    if (priv->child_count <= 16)
+        ret = __afr_inode_read_subvol_get_small(inode, this, data, metadata,
+                                                event_p);
+    else
+        /* TBD: allocate structure with array and read from it */
+        ret = -1;
 
-	return ret;
+    return ret;
 }
 
 int
-__afr_inode_split_brain_choice_get (inode_t *inode, xlator_t *this,
-			            int *spb_choice)
+__afr_inode_split_brain_choice_get(inode_t *inode, xlator_t *this,
+                                   int *spb_choice)
 {
-        afr_inode_ctx_t *ctx = NULL;
-        int ret = -1;
+    afr_inode_ctx_t *ctx = NULL;
+    int ret = -1;
 
-        ret = __afr_inode_ctx_get (this, inode, &ctx);
-        if (ret < 0)
-                return ret;
+    ret = __afr_inode_ctx_get(this, inode, &ctx);
+    if (ret < 0)
+        return ret;
 
-        *spb_choice = ctx->spb_choice;
-        return 0;
+    *spb_choice = ctx->spb_choice;
+    return 0;
 }
 
 int
-__afr_inode_read_subvol_set (inode_t *inode, xlator_t *this, unsigned char *data,
-			     unsigned char *metadata, int event)
+__afr_inode_read_subvol_set(inode_t *inode, xlator_t *this, unsigned char *data,
+                            unsigned char *metadata, int event)
 {
-	afr_private_t *priv = NULL;
-	int ret = -1;
+    afr_private_t *priv = NULL;
+    int ret = -1;
 
-	priv = this->private;
+    priv = this->private;
 
-	if (priv->child_count <= 16)
-		ret = __afr_inode_read_subvol_set_small (inode, this, data,
-							 metadata, event);
-	else
-		ret = -1;
+    if (priv->child_count <= 16)
+        ret = __afr_inode_read_subvol_set_small(inode, this, data, metadata,
+                                                event);
+    else
+        ret = -1;
 
-	return ret;
+    return ret;
 }
 
 int
-__afr_inode_split_brain_choice_set (inode_t *inode, xlator_t *this,
-                                    int spb_choice)
+__afr_inode_split_brain_choice_set(inode_t *inode, xlator_t *this,
+                                   int spb_choice)
 {
-        afr_inode_ctx_t *ctx = NULL;
-	int ret = -1;
+    afr_inode_ctx_t *ctx = NULL;
+    int ret = -1;
 
-	ret = __afr_inode_ctx_get (this, inode, &ctx);
-        if (ret)
-                goto out;
+    ret = __afr_inode_ctx_get(this, inode, &ctx);
+    if (ret)
+        goto out;
 
-        ctx->spb_choice = spb_choice;
+    ctx->spb_choice = spb_choice;
 
-        ret = 0;
+    ret = 0;
 out:
-        return ret;
+    return ret;
 }
 
 int
-__afr_inode_read_subvol_reset (inode_t *inode, xlator_t *this)
+afr_inode_read_subvol_get(inode_t *inode, xlator_t *this, unsigned char *data,
+                          unsigned char *metadata, int *event_p)
 {
-	afr_private_t *priv = NULL;
-	int ret = -1;
+    int ret = -1;
 
-	priv = this->private;
+    GF_VALIDATE_OR_GOTO(this->name, inode, out);
 
-	if (priv->child_count <= 16)
-		ret = __afr_inode_read_subvol_reset_small (inode, this);
-	else
-		ret = -1;
+    LOCK(&inode->lock);
+    {
+        ret = __afr_inode_read_subvol_get(inode, this, data, metadata, event_p);
+    }
+    UNLOCK(&inode->lock);
+out:
+    return ret;
+}
 
-	return ret;
+int
+afr_inode_get_readable(call_frame_t *frame, inode_t *inode, xlator_t *this,
+                       unsigned char *readable, int *event_p, int type)
+{
+    afr_private_t *priv = this->private;
+    afr_local_t *local = frame->local;
+    unsigned char *data = alloca0(priv->child_count);
+    unsigned char *metadata = alloca0(priv->child_count);
+    int data_count = 0;
+    int metadata_count = 0;
+    int event_generation = 0;
+    int ret = 0;
+
+    ret = afr_inode_read_subvol_get(inode, this, data, metadata,
+                                    &event_generation);
+    if (ret == -1)
+        return -EIO;
+
+    data_count = AFR_COUNT(data, priv->child_count);
+    metadata_count = AFR_COUNT(metadata, priv->child_count);
+
+    if (inode->ia_type == IA_IFDIR) {
+        /* For directories, allow even if it is in data split-brain. */
+        if (type == AFR_METADATA_TRANSACTION || local->op == GF_FOP_STAT ||
+            local->op == GF_FOP_FSTAT) {
+            if (!metadata_count)
+                return -EIO;
+        }
+    } else {
+        /* For files, abort in case of data/metadata split-brain. */
+        if (!data_count || !metadata_count) {
+            return -EIO;
+        }
+    }
+
+    if (type == AFR_METADATA_TRANSACTION && readable)
+        memcpy(readable, metadata, priv->child_count * sizeof *metadata);
+    if (type == AFR_DATA_TRANSACTION && readable) {
+        if (!data_count)
+            memcpy(readable, local->child_up,
+                   priv->child_count * sizeof *readable);
+        else
+            memcpy(readable, data, priv->child_count * sizeof *data);
+    }
+    if (event_p)
+        *event_p = event_generation;
+    return 0;
 }
 
+static int
+afr_inode_split_brain_choice_get(inode_t *inode, xlator_t *this,
+                                 int *spb_choice)
+{
+    int ret = -1;
+    GF_VALIDATE_OR_GOTO(this->name, inode, out);
+
+    LOCK(&inode->lock);
+    {
+        ret = __afr_inode_split_brain_choice_get(inode, this, spb_choice);
+    }
+    UNLOCK(&inode->lock);
+out:
+    return ret;
+}
 
+/*
+ * frame is used to get the favourite policy. Since
+ * afr_inode_split_brain_choice_get was called with afr_open, it is possible to
+ * have a frame with out local->replies. So in that case, frame is passed as
+ * null, hence this function will handle the frame NULL case.
+ */
 int
-afr_inode_read_subvol_get (inode_t *inode, xlator_t *this, unsigned char *data,
-			   unsigned char *metadata, int *event_p)
+afr_split_brain_read_subvol_get(inode_t *inode, xlator_t *this,
+                                call_frame_t *frame, int *spb_subvol)
 {
-	int ret = -1;
+    int ret = -1;
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
+
+    GF_VALIDATE_OR_GOTO("afr", this, out);
+    GF_VALIDATE_OR_GOTO(this->name, this->private, out);
+    GF_VALIDATE_OR_GOTO(this->name, inode, out);
+    GF_VALIDATE_OR_GOTO(this->name, spb_subvol, out);
 
-        GF_VALIDATE_OR_GOTO (this->name, inode, out);
+    priv = this->private;
+
+    ret = afr_inode_split_brain_choice_get(inode, this, spb_subvol);
+    if (*spb_subvol < 0 && priv->fav_child_policy && frame && frame->local) {
+        local = frame->local;
+        *spb_subvol = afr_sh_get_fav_by_policy(this, local->replies, inode,
+                                               NULL);
+        if (*spb_subvol >= 0) {
+            ret = 0;
+        }
+    }
 
-        LOCK(&inode->lock);
-	{
-		ret = __afr_inode_read_subvol_get (inode, this, data,
-						   metadata, event_p);
-	}
-	UNLOCK(&inode->lock);
 out:
-        return ret;
+    return ret;
 }
-
 int
-afr_inode_split_brain_choice_get (inode_t *inode, xlator_t *this,
-                                  int *spb_choice)
+afr_inode_read_subvol_set(inode_t *inode, xlator_t *this, unsigned char *data,
+                          unsigned char *metadata, int event)
 {
-	int ret = -1;
+    int ret = -1;
 
-        GF_VALIDATE_OR_GOTO (this->name, inode, out);
+    GF_VALIDATE_OR_GOTO(this->name, inode, out);
 
-	LOCK(&inode->lock);
-	{
-		ret = __afr_inode_split_brain_choice_get (inode, this,
-                                                          spb_choice);
-	}
-	UNLOCK(&inode->lock);
+    LOCK(&inode->lock);
+    {
+        ret = __afr_inode_read_subvol_set(inode, this, data, metadata, event);
+    }
+    UNLOCK(&inode->lock);
 out:
-        return ret;
+    return ret;
 }
 
-
 int
-afr_inode_read_subvol_set (inode_t *inode, xlator_t *this, unsigned char *data,
-			   unsigned char *metadata, int event)
+afr_inode_split_brain_choice_set(inode_t *inode, xlator_t *this, int spb_choice)
 {
-	int ret = -1;
+    int ret = -1;
 
-        GF_VALIDATE_OR_GOTO (this->name, inode, out);
+    GF_VALIDATE_OR_GOTO(this->name, inode, out);
 
-        LOCK(&inode->lock);
-	{
-		ret = __afr_inode_read_subvol_set (inode, this, data, metadata,
-						   event);
-	}
-	UNLOCK(&inode->lock);
+    LOCK(&inode->lock);
+    {
+        ret = __afr_inode_split_brain_choice_set(inode, this, spb_choice);
+    }
+    UNLOCK(&inode->lock);
 out:
-        return ret;
+    return ret;
 }
 
-
-int
-afr_inode_split_brain_choice_set (inode_t *inode, xlator_t *this,
-                                  int spb_choice)
+/* The caller of this should perform afr_inode_refresh, if this function
+ * returns _gf_true
+ */
+gf_boolean_t
+afr_is_inode_refresh_reqd(inode_t *inode, xlator_t *this, int event_gen1,
+                          int event_gen2)
 {
-	int ret = -1;
+    gf_boolean_t need_refresh = _gf_false;
+    afr_inode_ctx_t *ctx = NULL;
+    int ret = -1;
+
+    GF_VALIDATE_OR_GOTO(this->name, inode, out);
 
-        GF_VALIDATE_OR_GOTO (this->name, inode, out);
+    LOCK(&inode->lock);
+    {
+        ret = __afr_inode_ctx_get(this, inode, &ctx);
+        if (ret)
+            goto unlock;
 
-	LOCK(&inode->lock);
-	{
-		ret = __afr_inode_split_brain_choice_set (inode, this,
-                                                          spb_choice);
-	}
-	UNLOCK(&inode->lock);
+        need_refresh = ctx->need_refresh;
+        /* Hoping that the caller will do inode_refresh followed by
+         * this, hence setting the need_refresh to false */
+        ctx->need_refresh = _gf_false;
+    }
+unlock:
+    UNLOCK(&inode->lock);
+
+    if (event_gen1 != event_gen2)
+        need_refresh = _gf_true;
 out:
-	return ret;
+    return need_refresh;
 }
 
+int
+__afr_inode_need_refresh_set(inode_t *inode, xlator_t *this)
+{
+    int ret = -1;
+    afr_inode_ctx_t *ctx = NULL;
+
+    ret = __afr_inode_ctx_get(this, inode, &ctx);
+    if (ret == 0) {
+        ctx->need_refresh = _gf_true;
+    }
+
+    return ret;
+}
 
 int
-afr_inode_read_subvol_reset (inode_t *inode, xlator_t *this)
+afr_inode_need_refresh_set(inode_t *inode, xlator_t *this)
 {
-	int ret = -1;
+    int ret = -1;
 
-        GF_VALIDATE_OR_GOTO (this->name, inode, out);
+    GF_VALIDATE_OR_GOTO(this->name, inode, out);
 
-        LOCK(&inode->lock);
-        {
-                ret = __afr_inode_read_subvol_reset (inode, this);
-        }
-        UNLOCK(&inode->lock);
+    LOCK(&inode->lock);
+    {
+        ret = __afr_inode_need_refresh_set(inode, this);
+    }
+    UNLOCK(&inode->lock);
 out:
-	return ret;
+    return ret;
 }
 
-
 int
-afr_accused_fill (xlator_t *this, dict_t *xdata, unsigned char *accused,
-		  afr_transaction_type type)
+afr_spb_choice_timeout_cancel(xlator_t *this, inode_t *inode)
 {
-	afr_private_t *priv = NULL;
-	int i = 0;
-	int idx = afr_index_for_transaction_type (type);
-	void *pending_raw = NULL;
-	int pending[3];
-	int ret = 0;
+    afr_inode_ctx_t *ctx = NULL;
+    int ret = -1;
 
-	priv = this->private;
+    if (!inode)
+        return ret;
 
-	for (i = 0; i < priv->child_count; i++) {
-		ret = dict_get_ptr (xdata, priv->pending_key[i],
-				    &pending_raw);
-		if (ret) /* no pending flags */
-			continue;
-		memcpy (pending, pending_raw, sizeof(pending));
+    LOCK(&inode->lock);
+    {
+        ret = __afr_inode_ctx_get(this, inode, &ctx);
+        if (ret < 0 || !ctx) {
+            UNLOCK(&inode->lock);
+            gf_msg(this->name, GF_LOG_WARNING, 0,
+                   AFR_MSG_SPLIT_BRAIN_CHOICE_ERROR,
+                   "Failed to cancel split-brain choice timer.");
+            goto out;
+        }
+        ctx->spb_choice = -1;
+        if (ctx->timer) {
+            gf_timer_call_cancel(this->ctx, ctx->timer);
+            ctx->timer = NULL;
+        }
+        ret = 0;
+    }
+    UNLOCK(&inode->lock);
+out:
+    return ret;
+}
 
-		if (ntoh32 (pending[idx]))
-			accused[i] = 1;
-	}
+void
+afr_set_split_brain_choice_cbk(void *data)
+{
+    inode_t *inode = data;
+    xlator_t *this = THIS;
 
-	return 0;
+    afr_spb_choice_timeout_cancel(this, inode);
+    inode_invalidate(inode);
+    inode_unref(inode);
+    return;
 }
 
-
 int
-afr_accuse_smallfiles (xlator_t *this, struct afr_reply *replies,
-		       unsigned char *data_accused)
+afr_set_split_brain_choice(int ret, call_frame_t *frame, void *opaque)
 {
-	int i = 0;
-	afr_private_t *priv = NULL;
-	uint64_t maxsize = 0;
+    int op_errno = ENOMEM;
+    afr_private_t *priv = NULL;
+    afr_inode_ctx_t *ctx = NULL;
+    inode_t *inode = NULL;
+    loc_t *loc = NULL;
+    xlator_t *this = NULL;
+    afr_spbc_timeout_t *data = opaque;
+    struct timespec delta = {
+        0,
+    };
+    gf_boolean_t timer_set = _gf_false;
+    gf_boolean_t timer_cancelled = _gf_false;
+    gf_boolean_t timer_reset = _gf_false;
+    int old_spb_choice = -1;
+
+    frame = data->frame;
+    loc = data->loc;
+    this = frame->this;
+    priv = this->private;
+
+    if (ret) {
+        op_errno = -ret;
+        ret = -1;
+        goto out;
+    }
+
+    delta.tv_sec = priv->spb_choice_timeout;
+    delta.tv_nsec = 0;
+
+    if (!loc->inode) {
+        ret = -1;
+        op_errno = EINVAL;
+        goto out;
+    }
+
+    if (!(data->d_spb || data->m_spb)) {
+        gf_msg(this->name, GF_LOG_WARNING, 0, AFR_MSG_SPLIT_BRAIN_CHOICE_ERROR,
+               "Cannot set "
+               "replica.split-brain-choice on %s. File is"
+               " not in data/metadata split-brain.",
+               uuid_utoa(loc->gfid));
+        ret = -1;
+        op_errno = EINVAL;
+        goto out;
+    }
+
+    /*
+     * we're ref'ing the inode before LOCK like it is done elsewhere in the
+     * code. If we ref after LOCK, coverity complains of possible deadlocks.
+     */
+    inode = inode_ref(loc->inode);
+
+    LOCK(&inode->lock);
+    {
+        ret = __afr_inode_ctx_get(this, inode, &ctx);
+        if (ret) {
+            UNLOCK(&inode->lock);
+            gf_msg(this->name, GF_LOG_ERROR, 0,
+                   AFR_MSG_SPLIT_BRAIN_CHOICE_ERROR,
+                   "Failed to get inode_ctx for %s", loc->name);
+            goto post_unlock;
+        }
 
-	priv = this->private;
+        old_spb_choice = ctx->spb_choice;
+        ctx->spb_choice = data->spb_child_index;
 
-	for (i = 0; i < priv->child_count; i++) {
-		if (data_accused[i])
-			continue;
-		if (replies[i].poststat.ia_size > maxsize)
-			maxsize = replies[i].poststat.ia_size;
-	}
+        /* Possible changes in spb-choice :
+         *         valid to -1    : cancel timer and unref
+         *         valid to valid : cancel timer and inject new one
+         *         -1    to -1    : unref and do not do anything
+         *         -1 to valid    : inject timer
+         */
 
-	for (i = 0; i < priv->child_count; i++) {
-		if (data_accused[i])
-			continue;
-		if (replies[i].poststat.ia_size < maxsize)
-			data_accused[i] = 1;
-	}
+        /* ctx->timer is NULL iff previous value of
+         * ctx->spb_choice is -1
+         */
+        if (ctx->timer) {
+            if (ctx->spb_choice == -1) {
+                if (!gf_timer_call_cancel(this->ctx, ctx->timer)) {
+                    ctx->timer = NULL;
+                    timer_cancelled = _gf_true;
+                }
+                /* If timer cancel failed here it means that the
+                 *  previous cbk will be executed which will set
+                 *  spb_choice to -1. So we can consider the
+                 *  'valid to -1' case to be a success
+                 *  (i.e. ret = 0) and goto unlock.
+                 */
+                goto unlock;
+            }
+            goto reset_timer;
+        } else {
+            if (ctx->spb_choice == -1)
+                goto unlock;
+            goto set_timer;
+        }
 
-	return 0;
+    reset_timer:
+        ret = gf_timer_call_cancel(this->ctx, ctx->timer);
+        if (ret != 0) {
+            /* We need to bail out now instead of launching a new
+             * timer. Otherwise the cbk of the previous timer event
+             * will cancel the new ctx->timer.
+             */
+            ctx->spb_choice = old_spb_choice;
+            ret = -1;
+            op_errno = EAGAIN;
+            goto unlock;
+        }
+        ctx->timer = NULL;
+        timer_reset = _gf_true;
+
+    set_timer:
+        ctx->timer = gf_timer_call_after(this->ctx, delta,
+                                         afr_set_split_brain_choice_cbk, inode);
+        if (!ctx->timer) {
+            ctx->spb_choice = old_spb_choice;
+            ret = -1;
+            op_errno = ENOMEM;
+        }
+        if (!timer_reset && ctx->timer)
+            timer_set = _gf_true;
+        if (timer_reset && !ctx->timer)
+            timer_cancelled = _gf_true;
+    }
+unlock:
+    UNLOCK(&inode->lock);
+post_unlock:
+    if (!timer_set)
+        inode_unref(inode);
+    if (timer_cancelled)
+        inode_unref(inode);
+    /*
+     * We need to invalidate the inode to prevent the kernel from serving
+     * reads from an older cached value despite a change in spb_choice to
+     * a new value.
+     */
+    inode_invalidate(inode);
+out:
+    GF_FREE(data);
+    AFR_STACK_UNWIND(setxattr, frame, ret, op_errno, NULL);
+    return 0;
 }
 
-
 int
-afr_replies_interpret (call_frame_t *frame, xlator_t *this, inode_t *inode)
-{
-	afr_local_t *local = NULL;
-	afr_private_t *priv = NULL;
-	struct afr_reply *replies = NULL;
-	int event_generation = 0;
-	int i = 0;
-	unsigned char *data_accused = NULL;
-	unsigned char *metadata_accused = NULL;
-	unsigned char *data_readable = NULL;
-	unsigned char *metadata_readable = NULL;
-	int ret = 0;
-
-	local = frame->local;
-	priv = this->private;
-	replies = local->replies;
-	event_generation = local->event_generation;
-
-	data_accused = alloca0 (priv->child_count);
-	data_readable = alloca0 (priv->child_count);
-	metadata_accused = alloca0 (priv->child_count);
-	metadata_readable = alloca0 (priv->child_count);
-
-	for (i = 0; i < priv->child_count; i++) {
-		data_readable[i] = 1;
-		metadata_readable[i] = 1;
-	}
-
-	for (i = 0; i < priv->child_count; i++) {
-		if (!replies[i].valid) {
-			data_readable[i] = 0;
-			metadata_readable[i] = 0;
-			continue;
-		}
-
-		if (replies[i].op_ret == -1) {
-			data_readable[i] = 0;
-			metadata_readable[i] = 0;
-			continue;
-		}
-
-		afr_accused_fill (this, replies[i].xdata, data_accused,
-				  (replies[i].poststat.ia_type == IA_IFDIR) ?
-				   AFR_ENTRY_TRANSACTION : AFR_DATA_TRANSACTION);
-
-		afr_accused_fill (this, replies[i].xdata,
-				  metadata_accused, AFR_METADATA_TRANSACTION);
-
-	}
-
-	if (inode->ia_type != IA_IFDIR)
-		afr_accuse_smallfiles (this, replies, data_accused);
-
-	for (i = 0; i < priv->child_count; i++) {
-		if (data_accused[i]) {
-			data_readable[i] = 0;
-			ret = 1;
-		}
-		if (metadata_accused[i]) {
-			metadata_readable[i] = 0;
-			ret = 1;
-		}
-	}
-
-	afr_inode_read_subvol_set (inode, this, data_readable,
-				   metadata_readable, event_generation);
-	return ret;
+afr_accused_fill(xlator_t *this, dict_t *xdata, unsigned char *accused,
+                 afr_transaction_type type)
+{
+    afr_private_t *priv = NULL;
+    int i = 0;
+    int idx = afr_index_for_transaction_type(type);
+    void *pending_raw = NULL;
+    int pending[3];
+    int ret = 0;
+
+    priv = this->private;
+
+    for (i = 0; i < priv->child_count; i++) {
+        ret = dict_get_ptr(xdata, priv->pending_key[i], &pending_raw);
+        if (ret) /* no pending flags */
+            continue;
+        memcpy(pending, pending_raw, sizeof(pending));
+
+        if (ntoh32(pending[idx]))
+            accused[i] = 1;
+    }
+
+    return 0;
 }
 
-
-
 int
-afr_refresh_selfheal_done (int ret, call_frame_t *heal, void *opaque)
+afr_accuse_smallfiles(xlator_t *this, struct afr_reply *replies,
+                      unsigned char *data_accused)
 {
-	if (heal)
-		STACK_DESTROY (heal->root);
-	return 0;
+    int i = 0;
+    afr_private_t *priv = NULL;
+    uint64_t maxsize = 0;
+
+    priv = this->private;
+
+    for (i = 0; i < priv->child_count; i++) {
+        if (replies[i].valid && replies[i].xdata &&
+            dict_get_sizen(replies[i].xdata, GLUSTERFS_BAD_INODE))
+            continue;
+        if (data_accused[i])
+            continue;
+        if (replies[i].poststat.ia_size > maxsize)
+            maxsize = replies[i].poststat.ia_size;
+    }
+
+    for (i = 0; i < priv->child_count; i++) {
+        if (data_accused[i])
+            continue;
+        if (AFR_IS_ARBITER_BRICK(priv, i))
+            continue;
+        if (replies[i].poststat.ia_size < maxsize)
+            data_accused[i] = 1;
+    }
+
+    return 0;
 }
 
 int
-afr_inode_refresh_err (call_frame_t *frame, xlator_t *this)
+afr_readables_fill(call_frame_t *frame, xlator_t *this, inode_t *inode,
+                   unsigned char *data_accused, unsigned char *metadata_accused,
+                   unsigned char *data_readable,
+                   unsigned char *metadata_readable, struct afr_reply *replies)
 {
-	afr_local_t *local = NULL;
-	afr_private_t *priv = NULL;
-	int i = 0;
-	int err = 0;
-
-	local = frame->local;
-	priv = this->private;
-
-	for (i = 0; i < priv->child_count; i++) {
-		if (local->replies[i].valid && !local->replies[i].op_ret) {
-			err = 0;
-			goto ret;
-		}
-	}
-
-	err = afr_final_errno (local, priv);
-ret:
-	return -err;
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
+    dict_t *xdata = NULL;
+    int i = 0;
+    int ret = 0;
+    ia_type_t ia_type = IA_INVAL;
+
+    local = frame->local;
+    priv = this->private;
+
+    for (i = 0; i < priv->child_count; i++) {
+        data_readable[i] = 1;
+        metadata_readable[i] = 1;
+    }
+    if (AFR_IS_ARBITER_BRICK(priv, ARBITER_BRICK_INDEX)) {
+        data_readable[ARBITER_BRICK_INDEX] = 0;
+        metadata_readable[ARBITER_BRICK_INDEX] = 0;
+    }
+
+    for (i = 0; i < priv->child_count; i++) {
+        if (replies) { /* Lookup */
+            if (!replies[i].valid || replies[i].op_ret == -1 ||
+                (replies[i].xdata &&
+                 dict_get_sizen(replies[i].xdata, GLUSTERFS_BAD_INODE))) {
+                data_readable[i] = 0;
+                metadata_readable[i] = 0;
+                continue;
+            }
+
+            xdata = replies[i].xdata;
+            ia_type = replies[i].poststat.ia_type;
+        } else { /* pre-op xattrop */
+            xdata = local->transaction.changelog_xdata[i];
+            ia_type = inode->ia_type;
+        }
+
+        if (!xdata)
+            continue; /* mkdir_cbk sends NULL xdata_rsp. */
+        afr_accused_fill(this, xdata, data_accused,
+                         (ia_type == IA_IFDIR) ? AFR_ENTRY_TRANSACTION
+                                               : AFR_DATA_TRANSACTION);
+
+        afr_accused_fill(this, xdata, metadata_accused,
+                         AFR_METADATA_TRANSACTION);
+    }
+
+    if (replies && ia_type != IA_INVAL && ia_type != IA_IFDIR &&
+        /* We want to accuse small files only when we know for
+         * sure that there is no IO happening. Otherwise, the
+         * ia_sizes obtained in post-refresh replies may
+         * mismatch due to a race between inode-refresh and
+         * ongoing writes, causing spurious heal launches*/
+        !afr_is_possibly_under_txn(AFR_DATA_TRANSACTION, local, this)) {
+        afr_accuse_smallfiles(this, replies, data_accused);
+    }
+
+    for (i = 0; i < priv->child_count; i++) {
+        if (data_accused[i]) {
+            data_readable[i] = 0;
+            ret = 1;
+        }
+        if (metadata_accused[i]) {
+            metadata_readable[i] = 0;
+            ret = 1;
+        }
+    }
+    return ret;
 }
 
-
 int
-afr_refresh_selfheal_wrap (void *opaque)
+afr_replies_interpret(call_frame_t *frame, xlator_t *this, inode_t *inode,
+                      gf_boolean_t *start_heal)
 {
-	call_frame_t *frame = opaque;
-	afr_local_t *local = NULL;
-	xlator_t *this = NULL;
-	int err = 0;
-
-	local = frame->local;
-	this = frame->this;
-
-	afr_selfheal (frame->this, local->refreshinode->gfid);
-
-	afr_selfheal_unlocked_discover (frame, local->refreshinode,
-					local->refreshinode->gfid,
-					local->replies);
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
+    struct afr_reply *replies = NULL;
+    int event_generation = 0;
+    int i = 0;
+    unsigned char *data_accused = NULL;
+    unsigned char *metadata_accused = NULL;
+    unsigned char *data_readable = NULL;
+    unsigned char *metadata_readable = NULL;
+    int ret = 0;
+
+    local = frame->local;
+    priv = this->private;
+    replies = local->replies;
+    event_generation = local->event_generation;
+
+    data_accused = alloca0(priv->child_count);
+    data_readable = alloca0(priv->child_count);
+    metadata_accused = alloca0(priv->child_count);
+    metadata_readable = alloca0(priv->child_count);
+
+    ret = afr_readables_fill(frame, this, inode, data_accused, metadata_accused,
+                             data_readable, metadata_readable, replies);
+
+    for (i = 0; i < priv->child_count; i++) {
+        if (start_heal && priv->child_up[i] &&
+            (data_accused[i] || metadata_accused[i])) {
+            *start_heal = _gf_true;
+            break;
+        }
+    }
+    afr_inode_read_subvol_set(inode, this, data_readable, metadata_readable,
+                              event_generation);
+    return ret;
+}
 
-	afr_replies_interpret (frame, this, local->refreshinode);
+int
+afr_refresh_selfheal_done(int ret, call_frame_t *heal, void *opaque)
+{
+    if (heal)
+        AFR_STACK_DESTROY(heal);
+    return 0;
+}
 
-	err = afr_inode_refresh_err (frame, this);
+int
+afr_inode_refresh_err(call_frame_t *frame, xlator_t *this)
+{
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
+    int i = 0;
+    int err = 0;
 
-        afr_local_replies_wipe (local, this->private);
+    local = frame->local;
+    priv = this->private;
 
-	local->refreshfn (frame, this, err);
+    for (i = 0; i < priv->child_count; i++) {
+        if (local->replies[i].valid && !local->replies[i].op_ret) {
+            err = 0;
+            goto ret;
+        }
+    }
 
-	return 0;
+    err = afr_final_errno(local, priv);
+ret:
+    return err;
 }
 
-
 gf_boolean_t
-afr_selfheal_enabled (xlator_t *this)
+afr_selfheal_enabled(const xlator_t *this)
 {
-	afr_private_t *priv = NULL;
-	gf_boolean_t data = _gf_false;
-        int ret = 0;
-
-	priv = this->private;
-
-	ret = gf_string2boolean (priv->data_self_heal, &data);
-        GF_ASSERT (!ret);
+    const afr_private_t *priv = this->private;
 
-	return data || priv->metadata_self_heal || priv->entry_self_heal;
+    return priv->data_self_heal || priv->metadata_self_heal ||
+           priv->entry_self_heal;
 }
 
-
 int
-afr_inode_refresh_done (call_frame_t *frame, xlator_t *this)
+afr_txn_refresh_done(call_frame_t *frame, xlator_t *this, int err)
 {
-	call_frame_t *heal = NULL;
-	afr_local_t *local = NULL;
-	int ret = 0;
-	int err = 0;
-
-	local = frame->local;
-
-	ret = afr_replies_interpret (frame, this, local->refreshinode);
-
-	err = afr_inode_refresh_err (frame, this);
-
-        afr_local_replies_wipe (local, this->private);
+    call_frame_t *heal_frame = NULL;
+    afr_local_t *heal_local = NULL;
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
+    inode_t *inode = NULL;
+    int event_generation = 0;
+    int read_subvol = -1;
+    int ret = 0;
+
+    local = frame->local;
+    inode = local->inode;
+    priv = this->private;
+
+    if (err)
+        goto refresh_done;
+
+    if (local->op == GF_FOP_LOOKUP)
+        goto refresh_done;
+
+    ret = afr_inode_get_readable(frame, inode, this, local->readable,
+                                 &event_generation, local->transaction.type);
+
+    if (ret == -EIO) {
+        /* No readable subvolume even after refresh ==> splitbrain.*/
+        if (!priv->fav_child_policy) {
+            err = EIO;
+            goto refresh_done;
+        }
+        read_subvol = afr_sh_get_fav_by_policy(this, local->replies, inode,
+                                               NULL);
+        if (read_subvol == -1) {
+            err = EIO;
+            goto refresh_done;
+        }
+
+        heal_frame = afr_frame_create(this, NULL);
+        if (!heal_frame) {
+            err = EIO;
+            goto refresh_done;
+        }
+        heal_local = heal_frame->local;
+        heal_local->xdata_req = dict_new();
+        if (!heal_local->xdata_req) {
+            err = EIO;
+            AFR_STACK_DESTROY(heal_frame);
+            goto refresh_done;
+        }
+        heal_local->heal_frame = frame;
+        ret = synctask_new(this->ctx->env, afr_fav_child_reset_sink_xattrs,
+                           afr_fav_child_reset_sink_xattrs_cbk, heal_frame,
+                           heal_frame);
+        return 0;
+    }
 
-	if (ret && afr_selfheal_enabled (this)) {
-		heal = copy_frame (frame);
-		if (heal)
-			heal->root->pid = GF_CLIENT_PID_AFR_SELF_HEALD;
-		ret = synctask_new (this->ctx->env, afr_refresh_selfheal_wrap,
-				    afr_refresh_selfheal_done, heal, frame);
-		if (ret)
-			goto refresh_done;
-	} else {
-	refresh_done:
-		local->refreshfn (frame, this, err);
-	}
+refresh_done:
+    afr_local_replies_wipe(local, this->private);
+    local->refreshfn(frame, this, err);
 
-	return 0;
+    return 0;
 }
 
-
 int
-afr_inode_refresh_subvol_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-			      int op_ret, int op_errno, inode_t *inode,
-			      struct iatt *buf, dict_t *xdata, struct iatt *par)
+afr_inode_refresh_done(call_frame_t *frame, xlator_t *this, int error)
 {
-	afr_local_t *local = NULL;
-	int call_child = (long) cookie;
-	int call_count = 0;
-
-	local = frame->local;
+    call_frame_t *heal_frame = NULL;
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
+    gf_boolean_t start_heal = _gf_false;
+    afr_local_t *heal_local = NULL;
+    unsigned char *success_replies = NULL;
+    int ret = 0;
+
+    if (error != 0) {
+        goto refresh_done;
+    }
+
+    local = frame->local;
+    priv = this->private;
+    success_replies = alloca0(priv->child_count);
+    afr_fill_success_replies(local, priv, success_replies);
+
+    if (priv->thin_arbiter_count && local->is_read_txn &&
+        AFR_COUNT(success_replies, priv->child_count) != priv->child_count) {
+        /* We need to query the good bricks and/or thin-arbiter.*/
+        if (success_replies[0]) {
+            local->read_txn_query_child = AFR_CHILD_ZERO;
+        } else if (success_replies[1]) {
+            local->read_txn_query_child = AFR_CHILD_ONE;
+        }
+        error = EINVAL;
+        goto refresh_done;
+    }
+
+    if (!afr_has_quorum(success_replies, this, frame)) {
+        error = afr_final_errno(frame->local, this->private);
+        if (!error)
+            error = afr_quorum_errno(priv);
+        goto refresh_done;
+    }
+
+    ret = afr_replies_interpret(frame, this, local->refreshinode, &start_heal);
+
+    if (ret && afr_selfheal_enabled(this) && start_heal) {
+        heal_frame = afr_frame_create(this, NULL);
+        if (!heal_frame)
+            goto refresh_done;
+        heal_local = heal_frame->local;
+        heal_local->refreshinode = inode_ref(local->refreshinode);
+        heal_local->heal_frame = heal_frame;
+        if (!afr_throttled_selfheal(heal_frame, this)) {
+            AFR_STACK_DESTROY(heal_frame);
+            goto refresh_done;
+        }
+    }
+
+refresh_done:
+    afr_txn_refresh_done(frame, this, error);
+
+    return 0;
+}
 
-	local->replies[call_child].valid = 1;
-	local->replies[call_child].op_ret = op_ret;
-	local->replies[call_child].op_errno = op_errno;
-	if (op_ret != -1) {
-		local->replies[call_child].poststat = *buf;
-		local->replies[call_child].postparent = *par;
-		local->replies[call_child].xdata = dict_ref (xdata);
-	}
+void
+afr_inode_refresh_subvol_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                             int op_ret, int op_errno, struct iatt *buf,
+                             dict_t *xdata, struct iatt *par)
+{
+    afr_local_t *local = NULL;
+    int call_child = (long)cookie;
+    int8_t need_heal = 1;
+    int call_count = 0;
+    int ret = 0;
+
+    local = frame->local;
+    local->replies[call_child].valid = 1;
+    local->replies[call_child].op_ret = op_ret;
+    local->replies[call_child].op_errno = op_errno;
+    if (op_ret != -1) {
+        local->replies[call_child].poststat = *buf;
+        if (par)
+            local->replies[call_child].postparent = *par;
+        if (xdata)
+            local->replies[call_child].xdata = dict_ref(xdata);
+    }
 
-	call_count = afr_frame_return (frame);
+    if (xdata) {
+        ret = dict_get_int8(xdata, "link-count", &need_heal);
+        if (ret) {
+            gf_msg_debug(this->name, -ret, "Unable to get link count");
+        }
+    }
 
-	if (call_count == 0)
-		afr_inode_refresh_done (frame, this);
+    local->replies[call_child].need_heal = need_heal;
+    call_count = afr_frame_return(frame);
+    if (call_count == 0) {
+        afr_set_need_heal(this, local);
+        ret = afr_inode_refresh_err(frame, this);
+        if (ret) {
+            gf_msg_debug(this->name, ret, "afr_inode_refresh_err failed");
+        }
+        afr_inode_refresh_done(frame, this, ret);
+    }
+}
 
-	return 0;
+int
+afr_inode_refresh_subvol_with_lookup_cbk(call_frame_t *frame, void *cookie,
+                                         xlator_t *this, int op_ret,
+                                         int op_errno, inode_t *inode,
+                                         struct iatt *buf, dict_t *xdata,
+                                         struct iatt *par)
+{
+    afr_inode_refresh_subvol_cbk(frame, cookie, this, op_ret, op_errno, buf,
+                                 xdata, par);
+    return 0;
 }
 
+int
+afr_inode_refresh_subvol_with_lookup(call_frame_t *frame, xlator_t *this, int i,
+                                     inode_t *inode, uuid_t gfid, dict_t *xdata)
+{
+    loc_t loc = {
+        0,
+    };
+    afr_private_t *priv = NULL;
+
+    priv = this->private;
+
+    loc.inode = inode;
+    if (gf_uuid_is_null(inode->gfid) && gfid) {
+        /* To handle setattr/setxattr on yet to be linked inode from
+         * dht */
+        gf_uuid_copy(loc.gfid, gfid);
+    } else {
+        gf_uuid_copy(loc.gfid, inode->gfid);
+    }
+
+    STACK_WIND_COOKIE(frame, afr_inode_refresh_subvol_with_lookup_cbk,
+                      (void *)(long)i, priv->children[i],
+                      priv->children[i]->fops->lookup, &loc, xdata);
+    return 0;
+}
 
 int
-afr_inode_refresh_subvol (call_frame_t *frame, xlator_t *this, int i,
-			  inode_t *inode, dict_t *xdata)
+afr_inode_refresh_subvol_with_fstat_cbk(call_frame_t *frame, void *cookie,
+                                        xlator_t *this, int32_t op_ret,
+                                        int32_t op_errno, struct iatt *buf,
+                                        dict_t *xdata)
 {
-	loc_t loc = {0, };
-	afr_private_t *priv = NULL;
+    afr_inode_refresh_subvol_cbk(frame, cookie, this, op_ret, op_errno, buf,
+                                 xdata, NULL);
+    return 0;
+}
 
-	priv = this->private;
+int
+afr_inode_refresh_subvol_with_fstat(call_frame_t *frame, xlator_t *this, int i,
+                                    dict_t *xdata)
+{
+    afr_private_t *priv = NULL;
+    afr_local_t *local = NULL;
 
-	loc.inode = inode;
-	gf_uuid_copy (loc.gfid, inode->gfid);
+    priv = this->private;
+    local = frame->local;
 
-	STACK_WIND_COOKIE (frame, afr_inode_refresh_subvol_cbk,
-			   (void *) (long) i, priv->children[i],
-			   priv->children[i]->fops->lookup, &loc, xdata);
-	return 0;
+    STACK_WIND_COOKIE(frame, afr_inode_refresh_subvol_with_fstat_cbk,
+                      (void *)(long)i, priv->children[i],
+                      priv->children[i]->fops->fstat, local->fd, xdata);
+    return 0;
 }
 
-
 int
-afr_inode_refresh_do (call_frame_t *frame, xlator_t *this)
+afr_inode_refresh_do(call_frame_t *frame, xlator_t *this)
 {
-	afr_local_t *local = NULL;
-	afr_private_t *priv = NULL;
-	int call_count = 0;
-	int i = 0;
-	dict_t *xdata = NULL;
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
+    int call_count = 0;
+    int i = 0;
+    int ret = 0;
+    dict_t *xdata = NULL;
+    afr_fd_ctx_t *fd_ctx = NULL;
+    unsigned char *wind_subvols = NULL;
+
+    priv = this->private;
+    local = frame->local;
+    wind_subvols = alloca0(priv->child_count);
+
+    afr_local_replies_wipe(local, priv);
+
+    if (local->fd) {
+        fd_ctx = afr_fd_ctx_get(local->fd, this);
+        if (!fd_ctx) {
+            afr_inode_refresh_done(frame, this, EINVAL);
+            return 0;
+        }
+    }
+
+    xdata = dict_new();
+    if (!xdata) {
+        afr_inode_refresh_done(frame, this, ENOMEM);
+        return 0;
+    }
 
-	priv = this->private;
-	local = frame->local;
+    ret = afr_xattr_req_prepare(this, xdata);
+    if (ret != 0) {
+        dict_unref(xdata);
+        afr_inode_refresh_done(frame, this, -ret);
+        return 0;
+    }
 
-        afr_local_replies_wipe (local, priv);
+    ret = dict_set_sizen_str_sizen(xdata, "link-count", GF_XATTROP_INDEX_COUNT);
+    if (ret) {
+        gf_msg_debug(this->name, -ret, "Unable to set link-count in dict ");
+    }
 
-	xdata = dict_new ();
-	if (!xdata) {
-		afr_inode_refresh_done (frame, this);
-		return 0;
-	}
+    ret = dict_set_str_sizen(xdata, GLUSTERFS_INODELK_DOM_COUNT, this->name);
+    if (ret) {
+        gf_msg_debug(this->name, -ret,
+                     "Unable to set inodelk-dom-count in dict ");
+    }
 
-	if (afr_xattr_req_prepare (this, xdata) != 0) {
-		dict_unref (xdata);
-		afr_inode_refresh_done (frame, this);
-		return 0;
-	}
+    if (local->fd) {
+        for (i = 0; i < priv->child_count; i++) {
+            if (local->child_up[i] && fd_ctx->opened_on[i] == AFR_FD_OPENED)
+                wind_subvols[i] = 1;
+        }
+    } else {
+        memcpy(wind_subvols, local->child_up,
+               sizeof(*local->child_up) * priv->child_count);
+    }
 
-	local->call_count = AFR_COUNT (local->child_up, priv->child_count);
+    local->call_count = AFR_COUNT(wind_subvols, priv->child_count);
 
-	call_count = local->call_count;
-	for (i = 0; i < priv->child_count; i++) {
-		if (!local->child_up[i])
-			continue;
+    call_count = local->call_count;
+    if (!call_count) {
+        dict_unref(xdata);
+        if (local->fd && AFR_COUNT(local->child_up, priv->child_count))
+            afr_inode_refresh_done(frame, this, EBADFD);
+        else
+            afr_inode_refresh_done(frame, this, ENOTCONN);
+        return 0;
+    }
+    for (i = 0; i < priv->child_count; i++) {
+        if (!wind_subvols[i])
+            continue;
 
-		afr_inode_refresh_subvol (frame, this, i, local->refreshinode,
-					  xdata);
+        if (local->fd)
+            afr_inode_refresh_subvol_with_fstat(frame, this, i, xdata);
+        else
+            afr_inode_refresh_subvol_with_lookup(
+                frame, this, i, local->refreshinode, local->refreshgfid, xdata);
 
-		if (!--call_count)
-			break;
-	}
+        if (!--call_count)
+            break;
+    }
 
-	dict_unref (xdata);
+    dict_unref(xdata);
 
-	return 0;
+    return 0;
 }
 
-
 int
-afr_inode_refresh (call_frame_t *frame, xlator_t *this, inode_t *inode,
-		   afr_inode_refresh_cbk_t refreshfn)
+afr_inode_refresh(call_frame_t *frame, xlator_t *this, inode_t *inode,
+                  uuid_t gfid, afr_inode_refresh_cbk_t refreshfn)
 {
-	afr_local_t *local = NULL;
+    afr_local_t *local = NULL;
 
-	local = frame->local;
+    local = frame->local;
 
-	local->refreshfn = refreshfn;
+    local->refreshfn = refreshfn;
 
-	if (local->refreshinode) {
-		inode_unref (local->refreshinode);
-		local->refreshinode = NULL;
-	}
+    if (local->refreshinode) {
+        inode_unref(local->refreshinode);
+        local->refreshinode = NULL;
+    }
 
-	local->refreshinode = inode_ref (inode);
+    local->refreshinode = inode_ref(inode);
 
-	afr_inode_refresh_do (frame, this);
+    if (gfid)
+        gf_uuid_copy(local->refreshgfid, gfid);
+    else
+        gf_uuid_clear(local->refreshgfid);
 
-	return 0;
-}
+    afr_inode_refresh_do(frame, this);
 
+    return 0;
+}
 
 int
-afr_xattr_req_prepare (xlator_t *this, dict_t *xattr_req)
+afr_xattr_req_prepare(xlator_t *this, dict_t *xattr_req)
 {
-        int             i           = 0;
-        afr_private_t   *priv       = NULL;
-        int             ret         = 0;
-
-        priv   = this->private;
+    int i = 0;
+    afr_private_t *priv = NULL;
+    int ret = 0;
 
-        for (i = 0; i < priv->child_count; i++) {
-                ret = dict_set_uint64 (xattr_req, priv->pending_key[i],
-                                       AFR_NUM_CHANGE_LOGS * sizeof(int));
-                if (ret < 0)
-                        gf_log (this->name, GF_LOG_WARNING,
-                                "Unable to set dict value for %s",
-                                priv->pending_key[i]);
-                /* 3 = data+metadata+entry */
-        }
-        ret = dict_set_uint64 (xattr_req, AFR_DIRTY,
-			       AFR_NUM_CHANGE_LOGS * sizeof(int));
-        if (ret) {
-                gf_log (this->name, GF_LOG_DEBUG, "failed to set dirty "
-                        "query flag");
-        }
+    priv = this->private;
 
-        ret = dict_set_int32 (xattr_req, "list-xattr", 1);
-        if (ret) {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "Unable to set list-xattr in dict ");
-        }
-
-	return ret;
+    for (i = 0; i < priv->child_count; i++) {
+        ret = dict_set_uint64(xattr_req, priv->pending_key[i],
+                              AFR_NUM_CHANGE_LOGS * sizeof(int));
+        if (ret < 0)
+            gf_msg(this->name, GF_LOG_WARNING, -ret, AFR_MSG_DICT_SET_FAILED,
+                   "Unable to set dict value for %s", priv->pending_key[i]);
+        /* 3 = data+metadata+entry */
+    }
+    ret = dict_set_uint64(xattr_req, AFR_DIRTY,
+                          AFR_NUM_CHANGE_LOGS * sizeof(int));
+    if (ret) {
+        gf_msg_debug(this->name, -ret,
+                     "failed to set dirty "
+                     "query flag");
+    }
+
+    ret = dict_set_int32_sizen(xattr_req, "list-xattr", 1);
+    if (ret) {
+        gf_msg_debug(this->name, -ret, "Unable to set list-xattr in dict ");
+    }
+
+    return ret;
 }
 
 int
-afr_lookup_xattr_req_prepare (afr_local_t *local, xlator_t *this,
-                              dict_t *xattr_req, loc_t *loc)
+afr_lookup_xattr_req_prepare(afr_local_t *local, xlator_t *this,
+                             dict_t *xattr_req, loc_t *loc)
 {
-        int     ret = -ENOMEM;
-
-        if (!local->xattr_req)
-                local->xattr_req = dict_new ();
-
-        if (!local->xattr_req)
-                goto out;
+    int ret = -ENOMEM;
+
+    if (!local->xattr_req)
+        local->xattr_req = dict_new();
+
+    if (!local->xattr_req)
+        goto out;
+
+    if (xattr_req && (xattr_req != local->xattr_req))
+        dict_copy(xattr_req, local->xattr_req);
+
+    ret = afr_xattr_req_prepare(this, local->xattr_req);
+
+    ret = dict_set_uint64(local->xattr_req, GLUSTERFS_INODELK_COUNT, 0);
+    if (ret < 0) {
+        gf_msg(this->name, GF_LOG_WARNING, -ret, AFR_MSG_DICT_SET_FAILED,
+               "%s: Unable to set dict value for %s", loc->path,
+               GLUSTERFS_INODELK_COUNT);
+    }
+    ret = dict_set_uint64(local->xattr_req, GLUSTERFS_ENTRYLK_COUNT, 0);
+    if (ret < 0) {
+        gf_msg(this->name, GF_LOG_WARNING, -ret, AFR_MSG_DICT_SET_FAILED,
+               "%s: Unable to set dict value for %s", loc->path,
+               GLUSTERFS_ENTRYLK_COUNT);
+    }
+
+    ret = dict_set_uint32(local->xattr_req, GLUSTERFS_PARENT_ENTRYLK, 0);
+    if (ret < 0) {
+        gf_msg(this->name, GF_LOG_WARNING, -ret, AFR_MSG_DICT_SET_FAILED,
+               "%s: Unable to set dict value for %s", loc->path,
+               GLUSTERFS_PARENT_ENTRYLK);
+    }
+
+    ret = dict_set_sizen_str_sizen(local->xattr_req, "link-count",
+                                   GF_XATTROP_INDEX_COUNT);
+    if (ret) {
+        gf_msg_debug(this->name, -ret, "Unable to set link-count in dict ");
+    }
+
+    ret = 0;
+out:
+    return ret;
+}
 
-        if (xattr_req != local->xattr_req)
-                dict_copy (xattr_req, local->xattr_req);
+int
+afr_least_pending_reads_child(afr_private_t *priv, unsigned char *readable)
+{
+    int i = 0;
+    int child = -1;
+    int64_t read_iter = -1;
+    int64_t pending_read = -1;
+
+    for (i = 0; i < priv->child_count; i++) {
+        if (AFR_IS_ARBITER_BRICK(priv, i) || !readable[i])
+            continue;
+        read_iter = GF_ATOMIC_GET(priv->pending_reads[i]);
+        if (child == -1 || read_iter < pending_read) {
+            pending_read = read_iter;
+            child = i;
+        }
+    }
+
+    return child;
+}
 
-        ret = afr_xattr_req_prepare (this, local->xattr_req);
-	if (ret < 0) {
-		gf_log (this->name, GF_LOG_WARNING,
-			"%s: Unable to prepare xattr_req", loc->path);
-	}
+static int32_t
+afr_least_latency_child(afr_private_t *priv, unsigned char *readable)
+{
+    int32_t i = 0;
+    int child = -1;
 
-        ret = dict_set_uint64 (local->xattr_req, GLUSTERFS_INODELK_COUNT, 0);
-        if (ret < 0) {
-                gf_log (this->name, GF_LOG_WARNING,
-                        "%s: Unable to set dict value for %s",
-                        loc->path, GLUSTERFS_INODELK_COUNT);
-        }
-        ret = dict_set_uint64 (local->xattr_req, GLUSTERFS_ENTRYLK_COUNT, 0);
-        if (ret < 0) {
-                gf_log (this->name, GF_LOG_WARNING,
-                        "%s: Unable to set dict value for %s",
-                        loc->path, GLUSTERFS_ENTRYLK_COUNT);
-        }
+    for (i = 0; i < priv->child_count; i++) {
+        if (AFR_IS_ARBITER_BRICK(priv, i) || !readable[i] ||
+            priv->child_latency[i] < 0)
+            continue;
 
-        ret = dict_set_uint32 (local->xattr_req, GLUSTERFS_PARENT_ENTRYLK, 0);
-        if (ret < 0) {
-                gf_log (this->name, GF_LOG_WARNING,
-                        "%s: Unable to set dict value for %s",
-                        loc->path, GLUSTERFS_PARENT_ENTRYLK);
+        if (child == -1 ||
+            priv->child_latency[i] < priv->child_latency[child]) {
+            child = i;
         }
-
-        ret = 0;
-out:
-        return ret;
+    }
+    return child;
 }
 
+static int32_t
+afr_least_latency_times_pending_reads_child(afr_private_t *priv,
+                                            unsigned char *readable)
+{
+    int32_t i = 0;
+    int child = -1;
+    int64_t pending_read = 0;
+    int64_t latency = -1;
+    int64_t least_latency = -1;
+
+    for (i = 0; i < priv->child_count; i++) {
+        if (AFR_IS_ARBITER_BRICK(priv, i) || !readable[i] ||
+            priv->child_latency[i] < 0)
+            continue;
+
+        pending_read = GF_ATOMIC_GET(priv->pending_reads[i]);
+        latency = (pending_read + 1) * priv->child_latency[i];
+
+        if (child == -1 || latency < least_latency) {
+            least_latency = latency;
+            child = i;
+        }
+    }
+    return child;
+}
 
 int
-afr_hash_child (inode_t *inode, int32_t child_count, int hashmode)
+afr_hash_child(afr_read_subvol_args_t *args, afr_private_t *priv,
+               unsigned char *readable)
 {
-        uuid_t gfid_copy = {0,};
-        pid_t pid;
-
-        if (!hashmode) {
-                return -1;
-        }
-
-        if (inode) {
-               gf_uuid_copy (gfid_copy, inode->gfid);
-        }
-
-        if (hashmode > 1 && inode->ia_type != IA_IFDIR) {
+    uuid_t gfid_copy = {
+        0,
+    };
+    pid_t pid;
+    int child = -1;
+
+    switch (priv->hash_mode) {
+        case AFR_READ_POLICY_FIRST_UP:
+            break;
+        case AFR_READ_POLICY_GFID_HASH:
+            gf_uuid_copy(gfid_copy, args->gfid);
+            child = SuperFastHash((char *)gfid_copy, sizeof(gfid_copy)) %
+                    priv->child_count;
+            break;
+        case AFR_READ_POLICY_GFID_PID_HASH:
+            if (args->ia_type != IA_IFDIR) {
                 /*
                  * Why getpid?  Because it's one of the cheapest calls
-                 * available - faster than gethostname etc. - and returns a
-                 * constant-length value that's sure to be shorter than a UUID.
-                 * It's still very unlikely to be the same across clients, so
-                 * it still provides good mixing.  We're not trying for
-                 * perfection here.  All we need is a low probability that
-                 * multiple clients won't converge on the same subvolume.
+                 * available - faster than gethostname etc. - and
+                 * returns a constant-length value that's sure to be
+                 * shorter than a UUID. It's still very unlikely to be
+                 * the same across clients, so it still provides good
+                 * mixing.  We're not trying for perfection here. All we
+                 * need is a low probability that multiple clients
+                 * won't converge on the same subvolume.
                  */
+                gf_uuid_copy(gfid_copy, args->gfid);
                 pid = getpid();
-                memcpy (gfid_copy, &pid, sizeof(pid));
-        }
-
-        return SuperFastHash((char *)gfid_copy,
-                             sizeof(gfid_copy)) % child_count;
+                *(pid_t *)gfid_copy ^= pid;
+            }
+            child = SuperFastHash((char *)gfid_copy, sizeof(gfid_copy)) %
+                    priv->child_count;
+            break;
+        case AFR_READ_POLICY_LESS_LOAD:
+            child = afr_least_pending_reads_child(priv, readable);
+            break;
+        case AFR_READ_POLICY_LEAST_LATENCY:
+            child = afr_least_latency_child(priv, readable);
+            break;
+        case AFR_READ_POLICY_LOAD_LATENCY_HYBRID:
+            child = afr_least_latency_times_pending_reads_child(priv, readable);
+            break;
+    }
+
+    return child;
 }
 
-
 int
-afr_read_subvol_select_by_policy (inode_t *inode, xlator_t *this,
-				  unsigned char *readable)
+afr_read_subvol_select_by_policy(inode_t *inode, xlator_t *this,
+                                 unsigned char *readable,
+                                 afr_read_subvol_args_t *args)
 {
-	afr_private_t *priv = NULL;
-	int read_subvol = -1;
-	int i = 0;
-
-	priv = this->private;
-
-	/* first preference - explicitly specified or local subvolume */
-	if (priv->read_child >= 0 && readable[priv->read_child])
-		return priv->read_child;
-
-	/* second preference - use hashed mode */
-	read_subvol = afr_hash_child (inode, priv->child_count,
-				      priv->hash_mode);
-	if (read_subvol >= 0 && readable[read_subvol])
-		return read_subvol;
-
-	for (i = 0; i < priv->child_count; i++) {
-		if (readable[i])
-			return i;
-	}
-
-	/* no readable subvolumes, either split brain or all subvols down */
-
-	return -1;
+    int i = 0;
+    int read_subvol = -1;
+    afr_private_t *priv = NULL;
+    afr_read_subvol_args_t local_args = {
+        0,
+    };
+
+    priv = this->private;
+
+    /* first preference - explicitly specified or local subvolume */
+    if (priv->read_child >= 0 && readable[priv->read_child])
+        return priv->read_child;
+
+    if (inode_is_linked(inode)) {
+        gf_uuid_copy(local_args.gfid, inode->gfid);
+        local_args.ia_type = inode->ia_type;
+    } else if (args) {
+        local_args = *args;
+    }
+
+    /* second preference - use hashed mode */
+    read_subvol = afr_hash_child(&local_args, priv, readable);
+    if (read_subvol >= 0 && readable[read_subvol])
+        return read_subvol;
+
+    for (i = 0; i < priv->child_count; i++) {
+        if (readable[i])
+            return i;
+    }
+
+    /* no readable subvolumes, either split brain or all subvols down */
+
+    return -1;
 }
 
-
 int
-afr_inode_read_subvol_type_get (inode_t *inode, xlator_t *this,
-				unsigned char *readable, int *event_p,
-				int type)
+afr_inode_read_subvol_type_get(inode_t *inode, xlator_t *this,
+                               unsigned char *readable, int *event_p, int type)
 {
-	int ret = -1;
+    int ret = -1;
 
-	if (type == AFR_METADATA_TRANSACTION)
-		ret = afr_inode_read_subvol_get (inode, this, 0, readable,
-						 event_p);
-	else
-		ret = afr_inode_read_subvol_get (inode, this, readable, 0,
-						 event_p);
-	return ret;
+    if (type == AFR_METADATA_TRANSACTION)
+        ret = afr_inode_read_subvol_get(inode, this, 0, readable, event_p);
+    else
+        ret = afr_inode_read_subvol_get(inode, this, readable, 0, event_p);
+    return ret;
 }
 
+void
+afr_readables_intersect_get(inode_t *inode, xlator_t *this, int *event,
+                            unsigned char *intersection)
+{
+    afr_private_t *priv = NULL;
+    unsigned char *data_readable = NULL;
+    unsigned char *metadata_readable = NULL;
+    unsigned char *intersect = NULL;
+
+    priv = this->private;
+    data_readable = alloca0(priv->child_count);
+    metadata_readable = alloca0(priv->child_count);
+    intersect = alloca0(priv->child_count);
+
+    afr_inode_read_subvol_get(inode, this, data_readable, metadata_readable,
+                              event);
+
+    AFR_INTERSECT(intersect, data_readable, metadata_readable,
+                  priv->child_count);
+    if (intersection)
+        memcpy(intersection, intersect,
+               sizeof(*intersection) * priv->child_count);
+}
 
 int
-afr_read_subvol_get (inode_t *inode, xlator_t *this, int *subvol_p,
-		     int *event_p, afr_transaction_type type)
+afr_read_subvol_get(inode_t *inode, xlator_t *this, int *subvol_p,
+                    unsigned char *readables, int *event_p,
+                    afr_transaction_type type, afr_read_subvol_args_t *args)
 {
-	afr_private_t *priv = NULL;
-	unsigned char *data_readable = NULL;
-	unsigned char *metadata_readable = NULL;
-	unsigned char *readable = NULL;
-	unsigned char *intersection = NULL;
-	int subvol = -1;
-	int event = 0;
-
-	priv = this->private;
-
-	readable = alloca0 (priv->child_count);
-	data_readable = alloca0 (priv->child_count);
-	metadata_readable = alloca0 (priv->child_count);
-	intersection = alloca0 (priv->child_count);
-
-	afr_inode_read_subvol_type_get (inode, this, readable, &event, type);
-
-	afr_inode_read_subvol_get (inode, this, data_readable, metadata_readable,
-				   &event);
-
-	AFR_INTERSECT (intersection, data_readable, metadata_readable,
-		       priv->child_count);
-
-	if (AFR_COUNT (intersection, priv->child_count) > 0)
-		subvol = afr_read_subvol_select_by_policy (inode, this,
-							   intersection);
-	else
-		subvol = afr_read_subvol_select_by_policy (inode, this,
-							   readable);
-	if (subvol_p)
-		*subvol_p = subvol;
-	if (event_p)
-		*event_p = event;
-	return subvol;
+    afr_private_t *priv = NULL;
+    unsigned char *readable = NULL;
+    unsigned char *intersection = NULL;
+    int subvol = -1;
+    int event = 0;
+
+    priv = this->private;
+
+    readable = alloca0(priv->child_count);
+    intersection = alloca0(priv->child_count);
+
+    afr_inode_read_subvol_type_get(inode, this, readable, &event, type);
+
+    afr_readables_intersect_get(inode, this, &event, intersection);
+
+    if (AFR_COUNT(intersection, priv->child_count) > 0)
+        subvol = afr_read_subvol_select_by_policy(inode, this, intersection,
+                                                  args);
+    else
+        subvol = afr_read_subvol_select_by_policy(inode, this, readable, args);
+    if (subvol_p)
+        *subvol_p = subvol;
+    if (event_p)
+        *event_p = event;
+    if (readables)
+        memcpy(readables, readable, sizeof(*readables) * priv->child_count);
+    return subvol;
 }
 
-
 void
-afr_local_transaction_cleanup (afr_local_t *local, xlator_t *this)
+afr_local_transaction_cleanup(afr_local_t *local, xlator_t *this)
 {
-        afr_private_t *priv    = NULL;
-        int           i        = 0;
+    afr_private_t *priv = NULL;
+    int i = 0;
 
-        priv = this->private;
+    priv = this->private;
 
-        afr_matrix_cleanup (local->pending, priv->child_count);
+    afr_matrix_cleanup(local->pending, priv->child_count);
 
-        GF_FREE (local->internal_lock.locked_nodes);
+    GF_FREE(local->internal_lock.lower_locked_nodes);
 
-        for (i = 0; local->internal_lock.inodelk[i].domain; i++) {
-                GF_FREE (local->internal_lock.inodelk[i].locked_nodes);
-        }
+    afr_lockees_cleanup(&local->internal_lock);
 
-        GF_FREE (local->internal_lock.lower_locked_nodes);
+    GF_FREE(local->transaction.pre_op);
 
-        afr_entry_lockee_cleanup (&local->internal_lock);
-
-        GF_FREE (local->transaction.pre_op);
-        GF_FREE (local->transaction.eager_lock);
-        GF_FREE (local->transaction.fop_subvols);
-        GF_FREE (local->transaction.failed_subvols);
+    GF_FREE(local->transaction.pre_op_sources);
+    if (local->transaction.changelog_xdata) {
+        for (i = 0; i < priv->child_count; i++) {
+            if (!local->transaction.changelog_xdata[i])
+                continue;
+            dict_unref(local->transaction.changelog_xdata[i]);
+        }
+        GF_FREE(local->transaction.changelog_xdata);
+    }
 
-        GF_FREE (local->transaction.basename);
-        GF_FREE (local->transaction.new_basename);
+    GF_FREE(local->transaction.failed_subvols);
 
-        loc_wipe (&local->transaction.parent_loc);
-        loc_wipe (&local->transaction.new_parent_loc);
+    GF_FREE(local->transaction.basename);
+    GF_FREE(local->transaction.new_basename);
 
+    loc_wipe(&local->transaction.parent_loc);
+    loc_wipe(&local->transaction.new_parent_loc);
 }
 
+void
+afr_reply_wipe(struct afr_reply *reply)
+{
+    if (reply->xdata) {
+        dict_unref(reply->xdata);
+        reply->xdata = NULL;
+    }
+
+    if (reply->xattr) {
+        dict_unref(reply->xattr);
+        reply->xattr = NULL;
+    }
+}
 
 void
-afr_replies_wipe (struct afr_reply *replies, int count)
+afr_replies_wipe(struct afr_reply *replies, int count)
 {
-        int i = 0;
+    int i = 0;
 
-        for (i = 0; i < count; i++) {
-                if (replies[i].xdata) {
-                        dict_unref (replies[i].xdata);
-                        replies[i].xdata = NULL;
-                }
-        }
+    for (i = 0; i < count; i++) {
+        afr_reply_wipe(&replies[i]);
+    }
 }
 
 void
-afr_local_replies_wipe (afr_local_t *local, afr_private_t *priv)
+afr_local_replies_wipe(afr_local_t *local, afr_private_t *priv)
 {
+    if (!local->replies)
+        return;
 
-	if (!local->replies)
-		return;
-
-        afr_replies_wipe (local->replies, priv->child_count);
+    afr_replies_wipe(local->replies, priv->child_count);
 
-	memset (local->replies, 0, sizeof(*local->replies) * priv->child_count);
+    memset(local->replies, 0, sizeof(*local->replies) * priv->child_count);
 }
 
-void
-afr_remove_eager_lock_stub (afr_local_t *local)
+static gf_boolean_t
+afr_fop_lock_is_unlock(call_frame_t *frame)
 {
-        LOCK (&local->fd->lock);
-        {
-                list_del_init (&local->transaction.eager_locked);
-        }
-        UNLOCK (&local->fd->lock);
+    afr_local_t *local = frame->local;
+    switch (local->op) {
+        case GF_FOP_INODELK:
+        case GF_FOP_FINODELK:
+            if ((F_UNLCK == local->cont.inodelk.in_flock.l_type) &&
+                (local->cont.inodelk.in_cmd == F_SETLKW ||
+                 local->cont.inodelk.in_cmd == F_SETLK))
+                return _gf_true;
+            break;
+        case GF_FOP_ENTRYLK:
+        case GF_FOP_FENTRYLK:
+            if (ENTRYLK_UNLOCK == local->cont.entrylk.in_cmd)
+                return _gf_true;
+            break;
+        default:
+            return _gf_false;
+    }
+    return _gf_false;
 }
 
-void
-afr_local_cleanup (afr_local_t *local, xlator_t *this)
+static gf_boolean_t
+afr_lk_is_unlock(int32_t cmd, struct gf_flock *flock)
 {
-        afr_private_t * priv = NULL;
-
-        if (!local)
-                return;
-
-	syncbarrier_destroy (&local->barrier);
+    switch (cmd) {
+        case F_RESLK_UNLCK:
+            return _gf_true;
+            break;
 
-        if (local->transaction.eager_lock_on &&
-            !list_empty (&local->transaction.eager_locked))
-                afr_remove_eager_lock_stub (local);
+#if F_SETLKW != F_SETLKW64
+        case F_SETLKW64:
+#endif
+        case F_SETLKW:
 
-        afr_local_transaction_cleanup (local, this);
+#if F_SETLK != F_SETLK64
+        case F_SETLK64:
+#endif
+        case F_SETLK:
+            if (F_UNLCK == flock->l_type)
+                return _gf_true;
+            break;
+        default:
+            return _gf_false;
+    }
+    return _gf_false;
+}
 
-        priv = this->private;
+void
+afr_handle_inconsistent_fop(call_frame_t *frame, int32_t *op_ret,
+                            int32_t *op_errno)
+{
+    afr_private_t *priv = NULL;
+    afr_local_t *local = NULL;
 
-        loc_wipe (&local->loc);
-        loc_wipe (&local->newloc);
+    if (!frame || !frame->this || !frame->local || !frame->this->private)
+        return;
 
-        if (local->fd)
-                fd_unref (local->fd);
+    if (*op_ret < 0)
+        return;
 
-        if (local->xattr_req)
-                dict_unref (local->xattr_req);
+    /* Failing inodelk/entrylk/lk here is not a good idea because we
+     * need to cleanup the locks on the other bricks if we choose to fail
+     * the fop here. The brick may go down just after unwind happens as well
+     * so anyways the fop will fail when the next fop is sent so leaving
+     * it like this for now.*/
+    local = frame->local;
+    switch (local->op) {
+        case GF_FOP_LOOKUP:
+        case GF_FOP_INODELK:
+        case GF_FOP_FINODELK:
+        case GF_FOP_ENTRYLK:
+        case GF_FOP_FENTRYLK:
+        case GF_FOP_LK:
+            return;
+        default:
+            break;
+    }
 
-        if (local->dict)
-                dict_unref (local->dict);
+    priv = frame->this->private;
+    if (!priv->consistent_io)
+        return;
 
-        afr_local_replies_wipe (local, priv);
-	GF_FREE(local->replies);
+    if (local->event_generation &&
+        (local->event_generation != priv->event_generation))
+        goto inconsistent;
 
-        GF_FREE (local->child_up);
+    return;
+inconsistent:
+    *op_ret = -1;
+    *op_errno = ENOTCONN;
+}
 
-        GF_FREE (local->read_attempted);
+void
+afr_local_cleanup(afr_local_t *local, xlator_t *this)
+{
+    afr_private_t *priv = NULL;
 
-        GF_FREE (local->readable);
+    if (!local)
+        return;
 
-	if (local->inode)
-		inode_unref (local->inode);
+    syncbarrier_destroy(&local->barrier);
 
-	if (local->parent)
-		inode_unref (local->parent);
+    afr_local_transaction_cleanup(local, this);
 
-	if (local->parent2)
-		inode_unref (local->parent2);
+    priv = this->private;
 
-	if (local->refreshinode)
-		inode_unref (local->refreshinode);
+    loc_wipe(&local->loc);
+    loc_wipe(&local->newloc);
 
-        { /* getxattr */
-                GF_FREE (local->cont.getxattr.name);
-        }
+    if (local->fd)
+        fd_unref(local->fd);
 
-        { /* lk */
-                GF_FREE (local->cont.lk.locked_nodes);
-        }
+    if (local->xattr_req)
+        dict_unref(local->xattr_req);
 
-        { /* create */
-                if (local->cont.create.fd)
-                        fd_unref (local->cont.create.fd);
-                if (local->cont.create.params)
-                        dict_unref (local->cont.create.params);
-        }
+    if (local->xattr_rsp)
+        dict_unref(local->xattr_rsp);
 
-        { /* mknod */
-                if (local->cont.mknod.params)
-                        dict_unref (local->cont.mknod.params);
-        }
+    if (local->dict)
+        dict_unref(local->dict);
 
-        { /* mkdir */
-                if (local->cont.mkdir.params)
-                        dict_unref (local->cont.mkdir.params);
-        }
+    afr_local_replies_wipe(local, priv);
+    GF_FREE(local->replies);
 
-        { /* symlink */
-                if (local->cont.symlink.params)
-                        dict_unref (local->cont.symlink.params);
-        }
+    GF_FREE(local->child_up);
 
-        { /* writev */
-                GF_FREE (local->cont.writev.vector);
-                if (local->cont.writev.iobref)
-                        iobref_unref (local->cont.writev.iobref);
-        }
+    GF_FREE(local->read_attempted);
+
+    GF_FREE(local->readable);
+    GF_FREE(local->readable2);
+
+    if (local->inode)
+        inode_unref(local->inode);
 
-        { /* setxattr */
-                if (local->cont.setxattr.dict)
-                        dict_unref (local->cont.setxattr.dict);
-        }
+    if (local->parent)
+        inode_unref(local->parent);
+
+    if (local->parent2)
+        inode_unref(local->parent2);
+
+    if (local->refreshinode)
+        inode_unref(local->refreshinode);
+
+    { /* getxattr */
+        GF_FREE(local->cont.getxattr.name);
+    }
+
+    { /* lk */
+        GF_FREE(local->cont.lk.locked_nodes);
+        GF_FREE(local->cont.lk.dom_locked_nodes);
+        GF_FREE(local->cont.lk.dom_lock_op_ret);
+        GF_FREE(local->cont.lk.dom_lock_op_errno);
+    }
+
+    { /* create */
+        if (local->cont.create.fd)
+            fd_unref(local->cont.create.fd);
+        if (local->cont.create.params)
+            dict_unref(local->cont.create.params);
+    }
+
+    { /* mknod */
+        if (local->cont.mknod.params)
+            dict_unref(local->cont.mknod.params);
+    }
+
+    { /* mkdir */
+        if (local->cont.mkdir.params)
+            dict_unref(local->cont.mkdir.params);
+    }
+
+    { /* symlink */
+        if (local->cont.symlink.params)
+            dict_unref(local->cont.symlink.params);
+    }
+
+    { /* writev */
+        GF_FREE(local->cont.writev.vector);
+        if (local->cont.writev.iobref)
+            iobref_unref(local->cont.writev.iobref);
+    }
+
+    { /* setxattr */
+        if (local->cont.setxattr.dict)
+            dict_unref(local->cont.setxattr.dict);
+    }
+
+    { /* fsetxattr */
+        if (local->cont.fsetxattr.dict)
+            dict_unref(local->cont.fsetxattr.dict);
+    }
+
+    { /* removexattr */
+        GF_FREE(local->cont.removexattr.name);
+    }
+    { /* xattrop */
+        if (local->cont.xattrop.xattr)
+            dict_unref(local->cont.xattrop.xattr);
+    }
+    { /* symlink */
+        GF_FREE(local->cont.symlink.linkpath);
+    }
+
+    { /* opendir */
+        GF_FREE(local->cont.opendir.checksum);
+    }
+
+    { /* open */
+        if (local->cont.open.fd)
+            fd_unref(local->cont.open.fd);
+    }
+
+    { /* readdirp */
+        if (local->cont.readdir.dict)
+            dict_unref(local->cont.readdir.dict);
+    }
+
+    { /* inodelk */
+        GF_FREE(local->cont.inodelk.volume);
+        if (local->cont.inodelk.xdata)
+            dict_unref(local->cont.inodelk.xdata);
+    }
+
+    { /* entrylk */
+        GF_FREE(local->cont.entrylk.volume);
+        GF_FREE(local->cont.entrylk.basename);
+        if (local->cont.entrylk.xdata)
+            dict_unref(local->cont.entrylk.xdata);
+    }
+
+    if (local->xdata_req)
+        dict_unref(local->xdata_req);
+
+    if (local->xdata_rsp)
+        dict_unref(local->xdata_rsp);
+}
 
-        { /* fsetxattr */
-                if (local->cont.fsetxattr.dict)
-                        dict_unref (local->cont.fsetxattr.dict);
-        }
+int
+afr_frame_return(call_frame_t *frame)
+{
+    afr_local_t *local = NULL;
+    int call_count = 0;
 
-        { /* removexattr */
-                GF_FREE (local->cont.removexattr.name);
-        }
-        { /* xattrop */
-                if (local->cont.xattrop.xattr)
-                        dict_unref (local->cont.xattrop.xattr);
-        }
-        { /* fxattrop */
-                if (local->cont.fxattrop.xattr)
-                        dict_unref (local->cont.fxattrop.xattr);
-        }
-        { /* symlink */
-                GF_FREE (local->cont.symlink.linkpath);
-        }
+    local = frame->local;
 
-        { /* opendir */
-                GF_FREE (local->cont.opendir.checksum);
-        }
+    LOCK(&frame->lock);
+    {
+        call_count = --local->call_count;
+    }
+    UNLOCK(&frame->lock);
 
-        { /* readdirp */
-                if (local->cont.readdir.dict)
-                        dict_unref (local->cont.readdir.dict);
-        }
+    return call_count;
+}
 
-        { /* inodelk */
-                GF_FREE (local->cont.inodelk.volume);
-        }
+static char *afr_ignore_xattrs[] = {GF_SELINUX_XATTR_KEY, QUOTA_SIZE_KEY, NULL};
 
-        if (local->xdata_req)
-                dict_unref (local->xdata_req);
+gf_boolean_t
+afr_is_xattr_ignorable(char *key)
+{
+    int i = 0;
 
-        if (local->xdata_rsp)
-                dict_unref (local->xdata_rsp);
+    if (!strncmp(key, AFR_XATTR_PREFIX, SLEN(AFR_XATTR_PREFIX)))
+        return _gf_true;
+    for (i = 0; afr_ignore_xattrs[i]; i++) {
+        if (!strcmp(key, afr_ignore_xattrs[i]))
+            return _gf_true;
+    }
+    return _gf_false;
 }
 
-
-int
-afr_frame_return (call_frame_t *frame)
+static gf_boolean_t
+afr_xattr_match_needed(dict_t *this, char *key1, data_t *value1, void *data)
 {
-        afr_local_t *local = NULL;
-        int          call_count = 0;
-
-        local = frame->local;
+    /* Ignore all non-disk (i.e. virtual) xattrs right away. */
+    if (!gf_is_valid_xattr_namespace(key1))
+        return _gf_false;
 
-        LOCK (&frame->lock);
-        {
-                call_count = --local->call_count;
-        }
-        UNLOCK (&frame->lock);
+    /* Ignore on-disk xattrs that AFR doesn't need to heal. */
+    if (!afr_is_xattr_ignorable(key1))
+        return _gf_true;
 
-        return call_count;
+    return _gf_false;
 }
 
-
 gf_boolean_t
-afr_is_entry_possibly_under_txn (afr_local_t *local, xlator_t *this)
+afr_xattrs_are_equal(dict_t *dict1, dict_t *dict2)
 {
-	int i = 0;
-	int tmp = 0;
-	afr_private_t *priv = NULL;
-
-	priv = this->private;
+    return are_dicts_equal(dict1, dict2, afr_xattr_match_needed, NULL);
+}
 
-	for (i = 0; i < priv->child_count; i++) {
-		if (!local->replies[i].xdata)
-			continue;
-		if (dict_get_int32 (local->replies[i].xdata,
-				    GLUSTERFS_PARENT_ENTRYLK,
-				    &tmp) == 0)
-			if (tmp)
-				return _gf_true;
-	}
+static int
+afr_get_parent_read_subvol(xlator_t *this, inode_t *parent,
+                           struct afr_reply *replies, unsigned char *readable)
+{
+    int i = 0;
+    int par_read_subvol = -1;
+    int par_read_subvol_iter = -1;
+    afr_private_t *priv = NULL;
 
-	return _gf_false;
-}
+    priv = this->private;
 
+    if (parent)
+        par_read_subvol = afr_data_subvol_get(parent, this, NULL, NULL, NULL,
+                                              NULL);
 
-static char *afr_ignore_xattrs[] = {
-        GLUSTERFS_OPEN_FD_COUNT,
-        GLUSTERFS_PARENT_ENTRYLK,
-        GLUSTERFS_ENTRYLK_COUNT,
-        GLUSTERFS_INODELK_COUNT,
-        GF_SELINUX_XATTR_KEY,
-        QUOTA_SIZE_KEY,
-        NULL
-};
+    for (i = 0; i < priv->child_count; i++) {
+        if (!replies[i].valid)
+            continue;
 
-gf_boolean_t
-afr_is_xattr_ignorable (char *key)
-{
-        int i = 0;
+        if (replies[i].op_ret < 0)
+            continue;
 
-        if (!strncmp (key, AFR_XATTR_PREFIX, strlen(AFR_XATTR_PREFIX)))
-                return _gf_true;
-        for (i = 0; afr_ignore_xattrs[i]; i++) {
-                if (!strcmp (key, afr_ignore_xattrs[i]))
-                       return _gf_true;
+        if (par_read_subvol_iter == -1) {
+            par_read_subvol_iter = i;
+            continue;
         }
-        return _gf_false;
+
+        if ((par_read_subvol_iter != par_read_subvol) && readable[i])
+            par_read_subvol_iter = i;
+
+        if (i == par_read_subvol)
+            par_read_subvol_iter = i;
+    }
+    /* At the end of the for-loop, the only reason why @par_read_subvol_iter
+     * could be -1 is when this LOOKUP has failed on all sub-volumes.
+     * So it is okay to send an arbitrary subvolume (0 in this case)
+     * as parent read subvol.
+     */
+    if (par_read_subvol_iter == -1)
+        par_read_subvol_iter = 0;
+
+    return par_read_subvol_iter;
 }
 
 int
-xattr_is_equal (dict_t *this, char *key1, data_t *value1, void *data)
+afr_read_subvol_decide(inode_t *inode, xlator_t *this,
+                       afr_read_subvol_args_t *args, unsigned char *readable)
 {
-        dict_t *xattr2 = (dict_t *)data;
-        data_t *value2 = NULL;
+    int event = 0;
+    afr_private_t *priv = NULL;
+    unsigned char *intersection = NULL;
 
-        if (afr_is_xattr_ignorable (key1))
-                return 0;
+    priv = this->private;
+    intersection = alloca0(priv->child_count);
 
-        value2 = dict_get (xattr2, key1);
-        if (!value2)
-                return -1;
+    afr_readables_intersect_get(inode, this, &event, intersection);
 
-        if (value1->len != value2->len)
-                return -1;
-        if(memcmp(value1->data, value2->data, value1->len))
-                return -1;
-        else
-                return 0;
+    if (AFR_COUNT(intersection, priv->child_count) <= 0) {
+        /* TODO: If we have one brick with valid data_readable and
+         * another with metadata_readable, try to send an iatt with
+         * valid bits from both.*/
+        return -1;
+    }
 
+    memcpy(readable, intersection, sizeof(*readable) * priv->child_count);
+
+    return afr_read_subvol_select_by_policy(inode, this, intersection, args);
 }
 
-/* To conclude that both dicts are equal, we need to check if
- * 1) For every key-val pair in dict1, a match is present in dict2
- * 2) For every key-val pair in dict2, a match is present in dict1
- * We need to do both because ignoring glusterfs' internal xattrs
- * happens only in xattr_is_equal().
- */
-gf_boolean_t
-afr_xattrs_are_equal (dict_t *dict1, dict_t *dict2)
+static inline int
+afr_first_up_child(call_frame_t *frame, xlator_t *this)
 {
-        int ret = 0;
+    afr_private_t *priv = NULL;
+    afr_local_t *local = NULL;
+    int i = 0;
 
-        ret = dict_foreach (dict1, xattr_is_equal, dict2);
-        if (ret == -1)
-                return _gf_false;
+    local = frame->local;
+    priv = this->private;
 
-        ret = dict_foreach (dict2, xattr_is_equal, dict1);
-        if (ret == -1)
-                 return _gf_false;
-
-        return _gf_true;
+    for (i = 0; i < priv->child_count; i++)
+        if (local->replies[i].valid && local->replies[i].op_ret == 0)
+            return i;
+    return -1;
 }
 
-static int
-afr_get_parent_read_subvol (xlator_t *this, inode_t *parent,
-                            struct afr_reply *replies, unsigned char *readable)
+static void
+afr_attempt_readsubvol_set(call_frame_t *frame, xlator_t *this,
+                           unsigned char *success_replies,
+                           unsigned char *data_readable, int *read_subvol)
 {
-        int             i                    = 0;
-        int             par_read_subvol      = -1;
-        int             par_read_subvol_iter = -1;
-        afr_private_t  *priv                 = NULL;
+    afr_private_t *priv = NULL;
+    afr_local_t *local = NULL;
+    int spb_subvol = -1;
+    int child_count = -1;
 
-        priv = this->private;
+    if (*read_subvol != -1)
+        return;
 
-        if (parent)
-                par_read_subvol = afr_data_subvol_get (parent, this, 0, 0);
+    priv = this->private;
+    local = frame->local;
+    child_count = priv->child_count;
+
+    afr_split_brain_read_subvol_get(local->inode, this, frame, &spb_subvol);
+    if ((spb_subvol >= 0) &&
+        (AFR_COUNT(success_replies, child_count) == child_count)) {
+        *read_subvol = spb_subvol;
+    } else if (!priv->quorum_count ||
+               frame->root->pid == GF_CLIENT_PID_GLFS_HEAL) {
+        *read_subvol = afr_first_up_child(frame, this);
+    } else if (priv->quorum_count &&
+               afr_has_quorum(data_readable, this, NULL)) {
+        /* read_subvol is guaranteed to be valid if we hit this path. */
+        *read_subvol = afr_first_up_child(frame, this);
+    } else {
+        /* If quorum is enabled and we do not have a
+           readable yet, it means all good copies are down.
+        */
+        local->op_ret = -1;
+        local->op_errno = ENOTCONN;
+        gf_msg(this->name, GF_LOG_WARNING, 0, AFR_MSG_READ_SUBVOL_ERROR,
+               "no read "
+               "subvols for %s",
+               local->loc.path);
+    }
+    if (*read_subvol >= 0)
+        dict_del_sizen(local->replies[*read_subvol].xdata, GF_CONTENT_KEY);
+}
 
-        for (i = 0; i < priv->child_count; i++) {
-                if (!replies[i].valid)
-                        continue;
+static void
+afr_lookup_done(call_frame_t *frame, xlator_t *this)
+{
+    afr_private_t *priv = NULL;
+    afr_local_t *local = NULL;
+    int i = -1;
+    int op_errno = 0;
+    int read_subvol = 0;
+    int par_read_subvol = 0;
+    int ret = -1;
+    unsigned char *readable = NULL;
+    unsigned char *success_replies = NULL;
+    int event = 0;
+    struct afr_reply *replies = NULL;
+    uuid_t read_gfid = {
+        0,
+    };
+    gf_boolean_t locked_entry = _gf_false;
+    gf_boolean_t in_flight_create = _gf_false;
+    gf_boolean_t can_interpret = _gf_true;
+    inode_t *parent = NULL;
+    ia_type_t ia_type = IA_INVAL;
+    afr_read_subvol_args_t args = {
+        0,
+    };
+    char *gfid_heal_msg = NULL;
+
+    priv = this->private;
+    local = frame->local;
+    replies = local->replies;
+    parent = local->loc.parent;
+
+    locked_entry = afr_is_possibly_under_txn(AFR_ENTRY_TRANSACTION, local,
+                                             this);
+
+    readable = alloca0(priv->child_count);
+    success_replies = alloca0(priv->child_count);
+
+    afr_inode_read_subvol_get(parent, this, readable, NULL, &event);
+    par_read_subvol = afr_get_parent_read_subvol(this, parent, replies,
+                                                 readable);
+
+    /* First, check if we have a gfid-change from somewhere,
+       If so, propagate that so that a fresh lookup can be
+       issued
+    */
+    if (local->cont.lookup.needs_fresh_lookup) {
+        local->op_ret = -1;
+        local->op_errno = ESTALE;
+        goto error;
+    }
 
-                if (replies[i].op_ret < 0)
-                        continue;
+    op_errno = afr_final_errno(frame->local, this->private);
+    local->op_errno = op_errno;
 
-                if (par_read_subvol_iter == -1) {
-                        par_read_subvol_iter = i;
-                        continue;
-                }
+    read_subvol = -1;
+    afr_fill_success_replies(local, priv, success_replies);
 
-                if ((par_read_subvol_iter != par_read_subvol) && readable[i])
-                        par_read_subvol_iter = i;
+    for (i = 0; i < priv->child_count; i++) {
+        if (!replies[i].valid)
+            continue;
 
-                if (i == par_read_subvol)
-                        par_read_subvol_iter = i;
+        if (replies[i].op_ret == -1) {
+            if (locked_entry && replies[i].op_errno == ENOENT) {
+                in_flight_create = _gf_true;
+            }
+            continue;
         }
-        /* At the end of the for-loop, the only reason why @par_read_subvol_iter
-         * could be -1 is when this LOOKUP has failed on all sub-volumes.
-         * So it is okay to send an arbitrary subvolume (0 in this case)
-         * as parent read subvol.
-         */
-        if (par_read_subvol_iter == -1)
-                par_read_subvol_iter = 0;
-
-        return par_read_subvol_iter;
-
-}
 
-static void
-afr_lookup_done (call_frame_t *frame, xlator_t *this)
-{
-        afr_private_t       *priv  = NULL;
-        afr_local_t         *local = NULL;
-	int                 i = -1;
-	int                 op_errno = 0;
-	int                 read_subvol = 0;
-        int                 par_read_subvol = 0;
-	unsigned char      *readable = NULL;
-	int                 event = 0;
-	struct afr_reply   *replies = NULL;
-	uuid_t              read_gfid = {0, };
-	gf_boolean_t        locked_entry = _gf_false;
-	gf_boolean_t        can_interpret = _gf_true;
-        inode_t            *parent = NULL;
-        int                 spb_choice = -1;
-
-        priv  = this->private;
-        local = frame->local;
-	replies = local->replies;
-        parent = local->loc.parent;
-
-	locked_entry = afr_is_entry_possibly_under_txn (local, this);
-
-	readable = alloca0 (priv->child_count);
-
-	afr_inode_read_subvol_get (parent, this, readable, NULL, &event);
-
-        afr_inode_split_brain_choice_get (local->inode, this,
-                                                &spb_choice);
-	/* First, check if we have a gfid-change from somewhere,
-	   If so, propagate that so that a fresh lookup can be
-	   issued
-	*/
-        if (local->cont.lookup.needs_fresh_lookup) {
-                local->op_ret = -1;
-                local->op_errno = ESTALE;
-                goto unwind;
-        }
-
-	op_errno = afr_final_errno (frame->local, this->private);
-	local->op_errno = op_errno;
-
-	read_subvol = -1;
-	for (i = 0; i < priv->child_count; i++) {
-		if (!replies[i].valid)
-			continue;
-
-		if (locked_entry && replies[i].op_ret == -1 &&
-		    replies[i].op_errno == ENOENT) {
-			/* Second, check entry is still
-			   "underway" in creation */
-			local->op_ret = -1;
-			local->op_errno = ENOENT;
-			goto unwind;
-		}
-
-		if (replies[i].op_ret == -1)
-			continue;
-
-		if (read_subvol == -1 || !readable[read_subvol]) {
-			read_subvol = i;
-			gf_uuid_copy (read_gfid, replies[i].poststat.ia_gfid);
-			local->op_ret = 0;
-		}
-	}
-
-	if (read_subvol == -1)
-		goto unwind;
-	/* We now have a read_subvol, which is readable[] (if there
-	   were any). Next we look for GFID mismatches. We don't
-	   consider a GFID mismatch as an error if read_subvol is
-	   readable[] but the mismatching GFID subvol is not.
-	*/
-	for (i = 0; i < priv->child_count; i++) {
-		if (!replies[i].valid || replies[i].op_ret == -1) {
-			if (priv->child_up[i])
-				can_interpret = _gf_false;
-			continue;
-		}
-
-		if (!gf_uuid_compare (replies[i].poststat.ia_gfid, read_gfid))
-                        continue;
-
-		can_interpret = _gf_false;
-
-		if (locked_entry)
-			continue;
-
-		/* Now GFIDs mismatch. It's OK as long as this subvol
-		   is not readable[] but read_subvol is */
-		if (readable[read_subvol] && !readable[i])
-			continue;
-
-		/* LOG ERROR */
-		local->op_ret = -1;
-		local->op_errno = EIO;
-		goto unwind;
-	}
-
-	/* Forth, for the finalized GFID, pick the best subvolume
-	   to return stats from.
-	*/
-	if (can_interpret) {
-		/* It is safe to call afr_replies_interpret() because we have
-		   a response from all the UP subvolumes and all of them resolved
-		   to the same GFID
-		*/
-		if (afr_replies_interpret (frame, this, local->inode)) {
-			read_subvol = afr_data_subvol_get (local->inode, this,
-							   0, 0);
-			afr_inode_read_subvol_reset (local->inode, this);
-			goto cant_interpret;
-		} else {
-			read_subvol = afr_data_subvol_get (local->inode, this,
-							   0, 0);
-		}
-	} else {
-	cant_interpret:
-                if (read_subvol == -1) {
-                        if (spb_choice >= 0)
-                                read_subvol = spb_choice;
-                        else
-                                read_subvol = 0;
-                }
-		dict_del (replies[read_subvol].xdata, GF_CONTENT_KEY);
-	}
-
-	afr_handle_quota_size (frame, this);
+        if (read_subvol == -1 || !readable[read_subvol]) {
+            read_subvol = i;
+            gf_uuid_copy(read_gfid, replies[i].poststat.ia_gfid);
+            ia_type = replies[i].poststat.ia_type;
+            local->op_ret = 0;
+        }
+    }
 
-unwind:
+    if (in_flight_create && !afr_has_quorum(success_replies, this, NULL)) {
+        local->op_ret = -1;
+        local->op_errno = ENOENT;
+        goto error;
+    }
+
+    if (read_subvol == -1)
+        goto error;
+    /* We now have a read_subvol, which is readable[] (if there
+       were any). Next we look for GFID mismatches. We don't
+       consider a GFID mismatch as an error if read_subvol is
+       readable[] but the mismatching GFID subvol is not.
+    */
+    for (i = 0; i < priv->child_count; i++) {
+        if (!replies[i].valid || replies[i].op_ret == -1) {
+            continue;
+        }
+
+        if (!gf_uuid_compare(replies[i].poststat.ia_gfid, read_gfid))
+            continue;
+
+        can_interpret = _gf_false;
+
+        if (locked_entry)
+            continue;
+
+        /* Now GFIDs mismatch. It's OK as long as this subvol
+           is not readable[] but read_subvol is */
+        if (readable[read_subvol] && !readable[i])
+            continue;
+
+        /* If we were called from glfsheal and there is still a gfid
+         * mismatch, succeed the lookup and let glfsheal print the
+         * response via gfid-heal-msg.*/
+        if (!dict_get_str_sizen(local->xattr_req, "gfid-heal-msg",
+                                &gfid_heal_msg))
+            goto cant_interpret;
+
+        /* LOG ERROR */
+        local->op_ret = -1;
+        local->op_errno = EIO;
+        goto error;
+    }
+
+    /* Forth, for the finalized GFID, pick the best subvolume
+       to return stats from.
+    */
+    read_subvol = -1;
+    memset(readable, 0, sizeof(*readable) * priv->child_count);
+    if (can_interpret) {
+        if (!afr_has_quorum(success_replies, this, NULL))
+            goto cant_interpret;
+        /* It is safe to call afr_replies_interpret() because we have
+           a response from all the UP subvolumes and all of them resolved
+           to the same GFID
+        */
+        gf_uuid_copy(args.gfid, read_gfid);
+        args.ia_type = ia_type;
+        ret = afr_replies_interpret(frame, this, local->inode, NULL);
+        read_subvol = afr_read_subvol_decide(local->inode, this, &args,
+                                             readable);
+        if (read_subvol == -1)
+            goto cant_interpret;
+        if (ret) {
+            afr_inode_need_refresh_set(local->inode, this);
+            dict_del_sizen(local->replies[read_subvol].xdata, GF_CONTENT_KEY);
+        }
+    } else {
+    cant_interpret:
+        afr_attempt_readsubvol_set(frame, this, success_replies, readable,
+                                   &read_subvol);
         if (read_subvol == -1) {
-                if (spb_choice >= 0)
-                        read_subvol = spb_choice;
-                else
-                        read_subvol = 0;
+            goto error;
         }
-        par_read_subvol = afr_get_parent_read_subvol (this, parent, replies,
-                                                      readable);
+    }
 
-	AFR_STACK_UNWIND (lookup, frame, local->op_ret, local->op_errno,
-			  local->inode, &local->replies[read_subvol].poststat,
-			  local->replies[read_subvol].xdata,
-			  &local->replies[par_read_subvol].postparent);
+    afr_handle_quota_size(frame, this);
+
+    afr_set_need_heal(this, local);
+    if (AFR_IS_ARBITER_BRICK(priv, read_subvol) && local->op_ret == 0) {
+        local->op_ret = -1;
+        local->op_errno = ENOTCONN;
+        gf_msg_debug(this->name, 0,
+                     "Arbiter cannot be a read subvol "
+                     "for %s",
+                     local->loc.path);
+        goto error;
+    }
+
+    ret = dict_get_str_sizen(local->xattr_req, "gfid-heal-msg", &gfid_heal_msg);
+    if (!ret) {
+        ret = dict_set_str_sizen(local->replies[read_subvol].xdata,
+                                 "gfid-heal-msg", gfid_heal_msg);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, AFR_MSG_DICT_SET_FAILED,
+                   "Error setting gfid-heal-msg dict");
+            local->op_ret = -1;
+            local->op_errno = ENOMEM;
+        }
+    }
+
+    AFR_STACK_UNWIND(lookup, frame, local->op_ret, local->op_errno,
+                     local->inode, &local->replies[read_subvol].poststat,
+                     local->replies[read_subvol].xdata,
+                     &local->replies[par_read_subvol].postparent);
+    return;
+
+error:
+    AFR_STACK_UNWIND(lookup, frame, local->op_ret, local->op_errno, NULL, NULL,
+                     NULL, NULL);
 }
 
 /*
@@ -1479,606 +3122,845 @@ unwind:
  * others in that they must be given higher priority while
  * returning to the user.
  *
- * The hierarchy is ENODATA > ENOENT > ESTALE > others
+ * The hierarchy is ENODATA > ENOENT > ESTALE > ENOSPC others
  */
 
 int
-afr_higher_errno (int32_t old_errno, int32_t new_errno)
+afr_higher_errno(int32_t old_errno, int32_t new_errno)
 {
-	if (old_errno == ENODATA || new_errno == ENODATA)
-		return ENODATA;
-        if (old_errno == ENOENT || new_errno == ENOENT)
-                return ENOENT;
-	if (old_errno == ESTALE || new_errno == ESTALE)
-		return ESTALE;
-
-	return new_errno;
+    if (old_errno == ENODATA || new_errno == ENODATA)
+        return ENODATA;
+    if (old_errno == ENOENT || new_errno == ENOENT)
+        return ENOENT;
+    if (old_errno == ESTALE || new_errno == ESTALE)
+        return ESTALE;
+    if (old_errno == ENOSPC || new_errno == ENOSPC)
+        return ENOSPC;
+
+    return new_errno;
 }
 
-
 int
-afr_final_errno (afr_local_t *local, afr_private_t *priv)
+afr_final_errno(afr_local_t *local, afr_private_t *priv)
 {
-	int i = 0;
-	int op_errno = 0;
-	int tmp_errno = 0;
-
-	for (i = 0; i < priv->child_count; i++) {
-		if (!local->replies[i].valid)
-			continue;
-		if (local->replies[i].op_ret == 0)
-			continue;
-		tmp_errno = local->replies[i].op_errno;
-		op_errno = afr_higher_errno (op_errno, tmp_errno);
-	}
-
-	return op_errno;
+    int i = 0;
+    int op_errno = 0;
+    int tmp_errno = 0;
+
+    for (i = 0; i < priv->child_count; i++) {
+        if (!local->replies[i].valid)
+            continue;
+        if (local->replies[i].op_ret >= 0)
+            continue;
+        tmp_errno = local->replies[i].op_errno;
+        op_errno = afr_higher_errno(op_errno, tmp_errno);
+    }
+
+    return op_errno;
 }
 
 static int32_t
-afr_local_discovery_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-			 int32_t op_ret, int32_t op_errno, dict_t *dict,
-			 dict_t *xdata)
+afr_local_discovery_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                        int32_t op_ret, int32_t op_errno, dict_t *dict,
+                        dict_t *xdata)
 {
-        int              ret            = 0;
-        char            *pathinfo       = NULL;
-        gf_boolean_t     is_local        = _gf_false;
-        afr_private_t   *priv           = NULL;
-        int32_t          child_index    = -1;
-
-        if (op_ret != 0) {
-                goto out;
-        }
-
-	priv = this->private;
-	child_index = (int32_t)(long)cookie;
-
-        ret = dict_get_str (dict, GF_XATTR_PATHINFO_KEY, &pathinfo);
-        if (ret != 0) {
-                goto out;
-        }
-
-        ret = glusterfs_is_local_pathinfo (pathinfo, &is_local);
-        if (ret) {
-                goto out;
-        }
-
-        /*
-         * Note that one local subvolume will override another here.  The only
-         * way to avoid that would be to retain extra information about whether
-         * the previous read_child is local, and it's just not worth it.  Even
-         * the slowest local subvolume is far preferable to a remote one.
-         */
-        if (is_local) {
-                gf_log (this->name, GF_LOG_INFO,
-                        "selecting local read_child %s",
-                        priv->children[child_index]->name);
-                priv->read_child = child_index;
-        }
+    int ret = 0;
+    char *pathinfo = NULL;
+    gf_boolean_t is_local = _gf_false;
+    afr_private_t *priv = NULL;
+    int32_t child_index = -1;
+
+    if (op_ret != 0) {
+        goto out;
+    }
+
+    priv = this->private;
+    child_index = (int32_t)(long)cookie;
+
+    ret = dict_get_str_sizen(dict, GF_XATTR_PATHINFO_KEY, &pathinfo);
+    if (ret != 0) {
+        goto out;
+    }
+
+    ret = glusterfs_is_local_pathinfo(pathinfo, &is_local);
+    if (ret) {
+        goto out;
+    }
+
+    /*
+     * Note that one local subvolume will override another here.  The only
+     * way to avoid that would be to retain extra information about whether
+     * the previous read_child is local, and it's just not worth it.  Even
+     * the slowest local subvolume is far preferable to a remote one.
+     */
+    if (is_local) {
+        priv->local[child_index] = 1;
+        /* Don't set arbiter as read child. */
+        if (AFR_IS_ARBITER_BRICK(priv, child_index))
+            goto out;
+        gf_msg(this->name, GF_LOG_INFO, 0, AFR_MSG_LOCAL_CHILD,
+               "selecting local read_child %s",
+               priv->children[child_index]->name);
+
+        priv->read_child = child_index;
+    }
 out:
-        STACK_DESTROY(frame->root);
-        return 0;
+    STACK_DESTROY(frame->root);
+    return 0;
 }
 
 static void
-afr_attempt_local_discovery (xlator_t *this, int32_t child_index)
+afr_attempt_local_discovery(xlator_t *this, int32_t child_index)
 {
-        call_frame_t    *newframe = NULL;
-        loc_t            tmploc = {0,};
-        afr_private_t   *priv = this->private;
-
-        newframe = create_frame(this,this->ctx->pool);
-        if (!newframe) {
-                return;
-        }
+    call_frame_t *newframe = NULL;
+    loc_t tmploc = {
+        0,
+    };
+    afr_private_t *priv = this->private;
+
+    newframe = create_frame(this, this->ctx->pool);
+    if (!newframe) {
+        return;
+    }
 
-        tmploc.gfid[sizeof(tmploc.gfid)-1] = 1;
-        STACK_WIND_COOKIE (newframe, afr_local_discovery_cbk,
-                           (void *)(long)child_index,
-                           priv->children[child_index],
-                           priv->children[child_index]->fops->getxattr,
-                           &tmploc, GF_XATTR_PATHINFO_KEY, NULL);
+    tmploc.gfid[sizeof(tmploc.gfid) - 1] = 1;
+    STACK_WIND_COOKIE(newframe, afr_local_discovery_cbk,
+                      (void *)(long)child_index, priv->children[child_index],
+                      priv->children[child_index]->fops->getxattr, &tmploc,
+                      GF_XATTR_PATHINFO_KEY, NULL);
 }
 
 int
-afr_lookup_sh_metadata_wrap (void *opaque)
+afr_lookup_sh_metadata_wrap(void *opaque)
 {
-        call_frame_t *frame       = opaque;
-        afr_local_t  *local       = NULL;
-        xlator_t     *this        = NULL;
-        inode_t      *inode       = NULL;
-        afr_private_t *priv       = NULL;
-        struct afr_reply *replies = NULL;
-        int i= 0, first = -1;
+    call_frame_t *frame = opaque;
+    afr_local_t *local = NULL;
+    xlator_t *this = NULL;
+    inode_t *inode = NULL;
+    afr_private_t *priv = NULL;
+    struct afr_reply *replies = NULL;
+    int i = 0, first = -1;
+    int ret = -1;
+    dict_t *dict = NULL;
+
+    local = frame->local;
+    this = frame->this;
+    priv = this->private;
+    replies = local->replies;
+
+    for (i = 0; i < priv->child_count; i++) {
+        if (!replies[i].valid || replies[i].op_ret == -1)
+            continue;
+        first = i;
+        break;
+    }
+    if (first == -1)
+        goto out;
+
+    if (afr_selfheal_metadata_by_stbuf(this, &replies[first].poststat))
+        goto out;
+
+    afr_local_replies_wipe(local, this->private);
+
+    dict = dict_new();
+    if (!dict)
+        goto out;
+    if (local->xattr_req) {
+        dict_copy(local->xattr_req, dict);
+    }
+
+    ret = dict_set_sizen_str_sizen(dict, "link-count", GF_XATTROP_INDEX_COUNT);
+    if (ret) {
+        gf_msg_debug(this->name, -ret, "Unable to set link-count in dict ");
+    }
+
+    if (loc_is_nameless(&local->loc)) {
+        ret = afr_selfheal_unlocked_discover_on(frame, local->inode,
+                                                local->loc.gfid, local->replies,
+                                                local->child_up, dict);
+    } else {
+        inode = afr_selfheal_unlocked_lookup_on(frame, local->loc.parent,
+                                                local->loc.name, local->replies,
+                                                local->child_up, dict);
+    }
+    if (inode)
+        inode_unref(inode);
+out:
+    if (loc_is_nameless(&local->loc))
+        afr_discover_done(frame, this);
+    else
+        afr_lookup_done(frame, this);
 
-        local = frame->local;
-        this  = frame->this;
-        priv  = this->private;
-        replies = local->replies;
-
-        for (i =0; i < priv->child_count; i++) {
-                if(!replies[i].valid || replies[i].op_ret == -1)
-                        continue;
-                first = i;
-                break;
-        }
-        if (first == -1)
-                goto out;
+    if (dict)
+        dict_unref(dict);
 
-        inode = afr_inode_link (local->inode,&replies[first].poststat);
-        if(!inode)
-                goto out;
+    return 0;
+}
 
-        afr_selfheal_metadata (frame, this, inode);
-        inode_forget (inode, 1);
-        inode_unref (inode);
+gf_boolean_t
+afr_is_pending_set(xlator_t *this, dict_t *xdata, int type)
+{
+    int idx = -1;
+    afr_private_t *priv = NULL;
+    void *pending_raw = NULL;
+    int *pending_int = NULL;
+    int i = 0;
 
-        afr_local_replies_wipe (local, this->private);
-        inode = afr_selfheal_unlocked_lookup_on (frame, local->loc.parent,
-                                                 local->loc.name, local->replies,
-                                                 local->child_up, NULL);
-        if (inode)
-                inode_unref (inode);
-out:
-        afr_lookup_done (frame, this);
+    priv = this->private;
+    idx = afr_index_for_transaction_type(type);
 
-        return 0;
+    if (dict_get_ptr(xdata, AFR_DIRTY, &pending_raw) == 0) {
+        if (pending_raw) {
+            pending_int = pending_raw;
+
+            if (ntoh32(pending_int[idx]))
+                return _gf_true;
+        }
+    }
+
+    for (i = 0; i < priv->child_count; i++) {
+        if (dict_get_ptr(xdata, priv->pending_key[i], &pending_raw))
+            continue;
+        if (!pending_raw)
+            continue;
+        pending_int = pending_raw;
+
+        if (ntoh32(pending_int[idx]))
+            return _gf_true;
+    }
+
+    return _gf_false;
 }
 
 static gf_boolean_t
 afr_can_start_metadata_self_heal(call_frame_t *frame, xlator_t *this)
 {
-        afr_local_t *local = NULL;
-        afr_private_t *priv = NULL;
-        struct afr_reply *replies = NULL;
-        int i = 0, first = -1;
-        gf_boolean_t start = _gf_false;
-        struct iatt stbuf = {0, };
-
-        local = frame->local;
-        replies = local->replies;
-        priv = this->private;
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
+    struct afr_reply *replies = NULL;
+    int i = 0, first = -1;
+    gf_boolean_t start = _gf_false;
+    struct iatt stbuf = {
+        0,
+    };
+
+    local = frame->local;
+    replies = local->replies;
+    priv = this->private;
+
+    if (!priv->metadata_self_heal)
+        return _gf_false;
 
-        for (i = 0; i < priv->child_count; i++) {
-                if(!replies[i].valid || replies[i].op_ret == -1)
-                        continue;
-                if (first == -1) {
-                        first = i;
-                        stbuf = replies[i].poststat;
-                        continue;
-                }
+    for (i = 0; i < priv->child_count; i++) {
+        if (!replies[i].valid || replies[i].op_ret == -1)
+            continue;
+        if (first == -1) {
+            first = i;
+            stbuf = replies[i].poststat;
+            continue;
+        }
 
-                if (gf_uuid_compare (stbuf.ia_gfid, replies[i].poststat.ia_gfid)) {
-                        start = _gf_false;
-                        break;
-                }
-                if (!IA_EQUAL (stbuf, replies[i].poststat, type)) {
-                        start = _gf_false;
-                        break;
-                }
+        if (afr_is_pending_set(this, replies[i].xdata,
+                               AFR_METADATA_TRANSACTION)) {
+            /* Let shd do the heal so that lookup is not blocked
+             * on getting metadata lock/doing the heal */
+            start = _gf_false;
+            break;
+        }
 
-                /*Check if iattrs need heal*/
-                if ((!IA_EQUAL (stbuf, replies[i].poststat, uid)) ||
-                    (!IA_EQUAL (stbuf, replies[i].poststat, gid)) ||
-                    (!IA_EQUAL (stbuf, replies[i].poststat, prot))) {
-                        start = _gf_true;
-                        continue;
-                }
+        if (gf_uuid_compare(stbuf.ia_gfid, replies[i].poststat.ia_gfid)) {
+            start = _gf_false;
+            break;
+        }
+        if (!IA_EQUAL(stbuf, replies[i].poststat, type)) {
+            start = _gf_false;
+            break;
+        }
 
-                /*Check if xattrs need heal*/
-                if (!afr_xattrs_are_equal (replies[first].xdata,
-                                           replies[i].xdata))
-                        start = _gf_true;
+        /*Check if iattrs need heal*/
+        if ((!IA_EQUAL(stbuf, replies[i].poststat, uid)) ||
+            (!IA_EQUAL(stbuf, replies[i].poststat, gid)) ||
+            (!IA_EQUAL(stbuf, replies[i].poststat, prot))) {
+            start = _gf_true;
+            continue;
         }
 
-        return start;
+        /*Check if xattrs need heal*/
+        if (!afr_xattrs_are_equal(replies[first].xdata, replies[i].xdata))
+            start = _gf_true;
+    }
+
+    return start;
 }
 
 int
-afr_lookup_metadata_heal_check (call_frame_t *frame, xlator_t *this)
+afr_lookup_metadata_heal_check(call_frame_t *frame, xlator_t *this)
 
 {
-        call_frame_t *heal = NULL;
-        int ret            = 0;
-
-        if (!afr_can_start_metadata_self_heal (frame, this))
-                goto out;
-
-        heal = copy_frame (frame);
-        if (heal)
-                heal->root->pid = GF_CLIENT_PID_AFR_SELF_HEALD;
-        ret = synctask_new (this->ctx->env, afr_lookup_sh_metadata_wrap,
-                            afr_refresh_selfheal_done, heal, frame);
-        if(ret)
-                goto out;
-        return ret;
+    call_frame_t *heal = NULL;
+    afr_local_t *local = NULL;
+    int ret = 0;
+
+    local = frame->local;
+    if (!afr_can_start_metadata_self_heal(frame, this))
+        goto out;
+
+    heal = afr_frame_create(this, &ret);
+    if (!heal) {
+        ret = -ret;
+        goto out;
+    }
+
+    ret = synctask_new(this->ctx->env, afr_lookup_sh_metadata_wrap,
+                       afr_refresh_selfheal_done, heal, frame);
+    if (ret)
+        goto out;
+    return ret;
 out:
-        afr_lookup_done (frame, this);
-        return ret;
+    if (loc_is_nameless(&local->loc))
+        afr_discover_done(frame, this);
+    else
+        afr_lookup_done(frame, this);
+    if (heal)
+        AFR_STACK_DESTROY(heal);
+    return ret;
 }
 
 int
-afr_lookup_selfheal_wrap (void *opaque)
+afr_lookup_selfheal_wrap(void *opaque)
 {
-        int ret = 0;
-	call_frame_t *frame = opaque;
-	afr_local_t *local = NULL;
-	xlator_t *this = NULL;
-	inode_t *inode = NULL;
+    int ret = 0;
+    call_frame_t *frame = opaque;
+    afr_local_t *local = NULL;
+    xlator_t *this = NULL;
+    inode_t *inode = NULL;
+    uuid_t pargfid = {
+        0,
+    };
+
+    local = frame->local;
+    this = frame->this;
+    loc_pargfid(&local->loc, pargfid);
+
+    ret = afr_selfheal_name(frame->this, pargfid, local->loc.name,
+                            &local->cont.lookup.gfid_req, local->xattr_req);
+    if (ret == -EIO)
+        goto unwind;
+
+    afr_local_replies_wipe(local, this->private);
+
+    inode = afr_selfheal_unlocked_lookup_on(frame, local->loc.parent,
+                                            local->loc.name, local->replies,
+                                            local->child_up, local->xattr_req);
+    if (inode)
+        inode_unref(inode);
+
+    afr_lookup_metadata_heal_check(frame, this);
+    return 0;
 
-	local = frame->local;
-	this = frame->this;
+unwind:
+    AFR_STACK_UNWIND(lookup, frame, -1, EIO, NULL, NULL, NULL, NULL);
+    return 0;
+}
 
-	ret = afr_selfheal_name (frame->this, local->loc.pargfid,
-                                 local->loc.name, &local->cont.lookup.gfid_req);
-        if (ret == -EIO)
-                goto unwind;
+int
+afr_lookup_entry_heal(call_frame_t *frame, xlator_t *this)
+{
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
+    call_frame_t *heal = NULL;
+    int i = 0, first = -1;
+    gf_boolean_t name_state_mismatch = _gf_false;
+    struct afr_reply *replies = NULL;
+    int ret = 0;
+    unsigned char *par_readables = NULL;
+    unsigned char *success = NULL;
+    int32_t op_errno = 0;
+    uuid_t gfid = {0};
+
+    local = frame->local;
+    replies = local->replies;
+    priv = this->private;
+    par_readables = alloca0(priv->child_count);
+    success = alloca0(priv->child_count);
+
+    ret = afr_inode_read_subvol_get(local->loc.parent, this, par_readables,
+                                    NULL, NULL);
+    if (ret < 0 || AFR_COUNT(par_readables, priv->child_count) == 0) {
+        /* In this case set par_readables to all 1 so that name_heal
+         * need checks at the end of this function will flag missing
+         * entry when name state mismatches*/
+        memset(par_readables, 1, priv->child_count);
+    }
+
+    for (i = 0; i < priv->child_count; i++) {
+        if (!replies[i].valid)
+            continue;
+
+        if (replies[i].op_ret == 0) {
+            if (gf_uuid_is_null(gfid)) {
+                gf_uuid_copy(gfid, replies[i].poststat.ia_gfid);
+            }
+            success[i] = 1;
+        } else {
+            if ((replies[i].op_errno != ENOTCONN) &&
+                (replies[i].op_errno != ENOENT) &&
+                (replies[i].op_errno != ESTALE)) {
+                op_errno = replies[i].op_errno;
+            }
+        }
 
-        afr_local_replies_wipe (local, this->private);
+        /*gfid is missing, needs heal*/
+        if ((replies[i].op_ret == -1) && (replies[i].op_errno == ENODATA)) {
+            goto name_heal;
+        }
 
-	inode = afr_selfheal_unlocked_lookup_on (frame, local->loc.parent,
-						 local->loc.name, local->replies,
-						 local->child_up, NULL);
-	if (inode)
-		inode_unref (inode);
+        if (first == -1) {
+            first = i;
+            continue;
+        }
 
-        afr_lookup_metadata_heal_check(frame, this);
-        return 0;
+        if (replies[i].op_ret != replies[first].op_ret) {
+            name_state_mismatch = _gf_true;
+        }
 
-unwind:
-	AFR_STACK_UNWIND (lookup, frame, -1, EIO, NULL, NULL, NULL, NULL);
-        return 0;
-}
+        if (replies[i].op_ret == 0) {
+            /* Rename after this lookup may succeed if we don't do
+             * a name-heal and the destination may not have pending xattrs
+             * to indicate which name is good and which is bad so always do
+             * this heal*/
+            if (gf_uuid_compare(replies[i].poststat.ia_gfid, gfid)) {
+                goto name_heal;
+            }
+        }
+    }
 
+    if (name_state_mismatch) {
+        if (!priv->quorum_count)
+            goto name_heal;
+        if (!afr_has_quorum(success, this, NULL))
+            goto name_heal;
+        if (op_errno)
+            goto name_heal;
+        for (i = 0; i < priv->child_count; i++) {
+            if (!replies[i].valid)
+                continue;
+            if (par_readables[i] && replies[i].op_ret < 0 &&
+                replies[i].op_errno != ENOTCONN) {
+                goto name_heal;
+            }
+        }
+    }
+
+    goto metadata_heal;
+
+name_heal:
+    heal = afr_frame_create(this, NULL);
+    if (!heal)
+        goto metadata_heal;
+
+    ret = synctask_new(this->ctx->env, afr_lookup_selfheal_wrap,
+                       afr_refresh_selfheal_done, heal, frame);
+    if (ret) {
+        AFR_STACK_DESTROY(heal);
+        goto metadata_heal;
+    }
+    return ret;
 
-int
-afr_lookup_entry_heal (call_frame_t *frame, xlator_t *this)
-{
-	afr_local_t *local = NULL;
-	afr_private_t *priv = NULL;
-	call_frame_t *heal = NULL;
-	int i = 0, first = -1;
-	gf_boolean_t need_heal = _gf_false;
-	struct afr_reply *replies = NULL;
-	int ret = 0;
-
-	local = frame->local;
-	replies = local->replies;
-	priv = this->private;
-
-	for (i = 0; i < priv->child_count; i++) {
-		if (!replies[i].valid)
-			continue;
-
-                if ((replies[i].op_ret == -1) &&
-                    (replies[i].op_errno == ENODATA))
-                        need_heal = _gf_true;
-
-		if (first == -1) {
-			first = i;
-			continue;
-		}
-
-		if (replies[i].op_ret != replies[first].op_ret) {
-			need_heal = _gf_true;
-			break;
-		}
-
-		if (gf_uuid_compare (replies[i].poststat.ia_gfid,
-				  replies[first].poststat.ia_gfid)) {
-			need_heal = _gf_true;
-			break;
-		}
-	}
-
-	if (need_heal) {
-		heal = copy_frame (frame);
-		if (heal)
-			heal->root->pid = GF_CLIENT_PID_AFR_SELF_HEALD;
-		ret = synctask_new (this->ctx->env, afr_lookup_selfheal_wrap,
-				    afr_refresh_selfheal_done, heal, frame);
-		if (ret)
-			goto metadata_heal;
-                return ret;
-	}
 metadata_heal:
-        ret = afr_lookup_metadata_heal_check (frame, this);
+    ret = afr_lookup_metadata_heal_check(frame, this);
 
-	return ret;
+    return ret;
 }
 
-
 int
-afr_lookup_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-		int op_ret, int op_errno, inode_t *inode, struct iatt *buf,
-		dict_t *xdata, struct iatt *postparent)
+afr_lookup_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int op_ret,
+               int op_errno, inode_t *inode, struct iatt *buf, dict_t *xdata,
+               struct iatt *postparent)
 {
-        afr_local_t *   local = NULL;
-        int             call_count      = -1;
-        int             child_index     = -1;
-
-	child_index = (long) cookie;
-
-	local = frame->local;
-
-	local->replies[child_index].valid = 1;
-	local->replies[child_index].op_ret = op_ret;
-	local->replies[child_index].op_errno = op_errno;
-        /*
-         * On revalidate lookup if the gfid-changed, afr should unwind the fop
-         * with ESTALE so that a fresh lookup will be sent by the top xlator.
-         * So remember it.
-         */
-        if (xdata && dict_get (xdata, "gfid-changed"))
-                local->cont.lookup.needs_fresh_lookup = _gf_true;
+    afr_local_t *local = NULL;
+    int call_count = -1;
+    int child_index = -1;
+    GF_UNUSED int ret = 0;
+    int8_t need_heal = 1;
+
+    child_index = (long)cookie;
+
+    local = frame->local;
+
+    local->replies[child_index].valid = 1;
+    local->replies[child_index].op_ret = op_ret;
+    local->replies[child_index].op_errno = op_errno;
+    /*
+     * On revalidate lookup if the gfid-changed, afr should unwind the fop
+     * with ESTALE so that a fresh lookup will be sent by the top xlator.
+     * So remember it.
+     */
+    if (xdata && dict_get_sizen(xdata, "gfid-changed"))
+        local->cont.lookup.needs_fresh_lookup = _gf_true;
+
+    if (xdata) {
+        ret = dict_get_int8(xdata, "link-count", &need_heal);
+        local->replies[child_index].need_heal = need_heal;
+    } else {
+        local->replies[child_index].need_heal = need_heal;
+    }
+    if (op_ret != -1) {
+        local->replies[child_index].poststat = *buf;
+        local->replies[child_index].postparent = *postparent;
+        if (xdata)
+            local->replies[child_index].xdata = dict_ref(xdata);
+    }
 
-	if (op_ret != -1) {
-		local->replies[child_index].poststat = *buf;
-		local->replies[child_index].postparent = *postparent;
-		if (xdata)
-			local->replies[child_index].xdata = dict_ref (xdata);
-	}
+    call_count = afr_frame_return(frame);
+    if (call_count == 0) {
+        afr_set_need_heal(this, local);
+        afr_lookup_entry_heal(frame, this);
+    }
 
-        call_count = afr_frame_return (frame);
-        if (call_count == 0) {
-		afr_lookup_entry_heal (frame, this);
-        }
-
-	return 0;
+    return 0;
 }
 
-
-
 static void
-afr_discover_done (call_frame_t *frame, xlator_t *this)
+afr_discover_unwind(call_frame_t *frame, xlator_t *this)
 {
-        afr_private_t       *priv  = NULL;
-        afr_local_t         *local = NULL;
-	int                 i = -1;
-	int                 op_errno = 0;
-	int                 read_subvol = 0;
+    afr_private_t *priv = NULL;
+    afr_local_t *local = NULL;
+    int read_subvol = -1;
+    int ret = 0;
+    unsigned char *data_readable = NULL;
+    unsigned char *success_replies = NULL;
+
+    priv = this->private;
+    local = frame->local;
+    data_readable = alloca0(priv->child_count);
+    success_replies = alloca0(priv->child_count);
+
+    afr_fill_success_replies(local, priv, success_replies);
+    if (AFR_COUNT(success_replies, priv->child_count) > 0)
+        local->op_ret = 0;
+
+    if (local->op_ret < 0) {
+        local->op_ret = -1;
+        local->op_errno = afr_final_errno(frame->local, this->private);
+        goto error;
+    }
 
-        priv  = this->private;
-        local = frame->local;
+    if (!afr_has_quorum(success_replies, this, frame))
+        goto unwind;
 
-	for (i = 0; i < priv->child_count; i++) {
-		if (!local->replies[i].valid)
-			continue;
-		if (local->replies[i].op_ret == 0)
-			local->op_ret = 0;
-	}
-
-	op_errno = afr_final_errno (frame->local, this->private);
-
-        if (local->op_ret < 0) {
-		local->op_errno = op_errno;
-		local->op_ret = -1;
-                goto unwind;
-	}
-
-	afr_replies_interpret (frame, this, local->inode);
-
-	read_subvol = afr_data_subvol_get (local->inode, this, 0, 0);
-	if (read_subvol == -1) {
-		gf_log (this->name, GF_LOG_WARNING, "no read subvols for %s",
-			local->loc.path);
-
-		for (i = 0; i < priv->child_count; i++) {
-			if (!local->replies[i].valid ||
-			    local->replies[i].op_ret == -1)
-				continue;
-			read_subvol = i;
-			break;
-		}
-	}
+    ret = afr_replies_interpret(frame, this, local->inode, NULL);
+    if (ret) {
+        afr_inode_need_refresh_set(local->inode, this);
+    }
+
+    read_subvol = afr_read_subvol_decide(local->inode, this, NULL,
+                                         data_readable);
 
 unwind:
-	if (read_subvol == -1) {
-                afr_inode_split_brain_choice_get (local->inode, this,
-                                                        &read_subvol);
-                if (read_subvol == -1)
-                        read_subvol = 0;
-        }
+    afr_attempt_readsubvol_set(frame, this, success_replies, data_readable,
+                               &read_subvol);
+    if (read_subvol == -1)
+        goto error;
 
-	AFR_STACK_UNWIND (lookup, frame, local->op_ret, local->op_errno,
-			  local->inode, &local->replies[read_subvol].poststat,
-			  local->replies[read_subvol].xdata,
-			  &local->replies[read_subvol].postparent);
+    if (AFR_IS_ARBITER_BRICK(priv, read_subvol) && local->op_ret == 0) {
+        local->op_ret = -1;
+        local->op_errno = ENOTCONN;
+        gf_msg_debug(this->name, 0,
+                     "Arbiter cannot be a read subvol "
+                     "for %s",
+                     local->loc.path);
+    }
+
+    AFR_STACK_UNWIND(lookup, frame, local->op_ret, local->op_errno,
+                     local->inode, &local->replies[read_subvol].poststat,
+                     local->replies[read_subvol].xdata,
+                     &local->replies[read_subvol].postparent);
+    return;
+
+error:
+    AFR_STACK_UNWIND(lookup, frame, local->op_ret, local->op_errno, NULL, NULL,
+                     NULL, NULL);
 }
 
-
-int
-afr_discover_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-		  int op_ret, int op_errno, inode_t *inode, struct iatt *buf,
-		  dict_t *xdata, struct iatt *postparent)
+static int
+afr_ta_id_file_check(void *opaque)
 {
-        afr_local_t *   local = NULL;
-        int             call_count      = -1;
-        int             child_index     = -1;
+    afr_private_t *priv = NULL;
+    xlator_t *this = NULL;
+    loc_t loc = {
+        0,
+    };
+    struct iatt stbuf = {
+        0,
+    };
+    dict_t *dict = NULL;
+    uuid_t gfid = {
+        0,
+    };
+    fd_t *fd = NULL;
+    int ret = 0;
+
+    this = opaque;
+    priv = this->private;
+
+    ret = afr_fill_ta_loc(this, &loc, _gf_false);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, -ret, AFR_MSG_THIN_ARB,
+               "Failed to populate thin-arbiter loc for: %s.", loc.name);
+        goto out;
+    }
+
+    ret = syncop_lookup(priv->children[THIN_ARBITER_BRICK_INDEX], &loc, &stbuf,
+                        0, 0, 0);
+    if (ret == 0) {
+        goto out;
+    } else if (ret == -ENOENT) {
+        fd = fd_create(loc.inode, getpid());
+        if (!fd)
+            goto out;
+        dict = dict_new();
+        if (!dict)
+            goto out;
+        gf_uuid_generate(gfid);
+        ret = dict_set_gfuuid(dict, "gfid-req", gfid, true);
+        ret = syncop_create(priv->children[THIN_ARBITER_BRICK_INDEX], &loc,
+                            O_RDWR, 0664, fd, &stbuf, dict, NULL);
+    }
 
-	child_index = (long) cookie;
-
-	local = frame->local;
-
-	local->replies[child_index].valid = 1;
-	local->replies[child_index].op_ret = op_ret;
-	local->replies[child_index].op_errno = op_errno;
-	if (op_ret != -1) {
-		local->replies[child_index].poststat = *buf;
-		local->replies[child_index].postparent = *postparent;
-		if (xdata)
-			local->replies[child_index].xdata = dict_ref (xdata);
-	}
-
-        if (local->do_discovery && (op_ret == 0))
-                afr_attempt_local_discovery (this, child_index);
-
-        call_count = afr_frame_return (frame);
-        if (call_count == 0) {
-               afr_discover_done (frame, this);
-        }
+out:
+    if (ret == 0) {
+        gf_uuid_copy(priv->ta_gfid, stbuf.ia_gfid);
+    } else {
+        gf_msg(this->name, GF_LOG_ERROR, -ret, AFR_MSG_THIN_ARB,
+               "Failed to lookup/create thin-arbiter id file.");
+    }
+    if (dict)
+        dict_unref(dict);
+    if (fd)
+        fd_unref(fd);
+    loc_wipe(&loc);
+
+    return 0;
+}
 
-	return 0;
+static int
+afr_ta_id_file_check_cbk(int ret, call_frame_t *ta_frame, void *opaque)
+{
+    return 0;
 }
 
+static void
+afr_discover_done(call_frame_t *frame, xlator_t *this)
+{
+    int ret = 0;
+    afr_private_t *priv = NULL;
+
+    priv = this->private;
+    if (!priv->thin_arbiter_count)
+        goto unwind;
+    if (!gf_uuid_is_null(priv->ta_gfid))
+        goto unwind;
+
+    ret = synctask_new(this->ctx->env, afr_ta_id_file_check,
+                       afr_ta_id_file_check_cbk, NULL, this);
+    if (ret)
+        goto unwind;
+unwind:
+    afr_discover_unwind(frame, this);
+}
 
 int
-afr_discover_do (call_frame_t *frame, xlator_t *this, int err)
+afr_discover_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int op_ret,
+                 int op_errno, inode_t *inode, struct iatt *buf, dict_t *xdata,
+                 struct iatt *postparent)
 {
-	int ret = 0;
-	int i = 0;
-	afr_local_t *local = NULL;
-	afr_private_t *priv = NULL;
-	int call_count = 0;
-
-	local = frame->local;
-	priv = this->private;
-
-	if (err) {
-		local->op_errno = -err;
-		ret = -1;
-		goto out;
-	}
-
-	call_count = local->call_count = AFR_COUNT (local->child_up,
-						    priv->child_count);
+    afr_local_t *local = NULL;
+    int call_count = -1;
+    int child_index = -1;
+    GF_UNUSED int ret = 0;
+    int8_t need_heal = 1;
+
+    child_index = (long)cookie;
+
+    local = frame->local;
+
+    local->replies[child_index].valid = 1;
+    local->replies[child_index].op_ret = op_ret;
+    local->replies[child_index].op_errno = op_errno;
+    if (op_ret != -1) {
+        local->replies[child_index].poststat = *buf;
+        local->replies[child_index].postparent = *postparent;
+        if (xdata)
+            local->replies[child_index].xdata = dict_ref(xdata);
+    }
+
+    if (local->do_discovery && (op_ret == 0))
+        afr_attempt_local_discovery(this, child_index);
+
+    if (xdata) {
+        ret = dict_get_int8(xdata, "link-count", &need_heal);
+        local->replies[child_index].need_heal = need_heal;
+    } else {
+        local->replies[child_index].need_heal = need_heal;
+    }
+
+    call_count = afr_frame_return(frame);
+    if (call_count == 0) {
+        afr_set_need_heal(this, local);
+        afr_lookup_metadata_heal_check(frame, this);
+    }
 
-        ret = afr_lookup_xattr_req_prepare (local, this, local->xattr_req,
-					    &local->loc);
-        if (ret) {
-                local->op_errno = -ret;
-		ret = -1;
-                goto out;
-        }
+    return 0;
+}
 
-        for (i = 0; i < priv->child_count; i++) {
-                if (local->child_up[i]) {
-                        STACK_WIND_COOKIE (frame, afr_discover_cbk,
-                                           (void *) (long) i,
-                                           priv->children[i],
-                                           priv->children[i]->fops->lookup,
-                                           &local->loc, local->xattr_req);
-                        if (!--call_count)
-                                break;
-                }
+int
+afr_discover_do(call_frame_t *frame, xlator_t *this, int err)
+{
+    int ret = 0;
+    int i = 0;
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
+    int call_count = 0;
+
+    local = frame->local;
+    priv = this->private;
+
+    if (err) {
+        local->op_errno = err;
+        goto out;
+    }
+
+    call_count = local->call_count = AFR_COUNT(local->child_up,
+                                               priv->child_count);
+
+    ret = afr_lookup_xattr_req_prepare(local, this, local->xattr_req,
+                                       &local->loc);
+    if (ret) {
+        local->op_errno = -ret;
+        goto out;
+    }
+
+    for (i = 0; i < priv->child_count; i++) {
+        if (local->child_up[i]) {
+            STACK_WIND_COOKIE(
+                frame, afr_discover_cbk, (void *)(long)i, priv->children[i],
+                priv->children[i]->fops->lookup, &local->loc, local->xattr_req);
+            if (!--call_count)
+                break;
         }
+    }
 
-	return 0;
+    return 0;
 out:
-	AFR_STACK_UNWIND (lookup, frame, -1, local->op_errno, 0, 0, 0, 0);
-	return 0;
+    AFR_STACK_UNWIND(lookup, frame, -1, local->op_errno, 0, 0, 0, 0);
+    return 0;
 }
 
-
 int
-afr_discover (call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xattr_req)
+afr_discover(call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xattr_req)
 {
-	int op_errno = ENOMEM;
-	afr_private_t *priv = NULL;
-	afr_local_t *local = NULL;
-	int event = 0;
+    int op_errno = ENOMEM;
+    afr_private_t *priv = NULL;
+    afr_local_t *local = NULL;
+    int event = 0;
 
-	priv = this->private;
+    priv = this->private;
 
-	local = AFR_FRAME_INIT (frame, op_errno);
-	if (!local)
-		goto out;
+    local = AFR_FRAME_INIT(frame, op_errno);
+    if (!local)
+        goto out;
 
-        if (!local->call_count) {
-                op_errno = ENOTCONN;
-                goto out;
-        }
+    if (!local->call_count) {
+        op_errno = ENOTCONN;
+        goto out;
+    }
 
-	if (__is_root_gfid (loc->inode->gfid)) {
-		if (!this->itable)
-			this->itable = loc->inode->table;
-		if (!priv->root_inode)
-			priv->root_inode = inode_ref (loc->inode);
+    if (__is_root_gfid(loc->inode->gfid)) {
+        if (!priv->root_inode)
+            priv->root_inode = inode_ref(loc->inode);
 
-		if (priv->choose_local && !priv->did_discovery) {
-			/* Logic to detect which subvolumes of AFR are
-			   local, in order to prefer them for reads
-			*/
-			local->do_discovery = _gf_true;
-                        priv->did_discovery = _gf_true;
-                }
-	}
+        if (priv->choose_local && !priv->did_discovery) {
+            /* Logic to detect which subvolumes of AFR are
+               local, in order to prefer them for reads
+            */
+            local->do_discovery = _gf_true;
+            priv->did_discovery = _gf_true;
+        }
+    }
 
-        local->op = GF_FOP_LOOKUP;
+    local->op = GF_FOP_LOOKUP;
 
-        loc_copy (&local->loc, loc);
+    loc_copy(&local->loc, loc);
 
-	local->inode = inode_ref (loc->inode);
+    local->inode = inode_ref(loc->inode);
 
-	if (xattr_req)
-		/* If xattr_req was null, afr_lookup_xattr_req_prepare() will
-		   allocate one for us */
-		local->xattr_req = dict_ref (xattr_req);
+    if (xattr_req) {
+        /* If xattr_req was null, afr_lookup_xattr_req_prepare() will
+           allocate one for us */
+        local->xattr_req = dict_copy_with_ref(xattr_req, NULL);
+        if (!local->xattr_req) {
+            op_errno = ENOMEM;
+            goto out;
+        }
+    }
 
-	if (gf_uuid_is_null (loc->inode->gfid)) {
-		afr_discover_do (frame, this, 0);
-		return 0;
-	}
+    if (gf_uuid_is_null(loc->inode->gfid)) {
+        afr_discover_do(frame, this, 0);
+        return 0;
+    }
 
-	afr_read_subvol_get (loc->inode, this, NULL, &event,
-			     AFR_DATA_TRANSACTION);
+    afr_read_subvol_get(loc->inode, this, NULL, NULL, &event,
+                        AFR_DATA_TRANSACTION, NULL);
 
-	if (event != local->event_generation)
-		afr_inode_refresh (frame, this, loc->inode, afr_discover_do);
-	else
-		afr_discover_do (frame, this, 0);
+    afr_discover_do(frame, this, 0);
 
-	return 0;
+    return 0;
 out:
-	AFR_STACK_UNWIND (lookup, frame, -1, op_errno, NULL, NULL, NULL, NULL);
-	return 0;
+    AFR_STACK_UNWIND(lookup, frame, -1, op_errno, NULL, NULL, NULL, NULL);
+    return 0;
 }
 
-
 int
-afr_lookup_do (call_frame_t *frame, xlator_t *this, int err)
+afr_lookup_do(call_frame_t *frame, xlator_t *this, int err)
 {
-	int ret = 0;
-	int i = 0;
-	afr_local_t *local = NULL;
-	afr_private_t *priv = NULL;
-	int call_count = 0;
-
-	local = frame->local;
-	priv = this->private;
-
-	if (err < 0) {
-		local->op_errno = -err;
-		ret = -1;
-		goto out;
-	}
-
-	call_count = local->call_count = AFR_COUNT (local->child_up,
-						    priv->child_count);
-
-        ret = afr_lookup_xattr_req_prepare (local, this, local->xattr_req,
-					    &local->loc);
-        if (ret) {
-                local->op_errno = -ret;
-		ret = -1;
-                goto out;
-        }
-
-        for (i = 0; i < priv->child_count; i++) {
-                if (local->child_up[i]) {
-                        STACK_WIND_COOKIE (frame, afr_lookup_cbk,
-                                           (void *) (long) i,
-                                           priv->children[i],
-                                           priv->children[i]->fops->lookup,
-                                           &local->loc, local->xattr_req);
-                        if (!--call_count)
-                                break;
-                }
+    int ret = 0;
+    int i = 0;
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
+    int call_count = 0;
+
+    local = frame->local;
+    priv = this->private;
+
+    if (err < 0) {
+        local->op_errno = err;
+        goto out;
+    }
+
+    call_count = local->call_count = AFR_COUNT(local->child_up,
+                                               priv->child_count);
+
+    ret = afr_lookup_xattr_req_prepare(local, this, local->xattr_req,
+                                       &local->loc);
+    if (ret) {
+        local->op_errno = -ret;
+        goto out;
+    }
+
+    for (i = 0; i < priv->child_count; i++) {
+        if (local->child_up[i]) {
+            STACK_WIND_COOKIE(
+                frame, afr_lookup_cbk, (void *)(long)i, priv->children[i],
+                priv->children[i]->fops->lookup, &local->loc, local->xattr_req);
+            if (!--call_count)
+                break;
         }
-	return 0;
+    }
+    return 0;
 out:
-	AFR_STACK_UNWIND (lookup, frame, -1, local->op_errno, 0, 0, 0, 0);
-	return 0;
+    AFR_STACK_UNWIND(lookup, frame, -1, local->op_errno, 0, 0, 0, 0);
+    return 0;
 }
 
 /*
@@ -2118,2635 +4000,3879 @@ out:
  */
 
 int
-afr_lookup (call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xattr_req)
-{
-        afr_local_t   *local = NULL;
-        int32_t        op_errno = 0;
-	int            event = 0;
-        void          *gfid_req = NULL;
-        int            ret = 0;
-
-	if (!loc->parent && gf_uuid_is_null (loc->pargfid)) {
-                if (xattr_req)
-                        dict_del (xattr_req, "gfid-req");
-		afr_discover (frame, this, loc, xattr_req);
-		return 0;
-	}
-
-	if (__is_root_gfid (loc->parent->gfid)) {
-		if (!strcmp (loc->name, GF_REPLICATE_TRASH_DIR)) {
-			op_errno = EPERM;
-			goto out;
-		}
-	}
-
-	local = AFR_FRAME_INIT (frame, op_errno);
-	if (!local)
-		goto out;
-
-        if (!local->call_count) {
-                op_errno = ENOTCONN;
-                goto out;
-        }
+afr_lookup(call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xattr_req)
+{
+    afr_local_t *local = NULL;
+    int32_t op_errno = 0;
+    int event = 0;
+    int ret = 0;
+
+    if (loc_is_nameless(loc)) {
+        if (xattr_req)
+            dict_del_sizen(xattr_req, "gfid-req");
+        afr_discover(frame, this, loc, xattr_req);
+        return 0;
+    }
 
-        local->op = GF_FOP_LOOKUP;
+    if (afr_is_private_directory(this->private, loc->parent->gfid, loc->name,
+                                 frame->root->pid)) {
+        op_errno = EPERM;
+        goto out;
+    }
 
-        loc_copy (&local->loc, loc);
+    local = AFR_FRAME_INIT(frame, op_errno);
+    if (!local)
+        goto out;
 
-	local->inode = inode_ref (loc->inode);
+    if (!local->call_count) {
+        op_errno = ENOTCONN;
+        goto out;
+    }
 
-	if (xattr_req) {
-		/* If xattr_req was null, afr_lookup_xattr_req_prepare() will
-		   allocate one for us */
-                ret = dict_get_ptr (xattr_req, "gfid-req", &gfid_req);
-                if (ret == 0) {
-                        gf_uuid_copy (local->cont.lookup.gfid_req, gfid_req);
-                        dict_del (xattr_req, "gfid-req");
-                }
-		local->xattr_req = dict_ref (xattr_req);
+    local->op = GF_FOP_LOOKUP;
+
+    loc_copy(&local->loc, loc);
+
+    local->inode = inode_ref(loc->inode);
+
+    if (xattr_req) {
+        /* If xattr_req was null, afr_lookup_xattr_req_prepare() will
+           allocate one for us */
+        local->xattr_req = dict_copy_with_ref(xattr_req, NULL);
+        if (!local->xattr_req) {
+            op_errno = ENOMEM;
+            goto out;
+        }
+        ret = dict_get_gfuuid(local->xattr_req, "gfid-req",
+                              &local->cont.lookup.gfid_req);
+        if (ret == 0) {
+            dict_del_sizen(local->xattr_req, "gfid-req");
         }
+    }
 
-	afr_read_subvol_get (loc->parent, this, NULL, &event,
-			     AFR_DATA_TRANSACTION);
+    afr_read_subvol_get(loc->parent, this, NULL, NULL, &event,
+                        AFR_DATA_TRANSACTION, NULL);
 
-	if (event != local->event_generation)
-		afr_inode_refresh (frame, this, loc->parent, afr_lookup_do);
-	else
-		afr_lookup_do (frame, this, 0);
+    afr_lookup_do(frame, this, 0);
 
-	return 0;
+    return 0;
 out:
-	AFR_STACK_UNWIND (lookup, frame, -1, op_errno, NULL, NULL, NULL, NULL);
+    AFR_STACK_UNWIND(lookup, frame, -1, op_errno, NULL, NULL, NULL, NULL);
 
-        return 0;
+    return 0;
 }
 
+void
+_afr_cleanup_fd_ctx(xlator_t *this, afr_fd_ctx_t *fd_ctx)
+{
+    afr_private_t *priv = this->private;
 
-/* {{{ open */
+    if (fd_ctx->lk_heal_info) {
+        LOCK(&priv->lock);
+        {
+            list_del(&fd_ctx->lk_heal_info->pos);
+        }
+        afr_lk_heal_info_cleanup(fd_ctx->lk_heal_info);
+        fd_ctx->lk_heal_info = NULL;
+    }
+    GF_FREE(fd_ctx->opened_on);
+    GF_FREE(fd_ctx);
+    return;
+}
 
-afr_fd_ctx_t *
-__afr_fd_ctx_get (fd_t *fd, xlator_t *this)
+int
+afr_cleanup_fd_ctx(xlator_t *this, fd_t *fd)
 {
-        uint64_t       ctx = 0;
-        int            ret = 0;
-        afr_fd_ctx_t  *fd_ctx = NULL;
+    uint64_t ctx = 0;
+    afr_fd_ctx_t *fd_ctx = NULL;
+    int ret = 0;
 
-        ret = __fd_ctx_get (fd, this, &ctx);
+    ret = fd_ctx_get(fd, this, &ctx);
+    if (ret < 0)
+        goto out;
 
-        if (ret < 0) {
-                ret = __afr_fd_ctx_set (this, fd);
-                if (ret < 0)
-                        goto out;
+    fd_ctx = (afr_fd_ctx_t *)(long)ctx;
 
-                ret = __fd_ctx_get (fd, this, &ctx);
-                if (ret < 0)
-                        goto out;
-        }
+    if (fd_ctx) {
+        _afr_cleanup_fd_ctx(this, fd_ctx);
+    }
 
-        fd_ctx = (afr_fd_ctx_t *)(long) ctx;
 out:
-        return fd_ctx;
+    return 0;
 }
 
-
-afr_fd_ctx_t *
-afr_fd_ctx_get (fd_t *fd, xlator_t *this)
+int
+afr_release(xlator_t *this, fd_t *fd)
 {
-        afr_fd_ctx_t  *fd_ctx = NULL;
-
-        LOCK(&fd->lock);
-        {
-                fd_ctx = __afr_fd_ctx_get (fd, this);
-        }
-        UNLOCK(&fd->lock);
+    afr_cleanup_fd_ctx(this, fd);
 
-        return fd_ctx;
+    return 0;
 }
 
-
-int
-__afr_fd_ctx_set (xlator_t *this, fd_t *fd)
+afr_fd_ctx_t *
+__afr_fd_ctx_get(fd_t *fd, xlator_t *this)
 {
-        afr_private_t * priv   = NULL;
-        int             ret    = -1;
-        uint64_t        ctx    = 0;
-        afr_fd_ctx_t *  fd_ctx = NULL;
-	int             i = 0;
+    uint64_t ctx = 0;
+    int ret = 0;
+    afr_fd_ctx_t *fd_ctx = NULL;
 
-        VALIDATE_OR_GOTO (this->private, out);
-        VALIDATE_OR_GOTO (fd, out);
+    ret = __fd_ctx_get(fd, this, &ctx);
 
-        priv = this->private;
+    if (ret < 0) {
+        ret = __afr_fd_ctx_set(this, fd);
+        if (ret < 0)
+            goto out;
 
-        ret = __fd_ctx_get (fd, this, &ctx);
+        ret = __fd_ctx_get(fd, this, &ctx);
+        if (ret < 0)
+            goto out;
+    }
 
-        if (ret == 0)
-                goto out;
+    fd_ctx = (afr_fd_ctx_t *)(long)ctx;
+out:
+    return fd_ctx;
+}
 
-        fd_ctx = GF_CALLOC (1, sizeof (afr_fd_ctx_t),
-                            gf_afr_mt_afr_fd_ctx_t);
-        if (!fd_ctx) {
-                ret = -ENOMEM;
-                goto out;
-        }
+afr_fd_ctx_t *
+afr_fd_ctx_get(fd_t *fd, xlator_t *this)
+{
+    afr_fd_ctx_t *fd_ctx = NULL;
 
-	for (i = 0; i < AFR_NUM_CHANGE_LOGS; i++) {
-		fd_ctx->pre_op_done[i] = GF_CALLOC (sizeof (*fd_ctx->pre_op_done[i]),
-						    priv->child_count,
-						    gf_afr_mt_int32_t);
-		if (!fd_ctx->pre_op_done[i]) {
-			ret = -ENOMEM;
-			goto out;
-		}
-	}
-
-        fd_ctx->opened_on = GF_CALLOC (sizeof (*fd_ctx->opened_on),
-                                       priv->child_count,
-                                       gf_afr_mt_int32_t);
-        if (!fd_ctx->opened_on) {
-                ret = -ENOMEM;
-                goto out;
-        }
+    LOCK(&fd->lock);
+    {
+        fd_ctx = __afr_fd_ctx_get(fd, this);
+    }
+    UNLOCK(&fd->lock);
 
-	for (i = 0; i < priv->child_count; i++) {
-		if (fd_is_anonymous (fd))
-			fd_ctx->opened_on[i] = AFR_FD_OPENED;
-		else
-			fd_ctx->opened_on[i] = AFR_FD_NOT_OPENED;
-	}
+    return fd_ctx;
+}
 
-        fd_ctx->lock_piggyback = GF_CALLOC (sizeof (*fd_ctx->lock_piggyback),
-                                            priv->child_count,
-                                            gf_afr_mt_char);
-        if (!fd_ctx->lock_piggyback) {
-                ret = -ENOMEM;
-                goto out;
-        }
+int
+__afr_fd_ctx_set(xlator_t *this, fd_t *fd)
+{
+    afr_private_t *priv = NULL;
+    int ret = -1;
+    uint64_t ctx = 0;
+    afr_fd_ctx_t *fd_ctx = NULL;
+    int i = 0;
 
-        fd_ctx->lock_acquired = GF_CALLOC (sizeof (*fd_ctx->lock_acquired),
-                                           priv->child_count,
-                                           gf_afr_mt_char);
-        if (!fd_ctx->lock_acquired) {
-                ret = -ENOMEM;
-                goto out;
-        }
+    VALIDATE_OR_GOTO(this->private, out);
+    VALIDATE_OR_GOTO(fd, out);
 
-	fd_ctx->readdir_subvol = -1;
+    priv = this->private;
 
-	pthread_mutex_init (&fd_ctx->delay_lock, NULL);
+    ret = __fd_ctx_get(fd, this, &ctx);
 
-        INIT_LIST_HEAD (&fd_ctx->eager_locked);
+    if (ret == 0)
+        goto out;
 
-        ret = __fd_ctx_set (fd, this, (uint64_t)(long) fd_ctx);
-        if (ret)
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "failed to set fd ctx (%p)", fd);
-out:
-        return ret;
-}
+    fd_ctx = GF_CALLOC(1, sizeof(afr_fd_ctx_t), gf_afr_mt_afr_fd_ctx_t);
+    if (!fd_ctx) {
+        ret = -ENOMEM;
+        goto out;
+    }
 
+    fd_ctx->opened_on = GF_CALLOC(sizeof(*fd_ctx->opened_on), priv->child_count,
+                                  gf_afr_mt_int32_t);
+    if (!fd_ctx->opened_on) {
+        ret = -ENOMEM;
+        goto out;
+    }
 
-int
-afr_fd_ctx_set (xlator_t *this, fd_t *fd)
-{
-        int ret = -1;
+    for (i = 0; i < priv->child_count; i++) {
+        if (fd_is_anonymous(fd))
+            fd_ctx->opened_on[i] = AFR_FD_OPENED;
+        else
+            fd_ctx->opened_on[i] = AFR_FD_NOT_OPENED;
+    }
 
-        LOCK (&fd->lock);
-        {
-                ret = __afr_fd_ctx_set (this, fd);
-        }
-        UNLOCK (&fd->lock);
+    fd_ctx->readdir_subvol = -1;
+    fd_ctx->lk_heal_info = NULL;
 
-        return ret;
+    ret = __fd_ctx_set(fd, this, (uint64_t)(long)fd_ctx);
+    if (ret)
+        gf_msg_debug(this->name, 0, "failed to set fd ctx (%p)", fd);
+out:
+    if (ret && fd_ctx)
+        _afr_cleanup_fd_ctx(this, fd_ctx);
+    return ret;
 }
 
 /* {{{ flush */
 
 int
-afr_flush_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-               int32_t op_ret, int32_t op_errno, dict_t *xdata)
+afr_flush_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int32_t op_ret,
+              int32_t op_errno, dict_t *xdata)
 {
-        afr_local_t *local = NULL;
-        int call_count = -1;
+    afr_local_t *local = NULL;
+    int call_count = -1;
 
-        local = frame->local;
+    local = frame->local;
 
-        LOCK (&frame->lock);
-        {
-                if (op_ret != -1) {
-			local->op_ret = op_ret;
-			if (!local->xdata_rsp && xdata)
-				local->xdata_rsp = dict_ref (xdata);
-		} else {
-			local->op_errno = op_errno;
-		}
+    LOCK(&frame->lock);
+    {
+        if (op_ret != -1) {
+            local->op_ret = op_ret;
+            if (!local->xdata_rsp && xdata)
+                local->xdata_rsp = dict_ref(xdata);
+        } else {
+            local->op_errno = op_errno;
         }
-        UNLOCK (&frame->lock);
+        call_count = --local->call_count;
+    }
+    UNLOCK(&frame->lock);
 
-	call_count = afr_frame_return (frame);
+    if (call_count == 0)
+        AFR_STACK_UNWIND(flush, frame, local->op_ret, local->op_errno,
+                         local->xdata_rsp);
 
-	if (call_count == 0)
-		AFR_STACK_UNWIND (flush, frame, local->op_ret,
-				  local->op_errno, local->xdata_rsp);
-
-        return 0;
+    return 0;
 }
 
 static int
-afr_flush_wrapper (call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *xdata)
+afr_flush_wrapper(call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *xdata)
 {
-        int           i      = 0;
-        afr_local_t   *local = NULL;
-        afr_private_t *priv  = NULL;
-        int call_count       = -1;
-
-        priv = this->private;
-        local = frame->local;
-        call_count = local->call_count;
+    int i = 0;
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
+    int call_count = -1;
+
+    priv = this->private;
+    local = frame->local;
+    call_count = local->call_count;
+
+    for (i = 0; i < priv->child_count; i++) {
+        if (local->child_up[i]) {
+            STACK_WIND_COOKIE(frame, afr_flush_cbk, (void *)(long)i,
+                              priv->children[i], priv->children[i]->fops->flush,
+                              local->fd, xdata);
+            if (!--call_count)
+                break;
+        }
+    }
 
-        for (i = 0; i < priv->child_count; i++) {
-                if (local->child_up[i]) {
-                        STACK_WIND_COOKIE (frame, afr_flush_cbk,
-                                           (void *) (long) i,
-                                           priv->children[i],
-                                           priv->children[i]->fops->flush,
-                                           local->fd, xdata);
-                        if (!--call_count)
-                                break;
+    return 0;
+}
 
-                }
+afr_local_t *
+afr_wakeup_same_fd_delayed_op(xlator_t *this, afr_lock_t *lock, fd_t *fd)
+{
+    afr_local_t *local = NULL;
+
+    if (lock->delay_timer) {
+        local = list_entry(lock->post_op.next, afr_local_t,
+                           transaction.owner_list);
+        if (fd == local->fd) {
+            if (gf_timer_call_cancel(this->ctx, lock->delay_timer)) {
+                local = NULL;
+            } else {
+                lock->delay_timer = NULL;
+            }
+        } else {
+            local = NULL;
         }
+    }
 
-        return 0;
+    return local;
+}
+
+void
+afr_delayed_changelog_wake_resume(xlator_t *this, inode_t *inode,
+                                  call_stub_t *stub)
+{
+    afr_inode_ctx_t *ctx = NULL;
+    afr_lock_t *lock = NULL;
+    afr_local_t *metadata_local = NULL;
+    afr_local_t *data_local = NULL;
+    LOCK(&inode->lock);
+    {
+        (void)__afr_inode_ctx_get(this, inode, &ctx);
+        lock = &ctx->lock[AFR_DATA_TRANSACTION];
+        data_local = afr_wakeup_same_fd_delayed_op(this, lock, stub->args.fd);
+        lock = &ctx->lock[AFR_METADATA_TRANSACTION];
+        metadata_local = afr_wakeup_same_fd_delayed_op(this, lock,
+                                                       stub->args.fd);
+    }
+    UNLOCK(&inode->lock);
+
+    if (data_local) {
+        data_local->transaction.resume_stub = stub;
+    } else if (metadata_local) {
+        metadata_local->transaction.resume_stub = stub;
+    } else {
+        call_resume(stub);
+    }
+    if (data_local) {
+        afr_delayed_changelog_wake_up_cbk(data_local);
+    }
+    if (metadata_local) {
+        afr_delayed_changelog_wake_up_cbk(metadata_local);
+    }
 }
 
 int
-afr_flush (call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *xdata)
+afr_flush(call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *xdata)
 {
-        afr_local_t   *local = NULL;
-        call_stub_t   *stub = NULL;
-        int            op_errno   = ENOMEM;
+    afr_local_t *local = NULL;
+    call_stub_t *stub = NULL;
+    int op_errno = ENOMEM;
 
-	local = AFR_FRAME_INIT (frame, op_errno);
-	if (!local)
-		goto out;
+    AFR_ERROR_OUT_IF_FDCTX_INVALID(fd, this, op_errno, out);
+    local = AFR_FRAME_INIT(frame, op_errno);
+    if (!local)
+        goto out;
 
-	if (!local->call_count) {
-		op_errno = ENOTCONN;
-		goto out;
-	}
+    local->op = GF_FOP_FLUSH;
+    if (!afr_is_consistent_io_possible(local, this->private, &op_errno))
+        goto out;
 
-	local->fd = fd_ref(fd);
+    local->fd = fd_ref(fd);
 
-        stub = fop_flush_stub (frame, afr_flush_wrapper, fd, xdata);
-        if (!stub)
-                goto out;
+    stub = fop_flush_stub(frame, afr_flush_wrapper, fd, xdata);
+    if (!stub)
+        goto out;
 
-        afr_delayed_changelog_wake_resume (this, fd, stub);
+    afr_delayed_changelog_wake_resume(this, fd->inode, stub);
 
-	return 0;
+    return 0;
 out:
-	AFR_STACK_UNWIND (flush, frame, -1, op_errno, NULL);
-        return 0;
+    AFR_STACK_UNWIND(flush, frame, -1, op_errno, NULL);
+    return 0;
 }
 
-/* }}} */
-
-
 int
-afr_cleanup_fd_ctx (xlator_t *this, fd_t *fd)
+afr_fsyncdir_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                 int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
-        uint64_t        ctx = 0;
-        afr_fd_ctx_t    *fd_ctx = NULL;
-        int             ret = 0;
-	int             i = 0;
-
-        ret = fd_ctx_get (fd, this, &ctx);
-        if (ret < 0)
-                goto out;
-
-        fd_ctx = (afr_fd_ctx_t *)(long) ctx;
-
-        if (fd_ctx) {
-                //no need to take any locks
-                if (!list_empty (&fd_ctx->eager_locked))
-                        gf_log (this->name, GF_LOG_WARNING, "%s: Stale "
-                                "Eager-lock stubs found",
-                                uuid_utoa (fd->inode->gfid));
-
-		for (i = 0; i < AFR_NUM_CHANGE_LOGS; i++)
-			GF_FREE (fd_ctx->pre_op_done[i]);
+    afr_local_t *local = NULL;
+    int call_count = -1;
 
-                GF_FREE (fd_ctx->opened_on);
+    local = frame->local;
 
-                GF_FREE (fd_ctx->lock_piggyback);
-
-                GF_FREE (fd_ctx->lock_acquired);
-
-		pthread_mutex_destroy (&fd_ctx->delay_lock);
-
-                GF_FREE (fd_ctx);
+    LOCK(&frame->lock);
+    {
+        if (op_ret == 0) {
+            local->op_ret = 0;
+            if (!local->xdata_rsp && xdata)
+                local->xdata_rsp = dict_ref(xdata);
+        } else {
+            local->op_errno = op_errno;
         }
+        call_count = --local->call_count;
+    }
+    UNLOCK(&frame->lock);
 
-out:
-        return 0;
-}
+    if (call_count == 0)
+        AFR_STACK_UNWIND(fsyncdir, frame, local->op_ret, local->op_errno,
+                         local->xdata_rsp);
 
+    return 0;
+}
 
 int
-afr_release (xlator_t *this, fd_t *fd)
+afr_fsyncdir(call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t datasync,
+             dict_t *xdata)
 {
-        afr_cleanup_fd_ctx (this, fd);
+    afr_private_t *priv = NULL;
+    afr_local_t *local = NULL;
+    int i = 0;
+    int32_t call_count = 0;
+    int32_t op_errno = ENOMEM;
+
+    priv = this->private;
+
+    local = AFR_FRAME_INIT(frame, op_errno);
+    if (!local)
+        goto out;
+
+    local->op = GF_FOP_FSYNCDIR;
+    if (!afr_is_consistent_io_possible(local, priv, &op_errno))
+        goto out;
+
+    call_count = local->call_count;
+    for (i = 0; i < priv->child_count; i++) {
+        if (local->child_up[i]) {
+            STACK_WIND(frame, afr_fsyncdir_cbk, priv->children[i],
+                       priv->children[i]->fops->fsyncdir, fd, datasync, xdata);
+            if (!--call_count)
+                break;
+        }
+    }
 
-        return 0;
+    return 0;
+out:
+    AFR_STACK_UNWIND(fsyncdir, frame, -1, op_errno, NULL);
+
+    return 0;
 }
 
+/* }}} */
 
-/* {{{ fsync */
+static int
+afr_serialized_lock_wind(call_frame_t *frame, xlator_t *this);
 
-int
-afr_fsync_unwind_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                      int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
-                      struct iatt *postbuf, dict_t *xdata)
+static gf_boolean_t
+afr_is_conflicting_lock_present(int32_t op_ret, int32_t op_errno)
 {
-        AFR_STACK_UNWIND (fsync, frame, op_ret, op_errno, prebuf, postbuf,
-                          xdata);
-        return 0;
+    if (op_ret == -1 && op_errno == EAGAIN)
+        return _gf_true;
+    return _gf_false;
 }
 
-int
-afr_fsync_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-               int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
-               struct iatt *postbuf, dict_t *xdata)
+static void
+afr_fop_lock_unwind(call_frame_t *frame, glusterfs_fop_t op, int32_t op_ret,
+                    int32_t op_errno, dict_t *xdata)
 {
-        afr_local_t *local = NULL;
-        int call_count = -1;
-        int child_index = (long) cookie;
-	int read_subvol = 0;
-	call_stub_t *stub = NULL;
+    switch (op) {
+        case GF_FOP_INODELK:
+            AFR_STACK_UNWIND(inodelk, frame, op_ret, op_errno, xdata);
+            break;
+        case GF_FOP_FINODELK:
+            AFR_STACK_UNWIND(finodelk, frame, op_ret, op_errno, xdata);
+            break;
+        case GF_FOP_ENTRYLK:
+            AFR_STACK_UNWIND(entrylk, frame, op_ret, op_errno, xdata);
+            break;
+        case GF_FOP_FENTRYLK:
+            AFR_STACK_UNWIND(fentrylk, frame, op_ret, op_errno, xdata);
+            break;
+        default:
+            break;
+    }
+}
 
-        local = frame->local;
+static void
+afr_fop_lock_wind(call_frame_t *frame, xlator_t *this, int child_index,
+                  int32_t (*lock_cbk)(call_frame_t *, void *, xlator_t *,
+                                      int32_t, int32_t, dict_t *))
+{
+    afr_local_t *local = frame->local;
+    afr_private_t *priv = this->private;
+    int i = child_index;
+
+    switch (local->op) {
+        case GF_FOP_INODELK:
+            STACK_WIND_COOKIE(
+                frame, lock_cbk, (void *)(long)i, priv->children[i],
+                priv->children[i]->fops->inodelk,
+                (const char *)local->cont.inodelk.volume, &local->loc,
+                local->cont.inodelk.cmd, &local->cont.inodelk.flock,
+                local->cont.inodelk.xdata);
+            break;
+        case GF_FOP_FINODELK:
+            STACK_WIND_COOKIE(
+                frame, lock_cbk, (void *)(long)i, priv->children[i],
+                priv->children[i]->fops->finodelk,
+                (const char *)local->cont.inodelk.volume, local->fd,
+                local->cont.inodelk.cmd, &local->cont.inodelk.flock,
+                local->cont.inodelk.xdata);
+            break;
+        case GF_FOP_ENTRYLK:
+            STACK_WIND_COOKIE(
+                frame, lock_cbk, (void *)(long)i, priv->children[i],
+                priv->children[i]->fops->entrylk, local->cont.entrylk.volume,
+                &local->loc, local->cont.entrylk.basename,
+                local->cont.entrylk.cmd, local->cont.entrylk.type,
+                local->cont.entrylk.xdata);
+            break;
+        case GF_FOP_FENTRYLK:
+            STACK_WIND_COOKIE(
+                frame, lock_cbk, (void *)(long)i, priv->children[i],
+                priv->children[i]->fops->fentrylk, local->cont.entrylk.volume,
+                local->fd, local->cont.entrylk.basename,
+                local->cont.entrylk.cmd, local->cont.entrylk.type,
+                local->cont.entrylk.xdata);
+            break;
+        default:
+            break;
+    }
+}
 
-	read_subvol = afr_data_subvol_get (local->inode, this, 0, 0);
+void
+afr_fop_lock_proceed(call_frame_t *frame)
+{
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
 
-        LOCK (&frame->lock);
-        {
-                if (op_ret == 0) {
-                        if (local->op_ret == -1) {
-				local->op_ret = 0;
-
-                                local->cont.inode_wfop.prebuf  = *prebuf;
-                                local->cont.inode_wfop.postbuf = *postbuf;
-
-				if (xdata)
-					local->xdata_rsp = dict_ref (xdata);
-                        }
-
-                        if (child_index == read_subvol) {
-                                local->cont.inode_wfop.prebuf  = *prebuf;
-                                local->cont.inode_wfop.postbuf = *postbuf;
-				if (xdata) {
-					if (local->xdata_rsp)
-						dict_unref (local->xdata_rsp);
-					local->xdata_rsp = dict_ref (xdata);
-				}
-                        }
-                } else {
-			local->op_errno = op_errno;
-		}
-        }
-        UNLOCK (&frame->lock);
-
-        call_count = afr_frame_return (frame);
-
-        if (call_count == 0) {
-		/* Make a stub out of the frame, and register it
-		   with the waking up post-op. When the call-stub resumes,
-		   we are guaranteed that there was no post-op pending
-		   (i.e changelogs were unset in the server). This is an
-		   essential "guarantee", that fsync() returns only after
-		   completely finishing EVERYTHING, including the delayed
-		   post-op. This guarantee is expected by FUSE graph switching
-		   for example.
-		*/
-		stub = fop_fsync_cbk_stub (frame, afr_fsync_unwind_cbk,
-                                           local->op_ret, local->op_errno,
-                                           &local->cont.inode_wfop.prebuf,
-                                           &local->cont.inode_wfop.postbuf,
-                                           local->xdata_rsp);
-		if (!stub) {
-			AFR_STACK_UNWIND (fsync, frame, -1, ENOMEM, 0, 0, 0);
-			return 0;
-		}
-
-		/* If no new unstable writes happened between the
-		   time we cleared the unstable write witness flag in afr_fsync
-		   and now, calling afr_delayed_changelog_wake_up() should
-		   wake up and skip over the fsync phase and go straight to
-		   afr_changelog_post_op_now()
-		*/
-		afr_delayed_changelog_wake_resume (this, local->fd, stub);
-        }
+    local = frame->local;
+    priv = frame->this->private;
 
-        return 0;
+    if (local->fop_lock_state != AFR_FOP_LOCK_PARALLEL) {
+        afr_fop_lock_unwind(frame, local->op, local->op_ret, local->op_errno,
+                            local->xdata_rsp);
+        return;
+    }
+    /* At least one child is up */
+    /*
+     * Non-blocking locks also need to be serialized.  Otherwise there is
+     * a chance that both the mounts which issued same non-blocking inodelk
+     * may endup not acquiring the lock on any-brick.
+     * Ex: Mount1 and Mount2
+     * request for full length lock on file f1.  Mount1 afr may acquire the
+     * partial lock on brick-1 and may not acquire the lock on brick-2
+     * because Mount2 already got the lock on brick-2, vice versa.  Since
+     * both the mounts only got partial locks, afr treats them as failure in
+     * gaining the locks and unwinds with EAGAIN errno.
+     */
+    local->op_ret = -1;
+    local->op_errno = EUCLEAN;
+    local->fop_lock_state = AFR_FOP_LOCK_SERIAL;
+    afr_local_replies_wipe(local, priv);
+    if (local->xdata_rsp)
+        dict_unref(local->xdata_rsp);
+    local->xdata_rsp = NULL;
+    switch (local->op) {
+        case GF_FOP_INODELK:
+        case GF_FOP_FINODELK:
+            local->cont.inodelk.cmd = local->cont.inodelk.in_cmd;
+            local->cont.inodelk.flock = local->cont.inodelk.in_flock;
+            if (local->cont.inodelk.xdata)
+                dict_unref(local->cont.inodelk.xdata);
+            local->cont.inodelk.xdata = NULL;
+            if (local->xdata_req)
+                local->cont.inodelk.xdata = dict_ref(local->xdata_req);
+            break;
+        case GF_FOP_ENTRYLK:
+        case GF_FOP_FENTRYLK:
+            local->cont.entrylk.cmd = local->cont.entrylk.in_cmd;
+            if (local->cont.entrylk.xdata)
+                dict_unref(local->cont.entrylk.xdata);
+            local->cont.entrylk.xdata = NULL;
+            if (local->xdata_req)
+                local->cont.entrylk.xdata = dict_ref(local->xdata_req);
+            break;
+        default:
+            break;
+    }
+    afr_serialized_lock_wind(frame, frame->this);
 }
 
+static int32_t
+afr_unlock_partial_lock_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                            int32_t op_ret, int32_t op_errno, dict_t *xdata)
 
-int
-afr_fsync (call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t datasync,
-	   dict_t *xdata)
 {
-	afr_private_t *priv = NULL;
-        afr_local_t *local = NULL;
-        int i = 0;
-        int32_t call_count = 0;
-        int32_t op_errno = ENOMEM;
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
+    int call_count = -1;
+    int child_index = (long)cookie;
+    uuid_t gfid = {0};
 
-	priv = this->private;
+    local = frame->local;
+    priv = this->private;
 
-	local = AFR_FRAME_INIT (frame, op_errno);
-	if (!local)
-		goto out;
+    if (op_ret < 0 && op_errno != ENOTCONN) {
+        if (local->fd)
+            gf_uuid_copy(gfid, local->fd->inode->gfid);
+        else
+            loc_gfid(&local->loc, gfid);
+        gf_msg(this->name, GF_LOG_ERROR, op_errno, AFR_MSG_UNLOCK_FAIL,
+               "%s: Failed to unlock %s on %s "
+               "with lk_owner: %s",
+               uuid_utoa(gfid), gf_fop_list[local->op],
+               priv->children[child_index]->name,
+               lkowner_utoa(&frame->root->lk_owner));
+    }
+
+    call_count = afr_frame_return(frame);
+    if (call_count == 0)
+        afr_fop_lock_proceed(frame);
+
+    return 0;
+}
 
-        call_count = local->call_count;
-	if (!call_count) {
-		op_errno = ENOTCONN;
-		goto out;
-	}
+static int32_t
+afr_unlock_locks_and_proceed(call_frame_t *frame, xlator_t *this,
+                             int call_count)
+{
+    int i = 0;
+    afr_private_t *priv = NULL;
+    afr_local_t *local = NULL;
+
+    if (call_count == 0) {
+        afr_fop_lock_proceed(frame);
+        goto out;
+    }
+
+    local = frame->local;
+    priv = this->private;
+    local->call_count = call_count;
+    switch (local->op) {
+        case GF_FOP_INODELK:
+        case GF_FOP_FINODELK:
+            local->cont.inodelk.flock.l_type = F_UNLCK;
+            local->cont.inodelk.cmd = F_SETLK;
+            if (local->cont.inodelk.xdata)
+                dict_unref(local->cont.inodelk.xdata);
+            local->cont.inodelk.xdata = NULL;
+            break;
+        case GF_FOP_ENTRYLK:
+        case GF_FOP_FENTRYLK:
+            local->cont.entrylk.cmd = ENTRYLK_UNLOCK;
+            if (local->cont.entrylk.xdata)
+                dict_unref(local->cont.entrylk.xdata);
+            local->cont.entrylk.xdata = NULL;
+            break;
+        default:
+            break;
+    }
 
-        local->fd = fd_ref (fd);
+    for (i = 0; i < priv->child_count; i++) {
+        if (!local->replies[i].valid)
+            continue;
 
-	if (afr_fd_has_witnessed_unstable_write (this, fd)) {
-		/* don't care. we only wanted to CLEAR the bit */
-	}
+        if (local->replies[i].op_ret == -1)
+            continue;
 
-	local->inode = inode_ref (fd->inode);
+        afr_fop_lock_wind(frame, this, i, afr_unlock_partial_lock_cbk);
 
-        for (i = 0; i < priv->child_count; i++) {
-                if (local->child_up[i]) {
-                        STACK_WIND_COOKIE (frame, afr_fsync_cbk,
-                                           (void *) (long) i,
-                                           priv->children[i],
-                                           priv->children[i]->fops->fsync,
-                                           fd, datasync, xdata);
-                        if (!--call_count)
-                                break;
-                }
-        }
+        if (!--call_count)
+            break;
+    }
 
-	return 0;
 out:
-	AFR_STACK_UNWIND (fsync, frame, -1, op_errno, NULL, NULL, NULL);
-
-        return 0;
+    return 0;
 }
 
-/* }}} */
+int32_t
+afr_fop_lock_done(call_frame_t *frame, xlator_t *this)
+{
+    int i = 0;
+    int lock_count = 0;
+    unsigned char *success = NULL;
 
-/* {{{ fsync */
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
 
-int
-afr_fsyncdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-		  int32_t op_ret, int32_t op_errno, dict_t *xdata)
-{
-        afr_local_t *local = NULL;
-        int call_count = -1;
+    local = frame->local;
+    priv = this->private;
+    success = alloca0(priv->child_count);
 
-        local = frame->local;
+    for (i = 0; i < priv->child_count; i++) {
+        if (!local->replies[i].valid)
+            continue;
 
-        LOCK (&frame->lock);
-        {
-                if (op_ret == 0) {
-                        local->op_ret = 0;
-			if (!local->xdata_rsp && xdata)
-				local->xdata_rsp = dict_ref (xdata);
-		} else {
-			local->op_errno = op_errno;
-		}
+        if (local->replies[i].op_ret == 0) {
+            lock_count++;
+            success[i] = 1;
         }
-        UNLOCK (&frame->lock);
 
-        call_count = afr_frame_return (frame);
+        if (local->op_ret == -1 && local->op_errno == EAGAIN)
+            continue;
 
-        if (call_count == 0)
-                AFR_STACK_UNWIND (fsyncdir, frame, local->op_ret,
-				  local->op_errno, local->xdata_rsp);
+        if ((local->replies[i].op_ret == -1) &&
+            (local->replies[i].op_errno == EAGAIN)) {
+            local->op_ret = -1;
+            local->op_errno = EAGAIN;
+            continue;
+        }
 
-        return 0;
-}
+        if (local->replies[i].op_ret == 0)
+            local->op_ret = 0;
 
+        local->op_errno = local->replies[i].op_errno;
+    }
 
-int
-afr_fsyncdir (call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t datasync,
-	      dict_t *xdata)
-{
-	afr_private_t *priv = NULL;
-        afr_local_t *local = NULL;
-        int i = 0;
-        int32_t call_count = 0;
-        int32_t op_errno = ENOMEM;
+    if (afr_fop_lock_is_unlock(frame))
+        goto unwind;
 
-	priv = this->private;
+    if (afr_is_conflicting_lock_present(local->op_ret, local->op_errno)) {
+        afr_unlock_locks_and_proceed(frame, this, lock_count);
+    } else if (priv->quorum_count && !afr_has_quorum(success, this, NULL)) {
+        local->fop_lock_state = AFR_FOP_LOCK_QUORUM_FAILED;
+        local->op_ret = -1;
+        local->op_errno = afr_final_errno(local, priv);
+        if (local->op_errno == 0)
+            local->op_errno = afr_quorum_errno(priv);
+        afr_unlock_locks_and_proceed(frame, this, lock_count);
+    } else {
+        goto unwind;
+    }
+
+    return 0;
+unwind:
+    afr_fop_lock_unwind(frame, local->op, local->op_ret, local->op_errno,
+                        local->xdata_rsp);
+    return 0;
+}
 
-        local = AFR_FRAME_INIT (frame, op_errno);
-	if (!local)
-		goto out;
+static int
+afr_common_lock_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                    int32_t op_ret, int32_t op_errno, dict_t *xdata)
+{
+    afr_local_t *local = NULL;
+    int child_index = (long)cookie;
 
-        call_count = local->call_count;
-	if (!call_count) {
-		op_errno = ENOTCONN;
-		goto out;
-	}
+    local = frame->local;
 
-        for (i = 0; i < priv->child_count; i++) {
-                if (local->child_up[i]) {
-                        STACK_WIND (frame, afr_fsyncdir_cbk,
-                                    priv->children[i],
-                                    priv->children[i]->fops->fsyncdir,
-                                    fd, datasync, xdata);
-                        if (!--call_count)
-                                break;
-                }
+    local->replies[child_index].valid = 1;
+    local->replies[child_index].op_ret = op_ret;
+    local->replies[child_index].op_errno = op_errno;
+    if (op_ret == 0 && xdata) {
+        local->replies[child_index].xdata = dict_ref(xdata);
+        LOCK(&frame->lock);
+        {
+            if (!local->xdata_rsp)
+                local->xdata_rsp = dict_ref(xdata);
         }
-
-	return 0;
-out:
-	AFR_STACK_UNWIND (fsyncdir, frame, -1, op_errno, NULL);
-
-        return 0;
+        UNLOCK(&frame->lock);
+    }
+    return 0;
 }
 
-/* }}} */
-
-/* {{{ xattrop */
+static int32_t
+afr_serialized_lock_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                        int32_t op_ret, int32_t op_errno, dict_t *xdata)
 
-int32_t
-afr_xattrop_cbk (call_frame_t *frame, void *cookie,
-                 xlator_t *this, int32_t op_ret, int32_t op_errno,
-                 dict_t *xattr, dict_t *xdata)
 {
-        afr_local_t *local = NULL;
-        int call_count = -1;
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
+    int child_index = (long)cookie;
+    int next_child = 0;
+
+    local = frame->local;
+    priv = this->private;
+
+    afr_common_lock_cbk(frame, cookie, this, op_ret, op_errno, xdata);
+
+    for (next_child = child_index + 1; next_child < priv->child_count;
+         next_child++) {
+        if (local->child_up[next_child])
+            break;
+    }
+
+    if (afr_is_conflicting_lock_present(op_ret, op_errno) ||
+        (next_child == priv->child_count)) {
+        afr_fop_lock_done(frame, this);
+    } else {
+        afr_fop_lock_wind(frame, this, next_child, afr_serialized_lock_cbk);
+    }
+
+    return 0;
+}
 
-        local = frame->local;
+static int
+afr_serialized_lock_wind(call_frame_t *frame, xlator_t *this)
+{
+    afr_private_t *priv = NULL;
+    afr_local_t *local = NULL;
+    int i = 0;
 
-        LOCK (&frame->lock);
-        {
-                if (op_ret == 0) {
-                        if (!local->cont.xattrop.xattr)
-                                local->cont.xattrop.xattr = dict_ref (xattr);
+    priv = this->private;
+    local = frame->local;
 
-			if (!local->xdata_rsp && xdata)
-				local->xdata_rsp = dict_ref (xdata);
+    for (i = 0; i < priv->child_count; i++) {
+        if (local->child_up[i]) {
+            afr_fop_lock_wind(frame, this, i, afr_serialized_lock_cbk);
+            break;
+        }
+    }
+    return 0;
+}
 
-                        local->op_ret = 0;
-                }
+static int32_t
+afr_parallel_lock_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                      int32_t op_ret, int32_t op_errno, dict_t *xdata)
 
-                local->op_errno = op_errno;
-        }
-        UNLOCK (&frame->lock);
+{
+    int call_count = 0;
 
-        call_count = afr_frame_return (frame);
+    afr_common_lock_cbk(frame, cookie, this, op_ret, op_errno, xdata);
 
-        if (call_count == 0)
-                AFR_STACK_UNWIND (xattrop, frame, local->op_ret, local->op_errno,
-                local->cont.xattrop.xattr, local->xdata_rsp);
+    call_count = afr_frame_return(frame);
+    if (call_count == 0)
+        afr_fop_lock_done(frame, this);
 
-        return 0;
+    return 0;
 }
 
-
-int32_t
-afr_xattrop (call_frame_t *frame, xlator_t *this, loc_t *loc,
-             gf_xattrop_flags_t optype, dict_t *xattr, dict_t *xdata)
+static int
+afr_parallel_lock_wind(call_frame_t *frame, xlator_t *this)
 {
-        afr_private_t *priv = NULL;
-        afr_local_t *local  = NULL;
-        int i = 0;
-        int32_t call_count = 0;
-        int32_t op_errno = ENOMEM;
+    afr_private_t *priv = NULL;
+    afr_local_t *local = NULL;
+    int call_count = 0;
+    int i = 0;
+
+    priv = this->private;
+    local = frame->local;
+    call_count = local->call_count;
+
+    for (i = 0; i < priv->child_count; i++) {
+        if (!local->child_up[i])
+            continue;
+        afr_fop_lock_wind(frame, this, i, afr_parallel_lock_cbk);
+        if (!--call_count)
+            break;
+    }
+    return 0;
+}
 
-        priv = this->private;
+static int
+afr_fop_handle_lock(call_frame_t *frame, xlator_t *this)
+{
+    afr_local_t *local = frame->local;
+    int op_errno = 0;
 
-	local = AFR_FRAME_INIT (frame, op_errno);
-	if (!local)
-		goto out;
+    if (!afr_fop_lock_is_unlock(frame)) {
+        if (!afr_is_consistent_io_possible(local, this->private, &op_errno))
+            goto out;
 
-        call_count = local->call_count;
-	if (!call_count) {
-		op_errno = ENOTCONN;
-		goto out;
-	}
+        switch (local->op) {
+            case GF_FOP_INODELK:
+            case GF_FOP_FINODELK:
+                local->cont.inodelk.cmd = F_SETLK;
+                break;
+            case GF_FOP_ENTRYLK:
+            case GF_FOP_FENTRYLK:
+                local->cont.entrylk.cmd = ENTRYLK_LOCK_NB;
+                break;
+            default:
+                break;
+        }
+    }
 
-        for (i = 0; i < priv->child_count; i++) {
-                if (local->child_up[i]) {
-                        STACK_WIND (frame, afr_xattrop_cbk,
-                                    priv->children[i],
-                                    priv->children[i]->fops->xattrop,
-                                    loc, optype, xattr, xdata);
-                        if (!--call_count)
-                                break;
-                }
+    if (local->xdata_req) {
+        switch (local->op) {
+            case GF_FOP_INODELK:
+            case GF_FOP_FINODELK:
+                local->cont.inodelk.xdata = dict_ref(local->xdata_req);
+                break;
+            case GF_FOP_ENTRYLK:
+            case GF_FOP_FENTRYLK:
+                local->cont.entrylk.xdata = dict_ref(local->xdata_req);
+                break;
+            default:
+                break;
         }
+    }
 
-	return 0;
+    local->fop_lock_state = AFR_FOP_LOCK_PARALLEL;
+    afr_parallel_lock_wind(frame, this);
 out:
-	AFR_STACK_UNWIND (xattrop, frame, -1, op_errno, NULL, NULL);
-
-        return 0;
+    return -op_errno;
 }
 
-/* }}} */
+static int32_t
+afr_handle_inodelk(call_frame_t *frame, xlator_t *this, glusterfs_fop_t fop,
+                   const char *volume, loc_t *loc, fd_t *fd, int32_t cmd,
+                   struct gf_flock *flock, dict_t *xdata)
+{
+    afr_local_t *local = NULL;
+    int32_t op_errno = ENOMEM;
+
+    local = AFR_FRAME_INIT(frame, op_errno);
+    if (!local)
+        goto out;
+
+    local->op = fop;
+    if (loc)
+        loc_copy(&local->loc, loc);
+    if (fd && (flock->l_type != F_UNLCK)) {
+        AFR_ERROR_OUT_IF_FDCTX_INVALID(fd, this, op_errno, out);
+        local->fd = fd_ref(fd);
+    }
+
+    local->cont.inodelk.volume = gf_strdup(volume);
+    if (!local->cont.inodelk.volume) {
+        op_errno = ENOMEM;
+        goto out;
+    }
+
+    local->cont.inodelk.in_cmd = cmd;
+    local->cont.inodelk.cmd = cmd;
+    local->cont.inodelk.in_flock = *flock;
+    local->cont.inodelk.flock = *flock;
+    if (xdata)
+        local->xdata_req = dict_ref(xdata);
+
+    op_errno = -afr_fop_handle_lock(frame, frame->this);
+    if (op_errno)
+        goto out;
+    return 0;
+out:
+    afr_fop_lock_unwind(frame, fop, -1, op_errno, NULL);
 
-/* {{{ fxattrop */
+    return 0;
+}
 
 int32_t
-afr_fxattrop_cbk (call_frame_t *frame, void *cookie,
-                  xlator_t *this, int32_t op_ret, int32_t op_errno,
-                  dict_t *xattr, dict_t *xdata)
+afr_inodelk(call_frame_t *frame, xlator_t *this, const char *volume, loc_t *loc,
+            int32_t cmd, struct gf_flock *flock, dict_t *xdata)
 {
-        afr_local_t *local = NULL;
-
-        int call_count = -1;
-
-        local = frame->local;
-
-        LOCK (&frame->lock);
-        {
-                if (op_ret == 0) {
-                        if (!local->cont.fxattrop.xattr)
-                                local->cont.fxattrop.xattr = dict_ref (xattr);
-
-			if (!local->xdata_rsp && xdata)
-				local->xdata_rsp = dict_ref (xdata);
-                        local->op_ret = 0;
-                }
-
-                local->op_errno = op_errno;
-        }
-        UNLOCK (&frame->lock);
+    afr_handle_inodelk(frame, this, GF_FOP_INODELK, volume, loc, NULL, cmd,
+                       flock, xdata);
+    return 0;
+}
 
-        call_count = afr_frame_return (frame);
+int32_t
+afr_finodelk(call_frame_t *frame, xlator_t *this, const char *volume, fd_t *fd,
+             int32_t cmd, struct gf_flock *flock, dict_t *xdata)
+{
+    afr_handle_inodelk(frame, this, GF_FOP_FINODELK, volume, NULL, fd, cmd,
+                       flock, xdata);
+    return 0;
+}
 
-        if (call_count == 0)
-                AFR_STACK_UNWIND (fxattrop, frame, local->op_ret, local->op_errno,
-                                  local->cont.fxattrop.xattr, local->xdata_rsp);
+static int
+afr_handle_entrylk(call_frame_t *frame, xlator_t *this, glusterfs_fop_t fop,
+                   const char *volume, loc_t *loc, fd_t *fd,
+                   const char *basename, entrylk_cmd cmd, entrylk_type type,
+                   dict_t *xdata)
+{
+    afr_local_t *local = NULL;
+    int32_t op_errno = ENOMEM;
+
+    local = AFR_FRAME_INIT(frame, op_errno);
+    if (!local)
+        goto out;
+
+    local->op = fop;
+    if (loc)
+        loc_copy(&local->loc, loc);
+    if (fd && (cmd != ENTRYLK_UNLOCK)) {
+        AFR_ERROR_OUT_IF_FDCTX_INVALID(fd, this, op_errno, out);
+        local->fd = fd_ref(fd);
+    }
+    local->cont.entrylk.cmd = cmd;
+    local->cont.entrylk.in_cmd = cmd;
+    local->cont.entrylk.type = type;
+    local->cont.entrylk.volume = gf_strdup(volume);
+    local->cont.entrylk.basename = gf_strdup(basename);
+    if (!local->cont.entrylk.volume || !local->cont.entrylk.basename) {
+        op_errno = ENOMEM;
+        goto out;
+    }
+    if (xdata)
+        local->xdata_req = dict_ref(xdata);
+    op_errno = -afr_fop_handle_lock(frame, frame->this);
+    if (op_errno)
+        goto out;
+
+    return 0;
+out:
+    afr_fop_lock_unwind(frame, fop, -1, op_errno, NULL);
+    return 0;
+}
 
-        return 0;
+int
+afr_entrylk(call_frame_t *frame, xlator_t *this, const char *volume, loc_t *loc,
+            const char *basename, entrylk_cmd cmd, entrylk_type type,
+            dict_t *xdata)
+{
+    afr_handle_entrylk(frame, this, GF_FOP_ENTRYLK, volume, loc, NULL, basename,
+                       cmd, type, xdata);
+    return 0;
 }
 
+int
+afr_fentrylk(call_frame_t *frame, xlator_t *this, const char *volume, fd_t *fd,
+             const char *basename, entrylk_cmd cmd, entrylk_type type,
+             dict_t *xdata)
+{
+    afr_handle_entrylk(frame, this, GF_FOP_FENTRYLK, volume, NULL, fd, basename,
+                       cmd, type, xdata);
+    return 0;
+}
 
-int32_t
-afr_fxattrop (call_frame_t *frame, xlator_t *this, fd_t *fd,
-              gf_xattrop_flags_t optype, dict_t *xattr, dict_t *xdata)
+int
+afr_statfs_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int op_ret,
+               int op_errno, struct statvfs *statvfs, dict_t *xdata)
 {
-        afr_private_t *priv = NULL;
-        afr_local_t *local  = NULL;
-        int i = 0;
-        int32_t call_count = 0;
-        int32_t op_errno = 0;
+    afr_local_t *local = NULL;
+    int call_count = 0;
+    struct statvfs *buf = NULL;
 
-        priv = this->private;
+    local = frame->local;
 
-	local = AFR_FRAME_INIT (frame, op_errno);
-	if (!local)
-                goto out;
+    LOCK(&frame->lock);
+    {
+        if (op_ret != 0) {
+            local->op_errno = op_errno;
+            goto unlock;
+        }
 
-        call_count = local->call_count;
-	if (!call_count) {
-		op_errno = ENOTCONN;
-		goto out;
-	}
+        local->op_ret = op_ret;
 
-        for (i = 0; i < priv->child_count; i++) {
-                if (local->child_up[i]) {
-                        STACK_WIND (frame, afr_fxattrop_cbk,
-                                    priv->children[i],
-                                    priv->children[i]->fops->fxattrop,
-                                    fd, optype, xattr, xdata);
-                        if (!--call_count)
-                                break;
+        buf = &local->cont.statfs.buf;
+        if (local->cont.statfs.buf_set) {
+            if (statvfs->f_bavail < buf->f_bavail) {
+                *buf = *statvfs;
+                if (xdata) {
+                    if (local->xdata_rsp)
+                        dict_unref(local->xdata_rsp);
+                    local->xdata_rsp = dict_ref(xdata);
                 }
+            }
+        } else {
+            *buf = *statvfs;
+            local->cont.statfs.buf_set = 1;
+            if (xdata)
+                local->xdata_rsp = dict_ref(xdata);
         }
+    }
+unlock:
+    call_count = --local->call_count;
+    UNLOCK(&frame->lock);
 
-	return 0;
-out:
-	AFR_STACK_UNWIND (fxattrop, frame, -1, op_errno, NULL, NULL);
+    if (call_count == 0)
+        AFR_STACK_UNWIND(statfs, frame, local->op_ret, local->op_errno,
+                         &local->cont.statfs.buf, local->xdata_rsp);
 
-        return 0;
+    return 0;
 }
 
-/* }}} */
-
-int32_t
-afr_unlock_partial_inodelk_cbk (call_frame_t *frame, void *cookie,
-                                xlator_t *this, int32_t op_ret,
-                                int32_t op_errno, dict_t *xdata)
-
+int
+afr_statfs(call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xdata)
 {
-        afr_local_t *local = NULL;
-        afr_private_t *priv = NULL;
-        int call_count = -1;
-        int child_index = (long)cookie;
-        uuid_t  gfid = {0};
-
-        local = frame->local;
-        priv = this->private;
-
-        if (op_ret < 0 && op_errno != ENOTCONN) {
-                loc_gfid (&local->loc, gfid);
-                gf_log (this->name, GF_LOG_ERROR, "%s: Failed to unlock %s "
-                        "with lk_owner: %s (%s)", uuid_utoa (gfid),
-                        priv->children[child_index]->name,
-                        lkowner_utoa (&frame->root->lk_owner),
-                        strerror (op_errno));
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
+    int i = 0;
+    int call_count = 0;
+    int32_t op_errno = ENOMEM;
+
+    priv = this->private;
+
+    local = AFR_FRAME_INIT(frame, op_errno);
+    if (!local)
+        goto out;
+
+    local->op = GF_FOP_STATFS;
+    if (!afr_is_consistent_io_possible(local, priv, &op_errno))
+        goto out;
+
+    if (priv->arbiter_count == 1 && local->child_up[ARBITER_BRICK_INDEX])
+        local->call_count--;
+    call_count = local->call_count;
+    if (!call_count) {
+        op_errno = ENOTCONN;
+        goto out;
+    }
+
+    for (i = 0; i < priv->child_count; i++) {
+        if (local->child_up[i]) {
+            if (AFR_IS_ARBITER_BRICK(priv, i))
+                continue;
+            STACK_WIND(frame, afr_statfs_cbk, priv->children[i],
+                       priv->children[i]->fops->statfs, loc, xdata);
+            if (!--call_count)
+                break;
         }
+    }
 
-        call_count = afr_frame_return (frame);
-        if (call_count == 0) {
-                AFR_STACK_UNWIND (inodelk, frame, local->op_ret,
-                                  local->op_errno, local->xdata_rsp);
-        }
+    return 0;
+out:
+    AFR_STACK_UNWIND(statfs, frame, -1, op_errno, NULL, NULL);
 
-        return 0;
+    return 0;
 }
 
 int32_t
-afr_unlock_inodelks_and_unwind (call_frame_t *frame, xlator_t *this,
-                                int call_count)
+afr_lk_unlock_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                  int32_t op_ret, int32_t op_errno, struct gf_flock *lock,
+                  dict_t *xdata)
 {
-        int i = 0;
-        afr_private_t *priv = NULL;
-        afr_local_t *local = NULL;
-
-        local = frame->local;
-        priv = this->private;
-        local->call_count = call_count;
-        local->cont.inodelk.flock.l_type = F_UNLCK;
-
-        for (i = 0; i < priv->child_count; i++) {
-                if (!local->replies[i].valid)
-                        continue;
-
-                if (local->replies[i].op_ret == -1)
-                        continue;
-
-                STACK_WIND_COOKIE (frame, afr_unlock_partial_inodelk_cbk,
-                                   (void*) (long) i,
-                                   priv->children[i],
-                                   priv->children[i]->fops->inodelk,
-                                   local->cont.inodelk.volume,
-                                   &local->loc, local->cont.inodelk.cmd,
-                                   &local->cont.inodelk.flock, 0);
-
-                if (!--call_count)
-                        break;
-        }
-
-        return 0;
+    afr_local_t *local = NULL;
+    afr_private_t *priv = this->private;
+    int call_count = -1;
+    int child_index = (long)cookie;
+
+    local = frame->local;
+
+    if (op_ret < 0 && op_errno != ENOTCONN && op_errno != EBADFD) {
+        gf_msg(this->name, GF_LOG_ERROR, op_errno, AFR_MSG_UNLOCK_FAIL,
+               "gfid=%s: unlock failed on subvolume %s "
+               "with lock owner %s",
+               uuid_utoa(local->fd->inode->gfid),
+               priv->children[child_index]->name,
+               lkowner_utoa(&frame->root->lk_owner));
+    }
+
+    call_count = afr_frame_return(frame);
+    if (call_count == 0) {
+        AFR_STACK_UNWIND(lk, frame, local->op_ret, local->op_errno, NULL,
+                         local->xdata_rsp);
+    }
+
+    return 0;
 }
 
 int32_t
-afr_inodelk_done (call_frame_t *frame, xlator_t *this)
+afr_lk_unlock(call_frame_t *frame, xlator_t *this)
 {
-        int i = 0;
-        int lock_count = 0;
-
-        afr_local_t *local = NULL;
-        afr_private_t *priv = NULL;
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
+    int i = 0;
+    int call_count = 0;
 
-        local = frame->local;
-        priv = this->private;
+    local = frame->local;
+    priv = this->private;
 
-        for (i = 0; i < priv->child_count; i++) {
-                if (!local->replies[i].valid)
-                        continue;
+    call_count = afr_locked_nodes_count(local->cont.lk.locked_nodes,
+                                        priv->child_count);
 
-                if (local->replies[i].op_ret == 0)
-                        lock_count++;
+    if (call_count == 0) {
+        AFR_STACK_UNWIND(lk, frame, local->op_ret, local->op_errno, NULL,
+                         local->xdata_rsp);
+        return 0;
+    }
 
-                if (local->op_ret == -1 && local->op_errno == EAGAIN)
-                        continue;
+    local->call_count = call_count;
 
-                if ((local->replies[i].op_ret == -1) &&
-                    (local->replies[i].op_errno == EAGAIN)) {
-                        local->op_ret = -1;
-                        local->op_errno = EAGAIN;
-                        continue;
-                }
+    local->cont.lk.user_flock.l_type = F_UNLCK;
 
-                if (local->replies[i].op_ret == 0)
-                        local->op_ret = 0;
+    for (i = 0; i < priv->child_count; i++) {
+        if (local->cont.lk.locked_nodes[i]) {
+            STACK_WIND_COOKIE(frame, afr_lk_unlock_cbk, (void *)(long)i,
+                              priv->children[i], priv->children[i]->fops->lk,
+                              local->fd, F_SETLK, &local->cont.lk.user_flock,
+                              NULL);
 
-                local->op_errno = local->replies[i].op_errno;
-        }
-
-        if (lock_count && local->cont.inodelk.flock.l_type != F_UNLCK &&
-            (local->op_ret == -1 && local->op_errno == EAGAIN)) {
-                afr_unlock_inodelks_and_unwind (frame, this,
-                                                lock_count);
-        } else {
-                AFR_STACK_UNWIND (inodelk, frame, local->op_ret,
-                                  local->op_errno, local->xdata_rsp);
+            if (!--call_count)
+                break;
         }
+    }
 
-        return 0;
+    return 0;
 }
 
-int
-afr_common_inodelk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                        int32_t op_ret, int32_t op_errno, dict_t *xdata)
+int32_t
+afr_lk_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int32_t op_ret,
+           int32_t op_errno, struct gf_flock *lock, dict_t *xdata)
 {
-        afr_local_t *local = NULL;
-        int child_index = (long)cookie;
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
+    int child_index = -1;
 
-        local = frame->local;
+    local = frame->local;
+    priv = this->private;
 
-        local->replies[child_index].valid = 1;
-        local->replies[child_index].op_ret = op_ret;
-        local->replies[child_index].op_errno = op_errno;
-        if (op_ret == 0 && xdata) {
-                local->replies[child_index].xdata = dict_ref (xdata);
-                LOCK (&frame->lock);
-                {
-                        if (!local->xdata_rsp)
-                                local->xdata_rsp = dict_ref (xdata);
-                }
-                UNLOCK (&frame->lock);
-        }
-        return 0;
-}
+    child_index = (long)cookie;
 
-static int32_t
-afr_parallel_inodelk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                          int32_t op_ret, int32_t op_errno, dict_t *xdata)
+    afr_common_lock_cbk(frame, cookie, this, op_ret, op_errno, xdata);
+    if (op_ret < 0 && op_errno == EAGAIN) {
+        local->op_ret = -1;
+        local->op_errno = EAGAIN;
 
-{
-        int     call_count = 0;
+        afr_lk_unlock(frame, this);
+        return 0;
+    }
+
+    if (op_ret == 0) {
+        local->op_ret = 0;
+        local->op_errno = 0;
+        local->cont.lk.locked_nodes[child_index] = 1;
+        local->cont.lk.ret_flock = *lock;
+    }
+
+    child_index++;
+
+    if (child_index < priv->child_count) {
+        STACK_WIND_COOKIE(frame, afr_lk_cbk, (void *)(long)child_index,
+                          priv->children[child_index],
+                          priv->children[child_index]->fops->lk, local->fd,
+                          local->cont.lk.cmd, &local->cont.lk.user_flock,
+                          local->xdata_req);
+    } else if (priv->quorum_count &&
+               !afr_has_quorum(local->cont.lk.locked_nodes, this, NULL)) {
+        local->op_ret = -1;
+        local->op_errno = afr_final_errno(local, priv);
 
-        afr_common_inodelk_cbk (frame, cookie, this, op_ret, op_errno, xdata);
+        afr_lk_unlock(frame, this);
+    } else {
+        if (local->op_ret < 0)
+            local->op_errno = afr_final_errno(local, priv);
 
-        call_count = afr_frame_return (frame);
-        if (call_count == 0)
-                afr_inodelk_done (frame, this);
+        AFR_STACK_UNWIND(lk, frame, local->op_ret, local->op_errno,
+                         &local->cont.lk.ret_flock, local->xdata_rsp);
+    }
 
-        return 0;
+    return 0;
 }
 
-static inline gf_boolean_t
-afr_is_conflicting_lock_present (int32_t op_ret, int32_t op_errno)
+int
+afr_lk_transaction_cbk(int ret, call_frame_t *frame, void *opaque)
 {
-        if (op_ret == -1 && op_errno == EAGAIN)
-                return _gf_true;
-        return _gf_false;
+    return 0;
 }
 
-static int32_t
-afr_serialized_inodelk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-		            int32_t op_ret, int32_t op_errno, dict_t *xdata)
-
+int
+afr_lk_txn_wind_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                    int32_t op_ret, int32_t op_errno, struct gf_flock *lock,
+                    dict_t *xdata)
 {
-        afr_local_t *local = NULL;
-        afr_private_t *priv = NULL;
-        int child_index = (long)cookie;
-        int next_child  = 0;
-
-        local = frame->local;
-        priv = this->private;
-
-        afr_common_inodelk_cbk (frame, cookie, this, op_ret, op_errno, xdata);
+    afr_local_t *local = NULL;
+    int child_index = -1;
+
+    local = frame->local;
+    child_index = (long)cookie;
+    afr_common_lock_cbk(frame, cookie, this, op_ret, op_errno, xdata);
+    if (op_ret == 0) {
+        local->op_ret = 0;
+        local->op_errno = 0;
+        local->cont.lk.locked_nodes[child_index] = 1;
+        local->cont.lk.ret_flock = *lock;
+    }
+    syncbarrier_wake(&local->barrier);
+    return 0;
+}
 
-        for (next_child = child_index + 1; next_child < priv->child_count;
-             next_child++) {
-                if (local->child_up[next_child])
-                        break;
+int
+afr_lk_txn_unlock_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                      int32_t op_ret, int32_t op_errno, struct gf_flock *lock,
+                      dict_t *xdata)
+{
+    afr_local_t *local = frame->local;
+    afr_private_t *priv = this->private;
+    int child_index = (long)cookie;
+
+    if (op_ret < 0 && op_errno != ENOTCONN && op_errno != EBADFD) {
+        gf_msg(this->name, GF_LOG_ERROR, op_errno, AFR_MSG_UNLOCK_FAIL,
+               "gfid=%s: unlock failed on subvolume %s "
+               "with lock owner %s",
+               uuid_utoa(local->fd->inode->gfid),
+               priv->children[child_index]->name,
+               lkowner_utoa(&frame->root->lk_owner));
+    }
+    return 0;
+}
+int
+afr_lk_transaction(void *opaque)
+{
+    call_frame_t *frame = NULL;
+    xlator_t *this = NULL;
+    afr_private_t *priv = NULL;
+    afr_local_t *local = NULL;
+    char *wind_on = NULL;
+    int op_errno = 0;
+    int i = 0;
+    int ret = 0;
+
+    frame = (call_frame_t *)opaque;
+    local = frame->local;
+    this = frame->this;
+    priv = this->private;
+    wind_on = alloca0(priv->child_count);
+
+    if (priv->arbiter_count || priv->child_count != 3) {
+        op_errno = ENOTSUP;
+        gf_msg(frame->this->name, GF_LOG_ERROR, op_errno, AFR_MSG_LK_HEAL_DOM,
+               "%s: Lock healing supported only for replica 3 volumes.",
+               uuid_utoa(local->fd->inode->gfid));
+        goto err;
+    }
+
+    op_errno = -afr_dom_lock_acquire(frame);  // Released during
+                                              // AFR_STACK_UNWIND
+    if (op_errno != 0) {
+        goto err;
+    }
+    if (priv->quorum_count &&
+        !afr_has_quorum(local->cont.lk.dom_locked_nodes, this, NULL)) {
+        op_errno = afr_final_errno(local, priv);
+        goto err;
+    }
+
+    for (i = 0; i < priv->child_count; i++) {
+        if (priv->child_up[i] && local->cont.lk.dom_locked_nodes[i])
+            wind_on[i] = 1;
+    }
+    AFR_ONLIST(wind_on, frame, afr_lk_txn_wind_cbk, lk, local->fd,
+               local->cont.lk.cmd, &local->cont.lk.user_flock,
+               local->xdata_req);
+
+    if (priv->quorum_count &&
+        !afr_has_quorum(local->cont.lk.locked_nodes, this, NULL)) {
+        local->op_ret = -1;
+        local->op_errno = afr_final_errno(local, priv);
+        goto unlock;
+    } else {
+        if (local->cont.lk.user_flock.l_type == F_UNLCK)
+            ret = afr_remove_lock_from_saved_locks(local, this);
+        else
+            ret = afr_add_lock_to_saved_locks(frame, this);
+        if (ret) {
+            local->op_ret = -1;
+            local->op_errno = -ret;
+            goto unlock;
         }
+        AFR_STACK_UNWIND(lk, frame, local->op_ret, local->op_errno,
+                         &local->cont.lk.ret_flock, local->xdata_rsp);
+    }
 
-        if (afr_is_conflicting_lock_present (op_ret, op_errno) ||
-            (next_child == priv->child_count)) {
-                afr_inodelk_done (frame, this);
-        } else {
-                STACK_WIND_COOKIE (frame, afr_serialized_inodelk_cbk,
-                                   (void *) (long) next_child,
-                                   priv->children[next_child],
-                                   priv->children[next_child]->fops->inodelk,
-                                   (const char *)local->cont.inodelk.volume,
-                                   &local->loc, local->cont.inodelk.cmd,
-                                   &local->cont.inodelk.flock,
-                                   local->xdata_req);
-        }
+    return 0;
 
-        return 0;
+unlock:
+    local->cont.lk.user_flock.l_type = F_UNLCK;
+    AFR_ONLIST(local->cont.lk.locked_nodes, frame, afr_lk_txn_unlock_cbk, lk,
+               local->fd, F_SETLK, &local->cont.lk.user_flock, NULL);
+err:
+    AFR_STACK_UNWIND(lk, frame, -1, op_errno, NULL, NULL);
+    return -1;
 }
 
-static int
-afr_parallel_inodelk_wind (call_frame_t *frame, xlator_t *this)
+int
+afr_lk(call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t cmd,
+       struct gf_flock *flock, dict_t *xdata)
 {
-        afr_private_t *priv = NULL;
-        afr_local_t *local  = NULL;
-        int         call_count = 0;
-        int i = 0;
-
-        priv = this->private;
-        local = frame->local;
-        call_count = local->call_count;
-
-        for (i = 0; i < priv->child_count; i++) {
-                if (!local->child_up[i])
-                        continue;
-                STACK_WIND_COOKIE (frame, afr_parallel_inodelk_cbk,
-                                   (void *) (long) i,
-                                   priv->children[i],
-                                   priv->children[i]->fops->inodelk,
-                                   (const char *)local->cont.inodelk.volume,
-                                   &local->loc, local->cont.inodelk.cmd,
-                                   &local->cont.inodelk.flock,
-                                   local->xdata_req);
-                if (!--call_count)
-                        break;
+    afr_private_t *priv = NULL;
+    afr_local_t *local = NULL;
+    int ret = 0;
+    int i = 0;
+    int32_t op_errno = ENOMEM;
+
+    priv = this->private;
+
+    local = AFR_FRAME_INIT(frame, op_errno);
+    if (!local)
+        goto out;
+
+    local->op = GF_FOP_LK;
+    if (!afr_lk_is_unlock(cmd, flock)) {
+        AFR_ERROR_OUT_IF_FDCTX_INVALID(fd, this, op_errno, out);
+        if (!afr_is_consistent_io_possible(local, priv, &op_errno))
+            goto out;
+    }
+
+    local->cont.lk.locked_nodes = GF_CALLOC(
+        priv->child_count, sizeof(*local->cont.lk.locked_nodes),
+        gf_afr_mt_char);
+
+    if (!local->cont.lk.locked_nodes) {
+        op_errno = ENOMEM;
+        goto out;
+    }
+
+    local->fd = fd_ref(fd);
+    local->cont.lk.cmd = cmd;
+    local->cont.lk.user_flock = *flock;
+    local->cont.lk.ret_flock = *flock;
+    if (xdata)
+        local->xdata_req = dict_ref(xdata);
+
+    if (afr_is_lock_mode_mandatory(xdata)) {
+        ret = synctask_new(this->ctx->env, afr_lk_transaction,
+                           afr_lk_transaction_cbk, frame, frame);
+        if (ret) {
+            op_errno = ENOMEM;
+            goto out;
         }
         return 0;
-}
+    }
 
-static int
-afr_serialized_inodelk_wind (call_frame_t *frame, xlator_t *this)
-{
-        afr_private_t *priv = NULL;
-        afr_local_t *local  = NULL;
-        int i = 0;
+    STACK_WIND_COOKIE(frame, afr_lk_cbk, (void *)(long)0, priv->children[i],
+                      priv->children[i]->fops->lk, fd, cmd, flock,
+                      local->xdata_req);
 
-        priv = this->private;
-        local = frame->local;
+    return 0;
+out:
+    AFR_STACK_UNWIND(lk, frame, -1, op_errno, NULL, NULL);
 
-        for (i = 0; i < priv->child_count; i++) {
-                if (local->child_up[i]) {
-                        STACK_WIND_COOKIE (frame, afr_serialized_inodelk_cbk,
-                                           (void *) (long) i,
-                                           priv->children[i],
-                                           priv->children[i]->fops->inodelk,
-                                       (const char *)local->cont.inodelk.volume,
-                                           &local->loc, local->cont.inodelk.cmd,
-                                           &local->cont.inodelk.flock,
-                                           local->xdata_req);
-                        break;
-                }
-        }
-        return 0;
+    return 0;
 }
 
 int32_t
-afr_inodelk (call_frame_t *frame, xlator_t *this,
-             const char *volume, loc_t *loc, int32_t cmd,
-             struct gf_flock *flock, dict_t *xdata)
+afr_lease_unlock_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                     int32_t op_ret, int32_t op_errno, struct gf_lease *lease,
+                     dict_t *xdata)
 {
-        afr_local_t *local  = NULL;
-        int32_t op_errno = ENOMEM;
+    afr_local_t *local = NULL;
+    int call_count = -1;
 
-        local = AFR_FRAME_INIT (frame, op_errno);
-        if (!local)
-                goto out;
+    local = frame->local;
+    call_count = afr_frame_return(frame);
 
-        loc_copy (&local->loc, loc);
-        local->cont.inodelk.volume = gf_strdup (volume);
-        if (!local->cont.inodelk.volume) {
-                op_errno = ENOMEM;
-                goto out;
-        }
+    if (call_count == 0)
+        AFR_STACK_UNWIND(lease, frame, local->op_ret, local->op_errno, lease,
+                         xdata);
 
-        local->cont.inodelk.cmd = cmd;
-        local->cont.inodelk.flock = *flock;
-        if (xdata)
-                local->xdata_req = dict_ref (xdata);
-
-        /* At least one child is up */
-        /*
-         * Non-blocking locks also need to be serialized.  Otherwise there is
-         * a chance that both the mounts which issued same non-blocking inodelk
-         * may endup not acquiring the lock on any-brick.
-         * Ex: Mount1 and Mount2
-         * request for full length lock on file f1.  Mount1 afr may acquire the
-         * partial lock on brick-1 and may not acquire the lock on brick-2
-         * because Mount2 already got the lock on brick-2, vice versa.  Since
-         * both the mounts only got partial locks, afr treats them as failure in
-         * gaining the locks and unwinds with EAGAIN errno.
-         */
-        if (flock->l_type == F_UNLCK) {
-                afr_parallel_inodelk_wind (frame, this);
-        } else {
-                afr_serialized_inodelk_wind (frame, this);
-        }
-
-	return 0;
-out:
-	AFR_STACK_UNWIND (inodelk, frame, -1, op_errno, NULL);
-
-        return 0;
+    return 0;
 }
 
-
 int32_t
-afr_finodelk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-		  int32_t op_ret, int32_t op_errno, dict_t *xdata)
-
+afr_lease_unlock(call_frame_t *frame, xlator_t *this)
 {
-        afr_local_t *local = NULL;
-        int call_count = -1;
-
-        local = frame->local;
-
-        LOCK (&frame->lock);
-        {
-                if (op_ret == 0)
-                        local->op_ret = 0;
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
+    int i = 0;
+    int call_count = 0;
 
-                local->op_errno = op_errno;
-        }
-        UNLOCK (&frame->lock);
-
-        call_count = afr_frame_return (frame);
+    local = frame->local;
+    priv = this->private;
 
-        if (call_count == 0)
-                AFR_STACK_UNWIND (finodelk, frame, local->op_ret,
-                                  local->op_errno, xdata);
+    call_count = afr_locked_nodes_count(local->cont.lease.locked_nodes,
+                                        priv->child_count);
 
+    if (call_count == 0) {
+        AFR_STACK_UNWIND(lease, frame, local->op_ret, local->op_errno,
+                         &local->cont.lease.ret_lease, NULL);
         return 0;
-}
-
-
-int32_t
-afr_finodelk (call_frame_t *frame, xlator_t *this, const char *volume, fd_t *fd,
-	      int32_t cmd, struct gf_flock *flock, dict_t *xdata)
-{
-        afr_private_t *priv = NULL;
-        afr_local_t *local  = NULL;
-        int i = 0;
-        int32_t call_count = 0;
-        int32_t op_errno = ENOMEM;
+    }
 
-        priv = this->private;
+    local->call_count = call_count;
 
-	local = AFR_FRAME_INIT (frame, op_errno);
-	if (!local)
-		goto out;
+    local->cont.lease.user_lease.cmd = GF_UNLK_LEASE;
 
-        call_count = local->call_count;
-	if (!call_count) {
-		op_errno = ENOTCONN;
-		goto out;
-	}
+    for (i = 0; i < priv->child_count; i++) {
+        if (local->cont.lease.locked_nodes[i]) {
+            STACK_WIND(frame, afr_lease_unlock_cbk, priv->children[i],
+                       priv->children[i]->fops->lease, &local->loc,
+                       &local->cont.lease.user_lease, NULL);
 
-        for (i = 0; i < priv->child_count; i++) {
-                if (local->child_up[i]) {
-                        STACK_WIND (frame, afr_finodelk_cbk,
-                                    priv->children[i],
-                                    priv->children[i]->fops->finodelk,
-                                    volume, fd, cmd, flock, xdata);
-
-                        if (!--call_count)
-                                break;
-                }
+            if (!--call_count)
+                break;
         }
+    }
 
-	return 0;
-out:
-	AFR_STACK_UNWIND (finodelk, frame, -1, op_errno, NULL);
-
-        return 0;
+    return 0;
 }
 
-
 int32_t
-afr_entrylk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                 int32_t op_ret, int32_t op_errno, dict_t *xdata)
+afr_lease_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int32_t op_ret,
+              int32_t op_errno, struct gf_lease *lease, dict_t *xdata)
 {
-        afr_local_t *local = NULL;
-        int call_count = -1;
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
+    int child_index = -1;
 
-        local = frame->local;
+    local = frame->local;
+    priv = this->private;
 
-        LOCK (&frame->lock);
-        {
-                if (op_ret == 0)
-                        local->op_ret = 0;
+    child_index = (long)cookie;
 
-                local->op_errno = op_errno;
-        }
-        UNLOCK (&frame->lock);
+    afr_common_lock_cbk(frame, cookie, this, op_ret, op_errno, xdata);
+    if (op_ret < 0 && op_errno == EAGAIN) {
+        local->op_ret = -1;
+        local->op_errno = EAGAIN;
 
-        call_count = afr_frame_return (frame);
+        afr_lease_unlock(frame, this);
+        return 0;
+    }
+
+    if (op_ret == 0) {
+        local->op_ret = 0;
+        local->op_errno = 0;
+        local->cont.lease.locked_nodes[child_index] = 1;
+        local->cont.lease.ret_lease = *lease;
+    }
+
+    child_index++;
+    if (child_index < priv->child_count) {
+        STACK_WIND_COOKIE(frame, afr_lease_cbk, (void *)(long)child_index,
+                          priv->children[child_index],
+                          priv->children[child_index]->fops->lease, &local->loc,
+                          &local->cont.lease.user_lease, xdata);
+    } else if (priv->quorum_count &&
+               !afr_has_quorum(local->cont.lease.locked_nodes, this, NULL)) {
+        local->op_ret = -1;
+        local->op_errno = afr_final_errno(local, priv);
 
-        if (call_count == 0)
-                AFR_STACK_UNWIND (entrylk, frame, local->op_ret,
-                                  local->op_errno, xdata);
+        afr_lease_unlock(frame, this);
+    } else {
+        if (local->op_ret < 0)
+            local->op_errno = afr_final_errno(local, priv);
+        AFR_STACK_UNWIND(lease, frame, local->op_ret, local->op_errno,
+                         &local->cont.lease.ret_lease, NULL);
+    }
 
-        return 0;
+    return 0;
 }
 
-
 int
-afr_entrylk (call_frame_t *frame, xlator_t *this, const char *volume,
-	     loc_t *loc, const char *basename, entrylk_cmd cmd,
-	     entrylk_type type, dict_t *xdata)
+afr_lease(call_frame_t *frame, xlator_t *this, loc_t *loc,
+          struct gf_lease *lease, dict_t *xdata)
 {
-        afr_private_t *priv = NULL;
-        afr_local_t *local  = NULL;
-        int i = 0;
-        int32_t call_count = 0;
-        int32_t op_errno = 0;
+    afr_private_t *priv = NULL;
+    afr_local_t *local = NULL;
+    int32_t op_errno = ENOMEM;
 
-        priv = this->private;
+    priv = this->private;
 
-	local = AFR_FRAME_INIT (frame, op_errno);
-	if (!local)
-		goto out;
+    local = AFR_FRAME_INIT(frame, op_errno);
+    if (!local)
+        goto out;
 
-        call_count = local->call_count;
-	if (!call_count) {
-		op_errno = ENOTCONN;
-		goto out;
-	}
+    local->op = GF_FOP_LEASE;
+    local->cont.lease.locked_nodes = GF_CALLOC(
+        priv->child_count, sizeof(*local->cont.lease.locked_nodes),
+        gf_afr_mt_char);
 
-        for (i = 0; i < priv->child_count; i++) {
-                if (local->child_up[i]) {
-                        STACK_WIND (frame, afr_entrylk_cbk,
-                                    priv->children[i],
-                                    priv->children[i]->fops->entrylk,
-                                    volume, loc, basename, cmd, type, xdata);
-
-                        if (!--call_count)
-                                break;
-                }
-        }
+    if (!local->cont.lease.locked_nodes) {
+        op_errno = ENOMEM;
+        goto out;
+    }
 
-	return 0;
-out:
-	AFR_STACK_UNWIND (entrylk, frame, -1, op_errno, NULL);
+    loc_copy(&local->loc, loc);
+    local->cont.lease.user_lease = *lease;
+    local->cont.lease.ret_lease = *lease;
 
-        return 0;
-}
+    STACK_WIND_COOKIE(frame, afr_lease_cbk, (void *)(long)0, priv->children[0],
+                      priv->children[0]->fops->lease, loc, lease, xdata);
 
+    return 0;
+out:
+    AFR_STACK_UNWIND(lease, frame, -1, op_errno, NULL, NULL);
 
+    return 0;
+}
 
 int
-afr_fentrylk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-		  int32_t op_ret, int32_t op_errno, dict_t *xdata)
-
+afr_ipc_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int32_t op_ret,
+            int32_t op_errno, dict_t *xdata)
 {
-        afr_local_t *local = NULL;
-        int call_count = -1;
-
-        local = frame->local;
-
-        LOCK (&frame->lock);
-        {
-                if (op_ret == 0)
-                        local->op_ret = 0;
-
-                local->op_errno = op_errno;
-        }
-        UNLOCK (&frame->lock);
-
-        call_count = afr_frame_return (frame);
+    afr_local_t *local = NULL;
+    int child_index = (long)cookie;
+    int call_count = 0;
+    gf_boolean_t failed = _gf_false;
+    gf_boolean_t succeeded = _gf_false;
+    int i = 0;
+    afr_private_t *priv = NULL;
+
+    local = frame->local;
+    priv = this->private;
+
+    local->replies[child_index].valid = 1;
+    local->replies[child_index].op_ret = op_ret;
+    local->replies[child_index].op_errno = op_errno;
+    if (xdata)
+        local->replies[child_index].xdata = dict_ref(xdata);
+
+    call_count = afr_frame_return(frame);
+    if (call_count)
+        goto out;
+    /* If any of the subvolumes failed with other than ENOTCONN
+     * return error else return success unless all the subvolumes
+     * failed.
+     * TODO: In case of failure, we need to unregister the xattrs
+     * from the other subvolumes where it succeeded (once upcall
+     * fixes the Bz-1371622)*/
+    for (i = 0; i < priv->child_count; i++) {
+        if (!local->replies[i].valid)
+            continue;
+        if (local->replies[i].op_ret < 0 &&
+            local->replies[i].op_errno != ENOTCONN) {
+            local->op_ret = local->replies[i].op_ret;
+            local->op_errno = local->replies[i].op_errno;
+            if (local->xdata_rsp)
+                dict_unref(local->xdata_rsp);
+            local->xdata_rsp = NULL;
+            if (local->replies[i].xdata) {
+                local->xdata_rsp = dict_ref(local->replies[i].xdata);
+            }
+            failed = _gf_true;
+            break;
+        }
+        if (local->replies[i].op_ret == 0) {
+            succeeded = _gf_true;
+            local->op_ret = 0;
+            local->op_errno = 0;
+            if (!local->xdata_rsp && local->replies[i].xdata) {
+                local->xdata_rsp = dict_ref(local->replies[i].xdata);
+            }
+        }
+    }
+
+    if (!succeeded && !failed) {
+        local->op_ret = -1;
+        local->op_errno = ENOTCONN;
+    }
 
-        if (call_count == 0)
-                AFR_STACK_UNWIND (fentrylk, frame, local->op_ret,
-                                  local->op_errno, xdata);
+    AFR_STACK_UNWIND(ipc, frame, local->op_ret, local->op_errno,
+                     local->xdata_rsp);
 
-        return 0;
+out:
+    return 0;
 }
 
-
 int
-afr_fentrylk (call_frame_t *frame, xlator_t *this, const char *volume, fd_t *fd,
-              const char *basename, entrylk_cmd cmd, entrylk_type type,
-	      dict_t *xdata)
+afr_ipc(call_frame_t *frame, xlator_t *this, int32_t op, dict_t *xdata)
 {
-        afr_private_t *priv = NULL;
-        afr_local_t *local  = NULL;
-        int i = 0;
-        int32_t call_count = 0;
-        int32_t op_errno = ENOMEM;
+    afr_local_t *local = NULL;
+    int32_t op_errno = -1;
+    afr_private_t *priv = NULL;
+    int i = 0;
+    int call_cnt = -1;
+
+    VALIDATE_OR_GOTO(frame, err);
+    VALIDATE_OR_GOTO(this, err);
 
-        priv = this->private;
+    if (op != GF_IPC_TARGET_UPCALL)
+        goto wind_default;
 
-        local = AFR_FRAME_INIT (frame, op_errno);
-	if (!local)
-		goto out;
+    VALIDATE_OR_GOTO(this->private, err);
+    priv = this->private;
 
-        call_count = local->call_count;
-	if (!call_count) {
-		op_errno = ENOTCONN;
-		goto out;
-	}
+    local = AFR_FRAME_INIT(frame, op_errno);
+    if (!local)
+        goto err;
 
+    call_cnt = local->call_count;
+
+    if (xdata) {
         for (i = 0; i < priv->child_count; i++) {
-                if (local->child_up[i]) {
-                        STACK_WIND (frame, afr_fentrylk_cbk,
-                                    priv->children[i],
-                                    priv->children[i]->fops->fentrylk,
-                                    volume, fd, basename, cmd, type, xdata);
-
-                        if (!--call_count)
-                                break;
-                }
+            if (dict_set_int8(xdata, priv->pending_key[i], 0) < 0)
+                goto err;
         }
+    }
 
-	return 0;
-out:
-	AFR_STACK_UNWIND (fentrylk, frame, -1, op_errno, NULL);
+    for (i = 0; i < priv->child_count; i++) {
+        if (!local->child_up[i])
+            continue;
 
-        return 0;
+        STACK_WIND_COOKIE(frame, afr_ipc_cbk, (void *)(long)i,
+                          priv->children[i], priv->children[i]->fops->ipc, op,
+                          xdata);
+        if (!--call_cnt)
+            break;
+    }
+    return 0;
+
+err:
+    if (op_errno == -1)
+        op_errno = errno;
+    AFR_STACK_UNWIND(ipc, frame, -1, op_errno, NULL);
+
+    return 0;
+
+wind_default:
+    STACK_WIND(frame, default_ipc_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->ipc, op, xdata);
+    return 0;
 }
 
-
 int
-afr_statfs_cbk (call_frame_t *frame, void *cookie, xlator_t *this, int op_ret,
-		int op_errno, struct statvfs *statvfs, dict_t *xdata)
+afr_forget(xlator_t *this, inode_t *inode)
 {
-        afr_local_t *local = NULL;
-        int call_count = 0;
-	struct statvfs *buf = NULL;
-
-        LOCK (&frame->lock);
-        {
-                local = frame->local;
-
-                if (op_ret != 0) {
-                        local->op_errno = op_errno;
-			goto unlock;
-		}
-
-		local->op_ret = op_ret;
-
-		buf = &local->cont.statfs.buf;
-		if (local->cont.statfs.buf_set) {
-			if (statvfs->f_bavail < buf->f_bavail) {
-				*buf = *statvfs;
-				if (xdata) {
-					if (local->xdata_rsp)
-						dict_unref (local->xdata_rsp);
-					local->xdata_rsp = dict_ref (xdata);
-				}
-			}
-		} else {
-			*buf = *statvfs;
-			local->cont.statfs.buf_set = 1;
-			if (xdata)
-				local->xdata_rsp = dict_ref (xdata);
-		}
-        }
-unlock:
-        UNLOCK (&frame->lock);
-
-        call_count = afr_frame_return (frame);
-
-        if (call_count == 0)
-                AFR_STACK_UNWIND (statfs, frame, local->op_ret, local->op_errno,
-                                  &local->cont.statfs.buf, local->xdata_rsp);
+    uint64_t ctx_int = 0;
+    afr_inode_ctx_t *ctx = NULL;
 
+    afr_spb_choice_timeout_cancel(this, inode);
+    inode_ctx_del(inode, this, &ctx_int);
+    if (!ctx_int)
         return 0;
-}
 
+    ctx = (afr_inode_ctx_t *)(uintptr_t)ctx_int;
+    afr_inode_ctx_destroy(ctx);
+    return 0;
+}
 
 int
-afr_statfs (call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xdata)
+afr_priv_dump(xlator_t *this)
 {
-        afr_local_t   *  local       = NULL;
-	afr_private_t   *priv        = NULL;
-        int              i           = 0;
-        int              call_count = 0;
-        int32_t          op_errno    = ENOMEM;
-
-	priv = this->private;
-
-	local = AFR_FRAME_INIT (frame, op_errno);
-	if (!local)
-		goto out;
-
-        call_count = local->call_count;
-	if (!call_count) {
-		op_errno = ENOTCONN;
-		goto out;
-	}
-
-        for (i = 0; i < priv->child_count; i++) {
-                if (local->child_up[i]) {
-                        STACK_WIND (frame, afr_statfs_cbk,
-                                    priv->children[i],
-                                    priv->children[i]->fops->statfs,
-                                    loc, xdata);
-                        if (!--call_count)
-                                break;
-                }
-        }
-
-	return 0;
-out:
-	AFR_STACK_UNWIND (statfs, frame, -1, op_errno, NULL, NULL);
-
-        return 0;
+    afr_private_t *priv = NULL;
+    char key_prefix[GF_DUMP_MAX_BUF_LEN];
+    char key[GF_DUMP_MAX_BUF_LEN];
+    int i = 0;
+
+    GF_ASSERT(this);
+    priv = this->private;
+
+    GF_ASSERT(priv);
+    snprintf(key_prefix, GF_DUMP_MAX_BUF_LEN, "%s.%s", this->type, this->name);
+    gf_proc_dump_add_section("%s", key_prefix);
+    gf_proc_dump_write("child_count", "%u", priv->child_count);
+    for (i = 0; i < priv->child_count; i++) {
+        sprintf(key, "child_up[%d]", i);
+        gf_proc_dump_write(key, "%d", priv->child_up[i]);
+        sprintf(key, "pending_key[%d]", i);
+        gf_proc_dump_write(key, "%s", priv->pending_key[i]);
+        sprintf(key, "pending_reads[%d]", i);
+        gf_proc_dump_write(key, "%" PRId64,
+                           GF_ATOMIC_GET(priv->pending_reads[i]));
+        sprintf(key, "child_latency[%d]", i);
+        gf_proc_dump_write(key, "%" PRId64, priv->child_latency[i]);
+        sprintf(key, "halo_child_up[%d]", i);
+        gf_proc_dump_write(key, "%d", priv->halo_child_up[i]);
+    }
+    gf_proc_dump_write("data_self_heal", "%d", priv->data_self_heal);
+    gf_proc_dump_write("metadata_self_heal", "%d", priv->metadata_self_heal);
+    gf_proc_dump_write("entry_self_heal", "%d", priv->entry_self_heal);
+    gf_proc_dump_write("read_child", "%d", priv->read_child);
+    gf_proc_dump_write("wait_count", "%u", priv->wait_count);
+    gf_proc_dump_write("heal-wait-queue-length", "%d", priv->heal_wait_qlen);
+    gf_proc_dump_write("heal-waiters", "%d", priv->heal_waiters);
+    gf_proc_dump_write("background-self-heal-count", "%d",
+                       priv->background_self_heal_count);
+    gf_proc_dump_write("healers", "%d", priv->healers);
+    gf_proc_dump_write("read-hash-mode", "%d", priv->hash_mode);
+    gf_proc_dump_write("use-anonymous-inode", "%d", priv->use_anon_inode);
+    if (priv->quorum_count == AFR_QUORUM_AUTO) {
+        gf_proc_dump_write("quorum-type", "auto");
+    } else if (priv->quorum_count == 0) {
+        gf_proc_dump_write("quorum-type", "none");
+    } else {
+        gf_proc_dump_write("quorum-type", "fixed");
+        gf_proc_dump_write("quorum-count", "%d", priv->quorum_count);
+    }
+    gf_proc_dump_write("up", "%u", afr_has_quorum(priv->child_up, this, NULL));
+    if (priv->thin_arbiter_count) {
+        gf_proc_dump_write("ta_child_up", "%d", priv->ta_child_up);
+        gf_proc_dump_write("ta_bad_child_index", "%d",
+                           priv->ta_bad_child_index);
+        gf_proc_dump_write("ta_notify_dom_lock_offset", "%" PRId64,
+                           priv->ta_notify_dom_lock_offset);
+    }
+
+    return 0;
 }
 
+/**
+ * find_child_index - find the child's index in the array of subvolumes
+ * @this: AFR
+ * @child: child
+ */
 
-int32_t
-afr_lk_unlock_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                   int32_t op_ret, int32_t op_errno, struct gf_flock *lock,
-                   dict_t *xdata)
+static int
+afr_find_child_index(xlator_t *this, xlator_t *child)
 {
-        afr_local_t * local = NULL;
-        int call_count = -1;
-
-        local = frame->local;
-        call_count = afr_frame_return (frame);
-
-        if (call_count == 0)
-                AFR_STACK_UNWIND (lk, frame, local->op_ret, local->op_errno,
-                                  lock, xdata);
-
-        return 0;
+    afr_private_t *priv = NULL;
+    int child_count = -1;
+    int i = -1;
+
+    priv = this->private;
+    child_count = priv->child_count;
+    if (priv->thin_arbiter_count) {
+        child_count++;
+    }
+
+    for (i = 0; i < child_count; i++) {
+        if ((xlator_t *)child == priv->children[i])
+            break;
+    }
+
+    return i;
 }
 
-
-int32_t
-afr_lk_unlock (call_frame_t *frame, xlator_t *this)
+int
+__afr_get_up_children_count(afr_private_t *priv)
 {
-        afr_local_t   * local = NULL;
-        afr_private_t * priv  = NULL;
-        int i = 0;
-        int call_count = 0;
-
-        local = frame->local;
-        priv  = this->private;
+    int up_children = 0;
+    int i = 0;
 
-        call_count = afr_locked_nodes_count (local->cont.lk.locked_nodes,
-                                             priv->child_count);
+    for (i = 0; i < priv->child_count; i++)
+        if (priv->child_up[i] == 1)
+            up_children++;
 
-        if (call_count == 0) {
-                AFR_STACK_UNWIND (lk, frame, local->op_ret, local->op_errno,
-                                  &local->cont.lk.ret_flock, NULL);
-                return 0;
-        }
-
-        local->call_count = call_count;
+    return up_children;
+}
 
-        local->cont.lk.user_flock.l_type = F_UNLCK;
+static int
+__get_heard_from_all_status(xlator_t *this)
+{
+    afr_private_t *priv = this->private;
+    int i;
 
-        for (i = 0; i < priv->child_count; i++) {
-                if (local->cont.lk.locked_nodes[i]) {
-                        STACK_WIND (frame, afr_lk_unlock_cbk,
-                                    priv->children[i],
-                                    priv->children[i]->fops->lk,
-                                    local->fd, F_SETLK,
-                                    &local->cont.lk.user_flock, NULL);
-
-                        if (!--call_count)
-                                break;
-                }
+    for (i = 0; i < priv->child_count; i++) {
+        if (!priv->last_event[i]) {
+            return 0;
         }
-
+    }
+    if (priv->thin_arbiter_count && !priv->ta_child_up) {
         return 0;
+    }
+    return 1;
 }
 
-
-int32_t
-afr_lk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-            int32_t op_ret, int32_t op_errno, struct gf_flock *lock, dict_t *xdata)
+glusterfs_event_t
+__afr_transform_event_from_state(xlator_t *this)
 {
-        afr_local_t *local = NULL;
-        afr_private_t *priv = NULL;
-        int child_index = -1;
-/*        int            ret  = 0; */
+    int i = 0;
+    int up_children = 0;
+    afr_private_t *priv = this->private;
+
+    if (__get_heard_from_all_status(this))
+        /* have_heard_from_all. Let afr_notify() do the propagation. */
+        return GF_EVENT_MAXVAL;
+
+    up_children = __afr_get_up_children_count(priv);
+    /* Treat the children with pending notification, as having sent a
+     * GF_EVENT_CHILD_DOWN. i.e. set the event as GF_EVENT_SOME_DESCENDENT_DOWN,
+     * as done in afr_notify() */
+    for (i = 0; i < priv->child_count; i++) {
+        if (priv->last_event[i])
+            continue;
+        priv->last_event[i] = GF_EVENT_SOME_DESCENDENT_DOWN;
+        priv->child_up[i] = 0;
+    }
+
+    if (up_children)
+        /* We received at least one child up */
+        return GF_EVENT_CHILD_UP;
+    else
+        return GF_EVENT_CHILD_DOWN;
+
+    return GF_EVENT_MAXVAL;
+}
 
+static void
+afr_notify_cbk(void *data)
+{
+    xlator_t *this = data;
+    afr_private_t *priv = this->private;
+    glusterfs_event_t event = GF_EVENT_MAXVAL;
+    gf_boolean_t propagate = _gf_false;
+
+    LOCK(&priv->lock);
+    {
+        if (!priv->timer) {
+            /*
+             * Either child_up/child_down is already sent to parent.
+             * This is a spurious wake up.
+             */
+            goto unlock;
+        }
+        priv->timer = NULL;
+        event = __afr_transform_event_from_state(this);
+        if (event != GF_EVENT_MAXVAL)
+            propagate = _gf_true;
+    }
+unlock:
+    UNLOCK(&priv->lock);
+    if (propagate)
+        default_notify(this, event, NULL);
+}
 
-        local = frame->local;
-        priv  = this->private;
+static void
+__afr_launch_notify_timer(xlator_t *this, afr_private_t *priv)
+{
+    struct timespec delay = {
+        0,
+    };
+
+    gf_msg_debug(this->name, 0, "Initiating child-down timer");
+    delay.tv_sec = 10;
+    delay.tv_nsec = 0;
+    priv->timer = gf_timer_call_after(this->ctx, delay, afr_notify_cbk, this);
+    if (priv->timer == NULL) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, AFR_MSG_TIMER_CREATE_FAIL,
+               "Cannot create timer for delayed initialization");
+    }
+}
 
-        child_index = (long) cookie;
+static int
+find_best_down_child(xlator_t *this)
+{
+    afr_private_t *priv = NULL;
+    int i = -1;
+    int32_t best_child = -1;
+    int64_t best_latency = INT64_MAX;
+
+    priv = this->private;
+
+    for (i = 0; i < priv->child_count; i++) {
+        if (!priv->child_up[i] && priv->child_latency[i] >= 0 &&
+            priv->child_latency[i] < best_latency) {
+            best_child = i;
+            best_latency = priv->child_latency[i];
+        }
+    }
+    if (best_child >= 0) {
+        gf_msg_debug(this->name, 0,
+                     "Found best down child (%d) @ %" PRId64 " ms latency",
+                     best_child, best_latency);
+    }
+    return best_child;
+}
 
-        if (!child_went_down (op_ret, op_errno) && (op_ret == -1)) {
-                local->op_ret   = -1;
-                local->op_errno = op_errno;
+int
+find_worst_up_child(xlator_t *this)
+{
+    afr_private_t *priv = NULL;
+    int i = -1;
+    int32_t worst_child = -1;
+    int64_t worst_latency = INT64_MIN;
+
+    priv = this->private;
+
+    for (i = 0; i < priv->child_count; i++) {
+        if (priv->child_up[i] && priv->child_latency[i] >= 0 &&
+            priv->child_latency[i] > worst_latency) {
+            worst_child = i;
+            worst_latency = priv->child_latency[i];
+        }
+    }
+    if (worst_child >= 0) {
+        gf_msg_debug(this->name, 0,
+                     "Found worst up child (%d) @ %" PRId64 " ms latency",
+                     worst_child, worst_latency);
+    }
+    return worst_child;
+}
 
-                afr_lk_unlock (frame, this);
-                return 0;
-        }
+void
+__afr_handle_ping_event(xlator_t *this, xlator_t *child_xlator, const int idx,
+                        int64_t halo_max_latency_msec, int32_t *event,
+                        int64_t child_latency_msec)
+{
+    afr_private_t *priv = NULL;
+    int up_children = 0;
 
-        if (op_ret == 0) {
-                local->op_ret        = 0;
-                local->op_errno      = 0;
-                local->cont.lk.locked_nodes[child_index] = 1;
-                local->cont.lk.ret_flock = *lock;
-        }
+    priv = this->private;
 
-        child_index++;
+    priv->child_latency[idx] = child_latency_msec;
+    gf_msg_debug(child_xlator->name, 0, "Client ping @ %" PRId64 " ms",
+                 child_latency_msec);
+    if (priv->shd.iamshd)
+        return;
 
-        if (child_index < priv->child_count) {
-                STACK_WIND_COOKIE (frame, afr_lk_cbk, (void *) (long) child_index,
-                                   priv->children[child_index],
-                                   priv->children[child_index]->fops->lk,
-                                   local->fd, local->cont.lk.cmd,
-                                   &local->cont.lk.user_flock, xdata);
-        } else if (local->op_ret == -1) {
-                /* all nodes have gone down */
+    up_children = __afr_get_up_children_count(priv);
 
-                AFR_STACK_UNWIND (lk, frame, -1, ENOTCONN,
-                                  &local->cont.lk.ret_flock, NULL);
+    if (child_latency_msec > halo_max_latency_msec &&
+        priv->child_up[idx] == 1 && up_children > priv->halo_min_replicas) {
+        if ((up_children - 1) < priv->halo_min_replicas) {
+            gf_log(child_xlator->name, GF_LOG_INFO,
+                   "Overriding halo threshold, "
+                   "min replicas: %d",
+                   priv->halo_min_replicas);
         } else {
-                AFR_STACK_UNWIND (lk, frame, local->op_ret, local->op_errno,
-                                  &local->cont.lk.ret_flock, NULL);
+            gf_log(child_xlator->name, GF_LOG_INFO,
+                   "Child latency (%" PRId64
+                   " ms) "
+                   "exceeds halo threshold (%" PRId64
+                   "), "
+                   "marking child down.",
+                   child_latency_msec, halo_max_latency_msec);
+            if (priv->halo_child_up[idx]) {
+                *event = GF_EVENT_CHILD_DOWN;
+            }
+        }
+    } else if (child_latency_msec < halo_max_latency_msec &&
+               priv->child_up[idx] == 0) {
+        if (up_children < priv->halo_max_replicas) {
+            gf_log(child_xlator->name, GF_LOG_INFO,
+                   "Child latency (%" PRId64
+                   " ms) "
+                   "below halo threshold (%" PRId64
+                   "), "
+                   "marking child up.",
+                   child_latency_msec, halo_max_latency_msec);
+            if (priv->halo_child_up[idx]) {
+                *event = GF_EVENT_CHILD_UP;
+            }
+        } else {
+            gf_log(child_xlator->name, GF_LOG_INFO,
+                   "Not marking child %d up, "
+                   "max replicas (%d) reached.",
+                   idx, priv->halo_max_replicas);
         }
-
-        return 0;
+    }
 }
 
-
-int
-afr_lk (call_frame_t *frame, xlator_t *this,
-        fd_t *fd, int32_t cmd, struct gf_flock *flock, dict_t *xdata)
+static int64_t
+afr_get_halo_latency(xlator_t *this)
 {
-        afr_private_t *priv = NULL;
-        afr_local_t *local = NULL;
-        int i = 0;
-        int32_t op_errno = ENOMEM;
-
-        priv = this->private;
-
-        local = AFR_FRAME_INIT (frame, op_errno);
-        if (!local)
-                goto out;
-
-        local->cont.lk.locked_nodes = GF_CALLOC (priv->child_count,
-                                                 sizeof (*local->cont.lk.locked_nodes),
-                                                 gf_afr_mt_char);
-
-        if (!local->cont.lk.locked_nodes) {
-                op_errno = ENOMEM;
-                goto out;
-        }
-
-        local->fd            = fd_ref (fd);
-        local->cont.lk.cmd   = cmd;
-        local->cont.lk.user_flock = *flock;
-        local->cont.lk.ret_flock = *flock;
-
-        STACK_WIND_COOKIE (frame, afr_lk_cbk, (void *) (long) 0,
-                           priv->children[i],
-                           priv->children[i]->fops->lk,
-                           fd, cmd, flock, xdata);
+    afr_private_t *priv = NULL;
+    int64_t halo_max_latency_msec = 0;
+
+    priv = this->private;
+
+    if (priv->shd.iamshd) {
+        halo_max_latency_msec = priv->shd.halo_max_latency_msec;
+    } else if (priv->nfsd.iamnfsd) {
+        halo_max_latency_msec = priv->nfsd.halo_max_latency_msec;
+    } else {
+        halo_max_latency_msec = priv->halo_max_latency_msec;
+    }
+    gf_msg_debug(this->name, 0, "Using halo latency %" PRId64,
+                 halo_max_latency_msec);
+    return halo_max_latency_msec;
+}
 
-	return 0;
+void
+__afr_handle_child_up_event(xlator_t *this, xlator_t *child_xlator,
+                            const int idx, int64_t child_latency_msec,
+                            int32_t *event, int32_t *call_psh,
+                            int32_t *up_child)
+{
+    afr_private_t *priv = NULL;
+    int up_children = 0;
+    int worst_up_child = -1;
+    int64_t halo_max_latency_msec = afr_get_halo_latency(this);
+
+    priv = this->private;
+
+    /*
+     * This only really counts if the child was never up
+     * (value = -1) or had been down (value = 0).  See
+     * comment at GF_EVENT_CHILD_DOWN for a more detailed
+     * explanation.
+     */
+    if (priv->child_up[idx] != 1) {
+        priv->event_generation++;
+    }
+    priv->child_up[idx] = 1;
+
+    *call_psh = 1;
+    *up_child = idx;
+    up_children = __afr_get_up_children_count(priv);
+    /*
+     * If this is an _actual_ CHILD_UP event, we
+     * want to set the child_latency to MAX to indicate
+     * the child needs ping data to be available before doing child-up
+     */
+    if (!priv->halo_enabled)
+        goto out;
+
+    if (child_latency_msec < 0) {
+        /*set to INT64_MAX-1 so that it is found for best_down_child*/
+        priv->halo_child_up[idx] = 1;
+        if (priv->child_latency[idx] < 0) {
+            priv->child_latency[idx] = AFR_HALO_MAX_LATENCY;
+        }
+    }
+
+    /*
+     * Handle the edge case where we exceed
+     * halo_min_replicas and we've got a child which is
+     * marked up as it was helping to satisfy the
+     * halo_min_replicas even though it's latency exceeds
+     * halo_max_latency_msec.
+     */
+    if (up_children > priv->halo_min_replicas) {
+        worst_up_child = find_worst_up_child(this);
+        if (worst_up_child >= 0 &&
+            priv->child_latency[worst_up_child] > halo_max_latency_msec) {
+            gf_msg_debug(this->name, 0,
+                         "Marking child %d down, "
+                         "doesn't meet halo threshold (%" PRId64
+                         "), and > "
+                         "halo_min_replicas (%d)",
+                         worst_up_child, halo_max_latency_msec,
+                         priv->halo_min_replicas);
+            priv->child_up[worst_up_child] = 0;
+            up_children--;
+        }
+    }
+
+    if (up_children > priv->halo_max_replicas && !priv->shd.iamshd) {
+        worst_up_child = find_worst_up_child(this);
+        if (worst_up_child < 0) {
+            worst_up_child = idx;
+        }
+        priv->child_up[worst_up_child] = 0;
+        up_children--;
+        gf_msg_debug(this->name, 0,
+                     "Marking child %d down, "
+                     "up_children (%d) > halo_max_replicas (%d)",
+                     worst_up_child, up_children, priv->halo_max_replicas);
+    }
 out:
-	AFR_STACK_UNWIND (lk, frame, -1, op_errno, NULL, NULL);
-
-        return 0;
+    if (up_children == 1) {
+        gf_msg(this->name, GF_LOG_INFO, 0, AFR_MSG_SUBVOL_UP,
+               "Subvolume '%s' came back up; "
+               "going online.",
+               child_xlator->name);
+        gf_event(EVENT_AFR_SUBVOL_UP, "client-pid=%d; subvol=%s",
+                 this->ctx->cmd_args.client_pid, this->name);
+    } else {
+        *event = GF_EVENT_SOME_DESCENDENT_UP;
+    }
+
+    priv->last_event[idx] = *event;
 }
 
-int
-afr_forget (xlator_t *this, inode_t *inode)
+void
+__afr_handle_child_down_event(xlator_t *this, xlator_t *child_xlator, int idx,
+                              int64_t child_latency_msec, int32_t *event,
+                              int32_t *call_psh, int32_t *up_child)
 {
-        uint64_t        ctx_int = 0;
-        afr_inode_ctx_t *ctx    = NULL;
+    afr_private_t *priv = NULL;
+    int i = 0;
+    int up_children = 0;
+    int down_children = 0;
+    int best_down_child = -1;
+
+    priv = this->private;
+
+    /*
+     * If a brick is down when we start, we'll get a
+     * CHILD_DOWN to indicate its initial state.  There
+     * was never a CHILD_UP in this case, so if we
+     * increment "down_count" the difference between than
+     * and "up_count" will no longer be the number of
+     * children that are currently up.  This has serious
+     * implications e.g. for quorum enforcement, so we
+     * don't increment these values unless the event
+     * represents an actual state transition between "up"
+     * (value = 1) and anything else.
+     */
+    if (priv->child_up[idx] == 1) {
+        priv->event_generation++;
+    }
+
+    /*
+     * If this is an _actual_ CHILD_DOWN event, we
+     * want to set the child_latency to < 0 to indicate
+     * the child is really disconnected.
+     */
+    if (child_latency_msec < 0) {
+        priv->child_latency[idx] = child_latency_msec;
+        priv->halo_child_up[idx] = 0;
+    }
+    priv->child_up[idx] = 0;
+
+    up_children = __afr_get_up_children_count(priv);
+    /*
+     * Handle the edge case where we need to find the
+     * next best child (to mark up) as marking this child
+     * down would cause us to fall below halo_min_replicas.
+     * We will also force the SHD to heal this child _now_
+     * as we want it to be up to date if we are going to
+     * begin using it synchronously.
+     */
+    if (priv->halo_enabled && up_children < priv->halo_min_replicas) {
+        best_down_child = find_best_down_child(this);
+        if (best_down_child >= 0) {
+            gf_msg_debug(this->name, 0,
+                         "Swapping out child %d for "
+                         "child %d to satisfy halo_min_replicas (%d).",
+                         idx, best_down_child, priv->halo_min_replicas);
+            priv->child_up[best_down_child] = 1;
+            *call_psh = 1;
+            *up_child = best_down_child;
+        }
+    }
+    for (i = 0; i < priv->child_count; i++)
+        if (priv->child_up[i] == 0)
+            down_children++;
+    if (down_children == priv->child_count) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, AFR_MSG_SUBVOLS_DOWN,
+               "All subvolumes are down. Going "
+               "offline until at least one of them "
+               "comes back up.");
+        gf_event(EVENT_AFR_SUBVOLS_DOWN, "client-pid=%d; subvol=%s",
+                 this->ctx->cmd_args.client_pid, this->name);
+    } else {
+        *event = GF_EVENT_SOME_DESCENDENT_DOWN;
+    }
+    priv->last_event[idx] = *event;
+}
 
-        inode_ctx_del (inode, this, &ctx_int);
-        if (!ctx_int)
-                return 0;
+void
+afr_ta_lock_release_synctask(xlator_t *this)
+{
+    call_frame_t *ta_frame = NULL;
+    int ret = 0;
 
-        ctx = (afr_inode_ctx_t *)ctx_int;
-        GF_FREE (ctx);
-        return 0;
+    ta_frame = afr_ta_frame_create(this);
+    if (!ta_frame) {
+        gf_msg(this->name, GF_LOG_ERROR, ENOMEM, AFR_MSG_THIN_ARB,
+               "Failed to create ta_frame");
+        return;
+    }
+
+    ret = synctask_new(this->ctx->env, afr_release_notify_lock_for_ta,
+                       afr_ta_lock_release_done, ta_frame, this);
+    if (ret) {
+        STACK_DESTROY(ta_frame->root);
+        gf_msg(this->name, GF_LOG_ERROR, ENOMEM, AFR_MSG_THIN_ARB,
+               "Failed to release "
+               "AFR_TA_DOM_NOTIFY lock.");
+    }
 }
 
-int
-afr_priv_dump (xlator_t *this)
+static void
+afr_handle_inodelk_contention(xlator_t *this, struct gf_upcall *upcall)
 {
-        afr_private_t *priv = NULL;
-        char  key_prefix[GF_DUMP_MAX_BUF_LEN];
-        char  key[GF_DUMP_MAX_BUF_LEN];
-        int   i = 0;
+    struct gf_upcall_inodelk_contention *lc = NULL;
+    unsigned int inmem_count = 0;
+    unsigned int onwire_count = 0;
+    afr_private_t *priv = this->private;
 
+    lc = upcall->data;
 
-        GF_ASSERT (this);
-        priv = this->private;
+    if (strcmp(lc->domain, AFR_TA_DOM_NOTIFY) != 0)
+        return;
 
-        GF_ASSERT (priv);
-        snprintf(key_prefix, GF_DUMP_MAX_BUF_LEN, "%s.%s", this->type, this->name);
-        gf_proc_dump_add_section(key_prefix);
-        gf_proc_dump_write("child_count", "%u", priv->child_count);
-        for (i = 0; i < priv->child_count; i++) {
-                sprintf (key, "child_up[%d]", i);
-                gf_proc_dump_write(key, "%d", priv->child_up[i]);
-                sprintf (key, "pending_key[%d]", i);
-                gf_proc_dump_write(key, "%s", priv->pending_key[i]);
-        }
-        gf_proc_dump_write("data_self_heal", "%s", priv->data_self_heal);
-        gf_proc_dump_write("metadata_self_heal", "%d", priv->metadata_self_heal);
-        gf_proc_dump_write("entry_self_heal", "%d", priv->entry_self_heal);
-        gf_proc_dump_write("data_change_log", "%d", priv->data_change_log);
-        gf_proc_dump_write("metadata_change_log", "%d", priv->metadata_change_log);
-        gf_proc_dump_write("entry-change_log", "%d", priv->entry_change_log);
-        gf_proc_dump_write("read_child", "%d", priv->read_child);
-        gf_proc_dump_write("favorite_child", "%d", priv->favorite_child);
-        gf_proc_dump_write("wait_count", "%u", priv->wait_count);
-        gf_proc_dump_write("quorum-reads", "%d", priv->quorum_reads);
+    if (priv->shd.iamshd) {
+        /* shd should ignore AFR_TA_DOM_NOTIFY release requests. */
+        return;
+    }
+    LOCK(&priv->lock);
+    {
+        if (priv->release_ta_notify_dom_lock == _gf_true) {
+            /* Ignore multiple release requests from shds.*/
+            UNLOCK(&priv->lock);
+            return;
+        }
+        priv->release_ta_notify_dom_lock = _gf_true;
+        inmem_count = priv->ta_in_mem_txn_count;
+        onwire_count = priv->ta_on_wire_txn_count;
+    }
+    UNLOCK(&priv->lock);
+    if (inmem_count || onwire_count)
+        /* lock release will happen in txn code path after
+         * in-memory or on-wire txns are over.*/
+        return;
 
-        return 0;
+    afr_ta_lock_release_synctask(this);
 }
 
-
-/**
- * find_child_index - find the child's index in the array of subvolumes
- * @this: AFR
- * @child: child
- */
-
-static int
-find_child_index (xlator_t *this, xlator_t *child)
+static void
+afr_handle_upcall_event(xlator_t *this, struct gf_upcall *upcall)
 {
-        afr_private_t *priv = NULL;
-        int i = -1;
+    struct gf_upcall_cache_invalidation *up_ci = NULL;
+    afr_private_t *priv = this->private;
+    inode_t *inode = NULL;
+    inode_table_t *itable = NULL;
+    int i = 0;
+
+    switch (upcall->event_type) {
+        case GF_UPCALL_INODELK_CONTENTION:
+            afr_handle_inodelk_contention(this, upcall);
+            break;
+        case GF_UPCALL_CACHE_INVALIDATION:
+            up_ci = (struct gf_upcall_cache_invalidation *)upcall->data;
+
+            /* Since md-cache will be aggressively filtering
+             * lookups, the stale read issue will be more
+             * pronounced. Hence when a pending xattr is set notify
+             * all the md-cache clients to invalidate the existing
+             * stat cache and send the lookup next time */
+            if (!up_ci->dict)
+                break;
+            for (i = 0; i < priv->child_count; i++) {
+                if (!dict_get(up_ci->dict, priv->pending_key[i]))
+                    continue;
+                up_ci->flags |= UP_INVAL_ATTR;
+                itable = ((xlator_t *)this->graph->top)->itable;
+                /*Internal processes may not have itable for
+                 *top xlator*/
+                if (itable)
+                    inode = inode_find(itable, upcall->gfid);
+                if (inode)
+                    afr_inode_need_refresh_set(inode, this);
+                break;
+            }
+            break;
+        default:
+            break;
+    }
+}
 
-        priv = this->private;
+int32_t
+afr_notify(xlator_t *this, int32_t event, void *data, void *data2)
+{
+    afr_private_t *priv = NULL;
+    xlator_t *child_xlator = NULL;
+    int i = -1;
+    int propagate = 0;
+    int had_heard_from_all = 0;
+    int have_heard_from_all = 0;
+    int idx = -1;
+    int ret = -1;
+    int call_psh = 0;
+    int up_child = -1;
+    dict_t *input = NULL;
+    dict_t *output = NULL;
+    gf_boolean_t had_quorum = _gf_false;
+    gf_boolean_t has_quorum = _gf_false;
+    int64_t halo_max_latency_msec = 0;
+    int64_t child_latency_msec = -1;
+
+    child_xlator = (xlator_t *)data;
+
+    priv = this->private;
+
+    if (!priv)
+        return 0;
 
-        for (i = 0; i < priv->child_count; i++) {
-                if ((xlator_t *) child == priv->children[i])
-                        break;
+    /*
+     * We need to reset this in case children come up in "staggered"
+     * fashion, so that we discover a late-arriving local subvolume.  Note
+     * that we could end up issuing N lookups to the first subvolume, and
+     * O(N^2) overall, but N is small for AFR so it shouldn't be an issue.
+     */
+    priv->did_discovery = _gf_false;
+
+    /* parent xlators don't need to know about every child_up, child_down
+     * because of afr ha. If all subvolumes go down, child_down has
+     * to be triggered. In that state when 1 subvolume comes up child_up
+     * needs to be triggered. dht optimizes revalidate lookup by sending
+     * it only to one of its subvolumes. When child up/down happens
+     * for afr's subvolumes dht should be notified by child_modified. The
+     * subsequent revalidate lookup happens on all the dht's subvolumes
+     * which triggers afr self-heals if any.
+     */
+    idx = afr_find_child_index(this, child_xlator);
+    if (idx < 0) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, AFR_MSG_INVALID_CHILD_UP,
+               "Received child_up from invalid subvolume");
+        goto out;
+    }
+
+    had_quorum = priv->quorum_count &&
+                 afr_has_quorum(priv->child_up, this, NULL);
+    if (event == GF_EVENT_CHILD_PING) {
+        child_latency_msec = (int64_t)(uintptr_t)data2;
+        if (priv->halo_enabled) {
+            halo_max_latency_msec = afr_get_halo_latency(this);
+
+            /* Calculates the child latency and sets event
+             */
+            LOCK(&priv->lock);
+            {
+                __afr_handle_ping_event(this, child_xlator, idx,
+                                        halo_max_latency_msec, &event,
+                                        child_latency_msec);
+            }
+            UNLOCK(&priv->lock);
+        } else {
+            LOCK(&priv->lock);
+            {
+                priv->child_latency[idx] = child_latency_msec;
+            }
+            UNLOCK(&priv->lock);
         }
+    }
 
-        return i;
-}
-
-int32_t
-afr_notify (xlator_t *this, int32_t event,
-            void *data, void *data2)
-{
-        afr_private_t   *priv               = NULL;
-        int             i                   = -1;
-        int             up_children         = 0;
-        int             down_children       = 0;
-        int             propagate           = 0;
-        int             had_heard_from_all  = 0;
-        int             have_heard_from_all = 0;
-        int             idx                 = -1;
-        int             ret                 = -1;
-        int             call_psh            = 0;
-        int             up_child            = -1;
-        dict_t          *input              = NULL;
-        dict_t          *output             = NULL;
-        gf_boolean_t    had_quorum          = _gf_false;
-        gf_boolean_t    has_quorum          = _gf_false;
-
-        priv = this->private;
-
-        if (!priv)
-                return 0;
-
-        /*
-         * We need to reset this in case children come up in "staggered"
-         * fashion, so that we discover a late-arriving local subvolume.  Note
-         * that we could end up issuing N lookups to the first subvolume, and
-         * O(N^2) overall, but N is small for AFR so it shouldn't be an issue.
+    if (event == GF_EVENT_CHILD_PING) {
+        /* This is the only xlator that handles PING, no reason to
+         * propagate.
          */
-        priv->did_discovery = _gf_false;
+        goto out;
+    }
 
-        had_heard_from_all = 1;
-        for (i = 0; i < priv->child_count; i++) {
-                if (!priv->last_event[i]) {
-                        had_heard_from_all = 0;
-                }
+    if (event == GF_EVENT_TRANSLATOR_OP) {
+        LOCK(&priv->lock);
+        {
+            had_heard_from_all = __get_heard_from_all_status(this);
         }
+        UNLOCK(&priv->lock);
 
-        /* parent xlators dont need to know about every child_up, child_down
-         * because of afr ha. If all subvolumes go down, child_down has
-         * to be triggered. In that state when 1 subvolume comes up child_up
-         * needs to be triggered. dht optimizes revalidate lookup by sending
-         * it only to one of its subvolumes. When child up/down happens
-         * for afr's subvolumes dht should be notified by child_modified. The
-         * subsequent revalidate lookup happens on all the dht's subvolumes
-         * which triggers afr self-heals if any.
-         */
-        idx = find_child_index (this, data);
-        if (idx < 0) {
-                gf_msg (this->name, GF_LOG_ERROR, 0, AFR_MSG_INVALID_CHILD_UP,
-                        "Received child_up from invalid subvolume");
-                goto out;
+        if (!had_heard_from_all) {
+            ret = -1;
+        } else {
+            input = data;
+            output = data2;
+            ret = afr_xl_op(this, input, output);
         }
+        goto out;
+    }
+
+    if (event == GF_EVENT_UPCALL) {
+        afr_handle_upcall_event(this, data);
+    }
 
-        had_quorum = priv->quorum_count && afr_has_quorum (priv->child_up,
-                                                           this);
+    LOCK(&priv->lock);
+    {
+        had_heard_from_all = __get_heard_from_all_status(this);
         switch (event) {
-        case GF_EVENT_CHILD_UP:
-                LOCK (&priv->lock);
-                {
-                        /*
-                         * This only really counts if the child was never up
-                         * (value = -1) or had been down (value = 0).  See
-                         * comment at GF_EVENT_CHILD_DOWN for a more detailed
-                         * explanation.
-                         */
-                        if (priv->child_up[idx] != 1) {
-                                priv->up_count++;
-				priv->event_generation++;
-                        }
-                        priv->child_up[idx] = 1;
-
-                        call_psh = 1;
-                        up_child = idx;
-                        for (i = 0; i < priv->child_count; i++)
-                                if (priv->child_up[i] == 1)
-                                        up_children++;
-                        if (up_children == 1) {
-                                gf_msg (this->name, GF_LOG_INFO, 0,
-                                        AFR_MSG_SUBVOL_UP,
-                                        "Subvolume '%s' came back up; "
-                                        "going online.", ((xlator_t *)data)->name);
-                        } else {
-                                event = GF_EVENT_CHILD_MODIFIED;
-                        }
-
-                        priv->last_event[idx] = event;
+            case GF_EVENT_PARENT_UP:
+                __afr_launch_notify_timer(this, priv);
+                propagate = 1;
+                break;
+            case GF_EVENT_CHILD_UP:
+                if (priv->thin_arbiter_count &&
+                    (idx == AFR_CHILD_THIN_ARBITER)) {
+                    priv->ta_child_up = 1;
+                    priv->ta_event_gen++;
+                    break;
                 }
-                UNLOCK (&priv->lock);
-
+                __afr_handle_child_up_event(this, child_xlator, idx,
+                                            child_latency_msec, &event,
+                                            &call_psh, &up_child);
+                __afr_lock_heal_synctask(this, priv, idx);
                 break;
 
-        case GF_EVENT_CHILD_DOWN:
-                LOCK (&priv->lock);
-                {
-                        /*
-                         * If a brick is down when we start, we'll get a
-                         * CHILD_DOWN to indicate its initial state.  There
-                         * was never a CHILD_UP in this case, so if we
-                         * increment "down_count" the difference between than
-                         * and "up_count" will no longer be the number of
-                         * children that are currently up.  This has serious
-                         * implications e.g. for quorum enforcement, so we
-                         * don't increment these values unless the event
-                         * represents an actual state transition between "up"
-                         * (value = 1) and anything else.
-                         */
-                        if (priv->child_up[idx] == 1) {
-                                priv->down_count++;
-				priv->event_generation++;
-                        }
-                        priv->child_up[idx] = 0;
-
-                        for (i = 0; i < priv->child_count; i++)
-                                if (priv->child_up[i] == 0)
-                                        down_children++;
-                        if (down_children == priv->child_count) {
-                                gf_msg (this->name, GF_LOG_ERROR, 0,
-                                        AFR_MSG_ALL_SUBVOLS_DOWN,
-                                        "All subvolumes are down. Going offline "
-                                        "until atleast one of them comes back up.");
-                        } else {
-                                event = GF_EVENT_CHILD_MODIFIED;
-                        }
-
-                        priv->last_event[idx] = event;
+            case GF_EVENT_CHILD_DOWN:
+                if (priv->thin_arbiter_count &&
+                    (idx == AFR_CHILD_THIN_ARBITER)) {
+                    priv->ta_child_up = 0;
+                    priv->ta_event_gen++;
+                    afr_ta_locked_priv_invalidate(priv);
+                    break;
                 }
-                UNLOCK (&priv->lock);
-
+                __afr_handle_child_down_event(this, child_xlator, idx,
+                                              child_latency_msec, &event,
+                                              &call_psh, &up_child);
+                __afr_mark_pending_lk_heal(this, priv, idx);
                 break;
 
-        case GF_EVENT_CHILD_CONNECTING:
-                LOCK (&priv->lock);
-                {
-                        priv->last_event[idx] = event;
-                }
-                UNLOCK (&priv->lock);
+            case GF_EVENT_CHILD_CONNECTING:
+                priv->last_event[idx] = event;
 
                 break;
 
-        case GF_EVENT_TRANSLATOR_OP:
-                input = data;
-                output = data2;
-                if (!had_heard_from_all) {
-                        ret = -1;
-                        goto out;
-                }
-                ret = afr_xl_op (this, input, output);
-                goto out;
+            case GF_EVENT_SOME_DESCENDENT_DOWN:
+                priv->last_event[idx] = event;
                 break;
-
-        default:
+            default:
                 propagate = 1;
                 break;
         }
+        have_heard_from_all = __get_heard_from_all_status(this);
+        if (!had_heard_from_all && have_heard_from_all) {
+            if (priv->timer) {
+                gf_timer_call_cancel(this->ctx, priv->timer);
+                priv->timer = NULL;
+            }
+            /* This is the first event which completes aggregation
+               of events from all subvolumes. If at least one subvol
+               had come up, propagate CHILD_UP, but only this time
+            */
+            event = GF_EVENT_CHILD_DOWN;
+            for (i = 0; i < priv->child_count; i++) {
+                if (priv->last_event[i] == GF_EVENT_CHILD_UP) {
+                    event = GF_EVENT_CHILD_UP;
+                    break;
+                }
 
-        if (priv->quorum_count) {
-                has_quorum = afr_has_quorum (priv->child_up, this);
-                if (!had_quorum && has_quorum)
-                        gf_msg (this->name, GF_LOG_INFO, 0, AFR_MSG_QUORUM_MET,
-                                "Client-quorum is met");
-                if (had_quorum && !has_quorum)
-                        gf_msg (this->name, GF_LOG_WARNING, 0,
-                                AFR_MSG_QUORUM_FAIL,
-                                "Client-quorum is not met");
+                if (priv->last_event[i] == GF_EVENT_CHILD_CONNECTING) {
+                    event = GF_EVENT_CHILD_CONNECTING;
+                    /* continue to check other events for CHILD_UP */
+                }
+            }
+        }
+    }
+    UNLOCK(&priv->lock);
+
+    if (priv->quorum_count) {
+        has_quorum = afr_has_quorum(priv->child_up, this, NULL);
+        if (!had_quorum && has_quorum) {
+            gf_msg(this->name, GF_LOG_INFO, 0, AFR_MSG_QUORUM_MET,
+                   "Client-quorum is met");
+            gf_event(EVENT_AFR_QUORUM_MET, "client-pid=%d; subvol=%s",
+                     this->ctx->cmd_args.client_pid, this->name);
+        }
+        if (had_quorum && !has_quorum) {
+            gf_msg(this->name, GF_LOG_WARNING, 0, AFR_MSG_QUORUM_FAIL,
+                   "Client-quorum is not met");
+            gf_event(EVENT_AFR_QUORUM_FAIL, "client-pid=%d; subvol=%s",
+                     this->ctx->cmd_args.client_pid, this->name);
+        }
+    }
+
+    /* if all subvols have reported status, no need to hide anything
+       or wait for anything else. Just propagate blindly */
+    if (have_heard_from_all)
+        propagate = 1;
+
+    ret = 0;
+    if (propagate)
+        ret = default_notify(this, event, data);
+
+    if ((!had_heard_from_all) || call_psh) {
+        /* Launch self-heal on all local subvolumes if:
+         * a) We have_heard_from_all for the first time
+         * b) Already heard from everyone, but we now got a child-up
+         *    event.
+         */
+        if (have_heard_from_all) {
+            afr_selfheal_childup(this, priv);
         }
+    }
+out:
+    return ret;
+}
 
-        /* have all subvolumes reported status once by now? */
-        have_heard_from_all = 1;
-        for (i = 0; i < priv->child_count; i++) {
-                if (!priv->last_event[i])
-                        have_heard_from_all = 0;
-        }
+int
+afr_local_init(afr_local_t *local, afr_private_t *priv, int32_t *op_errno)
+{
+    int __ret = -1;
+    local->op_ret = -1;
+    local->op_errno = EUCLEAN;
+
+    __ret = syncbarrier_init(&local->barrier);
+    if (__ret) {
+        if (op_errno)
+            *op_errno = __ret;
+        goto out;
+    }
+
+    local->child_up = GF_MALLOC(priv->child_count * sizeof(*local->child_up),
+                                gf_afr_mt_char);
+    if (!local->child_up) {
+        if (op_errno)
+            *op_errno = ENOMEM;
+        goto out;
+    }
+
+    memcpy(local->child_up, priv->child_up,
+           sizeof(*local->child_up) * priv->child_count);
+    local->call_count = AFR_COUNT(local->child_up, priv->child_count);
+    if (local->call_count == 0) {
+        gf_msg(THIS->name, GF_LOG_INFO, 0, AFR_MSG_SUBVOLS_DOWN,
+               "no subvolumes up");
+        if (op_errno)
+            *op_errno = ENOTCONN;
+        goto out;
+    }
+
+    local->event_generation = priv->event_generation;
+
+    local->read_attempted = GF_CALLOC(priv->child_count, sizeof(char),
+                                      gf_afr_mt_char);
+    if (!local->read_attempted) {
+        if (op_errno)
+            *op_errno = ENOMEM;
+        goto out;
+    }
+
+    local->readable = GF_CALLOC(priv->child_count, sizeof(char),
+                                gf_afr_mt_char);
+    if (!local->readable) {
+        if (op_errno)
+            *op_errno = ENOMEM;
+        goto out;
+    }
+
+    local->readable2 = GF_CALLOC(priv->child_count, sizeof(char),
+                                 gf_afr_mt_char);
+    if (!local->readable2) {
+        if (op_errno)
+            *op_errno = ENOMEM;
+        goto out;
+    }
+
+    local->read_subvol = -1;
+
+    local->replies = GF_CALLOC(priv->child_count, sizeof(*local->replies),
+                               gf_afr_mt_reply_t);
+    if (!local->replies) {
+        if (op_errno)
+            *op_errno = ENOMEM;
+        goto out;
+    }
+
+    local->need_full_crawl = _gf_false;
+    if (priv->thin_arbiter_count) {
+        local->ta_child_up = priv->ta_child_up;
+        local->ta_failed_subvol = AFR_CHILD_UNKNOWN;
+        local->read_txn_query_child = AFR_CHILD_UNKNOWN;
+        local->ta_event_gen = priv->ta_event_gen;
+        local->fop_state = TA_SUCCESS;
+    }
+    local->is_new_entry = _gf_false;
+
+    INIT_LIST_HEAD(&local->healer);
+    return 0;
+out:
+    return -1;
+}
 
-        /* if all subvols have reported status, no need to hide anything
-           or wait for anything else. Just propagate blindly */
-        if (have_heard_from_all)
-                propagate = 1;
+int
+afr_internal_lock_init(afr_internal_lock_t *lk, size_t child_count)
+{
+    int ret = -ENOMEM;
 
-        if (!had_heard_from_all && have_heard_from_all) {
-                /* This is the first event which completes aggregation
-                   of events from all subvolumes. If at least one subvol
-                   had come up, propagate CHILD_UP, but only this time
-                */
-                event = GF_EVENT_CHILD_DOWN;
-
-                LOCK (&priv->lock);
-                {
-                        up_children = AFR_COUNT (priv->child_up, priv->child_count);
-                        for (i = 0; i < priv->child_count; i++) {
-                                if (priv->last_event[i] == GF_EVENT_CHILD_UP) {
-                                        event = GF_EVENT_CHILD_UP;
-                                        break;
-                                }
-
-                                if (priv->last_event[i] ==
-                                                GF_EVENT_CHILD_CONNECTING) {
-                                        event = GF_EVENT_CHILD_CONNECTING;
-                                        /* continue to check other events for CHILD_UP */
-                                }
-                        }
-                }
-                UNLOCK (&priv->lock);
-        }
+    lk->lower_locked_nodes = GF_CALLOC(sizeof(*lk->lower_locked_nodes),
+                                       child_count, gf_afr_mt_char);
+    if (NULL == lk->lower_locked_nodes)
+        goto out;
 
-        ret = 0;
-        if (propagate)
-                ret = default_notify (this, event, data);
+    lk->lock_op_ret = -1;
+    lk->lock_op_errno = EUCLEAN;
 
-        if (!had_heard_from_all && have_heard_from_all && priv->shd.iamshd) {
-                /*
-                 * Since self-heal is supposed to be launched only after
-                 * the responses from all the bricks are collected,
-                 * launch self-heals now on all up subvols.
-                 */
-                for (i = 0; i < priv->child_count; i++)
-                        if (priv->child_up[i])
-                                afr_selfheal_childup (this, i);
-        } else if (have_heard_from_all && call_psh && priv->shd.iamshd) {
-                /*
-                 * Already heard from everyone. Just launch heal on now up
-                 * subvolume.
-                 */
-                 afr_selfheal_childup (this, up_child);
-	}
+    ret = 0;
 out:
-        return ret;
+    return ret;
 }
 
-
-int
-afr_local_init (afr_local_t *local, afr_private_t *priv, int32_t *op_errno)
+void
+afr_matrix_cleanup(int32_t **matrix, unsigned int m)
 {
-        local->op_ret = -1;
-        local->op_errno = EUCLEAN;
+    int i = 0;
 
-	syncbarrier_init (&local->barrier);
+    if (!matrix)
+        goto out;
+    for (i = 0; i < m; i++) {
+        GF_FREE(matrix[i]);
+    }
 
-        local->child_up = GF_CALLOC (priv->child_count,
-                                     sizeof (*local->child_up),
-                                     gf_afr_mt_char);
-        if (!local->child_up) {
-                if (op_errno)
-                        *op_errno = ENOMEM;
-                goto out;
-        }
+    GF_FREE(matrix);
+out:
+    return;
+}
 
-        memcpy (local->child_up, priv->child_up,
-                sizeof (*local->child_up) * priv->child_count);
-        local->call_count = AFR_COUNT (local->child_up, priv->child_count);
-        if (local->call_count == 0) {
-                gf_log (THIS->name, GF_LOG_INFO, "no subvolumes up");
-                if (op_errno)
-                        *op_errno = ENOTCONN;
-                goto out;
-        }
-	local->event_generation = priv->event_generation;
-
-	local->read_attempted = GF_CALLOC (priv->child_count, sizeof (char),
-					   gf_afr_mt_char);
-	if (!local->read_attempted) {
-		if (op_errno)
-			*op_errno = ENOMEM;
-		goto out;
-	}
-
-	local->readable = GF_CALLOC (priv->child_count, sizeof (char),
-				     gf_afr_mt_char);
-	if (!local->readable) {
-		if (op_errno)
-			*op_errno = ENOMEM;
-		goto out;
-	}
-
-	local->replies = GF_CALLOC(priv->child_count, sizeof(*local->replies),
-				   gf_afr_mt_reply_t);
-	if (!local->replies) {
-		if (op_errno)
-			*op_errno = ENOMEM;
-		goto out;
-	}
-
-	return 0;
+int32_t **
+afr_matrix_create(unsigned int m, unsigned int n)
+{
+    int32_t **matrix = NULL;
+    int i = 0;
+
+    matrix = GF_CALLOC(sizeof(*matrix), m, gf_afr_mt_int32_t);
+    if (!matrix)
+        goto out;
+
+    for (i = 0; i < m; i++) {
+        matrix[i] = GF_CALLOC(sizeof(*matrix[i]), n, gf_afr_mt_int32_t);
+        if (!matrix[i])
+            goto out;
+    }
+    return matrix;
 out:
-        return -1;
+    afr_matrix_cleanup(matrix, m);
+    return NULL;
 }
 
 int
-afr_internal_lock_init (afr_internal_lock_t *lk, size_t child_count,
-                        transaction_lk_type_t lk_type)
+afr_transaction_local_init(afr_local_t *local, xlator_t *this)
 {
-        int             ret = -ENOMEM;
+    int ret = -ENOMEM;
+    afr_private_t *priv = NULL;
+
+    priv = this->private;
+    INIT_LIST_HEAD(&local->transaction.wait_list);
+    INIT_LIST_HEAD(&local->transaction.owner_list);
+    INIT_LIST_HEAD(&local->ta_waitq);
+    INIT_LIST_HEAD(&local->ta_onwireq);
+    ret = afr_internal_lock_init(&local->internal_lock, priv->child_count);
+    if (ret < 0)
+        goto out;
+
+    ret = -ENOMEM;
+    local->pre_op_compat = priv->pre_op_compat;
+
+    local->transaction.pre_op = GF_CALLOC(sizeof(*local->transaction.pre_op),
+                                          priv->child_count, gf_afr_mt_char);
+    if (!local->transaction.pre_op)
+        goto out;
+
+    local->transaction.changelog_xdata = GF_CALLOC(
+        sizeof(*local->transaction.changelog_xdata), priv->child_count,
+        gf_afr_mt_dict_t);
+    if (!local->transaction.changelog_xdata)
+        goto out;
+
+    if (priv->arbiter_count == 1) {
+        local->transaction.pre_op_sources = GF_CALLOC(
+            sizeof(*local->transaction.pre_op_sources), priv->child_count,
+            gf_afr_mt_char);
+        if (!local->transaction.pre_op_sources)
+            goto out;
+    }
+
+    local->transaction.failed_subvols = GF_CALLOC(
+        sizeof(*local->transaction.failed_subvols), priv->child_count,
+        gf_afr_mt_char);
+    if (!local->transaction.failed_subvols)
+        goto out;
+
+    local->pending = afr_matrix_create(priv->child_count, AFR_NUM_CHANGE_LOGS);
+    if (!local->pending)
+        goto out;
+
+    ret = 0;
+out:
+    return ret;
+}
 
-        lk->locked_nodes = GF_CALLOC (sizeof (*lk->locked_nodes),
-                                      child_count, gf_afr_mt_char);
-        if (NULL == lk->locked_nodes)
-                goto out;
+void
+afr_set_low_priority(call_frame_t *frame)
+{
+    frame->root->pid = LOW_PRIO_PROC_PID;
+}
 
-        lk->lower_locked_nodes = GF_CALLOC (sizeof (*lk->lower_locked_nodes),
-                                            child_count, gf_afr_mt_char);
-        if (NULL == lk->lower_locked_nodes)
-                goto out;
+void
+afr_priv_destroy(afr_private_t *priv)
+{
+    int i = 0;
+    int child_count = -1;
+
+    if (!priv)
+        goto out;
+
+    GF_FREE(priv->sh_domain);
+    GF_FREE(priv->last_event);
+
+    child_count = priv->child_count;
+    if (priv->thin_arbiter_count) {
+        child_count++;
+    }
+    if (priv->pending_key) {
+        for (i = 0; i < child_count; i++)
+            GF_FREE(priv->pending_key[i]);
+    }
+
+    GF_FREE(priv->pending_reads);
+    GF_FREE(priv->local);
+    GF_FREE(priv->pending_key);
+    GF_FREE(priv->children);
+    GF_FREE(priv->anon_inode);
+    GF_FREE(priv->child_up);
+    GF_FREE(priv->halo_child_up);
+    GF_FREE(priv->child_latency);
+    LOCK_DESTROY(&priv->lock);
+
+    GF_FREE(priv);
+out:
+    return;
+}
 
-        lk->lock_op_ret   = -1;
-        lk->lock_op_errno = EUCLEAN;
-        lk->transaction_lk_type = lk_type;
+int **
+afr_mark_pending_changelog(afr_private_t *priv, unsigned char *pending,
+                           dict_t *xattr, ia_type_t iat)
+{
+    int i = 0;
+    int **changelog = NULL;
+    int idx = -1;
+    int m_idx = 0;
+    int d_idx = 0;
+    int ret = 0;
+
+    m_idx = afr_index_for_transaction_type(AFR_METADATA_TRANSACTION);
+    d_idx = afr_index_for_transaction_type(AFR_DATA_TRANSACTION);
+
+    idx = afr_index_from_ia_type(iat);
+
+    changelog = afr_matrix_create(priv->child_count, AFR_NUM_CHANGE_LOGS);
+    if (!changelog)
+        goto out;
+
+    for (i = 0; i < priv->child_count; i++) {
+        if (!pending[i])
+            continue;
+
+        changelog[i][m_idx] = hton32(1);
+        if (idx != -1)
+            changelog[i][idx] = hton32(1);
+        /* If the newentry marking is on a newly created directory,
+         * then mark it with the full-heal indicator.
+         */
+        if ((IA_ISDIR(iat)) && (priv->esh_granular))
+            changelog[i][d_idx] = hton32(1);
+    }
+    ret = afr_set_pending_dict(priv, xattr, changelog);
+    if (ret < 0) {
+        afr_matrix_cleanup(changelog, priv->child_count);
+        return NULL;
+    }
+out:
+    return changelog;
+}
 
-        ret = 0;
+static dict_t *
+afr_set_heal_info(char *status)
+{
+    dict_t *dict = NULL;
+    int ret = -1;
+
+    dict = dict_new();
+    if (!dict) {
+        ret = -ENOMEM;
+        goto out;
+    }
+
+    ret = dict_set_dynstr_sizen(dict, "heal-info", status);
+    if (ret)
+        gf_msg("", GF_LOG_WARNING, -ret, AFR_MSG_DICT_SET_FAILED,
+               "Failed to set heal-info key to "
+               "%s",
+               status);
 out:
-        return ret;
+    /* Any error other than EINVAL, dict_set_dynstr frees status */
+    if (ret == -ENOMEM || ret == -EINVAL) {
+        GF_FREE(status);
+    }
+
+    if (ret && dict) {
+        dict_unref(dict);
+        dict = NULL;
+    }
+    return dict;
 }
 
-void
-afr_matrix_cleanup (int32_t **matrix, unsigned int m)
+static gf_boolean_t
+afr_is_dirty_count_non_unary_for_txn(xlator_t *this, struct afr_reply *replies,
+                                     afr_transaction_type type)
 {
-        int             i         = 0;
+    afr_private_t *priv = this->private;
+    int *dirty = alloca0(priv->child_count * sizeof(int));
+    int i = 0;
 
-        if (!matrix)
-                goto out;
-        for (i = 0; i < m; i++) {
-                GF_FREE (matrix[i]);
-        }
+    afr_selfheal_extract_xattr(this, replies, type, dirty, NULL);
+    for (i = 0; i < priv->child_count; i++) {
+        if (dirty[i] > 1)
+            return _gf_true;
+    }
 
-        GF_FREE (matrix);
-out:
-        return;
+    return _gf_false;
 }
 
-int32_t**
-afr_matrix_create (unsigned int m, unsigned int n)
+static gf_boolean_t
+afr_is_dirty_count_non_unary(xlator_t *this, struct afr_reply *replies,
+                             ia_type_t ia_type)
 {
-        int32_t         **matrix = NULL;
-        int             i       = 0;
+    gf_boolean_t data_chk = _gf_false;
+    gf_boolean_t mdata_chk = _gf_false;
+    gf_boolean_t entry_chk = _gf_false;
+
+    switch (ia_type) {
+        case IA_IFDIR:
+            mdata_chk = _gf_true;
+            entry_chk = _gf_true;
+            break;
+        case IA_IFREG:
+            mdata_chk = _gf_true;
+            data_chk = _gf_true;
+            break;
+        default:
+            /*IA_IFBLK, IA_IFCHR, IA_IFLNK, IA_IFIFO, IA_IFSOCK*/
+            mdata_chk = _gf_true;
+            break;
+    }
 
-        matrix = GF_CALLOC (sizeof (*matrix), m, gf_afr_mt_int32_t);
-        if (!matrix)
-                goto out;
+    if (data_chk && afr_is_dirty_count_non_unary_for_txn(
+                        this, replies, AFR_DATA_TRANSACTION)) {
+        return _gf_true;
+    } else if (mdata_chk && afr_is_dirty_count_non_unary_for_txn(
+                                this, replies, AFR_METADATA_TRANSACTION)) {
+        return _gf_true;
+    } else if (entry_chk && afr_is_dirty_count_non_unary_for_txn(
+                                this, replies, AFR_ENTRY_TRANSACTION)) {
+        return _gf_true;
+    }
+
+    return _gf_false;
+}
 
-        for (i = 0; i < m; i++) {
-                matrix[i] = GF_CALLOC (sizeof (*matrix[i]), n,
-                                       gf_afr_mt_int32_t);
-                if (!matrix[i])
-                        goto out;
+static int
+afr_update_heal_status(xlator_t *this, struct afr_reply *replies,
+                       ia_type_t ia_type, gf_boolean_t *esh, gf_boolean_t *dsh,
+                       gf_boolean_t *msh, unsigned char pending)
+{
+    int ret = -1;
+    GF_UNUSED int ret1 = 0;
+    int i = 0;
+    int io_domain_lk_count = 0;
+    int shd_domain_lk_count = 0;
+    afr_private_t *priv = NULL;
+    char *key1 = NULL;
+    char *key2 = NULL;
+
+    priv = this->private;
+    key1 = alloca0(strlen(GLUSTERFS_INODELK_DOM_PREFIX) + 2 +
+                   strlen(this->name));
+    key2 = alloca0(strlen(GLUSTERFS_INODELK_DOM_PREFIX) + 2 +
+                   strlen(priv->sh_domain));
+    sprintf(key1, "%s:%s", GLUSTERFS_INODELK_DOM_PREFIX, this->name);
+    sprintf(key2, "%s:%s", GLUSTERFS_INODELK_DOM_PREFIX, priv->sh_domain);
+
+    for (i = 0; i < priv->child_count; i++) {
+        if ((replies[i].valid != 1) || (replies[i].op_ret != 0))
+            continue;
+        if (!io_domain_lk_count) {
+            ret1 = dict_get_int32(replies[i].xdata, key1, &io_domain_lk_count);
+        }
+        if (!shd_domain_lk_count) {
+            ret1 = dict_get_int32(replies[i].xdata, key2, &shd_domain_lk_count);
+        }
+    }
+
+    if (!pending) {
+        if ((afr_is_dirty_count_non_unary(this, replies, ia_type)) ||
+            (!io_domain_lk_count)) {
+            /* Needs heal. */
+            ret = 0;
+        } else {
+            /* No heal needed. */
+            *dsh = *esh = *msh = 0;
         }
-        return matrix;
-out:
-        afr_matrix_cleanup (matrix, m);
-        return NULL;
+    } else {
+        if (shd_domain_lk_count) {
+            ret = -EAGAIN; /*For 'possibly-healing'. */
+        } else {
+            ret = 0; /*needs heal. Just set a non -ve value so that it is
+                       assumed as the source index.*/
+        }
+    }
+    return ret;
 }
 
+/*return EIO, EAGAIN or pending*/
 int
-afr_inodelk_init (afr_inodelk_t *lk, char *dom, size_t child_count)
+afr_lockless_inspect(call_frame_t *frame, xlator_t *this, uuid_t gfid,
+                     inode_t **inode, gf_boolean_t *entry_selfheal,
+                     gf_boolean_t *data_selfheal,
+                     gf_boolean_t *metadata_selfheal, unsigned char *pending)
 {
-        int             ret = -ENOMEM;
+    int ret = -1;
+    int i = 0;
+    afr_private_t *priv = NULL;
+    struct afr_reply *replies = NULL;
+    gf_boolean_t dsh = _gf_false;
+    gf_boolean_t msh = _gf_false;
+    gf_boolean_t esh = _gf_false;
+    unsigned char *sources = NULL;
+    unsigned char *sinks = NULL;
+    unsigned char *valid_on = NULL;
+    uint64_t *witness = NULL;
+
+    priv = this->private;
+    replies = alloca0(sizeof(*replies) * priv->child_count);
+    sources = alloca0(sizeof(*sources) * priv->child_count);
+    sinks = alloca0(sizeof(*sinks) * priv->child_count);
+    witness = alloca0(sizeof(*witness) * priv->child_count);
+    valid_on = alloca0(sizeof(*valid_on) * priv->child_count);
+
+    ret = afr_selfheal_unlocked_inspect(frame, this, gfid, inode, &dsh, &msh,
+                                        &esh, replies);
+    if (ret)
+        goto out;
+    for (i = 0; i < priv->child_count; i++) {
+        if (replies[i].valid && replies[i].op_ret == 0) {
+            valid_on[i] = 1;
+        }
+    }
+    if (msh) {
+        ret = afr_selfheal_find_direction(frame, this, replies,
+                                          AFR_METADATA_TRANSACTION, valid_on,
+                                          sources, sinks, witness, pending);
+        if (*pending & PFLAG_SBRAIN)
+            ret = -EIO;
+        if (ret)
+            goto out;
+    }
+    if (dsh) {
+        ret = afr_selfheal_find_direction(frame, this, replies,
+                                          AFR_DATA_TRANSACTION, valid_on,
+                                          sources, sinks, witness, pending);
+        if (*pending & PFLAG_SBRAIN)
+            ret = -EIO;
+        if (ret)
+            goto out;
+    }
+    if (esh) {
+        ret = afr_selfheal_find_direction(frame, this, replies,
+                                          AFR_ENTRY_TRANSACTION, valid_on,
+                                          sources, sinks, witness, pending);
+        if (*pending & PFLAG_SBRAIN)
+            ret = -EIO;
+        if (ret)
+            goto out;
+    }
 
-        lk->domain = dom;
-        lk->locked_nodes = GF_CALLOC (sizeof (*lk->locked_nodes),
-                                      child_count, gf_afr_mt_char);
-        if (NULL == lk->locked_nodes)
-                goto out;
-        ret = 0;
+    ret = afr_update_heal_status(this, replies, (*inode)->ia_type, &esh, &dsh,
+                                 &msh, *pending);
 out:
-        return ret;
+    *data_selfheal = dsh;
+    *entry_selfheal = esh;
+    *metadata_selfheal = msh;
+    if (replies)
+        afr_replies_wipe(replies, priv->child_count);
+    return ret;
 }
 
 int
-afr_transaction_local_init (afr_local_t *local, xlator_t *this)
+afr_get_heal_info(call_frame_t *frame, xlator_t *this, loc_t *loc)
 {
-        int            child_up_count = 0;
-        int            ret = -ENOMEM;
-        afr_private_t *priv = NULL;
-
-        priv = this->private;
-        ret = afr_internal_lock_init (&local->internal_lock, priv->child_count,
-                                      AFR_TRANSACTION_LK);
-        if (ret < 0)
+    gf_boolean_t data_selfheal = _gf_false;
+    gf_boolean_t metadata_selfheal = _gf_false;
+    gf_boolean_t entry_selfheal = _gf_false;
+    unsigned char pending = 0;
+    dict_t *dict = NULL;
+    int ret = -1;
+    int op_errno = ENOMEM;
+    inode_t *inode = NULL;
+    char *substr = NULL;
+    char *status = NULL;
+    call_frame_t *heal_frame = NULL;
+    afr_local_t *heal_local = NULL;
+
+    /*Use frame with lk-owner set*/
+    heal_frame = afr_frame_create(frame->this, &op_errno);
+    if (!heal_frame) {
+        ret = -1;
+        goto out;
+    }
+    heal_local = heal_frame->local;
+    heal_frame->local = frame->local;
+
+    ret = afr_lockless_inspect(heal_frame, this, loc->gfid, &inode,
+                               &entry_selfheal, &data_selfheal,
+                               &metadata_selfheal, &pending);
+
+    if (ret == -ENOMEM) {
+        ret = -1;
+        goto out;
+    }
+
+    if (pending & PFLAG_PENDING) {
+        gf_asprintf(&substr, "-pending");
+        if (!substr)
+            goto out;
+    }
+
+    if (ret == -EIO) {
+        ret = gf_asprintf(&status, "split-brain%s", substr ? substr : "");
+        if (ret < 0) {
+            goto out;
+        }
+        dict = afr_set_heal_info(status);
+        if (!dict) {
+            ret = -1;
+            goto out;
+        }
+    } else if (ret == -EAGAIN) {
+        ret = gf_asprintf(&status, "possibly-healing%s", substr ? substr : "");
+        if (ret < 0) {
+            goto out;
+        }
+        dict = afr_set_heal_info(status);
+        if (!dict) {
+            ret = -1;
+            goto out;
+        }
+    } else if (ret >= 0) {
+        /* value of ret = source index
+         * so ret >= 0 and at least one of the 3 booleans set to
+         * true means a source is identified; heal is required.
+         */
+        if (!data_selfheal && !entry_selfheal && !metadata_selfheal) {
+            status = gf_strdup("no-heal");
+            if (!status) {
+                ret = -1;
                 goto out;
-
-        if ((local->transaction.type == AFR_DATA_TRANSACTION) ||
-            (local->transaction.type == AFR_METADATA_TRANSACTION)) {
-                ret = afr_inodelk_init (&local->internal_lock.inodelk[0],
-                                        this->name, priv->child_count);
-                if (ret < 0)
-                        goto out;
+            }
+            dict = afr_set_heal_info(status);
+            if (!dict) {
+                ret = -1;
+                goto out;
+            }
+        } else {
+            ret = gf_asprintf(&status, "heal%s", substr ? substr : "");
+            if (ret < 0) {
+                goto out;
+            }
+            dict = afr_set_heal_info(status);
+            if (!dict) {
+                ret = -1;
+                goto out;
+            }
+        }
+    } else if (ret < 0) {
+        /* Apart from above checked -ve ret values, there are
+         * other possible ret values like ENOTCONN
+         * (returned when number of valid replies received are
+         * less than 2)
+         * in which case heal is required when one of the
+         * selfheal booleans is set.
+         */
+        if (data_selfheal || entry_selfheal || metadata_selfheal) {
+            ret = gf_asprintf(&status, "heal%s", substr ? substr : "");
+            if (ret < 0) {
+                goto out;
+            }
+            dict = afr_set_heal_info(status);
+            if (!dict) {
+                ret = -1;
+                goto out;
+            }
         }
+    }
 
-        ret = -ENOMEM;
-        child_up_count = AFR_COUNT (local->child_up, priv->child_count);
-        if (priv->optimistic_change_log && child_up_count == priv->child_count)
-                local->optimistic_change_log = 1;
+    ret = 0;
+    op_errno = 0;
+
+out:
+    if (heal_frame) {
+        heal_frame->local = heal_local;
+        AFR_STACK_DESTROY(heal_frame);
+    }
+    AFR_STACK_UNWIND(getxattr, frame, ret, op_errno, dict, NULL);
+    if (dict)
+        dict_unref(dict);
+    if (inode)
+        inode_unref(inode);
+    GF_FREE(substr);
+    return ret;
+}
 
-	local->pre_op_compat = priv->pre_op_compat;
+int
+_afr_is_split_brain(call_frame_t *frame, xlator_t *this,
+                    struct afr_reply *replies, afr_transaction_type type,
+                    gf_boolean_t *spb)
+{
+    afr_private_t *priv = NULL;
+    uint64_t *witness = NULL;
+    unsigned char *sources = NULL;
+    unsigned char *sinks = NULL;
+    int sources_count = 0;
+    int ret = 0;
+
+    priv = this->private;
+
+    sources = alloca0(priv->child_count);
+    sinks = alloca0(priv->child_count);
+    witness = alloca0(priv->child_count * sizeof(*witness));
+
+    ret = afr_selfheal_find_direction(frame, this, replies, type,
+                                      priv->child_up, sources, sinks, witness,
+                                      NULL);
+    if (ret)
+        return ret;
 
-        local->transaction.eager_lock =
-                GF_CALLOC (sizeof (*local->transaction.eager_lock),
-                           priv->child_count,
-                           gf_afr_mt_int32_t);
+    sources_count = AFR_COUNT(sources, priv->child_count);
+    if (!sources_count)
+        *spb = _gf_true;
 
-        if (!local->transaction.eager_lock)
-                goto out;
+    return ret;
+}
 
-        local->transaction.pre_op = GF_CALLOC (sizeof (*local->transaction.pre_op),
-                                               priv->child_count,
-                                               gf_afr_mt_char);
-        if (!local->transaction.pre_op)
-                goto out;
+int
+afr_is_split_brain(call_frame_t *frame, xlator_t *this, inode_t *inode,
+                   uuid_t gfid, gf_boolean_t *d_spb, gf_boolean_t *m_spb)
+{
+    int ret = -1;
+    afr_private_t *priv = NULL;
+    struct afr_reply *replies = NULL;
 
-        local->transaction.fop_subvols = GF_CALLOC (sizeof (*local->transaction.fop_subvols),
-						    priv->child_count,
-						    gf_afr_mt_char);
-        if (!local->transaction.fop_subvols)
-                goto out;
+    priv = this->private;
 
-        local->transaction.failed_subvols = GF_CALLOC (sizeof (*local->transaction.failed_subvols),
-						       priv->child_count,
-						       gf_afr_mt_char);
-        if (!local->transaction.failed_subvols)
-                goto out;
+    replies = alloca0(sizeof(*replies) * priv->child_count);
 
-        local->pending = afr_matrix_create (priv->child_count,
-                                            AFR_NUM_CHANGE_LOGS);
-        if (!local->pending)
-                goto out;
+    ret = afr_selfheal_unlocked_discover(frame, inode, gfid, replies);
+    if (ret)
+        goto out;
 
-	INIT_LIST_HEAD (&local->transaction.eager_locked);
+    if (!afr_can_decide_split_brain_source_sinks(replies, priv->child_count)) {
+        ret = -EAGAIN;
+        goto out;
+    }
 
-        ret = 0;
+    ret = _afr_is_split_brain(frame, this, replies, AFR_DATA_TRANSACTION,
+                              d_spb);
+    if (ret)
+        goto out;
+
+    ret = _afr_is_split_brain(frame, this, replies, AFR_METADATA_TRANSACTION,
+                              m_spb);
 out:
-        return ret;
+    if (replies) {
+        afr_replies_wipe(replies, priv->child_count);
+        replies = NULL;
+    }
+    return ret;
 }
 
-
-void
-afr_set_low_priority (call_frame_t *frame)
+int
+afr_get_split_brain_status_cbk(int ret, call_frame_t *frame, void *opaque)
 {
-        frame->root->pid = LOW_PRIO_PROC_PID;
+    GF_FREE(opaque);
+    return 0;
 }
 
-
-gf_boolean_t
-afr_have_quorum (char *logname, afr_private_t *priv)
+int
+afr_get_split_brain_status(void *opaque)
 {
-        unsigned int        quorum = 0;
+    gf_boolean_t d_spb = _gf_false;
+    gf_boolean_t m_spb = _gf_false;
+    int ret = -1;
+    int op_errno = 0;
+    int i = 0;
+    char *choices = NULL;
+    char *status = NULL;
+    dict_t *dict = NULL;
+    inode_t *inode = NULL;
+    afr_private_t *priv = NULL;
+    xlator_t **children = NULL;
+    call_frame_t *frame = NULL;
+    xlator_t *this = NULL;
+    loc_t *loc = NULL;
+    afr_spb_status_t *data = NULL;
+
+    data = opaque;
+    frame = data->frame;
+    this = frame->this;
+    loc = data->loc;
+    priv = this->private;
+    children = priv->children;
+
+    inode = afr_inode_find(this, loc->gfid);
+    if (!inode)
+        goto out;
+
+    dict = dict_new();
+    if (!dict) {
+        op_errno = ENOMEM;
+        ret = -1;
+        goto out;
+    }
+
+    /* Calculation for string length :
+     * (child_count X length of child-name) + SLEN("    Choices :")
+     * child-name consists of :
+     * a) 251 = max characters for volname according to GD_VOLUME_NAME_MAX
+     * b) strlen("-client-00,") assuming 16 replicas
+     */
+    choices = alloca0(priv->child_count * (256 + SLEN("-client-00,")) +
+                      SLEN("    Choices:"));
+
+    ret = afr_is_split_brain(frame, this, inode, loc->gfid, &d_spb, &m_spb);
+    if (ret) {
+        op_errno = -ret;
+        if (ret == -EAGAIN) {
+            ret = dict_set_sizen_str_sizen(dict, GF_AFR_SBRAIN_STATUS,
+                                           SBRAIN_HEAL_NO_GO_MSG);
+            if (ret) {
+                gf_msg(this->name, GF_LOG_WARNING, -ret,
+                       AFR_MSG_DICT_SET_FAILED,
+                       "Failed to set GF_AFR_SBRAIN_STATUS in dict");
+            }
+        }
+        ret = -1;
+        goto out;
+    }
+
+    if (d_spb || m_spb) {
+        sprintf(choices, "    Choices:");
+        for (i = 0; i < priv->child_count; i++) {
+            strcat(choices, children[i]->name);
+            strcat(choices, ",");
+        }
+        choices[strlen(choices) - 1] = '\0';
 
-        GF_VALIDATE_OR_GOTO(logname,priv,out);
+        ret = gf_asprintf(&status,
+                          "data-split-brain:%s    "
+                          "metadata-split-brain:%s%s",
+                          (d_spb) ? "yes" : "no", (m_spb) ? "yes" : "no",
+                          choices);
 
-        quorum = priv->quorum_count;
-        if (quorum != AFR_QUORUM_AUTO) {
-                return (priv->up_count >= (priv->down_count + quorum));
+        if (-1 == ret) {
+            op_errno = ENOMEM;
+            goto out;
         }
-
-        quorum = priv->child_count / 2 + 1;
-        if (priv->up_count >= (priv->down_count + quorum)) {
-                return _gf_true;
+        ret = dict_set_dynstr_sizen(dict, GF_AFR_SBRAIN_STATUS, status);
+        if (ret) {
+            op_errno = -ret;
+            ret = -1;
+            goto out;
         }
-
-        /*
-         * Special case for even numbers of nodes: if we have exactly half
-         * and that includes the first ("senior-most") node, then that counts
-         * as quorum even if it wouldn't otherwise.  This supports e.g. N=2
-         * while preserving the critical property that there can only be one
-         * such group.
-         */
-        if ((priv->child_count % 2) == 0) {
-                quorum = priv->child_count / 2;
-                if (priv->up_count >= (priv->down_count + quorum)) {
-                        if (priv->child_up[0]) {
-                                return _gf_true;
-                        }
-                }
+    } else {
+        ret = dict_set_sizen_str_sizen(dict, GF_AFR_SBRAIN_STATUS,
+                                       SFILE_NOT_UNDER_DATA);
+        if (ret) {
+            op_errno = -ret;
+            ret = -1;
+            goto out;
         }
+    }
 
+    ret = 0;
 out:
-        return _gf_false;
+    AFR_STACK_UNWIND(getxattr, frame, ret, op_errno, dict, NULL);
+    if (dict)
+        dict_unref(dict);
+    if (inode)
+        inode_unref(inode);
+    return ret;
 }
 
-void
-afr_priv_destroy (afr_private_t *priv)
+int32_t
+afr_heal_splitbrain_file(call_frame_t *frame, xlator_t *this, loc_t *loc)
 {
-        int            i           = 0;
-
-        if (!priv)
-                goto out;
-        GF_FREE (priv->last_event);
-        if (priv->pending_key) {
-                for (i = 0; i < priv->child_count; i++)
-                        GF_FREE (priv->pending_key[i]);
+    int ret = 0;
+    int op_errno = 0;
+    dict_t *dict = NULL;
+    afr_local_t *local = NULL;
+    afr_local_t *heal_local = NULL;
+    call_frame_t *heal_frame = NULL;
+
+    local = frame->local;
+    dict = dict_new();
+    if (!dict) {
+        op_errno = ENOMEM;
+        ret = -1;
+        goto out;
+    }
+
+    heal_frame = afr_frame_create(this, &op_errno);
+    if (!heal_frame) {
+        ret = -1;
+        goto out;
+    }
+    heal_local = heal_frame->local;
+    heal_frame->local = frame->local;
+    /*Initiate heal with heal_frame with lk-owner set so that inodelk/entrylk
+     * work correctly*/
+    ret = afr_selfheal_do(heal_frame, this, loc->gfid);
+
+    if (ret == 1 || ret == 2) {
+        ret = dict_set_sizen_str_sizen(dict, "sh-fail-msg",
+                                       SFILE_NOT_IN_SPLIT_BRAIN);
+        if (ret)
+            gf_msg(this->name, GF_LOG_WARNING, -ret, AFR_MSG_DICT_SET_FAILED,
+                   "Failed to set sh-fail-msg in dict");
+        ret = 0;
+        goto out;
+    } else {
+        if (local->xdata_rsp) {
+            /* 'sh-fail-msg' has been set in the dict during self-heal.*/
+            dict_copy(local->xdata_rsp, dict);
+            ret = 0;
+        } else if (ret < 0) {
+            op_errno = -ret;
+            ret = -1;
         }
-        GF_FREE (priv->pending_key);
-        GF_FREE (priv->children);
-        GF_FREE (priv->child_up);
-        LOCK_DESTROY (&priv->lock);
+    }
 
-        GF_FREE (priv);
 out:
-        return;
+    if (heal_frame) {
+        heal_frame->local = heal_local;
+        AFR_STACK_DESTROY(heal_frame);
+    }
+    if (local->op == GF_FOP_GETXATTR)
+        AFR_STACK_UNWIND(getxattr, frame, ret, op_errno, dict, NULL);
+    else if (local->op == GF_FOP_SETXATTR)
+        AFR_STACK_UNWIND(setxattr, frame, ret, op_errno, NULL);
+    if (dict)
+        dict_unref(dict);
+    return ret;
 }
 
-void
-afr_handle_open_fd_count (call_frame_t *frame, xlator_t *this)
+int
+afr_get_child_index_from_name(xlator_t *this, char *name)
 {
-        afr_local_t     *local = NULL;
-        afr_fd_ctx_t    *fd_ctx   = NULL;
-
-        local = frame->local;
-
-        if (!local->fd)
-		return;
-
-	fd_ctx = afr_fd_ctx_get (local->fd, this);
-	if (!fd_ctx)
-		return;
-
-	fd_ctx->open_fd_count = local->open_fd_count;
+    afr_private_t *priv = this->private;
+    int index = -1;
+
+    for (index = 0; index < priv->child_count; index++) {
+        if (!strcmp(priv->children[index]->name, name))
+            goto out;
+    }
+    index = -1;
+out:
+    return index;
 }
 
-int**
-afr_mark_pending_changelog (afr_private_t *priv, unsigned char *pending,
-                            dict_t *xattr, ia_type_t iat)
+void
+afr_priv_need_heal_set(afr_private_t *priv, gf_boolean_t need_heal)
 {
-       int i = 0;
-       int **changelog = NULL;
-       int idx = -1;
-       int m_idx = 0;
-       int ret = 0;
-
-       m_idx = afr_index_for_transaction_type (AFR_METADATA_TRANSACTION);
-
-       idx = afr_index_from_ia_type (iat);
-
-       changelog = afr_matrix_create (priv->child_count, AFR_NUM_CHANGE_LOGS);
-       if (!changelog)
-                goto out;
+    LOCK(&priv->lock);
+    {
+        priv->need_heal = need_heal;
+    }
+    UNLOCK(&priv->lock);
+}
 
-       for (i = 0; i < priv->child_count; i++) {
-               if (!pending[i])
-                       continue;
-
-               changelog[i][m_idx] = hton32(1);
-               if (idx != -1)
-                       changelog[i][idx] = hton32(1);
-       }
-       ret = afr_set_pending_dict (priv, xattr, changelog);
-       if (ret < 0) {
-               afr_matrix_cleanup (changelog, priv->child_count);
-               return NULL;
-       }
-out:
-       return changelog;
+void
+afr_set_need_heal(xlator_t *this, afr_local_t *local)
+{
+    int i = 0;
+    afr_private_t *priv = this->private;
+    gf_boolean_t need_heal = _gf_false;
+
+    for (i = 0; i < priv->child_count; i++) {
+        if (local->replies[i].valid && local->replies[i].need_heal) {
+            need_heal = _gf_true;
+            break;
+        }
+    }
+    afr_priv_need_heal_set(priv, need_heal);
+    return;
 }
 
 gf_boolean_t
-afr_decide_heal_info (afr_private_t *priv, unsigned char *sources, int ret)
+afr_get_need_heal(xlator_t *this)
 {
-        int sources_count = 0;
-
-        if (ret)
-                goto out;
+    afr_private_t *priv = this->private;
+    gf_boolean_t need_heal = _gf_true;
+
+    LOCK(&priv->lock);
+    {
+        need_heal = priv->need_heal;
+    }
+    UNLOCK(&priv->lock);
+    return need_heal;
+}
 
-        sources_count = AFR_COUNT (sources, priv->child_count);
-        if (sources_count == priv->child_count)
-                return _gf_false;
-out:
-        return _gf_true;
+int
+afr_get_msg_id(char *op_type)
+{
+    if (!strcmp(op_type, GF_AFR_REPLACE_BRICK))
+        return AFR_MSG_REPLACE_BRICK_STATUS;
+    else if (!strcmp(op_type, GF_AFR_ADD_BRICK))
+        return AFR_MSG_ADD_BRICK_STATUS;
+    return -1;
 }
 
 int
-afr_selfheal_locked_metadata_inspect (call_frame_t *frame, xlator_t *this,
-                                      inode_t *inode, gf_boolean_t *msh)
+afr_fav_child_reset_sink_xattrs_cbk(int ret, call_frame_t *heal_frame,
+                                    void *opaque)
 {
-        int ret = -1;
-        unsigned char *locked_on = NULL;
-        unsigned char *sources = NULL;
-        unsigned char *sinks = NULL;
-        unsigned char *healed_sinks = NULL;
-        struct afr_reply *locked_replies = NULL;
+    call_frame_t *txn_frame = NULL;
+    afr_local_t *local = NULL;
+    afr_local_t *heal_local = NULL;
+    xlator_t *this = NULL;
 
-        afr_private_t *priv = this->private;
+    heal_local = heal_frame->local;
+    txn_frame = heal_local->heal_frame;
+    local = txn_frame->local;
+    this = txn_frame->this;
 
-        locked_on = alloca0 (priv->child_count);
-        sources = alloca0 (priv->child_count);
-        sinks = alloca0 (priv->child_count);
-        healed_sinks = alloca0 (priv->child_count);
+    /* Refresh the inode agan and proceed with the transaction.*/
+    afr_inode_refresh(txn_frame, this, local->inode, NULL, local->refreshfn);
 
-        locked_replies = alloca0 (sizeof (*locked_replies) * priv->child_count);
+    AFR_STACK_DESTROY(heal_frame);
 
-        ret = afr_selfheal_inodelk (frame, this, inode, this->name,
-                                    LLONG_MAX - 1, 0, locked_on);
-        {
-                if (ret == 0) {
-                        /* Not a single lock */
-                        ret = -afr_final_errno (frame->local, priv);
-                        if (ret == 0)
-                                ret = -ENOTCONN;/* all invalid responses */
-                        goto out;
-                }
-                ret = __afr_selfheal_metadata_prepare (frame, this, inode,
-                                                       locked_on, sources,
-                                                       sinks, healed_sinks,
-                                                       locked_replies);
-                *msh = afr_decide_heal_info (priv, sources, ret);
-        }
-        afr_selfheal_uninodelk (frame, this, inode, this->name,
-                                LLONG_MAX - 1, 0, locked_on);
-out:
-        if (locked_replies)
-                afr_replies_wipe (locked_replies, priv->child_count);
-        return ret;
+    return 0;
 }
 
 int
-afr_selfheal_locked_data_inspect (call_frame_t *frame, xlator_t *this,
-                                  inode_t *inode, gf_boolean_t *dsh)
-{
-        int ret = -1;
-        afr_private_t   *priv = NULL;
-        unsigned char *locked_on = NULL;
-        unsigned char *data_lock = NULL;
-        unsigned char *sources = NULL;
-        unsigned char *sinks = NULL;
-        unsigned char *healed_sinks = NULL;
-        struct afr_reply *locked_replies = NULL;
-
-        priv = this->private;
-        locked_on = alloca0 (priv->child_count);
-        data_lock = alloca0 (priv->child_count);
-        sources = alloca0 (priv->child_count);
-        sinks = alloca0 (priv->child_count);
-        healed_sinks = alloca0 (priv->child_count);
-
-        locked_replies = alloca0 (sizeof (*locked_replies) * priv->child_count);
-
-        ret = afr_selfheal_tryinodelk (frame, this, inode, priv->sh_domain,
-                                       0, 0, locked_on);
+afr_fav_child_reset_sink_xattrs(void *opaque)
+{
+    call_frame_t *heal_frame = NULL;
+    call_frame_t *txn_frame = NULL;
+    xlator_t *this = NULL;
+    gf_boolean_t d_spb = _gf_false;
+    gf_boolean_t m_spb = _gf_false;
+    afr_local_t *heal_local = NULL;
+    afr_local_t *txn_local = NULL;
+    afr_private_t *priv = NULL;
+    inode_t *inode = NULL;
+    unsigned char *locked_on = NULL;
+    unsigned char *sources = NULL;
+    unsigned char *sinks = NULL;
+    unsigned char *healed_sinks = NULL;
+    unsigned char *undid_pending = NULL;
+    struct afr_reply *locked_replies = NULL;
+    int ret = 0;
+
+    heal_frame = (call_frame_t *)opaque;
+    heal_local = heal_frame->local;
+    txn_frame = heal_local->heal_frame;
+    txn_local = txn_frame->local;
+    this = txn_frame->this;
+    inode = txn_local->inode;
+    priv = this->private;
+    locked_on = alloca0(priv->child_count);
+    sources = alloca0(priv->child_count);
+    sinks = alloca0(priv->child_count);
+    healed_sinks = alloca0(priv->child_count);
+    undid_pending = alloca0(priv->child_count);
+    locked_replies = alloca0(sizeof(*locked_replies) * priv->child_count);
+
+    ret = _afr_is_split_brain(txn_frame, this, txn_local->replies,
+                              AFR_DATA_TRANSACTION, &d_spb);
+
+    ret = _afr_is_split_brain(txn_frame, this, txn_local->replies,
+                              AFR_METADATA_TRANSACTION, &m_spb);
+
+    /* Take appropriate locks and reset sink xattrs. */
+    if (d_spb) {
+        ret = afr_selfheal_inodelk(heal_frame, this, inode, this->name, 0, 0,
+                                   locked_on);
         {
-                if (ret == 0) {
-                        ret = -afr_final_errno (frame->local, priv);
-                        if (ret == 0)
-                                ret = -ENOTCONN;/* all invalid responses */
-                        goto out;
-                }
-                ret = afr_selfheal_inodelk (frame, this, inode, this->name,
-                                            0, 0, data_lock);
-                {
-                        if (ret == 0) {
-                                ret = -afr_final_errno (frame->local, priv);
-                                if (ret == 0)
-                                        ret = -ENOTCONN;
-                                /* all invalid responses */
-                                goto unlock;
-                        }
-                        ret = __afr_selfheal_data_prepare (frame, this, inode,
-                                                           data_lock, sources,
-                                                           sinks, healed_sinks,
-                                                           locked_replies);
-                        *dsh = afr_decide_heal_info (priv, sources, ret);
-                }
-                afr_selfheal_uninodelk (frame, this, inode, this->name, 0, 0,
-                                        data_lock);
-        }
-unlock:
-        afr_selfheal_uninodelk (frame, this, inode, priv->sh_domain, 0, 0,
-                                locked_on);
-out:
-        if (locked_replies)
-                afr_replies_wipe (locked_replies, priv->child_count);
-        return ret;
-}
-
-int
-afr_selfheal_locked_entry_inspect (call_frame_t *frame, xlator_t *this,
-                                   inode_t *inode,
-                                   gf_boolean_t *esh)
-{
-        int ret = -1;
-        int source = -1;
-        afr_private_t   *priv = NULL;
-        unsigned char *locked_on = NULL;
-        unsigned char *data_lock = NULL;
-        unsigned char *sources = NULL;
-        unsigned char *sinks = NULL;
-        unsigned char *healed_sinks = NULL;
-        struct afr_reply *locked_replies = NULL;
-
-        priv = this->private;
-        locked_on = alloca0 (priv->child_count);
-        data_lock = alloca0 (priv->child_count);
-        sources = alloca0 (priv->child_count);
-        sinks = alloca0 (priv->child_count);
-        healed_sinks = alloca0 (priv->child_count);
-
-        locked_replies = alloca0 (sizeof (*locked_replies) * priv->child_count);
-
-        ret = afr_selfheal_tryentrylk (frame, this, inode, priv->sh_domain,
-                                       NULL, locked_on);
+            if (ret < priv->child_count)
+                goto data_unlock;
+            ret = __afr_selfheal_data_prepare(
+                heal_frame, this, inode, locked_on, sources, sinks,
+                healed_sinks, undid_pending, locked_replies, NULL);
+        }
+    data_unlock:
+        afr_selfheal_uninodelk(heal_frame, this, inode, this->name, 0, 0,
+                               locked_on);
+    }
+
+    if (m_spb) {
+        memset(locked_on, 0, sizeof(*locked_on) * priv->child_count);
+        memset(undid_pending, 0, sizeof(*undid_pending) * priv->child_count);
+        ret = afr_selfheal_inodelk(heal_frame, this, inode, this->name,
+                                   LLONG_MAX - 1, 0, locked_on);
         {
-                if (ret == 0) {
-                        ret = -afr_final_errno (frame->local, priv);
-                        if (ret == 0)
-                                ret = -ENOTCONN;/* all invalid responses */
-                        goto out;
-                }
-
-                ret = afr_selfheal_entrylk (frame, this, inode, this->name,
-                                            NULL, data_lock);
-                {
-                        if (ret == 0) {
-                                ret = -afr_final_errno (frame->local, priv);
-                                if (ret == 0)
-                                        ret = -ENOTCONN;
-                                /* all invalid responses */
-                                goto unlock;
-                        }
-                        ret = __afr_selfheal_entry_prepare (frame, this, inode,
-                                                            data_lock, sources,
-                                                            sinks, healed_sinks,
-                                                            locked_replies,
-                                                            &source);
-                        if ((ret == 0) && source < 0)
-                                ret = -EIO;
-                        *esh = afr_decide_heal_info (priv, sources, ret);
-                }
-                afr_selfheal_unentrylk (frame, this, inode, this->name, NULL,
-                                        data_lock);
-        }
-unlock:
-        afr_selfheal_unentrylk (frame, this, inode, priv->sh_domain, NULL,
-                                locked_on);
-out:
-        if (locked_replies)
-                afr_replies_wipe (locked_replies, priv->child_count);
-        return ret;
+            if (ret < priv->child_count)
+                goto mdata_unlock;
+            ret = __afr_selfheal_metadata_prepare(
+                heal_frame, this, inode, locked_on, sources, sinks,
+                healed_sinks, undid_pending, locked_replies, NULL);
+        }
+    mdata_unlock:
+        afr_selfheal_uninodelk(heal_frame, this, inode, this->name,
+                               LLONG_MAX - 1, 0, locked_on);
+    }
+
+    return ret;
 }
 
+/*
+ * Concatenates the xattrs in local->replies separated by a delimiter.
+ */
 int
-afr_selfheal_locked_inspect (call_frame_t *frame, xlator_t *this, uuid_t gfid,
-                             inode_t **inode,
-                             gf_boolean_t *entry_selfheal,
-                             gf_boolean_t *data_selfheal,
-                             gf_boolean_t *metadata_selfheal)
-
+afr_serialize_xattrs_with_delimiter(call_frame_t *frame, xlator_t *this,
+                                    char *buf, const char *default_str,
+                                    int32_t *serz_len, char delimiter)
 {
-        int ret             = -1;
-        gf_boolean_t    dsh = _gf_false;
-        gf_boolean_t    msh = _gf_false;
-        gf_boolean_t    esh = _gf_false;
-
-        ret = afr_selfheal_unlocked_inspect (frame, this, gfid, inode,
-                                             &dsh, &msh, &esh);
-        if (ret)
+    afr_private_t *priv = NULL;
+    afr_local_t *local = NULL;
+    char *xattr = NULL;
+    int i = 0;
+    int len = 0;
+    int keylen = 0;
+    size_t str_len = 0;
+    int ret = -1;
+
+    priv = this->private;
+    local = frame->local;
+
+    keylen = strlen(local->cont.getxattr.name);
+    for (i = 0; i < priv->child_count; i++) {
+        if (!local->replies[i].valid || local->replies[i].op_ret) {
+            str_len = strlen(default_str);
+            buf = strncat(buf, default_str, str_len);
+            len += str_len;
+            buf[len++] = delimiter;
+            buf[len] = '\0';
+        } else {
+            ret = dict_get_strn(local->replies[i].xattr,
+                                local->cont.getxattr.name, keylen, &xattr);
+            if (ret) {
+                gf_msg("TEST", GF_LOG_ERROR, -ret, AFR_MSG_DICT_GET_FAILED,
+                       "Failed to get the node_uuid of brick "
+                       "%d",
+                       i);
                 goto out;
+            }
+            str_len = strlen(xattr);
+            buf = strncat(buf, xattr, str_len);
+            len += str_len;
+            buf[len++] = delimiter;
+            buf[len] = '\0';
+        }
+    }
+    buf[--len] = '\0'; /*remove the last delimiter*/
+    if (serz_len)
+        *serz_len = ++len;
+    ret = 0;
 
-        /* For every heal type hold locks and check if it indeed needs heal */
+out:
+    return ret;
+}
 
-        if (msh) {
-                ret = afr_selfheal_locked_metadata_inspect (frame, this,
-                                                            *inode, &msh);
-                if (ret == -EIO)
-                        goto out;
-        }
+uint64_t
+afr_write_subvol_get(call_frame_t *frame, xlator_t *this)
+{
+    afr_local_t *local = NULL;
+    uint64_t write_subvol = 0;
 
-        if (dsh) {
-                ret = afr_selfheal_locked_data_inspect (frame, this, *inode,
-                                                        &dsh);
-                if (ret == -EIO || (ret == -EAGAIN))
-                        goto out;
-        }
+    local = frame->local;
+    LOCK(&local->inode->lock);
+    write_subvol = local->inode_ctx->write_subvol;
+    UNLOCK(&local->inode->lock);
 
-        if (esh) {
-                ret = afr_selfheal_locked_entry_inspect (frame, this, *inode,
-                                                         &esh);
-        }
+    return write_subvol;
+}
 
-out:
-        *data_selfheal = dsh;
-        *entry_selfheal = esh;
-        *metadata_selfheal = msh;
-        return ret;
+int
+afr_write_subvol_set(call_frame_t *frame, xlator_t *this)
+{
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
+    unsigned char *data_accused = NULL;
+    unsigned char *metadata_accused = NULL;
+    unsigned char *data_readable = NULL;
+    unsigned char *metadata_readable = NULL;
+    uint16_t datamap = 0;
+    uint16_t metadatamap = 0;
+    uint64_t val = 0;
+    int event = 0;
+    int i = 0;
+
+    local = frame->local;
+    priv = this->private;
+    data_accused = alloca0(priv->child_count);
+    metadata_accused = alloca0(priv->child_count);
+    data_readable = alloca0(priv->child_count);
+    metadata_readable = alloca0(priv->child_count);
+    event = local->event_generation;
+
+    afr_readables_fill(frame, this, local->inode, data_accused,
+                       metadata_accused, data_readable, metadata_readable,
+                       NULL);
+
+    for (i = 0; i < priv->child_count; i++) {
+        if (data_readable[i])
+            datamap |= (1 << i);
+        if (metadata_readable[i])
+            metadatamap |= (1 << i);
+    }
+
+    val = ((uint64_t)metadatamap) | (((uint64_t)datamap) << 16) |
+          (((uint64_t)event) << 32);
+
+    LOCK(&local->inode->lock);
+    {
+        if (local->inode_ctx->write_subvol == 0 &&
+            local->transaction.type == AFR_DATA_TRANSACTION) {
+            local->inode_ctx->write_subvol = val;
+        }
+    }
+    UNLOCK(&local->inode->lock);
+
+    return 0;
 }
 
-dict_t*
-afr_set_heal_info (char *status)
+int
+afr_write_subvol_reset(call_frame_t *frame, xlator_t *this)
 {
-        dict_t *dict = NULL;
-        int    ret   = -1;
+    afr_local_t *local = NULL;
 
-        dict = dict_new ();
-        if (!dict) {
-                ret = -ENOMEM;
-                goto out;
-        }
+    local = frame->local;
+    LOCK(&local->inode->lock);
+    {
+        GF_ASSERT(local->inode_ctx->lock_count > 0);
+        local->inode_ctx->lock_count--;
 
-        if (!strcmp (status, "heal")) {
-                ret = dict_set_str (dict, "heal-info", "heal");
-                if (ret)
-                        gf_log ("", GF_LOG_WARNING,
-                                "Failed to set heal-info key to"
-                                "heal");
-        } else if (!strcmp (status, "split-brain")) {
-                ret = dict_set_str (dict, "heal-info", "split-brain");
-                if (ret)
-                        gf_log ("", GF_LOG_WARNING,
-                                "Failed to set heal-info key to"
-                                "split-brain");
-        } else if (!strcmp (status, "possibly-healing")) {
-                ret = dict_set_str (dict, "heal-info", "possibly-healing");
-                if (ret)
-                       gf_log ("", GF_LOG_WARNING,
-                                "Failed to set heal-info key to"
-                                "possibly-healing");
-        }
-out:
-        return dict;
+        if (!local->inode_ctx->lock_count)
+            local->inode_ctx->write_subvol = 0;
+    }
+    UNLOCK(&local->inode->lock);
+
+    return 0;
 }
 
 int
-afr_get_heal_info (call_frame_t *frame, xlator_t *this, loc_t *loc,
-                   dict_t *xdata)
+afr_set_inode_local(xlator_t *this, afr_local_t *local, inode_t *inode)
 {
-        gf_boolean_t    data_selfheal     = _gf_false;
-        gf_boolean_t    metadata_selfheal = _gf_false;
-        gf_boolean_t    entry_selfheal    = _gf_false;
-        dict_t         *dict              = NULL;
-        int             ret               = -1;
-        int             op_errno          = 0;
-        inode_t        *inode             = NULL;
+    int ret = 0;
+
+    local->inode = inode_ref(inode);
+    LOCK(&local->inode->lock);
+    {
+        ret = __afr_inode_ctx_get(this, local->inode, &local->inode_ctx);
+    }
+    UNLOCK(&local->inode->lock);
+    if (ret < 0) {
+        gf_msg_callingfn(
+            this->name, GF_LOG_ERROR, ENOMEM, AFR_MSG_INODE_CTX_GET_FAILED,
+            "Error getting inode ctx %s", uuid_utoa(local->inode->gfid));
+    }
+    return ret;
+}
 
-        ret = afr_selfheal_locked_inspect (frame, this, loc->gfid, &inode,
-                                           &entry_selfheal,
-                                           &data_selfheal, &metadata_selfheal);
+gf_boolean_t
+afr_ta_is_fop_called_from_synctask(xlator_t *this)
+{
+    struct synctask *task = NULL;
+    gf_lkowner_t tmp_owner = {
+        0,
+    };
 
-        if (ret == -ENOMEM) {
-                op_errno = -ret;
-                ret = -1;
-                goto out;
-        }
+    task = synctask_get();
+    if (!task)
+        return _gf_false;
 
-        if (ret == -EIO) {
-                dict = afr_set_heal_info ("split-brain");
-        } else if (ret == -EAGAIN) {
-                dict = afr_set_heal_info ("possibly-healing");
-        } else if (ret >= 0) {
-                /* value of ret = source index
-                 * so ret >= 0 and at least one of the 3 booleans set to
-                 * true means a source is identified; heal is required.
-                 */
-                if (!data_selfheal && !entry_selfheal &&
-                    !metadata_selfheal) {
-                        dict = afr_set_heal_info ("no-heal");
-                } else {
-                        dict = afr_set_heal_info ("heal");
-                }
-        } else if (ret < 0) {
-                /* Apart from above checked -ve ret values, there are
-                 * other possible ret values like ENOTCONN
-                 * (returned when number of valid replies received are
-                 * less than 2)
-                 * in which case heal is required when one of the
-                 * selfheal booleans is set.
-                 */
-                if (data_selfheal || entry_selfheal ||
-                    metadata_selfheal) {
-                        dict = afr_set_heal_info ("heal");
-                }
-        }
-        ret = 0;
+    set_lk_owner_from_ptr(&tmp_owner, (void *)this);
+
+    if (!is_same_lkowner(&tmp_owner, &task->frame->root->lk_owner))
+        return _gf_false;
 
+    return _gf_true;
+}
+
+int
+afr_ta_post_op_lock(xlator_t *this, loc_t *loc)
+{
+    int ret = 0;
+    uuid_t gfid = {
+        0,
+    };
+    afr_private_t *priv = this->private;
+    gf_boolean_t locked = _gf_false;
+    struct gf_flock flock1 = {
+        0,
+    };
+    struct gf_flock flock2 = {
+        0,
+    };
+    int32_t cmd = 0;
+
+    /* Clients must take AFR_TA_DOM_NOTIFY lock only when the previous lock
+     * has been released in afr_notify due to upcall notification from shd.
+     */
+    GF_ASSERT(priv->ta_notify_dom_lock_offset == 0);
+
+    if (!priv->shd.iamshd)
+        GF_ASSERT(afr_ta_is_fop_called_from_synctask(this));
+    flock1.l_type = F_WRLCK;
+
+    while (!locked) {
+        if (priv->shd.iamshd) {
+            cmd = F_SETLKW;
+            flock1.l_start = 0;
+            flock1.l_len = 0;
+        } else {
+            cmd = F_SETLK;
+            gf_uuid_generate(gfid);
+            flock1.l_start = gfid_to_ino(gfid);
+            if (flock1.l_start < 0)
+                flock1.l_start = -flock1.l_start;
+            flock1.l_len = 1;
+        }
+        ret = syncop_inodelk(priv->children[THIN_ARBITER_BRICK_INDEX],
+                             AFR_TA_DOM_NOTIFY, loc, cmd, &flock1, NULL, NULL);
+        if (!ret) {
+            locked = _gf_true;
+            priv->ta_notify_dom_lock_offset = flock1.l_start;
+        } else if (ret == -EAGAIN) {
+            continue;
+        } else {
+            gf_msg(this->name, GF_LOG_ERROR, -ret, AFR_MSG_THIN_ARB,
+                   "Failed to get "
+                   "AFR_TA_DOM_NOTIFY lock on %s.",
+                   loc->name);
+            goto out;
+        }
+    }
+
+    flock2.l_type = F_WRLCK;
+    flock2.l_start = 0;
+    flock2.l_len = 0;
+    ret = syncop_inodelk(priv->children[THIN_ARBITER_BRICK_INDEX],
+                         AFR_TA_DOM_MODIFY, loc, F_SETLKW, &flock2, NULL, NULL);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, -ret, AFR_MSG_THIN_ARB,
+               "Failed to get AFR_TA_DOM_MODIFY lock on %s.", loc->name);
+        flock1.l_type = F_UNLCK;
+        ret = syncop_inodelk(priv->children[THIN_ARBITER_BRICK_INDEX],
+                             AFR_TA_DOM_NOTIFY, loc, F_SETLK, &flock1, NULL,
+                             NULL);
+    }
 out:
-        AFR_STACK_UNWIND (getxattr, frame, ret, op_errno, dict, NULL);
-        if (dict)
-               dict_unref (dict);
-        if (inode) {
-                inode_forget (inode, 1);
-                inode_unref (inode);
-        }
-        return ret;
+    return ret;
 }
 
 int
-afr_set_split_brain_status (call_frame_t *frame, xlator_t *this,
-                            struct afr_reply *replies,
-                            afr_transaction_type type,
-                            gf_boolean_t *spb)
-{
-        afr_private_t    *priv              = NULL;
-        uint64_t         *witness           = NULL;
-        unsigned char    *sources           = NULL;
-        unsigned char    *sinks             = NULL;
-        int               sources_count     = 0;
-        int               ret               = 0;
-
-        priv = this->private;
-
-        sources = alloca0 (priv->child_count);
-        sinks = alloca0 (priv->child_count);
-        witness = alloca0(priv->child_count * sizeof (*witness));
-
-        ret = afr_selfheal_find_direction (frame, this, replies,
-					   type, priv->child_up, sources,
-                                           sinks, witness);
-        if (ret)
-                return ret;
+afr_ta_post_op_unlock(xlator_t *this, loc_t *loc)
+{
+    afr_private_t *priv = this->private;
+    struct gf_flock flock = {
+        0,
+    };
+    int ret = 0;
+
+    if (!priv->shd.iamshd)
+        GF_ASSERT(afr_ta_is_fop_called_from_synctask(this));
+    flock.l_type = F_UNLCK;
+    flock.l_start = 0;
+    flock.l_len = 0;
+
+    ret = syncop_inodelk(priv->children[THIN_ARBITER_BRICK_INDEX],
+                         AFR_TA_DOM_MODIFY, loc, F_SETLK, &flock, NULL, NULL);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, -ret, AFR_MSG_THIN_ARB,
+               "Failed to unlock AFR_TA_DOM_MODIFY lock.");
+        goto out;
+    }
+
+    if (!priv->shd.iamshd)
+        /* Mounts (clients) will not release the AFR_TA_DOM_NOTIFY lock
+         * in post-op as they use it as a notification mechanism. When
+         * shd sends a lock request on TA during heal, the clients will
+         * receive a lock-contention upcall notification upon which they
+         * will release the AFR_TA_DOM_NOTIFY lock after completing the
+         * in flight I/O.*/
+        goto out;
+
+    ret = syncop_inodelk(priv->children[THIN_ARBITER_BRICK_INDEX],
+                         AFR_TA_DOM_NOTIFY, loc, F_SETLK, &flock, NULL, NULL);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, -ret, AFR_MSG_THIN_ARB,
+               "Failed to unlock AFR_TA_DOM_NOTIFY lock.");
+    }
+out:
+    return ret;
+}
 
-        sources_count = AFR_COUNT (sources, priv->child_count);
-        if (!sources_count)
-                *spb = _gf_true;
+call_frame_t *
+afr_ta_frame_create(xlator_t *this)
+{
+    call_frame_t *frame = NULL;
+    void *lk_owner = NULL;
 
-        return ret;
+    frame = create_frame(this, this->ctx->pool);
+    if (!frame)
+        return NULL;
+    lk_owner = (void *)this;
+    afr_set_lk_owner(frame, this, lk_owner);
+    return frame;
 }
 
-int
-afr_get_split_brain_status (call_frame_t *frame, xlator_t *this, loc_t *loc)
-{
-        gf_boolean_t      d_spb             = _gf_false;
-        gf_boolean_t      m_spb             = _gf_false;
-        int               ret               = -1;
-        int               op_errno          = 0;
-        int               i                 = 0;
-        char             *choices           = NULL;
-        char             *status            = NULL;
-        dict_t           *dict              = NULL;
-        struct afr_reply *replies           = NULL;
-        inode_t          *inode             = NULL;
-        afr_private_t    *priv              = NULL;
-        xlator_t         **children         = NULL;
-
-        priv     = this->private;
-        children = priv->children;
-
-        inode = afr_inode_find (this, loc->gfid);
-        if (!inode)
-                goto out;
-        replies = alloca0 (sizeof (*replies) * priv->child_count);
+gf_boolean_t
+afr_ta_has_quorum(afr_private_t *priv, afr_local_t *local)
+{
+    int data_count = 0;
 
-        /* Calculation for string length :
-        * (child_count X length of child-name) + strlen ("    Choices :")
-        * child-name consists of :
-        * a) 256 = max characters for volname according to GD_VOLUME_NAME_MAX
-        * b) strlen ("-client-00,") assuming 16 replicas
-        */
-        choices = alloca0 (priv->child_count * (256 + strlen ("-client-00,")) +
-                           strlen ("    Choices:"));
-        ret = afr_selfheal_unlocked_discover (frame, inode, loc->gfid, replies);
-        if (ret) {
-                op_errno = -ret;
-                ret = -1;
-                goto out;
-        }
+    data_count = AFR_COUNT(local->child_up, priv->child_count);
+    if (data_count == 2) {
+        return _gf_true;
+    } else if (data_count == 1 && local->ta_child_up) {
+        return _gf_true;
+    }
 
-        ret = afr_set_split_brain_status (frame, this, replies,
-                                          AFR_DATA_TRANSACTION, &d_spb);
-        if (ret) {
-                op_errno = -ret;
-                ret = -1;
-                goto out;
-        }
+    return _gf_false;
+}
 
-        ret = afr_set_split_brain_status (frame, this, replies,
-                                          AFR_METADATA_TRANSACTION, &m_spb);
-        if (ret) {
-                op_errno = -ret;
-                ret = -1;
-                goto out;
-        }
+static gf_boolean_t
+afr_is_add_replica_mount_lookup_on_root(call_frame_t *frame)
+{
+    afr_local_t *local = NULL;
 
-        dict = dict_new ();
-        if (!dict) {
-                op_errno = ENOMEM;
-                ret = -1;
-                goto out;
-        }
+    if (frame->root->pid != GF_CLIENT_PID_ADD_REPLICA_MOUNT)
+        return _gf_false;
 
-        if (d_spb || m_spb) {
-                sprintf (choices, "    Choices:");
-                for (i = 0; i < priv->child_count; i++) {
-                        strcat (choices, children[i]->name);
-                        strcat (choices, ",");
-                }
-                choices[strlen (choices) - 1] = '\0';
+    local = frame->local;
 
-                ret = gf_asprintf (&status, "data-split-brain:%s    "
-                                    "metadata-split-brain:%s%s",
-                                    (d_spb) ? "yes" : "no",
-                                    (m_spb) ? "yes" : "no", choices);
+    if (local->op != GF_FOP_LOOKUP)
+        /* TODO:If the replica count is being increased on a plain distribute
+         * volume that was never mounted, we need to allow setxattr on '/' with
+         * GF_CLIENT_PID_NO_ROOT_SQUASH to accomodate for DHT layout setting */
+        return _gf_false;
 
-                if (-1 == ret) {
-                        op_errno = ENOMEM;
-                        goto out;
-                }
-                ret = dict_set_dynstr (dict, GF_AFR_SBRAIN_STATUS, status);
-                if (ret)
-                        goto out;
-        } else {
-                ret = dict_set_str (dict, GF_AFR_SBRAIN_STATUS,
-                                    "The file is not under data or"
-                                    " metadata split-brain");
-                if (ret)
-                        goto out;
-        }
+    if (local->inode == NULL)
+        return _gf_false;
 
-        ret = 0;
-out:
-        AFR_STACK_UNWIND (getxattr, frame, ret, op_errno, dict, NULL);
-        if (dict)
-               dict_unref (dict);
-        if (replies)
-                afr_replies_wipe (replies, priv->child_count);
-        if (inode)
-                inode_unref (inode);
-        return ret;
+    if (!__is_root_gfid(local->inode->gfid))
+        return _gf_false;
+
+    return _gf_true;
 }
 
-int32_t
-afr_heal_splitbrain_file(call_frame_t *frame, xlator_t *this, loc_t *loc)
+gf_boolean_t
+afr_lookup_has_quorum(call_frame_t *frame, const unsigned int up_children_count)
 {
-        int             ret               = 0;
-        int             op_errno          = 0;
-        dict_t         *dict              = NULL;
-        afr_local_t    *local             = NULL;
+    if (frame && (up_children_count > 0) &&
+        afr_is_add_replica_mount_lookup_on_root(frame))
+        return _gf_true;
 
-        local = frame->local;
-        dict = dict_new ();
-        if (!dict) {
-                op_errno = ENOMEM;
-                ret = -1;
-                goto out;
-        }
+    return _gf_false;
+}
 
-        ret = afr_selfheal_do (frame, this, loc->gfid);
+void
+afr_handle_replies_quorum(call_frame_t *frame, xlator_t *this)
+{
+    afr_local_t *local = frame->local;
+    afr_private_t *priv = this->private;
+    unsigned char *success_replies = NULL;
 
-        if (ret == 1 || ret == 2) {
-                ret = dict_set_str (dict, "sh-fail-msg",
-                                    "File not in split-brain");
-                if (ret)
-                        gf_log (this->name, GF_LOG_WARNING,
-                                "Failed to set sh-fail-msg in dict");
-                ret = 0;
-                goto out;
-        } else {
-                if (local->xdata_rsp) {
-                        /* 'sh-fail-msg' has been set in the dict during self-heal.*/
-                        dict_copy (local->xdata_rsp, dict);
-                        ret = 0;
-                } else if (ret < 0) {
-                        op_errno = -ret;
-                        ret = -1;
-                }
-        }
+    success_replies = alloca0(priv->child_count);
+    afr_fill_success_replies(local, priv, success_replies);
 
-out:
-        if (local->op == GF_FOP_GETXATTR)
-                AFR_STACK_UNWIND (getxattr, frame, ret, op_errno, dict, NULL);
-        else if (local->op == GF_FOP_SETXATTR)
-                AFR_STACK_UNWIND (setxattr, frame, ret, op_errno, NULL);
-        if (dict)
-                dict_unref(dict);
-        return ret;
+    if (priv->quorum_count && !afr_has_quorum(success_replies, this, NULL)) {
+        local->op_errno = afr_final_errno(local, priv);
+        if (!local->op_errno)
+            local->op_errno = afr_quorum_errno(priv);
+        local->op_ret = -1;
+    }
 }
 
-int
-afr_get_child_index_from_name (xlator_t *this, char *name)
+gf_boolean_t
+afr_ta_dict_contains_pending_xattr(dict_t *dict, afr_private_t *priv, int child)
 {
-        afr_private_t *priv  = this->private;
-        int            index = -1;
-
-        for (index = 0; index < priv->child_count; index++) {
-                if (!strcmp (priv->children[index]->name, name))
-                        goto out;
+    int *pending = NULL;
+    int ret = 0;
+    int i = 0;
+
+    ret = dict_get_ptr(dict, priv->pending_key[child], (void *)&pending);
+    if (ret == 0) {
+        for (i = 0; i < AFR_NUM_CHANGE_LOGS; i++) {
+            /* Not doing a ntoh32(pending) as we just want to check
+             * if it is non-zero or not. */
+            if (pending[i]) {
+                return _gf_true;
+            }
         }
-        index = -1;
-out:
-        return index;
+    }
+
+    return _gf_false;
 }
diff --git a/xlators/cluster/afr/src/afr-dir-read.c b/xlators/cluster/afr/src/afr-dir-read.c
index 984ed9c6095..f8bf8340dab 100644
--- a/xlators/cluster/afr/src/afr-dir-read.c
+++ b/xlators/cluster/afr/src/afr-dir-read.c
@@ -8,334 +8,339 @@
   cases as published by the Free Software Foundation.
 */
 
-
 #include <libgen.h>
 #include <unistd.h>
-#include <fnmatch.h>
 #include <sys/time.h>
 #include <stdlib.h>
 #include <signal.h>
 #include <string.h>
 
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
-
-#include "glusterfs.h"
-#include "dict.h"
-#include "xlator.h"
-#include "hashfn.h"
-#include "logging.h"
-#include "stack.h"
-#include "list.h"
-#include "call-stub.h"
-#include "defaults.h"
-#include "common-utils.h"
-#include "compat-errno.h"
-#include "compat.h"
-#include "checksum.h"
+#include <glusterfs/glusterfs.h>
+#include <glusterfs/dict.h>
+#include <glusterfs/list.h>
+#include <glusterfs/common-utils.h>
+#include <glusterfs/compat-errno.h>
+#include <glusterfs/compat.h>
 
 #include "afr.h"
 #include "afr-transaction.h"
 
-
 int32_t
-afr_opendir_cbk (call_frame_t *frame, void *cookie,
-                 xlator_t *this, int32_t op_ret, int32_t op_errno,
-                 fd_t *fd, dict_t *xdata)
+afr_opendir_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                int32_t op_ret, int32_t op_errno, fd_t *fd, dict_t *xdata)
 {
-        afr_local_t   *local             = NULL;
-        int            call_count        = -1;
-        int32_t        child_index       = 0;
-	afr_fd_ctx_t  *fd_ctx = NULL;
-
-        local = frame->local;
-	fd_ctx = local->fd_ctx;
-        child_index = (long) cookie;
-
-        LOCK (&frame->lock);
-        {
-                if (op_ret == -1) {
-                        local->op_errno = op_errno;
-			fd_ctx->opened_on[child_index] = AFR_FD_NOT_OPENED;
-                } else {
-                        local->op_ret = op_ret;
-			fd_ctx->opened_on[child_index] = AFR_FD_OPENED;
-			if (!local->xdata_rsp && xdata)
-				local->xdata_rsp = dict_ref (xdata);
-                }
+    afr_local_t *local = NULL;
+    int call_count = -1;
+    int32_t child_index = 0;
+    afr_fd_ctx_t *fd_ctx = NULL;
+
+    local = frame->local;
+    fd_ctx = local->fd_ctx;
+    child_index = (long)cookie;
+
+    local->replies[child_index].valid = 1;
+    local->replies[child_index].op_ret = op_ret;
+    local->replies[child_index].op_errno = op_errno;
+
+    LOCK(&frame->lock);
+    {
+        if (op_ret == -1) {
+            local->op_errno = op_errno;
+            fd_ctx->opened_on[child_index] = AFR_FD_NOT_OPENED;
+        } else {
+            local->op_ret = op_ret;
+            fd_ctx->opened_on[child_index] = AFR_FD_OPENED;
+            if (!local->xdata_rsp && xdata)
+                local->xdata_rsp = dict_ref(xdata);
         }
-        UNLOCK (&frame->lock);
+        call_count = --local->call_count;
+    }
+    UNLOCK(&frame->lock);
 
-        call_count = afr_frame_return (frame);
+    if (call_count == 0) {
+        afr_handle_replies_quorum(frame, this);
+        AFR_STACK_UNWIND(opendir, frame, local->op_ret, local->op_errno,
+                         local->fd, NULL);
+    }
 
-        if (call_count == 0)
-		AFR_STACK_UNWIND (opendir, frame, local->op_ret,
-				  local->op_errno, local->fd, NULL);
-        return 0;
+    return 0;
 }
 
-
 int
-afr_opendir (call_frame_t *frame, xlator_t *this, loc_t *loc, fd_t *fd)
+afr_opendir(call_frame_t *frame, xlator_t *this, loc_t *loc, fd_t *fd,
+            dict_t *xdata)
 {
-        afr_private_t * priv        = NULL;
-        afr_local_t   * local       = NULL;
-        int             i           = 0;
-        int             call_count  = -1;
-        int32_t         op_errno    = ENOMEM;
-	afr_fd_ctx_t *fd_ctx = NULL;
+    afr_private_t *priv = NULL;
+    afr_local_t *local = NULL;
+    int i = 0;
+    int call_count = -1;
+    int32_t op_errno = ENOMEM;
+    afr_fd_ctx_t *fd_ctx = NULL;
+
+    priv = this->private;
 
-        priv = this->private;
+    local = AFR_FRAME_INIT(frame, op_errno);
+    if (!local)
+        goto out;
 
-        local = AFR_FRAME_INIT (frame, op_errno);
-	if (!local)
-		goto out;
+    local->op = GF_FOP_OPENDIR;
 
-	fd_ctx = afr_fd_ctx_get (fd, this);
-	if (!fd_ctx)
-		goto out;
+    if (priv->quorum_count && !afr_has_quorum(local->child_up, this, NULL)) {
+        op_errno = afr_quorum_errno(priv);
+        goto out;
+    }
 
-        loc_copy (&local->loc, loc);
+    if (!afr_is_consistent_io_possible(local, priv, &op_errno))
+        goto out;
 
-        local->fd    = fd_ref (fd);
-	local->fd_ctx = fd_ctx;
+    fd_ctx = afr_fd_ctx_get(fd, this);
+    if (!fd_ctx)
+        goto out;
 
-        call_count = local->call_count;
+    loc_copy(&local->loc, loc);
 
-        for (i = 0; i < priv->child_count; i++) {
-                if (local->child_up[i]) {
-                        STACK_WIND_COOKIE (frame, afr_opendir_cbk,
-                                           (void*) (long) i,
-                                           priv->children[i],
-                                           priv->children[i]->fops->opendir,
-                                           loc, fd, NULL);
+    local->fd = fd_ref(fd);
+    local->fd_ctx = fd_ctx;
 
-                        if (!--call_count)
-                                break;
-                }
+    call_count = local->call_count;
+
+    for (i = 0; i < priv->child_count; i++) {
+        if (local->child_up[i]) {
+            STACK_WIND_COOKIE(frame, afr_opendir_cbk, (void *)(long)i,
+                              priv->children[i],
+                              priv->children[i]->fops->opendir, loc, fd, NULL);
+
+            if (!--call_count)
+                break;
         }
+    }
 
-	return 0;
+    return 0;
 out:
-	AFR_STACK_UNWIND (opendir, frame, -1, op_errno, fd, NULL);
-        return 0;
+    AFR_STACK_UNWIND(opendir, frame, -1, op_errno, fd, NULL);
+    return 0;
 }
 
 static int
-afr_validate_read_subvol (inode_t *inode, xlator_t *this, int par_read_subvol)
+afr_validate_read_subvol(inode_t *inode, xlator_t *this, int par_read_subvol)
 {
-	int             gen               = 0;
-        int             entry_read_subvol = 0;
-	unsigned char  *data_readable     = NULL;
-	unsigned char  *metadata_readable = NULL;
-        afr_private_t  *priv              = NULL;
-
-        priv = this->private;
-	data_readable = alloca0 (priv->child_count);
-	metadata_readable = alloca0 (priv->child_count);
-
-	afr_inode_read_subvol_get (inode, this, data_readable,
-				   metadata_readable, &gen);
-
-        if (gen != priv->event_generation ||
-                !data_readable[par_read_subvol] ||
-                !metadata_readable[par_read_subvol])
-                return -1;
-
-        /* Once the control reaches the following statement, it means that the
-         * parent's read subvol is perfectly readable. So calling
-         * either afr_data_subvol_get() or afr_metadata_subvol_get() would
-         * yield the same result. Hence, choosing afr_data_subvol_get() below.
-         */
-
-        if (!priv->consistent_metadata)
-                return 0;
-
-        entry_read_subvol = afr_data_subvol_get (inode, this, 0, 0);
-        if (entry_read_subvol != par_read_subvol)
-                return -1;
-
+    int gen = 0;
+    int entry_read_subvol = 0;
+    unsigned char *data_readable = NULL;
+    unsigned char *metadata_readable = NULL;
+    afr_private_t *priv = NULL;
+
+    priv = this->private;
+    data_readable = alloca0(priv->child_count);
+    metadata_readable = alloca0(priv->child_count);
+
+    afr_inode_read_subvol_get(inode, this, data_readable, metadata_readable,
+                              &gen);
+
+    if (gen != priv->event_generation || !data_readable[par_read_subvol] ||
+        !metadata_readable[par_read_subvol])
+        return -1;
+
+    /* Once the control reaches the following statement, it means that the
+     * parent's read subvol is perfectly readable. So calling
+     * either afr_data_subvol_get() or afr_metadata_subvol_get() would
+     * yield the same result. Hence, choosing afr_data_subvol_get() below.
+     */
+
+    if (!priv->consistent_metadata)
         return 0;
 
+    /* For an inode fetched through readdirp which is yet to be linked,
+     * inode ctx would not be initialised (yet). So this function returns
+     * -1 above due to gen being 0, which is why it is OK to pass NULL for
+     *  read_subvol_args here.
+     */
+    entry_read_subvol = afr_data_subvol_get(inode, this, NULL, NULL, NULL,
+                                            NULL);
+    if (entry_read_subvol != par_read_subvol)
+        return -1;
+
+    return 0;
 }
 
 static void
-afr_readdir_transform_entries (gf_dirent_t *subvol_entries, int subvol,
-			       gf_dirent_t *entries, fd_t *fd)
+afr_readdir_transform_entries(call_frame_t *frame, gf_dirent_t *subvol_entries,
+                              int subvol, gf_dirent_t *entries, fd_t *fd)
 {
-        int            ret   = -1;
-        gf_dirent_t   *entry = NULL;
-        gf_dirent_t   *tmp   = NULL;
-        xlator_t      *this  = NULL;
-
-        this = THIS;
-
-        list_for_each_entry_safe (entry, tmp, &subvol_entries->list, list) {
-                if (__is_root_gfid (fd->inode->gfid) &&
-                    !strcmp (entry->d_name, GF_REPLICATE_TRASH_DIR)) {
-			continue;
-                }
-
-		list_del_init (&entry->list);
-		list_add_tail (&entry->list, &entries->list);
-
-		if (entry->inode) {
-                        ret = afr_validate_read_subvol (entry->inode, this,
-                                                        subvol);
-                        if (ret == -1) {
-				inode_unref (entry->inode);
-				entry->inode = NULL;
-                                continue;
-			}
-		}
+    int ret = -1;
+    gf_dirent_t *entry = NULL;
+    gf_dirent_t *tmp = NULL;
+    xlator_t *this = NULL;
+    afr_private_t *priv = NULL;
+    gf_boolean_t need_heal = _gf_false;
+    gf_boolean_t validate_subvol = _gf_false;
+
+    this = THIS;
+    priv = this->private;
+
+    need_heal = afr_get_need_heal(this);
+    validate_subvol = need_heal | priv->consistent_metadata;
+
+    list_for_each_entry_safe(entry, tmp, &subvol_entries->list, list)
+    {
+        if (afr_is_private_directory(priv, fd->inode->gfid, entry->d_name,
+                                     frame->root->pid)) {
+            continue;
         }
-}
 
+        list_del_init(&entry->list);
+        list_add_tail(&entry->list, &entries->list);
+
+        if (!validate_subvol)
+            continue;
+
+        if (entry->inode) {
+            ret = afr_validate_read_subvol(entry->inode, this, subvol);
+            if (ret == -1) {
+                inode_unref(entry->inode);
+                entry->inode = NULL;
+                continue;
+            }
+        }
+    }
+}
 
 int32_t
-afr_readdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-		 int32_t op_ret, int32_t op_errno, gf_dirent_t *subvol_entries,
-		 dict_t *xdata)
+afr_readdir_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                int32_t op_ret, int32_t op_errno, gf_dirent_t *subvol_entries,
+                dict_t *xdata)
 {
-        afr_local_t *local = NULL;
-	gf_dirent_t  entries;
+    afr_local_t *local = NULL;
+    gf_dirent_t entries;
 
-	INIT_LIST_HEAD (&entries.list);
+    INIT_LIST_HEAD(&entries.list);
 
-        local = frame->local;
+    local = frame->local;
 
-        if (op_ret < 0 && !local->cont.readdir.offset) {
-		/* failover only if this was first readdir, detected
-		   by offset == 0 */
-		local->op_ret = op_ret;
-		local->op_errno = op_errno;
+    if (op_ret < 0 && !local->cont.readdir.offset) {
+        /* failover only if this was first readdir, detected
+           by offset == 0 */
+        local->op_ret = op_ret;
+        local->op_errno = op_errno;
 
-		afr_read_txn_continue (frame, this, (long) cookie);
-		return 0;
-	}
+        afr_read_txn_continue(frame, this, (long)cookie);
+        return 0;
+    }
 
-	if (op_ret >= 0)
-		afr_readdir_transform_entries (subvol_entries, (long) cookie,
-					       &entries, local->fd);
+    if (op_ret >= 0)
+        afr_readdir_transform_entries(frame, subvol_entries, (long)cookie,
+                                      &entries, local->fd);
 
-        AFR_STACK_UNWIND (readdir, frame, op_ret, op_errno, &entries, xdata);
+    AFR_STACK_UNWIND(readdir, frame, op_ret, op_errno, &entries, xdata);
 
-        gf_dirent_free (&entries);
+    gf_dirent_free(&entries);
 
-        return 0;
+    return 0;
 }
 
-
 int
-afr_readdir_wind (call_frame_t *frame, xlator_t *this, int subvol)
+afr_readdir_wind(call_frame_t *frame, xlator_t *this, int subvol)
 {
-	afr_local_t *local = NULL;
-	afr_private_t *priv = NULL;
-	afr_fd_ctx_t *fd_ctx = NULL;
-
-	priv = this->private;
-	local = frame->local;
-	fd_ctx = afr_fd_ctx_get (local->fd, this);
-
-	if (subvol == -1) {
-		AFR_STACK_UNWIND (readdir, frame, local->op_ret,
-				  local->op_errno, 0, 0);
-		return 0;
-	}
-
-	fd_ctx->readdir_subvol = subvol;
-
-        if (local->op == GF_FOP_READDIR)
-                STACK_WIND_COOKIE (frame, afr_readdir_cbk,
-                                   (void *) (long) subvol,
-                                   priv->children[subvol],
-                                   priv->children[subvol]->fops->readdir,
-				   local->fd, local->cont.readdir.size,
-                                   local->cont.readdir.offset,
-				   local->xdata_req);
-        else
-                STACK_WIND_COOKIE (frame, afr_readdir_cbk,
-                                   (void *) (long) subvol,
-                                   priv->children[subvol],
-                                   priv->children[subvol]->fops->readdirp,
-				   local->fd, local->cont.readdir.size,
-				   local->cont.readdir.offset,
-				   local->xdata_req);
-	return 0;
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
+    afr_fd_ctx_t *fd_ctx = NULL;
+
+    priv = this->private;
+    local = frame->local;
+    fd_ctx = afr_fd_ctx_get(local->fd, this);
+    if (!fd_ctx) {
+        local->op_errno = EINVAL;
+        local->op_ret = -1;
+    }
+
+    if (subvol == -1 || !fd_ctx) {
+        AFR_STACK_UNWIND(readdir, frame, local->op_ret, local->op_errno, 0, 0);
+        return 0;
+    }
+
+    fd_ctx->readdir_subvol = subvol;
+
+    if (local->op == GF_FOP_READDIR)
+        STACK_WIND_COOKIE(frame, afr_readdir_cbk, (void *)(long)subvol,
+                          priv->children[subvol],
+                          priv->children[subvol]->fops->readdir, local->fd,
+                          local->cont.readdir.size, local->cont.readdir.offset,
+                          local->xdata_req);
+    else
+        STACK_WIND_COOKIE(frame, afr_readdir_cbk, (void *)(long)subvol,
+                          priv->children[subvol],
+                          priv->children[subvol]->fops->readdirp, local->fd,
+                          local->cont.readdir.size, local->cont.readdir.offset,
+                          local->xdata_req);
+    return 0;
 }
 
-
 int
-afr_do_readdir (call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
-		off_t offset, int whichop, dict_t *dict)
+afr_do_readdir(call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
+               off_t offset, int whichop, dict_t *dict)
 {
-        afr_local_t   *local     = NULL;
-        int32_t       op_errno   = 0;
-	int           subvol = -1;
-	afr_fd_ctx_t *fd_ctx = NULL;
-
-	local = AFR_FRAME_INIT (frame, op_errno);
-	if (!local)
-		goto out;
-
-	fd_ctx = afr_fd_ctx_get (fd, this);
-	if (!fd_ctx) {
-	        op_errno = EINVAL;
-		goto out;
-        }
-
-	local->op = whichop;
-        local->fd = fd_ref (fd);
-        local->cont.readdir.size = size;
-	local->cont.readdir.offset = offset;
-        local->xdata_req = (dict)? dict_ref (dict) : NULL;
-
-	subvol = fd_ctx->readdir_subvol;
-
-	if (offset == 0 || subvol == -1) {
-		/* First readdir has option of failing over and selecting
-		   an appropriate read subvolume */
-		afr_read_txn (frame, this, fd->inode, afr_readdir_wind,
-			      AFR_DATA_TRANSACTION);
-	} else {
-		/* But continued readdirs MUST stick to the same subvolume
-		   without an option to failover */
-		afr_readdir_wind (frame, this, subvol);
-	}
-
-        return 0;
+    afr_local_t *local = NULL;
+    int32_t op_errno = 0;
+    int subvol = -1;
+    afr_fd_ctx_t *fd_ctx = NULL;
+
+    local = AFR_FRAME_INIT(frame, op_errno);
+    if (!local)
+        goto out;
+
+    fd_ctx = afr_fd_ctx_get(fd, this);
+    if (!fd_ctx) {
+        op_errno = EINVAL;
+        goto out;
+    }
+
+    local->op = whichop;
+    local->fd = fd_ref(fd);
+    local->cont.readdir.size = size;
+    local->cont.readdir.offset = offset;
+    local->xdata_req = (dict) ? dict_ref(dict) : NULL;
+
+    subvol = fd_ctx->readdir_subvol;
+
+    if (offset == 0 || subvol == -1) {
+        /* First readdir has option of failing over and selecting
+           an appropriate read subvolume */
+        afr_read_txn(frame, this, fd->inode, afr_readdir_wind,
+                     AFR_DATA_TRANSACTION);
+    } else {
+        /* But continued readdirs MUST stick to the same subvolume
+           without an option to failover */
+        afr_readdir_wind(frame, this, subvol);
+    }
+
+    return 0;
 out:
-        AFR_STACK_UNWIND (readdir, frame, -1, op_errno, NULL, NULL);
-        return 0;
+    AFR_STACK_UNWIND(readdir, frame, -1, op_errno, NULL, NULL);
+    return 0;
 }
 
-
 int32_t
-afr_readdir (call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
-             off_t offset, dict_t *xdata)
+afr_readdir(call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
+            off_t offset, dict_t *xdata)
 {
-	afr_do_readdir (frame, this, fd, size, offset, GF_FOP_READDIR, xdata);
+    afr_do_readdir(frame, this, fd, size, offset, GF_FOP_READDIR, xdata);
 
-        return 0;
+    return 0;
 }
 
-
 int32_t
-afr_readdirp (call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
-              off_t offset, dict_t *dict)
+afr_readdirp(call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
+             off_t offset, dict_t *dict)
 {
-        afr_do_readdir (frame, this, fd, size, offset, GF_FOP_READDIRP, dict);
+    afr_do_readdir(frame, this, fd, size, offset, GF_FOP_READDIRP, dict);
 
-        return 0;
+    return 0;
 }
 
-
 int32_t
-afr_releasedir (xlator_t *this, fd_t *fd)
+afr_releasedir(xlator_t *this, fd_t *fd)
 {
-        afr_cleanup_fd_ctx (this, fd);
+    afr_cleanup_fd_ctx(this, fd);
 
-        return 0;
+    return 0;
 }
diff --git a/xlators/cluster/afr/src/afr-dir-read.h b/xlators/cluster/afr/src/afr-dir-read.h
index 09456d15949..773e925ec6c 100644
--- a/xlators/cluster/afr/src/afr-dir-read.h
+++ b/xlators/cluster/afr/src/afr-dir-read.h
@@ -11,26 +11,23 @@
 #ifndef __DIR_READ_H__
 #define __DIR_READ_H__
 
-
 int32_t
-afr_opendir (call_frame_t *frame, xlator_t *this,
-	     loc_t *loc, fd_t *fd, dict_t *xdata);
+afr_opendir(call_frame_t *frame, xlator_t *this, loc_t *loc, fd_t *fd,
+            dict_t *xdata);
 
 int32_t
-afr_releasedir (xlator_t *this, fd_t *fd);
+afr_releasedir(xlator_t *this, fd_t *fd);
 
 int32_t
-afr_readdir (call_frame_t *frame, xlator_t *this,
-	     fd_t *fd, size_t size, off_t offset, dict_t *xdata);
-
+afr_readdir(call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
+            off_t offset, dict_t *xdata);
 
 int32_t
-afr_readdirp (call_frame_t *frame, xlator_t *this,
-              fd_t *fd, size_t size, off_t offset, dict_t *dict);
+afr_readdirp(call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
+             off_t offset, dict_t *dict);
 
 int32_t
-afr_checksum (call_frame_t *frame, xlator_t *this,
-	      loc_t *loc, int32_t flags, dict_t *xdata);
-
+afr_checksum(call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t flags,
+             dict_t *xdata);
 
 #endif /* __DIR_READ_H__ */
diff --git a/xlators/cluster/afr/src/afr-dir-write.c b/xlators/cluster/afr/src/afr-dir-write.c
index f7ca9108092..b7cceb79158 100644
--- a/xlators/cluster/afr/src/afr-dir-write.c
+++ b/xlators/cluster/afr/src/afr-dir-write.c
@@ -8,506 +8,493 @@
   cases as published by the Free Software Foundation.
 */
 
-
 #include <libgen.h>
 #include <unistd.h>
-#include <fnmatch.h>
 #include <sys/time.h>
 #include <stdlib.h>
 #include <signal.h>
 
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
-
-#include "glusterfs.h"
+#include <glusterfs/glusterfs.h>
 #include "afr.h"
-#include "dict.h"
-#include "xlator.h"
-#include "hashfn.h"
-#include "logging.h"
-#include "stack.h"
-#include "list.h"
-#include "call-stub.h"
-#include "defaults.h"
-#include "common-utils.h"
-#include "compat-errno.h"
-#include "compat.h"
-#include "byte-order.h"
+#include <glusterfs/dict.h>
+#include <glusterfs/logging.h>
+#include <glusterfs/list.h>
+#include <glusterfs/defaults.h>
+#include <glusterfs/common-utils.h>
+#include <glusterfs/compat-errno.h>
+#include <glusterfs/compat.h>
+#include <glusterfs/byte-order.h>
 
 #include "afr.h"
 #include "afr-transaction.h"
 
 void
-afr_mark_entry_pending_changelog (call_frame_t *frame, xlator_t *this);
+afr_mark_entry_pending_changelog(call_frame_t *frame, xlator_t *this);
 
 int
-afr_build_parent_loc (loc_t *parent, loc_t *child, int32_t *op_errno)
+afr_build_parent_loc(loc_t *parent, loc_t *child, int32_t *op_errno)
 {
-        int     ret = -1;
-        char    *child_path = NULL;
+    int ret = -1;
+    char *child_path = NULL;
+
+    if (!child->parent) {
+        if (op_errno)
+            *op_errno = EINVAL;
+        goto out;
+    }
+
+    child_path = gf_strdup(child->path);
+    if (!child_path) {
+        if (op_errno)
+            *op_errno = ENOMEM;
+        goto out;
+    }
+
+    parent->path = gf_strdup(dirname(child_path));
+    if (!parent->path) {
+        if (op_errno)
+            *op_errno = ENOMEM;
+        goto out;
+    }
+
+    parent->inode = inode_ref(child->parent);
+    gf_uuid_copy(parent->gfid, child->pargfid);
+
+    ret = 0;
+out:
+    GF_FREE(child_path);
 
-        if (!child->parent) {
-                if (op_errno)
-                        *op_errno = EINVAL;
-                goto out;
-        }
+    return ret;
+}
 
-        child_path = gf_strdup (child->path);
-        if (!child_path) {
-                if (op_errno)
-                        *op_errno = ENOMEM;
-                goto out;
+static void
+__afr_dir_write_finalize(call_frame_t *frame, xlator_t *this)
+{
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
+    int inode_read_subvol = -1;
+    int parent_read_subvol = -1;
+    int parent2_read_subvol = -1;
+    int i = 0;
+    afr_read_subvol_args_t args = {
+        0,
+    };
+
+    local = frame->local;
+    priv = this->private;
+
+    for (i = 0; i < priv->child_count; i++) {
+        if (!local->replies[i].valid)
+            continue;
+        if (local->replies[i].op_ret == -1)
+            continue;
+        gf_uuid_copy(args.gfid, local->replies[i].poststat.ia_gfid);
+        args.ia_type = local->replies[i].poststat.ia_type;
+        break;
+    }
+
+    if (local->inode) {
+        if (local->op != GF_FOP_RENAME && local->op != GF_FOP_LINK)
+            afr_replies_interpret(frame, this, local->inode, NULL);
+
+        inode_read_subvol = afr_data_subvol_get(local->inode, this, NULL, NULL,
+                                                NULL, &args);
+    }
+
+    if (local->parent)
+        parent_read_subvol = afr_data_subvol_get(local->parent, this, NULL,
+                                                 local->readable, NULL, NULL);
+
+    if (local->parent2)
+        parent2_read_subvol = afr_data_subvol_get(local->parent2, this, NULL,
+                                                  local->readable2, NULL, NULL);
+
+    local->op_ret = -1;
+    local->op_errno = afr_final_errno(local, priv);
+    afr_pick_error_xdata(local, priv, local->parent, local->readable,
+                         local->parent2, local->readable2);
+
+    for (i = 0; i < priv->child_count; i++) {
+        if (!local->replies[i].valid)
+            continue;
+        if (local->replies[i].op_ret < 0) {
+            if (local->inode)
+                afr_inode_need_refresh_set(local->inode, this);
+            if (local->parent)
+                afr_inode_need_refresh_set(local->parent, this);
+            if (local->parent2)
+                afr_inode_need_refresh_set(local->parent2, this);
+            continue;
         }
 
-        parent->path = gf_strdup (dirname (child_path));
-        if (!parent->path) {
-                if (op_errno)
-                        *op_errno = ENOMEM;
-                goto out;
+        if (local->op_ret == -1) {
+            local->op_ret = local->replies[i].op_ret;
+            local->op_errno = local->replies[i].op_errno;
+
+            local->cont.dir_fop.buf = local->replies[i].poststat;
+            local->cont.dir_fop.preparent = local->replies[i].preparent;
+            local->cont.dir_fop.postparent = local->replies[i].postparent;
+            local->cont.dir_fop.prenewparent = local->replies[i].preparent2;
+            local->cont.dir_fop.postnewparent = local->replies[i].postparent2;
+            if (local->xdata_rsp) {
+                dict_unref(local->xdata_rsp);
+                local->xdata_rsp = NULL;
+            }
+
+            if (local->replies[i].xdata)
+                local->xdata_rsp = dict_ref(local->replies[i].xdata);
+            continue;
         }
 
-        parent->inode = inode_ref (child->parent);
-	gf_uuid_copy (parent->gfid, child->pargfid);
-
-        ret = 0;
-out:
-        GF_FREE (child_path);
-
-        return ret;
-}
+        if (i == inode_read_subvol) {
+            local->cont.dir_fop.buf = local->replies[i].poststat;
+            if (local->replies[i].xdata) {
+                if (local->xdata_rsp)
+                    dict_unref(local->xdata_rsp);
+                local->xdata_rsp = dict_ref(local->replies[i].xdata);
+            }
+        }
 
+        if (i == parent_read_subvol) {
+            local->cont.dir_fop.preparent = local->replies[i].preparent;
+            local->cont.dir_fop.postparent = local->replies[i].postparent;
+        }
 
-static void
-__afr_dir_write_finalize (call_frame_t *frame, xlator_t *this)
-{
-	afr_local_t *local = NULL;
-	afr_private_t *priv = NULL;
-	int inode_read_subvol = -1;
-	int parent_read_subvol = -1;
-	int parent2_read_subvol = -1;
-	int i = 0;
-
-	local = frame->local;
-	priv = this->private;
-
-	if (local->inode) {
-		afr_replies_interpret (frame, this, local->inode);
-		inode_read_subvol = afr_data_subvol_get (local->inode, this,
-							 NULL, NULL);
-	}
-	if (local->parent)
-		parent_read_subvol = afr_data_subvol_get (local->parent, this,
-							  NULL, NULL);
-	if (local->parent2)
-		parent2_read_subvol = afr_data_subvol_get (local->parent2, this,
-							   NULL, NULL);
-
-	local->op_ret = -1;
-	local->op_errno = afr_final_errno (local, priv);
-
-	for (i = 0; i < priv->child_count; i++) {
-		if (!local->replies[i].valid)
-			continue;
-		if (local->replies[i].op_ret < 0) {
-			if (local->inode)
-				afr_inode_read_subvol_reset (local->inode,
-							     this);
-			if (local->parent)
-				afr_inode_read_subvol_reset (local->parent,
-							     this);
-			if (local->parent2)
-				afr_inode_read_subvol_reset (local->parent2,
-							     this);
-			continue;
-		}
-
-		if (local->op_ret == -1) {
-			local->op_ret = local->replies[i].op_ret;
-			local->op_errno = local->replies[i].op_errno;
-
-			local->cont.dir_fop.buf =
-				local->replies[i].poststat;
-			local->cont.dir_fop.preparent =
-				local->replies[i].preparent;
-			local->cont.dir_fop.postparent =
-				local->replies[i].postparent;
-			local->cont.dir_fop.prenewparent =
-				local->replies[i].preparent2;
-			local->cont.dir_fop.postnewparent =
-				local->replies[i].postparent2;
-			if (local->replies[i].xdata)
-				local->xdata_rsp =
-					dict_ref (local->replies[i].xdata);
-			continue;
-		}
-
-		if (i == inode_read_subvol) {
-			local->cont.dir_fop.buf =
-				local->replies[i].poststat;
-			if (local->replies[i].xdata) {
-				if (local->xdata_rsp)
-					dict_unref (local->xdata_rsp);
-				local->xdata_rsp =
-					dict_ref (local->replies[i].xdata);
-			}
-		}
-
-		if (i == parent_read_subvol) {
-			local->cont.dir_fop.preparent =
-				local->replies[i].preparent;
-			local->cont.dir_fop.postparent =
-				local->replies[i].postparent;
-		}
-
-		if (i == parent2_read_subvol) {
-			local->cont.dir_fop.prenewparent =
-				local->replies[i].preparent2;
-			local->cont.dir_fop.postnewparent =
-				local->replies[i].postparent2;
-		}
-	}
+        if (i == parent2_read_subvol) {
+            local->cont.dir_fop.prenewparent = local->replies[i].preparent2;
+            local->cont.dir_fop.postnewparent = local->replies[i].postparent2;
+        }
+    }
 }
 
-
 static void
-__afr_dir_write_fill (call_frame_t *frame, xlator_t *this, int child_index,
-		      int op_ret, int op_errno, struct iatt *poststat,
-		      struct iatt *preparent, struct iatt *postparent,
-		      struct iatt *preparent2, struct iatt *postparent2,
-		      dict_t *xdata)
+__afr_dir_write_fill(call_frame_t *frame, xlator_t *this, int child_index,
+                     int op_ret, int op_errno, struct iatt *poststat,
+                     struct iatt *preparent, struct iatt *postparent,
+                     struct iatt *preparent2, struct iatt *postparent2,
+                     dict_t *xdata)
 {
-        afr_local_t *local = NULL;
-	afr_fd_ctx_t *fd_ctx = NULL;
-
-        local = frame->local;
-	fd_ctx = local->fd_ctx;
-
-	local->replies[child_index].valid = 1;
-	local->replies[child_index].op_ret = op_ret;
-	local->replies[child_index].op_errno = op_errno;
-
-	if (op_ret >= 0) {
-		if (poststat)
-			local->replies[child_index].poststat = *poststat;
-		if (preparent)
-			local->replies[child_index].preparent = *preparent;
-		if (postparent)
-			local->replies[child_index].postparent = *postparent;
-		if (preparent2)
-			local->replies[child_index].preparent2 = *preparent2;
-		if (postparent2)
-			local->replies[child_index].postparent2 = *postparent2;
-		if (xdata)
-			local->replies[child_index].xdata = dict_ref (xdata);
-
-		if (fd_ctx)
-			fd_ctx->opened_on[child_index] = AFR_FD_OPENED;
-	} else {
-		if (op_errno != ENOTEMPTY)
-			afr_transaction_fop_failed (frame, this, child_index);
-		if (fd_ctx)
-			fd_ctx->opened_on[child_index] = AFR_FD_NOT_OPENED;
-	}
-
-        return;
+    afr_local_t *local = NULL;
+    afr_fd_ctx_t *fd_ctx = NULL;
+
+    local = frame->local;
+    fd_ctx = local->fd_ctx;
+
+    local->replies[child_index].valid = 1;
+    local->replies[child_index].op_ret = op_ret;
+    local->replies[child_index].op_errno = op_errno;
+    if (xdata)
+        local->replies[child_index].xdata = dict_ref(xdata);
+
+    if (op_ret >= 0) {
+        if (poststat)
+            local->replies[child_index].poststat = *poststat;
+        if (preparent)
+            local->replies[child_index].preparent = *preparent;
+        if (postparent)
+            local->replies[child_index].postparent = *postparent;
+        if (preparent2)
+            local->replies[child_index].preparent2 = *preparent2;
+        if (postparent2)
+            local->replies[child_index].postparent2 = *postparent2;
+        if (fd_ctx)
+            fd_ctx->opened_on[child_index] = AFR_FD_OPENED;
+    } else {
+        if (op_errno != ENOTEMPTY)
+            afr_transaction_fop_failed(frame, this, child_index);
+        if (fd_ctx)
+            fd_ctx->opened_on[child_index] = AFR_FD_NOT_OPENED;
+    }
+
+    return;
 }
 
-
 static int
-__afr_dir_write_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-		     int op_ret, int op_errno, struct iatt *buf,
-		     struct iatt *preparent, struct iatt *postparent,
-		     struct iatt *preparent2, struct iatt *postparent2,
-                     dict_t *xdata)
+__afr_dir_write_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                    int op_ret, int op_errno, struct iatt *buf,
+                    struct iatt *preparent, struct iatt *postparent,
+                    struct iatt *preparent2, struct iatt *postparent2,
+                    dict_t *xdata)
 {
-        afr_local_t *local = NULL;
-        int child_index = (long) cookie;
-        int call_count = -1;
-
-        local = frame->local;
-
-	LOCK (&frame->lock);
-	{
-		__afr_dir_write_fill (frame, this, child_index, op_ret,
-				      op_errno, buf, preparent, postparent,
-				      preparent2, postparent2, xdata);
-	}
-	UNLOCK (&frame->lock);
-        call_count = afr_frame_return (frame);
-
-        if (call_count == 0) {
-		__afr_dir_write_finalize (frame, this);
-
-		if (afr_txn_nothing_failed (frame, this))
-			local->transaction.unwind (frame, this);
+    afr_local_t *local = NULL;
+    int child_index = (long)cookie;
+    int call_count = -1;
+    afr_private_t *priv = NULL;
+
+    priv = this->private;
+    local = frame->local;
+
+    LOCK(&frame->lock);
+    {
+        __afr_dir_write_fill(frame, this, child_index, op_ret, op_errno, buf,
+                             preparent, postparent, preparent2, postparent2,
+                             xdata);
+        call_count = --local->call_count;
+    }
+    UNLOCK(&frame->lock);
+
+    if (call_count == 0) {
+        __afr_dir_write_finalize(frame, this);
+
+        if (afr_txn_nothing_failed(frame, this)) {
+            /*if it did pre-op, it will do post-op changing ctime*/
+            if (priv->consistent_metadata && afr_needs_changelog_update(local))
+                afr_zero_fill_stat(local);
+            local->transaction.unwind(frame, this);
+        }
 
-		afr_mark_entry_pending_changelog (frame, this);
+        afr_mark_entry_pending_changelog(frame, this);
 
-                local->transaction.resume (frame, this);
-        }
+        afr_transaction_resume(frame, this);
+    }
 
-        return 0;
+    return 0;
 }
 
-
 int
-afr_mark_new_entry_changelog_cbk (call_frame_t *frame, void *cookie,
-				  xlator_t *this, int op_ret, int op_errno,
-                                  dict_t *xattr, dict_t *xdata)
+afr_mark_new_entry_changelog_cbk(call_frame_t *frame, void *cookie,
+                                 xlator_t *this, int op_ret, int op_errno,
+                                 dict_t *xattr, dict_t *xdata)
 {
-        int call_count = 0;
+    int call_count = 0;
 
-        call_count = afr_frame_return (frame);
+    call_count = afr_frame_return(frame);
 
-        if (call_count == 0)
-                AFR_STACK_DESTROY (frame);
+    if (call_count == 0)
+        AFR_STACK_DESTROY(frame);
 
-        return 0;
+    return 0;
 }
 
-
 void
-afr_mark_new_entry_changelog (call_frame_t *frame, xlator_t *this)
+afr_mark_new_entry_changelog(call_frame_t *frame, xlator_t *this)
 {
-        call_frame_t  *new_frame  = NULL;
-        afr_local_t   *local      = NULL;
-        afr_local_t   *new_local  = NULL;
-        afr_private_t *priv       = NULL;
-        dict_t        *xattr      = NULL;
-        int32_t       **changelog = NULL;
-        int           i           = 0;
-        int           op_errno    = ENOMEM;
-	unsigned char *pending    = NULL;
-	int           call_count   = 0;
-
-        local = frame->local;
-        priv = this->private;
-
-        new_frame = copy_frame (frame);
-        if (!new_frame)
-                goto out;
-
-	new_local = AFR_FRAME_INIT (new_frame, op_errno);
-	if (!new_local)
-		goto out;
-
-        xattr = dict_new ();
-	if (!xattr)
-		goto out;
-
-	pending = alloca0 (priv->child_count);
-
-	for (i = 0; i < priv->child_count; i++) {
-		if (local->transaction.pre_op[i] &&
-		    !local->transaction.failed_subvols[i]) {
-			call_count ++;
-			continue;
-		}
-		pending[i] = 1;
-	}
-
-        changelog = afr_mark_pending_changelog (priv, pending, xattr,
-                                            local->cont.dir_fop.buf.ia_type);
-        if (!changelog)
-                goto out;
-
-        new_local->pending = changelog;
-        gf_uuid_copy (new_local->loc.gfid, local->cont.dir_fop.buf.ia_gfid);
-        new_local->loc.inode = inode_ref (local->inode);
-
-        new_local->call_count = call_count;
-
-        for (i = 0; i < priv->child_count; i++) {
-		if (pending[i])
-                        continue;
-
-                STACK_WIND_COOKIE (new_frame, afr_mark_new_entry_changelog_cbk,
-                                   (void *) (long) i, priv->children[i],
-                                   priv->children[i]->fops->xattrop,
-                                   &new_local->loc, GF_XATTROP_ADD_ARRAY,
-                                   xattr, NULL);
-		if (!--call_count)
-			break;
+    call_frame_t *new_frame = NULL;
+    afr_local_t *local = NULL;
+    afr_local_t *new_local = NULL;
+    afr_private_t *priv = NULL;
+    dict_t *xattr = NULL;
+    int32_t **changelog = NULL;
+    int i = 0;
+    int op_errno = ENOMEM;
+    unsigned char *pending = NULL;
+    int call_count = 0;
+
+    local = frame->local;
+    priv = this->private;
+
+    new_frame = copy_frame(frame);
+    if (!new_frame)
+        goto out;
+
+    new_local = AFR_FRAME_INIT(new_frame, op_errno);
+    if (!new_local)
+        goto out;
+
+    xattr = dict_new();
+    if (!xattr)
+        goto out;
+
+    pending = alloca0(priv->child_count);
+
+    for (i = 0; i < priv->child_count; i++) {
+        if (local->transaction.pre_op[i] &&
+            !local->transaction.failed_subvols[i]) {
+            call_count++;
+            continue;
         }
+        pending[i] = 1;
+    }
+
+    changelog = afr_mark_pending_changelog(priv, pending, xattr,
+                                           local->cont.dir_fop.buf.ia_type);
+    if (!changelog)
+        goto out;
+
+    new_local->pending = changelog;
+    gf_uuid_copy(new_local->loc.gfid, local->cont.dir_fop.buf.ia_gfid);
+    new_local->loc.inode = inode_ref(local->inode);
+
+    new_local->call_count = call_count;
 
-        new_frame = NULL;
+    for (i = 0; i < priv->child_count; i++) {
+        if (pending[i])
+            continue;
+
+        STACK_WIND_COOKIE(new_frame, afr_mark_new_entry_changelog_cbk,
+                          (void *)(long)i, priv->children[i],
+                          priv->children[i]->fops->xattrop, &new_local->loc,
+                          GF_XATTROP_ADD_ARRAY, xattr, NULL);
+        if (!--call_count)
+            break;
+    }
+
+    new_frame = NULL;
 out:
-        if (new_frame)
-                AFR_STACK_DESTROY (new_frame);
-	if (xattr)
-		dict_unref (xattr);
-        return;
+    if (new_frame)
+        AFR_STACK_DESTROY(new_frame);
+    if (xattr)
+        dict_unref(xattr);
+    return;
 }
 
-
 void
-afr_mark_entry_pending_changelog (call_frame_t *frame, xlator_t *this)
+afr_mark_entry_pending_changelog(call_frame_t *frame, xlator_t *this)
 {
-        afr_local_t *local = NULL;
-        afr_private_t *priv = NULL;
-	int pre_op_count = 0;
-	int failed_count = 0;
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
+    int pre_op_count = 0;
+    int failed_count = 0;
+    unsigned char *success_replies = NULL;
 
-        local = frame->local;
-        priv  = this->private;
+    local = frame->local;
+    priv = this->private;
 
-        if (local->op_ret < 0)
-		return;
+    if (local->op_ret < 0)
+        return;
 
-	if (local->op != GF_FOP_CREATE && local->op != GF_FOP_MKNOD)
-		return;
+    if (local->op != GF_FOP_CREATE && local->op != GF_FOP_MKNOD &&
+        local->op != GF_FOP_MKDIR)
+        return;
 
-	pre_op_count = AFR_COUNT (local->transaction.pre_op, priv->child_count);
-	failed_count = AFR_COUNT (local->transaction.failed_subvols,
-				  priv->child_count);
+    pre_op_count = AFR_COUNT(local->transaction.pre_op, priv->child_count);
+    failed_count = AFR_COUNT(local->transaction.failed_subvols,
+                             priv->child_count);
 
-	if (pre_op_count == priv->child_count && !failed_count)
-		return;
+    /* FOP succeeded on all bricks. */
+    if (pre_op_count == priv->child_count && !failed_count)
+        return;
 
-        afr_mark_new_entry_changelog (frame, this);
+    /* FOP did not suceed on quorum no. of bricks. */
+    success_replies = alloca0(priv->child_count);
+    afr_fill_success_replies(local, priv, success_replies);
+    if (!afr_has_quorum(success_replies, this, NULL))
+        return;
 
+    if (priv->thin_arbiter_count) {
+        /*Mark new entry using ta file*/
+        local->is_new_entry = _gf_true;
         return;
-}
+    }
+
+    afr_mark_new_entry_changelog(frame, this);
 
+    return;
+}
 
 /* {{{ create */
 
 int
-afr_create_unwind (call_frame_t *frame, xlator_t *this)
+afr_create_unwind(call_frame_t *frame, xlator_t *this)
 {
-        call_frame_t *main_frame = NULL;
-        afr_local_t  *local = NULL;
-
-        local = frame->local;
+    call_frame_t *main_frame = NULL;
+    afr_local_t *local = NULL;
 
-	main_frame = afr_transaction_detach_fop_frame (frame);
+    local = frame->local;
 
-        if (!main_frame)
-		return 0;
+    main_frame = afr_transaction_detach_fop_frame(frame);
 
-	AFR_STACK_UNWIND (create, main_frame, local->op_ret, local->op_errno,
-			  local->cont.create.fd, local->inode,
-			  &local->cont.dir_fop.buf,
-			  &local->cont.dir_fop.preparent,
-			  &local->cont.dir_fop.postparent, local->xdata_rsp);
+    if (!main_frame)
         return 0;
-}
 
+    AFR_STACK_UNWIND(create, main_frame, local->op_ret, local->op_errno,
+                     local->cont.create.fd, local->inode,
+                     &local->cont.dir_fop.buf, &local->cont.dir_fop.preparent,
+                     &local->cont.dir_fop.postparent, local->xdata_rsp);
+    return 0;
+}
 
 int
-afr_create_wind_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                     int32_t op_ret, int32_t op_errno,
-                     fd_t *fd, inode_t *inode, struct iatt *buf,
-                     struct iatt *preparent, struct iatt *postparent,
-                     dict_t *xdata)
+afr_create_wind_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                    int32_t op_ret, int32_t op_errno, fd_t *fd, inode_t *inode,
+                    struct iatt *buf, struct iatt *preparent,
+                    struct iatt *postparent, dict_t *xdata)
 {
-        return __afr_dir_write_cbk (frame, cookie, this, op_ret, op_errno, buf,
-				    preparent, postparent, NULL, NULL, xdata);
+    return __afr_dir_write_cbk(frame, cookie, this, op_ret, op_errno, buf,
+                               preparent, postparent, NULL, NULL, xdata);
 }
 
-
 int
-afr_create_wind (call_frame_t *frame, xlator_t *this, int subvol)
+afr_create_wind(call_frame_t *frame, xlator_t *this, int subvol)
 {
-        afr_local_t *local = NULL;
-        afr_private_t *priv = NULL;
-
-        local = frame->local;
-        priv = this->private;
-
-	STACK_WIND_COOKIE (frame, afr_create_wind_cbk, (void *) (long) subvol,
-			   priv->children[subvol],
-			   priv->children[subvol]->fops->create,
-			   &local->loc, local->cont.create.flags,
-			   local->cont.create.mode, local->umask,
-			   local->cont.create.fd, local->xdata_req);
-        return 0;
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
+
+    local = frame->local;
+    priv = this->private;
+
+    STACK_WIND_COOKIE(frame, afr_create_wind_cbk, (void *)(long)subvol,
+                      priv->children[subvol],
+                      priv->children[subvol]->fops->create, &local->loc,
+                      local->cont.create.flags, local->cont.create.mode,
+                      local->umask, local->cont.create.fd, local->xdata_req);
+    return 0;
 }
 
-
 int
-afr_create (call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t flags,
-	    mode_t mode, mode_t umask, fd_t *fd, dict_t *xdata)
+afr_create(call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t flags,
+           mode_t mode, mode_t umask, fd_t *fd, dict_t *xdata)
 {
-        afr_private_t           *priv                   = NULL;
-        afr_local_t             *local                  = NULL;
-        afr_internal_lock_t     *int_lock               = NULL;
-        call_frame_t            *transaction_frame      = NULL;
-        int                     ret                     = -1;
-        int                     op_errno                = ENOMEM;
-
-        priv = this->private;
-
-        transaction_frame = copy_frame (frame);
-        if (!transaction_frame)
-                goto out;
-
-	local = AFR_FRAME_INIT (transaction_frame, op_errno);
-	if (!local)
-		goto out;
-
-        loc_copy (&local->loc, loc);
-
-	local->fd_ctx = afr_fd_ctx_get (fd, this);
-	if (!local->fd_ctx)
-		goto out;
-
-	local->inode = inode_ref (loc->inode);
-	local->parent = inode_ref (loc->parent);
-
-        local->op                = GF_FOP_CREATE;
-        local->cont.create.flags = flags;
-        local->cont.create.mode  = mode;
-        local->cont.create.fd    = fd_ref (fd);
-        local->umask  = umask;
-
-        if (xdata)
-                local->xdata_req = dict_copy_with_ref (xdata, NULL);
-	else
-		local->xdata_req = dict_new ();
-
-	if (!local->xdata_req)
-		goto out;
-
-        local->transaction.wind   = afr_create_wind;
-        local->transaction.fop    = __afr_txn_write_fop;
-        local->transaction.done   = __afr_txn_write_done;
-        local->transaction.unwind = afr_create_unwind;
-
-        ret = afr_build_parent_loc (&local->transaction.parent_loc, loc,
-                                    &op_errno);
-        if (ret)
-                goto out;
-
-        local->transaction.main_frame = frame;
-        local->transaction.basename = AFR_BASENAME (loc->path);
-        int_lock = &local->internal_lock;
-
-        int_lock->lockee_count = 0;
-        ret = afr_init_entry_lockee (&int_lock->lockee[0], local,
-                                     &local->transaction.parent_loc,
-                                     local->transaction.basename,
-                                     priv->child_count);
-        if (ret)
-                goto out;
-
-        int_lock->lockee_count++;
-        ret = afr_transaction (transaction_frame, this, AFR_ENTRY_TRANSACTION);
-        if (ret < 0) {
-            op_errno = -ret;
-            goto out;
-        }
-
-	return 0;
+    afr_local_t *local = NULL;
+    call_frame_t *transaction_frame = NULL;
+    int ret = -1;
+    int op_errno = ENOMEM;
+
+    transaction_frame = copy_frame(frame);
+    if (!transaction_frame)
+        goto out;
+
+    local = AFR_FRAME_INIT(transaction_frame, op_errno);
+    if (!local)
+        goto out;
+
+    loc_copy(&local->loc, loc);
+
+    local->fd_ctx = afr_fd_ctx_get(fd, this);
+    if (!local->fd_ctx)
+        goto out;
+
+    local->inode = inode_ref(loc->inode);
+    local->parent = inode_ref(loc->parent);
+
+    local->op = GF_FOP_CREATE;
+    local->cont.create.flags = flags;
+    local->fd_ctx->flags = flags;
+    local->cont.create.mode = mode;
+    local->cont.create.fd = fd_ref(fd);
+    local->umask = umask;
+
+    if (xdata)
+        local->xdata_req = dict_copy_with_ref(xdata, NULL);
+    else
+        local->xdata_req = dict_new();
+
+    if (!local->xdata_req)
+        goto out;
+
+    local->transaction.wind = afr_create_wind;
+    local->transaction.unwind = afr_create_unwind;
+
+    ret = afr_build_parent_loc(&local->transaction.parent_loc, loc, &op_errno);
+    if (ret)
+        goto out;
+
+    local->transaction.main_frame = frame;
+    local->transaction.basename = AFR_BASENAME(loc->path);
+    ret = afr_transaction(transaction_frame, this, AFR_ENTRY_TRANSACTION);
+    if (ret < 0) {
+        op_errno = -ret;
+        goto out;
+    }
+
+    return 0;
 out:
-	if (transaction_frame)
-		AFR_STACK_DESTROY (transaction_frame);
+    if (transaction_frame)
+        AFR_STACK_DESTROY(transaction_frame);
 
-	AFR_STACK_UNWIND (create, frame, -1, op_errno, NULL, NULL, NULL, NULL,
-			  NULL, NULL);
-        return 0;
+    AFR_STACK_UNWIND(create, frame, -1, op_errno, NULL, NULL, NULL, NULL, NULL,
+                     NULL);
+    return 0;
 }
 
 /* }}} */
@@ -515,518 +502,436 @@ out:
 /* {{{ mknod */
 
 int
-afr_mknod_unwind (call_frame_t *frame, xlator_t *this)
+afr_mknod_unwind(call_frame_t *frame, xlator_t *this)
 {
-        call_frame_t *main_frame = NULL;
-        afr_local_t  *local = NULL;
-
-        local = frame->local;
+    call_frame_t *main_frame = NULL;
+    afr_local_t *local = NULL;
 
-	main_frame = afr_transaction_detach_fop_frame (frame);
-        if (!main_frame)
-		return 0;
+    local = frame->local;
 
-	AFR_STACK_UNWIND (mknod, main_frame, local->op_ret, local->op_errno,
-			  local->inode, &local->cont.dir_fop.buf,
-			  &local->cont.dir_fop.preparent,
-			  &local->cont.dir_fop.postparent, local->xdata_rsp);
+    main_frame = afr_transaction_detach_fop_frame(frame);
+    if (!main_frame)
         return 0;
-}
 
+    AFR_STACK_UNWIND(mknod, main_frame, local->op_ret, local->op_errno,
+                     local->inode, &local->cont.dir_fop.buf,
+                     &local->cont.dir_fop.preparent,
+                     &local->cont.dir_fop.postparent, local->xdata_rsp);
+    return 0;
+}
 
 int
-afr_mknod_wind_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                    int32_t op_ret, int32_t op_errno, inode_t *inode,
-                    struct iatt *buf, struct iatt *preparent,
-                    struct iatt *postparent, dict_t *xdata)
+afr_mknod_wind_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                   int32_t op_ret, int32_t op_errno, inode_t *inode,
+                   struct iatt *buf, struct iatt *preparent,
+                   struct iatt *postparent, dict_t *xdata)
 {
-	return __afr_dir_write_cbk (frame, cookie, this, op_ret, op_errno, buf,
-				    preparent, postparent, NULL, NULL, xdata);
+    return __afr_dir_write_cbk(frame, cookie, this, op_ret, op_errno, buf,
+                               preparent, postparent, NULL, NULL, xdata);
 }
 
-
 int
-afr_mknod_wind (call_frame_t *frame, xlator_t *this, int subvol)
+afr_mknod_wind(call_frame_t *frame, xlator_t *this, int subvol)
 {
-        afr_local_t *local = NULL;
-        afr_private_t *priv = NULL;
-
-        local = frame->local;
-        priv  = this->private;
-
-	STACK_WIND_COOKIE (frame, afr_mknod_wind_cbk, (void *) (long) subvol,
-			   priv->children[subvol],
-			   priv->children[subvol]->fops->mknod,
-			   &local->loc, local->cont.mknod.mode,
-			   local->cont.mknod.dev, local->umask,
-			   local->xdata_req);
-        return 0;
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
+
+    local = frame->local;
+    priv = this->private;
+
+    STACK_WIND_COOKIE(frame, afr_mknod_wind_cbk, (void *)(long)subvol,
+                      priv->children[subvol],
+                      priv->children[subvol]->fops->mknod, &local->loc,
+                      local->cont.mknod.mode, local->cont.mknod.dev,
+                      local->umask, local->xdata_req);
+    return 0;
 }
 
 int
-afr_mknod (call_frame_t *frame, xlator_t *this, loc_t *loc, mode_t mode,
-           dev_t dev, mode_t umask, dict_t *xdata)
+afr_mknod(call_frame_t *frame, xlator_t *this, loc_t *loc, mode_t mode,
+          dev_t dev, mode_t umask, dict_t *xdata)
 {
-        afr_private_t           *priv                   = NULL;
-        afr_local_t             *local                  = NULL;
-        afr_internal_lock_t     *int_lock               = NULL;
-        call_frame_t            *transaction_frame      = NULL;
-        int                     ret                     = -1;
-        int                     op_errno                = ENOMEM;
-
-        priv = this->private;
-
-        transaction_frame = copy_frame (frame);
-        if (!transaction_frame)
-                goto out;
-
-	local = AFR_FRAME_INIT (transaction_frame, op_errno);
-	if (!local)
-		goto out;
-
-        loc_copy (&local->loc, loc);
-	local->inode = inode_ref (loc->inode);
-	local->parent = inode_ref (loc->parent);
-
-        local->op               = GF_FOP_MKNOD;
-        local->cont.mknod.mode  = mode;
-        local->cont.mknod.dev   = dev;
-        local->umask = umask;
-
-        if (xdata)
-                local->xdata_req = dict_copy_with_ref (xdata, NULL);
-	else
-		local->xdata_req = dict_new ();
-
-	if (!local->xdata_req)
-		goto out;
-
-        local->transaction.wind   = afr_mknod_wind;
-        local->transaction.fop    = __afr_txn_write_fop;
-        local->transaction.done   = __afr_txn_write_done;
-        local->transaction.unwind = afr_mknod_unwind;
-
-        ret = afr_build_parent_loc (&local->transaction.parent_loc, loc,
-                                    &op_errno);
-        if (ret)
-                goto out;
-
-        local->transaction.main_frame = frame;
-        local->transaction.basename = AFR_BASENAME (loc->path);
-        int_lock = &local->internal_lock;
-
-        int_lock->lockee_count = 0;
-        ret = afr_init_entry_lockee (&int_lock->lockee[0], local,
-                                     &local->transaction.parent_loc,
-                                     local->transaction.basename,
-                                     priv->child_count);
-        if (ret)
-                goto out;
-
-        int_lock->lockee_count++;
-        ret = afr_transaction (transaction_frame, this, AFR_ENTRY_TRANSACTION);
-        if (ret < 0) {
-		op_errno = -ret;
-		goto out;
-        }
-
-	return 0;
+    afr_local_t *local = NULL;
+    call_frame_t *transaction_frame = NULL;
+    int ret = -1;
+    int op_errno = ENOMEM;
+
+    transaction_frame = copy_frame(frame);
+    if (!transaction_frame)
+        goto out;
+
+    local = AFR_FRAME_INIT(transaction_frame, op_errno);
+    if (!local)
+        goto out;
+
+    loc_copy(&local->loc, loc);
+    local->inode = inode_ref(loc->inode);
+    local->parent = inode_ref(loc->parent);
+
+    local->op = GF_FOP_MKNOD;
+    local->cont.mknod.mode = mode;
+    local->cont.mknod.dev = dev;
+    local->umask = umask;
+
+    if (xdata)
+        local->xdata_req = dict_copy_with_ref(xdata, NULL);
+    else
+        local->xdata_req = dict_new();
+
+    if (!local->xdata_req)
+        goto out;
+
+    local->transaction.wind = afr_mknod_wind;
+    local->transaction.unwind = afr_mknod_unwind;
+
+    ret = afr_build_parent_loc(&local->transaction.parent_loc, loc, &op_errno);
+    if (ret)
+        goto out;
+
+    local->transaction.main_frame = frame;
+    local->transaction.basename = AFR_BASENAME(loc->path);
+    ret = afr_transaction(transaction_frame, this, AFR_ENTRY_TRANSACTION);
+    if (ret < 0) {
+        op_errno = -ret;
+        goto out;
+    }
+
+    return 0;
 out:
-	if (transaction_frame)
-		AFR_STACK_DESTROY (transaction_frame);
+    if (transaction_frame)
+        AFR_STACK_DESTROY(transaction_frame);
 
-	AFR_STACK_UNWIND (mknod, frame, -1, op_errno, NULL, NULL, NULL, NULL,
-			  NULL);
-        return 0;
+    AFR_STACK_UNWIND(mknod, frame, -1, op_errno, NULL, NULL, NULL, NULL, NULL);
+    return 0;
 }
 
 /* }}} */
 
 /* {{{ mkdir */
 
-
 int
-afr_mkdir_unwind (call_frame_t *frame, xlator_t *this)
+afr_mkdir_unwind(call_frame_t *frame, xlator_t *this)
 {
-        call_frame_t *main_frame = NULL;
-        afr_local_t  *local = NULL;
+    call_frame_t *main_frame = NULL;
+    afr_local_t *local = NULL;
 
-        local = frame->local;
+    local = frame->local;
 
-	main_frame = afr_transaction_detach_fop_frame (frame);
-        if (!main_frame)
-		return 0;
-
-	AFR_STACK_UNWIND (mkdir, main_frame, local->op_ret, local->op_errno,
-			  local->inode, &local->cont.dir_fop.buf,
-			  &local->cont.dir_fop.preparent,
-			  &local->cont.dir_fop.postparent, local->xdata_rsp);
+    main_frame = afr_transaction_detach_fop_frame(frame);
+    if (!main_frame)
         return 0;
-}
 
+    AFR_STACK_UNWIND(mkdir, main_frame, local->op_ret, local->op_errno,
+                     local->inode, &local->cont.dir_fop.buf,
+                     &local->cont.dir_fop.preparent,
+                     &local->cont.dir_fop.postparent, local->xdata_rsp);
+    return 0;
+}
 
 int
-afr_mkdir_wind_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                    int32_t op_ret, int32_t op_errno, inode_t *inode,
-                    struct iatt *buf, struct iatt *preparent,
-                    struct iatt *postparent, dict_t *xdata)
+afr_mkdir_wind_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                   int32_t op_ret, int32_t op_errno, inode_t *inode,
+                   struct iatt *buf, struct iatt *preparent,
+                   struct iatt *postparent, dict_t *xdata)
 {
-	return __afr_dir_write_cbk (frame, cookie, this, op_ret, op_errno, buf,
-				    preparent, postparent, NULL, NULL, xdata);
+    return __afr_dir_write_cbk(frame, cookie, this, op_ret, op_errno, buf,
+                               preparent, postparent, NULL, NULL, xdata);
 }
 
-
 int
-afr_mkdir_wind (call_frame_t *frame, xlator_t *this, int subvol)
+afr_mkdir_wind(call_frame_t *frame, xlator_t *this, int subvol)
 {
-        afr_local_t *local = NULL;
-        afr_private_t *priv = NULL;
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
 
-        local = frame->local;
-        priv  = this->private;
+    local = frame->local;
+    priv = this->private;
 
-	STACK_WIND_COOKIE (frame, afr_mkdir_wind_cbk, (void *) (long) subvol,
-			   priv->children[subvol],
-			   priv->children[subvol]->fops->mkdir, &local->loc,
-			   local->cont.mkdir.mode, local->umask,
-			   local->xdata_req);
-        return 0;
+    STACK_WIND_COOKIE(frame, afr_mkdir_wind_cbk, (void *)(long)subvol,
+                      priv->children[subvol],
+                      priv->children[subvol]->fops->mkdir, &local->loc,
+                      local->cont.mkdir.mode, local->umask, local->xdata_req);
+    return 0;
 }
 
-
 int
-afr_mkdir (call_frame_t *frame, xlator_t *this, loc_t *loc, mode_t mode,
-	   mode_t umask, dict_t *xdata)
+afr_mkdir(call_frame_t *frame, xlator_t *this, loc_t *loc, mode_t mode,
+          mode_t umask, dict_t *xdata)
 {
-        afr_private_t           *priv                   = NULL;
-        afr_local_t             *local                  = NULL;
-        afr_internal_lock_t     *int_lock               = NULL;
-        call_frame_t            *transaction_frame      = NULL;
-        int                     ret                     = -1;
-        int                     op_errno                = ENOMEM;
-
-        priv = this->private;
-
-        transaction_frame = copy_frame (frame);
-        if (!transaction_frame)
-                goto out;
-
-	local = AFR_FRAME_INIT (transaction_frame, op_errno);
-	if (!local)
-		goto out;
-
-        loc_copy (&local->loc, loc);
-	local->inode = inode_ref (loc->inode);
-	local->parent = inode_ref (loc->parent);
-
-        local->cont.mkdir.mode  = mode;
-        local->umask = umask;
-
-        if (xdata)
-                local->xdata_req = dict_copy_with_ref (xdata, NULL);
-	else
-		local->xdata_req = dict_new ();
-
-	if (!local->xdata_req)
-		goto out;
-
-        local->op = GF_FOP_MKDIR;
-        local->transaction.wind   = afr_mkdir_wind;
-        local->transaction.fop    = __afr_txn_write_fop;
-        local->transaction.done   = __afr_txn_write_done;
-        local->transaction.unwind = afr_mkdir_unwind;
-
-        ret = afr_build_parent_loc (&local->transaction.parent_loc, loc,
-                                    &op_errno);
-        if (ret)
-                goto out;
-
-        local->transaction.main_frame = frame;
-        local->transaction.basename = AFR_BASENAME (loc->path);
-        int_lock = &local->internal_lock;
-
-        int_lock->lockee_count = 0;
-        ret = afr_init_entry_lockee (&int_lock->lockee[0], local,
-                                     &local->transaction.parent_loc,
-                                     local->transaction.basename,
-                                     priv->child_count);
-        if (ret)
-                goto out;
-
-        int_lock->lockee_count++;
-        ret = afr_transaction (transaction_frame, this, AFR_ENTRY_TRANSACTION);
-        if (ret < 0) {
-		op_errno = -ret;
-		goto out;
-        }
-
-	return 0;
+    afr_local_t *local = NULL;
+    call_frame_t *transaction_frame = NULL;
+    int ret = -1;
+    int op_errno = ENOMEM;
+
+    transaction_frame = copy_frame(frame);
+    if (!transaction_frame)
+        goto out;
+
+    local = AFR_FRAME_INIT(transaction_frame, op_errno);
+    if (!local)
+        goto out;
+
+    loc_copy(&local->loc, loc);
+    local->inode = inode_ref(loc->inode);
+    local->parent = inode_ref(loc->parent);
+
+    local->cont.mkdir.mode = mode;
+    local->umask = umask;
+
+    if (!xdata || !dict_get_sizen(xdata, "gfid-req")) {
+        op_errno = EPERM;
+        gf_msg_callingfn(this->name, GF_LOG_WARNING, op_errno,
+                         AFR_MSG_GFID_NULL,
+                         "mkdir: %s is received "
+                         "without gfid-req %p",
+                         loc->path, xdata);
+        goto out;
+    }
+
+    local->xdata_req = dict_copy_with_ref(xdata, NULL);
+    if (!local->xdata_req) {
+        op_errno = ENOMEM;
+        goto out;
+    }
+
+    local->op = GF_FOP_MKDIR;
+    local->transaction.wind = afr_mkdir_wind;
+    local->transaction.unwind = afr_mkdir_unwind;
+
+    ret = afr_build_parent_loc(&local->transaction.parent_loc, loc, &op_errno);
+    if (ret)
+        goto out;
+
+    local->transaction.main_frame = frame;
+    local->transaction.basename = AFR_BASENAME(loc->path);
+    ret = afr_transaction(transaction_frame, this, AFR_ENTRY_TRANSACTION);
+    if (ret < 0) {
+        op_errno = -ret;
+        goto out;
+    }
+
+    return 0;
 out:
-	if (transaction_frame)
-		AFR_STACK_DESTROY (transaction_frame);
+    if (transaction_frame)
+        AFR_STACK_DESTROY(transaction_frame);
 
-	AFR_STACK_UNWIND (mkdir, frame, -1, op_errno, NULL, NULL, NULL, NULL,
-			  NULL);
-        return 0;
+    AFR_STACK_UNWIND(mkdir, frame, -1, op_errno, NULL, NULL, NULL, NULL, NULL);
+    return 0;
 }
 
 /* }}} */
 
 /* {{{ link */
 
-
 int
-afr_link_unwind (call_frame_t *frame, xlator_t *this)
+afr_link_unwind(call_frame_t *frame, xlator_t *this)
 {
-        call_frame_t *main_frame = NULL;
-        afr_local_t  *local = NULL;
-
-        local = frame->local;
+    call_frame_t *main_frame = NULL;
+    afr_local_t *local = NULL;
 
-	main_frame = afr_transaction_detach_fop_frame (frame);
-        if (!main_frame)
-		return 0;
+    local = frame->local;
 
-	AFR_STACK_UNWIND (link, main_frame, local->op_ret, local->op_errno,
-			  local->inode, &local->cont.dir_fop.buf,
-			  &local->cont.dir_fop.preparent,
-			  &local->cont.dir_fop.postparent, local->xdata_rsp);
+    main_frame = afr_transaction_detach_fop_frame(frame);
+    if (!main_frame)
         return 0;
-}
 
+    AFR_STACK_UNWIND(link, main_frame, local->op_ret, local->op_errno,
+                     local->inode, &local->cont.dir_fop.buf,
+                     &local->cont.dir_fop.preparent,
+                     &local->cont.dir_fop.postparent, local->xdata_rsp);
+    return 0;
+}
 
 int
-afr_link_wind_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                   int32_t op_ret, int32_t op_errno, inode_t *inode,
-                   struct iatt *buf, struct iatt *preparent,
-                   struct iatt *postparent, dict_t *xdata)
+afr_link_wind_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                  int32_t op_ret, int32_t op_errno, inode_t *inode,
+                  struct iatt *buf, struct iatt *preparent,
+                  struct iatt *postparent, dict_t *xdata)
 {
-        return __afr_dir_write_cbk (frame, cookie, this, op_ret, op_errno, buf,
-				    preparent, postparent, NULL, NULL, xdata);
+    return __afr_dir_write_cbk(frame, cookie, this, op_ret, op_errno, buf,
+                               preparent, postparent, NULL, NULL, xdata);
 }
 
-
 int
-afr_link_wind (call_frame_t *frame, xlator_t *this, int subvol)
+afr_link_wind(call_frame_t *frame, xlator_t *this, int subvol)
 {
-        afr_local_t *local = NULL;
-        afr_private_t *priv = NULL;
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
 
-        local = frame->local;
-        priv  = this->private;
+    local = frame->local;
+    priv = this->private;
 
-	STACK_WIND_COOKIE (frame, afr_link_wind_cbk, (void *) (long) subvol,
-			   priv->children[subvol],
-			   priv->children[subvol]->fops->link,
-			   &local->loc, &local->newloc, local->xdata_req);
-        return 0;
+    STACK_WIND_COOKIE(frame, afr_link_wind_cbk, (void *)(long)subvol,
+                      priv->children[subvol],
+                      priv->children[subvol]->fops->link, &local->loc,
+                      &local->newloc, local->xdata_req);
+    return 0;
 }
 
-
 int
-afr_link (call_frame_t *frame, xlator_t *this, loc_t *oldloc, loc_t *newloc,
-	  dict_t *xdata)
+afr_link(call_frame_t *frame, xlator_t *this, loc_t *oldloc, loc_t *newloc,
+         dict_t *xdata)
 {
-        afr_private_t           *priv                   = NULL;
-        afr_local_t             *local                  = NULL;
-        afr_internal_lock_t     *int_lock               = NULL;
-        call_frame_t            *transaction_frame      = NULL;
-        int                     ret                     = -1;
-        int                     op_errno                = ENOMEM;
-
-        priv = this->private;
-
-        transaction_frame = copy_frame (frame);
-        if (!transaction_frame)
-                goto out;
-
-	local = AFR_FRAME_INIT (transaction_frame, op_errno);
-	if (!local)
-		goto out;
-
-        loc_copy (&local->loc,    oldloc);
-        loc_copy (&local->newloc, newloc);
-
-	local->inode = inode_ref (oldloc->inode);
-	local->parent = inode_ref (newloc->parent);
-
-        if (xdata)
-                local->xdata_req = dict_copy_with_ref (xdata, NULL);
-	else
-		local->xdata_req = dict_new ();
-
-	if (!local->xdata_req)
-		goto out;
-
-        local->op = GF_FOP_LINK;
-
-        local->transaction.wind   = afr_link_wind;
-        local->transaction.fop    = __afr_txn_write_fop;
-        local->transaction.done   = __afr_txn_write_done;
-        local->transaction.unwind = afr_link_unwind;
-
-        ret = afr_build_parent_loc (&local->transaction.parent_loc, newloc,
-                                    &op_errno);
-        if (ret)
-                goto out;
-
-        local->transaction.main_frame   = frame;
-        local->transaction.basename     = AFR_BASENAME (newloc->path);
-        int_lock = &local->internal_lock;
-
-        int_lock->lockee_count = 0;
-        ret = afr_init_entry_lockee (&int_lock->lockee[0], local,
-                                     &local->transaction.parent_loc,
-                                     local->transaction.basename,
-                                     priv->child_count);
-        if (ret)
-                goto out;
-
-        int_lock->lockee_count++;
-        ret = afr_transaction (transaction_frame, this, AFR_ENTRY_TRANSACTION);
-        if (ret < 0) {
-		op_errno = -ret;
-		goto out;
-        }
+    afr_local_t *local = NULL;
+    call_frame_t *transaction_frame = NULL;
+    int ret = -1;
+    int op_errno = ENOMEM;
+
+    transaction_frame = copy_frame(frame);
+    if (!transaction_frame)
+        goto out;
+
+    local = AFR_FRAME_INIT(transaction_frame, op_errno);
+    if (!local)
+        goto out;
+
+    loc_copy(&local->loc, oldloc);
+    loc_copy(&local->newloc, newloc);
 
-	return 0;
+    local->inode = inode_ref(oldloc->inode);
+    local->parent = inode_ref(newloc->parent);
+
+    if (xdata)
+        local->xdata_req = dict_copy_with_ref(xdata, NULL);
+    else
+        local->xdata_req = dict_new();
+
+    if (!local->xdata_req)
+        goto out;
+
+    local->op = GF_FOP_LINK;
+
+    local->transaction.wind = afr_link_wind;
+    local->transaction.unwind = afr_link_unwind;
+
+    ret = afr_build_parent_loc(&local->transaction.parent_loc, newloc,
+                               &op_errno);
+    if (ret)
+        goto out;
+
+    local->transaction.main_frame = frame;
+    local->transaction.basename = AFR_BASENAME(newloc->path);
+    ret = afr_transaction(transaction_frame, this, AFR_ENTRY_TRANSACTION);
+    if (ret < 0) {
+        op_errno = -ret;
+        goto out;
+    }
+
+    return 0;
 out:
-	if (transaction_frame)
-		AFR_STACK_DESTROY (transaction_frame);
+    if (transaction_frame)
+        AFR_STACK_DESTROY(transaction_frame);
 
-	AFR_STACK_UNWIND (link, frame, -1, op_errno, NULL, NULL, NULL, NULL,
-			  NULL);
-        return 0;
+    AFR_STACK_UNWIND(link, frame, -1, op_errno, NULL, NULL, NULL, NULL, NULL);
+    return 0;
 }
 
 /* }}} */
 
 /* {{{ symlink */
 
-
 int
-afr_symlink_unwind (call_frame_t *frame, xlator_t *this)
+afr_symlink_unwind(call_frame_t *frame, xlator_t *this)
 {
-        call_frame_t *main_frame = NULL;
-        afr_local_t  *local = NULL;
-
-        local = frame->local;
+    call_frame_t *main_frame = NULL;
+    afr_local_t *local = NULL;
 
-	main_frame = afr_transaction_detach_fop_frame (frame);
-        if (!main_frame)
-		return 0;
+    local = frame->local;
 
-	AFR_STACK_UNWIND (symlink, main_frame, local->op_ret, local->op_errno,
-			  local->inode, &local->cont.dir_fop.buf,
-			  &local->cont.dir_fop.preparent,
-			  &local->cont.dir_fop.postparent, local->xdata_rsp);
+    main_frame = afr_transaction_detach_fop_frame(frame);
+    if (!main_frame)
         return 0;
-}
 
+    AFR_STACK_UNWIND(symlink, main_frame, local->op_ret, local->op_errno,
+                     local->inode, &local->cont.dir_fop.buf,
+                     &local->cont.dir_fop.preparent,
+                     &local->cont.dir_fop.postparent, local->xdata_rsp);
+    return 0;
+}
 
 int
-afr_symlink_wind_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                      int32_t op_ret, int32_t op_errno, inode_t *inode,
-                      struct iatt *buf, struct iatt *preparent,
-                      struct iatt *postparent, dict_t *xdata)
+afr_symlink_wind_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                     int32_t op_ret, int32_t op_errno, inode_t *inode,
+                     struct iatt *buf, struct iatt *preparent,
+                     struct iatt *postparent, dict_t *xdata)
 {
-        return __afr_dir_write_cbk (frame, cookie, this, op_ret, op_errno, buf,
-				    preparent, postparent, NULL, NULL, xdata);
+    return __afr_dir_write_cbk(frame, cookie, this, op_ret, op_errno, buf,
+                               preparent, postparent, NULL, NULL, xdata);
 }
 
-
 int
-afr_symlink_wind (call_frame_t *frame, xlator_t *this, int subvol)
+afr_symlink_wind(call_frame_t *frame, xlator_t *this, int subvol)
 {
-        afr_local_t *local = NULL;
-        afr_private_t *priv = NULL;
-
-        local = frame->local;
-        priv = this->private;
-
-	STACK_WIND_COOKIE (frame, afr_symlink_wind_cbk, (void *) (long) subvol,
-			   priv->children[subvol],
-			   priv->children[subvol]->fops->symlink,
-			   local->cont.symlink.linkpath, &local->loc,
-			   local->umask, local->xdata_req);
-        return 0;
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
+
+    local = frame->local;
+    priv = this->private;
+
+    STACK_WIND_COOKIE(frame, afr_symlink_wind_cbk, (void *)(long)subvol,
+                      priv->children[subvol],
+                      priv->children[subvol]->fops->symlink,
+                      local->cont.symlink.linkpath, &local->loc, local->umask,
+                      local->xdata_req);
+    return 0;
 }
 
-
 int
-afr_symlink (call_frame_t *frame, xlator_t *this, const char *linkpath,
-	     loc_t *loc, mode_t umask, dict_t *xdata)
+afr_symlink(call_frame_t *frame, xlator_t *this, const char *linkpath,
+            loc_t *loc, mode_t umask, dict_t *xdata)
 {
-        afr_private_t           *priv                   = NULL;
-        afr_local_t             *local                  = NULL;
-        afr_internal_lock_t     *int_lock               = NULL;
-        call_frame_t            *transaction_frame      = NULL;
-        int                     ret                     = -1;
-        int                     op_errno                = ENOMEM;
-
-        priv = this->private;
-
-        transaction_frame = copy_frame (frame);
-        if (!transaction_frame)
-                goto out;
-
-	local = AFR_FRAME_INIT (transaction_frame, op_errno);
-	if (!local)
-		goto out;
-
-        loc_copy (&local->loc, loc);
-	local->inode = inode_ref (loc->inode);
-	local->parent = inode_ref (loc->parent);
-
-        local->cont.symlink.linkpath = gf_strdup (linkpath);
-        local->umask = umask;
-
-        if (xdata)
-                local->xdata_req = dict_copy_with_ref (xdata, NULL);
-	else
-		local->xdata_req = dict_new ();
-
-	if (!local->xdata_req)
-		goto out;
-
-        local->op = GF_FOP_SYMLINK;
-        local->transaction.wind   = afr_symlink_wind;
-        local->transaction.fop    = __afr_txn_write_fop;
-        local->transaction.done   = __afr_txn_write_done;
-        local->transaction.unwind = afr_symlink_unwind;
-
-        ret = afr_build_parent_loc (&local->transaction.parent_loc, loc,
-                                    &op_errno);
-        if (ret)
-                goto out;
-
-        local->transaction.main_frame   = frame;
-        local->transaction.basename     = AFR_BASENAME (loc->path);
-        int_lock = &local->internal_lock;
-
-        int_lock->lockee_count = 0;
-        ret = afr_init_entry_lockee (&int_lock->lockee[0], local,
-                                     &local->transaction.parent_loc,
-                                     local->transaction.basename,
-                                     priv->child_count);
-        if (ret)
-                goto out;
-
-        int_lock->lockee_count++;
-        ret = afr_transaction (transaction_frame, this, AFR_ENTRY_TRANSACTION);
-        if (ret < 0) {
-		op_errno = -ret;
-		goto out;
-        }
-
-	return 0;
+    afr_local_t *local = NULL;
+    call_frame_t *transaction_frame = NULL;
+    int ret = -1;
+    int op_errno = ENOMEM;
+
+    transaction_frame = copy_frame(frame);
+    if (!transaction_frame)
+        goto out;
+
+    local = AFR_FRAME_INIT(transaction_frame, op_errno);
+    if (!local)
+        goto out;
+
+    loc_copy(&local->loc, loc);
+    local->inode = inode_ref(loc->inode);
+    local->parent = inode_ref(loc->parent);
+
+    local->cont.symlink.linkpath = gf_strdup(linkpath);
+    local->umask = umask;
+
+    if (xdata)
+        local->xdata_req = dict_copy_with_ref(xdata, NULL);
+    else
+        local->xdata_req = dict_new();
+
+    if (!local->xdata_req)
+        goto out;
+
+    local->op = GF_FOP_SYMLINK;
+    local->transaction.wind = afr_symlink_wind;
+    local->transaction.unwind = afr_symlink_unwind;
+
+    ret = afr_build_parent_loc(&local->transaction.parent_loc, loc, &op_errno);
+    if (ret)
+        goto out;
+
+    local->transaction.main_frame = frame;
+    local->transaction.basename = AFR_BASENAME(loc->path);
+    ret = afr_transaction(transaction_frame, this, AFR_ENTRY_TRANSACTION);
+    if (ret < 0) {
+        op_errno = -ret;
+        goto out;
+    }
+
+    return 0;
 out:
-	if (transaction_frame)
-		AFR_STACK_DESTROY (transaction_frame);
+    if (transaction_frame)
+        AFR_STACK_DESTROY(transaction_frame);
 
-	AFR_STACK_UNWIND (symlink, frame, -1, op_errno, NULL, NULL, NULL,
-			  NULL, NULL);
-        return 0;
+    AFR_STACK_UNWIND(symlink, frame, -1, op_errno, NULL, NULL, NULL, NULL,
+                     NULL);
+    return 0;
 }
 
 /* }}} */
@@ -1034,161 +939,118 @@ out:
 /* {{{ rename */
 
 int
-afr_rename_unwind (call_frame_t *frame, xlator_t *this)
+afr_rename_unwind(call_frame_t *frame, xlator_t *this)
 {
-        call_frame_t *main_frame = NULL;
-        afr_local_t  *local = NULL;
-
-        local = frame->local;
+    call_frame_t *main_frame = NULL;
+    afr_local_t *local = NULL;
 
-	main_frame = afr_transaction_detach_fop_frame (frame);
-        if (!main_frame)
-		return 0;
+    local = frame->local;
 
-	AFR_STACK_UNWIND (rename, main_frame, local->op_ret, local->op_errno,
-			  &local->cont.dir_fop.buf,
-			  &local->cont.dir_fop.preparent,
-			  &local->cont.dir_fop.postparent,
-			  &local->cont.dir_fop.prenewparent,
-			  &local->cont.dir_fop.postnewparent, local->xdata_rsp);
+    main_frame = afr_transaction_detach_fop_frame(frame);
+    if (!main_frame)
         return 0;
-}
 
+    AFR_STACK_UNWIND(rename, main_frame, local->op_ret, local->op_errno,
+                     &local->cont.dir_fop.buf, &local->cont.dir_fop.preparent,
+                     &local->cont.dir_fop.postparent,
+                     &local->cont.dir_fop.prenewparent,
+                     &local->cont.dir_fop.postnewparent, local->xdata_rsp);
+    return 0;
+}
 
 int
-afr_rename_wind_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                     int32_t op_ret, int32_t op_errno, struct iatt *buf,
-                     struct iatt *preoldparent, struct iatt *postoldparent,
-                     struct iatt *prenewparent, struct iatt *postnewparent,
-                     dict_t *xdata)
+afr_rename_wind_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                    int32_t op_ret, int32_t op_errno, struct iatt *buf,
+                    struct iatt *preoldparent, struct iatt *postoldparent,
+                    struct iatt *prenewparent, struct iatt *postnewparent,
+                    dict_t *xdata)
 {
-        return __afr_dir_write_cbk (frame, cookie, this, op_ret, op_errno, buf,
-				    preoldparent, postoldparent, prenewparent,
-				    postnewparent, xdata);
+    return __afr_dir_write_cbk(frame, cookie, this, op_ret, op_errno, buf,
+                               preoldparent, postoldparent, prenewparent,
+                               postnewparent, xdata);
 }
 
-
 int
-afr_rename_wind (call_frame_t *frame, xlator_t *this, int subvol)
+afr_rename_wind(call_frame_t *frame, xlator_t *this, int subvol)
 {
-        afr_local_t *local = NULL;
-        afr_private_t *priv = NULL;
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
 
-	local = frame->local;
-	priv = this->private;
+    local = frame->local;
+    priv = this->private;
 
-	STACK_WIND_COOKIE (frame, afr_rename_wind_cbk, (void *) (long) subvol,
-			   priv->children[subvol],
-			   priv->children[subvol]->fops->rename,
-			   &local->loc, &local->newloc, local->xdata_req);
-        return 0;
+    STACK_WIND_COOKIE(frame, afr_rename_wind_cbk, (void *)(long)subvol,
+                      priv->children[subvol],
+                      priv->children[subvol]->fops->rename, &local->loc,
+                      &local->newloc, local->xdata_req);
+    return 0;
 }
 
-
 int
-afr_rename (call_frame_t *frame, xlator_t *this, loc_t *oldloc, loc_t *newloc,
-	    dict_t *xdata)
+afr_rename(call_frame_t *frame, xlator_t *this, loc_t *oldloc, loc_t *newloc,
+           dict_t *xdata)
 {
-        afr_private_t           *priv                   = NULL;
-        afr_local_t             *local                  = NULL;
-        afr_internal_lock_t     *int_lock               = NULL;
-        call_frame_t            *transaction_frame      = NULL;
-        int                     ret                     = -1;
-        int                     op_errno                = ENOMEM;
-        int                     nlockee                 = 0;
-
-        priv = this->private;
-
-        transaction_frame = copy_frame (frame);
-        if (!transaction_frame) {
-                op_errno = ENOMEM;
-                goto out;
-        }
-
-	local = AFR_FRAME_INIT (transaction_frame, op_errno);
-	if (!local)
-		goto out;
-
-        loc_copy (&local->loc,    oldloc);
-        loc_copy (&local->newloc, newloc);
-
-	local->inode = inode_ref (oldloc->inode);
-	local->parent = inode_ref (oldloc->parent);
-	local->parent2 = inode_ref (newloc->parent);
-
-        if (xdata)
-                local->xdata_req = dict_copy_with_ref (xdata, NULL);
-	else
-		local->xdata_req = dict_new ();
-
-	if (!local->xdata_req)
-		goto out;
-
-        local->op = GF_FOP_RENAME;
-        local->transaction.wind   = afr_rename_wind;
-        local->transaction.fop    = __afr_txn_write_fop;
-        local->transaction.done   = __afr_txn_write_done;
-        local->transaction.unwind = afr_rename_unwind;
-
-        ret = afr_build_parent_loc (&local->transaction.parent_loc, oldloc,
-                                    &op_errno);
-        if (ret)
-                goto out;
-        ret = afr_build_parent_loc (&local->transaction.new_parent_loc, newloc,
-                                    &op_errno);
-        if (ret)
-                goto out;
-
-        local->transaction.main_frame   = frame;
-        local->transaction.basename     = AFR_BASENAME (oldloc->path);
-        local->transaction.new_basename = AFR_BASENAME (newloc->path);
-        int_lock = &local->internal_lock;
-
-        int_lock->lockee_count = nlockee = 0;
-        ret = afr_init_entry_lockee (&int_lock->lockee[nlockee], local,
-                                     &local->transaction.new_parent_loc,
-                                     local->transaction.new_basename,
-                                     priv->child_count);
-        if (ret)
-                goto out;
-
-        nlockee++;
-        ret = afr_init_entry_lockee (&int_lock->lockee[nlockee], local,
-                                     &local->transaction.parent_loc,
-                                     local->transaction.basename,
-                                     priv->child_count);
-        if (ret)
-                goto out;
-
-        nlockee++;
-        if (local->newloc.inode && IA_ISDIR (local->newloc.inode->ia_type)) {
-                ret = afr_init_entry_lockee (&int_lock->lockee[nlockee], local,
-                                             &local->newloc,
-                                             NULL,
-                                             priv->child_count);
-                if (ret)
-                        goto out;
-
-                nlockee++;
-        }
-        qsort (int_lock->lockee, nlockee, sizeof (*int_lock->lockee),
-               afr_entry_lockee_cmp);
-        int_lock->lockee_count = nlockee;
-
-        ret = afr_transaction (transaction_frame, this, AFR_ENTRY_RENAME_TRANSACTION);
-        if (ret < 0) {
-		op_errno = -ret;
-		goto out;
-        }
-
-	return 0;
+    afr_local_t *local = NULL;
+    call_frame_t *transaction_frame = NULL;
+    int ret = -1;
+    int op_errno = ENOMEM;
+
+    transaction_frame = copy_frame(frame);
+    if (!transaction_frame) {
+        op_errno = ENOMEM;
+        goto out;
+    }
+
+    local = AFR_FRAME_INIT(transaction_frame, op_errno);
+    if (!local)
+        goto out;
+
+    loc_copy(&local->loc, oldloc);
+    loc_copy(&local->newloc, newloc);
+
+    local->inode = inode_ref(oldloc->inode);
+    local->parent = inode_ref(oldloc->parent);
+    local->parent2 = inode_ref(newloc->parent);
+
+    if (xdata)
+        local->xdata_req = dict_copy_with_ref(xdata, NULL);
+    else
+        local->xdata_req = dict_new();
+
+    if (!local->xdata_req)
+        goto out;
+
+    local->op = GF_FOP_RENAME;
+    local->transaction.wind = afr_rename_wind;
+    local->transaction.unwind = afr_rename_unwind;
+
+    ret = afr_build_parent_loc(&local->transaction.parent_loc, oldloc,
+                               &op_errno);
+    if (ret)
+        goto out;
+    ret = afr_build_parent_loc(&local->transaction.new_parent_loc, newloc,
+                               &op_errno);
+    if (ret)
+        goto out;
+
+    local->transaction.main_frame = frame;
+    local->transaction.basename = AFR_BASENAME(oldloc->path);
+    local->transaction.new_basename = AFR_BASENAME(newloc->path);
+    ret = afr_transaction(transaction_frame, this,
+                          AFR_ENTRY_RENAME_TRANSACTION);
+    if (ret < 0) {
+        op_errno = -ret;
+        goto out;
+    }
+
+    return 0;
 out:
-	if (transaction_frame)
-		AFR_STACK_DESTROY (transaction_frame);
+    if (transaction_frame)
+        AFR_STACK_DESTROY(transaction_frame);
 
-	AFR_STACK_UNWIND (rename, frame, -1, op_errno, NULL, NULL, NULL, NULL,
-			  NULL, NULL);
-        return 0;
+    AFR_STACK_UNWIND(rename, frame, -1, op_errno, NULL, NULL, NULL, NULL, NULL,
+                     NULL);
+    return 0;
 }
 
 /* }}} */
@@ -1196,263 +1058,205 @@ out:
 /* {{{ unlink */
 
 int
-afr_unlink_unwind (call_frame_t *frame, xlator_t *this)
+afr_unlink_unwind(call_frame_t *frame, xlator_t *this)
 {
-        call_frame_t *main_frame = NULL;
-        afr_local_t  *local = NULL;
-
-        local = frame->local;
+    call_frame_t *main_frame = NULL;
+    afr_local_t *local = NULL;
 
-	main_frame = afr_transaction_detach_fop_frame (frame);
-        if (!main_frame)
-		return 0;
+    local = frame->local;
 
-	AFR_STACK_UNWIND (unlink, main_frame, local->op_ret, local->op_errno,
-			  &local->cont.dir_fop.preparent,
-			  &local->cont.dir_fop.postparent, local->xdata_rsp);
+    main_frame = afr_transaction_detach_fop_frame(frame);
+    if (!main_frame)
         return 0;
-}
 
+    AFR_STACK_UNWIND(unlink, main_frame, local->op_ret, local->op_errno,
+                     &local->cont.dir_fop.preparent,
+                     &local->cont.dir_fop.postparent, local->xdata_rsp);
+    return 0;
+}
 
 int
-afr_unlink_wind_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                     int32_t op_ret, int32_t op_errno, struct iatt *preparent,
-                     struct iatt *postparent, dict_t *xdata)
+afr_unlink_wind_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                    int32_t op_ret, int32_t op_errno, struct iatt *preparent,
+                    struct iatt *postparent, dict_t *xdata)
 {
-        return __afr_dir_write_cbk (frame, cookie, this, op_ret, op_errno, NULL,
-				    preparent, postparent, NULL, NULL, xdata);
+    return __afr_dir_write_cbk(frame, cookie, this, op_ret, op_errno, NULL,
+                               preparent, postparent, NULL, NULL, xdata);
 }
 
-
 int
-afr_unlink_wind (call_frame_t *frame, xlator_t *this, int subvol)
+afr_unlink_wind(call_frame_t *frame, xlator_t *this, int subvol)
 {
-        afr_local_t *local = NULL;
-        afr_private_t *priv = NULL;
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
 
-        local = frame->local;
-        priv  = this->private;
+    local = frame->local;
+    priv = this->private;
 
-	STACK_WIND_COOKIE (frame, afr_unlink_wind_cbk, (void *) (long) subvol,
-			   priv->children[subvol],
-			   priv->children[subvol]->fops->unlink,
-			   &local->loc, local->xflag, local->xdata_req);
-        return 0;
+    STACK_WIND_COOKIE(frame, afr_unlink_wind_cbk, (void *)(long)subvol,
+                      priv->children[subvol],
+                      priv->children[subvol]->fops->unlink, &local->loc,
+                      local->xflag, local->xdata_req);
+    return 0;
 }
 
-
 int
-afr_unlink (call_frame_t *frame, xlator_t *this, loc_t *loc, int xflag,
-	    dict_t *xdata)
+afr_unlink(call_frame_t *frame, xlator_t *this, loc_t *loc, int xflag,
+           dict_t *xdata)
 {
-        afr_private_t           *priv                   = NULL;
-        afr_local_t             *local                  = NULL;
-        afr_internal_lock_t     *int_lock               = NULL;
-        call_frame_t            *transaction_frame      = NULL;
-        int                     ret                     = -1;
-        int                     op_errno                = ENOMEM;
-
-        priv = this->private;
-
-        transaction_frame = copy_frame (frame);
-        if (!transaction_frame)
-                goto out;
-
-	local = AFR_FRAME_INIT (transaction_frame, op_errno);
-	if (!local)
-		goto out;
-
-        loc_copy (&local->loc, loc);
-        local->xflag = xflag;
-
-	local->inode = inode_ref (loc->inode);
-	local->parent = inode_ref (loc->parent);
-
-        if (xdata)
-                local->xdata_req = dict_copy_with_ref (xdata, NULL);
-	else
-		local->xdata_req = dict_new ();
-
-	if (!local->xdata_req)
-		goto out;
-
-        local->op = GF_FOP_UNLINK;
-        local->transaction.wind   = afr_unlink_wind;
-        local->transaction.fop    = __afr_txn_write_fop;
-        local->transaction.done   = __afr_txn_write_done;
-        local->transaction.unwind = afr_unlink_unwind;
-
-        ret = afr_build_parent_loc (&local->transaction.parent_loc, loc,
-                                    &op_errno);
-        if (ret)
-                goto out;
-
-        local->transaction.main_frame = frame;
-        local->transaction.basename = AFR_BASENAME (loc->path);
-        int_lock = &local->internal_lock;
-
-        int_lock->lockee_count = 0;
-        ret = afr_init_entry_lockee (&int_lock->lockee[0], local,
-                                     &local->transaction.parent_loc,
-                                     local->transaction.basename,
-                                     priv->child_count);
-        if (ret)
-                goto out;
-
-        int_lock->lockee_count++;
-        ret = afr_transaction (transaction_frame, this, AFR_ENTRY_TRANSACTION);
-        if (ret < 0) {
-		op_errno = -ret;
-		goto out;
-        }
-
-	return 0;
+    afr_local_t *local = NULL;
+    call_frame_t *transaction_frame = NULL;
+    int ret = -1;
+    int op_errno = ENOMEM;
+
+    transaction_frame = copy_frame(frame);
+    if (!transaction_frame)
+        goto out;
+
+    local = AFR_FRAME_INIT(transaction_frame, op_errno);
+    if (!local)
+        goto out;
+
+    loc_copy(&local->loc, loc);
+    local->xflag = xflag;
+
+    local->inode = inode_ref(loc->inode);
+    local->parent = inode_ref(loc->parent);
+
+    if (xdata)
+        local->xdata_req = dict_copy_with_ref(xdata, NULL);
+    else
+        local->xdata_req = dict_new();
+
+    if (!local->xdata_req)
+        goto out;
+
+    local->op = GF_FOP_UNLINK;
+    local->transaction.wind = afr_unlink_wind;
+    local->transaction.unwind = afr_unlink_unwind;
+
+    ret = afr_build_parent_loc(&local->transaction.parent_loc, loc, &op_errno);
+    if (ret)
+        goto out;
+
+    local->transaction.main_frame = frame;
+    local->transaction.basename = AFR_BASENAME(loc->path);
+    ret = afr_transaction(transaction_frame, this, AFR_ENTRY_TRANSACTION);
+    if (ret < 0) {
+        op_errno = -ret;
+        goto out;
+    }
+
+    return 0;
 out:
-	if (transaction_frame)
-		AFR_STACK_DESTROY (transaction_frame);
+    if (transaction_frame)
+        AFR_STACK_DESTROY(transaction_frame);
 
-	AFR_STACK_UNWIND (unlink, frame, -1, op_errno, NULL, NULL, NULL);
-        return 0;
+    AFR_STACK_UNWIND(unlink, frame, -1, op_errno, NULL, NULL, NULL);
+    return 0;
 }
 
 /* }}} */
 
 /* {{{ rmdir */
 
-
-
 int
-afr_rmdir_unwind (call_frame_t *frame, xlator_t *this)
+afr_rmdir_unwind(call_frame_t *frame, xlator_t *this)
 {
-        call_frame_t *main_frame = NULL;
-        afr_local_t  *local = NULL;
+    call_frame_t *main_frame = NULL;
+    afr_local_t *local = NULL;
 
-        local = frame->local;
+    local = frame->local;
 
-	main_frame = afr_transaction_detach_fop_frame (frame);
-	if (!main_frame)
-		return 0;
-
-	AFR_STACK_UNWIND (rmdir, main_frame, local->op_ret, local->op_errno,
-			  &local->cont.dir_fop.preparent,
-			  &local->cont.dir_fop.postparent, local->xdata_rsp);
+    main_frame = afr_transaction_detach_fop_frame(frame);
+    if (!main_frame)
         return 0;
-}
 
+    AFR_STACK_UNWIND(rmdir, main_frame, local->op_ret, local->op_errno,
+                     &local->cont.dir_fop.preparent,
+                     &local->cont.dir_fop.postparent, local->xdata_rsp);
+    return 0;
+}
 
 int
-afr_rmdir_wind_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                    int32_t op_ret, int32_t op_errno, struct iatt *preparent,
-                    struct iatt *postparent, dict_t *xdata)
+afr_rmdir_wind_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                   int32_t op_ret, int32_t op_errno, struct iatt *preparent,
+                   struct iatt *postparent, dict_t *xdata)
 {
-        return __afr_dir_write_cbk (frame, cookie, this, op_ret, op_errno, NULL,
-				    preparent, postparent, NULL, NULL, xdata);
+    return __afr_dir_write_cbk(frame, cookie, this, op_ret, op_errno, NULL,
+                               preparent, postparent, NULL, NULL, xdata);
 }
 
-
 int
-afr_rmdir_wind (call_frame_t *frame, xlator_t *this, int subvol)
+afr_rmdir_wind(call_frame_t *frame, xlator_t *this, int subvol)
 {
-        afr_local_t *local = NULL;
-        afr_private_t *priv = NULL;
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
 
-        local = frame->local;
-        priv  = this->private;
+    local = frame->local;
+    priv = this->private;
 
-	STACK_WIND_COOKIE (frame, afr_rmdir_wind_cbk, (void *) (long) subvol,
-			   priv->children[subvol],
-			   priv->children[subvol]->fops->rmdir,
-			   &local->loc, local->cont.rmdir.flags, local->xdata_req);
-        return 0;
+    STACK_WIND_COOKIE(frame, afr_rmdir_wind_cbk, (void *)(long)subvol,
+                      priv->children[subvol],
+                      priv->children[subvol]->fops->rmdir, &local->loc,
+                      local->cont.rmdir.flags, local->xdata_req);
+    return 0;
 }
 
-
 int
-afr_rmdir (call_frame_t *frame, xlator_t *this, loc_t *loc, int flags,
-	   dict_t *xdata)
+afr_rmdir(call_frame_t *frame, xlator_t *this, loc_t *loc, int flags,
+          dict_t *xdata)
 {
-        afr_private_t           *priv                   = NULL;
-        afr_local_t             *local                  = NULL;
-        afr_internal_lock_t     *int_lock               = NULL;
-        call_frame_t            *transaction_frame      = NULL;
-        int                     ret                     = -1;
-        int                     op_errno                = ENOMEM;
-        int                     nlockee                 = 0;
-
-        priv = this->private;
-
-        transaction_frame = copy_frame (frame);
-        if (!transaction_frame)
-                goto out;
-
-	local = AFR_FRAME_INIT (transaction_frame, op_errno);
-	if (!local)
-		goto out;
-
-
-        loc_copy (&local->loc, loc);
-	local->inode = inode_ref (loc->inode);
-	local->parent = inode_ref (loc->parent);
-
-        local->cont.rmdir.flags = flags;
-
-        if (xdata)
-                local->xdata_req = dict_copy_with_ref (xdata, NULL);
-	else
-		local->xdata_req = dict_new ();
-
-	if (!local->xdata_req)
-		goto out;
-
-        local->op = GF_FOP_RMDIR;
-        local->transaction.wind   = afr_rmdir_wind;
-        local->transaction.fop    = __afr_txn_write_fop;
-        local->transaction.done   = __afr_txn_write_done;
-        local->transaction.unwind = afr_rmdir_unwind;
-
-        ret = afr_build_parent_loc (&local->transaction.parent_loc, loc,
-                                    &op_errno);
-        if (ret)
-                goto out;
-
-        local->transaction.main_frame = frame;
-        local->transaction.basename = AFR_BASENAME (loc->path);
-        int_lock = &local->internal_lock;
-
-        int_lock->lockee_count = nlockee = 0;
-        ret = afr_init_entry_lockee (&int_lock->lockee[nlockee], local,
-                                     &local->transaction.parent_loc,
-                                     local->transaction.basename,
-                                     priv->child_count);
-        if (ret)
-                goto out;
-
-        nlockee++;
-        ret = afr_init_entry_lockee (&int_lock->lockee[nlockee], local,
-                                     &local->loc,
-                                     NULL,
-                                     priv->child_count);
-        if (ret)
-                goto out;
-
-        nlockee++;
-        qsort (int_lock->lockee, nlockee, sizeof (*int_lock->lockee),
-               afr_entry_lockee_cmp);
-        int_lock->lockee_count = nlockee;
-
-        ret = afr_transaction (transaction_frame, this, AFR_ENTRY_TRANSACTION);
-        if (ret < 0) {
-		op_errno = -ret;
-		goto out;
-        }
-
-	return 0;
+    afr_local_t *local = NULL;
+    call_frame_t *transaction_frame = NULL;
+    int ret = -1;
+    int op_errno = ENOMEM;
+
+    transaction_frame = copy_frame(frame);
+    if (!transaction_frame)
+        goto out;
+
+    local = AFR_FRAME_INIT(transaction_frame, op_errno);
+    if (!local)
+        goto out;
+
+    loc_copy(&local->loc, loc);
+    local->inode = inode_ref(loc->inode);
+    local->parent = inode_ref(loc->parent);
+
+    local->cont.rmdir.flags = flags;
+
+    if (xdata)
+        local->xdata_req = dict_copy_with_ref(xdata, NULL);
+    else
+        local->xdata_req = dict_new();
+
+    if (!local->xdata_req)
+        goto out;
+
+    local->op = GF_FOP_RMDIR;
+    local->transaction.wind = afr_rmdir_wind;
+    local->transaction.unwind = afr_rmdir_unwind;
+
+    ret = afr_build_parent_loc(&local->transaction.parent_loc, loc, &op_errno);
+    if (ret)
+        goto out;
+
+    local->transaction.main_frame = frame;
+    local->transaction.basename = AFR_BASENAME(loc->path);
+    ret = afr_transaction(transaction_frame, this, AFR_ENTRY_TRANSACTION);
+    if (ret < 0) {
+        op_errno = -ret;
+        goto out;
+    }
+
+    return 0;
 out:
-	if (transaction_frame)
-		AFR_STACK_DESTROY (transaction_frame);
+    if (transaction_frame)
+        AFR_STACK_DESTROY(transaction_frame);
 
-	AFR_STACK_UNWIND (rmdir, frame, -1, op_errno, NULL, NULL, NULL);
-        return 0;
+    AFR_STACK_UNWIND(rmdir, frame, -1, op_errno, NULL, NULL, NULL);
+    return 0;
 }
 
 /* }}} */
diff --git a/xlators/cluster/afr/src/afr-dir-write.h b/xlators/cluster/afr/src/afr-dir-write.h
index 02f0a3682d9..1d88c3b9b26 100644
--- a/xlators/cluster/afr/src/afr-dir-write.h
+++ b/xlators/cluster/afr/src/afr-dir-write.h
@@ -12,36 +12,35 @@
 #define __DIR_WRITE_H__
 
 int32_t
-afr_create (call_frame_t *frame, xlator_t *this,
-            loc_t *loc, int32_t flags, mode_t mode,
-            mode_t umask, fd_t *fd, dict_t *xdata);
+afr_create(call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t flags,
+           mode_t mode, mode_t umask, fd_t *fd, dict_t *xdata);
 
 int32_t
-afr_mknod (call_frame_t *frame, xlator_t *this,
-	   loc_t *loc, mode_t mode, dev_t dev, mode_t umask, dict_t *xdata);
+afr_mknod(call_frame_t *frame, xlator_t *this, loc_t *loc, mode_t mode,
+          dev_t dev, mode_t umask, dict_t *xdata);
 
 int32_t
-afr_mkdir (call_frame_t *frame, xlator_t *this,
-	   loc_t *loc, mode_t mode, mode_t umask, dict_t *xdata);
+afr_mkdir(call_frame_t *frame, xlator_t *this, loc_t *loc, mode_t mode,
+          mode_t umask, dict_t *xdata);
 
 int32_t
-afr_unlink (call_frame_t *frame, xlator_t *this,
-	    loc_t *loc, int xflag, dict_t *xdata);
+afr_unlink(call_frame_t *frame, xlator_t *this, loc_t *loc, int xflag,
+           dict_t *xdata);
 
 int32_t
-afr_rmdir (call_frame_t *frame, xlator_t *this,
-	   loc_t *loc, int flags, dict_t *xdata);
+afr_rmdir(call_frame_t *frame, xlator_t *this, loc_t *loc, int flags,
+          dict_t *xdata);
 
 int32_t
-afr_link (call_frame_t *frame, xlator_t *this,
-	  loc_t *oldloc, loc_t *newloc, dict_t *xdata);
+afr_link(call_frame_t *frame, xlator_t *this, loc_t *oldloc, loc_t *newloc,
+         dict_t *xdata);
 
 int32_t
-afr_rename (call_frame_t *frame, xlator_t *this,
-	    loc_t *oldloc, loc_t *newloc, dict_t *xdata);
+afr_rename(call_frame_t *frame, xlator_t *this, loc_t *oldloc, loc_t *newloc,
+           dict_t *xdata);
 
 int
-afr_symlink (call_frame_t *frame, xlator_t *this,
-	     const char *linkpath, loc_t *oldloc, mode_t umask, dict_t *params);
+afr_symlink(call_frame_t *frame, xlator_t *this, const char *linkpath,
+            loc_t *oldloc, mode_t umask, dict_t *params);
 
 #endif /* __DIR_WRITE_H__ */
diff --git a/xlators/cluster/afr/src/afr-inode-read.c b/xlators/cluster/afr/src/afr-inode-read.c
index 7fed62a5361..c5521704de2 100644
--- a/xlators/cluster/afr/src/afr-inode-read.c
+++ b/xlators/cluster/afr/src/afr-inode-read.c
@@ -8,7 +8,6 @@
   cases as published by the Free Software Foundation.
 */
 
-
 #include <libgen.h>
 #include <unistd.h>
 #include <fnmatch.h>
@@ -16,28 +15,20 @@
 #include <stdlib.h>
 #include <signal.h>
 
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
-
-#include "glusterfs.h"
+#include <glusterfs/glusterfs.h>
 #include "afr.h"
-#include "dict.h"
-#include "xlator.h"
-#include "hashfn.h"
-#include "logging.h"
-#include "stack.h"
-#include "list.h"
-#include "call-stub.h"
-#include "byte-order.h"
-#include "defaults.h"
-#include "common-utils.h"
-#include "compat-errno.h"
-#include "compat.h"
-#include "quota-common-utils.h"
+#include <glusterfs/dict.h>
+#include <glusterfs/logging.h>
+#include <glusterfs/list.h>
+#include <glusterfs/byte-order.h>
+#include <glusterfs/defaults.h>
+#include <glusterfs/common-utils.h>
+#include <glusterfs/compat-errno.h>
+#include <glusterfs/compat.h>
+#include <glusterfs/quota-common-utils.h>
 
 #include "afr-transaction.h"
+#include "afr-messages.h"
 
 /*
  * Quota size xattrs are not maintained by afr. There is a
@@ -49,146 +40,146 @@
  * */
 
 int
-afr_handle_quota_size (call_frame_t *frame, xlator_t *this)
+afr_handle_quota_size(call_frame_t *frame, xlator_t *this)
 {
-        unsigned char *readable   = NULL;
-        afr_local_t *local        = NULL;
-        afr_private_t *priv       = NULL;
-        struct afr_reply *replies = NULL;
-        int i                     = 0;
-        int ret                   = 0;
-        quota_meta_t size         = {0, };
-        quota_meta_t max_size     = {0, };
-        int readable_cnt          = 0;
-        int read_subvol           = -1;
-
-        local = frame->local;
-        priv = this->private;
-        replies = local->replies;
-
-        readable = alloca0 (priv->child_count);
-
-        afr_inode_read_subvol_get (local->inode, this, readable, 0, 0);
-
-        readable_cnt = AFR_COUNT (readable, priv->child_count);
-
-        for (i = 0; i < priv->child_count; i++) {
-                if (!replies[i].valid || replies[i].op_ret == -1)
-                        continue;
-                if (readable_cnt && !readable[i])
-                        continue;
-                if (!replies[i].xdata)
-                        continue;
-                ret = quota_dict_get_meta (replies[i].xdata, QUOTA_SIZE_KEY,
-                                           &size);
-                if (ret == -1)
-                        continue;
-                if (read_subvol == -1)
-                        read_subvol = i;
-                if (size.size > max_size.size ||
-                    (size.file_count + size.dir_count) >
-                    (max_size.file_count + max_size.dir_count))
-                        read_subvol = i;
-
-                if (size.size > max_size.size)
-                        max_size.size = size.size;
-                if (size.file_count > max_size.file_count)
-                        max_size.file_count = size.file_count;
-                if (size.dir_count > max_size.dir_count)
-                        max_size.dir_count = size.dir_count;
-        }
-
-        if (max_size.size == 0 && max_size.file_count == 0 &&
-            max_size.dir_count == 0)
-                return read_subvol;
-
-        for (i = 0; i < priv->child_count; i++) {
-                if (!replies[i].valid || replies[i].op_ret == -1)
-                        continue;
-                if (readable_cnt && !readable[i])
-                        continue;
-                if (!replies[i].xdata)
-                        continue;
-                quota_dict_set_meta (replies[i].xdata, QUOTA_SIZE_KEY,
-                                     &max_size, IA_IFDIR);
-        }
-
+    unsigned char *readable = NULL;
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
+    struct afr_reply *replies = NULL;
+    int i = 0;
+    int ret = 0;
+    quota_meta_t size = {
+        0,
+    };
+    quota_meta_t max_size = {
+        0,
+    };
+    int readable_cnt = 0;
+    int read_subvol = -1;
+
+    local = frame->local;
+    priv = this->private;
+    replies = local->replies;
+
+    readable = alloca0(priv->child_count);
+
+    afr_inode_read_subvol_get(local->inode, this, readable, 0, 0);
+
+    readable_cnt = AFR_COUNT(readable, priv->child_count);
+
+    for (i = 0; i < priv->child_count; i++) {
+        if (!replies[i].valid || replies[i].op_ret == -1)
+            continue;
+        if (readable_cnt && !readable[i])
+            continue;
+        if (!replies[i].xdata)
+            continue;
+        ret = quota_dict_get_meta(replies[i].xdata, QUOTA_SIZE_KEY,
+                                  SLEN(QUOTA_SIZE_KEY), &size);
+        if (ret == -1)
+            continue;
+        if (read_subvol == -1)
+            read_subvol = i;
+        if (size.size > max_size.size ||
+            (size.file_count + size.dir_count) >
+                (max_size.file_count + max_size.dir_count))
+            read_subvol = i;
+
+        if (size.size > max_size.size)
+            max_size.size = size.size;
+        if (size.file_count > max_size.file_count)
+            max_size.file_count = size.file_count;
+        if (size.dir_count > max_size.dir_count)
+            max_size.dir_count = size.dir_count;
+    }
+
+    if (max_size.size == 0 && max_size.file_count == 0 &&
+        max_size.dir_count == 0)
         return read_subvol;
-}
 
+    for (i = 0; i < priv->child_count; i++) {
+        if (!replies[i].valid || replies[i].op_ret == -1)
+            continue;
+        if (readable_cnt && !readable[i])
+            continue;
+        if (!replies[i].xdata)
+            continue;
+        quota_dict_set_meta(replies[i].xdata, QUOTA_SIZE_KEY, &max_size,
+                            IA_IFDIR);
+    }
+
+    return read_subvol;
+}
 
 /* {{{ access */
 
 int
-afr_access_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-		int op_ret, int op_errno, dict_t *xdata)
+afr_access_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int op_ret,
+               int op_errno, dict_t *xdata)
 {
-        afr_local_t     *local = NULL;
+    afr_local_t *local = NULL;
 
-        local = frame->local;
+    local = frame->local;
 
-	if (op_ret < 0) {
-		local->op_ret = op_ret;
-		local->op_errno = op_errno;
+    if (op_ret < 0) {
+        local->op_ret = op_ret;
+        local->op_errno = op_errno;
 
-		afr_read_txn_continue (frame, this, (long) cookie);
-		return 0;
-	}
+        afr_read_txn_continue(frame, this, (long)cookie);
+        return 0;
+    }
 
-	AFR_STACK_UNWIND (access, frame, op_ret, op_errno, xdata);
+    AFR_STACK_UNWIND(access, frame, op_ret, op_errno, xdata);
 
-        return 0;
+    return 0;
 }
 
-
 int
-afr_access_wind (call_frame_t *frame, xlator_t *this, int subvol)
+afr_access_wind(call_frame_t *frame, xlator_t *this, int subvol)
 {
-	afr_private_t *priv = NULL;
-	afr_local_t *local = NULL;
-
-	priv = this->private;
-	local = frame->local;
-
-	if (subvol == -1) {
-		AFR_STACK_UNWIND (access, frame, local->op_ret,
-				  local->op_errno, 0);
-		return 0;
-	}
-
-        STACK_WIND_COOKIE (frame, afr_access_cbk, (void *) (long) subvol,
-                           priv->children[subvol],
-                           priv->children[subvol]->fops->access,
-                           &local->loc, local->cont.access.mask,
-			   local->xdata_req);
-	return 0;
+    afr_private_t *priv = NULL;
+    afr_local_t *local = NULL;
+
+    priv = this->private;
+    local = frame->local;
+
+    if (subvol == -1) {
+        AFR_STACK_UNWIND(access, frame, local->op_ret, local->op_errno, 0);
+        return 0;
+    }
+
+    STACK_WIND_COOKIE(frame, afr_access_cbk, (void *)(long)subvol,
+                      priv->children[subvol],
+                      priv->children[subvol]->fops->access, &local->loc,
+                      local->cont.access.mask, local->xdata_req);
+    return 0;
 }
 
 int
-afr_access (call_frame_t *frame, xlator_t *this, loc_t *loc,
-	    int mask, dict_t *xdata)
+afr_access(call_frame_t *frame, xlator_t *this, loc_t *loc, int mask,
+           dict_t *xdata)
 {
-        afr_local_t *local = NULL;
-	int op_errno = 0;
+    afr_local_t *local = NULL;
+    int op_errno = 0;
 
-        local = AFR_FRAME_INIT (frame, op_errno);
-	if (!local)
-		goto out;
+    local = AFR_FRAME_INIT(frame, op_errno);
+    if (!local)
+        goto out;
 
-	local->op = GF_FOP_ACCESS;
-	loc_copy (&local->loc, loc);
-	local->cont.access.mask = mask;
-	if (xdata)
-		local->xdata_req = dict_ref (xdata);
+    local->op = GF_FOP_ACCESS;
+    loc_copy(&local->loc, loc);
+    local->cont.access.mask = mask;
+    if (xdata)
+        local->xdata_req = dict_ref(xdata);
 
-	afr_read_txn (frame, this, loc->inode, afr_access_wind,
-		      AFR_METADATA_TRANSACTION);
+    afr_read_txn(frame, this, loc->inode, afr_access_wind,
+                 AFR_METADATA_TRANSACTION);
 
-	return 0;
+    return 0;
 out:
-	AFR_STACK_UNWIND (access, frame, -1, op_errno, NULL);
+    AFR_STACK_UNWIND(access, frame, -1, op_errno, NULL);
 
-        return 0;
+    return 0;
 }
 
 /* }}} */
@@ -196,152 +187,140 @@ out:
 /* {{{ stat */
 
 int
-afr_stat_cbk (call_frame_t *frame, void *cookie,
-              xlator_t *this, int32_t op_ret, int32_t op_errno,
-              struct iatt *buf, dict_t *xdata)
+afr_stat_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int32_t op_ret,
+             int32_t op_errno, struct iatt *buf, dict_t *xdata)
 {
-        afr_local_t     *local = NULL;
+    afr_local_t *local = NULL;
 
-        local = frame->local;
+    local = frame->local;
 
-	if (op_ret < 0) {
-		local->op_ret = op_ret;
-		local->op_errno = op_errno;
+    if (op_ret < 0) {
+        local->op_ret = op_ret;
+        local->op_errno = op_errno;
 
-		afr_read_txn_continue (frame, this, (long) cookie);
-		return 0;
-	}
+        afr_read_txn_continue(frame, this, (long)cookie);
+        return 0;
+    }
 
-	AFR_STACK_UNWIND (stat, frame, op_ret, op_errno, buf, xdata);
+    AFR_STACK_UNWIND(stat, frame, op_ret, op_errno, buf, xdata);
 
-        return 0;
+    return 0;
 }
 
-
 int
-afr_stat_wind (call_frame_t *frame, xlator_t *this, int subvol)
+afr_stat_wind(call_frame_t *frame, xlator_t *this, int subvol)
 {
-	afr_private_t *priv = NULL;
-	afr_local_t *local = NULL;
-
-	priv = this->private;
-	local = frame->local;
-
-	if (subvol == -1) {
-		AFR_STACK_UNWIND (stat, frame, local->op_ret, local->op_errno,
-				  0, 0);
-		return 0;
-	}
-
-        STACK_WIND_COOKIE (frame, afr_stat_cbk, (void *) (long) subvol,
-                           priv->children[subvol],
-                           priv->children[subvol]->fops->stat,
-                           &local->loc, local->xdata_req);
-	return 0;
+    afr_private_t *priv = NULL;
+    afr_local_t *local = NULL;
+
+    priv = this->private;
+    local = frame->local;
+
+    if (subvol == -1) {
+        AFR_STACK_UNWIND(stat, frame, local->op_ret, local->op_errno, 0, 0);
+        return 0;
+    }
+
+    STACK_WIND_COOKIE(
+        frame, afr_stat_cbk, (void *)(long)subvol, priv->children[subvol],
+        priv->children[subvol]->fops->stat, &local->loc, local->xdata_req);
+    return 0;
 }
 
 int
-afr_stat (call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xdata)
+afr_stat(call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xdata)
 {
-        afr_local_t *local = NULL;
-	int op_errno = 0;
+    afr_local_t *local = NULL;
+    int op_errno = 0;
 
-        local = AFR_FRAME_INIT (frame, op_errno);
-	if (!local)
-		goto out;
+    local = AFR_FRAME_INIT(frame, op_errno);
+    if (!local)
+        goto out;
 
-	local->op = GF_FOP_STAT;
-	loc_copy (&local->loc, loc);
-	if (xdata)
-		local->xdata_req = dict_ref (xdata);
+    local->op = GF_FOP_STAT;
+    loc_copy(&local->loc, loc);
+    if (xdata)
+        local->xdata_req = dict_ref(xdata);
 
-	afr_read_txn (frame, this, loc->inode, afr_stat_wind,
-		      AFR_DATA_TRANSACTION);
+    afr_read_txn(frame, this, loc->inode, afr_stat_wind, AFR_DATA_TRANSACTION);
 
-	return 0;
+    return 0;
 out:
-	AFR_STACK_UNWIND (stat, frame, -1, op_errno, NULL, NULL);
+    AFR_STACK_UNWIND(stat, frame, -1, op_errno, NULL, NULL);
 
-        return 0;
+    return 0;
 }
 
-
 /* }}} */
 
 /* {{{ fstat */
 
 int
-afr_fstat_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-               int32_t op_ret, int32_t op_errno, struct iatt *buf,
-               dict_t *xdata)
+afr_fstat_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int32_t op_ret,
+              int32_t op_errno, struct iatt *buf, dict_t *xdata)
 {
-        afr_local_t     *local = NULL;
+    afr_local_t *local = NULL;
 
-        local = frame->local;
+    local = frame->local;
 
-	if (op_ret < 0) {
-		local->op_ret = op_ret;
-		local->op_errno = op_errno;
+    if (op_ret < 0) {
+        local->op_ret = op_ret;
+        local->op_errno = op_errno;
 
-		afr_read_txn_continue (frame, this, (long) cookie);
-		return 0;
-	}
+        afr_read_txn_continue(frame, this, (long)cookie);
+        return 0;
+    }
 
-	AFR_STACK_UNWIND (fstat, frame, op_ret, op_errno, buf, xdata);
+    AFR_STACK_UNWIND(fstat, frame, op_ret, op_errno, buf, xdata);
 
-        return 0;
+    return 0;
 }
 
-
 int
-afr_fstat_wind (call_frame_t *frame, xlator_t *this, int subvol)
+afr_fstat_wind(call_frame_t *frame, xlator_t *this, int subvol)
 {
-	afr_private_t *priv = NULL;
-	afr_local_t *local = NULL;
-
-	priv = this->private;
-	local = frame->local;
-
-	if (subvol == -1) {
-		AFR_STACK_UNWIND (fstat, frame, local->op_ret, local->op_errno,
-				  0, 0);
-		return 0;
-	}
-
-        STACK_WIND_COOKIE (frame, afr_fstat_cbk, (void *) (long) subvol,
-                           priv->children[subvol],
-                           priv->children[subvol]->fops->fstat,
-                           local->fd, local->xdata_req);
-	return 0;
-}
+    afr_private_t *priv = NULL;
+    afr_local_t *local = NULL;
 
+    priv = this->private;
+    local = frame->local;
+
+    if (subvol == -1) {
+        AFR_STACK_UNWIND(fstat, frame, local->op_ret, local->op_errno, 0, 0);
+        return 0;
+    }
+
+    STACK_WIND_COOKIE(
+        frame, afr_fstat_cbk, (void *)(long)subvol, priv->children[subvol],
+        priv->children[subvol]->fops->fstat, local->fd, local->xdata_req);
+    return 0;
+}
 
 int32_t
-afr_fstat (call_frame_t *frame, xlator_t *this,
-           fd_t *fd, dict_t *xdata)
+afr_fstat(call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *xdata)
 {
-        afr_local_t *local = NULL;
-	int op_errno = 0;
+    afr_local_t *local = NULL;
+    int op_errno = 0;
 
-        local = AFR_FRAME_INIT (frame, op_errno);
-	if (!local)
-		goto out;
+    AFR_ERROR_OUT_IF_FDCTX_INVALID(fd, this, op_errno, out);
+    local = AFR_FRAME_INIT(frame, op_errno);
+    if (!local)
+        goto out;
 
-	local->op = GF_FOP_FSTAT;
-	local->fd = fd_ref (fd);
-	if (xdata)
-		local->xdata_req = dict_ref (xdata);
+    local->op = GF_FOP_FSTAT;
+    local->fd = fd_ref(fd);
+    if (xdata)
+        local->xdata_req = dict_ref(xdata);
 
-        afr_fix_open (fd, this);
+    afr_fix_open(fd, this);
 
-	afr_read_txn (frame, this, fd->inode, afr_fstat_wind,
-		      AFR_DATA_TRANSACTION);
+    afr_read_txn(frame, this, fd->inode, afr_fstat_wind, AFR_DATA_TRANSACTION);
 
-	return 0;
+    return 0;
 out:
-	AFR_STACK_UNWIND (fstat, frame, -1, op_errno, NULL, NULL);
+    AFR_STACK_UNWIND(fstat, frame, -1, op_errno, NULL, NULL);
 
-        return 0;
+    return 0;
 }
 
 /* }}} */
@@ -349,1394 +328,1567 @@ out:
 /* {{{ readlink */
 
 int
-afr_readlink_cbk (call_frame_t *frame, void *cookie,
-                  xlator_t *this, int32_t op_ret, int32_t op_errno,
-                  const char *buf, struct iatt *sbuf, dict_t *xdata)
+afr_readlink_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                 int32_t op_ret, int32_t op_errno, const char *buf,
+                 struct iatt *sbuf, dict_t *xdata)
 {
-	afr_local_t *local = NULL;
+    afr_local_t *local = NULL;
 
-	local = frame->local;
+    local = frame->local;
 
-	if (op_ret < 0) {
-		local->op_ret = -1;
-		local->op_errno = op_errno;
+    if (op_ret < 0) {
+        local->op_ret = -1;
+        local->op_errno = op_errno;
 
-		afr_read_txn_continue (frame, this, (long) cookie);
-		return 0;
-	}
+        afr_read_txn_continue(frame, this, (long)cookie);
+        return 0;
+    }
 
-	AFR_STACK_UNWIND (readlink, frame, op_ret, op_errno,
-			  buf, sbuf, xdata);
-	return 0;
+    AFR_STACK_UNWIND(readlink, frame, op_ret, op_errno, buf, sbuf, xdata);
+    return 0;
 }
 
 int
-afr_readlink_wind (call_frame_t *frame, xlator_t *this, int subvol)
+afr_readlink_wind(call_frame_t *frame, xlator_t *this, int subvol)
 {
-	afr_local_t *local = NULL;
-	afr_private_t *priv = NULL;
-
-	local = frame->local;
-	priv = this->private;
-
-	if (subvol == -1) {
-		AFR_STACK_UNWIND (readlink, frame, local->op_ret,
-				  local->op_errno, 0, 0, 0);
-		return 0;
-	}
-
-	STACK_WIND_COOKIE (frame, afr_readlink_cbk, (void *) (long) subvol,
-			   priv->children[subvol],
-			   priv->children[subvol]->fops->readlink,
-			   &local->loc, local->cont.readlink.size,
-			   local->xdata_req);
-	return 0;
-}
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
 
+    local = frame->local;
+    priv = this->private;
+
+    if (subvol == -1) {
+        AFR_STACK_UNWIND(readlink, frame, local->op_ret, local->op_errno, 0, 0,
+                         0);
+        return 0;
+    }
+
+    STACK_WIND_COOKIE(frame, afr_readlink_cbk, (void *)(long)subvol,
+                      priv->children[subvol],
+                      priv->children[subvol]->fops->readlink, &local->loc,
+                      local->cont.readlink.size, local->xdata_req);
+    return 0;
+}
 
 int
-afr_readlink (call_frame_t *frame, xlator_t *this,
-              loc_t *loc, size_t size, dict_t *xdata)
+afr_readlink(call_frame_t *frame, xlator_t *this, loc_t *loc, size_t size,
+             dict_t *xdata)
 {
-        afr_local_t   * local      = NULL;
-        int32_t         op_errno   = 0;
+    afr_local_t *local = NULL;
+    int32_t op_errno = 0;
 
-	local = AFR_FRAME_INIT (frame, op_errno);
-	if (!local)
-		goto out;
+    local = AFR_FRAME_INIT(frame, op_errno);
+    if (!local)
+        goto out;
 
-	local->op = GF_FOP_READLINK;
-        loc_copy (&local->loc, loc);
-        local->cont.readlink.size = size;
-	if (xdata)
-		local->xdata_req = dict_ref (xdata);
+    local->op = GF_FOP_READLINK;
+    loc_copy(&local->loc, loc);
+    local->cont.readlink.size = size;
+    if (xdata)
+        local->xdata_req = dict_ref(xdata);
 
-	afr_read_txn (frame, this, loc->inode, afr_readlink_wind,
-		      AFR_DATA_TRANSACTION);
+    afr_read_txn(frame, this, loc->inode, afr_readlink_wind,
+                 AFR_DATA_TRANSACTION);
 
-        return 0;
+    return 0;
 out:
-	AFR_STACK_UNWIND(readlink, frame, -1, op_errno, 0, 0, 0);
+    AFR_STACK_UNWIND(readlink, frame, -1, op_errno, 0, 0, 0);
 
-	return 0;
+    return 0;
 }
 
-
 /* }}} */
 
 /* {{{ getxattr */
 
 struct _xattr_key {
-        char *key;
-        struct list_head list;
+    char *key;
+    struct list_head list;
 };
 
-
 int
-__gather_xattr_keys (dict_t *dict, char *key, data_t *value,
-                     void *data)
+__gather_xattr_keys(dict_t *dict, char *key, data_t *value, void *data)
 {
-        struct list_head *  list  = data;
-        struct _xattr_key * xkey  = NULL;
+    struct list_head *list = data;
+    struct _xattr_key *xkey = NULL;
 
-        if (!strncmp (key, AFR_XATTR_PREFIX,
-                      strlen (AFR_XATTR_PREFIX))) {
+    if (!strncmp(key, AFR_XATTR_PREFIX, SLEN(AFR_XATTR_PREFIX))) {
+        xkey = GF_MALLOC(sizeof(*xkey), gf_afr_mt_xattr_key);
+        if (!xkey)
+            return -1;
 
-                xkey = GF_CALLOC (1, sizeof (*xkey), gf_afr_mt_xattr_key);
-                if (!xkey)
-                        return -1;
+        xkey->key = key;
+        INIT_LIST_HEAD(&xkey->list);
 
-                xkey->key = key;
-                INIT_LIST_HEAD (&xkey->list);
-
-                list_add_tail (&xkey->list, list);
-        }
-        return 0;
+        list_add_tail(&xkey->list, list);
+    }
+    return 0;
 }
 
-
 void
-afr_filter_xattrs (dict_t *dict)
+afr_filter_xattrs(dict_t *dict)
 {
-        struct list_head   keys = {0,};
-        struct _xattr_key *key  = NULL;
-        struct _xattr_key *tmp  = NULL;
+    struct list_head keys = {
+        0,
+    };
+    struct _xattr_key *key = NULL;
+    struct _xattr_key *tmp = NULL;
 
-        INIT_LIST_HEAD (&keys);
+    INIT_LIST_HEAD(&keys);
 
-        dict_foreach (dict, __gather_xattr_keys,
-                      (void *) &keys);
+    dict_foreach(dict, __gather_xattr_keys, (void *)&keys);
 
-        list_for_each_entry_safe (key, tmp, &keys, list) {
-                dict_del (dict, key->key);
+    list_for_each_entry_safe(key, tmp, &keys, list)
+    {
+        dict_del(dict, key->key);
 
-                list_del_init (&key->list);
+        list_del_init(&key->list);
 
-                GF_FREE (key);
-        }
+        GF_FREE(key);
+    }
 }
 
+static gf_boolean_t
+afr_getxattr_ignorable_errnos(int32_t op_errno)
+{
+    if (op_errno == ENODATA || op_errno == ENOTSUP || op_errno == ERANGE ||
+        op_errno == ENAMETOOLONG)
+        return _gf_true;
 
+    return _gf_false;
+}
 int
-afr_getxattr_cbk (call_frame_t *frame, void *cookie,
-                  xlator_t *this, int32_t op_ret, int32_t op_errno,
-                  dict_t *dict, dict_t *xdata)
+afr_getxattr_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                 int32_t op_ret, int32_t op_errno, dict_t *dict, dict_t *xdata)
 {
-        afr_local_t *local = NULL;
+    afr_local_t *local = NULL;
 
-        local = frame->local;
+    local = frame->local;
 
-	if (op_ret < 0) {
-		local->op_ret = op_ret;
-		local->op_errno = op_errno;
+    if (op_ret < 0 && !afr_getxattr_ignorable_errnos(op_errno)) {
+        local->op_ret = op_ret;
+        local->op_errno = op_errno;
 
-		afr_read_txn_continue (frame, this, (long) cookie);
-		return 0;
-	}
+        afr_read_txn_continue(frame, this, (long)cookie);
+        return 0;
+    }
 
-	if (dict)
-		afr_filter_xattrs (dict);
+    if (dict)
+        afr_filter_xattrs(dict);
 
-	AFR_STACK_UNWIND (getxattr, frame, op_ret, op_errno, dict, xdata);
+    AFR_STACK_UNWIND(getxattr, frame, op_ret, op_errno, dict, xdata);
 
-        return 0;
+    return 0;
 }
 
-
 int
-afr_getxattr_wind (call_frame_t *frame, xlator_t *this, int subvol)
+afr_getxattr_wind(call_frame_t *frame, xlator_t *this, int subvol)
 {
-	afr_local_t *local = NULL;
-	afr_private_t *priv = NULL;
-
-	local = frame->local;
-	priv = this->private;
-
-	if (subvol == -1) {
-                AFR_STACK_UNWIND (getxattr, frame, local->op_ret,
-				  local->op_errno, NULL, NULL);
-		return 0;
-	}
-
-	STACK_WIND_COOKIE (frame, afr_getxattr_cbk, (void *) (long) subvol,
-			   priv->children[subvol],
-			   priv->children[subvol]->fops->getxattr,
-			   &local->loc, local->cont.getxattr.name,
-			   local->xdata_req);
-	return 0;
-}
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
 
+    local = frame->local;
+    priv = this->private;
+
+    if (subvol == -1) {
+        AFR_STACK_UNWIND(getxattr, frame, local->op_ret, local->op_errno, NULL,
+                         NULL);
+        return 0;
+    }
+
+    STACK_WIND_COOKIE(frame, afr_getxattr_cbk, (void *)(long)subvol,
+                      priv->children[subvol],
+                      priv->children[subvol]->fops->getxattr, &local->loc,
+                      local->cont.getxattr.name, local->xdata_req);
+    return 0;
+}
 
 int32_t
-afr_getxattr_unwind (call_frame_t *frame, int op_ret, int op_errno,
-                     dict_t *dict, dict_t *xdata)
+afr_getxattr_unwind(call_frame_t *frame, int op_ret, int op_errno, dict_t *dict,
+                    dict_t *xdata)
 
 {
-        AFR_STACK_UNWIND (getxattr, frame, op_ret, op_errno, dict, xdata);
-        return 0;
+    AFR_STACK_UNWIND(getxattr, frame, op_ret, op_errno, dict, xdata);
+    return 0;
 }
 
 int32_t
-afr_fgetxattr_clrlk_cbk (call_frame_t *frame, void *cookie,
-                         xlator_t *this, int32_t op_ret, int32_t op_errno,
-                         dict_t *dict, dict_t *xdata)
+afr_fgetxattr_clrlk_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                        int32_t op_ret, int32_t op_errno, dict_t *dict,
+                        dict_t *xdata)
 {
-        afr_local_t     *local                  = NULL;
-        afr_private_t   *priv                   = NULL;
-        xlator_t        **children              = NULL;
-        dict_t          *xattr                  = NULL;
-        char            *tmp_report             = NULL;
-        char            lk_summary[1024]        = {0,};
-        int             serz_len                = 0;
-        int32_t         callcnt                 = 0;
-        long int        cky                     = 0;
-        int             ret                     = 0;
-
-        priv     = this->private;
-        children = priv->children;
-
-        local = frame->local;
-        cky = (long) cookie;
-
-        LOCK (&frame->lock);
-        {
-                callcnt = --local->call_count;
-                if (op_ret == -1)
-                        local->replies[cky].op_errno = op_errno;
-
-                if (!local->dict)
-                        local->dict = dict_new ();
-                if (local->dict) {
-                        ret = dict_get_str (dict, local->cont.getxattr.name,
-                                            &tmp_report);
-                        if (ret)
-                                goto unlock;
-                        ret = dict_set_dynstr (local->dict,
-                                               children[cky]->name,
-                                               gf_strdup (tmp_report));
-                        if (ret)
-                                goto unlock;
-                }
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
+    xlator_t **children = NULL;
+    dict_t *xattr = NULL;
+    char *tmp_report = NULL;
+    char lk_summary[1024] = {
+        0,
+    };
+    int serz_len = 0;
+    int32_t callcnt = 0;
+    long int cky = 0;
+    int ret = 0;
+    int keylen = 0;
+    int children_keylen = 0;
+
+    priv = this->private;
+    children = priv->children;
+
+    local = frame->local;
+    cky = (long)cookie;
+    keylen = strlen(local->cont.getxattr.name);
+    children_keylen = strlen(children[cky]->name);
+
+    LOCK(&frame->lock);
+    {
+        callcnt = --local->call_count;
+        if (op_ret == -1)
+            local->replies[cky].op_errno = op_errno;
+
+        if (!local->dict)
+            local->dict = dict_new();
+        if (local->dict) {
+            ret = dict_get_strn(dict, local->cont.getxattr.name, keylen,
+                                &tmp_report);
+            if (ret)
+                goto unlock;
+            ret = dict_set_dynstrn(local->dict, children[cky]->name,
+                                   children_keylen, gf_strdup(tmp_report));
+            if (ret)
+                goto unlock;
         }
+    }
 unlock:
-        UNLOCK (&frame->lock);
-
-        if (!callcnt) {
-                xattr = dict_new ();
-                if (!xattr) {
-                        op_ret = -1;
-                        op_errno = ENOMEM;
-                        goto unwind;
-                }
-                ret = dict_serialize_value_with_delim (local->dict,
-                                                       lk_summary,
-                                                       &serz_len, '\n');
-                if (ret) {
-                        op_ret = -1;
-                        op_errno = ENOMEM;
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "Error serializing dictionary");
-                        goto unwind;
-                }
-                if (serz_len == -1)
-                        snprintf (lk_summary, sizeof (lk_summary),
-                                  "No locks cleared.");
-                ret = dict_set_dynstr (xattr, local->cont.getxattr.name,
-                                       gf_strdup (lk_summary));
-                if (ret) {
-                        op_ret = -1;
-                        op_errno = ENOMEM;
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "Error setting dictionary");
-                        goto unwind;
-                }
+    UNLOCK(&frame->lock);
+
+    if (!callcnt) {
+        xattr = dict_new();
+        if (!xattr) {
+            op_ret = -1;
+            op_errno = ENOMEM;
+            goto unwind;
+        }
+        ret = dict_serialize_value_with_delim(local->dict, lk_summary,
+                                              &serz_len, '\n');
+        if (ret) {
+            op_ret = -1;
+            op_errno = ENOMEM;
+            goto unwind;
+        }
+        if (serz_len == -1)
+            snprintf(lk_summary, sizeof(lk_summary), "No locks cleared.");
+        ret = dict_set_dynstrn(xattr, local->cont.getxattr.name, keylen,
+                               gf_strdup(lk_summary));
+        if (ret) {
+            op_ret = -1;
+            op_errno = ENOMEM;
+            gf_msg(this->name, GF_LOG_ERROR, ENOMEM, AFR_MSG_DICT_SET_FAILED,
+                   "Error setting dictionary");
+            goto unwind;
+        }
 
-        unwind:
-                // Updating child_errno with more recent 'events'
-                op_errno = afr_final_errno (local, priv);
+        op_errno = afr_final_errno(local, priv);
 
-                AFR_STACK_UNWIND (fgetxattr, frame, op_ret, op_errno, xattr,
-                                  xdata);
-                if (xattr)
-                        dict_unref (xattr);
-        }
+    unwind:
+        AFR_STACK_UNWIND(fgetxattr, frame, op_ret, op_errno, xattr, xdata);
+        if (xattr)
+            dict_unref(xattr);
+    }
 
-        return ret;
+    return ret;
 }
 
 int32_t
-afr_getxattr_clrlk_cbk (call_frame_t *frame, void *cookie,
-                        xlator_t *this, int32_t op_ret, int32_t op_errno,
-                        dict_t *dict, dict_t *xdata)
+afr_getxattr_clrlk_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                       int32_t op_ret, int32_t op_errno, dict_t *dict,
+                       dict_t *xdata)
 {
-        afr_local_t     *local                  = NULL;
-        afr_private_t   *priv                   = NULL;
-        xlator_t        **children              = NULL;
-        dict_t          *xattr                  = NULL;
-        char            *tmp_report             = NULL;
-        char            lk_summary[1024]        = {0,};
-        int             serz_len                = 0;
-        int32_t         callcnt                 = 0;
-        long int        cky                     = 0;
-        int             ret                     = 0;
-
-        priv     = this->private;
-        children = priv->children;
-
-        local = frame->local;
-        cky = (long) cookie;
-
-        LOCK (&frame->lock);
-        {
-                callcnt = --local->call_count;
-                if (op_ret == -1)
-                        local->replies[cky].op_errno = op_errno;
-
-                if (!local->dict)
-                        local->dict = dict_new ();
-                if (local->dict) {
-                        ret = dict_get_str (dict, local->cont.getxattr.name,
-                                            &tmp_report);
-                        if (ret)
-                                goto unlock;
-                        ret = dict_set_dynstr (local->dict,
-                                               children[cky]->name,
-                                               gf_strdup (tmp_report));
-                        if (ret)
-                                goto unlock;
-                }
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
+    xlator_t **children = NULL;
+    dict_t *xattr = NULL;
+    char *tmp_report = NULL;
+    char lk_summary[1024] = {
+        0,
+    };
+    int serz_len = 0;
+    int32_t callcnt = 0;
+    long int cky = 0;
+    int ret = 0;
+    int keylen = 0;
+    int children_keylen = 0;
+
+    priv = this->private;
+    children = priv->children;
+
+    local = frame->local;
+    cky = (long)cookie;
+
+    keylen = strlen(local->cont.getxattr.name);
+    children_keylen = strlen(children[cky]->name);
+
+    LOCK(&frame->lock);
+    {
+        callcnt = --local->call_count;
+        if (op_ret == -1)
+            local->replies[cky].op_errno = op_errno;
+
+        if (!local->dict)
+            local->dict = dict_new();
+        if (local->dict) {
+            ret = dict_get_strn(dict, local->cont.getxattr.name, keylen,
+                                &tmp_report);
+            if (ret)
+                goto unlock;
+            ret = dict_set_dynstrn(local->dict, children[cky]->name,
+                                   children_keylen, gf_strdup(tmp_report));
+            if (ret)
+                goto unlock;
         }
+    }
 unlock:
-        UNLOCK (&frame->lock);
-
-        if (!callcnt) {
-                xattr = dict_new ();
-                if (!xattr) {
-                        op_ret = -1;
-                        op_errno = ENOMEM;
-                        goto unwind;
-                }
-                ret = dict_serialize_value_with_delim (local->dict,
-                                                       lk_summary,
-                                                       &serz_len, '\n');
-                if (ret) {
-                        op_ret = -1;
-                        op_errno = ENOMEM;
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "Error serializing dictionary");
-                        goto unwind;
-                }
-                if (serz_len == -1)
-                        snprintf (lk_summary, sizeof (lk_summary),
-                                  "No locks cleared.");
-                ret = dict_set_dynstr (xattr, local->cont.getxattr.name,
-                                       gf_strdup (lk_summary));
-                if (ret) {
-                        op_ret = -1;
-                        op_errno = ENOMEM;
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "Error setting dictionary");
-                        goto unwind;
-                }
+    UNLOCK(&frame->lock);
+
+    if (!callcnt) {
+        xattr = dict_new();
+        if (!xattr) {
+            op_ret = -1;
+            op_errno = ENOMEM;
+            goto unwind;
+        }
+        ret = dict_serialize_value_with_delim(local->dict, lk_summary,
+                                              &serz_len, '\n');
+        if (ret) {
+            op_ret = -1;
+            op_errno = ENOMEM;
+            goto unwind;
+        }
+        if (serz_len == -1)
+            snprintf(lk_summary, sizeof(lk_summary), "No locks cleared.");
+        ret = dict_set_dynstrn(xattr, local->cont.getxattr.name, keylen,
+                               gf_strdup(lk_summary));
+        if (ret) {
+            op_ret = -1;
+            op_errno = ENOMEM;
+            gf_msg(this->name, GF_LOG_ERROR, ENOMEM, AFR_MSG_DICT_SET_FAILED,
+                   "Error setting dictionary");
+            goto unwind;
+        }
 
-        unwind:
-                // Updating child_errno with more recent 'events'
-                op_errno = afr_final_errno (local, priv);
+        op_errno = afr_final_errno(local, priv);
 
-                AFR_STACK_UNWIND (getxattr, frame, op_ret, op_errno, xattr, xdata);
+    unwind:
+        AFR_STACK_UNWIND(getxattr, frame, op_ret, op_errno, xattr, xdata);
 
-                if (xattr)
-                        dict_unref (xattr);
-        }
+        if (xattr)
+            dict_unref(xattr);
+    }
 
-        return ret;
+    return ret;
 }
 
 /**
  * node-uuid cbk uses next child querying mechanism
  */
 int32_t
-afr_getxattr_node_uuid_cbk (call_frame_t *frame, void *cookie,
-                            xlator_t *this, int32_t op_ret, int32_t op_errno,
-                            dict_t *dict, dict_t *xdata)
+afr_getxattr_node_uuid_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                           int32_t op_ret, int32_t op_errno, dict_t *dict,
+                           dict_t *xdata)
 {
-        afr_private_t  *priv            = NULL;
-        afr_local_t    *local           = NULL;
-        xlator_t      **children        = NULL;
-        int             unwind          = 1;
-        int             curr_call_child = 0;
+    afr_private_t *priv = NULL;
+    afr_local_t *local = NULL;
+    xlator_t **children = NULL;
+    int unwind = 1;
+    int curr_call_child = 0;
 
-        priv = this->private;
-        children = priv->children;
+    priv = this->private;
+    children = priv->children;
 
-        local = frame->local;
+    local = frame->local;
 
-        if (op_ret == -1) { /** query the _next_ child */
-
-                /**
-                 * _current_ becomes _next_
-                 * If done with all childs and yet no success; give up !
-                 */
-                curr_call_child = (int) ((long)cookie);
-                if (++curr_call_child == priv->child_count)
-                        goto unwind;
-
-                gf_log (this->name, GF_LOG_WARNING,
-                        "op_ret (-1): Re-querying afr-child (%d/%d)",
-                        curr_call_child, priv->child_count);
-
-                unwind = 0;
-                STACK_WIND_COOKIE (frame, afr_getxattr_node_uuid_cbk,
-                                   (void *) (long) curr_call_child,
-                                   children[curr_call_child],
-                                   children[curr_call_child]->fops->getxattr,
-                                   &local->loc,
-                                   local->cont.getxattr.name,
-                                   NULL);
+    if (op_ret == -1) { /** query the _next_ child */
+
+        /**
+         * _current_ becomes _next_
+         * If done with all children and yet no success; give up !
+         */
+        curr_call_child = (int)((long)cookie);
+        if (++curr_call_child == priv->child_count)
+            goto unwind;
+
+        gf_msg_debug(this->name, op_errno,
+                     "op_ret (-1): Re-querying afr-child (%d/%d)",
+                     curr_call_child, priv->child_count);
+
+        unwind = 0;
+        STACK_WIND_COOKIE(
+            frame, afr_getxattr_node_uuid_cbk, (void *)(long)curr_call_child,
+            children[curr_call_child],
+            children[curr_call_child]->fops->getxattr, &local->loc,
+            local->cont.getxattr.name, local->xdata_req);
+    }
+
+unwind:
+    if (unwind)
+        AFR_STACK_UNWIND(getxattr, frame, op_ret, op_errno, dict, xdata);
+
+    return 0;
+}
+
+/**
+ * list-node-uuids cbk returns the list of node_uuids for the subvolume.
+ */
+int32_t
+afr_getxattr_list_node_uuids_cbk(call_frame_t *frame, void *cookie,
+                                 xlator_t *this, int32_t op_ret,
+                                 int32_t op_errno, dict_t *dict, dict_t *xdata)
+{
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
+    int32_t callcnt = 0;
+    int ret = 0;
+    char *xattr_serz = NULL;
+    long cky = 0;
+    int32_t tlen = 0;
+
+    local = frame->local;
+    priv = this->private;
+    cky = (long)cookie;
+
+    LOCK(&frame->lock);
+    {
+        callcnt = --local->call_count;
+        local->replies[cky].valid = 1;
+        local->replies[cky].op_ret = op_ret;
+        local->replies[cky].op_errno = op_errno;
+
+        if (op_ret < 0)
+            goto unlock;
+
+        local->op_ret = 0;
+
+        if (!local->xdata_rsp && xdata)
+            local->xdata_rsp = dict_ref(xdata);
+        local->replies[cky].xattr = dict_ref(dict);
+    }
+
+unlock:
+    UNLOCK(&frame->lock);
+
+    if (!callcnt) {
+        if (local->op_ret != 0) {
+            /* All bricks gave an error. */
+            local->op_errno = afr_final_errno(local, priv);
+            goto unwind;
         }
 
- unwind:
-        if (unwind)
-                AFR_STACK_UNWIND (getxattr, frame, op_ret, op_errno, dict,
-                                  NULL);
+        /*Since we store the UUID0_STR as node uuid for down bricks and
+         *for non zero op_ret, assigning length to  priv->child_count
+         *number of uuids*/
+        local->cont.getxattr.xattr_len = (SLEN(UUID0_STR) + 2) *
+                                         priv->child_count;
+
+        if (!local->dict)
+            local->dict = dict_new();
+        if (!local->dict) {
+            local->op_ret = -1;
+            local->op_errno = ENOMEM;
+            goto unwind;
+        }
 
-        return 0;
+        xattr_serz = GF_CALLOC(local->cont.getxattr.xattr_len, sizeof(char),
+                               gf_common_mt_char);
+
+        if (!xattr_serz) {
+            local->op_ret = -1;
+            local->op_errno = ENOMEM;
+            goto unwind;
+        }
+
+        ret = afr_serialize_xattrs_with_delimiter(frame, this, xattr_serz,
+                                                  UUID0_STR, &tlen, ' ');
+        if (ret) {
+            local->op_ret = -1;
+            local->op_errno = ENOMEM;
+            GF_FREE(xattr_serz);
+            goto unwind;
+        }
+        ret = dict_set_dynstr_sizen(local->dict, GF_XATTR_LIST_NODE_UUIDS_KEY,
+                                    xattr_serz);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, -ret, AFR_MSG_DICT_SET_FAILED,
+                   "Cannot set node_uuid key in dict");
+            local->op_ret = -1;
+            local->op_errno = ENOMEM;
+            if (ret == -EINVAL)
+                GF_FREE(xattr_serz);
+        } else {
+            local->op_ret = local->cont.getxattr.xattr_len - 1;
+            local->op_errno = 0;
+        }
+
+    unwind:
+        AFR_STACK_UNWIND(getxattr, frame, local->op_ret, local->op_errno,
+                         local->dict, local->xdata_rsp);
+    }
+
+    return ret;
 }
 
 int32_t
-afr_getxattr_quota_size_cbk (call_frame_t *frame, void *cookie,
-                             xlator_t *this, int32_t op_ret, int32_t op_errno,
-                             dict_t *dict, dict_t *xdata)
+afr_getxattr_quota_size_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                            int32_t op_ret, int32_t op_errno, dict_t *dict,
+                            dict_t *xdata)
 {
-        int         idx         = (long) cookie;
-        int         call_count  = 0;
-        afr_local_t *local      = frame->local;
-        int         read_subvol = -1;
-
-        local->replies[idx].valid = 1;
-        local->replies[idx].op_ret = op_ret;
-        local->replies[idx].op_errno = op_errno;
-        if (dict)
-                local->replies[idx].xdata = dict_ref (dict);
-        call_count = afr_frame_return (frame);
-        if (call_count == 0) {
-                local->inode = inode_ref (local->loc.inode);
-                read_subvol = afr_handle_quota_size (frame, this);
-                if (read_subvol != -1) {
-                        op_ret = local->replies[read_subvol].op_ret;
-                        op_errno = local->replies[read_subvol].op_errno;
-                        dict = local->replies[read_subvol].xdata;
-                }
-                AFR_STACK_UNWIND (getxattr, frame, op_ret, op_errno, dict,
-                                  xdata);
+    int idx = (long)cookie;
+    int call_count = 0;
+    afr_local_t *local = frame->local;
+    int read_subvol = -1;
+
+    local->replies[idx].valid = 1;
+    local->replies[idx].op_ret = op_ret;
+    local->replies[idx].op_errno = op_errno;
+    if (dict)
+        local->replies[idx].xdata = dict_ref(dict);
+    call_count = afr_frame_return(frame);
+    if (call_count == 0) {
+        local->inode = inode_ref(local->loc.inode);
+        read_subvol = afr_handle_quota_size(frame, this);
+        if (read_subvol != -1) {
+            op_ret = local->replies[read_subvol].op_ret;
+            op_errno = local->replies[read_subvol].op_errno;
+            dict = local->replies[read_subvol].xdata;
         }
+        AFR_STACK_UNWIND(getxattr, frame, op_ret, op_errno, dict, xdata);
+    }
 
-        return 0;
+    return 0;
 }
 
 int32_t
-afr_getxattr_lockinfo_cbk (call_frame_t *frame, void *cookie,
-                           xlator_t *this, int32_t op_ret, int32_t op_errno,
-                           dict_t *dict, dict_t *xdata)
+afr_getxattr_lockinfo_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                          int32_t op_ret, int32_t op_errno, dict_t *dict,
+                          dict_t *xdata)
 {
-        int          call_cnt     = 0, len = 0;
-        char        *lockinfo_buf = NULL;
-        dict_t      *lockinfo     = NULL, *newdict = NULL;
-        afr_local_t *local        = NULL;
-
-        LOCK (&frame->lock);
-        {
-                local = frame->local;
+    int call_cnt = 0, len = 0;
+    char *lockinfo_buf = NULL;
+    dict_t *lockinfo = NULL, *newdict = NULL;
+    afr_local_t *local = NULL;
 
-                call_cnt = --local->call_count;
+    LOCK(&frame->lock);
+    {
+        local = frame->local;
 
-                if ((op_ret < 0) || (!dict && !xdata)) {
-                        goto unlock;
-                }
+        call_cnt = --local->call_count;
 
-                if (xdata) {
-                        if (!local->xdata_rsp) {
-                                local->xdata_rsp = dict_new ();
-                                if (!local->xdata_rsp) {
-                                        local->op_ret = -1;
-                                        local->op_errno = ENOMEM;
-                                        goto unlock;
-                                }
-                        }
-                }
+        if ((op_ret < 0) || (!dict && !xdata)) {
+            goto unlock;
+        }
 
-                if (!dict) {
-                        goto unlock;
+        if (xdata) {
+            if (!local->xdata_rsp) {
+                local->xdata_rsp = dict_new();
+                if (!local->xdata_rsp) {
+                    local->op_ret = -1;
+                    local->op_errno = ENOMEM;
+                    goto unlock;
                 }
+            }
+        }
 
-                op_ret = dict_get_ptr_and_len (dict, GF_XATTR_LOCKINFO_KEY,
-                                               (void **)&lockinfo_buf, &len);
+        if (!dict) {
+            goto unlock;
+        }
 
-                if (!lockinfo_buf) {
-                        goto unlock;
-                }
+        op_ret = dict_get_ptr_and_len(dict, GF_XATTR_LOCKINFO_KEY,
+                                      (void **)&lockinfo_buf, &len);
 
-                if (!local->dict) {
-                        local->dict = dict_new ();
-                        if (!local->dict) {
-                                local->op_ret = -1;
-                                local->op_errno = ENOMEM;
-                                goto unlock;
-                        }
-                }
-        }
-unlock:
-        UNLOCK (&frame->lock);
-
-        if (lockinfo_buf != NULL) {
-                lockinfo = dict_new ();
-                if (lockinfo == NULL) {
-                        local->op_ret = -1;
-                        local->op_errno = ENOMEM;
-                } else {
-                        op_ret = dict_unserialize (lockinfo_buf, len,
-                                                   &lockinfo);
-
-                        if (lockinfo && local->dict) {
-                                dict_copy (lockinfo, local->dict);
-                        }
-                }
+        if (!lockinfo_buf) {
+            goto unlock;
         }
 
-        if (xdata && local->xdata_rsp) {
-                dict_copy (xdata, local->xdata_rsp);
+        if (!local->dict) {
+            local->dict = dict_new();
+            if (!local->dict) {
+                local->op_ret = -1;
+                local->op_errno = ENOMEM;
+                goto unlock;
+            }
         }
+    }
+unlock:
+    UNLOCK(&frame->lock);
 
-        if (!call_cnt) {
-                newdict = dict_new ();
-                if (!newdict) {
-                        local->op_ret = -1;
-                        local->op_errno = ENOMEM;
-                        goto unwind;
-                }
-
-                len = dict_serialized_length (local->dict);
-                if (len <= 0) {
-                        goto unwind;
-                }
-
-                lockinfo_buf = GF_CALLOC (1, len, gf_common_mt_char);
-                if (!lockinfo_buf) {
-                        local->op_ret = -1;
-                        local->op_errno = ENOMEM;
-                        goto unwind;
-                }
+    if (lockinfo_buf != NULL) {
+        lockinfo = dict_new();
+        if (lockinfo == NULL) {
+            local->op_ret = -1;
+            local->op_errno = ENOMEM;
+        } else {
+            op_ret = dict_unserialize(lockinfo_buf, len, &lockinfo);
 
-                op_ret = dict_serialize (local->dict, lockinfo_buf);
-                if (op_ret < 0) {
-                        local->op_ret = -1;
-                        local->op_errno = -op_ret;
-                }
+            if (lockinfo && local->dict) {
+                dict_copy(lockinfo, local->dict);
+            }
+        }
+    }
+
+    if (xdata && local->xdata_rsp) {
+        dict_copy(xdata, local->xdata_rsp);
+    }
+
+    if (!call_cnt) {
+        newdict = dict_new();
+        if (!newdict) {
+            local->op_ret = -1;
+            local->op_errno = ENOMEM;
+            goto unwind;
+        }
 
-                op_ret = dict_set_dynptr (newdict, GF_XATTR_LOCKINFO_KEY,
-                                          (void *)lockinfo_buf, len);
-                if (op_ret < 0) {
-                        local->op_ret = -1;
-                        local->op_errno = -op_ret;
-                        goto unwind;
-                }
+        op_ret = dict_allocate_and_serialize(
+            local->dict, (char **)&lockinfo_buf, (unsigned int *)&len);
+        if (op_ret != 0) {
+            local->op_ret = -1;
+            goto unwind;
+        }
 
-        unwind:
-                AFR_STACK_UNWIND (getxattr, frame, op_ret,
-                                  op_errno, newdict,
-                                  local->xdata_rsp);
+        op_ret = dict_set_dynptr(newdict, GF_XATTR_LOCKINFO_KEY,
+                                 (void *)lockinfo_buf, len);
+        if (op_ret < 0) {
+            local->op_ret = -1;
+            local->op_errno = -op_ret;
+            goto unwind;
         }
 
-        dict_unref (lockinfo);
+    unwind:
+        AFR_STACK_UNWIND(getxattr, frame, op_ret, op_errno, newdict,
+                         local->xdata_rsp);
+    }
 
-        return 0;
+    dict_unref(lockinfo);
+
+    return 0;
 }
 
 int32_t
-afr_fgetxattr_lockinfo_cbk (call_frame_t *frame, void *cookie,
-                            xlator_t *this, int32_t op_ret, int32_t op_errno,
-                            dict_t *dict, dict_t *xdata)
+afr_fgetxattr_lockinfo_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                           int32_t op_ret, int32_t op_errno, dict_t *dict,
+                           dict_t *xdata)
 {
-        int          call_cnt     = 0, len = 0;
-        char        *lockinfo_buf = NULL;
-        dict_t      *lockinfo     = NULL, *newdict = NULL;
-        afr_local_t *local        = NULL;
-
-        LOCK (&frame->lock);
-        {
-                local = frame->local;
+    int call_cnt = 0, len = 0;
+    char *lockinfo_buf = NULL;
+    dict_t *lockinfo = NULL, *newdict = NULL;
+    afr_local_t *local = NULL;
 
-                call_cnt = --local->call_count;
+    LOCK(&frame->lock);
+    {
+        local = frame->local;
 
-                if ((op_ret < 0) || (!dict && !xdata)) {
-                        goto unlock;
-                }
+        call_cnt = --local->call_count;
 
-                if (xdata) {
-                        if (!local->xdata_rsp) {
-                                local->xdata_rsp = dict_new ();
-                                if (!local->xdata_rsp) {
-                                        local->op_ret = -1;
-                                        local->op_errno = ENOMEM;
-                                        goto unlock;
-                                }
-                        }
-                }
+        if ((op_ret < 0) || (!dict && !xdata)) {
+            goto unlock;
+        }
 
-                if (!dict) {
-                        goto unlock;
+        if (xdata) {
+            if (!local->xdata_rsp) {
+                local->xdata_rsp = dict_new();
+                if (!local->xdata_rsp) {
+                    local->op_ret = -1;
+                    local->op_errno = ENOMEM;
+                    goto unlock;
                 }
+            }
+        }
 
-                op_ret = dict_get_ptr_and_len (dict, GF_XATTR_LOCKINFO_KEY,
-                                               (void **)&lockinfo_buf, &len);
+        if (!dict) {
+            goto unlock;
+        }
 
-                if (!lockinfo_buf) {
-                        goto unlock;
-                }
+        op_ret = dict_get_ptr_and_len(dict, GF_XATTR_LOCKINFO_KEY,
+                                      (void **)&lockinfo_buf, &len);
 
-                if (!local->dict) {
-                        local->dict = dict_new ();
-                        if (!local->dict) {
-                                local->op_ret = -1;
-                                local->op_errno = ENOMEM;
-                                goto unlock;
-                        }
-                }
-        }
-unlock:
-        UNLOCK (&frame->lock);
-
-        if (lockinfo_buf != NULL) {
-                lockinfo = dict_new ();
-                if (lockinfo == NULL) {
-                        local->op_ret = -1;
-                        local->op_errno = ENOMEM;
-                } else {
-                        op_ret = dict_unserialize (lockinfo_buf, len,
-                                                   &lockinfo);
-
-                        if (lockinfo && local->dict) {
-                                dict_copy (lockinfo, local->dict);
-                        }
-                }
+        if (!lockinfo_buf) {
+            goto unlock;
         }
 
-        if (xdata && local->xdata_rsp) {
-                dict_copy (xdata, local->xdata_rsp);
+        if (!local->dict) {
+            local->dict = dict_new();
+            if (!local->dict) {
+                local->op_ret = -1;
+                local->op_errno = ENOMEM;
+                goto unlock;
+            }
         }
+    }
+unlock:
+    UNLOCK(&frame->lock);
 
-        if (!call_cnt) {
-                newdict = dict_new ();
-                if (!newdict) {
-                        local->op_ret = -1;
-                        local->op_errno = ENOMEM;
-                        goto unwind;
-                }
-
-                len = dict_serialized_length (local->dict);
-                if (len <= 0) {
-                        goto unwind;
-                }
-
-                lockinfo_buf = GF_CALLOC (1, len, gf_common_mt_char);
-                if (!lockinfo_buf) {
-                        local->op_ret = -1;
-                        local->op_errno = ENOMEM;
-                        goto unwind;
-                }
+    if (lockinfo_buf != NULL) {
+        lockinfo = dict_new();
+        if (lockinfo == NULL) {
+            local->op_ret = -1;
+            local->op_errno = ENOMEM;
+        } else {
+            op_ret = dict_unserialize(lockinfo_buf, len, &lockinfo);
 
-                op_ret = dict_serialize (local->dict, lockinfo_buf);
-                if (op_ret < 0) {
-                        local->op_ret = -1;
-                        local->op_errno = -op_ret;
-                }
+            if (lockinfo && local->dict) {
+                dict_copy(lockinfo, local->dict);
+            }
+        }
+    }
+
+    if (xdata && local->xdata_rsp) {
+        dict_copy(xdata, local->xdata_rsp);
+    }
+
+    if (!call_cnt) {
+        newdict = dict_new();
+        if (!newdict) {
+            local->op_ret = -1;
+            local->op_errno = ENOMEM;
+            goto unwind;
+        }
 
-                op_ret = dict_set_dynptr (newdict, GF_XATTR_LOCKINFO_KEY,
-                                          (void *)lockinfo_buf, len);
-                if (op_ret < 0) {
-                        local->op_ret = -1;
-                        local->op_errno = -op_ret;
-                        goto unwind;
-                }
+        op_ret = dict_allocate_and_serialize(
+            local->dict, (char **)&lockinfo_buf, (unsigned int *)&len);
+        if (op_ret != 0) {
+            local->op_ret = -1;
+            goto unwind;
+        }
 
-        unwind:
-                AFR_STACK_UNWIND (fgetxattr, frame, op_ret,
-                                  op_errno, newdict,
-                                  local->xdata_rsp);
+        op_ret = dict_set_dynptr(newdict, GF_XATTR_LOCKINFO_KEY,
+                                 (void *)lockinfo_buf, len);
+        if (op_ret < 0) {
+            local->op_ret = -1;
+            local->op_errno = -op_ret;
+            goto unwind;
         }
 
-        dict_unref (lockinfo);
+    unwind:
+        AFR_STACK_UNWIND(fgetxattr, frame, op_ret, op_errno, newdict,
+                         local->xdata_rsp);
+    }
 
-        return 0;
+    dict_unref(lockinfo);
+
+    return 0;
 }
 
 int32_t
-afr_fgetxattr_pathinfo_cbk (call_frame_t *frame, void *cookie,
-                            xlator_t *this, int32_t op_ret, int32_t op_errno,
-                            dict_t *dict, dict_t *xdata)
+afr_fgetxattr_pathinfo_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                           int32_t op_ret, int32_t op_errno, dict_t *dict,
+                           dict_t *xdata)
 {
-        afr_local_t *local          = NULL;
-        int32_t      callcnt        = 0;
-        int          ret            = 0;
-        char        *xattr          = NULL;
-        char        *xattr_serz     = NULL;
-        char        xattr_cky[1024] = {0,};
-        dict_t      *nxattr         = NULL;
-        long         cky            = 0;
-        int32_t      padding        = 0;
-        int32_t      tlen           = 0;
-
-        if (!frame || !frame->local || !this) {
-                gf_log ("", GF_LOG_ERROR, "possible NULL deref");
-                goto out;
+    afr_local_t *local = NULL;
+    int32_t callcnt = 0;
+    int ret = 0;
+    char *xattr = NULL;
+    char *xattr_serz = NULL;
+    int keylen = 0;
+    char xattr_cky[1024] = {
+        0,
+    };
+    int xattr_cky_len = 0;
+    dict_t *nxattr = NULL;
+    long cky = 0;
+    int32_t padding = 0;
+    int32_t tlen = 0;
+
+    if (!frame || !frame->local || !this) {
+        gf_msg("", GF_LOG_ERROR, 0, AFR_MSG_INVALID_ARG, "possible NULL deref");
+        goto out;
+    }
+
+    local = frame->local;
+    cky = (long)cookie;
+    keylen = strlen(local->cont.getxattr.name);
+    xattr_cky_len = snprintf(xattr_cky, sizeof(xattr_cky), "%s-%ld",
+                             local->cont.getxattr.name, cky);
+    LOCK(&frame->lock);
+    {
+        callcnt = --local->call_count;
+
+        if (op_ret < 0) {
+            local->op_errno = op_errno;
+        } else {
+            local->op_ret = op_ret;
+            if (!local->xdata_rsp && xdata)
+                local->xdata_rsp = dict_ref(xdata);
         }
 
-        local = frame->local;
-        cky = (long) cookie;
-
-        LOCK (&frame->lock);
-        {
-                callcnt = --local->call_count;
-
-                if (op_ret < 0) {
-                        local->op_errno = op_errno;
-                } else {
-                        local->op_ret = op_ret;
-                        if (!local->xdata_rsp && xdata)
-                                local->xdata_rsp = dict_ref (xdata);
-                }
+        if (!dict || (op_ret < 0))
+            goto unlock;
 
-                if (!dict || (op_ret < 0))
-                        goto unlock;
-
-                if (!local->dict)
-                        local->dict = dict_new ();
-
-                if (local->dict) {
-                        ret = dict_get_str (dict,
-                                            local->cont.getxattr.name,
-                                            &xattr);
-                        if (ret)
-                                goto unlock;
-
-                        xattr = gf_strdup (xattr);
-
-                        (void)snprintf (xattr_cky, 1024, "%s-%ld",
-                                        local->cont.getxattr.name, cky);
-                        ret = dict_set_dynstr (local->dict,
-                                               xattr_cky, xattr);
-                        if (ret) {
-                                gf_log (this->name, GF_LOG_ERROR,
-                                        "Cannot set xattr cookie key");
-                                goto unlock;
-                        }
-
-                        local->cont.getxattr.xattr_len
-                                += strlen (xattr) + 1;
-                }
+        if (!local->dict) {
+            local->dict = dict_new();
+            if (!local->dict)
+                goto unlock;
+        }
+        ret = dict_get_strn(dict, local->cont.getxattr.name, keylen, &xattr);
+        if (ret)
+            goto unlock;
+
+        xattr = gf_strdup(xattr);
+
+        ret = dict_set_dynstrn(local->dict, xattr_cky, xattr_cky_len, xattr);
+        if (ret) {
+            UNLOCK(&frame->lock);
+            gf_msg(this->name, GF_LOG_ERROR, -ret, AFR_MSG_DICT_SET_FAILED,
+                   "Cannot set xattr cookie key");
+            goto post_unlock;
         }
-unlock:
-        UNLOCK (&frame->lock);
-
-        if (!callcnt) {
-                if (!local->cont.getxattr.xattr_len)
-                        goto unwind;
-
-                nxattr = dict_new ();
-                if (!nxattr)
-                        goto unwind;
-
-                /* extra bytes for decorations (brackets and <>'s) */
-                padding += strlen (this->name)
-                        + strlen (AFR_PATHINFO_HEADER) + 4;
-                local->cont.getxattr.xattr_len += (padding + 2);
-
-                xattr_serz = GF_CALLOC (local->cont.getxattr.xattr_len,
-                                        sizeof (char), gf_common_mt_char);
-
-                if (!xattr_serz)
-                        goto unwind;
-
-                /* the xlator info */
-                (void) sprintf (xattr_serz, "(<"AFR_PATHINFO_HEADER"%s> ",
-                                this->name);
-
-                /* actual series of pathinfo */
-                ret = dict_serialize_value_with_delim (local->dict,
-                                                       xattr_serz
-                                                       + strlen (xattr_serz),
-                                                       &tlen, ' ');
-                if (ret) {
-                        gf_log (this->name, GF_LOG_ERROR, "Error serializing"
-                                " dictionary");
-                        goto unwind;
-                }
 
-                /* closing part */
-                *(xattr_serz + padding + tlen) = ')';
-                *(xattr_serz + padding + tlen + 1) = '\0';
+        local->cont.getxattr.xattr_len += strlen(xattr) + 1;
+    }
+unlock:
+    UNLOCK(&frame->lock);
+post_unlock:
+    if (!callcnt) {
+        if (!local->cont.getxattr.xattr_len)
+            goto unwind;
+
+        nxattr = dict_new();
+        if (!nxattr)
+            goto unwind;
+
+        /* extra bytes for decorations (brackets and <>'s) */
+        padding += strlen(this->name) + SLEN(AFR_PATHINFO_HEADER) + 4;
+        local->cont.getxattr.xattr_len += (padding + 2);
+
+        xattr_serz = GF_MALLOC(local->cont.getxattr.xattr_len,
+                               gf_common_mt_char);
+
+        if (!xattr_serz)
+            goto unwind;
+
+        /* the xlator info */
+        int xattr_serz_len = sprintf(
+            xattr_serz, "(<" AFR_PATHINFO_HEADER "%s> ", this->name);
+
+        /* actual series of pathinfo */
+        ret = dict_serialize_value_with_delim(
+            local->dict, xattr_serz + xattr_serz_len, &tlen, ' ');
+        if (ret) {
+            GF_FREE(xattr_serz);
+            goto unwind;
+        }
 
-                ret = dict_set_dynstr (nxattr, local->cont.getxattr.name,
-                                       xattr_serz);
-                if (ret)
-                        gf_log (this->name, GF_LOG_ERROR, "Cannot set pathinfo"
-                                " key in dict");
+        /* closing part */
+        *(xattr_serz + padding + tlen) = ')';
+        *(xattr_serz + padding + tlen + 1) = '\0';
+
+        ret = dict_set_dynstrn(nxattr, local->cont.getxattr.name, keylen,
+                               xattr_serz);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, -ret, AFR_MSG_DICT_SET_FAILED,
+                   "Cannot set pathinfo key in dict");
+            if (ret == -EINVAL)
+                GF_FREE(xattr_serz);
+        }
 
-        unwind:
-                AFR_STACK_UNWIND (fgetxattr, frame, local->op_ret,
-                                  local->op_errno, nxattr, local->xdata_rsp);
+    unwind:
+        AFR_STACK_UNWIND(fgetxattr, frame, local->op_ret, local->op_errno,
+                         nxattr, local->xdata_rsp);
 
-                if (nxattr)
-                        dict_unref (nxattr);
-        }
+        if (nxattr)
+            dict_unref(nxattr);
+    }
 
 out:
-        return ret;
+    return ret;
 }
 
 int32_t
-afr_getxattr_pathinfo_cbk (call_frame_t *frame, void *cookie,
-                           xlator_t *this, int32_t op_ret, int32_t op_errno,
-                           dict_t *dict, dict_t *xdata)
+afr_getxattr_pathinfo_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                          int32_t op_ret, int32_t op_errno, dict_t *dict,
+                          dict_t *xdata)
 {
-        afr_local_t *local          = NULL;
-        int32_t      callcnt        = 0;
-        int          ret            = 0;
-        char        *xattr          = NULL;
-        char        *xattr_serz     = NULL;
-        char        xattr_cky[1024] = {0,};
-        dict_t      *nxattr         = NULL;
-        long         cky            = 0;
-        int32_t      padding        = 0;
-        int32_t      tlen           = 0;
-
-        if (!frame || !frame->local || !this) {
-                gf_log ("", GF_LOG_ERROR, "possible NULL deref");
-                goto out;
+    afr_local_t *local = NULL;
+    int32_t callcnt = 0;
+    int ret = 0;
+    char *xattr = NULL;
+    char *xattr_serz = NULL;
+    char xattr_cky[1024] = {
+        0,
+    };
+    int keylen = 0;
+    int xattr_cky_len = 0;
+    dict_t *nxattr = NULL;
+    long cky = 0;
+    int32_t padding = 0;
+    int32_t tlen = 0;
+
+    if (!frame || !frame->local || !this) {
+        gf_msg("", GF_LOG_ERROR, 0, AFR_MSG_INVALID_ARG, "possible NULL deref");
+        goto out;
+    }
+
+    local = frame->local;
+    cky = (long)cookie;
+    keylen = strlen(local->cont.getxattr.name);
+    xattr_cky_len = snprintf(xattr_cky, sizeof(xattr_cky), "%s-%ld",
+                             local->cont.getxattr.name, cky);
+    LOCK(&frame->lock);
+    {
+        callcnt = --local->call_count;
+
+        if (op_ret < 0) {
+            local->op_errno = op_errno;
+        } else {
+            local->op_ret = op_ret;
+            if (!local->xdata_rsp && xdata)
+                local->xdata_rsp = dict_ref(xdata);
         }
 
-        local = frame->local;
-        cky = (long) cookie;
-
-        LOCK (&frame->lock);
-                {
-                        callcnt = --local->call_count;
-
-                        if (op_ret < 0) {
-                                local->op_errno = op_errno;
-                        } else {
-                                local->op_ret = op_ret;
-                                if (!local->xdata_rsp && xdata)
-                                        local->xdata_rsp = dict_ref (xdata);
-                        }
-
-                        if (!dict || (op_ret < 0))
-                                goto unlock;
-
-                        if (!local->dict)
-                                local->dict = dict_new ();
-
-                        if (local->dict) {
-                                ret = dict_get_str (dict,
-                                                    local->cont.getxattr.name,
-                                                    &xattr);
-                                if (ret)
-                                        goto unlock;
-
-                                xattr = gf_strdup (xattr);
-
-                                (void)snprintf (xattr_cky, 1024, "%s-%ld",
-                                                local->cont.getxattr.name, cky);
-                                ret = dict_set_dynstr (local->dict,
-                                                       xattr_cky, xattr);
-                                if (ret) {
-                                        gf_log (this->name, GF_LOG_ERROR,
-                                                "Cannot set xattr cookie key");
-                                        goto unlock;
-                                }
-
-                                local->cont.getxattr.xattr_len += strlen (xattr) + 1;
-                        }
-                }
-unlock:
-        UNLOCK (&frame->lock);
-
-        if (!callcnt) {
-                if (!local->cont.getxattr.xattr_len)
-                        goto unwind;
-
-                nxattr = dict_new ();
-                if (!nxattr)
-                        goto unwind;
-
-                /* extra bytes for decorations (brackets and <>'s) */
-                padding += strlen (this->name) + strlen (AFR_PATHINFO_HEADER) + 4;
-                local->cont.getxattr.xattr_len += (padding + 2);
-
-                xattr_serz = GF_CALLOC (local->cont.getxattr.xattr_len,
-                                        sizeof (char), gf_common_mt_char);
-
-                if (!xattr_serz)
-                        goto unwind;
-
-                /* the xlator info */
-                (void) sprintf (xattr_serz, "(<"AFR_PATHINFO_HEADER"%s> ",
-                                this->name);
-
-                /* actual series of pathinfo */
-                ret = dict_serialize_value_with_delim (local->dict,
-                                                       xattr_serz + strlen (xattr_serz),
-                                                       &tlen, ' ');
-                if (ret) {
-                        gf_log (this->name, GF_LOG_ERROR, "Error serializing"
-                                " dictionary");
-                        goto unwind;
-                }
-
-                /* closing part */
-                *(xattr_serz + padding + tlen) = ')';
-                *(xattr_serz + padding + tlen + 1) = '\0';
+        if (!dict || (op_ret < 0))
+            goto unlock;
 
-                ret = dict_set_dynstr (nxattr, local->cont.getxattr.name,
-                                       xattr_serz);
-                if (ret)
-                        gf_log (this->name, GF_LOG_ERROR, "Cannot set pathinfo"
-                                " key in dict");
+        if (!local->dict) {
+            local->dict = dict_new();
+            if (!local->dict)
+                goto unlock;
+        }
+        ret = dict_get_strn(dict, local->cont.getxattr.name, keylen, &xattr);
+        if (ret)
+            goto unlock;
+
+        xattr = gf_strdup(xattr);
+
+        ret = dict_set_dynstrn(local->dict, xattr_cky, xattr_cky_len, xattr);
+        if (ret) {
+            UNLOCK(&frame->lock);
+            gf_msg(this->name, GF_LOG_ERROR, -ret, AFR_MSG_DICT_SET_FAILED,
+                   "Cannot set xattr cookie key");
+            goto post_unlock;
+        }
 
-        unwind:
-                AFR_STACK_UNWIND (getxattr, frame, local->op_ret,
-                                  local->op_errno, nxattr, local->xdata_rsp);
+        local->cont.getxattr.xattr_len += strlen(xattr) + 1;
+    }
+unlock:
+    UNLOCK(&frame->lock);
+post_unlock:
+    if (!callcnt) {
+        if (!local->cont.getxattr.xattr_len)
+            goto unwind;
+
+        nxattr = dict_new();
+        if (!nxattr)
+            goto unwind;
+
+        /* extra bytes for decorations (brackets and <>'s) */
+        padding += strlen(this->name) + SLEN(AFR_PATHINFO_HEADER) + 4;
+        local->cont.getxattr.xattr_len += (padding + 2);
+
+        xattr_serz = GF_MALLOC(local->cont.getxattr.xattr_len,
+                               gf_common_mt_char);
+
+        if (!xattr_serz)
+            goto unwind;
+
+        /* the xlator info */
+        int xattr_serz_len = sprintf(
+            xattr_serz, "(<" AFR_PATHINFO_HEADER "%s> ", this->name);
+
+        /* actual series of pathinfo */
+        ret = dict_serialize_value_with_delim(
+            local->dict, xattr_serz + xattr_serz_len, &tlen, ' ');
+        if (ret) {
+            GF_FREE(xattr_serz);
+            goto unwind;
+        }
 
-                if (nxattr)
-                        dict_unref (nxattr);
+        /* closing part */
+        *(xattr_serz + padding + tlen) = ')';
+        *(xattr_serz + padding + tlen + 1) = '\0';
+
+        ret = dict_set_dynstrn(nxattr, local->cont.getxattr.name, keylen,
+                               xattr_serz);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, -ret, AFR_MSG_DICT_SET_FAILED,
+                   "Cannot set pathinfo key in dict");
+            if (ret == -EINVAL)
+                GF_FREE(xattr_serz);
         }
 
+    unwind:
+        AFR_STACK_UNWIND(getxattr, frame, local->op_ret, local->op_errno,
+                         nxattr, local->xdata_rsp);
+
+        if (nxattr)
+            dict_unref(nxattr);
+    }
+
 out:
-        return ret;
+    return ret;
 }
 
 static int
-afr_aggregate_stime_xattr (dict_t *this, char *key, data_t *value, void *data)
+afr_aggregate_stime_xattr(dict_t *this, char *key, data_t *value, void *data)
 {
-        int ret = 0;
+    int ret = 0;
 
-        if (fnmatch (GF_XATTR_STIME_PATTERN, key, FNM_NOESCAPE) == 0)
-                ret = gf_get_max_stime (THIS, data, key, value);
+    if (fnmatch(GF_XATTR_STIME_PATTERN, key, FNM_NOESCAPE) == 0)
+        ret = gf_get_max_stime(THIS, data, key, value);
 
-        return ret;
+    return ret;
 }
 
 int32_t
-afr_common_getxattr_stime_cbk (call_frame_t *frame, void *cookie,
-                               xlator_t *this, int32_t op_ret, int32_t op_errno,
-                               dict_t *dict, dict_t *xdata)
+afr_common_getxattr_stime_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                              int32_t op_ret, int32_t op_errno, dict_t *dict,
+                              dict_t *xdata)
 {
-        afr_local_t *local          = NULL;
-        int32_t      callcnt        = 0;
-
-        if (!frame || !frame->local || !this) {
-                gf_log ("", GF_LOG_ERROR, "possible NULL deref");
-                goto out;
-        }
+    afr_local_t *local = NULL;
+    int32_t callcnt = 0;
 
-        local = frame->local;
+    if (!frame || !frame->local || !this) {
+        gf_msg("", GF_LOG_ERROR, 0, AFR_MSG_INVALID_ARG, "possible NULL deref");
+        goto out;
+    }
 
-        LOCK (&frame->lock);
-        {
-                callcnt = --local->call_count;
+    local = frame->local;
 
-                if (!dict || (op_ret < 0)) {
-                        local->op_errno = op_errno;
-                        goto cleanup;
-                }
+    LOCK(&frame->lock);
+    {
+        callcnt = --local->call_count;
 
-                if (!local->dict)
-                        local->dict = dict_copy_with_ref (dict, NULL);
-                else
-                        dict_foreach (dict, afr_aggregate_stime_xattr,
-                                      local->dict);
-                local->op_ret = 0;
+        if (!dict || (op_ret < 0)) {
+            local->op_errno = op_errno;
+            goto cleanup;
         }
 
+        if (!local->dict)
+            local->dict = dict_copy_with_ref(dict, NULL);
+        else
+            dict_foreach(dict, afr_aggregate_stime_xattr, local->dict);
+        local->op_ret = 0;
+    }
+
 cleanup:
-        UNLOCK (&frame->lock);
+    UNLOCK(&frame->lock);
 
-        if (!callcnt) {
-                AFR_STACK_UNWIND (getxattr, frame, local->op_ret,
-                                  local->op_errno, local->dict, xdata);
-        }
+    if (!callcnt) {
+        AFR_STACK_UNWIND(getxattr, frame, local->op_ret, local->op_errno,
+                         local->dict, xdata);
+    }
 
 out:
-        return 0;
+    return 0;
 }
 
-
 static gf_boolean_t
-afr_is_special_xattr (const char *name, fop_getxattr_cbk_t *cbk,
-                      gf_boolean_t is_fgetxattr)
+afr_is_special_xattr(const char *name, fop_getxattr_cbk_t *cbk,
+                     gf_boolean_t is_fgetxattr)
 {
-        gf_boolean_t    is_spl = _gf_true;
-
-        GF_ASSERT (cbk);
-        if (!cbk || !name) {
-                is_spl = _gf_false;
-                goto out;
+    gf_boolean_t is_spl = _gf_true;
+
+    GF_ASSERT(cbk);
+    if (!cbk || !name) {
+        is_spl = _gf_false;
+        goto out;
+    }
+
+    if (!strcmp(name, GF_XATTR_PATHINFO_KEY) ||
+        !strcmp(name, GF_XATTR_USER_PATHINFO_KEY)) {
+        if (is_fgetxattr) {
+            *cbk = afr_fgetxattr_pathinfo_cbk;
+        } else {
+            *cbk = afr_getxattr_pathinfo_cbk;
         }
-
-        if (!strcmp (name, GF_XATTR_PATHINFO_KEY) ||
-	    !strcmp (name, GF_XATTR_USER_PATHINFO_KEY)) {
-                if (is_fgetxattr) {
-                        *cbk = afr_fgetxattr_pathinfo_cbk;
-                } else {
-                        *cbk = afr_getxattr_pathinfo_cbk;
-                }
-        } else if (!strncmp (name, GF_XATTR_CLRLK_CMD,
-                             strlen (GF_XATTR_CLRLK_CMD))) {
-                if (is_fgetxattr) {
-                        *cbk = afr_fgetxattr_clrlk_cbk;
-                } else {
-                        *cbk = afr_getxattr_clrlk_cbk;
-                }
-        } else if (!strncmp (name, GF_XATTR_LOCKINFO_KEY,
-                             strlen (GF_XATTR_LOCKINFO_KEY))) {
-                if (is_fgetxattr) {
-                        *cbk = afr_fgetxattr_lockinfo_cbk;
-                } else {
-                        *cbk = afr_getxattr_lockinfo_cbk;
-                }
-        } else if (fnmatch (GF_XATTR_STIME_PATTERN, name, FNM_NOESCAPE) == 0) {
-                *cbk = afr_common_getxattr_stime_cbk;
-        } else if (strcmp (name, QUOTA_SIZE_KEY) == 0) {
-                *cbk = afr_getxattr_quota_size_cbk;
+    } else if (!strncmp(name, GF_XATTR_CLRLK_CMD, SLEN(GF_XATTR_CLRLK_CMD))) {
+        if (is_fgetxattr) {
+            *cbk = afr_fgetxattr_clrlk_cbk;
         } else {
-                is_spl = _gf_false;
+            *cbk = afr_getxattr_clrlk_cbk;
         }
+    } else if (!strncmp(name, GF_XATTR_LOCKINFO_KEY,
+                        SLEN(GF_XATTR_LOCKINFO_KEY))) {
+        if (is_fgetxattr) {
+            *cbk = afr_fgetxattr_lockinfo_cbk;
+        } else {
+            *cbk = afr_getxattr_lockinfo_cbk;
+        }
+    } else if (fnmatch(GF_XATTR_STIME_PATTERN, name, FNM_NOESCAPE) == 0) {
+        *cbk = afr_common_getxattr_stime_cbk;
+    } else if (strcmp(name, QUOTA_SIZE_KEY) == 0) {
+        *cbk = afr_getxattr_quota_size_cbk;
+    } else if (!strcmp(name, GF_XATTR_LIST_NODE_UUIDS_KEY)) {
+        *cbk = afr_getxattr_list_node_uuids_cbk;
+    } else {
+        is_spl = _gf_false;
+    }
 
 out:
-        return is_spl;
+    return is_spl;
 }
 
 static void
-afr_getxattr_all_subvols (xlator_t *this, call_frame_t *frame,
-			  const char *name, loc_t *loc,
-			  fop_getxattr_cbk_t cbk)
+afr_getxattr_all_subvols(xlator_t *this, call_frame_t *frame, const char *name,
+                         loc_t *loc, fop_getxattr_cbk_t cbk)
 {
-        afr_private_t   *priv           = NULL;
-        afr_local_t     *local          = NULL;
-        int             i               = 0;
-        int             call_count      = 0;
-
-        priv     = this->private;
-
-        local = frame->local;
-        //local->call_count set in afr_local_init
-        call_count = local->call_count;
-
-        //If up-children count is 0, afr_local_init would have failed already
-        //and the call would have unwound so not handling it here.
-
-        for (i = 0; i < priv->child_count; i++) {
-                if (local->child_up[i]) {
-                        STACK_WIND_COOKIE (frame, cbk,
-                                           (void *) (long) i, priv->children[i],
-                                           priv->children[i]->fops->getxattr,
-                                           loc, name, NULL);
-                        if (!--call_count)
-                                break;
-                }
+    afr_private_t *priv = NULL;
+    afr_local_t *local = NULL;
+    int i = 0;
+    int call_count = 0;
+
+    priv = this->private;
+
+    local = frame->local;
+    // local->call_count set in afr_local_init
+    call_count = local->call_count;
+
+    if (!strcmp(name, GF_XATTR_LIST_NODE_UUIDS_KEY)) {
+        GF_FREE(local->cont.getxattr.name);
+        local->cont.getxattr.name = gf_strdup(GF_XATTR_NODE_UUID_KEY);
+    }
+
+    // If up-children count is 0, afr_local_init would have failed already
+    // and the call would have unwound so not handling it here.
+    for (i = 0; i < priv->child_count; i++) {
+        if (local->child_up[i]) {
+            STACK_WIND_COOKIE(frame, cbk, (void *)(long)i, priv->children[i],
+                              priv->children[i]->fops->getxattr, loc,
+                              local->cont.getxattr.name, NULL);
+            if (!--call_count)
+                break;
         }
-        return;
+    }
+    return;
 }
 
 int
-afr_marker_populate_args (call_frame_t *frame, int type, int *gauge,
-                          xlator_t **subvols)
+afr_marker_populate_args(call_frame_t *frame, int type, int *gauge,
+                         xlator_t **subvols)
 {
-        xlator_t *this = frame->this;
-        afr_private_t *priv = this->private;
+    xlator_t *this = frame->this;
+    afr_private_t *priv = this->private;
 
-        memcpy (subvols, priv->children, sizeof (*subvols) * priv->child_count);
+    memcpy(subvols, priv->children, sizeof(*subvols) * priv->child_count);
 
-        if (type == MARKER_XTIME_TYPE) {
-                /*Don't error out on ENOENT/ENOTCONN */
-                gauge[MCNT_NOTFOUND] = 0;
-                gauge[MCNT_ENOTCONN] = 0;
-        }
-        return priv->child_count;
+    if (type == MARKER_XTIME_TYPE) {
+        /*Don't error out on ENOENT/ENOTCONN */
+        gauge[MCNT_NOTFOUND] = 0;
+        gauge[MCNT_ENOTCONN] = 0;
+    }
+    return priv->child_count;
 }
 
-int32_t
-afr_getxattr (call_frame_t *frame, xlator_t *this,
-              loc_t *loc, const char *name, dict_t *xdata)
+static int
+afr_handle_heal_xattrs(call_frame_t *frame, xlator_t *this, loc_t *loc,
+                       const char *heal_op)
 {
-        afr_private_t           *priv         = NULL;
-        xlator_t                **children    = NULL;
-        afr_local_t             *local        = NULL;
-        int                     i             = 0;
-        int32_t                 op_errno      = 0;
-        int                     ret           = -1;
-        fop_getxattr_cbk_t      cbk           = NULL;
+    int ret = -1;
+    afr_spb_status_t *data = NULL;
 
+    if (!strcmp(heal_op, GF_HEAL_INFO)) {
+        afr_get_heal_info(frame, this, loc);
+        ret = 0;
+        goto out;
+    }
 
-	local = AFR_FRAME_INIT (frame, op_errno);
-	if (!local)
-		goto out;
+    if (!strcmp(heal_op, GF_AFR_HEAL_SBRAIN)) {
+        afr_heal_splitbrain_file(frame, this, loc);
+        ret = 0;
+        goto out;
+    }
+
+    if (!strcmp(heal_op, GF_AFR_SBRAIN_STATUS)) {
+        data = GF_CALLOC(1, sizeof(*data), gf_afr_mt_spb_status_t);
+        if (!data) {
+            ret = 1;
+            goto out;
+        }
+        data->frame = frame;
+        data->loc = loc;
+        ret = synctask_new(this->ctx->env, afr_get_split_brain_status,
+                           afr_get_split_brain_status_cbk, NULL, data);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, AFR_MSG_SPLIT_BRAIN_STATUS,
+                   "Failed to create"
+                   " synctask. Unable to fetch split-brain status"
+                   " for %s.",
+                   loc->name);
+            ret = 1;
+            goto out;
+        }
+        goto out;
+    }
 
-        priv     = this->private;
+out:
+    if (ret == 1) {
+        AFR_STACK_UNWIND(getxattr, frame, -1, ENOMEM, NULL, NULL);
+        if (data)
+            GF_FREE(data);
+        ret = 0;
+    }
+    return ret;
+}
 
-        children = priv->children;
+int32_t
+afr_getxattr(call_frame_t *frame, xlator_t *this, loc_t *loc, const char *name,
+             dict_t *xdata)
+{
+    afr_private_t *priv = NULL;
+    afr_local_t *local = NULL;
+    xlator_t **children = NULL;
+    int i = 0;
+    int32_t op_errno = 0;
+    int ret = -1;
+    fop_getxattr_cbk_t cbk = NULL;
 
-        loc_copy (&local->loc, loc);
+    local = AFR_FRAME_INIT(frame, op_errno);
+    if (!local)
+        goto out;
 
-	local->op = GF_FOP_GETXATTR;
+    priv = this->private;
 
-	if (xdata)
-		local->xdata_req = dict_ref (xdata);
+    children = priv->children;
 
-        if (!name)
-                goto no_name;
+    loc_copy(&local->loc, loc);
 
-        local->cont.getxattr.name = gf_strdup (name);
+    local->op = GF_FOP_GETXATTR;
 
-	if (!local->cont.getxattr.name) {
-		op_errno = ENOMEM;
-		goto out;
-	}
+    if (xdata)
+        local->xdata_req = dict_ref(xdata);
 
-        if (!strncmp (name, AFR_XATTR_PREFIX,
-                      strlen (AFR_XATTR_PREFIX))) {
-                gf_log (this->name, GF_LOG_INFO,
-                        "%s: no data present for key %s",
-                        loc->path, name);
-                op_errno = ENODATA;
-                goto out;
-        }
+    if (!name)
+        goto no_name;
 
-        if (cluster_handle_marker_getxattr (frame, loc, name, priv->vol_uuid,
-                                            afr_getxattr_unwind,
-                                            afr_marker_populate_args) == 0)
-                return 0;
+    local->cont.getxattr.name = gf_strdup(name);
 
-        if (!strcmp (name, GF_HEAL_INFO)) {
-                afr_get_heal_info (frame, this, loc, xdata);
-                return 0;
-        }
+    if (!local->cont.getxattr.name) {
+        op_errno = ENOMEM;
+        goto out;
+    }
 
-        if (!strcmp (name, GF_AFR_HEAL_SBRAIN)) {
-                afr_heal_splitbrain_file (frame, this, loc);
-                return 0;
-        }
+    if (!strncmp(name, AFR_XATTR_PREFIX, SLEN(AFR_XATTR_PREFIX))) {
+        op_errno = ENODATA;
+        goto out;
+    }
 
-        if (!strcmp (name, GF_AFR_SBRAIN_STATUS)) {
-                afr_get_split_brain_status (frame, this, loc);
-                return 0;
-        }
-        /*
-         * Special xattrs which need responses from all subvols
-         */
-        if (afr_is_special_xattr (name, &cbk, 0)) {
-                afr_getxattr_all_subvols (this, frame, name, loc, cbk);
-                return 0;
-        }
+    if (cluster_handle_marker_getxattr(frame, loc, name, priv->vol_uuid,
+                                       afr_getxattr_unwind,
+                                       afr_marker_populate_args) == 0)
+        return 0;
 
-        if (XATTR_IS_NODE_UUID (name)) {
-                i = 0;
-                STACK_WIND_COOKIE (frame, afr_getxattr_node_uuid_cbk,
-                                   (void *) (long) i,
-                                   children[i],
-                                   children[i]->fops->getxattr,
-                                   loc, name, xdata);
-                return 0;
-        }
+    ret = afr_handle_heal_xattrs(frame, this, &local->loc, name);
+    if (ret == 0)
+        return 0;
+
+    /*
+     * Heal daemons don't have IO threads ... and as a result they
+     * send this getxattr down and eventually crash :(
+     */
+    op_errno = -1;
+    GF_CHECK_XATTR_KEY_AND_GOTO(name, IO_THREADS_QUEUE_SIZE_KEY, op_errno, out);
+
+    /*
+     * Special xattrs which need responses from all subvols
+     */
+    if (afr_is_special_xattr(name, &cbk, 0)) {
+        afr_getxattr_all_subvols(this, frame, name, loc, cbk);
+        return 0;
+    }
+
+    if (XATTR_IS_NODE_UUID(name)) {
+        i = 0;
+        STACK_WIND_COOKIE(frame, afr_getxattr_node_uuid_cbk, (void *)(long)i,
+                          children[i], children[i]->fops->getxattr, loc, name,
+                          xdata);
+        return 0;
+    }
 
 no_name:
 
-	afr_read_txn (frame, this, local->loc.inode, afr_getxattr_wind,
-		      AFR_METADATA_TRANSACTION);
+    afr_read_txn(frame, this, local->loc.inode, afr_getxattr_wind,
+                 AFR_METADATA_TRANSACTION);
 
-        ret = 0;
+    ret = 0;
 out:
-        if (ret < 0)
-                AFR_STACK_UNWIND (getxattr, frame, -1, op_errno, NULL, NULL);
-        return 0;
+    if (ret < 0)
+        AFR_STACK_UNWIND(getxattr, frame, -1, op_errno, NULL, NULL);
+    return 0;
 }
 
 /* {{{ fgetxattr */
 
-
 int32_t
-afr_fgetxattr_cbk (call_frame_t *frame, void *cookie,
-                   xlator_t *this, int32_t op_ret, int32_t op_errno,
-                   dict_t *dict, dict_t *xdata)
+afr_fgetxattr_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                  int32_t op_ret, int32_t op_errno, dict_t *dict, dict_t *xdata)
 {
-	afr_local_t *local = NULL;
+    afr_local_t *local = NULL;
 
-	local = frame->local;
+    local = frame->local;
 
-	if (op_ret < 0) {
-		local->op_ret = -1;
-		local->op_errno = op_errno;
+    if (op_ret < 0) {
+        local->op_ret = -1;
+        local->op_errno = op_errno;
 
-		afr_read_txn_continue (frame, this, (long) cookie);
-		return 0;
-	}
+        afr_read_txn_continue(frame, this, (long)cookie);
+        return 0;
+    }
 
-	if (dict)
-		afr_filter_xattrs (dict);
+    if (dict)
+        afr_filter_xattrs(dict);
 
-	AFR_STACK_UNWIND (fgetxattr, frame, op_ret, op_errno, dict, xdata);
+    AFR_STACK_UNWIND(fgetxattr, frame, op_ret, op_errno, dict, xdata);
 
-	return 0;
+    return 0;
 }
 
 int
-afr_fgetxattr_wind (call_frame_t *frame, xlator_t *this, int subvol)
+afr_fgetxattr_wind(call_frame_t *frame, xlator_t *this, int subvol)
 {
-	afr_local_t *local = NULL;
-	afr_private_t *priv = NULL;
-
-	local = frame->local;
-	priv = this->private;
-
-	if (subvol == -1) {
-		AFR_STACK_UNWIND (fgetxattr, frame, local->op_ret,
-				  local->op_errno, NULL, NULL);
-		return 0;
-	}
-
-	STACK_WIND_COOKIE (frame, afr_fgetxattr_cbk, (void *) (long) subvol,
-			   priv->children[subvol],
-			   priv->children[subvol]->fops->fgetxattr,
-			   local->fd, local->cont.getxattr.name,
-			   local->xdata_req);
-	return 0;
-}
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
 
+    local = frame->local;
+    priv = this->private;
 
-static void
-afr_fgetxattr_all_subvols (xlator_t *this, call_frame_t *frame,
-			   fop_fgetxattr_cbk_t cbk)
-{
-        afr_private_t   *priv           = NULL;
-        afr_local_t     *local          = NULL;
-        int             i               = 0;
-        int             call_count      = 0;
+    if (subvol == -1) {
+        AFR_STACK_UNWIND(fgetxattr, frame, local->op_ret, local->op_errno, NULL,
+                         NULL);
+        return 0;
+    }
 
-        priv     = this->private;
+    STACK_WIND_COOKIE(frame, afr_fgetxattr_cbk, (void *)(long)subvol,
+                      priv->children[subvol],
+                      priv->children[subvol]->fops->fgetxattr, local->fd,
+                      local->cont.getxattr.name, local->xdata_req);
+    return 0;
+}
 
-        local = frame->local;
-        //local->call_count set in afr_local_init
-        call_count = local->call_count;
-
-        //If up-children count is 0, afr_local_init would have failed already
-        //and the call would have unwound so not handling it here.
-
-        for (i = 0; i < priv->child_count; i++) {
-                if (local->child_up[i]) {
-                        STACK_WIND_COOKIE (frame, cbk,
-                                           (void *) (long) i,
-                                           priv->children[i],
-                                           priv->children[i]->fops->fgetxattr,
-                                           local->fd, local->cont.getxattr.name,
-					   NULL);
-                        if (!--call_count)
-                                break;
-                }
+static void
+afr_fgetxattr_all_subvols(xlator_t *this, call_frame_t *frame,
+                          fop_fgetxattr_cbk_t cbk)
+{
+    afr_private_t *priv = NULL;
+    afr_local_t *local = NULL;
+    int i = 0;
+    int call_count = 0;
+
+    priv = this->private;
+
+    local = frame->local;
+    // local->call_count set in afr_local_init
+    call_count = local->call_count;
+
+    // If up-children count is 0, afr_local_init would have failed already
+    // and the call would have unwound so not handling it here.
+
+    for (i = 0; i < priv->child_count; i++) {
+        if (local->child_up[i]) {
+            STACK_WIND_COOKIE(frame, cbk, (void *)(long)i, priv->children[i],
+                              priv->children[i]->fops->fgetxattr, local->fd,
+                              local->cont.getxattr.name, NULL);
+            if (!--call_count)
+                break;
         }
+    }
 
-        return;
+    return;
 }
 
-
 int
-afr_fgetxattr (call_frame_t *frame, xlator_t *this,
-               fd_t *fd, const char *name, dict_t *xdata)
+afr_fgetxattr(call_frame_t *frame, xlator_t *this, fd_t *fd, const char *name,
+              dict_t *xdata)
 {
-        afr_local_t          *local      = NULL;
-        int32_t               op_errno   = 0;
-        fop_fgetxattr_cbk_t   cbk        = NULL;
-
-        local = AFR_FRAME_INIT (frame, op_errno);
-	if (!local)
-		goto out;
-
-	local->op = GF_FOP_FGETXATTR;
-        local->fd = fd_ref (fd);
-        if (name) {
-                local->cont.getxattr.name = gf_strdup (name);
-		if (!local->cont.getxattr.name) {
-			op_errno = ENOMEM;
-			goto out;
-		}
-	}
-	if (xdata)
-		local->xdata_req = dict_ref (xdata);
-
-        /* pathinfo gets handled only in getxattr(), but we need to handle
-         * lockinfo.
-         * If we are doing fgetxattr with lockinfo as the key then we
-         * collect information from all children.
-         */
-        if (afr_is_special_xattr (name, &cbk, 1)) {
-                afr_fgetxattr_all_subvols (this, frame, cbk);
-                return 0;
+    afr_local_t *local = NULL;
+    int32_t op_errno = 0;
+    fop_fgetxattr_cbk_t cbk = NULL;
+
+    AFR_ERROR_OUT_IF_FDCTX_INVALID(fd, this, op_errno, out);
+    local = AFR_FRAME_INIT(frame, op_errno);
+    if (!local)
+        goto out;
+
+    local->op = GF_FOP_FGETXATTR;
+    local->fd = fd_ref(fd);
+    if (name) {
+        local->cont.getxattr.name = gf_strdup(name);
+        if (!local->cont.getxattr.name) {
+            op_errno = ENOMEM;
+            goto out;
         }
+    }
+    if (xdata)
+        local->xdata_req = dict_ref(xdata);
+
+    /* pathinfo gets handled only in getxattr(), but we need to handle
+     * lockinfo.
+     * If we are doing fgetxattr with lockinfo as the key then we
+     * collect information from all children.
+     */
+    if (afr_is_special_xattr(name, &cbk, 1)) {
+        afr_fgetxattr_all_subvols(this, frame, cbk);
+        return 0;
+    }
 
-        afr_fix_open (fd, this);
+    afr_fix_open(fd, this);
 
-	afr_read_txn (frame, this, fd->inode, afr_fgetxattr_wind,
-		      AFR_METADATA_TRANSACTION);
+    afr_read_txn(frame, this, fd->inode, afr_fgetxattr_wind,
+                 AFR_METADATA_TRANSACTION);
 
-	return 0;
+    return 0;
 out:
-	AFR_STACK_UNWIND (fgetxattr, frame, -1, op_errno, NULL, NULL);
+    AFR_STACK_UNWIND(fgetxattr, frame, -1, op_errno, NULL, NULL);
 
-        return 0;
+    return 0;
 }
 
-
 /* }}} */
 
 /* {{{ readv */
 
 int
-afr_readv_cbk (call_frame_t *frame, void *cookie,
-               xlator_t *this, int32_t op_ret, int32_t op_errno,
-               struct iovec *vector, int32_t count, struct iatt *buf,
-               struct iobref *iobref, dict_t *xdata)
+afr_readv_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int32_t op_ret,
+              int32_t op_errno, struct iovec *vector, int32_t count,
+              struct iatt *buf, struct iobref *iobref, dict_t *xdata)
 {
-	afr_local_t *local = NULL;
+    afr_local_t *local = NULL;
 
-	local = frame->local;
+    local = frame->local;
 
-	if (op_ret < 0) {
-		local->op_ret = -1;
-		local->op_errno = op_errno;
+    if (op_ret < 0) {
+        local->op_ret = -1;
+        local->op_errno = op_errno;
 
-		afr_read_txn_continue (frame, this, (long) cookie);
-		return 0;
-	}
+        afr_read_txn_continue(frame, this, (long)cookie);
+        return 0;
+    }
 
-	AFR_STACK_UNWIND (readv, frame, op_ret, op_errno,
-			  vector, count, buf, iobref, xdata);
-	return 0;
+    AFR_STACK_UNWIND(readv, frame, op_ret, op_errno, vector, count, buf, iobref,
+                     xdata);
+    return 0;
 }
 
+int
+afr_readv_wind(call_frame_t *frame, xlator_t *this, int subvol)
+{
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
+
+    local = frame->local;
+    priv = this->private;
+
+    if (subvol == -1) {
+        AFR_STACK_UNWIND(readv, frame, local->op_ret, local->op_errno, 0, 0, 0,
+                         0, 0);
+        return 0;
+    }
+
+    STACK_WIND_COOKIE(
+        frame, afr_readv_cbk, (void *)(long)subvol, priv->children[subvol],
+        priv->children[subvol]->fops->readv, local->fd, local->cont.readv.size,
+        local->cont.readv.offset, local->cont.readv.flags, local->xdata_req);
+    return 0;
+}
 
 int
-afr_readv_wind (call_frame_t *frame, xlator_t *this, int subvol)
+afr_readv(call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
+          off_t offset, uint32_t flags, dict_t *xdata)
 {
-	afr_local_t *local = NULL;
-	afr_private_t *priv = NULL;
-
-	local = frame->local;
-	priv = this->private;
-
-	if (subvol == -1) {
-		AFR_STACK_UNWIND (readv, frame, local->op_ret, local->op_errno,
-				  0, 0, 0, 0, 0);
-		return 0;
-	}
-
-	STACK_WIND_COOKIE (frame, afr_readv_cbk, (void *) (long) subvol,
-			   priv->children[subvol],
-			   priv->children[subvol]->fops->readv,
-			   local->fd, local->cont.readv.size,
-			   local->cont.readv.offset, local->cont.readv.flags,
-			   local->xdata_req);
-	return 0;
+    afr_local_t *local = NULL;
+    int32_t op_errno = 0;
+
+    AFR_ERROR_OUT_IF_FDCTX_INVALID(fd, this, op_errno, out);
+    local = AFR_FRAME_INIT(frame, op_errno);
+    if (!local)
+        goto out;
+
+    local->op = GF_FOP_READ;
+    local->fd = fd_ref(fd);
+    local->cont.readv.size = size;
+    local->cont.readv.offset = offset;
+    local->cont.readv.flags = flags;
+    if (xdata)
+        local->xdata_req = dict_ref(xdata);
+
+    afr_fix_open(fd, this);
+
+    afr_read_txn(frame, this, fd->inode, afr_readv_wind, AFR_DATA_TRANSACTION);
+
+    return 0;
+out:
+    AFR_STACK_UNWIND(readv, frame, -1, op_errno, 0, 0, 0, 0, 0);
+
+    return 0;
 }
 
+/* }}} */
+
+/* {{{ seek */
 
 int
-afr_readv (call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
-	   off_t offset, uint32_t flags, dict_t *xdata)
+afr_seek_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int32_t op_ret,
+             int32_t op_errno, off_t offset, dict_t *xdata)
 {
-        afr_local_t   * local      = NULL;
-        int32_t         op_errno   = 0;
+    afr_local_t *local = NULL;
 
-	local = AFR_FRAME_INIT (frame, op_errno);
-	if (!local)
-		goto out;
+    local = frame->local;
 
-	local->op = GF_FOP_READ;
-        local->fd = fd_ref (fd);
-        local->cont.readv.size = size;
-        local->cont.readv.offset = offset;
-        local->cont.readv.flags = flags;
-	if (xdata)
-		local->xdata_req = dict_ref (xdata);
+    if (op_ret < 0) {
+        local->op_ret = -1;
+        local->op_errno = op_errno;
 
-        afr_fix_open (fd, this);
+        afr_read_txn_continue(frame, this, (long)cookie);
+        return 0;
+    }
+
+    AFR_STACK_UNWIND(seek, frame, op_ret, op_errno, offset, xdata);
+    return 0;
+}
 
-	afr_read_txn (frame, this, fd->inode, afr_readv_wind,
-		      AFR_DATA_TRANSACTION);
+int
+afr_seek_wind(call_frame_t *frame, xlator_t *this, int subvol)
+{
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
 
+    local = frame->local;
+    priv = this->private;
+
+    if (subvol == -1) {
+        AFR_STACK_UNWIND(seek, frame, local->op_ret, local->op_errno, 0, NULL);
         return 0;
-out:
-	AFR_STACK_UNWIND(readv, frame, -1, op_errno, 0, 0, 0, 0, 0);
+    }
 
-	return 0;
+    STACK_WIND_COOKIE(
+        frame, afr_seek_cbk, (void *)(long)subvol, priv->children[subvol],
+        priv->children[subvol]->fops->seek, local->fd, local->cont.seek.offset,
+        local->cont.seek.what, local->xdata_req);
+    return 0;
 }
 
+int
+afr_seek(call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
+         gf_seek_what_t what, dict_t *xdata)
+{
+    afr_local_t *local = NULL;
+    int32_t op_errno = 0;
+
+    AFR_ERROR_OUT_IF_FDCTX_INVALID(fd, this, op_errno, out);
+    local = AFR_FRAME_INIT(frame, op_errno);
+    if (!local)
+        goto out;
+
+    local->op = GF_FOP_SEEK;
+    local->fd = fd_ref(fd);
+    local->cont.seek.offset = offset;
+    local->cont.seek.what = what;
+    if (xdata)
+        local->xdata_req = dict_ref(xdata);
+
+    afr_fix_open(fd, this);
+
+    afr_read_txn(frame, this, fd->inode, afr_seek_wind, AFR_DATA_TRANSACTION);
+
+    return 0;
+out:
+    AFR_STACK_UNWIND(seek, frame, -1, op_errno, 0, NULL);
+
+    return 0;
+}
 /* }}} */
diff --git a/xlators/cluster/afr/src/afr-inode-read.h b/xlators/cluster/afr/src/afr-inode-read.h
index d128134ef2a..8c982bc7e6f 100644
--- a/xlators/cluster/afr/src/afr-inode-read.h
+++ b/xlators/cluster/afr/src/afr-inode-read.h
@@ -12,34 +12,34 @@
 #define __INODE_READ_H__
 
 int32_t
-afr_access (call_frame_t *frame, xlator_t *this,
-	    loc_t *loc, int32_t mask, dict_t *xdata);
+afr_access(call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t mask,
+           dict_t *xdata);
 
 int32_t
-afr_stat (call_frame_t *frame, xlator_t *this,
-	  loc_t *loc, dict_t *xdata);
+afr_stat(call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xdata);
 
 int32_t
-afr_fstat (call_frame_t *frame, xlator_t *this,
-	   fd_t *fd, dict_t *xdata);
+afr_fstat(call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *xdata);
 
 int32_t
-afr_readlink (call_frame_t *frame, xlator_t *this,
-	      loc_t *loc, size_t size, dict_t *xdata);
+afr_readlink(call_frame_t *frame, xlator_t *this, loc_t *loc, size_t size,
+             dict_t *xdata);
 
 int32_t
-afr_readv (call_frame_t *frame, xlator_t *this,
-	   fd_t *fd, size_t size, off_t offset, uint32_t flags, dict_t *xdata);
+afr_readv(call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
+          off_t offset, uint32_t flags, dict_t *xdata);
 
 int32_t
-afr_getxattr (call_frame_t *frame, xlator_t *this,
-	      loc_t *loc, const char *name, dict_t *xdata);
+afr_getxattr(call_frame_t *frame, xlator_t *this, loc_t *loc, const char *name,
+             dict_t *xdata);
 
 int32_t
-afr_fgetxattr (call_frame_t *frame, xlator_t *this,
-               fd_t *fd, const char *name, dict_t *xdata);
-
+afr_fgetxattr(call_frame_t *frame, xlator_t *this, fd_t *fd, const char *name,
+              dict_t *xdata);
 
 int
-afr_handle_quota_size (call_frame_t *frame, xlator_t *this);
+afr_seek(call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
+         gf_seek_what_t what, dict_t *xdata);
+int
+afr_handle_quota_size(call_frame_t *frame, xlator_t *this);
 #endif /* __INODE_READ_H__ */
diff --git a/xlators/cluster/afr/src/afr-inode-write.c b/xlators/cluster/afr/src/afr-inode-write.c
index f712166e44d..1d6e4f3570a 100644
--- a/xlators/cluster/afr/src/afr-inode-write.c
+++ b/xlators/cluster/afr/src/afr-inode-write.c
@@ -8,694 +8,779 @@
   cases as published by the Free Software Foundation.
 */
 
-
-#include <libgen.h>
 #include <unistd.h>
-#include <fnmatch.h>
 #include <sys/time.h>
 #include <stdlib.h>
 #include <signal.h>
 
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
-
-#include "glusterfs.h"
+#include <glusterfs/glusterfs.h>
 #include "afr.h"
-#include "dict.h"
-#include "xlator.h"
-#include "hashfn.h"
-#include "logging.h"
-#include "stack.h"
-#include "list.h"
-#include "call-stub.h"
-#include "defaults.h"
-#include "common-utils.h"
-#include "compat-errno.h"
-#include "compat.h"
+#include <glusterfs/dict.h>
+#include <glusterfs/logging.h>
+#include <glusterfs/defaults.h>
+#include <glusterfs/common-utils.h>
+#include <glusterfs/compat-errno.h>
+#include <glusterfs/compat.h>
 #include "protocol-common.h"
-
+#include <glusterfs/byte-order.h>
 #include "afr-transaction.h"
-
+#include "afr-self-heal.h"
+#include "afr-messages.h"
 
 static void
-__afr_inode_write_finalize (call_frame_t *frame, xlator_t *this)
-{
-	afr_local_t *local = NULL;
-	afr_private_t *priv = NULL;
-	int read_subvol = 0;
-	int i = 0;
-
-	local = frame->local;
-	priv = this->private;
-
-	if (local->inode) {
-		if (local->transaction.type == AFR_METADATA_TRANSACTION)
-			read_subvol = afr_metadata_subvol_get (local->inode, this,
-							       NULL, NULL);
-		else
-			read_subvol = afr_data_subvol_get (local->inode, this,
-							   NULL, NULL);
-	}
-
-	local->op_ret = -1;
-	local->op_errno = afr_final_errno (local, priv);
-
-	for (i = 0; i < priv->child_count; i++) {
-		if (!local->replies[i].valid)
-			continue;
-		if (local->replies[i].op_ret < 0) {
-			afr_inode_read_subvol_reset (local->inode, this);
-			continue;
-		}
-
-		/* Order of checks in the compound conditional
-		   below is important.
-
-		   - Highest precedence: largest op_ret
-		   - Next precendence: if all op_rets are equal, read subvol
-		   - Least precedence: any succeeded subvol
-		*/
-		if ((local->op_ret < local->replies[i].op_ret) ||
-		    ((local->op_ret == local->replies[i].op_ret) &&
-		     (i == read_subvol))) {
-
-			local->op_ret = local->replies[i].op_ret;
-			local->op_errno = local->replies[i].op_errno;
-
-			local->cont.inode_wfop.prebuf =
-				local->replies[i].prestat;
-			local->cont.inode_wfop.postbuf =
-				local->replies[i].poststat;
-
-			if (local->replies[i].xdata) {
-				if (local->xdata_rsp)
-					dict_unref (local->xdata_rsp);
-				local->xdata_rsp =
-					dict_ref (local->replies[i].xdata);
-			}
-		}
-	}
-}
+__afr_inode_write_finalize(call_frame_t *frame, xlator_t *this)
+{
+    int i = 0;
+    int ret = 0;
+    int read_subvol = 0;
+    struct iatt *stbuf = NULL;
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
+    afr_read_subvol_args_t args = {
+        0,
+    };
+
+    local = frame->local;
+    priv = this->private;
+    GF_VALIDATE_OR_GOTO(this->name, local->inode, out);
+
+    /*This code needs to stay till DHT sends fops on linked
+     * inodes*/
+    if (!inode_is_linked(local->inode)) {
+        for (i = 0; i < priv->child_count; i++) {
+            if (!local->replies[i].valid)
+                continue;
+            if (local->replies[i].op_ret == -1)
+                continue;
+            if (!gf_uuid_is_null(local->replies[i].poststat.ia_gfid)) {
+                gf_uuid_copy(args.gfid, local->replies[i].poststat.ia_gfid);
+                args.ia_type = local->replies[i].poststat.ia_type;
+                break;
+            } else {
+                ret = dict_get_bin(local->replies[i].xdata,
+                                   DHT_IATT_IN_XDATA_KEY, (void **)&stbuf);
+                if (ret)
+                    continue;
+                gf_uuid_copy(args.gfid, stbuf->ia_gfid);
+                args.ia_type = stbuf->ia_type;
+                break;
+            }
+        }
+    }
+
+    if (local->transaction.type == AFR_METADATA_TRANSACTION) {
+        read_subvol = afr_metadata_subvol_get(local->inode, this, NULL,
+                                              local->readable, NULL, &args);
+    } else {
+        read_subvol = afr_data_subvol_get(local->inode, this, NULL,
+                                          local->readable, NULL, &args);
+    }
+
+    local->op_ret = -1;
+    local->op_errno = afr_final_errno(local, priv);
+    afr_pick_error_xdata(local, priv, local->inode, local->readable, NULL,
+                         NULL);
+
+    for (i = 0; i < priv->child_count; i++) {
+        if (!local->replies[i].valid)
+            continue;
+        if (local->replies[i].op_ret < 0)
+            continue;
+
+        /* Order of checks in the compound conditional
+           below is important.
+
+           - Highest precedence: largest op_ret
+           - Next precedence: if all op_rets are equal, read subvol
+           - Least precedence: any succeeded subvol
+        */
+        if ((local->op_ret < local->replies[i].op_ret) ||
+            ((local->op_ret == local->replies[i].op_ret) &&
+             (i == read_subvol))) {
+            local->op_ret = local->replies[i].op_ret;
+            local->op_errno = local->replies[i].op_errno;
+
+            local->cont.inode_wfop.prebuf = local->replies[i].prestat;
+            local->cont.inode_wfop.postbuf = local->replies[i].poststat;
+
+            if (local->replies[i].xdata) {
+                if (local->xdata_rsp)
+                    dict_unref(local->xdata_rsp);
+                local->xdata_rsp = dict_ref(local->replies[i].xdata);
+            }
+            if (local->replies[i].xattr) {
+                if (local->xattr_rsp)
+                    dict_unref(local->xattr_rsp);
+                local->xattr_rsp = dict_ref(local->replies[i].xattr);
+            }
+        }
+    }
 
+    afr_set_in_flight_sb_status(this, frame, local->inode);
+out:
+    return;
+}
 
 static void
-__afr_inode_write_fill (call_frame_t *frame, xlator_t *this, int child_index,
-			int op_ret, int op_errno,
-			struct iatt *prebuf, struct iatt *postbuf, dict_t *xdata)
+__afr_inode_write_fill(call_frame_t *frame, xlator_t *this, int child_index,
+                       int op_ret, int op_errno, struct iatt *prebuf,
+                       struct iatt *postbuf, dict_t *xattr, dict_t *xdata)
 {
-        afr_local_t *local = NULL;
-
-        local = frame->local;
-
-	local->replies[child_index].valid = 1;
-	local->replies[child_index].op_ret = op_ret;
-	local->replies[child_index].op_errno = op_errno;
-
-	if (op_ret >= 0) {
-		if (prebuf)
-			local->replies[child_index].prestat = *prebuf;
-		if (postbuf)
-			local->replies[child_index].poststat = *postbuf;
-		if (xdata)
-			local->replies[child_index].xdata = dict_ref (xdata);
-	} else {
-		afr_transaction_fop_failed (frame, this, child_index);
-	}
-
-        return;
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
+
+    local = frame->local;
+    priv = this->private;
+
+    local->replies[child_index].valid = 1;
+
+    if (AFR_IS_ARBITER_BRICK(priv, child_index) && op_ret == 1)
+        op_ret = iov_length(local->cont.writev.vector,
+                            local->cont.writev.count);
+
+    local->replies[child_index].op_ret = op_ret;
+    local->replies[child_index].op_errno = op_errno;
+    if (xdata)
+        local->replies[child_index].xdata = dict_ref(xdata);
+
+    if (op_ret >= 0) {
+        if (prebuf)
+            local->replies[child_index].prestat = *prebuf;
+        if (postbuf)
+            local->replies[child_index].poststat = *postbuf;
+        if (xattr)
+            local->replies[child_index].xattr = dict_ref(xattr);
+    } else {
+        afr_transaction_fop_failed(frame, this, child_index);
+    }
+
+    return;
 }
 
-
 static int
-__afr_inode_write_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                       int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
-                       struct iatt *postbuf, dict_t *xdata)
+__afr_inode_write_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                      int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
+                      struct iatt *postbuf, dict_t *xattr, dict_t *xdata)
 {
-        afr_local_t *local = NULL;
-        int child_index = (long) cookie;
-        int call_count = -1;
-
-        local = frame->local;
-
-        LOCK (&frame->lock);
-        {
-                __afr_inode_write_fill (frame, this, child_index, op_ret,
-					op_errno, prebuf, postbuf, xdata);
+    afr_local_t *local = NULL;
+    int child_index = (long)cookie;
+    int call_count = -1;
+    afr_private_t *priv = NULL;
+
+    priv = this->private;
+    local = frame->local;
+
+    LOCK(&frame->lock);
+    {
+        __afr_inode_write_fill(frame, this, child_index, op_ret, op_errno,
+                               prebuf, postbuf, xattr, xdata);
+        call_count = --local->call_count;
+    }
+    UNLOCK(&frame->lock);
+
+    if (call_count == 0) {
+        __afr_inode_write_finalize(frame, this);
+
+        if (afr_txn_nothing_failed(frame, this)) {
+            /*if it did pre-op, it will do post-op changing ctime*/
+            if (priv->consistent_metadata && afr_needs_changelog_update(local))
+                afr_zero_fill_stat(local);
+            local->transaction.unwind(frame, this);
         }
-        UNLOCK (&frame->lock);
-
-        call_count = afr_frame_return (frame);
-
-        if (call_count == 0) {
-		__afr_inode_write_finalize (frame, this);
 
-		if (afr_txn_nothing_failed (frame, this))
-			local->transaction.unwind (frame, this);
+        afr_transaction_resume(frame, this);
+    }
 
-                local->transaction.resume (frame, this);
-        }
-
-        return 0;
+    return 0;
 }
 
 /* {{{ writev */
 
 void
-afr_writev_copy_outvars (call_frame_t *src_frame, call_frame_t *dst_frame)
+afr_writev_copy_outvars(call_frame_t *src_frame, call_frame_t *dst_frame)
 {
-        afr_local_t *src_local = NULL;
-        afr_local_t *dst_local = NULL;
-
-        src_local = src_frame->local;
-        dst_local = dst_frame->local;
-
-        dst_local->op_ret = src_local->op_ret;
-        dst_local->op_errno = src_local->op_errno;
-        dst_local->cont.inode_wfop.prebuf = src_local->cont.inode_wfop.prebuf;
-        dst_local->cont.inode_wfop.postbuf = src_local->cont.inode_wfop.postbuf;
-	if (src_local->xdata_rsp)
-		dst_local->xdata_rsp = dict_ref (src_local->xdata_rsp);
+    afr_local_t *src_local = NULL;
+    afr_local_t *dst_local = NULL;
+
+    src_local = src_frame->local;
+    dst_local = dst_frame->local;
+
+    dst_local->op_ret = src_local->op_ret;
+    dst_local->op_errno = src_local->op_errno;
+    dst_local->cont.inode_wfop.prebuf = src_local->cont.inode_wfop.prebuf;
+    dst_local->cont.inode_wfop.postbuf = src_local->cont.inode_wfop.postbuf;
+    if (src_local->xdata_rsp)
+        dst_local->xdata_rsp = dict_ref(src_local->xdata_rsp);
 }
 
 void
-afr_writev_unwind (call_frame_t *frame, xlator_t *this)
+afr_writev_unwind(call_frame_t *frame, xlator_t *this)
 {
-        afr_local_t *   local = NULL;
-        local = frame->local;
+    afr_local_t *local = NULL;
+    afr_private_t *priv = this->private;
 
-        AFR_STACK_UNWIND (writev, frame,
-                          local->op_ret, local->op_errno,
-                          &local->cont.inode_wfop.prebuf,
-                          &local->cont.inode_wfop.postbuf,
-                          local->xdata_rsp);
-}
+    local = frame->local;
 
+    if (priv->consistent_metadata)
+        afr_zero_fill_stat(local);
+
+    AFR_STACK_UNWIND(writev, frame, local->op_ret, local->op_errno,
+                     &local->cont.inode_wfop.prebuf,
+                     &local->cont.inode_wfop.postbuf, local->xdata_rsp);
+}
 
 int
-afr_transaction_writev_unwind (call_frame_t *frame, xlator_t *this)
+afr_transaction_writev_unwind(call_frame_t *frame, xlator_t *this)
 {
-        call_frame_t *fop_frame = NULL;
+    call_frame_t *fop_frame = NULL;
 
-        fop_frame = afr_transaction_detach_fop_frame (frame);
+    fop_frame = afr_transaction_detach_fop_frame(frame);
 
-        if (fop_frame) {
-                afr_writev_copy_outvars (frame, fop_frame);
-                afr_writev_unwind (fop_frame, this);
-        }
-        return 0;
+    if (fop_frame) {
+        afr_writev_copy_outvars(frame, fop_frame);
+        afr_writev_unwind(fop_frame, this);
+    }
+    return 0;
 }
 
 static void
-afr_writev_handle_short_writes (call_frame_t *frame, xlator_t *this)
+afr_writev_handle_short_writes(call_frame_t *frame, xlator_t *this)
 {
-        afr_local_t   *local = NULL;
-        afr_private_t *priv  = NULL;
-        int           i      = 0;
-
-        local = frame->local;
-        priv = this->private;
-        /*
-         * We already have the best case result of the writev calls staged
-         * as the return value. Any writev that returns some value less
-         * than the best case is now out of sync, so mark the fop as
-         * failed. Note that fops that have returned with errors have
-         * already been marked as failed.
-         */
-        for (i = 0; i < priv->child_count; i++) {
-                if ((!local->replies[i].valid) ||
-                    (local->replies[i].op_ret == -1))
-                        continue;
-
-                if (local->replies[i].op_ret < local->op_ret)
-                        afr_transaction_fop_failed (frame, this, i);
-        }
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
+    int i = 0;
+
+    local = frame->local;
+    priv = this->private;
+    /*
+     * We already have the best case result of the writev calls staged
+     * as the return value. Any writev that returns some value less
+     * than the best case is now out of sync, so mark the fop as
+     * failed. Note that fops that have returned with errors have
+     * already been marked as failed.
+     */
+    for (i = 0; i < priv->child_count; i++) {
+        if ((!local->replies[i].valid) || (local->replies[i].op_ret == -1))
+            continue;
+
+        if (local->replies[i].op_ret < local->op_ret)
+            afr_transaction_fop_failed(frame, this, i);
+    }
 }
 
-int
-afr_writev_wind_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+void
+afr_inode_write_fill(call_frame_t *frame, xlator_t *this, int child_index,
                      int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
                      struct iatt *postbuf, dict_t *xdata)
 {
-        afr_local_t *   local = NULL;
-        call_frame_t    *fop_frame = NULL;
-        int child_index = (long) cookie;
-        int call_count  = -1;
-        int ret = 0;
-        uint32_t open_fd_count = 0;
-        uint32_t write_is_append = 0;
-
-        local = frame->local;
-
-        LOCK (&frame->lock);
-        {
-                __afr_inode_write_fill (frame, this, child_index, op_ret,
-					op_errno, prebuf, postbuf, xdata);
-		if (op_ret == -1 || !xdata)
-			goto unlock;
-
-		write_is_append = 0;
-		ret = dict_get_uint32 (xdata, GLUSTERFS_WRITE_IS_APPEND,
-				       &write_is_append);
-		if (ret || !write_is_append)
-			local->append_write = _gf_false;
-
-		ret = dict_get_uint32 (xdata, GLUSTERFS_OPEN_FD_COUNT,
-				       &open_fd_count);
-		if (ret == -1)
-			goto unlock;
-		if ((open_fd_count > local->open_fd_count)) {
-			local->open_fd_count = open_fd_count;
-			local->update_open_fd_count = _gf_true;
-		}
+    int ret = 0;
+    afr_local_t *local = frame->local;
+    uint32_t open_fd_count = 0;
+    uint32_t write_is_append = 0;
+    int32_t num_inodelks = 0;
+
+    LOCK(&frame->lock);
+    {
+        __afr_inode_write_fill(frame, this, child_index, op_ret, op_errno,
+                               prebuf, postbuf, NULL, xdata);
+        if (op_ret == -1 || !xdata)
+            goto unlock;
+
+        write_is_append = 0;
+        ret = dict_get_uint32(xdata, GLUSTERFS_WRITE_IS_APPEND,
+                              &write_is_append);
+        if (ret || !write_is_append)
+            local->append_write = _gf_false;
+
+        ret = dict_get_uint32(xdata, GLUSTERFS_ACTIVE_FD_COUNT, &open_fd_count);
+        if (ret < 0)
+            goto unlock;
+        if (open_fd_count > local->open_fd_count) {
+            local->open_fd_count = open_fd_count;
+            local->update_open_fd_count = _gf_true;
         }
-unlock:
-        UNLOCK (&frame->lock);
-
-        call_count = afr_frame_return (frame);
-
-        if (call_count == 0) {
-		if (!local->stable_write && !local->append_write)
-			/* An appended write removes the necessity to
-			   fsync() the file. This is because self-heal
-			   has the logic to check for larger file when
-			   the xattrs are not reliably pointing at
-			   a stale file.
-			*/
-			afr_fd_report_unstable_write (this, local->fd);
-
-		__afr_inode_write_finalize (frame, this);
-
-                afr_writev_handle_short_writes (frame, this);
-
-                if (local->update_open_fd_count)
-                        afr_handle_open_fd_count (frame, this);
-
-                if (!afr_txn_nothing_failed (frame, this)) {
-                        //Don't unwind until post-op is complete
-                        local->transaction.resume (frame, this);
-                } else {
-                /*
-                 * Generally inode-write fops do transaction.unwind then
-                 * transaction.resume, but writev needs to make sure that
-                 * delayed post-op frame is placed in fdctx before unwind
-                 * happens. This prevents the race of flush doing the
-                 * changelog wakeup first in fuse thread and then this
-                 * writev placing its delayed post-op frame in fdctx.
-                 * This helps flush make sure all the delayed post-ops are
-                 * completed.
-                 */
-
-                        fop_frame = afr_transaction_detach_fop_frame (frame);
-                        afr_writev_copy_outvars (frame, fop_frame);
-                        local->transaction.resume (frame, this);
-                        afr_writev_unwind (fop_frame, this);
-                }
+
+        ret = dict_get_int32_sizen(xdata, GLUSTERFS_INODELK_COUNT,
+                                   &num_inodelks);
+        if (ret < 0)
+            goto unlock;
+        if (num_inodelks > local->num_inodelks) {
+            local->num_inodelks = num_inodelks;
+            local->update_num_inodelks = _gf_true;
         }
-        return 0;
+    }
+unlock:
+    UNLOCK(&frame->lock);
 }
 
-
-int
-afr_writev_wind (call_frame_t *frame, xlator_t *this, int subvol)
+void
+afr_process_post_writev(call_frame_t *frame, xlator_t *this)
 {
-        afr_local_t *local = NULL;
-        afr_private_t *priv = NULL;
-
-        local = frame->local;
-        priv = this->private;
-
-	STACK_WIND_COOKIE (frame, afr_writev_wind_cbk, (void *) (long) subvol,
-			   priv->children[subvol],
-			   priv->children[subvol]->fops->writev,
-			   local->fd, local->cont.writev.vector,
-			   local->cont.writev.count, local->cont.writev.offset,
-			   local->cont.writev.flags, local->cont.writev.iobref,
-			   local->xdata_req);
-        return 0;
+    afr_local_t *local = NULL;
+    afr_lock_t *lock = NULL;
+
+    local = frame->local;
+
+    if (!local->stable_write && !local->append_write)
+        /* An appended write removes the necessity to
+           fsync() the file. This is because self-heal
+           has the logic to check for larger file when
+           the xattrs are not reliably pointing at
+           a stale file.
+        */
+        afr_fd_report_unstable_write(this, local);
+
+    __afr_inode_write_finalize(frame, this);
+
+    afr_writev_handle_short_writes(frame, this);
+
+    if (local->update_open_fd_count)
+        local->inode_ctx->open_fd_count = local->open_fd_count;
+    if (local->update_num_inodelks &&
+        local->transaction.type == AFR_DATA_TRANSACTION) {
+        lock = &local->inode_ctx->lock[local->transaction.type];
+        lock->num_inodelks = local->num_inodelks;
+    }
 }
 
-
 int
-afr_do_writev (call_frame_t *frame, xlator_t *this)
+afr_writev_wind_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                    int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
+                    struct iatt *postbuf, dict_t *xdata)
 {
-        call_frame_t    *transaction_frame = NULL;
-        afr_local_t     *local             = NULL;
-        int             ret   = -1;
-        int             op_errno = ENOMEM;
-
-        transaction_frame = copy_frame (frame);
-        if (!transaction_frame)
-                goto out;
-
-        local = frame->local;
-        transaction_frame->local = local;
-	frame->local = NULL;
-
-	if (!AFR_FRAME_INIT (frame, op_errno))
-		goto out;
+    call_frame_t *fop_frame = NULL;
+    int child_index = (long)cookie;
+    int call_count = -1;
 
-        local->op = GF_FOP_WRITE;
+    afr_inode_write_fill(frame, this, child_index, op_ret, op_errno, prebuf,
+                         postbuf, xdata);
 
-        local->transaction.wind   = afr_writev_wind;
-        local->transaction.fop    = __afr_txn_write_fop;
-        local->transaction.done   = __afr_txn_write_done;
-        local->transaction.unwind = afr_transaction_writev_unwind;
+    call_count = afr_frame_return(frame);
 
-        local->transaction.main_frame = frame;
+    if (call_count == 0) {
+        afr_process_post_writev(frame, this);
 
-        if (local->fd->flags & O_APPEND) {
-               /*
-                * Backend vfs ignores the 'offset' for append mode fd so
-                * locking just the region provided for the writev does not
-                * give consistency guarantee. The actual write may happen at a
-                * completely different range than the one provided by the
-                * offset, len in the fop. So lock the entire file.
-                */
-                local->transaction.start   = 0;
-                local->transaction.len     = 0;
+        if (!afr_txn_nothing_failed(frame, this)) {
+            // Don't unwind until post-op is complete
+            afr_transaction_resume(frame, this);
         } else {
-                local->transaction.start   = local->cont.writev.offset;
-                local->transaction.len     = iov_length (local->cont.writev.vector,
-                                                         local->cont.writev.count);
+            /*
+             * Generally inode-write fops do transaction.unwind then
+             * transaction.resume, but writev needs to make sure that
+             * delayed post-op frame is placed in fdctx before unwind
+             * happens. This prevents the race of flush doing the
+             * changelog wakeup first in fuse thread and then this
+             * writev placing its delayed post-op frame in fdctx.
+             * This helps flush make sure all the delayed post-ops are
+             * completed.
+             */
+
+            fop_frame = afr_transaction_detach_fop_frame(frame);
+            afr_writev_copy_outvars(frame, fop_frame);
+            afr_transaction_resume(frame, this);
+            afr_writev_unwind(fop_frame, this);
         }
-
-        ret = afr_transaction (transaction_frame, this, AFR_DATA_TRANSACTION);
-        if (ret < 0) {
-            op_errno = -ret;
-            goto out;
-        }
-
-	return 0;
-out:
-	if (transaction_frame)
-		AFR_STACK_DESTROY (transaction_frame);
-
-	AFR_STACK_UNWIND (writev, frame, -1, op_errno, NULL, NULL, NULL);
-        return 0;
+    }
+    return 0;
 }
 
+static int
+afr_arbiter_writev_wind(call_frame_t *frame, xlator_t *this, int subvol)
+{
+    afr_local_t *local = frame->local;
+    afr_private_t *priv = this->private;
+    static char byte = 0xFF;
+    static struct iovec vector = {&byte, 1};
+    int32_t count = 1;
+
+    STACK_WIND_COOKIE(
+        frame, afr_writev_wind_cbk, (void *)(long)subvol,
+        priv->children[subvol], priv->children[subvol]->fops->writev, local->fd,
+        &vector, count, local->cont.writev.offset, local->cont.writev.flags,
+        local->cont.writev.iobref, local->xdata_req);
+
+    return 0;
+}
 
 int
-afr_writev (call_frame_t *frame, xlator_t *this, fd_t *fd,
-            struct iovec *vector, int32_t count, off_t offset,
-            uint32_t flags, struct iobref *iobref, dict_t *xdata)
+afr_writev_wind(call_frame_t *frame, xlator_t *this, int subvol)
 {
-        afr_local_t *local = NULL;
-        int op_errno = ENOMEM;
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
 
-	local = AFR_FRAME_INIT (frame, op_errno);
-	if (!local)
-		goto out;
+    local = frame->local;
+    priv = this->private;
 
-        local->cont.writev.vector = iov_dup (vector, count);
-	if (!local->cont.writev.vector)
-		goto out;
-        local->cont.writev.count      = count;
-        local->cont.writev.offset     = offset;
-        local->cont.writev.flags      = flags;
-        local->cont.writev.iobref     = iobref_ref (iobref);
+    if (AFR_IS_ARBITER_BRICK(priv, subvol)) {
+        afr_arbiter_writev_wind(frame, this, subvol);
+        return 0;
+    }
+
+    STACK_WIND_COOKIE(frame, afr_writev_wind_cbk, (void *)(long)subvol,
+                      priv->children[subvol],
+                      priv->children[subvol]->fops->writev, local->fd,
+                      local->cont.writev.vector, local->cont.writev.count,
+                      local->cont.writev.offset, local->cont.writev.flags,
+                      local->cont.writev.iobref, local->xdata_req);
+    return 0;
+}
 
-	if (xdata)
-		local->xdata_req = dict_copy_with_ref (xdata, NULL);
-	else
-		local->xdata_req = dict_new ();
+int
+afr_do_writev(call_frame_t *frame, xlator_t *this)
+{
+    call_frame_t *transaction_frame = NULL;
+    afr_local_t *local = NULL;
+    int ret = -1;
+    int op_errno = ENOMEM;
 
-	if (!local->xdata_req)
-		goto out;
+    transaction_frame = copy_frame(frame);
+    if (!transaction_frame)
+        goto out;
 
-        local->fd = fd_ref (fd);
-	local->inode = inode_ref (fd->inode);
+    local = frame->local;
+    transaction_frame->local = local;
+    frame->local = NULL;
 
-	if (dict_set_uint32 (local->xdata_req, GLUSTERFS_OPEN_FD_COUNT, 4)) {
-		op_errno = ENOMEM;
-		goto out;
-	}
+    if (!AFR_FRAME_INIT(frame, op_errno))
+        goto out;
 
-	if (dict_set_uint32 (local->xdata_req, GLUSTERFS_WRITE_IS_APPEND, 4)) {
-		op_errno = ENOMEM;
-		goto out;
-	}
+    local->op = GF_FOP_WRITE;
 
-	/* Set append_write to be true speculatively. If on any
-	   server it turns not be true, we unset it in the
-	   callback.
-	*/
-	local->append_write = _gf_true;
+    local->transaction.wind = afr_writev_wind;
+    local->transaction.unwind = afr_transaction_writev_unwind;
 
-	/* detect here, but set it in writev_wind_cbk *after* the unstable
-	   write is performed
-	*/
-	local->stable_write = !!((fd->flags|flags)&(O_SYNC|O_DSYNC));
+    local->transaction.main_frame = frame;
 
-        afr_fix_open (fd, this);
+    if (local->fd->flags & O_APPEND) {
+        /*
+         * Backend vfs ignores the 'offset' for append mode fd so
+         * locking just the region provided for the writev does not
+         * give consistency guarantee. The actual write may happen at a
+         * completely different range than the one provided by the
+         * offset, len in the fop. So lock the entire file.
+         */
+        local->transaction.start = 0;
+        local->transaction.len = 0;
+    } else {
+        local->transaction.start = local->cont.writev.offset;
+        local->transaction.len = iov_length(local->cont.writev.vector,
+                                            local->cont.writev.count);
+    }
+
+    ret = afr_transaction(transaction_frame, this, AFR_DATA_TRANSACTION);
+    if (ret < 0) {
+        op_errno = -ret;
+        goto out;
+    }
+
+    return 0;
+out:
+    if (transaction_frame)
+        AFR_STACK_DESTROY(transaction_frame);
 
-        afr_do_writev (frame, this);
+    AFR_STACK_UNWIND(writev, frame, -1, op_errno, NULL, NULL, NULL);
+    return 0;
+}
 
-	return 0;
+int
+afr_writev(call_frame_t *frame, xlator_t *this, fd_t *fd, struct iovec *vector,
+           int32_t count, off_t offset, uint32_t flags, struct iobref *iobref,
+           dict_t *xdata)
+{
+    afr_local_t *local = NULL;
+    int op_errno = ENOMEM;
+    int ret = -1;
+
+    AFR_ERROR_OUT_IF_FDCTX_INVALID(fd, this, op_errno, out);
+    local = AFR_FRAME_INIT(frame, op_errno);
+    if (!local)
+        goto out;
+
+    local->cont.writev.vector = iov_dup(vector, count);
+    if (!local->cont.writev.vector)
+        goto out;
+    local->cont.writev.count = count;
+    local->cont.writev.offset = offset;
+    local->cont.writev.flags = flags;
+    local->cont.writev.iobref = iobref_ref(iobref);
+
+    if (xdata)
+        local->xdata_req = dict_copy_with_ref(xdata, NULL);
+    else
+        local->xdata_req = dict_new();
+
+    if (!local->xdata_req)
+        goto out;
+
+    local->fd = fd_ref(fd);
+    ret = afr_set_inode_local(this, local, fd->inode);
+    if (ret)
+        goto out;
+
+    if (dict_set_uint32(local->xdata_req, GLUSTERFS_ACTIVE_FD_COUNT, 4)) {
+        op_errno = ENOMEM;
+        goto out;
+    }
+
+    if (dict_set_str_sizen(local->xdata_req, GLUSTERFS_INODELK_DOM_COUNT,
+                           this->name)) {
+        op_errno = ENOMEM;
+        goto out;
+    }
+
+    if (dict_set_uint32(local->xdata_req, GLUSTERFS_WRITE_IS_APPEND, 4)) {
+        op_errno = ENOMEM;
+        goto out;
+    }
+
+    /* Set append_write to be true speculatively. If on any
+       server it turns not be true, we unset it in the
+       callback.
+    */
+    local->append_write = _gf_true;
+
+    /* detect here, but set it in writev_wind_cbk *after* the unstable
+       write is performed
+    */
+    local->stable_write = !!((fd->flags | flags) & (O_SYNC | O_DSYNC));
+
+    afr_fix_open(fd, this);
+
+    afr_do_writev(frame, this);
+
+    return 0;
 out:
-	AFR_STACK_UNWIND (writev, frame, -1, op_errno, NULL, NULL, NULL);
+    AFR_STACK_UNWIND(writev, frame, -1, op_errno, NULL, NULL, NULL);
 
-        return 0;
+    return 0;
 }
 
-
 /* }}} */
 
 /* {{{ truncate */
 
 int
-afr_truncate_unwind (call_frame_t *frame, xlator_t *this)
+afr_truncate_unwind(call_frame_t *frame, xlator_t *this)
 {
-        afr_local_t *   local = NULL;
-        call_frame_t   *main_frame = NULL;
+    afr_local_t *local = NULL;
+    call_frame_t *main_frame = NULL;
 
-        local = frame->local;
+    local = frame->local;
 
-	main_frame = afr_transaction_detach_fop_frame (frame);
-	if (!main_frame)
-		return 0;
-
-	AFR_STACK_UNWIND (truncate, main_frame, local->op_ret, local->op_errno,
-			  &local->cont.inode_wfop.prebuf,
-			  &local->cont.inode_wfop.postbuf, local->xdata_rsp);
+    main_frame = afr_transaction_detach_fop_frame(frame);
+    if (!main_frame)
         return 0;
-}
 
+    AFR_STACK_UNWIND(truncate, main_frame, local->op_ret, local->op_errno,
+                     &local->cont.inode_wfop.prebuf,
+                     &local->cont.inode_wfop.postbuf, local->xdata_rsp);
+    return 0;
+}
 
 int
-afr_truncate_wind_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                       int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
-                       struct iatt *postbuf, dict_t *xdata)
+afr_truncate_wind_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                      int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
+                      struct iatt *postbuf, dict_t *xdata)
 {
-	afr_local_t *local = NULL;
+    afr_local_t *local = NULL;
 
-	local = frame->local;
+    local = frame->local;
 
-	if (op_ret == 0 && prebuf->ia_size != postbuf->ia_size)
-		local->stable_write = _gf_false;
+    if (op_ret == 0 && prebuf->ia_size != postbuf->ia_size)
+        local->stable_write = _gf_false;
 
-	return __afr_inode_write_cbk (frame, cookie, this, op_ret, op_errno,
-				      prebuf, postbuf, xdata);
+    return __afr_inode_write_cbk(frame, cookie, this, op_ret, op_errno, prebuf,
+                                 postbuf, NULL, xdata);
 }
 
-
 int
-afr_truncate_wind (call_frame_t *frame, xlator_t *this, int subvol)
+afr_truncate_wind(call_frame_t *frame, xlator_t *this, int subvol)
 {
-        afr_local_t *local = NULL;
-        afr_private_t *priv = NULL;
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
 
-        local = frame->local;
-        priv = this->private;
+    local = frame->local;
+    priv = this->private;
 
-	STACK_WIND_COOKIE (frame, afr_truncate_wind_cbk, (void *) (long) subvol,
-			   priv->children[subvol],
-			   priv->children[subvol]->fops->truncate,
-			   &local->loc, local->cont.truncate.offset,
-			   local->xdata_req);
-        return 0;
+    STACK_WIND_COOKIE(frame, afr_truncate_wind_cbk, (void *)(long)subvol,
+                      priv->children[subvol],
+                      priv->children[subvol]->fops->truncate, &local->loc,
+                      local->cont.truncate.offset, local->xdata_req);
+    return 0;
 }
 
-
 int
-afr_truncate (call_frame_t *frame, xlator_t *this,
-              loc_t *loc, off_t offset, dict_t *xdata)
+afr_truncate(call_frame_t *frame, xlator_t *this, loc_t *loc, off_t offset,
+             dict_t *xdata)
 {
-        afr_local_t   * local = NULL;
-        call_frame_t   *transaction_frame = NULL;
-	int ret = -1;
-        int op_errno = ENOMEM;
+    afr_local_t *local = NULL;
+    call_frame_t *transaction_frame = NULL;
+    int ret = -1;
+    int op_errno = ENOMEM;
 
-        transaction_frame = copy_frame (frame);
-        if (!transaction_frame)
-                goto out;
+    transaction_frame = copy_frame(frame);
+    if (!transaction_frame)
+        goto out;
 
-	local = AFR_FRAME_INIT (transaction_frame, op_errno);
-	if (!local)
-		goto out;
+    local = AFR_FRAME_INIT(transaction_frame, op_errno);
+    if (!local)
+        goto out;
 
-        local->cont.truncate.offset  = offset;
-	if (xdata)
-		local->xdata_req = dict_copy_with_ref (xdata, NULL);
-	else
-		local->xdata_req = dict_new ();
+    local->cont.truncate.offset = offset;
+    if (xdata)
+        local->xdata_req = dict_copy_with_ref(xdata, NULL);
+    else
+        local->xdata_req = dict_new();
 
-	if (!local->xdata_req)
-		goto out;
+    if (!local->xdata_req)
+        goto out;
 
-	local->transaction.wind   = afr_truncate_wind;
-        local->transaction.fop    = __afr_txn_write_fop;
-        local->transaction.done   = __afr_txn_write_done;
-        local->transaction.unwind = afr_truncate_unwind;
+    local->transaction.wind = afr_truncate_wind;
+    local->transaction.unwind = afr_truncate_unwind;
 
-        loc_copy (&local->loc, loc);
-	local->inode = inode_ref (loc->inode);
+    loc_copy(&local->loc, loc);
+    ret = afr_set_inode_local(this, local, loc->inode);
+    if (ret)
+        goto out;
 
-        local->op = GF_FOP_TRUNCATE;
+    local->op = GF_FOP_TRUNCATE;
 
-        local->transaction.main_frame = frame;
-        local->transaction.start   = offset;
-        local->transaction.len     = 0;
+    local->transaction.main_frame = frame;
+    local->transaction.start = offset;
+    local->transaction.len = 0;
 
-	/* Set it true speculatively, will get reset in afr_truncate_wind_cbk
-	   if truncate was not a NOP */
-	local->stable_write = _gf_true;
+    /* Set it true speculatively, will get reset in afr_truncate_wind_cbk
+       if truncate was not a NOP */
+    local->stable_write = _gf_true;
 
-        ret = afr_transaction (transaction_frame, this, AFR_DATA_TRANSACTION);
-        if (ret < 0) {
-		op_errno = -ret;
-		goto out;
-        }
+    ret = afr_transaction(transaction_frame, this, AFR_DATA_TRANSACTION);
+    if (ret < 0) {
+        op_errno = -ret;
+        goto out;
+    }
 
-	return 0;
+    return 0;
 out:
-	if (transaction_frame)
-		AFR_STACK_DESTROY (transaction_frame);
+    if (transaction_frame)
+        AFR_STACK_DESTROY(transaction_frame);
 
-	AFR_STACK_UNWIND (truncate, frame, -1, op_errno, NULL, NULL, NULL);
-        return 0;
+    AFR_STACK_UNWIND(truncate, frame, -1, op_errno, NULL, NULL, NULL);
+    return 0;
 }
 
-
 /* }}} */
 
 /* {{{ ftruncate */
 
-
 int
-afr_ftruncate_unwind (call_frame_t *frame, xlator_t *this)
+afr_ftruncate_unwind(call_frame_t *frame, xlator_t *this)
 {
-        afr_local_t *   local = NULL;
-        call_frame_t   *main_frame = NULL;
-
-        local = frame->local;
+    afr_local_t *local = NULL;
+    call_frame_t *main_frame = NULL;
 
-	main_frame = afr_transaction_detach_fop_frame (frame);
-	if (!main_frame)
-		return 0;
+    local = frame->local;
 
-	AFR_STACK_UNWIND (ftruncate, main_frame, local->op_ret, local->op_errno,
-			  &local->cont.inode_wfop.prebuf,
-			  &local->cont.inode_wfop.postbuf, local->xdata_rsp);
+    main_frame = afr_transaction_detach_fop_frame(frame);
+    if (!main_frame)
         return 0;
-}
 
+    AFR_STACK_UNWIND(ftruncate, main_frame, local->op_ret, local->op_errno,
+                     &local->cont.inode_wfop.prebuf,
+                     &local->cont.inode_wfop.postbuf, local->xdata_rsp);
+    return 0;
+}
 
 int
-afr_ftruncate_wind_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                        int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
-                        struct iatt *postbuf, dict_t *xdata)
+afr_ftruncate_wind_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                       int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
+                       struct iatt *postbuf, dict_t *xdata)
 {
-	afr_local_t *local = NULL;
+    afr_local_t *local = NULL;
 
-	local = frame->local;
+    local = frame->local;
 
-	if (op_ret == 0 && prebuf->ia_size != postbuf->ia_size)
-		local->stable_write = _gf_false;
+    if (op_ret == 0 && prebuf->ia_size != postbuf->ia_size)
+        local->stable_write = _gf_false;
 
-	return __afr_inode_write_cbk (frame, cookie, this, op_ret, op_errno,
-				      prebuf, postbuf, xdata);
+    return __afr_inode_write_cbk(frame, cookie, this, op_ret, op_errno, prebuf,
+                                 postbuf, NULL, xdata);
 }
 
-
 int
-afr_ftruncate_wind (call_frame_t *frame, xlator_t *this, int subvol)
+afr_ftruncate_wind(call_frame_t *frame, xlator_t *this, int subvol)
 {
-        afr_local_t *local = NULL;
-        afr_private_t *priv = NULL;
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
 
-	local = frame->local;
-	priv = this->private;
+    local = frame->local;
+    priv = this->private;
 
-	STACK_WIND_COOKIE (frame, afr_ftruncate_wind_cbk, (void *) (long) subvol,
-			   priv->children[subvol],
-			   priv->children[subvol]->fops->ftruncate,
-			   local->fd, local->cont.ftruncate.offset,
-			   local->xdata_req);
-        return 0;
+    STACK_WIND_COOKIE(frame, afr_ftruncate_wind_cbk, (void *)(long)subvol,
+                      priv->children[subvol],
+                      priv->children[subvol]->fops->ftruncate, local->fd,
+                      local->cont.ftruncate.offset, local->xdata_req);
+    return 0;
 }
 
-
 int
-afr_ftruncate (call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
-	       dict_t *xdata)
+afr_ftruncate(call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
+              dict_t *xdata)
 {
-        afr_local_t *local = NULL;
-	call_frame_t *transaction_frame = NULL;
-	int ret = -1;
-        int op_errno = ENOMEM;
+    afr_local_t *local = NULL;
+    call_frame_t *transaction_frame = NULL;
+    int ret = -1;
+    int op_errno = ENOMEM;
 
-	transaction_frame = copy_frame (frame);
-	if (!transaction_frame)
-		goto out;
+    AFR_ERROR_OUT_IF_FDCTX_INVALID(fd, this, op_errno, out);
+    transaction_frame = copy_frame(frame);
+    if (!transaction_frame)
+        goto out;
 
-        local = AFR_FRAME_INIT (transaction_frame, op_errno);
-        if (!local)
-		goto out;
+    local = AFR_FRAME_INIT(transaction_frame, op_errno);
+    if (!local)
+        goto out;
 
-        local->cont.ftruncate.offset  = offset;
-	if (xdata)
-		local->xdata_req = dict_copy_with_ref (xdata, NULL);
-	else
-		local->xdata_req = dict_new ();
+    local->cont.ftruncate.offset = offset;
+    if (xdata)
+        local->xdata_req = dict_copy_with_ref(xdata, NULL);
+    else
+        local->xdata_req = dict_new();
 
-	if (!local->xdata_req)
-		goto out;
+    if (!local->xdata_req)
+        goto out;
 
-        local->fd = fd_ref (fd);
-	local->inode = inode_ref (fd->inode);
+    local->fd = fd_ref(fd);
+    ret = afr_set_inode_local(this, local, fd->inode);
+    if (ret)
+        goto out;
 
-        local->op = GF_FOP_FTRUNCATE;
+    local->op = GF_FOP_FTRUNCATE;
 
-	local->transaction.wind   = afr_ftruncate_wind;
-        local->transaction.fop    = __afr_txn_write_fop;
-        local->transaction.done   = __afr_txn_write_done;
-        local->transaction.unwind = afr_ftruncate_unwind;
+    local->transaction.wind = afr_ftruncate_wind;
+    local->transaction.unwind = afr_ftruncate_unwind;
 
-        local->transaction.main_frame = frame;
+    local->transaction.main_frame = frame;
 
-        local->transaction.start   = local->cont.ftruncate.offset;
-        local->transaction.len     = 0;
+    local->transaction.start = local->cont.ftruncate.offset;
+    local->transaction.len = 0;
 
-        afr_fix_open (fd, this);
+    afr_fix_open(fd, this);
 
-	/* Set it true speculatively, will get reset in afr_ftruncate_wind_cbk
-	   if truncate was not a NOP */
-	local->stable_write = _gf_true;
+    /* Set it true speculatively, will get reset in afr_ftruncate_wind_cbk
+       if truncate was not a NOP */
+    local->stable_write = _gf_true;
 
-        ret = afr_transaction (transaction_frame, this, AFR_DATA_TRANSACTION);
-        if (ret < 0) {
-		op_errno = -ret;
-		goto out;
-        }
+    ret = afr_transaction(transaction_frame, this, AFR_DATA_TRANSACTION);
+    if (ret < 0) {
+        op_errno = -ret;
+        goto out;
+    }
 
-	return 0;
+    return 0;
 out:
-	AFR_STACK_UNWIND (ftruncate, frame, -1, op_errno, NULL, NULL, NULL);
+    AFR_STACK_UNWIND(ftruncate, frame, -1, op_errno, NULL, NULL, NULL);
 
-        return 0;
+    return 0;
 }
 
 /* }}} */
@@ -703,1133 +788,1778 @@ out:
 /* {{{ setattr */
 
 int
-afr_setattr_unwind (call_frame_t *frame, xlator_t *this)
+afr_setattr_unwind(call_frame_t *frame, xlator_t *this)
 {
-        afr_local_t *local = NULL;
-        call_frame_t *main_frame = NULL;
-
-        local = frame->local;
+    afr_local_t *local = NULL;
+    call_frame_t *main_frame = NULL;
 
-	main_frame = afr_transaction_detach_fop_frame (frame);
-	if (!main_frame)
-		return 0;
+    local = frame->local;
 
-	AFR_STACK_UNWIND (setattr, main_frame, local->op_ret, local->op_errno,
-			  &local->cont.inode_wfop.prebuf,
-			  &local->cont.inode_wfop.postbuf,
-			  local->xdata_rsp);
+    main_frame = afr_transaction_detach_fop_frame(frame);
+    if (!main_frame)
         return 0;
+
+    AFR_STACK_UNWIND(setattr, main_frame, local->op_ret, local->op_errno,
+                     &local->cont.inode_wfop.prebuf,
+                     &local->cont.inode_wfop.postbuf, local->xdata_rsp);
+    return 0;
 }
 
+int
+afr_setattr_wind_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                     int op_ret, int op_errno, struct iatt *preop,
+                     struct iatt *postop, dict_t *xdata)
+{
+    return __afr_inode_write_cbk(frame, cookie, this, op_ret, op_errno, preop,
+                                 postop, NULL, xdata);
+}
+
+int
+afr_setattr_wind(call_frame_t *frame, xlator_t *this, int subvol)
+{
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
+
+    local = frame->local;
+    priv = this->private;
+
+    STACK_WIND_COOKIE(frame, afr_setattr_wind_cbk, (void *)(long)subvol,
+                      priv->children[subvol],
+                      priv->children[subvol]->fops->setattr, &local->loc,
+                      &local->cont.setattr.in_buf, local->cont.setattr.valid,
+                      local->xdata_req);
+    return 0;
+}
 
 int
-afr_setattr_wind_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                      int op_ret, int op_errno,
-                      struct iatt *preop, struct iatt *postop, dict_t *xdata)
+afr_setattr(call_frame_t *frame, xlator_t *this, loc_t *loc, struct iatt *buf,
+            int32_t valid, dict_t *xdata)
 {
-	return __afr_inode_write_cbk (frame, cookie, this, op_ret, op_errno,
-				      preop, postop, xdata);
+    afr_local_t *local = NULL;
+    call_frame_t *transaction_frame = NULL;
+    int ret = -1;
+    int op_errno = ENOMEM;
+
+    transaction_frame = copy_frame(frame);
+    if (!transaction_frame)
+        goto out;
+
+    local = AFR_FRAME_INIT(transaction_frame, op_errno);
+    if (!local)
+        goto out;
+
+    local->cont.setattr.in_buf = *buf;
+    local->cont.setattr.valid = valid;
+    if (xdata)
+        local->xdata_req = dict_copy_with_ref(xdata, NULL);
+    else
+        local->xdata_req = dict_new();
+
+    if (!local->xdata_req)
+        goto out;
+
+    local->transaction.wind = afr_setattr_wind;
+    local->transaction.unwind = afr_setattr_unwind;
+
+    loc_copy(&local->loc, loc);
+    ret = afr_set_inode_local(this, local, loc->inode);
+    if (ret)
+        goto out;
+
+    local->op = GF_FOP_SETATTR;
+
+    local->transaction.main_frame = frame;
+    local->transaction.start = LLONG_MAX - 1;
+    local->transaction.len = 0;
+
+    ret = afr_transaction(transaction_frame, this, AFR_METADATA_TRANSACTION);
+    if (ret < 0) {
+        op_errno = -ret;
+        goto out;
+    }
+
+    return 0;
+out:
+    if (transaction_frame)
+        AFR_STACK_DESTROY(transaction_frame);
+
+    AFR_STACK_UNWIND(setattr, frame, -1, op_errno, NULL, NULL, NULL);
+    return 0;
 }
 
+/* {{{ fsetattr */
 
 int
-afr_setattr_wind (call_frame_t *frame, xlator_t *this, int subvol)
+afr_fsetattr_unwind(call_frame_t *frame, xlator_t *this)
 {
-        afr_local_t *local = NULL;
-        afr_private_t *priv = NULL;
+    afr_local_t *local = NULL;
+    call_frame_t *main_frame = NULL;
 
-        local = frame->local;
-        priv = this->private;
+    local = frame->local;
 
-	STACK_WIND_COOKIE (frame, afr_setattr_wind_cbk, (void *) (long) subvol,
-			   priv->children[subvol],
-			   priv->children[subvol]->fops->setattr,
-			   &local->loc, &local->cont.setattr.in_buf,
-			   local->cont.setattr.valid, local->xdata_req);
+    main_frame = afr_transaction_detach_fop_frame(frame);
+    if (!main_frame)
         return 0;
+
+    AFR_STACK_UNWIND(fsetattr, main_frame, local->op_ret, local->op_errno,
+                     &local->cont.inode_wfop.prebuf,
+                     &local->cont.inode_wfop.postbuf, local->xdata_rsp);
+    return 0;
 }
 
+int
+afr_fsetattr_wind_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                      int32_t op_ret, int32_t op_errno, struct iatt *preop,
+                      struct iatt *postop, dict_t *xdata)
+{
+    return __afr_inode_write_cbk(frame, cookie, this, op_ret, op_errno, preop,
+                                 postop, NULL, xdata);
+}
 
 int
-afr_setattr (call_frame_t *frame, xlator_t *this, loc_t *loc, struct iatt *buf,
-	     int32_t valid, dict_t *xdata)
+afr_fsetattr_wind(call_frame_t *frame, xlator_t *this, int subvol)
 {
-        afr_local_t *local = NULL;
-        call_frame_t *transaction_frame = NULL;
-        int ret = -1;
-        int op_errno = ENOMEM;
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
+
+    local = frame->local;
+    priv = this->private;
+
+    STACK_WIND_COOKIE(frame, afr_fsetattr_wind_cbk, (void *)(long)subvol,
+                      priv->children[subvol],
+                      priv->children[subvol]->fops->fsetattr, local->fd,
+                      &local->cont.fsetattr.in_buf, local->cont.fsetattr.valid,
+                      local->xdata_req);
+    return 0;
+}
 
-        transaction_frame = copy_frame (frame);
-        if (!transaction_frame)
-                goto out;
+int
+afr_fsetattr(call_frame_t *frame, xlator_t *this, fd_t *fd, struct iatt *buf,
+             int32_t valid, dict_t *xdata)
+{
+    afr_local_t *local = NULL;
+    call_frame_t *transaction_frame = NULL;
+    int ret = -1;
+    int op_errno = ENOMEM;
 
-	local = AFR_FRAME_INIT (transaction_frame, op_errno);
-	if (!local)
-		goto out;
+    AFR_ERROR_OUT_IF_FDCTX_INVALID(fd, this, op_errno, out);
+    transaction_frame = copy_frame(frame);
+    if (!transaction_frame)
+        goto out;
 
-        local->cont.setattr.in_buf = *buf;
-        local->cont.setattr.valid  = valid;
-	if (xdata)
-		local->xdata_req = dict_copy_with_ref (xdata, NULL);
-	else
-		local->xdata_req = dict_new ();
+    local = AFR_FRAME_INIT(transaction_frame, op_errno);
+    if (!local)
+        goto out;
 
-	if (!local->xdata_req)
-		goto out;
+    local->cont.fsetattr.in_buf = *buf;
+    local->cont.fsetattr.valid = valid;
+    if (xdata)
+        local->xdata_req = dict_copy_with_ref(xdata, NULL);
+    else
+        local->xdata_req = dict_new();
 
-        local->transaction.wind   = afr_setattr_wind;
-        local->transaction.fop    = __afr_txn_write_fop;
-        local->transaction.done   = __afr_txn_write_done;
-        local->transaction.unwind = afr_setattr_unwind;
+    if (!local->xdata_req)
+        goto out;
 
-        loc_copy (&local->loc, loc);
-	local->inode = inode_ref (loc->inode);
+    local->transaction.wind = afr_fsetattr_wind;
+    local->transaction.unwind = afr_fsetattr_unwind;
 
-	local->op = GF_FOP_SETATTR;
+    local->fd = fd_ref(fd);
+    ret = afr_set_inode_local(this, local, fd->inode);
+    if (ret)
+        goto out;
 
-        local->transaction.main_frame = frame;
-        local->transaction.start   = LLONG_MAX - 1;
-        local->transaction.len     = 0;
+    local->op = GF_FOP_FSETATTR;
 
-        ret = afr_transaction (transaction_frame, this, AFR_METADATA_TRANSACTION);
-        if (ret < 0) {
-		op_errno = -ret;
-		goto out;
-        }
+    afr_fix_open(fd, this);
 
-	return 0;
+    local->transaction.main_frame = frame;
+    local->transaction.start = LLONG_MAX - 1;
+    local->transaction.len = 0;
+
+    ret = afr_transaction(transaction_frame, this, AFR_METADATA_TRANSACTION);
+    if (ret < 0) {
+        op_errno = -ret;
+        goto out;
+    }
+
+    return 0;
 out:
-	if (transaction_frame)
-		AFR_STACK_DESTROY (transaction_frame);
+    if (transaction_frame)
+        AFR_STACK_DESTROY(transaction_frame);
 
-	AFR_STACK_UNWIND (setattr, frame, -1, op_errno, NULL, NULL, NULL);
-        return 0;
+    AFR_STACK_UNWIND(fsetattr, frame, -1, op_errno, NULL, NULL, NULL);
+    return 0;
 }
 
-/* {{{ fsetattr */
+/* {{{ setxattr */
 
 int
-afr_fsetattr_unwind (call_frame_t *frame, xlator_t *this)
+afr_setxattr_unwind(call_frame_t *frame, xlator_t *this)
 {
-        afr_local_t *   local = NULL;
-        call_frame_t   *main_frame = NULL;
-
-        local = frame->local;
+    afr_local_t *local = NULL;
+    call_frame_t *main_frame = NULL;
 
-	main_frame = afr_transaction_detach_fop_frame (frame);
-        if (!main_frame)
-		return 0;
+    local = frame->local;
 
-	AFR_STACK_UNWIND (fsetattr, main_frame, local->op_ret, local->op_errno,
-			  &local->cont.inode_wfop.prebuf,
-			  &local->cont.inode_wfop.postbuf, local->xdata_rsp);
+    main_frame = afr_transaction_detach_fop_frame(frame);
+    if (!main_frame)
         return 0;
-}
 
+    AFR_STACK_UNWIND(setxattr, main_frame, local->op_ret, local->op_errno,
+                     local->xdata_rsp);
+    return 0;
+}
 
 int
-afr_fsetattr_wind_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                       int32_t op_ret, int32_t op_errno,
-                       struct iatt *preop, struct iatt *postop, dict_t *xdata)
+afr_setxattr_wind_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                      int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
-	return __afr_inode_write_cbk (frame, cookie, this, op_ret, op_errno,
-				      preop, postop, xdata);
+    return __afr_inode_write_cbk(frame, cookie, this, op_ret, op_errno, NULL,
+                                 NULL, NULL, xdata);
 }
 
-
 int
-afr_fsetattr_wind (call_frame_t *frame, xlator_t *this, int subvol)
+afr_setxattr_wind(call_frame_t *frame, xlator_t *this, int subvol)
 {
-        afr_local_t *local = NULL;
-        afr_private_t *priv = NULL;
-
-        local = frame->local;
-        priv = this->private;
-
-	STACK_WIND_COOKIE (frame, afr_fsetattr_wind_cbk, (void *) (long) subvol,
-			   priv->children[subvol],
-			   priv->children[subvol]->fops->fsetattr,
-			   local->fd, &local->cont.fsetattr.in_buf,
-			   local->cont.fsetattr.valid, local->xdata_req);
-        return 0;
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
+
+    local = frame->local;
+    priv = this->private;
+
+    STACK_WIND_COOKIE(frame, afr_setxattr_wind_cbk, (void *)(long)subvol,
+                      priv->children[subvol],
+                      priv->children[subvol]->fops->setxattr, &local->loc,
+                      local->cont.setxattr.dict, local->cont.setxattr.flags,
+                      local->xdata_req);
+    return 0;
 }
 
-
 int
-afr_fsetattr (call_frame_t *frame, xlator_t *this,
-              fd_t *fd, struct iatt *buf, int32_t valid, dict_t *xdata)
+afr_emptyb_set_pending_changelog_cbk(call_frame_t *frame, void *cookie,
+                                     xlator_t *this, int op_ret, int op_errno,
+                                     dict_t *xattr, dict_t *xdata)
+
 {
-        afr_local_t *local = NULL;
-        call_frame_t *transaction_frame = NULL;
-        int ret = -1;
-        int op_errno = ENOMEM;
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
+    int i, ret = 0;
+    char *op_type = NULL;
 
-        transaction_frame = copy_frame (frame);
-        if (!transaction_frame)
-                goto out;
+    local = frame->local;
+    priv = this->private;
+    i = (long)cookie;
 
-	local = AFR_FRAME_INIT (transaction_frame, op_errno);
-	if (!local)
-		goto out;
+    local->replies[i].valid = 1;
+    local->replies[i].op_ret = op_ret;
+    local->replies[i].op_errno = op_errno;
 
-        local->cont.fsetattr.in_buf = *buf;
-        local->cont.fsetattr.valid  = valid;
-	if (xdata)
-		local->xdata_req = dict_copy_with_ref (xdata, NULL);
-	else
-		local->xdata_req = dict_new ();
+    ret = dict_get_str_sizen(local->xdata_req, "replicate-brick-op", &op_type);
+    if (ret)
+        goto out;
 
-	if (!local->xdata_req)
-		goto out;
+    gf_smsg(this->name, op_ret ? GF_LOG_ERROR : GF_LOG_INFO,
+            op_ret ? op_errno : 0, AFR_MSG_SET_PEND_XATTR, "name=%s",
+            priv->children[i]->name, "op_ret=%s",
+            op_ret ? "failed" : "succeeded", NULL);
 
-        local->transaction.wind   = afr_fsetattr_wind;
-        local->transaction.fop    = __afr_txn_write_fop;
-        local->transaction.done   = __afr_txn_write_done;
-        local->transaction.unwind = afr_fsetattr_unwind;
+out:
+    syncbarrier_wake(&local->barrier);
+    return 0;
+}
 
-        local->fd                 = fd_ref (fd);
-	local->inode = inode_ref (fd->inode);
+int
+afr_emptyb_set_pending_changelog(call_frame_t *frame, xlator_t *this,
+                                 unsigned char *locked_nodes)
+{
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
+    int ret = 0, i = 0;
 
-	local->op = GF_FOP_FSETATTR;
+    local = frame->local;
+    priv = this->private;
 
-        afr_fix_open (fd, this);
+    AFR_ONLIST(locked_nodes, frame, afr_emptyb_set_pending_changelog_cbk,
+               xattrop, &local->loc, GF_XATTROP_ADD_ARRAY, local->xattr_req,
+               NULL);
 
-        local->transaction.main_frame = frame;
-        local->transaction.start   = LLONG_MAX - 1;
-        local->transaction.len     = 0;
+    /* It is sufficient if xattrop was successful on one child */
+    for (i = 0; i < priv->child_count; i++) {
+        if (!local->replies[i].valid)
+            continue;
 
-        ret = afr_transaction (transaction_frame, this, AFR_METADATA_TRANSACTION);
-        if (ret < 0) {
-		op_errno = -ret;
-		goto out;
+        if (local->replies[i].op_ret == 0) {
+            ret = 0;
+            goto out;
+        } else {
+            ret = afr_higher_errno(ret, local->replies[i].op_errno);
         }
-
-	return 0;
+    }
 out:
-	if (transaction_frame)
-		AFR_STACK_DESTROY (transaction_frame);
-
-	AFR_STACK_UNWIND (fsetattr, frame, -1, op_errno, NULL, NULL, NULL);
-        return 0;
+    return -ret;
 }
 
+static int
+_afr_handle_empty_brick_type(xlator_t *this, call_frame_t *frame, loc_t *loc,
+                             int empty_index, afr_transaction_type type,
+                             char *op_type, const int op_type_len)
+{
+    int count = 0;
+    int ret = -ENOMEM;
+    int idx = -1;
+    int d_idx = -1;
+    unsigned char *locked_nodes = NULL;
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
+
+    priv = this->private;
+    local = frame->local;
+
+    locked_nodes = alloca0(priv->child_count);
+
+    idx = afr_index_for_transaction_type(type);
+    d_idx = afr_index_for_transaction_type(AFR_DATA_TRANSACTION);
+
+    local->pending = afr_matrix_create(priv->child_count, AFR_NUM_CHANGE_LOGS);
+    if (!local->pending)
+        goto out;
+
+    local->pending[empty_index][idx] = hton32(1);
+
+    if ((priv->esh_granular) && (type == AFR_ENTRY_TRANSACTION))
+        local->pending[empty_index][d_idx] = hton32(1);
+
+    local->xdata_req = dict_new();
+    if (!local->xdata_req)
+        goto out;
+
+    ret = dict_set_nstrn(local->xdata_req, "replicate-brick-op",
+                         SLEN("replicate-brick-op"), op_type, op_type_len);
+    if (ret)
+        goto out;
+
+    local->xattr_req = dict_new();
+    if (!local->xattr_req)
+        goto out;
+
+    ret = afr_set_pending_dict(priv, local->xattr_req, local->pending);
+    if (ret < 0)
+        goto out;
+
+    if (AFR_ENTRY_TRANSACTION == type) {
+        count = afr_selfheal_entrylk(frame, this, loc->inode, this->name, NULL,
+                                     locked_nodes);
+    } else {
+        count = afr_selfheal_inodelk(frame, this, loc->inode, this->name,
+                                     LLONG_MAX - 1, 0, locked_nodes);
+    }
+
+    if (!count) {
+        gf_smsg(this->name, GF_LOG_ERROR, EAGAIN, AFR_MSG_REPLACE_BRICK_STATUS,
+                NULL);
+        ret = -EAGAIN;
+        goto unlock;
+    }
+
+    ret = afr_emptyb_set_pending_changelog(frame, this, locked_nodes);
+    if (ret)
+        goto unlock;
+    ret = 0;
+unlock:
+    if (AFR_ENTRY_TRANSACTION == type) {
+        afr_selfheal_unentrylk(frame, this, loc->inode, this->name, NULL,
+                               locked_nodes, NULL);
+    } else {
+        afr_selfheal_uninodelk(frame, this, loc->inode, this->name,
+                               LLONG_MAX - 1, 0, locked_nodes);
+    }
+out:
+    return ret;
+}
 
-/* {{{ setxattr */
+void
+afr_brick_args_cleanup(void *opaque)
+{
+    afr_empty_brick_args_t *data = NULL;
 
+    data = opaque;
+    loc_wipe(&data->loc);
+    GF_FREE(data);
+}
 
 int
-afr_setxattr_unwind (call_frame_t *frame, xlator_t *this)
+_afr_handle_empty_brick_cbk(int ret, call_frame_t *frame, void *opaque)
 {
-        afr_local_t *   local = NULL;
-        call_frame_t   *main_frame = NULL;
-
-        local = frame->local;
-
-	main_frame = afr_transaction_detach_fop_frame (frame);
-	if (!main_frame)
-		return 0;
-
-	AFR_STACK_UNWIND (setxattr, main_frame, local->op_ret, local->op_errno,
-			  local->xdata_rsp);
-        return 0;
+    afr_brick_args_cleanup(opaque);
+    return 0;
 }
 
-
 int
-afr_setxattr_wind_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                       int32_t op_ret, int32_t op_errno, dict_t *xdata)
+_afr_handle_empty_brick(void *opaque)
 {
-        return __afr_inode_write_cbk (frame, cookie, this, op_ret, op_errno,
-				      NULL, NULL, xdata);
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
+    int empty_index = -1;
+    int ret = -1;
+    int op_errno = ENOMEM;
+    call_frame_t *frame = NULL;
+    xlator_t *this = NULL;
+    char *op_type = NULL;
+    int op_type_len = 0;
+    afr_empty_brick_args_t *data = NULL;
+    call_frame_t *op_frame = NULL;
+
+    data = opaque;
+    frame = data->frame;
+    empty_index = data->empty_index;
+    if (!data->op_type)
+        goto out;
+
+    op_frame = copy_frame(frame);
+    if (!op_frame) {
+        ret = -1;
+        op_errno = ENOMEM;
+        goto out;
+    }
+
+    op_type = data->op_type;
+    op_type_len = strlen(op_type);
+    this = op_frame->this;
+    priv = this->private;
+
+    afr_set_lk_owner(op_frame, this, op_frame->root);
+    local = AFR_FRAME_INIT(op_frame, op_errno);
+    if (!local)
+        goto out;
+
+    loc_copy(&local->loc, &data->loc);
+
+    gf_smsg(this->name, GF_LOG_INFO, 0, AFR_MSG_NEW_BRICK, "name=%s",
+            priv->children[empty_index]->name, NULL);
+
+    ret = _afr_handle_empty_brick_type(this, op_frame, &local->loc, empty_index,
+                                       AFR_METADATA_TRANSACTION, op_type,
+                                       op_type_len);
+    if (ret) {
+        op_errno = -ret;
+        ret = -1;
+        goto out;
+    }
+
+    dict_unref(local->xdata_req);
+    dict_unref(local->xattr_req);
+    afr_matrix_cleanup(local->pending, priv->child_count);
+    local->pending = NULL;
+    local->xattr_req = NULL;
+    local->xdata_req = NULL;
+
+    ret = _afr_handle_empty_brick_type(this, op_frame, &local->loc, empty_index,
+                                       AFR_ENTRY_TRANSACTION, op_type,
+                                       op_type_len);
+    if (ret) {
+        op_errno = -ret;
+        ret = -1;
+        goto out;
+    }
+    ret = 0;
+out:
+    if (op_frame) {
+        AFR_STACK_DESTROY(op_frame);
+    }
+    AFR_STACK_UNWIND(setxattr, frame, ret, op_errno, NULL);
+    return 0;
 }
 
-
 int
-afr_setxattr_wind (call_frame_t *frame, xlator_t *this, int subvol)
+afr_split_brain_resolve_do(call_frame_t *frame, xlator_t *this, loc_t *loc,
+                           char *data)
 {
-        afr_local_t      *local         = NULL;
-        afr_private_t    *priv          = NULL;
-
-        local = frame->local;
-        priv = this->private;
-
-	STACK_WIND_COOKIE (frame, afr_setxattr_wind_cbk, (void *) (long) subvol,
-			   priv->children[subvol],
-			   priv->children[subvol]->fops->setxattr,
-			   &local->loc, local->cont.setxattr.dict,
-			   local->cont.setxattr.flags, local->xdata_req);
-        return 0;
+    afr_local_t *local = NULL;
+    int ret = -1;
+    int op_errno = EINVAL;
+
+    local = frame->local;
+    local->xdata_req = dict_new();
+
+    if (!local->xdata_req) {
+        op_errno = ENOMEM;
+        goto out;
+    }
+
+    ret = dict_set_int32_sizen(local->xdata_req, "heal-op",
+                               GF_SHD_OP_SBRAIN_HEAL_FROM_BRICK);
+    if (ret) {
+        op_errno = -ret;
+        ret = -1;
+        goto out;
+    }
+    ret = dict_set_str_sizen(local->xdata_req, "child-name", data);
+    if (ret) {
+        op_errno = -ret;
+        ret = -1;
+        goto out;
+    }
+    /* set spb choice to -1 whether heal succeeds or not:
+     * If heal succeeds : spb-choice should be set to -1 as
+     *                    it is no longer valid; file is not
+     *                    in split-brain anymore.
+     * If heal doesn't succeed:
+     *                    spb-choice should be set to -1
+     *                    otherwise reads will be served
+     *                    from spb-choice which is misleading.
+     */
+    ret = afr_inode_split_brain_choice_set(loc->inode, this, -1);
+    if (ret)
+        gf_smsg(this->name, GF_LOG_WARNING, 0, AFR_MSG_SPLIT_BRAIN_SET_FAILED,
+                NULL);
+    afr_heal_splitbrain_file(frame, this, loc);
+    ret = 0;
+out:
+    if (ret < 0)
+        AFR_STACK_UNWIND(setxattr, frame, -1, op_errno, NULL);
+    return 0;
 }
 
 int
-afr_split_brain_resolve_do (call_frame_t *frame, xlator_t *this, loc_t *loc,
-                            char *data)
+afr_get_split_brain_child_index(xlator_t *this, void *value, size_t len)
 {
-        afr_local_t    *local             = NULL;
-        int     ret                       = -1;
-        int     op_errno                  = EINVAL;
+    int spb_child_index = -1;
+    char *spb_child_str = NULL;
 
-        local = AFR_FRAME_INIT (frame, op_errno);
-        if (!local)
-                goto out;
+    spb_child_str = alloca0(len + 1);
+    memcpy(spb_child_str, value, len);
+
+    if (!strcmp(spb_child_str, "none"))
+        return -2;
 
-        local->op = GF_FOP_SETXATTR;
+    spb_child_index = afr_get_child_index_from_name(this, spb_child_str);
+    if (spb_child_index < 0) {
+        gf_smsg(this->name, GF_LOG_ERROR, 0, AFR_MSG_INVALID_SUBVOL,
+                "subvol=%s", spb_child_str, NULL);
+    }
+    return spb_child_index;
+}
 
-        local->xdata_req = dict_new ();
+int
+afr_can_set_split_brain_choice(void *opaque)
+{
+    afr_spbc_timeout_t *data = opaque;
+    call_frame_t *frame = NULL;
+    xlator_t *this = NULL;
+    loc_t *loc = NULL;
+    int ret = -1;
+
+    frame = data->frame;
+    loc = data->loc;
+    this = frame->this;
+
+    ret = afr_is_split_brain(frame, this, loc->inode, loc->gfid, &data->d_spb,
+                             &data->m_spb);
+
+    if (ret)
+        gf_smsg(this->name, GF_LOG_ERROR, 0,
+                AFR_MSG_SPLIT_BRAIN_DETERMINE_FAILED, "gfid=%s",
+                uuid_utoa(loc->gfid), NULL);
+    return ret;
+}
 
-        if (!local->xdata_req) {
-                op_errno = ENOMEM;
+int
+afr_handle_split_brain_commands(xlator_t *this, call_frame_t *frame, loc_t *loc,
+                                dict_t *dict)
+{
+    void *choice_value = NULL;
+    void *resolve_value = NULL;
+    afr_private_t *priv = NULL;
+    afr_local_t *local = NULL;
+    afr_spbc_timeout_t *data = NULL;
+    int len = 0;
+    int spb_child_index = -1;
+    int ret = -1;
+    int op_errno = EINVAL;
+
+    priv = this->private;
+
+    ret = dict_get_ptr_and_len(dict, GF_AFR_SBRAIN_CHOICE, &choice_value, &len);
+    ret = dict_get_ptr_and_len(dict, GF_AFR_SBRAIN_RESOLVE, &resolve_value,
+                               &len);
+    if (!choice_value && !resolve_value) {
+        ret = -1;
+        goto out;
+    }
+
+    local = AFR_FRAME_INIT(frame, op_errno);
+    if (!local) {
+        ret = 1;
+        goto out;
+    }
+
+    local->op = GF_FOP_SETXATTR;
+
+    if (choice_value) {
+        spb_child_index = afr_get_split_brain_child_index(this, choice_value,
+                                                          len);
+        if (spb_child_index < 0) {
+            /* Case where value was "none" */
+            if (spb_child_index == -2)
+                spb_child_index = -1;
+            else {
+                ret = 1;
+                op_errno = EINVAL;
                 goto out;
+            }
         }
 
-        ret = dict_set_int32 (local->xdata_req, "heal-op",
-                              GF_SHD_OP_SBRAIN_HEAL_FROM_BRICK);
-        if (ret) {
-                op_errno = -ret;
-                ret = -1;
-                goto out;
+        data = GF_CALLOC(1, sizeof(*data), gf_afr_mt_spbc_timeout_t);
+        if (!data) {
+            ret = 1;
+            goto out;
         }
-        ret = dict_set_str (local->xdata_req, "child-name", data);
+        data->spb_child_index = spb_child_index;
+        data->frame = frame;
+        loc_copy(&local->loc, loc);
+        data->loc = &local->loc;
+        ret = synctask_new(this->ctx->env, afr_can_set_split_brain_choice,
+                           afr_set_split_brain_choice, NULL, data);
         if (ret) {
-                op_errno = -ret;
-                ret = -1;
-                goto out;
+            gf_smsg(this->name, GF_LOG_ERROR, 0, AFR_MSG_SPLIT_BRAIN_STATUS,
+                    "name=%s", loc->name, NULL);
+            ret = 1;
+            op_errno = ENOMEM;
+            goto out;
+        }
+        ret = 0;
+        goto out;
+    }
+
+    if (resolve_value) {
+        spb_child_index = afr_get_split_brain_child_index(this, resolve_value,
+                                                          len);
+        if (spb_child_index < 0) {
+            ret = 1;
+            goto out;
         }
-        afr_heal_splitbrain_file (frame, this, loc);
+
+        afr_split_brain_resolve_do(frame, this, loc,
+                                   priv->children[spb_child_index]->name);
+        ret = 0;
+    }
 out:
-        if (ret < 0)
-                AFR_STACK_UNWIND (setxattr, frame, -1, op_errno, NULL);
-        return 0;
+    /* key was correct but value was invalid when ret == 1 */
+    if (ret == 1) {
+        AFR_STACK_UNWIND(setxattr, frame, -1, op_errno, NULL);
+        if (data)
+            GF_FREE(data);
+        ret = 0;
+    }
+    return ret;
 }
 
 int
-afr_set_split_brain_choice (call_frame_t *frame, xlator_t *this, loc_t *loc,
-                            int spb_choice)
+afr_handle_spb_choice_timeout(xlator_t *this, call_frame_t *frame, dict_t *dict)
 {
-        int     ret       = -1;
-        int     op_errno  = ENOMEM;
-        afr_private_t *priv = NULL;
+    int ret = -1;
+    int op_errno = 0;
+    uint64_t timeout = 0;
+    afr_private_t *priv = NULL;
 
-        priv = this->private;
+    priv = this->private;
 
-        ret = afr_inode_split_brain_choice_set (loc->inode, this, spb_choice);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "Failed to set"
-                        "split-brain choice as %s for %s",
-                        priv->children[spb_choice]->name,
-                        loc->name);
-        }
-        inode_invalidate (loc->inode);
-        AFR_STACK_UNWIND (setxattr, frame, ret, op_errno, NULL);
-        return ret;
+    ret = dict_get_uint64(dict, GF_AFR_SPB_CHOICE_TIMEOUT, &timeout);
+    if (!ret) {
+        priv->spb_choice_timeout = timeout * 60;
+        AFR_STACK_UNWIND(setxattr, frame, ret, op_errno, NULL);
+    }
+
+    return ret;
 }
 
 int
-afr_get_split_brain_child_index (xlator_t *this, void *value, size_t len)
+afr_handle_empty_brick(xlator_t *this, call_frame_t *frame, loc_t *loc,
+                       dict_t *dict)
 {
-        int             spb_child_index   = -1;
-        char           *spb_child_str     = NULL;
+    int ret = -1;
+    int ab_ret = -1;
+    int empty_index = -1;
+    int op_errno = EPERM;
+    char *empty_brick = NULL;
+    char *op_type = NULL;
+    afr_empty_brick_args_t *data = NULL;
+
+    ret = dict_get_str_sizen(dict, GF_AFR_REPLACE_BRICK, &empty_brick);
+    if (!ret)
+        op_type = GF_AFR_REPLACE_BRICK;
+
+    ab_ret = dict_get_str_sizen(dict, GF_AFR_ADD_BRICK, &empty_brick);
+    if (!ab_ret)
+        op_type = GF_AFR_ADD_BRICK;
+
+    if (ret && ab_ret)
+        goto out;
+
+    if (frame->root->pid != GF_CLIENT_PID_ADD_REPLICA_MOUNT) {
+        gf_smsg(this->name, GF_LOG_ERROR, EPERM, AFR_MSG_INTERNAL_ATTR,
+                "op_type=%s", op_type, NULL);
+        ret = 1;
+        goto out;
+    }
+    empty_index = afr_get_child_index_from_name(this, empty_brick);
+
+    if (empty_index < 0) {
+        /* Didn't belong to this replica pair
+         * Just do a no-op
+         */
+        AFR_STACK_UNWIND(setxattr, frame, 0, 0, NULL);
+        return 0;
+    } else {
+        data = GF_CALLOC(1, sizeof(*data), gf_afr_mt_empty_brick_t);
+        if (!data) {
+            ret = 1;
+            op_errno = ENOMEM;
+            goto out;
+        }
+        data->frame = frame;
+        loc_copy(&data->loc, loc);
+        data->empty_index = empty_index;
+        data->op_type = op_type;
+        ret = synctask_new(this->ctx->env, _afr_handle_empty_brick,
+                           _afr_handle_empty_brick_cbk, NULL, data);
+        if (ret) {
+            gf_smsg(this->name, GF_LOG_ERROR, 0, AFR_MSG_SPLIT_BRAIN_STATUS,
+                    NULL);
+            ret = 1;
+            op_errno = ENOMEM;
+            afr_brick_args_cleanup(data);
+            goto out;
+        }
+    }
+    ret = 0;
+out:
+    if (ret == 1) {
+        AFR_STACK_UNWIND(setxattr, frame, -1, op_errno, NULL);
+        ret = 0;
+    }
+    return ret;
+}
 
-        spb_child_str =  alloca0 (len + 1);
-        memcpy (spb_child_str, value, len);
+static int
+afr_handle_special_xattr(xlator_t *this, call_frame_t *frame, loc_t *loc,
+                         dict_t *dict)
+{
+    int ret = -1;
 
-        if (!strcmp (spb_child_str, "none"))
-                return -2;
+    ret = afr_handle_split_brain_commands(this, frame, loc, dict);
+    if (ret == 0)
+        goto out;
 
-        spb_child_index = afr_get_child_index_from_name (this,
-                                                         spb_child_str);
-        if (spb_child_index < 0) {
-                gf_log (this->name, GF_LOG_ERROR, "Invalid subvol: %s",
-                        spb_child_str);
-        }
-        return spb_child_index;
-}
-
-int
-afr_handle_split_brain_commands (xlator_t *this, call_frame_t *frame,
-                                loc_t *loc, dict_t *dict)
-{
-        int             len               = 0;
-        void           *value             = NULL;
-        int             spb_child_index   = -1;
-        int             ret               = -1;
-        int             op_errno          = EINVAL;
-        afr_private_t  *priv              = NULL;
-
-        priv = this->private;
-
-        ret =  dict_get_ptr_and_len (dict, GF_AFR_SBRAIN_CHOICE, &value,
-                                     &len);
-        if (value) {
-                spb_child_index = afr_get_split_brain_child_index (this, value,
-                                                                   len);
-                if (spb_child_index < 0) {
-                        /* Case where value was "none" */
-                        if (spb_child_index == -2)
-                                spb_child_index = -1;
-                        else {
-                                ret = 1;
-                                goto out;
-                        }
-                }
-
-                afr_set_split_brain_choice (frame, this, loc,
-                                            spb_child_index);
-                ret = 0;
-                goto out;
-        }
+    ret = afr_handle_spb_choice_timeout(this, frame, dict);
+    if (ret == 0)
+        goto out;
 
-        ret = dict_get_ptr_and_len (dict, GF_AFR_SBRAIN_RESOLVE, &value, &len);
-        if (value) {
-                spb_child_index = afr_get_split_brain_child_index (this, value,
-                                                                   len);
-                if (spb_child_index < 0) {
-                        ret = 1;
-                        goto out;
-                }
-
-                afr_split_brain_resolve_do (frame, this, loc,
-                                            priv->children[spb_child_index]->name);
-                ret = 0;
-        }
+    /* Applicable for replace-brick and add-brick commands */
+    ret = afr_handle_empty_brick(this, frame, loc, dict);
 out:
-        /* key was correct but value was invalid when ret == 1 */
-        if (ret == 1) {
-                AFR_STACK_UNWIND (setxattr, frame, -1, op_errno, NULL);
-                ret = 0;
-        }
-        return ret;
+    return ret;
 }
 
 int
-afr_setxattr (call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *dict,
-	      int32_t flags, dict_t *xdata)
+afr_setxattr(call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *dict,
+             int32_t flags, dict_t *xdata)
 {
-        afr_local_t    *local             = NULL;
-        call_frame_t   *transaction_frame = NULL;
-        int             ret               = -1;
-        int             op_errno          = EINVAL;
-
-        GF_IF_INTERNAL_XATTR_GOTO ("trusted.afr.*", dict,
-                                   op_errno, out);
+    afr_local_t *local = NULL;
+    call_frame_t *transaction_frame = NULL;
+    int ret = -1;
+    int op_errno = EINVAL;
 
-        GF_IF_INTERNAL_XATTR_GOTO ("trusted.glusterfs.afr.*", dict,
-                                   op_errno, out);
+    GF_IF_INTERNAL_XATTR_GOTO("trusted.afr.*", dict, op_errno, out);
 
-        ret = afr_handle_split_brain_commands (this, frame, loc, dict);
+    GF_IF_INTERNAL_XATTR_GOTO("trusted.glusterfs.afr.*", dict, op_errno, out);
 
-        if (ret == 0)
-                return 0;
+    ret = afr_handle_special_xattr(this, frame, loc, dict);
+    if (ret == 0)
+        return 0;
 
-        transaction_frame = copy_frame (frame);
-        if (!transaction_frame)
-                goto out;
+    transaction_frame = copy_frame(frame);
+    if (!transaction_frame)
+        goto out;
 
-	local = AFR_FRAME_INIT (transaction_frame, op_errno);
-	if (!local)
-                goto out;
+    local = AFR_FRAME_INIT(transaction_frame, op_errno);
+    if (!local)
+        goto out;
 
-        local->cont.setxattr.dict  = dict_ref (dict);
-        local->cont.setxattr.flags = flags;
-	if (xdata)
-		local->xdata_req = dict_copy_with_ref (xdata, NULL);
-	else
-		local->xdata_req = dict_new ();
+    local->cont.setxattr.dict = dict_ref(dict);
+    local->cont.setxattr.flags = flags;
+    if (xdata)
+        local->xdata_req = dict_copy_with_ref(xdata, NULL);
+    else
+        local->xdata_req = dict_new();
 
-	if (!local->xdata_req)
-		goto out;
+    if (!local->xdata_req)
+        goto out;
 
-        local->transaction.wind   = afr_setxattr_wind;
-        local->transaction.fop    = __afr_txn_write_fop;
-        local->transaction.done   = __afr_txn_write_done;
-        local->transaction.unwind = afr_setxattr_unwind;
+    local->transaction.wind = afr_setxattr_wind;
+    local->transaction.unwind = afr_setxattr_unwind;
 
-        loc_copy (&local->loc, loc);
-	local->inode = inode_ref (loc->inode);
+    loc_copy(&local->loc, loc);
+    ret = afr_set_inode_local(this, local, loc->inode);
+    if (ret)
+        goto out;
 
-        local->transaction.main_frame = frame;
-        local->transaction.start   = LLONG_MAX - 1;
-        local->transaction.len     = 0;
+    local->transaction.main_frame = frame;
+    local->transaction.start = LLONG_MAX - 1;
+    local->transaction.len = 0;
 
-	local->op = GF_FOP_SETXATTR;
+    local->op = GF_FOP_SETXATTR;
 
-        ret = afr_transaction (transaction_frame, this, AFR_METADATA_TRANSACTION);
-        if (ret < 0) {
-		op_errno = -ret;
-		goto out;
-        }
+    ret = afr_transaction(transaction_frame, this, AFR_METADATA_TRANSACTION);
+    if (ret < 0) {
+        op_errno = -ret;
+        goto out;
+    }
 
-	return 0;
+    return 0;
 out:
-	if (transaction_frame)
-		AFR_STACK_DESTROY (transaction_frame);
+    if (transaction_frame)
+        AFR_STACK_DESTROY(transaction_frame);
 
-	AFR_STACK_UNWIND (setxattr, frame, -1, op_errno, NULL);
+    AFR_STACK_UNWIND(setxattr, frame, -1, op_errno, NULL);
 
-        return 0;
+    return 0;
 }
 
 /* {{{ fsetxattr */
 
-
 int
-afr_fsetxattr_unwind (call_frame_t *frame, xlator_t *this)
+afr_fsetxattr_unwind(call_frame_t *frame, xlator_t *this)
 {
-        afr_local_t    *local         = NULL;
-        call_frame_t   *main_frame    = NULL;
-
-        local = frame->local;
+    afr_local_t *local = NULL;
+    call_frame_t *main_frame = NULL;
 
-	main_frame = afr_transaction_detach_fop_frame (frame);
-        if (!main_frame)
-		return 0;
+    local = frame->local;
 
-	AFR_STACK_UNWIND (fsetxattr, main_frame, local->op_ret, local->op_errno,
-			  local->xdata_rsp);
+    main_frame = afr_transaction_detach_fop_frame(frame);
+    if (!main_frame)
         return 0;
-}
 
+    AFR_STACK_UNWIND(fsetxattr, main_frame, local->op_ret, local->op_errno,
+                     local->xdata_rsp);
+    return 0;
+}
 
 int
-afr_fsetxattr_wind_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                        int32_t op_ret, int32_t op_errno, dict_t *xdata)
+afr_fsetxattr_wind_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                       int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
-        return __afr_inode_write_cbk (frame, cookie, this, op_ret, op_errno,
-				      NULL, NULL, xdata);
+    return __afr_inode_write_cbk(frame, cookie, this, op_ret, op_errno, NULL,
+                                 NULL, NULL, xdata);
 }
 
-
 int
-afr_fsetxattr_wind (call_frame_t *frame, xlator_t *this, int subvol)
+afr_fsetxattr_wind(call_frame_t *frame, xlator_t *this, int subvol)
 {
-        afr_local_t        *local       = NULL;
-        afr_private_t      *priv        = NULL;
-
-        local = frame->local;
-        priv = this->private;
-
-	STACK_WIND_COOKIE (frame, afr_fsetxattr_wind_cbk, (void *) (long) subvol,
-			   priv->children[subvol],
-			   priv->children[subvol]->fops->fsetxattr,
-			   local->fd, local->cont.fsetxattr.dict,
-			   local->cont.fsetxattr.flags, local->xdata_req);
-        return 0;
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
+
+    local = frame->local;
+    priv = this->private;
+
+    STACK_WIND_COOKIE(frame, afr_fsetxattr_wind_cbk, (void *)(long)subvol,
+                      priv->children[subvol],
+                      priv->children[subvol]->fops->fsetxattr, local->fd,
+                      local->cont.fsetxattr.dict, local->cont.fsetxattr.flags,
+                      local->xdata_req);
+    return 0;
 }
 
-
 int
-afr_fsetxattr (call_frame_t *frame, xlator_t *this,
-               fd_t *fd, dict_t *dict, int32_t flags, dict_t *xdata)
+afr_fsetxattr(call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *dict,
+              int32_t flags, dict_t *xdata)
 {
-        afr_local_t      *local             = NULL;
-        call_frame_t     *transaction_frame = NULL;
-        int               ret               = -1;
-        int               op_errno          = ENOMEM;
+    afr_local_t *local = NULL;
+    call_frame_t *transaction_frame = NULL;
+    int ret = -1;
+    int op_errno = ENOMEM;
 
-        GF_IF_INTERNAL_XATTR_GOTO ("trusted.afr.*", dict,
-                                   op_errno, out);
+    GF_IF_INTERNAL_XATTR_GOTO("trusted.afr.*", dict, op_errno, out);
 
-        GF_IF_INTERNAL_XATTR_GOTO ("trusted.glusterfs.afr.*", dict,
-                                   op_errno, out);
+    GF_IF_INTERNAL_XATTR_GOTO("trusted.glusterfs.afr.*", dict, op_errno, out);
 
-        transaction_frame = copy_frame (frame);
-        if (!transaction_frame)
-                goto out;
+    AFR_ERROR_OUT_IF_FDCTX_INVALID(fd, this, op_errno, out);
+    transaction_frame = copy_frame(frame);
+    if (!transaction_frame)
+        goto out;
 
-	local = AFR_FRAME_INIT (transaction_frame, op_errno);
-	if (!local)
-		goto out;
+    local = AFR_FRAME_INIT(transaction_frame, op_errno);
+    if (!local)
+        goto out;
 
-        local->cont.fsetxattr.dict  = dict_ref (dict);
-        local->cont.fsetxattr.flags = flags;
+    local->cont.fsetxattr.dict = dict_ref(dict);
+    local->cont.fsetxattr.flags = flags;
 
-	if (xdata)
-		local->xdata_req = dict_copy_with_ref (xdata, NULL);
-	else
-		local->xdata_req = dict_new ();
+    if (xdata)
+        local->xdata_req = dict_copy_with_ref(xdata, NULL);
+    else
+        local->xdata_req = dict_new();
 
-	if (!local->xdata_req)
-		goto out;
+    if (!local->xdata_req)
+        goto out;
 
-        local->transaction.wind   = afr_fsetxattr_wind;
-        local->transaction.fop    = __afr_txn_write_fop;
-        local->transaction.done   = __afr_txn_write_done;
-        local->transaction.unwind = afr_fsetxattr_unwind;
+    local->transaction.wind = afr_fsetxattr_wind;
+    local->transaction.unwind = afr_fsetxattr_unwind;
 
-        local->fd                 = fd_ref (fd);
-	local->inode = inode_ref (fd->inode);
+    local->fd = fd_ref(fd);
+    ret = afr_set_inode_local(this, local, fd->inode);
+    if (ret)
+        goto out;
 
-	local->op = GF_FOP_FSETXATTR;
+    local->op = GF_FOP_FSETXATTR;
 
-        local->transaction.main_frame = frame;
-        local->transaction.start  = LLONG_MAX - 1;
-        local->transaction.len    = 0;
+    local->transaction.main_frame = frame;
+    local->transaction.start = LLONG_MAX - 1;
+    local->transaction.len = 0;
 
-        ret = afr_transaction (transaction_frame, this, AFR_METADATA_TRANSACTION);
-        if (ret < 0) {
-		op_errno = -ret;
-		goto out;
-        }
+    ret = afr_transaction(transaction_frame, this, AFR_METADATA_TRANSACTION);
+    if (ret < 0) {
+        op_errno = -ret;
+        goto out;
+    }
 
-	return 0;
+    return 0;
 out:
-	if (transaction_frame)
-		AFR_STACK_DESTROY (transaction_frame);
+    if (transaction_frame)
+        AFR_STACK_DESTROY(transaction_frame);
 
-	AFR_STACK_UNWIND (fsetxattr, frame, -1, op_errno, NULL);
-        return 0;
+    AFR_STACK_UNWIND(fsetxattr, frame, -1, op_errno, NULL);
+    return 0;
 }
 
 /* }}} */
 
-
 /* {{{ removexattr */
 
-
 int
-afr_removexattr_unwind (call_frame_t *frame, xlator_t *this)
+afr_removexattr_unwind(call_frame_t *frame, xlator_t *this)
 {
-        afr_local_t *   local = NULL;
-        call_frame_t   *main_frame = NULL;
-
-        local = frame->local;
+    afr_local_t *local = NULL;
+    call_frame_t *main_frame = NULL;
 
-	main_frame = afr_transaction_detach_fop_frame (frame);
-	if (!main_frame)
-		return 0;
+    local = frame->local;
 
-	AFR_STACK_UNWIND (removexattr, main_frame, local->op_ret, local->op_errno,
-			  local->xdata_rsp);
+    main_frame = afr_transaction_detach_fop_frame(frame);
+    if (!main_frame)
         return 0;
-}
 
+    AFR_STACK_UNWIND(removexattr, main_frame, local->op_ret, local->op_errno,
+                     local->xdata_rsp);
+    return 0;
+}
 
 int
-afr_removexattr_wind_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                          int32_t op_ret, int32_t op_errno, dict_t *xdata)
+afr_removexattr_wind_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                         int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
-        return __afr_inode_write_cbk (frame, cookie, this, op_ret, op_errno,
-				      NULL, NULL, xdata);
+    return __afr_inode_write_cbk(frame, cookie, this, op_ret, op_errno, NULL,
+                                 NULL, NULL, xdata);
 }
 
-
 int
-afr_removexattr_wind (call_frame_t *frame, xlator_t *this, int subvol)
+afr_removexattr_wind(call_frame_t *frame, xlator_t *this, int subvol)
 {
-        afr_local_t *local = NULL;
-        afr_private_t *priv = NULL;
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
 
-	local = frame->local;
-	priv = this->private;
+    local = frame->local;
+    priv = this->private;
 
-	STACK_WIND_COOKIE (frame, afr_removexattr_wind_cbk, (void *) (long) subvol,
-			   priv->children[subvol],
-			   priv->children[subvol]->fops->removexattr,
-			   &local->loc, local->cont.removexattr.name,
-			   local->xdata_req);
-        return 0;
+    STACK_WIND_COOKIE(frame, afr_removexattr_wind_cbk, (void *)(long)subvol,
+                      priv->children[subvol],
+                      priv->children[subvol]->fops->removexattr, &local->loc,
+                      local->cont.removexattr.name, local->xdata_req);
+    return 0;
 }
 
-
 int
-afr_removexattr (call_frame_t *frame, xlator_t *this,
-                 loc_t *loc, const char *name, dict_t *xdata)
+afr_removexattr(call_frame_t *frame, xlator_t *this, loc_t *loc,
+                const char *name, dict_t *xdata)
 {
-        afr_local_t     *local             = NULL;
-        call_frame_t    *transaction_frame = NULL;
-        int              ret               = -1;
-        int              op_errno          = ENOMEM;
+    afr_local_t *local = NULL;
+    call_frame_t *transaction_frame = NULL;
+    int ret = -1;
+    int op_errno = ENOMEM;
 
-        GF_IF_NATIVE_XATTR_GOTO ("trusted.afr.*",
-                                 name, op_errno, out);
+    GF_IF_NATIVE_XATTR_GOTO("trusted.afr.*", name, op_errno, out);
 
-        GF_IF_NATIVE_XATTR_GOTO ("trusted.glusterfs.afr.*",
-                                 name, op_errno, out);
+    GF_IF_NATIVE_XATTR_GOTO("trusted.glusterfs.afr.*", name, op_errno, out);
 
-        transaction_frame = copy_frame (frame);
-        if (!transaction_frame)
-                goto out;
+    transaction_frame = copy_frame(frame);
+    if (!transaction_frame)
+        goto out;
 
-	local = AFR_FRAME_INIT (transaction_frame, op_errno);
-	if (!local)
-		goto out;
+    local = AFR_FRAME_INIT(transaction_frame, op_errno);
+    if (!local)
+        goto out;
 
-        local->cont.removexattr.name = gf_strdup (name);
+    local->cont.removexattr.name = gf_strdup(name);
 
-	if (xdata)
-		local->xdata_req = dict_copy_with_ref (xdata, NULL);
-	else
-		local->xdata_req = dict_new ();
+    if (xdata)
+        local->xdata_req = dict_copy_with_ref(xdata, NULL);
+    else
+        local->xdata_req = dict_new();
 
-	if (!local->xdata_req)
-		goto out;
+    if (!local->xdata_req)
+        goto out;
 
-        local->transaction.wind   = afr_removexattr_wind;
-        local->transaction.fop    = __afr_txn_write_fop;
-        local->transaction.done   = __afr_txn_write_done;
-        local->transaction.unwind = afr_removexattr_unwind;
+    local->transaction.wind = afr_removexattr_wind;
+    local->transaction.unwind = afr_removexattr_unwind;
 
-        loc_copy (&local->loc, loc);
-	local->inode = inode_ref (loc->inode);
+    loc_copy(&local->loc, loc);
+    ret = afr_set_inode_local(this, local, loc->inode);
+    if (ret)
+        goto out;
 
-	local->op = GF_FOP_REMOVEXATTR;
+    local->op = GF_FOP_REMOVEXATTR;
 
-        local->transaction.main_frame = frame;
-        local->transaction.start   = LLONG_MAX - 1;
-        local->transaction.len     = 0;
+    local->transaction.main_frame = frame;
+    local->transaction.start = LLONG_MAX - 1;
+    local->transaction.len = 0;
 
-        ret = afr_transaction (transaction_frame, this, AFR_METADATA_TRANSACTION);
-        if (ret < 0) {
-		op_errno = -ret;
-		goto out;
-        }
+    ret = afr_transaction(transaction_frame, this, AFR_METADATA_TRANSACTION);
+    if (ret < 0) {
+        op_errno = -ret;
+        goto out;
+    }
 
-	return 0;
+    return 0;
 out:
-	if (transaction_frame)
-		AFR_STACK_DESTROY (transaction_frame);
+    if (transaction_frame)
+        AFR_STACK_DESTROY(transaction_frame);
 
-	AFR_STACK_UNWIND (removexattr, frame, -1, op_errno, NULL);
-        return 0;
+    AFR_STACK_UNWIND(removexattr, frame, -1, op_errno, NULL);
+    return 0;
 }
 
 /* ffremovexattr */
 int
-afr_fremovexattr_unwind (call_frame_t *frame, xlator_t *this)
+afr_fremovexattr_unwind(call_frame_t *frame, xlator_t *this)
 {
-        afr_local_t *   local = NULL;
-        call_frame_t   *main_frame = NULL;
+    afr_local_t *local = NULL;
+    call_frame_t *main_frame = NULL;
 
-        local = frame->local;
+    local = frame->local;
 
-	main_frame = afr_transaction_detach_fop_frame (frame);
-        if (!main_frame)
-		return 0;
-
-	AFR_STACK_UNWIND (fremovexattr, main_frame, local->op_ret, local->op_errno,
-			  local->xdata_rsp);
+    main_frame = afr_transaction_detach_fop_frame(frame);
+    if (!main_frame)
         return 0;
-}
 
+    AFR_STACK_UNWIND(fremovexattr, main_frame, local->op_ret, local->op_errno,
+                     local->xdata_rsp);
+    return 0;
+}
 
 int
-afr_fremovexattr_wind_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+afr_fremovexattr_wind_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
                           int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
-	return __afr_inode_write_cbk (frame, cookie, this, op_ret, op_errno,
-				      NULL, NULL, xdata);
+    return __afr_inode_write_cbk(frame, cookie, this, op_ret, op_errno, NULL,
+                                 NULL, NULL, xdata);
 }
 
-
 int
-afr_fremovexattr_wind (call_frame_t *frame, xlator_t *this, int subvol)
+afr_fremovexattr_wind(call_frame_t *frame, xlator_t *this, int subvol)
 {
-        afr_local_t *local = NULL;
-        afr_private_t *priv = NULL;
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
 
-        local = frame->local;
-        priv = this->private;
+    local = frame->local;
+    priv = this->private;
 
-	STACK_WIND_COOKIE (frame, afr_fremovexattr_wind_cbk, (void *) (long) subvol,
-			   priv->children[subvol],
-			   priv->children[subvol]->fops->fremovexattr,
-			   local->fd, local->cont.removexattr.name,
-			   local->xdata_req);
-        return 0;
+    STACK_WIND_COOKIE(frame, afr_fremovexattr_wind_cbk, (void *)(long)subvol,
+                      priv->children[subvol],
+                      priv->children[subvol]->fops->fremovexattr, local->fd,
+                      local->cont.removexattr.name, local->xdata_req);
+    return 0;
 }
 
-
 int
-afr_fremovexattr (call_frame_t *frame, xlator_t *this, fd_t *fd,
-		  const char *name, dict_t *xdata)
+afr_fremovexattr(call_frame_t *frame, xlator_t *this, fd_t *fd,
+                 const char *name, dict_t *xdata)
 {
-        afr_local_t *local = NULL;
-        call_frame_t *transaction_frame = NULL;
-        int ret = -1;
-        int op_errno = ENOMEM;
+    afr_local_t *local = NULL;
+    call_frame_t *transaction_frame = NULL;
+    int ret = -1;
+    int op_errno = ENOMEM;
 
-        GF_IF_NATIVE_XATTR_GOTO ("trusted.afr.*",
-                                 name, op_errno, out);
+    GF_IF_NATIVE_XATTR_GOTO("trusted.afr.*", name, op_errno, out);
 
-        GF_IF_NATIVE_XATTR_GOTO ("trusted.glusterfs.afr.*",
-                                 name, op_errno, out);
+    GF_IF_NATIVE_XATTR_GOTO("trusted.glusterfs.afr.*", name, op_errno, out);
 
-        transaction_frame = copy_frame (frame);
-        if (!transaction_frame)
-                goto out;
+    AFR_ERROR_OUT_IF_FDCTX_INVALID(fd, this, op_errno, out);
+    transaction_frame = copy_frame(frame);
+    if (!transaction_frame)
+        goto out;
 
-	local = AFR_FRAME_INIT (transaction_frame, op_errno);
-	if (!local)
-                goto out;
+    local = AFR_FRAME_INIT(transaction_frame, op_errno);
+    if (!local)
+        goto out;
 
-        local->cont.removexattr.name = gf_strdup (name);
-	if (xdata)
-		local->xdata_req = dict_copy_with_ref (xdata, NULL);
-	else
-		local->xdata_req = dict_new ();
+    local->cont.removexattr.name = gf_strdup(name);
+    if (xdata)
+        local->xdata_req = dict_copy_with_ref(xdata, NULL);
+    else
+        local->xdata_req = dict_new();
 
-	if (!local->xdata_req)
-		goto out;
+    if (!local->xdata_req)
+        goto out;
 
-        local->transaction.wind   = afr_fremovexattr_wind;
-        local->transaction.fop    = __afr_txn_write_fop;
-        local->transaction.done   = __afr_txn_write_done;
-        local->transaction.unwind = afr_fremovexattr_unwind;
+    local->transaction.wind = afr_fremovexattr_wind;
+    local->transaction.unwind = afr_fremovexattr_unwind;
 
-        local->fd = fd_ref (fd);
-	local->inode = inode_ref (fd->inode);
+    local->fd = fd_ref(fd);
+    ret = afr_set_inode_local(this, local, fd->inode);
+    if (ret)
+        goto out;
 
-	local->op = GF_FOP_FREMOVEXATTR;
+    local->op = GF_FOP_FREMOVEXATTR;
 
-        local->transaction.main_frame = frame;
-        local->transaction.start   = LLONG_MAX - 1;
-        local->transaction.len     = 0;
+    local->transaction.main_frame = frame;
+    local->transaction.start = LLONG_MAX - 1;
+    local->transaction.len = 0;
 
-        ret = afr_transaction (transaction_frame, this, AFR_METADATA_TRANSACTION);
-        if (ret < 0) {
-		op_errno = -ret;
-		goto out;
-        }
+    ret = afr_transaction(transaction_frame, this, AFR_METADATA_TRANSACTION);
+    if (ret < 0) {
+        op_errno = -ret;
+        goto out;
+    }
 
-	return 0;
+    return 0;
 out:
-	if (transaction_frame)
-		AFR_STACK_DESTROY (transaction_frame);
+    if (transaction_frame)
+        AFR_STACK_DESTROY(transaction_frame);
 
-	AFR_STACK_UNWIND (fremovexattr, frame, -1, op_errno, NULL);
+    AFR_STACK_UNWIND(fremovexattr, frame, -1, op_errno, NULL);
 
-        return 0;
+    return 0;
 }
 
-
 int
-afr_fallocate_unwind (call_frame_t *frame, xlator_t *this)
+afr_fallocate_unwind(call_frame_t *frame, xlator_t *this)
 {
-        afr_local_t *   local = NULL;
-        call_frame_t   *main_frame = NULL;
+    afr_local_t *local = NULL;
+    call_frame_t *main_frame = NULL;
 
-        local = frame->local;
+    local = frame->local;
 
-	main_frame = afr_transaction_detach_fop_frame (frame);
-	if (!main_frame)
-		return 0;
-
-	AFR_STACK_UNWIND (fallocate, main_frame, local->op_ret, local->op_errno,
-			  &local->cont.inode_wfop.prebuf,
-			  &local->cont.inode_wfop.postbuf, local->xdata_rsp);
+    main_frame = afr_transaction_detach_fop_frame(frame);
+    if (!main_frame)
         return 0;
-}
 
+    AFR_STACK_UNWIND(fallocate, main_frame, local->op_ret, local->op_errno,
+                     &local->cont.inode_wfop.prebuf,
+                     &local->cont.inode_wfop.postbuf, local->xdata_rsp);
+    return 0;
+}
 
 int
-afr_fallocate_wind_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                        int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
-                        struct iatt *postbuf, dict_t *xdata)
+afr_fallocate_wind_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                       int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
+                       struct iatt *postbuf, dict_t *xdata)
 {
-        return __afr_inode_write_cbk (frame, cookie, this, op_ret, op_errno,
-				      prebuf, postbuf, xdata);
+    return __afr_inode_write_cbk(frame, cookie, this, op_ret, op_errno, prebuf,
+                                 postbuf, NULL, xdata);
 }
 
-
 int
-afr_fallocate_wind (call_frame_t *frame, xlator_t *this, int subvol)
+afr_fallocate_wind(call_frame_t *frame, xlator_t *this, int subvol)
 {
-        afr_local_t *local = NULL;
-        afr_private_t *priv = NULL;
-
-        local = frame->local;
-        priv = this->private;
-
-	STACK_WIND_COOKIE (frame, afr_fallocate_wind_cbk, (void *) (long) subvol,
-			   priv->children[subvol],
-			   priv->children[subvol]->fops->fallocate,
-			   local->fd, local->cont.fallocate.mode,
-			   local->cont.fallocate.offset,
-			   local->cont.fallocate.len, local->xdata_req);
-        return 0;
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
+
+    local = frame->local;
+    priv = this->private;
+
+    STACK_WIND_COOKIE(frame, afr_fallocate_wind_cbk, (void *)(long)subvol,
+                      priv->children[subvol],
+                      priv->children[subvol]->fops->fallocate, local->fd,
+                      local->cont.fallocate.mode, local->cont.fallocate.offset,
+                      local->cont.fallocate.len, local->xdata_req);
+    return 0;
 }
 
-
 int
-afr_fallocate (call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t mode,
-               off_t offset, size_t len, dict_t *xdata)
+afr_fallocate(call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t mode,
+              off_t offset, size_t len, dict_t *xdata)
 {
-        call_frame_t *transaction_frame = NULL;
-        afr_local_t *local = NULL;
-        int ret = -1;
-        int op_errno = ENOMEM;
+    call_frame_t *transaction_frame = NULL;
+    afr_local_t *local = NULL;
+    int ret = -1;
+    int op_errno = ENOMEM;
 
-        transaction_frame = copy_frame (frame);
-        if (!transaction_frame)
-                goto out;
+    AFR_ERROR_OUT_IF_FDCTX_INVALID(fd, this, op_errno, out);
+    transaction_frame = copy_frame(frame);
+    if (!transaction_frame)
+        goto out;
 
-	local = AFR_FRAME_INIT (transaction_frame, op_errno);
-	if (!local)
-		goto out;
+    local = AFR_FRAME_INIT(transaction_frame, op_errno);
+    if (!local)
+        goto out;
 
-        local->cont.fallocate.mode = mode;
-        local->cont.fallocate.offset  = offset;
-        local->cont.fallocate.len = len;
+    local->cont.fallocate.mode = mode;
+    local->cont.fallocate.offset = offset;
+    local->cont.fallocate.len = len;
 
-        local->fd = fd_ref (fd);
-	local->inode = inode_ref (fd->inode);
+    local->fd = fd_ref(fd);
+    ret = afr_set_inode_local(this, local, fd->inode);
+    if (ret)
+        goto out;
 
-	if (xdata)
-		local->xdata_req = dict_copy_with_ref (xdata, NULL);
-	else
-		local->xdata_req = dict_new ();
+    if (xdata)
+        local->xdata_req = dict_copy_with_ref(xdata, NULL);
+    else
+        local->xdata_req = dict_new();
 
-	if (!local->xdata_req)
-		goto out;
+    if (!local->xdata_req)
+        goto out;
 
-        local->op = GF_FOP_FALLOCATE;
+    local->op = GF_FOP_FALLOCATE;
 
-        local->transaction.wind   = afr_fallocate_wind;
-        local->transaction.fop    = __afr_txn_write_fop;
-        local->transaction.done   = __afr_txn_write_done;
-        local->transaction.unwind = afr_fallocate_unwind;
+    local->transaction.wind = afr_fallocate_wind;
+    local->transaction.unwind = afr_fallocate_unwind;
 
-        local->transaction.main_frame = frame;
+    local->transaction.main_frame = frame;
 
-        local->transaction.start   = local->cont.fallocate.offset;
-        local->transaction.len     = 0;
+    local->transaction.start = local->cont.fallocate.offset;
+    local->transaction.len = 0;
 
-        afr_fix_open (fd, this);
+    afr_fix_open(fd, this);
 
-        ret = afr_transaction (transaction_frame, this, AFR_DATA_TRANSACTION);
-        if (ret < 0) {
-		op_errno = -ret;
-		goto out;
-        }
+    ret = afr_transaction(transaction_frame, this, AFR_DATA_TRANSACTION);
+    if (ret < 0) {
+        op_errno = -ret;
+        goto out;
+    }
 
-	return 0;
+    return 0;
 out:
-	if (transaction_frame)
-		AFR_STACK_DESTROY (transaction_frame);
+    if (transaction_frame)
+        AFR_STACK_DESTROY(transaction_frame);
 
-	AFR_STACK_UNWIND (fallocate, frame, -1, op_errno, NULL, NULL, NULL);
-        return 0;
+    AFR_STACK_UNWIND(fallocate, frame, -1, op_errno, NULL, NULL, NULL);
+    return 0;
 }
 
-
 /* }}} */
 
 /* {{{ discard */
 
 int
-afr_discard_unwind (call_frame_t *frame, xlator_t *this)
+afr_discard_unwind(call_frame_t *frame, xlator_t *this)
 {
-        afr_local_t *   local = NULL;
-        call_frame_t   *main_frame = NULL;
+    afr_local_t *local = NULL;
+    call_frame_t *main_frame = NULL;
 
-        local = frame->local;
+    local = frame->local;
 
-	main_frame = afr_transaction_detach_fop_frame (frame);
-        if (!main_frame)
-		return 0;
-
-	AFR_STACK_UNWIND (discard, main_frame, local->op_ret, local->op_errno,
-			  &local->cont.inode_wfop.prebuf,
-			  &local->cont.inode_wfop.postbuf, local->xdata_rsp);
+    main_frame = afr_transaction_detach_fop_frame(frame);
+    if (!main_frame)
         return 0;
+
+    AFR_STACK_UNWIND(discard, main_frame, local->op_ret, local->op_errno,
+                     &local->cont.inode_wfop.prebuf,
+                     &local->cont.inode_wfop.postbuf, local->xdata_rsp);
+    return 0;
 }
 
+int
+afr_discard_wind_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                     int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
+                     struct iatt *postbuf, dict_t *xdata)
+{
+    return __afr_inode_write_cbk(frame, cookie, this, op_ret, op_errno, prebuf,
+                                 postbuf, NULL, xdata);
+}
 
 int
-afr_discard_wind_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                      int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
-                      struct iatt *postbuf, dict_t *xdata)
+afr_discard_wind(call_frame_t *frame, xlator_t *this, int subvol)
 {
-	return __afr_inode_write_cbk (frame, cookie, this, op_ret, op_errno,
-				      prebuf, postbuf, xdata);
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
+
+    local = frame->local;
+    priv = this->private;
+
+    STACK_WIND_COOKIE(frame, afr_discard_wind_cbk, (void *)(long)subvol,
+                      priv->children[subvol],
+                      priv->children[subvol]->fops->discard, local->fd,
+                      local->cont.discard.offset, local->cont.discard.len,
+                      local->xdata_req);
+    return 0;
 }
 
+int
+afr_discard(call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
+            size_t len, dict_t *xdata)
+{
+    afr_local_t *local = NULL;
+    call_frame_t *transaction_frame = NULL;
+    int ret = -1;
+    int op_errno = ENOMEM;
+
+    AFR_ERROR_OUT_IF_FDCTX_INVALID(fd, this, op_errno, out);
+    transaction_frame = copy_frame(frame);
+    if (!transaction_frame)
+        goto out;
+
+    local = AFR_FRAME_INIT(transaction_frame, op_errno);
+    if (!local)
+        goto out;
+
+    local->cont.discard.offset = offset;
+    local->cont.discard.len = len;
+
+    local->fd = fd_ref(fd);
+    ret = afr_set_inode_local(this, local, fd->inode);
+    if (ret)
+        goto out;
+
+    if (xdata)
+        local->xdata_req = dict_copy_with_ref(xdata, NULL);
+    else
+        local->xdata_req = dict_new();
+
+    if (!local->xdata_req)
+        goto out;
+
+    local->op = GF_FOP_DISCARD;
+
+    local->transaction.wind = afr_discard_wind;
+    local->transaction.unwind = afr_discard_unwind;
+
+    local->transaction.main_frame = frame;
+
+    local->transaction.start = local->cont.discard.offset;
+    local->transaction.len = 0;
+
+    afr_fix_open(fd, this);
+
+    ret = afr_transaction(transaction_frame, this, AFR_DATA_TRANSACTION);
+    if (ret < 0) {
+        op_errno = -ret;
+        goto out;
+    }
+
+    return 0;
+out:
+    if (transaction_frame)
+        AFR_STACK_DESTROY(transaction_frame);
+
+    AFR_STACK_UNWIND(discard, frame, -1, op_errno, NULL, NULL, NULL);
+    return 0;
+}
+
+/* {{{ zerofill */
 
 int
-afr_discard_wind (call_frame_t *frame, xlator_t *this, int subvol)
+afr_zerofill_unwind(call_frame_t *frame, xlator_t *this)
 {
-        afr_local_t *local = NULL;
-        afr_private_t *priv = NULL;
+    afr_local_t *local = NULL;
+    call_frame_t *main_frame = NULL;
 
-        local = frame->local;
-        priv = this->private;
+    local = frame->local;
 
-	STACK_WIND_COOKIE (frame, afr_discard_wind_cbk, (void *) (long) subvol,
-			   priv->children[subvol],
-			   priv->children[subvol]->fops->discard,
-			   local->fd, local->cont.discard.offset,
-			   local->cont.discard.len, local->xdata_req);
+    main_frame = afr_transaction_detach_fop_frame(frame);
+    if (!main_frame)
         return 0;
+
+    AFR_STACK_UNWIND(discard, main_frame, local->op_ret, local->op_errno,
+                     &local->cont.inode_wfop.prebuf,
+                     &local->cont.inode_wfop.postbuf, local->xdata_rsp);
+    return 0;
 }
 
+int
+afr_zerofill_wind_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                      int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
+                      struct iatt *postbuf, dict_t *xdata)
+{
+    return __afr_inode_write_cbk(frame, cookie, this, op_ret, op_errno, prebuf,
+                                 postbuf, NULL, xdata);
+}
+
+int
+afr_zerofill_wind(call_frame_t *frame, xlator_t *this, int subvol)
+{
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
+
+    local = frame->local;
+    priv = this->private;
+
+    STACK_WIND_COOKIE(frame, afr_zerofill_wind_cbk, (void *)(long)subvol,
+                      priv->children[subvol],
+                      priv->children[subvol]->fops->zerofill, local->fd,
+                      local->cont.zerofill.offset, local->cont.zerofill.len,
+                      local->xdata_req);
+    return 0;
+}
 
 int
-afr_discard (call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
+afr_zerofill(call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
              size_t len, dict_t *xdata)
 {
-        afr_local_t *local = NULL;
-        call_frame_t *transaction_frame = NULL;
-        int ret = -1;
-        int op_errno = ENOMEM;
+    afr_local_t *local = NULL;
+    call_frame_t *transaction_frame = NULL;
+    int ret = -1;
+    int op_errno = ENOMEM;
 
-	transaction_frame = copy_frame (frame);
-	if (!transaction_frame)
-		goto out;
+    AFR_ERROR_OUT_IF_FDCTX_INVALID(fd, this, op_errno, out);
+    transaction_frame = copy_frame(frame);
+    if (!transaction_frame)
+        goto out;
 
-	local = AFR_FRAME_INIT (transaction_frame, op_errno);
-	if (!local)
-		goto out;
+    local = AFR_FRAME_INIT(transaction_frame, op_errno);
+    if (!local)
+        goto out;
 
-        local->cont.discard.offset  = offset;
-        local->cont.discard.len = len;
+    local->cont.zerofill.offset = offset;
+    local->cont.zerofill.len = len;
 
-        local->fd = fd_ref (fd);
-	local->inode = inode_ref (fd->inode);
+    local->fd = fd_ref(fd);
+    ret = afr_set_inode_local(this, local, fd->inode);
+    if (ret)
+        goto out;
 
-	if (xdata)
-		local->xdata_req = dict_copy_with_ref (xdata, NULL);
-	else
-		local->xdata_req = dict_new ();
+    if (xdata)
+        local->xdata_req = dict_copy_with_ref(xdata, NULL);
+    else
+        local->xdata_req = dict_new();
 
-	if (!local->xdata_req)
-		goto out;
+    if (!local->xdata_req)
+        goto out;
 
-        local->op = GF_FOP_DISCARD;
+    local->op = GF_FOP_ZEROFILL;
 
-        local->transaction.wind   = afr_discard_wind;
-        local->transaction.fop    = __afr_txn_write_fop;
-        local->transaction.done   = __afr_txn_write_done;
-        local->transaction.unwind = afr_discard_unwind;
+    local->transaction.wind = afr_zerofill_wind;
+    local->transaction.unwind = afr_zerofill_unwind;
 
-        local->transaction.main_frame = frame;
+    local->transaction.main_frame = frame;
 
-        local->transaction.start   = local->cont.discard.offset;
-        local->transaction.len     = 0;
+    local->transaction.start = local->cont.zerofill.offset;
+    local->transaction.len = len;
 
-        afr_fix_open (fd, this);
+    afr_fix_open(fd, this);
 
-        ret = afr_transaction (transaction_frame, this, AFR_DATA_TRANSACTION);
-        if (ret < 0) {
-		op_errno = -ret;
-		goto out;
-        }
+    ret = afr_transaction(transaction_frame, this, AFR_DATA_TRANSACTION);
+    if (ret < 0) {
+        op_errno = -ret;
+        goto out;
+    }
 
-	return 0;
+    return 0;
 out:
-	if (transaction_frame)
-		AFR_STACK_DESTROY (transaction_frame);
+    if (transaction_frame)
+        AFR_STACK_DESTROY(transaction_frame);
 
-	AFR_STACK_UNWIND (discard, frame, -1, op_errno, NULL, NULL, NULL);
-        return 0;
+    AFR_STACK_UNWIND(zerofill, frame, -1, op_errno, NULL, NULL, NULL);
+    return 0;
 }
 
+/* }}} */
 
-/* {{{ zerofill */
+int32_t
+afr_xattrop_wind_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                     int32_t op_ret, int32_t op_errno, dict_t *xattr,
+                     dict_t *xdata)
+{
+    return __afr_inode_write_cbk(frame, cookie, this, op_ret, op_errno, NULL,
+                                 NULL, xattr, xdata);
+}
 
 int
-afr_zerofill_unwind (call_frame_t *frame, xlator_t *this)
+afr_xattrop_wind(call_frame_t *frame, xlator_t *this, int subvol)
 {
-        afr_local_t *   local = NULL;
-        call_frame_t   *main_frame = NULL;
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
+
+    local = frame->local;
+    priv = this->private;
+
+    STACK_WIND_COOKIE(frame, afr_xattrop_wind_cbk, (void *)(long)subvol,
+                      priv->children[subvol],
+                      priv->children[subvol]->fops->xattrop, &local->loc,
+                      local->cont.xattrop.optype, local->cont.xattrop.xattr,
+                      local->xdata_req);
+    return 0;
+}
 
-        local = frame->local;
+int
+afr_xattrop_unwind(call_frame_t *frame, xlator_t *this)
+{
+    afr_local_t *local = NULL;
+    call_frame_t *main_frame = NULL;
 
-	main_frame = afr_transaction_detach_fop_frame (frame);
-        if (!main_frame)
-		return 0;
+    local = frame->local;
 
-	AFR_STACK_UNWIND (discard, main_frame, local->op_ret, local->op_errno,
-			  &local->cont.inode_wfop.prebuf,
-			  &local->cont.inode_wfop.postbuf, local->xdata_rsp);
+    main_frame = afr_transaction_detach_fop_frame(frame);
+    if (!main_frame)
         return 0;
+
+    AFR_STACK_UNWIND(xattrop, main_frame, local->op_ret, local->op_errno,
+                     local->xattr_rsp, local->xdata_rsp);
+    return 0;
+}
+
+int32_t
+afr_xattrop(call_frame_t *frame, xlator_t *this, loc_t *loc,
+            gf_xattrop_flags_t optype, dict_t *xattr, dict_t *xdata)
+{
+    afr_local_t *local = NULL;
+    call_frame_t *transaction_frame = NULL;
+    int ret = -1;
+    int op_errno = ENOMEM;
+
+    transaction_frame = copy_frame(frame);
+    if (!transaction_frame)
+        goto out;
+
+    local = AFR_FRAME_INIT(transaction_frame, op_errno);
+    if (!local)
+        goto out;
+
+    local->cont.xattrop.xattr = dict_ref(xattr);
+    local->cont.xattrop.optype = optype;
+    if (xdata)
+        local->xdata_req = dict_ref(xdata);
+
+    local->transaction.wind = afr_xattrop_wind;
+    local->transaction.unwind = afr_xattrop_unwind;
+
+    loc_copy(&local->loc, loc);
+    ret = afr_set_inode_local(this, local, loc->inode);
+    if (ret)
+        goto out;
+
+    local->op = GF_FOP_XATTROP;
+
+    local->transaction.main_frame = frame;
+    local->transaction.start = LLONG_MAX - 1;
+    local->transaction.len = 0;
+
+    ret = afr_transaction(transaction_frame, this, AFR_METADATA_TRANSACTION);
+    if (ret < 0) {
+        op_errno = -ret;
+        goto out;
+    }
+
+    return 0;
+out:
+    if (transaction_frame)
+        AFR_STACK_DESTROY(transaction_frame);
+
+    AFR_STACK_UNWIND(xattrop, frame, -1, op_errno, NULL, NULL);
+    return 0;
 }
 
+int32_t
+afr_fxattrop_wind_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                      int32_t op_ret, int32_t op_errno, dict_t *xattr,
+                      dict_t *xdata)
+{
+    return __afr_inode_write_cbk(frame, cookie, this, op_ret, op_errno, NULL,
+                                 NULL, xattr, xdata);
+}
 
 int
-afr_zerofill_wind_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                      int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
-                      struct iatt *postbuf, dict_t *xdata)
+afr_fxattrop_wind(call_frame_t *frame, xlator_t *this, int subvol)
 {
-	return __afr_inode_write_cbk (frame, cookie, this, op_ret, op_errno,
-				      prebuf, postbuf, xdata);
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
+
+    local = frame->local;
+    priv = this->private;
+
+    STACK_WIND_COOKIE(frame, afr_fxattrop_wind_cbk, (void *)(long)subvol,
+                      priv->children[subvol],
+                      priv->children[subvol]->fops->fxattrop, local->fd,
+                      local->cont.xattrop.optype, local->cont.xattrop.xattr,
+                      local->xdata_req);
+    return 0;
 }
 
-
 int
-afr_zerofill_wind (call_frame_t *frame, xlator_t *this, int subvol)
+afr_fxattrop_unwind(call_frame_t *frame, xlator_t *this)
 {
-        afr_local_t *local = NULL;
-        afr_private_t *priv = NULL;
+    afr_local_t *local = NULL;
+    call_frame_t *main_frame = NULL;
 
-        local = frame->local;
-        priv = this->private;
+    local = frame->local;
 
-	STACK_WIND_COOKIE (frame, afr_zerofill_wind_cbk, (void *) (long) subvol,
-			   priv->children[subvol],
-			   priv->children[subvol]->fops->zerofill,
-			   local->fd, local->cont.zerofill.offset,
-			   local->cont.zerofill.len, local->xdata_req);
+    main_frame = afr_transaction_detach_fop_frame(frame);
+    if (!main_frame)
         return 0;
+
+    AFR_STACK_UNWIND(fxattrop, main_frame, local->op_ret, local->op_errno,
+                     local->xattr_rsp, local->xdata_rsp);
+    return 0;
+}
+
+int32_t
+afr_fxattrop(call_frame_t *frame, xlator_t *this, fd_t *fd,
+             gf_xattrop_flags_t optype, dict_t *xattr, dict_t *xdata)
+{
+    afr_local_t *local = NULL;
+    call_frame_t *transaction_frame = NULL;
+    int ret = -1;
+    int op_errno = ENOMEM;
+
+    AFR_ERROR_OUT_IF_FDCTX_INVALID(fd, this, op_errno, out);
+    transaction_frame = copy_frame(frame);
+    if (!transaction_frame)
+        goto out;
+
+    local = AFR_FRAME_INIT(transaction_frame, op_errno);
+    if (!local)
+        goto out;
+
+    local->cont.xattrop.xattr = dict_ref(xattr);
+    local->cont.xattrop.optype = optype;
+    if (xdata)
+        local->xdata_req = dict_ref(xdata);
+
+    local->transaction.wind = afr_fxattrop_wind;
+    local->transaction.unwind = afr_fxattrop_unwind;
+
+    local->fd = fd_ref(fd);
+    ret = afr_set_inode_local(this, local, fd->inode);
+    if (ret)
+        goto out;
+
+    local->op = GF_FOP_FXATTROP;
+
+    local->transaction.main_frame = frame;
+    local->transaction.start = LLONG_MAX - 1;
+    local->transaction.len = 0;
+
+    ret = afr_transaction(transaction_frame, this, AFR_METADATA_TRANSACTION);
+    if (ret < 0) {
+        op_errno = -ret;
+        goto out;
+    }
+
+    return 0;
+out:
+    if (transaction_frame)
+        AFR_STACK_DESTROY(transaction_frame);
+
+    AFR_STACK_UNWIND(fxattrop, frame, -1, op_errno, NULL, NULL);
+    return 0;
 }
 
 int
-afr_zerofill (call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
-             size_t len, dict_t *xdata)
+afr_fsync_unwind(call_frame_t *frame, xlator_t *this)
 {
-        afr_local_t *local = NULL;
-        call_frame_t *transaction_frame = NULL;
-        int ret = -1;
-        int op_errno = ENOMEM;
+    afr_local_t *local = NULL;
+    call_frame_t *main_frame = NULL;
+
+    local = frame->local;
+
+    main_frame = afr_transaction_detach_fop_frame(frame);
+    if (!main_frame)
+        return 0;
+
+    AFR_STACK_UNWIND(fsync, main_frame, local->op_ret, local->op_errno,
+                     &local->cont.inode_wfop.prebuf,
+                     &local->cont.inode_wfop.postbuf, local->xdata_rsp);
 
-	transaction_frame = copy_frame (frame);
-	if (!transaction_frame)
-		goto out;
+    return 0;
+}
+
+int
+afr_fsync_wind_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                   int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
+                   struct iatt *postbuf, dict_t *xdata)
+{
+    return __afr_inode_write_cbk(frame, cookie, this, op_ret, op_errno, prebuf,
+                                 postbuf, NULL, xdata);
+}
 
-	local = AFR_FRAME_INIT (transaction_frame, op_errno);
-	if (!local)
-		goto out;
+int
+afr_fsync_wind(call_frame_t *frame, xlator_t *this, int subvol)
+{
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
 
-        local->cont.zerofill.offset  = offset;
-        local->cont.zerofill.len = len;
+    local = frame->local;
+    priv = this->private;
 
-        local->fd = fd_ref (fd);
-	local->inode = inode_ref (fd->inode);
+    STACK_WIND_COOKIE(frame, afr_fsync_wind_cbk, (void *)(long)subvol,
+                      priv->children[subvol],
+                      priv->children[subvol]->fops->fsync, local->fd,
+                      local->cont.fsync.datasync, local->xdata_req);
+    return 0;
+}
 
-	if (xdata)
-		local->xdata_req = dict_copy_with_ref (xdata, NULL);
-	else
-		local->xdata_req = dict_new ();
+int
+afr_fsync(call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t datasync,
+          dict_t *xdata)
+{
+    afr_local_t *local = NULL;
+    call_frame_t *transaction_frame = NULL;
+    int ret = -1;
+    int32_t op_errno = ENOMEM;
+    int8_t last_fsync = 0;
+
+    AFR_ERROR_OUT_IF_FDCTX_INVALID(fd, this, op_errno, out);
+    transaction_frame = copy_frame(frame);
+    if (!transaction_frame)
+        goto out;
+
+    local = AFR_FRAME_INIT(transaction_frame, op_errno);
+    if (!local)
+        goto out;
+
+    if (xdata) {
+        local->xdata_req = dict_copy_with_ref(xdata, NULL);
+        if (dict_get_int8(xdata, "last-fsync", &last_fsync) == 0) {
+            if (last_fsync) {
+                local->transaction.disable_delayed_post_op = _gf_true;
+            }
+        }
+    } else {
+        local->xdata_req = dict_new();
+    }
 
-	if (!local->xdata_req)
-		goto out;
+    if (!local->xdata_req)
+        goto out;
 
-        local->op = GF_FOP_ZEROFILL;
+    local->fd = fd_ref(fd);
+    ret = afr_set_inode_local(this, local, fd->inode);
+    if (ret)
+        goto out;
 
-        local->transaction.wind   = afr_zerofill_wind;
-        local->transaction.fop    = __afr_txn_write_fop;
-        local->transaction.done   = __afr_txn_write_done;
-        local->transaction.unwind = afr_zerofill_unwind;
+    local->op = GF_FOP_FSYNC;
+    local->cont.fsync.datasync = datasync;
 
-        local->transaction.main_frame = frame;
+    if (afr_fd_has_witnessed_unstable_write(this, fd->inode)) {
+        /* don't care. we only wanted to CLEAR the bit */
+    }
 
-        local->transaction.start   = local->cont.discard.offset;
-        local->transaction.len     = len;
+    local->transaction.wind = afr_fsync_wind;
+    local->transaction.unwind = afr_fsync_unwind;
 
-        afr_fix_open (fd, this);
+    local->transaction.main_frame = frame;
 
-        ret = afr_transaction (transaction_frame, this, AFR_DATA_TRANSACTION);
-        if (ret < 0) {
-		op_errno = -ret;
-		goto out;
-        }
+    ret = afr_transaction(transaction_frame, this, AFR_DATA_TRANSACTION);
+    if (ret < 0) {
+        op_errno = -ret;
+        goto out;
+    }
 
-	return 0;
+    return 0;
 out:
-	if (transaction_frame)
-		AFR_STACK_DESTROY (transaction_frame);
+    if (transaction_frame)
+        AFR_STACK_DESTROY(transaction_frame);
 
-	AFR_STACK_UNWIND (zerofill, frame, -1, op_errno, NULL, NULL, NULL);
-        return 0;
-}
+    AFR_STACK_UNWIND(fsync, frame, -1, op_errno, NULL, NULL, NULL);
 
-/* }}} */
+    return 0;
+}
diff --git a/xlators/cluster/afr/src/afr-inode-write.h b/xlators/cluster/afr/src/afr-inode-write.h
index 7b1fc552880..a787069b7a1 100644
--- a/xlators/cluster/afr/src/afr-inode-write.h
+++ b/xlators/cluster/afr/src/afr-inode-write.h
@@ -12,71 +12,83 @@
 #define __INODE_WRITE_H__
 
 int32_t
-afr_chmod (call_frame_t *frame, xlator_t *this,
-	   loc_t *loc, mode_t mode, dict_t *xdata);
+afr_chmod(call_frame_t *frame, xlator_t *this, loc_t *loc, mode_t mode,
+          dict_t *xdata);
 
 int32_t
-afr_chown (call_frame_t *frame, xlator_t *this,
-	   loc_t *loc, uid_t uid, gid_t gid, dict_t *xdata);
+afr_chown(call_frame_t *frame, xlator_t *this, loc_t *loc, uid_t uid, gid_t gid,
+          dict_t *xdata);
 
 int
-afr_fchown (call_frame_t *frame, xlator_t *this,
-	    fd_t *fd, uid_t uid, gid_t gid, dict_t *xdata);
+afr_fchown(call_frame_t *frame, xlator_t *this, fd_t *fd, uid_t uid, gid_t gid,
+           dict_t *xdata);
 
 int32_t
-afr_fchmod (call_frame_t *frame, xlator_t *this,
-	    fd_t *fd, mode_t mode, dict_t *xdata);
+afr_fchmod(call_frame_t *frame, xlator_t *this, fd_t *fd, mode_t mode,
+           dict_t *xdata);
 
 int32_t
-afr_writev (call_frame_t *frame, xlator_t *this, fd_t *fd,
-	    struct iovec *vector, int32_t count, off_t offset,
-            uint32_t flags, struct iobref *iobref, dict_t *xdata);
+afr_writev(call_frame_t *frame, xlator_t *this, fd_t *fd, struct iovec *vector,
+           int32_t count, off_t offset, uint32_t flags, struct iobref *iobref,
+           dict_t *xdata);
 
 int32_t
-afr_truncate (call_frame_t *frame, xlator_t *this,
-	      loc_t *loc, off_t offset, dict_t *xdata);
+afr_truncate(call_frame_t *frame, xlator_t *this, loc_t *loc, off_t offset,
+             dict_t *xdata);
 
 int32_t
-afr_ftruncate (call_frame_t *frame, xlator_t *this,
-	       fd_t *fd, off_t offset, dict_t *xdata);
+afr_ftruncate(call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
+              dict_t *xdata);
 
 int32_t
-afr_utimens (call_frame_t *frame, xlator_t *this,
-	     loc_t *loc, struct timespec tv[2], dict_t *xdata);
+afr_utimens(call_frame_t *frame, xlator_t *this, loc_t *loc,
+            struct timespec tv[2], dict_t *xdata);
 
 int
-afr_setattr (call_frame_t *frame, xlator_t *this,
-             loc_t *loc, struct iatt *buf, int32_t valid, dict_t *xdata);
+afr_setattr(call_frame_t *frame, xlator_t *this, loc_t *loc, struct iatt *buf,
+            int32_t valid, dict_t *xdata);
 
 int
-afr_fsetattr (call_frame_t *frame, xlator_t *this,
-              fd_t *fd, struct iatt *buf, int32_t valid, dict_t *xdata);
+afr_fsetattr(call_frame_t *frame, xlator_t *this, fd_t *fd, struct iatt *buf,
+             int32_t valid, dict_t *xdata);
 
 int32_t
-afr_setxattr (call_frame_t *frame, xlator_t *this,
-              loc_t *loc, dict_t *dict, int32_t flags, dict_t *xdata);
+afr_setxattr(call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *dict,
+             int32_t flags, dict_t *xdata);
 
 int32_t
-afr_fsetxattr (call_frame_t *frame, xlator_t *this,
-               fd_t *fd, dict_t *dict, int32_t flags, dict_t *xdata);
+afr_fsetxattr(call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *dict,
+              int32_t flags, dict_t *xdata);
 
 int32_t
-afr_removexattr (call_frame_t *frame, xlator_t *this,
-		 loc_t *loc, const char *name, dict_t *xdata);
+afr_removexattr(call_frame_t *frame, xlator_t *this, loc_t *loc,
+                const char *name, dict_t *xdata);
 
 int32_t
-afr_fremovexattr (call_frame_t *frame, xlator_t *this,
-                  fd_t *fd, const char *name, dict_t *xdata);
+afr_fremovexattr(call_frame_t *frame, xlator_t *this, fd_t *fd,
+                 const char *name, dict_t *xdata);
 
 int
-afr_discard (call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
-             size_t len, dict_t *xdata);
+afr_discard(call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
+            size_t len, dict_t *xdata);
 
 int
-afr_fallocate (call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t mode,
-               off_t offset, size_t len, dict_t *xdata);
+afr_fallocate(call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t mode,
+              off_t offset, size_t len, dict_t *xdata);
 
 int
 afr_zerofill(call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
              off_t len, dict_t *xdata);
+
+int32_t
+afr_xattrop(call_frame_t *frame, xlator_t *this, loc_t *loc,
+            gf_xattrop_flags_t optype, dict_t *xattr, dict_t *xdata);
+
+int32_t
+afr_fxattrop(call_frame_t *frame, xlator_t *this, fd_t *fd,
+             gf_xattrop_flags_t optype, dict_t *xattr, dict_t *xdata);
+
+int
+afr_fsync(call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t datasync,
+          dict_t *xdata);
 #endif /* __INODE_WRITE_H__ */
diff --git a/xlators/cluster/afr/src/afr-lk-common.c b/xlators/cluster/afr/src/afr-lk-common.c
index dc4dfbc42c0..bc8eabe0f43 100644
--- a/xlators/cluster/afr/src/afr-lk-common.c
+++ b/xlators/cluster/afr/src/afr-lk-common.c
@@ -8,1664 +8,784 @@
   cases as published by the Free Software Foundation.
 */
 
-#include "dict.h"
-#include "byte-order.h"
-#include "common-utils.h"
+#include <glusterfs/dict.h>
+#include <glusterfs/byte-order.h>
+#include <glusterfs/common-utils.h>
 
 #include "afr.h"
 #include "afr-transaction.h"
+#include "afr-messages.h"
 
 #include <signal.h>
 
-
-#define LOCKED_NO       0x0        /* no lock held */
-#define LOCKED_YES      0x1        /* for DATA, METADATA, ENTRY and higher_path */
-#define LOCKED_LOWER    0x2        /* for lower path */
-
-#define AFR_TRACE_INODELK_IN(frame, this, params ...)           \
-        do {                                                    \
-                afr_private_t *_priv = this->private;           \
-                if (!_priv->inodelk_trace)                      \
-                        break;                                  \
-                afr_trace_inodelk_in (frame, this, params);     \
-        } while (0);
-
-#define AFR_TRACE_INODELK_OUT(frame, this, params ...)          \
-        do {                                                    \
-                afr_private_t *_priv = this->private;           \
-                if (!_priv->inodelk_trace)                      \
-                        break;                                  \
-                afr_trace_inodelk_out (frame, this, params);    \
-        } while (0);
-
-#define AFR_TRACE_ENTRYLK_IN(frame, this, params ...)           \
-        do {                                                    \
-                afr_private_t *_priv = this->private;           \
-                if (!_priv->entrylk_trace)                      \
-                        break;                                  \
-                afr_trace_entrylk_in (frame, this, params);     \
-        } while (0);
-
-#define AFR_TRACE_ENTRYLK_OUT(frame, this, params ...)          \
-        do {                                                    \
-                afr_private_t *_priv = this->private;           \
-                if (!_priv->entrylk_trace)                      \
-                        break;                                  \
-                afr_trace_entrylk_out (frame, this, params);    \
-        } while (0);
-
-int
-afr_entry_lockee_cmp (const void *l1, const void *l2)
-{
-        const afr_entry_lockee_t       *r1 = l1;
-        const afr_entry_lockee_t       *r2 = l2;
-        int                            ret = 0;
-        uuid_t                         gfid1 = {0};
-        uuid_t                         gfid2 = {0};
-
-        loc_gfid ((loc_t*)&r1->loc, gfid1);
-        loc_gfid ((loc_t*)&r2->loc, gfid2);
-        ret = gf_uuid_compare (gfid1, gfid2);
-        /*Entrylks with NULL basename are the 'smallest'*/
-        if (ret == 0) {
-                if (!r1->basename)
-                        return -1;
-                if (!r2->basename)
-                        return 1;
-                ret = strcmp (r1->basename, r2->basename);
-        }
-
-        if (ret <= 0)
-                return -1;
-        else
-                return 1;
-}
-
-int afr_lock_blocking (call_frame_t *frame, xlator_t *this, int child_index);
-
-static int
-afr_copy_locked_nodes (call_frame_t *frame, xlator_t *this);
-
-static uint64_t afr_lock_number = 1;
-
-static uint64_t
-get_afr_lock_number ()
-{
-        return (++afr_lock_number);
-}
-
-int
-afr_set_lock_number (call_frame_t *frame, xlator_t *this)
-{
-        afr_local_t         *local    = NULL;
-        afr_internal_lock_t *int_lock = NULL;
-
-        local    = frame->local;
-        int_lock = &local->internal_lock;
-
-        int_lock->lock_number = get_afr_lock_number ();
-
-        return 0;
-}
+#define LOCKED_NO 0x0    /* no lock held */
+#define LOCKED_YES 0x1   /* for DATA, METADATA, ENTRY and higher_path */
+#define LOCKED_LOWER 0x2 /* for lower path */
 
 void
-afr_set_lk_owner (call_frame_t *frame, xlator_t *this, void *lk_owner)
-{
-        gf_log (this->name, GF_LOG_TRACE,
-                "Setting lk-owner=%llu",
-                (unsigned long long) (unsigned long)lk_owner);
-
-        set_lk_owner_from_ptr (&frame->root->lk_owner, lk_owner);
-}
-
-static int
-is_afr_lock_selfheal (afr_local_t *local)
-{
-        afr_internal_lock_t *int_lock = NULL;
-        int                  ret      = -1;
-
-        int_lock = &local->internal_lock;
-
-        switch (int_lock->selfheal_lk_type) {
-        case AFR_DATA_SELF_HEAL_LK:
-        case AFR_METADATA_SELF_HEAL_LK:
-                ret = 1;
-                break;
-        case AFR_ENTRY_SELF_HEAL_LK:
-                ret = 0;
-                break;
-        }
-
-        return ret;
-
-}
-
-int32_t
-internal_lock_count (call_frame_t *frame, xlator_t *this)
+afr_lockee_cleanup(afr_lockee_t *lockee)
 {
-        afr_local_t   *local = NULL;
-        afr_private_t *priv  = NULL;
-        int32_t call_count = 0;
-        int i = 0;
+    if (lockee->fd) {
+        fd_unref(lockee->fd);
+        lockee->fd = NULL;
+    } else {
+        loc_wipe(&lockee->loc);
+    }
 
-        local = frame->local;
-        priv  = this->private;
+    GF_FREE(lockee->basename);
+    lockee->basename = NULL;
+    GF_FREE(lockee->locked_nodes);
+    lockee->locked_nodes = NULL;
 
-        for (i = 0; i < priv->child_count; i++) {
-                if (local->child_up[i])
-                        ++call_count;
-        }
-
-        return call_count;
-}
-
-static void
-afr_print_inodelk (char *str, int size, int cmd,
-                   struct gf_flock *flock, gf_lkowner_t *owner)
-{
-        char *cmd_str = NULL;
-        char *type_str = NULL;
-
-        switch (cmd) {
-#if F_GETLK != F_GETLK64
-        case F_GETLK64:
-#endif
-        case F_GETLK:
-                cmd_str = "GETLK";
-                break;
-
-#if F_SETLK != F_SETLK64
-        case F_SETLK64:
-#endif
-        case F_SETLK:
-                cmd_str = "SETLK";
-                break;
-
-#if F_SETLKW != F_SETLKW64
-        case F_SETLKW64:
-#endif
-        case F_SETLKW:
-                cmd_str = "SETLKW";
-                break;
-
-        default:
-                cmd_str = "<null>";
-                break;
-        }
-
-        switch (flock->l_type) {
-        case F_RDLCK:
-                type_str = "READ";
-                break;
-        case F_WRLCK:
-                type_str = "WRITE";
-                break;
-        case F_UNLCK:
-                type_str = "UNLOCK";
-                break;
-        default:
-                type_str = "UNKNOWN";
-                break;
-        }
-
-        snprintf (str, size, "lock=INODELK, cmd=%s, type=%s, "
-                  "start=%llu, len=%llu, pid=%llu, lk-owner=%s",
-                  cmd_str, type_str, (unsigned long long) flock->l_start,
-                  (unsigned long long) flock->l_len,
-                  (unsigned long long) flock->l_pid,
-                  lkowner_utoa (owner));
-
-}
-
-static void
-afr_print_lockee (char *str, int size, loc_t *loc, fd_t *fd,
-                  int child_index)
-{
-        snprintf (str, size, "path=%s, fd=%p, child=%d",
-                  loc->path ? loc->path : "<nul>",
-                  fd ? fd : NULL,
-                  child_index);
+    return;
 }
 
 void
-afr_print_entrylk (char *str, int size, const char *basename,
-                   gf_lkowner_t *owner)
-{
-        snprintf (str, size, "Basename=%s, lk-owner=%s",
-                  basename ? basename : "<nul>",
-                  lkowner_utoa (owner));
-}
-
-static void
-afr_print_verdict (int op_ret, int op_errno, char *str)
+afr_lockees_cleanup(afr_internal_lock_t *int_lock)
 {
-        if (op_ret < 0) {
-                if (op_errno == EAGAIN)
-                        strcpy (str, "EAGAIN");
-                else
-                        strcpy (str, "FAILED");
-        }
-        else
-                strcpy (str, "GRANTED");
-}
+    int i = 0;
 
-static void
-afr_set_lock_call_type (afr_lock_call_type_t lock_call_type,
-                        char *lock_call_type_str,
-                        afr_internal_lock_t *int_lock)
-{
-        switch (lock_call_type) {
-        case AFR_INODELK_TRANSACTION:
-                if (int_lock->transaction_lk_type == AFR_TRANSACTION_LK)
-                        strcpy (lock_call_type_str, "AFR_INODELK_TRANSACTION");
-                else
-                        strcpy (lock_call_type_str, "AFR_INODELK_SELFHEAL");
-                break;
-        case AFR_INODELK_NB_TRANSACTION:
-                if (int_lock->transaction_lk_type == AFR_TRANSACTION_LK)
-                        strcpy (lock_call_type_str, "AFR_INODELK_NB_TRANSACTION");
-                else
-                        strcpy (lock_call_type_str, "AFR_INODELK_NB_SELFHEAL");
-                break;
-        case AFR_ENTRYLK_TRANSACTION:
-                if (int_lock->transaction_lk_type == AFR_TRANSACTION_LK)
-                        strcpy (lock_call_type_str, "AFR_ENTRYLK_TRANSACTION");
-                else
-                        strcpy (lock_call_type_str, "AFR_ENTRYLK_SELFHEAL");
-                break;
-        case AFR_ENTRYLK_NB_TRANSACTION:
-                if (int_lock->transaction_lk_type == AFR_TRANSACTION_LK)
-                        strcpy (lock_call_type_str, "AFR_ENTRYLK_NB_TRANSACTION");
-                else
-                        strcpy (lock_call_type_str, "AFR_ENTRYLK_NB_SELFHEAL");
-                break;
-        default:
-                strcpy (lock_call_type_str, "UNKNOWN");
-                break;
-        }
+    for (i = 0; i < int_lock->lockee_count; i++) {
+        afr_lockee_cleanup(&int_lock->lockee[i]);
+    }
 
+    return;
 }
-
-static void
-afr_trace_inodelk_out (call_frame_t *frame, xlator_t *this,
-                       afr_lock_call_type_t lock_call_type,
-                       afr_lock_op_type_t lk_op_type, struct gf_flock *flock,
-                       int op_ret, int op_errno, int32_t child_index)
-{
-        afr_internal_lock_t *int_lock = NULL;
-        afr_local_t         *local    = NULL;
-
-        char lockee[256];
-        char lock_call_type_str[256];
-        char verdict[16];
-
-        local    = frame->local;
-        int_lock = &local->internal_lock;
-
-        afr_print_lockee (lockee, 256, &local->loc, local->fd, child_index);
-
-        afr_set_lock_call_type (lock_call_type, lock_call_type_str, int_lock);
-
-        afr_print_verdict (op_ret, op_errno, verdict);
-
-        gf_log (this->name, GF_LOG_INFO,
-                "[%s %s] [%s] lk-owner=%s Lockee={%s} Number={%llu}",
-                lock_call_type_str,
-                lk_op_type == AFR_LOCK_OP ? "LOCK REPLY" : "UNLOCK REPLY",
-                verdict, lkowner_utoa (&frame->root->lk_owner), lockee,
-                (unsigned long long) int_lock->lock_number);
-
-}
-
-static void
-afr_trace_inodelk_in (call_frame_t *frame, xlator_t *this,
-                      afr_lock_call_type_t lock_call_type,
-                      afr_lock_op_type_t lk_op_type, struct gf_flock *flock,
-                      int32_t cmd, int32_t child_index)
-{
-        afr_local_t         *local    = NULL;
-        afr_internal_lock_t *int_lock = NULL;
-
-        char lock[256];
-        char lockee[256];
-        char lock_call_type_str[256];
-
-        local    = frame->local;
-        int_lock = &local->internal_lock;
-
-        afr_print_inodelk (lock, 256, cmd, flock, &frame->root->lk_owner);
-        afr_print_lockee (lockee, 256, &local->loc, local->fd, child_index);
-
-        afr_set_lock_call_type (lock_call_type, lock_call_type_str, int_lock);
-
-        gf_log (this->name, GF_LOG_INFO,
-                "[%s %s] Lock={%s} Lockee={%s} Number={%llu}",
-                lock_call_type_str,
-                lk_op_type == AFR_LOCK_OP ? "LOCK REQUEST" : "UNLOCK REQUEST",
-                lock, lockee,
-                (unsigned long long) int_lock->lock_number);
-
+int
+afr_entry_lockee_cmp(const void *l1, const void *l2)
+{
+    const afr_lockee_t *r1 = l1;
+    const afr_lockee_t *r2 = l2;
+    int ret = 0;
+    uuid_t gfid1 = {0};
+    uuid_t gfid2 = {0};
+
+    loc_gfid((loc_t *)&r1->loc, gfid1);
+    loc_gfid((loc_t *)&r2->loc, gfid2);
+    ret = gf_uuid_compare(gfid1, gfid2);
+    /*Entrylks with NULL basename are the 'smallest'*/
+    if (ret == 0) {
+        if (!r1->basename)
+            return -1;
+        if (!r2->basename)
+            return 1;
+        ret = strcmp(r1->basename, r2->basename);
+    }
+
+    if (ret <= 0)
+        return -1;
+    else
+        return 1;
 }
 
-static void
-afr_trace_entrylk_in (call_frame_t *frame, xlator_t *this,
-                      afr_lock_call_type_t lock_call_type,
-                      afr_lock_op_type_t lk_op_type, const char *basename,
-                      int32_t cookie)
-{
-        afr_local_t         *local    = NULL;
-        afr_internal_lock_t *int_lock = NULL;
-        afr_private_t       *priv     = NULL;
-        int                 child_index = 0;
-        int                 lockee_no = 0;
-
-        char lock[256];
-        char lockee[256];
-        char lock_call_type_str[256];
-
-        local    = frame->local;
-        int_lock = &local->internal_lock;
-        priv     = this->private;
-
-        if (!priv->entrylk_trace) {
-                return;
-        }
-        lockee_no = cookie / priv->child_count;
-        child_index = cookie % priv->child_count;
-
-        afr_print_entrylk (lock, 256, basename, &frame->root->lk_owner);
-        afr_print_lockee (lockee, 256, &int_lock->lockee[lockee_no].loc, local->fd,
-                          child_index);
-
-        afr_set_lock_call_type (lock_call_type, lock_call_type_str, int_lock);
-
-        gf_log (this->name, GF_LOG_INFO,
-                "[%s %s] Lock={%s} Lockee={%s} Number={%llu}, Cookie={%d}",
-                lock_call_type_str,
-                lk_op_type == AFR_LOCK_OP ? "LOCK REQUEST" : "UNLOCK REQUEST",
-                lock, lockee,
-                (unsigned long long) int_lock->lock_number,
-                cookie);
-}
+int
+afr_lock_blocking(call_frame_t *frame, xlator_t *this, int child_index);
 
-static void
-afr_trace_entrylk_out (call_frame_t *frame, xlator_t *this,
-                       afr_lock_call_type_t lock_call_type,
-                       afr_lock_op_type_t lk_op_type, const char *basename,
-                       int op_ret, int op_errno, int32_t cookie)
+void
+afr_set_lk_owner(call_frame_t *frame, xlator_t *this, void *lk_owner)
 {
-        afr_internal_lock_t *int_lock = NULL;
-        afr_local_t         *local    = NULL;
-        afr_private_t       *priv     = NULL;
-        int                 lockee_no = 0;
-        int                 child_index = 0;
-
-        char lock[256];
-        char lockee[256];
-        char lock_call_type_str[256];
-        char verdict[16];
-
-        local    = frame->local;
-        int_lock = &local->internal_lock;
-        priv     = this->private;
-
-        if (!priv->entrylk_trace) {
-                return;
-        }
-        lockee_no = cookie / priv->child_count;
-        child_index = cookie % priv->child_count;
-
-        afr_print_entrylk (lock, 256, basename, &frame->root->lk_owner);
-        afr_print_lockee (lockee, 256, &int_lock->lockee[lockee_no].loc, local->fd,
-                          child_index);
-
-        afr_set_lock_call_type (lock_call_type, lock_call_type_str, int_lock);
-
-        afr_print_verdict (op_ret, op_errno, verdict);
-
-        gf_log (this->name, GF_LOG_INFO,
-                "[%s %s] [%s] Lock={%s} Lockee={%s} Number={%llu} Cookie={%d}",
-                lock_call_type_str,
-                lk_op_type == AFR_LOCK_OP ? "LOCK REPLY" : "UNLOCK REPLY",
-                verdict,
-                lock, lockee,
-                (unsigned long long) int_lock->lock_number,
-                cookie);
+    gf_msg_trace(this->name, 0, "Setting lk-owner=%llu",
+                 (unsigned long long)(unsigned long)lk_owner);
 
+    set_lk_owner_from_ptr(&frame->root->lk_owner, lk_owner);
 }
 
-static int
-transaction_lk_op (afr_local_t *local)
+int32_t
+internal_lock_count(call_frame_t *frame, xlator_t *this)
 {
-        afr_internal_lock_t *int_lock = NULL;
-        int ret = -1;
-
-        int_lock = &local->internal_lock;
-
-        if (int_lock->transaction_lk_type == AFR_TRANSACTION_LK) {
-                gf_log (THIS->name, GF_LOG_DEBUG,
-                        "lk op is for a transaction");
-                ret = 1;
-        }
-        else if (int_lock->transaction_lk_type == AFR_SELFHEAL_LK) {
-                gf_log (THIS->name, GF_LOG_DEBUG,
-                        "lk op is for a self heal");
-
-                ret = 0;
-        }
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
+    int32_t call_count = 0;
+    int i = 0;
 
-        if (ret == -1)
-                gf_log (THIS->name, GF_LOG_DEBUG,
-                        "lk op is not set");
+    local = frame->local;
+    priv = this->private;
 
-        return ret;
+    for (i = 0; i < priv->child_count; i++) {
+        if (local->child_up[i])
+            ++call_count;
+    }
 
+    return call_count;
 }
 
-static int
-is_afr_lock_transaction (afr_local_t *local)
+int
+afr_add_entry_lockee(afr_local_t *local, loc_t *loc, char *basename,
+                     int child_count)
 {
-        int ret = 0;
+    int ret = -ENOMEM;
+    afr_internal_lock_t *int_lock = &local->internal_lock;
+    afr_lockee_t *lockee = &int_lock->lockee[int_lock->lockee_count];
 
-        switch (local->transaction.type) {
-        case AFR_DATA_TRANSACTION:
-        case AFR_METADATA_TRANSACTION:
-                ret = 1;
-                break;
+    GF_ASSERT(int_lock->lockee_count < AFR_LOCKEE_COUNT_MAX);
+    loc_copy(&lockee->loc, loc);
+    lockee->basename = (basename) ? gf_strdup(basename) : NULL;
+    if (basename && !lockee->basename)
+        goto out;
 
-        case AFR_ENTRY_RENAME_TRANSACTION:
-        case AFR_ENTRY_TRANSACTION:
-                ret = 0;
-                break;
+    lockee->locked_count = 0;
+    lockee->locked_nodes = GF_CALLOC(child_count, sizeof(*lockee->locked_nodes),
+                                     gf_afr_mt_afr_node_character);
 
-        }
+    if (!lockee->locked_nodes)
+        goto out;
 
-        return ret;
+    ret = 0;
+    int_lock->lockee_count++;
+out:
+    if (ret) {
+        afr_lockee_cleanup(lockee);
+    }
+    return ret;
 }
 
 int
-afr_init_entry_lockee (afr_entry_lockee_t *lockee, afr_local_t *local,
-                       loc_t *loc, char *basename, int child_count)
+afr_add_inode_lockee(afr_local_t *local, int child_count)
 {
-        int     ret     = -1;
+    int ret = -ENOMEM;
+    afr_internal_lock_t *int_lock = &local->internal_lock;
+    afr_lockee_t *lockee = &int_lock->lockee[int_lock->lockee_count];
 
-        loc_copy (&lockee->loc, loc);
-        lockee->basename        = (basename)? gf_strdup (basename): NULL;
-        if (basename && !lockee->basename)
-                goto out;
+    if (local->fd) {
+        lockee->fd = fd_ref(local->fd);
+    } else {
+        loc_copy(&lockee->loc, &local->loc);
+    }
 
-        lockee->locked_count    = 0;
-        lockee->locked_nodes    = GF_CALLOC (child_count,
-                                             sizeof (*lockee->locked_nodes),
-                                             gf_afr_mt_afr_node_character);
+    lockee->locked_count = 0;
+    lockee->locked_nodes = GF_CALLOC(child_count, sizeof(*lockee->locked_nodes),
+                                     gf_afr_mt_afr_node_character);
 
-        if (!lockee->locked_nodes)
-                goto out;
+    if (!lockee->locked_nodes)
+        goto out;
 
-        ret = 0;
+    ret = 0;
+    int_lock->lockee_count++;
 out:
-        return ret;
-
-}
-
-void
-afr_entry_lockee_cleanup (afr_internal_lock_t *int_lock)
-{
-        int     i   = 0;
-
-        for (i = 0; i < int_lock->lockee_count; i++) {
-                loc_wipe (&int_lock->lockee[i].loc);
-                if (int_lock->lockee[i].basename)
-                        GF_FREE (int_lock->lockee[i].basename);
-                if (int_lock->lockee[i].locked_nodes)
-                       GF_FREE (int_lock->lockee[i].locked_nodes);
-        }
-
-        return;
-}
-
-static int
-initialize_entrylk_variables (call_frame_t *frame, xlator_t *this)
-{
-        afr_local_t         *local    = NULL;
-        afr_internal_lock_t *int_lock = NULL;
-        afr_private_t       *priv     = NULL;
-
-        int i = 0;
-
-        priv     = this->private;
-        local    = frame->local;
-        int_lock = &local->internal_lock;
-
-        int_lock->entrylk_lock_count = 0;
-        int_lock->lock_op_ret        = -1;
-        int_lock->lock_op_errno      = 0;
-
-        for (i = 0; i < AFR_LOCKEE_COUNT_MAX; i++) {
-                if (!int_lock->lockee[i].locked_nodes)
-                        break;
-                int_lock->lockee[i].locked_count = 0;
-                memset (int_lock->lockee[i].locked_nodes, 0,
-                        sizeof (*int_lock->lockee[i].locked_nodes) *
-                        priv->child_count);
-        }
-
-        return 0;
+    if (ret) {
+        afr_lockee_cleanup(lockee);
+    }
+    return ret;
 }
 
 static int
-initialize_inodelk_variables (call_frame_t *frame, xlator_t *this)
+initialize_internal_lock_variables(call_frame_t *frame, xlator_t *this)
 {
-        afr_local_t         *local    = NULL;
-        afr_internal_lock_t *int_lock = NULL;
-        afr_private_t       *priv     = NULL;
-        afr_inodelk_t       *inodelk  = NULL;
+    afr_local_t *local = NULL;
+    afr_internal_lock_t *int_lock = NULL;
+    afr_private_t *priv = NULL;
 
-        priv     = this->private;
-        local    = frame->local;
-        int_lock = &local->internal_lock;
+    int i = 0;
 
-        inodelk = afr_get_inodelk (int_lock, int_lock->domain);
+    priv = this->private;
+    local = frame->local;
+    int_lock = &local->internal_lock;
 
-        inodelk->lock_count    = 0;
-        int_lock->lk_attempted_count = 0;
-        int_lock->lock_op_ret   = -1;
-        int_lock->lock_op_errno = 0;
+    int_lock->lock_count = 0;
+    int_lock->lock_op_ret = -1;
+    int_lock->lock_op_errno = 0;
+    int_lock->lk_attempted_count = 0;
 
-        memset (inodelk->locked_nodes, 0,
-                sizeof (*inodelk->locked_nodes) * priv->child_count);
-        memset (int_lock->locked_nodes, 0,
-                sizeof (*int_lock->locked_nodes) * priv->child_count);
+    for (i = 0; i < AFR_LOCKEE_COUNT_MAX; i++) {
+        if (!int_lock->lockee[i].locked_nodes)
+            break;
+        int_lock->lockee[i].locked_count = 0;
+        memset(int_lock->lockee[i].locked_nodes, 0,
+               sizeof(*int_lock->lockee[i].locked_nodes) * priv->child_count);
+    }
 
-        return 0;
+    return 0;
 }
 
 int
-afr_lockee_locked_nodes_count (afr_internal_lock_t *int_lock)
+afr_lockee_locked_nodes_count(afr_internal_lock_t *int_lock)
 {
-        int call_count  = 0;
-        int i           = 0;
+    int call_count = 0;
+    int i = 0;
 
-        for (i = 0; i < int_lock->lockee_count; i++)
-                call_count += int_lock->lockee[i].locked_count;
+    for (i = 0; i < int_lock->lockee_count; i++)
+        call_count += int_lock->lockee[i].locked_count;
 
-        return call_count;
+    return call_count;
 }
 
 int
-afr_locked_nodes_count (unsigned char *locked_nodes, int child_count)
+afr_locked_nodes_count(unsigned char *locked_nodes, int child_count)
 
 {
-        int i = 0;
-        int call_count = 0;
+    int i = 0;
+    int call_count = 0;
 
-        for (i = 0; i < child_count; i++) {
-                if (locked_nodes[i] & LOCKED_YES)
-                        call_count++;
-        }
+    for (i = 0; i < child_count; i++) {
+        if (locked_nodes[i] & LOCKED_YES)
+            call_count++;
+    }
 
-        return call_count;
+    return call_count;
 }
 
-/* FIXME: What if UNLOCK fails */
-static int32_t
-afr_unlock_common_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                       int32_t op_ret, int32_t op_errno, dict_t *xdata)
+static void
+afr_log_locks_failure(call_frame_t *frame, char *where, char *what,
+                      int op_errno)
 {
-        afr_local_t         *local    = NULL;
-        afr_internal_lock_t *int_lock = NULL;
-        int call_count = 0;
-
-        local    = frame->local;
-        int_lock = &local->internal_lock;
-
-        LOCK (&frame->lock);
-        {
-                call_count = --int_lock->lk_call_count;
-        }
-        UNLOCK (&frame->lock);
+    xlator_t *this = frame->this;
+    gf_lkowner_t *lk_owner = &frame->root->lk_owner;
+    afr_local_t *local = frame->local;
+    const char *fop = NULL;
+    char *gfid = NULL;
+    const char *name = NULL;
 
-        if (call_count == 0) {
-                gf_log (this->name, GF_LOG_TRACE,
-                        "All internal locks unlocked");
-                int_lock->lock_cbk (frame, this);
-        }
+    fop = gf_fop_list[local->op];
 
-        return 0;
+    switch (local->transaction.type) {
+        case AFR_ENTRY_RENAME_TRANSACTION:
+        case AFR_ENTRY_TRANSACTION:
+            switch (local->op) {
+                case GF_FOP_LINK:
+                    gfid = uuid_utoa(local->newloc.pargfid);
+                    name = local->newloc.name;
+                    break;
+                default:
+                    gfid = uuid_utoa(local->loc.pargfid);
+                    name = local->loc.name;
+                    break;
+            }
+            gf_msg(this->name, GF_LOG_WARNING, op_errno,
+                   AFR_MSG_INTERNAL_LKS_FAILED,
+                   "Unable to do entry %s with lk-owner:%s on %s "
+                   "while attempting %s on {pgfid:%s, name:%s}.",
+                   what, lkowner_utoa(lk_owner), where, fop, gfid, name);
+            break;
+        case AFR_DATA_TRANSACTION:
+        case AFR_METADATA_TRANSACTION:
+            gfid = uuid_utoa(local->inode->gfid);
+            gf_msg(this->name, GF_LOG_WARNING, op_errno,
+                   AFR_MSG_INTERNAL_LKS_FAILED,
+                   "Unable to do inode %s with lk-owner:%s on %s "
+                   "while attempting %s on gfid:%s.",
+                   what, lkowner_utoa(lk_owner), where, fop, gfid);
+            break;
+    }
 }
 
 static int32_t
-afr_unlock_inodelk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                        int32_t op_ret, int32_t op_errno, dict_t *xdata)
+afr_unlock_common_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                      int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
-        afr_local_t         *local = NULL;
-        afr_internal_lock_t *int_lock = NULL;
-        afr_inodelk_t       *inodelk = NULL;
-        int32_t             child_index = (long)cookie;
-        afr_private_t       *priv = NULL;
-
-        local = frame->local;
-        int_lock = &local->internal_lock;
-
-        AFR_TRACE_INODELK_OUT (frame, this, AFR_INODELK_TRANSACTION,
-                               AFR_UNLOCK_OP, NULL, op_ret,
-                               op_errno, child_index);
-
-        priv = this->private;
-
-        if (op_ret < 0 && op_errno != ENOTCONN && op_errno != EBADFD) {
-                gf_msg (this->name, GF_LOG_ERROR, op_errno,
-                        AFR_MSG_INODE_UNLOCK_FAIL,
-                        "path=%s gfid=%s: unlock failed on subvolume %s "
-                        "with lock owner %s", local->loc.path,
-                        loc_gfid_utoa (&(local->loc)),
-                        priv->children[child_index]->name,
-                        lkowner_utoa (&frame->root->lk_owner));
-        }
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
+    afr_internal_lock_t *int_lock = NULL;
+    int lockee_num = 0;
+    int call_count = 0;
+    int child_index = 0;
+    int ret = 0;
 
+    local = frame->local;
+    int_lock = &local->internal_lock;
+    priv = this->private;
+    lockee_num = (int)((long)cookie) / priv->child_count;
+    child_index = (int)((long)cookie) % priv->child_count;
 
-        inodelk = afr_get_inodelk (int_lock, int_lock->domain);
-        inodelk->locked_nodes[child_index] &= LOCKED_NO;
-        if (local->transaction.eager_lock)
-                local->transaction.eager_lock[child_index] = 0;
+    if (op_ret < 0 && op_errno != ENOTCONN && op_errno != EBADFD) {
+        afr_log_locks_failure(frame, priv->children[child_index]->name,
+                              "unlock", op_errno);
+    }
 
-        afr_unlock_common_cbk (frame, cookie, this, op_ret, op_errno, xdata);
+    int_lock->lockee[lockee_num].locked_nodes[child_index] &= LOCKED_NO;
+    if (local->transaction.type == AFR_DATA_TRANSACTION && op_ret != 1)
+        ret = afr_write_subvol_reset(frame, this);
 
-        return 0;
+    LOCK(&frame->lock);
+    {
+        call_count = --int_lock->lk_call_count;
+    }
+    UNLOCK(&frame->lock);
 
-}
+    if (call_count == 0) {
+        int_lock->lock_cbk(frame, this);
+    }
 
-static int
-afr_unlock_inodelk (call_frame_t *frame, xlator_t *this)
-{
-        afr_internal_lock_t *int_lock = NULL;
-        afr_inodelk_t       *inodelk  = NULL;
-        afr_local_t         *local    = NULL;
-        afr_private_t       *priv     = NULL;
-        struct gf_flock flock = {0,};
-        struct gf_flock full_flock = {0,};
-        struct gf_flock *flock_use = NULL;
-        int call_count = 0;
-        int i = 0;
-        int piggyback = 0;
-        afr_fd_ctx_t        *fd_ctx      = NULL;
-
-
-        local    = frame->local;
-        int_lock = &local->internal_lock;
-        priv     = this->private;
-
-        inodelk = afr_get_inodelk (int_lock, int_lock->domain);
-
-        flock.l_start = inodelk->flock.l_start;
-        flock.l_len   = inodelk->flock.l_len;
-        flock.l_type  = F_UNLCK;
-
-        full_flock.l_type = F_UNLCK;
-        call_count = afr_locked_nodes_count (inodelk->locked_nodes,
-                                             priv->child_count);
-
-        int_lock->lk_call_count = call_count;
-
-        if (!call_count) {
-                gf_log (this->name, GF_LOG_TRACE,
-                        "No internal locks unlocked");
-                int_lock->lock_cbk (frame, this);
-                goto out;
-        }
-
-        if (local->fd)
-                fd_ctx = afr_fd_ctx_get (local->fd, this);
-
-        for (i = 0; i < priv->child_count; i++) {
-                if ((inodelk->locked_nodes[i] & LOCKED_YES) != LOCKED_YES)
-                        continue;
-
-                if (local->fd) {
-                        flock_use = &flock;
-                        if (!local->transaction.eager_lock[i]) {
-                                goto wind;
-                        }
-
-                        piggyback = 0;
-
-                        LOCK (&local->fd->lock);
-                        {
-                                if (fd_ctx->lock_piggyback[i]) {
-                                        fd_ctx->lock_piggyback[i]--;
-                                        piggyback = 1;
-                                } else {
-                                        fd_ctx->lock_acquired[i]--;
-                                }
-                        }
-                        UNLOCK (&local->fd->lock);
-
-                        if (piggyback) {
-                                afr_unlock_inodelk_cbk (frame, (void *) (long) i,
-                                                        this, 1, 0, NULL);
-                                if (!--call_count)
-                                        break;
-                                continue;
-                        }
-
-                        flock_use = &full_flock;
-                wind:
-                        AFR_TRACE_INODELK_IN (frame, this,
-                                              AFR_INODELK_TRANSACTION,
-                                              AFR_UNLOCK_OP, flock_use, F_SETLK,
-                                              i);
-
-                        STACK_WIND_COOKIE (frame, afr_unlock_inodelk_cbk,
-                                           (void *) (long)i,
-                                           priv->children[i],
-                                           priv->children[i]->fops->finodelk,
-                                           int_lock->domain, local->fd,
-                                           F_SETLK, flock_use, NULL);
-
-                        if (!--call_count)
-                                break;
-
-                } else {
-                        AFR_TRACE_INODELK_IN (frame, this,
-                                              AFR_INODELK_TRANSACTION,
-                                              AFR_UNLOCK_OP, &flock, F_SETLK, i);
-
-                        STACK_WIND_COOKIE (frame, afr_unlock_inodelk_cbk,
-                                           (void *) (long)i,
-                                           priv->children[i],
-                                           priv->children[i]->fops->inodelk,
-                                           int_lock->domain, &local->loc,
-                                           F_SETLK, &flock, NULL);
-
-                        if (!--call_count)
-                                break;
-                }
-        }
-out:
-        return 0;
+    return ret;
 }
 
-static int32_t
-afr_unlock_entrylk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                        int32_t op_ret, int32_t op_errno, dict_t *xdata)
-{
-        afr_local_t         *local = NULL;
-        afr_private_t       *priv  = NULL;
-        afr_internal_lock_t *int_lock = NULL;
-        int32_t             child_index = 0;
-        int                 lockee_no   = 0;
-
-        priv = this->private;
-        lockee_no = (int)((long) cookie) / priv->child_count;
-        child_index = (int) ((long) cookie) % priv->child_count;
-
-        local = frame->local;
-        int_lock = &local->internal_lock;
-
-        AFR_TRACE_ENTRYLK_OUT (frame, this, AFR_ENTRYLK_TRANSACTION,
-                               AFR_UNLOCK_OP,
-                               int_lock->lockee[lockee_no].basename, op_ret,
-                               op_errno, (int) ((long)cookie));
-
-        if (op_ret < 0) {
-                gf_msg (this->name, GF_LOG_ERROR, op_errno,
-                        AFR_MSG_ENTRY_UNLOCK_FAIL,
-                        "%s: unlock failed on %s", local->loc.path,
-                        priv->children[child_index]->name);
-        }
-
-        int_lock->lockee[lockee_no].locked_nodes[child_index] &= LOCKED_NO;
-        afr_unlock_common_cbk (frame, cookie, this, op_ret, op_errno, NULL);
+void
+afr_internal_lock_wind(call_frame_t *frame,
+                       int32_t (*cbk)(call_frame_t *, void *, xlator_t *,
+                                      int32_t, int32_t, dict_t *),
+                       void *cookie, int child, int lockee_num,
+                       gf_boolean_t blocking, gf_boolean_t unlock)
+{
+    afr_local_t *local = frame->local;
+    xlator_t *this = frame->this;
+    afr_private_t *priv = this->private;
+    afr_internal_lock_t *int_lock = &local->internal_lock;
+    entrylk_cmd cmd = ENTRYLK_LOCK_NB;
+    int32_t cmd1 = F_SETLK;
+    struct gf_flock flock = {
+        0,
+    };
+
+    switch (local->transaction.type) {
+        case AFR_ENTRY_TRANSACTION:
+        case AFR_ENTRY_RENAME_TRANSACTION:
+            if (unlock) {
+                cmd = ENTRYLK_UNLOCK;
+            } else if (blocking) { /*Doesn't make sense to have blocking
+                                      unlock*/
+                cmd = ENTRYLK_LOCK;
+            }
+
+            if (local->fd) {
+                STACK_WIND_COOKIE(frame, cbk, cookie, priv->children[child],
+                                  priv->children[child]->fops->fentrylk,
+                                  int_lock->domain,
+                                  int_lock->lockee[lockee_num].fd,
+                                  int_lock->lockee[lockee_num].basename, cmd,
+                                  ENTRYLK_WRLCK, NULL);
+            } else {
+                STACK_WIND_COOKIE(frame, cbk, cookie, priv->children[child],
+                                  priv->children[child]->fops->entrylk,
+                                  int_lock->domain,
+                                  &int_lock->lockee[lockee_num].loc,
+                                  int_lock->lockee[lockee_num].basename, cmd,
+                                  ENTRYLK_WRLCK, NULL);
+            }
+            break;
 
-        return 0;
+        case AFR_DATA_TRANSACTION:
+        case AFR_METADATA_TRANSACTION:
+            flock = int_lock->lockee[lockee_num].flock;
+            if (unlock) {
+                flock.l_type = F_UNLCK;
+            } else if (blocking) { /*Doesn't make sense to have blocking
+                                      unlock*/
+                cmd1 = F_SETLKW;
+            }
+
+            if (local->fd) {
+                STACK_WIND_COOKIE(
+                    frame, cbk, cookie, priv->children[child],
+                    priv->children[child]->fops->finodelk, int_lock->domain,
+                    int_lock->lockee[lockee_num].fd, cmd1, &flock, NULL);
+            } else {
+                STACK_WIND_COOKIE(
+                    frame, cbk, cookie, priv->children[child],
+                    priv->children[child]->fops->inodelk, int_lock->domain,
+                    &int_lock->lockee[lockee_num].loc, cmd1, &flock, NULL);
+            }
+            break;
+    }
 }
 
 static int
-afr_unlock_entrylk (call_frame_t *frame, xlator_t *this)
-{
-        afr_internal_lock_t     *int_lock       = NULL;
-        afr_local_t             *local          = NULL;
-        afr_private_t           *priv           = NULL;
-        int                     call_count      = 0;
-        int                     index           = 0;
-        int                     lockee_no       = 0;
-        int                     copies          = 0;
-        int                     i               = -1;
-
-        local    = frame->local;
-        int_lock = &local->internal_lock;
-        priv     = this->private;
-        copies   = priv->child_count;
-
-        call_count = afr_lockee_locked_nodes_count (int_lock);
-
-        int_lock->lk_call_count = call_count;
-
-        if (!call_count){
-                gf_log (this->name, GF_LOG_TRACE,
-                        "No internal locks unlocked");
-                int_lock->lock_cbk (frame, this);
-                goto out;
-        }
-
-        for (i = 0; i < int_lock->lockee_count * priv->child_count; i++) {
-                lockee_no = i / copies;
-                index     = i % copies;
-                if (int_lock->lockee[lockee_no].locked_nodes[index] & LOCKED_YES) {
-                        AFR_TRACE_ENTRYLK_IN (frame, this, AFR_ENTRYLK_NB_TRANSACTION,
-                                              AFR_UNLOCK_OP,
-                                              int_lock->lockee[lockee_no].basename,
-                                              i);
-
-                        STACK_WIND_COOKIE (frame, afr_unlock_entrylk_cbk,
-                                           (void *) (long) i,
-                                           priv->children[index],
-                                           priv->children[index]->fops->entrylk,
-                                           int_lock->domain,
-                                           &int_lock->lockee[lockee_no].loc,
-                                           int_lock->lockee[lockee_no].basename,
-                                           ENTRYLK_UNLOCK, ENTRYLK_WRLCK, NULL);
-
-                        if (!--call_count)
-                                break;
-                }
+afr_unlock_now(call_frame_t *frame, xlator_t *this)
+{
+    afr_internal_lock_t *int_lock = NULL;
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
+    int call_count = 0;
+    int child_index = 0;
+    int lockee_num = 0;
+    int i = -1;
+
+    local = frame->local;
+    int_lock = &local->internal_lock;
+    priv = this->private;
+
+    call_count = afr_lockee_locked_nodes_count(int_lock);
+
+    int_lock->lk_call_count = call_count;
+
+    if (!call_count) {
+        gf_msg_trace(this->name, 0, "No internal locks unlocked");
+        int_lock->lock_cbk(frame, this);
+        goto out;
+    }
+
+    for (i = 0; i < int_lock->lockee_count * priv->child_count; i++) {
+        lockee_num = i / priv->child_count;
+        child_index = i % priv->child_count;
+        if (int_lock->lockee[lockee_num].locked_nodes[child_index] &
+            LOCKED_YES) {
+            afr_internal_lock_wind(frame, afr_unlock_common_cbk,
+                                   (void *)(long)i, child_index, lockee_num,
+                                   _gf_false, _gf_true);
+            if (!--call_count)
+                break;
         }
+    }
 
 out:
-        return 0;
-
+    return 0;
 }
 
 static int32_t
-afr_lock_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-              int32_t op_ret, int32_t op_errno, dict_t *xdata)
-{
-        afr_internal_lock_t     *int_lock       = NULL;
-        afr_local_t             *local          = NULL;
-        afr_private_t           *priv           = NULL;
-        int                     cky             = (long) cookie;
-        int                     child_index     = 0;
-        int                     lockee_no       = 0;
-
-        priv     = this->private;
-        local    = frame->local;
-        int_lock = &local->internal_lock;
-
-        child_index = ((int)cky) % priv->child_count;
-        lockee_no   = ((int)cky) / priv->child_count;
-
-        LOCK (&frame->lock);
-        {
-                if (op_ret == -1) {
-                        if (op_errno == ENOSYS) {
-                                /* return ENOTSUP */
-                                gf_log (this->name, GF_LOG_ERROR,
-                                        "subvolume does not support locking. "
-                                        "please load features/locks xlator on server");
-                                local->op_ret = op_ret;
-                                int_lock->lock_op_ret = op_ret;
-                        }
-
-                        local->op_errno              = op_errno;
-                        int_lock->lock_op_errno      = op_errno;
-                }
-
-		int_lock->lk_attempted_count++;
-        }
-        UNLOCK (&frame->lock);
-
-        if ((op_ret == -1) &&
-            (op_errno == ENOSYS)) {
-                afr_unlock (frame, this);
-        } else {
-                if (op_ret == 0) {
-                        if (local->transaction.type == AFR_ENTRY_TRANSACTION ||
-                            local->transaction.type == AFR_ENTRY_RENAME_TRANSACTION) {
-                                int_lock->lockee[lockee_no].locked_nodes[child_index] |= LOCKED_YES;
-                                int_lock->lockee[lockee_no].locked_count++;
-                                int_lock->entrylk_lock_count++;
-                        } else {
-                                int_lock->locked_nodes[child_index] |= LOCKED_YES;
-                                int_lock->lock_count++;
-                        }
+afr_lock_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int32_t op_ret,
+             int32_t op_errno, dict_t *xdata)
+{
+    afr_internal_lock_t *int_lock = NULL;
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
+    int cky = (long)cookie;
+    int child_index = 0;
+    int lockee_num = 0;
+
+    priv = this->private;
+    local = frame->local;
+    int_lock = &local->internal_lock;
+
+    child_index = ((int)cky) % priv->child_count;
+    lockee_num = ((int)cky) / priv->child_count;
+
+    LOCK(&frame->lock);
+    {
+        if (op_ret == -1) {
+            if (op_errno == ENOSYS) {
+                /* return ENOTSUP */
+                gf_msg(this->name, GF_LOG_ERROR, ENOSYS,
+                       AFR_MSG_LOCK_XLATOR_NOT_LOADED,
+                       "subvolume does not support locking. "
+                       "please load features/locks xlator on server");
+                local->op_ret = op_ret;
+                int_lock->lock_op_ret = op_ret;
+            }
+
+            local->op_errno = op_errno;
+            int_lock->lock_op_errno = op_errno;
+        }
+
+        int_lock->lk_attempted_count++;
+    }
+    UNLOCK(&frame->lock);
+
+    if ((op_ret == -1) && (op_errno == ENOSYS)) {
+        afr_unlock_now(frame, this);
+    } else {
+        if (op_ret == 0) {
+            int_lock->lockee[lockee_num]
+                .locked_nodes[child_index] |= LOCKED_YES;
+            int_lock->lockee[lockee_num].locked_count++;
+            int_lock->lock_count++;
+            if (local->transaction.type == AFR_DATA_TRANSACTION) {
+                LOCK(&local->inode->lock);
+                {
+                    local->inode_ctx->lock_count++;
                 }
-                afr_lock_blocking (frame, this, cky + 1);
-        }
-
-        return 0;
-}
-
-static int32_t
-afr_blocking_inodelk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                          int32_t op_ret, int32_t op_errno, dict_t *xdata)
-{
-        AFR_TRACE_INODELK_OUT (frame, this, AFR_INODELK_TRANSACTION,
-                               AFR_LOCK_OP, NULL, op_ret,
-                               op_errno, (long) cookie);
-
-        afr_lock_cbk (frame, cookie, this, op_ret, op_errno, xdata);
-        return 0;
-
-}
-
-static int32_t
-afr_blocking_entrylk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                          int32_t op_ret, int32_t op_errno, dict_t *xdata)
-{
-        AFR_TRACE_ENTRYLK_OUT (frame, this, AFR_ENTRYLK_TRANSACTION,
-                               AFR_LOCK_OP, NULL, op_ret,
-                               op_errno, (long)cookie);
-
-        afr_lock_cbk (frame, cookie, this, op_ret, op_errno, xdata);
-        return 0;
-}
-
-static int
-afr_copy_locked_nodes (call_frame_t *frame, xlator_t *this)
-{
-        afr_internal_lock_t *int_lock = NULL;
-        afr_inodelk_t       *inodelk  = NULL;
-        afr_local_t         *local    = NULL;
-        afr_private_t       *priv     = NULL;
-
-        priv     = this->private;
-        local    = frame->local;
-        int_lock = &local->internal_lock;
-
-        switch (local->transaction.type) {
-        case AFR_DATA_TRANSACTION:
-        case AFR_METADATA_TRANSACTION:
-                inodelk = afr_get_inodelk (int_lock, int_lock->domain);
-                memcpy (inodelk->locked_nodes, int_lock->locked_nodes,
-                        sizeof (*inodelk->locked_nodes) * priv->child_count);
-                inodelk->lock_count = int_lock->lock_count;
-                break;
-
-        case AFR_ENTRY_RENAME_TRANSACTION:
-        case AFR_ENTRY_TRANSACTION:
-                /*entrylk_count is being used in both non-blocking and blocking
-                 * modes */
-                break;
+                UNLOCK(&local->inode->lock);
+            }
         }
+        afr_lock_blocking(frame, this, cky + 1);
+    }
 
-        return 0;
-
+    return 0;
 }
 
-static inline gf_boolean_t
-afr_is_entrylk (afr_internal_lock_t *int_lock,
-                afr_transaction_type trans_type)
+static gf_boolean_t
+_is_lock_wind_needed(afr_local_t *local, int child_index)
 {
-        gf_boolean_t is_entrylk = _gf_false;
-
-        if ((int_lock->transaction_lk_type == AFR_SELFHEAL_LK) &&
-            int_lock->selfheal_lk_type == AFR_ENTRY_SELF_HEAL_LK) {
+    if (!local->child_up[child_index])
+        return _gf_false;
 
-                is_entrylk = _gf_true;
-
-        } else if ((int_lock->transaction_lk_type == AFR_TRANSACTION_LK) &&
-                 (trans_type == AFR_ENTRY_TRANSACTION ||
-                  trans_type == AFR_ENTRY_RENAME_TRANSACTION)) {
-
-                is_entrylk = _gf_true;
-
-        } else {
-                is_entrylk = _gf_false;
-        }
-
-        return is_entrylk;
+    return _gf_true;
 }
 
 static gf_boolean_t
-_is_lock_wind_needed (afr_local_t *local, int child_index)
-{
-        if (!local->child_up[child_index])
-                return _gf_false;
-
-        return _gf_true;
+is_blocking_locks_count_sufficient(call_frame_t *frame, xlator_t *this)
+{
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
+    afr_internal_lock_t *int_lock = NULL;
+    int child = 0;
+    int nlockee = 0;
+    int lockee_count = 0;
+    gf_boolean_t ret = _gf_true;
+
+    local = frame->local;
+    priv = this->private;
+    int_lock = &local->internal_lock;
+    lockee_count = int_lock->lockee_count;
+
+    if (int_lock->lock_count == 0) {
+        afr_log_locks_failure(frame, "any subvolume", "lock",
+                              int_lock->lock_op_errno);
+        return _gf_false;
+    }
+    /* For FOPS that take multiple sets of locks (mkdir, rename),
+     * there must be at least one brick on which the locks from
+     * all lock sets were successful. */
+    for (child = 0; child < priv->child_count; child++) {
+        ret = _gf_true;
+        for (nlockee = 0; nlockee < lockee_count; nlockee++) {
+            if (!(int_lock->lockee[nlockee].locked_nodes[child] & LOCKED_YES))
+                ret = _gf_false;
+        }
+        if (ret)
+            return ret;
+    }
+    if (!ret)
+        afr_log_locks_failure(frame, "all", "lock", int_lock->lock_op_errno);
+
+    return ret;
 }
 
 int
-afr_lock_blocking (call_frame_t *frame, xlator_t *this, int cookie)
+afr_lock_blocking(call_frame_t *frame, xlator_t *this, int cookie)
 {
-        afr_internal_lock_t *int_lock    = NULL;
-        afr_inodelk_t       *inodelk     = NULL;
-        afr_local_t         *local       = NULL;
-        afr_private_t       *priv        = NULL;
-        struct gf_flock flock = {0,};
-        uint64_t ctx = 0;
-        int ret = 0;
-        int child_index = 0;
-        int lockee_no   = 0;
-        gf_boolean_t is_entrylk = _gf_false;
-
-        local         = frame->local;
-        int_lock      = &local->internal_lock;
-        priv          = this->private;
-        child_index   = cookie % priv->child_count;
-        lockee_no     = cookie / priv->child_count;
-        is_entrylk    = afr_is_entrylk (int_lock, local->transaction.type);
-
-
-        if (!is_entrylk) {
-                inodelk = afr_get_inodelk (int_lock, int_lock->domain);
-                flock.l_start = inodelk->flock.l_start;
-                flock.l_len   = inodelk->flock.l_len;
-                flock.l_type  = inodelk->flock.l_type;
-        }
-
-        if (local->fd) {
-                ret = fd_ctx_get (local->fd, this, &ctx);
-
-                if (ret < 0) {
-                        gf_log (this->name, GF_LOG_INFO,
-                                "unable to get fd ctx for fd=%p",
-                                local->fd);
+    afr_internal_lock_t *int_lock = NULL;
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
+    uint64_t ctx = 0;
+    int ret = 0;
+    int child_index = 0;
+    int lockee_num = 0;
 
-                        local->op_ret           = -1;
-                        int_lock->lock_op_ret   = -1;
+    local = frame->local;
+    int_lock = &local->internal_lock;
+    priv = this->private;
+    child_index = cookie % priv->child_count;
+    lockee_num = cookie / priv->child_count;
 
-                        afr_copy_locked_nodes (frame, this);
+    if (local->fd) {
+        ret = fd_ctx_get(local->fd, this, &ctx);
 
-                        afr_unlock (frame, this);
+        if (ret < 0) {
+            gf_msg(this->name, GF_LOG_INFO, 0, AFR_MSG_FD_CTX_GET_FAILED,
+                   "unable to get fd ctx for fd=%p", local->fd);
 
-                        return 0;
-                }
-        }
-
-        if (int_lock->lk_expected_count == int_lock->lk_attempted_count) {
-                if ((is_entrylk && int_lock->entrylk_lock_count == 0) ||
-                    (!is_entrylk && int_lock->lock_count == 0)) {
-                        gf_log (this->name, GF_LOG_INFO,
-                                "unable to lock on even one child");
-
-                        local->op_ret           = -1;
-                        int_lock->lock_op_ret   = -1;
-
-                        afr_copy_locked_nodes (frame, this);
+            local->op_ret = -1;
+            int_lock->lock_op_ret = -1;
 
-                        afr_unlock(frame, this);
+            afr_unlock_now(frame, this);
 
-                        return 0;
-                }
+            return 0;
         }
+    }
 
-        if (int_lock->lk_expected_count == int_lock->lk_attempted_count) {
-                /* we're done locking */
-
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "we're done locking");
+    if (int_lock->lk_expected_count == int_lock->lk_attempted_count) {
+        if (!is_blocking_locks_count_sufficient(frame, this)) {
+            local->op_ret = -1;
+            int_lock->lock_op_ret = -1;
 
-                afr_copy_locked_nodes (frame, this);
+            afr_unlock_now(frame, this);
 
-                int_lock->lock_op_ret = 0;
-                int_lock->lock_cbk (frame, this);
-                return 0;
+            return 0;
         }
+    }
 
-        if (!_is_lock_wind_needed (local, child_index)) {
-                afr_lock_blocking (frame, this, cookie + 1);
-                return 0;
-        }
-
-        switch (local->transaction.type) {
-        case AFR_DATA_TRANSACTION:
-        case AFR_METADATA_TRANSACTION:
+    if (int_lock->lk_expected_count == int_lock->lk_attempted_count) {
+        /* we're done locking */
 
-                if (local->fd) {
-                        AFR_TRACE_INODELK_IN (frame, this,
-                                              AFR_INODELK_TRANSACTION,
-                                              AFR_LOCK_OP, &flock, F_SETLKW,
-                                              child_index);
-
-                        STACK_WIND_COOKIE (frame, afr_blocking_inodelk_cbk,
-                                           (void *) (long) child_index,
-                                           priv->children[child_index],
-                                           priv->children[child_index]->fops->finodelk,
-                                           int_lock->domain, local->fd,
-                                           F_SETLKW, &flock, NULL);
-
-                } else {
-                        AFR_TRACE_INODELK_IN (frame, this,
-                                              AFR_INODELK_TRANSACTION,
-                                              AFR_LOCK_OP, &flock, F_SETLKW,
-                                              child_index);
-
-                        STACK_WIND_COOKIE (frame, afr_blocking_inodelk_cbk,
-                                           (void *) (long) child_index,
-                                           priv->children[child_index],
-                                           priv->children[child_index]->fops->inodelk,
-                                           int_lock->domain, &local->loc,
-                                           F_SETLKW, &flock, NULL);
-                }
+        gf_msg_debug(this->name, 0, "we're done locking");
 
-                break;
+        int_lock->lock_op_ret = 0;
+        int_lock->lock_cbk(frame, this);
+        return 0;
+    }
 
-        case AFR_ENTRY_RENAME_TRANSACTION:
-        case AFR_ENTRY_TRANSACTION:
-                /*Accounting for child_index increments on 'down'
-                 *and 'fd-less' children */
-
-                if (local->fd) {
-                        AFR_TRACE_ENTRYLK_IN (frame, this, AFR_ENTRYLK_TRANSACTION,
-                                              AFR_LOCK_OP,
-                                              int_lock->lockee[lockee_no].basename,
-                                              cookie);
-
-                        STACK_WIND_COOKIE (frame, afr_blocking_entrylk_cbk,
-                                           (void *) (long) cookie,
-                                           priv->children[child_index],
-                                           priv->children[child_index]->fops->fentrylk,
-                                           int_lock->domain, local->fd,
-                                           int_lock->lockee[lockee_no].basename,
-                                           ENTRYLK_LOCK, ENTRYLK_WRLCK, NULL);
-                } else {
-                        AFR_TRACE_ENTRYLK_IN (frame, this,
-                                              AFR_ENTRYLK_TRANSACTION,
-                                              AFR_LOCK_OP, local->transaction.basename,
-                                              child_index);
-
-                        STACK_WIND_COOKIE (frame, afr_blocking_entrylk_cbk,
-                                           (void *) (long) cookie,
-                                           priv->children[child_index],
-                                           priv->children[child_index]->fops->entrylk,
-                                           int_lock->domain,
-                                           &int_lock->lockee[lockee_no].loc,
-                                           int_lock->lockee[lockee_no].basename,
-                                           ENTRYLK_LOCK, ENTRYLK_WRLCK, NULL);
-                }
+    if (!_is_lock_wind_needed(local, child_index)) {
+        afr_lock_blocking(frame, this, cookie + 1);
+        return 0;
+    }
 
-                break;
-        }
+    afr_internal_lock_wind(frame, afr_lock_cbk, (void *)(long)cookie,
+                           child_index, lockee_num, _gf_true, _gf_false);
 
-        return 0;
+    return 0;
 }
 
 int32_t
-afr_blocking_lock (call_frame_t *frame, xlator_t *this)
+afr_blocking_lock(call_frame_t *frame, xlator_t *this)
 {
-        afr_internal_lock_t *int_lock = NULL;
-        afr_local_t         *local    = NULL;
-        afr_private_t       *priv     = NULL;
-        int                  up_count = 0;
-
-        priv     = this->private;
-        local    = frame->local;
-        int_lock = &local->internal_lock;
+    afr_internal_lock_t *int_lock = NULL;
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
+    int up_count = 0;
 
-        switch (local->transaction.type) {
-        case AFR_DATA_TRANSACTION:
-        case AFR_METADATA_TRANSACTION:
-                initialize_inodelk_variables (frame, this);
-                break;
+    priv = this->private;
+    local = frame->local;
+    int_lock = &local->internal_lock;
 
-        case AFR_ENTRY_RENAME_TRANSACTION:
-        case AFR_ENTRY_TRANSACTION:
-                up_count = AFR_COUNT (local->child_up, priv->child_count);
-                int_lock->lk_call_count = int_lock->lk_expected_count
-                                        = (int_lock->lockee_count *
-                                           up_count);
-                initialize_entrylk_variables (frame, this);
-                break;
-        }
+    up_count = AFR_COUNT(local->child_up, priv->child_count);
+    int_lock->lk_call_count = int_lock->lk_expected_count =
+        (int_lock->lockee_count * up_count);
+    initialize_internal_lock_variables(frame, this);
 
-        afr_lock_blocking (frame, this, 0);
+    afr_lock_blocking(frame, this, 0);
 
-        return 0;
+    return 0;
 }
 
 static int32_t
-afr_nonblocking_entrylk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                             int32_t op_ret, int32_t op_errno, dict_t *xdata)
-{
-        afr_internal_lock_t *int_lock = NULL;
-        afr_local_t         *local    = NULL;
-        int call_count          = 0;
-        int child_index         = (long) cookie;
-        int copies              = 0;
-        int index               = 0;
-        int lockee_no           = 0;
-        afr_private_t       *priv = NULL;
-
-        priv = this->private;
-
-        copies = priv->child_count;
-        index = child_index % copies;
-        lockee_no = child_index / copies;
-
-        local    = frame->local;
-        int_lock = &local->internal_lock;
-
-        AFR_TRACE_ENTRYLK_OUT (frame, this, AFR_ENTRYLK_TRANSACTION,
-                               AFR_LOCK_OP,
-                               int_lock->lockee[lockee_no].basename, op_ret,
-                               op_errno, (long) cookie);
-
-	LOCK (&frame->lock);
-	{
-		if (op_ret < 0 ) {
-			if (op_errno == ENOSYS) {
-                        /* return ENOTSUP */
-				gf_log (this->name, GF_LOG_ERROR,
-					"subvolume does not support locking. "
-					"please load features/locks xlator on server");
-				local->op_ret         = op_ret;
-				int_lock->lock_op_ret = op_ret;
-
-				int_lock->lock_op_errno      = op_errno;
-				local->op_errno              = op_errno;
-			}
-		} else if (op_ret == 0) {
-			int_lock->lockee[lockee_no].locked_nodes[index] |= \
-				LOCKED_YES;
-			int_lock->lockee[lockee_no].locked_count++;
-			int_lock->entrylk_lock_count++;
-		}
-
-                call_count = --int_lock->lk_call_count;
-        }
-        UNLOCK (&frame->lock);
-
-        if (call_count == 0) {
-                gf_log (this->name, GF_LOG_TRACE,
-                        "Last locking reply received");
-                /* all locks successful. Proceed to call FOP */
-                if (int_lock->entrylk_lock_count ==
-                                int_lock->lk_expected_count) {
-                        gf_log (this->name, GF_LOG_TRACE,
-                                "All servers locked. Calling the cbk");
-                        int_lock->lock_op_ret = 0;
-                        int_lock->lock_cbk (frame, this);
-                }
-                /* Not all locks were successful. Unlock and try locking
-                   again, this time with serially blocking locks */
-                else {
-                        gf_log (this->name, GF_LOG_TRACE,
-                                "%d servers locked. Trying again with blocking calls",
-                                int_lock->lock_count);
-
-                        afr_unlock(frame, this);
-                }
-        }
-
-        return 0;
-}
-
-int
-afr_nonblocking_entrylk (call_frame_t *frame, xlator_t *this)
+afr_nb_internal_lock_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                         int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
-        afr_internal_lock_t *int_lock   = NULL;
-        afr_local_t         *local      = NULL;
-        afr_private_t       *priv       = NULL;
-        afr_fd_ctx_t        *fd_ctx     = NULL;
-        int                 copies      = 0;
-        int                 index       = 0;
-        int                 lockee_no   = 0;
-        int32_t             call_count  = 0;
-        int i = 0;
-
-        local    = frame->local;
-        int_lock = &local->internal_lock;
-        priv     = this->private;
-
-        copies = priv->child_count;
-        initialize_entrylk_variables (frame, this);
-
-        if (local->fd) {
-                fd_ctx = afr_fd_ctx_get (local->fd, this);
-                if (!fd_ctx) {
-                        gf_log (this->name, GF_LOG_INFO,
-                                "unable to get fd ctx for fd=%p",
-                                local->fd);
-
-                        local->op_ret           = -1;
-                        int_lock->lock_op_ret   = -1;
-                        local->op_errno         = EINVAL;
-                        int_lock->lock_op_errno = EINVAL;
-
-			afr_unlock (frame, this);
-                        return -1;
-                }
-
-                call_count = int_lock->lockee_count * internal_lock_count (frame, this);
-                int_lock->lk_call_count = call_count;
-                int_lock->lk_expected_count = call_count;
-
-                if (!call_count) {
-                        gf_log (this->name, GF_LOG_INFO,
-                                "fd not open on any subvolumes. aborting.");
-                        afr_unlock (frame, this);
-                        goto out;
-                }
-
-                /* Send non-blocking entrylk calls only on up children
-                   and where the fd has been opened */
-                for (i = 0; i < int_lock->lockee_count*priv->child_count; i++) {
-                        index = i%copies;
-                        lockee_no = i/copies;
-                        if (local->child_up[index]) {
-                                AFR_TRACE_ENTRYLK_IN (frame, this, AFR_ENTRYLK_NB_TRANSACTION,
-                                                      AFR_LOCK_OP,
-                                                      int_lock->lockee[lockee_no].basename,
-                                                      i);
-
-                                STACK_WIND_COOKIE (frame, afr_nonblocking_entrylk_cbk,
-                                                   (void *) (long) i,
-                                                   priv->children[index],
-                                                   priv->children[index]->fops->fentrylk,
-                                                   this->name, local->fd,
-                                                   int_lock->lockee[lockee_no].basename,
-                                                   ENTRYLK_LOCK_NB, ENTRYLK_WRLCK,
-                                                   NULL);
-                                if (!--call_count)
-                                        break;
-                        }
-                }
-        } else {
-                call_count = int_lock->lockee_count * internal_lock_count (frame, this);
-                int_lock->lk_call_count = call_count;
-                int_lock->lk_expected_count = call_count;
-
-                for (i = 0; i < int_lock->lockee_count*priv->child_count; i++) {
-                        index = i%copies;
-                        lockee_no = i/copies;
-                        if (local->child_up[index]) {
-                                AFR_TRACE_ENTRYLK_IN (frame, this, AFR_ENTRYLK_NB_TRANSACTION,
-                                                      AFR_LOCK_OP,
-                                                      int_lock->lockee[lockee_no].basename,
-                                                      i);
-
-                                STACK_WIND_COOKIE (frame, afr_nonblocking_entrylk_cbk,
-                                                   (void *) (long) i,
-                                                   priv->children[index],
-                                                   priv->children[index]->fops->entrylk,
-                                                   this->name, &int_lock->lockee[lockee_no].loc,
-                                                   int_lock->lockee[lockee_no].basename,
-                                                   ENTRYLK_LOCK_NB, ENTRYLK_WRLCK,
-                                                   NULL);
-
-                                if (!--call_count)
-                                        break;
-                        }
-                }
-        }
-out:
-        return 0;
-}
-
-int32_t
-afr_nonblocking_inodelk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                             int32_t op_ret, int32_t op_errno, dict_t *xdata)
-{
-        afr_internal_lock_t *int_lock = NULL;
-        afr_inodelk_t       *inodelk  = NULL;
-        afr_local_t         *local    = NULL;
-        int call_count  = 0;
-        int child_index = (long) cookie;
-        afr_fd_ctx_t        *fd_ctx = NULL;
-
+    afr_internal_lock_t *int_lock = NULL;
+    afr_local_t *local = NULL;
+    int call_count = 0;
+    int child_index = 0;
+    int lockee_num = 0;
+    afr_private_t *priv = NULL;
 
-        local    = frame->local;
-        int_lock = &local->internal_lock;
-        inodelk = afr_get_inodelk (int_lock, int_lock->domain);
+    priv = this->private;
 
-        AFR_TRACE_INODELK_OUT (frame, this, AFR_INODELK_NB_TRANSACTION,
-                               AFR_LOCK_OP, NULL, op_ret,
-                               op_errno, (long) cookie);
+    child_index = ((long)cookie) % priv->child_count;
+    lockee_num = ((long)cookie) / priv->child_count;
 
-	if (local->fd)
-		fd_ctx = afr_fd_ctx_get (local->fd, this);
+    local = frame->local;
+    int_lock = &local->internal_lock;
 
-        LOCK (&frame->lock);
+    if (op_ret == 0 && local->transaction.type == AFR_DATA_TRANSACTION) {
+        LOCK(&local->inode->lock);
         {
-		if (op_ret < 0) {
-			if (op_errno == ENOSYS) {
-				/* return ENOTSUP */
-				gf_log (this->name, GF_LOG_ERROR,
-					"subvolume does not support locking. "
-					"please load features/locks xlator on "
-					"server");
-				local->op_ret                = op_ret;
-				int_lock->lock_op_ret        = op_ret;
-				int_lock->lock_op_errno      = op_errno;
-				local->op_errno              = op_errno;
-			}
-			if (local->transaction.eager_lock)
-				local->transaction.eager_lock[child_index] = 0;
-		} else {
-			inodelk->locked_nodes[child_index] |= LOCKED_YES;
-			inodelk->lock_count++;
-
-			if (local->transaction.eager_lock &&
-			    local->transaction.eager_lock[child_index] &&
-			    local->fd) {
-				/* piggybacked */
-				if (op_ret == 1) {
-					/* piggybacked */
-				} else if (op_ret == 0) {
-					/* lock acquired from server */
-                                        fd_ctx->lock_acquired[child_index]++;
-				}
-			}
-		}
-
-                call_count = --int_lock->lk_call_count;
-        }
-        UNLOCK (&frame->lock);
-
-        if (call_count == 0) {
-                gf_log (this->name, GF_LOG_TRACE,
-                        "Last inode locking reply received");
-                /* all locks successful. Proceed to call FOP */
-                if (inodelk->lock_count == int_lock->lk_expected_count) {
-                        gf_log (this->name, GF_LOG_TRACE,
-                                "All servers locked. Calling the cbk");
-                        int_lock->lock_op_ret = 0;
-                        int_lock->lock_cbk (frame, this);
-                }
-                /* Not all locks were successful. Unlock and try locking
-                   again, this time with serially blocking locks */
-                else {
-                        gf_log (this->name, GF_LOG_TRACE,
-                                "%d servers locked. Trying again with blocking calls",
-                                int_lock->lock_count);
-
-                        afr_unlock(frame, this);
-                }
+            local->inode_ctx->lock_count++;
         }
+        UNLOCK(&local->inode->lock);
+    }
 
-        return 0;
+    LOCK(&frame->lock);
+    {
+        if (op_ret < 0) {
+            if (op_errno == ENOSYS) {
+                /* return ENOTSUP */
+                gf_msg(this->name, GF_LOG_ERROR, ENOSYS,
+                       AFR_MSG_LOCK_XLATOR_NOT_LOADED,
+                       "subvolume does not support "
+                       "locking. please load features/locks"
+                       " xlator on server");
+                local->op_ret = op_ret;
+                int_lock->lock_op_ret = op_ret;
+
+                int_lock->lock_op_errno = op_errno;
+                local->op_errno = op_errno;
+            }
+        } else if (op_ret == 0) {
+            int_lock->lockee[lockee_num]
+                .locked_nodes[child_index] |= LOCKED_YES;
+            int_lock->lockee[lockee_num].locked_count++;
+            int_lock->lock_count++;
+        }
+
+        call_count = --int_lock->lk_call_count;
+    }
+    UNLOCK(&frame->lock);
+
+    if (call_count == 0) {
+        gf_msg_trace(this->name, 0, "Last locking reply received");
+        /* all locks successful. Proceed to call FOP */
+        if (int_lock->lock_count == int_lock->lk_expected_count) {
+            gf_msg_trace(this->name, 0, "All servers locked. Calling the cbk");
+            int_lock->lock_op_ret = 0;
+            int_lock->lock_cbk(frame, this);
+        }
+        /* Not all locks were successful. Unlock and try locking
+           again, this time with serially blocking locks */
+        else {
+            gf_msg_trace(this->name, 0,
+                         "%d servers locked. Trying again "
+                         "with blocking calls",
+                         int_lock->lock_count);
+
+            afr_unlock_now(frame, this);
+        }
+    }
+
+    return 0;
 }
 
 int
-afr_nonblocking_inodelk (call_frame_t *frame, xlator_t *this)
-{
-        afr_internal_lock_t *int_lock = NULL;
-        afr_inodelk_t       *inodelk  = NULL;
-        afr_local_t         *local    = NULL;
-        afr_private_t       *priv     = NULL;
-        afr_fd_ctx_t        *fd_ctx   = NULL;
-        int32_t             call_count = 0;
-        int                 i          = 0;
-        int                 ret        = 0;
-        struct              gf_flock flock = {0,};
-        struct              gf_flock full_flock = {0,};
-        struct              gf_flock *flock_use = NULL;
-        int                 piggyback = 0;
-
-        local    = frame->local;
-        int_lock = &local->internal_lock;
-        priv     = this->private;
-
-        inodelk = afr_get_inodelk (int_lock, int_lock->domain);
-
-        flock.l_start = inodelk->flock.l_start;
-        flock.l_len   = inodelk->flock.l_len;
-        flock.l_type  = inodelk->flock.l_type;
-
-        full_flock.l_type = inodelk->flock.l_type;
-
-        initialize_inodelk_variables (frame, this);
-
-        if (local->fd) {
-                fd_ctx = afr_fd_ctx_get (local->fd, this);
-                if (!fd_ctx) {
-                        gf_log (this->name, GF_LOG_INFO,
-                                "unable to get fd ctx for fd=%p",
-                                local->fd);
-
-                        local->op_ret           = -1;
-                        int_lock->lock_op_ret   = -1;
-                        local->op_errno         = EINVAL;
-                        int_lock->lock_op_errno = EINVAL;
-
-			afr_unlock (frame, this);
-                        ret = -1;
-                        goto out;
-                }
-
-                call_count = internal_lock_count (frame, this);
-                int_lock->lk_call_count = call_count;
-                int_lock->lk_expected_count = call_count;
-
-                if (!call_count) {
-                        gf_log (this->name, GF_LOG_INFO,
-                                "fd not open on any subvolumes. aborting.");
-                        afr_unlock (frame, this);
-                        goto out;
-                }
-
-                /* Send non-blocking inodelk calls only on up children
-                   and where the fd has been opened */
-                for (i = 0; i < priv->child_count; i++) {
-                        if (!local->child_up[i])
-                                continue;
-
-                        flock_use = &flock;
-                        if (!local->transaction.eager_lock_on) {
-                                goto wind;
-                        }
-
-                        piggyback = 0;
-                        local->transaction.eager_lock[i] = 1;
-
-			afr_set_delayed_post_op (frame, this);
-
-                        LOCK (&local->fd->lock);
-                        {
-                                if (fd_ctx->lock_acquired[i]) {
-                                        fd_ctx->lock_piggyback[i]++;
-                                        piggyback = 1;
-                                }
-                        }
-                        UNLOCK (&local->fd->lock);
-
-                        if (piggyback) {
-                                /* (op_ret == 1) => indicate piggybacked lock */
-                                afr_nonblocking_inodelk_cbk (frame, (void *) (long) i,
-                                                             this, 1, 0, NULL);
-                                if (!--call_count)
-                                        break;
-                                continue;
-                        }
-                        flock_use = &full_flock;
-                wind:
-                        AFR_TRACE_INODELK_IN (frame, this,
-                                              AFR_INODELK_NB_TRANSACTION,
-                                              AFR_LOCK_OP, flock_use, F_SETLK, i);
-
-                        STACK_WIND_COOKIE (frame, afr_nonblocking_inodelk_cbk,
-                                           (void *) (long) i,
-                                           priv->children[i],
-                                           priv->children[i]->fops->finodelk,
-                                           int_lock->domain, local->fd,
-                                           F_SETLK, flock_use, NULL);
-
-                        if (!--call_count)
-                                break;
-                }
-        } else {
-                call_count = internal_lock_count (frame, this);
-                int_lock->lk_call_count = call_count;
-                int_lock->lk_expected_count = call_count;
-
-                for (i = 0; i < priv->child_count; i++) {
-                        if (!local->child_up[i])
-                                continue;
-                        AFR_TRACE_INODELK_IN (frame, this,
-                                              AFR_INODELK_NB_TRANSACTION,
-                                              AFR_LOCK_OP, &flock, F_SETLK, i);
-
-                        STACK_WIND_COOKIE (frame, afr_nonblocking_inodelk_cbk,
-                                           (void *) (long) i,
-                                           priv->children[i],
-                                           priv->children[i]->fops->inodelk,
-                                           int_lock->domain, &local->loc,
-                                           F_SETLK, &flock, NULL);
-
-                        if (!--call_count)
-                                break;
-                }
+afr_lock_nonblocking(call_frame_t *frame, xlator_t *this)
+{
+    afr_internal_lock_t *int_lock = NULL;
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
+    afr_fd_ctx_t *fd_ctx = NULL;
+    int child = 0;
+    int lockee_num = 0;
+    int32_t call_count = 0;
+    int i = 0;
+    int ret = 0;
+
+    local = frame->local;
+    int_lock = &local->internal_lock;
+    priv = this->private;
+
+    initialize_internal_lock_variables(frame, this);
+
+    if (local->fd) {
+        fd_ctx = afr_fd_ctx_get(local->fd, this);
+        if (!fd_ctx) {
+            gf_msg(this->name, GF_LOG_INFO, 0, AFR_MSG_FD_CTX_GET_FAILED,
+                   "unable to get fd ctx for fd=%p", local->fd);
+
+            local->op_ret = -1;
+            int_lock->lock_op_ret = -1;
+            local->op_errno = EINVAL;
+            int_lock->lock_op_errno = EINVAL;
+
+            afr_unlock_now(frame, this);
+            ret = -1;
+            goto out;
+        }
+    }
+
+    call_count = int_lock->lockee_count * internal_lock_count(frame, this);
+    int_lock->lk_call_count = call_count;
+    int_lock->lk_expected_count = call_count;
+
+    if (!call_count) {
+        gf_msg(this->name, GF_LOG_INFO, 0, AFR_MSG_INFO_COMMON,
+               "fd not open on any subvolumes. aborting.");
+        afr_unlock_now(frame, this);
+        goto out;
+    }
+
+    /* Send non-blocking lock calls only on up children
+       and where the fd has been opened */
+    for (i = 0; i < int_lock->lockee_count * priv->child_count; i++) {
+        child = i % priv->child_count;
+        lockee_num = i / priv->child_count;
+        if (local->child_up[child]) {
+            afr_internal_lock_wind(frame, afr_nb_internal_lock_cbk,
+                                   (void *)(long)i, child, lockee_num,
+                                   _gf_false, _gf_false);
+            if (!--call_count)
+                break;
         }
+    }
 out:
-        return ret;
+    return ret;
 }
 
 int32_t
-afr_unlock (call_frame_t *frame, xlator_t *this)
-{
-        afr_local_t *local = NULL;
-
-        local = frame->local;
-
-        if (transaction_lk_op (local)) {
-                if (is_afr_lock_transaction (local))
-                        afr_unlock_inodelk (frame, this);
-                else
-                        afr_unlock_entrylk (frame, this);
-
-        } else {
-                if (is_afr_lock_selfheal (local))
-                        afr_unlock_inodelk (frame, this);
-                else
-                        afr_unlock_entrylk (frame, this);
-        }
-
+afr_unlock(call_frame_t *frame, xlator_t *this)
+{
+    afr_local_t *local = NULL;
+    afr_lock_t *lock = NULL;
+
+    local = frame->local;
+
+    if (!local->transaction.eager_lock_on)
+        goto out;
+    lock = &local->inode_ctx->lock[local->transaction.type];
+    LOCK(&local->inode->lock);
+    {
+        list_del_init(&local->transaction.owner_list);
+        if (list_empty(&lock->owners) && list_empty(&lock->post_op)) {
+            local->transaction.do_eager_unlock = _gf_true;
+            /*TODO: Need to get metadata use on_disk and inherit/uninherit
+             *GF_ASSERT (!local->inode_ctx->on_disk[local->transaction.type]);
+             *GF_ASSERT (!local->inode_ctx->inherited[local->transaction.type]);
+             */
+            GF_ASSERT(lock->release);
+        }
+    }
+    UNLOCK(&local->inode->lock);
+    if (!local->transaction.do_eager_unlock) {
+        local->internal_lock.lock_cbk(frame, this);
         return 0;
-}
-
-int
-afr_lk_transfer_datalock (call_frame_t *dst, call_frame_t *src, char *dom,
-                          unsigned int child_count)
-{
-        afr_local_t         *dst_local   = NULL;
-        afr_local_t         *src_local   = NULL;
-        afr_internal_lock_t *dst_lock    = NULL;
-        afr_internal_lock_t *src_lock    = NULL;
-        afr_inodelk_t       *dst_inodelk = NULL;
-        afr_inodelk_t       *src_inodelk = NULL;
-        int                 ret = -1;
-
-        src_local = src->local;
-        src_lock  = &src_local->internal_lock;
-        src_inodelk = afr_get_inodelk (src_lock, dom);
-        dst_local = dst->local;
-        dst_lock  = &dst_local->internal_lock;
-        dst_inodelk = afr_get_inodelk (dst_lock, dom);
-        if (!dst_inodelk || !src_inodelk)
-                goto out;
-        if (src_inodelk->locked_nodes) {
-                memcpy (dst_inodelk->locked_nodes, src_inodelk->locked_nodes,
-                        sizeof (*dst_inodelk->locked_nodes) * child_count);
-                memset (src_inodelk->locked_nodes, 0,
-                        sizeof (*src_inodelk->locked_nodes) * child_count);
-        }
+    }
 
-        dst_lock->transaction_lk_type = src_lock->transaction_lk_type;
-        dst_lock->selfheal_lk_type    = src_lock->selfheal_lk_type;
-        dst_inodelk->lock_count = src_inodelk->lock_count;
-        src_inodelk->lock_count = 0;
-        ret = 0;
 out:
-        return ret;
+    afr_unlock_now(frame, this);
+    return 0;
 }
diff --git a/xlators/cluster/afr/src/afr-mem-types.h b/xlators/cluster/afr/src/afr-mem-types.h
index 05df90cc0ee..816065fb57a 100644
--- a/xlators/cluster/afr/src/afr-mem-types.h
+++ b/xlators/cluster/afr/src/afr-mem-types.h
@@ -8,42 +8,31 @@
   cases as published by the Free Software Foundation.
 */
 
-
 #ifndef __AFR_MEM_TYPES_H__
 #define __AFR_MEM_TYPES_H__
 
-#include "mem-types.h"
+#include <glusterfs/mem-types.h>
 
 enum gf_afr_mem_types_ {
-        gf_afr_mt_iovec  = gf_common_mt_end + 1,
-        gf_afr_mt_afr_fd_ctx_t,
-        gf_afr_mt_afr_private_t,
-        gf_afr_mt_int32_t,
-        gf_afr_mt_char,
-        gf_afr_mt_xattr_key,
-        gf_afr_mt_dict_t,
-        gf_afr_mt_xlator_t,
-        gf_afr_mt_iatt,
-        gf_afr_mt_int,
-        gf_afr_mt_afr_node_character,
-        gf_afr_mt_sh_diff_loop_state,
-        gf_afr_mt_uint8_t,
-        gf_afr_mt_loc_t,
-        gf_afr_mt_entry_name,
-        gf_afr_mt_pump_priv,
-        gf_afr_mt_locked_fd,
-        gf_afr_mt_inode_ctx_t,
-        gf_afr_fd_paused_call_t,
-        gf_afr_mt_crawl_data_t,
-        gf_afr_mt_brick_pos_t,
-        gf_afr_mt_shd_bool_t,
-        gf_afr_mt_shd_timer_t,
-        gf_afr_mt_shd_event_t,
-        gf_afr_mt_time_t,
-        gf_afr_mt_pos_data_t,
-	gf_afr_mt_reply_t,
-	gf_afr_mt_subvol_healer_t,
-        gf_afr_mt_end
+    gf_afr_mt_afr_fd_ctx_t = gf_common_mt_end + 1,
+    gf_afr_mt_afr_private_t,
+    gf_afr_mt_int32_t,
+    gf_afr_mt_char,
+    gf_afr_mt_xattr_key,
+    gf_afr_mt_dict_t,
+    gf_afr_mt_xlator_t,
+    gf_afr_mt_afr_node_character,
+    gf_afr_mt_inode_ctx_t,
+    gf_afr_mt_shd_event_t,
+    gf_afr_mt_reply_t,
+    gf_afr_mt_subvol_healer_t,
+    gf_afr_mt_spbc_timeout_t,
+    gf_afr_mt_spb_status_t,
+    gf_afr_mt_empty_brick_t,
+    gf_afr_mt_child_latency_t,
+    gf_afr_mt_atomic_t,
+    gf_afr_mt_lk_heal_info_t,
+    gf_afr_mt_gf_lock,
+    gf_afr_mt_end
 };
 #endif
-
diff --git a/xlators/cluster/afr/src/afr-messages.h b/xlators/cluster/afr/src/afr-messages.h
index 52bdead934b..e73fd997765 100644
--- a/xlators/cluster/afr/src/afr-messages.h
+++ b/xlators/cluster/afr/src/afr-messages.h
@@ -1,5 +1,5 @@
 /*
- Copyright (c) 2014 Red Hat, Inc. <http://www.redhat.com>
+ Copyright (c) 2015 Red Hat, Inc. <http://www.redhat.com>
  This file is part of GlusterFS.
 
  This file is licensed to you under your choice of the GNU Lesser
@@ -11,137 +11,157 @@
 #ifndef _AFR_MESSAGES_H_
 #define _AFR_MESSAGES_H_
 
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
+#include <glusterfs/glfs-message-id.h>
 
-#include "glfs-message-id.h"
-
-/*! \file afr-messages.h
- *  \brief AFR log-message IDs and their descriptions.
- */
-
-/* NOTE: Rules for message additions
- * 1) Each instance of a message is _better_ left with a unique message ID, even
- * if the message format is the same. Reasoning is that, if the message
- * format needs to change in one instance, the other instances are not
- * impacted or the new change does not change the ID of the instance being
- * modified.
- * 2) Addition of a message,
- * - Should increment the GLFS_NUM_MESSAGES
- * - Append to the list of messages defined, towards the end
- * - Retain macro naming as glfs_msg_X (for redability across developers)
- * NOTE: Rules for message format modifications
- * 3) Check acorss the code if the message ID macro in question is reused
- * anywhere. If reused then then the modifications should ensure correctness
- * everywhere, or needs a new message ID as (1) above was not adhered to. If
- * not used anywhere, proceed with the required modification.
- * NOTE: Rules for message deletion
- * 4) Check (3) and if used anywhere else, then cannot be deleted. If not used
- * anywhere, then can be deleted, but will leave a hole by design, as
- * addition rules specify modification to the end of the list and not filling
- * holes.
- */
-
-#define GLFS_COMP_BASE_AFR GLFS_MSGID_COMP_AFR
-#define GLFS_NUM_MESSAGES 10
-#define GLFS_MSGID_END (GLFS_COMP_BASE_AFR + GLFS_NUM_MESSAGES + 1)
-
-#define glfs_msg_start_x GLFS_COMP_BASE_AFR, "Invalid: Start of messages"
-
-/*!
- * @messageid 108001
- * @diagnosis Client quorum is not met due to which file modification
- * operations are disallowed.
- * @recommendedaction Some brick processes are down/ not visible from the
- * client. Ensure that the bricks are up/ network traffic is not blocked.
- */
-#define AFR_MSG_QUORUM_FAIL             (GLFS_COMP_BASE_AFR + 1)
-
-
-/*!
- * @messageid 108002
- * @diagnosis The bricks that were down are now up and quorum is restored.
- * @recommendedaction Possibly check why the bricks went down to begin with.
- */
-#define AFR_MSG_QUORUM_MET              (GLFS_COMP_BASE_AFR + 2)
-
-
-/*!
- * @messageid 108003
- * @diagnosis Client quorum-type was set to auto due to which the quorum-count
- * option is no longer valid.
- * @recommendedaction None.
- */
-#define AFR_MSG_QUORUM_OVERRIDE         (GLFS_COMP_BASE_AFR + 3)
-
-
-/*!
- * @messageid 108004
- * @diagnosis Replication sub volume witnessed a connection notification
- * from a brick which does not belong to its replica set.
- * @recommendedaction None. This is a safety check in code.
- */
-#define AFR_MSG_INVALID_CHILD_UP        (GLFS_COMP_BASE_AFR + 4)
-
-
-/*!
- * @messageid 108005
- * @diagnosis A replica set that was inaccessible because all its bricks were
- * down is now accessible because at least one of its bricks came back up.
- * @recommendedaction Possibly check why all the bricks of that replica set
- * went down to begin with.
- */
-#define AFR_MSG_SUBVOL_UP               (GLFS_COMP_BASE_AFR + 5)
-
-
-/*!
- * @messageid 108006
- * @diagnosis All bricks of a replica set are down. Data residing in that
- * replica cannot be accessed until one of the bricks come back up.
- * @recommendedaction Ensure that the bricks are up.
+/* To add new message IDs, append new identifiers at the end of the list.
+ *
+ * Never remove a message ID. If it's not used anymore, you can rename it or
+ * leave it as it is, but not delete it. This is to prevent reutilization of
+ * IDs by other messages.
+ *
+ * The component name must match one of the entries defined in
+ * glfs-message-id.h.
  */
-#define AFR_MSG_ALL_SUBVOLS_DOWN        (GLFS_COMP_BASE_AFR + 6)
-
-
-/*!
- * @messageid 108007
- * @diagnosis Entry unlocks failed on a brick.
- * @recommendedaction Error number in the log should give the reason why it
- * failed. Also observe brick logs for more information.
-*/
-#define AFR_MSG_ENTRY_UNLOCK_FAIL       (GLFS_COMP_BASE_AFR + 7)
-
-
-/*!
- * @messageid 108008
- * @diagnosis There is an inconsistency in the file's data/metadata/gfid
- * amongst the bricks of a replica set.
- * @recommendedaction Resolve the split brain by clearing the AFR changelog
- * attributes from the appropriate brick and trigger self-heal.
- */
-#define AFR_MSG_SPLIT_BRAIN             (GLFS_COMP_BASE_AFR + 8)
-
-
-/*!
- * @messageid 108009
- * @diagnosis open/opendir failed on a brick.
- * @recommendedaction Error number in the log should give the reason why it
- * failed. Also observe brick logs for more information.
- */
-#define AFR_MSG_OPEN_FAIL               (GLFS_COMP_BASE_AFR + 9)
-
-
-/*!
- * @messageid 108010
- * @diagnosis Inode unlocks failed on a brick.
- * @recommendedaction Error number in the log should give the reason why it
- * failed. Also observe brick logs for more information.
-*/
-#define AFR_MSG_INODE_UNLOCK_FAIL       (GLFS_COMP_BASE_AFR + 10)
-
-
-#define glfs_msg_end_x GLFS_MSGID_END, "Invalid: End of messages"
 
+GLFS_MSGID(
+    AFR, AFR_MSG_QUORUM_FAIL, AFR_MSG_QUORUM_MET, AFR_MSG_QUORUM_OVERRIDE,
+    AFR_MSG_INVALID_CHILD_UP, AFR_MSG_SUBVOL_UP, AFR_MSG_SUBVOLS_DOWN,
+    AFR_MSG_ENTRY_UNLOCK_FAIL, AFR_MSG_SPLIT_BRAIN, AFR_MSG_OPEN_FAIL,
+    AFR_MSG_UNLOCK_FAIL, AFR_MSG_REPLACE_BRICK_STATUS, AFR_MSG_GFID_NULL,
+    AFR_MSG_FD_CREATE_FAILED, AFR_MSG_DICT_SET_FAILED,
+    AFR_MSG_EXPUNGING_FILE_OR_DIR, AFR_MSG_MIGRATION_IN_PROGRESS,
+    AFR_MSG_CHILD_MISCONFIGURED, AFR_MSG_VOL_MISCONFIGURED,
+    AFR_MSG_INTERNAL_LKS_FAILED, AFR_MSG_INVALID_FD, AFR_MSG_LOCK_INFO,
+    AFR_MSG_LOCK_XLATOR_NOT_LOADED, AFR_MSG_FD_CTX_GET_FAILED,
+    AFR_MSG_INVALID_SUBVOL, AFR_MSG_PUMP_XLATOR_ERROR, AFR_MSG_SELF_HEAL_INFO,
+    AFR_MSG_READ_SUBVOL_ERROR, AFR_MSG_DICT_GET_FAILED, AFR_MSG_INFO_COMMON,
+    AFR_MSG_SPLIT_BRAIN_CHOICE_ERROR, AFR_MSG_LOCAL_CHILD, AFR_MSG_INVALID_DATA,
+    AFR_MSG_INVALID_ARG, AFR_MSG_INDEX_DIR_GET_FAILED, AFR_MSG_FSYNC_FAILED,
+    AFR_MSG_FAVORITE_CHILD, AFR_MSG_SELF_HEAL_FAILED,
+    AFR_MSG_SPLIT_BRAIN_STATUS, AFR_MSG_ADD_BRICK_STATUS, AFR_MSG_NO_CHANGELOG,
+    AFR_MSG_TIMER_CREATE_FAIL, AFR_MSG_SBRAIN_FAV_CHILD_POLICY,
+    AFR_MSG_INODE_CTX_GET_FAILED, AFR_MSG_THIN_ARB,
+    AFR_MSG_THIN_ARB_XATTROP_FAILED, AFR_MSG_THIN_ARB_LOC_POP_FAILED,
+    AFR_MSG_GET_PEND_VAL, AFR_MSG_THIN_ARB_SKIP_SHD, AFR_MSG_UNKNOWN_SET,
+    AFR_MSG_NO_XL_ID, AFR_MSG_SELF_HEAL_INFO_START,
+    AFR_MSG_SELF_HEAL_INFO_FINISH, AFR_MSG_INCRE_COUNT,
+    AFR_MSG_ADD_TO_OUTPUT_FAILED, AFR_MSG_SET_TIME_FAILED,
+    AFR_MSG_GFID_MISMATCH_DETECTED, AFR_MSG_GFID_HEAL_MSG,
+    AFR_MSG_THIN_ARB_LOOKUP_FAILED, AFR_MSG_DICT_CREATE_FAILED,
+    AFR_MSG_NO_MAJORITY_TO_RESOLVE, AFR_MSG_TYPE_MISMATCH,
+    AFR_MSG_SIZE_POLICY_NOT_APPLICABLE, AFR_MSG_NO_CHILD_SELECTED,
+    AFR_MSG_INVALID_CHILD, AFR_MSG_RESOLVE_CONFLICTING_DATA,
+    SERROR_GETTING_SRC_BRICK, SNO_DIFF_IN_MTIME, SNO_BIGGER_FILE,
+    SALL_BRICKS_UP_TO_RESOLVE, AFR_MSG_UNLOCK_FAILED, AFR_MSG_POST_OP_FAILED,
+    AFR_MSG_TA_FRAME_CREATE_FAILED, AFR_MSG_SET_KEY_XATTROP_FAILED,
+    AFR_MSG_BLOCKING_ENTRYLKS_FAILED, AFR_MSG_FOP_FAILED,
+    AFR_MSG_CLEAN_UP_FAILED, AFR_MSG_UNABLE_TO_FETCH, AFR_MSG_XATTR_SET_FAILED,
+    AFR_MSG_SPLIT_BRAIN_REPLICA, AFR_MSG_INODE_CTX_FAILED,
+    AFR_MSG_LOOKUP_FAILED, AFR_MSG_ALL_SUBVOLS_DOWN,
+    AFR_MSG_RELEASE_LOCK_FAILED, AFR_MSG_CLEAR_TIME_SPLIT_BRAIN,
+    AFR_MSG_READ_FAILED, AFR_MSG_LAUNCH_FAILED, AFR_MSG_READ_SUBVOL_NOT_UP,
+    AFR_MSG_LK_HEAL_DOM, AFR_MSG_NEW_BRICK, AFR_MSG_SPLIT_BRAIN_SET_FAILED,
+    AFR_MSG_SPLIT_BRAIN_DETERMINE_FAILED, AFR_MSG_HEALER_SPAWN_FAILED,
+    AFR_MSG_ADD_CRAWL_EVENT_FAILED, AFR_MSG_NULL_DEREF, AFR_MSG_SET_PEND_XATTR,
+    AFR_MSG_INTERNAL_ATTR);
+
+#define AFR_MSG_DICT_GET_FAILED_STR "Dict get failed"
+#define AFR_MSG_DICT_SET_FAILED_STR "Dict set failed"
+#define AFR_MSG_HEALER_SPAWN_FAILED_STR "Healer spawn failed"
+#define AFR_MSG_ADD_CRAWL_EVENT_FAILED_STR "Adding crawl event failed"
+#define AFR_MSG_INVALID_ARG_STR "Invalid argument"
+#define AFR_MSG_INDEX_DIR_GET_FAILED_STR "unable to get index-dir on "
+#define AFR_MSG_THIN_ARB_LOOKUP_FAILED_STR "Failed lookup on file"
+#define AFR_MSG_DICT_CREATE_FAILED_STR "Failed to create dict."
+#define AFR_MSG_THIN_ARB_XATTROP_FAILED_STR "Xattrop failed."
+#define AFR_MSG_THIN_ARB_LOC_POP_FAILED_STR                                    \
+    "Failed to populate loc for thin-arbiter"
+#define AFR_MSG_GET_PEND_VAL_STR "Error getting value of pending"
+#define AFR_MSG_THIN_ARB_SKIP_SHD_STR "I am not the god shd. skipping."
+#define AFR_MSG_UNKNOWN_SET_STR "Unknown set"
+#define AFR_MSG_NO_XL_ID_STR "xl does not have id"
+#define AFR_MSG_SELF_HEAL_INFO_START_STR "starting full sweep on"
+#define AFR_MSG_SELF_HEAL_INFO_FINISH_STR "finished full sweep on"
+#define AFR_MSG_INCRE_COUNT_STR "Could not increment the counter."
+#define AFR_MSG_ADD_TO_OUTPUT_FAILED_STR "Could not add to output"
+#define AFR_MSG_SET_TIME_FAILED_STR "Could not set time"
+#define AFR_MSG_GFID_HEAL_MSG_STR "Error setting gfid-heal-msg dict"
+#define AFR_MSG_NO_MAJORITY_TO_RESOLVE_STR                                     \
+    "No majority to resolve gfid split brain"
+#define AFR_MSG_GFID_MISMATCH_DETECTED_STR "Gfid mismatch dectected"
+#define AFR_MSG_SELF_HEAL_INFO_STR "performing selfheal"
+#define AFR_MSG_TYPE_MISMATCH_STR "TYPE mismatch"
+#define AFR_MSG_SIZE_POLICY_NOT_APPLICABLE_STR                                 \
+    "Size policy is not applicable to directories."
+#define AFR_MSG_NO_CHILD_SELECTED_STR                                          \
+    "No child selected by favorite-child policy"
+#define AFR_MSG_INVALID_CHILD_STR "Invalid child"
+#define AFR_MSG_RESOLVE_CONFLICTING_DATA_STR                                   \
+    "selected as authentic to resolve conflicting data"
+#define SERROR_GETTING_SRC_BRICK_STR "Error getting the source brick"
+#define SNO_DIFF_IN_MTIME_STR "No difference in mtime"
+#define SNO_BIGGER_FILE_STR "No bigger file"
+#define SALL_BRICKS_UP_TO_RESOLVE_STR                                          \
+    "All the bricks should be up to resolve the gfid split brain"
+#define AFR_MSG_UNLOCK_FAILED_STR "Failed to unlock"
+#define AFR_MSG_POST_OP_FAILED_STR "Post-op on thin-arbiter failed"
+#define AFR_MSG_TA_FRAME_CREATE_FAILED_STR "Failed to create ta_frame"
+#define AFR_MSG_SET_KEY_XATTROP_FAILED_STR "Could not set key during xattrop"
+#define AFR_MSG_BLOCKING_ENTRYLKS_FAILED_STR "Blocking entrylks failed"
+#define AFR_MSG_FSYNC_FAILED_STR "fsync failed"
+#define AFR_MSG_QUORUM_FAIL_STR "quorum is not met"
+#define AFR_MSG_FOP_FAILED_STR "Failing Fop"
+#define AFR_MSG_INVALID_SUBVOL_STR "not a subvolume"
+#define AFR_MSG_VOL_MISCONFIGURED_STR "Volume is dangling"
+#define AFR_MSG_CHILD_MISCONFIGURED_STR                                        \
+    "replicate translator needs more than one subvolume defined"
+#define AFR_MSG_CLEAN_UP_FAILED_STR "Failed to clean up healer threads"
+#define AFR_MSG_QUORUM_OVERRIDE_STR "overriding quorum-count"
+#define AFR_MSG_UNABLE_TO_FETCH_STR                                            \
+    "Unable to fetch afr-pending-xattr option from volfile. Falling back to "  \
+    "using client translator names"
+#define AFR_MSG_NULL_DEREF_STR "possible NULL deref"
+#define AFR_MSG_XATTR_SET_FAILED_STR "Cannot set xattr cookie key"
+#define AFR_MSG_SPLIT_BRAIN_STATUS_STR "Failed to create synctask"
+#define AFR_MSG_SUBVOLS_DOWN_STR "All subvolumes are not up"
+#define AFR_MSG_SPLIT_BRAIN_CHOICE_ERROR_STR                                   \
+    "Failed to cancel split-brain choice"
+#define AFR_MSG_SPLIT_BRAIN_REPLICA_STR                                        \
+    "Cannot set replica. File is not in data/metadata split-brain"
+#define AFR_MSG_INODE_CTX_FAILED_STR "Failed to get inode_ctx"
+#define AFR_MSG_READ_SUBVOL_ERROR_STR "no read subvols"
+#define AFR_MSG_LOCAL_CHILD_STR "selecting local read-child"
+#define AFR_MSG_LOOKUP_FAILED_STR "Failed to lookup/create thin-arbiter id file"
+#define AFR_MSG_TIMER_CREATE_FAIL_STR                                          \
+    "Cannot create timer for delayed initialization"
+#define AFR_MSG_SUBVOL_UP_STR "Subvolume came back up; going online"
+#define AFR_MSG_ALL_SUBVOLS_DOWN_STR                                           \
+    "All subvolumes are down. Going offline until atleast one of them is up"
+#define AFR_MSG_RELEASE_LOCK_FAILED_STR "Failed to release lock"
+#define AFR_MSG_INVALID_CHILD_UP_STR "Received child_up from invalid subvolume"
+#define AFR_MSG_QUORUM_MET_STR "Client-quorum is met"
+#define AFR_MSG_EXPUNGING_FILE_OR_DIR_STR "expunging file or dir"
+#define AFR_MSG_SELF_HEAL_FAILED_STR "Invalid"
+#define AFR_MSG_SPLIT_BRAIN_STR "Skipping conservative mergeon the file"
+#define AFR_MSG_CLEAR_TIME_SPLIT_BRAIN_STR "clear time split brain"
+#define AFR_MSG_READ_FAILED_STR "Failing read since good brick is down"
+#define AFR_MSG_LAUNCH_FAILED_STR "Failed to launch synctask"
+#define AFR_MSG_READ_SUBVOL_NOT_UP_STR                                         \
+    "read subvolume in this generation is not up"
+#define AFR_MSG_INTERNAL_LKS_FAILED_STR                                        \
+    "Unable to work with lk-owner while attempting fop"
+#define AFR_MSG_LOCK_XLATOR_NOT_LOADED_STR                                     \
+    "subvolume does not support locking. please load features/locks xlator "   \
+    "on server."
+#define AFR_MSG_FD_CTX_GET_FAILED_STR "unable to get fd ctx"
+#define AFR_MSG_INFO_COMMON_STR "fd not open on any subvolumes, aborting."
+#define AFR_MSG_REPLACE_BRICK_STATUS_STR "Couldn't acquire lock on any child."
+#define AFR_MSG_NEW_BRICK_STR "New brick"
+#define AFR_MSG_SPLIT_BRAIN_SET_FAILED_STR                                     \
+    "Failed to set split-brain choice to -1"
+#define AFR_MSG_SPLIT_BRAIN_DETERMINE_FAILED_STR                               \
+    "Failed to determine split-brain. Aborting split-brain-choice set"
+#define AFR_MSG_OPEN_FAIL_STR "Failed to open subvolume"
+#define AFR_MSG_SET_PEND_XATTR_STR "Set of pending xattr"
+#define AFR_MSG_INTERNAL_ATTR_STR "is an internal extended attribute"
 #endif /* !_AFR_MESSAGES_H_ */
diff --git a/xlators/cluster/afr/src/afr-open.c b/xlators/cluster/afr/src/afr-open.c
index f1da8b05201..64856042b65 100644
--- a/xlators/cluster/afr/src/afr-open.c
+++ b/xlators/cluster/afr/src/afr-open.c
@@ -8,327 +8,346 @@
   cases as published by the Free Software Foundation.
 */
 
-#include <libgen.h>
 #include <unistd.h>
-#include <fnmatch.h>
 #include <sys/time.h>
 #include <stdlib.h>
 #include <signal.h>
 
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
-
-#include "glusterfs.h"
+#include <glusterfs/glusterfs.h>
 #include "afr.h"
-#include "dict.h"
-#include "xlator.h"
-#include "hashfn.h"
-#include "logging.h"
-#include "stack.h"
-#include "list.h"
-#include "call-stub.h"
-#include "defaults.h"
-#include "common-utils.h"
-#include "compat-errno.h"
-#include "compat.h"
-#include "byte-order.h"
-#include "statedump.h"
-
-#include "fd.h"
-
-#include "afr-inode-read.h"
-#include "afr-inode-write.h"
-#include "afr-dir-read.h"
-#include "afr-dir-write.h"
-#include "afr-transaction.h"
+#include <glusterfs/dict.h>
+#include <glusterfs/logging.h>
+#include <glusterfs/defaults.h>
+#include <glusterfs/common-utils.h>
+#include <glusterfs/compat-errno.h>
+#include <glusterfs/compat.h>
+#include <glusterfs/byte-order.h>
+#include <glusterfs/statedump.h>
 
+#include "afr-transaction.h"
 
 gf_boolean_t
-afr_is_fd_fixable (fd_t *fd)
+afr_is_fd_fixable(fd_t *fd)
 {
-        if (!fd || !fd->inode)
-                return _gf_false;
-        else if (fd_is_anonymous (fd))
-                return _gf_false;
-        else if (gf_uuid_is_null (fd->inode->gfid))
-                return _gf_false;
-
-        return _gf_true;
+    if (!fd || !fd->inode)
+        return _gf_false;
+    else if (fd_is_anonymous(fd))
+        return _gf_false;
+    else if (gf_uuid_is_null(fd->inode->gfid))
+        return _gf_false;
+
+    return _gf_true;
 }
 
-
 int
-afr_open_ftruncate_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                        int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
-                        struct iatt *postbuf, dict_t *xdata)
+afr_open_ftruncate_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                       int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
+                       struct iatt *postbuf, dict_t *xdata)
 {
-        afr_local_t * local = frame->local;
+    afr_local_t *local = frame->local;
 
-        AFR_STACK_UNWIND (open, frame, local->op_ret, local->op_errno,
-                          local->fd, xdata);
-        return 0;
+    AFR_STACK_UNWIND(open, frame, local->op_ret, local->op_errno,
+                     local->cont.open.fd, xdata);
+    return 0;
 }
 
-
 int
-afr_open_cbk (call_frame_t *frame, void *cookie,
-              xlator_t *this, int32_t op_ret, int32_t op_errno,
-              fd_t *fd, dict_t *xdata)
+afr_open_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int32_t op_ret,
+             int32_t op_errno, fd_t *fd, dict_t *xdata)
 {
-        afr_local_t *  local       = NULL;
-        int            call_count  = -1;
-        int            child_index = (long) cookie;
-	afr_fd_ctx_t  *fd_ctx = NULL;
-
-        local = frame->local;
-	fd_ctx = local->fd_ctx;
-
-        LOCK (&frame->lock);
-        {
-                if (op_ret == -1) {
-                        local->op_errno = op_errno;
-			fd_ctx->opened_on[child_index] = AFR_FD_NOT_OPENED;
-                } else {
-                        local->op_ret = op_ret;
-			fd_ctx->opened_on[child_index] = AFR_FD_OPENED;
-			if (!local->xdata_rsp && xdata)
-				local->xdata_rsp = dict_ref (xdata);
-                }
+    afr_local_t *local = NULL;
+    int call_count = -1;
+    int child_index = (long)cookie;
+    afr_fd_ctx_t *fd_ctx = NULL;
+
+    local = frame->local;
+    fd_ctx = local->fd_ctx;
+
+    local->replies[child_index].valid = 1;
+    local->replies[child_index].op_ret = op_ret;
+    local->replies[child_index].op_errno = op_errno;
+
+    LOCK(&frame->lock);
+    {
+        if (op_ret == -1) {
+            local->op_errno = op_errno;
+            fd_ctx->opened_on[child_index] = AFR_FD_NOT_OPENED;
+        } else {
+            local->op_ret = op_ret;
+            fd_ctx->opened_on[child_index] = AFR_FD_OPENED;
+            if (!local->xdata_rsp && xdata)
+                local->xdata_rsp = dict_ref(xdata);
         }
-        UNLOCK (&frame->lock);
-
-        call_count = afr_frame_return (frame);
-
-        if (call_count == 0) {
-                if ((fd_ctx->flags & O_TRUNC) && (local->op_ret >= 0)) {
-                        STACK_WIND (frame, afr_open_ftruncate_cbk,
-                                    this, this->fops->ftruncate,
-                                    fd, 0, NULL);
-                } else {
-                        AFR_STACK_UNWIND (open, frame, local->op_ret,
-                                          local->op_errno, local->fd,
-					  local->xdata_rsp);
-                }
+        call_count = --local->call_count;
+    }
+    UNLOCK(&frame->lock);
+
+    if (call_count == 0) {
+        afr_handle_replies_quorum(frame, this);
+        if (local->op_ret == -1) {
+            AFR_STACK_UNWIND(open, frame, local->op_ret, local->op_errno, NULL,
+                             NULL);
+        } else if (fd_ctx->flags & O_TRUNC) {
+            STACK_WIND(frame, afr_open_ftruncate_cbk, this,
+                       this->fops->ftruncate, fd, 0, NULL);
+        } else {
+            AFR_STACK_UNWIND(open, frame, local->op_ret, local->op_errno,
+                             local->cont.open.fd, local->xdata_rsp);
         }
+    }
 
-        return 0;
+    return 0;
 }
 
 int
-afr_open (call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t flags,
-          fd_t *fd, dict_t *xdata)
+afr_open_continue(call_frame_t *frame, xlator_t *this, int err)
 {
-        afr_private_t * priv       = NULL;
-        afr_local_t *   local      = NULL;
-        int             i          = 0;
-        int32_t         call_count = 0;
-        int32_t         op_errno   = 0;
-	afr_fd_ctx_t   *fd_ctx = NULL;
-
-        //We can't let truncation to happen outside transaction.
-
-        priv = this->private;
-
-	local = AFR_FRAME_INIT (frame, op_errno);
-	if (!local)
-		goto out;
-
-	fd_ctx = afr_fd_ctx_get (fd, this);
-	if (!fd_ctx) {
-		op_errno = ENOMEM;
-		goto out;
-	}
-
-        local->fd = fd_ref (fd);
-	local->fd_ctx = fd_ctx;
-	fd_ctx->flags = flags;
-
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
+    int call_count = 0;
+    int i = 0;
+
+    local = frame->local;
+    priv = this->private;
+
+    if (err) {
+        AFR_STACK_UNWIND(open, frame, -1, err, NULL, NULL);
+    } else {
+        local->call_count = AFR_COUNT(local->child_up, priv->child_count);
         call_count = local->call_count;
 
-        local->cont.open.flags = flags;
-
         for (i = 0; i < priv->child_count; i++) {
-                if (local->child_up[i]) {
-                        STACK_WIND_COOKIE (frame, afr_open_cbk, (void *) (long) i,
-                                           priv->children[i],
-                                           priv->children[i]->fops->open,
-                                           loc, (flags & ~O_TRUNC), fd, xdata);
-                        if (!--call_count)
-                                break;
-                }
+            if (local->child_up[i]) {
+                STACK_WIND_COOKIE(frame, afr_open_cbk, (void *)(long)i,
+                                  priv->children[i],
+                                  priv->children[i]->fops->open, &local->loc,
+                                  (local->cont.open.flags & ~O_TRUNC),
+                                  local->cont.open.fd, local->xdata_req);
+                if (!--call_count)
+                    break;
+            }
         }
+    }
+    return 0;
+}
 
-	return 0;
+int
+afr_open(call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t flags,
+         fd_t *fd, dict_t *xdata)
+{
+    afr_private_t *priv = NULL;
+    afr_local_t *local = NULL;
+    int spb_subvol = 0;
+    int event_generation = 0;
+    int ret = 0;
+    int32_t op_errno = 0;
+    afr_fd_ctx_t *fd_ctx = NULL;
+
+    // We can't let truncation to happen outside transaction.
+
+    priv = this->private;
+
+    local = AFR_FRAME_INIT(frame, op_errno);
+    if (!local)
+        goto out;
+
+    local->op = GF_FOP_OPEN;
+    fd_ctx = afr_fd_ctx_get(fd, this);
+    if (!fd_ctx) {
+        op_errno = ENOMEM;
+        goto out;
+    }
+
+    if (priv->quorum_count && !afr_has_quorum(local->child_up, this, NULL)) {
+        op_errno = afr_quorum_errno(priv);
+        goto out;
+    }
+
+    if (!afr_is_consistent_io_possible(local, priv, &op_errno))
+        goto out;
+
+    local->inode = inode_ref(loc->inode);
+    loc_copy(&local->loc, loc);
+    local->fd_ctx = fd_ctx;
+    fd_ctx->flags = flags;
+    if (xdata)
+        local->xdata_req = dict_ref(xdata);
+
+    local->cont.open.flags = flags;
+    local->cont.open.fd = fd_ref(fd);
+
+    ret = afr_inode_get_readable(frame, local->inode, this, NULL,
+                                 &event_generation, AFR_DATA_TRANSACTION);
+    if ((ret < 0) &&
+        (afr_split_brain_read_subvol_get(local->inode, this, NULL,
+                                         &spb_subvol) == 0) &&
+        spb_subvol < 0) {
+        afr_inode_refresh(frame, this, local->inode, local->inode->gfid,
+                          afr_open_continue);
+    } else {
+        afr_open_continue(frame, this, 0);
+    }
+
+    return 0;
 out:
-	AFR_STACK_UNWIND (open, frame, -1, op_errno, fd, NULL);
+    AFR_STACK_UNWIND(open, frame, -1, op_errno, fd, NULL);
 
-        return 0;
+    return 0;
 }
 
 int
-afr_openfd_fix_open_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                         int32_t op_ret, int32_t op_errno, fd_t *fd,
-                         dict_t *xdata)
+afr_openfd_fix_open_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                        int32_t op_ret, int32_t op_errno, fd_t *fd,
+                        dict_t *xdata)
 {
-        afr_local_t   *local      = NULL;
-        afr_private_t *priv       = NULL;
-        afr_fd_ctx_t  *fd_ctx     = NULL;
-        int           call_count  = 0;
-        int           child_index = (long) cookie;
-
-        priv     = this->private;
-        local    = frame->local;
-
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
+    afr_fd_ctx_t *fd_ctx = NULL;
+    int call_count = 0;
+    int child_index = (long)cookie;
+
+    priv = this->private;
+    local = frame->local;
+
+    if (op_ret >= 0) {
+        gf_msg_debug(this->name, 0,
+                     "fd for %s opened "
+                     "successfully on subvolume %s",
+                     local->loc.path, priv->children[child_index]->name);
+    } else {
+        gf_smsg(this->name, fop_log_level(GF_FOP_OPEN, op_errno), op_errno,
+                AFR_MSG_OPEN_FAIL, "path=%s", local->loc.path, "subvolume=%s",
+                priv->children[child_index]->name, NULL);
+    }
+
+    fd_ctx = local->fd_ctx;
+
+    LOCK(&local->fd->lock);
+    {
         if (op_ret >= 0) {
-                gf_log (this->name, GF_LOG_DEBUG, "fd for %s opened "
-                        "successfully on subvolume %s", local->loc.path,
-                        priv->children[child_index]->name);
+            fd_ctx->opened_on[child_index] = AFR_FD_OPENED;
         } else {
-                gf_msg (this->name, fop_log_level (GF_FOP_OPEN, op_errno),
-                        op_errno, AFR_MSG_OPEN_FAIL, "Failed to open %s on "
-                        "subvolume %s", local->loc.path,
-                        priv->children[child_index]->name);
+            fd_ctx->opened_on[child_index] = AFR_FD_NOT_OPENED;
         }
+    }
+    UNLOCK(&local->fd->lock);
 
-        fd_ctx = local->fd_ctx;
+    call_count = afr_frame_return(frame);
+    if (call_count == 0)
+        AFR_STACK_DESTROY(frame);
 
-        LOCK (&local->fd->lock);
-        {
-                if (op_ret >= 0) {
-                        fd_ctx->opened_on[child_index] = AFR_FD_OPENED;
-                } else {
-                        fd_ctx->opened_on[child_index] = AFR_FD_NOT_OPENED;
-                }
-        }
-        UNLOCK (&local->fd->lock);
+    return 0;
+}
 
-        call_count = afr_frame_return (frame);
-        if (call_count == 0)
-                AFR_STACK_DESTROY (frame);
+static int
+afr_fd_ctx_need_open(fd_t *fd, xlator_t *this, unsigned char *need_open)
+{
+    afr_fd_ctx_t *fd_ctx = NULL;
+    afr_private_t *priv = NULL;
+    int i = 0;
+    int count = 0;
+
+    priv = this->private;
 
+    fd_ctx = afr_fd_ctx_get(fd, this);
+    if (!fd_ctx)
         return 0;
-}
 
+    LOCK(&fd->lock);
+    {
+        for (i = 0; i < priv->child_count; i++) {
+            if (fd_ctx->opened_on[i] == AFR_FD_NOT_OPENED &&
+                priv->child_up[i]) {
+                fd_ctx->opened_on[i] = AFR_FD_OPENING;
+                need_open[i] = 1;
+                count++;
+            } else {
+                need_open[i] = 0;
+            }
+        }
+    }
+    UNLOCK(&fd->lock);
 
-static int
-afr_fd_ctx_need_open (fd_t *fd, xlator_t *this, unsigned char *need_open)
-{
-	afr_fd_ctx_t *fd_ctx = NULL;
-	afr_private_t *priv = NULL;
-	int i = 0;
-	int count = 0;
-
-	priv = this->private;
-
-	fd_ctx = afr_fd_ctx_get (fd, this);
-	if (!fd_ctx)
-		return 0;
-
-	LOCK (&fd->lock);
-	{
-		for (i = 0; i < priv->child_count; i++) {
-			if (fd_ctx->opened_on[i] == AFR_FD_NOT_OPENED &&
-			    priv->child_up[i]) {
-				fd_ctx->opened_on[i] = AFR_FD_OPENING;
-				need_open[i] = 1;
-				count++;
-			} else {
-				need_open[i] = 0;
-			}
-		}
-	}
-	UNLOCK (&fd->lock);
-
-	return count;
+    return count;
 }
 
-
 void
-afr_fix_open (fd_t *fd, xlator_t *this)
+afr_fix_open(fd_t *fd, xlator_t *this)
 {
-        afr_private_t *priv    = NULL;
-        int           i        = 0;
-        call_frame_t  *frame   = NULL;
-        afr_local_t   *local   = NULL;
-        int           ret      = -1;
-        int32_t       op_errno = 0;
-        afr_fd_ctx_t  *fd_ctx  = NULL;
-	unsigned char *need_open = NULL;
-	int call_count = 0;
+    afr_private_t *priv = NULL;
+    int i = 0;
+    call_frame_t *frame = NULL;
+    afr_local_t *local = NULL;
+    int ret = -1;
+    int32_t op_errno = 0;
+    afr_fd_ctx_t *fd_ctx = NULL;
+    unsigned char *need_open = NULL;
+    int call_count = 0;
 
-        priv  = this->private;
+    priv = this->private;
 
-        if (!afr_is_fd_fixable (fd))
-                goto out;
+    if (!afr_is_fd_fixable(fd))
+        goto out;
 
-        fd_ctx = afr_fd_ctx_get (fd, this);
-        if (!fd_ctx)
-                goto out;
+    fd_ctx = afr_fd_ctx_get(fd, this);
+    if (!fd_ctx)
+        goto out;
 
-	need_open = alloca0 (priv->child_count);
+    need_open = alloca0(priv->child_count);
 
-	call_count = afr_fd_ctx_need_open (fd, this, need_open);
-	if (!call_count)
-		goto out;
+    call_count = afr_fd_ctx_need_open(fd, this, need_open);
+    if (!call_count)
+        goto out;
 
-        frame = create_frame (this, this->ctx->pool);
-        if (!frame)
-                goto out;
+    frame = create_frame(this, this->ctx->pool);
+    if (!frame)
+        goto out;
 
-	local = AFR_FRAME_INIT (frame, op_errno);
-	if (!local)
-		goto out;
+    local = AFR_FRAME_INIT(frame, op_errno);
+    if (!local)
+        goto out;
 
-        local->loc.inode = inode_ref (fd->inode);
-        ret = loc_path (&local->loc, NULL);
-        if (ret < 0)
-                goto out;
+    local->loc.inode = inode_ref(fd->inode);
+    ret = loc_path(&local->loc, NULL);
+    if (ret < 0)
+        goto out;
 
-        local->fd = fd_ref (fd);
-	local->fd_ctx = fd_ctx;
+    local->fd = fd_ref(fd);
+    local->fd_ctx = fd_ctx;
 
-        local->call_count = call_count;
+    local->call_count = call_count;
 
-        gf_log (this->name, GF_LOG_DEBUG, "need open count: %d",
-                call_count);
+    gf_msg_debug(this->name, 0, "need open count: %d", call_count);
 
-        for (i = 0; i < priv->child_count; i++) {
-                if (!need_open[i])
-                        continue;
-
-                if (IA_IFDIR == fd->inode->ia_type) {
-                        gf_log (this->name, GF_LOG_DEBUG,
-                                "opening fd for dir %s on subvolume %s",
-                                local->loc.path, priv->children[i]->name);
-
-                        STACK_WIND_COOKIE (frame, afr_openfd_fix_open_cbk,
-                                           (void*) (long) i,
-                                           priv->children[i],
-                                           priv->children[i]->fops->opendir,
-                                           &local->loc, local->fd,
-                                           NULL);
-                } else {
-                        gf_log (this->name, GF_LOG_DEBUG,
-                                "opening fd for file %s on subvolume %s",
-                                local->loc.path, priv->children[i]->name);
-
-                        STACK_WIND_COOKIE (frame, afr_openfd_fix_open_cbk,
-                                           (void *)(long) i,
-                                           priv->children[i],
-                                           priv->children[i]->fops->open,
-                                           &local->loc,
-                                           fd_ctx->flags & (~O_TRUNC),
-                                           local->fd, NULL);
-                }
-
-		if (!--call_count)
-			break;
+    for (i = 0; i < priv->child_count; i++) {
+        if (!need_open[i])
+            continue;
+
+        if (IA_IFDIR == fd->inode->ia_type) {
+            gf_msg_debug(this->name, 0, "opening fd for dir %s on subvolume %s",
+                         local->loc.path, priv->children[i]->name);
+
+            STACK_WIND_COOKIE(frame, afr_openfd_fix_open_cbk, (void *)(long)i,
+                              priv->children[i],
+                              priv->children[i]->fops->opendir, &local->loc,
+                              local->fd, NULL);
+        } else {
+            gf_msg_debug(this->name, 0,
+                         "opening fd for file %s on subvolume %s",
+                         local->loc.path, priv->children[i]->name);
+
+            STACK_WIND_COOKIE(frame, afr_openfd_fix_open_cbk, (void *)(long)i,
+                              priv->children[i], priv->children[i]->fops->open,
+                              &local->loc, fd_ctx->flags & (~O_TRUNC),
+                              local->fd, NULL);
         }
 
-	return;
+        if (!--call_count)
+            break;
+    }
+
+    return;
 out:
-        if (frame)
-                AFR_STACK_DESTROY (frame);
+    if (frame)
+        AFR_STACK_DESTROY(frame);
 }
diff --git a/xlators/cluster/afr/src/afr-read-txn.c b/xlators/cluster/afr/src/afr-read-txn.c
index eaa73d9be20..6fc2c75145c 100644
--- a/xlators/cluster/afr/src/afr-read-txn.c
+++ b/xlators/cluster/afr/src/afr-read-txn.c
@@ -10,123 +10,329 @@
 
 #include "afr.h"
 #include "afr-transaction.h"
+#include "afr-messages.h"
 
-int
-afr_read_txn_next_subvol (call_frame_t *frame, xlator_t *this)
+void
+afr_pending_read_increment(afr_private_t *priv, int child_index)
 {
-	afr_local_t *local = NULL;
-	afr_private_t *priv = NULL;
-	int i = 0;
-	int subvol = -1;
-
-	local = frame->local;
-	priv = this->private;
-
-
-	for (i = 0; i < priv->child_count; i++) {
-		if (!local->readable[i]) {
-			/* don't even bother trying here.
-			   just mark as attempted and move on. */
-			local->read_attempted[i] = 1;
-			continue;
-		}
-
-		if (!local->read_attempted[i]) {
-			subvol = i;
-			break;
-		}
-	}
-
-	/* If no more subvols were available for reading, we leave
-	   @subvol as -1, which is an indication we have run out of
-	   readable subvols. */
-	if (subvol != -1)
-		local->read_attempted[subvol] = 1;
-	local->readfn (frame, this, subvol);
-
-	return 0;
+    if (child_index < 0 || child_index > priv->child_count)
+        return;
+
+    GF_ATOMIC_INC(priv->pending_reads[child_index]);
 }
 
+void
+afr_pending_read_decrement(afr_private_t *priv, int child_index)
+{
+    if (child_index < 0 || child_index > priv->child_count)
+        return;
+
+    GF_ATOMIC_DEC(priv->pending_reads[child_index]);
+}
+
+void
+afr_read_txn_wind(call_frame_t *frame, xlator_t *this, int subvol)
+{
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
+
+    local = frame->local;
+    priv = this->private;
+
+    afr_pending_read_decrement(priv, local->read_subvol);
+    local->read_subvol = subvol;
+    afr_pending_read_increment(priv, subvol);
+    local->readfn(frame, this, subvol);
+}
 
 int
-afr_read_txn_refresh_done (call_frame_t *frame, xlator_t *this, int err)
+afr_read_txn_next_subvol(call_frame_t *frame, xlator_t *this)
 {
-	afr_local_t *local = NULL;
-	int read_subvol = 0;
-	int event_generation = 0;
-	inode_t *inode = NULL;
-	int ret = -1;
-        int spb_choice = -1;
-
-	local = frame->local;
-	inode = local->inode;
-
-	if (err) {
-		local->op_errno = -err;
-		local->op_ret = -1;
-		read_subvol = -1;
-		goto readfn;
-	}
-
-	ret = afr_inode_read_subvol_type_get (inode, this, local->readable,
-					      &event_generation,
-					      local->transaction.type);
-
-	if (ret == -1 || !event_generation) {
-		/* Even after refresh, we don't have a good
-		   read subvolume. Time to bail */
-		local->op_ret = -1;
-		local->op_errno = EIO;
-		read_subvol = -1;
-		goto readfn;
-	}
-
-	read_subvol = afr_read_subvol_select_by_policy (inode, this,
-							local->readable);
-
-	if (read_subvol == -1) {
-		local->op_ret = -1;
-		local->op_errno = EIO;
-		goto readfn;
-	}
-
-	if (local->read_attempted[read_subvol]) {
-		afr_read_txn_next_subvol (frame, this);
-		return 0;
-	}
-
-	local->read_attempted[read_subvol] = 1;
-readfn:
-        if (read_subvol == -1) {
-                ret = afr_inode_split_brain_choice_get (inode, this,
-                                                        &spb_choice);
-                if ((ret == 0) && spb_choice >= 0)
-                        read_subvol = spb_choice;
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
+    int i = 0;
+    int subvol = -1;
+
+    local = frame->local;
+    priv = this->private;
+
+    for (i = 0; i < priv->child_count; i++) {
+        if (!local->readable[i]) {
+            /* don't even bother trying here.
+               just mark as attempted and move on. */
+            local->read_attempted[i] = 1;
+            continue;
         }
-	local->readfn (frame, this, read_subvol);
 
-	return 0;
+        if (!local->read_attempted[i]) {
+            subvol = i;
+            break;
+        }
+    }
+
+    /* If no more subvols were available for reading, we leave
+       @subvol as -1, which is an indication we have run out of
+       readable subvols. */
+    if (subvol != -1)
+        local->read_attempted[subvol] = 1;
+    afr_read_txn_wind(frame, this, subvol);
+
+    return 0;
 }
 
+static int
+afr_ta_read_txn_done(int ret, call_frame_t *ta_frame, void *opaque)
+{
+    STACK_DESTROY(ta_frame->root);
+    return 0;
+}
+
+static int
+afr_ta_read_txn(void *opaque)
+{
+    call_frame_t *frame = NULL;
+    xlator_t *this = NULL;
+    int read_subvol = -1;
+    int query_child = AFR_CHILD_UNKNOWN;
+    int possible_bad_child = AFR_CHILD_UNKNOWN;
+    int ret = 0;
+    int op_errno = ENOMEM;
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
+    struct gf_flock flock = {
+        0,
+    };
+    dict_t *xdata_req = NULL;
+    dict_t *xdata_rsp = NULL;
+    int **pending = NULL;
+    loc_t loc = {
+        0,
+    };
+
+    frame = (call_frame_t *)opaque;
+    this = frame->this;
+    local = frame->local;
+    priv = this->private;
+    query_child = local->read_txn_query_child;
+
+    if (query_child == AFR_CHILD_ZERO) {
+        possible_bad_child = AFR_CHILD_ONE;
+    } else if (query_child == AFR_CHILD_ONE) {
+        possible_bad_child = AFR_CHILD_ZERO;
+    } else {
+        /*read_txn_query_child is AFR_CHILD_UNKNOWN*/
+        goto out;
+    }
+
+    /* Ask the query_child to see if it blames the possibly bad one. */
+    xdata_req = dict_new();
+    if (!xdata_req)
+        goto out;
+
+    pending = afr_matrix_create(priv->child_count, AFR_NUM_CHANGE_LOGS);
+    if (!pending)
+        goto out;
+
+    ret = afr_set_pending_dict(priv, xdata_req, pending);
+    if (ret < 0)
+        goto out;
+
+    if (local->fd) {
+        ret = syncop_fxattrop(priv->children[query_child], local->fd,
+                              GF_XATTROP_ADD_ARRAY, xdata_req, NULL, &xdata_rsp,
+                              NULL);
+    } else {
+        ret = syncop_xattrop(priv->children[query_child], &local->loc,
+                             GF_XATTROP_ADD_ARRAY, xdata_req, NULL, &xdata_rsp,
+                             NULL);
+    }
+    if (ret || !xdata_rsp) {
+        gf_msg(this->name, GF_LOG_ERROR, -ret, AFR_MSG_THIN_ARB,
+               "Failed xattrop for gfid %s on %s",
+               uuid_utoa(local->inode->gfid),
+               priv->children[query_child]->name);
+        op_errno = -ret;
+        goto out;
+    }
+
+    if (afr_ta_dict_contains_pending_xattr(xdata_rsp, priv,
+                                           possible_bad_child)) {
+        read_subvol = query_child;
+        goto out;
+    }
+    dict_unref(xdata_rsp);
+    xdata_rsp = NULL;
+
+    /* It doesn't. So query thin-arbiter to see if it blames any data brick. */
+    ret = afr_fill_ta_loc(this, &loc, _gf_true);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, -ret, AFR_MSG_THIN_ARB,
+               "Failed to populate thin-arbiter loc for: %s.", loc.name);
+        goto out;
+    }
+    flock.l_type = F_WRLCK; /*start and length are already zero. */
+    ret = syncop_inodelk(priv->children[THIN_ARBITER_BRICK_INDEX],
+                         AFR_TA_DOM_MODIFY, &loc, F_SETLKW, &flock, NULL, NULL);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, -ret, AFR_MSG_THIN_ARB,
+               "gfid:%s: Failed to get AFR_TA_DOM_MODIFY lock on %s.",
+               uuid_utoa(local->inode->gfid),
+               priv->pending_key[THIN_ARBITER_BRICK_INDEX]);
+        op_errno = -ret;
+        goto out;
+    }
+
+    ret = syncop_xattrop(priv->children[THIN_ARBITER_BRICK_INDEX], &loc,
+                         GF_XATTROP_ADD_ARRAY, xdata_req, NULL, &xdata_rsp,
+                         NULL);
+    if (ret || !xdata_rsp) {
+        gf_msg(this->name, GF_LOG_ERROR, -ret, AFR_MSG_THIN_ARB,
+               "gfid:%s: Failed xattrop on %s.", uuid_utoa(local->inode->gfid),
+               priv->pending_key[THIN_ARBITER_BRICK_INDEX]);
+        op_errno = -ret;
+        goto unlock;
+    }
+
+    if (!afr_ta_dict_contains_pending_xattr(xdata_rsp, priv, query_child)) {
+        read_subvol = query_child;
+    } else {
+        gf_msg(this->name, GF_LOG_ERROR, EIO, AFR_MSG_THIN_ARB,
+               "Failing read for gfid %s since good brick %s is down",
+               uuid_utoa(local->inode->gfid),
+               priv->children[possible_bad_child]->name);
+        op_errno = EIO;
+    }
+
+unlock:
+    flock.l_type = F_UNLCK;
+    ret = syncop_inodelk(priv->children[THIN_ARBITER_BRICK_INDEX],
+                         AFR_TA_DOM_MODIFY, &loc, F_SETLK, &flock, NULL, NULL);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, -ret, AFR_MSG_THIN_ARB,
+               "gfid:%s: Failed to unlock AFR_TA_DOM_MODIFY lock on "
+               "%s.",
+               uuid_utoa(local->inode->gfid),
+               priv->pending_key[THIN_ARBITER_BRICK_INDEX]);
+    }
+out:
+    if (xdata_req)
+        dict_unref(xdata_req);
+    if (xdata_rsp)
+        dict_unref(xdata_rsp);
+    if (pending)
+        afr_matrix_cleanup(pending, priv->child_count);
+    loc_wipe(&loc);
+
+    if (read_subvol == -1) {
+        local->op_ret = -1;
+        local->op_errno = op_errno;
+    }
+    afr_read_txn_wind(frame, this, read_subvol);
+    return ret;
+}
+
+void
+afr_ta_read_txn_synctask(call_frame_t *frame, xlator_t *this)
+{
+    call_frame_t *ta_frame = NULL;
+    afr_local_t *local = NULL;
+    int ret = 0;
+
+    local = frame->local;
+    ta_frame = afr_ta_frame_create(this);
+    if (!ta_frame) {
+        local->op_ret = -1;
+        local->op_errno = ENOMEM;
+        gf_msg(this->name, GF_LOG_ERROR, ENOMEM, AFR_MSG_THIN_ARB,
+               "Failed to create ta_frame");
+        goto out;
+    }
+    ret = synctask_new(this->ctx->env, afr_ta_read_txn, afr_ta_read_txn_done,
+                       ta_frame, frame);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, ENOMEM, AFR_MSG_THIN_ARB,
+               "Failed to launch "
+               "afr_ta_read_txn synctask for gfid %s.",
+               uuid_utoa(local->inode->gfid));
+        local->op_ret = -1;
+        local->op_errno = ENOMEM;
+        STACK_DESTROY(ta_frame->root);
+        goto out;
+    }
+    return;
+out:
+    afr_read_txn_wind(frame, this, -1);
+}
 
 int
-afr_read_txn_continue (call_frame_t *frame, xlator_t *this, int subvol)
+afr_read_txn_refresh_done(call_frame_t *frame, xlator_t *this, int err)
 {
-	afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
+    afr_local_t *local = NULL;
+    int read_subvol = -1;
+    inode_t *inode = NULL;
+    int ret = -1;
+    int spb_subvol = -1;
+
+    local = frame->local;
+    inode = local->inode;
+    priv = this->private;
+
+    if (err) {
+        if (!priv->thin_arbiter_count)
+            goto readfn;
+        if (err != EINVAL)
+            goto readfn;
+        /* We need to query the good bricks and/or thin-arbiter.*/
+        afr_ta_read_txn_synctask(frame, this);
+        return 0;
+    }
+
+    read_subvol = afr_read_subvol_select_by_policy(inode, this, local->readable,
+                                                   NULL);
+    if (read_subvol == -1) {
+        err = EIO;
+        goto readfn;
+    }
+
+    if (local->read_attempted[read_subvol]) {
+        afr_read_txn_next_subvol(frame, this);
+        return 0;
+    }
+
+    local->read_attempted[read_subvol] = 1;
+readfn:
+    if (read_subvol == -1) {
+        ret = afr_split_brain_read_subvol_get(inode, this, frame, &spb_subvol);
+        if ((ret == 0) && spb_subvol >= 0)
+            read_subvol = spb_subvol;
+    }
+
+    if (read_subvol == -1) {
+        AFR_SET_ERROR_AND_CHECK_SPLIT_BRAIN(-1, err);
+    }
+    afr_read_txn_wind(frame, this, read_subvol);
+
+    return 0;
+}
 
-	local = frame->local;
+int
+afr_read_txn_continue(call_frame_t *frame, xlator_t *this, int subvol)
+{
+    afr_local_t *local = NULL;
 
-	if (!local->refreshed) {
-		local->refreshed = _gf_true;
-		afr_inode_refresh (frame, this, local->inode,
-				   afr_read_txn_refresh_done);
-	} else {
-		afr_read_txn_next_subvol (frame, this);
-	}
+    local = frame->local;
 
-	return 0;
-}
+    if (!local->refreshed) {
+        local->refreshed = _gf_true;
+        afr_inode_refresh(frame, this, local->inode, NULL,
+                          afr_read_txn_refresh_done);
+    } else {
+        afr_read_txn_next_subvol(frame, this);
+    }
 
+    return 0;
+}
 
 /* afr_read_txn_wipe:
 
@@ -136,27 +342,26 @@ afr_read_txn_continue (call_frame_t *frame, xlator_t *this, int subvol)
 */
 
 void
-afr_read_txn_wipe (call_frame_t *frame, xlator_t *this)
+afr_read_txn_wipe(call_frame_t *frame, xlator_t *this)
 {
-	afr_local_t *local = NULL;
-	afr_private_t *priv = NULL;
-	int i = 0;
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
+    int i = 0;
 
-	local = frame->local;
-	priv = this->private;
+    local = frame->local;
+    priv = this->private;
 
-	local->readfn = NULL;
+    local->readfn = NULL;
 
-	if (local->inode)
-		inode_unref (local->inode);
+    if (local->inode)
+        inode_unref(local->inode);
 
-	for (i = 0; i < priv->child_count; i++) {
-		local->read_attempted[i] = 0;
-		local->readable[i] = 0;
-	}
+    for (i = 0; i < priv->child_count; i++) {
+        local->read_attempted[i] = 0;
+        local->readable[i] = 0;
+    }
 }
 
-
 /*
   afr_read_txn:
 
@@ -185,76 +390,105 @@ afr_read_txn_wipe (call_frame_t *frame, xlator_t *this)
 */
 
 int
-afr_read_txn (call_frame_t *frame, xlator_t *this, inode_t *inode,
-	      afr_read_txn_wind_t readfn, afr_transaction_type type)
+afr_read_txn(call_frame_t *frame, xlator_t *this, inode_t *inode,
+             afr_read_txn_wind_t readfn, afr_transaction_type type)
 {
-	afr_local_t *local = NULL;
-	afr_private_t *priv = NULL;
-	int read_subvol = -1;
-	int event_generation = 0;
-	int ret = -1;
-
-	priv = this->private;
-	local = frame->local;
-
-	afr_read_txn_wipe (frame, this);
-
-	local->readfn = readfn;
-	local->inode = inode_ref (inode);
-
-        if (priv->quorum_reads &&
-            priv->quorum_count && !afr_has_quorum (priv->child_up, this)) {
-                local->op_ret = -1;
-                local->op_errno = ENOTCONN;
-                read_subvol = -1;
-                goto read;
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
+    unsigned char *data = NULL;
+    unsigned char *metadata = NULL;
+    int read_subvol = -1;
+    int event_generation = 0;
+    int ret = -1;
+
+    priv = this->private;
+    local = frame->local;
+    data = alloca0(priv->child_count);
+    metadata = alloca0(priv->child_count);
+
+    afr_read_txn_wipe(frame, this);
+
+    local->readfn = readfn;
+    local->inode = inode_ref(inode);
+    local->is_read_txn = _gf_true;
+    local->transaction.type = type;
+
+    if (priv->quorum_count && !afr_has_quorum(local->child_up, this, NULL)) {
+        local->op_ret = -1;
+        local->op_errno = afr_quorum_errno(priv);
+        goto read;
+    }
+
+    if (!afr_is_consistent_io_possible(local, priv, &local->op_errno)) {
+        local->op_ret = -1;
+        goto read;
+    }
+
+    if (priv->thin_arbiter_count && !afr_ta_has_quorum(priv, local)) {
+        local->op_ret = -1;
+        local->op_errno = -afr_quorum_errno(priv);
+        goto read;
+    }
+
+    if (priv->thin_arbiter_count &&
+        AFR_COUNT(local->child_up, priv->child_count) != priv->child_count) {
+        if (local->child_up[0]) {
+            local->read_txn_query_child = AFR_CHILD_ZERO;
+        } else if (local->child_up[1]) {
+            local->read_txn_query_child = AFR_CHILD_ONE;
         }
-
-
-	local->transaction.type = type;
-	ret = afr_inode_read_subvol_type_get (inode, this, local->readable,
-					      &event_generation, type);
-	if (ret == -1)
-		/* very first transaction on this inode */
-		goto refresh;
-
-        gf_log (this->name, GF_LOG_DEBUG, "%s: generation now vs cached: %d, "
-                "%d", uuid_utoa (inode->gfid), local->event_generation,
-                event_generation);
-	if (local->event_generation != event_generation)
-		/* servers have disconnected / reconnected, and possibly
-		   rebooted, very likely changing the state of freshness
-		   of copies */
-		goto refresh;
-
-	read_subvol = afr_read_subvol_select_by_policy (inode, this,
-							local->readable);
-
-	if (read_subvol < 0 || read_subvol > priv->child_count) {
-		gf_msg (this->name, GF_LOG_WARNING, 0, AFR_MSG_SPLIT_BRAIN,
-                       "Unreadable subvolume %d found with event generation "
-                       "%d. (Possible split-brain)",
-                        read_subvol, event_generation);
-		goto refresh;
-	}
-
-	if (!local->child_up[read_subvol]) {
-		/* should never happen, just in case */
-		gf_log (this->name, GF_LOG_WARNING, "subvolume %d is the "
-			"read subvolume in this generation, but is not up",
-			read_subvol);
-		goto refresh;
-	}
-
-	local->read_attempted[read_subvol] = 1;
+        afr_ta_read_txn_synctask(frame, this);
+        return 0;
+    }
+
+    ret = afr_inode_read_subvol_get(inode, this, data, metadata,
+                                    &event_generation);
+    if (ret == -1)
+        /* very first transaction on this inode */
+        goto refresh;
+    AFR_INTERSECT(local->readable, data, metadata, priv->child_count);
+
+    gf_msg_debug(this->name, 0,
+                 "%s: generation now vs cached: %d, "
+                 "%d",
+                 uuid_utoa(inode->gfid), local->event_generation,
+                 event_generation);
+    if (afr_is_inode_refresh_reqd(inode, this, local->event_generation,
+                                  event_generation))
+        /* servers have disconnected / reconnected, and possibly
+           rebooted, very likely changing the state of freshness
+           of copies */
+        goto refresh;
+
+    read_subvol = afr_read_subvol_select_by_policy(inode, this, local->readable,
+                                                   NULL);
+
+    if (read_subvol < 0 || read_subvol > priv->child_count) {
+        gf_msg_debug(this->name, 0,
+                     "Unreadable subvolume %d found "
+                     "with event generation %d for gfid %s.",
+                     read_subvol, event_generation, uuid_utoa(inode->gfid));
+        goto refresh;
+    }
+
+    if (!local->child_up[read_subvol]) {
+        /* should never happen, just in case */
+        gf_msg(this->name, GF_LOG_WARNING, 0, AFR_MSG_READ_SUBVOL_ERROR,
+               "subvolume %d is the "
+               "read subvolume in this generation, but is not up",
+               read_subvol);
+        goto refresh;
+    }
+
+    local->read_attempted[read_subvol] = 1;
 
 read:
-	local->readfn (frame, this, read_subvol);
+    afr_read_txn_wind(frame, this, read_subvol);
 
-	return 0;
+    return 0;
 
 refresh:
-	afr_inode_refresh (frame, this, inode, afr_read_txn_refresh_done);
+    afr_inode_refresh(frame, this, inode, NULL, afr_read_txn_refresh_done);
 
-	return 0;
+    return 0;
 }
diff --git a/xlators/cluster/afr/src/afr-self-heal-common.c b/xlators/cluster/afr/src/afr-self-heal-common.c
index f3d1f8b0b14..a580a1584cc 100644
--- a/xlators/cluster/afr/src/afr-self-heal-common.c
+++ b/xlators/cluster/afr/src/afr-self-heal-common.c
@@ -8,283 +8,808 @@
   cases as published by the Free Software Foundation.
 */
 
-
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
-
 #include "afr.h"
 #include "afr-self-heal.h"
-#include "byte-order.h"
+#include <glusterfs/byte-order.h>
 #include "protocol-common.h"
+#include "afr-messages.h"
+#include <glusterfs/events.h>
+
+void
+afr_heal_synctask(xlator_t *this, afr_local_t *local);
 
 int
-afr_selfheal_post_op_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-			  int op_ret, int op_errno, dict_t *xattr, dict_t *xdata)
+afr_lookup_and_heal_gfid(xlator_t *this, inode_t *parent, const char *name,
+                         inode_t *inode, struct afr_reply *replies, int source,
+                         unsigned char *sources, void *gfid, int *gfid_idx)
 {
-	afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
+    call_frame_t *frame = NULL;
+    afr_local_t *local = NULL;
+    unsigned char *wind_on = NULL;
+    ia_type_t ia_type = IA_INVAL;
+    dict_t *xdata = NULL;
+    loc_t loc = {
+        0,
+    };
+    int ret = 0;
+    int i = 0;
+
+    priv = this->private;
+    wind_on = alloca0(priv->child_count);
+    if (source >= 0 && replies[source].valid && replies[source].op_ret == 0)
+        ia_type = replies[source].poststat.ia_type;
+
+    if (ia_type != IA_INVAL)
+        goto heal;
+
+    /* If ia_type is still invalid, it means either
+     * (a)'source' was -1, i.e. parent dir pending xattrs are in split-brain
+     * (or) (b) The parent dir pending xattrs are all zeroes (i.e. all bricks
+     * are sources) and the 'source' we selected earlier might be the one where
+     * the file is not actually present.
+     *
+     * In both cases, let us pick a brick with a successful reply and use its
+     * ia_type.
+     * */
+    for (i = 0; i < priv->child_count; i++) {
+        if (source == -1) {
+            /* case (a) above. */
+            if (replies[i].valid && replies[i].op_ret == 0 &&
+                replies[i].poststat.ia_type != IA_INVAL) {
+                ia_type = replies[i].poststat.ia_type;
+                break;
+            }
+        } else {
+            /* case (b) above. */
+            if (i == source)
+                continue;
+            if (sources[i] && replies[i].valid && replies[i].op_ret == 0 &&
+                replies[i].poststat.ia_type != IA_INVAL) {
+                ia_type = replies[i].poststat.ia_type;
+                break;
+            }
+        }
+    }
+
+heal:
+    /* gfid heal on those subvolumes that do not have gfid associated
+     * with the inode and update those replies.
+     */
+    for (i = 0; i < priv->child_count; i++) {
+        if (!replies[i].valid || replies[i].op_ret != 0)
+            continue;
+
+        if (gf_uuid_is_null(gfid) &&
+            !gf_uuid_is_null(replies[i].poststat.ia_gfid) &&
+            replies[i].poststat.ia_type == ia_type)
+            gfid = replies[i].poststat.ia_gfid;
+
+        if (!gf_uuid_is_null(replies[i].poststat.ia_gfid) ||
+            replies[i].poststat.ia_type != ia_type)
+            continue;
+
+        wind_on[i] = 1;
+    }
+
+    if (AFR_COUNT(wind_on, priv->child_count) == 0)
+        return 0;
+
+    xdata = dict_new();
+    if (!xdata) {
+        ret = -ENOMEM;
+        goto out;
+    }
+
+    ret = dict_set_gfuuid(xdata, "gfid-req", gfid, true);
+    if (ret) {
+        ret = -ENOMEM;
+        goto out;
+    }
+
+    frame = afr_frame_create(this, &ret);
+    if (!frame) {
+        ret = -ret;
+        goto out;
+    }
+
+    local = frame->local;
+    loc.parent = inode_ref(parent);
+    gf_uuid_copy(loc.pargfid, parent->gfid);
+    loc.name = name;
+    loc.inode = inode_ref(inode);
+
+    AFR_ONLIST(wind_on, frame, afr_selfheal_discover_cbk, lookup, &loc, xdata);
+
+    for (i = 0; i < priv->child_count; i++) {
+        if (!wind_on[i])
+            continue;
+        afr_reply_wipe(&replies[i]);
+        afr_reply_copy(&replies[i], &local->replies[i]);
+    }
+    if (gfid_idx && (*gfid_idx == -1)) {
+        /*Pick a brick where the gifd heal was successful.*/
+        for (i = 0; i < priv->child_count; i++) {
+            if (!wind_on[i])
+                continue;
+            if (replies[i].valid && replies[i].op_ret == 0 &&
+                !gf_uuid_is_null(replies[i].poststat.ia_gfid)) {
+                *gfid_idx = i;
+                break;
+            }
+        }
+    }
+out:
+    if (gfid_idx && (*gfid_idx == -1) && (ret == 0) && local) {
+        ret = -afr_final_errno(local, priv);
+    }
+    loc_wipe(&loc);
+    if (frame)
+        AFR_STACK_DESTROY(frame);
+    if (xdata)
+        dict_unref(xdata);
+
+    return ret;
+}
 
-	local = frame->local;
+int
+afr_gfid_sbrain_source_from_src_brick(xlator_t *this, struct afr_reply *replies,
+                                      char *src_brick)
+{
+    int i = 0;
+    afr_private_t *priv = NULL;
+
+    priv = this->private;
+    for (i = 0; i < priv->child_count; i++) {
+        if (!replies[i].valid || replies[i].op_ret == -1)
+            continue;
+        if (strcmp(priv->children[i]->name, src_brick) == 0)
+            return i;
+    }
+    return -1;
+}
 
-	syncbarrier_wake (&local->barrier);
+int
+afr_selfheal_gfid_mismatch_by_majority(struct afr_reply *replies,
+                                       int child_count)
+{
+    int j = 0;
+    int i = 0;
+    int votes;
+
+    for (i = 0; i < child_count; i++) {
+        if (!replies[i].valid || replies[i].op_ret == -1)
+            continue;
+
+        votes = 1;
+        for (j = i + 1; j < child_count; j++) {
+            if ((!gf_uuid_compare(replies[i].poststat.ia_gfid,
+                                  replies[j].poststat.ia_gfid)))
+                votes++;
+            if (votes > child_count / 2)
+                return i;
+        }
+    }
 
-	return 0;
+    return -1;
 }
 
+int
+afr_gfid_sbrain_source_from_bigger_file(struct afr_reply *replies,
+                                        int child_count)
+{
+    int i = 0;
+    int src = -1;
+    uint64_t size = 0;
+
+    for (i = 0; i < child_count; i++) {
+        if (!replies[i].valid || replies[i].op_ret == -1)
+            continue;
+        if (size < replies[i].poststat.ia_size) {
+            src = i;
+            size = replies[i].poststat.ia_size;
+        } else if (replies[i].poststat.ia_size == size) {
+            src = -1;
+        }
+    }
+    return src;
+}
 
 int
-afr_selfheal_post_op (call_frame_t *frame, xlator_t *this, inode_t *inode,
-		      int subvol, dict_t *xattr)
+afr_gfid_sbrain_source_from_latest_mtime(struct afr_reply *replies,
+                                         int child_count)
 {
-	afr_private_t *priv = NULL;
-	afr_local_t *local = NULL;
-	loc_t loc = {0, };
+    int i = 0;
+    int src = -1;
+    uint32_t mtime = 0;
+    uint32_t mtime_nsec = 0;
+
+    for (i = 0; i < child_count; i++) {
+        if (!replies[i].valid || replies[i].op_ret != 0)
+            continue;
+        if ((mtime < replies[i].poststat.ia_mtime) ||
+            ((mtime == replies[i].poststat.ia_mtime) &&
+             (mtime_nsec < replies[i].poststat.ia_mtime_nsec))) {
+            src = i;
+            mtime = replies[i].poststat.ia_mtime;
+            mtime_nsec = replies[i].poststat.ia_mtime_nsec;
+        } else if ((mtime == replies[i].poststat.ia_mtime) &&
+                   (mtime_nsec == replies[i].poststat.ia_mtime_nsec)) {
+            src = -1;
+        }
+    }
+    return src;
+}
 
-	priv = this->private;
-	local = frame->local;
+int
+afr_gfid_split_brain_source(xlator_t *this, struct afr_reply *replies,
+                            inode_t *inode, uuid_t pargfid, const char *bname,
+                            int src_idx, int child_idx,
+                            unsigned char *locked_on, int *src, dict_t *xdata)
+{
+    afr_private_t *priv = NULL;
+    char g1[64] = {
+        0,
+    };
+    char g2[64] = {
+        0,
+    };
+    int up_count = 0;
+    int heal_op = -1;
+    int ret = -1;
+    char *src_brick = NULL;
+
+    *src = -1;
+    priv = this->private;
+    up_count = AFR_COUNT(locked_on, priv->child_count);
+    if (up_count != priv->child_count) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, AFR_MSG_SPLIT_BRAIN,
+               "All the bricks should be up to resolve the gfid split "
+               "barin");
+        if (xdata) {
+            ret = dict_set_sizen_str_sizen(xdata, "gfid-heal-msg",
+                                           SALL_BRICKS_UP_TO_RESOLVE);
+            if (ret)
+                gf_msg(this->name, GF_LOG_ERROR, 0, AFR_MSG_DICT_SET_FAILED,
+                       "Error setting"
+                       " gfid-heal-msg dict");
+        }
+        goto out;
+    }
 
-	loc.inode = inode_ref (inode);
-	gf_uuid_copy (loc.gfid, inode->gfid);
+    if (xdata) {
+        ret = dict_get_int32_sizen(xdata, "heal-op", &heal_op);
+        if (ret)
+            goto fav_child;
+    } else {
+        goto fav_child;
+    }
 
-	STACK_WIND (frame, afr_selfheal_post_op_cbk, priv->children[subvol],
-		    priv->children[subvol]->fops->xattrop, &loc,
-		    GF_XATTROP_ADD_ARRAY, xattr, NULL);
+    switch (heal_op) {
+        case GF_SHD_OP_SBRAIN_HEAL_FROM_BIGGER_FILE:
+            *src = afr_gfid_sbrain_source_from_bigger_file(replies,
+                                                           priv->child_count);
+            if (*src == -1) {
+                gf_msg(this->name, GF_LOG_ERROR, 0, AFR_MSG_SPLIT_BRAIN,
+                       SNO_BIGGER_FILE);
+                if (xdata) {
+                    ret = dict_set_sizen_str_sizen(xdata, "gfid-heal-msg",
+                                                   SNO_BIGGER_FILE);
+                    if (ret)
+                        gf_msg(this->name, GF_LOG_ERROR, 0,
+                               AFR_MSG_DICT_SET_FAILED,
+                               "Error"
+                               " setting gfid-heal-msg dict");
+                }
+            }
+            break;
+
+        case GF_SHD_OP_SBRAIN_HEAL_FROM_LATEST_MTIME:
+            *src = afr_gfid_sbrain_source_from_latest_mtime(replies,
+                                                            priv->child_count);
+            if (*src == -1) {
+                gf_msg(this->name, GF_LOG_ERROR, 0, AFR_MSG_SPLIT_BRAIN,
+                       SNO_DIFF_IN_MTIME);
+                if (xdata) {
+                    ret = dict_set_sizen_str_sizen(xdata, "gfid-heal-msg",
+                                                   SNO_DIFF_IN_MTIME);
+                    if (ret)
+                        gf_msg(this->name, GF_LOG_ERROR, 0,
+                               AFR_MSG_DICT_SET_FAILED,
+                               "Error"
+                               "setting gfid-heal-msg dict");
+                }
+            }
+            break;
 
-	syncbarrier_wait (&local->barrier, 1);
+        case GF_SHD_OP_SBRAIN_HEAL_FROM_BRICK:
+            ret = dict_get_str_sizen(xdata, "child-name", &src_brick);
+            if (ret) {
+                gf_msg(this->name, GF_LOG_ERROR, 0, AFR_MSG_SPLIT_BRAIN,
+                       "Error getting the source "
+                       "brick");
+                break;
+            }
+            *src = afr_gfid_sbrain_source_from_src_brick(this, replies,
+                                                         src_brick);
+            if (*src == -1) {
+                gf_msg(this->name, GF_LOG_ERROR, 0, AFR_MSG_SPLIT_BRAIN,
+                       SERROR_GETTING_SRC_BRICK);
+                if (xdata) {
+                    ret = dict_set_sizen_str_sizen(xdata, "gfid-heal-msg",
+                                                   SERROR_GETTING_SRC_BRICK);
+                    if (ret)
+                        gf_msg(this->name, GF_LOG_ERROR, 0,
+                               AFR_MSG_DICT_SET_FAILED,
+                               "Error"
+                               " setting gfid-heal-msg dict");
+                }
+            }
+            break;
 
-        loc_wipe (&loc);
+        default:
+            break;
+    }
+    goto out;
+
+fav_child:
+    switch (priv->fav_child_policy) {
+        case AFR_FAV_CHILD_BY_SIZE:
+            *src = afr_sh_fav_by_size(this, replies, inode);
+            break;
+        case AFR_FAV_CHILD_BY_MTIME:
+            *src = afr_sh_fav_by_mtime(this, replies, inode);
+            break;
+        case AFR_FAV_CHILD_BY_CTIME:
+            *src = afr_sh_fav_by_ctime(this, replies, inode);
+            break;
+        case AFR_FAV_CHILD_BY_MAJORITY:
+            if (priv->child_count != 2)
+                *src = afr_selfheal_gfid_mismatch_by_majority(
+                    replies, priv->child_count);
+            else
+                *src = -1;
+
+            if (*src == -1) {
+                gf_msg(this->name, GF_LOG_ERROR, 0, AFR_MSG_SPLIT_BRAIN,
+                       "No majority to resolve "
+                       "gfid split brain");
+            }
+            break;
+        default:
+            break;
+    }
 
-	return 0;
+out:
+    if (*src == -1) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, AFR_MSG_SPLIT_BRAIN,
+               "Gfid mismatch detected for <gfid:%s>/%s>, %s on %s and"
+               " %s on %s.",
+               uuid_utoa(pargfid), bname,
+               uuid_utoa_r(replies[child_idx].poststat.ia_gfid, g1),
+               priv->children[child_idx]->name,
+               uuid_utoa_r(replies[src_idx].poststat.ia_gfid, g2),
+               priv->children[src_idx]->name);
+        gf_event(EVENT_AFR_SPLIT_BRAIN,
+                 "client-pid=%d;"
+                 "subvol=%s;type=gfid;file="
+                 "<gfid:%s>/%s>;count=2;child-%d=%s;gfid-%d=%s;"
+                 "child-%d=%s;gfid-%d=%s",
+                 this->ctx->cmd_args.client_pid, this->name, uuid_utoa(pargfid),
+                 bname, child_idx, priv->children[child_idx]->name, child_idx,
+                 uuid_utoa_r(replies[child_idx].poststat.ia_gfid, g1), src_idx,
+                 priv->children[src_idx]->name, src_idx,
+                 uuid_utoa_r(replies[src_idx].poststat.ia_gfid, g2));
+        return -1;
+    }
+    return 0;
 }
 
+int
+afr_selfheal_post_op_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                         int op_ret, int op_errno, dict_t *xattr, dict_t *xdata)
+{
+    afr_local_t *local = NULL;
 
-dict_t *
-afr_selfheal_output_xattr (xlator_t *this, afr_transaction_type type,
-			   int *output_dirty, int **output_matrix, int subvol)
-{
-	dict_t *xattr = NULL;
-	afr_private_t *priv = NULL;
-	int j = 0;
-	int idx = 0;
-	int ret = 0;
-	int *raw = 0;
-
-	priv = this->private;
-	idx = afr_index_for_transaction_type (type);
-
-	xattr = dict_new ();
-	if (!xattr)
-		return NULL;
-
-	/* clear dirty */
-	raw = GF_CALLOC (sizeof(int), AFR_NUM_CHANGE_LOGS, gf_afr_mt_int32_t);
-	if (!raw)
-		goto err;
-
-	raw[idx] = hton32 (output_dirty[subvol]);
-	ret = dict_set_bin (xattr, AFR_DIRTY, raw,
-			    sizeof(int) * AFR_NUM_CHANGE_LOGS);
-	if (ret)
-		goto err;
-
-	/* clear/set pending */
-	for (j = 0; j < priv->child_count; j++) {
-		raw = GF_CALLOC (sizeof(int), AFR_NUM_CHANGE_LOGS,
-				 gf_afr_mt_int32_t);
-		if (!raw)
-			goto err;
-
-		raw[idx] = hton32 (output_matrix[subvol][j]);
-
-		ret = dict_set_bin (xattr, priv->pending_key[j],
-				    raw, sizeof(int) * AFR_NUM_CHANGE_LOGS);
-		if (ret)
-			goto err;
-	}
-
-	return xattr;
-err:
-	if (xattr)
-		dict_unref (xattr);
-	return NULL;
+    local = frame->local;
+
+    local->op_ret = op_ret;
+    local->op_errno = op_errno;
+    syncbarrier_wake(&local->barrier);
+
+    return 0;
 }
 
+int
+afr_selfheal_post_op(call_frame_t *frame, xlator_t *this, inode_t *inode,
+                     int subvol, dict_t *xattr, dict_t *xdata)
+{
+    afr_private_t *priv = NULL;
+    afr_local_t *local = NULL;
+    loc_t loc = {
+        0,
+    };
+    int ret = 0;
+
+    priv = this->private;
+    local = frame->local;
+
+    loc.inode = inode_ref(inode);
+    gf_uuid_copy(loc.gfid, inode->gfid);
+
+    local->op_ret = 0;
+
+    STACK_WIND(frame, afr_selfheal_post_op_cbk, priv->children[subvol],
+               priv->children[subvol]->fops->xattrop, &loc,
+               GF_XATTROP_ADD_ARRAY, xattr, xdata);
+
+    syncbarrier_wait(&local->barrier, 1);
+    if (local->op_ret < 0)
+        ret = -local->op_errno;
+
+    loc_wipe(&loc);
+    local->op_ret = 0;
+
+    return ret;
+}
 
 int
-afr_selfheal_undo_pending (call_frame_t *frame, xlator_t *this, inode_t *inode,
-			   unsigned char *sources, unsigned char *sinks,
-			   unsigned char *healed_sinks, afr_transaction_type type,
-			   struct afr_reply *replies, unsigned char *locked_on)
-{
-	afr_private_t *priv = NULL;
-	int i = 0;
-	int j = 0;
-	unsigned char *pending = NULL;
-	int *input_dirty = NULL;
-	int **input_matrix = NULL;
-	int *output_dirty = NULL;
-	int **output_matrix = NULL;
-	dict_t *xattr = NULL;
-
-	priv = this->private;
-
-	pending = alloca0 (priv->child_count);
-
-	input_dirty = alloca0 (priv->child_count * sizeof (int));
-	input_matrix = ALLOC_MATRIX (priv->child_count, int);
-	output_dirty = alloca0 (priv->child_count * sizeof (int));
-	output_matrix = ALLOC_MATRIX (priv->child_count, int);
-
-	afr_selfheal_extract_xattr (this, replies, type, input_dirty,
-				    input_matrix);
-
-	for (i = 0; i < priv->child_count; i++)
-		if (sinks[i] && !healed_sinks[i])
-			pending[i] = 1;
-
-	for (i = 0; i < priv->child_count; i++) {
-		for (j = 0; j < priv->child_count; j++) {
-			if (pending[j])
-				output_matrix[i][j] = 1;
-			else
-				output_matrix[i][j] = -input_matrix[i][j];
-		}
-	}
-
-	for (i = 0; i < priv->child_count; i++) {
-		if (!pending[i])
-			output_dirty[i] = -input_dirty[i];
-	}
-
-	for (i = 0; i < priv->child_count; i++) {
-		if (!locked_on[i])
-			/* perform post-op only on subvols we had locked
-			   and inspected on.
-			*/
-			continue;
-
-		xattr = afr_selfheal_output_xattr (this, type, output_dirty,
-						   output_matrix, i);
-		if (!xattr) {
-			gf_log (this->name, GF_LOG_ERROR,
-				"unable to allocate xdata for subvol %d", i);
-			continue;
-		}
-
-		afr_selfheal_post_op (frame, this, inode, i, xattr);
-
-		dict_unref (xattr);
-	}
-
-	return 0;
+afr_check_stale_error(struct afr_reply *replies, afr_private_t *priv)
+{
+    int i = 0;
+    int op_errno = 0;
+    int tmp_errno = 0;
+    int stale_count = 0;
+
+    for (i = 0; i < priv->child_count; i++) {
+        tmp_errno = replies[i].op_errno;
+        if (tmp_errno == ENOENT || tmp_errno == ESTALE) {
+            op_errno = afr_higher_errno(op_errno, tmp_errno);
+            stale_count++;
+        }
+    }
+    if (stale_count != priv->child_count)
+        return -ENOTCONN;
+    else
+        return -op_errno;
 }
 
+int
+afr_sh_generic_fop_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                       int op_ret, int op_errno, struct iatt *pre,
+                       struct iatt *post, dict_t *xdata)
+{
+    int i = (long)cookie;
+    afr_local_t *local = NULL;
 
-void
-afr_replies_copy (struct afr_reply *dst, struct afr_reply *src, int count)
-{
-	int i = 0;
-	dict_t *xdata = NULL;
-
-	if (dst == src)
-		return;
-
-	for (i = 0; i < count; i++) {
-		dst[i].valid = src[i].valid;
-		dst[i].op_ret = src[i].op_ret;
-		dst[i].op_errno = src[i].op_errno;
-		dst[i].prestat = src[i].prestat;
-		dst[i].poststat = src[i].poststat;
-		dst[i].preparent = src[i].preparent;
-		dst[i].postparent = src[i].postparent;
-		dst[i].preparent2 = src[i].preparent2;
-		dst[i].postparent2 = src[i].postparent2;
-		if (src[i].xdata)
-			xdata = dict_ref (src[i].xdata);
-		else
-			xdata = NULL;
-		if (dst[i].xdata)
-			dict_unref (dst[i].xdata);
-		dst[i].xdata = xdata;
-		memcpy (dst[i].checksum, src[i].checksum,
-			MD5_DIGEST_LENGTH);
-	}
+    local = frame->local;
+
+    local->replies[i].valid = 1;
+    local->replies[i].op_ret = op_ret;
+    local->replies[i].op_errno = op_errno;
+    if (pre)
+        local->replies[i].prestat = *pre;
+    if (post)
+        local->replies[i].poststat = *post;
+    if (xdata)
+        local->replies[i].xdata = dict_ref(xdata);
+
+    syncbarrier_wake(&local->barrier);
+
+    return 0;
 }
 
+int
+afr_selfheal_restore_time(call_frame_t *frame, xlator_t *this, inode_t *inode,
+                          int source, unsigned char *healed_sinks,
+                          struct afr_reply *replies)
+{
+    loc_t loc = {
+        0,
+    };
+
+    loc.inode = inode_ref(inode);
+    gf_uuid_copy(loc.gfid, inode->gfid);
+
+    AFR_ONLIST(healed_sinks, frame, afr_sh_generic_fop_cbk, setattr, &loc,
+               &replies[source].poststat,
+               (GF_SET_ATTR_ATIME | GF_SET_ATTR_MTIME | GF_SET_ATTR_CTIME),
+               NULL);
+
+    loc_wipe(&loc);
+
+    return 0;
+}
+
+dict_t *
+afr_selfheal_output_xattr(xlator_t *this, gf_boolean_t is_full_crawl,
+                          afr_transaction_type type, int *output_dirty,
+                          int **output_matrix, int subvol,
+                          int **full_heal_mtx_out)
+{
+    int j = 0;
+    int idx = 0;
+    int d_idx = 0;
+    int ret = 0;
+    int *raw = 0;
+    dict_t *xattr = NULL;
+    afr_private_t *priv = NULL;
+
+    priv = this->private;
+    idx = afr_index_for_transaction_type(type);
+    d_idx = afr_index_for_transaction_type(AFR_DATA_TRANSACTION);
+
+    xattr = dict_new();
+    if (!xattr)
+        return NULL;
+
+    /* clear dirty */
+    raw = GF_CALLOC(sizeof(int), AFR_NUM_CHANGE_LOGS, gf_afr_mt_int32_t);
+    if (!raw)
+        goto err;
+
+    raw[idx] = hton32(output_dirty[subvol]);
+    ret = dict_set_bin(xattr, AFR_DIRTY, raw,
+                       sizeof(int) * AFR_NUM_CHANGE_LOGS);
+    if (ret) {
+        GF_FREE(raw);
+        goto err;
+    }
+
+    /* clear/set pending */
+    for (j = 0; j < priv->child_count; j++) {
+        raw = GF_CALLOC(sizeof(int), AFR_NUM_CHANGE_LOGS, gf_afr_mt_int32_t);
+        if (!raw)
+            goto err;
+
+        raw[idx] = hton32(output_matrix[subvol][j]);
+        if (is_full_crawl)
+            raw[d_idx] = hton32(full_heal_mtx_out[subvol][j]);
+
+        ret = dict_set_bin(xattr, priv->pending_key[j], raw,
+                           sizeof(int) * AFR_NUM_CHANGE_LOGS);
+        if (ret) {
+            GF_FREE(raw);
+            goto err;
+        }
+    }
+
+    return xattr;
+err:
+    if (xattr)
+        dict_unref(xattr);
+    return NULL;
+}
 
 int
-afr_selfheal_fill_dirty (xlator_t *this, int *dirty, int subvol,
-			 int idx, dict_t *xdata)
+afr_selfheal_undo_pending(call_frame_t *frame, xlator_t *this, inode_t *inode,
+                          unsigned char *sources, unsigned char *sinks,
+                          unsigned char *healed_sinks,
+                          unsigned char *undid_pending,
+                          afr_transaction_type type, struct afr_reply *replies,
+                          unsigned char *locked_on)
 {
-	void *pending_raw = NULL;
-	int pending[3] = {0, };
+    afr_private_t *priv = NULL;
+    afr_local_t *local = NULL;
+    int i = 0;
+    int j = 0;
+    unsigned char *pending = NULL;
+    int *input_dirty = NULL;
+    int **input_matrix = NULL;
+    int **full_heal_mtx_in = NULL;
+    int **full_heal_mtx_out = NULL;
+    int *output_dirty = NULL;
+    int **output_matrix = NULL;
+    dict_t *xattr = NULL;
+    dict_t *xdata = NULL;
+
+    priv = this->private;
+    local = frame->local;
+
+    pending = alloca0(priv->child_count);
+
+    input_dirty = alloca0(priv->child_count * sizeof(int));
+    input_matrix = ALLOC_MATRIX(priv->child_count, int);
+    full_heal_mtx_in = ALLOC_MATRIX(priv->child_count, int);
+    full_heal_mtx_out = ALLOC_MATRIX(priv->child_count, int);
+    output_dirty = alloca0(priv->child_count * sizeof(int));
+    output_matrix = ALLOC_MATRIX(priv->child_count, int);
+
+    xdata = dict_new();
+    if (!xdata)
+        return -1;
+
+    afr_selfheal_extract_xattr(this, replies, type, input_dirty, input_matrix);
+
+    if (local->need_full_crawl)
+        afr_selfheal_extract_xattr(this, replies, AFR_DATA_TRANSACTION, NULL,
+                                   full_heal_mtx_in);
+
+    for (i = 0; i < priv->child_count; i++)
+        if (sinks[i] && !healed_sinks[i])
+            pending[i] = 1;
+
+    for (i = 0; i < priv->child_count; i++) {
+        for (j = 0; j < priv->child_count; j++) {
+            if (pending[j]) {
+                output_matrix[i][j] = 1;
+                if (type == AFR_ENTRY_TRANSACTION)
+                    full_heal_mtx_out[i][j] = 1;
+            } else if (locked_on[j]) {
+                output_matrix[i][j] = -input_matrix[i][j];
+                if (type == AFR_ENTRY_TRANSACTION)
+                    full_heal_mtx_out[i][j] = -full_heal_mtx_in[i][j];
+            }
+        }
+    }
+
+    for (i = 0; i < priv->child_count; i++) {
+        if (!pending[i])
+            output_dirty[i] = -input_dirty[i];
+    }
+
+    for (i = 0; i < priv->child_count; i++) {
+        if (!locked_on[i])
+            /* perform post-op only on subvols we had locked
+               and inspected on.
+            */
+            continue;
+        if (undid_pending[i])
+            /* We already unset the pending xattrs in
+             * _afr_fav_child_reset_sink_xattrs(). */
+            continue;
+
+        xattr = afr_selfheal_output_xattr(this, local->need_full_crawl, type,
+                                          output_dirty, output_matrix, i,
+                                          full_heal_mtx_out);
+        if (!xattr) {
+            continue;
+        }
 
-	if (dict_get_ptr (xdata, AFR_DIRTY, &pending_raw))
-		return -1;
+        if ((type == AFR_ENTRY_TRANSACTION) && (priv->esh_granular)) {
+            if (xdata && dict_set_int8(xdata, GF_XATTROP_PURGE_INDEX, 1))
+                gf_msg(this->name, GF_LOG_WARNING, 0, AFR_MSG_DICT_SET_FAILED,
+                       "Failed to set"
+                       " dict value for %s",
+                       GF_XATTROP_PURGE_INDEX);
+        }
 
-	if (!pending_raw)
-		return -1;
+        afr_selfheal_post_op(frame, this, inode, i, xattr, xdata);
+        dict_unref(xattr);
+    }
 
-	memcpy (pending, pending_raw, sizeof(pending));
+    if (xdata)
+        dict_unref(xdata);
 
-	dirty[subvol] = ntoh32 (pending[idx]);
+    return 0;
+}
 
-	return 0;
+void
+afr_reply_copy(struct afr_reply *dst, struct afr_reply *src)
+{
+    dict_t *xdata = NULL;
+
+    dst->valid = src->valid;
+    dst->op_ret = src->op_ret;
+    dst->op_errno = src->op_errno;
+    dst->prestat = src->prestat;
+    dst->poststat = src->poststat;
+    dst->preparent = src->preparent;
+    dst->postparent = src->postparent;
+    dst->preparent2 = src->preparent2;
+    dst->postparent2 = src->postparent2;
+    if (src->xdata)
+        xdata = dict_ref(src->xdata);
+    else
+        xdata = NULL;
+    if (dst->xdata)
+        dict_unref(dst->xdata);
+    dst->xdata = xdata;
+    if (xdata && dict_get_str_boolean(xdata, "fips-mode-rchecksum",
+                                      _gf_false) == _gf_true) {
+        memcpy(dst->checksum, src->checksum, SHA256_DIGEST_LENGTH);
+    } else {
+        memcpy(dst->checksum, src->checksum, MD5_DIGEST_LENGTH);
+    }
+    dst->fips_mode_rchecksum = src->fips_mode_rchecksum;
 }
 
+void
+afr_replies_copy(struct afr_reply *dst, struct afr_reply *src, int count)
+{
+    int i = 0;
+
+    if (dst == src)
+        return;
+
+    for (i = 0; i < count; i++) {
+        afr_reply_copy(&dst[i], &src[i]);
+    }
+}
 
 int
-afr_selfheal_fill_matrix (xlator_t *this, int **matrix, int subvol,
-			  int idx, dict_t *xdata)
+afr_selfheal_fill_dirty(xlator_t *this, int *dirty, int subvol, int idx,
+                        dict_t *xdata)
 {
-	int i = 0;
-	void *pending_raw = NULL;
-	int pending[3] = {0, };
-	afr_private_t *priv = NULL;
+    void *pending_raw = NULL;
+    int pending[3] = {
+        0,
+    };
 
-	priv = this->private;
+    if (!dirty)
+        return 0;
 
-	for (i = 0; i < priv->child_count; i++) {
-		if (dict_get_ptr (xdata, priv->pending_key[i], &pending_raw))
-			continue;
+    if (dict_get_ptr(xdata, AFR_DIRTY, &pending_raw))
+        return -1;
 
-		if (!pending_raw)
-			continue;
+    if (!pending_raw)
+        return -1;
 
-		memcpy (pending, pending_raw, sizeof(pending));
+    memcpy(pending, pending_raw, sizeof(pending));
 
-		matrix[subvol][i] = ntoh32 (pending[idx]);
-	}
+    dirty[subvol] = ntoh32(pending[idx]);
 
-	return 0;
+    return 0;
 }
 
+int
+afr_selfheal_fill_matrix(xlator_t *this, int **matrix, int subvol, int idx,
+                         dict_t *xdata)
+{
+    int i = 0;
+    void *pending_raw = NULL;
+    int pending[3] = {
+        0,
+    };
+    afr_private_t *priv = NULL;
+
+    priv = this->private;
+
+    if (!matrix)
+        return 0;
+
+    for (i = 0; i < priv->child_count; i++) {
+        if (dict_get_ptr(xdata, priv->pending_key[i], &pending_raw))
+            continue;
+
+        if (!pending_raw)
+            continue;
+
+        memcpy(pending, pending_raw, sizeof(pending));
+
+        matrix[subvol][i] = ntoh32(pending[idx]);
+    }
+
+    return 0;
+}
 
 int
-afr_selfheal_extract_xattr (xlator_t *this, struct afr_reply *replies,
-			    afr_transaction_type type, int *dirty, int **matrix)
+afr_selfheal_extract_xattr(xlator_t *this, struct afr_reply *replies,
+                           afr_transaction_type type, int *dirty, int **matrix)
 {
-	afr_private_t *priv = NULL;
-	int i = 0;
-	dict_t *xdata = NULL;
-	int idx = -1;
+    afr_private_t *priv = NULL;
+    int i = 0;
+    dict_t *xdata = NULL;
+    int idx = -1;
+
+    idx = afr_index_for_transaction_type(type);
 
-	idx = afr_index_for_transaction_type (type);
+    priv = this->private;
 
-	priv = this->private;
+    for (i = 0; i < priv->child_count; i++) {
+        if (!replies[i].valid || replies[i].op_ret != 0)
+            continue;
 
-	for (i = 0; i < priv->child_count; i++) {
-		if (!replies[i].xdata)
-			continue;
+        if (!replies[i].xdata)
+            continue;
 
-		xdata = replies[i].xdata;
+        xdata = replies[i].xdata;
 
-		afr_selfheal_fill_dirty (this, dirty, i, idx, xdata);
-		afr_selfheal_fill_matrix (this, matrix, i, idx, xdata);
-	}
+        afr_selfheal_fill_dirty(this, dirty, i, idx, xdata);
+        afr_selfheal_fill_matrix(this, matrix, i, idx, xdata);
+    }
 
-	return 0;
+    return 0;
 }
 
 /*
@@ -294,70 +819,566 @@ afr_selfheal_extract_xattr (xlator_t *this, struct afr_reply *replies,
  * This can happen if data was directly modified in the backend or for snapshots
  */
 void
-afr_mark_largest_file_as_source (xlator_t *this, unsigned char *sources,
-                                 struct afr_reply *replies)
+afr_mark_largest_file_as_source(xlator_t *this, unsigned char *sources,
+                                struct afr_reply *replies)
 {
-        int i = 0;
-        afr_private_t *priv = NULL;
-        uint64_t size = 0;
-
-        /* Find source with biggest file size */
-        priv = this->private;
-        for (i = 0; i < priv->child_count; i++) {
-                if (!sources[i])
-                        continue;
-                if (size <= replies[i].poststat.ia_size) {
-                        size = replies[i].poststat.ia_size;
-                }
+    int i = 0;
+    afr_private_t *priv = NULL;
+    uint64_t size = 0;
+
+    /* Find source with biggest file size */
+    priv = this->private;
+    for (i = 0; i < priv->child_count; i++) {
+        if (!sources[i])
+            continue;
+        if (!replies[i].valid || replies[i].op_ret != 0) {
+            sources[i] = 0;
+            continue;
+        }
+        if (size <= replies[i].poststat.ia_size) {
+            size = replies[i].poststat.ia_size;
         }
+    }
+
+    /* Mark sources with less size as not source */
+    for (i = 0; i < priv->child_count; i++) {
+        if (!sources[i])
+            continue;
+        if (size > replies[i].poststat.ia_size)
+            sources[i] = 0;
+    }
+}
 
-        /* Mark sources with less size as not source */
-        for (i = 0; i < priv->child_count; i++) {
-                if (!sources[i])
-                        continue;
-                if (size > replies[i].poststat.ia_size)
-                        sources[i] = 0;
+void
+afr_mark_latest_mtime_file_as_source(xlator_t *this, unsigned char *sources,
+                                     struct afr_reply *replies)
+{
+    int i = 0;
+    afr_private_t *priv = NULL;
+    uint32_t mtime = 0;
+    uint32_t mtime_nsec = 0;
+
+    priv = this->private;
+    for (i = 0; i < priv->child_count; i++) {
+        if (!sources[i])
+            continue;
+        if (!replies[i].valid || replies[i].op_ret != 0) {
+            sources[i] = 0;
+            continue;
+        }
+        if ((mtime < replies[i].poststat.ia_mtime) ||
+            ((mtime == replies[i].poststat.ia_mtime) &&
+             (mtime_nsec < replies[i].poststat.ia_mtime_nsec))) {
+            mtime = replies[i].poststat.ia_mtime;
+            mtime_nsec = replies[i].poststat.ia_mtime_nsec;
+        }
+    }
+    for (i = 0; i < priv->child_count; i++) {
+        if (!sources[i])
+            continue;
+        if ((mtime > replies[i].poststat.ia_mtime) ||
+            ((mtime == replies[i].poststat.ia_mtime) &&
+             (mtime_nsec > replies[i].poststat.ia_mtime_nsec))) {
+            sources[i] = 0;
         }
+    }
 }
 
 void
-afr_mark_active_sinks (xlator_t *this, unsigned char *sources,
-                       unsigned char *locked_on, unsigned char *sinks)
+afr_mark_active_sinks(xlator_t *this, unsigned char *sources,
+                      unsigned char *locked_on, unsigned char *sinks)
 {
-        int i = 0;
-        afr_private_t *priv = NULL;
+    int i = 0;
+    afr_private_t *priv = NULL;
 
-        priv = this->private;
+    priv = this->private;
 
-        memset (sinks, 0, sizeof (*sinks) * priv->child_count);
-        for (i = 0; i < priv->child_count; i++) {
-                if (!sources[i] && locked_on[i])
-                        sinks[i] = 1;
+    for (i = 0; i < priv->child_count; i++) {
+        if (!sources[i] && locked_on[i])
+            sinks[i] = 1;
+        else
+            sinks[i] = 0;
+    }
+}
+
+gf_boolean_t
+afr_dict_contains_heal_op(call_frame_t *frame)
+{
+    afr_local_t *local = NULL;
+    dict_t *xdata_req = NULL;
+    int ret = 0;
+    int heal_op = -1;
+
+    local = frame->local;
+    xdata_req = local->xdata_req;
+    ret = dict_get_int32_sizen(xdata_req, "heal-op", &heal_op);
+    if (ret)
+        return _gf_false;
+    if (local->xdata_rsp == NULL) {
+        local->xdata_rsp = dict_new();
+        if (!local->xdata_rsp)
+            return _gf_true;
+    }
+    ret = dict_set_sizen_str_sizen(local->xdata_rsp, "sh-fail-msg",
+                                   SFILE_NOT_IN_SPLIT_BRAIN);
+
+    return _gf_true;
+}
+
+gf_boolean_t
+afr_can_decide_split_brain_source_sinks(struct afr_reply *replies,
+                                        int child_count)
+{
+    int i = 0;
+
+    for (i = 0; i < child_count; i++)
+        if (replies[i].valid != 1 || replies[i].op_ret != 0)
+            return _gf_false;
+
+    return _gf_true;
+}
+
+int
+afr_mark_split_brain_source_sinks_by_heal_op(
+    call_frame_t *frame, xlator_t *this, unsigned char *sources,
+    unsigned char *sinks, unsigned char *healed_sinks, unsigned char *locked_on,
+    struct afr_reply *replies, afr_transaction_type type, int heal_op)
+{
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
+    dict_t *xdata_req = NULL;
+    dict_t *xdata_rsp = NULL;
+    int ret = 0;
+    int i = 0;
+    char *name = NULL;
+    int source = -1;
+
+    local = frame->local;
+    priv = this->private;
+    xdata_req = local->xdata_req;
+
+    for (i = 0; i < priv->child_count; i++) {
+        if (locked_on[i])
+            if (sources[i] || !sinks[i] || !healed_sinks[i]) {
+                ret = -1;
+                goto out;
+            }
+    }
+    if (local->xdata_rsp == NULL) {
+        local->xdata_rsp = dict_new();
+        if (!local->xdata_rsp) {
+            ret = -1;
+            goto out;
+        }
+    }
+    xdata_rsp = local->xdata_rsp;
+
+    if (!afr_can_decide_split_brain_source_sinks(replies, priv->child_count)) {
+        ret = dict_set_sizen_str_sizen(xdata_rsp, "sh-fail-msg",
+                                       SBRAIN_HEAL_NO_GO_MSG);
+        ret = -1;
+        goto out;
+    }
+
+    for (i = 0; i < priv->child_count; i++)
+        if (locked_on[i])
+            sources[i] = 1;
+    switch (heal_op) {
+        case GF_SHD_OP_SBRAIN_HEAL_FROM_BIGGER_FILE:
+            if (type == AFR_METADATA_TRANSACTION) {
+                ret = dict_set_sizen_str_sizen(xdata_rsp, "sh-fail-msg",
+                                               SUSE_SOURCE_BRICK_TO_HEAL);
+                if (!ret)
+                    ret = -1;
+                goto out;
+            }
+            afr_mark_largest_file_as_source(this, sources, replies);
+            if (AFR_COUNT(sources, priv->child_count) != 1) {
+                ret = dict_set_sizen_str_sizen(xdata_rsp, "sh-fail-msg",
+                                               SNO_BIGGER_FILE);
+                if (!ret)
+                    ret = -1;
+                goto out;
+            }
+            break;
+        case GF_SHD_OP_SBRAIN_HEAL_FROM_LATEST_MTIME:
+            if (type == AFR_METADATA_TRANSACTION) {
+                ret = dict_set_sizen_str_sizen(xdata_rsp, "sh-fail-msg",
+                                               SUSE_SOURCE_BRICK_TO_HEAL);
+                if (!ret)
+                    ret = -1;
+                goto out;
+            }
+            afr_mark_latest_mtime_file_as_source(this, sources, replies);
+            if (AFR_COUNT(sources, priv->child_count) != 1) {
+                ret = dict_set_sizen_str_sizen(xdata_rsp, "sh-fail-msg",
+                                               SNO_DIFF_IN_MTIME);
+                if (!ret)
+                    ret = -1;
+                goto out;
+            }
+            break;
+        case GF_SHD_OP_SBRAIN_HEAL_FROM_BRICK:
+            ret = dict_get_str_sizen(xdata_req, "child-name", &name);
+            if (ret)
+                goto out;
+            source = afr_get_child_index_from_name(this, name);
+            if (source < 0) {
+                ret = dict_set_sizen_str_sizen(xdata_rsp, "sh-fail-msg",
+                                               SINVALID_BRICK_NAME);
+                if (!ret)
+                    ret = -1;
+                goto out;
+            }
+            if (locked_on[source] != 1) {
+                ret = dict_set_sizen_str_sizen(xdata_rsp, "sh-fail-msg",
+                                               SBRICK_IS_NOT_UP);
+                if (!ret)
+                    ret = -1;
+                goto out;
+            }
+            memset(sources, 0, sizeof(*sources) * priv->child_count);
+            sources[source] = 1;
+            break;
+        default:
+            ret = -1;
+            goto out;
+    }
+    for (i = 0; i < priv->child_count; i++) {
+        if (sources[i]) {
+            source = i;
+            break;
+        }
+    }
+    sinks[source] = 0;
+    healed_sinks[source] = 0;
+    ret = source;
+out:
+    if (ret < 0)
+        memset(sources, 0, sizeof(*sources) * priv->child_count);
+    return ret;
+}
+
+int
+afr_sh_fav_by_majority(xlator_t *this, struct afr_reply *replies,
+                       inode_t *inode)
+{
+    afr_private_t *priv;
+    int vote_count = -1;
+    int fav_child = -1;
+    int i = 0;
+    int k = 0;
+
+    priv = this->private;
+
+    for (i = 0; i < priv->child_count; i++) {
+        if (replies[i].valid == 1) {
+            gf_msg_debug(this->name, 0,
+                         "Child:%s mtime_sec = %" PRId64 ", size = %" PRIu64
+                         " for gfid %s",
+                         priv->children[i]->name, replies[i].poststat.ia_mtime,
+                         replies[i].poststat.ia_size, uuid_utoa(inode->gfid));
+            vote_count = 0;
+            for (k = 0; k < priv->child_count; k++) {
+                if ((replies[k].poststat.ia_mtime ==
+                     replies[i].poststat.ia_mtime) &&
+                    (replies[k].poststat.ia_size ==
+                     replies[i].poststat.ia_size)) {
+                    vote_count++;
+                }
+            }
+            if (vote_count > priv->child_count / 2) {
+                fav_child = i;
+                break;
+            }
+        }
+    }
+    return fav_child;
+}
+
+/*
+ * afr_sh_fav_by_mtime: Choose favorite child by mtime.
+ */
+int
+afr_sh_fav_by_mtime(xlator_t *this, struct afr_reply *replies, inode_t *inode)
+{
+    afr_private_t *priv;
+    int fav_child = -1;
+    int i = 0;
+    uint32_t cmp_mtime = 0;
+    uint32_t cmp_mtime_nsec = 0;
+
+    priv = this->private;
+
+    for (i = 0; i < priv->child_count; i++) {
+        if (replies[i].valid == 1) {
+            gf_msg_debug(this->name, 0,
+                         "Child:%s mtime = %" PRId64
+                         ", mtime_nsec = %d for "
+                         "gfid %s",
+                         priv->children[i]->name, replies[i].poststat.ia_mtime,
+                         replies[i].poststat.ia_mtime_nsec,
+                         uuid_utoa(inode->gfid));
+            if (replies[i].poststat.ia_mtime > cmp_mtime) {
+                cmp_mtime = replies[i].poststat.ia_mtime;
+                cmp_mtime_nsec = replies[i].poststat.ia_mtime_nsec;
+                fav_child = i;
+            } else if ((replies[i].poststat.ia_mtime == cmp_mtime) &&
+                       (replies[i].poststat.ia_mtime_nsec > cmp_mtime_nsec)) {
+                cmp_mtime = replies[i].poststat.ia_mtime;
+                cmp_mtime_nsec = replies[i].poststat.ia_mtime_nsec;
+                fav_child = i;
+            }
+        }
+    }
+    return fav_child;
+}
+
+/*
+ * afr_sh_fav_by_ctime: Choose favorite child by ctime.
+ */
+int
+afr_sh_fav_by_ctime(xlator_t *this, struct afr_reply *replies, inode_t *inode)
+{
+    afr_private_t *priv;
+    int fav_child = -1;
+    int i = 0;
+    uint32_t cmp_ctime = 0;
+    uint32_t cmp_ctime_nsec = 0;
+
+    priv = this->private;
+
+    for (i = 0; i < priv->child_count; i++) {
+        if (replies[i].valid == 1) {
+            gf_msg_debug(this->name, 0,
+                         "Child:%s ctime = %" PRId64
+                         ", ctime_nsec = %d for "
+                         "gfid %s",
+                         priv->children[i]->name, replies[i].poststat.ia_ctime,
+                         replies[i].poststat.ia_ctime_nsec,
+                         uuid_utoa(inode->gfid));
+            if (replies[i].poststat.ia_ctime > cmp_ctime) {
+                cmp_ctime = replies[i].poststat.ia_ctime;
+                cmp_ctime_nsec = replies[i].poststat.ia_ctime_nsec;
+                fav_child = i;
+            } else if ((replies[i].poststat.ia_ctime == cmp_ctime) &&
+                       (replies[i].poststat.ia_ctime_nsec > cmp_ctime_nsec)) {
+                cmp_ctime = replies[i].poststat.ia_ctime;
+                cmp_ctime_nsec = replies[i].poststat.ia_ctime_nsec;
+                fav_child = i;
+            }
         }
+    }
+    return fav_child;
+}
+
+/*
+ * afr_sh_fav_by_size: Choose favorite child by size
+ * when not all files are of zero size.
+ */
+int
+afr_sh_fav_by_size(xlator_t *this, struct afr_reply *replies, inode_t *inode)
+{
+    afr_private_t *priv;
+    int fav_child = -1;
+    int i = 0;
+    uint64_t cmp_sz = 0;
+
+    priv = this->private;
+    for (i = 0; i < priv->child_count; i++) {
+        if (!replies[i].valid) {
+            continue;
+        }
+        gf_msg_debug(this->name, 0,
+                     "Child:%s file size = %" PRIu64 " for gfid %s",
+                     priv->children[i]->name, replies[i].poststat.ia_size,
+                     uuid_utoa(inode->gfid));
+        if (replies[i].poststat.ia_type == IA_IFDIR) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, AFR_MSG_SBRAIN_FAV_CHILD_POLICY,
+                   "Cannot perform selfheal on %s. "
+                   "Size policy is not applicable to directories.",
+                   uuid_utoa(inode->gfid));
+            break;
+        }
+        if (replies[i].poststat.ia_size > cmp_sz) {
+            cmp_sz = replies[i].poststat.ia_size;
+            fav_child = i;
+        } else if (replies[i].poststat.ia_size == cmp_sz) {
+            fav_child = -1;
+        }
+    }
+    if (fav_child == -1) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, AFR_MSG_SPLIT_BRAIN,
+               "No bigger file");
+    }
+    return fav_child;
+}
+
+int
+afr_sh_get_fav_by_policy(xlator_t *this, struct afr_reply *replies,
+                         inode_t *inode, char **policy_str)
+{
+    afr_private_t *priv = NULL;
+    int fav_child = -1;
+
+    priv = this->private;
+    if (!afr_can_decide_split_brain_source_sinks(replies, priv->child_count)) {
+        return -1;
+    }
+
+    switch (priv->fav_child_policy) {
+        case AFR_FAV_CHILD_BY_SIZE:
+            fav_child = afr_sh_fav_by_size(this, replies, inode);
+            if (policy_str && fav_child >= 0) {
+                *policy_str = "SIZE";
+            }
+            break;
+        case AFR_FAV_CHILD_BY_CTIME:
+            fav_child = afr_sh_fav_by_ctime(this, replies, inode);
+            if (policy_str && fav_child >= 0) {
+                *policy_str = "CTIME";
+            }
+            break;
+        case AFR_FAV_CHILD_BY_MTIME:
+            fav_child = afr_sh_fav_by_mtime(this, replies, inode);
+            if (policy_str && fav_child >= 0) {
+                *policy_str = "MTIME";
+            }
+            break;
+        case AFR_FAV_CHILD_BY_MAJORITY:
+            fav_child = afr_sh_fav_by_majority(this, replies, inode);
+            if (policy_str && fav_child >= 0) {
+                *policy_str = "MAJORITY";
+            }
+            break;
+        case AFR_FAV_CHILD_NONE:
+        default:
+            break;
+    }
+
+    return fav_child;
+}
+
+int
+afr_mark_split_brain_source_sinks_by_policy(
+    call_frame_t *frame, xlator_t *this, inode_t *inode, unsigned char *sources,
+    unsigned char *sinks, unsigned char *healed_sinks, unsigned char *locked_on,
+    struct afr_reply *replies, afr_transaction_type type)
+{
+    afr_private_t *priv = NULL;
+    int fav_child = -1;
+    char mtime_str[256];
+    char ctime_str[256];
+    char *policy_str = NULL;
+    struct tm *tm_ptr;
+    time_t time;
+
+    priv = this->private;
+
+    fav_child = afr_sh_get_fav_by_policy(this, replies, inode, &policy_str);
+    if (fav_child == -1) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, AFR_MSG_SBRAIN_FAV_CHILD_POLICY,
+               "No child selected by favorite-child policy.");
+    } else if (fav_child > priv->child_count - 1) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, AFR_MSG_SBRAIN_FAV_CHILD_POLICY,
+               "Invalid child (%d) "
+               "selected by policy %s.",
+               fav_child, policy_str);
+    } else if (fav_child >= 0) {
+        time = replies[fav_child].poststat.ia_mtime;
+        tm_ptr = localtime(&time);
+        strftime(mtime_str, sizeof(mtime_str), "%Y-%m-%d %H:%M:%S", tm_ptr);
+        time = replies[fav_child].poststat.ia_ctime;
+        tm_ptr = localtime(&time);
+        strftime(ctime_str, sizeof(ctime_str), "%Y-%m-%d %H:%M:%S", tm_ptr);
+
+        gf_msg(this->name, GF_LOG_WARNING, 0, AFR_MSG_SBRAIN_FAV_CHILD_POLICY,
+               "Source %s selected as authentic to resolve conflicting data "
+               "in file (gfid:%s) by %s (%" PRIu64
+               " bytes @ %s mtime, %s "
+               "ctime).",
+               priv->children[fav_child]->name, uuid_utoa(inode->gfid),
+               policy_str, replies[fav_child].poststat.ia_size, mtime_str,
+               ctime_str);
+
+        sources[fav_child] = 1;
+        sinks[fav_child] = 0;
+        healed_sinks[fav_child] = 0;
+    }
+    return fav_child;
 }
 
 gf_boolean_t
-afr_dict_contains_heal_op (call_frame_t *frame)
+afr_is_file_empty_on_all_children(afr_private_t *priv,
+                                  struct afr_reply *replies)
 {
-        afr_local_t   *local     = NULL;
-        dict_t        *xdata_req = NULL;
-        int            ret       = 0;
-        int            heal_op   = -1;
+    int i = 0;
 
-        local = frame->local;
-        xdata_req = local->xdata_req;
-        ret = dict_get_int32 (xdata_req, "heal-op", &heal_op);
-        if (ret)
-                return _gf_false;
-        if (local->xdata_rsp == NULL) {
-                local->xdata_rsp = dict_new();
-                if (!local->xdata_rsp)
-                        return _gf_true;
+    for (i = 0; i < priv->child_count; i++) {
+        if ((!replies[i].valid) || (replies[i].op_ret != 0) ||
+            (replies[i].poststat.ia_size != 0))
+            return _gf_false;
+    }
+
+    return _gf_true;
+}
+
+int
+afr_mark_source_sinks_if_file_empty(xlator_t *this, unsigned char *sources,
+                                    unsigned char *sinks,
+                                    unsigned char *healed_sinks,
+                                    unsigned char *locked_on,
+                                    struct afr_reply *replies,
+                                    afr_transaction_type type)
+{
+    int source = -1;
+    int i = 0;
+    afr_private_t *priv = this->private;
+    struct iatt stbuf = {
+        0,
+    };
+
+    if ((AFR_COUNT(locked_on, priv->child_count) < priv->child_count) ||
+        (afr_success_count(replies, priv->child_count) < priv->child_count))
+        return -1;
+
+    if (type == AFR_DATA_TRANSACTION) {
+        if (!afr_is_file_empty_on_all_children(priv, replies))
+            return -1;
+        goto mark;
+    }
+
+    /*For AFR_METADATA_TRANSACTION, metadata must be same on all bricks.*/
+    stbuf = replies[0].poststat;
+    for (i = 1; i < priv->child_count; i++) {
+        if ((!IA_EQUAL(stbuf, replies[i].poststat, type)) ||
+            (!IA_EQUAL(stbuf, replies[i].poststat, uid)) ||
+            (!IA_EQUAL(stbuf, replies[i].poststat, gid)) ||
+            (!IA_EQUAL(stbuf, replies[i].poststat, prot)))
+            return -1;
+    }
+    for (i = 1; i < priv->child_count; i++) {
+        if (!afr_xattrs_are_equal(replies[0].xdata, replies[i].xdata))
+            return -1;
+    }
+
+mark:
+    /* data/metadata is same on all bricks. Pick one of them as source. Rest
+     * are sinks.*/
+    for (i = 0; i < priv->child_count; i++) {
+        if (source == -1) {
+            source = i;
+            sources[i] = 1;
+            sinks[i] = 0;
+            healed_sinks[i] = 0;
+            continue;
         }
-        ret = dict_set_str (local->xdata_rsp, "sh-fail-msg",
-                            "File not in split-brain");
+        sources[i] = 0;
+        sinks[i] = 1;
+        healed_sinks[i] = 1;
+    }
 
-        return _gf_true;
+    return source;
 }
 
 /* Return a source depending on the type of heal_op, and set sources[source],
@@ -368,121 +1389,156 @@ afr_dict_contains_heal_op (call_frame_t *frame)
  * sinks[node] are 1. This should be the case if the file is in split-brain.
  */
 int
-afr_mark_split_brain_source_sinks (call_frame_t *frame, xlator_t *this,
-                                   unsigned char *sources,
-                                   unsigned char *sinks,
-                                   unsigned char *healed_sinks,
-                                   unsigned char *locked_on,
-                                   struct afr_reply *replies,
-                                   afr_transaction_type type)
-{
-        afr_local_t   *local     = NULL;
-        afr_private_t *priv      = NULL;
-        dict_t        *xdata_req = NULL;
-        dict_t        *xdata_rsp = NULL;
-        int            ret       = 0;
-        int            heal_op   = -1;
-        int            i         = 0;
-        char          *name      = NULL;
-        int            source     = -1;
-
-        local = frame->local;
-        priv = this->private;
-        xdata_req = local->xdata_req;
-
-        ret = dict_get_int32 (xdata_req, "heal-op", &heal_op);
-        if (ret)
-                goto out;
-
-        for (i = 0; i < priv->child_count; i++) {
-                if (locked_on[i])
-                        if (sources[i] || !sinks[i] || !healed_sinks[i]) {
-                                ret = -1;
-                                goto out;
-                        }
-        }
-        if (local->xdata_rsp == NULL) {
-                local->xdata_rsp = dict_new();
-                if (!local->xdata_rsp) {
-                        ret = -1;
-                        goto out;
-                }
+afr_mark_split_brain_source_sinks(
+    call_frame_t *frame, xlator_t *this, inode_t *inode, unsigned char *sources,
+    unsigned char *sinks, unsigned char *healed_sinks, unsigned char *locked_on,
+    struct afr_reply *replies, afr_transaction_type type)
+{
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
+    dict_t *xdata_req = NULL;
+    int heal_op = -1;
+    int ret = -1;
+    int source = -1;
+
+    local = frame->local;
+    priv = this->private;
+    xdata_req = local->xdata_req;
+
+    source = afr_mark_source_sinks_if_file_empty(
+        this, sources, sinks, healed_sinks, locked_on, replies, type);
+    if (source >= 0)
+        return source;
+
+    ret = dict_get_int32_sizen(xdata_req, "heal-op", &heal_op);
+    if (ret)
+        goto autoheal;
+
+    source = afr_mark_split_brain_source_sinks_by_heal_op(
+        frame, this, sources, sinks, healed_sinks, locked_on, replies, type,
+        heal_op);
+    return source;
+
+autoheal:
+    /* Automatically heal if fav_child_policy is set. */
+    if (priv->fav_child_policy != AFR_FAV_CHILD_NONE) {
+        source = afr_mark_split_brain_source_sinks_by_policy(
+            frame, this, inode, sources, sinks, healed_sinks, locked_on,
+            replies, type);
+        if (source != -1) {
+            ret = dict_set_int32_sizen(xdata_req, "fav-child-policy", 1);
+            if (ret)
+                return -1;
         }
-        xdata_rsp = local->xdata_rsp;
+    }
 
-        switch (heal_op) {
-        case GF_SHD_OP_SBRAIN_HEAL_FROM_BIGGER_FILE:
-                if (type == AFR_METADATA_TRANSACTION) {
-                        ret = dict_set_str (xdata_rsp, "sh-fail-msg",
-                                            "Use source-brick option to"
-                                            " heal metadata split-brain");
-                        if (!ret)
-                                ret = -1;
-                        goto out;
-                }
-                for (i = 0 ; i < priv->child_count; i++)
-                        if (locked_on[i])
-                                sources[i] = 1;
-                afr_mark_largest_file_as_source (this, sources, replies);
-                if (AFR_COUNT (sources, priv->child_count) != 1) {
-                        ret = dict_set_str (xdata_rsp, "sh-fail-msg",
-                                            "No bigger file");
-                        if (!ret)
-                                ret = -1;
-                        goto out;
-                }
-                for (i = 0 ; i < priv->child_count; i++)
-                        if (sources[i])
-                                source = i;
-                sinks[source] = 0;
-                healed_sinks[source] = 0;
-                break;
-        case GF_SHD_OP_SBRAIN_HEAL_FROM_BRICK:
-                ret = dict_get_str (xdata_req, "child-name", &name);
-                if (ret)
-                        goto out;
-                source = afr_get_child_index_from_name (this, name);
-                if (source < 0) {
-                        ret = dict_set_str (xdata_rsp, "sh-fail-msg",
-                                            "Invalid brick name");
-                        if (!ret)
-                                ret = -1;
-                        goto out;
-                }
-                if (locked_on[source] != 1) {
-                        ret = dict_set_str (xdata_rsp, "sh-fail-msg",
-                                            "Brick is not up");
-                        if (!ret)
-                                ret = -1;
-                        goto out;
-                }
-                sources[source] = 1;
-                sinks[source] = 0;
-                healed_sinks[source] = 0;
-                break;
-        default:
-                ret = -1;
-                goto out;
-        }
-        ret = source;
-out:
-        return ret;
+    return source;
+}
 
+int
+_afr_fav_child_reset_sink_xattrs(call_frame_t *frame, xlator_t *this,
+                                 inode_t *inode, int source,
+                                 unsigned char *healed_sinks,
+                                 unsigned char *undid_pending,
+                                 afr_transaction_type type,
+                                 unsigned char *locked_on,
+                                 struct afr_reply *replies)
+{
+    afr_private_t *priv = NULL;
+    afr_local_t *local = NULL;
+    int *input_dirty = NULL;
+    int **input_matrix = NULL;
+    int *output_dirty = NULL;
+    int **output_matrix = NULL;
+    dict_t *xattr = NULL;
+    dict_t *xdata = NULL;
+    int i = 0;
+
+    priv = this->private;
+    local = frame->local;
+
+    if (!dict_get_sizen(local->xdata_req, "fav-child-policy"))
+        return 0;
+
+    xdata = dict_new();
+    if (!xdata)
+        return -1;
+
+    input_dirty = alloca0(priv->child_count * sizeof(int));
+    input_matrix = ALLOC_MATRIX(priv->child_count, int);
+    output_dirty = alloca0(priv->child_count * sizeof(int));
+    output_matrix = ALLOC_MATRIX(priv->child_count, int);
+
+    afr_selfheal_extract_xattr(this, replies, type, input_dirty, input_matrix);
+
+    for (i = 0; i < priv->child_count; i++) {
+        if (i == source || !healed_sinks[i])
+            continue;
+        output_dirty[i] = -input_dirty[i];
+        output_matrix[i][source] = -input_matrix[i][source];
+    }
+
+    for (i = 0; i < priv->child_count; i++) {
+        if (!healed_sinks[i] || !locked_on[i])
+            continue;
+        xattr = afr_selfheal_output_xattr(this, _gf_false, type, output_dirty,
+                                          output_matrix, i, NULL);
+
+        afr_selfheal_post_op(frame, this, inode, i, xattr, xdata);
+
+        undid_pending[i] = 1;
+        dict_unref(xattr);
+    }
+
+    if (xdata)
+        dict_unref(xdata);
+
+    return 0;
 }
 
 gf_boolean_t
-afr_does_witness_exist (xlator_t *this, uint64_t *witness)
+afr_does_witness_exist(xlator_t *this, uint64_t *witness)
 {
-        int i = 0;
-        afr_private_t *priv = NULL;
+    int i = 0;
+    afr_private_t *priv = NULL;
 
-        priv = this->private;
+    priv = this->private;
 
-        for (i = 0; i < priv->child_count; i++) {
-                if (witness[i])
-                        return _gf_true;
+    for (i = 0; i < priv->child_count; i++) {
+        if (witness[i])
+            return _gf_true;
+    }
+    return _gf_false;
+}
+
+unsigned int
+afr_get_quorum_count(afr_private_t *priv)
+{
+    if (priv->quorum_count == AFR_QUORUM_AUTO) {
+        return priv->child_count / 2 + 1;
+    } else {
+        return priv->quorum_count;
+    }
+}
+
+void
+afr_selfheal_post_op_failure_accounting(afr_private_t *priv, char *accused,
+                                        unsigned char *sources,
+                                        unsigned char *locked_on)
+{
+    int i = 0;
+    unsigned int quorum_count = 0;
+
+    if (AFR_COUNT(sources, priv->child_count) != 0)
+        return;
+
+    quorum_count = afr_get_quorum_count(priv);
+    for (i = 0; i < priv->child_count; i++) {
+        if ((accused[i] < quorum_count) && locked_on[i]) {
+            sources[i] = 1;
         }
-        return _gf_false;
+    }
+    return;
 }
 
 /*
@@ -505,566 +1561,713 @@ afr_does_witness_exist (xlator_t *this, uint64_t *witness)
  */
 
 int
-afr_selfheal_find_direction (call_frame_t *frame, xlator_t *this,
-                             struct afr_reply *replies,
-                             afr_transaction_type type,
-                             unsigned char *locked_on, unsigned char *sources,
-                             unsigned char *sinks, uint64_t *witness)
-{
-        afr_private_t *priv = NULL;
-        int i = 0;
-        int j = 0;
-        int *dirty = NULL; /* Denotes if dirty xattr is set */
-        int **matrix = NULL;/* Changelog matrix */
-        char *accused = NULL;/* Accused others without any self-accusal */
-        char *pending = NULL;/* Have pending operations on others */
-        char *self_accused = NULL; /* Accused itself */
-
-	priv = this->private;
-
-	dirty = alloca0 (priv->child_count * sizeof (int));
-	accused = alloca0 (priv->child_count);
-        pending = alloca0 (priv->child_count);
-        self_accused = alloca0 (priv->child_count);
-	matrix = ALLOC_MATRIX(priv->child_count, int);
-        memset (witness, 0, sizeof (*witness) * priv->child_count);
-
-        if (afr_success_count (replies,
-                               priv->child_count) < AFR_SH_MIN_PARTICIPANTS) {
-                /* Treat this just like locks not being acquired */
-                return -ENOTCONN;
-        }
-
-	/* First construct the pending matrix for further analysis */
-	afr_selfheal_extract_xattr (this, replies, type, dirty, matrix);
-
-        /* short list all self-accused */
+afr_selfheal_find_direction(call_frame_t *frame, xlator_t *this,
+                            struct afr_reply *replies,
+                            afr_transaction_type type, unsigned char *locked_on,
+                            unsigned char *sources, unsigned char *sinks,
+                            uint64_t *witness, unsigned char *pflag)
+{
+    afr_private_t *priv = NULL;
+    int i = 0;
+    int j = 0;
+    int *dirty = NULL;         /* Denotes if dirty xattr is set */
+    int **matrix = NULL;       /* Changelog matrix */
+    char *accused = NULL;      /* Accused others without any self-accusal */
+    char *pending = NULL;      /* Have pending operations on others */
+    char *self_accused = NULL; /* Accused itself */
+
+    priv = this->private;
+
+    dirty = alloca0(priv->child_count * sizeof(int));
+    accused = alloca0(priv->child_count);
+    pending = alloca0(priv->child_count);
+    self_accused = alloca0(priv->child_count);
+    matrix = ALLOC_MATRIX(priv->child_count, int);
+    memset(witness, 0, sizeof(*witness) * priv->child_count);
+
+    /* First construct the pending matrix for further analysis */
+    afr_selfheal_extract_xattr(this, replies, type, dirty, matrix);
+
+    if (pflag) {
         for (i = 0; i < priv->child_count; i++) {
-                if (matrix[i][i])
-                        self_accused[i] = 1;
+            for (j = 0; j < priv->child_count; j++)
+                if (matrix[i][j])
+                    *pflag |= PFLAG_PENDING;
+            if (*pflag)
+                break;
         }
+    }
+
+    if (afr_success_count(replies, priv->child_count) < priv->child_count) {
+        /* Treat this just like locks not being acquired */
+        return -ENOTCONN;
+    }
+
+    /* short list all self-accused */
+    for (i = 0; i < priv->child_count; i++) {
+        if (matrix[i][i])
+            self_accused[i] = 1;
+    }
+
+    /* Next short list all accused to exclude them from being sources */
+    /* Self-accused can't accuse others as they are FOOLs */
+    for (i = 0; i < priv->child_count; i++) {
+        for (j = 0; j < priv->child_count; j++) {
+            if (matrix[i][j]) {
+                if (!self_accused[i])
+                    accused[j] += 1;
+                if (i != j)
+                    pending[i] += 1;
+            }
+        }
+    }
 
-	/* Next short list all accused to exclude them from being sources */
-        /* Self-accused can't accuse others as they are FOOLs */
-	for (i = 0; i < priv->child_count; i++) {
-		for (j = 0; j < priv->child_count; j++) {
-                        if (matrix[i][j]) {
-                                 if (!self_accused[i])
-                                         accused[j] = 1;
-
-                                 if (i != j)
-                                         pending[i] = 1;
-                         }
-		}
-	}
-
-	/* Short list all non-accused as sources */
-	memset (sources, 0, priv->child_count);
-	for (i = 0; i < priv->child_count; i++) {
-		if (!accused[i] && locked_on[i])
-			sources[i] = 1;
-	}
-
-        /* Everyone accused by non-self-accused sources are sinks */
-        memset (sinks, 0, priv->child_count);
-        for (i = 0; i < priv->child_count; i++) {
-                if (!sources[i])
-                        continue;
-                if (self_accused[i])
-                        continue;
-                for (j = 0; j < priv->child_count; j++) {
-                        if (matrix[i][j])
-                                sinks[j] = 1;
-                }
+    /* Short list all non-accused as sources */
+    for (i = 0; i < priv->child_count; i++) {
+        if (!accused[i] && locked_on[i])
+            sources[i] = 1;
+        else
+            sources[i] = 0;
+    }
+
+    /* Everyone accused by non-self-accused sources are sinks */
+    memset(sinks, 0, priv->child_count);
+    for (i = 0; i < priv->child_count; i++) {
+        if (!sources[i])
+            continue;
+        if (self_accused[i])
+            continue;
+        for (j = 0; j < priv->child_count; j++) {
+            if (matrix[i][j])
+                sinks[j] = 1;
+        }
+    }
+
+    /* For breaking ties provide with number of fops they witnessed */
+
+    /*
+     * count the pending fops witnessed from itself to others when it is
+     * self-accused
+     */
+    for (i = 0; i < priv->child_count; i++) {
+        if (!self_accused[i])
+            continue;
+        for (j = 0; j < priv->child_count; j++) {
+            if (i == j)
+                continue;
+            witness[i] += matrix[i][j];
         }
+    }
 
-        /* For breaking ties provide with number of fops they witnessed */
+    if (type == AFR_DATA_TRANSACTION || type == AFR_METADATA_TRANSACTION)
+        afr_selfheal_post_op_failure_accounting(priv, accused, sources,
+                                                locked_on);
 
-        /*
-         * count the pending fops witnessed from itself to others when it is
-         * self-accused
-         */
+    /* If no sources, all locked nodes are sinks - split brain */
+    if (AFR_COUNT(sources, priv->child_count) == 0) {
         for (i = 0; i < priv->child_count; i++) {
-                if (!self_accused[i])
-                        continue;
-                for (j = 0; j < priv->child_count; j++) {
-                        if (i == j)
-                                continue;
-                        witness[i] += matrix[i][j];
-                }
+            if (locked_on[i])
+                sinks[i] = 1;
         }
-
-         /* If no sources, all locked nodes are sinks - split brain */
-         if (AFR_COUNT (sources, priv->child_count) == 0) {
-                for (i = 0; i < priv->child_count; i++) {
-                        if (locked_on[i])
-                                sinks[i] = 1;
-                }
+        if (pflag)
+            *pflag |= PFLAG_SBRAIN;
+    }
+
+    /* One more class of witness similar to dirty in v2 is where no pending
+     * exists but we have self-accusing markers. This can happen in afr-v1
+     * if the brick crashes just after doing xattrop on self but
+     * before xattrop on the other xattrs on the brick in pre-op. */
+    if (AFR_COUNT(pending, priv->child_count) == 0) {
+        for (i = 0; i < priv->child_count; i++) {
+            if (self_accused[i])
+                witness[i] += matrix[i][i];
         }
-
-        /* In afr-v1 if a file is self-accused but didn't have any pending
+    } else {
+        /* In afr-v1 if a file is self-accused and has pending
          * operations on others then it is similar to 'dirty' in afr-v2.
          * Consider such cases as witness.
          */
         for (i = 0; i < priv->child_count; i++) {
-                if (self_accused[i] && !pending[i])
-                        witness[i] += matrix[i][i];
+            if (self_accused[i] && pending[i])
+                witness[i] += matrix[i][i];
         }
+    }
 
-        /* count the number of dirty fops witnessed */
-        for (i = 0; i < priv->child_count; i++)
-                witness[i] += dirty[i];
+    /* count the number of dirty fops witnessed */
+    for (i = 0; i < priv->child_count; i++)
+        witness[i] += dirty[i];
 
-	return 0;
+    return 0;
 }
 
 void
-afr_log_selfheal (uuid_t gfid, xlator_t *this, int ret, char *type,
-                  int source, unsigned char *healed_sinks)
-{
-        char *status = NULL;
-        char *sinks_str = NULL;
-        char *p = NULL;
-        afr_private_t *priv = NULL;
-        gf_loglevel_t loglevel = GF_LOG_NONE;
-        int i = 0;
-
-        priv = this->private;
-        sinks_str = alloca0 (priv->child_count * 8);
-        p = sinks_str;
-        for (i = 0; i < priv->child_count; i++) {
-                if (!healed_sinks[i])
-                        continue;
-                p += sprintf (p, "%d ", i);
-        }
-
-        if (ret < 0) {
-                status = "Failed";
-                loglevel = GF_LOG_DEBUG;
-        } else {
-                status = "Completed";
-                loglevel = GF_LOG_INFO;
+afr_log_selfheal(uuid_t gfid, xlator_t *this, int ret, char *type, int source,
+                 unsigned char *sources, unsigned char *healed_sinks)
+{
+    char *status = NULL;
+    char *sinks_str = NULL;
+    char *p = NULL;
+    char *sources_str = NULL;
+    char *q = NULL;
+    afr_private_t *priv = NULL;
+    gf_loglevel_t loglevel = GF_LOG_NONE;
+    int i = 0;
+
+    priv = this->private;
+    sinks_str = alloca0(priv->child_count * 8);
+    p = sinks_str;
+    sources_str = alloca0(priv->child_count * 8);
+    q = sources_str;
+    for (i = 0; i < priv->child_count; i++) {
+        if (healed_sinks[i])
+            p += sprintf(p, "%d ", i);
+        if (sources[i]) {
+            if (source == i) {
+                q += sprintf(q, "[%d] ", i);
+            } else {
+                q += sprintf(q, "%d ", i);
+            }
         }
-
-        gf_log (this->name, loglevel, "%s %s selfheal on %s. "
-                "source=%d sinks=%s", status, type, uuid_utoa (gfid),
-                source, sinks_str);
+    }
+
+    if (ret < 0) {
+        status = "Failed";
+        loglevel = GF_LOG_DEBUG;
+    } else {
+        status = "Completed";
+        loglevel = GF_LOG_INFO;
+    }
+
+    gf_msg(this->name, loglevel, 0, AFR_MSG_SELF_HEAL_INFO,
+           "%s %s selfheal on %s. "
+           "sources=%s sinks=%s",
+           status, type, uuid_utoa(gfid), sources_str, sinks_str);
 }
 
 int
-afr_selfheal_discover_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-			   int op_ret, int op_errno, inode_t *inode,
-			   struct iatt *buf, dict_t *xdata, struct iatt *parbuf)
+afr_selfheal_discover_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                          int op_ret, int op_errno, inode_t *inode,
+                          struct iatt *buf, dict_t *xdata, struct iatt *parbuf)
 {
-	afr_local_t *local = NULL;
-	int i = -1;
+    afr_local_t *local = NULL;
+    int i = -1;
+    GF_UNUSED int ret = -1;
+    int8_t need_heal = 1;
+
+    local = frame->local;
+    i = (long)cookie;
+
+    local->replies[i].valid = 1;
+    local->replies[i].op_ret = op_ret;
+    local->replies[i].op_errno = op_errno;
+    if (buf)
+        local->replies[i].poststat = *buf;
+    if (parbuf)
+        local->replies[i].postparent = *parbuf;
+    if (xdata) {
+        local->replies[i].xdata = dict_ref(xdata);
+        ret = dict_get_int8(xdata, "link-count", &need_heal);
+    }
+
+    local->replies[i].need_heal = need_heal;
+    syncbarrier_wake(&local->barrier);
+
+    return 0;
+}
 
-	local = frame->local;
-	i = (long) cookie;
+inode_t *
+afr_selfheal_unlocked_lookup_on(call_frame_t *frame, inode_t *parent,
+                                const char *name, struct afr_reply *replies,
+                                unsigned char *lookup_on, dict_t *xattr)
+{
+    loc_t loc = {
+        0,
+    };
+    dict_t *xattr_req = NULL;
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
+    inode_t *inode = NULL;
 
-	local->replies[i].valid = 1;
-	local->replies[i].op_ret = op_ret;
-	local->replies[i].op_errno = op_errno;
-	if (buf)
-		local->replies[i].poststat = *buf;
-	if (parbuf)
-		local->replies[i].postparent = *parbuf;
-	if (xdata)
-		local->replies[i].xdata = dict_ref (xdata);
+    local = frame->local;
+    priv = frame->this->private;
 
-	syncbarrier_wake (&local->barrier);
+    xattr_req = dict_new();
+    if (!xattr_req)
+        return NULL;
 
-	return 0;
-}
+    if (xattr)
+        dict_copy(xattr, xattr_req);
 
+    if (afr_xattr_req_prepare(frame->this, xattr_req) != 0) {
+        dict_unref(xattr_req);
+        return NULL;
+    }
 
-inode_t *
-afr_selfheal_unlocked_lookup_on (call_frame_t *frame, inode_t *parent,
-				 const char *name, struct afr_reply *replies,
-				 unsigned char *lookup_on, dict_t *xattr)
-{
-	loc_t loc = {0, };
-	dict_t *xattr_req = NULL;
-	afr_local_t *local = NULL;
-	afr_private_t *priv = NULL;
-	inode_t *inode = NULL;
+    inode = inode_new(parent->table);
+    if (!inode) {
+        dict_unref(xattr_req);
+        return NULL;
+    }
 
-	local = frame->local;
-	priv = frame->this->private;
+    loc.parent = inode_ref(parent);
+    gf_uuid_copy(loc.pargfid, parent->gfid);
+    loc.name = name;
+    loc.inode = inode_ref(inode);
 
-	xattr_req = dict_new ();
-	if (!xattr_req)
-		return NULL;
+    AFR_ONLIST(lookup_on, frame, afr_selfheal_discover_cbk, lookup, &loc,
+               xattr_req);
 
-        if (xattr)
-                dict_copy (xattr, xattr_req);
+    afr_replies_copy(replies, local->replies, priv->child_count);
 
-	if (afr_xattr_req_prepare (frame->this, xattr_req) != 0) {
-		dict_destroy (xattr_req);
-		return NULL;
-	}
+    loc_wipe(&loc);
+    dict_unref(xattr_req);
 
-	inode = inode_new (parent->table);
-	if (!inode) {
-		dict_destroy (xattr_req);
-		return NULL;
-	}
-
-	loc.parent = inode_ref (parent);
-	gf_uuid_copy (loc.pargfid, parent->gfid);
-	loc.name = name;
-	loc.inode = inode_ref (inode);
+    return inode;
+}
 
-	AFR_ONLIST (lookup_on, frame, afr_selfheal_discover_cbk, lookup, &loc,
-		    xattr_req);
+static int
+afr_set_multi_dom_lock_count_request(xlator_t *this, dict_t *dict)
+{
+    int ret = 0;
+    afr_private_t *priv = NULL;
+    char *key1 = NULL;
+    char *key2 = NULL;
+
+    priv = this->private;
+    key1 = alloca0(strlen(GLUSTERFS_INODELK_DOM_PREFIX) + 2 +
+                   strlen(this->name));
+    key2 = alloca0(strlen(GLUSTERFS_INODELK_DOM_PREFIX) + 2 +
+                   strlen(priv->sh_domain));
+
+    ret = dict_set_uint32(dict, GLUSTERFS_MULTIPLE_DOM_LK_CNT_REQUESTS, 1);
+    if (ret)
+        return ret;
 
-	afr_replies_copy (replies, local->replies, priv->child_count);
+    sprintf(key1, "%s:%s", GLUSTERFS_INODELK_DOM_PREFIX, this->name);
+    ret = dict_set_uint32(dict, key1, 1);
+    if (ret)
+        return ret;
 
-	loc_wipe (&loc);
-	dict_unref (xattr_req);
+    sprintf(key2, "%s:%s", GLUSTERFS_INODELK_DOM_PREFIX, priv->sh_domain);
+    ret = dict_set_uint32(dict, key2, 1);
+    if (ret)
+        return ret;
 
-	return inode;
+    return 0;
 }
 
 int
-afr_selfheal_unlocked_discover_on (call_frame_t *frame, inode_t *inode,
-				   uuid_t gfid, struct afr_reply *replies,
-				   unsigned char *discover_on)
+afr_selfheal_unlocked_discover_on(call_frame_t *frame, inode_t *inode,
+                                  uuid_t gfid, struct afr_reply *replies,
+                                  unsigned char *discover_on, dict_t *dict)
 {
-	loc_t loc = {0, };
-	dict_t *xattr_req = NULL;
-	afr_local_t *local = NULL;
-	afr_private_t *priv = NULL;
+    loc_t loc = {
+        0,
+    };
+    dict_t *xattr_req = NULL;
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
 
-	local = frame->local;
-	priv = frame->this->private;
+    local = frame->local;
+    priv = frame->this->private;
 
-	xattr_req = dict_new ();
-	if (!xattr_req)
-		return -ENOMEM;
+    xattr_req = dict_new();
+    if (!xattr_req)
+        return -ENOMEM;
+    if (dict)
+        dict_copy(dict, xattr_req);
 
-	if (afr_xattr_req_prepare (frame->this, xattr_req) != 0) {
-		dict_destroy (xattr_req);
-		return -ENOMEM;
-	}
+    if (afr_xattr_req_prepare(frame->this, xattr_req) != 0) {
+        dict_unref(xattr_req);
+        return -ENOMEM;
+    }
 
-	loc.inode = inode_ref (inode);
-	gf_uuid_copy (loc.gfid, gfid);
+    if (afr_set_multi_dom_lock_count_request(frame->this, xattr_req)) {
+        dict_unref(xattr_req);
+        return -1;
+    }
 
-	AFR_ONLIST (discover_on, frame, afr_selfheal_discover_cbk, lookup, &loc,
-		    xattr_req);
+    loc.inode = inode_ref(inode);
+    gf_uuid_copy(loc.gfid, gfid);
 
-	afr_replies_copy (replies, local->replies, priv->child_count);
+    AFR_ONLIST(discover_on, frame, afr_selfheal_discover_cbk, lookup, &loc,
+               xattr_req);
 
-	loc_wipe (&loc);
-	dict_unref (xattr_req);
+    afr_replies_copy(replies, local->replies, priv->child_count);
 
-	return 0;
+    loc_wipe(&loc);
+    dict_unref(xattr_req);
+
+    return 0;
 }
 
 int
-afr_selfheal_unlocked_discover (call_frame_t *frame, inode_t *inode,
-				uuid_t gfid, struct afr_reply *replies)
+afr_selfheal_unlocked_discover(call_frame_t *frame, inode_t *inode, uuid_t gfid,
+                               struct afr_reply *replies)
 {
-	afr_private_t *priv = NULL;
+    afr_local_t *local = NULL;
+    dict_t *dict = NULL;
+
+    local = frame->local;
 
-	priv = frame->this->private;
+    if (local->xattr_req)
+        dict = local->xattr_req;
 
-	return afr_selfheal_unlocked_discover_on (frame, inode, gfid, replies,
-						  priv->child_up);
+    return afr_selfheal_unlocked_discover_on(frame, inode, gfid, replies,
+                                             local->child_up, dict);
 }
 
 unsigned int
-afr_success_count (struct afr_reply *replies, unsigned int count)
+afr_success_count(struct afr_reply *replies, unsigned int count)
 {
-        int     i = 0;
-        unsigned int success = 0;
+    int i = 0;
+    unsigned int success = 0;
 
-        for (i = 0; i < count; i++)
-                if (replies[i].valid && replies[i].op_ret == 0)
-                        success++;
-        return success;
+    for (i = 0; i < count; i++)
+        if (replies[i].valid && replies[i].op_ret == 0)
+            success++;
+    return success;
 }
 
 int
-afr_selfheal_lock_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-		       int op_ret, int op_errno, dict_t *xdata)
+afr_selfheal_lock_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                      int op_ret, int op_errno, dict_t *xdata)
 {
-	afr_local_t *local = NULL;
-	int i = 0;
+    afr_local_t *local = NULL;
+    int i = 0;
 
-	local = frame->local;
-	i = (long) cookie;
+    local = frame->local;
+    i = (long)cookie;
 
-	local->replies[i].valid = 1;
-	local->replies[i].op_ret = op_ret;
-	local->replies[i].op_errno = op_errno;
+    local->replies[i].valid = 1;
+    local->replies[i].op_ret = op_ret;
+    local->replies[i].op_errno = op_errno;
 
-	syncbarrier_wake (&local->barrier);
+    syncbarrier_wake(&local->barrier);
 
-	return 0;
+    return 0;
 }
 
-
 int
-afr_selfheal_locked_fill (call_frame_t *frame, xlator_t *this,
-			  unsigned char *locked_on)
+afr_locked_fill(call_frame_t *frame, xlator_t *this, unsigned char *locked_on)
 {
-	int i = 0;
-	afr_private_t *priv = NULL;
-	afr_local_t *local = NULL;
-	int count = 0;
-
-	local = frame->local;
-	priv = this->private;
-
-	for (i = 0; i < priv->child_count; i++) {
-		if (local->replies[i].valid && local->replies[i].op_ret == 0) {
-			locked_on[i] = 1;
-			count++;
-		} else {
-			locked_on[i] = 0;
-		}
-	}
+    int i = 0;
+    afr_private_t *priv = NULL;
+    afr_local_t *local = NULL;
+    int count = 0;
+
+    local = frame->local;
+    priv = this->private;
+
+    for (i = 0; i < priv->child_count; i++) {
+        if (local->replies[i].valid && local->replies[i].op_ret == 0) {
+            locked_on[i] = 1;
+            count++;
+        } else {
+            locked_on[i] = 0;
+        }
+    }
 
-	return count;
+    return count;
 }
 
-
 int
-afr_selfheal_tryinodelk (call_frame_t *frame, xlator_t *this, inode_t *inode,
-			 char *dom, off_t off, size_t size,
-			 unsigned char *locked_on)
+afr_selfheal_tryinodelk(call_frame_t *frame, xlator_t *this, inode_t *inode,
+                        char *dom, off_t off, size_t size,
+                        unsigned char *locked_on)
 {
-	loc_t loc = {0,};
-	struct gf_flock flock = {0, };
+    loc_t loc = {
+        0,
+    };
+    struct gf_flock flock = {
+        0,
+    };
 
-	loc.inode = inode_ref (inode);
-	gf_uuid_copy (loc.gfid, inode->gfid);
+    loc.inode = inode_ref(inode);
+    gf_uuid_copy(loc.gfid, inode->gfid);
 
-	flock.l_type = F_WRLCK;
-	flock.l_start = off;
-	flock.l_len = size;
+    flock.l_type = F_WRLCK;
+    flock.l_start = off;
+    flock.l_len = size;
 
-	AFR_ONALL (frame, afr_selfheal_lock_cbk, inodelk, dom,
-		    &loc, F_SETLK, &flock, NULL);
+    AFR_ONALL(frame, afr_selfheal_lock_cbk, inodelk, dom, &loc, F_SETLK, &flock,
+              NULL);
 
-	loc_wipe (&loc);
+    loc_wipe(&loc);
 
-	return afr_selfheal_locked_fill (frame, this, locked_on);
+    return afr_locked_fill(frame, this, locked_on);
 }
 
-
 int
-afr_selfheal_inodelk (call_frame_t *frame, xlator_t *this, inode_t *inode,
-		      char *dom, off_t off, size_t size,
-		      unsigned char *locked_on)
+afr_selfheal_inodelk(call_frame_t *frame, xlator_t *this, inode_t *inode,
+                     char *dom, off_t off, size_t size,
+                     unsigned char *locked_on)
 {
-	loc_t loc = {0,};
-	struct gf_flock flock = {0, };
-	afr_local_t *local = NULL;
-	int i = 0;
-	afr_private_t *priv = NULL;
-
-	priv = this->private;
-	local = frame->local;
-
-	loc.inode = inode_ref (inode);
-	gf_uuid_copy (loc.gfid, inode->gfid);
-
-	flock.l_type = F_WRLCK;
-	flock.l_start = off;
-	flock.l_len = size;
-
-	AFR_ONALL (frame, afr_selfheal_lock_cbk, inodelk, dom,
-		    &loc, F_SETLK, &flock, NULL);
-
-	for (i = 0; i < priv->child_count; i++) {
-		if (local->replies[i].op_ret == -1 &&
-		    local->replies[i].op_errno == EAGAIN) {
-			afr_selfheal_locked_fill (frame, this, locked_on);
-			afr_selfheal_uninodelk (frame, this, inode, dom, off,
-						size, locked_on);
-
-			AFR_SEQ (frame, afr_selfheal_lock_cbk, inodelk, dom,
-				 &loc, F_SETLKW, &flock, NULL);
-			break;
-		}
-	}
+    loc_t loc = {
+        0,
+    };
+    struct gf_flock flock = {
+        0,
+    };
+    afr_local_t *local = NULL;
+    int i = 0;
+    afr_private_t *priv = NULL;
+
+    priv = this->private;
+    local = frame->local;
+
+    loc.inode = inode_ref(inode);
+    gf_uuid_copy(loc.gfid, inode->gfid);
+
+    flock.l_type = F_WRLCK;
+    flock.l_start = off;
+    flock.l_len = size;
+
+    AFR_ONALL(frame, afr_selfheal_lock_cbk, inodelk, dom, &loc, F_SETLK, &flock,
+              NULL);
+
+    for (i = 0; i < priv->child_count; i++) {
+        if (local->replies[i].op_ret == -1 &&
+            local->replies[i].op_errno == EAGAIN) {
+            afr_locked_fill(frame, this, locked_on);
+            afr_selfheal_uninodelk(frame, this, inode, dom, off, size,
+                                   locked_on);
+
+            AFR_SEQ(frame, afr_selfheal_lock_cbk, inodelk, dom, &loc, F_SETLKW,
+                    &flock, NULL);
+            break;
+        }
+    }
 
-	loc_wipe (&loc);
+    loc_wipe(&loc);
 
-	return afr_selfheal_locked_fill (frame, this, locked_on);
+    return afr_locked_fill(frame, this, locked_on);
 }
 
+static void
+afr_get_lock_and_eagain_counts(afr_private_t *priv, struct afr_reply *replies,
+                               int *lock_count, int *eagain_count)
+{
+    int i = 0;
+
+    for (i = 0; i < priv->child_count; i++) {
+        if (!replies[i].valid)
+            continue;
+        if (replies[i].op_ret == 0) {
+            (*lock_count)++;
+        } else if (replies[i].op_ret == -1 && replies[i].op_errno == EAGAIN) {
+            (*eagain_count)++;
+        }
+    }
+}
 
+/*Do blocking locks if number of locks acquired is majority and there were some
+ * EAGAINs. Useful for odd-way replication*/
 int
-afr_selfheal_uninodelk (call_frame_t *frame, xlator_t *this, inode_t *inode,
-			char *dom, off_t off, size_t size,
-			const unsigned char *locked_on)
+afr_selfheal_tie_breaker_inodelk(call_frame_t *frame, xlator_t *this,
+                                 inode_t *inode, char *dom, off_t off,
+                                 size_t size, unsigned char *locked_on)
 {
-	loc_t loc = {0,};
-	struct gf_flock flock = {0, };
+    loc_t loc = {
+        0,
+    };
+    struct gf_flock flock = {
+        0,
+    };
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
+    int lock_count = 0;
+    int eagain_count = 0;
 
+    priv = this->private;
+    local = frame->local;
 
-	loc.inode = inode_ref (inode);
-	gf_uuid_copy (loc.gfid, inode->gfid);
+    loc.inode = inode_ref(inode);
+    gf_uuid_copy(loc.gfid, inode->gfid);
 
-	flock.l_type = F_UNLCK;
-	flock.l_start = off;
-	flock.l_len = size;
+    flock.l_type = F_WRLCK;
+    flock.l_start = off;
+    flock.l_len = size;
 
-	AFR_ONLIST (locked_on, frame, afr_selfheal_lock_cbk, inodelk,
-		    dom, &loc, F_SETLK, &flock, NULL);
+    AFR_ONALL(frame, afr_selfheal_lock_cbk, inodelk, dom, &loc, F_SETLK, &flock,
+              NULL);
 
-	loc_wipe (&loc);
+    afr_get_lock_and_eagain_counts(priv, local->replies, &lock_count,
+                                   &eagain_count);
 
-	return 0;
-}
+    if (lock_count > priv->child_count / 2 && eagain_count) {
+        afr_locked_fill(frame, this, locked_on);
+        afr_selfheal_uninodelk(frame, this, inode, dom, off, size, locked_on);
+
+        AFR_SEQ(frame, afr_selfheal_lock_cbk, inodelk, dom, &loc, F_SETLKW,
+                &flock, NULL);
+    }
 
+    loc_wipe(&loc);
+
+    return afr_locked_fill(frame, this, locked_on);
+}
 
 int
-afr_selfheal_tryentrylk (call_frame_t *frame, xlator_t *this, inode_t *inode,
-			 char *dom, const char *name, unsigned char *locked_on)
+afr_selfheal_uninodelk(call_frame_t *frame, xlator_t *this, inode_t *inode,
+                       char *dom, off_t off, size_t size,
+                       const unsigned char *locked_on)
 {
-	loc_t loc = {0,};
+    loc_t loc = {
+        0,
+    };
+    struct gf_flock flock = {
+        0,
+    };
 
-	loc.inode = inode_ref (inode);
-	gf_uuid_copy (loc.gfid, inode->gfid);
+    loc.inode = inode_ref(inode);
+    gf_uuid_copy(loc.gfid, inode->gfid);
 
-	AFR_ONALL (frame, afr_selfheal_lock_cbk, entrylk, dom,
-		   &loc, name, ENTRYLK_LOCK_NB, ENTRYLK_WRLCK, NULL);
+    flock.l_type = F_UNLCK;
+    flock.l_start = off;
+    flock.l_len = size;
 
-	loc_wipe (&loc);
+    AFR_ONLIST(locked_on, frame, afr_selfheal_lock_cbk, inodelk, dom, &loc,
+               F_SETLK, &flock, NULL);
 
-	return afr_selfheal_locked_fill (frame, this, locked_on);
-}
+    loc_wipe(&loc);
 
+    return 0;
+}
 
 int
-afr_selfheal_entrylk (call_frame_t *frame, xlator_t *this, inode_t *inode,
-		      char *dom, const char *name, unsigned char *locked_on)
+afr_selfheal_tryentrylk(call_frame_t *frame, xlator_t *this, inode_t *inode,
+                        char *dom, const char *name, unsigned char *locked_on)
 {
-	loc_t loc = {0,};
-	afr_local_t *local = NULL;
-	int i = 0;
-	afr_private_t *priv = NULL;
+    loc_t loc = {
+        0,
+    };
 
-	priv = this->private;
-	local = frame->local;
+    loc.inode = inode_ref(inode);
+    gf_uuid_copy(loc.gfid, inode->gfid);
 
-	loc.inode = inode_ref (inode);
-	gf_uuid_copy (loc.gfid, inode->gfid);
+    AFR_ONALL(frame, afr_selfheal_lock_cbk, entrylk, dom, &loc, name,
+              ENTRYLK_LOCK_NB, ENTRYLK_WRLCK, NULL);
 
-	AFR_ONALL (frame, afr_selfheal_lock_cbk, entrylk, dom, &loc,
-		   name, ENTRYLK_LOCK_NB, ENTRYLK_WRLCK, NULL);
+    loc_wipe(&loc);
 
-	for (i = 0; i < priv->child_count; i++) {
-		if (local->replies[i].op_ret == -1 &&
-		    local->replies[i].op_errno == EAGAIN) {
-			afr_selfheal_locked_fill (frame, this, locked_on);
-			afr_selfheal_unentrylk (frame, this, inode, dom, name,
-						locked_on);
+    return afr_locked_fill(frame, this, locked_on);
+}
 
-			AFR_SEQ (frame, afr_selfheal_lock_cbk, entrylk, dom,
-				 &loc, name, ENTRYLK_LOCK, ENTRYLK_WRLCK, NULL);
-			break;
-		}
-	}
+int
+afr_selfheal_entrylk(call_frame_t *frame, xlator_t *this, inode_t *inode,
+                     char *dom, const char *name, unsigned char *locked_on)
+{
+    loc_t loc = {
+        0,
+    };
+    afr_local_t *local = NULL;
+    int i = 0;
+    afr_private_t *priv = NULL;
+
+    priv = this->private;
+    local = frame->local;
+
+    loc.inode = inode_ref(inode);
+    gf_uuid_copy(loc.gfid, inode->gfid);
+
+    AFR_ONALL(frame, afr_selfheal_lock_cbk, entrylk, dom, &loc, name,
+              ENTRYLK_LOCK_NB, ENTRYLK_WRLCK, NULL);
+
+    for (i = 0; i < priv->child_count; i++) {
+        if (local->replies[i].op_ret == -1 &&
+            local->replies[i].op_errno == EAGAIN) {
+            afr_locked_fill(frame, this, locked_on);
+            afr_selfheal_unentrylk(frame, this, inode, dom, name, locked_on,
+                                   NULL);
+
+            AFR_SEQ(frame, afr_selfheal_lock_cbk, entrylk, dom, &loc, name,
+                    ENTRYLK_LOCK, ENTRYLK_WRLCK, NULL);
+            break;
+        }
+    }
 
-	loc_wipe (&loc);
+    loc_wipe(&loc);
 
-	return afr_selfheal_locked_fill (frame, this, locked_on);
+    return afr_locked_fill(frame, this, locked_on);
 }
 
-
 int
-afr_selfheal_unentrylk (call_frame_t *frame, xlator_t *this, inode_t *inode,
-			char *dom, const char *name, unsigned char *locked_on)
+afr_selfheal_tie_breaker_entrylk(call_frame_t *frame, xlator_t *this,
+                                 inode_t *inode, char *dom, const char *name,
+                                 unsigned char *locked_on)
 {
-	loc_t loc = {0,};
+    loc_t loc = {
+        0,
+    };
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
+    int lock_count = 0;
+    int eagain_count = 0;
 
-	loc.inode = inode_ref (inode);
-	gf_uuid_copy (loc.gfid, inode->gfid);
+    priv = this->private;
+    local = frame->local;
 
-	AFR_ONLIST (locked_on, frame, afr_selfheal_lock_cbk, entrylk,
-		    dom, &loc, name, ENTRYLK_UNLOCK, ENTRYLK_WRLCK, NULL);
+    loc.inode = inode_ref(inode);
+    gf_uuid_copy(loc.gfid, inode->gfid);
 
-	loc_wipe (&loc);
-
-	return 0;
-}
+    AFR_ONALL(frame, afr_selfheal_lock_cbk, entrylk, dom, &loc, name,
+              ENTRYLK_LOCK_NB, ENTRYLK_WRLCK, NULL);
 
+    afr_get_lock_and_eagain_counts(priv, local->replies, &lock_count,
+                                   &eagain_count);
 
-gf_boolean_t
-afr_is_pending_set (xlator_t *this, dict_t *xdata, int type)
-{
-	int idx = -1;
-	afr_private_t *priv = NULL;
-	void *pending_raw = NULL;
-	int *pending_int = NULL;
-	int i = 0;
+    if (lock_count > priv->child_count / 2 && eagain_count) {
+        afr_locked_fill(frame, this, locked_on);
+        afr_selfheal_unentrylk(frame, this, inode, dom, name, locked_on, NULL);
 
-	priv = this->private;
-	idx = afr_index_for_transaction_type (type);
+        AFR_SEQ(frame, afr_selfheal_lock_cbk, entrylk, dom, &loc, name,
+                ENTRYLK_LOCK, ENTRYLK_WRLCK, NULL);
+    }
 
-	if (dict_get_ptr (xdata, AFR_DIRTY, &pending_raw) == 0) {
-		if (pending_raw) {
-			pending_int = pending_raw;
+    loc_wipe(&loc);
 
-			if (ntoh32 (pending_int[idx]))
-				return _gf_true;
-		}
-	}
+    return afr_locked_fill(frame, this, locked_on);
+}
 
-	for (i = 0; i < priv->child_count; i++) {
-		if (dict_get_ptr (xdata, priv->pending_key[i],
-				  &pending_raw))
-			continue;
-		if (!pending_raw)
-			continue;
-		pending_int = pending_raw;
+int
+afr_selfheal_unentrylk(call_frame_t *frame, xlator_t *this, inode_t *inode,
+                       char *dom, const char *name, unsigned char *locked_on,
+                       dict_t *xdata)
+{
+    loc_t loc = {
+        0,
+    };
 
-		if (ntoh32 (pending_int[idx]))
-			return _gf_true;
-	}
+    loc.inode = inode_ref(inode);
+    gf_uuid_copy(loc.gfid, inode->gfid);
 
-	return _gf_false;
-}
+    AFR_ONLIST(locked_on, frame, afr_selfheal_lock_cbk, entrylk, dom, &loc,
+               name, ENTRYLK_UNLOCK, ENTRYLK_WRLCK, xdata);
 
+    loc_wipe(&loc);
 
-gf_boolean_t
-afr_is_data_set (xlator_t *this, dict_t *xdata)
-{
-	return afr_is_pending_set (this, xdata, AFR_DATA_TRANSACTION);
+    return 0;
 }
 
 gf_boolean_t
-afr_is_metadata_set (xlator_t *this, dict_t *xdata)
+afr_is_data_set(xlator_t *this, dict_t *xdata)
 {
-	return afr_is_pending_set (this, xdata, AFR_METADATA_TRANSACTION);
+    return afr_is_pending_set(this, xdata, AFR_DATA_TRANSACTION);
 }
 
 gf_boolean_t
-afr_is_entry_set (xlator_t *this, dict_t *xdata)
+afr_is_metadata_set(xlator_t *this, dict_t *xdata)
 {
-	return afr_is_pending_set (this, xdata, AFR_ENTRY_TRANSACTION);
+    return afr_is_pending_set(this, xdata, AFR_METADATA_TRANSACTION);
 }
 
-
-inode_t*
-afr_inode_link (inode_t *inode, struct iatt *iatt)
+gf_boolean_t
+afr_is_entry_set(xlator_t *this, dict_t *xdata)
 {
-	inode_t *linked_inode = NULL;
-
-	linked_inode = inode_link (inode, NULL, NULL, iatt);
-
-	if (linked_inode)
-		inode_lookup (linked_inode);
-        return linked_inode;
+    return afr_is_pending_set(this, xdata, AFR_ENTRY_TRANSACTION);
 }
 
-
 /*
  * This function inspects the looked up replies (in an unlocked manner)
  * and decides whether a locked verification and possible healing is
@@ -1075,278 +2278,310 @@ afr_inode_link (inode_t *inode, struct iatt *iatt)
  */
 
 int
-afr_selfheal_unlocked_inspect (call_frame_t *frame, xlator_t *this,
-			       uuid_t gfid, inode_t **link_inode,
-			       gf_boolean_t *data_selfheal,
-			       gf_boolean_t *metadata_selfheal,
-			       gf_boolean_t *entry_selfheal)
-{
-	afr_private_t *priv = NULL;
-        inode_t *inode = NULL;
-	int i = 0;
-	int valid_cnt = 0;
-	struct iatt first = {0, };
-	struct afr_reply *replies = NULL;
-	int ret = -1;
-
-	priv = this->private;
-
-        inode = afr_inode_find (this, gfid);
-        if (!inode)
-                goto out;
+afr_selfheal_unlocked_inspect(call_frame_t *frame, xlator_t *this, uuid_t gfid,
+                              inode_t **link_inode, gf_boolean_t *data_selfheal,
+                              gf_boolean_t *metadata_selfheal,
+                              gf_boolean_t *entry_selfheal,
+                              struct afr_reply *replies_dst)
+{
+    afr_private_t *priv = NULL;
+    inode_t *inode = NULL;
+    int i = 0;
+    int valid_cnt = 0;
+    struct iatt first = {
+        0,
+    };
+    int first_idx = 0;
+    struct afr_reply *replies = NULL;
+    int ret = -1;
+
+    priv = this->private;
+
+    inode = afr_inode_find(this, gfid);
+    if (!inode)
+        goto out;
+
+    replies = alloca0(sizeof(*replies) * priv->child_count);
+
+    ret = afr_selfheal_unlocked_discover(frame, inode, gfid, replies);
+    if (ret)
+        goto out;
+
+    for (i = 0; i < priv->child_count; i++) {
+        if (!replies[i].valid)
+            continue;
+        if (replies[i].op_ret == -1)
+            continue;
+
+        /* The data segment of the changelog can be non-zero to indicate
+         * the directory needs a full heal. So the check below ensures
+         * it's not a directory before setting the data_selfheal boolean.
+         */
+        if (data_selfheal && !IA_ISDIR(replies[i].poststat.ia_type) &&
+            afr_is_data_set(this, replies[i].xdata))
+            *data_selfheal = _gf_true;
 
-	replies = alloca0 (sizeof (*replies) * priv->child_count);
+        if (metadata_selfheal && afr_is_metadata_set(this, replies[i].xdata))
+            *metadata_selfheal = _gf_true;
 
-	ret = afr_selfheal_unlocked_discover (frame, inode, gfid, replies);
-	if (ret)
-                goto out;
+        if (entry_selfheal && afr_is_entry_set(this, replies[i].xdata))
+            *entry_selfheal = _gf_true;
 
-	for (i = 0; i < priv->child_count; i++) {
-		if (!replies[i].valid)
-			continue;
-		if (replies[i].op_ret == -1)
-			continue;
-
-		if (data_selfheal && afr_is_data_set (this, replies[i].xdata))
-			*data_selfheal = _gf_true;
-
-		if (metadata_selfheal &&
-                    afr_is_metadata_set (this, replies[i].xdata))
-			*metadata_selfheal = _gf_true;
-
-		if (entry_selfheal && afr_is_entry_set (this, replies[i].xdata))
-			*entry_selfheal = _gf_true;
-
-		valid_cnt ++;
-		if (valid_cnt == 1) {
-			first = replies[i].poststat;
-			continue;
-		}
-
-		if (!IA_EQUAL (first, replies[i].poststat, type)) {
-			gf_msg (this->name, GF_LOG_ERROR, 0,
-                                AFR_MSG_SPLIT_BRAIN,
-				"TYPE mismatch %d vs %d on %s for gfid:%s",
-				(int) first.ia_type,
-				(int) replies[i].poststat.ia_type,
-				priv->children[i]->name,
-				uuid_utoa (replies[i].poststat.ia_gfid));
-                        ret = -EIO;
-                        goto out;
-		}
-
-		if (!IA_EQUAL (first, replies[i].poststat, uid)) {
-			gf_log (this->name, GF_LOG_DEBUG,
-				"UID mismatch %d vs %d on %s for gfid:%s",
-				(int) first.ia_uid,
-				(int) replies[i].poststat.ia_uid,
-				priv->children[i]->name,
-				uuid_utoa (replies[i].poststat.ia_gfid));
-
-                        if (metadata_selfheal)
-                                *metadata_selfheal = _gf_true;
-		}
-
-		if (!IA_EQUAL (first, replies[i].poststat, gid)) {
-			gf_log (this->name, GF_LOG_DEBUG,
-				"GID mismatch %d vs %d on %s for gfid:%s",
-				(int) first.ia_uid,
-				(int) replies[i].poststat.ia_uid,
-				priv->children[i]->name,
-				uuid_utoa (replies[i].poststat.ia_gfid));
-
-                        if (metadata_selfheal)
-                                *metadata_selfheal = _gf_true;
-		}
-
-		if (!IA_EQUAL (first, replies[i].poststat, prot)) {
-			gf_log (this->name, GF_LOG_DEBUG,
-				"MODE mismatch %d vs %d on %s for gfid:%s",
-				(int) st_mode_from_ia (first.ia_prot, 0),
-				(int) st_mode_from_ia (replies[i].poststat.ia_prot, 0),
-				priv->children[i]->name,
-				uuid_utoa (replies[i].poststat.ia_gfid));
-
-                        if (metadata_selfheal)
-                                *metadata_selfheal = _gf_true;
-		}
-
-		if (IA_ISREG(first.ia_type) &&
-		    !IA_EQUAL (first, replies[i].poststat, size)) {
-			gf_log (this->name, GF_LOG_DEBUG,
-				"SIZE mismatch %lld vs %lld on %s for gfid:%s",
-				(long long) first.ia_size,
-				(long long) replies[i].poststat.ia_size,
-				priv->children[i]->name,
-				uuid_utoa (replies[i].poststat.ia_gfid));
-
-                        if (data_selfheal)
-                                *data_selfheal = _gf_true;
-		}
-	}
-
-	if (valid_cnt > 0 && link_inode) {
-		*link_inode = afr_inode_link (inode, &first);
-                if (!*link_inode) {
-                        ret = -EINVAL;
-                        goto out;
-                }
-        } else if (valid_cnt < 2) {
-                ret = -ENOTCONN;
-                goto out;
+        valid_cnt++;
+        if (valid_cnt == 1) {
+            first = replies[i].poststat;
+            first_idx = i;
+            continue;
         }
 
-        ret = 0;
-out:
-        if (inode)
-                inode_unref (inode);
-        if (replies)
-                afr_replies_wipe (replies, priv->child_count);
+        if (!IA_EQUAL(first, replies[i].poststat, type)) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, AFR_MSG_SPLIT_BRAIN,
+                   "TYPE mismatch %d vs %d on %s for gfid:%s",
+                   (int)first.ia_type, (int)replies[i].poststat.ia_type,
+                   priv->children[i]->name,
+                   uuid_utoa(replies[i].poststat.ia_gfid));
+            gf_event(EVENT_AFR_SPLIT_BRAIN,
+                     "client-pid=%d;"
+                     "subvol=%s;"
+                     "type=file;gfid=%s;"
+                     "ia_type-%d=%s;ia_type-%d=%s",
+                     this->ctx->cmd_args.client_pid, this->name,
+                     uuid_utoa(replies[i].poststat.ia_gfid), first_idx,
+                     gf_inode_type_to_str(first.ia_type), i,
+                     gf_inode_type_to_str(replies[i].poststat.ia_type));
+            ret = -EIO;
+            goto out;
+        }
 
-	return ret;
-}
+        if (!IA_EQUAL(first, replies[i].poststat, uid)) {
+            gf_msg_debug(this->name, 0,
+                         "UID mismatch "
+                         "%d vs %d on %s for gfid:%s",
+                         (int)first.ia_uid, (int)replies[i].poststat.ia_uid,
+                         priv->children[i]->name,
+                         uuid_utoa(replies[i].poststat.ia_gfid));
 
+            if (metadata_selfheal)
+                *metadata_selfheal = _gf_true;
+        }
+
+        if (!IA_EQUAL(first, replies[i].poststat, gid)) {
+            gf_msg_debug(this->name, 0,
+                         "GID mismatch "
+                         "%d vs %d on %s for gfid:%s",
+                         (int)first.ia_uid, (int)replies[i].poststat.ia_uid,
+                         priv->children[i]->name,
+                         uuid_utoa(replies[i].poststat.ia_gfid));
+
+            if (metadata_selfheal)
+                *metadata_selfheal = _gf_true;
+        }
+
+        if (!IA_EQUAL(first, replies[i].poststat, prot)) {
+            gf_msg_debug(this->name, 0,
+                         "MODE mismatch "
+                         "%d vs %d on %s for gfid:%s",
+                         (int)st_mode_from_ia(first.ia_prot, 0),
+                         (int)st_mode_from_ia(replies[i].poststat.ia_prot, 0),
+                         priv->children[i]->name,
+                         uuid_utoa(replies[i].poststat.ia_gfid));
+
+            if (metadata_selfheal)
+                *metadata_selfheal = _gf_true;
+        }
+
+        if (IA_ISREG(first.ia_type) &&
+            !IA_EQUAL(first, replies[i].poststat, size)) {
+            gf_msg_debug(this->name, 0,
+                         "SIZE mismatch "
+                         "%lld vs %lld on %s for gfid:%s",
+                         (long long)first.ia_size,
+                         (long long)replies[i].poststat.ia_size,
+                         priv->children[i]->name,
+                         uuid_utoa(replies[i].poststat.ia_gfid));
+
+            if (data_selfheal)
+                *data_selfheal = _gf_true;
+        }
+    }
+
+    if (valid_cnt > 0 && link_inode) {
+        *link_inode = inode_link(inode, NULL, NULL, &first);
+        if (!*link_inode) {
+            ret = -EINVAL;
+            goto out;
+        }
+    } else if (valid_cnt < 2) {
+        ret = afr_check_stale_error(replies, priv);
+        goto out;
+    }
+
+    ret = 0;
+out:
+    if (replies && replies_dst)
+        afr_replies_copy(replies_dst, replies, priv->child_count);
+    if (inode)
+        inode_unref(inode);
+    if (replies)
+        afr_replies_wipe(replies, priv->child_count);
+
+    return ret;
+}
 
 inode_t *
-afr_inode_find (xlator_t *this, uuid_t gfid)
+afr_inode_find(xlator_t *this, uuid_t gfid)
 {
-	inode_table_t *table = NULL;
-	inode_t *inode = NULL;
+    inode_table_t *table = NULL;
+    inode_t *inode = NULL;
 
-	table = this->itable;
-	if (!table)
-		return NULL;
+    table = this->itable;
+    if (!table)
+        return NULL;
 
-	inode = inode_find (table, gfid);
-	if (inode)
-		return inode;
+    inode = inode_find(table, gfid);
+    if (inode)
+        return inode;
 
-	inode = inode_new (table);
-	if (!inode)
-		return NULL;
+    inode = inode_new(table);
+    if (!inode)
+        return NULL;
 
-	gf_uuid_copy (inode->gfid, gfid);
+    gf_uuid_copy(inode->gfid, gfid);
 
-	return inode;
+    return inode;
 }
 
-
 call_frame_t *
-afr_frame_create (xlator_t *this)
+afr_frame_create(xlator_t *this, int32_t *op_errno)
 {
-	call_frame_t *frame    = NULL;
-	afr_local_t  *local    = NULL;
-	int           op_errno = 0;
-	pid_t         pid      = GF_CLIENT_PID_AFR_SELF_HEALD;
+    call_frame_t *frame = NULL;
+    afr_local_t *local = NULL;
+    pid_t pid = GF_CLIENT_PID_SELF_HEALD;
 
-	frame = create_frame (this, this->ctx->pool);
-	if (!frame)
-		return NULL;
+    frame = create_frame(this, this->ctx->pool);
+    if (!frame) {
+        if (op_errno)
+            *op_errno = ENOMEM;
+        return NULL;
+    }
 
-	local = AFR_FRAME_INIT (frame, op_errno);
-	if (!local) {
-		STACK_DESTROY (frame->root);
-		return NULL;
-	}
+    local = AFR_FRAME_INIT(frame, (*op_errno));
+    if (!local) {
+        STACK_DESTROY(frame->root);
+        return NULL;
+    }
 
-	syncopctx_setfspid (&pid);
+    syncopctx_setfspid(&pid);
 
-	frame->root->pid = pid;
+    frame->root->pid = pid;
 
-	afr_set_lk_owner (frame, this, frame->root);
+    afr_set_lk_owner(frame, this, frame->root);
 
-	return frame;
+    return frame;
 }
 
 int
-afr_selfheal_newentry_mark (call_frame_t *frame, xlator_t *this, inode_t *inode,
-			    int source, struct afr_reply *replies,
-			    unsigned char *sources, unsigned char *newentry)
+afr_selfheal_newentry_mark(call_frame_t *frame, xlator_t *this, inode_t *inode,
+                           int source, struct afr_reply *replies,
+                           unsigned char *sources, unsigned char *newentry)
 {
-	int ret = 0;
-	int i = 0;
-	afr_private_t *priv = NULL;
-	dict_t *xattr = NULL;
-	int **changelog = NULL;
+    int ret = 0;
+    int i = 0;
+    afr_private_t *priv = NULL;
+    dict_t *xattr = NULL;
+    int **changelog = NULL;
 
-	priv = this->private;
+    priv = this->private;
 
-	gf_uuid_copy (inode->gfid, replies[source].poststat.ia_gfid);
+    gf_uuid_copy(inode->gfid, replies[source].poststat.ia_gfid);
 
-	xattr = dict_new();
-	if (!xattr)
-		return -ENOMEM;
+    xattr = dict_new();
+    if (!xattr)
+        return -ENOMEM;
 
-        changelog = afr_mark_pending_changelog (priv, newentry, xattr,
-                                            replies[source].poststat.ia_type);
+    changelog = afr_mark_pending_changelog(priv, newentry, xattr,
+                                           replies[source].poststat.ia_type);
 
-        if (!changelog)
-                goto out;
+    if (!changelog) {
+        ret = -ENOMEM;
+        goto out;
+    }
 
-	for (i = 0; i < priv->child_count; i++) {
-		if (!sources[i])
-			continue;
-		afr_selfheal_post_op (frame, this, inode, i, xattr);
-	}
+    for (i = 0; i < priv->child_count; i++) {
+        if (!sources[i])
+            continue;
+        ret |= afr_selfheal_post_op(frame, this, inode, i, xattr, NULL);
+    }
 out:
-        if (changelog)
-                afr_matrix_cleanup (changelog, priv->child_count);
-        if (xattr)
-                dict_unref (xattr);
-	return ret;
+    if (changelog)
+        afr_matrix_cleanup(changelog, priv->child_count);
+    if (xattr)
+        dict_unref(xattr);
+    return ret;
 }
 
 int
-afr_selfheal_do (call_frame_t *frame, xlator_t *this, uuid_t gfid)
-{
-	int           ret               = -1;
-        int           entry_ret         = 1;
-        int           metadata_ret      = 1;
-        int           data_ret          = 1;
-        int           or_ret            = 0;
-        inode_t      *inode             = NULL;
-	gf_boolean_t  data_selfheal     = _gf_false;
-	gf_boolean_t  metadata_selfheal = _gf_false;
-	gf_boolean_t  entry_selfheal    = _gf_false;
-
-	ret = afr_selfheal_unlocked_inspect (frame, this, gfid, &inode,
-					     &data_selfheal,
-					     &metadata_selfheal,
-					     &entry_selfheal);
-	if (ret)
-		goto out;
-
-        if (!(data_selfheal || metadata_selfheal || entry_selfheal)) {
-                ret = 2;
-                goto out;
+afr_selfheal_do(call_frame_t *frame, xlator_t *this, uuid_t gfid)
+{
+    int ret = -1;
+    int entry_ret = 1;
+    int metadata_ret = 1;
+    int data_ret = 1;
+    int or_ret = 0;
+    inode_t *inode = NULL;
+    fd_t *fd = NULL;
+    gf_boolean_t data_selfheal = _gf_false;
+    gf_boolean_t metadata_selfheal = _gf_false;
+    gf_boolean_t entry_selfheal = _gf_false;
+    afr_private_t *priv = NULL;
+
+    priv = this->private;
+
+    ret = afr_selfheal_unlocked_inspect(frame, this, gfid, &inode,
+                                        &data_selfheal, &metadata_selfheal,
+                                        &entry_selfheal, NULL);
+    if (ret)
+        goto out;
+
+    if (!(data_selfheal || metadata_selfheal || entry_selfheal)) {
+        ret = 2;
+        goto out;
+    }
+
+    if (inode->ia_type == IA_IFREG) {
+        ret = afr_selfheal_data_open(this, inode, &fd);
+        if (!fd) {
+            ret = -EIO;
+            goto out;
         }
+    }
 
-	if (data_selfheal)
-                data_ret = afr_selfheal_data (frame, this, inode);
+    if (data_selfheal && priv->data_self_heal)
+        data_ret = afr_selfheal_data(frame, this, fd);
 
-	if (metadata_selfheal)
-                metadata_ret = afr_selfheal_metadata (frame, this, inode);
+    if (metadata_selfheal && priv->metadata_self_heal)
+        metadata_ret = afr_selfheal_metadata(frame, this, inode);
 
-	if (entry_selfheal)
-                entry_ret = afr_selfheal_entry (frame, this, inode);
+    if (entry_selfheal && priv->entry_self_heal)
+        entry_ret = afr_selfheal_entry(frame, this, inode);
 
-        or_ret = (data_ret | metadata_ret | entry_ret);
+    or_ret = (data_ret | metadata_ret | entry_ret);
 
-        if (data_ret == -EIO || metadata_ret == -EIO || entry_ret == -EIO)
-                ret = -EIO;
-        else if (data_ret == 1 && metadata_ret == 1 && entry_ret == 1)
-                ret = 1;
-        else if (or_ret < 0)
-                ret = or_ret;
-        else
-                ret = 0;
+    if (data_ret == -EIO || metadata_ret == -EIO || entry_ret == -EIO)
+        ret = -EIO;
+    else if (data_ret == 1 && metadata_ret == 1 && entry_ret == 1)
+        ret = 1;
+    else if (or_ret < 0)
+        ret = or_ret;
+    else
+        ret = 0;
 
 out:
-        if (inode) {
-                inode_forget (inode, 1);
-                inode_unref (inode);
-        }
-        return ret;
+    if (inode)
+        inode_unref(inode);
+    if (fd)
+        fd_unref(fd);
+    return ret;
 }
 /*
  * This is the entry point for healing a given GFID. The return values for this
@@ -1358,19 +2593,342 @@ out:
  */
 
 int
-afr_selfheal (xlator_t *this, uuid_t gfid)
+afr_selfheal(xlator_t *this, uuid_t gfid)
+{
+    int ret = -1;
+    call_frame_t *frame = NULL;
+    afr_local_t *local = NULL;
+
+    frame = afr_frame_create(this, NULL);
+    if (!frame)
+        return ret;
+
+    local = frame->local;
+    local->xdata_req = dict_new();
+
+    ret = afr_selfheal_do(frame, this, gfid);
+
+    if (frame)
+        AFR_STACK_DESTROY(frame);
+
+    return ret;
+}
+
+afr_local_t *
+__afr_dequeue_heals(afr_private_t *priv)
+{
+    afr_local_t *local = NULL;
+
+    if (list_empty(&priv->heal_waiting))
+        goto none;
+    if ((priv->background_self_heal_count > 0) &&
+        (priv->healers >= priv->background_self_heal_count))
+        goto none;
+
+    local = list_entry(priv->heal_waiting.next, afr_local_t, healer);
+    priv->heal_waiters--;
+    GF_ASSERT(priv->heal_waiters >= 0);
+    list_del_init(&local->healer);
+    list_add(&local->healer, &priv->healing);
+    priv->healers++;
+    return local;
+none:
+    gf_msg_debug(THIS->name, 0,
+                 "Nothing dequeued. "
+                 "Num healers: %d, Num Waiters: %d",
+                 priv->healers, priv->heal_waiters);
+    return NULL;
+}
+
+int
+afr_refresh_selfheal_wrap(void *opaque)
 {
-        int           ret   = -1;
-	call_frame_t *frame = NULL;
+    call_frame_t *heal_frame = opaque;
+    afr_local_t *local = heal_frame->local;
+    int ret = 0;
 
-	frame = afr_frame_create (this);
-	if (!frame)
-		return ret;
+    ret = afr_selfheal(heal_frame->this, local->refreshinode->gfid);
+    return ret;
+}
 
-        ret = afr_selfheal_do (frame, this, gfid);
+int
+afr_refresh_heal_done(int ret, call_frame_t *frame, void *opaque)
+{
+    call_frame_t *heal_frame = opaque;
+    xlator_t *this = heal_frame->this;
+    afr_private_t *priv = this->private;
+    afr_local_t *local = heal_frame->local;
+
+    LOCK(&priv->lock);
+    {
+        list_del_init(&local->healer);
+        priv->healers--;
+        GF_ASSERT(priv->healers >= 0);
+        local = __afr_dequeue_heals(priv);
+    }
+    UNLOCK(&priv->lock);
+
+    AFR_STACK_DESTROY(heal_frame);
+
+    if (local)
+        afr_heal_synctask(this, local);
+    return 0;
+}
 
-	if (frame)
-		AFR_STACK_DESTROY (frame);
+void
+afr_heal_synctask(xlator_t *this, afr_local_t *local)
+{
+    int ret = 0;
+    call_frame_t *heal_frame = NULL;
+
+    heal_frame = local->heal_frame;
+    ret = synctask_new(this->ctx->env, afr_refresh_selfheal_wrap,
+                       afr_refresh_heal_done, heal_frame, heal_frame);
+    if (ret < 0)
+        /* Heal not launched. Will be queued when the next inode
+         * refresh happens and shd hasn't healed it yet. */
+        afr_refresh_heal_done(ret, heal_frame, heal_frame);
+}
 
-	return ret;
+gf_boolean_t
+afr_throttled_selfheal(call_frame_t *frame, xlator_t *this)
+{
+    gf_boolean_t can_heal = _gf_true;
+    afr_private_t *priv = this->private;
+    afr_local_t *local = frame->local;
+
+    LOCK(&priv->lock);
+    {
+        if ((priv->background_self_heal_count > 0) &&
+            (priv->heal_wait_qlen + priv->background_self_heal_count) >
+                (priv->heal_waiters + priv->healers)) {
+            list_add_tail(&local->healer, &priv->heal_waiting);
+            priv->heal_waiters++;
+            local = __afr_dequeue_heals(priv);
+        } else {
+            can_heal = _gf_false;
+        }
+    }
+    UNLOCK(&priv->lock);
+
+    if (can_heal) {
+        if (local)
+            afr_heal_synctask(this, local);
+        else
+            gf_msg_debug(this->name, 0,
+                         "Max number of heals are "
+                         "pending, background self-heal rejected.");
+    }
+
+    return can_heal;
+}
+
+int
+afr_choose_source_by_policy(afr_private_t *priv, unsigned char *sources,
+                            afr_transaction_type type)
+{
+    int source = -1;
+    int i = 0;
+
+    /* Give preference to local child to save on bandwidth */
+    for (i = 0; i < priv->child_count; i++) {
+        if (priv->local[i] && sources[i]) {
+            if ((type == AFR_DATA_TRANSACTION) && AFR_IS_ARBITER_BRICK(priv, i))
+                continue;
+
+            source = i;
+            goto out;
+        }
+    }
+
+    for (i = 0; i < priv->child_count; i++) {
+        if (sources[i]) {
+            source = i;
+            goto out;
+        }
+    }
+out:
+    return source;
+}
+
+static int
+afr_anon_inode_mkdir_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                         int32_t op_ret, int32_t op_errno, inode_t *inode,
+                         struct iatt *buf, struct iatt *preparent,
+                         struct iatt *postparent, dict_t *xdata)
+{
+    afr_local_t *local = frame->local;
+    int i = (long)cookie;
+
+    local->replies[i].valid = 1;
+    local->replies[i].op_ret = op_ret;
+    local->replies[i].op_errno = op_errno;
+    if (op_ret == 0) {
+        local->op_ret = 0;
+        local->replies[i].poststat = *buf;
+        local->replies[i].preparent = *preparent;
+        local->replies[i].postparent = *postparent;
+    }
+    if (xdata) {
+        local->replies[i].xdata = dict_ref(xdata);
+    }
+
+    syncbarrier_wake(&local->barrier);
+    return 0;
+}
+
+int
+afr_anon_inode_create(xlator_t *this, int child, inode_t **linked_inode)
+{
+    call_frame_t *frame = NULL;
+    afr_local_t *local = NULL;
+    afr_private_t *priv = this->private;
+    unsigned char *mkdir_on = alloca0(priv->child_count);
+    unsigned char *lookup_on = alloca0(priv->child_count);
+    loc_t loc = {0};
+    int32_t op_errno = 0;
+    int32_t child_op_errno = 0;
+    struct iatt iatt = {0};
+    dict_t *xdata = NULL;
+    uuid_t anon_inode_gfid = {0};
+    int mkdir_count = 0;
+    int i = 0;
+
+    /*Try to mkdir everywhere and return success if the dir exists on 'child'
+     */
+
+    if (!priv->use_anon_inode) {
+        op_errno = EINVAL;
+        goto out;
+    }
+
+    frame = afr_frame_create(this, &op_errno);
+    if (op_errno) {
+        goto out;
+    }
+    local = frame->local;
+    if (!local->child_up[child]) {
+        /*Other bricks may need mkdir so don't error out yet*/
+        child_op_errno = ENOTCONN;
+    }
+    gf_uuid_parse(priv->anon_gfid_str, anon_inode_gfid);
+    for (i = 0; i < priv->child_count; i++) {
+        if (!local->child_up[i])
+            continue;
+
+        if (priv->anon_inode[i]) {
+            mkdir_on[i] = 0;
+        } else {
+            mkdir_on[i] = 1;
+            mkdir_count++;
+        }
+    }
+
+    if (mkdir_count == 0) {
+        *linked_inode = inode_find(this->itable, anon_inode_gfid);
+        if (*linked_inode) {
+            op_errno = 0;
+            goto out;
+        }
+    }
+
+    loc.parent = inode_ref(this->itable->root);
+    loc.name = priv->anon_inode_name;
+    loc.inode = inode_new(this->itable);
+    if (!loc.inode) {
+        op_errno = ENOMEM;
+        goto out;
+    }
+
+    xdata = dict_new();
+    if (!xdata) {
+        op_errno = ENOMEM;
+        goto out;
+    }
+
+    op_errno = -dict_set_gfuuid(xdata, "gfid-req", anon_inode_gfid, _gf_true);
+    if (op_errno) {
+        goto out;
+    }
+
+    if (mkdir_count == 0) {
+        memcpy(lookup_on, local->child_up, priv->child_count);
+        goto lookup;
+    }
+
+    AFR_ONLIST(mkdir_on, frame, afr_anon_inode_mkdir_cbk, mkdir, &loc, 0755, 0,
+               xdata);
+
+    for (i = 0; i < priv->child_count; i++) {
+        if (!mkdir_on[i]) {
+            continue;
+        }
+
+        if (local->replies[i].op_ret == 0) {
+            priv->anon_inode[i] = 1;
+            iatt = local->replies[i].poststat;
+        } else if (local->replies[i].op_ret < 0 &&
+                   local->replies[i].op_errno == EEXIST) {
+            lookup_on[i] = 1;
+        } else if (i == child) {
+            child_op_errno = local->replies[i].op_errno;
+        }
+    }
+
+    if (AFR_COUNT(lookup_on, priv->child_count) == 0) {
+        goto link;
+    }
+
+lookup:
+    AFR_ONLIST(lookup_on, frame, afr_selfheal_discover_cbk, lookup, &loc,
+               xdata);
+    for (i = 0; i < priv->child_count; i++) {
+        if (!lookup_on[i]) {
+            continue;
+        }
+
+        if (local->replies[i].op_ret == 0) {
+            if (gf_uuid_compare(anon_inode_gfid,
+                                local->replies[i].poststat.ia_gfid) == 0) {
+                priv->anon_inode[i] = 1;
+                iatt = local->replies[i].poststat;
+            } else {
+                if (i == child)
+                    child_op_errno = EINVAL;
+                gf_msg(this->name, GF_LOG_ERROR, 0, AFR_MSG_INVALID_DATA,
+                       "%s has gfid: %s", priv->anon_inode_name,
+                       uuid_utoa(local->replies[i].poststat.ia_gfid));
+            }
+        } else if (i == child) {
+            child_op_errno = local->replies[i].op_errno;
+        }
+    }
+link:
+    if (!gf_uuid_is_null(iatt.ia_gfid)) {
+        *linked_inode = inode_link(loc.inode, loc.parent, loc.name, &iatt);
+        if (*linked_inode) {
+            op_errno = 0;
+            inode_lookup(*linked_inode);
+        } else {
+            op_errno = ENOMEM;
+        }
+        goto out;
+    }
+
+out:
+    if (xdata)
+        dict_unref(xdata);
+    loc_wipe(&loc);
+    /*child_op_errno takes precedence*/
+    if (child_op_errno == 0) {
+        child_op_errno = op_errno;
+    }
+
+    if (child_op_errno && *linked_inode) {
+        inode_unref(*linked_inode);
+        *linked_inode = NULL;
+    }
+    if (frame)
+        AFR_STACK_DESTROY(frame);
+    return -child_op_errno;
 }
diff --git a/xlators/cluster/afr/src/afr-self-heal-data.c b/xlators/cluster/afr/src/afr-self-heal-data.c
index a8a7326e4ec..37bcc2b3f9e 100644
--- a/xlators/cluster/afr/src/afr-self-heal-data.c
+++ b/xlators/cluster/afr/src/afr-self-heal-data.c
@@ -8,572 +8,592 @@
   cases as published by the Free Software Foundation.
 */
 
-
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
-
 #include "afr.h"
 #include "afr-self-heal.h"
-#include "byte-order.h"
+#include <glusterfs/byte-order.h>
 #include "protocol-common.h"
-
-enum {
-	AFR_SELFHEAL_DATA_FULL = 0,
-	AFR_SELFHEAL_DATA_DIFF,
-};
-
+#include "afr-messages.h"
+#include <glusterfs/events.h>
 
 #define HAS_HOLES(i) ((i->ia_blocks * 512) < (i->ia_size))
 static int
-__checksum_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-		int op_ret, int op_errno, uint32_t weak, uint8_t *strong,
-		dict_t *xdata)
+__checksum_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int op_ret,
+               int op_errno, uint32_t weak, uint8_t *strong, dict_t *xdata)
 {
-	afr_local_t *local = NULL;
-	int i = (long) cookie;
-
-	local = frame->local;
-
-	local->replies[i].valid = 1;
-	local->replies[i].op_ret = op_ret;
-	local->replies[i].op_errno = op_errno;
-	if (strong)
-		memcpy (local->replies[i].checksum, strong, MD5_DIGEST_LENGTH);
+    afr_local_t *local = NULL;
+    struct afr_reply *replies = NULL;
+    int i = (long)cookie;
+
+    local = frame->local;
+    replies = local->replies;
+
+    replies[i].valid = 1;
+    replies[i].op_ret = op_ret;
+    replies[i].op_errno = op_errno;
+    if (xdata) {
+        replies[i].buf_has_zeroes = dict_get_str_boolean(
+            xdata, "buf-has-zeroes", _gf_false);
+        replies[i].fips_mode_rchecksum = dict_get_str_boolean(
+            xdata, "fips-mode-rchecksum", _gf_false);
+    }
+    if (strong) {
+        if (replies[i].fips_mode_rchecksum) {
+            memcpy(local->replies[i].checksum, strong, SHA256_DIGEST_LENGTH);
+        } else {
+            memcpy(local->replies[i].checksum, strong, MD5_DIGEST_LENGTH);
+        }
+    }
 
-	syncbarrier_wake (&local->barrier);
-	return 0;
+    syncbarrier_wake(&local->barrier);
+    return 0;
 }
 
-
-static int
-attr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-	  int op_ret, int op_errno, struct iatt *pre, struct iatt *post,
-	  dict_t *xdata)
+static gf_boolean_t
+__afr_can_skip_data_block_heal(call_frame_t *frame, xlator_t *this, fd_t *fd,
+                               int source, unsigned char *healed_sinks,
+                               off_t offset, size_t size, struct iatt *poststat)
 {
-	int i = (long) cookie;
-	afr_local_t *local = NULL;
-
-	local = frame->local;
-
-	local->replies[i].valid = 1;
-	local->replies[i].op_ret = op_ret;
-	local->replies[i].op_errno = op_errno;
-	if (pre)
-		local->replies[i].prestat = *pre;
-	if (post)
-		local->replies[i].poststat = *post;
-	if (xdata)
-		local->replies[i].xdata = dict_ref (xdata);
-
-	syncbarrier_wake (&local->barrier);
+    afr_private_t *priv = NULL;
+    afr_local_t *local = NULL;
+    unsigned char *wind_subvols = NULL;
+    gf_boolean_t checksum_match = _gf_true;
+    struct afr_reply *replies = NULL;
+    dict_t *xdata = NULL;
+    int i = 0;
+
+    priv = this->private;
+    local = frame->local;
+    replies = local->replies;
+
+    xdata = dict_new();
+    if (!xdata)
+        goto out;
+    if (dict_set_int32_sizen(xdata, "check-zero-filled", 1)) {
+        dict_unref(xdata);
+        goto out;
+    }
+
+    wind_subvols = alloca0(priv->child_count);
+    for (i = 0; i < priv->child_count; i++) {
+        if (i == source || healed_sinks[i])
+            wind_subvols[i] = 1;
+    }
+
+    AFR_ONLIST(wind_subvols, frame, __checksum_cbk, rchecksum, fd, offset, size,
+               xdata);
+    if (xdata)
+        dict_unref(xdata);
+
+    if (!replies[source].valid || replies[source].op_ret != 0)
+        return _gf_false;
 
-	return 0;
-}
+    for (i = 0; i < priv->child_count; i++) {
+        if (i == source)
+            continue;
+        if (replies[i].valid) {
+            if (memcmp(replies[source].checksum, replies[i].checksum,
+                       replies[source].fips_mode_rchecksum
+                           ? SHA256_DIGEST_LENGTH
+                           : MD5_DIGEST_LENGTH)) {
+                checksum_match = _gf_false;
+                break;
+            }
+        }
+    }
 
+    if (checksum_match) {
+        if (HAS_HOLES(poststat))
+            return _gf_true;
 
-static gf_boolean_t
-__afr_selfheal_data_checksums_match (call_frame_t *frame, xlator_t *this,
-				     fd_t *fd, int source,
-				     unsigned char *healed_sinks,
-				     off_t offset, size_t size)
-{
-	afr_private_t *priv = NULL;
-	afr_local_t *local = NULL;
-	unsigned char *wind_subvols = NULL;
-	int i = 0;
-
-	priv = this->private;
-	local = frame->local;
-
-	wind_subvols = alloca0 (priv->child_count);
-	for (i = 0; i < priv->child_count; i++) {
-		if (i == source || healed_sinks[i])
-			wind_subvols[i] = 1;
-	}
-
-	AFR_ONLIST (wind_subvols, frame, __checksum_cbk, rchecksum, fd,
-		    offset, size, NULL);
-
-	if (!local->replies[source].valid || local->replies[source].op_ret != 0)
-		return _gf_false;
-
-	for (i = 0; i < priv->child_count; i++) {
-		if (i == source)
-			continue;
-		if (memcmp (local->replies[source].checksum,
-			    local->replies[i].checksum,
-			    MD5_DIGEST_LENGTH))
-			return _gf_false;
-	}
-
-	return _gf_true;
+        /* For non-sparse files, we might be better off writing the
+         * zeroes to sinks to avoid mismatch of disk-usage in bricks. */
+        if (local->replies[source].buf_has_zeroes)
+            return _gf_false;
+        else
+            return _gf_true;
+    }
+out:
+    return _gf_false;
 }
 
-
 static gf_boolean_t
-__afr_is_sink_zero_filled (xlator_t *this, fd_t *fd, size_t size,
-                           off_t offset, int sink)
+__afr_is_sink_zero_filled(xlator_t *this, fd_t *fd, size_t size, off_t offset,
+                          int sink)
 {
-        afr_private_t *priv   = NULL;
-        struct iobref *iobref = NULL;
-        struct iovec  *iovec  = NULL;
-        int            count  = 0;
-        int            ret    = 0;
-        gf_boolean_t   zero_filled   = _gf_false;
-
-        priv = this->private;
-        ret = syncop_readv (priv->children[sink], fd, size, offset, 0, &iovec,
-                            &count, &iobref, NULL, NULL);
-        if (ret < 0)
-                goto out;
-        ret = iov_0filled (iovec, count);
-        if (!ret)
-                zero_filled = _gf_true;
+    afr_private_t *priv = NULL;
+    struct iobref *iobref = NULL;
+    struct iovec *iovec = NULL;
+    int count = 0;
+    int ret = 0;
+    gf_boolean_t zero_filled = _gf_false;
+
+    priv = this->private;
+    ret = syncop_readv(priv->children[sink], fd, size, offset, 0, &iovec,
+                       &count, &iobref, NULL, NULL, NULL);
+    if (ret < 0)
+        goto out;
+    ret = iov_0filled(iovec, count);
+    if (!ret)
+        zero_filled = _gf_true;
 out:
-        if (iovec)
-                GF_FREE (iovec);
-        if (iobref)
-                iobref_unref (iobref);
-        return zero_filled;
+    if (iovec)
+        GF_FREE(iovec);
+    if (iobref)
+        iobref_unref(iobref);
+    return zero_filled;
 }
 
 static int
-__afr_selfheal_data_read_write (call_frame_t *frame, xlator_t *this, fd_t *fd,
-				int source, unsigned char *healed_sinks,
-				off_t offset, size_t size,
-				struct afr_reply *replies, int type)
+__afr_selfheal_data_read_write(call_frame_t *frame, xlator_t *this, fd_t *fd,
+                               int source, unsigned char *healed_sinks,
+                               off_t offset, size_t size,
+                               struct afr_reply *replies, int type)
 {
-	struct iovec *iovec = NULL;
-	int count = 0;
-	struct iobref *iobref = NULL;
-	int ret = 0;
-	int i = 0;
-	afr_private_t *priv = NULL;
-
-	priv = this->private;
-
-	ret = syncop_readv (priv->children[source], fd, size, offset, 0,
-			    &iovec, &count, &iobref, NULL, NULL);
-	if (ret <= 0)
-		return ret;
-
-	for (i = 0; i < priv->child_count; i++) {
-		if (!healed_sinks[i])
-			continue;
-
-		/*
-		 * TODO: Use fiemap() and discard() to heal holes
-		 * in the future.
-		 *
-		 * For now,
-		 *
-		 * - if the source had any holes at all,
-		 * AND
-		 * - if we are writing past the original file size
-		 *   of the sink
-		 * AND
-		 * - is NOT the last block of the source file. if
-		 *   the block contains EOF, it has to be written
-		 *   in order to set the file size even if the
-		 *   last block is 0-filled.
-		 * AND
-		 * - if the read buffer is filled with only 0's
-		 *
-		 * then, skip writing to this source. We don't depend
-		 * on the write to happen to update the size as we
-		 * have performed an ftruncate() upfront anyways.
-		 */
-#define is_last_block(o,b,s) ((s >= o) && (s <= (o + b)))
-		if (HAS_HOLES ((&replies[source].poststat)) &&
-		    offset >= replies[i].poststat.ia_size &&
-		    !is_last_block (offset, size,
-				    replies[source].poststat.ia_size) &&
-		    (iov_0filled (iovec, count) == 0))
-			continue;
-
-                /* Avoid filling up sparse regions of the sink with 0-filled
-                 * writes.*/
-                if (type == AFR_SELFHEAL_DATA_FULL &&
-                    HAS_HOLES ((&replies[source].poststat)) &&
-                    ((offset + size) <= replies[i].poststat.ia_size) &&
-                    (iov_0filled (iovec, count) == 0) &&
-                    __afr_is_sink_zero_filled (this, fd, size, offset, i)) {
-                                continue;
-                }
-
-		ret = syncop_writev (priv->children[i], fd, iovec, count,
-				     offset, iobref, 0, NULL, NULL);
-		if (ret != iov_length (iovec, count)) {
-			/* write() failed on this sink. unset the corresponding
-			   member in sinks[] (which is healed_sinks[] in the
-			   caller) so that this server does NOT get considered
-			   as successfully healed.
-			*/
-			healed_sinks[i] = 0;
-		}
-	}
-        if (iovec)
-                GF_FREE (iovec);
-	if (iobref)
-		iobref_unref (iobref);
-
-	return ret;
-}
+    struct iovec *iovec = NULL;
+    int count = 0;
+    struct iobref *iobref = NULL;
+    int ret = 0;
+    int i = 0;
+    afr_private_t *priv = NULL;
+
+    priv = this->private;
+
+    ret = syncop_readv(priv->children[source], fd, size, offset, 0, &iovec,
+                       &count, &iobref, NULL, NULL, NULL);
+    if (ret <= 0)
+        return ret;
+
+    for (i = 0; i < priv->child_count; i++) {
+        if (!healed_sinks[i])
+            continue;
+
+            /*
+             * TODO: Use fiemap() and discard() to heal holes
+             * in the future.
+             *
+             * For now,
+             *
+             * - if the source had any holes at all,
+             * AND
+             * - if we are writing past the original file size
+             *   of the sink
+             * AND
+             * - is NOT the last block of the source file. if
+             *   the block contains EOF, it has to be written
+             *   in order to set the file size even if the
+             *   last block is 0-filled.
+             * AND
+             * - if the read buffer is filled with only 0's
+             *
+             * then, skip writing to this source. We don't depend
+             * on the write to happen to update the size as we
+             * have performed an ftruncate() upfront anyways.
+             */
+#define is_last_block(o, b, s) ((s >= o) && (s <= (o + b)))
+        if (HAS_HOLES((&replies[source].poststat)) &&
+            offset >= replies[i].poststat.ia_size &&
+            !is_last_block(offset, size, replies[source].poststat.ia_size) &&
+            (iov_0filled(iovec, count) == 0))
+            continue;
+
+        /* Avoid filling up sparse regions of the sink with 0-filled
+         * writes.*/
+        if (type == AFR_SELFHEAL_DATA_FULL &&
+            HAS_HOLES((&replies[source].poststat)) &&
+            ((offset + size) <= replies[i].poststat.ia_size) &&
+            (iov_0filled(iovec, count) == 0) &&
+            __afr_is_sink_zero_filled(this, fd, size, offset, i)) {
+            continue;
+        }
+
+        ret = syncop_writev(priv->children[i], fd, iovec, count, offset, iobref,
+                            0, NULL, NULL, NULL, NULL);
+        if (ret != iov_length(iovec, count)) {
+            /* write() failed on this sink. unset the corresponding
+               member in sinks[] (which is healed_sinks[] in the
+               caller) so that this server does NOT get considered
+               as successfully healed.
+            */
+            healed_sinks[i] = 0;
+        }
+    }
+    if (iovec)
+        GF_FREE(iovec);
+    if (iobref)
+        iobref_unref(iobref);
 
+    return ret;
+}
 
-static int
-afr_selfheal_data_block (call_frame_t *frame, xlator_t *this, fd_t *fd,
-			 int source, unsigned char *healed_sinks, off_t offset,
-			 size_t size, int type, struct afr_reply *replies)
+static gf_boolean_t
+afr_source_sinks_locked(xlator_t *this, unsigned char *locked_on, int source,
+                        unsigned char *healed_sinks)
 {
-	int ret = -1;
-	int sink_count = 0;
-	afr_private_t *priv = NULL;
-	unsigned char *data_lock = NULL;
-
-	priv = this->private;
-	sink_count = AFR_COUNT (healed_sinks, priv->child_count);
-	data_lock = alloca0 (priv->child_count);
-
-	ret = afr_selfheal_inodelk (frame, this, fd->inode, this->name,
-				    offset, size, data_lock);
-	{
-		if (ret < sink_count) {
-			ret = -ENOTCONN;
-			goto unlock;
-		}
-
-		if (type == AFR_SELFHEAL_DATA_DIFF &&
-		    __afr_selfheal_data_checksums_match (frame, this, fd, source,
-							 healed_sinks, offset, size)) {
-			ret = 0;
-			goto unlock;
-		}
-
-		ret = __afr_selfheal_data_read_write (frame, this, fd, source,
-						      healed_sinks, offset, size,
-						      replies, type);
-	}
-unlock:
-	afr_selfheal_uninodelk (frame, this, fd->inode, this->name,
-				offset, size, data_lock);
-	return ret;
-}
+    afr_private_t *priv = this->private;
+    int i = 0;
 
+    if (!locked_on[source])
+        return _gf_false;
 
+    for (i = 0; i < priv->child_count; i++) {
+        if (healed_sinks[i] && locked_on[i])
+            return _gf_true;
+    }
+
+    return _gf_false;
+}
 
 static int
-afr_selfheal_data_fsync (call_frame_t *frame, xlator_t *this, fd_t *fd,
-			 unsigned char *healed_sinks)
+afr_selfheal_data_block(call_frame_t *frame, xlator_t *this, fd_t *fd,
+                        int source, unsigned char *healed_sinks, off_t offset,
+                        size_t size, int type, struct afr_reply *replies)
 {
-	afr_local_t *local = NULL;
-	afr_private_t *priv = NULL;
-	int i = 0;
-
-	local = frame->local;
-	priv = this->private;
-
-	AFR_ONLIST (healed_sinks, frame, attr_cbk, fsync, fd, 0, NULL);
-
-	for (i = 0; i < priv->child_count; i++)
-		if (healed_sinks[i] && local->replies[i].op_ret != 0)
-			/* fsync() failed. Do NOT consider this server
-			   as successfully healed. Mark it so.
-			*/
-			healed_sinks[i] = 0;
-	return 0;
-}
+    int ret = -1;
+    afr_private_t *priv = NULL;
+    unsigned char *data_lock = NULL;
+
+    priv = this->private;
+    data_lock = alloca0(priv->child_count);
+
+    ret = afr_selfheal_inodelk(frame, this, fd->inode, this->name, offset, size,
+                               data_lock);
+    {
+        if (!afr_source_sinks_locked(this, data_lock, source, healed_sinks)) {
+            ret = -ENOTCONN;
+            goto unlock;
+        }
 
+        if (type == AFR_SELFHEAL_DATA_DIFF &&
+            __afr_can_skip_data_block_heal(frame, this, fd, source,
+                                           healed_sinks, offset, size,
+                                           &replies[source].poststat)) {
+            ret = 0;
+            goto unlock;
+        }
+
+        ret = __afr_selfheal_data_read_write(
+            frame, this, fd, source, healed_sinks, offset, size, replies, type);
+    }
+unlock:
+    afr_selfheal_uninodelk(frame, this, fd->inode, this->name, offset, size,
+                           data_lock);
+    return ret;
+}
 
 static int
-afr_selfheal_data_restore_time (call_frame_t *frame, xlator_t *this,
-				inode_t *inode, int source,
-				unsigned char *healed_sinks,
-				struct afr_reply *replies)
+afr_selfheal_data_fsync(call_frame_t *frame, xlator_t *this, fd_t *fd,
+                        unsigned char *healed_sinks)
 {
-	loc_t loc = {0, };
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
+    int i = 0;
 
-	loc.inode = inode_ref (inode);
-	gf_uuid_copy (loc.gfid, inode->gfid);
+    local = frame->local;
+    priv = this->private;
 
-	AFR_ONLIST (healed_sinks, frame, attr_cbk, setattr, &loc,
-		    &replies[source].poststat,
-		    (GF_SET_ATTR_ATIME|GF_SET_ATTR_MTIME), NULL);
+    if (!priv->ensure_durability)
+        return 0;
 
-	loc_wipe (&loc);
+    AFR_ONLIST(healed_sinks, frame, afr_sh_generic_fop_cbk, fsync, fd, 0, NULL);
 
-	return 0;
+    for (i = 0; i < priv->child_count; i++)
+        if (healed_sinks[i] && local->replies[i].op_ret != 0)
+            /* fsync() failed. Do NOT consider this server
+               as successfully healed. Mark it so.
+            */
+            healed_sinks[i] = 0;
+    return 0;
 }
 
 static int
-afr_data_self_heal_type_get (afr_private_t *priv, unsigned char *healed_sinks,
-                             int source, struct afr_reply *replies)
+afr_data_self_heal_type_get(afr_private_t *priv, unsigned char *healed_sinks,
+                            int source, struct afr_reply *replies)
 {
-        int type = AFR_SELFHEAL_DATA_FULL;
-        int i = 0;
-
-        if (priv->data_self_heal_algorithm == NULL) {
-                type = AFR_SELFHEAL_DATA_FULL;
-                for (i = 0; i < priv->child_count; i++) {
-                        if (!healed_sinks[i] && i != source)
-                                continue;
-                        if (replies[i].poststat.ia_size) {
-                                type = AFR_SELFHEAL_DATA_DIFF;
-                                break;
-                        }
-                }
-        } else if (strcmp (priv->data_self_heal_algorithm, "full") == 0) {
-                type = AFR_SELFHEAL_DATA_FULL;
-        } else if (strcmp (priv->data_self_heal_algorithm, "diff") == 0) {
+    int type = AFR_SELFHEAL_DATA_FULL;
+    int i = 0;
+
+    if (priv->data_self_heal_algorithm == AFR_SELFHEAL_DATA_DYNAMIC) {
+        type = AFR_SELFHEAL_DATA_FULL;
+        for (i = 0; i < priv->child_count; i++) {
+            if (!healed_sinks[i] && i != source)
+                continue;
+            if (replies[i].poststat.ia_size) {
                 type = AFR_SELFHEAL_DATA_DIFF;
+                break;
+            }
         }
-        return type;
+    } else {
+        type = priv->data_self_heal_algorithm;
+    }
+    return type;
 }
 
 static int
-afr_selfheal_data_do (call_frame_t *frame, xlator_t *this, fd_t *fd,
-		      int source, unsigned char *healed_sinks,
-		      struct afr_reply *replies)
+afr_selfheal_data_do(call_frame_t *frame, xlator_t *this, fd_t *fd, int source,
+                     unsigned char *healed_sinks, struct afr_reply *replies)
 {
-	afr_private_t *priv = NULL;
-	off_t off = 0;
-	size_t block = 128 * 1024;
-	int type = AFR_SELFHEAL_DATA_FULL;
-	int ret = -1;
-	call_frame_t *iter_frame = NULL;
-
-	priv = this->private;
-
-        type = afr_data_self_heal_type_get (priv, healed_sinks, source,
-                                            replies);
-
-	iter_frame = afr_copy_frame (frame);
-	if (!iter_frame)
-		return -ENOMEM;
-
-	for (off = 0; off < replies[source].poststat.ia_size; off += block) {
-                if (AFR_COUNT (healed_sinks, priv->child_count) == 0) {
-                        ret = -ENOTCONN;
-                        goto out;
-                }
-
-		ret = afr_selfheal_data_block (iter_frame, this, fd, source,
-					       healed_sinks, off, block, type,
-					       replies);
-		if (ret < 0)
-			goto out;
+    afr_private_t *priv = NULL;
+    off_t off = 0;
+    size_t block = 0;
+    int type = AFR_SELFHEAL_DATA_FULL;
+    int ret = -1;
+    call_frame_t *iter_frame = NULL;
+    unsigned char arbiter_sink_status = 0;
+
+    gf_msg(this->name, GF_LOG_INFO, 0, AFR_MSG_SELF_HEAL_INFO,
+           "performing data selfheal on %s", uuid_utoa(fd->inode->gfid));
+
+    priv = this->private;
+    if (priv->arbiter_count) {
+        arbiter_sink_status = healed_sinks[ARBITER_BRICK_INDEX];
+        healed_sinks[ARBITER_BRICK_INDEX] = 0;
+    }
+
+    block = 128 * 1024 * priv->data_self_heal_window_size;
+
+    type = afr_data_self_heal_type_get(priv, healed_sinks, source, replies);
+
+    iter_frame = afr_copy_frame(frame);
+    if (!iter_frame) {
+        ret = -ENOMEM;
+        goto out;
+    }
+
+    for (off = 0; off < replies[source].poststat.ia_size; off += block) {
+        if (AFR_COUNT(healed_sinks, priv->child_count) == 0) {
+            ret = -ENOTCONN;
+            goto out;
+        }
 
-		AFR_STACK_RESET (iter_frame);
-	}
+        ret = afr_selfheal_data_block(iter_frame, this, fd, source,
+                                      healed_sinks, off, block, type, replies);
+        if (ret < 0)
+            goto out;
 
-	afr_selfheal_data_restore_time (frame, this, fd->inode, source,
-					healed_sinks, replies);
+        AFR_STACK_RESET(iter_frame);
+        if (iter_frame->local == NULL) {
+            ret = -ENOTCONN;
+            goto out;
+        }
+    }
 
-	ret = afr_selfheal_data_fsync (frame, this, fd, healed_sinks);
+    ret = afr_selfheal_data_fsync(frame, this, fd, healed_sinks);
 
 out:
-	if (iter_frame)
-		AFR_STACK_DESTROY (iter_frame);
-	return ret;
-}
+    if (arbiter_sink_status)
+        healed_sinks[ARBITER_BRICK_INDEX] = arbiter_sink_status;
 
+    if (iter_frame)
+        AFR_STACK_DESTROY(iter_frame);
+    return ret;
+}
 
 static int
-__afr_selfheal_truncate_sinks (call_frame_t *frame, xlator_t *this,
-			       fd_t *fd, unsigned char *healed_sinks,
-			       struct afr_reply *replies, uint64_t size)
+__afr_selfheal_truncate_sinks(call_frame_t *frame, xlator_t *this, fd_t *fd,
+                              unsigned char *healed_sinks, uint64_t size)
 {
-	afr_local_t *local = NULL;
-	afr_private_t *priv = NULL;
-	unsigned char *larger_sinks = 0;
-	int i = 0;
-
-	local = frame->local;
-	priv = this->private;
-
-	larger_sinks = alloca0 (priv->child_count);
-	for (i = 0; i < priv->child_count; i++) {
-		if (healed_sinks[i] && replies[i].poststat.ia_size > size)
-			larger_sinks[i] = 1;
-	}
-
-	AFR_ONLIST (larger_sinks, frame, attr_cbk, ftruncate, fd, size, NULL);
-
-	for (i = 0; i < priv->child_count; i++)
-		if (healed_sinks[i] && local->replies[i].op_ret == -1)
-			/* truncate() failed. Do NOT consider this server
-			   as successfully healed. Mark it so.
-			*/
-			healed_sinks[i] = 0;
-	return 0;
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
+    int i = 0;
+
+    local = frame->local;
+    priv = this->private;
+
+    /* This will send truncate on the arbiter brick as well if it is marked as
+     * sink. If changelog is enabled on the volume it captures truncate as a
+     * data transactions on the arbiter brick. This will help geo-rep to
+     * properly sync the data from master to slave if arbiter is the ACTIVE
+     * brick during syncing and which had got some entries healed for data as
+     * part of self heal.
+     */
+    AFR_ONLIST(healed_sinks, frame, afr_sh_generic_fop_cbk, ftruncate, fd, size,
+               NULL);
+
+    for (i = 0; i < priv->child_count; i++)
+        if (healed_sinks[i] && local->replies[i].op_ret == -1)
+            /* truncate() failed. Do NOT consider this server
+               as successfully healed. Mark it so.
+            */
+            healed_sinks[i] = 0;
+
+    return 0;
 }
 
 gf_boolean_t
-afr_has_source_witnesses (xlator_t *this, unsigned char *sources,
-                          uint64_t *witness)
+afr_has_source_witnesses(xlator_t *this, unsigned char *sources,
+                         uint64_t *witness)
 {
-        int i = 0;
-        afr_private_t *priv = NULL;
+    int i = 0;
+    afr_private_t *priv = NULL;
 
-        priv = this->private;
+    priv = this->private;
 
-        for (i = 0; i < priv->child_count; i++) {
-                if (sources[i] && witness[i])
-                        return _gf_true;
-        }
-        return _gf_false;
+    for (i = 0; i < priv->child_count; i++) {
+        if (sources[i] && witness[i])
+            return _gf_true;
+    }
+    return _gf_false;
 }
 
 static gf_boolean_t
-afr_does_size_mismatch (xlator_t *this, unsigned char *sources,
-                        struct afr_reply *replies)
+afr_does_size_mismatch(xlator_t *this, unsigned char *sources,
+                       struct afr_reply *replies)
 {
-        int     i = 0;
-        afr_private_t *priv = NULL;
-        struct iatt *min = NULL;
-        struct iatt *max = NULL;
+    int i = 0;
+    afr_private_t *priv = NULL;
+    struct iatt *min = NULL;
+    struct iatt *max = NULL;
 
-        priv = this->private;
+    priv = this->private;
 
-        for (i = 0; i < priv->child_count; i++) {
-                if (!replies[i].valid)
-                        continue;
+    for (i = 0; i < priv->child_count; i++) {
+        if (!replies[i].valid)
+            continue;
 
-                if (replies[i].op_ret < 0)
-                        continue;
+        if (replies[i].op_ret < 0)
+            continue;
 
-                if (!min)
-                        min = &replies[i].poststat;
+        if (!sources[i])
+            continue;
 
-                if (!max)
-                        max = &replies[i].poststat;
+        if (AFR_IS_ARBITER_BRICK(priv, i) && (replies[i].poststat.ia_size == 0))
+            continue;
 
-                if (min->ia_size > replies[i].poststat.ia_size)
-                        min = &replies[i].poststat;
+        if (!min)
+            min = &replies[i].poststat;
 
-                if (max->ia_size < replies[i].poststat.ia_size)
-                        max = &replies[i].poststat;
-        }
+        if (!max)
+            max = &replies[i].poststat;
 
-        if (min && max) {
-                if (min->ia_size != max->ia_size)
-                        return _gf_true;
-        }
+        if (min->ia_size > replies[i].poststat.ia_size)
+            min = &replies[i].poststat;
 
-        return _gf_false;
+        if (max->ia_size < replies[i].poststat.ia_size)
+            max = &replies[i].poststat;
+    }
+
+    if (min && max) {
+        if (min->ia_size != max->ia_size)
+            return _gf_true;
+    }
+
+    return _gf_false;
 }
 
 static void
-afr_mark_biggest_witness_as_source (xlator_t *this, unsigned char *sources,
-                                    uint64_t *witness)
+afr_mark_biggest_witness_as_source(xlator_t *this, unsigned char *sources,
+                                   uint64_t *witness)
 {
-        int i = 0;
-        afr_private_t *priv = NULL;
-        uint64_t biggest_witness = 0;
-
-        priv = this->private;
-        /* Find source with biggest witness count */
-        for (i = 0; i < priv->child_count; i++) {
-                if (!sources[i])
-                        continue;
-                if (biggest_witness < witness[i])
-                        biggest_witness = witness[i];
-        }
-
-        /* Mark files with less witness count as not source */
-        for (i = 0; i < priv->child_count; i++) {
-                if (!sources[i])
-                        continue;
-                if (witness[i] < biggest_witness)
-                        sources[i] = 0;
-        }
-
-        return;
+    int i = 0;
+    afr_private_t *priv = NULL;
+    uint64_t biggest_witness = 0;
+
+    priv = this->private;
+    /* Find source with biggest witness count */
+    for (i = 0; i < priv->child_count; i++) {
+        if (!sources[i])
+            continue;
+        if (biggest_witness < witness[i])
+            biggest_witness = witness[i];
+    }
+
+    /* Mark files with less witness count as not source */
+    for (i = 0; i < priv->child_count; i++) {
+        if (!sources[i])
+            continue;
+        if (witness[i] < biggest_witness)
+            sources[i] = 0;
+    }
+
+    return;
 }
 
 /* This is a tie breaker function. Only one source be assigned here */
 static void
-afr_mark_newest_file_as_source (xlator_t *this, unsigned char *sources,
-                                struct afr_reply *replies)
+afr_mark_newest_file_as_source(xlator_t *this, unsigned char *sources,
+                               struct afr_reply *replies)
 {
-        int i = 0;
-        afr_private_t *priv = NULL;
-        int source = -1;
-        uint32_t max_ctime = 0;
-
-        priv = this->private;
-        /* Find source with latest ctime */
-        for (i = 0; i < priv->child_count; i++) {
-                if (!sources[i])
-                        continue;
-
-                if (max_ctime <= replies[i].poststat.ia_ctime) {
-                        source = i;
-                        max_ctime = replies[i].poststat.ia_ctime;
-                }
+    int i = 0;
+    afr_private_t *priv = NULL;
+    int source = -1;
+    uint32_t max_ctime = 0;
+
+    priv = this->private;
+    /* Find source with latest ctime */
+    for (i = 0; i < priv->child_count; i++) {
+        if (!sources[i])
+            continue;
+
+        if (max_ctime <= replies[i].poststat.ia_ctime) {
+            source = i;
+            max_ctime = replies[i].poststat.ia_ctime;
         }
+    }
 
-        /* Only mark one of the files as source to break ties */
-        memset (sources, 0, sizeof (*sources) * priv->child_count);
-        sources[source] = 1;
+    /* Only mark one of the files as source to break ties */
+    memset(sources, 0, sizeof(*sources) * priv->child_count);
+    sources[source] = 1;
 }
 
 static int
-__afr_selfheal_data_finalize_source (call_frame_t *frame, xlator_t *this,
-                                     unsigned char *sources,
-                                     unsigned char *sinks,
-				     unsigned char *healed_sinks,
-				     unsigned char *locked_on,
-				     struct afr_reply *replies,
-                                     uint64_t *witness)
+__afr_selfheal_data_finalize_source(
+    call_frame_t *frame, xlator_t *this, inode_t *inode, unsigned char *sources,
+    unsigned char *sinks, unsigned char *healed_sinks, unsigned char *locked_on,
+    unsigned char *undid_pending, struct afr_reply *replies, uint64_t *witness)
 {
-	int i = 0;
-	afr_private_t *priv = NULL;
-	int source = -1;
-	int sources_count = 0;
-	priv = this->private;
-
-	sources_count = AFR_COUNT (sources, priv->child_count);
-
-	if ((AFR_CMP (locked_on, healed_sinks, priv->child_count) == 0)
-            || !sources_count) {
-		/* split brain */
-                source = afr_mark_split_brain_source_sinks (frame, this,
-                                                            sources, sinks,
-                                                            healed_sinks,
-                                                            locked_on, replies,
-                                                          AFR_DATA_TRANSACTION);
-                if (source < 0)
-                        return -EIO;
-                return source;
-	}
-
-        /* No split brain at this point. If we were called from
-         * afr_heal_splitbrain_file(), abort.*/
-        if (afr_dict_contains_heal_op(frame))
-                return -EIO;
-
-        /* If there are no witnesses/size-mismatches on sources we are done*/
-        if (!afr_does_size_mismatch (this, sources, replies) &&
-            !afr_has_source_witnesses (this, sources, witness))
-                goto out;
-
-        afr_mark_largest_file_as_source (this, sources, replies);
-        afr_mark_biggest_witness_as_source (this, sources, witness);
-        afr_mark_newest_file_as_source (this, sources, replies);
+    afr_private_t *priv = NULL;
+    int source = -1;
+    int sources_count = 0;
+    priv = this->private;
+
+    sources_count = AFR_COUNT(sources, priv->child_count);
+
+    if ((AFR_CMP(locked_on, healed_sinks, priv->child_count) == 0) ||
+        !sources_count) {
+        /* split brain */
+        source = afr_mark_split_brain_source_sinks(
+            frame, this, inode, sources, sinks, healed_sinks, locked_on,
+            replies, AFR_DATA_TRANSACTION);
+        if (source < 0) {
+            gf_event(EVENT_AFR_SPLIT_BRAIN,
+                     "client-pid=%d;"
+                     "subvol=%s;type=data;"
+                     "file=%s",
+                     this->ctx->cmd_args.client_pid, this->name,
+                     uuid_utoa(inode->gfid));
+            return -EIO;
+        }
+
+        _afr_fav_child_reset_sink_xattrs(
+            frame, this, inode, source, healed_sinks, undid_pending,
+            AFR_DATA_TRANSACTION, locked_on, replies);
+        goto out;
+    }
+
+    /* No split brain at this point. If we were called from
+     * afr_heal_splitbrain_file(), abort.*/
+    if (afr_dict_contains_heal_op(frame))
+        return -EIO;
+
+    /* If there are no witnesses/size-mismatches on sources we are done*/
+    if (!afr_does_size_mismatch(this, sources, replies) &&
+        !afr_has_source_witnesses(this, sources, witness))
+        goto out;
+
+    afr_mark_largest_file_as_source(this, sources, replies);
+    afr_mark_biggest_witness_as_source(this, sources, witness);
+    afr_mark_newest_file_as_source(this, sources, replies);
+    if (priv->arbiter_count)
+        /* Choose non-arbiter brick as source for empty files. */
+        afr_mark_source_sinks_if_file_empty(this, sources, sinks, healed_sinks,
+                                            locked_on, replies,
+                                            AFR_DATA_TRANSACTION);
 
 out:
-        afr_mark_active_sinks (this, sources, locked_on, healed_sinks);
+    afr_mark_active_sinks(this, sources, locked_on, healed_sinks);
+    source = afr_choose_source_by_policy(priv, sources, AFR_DATA_TRANSACTION);
 
-        for (i = 0; i < priv->child_count; i++) {
-                if (sources[i]) {
-                        source = i;
-                        break;
-                }
-        }
-	return source;
+    return source;
 }
 
 /*
@@ -586,225 +606,286 @@ out:
  * for self-healing, or -1 if no healing is necessary/split brain.
  */
 int
-__afr_selfheal_data_prepare (call_frame_t *frame, xlator_t *this,
-                             inode_t *inode, unsigned char *locked_on,
-                             unsigned char *sources, unsigned char *sinks,
-                             unsigned char *healed_sinks,
-			     struct afr_reply *replies)
+__afr_selfheal_data_prepare(call_frame_t *frame, xlator_t *this, inode_t *inode,
+                            unsigned char *locked_on, unsigned char *sources,
+                            unsigned char *sinks, unsigned char *healed_sinks,
+                            unsigned char *undid_pending,
+                            struct afr_reply *replies, unsigned char *pflag)
 {
-	int ret = -1;
-	int source = -1;
-	afr_private_t *priv = NULL;
-        uint64_t *witness = NULL;
-
-	priv = this->private;
-
-	ret = afr_selfheal_unlocked_discover (frame, inode, inode->gfid,
-					      replies);
-
-        if (ret)
-                return ret;
-
-        witness = alloca0(priv->child_count * sizeof (*witness));
-	ret = afr_selfheal_find_direction (frame, this, replies,
-					   AFR_DATA_TRANSACTION,
-					   locked_on, sources, sinks, witness);
-	if (ret)
-		return ret;
-
-        /* Initialize the healed_sinks[] array optimistically to
-           the intersection of to-be-healed (i.e sinks[]) and
-           the list of servers which are up (i.e locked_on[]).
-           As we encounter failures in the healing process, we
-           will unmark the respective servers in the healed_sinks[]
-           array.
-        */
-        AFR_INTERSECT (healed_sinks, sinks, locked_on, priv->child_count);
-
-	source = __afr_selfheal_data_finalize_source (frame, this, sources,
-                                                      sinks, healed_sinks,
-                                                      locked_on, replies,
-                                                      witness);
-	if (source < 0)
-		return -EIO;
-
-	return source;
+    int ret = -1;
+    int source = -1;
+    afr_private_t *priv = NULL;
+    uint64_t *witness = NULL;
+
+    priv = this->private;
+
+    ret = afr_selfheal_unlocked_discover(frame, inode, inode->gfid, replies);
+
+    if (ret)
+        return ret;
+
+    witness = alloca0(priv->child_count * sizeof(*witness));
+    ret = afr_selfheal_find_direction(frame, this, replies,
+                                      AFR_DATA_TRANSACTION, locked_on, sources,
+                                      sinks, witness, pflag);
+    if (ret)
+        return ret;
+
+    /* Initialize the healed_sinks[] array optimistically to
+       the intersection of to-be-healed (i.e sinks[]) and
+       the list of servers which are up (i.e locked_on[]).
+       As we encounter failures in the healing process, we
+       will unmark the respective servers in the healed_sinks[]
+       array.
+    */
+    AFR_INTERSECT(healed_sinks, sinks, locked_on, priv->child_count);
+
+    source = __afr_selfheal_data_finalize_source(
+        frame, this, inode, sources, sinks, healed_sinks, locked_on,
+        undid_pending, replies, witness);
+    if (source < 0)
+        return -EIO;
+
+    return source;
 }
 
-
 static int
-__afr_selfheal_data (call_frame_t *frame, xlator_t *this, fd_t *fd,
-		     unsigned char *locked_on)
+__afr_selfheal_data(call_frame_t *frame, xlator_t *this, fd_t *fd,
+                    unsigned char *locked_on)
 {
-	afr_private_t *priv = NULL;
-	int ret = -1;
-	unsigned char *sources = NULL;
-	unsigned char *sinks = NULL;
-	unsigned char *data_lock = NULL;
-	unsigned char *healed_sinks = NULL;
-	struct afr_reply *locked_replies = NULL;
-	int source = -1;
-	gf_boolean_t compat = _gf_false;
-        gf_boolean_t did_sh = _gf_true;
-	unsigned char *compat_lock = NULL;
-
-	priv = this->private;
-
-	sources = alloca0 (priv->child_count);
-	sinks = alloca0 (priv->child_count);
-	healed_sinks = alloca0 (priv->child_count);
-	data_lock = alloca0 (priv->child_count);
-	compat_lock = alloca0 (priv->child_count);
-
-	locked_replies = alloca0 (sizeof (*locked_replies) * priv->child_count);
-
-	ret = afr_selfheal_inodelk (frame, this, fd->inode, this->name, 0, 0,
-				    data_lock);
-	{
-		if (ret < AFR_SH_MIN_PARTICIPANTS) {
-                        gf_log (this->name, GF_LOG_DEBUG, "%s: Skipping "
-                                "self-heal as only %d number of subvolumes "
-                                "could be locked", uuid_utoa (fd->inode->gfid),
-                                ret);
-			ret = -ENOTCONN;
-			goto unlock;
-		}
-
-		ret = __afr_selfheal_data_prepare (frame, this, fd->inode,
-                                                   data_lock, sources, sinks,
-                                                   healed_sinks,
-						   locked_replies);
-		if (ret < 0)
-			goto unlock;
-
-                if (AFR_COUNT(healed_sinks, priv->child_count) == 0) {
-                        did_sh = _gf_false;
-                        goto unlock;
-                }
-
-		source = ret;
-
-		ret = __afr_selfheal_truncate_sinks (frame, this, fd, healed_sinks,
-						     locked_replies,
-						     locked_replies[source].poststat.ia_size);
-		if (ret < 0)
-			goto unlock;
-
-		ret = 0;
-
-		/* Locking from (LLONG_MAX - 2) to (LLONG_MAX - 1) is for
-		   compatibility with older self-heal clients which do not
-		   hold a lock in the @priv->sh_domain domain to guard
-		   against concurrent ongoing self-heals
-		*/
-		afr_selfheal_inodelk (frame, this, fd->inode, this->name,
-				      LLONG_MAX - 2, 1, compat_lock);
-		compat = _gf_true;
-	}
-unlock:
-	afr_selfheal_uninodelk (frame, this, fd->inode, this->name, 0, 0,
-				data_lock);
+    afr_private_t *priv = NULL;
+    int ret = -1;
+    unsigned char *sources = NULL;
+    unsigned char *sinks = NULL;
+    unsigned char *data_lock = NULL;
+    unsigned char *healed_sinks = NULL;
+    unsigned char *undid_pending = NULL;
+    struct afr_reply *locked_replies = NULL;
+    int source = -1;
+    gf_boolean_t did_sh = _gf_true;
+    gf_boolean_t is_arbiter_the_only_sink = _gf_false;
+    gf_boolean_t empty_file = _gf_false;
+
+    priv = this->private;
+
+    sources = alloca0(priv->child_count);
+    sinks = alloca0(priv->child_count);
+    healed_sinks = alloca0(priv->child_count);
+    data_lock = alloca0(priv->child_count);
+    undid_pending = alloca0(priv->child_count);
+
+    locked_replies = alloca0(sizeof(*locked_replies) * priv->child_count);
+
+    ret = afr_selfheal_inodelk(frame, this, fd->inode, this->name, 0, 0,
+                               data_lock);
+    {
+        if (ret < priv->child_count) {
+            gf_msg_debug(this->name, 0,
+                         "%s: Skipping "
+                         "self-heal as only %d number "
+                         "of subvolumes "
+                         "could be locked",
+                         uuid_utoa(fd->inode->gfid), ret);
+            ret = -ENOTCONN;
+            goto unlock;
+        }
+
+        ret = __afr_selfheal_data_prepare(frame, this, fd->inode, data_lock,
+                                          sources, sinks, healed_sinks,
+                                          undid_pending, locked_replies, NULL);
         if (ret < 0)
-		goto out;
+            goto unlock;
 
-        if (!did_sh)
-                goto out;
+        if (AFR_COUNT(healed_sinks, priv->child_count) == 0) {
+            did_sh = _gf_false;
+            goto unlock;
+        }
+
+        source = ret;
 
-	ret = afr_selfheal_data_do (frame, this, fd, source, healed_sinks,
-				    locked_replies);
-	if (ret)
-		goto out;
+        if (AFR_IS_ARBITER_BRICK(priv, source)) {
+            empty_file = afr_is_file_empty_on_all_children(priv,
+                                                           locked_replies);
+            if (empty_file)
+                goto restore_time;
 
-	ret = afr_selfheal_undo_pending (frame, this, fd->inode, sources, sinks,
-					 healed_sinks, AFR_DATA_TRANSACTION,
-					 locked_replies, data_lock);
+            did_sh = _gf_false;
+            goto unlock;
+        }
+
+        ret = __afr_selfheal_truncate_sinks(
+            frame, this, fd, healed_sinks,
+            locked_replies[source].poststat.ia_size);
+        if (ret < 0)
+            goto unlock;
+
+        if (priv->arbiter_count &&
+            AFR_COUNT(healed_sinks, priv->child_count) == 1 &&
+            healed_sinks[ARBITER_BRICK_INDEX]) {
+            is_arbiter_the_only_sink = _gf_true;
+            goto restore_time;
+        }
+        ret = 0;
+    }
+unlock:
+    afr_selfheal_uninodelk(frame, this, fd->inode, this->name, 0, 0, data_lock);
+    if (ret < 0)
+        goto out;
+
+    if (!did_sh)
+        goto out;
+
+    ret = afr_selfheal_data_do(frame, this, fd, source, healed_sinks,
+                               locked_replies);
+    if (ret)
+        goto out;
+restore_time:
+    afr_selfheal_restore_time(frame, this, fd->inode, source, healed_sinks,
+                              locked_replies);
+
+    if (!is_arbiter_the_only_sink && !empty_file) {
+        ret = afr_selfheal_inodelk(frame, this, fd->inode, this->name, 0, 0,
+                                   data_lock);
+        if (ret < priv->child_count) {
+            ret = -ENOTCONN;
+            did_sh = _gf_false;
+            goto skip_undo_pending;
+        }
+    }
+    ret = afr_selfheal_undo_pending(
+        frame, this, fd->inode, sources, sinks, healed_sinks, undid_pending,
+        AFR_DATA_TRANSACTION, locked_replies, data_lock);
+skip_undo_pending:
+    afr_selfheal_uninodelk(frame, this, fd->inode, this->name, 0, 0, data_lock);
 out:
-	if (compat)
-		afr_selfheal_uninodelk (frame, this, fd->inode, this->name,
-					LLONG_MAX - 2, 1, compat_lock);
 
-        if (did_sh)
-                afr_log_selfheal (fd->inode->gfid, this, ret, "data", source,
-                                  healed_sinks);
-        else
-                ret = 1;
+    if (did_sh)
+        afr_log_selfheal(fd->inode->gfid, this, ret, "data", source, sources,
+                         healed_sinks);
+    else
+        ret = 1;
 
-        if (locked_replies)
-                afr_replies_wipe (locked_replies, priv->child_count);
+    if (locked_replies)
+        afr_replies_wipe(locked_replies, priv->child_count);
 
-	return ret;
+    return ret;
 }
 
-
-static fd_t *
-afr_selfheal_data_open (xlator_t *this, inode_t *inode)
+int
+afr_selfheal_data_open_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                           int32_t op_ret, int32_t op_errno, fd_t *fd,
+                           dict_t *xdata)
 {
-	loc_t loc = {0,};
-	int ret = 0;
-	fd_t *fd = NULL;
+    afr_local_t *local = NULL;
+    int i = (long)cookie;
+
+    local = frame->local;
+
+    local->replies[i].valid = 1;
+    local->replies[i].op_ret = op_ret;
+    local->replies[i].op_errno = op_errno;
+
+    syncbarrier_wake(&local->barrier);
 
-	fd = fd_create (inode, 0);
-	if (!fd)
-		return NULL;
+    return 0;
+}
 
-	loc.inode = inode_ref (inode);
-	gf_uuid_copy (loc.gfid, inode->gfid);
+int
+afr_selfheal_data_open(xlator_t *this, inode_t *inode, fd_t **fd)
+{
+    int ret = 0;
+    fd_t *fd_tmp = NULL;
+    loc_t loc = {
+        0,
+    };
+    call_frame_t *frame = NULL;
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
+    int i = 0;
+
+    priv = this->private;
+
+    fd_tmp = fd_create(inode, 0);
+    if (!fd_tmp)
+        return -ENOMEM;
+
+    loc.inode = inode_ref(inode);
+    gf_uuid_copy(loc.gfid, inode->gfid);
+
+    frame = afr_frame_create(this, &ret);
+    if (!frame) {
+        ret = -ret;
+        fd_unref(fd_tmp);
+        goto out;
+    }
+    local = frame->local;
+
+    AFR_ONLIST(local->child_up, frame, afr_selfheal_data_open_cbk, open, &loc,
+               O_RDWR | O_LARGEFILE, fd_tmp, NULL);
+
+    ret = -ENOTCONN;
+    for (i = 0; i < priv->child_count; i++) {
+        if (!local->replies[i].valid)
+            continue;
+
+        if (local->replies[i].op_ret < 0) {
+            ret = -local->replies[i].op_errno;
+            continue;
+        }
 
-	ret = syncop_open (this, &loc, O_RDWR|O_LARGEFILE, fd, NULL, NULL);
-	if (ret) {
-		fd_unref (fd);
-		fd = NULL;
-	} else {
-		fd_bind (fd);
-	}
+        ret = 0;
+        break;
+    }
 
-	loc_wipe (&loc);
+    if (ret < 0) {
+        fd_unref(fd_tmp);
+        goto out;
+    } else {
+        fd_bind(fd_tmp);
+    }
 
-	return fd;
+    *fd = fd_tmp;
+out:
+    loc_wipe(&loc);
+    if (frame)
+        AFR_STACK_DESTROY(frame);
+    return ret;
 }
 
 int
-afr_selfheal_data (call_frame_t *frame, xlator_t *this, inode_t *inode)
+afr_selfheal_data(call_frame_t *frame, xlator_t *this, fd_t *fd)
 {
-	afr_private_t *priv = NULL;
-	unsigned char *locked_on = NULL;
-	int ret = 0;
-	fd_t *fd = NULL;
-
-	priv = this->private;
-
-	fd = afr_selfheal_data_open (this, inode);
-	if (!fd) {
-                gf_log (this->name, GF_LOG_DEBUG, "%s: Failed to open",
-                        uuid_utoa (inode->gfid));
-                return -EIO;
+    afr_private_t *priv = NULL;
+    unsigned char *locked_on = NULL;
+    int ret = 0;
+    inode_t *inode = fd->inode;
+
+    priv = this->private;
+
+    locked_on = alloca0(priv->child_count);
+
+    ret = afr_selfheal_tie_breaker_inodelk(frame, this, inode, priv->sh_domain,
+                                           0, 0, locked_on);
+    {
+        if (ret < priv->child_count) {
+            gf_msg_debug(this->name, 0,
+                         "%s: Skipping "
+                         "self-heal as only %d number of "
+                         "subvolumes could be locked",
+                         uuid_utoa(fd->inode->gfid), ret);
+            /* Either less than two subvols available, or another
+               selfheal (from another server) is in progress. Skip
+               for now in any case there isn't anything to do.
+            */
+            ret = -ENOTCONN;
+            goto unlock;
         }
 
-	locked_on = alloca0 (priv->child_count);
-
-	ret = afr_selfheal_tryinodelk (frame, this, inode, priv->sh_domain, 0, 0,
-				       locked_on);
-	{
-		if (ret < AFR_SH_MIN_PARTICIPANTS) {
-                        gf_log (this->name, GF_LOG_DEBUG, "%s: Skipping "
-                                "self-heal as only %d number of subvolumes "
-                                "could be locked", uuid_utoa (fd->inode->gfid),
-                                ret);
-			/* Either less than two subvols available, or another
-			   selfheal (from another server) is in progress. Skip
-			   for now in any case there isn't anything to do.
-			*/
-			ret = -ENOTCONN;
-			goto unlock;
-		}
-
-		ret = __afr_selfheal_data (frame, this, fd, locked_on);
-	}
+        ret = __afr_selfheal_data(frame, this, fd, locked_on);
+    }
 unlock:
-	afr_selfheal_uninodelk (frame, this, inode, priv->sh_domain, 0, 0, locked_on);
-
-	if (fd)
-		fd_unref (fd);
+    afr_selfheal_uninodelk(frame, this, inode, priv->sh_domain, 0, 0,
+                           locked_on);
 
-	return ret;
+    return ret;
 }
diff --git a/xlators/cluster/afr/src/afr-self-heal-entry.c b/xlators/cluster/afr/src/afr-self-heal-entry.c
index ab0d98bd014..64893f441e3 100644
--- a/xlators/cluster/afr/src/afr-self-heal-entry.c
+++ b/xlators/cluster/afr/src/afr-self-heal-entry.c
@@ -8,754 +8,1269 @@
   cases as published by the Free Software Foundation.
 */
 
-
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
-
 #include "afr.h"
 #include "afr-self-heal.h"
-#include "byte-order.h"
+#include <glusterfs/byte-order.h>
 #include "afr-transaction.h"
+#include "afr-messages.h"
+#include <glusterfs/syncop-utils.h>
+#include <glusterfs/events.h>
 
-/* Max file name length is 255 this filename is of length 256. No file with
- * this name can ever come, entry-lock with this name is going to prevent
- * self-heals from older versions while the granular entry-self-heal is going
- * on in newer version.*/
-#define LONG_FILENAME "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"\
-                      "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"\
-                      "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"\
-                      "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"\
-                      "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"
-
-static int
-afr_selfheal_entry_delete (xlator_t *this, inode_t *dir, const char *name,
-                           inode_t *inode, int child, struct afr_reply *replies)
+int
+afr_selfheal_entry_anon_inode(xlator_t *this, inode_t *dir, const char *name,
+                              inode_t *inode, int child,
+                              struct afr_reply *replies,
+                              gf_boolean_t *anon_inode)
 {
-	afr_private_t *priv = NULL;
-	xlator_t *subvol = NULL;
-	int ret = 0;
-	loc_t loc = {0, };
-	char g[64];
-
-	priv = this->private;
-
-	subvol = priv->children[child];
-
-	loc.parent = inode_ref (dir);
-	gf_uuid_copy (loc.pargfid, dir->gfid);
-	loc.name = name;
-	loc.inode = inode_ref (inode);
-
-	if (replies[child].valid && replies[child].op_ret == 0) {
-		switch (replies[child].poststat.ia_type) {
-		case IA_IFDIR:
-			gf_log (this->name, GF_LOG_WARNING,
-				"expunging dir %s/%s (%s) on %s",
-				uuid_utoa (dir->gfid), name,
-				uuid_utoa_r (replies[child].poststat.ia_gfid, g),
-				subvol->name);
-			ret = syncop_rmdir (subvol, &loc, 1, NULL, NULL);
-			break;
-		default:
-			gf_log (this->name, GF_LOG_WARNING,
-				"expunging file %s/%s (%s) on %s",
-				uuid_utoa (dir->gfid), name,
-				uuid_utoa_r (replies[child].poststat.ia_gfid, g),
-				subvol->name);
-			ret = syncop_unlink (subvol, &loc, NULL, NULL);
-			break;
-		}
-	}
-
-	loc_wipe (&loc);
-
-	return ret;
+    afr_private_t *priv = NULL;
+    afr_local_t *local = NULL;
+    xlator_t *subvol = NULL;
+    int ret = 0;
+    int i = 0;
+    char g[64] = {0};
+    unsigned char *lookup_success = NULL;
+    call_frame_t *frame = NULL;
+    loc_t loc2 = {
+        0,
+    };
+    loc_t loc = {
+        0,
+    };
+
+    priv = this->private;
+    subvol = priv->children[child];
+    lookup_success = alloca0(priv->child_count);
+    uuid_utoa_r(replies[child].poststat.ia_gfid, g);
+    loc.inode = inode_new(inode->table);
+    if (!loc.inode) {
+        ret = -ENOMEM;
+        goto out;
+    }
+
+    if (replies[child].poststat.ia_type == IA_IFDIR) {
+        /* This directory may have sub-directory hierarchy which may need to
+         * be preserved for subsequent heals. So unconditionally move the
+         * directory to anonymous-inode directory*/
+        *anon_inode = _gf_true;
+        goto anon_inode;
+    }
+
+    frame = afr_frame_create(this, &ret);
+    if (!frame) {
+        ret = -ret;
+        goto out;
+    }
+    local = frame->local;
+    gf_uuid_copy(loc.gfid, replies[child].poststat.ia_gfid);
+    AFR_ONLIST(local->child_up, frame, afr_selfheal_discover_cbk, lookup, &loc,
+               NULL);
+    for (i = 0; i < priv->child_count; i++) {
+        if (local->replies[i].op_ret == 0) {
+            lookup_success[i] = 1;
+        } else if (local->replies[i].op_errno != ENOENT &&
+                   local->replies[i].op_errno != ESTALE) {
+            ret = -local->replies[i].op_errno;
+        }
+    }
+
+    if (priv->quorum_count) {
+        if (afr_has_quorum(lookup_success, this, NULL)) {
+            *anon_inode = _gf_true;
+        }
+    } else if (AFR_COUNT(lookup_success, priv->child_count) > 1) {
+        *anon_inode = _gf_true;
+    } else if (ret) {
+        goto out;
+    }
+
+anon_inode:
+    if (!*anon_inode) {
+        ret = 0;
+        goto out;
+    }
+
+    loc.parent = inode_ref(dir);
+    gf_uuid_copy(loc.pargfid, dir->gfid);
+    loc.name = name;
+
+    ret = afr_anon_inode_create(this, child, &loc2.parent);
+    if (ret < 0)
+        goto out;
+
+    loc2.name = g;
+    ret = syncop_rename(subvol, &loc, &loc2, NULL, NULL);
+    if (ret < 0) {
+        gf_msg(this->name, GF_LOG_WARNING, -ret, AFR_MSG_EXPUNGING_FILE_OR_DIR,
+               "Rename to %s dir %s/%s (%s) on %s failed",
+               priv->anon_inode_name, uuid_utoa(dir->gfid), name, g,
+               subvol->name);
+    } else {
+        gf_msg(this->name, GF_LOG_WARNING, 0, AFR_MSG_EXPUNGING_FILE_OR_DIR,
+               "Rename to %s dir %s/%s (%s) on %s successful",
+               priv->anon_inode_name, uuid_utoa(dir->gfid), name, g,
+               subvol->name);
+    }
+
+out:
+    loc_wipe(&loc);
+    loc_wipe(&loc2);
+    if (frame) {
+        AFR_STACK_DESTROY(frame);
+    }
+
+    return ret;
 }
 
+int
+afr_selfheal_entry_delete(xlator_t *this, inode_t *dir, const char *name,
+                          inode_t *inode, int child, struct afr_reply *replies)
+{
+    char g[64] = {0};
+    afr_private_t *priv = NULL;
+    xlator_t *subvol = NULL;
+    int ret = 0;
+    loc_t loc = {
+        0,
+    };
+    gf_boolean_t anon_inode = _gf_false;
+
+    priv = this->private;
+    subvol = priv->children[child];
+
+    if ((!replies[child].valid) || (replies[child].op_ret < 0)) {
+        /*Nothing to do*/
+        ret = 0;
+        goto out;
+    }
+
+    if (priv->use_anon_inode) {
+        ret = afr_selfheal_entry_anon_inode(this, dir, name, inode, child,
+                                            replies, &anon_inode);
+        if (ret < 0 || anon_inode)
+            goto out;
+    }
+
+    loc.parent = inode_ref(dir);
+    loc.inode = inode_new(inode->table);
+    if (!loc.inode) {
+        ret = -ENOMEM;
+        goto out;
+    }
+    loc.name = name;
+    switch (replies[child].poststat.ia_type) {
+        case IA_IFDIR:
+            gf_msg(this->name, GF_LOG_WARNING, 0, AFR_MSG_EXPUNGING_FILE_OR_DIR,
+                   "expunging dir %s/%s (%s) on %s", uuid_utoa(dir->gfid), name,
+                   uuid_utoa_r(replies[child].poststat.ia_gfid, g),
+                   subvol->name);
+            ret = syncop_rmdir(subvol, &loc, 1, NULL, NULL);
+            break;
+        default:
+            gf_msg(this->name, GF_LOG_WARNING, 0, AFR_MSG_EXPUNGING_FILE_OR_DIR,
+                   "expunging file %s/%s (%s) on %s", uuid_utoa(dir->gfid),
+                   name, uuid_utoa_r(replies[child].poststat.ia_gfid, g),
+                   subvol->name);
+            ret = syncop_unlink(subvol, &loc, NULL, NULL);
+            break;
+    }
+
+out:
+    loc_wipe(&loc);
+    return ret;
+}
 
 int
-afr_selfheal_recreate_entry (xlator_t *this, int dst, int source, inode_t *dir,
-                             const char *name, inode_t *inode,
-                             struct afr_reply *replies,
-                             unsigned char *newentry)
+afr_selfheal_recreate_entry(call_frame_t *frame, int dst, int source,
+                            unsigned char *sources, inode_t *dir,
+                            const char *name, inode_t *inode,
+                            struct afr_reply *replies)
 {
-	int ret = 0;
-	loc_t loc = {0,};
-	loc_t srcloc = {0,};
-	afr_private_t *priv = NULL;
-	dict_t *xdata = NULL;
-	struct iatt *iatt = NULL;
-	char *linkname = NULL;
-	mode_t mode = 0;
-	struct iatt newent = {0,};
-	priv = this->private;
-
-	xdata = dict_new();
-	if (!xdata)
-		return -ENOMEM;
-
-	loc.parent = inode_ref (dir);
-	gf_uuid_copy (loc.pargfid, dir->gfid);
-	loc.name = name;
-	loc.inode = inode_ref (inode);
-
-	ret = afr_selfheal_entry_delete (this, dir, name, inode, dst, replies);
-	if (ret)
-		goto out;
-
-	ret = dict_set_static_bin (xdata, "gfid-req",
-				   replies[source].poststat.ia_gfid, 16);
-	if (ret)
-		goto out;
-
-	iatt = &replies[source].poststat;
-
-	srcloc.inode = inode_ref (inode);
-	gf_uuid_copy (srcloc.gfid, iatt->ia_gfid);
-
-	mode = st_mode_from_ia (iatt->ia_prot, iatt->ia_type);
-
-	switch (iatt->ia_type) {
-	case IA_IFDIR:
-		ret = syncop_mkdir (priv->children[dst], &loc, mode, 0,
-                                    xdata, NULL);
-                if (ret == 0)
-                        newentry[dst] = 1;
-		break;
-	case IA_IFLNK:
-		ret = syncop_lookup (priv->children[dst], &srcloc, 0, 0, 0, 0);
-		if (ret == 0) {
-			ret = syncop_link (priv->children[dst], &srcloc, &loc,
-					   NULL, NULL);
-		} else {
-			ret = syncop_readlink (priv->children[source], &srcloc,
-					       &linkname, 4096, NULL, NULL);
-			if (ret <= 0)
-				goto out;
-			ret = syncop_symlink (priv->children[dst], &loc,
-                                              linkname, NULL, xdata, NULL);
-                        if (ret == 0)
-                                newentry[dst] = 1;
-                }
-		break;
-	default:
-		ret = dict_set_int32 (xdata, GLUSTERFS_INTERNAL_FOP_KEY, 1);
-		if (ret)
-			goto out;
-		ret = syncop_mknod (priv->children[dst], &loc, mode,
-				    iatt->ia_rdev, &newent, xdata, NULL);
-		if (ret == 0 && newent.ia_nlink == 1) {
-			/* New entry created. Mark @dst pending on all sources */
-                        newentry[dst] = 1;
-		}
-		break;
-	}
+    int ret = 0;
+    loc_t loc = {
+        0,
+    };
+    loc_t srcloc = {
+        0,
+    };
+    loc_t anonloc = {
+        0,
+    };
+    xlator_t *this = frame->this;
+    afr_private_t *priv = NULL;
+    dict_t *xdata = NULL;
+    struct iatt *iatt = NULL;
+    char *linkname = NULL;
+    mode_t mode = 0;
+    struct iatt newent = {
+        0,
+    };
+    unsigned char *newentry = NULL;
+    char iatt_uuid_str[64] = {0};
+    char dir_uuid_str[64] = {0};
+
+    priv = this->private;
+    iatt = &replies[source].poststat;
+    uuid_utoa_r(iatt->ia_gfid, iatt_uuid_str);
+    if (iatt->ia_type == IA_INVAL || gf_uuid_is_null(iatt->ia_gfid)) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, AFR_MSG_SELF_HEAL_FAILED,
+               "Invalid ia_type (%d) or gfid(%s). source brick=%d, "
+               "pargfid=%s, name=%s",
+               iatt->ia_type, iatt_uuid_str, source,
+               uuid_utoa_r(dir->gfid, dir_uuid_str), name);
+        ret = -EINVAL;
+        goto out;
+    }
+
+    xdata = dict_new();
+    if (!xdata)
+        return -ENOMEM;
+    newentry = alloca0(priv->child_count);
+    loc.parent = inode_ref(dir);
+    gf_uuid_copy(loc.pargfid, dir->gfid);
+    loc.name = name;
+    loc.inode = inode_ref(inode);
+
+    ret = afr_selfheal_entry_delete(this, dir, name, inode, dst, replies);
+    if (ret)
+        goto out;
+
+    ret = dict_set_gfuuid(xdata, "gfid-req", replies[source].poststat.ia_gfid,
+                          true);
+    if (ret)
+        goto out;
+
+    srcloc.inode = inode_ref(inode);
+    gf_uuid_copy(srcloc.gfid, iatt->ia_gfid);
+    ret = syncop_lookup(priv->children[dst], &srcloc, 0, 0, 0, 0);
+    if (ret == -ENOENT || ret == -ESTALE) {
+        newentry[dst] = 1;
+        ret = afr_selfheal_newentry_mark(frame, this, inode, source, replies,
+                                         sources, newentry);
+        if (ret)
+            goto out;
+    } else if (ret == 0 && iatt->ia_type == IA_IFDIR && priv->use_anon_inode) {
+        // Try rename from hidden directory
+        ret = afr_anon_inode_create(this, dst, &anonloc.parent);
+        if (ret < 0)
+            goto out;
+        anonloc.inode = inode_ref(inode);
+        anonloc.name = iatt_uuid_str;
+        ret = syncop_rename(priv->children[dst], &anonloc, &loc, NULL, NULL);
+        if (ret == -ENOENT || ret == -ESTALE)
+            ret = -1; /*This sets 'mismatch' to true*/
+        goto out;
+    }
+
+    mode = st_mode_from_ia(iatt->ia_prot, iatt->ia_type);
+
+    switch (iatt->ia_type) {
+        case IA_IFDIR:
+            ret = syncop_mkdir(priv->children[dst], &loc, mode, 0, xdata, NULL);
+            break;
+        case IA_IFLNK:
+            if (!newentry[dst]) {
+                ret = syncop_link(priv->children[dst], &srcloc, &loc, &newent,
+                                  NULL, NULL);
+            } else {
+                ret = syncop_readlink(priv->children[source], &srcloc,
+                                      &linkname, 4096, NULL, NULL);
+                if (ret <= 0)
+                    goto out;
+                ret = syncop_symlink(priv->children[dst], &loc, linkname, NULL,
+                                     xdata, NULL);
+            }
+            break;
+        default:
+            ret = dict_set_int32_sizen(xdata, GLUSTERFS_INTERNAL_FOP_KEY, 1);
+            if (ret)
+                goto out;
+            ret = syncop_mknod(
+                priv->children[dst], &loc, mode,
+                makedev(ia_major(iatt->ia_rdev), ia_minor(iatt->ia_rdev)),
+                &newent, xdata, NULL);
+            break;
+    }
 
 out:
-	if (xdata)
-		dict_unref (xdata);
-	loc_wipe (&loc);
-	loc_wipe (&srcloc);
-	return ret;
+    if (xdata)
+        dict_unref(xdata);
+    GF_FREE(linkname);
+    loc_wipe(&loc);
+    loc_wipe(&srcloc);
+    loc_wipe(&anonloc);
+    return ret;
 }
 
+static int
+__afr_selfheal_heal_dirent(call_frame_t *frame, xlator_t *this, fd_t *fd,
+                           char *name, inode_t *inode, int source,
+                           unsigned char *sources, unsigned char *healed_sinks,
+                           unsigned char *locked_on, struct afr_reply *replies)
+{
+    int ret = 0;
+    afr_private_t *priv = NULL;
+    int i = 0;
+
+    priv = this->private;
+
+    if (!replies[source].valid)
+        return -EIO;
+
+    /* Skip healing this entry if the last lookup on it failed for reasons
+     * other than ENOENT.
+     */
+    if ((replies[source].op_ret < 0) && (replies[source].op_errno != ENOENT))
+        return -replies[source].op_errno;
+
+    if (replies[source].op_ret == 0) {
+        ret = afr_lookup_and_heal_gfid(this, fd->inode, name, inode, replies,
+                                       source, sources,
+                                       &replies[source].poststat.ia_gfid, NULL);
+        if (ret)
+            return ret;
+    }
+
+    for (i = 0; i < priv->child_count; i++) {
+        if (!healed_sinks[i])
+            continue;
+        if (replies[source].op_ret == -1 &&
+            replies[source].op_errno == ENOENT) {
+            ret = afr_selfheal_entry_delete(this, fd->inode, name, inode, i,
+                                            replies);
+        } else {
+            if (!gf_uuid_compare(replies[i].poststat.ia_gfid,
+                                 replies[source].poststat.ia_gfid))
+                continue;
+
+            ret = afr_selfheal_recreate_entry(frame, i, source, sources,
+                                              fd->inode, name, inode, replies);
+        }
+        if (ret < 0)
+            break;
+    }
+
+    return ret;
+}
 
 static int
-__afr_selfheal_heal_dirent (call_frame_t *frame, xlator_t *this, fd_t *fd,
-			    char *name, inode_t *inode, int source,
-			    unsigned char *sources, unsigned char *healed_sinks,
-			    unsigned char *locked_on, struct afr_reply *replies)
+afr_selfheal_detect_gfid_and_type_mismatch(xlator_t *this,
+                                           struct afr_reply *replies,
+                                           inode_t *inode, uuid_t pargfid,
+                                           char *bname, int src_idx,
+                                           unsigned char *locked_on, int *src)
 {
-	int ret = 0;
-	afr_private_t *priv = NULL;
-	int i = 0;
-	unsigned char *newentry = NULL;
-
-	priv = this->private;
-
-	newentry = alloca0 (priv->child_count);
-
-	if (!replies[source].valid)
-		return -EIO;
-
-	for (i = 0; i < priv->child_count; i++) {
-		if (!healed_sinks[i])
-			continue;
-		if (replies[source].op_ret == -1 &&
-		    replies[source].op_errno == ENOENT) {
-			ret = afr_selfheal_entry_delete (this, fd->inode, name,
-                                                         inode, i, replies);
-		} else {
-			if (!gf_uuid_compare (replies[i].poststat.ia_gfid,
-					   replies[source].poststat.ia_gfid))
-				continue;
-
-			ret = afr_selfheal_recreate_entry (this, i, source,
-							   fd->inode, name, inode,
-							   replies, newentry);
-		}
-		if (ret < 0)
-			break;
-	}
-
-	if (AFR_COUNT (newentry, priv->child_count))
-		afr_selfheal_newentry_mark (frame, this, inode, source, replies,
-					    sources, newentry);
-	return ret;
+    int i = 0;
+    int ret = -1;
+    afr_private_t *priv = NULL;
+    void *gfid = NULL;
+    ia_type_t ia_type = IA_INVAL;
+
+    priv = this->private;
+    gfid = &replies[src_idx].poststat.ia_gfid;
+    ia_type = replies[src_idx].poststat.ia_type;
+
+    for (i = 0; i < priv->child_count; i++) {
+        if (i == src_idx)
+            continue;
+
+        if (!replies[i].valid)
+            continue;
+
+        if (replies[i].op_ret != 0)
+            continue;
+
+        if (gf_uuid_is_null(replies[i].poststat.ia_gfid))
+            continue;
+
+        if (replies[i].poststat.ia_type == IA_INVAL)
+            continue;
+
+        if (ia_type == IA_INVAL || gf_uuid_is_null(gfid)) {
+            src_idx = i;
+            ia_type = replies[src_idx].poststat.ia_type;
+            gfid = &replies[src_idx].poststat.ia_gfid;
+            continue;
+        }
+
+        if (gf_uuid_compare(gfid, replies[i].poststat.ia_gfid) &&
+            (ia_type == replies[i].poststat.ia_type)) {
+            ret = afr_gfid_split_brain_source(this, replies, inode, pargfid,
+                                              bname, src_idx, i, locked_on, src,
+                                              NULL);
+            if (ret)
+                gf_msg(this->name, GF_LOG_ERROR, 0, AFR_MSG_SPLIT_BRAIN,
+                       "Skipping conservative merge on the "
+                       "file.");
+            return ret;
+        }
+
+        if (ia_type != replies[i].poststat.ia_type) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, AFR_MSG_SPLIT_BRAIN,
+                   "Type mismatch detected "
+                   "for <gfid:%s>/%s>, %s on %s and %s on %s. "
+                   "Skipping conservative merge on the file.",
+                   uuid_utoa(pargfid), bname,
+                   gf_inode_type_to_str(replies[i].poststat.ia_type),
+                   priv->children[i]->name,
+                   gf_inode_type_to_str(replies[src_idx].poststat.ia_type),
+                   priv->children[src_idx]->name);
+            gf_event(EVENT_AFR_SPLIT_BRAIN,
+                     "client-pid=%d;"
+                     "subvol=%s;type=file;"
+                     "file=<gfid:%s>/%s>;count=2;child-%d=%s;type-"
+                     "%d=%s;child-%d=%s;type-%d=%s",
+                     this->ctx->cmd_args.client_pid, this->name,
+                     uuid_utoa(pargfid), bname, i, priv->children[i]->name, i,
+                     gf_inode_type_to_str(replies[i].poststat.ia_type), src_idx,
+                     priv->children[src_idx]->name, src_idx,
+                     gf_inode_type_to_str(replies[src_idx].poststat.ia_type));
+            return -1;
+        }
+    }
+
+    return 0;
 }
 
 static int
-afr_selfheal_detect_gfid_and_type_mismatch (xlator_t *this,
-                                            struct afr_reply *replies,
-                                            uuid_t pargfid, char *bname,
-                                            int src_idx)
+__afr_selfheal_merge_dirent(call_frame_t *frame, xlator_t *this, fd_t *fd,
+                            char *name, inode_t *inode, unsigned char *sources,
+                            unsigned char *healed_sinks,
+                            unsigned char *locked_on, struct afr_reply *replies)
 {
-        int             i      = 0;
-        char            g1[64] = {0,};
-        char            g2[64] = {0,};
-        afr_private_t  *priv   = NULL;
+    int ret = 0;
+    int i = 0;
+    int source = -1;
+    int src = -1;
+    afr_private_t *priv = NULL;
+
+    priv = this->private;
+
+    for (i = 0; i < priv->child_count; i++) {
+        if (replies[i].valid && replies[i].op_ret == 0) {
+            source = i;
+            break;
+        }
+    }
 
-        priv = this->private;
+    if (source == -1) {
+        /* entry got deleted in the mean time? */
+        return 0;
+    }
 
+    /* Set all the sources as 1, otheriwse newentry_mark won't be set */
+    for (i = 0; i < priv->child_count; i++) {
+        if (replies[i].valid && replies[i].op_ret == 0) {
+            sources[i] = 1;
+        }
+    }
+
+    ret = afr_lookup_and_heal_gfid(this, fd->inode, name, inode, replies,
+                                   source, sources,
+                                   &replies[source].poststat.ia_gfid, NULL);
+    if (ret)
+        return ret;
+
+    /* In case of type mismatch / unable to resolve gfid mismatch on the
+     * entry, return -1.*/
+    ret = afr_selfheal_detect_gfid_and_type_mismatch(
+        this, replies, inode, fd->inode->gfid, name, source, locked_on, &src);
+
+    if (ret < 0)
+        return ret;
+    if (src != -1) {
+        source = src;
         for (i = 0; i < priv->child_count; i++) {
-                if (i == src_idx)
-                        continue;
-
-                if (!replies[i].valid)
-                        continue;
-
-                if (replies[i].op_ret != 0)
-                        continue;
-
-                if (gf_uuid_compare (replies[src_idx].poststat.ia_gfid,
-                                  replies[i].poststat.ia_gfid)) {
-                        gf_log (this->name, GF_LOG_ERROR, "Gfid mismatch "
-                                "detected for <%s/%s>, %s on %s and %s on %s. "
-                                "Skipping conservative merge on the file.",
-                                uuid_utoa (pargfid), bname,
-                                uuid_utoa_r (replies[i].poststat.ia_gfid, g1),
-                                priv->children[i]->name,
-                                uuid_utoa_r (replies[src_idx].poststat.ia_gfid,
-                                g2), priv->children[src_idx]->name);
-                        return -1;
-                }
-
-                if ((replies[src_idx].poststat.ia_type) !=
-                    (replies[i].poststat.ia_type)) {
-                        gf_log (this->name, GF_LOG_ERROR, "Type mismatch "
-                                "detected for <%s/%s>, %d on %s and %d on %s. "
-                                "Skipping conservative merge on the file.",
-                                uuid_utoa (pargfid), bname,
-                                replies[i].poststat.ia_type,
-                                priv->children[i]->name,
-                                replies[src_idx].poststat.ia_type,
-                                priv->children[src_idx]->name);
-                        return -1;
-                }
+            if (i != src && replies[i].valid &&
+                gf_uuid_compare(replies[src].poststat.ia_gfid,
+                                replies[i].poststat.ia_gfid)) {
+                sources[i] = 0;
+            }
+        }
+    }
+
+    for (i = 0; i < priv->child_count; i++) {
+        if (i == source || !healed_sinks[i])
+            continue;
+
+        if (src != -1) {
+            if (!gf_uuid_compare(replies[src].poststat.ia_gfid,
+                                 replies[i].poststat.ia_gfid))
+                continue;
+        } else if (replies[i].op_errno != ENOENT) {
+            continue;
         }
 
-        return 0;
+        ret |= afr_selfheal_recreate_entry(frame, i, source, sources, fd->inode,
+                                           name, inode, replies);
+    }
+
+    return ret;
 }
 
 static int
-__afr_selfheal_merge_dirent (call_frame_t *frame, xlator_t *this, fd_t *fd,
-			     char *name, inode_t *inode, unsigned char *sources,
-			     unsigned char *healed_sinks, unsigned char *locked_on,
-			     struct afr_reply *replies)
+__afr_selfheal_entry_dirent(call_frame_t *frame, xlator_t *this, fd_t *fd,
+                            char *name, inode_t *inode, int source,
+                            unsigned char *sources, unsigned char *healed_sinks,
+                            unsigned char *locked_on, struct afr_reply *replies)
 {
-        int             ret       = 0;
-        int             i         = 0;
-        int             source    = -1;
-        unsigned char  *newentry  = NULL;
-        afr_private_t  *priv      = NULL;
+    int ret = -1;
+
+    if (source < 0)
+        ret = __afr_selfheal_merge_dirent(frame, this, fd, name, inode, sources,
+                                          healed_sinks, locked_on, replies);
+    else
+        ret = __afr_selfheal_heal_dirent(frame, this, fd, name, inode, source,
+                                         sources, healed_sinks, locked_on,
+                                         replies);
+    return ret;
+}
 
-	priv = this->private;
+static gf_boolean_t
+is_full_heal_marker_present(xlator_t *this, dict_t *xdata, int idx)
+{
+    int i = 0;
+    int pending[3] = {
+        0,
+    };
+    void *pending_raw = NULL;
+    afr_private_t *priv = NULL;
+
+    priv = this->private;
+
+    if (!xdata)
+        return _gf_false;
+
+    /* Iterate over each of the priv->pending_keys[] elements and then
+     * see if any of them have data segment non-zero. If they do, return
+     * true. Else return false.
+     */
+    for (i = 0; i < priv->child_count; i++) {
+        if (dict_get_ptr(xdata, priv->pending_key[i], &pending_raw))
+            continue;
+
+        if (!pending_raw)
+            continue;
+
+        memcpy(pending, pending_raw, sizeof(pending));
+        if (ntoh32(pending[idx]))
+            return _gf_true;
+    }
+
+    return _gf_false;
+}
 
-	newentry = alloca0 (priv->child_count);
+static gf_boolean_t
+afr_need_full_heal(xlator_t *this, struct afr_reply *replies, int source,
+                   unsigned char *healed_sinks, afr_transaction_type type)
+{
+    int i = 0;
+    int idx = 0;
+    afr_private_t *priv = NULL;
 
-	for (i = 0; i < priv->child_count; i++) {
-		if (replies[i].valid && replies[i].op_ret == 0) {
-			source = i;
-			break;
-		}
-	}
+    priv = this->private;
 
-	if (source == -1) {
-		/* entry got deleted in the mean time? */
-		return 0;
-	}
+    if (!priv->esh_granular)
+        return _gf_true;
 
-        /* In case of a gfid or type mismatch on the entry, return -1.*/
-        ret = afr_selfheal_detect_gfid_and_type_mismatch (this, replies,
-                                                          fd->inode->gfid,
-                                                          name, source);
+    if (type != AFR_ENTRY_TRANSACTION)
+        return _gf_true;
 
-        if (ret < 0)
-                return ret;
+    priv = this->private;
+    idx = afr_index_for_transaction_type(AFR_DATA_TRANSACTION);
 
-	for (i = 0; i < priv->child_count; i++) {
-		if (i == source || !healed_sinks[i])
-			continue;
+    /* If there is a clear source, check whether the full-heal-indicator
+     * is present in its xdata. Otherwise, we need to examine all the
+     * participating bricks and then figure if *even* one of them has a
+     * full-heal-indicator.
+     */
 
-		if (replies[i].op_errno != ENOENT)
-			continue;
+    if (source != -1) {
+        if (is_full_heal_marker_present(this, replies[source].xdata, idx))
+            return _gf_true;
+    }
 
-		ret = afr_selfheal_recreate_entry (this, i, source, fd->inode,
-                                                   name, inode, replies,
-                                                   newentry);
-	}
+    /* else ..*/
 
-	if (AFR_COUNT (newentry, priv->child_count))
-		afr_selfheal_newentry_mark (frame, this, inode, source, replies,
-					    sources, newentry);
-	return ret;
-}
+    for (i = 0; i < priv->child_count; i++) {
+        if (!healed_sinks[i])
+            continue;
 
+        if (is_full_heal_marker_present(this, replies[i].xdata, idx))
+            return _gf_true;
+    }
 
-static int
-__afr_selfheal_entry_dirent (call_frame_t *frame, xlator_t *this, fd_t *fd,
-                             char *name, inode_t *inode, int source,
-                             unsigned char *sources, unsigned char *healed_sinks,
-                             unsigned char *locked_on,
-                             struct afr_reply *replies)
-{
-	int ret = -1;
-
-	if (source < 0)
-		ret = __afr_selfheal_merge_dirent (frame, this, fd, name, inode,
-						   sources, healed_sinks,
-						   locked_on, replies);
-	else
-		ret = __afr_selfheal_heal_dirent (frame, this, fd, name, inode,
-						  source, sources, healed_sinks,
-						  locked_on, replies);
-	return ret;
+    return _gf_false;
 }
 
 static int
-__afr_selfheal_entry_finalize_source (xlator_t *this, unsigned char *sources,
-				      unsigned char *healed_sinks,
-                                      unsigned char *locked_on,
-                                      struct afr_reply *replies,
-                                      uint64_t *witness)
+__afr_selfheal_entry_finalize_source(xlator_t *this, unsigned char *sources,
+                                     unsigned char *healed_sinks,
+                                     unsigned char *locked_on,
+                                     struct afr_reply *replies,
+                                     uint64_t *witness)
 {
-	int i = 0;
-	afr_private_t *priv = NULL;
-	int source = -1;
-	int sources_count = 0;
+    afr_private_t *priv = NULL;
+    int source = -1;
+    int sources_count = 0;
+    int i = 0;
 
-	priv = this->private;
+    priv = this->private;
 
-	sources_count = AFR_COUNT (sources, priv->child_count);
+    sources_count = AFR_COUNT(sources, priv->child_count);
 
-	if ((AFR_CMP (locked_on, healed_sinks, priv->child_count) == 0)
-            || !sources_count || afr_does_witness_exist (this, witness)) {
+    if ((AFR_CMP(locked_on, healed_sinks, priv->child_count) == 0) ||
+        !sources_count || afr_does_witness_exist(this, witness)) {
+        memset(sources, 0, sizeof(*sources) * priv->child_count);
+        afr_mark_active_sinks(this, sources, locked_on, healed_sinks);
+        return -1;
+    }
 
-                memset (sources, 0, sizeof (*sources) * priv->child_count);
-                afr_mark_active_sinks (this, sources, locked_on, healed_sinks);
-		return -1;
-	}
+    source = afr_choose_source_by_policy(priv, sources, AFR_ENTRY_TRANSACTION);
 
-	for (i = 0; i < priv->child_count; i++) {
-		if (sources[i]) {
-			source = i;
-			break;
-		}
-	}
+    /*If the selected source does not blame any other brick, then mark
+     * everything as sink to trigger conservative merge.
+     */
+    if (source != -1 && !AFR_COUNT(healed_sinks, priv->child_count)) {
+        for (i = 0; i < priv->child_count; i++) {
+            if (locked_on[i]) {
+                sources[i] = 0;
+                healed_sinks[i] = 1;
+            }
+        }
+        return -1;
+    }
 
-	return source;
+    return source;
 }
 
 int
-__afr_selfheal_entry_prepare (call_frame_t *frame, xlator_t *this,
-                              inode_t *inode, unsigned char *locked_on,
-                              unsigned char *sources, unsigned char *sinks,
-                              unsigned char *healed_sinks,
-			      struct afr_reply *replies, int *source_p)
+__afr_selfheal_entry_prepare(call_frame_t *frame, xlator_t *this,
+                             inode_t *inode, unsigned char *locked_on,
+                             unsigned char *sources, unsigned char *sinks,
+                             unsigned char *healed_sinks,
+                             struct afr_reply *replies, int *source_p,
+                             unsigned char *pflag)
 {
-	int ret = -1;
-	int source = -1;
-	afr_private_t *priv = NULL;
-        uint64_t *witness = NULL;
-
-	priv = this->private;
-
-	ret = afr_selfheal_unlocked_discover (frame, inode, inode->gfid,
-					      replies);
-        if (ret)
-                return ret;
-
-        witness = alloca0 (sizeof (*witness) * priv->child_count);
-	ret = afr_selfheal_find_direction (frame, this, replies,
-					   AFR_ENTRY_TRANSACTION,
-					   locked_on, sources, sinks, witness);
-	if (ret)
-		return ret;
-
-        /* Initialize the healed_sinks[] array optimistically to
-           the intersection of to-be-healed (i.e sinks[]) and
-           the list of servers which are up (i.e locked_on[]).
-
-           As we encounter failures in the healing process, we
-           will unmark the respective servers in the healed_sinks[]
-           array.
-        */
-        AFR_INTERSECT (healed_sinks, sinks, locked_on, priv->child_count);
-
-	source = __afr_selfheal_entry_finalize_source (this, sources,
-                                                       healed_sinks,
-						       locked_on, replies,
-                                                       witness);
-
-	if (source < 0) {
-		/* If source is < 0 (typically split-brain), we perform a
-		   conservative merge of entries rather than erroring out */
-	}
-	*source_p = source;
-
-	return ret;
+    int ret = -1;
+    int source = -1;
+    afr_private_t *priv = NULL;
+    uint64_t *witness = NULL;
+
+    priv = this->private;
+
+    ret = afr_selfheal_unlocked_discover(frame, inode, inode->gfid, replies);
+    if (ret)
+        return ret;
+
+    witness = alloca0(sizeof(*witness) * priv->child_count);
+    ret = afr_selfheal_find_direction(frame, this, replies,
+                                      AFR_ENTRY_TRANSACTION, locked_on, sources,
+                                      sinks, witness, pflag);
+    if (ret)
+        return ret;
+
+    /* Initialize the healed_sinks[] array optimistically to
+       the intersection of to-be-healed (i.e sinks[]) and
+       the list of servers which are up (i.e locked_on[]).
+
+       As we encounter failures in the healing process, we
+       will unmark the respective servers in the healed_sinks[]
+       array.
+    */
+    AFR_INTERSECT(healed_sinks, sinks, locked_on, priv->child_count);
+
+    source = __afr_selfheal_entry_finalize_source(this, sources, healed_sinks,
+                                                  locked_on, replies, witness);
+
+    if (source < 0) {
+        /* If source is < 0 (typically split-brain), we perform a
+           conservative merge of entries rather than erroring out */
+    }
+    *source_p = source;
+
+    return ret;
 }
 
 static int
-afr_selfheal_entry_dirent (call_frame_t *frame, xlator_t *this,
-                           fd_t *fd, char *name)
+afr_selfheal_entry_dirent(call_frame_t *frame, xlator_t *this, fd_t *fd,
+                          char *name, inode_t *parent_idx_inode,
+                          xlator_t *subvol, gf_boolean_t full_crawl)
 {
-        int                ret          = 0;
-        int                source       = -1;
-        unsigned char     *locked_on    = NULL;
-        unsigned char     *sources      = NULL;
-        unsigned char     *sinks        = NULL;
-        unsigned char     *healed_sinks = NULL;
-        inode_t           *inode        = NULL;
-        struct afr_reply  *replies      = NULL;
-        struct afr_reply  *par_replies  = NULL;
-        afr_private_t     *priv         = NULL;
-
-	priv = this->private;
-
-        sources = alloca0 (priv->child_count);
-        sinks   = alloca0 (priv->child_count);
-        healed_sinks = alloca0 (priv->child_count);
-	locked_on = alloca0 (priv->child_count);
-
-	replies = alloca0 (priv->child_count * sizeof(*replies));
-	par_replies = alloca0 (priv->child_count * sizeof(*par_replies));
-
-        ret = afr_selfheal_entrylk (frame, this, fd->inode, this->name, NULL,
-                                    locked_on);
-	{
-		if (ret < AFR_SH_MIN_PARTICIPANTS) {
-                        gf_log (this->name, GF_LOG_DEBUG, "%s: Skipping "
-                                "entry self-heal as only %d sub-volumes could "
-                                "be locked in %s domain",
-                                uuid_utoa (fd->inode->gfid), ret, this->name);
-			ret = -ENOTCONN;
-			goto unlock;
-		}
-
-                ret = __afr_selfheal_entry_prepare (frame, this, fd->inode,
-                                                    locked_on,
-                                                    sources, sinks,
-                                                    healed_sinks, par_replies,
-                                                    &source);
-                if (ret < 0)
-                        goto unlock;
-
-		inode = afr_selfheal_unlocked_lookup_on (frame, fd->inode, name,
-							 replies, locked_on,
-                                                         NULL);
-		if (!inode) {
-			ret = -ENOMEM;
-			goto unlock;
-		}
-
-		ret = __afr_selfheal_entry_dirent (frame, this, fd, name, inode,
-						   source, sources, healed_sinks,
-						   locked_on, replies);
-	}
+    int ret = 0;
+    int source = -1;
+    unsigned char *locked_on = NULL;
+    unsigned char *sources = NULL;
+    unsigned char *sinks = NULL;
+    unsigned char *healed_sinks = NULL;
+    inode_t *inode = NULL;
+    struct afr_reply *replies = NULL;
+    struct afr_reply *par_replies = NULL;
+    afr_private_t *priv = NULL;
+    dict_t *xattr = NULL;
+
+    priv = this->private;
+
+    if (afr_is_private_directory(priv, fd->inode->gfid, name,
+                                 GF_CLIENT_PID_SELF_HEALD)) {
+        return 0;
+    }
+
+    xattr = dict_new();
+    if (!xattr)
+        return -ENOMEM;
+    ret = dict_set_int32_sizen(xattr, GF_GFIDLESS_LOOKUP, 1);
+    if (ret) {
+        dict_unref(xattr);
+        return -1;
+    }
+
+    sources = alloca0(priv->child_count);
+    sinks = alloca0(priv->child_count);
+    healed_sinks = alloca0(priv->child_count);
+    locked_on = alloca0(priv->child_count);
+
+    replies = alloca0(priv->child_count * sizeof(*replies));
+    par_replies = alloca0(priv->child_count * sizeof(*par_replies));
+
+    ret = afr_selfheal_entrylk(frame, this, fd->inode, this->name, NULL,
+                               locked_on);
+    {
+        if (ret < priv->child_count) {
+            gf_msg_debug(this->name, 0,
+                         "%s: Skipping "
+                         "entry self-heal as only %d sub-volumes "
+                         " could be locked in %s domain",
+                         uuid_utoa(fd->inode->gfid), ret, this->name);
+            ret = -ENOTCONN;
+            goto unlock;
+        }
+
+        ret = __afr_selfheal_entry_prepare(frame, this, fd->inode, locked_on,
+                                           sources, sinks, healed_sinks,
+                                           par_replies, &source, NULL);
+        if (ret < 0)
+            goto unlock;
+
+        inode = afr_selfheal_unlocked_lookup_on(frame, fd->inode, name, replies,
+                                                locked_on, xattr);
+        if (!inode) {
+            ret = -ENOMEM;
+            goto unlock;
+        }
+
+        ret = __afr_selfheal_entry_dirent(frame, this, fd, name, inode, source,
+                                          sources, healed_sinks, locked_on,
+                                          replies);
+
+        if ((ret == 0) && (priv->esh_granular) && parent_idx_inode) {
+            ret = afr_shd_entry_purge(subvol, parent_idx_inode, name,
+                                      inode->ia_type);
+            /* Why is ret force-set to 0? We do not care about
+             * index purge failing for full heal as it is quite
+             * possible during replace-brick that not all files
+             * and directories have their name indices present in
+             * entry-changes/.
+             */
+            ret = 0;
+        }
+    }
+
 unlock:
-        afr_selfheal_unentrylk (frame, this, fd->inode, this->name, NULL,
-                                locked_on);
-	if (inode)
-		inode_unref (inode);
-        if (replies)
-                afr_replies_wipe (replies, priv->child_count);
-        if (par_replies)
-                afr_replies_wipe (par_replies, priv->child_count);
-
-	return ret;
+    afr_selfheal_unentrylk(frame, this, fd->inode, this->name, NULL, locked_on,
+                           NULL);
+    if (inode)
+        inode_unref(inode);
+    if (replies)
+        afr_replies_wipe(replies, priv->child_count);
+    if (par_replies)
+        afr_replies_wipe(par_replies, priv->child_count);
+    if (xattr)
+        dict_unref(xattr);
+
+    return ret;
 }
 
+static inode_t *
+afr_shd_entry_changes_index_inode(xlator_t *this, xlator_t *subvol,
+                                  uuid_t pargfid)
+{
+    int ret = -1;
+    void *index_gfid = NULL;
+    loc_t rootloc = {
+        0,
+    };
+    loc_t loc = {
+        0,
+    };
+    dict_t *xattr = NULL;
+    inode_t *inode = NULL;
+    struct iatt iatt = {
+        0,
+    };
+
+    rootloc.inode = inode_ref(this->itable->root);
+    gf_uuid_copy(rootloc.gfid, rootloc.inode->gfid);
+
+    ret = syncop_getxattr(subvol, &rootloc, &xattr,
+                          GF_XATTROP_ENTRY_CHANGES_GFID, NULL, NULL);
+    if (ret || !xattr) {
+        errno = -ret;
+        goto out;
+    }
+
+    ret = dict_get_ptr(xattr, GF_XATTROP_ENTRY_CHANGES_GFID, &index_gfid);
+    if (ret) {
+        errno = EINVAL;
+        goto out;
+    }
+
+    loc.inode = inode_new(this->itable);
+    if (!loc.inode) {
+        errno = ENOMEM;
+        goto out;
+    }
+
+    gf_uuid_copy(loc.pargfid, index_gfid);
+    loc.name = gf_strdup(uuid_utoa(pargfid));
+
+    ret = syncop_lookup(subvol, &loc, &iatt, NULL, NULL, NULL);
+    if (ret < 0) {
+        errno = -ret;
+        goto out;
+    }
+
+    inode = inode_link(loc.inode, NULL, NULL, &iatt);
+
+out:
+    if (xattr)
+        dict_unref(xattr);
+    loc_wipe(&rootloc);
+    GF_FREE((char *)loc.name);
+    loc_wipe(&loc);
+
+    return inode;
+}
+
+static int
+afr_selfheal_entry_do_subvol(call_frame_t *frame, xlator_t *this, fd_t *fd,
+                             int child)
+{
+    int ret = 0;
+    gf_dirent_t entries;
+    gf_dirent_t *entry = NULL;
+    off_t offset = 0;
+    call_frame_t *iter_frame = NULL;
+    xlator_t *subvol = NULL;
+    afr_private_t *priv = NULL;
+    gf_boolean_t mismatch = _gf_false;
+    afr_local_t *local = NULL;
+    loc_t loc = {
+        0,
+    };
+
+    priv = this->private;
+    subvol = priv->children[child];
+
+    INIT_LIST_HEAD(&entries.list);
+
+    local = frame->local;
+
+    iter_frame = afr_copy_frame(frame);
+    if (!iter_frame)
+        return -ENOMEM;
+
+    loc.inode = afr_shd_entry_changes_index_inode(this, subvol,
+                                                  fd->inode->gfid);
+
+    while ((ret = syncop_readdir(subvol, fd, 131072, offset, &entries, NULL,
+                                 NULL))) {
+        if (ret > 0)
+            ret = 0;
+        list_for_each_entry(entry, &entries.list, list)
+        {
+            offset = entry->d_off;
+
+            if (!strcmp(entry->d_name, ".") || !strcmp(entry->d_name, ".."))
+                continue;
+
+            ret = afr_selfheal_entry_dirent(iter_frame, this, fd, entry->d_name,
+                                            loc.inode, subvol,
+                                            local->need_full_crawl);
+            AFR_STACK_RESET(iter_frame);
+            if (iter_frame->local == NULL) {
+                ret = -ENOTCONN;
+                break;
+            }
+
+            if (ret == -1) {
+                /* gfid or type mismatch. */
+                mismatch = _gf_true;
+                ret = 0;
+            }
+            if (ret)
+                break;
+        }
+
+        gf_dirent_free(&entries);
+        if (ret)
+            break;
+    }
+
+    loc_wipe(&loc);
+
+    AFR_STACK_DESTROY(iter_frame);
+    if (mismatch == _gf_true)
+        /* undo pending will be skipped */
+        ret = -1;
+    return ret;
+}
 
 static int
-afr_selfheal_entry_do_subvol (call_frame_t *frame, xlator_t *this,
-                              fd_t *fd, int child)
+afr_selfheal_entry_granular_dirent(xlator_t *subvol, gf_dirent_t *entry,
+                                   loc_t *parent, void *data)
 {
-	int ret = 0;
-	gf_dirent_t entries;
-	gf_dirent_t *entry = NULL;
-	off_t offset = 0;
-	call_frame_t *iter_frame = NULL;
-	xlator_t *subvol = NULL;
-	afr_private_t *priv = NULL;
-
-	priv = this->private;
-	subvol = priv->children[child];
-
-	INIT_LIST_HEAD (&entries.list);
-
-	iter_frame = afr_copy_frame (frame);
-	if (!iter_frame)
-		return -ENOMEM;
-
-	while ((ret = syncop_readdir (subvol, fd, 131072, offset, &entries,
-                                      NULL, NULL))) {
-		if (ret > 0)
-			ret = 0;
-		list_for_each_entry (entry, &entries.list, list) {
-			offset = entry->d_off;
-
-			if (!strcmp (entry->d_name, ".") ||
-			    !strcmp (entry->d_name, ".."))
-				continue;
-
-			if (__is_root_gfid (fd->inode->gfid) &&
-			    !strcmp (entry->d_name, GF_REPLICATE_TRASH_DIR))
-				continue;
-
-			ret = afr_selfheal_entry_dirent (iter_frame, this, fd,
-                                                         entry->d_name);
-			AFR_STACK_RESET (iter_frame);
-
-			if (ret)
-				break;
-		}
-
-		gf_dirent_free (&entries);
-		if (ret)
-			break;
-	}
-
-	AFR_STACK_DESTROY (iter_frame);
-	return ret;
+    int ret = 0;
+    loc_t loc = {
+        0,
+    };
+    struct iatt iatt = {
+        0,
+    };
+    afr_granular_esh_args_t *args = data;
+
+    /* Look up the actual inode associated with entry. If the lookup returns
+     * ESTALE or ENOENT, then it means we have a stale index. Remove it.
+     * This is analogous to the check in afr_shd_index_heal() except that
+     * here it is achieved through LOOKUP and in afr_shd_index_heal() through
+     * a GETXATTR.
+     */
+
+    loc.inode = inode_new(args->xl->itable);
+    loc.parent = inode_ref(args->heal_fd->inode);
+    gf_uuid_copy(loc.pargfid, loc.parent->gfid);
+    loc.name = entry->d_name;
+
+    ret = syncop_lookup(args->xl, &loc, &iatt, NULL, NULL, NULL);
+    if ((ret == -ENOENT) || (ret == -ESTALE)) {
+        /* The name indices under the pgfid index dir are guaranteed
+         * to be regular files. Hence the hardcoding.
+         */
+        afr_shd_entry_purge(subvol, parent->inode, entry->d_name, IA_IFREG);
+        ret = 0;
+        goto out;
+    }
+    /* TBD: afr_shd_zero_xattrop? */
+
+    ret = afr_selfheal_entry_dirent(args->frame, args->xl, args->heal_fd,
+                                    entry->d_name, parent->inode, subvol,
+                                    _gf_false);
+    AFR_STACK_RESET(args->frame);
+    if (args->frame->local == NULL)
+        ret = -ENOTCONN;
+
+    if (ret == -1)
+        args->mismatch = _gf_true;
+
+out:
+    loc_wipe(&loc);
+    return 0;
 }
 
 static int
-afr_selfheal_entry_do (call_frame_t *frame, xlator_t *this, fd_t *fd,
-		       int source, unsigned char *sources,
-		       unsigned char *healed_sinks)
+afr_selfheal_entry_granular(call_frame_t *frame, xlator_t *this, fd_t *fd,
+                            int subvol_idx, gf_boolean_t is_src)
 {
-	int i = 0;
-	afr_private_t *priv = NULL;
-	int ret = 0;
-
-	priv = this->private;
-
-	gf_log (this->name, GF_LOG_INFO, "performing entry selfheal on %s",
-		uuid_utoa (fd->inode->gfid));
-
-	for (i = 0; i < priv->child_count; i++) {
-		if (i != source && !healed_sinks[i])
-			continue;
-		ret = afr_selfheal_entry_do_subvol (frame, this, fd, i);
-		if (ret)
-			break;
-	}
-	return ret;
+    int ret = 0;
+    loc_t loc = {
+        0,
+    };
+    xlator_t *subvol = NULL;
+    afr_private_t *priv = NULL;
+    afr_granular_esh_args_t args = {
+        0,
+    };
+
+    priv = this->private;
+    subvol = priv->children[subvol_idx];
+
+    args.frame = afr_copy_frame(frame);
+    if (!args.frame)
+        goto out;
+    args.xl = this;
+    /* args.heal_fd represents the fd associated with the original directory
+     * on which entry heal is being attempted.
+     */
+    args.heal_fd = fd;
+
+    /* @subvol here represents the subvolume of AFR where
+     * indices/entry-changes/<pargfid> will be processed
+     */
+    loc.inode = afr_shd_entry_changes_index_inode(this, subvol,
+                                                  fd->inode->gfid);
+    if (!loc.inode) {
+        /* If granular heal failed on the sink (as it might sometimes
+         * because it is the src that would mostly contain the granular
+         * changelogs and the sink's entry-changes would be empty),
+         * do not treat heal as failure.
+         */
+        if (is_src)
+            ret = -errno;
+        else
+            ret = 0;
+        goto out;
+    }
+
+    ret = syncop_dir_scan(subvol, &loc, GF_CLIENT_PID_SELF_HEALD, &args,
+                          afr_selfheal_entry_granular_dirent);
+
+    loc_wipe(&loc);
+
+    if (args.mismatch == _gf_true)
+        ret = -1;
+out:
+    if (args.frame)
+        AFR_STACK_DESTROY(args.frame);
+    return ret;
 }
 
+static int
+afr_selfheal_entry_do(call_frame_t *frame, xlator_t *this, fd_t *fd, int source,
+                      unsigned char *sources, unsigned char *healed_sinks)
+{
+    int i = 0;
+    int ret = 0;
+    gf_boolean_t mismatch = _gf_false;
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
+
+    priv = this->private;
+    local = frame->local;
+
+    gf_msg(this->name, GF_LOG_INFO, 0, AFR_MSG_SELF_HEAL_INFO,
+           "performing entry selfheal on %s", uuid_utoa(fd->inode->gfid));
+
+    for (i = 0; i < priv->child_count; i++) {
+        /* Expunge */
+        if (!healed_sinks[i])
+            continue;
+
+        if (!local->need_full_crawl)
+            /* Why call afr_selfheal_entry_granular() on a "healed sink",
+             * given that it is the source that contains the granular
+             * indices?
+             * If the index for this directory is non-existent or empty on
+             * this subvol (=> clear sink), then it will return early
+             * without failure status.
+             * If the index is non-empty and it is yet a 'healed sink', then
+             * it is due to a split-brain in which case we anyway need to
+             * crawl the indices/entry-changes/pargfid directory.
+             */
+            ret = afr_selfheal_entry_granular(frame, this, fd, i, _gf_false);
+        else
+            ret = afr_selfheal_entry_do_subvol(frame, this, fd, i);
 
+        if (ret == -1) {
+            /* gfid or type mismatch. */
+            mismatch = _gf_true;
+            ret = 0;
+        }
+        if (ret)
+            break;
+    }
+
+    if (!ret && source != -1) {
+        /* Impunge */
+        if (local->need_full_crawl)
+            ret = afr_selfheal_entry_do_subvol(frame, this, fd, source);
+        else
+            ret = afr_selfheal_entry_granular(frame, this, fd, source,
+                                              _gf_true);
+    }
+
+    if (mismatch == _gf_true)
+        /* undo pending will be skipped */
+        ret = -1;
+    return ret;
+}
 
 static int
-__afr_selfheal_entry (call_frame_t *frame, xlator_t *this, fd_t *fd,
-		      unsigned char *locked_on)
+__afr_selfheal_entry(call_frame_t *frame, xlator_t *this, fd_t *fd,
+                     unsigned char *locked_on)
 {
-	int                     ret                   = -1;
-	int                     source                = -1;
-	unsigned char          *sources               = NULL;
-	unsigned char          *sinks                 = NULL;
-	unsigned char          *data_lock             = NULL;
-        unsigned char          *postop_lock           = NULL;
-	unsigned char          *healed_sinks          = NULL;
-	struct afr_reply       *locked_replies        = NULL;
-	afr_private_t          *priv                  = NULL;
-        gf_boolean_t            did_sh                = _gf_true;
-
-	priv = this->private;
-
-	sources = alloca0 (priv->child_count);
-	sinks = alloca0 (priv->child_count);
-	healed_sinks = alloca0 (priv->child_count);
-	data_lock = alloca0 (priv->child_count);
-        postop_lock = alloca0 (priv->child_count);
-
-	locked_replies = alloca0 (sizeof (*locked_replies) * priv->child_count);
-
-	ret = afr_selfheal_entrylk (frame, this, fd->inode, this->name, NULL,
-				    data_lock);
-	{
-		if (ret < AFR_SH_MIN_PARTICIPANTS) {
-                        gf_log (this->name, GF_LOG_DEBUG, "%s: Skipping "
-                                "entry self-heal as only %d sub-volumes could "
-                                "be locked in %s domain",
-                                uuid_utoa (fd->inode->gfid), ret,
-                                this->name);
-			ret = -ENOTCONN;
-			goto unlock;
-		}
-
-		ret = __afr_selfheal_entry_prepare (frame, this, fd->inode,
-                                                    data_lock, sources, sinks,
-                                                    healed_sinks,
-						    locked_replies, &source);
-                if (AFR_COUNT(healed_sinks, priv->child_count) == 0) {
-                        did_sh = _gf_false;
-                        goto unlock;
-                }
-	}
-unlock:
-	afr_selfheal_unentrylk (frame, this, fd->inode, this->name, NULL,
-				data_lock);
-	if (ret < 0)
-		goto out;
+    int ret = -1;
+    int source = -1;
+    unsigned char *sources = NULL;
+    unsigned char *sinks = NULL;
+    unsigned char *data_lock = NULL;
+    unsigned char *postop_lock = NULL;
+    unsigned char *healed_sinks = NULL;
+    unsigned char *undid_pending = NULL;
+    struct afr_reply *locked_replies = NULL;
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
+    gf_boolean_t did_sh = _gf_true;
+
+    priv = this->private;
+    local = frame->local;
+
+    sources = alloca0(priv->child_count);
+    sinks = alloca0(priv->child_count);
+    healed_sinks = alloca0(priv->child_count);
+    undid_pending = alloca0(priv->child_count);
+    data_lock = alloca0(priv->child_count);
+    postop_lock = alloca0(priv->child_count);
+
+    locked_replies = alloca0(sizeof(*locked_replies) * priv->child_count);
+
+    ret = afr_selfheal_entrylk(frame, this, fd->inode, this->name, NULL,
+                               data_lock);
+    {
+        if (ret < priv->child_count) {
+            gf_msg_debug(this->name, 0,
+                         "%s: Skipping "
+                         "entry self-heal as only %d sub-volumes could "
+                         "be locked in %s domain",
+                         uuid_utoa(fd->inode->gfid), ret, this->name);
+            ret = -ENOTCONN;
+            goto unlock;
+        }
 
-        if (!did_sh)
-                goto out;
+        ret = __afr_selfheal_entry_prepare(frame, this, fd->inode, data_lock,
+                                           sources, sinks, healed_sinks,
+                                           locked_replies, &source, NULL);
+        if (AFR_COUNT(healed_sinks, priv->child_count) == 0) {
+            did_sh = _gf_false;
+            goto unlock;
+        }
 
-	ret = afr_selfheal_entry_do (frame, this, fd, source, sources,
-                                     healed_sinks);
-	if (ret)
-		goto out;
-
-        /* Take entrylks in xlator domain before doing post-op (undo-pending) in
-         * entry self-heal. This is to prevent a parallel name self-heal on
-         * an entry under @fd->inode from reading pending xattrs while it is
-         * being modified by SHD after entry sh below, given that
-         * name self-heal takes locks ONLY in xlator domain and is free to read
-         * pending changelog in the absence of the following locking.
-         */
-        ret = afr_selfheal_entrylk (frame, this, fd->inode, this->name, NULL,
-                                    postop_lock);
-        {
-                if (AFR_CMP (data_lock, postop_lock, priv->child_count) != 0) {
-                        gf_log (this->name, GF_LOG_DEBUG, "%s: Skipping "
-                                "post-op after entry self-heal as %d "
-                                "sub-volumes, as opposed to %d, could be locked"
-                                " in %s domain", uuid_utoa (fd->inode->gfid),
-                                ret, AFR_COUNT (data_lock, priv->child_count),
-                                this->name);
-                        ret = -ENOTCONN;
-                        goto postop_unlock;
-                }
-
-                ret = afr_selfheal_undo_pending (frame, this, fd->inode,
-                                                 sources, sinks, healed_sinks,
-                                                 AFR_ENTRY_TRANSACTION,
-                                                 locked_replies, postop_lock);
+        local->need_full_crawl = afr_need_full_heal(
+            this, locked_replies, source, healed_sinks, AFR_ENTRY_TRANSACTION);
+    }
+unlock:
+    afr_selfheal_unentrylk(frame, this, fd->inode, this->name, NULL, data_lock,
+                           NULL);
+    if (ret < 0)
+        goto out;
+
+    if (!did_sh)
+        goto out;
+
+    ret = afr_selfheal_entry_do(frame, this, fd, source, sources, healed_sinks);
+    if (ret)
+        goto out;
+
+    /* Take entrylks in xlator domain before doing post-op (undo-pending) in
+     * entry self-heal. This is to prevent a parallel name self-heal on
+     * an entry under @fd->inode from reading pending xattrs while it is
+     * being modified by SHD after entry sh below, given that
+     * name self-heal takes locks ONLY in xlator domain and is free to read
+     * pending changelog in the absence of the following locking.
+     */
+    ret = afr_selfheal_entrylk(frame, this, fd->inode, this->name, NULL,
+                               postop_lock);
+    {
+        if (AFR_CMP(data_lock, postop_lock, priv->child_count) != 0) {
+            gf_msg_debug(this->name, 0,
+                         "%s: Skipping "
+                         "post-op after entry self-heal as %d "
+                         "sub-volumes, as opposed to %d, "
+                         "could be locked in %s domain",
+                         uuid_utoa(fd->inode->gfid), ret,
+                         AFR_COUNT(data_lock, priv->child_count), this->name);
+            ret = -ENOTCONN;
+            goto postop_unlock;
         }
+
+        afr_selfheal_restore_time(frame, this, fd->inode, source, healed_sinks,
+                                  locked_replies);
+        ret = afr_selfheal_undo_pending(
+            frame, this, fd->inode, sources, sinks, healed_sinks, undid_pending,
+            AFR_ENTRY_TRANSACTION, locked_replies, postop_lock);
+    }
 postop_unlock:
-        afr_selfheal_unentrylk (frame, this, fd->inode, this->name, NULL,
-                                postop_lock);
+    afr_selfheal_unentrylk(frame, this, fd->inode, this->name, NULL,
+                           postop_lock, NULL);
 out:
-        if (did_sh)
-                afr_log_selfheal (fd->inode->gfid, this, ret, "entry", source,
-                                  healed_sinks);
-        else
-                ret = 1;
-
-        if (locked_replies)
-                afr_replies_wipe (locked_replies, priv->child_count);
-	return ret;
+    if (did_sh)
+        afr_log_selfheal(fd->inode->gfid, this, ret, "entry", source, sources,
+                         healed_sinks);
+    else
+        ret = 1;
+
+    if (locked_replies)
+        afr_replies_wipe(locked_replies, priv->child_count);
+    return ret;
 }
 
-
 static fd_t *
-afr_selfheal_data_opendir (xlator_t *this, inode_t *inode)
+afr_selfheal_data_opendir(xlator_t *this, inode_t *inode)
 {
-	loc_t loc = {0,};
-	int ret = 0;
-	fd_t *fd = NULL;
-
-	fd = fd_create (inode, 0);
-	if (!fd)
-		return NULL;
-
-	loc.inode = inode_ref (inode);
-	gf_uuid_copy (loc.gfid, inode->gfid);
-
-	ret = syncop_opendir (this, &loc, fd, NULL, NULL);
-	if (ret) {
-		fd_unref (fd);
-		fd = NULL;
-	} else {
-		fd_bind (fd);
-	}
-
-	loc_wipe (&loc);
-        return fd;
+    loc_t loc = {
+        0,
+    };
+    int ret = 0;
+    fd_t *fd = NULL;
+
+    fd = fd_create(inode, 0);
+    if (!fd)
+        return NULL;
+
+    loc.inode = inode_ref(inode);
+    gf_uuid_copy(loc.gfid, inode->gfid);
+
+    ret = syncop_opendir(this, &loc, fd, NULL, NULL);
+    if (ret) {
+        fd_unref(fd);
+        fd = NULL;
+    } else {
+        fd_bind(fd);
+    }
+
+    loc_wipe(&loc);
+    return fd;
 }
 
-
 int
-afr_selfheal_entry (call_frame_t *frame, xlator_t *this, inode_t *inode)
+afr_selfheal_entry(call_frame_t *frame, xlator_t *this, inode_t *inode)
 {
-	afr_private_t *priv = NULL;
-	unsigned char *locked_on = NULL;
-        unsigned char *long_name_locked = NULL;
-	fd_t *fd = NULL;
-	int ret = 0;
-
-	priv = this->private;
-
-	fd = afr_selfheal_data_opendir (this, inode);
-	if (!fd)
-		return -EIO;
-
-	locked_on = alloca0 (priv->child_count);
-	long_name_locked = alloca0 (priv->child_count);
-
-	ret = afr_selfheal_tryentrylk (frame, this, inode, priv->sh_domain, NULL,
-				       locked_on);
-	{
-		if (ret < AFR_SH_MIN_PARTICIPANTS) {
-                        gf_log (this->name, GF_LOG_DEBUG, "%s: Skipping "
-                                "entry self-heal as only %d sub-volumes could "
-                                "be locked in %s domain",
-                                uuid_utoa (fd->inode->gfid), ret,
-                                priv->sh_domain);
-			/* Either less than two subvols available, or another
-			   selfheal (from another server) is in progress. Skip
-			   for now in any case there isn't anything to do.
-			*/
-			ret = -ENOTCONN;
-			goto unlock;
-		}
-
-                ret = afr_selfheal_tryentrylk (frame, this, inode, this->name,
-                                               LONG_FILENAME, long_name_locked);
-                {
-                        if (ret < 1) {
-                                gf_log (this->name, GF_LOG_DEBUG, "%s: Skipping"
-                                        " entry self-heal as only %d "
-                                        "sub-volumes could be locked in "
-                                        "special-filename domain",
-                                        uuid_utoa (fd->inode->gfid),
-                                        ret);
-                                ret = -ENOTCONN;
-                                goto unlock;
-                        }
-                        ret = __afr_selfheal_entry (frame, this, fd, locked_on);
-                }
-                afr_selfheal_unentrylk (frame, this, inode, this->name,
-                                        LONG_FILENAME, long_name_locked);
-	}
+    afr_private_t *priv = NULL;
+    unsigned char *locked_on = NULL;
+    fd_t *fd = NULL;
+    int ret = 0;
+
+    priv = this->private;
+
+    fd = afr_selfheal_data_opendir(this, inode);
+    if (!fd)
+        return -EIO;
+
+    locked_on = alloca0(priv->child_count);
+
+    ret = afr_selfheal_tie_breaker_entrylk(frame, this, inode, priv->sh_domain,
+                                           NULL, locked_on);
+    {
+        if (ret < priv->child_count) {
+            gf_msg_debug(this->name, 0,
+                         "%s: Skipping "
+                         "entry self-heal as only %d sub-volumes could "
+                         "be locked in %s domain",
+                         uuid_utoa(fd->inode->gfid), ret, priv->sh_domain);
+            /* Either less than two subvols available, or another
+               selfheal (from another server) is in progress. Skip
+               for now in any case there isn't anything to do.
+            */
+            ret = -ENOTCONN;
+            goto unlock;
+        }
+
+        ret = __afr_selfheal_entry(frame, this, fd, locked_on);
+    }
 unlock:
-	afr_selfheal_unentrylk (frame, this, inode, priv->sh_domain, NULL, locked_on);
+    afr_selfheal_unentrylk(frame, this, inode, priv->sh_domain, NULL, locked_on,
+                           NULL);
 
-	if (fd)
-		fd_unref (fd);
+    if (fd)
+        fd_unref(fd);
 
-	return ret;
+    return ret;
 }
diff --git a/xlators/cluster/afr/src/afr-self-heal-metadata.c b/xlators/cluster/afr/src/afr-self-heal-metadata.c
index 11c0c9c8cd9..03f43bad16e 100644
--- a/xlators/cluster/afr/src/afr-self-heal-metadata.c
+++ b/xlators/cluster/afr/src/afr-self-heal-metadata.c
@@ -8,110 +8,108 @@
   cases as published by the Free Software Foundation.
 */
 
-
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
-
 #include "afr.h"
 #include "afr-self-heal.h"
-#include "byte-order.h"
+#include <glusterfs/byte-order.h>
 #include "protocol-common.h"
+#include <glusterfs/events.h>
 
-#define AFR_HEAL_ATTR (GF_SET_ATTR_UID|GF_SET_ATTR_GID|GF_SET_ATTR_MODE)
+#define AFR_HEAL_ATTR (GF_SET_ATTR_UID | GF_SET_ATTR_GID | GF_SET_ATTR_MODE)
 
 static gf_boolean_t
-_afr_ignorable_key_match (dict_t *d, char *k, data_t *val, void *mdata)
+_afr_ignorable_key_match(dict_t *d, char *k, data_t *val, void *mdata)
 {
-        return afr_is_xattr_ignorable (k);
+    return afr_is_xattr_ignorable(k);
 }
 
 void
-afr_delete_ignorable_xattrs (dict_t *xattr)
+afr_delete_ignorable_xattrs(dict_t *xattr)
 {
-        dict_foreach_match (xattr, _afr_ignorable_key_match, NULL,
-                            dict_remove_foreach_fn, NULL);
+    dict_foreach_match(xattr, _afr_ignorable_key_match, NULL,
+                       dict_remove_foreach_fn, NULL);
 }
 
 int
-__afr_selfheal_metadata_do (call_frame_t *frame, xlator_t *this, inode_t *inode,
-                            int source, unsigned char *healed_sinks,
-                            struct afr_reply *locked_replies)
+__afr_selfheal_metadata_do(call_frame_t *frame, xlator_t *this, inode_t *inode,
+                           int source, unsigned char *healed_sinks,
+                           struct afr_reply *locked_replies)
 {
-	int ret = -1;
-	loc_t loc = {0,};
-	dict_t *xattr = NULL;
-	dict_t *old_xattr = NULL;
-	afr_private_t *priv = NULL;
-	int i = 0;
-
-	priv = this->private;
-
-	loc.inode = inode_ref (inode);
-	gf_uuid_copy (loc.gfid, inode->gfid);
-
-	gf_log (this->name, GF_LOG_INFO, "performing metadata selfheal on %s",
-		uuid_utoa (inode->gfid));
-
-	ret = syncop_getxattr (priv->children[source], &loc, &xattr, NULL,
-                               NULL, NULL);
-	if (ret < 0) {
-		ret = -EIO;
-                goto out;
-	}
-
-	afr_delete_ignorable_xattrs (xattr);
-
-	for (i = 0; i < priv->child_count; i++) {
-                if (old_xattr) {
-                        dict_unref (old_xattr);
-                        old_xattr = NULL;
-                }
-
-		if (!healed_sinks[i])
-			continue;
-
-		ret = syncop_setattr (priv->children[i], &loc,
-				      &locked_replies[source].poststat,
-				      AFR_HEAL_ATTR, NULL, NULL, NULL, NULL);
-		if (ret)
-			healed_sinks[i] = 0;
-
-		ret = syncop_getxattr (priv->children[i], &loc, &old_xattr, 0,
-                                       NULL, NULL);
-		if (old_xattr) {
-			afr_delete_ignorable_xattrs (old_xattr);
-			ret = syncop_removexattr (priv->children[i], &loc, "",
-						  old_xattr, NULL);
-		}
-
-		ret = syncop_setxattr (priv->children[i], &loc, xattr, 0, NULL,
-                                       NULL);
-		if (ret)
-			healed_sinks[i] = 0;
-	}
-        ret = 0;
+    int ret = -1;
+    loc_t loc = {
+        0,
+    };
+    dict_t *xattr = NULL;
+    dict_t *old_xattr = NULL;
+    afr_private_t *priv = NULL;
+    int i = 0;
+
+    priv = this->private;
+
+    loc.inode = inode_ref(inode);
+    gf_uuid_copy(loc.gfid, inode->gfid);
+
+    gf_msg(this->name, GF_LOG_INFO, 0, AFR_MSG_SELF_HEAL_INFO,
+           "performing metadata selfheal on %s", uuid_utoa(inode->gfid));
+
+    ret = syncop_getxattr(priv->children[source], &loc, &xattr, NULL, NULL,
+                          NULL);
+    if (ret < 0) {
+        ret = -EIO;
+        goto out;
+    }
+
+    afr_delete_ignorable_xattrs(xattr);
+
+    for (i = 0; i < priv->child_count; i++) {
+        if (old_xattr) {
+            dict_unref(old_xattr);
+            old_xattr = NULL;
+        }
+
+        if (!healed_sinks[i])
+            continue;
+
+        ret = syncop_setattr(priv->children[i], &loc,
+                             &locked_replies[source].poststat, AFR_HEAL_ATTR,
+                             NULL, NULL, NULL, NULL);
+        if (ret)
+            healed_sinks[i] = 0;
+
+        ret = syncop_getxattr(priv->children[i], &loc, &old_xattr, 0, NULL,
+                              NULL);
+        if (old_xattr) {
+            afr_delete_ignorable_xattrs(old_xattr);
+            ret = syncop_removexattr(priv->children[i], &loc, "", old_xattr,
+                                     NULL);
+            if (ret)
+                healed_sinks[i] = 0;
+        }
+
+        ret = syncop_setxattr(priv->children[i], &loc, xattr, 0, NULL, NULL);
+        if (ret)
+            healed_sinks[i] = 0;
+    }
+    ret = 0;
 
 out:
-	loc_wipe (&loc);
-	if (xattr)
-		dict_unref (xattr);
-        if (old_xattr)
-                dict_unref (old_xattr);
+    loc_wipe(&loc);
+    if (xattr)
+        dict_unref(xattr);
+    if (old_xattr)
+        dict_unref(old_xattr);
 
-	return ret;
+    return ret;
 }
 
-static inline uint64_t
+static uint64_t
 mtime_ns(struct iatt *ia)
 {
-        uint64_t ret;
+    uint64_t ret;
 
-        ret = (((uint64_t)(ia->ia_mtime)) * 1000000000)
-            + (uint64_t)(ia->ia_mtime_nsec);
+    ret = (((uint64_t)(ia->ia_mtime)) * 1000000000) +
+          (uint64_t)(ia->ia_mtime_nsec);
 
-        return ret;
+    return ret;
 }
 
 /*
@@ -124,315 +122,425 @@ mtime_ns(struct iatt *ia)
  * the source with the most recent modification date.
  */
 static int
-afr_dirtime_splitbrain_source (call_frame_t *frame, xlator_t *this,
-                               struct afr_reply *replies,
-                               unsigned char *locked_on)
+afr_dirtime_splitbrain_source(call_frame_t *frame, xlator_t *this,
+                              struct afr_reply *replies,
+                              unsigned char *locked_on)
 {
-        afr_private_t *priv  = NULL;
-        int            source = -1;
-        struct iatt    source_ia;
-        struct iatt    child_ia;
-        uint64_t       mtime = 0;
-        int            i;
-        int            ret   = -1;
-
-        priv = this->private;
+    afr_private_t *priv = NULL;
+    int source = -1;
+    struct iatt source_ia;
+    struct iatt child_ia;
+    uint64_t mtime = 0;
+    int i;
+    int ret = -1;
+
+    priv = this->private;
+
+    for (i = 0; i < priv->child_count; i++) {
+        if (!locked_on[i])
+            continue;
+
+        if (!replies[i].valid)
+            continue;
+
+        if (replies[i].op_ret != 0)
+            continue;
+
+        if (mtime_ns(&replies[i].poststat) <= mtime)
+            continue;
+
+        mtime = mtime_ns(&replies[i].poststat);
+        source = i;
+    }
+
+    if (source == -1)
+        goto out;
+
+    source_ia = replies[source].poststat;
+    if (source_ia.ia_type != IA_IFDIR)
+        goto out;
+
+    for (i = 0; i < priv->child_count; i++) {
+        if (i == source)
+            continue;
+
+        if (!replies[i].valid)
+            continue;
+
+        if (replies[i].op_ret != 0)
+            continue;
+
+        child_ia = replies[i].poststat;
+
+        if (!IA_EQUAL(source_ia, child_ia, gfid) ||
+            !IA_EQUAL(source_ia, child_ia, type) ||
+            !IA_EQUAL(source_ia, child_ia, prot) ||
+            !IA_EQUAL(source_ia, child_ia, uid) ||
+            !IA_EQUAL(source_ia, child_ia, gid) ||
+            !afr_xattrs_are_equal(replies[source].xdata, replies[i].xdata))
+            goto out;
+    }
+
+    /*
+     * Metadata split brain is just about [amc]time
+     * We return our source.
+     */
+    ret = source;
+out:
+    return ret;
+}
 
-        for (i = 0; i < priv->child_count; i++) {
-                if (!locked_on[i])
-                        continue;
+static int
+__afr_selfheal_metadata_mark_pending_xattrs(call_frame_t *frame, xlator_t *this,
+                                            inode_t *inode,
+                                            struct afr_reply *replies,
+                                            unsigned char *sources)
+{
+    int ret = 0;
+    int i = 0;
+    int m_idx = 0;
+    afr_private_t *priv = NULL;
+    int raw[AFR_NUM_CHANGE_LOGS] = {0};
+    dict_t *xattr = NULL;
+
+    priv = this->private;
+    m_idx = afr_index_for_transaction_type(AFR_METADATA_TRANSACTION);
+    raw[m_idx] = 1;
+
+    xattr = dict_new();
+    if (!xattr)
+        return -ENOMEM;
+
+    for (i = 0; i < priv->child_count; i++) {
+        if (sources[i])
+            continue;
+        ret = dict_set_static_bin(xattr, priv->pending_key[i], raw,
+                                  sizeof(int) * AFR_NUM_CHANGE_LOGS);
+        if (ret) {
+            ret = -1;
+            goto out;
+        }
+    }
+
+    for (i = 0; i < priv->child_count; i++) {
+        if (!sources[i])
+            continue;
+        ret = afr_selfheal_post_op(frame, this, inode, i, xattr, NULL);
+        if (ret < 0) {
+            gf_msg(this->name, GF_LOG_INFO, -ret, AFR_MSG_SELF_HEAL_INFO,
+                   "Failed to set pending metadata xattr on child %d for %s", i,
+                   uuid_utoa(inode->gfid));
+            goto out;
+        }
+    }
 
-                if (!replies[i].valid)
-                        continue;
+    afr_replies_wipe(replies, priv->child_count);
+    ret = afr_selfheal_unlocked_discover(frame, inode, inode->gfid, replies);
 
-                if (replies[i].op_ret != 0)
-                        continue;
+out:
+    if (xattr)
+        dict_unref(xattr);
+    return ret;
+}
 
-                if (mtime_ns(&replies[i].poststat) <= mtime)
-                        continue;
+/*
+ * Look for mismatching uid/gid or mode or user xattrs even if
+ * AFR xattrs don't say so, and pick one arbitrarily as winner. */
 
-                mtime = mtime_ns(&replies[i].poststat);
-                source = i;
+static int
+__afr_selfheal_metadata_finalize_source(call_frame_t *frame, xlator_t *this,
+                                        inode_t *inode, unsigned char *sources,
+                                        unsigned char *sinks,
+                                        unsigned char *healed_sinks,
+                                        unsigned char *undid_pending,
+                                        unsigned char *locked_on,
+                                        struct afr_reply *replies)
+{
+    int i = 0;
+    afr_private_t *priv = NULL;
+    struct iatt srcstat = {
+        0,
+    };
+    int source = -1;
+    int sources_count = 0;
+    int ret = 0;
+
+    priv = this->private;
+
+    sources_count = AFR_COUNT(sources, priv->child_count);
+
+    if ((AFR_CMP(locked_on, healed_sinks, priv->child_count) == 0) ||
+        !sources_count) {
+        source = afr_mark_split_brain_source_sinks(
+            frame, this, inode, sources, sinks, healed_sinks, locked_on,
+            replies, AFR_METADATA_TRANSACTION);
+        if (source >= 0) {
+            _afr_fav_child_reset_sink_xattrs(
+                frame, this, inode, source, healed_sinks, undid_pending,
+                AFR_METADATA_TRANSACTION, locked_on, replies);
+            goto out;
         }
 
-        if (source == -1)
-                goto out;
+        /* If this is a directory mtime/ctime only split brain
+           use the most recent */
+        source = afr_dirtime_splitbrain_source(frame, this, replies, locked_on);
+        if (source != -1) {
+            gf_msg(this->name, GF_LOG_INFO, 0, AFR_MSG_SPLIT_BRAIN,
+                   "clear time "
+                   "split brain on %s",
+                   uuid_utoa(replies[source].poststat.ia_gfid));
+            sources[source] = 1;
+            healed_sinks[source] = 0;
+            goto out;
+        }
 
-        source_ia = replies[source].poststat;
-        if (source_ia.ia_type != IA_IFDIR)
-                goto out;
+        if (!priv->metadata_splitbrain_forced_heal) {
+            gf_event(EVENT_AFR_SPLIT_BRAIN,
+                     "client-pid=%d;"
+                     "subvol=%s;"
+                     "type=metadata;file=%s",
+                     this->ctx->cmd_args.client_pid, this->name,
+                     uuid_utoa(inode->gfid));
+            return -EIO;
+        }
 
+        /* Metadata split brain, select one subvol
+           arbitrarily */
         for (i = 0; i < priv->child_count; i++) {
-                if (i == source)
-                        continue;
-
-                if (!replies[i].valid)
-                        continue;
-
-                if (replies[i].op_ret != 0)
-                        continue;
+            if (locked_on[i] && healed_sinks[i]) {
+                sources[i] = 1;
+                healed_sinks[i] = 0;
+                break;
+            }
+        }
+    }
+
+    /* No split brain at this point. If we were called from
+     * afr_heal_splitbrain_file(), abort.*/
+    if (afr_dict_contains_heal_op(frame))
+        return -EIO;
+
+    source = afr_choose_source_by_policy(priv, sources,
+                                         AFR_METADATA_TRANSACTION);
+    srcstat = replies[source].poststat;
+
+    for (i = 0; i < priv->child_count; i++) {
+        if (!sources[i] || i == source)
+            continue;
+        if (!IA_EQUAL(srcstat, replies[i].poststat, type) ||
+            !IA_EQUAL(srcstat, replies[i].poststat, uid) ||
+            !IA_EQUAL(srcstat, replies[i].poststat, gid) ||
+            !IA_EQUAL(srcstat, replies[i].poststat, prot)) {
+            gf_msg_debug(this->name, 0,
+                         "%s: iatt mismatch "
+                         "for source(%d) vs (%d)",
+                         uuid_utoa(replies[source].poststat.ia_gfid), source,
+                         i);
+            sources[i] = 0;
+            healed_sinks[i] = 1;
+        }
+    }
+
+    for (i = 0; i < priv->child_count; i++) {
+        if (!sources[i] || i == source)
+            continue;
+        if (!afr_xattrs_are_equal(replies[source].xdata, replies[i].xdata)) {
+            gf_msg_debug(this->name, 0,
+                         "%s: xattr mismatch "
+                         "for source(%d) vs (%d)",
+                         uuid_utoa(replies[source].poststat.ia_gfid), source,
+                         i);
+            sources[i] = 0;
+            healed_sinks[i] = 1;
+        }
+    }
+    if ((sources_count == priv->child_count) && (source > -1) &&
+        (AFR_COUNT(healed_sinks, priv->child_count) != 0)) {
+        ret = __afr_selfheal_metadata_mark_pending_xattrs(frame, this, inode,
+                                                          replies, sources);
+        if (ret < 0)
+            return ret;
+    }
+out:
+    afr_mark_active_sinks(this, sources, locked_on, healed_sinks);
+    return source;
+}
 
-                child_ia = replies[i].poststat;
+int
+__afr_selfheal_metadata_prepare(call_frame_t *frame, xlator_t *this,
+                                inode_t *inode, unsigned char *locked_on,
+                                unsigned char *sources, unsigned char *sinks,
+                                unsigned char *healed_sinks,
+                                unsigned char *undid_pending,
+                                struct afr_reply *replies, unsigned char *pflag)
+{
+    int ret = -1;
+    int source = -1;
+    afr_private_t *priv = NULL;
+    int i = 0;
+    uint64_t *witness = NULL;
 
-                if (!IA_EQUAL(source_ia, child_ia, gfid) ||
-                    !IA_EQUAL(source_ia, child_ia, type) ||
-                    !IA_EQUAL(source_ia, child_ia, prot) ||
-                    !IA_EQUAL(source_ia, child_ia, uid) ||
-                    !IA_EQUAL(source_ia, child_ia, gid) ||
-                    !afr_xattrs_are_equal (replies[source].xdata,
-                                           replies[i].xdata))
-                        goto out;
-        }
+    priv = this->private;
 
-        /*
-         * Metadata split brain is just about [amc]time
-         * We return our source.
-         */
-        ret = source;
-out:
+    ret = afr_selfheal_unlocked_discover(frame, inode, inode->gfid, replies);
+    if (ret)
         return ret;
-}
 
+    witness = alloca0(sizeof(*witness) * priv->child_count);
+    ret = afr_selfheal_find_direction(frame, this, replies,
+                                      AFR_METADATA_TRANSACTION, locked_on,
+                                      sources, sinks, witness, pflag);
+    if (ret)
+        return ret;
 
-/*
- * Look for mismatching uid/gid or mode or user xattrs even if
- * AFR xattrs don't say so, and pick one arbitrarily as winner. */
+    /* Initialize the healed_sinks[] array optimistically to
+       the intersection of to-be-healed (i.e sinks[]) and
+       the list of servers which are up (i.e locked_on[]).
+
+       As we encounter failures in the healing process, we
+       will unmark the respective servers in the healed_sinks[]
+       array.
+    */
+    AFR_INTERSECT(healed_sinks, sinks, locked_on, priv->child_count);
+
+    /* If any source has witness, pick first
+     * witness source and make everybody else sinks */
+    for (i = 0; i < priv->child_count; i++) {
+        if (sources[i] && witness[i]) {
+            source = i;
+            break;
+        }
+    }
 
-static int
-__afr_selfheal_metadata_finalize_source (call_frame_t *frame, xlator_t *this,
-                                         unsigned char *sources,
-                                         unsigned char *sinks,
-					 unsigned char *healed_sinks,
-					 unsigned char *locked_on,
-					 struct afr_reply *replies)
-{
-	int i = 0;
-	afr_private_t *priv = NULL;
-	struct iatt first = {0, };
-	int source = -1;
-	int sources_count = 0;
-
-	priv = this->private;
-
-	sources_count = AFR_COUNT (sources, priv->child_count);
-
-	if ((AFR_CMP (locked_on, healed_sinks, priv->child_count) == 0)
-	    || !sources_count) {
-
-                source = afr_mark_split_brain_source_sinks (frame, this,
-                                                            sources, sinks,
-                                                            healed_sinks,
-                                                            locked_on, replies,
-                                                      AFR_METADATA_TRANSACTION);
-                if (source >= 0)
-                        return source;
-
-		/* If this is a directory mtime/ctime only split brain
-		   use the most recent */
-		source = afr_dirtime_splitbrain_source (frame, this,
-							replies, locked_on);
-		if (source != -1) {
-			gf_log (this->name, GF_LOG_NOTICE, "clear time "
-				"split brain on %s",
-				 uuid_utoa (replies[source].poststat.ia_gfid));
-			sources[source] = 1;
-			healed_sinks[source] = 0;
-			return source;
-		}
-
-		if (!priv->metadata_splitbrain_forced_heal) {
-			return -EIO;
-		}
-
-		/* Metadata split brain, select one subvol
-		   arbitrarily */
-		for (i = 0; i < priv->child_count; i++) {
-			if (locked_on[i] && healed_sinks[i]) {
-				sources[i] = 1;
-				healed_sinks[i] = 0;
-				break;
-			}
-		}
-	}
-
-        /* No split brain at this point. If we were called from
-         * afr_heal_splitbrain_file(), abort.*/
-        if (afr_dict_contains_heal_op(frame))
-                return -EIO;
-
-	for (i = 0; i < priv->child_count; i++) {
-		if (!sources[i])
-			continue;
-		if (source == -1) {
-			source = i;
-			first = replies[i].poststat;
-                        break;
-		}
-	}
-
-	for (i = 0; i < priv->child_count; i++) {
-		if (!sources[i] || i == source)
-			continue;
-		if (!IA_EQUAL (first, replies[i].poststat, type) ||
-		    !IA_EQUAL (first, replies[i].poststat, uid) ||
-		    !IA_EQUAL (first, replies[i].poststat, gid) ||
-		    !IA_EQUAL (first, replies[i].poststat, prot)) {
-                        gf_log (this->name, GF_LOG_DEBUG, "%s: iatt mismatch "
-                                "for source(%d) vs (%d)",
-                                uuid_utoa (replies[source].poststat.ia_gfid),
-                                source, i);
-			sources[i] = 0;
-			healed_sinks[i] = 1;
-		}
-	}
-
-        for (i =0; i < priv->child_count; i++) {
-		if (!sources[i] || i == source)
-			continue;
-                if (!afr_xattrs_are_equal (replies[source].xdata,
-                                           replies[i].xdata)) {
-                        gf_log (this->name, GF_LOG_DEBUG, "%s: xattr mismatch "
-                                "for source(%d) vs (%d)",
-                                uuid_utoa (replies[source].poststat.ia_gfid),
-                                source, i);
-                        sources[i] = 0;
-                        healed_sinks[i] = 1;
-                }
+    if (source != -1) {
+        for (i = 0; i < priv->child_count; i++) {
+            if (i != source && sources[i]) {
+                sources[i] = 0;
+                healed_sinks[i] = 1;
+            }
         }
+    }
 
-	return source;
-}
+    source = __afr_selfheal_metadata_finalize_source(
+        frame, this, inode, sources, sinks, healed_sinks, undid_pending,
+        locked_on, replies);
 
+    if (source < 0)
+        return -EIO;
+
+    return source;
+}
 
 int
-__afr_selfheal_metadata_prepare (call_frame_t *frame, xlator_t *this, inode_t *inode,
-				 unsigned char *locked_on, unsigned char *sources,
-				 unsigned char *sinks, unsigned char *healed_sinks,
-				 struct afr_reply *replies)
+afr_selfheal_metadata(call_frame_t *frame, xlator_t *this, inode_t *inode)
 {
-	int ret = -1;
-	int source = -1;
-	afr_private_t *priv = NULL;
-	int i = 0;
-        uint64_t *witness = NULL;
+    afr_private_t *priv = NULL;
+    int ret = -1;
+    unsigned char *sources = NULL;
+    unsigned char *sinks = NULL;
+    unsigned char *data_lock = NULL;
+    unsigned char *healed_sinks = NULL;
+    unsigned char *undid_pending = NULL;
+    struct afr_reply *locked_replies = NULL;
+    gf_boolean_t did_sh = _gf_true;
+    int source = -1;
+
+    priv = this->private;
+
+    sources = alloca0(priv->child_count);
+    sinks = alloca0(priv->child_count);
+    healed_sinks = alloca0(priv->child_count);
+    undid_pending = alloca0(priv->child_count);
+    data_lock = alloca0(priv->child_count);
+
+    locked_replies = alloca0(sizeof(*locked_replies) * priv->child_count);
+
+    ret = afr_selfheal_inodelk(frame, this, inode, this->name, LLONG_MAX - 1, 0,
+                               data_lock);
+    {
+        if (ret < priv->child_count) {
+            ret = -ENOTCONN;
+            goto unlock;
+        }
 
-	priv = this->private;
+        ret = __afr_selfheal_metadata_prepare(
+            frame, this, inode, data_lock, sources, sinks, healed_sinks,
+            undid_pending, locked_replies, NULL);
+        if (ret < 0)
+            goto unlock;
 
-	ret = afr_selfheal_unlocked_discover (frame, inode, inode->gfid,
-					      replies);
-        if (ret)
-                return ret;
-
-        witness = alloca0 (sizeof (*witness) * priv->child_count);
-        ret = afr_selfheal_find_direction (frame, this, replies,
-					   AFR_METADATA_TRANSACTION,
-					   locked_on, sources, sinks, witness);
-	if (ret)
-		return ret;
-
-        /* Initialize the healed_sinks[] array optimistically to
-           the intersection of to-be-healed (i.e sinks[]) and
-           the list of servers which are up (i.e locked_on[]).
-
-           As we encounter failures in the healing process, we
-           will unmark the respective servers in the healed_sinks[]
-           array.
-        */
-        AFR_INTERSECT (healed_sinks, sinks, locked_on, priv->child_count);
-
-        /* If any source has witness, pick first
-         * witness source and make everybody else sinks */
-        for (i = 0; i < priv->child_count; i++) {
-                if (sources[i] && witness[i]) {
-                        source = i;
-                        break;
-                }
-        }
+        source = ret;
 
-        if (source != -1) {
-                for (i = 0; i < priv->child_count; i++) {
-                        if (i != source && sources[i]) {
-                                sources[i] = 0;
-                                healed_sinks[i] = 1;
-                        }
-                }
+        if (AFR_COUNT(healed_sinks, priv->child_count) == 0) {
+            did_sh = _gf_false;
+            goto unlock;
         }
 
-	source = __afr_selfheal_metadata_finalize_source (frame, this, sources,
-                                                          sinks, healed_sinks,
-                                                          locked_on, replies);
+        ret = __afr_selfheal_metadata_do(frame, this, inode, source,
+                                         healed_sinks, locked_replies);
+        if (ret)
+            goto unlock;
 
-	if (source < 0)
-		return -EIO;
+        afr_selfheal_restore_time(frame, this, inode, source, healed_sinks,
+                                  locked_replies);
 
-	return source;
+        ret = afr_selfheal_undo_pending(
+            frame, this, inode, sources, sinks, healed_sinks, undid_pending,
+            AFR_METADATA_TRANSACTION, locked_replies, data_lock);
+    }
+unlock:
+    afr_selfheal_uninodelk(frame, this, inode, this->name, LLONG_MAX - 1, 0,
+                           data_lock);
+
+    if (did_sh)
+        afr_log_selfheal(inode->gfid, this, ret, "metadata", source, sources,
+                         healed_sinks);
+    else
+        ret = 1;
+
+    if (locked_replies)
+        afr_replies_wipe(locked_replies, priv->child_count);
+    return ret;
 }
 
 int
-afr_selfheal_metadata (call_frame_t *frame, xlator_t *this, inode_t *inode)
+afr_selfheal_metadata_by_stbuf(xlator_t *this, struct iatt *stbuf)
 {
-	afr_private_t *priv = NULL;
-	int ret = -1;
-	unsigned char *sources = NULL;
-	unsigned char *sinks = NULL;
-	unsigned char *data_lock = NULL;
-	unsigned char *healed_sinks = NULL;
-	struct afr_reply *locked_replies = NULL;
-        gf_boolean_t did_sh = _gf_true;
-	int source = -1;
-
-	priv = this->private;
-
-	sources = alloca0 (priv->child_count);
-	sinks = alloca0 (priv->child_count);
-	healed_sinks = alloca0 (priv->child_count);
-	data_lock = alloca0 (priv->child_count);
-
-	locked_replies = alloca0 (sizeof (*locked_replies) * priv->child_count);
-
-	ret = afr_selfheal_inodelk (frame, this, inode, this->name,
-				    LLONG_MAX - 1, 0, data_lock);
-	{
-		if (ret < AFR_SH_MIN_PARTICIPANTS) {
-			ret = -ENOTCONN;
-			goto unlock;
-		}
-
-		ret = __afr_selfheal_metadata_prepare (frame, this, inode,
-                                                       data_lock, sources,
-                                                       sinks, healed_sinks,
-						       locked_replies);
-		if (ret < 0)
-			goto unlock;
-
-		source = ret;
-
-                if (AFR_COUNT (healed_sinks, priv->child_count) == 0) {
-                        did_sh = _gf_false;
-                        goto unlock;
-                }
-
-                ret = __afr_selfheal_metadata_do (frame, this, inode, source,
-                                                  healed_sinks, locked_replies);
-                if (ret)
-                        goto unlock;
-
-                ret = afr_selfheal_undo_pending (frame, this, inode, sources,
-                                                 sinks, healed_sinks,
-                                                 AFR_METADATA_TRANSACTION,
-                                                 locked_replies, data_lock);
-	}
-unlock:
-	afr_selfheal_uninodelk (frame, this, inode, this->name,
-				LLONG_MAX -1, 0, data_lock);
-
-        if (did_sh)
-                afr_log_selfheal (inode->gfid, this, ret, "metadata", source,
-                                  healed_sinks);
-        else
-                ret = 1;
-
-        if (locked_replies)
-                afr_replies_wipe (locked_replies, priv->child_count);
-	return ret;
+    inode_t *inode = NULL;
+    inode_t *link_inode = NULL;
+    call_frame_t *frame = NULL;
+    int ret = 0;
+
+    if (gf_uuid_is_null(stbuf->ia_gfid)) {
+        ret = -EINVAL;
+        goto out;
+    }
+
+    inode = inode_new(this->itable);
+    if (!inode) {
+        ret = -ENOMEM;
+        goto out;
+    }
+
+    link_inode = inode_link(inode, NULL, NULL, stbuf);
+    if (!link_inode) {
+        ret = -ENOMEM;
+        goto out;
+    }
+
+    frame = afr_frame_create(this, &ret);
+    if (!frame) {
+        ret = -ret;
+        goto out;
+    }
+
+    ret = afr_selfheal_metadata(frame, this, link_inode);
+out:
+    if (inode)
+        inode_unref(inode);
+    if (link_inode)
+        inode_unref(link_inode);
+    if (frame)
+        AFR_STACK_DESTROY(frame);
+    return ret;
 }
diff --git a/xlators/cluster/afr/src/afr-self-heal-name.c b/xlators/cluster/afr/src/afr-self-heal-name.c
index ba9c3eeb6ea..834aac86d48 100644
--- a/xlators/cluster/afr/src/afr-self-heal-name.c
+++ b/xlators/cluster/afr/src/afr-self-heal-name.c
@@ -8,701 +8,609 @@
   cases as published by the Free Software Foundation.
 */
 
-
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
-
+#include <glusterfs/events.h>
 #include "afr.h"
 #include "afr-self-heal.h"
+#include "afr-messages.h"
 
 int
-__afr_selfheal_assign_gfid (xlator_t *this, inode_t *parent, uuid_t pargfid,
-                            const char *bname, inode_t *inode,
-                            struct afr_reply *replies, void *gfid,
-                            unsigned char *locked_on,
-                            gf_boolean_t is_gfid_absent)
+__afr_selfheal_assign_gfid(xlator_t *this, inode_t *parent, uuid_t pargfid,
+                           const char *bname, inode_t *inode,
+                           struct afr_reply *replies, void *gfid,
+                           unsigned char *locked_on, int source,
+                           unsigned char *sources, gf_boolean_t is_gfid_absent,
+                           int *gfid_idx)
 {
-	int             ret          = 0;
-        int             up_count     = 0;
-        int             locked_count = 0;
-	afr_private_t  *priv         = NULL;
-	dict_t         *xdata        = NULL;
-	loc_t           loc          = {0, };
-        call_frame_t   *new_frame    = NULL;
-        afr_local_t    *new_local    = NULL;
-
-	priv = this->private;
-
-        new_frame = afr_frame_create (this);
-        if (!new_frame) {
-                ret = -ENOMEM;
-                goto out;
-        }
-
-        new_local = new_frame->local;
+    int ret = 0;
+    int up_count = 0;
+    int locked_count = 0;
+    afr_private_t *priv = NULL;
 
-	gf_uuid_copy (parent->gfid, pargfid);
+    priv = this->private;
 
-	xdata = dict_new ();
-	if (!xdata) {
-                ret = -ENOMEM;
-		goto out;
-	}
+    gf_uuid_copy(parent->gfid, pargfid);
 
-	ret = dict_set_static_bin (xdata, "gfid-req", gfid, 16);
-	if (ret) {
-		ret = -ENOMEM;
-		goto out;
-	}
-
-	loc.parent = inode_ref (parent);
-	loc.inode = inode_ref (inode);
-	gf_uuid_copy (loc.pargfid, pargfid);
-	loc.name = bname;
-
-        if (is_gfid_absent) {
-                /* Ensure all children of AFR are up before performing gfid heal, to
-                 * guard against the possibility of gfid split brain. */
-
-                up_count = AFR_COUNT (priv->child_up, priv->child_count);
-                if (up_count != priv->child_count) {
-                        ret = -EIO;
-                        goto out;
-                }
+    if (is_gfid_absent) {
+        /* Ensure all children of AFR are up before performing gfid heal, to
+         * guard against the possibility of gfid split brain. */
 
-                locked_count = AFR_COUNT (locked_on, priv->child_count);
-                if (locked_count != priv->child_count) {
-                        ret = -EIO;
-                        goto out;
-                }
+        up_count = AFR_COUNT(priv->child_up, priv->child_count);
+        if (up_count != priv->child_count) {
+            ret = -EIO;
+            goto out;
         }
 
-        /* Clear out old replies here and wind lookup on all locked
-         * subvolumes to achieve two things:
-         *   a. gfid heal on those subvolumes that do not have gfid associated
-         *      with the inode, and
-         *   b. refresh replies, which can be consumed by
-         *      __afr_selfheal_name_impunge().
-         */
-
-        afr_replies_wipe (replies, priv->child_count);
-
-        AFR_ONLIST (locked_on, new_frame, afr_selfheal_discover_cbk, lookup,
-                    &loc, xdata);
+        locked_count = AFR_COUNT(locked_on, priv->child_count);
+        if (locked_count != priv->child_count) {
+            ret = -EIO;
+            goto out;
+        }
+    }
 
-        afr_replies_copy (replies, new_local->replies, priv->child_count);
+    ret = afr_lookup_and_heal_gfid(this, parent, bname, inode, replies, source,
+                                   sources, gfid, gfid_idx);
 
 out:
-	loc_wipe (&loc);
-        if (xdata)
-                dict_unref (xdata);
-        if (new_frame)
-                AFR_STACK_DESTROY (new_frame);
-
-	return ret;
+    return ret;
 }
 
 int
-__afr_selfheal_name_impunge (call_frame_t *frame, xlator_t *this,
-                             inode_t *parent, uuid_t pargfid,
-                             const char *bname, inode_t *inode,
-                             struct afr_reply *replies, int gfid_idx)
+__afr_selfheal_name_impunge(call_frame_t *frame, xlator_t *this,
+                            inode_t *parent, uuid_t pargfid, const char *bname,
+                            inode_t *inode, struct afr_reply *replies,
+                            int gfid_idx)
 {
-	int i = 0;
-	afr_private_t *priv = NULL;
-        int ret = 0;
-        unsigned char *newentry = NULL;
-        unsigned char *sources = NULL;
+    int i = 0;
+    afr_private_t *priv = NULL;
+    int ret = 0;
+    unsigned char *sources = NULL;
 
-        priv = this->private;
+    priv = this->private;
 
-	newentry = alloca0 (priv->child_count);
-	sources = alloca0 (priv->child_count);
+    sources = alloca0(priv->child_count);
 
-	gf_uuid_copy (parent->gfid, pargfid);
+    gf_uuid_copy(parent->gfid, pargfid);
 
-	for (i = 0; i < priv->child_count; i++) {
-		if (!replies[i].valid)
-			continue;
+    for (i = 0; i < priv->child_count; i++) {
+        if (!replies[i].valid || replies[i].op_ret != 0)
+            continue;
 
-		if (gf_uuid_compare (replies[i].poststat.ia_gfid,
-				  replies[gfid_idx].poststat.ia_gfid) == 0) {
-                        sources[i] = 1;
-			continue;
-                }
-
-		ret |= afr_selfheal_recreate_entry (this, i, gfid_idx, parent,
-                                                    bname, inode, replies,
-                                                    newentry);
-	}
-
-	if (AFR_COUNT (newentry, priv->child_count))
-		afr_selfheal_newentry_mark (frame, this, inode, gfid_idx, replies,
-					    sources, newentry);
-	return ret;
-}
+        if (gf_uuid_compare(replies[i].poststat.ia_gfid,
+                            replies[gfid_idx].poststat.ia_gfid) == 0) {
+            sources[i] = 1;
+            continue;
+        }
+    }
 
+    for (i = 0; i < priv->child_count; i++) {
+        if (sources[i])
+            continue;
 
-int
-__afr_selfheal_name_expunge (xlator_t *this, inode_t *parent, uuid_t pargfid,
-                             const char *bname, inode_t *inode,
-			     struct afr_reply *replies)
-{
-	loc_t loc = {0, };
-	int i = 0;
-	afr_private_t *priv = NULL;
-	char g[64];
-	int ret = 0;
-
-	priv = this->private;
-
-	loc.parent = inode_ref (parent);
-	gf_uuid_copy (loc.pargfid, pargfid);
-	loc.name = bname;
-	loc.inode = inode_ref (inode);
-
-	for (i = 0; i < priv->child_count; i++) {
-		if (!replies[i].valid)
-			continue;
-
-		if (replies[i].op_ret)
-			continue;
-
-		switch (replies[i].poststat.ia_type) {
-		case IA_IFDIR:
-			gf_log (this->name, GF_LOG_WARNING,
-				"expunging dir %s/%s (%s) on %s",
-				uuid_utoa (pargfid), bname,
-				uuid_utoa_r (replies[i].poststat.ia_gfid, g),
-				priv->children[i]->name);
-			ret |= syncop_rmdir (priv->children[i], &loc, 1, NULL,
-                                             NULL);
-			break;
-		default:
-			gf_log (this->name, GF_LOG_WARNING,
-				"expunging file %s/%s (%s) on %s",
-				uuid_utoa (pargfid), bname,
-				uuid_utoa_r (replies[i].poststat.ia_gfid, g),
-				priv->children[i]->name);
-			ret |= syncop_unlink (priv->children[i], &loc, NULL,
-                                              NULL);
-			break;
-		}
-	}
-
-	loc_wipe (&loc);
-
-	return ret;
+        ret |= afr_selfheal_recreate_entry(frame, i, gfid_idx, sources, parent,
+                                           bname, inode, replies);
+    }
 
+    return ret;
 }
 
-/* This function is to be called after ensuring that there is no gfid mismatch
- * for the inode across multiple sources
- */
-static int
-afr_selfheal_gfid_idx_get (xlator_t *this, struct afr_reply *replies,
-                           unsigned char *sources)
+int
+__afr_selfheal_name_expunge(xlator_t *this, inode_t *parent, uuid_t pargfid,
+                            const char *bname, inode_t *inode,
+                            struct afr_reply *replies)
 {
-        int             i        =  0;
-        int             gfid_idx = -1;
-        afr_private_t  *priv     =  NULL;
+    int i = 0;
+    afr_private_t *priv = NULL;
+    int ret = 0;
 
-        priv = this->private;
+    priv = this->private;
 
-        for (i = 0; i < priv->child_count; i++) {
-                if (!replies[i].valid)
-                        continue;
+    for (i = 0; i < priv->child_count; i++) {
+        if (!replies[i].valid)
+            continue;
 
-                if (!sources[i])
-                        continue;
+        if (replies[i].op_ret)
+            continue;
 
-                if (gf_uuid_is_null (replies[i].poststat.ia_gfid))
-                        continue;
+        ret |= afr_selfheal_entry_delete(this, parent, bname, inode, i,
+                                         replies);
+    }
 
-                gfid_idx = i;
-                break;
-        }
-        return gfid_idx;
+    return ret;
 }
 
 static gf_boolean_t
-afr_selfheal_name_need_heal_check (xlator_t *this, struct afr_reply *replies)
+afr_selfheal_name_need_heal_check(xlator_t *this, struct afr_reply *replies)
 {
-        int             i           = 0;
-	int             first_idx   = -1;
-        gf_boolean_t    need_heal   = _gf_false;
-        afr_private_t  *priv        = NULL;
+    int i = 0;
+    int first_idx = -1;
+    gf_boolean_t need_heal = _gf_false;
+    afr_private_t *priv = NULL;
 
-        priv = this->private;
+    priv = this->private;
 
-	for (i = 0; i < priv->child_count; i++) {
-		if (!replies[i].valid)
-			continue;
+    for (i = 0; i < priv->child_count; i++) {
+        if (!replies[i].valid)
+            continue;
 
-                if ((replies[i].op_ret == -1) &&
-                    (replies[i].op_errno == ENODATA))
-                        need_heal = _gf_true;
+        if ((replies[i].op_ret == -1) && (replies[i].op_errno == ENODATA))
+            need_heal = _gf_true;
 
-		if (first_idx == -1) {
-			first_idx = i;
-			continue;
-		}
-
-		if (replies[i].op_ret != replies[first_idx].op_ret)
-			need_heal = _gf_true;
+        if (first_idx == -1) {
+            first_idx = i;
+            continue;
+        }
 
-		if (gf_uuid_compare (replies[i].poststat.ia_gfid,
-				  replies[first_idx].poststat.ia_gfid))
-			need_heal = _gf_true;
+        if (replies[i].op_ret != replies[first_idx].op_ret)
+            need_heal = _gf_true;
 
-                if ((replies[i].op_ret == 0) &&
-                    (gf_uuid_is_null(replies[i].poststat.ia_gfid)))
-                        need_heal = _gf_true;
+        if (gf_uuid_compare(replies[i].poststat.ia_gfid,
+                            replies[first_idx].poststat.ia_gfid))
+            need_heal = _gf_true;
 
-	}
+        if ((replies[i].op_ret == 0) &&
+            (gf_uuid_is_null(replies[i].poststat.ia_gfid)))
+            need_heal = _gf_true;
+    }
 
-        return need_heal;
+    return need_heal;
 }
 
 static int
-afr_selfheal_name_type_mismatch_check (xlator_t *this, struct afr_reply *replies,
-                                       int source, unsigned char *sources,
-                                       uuid_t pargfid, const char *bname)
+afr_selfheal_name_type_mismatch_check(xlator_t *this, struct afr_reply *replies,
+                                      int source, unsigned char *sources,
+                                      uuid_t pargfid, const char *bname)
 {
-        int             i           = 0;
-        int             type_idx    = -1;
-        ia_type_t       inode_type  = IA_INVAL;
-        afr_private_t  *priv        = NULL;
+    int i = 0;
+    int type_idx = -1;
+    ia_type_t inode_type = IA_INVAL;
+    ia_type_t inode_type1 = IA_INVAL;
+    afr_private_t *priv = NULL;
 
-        priv = this->private;
+    priv = this->private;
 
-        for (i = 0; i < priv->child_count; i++) {
-                if (!replies[i].valid)
-                        continue;
+    for (i = 0; i < priv->child_count; i++) {
+        if (!replies[i].valid || replies[i].op_ret != 0)
+            continue;
 
-                if (replies[i].poststat.ia_type == IA_INVAL)
-                        continue;
+        if (replies[i].poststat.ia_type == IA_INVAL)
+            continue;
 
-                if (inode_type == IA_INVAL) {
-                        inode_type = replies[i].poststat.ia_type;
-                        type_idx = i;
-                        continue;
-                }
-
-                if (sources[i] || source == -1) {
-                        if ((sources[type_idx] || source == -1) &&
-                            (inode_type != replies[i].poststat.ia_type)) {
-                                    gf_msg (this->name, GF_LOG_WARNING, 0,
-                                            AFR_MSG_SPLIT_BRAIN,
-                                            "Type mismatch for <gfid:%s>/%s: "
-                                            "%d on %s and %d on %s",
-                                            uuid_utoa(pargfid), bname,
-                                            replies[i].poststat.ia_type,
-                                            priv->children[i]->name,
-                                            replies[type_idx].poststat.ia_type,
-                                            priv->children[type_idx]->name);
-                                    return -EIO;
-                        }
-                        inode_type = replies[i].poststat.ia_type;
-                        type_idx = i;
-                }
+        if (inode_type == IA_INVAL) {
+            inode_type = replies[i].poststat.ia_type;
+            type_idx = i;
+            continue;
         }
-        return 0;
+        inode_type1 = replies[i].poststat.ia_type;
+        if (sources[i] || source == -1) {
+            if ((sources[type_idx] || source == -1) &&
+                (inode_type != inode_type1)) {
+                gf_msg(this->name, GF_LOG_WARNING, 0, AFR_MSG_SPLIT_BRAIN,
+                       "Type mismatch for <gfid:%s>/%s: "
+                       "%s on %s and %s on %s",
+                       uuid_utoa(pargfid), bname,
+                       gf_inode_type_to_str(inode_type1),
+                       priv->children[i]->name,
+                       gf_inode_type_to_str(inode_type),
+                       priv->children[type_idx]->name);
+                gf_event(EVENT_AFR_SPLIT_BRAIN,
+                         "client-pid=%d;"
+                         "subvol=%s;type=file;"
+                         "file=<gfid:%s>/%s;count=2;"
+                         "child-%d=%s;type-%d=%s;child-%d=%s;"
+                         "type-%d=%s",
+                         this->ctx->cmd_args.client_pid, this->name,
+                         uuid_utoa(pargfid), bname, i, priv->children[i]->name,
+                         i, gf_inode_type_to_str(inode_type1), type_idx,
+                         priv->children[type_idx]->name, type_idx,
+                         gf_inode_type_to_str(inode_type));
+                return -EIO;
+            }
+            inode_type = replies[i].poststat.ia_type;
+            type_idx = i;
+        }
+    }
+    return 0;
 }
 
 static int
-afr_selfheal_name_gfid_mismatch_check (xlator_t *this, struct afr_reply *replies,
-                                       int source, unsigned char *sources,
-                                       int *gfid_idx, uuid_t pargfid,
-                                       const char *bname)
+afr_selfheal_name_gfid_mismatch_check(xlator_t *this, struct afr_reply *replies,
+                                      int source, unsigned char *sources,
+                                      int *gfid_idx, uuid_t pargfid,
+                                      const char *bname, inode_t *inode,
+                                      unsigned char *locked_on, dict_t *xdata)
 {
-        int             i             = 0;
-	int             gfid_idx_iter = -1;
-        void           *gfid          = NULL;
-        afr_private_t  *priv          = NULL;
-	char g1[64], g2[64];
-
-        priv = this->private;
-
-	for (i = 0; i < priv->child_count; i++) {
-		if (!replies[i].valid)
-			continue;
-
-		if (gf_uuid_is_null (replies[i].poststat.ia_gfid))
-			continue;
-
-		if (!gfid) {
-			gfid = &replies[i].poststat.ia_gfid;
-			gfid_idx_iter = i;
-			continue;
-		}
-
-		if (sources[i] || source == -1) {
-			if ((sources[gfid_idx_iter] || source == -1) &&
-			    gf_uuid_compare (gfid, replies[i].poststat.ia_gfid)) {
-				gf_msg (this->name, GF_LOG_WARNING, 0,
-                                        AFR_MSG_SPLIT_BRAIN,
-					"GFID mismatch for <gfid:%s>/%s "
-					"%s on %s and %s on %s",
-					uuid_utoa (pargfid), bname,
-					uuid_utoa_r (replies[i].poststat.ia_gfid, g1),
-					priv->children[i]->name,
-					uuid_utoa_r (replies[gfid_idx_iter].poststat.ia_gfid, g2),
-					priv->children[gfid_idx_iter]->name);
-				return -EIO;
-			}
-
-                        gfid = &replies[i].poststat.ia_gfid;
-			gfid_idx_iter = i;
-		}
-	}
-
-        *gfid_idx = gfid_idx_iter;
-        return 0;
+    int i = 0;
+    int gfid_idx_iter = -1;
+    int ret = -1;
+    void *gfid = NULL;
+    void *gfid1 = NULL;
+    afr_private_t *priv = NULL;
+
+    priv = this->private;
+
+    for (i = 0; i < priv->child_count; i++) {
+        if (!replies[i].valid || replies[i].op_ret != 0)
+            continue;
+
+        if (gf_uuid_is_null(replies[i].poststat.ia_gfid))
+            continue;
+
+        if (!gfid) {
+            gfid = &replies[i].poststat.ia_gfid;
+            gfid_idx_iter = i;
+            continue;
+        }
+
+        gfid1 = &replies[i].poststat.ia_gfid;
+        if (sources[i] || source == -1) {
+            if ((sources[gfid_idx_iter] || source == -1) &&
+                gf_uuid_compare(gfid, gfid1)) {
+                ret = afr_gfid_split_brain_source(this, replies, inode, pargfid,
+                                                  bname, gfid_idx_iter, i,
+                                                  locked_on, gfid_idx, xdata);
+                if (!ret && *gfid_idx >= 0) {
+                    ret = dict_set_sizen_str_sizen(xdata, "gfid-heal-msg",
+                                                   "GFID split-brain resolved");
+                    if (ret)
+                        gf_msg(this->name, GF_LOG_ERROR, 0,
+                               AFR_MSG_DICT_SET_FAILED,
+                               "Error setting gfid-"
+                               "heal-msg dict");
+                }
+                return ret;
+            }
+            gfid = &replies[i].poststat.ia_gfid;
+            gfid_idx_iter = i;
+        }
+    }
+
+    *gfid_idx = gfid_idx_iter;
+    return 0;
 }
 
 static gf_boolean_t
-afr_selfheal_name_source_empty_check (xlator_t *this, struct afr_reply *replies,
-                                      unsigned char *sources, int source)
+afr_selfheal_name_source_empty_check(xlator_t *this, struct afr_reply *replies,
+                                     unsigned char *sources, int source)
 {
-	int             i               = 0;
-	afr_private_t  *priv            = NULL;
-	gf_boolean_t    source_is_empty = _gf_true;
+    int i = 0;
+    afr_private_t *priv = NULL;
+    gf_boolean_t source_is_empty = _gf_true;
 
-	priv = this->private;
+    priv = this->private;
 
-        if (source == -1) {
-                source_is_empty = _gf_false;
-                goto out;
-        }
+    if (source == -1) {
+        source_is_empty = _gf_false;
+        goto out;
+    }
 
-	for (i = 0; i < priv->child_count; i++) {
-                if (!sources[i])
-                        continue;
+    for (i = 0; i < priv->child_count; i++) {
+        if (!sources[i])
+            continue;
 
-                if (replies[i].op_ret == -1 && replies[i].op_errno == ENOENT)
-                        continue;
+        if (replies[i].op_ret == -1 && replies[i].op_errno == ENOENT)
+            continue;
 
-                source_is_empty = _gf_false;
-                break;
-	}
+        source_is_empty = _gf_false;
+        break;
+    }
 out:
-        return source_is_empty;
+    return source_is_empty;
 }
 
 int
-__afr_selfheal_name_do (call_frame_t *frame, xlator_t *this, inode_t *parent,
-                        uuid_t pargfid, const char *bname, inode_t *inode,
-                        unsigned char *sources, unsigned char *sinks,
-			unsigned char *healed_sinks, int source,
-			unsigned char *locked_on, struct afr_reply *replies,
-                        void *gfid_req)
+__afr_selfheal_name_do(call_frame_t *frame, xlator_t *this, inode_t *parent,
+                       uuid_t pargfid, const char *bname, inode_t *inode,
+                       unsigned char *sources, unsigned char *sinks,
+                       unsigned char *healed_sinks, int source,
+                       unsigned char *locked_on, struct afr_reply *replies,
+                       void *gfid_req, dict_t *xdata)
 {
-	int             gfid_idx        = -1;
-        int             ret             = -1;
-	void           *gfid            = NULL;
-	gf_boolean_t    source_is_empty = _gf_true;
-	gf_boolean_t    need_heal       = _gf_false;
-        gf_boolean_t    is_gfid_absent  = _gf_false;
-
-        need_heal = afr_selfheal_name_need_heal_check (this, replies);
-	if (!need_heal)
-		return 0;
-
-        source_is_empty = afr_selfheal_name_source_empty_check (this, replies,
-                                                                sources,
-                                                                source);
-	if (source_is_empty)
-		return __afr_selfheal_name_expunge (this, parent, pargfid,
-						    bname, inode, replies);
-
-        ret = afr_selfheal_name_type_mismatch_check (this, replies, source,
-                                                     sources, pargfid, bname);
-        if (ret)
-                return ret;
-
-        ret = afr_selfheal_name_gfid_mismatch_check (this, replies, source,
-                                                     sources, &gfid_idx,
-                                                     pargfid, bname);
-        if (ret)
-                return ret;
-
-	if (gfid_idx == -1) {
-                if (!gfid_req || gf_uuid_is_null (gfid_req))
-                        return -1;
-                gfid = gfid_req;
-        } else {
-                gfid = &replies[gfid_idx].poststat.ia_gfid;
-        }
-
-        is_gfid_absent = (gfid_idx == -1) ? _gf_true : _gf_false;
-	ret = __afr_selfheal_assign_gfid (this, parent, pargfid, bname, inode,
-                                          replies, gfid, locked_on,
-                                          is_gfid_absent);
-        if (ret)
-                return ret;
-
-        if (gfid_idx == -1) {
-                gfid_idx = afr_selfheal_gfid_idx_get (this, replies, sources);
-                if (gfid_idx == -1)
-                        return -1;
-        }
+    int gfid_idx = -1;
+    int ret = -1;
+    void *gfid = NULL;
+    gf_boolean_t source_is_empty = _gf_true;
+    gf_boolean_t need_heal = _gf_false;
+    gf_boolean_t is_gfid_absent = _gf_false;
+
+    need_heal = afr_selfheal_name_need_heal_check(this, replies);
+    if (!need_heal)
+        return 0;
 
-	return __afr_selfheal_name_impunge (frame, this, parent, pargfid,
-                                            bname, inode,
-                                            replies, gfid_idx);
+    source_is_empty = afr_selfheal_name_source_empty_check(this, replies,
+                                                           sources, source);
+    if (source_is_empty) {
+        ret = __afr_selfheal_name_expunge(this, parent, pargfid, bname, inode,
+                                          replies);
+        if (ret == -EIO)
+            ret = -1;
+        return ret;
+    }
+
+    ret = afr_selfheal_name_type_mismatch_check(this, replies, source, sources,
+                                                pargfid, bname);
+    if (ret)
+        return ret;
+
+    ret = afr_selfheal_name_gfid_mismatch_check(this, replies, source, sources,
+                                                &gfid_idx, pargfid, bname,
+                                                inode, locked_on, xdata);
+    if (ret)
+        return ret;
+
+    if (gfid_idx == -1) {
+        if (!gfid_req || gf_uuid_is_null(gfid_req))
+            return -1;
+        gfid = gfid_req;
+    } else {
+        gfid = &replies[gfid_idx].poststat.ia_gfid;
+        if (source == -1)
+            /* Either entry split-brain or dirty xattrs are present on parent.*/
+            source = gfid_idx;
+    }
+
+    is_gfid_absent = (gfid_idx == -1) ? _gf_true : _gf_false;
+    ret = __afr_selfheal_assign_gfid(this, parent, pargfid, bname, inode,
+                                     replies, gfid, locked_on, source, sources,
+                                     is_gfid_absent, &gfid_idx);
+    if (ret || (gfid_idx < 0))
+        return ret;
+
+    ret = __afr_selfheal_name_impunge(frame, this, parent, pargfid, bname,
+                                      inode, replies, gfid_idx);
+    if (ret == -EIO)
+        ret = -1;
+
+    return ret;
 }
 
-
 int
-__afr_selfheal_name_finalize_source (xlator_t *this, unsigned char *sources,
-				     unsigned char *healed_sinks,
-				     unsigned char *locked_on,
-				     struct afr_reply *replies,
-                                     uint64_t *witness)
+__afr_selfheal_name_finalize_source(xlator_t *this, unsigned char *sources,
+                                    unsigned char *healed_sinks,
+                                    unsigned char *locked_on, uint64_t *witness)
 {
-	int i = 0;
-	afr_private_t *priv = NULL;
-	int source = -1;
-	int sources_count = 0;
-
-	priv = this->private;
-
-	sources_count = AFR_COUNT (sources, priv->child_count);
-
-	if ((AFR_CMP (locked_on, healed_sinks, priv->child_count) == 0)
-            || !sources_count || afr_does_witness_exist (this, witness)) {
-                memset (sources, 0, sizeof (*sources) * priv->child_count);
-                afr_mark_active_sinks (this, sources, locked_on, healed_sinks);
-		return -1;
-	}
-
-	for (i = 0; i < priv->child_count; i++) {
-		if (sources[i]) {
-			source = i;
-			break;
-		}
-	}
-
-	return source;
+    int i = 0;
+    afr_private_t *priv = NULL;
+    int source = -1;
+    int sources_count = 0;
+
+    priv = this->private;
+
+    sources_count = AFR_COUNT(sources, priv->child_count);
+
+    if ((AFR_CMP(locked_on, healed_sinks, priv->child_count) == 0) ||
+        !sources_count || afr_does_witness_exist(this, witness)) {
+        memset(sources, 0, sizeof(*sources) * priv->child_count);
+        afr_mark_active_sinks(this, sources, locked_on, healed_sinks);
+        return -1;
+    }
+
+    for (i = 0; i < priv->child_count; i++) {
+        if (sources[i]) {
+            source = i;
+            break;
+        }
+    }
+
+    return source;
 }
 
 int
-__afr_selfheal_name_prepare (call_frame_t *frame, xlator_t *this, inode_t *parent,
-			     uuid_t pargfid, unsigned char *locked_on,
-			     unsigned char *sources, unsigned char *sinks,
-			     unsigned char *healed_sinks, int *source_p)
+__afr_selfheal_name_prepare(call_frame_t *frame, xlator_t *this,
+                            inode_t *parent, uuid_t pargfid,
+                            unsigned char *locked_on, unsigned char *sources,
+                            unsigned char *sinks, unsigned char *healed_sinks,
+                            int *source_p)
 {
-	int ret = -1;
-	int source = -1;
-	afr_private_t *priv = NULL;
-        struct afr_reply *replies = NULL;
-        uint64_t *witness = NULL;
-
-	priv = this->private;
-
-        replies = alloca0 (priv->child_count * sizeof(*replies));
-
-	ret = afr_selfheal_unlocked_discover (frame, parent, pargfid, replies);
-	if (ret)
-		goto out;
-
-        witness = alloca0 (sizeof (*witness) * priv->child_count);
-	ret = afr_selfheal_find_direction (frame, this, replies,
-					   AFR_ENTRY_TRANSACTION,
-					   locked_on, sources, sinks, witness);
-	if (ret)
-		goto out;
-
-        /* Initialize the healed_sinks[] array optimistically to
-           the intersection of to-be-healed (i.e sinks[]) and
-           the list of servers which are up (i.e locked_on[]).
-
-           As we encounter failures in the healing process, we
-           will unmark the respective servers in the healed_sinks[]
-           array.
-        */
-        AFR_INTERSECT (healed_sinks, sinks, locked_on, priv->child_count);
-
-	source = __afr_selfheal_name_finalize_source (this, sources,
-                                                      healed_sinks,
-						      locked_on, replies,
-                                                      witness);
-	if (source < 0) {
-		/* If source is < 0 (typically split-brain), we perform a
-		   conservative merge of entries rather than erroring out */
-	}
-	*source_p = source;
+    int ret = -1;
+    int source = -1;
+    afr_private_t *priv = NULL;
+    struct afr_reply *replies = NULL;
+    uint64_t *witness = NULL;
+
+    priv = this->private;
+
+    replies = alloca0(priv->child_count * sizeof(*replies));
+
+    ret = afr_selfheal_unlocked_discover(frame, parent, pargfid, replies);
+    if (ret)
+        goto out;
+
+    witness = alloca0(sizeof(*witness) * priv->child_count);
+    ret = afr_selfheal_find_direction(frame, this, replies,
+                                      AFR_ENTRY_TRANSACTION, locked_on, sources,
+                                      sinks, witness, NULL);
+    if (ret)
+        goto out;
+
+    /* Initialize the healed_sinks[] array optimistically to
+       the intersection of to-be-healed (i.e sinks[]) and
+       the list of servers which are up (i.e locked_on[]).
+
+       As we encounter failures in the healing process, we
+       will unmark the respective servers in the healed_sinks[]
+       array.
+    */
+    AFR_INTERSECT(healed_sinks, sinks, locked_on, priv->child_count);
+
+    source = __afr_selfheal_name_finalize_source(this, sources, healed_sinks,
+                                                 locked_on, witness);
+    if (source < 0) {
+        /* If source is < 0 (typically split-brain), we perform a
+           conservative merge of entries rather than erroring out */
+    }
+    *source_p = source;
 
 out:
-        if (replies)
-                afr_replies_wipe (replies, priv->child_count);
+    if (replies)
+        afr_replies_wipe(replies, priv->child_count);
 
-	return ret;
+    return ret;
 }
 
-
 int
-afr_selfheal_name_do (call_frame_t *frame, xlator_t *this, inode_t *parent,
-		      uuid_t pargfid, const char *bname, void *gfid_req)
+afr_selfheal_name_do(call_frame_t *frame, xlator_t *this, inode_t *parent,
+                     uuid_t pargfid, const char *bname, void *gfid_req,
+                     dict_t *xdata)
 {
-	afr_private_t *priv = NULL;
-	unsigned char *sources = NULL;
-	unsigned char *sinks = NULL;
-	unsigned char *healed_sinks = NULL;
-	unsigned char *locked_on = NULL;
-	int source = -1;
-        struct afr_reply *replies = NULL;
-	int ret = -1;
-	inode_t *inode = NULL;
-        dict_t *xattr = NULL;
-
-        xattr = dict_new ();
-        if (!xattr)
-                return -ENOMEM;
-
-        ret = dict_set_int32 (xattr, GF_GFIDLESS_LOOKUP, 1);
-        if (ret) {
-                dict_destroy (xattr);
-                return -1;
+    afr_private_t *priv = NULL;
+    unsigned char *sources = NULL;
+    unsigned char *sinks = NULL;
+    unsigned char *healed_sinks = NULL;
+    unsigned char *locked_on = NULL;
+    int source = -1;
+    struct afr_reply *replies = NULL;
+    int ret = -1;
+    inode_t *inode = NULL;
+    dict_t *xattr = NULL;
+
+    xattr = dict_new();
+    if (!xattr)
+        return -ENOMEM;
+
+    ret = dict_set_int32_sizen(xattr, GF_GFIDLESS_LOOKUP, 1);
+    if (ret) {
+        dict_unref(xattr);
+        return -1;
+    }
+
+    priv = this->private;
+
+    locked_on = alloca0(priv->child_count);
+    sources = alloca0(priv->child_count);
+    sinks = alloca0(priv->child_count);
+    healed_sinks = alloca0(priv->child_count);
+
+    replies = alloca0(priv->child_count * sizeof(*replies));
+
+    ret = afr_selfheal_entrylk(frame, this, parent, this->name, bname,
+                               locked_on);
+    {
+        if (ret < priv->child_count) {
+            ret = -ENOTCONN;
+            goto unlock;
         }
 
-	priv = this->private;
-
-	locked_on = alloca0 (priv->child_count);
-	sources = alloca0 (priv->child_count);
-	sinks = alloca0 (priv->child_count);
-	healed_sinks = alloca0 (priv->child_count);
-
-	replies = alloca0 (priv->child_count * sizeof(*replies));
-
-	ret = afr_selfheal_entrylk (frame, this, parent, this->name, bname,
-				    locked_on);
-	{
-		if (ret < AFR_SH_MIN_PARTICIPANTS) {
-			ret = -ENOTCONN;
-			goto unlock;
-		}
-
-		ret = __afr_selfheal_name_prepare (frame, this, parent, pargfid,
-						   locked_on, sources, sinks,
-						   healed_sinks, &source);
-		if (ret)
-			goto unlock;
-
-                inode = afr_selfheal_unlocked_lookup_on (frame, parent, bname,
-						         replies, locked_on,
-                                                         xattr);
-		if (!inode) {
-			ret = -ENOMEM;
-			goto unlock;
-		}
-
-		ret = __afr_selfheal_name_do (frame, this, parent, pargfid,
-                                              bname, inode, sources, sinks,
-                                              healed_sinks, source, locked_on,
-                                              replies, gfid_req);
-	}
+        ret = __afr_selfheal_name_prepare(frame, this, parent, pargfid,
+                                          locked_on, sources, sinks,
+                                          healed_sinks, &source);
+        if (ret)
+            goto unlock;
+
+        inode = afr_selfheal_unlocked_lookup_on(frame, parent, bname, replies,
+                                                locked_on, xattr);
+        if (!inode) {
+            ret = -ENOMEM;
+            goto unlock;
+        }
+
+        ret = __afr_selfheal_name_do(frame, this, parent, pargfid, bname, inode,
+                                     sources, sinks, healed_sinks, source,
+                                     locked_on, replies, gfid_req, xdata);
+    }
 unlock:
-	afr_selfheal_unentrylk (frame, this, parent, this->name, bname,
-				locked_on);
-	if (inode)
-		inode_unref (inode);
+    afr_selfheal_unentrylk(frame, this, parent, this->name, bname, locked_on,
+                           NULL);
+    if (inode)
+        inode_unref(inode);
 
-        if (replies)
-                afr_replies_wipe (replies, priv->child_count);
-        if (xattr)
-                dict_unref (xattr);
+    if (replies)
+        afr_replies_wipe(replies, priv->child_count);
+    if (xattr)
+        dict_unref(xattr);
 
-	return ret;
+    return ret;
 }
 
-
 int
-afr_selfheal_name_unlocked_inspect (call_frame_t *frame, xlator_t *this,
-				    inode_t *parent, uuid_t pargfid,
-				    const char *bname, gf_boolean_t *need_heal)
+afr_selfheal_name_unlocked_inspect(call_frame_t *frame, xlator_t *this,
+                                   inode_t *parent, uuid_t pargfid,
+                                   const char *bname, gf_boolean_t *need_heal)
 {
-	afr_private_t *priv = NULL;
-	int i = 0;
-	struct afr_reply *replies = NULL;
-	inode_t *inode = NULL;
-	int first_idx = -1;
-
-	priv = this->private;
-
-	replies = alloca0 (sizeof (*replies) * priv->child_count);
-
-	inode = afr_selfheal_unlocked_lookup_on (frame, parent, bname,
-						 replies, priv->child_up, NULL);
-	if (!inode)
-		return -ENOMEM;
-
-	for (i = 0; i < priv->child_count; i++) {
-		if (!replies[i].valid)
-			continue;
-
-                if ((replies[i].op_ret == -1) &&
-                    (replies[i].op_errno == ENODATA))
-                        *need_heal = _gf_true;
-
-		if (first_idx == -1) {
-			first_idx = i;
-			continue;
-		}
-
-		if (replies[i].op_ret != replies[first_idx].op_ret)
-			*need_heal = _gf_true;
-
-		if (gf_uuid_compare (replies[i].poststat.ia_gfid,
-				  replies[first_idx].poststat.ia_gfid))
-			*need_heal = _gf_true;
-	}
-
-	if (inode)
-		inode_unref (inode);
-        if (replies)
-                afr_replies_wipe (replies, priv->child_count);
-	return 0;
+    afr_private_t *priv = NULL;
+    int i = 0;
+    struct afr_reply *replies = NULL;
+    inode_t *inode = NULL;
+    int first_idx = -1;
+    afr_local_t *local = NULL;
+
+    priv = this->private;
+    local = frame->local;
+
+    replies = alloca0(sizeof(*replies) * priv->child_count);
+
+    inode = afr_selfheal_unlocked_lookup_on(frame, parent, bname, replies,
+                                            local->child_up, NULL);
+    if (!inode)
+        return -ENOMEM;
+
+    for (i = 0; i < priv->child_count; i++) {
+        if (!replies[i].valid)
+            continue;
+
+        if ((replies[i].op_ret == -1) && (replies[i].op_errno == ENODATA)) {
+            *need_heal = _gf_true;
+            break;
+        }
+
+        if (first_idx == -1) {
+            first_idx = i;
+            continue;
+        }
+
+        if (replies[i].op_ret != replies[first_idx].op_ret) {
+            *need_heal = _gf_true;
+            break;
+        }
+
+        if (gf_uuid_compare(replies[i].poststat.ia_gfid,
+                            replies[first_idx].poststat.ia_gfid)) {
+            *need_heal = _gf_true;
+            break;
+        }
+    }
+
+    if (inode)
+        inode_unref(inode);
+    if (replies)
+        afr_replies_wipe(replies, priv->child_count);
+    return 0;
 }
 
 int
-afr_selfheal_name (xlator_t *this, uuid_t pargfid, const char *bname,
-                   void *gfid_req)
+afr_selfheal_name(xlator_t *this, uuid_t pargfid, const char *bname,
+                  void *gfid_req, dict_t *xdata)
 {
-	inode_t *parent = NULL;
-	call_frame_t *frame = NULL;
-	int ret = -1;
-	gf_boolean_t need_heal = _gf_false;
-
-	parent = afr_inode_find (this, pargfid);
-	if (!parent)
-		goto out;
-
-	frame = afr_frame_create (this);
-	if (!frame)
-		goto out;
-
-	ret = afr_selfheal_name_unlocked_inspect (frame, this, parent, pargfid,
-						  bname, &need_heal);
-	if (ret)
-		goto out;
-
-	if (need_heal) {
-		ret = afr_selfheal_name_do (frame, this, parent, pargfid, bname,
-                                            gfid_req);
-                if (ret)
-                        goto out;
-        }
+    inode_t *parent = NULL;
+    call_frame_t *frame = NULL;
+    int ret = -1;
+    gf_boolean_t need_heal = _gf_false;
+
+    parent = afr_inode_find(this, pargfid);
+    if (!parent)
+        goto out;
+
+    frame = afr_frame_create(this, NULL);
+    if (!frame)
+        goto out;
+
+    ret = afr_selfheal_name_unlocked_inspect(frame, this, parent, pargfid,
+                                             bname, &need_heal);
+    if (ret)
+        goto out;
+
+    if (need_heal) {
+        ret = afr_selfheal_name_do(frame, this, parent, pargfid, bname,
+                                   gfid_req, xdata);
+        if (ret)
+            goto out;
+    }
 
-        ret = 0;
+    ret = 0;
 out:
-	if (parent)
-		inode_unref (parent);
-	if (frame)
-		AFR_STACK_DESTROY (frame);
+    if (parent)
+        inode_unref(parent);
+    if (frame)
+        AFR_STACK_DESTROY(frame);
 
-	return ret;
+    return ret;
 }
diff --git a/xlators/cluster/afr/src/afr-self-heal.h b/xlators/cluster/afr/src/afr-self-heal.h
index 32be2480234..48e6dbcfb18 100644
--- a/xlators/cluster/afr/src/afr-self-heal.h
+++ b/xlators/cluster/afr/src/afr-self-heal.h
@@ -8,244 +8,370 @@
   cases as published by the Free Software Foundation.
 */
 
-
 #ifndef _AFR_SELFHEAL_H
 #define _AFR_SELFHEAL_H
 
-#define AFR_SH_MIN_PARTICIPANTS 2
-
 /* Perform fop on all UP subvolumes and wait for all callbacks to return */
 
-#define AFR_ONALL(frame, rfn, fop, args ...) do {			\
-	afr_local_t *__local = frame->local;				\
-	afr_private_t *__priv = frame->this->private;			\
-	int __i = 0, __count = 0;					\
-									\
-	afr_local_replies_wipe (__local, __priv);				\
-									\
-	for (__i = 0; __i < __priv->child_count; __i++) {		\
-		if (!__priv->child_up[__i]) continue;			\
-		STACK_WIND_COOKIE (frame, rfn, (void *)(long) __i,	\
-				   __priv->children[__i],		\
-				   __priv->children[__i]->fops->fop, args); \
-		__count++;						\
-	}								\
-	syncbarrier_wait (&__local->barrier, __count);			\
-	} while (0)
-
+#define AFR_ONALL(frame, rfn, fop, args...)                                    \
+    do {                                                                       \
+        afr_local_t *__local = frame->local;                                   \
+        afr_private_t *__priv = frame->this->private;                          \
+        int __i = 0, __count = 0;                                              \
+        unsigned char *__child_up = alloca(__priv->child_count);               \
+                                                                               \
+        memcpy(__child_up, __priv->child_up,                                   \
+               sizeof(*__child_up) * __priv->child_count);                     \
+        __count = AFR_COUNT(__child_up, __priv->child_count);                  \
+                                                                               \
+        __local->barrier.waitfor = __count;                                    \
+        afr_local_replies_wipe(__local, __priv);                               \
+                                                                               \
+        for (__i = 0; __i < __priv->child_count; __i++) {                      \
+            if (!__child_up[__i])                                              \
+                continue;                                                      \
+            STACK_WIND_COOKIE(frame, rfn, (void *)(long)__i,                   \
+                              __priv->children[__i],                           \
+                              __priv->children[__i]->fops->fop, args);         \
+        }                                                                      \
+        syncbarrier_wait(&__local->barrier, __count);                          \
+    } while (0)
 
 /* Perform fop on all subvolumes represented by list[] array and wait
    for all callbacks to return */
 
-#define AFR_ONLIST(list, frame, rfn, fop, args ...) do {		\
-	afr_local_t *__local = frame->local;				\
-	afr_private_t *__priv = frame->this->private;			\
-	int __i = 0, __count = 0;					\
-									\
-	afr_local_replies_wipe (__local, __priv);				\
-									\
-	for (__i = 0; __i < __priv->child_count; __i++) {		\
-		if (!list[__i]) continue;				\
-		STACK_WIND_COOKIE (frame, rfn, (void *)(long) __i,	\
-				   __priv->children[__i],		\
-				   __priv->children[__i]->fops->fop, args); \
-		__count++;						\
-	}								\
-	syncbarrier_wait (&__local->barrier, __count);			\
-	} while (0)
-
-
-#define AFR_SEQ(frame, rfn, fop, args ...) do {				\
-	afr_local_t *__local = frame->local;				\
-	afr_private_t *__priv = frame->this->private;			\
-	int __i = 0;							\
-									\
-	afr_local_replies_wipe (__local, __priv);				\
-									\
-	for (__i = 0; __i < __priv->child_count; __i++) {		\
-		if (!__priv->child_up[__i]) continue;			\
-		STACK_WIND_COOKIE (frame, rfn, (void *)(long) __i,	\
-				   __priv->children[__i],		\
-				   __priv->children[__i]->fops->fop, args); \
-		syncbarrier_wait (&__local->barrier, 1);		\
-	}								\
-	} while (0)
-
-
-#define ALLOC_MATRIX(n, type) ({type **__ptr = NULL; \
-	int __i; \
-	__ptr = alloca0 (n * sizeof(type *)); \
-	for (__i = 0; __i < n; __i++) __ptr[__i] = alloca0 (n * sizeof(type)); \
-	__ptr;})
-
+#define AFR_ONLIST(list, frame, rfn, fop, args...)                             \
+    do {                                                                       \
+        afr_local_t *__local = frame->local;                                   \
+        afr_private_t *__priv = frame->this->private;                          \
+        int __i = 0;                                                           \
+        int __count = 0;                                                       \
+        unsigned char *__list = alloca(__priv->child_count);                   \
+                                                                               \
+        memcpy(__list, list, sizeof(*__list) * __priv->child_count);           \
+        __count = AFR_COUNT(__list, __priv->child_count);                      \
+        __local->barrier.waitfor = __count;                                    \
+        afr_local_replies_wipe(__local, __priv);                               \
+                                                                               \
+        for (__i = 0; __i < __priv->child_count; __i++) {                      \
+            if (!__list[__i])                                                  \
+                continue;                                                      \
+            STACK_WIND_COOKIE(frame, rfn, (void *)(long)__i,                   \
+                              __priv->children[__i],                           \
+                              __priv->children[__i]->fops->fop, args);         \
+        }                                                                      \
+        syncbarrier_wait(&__local->barrier, __count);                          \
+    } while (0)
+
+#define AFR_SEQ(frame, rfn, fop, args...)                                      \
+    do {                                                                       \
+        afr_local_t *__local = frame->local;                                   \
+        afr_private_t *__priv = frame->this->private;                          \
+        int __i = 0;                                                           \
+                                                                               \
+        afr_local_replies_wipe(__local, __priv);                               \
+                                                                               \
+        for (__i = 0; __i < __priv->child_count; __i++) {                      \
+            if (!__priv->child_up[__i])                                        \
+                continue;                                                      \
+            STACK_WIND_COOKIE(frame, rfn, (void *)(long)__i,                   \
+                              __priv->children[__i],                           \
+                              __priv->children[__i]->fops->fop, args);         \
+            syncbarrier_wait(&__local->barrier, 1);                            \
+        }                                                                      \
+    } while (0)
+
+#define ALLOC_MATRIX(n, type)                                                  \
+    ({                                                                         \
+        int __i;                                                               \
+        type **__ptr = alloca(n * sizeof(type *));                             \
+                                                                               \
+        for (__i = 0; __i < n; __i++)                                          \
+            __ptr[__i] = alloca0(n * sizeof(type));                            \
+        __ptr;                                                                 \
+    })
+
+#define IA_EQUAL(f, s, field)                                                  \
+    (memcmp(&(f.ia_##field), &(s.ia_##field), sizeof(s.ia_##field)) == 0)
+
+#define SBRAIN_HEAL_NO_GO_MSG                                                  \
+    "Failed to obtain replies from all bricks of "                             \
+    "the replica (are they up?). Cannot resolve split-brain."
+#define SFILE_NOT_IN_SPLIT_BRAIN "File not in split-brain"
+#define SNO_BIGGER_FILE "No bigger file"
+#define SNO_DIFF_IN_MTIME "No difference in mtime"
+#define SUSE_SOURCE_BRICK_TO_HEAL                                              \
+    "Use source-brick option to heal metadata"                                 \
+    " split-brain"
+#define SINVALID_BRICK_NAME "Invalid brick name"
+#define SBRICK_IS_NOT_UP "Brick is not up"
+#define SBRICK_NOT_CONNECTED "Brick is not connected"
+#define SLESS_THAN2_BRICKS_in_REP "< 2 bricks in replica are up"
+#define SBRICK_IS_REMOTE "Brick is remote"
+#define SSTARTED_SELF_HEAL "Started self-heal"
+#define SOP_NOT_SUPPORTED "Operation Not Supported"
+#define SFILE_NOT_UNDER_DATA                                                   \
+    "The file is not under data or metadata "                                  \
+    "split-brain"
+#define SFILE_NOT_IN_SPLIT_BRAIN "File not in split-brain"
+#define SALL_BRICKS_UP_TO_RESOLVE                                              \
+    "All the bricks should be up to resolve the"                               \
+    " gfid split brain"
+#define SERROR_GETTING_SRC_BRICK "Error getting the source brick"
+int
+afr_selfheal(xlator_t *this, uuid_t gfid);
 
-#define IA_EQUAL(f,s,field) (memcmp (&(f.ia_##field), &(s.ia_##field), sizeof (s.ia_##field)) == 0)
+gf_boolean_t
+afr_throttled_selfheal(call_frame_t *frame, xlator_t *this);
 
+int
+afr_selfheal_name(xlator_t *this, uuid_t gfid, const char *name, void *gfid_req,
+                  dict_t *xdata);
 
 int
-afr_selfheal (xlator_t *this, uuid_t gfid);
+afr_selfheal_data(call_frame_t *frame, xlator_t *this, fd_t *fd);
 
 int
-afr_selfheal_name (xlator_t *this, uuid_t gfid, const char *name,
-                   void *gfid_req);
+afr_selfheal_metadata(call_frame_t *frame, xlator_t *this, inode_t *inode);
 
 int
-afr_selfheal_data (call_frame_t *frame, xlator_t *this, inode_t *inode);
+afr_selfheal_entry(call_frame_t *frame, xlator_t *this, inode_t *inode);
 
 int
-afr_selfheal_metadata (call_frame_t *frame, xlator_t *this, inode_t *inode);
+afr_lookup_and_heal_gfid(xlator_t *this, inode_t *parent, const char *name,
+                         inode_t *inode, struct afr_reply *replies, int source,
+                         unsigned char *sources, void *gfid, int *gfid_idx);
 
 int
-afr_selfheal_entry (call_frame_t *frame, xlator_t *this, inode_t *inode);
+afr_selfheal_inodelk(call_frame_t *frame, xlator_t *this, inode_t *inode,
+                     char *dom, off_t off, size_t size,
+                     unsigned char *locked_on);
 
+int
+afr_selfheal_tryinodelk(call_frame_t *frame, xlator_t *this, inode_t *inode,
+                        char *dom, off_t off, size_t size,
+                        unsigned char *locked_on);
 
 int
-afr_selfheal_inodelk (call_frame_t *frame, xlator_t *this, inode_t *inode,
-		      char *dom, off_t off, size_t size,
-		      unsigned char *locked_on);
+afr_selfheal_tie_breaker_inodelk(call_frame_t *frame, xlator_t *this,
+                                 inode_t *inode, char *dom, off_t off,
+                                 size_t size, unsigned char *locked_on);
 
 int
-afr_selfheal_tryinodelk (call_frame_t *frame, xlator_t *this, inode_t *inode,
-			 char *dom, off_t off, size_t size,
-			 unsigned char *locked_on);
+afr_selfheal_uninodelk(call_frame_t *frame, xlator_t *this, inode_t *inode,
+                       char *dom, off_t off, size_t size,
+                       const unsigned char *locked_on);
 
 int
-afr_selfheal_uninodelk (call_frame_t *frame, xlator_t *this, inode_t *inode,
-			char *dom, off_t off, size_t size,
-			const unsigned char *locked_on);
+afr_selfheal_entrylk(call_frame_t *frame, xlator_t *this, inode_t *inode,
+                     char *dom, const char *name, unsigned char *locked_on);
 
 int
-afr_selfheal_entrylk (call_frame_t *frame, xlator_t *this, inode_t *inode,
-		      char *dom, const char *name, unsigned char *locked_on);
+afr_selfheal_tryentrylk(call_frame_t *frame, xlator_t *this, inode_t *inode,
+                        char *dom, const char *name, unsigned char *locked_on);
 
 int
-afr_selfheal_tryentrylk (call_frame_t *frame, xlator_t *this, inode_t *inode,
-			 char *dom, const char *name, unsigned char *locked_on);
+afr_selfheal_tie_breaker_entrylk(call_frame_t *frame, xlator_t *this,
+                                 inode_t *inode, char *dom, const char *name,
+                                 unsigned char *locked_on);
 
 int
-afr_selfheal_unentrylk (call_frame_t *frame, xlator_t *this, inode_t *inode,
-			char *dom, const char *name, unsigned char *locked_on);
+afr_selfheal_unentrylk(call_frame_t *frame, xlator_t *this, inode_t *inode,
+                       char *dom, const char *name, unsigned char *locked_on,
+                       dict_t *xdata);
 
 int
-afr_selfheal_unlocked_discover (call_frame_t *frame, inode_t *inode,
-				uuid_t gfid, struct afr_reply *replies);
+afr_selfheal_unlocked_discover(call_frame_t *frame, inode_t *inode, uuid_t gfid,
+                               struct afr_reply *replies);
 
+int
+afr_selfheal_unlocked_discover_on(call_frame_t *frame, inode_t *inode,
+                                  uuid_t gfid, struct afr_reply *replies,
+                                  unsigned char *discover_on, dict_t *dict);
 inode_t *
-afr_selfheal_unlocked_lookup_on (call_frame_t *frame, inode_t *parent,
-				 const char *name, struct afr_reply *replies,
-				 unsigned char *lookup_on, dict_t *xattr);
+afr_selfheal_unlocked_lookup_on(call_frame_t *frame, inode_t *parent,
+                                const char *name, struct afr_reply *replies,
+                                unsigned char *lookup_on, dict_t *xattr);
+
+int
+afr_selfheal_find_direction(call_frame_t *frame, xlator_t *this,
+                            struct afr_reply *replies,
+                            afr_transaction_type type, unsigned char *locked_on,
+                            unsigned char *sources, unsigned char *sinks,
+                            uint64_t *witness, unsigned char *flag);
+int
+afr_selfheal_fill_matrix(xlator_t *this, int **matrix, int subvol, int idx,
+                         dict_t *xdata);
 
 int
-afr_selfheal_find_direction (call_frame_t *frame, xlator_t *this,
-                             struct afr_reply *replies,
-                             afr_transaction_type type,
-                             unsigned char *locked_on, unsigned char *sources,
-                             unsigned char *sinks, uint64_t *witness);
+afr_selfheal_extract_xattr(xlator_t *this, struct afr_reply *replies,
+                           afr_transaction_type type, int *dirty, int **matrix);
 
 int
-afr_selfheal_extract_xattr (xlator_t *this, struct afr_reply *replies,
-			    afr_transaction_type type, int *dirty, int **matrix);
+afr_sh_generic_fop_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                       int op_ret, int op_errno, struct iatt *pre,
+                       struct iatt *post, dict_t *xdata);
 
 int
-afr_selfheal_undo_pending (call_frame_t *frame, xlator_t *this, inode_t *inode,
-			   unsigned char *sources, unsigned char *sinks,
-			   unsigned char *healed_sinks, afr_transaction_type type,
-			   struct afr_reply *replies, unsigned char *locked_on);
+afr_selfheal_restore_time(call_frame_t *frame, xlator_t *this, inode_t *inode,
+                          int source, unsigned char *healed_sinks,
+                          struct afr_reply *replies);
+int
+afr_selfheal_undo_pending(call_frame_t *frame, xlator_t *this, inode_t *inode,
+                          unsigned char *sources, unsigned char *sinks,
+                          unsigned char *healed_sinks,
+                          unsigned char *undid_pending,
+                          afr_transaction_type type, struct afr_reply *replies,
+                          unsigned char *locked_on);
 
 int
-afr_selfheal_recreate_entry (xlator_t *this, int dst, int source, inode_t *dir,
-                             const char *name, inode_t *inode,
-                             struct afr_reply *replies,
-                             unsigned char *newentry);
+afr_selfheal_recreate_entry(call_frame_t *frame, int dst, int source,
+                            unsigned char *sources, inode_t *dir,
+                            const char *name, inode_t *inode,
+                            struct afr_reply *replies);
 
 int
-afr_selfheal_post_op (call_frame_t *frame, xlator_t *this, inode_t *inode,
-		      int subvol, dict_t *xattr);
+afr_selfheal_post_op(call_frame_t *frame, xlator_t *this, inode_t *inode,
+                     int subvol, dict_t *xattr, dict_t *xdata);
 
 call_frame_t *
-afr_frame_create (xlator_t *this);
+afr_frame_create(xlator_t *this, int32_t *op_errno);
 
 inode_t *
-afr_inode_find (xlator_t *this, uuid_t gfid);
+afr_inode_find(xlator_t *this, uuid_t gfid);
 
 int
-afr_selfheal_discover_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-			   int op_ret, int op_errno, inode_t *inode,
-			   struct iatt *buf, dict_t *xdata,
-                           struct iatt *parbuf);
+afr_selfheal_discover_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                          int op_ret, int op_errno, inode_t *inode,
+                          struct iatt *buf, dict_t *xdata, struct iatt *parbuf);
+void
+afr_reply_copy(struct afr_reply *dst, struct afr_reply *src);
 
 void
-afr_replies_copy (struct afr_reply *dst, struct afr_reply *src, int count);
+afr_replies_copy(struct afr_reply *dst, struct afr_reply *src, int count);
 
 int
-afr_selfheal_newentry_mark (call_frame_t *frame, xlator_t *this, inode_t *inode,
-                            int source, struct afr_reply *replies,
-                            unsigned char *sources, unsigned char *newentry);
-
-inode_t*
-afr_inode_link (inode_t *inode, struct iatt *iatt);
+afr_selfheal_newentry_mark(call_frame_t *frame, xlator_t *this, inode_t *inode,
+                           int source, struct afr_reply *replies,
+                           unsigned char *sources, unsigned char *newentry);
 
 unsigned int
-afr_success_count (struct afr_reply *replies, unsigned int count);
+afr_success_count(struct afr_reply *replies, unsigned int count);
 
 void
-afr_log_selfheal (uuid_t gfid, xlator_t *this, int ret, char *type,
-                  int source, unsigned char *healed_sinks);
+afr_log_selfheal(uuid_t gfid, xlator_t *this, int ret, char *type, int source,
+                 unsigned char *sources, unsigned char *healed_sinks);
 
 void
-afr_mark_largest_file_as_source (xlator_t *this, unsigned char *sources,
-                                 struct afr_reply *replies);
+afr_mark_largest_file_as_source(xlator_t *this, unsigned char *sources,
+                                struct afr_reply *replies);
 void
-afr_mark_active_sinks (xlator_t *this, unsigned char *sources,
-                       unsigned char *locked_on, unsigned char *sinks);
+afr_mark_active_sinks(xlator_t *this, unsigned char *sources,
+                      unsigned char *locked_on, unsigned char *sinks);
+
+gf_boolean_t
+afr_dict_contains_heal_op(call_frame_t *frame);
 
 gf_boolean_t
-afr_dict_contains_heal_op (call_frame_t *frame);
+afr_can_decide_split_brain_source_sinks(struct afr_reply *replies,
+                                        int child_count);
+int
+afr_mark_split_brain_source_sinks(
+    call_frame_t *frame, xlator_t *this, inode_t *inode, unsigned char *sources,
+    unsigned char *sinks, unsigned char *healed_sinks, unsigned char *locked_on,
+    struct afr_reply *replies, afr_transaction_type type);
+
+int
+afr_sh_get_fav_by_policy(xlator_t *this, struct afr_reply *replies,
+                         inode_t *inode, char **policy_str);
 
 int
-afr_mark_split_brain_source_sinks (call_frame_t *frame, xlator_t *this,
-                                   unsigned char *sources,
-                                   unsigned char *sinks,
-                                   unsigned char *healed_sinks,
-                                   unsigned char *locked_on,
-                                   struct afr_reply *replies,
-                                   afr_transaction_type type);
+_afr_fav_child_reset_sink_xattrs(call_frame_t *frame, xlator_t *this,
+                                 inode_t *inode, int source,
+                                 unsigned char *healed_sinks,
+                                 unsigned char *undid_pending,
+                                 afr_transaction_type type,
+                                 unsigned char *locked_on,
+                                 struct afr_reply *replies);
 
 int
-afr_get_child_index_from_name (xlator_t *this, char *name);
+afr_get_child_index_from_name(xlator_t *this, char *name);
 
 gf_boolean_t
-afr_does_witness_exist (xlator_t *this, uint64_t *witness);
+afr_does_witness_exist(xlator_t *this, uint64_t *witness);
 
 int
-__afr_selfheal_data_prepare (call_frame_t *frame, xlator_t *this,
+__afr_selfheal_data_prepare(call_frame_t *frame, xlator_t *this, inode_t *inode,
+                            unsigned char *locked_on, unsigned char *sources,
+                            unsigned char *sinks, unsigned char *healed_sinks,
+                            unsigned char *undid_pending,
+                            struct afr_reply *replies, unsigned char *flag);
+
+int
+__afr_selfheal_metadata_prepare(call_frame_t *frame, xlator_t *this,
+                                inode_t *inode, unsigned char *locked_on,
+                                unsigned char *sources, unsigned char *sinks,
+                                unsigned char *healed_sinks,
+                                unsigned char *undid_pending,
+                                struct afr_reply *replies, unsigned char *flag);
+int
+__afr_selfheal_entry_prepare(call_frame_t *frame, xlator_t *this,
                              inode_t *inode, unsigned char *locked_on,
-                             unsigned char *sources,
-                             unsigned char *sinks, unsigned char *healed_sinks,
-                             struct afr_reply *replies);
+                             unsigned char *sources, unsigned char *sinks,
+                             unsigned char *healed_sinks,
+                             struct afr_reply *replies, int *source_p,
+                             unsigned char *flag);
 
 int
-__afr_selfheal_metadata_prepare (call_frame_t *frame, xlator_t *this,
-                                 inode_t *inode, unsigned char *locked_on,
-                                 unsigned char *sources,
-                                 unsigned char *sinks,
-                                 unsigned char *healed_sinks,
-                                 struct afr_reply *replies);
+afr_selfheal_unlocked_inspect(call_frame_t *frame, xlator_t *this, uuid_t gfid,
+                              inode_t **link_inode, gf_boolean_t *data_selfheal,
+                              gf_boolean_t *metadata_selfheal,
+                              gf_boolean_t *entry_selfheal,
+                              struct afr_reply *replies);
+
+int
+afr_selfheal_do(call_frame_t *frame, xlator_t *this, uuid_t gfid);
+
+int
+afr_selfheal_lock_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                      int op_ret, int op_errno, dict_t *xdata);
+
+int
+afr_locked_fill(call_frame_t *frame, xlator_t *this, unsigned char *locked_on);
+int
+afr_choose_source_by_policy(afr_private_t *priv, unsigned char *sources,
+                            afr_transaction_type type);
+
+int
+afr_selfheal_metadata_by_stbuf(xlator_t *this, struct iatt *stbuf);
+
 int
-__afr_selfheal_entry_prepare (call_frame_t *frame, xlator_t *this,
-                              inode_t *inode, unsigned char *locked_on,
-                              unsigned char *sources,
-                              unsigned char *sinks,
-                              unsigned char *healed_sinks,
-                              struct afr_reply *replies, int *source_p);
+afr_sh_fav_by_size(xlator_t *this, struct afr_reply *replies, inode_t *inode);
+int
+afr_sh_fav_by_mtime(xlator_t *this, struct afr_reply *replies, inode_t *inode);
+int
+afr_sh_fav_by_ctime(xlator_t *this, struct afr_reply *replies, inode_t *inode);
 
 int
-afr_selfheal_unlocked_inspect (call_frame_t *frame, xlator_t *this,
-                               uuid_t gfid, inode_t **link_inode,
-                               gf_boolean_t *data_selfheal,
-                               gf_boolean_t *metadata_selfheal,
-                               gf_boolean_t *entry_selfheal);
+afr_gfid_split_brain_source(xlator_t *this, struct afr_reply *replies,
+                            inode_t *inode, uuid_t pargfid, const char *bname,
+                            int src_idx, int child_idx,
+                            unsigned char *locked_on, int *src, dict_t *xdata);
+int
+afr_mark_source_sinks_if_file_empty(xlator_t *this, unsigned char *sources,
+                                    unsigned char *sinks,
+                                    unsigned char *healed_sinks,
+                                    unsigned char *locked_on,
+                                    struct afr_reply *replies,
+                                    afr_transaction_type type);
 
+gf_boolean_t
+afr_is_file_empty_on_all_children(afr_private_t *priv,
+                                  struct afr_reply *replies);
+
+int
+afr_selfheal_entry_delete(xlator_t *this, inode_t *dir, const char *name,
+                          inode_t *inode, int child, struct afr_reply *replies);
 int
-afr_selfheal_do (call_frame_t *frame, xlator_t *this, uuid_t gfid);
+afr_anon_inode_create(xlator_t *this, int child, inode_t **linked_inode);
 #endif /* !_AFR_SELFHEAL_H */
diff --git a/xlators/cluster/afr/src/afr-self-heald.c b/xlators/cluster/afr/src/afr-self-heald.c
index fe9367597a2..109fd4b7421 100644
--- a/xlators/cluster/afr/src/afr-self-heald.c
+++ b/xlators/cluster/afr/src/afr-self-heald.c
@@ -8,1189 +8,1709 @@
   cases as published by the Free Software Foundation.
 */
 
-
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
-
 #include "afr.h"
 #include "afr-self-heal.h"
 #include "afr-self-heald.h"
 #include "protocol-common.h"
-#include "syncop-utils.h"
-
-#define SHD_INODE_LRU_LIMIT          2048
-#define AFR_EH_SPLIT_BRAIN_LIMIT     1024
-#define AFR_STATISTICS_HISTORY_SIZE    50
-
-
-#define ASSERT_LOCAL(this, healer)				\
-	if (!afr_shd_is_subvol_local(this, healer->subvol)) {	\
-		healer->local = _gf_false;			\
-		if (safe_break (healer)) {			\
-			break;					\
-		} else {					\
-			continue;				\
-		}						\
-	} else {						\
-		healer->local = _gf_true;			\
-	}
-
-
-#define NTH_INDEX_HEALER(this, n) &((((afr_private_t *)this->private))->shd.index_healers[n])
-#define NTH_FULL_HEALER(this, n) &((((afr_private_t *)this->private))->shd.full_healers[n])
+#include <glusterfs/syncop-utils.h>
+#include "afr-messages.h"
+#include <glusterfs/byte-order.h>
+
+#define AFR_EH_SPLIT_BRAIN_LIMIT 1024
+#define AFR_STATISTICS_HISTORY_SIZE 50
+
+#define ASSERT_LOCAL(this, healer)                                             \
+    if (!afr_shd_is_subvol_local(this, healer->subvol)) {                      \
+        healer->local = _gf_false;                                             \
+        if (safe_break(healer)) {                                              \
+            break;                                                             \
+        } else {                                                               \
+            continue;                                                          \
+        }                                                                      \
+    } else {                                                                   \
+        healer->local = _gf_true;                                              \
+    }
+
+#define NTH_INDEX_HEALER(this, n)                                              \
+    &((((afr_private_t *)this->private))->shd.index_healers[n])
+#define NTH_FULL_HEALER(this, n)                                               \
+    &((((afr_private_t *)this->private))->shd.full_healers[n])
 
 char *
-afr_subvol_name (xlator_t *this, int subvol)
+afr_subvol_name(xlator_t *this, int subvol)
 {
-	afr_private_t *priv = NULL;
+    afr_private_t *priv = NULL;
 
-	priv = this->private;
-	if (subvol < 0 || subvol > priv->child_count)
-		return NULL;
+    priv = this->private;
+    if (subvol < 0 || subvol > priv->child_count)
+        return NULL;
 
-	return priv->children[subvol]->name;
+    return priv->children[subvol]->name;
 }
 
-
 void
-afr_destroy_crawl_event_data (void *data)
+afr_destroy_crawl_event_data(void *data)
 {
-        return;
+    return;
 }
 
-
 void
-afr_destroy_shd_event_data (void *data)
+afr_destroy_shd_event_data(void *data)
 {
-	shd_event_t *shd_event = data;
-
-	if (!shd_event)
-		return;
-	GF_FREE (shd_event->path);
+    shd_event_t *shd_event = data;
 
+    if (!shd_event)
         return;
-}
+    GF_FREE(shd_event->path);
 
+    return;
+}
 
 gf_boolean_t
-afr_shd_is_subvol_local (xlator_t *this, int subvol)
+afr_shd_is_subvol_local(xlator_t *this, int subvol)
 {
-	afr_private_t *priv    = NULL;
-	gf_boolean_t  is_local = _gf_false;
-        loc_t         loc      = {0, };
-
-        loc.inode = this->itable->root;
-        gf_uuid_copy (loc.gfid, loc.inode->gfid);
-	priv = this->private;
-        syncop_is_subvol_local(priv->children[subvol], &loc, &is_local);
-        return is_local;
+    afr_private_t *priv = NULL;
+    gf_boolean_t is_local = _gf_false;
+    loc_t loc = {
+        0,
+    };
+
+    loc.inode = this->itable->root;
+    gf_uuid_copy(loc.gfid, loc.inode->gfid);
+    priv = this->private;
+    syncop_is_subvol_local(priv->children[subvol], &loc, &is_local);
+    return is_local;
 }
 
-
 int
-__afr_shd_healer_wait (struct subvol_healer *healer)
+__afr_shd_healer_wait(struct subvol_healer *healer)
 {
-	afr_private_t *priv = NULL;
-	struct timespec wait_till = {0, };
-	int ret = 0;
+    afr_private_t *priv = NULL;
+    struct timespec wait_till = {
+        0,
+    };
+    int ret = 0;
 
-	priv = healer->this->private;
+    priv = healer->this->private;
 
 disabled_loop:
-	wait_till.tv_sec = time (NULL) + priv->shd.timeout;
+    wait_till.tv_sec = gf_time() + priv->shd.timeout;
 
-	while (!healer->rerun) {
-		ret = pthread_cond_timedwait (&healer->cond,
-					      &healer->mutex,
-					      &wait_till);
-		if (ret == ETIMEDOUT)
-			break;
-	}
+    while (!healer->rerun) {
+        ret = pthread_cond_timedwait(&healer->cond, &healer->mutex, &wait_till);
+        if (ret == ETIMEDOUT)
+            break;
+    }
 
-	ret = healer->rerun;
-	healer->rerun = 0;
+    ret = healer->rerun;
+    healer->rerun = 0;
 
-	if (!priv->shd.enabled)
-		goto disabled_loop;
+    if (!priv->shd.enabled)
+        goto disabled_loop;
 
-	return ret;
+    return ret;
 }
 
-
 int
-afr_shd_healer_wait (struct subvol_healer *healer)
+afr_shd_healer_wait(struct subvol_healer *healer)
 {
-	int ret = 0;
+    int ret = 0;
 
-	pthread_mutex_lock (&healer->mutex);
-	{
-		ret = __afr_shd_healer_wait (healer);
-	}
-	pthread_mutex_unlock (&healer->mutex);
+    pthread_mutex_lock(&healer->mutex);
+    {
+        ret = __afr_shd_healer_wait(healer);
+    }
+    pthread_mutex_unlock(&healer->mutex);
 
-	return ret;
+    return ret;
 }
 
-
 gf_boolean_t
-safe_break (struct subvol_healer *healer)
+safe_break(struct subvol_healer *healer)
 {
-	gf_boolean_t ret = _gf_false;
+    gf_boolean_t ret = _gf_false;
 
-	pthread_mutex_lock (&healer->mutex);
-	{
-		if (healer->rerun)
-			goto unlock;
+    pthread_mutex_lock(&healer->mutex);
+    {
+        if (healer->rerun)
+            goto unlock;
 
-		healer->running = _gf_false;
-		ret = _gf_true;
-	}
+        healer->running = _gf_false;
+        ret = _gf_true;
+    }
 unlock:
-	pthread_mutex_unlock (&healer->mutex);
+    pthread_mutex_unlock(&healer->mutex);
 
-	return ret;
+    return ret;
 }
 
-
 inode_t *
-afr_shd_inode_find (xlator_t *this, xlator_t *subvol, uuid_t gfid)
+afr_shd_inode_find(xlator_t *this, xlator_t *subvol, uuid_t gfid)
 {
-	inode_t *inode = NULL;
-	int ret = 0;
-	loc_t loc = {0, };
-	struct iatt iatt = {0, };
-
-	inode = inode_find (this->itable, gfid);
-	if (inode) {
-                inode_lookup (inode);
-		goto out;
-        }
-
-	loc.inode = inode_new (this->itable);
-	if (!loc.inode)
-		goto out;
-	gf_uuid_copy (loc.gfid, gfid);
-
-	ret = syncop_lookup (subvol, &loc, &iatt, NULL, NULL, NULL);
-	if (ret < 0)
-		goto out;
-
-	inode = inode_link (loc.inode, NULL, NULL, &iatt);
-	if (inode)
-		inode_lookup (inode);
+    int ret = 0;
+    uint64_t val = IA_INVAL;
+    dict_t *xdata = NULL;
+    dict_t *rsp_dict = NULL;
+    inode_t *inode = NULL;
+
+    xdata = dict_new();
+    if (!xdata)
+        goto out;
+
+    ret = dict_set_int8(xdata, GF_INDEX_IA_TYPE_GET_REQ, 1);
+    if (ret)
+        goto out;
+
+    ret = syncop_inode_find(this, subvol, gfid, &inode, xdata, &rsp_dict);
+    if (ret < 0)
+        goto out;
+
+    if (rsp_dict) {
+        ret = dict_get_uint64(rsp_dict, GF_INDEX_IA_TYPE_GET_RSP, &val);
+        if (ret)
+            goto out;
+    }
+    ret = inode_ctx_set2(inode, subvol, 0, &val);
 out:
-	loc_wipe (&loc);
-	return inode;
+    if (ret && inode) {
+        inode_unref(inode);
+        inode = NULL;
+    }
+    if (xdata)
+        dict_unref(xdata);
+    if (rsp_dict)
+        dict_unref(rsp_dict);
+    return inode;
 }
 
-
-inode_t*
-afr_shd_index_inode (xlator_t *this, xlator_t *subvol)
+inode_t *
+afr_shd_index_inode(xlator_t *this, xlator_t *subvol, char *vgfid)
 {
-	loc_t rootloc = {0, };
-	inode_t *inode = NULL;
-	int ret = 0;
-	dict_t *xattr = NULL;
-	void *index_gfid = NULL;
+    loc_t rootloc = {
+        0,
+    };
+    inode_t *inode = NULL;
+    int ret = 0;
+    dict_t *xattr = NULL;
+    void *index_gfid = NULL;
 
-	rootloc.inode = inode_ref (this->itable->root);
-	gf_uuid_copy (rootloc.gfid, rootloc.inode->gfid);
+    rootloc.inode = inode_ref(this->itable->root);
+    gf_uuid_copy(rootloc.gfid, rootloc.inode->gfid);
 
-	ret = syncop_getxattr (subvol, &rootloc, &xattr,
-			       GF_XATTROP_INDEX_GFID, NULL, NULL);
-	if (ret || !xattr) {
-		errno = -ret;
-		goto out;
-	}
+    ret = syncop_getxattr(subvol, &rootloc, &xattr, vgfid, NULL, NULL);
+    if (ret || !xattr) {
+        errno = -ret;
+        goto out;
+    }
 
-	ret = dict_get_ptr (xattr, GF_XATTROP_INDEX_GFID, &index_gfid);
-	if (ret)
-		goto out;
+    ret = dict_get_ptr(xattr, vgfid, &index_gfid);
+    if (ret)
+        goto out;
 
-	gf_log (this->name, GF_LOG_DEBUG, "index-dir gfid for %s: %s",
-		subvol->name, uuid_utoa (index_gfid));
+    gf_msg_debug(this->name, 0, "%s dir gfid for %s: %s", vgfid, subvol->name,
+                 uuid_utoa(index_gfid));
 
-	inode = afr_shd_inode_find (this, subvol, index_gfid);
+    inode = afr_shd_inode_find(this, subvol, index_gfid);
 
 out:
-	loc_wipe (&rootloc);
+    loc_wipe(&rootloc);
 
-	if (xattr)
-		dict_unref (xattr);
+    if (xattr)
+        dict_unref(xattr);
 
-	return inode;
+    return inode;
 }
 
 int
-afr_shd_index_purge (xlator_t *subvol, inode_t *inode, char *name)
+afr_shd_entry_purge(xlator_t *subvol, inode_t *inode, char *name,
+                    ia_type_t type)
 {
-	loc_t loc = {0, };
-	int ret = 0;
+    int ret = 0;
+    loc_t loc = {
+        0,
+    };
 
-	loc.parent = inode_ref (inode);
-	loc.name = name;
+    loc.parent = inode_ref(inode);
+    loc.name = name;
 
-	ret = syncop_unlink (subvol, &loc, NULL, NULL);
+    if (IA_ISDIR(type))
+        ret = syncop_rmdir(subvol, &loc, 1, NULL, NULL);
+    else
+        ret = syncop_unlink(subvol, &loc, NULL, NULL);
 
-	loc_wipe (&loc);
-	return ret;
+    loc_wipe(&loc);
+    return ret;
 }
 
 void
-afr_shd_zero_xattrop (xlator_t *this, uuid_t gfid)
+afr_shd_zero_xattrop(xlator_t *this, uuid_t gfid)
 {
-
-        call_frame_t *frame = NULL;
-        inode_t *inode = NULL;
-        afr_private_t *priv = NULL;
-        dict_t  *xattr = NULL;
-        int ret = 0;
-        int i = 0;
-        int raw[AFR_NUM_CHANGE_LOGS] = {0};
-
-        priv = this->private;
-        frame = afr_frame_create (this);
-        if (!frame)
-                goto out;
-        inode = afr_inode_find (this, gfid);
-        if (!inode)
-                goto out;
-        xattr = dict_new();
-        if (!xattr)
-                goto out;
-        ret = dict_set_static_bin (xattr, AFR_DIRTY, raw,
-                                   sizeof(int) * AFR_NUM_CHANGE_LOGS);
+    call_frame_t *frame = NULL;
+    inode_t *inode = NULL;
+    afr_private_t *priv = NULL;
+    dict_t *xattr = NULL;
+    int ret = 0;
+    int i = 0;
+    int raw[AFR_NUM_CHANGE_LOGS] = {0};
+
+    priv = this->private;
+    frame = afr_frame_create(this, NULL);
+    if (!frame)
+        goto out;
+    inode = afr_inode_find(this, gfid);
+    if (!inode)
+        goto out;
+    xattr = dict_new();
+    if (!xattr)
+        goto out;
+    ret = dict_set_static_bin(xattr, AFR_DIRTY, raw,
+                              sizeof(int) * AFR_NUM_CHANGE_LOGS);
+    if (ret)
+        goto out;
+    for (i = 0; i < priv->child_count; i++) {
+        ret = dict_set_static_bin(xattr, priv->pending_key[i], raw,
+                                  sizeof(int) * AFR_NUM_CHANGE_LOGS);
         if (ret)
-                goto out;
-        for (i = 0; i < priv->child_count; i++) {
-                ret = dict_set_static_bin (xattr, priv->pending_key[i], raw,
-                                           sizeof(int) * AFR_NUM_CHANGE_LOGS);
-                if (ret)
-                        goto out;
-        }
+            goto out;
+    }
 
-        /*Send xattrop to all bricks. Doing a lookup to see if bricks are up or
-        * has valid repies for this gfid seems a bit of an overkill.*/
-        for (i = 0; i < priv->child_count; i++)
-                afr_selfheal_post_op (frame, this, inode, i, xattr);
+    /*Send xattrop to all bricks. Doing a lookup to see if bricks are up or
+     * has valid repies for this gfid seems a bit of an overkill.*/
+    for (i = 0; i < priv->child_count; i++)
+        afr_selfheal_post_op(frame, this, inode, i, xattr, NULL);
 
 out:
-        if (frame)
-                AFR_STACK_DESTROY (frame);
-        if (inode)
-                inode_unref (inode);
-        if (xattr)
-                dict_unref (xattr);
-        return;
+    if (frame)
+        AFR_STACK_DESTROY(frame);
+    if (inode)
+        inode_unref(inode);
+    if (xattr)
+        dict_unref(xattr);
+    return;
 }
 
 int
-afr_shd_selfheal_name (struct subvol_healer *healer, int child, uuid_t parent,
-		       const char *bname)
+afr_shd_selfheal_name(struct subvol_healer *healer, int child, uuid_t parent,
+                      const char *bname)
 {
-	int ret = -1;
+    int ret = -1;
 
-	ret = afr_selfheal_name (THIS, parent, bname, NULL);
+    ret = afr_selfheal_name(THIS, parent, bname, NULL, NULL);
 
-	return ret;
+    return ret;
 }
 
 int
-afr_shd_selfheal (struct subvol_healer *healer, int child, uuid_t gfid)
+afr_shd_selfheal(struct subvol_healer *healer, int child, uuid_t gfid)
 {
-	int ret = 0;
-	eh_t *eh = NULL;
-	afr_private_t *priv = NULL;
-	afr_self_heald_t *shd = NULL;
-	shd_event_t *shd_event = NULL;
-	char *path = NULL;
-	xlator_t *subvol = NULL;
-	xlator_t *this = NULL;
-	crawl_event_t *crawl_event = NULL;
-
-	this = healer->this;
-	priv = this->private;
-	shd = &priv->shd;
-	crawl_event = &healer->crawl_event;
-
-	subvol = priv->children[child];
-
-        //If this fails with ENOENT/ESTALE index is stale
-        ret = syncop_gfid_to_path (this->itable, subvol, gfid, &path);
-        if (ret < 0)
-                return ret;
-
-	ret = afr_selfheal (this, gfid);
-
-	if (ret == -EIO) {
-		eh = shd->split_brain;
-		crawl_event->split_brain_count++;
-	} else if (ret < 0) {
-		crawl_event->heal_failed_count++;
-	} else if (ret == 0) {
-		crawl_event->healed_count++;
-	}
-
-	if (eh) {
-		shd_event = GF_CALLOC (1, sizeof(*shd_event),
-				       gf_afr_mt_shd_event_t);
-		if (!shd_event)
-                        goto out;
-
-		shd_event->child = child;
-		shd_event->path = path;
-
-		if (eh_save_history (eh, shd_event) < 0)
-                        goto out;
-
-                shd_event = NULL;
-                path = NULL;
-	}
+    int ret = 0;
+    eh_t *eh = NULL;
+    afr_private_t *priv = NULL;
+    afr_self_heald_t *shd = NULL;
+    shd_event_t *shd_event = NULL;
+    char *path = NULL;
+    xlator_t *subvol = NULL;
+    xlator_t *this = NULL;
+    crawl_event_t *crawl_event = NULL;
+
+    this = healer->this;
+    priv = this->private;
+    shd = &priv->shd;
+    crawl_event = &healer->crawl_event;
+
+    subvol = priv->children[child];
+
+    // If this fails with ENOENT/ESTALE index is stale
+    ret = syncop_gfid_to_path(this->itable, subvol, gfid, &path);
+    if (ret < 0)
+        return ret;
+
+    ret = afr_selfheal(this, gfid);
+
+    LOCK(&priv->lock);
+    {
+        if (ret == -EIO) {
+            eh = shd->split_brain;
+            crawl_event->split_brain_count++;
+        } else if (ret < 0) {
+            crawl_event->heal_failed_count++;
+        } else if (ret == 0) {
+            crawl_event->healed_count++;
+        }
+    }
+    UNLOCK(&priv->lock);
+
+    if (eh) {
+        shd_event = GF_CALLOC(1, sizeof(*shd_event), gf_afr_mt_shd_event_t);
+        if (!shd_event)
+            goto out;
+
+        shd_event->child = child;
+        shd_event->path = path;
+
+        if (eh_save_history(eh, shd_event) < 0)
+            goto out;
+
+        shd_event = NULL;
+        path = NULL;
+    }
 out:
-        GF_FREE (shd_event);
-        GF_FREE (path);
-	return ret;
+    GF_FREE(shd_event);
+    GF_FREE(path);
+    return ret;
 }
 
-
 void
-afr_shd_sweep_prepare (struct subvol_healer *healer)
+afr_shd_sweep_prepare(struct subvol_healer *healer)
 {
-	crawl_event_t *event = NULL;
+    crawl_event_t *event = NULL;
 
-	event = &healer->crawl_event;
+    event = &healer->crawl_event;
 
-	event->healed_count = 0;
-	event->split_brain_count = 0;
-	event->heal_failed_count = 0;
+    event->healed_count = 0;
+    event->split_brain_count = 0;
+    event->heal_failed_count = 0;
 
-	time (&event->start_time);
-	event->end_time = 0;
+    event->start_time = gf_time();
+    event->end_time = 0;
+    _mask_cancellation();
 }
 
-
 void
-afr_shd_sweep_done (struct subvol_healer *healer)
+afr_shd_sweep_done(struct subvol_healer *healer)
 {
-	crawl_event_t *event = NULL;
-	crawl_event_t *history = NULL;
-	afr_self_heald_t *shd = NULL;
+    crawl_event_t *event = NULL;
+    crawl_event_t *history = NULL;
+    afr_self_heald_t *shd = NULL;
 
-	event = &healer->crawl_event;
-	shd = &(((afr_private_t *)healer->this->private)->shd);
+    event = &healer->crawl_event;
+    shd = &(((afr_private_t *)healer->this->private)->shd);
 
-	time (&event->end_time);
-	history = memdup (event, sizeof (*event));
-	event->start_time = 0;
+    event->end_time = gf_time();
+    history = gf_memdup(event, sizeof(*event));
+    event->start_time = 0;
 
-	if (!history)
-		return;
+    if (!history)
+        return;
 
-	if (eh_save_history (shd->statistics[healer->subvol], history) < 0)
-		GF_FREE (history);
+    if (eh_save_history(shd->statistics[healer->subvol], history) < 0)
+        GF_FREE(history);
+    _unmask_cancellation();
 }
 
 int
-afr_shd_index_heal (xlator_t *subvol, gf_dirent_t *entry, loc_t *parent,
-                    void *data)
+afr_shd_index_heal(xlator_t *subvol, gf_dirent_t *entry, loc_t *parent,
+                   void *data)
 {
-        struct subvol_healer *healer = data;
-        afr_private_t        *priv   = NULL;
-        uuid_t               gfid    = {0};
-        int                  ret     = 0;
+    struct subvol_healer *healer = data;
+    afr_private_t *priv = NULL;
+    uuid_t gfid = {0};
+    int ret = 0;
+    uint64_t val = IA_INVAL;
 
-        priv = healer->this->private;
-        if (!priv->shd.enabled)
-                return -EBUSY;
+    priv = healer->this->private;
+    if (!priv->shd.enabled)
+        return -EBUSY;
 
-        gf_log (healer->this->name, GF_LOG_DEBUG, "got entry: %s",
-                entry->d_name);
+    gf_msg_debug(healer->this->name, 0, "got entry: %s from %s", entry->d_name,
+                 priv->children[healer->subvol]->name);
 
-        ret = gf_uuid_parse (entry->d_name, gfid);
-        if (ret)
-                return 0;
+    ret = gf_uuid_parse(entry->d_name, gfid);
+    if (ret)
+        return 0;
 
-        ret = afr_shd_selfheal (healer, healer->subvol, gfid);
+    inode_ctx_get2(parent->inode, subvol, NULL, &val);
 
-        if (ret == -ENOENT || ret == -ESTALE)
-                afr_shd_index_purge (subvol, parent->inode, entry->d_name);
-        if (ret == 2)
-                /* If bricks crashed in pre-op after creating indices/xattrop
-                 * link but before setting afr changelogs, we end up with stale
-                 * xattrop links but zero changelogs. Remove such entries by
-                 * sending a post-op with zero changelogs.
-                 */
-                afr_shd_zero_xattrop (healer->this, gfid);
+    ret = afr_shd_selfheal(healer, healer->subvol, gfid);
 
-        return 0;
+    if (ret == -ENOENT || ret == -ESTALE)
+        afr_shd_entry_purge(subvol, parent->inode, entry->d_name, val);
+
+    if (ret == 2)
+        /* If bricks crashed in pre-op after creating indices/xattrop
+         * link but before setting afr changelogs, we end up with stale
+         * xattrop links but zero changelogs. Remove such entries by
+         * sending a post-op with zero changelogs.
+         */
+        afr_shd_zero_xattrop(healer->this, gfid);
+
+    return 0;
 }
 
 int
-afr_shd_index_sweep (struct subvol_healer *healer)
+afr_shd_index_sweep(struct subvol_healer *healer, char *vgfid)
 {
-	loc_t         loc     = {0};
-	afr_private_t *priv   = NULL;
-	int           ret     = 0;
-	xlator_t      *subvol = NULL;
+    loc_t loc = {0};
+    afr_private_t *priv = NULL;
+    int ret = 0;
+    xlator_t *subvol = NULL;
+    dict_t *xdata = NULL;
+    call_frame_t *frame = NULL;
+
+    priv = healer->this->private;
+    subvol = priv->children[healer->subvol];
+
+    frame = afr_frame_create(healer->this, &ret);
+    if (!frame) {
+        ret = -ret;
+        goto out;
+    }
+
+    loc.inode = afr_shd_index_inode(healer->this, subvol, vgfid);
+    if (!loc.inode) {
+        gf_msg(healer->this->name, GF_LOG_WARNING, 0,
+               AFR_MSG_INDEX_DIR_GET_FAILED, "unable to get index-dir on %s",
+               subvol->name);
+        ret = -errno;
+        goto out;
+    }
+
+    xdata = dict_new();
+    if (!xdata || dict_set_int32_sizen(xdata, "get-gfid-type", 1)) {
+        ret = -ENOMEM;
+        goto out;
+    }
+
+    ret = syncop_mt_dir_scan(frame, subvol, &loc, GF_CLIENT_PID_SELF_HEALD,
+                             healer, afr_shd_index_heal, xdata,
+                             priv->shd.max_threads, priv->shd.wait_qlength);
+
+    if (ret == 0)
+        ret = healer->crawl_event.healed_count;
 
-	priv = healer->this->private;
-	subvol = priv->children[healer->subvol];
+out:
+    loc_wipe(&loc);
 
-	loc.inode = afr_shd_index_inode (healer->this, subvol);
-	if (!loc.inode) {
-		gf_log (healer->this->name, GF_LOG_WARNING,
-			"unable to get index-dir on %s", subvol->name);
-		return -errno;
-	}
+    if (xdata)
+        dict_unref(xdata);
+    if (frame)
+        AFR_STACK_DESTROY(frame);
+    return ret;
+}
 
-        ret = syncop_dir_scan (subvol, &loc, GF_CLIENT_PID_AFR_SELF_HEALD,
-                               healer, afr_shd_index_heal);
+int
+afr_shd_index_sweep_all(struct subvol_healer *healer)
+{
+    int ret = 0;
+    int count = 0;
+
+    ret = afr_shd_index_sweep(healer, GF_XATTROP_INDEX_GFID);
+    if (ret < 0)
+        goto out;
+    count = ret;
+
+    ret = afr_shd_index_sweep(healer, GF_XATTROP_DIRTY_GFID);
+    if (ret < 0)
+        goto out;
+    count += ret;
+
+    ret = afr_shd_index_sweep(healer, GF_XATTROP_ENTRY_CHANGES_GFID);
+    if (ret < 0)
+        goto out;
+    count += ret;
+out:
+    if (ret < 0)
+        return ret;
+    else
+        return count;
+}
 
-        inode_forget (loc.inode, 1);
-        loc_wipe (&loc);
+int
+afr_shd_full_heal(xlator_t *subvol, gf_dirent_t *entry, loc_t *parent,
+                  void *data)
+{
+    struct subvol_healer *healer = data;
+    xlator_t *this = healer->this;
+    afr_private_t *priv = NULL;
+
+    priv = this->private;
+
+    if (this->cleanup_starting) {
+        return -ENOTCONN;
+    }
 
-        if (ret == 0)
-                ret = healer->crawl_event.healed_count;
+    if (!priv->shd.enabled)
+        return -EBUSY;
 
-	return ret;
+    afr_shd_selfheal_name(healer, healer->subvol, parent->inode->gfid,
+                          entry->d_name);
+
+    afr_shd_selfheal(healer, healer->subvol, entry->d_stat.ia_gfid);
+
+    return 0;
 }
 
 int
-afr_shd_full_heal (xlator_t *subvol, gf_dirent_t *entry, loc_t *parent,
-                   void *data)
+afr_shd_full_sweep(struct subvol_healer *healer, inode_t *inode)
 {
-        struct subvol_healer *healer = data;
-        xlator_t             *this   = healer->this;
-        afr_private_t        *priv   = NULL;
+    afr_private_t *priv = NULL;
+    loc_t loc = {0};
 
-        priv = this->private;
-        if (!priv->shd.enabled)
-                return -EBUSY;
+    priv = healer->this->private;
+    loc.inode = inode;
+    return syncop_ftw(priv->children[healer->subvol], &loc,
+                      GF_CLIENT_PID_SELF_HEALD, healer, afr_shd_full_heal);
+}
 
-        afr_shd_selfheal_name (healer, healer->subvol,
-                               parent->inode->gfid, entry->d_name);
+int
+afr_shd_fill_ta_loc(xlator_t *this, loc_t *loc)
+{
+    afr_private_t *priv = NULL;
+    struct iatt stbuf = {
+        0,
+    };
+    int ret = -1;
+
+    priv = this->private;
+    loc->parent = inode_ref(this->itable->root);
+    gf_uuid_copy(loc->pargfid, loc->parent->gfid);
+    loc->name = priv->pending_key[THIN_ARBITER_BRICK_INDEX];
+    loc->inode = inode_new(loc->parent->table);
+    GF_CHECK_ALLOC(loc->inode, ret, out);
+
+    if (!gf_uuid_is_null(priv->ta_gfid))
+        goto assign_gfid;
+
+    ret = syncop_lookup(priv->children[THIN_ARBITER_BRICK_INDEX], loc, &stbuf,
+                        0, 0, 0);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, -ret, AFR_MSG_THIN_ARB,
+               "Failed lookup on file %s.", loc->name);
+        goto out;
+    }
+
+    gf_uuid_copy(priv->ta_gfid, stbuf.ia_gfid);
+
+assign_gfid:
+    gf_uuid_copy(loc->gfid, priv->ta_gfid);
+    ret = 0;
 
-        afr_shd_selfheal (healer, healer->subvol, entry->d_stat.ia_gfid);
+out:
+    if (ret)
+        loc_wipe(loc);
 
-        return 0;
+    return ret;
 }
 
 int
-afr_shd_full_sweep (struct subvol_healer *healer, inode_t *inode)
+_afr_shd_ta_get_xattrs(xlator_t *this, loc_t *loc, dict_t **xdata)
 {
-        afr_private_t *priv = NULL;
-        loc_t          loc  = {0};
-
-        priv = healer->this->private;
-        loc.inode = inode;
-        return syncop_ftw (priv->children[healer->subvol], &loc,
-                           GF_CLIENT_PID_AFR_SELF_HEALD, healer,
-                           afr_shd_full_heal);
-}
+    afr_private_t *priv = NULL;
+    dict_t *xattr = NULL;
+    int raw[AFR_NUM_CHANGE_LOGS] = {
+        0,
+    };
+    int ret = -1;
+    int i = 0;
+
+    priv = this->private;
+
+    xattr = dict_new();
+    if (!xattr) {
+        gf_msg(this->name, GF_LOG_ERROR, ENOMEM, AFR_MSG_DICT_GET_FAILED,
+               "Failed to create dict.");
+        goto out;
+    }
+    for (i = 0; i < priv->child_count; i++) {
+        ret = dict_set_static_bin(xattr, priv->pending_key[i], &raw,
+                                  AFR_NUM_CHANGE_LOGS * sizeof(int));
+        if (ret)
+            goto out;
+    }
 
+    ret = syncop_xattrop(priv->children[THIN_ARBITER_BRICK_INDEX], loc,
+                         GF_XATTROP_ADD_ARRAY, xattr, NULL, xdata, NULL);
+    if (ret || !(*xdata)) {
+        gf_msg(this->name, GF_LOG_ERROR, -ret, AFR_MSG_THIN_ARB,
+               "Xattrop failed on %s.", loc->name);
+    }
 
-void *
-afr_shd_index_healer (void *data)
-{
-	struct subvol_healer *healer = NULL;
-	xlator_t *this = NULL;
-	int ret = 0;
-
-	healer = data;
-	THIS = this = healer->this;
-
-	for (;;) {
-		afr_shd_healer_wait (healer);
-
-		ASSERT_LOCAL(this, healer);
-
-		do {
-			gf_log (this->name, GF_LOG_DEBUG,
-				"starting index sweep on subvol %s",
-				afr_subvol_name (this, healer->subvol));
-
-			afr_shd_sweep_prepare (healer);
-
-			ret = afr_shd_index_sweep (healer);
-
-			afr_shd_sweep_done (healer);
-			/*
-			  As long as at least one gfid was
-			  healed, keep retrying. We may have
-			  just healed a directory and thereby
-			  created entries for other gfids which
-			  could not be healed thus far.
-			*/
-
-			gf_log (this->name, GF_LOG_DEBUG,
-				"finished index sweep on subvol %s",
-				afr_subvol_name (this, healer->subvol));
-			/*
-			  Give a pause before retrying to avoid a busy loop
-			  in case the only entry in index is because of
-			  an ongoing I/O.
-			*/
-			sleep (1);
-		} while (ret > 0);
-	}
-
-	return NULL;
-}
+out:
+    if (xattr)
+        dict_unref(xattr);
 
+    return ret;
+}
 
-void *
-afr_shd_full_healer (void *data)
+void
+afr_shd_ta_get_xattrs(xlator_t *this, loc_t *loc, struct subvol_healer *healer,
+                      dict_t **xdata)
 {
-	struct subvol_healer *healer = NULL;
-	xlator_t *this = NULL;
-	int run = 0;
+    int ret = 0;
+
+    loc_wipe(loc);
+    if (afr_shd_fill_ta_loc(this, loc)) {
+        gf_msg(this->name, GF_LOG_ERROR, -ret, AFR_MSG_THIN_ARB,
+               "Failed to populate thin-arbiter loc for: %s.", loc->name);
+        ret = -1;
+        goto out;
+    }
+
+    ret = afr_ta_post_op_lock(this, loc);
+    if (ret)
+        goto out;
+
+    ret = _afr_shd_ta_get_xattrs(this, loc, xdata);
+    if (ret) {
+        if (*xdata) {
+            dict_unref(*xdata);
+            *xdata = NULL;
+        }
+    }
 
-	healer = data;
-	THIS = this = healer->this;
+    afr_ta_post_op_unlock(this, loc);
 
-	for (;;) {
-		pthread_mutex_lock (&healer->mutex);
-		{
-			run = __afr_shd_healer_wait (healer);
-			if (!run)
-				healer->running = _gf_false;
-		}
-		pthread_mutex_unlock (&healer->mutex);
+out:
+    if (ret)
+        healer->rerun = 1;
+}
 
-		if (!run)
-			break;
+int
+afr_shd_ta_unset_xattrs(xlator_t *this, loc_t *loc, dict_t **xdata, int healer)
+{
+    afr_private_t *priv = NULL;
+    dict_t *xattr = NULL;
+    gf_boolean_t need_xattrop = _gf_false;
+    void *pending_raw = NULL;
+    int *raw = NULL;
+    int pending[AFR_NUM_CHANGE_LOGS] = {
+        0,
+    };
+    int i = 0;
+    int j = 0;
+    int val = 0;
+    int ret = -1;
+
+    priv = this->private;
+
+    xattr = dict_new();
+    if (!xattr) {
+        goto out;
+    }
+
+    for (i = 0; i < priv->child_count; i++) {
+        raw = GF_CALLOC(AFR_NUM_CHANGE_LOGS, sizeof(int), gf_afr_mt_int32_t);
+        if (!raw) {
+            goto out;
+        }
 
-		ASSERT_LOCAL(this, healer);
+        ret = dict_get_ptr(*xdata, priv->pending_key[i], &pending_raw);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, -ret, AFR_MSG_DICT_GET_FAILED,
+                   "Error getting value "
+                   "of pending key %s",
+                   priv->pending_key[i]);
+            GF_FREE(raw);
+            goto out;
+        }
 
-		gf_log (this->name, GF_LOG_INFO,
-			"starting full sweep on subvol %s",
-			afr_subvol_name (this, healer->subvol));
+        memcpy(pending, pending_raw, sizeof(pending));
+        for (j = 0; j < AFR_NUM_CHANGE_LOGS; j++) {
+            val = ntoh32(pending[j]);
+            if (val) {
+                if (i == healer) {
+                    gf_msg(this->name, GF_LOG_INFO, 0, AFR_MSG_THIN_ARB,
+                           "I am "
+                           "not the good shd. Skipping. "
+                           "SHD = %d.",
+                           healer);
+                    ret = 0;
+                    GF_FREE(raw);
+                    goto out;
+                }
+                need_xattrop = _gf_true;
+                raw[j] = hton32(-val);
+            }
+        }
 
-		afr_shd_sweep_prepare (healer);
+        ret = dict_set_bin(xattr, priv->pending_key[i], raw,
+                           AFR_NUM_CHANGE_LOGS * sizeof(int));
+        if (ret) {
+            GF_FREE(raw);
+            goto out;
+        }
 
-		afr_shd_full_sweep (healer, this->itable->root);
+        if (need_xattrop)
+            break;
+    }
 
-		afr_shd_sweep_done (healer);
+    if (!need_xattrop) {
+        ret = 0;
+        goto out;
+    }
 
-		gf_log (this->name, GF_LOG_INFO,
-			"finished full sweep on subvol %s",
-			afr_subvol_name (this, healer->subvol));
-	}
+    ret = syncop_xattrop(priv->children[THIN_ARBITER_BRICK_INDEX], loc,
+                         GF_XATTROP_ADD_ARRAY, xattr, NULL, NULL, NULL);
+    if (ret)
+        gf_msg(this->name, GF_LOG_ERROR, -ret, AFR_MSG_THIN_ARB,
+               "Xattrop failed.");
 
-	return NULL;
-}
+out:
+    if (xattr)
+        dict_unref(xattr);
 
+    return ret;
+}
 
-int
-afr_shd_healer_init (xlator_t *this, struct subvol_healer *healer)
+void
+afr_shd_ta_check_and_unset_xattrs(xlator_t *this, loc_t *loc,
+                                  struct subvol_healer *healer,
+                                  dict_t *pre_crawl_xdata)
 {
-	int ret = 0;
+    int ret_lock = 0;
+    int ret = 0;
+    dict_t *post_crawl_xdata = NULL;
 
-	ret = pthread_mutex_init (&healer->mutex, NULL);
-	if (ret)
-		goto out;
+    ret_lock = afr_ta_post_op_lock(this, loc);
+    if (ret_lock)
+        goto unref;
 
-	ret = pthread_cond_init (&healer->cond, NULL);
-	if (ret)
-		goto out;
-
-	healer->this = this;
-	healer->running = _gf_false;
-	healer->rerun = _gf_false;
-	healer->local = _gf_false;
-out:
-	return ret;
-}
+    ret = _afr_shd_ta_get_xattrs(this, loc, &post_crawl_xdata);
+    if (ret)
+        goto unref;
 
+    if (!are_dicts_equal(pre_crawl_xdata, post_crawl_xdata, NULL, NULL)) {
+        ret = -1;
+        goto unref;
+    }
 
-int
-afr_shd_healer_spawn (xlator_t *this, struct subvol_healer *healer,
-		      void *(threadfn)(void *))
-{
-	int ret = 0;
-
-	pthread_mutex_lock (&healer->mutex);
-	{
-		if (healer->running) {
-			pthread_cond_signal (&healer->cond);
-		} else {
-			ret = gf_thread_create (&healer->thread, NULL,
-						threadfn, healer);
-			if (ret)
-				goto unlock;
-			healer->running = 1;
-		}
-
-		healer->rerun = 1;
-	}
-unlock:
-	pthread_mutex_unlock (&healer->mutex);
+    ret = afr_shd_ta_unset_xattrs(this, loc, &post_crawl_xdata, healer->subvol);
 
-	return ret;
-}
+unref:
+    if (post_crawl_xdata) {
+        dict_unref(post_crawl_xdata);
+        post_crawl_xdata = NULL;
+    }
 
+    if (ret || ret_lock)
+        healer->rerun = 1;
 
-int
-afr_shd_full_healer_spawn (xlator_t *this, int subvol)
-{
-	return afr_shd_healer_spawn (this, NTH_FULL_HEALER (this, subvol),
-				     afr_shd_full_healer);
+    if (!ret_lock)
+        afr_ta_post_op_unlock(this, loc);
 }
 
-
-int
-afr_shd_index_healer_spawn (xlator_t *this, int subvol)
+gf_boolean_t
+afr_bricks_available_for_heal(afr_private_t *priv)
 {
-	return afr_shd_healer_spawn (this, NTH_INDEX_HEALER (this, subvol),
-				     afr_shd_index_healer);
-}
+    int up_children = 0;
 
+    up_children = __afr_get_up_children_count(priv);
+    if (up_children < 2) {
+        return _gf_false;
+    }
+    return _gf_true;
+}
 
-int
-afr_shd_dict_add_crawl_event (xlator_t *this, dict_t *output,
-			      crawl_event_t *crawl_event)
+static gf_boolean_t
+afr_shd_ta_needs_heal(xlator_t *this, struct subvol_healer *healer)
 {
-        int             ret = 0;
-        uint64_t        count = 0;
-        char            key[256] = {0};
-        int             xl_id = 0;
-        uint64_t        healed_count = 0;
-        uint64_t        split_brain_count = 0;
-        uint64_t        heal_failed_count = 0;
-        char            *start_time_str = 0;
-        char            *end_time_str = NULL;
-        char            *crawl_type = NULL;
-        int             progress = -1;
-	int             child = -1;
-
-	child = crawl_event->child;
-        healed_count = crawl_event->healed_count;
-        split_brain_count = crawl_event->split_brain_count;
-        heal_failed_count = crawl_event->heal_failed_count;
-        crawl_type = crawl_event->crawl_type;
-
-	if (!crawl_event->start_time)
-		goto out;
-
-        start_time_str = gf_strdup (ctime (&crawl_event->start_time));
-
-	if (crawl_event->end_time)
-		end_time_str = gf_strdup (ctime (&crawl_event->end_time));
-
-        ret = dict_get_int32 (output, this->name, &xl_id);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "xl does not have id");
-                goto out;
+    dict_t *xdata = NULL;
+    afr_private_t *priv = NULL;
+    loc_t loc = {
+        0,
+    };
+    int ret = -1;
+    int i = 0;
+    gf_boolean_t need_heal = _gf_false;
+
+    priv = this->private;
+
+    ret = afr_shd_fill_ta_loc(this, &loc);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, -ret, AFR_MSG_THIN_ARB,
+               "Failed to populate thin-arbiter loc for: %s.", loc.name);
+        healer->rerun = 1;
+        goto out;
+    }
+
+    if (_afr_shd_ta_get_xattrs(this, &loc, &xdata)) {
+        healer->rerun = 1;
+        goto out;
+    }
+
+    for (i = 0; i < priv->child_count; i++) {
+        if (afr_ta_dict_contains_pending_xattr(xdata, priv, i)) {
+            need_heal = _gf_true;
+            break;
         }
+    }
 
-        snprintf (key, sizeof (key), "statistics-%d-%d-count", xl_id, child);
-        ret = dict_get_uint64 (output, key, &count);
+out:
+    if (xdata)
+        dict_unref(xdata);
+    loc_wipe(&loc);
 
+    return need_heal;
+}
 
-        snprintf (key, sizeof (key), "statistics_healed_cnt-%d-%d-%"PRIu64,
-                  xl_id, child, count);
-        ret = dict_set_uint64(output, key, healed_count);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR,
-			"Could not add statistics_healed_count to outout");
-                goto out;
-	}
-
-        snprintf (key, sizeof (key), "statistics_sb_cnt-%d-%d-%"PRIu64,
-                  xl_id, child, count);
-        ret = dict_set_uint64 (output, key, split_brain_count);
-	if (ret) {
-                gf_log (this->name, GF_LOG_ERROR,
-			"Could not add statistics_split_brain_count to outout");
-                goto out;
+static int
+afr_shd_anon_inode_cleaner(xlator_t *subvol, gf_dirent_t *entry, loc_t *parent,
+                           void *data)
+{
+    struct subvol_healer *healer = data;
+    afr_private_t *priv = healer->this->private;
+    call_frame_t *frame = NULL;
+    afr_local_t *local = NULL;
+    int ret = 0;
+    loc_t loc = {0};
+    int count = 0;
+    int i = 0;
+    int op_errno = 0;
+    struct iatt *iatt = NULL;
+    gf_boolean_t multiple_links = _gf_false;
+    unsigned char *gfid_present = alloca0(priv->child_count);
+    unsigned char *entry_present = alloca0(priv->child_count);
+    char *type = "file";
+
+    frame = afr_frame_create(healer->this, &ret);
+    if (!frame) {
+        ret = -ret;
+        goto out;
+    }
+    local = frame->local;
+    if (AFR_COUNT(local->child_up, priv->child_count) != priv->child_count) {
+        gf_msg_debug(healer->this->name, 0,
+                     "Not all bricks are up. Skipping "
+                     "cleanup of %s on %s",
+                     entry->d_name, subvol->name);
+        ret = 0;
+        goto out;
+    }
+
+    loc.inode = inode_new(parent->inode->table);
+    if (!loc.inode) {
+        ret = -ENOMEM;
+        goto out;
+    }
+    ret = gf_uuid_parse(entry->d_name, loc.gfid);
+    if (ret) {
+        ret = 0;
+        goto out;
+    }
+    AFR_ONLIST(local->child_up, frame, afr_selfheal_discover_cbk, lookup, &loc,
+               NULL);
+    for (i = 0; i < priv->child_count; i++) {
+        if (local->replies[i].op_ret == 0) {
+            count++;
+            gfid_present[i] = 1;
+            iatt = &local->replies[i].poststat;
+            if (iatt->ia_type == IA_IFDIR) {
+                type = "dir";
+            }
+
+            if (i == healer->subvol) {
+                if (local->replies[i].poststat.ia_nlink > 1) {
+                    multiple_links = _gf_true;
+                }
+            }
+        } else if (local->replies[i].op_errno != ENOENT &&
+                   local->replies[i].op_errno != ESTALE) {
+            /*We don't have complete view. Skip the entry*/
+            gf_msg_debug(healer->this->name, local->replies[i].op_errno,
+                         "Skipping cleanup of %s on %s", entry->d_name,
+                         subvol->name);
+            ret = 0;
+            goto out;
         }
-
-        snprintf (key, sizeof (key), "statistics_crawl_type-%d-%d-%"PRIu64,
-                  xl_id, child, count);
-        ret = dict_set_str (output, key, crawl_type);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR,
-			"Could not add statistics_crawl_type to output");
-                goto out;
+    }
+
+    /*Inode is deleted from subvol*/
+    if (count == 1 || (iatt->ia_type != IA_IFDIR && multiple_links)) {
+        gf_msg(healer->this->name, GF_LOG_WARNING, 0,
+               AFR_MSG_EXPUNGING_FILE_OR_DIR, "expunging %s %s/%s on %s", type,
+               priv->anon_inode_name, entry->d_name, subvol->name);
+        ret = afr_shd_entry_purge(subvol, parent->inode, entry->d_name,
+                                  iatt->ia_type);
+        if (ret == -ENOENT || ret == -ESTALE)
+            ret = 0;
+    } else if (count > 1) {
+        loc_wipe(&loc);
+        loc.parent = inode_ref(parent->inode);
+        loc.name = entry->d_name;
+        loc.inode = inode_new(parent->inode->table);
+        if (!loc.inode) {
+            ret = -ENOMEM;
+            goto out;
         }
-
-        snprintf (key, sizeof (key), "statistics_heal_failed_cnt-%d-%d-%"PRIu64,
-                  xl_id, child, count);
-        ret = dict_set_uint64 (output, key, heal_failed_count);
-	if (ret) {
-                gf_log (this->name, GF_LOG_ERROR,
-			"Could not add statistics_healed_failed_count to outout");
+        AFR_ONLIST(local->child_up, frame, afr_selfheal_discover_cbk, lookup,
+                   &loc, NULL);
+        count = 0;
+        for (i = 0; i < priv->child_count; i++) {
+            if (local->replies[i].op_ret == 0) {
+                count++;
+                entry_present[i] = 1;
+                iatt = &local->replies[i].poststat;
+            } else if (local->replies[i].op_errno != ENOENT &&
+                       local->replies[i].op_errno != ESTALE) {
+                /*We don't have complete view. Skip the entry*/
+                gf_msg_debug(healer->this->name, local->replies[i].op_errno,
+                             "Skipping cleanup of %s on %s", entry->d_name,
+                             subvol->name);
+                ret = 0;
                 goto out;
+            }
         }
-
-        snprintf (key, sizeof (key), "statistics_strt_time-%d-%d-%"PRIu64,
-                  xl_id, child, count);
-        ret = dict_set_dynstr (output, key, start_time_str);
-	if (ret) {
-                gf_log (this->name, GF_LOG_ERROR,
-			"Could not add statistics_crawl_start_time to outout");
-                goto out;
-        } else {
-		start_time_str = NULL;
-	}
-
-	if (!end_time_str)
-                progress = 1;
-        else
-                progress = 0;
-
-        snprintf (key, sizeof (key), "statistics_end_time-%d-%d-%"PRIu64,
-                  xl_id, child, count);
-        if (!end_time_str)
-                end_time_str = gf_strdup ("Could not determine the end time");
-        ret = dict_set_dynstr (output, key, end_time_str);
-	if (ret) {
-                gf_log (this->name, GF_LOG_ERROR,
-			"Could not add statistics_crawl_end_time to outout");
+        for (i = 0; i < priv->child_count; i++) {
+            if (gfid_present[i] && !entry_present[i]) {
+                /*Entry is not anonymous on at least one subvol*/
+                gf_msg_debug(healer->this->name, 0,
+                             "Valid entry present on %s "
+                             "Skipping cleanup of %s on %s",
+                             priv->children[i]->name, entry->d_name,
+                             subvol->name);
+                ret = 0;
                 goto out;
-        } else {
-		end_time_str = NULL;
-	}
-
-        snprintf (key, sizeof (key), "statistics_inprogress-%d-%d-%"PRIu64,
-                  xl_id, child, count);
+            }
+        }
 
-        ret = dict_set_int32 (output, key, progress);
-	if (ret) {
-                gf_log (this->name, GF_LOG_ERROR,
-			"Could not add statistics_inprogress to outout");
-                goto out;
+        gf_msg(healer->this->name, GF_LOG_WARNING, 0,
+               AFR_MSG_EXPUNGING_FILE_OR_DIR,
+               "expunging %s %s/%s on all subvols", type, priv->anon_inode_name,
+               entry->d_name);
+        ret = 0;
+        for (i = 0; i < priv->child_count; i++) {
+            op_errno = -afr_shd_entry_purge(priv->children[i], loc.parent,
+                                            entry->d_name, iatt->ia_type);
+            if (op_errno != ENOENT && op_errno != ESTALE) {
+                ret |= -op_errno;
+            }
         }
+    }
 
-	snprintf (key, sizeof (key), "statistics-%d-%d-count", xl_id, child);
-	ret = dict_set_uint64 (output, key, count + 1);
-	if (ret) {
-                gf_log (this->name, GF_LOG_ERROR,
-			"Could not increment the counter.");
-                goto out;
-	}
 out:
-	GF_FREE (start_time_str);
-	GF_FREE (end_time_str);
-        return ret;
+    if (frame)
+        AFR_STACK_DESTROY(frame);
+    loc_wipe(&loc);
+    return ret;
 }
 
-
-int
-afr_shd_dict_add_path (xlator_t *this, dict_t *output, int child, char *path,
-		       struct timeval *tv)
+static void
+afr_cleanup_anon_inode_dir(struct subvol_healer *healer)
 {
-        int             ret = -1;
-        uint64_t        count = 0;
-        char            key[256] = {0};
-        int             xl_id = 0;
+    int ret = 0;
+    call_frame_t *frame = NULL;
+    afr_private_t *priv = healer->this->private;
+    loc_t loc = {0};
+
+    ret = afr_anon_inode_create(healer->this, healer->subvol, &loc.inode);
+    if (ret)
+        goto out;
+
+    frame = afr_frame_create(healer->this, &ret);
+    if (!frame) {
+        ret = -ret;
+        goto out;
+    }
+
+    ret = syncop_mt_dir_scan(frame, priv->children[healer->subvol], &loc,
+                             GF_CLIENT_PID_SELF_HEALD, healer,
+                             afr_shd_anon_inode_cleaner, NULL,
+                             priv->shd.max_threads, priv->shd.wait_qlength);
+out:
+    if (frame)
+        AFR_STACK_DESTROY(frame);
+    loc_wipe(&loc);
+    return;
+}
 
-        ret = dict_get_int32 (output, this->name, &xl_id);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "xl does not have id");
-                goto out;
+void *
+afr_shd_index_healer(void *data)
+{
+    struct subvol_healer *healer = NULL;
+    xlator_t *this = NULL;
+    int ret = 0;
+    afr_private_t *priv = NULL;
+    dict_t *pre_crawl_xdata = NULL;
+    loc_t loc = {
+        0,
+    };
+
+    healer = data;
+    THIS = this = healer->this;
+    priv = this->private;
+
+    for (;;) {
+        afr_shd_healer_wait(healer);
+
+        if (!afr_bricks_available_for_heal(priv))
+            continue;
+
+        ASSERT_LOCAL(this, healer);
+        priv->local[healer->subvol] = healer->local;
+
+        if (priv->thin_arbiter_count) {
+            if (afr_shd_ta_needs_heal(this, healer))
+                afr_shd_ta_get_xattrs(this, &loc, healer, &pre_crawl_xdata);
         }
 
-        snprintf (key, sizeof (key), "%d-%d-count", xl_id, child);
-        ret = dict_get_uint64 (output, key, &count);
-
-        snprintf (key, sizeof (key), "%d-%d-%"PRIu64, xl_id, child, count);
-	ret = dict_set_dynstr (output, key, path);
-
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "%s: Could not add to output",
-                        path);
-                goto out;
+        do {
+            gf_msg_debug(this->name, 0, "starting index sweep on subvol %s",
+                         afr_subvol_name(this, healer->subvol));
+
+            afr_shd_sweep_prepare(healer);
+
+            ret = afr_shd_index_sweep_all(healer);
+
+            afr_shd_sweep_done(healer);
+            /*
+              As long as at least one gfid was
+              healed, keep retrying. We may have
+              just healed a directory and thereby
+              created entries for other gfids which
+              could not be healed thus far.
+            */
+
+            gf_msg_debug(this->name, 0, "finished index sweep on subvol %s",
+                         afr_subvol_name(this, healer->subvol));
+            /*
+              Give a pause before retrying to avoid a busy loop
+              in case the only entry in index is because of
+              an ongoing I/O.
+            */
+            sleep(1);
+        } while (ret > 0);
+
+        if (ret == 0) {
+            afr_cleanup_anon_inode_dir(healer);
         }
 
-	if (tv) {
-		snprintf (key, sizeof (key), "%d-%d-%"PRIu64"-time", xl_id,
-			  child, count);
-		ret = dict_set_uint32 (output, key, tv->tv_sec);
-		if (ret) {
-			gf_log (this->name, GF_LOG_ERROR, "%s: Could not set time",
-				path);
-			goto out;
-		}
-	}
-
-        snprintf (key, sizeof (key), "%d-%d-count", xl_id, child);
+        if (ret == 0 && pre_crawl_xdata &&
+            !healer->crawl_event.heal_failed_count) {
+            afr_shd_ta_check_and_unset_xattrs(this, &loc, healer,
+                                              pre_crawl_xdata);
+        }
 
-        ret = dict_set_uint64 (output, key, count + 1);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "Could not increment count");
-                goto out;
+        if (pre_crawl_xdata) {
+            dict_unref(pre_crawl_xdata);
+            pre_crawl_xdata = NULL;
         }
+    }
 
-        ret = 0;
-out:
-        return ret;
+    return NULL;
 }
 
-int
-afr_shd_gather_entry (xlator_t *subvol, gf_dirent_t *entry, loc_t *parent,
-                      void *data)
+void *
+afr_shd_full_healer(void *data)
 {
-        dict_t        *output = data;
-        xlator_t      *this   = NULL;
-        afr_private_t *priv   = NULL;
-        char          *path   = NULL;
-        int           ret     = 0;
-        int           child   = 0;
-        uuid_t        gfid    = {0};
+    struct subvol_healer *healer = NULL;
+    xlator_t *this = NULL;
+    int run = 0;
+
+    healer = data;
+    THIS = this = healer->this;
+
+    for (;;) {
+        pthread_mutex_lock(&healer->mutex);
+        {
+            run = __afr_shd_healer_wait(healer);
+            if (!run)
+                healer->running = _gf_false;
+        }
+        pthread_mutex_unlock(&healer->mutex);
 
-        this = THIS;
-        priv = this->private;
+        if (!run)
+            break;
 
-        gf_log (this->name, GF_LOG_DEBUG, "got entry: %s",
-                entry->d_name);
+        ASSERT_LOCAL(this, healer);
 
-        ret = gf_uuid_parse (entry->d_name, gfid);
-        if (ret)
-                return 0;
+        gf_msg(this->name, GF_LOG_INFO, 0, AFR_MSG_SELF_HEAL_INFO,
+               "starting full sweep on subvol %s",
+               afr_subvol_name(this, healer->subvol));
 
-        for (child = 0; child < priv->child_count; child++)
-                if (priv->children[child] == subvol)
-                        break;
+        afr_shd_sweep_prepare(healer);
 
-        if (child == priv->child_count)
-                return 0;
+        afr_shd_full_sweep(healer, this->itable->root);
 
-        ret = syncop_gfid_to_path (this->itable, subvol, gfid, &path);
+        afr_shd_sweep_done(healer);
 
-        if (ret == -ENOENT || ret == -ESTALE) {
-                afr_shd_index_purge (subvol, parent->inode, entry->d_name);
-        } else if (ret == 0) {
-                ret = afr_shd_dict_add_path (this, output, child, path, NULL);
-        }
+        gf_msg(this->name, GF_LOG_INFO, 0, AFR_MSG_SELF_HEAL_INFO,
+               "finished full sweep on subvol %s",
+               afr_subvol_name(this, healer->subvol));
+    }
 
-        return 0;
+    return NULL;
 }
 
 int
-afr_shd_gather_index_entries (xlator_t *this, int child, dict_t *output)
+afr_shd_healer_init(xlator_t *this, struct subvol_healer *healer)
 {
-        loc_t          loc    = {0};
-        afr_private_t *priv   = NULL;
-        xlator_t      *subvol = NULL;
-        int           ret     = 0;
+    int ret = 0;
 
-        priv = this->private;
-        subvol = priv->children[child];
+    ret = pthread_mutex_init(&healer->mutex, NULL);
+    if (ret)
+        goto out;
 
-        loc.inode = afr_shd_index_inode (this, subvol);
-        if (!loc.inode) {
-                gf_log (this->name, GF_LOG_WARNING,
-                        "unable to get index-dir on %s", subvol->name);
-                return -errno;
+    ret = pthread_cond_init(&healer->cond, NULL);
+    if (ret)
+        goto out;
+
+    healer->this = this;
+    healer->running = _gf_false;
+    healer->rerun = _gf_false;
+    healer->local = _gf_false;
+out:
+    return ret;
+}
+
+int
+afr_shd_healer_spawn(xlator_t *this, struct subvol_healer *healer,
+                     void *(threadfn)(void *))
+{
+    int ret = 0;
+
+    pthread_mutex_lock(&healer->mutex);
+    {
+        if (healer->running) {
+            pthread_cond_signal(&healer->cond);
+        } else {
+            ret = gf_thread_create(&healer->thread, NULL, threadfn, healer,
+                                   "shdheal");
+            if (ret)
+                goto unlock;
+            healer->running = 1;
         }
 
-        ret = syncop_dir_scan (subvol, &loc, GF_CLIENT_PID_AFR_SELF_HEALD,
-                               output, afr_shd_gather_entry);
-        inode_forget (loc.inode, 1);
-        loc_wipe (&loc);
-        return ret;
+        healer->rerun = 1;
+    }
+unlock:
+    pthread_mutex_unlock(&healer->mutex);
+
+    return ret;
 }
 
+int
+afr_shd_full_healer_spawn(xlator_t *this, int subvol)
+{
+    return afr_shd_healer_spawn(this, NTH_FULL_HEALER(this, subvol),
+                                afr_shd_full_healer);
+}
 
 int
-afr_add_shd_event (circular_buffer_t *cb, void *data)
+afr_shd_index_healer_spawn(xlator_t *this, int subvol)
 {
-	dict_t *output = NULL;
-	xlator_t *this = THIS;
-	afr_private_t *priv = NULL;
-	afr_self_heald_t *shd = NULL;
-	shd_event_t *shd_event = NULL;
-	char *path = NULL;
-
-	output = data;
-	priv = this->private;
-	shd = &priv->shd;
-	shd_event = cb->data;
-
-	if (!shd->index_healers[shd_event->child].local)
-		return 0;
-
-	path = gf_strdup (shd_event->path);
-	if (!path)
-		return -ENOMEM;
-
-	afr_shd_dict_add_path (this, output, shd_event->child, path,
-			       &cb->tv);
-	return 0;
+    return afr_shd_healer_spawn(this, NTH_INDEX_HEALER(this, subvol),
+                                afr_shd_index_healer);
 }
 
 int
-afr_add_crawl_event (circular_buffer_t *cb, void *data)
+afr_shd_dict_add_crawl_event(xlator_t *this, dict_t *output,
+                             crawl_event_t *crawl_event)
 {
-	dict_t *output = NULL;
-	xlator_t *this = THIS;
-	afr_private_t *priv = NULL;
-	afr_self_heald_t *shd = NULL;
-	crawl_event_t *crawl_event = NULL;
+    int ret = 0;
+    uint64_t count = 0;
+    char key[128] = {0};
+    int keylen = 0;
+    char suffix[64] = {0};
+    int xl_id = 0;
+    uint64_t healed_count = 0;
+    uint64_t split_brain_count = 0;
+    uint64_t heal_failed_count = 0;
+    char *start_time_str = 0;
+    char *end_time_str = NULL;
+    char *crawl_type = NULL;
+    int progress = -1;
+    int child = -1;
+
+    child = crawl_event->child;
+    healed_count = crawl_event->healed_count;
+    split_brain_count = crawl_event->split_brain_count;
+    heal_failed_count = crawl_event->heal_failed_count;
+    crawl_type = crawl_event->crawl_type;
+
+    if (!crawl_event->start_time)
+        goto out;
+
+    start_time_str = gf_strdup(ctime(&crawl_event->start_time));
+
+    if (crawl_event->end_time)
+        end_time_str = gf_strdup(ctime(&crawl_event->end_time));
+
+    ret = dict_get_int32(output, this->name, &xl_id);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, -ret, AFR_MSG_DICT_GET_FAILED,
+               "xl does not have id");
+        goto out;
+    }
+
+    snprintf(key, sizeof(key), "statistics-%d-%d-count", xl_id, child);
+    ret = dict_get_uint64(output, key, &count);
+
+    snprintf(suffix, sizeof(suffix), "%d-%d-%" PRIu64, xl_id, child, count);
+    snprintf(key, sizeof(key), "statistics_healed_cnt-%s", suffix);
+    ret = dict_set_uint64(output, key, healed_count);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, -ret, AFR_MSG_DICT_SET_FAILED,
+               "Could not add statistics_healed_count to output");
+        goto out;
+    }
+
+    snprintf(key, sizeof(key), "statistics_sb_cnt-%s", suffix);
+    ret = dict_set_uint64(output, key, split_brain_count);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, -ret, AFR_MSG_DICT_SET_FAILED,
+               "Could not add statistics_split_brain_count to output");
+        goto out;
+    }
+
+    keylen = snprintf(key, sizeof(key), "statistics_crawl_type-%s", suffix);
+    ret = dict_set_strn(output, key, keylen, crawl_type);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, -ret, AFR_MSG_DICT_SET_FAILED,
+               "Could not add statistics_crawl_type to output");
+        goto out;
+    }
+
+    snprintf(key, sizeof(key), "statistics_heal_failed_cnt-%s", suffix);
+    ret = dict_set_uint64(output, key, heal_failed_count);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, -ret, AFR_MSG_DICT_SET_FAILED,
+               "Could not add statistics_healed_failed_count to output");
+        goto out;
+    }
+
+    keylen = snprintf(key, sizeof(key), "statistics_strt_time-%s", suffix);
+    ret = dict_set_dynstrn(output, key, keylen, start_time_str);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, -ret, AFR_MSG_DICT_SET_FAILED,
+               "Could not add statistics_crawl_start_time to output");
+        goto out;
+    } else {
+        start_time_str = NULL;
+    }
+
+    if (!end_time_str)
+        progress = 1;
+    else
+        progress = 0;
+
+    keylen = snprintf(key, sizeof(key), "statistics_end_time-%s", suffix);
+    if (!end_time_str)
+        end_time_str = gf_strdup("Could not determine the end time");
+    ret = dict_set_dynstrn(output, key, keylen, end_time_str);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, -ret, AFR_MSG_DICT_SET_FAILED,
+               "Could not add statistics_crawl_end_time to output");
+        goto out;
+    } else {
+        end_time_str = NULL;
+    }
+
+    keylen = snprintf(key, sizeof(key), "statistics_inprogress-%s", suffix);
+
+    ret = dict_set_int32n(output, key, keylen, progress);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, -ret, AFR_MSG_DICT_SET_FAILED,
+               "Could not add statistics_inprogress to output");
+        goto out;
+    }
+
+    snprintf(key, sizeof(key), "statistics-%d-%d-count", xl_id, child);
+    ret = dict_set_uint64(output, key, count + 1);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, -ret, AFR_MSG_DICT_SET_FAILED,
+               "Could not increment the counter.");
+        goto out;
+    }
+out:
+    GF_FREE(start_time_str);
+    GF_FREE(end_time_str);
+    return ret;
+}
 
-	output = data;
-	priv = this->private;
-	shd = &priv->shd;
-	crawl_event = cb->data;
+int
+afr_shd_dict_add_path(xlator_t *this, dict_t *output, int child, char *path,
+                      struct timeval *tv)
+{
+    int ret = -1;
+    uint64_t count = 0;
+    char key[64] = {0};
+    int keylen = 0;
+    char xl_id_child_str[32] = {0};
+    int xl_id = 0;
+
+    ret = dict_get_int32(output, this->name, &xl_id);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, -ret, AFR_MSG_DICT_GET_FAILED,
+               "xl does not have id");
+        goto out;
+    }
+
+    snprintf(xl_id_child_str, sizeof(xl_id_child_str), "%d-%d", xl_id, child);
+    snprintf(key, sizeof(key), "%s-count", xl_id_child_str);
+    ret = dict_get_uint64(output, key, &count);
+
+    keylen = snprintf(key, sizeof(key), "%s-%" PRIu64, xl_id_child_str, count);
+    ret = dict_set_dynstrn(output, key, keylen, path);
+
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, -ret, AFR_MSG_DICT_SET_FAILED,
+               "%s: Could not add to output", path);
+        goto out;
+    }
+
+    if (tv) {
+        snprintf(key, sizeof(key), "%s-%" PRIu64 "-time", xl_id_child_str,
+                 count);
+        ret = dict_set_uint32(output, key, tv->tv_sec);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, -ret, AFR_MSG_DICT_SET_FAILED,
+                   "%s: Could not set time", path);
+            goto out;
+        }
+    }
 
-	if (!shd->index_healers[crawl_event->child].local)
-		return 0;
+    snprintf(key, sizeof(key), "%s-count", xl_id_child_str);
 
-	afr_shd_dict_add_crawl_event (this, output, crawl_event);
+    ret = dict_set_uint64(output, key, count + 1);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, -ret, AFR_MSG_DICT_SET_FAILED,
+               "Could not increment count");
+        goto out;
+    }
 
-	return 0;
+    ret = 0;
+out:
+    return ret;
 }
 
-
 int
-afr_selfheal_daemon_init (xlator_t *this)
+afr_add_shd_event(circular_buffer_t *cb, void *data)
 {
-	afr_private_t *priv = NULL;
-	afr_self_heald_t *shd = NULL;
-	int ret = -1;
-	int i = 0;
-
-	priv = this->private;
-	shd = &priv->shd;
-
-	this->itable = inode_table_new (SHD_INODE_LRU_LIMIT, this);
-	if (!this->itable)
-		goto out;
-
-	shd->index_healers = GF_CALLOC (sizeof(*shd->index_healers),
-					priv->child_count,
-					gf_afr_mt_subvol_healer_t);
-	if (!shd->index_healers)
-		goto out;
-
-	for (i = 0; i < priv->child_count; i++) {
-		shd->index_healers[i].subvol = i;
-		ret = afr_shd_healer_init (this, &shd->index_healers[i]);
-		if (ret)
-			goto out;
-	}
-
-	shd->full_healers = GF_CALLOC (sizeof(*shd->full_healers),
-				       priv->child_count,
-				       gf_afr_mt_subvol_healer_t);
-	if (!shd->full_healers)
-		goto out;
-	for (i = 0; i < priv->child_count; i++) {
-		shd->full_healers[i].subvol = i;
-		ret = afr_shd_healer_init (this, &shd->full_healers[i]);
-		if (ret)
-			goto out;
-	}
-
-	shd->split_brain = eh_new (AFR_EH_SPLIT_BRAIN_LIMIT, _gf_false,
-				   afr_destroy_shd_event_data);
-	if (!shd->split_brain)
-		goto out;
-
-        shd->statistics = GF_CALLOC (sizeof(eh_t *), priv->child_count,
-				     gf_common_mt_eh_t);
-        if (!shd->statistics)
-                goto out;
+    dict_t *output = NULL;
+    xlator_t *this = THIS;
+    afr_private_t *priv = NULL;
+    afr_self_heald_t *shd = NULL;
+    shd_event_t *shd_event = NULL;
+    char *path = NULL;
+
+    output = data;
+    priv = this->private;
+    shd = &priv->shd;
+    shd_event = cb->data;
+
+    if (!shd->index_healers[shd_event->child].local)
+        return 0;
 
-        for (i = 0; i < priv->child_count ; i++) {
-                shd->statistics[i] = eh_new (AFR_STATISTICS_HISTORY_SIZE,
-					     _gf_false,
-					     afr_destroy_crawl_event_data);
-                if (!shd->statistics[i])
-                        goto out;
-		shd->full_healers[i].crawl_event.child = i;
-		shd->full_healers[i].crawl_event.crawl_type = "FULL";
-		shd->index_healers[i].crawl_event.child = i;
-		shd->index_healers[i].crawl_event.crawl_type = "INDEX";
-        }
+    path = gf_strdup(shd_event->path);
+    if (!path)
+        return -ENOMEM;
 
-	ret = 0;
-out:
-	return ret;
+    afr_shd_dict_add_path(this, output, shd_event->child, path, &cb->tv);
+    return 0;
 }
 
-
 int
-afr_selfheal_childup (xlator_t *this, int subvol)
+afr_add_crawl_event(circular_buffer_t *cb, void *data)
 {
-	afr_shd_index_healer_spawn (this, subvol);
+    dict_t *output = NULL;
+    xlator_t *this = THIS;
+    afr_private_t *priv = NULL;
+    afr_self_heald_t *shd = NULL;
+    crawl_event_t *crawl_event = NULL;
+
+    output = data;
+    priv = this->private;
+    shd = &priv->shd;
+    crawl_event = cb->data;
+
+    if (!shd->index_healers[crawl_event->child].local)
+        return 0;
+
+    afr_shd_dict_add_crawl_event(this, output, crawl_event);
 
-	return 0;
+    return 0;
 }
 
+int
+afr_selfheal_daemon_init(xlator_t *this)
+{
+    afr_private_t *priv = NULL;
+    afr_self_heald_t *shd = NULL;
+    int ret = -1;
+    int i = 0;
+
+    priv = this->private;
+    shd = &priv->shd;
+
+    shd->index_healers = GF_CALLOC(sizeof(*shd->index_healers),
+                                   priv->child_count,
+                                   gf_afr_mt_subvol_healer_t);
+    if (!shd->index_healers)
+        goto out;
+
+    for (i = 0; i < priv->child_count; i++) {
+        shd->index_healers[i].subvol = i;
+        ret = afr_shd_healer_init(this, &shd->index_healers[i]);
+        if (ret)
+            goto out;
+    }
+
+    shd->full_healers = GF_CALLOC(sizeof(*shd->full_healers), priv->child_count,
+                                  gf_afr_mt_subvol_healer_t);
+    if (!shd->full_healers)
+        goto out;
+    for (i = 0; i < priv->child_count; i++) {
+        shd->full_healers[i].subvol = i;
+        ret = afr_shd_healer_init(this, &shd->full_healers[i]);
+        if (ret)
+            goto out;
+    }
+
+    shd->split_brain = eh_new(AFR_EH_SPLIT_BRAIN_LIMIT, _gf_false,
+                              afr_destroy_shd_event_data);
+    if (!shd->split_brain)
+        goto out;
+
+    shd->statistics = GF_CALLOC(sizeof(eh_t *), priv->child_count,
+                                gf_common_mt_eh_t);
+    if (!shd->statistics)
+        goto out;
+
+    for (i = 0; i < priv->child_count; i++) {
+        shd->statistics[i] = eh_new(AFR_STATISTICS_HISTORY_SIZE, _gf_false,
+                                    afr_destroy_crawl_event_data);
+        if (!shd->statistics[i])
+            goto out;
+        shd->full_healers[i].crawl_event.child = i;
+        shd->full_healers[i].crawl_event.crawl_type = "FULL";
+        shd->index_healers[i].crawl_event.child = i;
+        shd->index_healers[i].crawl_event.crawl_type = "INDEX";
+    }
+
+    ret = 0;
+out:
+    return ret;
+}
+
+void
+afr_selfheal_childup(xlator_t *this, afr_private_t *priv)
+{
+    int subvol = 0;
+
+    if (!priv->shd.iamshd)
+        return;
+    for (subvol = 0; subvol < priv->child_count; subvol++)
+        if (priv->child_up[subvol])
+            afr_shd_index_healer_spawn(this, subvol);
+
+    return;
+}
 
 int
-afr_shd_get_index_count (xlator_t *this, int i, uint64_t *count)
+afr_shd_get_index_count(xlator_t *this, int i, uint64_t *count)
 {
-	afr_private_t *priv = NULL;
-	xlator_t *subvol = NULL;
-	loc_t rootloc = {0, };
-	dict_t *xattr = NULL;
-	int ret = -1;
+    afr_private_t *priv = NULL;
+    xlator_t *subvol = NULL;
+    loc_t rootloc = {
+        0,
+    };
+    dict_t *xattr = NULL;
+    int ret = -1;
 
-	priv = this->private;
-	subvol = priv->children[i];
+    priv = this->private;
+    subvol = priv->children[i];
 
-	rootloc.inode = inode_ref (this->itable->root);
-	gf_uuid_copy (rootloc.gfid, rootloc.inode->gfid);
+    rootloc.inode = inode_ref(this->itable->root);
+    gf_uuid_copy(rootloc.gfid, rootloc.inode->gfid);
 
-	ret = syncop_getxattr (subvol, &rootloc, &xattr,
-			       GF_XATTROP_INDEX_COUNT, NULL, NULL);
-	if (ret < 0)
-		goto out;
+    ret = syncop_getxattr(subvol, &rootloc, &xattr, GF_XATTROP_INDEX_COUNT,
+                          NULL, NULL);
+    if (ret < 0)
+        goto out;
 
-	ret = dict_get_uint64 (xattr, GF_XATTROP_INDEX_COUNT, count);
-	if (ret)
-		goto out;
+    ret = dict_get_uint64(xattr, GF_XATTROP_INDEX_COUNT, count);
+    if (ret)
+        goto out;
 
-        ret = 0;
+    ret = 0;
 
 out:
-        if (xattr)
-                dict_unref (xattr);
-	loc_wipe (&rootloc);
+    if (xattr)
+        dict_unref(xattr);
+    loc_wipe(&rootloc);
 
-	return ret;
+    return ret;
 }
 
-
 int
-afr_xl_op (xlator_t *this, dict_t *input, dict_t *output)
+afr_xl_op(xlator_t *this, dict_t *input, dict_t *output)
 {
-        gf_xl_afr_op_t   op = GF_SHD_OP_INVALID;
-        int              ret = 0;
-        int              xl_id = 0;
-	afr_private_t   *priv = NULL;
-	afr_self_heald_t *shd = NULL;
-	struct subvol_healer *healer = NULL;
-	int i = 0;
-	char key[64];
-	int op_ret = 0;
-	uint64_t cnt = 0;
-
-	priv = this->private;
-	shd = &priv->shd;
-
-        ret = dict_get_int32 (input, "xl-op", (int32_t*)&op);
-        if (ret)
-                goto out;
-        ret = dict_get_int32 (input, this->name, &xl_id);
-        if (ret)
-                goto out;
-        ret = dict_set_int32 (output, this->name, xl_id);
-        if (ret)
-                goto out;
-        switch (op) {
+    gf_xl_afr_op_t op = GF_SHD_OP_INVALID;
+    int ret = 0;
+    int xl_id = 0;
+    afr_private_t *priv = NULL;
+    afr_self_heald_t *shd = NULL;
+    struct subvol_healer *healer = NULL;
+    int i = 0;
+    char key[64];
+    int keylen = 0;
+    int this_name_len = 0;
+    int op_ret = 0;
+    uint64_t cnt = 0;
+
+#define AFR_SET_DICT_AND_LOG(name, output, key, keylen, dict_str,              \
+                             dict_str_len)                                     \
+    {                                                                          \
+        int ret;                                                               \
+                                                                               \
+        ret = dict_set_nstrn(output, key, keylen, dict_str, dict_str_len);     \
+        if (ret) {                                                             \
+            gf_smsg(name, GF_LOG_ERROR, -ret, AFR_MSG_DICT_SET_FAILED,         \
+                    "key=%s", key, "value=%s", dict_str, NULL);                \
+        }                                                                      \
+    }
+
+    priv = this->private;
+    shd = &priv->shd;
+
+    ret = dict_get_int32_sizen(input, "xl-op", (int32_t *)&op);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_ERROR, -ret, AFR_MSG_DICT_GET_FAILED,
+                "key=xl-op", NULL);
+        goto out;
+    }
+    this_name_len = strlen(this->name);
+    ret = dict_get_int32n(input, this->name, this_name_len, &xl_id);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_ERROR, -ret, AFR_MSG_DICT_GET_FAILED,
+                "key=%s", this->name, NULL);
+        goto out;
+    }
+    ret = dict_set_int32n(output, this->name, this_name_len, xl_id);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_ERROR, -ret, AFR_MSG_DICT_SET_FAILED,
+                "key=%s", this->name, NULL);
+        goto out;
+    }
+    switch (op) {
         case GF_SHD_OP_HEAL_INDEX:
-		op_ret = -1;
-
-		for (i = 0; i < priv->child_count; i++) {
-			healer = &shd->index_healers[i];
-			snprintf (key, sizeof (key), "%d-%d-status", xl_id, i);
-
-			if (!priv->child_up[i]) {
-				ret = dict_set_str (output, key,
-						    "Brick is not connected");
-			} else if (AFR_COUNT (priv->child_up,
-					      priv->child_count) < 2) {
-				ret = dict_set_str (output, key,
-						    "< 2 bricks in replica are up");
-			} else if (!afr_shd_is_subvol_local (this, healer->subvol)) {
-				ret = dict_set_str (output, key,
-						    "Brick is remote");
-			} else {
-				ret = dict_set_str (output, key,
-						    "Started self-heal");
-				afr_shd_index_healer_spawn (this, i);
-				op_ret = 0;
-			}
-		}
-                break;
+            op_ret = 0;
+
+            for (i = 0; i < priv->child_count; i++) {
+                healer = &shd->index_healers[i];
+                keylen = snprintf(key, sizeof(key), "%d-%d-status", xl_id, i);
+
+                if (!priv->child_up[i]) {
+                    AFR_SET_DICT_AND_LOG(this->name, output, key, keylen,
+                                         SBRICK_NOT_CONNECTED,
+                                         SLEN(SBRICK_NOT_CONNECTED));
+                    op_ret = -1;
+                } else if (AFR_COUNT(priv->child_up, priv->child_count) < 2) {
+                    AFR_SET_DICT_AND_LOG(this->name, output, key, keylen,
+                                         SLESS_THAN2_BRICKS_in_REP,
+                                         SLEN(SLESS_THAN2_BRICKS_in_REP));
+                    op_ret = -1;
+                } else if (!afr_shd_is_subvol_local(this, healer->subvol)) {
+                    AFR_SET_DICT_AND_LOG(this->name, output, key, keylen,
+                                         SBRICK_IS_REMOTE,
+                                         SLEN(SBRICK_IS_REMOTE));
+                } else {
+                    AFR_SET_DICT_AND_LOG(this->name, output, key, keylen,
+                                         SSTARTED_SELF_HEAL,
+                                         SLEN(SSTARTED_SELF_HEAL));
+
+                    ret = afr_shd_index_healer_spawn(this, i);
+
+                    if (ret) {
+                        gf_smsg(this->name, GF_LOG_ERROR, -ret,
+                                AFR_MSG_HEALER_SPAWN_FAILED, NULL);
+                    }
+                }
+            }
+            break;
         case GF_SHD_OP_HEAL_FULL:
-		op_ret = -1;
-
-		for (i = 0; i < priv->child_count; i++) {
-			healer = &shd->full_healers[i];
-			snprintf (key, sizeof (key), "%d-%d-status", xl_id, i);
-
-			if (!priv->child_up[i]) {
-				ret = dict_set_str (output, key,
-						    "Brick is not connected");
-			} else if (AFR_COUNT (priv->child_up,
-					      priv->child_count) < 2) {
-				ret = dict_set_str (output, key,
-						    "< 2 bricks in replica are up");
-			} else if (!afr_shd_is_subvol_local (this, healer->subvol)) {
-				ret = dict_set_str (output, key,
-						    "Brick is remote");
-			} else {
-				ret = dict_set_str (output, key,
-						    "Started self-heal");
-				afr_shd_full_healer_spawn (this, i);
-				op_ret = 0;
-			}
-		}
-                break;
-        case GF_SHD_OP_INDEX_SUMMARY:
-		for (i = 0; i < priv->child_count; i++)
-			if (shd->index_healers[i].local)
-				afr_shd_gather_index_entries (this, i, output);
-                break;
-        case GF_SHD_OP_HEALED_FILES:
-        case GF_SHD_OP_HEAL_FAILED_FILES:
-                for (i = 0; i < priv->child_count; i++) {
-                        snprintf (key, sizeof (key), "%d-%d-status", xl_id, i);
-                        ret = dict_set_str (output, key, "Operation Not "
-                                            "Supported");
+            op_ret = -1;
+
+            for (i = 0; i < priv->child_count; i++) {
+                healer = &shd->full_healers[i];
+                keylen = snprintf(key, sizeof(key), "%d-%d-status", xl_id, i);
+
+                if (!priv->child_up[i]) {
+                    AFR_SET_DICT_AND_LOG(this->name, output, key, keylen,
+                                         SBRICK_NOT_CONNECTED,
+                                         SLEN(SBRICK_NOT_CONNECTED));
+                } else if (AFR_COUNT(priv->child_up, priv->child_count) < 2) {
+                    AFR_SET_DICT_AND_LOG(this->name, output, key, keylen,
+                                         SLESS_THAN2_BRICKS_in_REP,
+                                         SLEN(SLESS_THAN2_BRICKS_in_REP));
+                } else if (!afr_shd_is_subvol_local(this, healer->subvol)) {
+                    AFR_SET_DICT_AND_LOG(this->name, output, key, keylen,
+                                         SBRICK_IS_REMOTE,
+                                         SLEN(SBRICK_IS_REMOTE));
+                } else {
+                    AFR_SET_DICT_AND_LOG(this->name, output, key, keylen,
+                                         SSTARTED_SELF_HEAL,
+                                         SLEN(SSTARTED_SELF_HEAL));
+
+                    ret = afr_shd_full_healer_spawn(this, i);
+
+                    if (ret) {
+                        gf_smsg(this->name, GF_LOG_ERROR, -ret,
+                                AFR_MSG_HEALER_SPAWN_FAILED, NULL);
+                    }
+                    op_ret = 0;
                 }
-                break;
+            }
+            break;
+        case GF_SHD_OP_INDEX_SUMMARY:
+            /* this case has been handled in glfs-heal.c */
+            break;
         case GF_SHD_OP_SPLIT_BRAIN_FILES:
-		eh_dump (shd->split_brain, output, afr_add_shd_event);
-                break;
+            eh_dump(shd->split_brain, output, afr_add_shd_event);
+            break;
         case GF_SHD_OP_STATISTICS:
-		for (i = 0; i < priv->child_count; i++) {
-			eh_dump (shd->statistics[i], output,
-				 afr_add_crawl_event);
-			afr_shd_dict_add_crawl_event (this, output,
-						      &shd->index_healers[i].crawl_event);
-			afr_shd_dict_add_crawl_event (this, output,
-						      &shd->full_healers[i].crawl_event);
-		}
-                break;
+            for (i = 0; i < priv->child_count; i++) {
+                eh_dump(shd->statistics[i], output, afr_add_crawl_event);
+                ret = afr_shd_dict_add_crawl_event(
+                    this, output, &shd->index_healers[i].crawl_event);
+                if (ret) {
+                    gf_smsg(this->name, GF_LOG_ERROR, -ret,
+                            AFR_MSG_ADD_CRAWL_EVENT_FAILED, NULL);
+                }
+
+                ret = afr_shd_dict_add_crawl_event(
+                    this, output, &shd->full_healers[i].crawl_event);
+                if (ret) {
+                    gf_smsg(this->name, GF_LOG_ERROR, -ret,
+                            AFR_MSG_ADD_CRAWL_EVENT_FAILED, NULL);
+                }
+            }
+            break;
         case GF_SHD_OP_STATISTICS_HEAL_COUNT:
         case GF_SHD_OP_STATISTICS_HEAL_COUNT_PER_REPLICA:
-		op_ret = -1;
-
-		for (i = 0; i < priv->child_count; i++) {
-			if (!priv->child_up[i]) {
-				snprintf (key, sizeof (key), "%d-%d-status",
-                                          xl_id, i);
-				ret = dict_set_str (output, key,
-						    "Brick is not connected");
-			} else {
-				snprintf (key, sizeof (key), "%d-%d-hardlinks",
-                                          xl_id, i);
-				ret = afr_shd_get_index_count (this, i, &cnt);
-				if (ret == 0) {
-					ret = dict_set_uint64 (output, key, cnt);
-				}
-				op_ret = 0;
-			}
-		}
-
-//                ret = _do_crawl_op_on_local_subvols (this, INDEX_TO_BE_HEALED,
-//                                                     STATISTICS_TO_BE_HEALED,
-//                                                     output);
-                break;
+            op_ret = -1;
+
+            for (i = 0; i < priv->child_count; i++) {
+                if (!priv->child_up[i]) {
+                    keylen = snprintf(key, sizeof(key), "%d-%d-status", xl_id,
+                                      i);
+                    AFR_SET_DICT_AND_LOG(this->name, output, key, keylen,
+                                         SBRICK_NOT_CONNECTED,
+                                         SLEN(SBRICK_NOT_CONNECTED));
+                } else {
+                    snprintf(key, sizeof(key), "%d-%d-hardlinks", xl_id, i);
+                    ret = afr_shd_get_index_count(this, i, &cnt);
+                    if (ret == 0) {
+                        ret = dict_set_uint64(output, key, cnt);
+                    }
+                    if (ret) {
+                        gf_smsg(this->name, GF_LOG_ERROR, -ret,
+                                AFR_MSG_DICT_SET_FAILED, NULL);
+                    }
+                    op_ret = 0;
+                }
+            }
+
+            break;
 
         default:
-                gf_log (this->name, GF_LOG_ERROR, "Unknown set op %d", op);
-                break;
-        }
+            gf_smsg(this->name, GF_LOG_ERROR, 0, AFR_MSG_INVALID_ARG, "op=%d",
+                    op, NULL);
+            break;
+    }
 out:
-        dict_del (output, this->name);
-        return op_ret;
+    dict_deln(output, this->name, this_name_len);
+    return op_ret;
+
+#undef AFR_SET_DICT_AND_LOG
 }
diff --git a/xlators/cluster/afr/src/afr-self-heald.h b/xlators/cluster/afr/src/afr-self-heald.h
index 58b088e4cd7..18db728ea7b 100644
--- a/xlators/cluster/afr/src/afr-self-heald.h
+++ b/xlators/cluster/afr/src/afr-self-heald.h
@@ -8,70 +8,68 @@
   cases as published by the Free Software Foundation.
 */
 
-
 #ifndef _AFR_SELF_HEALD_H
 #define _AFR_SELF_HEALD_H
 
 #include <pthread.h>
 
-
 typedef struct {
-	int child;
-	char *path;
+    char *path;
+    int child;
 } shd_event_t;
 
 typedef struct {
-	int      child;
-	uint64_t healed_count;
-        uint64_t split_brain_count;
-        uint64_t heal_failed_count;
-
-	/* If start_time is 0, it means crawler is not in progress
-	   and stats are not valid */
-	time_t   start_time;
-	/* If start_time is NOT 0 and end_time is 0, it means
-	   cralwer is in progress */
-        time_t   end_time;
-        char     *crawl_type;
+    uint64_t healed_count;
+    uint64_t split_brain_count;
+    uint64_t heal_failed_count;
+
+    /* If start_time is 0, it means crawler is not in progress
+       and stats are not valid */
+    time_t start_time;
+    /* If start_time is NOT 0 and end_time is 0, it means
+       cralwer is in progress */
+    time_t end_time;
+    char *crawl_type;
+    int child;
 } crawl_event_t;
 
 struct subvol_healer {
-	xlator_t        *this;
-	int              subvol;
-	gf_boolean_t     local;
-	gf_boolean_t     running;
-	gf_boolean_t     rerun;
-	crawl_event_t    crawl_event;
-	pthread_mutex_t  mutex;
-	pthread_cond_t   cond;
-	pthread_t        thread;
+    xlator_t *this;
+    crawl_event_t crawl_event;
+    pthread_mutex_t mutex;
+    pthread_cond_t cond;
+    pthread_t thread;
+    int subvol;
+    gf_boolean_t local;
+    gf_boolean_t running;
+    gf_boolean_t rerun;
 };
 
 typedef struct {
-	gf_boolean_t            iamshd;
-	gf_boolean_t            enabled;
-	int                     timeout;
-	struct subvol_healer   *index_healers;
-	struct subvol_healer   *full_healers;
-
-        eh_t                    *split_brain;
-        eh_t                    **statistics;
+    struct subvol_healer *index_healers;
+    struct subvol_healer *full_healers;
+
+    eh_t *split_brain;
+    eh_t **statistics;
+    int timeout;
+    uint32_t max_threads;
+    uint32_t wait_qlength;
+    uint32_t halo_max_latency_msec;
+    gf_boolean_t iamshd;
+    gf_boolean_t enabled;
 } afr_self_heald_t;
 
-
-int
-afr_selfheal_childup (xlator_t *this, int subvol);
-
 int
-afr_selfheal_daemon_init (xlator_t *this);
+afr_selfheal_daemon_init(xlator_t *this);
 
 int
-afr_xl_op (xlator_t *this, dict_t *input, dict_t *output);
+afr_xl_op(xlator_t *this, dict_t *input, dict_t *output);
 
 int
-afr_shd_gfid_to_path (xlator_t *this, xlator_t *subvol, uuid_t gfid,
-                      char **path_p);
+afr_shd_gfid_to_path(xlator_t *this, xlator_t *subvol, uuid_t gfid,
+                     char **path_p);
 
 int
-afr_shd_index_purge (xlator_t *subvol, inode_t *inode, char *name);
+afr_shd_entry_purge(xlator_t *subvol, inode_t *inode, char *name,
+                    ia_type_t type);
 #endif /* !_AFR_SELF_HEALD_H */
diff --git a/xlators/cluster/afr/src/afr-transaction.c b/xlators/cluster/afr/src/afr-transaction.c
index d2111060035..a51f79b1f43 100644
--- a/xlators/cluster/afr/src/afr-transaction.c
+++ b/xlators/cluster/afr/src/afr-transaction.c
@@ -8,1868 +8,2920 @@
   cases as published by the Free Software Foundation.
 */
 
-#include "dict.h"
-#include "byte-order.h"
-#include "common-utils.h"
-#include "timer.h"
+#include <glusterfs/dict.h>
+#include <glusterfs/byte-order.h>
+#include <glusterfs/common-utils.h>
+#include <glusterfs/timer.h>
 
 #include "afr.h"
 #include "afr-transaction.h"
+#include "afr-self-heal.h"
+#include "afr-messages.h"
 
 #include <signal.h>
 
+typedef enum {
+    AFR_TRANSACTION_PRE_OP,
+    AFR_TRANSACTION_POST_OP,
+} afr_xattrop_type_t;
+
+static void
+afr_lock_resume_shared(struct list_head *list);
+
+static void
+afr_post_op_handle_success(call_frame_t *frame, xlator_t *this);
+
+static void
+afr_post_op_handle_failure(call_frame_t *frame, xlator_t *this, int op_errno);
+
+void
+__afr_transaction_wake_shared(afr_local_t *local, struct list_head *shared);
+
+void
+afr_changelog_post_op_do(call_frame_t *frame, xlator_t *this);
+
+int
+afr_changelog_post_op_safe(call_frame_t *frame, xlator_t *this);
+
 gf_boolean_t
-afr_changelog_pre_op_uninherit (call_frame_t *frame, xlator_t *this);
+afr_changelog_pre_op_uninherit(call_frame_t *frame, xlator_t *this);
 
 gf_boolean_t
-afr_changelog_pre_op_update (call_frame_t *frame, xlator_t *this);
+afr_changelog_pre_op_update(call_frame_t *frame, xlator_t *this);
 
 int
-afr_changelog_do (call_frame_t *frame, xlator_t *this, dict_t *xattr,
-		  afr_changelog_resume_t changelog_resume);
+afr_changelog_call_count(afr_transaction_type type,
+                         unsigned char *pre_op_subvols,
+                         unsigned char *failed_subvols,
+                         unsigned int child_count);
+int
+afr_changelog_do(call_frame_t *frame, xlator_t *this, dict_t *xattr,
+                 afr_changelog_resume_t changelog_resume,
+                 afr_xattrop_type_t op);
+
+static void
+afr_ta_decide_post_op_state(call_frame_t *frame, xlator_t *this);
+
+static int
+afr_ta_post_op_do(void *opaque);
 
-static int32_t
-afr_quorum_errno (afr_private_t *priv)
+static int
+afr_ta_post_op_synctask(xlator_t *this, afr_local_t *local);
+
+static int
+afr_changelog_post_op_done(call_frame_t *frame, xlator_t *this);
+
+static void
+afr_changelog_post_op_fail(call_frame_t *frame, xlator_t *this, int op_errno);
+
+void
+afr_ta_locked_priv_invalidate(afr_private_t *priv)
 {
-        if (priv->quorum_reads)
-                return ENOTCONN;
-        return EROFS;
+    priv->ta_bad_child_index = AFR_CHILD_UNKNOWN;
+    priv->release_ta_notify_dom_lock = _gf_false;
+    priv->ta_notify_dom_lock_offset = 0;
+}
+
+static void
+afr_ta_process_waitq(xlator_t *this)
+{
+    afr_local_t *entry = NULL;
+    afr_private_t *priv = this->private;
+    struct list_head waitq = {
+        0,
+    };
+
+    INIT_LIST_HEAD(&waitq);
+    LOCK(&priv->lock);
+    list_splice_init(&priv->ta_waitq, &waitq);
+    UNLOCK(&priv->lock);
+    list_for_each_entry(entry, &waitq, ta_waitq)
+    {
+        afr_ta_decide_post_op_state(entry->transaction.frame, this);
+    }
 }
 
 int
-__afr_txn_write_fop (call_frame_t *frame, xlator_t *this)
+afr_ta_lock_release_done(int ret, call_frame_t *ta_frame, void *opaque)
 {
-        afr_local_t *local = NULL;
-        afr_private_t *priv = NULL;
-        int call_count = -1;
-        int i = 0;
+    afr_ta_process_waitq(ta_frame->this);
+    STACK_DESTROY(ta_frame->root);
+    return 0;
+}
 
-        local = frame->local;
-        priv = this->private;
+int
+afr_release_notify_lock_for_ta(void *opaque)
+{
+    xlator_t *this = NULL;
+    afr_private_t *priv = NULL;
+    loc_t loc = {
+        0,
+    };
+    struct gf_flock flock = {
+        0,
+    };
+    int ret = -1;
+
+    this = (xlator_t *)opaque;
+    priv = this->private;
+    ret = afr_fill_ta_loc(this, &loc, _gf_true);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, -ret, AFR_MSG_THIN_ARB,
+               "Failed to populate loc for thin-arbiter.");
+        goto out;
+    }
+    flock.l_type = F_UNLCK;
+    flock.l_start = priv->ta_notify_dom_lock_offset;
+    flock.l_len = 1;
+    ret = syncop_inodelk(priv->children[THIN_ARBITER_BRICK_INDEX],
+                         AFR_TA_DOM_NOTIFY, &loc, F_SETLK, &flock, NULL, NULL);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, -ret, AFR_MSG_THIN_ARB,
+               "Failed to unlock AFR_TA_DOM_NOTIFY lock.");
+    }
+
+    LOCK(&priv->lock);
+    {
+        afr_ta_locked_priv_invalidate(priv);
+    }
+    UNLOCK(&priv->lock);
+out:
+    loc_wipe(&loc);
+    return ret;
+}
 
-        call_count = AFR_COUNT (local->transaction.pre_op, priv->child_count);
+void
+afr_zero_fill_stat(afr_local_t *local)
+{
+    if (!local)
+        return;
+    if (local->transaction.type == AFR_DATA_TRANSACTION ||
+        local->transaction.type == AFR_METADATA_TRANSACTION) {
+        gf_zero_fill_stat(&local->cont.inode_wfop.prebuf);
+        gf_zero_fill_stat(&local->cont.inode_wfop.postbuf);
+    } else if (local->transaction.type == AFR_ENTRY_TRANSACTION ||
+               local->transaction.type == AFR_ENTRY_RENAME_TRANSACTION) {
+        gf_zero_fill_stat(&local->cont.dir_fop.buf);
+        gf_zero_fill_stat(&local->cont.dir_fop.preparent);
+        gf_zero_fill_stat(&local->cont.dir_fop.postparent);
+        if (local->transaction.type == AFR_ENTRY_TRANSACTION)
+            return;
+        gf_zero_fill_stat(&local->cont.dir_fop.prenewparent);
+        gf_zero_fill_stat(&local->cont.dir_fop.postnewparent);
+    }
+}
 
-        if (call_count == 0) {
-                local->transaction.resume (frame, this);
-                return 0;
+/* In case of errors afr needs to choose which xdata from lower xlators it needs
+ * to unwind with. The way it is done is by checking if there are
+ * any good subvols which failed. Give preference to errnos other than
+ * ENOTCONN even if the child is source */
+void
+afr_pick_error_xdata(afr_local_t *local, afr_private_t *priv, inode_t *inode1,
+                     unsigned char *readable1, inode_t *inode2,
+                     unsigned char *readable2)
+{
+    int s = -1; /*selection*/
+    int i = 0;
+    unsigned char *readable = NULL;
+
+    if (local->xdata_rsp) {
+        dict_unref(local->xdata_rsp);
+        local->xdata_rsp = NULL;
+    }
+
+    readable = alloca0(priv->child_count * sizeof(*readable));
+    if (inode2 && readable2) { /*rename fop*/
+        AFR_INTERSECT(readable, readable1, readable2, priv->child_count);
+    } else {
+        memcpy(readable, readable1, sizeof(*readable) * priv->child_count);
+    }
+
+    for (i = 0; i < priv->child_count; i++) {
+        if (!local->replies[i].valid)
+            continue;
+
+        if (local->replies[i].op_ret >= 0)
+            continue;
+
+        if (local->replies[i].op_errno == ENOTCONN)
+            continue;
+
+        /*Order is important in the following condition*/
+        if ((s < 0) || (!readable[s] && readable[i]))
+            s = i;
+    }
+
+    if (s != -1 && local->replies[s].xdata) {
+        local->xdata_rsp = dict_ref(local->replies[s].xdata);
+    } else if (s == -1) {
+        for (i = 0; i < priv->child_count; i++) {
+            if (!local->replies[i].valid)
+                continue;
+
+            if (local->replies[i].op_ret >= 0)
+                continue;
+
+            if (!local->replies[i].xdata)
+                continue;
+            local->xdata_rsp = dict_ref(local->replies[i].xdata);
+            break;
         }
+    }
+}
 
-        local->call_count = call_count;
+gf_boolean_t
+afr_needs_changelog_update(afr_local_t *local)
+{
+    if (local->transaction.type == AFR_DATA_TRANSACTION)
+        return _gf_true;
+    if (!local->optimistic_change_log)
+        return _gf_true;
+    return _gf_false;
+}
 
-        for (i = 0; i < priv->child_count; i++) {
-                if (local->transaction.pre_op[i]) {
-			local->transaction.wind (frame, this, i);
+gf_boolean_t
+afr_changelog_has_quorum(afr_local_t *local, xlator_t *this)
+{
+    afr_private_t *priv = NULL;
+    int i = 0;
+    unsigned char *success_children = NULL;
 
-                        if (!--call_count)
-                                break;
-                }
+    priv = this->private;
+    success_children = alloca0(priv->child_count);
+
+    for (i = 0; i < priv->child_count; i++) {
+        if (!local->transaction.failed_subvols[i]) {
+            success_children[i] = 1;
         }
+    }
 
-        return 0;
+    if (afr_has_quorum(success_children, this, NULL)) {
+        return _gf_true;
+    }
+
+    return _gf_false;
 }
 
+gf_boolean_t
+afr_is_write_subvol_valid(call_frame_t *frame, xlator_t *this)
+{
+    int i = 0;
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
+    uint64_t write_subvol = 0;
+    unsigned char *writable = NULL;
+    uint16_t datamap = 0;
+
+    local = frame->local;
+    priv = this->private;
+    writable = alloca0(priv->child_count);
+
+    write_subvol = afr_write_subvol_get(frame, this);
+    datamap = (write_subvol & 0x00000000ffff0000) >> 16;
+    for (i = 0; i < priv->child_count; i++) {
+        if (datamap & (1 << i))
+            writable[i] = 1;
+
+        if (writable[i] && !local->transaction.failed_subvols[i])
+            return _gf_true;
+    }
+
+    return _gf_false;
+}
 
 int
-__afr_txn_write_done (call_frame_t *frame, xlator_t *this)
+afr_transaction_fop(call_frame_t *frame, xlator_t *this)
 {
-        afr_local_t *local = NULL;
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
+    int call_count = -1;
+    unsigned char *failed_subvols = NULL;
+    int i = 0;
+
+    local = frame->local;
+    priv = this->private;
+
+    failed_subvols = local->transaction.failed_subvols;
+    call_count = priv->child_count -
+                 AFR_COUNT(failed_subvols, priv->child_count);
+    /* Fail if pre-op did not succeed on quorum no. of bricks. */
+    if (!afr_changelog_has_quorum(local, this) || !call_count) {
+        local->op_ret = -1;
+        /* local->op_errno is already captured in changelog cbk. */
+        afr_transaction_resume(frame, this);
+        return 0;
+    }
 
-        local = frame->local;
+    /* Fail if at least one writeable brick isn't up.*/
+    if (local->transaction.type == AFR_DATA_TRANSACTION &&
+        !afr_is_write_subvol_valid(frame, this)) {
+        local->op_ret = -1;
+        local->op_errno = EIO;
+        afr_transaction_resume(frame, this);
+        return 0;
+    }
 
-        local->transaction.unwind (frame, this);
+    local->call_count = call_count;
+    for (i = 0; i < priv->child_count; i++) {
+        if (local->transaction.pre_op[i] && !failed_subvols[i]) {
+            local->transaction.wind(frame, this, i);
 
-        AFR_STACK_DESTROY (frame);
+            if (!--call_count)
+                break;
+        }
+    }
 
-        return 0;
+    return 0;
 }
 
-
-call_frame_t*
-afr_transaction_detach_fop_frame (call_frame_t *frame)
+int
+afr_transaction_done(call_frame_t *frame, xlator_t *this)
 {
-        afr_local_t *   local = NULL;
-        call_frame_t   *fop_frame = NULL;
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
+    gf_boolean_t unwind = _gf_false;
+    afr_lock_t *lock = NULL;
+    afr_local_t *lock_local = NULL;
 
-        local = frame->local;
+    priv = this->private;
+    local = frame->local;
 
-        LOCK (&frame->lock);
+    if (priv->consistent_metadata) {
+        LOCK(&frame->lock);
         {
-                fop_frame = local->transaction.main_frame;
-                local->transaction.main_frame = NULL;
+            unwind = (local->transaction.main_frame != NULL);
         }
-        UNLOCK (&frame->lock);
-
-        return fop_frame;
+        UNLOCK(&frame->lock);
+        if (unwind) /*It definitely did post-op*/
+            afr_zero_fill_stat(local);
+    }
+
+    if (local->transaction.do_eager_unlock) {
+        lock = &local->inode_ctx->lock[local->transaction.type];
+        LOCK(&local->inode->lock);
+        {
+            lock->acquired = _gf_false;
+            lock->release = _gf_false;
+            list_splice_init(&lock->frozen, &lock->waiting);
+            if (list_empty(&lock->waiting))
+                goto unlock;
+            lock_local = list_entry(lock->waiting.next, afr_local_t,
+                                    transaction.wait_list);
+            list_del_init(&lock_local->transaction.wait_list);
+            list_add(&lock_local->transaction.owner_list, &lock->owners);
+        }
+    unlock:
+        UNLOCK(&local->inode->lock);
+    }
+    if (lock_local) {
+        afr_lock(lock_local->transaction.frame,
+                 lock_local->transaction.frame->this);
+    }
+    local->transaction.unwind(frame, this);
+
+    GF_ASSERT(list_empty(&local->transaction.owner_list));
+    GF_ASSERT(list_empty(&local->transaction.wait_list));
+    AFR_STACK_DESTROY(frame);
+
+    return 0;
 }
 
-
 static void
-afr_save_lk_owner (call_frame_t *frame)
+afr_lock_fail_shared(afr_local_t *local, struct list_head *list)
 {
-        afr_local_t * local = NULL;
-
-        local = frame->local;
-
-        local->saved_lk_owner = frame->root->lk_owner;
+    afr_local_t *each = NULL;
+
+    while (!list_empty(list)) {
+        each = list_entry(list->next, afr_local_t, transaction.wait_list);
+        list_del_init(&each->transaction.wait_list);
+        each->op_ret = -1;
+        each->op_errno = local->op_errno;
+        afr_transaction_done(each->transaction.frame,
+                             each->transaction.frame->this);
+    }
 }
 
-
 static void
-afr_restore_lk_owner (call_frame_t *frame)
+afr_handle_lock_acquire_failure(afr_local_t *local)
 {
-        afr_local_t * local = NULL;
-
-        local = frame->local;
+    struct list_head shared;
+    afr_lock_t *lock = NULL;
 
-        frame->root->lk_owner = local->saved_lk_owner;
-}
+    if (!local->transaction.eager_lock_on)
+        goto out;
 
-void
-__mark_all_success (call_frame_t *frame, xlator_t *this)
-{
-	afr_private_t *priv = NULL;
-	afr_local_t *local = NULL;
-	int i;
+    lock = &local->inode_ctx->lock[local->transaction.type];
 
-	local = frame->local;
-	priv = this->private;
+    INIT_LIST_HEAD(&shared);
+    LOCK(&local->inode->lock);
+    {
+        lock->release = _gf_true;
+        list_splice_init(&lock->waiting, &shared);
+    }
+    UNLOCK(&local->inode->lock);
 
-	for (i = 0; i < priv->child_count; i++) {
-		local->transaction.failed_subvols[i] = 0;
-	}
+    afr_lock_fail_shared(local, &shared);
+    local->transaction.do_eager_unlock = _gf_true;
+out:
+    local->internal_lock.lock_cbk = afr_transaction_done;
+    afr_unlock(local->transaction.frame, local->transaction.frame->this);
 }
 
-
-int
-afr_transaction_perform_fop (call_frame_t *frame, xlator_t *this)
+call_frame_t *
+afr_transaction_detach_fop_frame(call_frame_t *frame)
 {
-        afr_local_t     *local = NULL;
-        fd_t            *fd   = NULL;
+    afr_local_t *local = NULL;
+    call_frame_t *fop_frame = NULL;
 
-        local = frame->local;
-        fd    = local->fd;
+    local = frame->local;
 
-        /*  Perform fops with the lk-owner from top xlator.
-         *  Eg: lk-owner of posix-lk and flush should be same,
-         *  flush cant clear the  posix-lks without that lk-owner.
-         */
-        afr_save_lk_owner (frame);
-        frame->root->lk_owner =
-                local->transaction.main_frame->root->lk_owner;
-
-	if (local->pre_op_compat)
-		/* old mode, pre-op was done as afr_changelog_do()
-		   just now, before OP */
-		afr_changelog_pre_op_update (frame, this);
-
-        /* The wake up needs to happen independent of
-           what type of fop arrives here. If it was
-           a write, then it has already inherited the
-           lock and changelog. If it was not a write,
-           then the presumption of the optimization (of
-           optimizing for successive write operations)
-           fails.
-        */
-        if (fd)
-                afr_delayed_changelog_wake_up (this, fd);
-        local->transaction.fop (frame, this);
+    afr_handle_inconsistent_fop(frame, &local->op_ret, &local->op_errno);
+    LOCK(&frame->lock);
+    {
+        fop_frame = local->transaction.main_frame;
+        local->transaction.main_frame = NULL;
+    }
+    UNLOCK(&frame->lock);
 
-	return 0;
+    return fop_frame;
 }
 
-
-static int
-__changelog_enabled (afr_private_t *priv, afr_transaction_type type)
+static void
+afr_save_lk_owner(call_frame_t *frame)
 {
-        int ret = 0;
+    afr_local_t *local = NULL;
 
-        switch (type) {
-        case AFR_DATA_TRANSACTION:
-                if (priv->data_change_log)
-                        ret = 1;
+    local = frame->local;
 
-                break;
+    local->saved_lk_owner = frame->root->lk_owner;
+}
 
-        case AFR_METADATA_TRANSACTION:
-                if (priv->metadata_change_log)
-                        ret = 1;
+static void
+afr_restore_lk_owner(call_frame_t *frame)
+{
+    afr_local_t *local = NULL;
 
-                break;
+    local = frame->local;
 
-        case AFR_ENTRY_TRANSACTION:
-        case AFR_ENTRY_RENAME_TRANSACTION:
-                if (priv->entry_change_log)
-                        ret = 1;
+    frame->root->lk_owner = local->saved_lk_owner;
+}
 
-                break;
-        }
+void
+__mark_all_success(call_frame_t *frame, xlator_t *this)
+{
+    afr_private_t *priv = NULL;
+    afr_local_t *local = NULL;
+    int i;
 
-        return ret;
-}
+    local = frame->local;
+    priv = this->private;
 
+    for (i = 0; i < priv->child_count; i++) {
+        local->transaction.failed_subvols[i] = 0;
+    }
+}
 
-static int
-__fop_changelog_needed (call_frame_t *frame, xlator_t *this)
+void
+afr_compute_pre_op_sources(call_frame_t *frame, xlator_t *this)
 {
-        afr_private_t * priv  = NULL;
-        afr_local_t   * local = NULL;
-        int op_ret = 0;
-        afr_transaction_type type = -1;
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
+    afr_transaction_type type = -1;
+    dict_t *xdata = NULL;
+    int **matrix = NULL;
+    int idx = -1;
+    int i = 0;
+    int j = 0;
+
+    priv = this->private;
+    local = frame->local;
+    type = local->transaction.type;
+    idx = afr_index_for_transaction_type(type);
+    matrix = ALLOC_MATRIX(priv->child_count, int);
+
+    for (i = 0; i < priv->child_count; i++) {
+        if (!local->transaction.changelog_xdata[i])
+            continue;
+        xdata = local->transaction.changelog_xdata[i];
+        afr_selfheal_fill_matrix(this, matrix, i, idx, xdata);
+    }
+
+    memset(local->transaction.pre_op_sources, 1, priv->child_count);
+
+    /*If lock or pre-op failed on a brick, it is not a source. */
+    for (i = 0; i < priv->child_count; i++) {
+        if (local->transaction.failed_subvols[i])
+            local->transaction.pre_op_sources[i] = 0;
+    }
+
+    /* If brick is blamed by others, it is not a source. */
+    for (i = 0; i < priv->child_count; i++)
+        for (j = 0; j < priv->child_count; j++)
+            if (matrix[i][j] != 0)
+                local->transaction.pre_op_sources[j] = 0;
+}
 
-        priv  = this->private;
-        local = frame->local;
-        type  = local->transaction.type;
+void
+afr_txn_arbitrate_fop(call_frame_t *frame, xlator_t *this)
+{
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
+    int pre_op_sources_count = 0;
+    int i = 0;
 
-        if (__changelog_enabled (priv, type)) {
-                switch (local->op) {
+    priv = this->private;
+    local = frame->local;
 
-                case GF_FOP_WRITE:
-                case GF_FOP_FTRUNCATE:
-                        op_ret = 1;
-                        break;
+    afr_compute_pre_op_sources(frame, this);
+    pre_op_sources_count = AFR_COUNT(local->transaction.pre_op_sources,
+                                     priv->child_count);
 
-                case GF_FOP_FLUSH:
-                        op_ret = 0;
-                        break;
+    /* If arbiter is the only source, do not proceed. */
+    if (pre_op_sources_count < 2 &&
+        local->transaction.pre_op_sources[ARBITER_BRICK_INDEX]) {
+        local->op_ret = -1;
+        local->op_errno = ENOTCONN;
+        for (i = 0; i < priv->child_count; i++)
+            local->transaction.failed_subvols[i] = 1;
+    }
 
-                default:
-                        op_ret = 1;
-                }
-        }
+    afr_transaction_fop(frame, this);
 
-        return op_ret;
+    return;
 }
 
-
 int
-afr_set_pending_dict (afr_private_t *priv, dict_t *xattr, int **pending)
+afr_transaction_perform_fop(call_frame_t *frame, xlator_t *this)
 {
-        int i = 0;
-        int ret = 0;
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
+    int i = 0;
+    int ret = 0;
+    int failure_count = 0;
+    struct list_head shared;
+    afr_lock_t *lock = NULL;
+
+    local = frame->local;
+    priv = this->private;
+
+    INIT_LIST_HEAD(&shared);
+    if (local->transaction.type == AFR_DATA_TRANSACTION &&
+        !local->transaction.inherited) {
+        ret = afr_write_subvol_set(frame, this);
+        if (ret) {
+            /*act as if operation failed on all subvols*/
+            local->op_ret = -1;
+            local->op_errno = -ret;
+            for (i = 0; i < priv->child_count; i++)
+                local->transaction.failed_subvols[i] = 1;
+        }
+    }
+
+    if (local->pre_op_compat)
+        /* old mode, pre-op was done as afr_changelog_do()
+           just now, before OP */
+        afr_changelog_pre_op_update(frame, this);
+
+    if (!local->transaction.eager_lock_on || local->transaction.inherited)
+        goto fop;
+    failure_count = AFR_COUNT(local->transaction.failed_subvols,
+                              priv->child_count);
+    if (failure_count == priv->child_count) {
+        afr_handle_lock_acquire_failure(local);
+        return 0;
+    } else {
+        lock = &local->inode_ctx->lock[local->transaction.type];
+        LOCK(&local->inode->lock);
+        {
+            lock->acquired = _gf_true;
+            __afr_transaction_wake_shared(local, &shared);
+        }
+        UNLOCK(&local->inode->lock);
+    }
+
+fop:
+    /*  Perform fops with the lk-owner from top xlator.
+     *  Eg: lk-owner of posix-lk and flush should be same,
+     *  flush cant clear the  posix-lks without that lk-owner.
+     */
+    afr_save_lk_owner(frame);
+    frame->root->lk_owner = local->transaction.main_frame->root->lk_owner;
+
+    if (priv->arbiter_count == 1) {
+        afr_txn_arbitrate_fop(frame, this);
+    } else {
+        afr_transaction_fop(frame, this);
+    }
+
+    afr_lock_resume_shared(&shared);
+    return 0;
+}
 
-        for (i = 0; i < priv->child_count; i++) {
+int
+afr_set_pending_dict(afr_private_t *priv, dict_t *xattr, int **pending)
+{
+    int i = 0;
+    int ret = 0;
 
-                ret = dict_set_static_bin (xattr, priv->pending_key[i],
-					   pending[i],
-					   AFR_NUM_CHANGE_LOGS * sizeof (int));
-                /* 3 = data+metadata+entry */
+    for (i = 0; i < priv->child_count; i++) {
+        ret = dict_set_static_bin(xattr, priv->pending_key[i], pending[i],
+                                  AFR_NUM_CHANGE_LOGS * sizeof(int));
+        /* 3 = data+metadata+entry */
 
-                if (ret)
-                        break;
-        }
+        if (ret)
+            break;
+    }
 
-        return ret;
+    return ret;
 }
 
-int
-afr_lock_server_count (afr_private_t *priv, afr_transaction_type type)
+static void
+afr_ta_dom_lock_check_and_release(afr_ta_fop_state_t fop_state, xlator_t *this)
 {
-        int ret = 0;
-
-        switch (type) {
-        case AFR_DATA_TRANSACTION:
-                ret = priv->child_count;
+    afr_private_t *priv = this->private;
+    unsigned int inmem_count = 0;
+    unsigned int onwire_count = 0;
+    gf_boolean_t release = _gf_false;
+
+    LOCK(&priv->lock);
+    {
+        /*Once we get notify lock release upcall notification,
+         if any of the fop state counters are non-zero, we will
+         not release the lock.
+         */
+        onwire_count = priv->ta_on_wire_txn_count;
+        inmem_count = priv->ta_in_mem_txn_count;
+        switch (fop_state) {
+            case TA_GET_INFO_FROM_TA_FILE:
+                onwire_count = --priv->ta_on_wire_txn_count;
                 break;
-
-        case AFR_METADATA_TRANSACTION:
-                ret = priv->child_count;
+            case TA_INFO_IN_MEMORY_SUCCESS:
+            case TA_INFO_IN_MEMORY_FAILED:
+                inmem_count = --priv->ta_in_mem_txn_count;
                 break;
-
-        case AFR_ENTRY_TRANSACTION:
-        case AFR_ENTRY_RENAME_TRANSACTION:
-                ret = priv->child_count;
+            case TA_WAIT_FOR_NOTIFY_LOCK_REL:
+                GF_ASSERT(0);
+                break;
+            case TA_SUCCESS:
                 break;
         }
+        release = priv->release_ta_notify_dom_lock;
+    }
+    UNLOCK(&priv->lock);
 
-        return ret;
-}
+    if (inmem_count != 0 || release == _gf_false || onwire_count != 0)
+        return;
 
-/* {{{ pending */
+    afr_ta_lock_release_synctask(this);
+}
 
+static void
+afr_ta_process_onwireq(afr_ta_fop_state_t fop_state, xlator_t *this)
+{
+    afr_private_t *priv = this->private;
+    afr_local_t *entry = NULL;
+    int bad_child = AFR_CHILD_UNKNOWN;
+
+    struct list_head onwireq = {
+        0,
+    };
+    INIT_LIST_HEAD(&onwireq);
+
+    LOCK(&priv->lock);
+    {
+        bad_child = priv->ta_bad_child_index;
+        if (bad_child == AFR_CHILD_UNKNOWN) {
+            /*The previous on-wire ta_post_op was a failure. Just dequeue
+             *one element to wind on-wire again. */
+            entry = list_entry(priv->ta_onwireq.next, afr_local_t, ta_onwireq);
+            list_del_init(&entry->ta_onwireq);
+        } else {
+            /* Prepare to process all fops based on bad_child_index. */
+            list_splice_init(&priv->ta_onwireq, &onwireq);
+        }
+    }
+    UNLOCK(&priv->lock);
+
+    if (entry) {
+        afr_ta_post_op_synctask(this, entry);
+        return;
+    } else {
+        while (!list_empty(&onwireq)) {
+            entry = list_entry(onwireq.next, afr_local_t, ta_onwireq);
+            list_del_init(&entry->ta_onwireq);
+            if (entry->ta_failed_subvol == bad_child) {
+                afr_post_op_handle_success(entry->transaction.frame, this);
+            } else {
+                afr_post_op_handle_failure(entry->transaction.frame, this, EIO);
+            }
+        }
+    }
+}
 
 int
-afr_changelog_post_op_done (call_frame_t *frame, xlator_t *this)
+afr_changelog_post_op_done(call_frame_t *frame, xlator_t *this)
 {
-	afr_local_t *local = NULL;
-	afr_private_t *priv = NULL;
-        afr_internal_lock_t *int_lock = NULL;
+    afr_local_t *local = NULL;
+    afr_internal_lock_t *int_lock = NULL;
+    afr_private_t *priv = NULL;
 
-	local = frame->local;
-	priv = this->private;
-        int_lock = &local->internal_lock;
+    local = frame->local;
+    priv = this->private;
+    int_lock = &local->internal_lock;
 
-	if (local->transaction.resume_stub) {
-		call_resume (local->transaction.resume_stub);
-		local->transaction.resume_stub = NULL;
-	}
+    if (priv->thin_arbiter_count) {
+        /*fop should not come here with TA_WAIT_FOR_NOTIFY_LOCK_REL state */
+        afr_ta_dom_lock_check_and_release(local->fop_state, this);
+    }
 
-	if (afr_lock_server_count (priv, local->transaction.type) == 0) {
-		local->transaction.done (frame, this);
-	} else {
-		int_lock->lock_cbk = local->transaction.done;
-		afr_unlock (frame, this);
-	}
+    /* Fail the FOP if post-op did not succeed on quorum no. of bricks. */
+    if (!afr_changelog_has_quorum(local, this)) {
+        local->op_ret = -1;
+        /*local->op_errno is already captured in changelog cbk*/
+    }
 
-	return 0;
-}
+    if (local->transaction.resume_stub) {
+        call_resume(local->transaction.resume_stub);
+        local->transaction.resume_stub = NULL;
+    }
 
+    int_lock->lock_cbk = afr_transaction_done;
+    afr_unlock(frame, this);
 
-afr_inodelk_t*
-afr_get_inodelk (afr_internal_lock_t *int_lock, char *dom)
-{
-        afr_inodelk_t *inodelk = NULL;
-        int           i = 0;
-
-        for (i = 0; int_lock->inodelk[i].domain; i++) {
-                inodelk = &int_lock->inodelk[i];
-                if (strcmp (dom, inodelk->domain) == 0)
-                        return inodelk;
-        }
-        return NULL;
+    return 0;
 }
 
-unsigned char*
-afr_locked_nodes_get (afr_transaction_type type, afr_internal_lock_t *int_lock)
+static void
+afr_changelog_post_op_fail(call_frame_t *frame, xlator_t *this, int op_errno)
 {
-        unsigned char *locked_nodes = NULL;
-        afr_inodelk_t *inodelk = NULL;
-        switch (type) {
-        case AFR_DATA_TRANSACTION:
-        case AFR_METADATA_TRANSACTION:
-                inodelk = afr_get_inodelk (int_lock, int_lock->domain);
-                locked_nodes = inodelk->locked_nodes;
-        break;
+    afr_local_t *local = frame->local;
+    local->op_ret = -1;
+    local->op_errno = op_errno;
 
-        case AFR_ENTRY_TRANSACTION:
-        case AFR_ENTRY_RENAME_TRANSACTION:
-                /*Because same set of subvols participate in all lockee
-                 * entities*/
-                locked_nodes = int_lock->lockee[0].locked_nodes;
-        break;
-        }
-        return locked_nodes;
+    gf_msg(this->name, GF_LOG_ERROR, op_errno, AFR_MSG_THIN_ARB,
+           "Failing %s for gfid %s. Fop state is:%d", gf_fop_list[local->op],
+           uuid_utoa(local->inode->gfid), local->fop_state);
+
+    afr_changelog_post_op_done(frame, this);
 }
 
+unsigned char *
+afr_locked_nodes_get(afr_transaction_type type, afr_internal_lock_t *int_lock)
+{
+    /*Because same set of subvols participate in all lockee
+     * entities*/
+    return int_lock->lockee[0].locked_nodes;
+}
 
 int
-afr_changelog_call_count (afr_transaction_type type,
-			  unsigned char *pre_op_subvols,
-			  unsigned int child_count)
+afr_changelog_call_count(afr_transaction_type type,
+                         unsigned char *pre_op_subvols,
+                         unsigned char *failed_subvols,
+                         unsigned int child_count)
 {
-        int call_count = 0;
+    int i = 0;
+    int call_count = 0;
 
-	call_count = AFR_COUNT(pre_op_subvols, child_count);
+    for (i = 0; i < child_count; i++) {
+        if (pre_op_subvols[i] && !failed_subvols[i]) {
+            call_count++;
+        }
+    }
 
-        if (type == AFR_ENTRY_RENAME_TRANSACTION)
-                call_count *= 2;
+    if (type == AFR_ENTRY_RENAME_TRANSACTION)
+        call_count *= 2;
 
-        return call_count;
+    return call_count;
 }
 
-
 gf_boolean_t
-afr_txn_nothing_failed (call_frame_t *frame, xlator_t *this)
+afr_txn_nothing_failed(call_frame_t *frame, xlator_t *this)
 {
-        afr_private_t *priv = NULL;
-        afr_local_t *local = NULL;
-        int i = 0;
-
-        local = frame->local;
-	priv = this->private;
-
-        for (i = 0; i < priv->child_count; i++) {
-                if (local->transaction.failed_subvols[i])
-                        return _gf_false;
-        }
-
-        return _gf_true;
+    afr_private_t *priv = NULL;
+    afr_local_t *local = NULL;
+    int i = 0;
+
+    local = frame->local;
+    priv = this->private;
+
+    if (priv->thin_arbiter_count) {
+        /* We need to perform post-op even if 1 data brick was down
+         * before the txn started.*/
+        if (AFR_COUNT(local->transaction.failed_subvols, priv->child_count))
+            return _gf_false;
+    }
+
+    for (i = 0; i < priv->child_count; i++) {
+        if (local->transaction.pre_op[i] &&
+            local->transaction.failed_subvols[i])
+            return _gf_false;
+    }
+
+    return _gf_true;
 }
 
-
 void
-afr_handle_symmetric_errors (call_frame_t *frame, xlator_t *this)
-{
-	afr_local_t *local = NULL;
-	afr_private_t *priv = NULL;
-	int op_errno = 0;
-	int i_errno = 0;
-	gf_boolean_t matching_errors = _gf_true;
-	int i = 0;
-
-	priv = this->private;
-	local = frame->local;
-
-	for (i = 0; i < priv->child_count; i++) {
-		if (!local->replies[i].valid)
-			continue;
-		if (local->replies[i].op_ret != -1) {
-			/* Operation succeeded on at least on subvol,
-			   so it is not a failed-everywhere situation.
-			*/
-			matching_errors = _gf_false;
-			break;
-		}
-		i_errno = local->replies[i].op_errno;
-
-		if (i_errno == ENOTCONN) {
-			/* ENOTCONN is not a symmetric error. We do not
-			   know if the operation was performed on the
-			   backend or not.
-			*/
-			matching_errors = _gf_false;
-			break;
-		}
-
-		if (!op_errno) {
-			op_errno = i_errno;
-		} else if (op_errno != i_errno) {
-			/* Mismatching op_errno's */
-			matching_errors = _gf_false;
-			break;
-		}
-	}
-
-	if (matching_errors)
-		__mark_all_success (frame, this);
+afr_handle_symmetric_errors(call_frame_t *frame, xlator_t *this)
+{
+    if (afr_is_symmetric_error(frame, this))
+        __mark_all_success(frame, this);
 }
 
 gf_boolean_t
-afr_has_quorum (unsigned char *subvols, xlator_t *this)
+afr_has_quorum(unsigned char *subvols, xlator_t *this, call_frame_t *frame)
 {
-        unsigned int  quorum_count = 0;
-        afr_private_t *priv  = NULL;
-        unsigned int  up_children_count = 0;
+    unsigned int quorum_count = 0;
+    afr_private_t *priv = NULL;
+    unsigned int up_children_count = 0;
 
-        priv = this->private;
-        up_children_count = AFR_COUNT (subvols, priv->child_count);
+    priv = this->private;
+    up_children_count = AFR_COUNT(subvols, priv->child_count);
 
-        if (priv->quorum_count == AFR_QUORUM_AUTO) {
+    if (afr_lookup_has_quorum(frame, up_children_count))
+        return _gf_true;
+
+    if (priv->quorum_count == AFR_QUORUM_AUTO) {
         /*
-         * Special case for even numbers of nodes in auto-quorum:
-         * if we have exactly half children up
-         * and that includes the first ("senior-most") node, then that counts
-         * as quorum even if it wouldn't otherwise.  This supports e.g. N=2
-         * while preserving the critical property that there can only be one
-         * such group.
+         * Special case for auto-quorum with an even number of nodes.
+         *
+         * A replica set with even count N can only handle the same
+         * number of failures as odd N-1 before losing "vanilla"
+         * quorum, and the probability of more simultaneous failures is
+         * actually higher.  For example, with a 1% chance of failure
+         * we'd have a 0.03% chance of two simultaneous failures with
+         * N=3 but a 0.06% chance with N=4.  However, the special case
+         * is necessary for N=2 because there's no real quorum in that
+         * case (i.e. can't normally survive *any* failures).  In that
+         * case, we treat the first node as a tie-breaker, allowing
+         * quorum to be retained in some cases while still honoring the
+         * all-important constraint that there can not simultaneously
+         * be two partitioned sets of nodes each believing they have
+         * quorum.  Of two equally sized sets, the one without that
+         * first node will lose.
+         *
+         * It turns out that the special case is beneficial for higher
+         * values of N as well.  Continuing the example above, the
+         * probability of losing quorum with N=4 and this type of
+         * quorum is (very) slightly lower than with N=3 and vanilla
+         * quorum.  The difference becomes even more pronounced with
+         * higher N.  Therefore, even though such replica counts are
+         * unlikely to be seen in practice, we might as well use the
+         * "special" quorum then as well.
          */
-                if ((priv->child_count % 2 == 0) &&
-                    (up_children_count == (priv->child_count/2)))
-                        return subvols[0];
+        if ((up_children_count * 2) == priv->child_count) {
+            return subvols[0];
         }
+    }
 
-        if (priv->quorum_count == AFR_QUORUM_AUTO) {
-                quorum_count = priv->child_count/2 + 1;
-        } else {
-                quorum_count = priv->quorum_count;
-        }
+    if (priv->quorum_count == AFR_QUORUM_AUTO) {
+        quorum_count = priv->child_count / 2 + 1;
+    } else {
+        quorum_count = priv->quorum_count;
+    }
 
-        if (up_children_count >= quorum_count)
-                return _gf_true;
+    if (up_children_count >= quorum_count)
+        return _gf_true;
 
-        return _gf_false;
+    return _gf_false;
 }
 
 static gf_boolean_t
-afr_has_fop_quorum (call_frame_t *frame)
+afr_has_fop_quorum(call_frame_t *frame)
 {
-        xlator_t        *this = frame->this;
-        afr_local_t     *local = frame->local;
-        unsigned char *locked_nodes = NULL;
+    xlator_t *this = frame->this;
+    afr_local_t *local = frame->local;
+    unsigned char *locked_nodes = NULL;
 
-        locked_nodes = afr_locked_nodes_get (local->transaction.type,
-                                             &local->internal_lock);
-        return afr_has_quorum (locked_nodes, this);
+    locked_nodes = afr_locked_nodes_get(local->transaction.type,
+                                        &local->internal_lock);
+    return afr_has_quorum(locked_nodes, this, NULL);
 }
 
 static gf_boolean_t
-afr_has_fop_cbk_quorum (call_frame_t *frame)
+afr_has_fop_cbk_quorum(call_frame_t *frame)
 {
-        afr_local_t   *local   = frame->local;
-        xlator_t      *this    = frame->this;
-        afr_private_t *priv    = this->private;
-        unsigned char *success = alloca0(priv->child_count);
-        int           i        = 0;
-
-        for (i = 0; i < priv->child_count; i++) {
-                if (local->transaction.pre_op[i])
-                        if (!local->transaction.failed_subvols[i])
-                                success[i] = 1;
-        }
-
-        return afr_has_quorum (success, this);
+    afr_local_t *local = frame->local;
+    xlator_t *this = frame->this;
+    afr_private_t *priv = this->private;
+    unsigned char *success = alloca0(priv->child_count);
+    int i = 0;
+
+    for (i = 0; i < priv->child_count; i++) {
+        if (local->transaction.pre_op[i])
+            if (!local->transaction.failed_subvols[i])
+                success[i] = 1;
+    }
+
+    return afr_has_quorum(success, this, NULL);
 }
 
-void
-afr_handle_quorum (call_frame_t *frame)
+gf_boolean_t
+afr_need_dirty_marking(call_frame_t *frame, xlator_t *this)
 {
-        afr_local_t   *local = NULL;
-        afr_private_t *priv  = NULL;
-        int           i      = 0;
-        const char    *file  = NULL;
-        uuid_t        gfid   = {0};
+    afr_private_t *priv = this->private;
+    afr_local_t *local = NULL;
+    gf_boolean_t need_dirty = _gf_false;
 
-        local = frame->local;
-        priv  = frame->this->private;
+    local = frame->local;
 
-        if (priv->quorum_count == 0)
-                return;
+    if (!priv->quorum_count || !local->optimistic_change_log)
+        return _gf_false;
 
-        /* If the fop already failed return right away to preserve errno */
-        if (local->op_ret == -1)
-                return;
+    if (local->transaction.type == AFR_DATA_TRANSACTION ||
+        local->transaction.type == AFR_METADATA_TRANSACTION)
+        return _gf_false;
 
-        /*
-         * Network split may happen just after the fops are unwound, so check
-         * if the fop succeeded in a way it still follows quorum. If it doesn't,
-         * mark the fop as failure, mark the changelogs so it reflects that
-         * failure.
-         *
-         * Scenario:
-         * There are 3 mounts on 3 machines(node1, node2, node3) all writing to
-         * single file. Network split happened in a way that node1 can't see
-         * node2, node3. Node2, node3 both of them can't see node1. Now at the
-         * time of sending write all the bricks are up. Just after write fop is
-         * wound on node1, network split happens. Node1 thinks write fop failed
-         * on node2, node3 so marks pending changelog for those 2 extended
-         * attributes on node1. Node2, node3 thinks writes failed on node1 so
-         * they mark pending changelog for node1. When the network is stable
-         * again the file already is in split-brain. These checks prevent
-         * marking pending changelog on other subvolumes if the fop doesn't
-         * succeed in a way it is still following quorum. So with this fix what
-         * is happening is, node1 will have all pending changelog(FOOL) because
-         * the write succeeded only on node1 but failed on node2, node3 so
-         * instead of marking pending changelogs on node2, node3 it just treats
-         * the fop as failure and goes into DIRTY state. Where as node2, node3
-         * say they are sources and have pending changelog to node1 so there is
-         * no split-brain with the fix. The problem is eliminated completely.
-         */
+    if (AFR_COUNT(local->transaction.failed_subvols, priv->child_count) ==
+        priv->child_count)
+        return _gf_false;
 
-        if (afr_has_fop_cbk_quorum (frame))
-                return;
+    if (!afr_has_fop_cbk_quorum(frame))
+        need_dirty = _gf_true;
 
-        if (local->fd) {
-                gf_uuid_copy (gfid, local->fd->inode->gfid);
-                file = uuid_utoa (gfid);
-        } else {
-                loc_path (&local->loc, local->loc.name);
-                file = local->loc.path;
-        }
+    return need_dirty;
+}
 
-        gf_msg (frame->this->name, GF_LOG_WARNING, 0, AFR_MSG_QUORUM_FAIL,
-                "%s: Failing %s as quorum is not met",
-                file, gf_fop_list[local->op]);
+void
+afr_handle_quorum(call_frame_t *frame, xlator_t *this)
+{
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
+    int i = 0;
+    const char *file = NULL;
+    uuid_t gfid = {0};
+
+    local = frame->local;
+    priv = frame->this->private;
+
+    if (priv->quorum_count == 0)
+        return;
+
+    /* If the fop already failed return right away to preserve errno */
+    if (local->op_ret == -1)
+        return;
+
+    /*
+     * Network split may happen just after the fops are unwound, so check
+     * if the fop succeeded in a way it still follows quorum. If it doesn't,
+     * mark the fop as failure, mark the changelogs so it reflects that
+     * failure.
+     *
+     * Scenario:
+     * There are 3 mounts on 3 machines(node1, node2, node3) all writing to
+     * single file. Network split happened in a way that node1 can't see
+     * node2, node3. Node2, node3 both of them can't see node1. Now at the
+     * time of sending write all the bricks are up. Just after write fop is
+     * wound on node1, network split happens. Node1 thinks write fop failed
+     * on node2, node3 so marks pending changelog for those 2 extended
+     * attributes on node1. Node2, node3 thinks writes failed on node1 so
+     * they mark pending changelog for node1. When the network is stable
+     * again the file already is in split-brain. These checks prevent
+     * marking pending changelog on other subvolumes if the fop doesn't
+     * succeed in a way it is still following quorum. So with this fix what
+     * is happening is, node1 will have all pending changelog(FOOL) because
+     * the write succeeded only on node1 but failed on node2, node3 so
+     * instead of marking pending changelogs on node2, node3 it just treats
+     * the fop as failure and goes into DIRTY state. Where as node2, node3
+     * say they are sources and have pending changelog to node1 so there is
+     * no split-brain with the fix. The problem is eliminated completely.
+     */
+
+    if (afr_has_fop_cbk_quorum(frame))
+        return;
+
+    if (afr_need_dirty_marking(frame, this))
+        goto set_response;
+
+    for (i = 0; i < priv->child_count; i++) {
+        if (local->transaction.pre_op[i])
+            afr_transaction_fop_failed(frame, frame->this, i);
+    }
+
+set_response:
+    local->op_ret = -1;
+    local->op_errno = afr_final_errno(local, priv);
+    if (local->op_errno == 0)
+        local->op_errno = afr_quorum_errno(priv);
+
+    if (local->fd) {
+        gf_uuid_copy(gfid, local->fd->inode->gfid);
+        file = uuid_utoa(gfid);
+    } else {
+        loc_path(&local->loc, local->loc.name);
+        file = local->loc.path;
+    }
+
+    gf_msg(frame->this->name, GF_LOG_WARNING, local->op_errno,
+           AFR_MSG_QUORUM_FAIL, "%s: Failing %s as quorum is not met", file,
+           gf_fop_list[local->op]);
+
+    switch (local->transaction.type) {
+        case AFR_ENTRY_TRANSACTION:
+        case AFR_ENTRY_RENAME_TRANSACTION:
+            afr_pick_error_xdata(local, priv, local->parent, local->readable,
+                                 local->parent2, local->readable2);
+            break;
+        default:
+            afr_pick_error_xdata(local, priv, local->inode, local->readable,
+                                 NULL, NULL);
+            break;
+    }
+}
 
-        for (i = 0; i < priv->child_count; i++) {
-                if (local->transaction.pre_op[i])
-                        afr_transaction_fop_failed (frame, frame->this, i);
-        }
+int
+afr_fill_ta_loc(xlator_t *this, loc_t *loc, gf_boolean_t is_gfid_based_fop)
+{
+    afr_private_t *priv = NULL;
+
+    priv = this->private;
+    loc->parent = inode_ref(priv->root_inode);
+    gf_uuid_copy(loc->pargfid, loc->parent->gfid);
+    loc->name = priv->pending_key[THIN_ARBITER_BRICK_INDEX];
+    if (is_gfid_based_fop && gf_uuid_is_null(priv->ta_gfid)) {
+        /* Except afr_ta_id_file_check() which is path based, all other gluster
+         * FOPS need gfid.*/
+        return -EINVAL;
+    }
+    gf_uuid_copy(loc->gfid, priv->ta_gfid);
+    loc->inode = inode_new(loc->parent->table);
+    if (!loc->inode) {
+        loc_wipe(loc);
+        return -ENOMEM;
+    }
+    return 0;
+}
 
-        local->op_ret = -1;
-        local->op_errno = afr_quorum_errno (priv);
+static int
+afr_ta_post_op_done(int ret, call_frame_t *frame, void *opaque)
+{
+    xlator_t *this = NULL;
+    afr_local_t *local = NULL;
+    call_frame_t *txn_frame = NULL;
+    afr_ta_fop_state_t fop_state;
+
+    local = (afr_local_t *)opaque;
+    fop_state = local->fop_state;
+    txn_frame = local->transaction.frame;
+    this = frame->this;
+
+    if (ret == 0) {
+        /*Mark pending xattrs on the up data brick.*/
+        afr_post_op_handle_success(txn_frame, this);
+    } else {
+        afr_post_op_handle_failure(txn_frame, this, -ret);
+    }
+
+    STACK_DESTROY(frame->root);
+    afr_ta_process_onwireq(fop_state, this);
+
+    return 0;
 }
 
-int
-afr_changelog_post_op_now (call_frame_t *frame, xlator_t *this)
-{
-        afr_private_t * priv = this->private;
-        int i          = 0;
-	int ret = 0;
-	int idx = 0;
-        afr_local_t *  local = NULL;
-        dict_t        *xattr = NULL;
-        int            nothing_failed = 1;
-	gf_boolean_t   need_undirty = _gf_false;
-
-        afr_handle_quorum (frame);
-        local = frame->local;
-	idx = afr_index_for_transaction_type (local->transaction.type);
-
-        nothing_failed = afr_txn_nothing_failed (frame, this);
-
-	if (afr_changelog_pre_op_uninherit (frame, this))
-		need_undirty = _gf_false;
-	else
-		need_undirty = _gf_true;
-
-        //If the fop fails on all the subvols then pending markers are placed
-        //for every subvol on all subvolumes. Which is nothing but split-brain.
-        //Avoid this by not doing post-op in case of failures.
-        if (local->op_ret < 0) {
-                afr_changelog_post_op_done (frame, this);
-                goto out;
+int **
+afr_set_changelog_xattr(afr_private_t *priv, unsigned char *pending,
+                        dict_t *xattr, afr_local_t *local)
+{
+    int **changelog = NULL;
+    int idx = 0;
+    int ret = 0;
+    int i;
+
+    if (local->is_new_entry == _gf_true) {
+        changelog = afr_mark_pending_changelog(priv, pending, xattr,
+                                               local->cont.dir_fop.buf.ia_type);
+    } else {
+        idx = afr_index_for_transaction_type(local->transaction.type);
+        changelog = afr_matrix_create(priv->child_count, AFR_NUM_CHANGE_LOGS);
+        if (!changelog) {
+            goto out;
+        }
+        for (i = 0; i < priv->child_count; i++) {
+            if (local->transaction.failed_subvols[i])
+                changelog[i][idx] = hton32(1);
+        }
+        ret = afr_set_pending_dict(priv, xattr, changelog);
+        if (ret < 0) {
+            afr_matrix_cleanup(changelog, priv->child_count);
+            return NULL;
         }
+    }
 
-	if (nothing_failed && !need_undirty) {
-		afr_changelog_post_op_done (frame, this);
-                goto out;
-	}
-
-	xattr = dict_new ();
-	if (!xattr) {
-		local->op_ret = -1;
-		local->op_errno = ENOMEM;
-		afr_changelog_post_op_done (frame, this);
-		goto out;
-	}
-
-	if (need_undirty)
-		local->dirty[idx] = hton32(-1);
-	else
-		local->dirty[idx] = hton32(0);
-
-	ret = dict_set_static_bin (xattr, AFR_DIRTY, local->dirty,
-				   sizeof(int) * AFR_NUM_CHANGE_LOGS);
-	if (ret) {
-		local->op_ret = -1;
-		local->op_errno = ENOMEM;
-		afr_changelog_post_op_done (frame, this);
-		goto out;
-	}
-
-	for (i = 0; i < priv->child_count; i++) {
-		if (local->transaction.failed_subvols[i])
-			local->pending[i][idx] = hton32(1);
-	}
-
-	ret = afr_set_pending_dict (priv, xattr, local->pending);
-	if (ret < 0) {
-		local->op_ret = -1;
-		local->op_errno = ENOMEM;
-		afr_changelog_post_op_done (frame, this);
-		goto out;
-	}
-
-	afr_changelog_do (frame, this, xattr, afr_changelog_post_op_done);
 out:
-	if (xattr)
-                dict_unref (xattr);
-
-        return 0;
+    return changelog;
 }
 
+static void
+afr_ta_locked_xattrop_validate(afr_private_t *priv, afr_local_t *local,
+                               gf_boolean_t *valid)
+{
+    if (priv->ta_event_gen > local->ta_event_gen) {
+        /* We can't trust the ta's response anymore.*/
+        afr_ta_locked_priv_invalidate(priv);
+        *valid = _gf_false;
+        return;
+    }
+    return;
+}
 
-gf_boolean_t
-afr_changelog_pre_op_uninherit (call_frame_t *frame, xlator_t *this)
-{
-	afr_local_t *local = NULL;
-	afr_private_t *priv = NULL;
-	fd_t *fd = NULL;
-	int i = 0;
-	gf_boolean_t ret = _gf_false;
-	afr_fd_ctx_t *fd_ctx = NULL;
-	int type = 0;
-
-	local = frame->local;
-	priv = this->private;
-	fd = local->fd;
-
-	type = afr_index_for_transaction_type (local->transaction.type);
-	if (type != AFR_DATA_TRANSACTION)
-		return !local->transaction.dirtied;
-
-	if (!fd)
-		return !local->transaction.dirtied;
-
-	fd_ctx = afr_fd_ctx_get (fd, this);
-	if (!fd_ctx)
-		return _gf_false;
-
-	if (local->transaction.no_uninherit)
-		return _gf_false;
-
-	/* This function must be idempotent. So check if we
-	   were called before and return the same answer again.
-
-	   It is important to keep this function idempotent for
-	   the call in afr_changelog_post_op_safe() to not have
-	   side effects on the call from afr_changelog_post_op_now()
-	*/
-	if (local->transaction.uninherit_done)
-		return local->transaction.uninherit_value;
-
-	LOCK(&fd->lock);
-	{
-		for (i = 0; i < priv->child_count; i++) {
-			if (local->transaction.pre_op[i] !=
-			    fd_ctx->pre_op_done[type][i]) {
-				ret = !local->transaction.dirtied;
-				goto unlock;
-			}
-		}
-
-		if (fd_ctx->inherited[type]) {
-			ret = _gf_true;
-			fd_ctx->inherited[type]--;
-		} else if (fd_ctx->on_disk[type]) {
-			ret = _gf_false;
-			fd_ctx->on_disk[type]--;
-		} else {
-			/* ASSERT */
-			ret = _gf_false;
-		}
-
-		if (!fd_ctx->inherited[type] && !fd_ctx->on_disk[type]) {
-			for (i = 0; i < priv->child_count; i++)
-				fd_ctx->pre_op_done[type][i] = 0;
-		}
-	}
-unlock:
-	UNLOCK(&fd->lock);
+static int
+afr_ta_post_op_do(void *opaque)
+{
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
+    xlator_t *this = NULL;
+    dict_t *xattr = NULL;
+    unsigned char *pending = NULL;
+    int **changelog = NULL;
+    int failed_subvol = -1;
+    int success_subvol = -1;
+    loc_t loc = {
+        0,
+    };
+    int i = 0;
+    int ret = 0;
+    gf_boolean_t valid = _gf_true;
+
+    local = (afr_local_t *)opaque;
+    this = local->transaction.frame->this;
+    priv = this->private;
+
+    ret = afr_fill_ta_loc(this, &loc, _gf_true);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, -ret, AFR_MSG_THIN_ARB,
+               "Failed to populate loc for thin-arbiter.");
+        goto out;
+    }
+
+    xattr = dict_new();
+    if (!xattr) {
+        ret = -ENOMEM;
+        goto out;
+    }
+
+    pending = alloca0(priv->child_count);
+
+    for (i = 0; i < priv->child_count; i++) {
+        if (local->transaction.failed_subvols[i]) {
+            pending[i] = 1;
+            failed_subvol = i;
+        } else {
+            success_subvol = i;
+        }
+    }
+
+    changelog = afr_set_changelog_xattr(priv, pending, xattr, local);
+
+    if (!changelog) {
+        ret = -ENOMEM;
+        goto out;
+    }
+
+    ret = afr_ta_post_op_lock(this, &loc);
+    if (ret)
+        goto out;
+
+    ret = syncop_xattrop(priv->children[THIN_ARBITER_BRICK_INDEX], &loc,
+                         GF_XATTROP_ADD_ARRAY, xattr, NULL, NULL, NULL);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, -ret, AFR_MSG_THIN_ARB,
+               "Post-op on thin-arbiter id file %s failed for gfid %s.",
+               priv->pending_key[THIN_ARBITER_BRICK_INDEX],
+               uuid_utoa(local->inode->gfid));
+    }
+    LOCK(&priv->lock);
+    {
+        if (ret == 0) {
+            priv->ta_bad_child_index = failed_subvol;
+        } else if (ret == -EINVAL) {
+            priv->ta_bad_child_index = success_subvol;
+            ret = -EIO; /* TA failed the fop. Return EIO to application. */
+        }
 
-	local->transaction.uninherit_done = _gf_true;
-	local->transaction.uninherit_value = ret;
+        afr_ta_locked_xattrop_validate(priv, local, &valid);
+    }
+    UNLOCK(&priv->lock);
+    if (valid == _gf_false) {
+        gf_msg(this->name, GF_LOG_ERROR, EIO, AFR_MSG_THIN_ARB,
+               "Post-op on thin-arbiter id file %s for gfid %s invalidated due "
+               "to event-gen mismatch.",
+               priv->pending_key[THIN_ARBITER_BRICK_INDEX],
+               uuid_utoa(local->inode->gfid));
+        ret = -EIO;
+    }
+
+    afr_ta_post_op_unlock(this, &loc);
+out:
+    if (xattr)
+        dict_unref(xattr);
 
-	return ret;
-}
+    if (changelog)
+        afr_matrix_cleanup(changelog, priv->child_count);
 
+    loc_wipe(&loc);
 
-gf_boolean_t
-afr_changelog_pre_op_inherit (call_frame_t *frame, xlator_t *this)
-{
-	afr_local_t *local = NULL;
-	afr_private_t *priv = NULL;
-	fd_t *fd = NULL;
-	int i = 0;
-	gf_boolean_t ret = _gf_false;
-	afr_fd_ctx_t *fd_ctx = NULL;
-	int type = 0;
-
-	local = frame->local;
-	priv = this->private;
-	fd = local->fd;
-
-	if (local->transaction.type != AFR_DATA_TRANSACTION)
-		return _gf_false;
-
-	type = afr_index_for_transaction_type (local->transaction.type);
-
-	if (!fd)
-		return _gf_false;
-
-	fd_ctx = afr_fd_ctx_get (fd, this);
-	if (!fd_ctx)
-		return _gf_false;
-
-	LOCK(&fd->lock);
-	{
-		if (!fd_ctx->on_disk[type]) {
-			/* nothing to inherit yet */
-			ret = _gf_false;
-			goto unlock;
-		}
-
-		for (i = 0; i < priv->child_count; i++) {
-			if (local->transaction.pre_op[i] !=
-			    fd_ctx->pre_op_done[type][i]) {
-				/* either inherit exactly, or don't */
-				ret = _gf_false;
-				goto unlock;
-			}
-		}
-
-		fd_ctx->inherited[type]++;
-
-		ret = _gf_true;
-
-		local->transaction.inherited = _gf_true;
-	}
-unlock:
-	UNLOCK(&fd->lock);
+    return ret;
+}
 
-	return ret;
+static int
+afr_ta_post_op_synctask(xlator_t *this, afr_local_t *local)
+{
+    call_frame_t *ta_frame = NULL;
+    int ret = 0;
+
+    ta_frame = afr_ta_frame_create(this);
+    if (!ta_frame) {
+        gf_msg(this->name, GF_LOG_ERROR, ENOMEM, AFR_MSG_THIN_ARB,
+               "Failed to create ta_frame");
+        goto err;
+    }
+    ret = synctask_new(this->ctx->env, afr_ta_post_op_do, afr_ta_post_op_done,
+                       ta_frame, local);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, ENOMEM, AFR_MSG_THIN_ARB,
+               "Failed to launch post-op on thin arbiter for gfid %s",
+               uuid_utoa(local->inode->gfid));
+        STACK_DESTROY(ta_frame->root);
+        goto err;
+    }
+
+    return ret;
+err:
+    afr_changelog_post_op_fail(local->transaction.frame, this, ENOMEM);
+    return ret;
 }
 
+static void
+afr_ta_set_fop_state(afr_private_t *priv, afr_local_t *local,
+                     int *on_wire_count)
+{
+    LOCK(&priv->lock);
+    {
+        if (priv->release_ta_notify_dom_lock == _gf_true) {
+            /* Put the fop in waitq until notify dom lock is released.*/
+            local->fop_state = TA_WAIT_FOR_NOTIFY_LOCK_REL;
+            list_add_tail(&local->ta_waitq, &priv->ta_waitq);
+        } else if (priv->ta_bad_child_index == AFR_CHILD_UNKNOWN) {
+            /* Post-op on thin-arbiter to decide success/failure. */
+            local->fop_state = TA_GET_INFO_FROM_TA_FILE;
+            *on_wire_count = ++priv->ta_on_wire_txn_count;
+            if (*on_wire_count > 1) {
+                /*Avoid sending multiple on-wire post-ops on TA*/
+                list_add_tail(&local->ta_onwireq, &priv->ta_onwireq);
+            }
+        } else if (local->ta_failed_subvol == priv->ta_bad_child_index) {
+            /* Post-op on TA not needed as the fop failed on the in-memory bad
+             * brick. Just mark pending xattrs on the good data brick.*/
+            local->fop_state = TA_INFO_IN_MEMORY_SUCCESS;
+            priv->ta_in_mem_txn_count++;
+        } else {
+            /* Post-op on TA not needed as the fop succeeded only on the
+             * in-memory bad data brick and not the good one. Fail the fop.*/
+            local->fop_state = TA_INFO_IN_MEMORY_FAILED;
+            priv->ta_in_mem_txn_count++;
+        }
+    }
+    UNLOCK(&priv->lock);
+}
 
-gf_boolean_t
-afr_changelog_pre_op_update (call_frame_t *frame, xlator_t *this)
-{
-	afr_local_t *local = NULL;
-	afr_private_t *priv = NULL;
-	fd_t *fd = NULL;
-	afr_fd_ctx_t *fd_ctx = NULL;
-	int i = 0;
-	gf_boolean_t ret = _gf_false;
-	int type = 0;
-
-	local = frame->local;
-	priv = this->private;
-	fd = local->fd;
-
-	if (!fd)
-		return _gf_false;
-
-	fd_ctx = afr_fd_ctx_get (fd, this);
-	if (!fd_ctx)
-		return _gf_false;
-
-	if (local->transaction.inherited)
-		/* was already inherited in afr_changelog_pre_op */
-		return _gf_false;
-
-	if (!local->transaction.dirtied)
-		return _gf_false;
-
-        if (!afr_txn_nothing_failed (frame, this))
-		return _gf_false;
-
-	type = afr_index_for_transaction_type (local->transaction.type);
-
-	ret = _gf_false;
-
-	LOCK(&fd->lock);
-	{
-		if (!fd_ctx->on_disk[type]) {
-			for (i = 0; i < priv->child_count; i++)
-				fd_ctx->pre_op_done[type][i] =
-					local->transaction.pre_op[i];
-		} else {
-			for (i = 0; i < priv->child_count; i++)
-				if (fd_ctx->pre_op_done[type][i] !=
-				    local->transaction.pre_op[i]) {
-					local->transaction.no_uninherit = 1;
-					goto unlock;
-				}
-		}
-		fd_ctx->on_disk[type]++;
-
-		ret = _gf_true;
-	}
-unlock:
-	UNLOCK(&fd->lock);
+static void
+afr_ta_fill_failed_subvol(afr_private_t *priv, afr_local_t *local)
+{
+    int i = 0;
 
-	return ret;
+    for (i = 0; i < priv->child_count; i++) {
+        if (local->transaction.failed_subvols[i]) {
+            local->ta_failed_subvol = i;
+            break;
+        }
+    }
 }
 
-
-int
-afr_changelog_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-		   int op_ret, int op_errno, dict_t *xattr, dict_t *xdata)
+static void
+afr_post_op_handle_success(call_frame_t *frame, xlator_t *this)
 {
-        afr_local_t *local = NULL;
-        int call_count = -1;
+    afr_local_t *local = NULL;
 
-        local = frame->local;
+    local = frame->local;
+    if (local->is_new_entry == _gf_true) {
+        afr_mark_new_entry_changelog(frame, this);
+    }
+    afr_changelog_post_op_do(frame, this);
 
-	if (op_ret == -1)
-		afr_transaction_fop_failed (frame, this, (long) cookie);
+    return;
+}
 
-	call_count = afr_frame_return (frame);
+static void
+afr_post_op_handle_failure(call_frame_t *frame, xlator_t *this, int op_errno)
+{
+    afr_changelog_post_op_fail(frame, this, op_errno);
 
-        if (call_count == 0)
-		local->transaction.changelog_resume (frame, this);
+    return;
+}
 
-        return 0;
+static void
+afr_ta_decide_post_op_state(call_frame_t *frame, xlator_t *this)
+{
+    afr_private_t *priv = NULL;
+    afr_local_t *local = NULL;
+    int on_wire_count = 0;
+
+    priv = this->private;
+    local = frame->local;
+
+    afr_ta_set_fop_state(priv, local, &on_wire_count);
+
+    switch (local->fop_state) {
+        case TA_GET_INFO_FROM_TA_FILE:
+            if (on_wire_count == 1)
+                afr_ta_post_op_synctask(this, local);
+            /*else, fop is queued in ta_onwireq.*/
+            break;
+        case TA_WAIT_FOR_NOTIFY_LOCK_REL:
+            /*Post releasing the notify lock, we will act on this queue*/
+            break;
+        case TA_INFO_IN_MEMORY_SUCCESS:
+            afr_post_op_handle_success(frame, this);
+            break;
+        case TA_INFO_IN_MEMORY_FAILED:
+            afr_post_op_handle_failure(frame, this, EIO);
+            break;
+        default:
+            break;
+    }
+    return;
 }
 
+static void
+afr_handle_failure_using_thin_arbiter(call_frame_t *frame, xlator_t *this)
+{
+    afr_private_t *priv = this->private;
+    afr_local_t *local = frame->local;
+
+    afr_ta_fill_failed_subvol(priv, local);
+    gf_msg_debug(this->name, 0,
+                 "Fop failed on data brick (%s) for gfid=%s. "
+                 "ta info needed to decide fop result.",
+                 priv->children[local->ta_failed_subvol]->name,
+                 uuid_utoa(local->inode->gfid));
+    afr_ta_decide_post_op_state(frame, this);
+}
 
-int
-afr_changelog_do (call_frame_t *frame, xlator_t *this, dict_t *xattr,
-		  afr_changelog_resume_t changelog_resume)
+void
+afr_changelog_post_op_do(call_frame_t *frame, xlator_t *this)
 {
-	afr_local_t *local = NULL;
-	afr_private_t *priv = NULL;
-	int i = 0;
-	int call_count = 0;
+    afr_private_t *priv = this->private;
+    afr_local_t *local = NULL;
+    dict_t *xattr = NULL;
+    int i = 0;
+    int ret = 0;
+    int idx = 0;
+    int nothing_failed = 1;
+    gf_boolean_t need_undirty = _gf_false;
+
+    afr_handle_quorum(frame, this);
+    local = frame->local;
+    idx = afr_index_for_transaction_type(local->transaction.type);
+
+    xattr = dict_new();
+    if (!xattr) {
+        afr_changelog_post_op_fail(frame, this, ENOMEM);
+        goto out;
+    }
+
+    nothing_failed = afr_txn_nothing_failed(frame, this);
+
+    if (afr_changelog_pre_op_uninherit(frame, this))
+        need_undirty = _gf_false;
+    else
+        need_undirty = _gf_true;
+
+    if (local->op_ret < 0 && !nothing_failed) {
+        if (afr_need_dirty_marking(frame, this)) {
+            local->dirty[idx] = hton32(1);
+            goto set_dirty;
+        }
+
+        afr_changelog_post_op_done(frame, this);
+        goto out;
+    }
+
+    if (nothing_failed && !need_undirty) {
+        afr_changelog_post_op_done(frame, this);
+        goto out;
+    }
+
+    if (local->transaction.in_flight_sb) {
+        afr_changelog_post_op_fail(frame, this,
+                                   local->transaction.in_flight_sb_errno);
+        goto out;
+    }
+
+    for (i = 0; i < priv->child_count; i++) {
+        if (local->transaction.failed_subvols[i])
+            local->pending[i][idx] = hton32(1);
+    }
+
+    ret = afr_set_pending_dict(priv, xattr, local->pending);
+    if (ret < 0) {
+        afr_changelog_post_op_fail(frame, this, ENOMEM);
+        goto out;
+    }
+
+    if (need_undirty)
+        local->dirty[idx] = hton32(-1);
+    else
+        local->dirty[idx] = hton32(0);
+
+set_dirty:
+    ret = dict_set_static_bin(xattr, AFR_DIRTY, local->dirty,
+                              sizeof(int) * AFR_NUM_CHANGE_LOGS);
+    if (ret) {
+        afr_changelog_post_op_fail(frame, this, ENOMEM);
+        goto out;
+    }
+
+    afr_changelog_do(frame, this, xattr, afr_changelog_post_op_done,
+                     AFR_TRANSACTION_POST_OP);
+out:
+    if (xattr)
+        dict_unref(xattr);
+
+    return;
+}
 
-	local = frame->local;
-	priv = this->private;
+static int
+afr_changelog_post_op_now(call_frame_t *frame, xlator_t *this)
+{
+    afr_private_t *priv = NULL;
+    afr_local_t *local = NULL;
+    int failed_count = 0;
+
+    priv = this->private;
+    local = frame->local;
+
+    if (priv->thin_arbiter_count) {
+        failed_count = AFR_COUNT(local->transaction.failed_subvols,
+                                 priv->child_count);
+        if (failed_count == 1) {
+            afr_handle_failure_using_thin_arbiter(frame, this);
+            return 0;
+        } else {
+            /* Txn either succeeded or failed on both data bricks. Let
+             * post_op_do handle it as the case might be. */
+        }
+    }
 
-        call_count = afr_changelog_call_count (local->transaction.type,
-					       local->transaction.pre_op,
-					       priv->child_count);
+    afr_changelog_post_op_do(frame, this);
+    return 0;
+}
 
-	if (call_count == 0) {
-		changelog_resume (frame, this);
-		return 0;
-	}
+gf_boolean_t
+afr_changelog_pre_op_uninherit(call_frame_t *frame, xlator_t *this)
+{
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
+    afr_inode_ctx_t *ctx = NULL;
+    int i = 0;
+    gf_boolean_t ret = _gf_false;
+    int type = 0;
+
+    local = frame->local;
+    priv = this->private;
+    ctx = local->inode_ctx;
+
+    type = afr_index_for_transaction_type(local->transaction.type);
+    if (type != AFR_DATA_TRANSACTION)
+        return !local->transaction.dirtied;
+
+    if (local->transaction.no_uninherit)
+        return _gf_false;
 
-	local->call_count = call_count;
+    /* This function must be idempotent. So check if we
+       were called before and return the same answer again.
 
-	local->transaction.changelog_resume = changelog_resume;
+       It is important to keep this function idempotent for
+       the call in afr_changelog_post_op_safe() to not have
+       side effects on the call from afr_changelog_post_op_now()
+    */
+    if (local->transaction.uninherit_done)
+        return local->transaction.uninherit_value;
 
+    LOCK(&local->inode->lock);
+    {
         for (i = 0; i < priv->child_count; i++) {
-                if (!local->transaction.pre_op[i])
-                        continue;
-
-                switch (local->transaction.type) {
-                case AFR_DATA_TRANSACTION:
-                case AFR_METADATA_TRANSACTION:
-                        if (!local->fd) {
-                                STACK_WIND_COOKIE (frame, afr_changelog_cbk,
-						   (void *) (long) i,
-						   priv->children[i],
-						   priv->children[i]->fops->xattrop,
-						   &local->loc,
-						   GF_XATTROP_ADD_ARRAY, xattr,
-						   NULL);
-                        } else {
-                                STACK_WIND_COOKIE (frame, afr_changelog_cbk,
-						   (void *) (long) i,
-						   priv->children[i],
-						   priv->children[i]->fops->fxattrop,
-						   local->fd,
-						   GF_XATTROP_ADD_ARRAY, xattr,
-						   NULL);
-                        }
-			break;
-                case AFR_ENTRY_RENAME_TRANSACTION:
-
-			STACK_WIND_COOKIE (frame, afr_changelog_cbk,
-					   (void *) (long) i,
-					   priv->children[i],
-					   priv->children[i]->fops->xattrop,
-					   &local->transaction.new_parent_loc,
-					   GF_XATTROP_ADD_ARRAY, xattr,
-					   NULL);
-                        call_count--;
+            if (local->transaction.pre_op[i] != ctx->pre_op_done[type][i]) {
+                ret = !local->transaction.dirtied;
+                goto unlock;
+            }
+        }
 
-                /* fall through */
+        if (ctx->inherited[type]) {
+            ret = _gf_true;
+            ctx->inherited[type]--;
+        } else if (ctx->on_disk[type]) {
+            ret = _gf_false;
+            ctx->on_disk[type]--;
+        } else {
+            /* ASSERT */
+            ret = _gf_false;
+        }
 
-                case AFR_ENTRY_TRANSACTION:
-                        if (local->fd)
-                                STACK_WIND_COOKIE (frame, afr_changelog_cbk,
-						   (void *) (long) i,
-						   priv->children[i],
-						   priv->children[i]->fops->fxattrop,
-						   local->fd,
-						   GF_XATTROP_ADD_ARRAY, xattr,
-						   NULL);
-                        else
-                                STACK_WIND_COOKIE (frame, afr_changelog_cbk,
-						   (void *) (long) i,
-						   priv->children[i],
-						   priv->children[i]->fops->xattrop,
-						   &local->transaction.parent_loc,
-						   GF_XATTROP_ADD_ARRAY, xattr,
-						   NULL);
-			break;
-		}
-
-                if (!--call_count)
-                        break;
+        if (!ctx->inherited[type] && !ctx->on_disk[type]) {
+            for (i = 0; i < priv->child_count; i++)
+                ctx->pre_op_done[type][i] = 0;
         }
+    }
+unlock:
+    UNLOCK(&local->inode->lock);
 
-	return 0;
+    local->transaction.uninherit_done = _gf_true;
+    local->transaction.uninherit_value = ret;
+
+    return ret;
 }
 
+gf_boolean_t
+afr_changelog_pre_op_inherit(call_frame_t *frame, xlator_t *this)
+{
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
+    int i = 0;
+    gf_boolean_t ret = _gf_false;
+    int type = 0;
+
+    local = frame->local;
+    priv = this->private;
 
-int
-afr_changelog_pre_op (call_frame_t *frame, xlator_t *this)
-{
-        afr_private_t * priv = this->private;
-        int i = 0;
-        int ret = 0;
-        int call_count = 0;
-	int op_errno = 0;
-        afr_local_t *local = NULL;
-        afr_internal_lock_t *int_lock = NULL;
-        unsigned char       *locked_nodes = NULL;
-	unsigned char       *pending_subvols = NULL;
-	int idx = -1;
-	gf_boolean_t pre_nop = _gf_true;
-	dict_t *xdata_req = NULL;
-
-        local = frame->local;
-        int_lock = &local->internal_lock;
-	idx = afr_index_for_transaction_type (local->transaction.type);
-
-        locked_nodes = afr_locked_nodes_get (local->transaction.type, int_lock);
-
-	pending_subvols = alloca0 (priv->child_count);
-
-	for (i = 0; i < priv->child_count; i++) {
-		if (locked_nodes[i]) {
-			local->transaction.pre_op[i] = 1;
-			call_count++;
-		} else {
-			pending_subvols[i] = 1;
-		}
-	}
-
-        /* This condition should not be met with present code, as
-         * transaction.done will be called if locks are not acquired on even a
-         * single node.
-         */
-        if (call_count == 0) {
-		op_errno = ENOTCONN;
-		goto err;
-	}
+    if (local->transaction.type != AFR_DATA_TRANSACTION)
+        return _gf_false;
 
-        /* Check if the fop can be performed on at least
-         * quorum number of nodes.
-         */
-        if (priv->quorum_count && !afr_has_fop_quorum (frame)) {
-                op_errno = afr_quorum_errno (priv);
-                goto err;
-        }
+    type = afr_index_for_transaction_type(local->transaction.type);
 
-	xdata_req = dict_new();
-	if (!xdata_req) {
-		op_errno = ENOMEM;
-		goto err;
-	}
-
-	pre_nop = _gf_true;
-
-	if (afr_changelog_pre_op_inherit (frame, this))
-		goto next;
-
-	if (call_count < priv->child_count) {
-		/* For subvols we are not performing operation on,
-		   mark them as pending up-front along with the FOP
-		   so that we can safely defer unmarking dirty until
-		   later.
-		*/
-		for (i = 0; i < priv->child_count; i++) {
-			if (pending_subvols[i])
-				local->pending[i][idx] = hton32(1);
-		}
-		ret = afr_set_pending_dict (priv, xdata_req,
-					    local->pending);
-		if (ret < 0) {
-			op_errno = ENOMEM;
-			goto err;
-		}
-		pre_nop = _gf_false;
-	}
-
-	if (call_count > 1 &&
-	    (local->transaction.type == AFR_DATA_TRANSACTION ||
-	     !local->optimistic_change_log)) {
-
-		/* If we are performing change on only one subvol, no
-		   need to mark dirty, because we are setting the pending
-		   counts already anyways
-		*/
-		local->dirty[idx] = hton32(1);
-
-		ret = dict_set_static_bin (xdata_req, AFR_DIRTY, local->dirty,
-					   sizeof(int) * AFR_NUM_CHANGE_LOGS);
-		if (ret) {
-			op_errno = ENOMEM;
-			goto err;
-		}
-
-		pre_nop = _gf_false;
-		local->transaction.dirtied = 1;
-	}
-
-	if (pre_nop)
-		goto next;
-
-	if (!local->pre_op_compat) {
-		dict_copy (xdata_req, local->xdata_req);
-		goto next;
-	}
-
-	afr_changelog_do (frame, this, xdata_req, afr_transaction_perform_fop);
-
-	if (xdata_req)
-		dict_unref (xdata_req);
-
-	return 0;
-next:
-	afr_transaction_perform_fop (frame, this);
+    LOCK(&local->inode->lock);
+    {
+        if (!local->inode_ctx->on_disk[type]) {
+            /* nothing to inherit yet */
+            ret = _gf_false;
+            goto unlock;
+        }
 
-	if (xdata_req)
-		dict_unref (xdata_req);
+        for (i = 0; i < priv->child_count; i++) {
+            if (local->transaction.pre_op[i] !=
+                local->inode_ctx->pre_op_done[type][i]) {
+                /* either inherit exactly, or don't */
+                ret = _gf_false;
+                goto unlock;
+            }
+        }
 
-        return 0;
-err:
-	local->internal_lock.lock_cbk = local->transaction.done;
-	local->op_ret = -1;
-	local->op_errno = op_errno;
+        local->inode_ctx->inherited[type]++;
 
-	afr_unlock (frame, this);
+        ret = _gf_true;
 
-	if (xdata_req)
-		dict_unref (xdata_req);
+        local->transaction.inherited = _gf_true;
+    }
+unlock:
+    UNLOCK(&local->inode->lock);
 
-	return 0;
+    return ret;
 }
 
-
-int
-afr_post_blocking_inodelk_cbk (call_frame_t *frame, xlator_t *this)
+gf_boolean_t
+afr_changelog_pre_op_update(call_frame_t *frame, xlator_t *this)
 {
-        afr_internal_lock_t *int_lock = NULL;
-        afr_local_t         *local    = NULL;
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
+    int i = 0;
+    gf_boolean_t ret = _gf_false;
+    int type = 0;
 
-        local    = frame->local;
-        int_lock = &local->internal_lock;
+    local = frame->local;
+    priv = this->private;
 
-        if (int_lock->lock_op_ret < 0) {
-                gf_log (this->name, GF_LOG_INFO,
-                        "Blocking inodelks failed.");
-                local->transaction.done (frame, this);
-        } else {
+    if (local->transaction.type == AFR_ENTRY_TRANSACTION ||
+        local->transaction.type == AFR_ENTRY_RENAME_TRANSACTION)
+        return _gf_false;
 
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "Blocking inodelks done. Proceeding to FOP");
-                afr_internal_lock_finish (frame, this);
-        }
+    if (local->transaction.inherited)
+        /* was already inherited in afr_changelog_pre_op */
+        return _gf_false;
 
-        return 0;
-}
+    if (!local->transaction.dirtied)
+        return _gf_false;
 
+    if (!afr_txn_nothing_failed(frame, this))
+        return _gf_false;
 
-int
-afr_post_nonblocking_inodelk_cbk (call_frame_t *frame, xlator_t *this)
-{
-        afr_internal_lock_t *int_lock = NULL;
-        afr_local_t         *local    = NULL;
+    type = afr_index_for_transaction_type(local->transaction.type);
 
-        local    = frame->local;
-        int_lock = &local->internal_lock;
+    ret = _gf_false;
 
-        /* Initiate blocking locks if non-blocking has failed */
-        if (int_lock->lock_op_ret < 0) {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "Non blocking inodelks failed. Proceeding to blocking");
-                int_lock->lock_cbk = afr_post_blocking_inodelk_cbk;
-                afr_blocking_lock (frame, this);
+    LOCK(&local->inode->lock);
+    {
+        if (!local->inode_ctx->on_disk[type]) {
+            for (i = 0; i < priv->child_count; i++)
+                local->inode_ctx->pre_op_done[type][i] =
+                    (!local->transaction.failed_subvols[i]);
         } else {
-
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "Non blocking inodelks done. Proceeding to FOP");
-                afr_internal_lock_finish (frame, this);
+            for (i = 0; i < priv->child_count; i++)
+                if (local->inode_ctx->pre_op_done[type][i] !=
+                    (!local->transaction.failed_subvols[i])) {
+                    local->transaction.no_uninherit = 1;
+                    goto unlock;
+                }
         }
+        local->inode_ctx->on_disk[type]++;
 
-        return 0;
-}
+        ret = _gf_true;
+    }
+unlock:
+    UNLOCK(&local->inode->lock);
 
+    return ret;
+}
 
 int
-afr_post_blocking_entrylk_cbk (call_frame_t *frame, xlator_t *this)
+afr_changelog_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int op_ret,
+                  int op_errno, dict_t *xattr, dict_t *xdata)
 {
-        afr_internal_lock_t *int_lock = NULL;
-        afr_local_t         *local    = NULL;
+    afr_local_t *local = NULL;
+    int call_count = -1;
+    int child_index = -1;
 
-        local    = frame->local;
-        int_lock = &local->internal_lock;
+    local = frame->local;
+    child_index = (long)cookie;
 
-        if (int_lock->lock_op_ret < 0) {
-                gf_log (this->name, GF_LOG_INFO,
-                        "Blocking entrylks failed.");
-                local->transaction.done (frame, this);
-        } else {
+    if (op_ret == -1) {
+        local->op_errno = op_errno;
+        afr_transaction_fop_failed(frame, this, child_index);
+    }
 
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "Blocking entrylks done. Proceeding to FOP");
-                afr_internal_lock_finish (frame, this);
-        }
-
-        return 0;
-}
+    if (xattr)
+        local->transaction.changelog_xdata[child_index] = dict_ref(xattr);
 
+    call_count = afr_frame_return(frame);
 
-int
-afr_post_nonblocking_entrylk_cbk (call_frame_t *frame, xlator_t *this)
-{
-        afr_internal_lock_t *int_lock = NULL;
-        afr_local_t         *local    = NULL;
+    if (call_count == 0) {
+        local->transaction.changelog_resume(frame, this);
+    }
 
-        local = frame->local;
-        int_lock = &local->internal_lock;
+    return 0;
+}
 
-        /* Initiate blocking locks if non-blocking has failed */
-        if (int_lock->lock_op_ret < 0) {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "Non blocking entrylks failed. Proceeding to blocking");
-                int_lock->lock_cbk = afr_post_blocking_entrylk_cbk;
-                afr_blocking_lock (frame, this);
-        } else {
+void
+afr_changelog_populate_xdata(call_frame_t *frame, afr_xattrop_type_t op,
+                             dict_t **xdata, dict_t **newloc_xdata)
+{
+    int i = 0;
+    int ret = 0;
+    char *key = NULL;
+    int keylen = 0;
+    const char *name = NULL;
+    dict_t *xdata1 = NULL;
+    dict_t *xdata2 = NULL;
+    xlator_t *this = NULL;
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
+    gf_boolean_t need_entry_key_set = _gf_true;
+
+    local = frame->local;
+    this = THIS;
+    priv = this->private;
+
+    if (local->transaction.type == AFR_DATA_TRANSACTION ||
+        local->transaction.type == AFR_METADATA_TRANSACTION)
+        goto out;
+
+    if (!priv->esh_granular)
+        goto out;
+
+    xdata1 = dict_new();
+    if (!xdata1)
+        goto out;
+
+    name = local->loc.name;
+    if (local->op == GF_FOP_LINK)
+        name = local->newloc.name;
+
+    switch (op) {
+        case AFR_TRANSACTION_PRE_OP:
+            key = GF_XATTROP_ENTRY_IN_KEY;
+            break;
+        case AFR_TRANSACTION_POST_OP:
+            if (afr_txn_nothing_failed(frame, this)) {
+                key = GF_XATTROP_ENTRY_OUT_KEY;
+                for (i = 0; i < priv->child_count; i++) {
+                    if (!local->transaction.failed_subvols[i])
+                        continue;
+                    need_entry_key_set = _gf_false;
+                    break;
+                }
+                /* If the transaction itself did not fail and there
+                 * are no failed subvolumes, check whether the fop
+                 * failed due to a symmetric error. If it did, do
+                 * not set the ENTRY_OUT xattr which would end up
+                 * deleting a name index which was created possibly by
+                 * an earlier entry txn that may have failed on some
+                 * of the sub-volumes.
+                 */
+                if (local->op_ret)
+                    need_entry_key_set = _gf_false;
+            } else {
+                key = GF_XATTROP_ENTRY_IN_KEY;
+            }
+            break;
+    }
+
+    if (need_entry_key_set) {
+        keylen = strlen(key);
+        ret = dict_set_strn(xdata1, key, keylen, (char *)name);
+        if (ret)
+            gf_msg(THIS->name, GF_LOG_ERROR, 0, AFR_MSG_DICT_SET_FAILED,
+                   "%s/%s: Could not set %s key during xattrop",
+                   uuid_utoa(local->loc.pargfid), local->loc.name, key);
+        if (local->transaction.type == AFR_ENTRY_RENAME_TRANSACTION) {
+            xdata2 = dict_new();
+            if (!xdata2)
+                goto out;
 
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "Non blocking entrylks done. Proceeding to FOP");
-                afr_internal_lock_finish (frame, this);
+            ret = dict_set_strn(xdata2, key, keylen,
+                                (char *)local->newloc.name);
+            if (ret)
+                gf_msg(THIS->name, GF_LOG_ERROR, 0, AFR_MSG_DICT_SET_FAILED,
+                       "%s/%s: Could not set %s key during "
+                       "xattrop",
+                       uuid_utoa(local->newloc.pargfid), local->newloc.name,
+                       key);
         }
+    }
 
-        return 0;
+    *xdata = xdata1;
+    *newloc_xdata = xdata2;
+    xdata1 = xdata2 = NULL;
+out:
+    if (xdata1)
+        dict_unref(xdata1);
+    return;
 }
 
-
 int
-afr_post_blocking_rename_cbk (call_frame_t *frame, xlator_t *this)
+afr_changelog_prepare(xlator_t *this, call_frame_t *frame, int *call_count,
+                      afr_changelog_resume_t changelog_resume,
+                      afr_xattrop_type_t op, dict_t **xdata,
+                      dict_t **newloc_xdata)
 {
-        afr_internal_lock_t *int_lock = NULL;
-        afr_local_t         *local    = NULL;
+    afr_private_t *priv = NULL;
+    afr_local_t *local = NULL;
 
-        local    = frame->local;
-        int_lock = &local->internal_lock;
+    local = frame->local;
+    priv = this->private;
 
-        if (int_lock->lock_op_ret < 0) {
-                gf_log (this->name, GF_LOG_INFO,
-                        "Blocking entrylks failed.");
-                local->transaction.done (frame, this);
-        } else {
+    *call_count = afr_changelog_call_count(
+        local->transaction.type, local->transaction.pre_op,
+        local->transaction.failed_subvols, priv->child_count);
 
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "Blocking entrylks done. Proceeding to FOP");
-                afr_internal_lock_finish (frame, this);
-        }
-        return 0;
+    if (*call_count == 0) {
+        changelog_resume(frame, this);
+        return -1;
+    }
+
+    afr_changelog_populate_xdata(frame, op, xdata, newloc_xdata);
+    local->call_count = *call_count;
+
+    local->transaction.changelog_resume = changelog_resume;
+    return 0;
 }
 
 int
-afr_post_lower_unlock_cbk (call_frame_t *frame, xlator_t *this)
+afr_changelog_do(call_frame_t *frame, xlator_t *this, dict_t *xattr,
+                 afr_changelog_resume_t changelog_resume, afr_xattrop_type_t op)
 {
-        afr_internal_lock_t *int_lock = NULL;
-        afr_local_t         *local    = NULL;
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
+    dict_t *xdata = NULL;
+    dict_t *newloc_xdata = NULL;
+    int i = 0;
+    int call_count = 0;
+    int ret = 0;
+
+    local = frame->local;
+    priv = this->private;
+
+    for (i = 0; i < priv->child_count; i++) {
+        if (local->transaction.changelog_xdata[i]) {
+            dict_unref(local->transaction.changelog_xdata[i]);
+            local->transaction.changelog_xdata[i] = NULL;
+        }
+    }
 
-        local    = frame->local;
-        int_lock = &local->internal_lock;
+    ret = afr_changelog_prepare(this, frame, &call_count, changelog_resume, op,
+                                &xdata, &newloc_xdata);
 
-        GF_ASSERT (!int_lock->higher_locked);
+    if (ret)
+        return 0;
 
-        int_lock->lock_cbk = afr_post_blocking_rename_cbk;
-        afr_blocking_lock (frame, this);
+    for (i = 0; i < priv->child_count; i++) {
+        if (!local->transaction.pre_op[i] ||
+            local->transaction.failed_subvols[i])
+            continue;
 
-        return 0;
-}
+        switch (local->transaction.type) {
+            case AFR_DATA_TRANSACTION:
+            case AFR_METADATA_TRANSACTION:
+                if (!local->fd) {
+                    STACK_WIND_COOKIE(
+                        frame, afr_changelog_cbk, (void *)(long)i,
+                        priv->children[i], priv->children[i]->fops->xattrop,
+                        &local->loc, GF_XATTROP_ADD_ARRAY, xattr, xdata);
+                } else {
+                    STACK_WIND_COOKIE(
+                        frame, afr_changelog_cbk, (void *)(long)i,
+                        priv->children[i], priv->children[i]->fops->fxattrop,
+                        local->fd, GF_XATTROP_ADD_ARRAY, xattr, xdata);
+                }
+                break;
+            case AFR_ENTRY_RENAME_TRANSACTION:
 
+                STACK_WIND_COOKIE(frame, afr_changelog_cbk, (void *)(long)i,
+                                  priv->children[i],
+                                  priv->children[i]->fops->xattrop,
+                                  &local->transaction.new_parent_loc,
+                                  GF_XATTROP_ADD_ARRAY, xattr, newloc_xdata);
+                call_count--;
 
-int
-afr_set_transaction_flock (afr_local_t *local)
+                /* fall through */
+
+            case AFR_ENTRY_TRANSACTION:
+                if (local->fd)
+                    STACK_WIND_COOKIE(
+                        frame, afr_changelog_cbk, (void *)(long)i,
+                        priv->children[i], priv->children[i]->fops->fxattrop,
+                        local->fd, GF_XATTROP_ADD_ARRAY, xattr, xdata);
+                else
+                    STACK_WIND_COOKIE(frame, afr_changelog_cbk, (void *)(long)i,
+                                      priv->children[i],
+                                      priv->children[i]->fops->xattrop,
+                                      &local->transaction.parent_loc,
+                                      GF_XATTROP_ADD_ARRAY, xattr, xdata);
+                break;
+        }
+
+        if (!--call_count)
+            break;
+    }
+
+    if (xdata)
+        dict_unref(xdata);
+    if (newloc_xdata)
+        dict_unref(newloc_xdata);
+    return 0;
+}
+
+static void
+afr_init_optimistic_changelog_for_txn(xlator_t *this, afr_local_t *local)
 {
-        afr_internal_lock_t *int_lock = NULL;
-        afr_inodelk_t       *inodelk  = NULL;
+    int locked_count = 0;
+    afr_private_t *priv = NULL;
 
-        int_lock = &local->internal_lock;
-        inodelk = afr_get_inodelk (int_lock, int_lock->domain);
+    priv = this->private;
 
-        inodelk->flock.l_len   = local->transaction.len;
-        inodelk->flock.l_start = local->transaction.start;
-        inodelk->flock.l_type  = F_WRLCK;
+    locked_count = AFR_COUNT(local->transaction.pre_op, priv->child_count);
+    if (priv->optimistic_change_log && locked_count == priv->child_count)
+        local->optimistic_change_log = 1;
 
-        return 0;
+    return;
 }
 
 int
-afr_lock_rec (call_frame_t *frame, xlator_t *this)
+afr_changelog_pre_op(call_frame_t *frame, xlator_t *this)
 {
-        afr_internal_lock_t *int_lock = NULL;
-        afr_local_t         *local    = NULL;
+    afr_private_t *priv = this->private;
+    int i = 0;
+    int ret = 0;
+    int call_count = 0;
+    int op_errno = 0;
+    afr_local_t *local = NULL;
+    afr_internal_lock_t *int_lock = NULL;
+    unsigned char *locked_nodes = NULL;
+    int idx = -1;
+    gf_boolean_t pre_nop = _gf_true;
+    dict_t *xdata_req = NULL;
+
+    local = frame->local;
+    int_lock = &local->internal_lock;
+    idx = afr_index_for_transaction_type(local->transaction.type);
+
+    locked_nodes = afr_locked_nodes_get(local->transaction.type, int_lock);
+
+    for (i = 0; i < priv->child_count; i++) {
+        if (locked_nodes[i]) {
+            local->transaction.pre_op[i] = 1;
+            call_count++;
+        } else {
+            local->transaction.failed_subvols[i] = 1;
+        }
+    }
+
+    afr_init_optimistic_changelog_for_txn(this, local);
+
+    if (afr_changelog_pre_op_inherit(frame, this))
+        goto next;
+
+    /* This condition should not be met with present code, as
+     * transaction.done will be called if locks are not acquired on even a
+     * single node.
+     */
+    if (call_count == 0) {
+        op_errno = ENOTCONN;
+        goto err;
+    }
+
+    /* Check if the fop can be performed on at least
+     * quorum number of nodes.
+     */
+    if (priv->quorum_count && !afr_has_fop_quorum(frame)) {
+        op_errno = int_lock->lock_op_errno;
+        if (op_errno == 0)
+            op_errno = afr_quorum_errno(priv);
+        goto err;
+    }
+
+    xdata_req = dict_new();
+    if (!xdata_req) {
+        op_errno = ENOMEM;
+        goto err;
+    }
+
+    if (call_count < priv->child_count)
+        pre_nop = _gf_false;
+
+    /* Set an all-zero pending changelog so that in the cbk, we can get the
+     * current on-disk values. In a replica 3 volume with arbiter enabled,
+     * these values are needed to arrive at a go/ no-go of the fop phase to
+     * avoid ending up in split-brain.*/
+
+    ret = afr_set_pending_dict(priv, xdata_req, local->pending);
+    if (ret < 0) {
+        op_errno = ENOMEM;
+        goto err;
+    }
+
+    if (afr_needs_changelog_update(local)) {
+        local->dirty[idx] = hton32(1);
+
+        ret = dict_set_static_bin(xdata_req, AFR_DIRTY, local->dirty,
+                                  sizeof(int) * AFR_NUM_CHANGE_LOGS);
+        if (ret) {
+            op_errno = ENOMEM;
+            goto err;
+        }
 
-        local    = frame->local;
-        int_lock = &local->internal_lock;
+        pre_nop = _gf_false;
+        local->transaction.dirtied = 1;
+    }
 
-        int_lock->transaction_lk_type = AFR_TRANSACTION_LK;
-        int_lock->domain = this->name;
+    if (pre_nop)
+        goto next;
 
-        switch (local->transaction.type) {
-        case AFR_DATA_TRANSACTION:
-        case AFR_METADATA_TRANSACTION:
-                afr_set_transaction_flock (local);
+    if (!local->pre_op_compat) {
+        dict_copy(xdata_req, local->xdata_req);
+        goto next;
+    }
 
-                int_lock->lock_cbk = afr_post_nonblocking_inodelk_cbk;
+    afr_changelog_do(frame, this, xdata_req, afr_transaction_perform_fop,
+                     AFR_TRANSACTION_PRE_OP);
 
-                afr_nonblocking_inodelk (frame, this);
-                break;
+    if (xdata_req)
+        dict_unref(xdata_req);
 
-        case AFR_ENTRY_RENAME_TRANSACTION:
+    return 0;
+next:
+    afr_transaction_perform_fop(frame, this);
 
-                int_lock->lock_cbk = afr_post_nonblocking_entrylk_cbk;
-                afr_nonblocking_entrylk (frame, this);
-                break;
+    if (xdata_req)
+        dict_unref(xdata_req);
 
-        case AFR_ENTRY_TRANSACTION:
-                int_lock->lk_basename = local->transaction.basename;
-                if (&local->transaction.parent_loc)
-                        int_lock->lk_loc = &local->transaction.parent_loc;
-                else
-                        GF_ASSERT (local->fd);
+    return 0;
+err:
+    local->internal_lock.lock_cbk = afr_transaction_done;
+    local->op_ret = -1;
+    local->op_errno = op_errno;
 
-                int_lock->lock_cbk = afr_post_nonblocking_entrylk_cbk;
-                afr_nonblocking_entrylk (frame, this);
-                break;
-        }
+    afr_handle_lock_acquire_failure(local);
 
-        return 0;
+    if (xdata_req)
+        dict_unref(xdata_req);
+
+    return 0;
 }
 
+int
+afr_post_nonblocking_lock_cbk(call_frame_t *frame, xlator_t *this)
+{
+    afr_internal_lock_t *int_lock = NULL;
+    afr_local_t *local = NULL;
+
+    local = frame->local;
+    int_lock = &local->internal_lock;
+
+    /* Initiate blocking locks if non-blocking has failed */
+    if (int_lock->lock_op_ret < 0) {
+        gf_msg_debug(this->name, 0,
+                     "Non blocking locks failed. Proceeding to blocking");
+        int_lock->lock_cbk = afr_internal_lock_finish;
+        afr_blocking_lock(frame, this);
+    } else {
+        gf_msg_debug(this->name, 0,
+                     "Non blocking locks done. Proceeding to FOP");
+
+        afr_internal_lock_finish(frame, this);
+    }
+
+    return 0;
+}
 
 int
-afr_lock (call_frame_t *frame, xlator_t *this)
+afr_post_blocking_rename_cbk(call_frame_t *frame, xlator_t *this)
 {
-        afr_set_lock_number (frame, this);
+    afr_internal_lock_t *int_lock = NULL;
+    afr_local_t *local = NULL;
 
-        return afr_lock_rec (frame, this);
-}
+    local = frame->local;
+    int_lock = &local->internal_lock;
 
+    if (int_lock->lock_op_ret < 0) {
+        gf_msg(this->name, GF_LOG_INFO, 0, AFR_MSG_INTERNAL_LKS_FAILED,
+               "Blocking entrylks failed.");
 
-/* }}} */
+        afr_transaction_done(frame, this);
+    } else {
+        gf_msg_debug(this->name, 0,
+                     "Blocking entrylks done. Proceeding to FOP");
+
+        afr_internal_lock_finish(frame, this);
+    }
+    return 0;
+}
 
 int
-afr_internal_lock_finish (call_frame_t *frame, xlator_t *this)
+afr_post_lower_unlock_cbk(call_frame_t *frame, xlator_t *this)
 {
-        if (__fop_changelog_needed (frame, this)) {
-                afr_changelog_pre_op (frame, this);
-        } else {
-                afr_transaction_perform_fop (frame, this);
-        }
+    afr_internal_lock_t *int_lock = NULL;
+    afr_local_t *local = NULL;
 
-        return 0;
-}
+    local = frame->local;
+    int_lock = &local->internal_lock;
 
+    GF_ASSERT(!int_lock->higher_locked);
 
-void
-afr_set_delayed_post_op (call_frame_t *frame, xlator_t *this)
+    int_lock->lock_cbk = afr_post_blocking_rename_cbk;
+    afr_blocking_lock(frame, this);
+
+    return 0;
+}
+
+int
+afr_set_transaction_flock(xlator_t *this, afr_local_t *local,
+                          afr_lockee_t *lockee)
 {
-        afr_local_t    *local = NULL;
-        afr_private_t  *priv = NULL;
+    afr_private_t *priv = NULL;
+    struct gf_flock *flock = NULL;
+
+    priv = this->private;
+    flock = &lockee->flock;
+
+    if ((priv->arbiter_count || local->transaction.eager_lock_on ||
+         priv->full_lock) &&
+        local->transaction.type == AFR_DATA_TRANSACTION) {
+        /*Lock entire file to avoid network split brains.*/
+        flock->l_len = 0;
+        flock->l_start = 0;
+    } else {
+        flock->l_len = local->transaction.len;
+        flock->l_start = local->transaction.start;
+    }
+    flock->l_type = F_WRLCK;
+
+    return 0;
+}
 
-        /* call this function from any of the related optimizations
-           which benefit from delaying post op are enabled, namely:
+int
+afr_lock(call_frame_t *frame, xlator_t *this)
+{
+    afr_internal_lock_t *int_lock = NULL;
+    afr_local_t *local = NULL;
+    int i = 0;
 
-           - changelog piggybacking
-           - eager locking
-        */
+    local = frame->local;
+    int_lock = &local->internal_lock;
 
-        priv = this->private;
-        if (!priv)
-                return;
+    int_lock->lock_cbk = afr_post_nonblocking_lock_cbk;
+    int_lock->domain = this->name;
 
-        if (!priv->post_op_delay_secs)
-                return;
+    switch (local->transaction.type) {
+        case AFR_DATA_TRANSACTION:
+        case AFR_METADATA_TRANSACTION:
+            for (i = 0; i < int_lock->lockee_count; i++) {
+                afr_set_transaction_flock(this, local, &int_lock->lockee[i]);
+            }
 
-        local = frame->local;
-        if (!local)
-                return;
+            break;
 
-        if (!local->transaction.eager_lock_on)
-                return;
+        case AFR_ENTRY_TRANSACTION:
+            int_lock->lk_basename = local->transaction.basename;
+            if (local->transaction.parent_loc.path)
+                int_lock->lk_loc = &local->transaction.parent_loc;
+            else
+                GF_ASSERT(local->fd);
+            break;
+        case AFR_ENTRY_RENAME_TRANSACTION:
+            break;
+    }
+    afr_lock_nonblocking(frame, this);
 
-        if (!local->fd)
-                return;
+    return 0;
+}
 
-        if (local->op == GF_FOP_WRITE)
-                local->delayed_post_op = _gf_true;
+static gf_boolean_t
+afr_locals_overlap(afr_local_t *local1, afr_local_t *local2)
+{
+    uint64_t start1 = local1->transaction.start;
+    uint64_t start2 = local2->transaction.start;
+    uint64_t end1 = 0;
+    uint64_t end2 = 0;
+
+    if (local1->transaction.len)
+        end1 = start1 + local1->transaction.len - 1;
+    else
+        end1 = ULLONG_MAX;
+
+    if (local2->transaction.len)
+        end2 = start2 + local2->transaction.len - 1;
+    else
+        end2 = ULLONG_MAX;
+
+    return ((end1 >= start2) && (end2 >= start1));
 }
 
 gf_boolean_t
-afr_are_multiple_fds_opened (fd_t *fd, xlator_t *this)
+afr_has_lock_conflict(afr_local_t *local, gf_boolean_t waitlist_check)
 {
-        afr_fd_ctx_t *fd_ctx = NULL;
-
-        if (!fd) {
-                /* If false is returned, it may keep on taking eager-lock
-                 * which may lead to starvation, so return true to avoid that.
-                 */
-                gf_log_callingfn (this->name, GF_LOG_ERROR, "Invalid fd");
-                return _gf_true;
+    afr_local_t *each = NULL;
+    afr_lock_t *lock = NULL;
+
+    lock = &local->inode_ctx->lock[local->transaction.type];
+    /*
+     * Once full file lock is acquired in eager-lock phase, overlapping
+     * writes do not compete for inode-locks, instead are transferred to the
+     * next writes. Because of this overlapping writes are not ordered.
+     * This can cause inconsistencies in replication.
+     * Example:
+     * Two overlapping writes w1, w2 are sent in parallel on same fd
+     * in two threads t1, t2.
+     * Both threads can execute afr_writev_wind in the following manner.
+     * t1 winds w1 on brick-0
+     * t2 winds w2 on brick-0
+     * t2 winds w2 on brick-1
+     * t1 winds w1 on brick-1
+     *
+     * This check makes sure the locks are not transferred for
+     * overlapping writes.
+     */
+    list_for_each_entry(each, &lock->owners, transaction.owner_list)
+    {
+        if (afr_locals_overlap(each, local)) {
+            return _gf_true;
         }
-        /* Lets say mount1 has eager-lock(full-lock) and after the eager-lock
-         * is taken mount2 opened the same file, it won't be able to
-         * perform any data operations until mount1 releases eager-lock.
-         * To avoid such scenario do not enable eager-lock for this transaction
-         * if open-fd-count is > 1
-         */
-
-        fd_ctx = afr_fd_ctx_get (fd, this);
-        if (!fd_ctx)
-                return _gf_true;
-
-        if (fd_ctx->open_fd_count > 1)
-                return _gf_true;
+    }
 
+    if (!waitlist_check)
         return _gf_false;
+    list_for_each_entry(each, &lock->waiting, transaction.wait_list)
+    {
+        if (afr_locals_overlap(each, local)) {
+            return _gf_true;
+        }
+    }
+    return _gf_false;
 }
 
+/* }}} */
+static void
+afr_copy_inodelk_vars(afr_internal_lock_t *dst, afr_internal_lock_t *src,
+                      xlator_t *this, int lockee_num)
+{
+    afr_private_t *priv = this->private;
+    afr_lockee_t *sl = &src->lockee[lockee_num];
+    afr_lockee_t *dl = &dst->lockee[lockee_num];
+
+    dst->domain = src->domain;
+    dl->flock.l_len = sl->flock.l_len;
+    dl->flock.l_start = sl->flock.l_start;
+    dl->flock.l_type = sl->flock.l_type;
+    dl->locked_count = sl->locked_count;
+    memcpy(dl->locked_nodes, sl->locked_nodes,
+           priv->child_count * sizeof(*dl->locked_nodes));
+}
 
-gf_boolean_t
-is_afr_delayed_changelog_post_op_needed (call_frame_t *frame, xlator_t *this)
+void
+__afr_transaction_wake_shared(afr_local_t *local, struct list_head *shared)
 {
-        afr_local_t      *local = NULL;
-        gf_boolean_t      res = _gf_false;
+    gf_boolean_t conflict = _gf_false;
+    afr_local_t *each = NULL;
+    afr_lock_t *lock = &local->inode_ctx->lock[local->transaction.type];
+
+    while (!conflict) {
+        if (list_empty(&lock->waiting))
+            return;
+        each = list_entry(lock->waiting.next, afr_local_t,
+                          transaction.wait_list);
+        if (afr_has_lock_conflict(each, _gf_false)) {
+            conflict = _gf_true;
+        }
+        if (conflict && !list_empty(&lock->owners))
+            return;
+        afr_copy_inodelk_vars(&each->internal_lock, &local->internal_lock,
+                              each->transaction.frame->this, 0);
+        list_move_tail(&each->transaction.wait_list, shared);
+        list_add_tail(&each->transaction.owner_list, &lock->owners);
+    }
+}
 
-        local = frame->local;
-        if (!local)
-                goto out;
+static void
+afr_lock_resume_shared(struct list_head *list)
+{
+    afr_local_t *each = NULL;
+
+    while (!list_empty(list)) {
+        each = list_entry(list->next, afr_local_t, transaction.wait_list);
+        list_del_init(&each->transaction.wait_list);
+        afr_changelog_pre_op(each->transaction.frame,
+                             each->transaction.frame->this);
+    }
+}
 
-        if (!local->delayed_post_op)
-                goto out;
+int
+afr_internal_lock_finish(call_frame_t *frame, xlator_t *this)
+{
+    afr_local_t *local = frame->local;
+    afr_lock_t *lock = NULL;
+
+    local->internal_lock.lock_cbk = NULL;
+    if (!local->transaction.eager_lock_on) {
+        if (local->internal_lock.lock_op_ret < 0) {
+            afr_transaction_done(frame, this);
+            return 0;
+        }
+        afr_changelog_pre_op(frame, this);
+    } else {
+        lock = &local->inode_ctx->lock[local->transaction.type];
+        if (local->internal_lock.lock_op_ret < 0) {
+            afr_handle_lock_acquire_failure(local);
+        } else {
+            lock->event_generation = local->event_generation;
+            afr_changelog_pre_op(frame, this);
+        }
+    }
 
-        //Mark pending changelog ASAP
-        if (!afr_txn_nothing_failed (frame, this))
-                goto out;
+    return 0;
+}
 
-        if (local->fd && afr_are_multiple_fds_opened (local->fd, this))
-                goto out;
+gf_boolean_t
+afr_are_conflicting_ops_waiting(afr_local_t *local, xlator_t *this)
+{
+    afr_lock_t *lock = NULL;
+    lock = &local->inode_ctx->lock[local->transaction.type];
+
+    /* Lets say mount1 has eager-lock(full-lock) and after the eager-lock
+     * is taken mount2 opened the same file, it won't be able to
+     * perform any {meta,}data operations until mount1 releases eager-lock.
+     * To avoid such scenario do not enable eager-lock for this transaction
+     * if open-fd-count is > 1 for metadata transactions and if num-inodelks > 1
+     * for data transactions
+     */
+
+    if (local->transaction.type == AFR_METADATA_TRANSACTION) {
+        if (local->inode_ctx->open_fd_count > 1) {
+            return _gf_true;
+        }
+    } else if (local->transaction.type == AFR_DATA_TRANSACTION) {
+        if (lock->num_inodelks > 1) {
+            return _gf_true;
+        }
+    }
 
-        res = _gf_true;
-out:
-        return res;
+    return _gf_false;
 }
 
+gf_boolean_t
+afr_is_delayed_changelog_post_op_needed(call_frame_t *frame, xlator_t *this,
+                                        int delay)
+{
+    afr_local_t *local = NULL;
+    afr_lock_t *lock = NULL;
+    gf_boolean_t res = _gf_false;
+
+    local = frame->local;
+    lock = &local->inode_ctx->lock[local->transaction.type];
+
+    if (!afr_txn_nothing_failed(frame, this)) {
+        lock->release = _gf_true;
+        goto out;
+    }
+
+    if (afr_are_conflicting_ops_waiting(local, this)) {
+        lock->release = _gf_true;
+        goto out;
+    }
+
+    if (!list_empty(&lock->owners))
+        goto out;
+    else
+        GF_ASSERT(list_empty(&lock->waiting));
+
+    if (lock->release) {
+        goto out;
+    }
+
+    if (!delay) {
+        goto out;
+    }
+
+    if (local->transaction.disable_delayed_post_op) {
+        goto out;
+    }
+
+    if ((local->op != GF_FOP_WRITE) && (local->op != GF_FOP_FXATTROP) &&
+        (local->op != GF_FOP_FSYNC)) {
+        /*Only allow writes/fsyncs but shard does [f]xattrops on writes, so
+         * they are fine too*/
+        goto out;
+    }
+
+    res = _gf_true;
+out:
+    return res;
+}
 
 void
-afr_delayed_changelog_post_op (xlator_t *this, call_frame_t *frame, fd_t *fd,
-                               call_stub_t *stub);
-
-void
-afr_delayed_changelog_wake_up_cbk (void *data)
+afr_delayed_changelog_wake_up_cbk(void *data)
 {
-        fd_t           *fd = NULL;
-
-        fd = data;
-
-        afr_delayed_changelog_wake_up (THIS, fd);
+    afr_lock_t *lock = NULL;
+    afr_local_t *local = data;
+    afr_local_t *timer_local = NULL;
+    struct list_head shared;
+
+    INIT_LIST_HEAD(&shared);
+    lock = &local->inode_ctx->lock[local->transaction.type];
+    LOCK(&local->inode->lock);
+    {
+        timer_local = list_entry(lock->post_op.next, afr_local_t,
+                                 transaction.owner_list);
+        if (list_empty(&lock->owners) && (local == timer_local)) {
+            GF_ASSERT(list_empty(&lock->waiting));
+            /*Last owner*/
+            lock->release = _gf_true;
+            lock->delay_timer = NULL;
+        }
+    }
+    UNLOCK(&local->inode->lock);
+    afr_changelog_post_op_now(local->transaction.frame,
+                              local->transaction.frame->this);
 }
 
-
 /* SET operation */
 int
-afr_fd_report_unstable_write (xlator_t *this, fd_t *fd)
+afr_fd_report_unstable_write(xlator_t *this, afr_local_t *local)
 {
-        afr_fd_ctx_t *fdctx = NULL;
-
-        fdctx = afr_fd_ctx_get (fd, this);
-
-        LOCK(&fd->lock);
-        {
-                fdctx->witnessed_unstable_write = _gf_true;
-        }
-        UNLOCK(&fd->lock);
+    LOCK(&local->inode->lock);
+    {
+        local->inode_ctx->witnessed_unstable_write = _gf_true;
+    }
+    UNLOCK(&local->inode->lock);
 
-        return 0;
+    return 0;
 }
 
 /* TEST and CLEAR operation */
 gf_boolean_t
-afr_fd_has_witnessed_unstable_write (xlator_t *this, fd_t *fd)
+afr_fd_has_witnessed_unstable_write(xlator_t *this, inode_t *inode)
 {
-        afr_fd_ctx_t *fdctx = NULL;
-        gf_boolean_t witness = _gf_false;
+    afr_inode_ctx_t *ctx = NULL;
+    gf_boolean_t witness = _gf_false;
 
-        fdctx = afr_fd_ctx_get (fd, this);
-        if (!fdctx)
-                return _gf_true;
+    LOCK(&inode->lock);
+    {
+        (void)__afr_inode_ctx_get(this, inode, &ctx);
 
-        LOCK(&fd->lock);
-        {
-                if (fdctx->witnessed_unstable_write) {
-                        witness = _gf_true;
-                        fdctx->witnessed_unstable_write = _gf_false;
-                }
+        if (ctx->witnessed_unstable_write) {
+            witness = _gf_true;
+            ctx->witnessed_unstable_write = _gf_false;
         }
-        UNLOCK (&fd->lock);
+    }
+    UNLOCK(&inode->lock);
 
-        return witness;
+    return witness;
 }
 
-
 int
-afr_changelog_fsync_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                         int op_ret, int op_errno, struct iatt *pre,
-                         struct iatt *post, dict_t *xdata)
+afr_changelog_fsync_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                        int op_ret, int op_errno, struct iatt *pre,
+                        struct iatt *post, dict_t *xdata)
 {
-        afr_private_t *priv = NULL;
-        int child_index = (long) cookie;
-        int call_count = -1;
-        afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
+    int child_index = (long)cookie;
+    int call_count = -1;
+    afr_local_t *local = NULL;
 
-        priv = this->private;
-        local = frame->local;
+    priv = this->private;
+    local = frame->local;
 
-        if (op_ret != 0) {
-                /* Failure of fsync() is as good as failure of previous
-                   write(). So treat it like one.
-		*/
-                gf_log (this->name, GF_LOG_WARNING,
-                        "fsync(%s) failed on subvolume %s. Transaction was %s",
-                        uuid_utoa (local->fd->inode->gfid),
-                        priv->children[child_index]->name,
-                        gf_fop_list[local->op]);
+    if (op_ret != 0) {
+        /* Failure of fsync() is as good as failure of previous
+           write(). So treat it like one.
+        */
+        gf_msg(this->name, GF_LOG_WARNING, op_errno, AFR_MSG_FSYNC_FAILED,
+               "fsync(%s) failed on subvolume %s. Transaction was %s",
+               uuid_utoa(local->fd->inode->gfid),
+               priv->children[child_index]->name, gf_fop_list[local->op]);
 
-                afr_transaction_fop_failed (frame, this, child_index);
-	}
+        afr_transaction_fop_failed(frame, this, child_index);
+    }
 
-	call_count = afr_frame_return (frame);
+    call_count = afr_frame_return(frame);
 
-	if (call_count == 0)
-		afr_changelog_post_op_now (frame, this);
+    if (call_count == 0)
+        afr_changelog_post_op_now(frame, this);
 
-	return 0;
+    return 0;
 }
 
-
 int
-afr_changelog_fsync (call_frame_t *frame, xlator_t *this)
+afr_changelog_fsync(call_frame_t *frame, xlator_t *this)
 {
-        afr_local_t *local = NULL;
-        int i = 0;
-        int call_count = 0;
-        afr_private_t *priv = NULL;
-        dict_t *xdata = NULL;
-        GF_UNUSED int ret = -1;
+    afr_local_t *local = NULL;
+    int i = 0;
+    int call_count = 0;
+    afr_private_t *priv = NULL;
+    dict_t *xdata = NULL;
+    GF_UNUSED int ret = -1;
 
-        local = frame->local;
-        priv = this->private;
+    local = frame->local;
+    priv = this->private;
 
-        call_count = AFR_COUNT (local->transaction.pre_op, priv->child_count);
+    call_count = AFR_COUNT(local->transaction.pre_op, priv->child_count);
 
-        if (!call_count) {
-                /* will go straight to unlock */
-                afr_changelog_post_op_now (frame, this);
-                return 0;
-        }
+    if (!call_count) {
+        /* will go straight to unlock */
+        afr_changelog_post_op_now(frame, this);
+        return 0;
+    }
 
-        local->call_count = call_count;
+    local->call_count = call_count;
 
-        xdata = dict_new();
-        if (xdata)
-                ret = dict_set_int32 (xdata, "batch-fsync", 1);
+    xdata = dict_new();
+    if (xdata) {
+        ret = dict_set_int32_sizen(xdata, "batch-fsync", 1);
+        ret = dict_set_str(xdata, GLUSTERFS_INTERNAL_FOP_KEY, "yes");
+    }
 
-        for (i = 0; i < priv->child_count; i++) {
-                if (!local->transaction.pre_op[i])
-                        continue;
+    for (i = 0; i < priv->child_count; i++) {
+        if (!local->transaction.pre_op[i])
+            continue;
 
-                STACK_WIND_COOKIE (frame, afr_changelog_fsync_cbk,
-				   (void *) (long) i, priv->children[i],
-				   priv->children[i]->fops->fsync, local->fd,
-				   1, xdata);
-                if (!--call_count)
-                        break;
-        }
+        STACK_WIND_COOKIE(frame, afr_changelog_fsync_cbk, (void *)(long)i,
+                          priv->children[i], priv->children[i]->fops->fsync,
+                          local->fd, 1, xdata);
+        if (!--call_count)
+            break;
+    }
 
-        if (xdata)
-                dict_unref (xdata);
+    if (xdata)
+        dict_unref(xdata);
 
-        return 0;
+    return 0;
 }
 
-
 int
-afr_changelog_post_op_safe (call_frame_t *frame, xlator_t *this)
+afr_changelog_post_op_safe(call_frame_t *frame, xlator_t *this)
 {
-	afr_local_t    *local = NULL;
-        afr_private_t  *priv = NULL;
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
 
-	local = frame->local;
-        priv = this->private;
+    local = frame->local;
+    priv = this->private;
 
-        if (!local->fd || local->transaction.type != AFR_DATA_TRANSACTION) {
-                afr_changelog_post_op_now (frame, this);
-                return 0;
-        }
-
-        if (afr_changelog_pre_op_uninherit (frame, this) &&
-	    afr_txn_nothing_failed (frame, this)) {
-                /* just detected that this post-op is about to
-                   be optimized away as a new write() has
-                   already piggybacked on this frame's changelog.
-                   */
-                afr_changelog_post_op_now (frame, this);
-                return 0;
-        }
+    if (!local->fd || local->transaction.type != AFR_DATA_TRANSACTION) {
+        afr_changelog_post_op_now(frame, this);
+        return 0;
+    }
 
-        /* Calling afr_changelog_post_op_now() now will result in
-           issuing ->[f]xattrop().
-
-           Performing a hard POST-OP (->[f]xattrop() FOP) is a more
-           responsible operation that what it might appear on the surface.
-
-           The changelog of a file (in the xattr of the file on the server)
-           stores information (pending count) about the state of the file
-           on the OTHER server. This changelog is blindly trusted, and must
-           therefore be updated in such a way it remains trustworthy. This
-           implies that decrementing the pending count (essentially "clearing
-           the dirty flag") must be done STRICTLY after we are sure that the
-           operation on the other server has reached stable storage.
-
-           While the backend filesystem on that server will eventually flush
-           it to stable storage, we (being in userspace) have no mechanism
-           to get notified when the write became "stable".
-
-           This means we need take matter into our own hands and issue an
-           fsync() EVEN IF THE APPLICATION WAS PERFORMING UNSTABLE WRITES,
-           and get an acknowledgement for it. And we need to wait for the
-           fsync() acknowledgement before initiating the hard POST-OP.
-
-           However if the FD itself was opened in O_SYNC or O_DSYNC then
-           we are already guaranteed that the writes were made stable as
-           part of the FOP itself. The same holds true for NFS stable
-           writes which happen on an anonymous FD with O_DSYNC or O_SYNC
-           flag set in the writev() @flags param. For all other write types,
-           mark a flag in the fdctx whenever an unstable write is witnessed.
+    if (afr_changelog_pre_op_uninherit(frame, this) &&
+        afr_txn_nothing_failed(frame, this)) {
+        /* just detected that this post-op is about to
+           be optimized away as a new write() has
+           already piggybacked on this frame's changelog.
            */
-
-        if (!afr_fd_has_witnessed_unstable_write (this, local->fd)) {
-                afr_changelog_post_op_now (frame, this);
-                return 0;
-        }
-
-        /* Check whether users want durability and perform fsync/post-op
-         * accordingly.
-         */
-        if (priv->ensure_durability) {
-                /* Time to fsync() */
-                afr_changelog_fsync (frame, this);
-        } else {
-                afr_changelog_post_op_now (frame, this);
-        }
-
+        afr_changelog_post_op_now(frame, this);
+        return 0;
+    }
+
+    /* Calling afr_changelog_post_op_now() now will result in
+       issuing ->[f]xattrop().
+
+       Performing a hard POST-OP (->[f]xattrop() FOP) is a more
+       responsible operation that what it might appear on the surface.
+
+       The changelog of a file (in the xattr of the file on the server)
+       stores information (pending count) about the state of the file
+       on the OTHER server. This changelog is blindly trusted, and must
+       therefore be updated in such a way it remains trustworthy. This
+       implies that decrementing the pending count (essentially "clearing
+       the dirty flag") must be done STRICTLY after we are sure that the
+       operation on the other server has reached stable storage.
+
+       While the backend filesystem on that server will eventually flush
+       it to stable storage, we (being in userspace) have no mechanism
+       to get notified when the write became "stable".
+
+       This means we need take matter into our own hands and issue an
+       fsync() EVEN IF THE APPLICATION WAS PERFORMING UNSTABLE WRITES,
+       and get an acknowledgement for it. And we need to wait for the
+       fsync() acknowledgement before initiating the hard POST-OP.
+
+       However if the FD itself was opened in O_SYNC or O_DSYNC then
+       we are already guaranteed that the writes were made stable as
+       part of the FOP itself. The same holds true for NFS stable
+       writes which happen on an anonymous FD with O_DSYNC or O_SYNC
+       flag set in the writev() @flags param. For all other write types,
+       mark a flag in the fdctx whenever an unstable write is witnessed.
+       */
+
+    if (!afr_fd_has_witnessed_unstable_write(this, local->inode)) {
+        afr_changelog_post_op_now(frame, this);
         return 0;
+    }
+
+    /* Check whether users want durability and perform fsync/post-op
+     * accordingly.
+     */
+    if (priv->ensure_durability) {
+        /* Time to fsync() */
+        afr_changelog_fsync(frame, this);
+    } else {
+        afr_changelog_post_op_now(frame, this);
+    }
+
+    return 0;
 }
 
-
 void
-afr_delayed_changelog_post_op (xlator_t *this, call_frame_t *frame, fd_t *fd,
-                               call_stub_t *stub)
+afr_changelog_post_op(call_frame_t *frame, xlator_t *this)
 {
-	afr_fd_ctx_t      *fd_ctx = NULL;
-	call_frame_t      *prev_frame = NULL;
-	struct timespec    delta = {0, };
-	afr_private_t     *priv = NULL;
-	afr_local_t       *local = NULL;
-
-	priv = this->private;
-
-	fd_ctx = afr_fd_ctx_get (fd, this);
-	if (!fd_ctx)
-                goto out;
+    struct timespec delta = {
+        0,
+    };
+    afr_private_t *priv = NULL;
+    afr_local_t *local = frame->local;
+    afr_lock_t *lock = NULL;
+    gf_boolean_t post_op = _gf_true;
+    struct list_head shared;
+
+    priv = this->private;
+    delta.tv_sec = priv->post_op_delay_secs;
+    delta.tv_nsec = 0;
+
+    INIT_LIST_HEAD(&shared);
+    if (!local->transaction.eager_lock_on)
+        goto out;
+
+    lock = &local->inode_ctx->lock[local->transaction.type];
+    LOCK(&local->inode->lock);
+    {
+        list_del_init(&local->transaction.owner_list);
+        list_add(&local->transaction.owner_list, &lock->post_op);
+        __afr_transaction_wake_shared(local, &shared);
+
+        if (!afr_is_delayed_changelog_post_op_needed(frame, this,
+                                                     delta.tv_sec)) {
+            if (list_empty(&lock->owners))
+                lock->release = _gf_true;
+            goto unlock;
+        }
 
-	delta.tv_sec = priv->post_op_delay_secs;
-	delta.tv_nsec = 0;
-
-	pthread_mutex_lock (&fd_ctx->delay_lock);
-	{
-		prev_frame = fd_ctx->delay_frame;
-		fd_ctx->delay_frame = NULL;
-		if (fd_ctx->delay_timer)
-			gf_timer_call_cancel (this->ctx, fd_ctx->delay_timer);
-		fd_ctx->delay_timer = NULL;
-		if (!frame)
-			goto unlock;
-		fd_ctx->delay_timer = gf_timer_call_after (this->ctx, delta,
-							   afr_delayed_changelog_wake_up_cbk,
-							   fd);
-		fd_ctx->delay_frame = frame;
-	}
+        GF_ASSERT(lock->delay_timer == NULL);
+        lock->delay_timer = gf_timer_call_after(
+            this->ctx, delta, afr_delayed_changelog_wake_up_cbk, local);
+        if (!lock->delay_timer) {
+            lock->release = _gf_true;
+        } else {
+            post_op = _gf_false;
+        }
+    }
 unlock:
-	pthread_mutex_unlock (&fd_ctx->delay_lock);
-
-out:
-	if (prev_frame) {
-		local = prev_frame->local;
-		local->transaction.resume_stub = stub;
-		afr_changelog_post_op_now (prev_frame, this);
-	} else if (stub) {
-		call_resume (stub);
-	}
-}
-
-
-void
-afr_changelog_post_op (call_frame_t *frame, xlator_t *this)
-{
-        afr_local_t  *local = NULL;
-
-        local = frame->local;
-
-        if (is_afr_delayed_changelog_post_op_needed (frame, this))
-                afr_delayed_changelog_post_op (this, frame, local->fd, NULL);
-        else
-                afr_changelog_post_op_safe (frame, this);
-}
+    UNLOCK(&local->inode->lock);
 
+    if (!list_empty(&shared)) {
+        afr_lock_resume_shared(&shared);
+    }
 
-
-/* Wake up the sleeping/delayed post-op, and also register
-   a stub to have it resumed after this transaction
-   completely finishes.
-
-   The @stub gets saved in @local and gets resumed in
-   afr_local_cleanup()
-   */
-void
-afr_delayed_changelog_wake_resume (xlator_t *this, fd_t *fd, call_stub_t *stub)
-{
-        afr_delayed_changelog_post_op (this, NULL, fd, stub);
-}
-
-
-void
-afr_delayed_changelog_wake_up (xlator_t *this, fd_t *fd)
-{
-        afr_delayed_changelog_post_op (this, NULL, fd, NULL);
+out:
+    if (post_op) {
+        if (!local->transaction.eager_lock_on || lock->release) {
+            afr_changelog_post_op_safe(frame, this);
+        } else {
+            afr_changelog_post_op_now(frame, this);
+        }
+    }
 }
 
-
 int
-afr_transaction_resume (call_frame_t *frame, xlator_t *this)
+afr_transaction_resume(call_frame_t *frame, xlator_t *this)
 {
-        afr_local_t         *local    = NULL;
+    afr_local_t *local = NULL;
 
-        local    = frame->local;
+    local = frame->local;
 
-        if (local->transaction.eager_lock_on) {
-                /* We don't need to retain "local" in the
-                   fd list anymore, writes to all subvols
-                   are finished by now */
-                afr_remove_eager_lock_stub (local);
-        }
-
-        afr_restore_lk_owner (frame);
+    afr_restore_lk_owner(frame);
 
-        afr_handle_symmetric_errors (frame, this);
+    afr_handle_symmetric_errors(frame, this);
 
-	if (!local->pre_op_compat)
-		/* new mode, pre-op was done along
-		   with OP */
-		afr_changelog_pre_op_update (frame, this);
+    if (!local->pre_op_compat)
+        /* new mode, pre-op was done along
+           with OP */
+        afr_changelog_pre_op_update(frame, this);
 
-        if (__fop_changelog_needed (frame, this)) {
-                afr_changelog_post_op (frame, this);
-        } else {
-		afr_changelog_post_op_done (frame, this);
-        }
+    afr_changelog_post_op(frame, this);
 
-        return 0;
+    return 0;
 }
 
-
 /**
  * afr_transaction_fop_failed - inform that an fop failed
  */
 
 void
-afr_transaction_fop_failed (call_frame_t *frame, xlator_t *this,
-                            int child_index)
+afr_transaction_fop_failed(call_frame_t *frame, xlator_t *this, int child_index)
 {
-        afr_local_t *   local = NULL;
+    afr_local_t *local = NULL;
 
-        local = frame->local;
+    local = frame->local;
 
-	local->transaction.failed_subvols[child_index] = 1;
+    local->transaction.failed_subvols[child_index] = 1;
 }
 
-
-
-        static gf_boolean_t
-afr_locals_overlap (afr_local_t *local1, afr_local_t *local2)
+static gf_boolean_t
+__need_previous_lock_unlocked(afr_local_t *local)
 {
-        uint64_t start1 = local1->transaction.start;
-        uint64_t start2 = local2->transaction.start;
-        uint64_t end1 = 0;
-        uint64_t end2 = 0;
+    afr_lock_t *lock = NULL;
 
-        if (local1->transaction.len)
-                end1 = start1 + local1->transaction.len - 1;
-        else
-                end1 = ULLONG_MAX;
-
-        if (local2->transaction.len)
-                end2 = start2 + local2->transaction.len - 1;
-        else
-                end2 = ULLONG_MAX;
-
-        return ((end1 >= start2) && (end2 >= start1));
+    lock = &local->inode_ctx->lock[local->transaction.type];
+    if (!lock->acquired)
+        return _gf_false;
+    if (lock->acquired && lock->event_generation != local->event_generation)
+        return _gf_true;
+    return _gf_false;
 }
 
 void
-afr_transaction_eager_lock_init (afr_local_t *local, xlator_t *this)
+__afr_eager_lock_handle(afr_local_t *local, gf_boolean_t *take_lock,
+                        gf_boolean_t *do_pre_op, afr_local_t **timer_local)
 {
-        afr_private_t *priv = NULL;
-        afr_fd_ctx_t  *fdctx = NULL;
-        afr_local_t   *each = NULL;
-
-        priv = this->private;
-
-        if (!local->fd)
-                return;
-
-        if (local->transaction.type != AFR_DATA_TRANSACTION)
-                return;
-
-        if (!priv->eager_lock)
-                return;
-
-        fdctx = afr_fd_ctx_get (local->fd, this);
-        if (!fdctx)
-                return;
-
-        if (afr_are_multiple_fds_opened (local->fd, this))
-                return;
-        /*
-         * Once full file lock is acquired in eager-lock phase, overlapping
-         * writes do not compete for inode-locks, instead are transferred to the
-         * next writes. Because of this overlapping writes are not ordered.
-         * This can cause inconsistencies in replication.
-         * Example:
-         * Two overlapping writes w1, w2 are sent in parallel on same fd
-         * in two threads t1, t2.
-         * Both threads can execute afr_writev_wind in the following manner.
-         * t1 winds w1 on brick-0
-         * t2 winds w2 on brick-0
-         * t2 winds w2 on brick-1
-         * t1 winds w1 on brick-1
-         *
-         * This check makes sure the locks are not transferred for
-         * overlapping writes.
-         */
-        LOCK (&local->fd->lock);
-        {
-                list_for_each_entry (each, &fdctx->eager_locked,
-                                     transaction.eager_locked) {
-                        if (afr_locals_overlap (each, local)) {
-                                local->transaction.eager_lock_on = _gf_false;
-                                goto unlock;
-                        }
-                }
+    afr_lock_t *lock = NULL;
+    afr_local_t *owner_local = NULL;
+    xlator_t *this = local->transaction.frame->this;
+
+    local->transaction.eager_lock_on = _gf_true;
+    afr_set_lk_owner(local->transaction.frame, this, local->inode);
+
+    lock = &local->inode_ctx->lock[local->transaction.type];
+    if (__need_previous_lock_unlocked(local)) {
+        if (!list_empty(&lock->owners)) {
+            lock->release = _gf_true;
+        } else if (lock->delay_timer) {
+            lock->release = _gf_true;
+            if (gf_timer_call_cancel(this->ctx, lock->delay_timer)) {
+                /* It will be put in frozen list
+                 * in the code flow below*/
+            } else {
+                *timer_local = list_entry(lock->post_op.next, afr_local_t,
+                                          transaction.owner_list);
+                lock->delay_timer = NULL;
+            }
+        }
+    }
+
+    if (lock->release) {
+        list_add_tail(&local->transaction.wait_list, &lock->frozen);
+        *take_lock = _gf_false;
+        goto out;
+    }
+
+    if (lock->delay_timer) {
+        *take_lock = _gf_false;
+        if (gf_timer_call_cancel(this->ctx, lock->delay_timer)) {
+            list_add_tail(&local->transaction.wait_list, &lock->frozen);
+        } else {
+            *timer_local = list_entry(lock->post_op.next, afr_local_t,
+                                      transaction.owner_list);
+            afr_copy_inodelk_vars(&local->internal_lock,
+                                  &(*timer_local)->internal_lock, this, 0);
+            lock->delay_timer = NULL;
+            *do_pre_op = _gf_true;
+            list_add_tail(&local->transaction.owner_list, &lock->owners);
+        }
+        goto out;
+    }
 
-                local->transaction.eager_lock_on = _gf_true;
-                list_add_tail (&local->transaction.eager_locked,
-                               &fdctx->eager_locked);
+    if (!list_empty(&lock->owners)) {
+        if (!lock->acquired || afr_has_lock_conflict(local, _gf_true)) {
+            list_add_tail(&local->transaction.wait_list, &lock->waiting);
+            *take_lock = _gf_false;
+            goto out;
         }
-unlock:
-        UNLOCK (&local->fd->lock);
+        owner_local = list_entry(lock->owners.next, afr_local_t,
+                                 transaction.owner_list);
+        afr_copy_inodelk_vars(&local->internal_lock,
+                              &owner_local->internal_lock, this, 0);
+        *take_lock = _gf_false;
+        *do_pre_op = _gf_true;
+    }
+
+    if (lock->acquired)
+        GF_ASSERT(!(*take_lock));
+    list_add_tail(&local->transaction.owner_list, &lock->owners);
+out:
+    return;
 }
 
+void
+afr_transaction_start(afr_local_t *local, xlator_t *this)
+{
+    afr_private_t *priv = NULL;
+    gf_boolean_t take_lock = _gf_true;
+    gf_boolean_t do_pre_op = _gf_false;
+    afr_local_t *timer_local = NULL;
+
+    priv = this->private;
+
+    if (local->transaction.type != AFR_DATA_TRANSACTION &&
+        local->transaction.type != AFR_METADATA_TRANSACTION)
+        goto lock_phase;
+
+    if (!priv->eager_lock)
+        goto lock_phase;
+
+    LOCK(&local->inode->lock);
+    {
+        __afr_eager_lock_handle(local, &take_lock, &do_pre_op, &timer_local);
+    }
+    UNLOCK(&local->inode->lock);
+lock_phase:
+    if (!local->transaction.eager_lock_on) {
+        afr_set_lk_owner(local->transaction.frame, this,
+                         local->transaction.frame->root);
+    }
+
+    if (take_lock) {
+        afr_lock(local->transaction.frame, this);
+    } else if (do_pre_op) {
+        afr_changelog_pre_op(local->transaction.frame, this);
+    }
+    /*Always call delayed_changelog_wake_up_cbk after calling pre-op above
+     * so that any inheriting can happen*/
+    if (timer_local)
+        afr_delayed_changelog_wake_up_cbk(timer_local);
+}
 
 int
-afr_transaction (call_frame_t *frame, xlator_t *this, afr_transaction_type type)
+afr_write_txn_refresh_done(call_frame_t *frame, xlator_t *this, int err)
 {
-        afr_local_t *   local = NULL;
-        afr_private_t * priv  = NULL;
-        fd_t            *fd   = NULL;
-        int             ret   = -1;
-
-        local = frame->local;
-        priv  = this->private;
-
-        local->transaction.resume = afr_transaction_resume;
-        local->transaction.type   = type;
-
-        ret = afr_transaction_local_init (local, this);
-        if (ret < 0)
-            goto out;
+    afr_local_t *local = frame->local;
+
+    if (err) {
+        AFR_SET_ERROR_AND_CHECK_SPLIT_BRAIN(-1, err);
+        goto fail;
+    }
+
+    afr_transaction_start(local, this);
+    return 0;
+fail:
+    local->transaction.unwind(frame, this);
+    AFR_STACK_DESTROY(frame);
+    return 0;
+}
 
-        afr_transaction_eager_lock_init (local, this);
+int
+afr_transaction_lockee_init(call_frame_t *frame)
+{
+    afr_local_t *local = frame->local;
+    afr_internal_lock_t *int_lock = &local->internal_lock;
+    afr_private_t *priv = frame->this->private;
+    int ret = 0;
 
-        if (local->fd && local->transaction.eager_lock_on)
-                afr_set_lk_owner (frame, this, local->fd);
-        else
-                afr_set_lk_owner (frame, this, frame->root);
+    switch (local->transaction.type) {
+        case AFR_DATA_TRANSACTION:
+        case AFR_METADATA_TRANSACTION:
+            ret = afr_add_inode_lockee(local, priv->child_count);
+            break;
 
-        if (!local->transaction.eager_lock_on && local->loc.inode) {
-                fd = fd_lookup (local->loc.inode, frame->root->pid);
-                if (fd == NULL)
-                        fd = fd_lookup_anonymous (local->loc.inode);
+        case AFR_ENTRY_TRANSACTION:
+        case AFR_ENTRY_RENAME_TRANSACTION:
+            ret = afr_add_entry_lockee(local, &local->transaction.parent_loc,
+                                       local->transaction.basename,
+                                       priv->child_count);
+            if (ret) {
+                goto out;
+            }
+            if (local->op == GF_FOP_RENAME) {
+                ret = afr_add_entry_lockee(
+                    local, &local->transaction.new_parent_loc,
+                    local->transaction.new_basename, priv->child_count);
+                if (ret) {
+                    goto out;
+                }
 
-                if (fd) {
-                        afr_delayed_changelog_wake_up (this, fd);
-                        fd_unref (fd);
+                if (local->newloc.inode &&
+                    IA_ISDIR(local->newloc.inode->ia_type)) {
+                    ret = afr_add_entry_lockee(local, &local->newloc, NULL,
+                                               priv->child_count);
+                    if (ret) {
+                        goto out;
+                    }
                 }
-        }
+            } else if (local->op == GF_FOP_RMDIR) {
+                ret = afr_add_entry_lockee(local, &local->loc, NULL,
+                                           priv->child_count);
+                if (ret) {
+                    goto out;
+                }
+            }
+
+            if (int_lock->lockee_count > 1) {
+                qsort(int_lock->lockee, int_lock->lockee_count,
+                      sizeof(*int_lock->lockee), afr_entry_lockee_cmp);
+            }
+            break;
+    }
+out:
+    return ret;
+}
 
-        if (afr_lock_server_count (priv, local->transaction.type) == 0) {
-                afr_internal_lock_finish (frame, this);
-        } else {
-                afr_lock (frame, this);
-        }
+int
+afr_transaction(call_frame_t *frame, xlator_t *this, afr_transaction_type type)
+{
+    afr_local_t *local = NULL;
+    afr_private_t *priv = NULL;
+    int ret = -1;
+    int event_generation = 0;
+
+    local = frame->local;
+    priv = this->private;
+    local->transaction.frame = frame;
+
+    local->transaction.type = type;
+
+    if (priv->quorum_count && !afr_has_quorum(local->child_up, this, NULL)) {
+        ret = -afr_quorum_errno(priv);
+        goto out;
+    }
+
+    if (!afr_is_consistent_io_possible(local, priv, &ret)) {
+        ret = -ret; /*op_errno to ret conversion*/
+        goto out;
+    }
+
+    if (priv->thin_arbiter_count && !afr_ta_has_quorum(priv, local)) {
+        ret = -afr_quorum_errno(priv);
+        goto out;
+    }
+
+    ret = afr_transaction_local_init(local, this);
+    if (ret < 0)
+        goto out;
+
+    ret = afr_transaction_lockee_init(frame);
+    if (ret)
+        goto out;
+
+    if (type != AFR_METADATA_TRANSACTION) {
+        goto txn_start;
+    }
+
+    ret = afr_inode_get_readable(frame, local->inode, this, local->readable,
+                                 &event_generation, type);
+    if (ret < 0 ||
+        afr_is_inode_refresh_reqd(local->inode, this, priv->event_generation,
+                                  event_generation)) {
+        afr_inode_refresh(frame, this, local->inode, local->loc.gfid,
+                          afr_write_txn_refresh_done);
         ret = 0;
+        goto out;
+    }
+
+txn_start:
+    ret = 0;
+    afr_transaction_start(local, this);
 out:
-        return ret;
+    return ret;
 }
diff --git a/xlators/cluster/afr/src/afr-transaction.h b/xlators/cluster/afr/src/afr-transaction.h
index c3ce333b771..beefa26f4a6 100644
--- a/xlators/cluster/afr/src/afr-transaction.h
+++ b/xlators/cluster/afr/src/afr-transaction.h
@@ -14,41 +14,62 @@
 #include "afr.h"
 
 void
-afr_transaction_fop_failed (call_frame_t *frame, xlator_t *this,
-			    int child_index);
+afr_transaction_fop_failed(call_frame_t *frame, xlator_t *this,
+                           int child_index);
+
+int32_t
+afr_transaction(call_frame_t *frame, xlator_t *this, afr_transaction_type type);
 
 int
-afr_lock_server_count (afr_private_t *priv, afr_transaction_type type);
+afr_set_pending_dict(afr_private_t *priv, dict_t *xattr, int32_t **pending);
 
-afr_inodelk_t*
-afr_get_inodelk (afr_internal_lock_t *int_lock, char *dom);
+void
+afr_delayed_changelog_wake_up(xlator_t *this, fd_t *fd);
 
-int32_t
-afr_transaction (call_frame_t *frame, xlator_t *this, afr_transaction_type type);
+void
+__mark_all_success(call_frame_t *frame, xlator_t *this);
+
+gf_boolean_t
+afr_txn_nothing_failed(call_frame_t *frame, xlator_t *this);
 
 int
-afr_set_pending_dict (afr_private_t *priv, dict_t *xattr, int32_t **pending);
+afr_read_txn(call_frame_t *frame, xlator_t *this, inode_t *inode,
+             afr_read_txn_wind_t readfn, afr_transaction_type type);
 
-void
-afr_set_delayed_post_op (call_frame_t *frame, xlator_t *this);
+int
+afr_read_txn_continue(call_frame_t *frame, xlator_t *this, int subvol);
 
 void
-afr_delayed_changelog_wake_up (xlator_t *this, fd_t *fd);
+afr_pending_read_increment(afr_private_t *priv, int child_index);
 
 void
-__mark_all_success (call_frame_t *frame, xlator_t *this);
+afr_pending_read_decrement(afr_private_t *priv, int child_index);
 
+call_frame_t *
+afr_transaction_detach_fop_frame(call_frame_t *frame);
 gf_boolean_t
-afr_txn_nothing_failed (call_frame_t *frame, xlator_t *this);
+afr_has_quorum(unsigned char *subvols, xlator_t *this, call_frame_t *frame);
+gf_boolean_t
+afr_needs_changelog_update(afr_local_t *local);
+void
+afr_zero_fill_stat(afr_local_t *local);
 
-int afr_read_txn (call_frame_t *frame, xlator_t *this, inode_t *inode,
-		  afr_read_txn_wind_t readfn, afr_transaction_type type);
+void
+afr_pick_error_xdata(afr_local_t *local, afr_private_t *priv, inode_t *inode1,
+                     unsigned char *readable1, inode_t *inode2,
+                     unsigned char *readable2);
+int
+afr_transaction_resume(call_frame_t *frame, xlator_t *this);
 
-int afr_read_txn_continue (call_frame_t *frame, xlator_t *this, int subvol);
+int
+afr_lock(call_frame_t *frame, xlator_t *this);
 
-int __afr_txn_write_fop (call_frame_t *frame, xlator_t *this);
-int __afr_txn_write_done (call_frame_t *frame, xlator_t *this);
-call_frame_t *afr_transaction_detach_fop_frame (call_frame_t *frame);
-gf_boolean_t afr_has_quorum (unsigned char *subvols, xlator_t *this);
+void
+afr_delayed_changelog_wake_up_cbk(void *data);
+
+int
+afr_release_notify_lock_for_ta(void *opaque);
 
+int
+afr_ta_lock_release_done(int ret, call_frame_t *ta_frame, void *opaque);
 #endif /* __TRANSACTION_H__ */
diff --git a/xlators/cluster/afr/src/afr.c b/xlators/cluster/afr/src/afr.c
index f962fb6494e..df7366f0a65 100644
--- a/xlators/cluster/afr/src/afr.c
+++ b/xlators/cluster/afr/src/afr.c
@@ -15,784 +15,1330 @@
 #include <stdlib.h>
 #include <signal.h>
 
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
 #include "afr-common.c"
+#include "afr-messages.h"
 
 struct volume_options options[];
 
+static char *afr_favorite_child_policies[AFR_FAV_CHILD_POLICY_MAX + 1] = {
+    [AFR_FAV_CHILD_NONE] = "none",
+    [AFR_FAV_CHILD_BY_SIZE] = "size",
+    [AFR_FAV_CHILD_BY_CTIME] = "ctime",
+    [AFR_FAV_CHILD_BY_MTIME] = "mtime",
+    [AFR_FAV_CHILD_BY_MAJORITY] = "majority",
+    [AFR_FAV_CHILD_POLICY_MAX] = NULL,
+};
+
 int32_t
-notify (xlator_t *this, int32_t event,
-        void *data, ...)
+notify(xlator_t *this, int32_t event, void *data, ...)
 {
-        int ret = -1;
-        va_list         ap;
-        void *data2 = NULL;
+    int ret = -1;
+    va_list ap;
+    void *data2 = NULL;
 
-        va_start (ap, data);
-        data2 = va_arg (ap, dict_t*);
-        va_end (ap);
-        ret = afr_notify (this, event, data, data2);
+    va_start(ap, data);
+    data2 = va_arg(ap, dict_t *);
+    va_end(ap);
+    ret = afr_notify(this, event, data, data2);
 
-        return ret;
+    return ret;
 }
 
 int32_t
-mem_acct_init (xlator_t *this)
+mem_acct_init(xlator_t *this)
 {
-        int     ret = -1;
+    int ret = -1;
 
-        if (!this)
-                return ret;
-
-        ret = xlator_mem_acct_init (this, gf_afr_mt_end + 1);
+    if (!this)
+        return ret;
 
-        if (ret != 0) {
-                gf_log(this->name, GF_LOG_ERROR, "Memory accounting init"
-                       "failed");
-                return ret;
-        }
+    ret = xlator_mem_acct_init(this, gf_afr_mt_end + 1);
 
+    if (ret != 0) {
         return ret;
-}
+    }
 
+    return ret;
+}
 
 int
-xlator_subvolume_index (xlator_t *this, xlator_t *subvol)
+xlator_subvolume_index(xlator_t *this, xlator_t *subvol)
 {
-        int index = -1;
-        int i = 0;
-        xlator_list_t *list = NULL;
-
-        list = this->children;
-
-        while (list) {
-                if (subvol == list->xlator ||
-                    strcmp (subvol->name, list->xlator->name) == 0) {
-                        index = i;
-                        break;
-                }
-                list = list->next;
-                i++;
+    int index = -1;
+    int i = 0;
+    xlator_list_t *list = NULL;
+
+    list = this->children;
+
+    while (list) {
+        if (subvol == list->xlator ||
+            strcmp(subvol->name, list->xlator->name) == 0) {
+            index = i;
+            break;
         }
+        list = list->next;
+        i++;
+    }
 
-        return index;
+    return index;
 }
 
 static void
-fix_quorum_options (xlator_t *this, afr_private_t *priv, char *qtype,
-                    dict_t *options)
+fix_quorum_options(xlator_t *this, afr_private_t *priv, char *qtype,
+                   dict_t *options)
 {
-        if (dict_get (options, "quorum-type") == NULL) {
-                /* If user doesn't configure anything enable auto-quorum if the
-                 * replica has odd number of subvolumes */
-                if (priv->child_count % 2)
-                        qtype = "auto";
-        }
-
-        if (priv->quorum_count && strcmp (qtype, "fixed")) {
-                gf_msg (this->name,GF_LOG_WARNING, 0, AFR_MSG_QUORUM_OVERRIDE,
-                       "quorum-type %s overriding quorum-count %u",
-                       qtype, priv->quorum_count);
-        }
-
-        if (!strcmp (qtype, "none")) {
-                priv->quorum_count = 0;
-        } else if (!strcmp (qtype, "auto")) {
-                priv->quorum_count = AFR_QUORUM_AUTO;
-        }
+    if (dict_get_sizen(options, "quorum-type") == NULL) {
+        /* If user doesn't configure anything enable auto-quorum if the
+         * replica has more than two subvolumes */
+        if (priv->child_count > 2)
+            qtype = "auto";
+    }
+
+    if (priv->quorum_count && strcmp(qtype, "fixed")) {
+        gf_msg(this->name, GF_LOG_WARNING, 0, AFR_MSG_QUORUM_OVERRIDE,
+               "quorum-type %s overriding quorum-count %u", qtype,
+               priv->quorum_count);
+    }
+
+    if (!strcmp(qtype, "none")) {
+        priv->quorum_count = 0;
+    } else if (!strcmp(qtype, "auto")) {
+        priv->quorum_count = AFR_QUORUM_AUTO;
+    }
 }
 
 int
-reconfigure (xlator_t *this, dict_t *options)
+afr_set_favorite_child_policy(afr_private_t *priv, char *policy)
 {
-        afr_private_t *priv        = NULL;
-        xlator_t      *read_subvol = NULL;
-        int            read_subvol_index = -1;
-        int            ret         = -1;
-        int            index       = -1;
-        char          *qtype       = NULL;
+    int index = -1;
 
-        priv = this->private;
+    index = gf_get_index_by_elem(afr_favorite_child_policies, policy);
+    if (index < 0 || index >= AFR_FAV_CHILD_POLICY_MAX)
+        return -1;
 
-	GF_OPTION_RECONF ("afr-dirty-xattr",
-			  priv->afr_dirty, options, str,
-			  out);
+    priv->fav_child_policy = index;
 
-	GF_OPTION_RECONF ("metadata-splitbrain-forced-heal",
-			  priv->metadata_splitbrain_forced_heal, options, bool,
-			  out);
-
-        GF_OPTION_RECONF ("background-self-heal-count",
-                          priv->background_self_heal_count, options, uint32,
-                          out);
-
-        GF_OPTION_RECONF ("metadata-self-heal",
-                          priv->metadata_self_heal, options, bool, out);
+    return 0;
+}
 
-        GF_OPTION_RECONF ("data-self-heal", priv->data_self_heal, options, str,
-                          out);
+static void
+set_data_self_heal_algorithm(afr_private_t *priv, char *algo)
+{
+    if (!algo) {
+        priv->data_self_heal_algorithm = AFR_SELFHEAL_DATA_DYNAMIC;
+    } else if (strcmp(algo, "full") == 0) {
+        priv->data_self_heal_algorithm = AFR_SELFHEAL_DATA_FULL;
+    } else if (strcmp(algo, "diff") == 0) {
+        priv->data_self_heal_algorithm = AFR_SELFHEAL_DATA_DIFF;
+    } else {
+        priv->data_self_heal_algorithm = AFR_SELFHEAL_DATA_DYNAMIC;
+    }
+}
 
-        GF_OPTION_RECONF ("entry-self-heal", priv->entry_self_heal, options,
-                          bool, out);
+void
+afr_handle_anon_inode_options(afr_private_t *priv, dict_t *options)
+{
+    char *volfile_id_str = NULL;
+    uuid_t anon_inode_gfid = {0};
+
+    /*If volume id is not present don't enable anything*/
+    if (dict_get_str(options, "volume-id", &volfile_id_str))
+        return;
+    GF_ASSERT(strlen(AFR_ANON_DIR_PREFIX) + strlen(volfile_id_str) <= NAME_MAX);
+    /*anon_inode_name is not supposed to change once assigned*/
+    if (!priv->anon_inode_name[0]) {
+        snprintf(priv->anon_inode_name, sizeof(priv->anon_inode_name), "%s-%s",
+                 AFR_ANON_DIR_PREFIX, volfile_id_str);
+        gf_uuid_parse(volfile_id_str, anon_inode_gfid);
+        /*Flip a bit to make sure volfile-id and anon-gfid are not same*/
+        anon_inode_gfid[0] ^= 1;
+        uuid_utoa_r(anon_inode_gfid, priv->anon_gfid_str);
+    }
+}
 
-        GF_OPTION_RECONF ("data-self-heal-window-size",
-                          priv->data_self_heal_window_size, options,
-                          uint32, out);
+int
+reconfigure(xlator_t *this, dict_t *options)
+{
+    afr_private_t *priv = NULL;
+    xlator_t *read_subvol = NULL;
+    int read_subvol_index = -1;
+    int timeout_old = 0;
+    int ret = -1;
+    int index = -1;
+    char *qtype = NULL;
+    char *fav_child_policy = NULL;
+    char *data_self_heal = NULL;
+    char *data_self_heal_algorithm = NULL;
+    char *locking_scheme = NULL;
+    gf_boolean_t consistent_io = _gf_false;
+    gf_boolean_t choose_local_old = _gf_false;
+    gf_boolean_t enabled_old = _gf_false;
 
-        GF_OPTION_RECONF ("data-change-log", priv->data_change_log, options,
-                          bool, out);
+    priv = this->private;
 
-        GF_OPTION_RECONF ("metadata-change-log",
-                          priv->metadata_change_log, options, bool, out);
+    GF_OPTION_RECONF("metadata-splitbrain-forced-heal",
+                     priv->metadata_splitbrain_forced_heal, options, bool, out);
 
-        GF_OPTION_RECONF ("entry-change-log", priv->entry_change_log, options,
-                          bool, out);
+    GF_OPTION_RECONF("background-self-heal-count",
+                     priv->background_self_heal_count, options, uint32, out);
 
-        GF_OPTION_RECONF ("data-self-heal-algorithm",
-                          priv->data_self_heal_algorithm, options, str, out);
+    GF_OPTION_RECONF("heal-wait-queue-length", priv->heal_wait_qlen, options,
+                     uint32, out);
 
-        GF_OPTION_RECONF ("read-subvolume", read_subvol, options, xlator, out);
+    GF_OPTION_RECONF("metadata-self-heal", priv->metadata_self_heal, options,
+                     bool, out);
 
-        GF_OPTION_RECONF ("read-hash-mode", priv->hash_mode,
-                          options, uint32, out);
+    GF_OPTION_RECONF("data-self-heal", data_self_heal, options, str, out);
+    if (gf_string2boolean(data_self_heal, &priv->data_self_heal) == -1)
+        goto out;
 
-        if (read_subvol) {
-                index = xlator_subvolume_index (this, read_subvol);
-                if (index == -1) {
-                        gf_log (this->name, GF_LOG_ERROR, "%s not a subvolume",
-                                read_subvol->name);
-                        goto out;
-                }
-                priv->read_child = index;
-        }
+    GF_OPTION_RECONF("entry-self-heal", priv->entry_self_heal, options, bool,
+                     out);
 
-        GF_OPTION_RECONF ("read-subvolume-index",read_subvol_index, options,int32,out);
+    GF_OPTION_RECONF("data-self-heal-window-size",
+                     priv->data_self_heal_window_size, options, uint32, out);
 
-        if (read_subvol_index >-1) {
-                index=read_subvol_index;
-                if (index >= priv->child_count) {
-                        gf_log (this->name, GF_LOG_ERROR, "%d not a subvolume-index",
-                                index);
-                        goto out;
-                }
-                priv->read_child = index;
-        }
+    GF_OPTION_RECONF("data-self-heal-algorithm", data_self_heal_algorithm,
+                     options, str, out);
+    set_data_self_heal_algorithm(priv, data_self_heal_algorithm);
 
-        GF_OPTION_RECONF ("pre-op-compat", priv->pre_op_compat, options, bool, out);
+    GF_OPTION_RECONF("halo-enabled", priv->halo_enabled, options, bool, out);
 
-        GF_OPTION_RECONF ("eager-lock", priv->eager_lock, options, bool, out);
-        GF_OPTION_RECONF ("quorum-type", qtype, options, str, out);
-        GF_OPTION_RECONF ("quorum-count", priv->quorum_count, options,
-                          uint32, out);
-        fix_quorum_options (this, priv, qtype, options);
-        if (priv->quorum_count && !afr_has_quorum (priv->child_up, this))
-                gf_msg (this->name, GF_LOG_WARNING, 0, AFR_MSG_QUORUM_FAIL,
-                        "Client-quorum is not met");
+    GF_OPTION_RECONF("halo-shd-max-latency", priv->shd.halo_max_latency_msec,
+                     options, uint32, out);
 
+    GF_OPTION_RECONF("halo-nfsd-max-latency", priv->nfsd.halo_max_latency_msec,
+                     options, uint32, out);
 
-	GF_OPTION_RECONF ("post-op-delay-secs", priv->post_op_delay_secs, options,
-			  uint32, out);
+    GF_OPTION_RECONF("halo-max-latency", priv->halo_max_latency_msec, options,
+                     uint32, out);
 
-        GF_OPTION_RECONF (AFR_SH_READDIR_SIZE_KEY, priv->sh_readdir_size,
-                          options, size_uint64, out);
-        /* Reset this so we re-discover in case the topology changed.  */
-        GF_OPTION_RECONF ("ensure-durability", priv->ensure_durability, options,
-                          bool, out);
+    GF_OPTION_RECONF("halo-max-replicas", priv->halo_max_replicas, options,
+                     uint32, out);
 
-	GF_OPTION_RECONF ("self-heal-daemon", priv->shd.enabled, options,
-			  bool, out);
+    GF_OPTION_RECONF("halo-min-replicas", priv->halo_min_replicas, options,
+                     uint32, out);
 
-	GF_OPTION_RECONF ("iam-self-heal-daemon", priv->shd.iamshd, options,
-			  bool, out);
+    GF_OPTION_RECONF("read-subvolume", read_subvol, options, xlator, out);
 
-        GF_OPTION_RECONF ("heal-timeout", priv->shd.timeout, options,
-                          int32, out);
+    choose_local_old = priv->choose_local;
+    GF_OPTION_RECONF("choose-local", priv->choose_local, options, bool, out);
 
-        GF_OPTION_RECONF ("quorum-reads", priv->quorum_reads, options,
-                          bool, out);
-        GF_OPTION_RECONF ("consistent-metadata", priv->consistent_metadata,
-                          options, bool, out);
+    if (choose_local_old != priv->choose_local) {
+        priv->read_child = -1;
+        if (choose_local_old == _gf_false)
+            priv->did_discovery = _gf_false;
+    }
+
+    GF_OPTION_RECONF("read-hash-mode", priv->hash_mode, options, uint32, out);
+
+    if (read_subvol) {
+        index = xlator_subvolume_index(this, read_subvol);
+        if (index == -1) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, AFR_MSG_INVALID_SUBVOL,
+                   "%s not a subvolume", read_subvol->name);
+            goto out;
+        }
+        priv->read_child = index;
+    }
+
+    GF_OPTION_RECONF("read-subvolume-index", read_subvol_index, options, int32,
+                     out);
+
+    if (read_subvol_index > -1) {
+        index = read_subvol_index;
+        if (index >= priv->child_count) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, AFR_MSG_INVALID_SUBVOL,
+                   "%d not a subvolume-index", index);
+            goto out;
+        }
+        priv->read_child = index;
+    }
+
+    GF_OPTION_RECONF("pre-op-compat", priv->pre_op_compat, options, bool, out);
+    GF_OPTION_RECONF("locking-scheme", locking_scheme, options, str, out);
+    priv->granular_locks = (strcmp(locking_scheme, "granular") == 0);
+    GF_OPTION_RECONF("full-lock", priv->full_lock, options, bool, out);
+    GF_OPTION_RECONF("granular-entry-heal", priv->esh_granular, options, bool,
+                     out);
+
+    GF_OPTION_RECONF("eager-lock", priv->eager_lock, options, bool, out);
+    GF_OPTION_RECONF("optimistic-change-log", priv->optimistic_change_log,
+                     options, bool, out);
+    GF_OPTION_RECONF("quorum-type", qtype, options, str, out);
+    GF_OPTION_RECONF("quorum-count", priv->quorum_count, options, uint32, out);
+    fix_quorum_options(this, priv, qtype, options);
+    if (priv->quorum_count && !afr_has_quorum(priv->child_up, this, NULL))
+        gf_msg(this->name, GF_LOG_WARNING, 0, AFR_MSG_QUORUM_FAIL,
+               "Client-quorum is not met");
+
+    GF_OPTION_RECONF("post-op-delay-secs", priv->post_op_delay_secs, options,
+                     uint32, out);
+
+    GF_OPTION_RECONF(AFR_SH_READDIR_SIZE_KEY, priv->sh_readdir_size, options,
+                     size_uint64, out);
+    /* Reset this so we re-discover in case the topology changed.  */
+    GF_OPTION_RECONF("ensure-durability", priv->ensure_durability, options,
+                     bool, out);
+
+    enabled_old = priv->shd.enabled;
+    GF_OPTION_RECONF("self-heal-daemon", priv->shd.enabled, options, bool, out);
+
+    GF_OPTION_RECONF("iam-self-heal-daemon", priv->shd.iamshd, options, bool,
+                     out);
+
+    timeout_old = priv->shd.timeout;
+    GF_OPTION_RECONF("heal-timeout", priv->shd.timeout, options, int32, out);
+
+    GF_OPTION_RECONF("consistent-metadata", priv->consistent_metadata, options,
+                     bool, out);
+
+    GF_OPTION_RECONF("shd-max-threads", priv->shd.max_threads, options, uint32,
+                     out);
+
+    GF_OPTION_RECONF("shd-wait-qlength", priv->shd.wait_qlength, options,
+                     uint32, out);
+
+    GF_OPTION_RECONF("favorite-child-policy", fav_child_policy, options, str,
+                     out);
+    if (afr_set_favorite_child_policy(priv, fav_child_policy) == -1)
+        goto out;
+
+    priv->did_discovery = _gf_false;
+
+    GF_OPTION_RECONF("consistent-io", consistent_io, options, bool, out);
+    if (priv->quorum_count != 0)
+        consistent_io = _gf_false;
+    priv->consistent_io = consistent_io;
+
+    afr_handle_anon_inode_options(priv, options);
+
+    GF_OPTION_RECONF("use-anonymous-inode", priv->use_anon_inode, options, bool,
+                     out);
+    if (priv->shd.enabled) {
+        if ((priv->shd.enabled != enabled_old) ||
+            (timeout_old != priv->shd.timeout))
+            afr_selfheal_childup(this, priv);
+    }
+
+    ret = 0;
+out:
+    return ret;
+}
 
-        priv->did_discovery = _gf_false;
+static int
+afr_pending_xattrs_init(afr_private_t *priv, xlator_t *this)
+{
+    int ret = -1;
+    int i = 0;
+    char *ptr = NULL;
+    char *ptr1 = NULL;
+    char *xattrs_list = NULL;
+    xlator_list_t *trav = NULL;
+    int child_count = -1;
+
+    trav = this->children;
+    child_count = priv->child_count;
+    if (priv->thin_arbiter_count) {
+        /* priv->pending_key[THIN_ARBITER_BRICK_INDEX] is used as the
+         * name of the thin arbiter file for persistence across add/
+         * removal of DHT subvols.*/
+        child_count++;
+    }
+
+    GF_OPTION_INIT("afr-pending-xattr", xattrs_list, str, out);
+    priv->pending_key = GF_CALLOC(sizeof(*priv->pending_key), child_count,
+                                  gf_afr_mt_char);
+    if (!priv->pending_key) {
+        ret = -ENOMEM;
+        goto out;
+    }
+    if (!xattrs_list) {
+        gf_msg(this->name, GF_LOG_WARNING, 0, AFR_MSG_NO_CHANGELOG,
+               "Unable to fetch afr-pending-xattr option from volfile."
+               " Falling back to using client translator names. ");
 
+        while (i < child_count) {
+            ret = gf_asprintf(&priv->pending_key[i], "%s.%s", AFR_XATTR_PREFIX,
+                              trav->xlator->name);
+            if (ret == -1) {
+                ret = -ENOMEM;
+                goto out;
+            }
+            trav = trav->next;
+            i++;
+        }
         ret = 0;
-out:
-        return ret;
+        goto out;
+    }
+
+    ptr = ptr1 = gf_strdup(xattrs_list);
+    if (!ptr) {
+        ret = -ENOMEM;
+        goto out;
+    }
+    for (i = 0, ptr = strtok(ptr, ","); ptr; ptr = strtok(NULL, ",")) {
+        ret = gf_asprintf(&priv->pending_key[i], "%s.%s", AFR_XATTR_PREFIX,
+                          ptr);
+        if (ret == -1) {
+            ret = -ENOMEM;
+            goto out;
+        }
+        i++;
+    }
+    ret = 0;
 
+out:
+    GF_FREE(ptr1);
+    return ret;
 }
 
-
-static const char *favorite_child_warning_str = "You have specified subvolume '%s' "
-        "as the 'favorite child'. This means that if a discrepancy in the content "
-        "or attributes (ownership, permission, etc.) of a file is detected among "
-        "the subvolumes, the file on '%s' will be considered the definitive "
-        "version and its contents will OVERWRITE the contents of the file on other "
-        "subvolumes. All versions of the file except that on '%s' "
-        "WILL BE LOST.";
-
+void
+afr_ta_init(afr_private_t *priv)
+{
+    priv->thin_arbiter_count = 1;
+    priv->child_count--;
+    priv->ta_child_up = 0;
+    priv->ta_bad_child_index = AFR_CHILD_UNKNOWN;
+    priv->ta_notify_dom_lock_offset = 0;
+    priv->ta_in_mem_txn_count = 0;
+    priv->ta_on_wire_txn_count = 0;
+    priv->release_ta_notify_dom_lock = _gf_false;
+    INIT_LIST_HEAD(&priv->ta_waitq);
+    INIT_LIST_HEAD(&priv->ta_onwireq);
+    gf_uuid_clear(priv->ta_gfid);
+}
 
 int32_t
-init (xlator_t *this)
+init(xlator_t *this)
 {
-        afr_private_t *priv        = NULL;
-        int            child_count = 0;
-        xlator_list_t *trav        = NULL;
-        int            i           = 0;
-        int            ret         = -1;
-        GF_UNUSED int  op_errno    = 0;
-        xlator_t      *read_subvol = NULL;
-        int            read_subvol_index = -1;
-        xlator_t      *fav_child   = NULL;
-        char          *qtype       = NULL;
-
-        if (!this->children) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "replicate translator needs more than one "
-                        "subvolume defined.");
-                return -1;
+    afr_private_t *priv = NULL;
+    int child_count = 0;
+    xlator_list_t *trav = NULL;
+    int i = 0;
+    int ret = -1;
+    GF_UNUSED int op_errno = 0;
+    xlator_t *read_subvol = NULL;
+    int read_subvol_index = -1;
+    char *qtype = NULL;
+    char *fav_child_policy = NULL;
+    char *thin_arbiter = NULL;
+    char *data_self_heal = NULL;
+    char *locking_scheme = NULL;
+    char *data_self_heal_algorithm = NULL;
+
+    if (!this->children) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, AFR_MSG_CHILD_MISCONFIGURED,
+               "replicate translator needs more than one "
+               "subvolume defined.");
+        return -1;
+    }
+
+    if (!this->parents) {
+        gf_msg(this->name, GF_LOG_WARNING, 0, AFR_MSG_VOL_MISCONFIGURED,
+               "Volume is dangling.");
+    }
+
+    this->private = GF_CALLOC(1, sizeof(afr_private_t),
+                              gf_afr_mt_afr_private_t);
+    if (!this->private)
+        goto out;
+
+    priv = this->private;
+    INIT_LIST_HEAD(&priv->saved_locks);
+    INIT_LIST_HEAD(&priv->lk_healq);
+    LOCK_INIT(&priv->lock);
+
+    child_count = xlator_subvolume_count(this);
+
+    priv->child_count = child_count;
+
+    priv->read_child = -1;
+
+    GF_OPTION_INIT("arbiter-count", priv->arbiter_count, uint32, out);
+    GF_OPTION_INIT("thin-arbiter", thin_arbiter, str, out);
+    if (thin_arbiter && strlen(thin_arbiter) > 0) {
+        afr_ta_init(priv);
+    }
+    INIT_LIST_HEAD(&priv->healing);
+    INIT_LIST_HEAD(&priv->heal_waiting);
+
+    priv->spb_choice_timeout = AFR_DEFAULT_SPB_CHOICE_TIMEOUT;
+
+    GF_OPTION_INIT("afr-dirty-xattr", priv->afr_dirty, str, out);
+
+    GF_OPTION_INIT("metadata-splitbrain-forced-heal",
+                   priv->metadata_splitbrain_forced_heal, bool, out);
+
+    GF_OPTION_INIT("read-subvolume", read_subvol, xlator, out);
+    if (read_subvol) {
+        priv->read_child = xlator_subvolume_index(this, read_subvol);
+        if (priv->read_child == -1) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, AFR_MSG_INVALID_SUBVOL,
+                   "%s not a subvolume", read_subvol->name);
+            goto out;
         }
-
-        if (!this->parents) {
-                gf_log (this->name, GF_LOG_WARNING,
-                        "Volume is dangling.");
+    }
+    GF_OPTION_INIT("read-subvolume-index", read_subvol_index, int32, out);
+    if (read_subvol_index > -1) {
+        if (read_subvol_index >= priv->child_count) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, AFR_MSG_INVALID_SUBVOL,
+                   "%d not a subvolume-index", read_subvol_index);
+            goto out;
         }
+        priv->read_child = read_subvol_index;
+    }
+    GF_OPTION_INIT("choose-local", priv->choose_local, bool, out);
 
-	this->private = GF_CALLOC (1, sizeof (afr_private_t),
-                                   gf_afr_mt_afr_private_t);
-        if (!this->private)
-                goto out;
+    priv->pending_reads = GF_CALLOC(sizeof(*priv->pending_reads),
+                                    priv->child_count, gf_afr_mt_atomic_t);
 
-        priv = this->private;
-        LOCK_INIT (&priv->lock);
+    GF_OPTION_INIT("read-hash-mode", priv->hash_mode, uint32, out);
 
-        child_count = xlator_subvolume_count (this);
+    priv->favorite_child = -1;
 
-        priv->child_count = child_count;
+    GF_OPTION_INIT("favorite-child-policy", fav_child_policy, str, out);
+    if (afr_set_favorite_child_policy(priv, fav_child_policy) == -1)
+        goto out;
 
-        priv->read_child = -1;
+    GF_OPTION_INIT("shd-max-threads", priv->shd.max_threads, uint32, out);
 
-	GF_OPTION_INIT ("afr-dirty-xattr", priv->afr_dirty, str, out);
+    GF_OPTION_INIT("shd-wait-qlength", priv->shd.wait_qlength, uint32, out);
 
-	GF_OPTION_INIT ("metadata-splitbrain-forced-heal",
-			priv->metadata_splitbrain_forced_heal, bool, out);
+    GF_OPTION_INIT("background-self-heal-count",
+                   priv->background_self_heal_count, uint32, out);
 
-        GF_OPTION_INIT ("read-subvolume", read_subvol, xlator, out);
-        if (read_subvol) {
-                priv->read_child = xlator_subvolume_index (this, read_subvol);
-                if (priv->read_child == -1) {
-                        gf_log (this->name, GF_LOG_ERROR, "%s not a subvolume",
-                                read_subvol->name);
-                        goto out;
-                }
-        }
-        GF_OPTION_INIT ("read-subvolume-index",read_subvol_index,int32,out);
-        if (read_subvol_index > -1) {
-                if (read_subvol_index >= priv->child_count) {
-                        gf_log (this->name, GF_LOG_ERROR, "%d not a subvolume-index",
-                                read_subvol_index);
-                        goto out;
-                }
-                priv->read_child = read_subvol_index;
-        }
-        GF_OPTION_INIT ("choose-local", priv->choose_local, bool, out);
-
-        GF_OPTION_INIT ("read-hash-mode", priv->hash_mode, uint32, out);
-
-        priv->favorite_child = -1;
-        GF_OPTION_INIT ("favorite-child", fav_child, xlator, out);
-        if (fav_child) {
-                priv->favorite_child = xlator_subvolume_index (this, fav_child);
-                if (priv->favorite_child == -1) {
-                        gf_log (this->name, GF_LOG_ERROR, "%s not a subvolume",
-                                fav_child->name);
-                        goto out;
-                }
-                gf_log (this->name, GF_LOG_WARNING,
-                        favorite_child_warning_str, fav_child->name,
-                        fav_child->name, fav_child->name);
-        }
+    GF_OPTION_INIT("heal-wait-queue-length", priv->heal_wait_qlen, uint32, out);
 
+    GF_OPTION_INIT("data-self-heal", data_self_heal, str, out);
+    if (gf_string2boolean(data_self_heal, &priv->data_self_heal) == -1)
+        goto out;
 
-        GF_OPTION_INIT ("background-self-heal-count",
-                        priv->background_self_heal_count, uint32, out);
+    GF_OPTION_INIT("data-self-heal-algorithm", data_self_heal_algorithm, str,
+                   out);
+    set_data_self_heal_algorithm(priv, data_self_heal_algorithm);
 
-        GF_OPTION_INIT ("data-self-heal", priv->data_self_heal, str, out);
+    GF_OPTION_INIT("data-self-heal-window-size",
+                   priv->data_self_heal_window_size, uint32, out);
 
-        GF_OPTION_INIT ("data-self-heal-algorithm",
-                        priv->data_self_heal_algorithm, str, out);
+    GF_OPTION_INIT("metadata-self-heal", priv->metadata_self_heal, bool, out);
 
-        GF_OPTION_INIT ("data-self-heal-window-size",
-                        priv->data_self_heal_window_size, uint32, out);
+    GF_OPTION_INIT("entry-self-heal", priv->entry_self_heal, bool, out);
 
-        GF_OPTION_INIT ("metadata-self-heal", priv->metadata_self_heal, bool,
-                        out);
+    GF_OPTION_INIT("halo-shd-max-latency", priv->shd.halo_max_latency_msec,
+                   uint32, out);
 
-        GF_OPTION_INIT ("entry-self-heal", priv->entry_self_heal, bool, out);
+    GF_OPTION_INIT("halo-max-latency", priv->halo_max_latency_msec, uint32,
+                   out);
+    GF_OPTION_INIT("halo-max-replicas", priv->halo_max_replicas, uint32, out);
+    GF_OPTION_INIT("halo-min-replicas", priv->halo_min_replicas, uint32, out);
 
-        GF_OPTION_INIT ("data-change-log", priv->data_change_log, bool, out);
+    GF_OPTION_INIT("halo-enabled", priv->halo_enabled, bool, out);
 
-        GF_OPTION_INIT ("metadata-change-log", priv->metadata_change_log, bool,
-                        out);
+    GF_OPTION_INIT("halo-nfsd-max-latency", priv->nfsd.halo_max_latency_msec,
+                   uint32, out);
 
-        GF_OPTION_INIT ("entry-change-log", priv->entry_change_log, bool, out);
+    GF_OPTION_INIT("iam-nfs-daemon", priv->nfsd.iamnfsd, bool, out);
 
-        GF_OPTION_INIT ("optimistic-change-log", priv->optimistic_change_log,
-                        bool, out);
+    GF_OPTION_INIT("optimistic-change-log", priv->optimistic_change_log, bool,
+                   out);
 
-        GF_OPTION_INIT ("inodelk-trace", priv->inodelk_trace, bool, out);
+    GF_OPTION_INIT("pre-op-compat", priv->pre_op_compat, bool, out);
+    GF_OPTION_INIT("locking-scheme", locking_scheme, str, out);
+    priv->granular_locks = (strcmp(locking_scheme, "granular") == 0);
+    GF_OPTION_INIT("full-lock", priv->full_lock, bool, out);
+    GF_OPTION_INIT("granular-entry-heal", priv->esh_granular, bool, out);
 
-        GF_OPTION_INIT ("entrylk-trace", priv->entrylk_trace, bool, out);
+    GF_OPTION_INIT("eager-lock", priv->eager_lock, bool, out);
+    GF_OPTION_INIT("quorum-type", qtype, str, out);
+    GF_OPTION_INIT("quorum-count", priv->quorum_count, uint32, out);
+    GF_OPTION_INIT(AFR_SH_READDIR_SIZE_KEY, priv->sh_readdir_size, size_uint64,
+                   out);
+    fix_quorum_options(this, priv, qtype, this->options);
 
-        GF_OPTION_INIT ("pre-op-compat", priv->pre_op_compat, bool, out);
+    GF_OPTION_INIT("post-op-delay-secs", priv->post_op_delay_secs, uint32, out);
+    GF_OPTION_INIT("ensure-durability", priv->ensure_durability, bool, out);
 
-        GF_OPTION_INIT ("eager-lock", priv->eager_lock, bool, out);
-        GF_OPTION_INIT ("quorum-type", qtype, str, out);
-        GF_OPTION_INIT ("quorum-count", priv->quorum_count, uint32, out);
-        GF_OPTION_INIT (AFR_SH_READDIR_SIZE_KEY, priv->sh_readdir_size, size_uint64,
-                        out);
-        fix_quorum_options (this, priv, qtype, this->options);
+    GF_OPTION_INIT("self-heal-daemon", priv->shd.enabled, bool, out);
 
-	GF_OPTION_INIT ("post-op-delay-secs", priv->post_op_delay_secs, uint32, out);
-        GF_OPTION_INIT ("ensure-durability", priv->ensure_durability, bool,
-                        out);
+    GF_OPTION_INIT("iam-self-heal-daemon", priv->shd.iamshd, bool, out);
+    GF_OPTION_INIT("heal-timeout", priv->shd.timeout, int32, out);
 
-	GF_OPTION_INIT ("self-heal-daemon", priv->shd.enabled, bool, out);
+    GF_OPTION_INIT("consistent-metadata", priv->consistent_metadata, bool, out);
+    GF_OPTION_INIT("consistent-io", priv->consistent_io, bool, out);
+    afr_handle_anon_inode_options(priv, this->options);
 
-	GF_OPTION_INIT ("iam-self-heal-daemon", priv->shd.iamshd, bool, out);
-        GF_OPTION_INIT ("heal-timeout", priv->shd.timeout, int32, out);
+    GF_OPTION_INIT("use-anonymous-inode", priv->use_anon_inode, bool, out);
+    if (priv->quorum_count != 0)
+        priv->consistent_io = _gf_false;
 
-        GF_OPTION_INIT ("quorum-reads", priv->quorum_reads, bool, out);
-        GF_OPTION_INIT ("consistent-metadata", priv->consistent_metadata, bool,
-                        out);
+    priv->wait_count = 1;
 
-        priv->wait_count = 1;
+    priv->local = GF_CALLOC(sizeof(unsigned char), child_count, gf_afr_mt_char);
+    if (!priv->local) {
+        ret = -ENOMEM;
+        goto out;
+    }
 
-        priv->child_up = GF_CALLOC (sizeof (unsigned char), child_count,
-                                    gf_afr_mt_char);
-        if (!priv->child_up) {
-                ret = -ENOMEM;
-                goto out;
-        }
+    priv->anon_inode = GF_CALLOC(sizeof(unsigned char), child_count,
+                                 gf_afr_mt_char);
 
-        for (i = 0; i < child_count; i++)
-                priv->child_up[i] = -1; /* start with unknown state.
-                                           this initialization needed
-                                           for afr_notify() to work
-                                           reliably
-                                        */
+    priv->child_up = GF_CALLOC(sizeof(unsigned char), child_count,
+                               gf_afr_mt_char);
 
-        priv->children = GF_CALLOC (sizeof (xlator_t *), child_count,
-                                    gf_afr_mt_xlator_t);
-        if (!priv->children) {
-                ret = -ENOMEM;
-                goto out;
-        }
+    priv->child_latency = GF_MALLOC(sizeof(*priv->child_latency) * child_count,
+                                    gf_afr_mt_child_latency_t);
+    priv->halo_child_up = GF_CALLOC(sizeof(unsigned char), child_count,
+                                    gf_afr_mt_char);
 
-        priv->pending_key = GF_CALLOC (sizeof (*priv->pending_key),
-                                       child_count,
-                                       gf_afr_mt_char);
-        if (!priv->pending_key) {
-                ret = -ENOMEM;
-                goto out;
+    if (!priv->child_up || !priv->child_latency || !priv->halo_child_up ||
+        !priv->anon_inode) {
+        ret = -ENOMEM;
+        goto out;
+    }
+    /*Initialize to -ve ping timeout so that they are not considered
+     * in child-up events until ping-event comes*/
+    for (i = 0; i < child_count; i++)
+        priv->child_latency[i] = -1;
+
+    priv->children = GF_CALLOC(sizeof(xlator_t *), child_count,
+                               gf_afr_mt_xlator_t);
+    if (!priv->children) {
+        ret = -ENOMEM;
+        goto out;
+    }
+
+    ret = afr_pending_xattrs_init(priv, this);
+    if (ret)
+        goto out;
+
+    trav = this->children;
+    i = 0;
+    while (i < child_count) {
+        priv->children[i] = trav->xlator;
+        trav = trav->next;
+        i++;
+    }
+
+    ret = gf_asprintf(&priv->sh_domain, AFR_SH_DATA_DOMAIN_FMT, this->name);
+    if (-1 == ret) {
+        ret = -ENOMEM;
+        goto out;
+    }
+
+    priv->last_event = GF_CALLOC(child_count, sizeof(*priv->last_event),
+                                 gf_afr_mt_int32_t);
+    if (!priv->last_event) {
+        ret = -ENOMEM;
+        goto out;
+    }
+
+    this->itable = inode_table_new(SHD_INODE_LRU_LIMIT, this);
+    if (!this->itable) {
+        ret = -ENOMEM;
+        goto out;
+    }
+
+    if (priv->shd.iamshd) {
+        ret = afr_selfheal_daemon_init(this);
+        if (ret) {
+            ret = -ENOMEM;
+            goto out;
         }
+    }
 
-        trav = this->children;
-        i = 0;
-        while (i < child_count) {
-                priv->children[i] = trav->xlator;
-
-                ret = gf_asprintf (&priv->pending_key[i], "%s.%s",
-                                   AFR_XATTR_PREFIX,
-                                   trav->xlator->name);
-                if (-1 == ret) {
-                        ret = -ENOMEM;
-                        goto out;
-                }
-
-                trav = trav->next;
-                i++;
-        }
+    /* keep more local here as we may need them for self-heal etc */
+    this->local_pool = mem_pool_new(afr_local_t, 512);
+    if (!this->local_pool) {
+        ret = -1;
+        goto out;
+    }
 
-        ret = gf_asprintf (&priv->sh_domain, AFR_SH_DATA_DOMAIN_FMT,
-                           this->name);
-        if (-1 == ret) {
-                ret = -ENOMEM;
-                goto out;
-        }
+    priv->root_inode = NULL;
 
-        priv->last_event = GF_CALLOC (child_count, sizeof (*priv->last_event),
-                                      gf_afr_mt_int32_t);
-        if (!priv->last_event) {
-                ret = -ENOMEM;
-                goto out;
-        }
+    ret = 0;
+out:
+    return ret;
+}
+void
+afr_destroy_healer_object(xlator_t *this, struct subvol_healer *healer)
+{
+    int ret = -1;
+
+    if (!healer)
+        return;
+
+    if (healer->running) {
+        /*
+         * If there are any resources to cleanup, We need
+         * to do that gracefully using pthread_cleanup_push
+         */
+        ret = gf_thread_cleanup_xint(healer->thread);
+        if (ret)
+            gf_msg(this->name, GF_LOG_WARNING, 0, AFR_MSG_SELF_HEAL_FAILED,
+                   "Failed to clean up healer threads.");
+        healer->thread = 0;
+    }
+    pthread_cond_destroy(&healer->cond);
+    pthread_mutex_destroy(&healer->mutex);
+}
 
-	ret = afr_selfheal_daemon_init (this);
-	if (ret) {
-		ret = -ENOMEM;
-		goto out;
-	}
-
-        /* keep more local here as we may need them for self-heal etc */
-        this->local_pool = mem_pool_new (afr_local_t, 512);
-        if (!this->local_pool) {
-                ret = -1;
-                gf_log (this->name, GF_LOG_ERROR,
-                        "failed to create local_t's memory pool");
-                goto out;
-        }
+void
+afr_selfheal_daemon_fini(xlator_t *this)
+{
+    struct subvol_healer *healer = NULL;
+    afr_self_heald_t *shd = NULL;
+    afr_private_t *priv = NULL;
+    int i = 0;
+
+    priv = this->private;
+    if (!priv)
+        return;
+
+    shd = &priv->shd;
+    if (!shd->iamshd)
+        return;
+
+    for (i = 0; i < priv->child_count; i++) {
+        healer = &shd->index_healers[i];
+        afr_destroy_healer_object(this, healer);
+
+        healer = &shd->full_healers[i];
+        afr_destroy_healer_object(this, healer);
+
+        if (shd->statistics[i])
+            eh_destroy(shd->statistics[i]);
+    }
+    GF_FREE(shd->index_healers);
+    GF_FREE(shd->full_healers);
+    GF_FREE(shd->statistics);
+    if (shd->split_brain)
+        eh_destroy(shd->split_brain);
+}
+void
+fini(xlator_t *this)
+{
+    afr_private_t *priv = NULL;
 
-        priv->root_inode = NULL;
+    priv = this->private;
 
-        ret = 0;
-out:
-        return ret;
-}
+    afr_selfheal_daemon_fini(this);
+    GF_ASSERT(list_empty(&priv->saved_locks));
 
+    LOCK(&priv->lock);
+    if (priv->timer != NULL) {
+        gf_timer_call_cancel(this->ctx, priv->timer);
+        priv->timer = NULL;
+    }
+    UNLOCK(&priv->lock);
 
-int
-fini (xlator_t *this)
-{
-        afr_private_t *priv = NULL;
+    if (this->local_pool != NULL) {
+        mem_pool_destroy(this->local_pool);
+        this->local_pool = NULL;
+    }
 
-        priv = this->private;
-        this->private = NULL;
-        afr_priv_destroy (priv);
-        //if (this->itable);//I dont see any destroy func
+    this->private = NULL;
+    afr_priv_destroy(priv);
+    if (this->itable) {
+        inode_table_destroy(this->itable);
+        this->itable = NULL;
+    }
 
-        return 0;
+    return;
 }
 
-
 struct xlator_fops fops = {
-        .lookup      = afr_lookup,
-        .open        = afr_open,
-        .lk          = afr_lk,
-        .flush       = afr_flush,
-        .statfs      = afr_statfs,
-        .fsync       = afr_fsync,
-        .fsyncdir    = afr_fsyncdir,
-        .xattrop     = afr_xattrop,
-        .fxattrop    = afr_fxattrop,
-        .inodelk     = afr_inodelk,
-        .finodelk    = afr_finodelk,
-        .entrylk     = afr_entrylk,
-        .fentrylk    = afr_fentrylk,
-
-        /* inode read */
-        .access      = afr_access,
-        .stat        = afr_stat,
-        .fstat       = afr_fstat,
-        .readlink    = afr_readlink,
-        .getxattr    = afr_getxattr,
-        .fgetxattr   = afr_fgetxattr,
-        .readv       = afr_readv,
-
-        /* inode write */
-        .writev      = afr_writev,
-        .truncate    = afr_truncate,
-        .ftruncate   = afr_ftruncate,
-        .setxattr    = afr_setxattr,
-        .fsetxattr   = afr_fsetxattr,
-        .setattr     = afr_setattr,
-        .fsetattr    = afr_fsetattr,
-        .removexattr = afr_removexattr,
-        .fremovexattr = afr_fremovexattr,
-        .fallocate   = afr_fallocate,
-        .discard     = afr_discard,
-        .zerofill    = afr_zerofill,
-
-        /* dir read */
-        .opendir     = afr_opendir,
-        .readdir     = afr_readdir,
-        .readdirp    = afr_readdirp,
-
-        /* dir write */
-        .create      = afr_create,
-        .mknod       = afr_mknod,
-        .mkdir       = afr_mkdir,
-        .unlink      = afr_unlink,
-        .rmdir       = afr_rmdir,
-        .link        = afr_link,
-        .symlink     = afr_symlink,
-        .rename      = afr_rename,
+    .lookup = afr_lookup,
+    .lk = afr_lk,
+    .flush = afr_flush,
+    .statfs = afr_statfs,
+    .fsyncdir = afr_fsyncdir,
+    .inodelk = afr_inodelk,
+    .finodelk = afr_finodelk,
+    .entrylk = afr_entrylk,
+    .fentrylk = afr_fentrylk,
+    .ipc = afr_ipc,
+    .lease = afr_lease,
+
+    /* inode read */
+    .access = afr_access,
+    .stat = afr_stat,
+    .fstat = afr_fstat,
+    .readlink = afr_readlink,
+    .getxattr = afr_getxattr,
+    .fgetxattr = afr_fgetxattr,
+    .readv = afr_readv,
+    .seek = afr_seek,
+
+    /* inode write */
+    .writev = afr_writev,
+    .truncate = afr_truncate,
+    .ftruncate = afr_ftruncate,
+    .setxattr = afr_setxattr,
+    .fsetxattr = afr_fsetxattr,
+    .setattr = afr_setattr,
+    .fsetattr = afr_fsetattr,
+    .removexattr = afr_removexattr,
+    .fremovexattr = afr_fremovexattr,
+    .fallocate = afr_fallocate,
+    .discard = afr_discard,
+    .zerofill = afr_zerofill,
+    .xattrop = afr_xattrop,
+    .fxattrop = afr_fxattrop,
+    .fsync = afr_fsync,
+
+    /*inode open*/
+    .opendir = afr_opendir,
+    .open = afr_open,
+
+    /* dir read */
+    .readdir = afr_readdir,
+    .readdirp = afr_readdirp,
+
+    /* dir write */
+    .create = afr_create,
+    .mknod = afr_mknod,
+    .mkdir = afr_mkdir,
+    .unlink = afr_unlink,
+    .rmdir = afr_rmdir,
+    .link = afr_link,
+    .symlink = afr_symlink,
+    .rename = afr_rename,
 };
 
-
 struct xlator_dumpops dumpops = {
-        .priv       = afr_priv_dump,
+    .priv = afr_priv_dump,
 };
 
-
 struct xlator_cbks cbks = {
-        .release     = afr_release,
-        .releasedir  = afr_releasedir,
-        .forget      = afr_forget,
+    .release = afr_release,
+    .releasedir = afr_releasedir,
+    .forget = afr_forget,
 };
 
-
 struct volume_options options[] = {
-        { .key  = {"read-subvolume" },
-          .type = GF_OPTION_TYPE_XLATOR,
-          .description = "inode-read fops happen only on one of the bricks in "
-                         "replicate. Afr will prefer the one specified using "
-                         "this option if it is not stale. Option value must be "
-                         "one of the xlator names of the children. "
-                         "Ex: <volname>-client-0 till "
-                         "<volname>-client-<number-of-bricks - 1>"
-        },
-        { .key  = {"read-subvolume-index" },
-          .type = GF_OPTION_TYPE_INT,
-          .default_value = "-1",
-          .description = "inode-read fops happen only on one of the bricks in "
-                         "replicate. AFR will prefer the one specified using "
-                         "this option if it is not stale. allowed options"
-                         " include -1 till replica-count - 1"
-        },
-        { .key = {"read-hash-mode" },
-          .type = GF_OPTION_TYPE_INT,
-          .min = 0,
-          .max = 2,
-          .default_value = "1",
-          .description = "inode-read fops happen only on one of the bricks in "
-                         "replicate. AFR will prefer the one computed using "
-                         "the method specified using this option"
-                         "0 = first up server, "
-                         "1 = hash by GFID of file (all clients use "
-                                                    "same subvolume), "
-                         "2 = hash by GFID of file and client PID",
-        },
-        { .key  = {"choose-local" },
-          .type = GF_OPTION_TYPE_BOOL,
-          .default_value = "true",
-          .description = "Choose a local subvolume (i.e. Brick) to read from"
-	                 " if read-subvolume is not explicitly set.",
-        },
-        { .key  = {"favorite-child"},
-          .type = GF_OPTION_TYPE_XLATOR,
-          .description = "If a split-brain happens choose subvol/brick set by "
-                         "this option as source."
-        },
-        { .key  = {"background-self-heal-count"},
-          .type = GF_OPTION_TYPE_INT,
-          .min  = 0,
-          .default_value = "16",
-          .validate = GF_OPT_VALIDATE_MIN,
-          .description = "This specifies the number of self-heals that can be "
-                         " performed in background without blocking the fop"
-        },
-        { .key  = {"data-self-heal"},
-          .type = GF_OPTION_TYPE_STR,
-          .value = {"1", "on", "yes", "true", "enable",
-                    "0", "off", "no", "false", "disable",
-                    "open"},
-          .default_value = "on",
-          .description   = "Using this option we can enable/disable data "
-                           "self-heal on the file. \"open\" means data "
-                           "self-heal action will only be triggered by file "
-                           "open operations."
-        },
-        { .key  = {"data-self-heal-algorithm"},
-          .type = GF_OPTION_TYPE_STR,
-          .description   = "Select between \"full\", \"diff\". The "
-                           "\"full\" algorithm copies the entire file from "
-                           "source to sink. The \"diff\" algorithm copies to "
-                           "sink only those blocks whose checksums don't match "
-                           "with those of source. If no option is configured "
-                           "the option is chosen dynamically as follows: "
-                           "If the file does not exist on one of the sinks "
-                           "or empty file exists or if the source file size is "
-                           "about the same as page size the entire file will "
-                           "be read and written i.e \"full\" algo, "
-                           "otherwise \"diff\" algo is chosen.",
-          .value = { "diff", "full"}
-        },
-        { .key  = {"data-self-heal-window-size"},
-          .type = GF_OPTION_TYPE_INT,
-          .min  = 1,
-          .max  = 1024,
-          .default_value = "1",
-          .description = "Maximum number blocks per file for which self-heal "
-                         "process would be applied simultaneously."
-        },
-        { .key  = {"metadata-self-heal"},
-          .type = GF_OPTION_TYPE_BOOL,
-          .default_value = "on",
-          .description = "Using this option we can enable/disable metadata "
-                         "i.e. Permissions, ownerships, xattrs self-heal on "
-                         "the file/directory."
-        },
-        { .key  = {"entry-self-heal"},
-          .type = GF_OPTION_TYPE_BOOL,
-          .default_value = "on",
-          .description = "Using this option we can enable/disable entry "
-                         "self-heal on the directory."
-        },
-        { .key  = {"data-change-log"},
-          .type = GF_OPTION_TYPE_BOOL,
-          .default_value = "on",
-          .description = "Data fops like write/truncate will not perform "
-                         "pre/post fop changelog operations in afr transaction "
-                         "if this option is disabled"
-        },
-        { .key  = {"metadata-change-log"},
-          .type = GF_OPTION_TYPE_BOOL,
-          .default_value = "on",
-          .description = "Metadata fops like setattr/setxattr will not perform "
-                         "pre/post fop changelog operations in afr transaction "
-                         "if this option is disabled"
-        },
-        { .key  = {"entry-change-log"},
-          .type = GF_OPTION_TYPE_BOOL,
-          .default_value = "on",
-          .description = "Entry fops like create/unlink will not perform "
-                         "pre/post fop changelog operations in afr transaction "
-                         "if this option is disabled"
-        },
-        { .key  = {"optimistic-change-log"},
-          .type = GF_OPTION_TYPE_BOOL,
-          .default_value = "on",
-          .description = "Entry/Metadata fops will not perform "
-                         "pre fop changelog operations in afr transaction "
-                         "if this option is enabled."
-        },
-        { .key = {"inodelk-trace"},
-          .type = GF_OPTION_TYPE_BOOL,
-          .default_value = "off",
-          .description = "Enabling this option logs inode lock/unlocks"
-        },
-        { .key = {"entrylk-trace"},
-          .type = GF_OPTION_TYPE_BOOL,
-          .default_value = "off",
-          .description = "Enabling this option logs entry lock/unlocks"
-        },
-	{ .key = {"pre-op-compat"},
-	  .type = GF_OPTION_TYPE_BOOL,
-	  .default_value = "on",
-	  .description = "Use separate pre-op xattrop() FOP rather than "
-	                 "overloading xdata of the OP"
-	},
-        { .key = {"eager-lock"},
-          .type = GF_OPTION_TYPE_BOOL,
-          .default_value = "on",
-          .description = "Lock phase of a transaction has two sub-phases. "
-                         "First is an attempt to acquire locks in parallel by "
-                         "broadcasting non-blocking lock requests. If lock "
-                         "acquisition fails on any server, then the held locks "
-                         "are unlocked and revert to a blocking locked mode "
-                         "sequentially on one server after another.  If this "
-                         "option is enabled the initial broadcasting lock "
-                         "request attempt to acquire lock on the entire file. "
-                         "If this fails, we revert back to the sequential "
-                         "\"regional\" blocking lock as before. In the case "
-                         "where such an \"eager\" lock is granted in the "
-                         "non-blocking phase, it gives rise to an opportunity "
-                         "for optimization. i.e, if the next write transaction "
-                         "on the same FD arrives before the unlock phase of "
-                         "the first transaction, it \"takes over\" the full "
-                         "file lock. Similarly if yet another data transaction "
-                         "arrives before the unlock phase of the \"optimized\" "
-                         "transaction, that in turn \"takes over\" the lock as "
-                         "well. The actual unlock now happens at the end of "
-                         "the last \"optimized\" transaction."
-
-        },
-        { .key = {"self-heal-daemon"},
-          .type = GF_OPTION_TYPE_BOOL,
-          .default_value = "on",
-          .description = "This option applies to only self-heal-daemon. "
-                         "Index directory crawl and automatic healing of files "
-                         "will not be performed if this option is turned off."
-        },
-        { .key = {"iam-self-heal-daemon"},
-          .type = GF_OPTION_TYPE_BOOL,
-          .default_value = "off",
-          .description = "This option differentiates if the replicate "
-                         "translator is running as part of self-heal-daemon "
-                         "or not."
-        },
-        { .key = {"quorum-type"},
-          .type = GF_OPTION_TYPE_STR,
-          .value = { "none", "auto", "fixed"},
-          .default_value = "none",
-          .description = "If value is \"fixed\" only allow writes if "
-                         "quorum-count bricks are present.  If value is "
-                         "\"auto\" only allow writes if more than half of "
-                         "bricks, or exactly half including the first, are "
-                         "present.",
-        },
-        { .key = {"quorum-count"},
-          .type = GF_OPTION_TYPE_INT,
-          .min = 1,
-          .max = INT_MAX,
-          .default_value = 0,
-          .description = "If quorum-type is \"fixed\" only allow writes if "
-                         "this many bricks or present.  Other quorum types "
-                         "will OVERWRITE this value.",
-        },
-        { .key = {"quorum-reads"},
-          .type = GF_OPTION_TYPE_BOOL,
-          .default_value = "no",
-          .description = "If quorum-reads is \"true\" only allow reads if "
-                         "quorum is met when quorum is enabled.",
-        },
-        { .key  = {"node-uuid"},
-          .type = GF_OPTION_TYPE_STR,
-          .description = "Local glusterd uuid string, used in starting "
-                         "self-heal-daemon so that it can crawl only on "
-                         "local index directories.",
-        },
-        { .key  = {"post-op-delay-secs"},
-          .type = GF_OPTION_TYPE_INT,
-          .min  = 0,
-          .max  = INT_MAX,
-          .default_value = "1",
-          .description = "Time interval induced artificially before "
-	                 "post-operation phase of the transaction to "
-                         "enhance overlap of adjacent write operations.",
-        },
-        { .key = {AFR_SH_READDIR_SIZE_KEY},
-          .type = GF_OPTION_TYPE_SIZET,
-          .description = "readdirp size for performing entry self-heal",
-          .min = 1024,
-          .max = 131072,
-          .default_value = "1KB",
-        },
-        { .key = {"ensure-durability"},
-          .type = GF_OPTION_TYPE_BOOL,
-          .description = "Afr performs fsyncs for transactions if this "
-                         "option is on to make sure the changelogs/data is "
-                         "written to the disk",
-          .default_value = "on",
-        },
-	{ .key = {"afr-dirty-xattr"},
-	  .type = GF_OPTION_TYPE_STR,
-	  .default_value = AFR_DIRTY_DEFAULT,
-	},
-	{ .key = {"metadata-splitbrain-forced-heal"},
-	  .type = GF_OPTION_TYPE_BOOL,
-	  .default_value = "off",
-	},
-        { .key  = {"heal-timeout"},
-          .type = GF_OPTION_TYPE_INT,
-          .min  = 60,
-          .max  = INT_MAX,
-          .default_value = "600",
-          .description = "time interval for checking the need to self-heal "
-                         "in self-heal-daemon"
-        },
-        { .key = {"consistent-metadata"},
-          .type = GF_OPTION_TYPE_BOOL,
-          .default_value = "no",
-          .description = "If this option is enabled, readdirp will force "
-                         "lookups on those entries read whose read child is "
-                         "not the same as that of the parent. This will "
-                         "guarantee that all read operations on a file serve "
-                         "attributes from the same subvol as long as it holds "
-                         " a good copy of the file/dir.",
-        },
-        { .key  = {NULL} },
+    {.key = {"read-subvolume"},
+     .type = GF_OPTION_TYPE_XLATOR,
+     .op_version = {1},
+     .flags = OPT_FLAG_CLIENT_OPT | OPT_FLAG_SETTABLE | OPT_FLAG_DOC,
+     .tags = {"replicate"},
+     .description = "inode-read fops happen only on one of the bricks in "
+                    "replicate. Afr will prefer the one specified using "
+                    "this option if it is not stale. Option value must be "
+                    "one of the xlator names of the children. "
+                    "Ex: <volname>-client-0 till "
+                    "<volname>-client-<number-of-bricks - 1>"},
+    {.key = {"read-subvolume-index"},
+     .type = GF_OPTION_TYPE_INT,
+     .default_value = "-1",
+     .op_version = {2},
+     .flags = OPT_FLAG_CLIENT_OPT | OPT_FLAG_SETTABLE | OPT_FLAG_DOC,
+     .tags = {"replicate"},
+     .description = "inode-read fops happen only on one of the bricks in "
+                    "replicate. AFR will prefer the one specified using "
+                    "this option if it is not stale. allowed options"
+                    " include -1 till replica-count - 1"},
+    {.key = {"read-hash-mode"},
+     .type = GF_OPTION_TYPE_INT,
+     .min = 0,
+     .max = 5,
+     .default_value = "1",
+     .op_version = {2},
+     .flags = OPT_FLAG_CLIENT_OPT | OPT_FLAG_SETTABLE | OPT_FLAG_DOC,
+     .tags = {"replicate"},
+     .description =
+         "inode-read fops happen only on one of the bricks in "
+         "replicate. AFR will prefer the one computed using "
+         "the method specified using this option.\n"
+         "0 = first readable child of AFR, starting from 1st child.\n"
+         "1 = hash by GFID of file (all clients use "
+         "same subvolume).\n"
+         "2 = hash by GFID of file and client PID.\n"
+         "3 = brick having the least outstanding read requests.\n"
+         "4 = brick having the least network ping latency.\n"
+         "5 = Hybrid mode between 3 and 4, ie least value among "
+         "network-latency multiplied by outstanding-read-requests."},
+    {
+        .key = {"choose-local"},
+        .type = GF_OPTION_TYPE_BOOL,
+        .default_value = "true",
+        .op_version = {2},
+        .flags = OPT_FLAG_CLIENT_OPT | OPT_FLAG_SETTABLE | OPT_FLAG_DOC,
+        .tags = {"replicate"},
+        .description = "Choose a local subvolume (i.e. Brick) to read from"
+                       " if read-subvolume is not explicitly set.",
+    },
+    {.key = {"background-self-heal-count"},
+     .type = GF_OPTION_TYPE_INT,
+     .min = 0,
+     .max = 256,
+     .default_value = "8",
+     .validate = GF_OPT_VALIDATE_MIN,
+     .op_version = {1},
+     .flags = OPT_FLAG_CLIENT_OPT | OPT_FLAG_SETTABLE | OPT_FLAG_DOC,
+     .tags = {"replicate"},
+     .description = "This specifies the number of per client self-heal "
+                    "jobs that can perform parallel heals in the "
+                    "background."},
+    {.key = {"halo-shd-max-latency"},
+     .type = GF_OPTION_TYPE_INT,
+     .min = 1,
+     .max = 99999,
+     .default_value = "99999",
+     .op_version = {GD_OP_VERSION_3_11_0},
+     .flags = OPT_FLAG_CLIENT_OPT | OPT_FLAG_SETTABLE | OPT_FLAG_DOC,
+     .tags = {"replicate", "halo"},
+     .description = "Maximum latency for shd halo replication in msec."},
+    {.key = {"halo-enabled"},
+     .type = GF_OPTION_TYPE_BOOL,
+     .default_value = "False",
+     .op_version = {GD_OP_VERSION_3_11_0},
+     .flags = OPT_FLAG_CLIENT_OPT | OPT_FLAG_SETTABLE | OPT_FLAG_DOC,
+     .tags = {"replicate", "halo"},
+     .description = "Enable Halo (geo) replication mode."},
+    {.key = {"halo-nfsd-max-latency"},
+     .type = GF_OPTION_TYPE_INT,
+     .min = 1,
+     .max = 99999,
+     .default_value = "5",
+     .op_version = {GD_OP_VERSION_3_11_0},
+     .flags = OPT_FLAG_CLIENT_OPT | OPT_FLAG_SETTABLE | OPT_FLAG_DOC,
+     .tags = {"replicate", "halo"},
+     .description = "Maximum latency for nfsd halo replication in msec."},
+    {.key = {"halo-max-latency"},
+     .type = GF_OPTION_TYPE_INT,
+     .min = 1,
+     .max = AFR_HALO_MAX_LATENCY,
+     .default_value = "5",
+     .op_version = {GD_OP_VERSION_3_11_0},
+     .flags = OPT_FLAG_CLIENT_OPT | OPT_FLAG_SETTABLE | OPT_FLAG_DOC,
+     .tags = {"replicate", "halo"},
+     .description = "Maximum latency for halo replication in msec."},
+    {.key = {"halo-max-replicas"},
+     .type = GF_OPTION_TYPE_INT,
+     .min = 1,
+     .max = 99999,
+     .default_value = "99999",
+     .op_version = {GD_OP_VERSION_3_11_0},
+     .flags = OPT_FLAG_CLIENT_OPT | OPT_FLAG_SETTABLE | OPT_FLAG_DOC,
+     .tags = {"replicate", "halo"},
+     .description = "The maximum number of halo replicas; replicas"
+                    " beyond this value will be written asynchronously"
+                    "via the SHD."},
+    {.key = {"halo-min-replicas"},
+     .type = GF_OPTION_TYPE_INT,
+     .min = 1,
+     .max = 99999,
+     .default_value = "2",
+     .op_version = {GD_OP_VERSION_3_11_0},
+     .flags = OPT_FLAG_CLIENT_OPT | OPT_FLAG_SETTABLE | OPT_FLAG_DOC,
+     .tags = {"replicate", "halo"},
+     .description = "The minimmum number of halo replicas, before adding "
+                    "out of region replicas."},
+    {.key = {"heal-wait-queue-length"},
+     .type = GF_OPTION_TYPE_INT,
+     .min = 0,
+     .max = 10000, /*Around 100MB with sizeof(afr_local_t)= 10496 bytes*/
+     .default_value = "128",
+     .validate = GF_OPT_VALIDATE_MIN,
+     .op_version = {GD_OP_VERSION_3_7_10},
+     .flags = OPT_FLAG_CLIENT_OPT | OPT_FLAG_SETTABLE | OPT_FLAG_DOC,
+     .tags = {"replicate"},
+     .description = "This specifies the number of heals that can be queued"
+                    " for the parallel background self heal jobs."},
+    {.key = {"data-self-heal"},
+     .type = GF_OPTION_TYPE_STR,
+     .value = {"1", "on", "yes", "true", "enable", "0", "off", "no", "false",
+               "disable", "open"},
+     .default_value = "off",
+     .op_version = {1},
+     .flags = OPT_FLAG_CLIENT_OPT | OPT_FLAG_SETTABLE | OPT_FLAG_DOC,
+     .tags = {"replicate"},
+     .description = "Using this option we can enable/disable data "
+                    "self-heal on the file. \"open\" means data "
+                    "self-heal action will only be triggered by file "
+                    "open operations."},
+    {.key = {"data-self-heal-algorithm"},
+     .type = GF_OPTION_TYPE_STR,
+     .op_version = {1},
+     .flags = OPT_FLAG_CLIENT_OPT | OPT_FLAG_SETTABLE | OPT_FLAG_DOC,
+     .tags = {"replicate"},
+     .description = "Select between \"full\", \"diff\". The "
+                    "\"full\" algorithm copies the entire file from "
+                    "source to sink. The \"diff\" algorithm copies to "
+                    "sink only those blocks whose checksums don't match "
+                    "with those of source. If no option is configured "
+                    "the option is chosen dynamically as follows: "
+                    "If the file does not exist on one of the sinks "
+                    "or empty file exists or if the source file size is "
+                    "about the same as page size the entire file will "
+                    "be read and written i.e \"full\" algo, "
+                    "otherwise \"diff\" algo is chosen.",
+     .value = {"diff", "full"}},
+    {.key = {"data-self-heal-window-size"},
+     .type = GF_OPTION_TYPE_INT,
+     .min = 1,
+     .max = 1024,
+     .default_value = "1",
+     .op_version = {1},
+     .flags = OPT_FLAG_CLIENT_OPT | OPT_FLAG_SETTABLE | OPT_FLAG_DOC,
+     .tags = {"replicate"},
+     .description = "Maximum number blocks per file for which self-heal "
+                    "process would be applied simultaneously."},
+    {.key = {"metadata-self-heal"},
+     .type = GF_OPTION_TYPE_BOOL,
+     .default_value = "off",
+     .op_version = {1},
+     .flags = OPT_FLAG_CLIENT_OPT | OPT_FLAG_SETTABLE | OPT_FLAG_DOC,
+     .tags = {"replicate"},
+     /*.validate_fn = validate_replica*/
+     .description = "Using this option we can enable/disable metadata "
+                    "i.e. Permissions, ownerships, xattrs self-heal on "
+                    "the file/directory."},
+    {.key = {"entry-self-heal"},
+     .type = GF_OPTION_TYPE_BOOL,
+     .default_value = "off",
+     .op_version = {1},
+     .flags = OPT_FLAG_CLIENT_OPT | OPT_FLAG_SETTABLE | OPT_FLAG_DOC,
+     .tags = {"replicate"},
+     /*.validate_fn = validate_replica*/
+     .description = "Using this option we can enable/disable entry "
+                    "self-heal on the directory."},
+    {.key = {"data-change-log"},
+     .type = GF_OPTION_TYPE_BOOL,
+     .default_value = "on",
+     .op_version = {1},
+     .flags = OPT_FLAG_CLIENT_OPT | OPT_FLAG_SETTABLE | OPT_FLAG_DOC,
+     .tags = {"replicate"},
+     .description = "This option exists only for backward compatibility "
+                    "and configuring it doesn't have any effect"},
+    {.key = {"metadata-change-log"},
+     .type = GF_OPTION_TYPE_BOOL,
+     .default_value = "on",
+     .op_version = {1},
+     .flags = OPT_FLAG_CLIENT_OPT | OPT_FLAG_SETTABLE | OPT_FLAG_DOC,
+     .tags = {"replicate"},
+     .description = "This option exists only for backward compatibility "
+                    "and configuring it doesn't have any effect"},
+    {.key = {"entry-change-log"},
+     .type = GF_OPTION_TYPE_BOOL,
+     .default_value = "on",
+     .op_version = {1},
+     .flags = OPT_FLAG_CLIENT_OPT | OPT_FLAG_SETTABLE | OPT_FLAG_DOC,
+     .tags = {"replicate"},
+     .description = "This option exists only for backward compatibility "
+                    "and configuring it doesn't have any effect"},
+    {.key = {"optimistic-change-log"},
+     .type = GF_OPTION_TYPE_BOOL,
+     .default_value = "on",
+     .description = "Entry/Metadata fops will not perform "
+                    "pre fop changelog operations in afr transaction "
+                    "if this option is enabled."},
+    {.key = {"inodelk-trace"},
+     .type = GF_OPTION_TYPE_BOOL,
+     .default_value = "off",
+     .description = "Enabling this option logs inode lock/unlocks"},
+    {.key = {"entrylk-trace"},
+     .type = GF_OPTION_TYPE_BOOL,
+     .default_value = "off",
+     .description = "Enabling this option logs entry lock/unlocks"},
+    {.key = {"pre-op-compat"},
+     .type = GF_OPTION_TYPE_BOOL,
+     .default_value = "on",
+     .description = "Use separate pre-op xattrop() FOP rather than "
+                    "overloading xdata of the OP"},
+    {.key = {"eager-lock"},
+     .type = GF_OPTION_TYPE_BOOL,
+     .default_value = "on",
+     .op_version = {1},
+     .flags = OPT_FLAG_CLIENT_OPT | OPT_FLAG_SETTABLE | OPT_FLAG_DOC,
+     .tags = {"replicate"},
+     .description =
+         "Enable/Disable eager lock for replica volume. "
+         "Lock phase of a transaction has two sub-phases. "
+         "First is an attempt to acquire locks in parallel by "
+         "broadcasting non-blocking lock requests. If lock "
+         "acquisition fails on any server, then the held locks "
+         "are unlocked and we revert to a blocking locks mode "
+         "sequentially on one server after another.  If this "
+         "option is enabled the initial broadcasting lock "
+         "request attempts to acquire a full lock on the entire file. "
+         "If this fails, we revert back to the sequential "
+         "\"regional\" blocking locks as before. In the case "
+         "where such an \"eager\" lock is granted in the "
+         "non-blocking phase, it gives rise to an opportunity "
+         "for optimization. i.e, if the next write transaction "
+         "on the same FD arrives before the unlock phase of "
+         "the first transaction, it \"takes over\" the full "
+         "file lock. Similarly if yet another data transaction "
+         "arrives before the unlock phase of the \"optimized\" "
+         "transaction, that in turn \"takes over\" the lock as "
+         "well. The actual unlock now happens at the end of "
+         "the last \"optimized\" transaction."
+
+    },
+    {.key = {"self-heal-daemon"},
+     .type = GF_OPTION_TYPE_BOOL,
+     .default_value = "on",
+     .op_version = {1},
+     .flags = OPT_FLAG_CLIENT_OPT | OPT_FLAG_SETTABLE,
+     .tags = {"replicate"},
+     /*.validate_fn   = validate_replica_heal_enable_disable*/
+     .description = "This option applies to only self-heal-daemon. "
+                    "Index directory crawl and automatic healing of files "
+                    "will not be performed if this option is turned off."},
+    {.key = {"iam-self-heal-daemon"},
+     .type = GF_OPTION_TYPE_BOOL,
+     .default_value = "off",
+     .description = "This option differentiates if the replicate "
+                    "translator is running as part of self-heal-daemon "
+                    "or not."},
+    {.key = {"iam-nfs-daemon"},
+     .type = GF_OPTION_TYPE_BOOL,
+     .default_value = "off",
+     .description = "This option differentiates if the replicate "
+                    "translator is running as part of an NFS daemon "
+                    "or not."},
+    {
+        .key = {"quorum-type"},
+        .type = GF_OPTION_TYPE_STR,
+        .value = {"none", "auto", "fixed"},
+        .default_value = "none",
+        .op_version = {1},
+        .flags = OPT_FLAG_CLIENT_OPT | OPT_FLAG_SETTABLE | OPT_FLAG_DOC,
+        .tags = {"replicate"},
+        /*.option = quorum-type*/
+        .description = "If value is \"fixed\" only allow writes if "
+                       "quorum-count bricks are present.  If value is "
+                       "\"auto\" only allow writes if more than half of "
+                       "bricks, or exactly half including the first, are "
+                       "present.",
+    },
+    {
+        .key = {"quorum-count"},
+        .type = GF_OPTION_TYPE_INT,
+        .min = 1,
+        .max = INT_MAX,
+        .default_value = 0,
+        .op_version = {1},
+        .flags = OPT_FLAG_CLIENT_OPT | OPT_FLAG_SETTABLE | OPT_FLAG_DOC,
+        .tags = {"replicate"},
+        /*.option = quorum-count*/
+        /*.validate_fn = validate_quorum_count*/
+        .description = "If quorum-type is \"fixed\" only allow writes if "
+                       "this many bricks are present.  Other quorum types "
+                       "will OVERWRITE this value.",
+    },
+    {
+        .key = {"quorum-reads"},
+        .type = GF_OPTION_TYPE_BOOL,
+        .default_value = "no",
+        .op_version = {GD_OP_VERSION_3_7_0},
+        .flags = OPT_FLAG_CLIENT_OPT | OPT_FLAG_SETTABLE | OPT_FLAG_DOC,
+        .tags = {"replicate"},
+        .description = "This option has been removed. Reads are not allowed "
+                       "if quorum is not met.",
+    },
+    {
+        .key = {"node-uuid"},
+        .type = GF_OPTION_TYPE_STR,
+        .description = "Local glusterd uuid string, used in starting "
+                       "self-heal-daemon so that it can crawl only on "
+                       "local index directories.",
+    },
+    {
+        .key = {"post-op-delay-secs"},
+        .type = GF_OPTION_TYPE_INT,
+        .min = 0,
+        .max = INT_MAX,
+        .default_value = "1",
+        .op_version = {2},
+        .flags = OPT_FLAG_CLIENT_OPT | OPT_FLAG_SETTABLE | OPT_FLAG_DOC,
+        .tags = {"replicate"},
+        .description = "Time interval induced artificially before "
+                       "post-operation phase of the transaction to "
+                       "enhance overlap of adjacent write operations.",
+    },
+    {
+        .key = {AFR_SH_READDIR_SIZE_KEY},
+        .type = GF_OPTION_TYPE_SIZET,
+        .description = "readdirp size for performing entry self-heal",
+        .min = 1024,
+        .max = 131072,
+        .op_version = {2},
+        .flags = OPT_FLAG_CLIENT_OPT | OPT_FLAG_SETTABLE,
+        .tags = {"replicate"},
+        .default_value = "1KB",
+    },
+    {
+        .key = {"ensure-durability"},
+        .type = GF_OPTION_TYPE_BOOL,
+        .op_version = {3},
+        .flags = OPT_FLAG_CLIENT_OPT | OPT_FLAG_SETTABLE | OPT_FLAG_DOC,
+        .tags = {"replicate"},
+        .description = "Afr performs fsyncs for transactions if this "
+                       "option is on to make sure the changelogs/data is "
+                       "written to the disk",
+        .default_value = "on",
+    },
+    {
+        .key = {"afr-dirty-xattr"},
+        .type = GF_OPTION_TYPE_STR,
+        .default_value = AFR_DIRTY_DEFAULT,
+    },
+    {.key = {"afr-pending-xattr"},
+     .type = GF_OPTION_TYPE_STR,
+     .description = "Comma separated list of xattrs that are used to  "
+                    "capture information on pending heals."},
+    {
+        .key = {"metadata-splitbrain-forced-heal"},
+        .type = GF_OPTION_TYPE_BOOL,
+        .default_value = "off",
+    },
+    {.key = {"heal-timeout"},
+     .type = GF_OPTION_TYPE_INT,
+     .min = 5,
+     .max = INT_MAX,
+     .default_value = "600",
+     .op_version = {2},
+     .flags = OPT_FLAG_CLIENT_OPT | OPT_FLAG_SETTABLE | OPT_FLAG_DOC,
+     .tags = {"replicate"},
+     .description = "time interval for checking the need to self-heal "
+                    "in self-heal-daemon"},
+    {
+        .key = {"consistent-metadata"},
+        .type = GF_OPTION_TYPE_BOOL,
+        .default_value = "no",
+        .op_version = {GD_OP_VERSION_3_7_0},
+        .flags = OPT_FLAG_CLIENT_OPT | OPT_FLAG_SETTABLE | OPT_FLAG_DOC,
+        .tags = {"replicate"},
+        .description = "If this option is enabled, readdirp will force "
+                       "lookups on those entries read whose read child is "
+                       "not the same as that of the parent. This will "
+                       "guarantee that all read operations on a file serve "
+                       "attributes from the same subvol as long as it holds "
+                       " a good copy of the file/dir.",
+    },
+    {.key = {"arbiter-count"},
+     .type = GF_OPTION_TYPE_INT,
+     .description = "subset of child_count. Has to be 0 or 1."},
+    {
+        .key = {"thin-arbiter"},
+        .type = GF_OPTION_TYPE_STR,
+        .op_version = {GD_OP_VERSION_4_1_0},
+        .flags = OPT_FLAG_SETTABLE,
+        .tags = {"replicate"},
+        .description = "contains host:path of thin abriter brick",
+    },
+    {.key = {"shd-max-threads"},
+     .type = GF_OPTION_TYPE_INT,
+     .min = 1,
+     .max = 64,
+     .default_value = "1",
+     .op_version = {GD_OP_VERSION_3_7_12},
+     .flags = OPT_FLAG_CLIENT_OPT | OPT_FLAG_SETTABLE | OPT_FLAG_DOC,
+     .tags = {"replicate"},
+     .description = "Maximum number of parallel heals SHD can do per "
+                    "local brick. This can substantially lower heal times"
+                    ", but can also crush your bricks if you don't have "
+                    "the storage hardware to support this."},
+    {
+        .key = {"shd-wait-qlength"},
+        .type = GF_OPTION_TYPE_INT,
+        .min = 1,
+        .max = 655536,
+        .default_value = "1024",
+        .op_version = {GD_OP_VERSION_3_7_12},
+        .flags = OPT_FLAG_CLIENT_OPT | OPT_FLAG_SETTABLE | OPT_FLAG_DOC,
+        .tags = {"replicate"},
+        .description = "This option can be used to control number of heals"
+                       " that can wait in SHD per subvolume",
+    },
+    {
+        .key = {"locking-scheme"},
+        .type = GF_OPTION_TYPE_STR,
+        .value = {"full", "granular"},
+        .default_value = "full",
+        .op_version = {GD_OP_VERSION_3_7_12},
+        .flags = OPT_FLAG_CLIENT_OPT | OPT_FLAG_SETTABLE | OPT_FLAG_DOC,
+        .tags = {"replicate"},
+        .description = "If this option is set to granular, self-heal will "
+                       "stop being compatible with afr-v1, which helps afr "
+                       "be more granular while self-healing",
+    },
+    {.key = {"full-lock"},
+     .type = GF_OPTION_TYPE_BOOL,
+     .default_value = "yes",
+     .op_version = {GD_OP_VERSION_3_13_2},
+     .flags = OPT_FLAG_CLIENT_OPT | OPT_FLAG_SETTABLE,
+     .tags = {"replicate"},
+     .description = "If this option is disabled, then the IOs will take "
+                    "range locks same as versions till 3.13.1."},
+    {
+        .key = {"granular-entry-heal"},
+        .type = GF_OPTION_TYPE_BOOL,
+        .default_value = "no",
+        .op_version = {GD_OP_VERSION_3_8_0},
+        .flags = OPT_FLAG_CLIENT_OPT | OPT_FLAG_SETTABLE | OPT_FLAG_DOC,
+        .tags = {"replicate"},
+        .description = "If this option is enabled, self-heal will resort to "
+                       "granular way of recording changelogs and doing entry "
+                       "self-heal.",
+    },
+    {
+        .key = {"favorite-child-policy"},
+        .type = GF_OPTION_TYPE_STR,
+        .value = {"none", "size", "ctime", "mtime", "majority"},
+        .default_value = "none",
+        .op_version = {GD_OP_VERSION_3_7_12},
+        .flags = OPT_FLAG_CLIENT_OPT | OPT_FLAG_SETTABLE | OPT_FLAG_DOC,
+        .tags = {"replicate"},
+        .description = "This option can be used to automatically resolve "
+                       "split-brains using various policies without user "
+                       "intervention. \"size\" picks the file with the "
+                       "biggest size as the source. \"ctime\" and \"mtime\" "
+                       "pick the file with the latest ctime and mtime "
+                       "respectively as the source. \"majority\" picks a file"
+                       " with identical mtime and size in more than half the "
+                       "number of bricks in the replica.",
+    },
+    {
+        .key = {"consistent-io"},
+        .type = GF_OPTION_TYPE_BOOL,
+        .default_value = "no",
+        .description = "If this option is enabled, i/o will fail even if "
+                       "one of the bricks is down in the replicas",
+    },
+    {.key = {"use-compound-fops"},
+     .type = GF_OPTION_TYPE_BOOL,
+     .default_value = "no",
+     .op_version = {GD_OP_VERSION_3_8_4},
+     .flags = OPT_FLAG_CLIENT_OPT | OPT_FLAG_SETTABLE | OPT_FLAG_DOC,
+     .tags = {"replicate"},
+     .description = "This option exists only for backward compatibility "
+                    "and configuring it doesn't have any effect"},
+    {.key = {"use-anonymous-inode"},
+     .type = GF_OPTION_TYPE_BOOL,
+     .default_value = "no",
+     .op_version = {GD_OP_VERSION_8_0},
+     .flags = OPT_FLAG_CLIENT_OPT | OPT_FLAG_SETTABLE,
+     .tags = {"replicate"},
+     .description = "Setting this option heals directory renames efficiently"},
+
+    {.key = {NULL}},
+};
+
+xlator_api_t xlator_api = {
+    .init = init,
+    .fini = fini,
+    .notify = notify,
+    .reconfigure = reconfigure,
+    .mem_acct_init = mem_acct_init,
+    .op_version = {1}, /* Present from the initial version */
+    .dumpops = &dumpops,
+    .fops = &fops,
+    .cbks = &cbks,
+    .options = options,
+    .identifier = "replicate",
+    .category = GF_MAINTAINED,
 };
diff --git a/xlators/cluster/afr/src/afr.h b/xlators/cluster/afr/src/afr.h
index de000e765ea..d62f9a9caf2 100644
--- a/xlators/cluster/afr/src/afr.h
+++ b/xlators/cluster/afr/src/afr.h
@@ -8,161 +8,293 @@
   cases as published by the Free Software Foundation.
 */
 
-
 #ifndef __AFR_H__
 #define __AFR_H__
 
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
-
-#include "call-stub.h"
-#include "compat-errno.h"
+#include <glusterfs/call-stub.h>
+#include <glusterfs/compat-errno.h>
 #include "afr-mem-types.h"
 
 #include "libxlator.h"
-#include "timer.h"
-#include "syncop.h"
+#include <glusterfs/timer.h>
+#include <glusterfs/syncop.h>
 
 #include "afr-self-heald.h"
 #include "afr-messages.h"
 
-#define AFR_XATTR_PREFIX "trusted.afr"
+#define SHD_INODE_LRU_LIMIT 1
 #define AFR_PATHINFO_HEADER "REPLICATE:"
 #define AFR_SH_READDIR_SIZE_KEY "self-heal-readdir-size"
 #define AFR_SH_DATA_DOMAIN_FMT "%s:self-heal"
 #define AFR_DIRTY_DEFAULT AFR_XATTR_PREFIX ".dirty"
-#define AFR_DIRTY (((afr_private_t *) (THIS->private))->afr_dirty)
+#define AFR_DIRTY (((afr_private_t *)(THIS->private))->afr_dirty)
+
+#define AFR_LOCKEE_COUNT_MAX 3
+#define AFR_DOM_COUNT_MAX 3
+#define AFR_NUM_CHANGE_LOGS 3              /*data + metadata + entry*/
+#define AFR_DEFAULT_SPB_CHOICE_TIMEOUT 300 /*in seconds*/
+
+#define ARBITER_BRICK_INDEX 2
+#define THIN_ARBITER_BRICK_INDEX 2
+#define AFR_TA_DOM_NOTIFY "afr.ta.dom-notify"
+#define AFR_TA_DOM_MODIFY "afr.ta.dom-modify"
+
+#define AFR_LK_HEAL_DOM "afr.lock-heal.domain"
+
+#define AFR_HALO_MAX_LATENCY 99999
+#define AFR_ANON_DIR_PREFIX ".glusterfs-anonymous-inode"
+
+#define PFLAG_PENDING (1 << 0)
+#define PFLAG_SBRAIN (1 << 1)
+
+typedef int (*afr_lock_cbk_t)(call_frame_t *frame, xlator_t *this);
+
+typedef int (*afr_read_txn_wind_t)(call_frame_t *frame, xlator_t *this,
+                                   int subvol);
+
+typedef int (*afr_inode_refresh_cbk_t)(call_frame_t *frame, xlator_t *this,
+                                       int err);
+
+typedef int (*afr_changelog_resume_t)(call_frame_t *frame, xlator_t *this);
+
+#define AFR_COUNT(array, max)                                                  \
+    ({                                                                         \
+        int __i;                                                               \
+        int __res = 0;                                                         \
+        for (__i = 0; __i < max; __i++)                                        \
+            if (array[__i])                                                    \
+                __res++;                                                       \
+        __res;                                                                 \
+    })
+#define AFR_INTERSECT(dst, src1, src2, max)                                    \
+    ({                                                                         \
+        int __i;                                                               \
+        for (__i = 0; __i < max; __i++)                                        \
+            dst[__i] = src1[__i] && src2[__i];                                 \
+    })
+#define AFR_CMP(a1, a2, len)                                                   \
+    ({                                                                         \
+        int __cmp = 0;                                                         \
+        int __i;                                                               \
+        for (__i = 0; __i < len; __i++)                                        \
+            if (a1[__i] != a2[__i]) {                                          \
+                __cmp = 1;                                                     \
+                break;                                                         \
+            }                                                                  \
+        __cmp;                                                                 \
+    })
+#define AFR_IS_ARBITER_BRICK(priv, index)                                      \
+    ((priv->arbiter_count == 1) && (index == ARBITER_BRICK_INDEX))
+
+#define AFR_SET_ERROR_AND_CHECK_SPLIT_BRAIN(ret, errnum)                       \
+    do {                                                                       \
+        local->op_ret = ret;                                                   \
+        local->op_errno = errnum;                                              \
+        if (local->op_errno == EIO)                                            \
+            gf_msg(this->name, GF_LOG_ERROR, local->op_errno,                  \
+                   AFR_MSG_SPLIT_BRAIN,                                        \
+                   "Failing %s on gfid %s: "                                   \
+                   "split-brain observed.",                                    \
+                   gf_fop_list[local->op], uuid_utoa(local->inode->gfid));     \
+    } while (0)
+
+#define AFR_ERROR_OUT_IF_FDCTX_INVALID(__fd, __this, __error, __label)         \
+    do {                                                                       \
+        afr_fd_ctx_t *__fd_ctx = NULL;                                         \
+        __fd_ctx = afr_fd_ctx_get(__fd, __this);                               \
+        if (__fd_ctx && __fd_ctx->is_fd_bad) {                                 \
+            __error = EBADF;                                                   \
+            goto __label;                                                      \
+        }                                                                      \
+    } while (0)
 
-#define AFR_LOCKEE_COUNT_MAX    3
-#define AFR_DOM_COUNT_MAX    3
-#define AFR_NUM_CHANGE_LOGS            3 /*data + metadata + entry*/
+typedef enum {
+    AFR_READ_POLICY_FIRST_UP,
+    AFR_READ_POLICY_GFID_HASH,
+    AFR_READ_POLICY_GFID_PID_HASH,
+    AFR_READ_POLICY_LESS_LOAD,
+    AFR_READ_POLICY_LEAST_LATENCY,
+    AFR_READ_POLICY_LOAD_LATENCY_HYBRID,
+} afr_read_hash_mode_t;
 
-typedef int (*afr_lock_cbk_t) (call_frame_t *frame, xlator_t *this);
+typedef enum {
+    AFR_FAV_CHILD_NONE,
+    AFR_FAV_CHILD_BY_SIZE,
+    AFR_FAV_CHILD_BY_CTIME,
+    AFR_FAV_CHILD_BY_MTIME,
+    AFR_FAV_CHILD_BY_MAJORITY,
+    AFR_FAV_CHILD_POLICY_MAX,
+} afr_favorite_child_policy;
 
-typedef int (*afr_read_txn_wind_t) (call_frame_t *frame, xlator_t *this, int subvol);
+typedef enum {
+    AFR_SELFHEAL_DATA_FULL = 0,
+    AFR_SELFHEAL_DATA_DIFF,
+    AFR_SELFHEAL_DATA_DYNAMIC,
+} afr_data_self_heal_type_t;
 
-typedef int (*afr_inode_refresh_cbk_t) (call_frame_t *frame, xlator_t *this, int err);
+typedef enum {
+    AFR_CHILD_UNKNOWN = -1,
+    AFR_CHILD_ZERO,
+    AFR_CHILD_ONE,
+    AFR_CHILD_THIN_ARBITER,
+} afr_child_index;
 
-typedef int (*afr_changelog_resume_t) (call_frame_t *frame, xlator_t *this);
+typedef enum {
+    TA_WAIT_FOR_NOTIFY_LOCK_REL, /*FOP came after notify domain lock upcall
+                                   notification and waiting for its release.*/
+    TA_GET_INFO_FROM_TA_FILE,    /*FOP needs post-op on ta file to get
+                                  *info about which brick is bad.*/
+    TA_INFO_IN_MEMORY_SUCCESS,   /*Bad brick info is in memory and fop failed
+                                  *on BAD brick - Success*/
+    TA_INFO_IN_MEMORY_FAILED,    /*Bad brick info is in memory and fop failed
+                                  *on GOOD brick - Failed*/
+    TA_SUCCESS,                  /*FOP succeeded on both data bricks.*/
+} afr_ta_fop_state_t;
+
+struct afr_nfsd {
+    uint32_t halo_max_latency_msec;
+    gf_boolean_t iamnfsd;
+};
 
-#define alloca0(size) ({void *__ptr; __ptr = alloca(size); memset(__ptr, 0, size); __ptr;})
-#define AFR_COUNT(array,max) ({int __i; int __res = 0; for (__i = 0; __i < max; __i++) if (array[__i]) __res++; __res;})
-#define AFR_INTERSECT(dst,src1,src2,max) ({int __i; for (__i = 0; __i < max; __i++) dst[__i] = src1[__i] && src2[__i];})
-#define AFR_CMP(a1,a2,len) ({int __cmp = 0; int __i; for (__i = 0; __i < len; __i++) if (a1[__i] != a2[__i]) { __cmp = 1; break;} __cmp;})
+typedef struct _afr_lk_heal_info {
+    fd_t *fd;
+    int32_t cmd;
+    struct gf_flock flock;
+    dict_t *xdata_req;
+    unsigned char *locked_nodes;
+    struct list_head pos;
+    gf_lkowner_t lk_owner;
+    pid_t pid;
+    int32_t *child_up_event_gen;
+    int32_t *child_down_event_gen;
+} afr_lk_heal_info_t;
 
 typedef struct _afr_private {
-        gf_lock_t lock;               /* to guard access to child_count, etc */
-        unsigned int child_count;     /* total number of children   */
-
-        xlator_t **children;
-
-        inode_t *root_inode;
-
-        unsigned char *child_up;
-
-        char **pending_key;
-
-        char         *data_self_heal;              /* on/off/open */
-        char *       data_self_heal_algorithm;    /* name of algorithm */
-        unsigned int data_self_heal_window_size;  /* max number of pipelined
-                                                     read/writes */
-
-        unsigned int background_self_heal_count;
-        unsigned int background_self_heals_started;
-        gf_boolean_t metadata_self_heal;   /* on/off */
-        gf_boolean_t entry_self_heal;      /* on/off */
-
-        gf_boolean_t data_change_log;       /* on/off */
-        gf_boolean_t metadata_change_log;   /* on/off */
-        gf_boolean_t entry_change_log;      /* on/off */
-
-	gf_boolean_t metadata_splitbrain_forced_heal; /* on/off */
-        int read_child;               /* read-subvolume */
-        unsigned int hash_mode;       /* for when read_child is not set */
-        int favorite_child;  /* subvolume to be preferred in resolving
-                                         split-brain cases */
-
-        gf_boolean_t inodelk_trace;
-        gf_boolean_t entrylk_trace;
-
-        unsigned int wait_count;      /* # of servers to wait for success */
-
-        uint64_t up_count;      /* number of CHILD_UPs we have seen */
-        uint64_t down_count;    /* number of CHILD_DOWNs we have seen */
-
-        gf_boolean_t      optimistic_change_log;
-        gf_boolean_t      eager_lock;
-        gf_boolean_t      pre_op_compat;      /* on/off */
-	uint32_t          post_op_delay_secs;
-        unsigned int      quorum_count;
-        gf_boolean_t      quorum_reads;
-
-        char                   vol_uuid[UUID_SIZE + 1];
-        int32_t                *last_event;
-
-	/* @event_generation: Keeps count of number of events received which can
-	   potentially impact consistency decisions. The events are CHILD_UP
-	   and CHILD_DOWN, when we have to recalculate the freshness/staleness
-	   of copies to detect if changes had happened while the other server
-	   was down. CHILD_DOWN and CHILD_UP can also be received on network
-	   disconnect/reconnects and not necessarily server going down/up.
-	   Recalculating freshness/staleness on network events is equally
-	   important as we might have had a network split brain.
-	*/
-	uint32_t               event_generation;
-
-        gf_boolean_t           choose_local;
-        gf_boolean_t           did_discovery;
-        uint64_t               sh_readdir_size;
-        gf_boolean_t           ensure_durability;
-        char                   *sh_domain;
-	char                   *afr_dirty;
-
-	afr_self_heald_t       shd;
-
-	/* pump dependencies */
-	void                   *pump_private;
-	gf_boolean_t           use_afr_in_pump;
-        gf_boolean_t           consistent_metadata;
+    gf_lock_t lock;             /* to guard access to child_count, etc */
+    unsigned int child_count;   /* total number of children   */
+    unsigned int arbiter_count; /*subset of child_count.
+                                  Has to be 0 or 1.*/
+
+    xlator_t **children;
+
+    inode_t *root_inode;
+
+    int favorite_child; /* subvolume to be preferred in resolving
+                                    split-brain cases */
+    /* For thin-arbiter. */
+    uuid_t ta_gfid;
+    unsigned int thin_arbiter_count; /* 0 or 1 at the moment.*/
+    int ta_bad_child_index;
+    int ta_event_gen;
+    unsigned int ta_in_mem_txn_count;
+    unsigned int ta_on_wire_txn_count;
+    struct list_head ta_waitq;
+    struct list_head ta_onwireq;
+
+    unsigned char *anon_inode;
+    unsigned char *child_up;
+    unsigned char *halo_child_up;
+    int64_t *child_latency;
+    unsigned char *local;
+
+    char **pending_key;
+
+    afr_data_self_heal_type_t data_self_heal_algorithm;
+    unsigned int data_self_heal_window_size; /* max number of pipelined
+                                                read/writes */
+
+    struct list_head heal_waiting; /*queue for files that need heal*/
+    uint32_t heal_wait_qlen; /*configurable queue length for heal_waiting*/
+    int32_t heal_waiters;    /* No. of elements currently in wait queue.*/
+
+    struct list_head healing;            /* queue for files that are undergoing
+                                            background heal*/
+    uint32_t background_self_heal_count; /*configurable queue length for
+                                           healing queue*/
+    int32_t healers; /* No. of elements currently undergoing background
+                      heal*/
+
+    gf_boolean_t release_ta_notify_dom_lock;
+
+    gf_boolean_t metadata_self_heal; /* on/off */
+    gf_boolean_t entry_self_heal;    /* on/off */
+
+    gf_boolean_t metadata_splitbrain_forced_heal; /* on/off */
+    int read_child;                               /* read-subvolume */
+    gf_atomic_t *pending_reads; /*No. of pending read cbks per child.*/
+
+    gf_timer_t *timer; /* launched when parent up is received */
+
+    unsigned int wait_count; /* # of servers to wait for success */
+
+    unsigned char ta_child_up;
+    gf_boolean_t optimistic_change_log;
+    gf_boolean_t eager_lock;
+    gf_boolean_t pre_op_compat; /* on/off */
+    uint32_t post_op_delay_secs;
+    unsigned int quorum_count;
+
+    off_t ta_notify_dom_lock_offset;
+    afr_favorite_child_policy fav_child_policy; /*Policy to use for automatic
+                                              resolution of split-brains.*/
+    afr_read_hash_mode_t hash_mode; /* for when read_child is not set */
+
+    int32_t *last_event;
+
+    /* @event_generation: Keeps count of number of events received which can
+       potentially impact consistency decisions. The events are CHILD_UP
+       and CHILD_DOWN, when we have to recalculate the freshness/staleness
+       of copies to detect if changes had happened while the other server
+       was down. CHILD_DOWN and CHILD_UP can also be received on network
+       disconnect/reconnects and not necessarily server going down/up.
+       Recalculating freshness/staleness on network events is equally
+       important as we might have had a network split brain.
+    */
+    uint32_t event_generation;
+    char vol_uuid[UUID_SIZE + 1];
+
+    gf_boolean_t choose_local;
+    gf_boolean_t did_discovery;
+    gf_boolean_t ensure_durability;
+    gf_boolean_t halo_enabled;
+    gf_boolean_t consistent_metadata;
+    gf_boolean_t need_heal;
+    gf_boolean_t granular_locks;
+    uint64_t sh_readdir_size;
+    char *sh_domain;
+    char *afr_dirty;
+
+    uint64_t spb_choice_timeout;
+
+    afr_self_heald_t shd;
+    struct afr_nfsd nfsd;
+
+    uint32_t halo_max_latency_msec;
+    uint32_t halo_max_replicas;
+    uint32_t halo_min_replicas;
+
+    gf_boolean_t full_lock;
+    gf_boolean_t esh_granular;
+    gf_boolean_t consistent_io;
+    gf_boolean_t data_self_heal; /* on/off */
+    gf_boolean_t use_anon_inode;
+
+    /*For lock healing.*/
+    struct list_head saved_locks;
+    struct list_head lk_healq;
+
+    /*For anon-inode handling */
+    char anon_inode_name[NAME_MAX + 1];
+    char anon_gfid_str[UUID_SIZE + 1];
 } afr_private_t;
 
-
 typedef enum {
-        AFR_DATA_TRANSACTION,          /* truncate, write, ... */
-        AFR_METADATA_TRANSACTION,      /* chmod, chown, ... */
-        AFR_ENTRY_TRANSACTION,         /* create, rmdir, ... */
-        AFR_ENTRY_RENAME_TRANSACTION,  /* rename */
+    AFR_DATA_TRANSACTION,         /* truncate, write, ... */
+    AFR_METADATA_TRANSACTION,     /* chmod, chown, ... */
+    AFR_ENTRY_TRANSACTION,        /* create, rmdir, ... */
+    AFR_ENTRY_RENAME_TRANSACTION, /* rename */
 } afr_transaction_type;
 
-typedef enum {
-        AFR_TRANSACTION_LK,
-        AFR_SELFHEAL_LK,
-} transaction_lk_type_t;
-
-typedef enum {
-        AFR_LOCK_OP,
-        AFR_UNLOCK_OP,
-} afr_lock_op_type_t;
-
-typedef enum {
-        AFR_DATA_SELF_HEAL_LK,
-        AFR_METADATA_SELF_HEAL_LK,
-        AFR_ENTRY_SELF_HEAL_LK,
-}selfheal_lk_type_t;
-
-typedef enum {
-        AFR_INODELK_TRANSACTION,
-        AFR_INODELK_NB_TRANSACTION,
-        AFR_ENTRYLK_TRANSACTION,
-        AFR_ENTRYLK_NB_TRANSACTION,
-        AFR_INODELK_SELFHEAL,
-        AFR_INODELK_NB_SELFHEAL,
-        AFR_ENTRYLK_SELFHEAL,
-        AFR_ENTRYLK_NB_SELFHEAL,
-} afr_lock_call_type_t;
-
 /*
   xattr format: trusted.afr.volume = [x y z]
   x - data pending
@@ -171,821 +303,940 @@ typedef enum {
 */
 
 static inline int
-afr_index_for_transaction_type (afr_transaction_type type)
+afr_index_for_transaction_type(afr_transaction_type type)
 {
-        switch (type) {
-
+    switch (type) {
         case AFR_DATA_TRANSACTION:
-                return 0;
+            return 0;
 
         case AFR_METADATA_TRANSACTION:
-                return 1;
+            return 1;
 
         case AFR_ENTRY_TRANSACTION:
         case AFR_ENTRY_RENAME_TRANSACTION:
-                return 2;
-        }
+            return 2;
+    }
 
-        return -1;  /* make gcc happy */
+    return -1; /* make gcc happy */
 }
 
 static inline int
-afr_index_from_ia_type (ia_type_t type)
+afr_index_from_ia_type(ia_type_t type)
 {
-        switch (type) {
+    switch (type) {
         case IA_IFDIR:
-                return afr_index_for_transaction_type (AFR_ENTRY_TRANSACTION);
+            return afr_index_for_transaction_type(AFR_ENTRY_TRANSACTION);
         case IA_IFREG:
-                return afr_index_for_transaction_type (AFR_DATA_TRANSACTION);
-        default: return -1;
-        }
+            return afr_index_for_transaction_type(AFR_DATA_TRANSACTION);
+        default:
+            return -1;
+    }
 }
 
 typedef struct {
-        loc_t                   loc;
-        char                    *basename;
-        unsigned char           *locked_nodes;
-        int                     locked_count;
+    struct gf_flock flock;
+    loc_t loc;
+    fd_t *fd;
+    char *basename;
+    unsigned char *locked_nodes;
+    int locked_count;
 
-} afr_entry_lockee_t;
+} afr_lockee_t;
 
 int
-afr_entry_lockee_cmp (const void *l1, const void *l2);
+afr_entry_lockee_cmp(const void *l1, const void *l2);
 
 typedef struct {
-        char    *domain; /* Domain on which inodelk is taken */
-        struct gf_flock flock;
-        unsigned char *locked_nodes;
-        int32_t lock_count;
-} afr_inodelk_t;
+    loc_t *lk_loc;
 
-typedef struct {
-        loc_t *lk_loc;
-
-        int                     lockee_count;
-        afr_entry_lockee_t      lockee[AFR_LOCKEE_COUNT_MAX];
+    afr_lockee_t lockee[AFR_LOCKEE_COUNT_MAX];
 
-        afr_inodelk_t       inodelk[AFR_DOM_COUNT_MAX];
-        const char *lk_basename;
-        const char *lower_basename;
-        const char *higher_basename;
-        char lower_locked;
-        char higher_locked;
+    const char *lk_basename;
+    const char *lower_basename;
+    const char *higher_basename;
 
-        unsigned char *locked_nodes;
-        unsigned char *lower_locked_nodes;
+    unsigned char *lower_locked_nodes;
 
-        selfheal_lk_type_t selfheal_lk_type;
-        transaction_lk_type_t transaction_lk_type;
+    afr_lock_cbk_t lock_cbk;
 
-        int32_t lock_count;
-        int32_t entrylk_lock_count;
+    int lockee_count;
 
-        uint64_t lock_number;
-        int32_t lk_call_count;
-        int32_t lk_expected_count;
-        int32_t lk_attempted_count;
+    int32_t lk_call_count;
+    int32_t lk_expected_count;
+    int32_t lk_attempted_count;
 
-        int32_t lock_op_ret;
-        int32_t lock_op_errno;
-        afr_lock_cbk_t lock_cbk;
-        char *domain; /* Domain on which inode/entry lock/unlock in progress.*/
+    int32_t lock_op_ret;
+    int32_t lock_op_errno;
+    char *domain; /* Domain on which inode/entry lock/unlock in progress.*/
+    int32_t lock_count;
+    char lower_locked;
+    char higher_locked;
 } afr_internal_lock_t;
 
 struct afr_reply {
-	int	valid;
-	int32_t	op_ret;
-	int32_t	op_errno;
-	dict_t *xdata;
-	struct iatt poststat;
-	struct iatt postparent;
-	struct iatt prestat;
-	struct iatt preparent;
-	struct iatt preparent2;
-	struct iatt postparent2;
-	uint8_t checksum[MD5_DIGEST_LENGTH];
+    int valid;
+    int32_t op_ret;
+    dict_t *xattr; /*For xattrop*/
+    dict_t *xdata;
+    struct iatt poststat;
+    struct iatt postparent;
+    struct iatt prestat;
+    struct iatt preparent;
+    struct iatt preparent2;
+    struct iatt postparent2;
+    int32_t op_errno;
+    /* For rchecksum */
+    uint8_t checksum[SHA256_DIGEST_LENGTH];
+    gf_boolean_t buf_has_zeroes;
+    gf_boolean_t fips_mode_rchecksum;
+    /* For lookup */
+    int8_t need_heal;
 };
 
 typedef enum {
-        AFR_FD_NOT_OPENED,
-        AFR_FD_OPENED,
-        AFR_FD_OPENING
+    AFR_FD_NOT_OPENED,
+    AFR_FD_OPENED,
+    AFR_FD_OPENING
 } afr_fd_open_status_t;
 
 typedef struct {
-        unsigned int *pre_op_done[AFR_NUM_CHANGE_LOGS];
-	int inherited[AFR_NUM_CHANGE_LOGS];
-	int on_disk[AFR_NUM_CHANGE_LOGS];
-        afr_fd_open_status_t *opened_on; /* which subvolumes the fd is open on */
-
-        unsigned int *lock_piggyback;
-        unsigned int *lock_acquired;
-
-        int flags;
-
-	/* used for delayed-post-op optimization */
-	pthread_mutex_t    delay_lock;
-	gf_timer_t        *delay_timer;
-	call_frame_t      *delay_frame;
-
-	/* set if any write on this fd was a non stable write
-	   (i.e, without O_SYNC or O_DSYNC)
-	*/
-	gf_boolean_t      witnessed_unstable_write;
-
-	/* @open_fd_count:
-	   Number of open FDs queried from the server, as queried through
-	   xdata in FOPs. Currently, used to decide if eager-locking must be
-	   temporarily disabled.
-	*/
-        uint32_t        open_fd_count;
-
-
-	/* list of frames currently in progress */
-	struct list_head  eager_locked;
-
-	/* the subvolume on which the latest sequence of readdirs (starting
-	   at offset 0) has begun. Till the next readdir request with 0 offset
-	   arrives, we continue to read off this subvol.
-	*/
-	int readdir_subvol;
+    afr_fd_open_status_t *opened_on; /* which subvolumes the fd is open on */
+    int flags;
+
+    /* the subvolume on which the latest sequence of readdirs (starting
+       at offset 0) has begun. Till the next readdir request with 0 offset
+       arrives, we continue to read off this subvol.
+    */
+    int readdir_subvol;
+    /* lock-healing related members. */
+    gf_boolean_t is_fd_bad;
+    afr_lk_heal_info_t *lk_heal_info;
+
 } afr_fd_ctx_t;
 
+typedef enum {
+    AFR_FOP_LOCK_PARALLEL,
+    AFR_FOP_LOCK_SERIAL,
+    AFR_FOP_LOCK_QUORUM_FAILED,
+} afr_fop_lock_state_t;
+
+typedef struct _afr_inode_lock_t {
+    /* @num_inodelks:
+       Number of inodelks queried from the server, as queried through
+       xdata in FOPs. Currently, used to decide if eager-locking must be
+       temporarily disabled.
+    */
+    int32_t num_inodelks;
+    unsigned int event_generation;
+    gf_timer_t *delay_timer;
+    struct list_head owners;  /*Transactions that are performing fop*/
+    struct list_head post_op; /*Transactions that are done with the fop
+                               *So can not conflict with the fops*/
+    struct list_head waiting; /*Transaction that are waiting for
+                               *conflicting transactions to complete*/
+    struct list_head frozen;  /*Transactions that need to go as part of
+                               * next batch of eager-lock*/
+    gf_boolean_t release;
+    gf_boolean_t acquired;
+} afr_lock_t;
+
+typedef struct _afr_inode_ctx {
+    uint64_t read_subvol;
+    uint64_t write_subvol;
+    int lock_count;
+    int spb_choice;
+    gf_timer_t *timer;
+    unsigned int *pre_op_done[AFR_NUM_CHANGE_LOGS];
+    int inherited[AFR_NUM_CHANGE_LOGS];
+    int on_disk[AFR_NUM_CHANGE_LOGS];
+    /*Only 2 types of transactions support eager-locks now. DATA/METADATA*/
+    afr_lock_t lock[2];
+
+    /* @open_fd_count:
+       Number of open FDs queried from the server, as queried through
+       xdata in FOPs. Currently, used to decide if eager-locking must be
+       temporarily disabled.
+    */
+    uint32_t open_fd_count;
+    gf_boolean_t need_refresh;
+
+    /* set if any write on this fd was a non stable write
+       (i.e, without O_SYNC or O_DSYNC)
+    */
+    gf_boolean_t witnessed_unstable_write;
+} afr_inode_ctx_t;
 
 typedef struct _afr_local {
-	glusterfs_fop_t  op;
-        unsigned int call_count;
+    glusterfs_fop_t op;
+    unsigned int call_count;
 
-	/* @event_generation: copy of priv->event_generation taken at the
-	   time of starting the transaction. The copy is made so that we
-	   have a stable value through the various phases of the transaction.
-	*/
-	unsigned int event_generation;
+    /* @event_generation: copy of priv->event_generation taken at the
+       time of starting the transaction. The copy is made so that we
+       have a stable value through the various phases of the transaction.
+    */
+    unsigned int event_generation;
 
-        uint32_t     open_fd_count;
-        gf_boolean_t update_open_fd_count;
+    uint32_t open_fd_count;
+    int32_t num_inodelks;
 
-	gf_lkowner_t  saved_lk_owner;
+    int32_t op_ret;
+    int32_t op_errno;
 
-        int32_t op_ret;
-        int32_t op_errno;
+    int dirty[AFR_NUM_CHANGE_LOGS];
 
-        int32_t **pending;
+    int32_t **pending;
 
-	int dirty[AFR_NUM_CHANGE_LOGS];
+    loc_t loc;
+    loc_t newloc;
 
-        loc_t loc;
-        loc_t newloc;
+    fd_t *fd;
+    afr_fd_ctx_t *fd_ctx;
 
-        fd_t *fd;
-	afr_fd_ctx_t *fd_ctx;
+    /* @child_up: copy of priv->child_up taken at the time of transaction
+       start. The copy is taken so that we have a stable child_up array
+       through the phases of the transaction as priv->child_up[i] can keep
+       changing through time.
+    */
+    unsigned char *child_up;
 
-	/* @child_up: copy of priv->child_up taken at the time of transaction
-	   start. The copy is taken so that we have a stable child_up array
-	   through the phases of the transaction as priv->child_up[i] can keep
-	   changing through time.
-	*/
-        unsigned char *child_up;
+    /* @read_attempted:
+       array of flags representing subvolumes where read operations of
+       the read transaction have already been attempted. The array is
+       first pre-filled with down subvolumes, and as reads are performed
+       on other subvolumes, those are set as well. This way if the read
+       operation fails we do not retry on that subvolume again.
+    */
+    unsigned char *read_attempted;
 
-	/* @read_attempted:
-	   array of flags representing subvolumes where read operations of
-	   the read transaction have already been attempted. The array is
-	   first pre-filled with down subvolumes, and as reads are performed
-	   on other subvolumes, those are set as well. This way if the read
-	   operation fails we do not retry on that subvolume again.
-	*/
-	unsigned char *read_attempted;
+    /* @readfn:
 
-	/* @readfn:
+       pointer to function which will perform the read operation on a given
+       subvolume. Used in read transactions.
+    */
 
-	   pointer to function which will perform the read operation on a given
-	   subvolume. Used in read transactions.
-	*/
+    afr_read_txn_wind_t readfn;
 
-	afr_read_txn_wind_t readfn;
+    /* @inode:
 
-	/* @refreshed:
+       the inode on which the read txn is performed on. ref'ed and copied
+       from either fd->inode or loc.inode
+    */
 
-	   the inode was "refreshed" (i.e, pending xattrs from all subvols
-	   freshly inspected and inode ctx updated accordingly) as part of
-	   this transaction already.
-	*/
-	gf_boolean_t refreshed;
+    inode_t *inode;
 
-	/* @inode:
+    /* @parent[2]:
 
-	   the inode on which the read txn is performed on. ref'ed and copied
-	   from either fd->inode or loc.inode
-	*/
+       parent inode[s] on which directory transactions are performed.
+    */
 
-	inode_t *inode;
+    inode_t *parent;
+    inode_t *parent2;
 
-	/* @parent[2]:
+    /* @readable:
 
-	   parent inode[s] on which directory transactions are performed.
-	*/
+       array of flags representing servers from which a read can be
+       performed. This is the output of afr_inode_refresh()
+    */
+    unsigned char *readable;
+    unsigned char *readable2; /*For rename transaction*/
 
-	inode_t *parent;
-	inode_t *parent2;
+    afr_inode_refresh_cbk_t refreshfn;
 
-	/* @readable:
+    /* @refreshinode:
 
-	   array of flags representing servers from which a read can be
-	   performed. This is the output of afr_inode_refresh()
-	*/
-	unsigned char *readable;
+       Inode currently getting refreshed.
+    */
+    inode_t *refreshinode;
 
-	afr_inode_refresh_cbk_t refreshfn;
+    dict_t *xattr_req;
 
-	/* @refreshinode:
+    dict_t *dict;
 
-	   Inode currently getting refreshed.
-	*/
-	inode_t *refreshinode;
+    int read_subvol; /* Current read subvolume */
 
-	/*
-	  @pre_op_compat:
+    int optimistic_change_log;
 
-	  compatibility mode of pre-op. send a separate pre-op and
-	  op operations as part of transaction, rather than combining
-	*/
+    afr_internal_lock_t internal_lock;
 
-	gf_boolean_t pre_op_compat;
+    /*To handle setattr/setxattr on yet to be linked inode from dht*/
+    uuid_t refreshgfid;
 
-        dict_t  *xattr_req;
+    /* @refreshed:
 
-        afr_internal_lock_t internal_lock;
+       the inode was "refreshed" (i.e, pending xattrs from all subvols
+       freshly inspected and inode ctx updated accordingly) as part of
+       this transaction already.
+    */
+    gf_boolean_t refreshed;
 
-        dict_t  *dict;
+    gf_boolean_t update_num_inodelks;
+    gf_boolean_t update_open_fd_count;
 
-        int      optimistic_change_log;
-	gf_boolean_t      delayed_post_op;
+    /*
+      @pre_op_compat:
 
-	/* Is the current writev() going to perform a stable write?
-	   i.e, is fd->flags or @flags writev param have O_SYNC or
-	   O_DSYNC?
-	*/
-	gf_boolean_t      stable_write;
+      compatibility mode of pre-op. send a separate pre-op and
+      op operations as part of transaction, rather than combining
+    */
 
-	/* This write appended to the file. Nnot necessarily O_APPEND,
-	   just means the offset of write was at the end of file.
-	*/
-	gf_boolean_t      append_write;
+    gf_boolean_t pre_op_compat;
 
-        /*
-          This struct contains the arguments for the "continuation"
-          (scheme-like) of fops
-        */
+    /* Is the current writev() going to perform a stable write?
+       i.e, is fd->flags or @flags writev param have O_SYNC or
+       O_DSYNC?
+    */
+    gf_boolean_t stable_write;
+
+    /* This write appended to the file. Nnot necessarily O_APPEND,
+       just means the offset of write was at the end of file.
+    */
+    gf_boolean_t append_write;
+
+    /*
+      This struct contains the arguments for the "continuation"
+      (scheme-like) of fops
+    */
+
+    struct {
+        struct {
+            struct statvfs buf;
+            unsigned char buf_set;
+        } statfs;
+
+        struct {
+            fd_t *fd;
+            int32_t flags;
+        } open;
+
+        struct {
+            struct gf_flock user_flock;
+            struct gf_flock ret_flock;
+            unsigned char *locked_nodes;
+            int32_t cmd;
+            /*For lock healing only.*/
+            unsigned char *dom_locked_nodes;
+            int32_t *dom_lock_op_ret;
+            int32_t *dom_lock_op_errno;
+            struct gf_flock *getlk_rsp;
+        } lk;
+
+        /* inode read */
+
+        struct {
+            int32_t mask;
+            int last_index; /* index of the child we tried previously */
+        } access;
+
+        struct {
+            int last_index;
+        } stat;
+
+        struct {
+            int last_index;
+        } fstat;
+
+        struct {
+            size_t size;
+            int last_index;
+        } readlink;
+
+        struct {
+            char *name;
+            long xattr_len;
+            int last_index;
+        } getxattr;
+
+        struct {
+            size_t size;
+            off_t offset;
+            int last_index;
+            uint32_t flags;
+        } readv;
+
+        /* dir read */
 
         struct {
-                struct {
-                        gf_boolean_t needs_fresh_lookup;
-                        uuid_t gfid_req;
-                } lookup;
-
-                struct {
-                        unsigned char buf_set;
-                        struct statvfs buf;
-                } statfs;
-
-                struct {
-                        int32_t flags;
-                } open;
-
-                struct {
-                        int32_t cmd;
-                        struct gf_flock user_flock;
-                        struct gf_flock ret_flock;
-                        unsigned char *locked_nodes;
-                } lk;
-
-                /* inode read */
-
-                struct {
-                        int32_t mask;
-                        int last_index;  /* index of the child we tried previously */
-                } access;
-
-                struct {
-                        int last_index;
-                } stat;
-
-                struct {
-                        int last_index;
-                } fstat;
-
-                struct {
-                        size_t size;
-                        int last_index;
-                } readlink;
-
-                struct {
-                        char *name;
-                        int last_index;
-                        long xattr_len;
-                } getxattr;
-
-                struct {
-                        size_t size;
-                        off_t offset;
-                        int last_index;
-                        uint32_t flags;
-                } readv;
-
-                /* dir read */
-
-                struct {
-                        int success_count;
-                        int32_t op_ret;
-                        int32_t op_errno;
-
-                        uint32_t *checksum;
-                } opendir;
-
-                struct {
-                        int32_t op_ret;
-                        int32_t op_errno;
-                        size_t size;
-                        off_t offset;
-                        dict_t *dict;
-                        gf_boolean_t failed;
-                        int last_index;
-                } readdir;
-                /* inode write */
-
-                struct {
-                        struct iatt prebuf;
-                        struct iatt postbuf;
-                } inode_wfop; //common structure for all inode-write-fops
-
-                struct {
-                        int32_t op_ret;
-
-                        struct iovec *vector;
-                        struct iobref *iobref;
-                        int32_t count;
-                        off_t offset;
-                        uint32_t flags;
-                } writev;
-
-                struct {
-                        off_t offset;
-                } truncate;
-
-                struct {
-                        off_t offset;
-                } ftruncate;
-
-                struct {
-                        struct iatt in_buf;
-                        int32_t valid;
-                } setattr;
-
-                struct {
-                        struct iatt in_buf;
-                        int32_t valid;
-                } fsetattr;
-
-                struct {
-                        dict_t *dict;
-                        int32_t flags;
-                } setxattr;
-
-                struct {
-                        dict_t *dict;
-                        int32_t flags;
-                } fsetxattr;
-
-                struct {
-                        char *name;
-                } removexattr;
-
-                struct {
-                        dict_t *xattr;
-                } xattrop;
-
-                struct {
-                        dict_t *xattr;
-                } fxattrop;
-
-                /* dir write */
-
-                struct {
-                        inode_t *inode;
-                        struct iatt buf;
-                        struct iatt preparent;
-                        struct iatt postparent;
-                        struct iatt prenewparent;
-                        struct iatt postnewparent;
-                } dir_fop; //common structure for all dir fops
-
-                struct {
-                        fd_t *fd;
-                        dict_t *params;
-                        int32_t flags;
-                        mode_t mode;
-                } create;
-
-                struct {
-                        dev_t dev;
-                        mode_t mode;
-                        dict_t *params;
-                } mknod;
-
-                struct {
-                        int32_t mode;
-                        dict_t *params;
-                } mkdir;
-
-                struct {
-                        int flags;
-                } rmdir;
-
-                struct {
-                        dict_t *params;
-                        char *linkpath;
-                } symlink;
-
-		struct {
-			int32_t mode;
-			off_t offset;
-			size_t len;
-		} fallocate;
-
-		struct {
-			off_t offset;
-			size_t len;
-		} discard;
-
-                struct {
-                        off_t offset;
-                        off_t len;
-                        struct iatt prebuf;
-                        struct iatt postbuf;
-                } zerofill;
-
-                struct {
-                        char *volume;
-                        int32_t cmd;
-                        struct gf_flock flock;
-                } inodelk;
-
-        } cont;
+            uint32_t *checksum;
+            int success_count;
+            int32_t op_ret;
+            int32_t op_errno;
+        } opendir;
 
         struct {
-                off_t start, len;
+            int32_t op_ret;
+            int32_t op_errno;
+            size_t size;
+            off_t offset;
+            dict_t *dict;
+            int last_index;
+            gf_boolean_t failed;
+        } readdir;
+        /* inode write */
 
-                gf_boolean_t    eager_lock_on;
-                int *eager_lock;
+        struct {
+            struct iatt prebuf;
+            struct iatt postbuf;
+        } inode_wfop;  // common structure for all inode-write-fops
+
+        struct {
+            struct iovec *vector;
+            struct iobref *iobref;
+            off_t offset;
+            int32_t op_ret;
+            int32_t count;
+            uint32_t flags;
+        } writev;
+
+        struct {
+            off_t offset;
+        } truncate;
+
+        struct {
+            off_t offset;
+        } ftruncate;
+
+        struct {
+            struct iatt in_buf;
+            int32_t valid;
+        } setattr;
+
+        struct {
+            struct iatt in_buf;
+            int32_t valid;
+        } fsetattr;
+
+        struct {
+            dict_t *dict;
+            int32_t flags;
+        } setxattr;
+
+        struct {
+            dict_t *dict;
+            int32_t flags;
+        } fsetxattr;
+
+        struct {
+            char *name;
+        } removexattr;
+
+        struct {
+            dict_t *xattr;
+            gf_xattrop_flags_t optype;
+        } xattrop;
+
+        /* dir write */
+
+        struct {
+            inode_t *inode;
+            struct iatt buf;
+            struct iatt preparent;
+            struct iatt postparent;
+            struct iatt prenewparent;
+            struct iatt postnewparent;
+        } dir_fop;  // common structure for all dir fops
+
+        struct {
+            fd_t *fd;
+            dict_t *params;
+            int32_t flags;
+            mode_t mode;
+        } create;
 
-                char *basename;
-                char *new_basename;
+        struct {
+            dict_t *params;
+            dev_t dev;
+            mode_t mode;
+        } mknod;
+
+        struct {
+            dict_t *params;
+            int32_t mode;
+        } mkdir;
+
+        struct {
+            dict_t *params;
+            char *linkpath;
+        } symlink;
+
+        struct {
+            off_t offset;
+            size_t len;
+            int32_t mode;
+        } fallocate;
+
+        struct {
+            off_t offset;
+            size_t len;
+        } discard;
 
-                loc_t parent_loc;
-                loc_t new_parent_loc;
+        struct {
+            off_t offset;
+            off_t len;
+            struct iatt prebuf;
+            struct iatt postbuf;
+        } zerofill;
+
+        struct {
+            char *volume;
+            int32_t cmd;
+            int32_t in_cmd;
+            struct gf_flock in_flock;
+            struct gf_flock flock;
+            void *xdata;
+        } inodelk;
 
-                afr_transaction_type type;
+        struct {
+            char *volume;
+            char *basename;
+            void *xdata;
+            entrylk_cmd in_cmd;
+            entrylk_cmd cmd;
+            entrylk_type type;
+        } entrylk;
 
-		/* stub to resume on destruction
-		   of the transaction frame */
-		call_stub_t      *resume_stub;
+        struct {
+            off_t offset;
+            gf_seek_what_t what;
+        } seek;
 
-		struct list_head  eager_locked;
+        struct {
+            struct gf_lease user_lease;
+            struct gf_lease ret_lease;
+            unsigned char *locked_nodes;
+        } lease;
 
-                unsigned char   *pre_op;
+        struct {
+            int flags;
+        } rmdir;
 
-		/* @fop_subvols: subvolumes on which FOP will be attempted */
-                unsigned char   *fop_subvols;
+        struct {
+            int32_t datasync;
+        } fsync;
 
-		/* @failed_subvols: subvolumes on which FOP failed. Always
-		   a subset of @fop_subvols */
-                unsigned char   *failed_subvols;
+        struct {
+            uuid_t gfid_req;
+            gf_boolean_t needs_fresh_lookup;
+        } lookup;
 
-		/* @dirtied: flag which indicates whether we set dirty flag
-		   in the OP. Typically true when we are performing operation
-		   on more than one subvol and optimistic changelog is disabled
+    } cont;
 
-		   A 'true' value set in @dirtied flag means an 'undirtying'
-		   has to be done in POST-OP phase.
-		*/
-		gf_boolean_t  dirtied;
+    struct {
+        char *basename;
+        char *new_basename;
 
-		/* @inherited: flag which indicates that the dirty flags
-		   of the previous transaction were inherited
-		*/
-		gf_boolean_t  inherited;
+        loc_t parent_loc;
+        loc_t new_parent_loc;
 
-		/*
-		  @no_uninherit: flag which indicates that a pre_op_uninherit()
-		  must _not_ be attempted (and returned as failure) always. This
-		  flag is set when a hard pre-op is performed, but not accounted
-		  for it in fd_ctx->on_disk[]. Such transactions are "isolated"
-		  from the pre-op piggybacking entirely and therefore uninherit
-		  must not be attempted.
-		*/
-		gf_boolean_t no_uninherit;
+        /* stub to resume on destruction
+           of the transaction frame */
+        call_stub_t *resume_stub;
 
-		/* @uninherit_done:
-		   @uninherit_value:
+        struct list_head owner_list;
+        struct list_head wait_list;
 
-		   The above pair variables make pre_op_uninherit() idempotent.
-		   Both are FALSE initially. The first call to pre_op_uninherit
-		   sets @uninherit_done to TRUE and the return value to
-		   @uninherit_value. Further calls will check for @uninherit_done
-		   to be TRUE and if so will simply return @uninherit_value.
-		*/
-		gf_boolean_t uninherit_done;
-		gf_boolean_t uninherit_value;
+        unsigned char *pre_op;
 
-		/* @changelog_resume: function to be called after changlogging
-		   (either pre-op or post-op) is done
-		*/
+        /* Changelog xattr dict for [f]xattrop*/
+        dict_t **changelog_xdata;
+        unsigned char *pre_op_sources;
 
-		afr_changelog_resume_t changelog_resume;
+        /* @failed_subvols: subvolumes on which a pre-op or a
+            FOP failed. */
+        unsigned char *failed_subvols;
 
-                call_frame_t *main_frame;
+        call_frame_t *main_frame; /*Fop frame*/
+        call_frame_t *frame;      /*Transaction frame*/
 
-                int (*wind) (call_frame_t *frame, xlator_t *this, int subvol);
+        int (*wind)(call_frame_t *frame, xlator_t *this, int subvol);
 
-                int (*fop) (call_frame_t *frame, xlator_t *this);
+        int (*unwind)(call_frame_t *frame, xlator_t *this);
 
-                int (*done) (call_frame_t *frame, xlator_t *this);
+        off_t start, len;
 
-                int (*resume) (call_frame_t *frame, xlator_t *this);
+        afr_transaction_type type;
 
-                int (*unwind) (call_frame_t *frame, xlator_t *this);
+        int32_t in_flight_sb_errno; /* This is where the cause of the
+                                       failure on the last good copy of
+                                       the file is stored.
+                                       */
 
-                /* post-op hook */
-        } transaction;
+        /* @changelog_resume: function to be called after changlogging
+           (either pre-op or post-op) is done
+        */
+        afr_changelog_resume_t changelog_resume;
 
-	syncbarrier_t barrier;
+        gf_boolean_t eager_lock_on;
+        gf_boolean_t do_eager_unlock;
 
-        /* extra data for fops */
-        dict_t         *xdata_req;
-        dict_t         *xdata_rsp;
+        /* @dirtied: flag which indicates whether we set dirty flag
+           in the OP. Typically true when we are performing operation
+           on more than one subvol and optimistic changelog is disabled
 
-        mode_t          umask;
-        int             xflag;
-        gf_boolean_t    do_discovery;
-	struct afr_reply *replies;
-} afr_local_t;
+           A 'true' value set in @dirtied flag means an 'undirtying'
+           has to be done in POST-OP phase.
+        */
+        gf_boolean_t dirtied;
 
+        /* @inherited: flag which indicates that the dirty flags
+           of the previous transaction were inherited
+        */
+        gf_boolean_t inherited;
 
-typedef struct _afr_inode_ctx {
-        uint64_t        read_subvol;
-        int             spb_choice;
-} afr_inode_ctx_t;
+        /*
+          @no_uninherit: flag which indicates that a pre_op_uninherit()
+          must _not_ be attempted (and returned as failure) always. This
+          flag is set when a hard pre-op is performed, but not accounted
+          for it in fd_ctx->on_disk[]. Such transactions are "isolated"
+          from the pre-op piggybacking entirely and therefore uninherit
+          must not be attempted.
+        */
+        gf_boolean_t no_uninherit;
+
+        gf_boolean_t in_flight_sb; /* Indicator for occurrence of
+                                      split-brain while in the middle of
+                                      a txn. */
+
+        /* @uninherit_done:
+           @uninherit_value:
+
+           The above pair variables make pre_op_uninherit() idempotent.
+           Both are FALSE initially. The first call to pre_op_uninherit
+           sets @uninherit_done to TRUE and the return value to
+           @uninherit_value. Further calls will check for @uninherit_done
+           to be TRUE and if so will simply return @uninherit_value.
+        */
+        gf_boolean_t uninherit_done;
+        gf_boolean_t uninherit_value;
+
+        gf_boolean_t disable_delayed_post_op;
+    } transaction;
+
+    syncbarrier_t barrier;
+
+    /* extra data for fops */
+    dict_t *xdata_req;
+    dict_t *xdata_rsp;
+
+    dict_t *xattr_rsp; /*for [f]xattrop*/
+
+    mode_t umask;
+    int xflag;
+    struct afr_reply *replies;
+
+    /* For  client side background heals. */
+    struct list_head healer;
+    call_frame_t *heal_frame;
+
+    afr_inode_ctx_t *inode_ctx;
+
+    /*For thin-arbiter transactions.*/
+    int ta_failed_subvol;
+    int ta_event_gen;
+    struct list_head ta_waitq;
+    struct list_head ta_onwireq;
+    afr_ta_fop_state_t fop_state;
+    afr_fop_lock_state_t fop_lock_state;
+    gf_lkowner_t saved_lk_owner;
+    unsigned char read_txn_query_child;
+    unsigned char ta_child_up;
+    gf_boolean_t do_discovery;
+    gf_boolean_t need_full_crawl;
+    gf_boolean_t is_read_txn;
+    gf_boolean_t is_new_entry;
+} afr_local_t;
 
-/* did a call fail due to a child failing? */
-#define child_went_down(op_ret, op_errno) (((op_ret) < 0) &&            \
-                                           ((op_errno == ENOTCONN) ||   \
-                                            (op_errno == EBADFD)))
+typedef struct afr_spbc_timeout {
+    call_frame_t *frame;
+    loc_t *loc;
+    int spb_child_index;
+    gf_boolean_t d_spb;
+    gf_boolean_t m_spb;
+} afr_spbc_timeout_t;
+
+typedef struct afr_spb_status {
+    call_frame_t *frame;
+    loc_t *loc;
+} afr_spb_status_t;
+
+typedef struct afr_empty_brick_args {
+    call_frame_t *frame;
+    char *op_type;
+    loc_t loc;
+    int empty_index;
+} afr_empty_brick_args_t;
+
+typedef struct afr_read_subvol_args {
+    ia_type_t ia_type;
+    uuid_t gfid;
+} afr_read_subvol_args_t;
+
+typedef struct afr_granular_esh_args {
+    fd_t *heal_fd;
+    xlator_t *xl;
+    call_frame_t *frame;
+    gf_boolean_t mismatch; /* flag to represent occurrence of type/gfid
+                              mismatch */
+} afr_granular_esh_args_t;
 
 int
-afr_inode_read_subvol_get (inode_t *inode, xlator_t *this,
-			   unsigned char *data_subvols,
-			   unsigned char *metadata_subvols,
-			   int *event_generation);
+afr_inode_get_readable(call_frame_t *frame, inode_t *inode, xlator_t *this,
+                       unsigned char *readable, int *event_p, int type);
+int
+afr_inode_read_subvol_get(inode_t *inode, xlator_t *this,
+                          unsigned char *data_subvols,
+                          unsigned char *metadata_subvols,
+                          int *event_generation);
 int
-__afr_inode_read_subvol_get (inode_t *inode, xlator_t *this,
-			     unsigned char *data_subvols,
-			     unsigned char *metadata_subvols,
-			     int *event_generation);
+__afr_inode_read_subvol_get(inode_t *inode, xlator_t *this,
+                            unsigned char *data_subvols,
+                            unsigned char *metadata_subvols,
+                            int *event_generation);
 
 int
-__afr_inode_read_subvol_set (inode_t *inode, xlator_t *this,
-			     unsigned char *data_subvols,
-			     unsigned char *metadata_subvol,
-			     int event_generation);
+__afr_inode_read_subvol_set(inode_t *inode, xlator_t *this,
+                            unsigned char *data_subvols,
+                            unsigned char *metadata_subvol,
+                            int event_generation);
+int
+afr_inode_read_subvol_set(inode_t *inode, xlator_t *this,
+                          unsigned char *data_subvols,
+                          unsigned char *metadata_subvols,
+                          int event_generation);
+
 int
-afr_inode_read_subvol_set (inode_t *inode, xlator_t *this,
-			   unsigned char *data_subvols,
-			   unsigned char *metadata_subvols,
-			   int event_generation);
+__afr_inode_need_refresh_set(inode_t *inode, xlator_t *this);
 
 int
-afr_inode_read_subvol_reset (inode_t *inode, xlator_t *this);
+afr_inode_need_refresh_set(inode_t *inode, xlator_t *this);
 
 int
-afr_read_subvol_select_by_policy (inode_t *inode, xlator_t *this,
-				  unsigned char *readable);
+afr_read_subvol_select_by_policy(inode_t *inode, xlator_t *this,
+                                 unsigned char *readable,
+                                 afr_read_subvol_args_t *args);
 
 int
-afr_inode_read_subvol_type_get (inode_t *inode, xlator_t *this,
-				unsigned char *readable, int *event_p,
-				int type);
+afr_inode_read_subvol_type_get(inode_t *inode, xlator_t *this,
+                               unsigned char *readable, int *event_p, int type);
 int
-afr_read_subvol_get (inode_t *inode, xlator_t *this, int *subvol_p,
-		     int *event_p, afr_transaction_type type);
+afr_read_subvol_get(inode_t *inode, xlator_t *this, int *subvol_p,
+                    unsigned char *readables, int *event_p,
+                    afr_transaction_type type, afr_read_subvol_args_t *args);
 
-#define afr_data_subvol_get(i, t, s, e) \
-	afr_read_subvol_get(i, t, s, e, AFR_DATA_TRANSACTION)
+#define afr_data_subvol_get(i, t, s, r, e, a)                                  \
+    afr_read_subvol_get(i, t, s, r, e, AFR_DATA_TRANSACTION, a)
 
-#define afr_metadata_subvol_get(i, t, s, e) \
-	afr_read_subvol_get(i, t, s, e, AFR_METADATA_TRANSACTION)
+#define afr_metadata_subvol_get(i, t, s, r, e, a)                              \
+    afr_read_subvol_get(i, t, s, r, e, AFR_METADATA_TRANSACTION, a)
 
 int
-afr_inode_refresh (call_frame_t *frame, xlator_t *this, inode_t *inode,
-		   afr_inode_refresh_cbk_t cbk);
+afr_inode_refresh(call_frame_t *frame, xlator_t *this, inode_t *inode,
+                  uuid_t gfid, afr_inode_refresh_cbk_t cbk);
 
 int32_t
-afr_notify (xlator_t *this, int32_t event, void *data, void *data2);
+afr_notify(xlator_t *this, int32_t event, void *data, void *data2);
+
+int
+xattr_is_equal(dict_t *this, char *key1, data_t *value1, void *data);
 
 int
-xattr_is_equal (dict_t *this, char *key1, data_t *value1, void *data);
+afr_add_entry_lockee(afr_local_t *local, loc_t *loc, char *basename,
+                     int child_count);
 
 int
-afr_init_entry_lockee (afr_entry_lockee_t *lockee, afr_local_t *local,
-                       loc_t *loc, char *basename, int child_count);
+afr_add_inode_lockee(afr_local_t *local, int child_count);
 
 void
-afr_entry_lockee_cleanup (afr_internal_lock_t *int_lock);
+afr_lockees_cleanup(afr_internal_lock_t *int_lock);
 
 int
-afr_attempt_lock_recovery (xlator_t *this, int32_t child_index);
+afr_attempt_lock_recovery(xlator_t *this, int32_t child_index);
 
 int
-afr_mark_locked_nodes (xlator_t *this, fd_t *fd,
-                       unsigned char *locked_nodes);
+afr_mark_locked_nodes(xlator_t *this, fd_t *fd, unsigned char *locked_nodes);
 
 void
-afr_set_lk_owner (call_frame_t *frame, xlator_t *this, void *lk_owner);
+afr_set_lk_owner(call_frame_t *frame, xlator_t *this, void *lk_owner);
 
 int
-afr_set_lock_number (call_frame_t *frame, xlator_t *this);
+afr_set_lock_number(call_frame_t *frame, xlator_t *this);
 
 int32_t
-afr_unlock (call_frame_t *frame, xlator_t *this);
+afr_unlock(call_frame_t *frame, xlator_t *this);
 
 int
-afr_nonblocking_entrylk (call_frame_t *frame, xlator_t *this);
+afr_lock_nonblocking(call_frame_t *frame, xlator_t *this);
 
 int
-afr_nonblocking_inodelk (call_frame_t *frame, xlator_t *this);
+afr_blocking_lock(call_frame_t *frame, xlator_t *this);
 
 int
-afr_blocking_lock (call_frame_t *frame, xlator_t *this);
+afr_internal_lock_finish(call_frame_t *frame, xlator_t *this);
 
 int
-afr_internal_lock_finish (call_frame_t *frame, xlator_t *this);
-
-int
-afr_lk_transfer_datalock (call_frame_t *dst, call_frame_t *src, char *dom,
-                          unsigned int child_count);
-
-int
-__afr_fd_ctx_set (xlator_t *this, fd_t *fd);
-
-int
-afr_fd_ctx_set (xlator_t *this, fd_t *fd);
+__afr_fd_ctx_set(xlator_t *this, fd_t *fd);
 
 afr_fd_ctx_t *
-afr_fd_ctx_get (fd_t *fd, xlator_t *this);
+afr_fd_ctx_get(fd_t *fd, xlator_t *this);
 
 int
-afr_build_parent_loc (loc_t *parent, loc_t *child, int32_t *op_errno);
+afr_build_parent_loc(loc_t *parent, loc_t *child, int32_t *op_errno);
 
 int
-afr_locked_nodes_count (unsigned char *locked_nodes, int child_count);
+afr_locked_nodes_count(unsigned char *locked_nodes, int child_count);
 
 int
-afr_replies_interpret (call_frame_t *frame, xlator_t *this, inode_t *inode);
+afr_replies_interpret(call_frame_t *frame, xlator_t *this, inode_t *inode,
+                      gf_boolean_t *start_heal);
 
 void
-afr_local_replies_wipe (afr_local_t *local, afr_private_t *priv);
+afr_local_replies_wipe(afr_local_t *local, afr_private_t *priv);
 
 void
-afr_local_cleanup (afr_local_t *local, xlator_t *this);
+afr_local_cleanup(afr_local_t *local, xlator_t *this);
 
 int
-afr_frame_return (call_frame_t *frame);
+afr_frame_return(call_frame_t *frame);
 
 int
-afr_open (call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t flags,
-          fd_t *fd, dict_t *xdata);
+afr_open(call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t flags,
+         fd_t *fd, dict_t *xdata);
 
 void
-afr_local_transaction_cleanup (afr_local_t *local, xlator_t *this);
+afr_local_transaction_cleanup(afr_local_t *local, xlator_t *this);
 
 int
-afr_cleanup_fd_ctx (xlator_t *this, fd_t *fd);
-
-#define AFR_STACK_UNWIND(fop, frame, params ...)                \
-        do {                                                    \
-                afr_local_t *__local = NULL;                    \
-                xlator_t    *__this = NULL;                     \
-                if (frame) {                                    \
-                        __local = frame->local;                 \
-                        __this = frame->this;                   \
-                        frame->local = NULL;                    \
-                }                                               \
-                STACK_UNWIND_STRICT (fop, frame, params);       \
-                if (__local) {                                  \
-                        afr_local_cleanup (__local, __this);    \
-                        mem_put (__local);                      \
-                }                                               \
-        } while (0)
-
-#define AFR_STACK_DESTROY(frame)                                \
-        do {                                                    \
-                afr_local_t *__local = NULL;                    \
-                xlator_t    *__this = NULL;                     \
-                __local = frame->local;                         \
-                __this = frame->this;                           \
-                frame->local = NULL;                            \
-                STACK_DESTROY (frame->root);                    \
-                if (__local) {                                  \
-                        afr_local_cleanup (__local, __this);    \
-                        mem_put (__local);                      \
-                }                                               \
-        } while (0);
-
-#define AFR_FRAME_INIT(frame, op_errno)				       \
-	({frame->local = mem_get0 (THIS->local_pool);		       \
-	if (afr_local_init (frame->local, THIS->private, &op_errno)) { \
-		afr_local_cleanup (frame->local, THIS);		       \
-		mem_put (frame->local);				       \
-		frame->local = NULL; };				       \
-	frame->local;})
-
-#define AFR_STACK_RESET(frame)                                         \
-        do {                                                           \
-                afr_local_t *__local = NULL;                           \
-                xlator_t    *__this = NULL;                            \
-                __local = frame->local;                                \
-                __this = frame->this;                                  \
-                frame->local = NULL;                                   \
-                int __opr;                                             \
-                STACK_RESET (frame->root);                             \
-                if (__local) {                                         \
-                        afr_local_cleanup (__local, __this);           \
-                        mem_put (__local);                             \
-                }                                                      \
-                AFR_FRAME_INIT (frame, __opr);                         \
-        } while (0)
+afr_cleanup_fd_ctx(xlator_t *this, fd_t *fd);
+
+#define AFR_STACK_UNWIND(fop, frame, op_ret, op_errno, params...)              \
+    do {                                                                       \
+        afr_local_t *__local = NULL;                                           \
+        xlator_t *__this = NULL;                                               \
+        int32_t __op_ret = 0;                                                  \
+        int32_t __op_errno = 0;                                                \
+                                                                               \
+        __op_ret = op_ret;                                                     \
+        __op_errno = op_errno;                                                 \
+        if (frame) {                                                           \
+            __local = frame->local;                                            \
+            __this = frame->this;                                              \
+            afr_handle_inconsistent_fop(frame, &__op_ret, &__op_errno);        \
+            if (__local && __local->is_read_txn)                               \
+                afr_pending_read_decrement(__this->private,                    \
+                                           __local->read_subvol);              \
+            if (__local && __local->xdata_req &&                               \
+                afr_is_lock_mode_mandatory(__local->xdata_req))                \
+                afr_dom_lock_release(frame);                                   \
+            frame->local = NULL;                                               \
+        }                                                                      \
+                                                                               \
+        STACK_UNWIND_STRICT(fop, frame, __op_ret, __op_errno, params);         \
+        if (__local) {                                                         \
+            afr_local_cleanup(__local, __this);                                \
+            mem_put(__local);                                                  \
+        }                                                                      \
+    } while (0)
+
+#define AFR_STACK_DESTROY(frame)                                               \
+    do {                                                                       \
+        afr_local_t *__local = NULL;                                           \
+        xlator_t *__this = NULL;                                               \
+        __local = frame->local;                                                \
+        __this = frame->this;                                                  \
+        frame->local = NULL;                                                   \
+        STACK_DESTROY(frame->root);                                            \
+        if (__local) {                                                         \
+            afr_local_cleanup(__local, __this);                                \
+            mem_put(__local);                                                  \
+        }                                                                      \
+    } while (0);
+
+#define AFR_FRAME_INIT(frame, op_errno)                                        \
+    ({                                                                         \
+        frame->local = mem_get0(THIS->local_pool);                             \
+        if (afr_local_init(frame->local, frame->this->private, &op_errno)) {   \
+            afr_local_cleanup(frame->local, frame->this);                      \
+            mem_put(frame->local);                                             \
+            frame->local = NULL;                                               \
+        };                                                                     \
+        frame->local;                                                          \
+    })
+
+#define AFR_STACK_RESET(frame)                                                 \
+    do {                                                                       \
+        afr_local_t *__local = NULL;                                           \
+        xlator_t *__this = NULL;                                               \
+        __local = frame->local;                                                \
+        __this = frame->this;                                                  \
+        frame->local = NULL;                                                   \
+        int __opr;                                                             \
+        STACK_RESET(frame->root);                                              \
+        if (__local) {                                                         \
+            afr_local_cleanup(__local, __this);                                \
+            mem_put(__local);                                                  \
+        }                                                                      \
+        AFR_FRAME_INIT(frame, __opr);                                          \
+    } while (0)
 
 /* allocate and return a string that is the basename of argument */
 static inline char *
-AFR_BASENAME (const char *str)
+AFR_BASENAME(const char *str)
 {
-        char *__tmp_str = NULL;
-        char *__basename_str = NULL;
-        __tmp_str = gf_strdup (str);
-        __basename_str = gf_strdup (basename (__tmp_str));
-        GF_FREE (__tmp_str);
-        return __basename_str;
+    char *__tmp_str = NULL;
+    char *__basename_str = NULL;
+    __tmp_str = gf_strdup(str);
+    __basename_str = gf_strdup(basename(__tmp_str));
+    GF_FREE(__tmp_str);
+    return __basename_str;
 }
 
 call_frame_t *
-afr_copy_frame (call_frame_t *base);
+afr_copy_frame(call_frame_t *base);
 
 int
-afr_transaction_local_init (afr_local_t *local, xlator_t *this);
+afr_transaction_local_init(afr_local_t *local, xlator_t *this);
 
 int32_t
-afr_marker_getxattr (call_frame_t *frame, xlator_t *this,
-                     loc_t *loc, const char *name,afr_local_t *local, afr_private_t *priv );
+afr_marker_getxattr(call_frame_t *frame, xlator_t *this, loc_t *loc,
+                    const char *name, afr_local_t *local, afr_private_t *priv);
 
 int
-afr_local_init (afr_local_t *local, afr_private_t *priv, int32_t *op_errno);
+afr_local_init(afr_local_t *local, afr_private_t *priv, int32_t *op_errno);
 
 int
-afr_internal_lock_init (afr_internal_lock_t *lk, size_t child_count,
-                        transaction_lk_type_t lk_type);
+afr_internal_lock_init(afr_internal_lock_t *lk, size_t child_count);
 
 int
-afr_higher_errno (int32_t old_errno, int32_t new_errno);
+afr_higher_errno(int32_t old_errno, int32_t new_errno);
 
 int
-afr_final_errno (afr_local_t *local, afr_private_t *priv);
+afr_final_errno(afr_local_t *local, afr_private_t *priv);
 
 int
-afr_xattr_req_prepare (xlator_t *this, dict_t *xattr_req);
+afr_xattr_req_prepare(xlator_t *this, dict_t *xattr_req);
 
 void
-afr_fix_open (fd_t *fd, xlator_t *this);
+afr_fix_open(fd_t *fd, xlator_t *this);
 
 afr_fd_ctx_t *
-afr_fd_ctx_get (fd_t *fd, xlator_t *this);
+afr_fd_ctx_get(fd_t *fd, xlator_t *this);
 
 void
-afr_set_low_priority (call_frame_t *frame);
+afr_set_low_priority(call_frame_t *frame);
 int
-afr_child_fd_ctx_set (xlator_t *this, fd_t *fd, int32_t child,
-                      int flags);
+afr_child_fd_ctx_set(xlator_t *this, fd_t *fd, int32_t child, int flags);
 
 void
-afr_matrix_cleanup (int32_t **pending, unsigned int m);
+afr_matrix_cleanup(int32_t **pending, unsigned int m);
 
-int32_t**
-afr_matrix_create (unsigned int m, unsigned int n);
+int32_t **
+afr_matrix_create(unsigned int m, unsigned int n);
 
-int**
-afr_mark_pending_changelog (afr_private_t *priv, unsigned char *pending,
-                            dict_t *xattr, ia_type_t iat);
+int **
+afr_mark_pending_changelog(afr_private_t *priv, unsigned char *pending,
+                           dict_t *xattr, ia_type_t iat);
 
 void
-afr_filter_xattrs (dict_t *xattr);
+afr_filter_xattrs(dict_t *xattr);
 
 /*
  * Special value indicating we should use the "auto" quorum method instead of
@@ -994,48 +1245,179 @@ afr_filter_xattrs (dict_t *xattr);
 #define AFR_QUORUM_AUTO INT_MAX
 
 int
-afr_fd_report_unstable_write (xlator_t *this, fd_t *fd);
+afr_fd_report_unstable_write(xlator_t *this, afr_local_t *local);
+
+gf_boolean_t
+afr_fd_has_witnessed_unstable_write(xlator_t *this, inode_t *inode);
+
+void
+afr_reply_wipe(struct afr_reply *reply);
+
+void
+afr_replies_wipe(struct afr_reply *replies, int count);
+
+gf_boolean_t
+afr_xattrs_are_equal(dict_t *dict1, dict_t *dict2);
+
+gf_boolean_t
+afr_is_xattr_ignorable(char *key);
+
+int
+afr_get_heal_info(call_frame_t *frame, xlator_t *this, loc_t *loc);
+
+int
+afr_heal_splitbrain_file(call_frame_t *frame, xlator_t *this, loc_t *loc);
+
+int
+afr_get_split_brain_status(void *opaque);
+
+int
+afr_get_split_brain_status_cbk(int ret, call_frame_t *frame, void *opaque);
+
+int
+afr_inode_split_brain_choice_set(inode_t *inode, xlator_t *this,
+                                 int spb_choice);
+int
+afr_split_brain_read_subvol_get(inode_t *inode, xlator_t *this,
+                                call_frame_t *frame, int *spb_subvol);
+int
+afr_get_child_index_from_name(xlator_t *this, char *name);
+
+int
+afr_is_split_brain(call_frame_t *frame, xlator_t *this, inode_t *inode,
+                   uuid_t gfid, gf_boolean_t *d_spb, gf_boolean_t *m_spb);
+int
+afr_spb_choice_timeout_cancel(xlator_t *this, inode_t *inode);
+
+int
+afr_set_split_brain_choice(int ret, call_frame_t *frame, void *opaque);
 
 gf_boolean_t
-afr_fd_has_witnessed_unstable_write (xlator_t *this, fd_t *fd);
+afr_get_need_heal(xlator_t *this);
 
 void
-afr_delayed_changelog_wake_resume (xlator_t *this, fd_t *fd, call_stub_t *stub);
+afr_set_need_heal(xlator_t *this, afr_local_t *local);
+
+int
+afr_selfheal_data_open(xlator_t *this, inode_t *inode, fd_t **fd);
 
 int
-afr_inodelk_init (afr_inodelk_t *lk, char *dom, size_t child_count);
+afr_get_msg_id(char *op_type);
+
+int
+afr_set_in_flight_sb_status(xlator_t *this, call_frame_t *frame,
+                            inode_t *inode);
+
+int32_t
+afr_quorum_errno(afr_private_t *priv);
+
+gf_boolean_t
+afr_is_consistent_io_possible(afr_local_t *local, afr_private_t *priv,
+                              int32_t *op_errno);
+void
+afr_handle_inconsistent_fop(call_frame_t *frame, int32_t *op_ret,
+                            int32_t *op_errno);
+
+void
+afr_inode_write_fill(call_frame_t *frame, xlator_t *this, int child_index,
+                     int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
+                     struct iatt *postbuf, dict_t *xdata);
+void
+afr_process_post_writev(call_frame_t *frame, xlator_t *this);
 
 void
-afr_handle_open_fd_count (call_frame_t *frame, xlator_t *this);
+afr_writev_unwind(call_frame_t *frame, xlator_t *this);
 
 void
-afr_remove_eager_lock_stub (afr_local_t *local);
+afr_writev_copy_outvars(call_frame_t *src_frame, call_frame_t *dst_frame);
 
 void
-afr_replies_wipe (struct afr_reply *replies, int count);
+afr_update_uninodelk(afr_local_t *local, afr_internal_lock_t *int_lock,
+                     int32_t child_index);
+afr_fd_ctx_t *
+__afr_fd_ctx_get(fd_t *fd, xlator_t *this);
 
 gf_boolean_t
-afr_xattrs_are_equal (dict_t *dict1, dict_t *dict2);
+afr_is_inode_refresh_reqd(inode_t *inode, xlator_t *this, int event_gen1,
+                          int event_gen2);
 
+int
+afr_serialize_xattrs_with_delimiter(call_frame_t *frame, xlator_t *this,
+                                    char *buf, const char *default_str,
+                                    int32_t *serz_len, char delimiter);
 gf_boolean_t
-afr_is_xattr_ignorable (char *key);
+afr_is_symmetric_error(call_frame_t *frame, xlator_t *this);
 
 int
-afr_get_heal_info (call_frame_t *frame, xlator_t *this, loc_t *loc,
-                   dict_t *xdata);
+__afr_inode_ctx_get(xlator_t *this, inode_t *inode, afr_inode_ctx_t **ctx);
+
+uint64_t
+afr_write_subvol_get(call_frame_t *frame, xlator_t *this);
 
 int
-afr_heal_splitbrain_file(call_frame_t *frame, xlator_t *this, loc_t *loc);
+afr_write_subvol_set(call_frame_t *frame, xlator_t *this);
+
+int
+afr_write_subvol_reset(call_frame_t *frame, xlator_t *this);
+
+int
+afr_set_inode_local(xlator_t *this, afr_local_t *local, inode_t *inode);
 
 int
-afr_get_split_brain_status (call_frame_t *frame, xlator_t *this, loc_t *loc);
+afr_fill_ta_loc(xlator_t *this, loc_t *loc, gf_boolean_t is_gfid_based_fop);
 
 int
-afr_inode_split_brain_choice_set (inode_t *inode, xlator_t *this,
-                                  int spb_choice);
+afr_ta_post_op_lock(xlator_t *this, loc_t *loc);
+
 int
-afr_inode_split_brain_choice_get (inode_t *inode, xlator_t *this,
-                                  int *spb_choice);
+afr_ta_post_op_unlock(xlator_t *this, loc_t *loc);
+
+gf_boolean_t
+afr_is_pending_set(xlator_t *this, dict_t *xdata, int type);
+
 int
-afr_get_child_index_from_name (xlator_t *this, char *name);
+__afr_get_up_children_count(afr_private_t *priv);
+
+call_frame_t *
+afr_ta_frame_create(xlator_t *this);
+
+gf_boolean_t
+afr_ta_has_quorum(afr_private_t *priv, afr_local_t *local);
+
+void
+afr_ta_lock_release_synctask(xlator_t *this);
+
+void
+afr_ta_locked_priv_invalidate(afr_private_t *priv);
+
+gf_boolean_t
+afr_lookup_has_quorum(call_frame_t *frame,
+                      const unsigned int up_children_count);
+
+void
+afr_mark_new_entry_changelog(call_frame_t *frame, xlator_t *this);
+
+void
+afr_handle_replies_quorum(call_frame_t *frame, xlator_t *this);
+
+gf_boolean_t
+afr_ta_dict_contains_pending_xattr(dict_t *dict, afr_private_t *priv,
+                                   int child);
+
+void
+afr_selfheal_childup(xlator_t *this, afr_private_t *priv);
+
+gf_boolean_t
+afr_is_lock_mode_mandatory(dict_t *xdata);
+
+void
+afr_dom_lock_release(call_frame_t *frame);
+
+void
+afr_fill_success_replies(afr_local_t *local, afr_private_t *priv,
+                         unsigned char *replies);
+
+gf_boolean_t
+afr_is_private_directory(afr_private_t *priv, uuid_t pargfid, const char *name,
+                         pid_t pid);
 #endif /* __AFR_H__ */
diff --git a/xlators/cluster/afr/src/pump.c b/xlators/cluster/afr/src/pump.c
deleted file mode 100644
index ff8f5866f10..00000000000
--- a/xlators/cluster/afr/src/pump.c
+++ /dev/null
@@ -1,2477 +0,0 @@
-/*
-  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
-  This file is part of GlusterFS.
-
-  This file is licensed to you under your choice of the GNU Lesser
-  General Public License, version 3 or any later version (LGPLv3 or
-  later), or the GNU General Public License, version 2 (GPLv2), in all
-  cases as published by the Free Software Foundation.
-*/
-
-#include <unistd.h>
-#include <sys/time.h>
-#include <stdlib.h>
-#include <fnmatch.h>
-
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
-
-#include "afr-common.c"
-#include "defaults.c"
-#include "glusterfs.h"
-#include "pump.h"
-
-
-static int
-afr_set_dict_gfid (dict_t *dict, uuid_t gfid)
-{
-        int ret       = 0;
-        uuid_t *pgfid = NULL;
-
-        GF_ASSERT (gfid);
-
-        pgfid = GF_CALLOC (1, sizeof (uuid_t), gf_common_mt_char);
-        if (!pgfid) {
-                ret = -1;
-                goto out;
-        }
-
-        gf_uuid_copy (*pgfid, gfid);
-
-        ret = dict_set_dynptr (dict, "gfid-req", pgfid, sizeof (uuid_t));
-        if (ret)
-                gf_log (THIS->name, GF_LOG_ERROR, "gfid set failed");
-
-out:
-        if (ret && pgfid)
-                GF_FREE (pgfid);
-	return ret;
-}
-
-static int
-afr_set_root_gfid (dict_t *dict)
-{
-        uuid_t gfid;
-        int ret = 0;
-
-        memset (gfid, 0, 16);
-        gfid[15] = 1;
-
-        ret = afr_set_dict_gfid (dict, gfid);
-
-        return ret;
-}
-
-static int
-afr_build_child_loc (xlator_t *this, loc_t *child, loc_t *parent, char *name)
-{
-        int   ret = -1;
-        uuid_t pargfid = {0};
-
-        if (!child)
-                goto out;
-
-        if (!gf_uuid_is_null (parent->inode->gfid))
-                gf_uuid_copy (pargfid, parent->inode->gfid);
-        else if (!gf_uuid_is_null (parent->gfid))
-                gf_uuid_copy (pargfid, parent->gfid);
-
-        if (gf_uuid_is_null (pargfid))
-                goto out;
-
-        if (strcmp (parent->path, "/") == 0)
-                ret = gf_asprintf ((char **)&child->path, "/%s", name);
-        else
-                ret = gf_asprintf ((char **)&child->path, "%s/%s", parent->path,
-                                   name);
-
-        if (-1 == ret) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "asprintf failed while setting child path");
-        }
-
-        child->name = strrchr (child->path, '/');
-        if (child->name)
-                child->name++;
-
-        child->parent = inode_ref (parent->inode);
-        child->inode = inode_new (parent->inode->table);
-        gf_uuid_copy (child->pargfid, pargfid);
-
-        if (!child->inode) {
-                ret = -1;
-                goto out;
-        }
-
-        ret = 0;
-out:
-        if ((ret == -1) && child)
-                loc_wipe (child);
-
-        return ret;
-}
-
-static void
-afr_build_root_loc (xlator_t *this, loc_t *loc)
-{
-        afr_private_t   *priv = NULL;
-
-        priv = this->private;
-        loc->path = gf_strdup ("/");
-        loc->name = "";
-        loc->inode = inode_ref (priv->root_inode);
-        gf_uuid_copy (loc->gfid, loc->inode->gfid);
-}
-
-static void
-afr_update_loc_gfids (loc_t *loc, struct iatt *buf, struct iatt *postparent)
-{
-        GF_ASSERT (loc);
-        GF_ASSERT (buf);
-
-        gf_uuid_copy (loc->gfid, buf->ia_gfid);
-        if (postparent)
-                gf_uuid_copy (loc->pargfid, postparent->ia_gfid);
-}
-
-static uint64_t pump_pid = 0;
-static inline void
-pump_fill_loc_info (loc_t *loc, struct iatt *iatt, struct iatt *parent)
-{
-        afr_update_loc_gfids (loc, iatt, parent);
-        gf_uuid_copy (loc->inode->gfid, iatt->ia_gfid);
-}
-
-static int
-pump_mark_start_pending (xlator_t *this)
-{
-        afr_private_t  *priv      = NULL;
-        pump_private_t *pump_priv = NULL;
-
-        priv      = this->private;
-        pump_priv = priv->pump_private;
-
-        pump_priv->pump_start_pending = 1;
-
-        return 0;
-}
-
-static int
-is_pump_start_pending (xlator_t *this)
-{
-        afr_private_t  *priv      = NULL;
-        pump_private_t *pump_priv = NULL;
-
-        priv      = this->private;
-        pump_priv = priv->pump_private;
-
-        return (pump_priv->pump_start_pending);
-}
-
-static int
-pump_remove_start_pending (xlator_t *this)
-{
-        afr_private_t  *priv      = NULL;
-        pump_private_t *pump_priv = NULL;
-
-        priv      = this->private;
-        pump_priv = priv->pump_private;
-
-        pump_priv->pump_start_pending = 0;
-
-        return 0;
-}
-
-static pump_state_t
-pump_get_state ()
-{
-        xlator_t *this = NULL;
-        afr_private_t *priv = NULL;
-        pump_private_t *pump_priv = NULL;
-
-        pump_state_t ret;
-
-        this = THIS;
-        priv = this->private;
-        pump_priv = priv->pump_private;
-
-        LOCK (&pump_priv->pump_state_lock);
-        {
-                ret = pump_priv->pump_state;
-        }
-        UNLOCK (&pump_priv->pump_state_lock);
-
-        return ret;
-}
-
-int
-pump_change_state (xlator_t *this, pump_state_t state)
-{
-        afr_private_t *priv = NULL;
-        pump_private_t *pump_priv = NULL;
-
-        pump_state_t state_old;
-        pump_state_t state_new;
-
-
-        priv = this->private;
-        pump_priv = priv->pump_private;
-
-        GF_ASSERT (pump_priv);
-
-        LOCK (&pump_priv->pump_state_lock);
-        {
-                state_old = pump_priv->pump_state;
-                state_new = state;
-
-                pump_priv->pump_state = state;
-
-        }
-        UNLOCK (&pump_priv->pump_state_lock);
-
-        gf_log (this->name, GF_LOG_DEBUG,
-                "Pump changing state from %d to %d",
-                state_old,
-                state_new);
-
-        return  0;
-}
-
-static int
-pump_set_resume_path (xlator_t *this, const char *path)
-{
-        int ret = 0;
-
-        afr_private_t *priv = NULL;
-        pump_private_t *pump_priv = NULL;
-
-        priv = this->private;
-        pump_priv = priv->pump_private;
-
-        GF_ASSERT (pump_priv);
-
-        LOCK (&pump_priv->resume_path_lock);
-        {
-                strncpy (pump_priv->resume_path, path, strlen (path) + 1);
-        }
-        UNLOCK (&pump_priv->resume_path_lock);
-
-        return ret;
-}
-
-static int
-pump_save_path (xlator_t *this, const char *path)
-{
-        afr_private_t *priv = NULL;
-        pump_state_t state;
-        dict_t *dict = NULL;
-        loc_t  loc = {0};
-        int dict_ret = 0;
-        int ret = -1;
-
-        state = pump_get_state ();
-        if (state == PUMP_STATE_RESUME)
-                return 0;
-
-        priv = this->private;
-
-        GF_ASSERT (priv->root_inode);
-
-        afr_build_root_loc (this, &loc);
-
-        dict = dict_new ();
-        dict_ret = dict_set_str (dict, PUMP_PATH, (char *)path);
-        if (dict_ret)
-                gf_log (this->name, GF_LOG_WARNING,
-                        "%s: failed to set the key %s", path, PUMP_PATH);
-
-        ret = syncop_setxattr (PUMP_SOURCE_CHILD (this), &loc, dict, 0, NULL,
-                               NULL);
-
-        if (ret < 0) {
-                gf_log (this->name, GF_LOG_INFO,
-                        "setxattr failed - could not save path=%s", path);
-        } else {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "setxattr succeeded - saved path=%s", path);
-        }
-
-        dict_unref (dict);
-
-        loc_wipe (&loc);
-        return 0;
-}
-
-static int
-pump_check_and_update_status (xlator_t *this)
-{
-        pump_state_t state;
-        int ret = -1;
-
-        state = pump_get_state ();
-
-        switch (state) {
-
-        case PUMP_STATE_RESUME:
-        case PUMP_STATE_RUNNING:
-        {
-                ret = 0;
-                break;
-        }
-        case PUMP_STATE_PAUSE:
-        {
-                ret = -1;
-                break;
-        }
-        case PUMP_STATE_ABORT:
-        {
-                pump_save_path (this, "/");
-                ret = -1;
-                break;
-        }
-        default:
-        {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "Unknown pump state");
-                ret = -1;
-                break;
-        }
-
-        }
-
-        return ret;
-}
-
-static const char *
-pump_get_resume_path (xlator_t *this)
-{
-        afr_private_t *priv = NULL;
-        pump_private_t *pump_priv = NULL;
-
-        const char *resume_path = NULL;
-
-        priv = this->private;
-        pump_priv = priv->pump_private;
-
-        resume_path = pump_priv->resume_path;
-
-        return resume_path;
-}
-
-static int
-pump_update_resume_state (xlator_t *this, const char *path)
-{
-        pump_state_t state;
-        const char *resume_path = NULL;
-
-        state = pump_get_state ();
-
-        if (state == PUMP_STATE_RESUME) {
-                resume_path = pump_get_resume_path (this);
-                if (strcmp (resume_path, "/") == 0) {
-                        gf_log (this->name, GF_LOG_DEBUG,
-                                "Reached the resume path (/). Proceeding to change state"
-                                " to running");
-                        pump_change_state (this, PUMP_STATE_RUNNING);
-                } else if (strcmp (resume_path, path) == 0) {
-                        gf_log (this->name, GF_LOG_DEBUG,
-                                "Reached the resume path. Proceeding to change state"
-                                " to running");
-                        pump_change_state (this, PUMP_STATE_RUNNING);
-                } else {
-                        gf_log (this->name, GF_LOG_DEBUG,
-                                "Not yet hit the resume path:res-path=%s,path=%s",
-                                resume_path, path);
-                }
-        }
-
-        return 0;
-}
-
-static gf_boolean_t
-is_pump_traversal_allowed (xlator_t *this, const char *path)
-{
-        pump_state_t state;
-        const char *resume_path = NULL;
-        gf_boolean_t ret = _gf_true;
-
-        state = pump_get_state ();
-
-        if (state == PUMP_STATE_RESUME) {
-                resume_path = pump_get_resume_path (this);
-                if (strstr (resume_path, path)) {
-                        gf_log (this->name, GF_LOG_DEBUG,
-                                "On the right path to resumption path");
-                        ret = _gf_true;
-                } else {
-                        gf_log (this->name, GF_LOG_DEBUG,
-                                "Not the right path to resuming=> ignoring traverse");
-                        ret = _gf_false;
-                }
-        }
-
-        return ret;
-}
-
-static int
-pump_save_file_stats (xlator_t *this, const char *path)
-{
-        afr_private_t  *priv        = NULL;
-        pump_private_t *pump_priv   = NULL;
-
-        priv      = this->private;
-        pump_priv = priv->pump_private;
-
-        LOCK (&pump_priv->resume_path_lock);
-        {
-                pump_priv->number_files_pumped++;
-
-                strncpy (pump_priv->current_file, path,
-                         PATH_MAX);
-        }
-        UNLOCK (&pump_priv->resume_path_lock);
-
-        return 0;
-}
-
-static int
-gf_pump_traverse_directory (loc_t *loc)
-{
-        xlator_t        *this              = NULL;
-        fd_t            *fd                = NULL;
-        off_t           offset             = 0;
-        loc_t           entry_loc          = {0};
-        gf_dirent_t     *entry             = NULL;
-        gf_dirent_t     *tmp               = NULL;
-        gf_dirent_t     entries;
-	struct iatt     iatt               = {0};
-        struct iatt     parent             = {0};
-	dict_t          *xattr_rsp         = NULL;
-        int             ret                = 0;
-        gf_boolean_t    is_directory_empty = _gf_true;
-        gf_boolean_t    free_entries       = _gf_false;
-
-        INIT_LIST_HEAD (&entries.list);
-        this = THIS;
-
-        GF_ASSERT (loc->inode);
-
-	fd = fd_create (loc->inode, pump_pid);
-        if (!fd) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "Failed to create fd for %s", loc->path);
-                goto out;
-        }
-
-        ret = syncop_opendir (this, loc, fd, NULL, NULL);
-        if (ret < 0) {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "opendir failed on %s", loc->path);
-                goto out;
-        }
-
-        gf_log (this->name, GF_LOG_TRACE,
-                "pump opendir on %s returned=%d",
-                loc->path, ret);
-
-        while (syncop_readdirp (this, fd, 131072, offset, &entries, NULL,
-                                NULL)) {
-                free_entries = _gf_true;
-
-                if (list_empty (&entries.list)) {
-                        gf_log (this->name, GF_LOG_TRACE,
-                                "no more entries in directory");
-                        goto out;
-                }
-
-                list_for_each_entry_safe (entry, tmp, &entries.list, list) {
-                        gf_log (this->name, GF_LOG_DEBUG,
-                                "found readdir entry=%s", entry->d_name);
-
-                        offset = entry->d_off;
-                        if (gf_uuid_is_null (entry->d_stat.ia_gfid)) {
-                                gf_log (this->name, GF_LOG_WARNING, "%s/%s: No "
-                                        "gfid present skipping",
-                                        loc->path, entry->d_name);
-                                continue;
-                        }
-                        loc_wipe (&entry_loc);
-                        ret = afr_build_child_loc (this, &entry_loc, loc,
-                                                   entry->d_name);
-                        if (ret)
-                                goto out;
-
-			if ((strcmp (entry->d_name, ".") == 0) ||
-			    (strcmp (entry->d_name, "..") == 0))
-				continue;
-
-			is_directory_empty = _gf_false;
-			gf_log (this->name, GF_LOG_DEBUG,
-				"lookup %s => %"PRId64,
-				entry_loc.path,
-				iatt.ia_ino);
-
-			ret = syncop_lookup (this, &entry_loc, &iatt, &parent,
-					     NULL, &xattr_rsp);
-
-			if (ret) {
-				gf_log (this->name, GF_LOG_ERROR,
-					"%s: lookup failed", entry_loc.path);
-				continue;
-			}
-
-			ret = afr_selfheal_name (this, loc->gfid, entry->d_name,
-                                                 NULL);
-			if (ret) {
-				gf_log (this->name, GF_LOG_ERROR,
-					"%s: name self-heal failed (%s/%s)",
-					entry_loc.path, uuid_utoa (loc->gfid),
-					entry->d_name);
-				continue;
-			}
-
-			ret = afr_selfheal (this, iatt.ia_gfid);
-			if (ret < 0) {
-				gf_log (this->name, GF_LOG_ERROR,
-					"%s: self-heal failed (%s)",
-					entry_loc.path, uuid_utoa (iatt.ia_gfid));
-				continue;
-			}
-
-			pump_fill_loc_info (&entry_loc, &iatt, &parent);
-
-			pump_update_resume_state (this, entry_loc.path);
-
-			pump_save_path (this, entry_loc.path);
-			pump_save_file_stats (this, entry_loc.path);
-
-			ret = pump_check_and_update_status (this);
-			if (ret < 0) {
-				gf_log (this->name, GF_LOG_DEBUG,
-					"Pump beginning to exit out");
-				goto out;
-			}
-
-			if (IA_ISDIR (iatt.ia_type)) {
-				if (is_pump_traversal_allowed (this, entry_loc.path)) {
-					gf_log (this->name, GF_LOG_TRACE,
-						"entering dir=%s", entry->d_name);
-					gf_pump_traverse_directory (&entry_loc);
-				}
-                        }
-                }
-
-                gf_dirent_free (&entries);
-                free_entries = _gf_false;
-                gf_log (this->name, GF_LOG_TRACE, "offset incremented to %d",
-                        (int32_t ) offset);
-
-        }
-
-        ret = syncop_close (fd);
-        if (ret < 0)
-                gf_log (this->name, GF_LOG_DEBUG, "closing the fd failed");
-
-        if (is_directory_empty && (strcmp (loc->path, "/") == 0)) {
-               pump_change_state (this, PUMP_STATE_RUNNING);
-               gf_log (this->name, GF_LOG_INFO, "Empty source brick. "
-                                "Nothing to be done.");
-        }
-
-out:
-        if (entry_loc.path)
-                loc_wipe (&entry_loc);
-        if (free_entries)
-                gf_dirent_free (&entries);
-        return 0;
-}
-
-static int
-pump_update_resume_path (xlator_t *this)
-{
-        const char *resume_path = NULL;
-
-        resume_path = pump_get_resume_path (this);
-
-        if (resume_path) {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "Found a path to resume from: %s",
-                        resume_path);
-
-        }else {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "Did not find a path=> setting to '/'");
-                pump_set_resume_path (this, "/");
-        }
-
-        pump_change_state (this, PUMP_STATE_RESUME);
-
-        return 0;
-}
-
-static int32_t
-pump_xattr_cleaner (call_frame_t *frame, void *cookie, xlator_t *this,
-                    int32_t op_ret, int32_t op_errno, dict_t *xdata)
-{
-        afr_private_t  *priv      = NULL;
-        loc_t           loc       = {0};
-        int             i         = 0;
-        int             ret       = 0;
-        int             source    = 0;
-        int             sink      = 1;
-
-        priv      = this->private;
-
-        afr_build_root_loc (this, &loc);
-
-        ret = syncop_removexattr (priv->children[source], &loc,
-				  PUMP_PATH, 0, NULL);
-
-        ret = syncop_removexattr (priv->children[sink], &loc,
-                                  PUMP_SINK_COMPLETE, 0, NULL);
-
-        for (i = 0; i < priv->child_count; i++) {
-                ret = syncop_removexattr (priv->children[i], &loc,
-                                          PUMP_SOURCE_COMPLETE, 0, NULL);
-                if (ret) {
-                        gf_log (this->name, GF_LOG_DEBUG, "removexattr "
-                                "failed with %s", strerror (-ret));
-                }
-        }
-
-        loc_wipe (&loc);
-        return pump_command_reply (frame, this);
-}
-
-static int
-pump_complete_migration (xlator_t *this)
-{
-        afr_private_t *priv = NULL;
-        pump_private_t *pump_priv = NULL;
-        dict_t *dict = NULL;
-        pump_state_t state;
-        loc_t  loc = {0};
-        int dict_ret = 0;
-        int ret = -1;
-
-        priv = this->private;
-        pump_priv = priv->pump_private;
-
-        GF_ASSERT (priv->root_inode);
-
-        afr_build_root_loc (this, &loc);
-
-        dict = dict_new ();
-
-        state = pump_get_state ();
-        if (state == PUMP_STATE_RUNNING) {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "Pump finished pumping");
-
-                pump_priv->pump_finished = _gf_true;
-
-                dict_ret = dict_set_str (dict, PUMP_SOURCE_COMPLETE, "jargon");
-                if (dict_ret)
-                        gf_log (this->name, GF_LOG_WARNING,
-                                "%s: failed to set the key %s",
-                                loc.path, PUMP_SOURCE_COMPLETE);
-
-                ret = syncop_setxattr (PUMP_SOURCE_CHILD (this), &loc, dict, 0,
-                                       NULL, NULL);
-                if (ret < 0) {
-                        gf_log (this->name, GF_LOG_DEBUG,
-                                "setxattr failed - while  notifying source complete");
-                }
-                dict_ret = dict_set_str (dict, PUMP_SINK_COMPLETE, "jargon");
-                if (dict_ret)
-                        gf_log (this->name, GF_LOG_WARNING,
-                                "%s: failed to set the key %s",
-                                loc.path, PUMP_SINK_COMPLETE);
-
-                ret = syncop_setxattr (PUMP_SINK_CHILD (this), &loc, dict, 0,
-                                       NULL, NULL);
-                if (ret < 0) {
-                        gf_log (this->name, GF_LOG_DEBUG,
-                                "setxattr failed - while notifying sink complete");
-                }
-
-                pump_save_path (this, "/");
-
-        } else if (state == PUMP_STATE_ABORT) {
-                gf_log (this->name, GF_LOG_DEBUG, "Starting cleanup "
-                        "of pump internal xattrs");
-                call_resume (pump_priv->cleaner);
-        }
-
-        loc_wipe (&loc);
-        return 0;
-}
-
-static int
-pump_lookup_sink (loc_t *loc)
-{
-        xlator_t *this = NULL;
-	struct iatt iatt, parent;
-	dict_t *xattr_rsp;
-        dict_t *xattr_req = NULL;
-        int ret = 0;
-
-        this = THIS;
-
-        xattr_req = dict_new ();
-
-        ret = afr_set_root_gfid (xattr_req);
-        if (ret)
-                goto out;
-
-        ret = syncop_lookup (PUMP_SINK_CHILD (this), loc, &iatt, &parent,
-                             xattr_req, &xattr_rsp);
-
-        if (ret) {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "Lookup on sink child failed");
-                ret = -1;
-                goto out;
-        }
-
-out:
-        if (xattr_req)
-                dict_unref (xattr_req);
-
-        return ret;
-}
-
-static int
-pump_task (void *data)
-{
-	xlator_t *this = NULL;
-        afr_private_t *priv = NULL;
-
-
-        loc_t loc = {0};
-	struct iatt iatt, parent;
-	dict_t *xattr_rsp = NULL;
-        dict_t *xattr_req = NULL;
-
-        int ret = -1;
-
-        this = THIS;
-        priv = this->private;
-
-        GF_ASSERT (priv->root_inode);
-
-        afr_build_root_loc (this, &loc);
-        xattr_req = dict_new ();
-        if (!xattr_req) {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "Out of memory");
-                ret = -1;
-                goto out;
-        }
-
-        afr_set_root_gfid (xattr_req);
-        ret = syncop_lookup (this, &loc, &iatt, &parent,
-                             xattr_req, &xattr_rsp);
-
-        gf_log (this->name, GF_LOG_TRACE,
-                "lookup: path=%s gfid=%s",
-                loc.path, uuid_utoa (loc.inode->gfid));
-
-        ret = pump_check_and_update_status (this);
-        if (ret < 0) {
-                goto out;
-        }
-
-        pump_update_resume_path (this);
-
-        afr_set_root_gfid (xattr_req);
-        ret = pump_lookup_sink (&loc);
-        if (ret) {
-                pump_update_resume_path (this);
-                goto out;
-        }
-
-        gf_pump_traverse_directory (&loc);
-
-        pump_complete_migration (this);
-out:
-        if (xattr_req)
-                dict_unref (xattr_req);
-
-        loc_wipe (&loc);
-	return 0;
-}
-
-
-static int
-pump_task_completion (int ret, call_frame_t *sync_frame, void *data)
-{
-        xlator_t *this = NULL;
-        afr_private_t *priv = NULL;
-
-        this = THIS;
-
-        priv = this->private;
-
-        inode_unref (priv->root_inode);
-        STACK_DESTROY (sync_frame->root);
-
-        gf_log (this->name, GF_LOG_DEBUG,
-                "Pump xlator exiting");
-	return 0;
-}
-
-int
-pump_start (call_frame_t *pump_frame, xlator_t *this)
-{
-	afr_private_t *priv = NULL;
-	pump_private_t *pump_priv = NULL;
-
-	int ret = -1;
-
-	priv = this->private;
-        pump_priv = priv->pump_private;
-
-        afr_set_lk_owner (pump_frame, this, pump_frame->root);
-	pump_pid = (uint64_t) (unsigned long)pump_frame->root;
-
-	ret = synctask_new (pump_priv->env, pump_task,
-                            pump_task_completion,
-                            pump_frame, NULL);
-        if (ret == -1) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "starting pump failed");
-                pump_change_state (this, PUMP_STATE_ABORT);
-                goto out;
-        }
-
-        gf_log (this->name, GF_LOG_DEBUG,
-                "setting pump as started lk_owner: %s %"PRIu64,
-                lkowner_utoa (&pump_frame->root->lk_owner), pump_pid);
-
-        priv->use_afr_in_pump = 1;
-out:
-	return ret;
-}
-
-static int
-pump_start_synctask (xlator_t *this)
-{
-        call_frame_t *frame = NULL;
-        int ret = 0;
-
-        frame = create_frame (this, this->ctx->pool);
-        if (!frame) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "Out of memory");
-                ret = -1;
-                goto out;
-        }
-
-        pump_change_state (this, PUMP_STATE_RUNNING);
-
-        ret = pump_start (frame, this);
-
-out:
-        return ret;
-}
-
-int32_t
-pump_cmd_start_setxattr_cbk (call_frame_t *frame,
-                             void *cookie,
-                             xlator_t *this,
-                             int32_t op_ret,
-                             int32_t op_errno, dict_t *xdata)
-
-{
-        call_frame_t *prev = NULL;
-        afr_local_t *local = NULL;
-        int ret = 0;
-
-        local = frame->local;
-
-        if (op_ret < 0) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "Could not initiate destination "
-                        "brick connect");
-                ret = op_ret;
-                goto out;
-        }
-
-        gf_log (this->name, GF_LOG_DEBUG,
-                "Successfully initiated destination "
-                "brick connect");
-
-        pump_mark_start_pending (this);
-
-        /* send the PARENT_UP as pump is ready now */
-        prev = cookie;
-        if (prev && prev->this)
-                prev->this->notify (prev->this, GF_EVENT_PARENT_UP, this);
-
-out:
-        local->op_ret = ret;
-        pump_command_reply (frame, this);
-
-        return 0;
-}
-
-static int
-pump_initiate_sink_connect (call_frame_t *frame, xlator_t *this)
-{
-        afr_local_t   *local     = NULL;
-        afr_private_t *priv      = NULL;
-        dict_t        *dict      = NULL;
-        data_t        *data      = NULL;
-        char          *clnt_cmd  = NULL;
-        loc_t loc = {0};
-
-        int ret = 0;
-
-        priv  = this->private;
-        local = frame->local;
-
-        GF_ASSERT (priv->root_inode);
-
-        afr_build_root_loc (this, &loc);
-
-        data = data_ref (dict_get (local->dict, RB_PUMP_CMD_START));
-        if (!data) {
-                ret = -1;
-                gf_log (this->name, GF_LOG_ERROR,
-                        "Could not get destination brick value");
-                goto out;
-        }
-
-        dict = dict_new ();
-        if (!dict) {
-                ret = -1;
-                goto out;
-        }
-
-        clnt_cmd = GF_CALLOC (1, data->len+1, gf_common_mt_char);
-        if (!clnt_cmd) {
-                ret = -1;
-                goto out;
-        }
-
-        memcpy (clnt_cmd, data->data, data->len);
-        clnt_cmd[data->len] = '\0';
-        gf_log (this->name, GF_LOG_DEBUG, "Got destination brick %s\n",
-                        clnt_cmd);
-
-        ret = dict_set_dynstr (dict, CLIENT_CMD_CONNECT, clnt_cmd);
-        if (ret < 0) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "Could not inititiate destination brick "
-                        "connect");
-                goto out;
-        }
-
-	STACK_WIND (frame,
-		    pump_cmd_start_setxattr_cbk,
-		    PUMP_SINK_CHILD(this),
-		    PUMP_SINK_CHILD(this)->fops->setxattr,
-		    &loc,
-		    dict,
-		    0, NULL);
-
-        ret = 0;
-
-out:
-        if (dict)
-                dict_unref (dict);
-
-        if (data)
-                data_unref (data);
-
-        if (ret && clnt_cmd)
-                GF_FREE (clnt_cmd);
-
-        loc_wipe (&loc);
-        return ret;
-}
-
-static int
-is_pump_aborted (xlator_t *this)
-{
-        pump_state_t state;
-
-        state = pump_get_state ();
-
-        return ((state == PUMP_STATE_ABORT));
-}
-
-int32_t
-pump_cmd_start_getxattr_cbk (call_frame_t *frame,
-                             void *cookie,
-                             xlator_t *this,
-                             int32_t op_ret,
-                             int32_t op_errno,
-                             dict_t *dict, dict_t *xdata)
-{
-        afr_local_t *local = NULL;
-        char *path = NULL;
-
-        pump_state_t state;
-        int ret = 0;
-        int need_unwind = 0;
-        int dict_ret = -1;
-
-        local = frame->local;
-
-        if (op_ret < 0) {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "getxattr failed - changing pump "
-                        "state to RUNNING with '/'");
-                path = "/";
-                ret = op_ret;
-        } else {
-                gf_log (this->name, GF_LOG_TRACE,
-                        "getxattr succeeded");
-
-                dict_ret =  dict_get_str (dict, PUMP_PATH, &path);
-                if (dict_ret < 0)
-                        path = "/";
-        }
-
-        state = pump_get_state ();
-        if ((state == PUMP_STATE_RUNNING) ||
-            (state == PUMP_STATE_RESUME)) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "Pump is already started");
-                ret = -1;
-                goto out;
-        }
-
-        pump_set_resume_path (this, path);
-
-        if (is_pump_aborted (this))
-                /* We're re-starting pump afresh */
-                ret = pump_initiate_sink_connect (frame, this);
-        else {
-                /* We're re-starting pump from a previous
-                   pause */
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "about to start synctask");
-                ret = pump_start_synctask (this);
-                need_unwind = 1;
-        }
-
-out:
-        if ((ret < 0) || (need_unwind == 1)) {
-                local->op_ret = ret;
-                pump_command_reply (frame, this);
-        }
-	return 0;
-}
-
-int
-pump_execute_status (call_frame_t *frame, xlator_t *this)
-{
-        afr_private_t *priv = NULL;
-        pump_private_t *pump_priv = NULL;
-
-        uint64_t number_files = 0;
-
-        char filename[PATH_MAX];
-        char summary[PATH_MAX+256];
-        char *dict_str = NULL;
-
-        int32_t op_ret = 0;
-        int32_t op_errno = 0;
-
-        dict_t *dict = NULL;
-        int ret = -1;
-
-        priv = this->private;
-        pump_priv = priv->pump_private;
-
-        LOCK (&pump_priv->resume_path_lock);
-        {
-                number_files  = pump_priv->number_files_pumped;
-                strncpy (filename, pump_priv->current_file, PATH_MAX);
-        }
-        UNLOCK (&pump_priv->resume_path_lock);
-
-        dict_str     = GF_CALLOC (1, PATH_MAX + 256, gf_afr_mt_char);
-        if (!dict_str) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "Out of memory");
-                op_ret = -1;
-                op_errno = ENOMEM;
-                goto out;
-        }
-
-        if (pump_priv->pump_finished) {
-                snprintf (summary, PATH_MAX+256,
-                          "no_of_files=%"PRIu64, number_files);
-        } else {
-                snprintf (summary, PATH_MAX+256,
-                          "no_of_files=%"PRIu64":current_file=%s",
-                          number_files, filename);
-        }
-        snprintf (dict_str, PATH_MAX+256, "status=%d:%s",
-                  (pump_priv->pump_finished)?1:0, summary);
-
-        dict = dict_new ();
-
-        ret = dict_set_dynstr (dict, RB_PUMP_CMD_STATUS, dict_str);
-        if (ret < 0) {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "dict_set_dynstr returned negative value");
-        } else {
-                dict_str = NULL;
-        }
-
-        op_ret = 0;
-
-out:
-
-        AFR_STACK_UNWIND (getxattr, frame, op_ret, op_errno, dict, NULL);
-
-        if (dict)
-                dict_unref (dict);
-
-        GF_FREE (dict_str);
-
-        return 0;
-}
-
-int
-pump_execute_pause (call_frame_t *frame, xlator_t *this)
-{
-        afr_local_t *local = NULL;
-
-        local = frame->local;
-
-        pump_change_state (this, PUMP_STATE_PAUSE);
-
-        local->op_ret = 0;
-        pump_command_reply (frame, this);
-
-        return 0;
-}
-
-int
-pump_execute_start (call_frame_t *frame, xlator_t *this)
-{
-        afr_private_t *priv = NULL;
-        afr_local_t   *local = NULL;
-
-        int ret = 0;
-        loc_t loc = {0};
-
-        priv = this->private;
-        local = frame->local;
-
-        if (!priv->root_inode) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "Pump xlator cannot be started without an initial "
-                        "lookup");
-                ret = -1;
-                goto out;
-        }
-
-        GF_ASSERT (priv->root_inode);
-
-        afr_build_root_loc (this, &loc);
-
-	STACK_WIND (frame,
-		    pump_cmd_start_getxattr_cbk,
-		    PUMP_SOURCE_CHILD(this),
-		    PUMP_SOURCE_CHILD(this)->fops->getxattr,
-		    &loc,
-		    PUMP_PATH, NULL);
-
-        ret = 0;
-
-out:
-        if (ret < 0) {
-                local->op_ret = ret;
-                pump_command_reply (frame, this);
-        }
-
-        loc_wipe (&loc);
-	return 0;
-}
-
-static int
-pump_cleanup_helper (void *data) {
-        call_frame_t *frame = data;
-
-        pump_xattr_cleaner (frame, 0, frame->this, 0, 0, NULL);
-
-        return 0;
-}
-
-static int
-pump_cleanup_done (int ret, call_frame_t *sync_frame, void *data)
-{
-        STACK_DESTROY (sync_frame->root);
-
-        return 0;
-}
-
-int
-pump_execute_commit (call_frame_t *frame, xlator_t *this)
-{
-        afr_private_t  *priv       = NULL;
-        pump_private_t *pump_priv  = NULL;
-        afr_local_t    *local      = NULL;
-        call_frame_t   *sync_frame = NULL;
-        int             ret        = 0;
-
-        priv      = this->private;
-        pump_priv = priv->pump_private;
-        local     = frame->local;
-
-        local->op_ret = 0;
-        if (pump_priv->pump_finished) {
-                pump_change_state (this, PUMP_STATE_COMMIT);
-                sync_frame = create_frame (this, this->ctx->pool);
-                ret = synctask_new (pump_priv->env, pump_cleanup_helper,
-                                    pump_cleanup_done, sync_frame, frame);
-                if (ret) {
-                        gf_log (this->name, GF_LOG_DEBUG, "Couldn't create "
-                                "synctask for cleaning up xattrs.");
-                }
-
-        } else {
-                gf_log (this->name, GF_LOG_ERROR, "Commit can't proceed. "
-                        "Migration in progress");
-                local->op_ret = -1;
-                local->op_errno = EINPROGRESS;
-                pump_command_reply (frame, this);
-        }
-
-        return 0;
-}
-int
-pump_execute_abort (call_frame_t *frame, xlator_t *this)
-{
-        afr_private_t  *priv       = NULL;
-        pump_private_t *pump_priv  = NULL;
-        afr_local_t    *local      = NULL;
-        call_frame_t   *sync_frame = NULL;
-        int             ret        = 0;
-
-        priv      = this->private;
-        pump_priv = priv->pump_private;
-        local     = frame->local;
-
-        pump_change_state (this, PUMP_STATE_ABORT);
-
-        LOCK (&pump_priv->resume_path_lock);
-        {
-                pump_priv->number_files_pumped = 0;
-                pump_priv->current_file[0] = '\0';
-        }
-        UNLOCK (&pump_priv->resume_path_lock);
-
-        local->op_ret = 0;
-        if (pump_priv->pump_finished) {
-                sync_frame = create_frame (this, this->ctx->pool);
-                ret = synctask_new (pump_priv->env, pump_cleanup_helper,
-                                    pump_cleanup_done, sync_frame, frame);
-                if (ret) {
-                        gf_log (this->name, GF_LOG_DEBUG, "Couldn't create "
-                                "synctask for cleaning up xattrs.");
-                }
-
-        } else {
-                pump_priv->cleaner = fop_setxattr_cbk_stub (frame,
-                                                            pump_xattr_cleaner,
-                                                            0, 0, NULL);
-        }
-
-        return 0;
-}
-
-gf_boolean_t
-pump_command_status (xlator_t *this, dict_t *dict)
-{
-        char *cmd = NULL;
-        int dict_ret = -1;
-        int ret = _gf_true;
-
-        dict_ret = dict_get_str (dict, RB_PUMP_CMD_STATUS, &cmd);
-        if (dict_ret < 0) {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "Not a pump status command");
-                ret = _gf_false;
-                goto out;
-        }
-
-        gf_log (this->name, GF_LOG_DEBUG,
-                "Hit a pump command - status");
-        ret = _gf_true;
-
-out:
-        return ret;
-
-}
-
-gf_boolean_t
-pump_command_pause (xlator_t *this, dict_t *dict)
-{
-        char *cmd = NULL;
-        int dict_ret = -1;
-        int ret = _gf_true;
-
-        dict_ret = dict_get_str (dict, RB_PUMP_CMD_PAUSE, &cmd);
-        if (dict_ret < 0) {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "Not a pump pause command");
-                ret = _gf_false;
-                goto out;
-        }
-
-        gf_log (this->name, GF_LOG_DEBUG,
-                "Hit a pump command - pause");
-        ret = _gf_true;
-
-out:
-        return ret;
-
-}
-
-gf_boolean_t
-pump_command_commit (xlator_t *this, dict_t *dict)
-{
-        char *cmd = NULL;
-        int dict_ret = -1;
-        int ret = _gf_true;
-
-        dict_ret = dict_get_str (dict, RB_PUMP_CMD_COMMIT, &cmd);
-        if (dict_ret < 0) {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "Not a pump commit command");
-                ret = _gf_false;
-                goto out;
-        }
-
-        gf_log (this->name, GF_LOG_DEBUG,
-                "Hit a pump command - commit");
-        ret = _gf_true;
-
-out:
-        return ret;
-
-}
-
-gf_boolean_t
-pump_command_abort (xlator_t *this, dict_t *dict)
-{
-        char *cmd = NULL;
-        int dict_ret = -1;
-        int ret = _gf_true;
-
-        dict_ret = dict_get_str (dict, RB_PUMP_CMD_ABORT, &cmd);
-        if (dict_ret < 0) {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "Not a pump abort command");
-                ret = _gf_false;
-                goto out;
-        }
-
-        gf_log (this->name, GF_LOG_DEBUG,
-                "Hit a pump command - abort");
-        ret = _gf_true;
-
-out:
-        return ret;
-
-}
-
-gf_boolean_t
-pump_command_start (xlator_t *this, dict_t *dict)
-{
-        char *cmd = NULL;
-        int dict_ret = -1;
-        int ret = _gf_true;
-
-        dict_ret = dict_get_str (dict, RB_PUMP_CMD_START, &cmd);
-        if (dict_ret < 0) {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "Not a pump start command");
-                ret = _gf_false;
-                goto out;
-        }
-
-        gf_log (this->name, GF_LOG_DEBUG,
-                "Hit a pump command - start");
-        ret = _gf_true;
-
-out:
-        return ret;
-
-}
-
-int
-pump_getxattr (call_frame_t *frame, xlator_t *this, loc_t *loc,
-	       const char *name, dict_t *xdata)
-{
-	afr_private_t *priv = NULL;
-	int op_errno = 0;
-	int ret = 0;
-
-	priv = this->private;
-
-        if (!priv->use_afr_in_pump) {
-                STACK_WIND (frame, default_getxattr_cbk,
-                            FIRST_CHILD (this),
-                            (FIRST_CHILD (this))->fops->getxattr,
-                            loc, name, xdata);
-                return 0;
-        }
-
-        if (name) {
-                if (!strncmp (name, AFR_XATTR_PREFIX,
-                              strlen (AFR_XATTR_PREFIX))) {
-
-                        op_errno = ENODATA;
-                        ret = -1;
-                        goto out;
-                }
-
-                if (!strcmp (name, RB_PUMP_CMD_STATUS)) {
-                        gf_log (this->name, GF_LOG_DEBUG,
-                                "Hit pump command - status");
-                        pump_execute_status (frame, this);
-                        goto out;
-                }
-        }
-
-	afr_getxattr (frame, this, loc, name, xdata);
-
-out:
-	if (ret < 0)
-		AFR_STACK_UNWIND (getxattr, frame, -1, op_errno, NULL, NULL);
-	return 0;
-}
-
-int
-pump_command_reply (call_frame_t *frame, xlator_t *this)
-{
-        afr_local_t *local = NULL;
-
-        local = frame->local;
-
-        if (local->op_ret < 0)
-                gf_log (this->name, GF_LOG_INFO,
-                        "Command failed");
-        else
-                gf_log (this->name, GF_LOG_INFO,
-                        "Command succeeded");
-
-        AFR_STACK_UNWIND (setxattr,
-                          frame,
-                          local->op_ret,
-                          local->op_errno, NULL);
-
-        return 0;
-}
-
-int
-pump_parse_command (call_frame_t *frame, xlator_t *this, dict_t *dict,
-		    int *op_errno_p)
-{
-	afr_local_t *local = NULL;
-        int ret = -1;
-	int op_errno = 0;
-
-        if (pump_command_start (this, dict)) {
-                local = AFR_FRAME_INIT (frame, op_errno);
-		if (!local)
-			goto out;
-                local->dict = dict_ref (dict);
-                ret = pump_execute_start (frame, this);
-
-        } else if (pump_command_pause (this, dict)) {
-		local = AFR_FRAME_INIT (frame, op_errno);
-		if (!local)
-			goto out;
-                local->dict = dict_ref (dict);
-                ret = pump_execute_pause (frame, this);
-
-        } else if (pump_command_abort (this, dict)) {
-		local = AFR_FRAME_INIT (frame, op_errno);
-		if (!local)
-			goto out;
-                local->dict = dict_ref (dict);
-                ret = pump_execute_abort (frame, this);
-
-        } else if (pump_command_commit (this, dict)) {
-		local = AFR_FRAME_INIT (frame, op_errno);
-		if (!local)
-			goto out;
-                local->dict = dict_ref (dict);
-                ret = pump_execute_commit (frame, this);
-        }
-out:
-	if (op_errno_p)
-		*op_errno_p = op_errno;
-        return ret;
-}
-
-int
-pump_setxattr (call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *dict,
-	       int32_t flags, dict_t *xdata)
-{
-	afr_private_t *priv = NULL;
-	int ret = -1;
-	int op_errno = 0;
-
-        GF_IF_INTERNAL_XATTR_GOTO ("trusted.glusterfs.pump*", dict, op_errno, out);
-
-	priv = this->private;
-        if (!priv->use_afr_in_pump) {
-                STACK_WIND (frame, default_setxattr_cbk,
-                            FIRST_CHILD (this),
-                            (FIRST_CHILD (this))->fops->setxattr,
-                            loc, dict, flags, xdata);
-                return 0;
-        }
-
-        ret = pump_parse_command (frame, this, dict, &op_errno);
-        if (ret >= 0)
-                goto out;
-
-	afr_setxattr (frame, this, loc, dict, flags, xdata);
-
-	ret = 0;
-out:
-	if (ret < 0) {
-		AFR_STACK_UNWIND (setxattr, frame, -1, op_errno, NULL);
-	}
-
-	return 0;
-}
-
-/* Defaults */
-static int32_t
-pump_lookup (call_frame_t *frame,
-             xlator_t *this,
-             loc_t *loc,
-             dict_t *xattr_req)
-{
-	afr_private_t *priv  = NULL;
-	priv = this->private;
-        if (!priv->use_afr_in_pump) {
-                STACK_WIND (frame,
-                            default_lookup_cbk,
-                            FIRST_CHILD(this),
-                            FIRST_CHILD(this)->fops->lookup,
-                            loc,
-                            xattr_req);
-                return 0;
-        }
-
-        afr_lookup (frame, this, loc, xattr_req);
-        return 0;
-}
-
-
-static int32_t
-pump_truncate (call_frame_t *frame,
-               xlator_t *this,
-               loc_t *loc,
-               off_t offset, dict_t *xdata)
-{
-        afr_private_t *priv  = NULL;
-	priv = this->private;
-        if (!priv->use_afr_in_pump) {
-                STACK_WIND (frame,
-                            default_truncate_cbk,
-                            FIRST_CHILD(this),
-                            FIRST_CHILD(this)->fops->truncate,
-                            loc,
-                            offset, xdata);
-                return 0;
-        }
-
-        afr_truncate (frame, this, loc, offset, xdata);
-        return 0;
-}
-
-
-static int32_t
-pump_ftruncate (call_frame_t *frame,
-                xlator_t *this,
-                fd_t *fd,
-                off_t offset, dict_t *xdata)
-{
-        afr_private_t *priv  = NULL;
-	priv = this->private;
-        if (!priv->use_afr_in_pump) {
-                STACK_WIND (frame,
-                            default_ftruncate_cbk,
-                            FIRST_CHILD(this),
-                            FIRST_CHILD(this)->fops->ftruncate,
-                            fd,
-                            offset, xdata);
-                return 0;
-        }
-
-        afr_ftruncate (frame, this, fd, offset, xdata);
-        return 0;
-}
-
-
-
-
-int
-pump_mknod (call_frame_t *frame, xlator_t *this,
-            loc_t *loc, mode_t mode, dev_t rdev, mode_t umask, dict_t *xdata)
-{
-        afr_private_t *priv  = NULL;
-	priv = this->private;
-        if (!priv->use_afr_in_pump) {
-                STACK_WIND (frame, default_mknod_cbk,
-                            FIRST_CHILD(this),
-                            FIRST_CHILD(this)->fops->mknod,
-                            loc, mode, rdev, umask, xdata);
-                return 0;
-        }
-        afr_mknod (frame, this, loc, mode, rdev, umask, xdata);
-        return 0;
-
-}
-
-
-
-int
-pump_mkdir (call_frame_t *frame, xlator_t *this,
-            loc_t *loc, mode_t mode, mode_t umask, dict_t *xdata)
-{
-        afr_private_t *priv  = NULL;
-	priv = this->private;
-        if (!priv->use_afr_in_pump) {
-                STACK_WIND (frame, default_mkdir_cbk,
-                            FIRST_CHILD(this),
-                            FIRST_CHILD(this)->fops->mkdir,
-                            loc, mode, umask, xdata);
-                return 0;
-        }
-        afr_mkdir (frame, this, loc, mode, umask, xdata);
-        return 0;
-
-}
-
-
-static int32_t
-pump_unlink (call_frame_t *frame,
-             xlator_t *this,
-             loc_t *loc, int xflag, dict_t *xdata)
-{
-        afr_private_t *priv  = NULL;
-	priv = this->private;
-        if (!priv->use_afr_in_pump) {
-                STACK_WIND (frame,
-                            default_unlink_cbk,
-                            FIRST_CHILD(this),
-                            FIRST_CHILD(this)->fops->unlink,
-                            loc, xflag, xdata);
-                return 0;
-        }
-        afr_unlink (frame, this, loc, xflag, xdata);
-        return 0;
-
-}
-
-
-static int
-pump_rmdir (call_frame_t *frame, xlator_t *this,
-            loc_t *loc, int flags, dict_t *xdata)
-{
-        afr_private_t *priv  = NULL;
-
-	priv = this->private;
-
-        if (!priv->use_afr_in_pump) {
-                STACK_WIND (frame, default_rmdir_cbk,
-                            FIRST_CHILD(this),
-                            FIRST_CHILD(this)->fops->rmdir,
-                            loc, flags, xdata);
-                return 0;
-        }
-
-        afr_rmdir (frame, this, loc, flags, xdata);
-        return 0;
-
-}
-
-
-
-int
-pump_symlink (call_frame_t *frame, xlator_t *this,
-              const char *linkpath, loc_t *loc, mode_t umask, dict_t *xdata)
-{
-        afr_private_t *priv  = NULL;
-	priv = this->private;
-        if (!priv->use_afr_in_pump) {
-                STACK_WIND (frame, default_symlink_cbk,
-                            FIRST_CHILD(this),
-                            FIRST_CHILD(this)->fops->symlink,
-                            linkpath, loc, umask, xdata);
-                return 0;
-        }
-        afr_symlink (frame, this, linkpath, loc, umask, xdata);
-        return 0;
-
-}
-
-
-static int32_t
-pump_rename (call_frame_t *frame,
-             xlator_t *this,
-             loc_t *oldloc,
-             loc_t *newloc, dict_t *xdata)
-{
-        afr_private_t *priv  = NULL;
-	priv = this->private;
-        if (!priv->use_afr_in_pump) {
-                STACK_WIND (frame,
-                            default_rename_cbk,
-                            FIRST_CHILD(this),
-                            FIRST_CHILD(this)->fops->rename,
-                            oldloc, newloc, xdata);
-                return 0;
-        }
-        afr_rename (frame, this, oldloc, newloc, xdata);
-        return 0;
-
-}
-
-
-static int32_t
-pump_link (call_frame_t *frame,
-           xlator_t *this,
-           loc_t *oldloc,
-           loc_t *newloc, dict_t *xdata)
-{
-        afr_private_t *priv  = NULL;
-	priv = this->private;
-        if (!priv->use_afr_in_pump) {
-                STACK_WIND (frame,
-                            default_link_cbk,
-                            FIRST_CHILD(this),
-                            FIRST_CHILD(this)->fops->link,
-                            oldloc, newloc, xdata);
-                return 0;
-        }
-        afr_link (frame, this, oldloc, newloc, xdata);
-        return 0;
-
-}
-
-
-static int32_t
-pump_create (call_frame_t *frame, xlator_t *this,
-             loc_t *loc, int32_t flags, mode_t mode,
-             mode_t umask, fd_t *fd, dict_t *xdata)
-{
-        afr_private_t *priv  = NULL;
-	priv = this->private;
-        if (!priv->use_afr_in_pump) {
-                STACK_WIND (frame, default_create_cbk,
-                            FIRST_CHILD(this),
-                            FIRST_CHILD(this)->fops->create,
-                            loc, flags, mode, umask, fd, xdata);
-                return 0;
-        }
-        afr_create (frame, this, loc, flags, mode, umask, fd, xdata);
-        return 0;
-
-}
-
-
-static int32_t
-pump_open (call_frame_t *frame,
-           xlator_t *this,
-           loc_t *loc,
-           int32_t flags, fd_t *fd, dict_t *xdata)
-{
-        afr_private_t *priv  = NULL;
-	priv = this->private;
-        if (!priv->use_afr_in_pump) {
-                STACK_WIND (frame,
-                            default_open_cbk,
-                            FIRST_CHILD(this),
-                            FIRST_CHILD(this)->fops->open,
-                            loc, flags, fd, xdata);
-                return 0;
-        }
-        afr_open (frame, this, loc, flags, fd, xdata);
-        return 0;
-
-}
-
-
-static int32_t
-pump_writev (call_frame_t *frame,
-             xlator_t *this,
-             fd_t *fd,
-             struct iovec *vector,
-             int32_t count,
-             off_t off, uint32_t flags,
-             struct iobref *iobref, dict_t *xdata)
-{
-        afr_private_t *priv  = NULL;
-	priv = this->private;
-        if (!priv->use_afr_in_pump) {
-                STACK_WIND (frame,
-                            default_writev_cbk,
-                            FIRST_CHILD(this),
-                            FIRST_CHILD(this)->fops->writev,
-                            fd,
-                            vector,
-                            count,
-                            off, flags,
-                            iobref, xdata);
-                return 0;
-        }
-
-        afr_writev (frame, this, fd, vector, count, off, flags, iobref, xdata);
-        return 0;
-}
-
-
-static int32_t
-pump_flush (call_frame_t *frame,
-            xlator_t *this,
-            fd_t *fd, dict_t *xdata)
-{
-        afr_private_t *priv  = NULL;
-	priv = this->private;
-        if (!priv->use_afr_in_pump) {
-                STACK_WIND (frame,
-                            default_flush_cbk,
-                            FIRST_CHILD(this),
-                            FIRST_CHILD(this)->fops->flush,
-                            fd, xdata);
-                return 0;
-        }
-        afr_flush (frame, this, fd, xdata);
-        return 0;
-
-}
-
-
-static int32_t
-pump_fsync (call_frame_t *frame,
-            xlator_t *this,
-            fd_t *fd,
-            int32_t flags, dict_t *xdata)
-{
-        afr_private_t *priv  = NULL;
-	priv = this->private;
-        if (!priv->use_afr_in_pump) {
-                STACK_WIND (frame,
-                            default_fsync_cbk,
-                            FIRST_CHILD(this),
-                            FIRST_CHILD(this)->fops->fsync,
-                            fd,
-                            flags, xdata);
-                return 0;
-        }
-        afr_fsync (frame, this, fd, flags, xdata);
-        return 0;
-
-}
-
-
-static int32_t
-pump_opendir (call_frame_t *frame,
-              xlator_t *this,
-              loc_t *loc, fd_t *fd, dict_t *xdata)
-{
-        afr_private_t *priv  = NULL;
-	priv = this->private;
-        if (!priv->use_afr_in_pump) {
-                STACK_WIND (frame,
-                            default_opendir_cbk,
-                            FIRST_CHILD(this),
-                            FIRST_CHILD(this)->fops->opendir,
-                            loc, fd, xdata);
-                return 0;
-        }
-        afr_opendir (frame, this, loc, fd, xdata);
-        return 0;
-
-}
-
-
-static int32_t
-pump_fsyncdir (call_frame_t *frame,
-               xlator_t *this,
-               fd_t *fd,
-               int32_t flags, dict_t *xdata)
-{
-        afr_private_t *priv  = NULL;
-	priv = this->private;
-        if (!priv->use_afr_in_pump) {
-                STACK_WIND (frame,
-                            default_fsyncdir_cbk,
-                            FIRST_CHILD(this),
-                            FIRST_CHILD(this)->fops->fsyncdir,
-                            fd,
-                            flags, xdata);
-                return 0;
-        }
-        afr_fsyncdir (frame, this, fd, flags, xdata);
-        return 0;
-
-}
-
-
-static int32_t
-pump_xattrop (call_frame_t *frame,
-              xlator_t *this,
-              loc_t *loc,
-              gf_xattrop_flags_t flags,
-              dict_t *dict, dict_t *xdata)
-{
-        afr_private_t *priv  = NULL;
-	priv = this->private;
-        if (!priv->use_afr_in_pump) {
-                STACK_WIND (frame,
-                            default_xattrop_cbk,
-                            FIRST_CHILD(this),
-                            FIRST_CHILD(this)->fops->xattrop,
-                            loc,
-                            flags,
-                            dict, xdata);
-                return 0;
-        }
-        afr_xattrop (frame, this, loc, flags, dict, xdata);
-        return 0;
-
-}
-
-static int32_t
-pump_fxattrop (call_frame_t *frame,
-               xlator_t *this,
-               fd_t *fd,
-               gf_xattrop_flags_t flags,
-               dict_t *dict, dict_t *xdata)
-{
-        afr_private_t *priv  = NULL;
-	priv = this->private;
-        if (!priv->use_afr_in_pump) {
-                STACK_WIND (frame,
-                            default_fxattrop_cbk,
-                            FIRST_CHILD(this),
-                            FIRST_CHILD(this)->fops->fxattrop,
-                            fd,
-                            flags,
-                            dict, xdata);
-                return 0;
-        }
-        afr_fxattrop (frame, this, fd, flags, dict, xdata);
-        return 0;
-
-}
-
-
-static int32_t
-pump_removexattr (call_frame_t *frame,
-                  xlator_t *this,
-                  loc_t *loc,
-                  const char *name, dict_t *xdata)
-{
-        afr_private_t *priv     = NULL;
-        int            op_errno = -1;
-
-        VALIDATE_OR_GOTO (this, out);
-
-        GF_IF_NATIVE_XATTR_GOTO ("trusted.glusterfs.pump*",
-                                 name, op_errno, out);
-
-        op_errno = 0;
-	priv = this->private;
-        if (!priv->use_afr_in_pump) {
-                STACK_WIND (frame,
-                            default_removexattr_cbk,
-                            FIRST_CHILD(this),
-                            FIRST_CHILD(this)->fops->removexattr,
-                            loc,
-                            name, xdata);
-                return 0;
-        }
-        afr_removexattr (frame, this, loc, name, xdata);
-
- out:
-        if (op_errno)
-                AFR_STACK_UNWIND (removexattr, frame, -1, op_errno, NULL);
-        return 0;
-
-}
-
-
-
-static int32_t
-pump_readdir (call_frame_t *frame,
-              xlator_t *this,
-              fd_t *fd,
-              size_t size,
-              off_t off, dict_t *xdata)
-{
-        afr_private_t *priv  = NULL;
-	priv = this->private;
-        if (!priv->use_afr_in_pump) {
-                STACK_WIND (frame,
-                            default_readdir_cbk,
-                            FIRST_CHILD(this),
-                            FIRST_CHILD(this)->fops->readdir,
-                            fd, size, off, xdata);
-                return 0;
-        }
-        afr_readdir (frame, this, fd, size, off, xdata);
-        return 0;
-
-}
-
-
-static int32_t
-pump_readdirp (call_frame_t *frame, xlator_t *this, fd_t *fd,
-               size_t size, off_t off, dict_t *dict)
-{
-        afr_private_t *priv  = NULL;
-	priv = this->private;
-        if (!priv->use_afr_in_pump) {
-                STACK_WIND (frame,
-                            default_readdirp_cbk,
-                            FIRST_CHILD(this),
-                            FIRST_CHILD(this)->fops->readdirp,
-                            fd, size, off, dict);
-                return 0;
-        }
-        afr_readdirp (frame, this, fd, size, off, dict);
-        return 0;
-
-}
-
-
-
-static int32_t
-pump_releasedir (xlator_t *this,
-                 fd_t *fd)
-{
-        afr_private_t *priv  = NULL;
-	priv = this->private;
-        if (priv->use_afr_in_pump)
-                afr_releasedir (this, fd);
-	return 0;
-
-}
-
-static int32_t
-pump_release (xlator_t *this,
-              fd_t *fd)
-{
-        afr_private_t *priv  = NULL;
-	priv = this->private;
-        if (priv->use_afr_in_pump)
-                afr_release (this, fd);
-	return 0;
-
-}
-
-static int32_t
-pump_forget (xlator_t *this, inode_t *inode)
-{
-        afr_private_t  *priv  = NULL;
-
-        priv = this->private;
-        if (priv->use_afr_in_pump)
-                afr_forget (this, inode);
-
-        return 0;
-}
-
-static int32_t
-pump_setattr (call_frame_t *frame,
-              xlator_t *this,
-              loc_t *loc,
-              struct iatt *stbuf,
-              int32_t valid, dict_t *xdata)
-{
-        afr_private_t *priv  = NULL;
-	priv = this->private;
-        if (!priv->use_afr_in_pump) {
-                STACK_WIND (frame,
-                            default_setattr_cbk,
-                            FIRST_CHILD (this),
-                            FIRST_CHILD (this)->fops->setattr,
-                            loc, stbuf, valid, xdata);
-                return 0;
-        }
-        afr_setattr (frame, this, loc, stbuf, valid, xdata);
-        return 0;
-
-}
-
-
-static int32_t
-pump_fsetattr (call_frame_t *frame,
-               xlator_t *this,
-               fd_t *fd,
-               struct iatt *stbuf,
-               int32_t valid, dict_t *xdata)
-{
-        afr_private_t *priv  = NULL;
-	priv = this->private;
-        if (!priv->use_afr_in_pump) {
-                STACK_WIND (frame,
-                            default_fsetattr_cbk,
-                            FIRST_CHILD (this),
-                            FIRST_CHILD (this)->fops->fsetattr,
-                            fd, stbuf, valid, xdata);
-                return 0;
-        }
-        afr_fsetattr (frame, this, fd, stbuf, valid, xdata);
-        return 0;
-
-}
-
-
-/* End of defaults */
-
-
-int32_t
-mem_acct_init (xlator_t *this)
-{
-        int     ret = -1;
-
-        if (!this)
-                return ret;
-
-        ret = xlator_mem_acct_init (this, gf_afr_mt_end + 1);
-
-        if (ret != 0) {
-                gf_log(this->name, GF_LOG_ERROR, "Memory accounting init"
-                                "failed");
-                return ret;
-        }
-
-        return ret;
-}
-
-static int
-is_xlator_pump_sink (xlator_t *child)
-{
-        return (child == PUMP_SINK_CHILD(THIS));
-}
-
-static int
-is_xlator_pump_source (xlator_t *child)
-{
-        return (child == PUMP_SOURCE_CHILD(THIS));
-}
-
-int32_t
-notify (xlator_t *this, int32_t event,
-	void *data, ...)
-{
-        int ret = -1;
-        xlator_t *child_xl = NULL;
-
-        child_xl = (xlator_t *) data;
-
-        ret = afr_notify (this, event, data, NULL);
-
-	switch (event) {
-	case GF_EVENT_CHILD_DOWN:
-                if (is_xlator_pump_source (child_xl))
-                        pump_change_state (this, PUMP_STATE_ABORT);
-                break;
-
-        case GF_EVENT_CHILD_UP:
-                if (is_xlator_pump_sink (child_xl))
-                        if (is_pump_start_pending (this)) {
-                                gf_log (this->name, GF_LOG_DEBUG,
-                                        "about to start synctask");
-                                ret = pump_start_synctask (this);
-                                if (ret < 0)
-                                        gf_log (this->name, GF_LOG_DEBUG,
-                                                "Could not start pump "
-                                                "synctask");
-                                else
-                                        pump_remove_start_pending (this);
-                        }
-        }
-
-        return ret;
-}
-
-int32_t
-init (xlator_t *this)
-{
-	afr_private_t * priv        = NULL;
-        pump_private_t *pump_priv   = NULL;
-	int             child_count = 0;
-	xlator_list_t * trav        = NULL;
-	int             i           = 0;
-	int             ret         = -1;
-	GF_UNUSED int   op_errno    = 0;
-
-        int source_child = 0;
-
-	if (!this->children) {
-		gf_log (this->name, GF_LOG_ERROR,
-			"pump translator needs a source and sink"
-                        "subvolumes defined.");
-		return -1;
-	}
-
-	if (!this->parents) {
-		gf_log (this->name, GF_LOG_WARNING,
-			"Volume is dangling.");
-	}
-
-	priv = GF_CALLOC (1, sizeof (afr_private_t), gf_afr_mt_afr_private_t);
-        if (!priv)
-                goto out;
-
-        LOCK_INIT (&priv->lock);
-
-        child_count = xlator_subvolume_count (this);
-        if (child_count != 2) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "There should be exactly 2 children - one source "
-                        "and one sink");
-                return -1;
-        }
-	priv->child_count = child_count;
-
-        priv->read_child = source_child;
-        priv->favorite_child = source_child;
-        priv->background_self_heal_count = 0;
-
-	priv->data_self_heal     = "on";
-	priv->metadata_self_heal = 1;
-	priv->entry_self_heal    = 1;
-
-        priv->data_self_heal_window_size = 16;
-
-	priv->data_change_log     = 1;
-	priv->metadata_change_log = 1;
-	priv->entry_change_log    = 1;
-        priv->use_afr_in_pump = 1;
-        priv->sh_readdir_size = 65536;
-
-	/* Locking options */
-
-        /* Lock server count infact does not matter. Locks are held
-           on all subvolumes, in this case being the source
-           and the sink.
-        */
-
-	priv->child_up = GF_CALLOC (sizeof (unsigned char), child_count,
-                                 gf_afr_mt_char);
-	if (!priv->child_up) {
-		gf_log (this->name, GF_LOG_ERROR,
-			"Out of memory.");
-		op_errno = ENOMEM;
-		goto out;
-	}
-
-	priv->children = GF_CALLOC (sizeof (xlator_t *), child_count,
-                                 gf_afr_mt_xlator_t);
-	if (!priv->children) {
-		gf_log (this->name, GF_LOG_ERROR,
-			"Out of memory.");
-		op_errno = ENOMEM;
-		goto out;
-	}
-
-        priv->pending_key = GF_CALLOC (sizeof (*priv->pending_key),
-                                       child_count,
-                                       gf_afr_mt_char);
-        if (!priv->pending_key) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "Out of memory.");
-                op_errno = ENOMEM;
-                goto out;
-        }
-
-	trav = this->children;
-	i = 0;
-	while (i < child_count) {
-		priv->children[i] = trav->xlator;
-
-                ret = gf_asprintf (&priv->pending_key[i], "%s.%s",
-                                   AFR_XATTR_PREFIX,
-                                   trav->xlator->name);
-                if (-1 == ret) {
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "asprintf failed to set pending key");
-                        op_errno = ENOMEM;
-                        goto out;
-                }
-
-		trav = trav->next;
-		i++;
-	}
-
-        ret = gf_asprintf (&priv->sh_domain, "%s-self-heal", this->name);
-        if (-1 == ret) {
-                op_errno = ENOMEM;
-                goto out;
-        }
-
-        priv->root_inode = NULL;
-
-        priv->last_event = GF_CALLOC (child_count, sizeof (*priv->last_event),
-                                      gf_afr_mt_int32_t);
-        if (!priv->last_event) {
-                ret = -ENOMEM;
-                goto out;
-        }
-
-	pump_priv = GF_CALLOC (1, sizeof (*pump_priv),
-                            gf_afr_mt_pump_priv);
-	if (!pump_priv) {
-		gf_log (this->name, GF_LOG_ERROR,
-			"Out of memory");
-                op_errno = ENOMEM;
-		goto out;
-	}
-
-        LOCK_INIT (&pump_priv->resume_path_lock);
-        LOCK_INIT (&pump_priv->pump_state_lock);
-
-        pump_priv->resume_path = GF_CALLOC (1, PATH_MAX,
-                                            gf_afr_mt_char);
-        if (!pump_priv->resume_path) {
-                gf_log (this->name, GF_LOG_ERROR, "Out of memory");
-                ret = -1;
-                goto out;
-        }
-
-	pump_priv->env = this->ctx->env;
-        if (!pump_priv->env) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "Could not create new sync-environment");
-                ret = -1;
-                goto out;
-        }
-
-        /* keep more local here as we may need them for self-heal etc */
-        this->local_pool = mem_pool_new (afr_local_t, 128);
-        if (!this->local_pool) {
-                ret = -1;
-                gf_log (this->name, GF_LOG_ERROR,
-                        "failed to create local_t's memory pool");
-                goto out;
-        }
-
-	priv->pump_private = pump_priv;
-        pump_priv = NULL;
-
-        this->private = priv;
-        priv = NULL;
-
-        pump_change_state (this, PUMP_STATE_ABORT);
-
-	ret = 0;
-out:
-
-        if (pump_priv) {
-                GF_FREE (pump_priv->resume_path);
-                LOCK_DESTROY (&pump_priv->resume_path_lock);
-                LOCK_DESTROY (&pump_priv->pump_state_lock);
-                GF_FREE (pump_priv);
-        }
-
-        if (priv) {
-                GF_FREE (priv->child_up);
-                GF_FREE (priv->children);
-                GF_FREE (priv->pending_key);
-                GF_FREE (priv->last_event);
-                LOCK_DESTROY (&priv->lock);
-                GF_FREE (priv);
-        }
-
-	return ret;
-}
-
-int
-fini (xlator_t *this)
-{
-        afr_private_t * priv        = NULL;
-        pump_private_t *pump_priv = NULL;
-
-        priv      = this->private;
-        this->private = NULL;
-        if (!priv)
-                goto out;
-
-        pump_priv = priv->pump_private;
-        if (!pump_priv)
-                goto afr_priv;
-
-        GF_FREE (pump_priv->resume_path);
-        LOCK_DESTROY (&pump_priv->resume_path_lock);
-        LOCK_DESTROY (&pump_priv->pump_state_lock);
-        GF_FREE (pump_priv);
-afr_priv:
-        afr_priv_destroy (priv);
-out:
-	return 0;
-}
-
-
-struct xlator_fops fops = {
-	.lookup      = pump_lookup,
-	.open        = pump_open,
-	.flush       = pump_flush,
-	.fsync       = pump_fsync,
-	.fsyncdir    = pump_fsyncdir,
-	.xattrop     = pump_xattrop,
-	.fxattrop    = pump_fxattrop,
-        .getxattr    = pump_getxattr,
-
-	/* inode write */
-	.writev      = pump_writev,
-	.truncate    = pump_truncate,
-	.ftruncate   = pump_ftruncate,
-	.setxattr    = pump_setxattr,
-        .setattr     = pump_setattr,
-	.fsetattr    = pump_fsetattr,
-	.removexattr = pump_removexattr,
-
-	/* dir read */
-	.opendir     = pump_opendir,
-	.readdir     = pump_readdir,
-	.readdirp    = pump_readdirp,
-
-	/* dir write */
-	.create      = pump_create,
-	.mknod       = pump_mknod,
-	.mkdir       = pump_mkdir,
-	.unlink      = pump_unlink,
-	.rmdir       = pump_rmdir,
-	.link        = pump_link,
-	.symlink     = pump_symlink,
-	.rename      = pump_rename,
-};
-
-struct xlator_dumpops dumpops = {
-        .priv       = afr_priv_dump,
-};
-
-
-struct xlator_cbks cbks = {
-	.release     = pump_release,
-	.releasedir  = pump_releasedir,
-        .forget      = pump_forget,
-};
-
-struct volume_options options[] = {
-	{ .key  = {NULL} },
-};
diff --git a/xlators/cluster/afr/src/pump.h b/xlators/cluster/afr/src/pump.h
deleted file mode 100644
index 7d5acd02bf6..00000000000
--- a/xlators/cluster/afr/src/pump.h
+++ /dev/null
@@ -1,81 +0,0 @@
-/*
-  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
-  This file is part of GlusterFS.
-
-  This file is licensed to you under your choice of the GNU Lesser
-  General Public License, version 3 or any later version (LGPLv3 or
-  later), or the GNU General Public License, version 2 (GPLv2), in all
-  cases as published by the Free Software Foundation.
-*/
-
-#ifndef __PUMP_H__
-#define __PUMP_H__
-
-#include "syncop.h"
-
-/* FIXME: Needs to be defined in a common file */
-#define CLIENT_CMD_CONNECT "trusted.glusterfs.client-connect"
-#define CLIENT_CMD_DISCONNECT "trusted.glusterfs.client-disconnect"
-
-#define PUMP_SOURCE_COMPLETE "trusted.glusterfs.pump-source-complete"
-#define PUMP_SINK_COMPLETE "trusted.glusterfs.pump-sink-complete"
-
-#define PUMP_PATH "trusted.glusterfs.pump-path"
-
-#define PUMP_SOURCE_CHILD(xl) (xl->children->xlator)
-#define PUMP_SINK_CHILD(xl) (xl->children->next->xlator)
-
-typedef enum {
-        PUMP_STATE_RUNNING,             /* Pump is running and migrating files */
-        PUMP_STATE_RESUME,              /* Pump is resuming from a previous pause */
-        PUMP_STATE_PAUSE,               /* Pump is paused */
-        PUMP_STATE_ABORT,               /* Pump is aborted */
-        PUMP_STATE_COMMIT,              /* Pump is committed */
-} pump_state_t;
-
-typedef struct _pump_private {
-	struct syncenv *env;            /* The env pointer to the pump synctask */
-        char *resume_path;              /* path to resume from the last pause */
-        gf_lock_t resume_path_lock;     /* Synchronize resume_path changes */
-        gf_lock_t pump_state_lock;      /* Synchronize pump_state changes */
-        pump_state_t pump_state;        /* State of pump */
-        char current_file[PATH_MAX];    /* Current file being pumped */
-        uint64_t number_files_pumped;   /* Number of files pumped */
-        gf_boolean_t pump_finished;     /* Boolean to indicate pump termination */
-        char pump_start_pending;        /* Boolean to mark start pending until
-                                           CHILD_UP */
-        call_stub_t *cleaner;
-} pump_private_t;
-
-void
-build_root_loc (inode_t *inode, loc_t *loc);
-int pump_start (call_frame_t *frame, xlator_t *this);
-
-gf_boolean_t
-pump_command_start (xlator_t *this, dict_t *dict);
-
-int
-pump_execute_start (call_frame_t *frame, xlator_t *this);
-
-gf_boolean_t
-pump_command_pause (xlator_t *this, dict_t *dict);
-
-int
-pump_execute_pause (call_frame_t *frame, xlator_t *this);
-
-gf_boolean_t
-pump_command_abort (xlator_t *this, dict_t *dict);
-
-int
-pump_execute_abort (call_frame_t *frame, xlator_t *this);
-
-gf_boolean_t
-pump_command_status (xlator_t *this, dict_t *dict);
-
-int
-pump_execute_status (call_frame_t *frame, xlator_t *this);
-
-int
-pump_command_reply (call_frame_t *frame, xlator_t *this);
-
-#endif /* __PUMP_H__ */
diff --git a/xlators/cluster/dht/src/Makefile.am b/xlators/cluster/dht/src/Makefile.am
index a8e1ec0d286..56f1f2ad7c8 100644
--- a/xlators/cluster/dht/src/Makefile.am
+++ b/xlators/cluster/dht/src/Makefile.am
@@ -1,7 +1,4 @@
 xlator_LTLIBRARIES = dht.la nufa.la switch.la
-if BUILD_GFDB
-   xlator_LTLIBRARIES += tier.la
-endif
 
 AM_CFLAGS = -Wall $(GF_CFLAGS)
 
@@ -10,36 +7,31 @@ xlatordir = $(libdir)/glusterfs/$(PACKAGE_VERSION)/xlator/cluster
 dht_common_source = dht-layout.c dht-helper.c dht-linkfile.c dht-rebalance.c \
 	dht-selfheal.c dht-rename.c dht-hashfn.c dht-diskusage.c \
 	dht-common.c dht-inode-write.c dht-inode-read.c dht-shared.c \
-	$(top_builddir)/xlators/lib/src/libxlator.c
+	dht-lock.c $(top_builddir)/xlators/lib/src/libxlator.c
 
 dht_la_SOURCES = $(dht_common_source) dht.c
 
 nufa_la_SOURCES = $(dht_common_source) nufa.c
 switch_la_SOURCES = $(dht_common_source) switch.c
-tier_la_SOURCES = $(dht_common_source) tier.c
 
-dht_la_LDFLAGS = -module -avoid-version
+dht_la_LDFLAGS = -module $(GF_XLATOR_DEFAULT_LDFLAGS)
 dht_la_LIBADD = $(top_builddir)/libglusterfs/src/libglusterfs.la
 
-nufa_la_LDFLAGS = -module -avoid-version
+nufa_la_LDFLAGS = -module $(GF_XLATOR_DEFAULT_LDFLAGS)
 nufa_la_LIBADD = $(top_builddir)/libglusterfs/src/libglusterfs.la
 
-switch_la_LDFLAGS = -module -avoid-version
+switch_la_LDFLAGS = -module $(GF_XLATOR_DEFAULT_LDFLAGS)
 switch_la_LIBADD = $(top_builddir)/libglusterfs/src/libglusterfs.la
 
-
-tier_la_LDFLAGS = -module -avoid-version
-tier_la_LIBADD = $(top_builddir)/libglusterfs/src/libglusterfs.la
-
-noinst_HEADERS = dht-common.h dht-mem-types.h dht-messages.h dht-helper.h tier.h\
-	$(top_builddir)/xlators/lib/src/libxlator.h
+noinst_HEADERS = dht-common.h dht-mem-types.h dht-messages.h \
+	dht-lock.h $(top_builddir)/xlators/lib/src/libxlator.h
 
 AM_CPPFLAGS = $(GF_CPPFLAGS) -I$(top_srcdir)/libglusterfs/src \
-	-I$(top_srcdir)/libglusterfs/src/gfdb \
+	-I$(top_srcdir)/rpc/xdr/src -I$(top_builddir)/rpc/xdr/src \
+	-I$(top_srcdir)/rpc/rpc-lib/src \
 	-I$(top_srcdir)/xlators/lib/src \
 	-DDATADIR=\"$(localstatedir)\" \
-	-DLIBDIR=\"$(libdir)\" \
-	-DLIBGFDB_VERSION=\"$(LIBGFDB_VERSION)\"
+	-DLIBDIR=\"$(libdir)\"
 
 CLEANFILES =
 
@@ -53,13 +45,4 @@ if UNITTEST
 CLEANFILES += *.gcda *.gcno *_xunit.xml
 noinst_PROGRAMS =
 TESTS =
-
-dht_layout_unittest_CPPFLAGS = $(AM_CPPFLAGS)
-dht_layout_unittest_SOURCES = unittest/dht_layout_unittest.c \
-                              unittest/dht_layout_mock.c \
-                              dht-layout.c
-dht_layout_unittest_CFLAGS = $(AM_CFLAGS) $(UNITTEST_CFLAGS)
-dht_layout_unittest_LDFLAGS = $(UNITTEST_LDFLAGS)
-noinst_PROGRAMS += dht_layout_unittest
-TESTS += dht_layout_unittest
 endif
diff --git a/xlators/cluster/dht/src/dht-common.c b/xlators/cluster/dht/src/dht-common.c
index e0b159ace32..8ba0cc4c732 100644
--- a/xlators/cluster/dht/src/dht-common.c
+++ b/xlators/cluster/dht/src/dht-common.c
@@ -8,151 +8,439 @@
   cases as published by the Free Software Foundation.
 */
 
-
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
-
 /* TODO: add NS locking */
 
-#include "glusterfs.h"
-#include "xlator.h"
 #include "libxlator.h"
 #include "dht-common.h"
-#include "defaults.h"
-#include "byte-order.h"
-#include "glusterfs-acl.h"
-#include "quota-common-utils.h"
+#include "dht-lock.h"
+#include <glusterfs/byte-order.h>
+#include <glusterfs/quota-common-utils.h>
+#include <glusterfs/upcall-utils.h>
+#include "glusterfs/compat-errno.h"  // for ENODATA on BSD
+#include <glusterfs/common-utils.h>
 
 #include <sys/time.h>
 #include <libgen.h>
 #include <signal.h>
 
-int dht_link2 (xlator_t *this, call_frame_t *frame, int op_ret);
+static int
+dht_rmdir_readdirp_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                       int op_ret, int op_errno, gf_dirent_t *entries,
+                       dict_t *xdata);
 
-int
-dht_aggregate_quota_xattr (dict_t *dst, char *key, data_t *value)
+static int
+dht_link2(xlator_t *this, xlator_t *subvol, call_frame_t *frame, int ret);
+
+static int
+dht_set_dir_xattr_req(xlator_t *this, loc_t *loc, dict_t *xattr_req);
+
+static int
+dht_lookup_everywhere_done(call_frame_t *frame, xlator_t *this);
+
+static int
+dht_common_mark_mdsxattr_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                             int op_ret, int op_errno, dict_t *xdata);
+
+static int
+dht_rmdir_unlock(call_frame_t *frame, xlator_t *this);
+
+static const char *dht_dbg_vxattrs[] = {DHT_DBG_HASHED_SUBVOL_PATTERN, NULL};
+
+/* Check the xdata to make sure EBADF has been set by client xlator */
+int32_t
+dht_check_remote_fd_failed_error(dht_local_t *local, int op_ret, int op_errno)
 {
-        int              ret            = -1;
-        quota_meta_t    *meta_dst       = NULL;
-        quota_meta_t    *meta_src       = NULL;
-        int64_t         *size           = NULL;
-        int64_t          dst_dir_count  = 0;
-        int64_t          src_dir_count  = 0;
+    if (op_ret == -1 && (op_errno == EBADF || op_errno == EBADFD) &&
+        !(local->fd_checked)) {
+        return 1;
+    }
+    return 0;
+}
 
-        if (value == NULL) {
-                gf_log ("dht", GF_LOG_WARNING, "data value is NULL");
-                ret = -1;
-                goto out;
-        }
+/* Sets the blocks and size values to fixed values. This is to be called
+ * only for dirs. The caller is responsible for checking the type
+ */
+int32_t
+dht_set_fixed_dir_stat(struct iatt *stat)
+{
+    if (stat) {
+        stat->ia_blocks = DHT_DIR_STAT_BLOCKS;
+        stat->ia_size = DHT_DIR_STAT_SIZE;
+        return 0;
+    }
+    return -1;
+}
+
+/* Return true if key exists in array
+ */
+static gf_boolean_t
+dht_match_xattr(const char *key)
+{
+    char **xattrs_to_heal = get_xattrs_to_heal();
+
+    return gf_get_index_by_elem(xattrs_to_heal, (char *)key) >= 0;
+}
 
-        ret = dict_get_bin (dst, key, (void **)&meta_dst);
+static int
+dht_aggregate_quota_xattr(dict_t *dst, char *key, data_t *value)
+{
+    int ret = -1;
+    quota_meta_t *meta_dst = NULL;
+    quota_meta_t *meta_src = NULL;
+    int64_t *size = NULL;
+    int64_t dst_dir_count = 0;
+    int64_t src_dir_count = 0;
+
+    if (value == NULL) {
+        gf_msg("dht", GF_LOG_WARNING, 0, DHT_MSG_DATA_NULL,
+               "data value is NULL");
+        ret = -1;
+        goto out;
+    }
+
+    ret = dict_get_bin(dst, key, (void **)&meta_dst);
+    if (ret < 0) {
+        meta_dst = GF_CALLOC(1, sizeof(quota_meta_t), gf_common_quota_meta_t);
+        if (meta_dst == NULL) {
+            gf_msg("dht", GF_LOG_WARNING, ENOMEM, DHT_MSG_NO_MEMORY,
+                   "Memory allocation failed");
+            ret = -1;
+            goto out;
+        }
+        ret = dict_set_bin(dst, key, meta_dst, sizeof(quota_meta_t));
         if (ret < 0) {
-                meta_dst = GF_CALLOC (1, sizeof (quota_meta_t),
-                                      gf_common_quota_meta_t);
-                if (meta_dst == NULL) {
-                        gf_msg ("dht", GF_LOG_WARNING, 0, DHT_MSG_NO_MEMORY,
-                                "Memory allocation failed");
-                        ret = -1;
-                        goto out;
-                }
-                ret = dict_set_bin (dst, key, meta_dst,
-                                    sizeof (quota_meta_t));
-                if (ret < 0) {
-                        gf_log ("dht", GF_LOG_WARNING,
-                                "dht aggregate dict set failed");
-                        GF_FREE (meta_dst);
-                        ret = -1;
-                        goto out;
-                }
+            gf_msg("dht", GF_LOG_WARNING, EINVAL, DHT_MSG_DICT_SET_FAILED,
+                   "dht aggregate dict set failed");
+            GF_FREE(meta_dst);
+            ret = -1;
+            goto out;
         }
+    }
 
-        if (value->len > sizeof (int64_t)) {
-                meta_src = data_to_bin (value);
+    if (value->len > sizeof(int64_t)) {
+        meta_src = data_to_bin(value);
 
-                meta_dst->size = hton64 (ntoh64 (meta_dst->size) +
-                                         ntoh64 (meta_src->size));
-                meta_dst->file_count = hton64 (ntoh64 (meta_dst->file_count) +
-                                               ntoh64 (meta_src->file_count));
+        meta_dst->size = hton64(ntoh64(meta_dst->size) +
+                                ntoh64(meta_src->size));
+        meta_dst->file_count = hton64(ntoh64(meta_dst->file_count) +
+                                      ntoh64(meta_src->file_count));
 
-                if (value->len > (2 * sizeof (int64_t))) {
-                        dst_dir_count = ntoh64 (meta_dst->dir_count);
-                        src_dir_count = ntoh64 (meta_src->dir_count);
+        if (value->len > (2 * sizeof(int64_t))) {
+            dst_dir_count = ntoh64(meta_dst->dir_count);
+            src_dir_count = ntoh64(meta_src->dir_count);
 
-                        if (src_dir_count > dst_dir_count)
-                                meta_dst->dir_count = meta_src->dir_count;
-                } else {
-                        meta_dst->dir_count = 0;
-                }
+            if (src_dir_count > dst_dir_count)
+                meta_dst->dir_count = meta_src->dir_count;
         } else {
-                size = data_to_bin (value);
-                meta_dst->size = hton64 (ntoh64 (meta_dst->size) +
-                                         ntoh64 (*size));
+            meta_dst->dir_count = 0;
         }
+    } else {
+        size = data_to_bin(value);
+        meta_dst->size = hton64(ntoh64(meta_dst->size) + ntoh64(*size));
+    }
 
-        ret = 0;
+    ret = 0;
 out:
-        return ret;
+    return ret;
 }
 
-int
-dht_aggregate (dict_t *this, char *key, data_t *value, void *data)
+static int
+add_opt(char **optsp, const char *opt)
 {
-        dict_t          *dst            = NULL;
-        int32_t          ret            = -1;
-        data_t          *dict_data      = NULL;
-
-        dst = data;
+    char *newopts = NULL;
+    unsigned oldsize = 0;
+    unsigned newsize = 0;
+
+    if (*optsp == NULL)
+        newopts = gf_strdup(opt);
+    else {
+        oldsize = strlen(*optsp);
+        newsize = oldsize + 1 + strlen(opt) + 1;
+        newopts = GF_REALLOC(*optsp, newsize);
+        if (newopts)
+            sprintf(newopts + oldsize, ",%s", opt);
+    }
+    if (newopts == NULL) {
+        gf_msg("dht", GF_LOG_WARNING, 0, DHT_MSG_NO_MEMORY,
+               "Error to add choices in buffer in add_opt");
+        return -1;
+    }
+    *optsp = newopts;
+    return 0;
+}
 
-        if (strcmp (key, QUOTA_SIZE_KEY) == 0) {
-                ret = dht_aggregate_quota_xattr (dst, key, value);
-                if (ret) {
-                        gf_log ("dht", GF_LOG_WARNING, "Failed to "
-                                "aggregate qutoa xattr");
-                        goto out;
-                }
-        } else if (fnmatch (GF_XATTR_STIME_PATTERN, key, FNM_NOESCAPE) == 0) {
-                ret = gf_get_min_stime (THIS, dst, key, value);
-                if (ret < 0)
-                        goto out;
-        } else {
-                /* compare user xattrs only */
-                if (!strncmp (key, "user.", strlen ("user."))) {
-                        ret = dict_lookup (dst, key, &dict_data);
-                        if (!ret && dict_data && value) {
-                                ret = is_data_equal (dict_data, value);
-                                if (!ret)
-                                        gf_msg_debug ("dht", 0,
-                                                      "xattr mismatch for %s",
-                                                      key);
-                        }
-                }
-                ret = dict_set (dst, key, value);
-                if (ret) {
-                        gf_msg ("dht", GF_LOG_WARNING, 0,
-                                DHT_MSG_DICT_SET_FAILED,
-                                "Failed to set dictionary value: key = %s",
-                                key);
-                }
-        }
+/* Return Choice list from Split brain status */
+static char *
+getChoices(const char *value)
+{
+    int i = 0;
+    char *ptr = NULL;
+    char *tok = NULL;
+    char *result = NULL;
+    char *newval = NULL;
+
+    ptr = strstr(value, "Choices:");
+    if (!ptr) {
+        result = ptr;
+        goto out;
+    }
+
+    newval = gf_strdup(ptr);
+    if (!newval) {
+        result = newval;
+        goto out;
+    }
+
+    tok = strtok(newval, ":");
+    if (!tok) {
+        result = tok;
+        goto out;
+    }
+
+    while (tok) {
+        i++;
+        if (i == 2)
+            break;
+        tok = strtok(NULL, ":");
+    }
+
+    result = gf_strdup(tok);
 
-        ret = 0;
 out:
-        return ret;
+    if (newval)
+        GF_FREE(newval);
+
+    return result;
 }
 
+/* This function prepare a list of choices for key
+   (replica.split-brain-status) in   case of metadata split brain
+   only on the basis of key-value passed to this function.
+   After prepare the list of choices it update the same key in dict
+   with this value to reflect the same in
+   replica.split-brain-status attr for file.
 
-void
-dht_aggregate_xattr (dict_t *dst, dict_t *src)
+*/
+
+static int
+dht_aggregate_split_brain_xattr(dict_t *dst, char *key, data_t *value)
 {
-        if ((dst == NULL) || (src == NULL)) {
+    int ret = 0;
+    char *oldvalue = NULL;
+    char *old_choice = NULL;
+    char *new_choice = NULL;
+    char *full_choice = NULL;
+    char *status = NULL;
+
+    if (value == NULL) {
+        gf_msg("dht", GF_LOG_WARNING, 0, DHT_MSG_DATA_NULL,
+               "GF_AFR_SBRAIN_STATUS value is NULL");
+        ret = -1;
+        goto out;
+    }
+
+    ret = dict_get_str(dst, key, &oldvalue);
+    if (ret)
+        goto out;
+
+    /* skip code that is irrelevant if !oldvalue */
+    if (!oldvalue)
+        goto out;
+
+    if (strstr(oldvalue, "not")) {
+        gf_msg_debug("dht", 0, "Need to update split-brain status in dict");
+        ret = -1;
+        goto out;
+    }
+    if (strstr(oldvalue, "metadata-split-brain:yes") &&
+        (strstr(oldvalue, "data-split-brain:no"))) {
+        if (strstr(value->data, "not")) {
+            gf_msg_debug("dht", 0, "No need to update split-brain status");
+            ret = 0;
+            goto out;
+        }
+        if (strstr(value->data, "yes") &&
+            (strncmp(oldvalue, value->data, strlen(oldvalue)))) {
+            old_choice = getChoices(oldvalue);
+            if (!old_choice) {
+                gf_msg("dht", GF_LOG_WARNING, 0, DHT_MSG_NO_MEMORY,
+                       "Error to get choices");
+                ret = -1;
+                goto out;
+            }
+
+            ret = add_opt(&full_choice, old_choice);
+            if (ret) {
+                gf_msg("dht", GF_LOG_WARNING, 0, DHT_MSG_NO_MEMORY,
+                       "Error to add choices");
+                ret = -1;
                 goto out;
+            }
+
+            new_choice = getChoices(value->data);
+            if (!new_choice) {
+                gf_msg("dht", GF_LOG_WARNING, 0, DHT_MSG_NO_MEMORY,
+                       "Error to get choices");
+                ret = -1;
+                goto out;
+            }
+
+            ret = add_opt(&full_choice, new_choice);
+            if (ret) {
+                gf_msg("dht", GF_LOG_WARNING, 0, DHT_MSG_NO_MEMORY,
+                       "Error to add choices ");
+                ret = -1;
+                goto out;
+            }
+            ret = gf_asprintf(&status,
+                              "data-split-brain:%s    "
+                              "metadata-split-brain:%s   Choices:%s",
+                              "no", "yes", full_choice);
+
+            if (-1 == ret) {
+                gf_msg("dht", GF_LOG_WARNING, 0, DHT_MSG_NO_MEMORY,
+                       "Error to prepare status ");
+                goto out;
+            }
+            ret = dict_set_dynstr(dst, key, status);
+            if (ret) {
+                gf_msg("dht", GF_LOG_WARNING, 0, DHT_MSG_DICT_SET_FAILED,
+                       "Failed to set full choice");
+            }
         }
+    }
 
-        dict_foreach (src, dht_aggregate, dst);
 out:
-        return;
+    if (old_choice)
+        GF_FREE(old_choice);
+    if (new_choice)
+        GF_FREE(new_choice);
+    if (full_choice)
+        GF_FREE(full_choice);
+
+    return ret;
+}
+
+static int
+dht_aggregate(dict_t *this, char *key, data_t *value, void *data)
+{
+    dict_t *dst = NULL;
+    int32_t ret = -1;
+    data_t *dict_data = NULL;
+
+    dst = data;
+
+    /* compare split brain xattr only */
+    if (strcmp(key, GF_AFR_SBRAIN_STATUS) == 0) {
+        ret = dht_aggregate_split_brain_xattr(dst, key, value);
+        if (!ret)
+            goto out;
+    } else if (strcmp(key, QUOTA_SIZE_KEY) == 0) {
+        ret = dht_aggregate_quota_xattr(dst, key, value);
+        if (ret) {
+            gf_msg("dht", GF_LOG_WARNING, 0,
+                   DHT_MSG_AGGREGATE_QUOTA_XATTR_FAILED,
+                   "Failed to aggregate quota xattr");
+        }
+        goto out;
+    } else if (fnmatch(GF_XATTR_STIME_PATTERN, key, FNM_NOESCAPE) == 0) {
+        ret = gf_get_min_stime(THIS, dst, key, value);
+        goto out;
+    } else {
+        /* compare user xattrs only */
+        if (!strncmp(key, "user.", SLEN("user."))) {
+            ret = dict_lookup(dst, key, &dict_data);
+            if (!ret && dict_data && value) {
+                ret = is_data_equal(dict_data, value);
+                if (!ret)
+                    gf_msg_debug("dht", 0, "xattr mismatch for %s", key);
+            }
+        }
+    }
+
+    ret = dict_set(dst, key, value);
+    if (ret) {
+        gf_msg("dht", GF_LOG_WARNING, 0, DHT_MSG_DICT_SET_FAILED,
+               "Failed to set dictionary value: key = %s", key);
+    }
+
+out:
+    return ret;
+}
+
+static void
+dht_aggregate_xattr(dict_t *dst, dict_t *src)
+{
+    if ((dst == NULL) || (src == NULL)) {
+        goto out;
+    }
+
+    dict_foreach(src, dht_aggregate, dst);
+out:
+    return;
+}
+
+/* Code to save hashed subvol on inode ctx as a mds subvol
+ */
+int
+dht_inode_ctx_mdsvol_set(inode_t *inode, xlator_t *this, xlator_t *mds_subvol)
+{
+    dht_inode_ctx_t *ctx = NULL;
+    int ret = -1;
+    uint64_t ctx_int = 0;
+    gf_boolean_t ctx_free = _gf_false;
+
+    LOCK(&inode->lock);
+    {
+        ret = __inode_ctx_get(inode, this, &ctx_int);
+        if (ctx_int) {
+            ctx = (dht_inode_ctx_t *)(uintptr_t)ctx_int;
+            ctx->mds_subvol = mds_subvol;
+        } else {
+            ctx = GF_CALLOC(1, sizeof(*ctx), gf_dht_mt_inode_ctx_t);
+            if (!ctx)
+                goto unlock;
+            ctx->mds_subvol = mds_subvol;
+            ctx_free = _gf_true;
+            ctx_int = (long)ctx;
+            ret = __inode_ctx_set(inode, this, &ctx_int);
+        }
+    }
+unlock:
+    UNLOCK(&inode->lock);
+    if (ret && ctx_free)
+        GF_FREE(ctx);
+    return ret;
+}
+
+/*Code to get mds subvol from inode ctx */
+
+int
+dht_inode_ctx_mdsvol_get(inode_t *inode, xlator_t *this, xlator_t **mdsvol)
+{
+    dht_inode_ctx_t *ctx = NULL;
+    int ret = -1;
+
+    if (!mdsvol)
+        return ret;
+
+    if (__is_root_gfid(inode->gfid)) {
+        (*mdsvol) = FIRST_CHILD(this);
+        return 0;
+    }
+
+    ret = dht_inode_ctx_get(inode, this, &ctx);
+
+    if (!ret && ctx) {
+        if (ctx->mds_subvol) {
+            *mdsvol = ctx->mds_subvol;
+            ret = 0;
+        } else {
+            ret = -1;
+        }
+    }
+
+    return ret;
 }
 
 /* TODO:
@@ -162,907 +450,1851 @@ out:
    - complete linkfile selfheal
 */
 
-
-int
-dht_lookup_selfheal_cbk (call_frame_t *frame, void *cookie,
-                         xlator_t *this,
-                         int op_ret, int op_errno, dict_t *xdata)
+static int
+dht_lookup_selfheal_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                        int op_ret, int op_errno, dict_t *xdata)
 {
-        dht_local_t  *local = NULL;
-        dht_layout_t *layout = NULL;
-	int           ret = -1;
+    dht_local_t *local = NULL;
+    dht_layout_t *layout = NULL;
+    dht_conf_t *conf = NULL;
+    int ret = -1;
 
-        GF_VALIDATE_OR_GOTO ("dht", frame, out);
-        GF_VALIDATE_OR_GOTO ("dht", this, out);
-        GF_VALIDATE_OR_GOTO ("dht", frame->local, out);
+    GF_VALIDATE_OR_GOTO("dht", frame, out);
+    GF_VALIDATE_OR_GOTO("dht", this, out);
+    GF_VALIDATE_OR_GOTO("dht", frame->local, out);
 
-        local = frame->local;
-        ret = op_ret;
+    local = frame->local;
+    conf = this->private;
+    ret = op_ret;
 
-        FRAME_SU_UNDO (frame, dht_local_t);
+    FRAME_SU_UNDO(frame, dht_local_t);
 
-        if (ret == 0) {
-                layout = local->selfheal.layout;
-                ret = dht_layout_set (this, local->inode, layout);
-        }
+    if (ret == 0) {
+        layout = local->selfheal.layout;
+        ret = dht_layout_set(this, local->inode, layout);
+    }
 
-        if (local->loc.parent) {
-                dht_inode_ctx_time_update (local->loc.parent, this,
-                                           &local->postparent, 1);
-        }
+    dht_inode_ctx_time_update(local->inode, this, &local->stbuf, 1);
+    if (local->loc.parent) {
+        dht_inode_ctx_time_update(local->loc.parent, this, &local->postparent,
+                                  1);
+    }
 
-        DHT_STRIP_PHASE1_FLAGS (&local->stbuf);
+    DHT_STRIP_PHASE1_FLAGS(&local->stbuf);
+    dht_set_fixed_dir_stat(&local->postparent);
+    /* Delete mds xattr at the time of STACK UNWIND */
+    GF_REMOVE_INTERNAL_XATTR(conf->mds_xattr_key, local->xattr);
 
-        DHT_STACK_UNWIND (lookup, frame, ret, local->op_errno, local->inode,
-                          &local->stbuf, local->xattr, &local->postparent);
+    DHT_STACK_UNWIND(lookup, frame, ret, local->op_errno, local->inode,
+                     &local->stbuf, local->xattr, &local->postparent);
 
 out:
-        return ret;
+    return ret;
 }
 
-
-int
-dht_discover_complete (xlator_t *this, call_frame_t *discover_frame)
+static int
+dht_discover_complete(xlator_t *this, call_frame_t *discover_frame)
 {
-        dht_local_t     *local = NULL;
-        call_frame_t    *main_frame = NULL;
-        int              op_errno = 0;
-        int              ret = -1;
-        dht_layout_t    *layout = NULL;
-        dht_conf_t      *conf = NULL;
+    dht_local_t *local = NULL;
+    dht_local_t *heal_local = NULL;
+    call_frame_t *main_frame = NULL;
+    call_frame_t *heal_frame = NULL;
+    int op_errno = 0;
+    int ret = -1;
+    dht_layout_t *layout = NULL;
+    dht_conf_t *conf = NULL;
+    uint32_t vol_commit_hash = 0;
+    xlator_t *source = NULL;
+    int heal_path = 0;
+    int error_while_marking_mds = 0;
+    int i = 0;
+    loc_t loc = {0};
+    int8_t is_read_only = 0, layout_anomalies = 0;
+    char gfid_local[GF_UUID_BUF_SIZE] = {0};
+
+    local = discover_frame->local;
+    layout = local->layout;
+    conf = this->private;
+    gf_uuid_unparse(local->gfid, gfid_local);
+
+    LOCK(&discover_frame->lock);
+    {
+        main_frame = local->main_frame;
+        local->main_frame = NULL;
+    }
+    UNLOCK(&discover_frame->lock);
 
-        local = discover_frame->local;
-        layout = local->layout;
-        conf = this->private;
+    if (!main_frame)
+        return 0;
 
-        LOCK(&discover_frame->lock);
-        {
-                main_frame = local->main_frame;
-                local->main_frame = NULL;
+    /* Code to update all extended attributed from
+       subvol to local->xattr on that internal xattr has found
+    */
+    if (conf->subvolume_cnt == 1)
+        local->need_xattr_heal = 0;
+    if (local->need_xattr_heal && (local->mds_xattr)) {
+        dht_dir_set_heal_xattr(this, local, local->xattr, local->mds_xattr,
+                               NULL, NULL);
+        dict_unref(local->mds_xattr);
+        local->mds_xattr = NULL;
+    }
+
+    ret = dict_get_int8(local->xattr_req, QUOTA_READ_ONLY_KEY, &is_read_only);
+    if (ret < 0)
+        gf_msg_debug(this->name, 0, "key = %s not present in dict",
+                     QUOTA_READ_ONLY_KEY);
+
+    if (local->file_count && local->dir_count) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, DHT_MSG_FILE_TYPE_MISMATCH,
+               "path %s exists as a file on one subvolume "
+               "and directory on another. "
+               "Please fix it manually",
+               local->loc.path);
+        op_errno = EIO;
+        goto out;
+    }
+
+    if (local->cached_subvol) {
+        ret = dht_layout_preset(this, local->cached_subvol, local->inode);
+        if (ret < 0) {
+            gf_msg(this->name, GF_LOG_WARNING, 0, DHT_MSG_LAYOUT_SET_FAILED,
+                   "failed to set layout for subvolume %s",
+                   local->cached_subvol ? local->cached_subvol->name : "<nil>");
+            op_errno = EINVAL;
+            goto out;
+        }
+    } else {
+        ret = dht_layout_normalize(this, &local->loc, layout);
+        if ((ret < 0) || ((ret > 0) && (local->op_ret != 0))) {
+            /* either the layout is incorrect or the directory is
+             * not found even in one subvolume.
+             */
+            gf_msg_debug(this->name, 0,
+                         "normalizing failed on %s "
+                         "(overlaps/holes present: %s, "
+                         "ENOENT errors: %d)",
+                         local->loc.path, (ret < 0) ? "yes" : "no",
+                         (ret > 0) ? ret : 0);
+            layout_anomalies = 1;
+        } else if (local->inode) {
+            dht_layout_set(this, local->inode, layout);
+        }
+    }
+
+    if (!conf->vch_forced) {
+        ret = dict_get_uint32(local->xattr, conf->commithash_xattr_name,
+                              &vol_commit_hash);
+        if (ret == 0) {
+            conf->vol_commit_hash = vol_commit_hash;
         }
-        UNLOCK(&discover_frame->lock);
-
-        if (!main_frame)
-                return 0;
+    }
 
-        if (local->file_count && local->dir_count) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        DHT_MSG_FILE_TYPE_MISMATCH,
-                        "path %s exists as a file on one subvolume "
-                        "and directory on another. "
-                        "Please fix it manually",
-                        local->loc.path);
-                op_errno = EIO;
-                goto out;
+    if (IA_ISDIR(local->stbuf.ia_type) && !is_read_only) {
+        for (i = 0; i < layout->cnt; i++) {
+            if (!source && !layout->list[i].err)
+                source = layout->list[i].xlator;
+            if (layout->list[i].err == ENOENT ||
+                layout->list[i].err == ESTALE) {
+                heal_path = 1;
+            }
+
+            if (source && heal_path)
+                break;
         }
+    }
 
-        if (local->cached_subvol) {
-                ret = dht_layout_preset (this, local->cached_subvol,
-                                         local->inode);
-                if (ret < 0) {
-                        gf_msg (this->name, GF_LOG_WARNING, 0,
-                                DHT_MSG_LAYOUT_SET_FAILED,
-                                "failed to set layout for subvolume %s",
-                                local->cached_subvol ? local->cached_subvol->name : "<nil>");
-                        op_errno = EINVAL;
-                        goto out;
-                }
-        } else {
-                ret = dht_layout_normalize (this, &local->loc, layout);
-                if ((ret < 0) || ((ret > 0) && (local->op_ret != 0))) {
-                        /* either the layout is incorrect or the directory is
-                         * not found even in one subvolume.
-                         */
-                        gf_msg_debug (this->name, 0,
-                                      "normalizing failed on %s "
-                                      "(overlaps/holes present: %s, "
-                                      "ENOENT errors: %d)", local->loc.path,
-                                      (ret < 0) ? "yes" : "no", (ret > 0) ? ret : 0);
-                        if ((ret > 0) && (ret == conf->subvolume_cnt)) {
-                                op_errno = ESTALE;
-                                goto out;
-                        }
+    if (IA_ISDIR(local->stbuf.ia_type)) {
+        /* Call function to save hashed subvol on inode ctx if
+           internal mds xattr is not present and all subvols are up
+        */
+        if (!local->op_ret && !__is_root_gfid(local->stbuf.ia_gfid))
+            (void)dht_common_mark_mdsxattr(discover_frame,
+                                           &error_while_marking_mds, 1);
+
+        if (local->need_xattr_heal && !heal_path) {
+            local->need_xattr_heal = 0;
+            ret = dht_dir_xattr_heal(this, local, &op_errno);
+            if (ret) {
+                gf_msg(this->name, GF_LOG_ERROR, op_errno,
+                       DHT_MSG_DIR_XATTR_HEAL_FAILED,
+                       "xattr heal failed for "
+                       "directory  gfid is %s ",
+                       gfid_local);
+            }
+        }
+    }
+
+    if (source && (heal_path || layout_anomalies || error_while_marking_mds)) {
+        gf_uuid_copy(loc.gfid, local->gfid);
+        if (gf_uuid_is_null(loc.gfid)) {
+            goto done;
+        }
+
+        if (local->inode)
+            loc.inode = inode_ref(local->inode);
+        else
+            goto done;
+
+        heal_frame = create_frame(this, this->ctx->pool);
+        if (heal_frame) {
+            heal_local = dht_local_init(heal_frame, &loc, NULL, 0);
+            if (!heal_local)
+                goto cleanup;
+
+            gf_uuid_copy(heal_local->gfid, local->gfid);
+            heal_frame->cookie = source;
+            heal_local->xattr = dict_ref(local->xattr);
+            heal_local->stbuf = local->stbuf;
+            heal_local->postparent = local->postparent;
+            heal_local->inode = inode_ref(loc.inode);
+            heal_local->main_frame = main_frame;
+            FRAME_SU_DO(heal_frame, dht_local_t);
+            ret = synctask_new(this->ctx->env, dht_heal_full_path,
+                               dht_heal_full_path_done, heal_frame, heal_frame);
+            if (!ret) {
+                loc_wipe(&loc);
+                return 0;
+            }
+            /*
+             * Failed to spawn the synctask. Returning
+             * with out doing heal.
+             */
+        cleanup:
+            loc_wipe(&loc);
+            DHT_STACK_DESTROY(heal_frame);
+        }
+    }
+done:
+    dht_set_fixed_dir_stat(&local->postparent);
+    /* Delete mds xattr at the time of STACK UNWIND */
+    if (local->xattr)
+        GF_REMOVE_INTERNAL_XATTR(conf->mds_xattr_key, local->xattr);
 
-                        /* For fixing the directory layout, we need to choose
-                         * the subvolume on which layout will be set first.
-                         * Because in nameless lookup, we have gfid only,
-                         * we are dependent on gfid. Therefore if conf->
-                         * randomize_by_gfid is set, then only we proceed for
-                         * healing layout of directory otherwise we don't heal.
-                         */
+    DHT_STACK_UNWIND(lookup, main_frame, local->op_ret, local->op_errno,
+                     local->inode, &local->stbuf, local->xattr,
+                     &local->postparent);
+    return 0;
 
-                        if (local->inode && conf->randomize_by_gfid)
-                                goto selfheal;
-                }
+out:
+    DHT_STACK_UNWIND(lookup, main_frame, -1, op_errno, NULL, NULL, NULL, NULL);
 
-                if (local->inode)
-                        dht_layout_set (this, local->inode, layout);
-        }
+    return ret;
+}
 
-        DHT_STACK_UNWIND (lookup, main_frame, local->op_ret, local->op_errno,
-                          local->inode, &local->stbuf, local->xattr,
-                          &local->postparent);
-        return 0;
+static int
+dht_common_mark_mdsxattr_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                             int op_ret, int op_errno, dict_t *xdata)
+{
+    dht_local_t *local = NULL;
+    xlator_t *prev = cookie;
+    int ret = -1;
+    dht_conf_t *conf = 0;
+    dht_layout_t *layout = NULL;
+    int32_t mds_heal_fresh_lookup = 0;
+
+    GF_VALIDATE_OR_GOTO(this->name, frame, out);
+    GF_VALIDATE_OR_GOTO(this->name, frame->local, out);
+
+    local = frame->local;
+    conf = this->private;
+    layout = local->selfheal.layout;
+    mds_heal_fresh_lookup = local->mds_heal_fresh_lookup;
+
+    if (op_ret) {
+        gf_msg_debug(this->name, op_ret,
+                     "Failed to set %s on the MDS %s for path %s. ",
+                     conf->mds_xattr_key, prev->name, local->loc.path);
+    } else {
+        /* Save mds subvol on inode ctx */
+        ret = dht_inode_ctx_mdsvol_set(local->inode, this, prev);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, DHT_MSG_SET_INODE_CTX_FAILED,
+                   "Failed to set mds subvol on inode ctx"
+                   " %s for %s ",
+                   prev->name, local->loc.path);
+        }
+    }
+    if (!local->mds_heal_fresh_lookup && layout) {
+        dht_selfheal_dir_setattr(frame, &local->loc, &local->stbuf, 0xffffffff,
+                                 layout);
+    }
 out:
-        DHT_STACK_UNWIND (lookup, main_frame, -1, op_errno, NULL, NULL, NULL,
-                          NULL);
+    if (mds_heal_fresh_lookup)
+        DHT_STACK_DESTROY(frame);
+    return 0;
+}
 
-        return ret;
+static xlator_t *
+dht_inode_get_hashed_subvol(inode_t *inode, xlator_t *this, loc_t *loc)
+{
+    char *path = NULL;
+    loc_t populate_loc = {
+        0,
+    };
+    char *name = NULL;
+    xlator_t *hash_subvol = NULL;
+
+    if (!inode)
+        return hash_subvol;
+
+    if (loc && loc->parent && loc->path) {
+        if (!loc->name) {
+            name = strrchr(loc->path, '/');
+            if (name) {
+                loc->name = name + 1;
+            } else {
+                goto out;
+            }
+        }
+        hash_subvol = dht_subvol_get_hashed(this, loc);
+        goto out;
+    }
 
-selfheal:
+    if (!gf_uuid_is_null(inode->gfid)) {
+        populate_loc.inode = inode_ref(inode);
+        populate_loc.parent = inode_parent(populate_loc.inode, NULL, NULL);
+        inode_path(populate_loc.inode, NULL, &path);
 
-        main_frame->local = local;
-        discover_frame->local =  NULL;
-        FRAME_SU_DO (main_frame, dht_local_t);
-        gf_uuid_copy (local->loc.gfid, local->gfid);
-        ret = dht_selfheal_directory_for_nameless_lookup (main_frame,
-                                                        dht_lookup_selfheal_cbk,
-                                                          &local->loc, layout);
-        return ret;
+        if (!path)
+            goto out;
+
+        populate_loc.path = path;
+        if (!populate_loc.name && populate_loc.path) {
+            name = strrchr(populate_loc.path, '/');
+            if (name) {
+                populate_loc.name = name + 1;
 
+            } else {
+                goto out;
+            }
+        }
+        hash_subvol = dht_subvol_get_hashed(this, &populate_loc);
+    }
+out:
+    if (populate_loc.inode)
+        loc_wipe(&populate_loc);
+    return hash_subvol;
 }
 
+/* Common function call by revalidate/selfheal code path to populate
+   internal xattr if it is not present, mark_during_fresh_lookup value
+   determines either function is call by revalidate_cbk(discover_complete)
+   or call by selfheal code path while fresh lookup.
+   Here we do wind a call serially in case of fresh lookup and
+   for other lookup code path we do wind a call parallel.The reason
+   to wind a call serially is at the time of fresh lookup directory is not
+   discovered and at the time of revalidate_lookup directory is
+   already discovered. So, revalidate codepath can race with setxattr
+   codepath and can get into spurious heals because of an ongoing setxattr.
+   This can slow down revalidates, if healing happens in foreground.
+   However, if healing happens in background, there is no direct performance
+   penalty.
+*/
 int
-dht_discover_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                  int op_ret, int op_errno,
-                  inode_t *inode, struct iatt *stbuf, dict_t *xattr,
-                  struct iatt *postparent)
-{
-        dht_local_t  *local                   = NULL;
-        int           this_call_cnt           = 0;
-        call_frame_t *prev                    = NULL;
-        dht_layout_t *layout                  = NULL;
-        int           ret                     = -1;
-        int           is_dir                  = 0;
-        int           is_linkfile             = 0;
-        int           attempt_unwind          = 0;
-        dht_conf_t   *conf                    = 0;
-        char         gfid_local[GF_UUID_BUF_SIZE]  = {0};
-        char         gfid_node[GF_UUID_BUF_SIZE]  = {0};
-
-        GF_VALIDATE_OR_GOTO ("dht", frame, out);
-        GF_VALIDATE_OR_GOTO ("dht", this, out);
-        GF_VALIDATE_OR_GOTO ("dht", frame->local, out);
-        GF_VALIDATE_OR_GOTO ("dht", this->private, out);
-        GF_VALIDATE_OR_GOTO ("dht", cookie, out);
-
-        local = frame->local;
-        prev  = cookie;
-        conf = this->private;
+dht_common_mark_mdsxattr(call_frame_t *frame, int *errst,
+                         int mark_during_fresh_lookup)
+{
+    dht_local_t *local = NULL;
+    xlator_t *this = NULL;
+    xlator_t *hashed_subvol = NULL;
+    int ret = 0;
+    int i = 0;
+    dict_t *xattrs = NULL;
+    char gfid_local[GF_UUID_BUF_SIZE] = {
+        0,
+    };
+    int32_t zero[1] = {0};
+    dht_conf_t *conf = 0;
+    dht_layout_t *layout = NULL;
+    dht_local_t *copy_local = NULL;
+    call_frame_t *xattr_frame = NULL;
+    gf_boolean_t vol_down = _gf_false;
+
+    GF_VALIDATE_OR_GOTO("dht", frame, out);
+    this = frame->this;
+    GF_VALIDATE_OR_GOTO("dht", this, out);
+    GF_VALIDATE_OR_GOTO(this->name, frame->local, out);
+    GF_VALIDATE_OR_GOTO(this->name, this->private, out);
+
+    local = frame->local;
+    conf = this->private;
+    layout = local->selfheal.layout;
+    local->mds_heal_fresh_lookup = mark_during_fresh_lookup;
+
+    gf_uuid_unparse(local->gfid, gfid_local);
+
+    /* Code to update hashed subvol consider as a mds subvol
+       and wind a setxattr call on hashed subvol to update
+       internal xattr
+    */
+    if (!local->xattr || !dict_get(local->xattr, conf->mds_xattr_key)) {
+        /* It means no internal MDS xattr has been set yet
+         */
+        /* Check the status of all subvol are up while call
+           this function call by lookup code path
+        */
+        if (mark_during_fresh_lookup) {
+            for (i = 0; i < conf->subvolume_cnt; i++) {
+                if (!conf->subvolume_status[i]) {
+                    vol_down = _gf_true;
+                    break;
+                }
+            }
+            if (vol_down) {
+                gf_msg_debug(this->name, 0,
+                             "subvol %s is down. Unable to "
+                             " save mds subvol on inode for "
+                             " path %s gfid is %s ",
+                             conf->subvolumes[i]->name, local->loc.path,
+                             gfid_local);
+                goto out;
+            }
+        }
 
-        layout = local->layout;
+        /* Calculate hashed subvol based on inode and parent node
+         */
+        hashed_subvol = dht_inode_get_hashed_subvol(local->inode, this,
+                                                    &local->loc);
+        if (!hashed_subvol) {
+            gf_msg(this->name, GF_LOG_DEBUG, 0,
+                   DHT_MSG_HASHED_SUBVOL_GET_FAILED,
+                   "Failed to get hashed subvol for path %s"
+                   "gfid is %s ",
+                   local->loc.path, gfid_local);
+            if (errst)
+                (*errst) = 1;
+            ret = -1;
+            goto out;
+        }
+        xattrs = dict_new();
+        if (!xattrs) {
+            gf_msg(this->name, GF_LOG_ERROR, ENOMEM, DHT_MSG_NO_MEMORY,
+                   "dict_new failed");
+            ret = -1;
+            goto out;
+        }
+        /* Add internal MDS xattr on disk for hashed subvol
+         */
+        ret = dht_dict_set_array(xattrs, conf->mds_xattr_key, zero, 1);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_WARNING, ENOMEM, DHT_MSG_DICT_SET_FAILED,
+                   "Failed to set dictionary"
+                   "  value:key = %s for "
+                   "path %s",
+                   conf->mds_xattr_key, local->loc.path);
+            ret = -1;
+            goto out;
+        }
+        /* Create a new frame to wind a call only while
+           this function call by revalidate_cbk code path
+           To wind a call parallel need to create a new frame
+        */
+        if (mark_during_fresh_lookup) {
+            xattr_frame = create_frame(this, this->ctx->pool);
+            if (!xattr_frame) {
+                ret = -1;
+                goto out;
+            }
+            copy_local = dht_local_init(xattr_frame, &(local->loc), NULL, 0);
+            if (!copy_local) {
+                ret = -1;
+                DHT_STACK_DESTROY(xattr_frame);
+                goto out;
+            }
+            copy_local->stbuf = local->stbuf;
+            copy_local->mds_heal_fresh_lookup = mark_during_fresh_lookup;
+            if (!copy_local->inode)
+                copy_local->inode = inode_ref(local->inode);
+            gf_uuid_copy(copy_local->loc.gfid, local->gfid);
+            FRAME_SU_DO(xattr_frame, dht_local_t);
+            STACK_WIND_COOKIE(xattr_frame, dht_common_mark_mdsxattr_cbk,
+                              hashed_subvol, hashed_subvol,
+                              hashed_subvol->fops->setxattr, &local->loc,
+                              xattrs, 0, NULL);
+        } else {
+            STACK_WIND_COOKIE(frame, dht_common_mark_mdsxattr_cbk,
+                              (void *)hashed_subvol, hashed_subvol,
+                              hashed_subvol->fops->setxattr, &local->loc,
+                              xattrs, 0, NULL);
+        }
+    } else {
+        gf_msg_debug(this->name, 0,
+                     "internal xattr %s is present on subvol"
+                     "on path %s gfid is %s ",
+                     conf->mds_xattr_key, local->loc.path, gfid_local);
+        if (!mark_during_fresh_lookup)
+            dht_selfheal_dir_setattr(frame, &local->loc, &local->stbuf,
+                                     0xffffffff, layout);
+    }
 
+out:
+    if (xattrs)
+        dict_unref(xattrs);
+    return ret;
+}
 
-        /* Check if the gfid is different for file from other node */
-        if (!op_ret && gf_uuid_compare (local->gfid, stbuf->ia_gfid)) {
+/* Get the value of key from dict in the bytewise and save in array after
+   convert from network byte order to host byte order
+*/
+static int32_t
+dht_dict_get_array(dict_t *dict, char *key, int32_t value[], int32_t size,
+                   int *errst)
+{
+    void *ptr = NULL;
+    int32_t len = -1;
+    int32_t vindex = -1;
+    int32_t err = -1;
+    int ret = 0;
+
+    if (dict == NULL) {
+        (*errst) = -1;
+        return -EINVAL;
+    }
+    err = dict_get_ptr_and_len(dict, key, &ptr, &len);
+    if (err != 0) {
+        (*errst) = -1;
+        return err;
+    }
+
+    if (len != (size * sizeof(int32_t))) {
+        (*errst) = -1;
+        return -EINVAL;
+    }
+
+    for (vindex = 0; vindex < size; vindex++) {
+        value[vindex] = ntoh32(*((int32_t *)ptr + vindex));
+        if (value[vindex] < 0)
+            ret = -1;
+    }
+
+    return ret;
+}
 
-                gf_uuid_unparse(stbuf->ia_gfid, gfid_node);
-                gf_uuid_unparse(local->gfid, gfid_local);
+static int
+dht_discover_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int op_ret,
+                 int op_errno, inode_t *inode, struct iatt *stbuf,
+                 dict_t *xattr, struct iatt *postparent)
+{
+    dht_local_t *local = NULL;
+    int this_call_cnt = 0;
+    xlator_t *prev = NULL;
+    dht_layout_t *layout = NULL;
+    int ret = -1;
+    int is_dir = 0;
+    int32_t check_mds = 0;
+    int is_linkfile = 0;
+    int attempt_unwind = 0;
+    dht_conf_t *conf = 0;
+    char gfid_local[GF_UUID_BUF_SIZE] = {0};
+    char gfid_node[GF_UUID_BUF_SIZE] = {0};
+    int32_t mds_xattr_val[1] = {0};
+    int errst = 0;
+
+    GF_VALIDATE_OR_GOTO("dht", frame, out);
+    GF_VALIDATE_OR_GOTO("dht", this, out);
+    GF_VALIDATE_OR_GOTO("dht", frame->local, out);
+    GF_VALIDATE_OR_GOTO("dht", this->private, out);
+    GF_VALIDATE_OR_GOTO("dht", cookie, out);
+
+    local = frame->local;
+    prev = cookie;
+    conf = this->private;
+
+    layout = local->layout;
+
+    /* Check if the gfid is different for file from other node */
+    if (!op_ret && gf_uuid_compare(local->gfid, stbuf->ia_gfid)) {
+        gf_uuid_unparse(stbuf->ia_gfid, gfid_node);
+        gf_uuid_unparse(local->gfid, gfid_local);
+
+        gf_msg(this->name, GF_LOG_WARNING, 0, DHT_MSG_GFID_MISMATCH,
+               "%s: gfid different on %s, gfid local = %s"
+               "gfid other = %s",
+               local->loc.path, prev->name, gfid_local, gfid_node);
+    }
+
+    LOCK(&frame->lock);
+    {
+        /* TODO: assert equal mode on stbuf->st_mode and
+           local->stbuf->st_mode
+
+           else mkdir/chmod/chown and fix
+        */
 
-                gf_msg (this->name, GF_LOG_WARNING, 0,
-                        DHT_MSG_GFID_MISMATCH,
-                        "%s: gfid different on %s, gfid local = %s"
-                        "gfid other = %s",
-                        local->loc.path, prev->this->name,
-                        gfid_local, gfid_node);
-        }
+        ret = dht_layout_merge(this, layout, prev, op_ret, op_errno, xattr);
+        if (ret)
+            gf_msg(this->name, GF_LOG_WARNING, 0, DHT_MSG_LAYOUT_MERGE_FAILED,
+                   "%s: failed to merge layouts for subvol %s", local->loc.path,
+                   prev->name);
 
+        if (op_ret == -1) {
+            local->op_errno = op_errno;
+            gf_msg_debug(this->name, op_errno,
+                         "lookup of %s on %s returned error", local->loc.path,
+                         prev->name);
 
-        LOCK (&frame->lock);
-        {
-                /* TODO: assert equal mode on stbuf->st_mode and
-                   local->stbuf->st_mode
+            goto unlock;
+        }
 
-                   else mkdir/chmod/chown and fix
-                */
-                ret = dht_layout_merge (this, layout, prev->this,
-                                        op_ret, op_errno, xattr);
-                if (ret)
-                        gf_msg (this->name, GF_LOG_WARNING, 0,
-                                DHT_MSG_LAYOUT_MERGE_FAILED,
-                                "%s: failed to merge layouts for subvol %s",
-                                local->loc.path, prev->this->name);
-
-                if (op_ret == -1) {
-                        local->op_errno = op_errno;
-                        gf_msg_debug (this->name, 0,
-                                      "lookup of %s on %s returned error (%s)",
-                                      local->loc.path, prev->this->name,
-                                      strerror (op_errno));
-
-                        goto unlock;
-                }
+        is_linkfile = check_is_linkfile(inode, stbuf, xattr,
+                                        conf->link_xattr_name);
+        is_dir = check_is_dir(inode, stbuf, xattr);
 
-                is_linkfile = check_is_linkfile (inode, stbuf, xattr,
-                                                 conf->link_xattr_name);
-                is_dir = check_is_dir (inode, stbuf, xattr);
+        if (is_dir) {
+            local->dir_count++;
+        } else {
+            local->file_count++;
+
+            if (!is_linkfile && !local->cached_subvol) {
+                /* real file */
+                /* Ok, we somehow managed to find a file on
+                 * more than one subvol. ignore this or we
+                 * will end up overwriting information while a
+                 * a thread is potentially unwinding from
+                 * dht_discover_complete
+                 */
+                local->cached_subvol = prev;
+                attempt_unwind = 1;
+            } else {
+                goto unlock;
+            }
+        }
 
-                if (is_dir) {
-                        local->dir_count ++;
-                } else {
-                        local->file_count ++;
-
-                        if (!is_linkfile) {
-                                /* real file */
-                                local->cached_subvol = prev->this;
-                                attempt_unwind = 1;
-                        } else {
-                                goto unlock;
-                        }
-                }
+        local->op_ret = 0;
 
-                local->op_ret = 0;
+        if (local->xattr == NULL) {
+            local->xattr = dict_ref(xattr);
+        } else {
+            /* Don't aggregate for files. See BZ#1484709 */
+            if (is_dir)
+                dht_aggregate_xattr(local->xattr, xattr);
+        }
 
-                if (local->xattr == NULL) {
-                        local->xattr = dict_ref (xattr);
-                } else {
-                        dht_aggregate_xattr (local->xattr, xattr);
-                }
+        if (local->inode == NULL)
+            local->inode = inode_ref(inode);
 
-                if (local->inode == NULL)
-                        local->inode = inode_ref (inode);
+        dht_iatt_merge(this, &local->stbuf, stbuf);
+        dht_iatt_merge(this, &local->postparent, postparent);
 
-                dht_iatt_merge (this, &local->stbuf, stbuf, prev->this);
-                dht_iatt_merge (this, &local->postparent, postparent,
-                                prev->this);
-        }
+        if (!dict_get(xattr, conf->mds_xattr_key)) {
+            goto unlock;
+        } else {
+            gf_msg_debug(this->name, 0,
+                         "internal xattr %s is present on subvol"
+                         "on path %s gfid is %s ",
+                         conf->mds_xattr_key, local->loc.path, gfid_local);
+        }
+        check_mds = dht_dict_get_array(xattr, conf->mds_xattr_key,
+                                       mds_xattr_val, 1, &errst);
+        /* save mds subvol on inode ctx */
+        ret = dht_inode_ctx_mdsvol_set(local->inode, this, prev);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, DHT_MSG_SET_INODE_CTX_FAILED,
+                   "Failed to set hashed subvol for %s vol is %s",
+                   local->loc.path, prev->name);
+        }
+
+        if ((check_mds < 0) && !errst) {
+            local->mds_xattr = dict_ref(xattr);
+            gf_msg_debug(this->name, 0,
+                         "Value of %s is not zero on mds subvol"
+                         "so xattr needs to be healed on non mds"
+                         " path is %s and vol name is %s "
+                         " gfid is %s",
+                         conf->mds_xattr_key, local->loc.path, prev->name,
+                         gfid_local);
+            local->need_xattr_heal = 1;
+            local->mds_subvol = prev;
+        }
+    }
 unlock:
-        UNLOCK (&frame->lock);
+    UNLOCK(&frame->lock);
 out:
-        /* Make sure, the thread executing dht_discover_complete is the one
-         * which calls STACK_DESTROY (frame). In the case of "attempt_unwind",
-         * this makes sure that the thread don't call dht_frame_return, till
-         * call to dht_discover_complete is done.
-         */
-        if (attempt_unwind) {
-                dht_discover_complete (this, frame);
-        }
+    /* Make sure, the thread executing dht_discover_complete is the one
+     * which calls STACK_DESTROY (frame). In the case of "attempt_unwind",
+     * this makes sure that the thread don't call dht_frame_return, till
+     * call to dht_discover_complete is done.
+     */
+    if (attempt_unwind) {
+        dht_discover_complete(this, frame);
+    }
 
-        this_call_cnt = dht_frame_return (frame);
+    this_call_cnt = dht_frame_return(frame);
 
-        if (is_last_call (this_call_cnt) && !attempt_unwind) {
-                dht_discover_complete (this, frame);
-        }
+    if (is_last_call(this_call_cnt) && !attempt_unwind) {
+        dht_discover_complete(this, frame);
+    }
 
-        if (is_last_call (this_call_cnt))
-                DHT_STACK_DESTROY (frame);
+    if (is_last_call(this_call_cnt))
+        DHT_STACK_DESTROY(frame);
 
-        return 0;
+    return 0;
 }
 
+static int
+dht_set_file_xattr_req(xlator_t *this, loc_t *loc, dict_t *xattr_req)
+{
+    int ret = -EINVAL;
+    dht_conf_t *conf = NULL;
+
+    conf = this->private;
+    if (!conf) {
+        goto err;
+    }
+
+    if (!xattr_req) {
+        goto err;
+    }
+
+    /* Used to check whether this is a linkto file.
+     */
+    ret = dict_set_uint32(xattr_req, conf->link_xattr_name, 256);
+    if (ret < 0) {
+        gf_msg(this->name, GF_LOG_WARNING, ENOMEM, DHT_MSG_DICT_SET_FAILED,
+               "Failed to set dictionary value:key = %s for "
+               "path %s",
+               conf->link_xattr_name, loc->path);
+        goto err;
+    }
+
+    /* This is used to make sure we don't unlink linkto files
+     * which are the target of an ongoing file migration.
+     */
+    ret = dict_set_uint32(xattr_req, GLUSTERFS_OPEN_FD_COUNT, 4);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_WARNING, ENOMEM, DHT_MSG_DICT_SET_FAILED,
+               "Failed to set dictionary value:key = %s for "
+               "path %s",
+               GLUSTERFS_OPEN_FD_COUNT, loc->path);
+        goto err;
+    }
+
+    ret = 0;
+err:
+    return ret;
+}
 
-int
-dht_discover (call_frame_t *frame, xlator_t *this, loc_t *loc)
+/* This is a gfid based nameless lookup. Without a name, the hashed subvol
+ * cannot be calculated so a lookup is sent to all subvols.
+ */
+static int
+dht_do_discover(call_frame_t *frame, xlator_t *this, loc_t *loc)
 {
-        int          ret;
-        dht_local_t *local = NULL;
-        dht_conf_t  *conf = NULL;
-        int          call_cnt = 0;
-        int          op_errno = EINVAL;
-        int          i = 0;
-        call_frame_t *discover_frame = NULL;
+    int ret;
+    dht_local_t *local = NULL;
+    dht_conf_t *conf = NULL;
+    int call_cnt = 0;
+    int op_errno = EINVAL;
+    int i = 0;
+    call_frame_t *discover_frame = NULL;
+
+    conf = this->private;
+    local = frame->local;
+
+    /* As we do not know if this is a file or directory, request
+     * both file and directory xattrs
+     */
+    ret = dht_set_file_xattr_req(this, loc, local->xattr_req);
+    if (ret) {
+        goto err;
+    }
+
+    ret = dht_set_dir_xattr_req(this, loc, local->xattr_req);
+    if (ret) {
+        goto err;
+    }
+
+    if (loc_is_root(loc)) {
+        /* Request the DHT commit hash xattr (trusted.glusterfs.dht.commithash)
+         * set on the brick root.
+         */
+        ret = dict_set_uint32(local->xattr_req, conf->commithash_xattr_name,
+                              sizeof(uint32_t));
+    }
 
-        conf = this->private;
-        local = frame->local;
+    call_cnt = conf->subvolume_cnt;
+    local->call_cnt = call_cnt;
 
-        ret = dict_set_uint32 (local->xattr_req, conf->xattr_name, 4 * 4);
-        if (ret)
-                gf_msg (this->name, GF_LOG_WARNING, 0,
-                        DHT_MSG_DICT_SET_FAILED,
-                        "%s: Failed to set dictionary value:key = %s",
-                        loc->path, conf->xattr_name);
+    local->layout = dht_layout_new(this, conf->subvolume_cnt);
 
-        ret = dict_set_uint32 (local->xattr_req, conf->link_xattr_name, 256);
-        if (ret)
-                gf_msg (this->name, GF_LOG_WARNING, 0,
-                        DHT_MSG_DICT_SET_FAILED,
-                        "%s: Failed to set dictionary value:key = %s",
-                        loc->path, conf->link_xattr_name);
+    if (!local->layout) {
+        op_errno = ENOMEM;
+        goto err;
+    }
 
-        call_cnt        = conf->subvolume_cnt;
-        local->call_cnt = call_cnt;
+    gf_uuid_copy(local->gfid, loc->gfid);
 
-        local->layout = dht_layout_new (this, conf->subvolume_cnt);
+    discover_frame = copy_frame(frame);
+    if (!discover_frame) {
+        op_errno = ENOMEM;
+        goto err;
+    }
 
-        if (!local->layout) {
-                op_errno = ENOMEM;
-                goto err;
-        }
+    discover_frame->local = local;
+    frame->local = NULL;
+    local->main_frame = frame;
 
-        gf_uuid_copy (local->gfid, loc->gfid);
+    for (i = 0; i < call_cnt; i++) {
+        STACK_WIND_COOKIE(discover_frame, dht_discover_cbk, conf->subvolumes[i],
+                          conf->subvolumes[i],
+                          conf->subvolumes[i]->fops->lookup, &local->loc,
+                          local->xattr_req);
+    }
 
-        discover_frame = copy_frame (frame);
-        if (!discover_frame) {
-                op_errno = ENOMEM;
-                goto err;
-        }
+    return 0;
 
-        discover_frame->local = local;
-        frame->local = NULL;
-        local->main_frame = frame;
+err:
+    DHT_STACK_UNWIND(lookup, frame, -1, op_errno, NULL, NULL, NULL, NULL);
 
-        for (i = 0; i < call_cnt; i++) {
-                STACK_WIND (discover_frame, dht_discover_cbk,
-                            conf->subvolumes[i],
-                            conf->subvolumes[i]->fops->lookup,
-                            &local->loc, local->xattr_req);
-        }
+    return 0;
+}
 
-        return 0;
+/* Code to call syntask to heal custom xattr from hashed subvol
+   to non hashed subvol
+*/
+int
+dht_dir_xattr_heal(xlator_t *this, dht_local_t *local, int *op_errno)
+{
+    dht_local_t *copy_local = NULL;
+    call_frame_t *copy = NULL;
+    int ret = -1;
+    char gfid_local[GF_UUID_BUF_SIZE] = {0};
+
+    if (gf_uuid_is_null(local->gfid)) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, DHT_MSG_DIR_XATTR_HEAL_FAILED,
+               "No gfid exists for path %s "
+               "so healing xattr is not possible",
+               local->loc.path);
+        *op_errno = EIO;
+        goto out;
+    }
+
+    gf_uuid_unparse(local->gfid, gfid_local);
+    copy = create_frame(this, this->ctx->pool);
+    if (copy) {
+        copy_local = dht_local_init(copy, &(local->loc), NULL, 0);
+        if (!copy_local) {
+            gf_msg(this->name, GF_LOG_ERROR, ENOMEM,
+                   DHT_MSG_DIR_XATTR_HEAL_FAILED,
+                   "Memory allocation failed "
+                   "for path %s gfid %s ",
+                   local->loc.path, gfid_local);
+            *op_errno = ENOMEM;
+            DHT_STACK_DESTROY(copy);
+        } else {
+            copy_local->stbuf = local->stbuf;
+            gf_uuid_copy(copy_local->loc.gfid, local->gfid);
+            copy_local->mds_subvol = local->mds_subvol;
+            FRAME_SU_DO(copy, dht_local_t);
+            ret = synctask_new(this->ctx->env, dht_dir_heal_xattrs,
+                               dht_dir_heal_xattrs_done, copy, copy);
+            if (ret) {
+                gf_msg(this->name, GF_LOG_ERROR, ENOMEM,
+                       DHT_MSG_DIR_XATTR_HEAL_FAILED,
+                       "Synctask creation failed to heal xattr "
+                       "for path %s gfid %s ",
+                       local->loc.path, gfid_local);
+                *op_errno = ENOMEM;
+                DHT_STACK_DESTROY(copy);
+            }
+        }
+    }
+out:
+    return ret;
+}
 
-err:
-        DHT_STACK_UNWIND (lookup, frame, -1, op_errno, NULL, NULL, NULL,
-                          NULL);
+static int
+dht_needs_selfheal(call_frame_t *frame, xlator_t *this)
+{
+    dht_local_t *local = NULL;
+    dht_layout_t *layout = NULL;
+    int needs_selfheal = 0;
+    int ret = 0;
+
+    local = frame->local;
+    layout = local->layout;
+
+    if (local->need_attrheal || local->need_xattr_heal ||
+        local->need_selfheal) {
+        needs_selfheal = 1;
+    }
+
+    ret = dht_layout_normalize(this, &local->loc, layout);
+
+    if (ret != 0) {
+        gf_msg_debug(this->name, 0, "fixing assignment on %s", local->loc.path);
+        needs_selfheal = 1;
+    }
+    return needs_selfheal;
+}
 
+static int
+is_permission_different(ia_prot_t *prot1, ia_prot_t *prot2)
+{
+    if ((prot1->owner.read != prot2->owner.read) ||
+        (prot1->owner.write != prot2->owner.write) ||
+        (prot1->owner.exec != prot2->owner.exec) ||
+        (prot1->group.read != prot2->group.read) ||
+        (prot1->group.write != prot2->group.write) ||
+        (prot1->group.exec != prot2->group.exec) ||
+        (prot1->other.read != prot2->other.read) ||
+        (prot1->other.write != prot2->other.write) ||
+        (prot1->other.exec != prot2->other.exec) ||
+        (prot1->suid != prot2->suid) || (prot1->sgid != prot2->sgid) ||
+        (prot1->sticky != prot2->sticky)) {
+        return 1;
+    } else {
         return 0;
+    }
 }
 
-
 int
-dht_lookup_dir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                    int op_ret, int op_errno,
-                    inode_t *inode, struct iatt *stbuf, dict_t *xattr,
-                    struct iatt *postparent)
-{
-        dht_local_t  *local                   = NULL;
-        int           this_call_cnt           = 0;
-        call_frame_t *prev                    = NULL;
-        dht_layout_t *layout                  = NULL;
-        int           ret                     = -1;
-        int           is_dir                  = 0;
-        char         gfid_local[GF_UUID_BUF_SIZE]  = {0};
-        char         gfid_node[GF_UUID_BUF_SIZE]  = {0};
-
-        GF_VALIDATE_OR_GOTO ("dht", frame, out);
-        GF_VALIDATE_OR_GOTO ("dht", this, out);
-        GF_VALIDATE_OR_GOTO ("dht", frame->local, out);
-        GF_VALIDATE_OR_GOTO ("dht", this->private, out);
-        GF_VALIDATE_OR_GOTO ("dht", cookie, out);
+dht_lookup_dir_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                   int op_ret, int op_errno, inode_t *inode, struct iatt *stbuf,
+                   dict_t *xattr, struct iatt *postparent)
+{
+    dht_local_t *local = NULL;
+    dht_conf_t *conf = NULL;
+    int this_call_cnt = 0;
+    xlator_t *prev = NULL;
+    dht_layout_t *layout = NULL;
+    int ret = -1;
+    int is_dir = 0;
+    int32_t check_mds = 0;
+    int errst = 0;
+    char gfid_local[GF_UUID_BUF_SIZE] = {0};
+    char gfid_node[GF_UUID_BUF_SIZE] = {0};
+    int32_t mds_xattr_val[1] = {0};
+
+    GF_VALIDATE_OR_GOTO("dht", frame, out);
+    GF_VALIDATE_OR_GOTO("dht", this, out);
+    GF_VALIDATE_OR_GOTO("dht", frame->local, out);
+    GF_VALIDATE_OR_GOTO("dht", this->private, out);
+    GF_VALIDATE_OR_GOTO("dht", cookie, out);
+
+    local = frame->local;
+    prev = cookie;
+    conf = this->private;
+
+    layout = local->layout;
+    gf_msg_debug(this->name, op_errno,
+                 "%s: lookup on %s returned with op_ret = %d, op_errno = %d",
+                 local->loc.path, prev->name, op_ret, op_errno);
+
+    /* The first successful lookup*/
+    if (!op_ret && gf_uuid_is_null(local->gfid)) {
+        memcpy(local->gfid, stbuf->ia_gfid, 16);
+    }
+    if (!gf_uuid_is_null(local->gfid)) {
+        gf_uuid_unparse(local->gfid, gfid_local);
+    }
+
+    /* Check if the gfid is different for file from other node */
+    if (!op_ret && gf_uuid_compare(local->gfid, stbuf->ia_gfid)) {
+        gf_uuid_unparse(stbuf->ia_gfid, gfid_node);
+
+        gf_msg(this->name, GF_LOG_WARNING, 0, DHT_MSG_GFID_MISMATCH,
+               "%s: gfid different on %s."
+               " gfid local = %s, gfid subvol = %s",
+               local->loc.path, prev->name, gfid_local, gfid_node);
+    }
+
+    LOCK(&frame->lock);
+    {
+        /* TODO: assert equal mode on stbuf->st_mode and
+           local->stbuf->st_mode
+           else mkdir/chmod/chown and fix
+        */
+        ret = dht_layout_merge(this, layout, prev, op_ret, op_errno, xattr);
 
-        local = frame->local;
-        prev  = cookie;
+        if (op_ret == -1) {
+            local->op_errno = op_errno;
 
-        layout = local->layout;
+            /* The GFID is missing on this subvol. Force a heal. */
+            if (op_errno == ENODATA) {
+                local->need_lookup_everywhere = 1;
+            }
+            goto unlock;
+        }
 
-        if (!op_ret && gf_uuid_is_null (local->gfid))
-                memcpy (local->gfid, stbuf->ia_gfid, 16);
+        is_dir = check_is_dir(inode, stbuf, xattr);
+        if (!is_dir) {
+            gf_msg_debug(this->name, 0,
+                         "%s: lookup on %s returned non dir 0%o"
+                         "calling lookup_everywhere",
+                         local->loc.path, prev->name, stbuf->ia_type);
 
+            local->need_lookup_everywhere = 1;
+            goto unlock;
+        }
 
-        /* Check if the gfid is different for file from other node */
-        if (!op_ret && gf_uuid_compare (local->gfid, stbuf->ia_gfid)) {
+        local->op_ret = 0;
+        if (local->xattr == NULL) {
+            local->xattr = dict_ref(xattr);
+        } else {
+            dht_aggregate_xattr(local->xattr, xattr);
+        }
 
-                gf_uuid_unparse(stbuf->ia_gfid, gfid_node);
-                gf_uuid_unparse(local->gfid, gfid_local);
+        if (__is_root_gfid(stbuf->ia_gfid)) {
+            ret = dht_dir_has_layout(xattr, conf->xattr_name);
+            if (ret >= 0) {
+                if (is_greater_time(local->prebuf.ia_ctime,
+                                    local->prebuf.ia_ctime_nsec,
+                                    stbuf->ia_ctime, stbuf->ia_ctime_nsec)) {
+                    /* Choose source */
+                    local->prebuf.ia_gid = stbuf->ia_gid;
+                    local->prebuf.ia_uid = stbuf->ia_uid;
 
-                gf_msg (this->name, GF_LOG_WARNING, 0,
-                        DHT_MSG_GFID_MISMATCH,
-                        "%s: gfid different on %s."
-                        " gfid local = %s, gfid subvol = %s",
-                        local->loc.path, prev->this->name,
-                        gfid_local, gfid_node);
+                    local->prebuf.ia_ctime = stbuf->ia_ctime;
+                    local->prebuf.ia_ctime_nsec = stbuf->ia_ctime_nsec;
+                    local->prebuf.ia_prot = stbuf->ia_prot;
+                }
+            }
         }
 
-        LOCK (&frame->lock);
-        {
-                /* TODO: assert equal mode on stbuf->st_mode and
-                   local->stbuf->st_mode
+        if (local->stbuf.ia_type != IA_INVAL) {
+            /* This is not the first subvol to respond
+             * Compare values to see if attrs need to be healed
+             */
+            if ((local->stbuf.ia_gid != stbuf->ia_gid) ||
+                (local->stbuf.ia_uid != stbuf->ia_uid) ||
+                (is_permission_different(&local->stbuf.ia_prot,
+                                         &stbuf->ia_prot))) {
+                local->need_attrheal = 1;
+            }
+        }
 
-                   else mkdir/chmod/chown and fix
-                */
-                ret = dht_layout_merge (this, layout, prev->this,
-                                        op_ret, op_errno, xattr);
+        if (local->inode == NULL)
+            local->inode = inode_ref(inode);
 
-                if (op_ret == -1) {
-                        local->op_errno = op_errno;
-                        gf_msg_debug (this->name, 0,
-                                      "lookup of %s on %s returned error (%s)",
-                                      local->loc.path, prev->this->name,
-                                      strerror (op_errno));
+        dht_iatt_merge(this, &local->stbuf, stbuf);
+        dht_iatt_merge(this, &local->postparent, postparent);
 
-                        goto unlock;
-                }
+        if (!dict_get(xattr, conf->mds_xattr_key)) {
+            gf_msg_debug(this->name, 0,
+                         "%s: mds xattr %s is not present "
+                         "on %s(gfid = %s)",
+                         local->loc.path, conf->mds_xattr_key, prev->name,
+                         gfid_local);
+            goto unlock;
+        }
 
-                is_dir = check_is_dir (inode, stbuf, xattr);
-                if (!is_dir) {
+        /* Save the mds subvol info and stbuf. This is the value that will
+         * be used for healing
+         */
+        local->mds_subvol = prev;
+        local->mds_stbuf = *stbuf;
 
-                        gf_msg_debug (this->name, 0,
-                                      "lookup of %s on %s returned non"
-                                      "dir 0%o"
-                                      "calling lookup_everywhere",
-                                      local->loc.path, prev->this->name,
-                                      stbuf->ia_type);
+        /* Save mds subvol on inode ctx */
 
-                        local->need_selfheal = 1;
-                        goto unlock;
-                }
+        ret = dht_inode_ctx_mdsvol_set(local->inode, this, prev);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, DHT_MSG_SET_INODE_CTX_FAILED,
+                   "%s: Failed to set mds (%s)", local->loc.path, prev->name);
+        }
+        check_mds = dht_dict_get_array(xattr, conf->mds_xattr_key,
+                                       mds_xattr_val, 1, &errst);
+        if ((check_mds < 0) && !errst) {
+            /* Check if xattrs need to be healed on the directories */
+            local->mds_xattr = dict_ref(xattr);
+            gf_msg_debug(this->name, 0,
+                         "%s: %s is not zero on %s. Xattrs need to be healed."
+                         "(gfid = %s)",
+                         local->loc.path, conf->mds_xattr_key, prev->name,
+                         gfid_local);
+            local->need_xattr_heal = 1;
+        }
+    }
 
-                local->op_ret = 0;
-                if (local->xattr == NULL) {
-                        local->xattr = dict_ref (xattr);
-                } else {
-                        dht_aggregate_xattr (local->xattr, xattr);
-                }
+unlock:
+    UNLOCK(&frame->lock);
 
-                if (local->inode == NULL)
-                        local->inode = inode_ref (inode);
+    this_call_cnt = dht_frame_return(frame);
 
-                dht_iatt_merge (this, &local->stbuf, stbuf, prev->this);
-                dht_iatt_merge (this, &local->postparent, postparent,
-                                prev->this);
+    if (is_last_call(this_call_cnt)) {
+        /* If the mds subvol is not set correctly*/
+        if (!__is_root_gfid(local->gfid) &&
+            (!dict_get(local->xattr, conf->mds_xattr_key))) {
+            local->need_selfheal = 1;
         }
-unlock:
-        UNLOCK (&frame->lock);
 
+        /* No need to call xattr heal code if volume count is 1
+         */
+        if (conf->subvolume_cnt == 1) {
+            local->need_xattr_heal = 0;
+        }
 
-        this_call_cnt = dht_frame_return (frame);
+        if (local->need_selfheal || local->need_lookup_everywhere) {
+            /* Set the gfid-req so posix will set the GFID*/
+            if (!gf_uuid_is_null(local->gfid)) {
+                /* Ok, this should _never_ happen */
+                ret = dict_set_static_bin(local->xattr_req, "gfid-req",
+                                          local->gfid, 16);
+            } else {
+                if (!gf_uuid_is_null(local->gfid_req))
+                    ret = dict_set_static_bin(local->xattr_req, "gfid-req",
+                                              local->gfid_req, 16);
+            }
+        }
 
-        if (is_last_call (this_call_cnt)) {
-                if (local->need_selfheal) {
-                        local->need_selfheal = 0;
-                        dht_lookup_everywhere (frame, this, &local->loc);
-                        return 0;
-                }
+        if (local->need_lookup_everywhere) {
+            local->need_lookup_everywhere = 0;
+            dht_lookup_everywhere(frame, this, &local->loc);
+            return 0;
+        }
 
-                if (local->op_ret == 0) {
-                        ret = dht_layout_normalize (this, &local->loc, layout);
+        if (local->op_ret == 0) {
+            if (dht_needs_selfheal(frame, this)) {
+                goto selfheal;
+            }
 
-                        if (ret != 0) {
-                                gf_msg_debug (this->name, 0,
-                                              "fixing assignment on %s",
-                                              local->loc.path);
-                                goto selfheal;
-                        }
+            dht_layout_set(this, local->inode, layout);
+            if (local->inode) {
+                dht_inode_ctx_time_update(local->inode, this, &local->stbuf, 1);
+            }
 
-                        dht_layout_set (this, local->inode, layout);
-                }
+            if (local->loc.parent) {
+                dht_inode_ctx_time_update(local->loc.parent, this,
+                                          &local->postparent, 1);
+            }
+        }
 
-                if (local->loc.parent) {
-                        dht_inode_ctx_time_update (local->loc.parent, this,
-                                                   &local->postparent, 1);
-                }
+        DHT_STRIP_PHASE1_FLAGS(&local->stbuf);
+        dht_set_fixed_dir_stat(&local->postparent);
+        /* Delete mds xattr at the time of STACK UNWIND */
+        if (local->xattr)
+            GF_REMOVE_INTERNAL_XATTR(conf->mds_xattr_key, local->xattr);
 
-                DHT_STRIP_PHASE1_FLAGS (&local->stbuf);
-                DHT_STACK_UNWIND (lookup, frame, local->op_ret, local->op_errno,
-                                  local->inode, &local->stbuf, local->xattr,
-                                  &local->postparent);
-        }
+        DHT_STACK_UNWIND(lookup, frame, local->op_ret, local->op_errno,
+                         local->inode, &local->stbuf, local->xattr,
+                         &local->postparent);
+    }
 
-        return 0;
+    return 0;
 
 selfheal:
-        FRAME_SU_DO (frame, dht_local_t);
-        gf_uuid_copy (local->loc.gfid, local->gfid);
-        ret = dht_selfheal_directory (frame, dht_lookup_selfheal_cbk,
-                                      &local->loc, layout);
+    FRAME_SU_DO(frame, dht_local_t);
+    ret = dht_selfheal_directory(frame, dht_lookup_selfheal_cbk, &local->loc,
+                                 layout);
 out:
-        return ret;
+    return ret;
+}
+
+static int
+dht_lookup_directory(call_frame_t *frame, xlator_t *this, loc_t *loc)
+{
+    int call_cnt = 0;
+    int i = 0;
+    dht_conf_t *conf = NULL;
+    dht_local_t *local = NULL;
+    int ret = 0;
+
+    GF_VALIDATE_OR_GOTO("dht", frame, out);
+    GF_VALIDATE_OR_GOTO("dht", this, unwind);
+    GF_VALIDATE_OR_GOTO("dht", frame->local, unwind);
+    GF_VALIDATE_OR_GOTO("dht", this->private, unwind);
+    GF_VALIDATE_OR_GOTO("dht", loc, unwind);
+
+    conf = this->private;
+    local = frame->local;
+
+    call_cnt = conf->subvolume_cnt;
+    local->call_cnt = call_cnt;
+
+    local->layout = dht_layout_new(this, conf->subvolume_cnt);
+    if (!local->layout) {
+        goto unwind;
+    }
+
+    if (local->xattr != NULL) {
+        dict_unref(local->xattr);
+        local->xattr = NULL;
+    }
+
+    if (!gf_uuid_is_null(local->gfid)) {
+        /* use this gfid in order to heal any missing ones */
+        ret = dict_set_gfuuid(local->xattr_req, "gfid-req", local->gfid, true);
+        if (ret)
+            gf_msg(this->name, GF_LOG_WARNING, 0, DHT_MSG_DICT_SET_FAILED,
+                   "%s: Failed to set dictionary value:"
+                   " key = gfid-req",
+                   local->loc.path);
+    }
+
+    for (i = 0; i < call_cnt; i++) {
+        STACK_WIND_COOKIE(
+            frame, dht_lookup_dir_cbk, conf->subvolumes[i], conf->subvolumes[i],
+            conf->subvolumes[i]->fops->lookup, &local->loc, local->xattr_req);
+    }
+    return 0;
+unwind:
+    DHT_STACK_UNWIND(lookup, frame, -1, ENOMEM, NULL, NULL, NULL, NULL);
+out:
+    return 0;
 }
 
 int
-dht_revalidate_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                    int op_ret, int op_errno,
-                    inode_t *inode, struct iatt *stbuf, dict_t *xattr,
-                    struct iatt *postparent)
-{
-        dht_local_t  *local         = NULL;
-        int           this_call_cnt = 0;
-        call_frame_t *prev          = NULL;
-        dht_layout_t *layout        = NULL;
-        dht_conf_t   *conf          = NULL;
-        int           ret  = -1;
-        int           is_dir = 0;
-        int           is_linkfile = 0;
-        call_frame_t *copy          = NULL;
-        dht_local_t  *copy_local    = NULL;
-        char gfid[GF_UUID_BUF_SIZE] = {0};
-
-        GF_VALIDATE_OR_GOTO ("dht", frame, err);
-        GF_VALIDATE_OR_GOTO ("dht", this, err);
-        GF_VALIDATE_OR_GOTO ("dht", frame->local, err);
-        GF_VALIDATE_OR_GOTO ("dht", cookie, err);
+dht_revalidate_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                   int op_ret, int op_errno, inode_t *inode, struct iatt *stbuf,
+                   dict_t *xattr, struct iatt *postparent)
+{
+    dht_local_t *local = NULL;
+    int this_call_cnt = 0;
+    xlator_t *prev = NULL;
+    dht_layout_t *layout = NULL;
+    dht_conf_t *conf = NULL;
+    int ret = -1;
+    int is_dir = 0;
+    int is_linkfile = 0;
+    int follow_link = 0;
+    char gfid[GF_UUID_BUF_SIZE] = {0};
+    uint32_t vol_commit_hash = 0;
+    xlator_t *subvol = NULL;
+    int32_t check_mds = 0;
+    int errst = 0, i = 0;
+    int32_t mds_xattr_val[1] = {0};
+
+    GF_VALIDATE_OR_GOTO("dht", frame, err);
+    GF_VALIDATE_OR_GOTO("dht", this, err);
+    GF_VALIDATE_OR_GOTO("dht", frame->local, err);
+    GF_VALIDATE_OR_GOTO("dht", cookie, err);
+    GF_VALIDATE_OR_GOTO("dht", this->private, err);
+
+    local = frame->local;
+    prev = cookie;
+    conf = this->private;
+
+    if (!conf->vch_forced) {
+        /* Update the commithash value if available
+         */
+        ret = dict_get_uint32(xattr, conf->commithash_xattr_name,
+                              &vol_commit_hash);
+        if (ret == 0) {
+            conf->vol_commit_hash = vol_commit_hash;
+        }
+    }
 
-        local = frame->local;
-        prev  = cookie;
-        conf = this->private;
-        if (!conf)
-                goto out;
+    gf_uuid_unparse(local->loc.gfid, gfid);
 
-        gf_uuid_unparse (local->loc.gfid, gfid);
-
-        LOCK (&frame->lock);
-        {
-
-                gf_msg_debug (this->name, 0,
-                              "revalidate lookup of %s "
-                              "returned with op_ret %d and op_errno %d",
-                              local->loc.path, op_ret, op_errno);
-
-                if (op_ret == -1) {
-                        local->op_errno = op_errno;
-
-                        if ((op_errno != ENOTCONN)
-                            && (op_errno != ENOENT)
-                            && (op_errno != ESTALE)) {
-				gf_log (this->name, GF_LOG_INFO,
-					"Revalidate: subvolume %s for %s "
-                                        "(gfid = %s) returned -1 (%s)",
-					prev->this->name, local->loc.path,
-                                        gfid, strerror (op_errno));
-			}
-                        if (op_errno == ESTALE) {
-                                /* propagate the ESTALE to parent.
-                                 * setting local->return_estale would send
-                                 * ESTALE to parent. */
-                                local->return_estale = 1;
-                        }
+    gf_msg_debug(this->name, op_errno,
+                 "%s: revalidate lookup on %s returned op_ret %d",
+                 local->loc.path, prev->name, op_ret);
+
+    LOCK(&frame->lock);
+    {
+        if (gf_uuid_is_null(local->gfid)) {
+            memcpy(local->gfid, local->loc.gfid, 16);
+        }
 
-                        /* if it is ENOENT, we may have to do a
-                         * 'lookup_everywhere()' to make sure
-                         * the file is not migrated */
-                        if (op_errno == ENOENT) {
-                                if (IA_ISREG (local->loc.inode->ia_type)) {
-
-                                        gf_msg_debug (this->name, 0,
-                                                      "found ENOENT for %s. "
-                                                      "Setting "
-                                                      "need_lookup_everywhere"
-                                                      " flag to 1",
-                                                      local->loc.path);
-
-                                        local->need_lookup_everywhere = 1;
-                                }
+        if (op_ret == -1) {
+            local->op_errno = op_errno;
+
+            if ((op_errno != ENOTCONN) && (op_errno != ENOENT) &&
+                (op_errno != ESTALE)) {
+                gf_msg(this->name, GF_LOG_INFO, op_errno,
+                       DHT_MSG_REVALIDATE_CBK_INFO,
+                       "Revalidate: subvolume %s for %s "
+                       "(gfid = %s) returned -1",
+                       prev->name, local->loc.path, gfid);
+            }
+            if (op_errno == ESTALE) {
+                /* propagate the ESTALE to parent.
+                 * setting local->return_estale would send
+                 * ESTALE to parent. */
+                local->return_estale = 1;
+            }
+
+            /* if it is ENOENT, we may have to do a
+             * 'lookup_everywhere()' to make sure
+             * the file is not migrated */
+            if (op_errno == ENOENT) {
+                if (IA_ISREG(local->loc.inode->ia_type)) {
+                    gf_msg_debug(this->name, 0,
+                                 "found ENOENT for %s. "
+                                 "Setting "
+                                 "need_lookup_everywhere"
+                                 " flag to 1",
+                                 local->loc.path);
+
+                    local->need_lookup_everywhere = 1;
+                } else if (IA_ISDIR(local->loc.inode->ia_type)) {
+                    layout = local->layout;
+                    for (i = 0; i < layout->cnt; i++) {
+                        if (layout->list[i].xlator == prev) {
+                            layout->list[i].err = op_errno;
+                            break;
                         }
-                        goto unlock;
+                    }
+
+                    local->need_selfheal = 1;
                 }
+            }
 
-                if (stbuf->ia_type != local->inode->ia_type) {
-                        gf_msg (this->name, GF_LOG_WARNING, 0,
-                                DHT_MSG_FILE_TYPE_MISMATCH,
-                                "mismatching filetypes 0%o v/s 0%o for %s,"
-                                " gfid = %s",
-                                (stbuf->ia_type), (local->inode->ia_type),
-                                local->loc.path, gfid);
+            /* The GFID is missing on this subvol. Lookup everywhere to force a
+             * gfid heal
+             */
+            if ((op_errno == ENODATA) &&
+                (IA_ISDIR(local->loc.inode->ia_type))) {
+                local->need_lookup_everywhere = 1;
+            }
 
-                        local->op_ret = -1;
-                        local->op_errno = EINVAL;
+            goto unlock;
+        }
 
-                        goto unlock;
-                }
+        if ((!IA_ISINVAL(local->inode->ia_type)) &&
+            stbuf->ia_type != local->inode->ia_type) {
+            gf_msg(this->name, GF_LOG_WARNING, 0, DHT_MSG_FILE_TYPE_MISMATCH,
+                   "mismatching filetypes 0%o v/s 0%o for %s,"
+                   " gfid = %s",
+                   (stbuf->ia_type), (local->inode->ia_type), local->loc.path,
+                   gfid);
 
-                layout = local->layout;
+            local->op_ret = -1;
+            local->op_errno = EINVAL;
 
-                is_dir = check_is_dir (inode, stbuf, xattr);
-                is_linkfile = check_is_linkfile (inode, stbuf, xattr,
-                                                 conf->link_xattr_name);
+            goto unlock;
+        }
 
-                if (is_linkfile) {
-                        gf_log (this->name, GF_LOG_INFO,
-                                "Revalidate: linkfile found %s, (gfid = %s)",
-                                local->loc.path, gfid);
-                        local->return_estale = 1;
+        layout = local->layout;
 
-                        goto unlock;
-                }
+        is_dir = check_is_dir(inode, stbuf, xattr);
+        is_linkfile = check_is_linkfile(inode, stbuf, xattr,
+                                        conf->link_xattr_name);
+        if (is_linkfile) {
+            follow_link = 1;
+            goto unlock;
+        }
+        if (is_dir) {
+            ret = dht_dir_has_layout(xattr, conf->xattr_name);
+            if (ret >= 0) {
+                if (is_greater_time(local->prebuf.ia_ctime,
+                                    local->prebuf.ia_ctime_nsec,
+                                    stbuf->ia_ctime, stbuf->ia_ctime_nsec)) {
+                    /* Choose source */
+                    local->prebuf.ia_gid = stbuf->ia_gid;
+                    local->prebuf.ia_uid = stbuf->ia_uid;
+
+                    local->prebuf.ia_ctime = stbuf->ia_ctime;
+                    local->prebuf.ia_ctime_nsec = stbuf->ia_ctime_nsec;
+
+                    if (__is_root_gfid(stbuf->ia_gfid))
+                        local->prebuf.ia_prot = stbuf->ia_prot;
+                }
+            }
+
+            if (local->stbuf.ia_type != IA_INVAL) {
+                if ((local->stbuf.ia_gid != stbuf->ia_gid) ||
+                    (local->stbuf.ia_uid != stbuf->ia_uid) ||
+                    is_permission_different(&local->stbuf.ia_prot,
+                                            &stbuf->ia_prot)) {
+                    local->need_attrheal = 1;
+                }
+            }
+
+            if (!dict_get(xattr, conf->mds_xattr_key)) {
+                gf_msg_debug(this->name, 0,
+                             "%s: internal xattr %s is not present"
+                             " on subvol %s(gfid is %s)",
+                             local->loc.path, conf->mds_xattr_key, prev->name,
+                             gfid);
+            } else {
+                check_mds = dht_dict_get_array(xattr, conf->mds_xattr_key,
+                                               mds_xattr_val, 1, &errst);
+                local->mds_subvol = prev;
+                local->mds_stbuf.ia_gid = stbuf->ia_gid;
+                local->mds_stbuf.ia_uid = stbuf->ia_uid;
+                local->mds_stbuf.ia_prot = stbuf->ia_prot;
+
+                /* save mds subvol on inode ctx */
+                ret = dht_inode_ctx_mdsvol_set(local->inode, this, prev);
+                if (ret) {
+                    gf_msg(this->name, GF_LOG_ERROR, 0,
+                           DHT_MSG_SET_INODE_CTX_FAILED,
+                           "Failed to set MDS subvol for %s vol is %s",
+                           local->loc.path, prev->name);
+                }
+                if ((check_mds < 0) && !errst) {
+                    /* Check if xattrs need to be healed on the directory
+                     */
+                    local->mds_xattr = dict_ref(xattr);
+                    gf_msg_debug(this->name, 0,
+                                 "Value of %s is not zero on "
+                                 "hashed subvol so xattr needs to"
+                                 " be healed on non hashed"
+                                 " path is %s and vol name is %s "
+                                 " gfid is %s",
+                                 conf->mds_xattr_key, local->loc.path,
+                                 prev->name, gfid);
+                    local->need_xattr_heal = 1;
+                }
+            }
+            ret = dht_layout_dir_mismatch(this, layout, prev, &local->loc,
+                                          xattr);
+            if (ret != 0) {
+                /* In memory layout does not match on-disk layout.
+                 */
+                gf_msg(this->name, GF_LOG_INFO, 0, DHT_MSG_LAYOUT_MISMATCH,
+                       "Mismatching layouts for %s, gfid = %s", local->loc.path,
+                       gfid);
 
-                if (is_dir) {
-                        ret = dht_dir_has_layout (xattr, conf->xattr_name);
-                        if (ret >= 0) {
-                                if (is_greater_time(local->stbuf.ia_ctime,
-                                                    local->stbuf.ia_ctime_nsec,
-                                                    stbuf->ia_ctime,
-                                                    stbuf->ia_ctime_nsec)) {
-                                        local->prebuf.ia_gid = stbuf->ia_gid;
-                                        local->prebuf.ia_uid = stbuf->ia_uid;
-                                }
-                        }
-                        if (local->stbuf.ia_type != IA_INVAL)
-                        {
-                                if ((local->stbuf.ia_gid != stbuf->ia_gid) ||
-                                    (local->stbuf.ia_uid != stbuf->ia_uid)) {
-                                        local->need_selfheal = 1;
-                                }
-                        }
-                        ret = dht_layout_dir_mismatch (this, layout,
-                                                       prev->this, &local->loc,
-                                                       xattr);
-                        if (ret != 0) {
-                                gf_msg (this->name, GF_LOG_INFO, 0,
-                                        DHT_MSG_LAYOUT_MISMATCH,
-                                        "Mismatching layouts for %s, gfid = %s",
-                                        local->loc.path, gfid);
-
-                                local->layout_mismatch = 1;
-
-                                goto unlock;
-                        }
-                }
+                local->layout_mismatch = 1;
 
-                dht_iatt_merge (this, &local->stbuf, stbuf, prev->this);
-                dht_iatt_merge (this, &local->postparent, postparent,
-                                prev->this);
+                goto unlock;
+            }
+        }
 
-                local->op_ret = 0;
+        gf_uuid_copy(local->stbuf.ia_gfid, stbuf->ia_gfid);
+        dht_iatt_merge(this, &local->stbuf, stbuf);
+        dht_iatt_merge(this, &local->postparent, postparent);
 
-                if (!local->xattr) {
-                        local->xattr = dict_ref (xattr);
-                } else if (is_dir) {
-                        dht_aggregate_xattr (local->xattr, xattr);
-                }
+        local->op_ret = 0;
+
+        if (!local->xattr) {
+            local->xattr = dict_ref(xattr);
+        } else if (is_dir) {
+            dht_aggregate_xattr(local->xattr, xattr);
         }
+    }
 unlock:
-        UNLOCK (&frame->lock);
-out:
-        this_call_cnt = dht_frame_return (frame);
-
-        if (is_last_call (this_call_cnt)) {
-                if (!IA_ISDIR (local->stbuf.ia_type)
-                    && (local->hashed_subvol != local->cached_subvol)
-                    && (local->stbuf.ia_nlink == 1)
-                    && (conf && conf->unhashed_sticky_bit)) {
-                        local->stbuf.ia_prot.sticky = 1;
-                }
-                if (local->need_selfheal) {
-                        local->need_selfheal = 0;
-                        gf_uuid_copy (local->gfid, local->stbuf.ia_gfid);
-                        local->stbuf.ia_gid = local->prebuf.ia_gid;
-                        local->stbuf.ia_uid = local->prebuf.ia_uid;
-                        copy = create_frame (this, this->ctx->pool);
-                        if (copy) {
-                                copy_local = dht_local_init (copy, &local->loc,
-                                                             NULL, 0);
-                                if (!copy_local)
-                                        goto cont;
-                                copy_local->stbuf = local->stbuf;
-                                copy->local = copy_local;
-                                FRAME_SU_DO (copy, dht_local_t);
-                                ret = synctask_new (this->ctx->env,
-                                                    dht_dir_attr_heal,
-                                                    dht_dir_attr_heal_done,
-                                                    copy, copy);
-                        }
-                }
-cont:
-		if (local->layout_mismatch) {
-                        /* Found layout mismatch in the directory, need to
-                           fix this in the inode context */
-                        dht_layout_unref (this, local->layout);
-                        local->layout = NULL;
-                        dht_lookup_directory (frame, this, &local->loc);
-                        return 0;
-                }
+    UNLOCK(&frame->lock);
 
-                if (local->need_lookup_everywhere) {
-                        /* As the current layout gave ENOENT error, we would
-                           need a new layout */
-                        dht_layout_unref (this, local->layout);
-                        local->layout = NULL;
-
-                        /* We know that current cached subvol is no more
-                           valid, get the new one */
-                        local->cached_subvol = NULL;
-                        dht_lookup_everywhere (frame, this, &local->loc);
-                        return 0;
-                }
-                if (local->return_estale) {
-                        local->op_ret = -1;
-                        local->op_errno = ESTALE;
-                }
-
-                if (local->loc.parent) {
-                        dht_inode_ctx_time_update (local->loc.parent, this,
-                                                   &local->postparent, 1);
-                }
+    if (follow_link) {
+        /* Found a linkto file. Follow it to see if the target file exists
+         */
+        gf_uuid_copy(local->gfid, stbuf->ia_gfid);
 
-                DHT_STRIP_PHASE1_FLAGS (&local->stbuf);
-                DHT_STACK_UNWIND (lookup, frame, local->op_ret, local->op_errno,
-                                  local->inode, &local->stbuf, local->xattr,
-                                  &local->postparent);
+        subvol = dht_linkfile_subvol(this, inode, stbuf, xattr);
+        if (!subvol) {
+            op_errno = ESTALE;
+            local->op_ret = -1;
+        } else {
+            STACK_WIND_COOKIE(frame, dht_lookup_linkfile_cbk, subvol, subvol,
+                              subvol->fops->lookup, &local->loc,
+                              local->xattr_req);
+            return 0;
         }
+    }
 
-err:
-        return ret;
-}
+    this_call_cnt = dht_frame_return(frame);
 
+    if (is_last_call(this_call_cnt)) {
+        if (!IA_ISDIR(local->stbuf.ia_type) &&
+            (local->hashed_subvol != local->cached_subvol) &&
+            (local->stbuf.ia_nlink == 1) &&
+            (conf && conf->unhashed_sticky_bit)) {
+            local->stbuf.ia_prot.sticky = 1;
+        }
+        /* No need to call heal code if volume count is 1
+         */
+        if (conf->subvolume_cnt == 1)
+            local->need_xattr_heal = 0;
+
+        if (IA_ISDIR(local->stbuf.ia_type)) {
+            /* No mds xattr found. Trigger a heal to set it */
+            if (!__is_root_gfid(local->loc.inode->gfid) &&
+                (!dict_get(local->xattr, conf->mds_xattr_key)))
+                local->need_selfheal = 1;
+
+            if (dht_needs_selfheal(frame, this)) {
+                if (!__is_root_gfid(local->loc.inode->gfid)) {
+                    if (local->mds_subvol) {
+                        local->stbuf.ia_gid = local->mds_stbuf.ia_gid;
+                        local->stbuf.ia_uid = local->mds_stbuf.ia_uid;
+                        local->stbuf.ia_prot = local->mds_stbuf.ia_prot;
+                    }
+                } else {
+                    local->stbuf.ia_gid = local->prebuf.ia_gid;
+                    local->stbuf.ia_uid = local->prebuf.ia_uid;
+                    local->stbuf.ia_prot = local->prebuf.ia_prot;
+                }
 
-int
-dht_lookup_linkfile_create_cbk (call_frame_t *frame, void *cookie,
-                                xlator_t *this,
-                                int32_t op_ret, int32_t op_errno,
-                                inode_t *inode, struct iatt *stbuf,
-                                struct iatt *preparent, struct iatt *postparent,
-                                dict_t *xdata)
-{
-        dht_local_t  *local = NULL;
-        xlator_t     *cached_subvol = NULL;
-        dht_conf_t   *conf = NULL;
-        int           ret = -1;
-        char          gfid[GF_UUID_BUF_SIZE] = {0};
-
-        GF_VALIDATE_OR_GOTO ("dht", frame, out);
-        GF_VALIDATE_OR_GOTO ("dht", this, out);
-        GF_VALIDATE_OR_GOTO ("dht", frame->local, out);
-        GF_VALIDATE_OR_GOTO ("dht", this->private, out);
-        GF_VALIDATE_OR_GOTO ("dht", cookie, out);
+                layout = local->layout;
+                dht_selfheal_directory(frame, dht_lookup_selfheal_cbk,
+                                       &local->loc, layout);
+                return 0;
+            }
+        }
 
-        local = frame->local;
-        cached_subvol = local->cached_subvol;
-        conf = this->private;
+        if (local->layout_mismatch) {
+            /* Found layout mismatch in the directory, need to
+               fix this in the inode context */
+            dht_layout_unref(this, local->layout);
+            local->layout = NULL;
+            dht_lookup_directory(frame, this, &local->loc);
+            return 0;
+        }
 
-        gf_uuid_unparse(local->loc.gfid, gfid);
+        if (local->need_lookup_everywhere) {
+            /* As the current layout gave ENOENT error, we would
+               need a new layout */
+            dht_layout_unref(this, local->layout);
+            local->layout = NULL;
 
-        ret = dht_layout_preset (this, local->cached_subvol, local->loc.inode);
-        if (ret < 0) {
-                gf_msg_debug (this->name, 0,
-                              "Failed to set layout for subvolume %s, "
-                              "(gfid = %s)",
-                              cached_subvol ? cached_subvol->name : "<nil>",
-                              gfid);
-                local->op_ret = -1;
-                local->op_errno = EINVAL;
-                goto unwind;
-        }
+            /* We know that current cached subvol is no longer
+               valid, get the new one */
+            local->cached_subvol = NULL;
+            if (local->xattr_req) {
+                if (!gf_uuid_is_null(local->gfid)) {
+                    ret = dict_set_static_bin(local->xattr_req, "gfid-req",
+                                              local->gfid, 16);
+                }
+            }
 
-        local->op_ret = 0;
-        if ((local->stbuf.ia_nlink == 1)
-            && (conf && conf->unhashed_sticky_bit)) {
-                local->stbuf.ia_prot.sticky = 1;
+            dht_lookup_everywhere(frame, this, &local->loc);
+            return 0;
+        }
+        if (local->return_estale) {
+            local->op_ret = -1;
+            local->op_errno = ESTALE;
         }
 
         if (local->loc.parent) {
-                dht_inode_ctx_time_update (local->loc.parent, this,
-                                           postparent, 1);
+            dht_inode_ctx_time_update(local->loc.parent, this,
+                                      &local->postparent, 1);
         }
 
+        DHT_STRIP_PHASE1_FLAGS(&local->stbuf);
+        dht_set_fixed_dir_stat(&local->postparent);
+
+        /* local->stbuf is updated only from subvols which have a layout
+         * The reason is to avoid choosing attr heal source from newly
+         * added bricks. In case e.g we have only one subvol and for
+         * some reason layout is not present on it, then local->stbuf
+         * will be EINVAL. This is an indication that the subvols
+         * active in the cluster do not have layouts on disk.
+         * Unwind with ESTALE to trigger a fresh lookup */
+        if (is_dir && local->stbuf.ia_type == IA_INVAL) {
+            local->op_ret = -1;
+            local->op_errno = ESTALE;
+        }
+        /* Delete mds xattr at the time of STACK UNWIND */
+        if (local->xattr)
+            GF_REMOVE_INTERNAL_XATTR(conf->mds_xattr_key, local->xattr);
+
+        DHT_STACK_UNWIND(lookup, frame, local->op_ret, local->op_errno,
+                         local->inode, &local->stbuf, local->xattr,
+                         &local->postparent);
+    }
+
+err:
+    return ret;
+}
+
+static int
+dht_lookup_linkfile_create_cbk(call_frame_t *frame, void *cooie, xlator_t *this,
+                               int32_t op_ret, int32_t op_errno, inode_t *inode,
+                               struct iatt *stbuf, struct iatt *preparent,
+                               struct iatt *postparent, dict_t *xdata)
+{
+    dht_local_t *local = NULL;
+    xlator_t *cached_subvol = NULL;
+    dht_conf_t *conf = NULL;
+    int ret = -1;
+    char gfid[GF_UUID_BUF_SIZE] = {0};
+
+    GF_VALIDATE_OR_GOTO("dht", frame, out);
+    GF_VALIDATE_OR_GOTO("dht", this, out);
+    GF_VALIDATE_OR_GOTO("dht", frame->local, out);
+    GF_VALIDATE_OR_GOTO("dht", this->private, out);
+
+    local = frame->local;
+    cached_subvol = local->cached_subvol;
+    conf = this->private;
+
+    gf_uuid_unparse(local->loc.gfid, gfid);
+
+    if (local->locked)
+        dht_unlock_namespace(frame, &local->lock[0]);
+
+    ret = dht_layout_preset(this, local->cached_subvol, local->loc.inode);
+    if (ret < 0) {
+        gf_msg_debug(this->name, EINVAL,
+                     "Failed to set layout for subvolume %s, "
+                     "(gfid = %s)",
+                     cached_subvol ? cached_subvol->name : "<nil>", gfid);
+        local->op_ret = -1;
+        local->op_errno = EINVAL;
+        goto unwind;
+    }
+
+    local->op_ret = 0;
+    if ((local->stbuf.ia_nlink == 1) && (conf && conf->unhashed_sticky_bit)) {
+        local->stbuf.ia_prot.sticky = 1;
+    }
+
+    if (local->loc.parent) {
+        dht_inode_ctx_time_update(local->loc.parent, this, postparent, 1);
+    }
+
 unwind:
-        gf_msg_debug (this->name, 0,
-                      "creation of linkto on hashed subvol:%s, "
-                      "returned with op_ret %d and op_errno %d: %s",
-                      local->hashed_subvol->name,
-                      op_ret, op_errno, uuid_utoa (local->loc.gfid));
+    gf_msg_debug(this->name, 0,
+                 "creation of linkto on hashed subvol:%s, "
+                 "returned with op_ret %d and op_errno %d: %s",
+                 local->hashed_subvol->name, op_ret, op_errno,
+                 uuid_utoa(local->loc.gfid));
 
-        if (local->linked == _gf_true)
-                dht_linkfile_attr_heal (frame, this);
+    if (local->linked == _gf_true)
+        dht_linkfile_attr_heal(frame, this);
 
+    dht_set_fixed_dir_stat(&local->postparent);
 
-        DHT_STRIP_PHASE1_FLAGS (&local->stbuf);
-        DHT_STACK_UNWIND (lookup, frame, local->op_ret, local->op_errno,
-                          local->inode, &local->stbuf, local->xattr,
-                          &local->postparent);
+    DHT_STRIP_PHASE1_FLAGS(&local->stbuf);
+    DHT_STACK_UNWIND(lookup, frame, local->op_ret, local->op_errno,
+                     local->inode, &local->stbuf, local->xattr,
+                     &local->postparent);
 out:
-        return ret;
+    return ret;
 }
 
-int
-dht_lookup_unlink_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                       int op_ret, int op_errno,
-                       struct iatt *preparent, struct iatt *postparent,
-                       dict_t *xdata)
+static int
+dht_lookup_unlink_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                      int op_ret, int op_errno, struct iatt *preparent,
+                      struct iatt *postparent, dict_t *xdata)
 {
-        int             this_call_cnt = 0;
-        dht_local_t     *local = NULL;
-        const char      *path =  NULL;
+    int this_call_cnt = 0;
+    dht_local_t *local = NULL;
+    const char *path = NULL;
 
-        local =  (dht_local_t*)frame->local;
-        path = local->loc.path;
+    local = (dht_local_t *)frame->local;
+    path = local->loc.path;
+    FRAME_SU_UNDO(frame, dht_local_t);
 
-        gf_log (this->name, GF_LOG_INFO, "lookup_unlink returned with "
-                "op_ret -> %d and op-errno -> %d for %s", op_ret, op_errno,
-                ((path == NULL)? "null" : path ));
+    gf_msg(this->name, GF_LOG_INFO, 0, DHT_MSG_UNLINK_LOOKUP_INFO,
+           "lookup_unlink returned with "
+           "op_ret -> %d and op-errno -> %d for %s",
+           op_ret, op_errno, ((path == NULL) ? "null" : path));
 
-        this_call_cnt = dht_frame_return (frame);
-        if (is_last_call (this_call_cnt)) {
-                dht_lookup_everywhere_done (frame, this);
-        }
+    this_call_cnt = dht_frame_return(frame);
+    if (is_last_call(this_call_cnt)) {
+        dht_lookup_everywhere_done(frame, this);
+    }
 
-        return 0;
+    return 0;
 }
 
-int
-dht_lookup_unlink_of_false_linkto_cbk (call_frame_t *frame, void *cookie,
-                                       xlator_t *this, int op_ret, int op_errno,
-                                       struct iatt *preparent,
-                                       struct iatt *postparent, dict_t *xdata)
+static int
+dht_lookup_unlink_of_false_linkto_cbk(call_frame_t *frame, void *cookie,
+                                      xlator_t *this, int op_ret, int op_errno,
+                                      struct iatt *preparent,
+                                      struct iatt *postparent, dict_t *xdata)
 {
-        int             this_call_cnt = 0;
-        dht_local_t     *local = NULL;
-        const char      *path =  NULL;
+    int this_call_cnt = 0;
+    dht_local_t *local = NULL;
+    const char *path = NULL;
 
-        local =  (dht_local_t*)frame->local;
-        path = local->loc.path;
+    local = (dht_local_t *)frame->local;
+    path = local->loc.path;
 
-        gf_log (this->name, GF_LOG_INFO, "lookup_unlink returned with "
-                "op_ret -> %d and op-errno -> %d for %s", op_ret, op_errno,
-                ((path == NULL)? "null" : path ));
+    FRAME_SU_UNDO(frame, dht_local_t);
 
-        this_call_cnt = dht_frame_return (frame);
-        if (is_last_call (this_call_cnt)) {
+    gf_msg(this->name, GF_LOG_INFO, 0, DHT_MSG_UNLINK_LOOKUP_INFO,
+           "lookup_unlink returned with "
+           "op_ret -> %d and op-errno -> %d for %s",
+           op_ret, op_errno, ((path == NULL) ? "null" : path));
 
-                if (op_ret == 0) {
-                        dht_lookup_everywhere_done (frame, this);
-                } else {
-                       /*When dht_lookup_everywhere is performed, one cached
-                         *and one hashed file was found and hashed file does
-                         *not point to the above mentioned cached node. So it
-                         *was considered as stale and an unlink was performed.
-                         *But unlink fails. So may be rebalance is in progress.
-                        *now ideally we have two data-files. One obtained during
-                         *lookup_everywhere and one where unlink-failed. So
-                         *at this point in time we cannot decide which one to
-                         *choose because there are chances of first cached
-                         *file is truncated after rebalance and if it is chosen
-                        *as cached node, application will fail. So return EIO.*/
-
-                        if (op_errno == EBUSY) {
-
-                                gf_log (this->name, GF_LOG_ERROR,
-                                        "Could not unlink the linkto file as "
-                                        "either fd is open and/or linkto xattr "
-                                        "is set for %s",
-                                        ((path == NULL)? "null":path));
+    this_call_cnt = dht_frame_return(frame);
+    if (is_last_call(this_call_cnt)) {
+        if ((op_ret == 0) || ((op_errno != EBUSY) && (op_errno != ENOTCONN))) {
+            dht_lookup_everywhere_done(frame, this);
+        } else {
+            /*When dht_lookup_everywhere is performed, one cached
+             *and one hashed file was found and hashed file does
+             *not point to the above mentioned cached node. So it
+             *was considered as stale and an unlink was performed.
+             *But unlink fails. So may be rebalance is in progress.
+             *now ideally we have two data-files. One obtained during
+             *lookup_everywhere and one where unlink-failed. So
+             *at this point in time we cannot decide which one to
+             *choose because there are chances of first cached
+             *file is truncated after rebalance and if it is chosen
+             *as cached node, application will fail. So return EIO.*/
+
+            if (op_errno == EBUSY) {
+                gf_msg(this->name, GF_LOG_ERROR, op_errno,
+                       DHT_MSG_UNLINK_FAILED,
+                       "Could not unlink the linkto file as "
+                       "either fd is open and/or linkto xattr "
+                       "is set for %s",
+                       ((path == NULL) ? "null" : path));
+            }
+            DHT_STACK_UNWIND(lookup, frame, -1, EIO, NULL, NULL, NULL, NULL);
+        }
+    }
+
+    return 0;
+}
 
-                        }
-                        DHT_STACK_UNWIND (lookup, frame, -1, EIO, NULL, NULL,
-                                          NULL, NULL);
+static int
+dht_lookup_unlink_stale_linkto_cbk(call_frame_t *frame, void *cookie,
+                                   xlator_t *this, int op_ret, int op_errno,
+                                   struct iatt *preparent,
+                                   struct iatt *postparent, dict_t *xdata)
+{
+    dht_local_t *local = NULL;
+    const char *path = NULL;
 
-                }
-        }
+    /* NOTE:
+     * If stale file unlink fails either there is an open-fd or is not an
+     * dht-linkto-file then posix_unlink returns EBUSY, which is overwritten
+     *  to ENOENT
+     */
 
-        return 0;
+    local = frame->local;
+
+    if (local) {
+        FRAME_SU_UNDO(frame, dht_local_t);
+        if (local->loc.path)
+            path = local->loc.path;
+    }
+
+    gf_msg(this->name, GF_LOG_INFO, 0, DHT_MSG_UNLINK_LOOKUP_INFO,
+           "Returned with op_ret %d and "
+           "op_errno %d for %s",
+           op_ret, op_errno, ((path == NULL) ? "null" : path));
+
+    DHT_STACK_UNWIND(lookup, frame, -1, ENOENT, NULL, NULL, NULL, NULL);
+
+    return 0;
 }
 
-int
-dht_lookup_unlink_stale_linkto_cbk (call_frame_t *frame, void *cookie,
-                                    xlator_t *this, int op_ret, int op_errno,
-                                    struct iatt *preparent,
-                                    struct iatt *postparent, dict_t *xdata)
+static int
+dht_fill_dict_to_avoid_unlink_of_migrating_file(dict_t *dict)
 {
+    int ret = 0;
 
-        dht_local_t     *local = NULL;
-        const char      *path  = NULL;
-
-        /* NOTE:
-         * If stale file unlink fails either there is an open-fd or is not an
-         * dht-linkto-file then posix_unlink returns EBUSY, which is overwritten
-         *  to ENOENT
-         */
+    ret = dict_set_int32_sizen(dict, DHT_SKIP_NON_LINKTO_UNLINK, 1);
 
-        local = frame->local;
+    if (ret)
+        return -1;
 
-        if (local && local->loc.path)
-                path = local->loc.path;
+    ret = dict_set_int32_sizen(dict, DHT_SKIP_OPEN_FD_UNLINK, 1);
 
-        gf_log (this->name, GF_LOG_INFO, "Returned with op_ret %d and "
-                "op_errno %d for %s", op_ret, op_errno,
-                ((path==NULL)?"null":path));
+    if (ret)
+        return -1;
 
-        DHT_STACK_UNWIND (lookup, frame, -1, ENOENT, NULL, NULL, NULL,
-                          NULL);
+    return 0;
+}
 
-        return 0;
+static int32_t
+dht_linkfile_create_lookup_cbk(call_frame_t *frame, void *cookie,
+                               xlator_t *this, int32_t op_ret, int32_t op_errno,
+                               inode_t *inode, struct iatt *buf, dict_t *xdata,
+                               struct iatt *postparent)
+{
+    dht_local_t *local = NULL;
+    int call_cnt = 0, ret = 0;
+    xlator_t *subvol = NULL;
+    uuid_t gfid = {
+        0,
+    };
+    char gfid_str[GF_UUID_BUF_SIZE] = {0};
+
+    subvol = cookie;
+    local = frame->local;
+
+    if (subvol == local->hashed_subvol) {
+        if ((op_ret == 0) || (op_errno != ENOENT))
+            local->dont_create_linkto = _gf_true;
+    } else {
+        if (gf_uuid_is_null(local->gfid))
+            gf_uuid_copy(gfid, local->loc.gfid);
+        else
+            gf_uuid_copy(gfid, local->gfid);
+
+        if ((op_ret == 0) && gf_uuid_compare(gfid, buf->ia_gfid)) {
+            gf_uuid_unparse(gfid, gfid_str);
+            gf_msg_debug(this->name, 0,
+                         "gfid (%s) different on cached subvol "
+                         "(%s) and looked up inode (%s), not "
+                         "creating linkto",
+                         uuid_utoa(buf->ia_gfid), subvol->name, gfid_str);
+            local->dont_create_linkto = _gf_true;
+        } else if (op_ret == -1) {
+            local->dont_create_linkto = _gf_true;
+        }
+    }
+
+    call_cnt = dht_frame_return(frame);
+    if (is_last_call(call_cnt)) {
+        if (local->dont_create_linkto)
+            goto no_linkto;
+        else {
+            gf_msg_debug(this->name, 0,
+                         "Creating linkto file on %s(hash) to "
+                         "%s on %s (gfid = %s)",
+                         local->hashed_subvol->name, local->loc.path,
+                         local->cached_subvol->name, gfid_str);
+
+            ret = dht_linkfile_create(frame, dht_lookup_linkfile_create_cbk,
+                                      this, local->cached_subvol,
+                                      local->hashed_subvol, &local->loc);
+
+            if (ret < 0)
+                goto no_linkto;
+        }
+    }
+
+    return 0;
+
+no_linkto:
+    gf_msg_debug(this->name, 0,
+                 "skipped linkto creation (path:%s) (gfid:%s) "
+                 "(hashed-subvol:%s) (cached-subvol:%s)",
+                 local->loc.path, gfid_str, local->hashed_subvol->name,
+                 local->cached_subvol->name);
+
+    dht_lookup_linkfile_create_cbk(frame, NULL, this, 0, 0, local->loc.inode,
+                                   &local->stbuf, &local->preparent,
+                                   &local->postparent, local->xattr);
+    return 0;
 }
 
-int
-dht_fill_dict_to_avoid_unlink_of_migrating_file (dict_t *dict) {
+static int32_t
+dht_call_lookup_linkfile_create(call_frame_t *frame, void *cookie,
+                                xlator_t *this, int32_t op_ret,
+                                int32_t op_errno, dict_t *xdata)
+{
+    dht_local_t *local = NULL;
+    char gfid[GF_UUID_BUF_SIZE] = {0};
+    int i = 0;
+    xlator_t *subvol = NULL;
 
-        int ret = 0;
+    local = frame->local;
+    if (gf_uuid_is_null(local->gfid))
+        gf_uuid_unparse(local->loc.gfid, gfid);
+    else
+        gf_uuid_unparse(local->gfid, gfid);
 
-        ret = dict_set_int32 (dict, DHT_SKIP_NON_LINKTO_UNLINK, 1);
+    if (op_ret < 0) {
+        gf_log(this->name, GF_LOG_WARNING,
+               "protecting namespace failed, skipping linkto "
+               "creation (path:%s)(gfid:%s)(hashed-subvol:%s)"
+               "(cached-subvol:%s)",
+               local->loc.path, gfid, local->hashed_subvol->name,
+               local->cached_subvol->name);
+        goto err;
+    }
 
-        if (ret)
-                goto err;
+    local->locked = _gf_true;
 
-        ret =  dict_set_int32 (dict, DHT_SKIP_OPEN_FD_UNLINK, 1);
+    local->call_cnt = 2;
 
-        if (ret)
-                goto err;
+    for (i = 0; i < 2; i++) {
+        subvol = (subvol == NULL) ? local->hashed_subvol : local->cached_subvol;
 
+        STACK_WIND_COOKIE(frame, dht_linkfile_create_lookup_cbk, subvol, subvol,
+                          subvol->fops->lookup, &local->loc, NULL);
+    }
 
-        return 0;
+    return 0;
 
 err:
-        return -1;
-
+    dht_lookup_linkfile_create_cbk(frame, NULL, this, 0, 0, local->loc.inode,
+                                   &local->stbuf, &local->preparent,
+                                   &local->postparent, local->xattr);
+    return 0;
 }
+
 /* Rebalance is performed from cached_node to hashed_node. Initial cached_node
  * contains a non-linkto file. After migration it is converted to linkto and
  * then unlinked. And at hashed_subvolume, first a linkto file is present,
@@ -1103,5415 +2335,9057 @@ err:
  * dht_lookup_everywhere_done takes decision based on any of the above case
  */
 
-int
-dht_lookup_everywhere_done (call_frame_t *frame, xlator_t *this)
+static int
+dht_lookup_everywhere_done(call_frame_t *frame, xlator_t *this)
 {
-        int           ret = 0;
-        dht_local_t  *local = NULL;
-        xlator_t     *hashed_subvol = NULL;
-        xlator_t     *cached_subvol = NULL;
-        dht_layout_t *layout = NULL;
-        char gfid[GF_UUID_BUF_SIZE] = {0};
-        gf_boolean_t  found_non_linkto_on_hashed = _gf_false;
+    int ret = 0;
+    dht_local_t *local = NULL;
+    xlator_t *hashed_subvol = NULL;
+    xlator_t *cached_subvol = NULL;
+    dht_layout_t *layout = NULL;
+    char gfid[GF_UUID_BUF_SIZE] = {0};
+    gf_boolean_t found_non_linkto_on_hashed = _gf_false;
+
+    local = frame->local;
+    hashed_subvol = local->hashed_subvol;
+    cached_subvol = local->cached_subvol;
+
+    gf_uuid_unparse(local->loc.gfid, gfid);
+
+    if (local->file_count && local->dir_count) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, DHT_MSG_FILE_TYPE_MISMATCH,
+               "path %s (gfid = %s)exists as a file on one "
+               "subvolume and directory on another. "
+               "Please fix it manually",
+               local->loc.path, gfid);
+        DHT_STACK_UNWIND(lookup, frame, -1, EIO, NULL, NULL, NULL, NULL);
+        return 0;
+    }
+    if (local->op_ret && local->gfid_missing) {
+        if (gf_uuid_is_null(local->gfid_req)) {
+            DHT_STACK_UNWIND(lookup, frame, -1, ENODATA, NULL, NULL, NULL,
+                             NULL);
+            return 0;
+        }
+        /* A hack */
+        dht_lookup_directory(frame, this, &local->loc);
+        return 0;
+    }
 
-        local = frame->local;
-        hashed_subvol = local->hashed_subvol;
-        cached_subvol = local->cached_subvol;
+    if (local->dir_count) {
+        dht_lookup_directory(frame, this, &local->loc);
+        return 0;
+    }
+
+    gf_msg_debug(this->name, 0,
+                 "STATUS: hashed_subvol %s "
+                 "cached_subvol %s",
+                 (hashed_subvol == NULL) ? "null" : hashed_subvol->name,
+                 (cached_subvol == NULL) ? "null" : cached_subvol->name);
+
+    if (!cached_subvol) {
+        if (local->skip_unlink.handle_valid_link && hashed_subvol) {
+            /*Purpose of "DHT_SKIP_NON_LINKTO_UNLINK":
+             * If this lookup is performed by rebalance and this
+             * rebalance process detected hashed file and by
+             * the time it sends the lookup request to cached node,
+             * file got migrated and now at initial hashed_node,
+             * final migrated file is present. With current logic,
+             * because this process fails to find the cached_node,
+             * it will unlink the file at initial hashed_node.
+             *
+             * So we avoid this by setting key, and checking at the
+             * posix_unlink that unlink the file only if file is a
+             * linkto file and not a migrated_file.
+             */
+
+            ret = dht_fill_dict_to_avoid_unlink_of_migrating_file(
+                local->xattr_req);
+
+            if (ret) {
+                /* If for some reason, setting key in the dict
+                 * fails, return with ENOENT, as with respect to
+                 * this process, it detected only a stale link
+                 * file.
+                 *
+                 * Next lookup will delete it.
+                 *
+                 * Performing deletion of stale link file when
+                 * setting key in dict fails, may cause the data
+                 * loss because of the above mentioned race.
+                 */
 
-        gf_uuid_unparse (local->loc.gfid, gfid);
-
-        if (local->file_count && local->dir_count) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        DHT_MSG_FILE_TYPE_MISMATCH,
-                        "path %s (gfid = %s)exists as a file on one "
-                        "subvolume and directory on another. "
-                        "Please fix it manually",
-                        local->loc.path, gfid);
-                DHT_STACK_UNWIND (lookup, frame, -1, EIO, NULL, NULL, NULL,
-                                  NULL);
-                return 0;
-        }
+                DHT_STACK_UNWIND(lookup, frame, -1, ENOENT, NULL, NULL, NULL,
+                                 NULL);
+            } else {
+                local->skip_unlink.handle_valid_link = _gf_false;
+
+                gf_msg_debug(this->name, 0,
+                             "No Cached was found and "
+                             "unlink on hashed was skipped"
+                             " so performing now: %s",
+                             local->loc.path);
+                FRAME_SU_DO(frame, dht_local_t);
+                STACK_WIND(frame, dht_lookup_unlink_stale_linkto_cbk,
+                           hashed_subvol, hashed_subvol->fops->unlink,
+                           &local->loc, 0, local->xattr_req);
+            }
 
-        if (local->dir_count) {
-                dht_lookup_directory (frame, this, &local->loc);
-                return 0;
+        } else {
+            gf_msg_debug(this->name, 0,
+                         "There was no cached file and  "
+                         "unlink on hashed is not skipped %s",
+                         local->loc.path);
+
+            DHT_STACK_UNWIND(lookup, frame, -1, ENOENT, NULL, NULL, NULL, NULL);
         }
+        return 0;
+    }
 
-        gf_msg_debug (this->name, 0, "STATUS: hashed_subvol %s "
-                      "cached_subvol %s",
-                      (hashed_subvol == NULL)?"null":hashed_subvol->name,
-                      (cached_subvol == NULL)?"null":cached_subvol->name);
+    /* At the time of dht_lookup, no file was found on hashed and that is
+     * why dht_lookup_everywhere is called, but by the time
+     * dht_lookup_everywhere
+     * reached to server, file might have already migrated. In that case we
+     * will find a migrated file at the hashed_node. In this case store the
+     * layout in context and return successfully.
+     */
 
-        if (!cached_subvol) {
+    if (hashed_subvol || local->need_lookup_everywhere) {
+        if (local->need_lookup_everywhere) {
+            found_non_linkto_on_hashed = _gf_true;
 
-                if (local->skip_unlink.handle_valid_link && hashed_subvol) {
-
-                        /*Purpose of "DHT_SKIP_NON_LINKTO_UNLINK":
-                         * If this lookup is performed by rebalance and this
-                         * rebalance process detected hashed file and by
-                         * the time it sends the lookup request to cached node,
-                         * file got migrated and now at initial hashed_node,
-                         * final migrated file is present. With current logic,
-                         * because this process fails to find the cached_node,
-                         * it will unlink the file at initial hashed_node.
-                         *
-                         * So we avoid this by setting key, and checking at the
-                         * posix_unlink that unlink the file only if file is a
-                         * linkto file and not a migrated_file.
-                         */
-
-
-                        ret = dht_fill_dict_to_avoid_unlink_of_migrating_file
-                              (local->xattr_req);
-
-                        if (ret) {
-                                /* If for some reason, setting key in the dict
-                                 * fails, return with ENOENT, as with respect to
-                                 * this process, it detected only a stale link
-                                 * file.
-                                 *
-                                 * Next lookup will delete it.
-                                 *
-                                 * Performing deletion of stale link file when
-                                 * setting key in dict fails, may cause the data
-                                 * loss becase of the above mentioned race.
-                                 */
-
-
-                                DHT_STACK_UNWIND (lookup, frame, -1, ENOENT,
-                                                  NULL, NULL, NULL, NULL);
-                        } else {
-                               local->skip_unlink.handle_valid_link = _gf_false;
-
-                                gf_msg_debug (this->name, 0,
-                                              "No Cached was found and "
-                                              "unlink on hashed was skipped"
-                                              " so performing now: %s",
-                                              local->loc.path);
-
-                               STACK_WIND (frame,
-                                            dht_lookup_unlink_stale_linkto_cbk,
-                                            hashed_subvol,
-                                            hashed_subvol->fops->unlink,
-                                            &local->loc, 0, local->xattr_req);
-                        }
+        } else if ((local->file_count == 1) &&
+                   (hashed_subvol == cached_subvol)) {
+            gf_msg_debug(this->name, 0,
+                         "found cached file on hashed subvolume "
+                         "so store in context and return for %s",
+                         local->loc.path);
 
-                } else  {
+            found_non_linkto_on_hashed = _gf_true;
+        }
 
-                        gf_msg_debug (this->name, 0,
-                                      "There was no cached file and  "
-                                      "unlink on hashed is not skipped %s",
-                                      local->loc.path);
+        if (found_non_linkto_on_hashed)
+            goto preset_layout;
+    }
 
-                        DHT_STACK_UNWIND (lookup, frame, -1, ENOENT, NULL, NULL,
-                                          NULL, NULL);
+    if (hashed_subvol) {
+        if (local->skip_unlink.handle_valid_link == _gf_true) {
+            if (cached_subvol == local->skip_unlink.hash_links_to) {
+                if (gf_uuid_compare(local->skip_unlink.cached_gfid,
+                                    local->skip_unlink.hashed_gfid)) {
+                    /*GFID different, return error*/
+                    DHT_STACK_UNWIND(lookup, frame, -1, ESTALE, NULL, NULL,
+                                     NULL, NULL);
+
+                    return 0;
                 }
-                return 0;
-        }
 
-        /* At the time of dht_lookup, no file was found on hashed and that is
-         * why dht_lookup_everywhere is called, but by the time
-         * dht_lookup_everywhere
-         * reached to server, file might have already migrated. In that case we
-         * will find a migrated file at the hashed_node. In this case store the
-         * layout in context and return successfully.
-         */
+                ret = dht_layout_preset(this, cached_subvol, local->loc.inode);
+                if (ret) {
+                    gf_msg(this->name, GF_LOG_INFO, 0,
+                           DHT_MSG_LAYOUT_PRESET_FAILED,
+                           "Could not set pre-set layout "
+                           "for subvolume %s",
+                           cached_subvol->name);
+                }
 
-        if (hashed_subvol || local->need_lookup_everywhere) {
+                local->op_ret = (ret == 0) ? ret : -1;
+                local->op_errno = (ret == 0) ? ret : EINVAL;
 
-                if (local->need_lookup_everywhere) {
+                /* Presence of local->cached_subvol validates
+                 * that lookup from cached node is successful
+                 */
 
-                        found_non_linkto_on_hashed = _gf_true;
+                if (!local->op_ret && local->loc.parent) {
+                    dht_inode_ctx_time_update(local->loc.parent, this,
+                                              &local->postparent, 1);
+                }
+
+                gf_msg_debug(this->name, 0,
+                             "Skipped unlinking linkto file "
+                             "on the hashed subvolume. "
+                             "Returning success as it is a "
+                             "valid linkto file. Path:%s",
+                             local->loc.path);
+
+                goto unwind_hashed_and_cached;
+            } else {
+                local->skip_unlink.handle_valid_link = _gf_false;
+
+                gf_msg_debug(this->name, 0,
+                             "Linkto file found on hashed "
+                             "subvol "
+                             "and data file found on cached "
+                             "subvolume. But linkto points to "
+                             "different cached subvolume (%s) "
+                             "path %s",
+                             (local->skip_unlink.hash_links_to
+                                  ? local->skip_unlink.hash_links_to->name
+                                  : " <nil>"),
+                             local->loc.path);
+
+                if (local->skip_unlink.opend_fd_count == 0) {
+                    ret = dht_fill_dict_to_avoid_unlink_of_migrating_file(
+                        local->xattr_req);
+
+                    if (ret) {
+                        DHT_STACK_UNWIND(lookup, frame, -1, EIO, NULL, NULL,
+                                         NULL, NULL);
+                    } else {
+                        local->call_cnt = 1;
+                        FRAME_SU_DO(frame, dht_local_t);
+                        STACK_WIND(frame, dht_lookup_unlink_of_false_linkto_cbk,
+                                   hashed_subvol, hashed_subvol->fops->unlink,
+                                   &local->loc, 0, local->xattr_req);
+                    }
+
+                    return 0;
+                }
+            }
+        }
+    }
 
-                } else if ((local->file_count == 1) &&
-                            (hashed_subvol == cached_subvol)) {
+preset_layout:
 
-                        gf_msg_debug (this->name, 0,
-                                      "found cached file on hashed subvolume "
-                                      "so store in context and return for %s",
-                                      local->loc.path);
+    if (found_non_linkto_on_hashed) {
+        if (local->need_lookup_everywhere) {
+            if (gf_uuid_compare(local->gfid, local->inode->gfid)) {
+                /* GFID different, return error */
+                DHT_STACK_UNWIND(lookup, frame, -1, ENOENT, NULL, NULL, NULL,
+                                 NULL);
+                return 0;
+            }
+        }
 
-                        found_non_linkto_on_hashed = _gf_true;
-                }
+        local->op_ret = 0;
+        local->op_errno = 0;
+        layout = dht_layout_for_subvol(this, cached_subvol);
+        if (!layout) {
+            gf_msg(this->name, GF_LOG_INFO, 0, DHT_MSG_SUBVOL_INFO,
+                   "%s: no pre-set layout for subvolume %s,"
+                   " gfid = %s",
+                   local->loc.path,
+                   (cached_subvol ? cached_subvol->name : "<nil>"), gfid);
+        }
 
-                if (found_non_linkto_on_hashed)
-                        goto preset_layout;
+        ret = dht_layout_set(this, local->inode, layout);
+        if (ret < 0) {
+            gf_msg(this->name, GF_LOG_INFO, 0, DHT_MSG_SUBVOL_INFO,
+                   "%s: failed to set layout for subvol %s, "
+                   "gfid = %s",
+                   local->loc.path,
+                   (cached_subvol ? cached_subvol->name : "<nil>"), gfid);
+        }
 
+        if (local->loc.parent) {
+            dht_inode_ctx_time_update(local->loc.parent, this,
+                                      &local->postparent, 1);
         }
 
+        DHT_STRIP_PHASE1_FLAGS(&local->stbuf);
+        dht_set_fixed_dir_stat(&local->postparent);
+        DHT_STACK_UNWIND(lookup, frame, local->op_ret, local->op_errno,
+                         local->inode, &local->stbuf, local->xattr,
+                         &local->postparent);
+        return 0;
+    }
 
-        if (hashed_subvol) {
-                if (local->skip_unlink.handle_valid_link == _gf_true) {
-                        if (cached_subvol == local->skip_unlink.hash_links_to) {
+    if (!hashed_subvol) {
+        gf_msg_debug(this->name, 0,
+                     "Cannot create linkfile for %s on %s: "
+                     "hashed subvolume cannot be found, gfid = %s.",
+                     local->loc.path, cached_subvol->name, gfid);
 
-                             if (gf_uuid_compare (local->skip_unlink.cached_gfid,
-                                               local->skip_unlink.hashed_gfid)){
+        local->op_ret = 0;
+        local->op_errno = 0;
 
-                                        /*GFID different, return error*/
-                                        DHT_STACK_UNWIND (lookup, frame, -1,
-                                                          ESTALE, NULL, NULL,
-                                                          NULL, NULL);
+        ret = dht_layout_preset(frame->this, cached_subvol, local->inode);
+        if (ret < 0) {
+            gf_msg(this->name, GF_LOG_INFO, 0, DHT_MSG_LAYOUT_PRESET_FAILED,
+                   "Failed to set layout for subvol %s"
+                   ", gfid = %s",
+                   cached_subvol ? cached_subvol->name : "<nil>", gfid);
+            local->op_ret = -1;
+            local->op_errno = EINVAL;
+        }
 
-                                        return 0;
-                                }
+        if (local->loc.parent) {
+            dht_inode_ctx_time_update(local->loc.parent, this,
+                                      &local->postparent, 1);
+        }
 
-                                ret = dht_layout_preset (this, cached_subvol,
-                                                         local->loc.inode);
-                                if (ret) {
-                                        gf_log (this->name, GF_LOG_INFO,
-                                                "Could not set pre-set layout "
-                                                "for subvolume %s",
-                                                cached_subvol->name);
-                                }
+        DHT_STRIP_PHASE1_FLAGS(&local->stbuf);
+        dht_set_fixed_dir_stat(&local->postparent);
+        DHT_STACK_UNWIND(lookup, frame, local->op_ret, local->op_errno,
+                         local->inode, &local->stbuf, local->xattr,
+                         &local->postparent);
+        return 0;
+    }
+
+    if (frame->root->op != GF_FOP_RENAME) {
+        local->current = &local->lock[0];
+        ret = dht_protect_namespace(frame, &local->loc, hashed_subvol,
+                                    &local->current->ns,
+                                    dht_call_lookup_linkfile_create);
+    } else {
+        gf_msg_debug(this->name, 0,
+                     "Creating linkto file on %s(hash) to %s on %s "
+                     "(gfid = %s)",
+                     hashed_subvol->name, local->loc.path, cached_subvol->name,
+                     gfid);
+
+        ret = dht_linkfile_create(frame, dht_lookup_linkfile_create_cbk, this,
+                                  cached_subvol, hashed_subvol, &local->loc);
+    }
+
+    return ret;
 
-                                local->op_ret = (ret == 0) ? ret : -1;
-                                local->op_errno = (ret == 0) ? ret : EINVAL;
+unwind_hashed_and_cached:
+    DHT_STRIP_PHASE1_FLAGS(&local->stbuf);
+    dht_set_fixed_dir_stat(&local->postparent);
+    DHT_STACK_UNWIND(lookup, frame, local->op_ret, local->op_errno,
+                     local->inode, &local->stbuf, local->xattr,
+                     &local->postparent);
+    return 0;
+}
 
-                                /* Presence of local->cached_subvol validates
-                                 * that lookup from cached node is successful
-                                 */
+static int
+dht_lookup_everywhere_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                          int32_t op_ret, int32_t op_errno, inode_t *inode,
+                          struct iatt *buf, dict_t *xattr,
+                          struct iatt *postparent)
+{
+    dht_local_t *local = NULL;
+    int this_call_cnt = 0;
+    xlator_t *prev = NULL;
+    int is_linkfile = 0;
+    int is_dir = 0;
+    loc_t *loc = NULL;
+    xlator_t *link_subvol = NULL;
+    int ret = -1;
+    int32_t fd_count = 0;
+    dht_conf_t *conf = NULL;
+    char gfid[GF_UUID_BUF_SIZE] = {0};
+    dict_t *dict_req = {0};
+
+    GF_VALIDATE_OR_GOTO("dht", frame, out);
+    GF_VALIDATE_OR_GOTO("dht", this, out);
+    GF_VALIDATE_OR_GOTO("dht", frame->local, out);
+    GF_VALIDATE_OR_GOTO("dht", cookie, out);
+    GF_VALIDATE_OR_GOTO("dht", this->private, out);
+
+    local = frame->local;
+    loc = &local->loc;
+    conf = this->private;
+
+    prev = cookie;
+
+    gf_msg_debug(this->name, 0,
+                 "returned with op_ret %d and op_errno %d (%s) "
+                 "from subvol %s",
+                 op_ret, op_errno, loc->path, prev->name);
+
+    LOCK(&frame->lock);
+    {
+        if (op_ret == -1) {
+            if (op_errno != ENOENT)
+                local->op_errno = op_errno;
+            if (op_errno == ENODATA)
+                local->gfid_missing = _gf_true;
+            goto unlock;
+        }
 
-                                if (!local->op_ret && local->loc.parent) {
-                                        dht_inode_ctx_time_update
-                                                (local->loc.parent, this,
-                                                 &local->postparent, 1);
-                                }
+        if (gf_uuid_is_null(local->gfid))
+            gf_uuid_copy(local->gfid, buf->ia_gfid);
 
-                                gf_msg_debug (this->name, 0,
-                                              "Skipped unlinking linkto file "
-                                              "on the hashed subvolume. "
-                                              "Returning success as it is a "
-                                              "valid linkto file. Path:%s"
-                                              ,local->loc.path);
+        gf_uuid_unparse(local->gfid, gfid);
 
-                                goto unwind_hashed_and_cached;
-                        } else {
+        if (gf_uuid_compare(local->gfid, buf->ia_gfid)) {
+            gf_msg(this->name, GF_LOG_WARNING, 0, DHT_MSG_GFID_MISMATCH,
+                   "%s: gfid differs on subvolume %s,"
+                   " gfid local = %s, gfid node = %s",
+                   loc->path, prev->name, gfid, uuid_utoa(buf->ia_gfid));
+        }
 
-                               local->skip_unlink.handle_valid_link = _gf_false;
+        is_linkfile = check_is_linkfile(inode, buf, xattr,
+                                        conf->link_xattr_name);
 
-                               gf_msg_debug (this->name, 0,
-                                             "Linkto file found on hashed "
-                                             "subvol "
-                                             "and data file found on cached "
-                                             "subvolume. But linkto points to "
-                                             "different cached subvolume (%s) "
-                                             "path %s",
-                                            (local->skip_unlink.hash_links_to ?
-                                        local->skip_unlink.hash_links_to->name :
-                                             " <nil>"), local->loc.path);
+        if (is_linkfile) {
+            link_subvol = dht_linkfile_subvol(this, inode, buf, xattr);
+            gf_msg_debug(this->name, 0, "found on %s linkfile %s (-> %s)",
+                         prev->name, loc->path,
+                         link_subvol ? link_subvol->name : "''");
+            goto unlock;
+        }
 
-                               if (local->skip_unlink.opend_fd_count == 0) {
+        is_dir = check_is_dir(inode, buf, xattr);
 
+        /* non linkfile GFID takes precedence but don't overwrite
+         gfid if we have already found a cached file*/
+        if (!local->cached_subvol)
+            gf_uuid_copy(local->gfid, buf->ia_gfid);
 
-                          ret = dht_fill_dict_to_avoid_unlink_of_migrating_file
-                                  (local->xattr_req);
+        if (is_dir) {
+            local->dir_count++;
 
+            gf_msg_debug(this->name, 0, "found on %s directory %s", prev->name,
+                         loc->path);
+        } else {
+            local->file_count++;
 
-                                        if (ret) {
-                                          DHT_STACK_UNWIND (lookup, frame, -1,
-                                                            EIO, NULL, NULL,
-                                                            NULL, NULL);
-                                        } else {
-                                                local->call_cnt = 1;
-                                                STACK_WIND (frame,
-                                          dht_lookup_unlink_of_false_linkto_cbk,
-                                                    hashed_subvol,
-                                                    hashed_subvol->fops->unlink,
-                                                    &local->loc, 0,
-                                                    local->xattr_req);
-                                        }
+            gf_msg_debug(this->name, 0, "found cached file on %s for %s",
+                         prev->name, loc->path);
 
-                                        return 0;
+            if (!local->cached_subvol) {
+                /* found one file */
+                dht_iatt_merge(this, &local->stbuf, buf);
 
-                                }
-                        }
+                local->xattr = dict_ref(xattr);
+                local->cached_subvol = prev;
 
-                }
-        }
+                gf_msg_debug(this->name, 0,
+                             "storing cached on %s file"
+                             " %s",
+                             prev->name, loc->path);
 
+                dht_iatt_merge(this, &local->postparent, postparent);
 
-preset_layout:
+                gf_uuid_copy(local->skip_unlink.cached_gfid, buf->ia_gfid);
+            } else {
+                /* This is where we need 'rename' both entries logic */
+                gf_msg(this->name, GF_LOG_WARNING, 0,
+                       DHT_MSG_FILE_ON_MULT_SUBVOL,
+                       "multiple subvolumes (%s and %s) have "
+                       "file %s (preferably rename the file "
+                       "in the backend,and do a fresh lookup)",
+                       local->cached_subvol->name, prev->name, local->loc.path);
+            }
+        }
+    }
+unlock:
+    UNLOCK(&frame->lock);
 
-        if (found_non_linkto_on_hashed) {
+    if (is_linkfile) {
+        ret = dict_get_int32(xattr, GLUSTERFS_OPEN_FD_COUNT, &fd_count);
 
-                if (local->need_lookup_everywhere) {
-                        if (gf_uuid_compare (local->gfid, local->inode->gfid)) {
-                                /* GFID different, return error */
-                                DHT_STACK_UNWIND (lookup, frame, -1, ENOENT,
-                                                  NULL, NULL, NULL, NULL);
-                                return 0;
-                        }
-                }
+        /*  Any linkto file found on the non-hashed subvolume should
+         *  be unlinked (performed in the "else if" block below)
+         *
+         *  But if a linkto file is found on hashed subvolume, it may be
+         *  pointing to valid cached node. So unlinking of linkto
+         *  file on hashed subvolume is skipped and inside
+         *  dht_lookup_everywhere_done, checks are performed. If this
+         *  linkto file is found as stale linkto file, it is deleted
+         *  otherwise unlink is skipped.
+         */
 
-                local->op_ret = 0;
-                local->op_errno = 0;
-                layout = dht_layout_for_subvol (this, cached_subvol);
-                if (!layout) {
-                        gf_log (this->name, GF_LOG_INFO,
-                                "%s: no pre-set layout for subvolume %s,"
-                                " gfid = %s",
-                                local->loc.path, (cached_subvol ?
-                                                  cached_subvol->name :
-                                                  "<nil>"), gfid);
-                }
+        if (local->hashed_subvol && local->hashed_subvol == prev) {
+            local->skip_unlink.handle_valid_link = _gf_true;
+            local->skip_unlink.opend_fd_count = fd_count;
+            local->skip_unlink.hash_links_to = link_subvol;
+            gf_uuid_copy(local->skip_unlink.hashed_gfid, buf->ia_gfid);
+
+            gf_msg_debug(this->name, 0,
+                         "Found"
+                         " one linkto file on hashed subvol %s "
+                         "for %s: Skipping unlinking till "
+                         "everywhere_done",
+                         prev->name, loc->path);
+
+        } else if (!ret && (fd_count == 0)) {
+            dict_req = dict_new();
+
+            ret = dht_fill_dict_to_avoid_unlink_of_migrating_file(dict_req);
+
+            if (ret) {
+                /* Skip unlinking for dict_failure
+                 *File is found as a linkto file on non-hashed,
+                 *subvolume. In the current implementation,
+                 *finding a linkto-file on non-hashed does not
+                 *always implies that it is stale. So deletion
+                 *of file should be done only when both fd is
+                 *closed and linkto-xattr is set. In case of
+                 *dict_set failure, avoid skipping of file.
+                 *NOTE: dht_frame_return should get called for
+                 *      this block.
+                 */
 
-                ret = dht_layout_set (this, local->inode, layout);
-                if (ret < 0) {
-                        gf_log (this->name, GF_LOG_INFO,
-                                "%s: failed to set layout for subvol %s, "
-                                "gfid = %s",
-                                local->loc.path, (cached_subvol ?
-                                                  cached_subvol->name :
-                                                  "<nil>"), gfid);
-                }
+                dict_unref(dict_req);
+
+            } else {
+                gf_msg(this->name, GF_LOG_INFO, 0, DHT_MSG_SUBVOL_INFO,
+                       "attempting deletion of stale linkfile "
+                       "%s on %s (hashed subvol is %s)",
+                       loc->path, prev->name,
+                       (local->hashed_subvol ? local->hashed_subvol->name
+                                             : "<null>"));
+                /* *
+                 * These stale files may be created using root
+                 * user. Hence deletion will work only with
+                 * root.
+                 */
+                FRAME_SU_DO(frame, dht_local_t);
+                STACK_WIND(frame, dht_lookup_unlink_cbk, prev,
+                           prev->fops->unlink, loc, 0, dict_req);
 
-                if (local->loc.parent) {
-                        dht_inode_ctx_time_update (local->loc.parent, this,
-                                                   &local->postparent, 1);
-                }
+                dict_unref(dict_req);
 
-                DHT_STRIP_PHASE1_FLAGS (&local->stbuf);
-                DHT_STACK_UNWIND (lookup, frame, local->op_ret,
-                                  local->op_errno, local->inode,
-                                  &local->stbuf, local->xattr,
-                                  &local->postparent);
                 return 0;
+            }
         }
+    }
 
-        if (!hashed_subvol) {
+    this_call_cnt = dht_frame_return(frame);
+    if (is_last_call(this_call_cnt)) {
+        dht_lookup_everywhere_done(frame, this);
+    }
 
-                gf_msg_debug (this->name, 0,
-                              "Cannot create linkfile for %s on %s: "
-                              "hashed subvolume cannot be found, gfid = %s.",
-                              local->loc.path, cached_subvol->name, gfid);
+out:
+    return ret;
+}
 
-                local->op_ret = 0;
-                local->op_errno = 0;
+int
+dht_lookup_everywhere(call_frame_t *frame, xlator_t *this, loc_t *loc)
+{
+    dht_conf_t *conf = NULL;
+    dht_local_t *local = NULL;
+    int i = 0;
+    int call_cnt = 0;
 
-                ret = dht_layout_preset (frame->this, cached_subvol,
-                                         local->inode);
-                if (ret < 0) {
-                        gf_log (this->name, GF_LOG_INFO,
-                                "Failed to set layout for subvol %s"
-                                ", gfid = %s",
-                                cached_subvol ? cached_subvol->name :
-                                "<nil>", gfid);
-                        local->op_ret = -1;
-                        local->op_errno = EINVAL;
-                }
+    GF_VALIDATE_OR_GOTO("dht", frame, err);
+    GF_VALIDATE_OR_GOTO("dht", this, out);
+    GF_VALIDATE_OR_GOTO("dht", frame->local, out);
+    GF_VALIDATE_OR_GOTO("dht", this->private, out);
+    GF_VALIDATE_OR_GOTO("dht", loc, out);
 
-                if (local->loc.parent) {
-                        dht_inode_ctx_time_update (local->loc.parent, this,
-                                                   &local->postparent, 1);
-                }
+    conf = this->private;
+    local = frame->local;
 
-                DHT_STRIP_PHASE1_FLAGS (&local->stbuf);
-                DHT_STACK_UNWIND (lookup, frame, local->op_ret,
-                                  local->op_errno, local->inode,
-                                  &local->stbuf, local->xattr,
-                                  &local->postparent);
-                return 0;
-        }
+    call_cnt = conf->subvolume_cnt;
+    local->call_cnt = call_cnt;
 
-        gf_msg_debug (this->name, 0,
-                      "Creating linkto file on %s(hash) to %s on %s (gfid = %s)",
-                      hashed_subvol->name, local->loc.path,
-                      cached_subvol->name, gfid);
+    if (!local->inode)
+        local->inode = inode_ref(loc->inode);
 
-        ret = dht_linkfile_create (frame,
-                                   dht_lookup_linkfile_create_cbk, this,
-                                   cached_subvol, hashed_subvol, &local->loc);
+    gf_msg_debug(this->name, 0, "winding lookup call to %d subvols", call_cnt);
 
-        return ret;
+    for (i = 0; i < call_cnt; i++) {
+        STACK_WIND_COOKIE(frame, dht_lookup_everywhere_cbk, conf->subvolumes[i],
+                          conf->subvolumes[i],
+                          conf->subvolumes[i]->fops->lookup, loc,
+                          local->xattr_req);
+    }
 
-unwind_hashed_and_cached:
-        DHT_STRIP_PHASE1_FLAGS (&local->stbuf);
-        DHT_STACK_UNWIND (lookup, frame, local->op_ret, local->op_errno,
-                          local->loc.inode, &local->stbuf, local->xattr,
-                          &local->postparent);
-        return 0;
+    return 0;
+out:
+    DHT_STACK_UNWIND(lookup, frame, -1, EINVAL, NULL, NULL, NULL, NULL);
+err:
+    return -1;
 }
 
 int
-dht_lookup_everywhere_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                           int32_t op_ret, int32_t op_errno,
-                           inode_t *inode, struct iatt *buf, dict_t *xattr,
-                           struct iatt *postparent)
-{
-        dht_local_t  *local         = NULL;
-        int           this_call_cnt = 0;
-        call_frame_t *prev          = NULL;
-        int           is_linkfile   = 0;
-        int           is_dir        = 0;
-        xlator_t     *subvol        = NULL;
-        loc_t        *loc           = NULL;
-        xlator_t     *link_subvol   = NULL;
-        int           ret           = -1;
-        int32_t       fd_count      = 0;
-        dht_conf_t   *conf          = NULL;
-        char         gfid[GF_UUID_BUF_SIZE] = {0};
-        dict_t       *dict_req      = {0};
-
-        GF_VALIDATE_OR_GOTO ("dht", frame, out);
-        GF_VALIDATE_OR_GOTO ("dht", this, out);
-        GF_VALIDATE_OR_GOTO ("dht", frame->local, out);
-        GF_VALIDATE_OR_GOTO ("dht", cookie, out);
-        GF_VALIDATE_OR_GOTO ("dht", this->private, out);
-
-        local  = frame->local;
-        loc    = &local->loc;
-        conf   = this->private;
-
-        prev   = cookie;
-        subvol = prev->this;
-
-        gf_msg_debug (this->name, 0,
-                      "returned with op_ret %d and op_errno %d (%s) "
-                      "from subvol %s", op_ret, op_errno, loc->path,
-                      subvol->name);
-
-        LOCK (&frame->lock);
-        {
-                if (op_ret == -1) {
-                        if (op_errno != ENOENT)
-                                local->op_errno = op_errno;
-                        goto unlock;
-                }
+dht_lookup_linkfile_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                        int op_ret, int op_errno, inode_t *inode,
+                        struct iatt *stbuf, dict_t *xattr,
+                        struct iatt *postparent)
+{
+    xlator_t *prev = NULL;
+    dht_local_t *local = NULL;
+    xlator_t *subvol = NULL;
+    loc_t *loc = NULL;
+    dht_conf_t *conf = NULL;
+    int ret = 0;
+    char gfid[GF_UUID_BUF_SIZE] = {0};
+
+    GF_VALIDATE_OR_GOTO("dht", frame, out);
+    GF_VALIDATE_OR_GOTO("dht", this, unwind);
+    GF_VALIDATE_OR_GOTO("dht", frame->local, unwind);
+    GF_VALIDATE_OR_GOTO("dht", this->private, unwind);
+    GF_VALIDATE_OR_GOTO("dht", cookie, unwind);
+
+    prev = cookie;
+    subvol = prev;
+    conf = this->private;
+    local = frame->local;
+    loc = &local->loc;
+
+    gf_uuid_unparse(loc->gfid, gfid);
+
+    if (op_ret == -1) {
+        gf_msg(this->name, GF_LOG_INFO, op_errno, DHT_MSG_LINK_FILE_LOOKUP_INFO,
+               "Lookup of %s on %s (following linkfile) failed "
+               ",gfid = %s",
+               local->loc.path, subvol->name, gfid);
+
+        /* If cached subvol returned ENOTCONN, do not do
+        lookup_everywhere. We need to make sure linkfile does not get
+        removed, which can take away the namespace, and subvol is
+        anyways down. */
+
+        local->cached_subvol = NULL;
+        if (op_errno != ENOTCONN)
+            goto err;
+        else
+            goto unwind;
+    }
+
+    if (check_is_dir(inode, stbuf, xattr)) {
+        gf_msg(this->name, GF_LOG_INFO, 0, DHT_MSG_LINK_FILE_LOOKUP_INFO,
+               "Lookup of %s on %s (following linkfile) reached dir,"
+               " gfid = %s",
+               local->loc.path, subvol->name, gfid);
+        goto err;
+    }
+
+    if (check_is_linkfile(inode, stbuf, xattr, conf->link_xattr_name)) {
+        gf_msg(this->name, GF_LOG_INFO, 0, DHT_MSG_LINK_FILE_LOOKUP_INFO,
+               "lookup of %s on %s (following linkfile) reached link,"
+               "gfid = %s",
+               local->loc.path, subvol->name, gfid);
+        goto err;
+    }
+
+    if (gf_uuid_compare(local->gfid, stbuf->ia_gfid)) {
+        gf_msg(this->name, GF_LOG_WARNING, 0, DHT_MSG_GFID_MISMATCH,
+               "%s: gfid different on data file on %s,"
+               " gfid local = %s, gfid node = %s ",
+               local->loc.path, subvol->name, gfid, uuid_utoa(stbuf->ia_gfid));
+        goto err;
+    }
+
+    if ((stbuf->ia_nlink == 1) && (conf && conf->unhashed_sticky_bit)) {
+        stbuf->ia_prot.sticky = 1;
+    }
+
+    ret = dht_layout_preset(this, prev, inode);
+    if (ret < 0) {
+        gf_msg(this->name, GF_LOG_INFO, 0, DHT_MSG_LAYOUT_PRESET_FAILED,
+               "Failed to set layout for subvolume %s,"
+               "gfid = %s",
+               prev->name, gfid);
+        op_ret = -1;
+        op_errno = EINVAL;
+    }
+
+    if (local->loc.parent) {
+        dht_inode_ctx_time_update(local->loc.parent, this, postparent, 1);
+    }
 
-                if (gf_uuid_is_null (local->gfid))
-                        gf_uuid_copy (local->gfid, buf->ia_gfid);
+unwind:
+    DHT_STRIP_PHASE1_FLAGS(stbuf);
+    dht_set_fixed_dir_stat(postparent);
+    DHT_STACK_UNWIND(lookup, frame, op_ret, op_errno, inode, stbuf, xattr,
+                     postparent);
 
-                gf_uuid_unparse(local->gfid, gfid);
+    return 0;
 
-                if (gf_uuid_compare (local->gfid, buf->ia_gfid)) {
-                        gf_msg (this->name, GF_LOG_WARNING, 0,
-                                DHT_MSG_GFID_MISMATCH,
-                                "%s: gfid differs on subvolume %s,"
-                                " gfid local = %s, gfid node = %s",
-                                loc->path, prev->this->name, gfid,
-                                uuid_utoa(buf->ia_gfid));
-                }
+err:
+    dht_lookup_everywhere(frame, this, loc);
+out:
+    return 0;
+}
 
-                is_linkfile = check_is_linkfile (inode, buf, xattr,
-                                                 conf->link_xattr_name);
-
-                if (is_linkfile) {
-                        link_subvol = dht_linkfile_subvol (this, inode, buf,
-                                                           xattr);
-                        gf_msg_debug (this->name, 0,
-                                      "found on %s linkfile %s (-> %s)",
-                                      subvol->name, loc->path,
-                                      link_subvol ? link_subvol->name : "''");
-                        goto unlock;
-                }
+/* Code to get hashed subvol based on inode and loc
+   First it check if loc->parent and loc->path exist then it get
+   hashed subvol based on loc.
+*/
 
-                is_dir = check_is_dir (inode, buf, xattr);
+static gf_boolean_t
+dht_should_lookup_everywhere(xlator_t *this, dht_conf_t *conf, loc_t *loc)
+{
+    dht_layout_t *parent_layout = NULL;
+    int ret = 0;
+    gf_boolean_t lookup_everywhere = _gf_true;
+
+    /* lookup-optimize supersedes lookup-unhashed settings.
+     * If it is set, do not process search_unhashed
+     * If lookup-optimize if enabled, lookup everywhere if:
+     *  - this is the rebalance daemon.
+     *  - loc->parent is unavailable.
+     *  - parent_layout is unavailable
+     *  - parent_layout->commit_hash != conf->vol_commit_hash
+     */
+
+    if (conf->lookup_optimize) {
+        if (!conf->defrag && loc->parent) {
+            ret = dht_inode_ctx_layout_get(loc->parent, this, &parent_layout);
+            if (!ret && parent_layout &&
+                (parent_layout->commit_hash == conf->vol_commit_hash)) {
+                lookup_everywhere = _gf_false;
+            }
+        }
+        goto out;
+    } else {
+        if (conf->search_unhashed == GF_DHT_LOOKUP_UNHASHED_AUTO) {
+            if (loc->parent) {
+                ret = dht_inode_ctx_layout_get(loc->parent, this,
+                                               &parent_layout);
+                if (ret || !parent_layout ||
+                    (!parent_layout->search_unhashed)) {
+                    lookup_everywhere = _gf_false;
+                }
+            } else {
+                lookup_everywhere = _gf_false;
+            }
+
+            goto out;
+        }
+    }
+out:
+    return lookup_everywhere;
+}
+
+int
+dht_lookup_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int op_ret,
+               int op_errno, inode_t *inode, struct iatt *stbuf, dict_t *xattr,
+               struct iatt *postparent)
+{
+    char is_linkfile = 0;
+    char is_dir = 0;
+    xlator_t *subvol = NULL;
+    dht_conf_t *conf = NULL;
+    dht_local_t *local = NULL;
+    loc_t *loc = NULL;
+    xlator_t *prev = NULL;
+    int ret = 0;
+    uint32_t vol_commit_hash = 0;
+
+    GF_VALIDATE_OR_GOTO("dht", frame, err);
+    GF_VALIDATE_OR_GOTO("dht", this, out);
+    GF_VALIDATE_OR_GOTO("dht", frame->local, out);
+    GF_VALIDATE_OR_GOTO("dht", cookie, out);
+    GF_VALIDATE_OR_GOTO("dht", this->private, out);
+
+    conf = this->private;
+
+    prev = cookie;
+    local = frame->local;
+    loc = &local->loc;
+
+    gf_msg_debug(this->name, op_errno,
+                 "%s: fresh_lookup on %s returned with op_ret %d", loc->path,
+                 prev->name, op_ret);
+
+    if (op_ret == -1) {
+        if (ENTRY_MISSING(op_ret, op_errno)) {
+            if (1 == conf->subvolume_cnt) {
+                /* No need to lookup again */
+                goto out;
+            }
 
-                /* non linkfile GFID takes precedence but don't overwrite
-                 gfid if we have already found a cached file*/
-                if (!local->cached_subvol)
-                        gf_uuid_copy (local->gfid, buf->ia_gfid);
+            gf_msg_debug(this->name, 0, "Entry %s missing on subvol %s",
+                         loc->path, prev->name);
 
-                if (is_dir) {
-                        local->dir_count++;
+            if (dht_should_lookup_everywhere(this, conf, loc)) {
+                local->op_errno = ENOENT;
+                dht_lookup_everywhere(frame, this, loc);
+                return 0;
+            }
 
-                        gf_msg_debug (this->name, 0,
-                                      "found on %s directory %s",
-                                      subvol->name, loc->path);
-                } else {
-                        local->file_count++;
-
-                        gf_msg_debug (this->name, 0,
-                                      "found cached file on %s for %s",
-                                      subvol->name, loc->path);
-
-                        if (!local->cached_subvol) {
-                                /* found one file */
-                                dht_iatt_merge (this, &local->stbuf, buf,
-                                                subvol);
-                                local->xattr = dict_ref (xattr);
-                                local->cached_subvol = subvol;
-
-                                gf_msg_debug (this->name, 0,
-                                              "storing cached on %s file"
-                                              " %s", subvol->name, loc->path);
-
-                                dht_iatt_merge (this, &local->postparent,
-                                                postparent, subvol);
-
-                                gf_uuid_copy (local->skip_unlink.cached_gfid,
-                                           buf->ia_gfid);
-                        } else {
-                                /* This is where we need 'rename' both entries logic */
-                                gf_msg (this->name, GF_LOG_WARNING, 0,
-                                        DHT_MSG_FILE_ON_MULT_SUBVOL,
-                                        "multiple subvolumes (%s and %s) have "
-                                        "file %s (preferably rename the file "
-                                        "in the backend,and do a fresh lookup)",
-                                        local->cached_subvol->name,
-                                        subvol->name, local->loc.path);
-                        }
-                }
+        } else {
+            /* posix returns ENODATA if the gfid is not set but the client and
+             * server protocol layers do not send the stbuf. We need to
+             * heal this so check if this is a directory on the other subvols.
+             */
+            if ((op_errno == ENOTCONN) || (op_errno == ENODATA)) {
+                dht_lookup_directory(frame, this, &local->loc);
+                return 0;
+            }
+        }
+        gf_msg_debug(this->name, op_errno, "%s: Lookup on subvolume %s failed",
+                     loc->path, prev->name);
+        goto out;
+    }
+
+    /* Lookup succeeded - op_ret = 0 */
+
+    /* This is required for handling stale linkfile deletion,
+     * or any more call which happens from this 'loc'.
+     */
+    if (gf_uuid_is_null(local->gfid)) {
+        /*This is set from the first successful response*/
+        memcpy(local->gfid, stbuf->ia_gfid, 16);
+    }
+
+    if (!conf->vch_forced) {
+        /* Update the commit hash in conf if it is found */
+        ret = dict_get_uint32(xattr, conf->commithash_xattr_name,
+                              &vol_commit_hash);
+        if (ret == 0) {
+            conf->vol_commit_hash = vol_commit_hash;
         }
-unlock:
-        UNLOCK (&frame->lock);
+    }
 
-        if (is_linkfile) {
-                ret = dict_get_int32 (xattr, GLUSTERFS_OPEN_FD_COUNT, &fd_count);
+    is_dir = check_is_dir(inode, stbuf, xattr);
+    if (is_dir) {
+        /* A directory is present on all subvols, send the lookup to
+         * all subvols now */
+        local->inode = inode_ref(inode);
+        local->xattr = dict_ref(xattr);
+        dht_lookup_directory(frame, this, &local->loc);
+        return 0;
+    }
 
-                /*  Any linkto file found on the non-hashed subvolume should
-                 *  be unlinked (performed in the "else if" block below)
-                 *
-                 *  But if a linkto file is found on hashed subvolume, it may be
-                 *  pointing to vaild cached node. So unlinking of linkto
-                 *  file on hashed subvolume is skipped and inside
-                 *  dht_lookup_everywhere_done, checks are performed. If this
-                 *  linkto file is found as stale linkto file, it is deleted
-                 *  otherwise unlink is skipped.
-                 */
+    is_linkfile = check_is_linkfile(inode, stbuf, xattr, conf->link_xattr_name);
 
-                if (local->hashed_subvol && local->hashed_subvol == subvol) {
+    if (!is_linkfile) {
+        /* non-directory and not a linkto file. This is a data file
+         * Update the layout to point to the cached subvol
+         */
 
-                        local->skip_unlink.handle_valid_link = _gf_true;
-                        local->skip_unlink.opend_fd_count = fd_count;
-                        local->skip_unlink.hash_links_to = link_subvol;
-                        gf_uuid_copy (local->skip_unlink.hashed_gfid,
-                                   buf->ia_gfid);
+        ret = dht_layout_preset(this, prev, inode);
+        if (ret < 0) {
+            gf_msg(this->name, GF_LOG_INFO, 0, DHT_MSG_LAYOUT_PRESET_FAILED,
+                   "%s: could not set pre-set layout for subvolume %s",
+                   loc->path, prev->name);
+            op_ret = -1;
+            op_errno = EINVAL;
+            goto out;
+        }
+        goto out;
+    }
+
+    /* This is a linkto file. Get the value of the target subvol from the
+     * linkto xattr and lookup there to see if the file exists
+     */
+    subvol = dht_linkfile_subvol(this, inode, stbuf, xattr);
+    if (!subvol) {
+        gf_msg(this->name, GF_LOG_INFO, 0, DHT_MSG_SUBVOL_INFO,
+               "%s: No link subvol for linkto", loc->path);
+        dht_lookup_everywhere(frame, this, loc);
+        return 0;
+    }
 
-                        gf_msg_debug (this->name, 0, "Found"
-                                      " one linkto file on hashed subvol %s "
-                                      "for %s: Skipping unlinking till "
-                                      "everywhere_done", subvol->name,
-                                      loc->path);
+    gf_msg_debug(this->name, 0, "%s: Calling lookup on linkto target %s",
+                 loc->path, subvol->name);
 
-                } else if (!ret && (fd_count == 0)) {
+    STACK_WIND_COOKIE(frame, dht_lookup_linkfile_cbk, subvol, subvol,
+                      subvol->fops->lookup, &local->loc, local->xattr_req);
 
-                        dict_req = dict_new ();
+    return 0;
 
-                        ret = dht_fill_dict_to_avoid_unlink_of_migrating_file
-                              (dict_req);
+out:
+    /*
+     * FIXME: postparent->ia_size and postparent->st_blocks do not have
+     * correct values. since, postparent corresponds to a directory these
+     * two members should have values equal to sum of corresponding values
+     * from each of the subvolume. See dht_iatt_merge for reference.
+     */
+
+    if (!op_ret && local && local->loc.parent) {
+        dht_inode_ctx_time_update(local->loc.parent, this, postparent, 1);
+    }
+
+    DHT_STRIP_PHASE1_FLAGS(stbuf);
+    dht_set_fixed_dir_stat(postparent);
+    DHT_STACK_UNWIND(lookup, frame, op_ret, op_errno, inode, stbuf, xattr,
+                     postparent);
+err:
+    return 0;
+}
 
-                        if (ret) {
+/* For directories, check if acl xattrs have been requested (by the acl
+ * xlator), if not, request for them. These xattrs are needed for dht dir
+ * self-heal to perform proper self-healing of dirs
+ */
+static void
+dht_check_and_set_acl_xattr_req(xlator_t *this, dict_t *xattr_req)
+{
+    int ret = 0;
 
-                                /* Skip unlinking for dict_failure
-                                 *File is found as a linkto file on non-hashed,
-                                 *subvolume. In the current implementation,
-                                 *finding a linkto-file on non-hashed does not
-                                 *always implies that it is stale. So deletion
-                                 *of file should be done only when both fd is
-                                 *closed and linkto-xattr is set. In case of
-                                 *dict_set failure, avoid skipping of file.
-                                 *NOTE: dht_frame_return should get called for
-                                 *      this block.
-                                 */
+    GF_ASSERT(xattr_req);
 
-                                dict_unref (dict_req);
+    if (!dict_get(xattr_req, POSIX_ACL_ACCESS_XATTR)) {
+        ret = dict_set_int8(xattr_req, POSIX_ACL_ACCESS_XATTR, 0);
+        if (ret)
+            gf_msg(this->name, GF_LOG_WARNING, -ret, DHT_MSG_DICT_SET_FAILED,
+                   "Failed to set dictionary value:key = %s",
+                   POSIX_ACL_ACCESS_XATTR);
+    }
 
-                        } else {
-                                gf_log (this->name, GF_LOG_INFO,
-                                        "attempting deletion of stale linkfile "
-                                        "%s on %s (hashed subvol is %s)",
-                                        loc->path, subvol->name,
-                                        (local->hashed_subvol?
-                                        local->hashed_subvol->name : "<null>"));
+    if (!dict_get(xattr_req, POSIX_ACL_DEFAULT_XATTR)) {
+        ret = dict_set_int8(xattr_req, POSIX_ACL_DEFAULT_XATTR, 0);
+        if (ret)
+            gf_msg(this->name, GF_LOG_WARNING, -ret, DHT_MSG_DICT_SET_FAILED,
+                   "Failed to set dictionary value:key = %s",
+                   POSIX_ACL_DEFAULT_XATTR);
+    }
+
+    return;
+}
 
-                                STACK_WIND (frame, dht_lookup_unlink_cbk,
-                                            subvol, subvol->fops->unlink, loc,
-                                            0, dict_req);
+/* for directories, we need the following info:
+ * the layout : trusted.glusterfs.dht
+ * the mds information : trusted.glusterfs.dht.mds
+ * the acl info: See above
+ */
+static int
+dht_set_dir_xattr_req(xlator_t *this, loc_t *loc, dict_t *xattr_req)
+{
+    int ret = -EINVAL;
+    dht_conf_t *conf = NULL;
+
+    conf = this->private;
+    if (!conf) {
+        goto err;
+    }
+
+    if (!xattr_req) {
+        goto err;
+    }
+
+    /* Xattr to get the layout for a directory
+     */
+    ret = dict_set_uint32(xattr_req, conf->xattr_name, 4 * 4);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_WARNING, ENOMEM, DHT_MSG_DICT_SET_FAILED,
+               "Failed to set dictionary value:key = %s for "
+               "path %s",
+               conf->xattr_name, loc->path);
+        goto err;
+    }
+
+    /*Non-fatal failure */
+    ret = dict_set_uint32(xattr_req, conf->mds_xattr_key, 4);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_WARNING, ENOMEM, DHT_MSG_DICT_SET_FAILED,
+               "Failed to set dictionary value:key = %s for "
+               "path %s",
+               conf->mds_xattr_key, loc->path);
+    }
+
+    dht_check_and_set_acl_xattr_req(this, xattr_req);
+    ret = 0;
+err:
+    return ret;
+}
 
-                                dict_unref (dict_req);
+/* If the hashed subvol is present, send the lookup to only that subvol first.
+ * If no hashed subvol, send a lookup to all subvols and proceed based on the
+ * responses.
+ */
+static int
+dht_do_fresh_lookup(call_frame_t *frame, xlator_t *this, loc_t *loc)
+{
+    int ret = -1;
+    dht_conf_t *conf = NULL;
+    xlator_t *hashed_subvol = NULL;
+    dht_local_t *local = NULL;
+    int op_errno = -1;
+    int call_cnt = 0;
+    int i = 0;
+
+    conf = this->private;
+    if (!conf) {
+        op_errno = EINVAL;
+        goto err;
+    }
+
+    local = frame->local;
+    if (!local) {
+        op_errno = EINVAL;
+        goto err;
+    }
+
+    /* Since we don't know whether this is a file or a directory,
+     * request all xattrs*/
+    ret = dht_set_file_xattr_req(this, loc, local->xattr_req);
+    if (ret) {
+        op_errno = -ret;
+        goto err;
+    }
+
+    ret = dht_set_dir_xattr_req(this, loc, local->xattr_req);
+    if (ret) {
+        op_errno = -ret;
+        goto err;
+    }
+
+    /* Fuse sets a random value in gfid-req. If the gfid is missing
+     * on one or more subvols, posix will set the gfid to this value,
+     * causing GFID mismatches for directories. Remove the value fuse
+     * has sent before sending the lookup.
+     */
+    ret = dict_get_gfuuid(local->xattr_req, "gfid-req", &local->gfid_req);
+    if (ret) {
+        gf_msg_debug(this->name, 0, "%s: No gfid-req available", loc->path);
+    } else {
+        dict_del(local->xattr_req, "gfid-req");
+    }
+    /* This should have been set in dht_lookup */
+    hashed_subvol = local->hashed_subvol;
+
+    if (!hashed_subvol) {
+        gf_msg_debug(this->name, 0,
+                     "%s: no subvolume in layout for path, "
+                     "checking on all the subvols to see if "
+                     "it is a directory",
+                     loc->path);
 
-                                return 0;
-                        }
-                }
+        call_cnt = conf->subvolume_cnt;
+        local->call_cnt = call_cnt;
+
+        /* Allocate a layout. This will be populated and saved in
+         * the dht inode_ctx on successful lookup
+         */
+        local->layout = dht_layout_new(this, conf->subvolume_cnt);
+        if (!local->layout) {
+            op_errno = ENOMEM;
+            goto err;
         }
 
-        this_call_cnt = dht_frame_return (frame);
-        if (is_last_call (this_call_cnt)) {
-                dht_lookup_everywhere_done (frame, this);
+        gf_msg_debug(this->name, 0,
+                     "%s: Found null hashed subvol. Calling lookup"
+                     " on all nodes.",
+                     loc->path);
+
+        for (i = 0; i < call_cnt; i++) {
+            STACK_WIND_COOKIE(frame, dht_lookup_dir_cbk, conf->subvolumes[i],
+                              conf->subvolumes[i],
+                              conf->subvolumes[i]->fops->lookup, &local->loc,
+                              local->xattr_req);
         }
+        return 0;
+    }
 
-out:
-        return ret;
-}
+    /* if the hashed_subvol is non-null, send the lookup there first so
+     * as to see whether we have a file or a directory */
+    gf_msg_debug(this->name, 0, "%s: Calling fresh lookup on %s", loc->path,
+                 hashed_subvol->name);
 
+    STACK_WIND_COOKIE(frame, dht_lookup_cbk, hashed_subvol, hashed_subvol,
+                      hashed_subvol->fops->lookup, loc, local->xattr_req);
+    return 0;
+err:
+    op_errno = (op_errno == -1) ? errno : op_errno;
+    DHT_STACK_UNWIND(lookup, frame, -1, op_errno, NULL, NULL, NULL, NULL);
+    return 0;
+}
 
-int
-dht_lookup_everywhere (call_frame_t *frame, xlator_t *this, loc_t *loc)
+static int
+dht_do_revalidate(call_frame_t *frame, xlator_t *this, loc_t *loc)
 {
-        dht_conf_t     *conf = NULL;
-        dht_local_t    *local = NULL;
-        int             i = 0;
-        int             call_cnt = 0;
+    xlator_t *subvol = NULL;
+    xlator_t *mds_subvol = NULL;
+    dht_local_t *local = NULL;
+    dht_conf_t *conf = NULL;
+    int ret = -1;
+    int op_errno = -1;
+    dht_layout_t *layout = NULL;
+    int i = 0;
+    int call_cnt = 0;
+    int gen = 0;
+
+    conf = this->private;
+    if (!conf) {
+        op_errno = EINVAL;
+        goto err;
+    }
+
+    local = frame->local;
+    if (!local) {
+        op_errno = EINVAL;
+        goto err;
+    }
+
+    layout = local->layout;
+    if (!layout) {
+        gf_msg_debug(this->name, 0,
+                     "path = %s. No layout found in the inode ctx.", loc->path);
+        op_errno = EINVAL;
+        goto err;
+    }
+
+    /* Generation number has changed. This layout may be stale. */
+    if (layout->gen && (layout->gen < conf->gen)) {
+        gen = layout->gen;
+        dht_layout_unref(this, local->layout);
+        local->layout = NULL;
+        local->cached_subvol = NULL;
+
+        gf_msg_debug(this->name, 0,
+                     "path = %s. In memory layout may be stale."
+                     "(layout->gen (%d) is less than "
+                     "conf->gen (%d)). Calling fresh lookup.",
+                     loc->path, gen, conf->gen);
+
+        dht_do_fresh_lookup(frame, this, loc);
+        return 0;
+    }
+
+    local->inode = inode_ref(loc->inode);
+
+    /* Since we don't know whether this has changed,
+     * request all xattrs*/
+    ret = dht_set_file_xattr_req(this, loc, local->xattr_req);
+    if (ret) {
+        op_errno = -ret;
+        goto err;
+    }
+
+    ret = dht_set_dir_xattr_req(this, loc, local->xattr_req);
+    if (ret) {
+        op_errno = -ret;
+        goto err;
+    }
+
+    if (IA_ISDIR(local->inode->ia_type)) {
+        ret = dht_inode_ctx_mdsvol_get(local->inode, this, &mds_subvol);
+        if (ret || !mds_subvol) {
+            gf_msg_debug(this->name, 0, "path = %s. No mds subvol in inode ctx",
+                         local->loc.path);
+        }
+        local->mds_subvol = mds_subvol;
+        local->call_cnt = conf->subvolume_cnt;
 
-        GF_VALIDATE_OR_GOTO ("dht", frame, err);
-        GF_VALIDATE_OR_GOTO ("dht", this, out);
-        GF_VALIDATE_OR_GOTO ("dht", frame->local, out);
-        GF_VALIDATE_OR_GOTO ("dht", this->private, out);
-        GF_VALIDATE_OR_GOTO ("dht", loc, out);
+        /* local->call_cnt will change as responses are processed. Always use a
+         * local copy to loop through the STACK_WIND calls
+         */
 
-        conf = this->private;
-        local = frame->local;
+        call_cnt = local->call_cnt;
 
-        call_cnt = conf->subvolume_cnt;
-        local->call_cnt = call_cnt;
+        for (i = 0; i < call_cnt; i++) {
+            STACK_WIND_COOKIE(frame, dht_revalidate_cbk, conf->subvolumes[i],
+                              conf->subvolumes[i],
+                              conf->subvolumes[i]->fops->lookup, loc,
+                              local->xattr_req);
+        }
+        return 0;
+    }
 
-        if (!local->inode)
-                local->inode = inode_ref (loc->inode);
+    /* If not a dir, this should be 1 */
+    local->call_cnt = layout->cnt;
+    call_cnt = local->call_cnt;
 
-        gf_msg_debug (this->name, 0,
-                      "winding lookup call to %d subvols", call_cnt);
+    for (i = 0; i < call_cnt; i++) {
+        subvol = layout->list[i].xlator;
 
-        for (i = 0; i < call_cnt; i++) {
-                STACK_WIND (frame, dht_lookup_everywhere_cbk,
-                            conf->subvolumes[i],
-                            conf->subvolumes[i]->fops->lookup,
-                            loc, local->xattr_req);
-        }
+        gf_msg_debug(this->name, 0,
+                     "path = %s. Calling "
+                     "revalidate lookup on %s",
+                     loc->path, subvol->name);
 
-        return 0;
-out:
-        DHT_STACK_UNWIND (lookup, frame, -1, EINVAL, NULL, NULL, NULL, NULL);
+        STACK_WIND_COOKIE(frame, dht_revalidate_cbk, subvol, subvol,
+                          subvol->fops->lookup, &local->loc, local->xattr_req);
+    }
+    return 0;
 err:
-        return -1;
+    op_errno = (op_errno == -1) ? errno : op_errno;
+    DHT_STACK_UNWIND(lookup, frame, -1, op_errno, NULL, NULL, NULL, NULL);
+    return 0;
 }
 
+/* Depending on the input, decide if this is a:
+ * fresh-lookup: loc->name is provided but no dht inode ctx
+ * revalidation: loc->name is provided, dht inode ctx is present
+ * discover: gfid based nameless lookup.
+ */
 
 int
-dht_lookup_linkfile_cbk (call_frame_t *frame, void *cookie,
-                         xlator_t *this, int op_ret, int op_errno,
-                         inode_t *inode, struct iatt *stbuf, dict_t *xattr,
-                         struct iatt *postparent)
-{
-        call_frame_t *prev          = NULL;
-        dht_local_t  *local         = NULL;
-        xlator_t     *subvol        = NULL;
-        loc_t        *loc           = NULL;
-        dht_conf_t   *conf          = NULL;
-        int           ret           = 0;
-        char          gfid[GF_UUID_BUF_SIZE]     = {0};
-
-        GF_VALIDATE_OR_GOTO ("dht", frame, out);
-        GF_VALIDATE_OR_GOTO ("dht", this, unwind);
-        GF_VALIDATE_OR_GOTO ("dht", frame->local, unwind);
-        GF_VALIDATE_OR_GOTO ("dht", this->private, unwind);
-        GF_VALIDATE_OR_GOTO ("dht", cookie, unwind);
-
-        prev   = cookie;
-        subvol = prev->this;
-        conf   = this->private;
-        local  = frame->local;
-        loc    = &local->loc;
+dht_lookup(call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xattr_req)
+{
+    xlator_t *hashed_subvol = NULL;
+    dht_local_t *local = NULL;
+    dht_conf_t *conf = NULL;
+    int ret = -1;
+    int op_errno = -1;
+    loc_t new_loc = {
+        0,
+    };
+
+    VALIDATE_OR_GOTO(frame, err);
+    VALIDATE_OR_GOTO(this, err);
+    VALIDATE_OR_GOTO(loc, err);
+    VALIDATE_OR_GOTO(loc->inode, err);
+
+    conf = this->private;
+    if (!conf)
+        goto err;
+
+    local = dht_local_init(frame, loc, NULL, GF_FOP_LOOKUP);
+    if (!local) {
+        op_errno = ENOMEM;
+        goto err;
+    }
+
+    ret = dht_filter_loc_subvol_key(this, loc, &new_loc, &hashed_subvol);
+    if (ret) {
+        loc_wipe(&local->loc);
+        ret = loc_dup(&new_loc, &local->loc);
+
+        /* we no longer need 'new_loc' entries */
+        loc_wipe(&new_loc);
+
+        /* check if loc_dup() is successful */
+        if (ret == -1) {
+            op_errno = errno;
+            gf_msg_debug(this->name, errno,
+                         "copying location failed for path=%s", loc->path);
+            goto err;
+        }
+    }
+
+    if (xattr_req) {
+        local->xattr_req = dict_ref(xattr_req);
+    } else {
+        local->xattr_req = dict_new();
+    }
+
+    /* Nameless lookup */
+
+    /* This is usually sent by NFS. Lookups are done based on the gfid and
+     * no name information is available. Without the name, dht cannot calculate
+     * the hash and has to send a lookup to all subvols.
+     */
+    if (gf_uuid_is_null(loc->pargfid) && !gf_uuid_is_null(loc->gfid) &&
+        !__is_root_gfid(loc->inode->gfid)) {
+        local->cached_subvol = NULL;
+        dht_do_discover(frame, this, loc);
+        return 0;
+    }
 
-        gf_uuid_unparse(loc->gfid, gfid);
+    if (loc_is_root(loc)) {
+        /* Request the DHT commit hash xattr (trusted.glusterfs.dht.commithash)
+         * set on the brick root.
+         */
+        ret = dict_set_uint32(local->xattr_req, conf->commithash_xattr_name,
+                              sizeof(uint32_t));
+    }
 
-        if (op_ret == -1) {
-                gf_log (this->name, GF_LOG_INFO,
-                        "Lookup of %s on %s (following linkfile) failed (%s)"
-                        ",gfid = %s", local->loc.path, subvol->name,
-                        strerror (op_errno), gfid);
-
-                /* If cached subvol returned ENOTCONN, do not do
-                lookup_everywhere. We need to make sure linkfile does not get
-                removed, which can take away the namespace, and subvol is
-                anyways down. */
-
-                if (op_errno != ENOTCONN)
-                        goto err;
-                else
-                        goto unwind;
-        }
+    if (!hashed_subvol)
+        hashed_subvol = dht_subvol_get_hashed(this, loc);
+    local->hashed_subvol = hashed_subvol;
 
-        if (check_is_dir (inode, stbuf, xattr)) {
-                gf_log (this->name, GF_LOG_INFO,
-                        "Lookup of %s on %s (following linkfile) reached dir,"
-                        " gfid = %s", local->loc.path, subvol->name, gfid);
-                goto err;
-        }
+    if (is_revalidate(loc)) {
+        /* The entry has been looked up before and has a dht inode_ctx
+         */
+        dht_do_revalidate(frame, this, loc);
+        return 0;
+    } else {
+        /* Entry has not been looked up before
+         */
+        dht_do_fresh_lookup(frame, this, loc);
+        return 0;
+    }
 
-        if (check_is_linkfile (inode, stbuf, xattr, conf->link_xattr_name)) {
-                gf_log (this->name, GF_LOG_INFO,
-                        "lookup of %s on %s (following linkfile) reached link,"
-                        "gfid = %s", local->loc.path, subvol->name, gfid);
-                goto err;
-        }
+    return 0;
+err:
+    op_errno = (op_errno == -1) ? errno : op_errno;
+    DHT_STACK_UNWIND(lookup, frame, -1, op_errno, NULL, NULL, NULL, NULL);
+    return 0;
+}
 
-        if (gf_uuid_compare (local->gfid, stbuf->ia_gfid)) {
-                gf_msg (this->name, GF_LOG_WARNING, 0,
-                        DHT_MSG_GFID_MISMATCH,
-                        "%s: gfid different on data file on %s,"
-                        " gfid local = %s, gfid node = %s ",
-                        local->loc.path, subvol->name, gfid,
-                        uuid_utoa(stbuf->ia_gfid));
-                goto err;
-        }
+static int
+dht_unlink_linkfile_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                        int op_ret, int op_errno, struct iatt *preparent,
+                        struct iatt *postparent, dict_t *xdata)
+{
+    dht_local_t *local = NULL;
+    xlator_t *prev = NULL;
 
-        if ((stbuf->ia_nlink == 1)
-            && (conf && conf->unhashed_sticky_bit)) {
-                stbuf->ia_prot.sticky = 1;
-        }
+    local = frame->local;
+    prev = cookie;
 
-        ret = dht_layout_preset (this, prev->this, inode);
-        if (ret < 0) {
-                gf_log (this->name, GF_LOG_INFO,
-                        "Failed to set layout for subvolume %s,"
-                        "gfid = %s", prev->this->name, gfid);
-                op_ret   = -1;
-                op_errno = EINVAL;
+    LOCK(&frame->lock);
+    {
+        if ((op_ret == -1) &&
+            !((op_errno == ENOENT) || (op_errno == ENOTCONN))) {
+            local->op_errno = op_errno;
+            UNLOCK(&frame->lock);
+            gf_msg_debug(this->name, op_errno,
+                         "Unlink link: subvolume %s returned -1", prev->name);
+            goto post_unlock;
         }
 
-        if (local->loc.parent) {
-                dht_inode_ctx_time_update (local->loc.parent, this,
-                                           postparent, 1);
+        local->op_ret = 0;
+    }
+    UNLOCK(&frame->lock);
+post_unlock:
+    dht_set_fixed_dir_stat(&local->preparent);
+    dht_set_fixed_dir_stat(&local->postparent);
+    DHT_STACK_UNWIND(unlink, frame, local->op_ret, local->op_errno,
+                     &local->preparent, &local->postparent, xdata);
+
+    return 0;
+}
+
+static int
+dht_unlink_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int op_ret,
+               int op_errno, struct iatt *preparent, struct iatt *postparent,
+               dict_t *xdata)
+{
+    dht_local_t *local = NULL;
+    xlator_t *prev = NULL;
+    xlator_t *hashed_subvol = NULL;
+
+    local = frame->local;
+    prev = cookie;
+
+    LOCK(&frame->lock);
+    {
+        if (op_ret == -1) {
+            if (op_errno != ENOENT) {
+                local->op_ret = -1;
+                local->op_errno = op_errno;
+            } else {
+                local->op_ret = 0;
+            }
+            UNLOCK(&frame->lock);
+            gf_msg_debug(this->name, op_errno,
+                         "Unlink: subvolume %s returned -1", prev->name);
+            goto post_unlock;
         }
 
-unwind:
-        DHT_STRIP_PHASE1_FLAGS (stbuf);
-        DHT_STACK_UNWIND (lookup, frame, op_ret, op_errno, inode, stbuf, xattr,
-                          postparent);
+        local->op_ret = 0;
 
-        return 0;
+        local->postparent = *postparent;
+        local->preparent = *preparent;
 
-err:
-        dht_lookup_everywhere (frame, this, loc);
-out:
-        return 0;
+        if (local->loc.parent) {
+            dht_inode_ctx_time_update(local->loc.parent, this,
+                                      &local->preparent, 0);
+            dht_inode_ctx_time_update(local->loc.parent, this,
+                                      &local->postparent, 1);
+        }
+    }
+    UNLOCK(&frame->lock);
+post_unlock:
+    if (!local->op_ret) {
+        hashed_subvol = dht_subvol_get_hashed(this, &local->loc);
+        if (hashed_subvol && hashed_subvol != local->cached_subvol) {
+            /*
+             * If hashed and cached are different, then we need
+             * to unlink linkfile from hashed subvol if data
+             * file is deleted successfully
+             */
+            STACK_WIND_COOKIE(frame, dht_unlink_linkfile_cbk, hashed_subvol,
+                              hashed_subvol, hashed_subvol->fops->unlink,
+                              &local->loc, local->flags, xdata);
+            return 0;
+        }
+    }
+
+    dht_set_fixed_dir_stat(&local->preparent);
+    dht_set_fixed_dir_stat(&local->postparent);
+    DHT_STACK_UNWIND(unlink, frame, local->op_ret, local->op_errno,
+                     &local->preparent, &local->postparent, xdata);
+
+    return 0;
 }
 
-
-int
-dht_lookup_directory (call_frame_t *frame, xlator_t *this, loc_t *loc)
+static int
+dht_common_setxattr_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                        int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
-        int           call_cnt = 0;
-        int           i = 0;
-        dht_conf_t   *conf = NULL;
-        dht_local_t  *local = NULL;
-        int           ret = 0;
+    DHT_STACK_UNWIND(setxattr, frame, op_ret, op_errno, xdata);
+    return 0;
+}
 
-        GF_VALIDATE_OR_GOTO ("dht", frame, out);
-        GF_VALIDATE_OR_GOTO ("dht", this, unwind);
-        GF_VALIDATE_OR_GOTO ("dht", frame->local, unwind);
-        GF_VALIDATE_OR_GOTO ("dht", this->private, unwind);
-        GF_VALIDATE_OR_GOTO ("dht", loc, unwind);
+static int
+dht_fix_layout_setxattr_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                            int32_t op_ret, int32_t op_errno, dict_t *xdata)
+{
+    dht_local_t *local = NULL;
+    dht_layout_t *layout = NULL;
 
-        conf = this->private;
+    if (op_ret == 0) {
+        /* update the layout in the inode ctx */
         local = frame->local;
+        layout = local->selfheal.layout;
 
-        call_cnt        = conf->subvolume_cnt;
-        local->call_cnt = call_cnt;
+        dht_layout_set(this, local->loc.inode, layout);
+    }
 
-        local->layout = dht_layout_new (this, conf->subvolume_cnt);
-        if (!local->layout) {
-                goto unwind;
-        }
+    DHT_STACK_UNWIND(setxattr, frame, op_ret, op_errno, xdata);
+    return 0;
+}
 
-        if (local->xattr != NULL) {
-                dict_unref (local->xattr);
-                local->xattr = NULL;
-        }
+static int
+dht_err_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int op_ret,
+            int op_errno, dict_t *xdata)
+{
+    dht_local_t *local = NULL;
+    int this_call_cnt = 0;
+    xlator_t *prev = NULL;
 
-        if (!gf_uuid_is_null (local->gfid)) {
-                ret = dict_set_static_bin (local->xattr_req, "gfid-req",
-                                           local->gfid, 16);
-                if (ret)
-                        gf_msg (this->name, GF_LOG_WARNING, 0,
-                                DHT_MSG_DICT_SET_FAILED,
-                                "%s: Failed to set dictionary value:"
-                                " key = gfid-req", local->loc.path);
-        }
+    local = frame->local;
+    prev = cookie;
 
-        for (i = 0; i < call_cnt; i++) {
-                STACK_WIND (frame, dht_lookup_dir_cbk,
-                            conf->subvolumes[i],
-                            conf->subvolumes[i]->fops->lookup,
-                            &local->loc, local->xattr_req);
+    LOCK(&frame->lock);
+    {
+        if (op_ret == -1) {
+            local->op_errno = op_errno;
+            UNLOCK(&frame->lock);
+            gf_msg_debug(this->name, op_errno, "subvolume %s returned -1",
+                         prev->name);
+            goto post_unlock;
         }
-        return 0;
-unwind:
-        DHT_STACK_UNWIND (lookup, frame, -1, ENOMEM, NULL, NULL, NULL, NULL);
+
+        local->op_ret = 0;
+    }
+    UNLOCK(&frame->lock);
+post_unlock:
+    this_call_cnt = dht_frame_return(frame);
+    if (is_last_call(this_call_cnt)) {
+        if ((local->fop == GF_FOP_SETXATTR) ||
+            (local->fop == GF_FOP_FSETXATTR)) {
+            DHT_STACK_UNWIND(setxattr, frame, local->op_ret, local->op_errno,
+                             NULL);
+            /* 'local' itself may not be valid after this */
+            goto out;
+        }
+        if ((local->fop == GF_FOP_REMOVEXATTR) ||
+            (local->fop == GF_FOP_FREMOVEXATTR)) {
+            DHT_STACK_UNWIND(removexattr, frame, local->op_ret, local->op_errno,
+                             NULL);
+        }
+    }
+
 out:
-        return 0;
+    return 0;
+}
 
+/* Set the value[] of key into dict after convert from
+   host byte order to network byte order
+*/
+int32_t
+dht_dict_set_array(dict_t *dict, char *key, int32_t value[], int32_t size)
+{
+    int ret = -1;
+    int32_t *ptr = NULL;
+    int32_t vindex;
+
+    if (value == NULL) {
+        return -EINVAL;
+    }
+
+    ptr = GF_MALLOC(sizeof(int32_t) * size, gf_common_mt_char);
+    if (ptr == NULL) {
+        return -ENOMEM;
+    }
+    for (vindex = 0; vindex < size; vindex++) {
+        ptr[vindex] = hton32(value[vindex]);
+    }
+    ret = dict_set_bin(dict, key, ptr, sizeof(int32_t) * size);
+    if (ret)
+        GF_FREE(ptr);
+    return ret;
 }
 
+static int
+dht_common_mds_xattrop_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                           int32_t op_ret, int32_t op_errno, dict_t *dict,
+                           dict_t *xdata)
+{
+    dht_local_t *local = NULL;
+    call_frame_t *prev = cookie;
 
-int
-dht_lookup_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                int op_ret, int op_errno,
-                inode_t *inode, struct iatt *stbuf, dict_t *xattr,
-                struct iatt *postparent)
-{
-        char          is_linkfile   = 0;
-        char          is_dir        = 0;
-        xlator_t     *subvol        = NULL;
-        dht_conf_t   *conf          = NULL;
-        dht_local_t  *local         = NULL;
-        loc_t        *loc           = NULL;
-        call_frame_t *prev          = NULL;
-        int           ret           = 0;
-        dht_layout_t *parent_layout = NULL;
-
-        GF_VALIDATE_OR_GOTO ("dht", frame, err);
-        GF_VALIDATE_OR_GOTO ("dht", this, out);
-        GF_VALIDATE_OR_GOTO ("dht", frame->local, out);
-        GF_VALIDATE_OR_GOTO ("dht", cookie, out);
-        GF_VALIDATE_OR_GOTO ("dht", this->private, out);
-
-        conf  = this->private;
-
-        prev  = cookie;
-        local = frame->local;
-        loc   = &local->loc;
+    local = frame->local;
 
-        /* This is required for handling stale linkfile deletion,
-         * or any more call which happens from this 'loc'.
-         */
-        if (!op_ret && gf_uuid_is_null (local->gfid))
-                memcpy (local->gfid, stbuf->ia_gfid, 16);
-
-        gf_msg_debug (this->name, 0,
-                      "fresh_lookup returned for %s with op_ret %d and "
-                      "op_errno %d", loc->path, op_ret, op_errno);
-
-        if (ENTRY_MISSING (op_ret, op_errno)) {
-                gf_msg_debug (this->name, 0,
-                              "Entry %s missing on subvol %s",
-                              loc->path, prev->this->name);
-
-                if (conf->search_unhashed == GF_DHT_LOOKUP_UNHASHED_ON) {
-                        local->op_errno = ENOENT;
-                        dht_lookup_everywhere (frame, this, loc);
-                        return 0;
-                }
-                if ((conf->search_unhashed == GF_DHT_LOOKUP_UNHASHED_AUTO) &&
-                    (loc->parent)) {
-                        ret = dht_inode_ctx_layout_get (loc->parent, this,
-                                                        &parent_layout);
-                        if (ret || !parent_layout)
-                                goto out;
-                        if (parent_layout->search_unhashed) {
-                                local->op_errno = ENOENT;
-                                dht_lookup_everywhere (frame, this, loc);
-                                return 0;
-                        }
-                }
+    if (op_ret)
+        gf_msg_debug(this->name, op_errno, "subvolume %s returned -1",
+                     prev->this->name);
+
+    if (local->fop == GF_FOP_SETXATTR) {
+        DHT_STACK_UNWIND(setxattr, frame, 0, op_errno, local->xdata);
+        /* 'local' itself may not be valid after this */
+        goto out;
+    }
+
+    if (local->fop == GF_FOP_FSETXATTR) {
+        DHT_STACK_UNWIND(fsetxattr, frame, 0, op_errno, local->xdata);
+        /* 'local' itself may not be valid after this */
+        goto out;
+    }
+
+    if (local->fop == GF_FOP_REMOVEXATTR) {
+        DHT_STACK_UNWIND(removexattr, frame, 0, op_errno, NULL);
+        /* 'local' itself may not be valid after this */
+        goto out;
+    }
+
+    if (local->fop == GF_FOP_FREMOVEXATTR) {
+        DHT_STACK_UNWIND(fremovexattr, frame, 0, op_errno, NULL);
+    }
+
+out:
+    return 0;
+}
+
+/* Code to wind a xattrop call to add 1 on current mds internal xattr
+   value
+*/
+static int
+dht_setxattr_non_mds_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                         int op_ret, int op_errno, dict_t *xdata)
+{
+    dht_local_t *local = NULL;
+    int this_call_cnt = 0;
+    int ret = 0;
+    dict_t *xattrop = NULL;
+    int32_t addone[1] = {1};
+    call_frame_t *prev = NULL;
+    dht_conf_t *conf = NULL;
+
+    local = frame->local;
+    prev = cookie;
+    conf = this->private;
+
+    LOCK(&frame->lock);
+    {
+        if (op_ret && !local->op_ret) {
+            local->op_ret = op_ret;
+            local->op_errno = op_errno;
+            UNLOCK(&frame->lock);
+            gf_msg_debug(this->name, op_errno, "subvolume %s returned -1",
+                         prev->this->name);
+            goto post_unlock;
+        }
+    }
+    UNLOCK(&frame->lock);
+post_unlock:
+    this_call_cnt = dht_frame_return(frame);
+
+    if (is_last_call(this_call_cnt)) {
+        if (!local->op_ret) {
+            xattrop = dict_new();
+            if (!xattrop) {
+                gf_msg(this->name, GF_LOG_ERROR, DHT_MSG_NO_MEMORY, 0,
+                       "dictionary creation failed");
+                ret = -1;
+                goto out;
+            }
+            ret = dht_dict_set_array(xattrop, conf->mds_xattr_key, addone, 1);
+            if (ret != 0) {
+                gf_msg(this->name, GF_LOG_ERROR, 0, DHT_MSG_DICT_SET_FAILED,
+                       "dictionary set array failed ");
+                ret = -1;
+                goto out;
+            }
+            if ((local->fop == GF_FOP_SETXATTR) ||
+                (local->fop == GF_FOP_REMOVEXATTR)) {
+                STACK_WIND(frame, dht_common_mds_xattrop_cbk, local->mds_subvol,
+                           local->mds_subvol->fops->xattrop, &local->loc,
+                           GF_XATTROP_ADD_ARRAY, xattrop, NULL);
+            } else {
+                STACK_WIND(frame, dht_common_mds_xattrop_cbk, local->mds_subvol,
+                           local->mds_subvol->fops->fxattrop, local->fd,
+                           GF_XATTROP_ADD_ARRAY, xattrop, NULL);
+            }
+        } else {
+            if (local->fop == GF_FOP_SETXATTR) {
+                DHT_STACK_UNWIND(setxattr, frame, 0, 0, local->xdata);
+                /* 'local' itself may not be valid after this */
+                goto just_return;
+            }
+
+            if (local->fop == GF_FOP_FSETXATTR) {
+                DHT_STACK_UNWIND(fsetxattr, frame, 0, 0, local->xdata);
+                /* 'local' itself may not be valid after this */
+                goto just_return;
+            }
+
+            if (local->fop == GF_FOP_REMOVEXATTR) {
+                DHT_STACK_UNWIND(removexattr, frame, 0, 0, NULL);
+                /* 'local' itself may not be valid after this */
+                goto just_return;
+            }
+
+            if (local->fop == GF_FOP_FREMOVEXATTR) {
+                DHT_STACK_UNWIND(fremovexattr, frame, 0, 0, NULL);
+                /* 'local' itself may not be valid after this */
+                goto just_return;
+            }
+        }
+    }
+out:
+    if (ret) {
+        if (local->fop == GF_FOP_SETXATTR) {
+            DHT_STACK_UNWIND(setxattr, frame, 0, 0, local->xdata);
+            /* 'local' itself may not be valid after this */
+            goto just_return;
         }
 
-        if (op_ret == 0) {
-                is_dir      = check_is_dir (inode, stbuf, xattr);
-                if (is_dir) {
-                        local->inode = inode_ref (inode);
-                        local->xattr = dict_ref (xattr);
-                }
+        if (local->fop == GF_FOP_FSETXATTR) {
+            DHT_STACK_UNWIND(fsetxattr, frame, 0, 0, local->xdata);
+            /* 'local' itself may not be valid after this */
+            goto just_return;
         }
 
-        if (is_dir || (op_ret == -1 && op_errno == ENOTCONN)) {
-                dht_lookup_directory (frame, this, &local->loc);
-                return 0;
+        if (local->fop == GF_FOP_REMOVEXATTR) {
+            DHT_STACK_UNWIND(removexattr, frame, 0, 0, NULL);
+            /* 'local' itself may not be valid after this */
+            goto just_return;
         }
 
-        if (op_ret == -1) {
-                gf_msg_debug (this->name, 0, "Lookup of %s for subvolume"
-                              " %s failed with error %s", loc->path,
-                              prev->this->name, strerror (op_errno));
-                goto out;
+        if (local->fop == GF_FOP_FREMOVEXATTR) {
+            DHT_STACK_UNWIND(fremovexattr, frame, 0, 0, NULL);
         }
+    }
+just_return:
+    if (xattrop)
+        dict_unref(xattrop);
+    return 0;
+}
 
-        is_linkfile = check_is_linkfile (inode, stbuf, xattr,
-                                         conf->link_xattr_name);
+static int
+dht_setxattr_mds_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                     int op_ret, int op_errno, dict_t *xdata)
+{
+    dht_local_t *local = NULL;
+    dht_conf_t *conf = NULL;
+    call_frame_t *prev = NULL;
+    xlator_t *mds_subvol = NULL;
+    int i = 0;
+
+    local = frame->local;
+    prev = cookie;
+    conf = this->private;
+    mds_subvol = local->mds_subvol;
+
+    if (op_ret == -1) {
+        local->op_ret = op_ret;
+        local->op_errno = op_errno;
+        gf_msg_debug(this->name, op_errno, "subvolume %s returned -1",
+                     prev->this->name);
+        goto out;
+    }
 
-        if (!is_linkfile) {
-                /* non-directory and not a linkfile */
+    local->op_ret = 0;
+    local->call_cnt = conf->subvolume_cnt - 1;
+    local->xdata = dict_ref(xdata);
 
-                ret = dht_layout_preset (this, prev->this, inode);
-                if (ret < 0) {
-                        gf_log (this->name, GF_LOG_INFO,
-                                "could not set pre-set layout for subvolume %s",
-                                prev->this->name);
-                        op_ret   = -1;
-                        op_errno = EINVAL;
-                        goto out;
-                }
-                goto out;
+    for (i = 0; i < conf->subvolume_cnt; i++) {
+        if (mds_subvol && (mds_subvol == conf->subvolumes[i]))
+            continue;
+        if (local->fop == GF_FOP_SETXATTR) {
+            STACK_WIND(frame, dht_setxattr_non_mds_cbk, conf->subvolumes[i],
+                       conf->subvolumes[i]->fops->setxattr, &local->loc,
+                       local->xattr, local->flags, local->xattr_req);
         }
 
-        subvol = dht_linkfile_subvol (this, inode, stbuf, xattr);
-        if (!subvol) {
+        if (local->fop == GF_FOP_FSETXATTR) {
+            STACK_WIND(frame, dht_setxattr_non_mds_cbk, conf->subvolumes[i],
+                       conf->subvolumes[i]->fops->fsetxattr, local->fd,
+                       local->xattr, local->flags, local->xattr_req);
+        }
 
-                gf_log (this->name, GF_LOG_INFO, "linkfile not having link "
-                        "subvol for %s", loc->path);
+        if (local->fop == GF_FOP_REMOVEXATTR) {
+            STACK_WIND(frame, dht_setxattr_non_mds_cbk, conf->subvolumes[i],
+                       conf->subvolumes[i]->fops->removexattr, &local->loc,
+                       local->key, local->xattr_req);
+        }
 
-                gf_msg_debug (this->name, 0,
-                              "linkfile not having link subvolume. path=%s",
-                              loc->path);
-                dht_lookup_everywhere (frame, this, loc);
-                return 0;
+        if (local->fop == GF_FOP_FREMOVEXATTR) {
+            STACK_WIND(frame, dht_setxattr_non_mds_cbk, conf->subvolumes[i],
+                       conf->subvolumes[i]->fops->fremovexattr, local->fd,
+                       local->key, local->xattr_req);
         }
+    }
 
-        gf_msg_debug (this->name, 0,
-                      "Calling lookup on linkto target %s for path %s",
-                      subvol->name, loc->path);
+    return 0;
+out:
+    if (local->fop == GF_FOP_SETXATTR) {
+        DHT_STACK_UNWIND(setxattr, frame, local->op_ret, local->op_errno,
+                         xdata);
+        /* 'local' itself may not be valid after this */
+        goto just_return;
+    }
+
+    if (local->fop == GF_FOP_FSETXATTR) {
+        DHT_STACK_UNWIND(fsetxattr, frame, local->op_ret, local->op_errno,
+                         xdata);
+        /* 'local' itself may not be valid after this */
+        goto just_return;
+    }
+
+    if (local->fop == GF_FOP_REMOVEXATTR) {
+        DHT_STACK_UNWIND(removexattr, frame, local->op_ret, local->op_errno,
+                         NULL);
+        /* 'local' itself may not be valid after this */
+        goto just_return;
+    }
+
+    if (local->fop == GF_FOP_FREMOVEXATTR) {
+        DHT_STACK_UNWIND(fremovexattr, frame, local->op_ret, local->op_errno,
+                         NULL);
+    }
+
+just_return:
+    return 0;
+}
 
-        STACK_WIND (frame, dht_lookup_linkfile_cbk,
-                    subvol, subvol->fops->lookup,
-                    &local->loc, local->xattr_req);
+static int
+dht_xattrop_mds_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                    int op_ret, int op_errno, dict_t *dict, dict_t *xdata)
+{
+    dht_local_t *local = NULL;
+    call_frame_t *prev = NULL;
 
-        return 0;
+    local = frame->local;
+    prev = cookie;
 
+    if (op_ret == -1) {
+        local->op_errno = op_errno;
+        local->op_ret = op_ret;
+        gf_msg_debug(this->name, op_errno, "subvolume %s returned -1",
+                     prev->this->name);
+        goto out;
+    }
+
+    if (local->fop == GF_FOP_SETXATTR) {
+        STACK_WIND(frame, dht_setxattr_mds_cbk, local->mds_subvol,
+                   local->mds_subvol->fops->setxattr, &local->loc, local->xattr,
+                   local->flags, local->xattr_req);
+    }
+
+    if (local->fop == GF_FOP_FSETXATTR) {
+        STACK_WIND(frame, dht_setxattr_mds_cbk, local->mds_subvol,
+                   local->mds_subvol->fops->fsetxattr, local->fd, local->xattr,
+                   local->flags, local->xattr_req);
+    }
+
+    if (local->fop == GF_FOP_REMOVEXATTR) {
+        STACK_WIND(frame, dht_setxattr_mds_cbk, local->mds_subvol,
+                   local->mds_subvol->fops->removexattr, &local->loc,
+                   local->key, local->xattr_req);
+    }
+
+    if (local->fop == GF_FOP_FREMOVEXATTR) {
+        STACK_WIND(frame, dht_setxattr_mds_cbk, local->mds_subvol,
+                   local->mds_subvol->fops->fremovexattr, local->fd, local->key,
+                   local->xattr_req);
+    }
+
+    return 0;
 out:
-        /*
-         * FIXME: postparent->ia_size and postparent->st_blocks do not have
-         * correct values. since, postparent corresponds to a directory these
-         * two members should have values equal to sum of corresponding values
-         * from each of the subvolume. See dht_iatt_merge for reference.
-         */
-
-        if (!op_ret && local && local->loc.parent) {
-                dht_inode_ctx_time_update (local->loc.parent, this,
-                                           postparent, 1);
-        }
+    if (local->fop == GF_FOP_SETXATTR) {
+        DHT_STACK_UNWIND(setxattr, frame, local->op_ret, local->op_errno,
+                         xdata);
+        /* 'local' itself may not be valid after this */
+        goto just_return;
+    }
+
+    if (local->fop == GF_FOP_FSETXATTR) {
+        DHT_STACK_UNWIND(fsetxattr, frame, local->op_ret, local->op_errno,
+                         xdata);
+        /* 'local' itself may not be valid after this */
+        goto just_return;
+    }
+
+    if (local->fop == GF_FOP_REMOVEXATTR) {
+        DHT_STACK_UNWIND(removexattr, frame, local->op_ret, local->op_errno,
+                         NULL);
+        /* 'local' itself may not be valid after this */
+        goto just_return;
+    }
+
+    if (local->fop == GF_FOP_FREMOVEXATTR) {
+        DHT_STACK_UNWIND(fremovexattr, frame, local->op_ret, local->op_errno,
+                         NULL);
+    }
+
+just_return:
+    return 0;
+}
 
-        DHT_STRIP_PHASE1_FLAGS (stbuf);
-        DHT_STACK_UNWIND (lookup, frame, op_ret, op_errno, inode, stbuf, xattr,
-                          postparent);
-err:
-        return 0;
+static void
+fill_layout_info(dht_layout_t *layout, char *buf)
+{
+    int i = 0;
+    char tmp_buf[128] = {
+        0,
+    };
+
+    for (i = 0; i < layout->cnt; i++) {
+        snprintf(tmp_buf, sizeof(tmp_buf), "(%s %u %u)",
+                 layout->list[i].xlator->name, layout->list[i].start,
+                 layout->list[i].stop);
+        if (i)
+            strcat(buf, " ");
+        strcat(buf, tmp_buf);
+    }
 }
 
-/* For directories, check if acl xattrs have been requested (by the acl xlator),
- * if not, request for them. These xattrs are needed for dht dir self-heal to
- * perform proper self-healing of dirs
- */
-void
-dht_check_and_set_acl_xattr_req (inode_t *inode, dict_t *xattr_req)
+static void
+dht_fill_pathinfo_xattr(xlator_t *this, dht_local_t *local, char *xattr_buf,
+                        int32_t alloc_len, int flag, char *layout_buf)
 {
-        int     ret = 0;
+    if (flag) {
+        if (local->xattr_val) {
+            snprintf(xattr_buf, alloc_len,
+                     "((<" DHT_PATHINFO_HEADER "%s> %s) (%s-layout %s))",
+                     this->name, local->xattr_val, this->name, layout_buf);
+        } else {
+            snprintf(xattr_buf, alloc_len, "(%s-layout %s)", this->name,
+                     layout_buf);
+        }
+    } else if (local->xattr_val) {
+        snprintf(xattr_buf, alloc_len, "(<" DHT_PATHINFO_HEADER "%s> %s)",
+                 this->name, local->xattr_val);
+    } else {
+        xattr_buf[0] = '\0';
+    }
+}
 
-        GF_ASSERT (inode);
-        GF_ASSERT (xattr_req);
+static int
+dht_vgetxattr_alloc_and_fill(dht_local_t *local, dict_t *xattr, xlator_t *this,
+                             int op_errno)
+{
+    int ret = -1;
+    char *value = NULL;
+
+    ret = dict_get_str(xattr, local->xsel, &value);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, op_errno, DHT_MSG_GET_XATTR_FAILED,
+               "Subvolume %s returned -1", this->name);
+        local->op_ret = -1;
+        local->op_errno = op_errno;
+        goto out;
+    }
 
-        if (inode->ia_type != IA_IFDIR)
-                return;
+    local->alloc_len += strlen(value);
 
-        if (!dict_get (xattr_req, POSIX_ACL_ACCESS_XATTR)) {
-                ret = dict_set_int8 (xattr_req, POSIX_ACL_ACCESS_XATTR, 0);
-                if (ret)
-                        gf_msg (THIS->name, GF_LOG_WARNING, 0,
-                                DHT_MSG_DICT_SET_FAILED,
-                                "Failed to set dictionary value:key = %s",
-                                POSIX_ACL_ACCESS_XATTR);
+    if (!local->xattr_val) {
+        local->alloc_len += (SLEN(DHT_PATHINFO_HEADER) + 10);
+        local->xattr_val = GF_MALLOC(local->alloc_len, gf_common_mt_char);
+        if (!local->xattr_val) {
+            ret = -1;
+            goto out;
         }
+        local->xattr_val[0] = '\0';
+    }
 
-        if (!dict_get (xattr_req, POSIX_ACL_DEFAULT_XATTR)) {
-                ret = dict_set_int8 (xattr_req, POSIX_ACL_DEFAULT_XATTR, 0);
-                if (ret)
-                        gf_msg (THIS->name, GF_LOG_WARNING, 0,
-                                DHT_MSG_DICT_SET_FAILED,
-                                "Failed to set dictionary value:key = %s",
-                                POSIX_ACL_DEFAULT_XATTR);
+    int plen = strlen(local->xattr_val);
+    if (plen) {
+        /* extra byte(s) for \0 to be safe */
+        local->alloc_len += (plen + 2);
+        local->xattr_val = GF_REALLOC(local->xattr_val, local->alloc_len);
+        if (!local->xattr_val) {
+            ret = -1;
+            goto out;
         }
+    }
 
-        return;
+    (void)strcat(local->xattr_val, value);
+    (void)strcat(local->xattr_val, " ");
+    local->op_ret = 0;
+
+    ret = 0;
+
+out:
+    return ret;
 }
 
-int
-dht_lookup (call_frame_t *frame, xlator_t *this,
-            loc_t *loc, dict_t *xattr_req)
-{
-        xlator_t     *subvol = NULL;
-        xlator_t     *hashed_subvol = NULL;
-        dht_local_t  *local  = NULL;
-        dht_conf_t   *conf = NULL;
-        int           ret    = -1;
-        int           op_errno = -1;
-        dht_layout_t *layout = NULL;
-        int           i = 0;
-        int           call_cnt = 0;
-        loc_t         new_loc = {0,};
-
-        VALIDATE_OR_GOTO (frame, err);
-        VALIDATE_OR_GOTO (this, err);
-        VALIDATE_OR_GOTO (loc, err);
-        VALIDATE_OR_GOTO (loc->inode, err);
-
-        conf = this->private;
-        if (!conf)
-                goto err;
+static int
+dht_vgetxattr_fill_and_set(dht_local_t *local, dict_t **dict, xlator_t *this,
+                           gf_boolean_t flag)
+{
+    int ret = -1;
+    char *xattr_buf = NULL;
+    char layout_buf[8192] = {
+        0,
+    };
+
+    if (flag)
+        fill_layout_info(local->layout, layout_buf);
+
+    *dict = dict_new();
+    if (!*dict)
+        goto out;
+
+    local->xattr_val[strlen(local->xattr_val) - 1] = '\0';
+
+    /* we would need max this many bytes to create xattr string
+     * extra 40 bytes is just an estimated amount of additional
+     * space required as we include translator name and some
+     * spaces, brackets etc. when forming the pathinfo string.
+     *
+     * For node-uuid we just don't have all the pretty formatting,
+     * but since this is a generic routine for pathinfo & node-uuid
+     * we don't have conditional space allocation and try to be
+     * generic
+     */
+    local->alloc_len += (2 * strlen(this->name)) + strlen(layout_buf) + 40;
+    xattr_buf = GF_MALLOC(local->alloc_len, gf_common_mt_char);
+    if (!xattr_buf)
+        goto out;
+
+    if (XATTR_IS_PATHINFO(local->xsel)) {
+        (void)dht_fill_pathinfo_xattr(this, local, xattr_buf, local->alloc_len,
+                                      flag, layout_buf);
+    } else if ((XATTR_IS_NODE_UUID(local->xsel)) ||
+               (XATTR_IS_NODE_UUID_LIST(local->xsel))) {
+        (void)snprintf(xattr_buf, local->alloc_len, "%s", local->xattr_val);
+    } else {
+        gf_msg(this->name, GF_LOG_WARNING, 0, DHT_MSG_GET_XATTR_FAILED,
+               "Unknown local->xsel (%s)", local->xsel);
+        GF_FREE(xattr_buf);
+        goto out;
+    }
+
+    ret = dict_set_dynstr(*dict, local->xsel, xattr_buf);
+    if (ret)
+        GF_FREE(xattr_buf);
+    GF_FREE(local->xattr_val);
 
-        local = dht_local_init (frame, loc, NULL, GF_FOP_LOOKUP);
-        if (!local) {
-                op_errno = ENOMEM;
-                goto err;
-        }
+out:
+    return ret;
+}
 
-        ret = dht_filter_loc_subvol_key (this, loc, &new_loc,
-                                         &hashed_subvol);
-        if (ret) {
-                loc_wipe (&local->loc);
-                ret = loc_dup (&new_loc, &local->loc);
-
-                /* we no more need 'new_loc' entries */
-                loc_wipe (&new_loc);
-
-                /* check if loc_dup() is successful */
-                if (ret == -1) {
-                        op_errno = errno;
-                        gf_msg_debug (this->name, 0,
-                                      "copying location failed for path=%s",
-                                      loc->path);
-                        goto err;
-                }
+static int
+dht_find_local_subvol_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                          int op_ret, int op_errno, dict_t *xattr,
+                          dict_t *xdata)
+{
+    dht_local_t *local = NULL;
+    dht_conf_t *conf = NULL;
+    xlator_t *prev = NULL;
+    int this_call_cnt = 0;
+    int ret = 0;
+    char *uuid_str = NULL;
+    char *uuid_list = NULL;
+    char *next_uuid_str = NULL;
+    char *saveptr = NULL;
+    uuid_t node_uuid = {
+        0,
+    };
+    char *uuid_list_copy = NULL;
+    int count = 0;
+    int i = 0;
+    int index = 0;
+    int found = 0;
+    nodeuuid_info_t *tmp_ptr = NULL;
+
+    VALIDATE_OR_GOTO(frame, out);
+    VALIDATE_OR_GOTO(frame->local, out);
+
+    local = frame->local;
+    prev = cookie;
+    conf = this->private;
+
+    VALIDATE_OR_GOTO(conf->defrag, out);
+
+    gf_msg_debug(this->name, 0, "subvol %s returned", prev->name);
+
+    LOCK(&frame->lock);
+    {
+        this_call_cnt = --local->call_cnt;
+        if (op_ret < 0) {
+            local->op_ret = -1;
+            local->op_errno = op_errno;
+            UNLOCK(&frame->lock);
+            if (op_errno == ENODATA)
+                gf_msg_debug(this->name, 0, "failed to get node-uuid");
+            else
+                gf_msg(this->name, GF_LOG_ERROR, op_errno,
+                       DHT_MSG_GET_XATTR_FAILED, "failed to get node-uuid");
+            goto post_unlock;
         }
 
-        if (xattr_req) {
-                local->xattr_req = dict_ref (xattr_req);
-        } else {
-                local->xattr_req = dict_new ();
+        ret = dict_get_str(xattr, local->xsel, &uuid_list);
+
+        if (ret < 0) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, DHT_MSG_DICT_GET_FAILED,
+                   "Failed to get %s", local->xsel);
+            local->op_ret = -1;
+            local->op_errno = EINVAL;
+            goto unlock;
         }
 
-        if (gf_uuid_is_null (loc->pargfid) && !gf_uuid_is_null (loc->gfid) &&
-            !__is_root_gfid (loc->inode->gfid)) {
-                local->cached_subvol = NULL;
-                dht_discover (frame, this, loc);
-                return 0;
+        /* As DHT will not know details of its child xlators
+         * we need to parse this twice to get the count first
+         * and allocate memory later.
+         */
+        count = 0;
+        index = conf->local_subvols_cnt;
+
+        uuid_list_copy = gf_strdup(uuid_list);
+        if (!uuid_list_copy)
+            goto unlock;
+
+        for (uuid_str = strtok_r(uuid_list, " ", &saveptr); uuid_str;
+             uuid_str = next_uuid_str) {
+            next_uuid_str = strtok_r(NULL, " ", &saveptr);
+            if (gf_uuid_parse(uuid_str, node_uuid)) {
+                local->op_ret = -1;
+                local->op_errno = EINVAL;
+                UNLOCK(&frame->lock);
+                gf_msg(this->name, GF_LOG_ERROR, 0, DHT_MSG_UUID_PARSE_ERROR,
+                       "Failed to parse uuid for %s", prev->name);
+                goto post_unlock;
+            }
+
+            count++;
+            if (gf_uuid_compare(node_uuid, conf->defrag->node_uuid)) {
+                gf_msg_debug(this->name, 0,
+                             "subvol %s does not"
+                             "belong to this node",
+                             prev->name);
+            } else {
+                /* handle multiple bricks of the same replica
+                 * on the same node */
+                if (found)
+                    continue;
+                conf->local_subvols[(conf->local_subvols_cnt)++] = prev;
+                found = 1;
+                gf_msg_debug(this->name, 0,
+                             "subvol %s belongs to"
+                             " this node",
+                             prev->name);
+            }
+        }
+
+        if (!found) {
+            local->op_ret = 0;
+            goto unlock;
+        }
+
+        conf->local_nodeuuids[index].count = count;
+        conf->local_nodeuuids[index].elements = GF_CALLOC(
+            count, sizeof(nodeuuid_info_t), 1);
+
+        /* The node-uuids are guaranteed to be returned in the same
+         * order as the bricks
+         * A null node-uuid is returned for a brick that is down.
+         */
+
+        saveptr = NULL;
+        i = 0;
+
+        for (uuid_str = strtok_r(uuid_list_copy, " ", &saveptr); uuid_str;
+             uuid_str = next_uuid_str) {
+            next_uuid_str = strtok_r(NULL, " ", &saveptr);
+            tmp_ptr = &(conf->local_nodeuuids[index].elements[i]);
+            gf_uuid_parse(uuid_str, tmp_ptr->uuid);
+
+            if (!gf_uuid_compare(tmp_ptr->uuid, conf->defrag->node_uuid)) {
+                tmp_ptr->info = REBAL_NODEUUID_MINE;
+            }
+            i++;
+            tmp_ptr = NULL;
         }
+    }
 
-        if (!hashed_subvol)
-                hashed_subvol = dht_subvol_get_hashed (this, loc);
-        local->hashed_subvol = hashed_subvol;
+    local->op_ret = 0;
+unlock:
+    UNLOCK(&frame->lock);
+post_unlock:
+    if (!is_last_call(this_call_cnt))
+        goto out;
 
-        if (is_revalidate (loc)) {
-                layout = local->layout;
-                if (!layout) {
-                        gf_msg_debug (this->name, 0,
-                                      "Revalidate lookup without cache."
-                                      " path=%s", loc->path);
-                        op_errno = EINVAL;
-                        goto err;
-                }
+    if (local->op_ret == -1) {
+        goto unwind;
+    }
 
-                if (layout->gen && (layout->gen < conf->gen)) {
-                        gf_msg_trace (this->name, 0,
-                                      "incomplete layout failure for path=%s",
-                                      loc->path);
+    DHT_STACK_UNWIND(getxattr, frame, 0, 0, xattr, xdata);
+    goto out;
 
-                        dht_layout_unref (this, local->layout);
-                        local->layout = NULL;
-                        local->cached_subvol = NULL;
+unwind:
 
-                        gf_msg_debug(this->name, 0,
-                                     "Called revalidate lookup for %s, "
-                                     "but layout->gen (%d) is less than "
-                                     "conf->gen (%d), calling fresh_lookup",
-                                     loc->path, layout->gen, conf->gen);
+    GF_FREE(conf->local_nodeuuids[index].elements);
+    conf->local_nodeuuids[index].elements = NULL;
 
-                        goto do_fresh_lookup;
-                }
+    DHT_STACK_UNWIND(getxattr, frame, -1, local->op_errno, NULL, xdata);
+out:
+    GF_FREE(uuid_list_copy);
+    return 0;
+}
 
-                local->inode = inode_ref (loc->inode);
+static int
+dht_vgetxattr_dir_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                      int op_ret, int op_errno, dict_t *xattr, dict_t *xdata)
+{
+    int ret = 0;
+    dht_local_t *local = NULL;
+    int this_call_cnt = 0;
+    dict_t *dict = NULL;
 
-                ret = dict_set_uint32 (local->xattr_req,
-                                       conf->xattr_name, 4 * 4);
-                if (ret) {
-                        gf_msg (this->name, GF_LOG_WARNING, ENOMEM,
-                                DHT_MSG_DICT_SET_FAILED,
-                                "Failed to set dictionary value:key = %s for "
-                                "path %s", conf->xattr_name, loc->path);
-                        goto err;
-                }
-                /* need it in case file is not found on cached file
-                 * on revalidate path and we may encounter linkto files on
-                 * with dht_lookup_everywhere*/
-                ret = dict_set_uint32 (local->xattr_req,
-                                       conf->link_xattr_name, 256);
-                if (ret < 0) {
-                        gf_msg (this->name, GF_LOG_WARNING, ENOMEM,
-                                DHT_MSG_DICT_SET_FAILED,
-                                "Failed to set dictionary value:key = %s for "
-                                "path %s", conf->link_xattr_name, loc->path);
-                        goto err;
-                }
-                if (IA_ISDIR (local->inode->ia_type)) {
-                        local->call_cnt = call_cnt = conf->subvolume_cnt;
-                        for (i = 0; i < call_cnt; i++) {
-                                STACK_WIND (frame, dht_revalidate_cbk,
-                                            conf->subvolumes[i],
-                                            conf->subvolumes[i]->fops->lookup,
-                                            loc, local->xattr_req);
-                        }
-                        return 0;
-                }
+    VALIDATE_OR_GOTO(frame, out);
+    VALIDATE_OR_GOTO(frame->local, out);
 
-                call_cnt = local->call_cnt = layout->cnt;
+    local = frame->local;
 
-                /* need it for self-healing linkfiles which is
-                   'in-migration' state */
-                ret = dict_set_uint32 (local->xattr_req,
-                                       GLUSTERFS_OPEN_FD_COUNT, 4);
-                if (ret) {
-                        gf_msg (this->name, GF_LOG_WARNING, ENOMEM,
-                                DHT_MSG_DICT_SET_FAILED,
-                                "Failed to set dictionary value:key = %s for "
-                                "path %s", GLUSTERFS_OPEN_FD_COUNT, loc->path);
-                        goto err;
-                }
-                /* need it for dir self-heal */
-                dht_check_and_set_acl_xattr_req (loc->inode, local->xattr_req);
+    LOCK(&frame->lock);
+    {
+        this_call_cnt = --local->call_cnt;
+        if (op_ret < 0) {
+            if (op_errno != ENOTCONN) {
+                local->op_ret = -1;
+                local->op_errno = op_errno;
+                UNLOCK(&frame->lock);
+                gf_msg(this->name, GF_LOG_ERROR, op_errno,
+                       DHT_MSG_GET_XATTR_FAILED, "getxattr err for dir");
+                goto post_unlock;
+            }
 
-		for (i = 0; i < call_cnt; i++) {
-			subvol = layout->list[i].xlator;
+            goto unlock;
+        }
 
-                        gf_msg_debug (this->name, 0, "calling "
-                                      "revalidate lookup for %s at %s",
-                                      loc->path, subvol->name);
+        ret = dht_vgetxattr_alloc_and_fill(local, xattr, this, op_errno);
+        if (ret) {
+            UNLOCK(&frame->lock);
+            gf_msg(this->name, GF_LOG_ERROR, op_errno, DHT_MSG_DICT_SET_FAILED,
+                   "alloc or fill failure");
+            goto post_unlock;
+        }
+    }
+unlock:
+    UNLOCK(&frame->lock);
+post_unlock:
+    if (!is_last_call(this_call_cnt))
+        goto out;
 
-			STACK_WIND (frame, dht_revalidate_cbk,
-				    subvol, subvol->fops->lookup,
-				    &local->loc, local->xattr_req);
+    /* -- last call: do patch ups -- */
 
-		}
-        } else {
-        do_fresh_lookup:
-                /* TODO: remove the hard-coding */
-                ret = dict_set_uint32 (local->xattr_req,
-                                       conf->xattr_name, 4 * 4);
-                if (ret) {
-                        gf_msg (this->name, GF_LOG_WARNING, ENOMEM,
-                                DHT_MSG_DICT_SET_FAILED,
-                                "Failed to set dictionary value:key = %s for "
-                                "path %s", conf->xattr_name, loc->path);
-                        goto err;
-                }
+    if (local->op_ret == -1) {
+        goto unwind;
+    }
 
-                ret = dict_set_uint32 (local->xattr_req,
-                                       conf->link_xattr_name, 256);
-                if (ret) {
-                        gf_msg (this->name, GF_LOG_WARNING, ENOMEM,
-                                DHT_MSG_DICT_SET_FAILED,
-                                "Failed to set dictionary value:key = %s for "
-                                "path %s", conf->link_xattr_name, loc->path);
-                        goto err;
-                }
-                /* need it for self-healing linkfiles which is
-                   'in-migration' state */
-                ret = dict_set_uint32 (local->xattr_req,
-                                       GLUSTERFS_OPEN_FD_COUNT, 4);
-                if (ret) {
-                        gf_msg (this->name, GF_LOG_WARNING, ENOMEM,
-                                DHT_MSG_DICT_SET_FAILED,
-                                "Failed to set dictionary value:key = %s for "
-                                "path %s", GLUSTERFS_OPEN_FD_COUNT, loc->path);
-                        goto err;
-                }
-                /* need it for dir self-heal */
-                dht_check_and_set_acl_xattr_req (loc->inode, local->xattr_req);
+    ret = dht_vgetxattr_fill_and_set(local, &dict, this, _gf_true);
+    if (ret)
+        goto unwind;
 
-                if (!hashed_subvol) {
+    DHT_STACK_UNWIND(getxattr, frame, 0, 0, dict, xdata);
+    goto cleanup;
 
-                        gf_msg_debug (this->name, 0,
-                                      "no subvolume in layout for path=%s, "
-                                      "checking on all the subvols to see if "
-                                      "it is a directory", loc->path);
+unwind:
+    DHT_STACK_UNWIND(getxattr, frame, -1, local->op_errno, NULL, NULL);
+cleanup:
+    if (dict)
+        dict_unref(dict);
+out:
+    return 0;
+}
 
-                        call_cnt        = conf->subvolume_cnt;
-                        local->call_cnt = call_cnt;
+static int
+dht_vgetxattr_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int op_ret,
+                  int op_errno, dict_t *xattr, dict_t *xdata)
+{
+    dht_local_t *local = NULL;
+    int ret = 0;
+    dict_t *dict = NULL;
+    xlator_t *prev = NULL;
+    gf_boolean_t flag = _gf_true;
 
-                        local->layout = dht_layout_new (this,
-                                                        conf->subvolume_cnt);
-                        if (!local->layout) {
-                                op_errno = ENOMEM;
-                                goto err;
-                        }
+    local = frame->local;
+    prev = cookie;
 
-                        gf_msg_debug (this->name, 0,
-                                      "Found null hashed subvol. Calling lookup"
-                                      " on all nodes.");
+    if (op_ret < 0) {
+        local->op_ret = -1;
+        local->op_errno = op_errno;
+        gf_msg(this->name, GF_LOG_ERROR, op_errno, DHT_MSG_GET_XATTR_FAILED,
+               "vgetxattr: Subvolume %s returned -1", prev->name);
+        goto unwind;
+    }
 
-                        for (i = 0; i < call_cnt; i++) {
-                                STACK_WIND (frame, dht_lookup_dir_cbk,
-                                            conf->subvolumes[i],
-                                            conf->subvolumes[i]->fops->lookup,
-                                            &local->loc, local->xattr_req);
-                        }
-                        return 0;
-                }
+    ret = dht_vgetxattr_alloc_and_fill(local, xattr, this, op_errno);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, DHT_MSG_NO_MEMORY,
+               "Allocation or fill failure");
+        goto unwind;
+    }
+
+    flag = (local->layout->cnt > 1) ? _gf_true : _gf_false;
+
+    ret = dht_vgetxattr_fill_and_set(local, &dict, this, flag);
+    if (ret)
+        goto unwind;
+
+    DHT_STACK_UNWIND(getxattr, frame, 0, 0, dict, xdata);
+    goto cleanup;
+
+unwind:
+    DHT_STACK_UNWIND(getxattr, frame, -1, local->op_errno, NULL, NULL);
+cleanup:
+    if (dict)
+        dict_unref(dict);
+
+    return 0;
+}
 
-                gf_msg_debug (this->name, 0, "Calling fresh lookup for %s on"
-                              " %s", loc->path, hashed_subvol->name);
+static int
+dht_linkinfo_getxattr_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                          int op_ret, int op_errno, dict_t *xattr,
+                          dict_t *xdata)
+{
+    int ret = 0;
+    char *value = NULL;
 
-                STACK_WIND (frame, dht_lookup_cbk,
-                            hashed_subvol, hashed_subvol->fops->lookup,
-                            loc, local->xattr_req);
+    if (op_ret != -1) {
+        ret = dict_get_str(xattr, GF_XATTR_PATHINFO_KEY, &value);
+        if (!ret) {
+            ret = dict_set_str(xattr, GF_XATTR_LINKINFO_KEY, value);
+            if (!ret)
+                gf_msg_trace(this->name, 0, "failed to set linkinfo");
         }
+    }
 
-        return 0;
+    DHT_STACK_UNWIND(getxattr, frame, op_ret, op_errno, xattr, xdata);
 
-err:
-        op_errno = (op_errno == -1) ? errno : op_errno;
-        DHT_STACK_UNWIND (lookup, frame, -1, op_errno, NULL, NULL, NULL,
-                          NULL);
-        return 0;
+    return 0;
 }
 
+static int
+dht_mds_getxattr_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                     int op_ret, int op_errno, dict_t *xattr, dict_t *xdata)
+{
+    dht_local_t *local = NULL;
+    dht_conf_t *conf = NULL;
+
+    VALIDATE_OR_GOTO(frame, err);
+    VALIDATE_OR_GOTO(frame->local, err);
+    VALIDATE_OR_GOTO(this->private, err);
+
+    conf = this->private;
+    local = frame->local;
+
+    if (!xattr || (op_ret == -1)) {
+        local->op_ret = op_ret;
+        goto out;
+    }
+    dict_del(xattr, conf->xattr_name);
+    local->op_ret = 0;
+
+    if (!local->xattr) {
+        local->xattr = dict_copy_with_ref(xattr, NULL);
+    }
+
+out:
+    DHT_STACK_UNWIND(getxattr, frame, local->op_ret, op_errno, local->xattr,
+                     xdata);
+    return 0;
+err:
+    DHT_STACK_UNWIND(getxattr, frame, -1, EINVAL, NULL, NULL);
+    return 0;
+}
 
 int
-dht_unlink_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                int op_ret, int op_errno, struct iatt *preparent,
-                struct iatt *postparent, dict_t *xdata)
+dht_getxattr_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int op_ret,
+                 int op_errno, dict_t *xattr, dict_t *xdata)
 {
-        dht_local_t  *local = NULL;
-        call_frame_t *prev  = NULL;
+    int this_call_cnt = 0;
+    dht_local_t *local = NULL;
+    dht_conf_t *conf = NULL;
+    int ret = 0;
 
-        local = frame->local;
-        prev  = cookie;
-
-        LOCK (&frame->lock);
-        {
-                if (op_ret == -1) {
-                        local->op_ret   = -1;
-                        local->op_errno = op_errno;
-                        gf_msg_debug (this->name, 0,
-                                      "Unlink: subvolume %s returned -1 (%s)",
-                                       prev->this->name, strerror (op_errno));
-                        goto unlock;
-                }
+    VALIDATE_OR_GOTO(frame, err);
+    VALIDATE_OR_GOTO(frame->local, err);
+    VALIDATE_OR_GOTO(this->private, err);
 
-                local->op_ret = 0;
+    conf = this->private;
+    local = frame->local;
 
-                local->postparent = *postparent;
-                local->preparent = *preparent;
+    if (dht_check_remote_fd_failed_error(local, op_ret, op_errno)) {
+        ret = dht_check_and_open_fd_on_subvol(this, frame);
+        if (ret)
+            goto err;
+        return 0;
+    }
 
-                if (local->loc.parent) {
-                        dht_inode_ctx_time_update (local->loc.parent, this,
-                                                   &local->preparent, 0);
-                        dht_inode_ctx_time_update (local->loc.parent, this,
-                                                   &local->postparent, 1);
-                }
+    LOCK(&frame->lock);
+    {
+        if (!xattr || (op_ret == -1)) {
+            local->op_ret = op_ret;
+            goto unlock;
+        }
+
+        dict_del(xattr, conf->xattr_name);
+        dict_del(xattr, conf->mds_xattr_key);
+
+        dict_del(xattr, conf->commithash_xattr_name);
+
+        if (frame->root->pid >= 0) {
+            GF_REMOVE_INTERNAL_XATTR("trusted.glusterfs.quota*", xattr);
+            GF_REMOVE_INTERNAL_XATTR("trusted.pgfid*", xattr);
         }
+
+        local->op_ret = 0;
+
+        if (!local->xattr) {
+            local->xattr = dict_copy_with_ref(xattr, NULL);
+        } else {
+            dht_aggregate_xattr(local->xattr, xattr);
+        }
+
+        if (!local->xdata) {
+            local->xdata = dict_ref(xdata);
+        } else if ((local->inode && IA_ISDIR(local->inode->ia_type)) ||
+                   (local->fd && IA_ISDIR(local->fd->inode->ia_type))) {
+            dht_aggregate_xattr(local->xdata, xdata);
+        }
+    }
 unlock:
-        UNLOCK (&frame->lock);
+    UNLOCK(&frame->lock);
 
-        DHT_STACK_UNWIND (unlink, frame, local->op_ret, local->op_errno,
-                          &local->preparent, &local->postparent, NULL);
+    this_call_cnt = dht_frame_return(frame);
+    if (is_last_call(this_call_cnt)) {
+        /* If we have a valid xattr received from any one of the
+         * subvolume, let's return it */
+        if (local->xattr) {
+            local->op_ret = 0;
+        }
 
-        return 0;
+        DHT_STACK_UNWIND(getxattr, frame, local->op_ret, op_errno, local->xattr,
+                         local->xdata);
+    }
+    return 0;
+err:
+    DHT_STACK_UNWIND(getxattr, frame, -1, EINVAL, NULL, NULL);
+    return 0;
 }
 
+static int32_t
+dht_getxattr_unwind(call_frame_t *frame, int op_ret, int op_errno, dict_t *dict,
+                    dict_t *xdata)
+{
+    DHT_STACK_UNWIND(getxattr, frame, op_ret, op_errno, dict, xdata);
+    return 0;
+}
 
-int
-dht_unlink_linkfile_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                         int op_ret, int op_errno, struct iatt *preparent,
-                         struct iatt *postparent, dict_t *xdata)
+static int
+dht_getxattr_get_real_filename_cbk(call_frame_t *frame, void *cookie,
+                                   xlator_t *this, int op_ret, int op_errno,
+                                   dict_t *xattr, dict_t *xdata)
 {
-        dht_local_t  *local = NULL;
-        call_frame_t *prev = NULL;
+    int this_call_cnt = 0;
+    dht_local_t *local = NULL;
 
-        xlator_t *cached_subvol = NULL;
+    local = frame->local;
 
-        local = frame->local;
-        prev  = cookie;
-
-        LOCK (&frame->lock);
-        {
-                if ((op_ret == -1) && !((op_errno == ENOENT) ||
-                                        (op_errno == ENOTCONN))) {
-                        local->op_errno = op_errno;
-                        gf_msg_debug (this->name, 0,
-                                      "Unlink link: subvolume %s"
-                                      " returned -1 (%s)",
-                                      prev->this->name, strerror (op_errno));
-                        goto unlock;
+    LOCK(&frame->lock);
+    {
+        if (local->op_errno == EOPNOTSUPP) {
+            /* Nothing to do here, we have already found
+             * a subvol which does not have the get_real_filename
+             * optimization. If condition is for simple logic.
+             */
+            goto unlock;
+        }
+
+        if (op_ret == -1) {
+            if (op_errno == EOPNOTSUPP) {
+                /* This subvol does not have the optimization.
+                 * Better let the user know we don't support it.
+                 * Remove previous results if any.
+                 */
+
+                if (local->xattr) {
+                    dict_unref(local->xattr);
+                    local->xattr = NULL;
                 }
 
-                local->op_ret = 0;
+                if (local->xattr_req) {
+                    dict_unref(local->xattr_req);
+                    local->xattr_req = NULL;
+                }
+
+                local->op_ret = op_ret;
+                local->op_errno = op_errno;
+                UNLOCK(&frame->lock);
+                gf_msg(this->name, GF_LOG_WARNING, op_errno,
+                       DHT_MSG_UPGRADE_BRICKS,
+                       "At least "
+                       "one of the bricks does not support "
+                       "this operation. Please upgrade all "
+                       "bricks.");
+                goto post_unlock;
+            }
+
+            if (op_errno == ENOATTR) {
+                /* Do nothing, our defaults are set to this.
+                 */
+                goto unlock;
+            }
+
+            /* This is a place holder for every other error
+             * case. I am not sure of how to interpret
+             * ENOTCONN etc. As of now, choosing to ignore
+             * down subvol and return a good result(if any)
+             * from other subvol.
+             */
+            UNLOCK(&frame->lock);
+            gf_msg(this->name, GF_LOG_WARNING, op_errno,
+                   DHT_MSG_GET_XATTR_FAILED, "Failed to get real filename.");
+            goto post_unlock;
+        }
+
+        /* This subvol has the required file.
+         * There could be other subvols which have returned
+         * success already, choosing to return the latest good
+         * result.
+         */
+        if (local->xattr)
+            dict_unref(local->xattr);
+        local->xattr = dict_ref(xattr);
+
+        if (local->xattr_req) {
+            dict_unref(local->xattr_req);
+            local->xattr_req = NULL;
         }
+        if (xdata)
+            local->xattr_req = dict_ref(xdata);
+
+        local->op_ret = op_ret;
+        local->op_errno = 0;
+        UNLOCK(&frame->lock);
+        gf_msg_debug(this->name, 0, "Found a matching file.");
+        goto post_unlock;
+    }
 unlock:
-        UNLOCK (&frame->lock);
+    UNLOCK(&frame->lock);
+post_unlock:
+    this_call_cnt = dht_frame_return(frame);
+    if (is_last_call(this_call_cnt)) {
+        DHT_STACK_UNWIND(getxattr, frame, local->op_ret, local->op_errno,
+                         local->xattr, local->xattr_req);
+    }
+
+    return 0;
+}
 
-        if (local->op_ret == -1)
-                goto err;
+static int
+dht_getxattr_get_real_filename(call_frame_t *frame, xlator_t *this, loc_t *loc,
+                               const char *key, dict_t *xdata)
+{
+    dht_local_t *local = NULL;
+    int i = 0;
+    dht_layout_t *layout = NULL;
+    int cnt = 0;
+    xlator_t *subvol = NULL;
 
-        cached_subvol = dht_subvol_get_cached (this, local->loc.inode);
-        if (!cached_subvol) {
-                gf_msg_debug (this->name, 0,
-                              "no cached subvolume for path=%s",
-                              local->loc.path);
-                local->op_errno = EINVAL;
-                goto err;
-        }
+    local = frame->local;
+    layout = local->layout;
 
-        STACK_WIND (frame, dht_unlink_cbk,
-                    cached_subvol, cached_subvol->fops->unlink,
-                    &local->loc, local->flags, NULL);
+    cnt = local->call_cnt = layout->cnt;
 
-        return 0;
+    local->op_ret = -1;
+    local->op_errno = ENOATTR;
 
-err:
-        DHT_STACK_UNWIND (unlink, frame, -1, local->op_errno,
-                          NULL, NULL, NULL);
-        return 0;
+    for (i = 0; i < cnt; i++) {
+        subvol = layout->list[i].xlator;
+        STACK_WIND(frame, dht_getxattr_get_real_filename_cbk, subvol,
+                   subvol->fops->getxattr, loc, key, xdata);
+    }
+
+    return 0;
 }
 
-int
-dht_err_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-             int op_ret, int op_errno, dict_t *xdata)
+static int
+dht_marker_populate_args(call_frame_t *frame, int type, int *gauge,
+                         xlator_t **subvols)
 {
-        dht_local_t  *local = NULL;
-        int           this_call_cnt = 0;
-        call_frame_t *prev = NULL;
+    dht_local_t *local = NULL;
+    int i = 0;
+    dht_layout_t *layout = NULL;
 
-        local = frame->local;
-        prev = cookie;
-
-        LOCK (&frame->lock);
-        {
-                if (op_ret == -1) {
-                        local->op_errno = op_errno;
-                        gf_msg_debug (this->name, 0,
-                                      "subvolume %s returned -1 (%s)",
-                                      prev->this->name, strerror (op_errno));
-                        goto unlock;
-                }
+    local = frame->local;
+    layout = local->layout;
 
-                local->op_ret = 0;
-        }
-unlock:
-        UNLOCK (&frame->lock);
+    for (i = 0; i < layout->cnt; i++)
+        subvols[i] = layout->list[i].xlator;
 
-        this_call_cnt = dht_frame_return (frame);
-        if (is_last_call (this_call_cnt)) {
-                DHT_STACK_UNWIND (setxattr, frame, local->op_ret,
-                                  local->op_errno, NULL);
-        }
+    return layout->cnt;
+}
 
-        return 0;
+static int
+dht_is_debug_xattr_key(const char **array, char *key)
+{
+    int i = 0;
+
+    for (i = 0; array[i]; i++) {
+        if (fnmatch(array[i], key, FNM_NOESCAPE) == 0)
+            return i;
+    }
+
+    return -1;
 }
 
-static void
-fill_layout_info (dht_layout_t *layout, char *buf)
+/* Note we already have frame->local initialised here*/
+
+static int
+dht_handle_debug_getxattr(call_frame_t *frame, xlator_t *this, loc_t *loc,
+                          const char *key)
 {
-        int i = 0;
-        char tmp_buf[128] = {0,};
+    dht_local_t *local = NULL;
+    int ret = -1;
+    int op_errno = ENODATA;
+    char *value = NULL;
+    loc_t file_loc = {0};
+    const char *name = NULL;
+
+    local = frame->local;
+
+    if (dht_is_debug_xattr_key(dht_dbg_vxattrs, (char *)key) == -1) {
+        goto out;
+    }
+
+    local->xattr = dict_new();
+    if (!local->xattr) {
+        op_errno = ENOMEM;
+        goto out;
+    }
+
+    if (strncmp(key, DHT_DBG_HASHED_SUBVOL_KEY,
+                SLEN(DHT_DBG_HASHED_SUBVOL_KEY)) == 0) {
+        name = key + strlen(DHT_DBG_HASHED_SUBVOL_KEY);
+        if (strlen(name) == 0) {
+            op_errno = EINVAL;
+            goto out;
+        }
+
+        ret = dht_build_child_loc(this, &file_loc, loc, (char *)name);
+        if (ret) {
+            op_errno = ENOMEM;
+            goto out;
+        }
 
-        for (i = 0; i < layout->cnt; i++) {
-                snprintf (tmp_buf, 128, "(%s %u %u)",
-                          layout->list[i].xlator->name,
-                          layout->list[i].start,
-                          layout->list[i].stop);
-                if (i)
-                        strcat (buf, " ");
-                strcat (buf, tmp_buf);
+        local->hashed_subvol = dht_subvol_get_hashed(this, &file_loc);
+        if (local->hashed_subvol == NULL) {
+            op_errno = ENODATA;
+            goto out;
         }
+
+        value = gf_strdup(local->hashed_subvol->name);
+        if (!value) {
+            op_errno = ENOMEM;
+            goto out;
+        }
+
+        ret = dict_set_dynstr(local->xattr, (char *)key, value);
+        if (ret < 0) {
+            op_errno = -ret;
+            ret = -1;
+            goto out;
+        }
+        ret = 0;
+        goto out;
+    }
+
+out:
+    loc_wipe(&file_loc);
+    DHT_STACK_UNWIND(getxattr, frame, ret, op_errno, local->xattr, NULL);
+    return 0;
 }
 
-void
-dht_fill_pathinfo_xattr (xlator_t *this, dht_local_t *local,
-                         char *xattr_buf, int32_t alloc_len,
-                         int flag, char *layout_buf)
-{
-        if (flag && local->xattr_val)
-                snprintf (xattr_buf, alloc_len,
-                          "((<"DHT_PATHINFO_HEADER"%s> %s) (%s-layout %s))",
-                          this->name, local->xattr_val, this->name,
-                          layout_buf);
-        else if (local->xattr_val)
-                snprintf (xattr_buf, alloc_len,
-                          "(<"DHT_PATHINFO_HEADER"%s> %s)",
-                          this->name, local->xattr_val);
-        else if (flag)
-                snprintf (xattr_buf, alloc_len, "(%s-layout %s)",
-                          this->name, layout_buf);
+/* Virtual Xattr which returns 1 if all subvols are up,
+   else returns 0. Geo-rep then uses this virtual xattr
+   after a fresh mount and starts the I/O.
+*/
+
+enum dht_vxattr_subvol {
+    DHT_VXATTR_SUBVOLS_UP = 1,
+    DHT_VXATTR_SUBVOLS_DOWN = 0,
+};
+
+int
+dht_vgetxattr_subvol_status(call_frame_t *frame, xlator_t *this,
+                            const char *key)
+{
+    dht_local_t *local = NULL;
+    int ret = -1;
+    int op_errno = ENODATA;
+    int value = DHT_VXATTR_SUBVOLS_UP;
+    int i = 0;
+    dht_conf_t *conf = NULL;
+
+    conf = this->private;
+    local = frame->local;
+
+    if (!key) {
+        op_errno = EINVAL;
+        goto out;
+    }
+    local->xattr = dict_new();
+    if (!local->xattr) {
+        op_errno = ENOMEM;
+        goto out;
+    }
+    for (i = 0; i < conf->subvolume_cnt; i++) {
+        if (!conf->subvolume_status[i]) {
+            value = DHT_VXATTR_SUBVOLS_DOWN;
+            gf_msg_debug(this->name, 0, "subvol %s is down ",
+                         conf->subvolumes[i]->name);
+            break;
+        }
+    }
+    ret = dict_set_int8(local->xattr, (char *)key, value);
+    if (ret < 0) {
+        op_errno = -ret;
+        ret = -1;
+        goto out;
+    }
+    ret = 0;
+
+out:
+    DHT_STACK_UNWIND(getxattr, frame, ret, op_errno, local->xattr, NULL);
+    return 0;
 }
 
 int
-dht_vgetxattr_alloc_and_fill (dht_local_t *local, dict_t *xattr, xlator_t *this,
-                              int op_errno)
+dht_getxattr(call_frame_t *frame, xlator_t *this, loc_t *loc, const char *key,
+             dict_t *xdata)
+#define DHT_IS_DIR(layout) (layout->cnt > 1)
 {
-        int      ret       = -1;
-        char    *value     = NULL;
-        int32_t  plen      = 0;
+    xlator_t *subvol = NULL;
+    xlator_t *hashed_subvol = NULL;
+    xlator_t *mds_subvol = NULL;
+    xlator_t *cached_subvol = NULL;
+    dht_conf_t *conf = NULL;
+    dht_local_t *local = NULL;
+    dht_layout_t *layout = NULL;
+    int op_errno = -1;
+    int i = 0;
+    int cnt = 0;
+    char *node_uuid_key = NULL;
+    int ret = -1;
+
+    GF_CHECK_XATTR_KEY_AND_GOTO(key, IO_THREADS_QUEUE_SIZE_KEY, op_errno, err);
+    VALIDATE_OR_GOTO(frame, err);
+    VALIDATE_OR_GOTO(this, err);
+    VALIDATE_OR_GOTO(loc, err);
+    VALIDATE_OR_GOTO(loc->inode, err);
+    VALIDATE_OR_GOTO(this->private, err);
+
+    conf = this->private;
+
+    local = dht_local_init(frame, loc, NULL, GF_FOP_GETXATTR);
+    if (!local) {
+        op_errno = ENOMEM;
+
+        goto err;
+    }
+
+    layout = local->layout;
+    if (!layout) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, DHT_MSG_LAYOUT_NULL,
+               "Layout is NULL");
+        op_errno = ENOENT;
+        goto err;
+    }
+
+    /* skip over code which is irrelevant without a valid key */
+    if (!key)
+        goto no_key;
+
+    local->key = gf_strdup(key);
+    if (!local->key) {
+        op_errno = ENOMEM;
+        goto err;
+    }
+
+    if (strncmp(key, conf->mds_xattr_key, strlen(key)) == 0) {
+        op_errno = ENOTSUP;
+        goto err;
+    }
+
+    if (strncmp(key, DHT_SUBVOL_STATUS_KEY, SLEN(DHT_SUBVOL_STATUS_KEY)) == 0) {
+        dht_vgetxattr_subvol_status(frame, this, key);
+        return 0;
+    }
 
-        ret = dict_get_str (xattr, local->xsel, &value);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "Subvolume %s returned -1 (%s)", this->name,
-                        strerror (op_errno));
-                local->op_ret = -1;
-                local->op_errno = op_errno;
-                goto out;
+    /* skip over code which is irrelevant if !DHT_IS_DIR(layout) */
+    if (!DHT_IS_DIR(layout))
+        goto no_dht_is_dir;
+
+    if ((strncmp(key, GF_XATTR_GET_REAL_FILENAME_KEY,
+                 SLEN(GF_XATTR_GET_REAL_FILENAME_KEY)) == 0) &&
+        DHT_IS_DIR(layout)) {
+        dht_getxattr_get_real_filename(frame, this, loc, key, xdata);
+        return 0;
+    }
+
+    if (!strcmp(key, GF_REBAL_FIND_LOCAL_SUBVOL)) {
+        ret = gf_asprintf(&node_uuid_key, "%s", GF_XATTR_LIST_NODE_UUIDS_KEY);
+        if (ret == -1 || !node_uuid_key) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, DHT_MSG_NO_MEMORY,
+                   "Failed to copy node uuid key");
+            op_errno = ENOMEM;
+            goto err;
+        }
+        (void)snprintf(local->xsel, sizeof(local->xsel), "%s", node_uuid_key);
+        cnt = local->call_cnt = conf->subvolume_cnt;
+        for (i = 0; i < cnt; i++) {
+            STACK_WIND_COOKIE(frame, dht_find_local_subvol_cbk,
+                              conf->subvolumes[i], conf->subvolumes[i],
+                              conf->subvolumes[i]->fops->getxattr, loc,
+                              node_uuid_key, xdata);
         }
+        if (node_uuid_key)
+            GF_FREE(node_uuid_key);
+        return 0;
+    }
+
+    if (!strcmp(key, GF_REBAL_OLD_FIND_LOCAL_SUBVOL)) {
+        ret = gf_asprintf(&node_uuid_key, "%s", GF_XATTR_NODE_UUID_KEY);
+        if (ret == -1 || !node_uuid_key) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, DHT_MSG_NO_MEMORY,
+                   "Failed to copy node uuid key");
+            op_errno = ENOMEM;
+            goto err;
+        }
+        (void)snprintf(local->xsel, sizeof(local->xsel), "%s", node_uuid_key);
+        cnt = local->call_cnt = conf->subvolume_cnt;
+        for (i = 0; i < cnt; i++) {
+            STACK_WIND_COOKIE(frame, dht_find_local_subvol_cbk,
+                              conf->subvolumes[i], conf->subvolumes[i],
+                              conf->subvolumes[i]->fops->getxattr, loc,
+                              node_uuid_key, xdata);
+        }
+        if (node_uuid_key)
+            GF_FREE(node_uuid_key);
+        return 0;
+    }
+
+    /* for file use cached subvolume (obviously!): see if {}
+     * below
+     * for directory:
+     *  wind to all subvolumes and exclude subvolumes which
+     *  return ENOTCONN (in callback)
+     *
+     * NOTE: Don't trust inode here, as that may not be valid
+     *       (until inode_link() happens)
+     */
+
+    if (XATTR_IS_PATHINFO(key) || (strcmp(key, GF_XATTR_NODE_UUID_KEY) == 0) ||
+        (strcmp(key, GF_XATTR_LIST_NODE_UUIDS_KEY) == 0)) {
+        (void)snprintf(local->xsel, sizeof(local->xsel), "%s", key);
+        cnt = local->call_cnt = layout->cnt;
+        for (i = 0; i < cnt; i++) {
+            subvol = layout->list[i].xlator;
+            STACK_WIND(frame, dht_vgetxattr_dir_cbk, subvol,
+                       subvol->fops->getxattr, loc, key, xdata);
+        }
+        return 0;
+    }
 
-        local->alloc_len += strlen(value);
+no_dht_is_dir:
+    /* node-uuid or pathinfo for files */
+    if (XATTR_IS_PATHINFO(key) || (strcmp(key, GF_XATTR_NODE_UUID_KEY) == 0)) {
+        cached_subvol = local->cached_subvol;
+        (void)snprintf(local->xsel, sizeof(local->xsel), "%s", key);
+        local->call_cnt = 1;
+        STACK_WIND_COOKIE(frame, dht_vgetxattr_cbk, cached_subvol,
+                          cached_subvol, cached_subvol->fops->getxattr, loc,
+                          key, xdata);
 
-        if (!local->xattr_val) {
-                local->alloc_len += (strlen (DHT_PATHINFO_HEADER) + 10);
-                local->xattr_val = GF_CALLOC (local->alloc_len, sizeof (char),
-                                              gf_common_mt_char);
-                if (!local->xattr_val) {
-                        ret = -1;
-                        goto out;
-                }
+        return 0;
+    }
+
+    if (strcmp(key, GF_XATTR_LINKINFO_KEY) == 0) {
+        hashed_subvol = dht_subvol_get_hashed(this, loc);
+        if (!hashed_subvol) {
+            gf_msg(this->name, GF_LOG_ERROR, 0,
+                   DHT_MSG_HASHED_SUBVOL_GET_FAILED,
+                   "Failed to get hashed subvol for %s", loc->path);
+            op_errno = EINVAL;
+            goto err;
         }
 
-        if (local->xattr_val) {
-                plen = strlen (local->xattr_val);
-                if (plen) {
-                        /* extra byte(s) for \0 to be safe */
-                        local->alloc_len += (plen + 2);
-                        local->xattr_val = GF_REALLOC (local->xattr_val,
-                                                       local->alloc_len);
-                        if (!local->xattr_val) {
-                                ret = -1;
-                                goto out;
-                        }
-                }
+        cached_subvol = dht_subvol_get_cached(this, loc->inode);
+        if (!cached_subvol) {
+            gf_msg(this->name, GF_LOG_ERROR, 0,
+                   DHT_MSG_CACHED_SUBVOL_GET_FAILED,
+                   "Failed to get cached subvol for %s", loc->path);
+            op_errno = EINVAL;
+            goto err;
+        }
 
-                (void) strcat (local->xattr_val, value);
-                (void) strcat (local->xattr_val, " ");
-                local->op_ret = 0;
+        if (hashed_subvol == cached_subvol) {
+            op_errno = ENODATA;
+            goto err;
         }
 
-        ret = 0;
+        STACK_WIND(frame, dht_linkinfo_getxattr_cbk, hashed_subvol,
+                   hashed_subvol->fops->getxattr, loc, GF_XATTR_PATHINFO_KEY,
+                   xdata);
+        return 0;
+    }
 
- out:
-        return ret;
+    if (dht_is_debug_xattr_key(dht_dbg_vxattrs, (char *)key) >= 0) {
+        dht_handle_debug_getxattr(frame, this, loc, key);
+        return 0;
+    }
+
+no_key:
+    if (cluster_handle_marker_getxattr(frame, loc, key, conf->vol_uuid,
+                                       dht_getxattr_unwind,
+                                       dht_marker_populate_args) == 0)
+        return 0;
+
+    if (DHT_IS_DIR(layout)) {
+        local->call_cnt = conf->subvolume_cnt;
+        cnt = conf->subvolume_cnt;
+        ret = dht_inode_ctx_mdsvol_get(loc->inode, this, &mds_subvol);
+        if (!mds_subvol) {
+            gf_msg(this->name, GF_LOG_INFO, 0, DHT_MSG_HASHED_SUBVOL_GET_FAILED,
+                   "Cannot determine MDS, fetching xattr %s randomly"
+                   " from a subvol for path %s ",
+                   key, loc->path);
+        } else {
+            /* TODO need to handle it, As of now we are
+               choosing availability instead of chossing
+               consistencty, in case of mds_subvol is
+               down winding a getxattr call on other subvol
+               and return xattr
+            */
+            local->mds_subvol = mds_subvol;
+            for (i = 0; i < cnt; i++) {
+                if (conf->subvolumes[i] == mds_subvol) {
+                    if (!conf->subvolume_status[i]) {
+                        gf_msg(this->name, GF_LOG_INFO, 0,
+                               DHT_MSG_HASHED_SUBVOL_DOWN,
+                               "MDS %s is down for path"
+                               " path %s so fetching xattr "
+                               "%s randomly from a subvol ",
+                               local->mds_subvol->name, loc->path, key);
+                        ret = 1;
+                    }
+                }
+            }
+        }
+
+        if (!ret && key && local->mds_subvol && dht_match_xattr(key)) {
+            STACK_WIND(frame, dht_mds_getxattr_cbk, local->mds_subvol,
+                       local->mds_subvol->fops->getxattr, loc, key, xdata);
+
+            return 0;
+        }
+    } else {
+        cnt = local->call_cnt = 1;
+    }
+
+    for (i = 0; i < cnt; i++) {
+        subvol = layout->list[i].xlator;
+        STACK_WIND(frame, dht_getxattr_cbk, subvol, subvol->fops->getxattr, loc,
+                   key, xdata);
+    }
+    return 0;
+
+err:
+    op_errno = (op_errno == -1) ? errno : op_errno;
+    DHT_STACK_UNWIND(getxattr, frame, -1, op_errno, NULL, NULL);
+
+    return 0;
 }
+#undef DHT_IS_DIR
 
 int
-dht_vgetxattr_fill_and_set (dht_local_t *local, dict_t **dict, xlator_t *this,
-                            gf_boolean_t flag)
+dht_fgetxattr(call_frame_t *frame, xlator_t *this, fd_t *fd, const char *key,
+              dict_t *xdata)
 {
-        int   ret             = -1;
-        char *xattr_buf       = NULL;
-        char layout_buf[8192] = {0,};
+    xlator_t *subvol = NULL;
+    dht_local_t *local = NULL;
+    dht_layout_t *layout = NULL;
+    int op_errno = -1;
+    int i = 0;
+    int cnt = 0;
+    xlator_t *mds_subvol = NULL;
+    int ret = -1;
+    dht_conf_t *conf = NULL;
+    char gfid[GF_UUID_BUF_SIZE] = {0};
+
+    VALIDATE_OR_GOTO(frame, err);
+    VALIDATE_OR_GOTO(this, err);
+    VALIDATE_OR_GOTO(fd, err);
+    VALIDATE_OR_GOTO(fd->inode, err);
+    VALIDATE_OR_GOTO(this->private, err);
+
+    conf = this->private;
+
+    local = dht_local_init(frame, NULL, fd, GF_FOP_FGETXATTR);
+    if (!local) {
+        op_errno = ENOMEM;
+
+        goto err;
+    }
+
+    layout = local->layout;
+    if (!layout) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, DHT_MSG_LAYOUT_NULL,
+               "Layout is NULL");
+        op_errno = ENOENT;
+        goto err;
+    }
+
+    if (key) {
+        local->key = gf_strdup(key);
+        if (!local->key) {
+            op_errno = ENOMEM;
+            goto err;
+        }
+    }
+
+    gf_uuid_unparse(fd->inode->gfid, gfid);
+
+    if ((fd->inode->ia_type == IA_IFDIR) && key &&
+        (strncmp(key, GF_XATTR_LOCKINFO_KEY, SLEN(GF_XATTR_LOCKINFO_KEY)) !=
+         0)) {
+        local->call_cnt = conf->subvolume_cnt;
+        cnt = conf->subvolume_cnt;
+        ret = dht_inode_ctx_mdsvol_get(fd->inode, this, &mds_subvol);
+
+        if (!mds_subvol) {
+            gf_msg(this->name, GF_LOG_ERROR, 0,
+                   DHT_MSG_HASHED_SUBVOL_GET_FAILED,
+                   "cannot determine MDS, fetching xattr %s "
+                   " randomly from a subvol for gfid %s ",
+                   key, gfid);
+        } else {
+            /* TODO need to handle it, As of now we are
+               choosing availability instead of chossing
+               consistencty, in case of hashed_subvol is
+               down winding a getxattr call on other subvol
+               and return xattr
+            */
+            local->mds_subvol = mds_subvol;
+            for (i = 0; i < cnt; i++) {
+                if (conf->subvolumes[i] == mds_subvol) {
+                    if (!conf->subvolume_status[i]) {
+                        gf_msg(this->name, GF_LOG_WARNING, 0,
+                               DHT_MSG_HASHED_SUBVOL_DOWN,
+                               "MDS subvolume %s is down"
+                               " for gfid %s so fetching xattr "
+                               " %s randomly from a subvol ",
+                               local->mds_subvol->name, gfid, key);
+                        ret = 1;
+                    }
+                }
+            }
+        }
+
+        if (!ret && key && local->mds_subvol && dht_match_xattr(key)) {
+            STACK_WIND(frame, dht_mds_getxattr_cbk, local->mds_subvol,
+                       local->mds_subvol->fops->fgetxattr, fd, key, NULL);
+
+            return 0;
+        }
+
+    } else {
+        cnt = local->call_cnt = 1;
+    }
+
+    for (i = 0; i < cnt; i++) {
+        subvol = layout->list[i].xlator;
+        STACK_WIND(frame, dht_getxattr_cbk, subvol, subvol->fops->fgetxattr, fd,
+                   key, NULL);
+    }
+    return 0;
+
+err:
+    op_errno = (op_errno == -1) ? errno : op_errno;
+    DHT_STACK_UNWIND(fgetxattr, frame, -1, op_errno, NULL, NULL);
 
-        if (flag)
-                fill_layout_info (local->layout, layout_buf);
+    return 0;
+}
 
-        *dict = dict_new ();
-        if (!*dict)
-                goto out;
+static int
+dht_setxattr2(xlator_t *this, xlator_t *subvol, call_frame_t *frame, int ret)
+{
+    dht_local_t *local = NULL;
+    int op_errno = EINVAL;
 
-        local->xattr_val[strlen (local->xattr_val) - 1] = '\0';
+    if (!frame || !frame->local)
+        goto err;
 
-        /* we would need max this many bytes to create xattr string
-         * extra 40 bytes is just an estimated amount of additional
-         * space required as we include translator name and some
-         * spaces, brackets etc. when forming the pathinfo string.
-         *
-         * For node-uuid we just don't have all the pretty formatting,
-         * but since this is a generic routine for pathinfo & node-uuid
-         * we dont have conditional space allocation and try to be
-         * generic
+    local = frame->local;
+    op_errno = local->op_errno;
+
+    if (we_are_not_migrating(ret)) {
+        /* This dht xlator is not migrating the file. Unwind and
+         * pass on the original mode bits so the higher DHT layer
+         * can handle this.
          */
-        local->alloc_len += (2 * strlen (this->name))
-                + strlen (layout_buf)
-                + 40;
-        xattr_buf = GF_CALLOC (local->alloc_len, sizeof (char),
-                               gf_common_mt_char);
-        if (!xattr_buf)
-                goto out;
+        DHT_STACK_UNWIND(setxattr, frame, local->op_ret, local->op_errno,
+                         local->rebalance.xdata);
+        return 0;
+    }
 
-        if (XATTR_IS_PATHINFO (local->xsel)) {
-                (void) dht_fill_pathinfo_xattr (this, local, xattr_buf,
-                                                local->alloc_len, flag,
-                                                layout_buf);
-        } else if (XATTR_IS_NODE_UUID (local->xsel)) {
-                (void) snprintf (xattr_buf, local->alloc_len, "%s",
-                                 local->xattr_val);
-        } else {
-                gf_log (this->name, GF_LOG_WARNING,
-                        "Unknown local->xsel (%s)", local->xsel);
-                GF_FREE (xattr_buf);
-                goto out;
-        }
+    if (subvol == NULL)
+        goto err;
 
-        ret = dict_set_dynstr (*dict, local->xsel, xattr_buf);
-        if (ret)
-                GF_FREE (xattr_buf);
-        GF_FREE (local->xattr_val);
+    local->call_cnt = 2; /* This is the second attempt */
 
- out:
-        return ret;
+    if (local->fop == GF_FOP_SETXATTR) {
+        STACK_WIND_COOKIE(frame, dht_file_setxattr_cbk, subvol, subvol,
+                          subvol->fops->setxattr, &local->loc,
+                          local->rebalance.xattr, local->rebalance.flags,
+                          local->xattr_req);
+    } else {
+        STACK_WIND_COOKIE(frame, dht_file_setxattr_cbk, subvol, subvol,
+                          subvol->fops->fsetxattr, local->fd,
+                          local->rebalance.xattr, local->rebalance.flags,
+                          local->xattr_req);
+    }
+
+    return 0;
+
+err:
+    DHT_STACK_UNWIND(setxattr, frame, (local ? local->op_ret : -1), op_errno,
+                     NULL);
+    return 0;
 }
 
 int
-dht_vgetxattr_dir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                       int op_ret, int op_errno, dict_t *xattr, dict_t *xdata)
+dht_file_setxattr_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                      int op_ret, int op_errno, dict_t *xdata)
 {
-        int          ret           = 0;
-        dht_local_t *local         = NULL;
-        int          this_call_cnt = 0;
-        dict_t      *dict          = NULL;
+    int ret = -1;
+    dht_local_t *local = NULL;
+    xlator_t *prev = NULL;
+    struct iatt *stbuf = NULL;
+    inode_t *inode = NULL;
+    xlator_t *subvol1 = NULL, *subvol2 = NULL;
 
-        VALIDATE_OR_GOTO (frame, out);
-        VALIDATE_OR_GOTO (frame->local, out);
+    local = frame->local;
+    prev = cookie;
 
-        local = frame->local;
+    local->op_errno = op_errno;
 
-        LOCK (&frame->lock);
-        {
-                this_call_cnt = --local->call_cnt;
-                if (op_ret < 0) {
-                        if (op_errno != ENOTCONN) {
-                                gf_log (this->name, GF_LOG_ERROR,
-                                        "getxattr err (%s) for dir",
-                                        strerror (op_errno));
-				local->op_ret = -1;
-				local->op_errno = op_errno;
-                        }
+    if ((local->fop == GF_FOP_FSETXATTR) &&
+        dht_check_remote_fd_failed_error(local, op_ret, op_errno)) {
+        ret = dht_check_and_open_fd_on_subvol(this, frame);
+        if (ret)
+            goto out;
+        return 0;
+    }
 
-                        goto unlock;
-                }
+    if ((op_ret == -1) && !dht_inode_missing(op_errno)) {
+        gf_msg_debug(this->name, op_errno, "subvolume %s returned -1.",
+                     prev->name);
+        goto out;
+    }
 
-                ret = dht_vgetxattr_alloc_and_fill (local, xattr, this,
-                                                    op_errno);
-                if (ret)
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "alloc or fill failure");
-        }
- unlock:
-        UNLOCK (&frame->lock);
+    if (local->call_cnt != 1)
+        goto out;
 
-        if (!is_last_call (this_call_cnt))
-                goto out;
+    ret = dict_get_bin(xdata, DHT_IATT_IN_XDATA_KEY, (void **)&stbuf);
 
-        /* -- last call: do patch ups -- */
+    if ((!op_ret) && !stbuf) {
+        goto out;
+    }
 
-        if (local->op_ret == -1) {
-                goto unwind;
+    local->op_ret = op_ret;
+    local->rebalance.target_op_fn = dht_setxattr2;
+    if (xdata)
+        local->rebalance.xdata = dict_ref(xdata);
+
+    /* Phase 2 of migration */
+    if ((op_ret == -1) || IS_DHT_MIGRATION_PHASE2(stbuf)) {
+        ret = dht_rebalance_complete_check(this, frame);
+        if (!ret)
+            return 0;
+    }
+
+    /* Phase 1 of migration */
+    if (IS_DHT_MIGRATION_PHASE1(stbuf)) {
+        inode = (local->fd) ? local->fd->inode : local->loc.inode;
+
+        ret = dht_inode_ctx_get_mig_info(this, inode, &subvol1, &subvol2);
+        if (!dht_mig_info_is_invalid(local->cached_subvol, subvol1, subvol2)) {
+            dht_setxattr2(this, subvol2, frame, 0);
+            return 0;
         }
 
-        ret = dht_vgetxattr_fill_and_set (local, &dict, this, _gf_true);
-        if (ret)
-                goto unwind;
+        ret = dht_rebalance_in_progress_check(this, frame);
+        if (!ret)
+            return 0;
+    }
 
-        DHT_STACK_UNWIND (getxattr, frame, 0, 0, dict, xdata);
-        goto cleanup;
+out:
 
- unwind:
-        DHT_STACK_UNWIND (getxattr, frame, -1, local->op_errno, NULL, NULL);
- cleanup:
-        if (dict)
-                dict_unref (dict);
- out:
-        return 0;
+    if (local->fop == GF_FOP_SETXATTR) {
+        DHT_STACK_UNWIND(setxattr, frame, op_ret, op_errno, xdata);
+    } else {
+        DHT_STACK_UNWIND(fsetxattr, frame, op_ret, op_errno, xdata);
+    }
+
+    return 0;
 }
 
-int
-dht_vgetxattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                   int op_ret, int op_errno, dict_t *xattr, dict_t *xdata)
+/* Function is call by dict_foreach_fnmatch if key is match with
+   user.* and set boolean flag to true
+*/
+static int
+dht_is_user_xattr(dict_t *this, char *key, data_t *value, void *data)
 {
-        dht_local_t  *local         = NULL;
-        int           ret           = 0;
-        dict_t       *dict          = NULL;
-        call_frame_t *prev          = NULL;
-        gf_boolean_t  flag          = _gf_true;
+    gf_boolean_t *user_xattr_found = data;
+    *user_xattr_found = _gf_true;
+    return 0;
+}
 
-        local = frame->local;
-        prev = cookie;
+/* Common code to wind a (f)(set|remove)xattr call to set xattr on directory
+ */
+static int
+dht_dir_common_set_remove_xattr(call_frame_t *frame, xlator_t *this, loc_t *loc,
+                                fd_t *fd, dict_t *xattr, int flags,
+                                dict_t *xdata, int *op_errno)
 
-        if (op_ret < 0) {
-                local->op_ret = -1;
-                local->op_errno = op_errno;
-                gf_log (this->name, GF_LOG_ERROR,
-                        "vgetxattr: Subvolume %s returned -1 (%s)",
-                         prev->this->name, strerror (op_errno));
-                goto unwind;
+{
+    dict_t *xattrop = NULL;
+    int32_t subone[1] = {-1};
+    gf_boolean_t uxattr_key_found = _gf_false;
+    xlator_t *mds_subvol = NULL;
+    xlator_t *travvol = NULL;
+    dht_conf_t *conf = NULL;
+    int ret = -1;
+    int i = 0;
+    int call_cnt = 0;
+    dht_local_t *local = NULL;
+    char gfid_local[GF_UUID_BUF_SIZE] = {0};
+    char **xattrs_to_heal;
+
+    conf = this->private;
+    local = frame->local;
+    call_cnt = conf->subvolume_cnt;
+    local->flags = flags;
+    xattrs_to_heal = get_xattrs_to_heal();
+
+    if (!gf_uuid_is_null(local->gfid)) {
+        gf_uuid_unparse(local->gfid, gfid_local);
+    }
+
+    if ((local->fop == GF_FOP_SETXATTR) || (local->fop == GF_FOP_FSETXATTR)) {
+        /* Check if any user xattr present in xattr
+         */
+        dict_foreach_fnmatch(xattr, "user*", dht_is_user_xattr,
+                             &uxattr_key_found);
+
+        /* Check if any custom key xattr present in dict xattr
+           and start index from 1 because user xattr already
+           checked in previous line
+        */
+        for (i = 1; xattrs_to_heal[i]; i++)
+            if (dict_get(xattr, xattrs_to_heal[i]))
+                uxattr_key_found = _gf_true;
+    }
+
+    if ((local->fop == GF_FOP_REMOVEXATTR) ||
+        (local->fop == GF_FOP_FREMOVEXATTR)) {
+        /* Check if any custom key xattr present in local->key
+         */
+        for (i = 0; xattrs_to_heal[i]; i++)
+            if (strstr(local->key, xattrs_to_heal[i]))
+                uxattr_key_found = _gf_true;
+    }
+
+    /* If there is no custom key xattr present or gfid is root
+       or call_cnt is 1 then wind a (f)setxattr call on all subvols
+    */
+    if (!uxattr_key_found || __is_root_gfid(local->gfid) || call_cnt == 1) {
+        for (i = 0; i < conf->subvolume_cnt; i++) {
+            travvol = conf->subvolumes[i];
+            if ((local->fop == GF_FOP_SETXATTR) ||
+                (local->fop == GF_FOP_FSETXATTR)) {
+                if (fd) {
+                    STACK_WIND_COOKIE(frame, dht_err_cbk, travvol, travvol,
+                                      travvol->fops->fsetxattr, fd, xattr,
+                                      flags, xdata);
+                } else {
+                    STACK_WIND_COOKIE(frame, dht_err_cbk, travvol, travvol,
+                                      travvol->fops->setxattr, loc, xattr,
+                                      flags, xdata);
+                }
+            }
+
+            if ((local->fop == GF_FOP_REMOVEXATTR) ||
+                (local->fop == GF_FOP_FREMOVEXATTR)) {
+                if (fd) {
+                    STACK_WIND_COOKIE(frame, dht_err_cbk, travvol, travvol,
+                                      travvol->fops->fremovexattr, fd,
+                                      local->key, local->xattr_req);
+                } else {
+                    STACK_WIND_COOKIE(frame, dht_err_cbk, travvol, travvol,
+                                      travvol->fops->removexattr, loc,
+                                      local->key, local->xattr_req);
+                }
+            }
         }
 
-        ret = dht_vgetxattr_alloc_and_fill (local, xattr, this,
-                                            op_errno);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        DHT_MSG_NO_MEMORY,
-                        "Allocation or fill failure");
-                goto unwind;
+        return 0;
+    }
+
+    /* Calculate hash subvol based on inode and parent inode
+     */
+    if (fd) {
+        ret = dht_inode_ctx_mdsvol_get(fd->inode, this, &mds_subvol);
+    } else {
+        ret = dht_inode_ctx_mdsvol_get(loc->inode, this, &mds_subvol);
+    }
+    if (ret || !mds_subvol) {
+        if (fd) {
+            gf_msg(this->name, GF_LOG_ERROR, 0,
+                   DHT_MSG_HASHED_SUBVOL_GET_FAILED,
+                   "Failed to get mds subvol for fd %p"
+                   "gfid is %s ",
+                   fd, gfid_local);
+        } else {
+            gf_msg(this->name, GF_LOG_ERROR, 0,
+                   DHT_MSG_HASHED_SUBVOL_GET_FAILED,
+                   "%s: Failed to get mds subvol. (gfid is %s)", loc->path,
+                   gfid_local);
+        }
+        (*op_errno) = ENOENT;
+        goto err;
+    }
+
+    local->mds_subvol = mds_subvol;
+
+    for (i = 0; i < conf->subvolume_cnt; i++) {
+        if (conf->subvolumes[i] == mds_subvol) {
+            if (!conf->subvolume_status[i]) {
+                gf_msg(this->name, GF_LOG_WARNING, 0,
+                       DHT_MSG_HASHED_SUBVOL_DOWN,
+                       "MDS subvol is down for path "
+                       " %s gfid is %s Unable to set xattr ",
+                       local->loc.path, gfid_local);
+                (*op_errno) = ENOTCONN;
+                goto err;
+            }
+        }
+    }
+
+    if (uxattr_key_found) {
+        xattrop = dict_new();
+        if (!xattrop) {
+            gf_msg(this->name, GF_LOG_ERROR, DHT_MSG_NO_MEMORY, 0,
+                   "dictionary creation failed for path %s "
+                   "for gfid is %s ",
+                   local->loc.path, gfid_local);
+            (*op_errno) = ENOMEM;
+            goto err;
+        }
+        local->xattr = dict_ref(xattr);
+        /* Subtract current MDS xattr value to -1 , value of MDS
+           xattr represents no. of times xattr modification failed
+           on non MDS subvols.
+        */
+        ret = dht_dict_set_array(xattrop, conf->mds_xattr_key, subone, 1);
+        if (ret != 0) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, DHT_MSG_DICT_SET_FAILED,
+                   "dictionary set array failed for path %s "
+                   "for gfid is %s ",
+                   local->loc.path, gfid_local);
+            if (xattrop)
+                dict_unref(xattrop);
+            (*op_errno) = ret;
+            goto err;
+        }
+        /* Wind a xattrop call to use ref counting approach
+           update mds xattr to -1 before update xattr on
+           hashed subvol and update mds xattr to +1 after update
+           xattr on all non hashed subvol
+        */
+        if (fd) {
+            STACK_WIND(frame, dht_xattrop_mds_cbk, local->mds_subvol,
+                       local->mds_subvol->fops->fxattrop, fd,
+                       GF_XATTROP_ADD_ARRAY, xattrop, NULL);
+        } else {
+            STACK_WIND(frame, dht_xattrop_mds_cbk, local->mds_subvol,
+                       local->mds_subvol->fops->xattrop, loc,
+                       GF_XATTROP_ADD_ARRAY, xattrop, NULL);
         }
+        if (xattrop)
+            dict_unref(xattrop);
+    }
 
-        flag = (local->layout->cnt > 1) ? _gf_true : _gf_false;
+    return 0;
+err:
+    return -1;
+}
 
-        ret = dht_vgetxattr_fill_and_set (local, &dict, this, flag);
+int
+dht_fsetxattr(call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *xattr,
+              int flags, dict_t *xdata)
+{
+    xlator_t *subvol = NULL;
+    dht_local_t *local = NULL;
+    int op_errno = EINVAL;
+    dht_conf_t *conf = NULL;
+    dht_layout_t *layout = NULL;
+    int ret = -1;
+    int call_cnt = 0;
+
+    VALIDATE_OR_GOTO(frame, err);
+    VALIDATE_OR_GOTO(this, err);
+    VALIDATE_OR_GOTO(fd, err);
+    VALIDATE_OR_GOTO(fd->inode, err);
+    VALIDATE_OR_GOTO(this->private, err);
+
+    conf = this->private;
+
+    if (!conf->defrag)
+        GF_IF_INTERNAL_XATTR_GOTO(conf->wild_xattr_name, xattr, op_errno, err);
+
+    local = dht_local_init(frame, NULL, fd, GF_FOP_FSETXATTR);
+    if (!local) {
+        op_errno = ENOMEM;
+        goto err;
+    }
+
+    subvol = local->cached_subvol;
+    if (!subvol) {
+        gf_msg_debug(this->name, 0, "no cached subvolume for fd=%p", fd);
+        op_errno = EINVAL;
+        goto err;
+    }
+
+    layout = local->layout;
+    if (!layout) {
+        gf_msg_debug(this->name, 0, "no layout for fd=%p", fd);
+        op_errno = EINVAL;
+        goto err;
+    }
+
+    local->xattr_req = xdata ? dict_ref(xdata) : dict_new();
+    local->call_cnt = call_cnt = layout->cnt;
+
+    if (IA_ISDIR(fd->inode->ia_type)) {
+        local->hashed_subvol = NULL;
+        ret = dht_dir_common_set_remove_xattr(frame, this, NULL, fd, xattr,
+                                              flags, xdata, &op_errno);
         if (ret)
-                goto unwind;
+            goto err;
+    } else {
+        local->call_cnt = 1;
+        local->rebalance.xattr = dict_ref(xattr);
+        local->rebalance.flags = flags;
+
+        ret = dict_set_int8(local->xattr_req, DHT_IATT_IN_XDATA_KEY, 1);
+        if (ret) {
+            gf_msg_debug(this->name, 0,
+                         "Failed to set dictionary key %s for fd=%p",
+                         DHT_IATT_IN_XDATA_KEY, fd);
+        }
 
-        DHT_STACK_UNWIND (getxattr, frame, 0, 0, dict, xdata);
-        goto cleanup;
+        STACK_WIND_COOKIE(frame, dht_file_setxattr_cbk, subvol, subvol,
+                          subvol->fops->fsetxattr, fd, xattr, flags,
+                          local->xattr_req);
+    }
+    return 0;
 
- unwind:
-        DHT_STACK_UNWIND (getxattr, frame, -1, local->op_errno,
-                          NULL, NULL);
- cleanup:
-        if (dict)
-                dict_unref (dict);
+err:
+    op_errno = (op_errno == -1) ? errno : op_errno;
+    DHT_STACK_UNWIND(fsetxattr, frame, -1, op_errno, NULL);
 
-        return 0;
+    return 0;
 }
 
-int
-dht_linkinfo_getxattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                           int op_ret, int op_errno, dict_t *xattr,
-                           dict_t *xdata)
+static int
+dht_checking_pathinfo_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                          int op_ret, int op_errno, dict_t *xattr,
+                          dict_t *xdata)
 {
-        int   ret   = 0;
-        char *value = NULL;
-
-        if (op_ret != -1) {
-                ret = dict_get_str (xattr, GF_XATTR_PATHINFO_KEY, &value);
-                if (!ret) {
-                        ret = dict_set_str (xattr, GF_XATTR_LINKINFO_KEY, value);
-                        if (!ret)
-                                gf_msg_trace (this->name, 0,
-                                              "failed to set linkinfo");
-                }
+    int i = -1;
+    int ret = -1;
+    char *value = NULL;
+    dht_local_t *local = NULL;
+    dht_conf_t *conf = NULL;
+    xlator_t *prev = NULL;
+    int this_call_cnt = 0;
+
+    local = frame->local;
+    prev = cookie;
+    conf = this->private;
+
+    if (op_ret == -1)
+        goto out;
+
+    ret = dict_get_str(xattr, GF_XATTR_PATHINFO_KEY, &value);
+    if (ret)
+        goto out;
+
+    if (!strcmp(value, local->key)) {
+        for (i = 0; i < conf->subvolume_cnt; i++) {
+            if (conf->subvolumes[i] == prev)
+                conf->decommissioned_bricks[i] = prev;
         }
+    }
+
+out:
+    this_call_cnt = dht_frame_return(frame);
+    if (is_last_call(this_call_cnt)) {
+        DHT_STACK_UNWIND(setxattr, frame, local->op_ret, ENOTSUP, NULL);
+    }
+    return 0;
+}
+
+static int
+dht_nuke_dir_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                 int32_t op_ret, int32_t op_errno, struct iatt *preparent,
+                 struct iatt *postparent, dict_t *xdata)
+{
+    STACK_UNWIND_STRICT(setxattr, frame, op_ret, op_errno, NULL);
+    return 0;
+}
 
-        DHT_STACK_UNWIND (getxattr, frame, op_ret, op_errno, xattr, xdata);
+static int
+dht_nuke_dir(call_frame_t *frame, xlator_t *this, loc_t *loc, data_t *tmp)
+{
+    if (!IA_ISDIR(loc->inode->ia_type)) {
+        DHT_STACK_UNWIND(setxattr, frame, -1, ENOTSUP, NULL);
+        return 0;
+    }
 
+    /* Setxattr didn't need the parent, but rmdir does. */
+    loc->parent = inode_parent(loc->inode, NULL, NULL);
+    if (!loc->parent) {
+        DHT_STACK_UNWIND(setxattr, frame, -1, ENOENT, NULL);
         return 0;
+    }
+    gf_uuid_copy(loc->pargfid, loc->parent->gfid);
+
+    if (!loc->name && loc->path) {
+        loc->name = strrchr(loc->path, '/');
+        if (loc->name) {
+            ++(loc->name);
+        }
+    }
+
+    /*
+     * We do this instead of calling dht_rmdir_do directly for two reasons.
+     * The first is that we want to reuse all of the initialization that
+     * dht_rmdir does, so if it ever changes we'll just follow along.  The
+     * second (i.e. why we don't use STACK_WIND_TAIL) is so that we don't
+     * obscure the fact that we came in via this path instead of a genuine
+     * rmdir.  That makes debugging just a tiny bit easier.
+     */
+    STACK_WIND(frame, dht_nuke_dir_cbk, this, this->fops->rmdir, loc, 1, NULL);
+
+    return 0;
 }
 
 int
-dht_getxattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                  int op_ret, int op_errno, dict_t *xattr, dict_t *xdata)
+dht_setxattr(call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xattr,
+             int flags, dict_t *xdata)
 {
-        int             this_call_cnt = 0;
-        dht_local_t     *local = NULL;
-        dht_conf_t      *conf = NULL;
+    xlator_t *subvol = NULL;
+    dht_local_t *local = NULL;
+    dht_conf_t *conf = NULL;
+    dht_methods_t *methods = NULL;
+    dht_layout_t *layout = NULL;
+    int i = 0;
+    int op_errno = EINVAL;
+    int ret = -1;
+    data_t *tmp = NULL;
+    uint32_t dir_spread = 0;
+    char value[4096] = {
+        0,
+    };
+    gf_dht_migrate_data_type_t forced_rebalance = GF_DHT_MIGRATE_DATA;
+    int call_cnt = 0;
+    uint32_t new_hash = 0;
+
+    VALIDATE_OR_GOTO(frame, err);
+    VALIDATE_OR_GOTO(this, err);
+    VALIDATE_OR_GOTO(loc, err);
+    VALIDATE_OR_GOTO(loc->inode, err);
+
+    conf = this->private;
+    GF_VALIDATE_OR_GOTO(this->name, conf, err);
+
+    methods = &(conf->methods);
+
+    /* Rebalance daemon is allowed to set internal keys */
+    if (!conf->defrag)
+        GF_IF_INTERNAL_XATTR_GOTO(conf->wild_xattr_name, xattr, op_errno, err);
+
+    local = dht_local_init(frame, loc, NULL, GF_FOP_SETXATTR);
+    if (!local) {
+        op_errno = ENOMEM;
+        goto err;
+    }
+
+    subvol = local->cached_subvol;
+    if (!subvol) {
+        gf_msg_debug(this->name, 0, "no cached subvolume for path=%s",
+                     loc->path);
+        op_errno = EINVAL;
+        goto err;
+    }
+
+    layout = local->layout;
+    if (!layout) {
+        gf_msg_debug(this->name, 0, "no layout for path=%s", loc->path);
+        op_errno = EINVAL;
+        goto err;
+    }
+
+    local->call_cnt = call_cnt = layout->cnt;
+    tmp = dict_get(xattr, conf->mds_xattr_key);
+    if (tmp) {
+        op_errno = ENOTSUP;
+        goto err;
+    }
+
+    tmp = dict_get(xattr, GF_XATTR_FILE_MIGRATE_KEY);
+    if (tmp) {
+        if (IA_ISDIR(loc->inode->ia_type)) {
+            op_errno = ENOTSUP;
+            goto err;
+        }
+
+        /* TODO: need to interpret the 'value' for more meaning
+           (ie, 'target' subvolume given there, etc) */
+        memcpy(value, tmp->data, tmp->len);
+        if (strcmp(value, "force") == 0)
+            forced_rebalance = GF_DHT_MIGRATE_DATA_EVEN_IF_LINK_EXISTS;
+
+        if (conf->decommission_in_progress)
+            forced_rebalance = GF_DHT_MIGRATE_HARDLINK;
+
+        if (!loc->path) {
+            op_errno = EINVAL;
+            goto err;
+        }
+
+        if (!local->loc.name)
+            local->loc.name = strrchr(local->loc.path, '/') + 1;
+
+        if (!local->loc.parent)
+            local->loc.parent = inode_parent(local->loc.inode, NULL, NULL);
+
+        if ((!local->loc.name) || (!local->loc.parent)) {
+            op_errno = EINVAL;
+            goto err;
+        }
+
+        if (gf_uuid_is_null(local->loc.pargfid))
+            gf_uuid_copy(local->loc.pargfid, local->loc.parent->gfid);
+
+        methods->migration_get_dst_subvol(this, local);
+
+        if (!local->rebalance.target_node) {
+            gf_msg(this->name, GF_LOG_ERROR, 0,
+                   DHT_MSG_HASHED_SUBVOL_GET_FAILED,
+                   "Failed to get hashed subvol for %s", loc->path);
+            op_errno = EINVAL;
+            goto err;
+        }
+
+        local->rebalance.from_subvol = local->cached_subvol;
+
+        if (local->rebalance.target_node == local->rebalance.from_subvol) {
+            op_errno = EEXIST;
+            goto err;
+        }
+        if (local->rebalance.target_node) {
+            local->flags = forced_rebalance;
 
-        VALIDATE_OR_GOTO (frame, out);
-        VALIDATE_OR_GOTO (frame->local, out);
-        VALIDATE_OR_GOTO (this->private, out);
+            frame->root->pid = GF_CLIENT_PID_DEFRAG;
 
-        conf = this->private;
-        local = frame->local;
+            ret = dht_start_rebalance_task(this, frame);
+            if (!ret)
+                return 0;
 
-        this_call_cnt = dht_frame_return (frame);
+            gf_msg(this->name, GF_LOG_ERROR, 0, DHT_MSG_REBALANCE_START_FAILED,
+                   "%s: failed to create a new rebalance synctask", loc->path);
+        }
+        op_errno = EINVAL;
+        goto err;
+    }
 
-        if (!xattr || (op_ret == -1)) {
-                local->op_ret = op_ret;
-                goto out;
+    tmp = dict_get(xattr, "decommission-brick");
+    if (tmp) {
+        /* This operation should happen only on '/' */
+        if (!__is_root_gfid(loc->inode->gfid)) {
+            op_errno = ENOTSUP;
+            goto err;
         }
 
-        if (dict_get (xattr, conf->xattr_name)) {
-                dict_del (xattr, conf->xattr_name);
+        memcpy(value, tmp->data, min(tmp->len, 4095));
+        local->key = gf_strdup(value);
+        local->call_cnt = conf->subvolume_cnt;
+
+        for (i = 0; i < conf->subvolume_cnt; i++) {
+            /* Get the pathinfo, and then compare */
+            STACK_WIND_COOKIE(frame, dht_checking_pathinfo_cbk,
+                              conf->subvolumes[i], conf->subvolumes[i],
+                              conf->subvolumes[i]->fops->getxattr, loc,
+                              GF_XATTR_PATHINFO_KEY, NULL);
         }
+        return 0;
+    }
 
-        if (frame->root->pid >= 0 ) {
-                GF_REMOVE_INTERNAL_XATTR("trusted.glusterfs.quota*", xattr);
-                GF_REMOVE_INTERNAL_XATTR("trusted.pgfid*", xattr);
+    tmp = dict_get(xattr, GF_XATTR_FIX_LAYOUT_KEY);
+    if (tmp) {
+        ret = dict_get_uint32(xattr, "new-commit-hash", &new_hash);
+        if (ret == 0) {
+            gf_msg_debug(this->name, 0,
+                         "updating commit hash for %s from %u to %u",
+                         uuid_utoa(loc->gfid), layout->commit_hash, new_hash);
+            layout->commit_hash = new_hash;
+
+            ret = dht_update_commit_hash_for_layout(frame);
+            if (ret) {
+                op_errno = ENOTCONN;
+                goto err;
+            }
+            return ret;
         }
 
-        local->op_ret = 0;
+        gf_msg(this->name, GF_LOG_INFO, 0, DHT_MSG_FIX_LAYOUT_INFO,
+               "fixing the layout of %s", loc->path);
 
-        if (!local->xattr) {
-                local->xattr = dict_copy_with_ref (xattr, NULL);
-        } else {
-                dht_aggregate_xattr (local->xattr, xattr);
-        }
-out:
-        if (is_last_call (this_call_cnt)) {
-                DHT_STACK_UNWIND (getxattr, frame, local->op_ret, op_errno,
-                                  local->xattr, NULL);
+        ret = dht_fix_directory_layout(frame, dht_fix_layout_setxattr_cbk,
+                                       layout);
+        if (ret) {
+            op_errno = ENOTCONN;
+            goto err;
         }
-        return 0;
+        return ret;
+    }
+
+    tmp = dict_get(xattr, "distribute.directory-spread-count");
+    if (tmp) {
+        /* Setxattr value is packed as 'binary', not string */
+        memcpy(value, tmp->data, min(tmp->len, 4095));
+        ret = gf_string2uint32(value, &dir_spread);
+        if (!ret && ((dir_spread <= conf->subvolume_cnt) && (dir_spread > 0))) {
+            layout->spread_cnt = dir_spread;
+
+            ret = dht_fix_directory_layout(frame, dht_common_setxattr_cbk,
+                                           layout);
+            if (ret) {
+                op_errno = ENOTCONN;
+                goto err;
+            }
+            return ret;
+        }
+        gf_msg(this->name, GF_LOG_ERROR, 0, DHT_MSG_OPERATION_NOT_SUP,
+               "wrong 'directory-spread-count' value (%s)", value);
+        op_errno = ENOTSUP;
+        goto err;
+    }
+
+    tmp = dict_get(xattr, "glusterfs.dht.nuke");
+    if (tmp) {
+        return dht_nuke_dir(frame, this, loc, tmp);
+    }
+    local->xattr_req = xdata ? dict_ref(xdata) : dict_new();
+
+    if (IA_ISDIR(loc->inode->ia_type)) {
+        local->hashed_subvol = NULL;
+        ret = dht_dir_common_set_remove_xattr(frame, this, loc, NULL, xattr,
+                                              flags, xdata, &op_errno);
+        if (ret)
+            goto err;
+    } else {
+        local->rebalance.xattr = dict_ref(xattr);
+        local->rebalance.flags = flags;
+        local->call_cnt = 1;
+
+        ret = dict_set_int8(local->xattr_req, DHT_IATT_IN_XDATA_KEY, 1);
+
+        STACK_WIND_COOKIE(frame, dht_file_setxattr_cbk, subvol, subvol,
+                          subvol->fops->setxattr, loc, xattr, flags,
+                          local->xattr_req);
+    }
+
+    return 0;
+
+err:
+    op_errno = (op_errno == -1) ? errno : op_errno;
+    DHT_STACK_UNWIND(setxattr, frame, -1, op_errno, NULL);
+
+    return 0;
 }
 
-int32_t
-dht_getxattr_unwind (call_frame_t *frame,
-                     int op_ret, int op_errno, dict_t *dict, dict_t *xdata)
+static int
+dht_removexattr2(xlator_t *this, xlator_t *subvol, call_frame_t *frame, int ret)
 {
-        DHT_STACK_UNWIND (getxattr, frame, op_ret, op_errno, dict, xdata);
+    dht_local_t *local = NULL;
+    int op_errno = EINVAL;
+
+    if (!frame || !frame->local)
+        goto err;
+
+    local = frame->local;
+    op_errno = local->op_errno;
+
+    local->call_cnt = 2; /* This is the second attempt */
+
+    if (we_are_not_migrating(ret)) {
+        /* This dht xlator is not migrating the file. Unwind and
+         * pass on the original mode bits so the higher DHT layer
+         * can handle this.
+         */
+        DHT_STACK_UNWIND(removexattr, frame, local->op_ret, local->op_errno,
+                         local->rebalance.xdata);
         return 0;
-}
+    }
+
+    if (subvol == NULL)
+        goto err;
+
+    if (local->fop == GF_FOP_REMOVEXATTR) {
+        STACK_WIND_COOKIE(frame, dht_file_removexattr_cbk, subvol, subvol,
+                          subvol->fops->removexattr, &local->loc, local->key,
+                          local->xattr_req);
+    } else {
+        STACK_WIND_COOKIE(frame, dht_file_removexattr_cbk, subvol, subvol,
+                          subvol->fops->fremovexattr, local->fd, local->key,
+                          local->xattr_req);
+    }
+
+    return 0;
 
+err:
+    DHT_STACK_UNWIND(removexattr, frame, -1, op_errno, NULL);
+    return 0;
+}
 
 int
-dht_getxattr_get_real_filename_cbk (call_frame_t *frame, void *cookie,
-				    xlator_t *this, int op_ret, int op_errno,
-				    dict_t *xattr, dict_t *xdata)
+dht_file_removexattr_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                         int op_ret, int op_errno, dict_t *xdata)
 {
-        int             this_call_cnt = 0;
-        dht_local_t     *local = NULL;
+    int ret = -1;
+    dht_local_t *local = NULL;
+    xlator_t *prev = NULL;
+    struct iatt *stbuf = NULL;
+    inode_t *inode = NULL;
+    xlator_t *subvol1 = NULL, *subvol2 = NULL;
 
+    local = frame->local;
+    prev = cookie;
 
-        local = frame->local;
+    local->op_errno = op_errno;
 
-        LOCK (&frame->lock);
-        {
-                if (local->op_errno == ENODATA ||
-                    local->op_errno == EOPNOTSUPP) {
-                        /* Nothing to do here, we have already found
-                         * a subvol which does not have the get_real_filename
-                         * optimization. If condition is for simple logic.
-                         */
-                        goto unlock;
-                }
+    if ((local->fop == GF_FOP_FREMOVEXATTR) &&
+        dht_check_remote_fd_failed_error(local, op_ret, op_errno)) {
+        ret = dht_check_and_open_fd_on_subvol(this, frame);
+        if (ret)
+            goto out;
+        return 0;
+    }
 
-                if (op_ret == -1) {
-
-                        if (op_errno == ENODATA || op_errno == EOPNOTSUPP) {
-                                /* This subvol does not have the optimization.
-                                 * Better let the user know we don't support it.
-                                 * Remove previous results if any.
-                                 */
-
-                                if (local->xattr) {
-                                        dict_unref (local->xattr);
-                                        local->xattr = NULL;
-                                }
-
-                                if (local->xattr_req) {
-                                        dict_unref (local->xattr_req);
-                                        local->xattr_req = NULL;
-                                }
-
-                                local->op_ret = op_ret;
-                                local->op_errno = op_errno;
-                                gf_log (this->name, GF_LOG_WARNING, "At least "
-                                        "one of the bricks does not support "
-                                        "this operation. Please upgrade all "
-                                        "bricks.");
-                                goto unlock;
-                        }
+    if ((op_ret == -1) && !dht_inode_missing(op_errno)) {
+        gf_msg_debug(this->name, op_errno, "subvolume %s returned -1",
+                     prev->name);
+        goto out;
+    }
 
-                        if (op_errno == ENOENT) {
-                                /* Do nothing, our defaults are set to this.
-                                 */
-                                goto unlock;
-                        }
+    if (local->call_cnt != 1)
+        goto out;
 
-                        /* This is a place holder for every other error
-                         * case. I am not sure of how to interpret
-                         * ENOTCONN etc. As of now, choosing to ignore
-                         * down subvol and return a good result(if any)
-                         * from other subvol.
-                         */
-                         gf_log (this->name, GF_LOG_WARNING,
-                                 "Failed to get real filename. "
-                                 "error:%s", strerror (op_errno));
-                        goto unlock;
+    ret = dict_get_bin(xdata, DHT_IATT_IN_XDATA_KEY, (void **)&stbuf);
 
-                }
+    if ((!op_ret) && !stbuf) {
+        goto out;
+    }
 
+    local->op_ret = 0;
 
-                /* This subvol has the required file.
-                 * There could be other subvols which have returned
-                 * success already, choosing to return the latest good
-                 * result.
-                 */
-                if (local->xattr)
-                        dict_unref (local->xattr);
-                local->xattr = dict_ref (xattr);
+    local->rebalance.target_op_fn = dht_removexattr2;
+    if (xdata)
+        local->rebalance.xdata = dict_ref(xdata);
 
-                if (local->xattr_req) {
-                        dict_unref (local->xattr_req);
-                        local->xattr_req = NULL;
-                }
-                if (xdata)
-                        local->xattr_req = dict_ref (xdata);
+    /* Phase 2 of migration */
+    if ((op_ret == -1) || IS_DHT_MIGRATION_PHASE2(stbuf)) {
+        ret = dht_rebalance_complete_check(this, frame);
+        if (!ret)
+            return 0;
+    }
 
-                local->op_ret = op_ret;
-                local->op_errno = 0;
-                gf_log (this->name, GF_LOG_DEBUG, "Found a matching "
-                        "file.");
-        }
-unlock:
-        UNLOCK (&frame->lock);
+    /* Phase 1 of migration */
+    if (IS_DHT_MIGRATION_PHASE1(stbuf)) {
+        inode = (local->fd) ? local->fd->inode : local->loc.inode;
 
+        ret = dht_inode_ctx_get_mig_info(this, inode, &subvol1, &subvol2);
+        if (!dht_mig_info_is_invalid(local->cached_subvol, subvol1, subvol2)) {
+            dht_removexattr2(this, subvol2, frame, 0);
+            return 0;
+        }
 
-	this_call_cnt = dht_frame_return (frame);
-	if (is_last_call (this_call_cnt)) {
-                DHT_STACK_UNWIND (getxattr, frame, local->op_ret,
-                                  local->op_errno, local->xattr,
-                                  local->xattr_req);
-	}
+        ret = dht_rebalance_in_progress_check(this, frame);
+        if (!ret)
+            return 0;
+    }
 
-	return 0;
+out:
+    if (local->fop == GF_FOP_REMOVEXATTR) {
+        DHT_STACK_UNWIND(removexattr, frame, op_ret, op_errno, xdata);
+    } else {
+        DHT_STACK_UNWIND(fremovexattr, frame, op_ret, op_errno, xdata);
+    }
+    return 0;
 }
 
-
 int
-dht_getxattr_get_real_filename (call_frame_t *frame, xlator_t *this,
-				loc_t *loc, const char *key, dict_t *xdata)
+dht_removexattr(call_frame_t *frame, xlator_t *this, loc_t *loc,
+                const char *key, dict_t *xdata)
 {
-	dht_local_t     *local = NULL;
-	int              i = 0;
-	dht_layout_t    *layout = NULL;
-	int              cnt = 0;
-	xlator_t        *subvol = NULL;
-
+    xlator_t *subvol = NULL;
+    int op_errno = -1;
+    dht_local_t *local = NULL;
+    dht_layout_t *layout = NULL;
+    int call_cnt = 0;
+    dht_conf_t *conf = NULL;
+    int ret = 0;
+
+    VALIDATE_OR_GOTO(this, err);
+    VALIDATE_OR_GOTO(this->private, err);
+
+    conf = this->private;
+
+    GF_IF_NATIVE_XATTR_GOTO(conf->wild_xattr_name, key, op_errno, err);
+
+    VALIDATE_OR_GOTO(frame, err);
+    VALIDATE_OR_GOTO(loc, err);
+    VALIDATE_OR_GOTO(loc->inode, err);
+
+    local = dht_local_init(frame, loc, NULL, GF_FOP_REMOVEXATTR);
+    if (!local) {
+        op_errno = ENOMEM;
+        goto err;
+    }
+
+    subvol = local->cached_subvol;
+    if (!subvol) {
+        gf_msg_debug(this->name, 0, "no cached subvolume for path=%s",
+                     loc->path);
+        op_errno = EINVAL;
+        goto err;
+    }
+
+    layout = local->layout;
+    if (!local->layout) {
+        gf_msg_debug(this->name, 0, "no layout for path=%s", loc->path);
+        op_errno = EINVAL;
+        goto err;
+    }
+    local->xattr_req = (xdata) ? dict_ref(xdata) : dict_new();
+
+    local->call_cnt = call_cnt = layout->cnt;
+    local->key = gf_strdup(key);
+
+    if (key && (strncmp(key, conf->mds_xattr_key, strlen(key)) == 0)) {
+        op_errno = ENOTSUP;
+        goto err;
+    }
+
+    if (IA_ISDIR(loc->inode->ia_type)) {
+        local->hashed_subvol = NULL;
+        ret = dht_dir_common_set_remove_xattr(frame, this, loc, NULL, NULL, 0,
+                                              local->xattr_req, &op_errno);
+        if (ret)
+            goto err;
 
-	local = frame->local;
-	layout = local->layout;
+    } else {
+        local->call_cnt = 1;
+        ret = dict_set_int8(local->xattr_req, DHT_IATT_IN_XDATA_KEY, 1);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, ENOMEM, DHT_MSG_DICT_SET_FAILED,
+                   "Failed to "
+                   "set dictionary key %s for %s",
+                   DHT_IATT_IN_XDATA_KEY, loc->path);
+        }
 
-	cnt = local->call_cnt = layout->cnt;
+        STACK_WIND_COOKIE(frame, dht_file_removexattr_cbk, subvol, subvol,
+                          subvol->fops->removexattr, loc, key,
+                          local->xattr_req);
+    }
 
-	local->op_ret = -1;
-	local->op_errno = ENOENT;
+    return 0;
 
-	for (i = 0; i < cnt; i++) {
-		subvol = layout->list[i].xlator;
-		STACK_WIND (frame, dht_getxattr_get_real_filename_cbk,
-			    subvol, subvol->fops->getxattr,
-			    loc, key, xdata);
-	}
+err:
+    op_errno = (op_errno == -1) ? errno : op_errno;
+    DHT_STACK_UNWIND(removexattr, frame, -1, op_errno, NULL);
 
-	return 0;
+    return 0;
 }
 
 int
-dht_marker_populate_args (call_frame_t *frame, int type, int *gauge,
-                          xlator_t **subvols)
+dht_fremovexattr(call_frame_t *frame, xlator_t *this, fd_t *fd, const char *key,
+                 dict_t *xdata)
 {
-        dht_local_t *local = NULL;
-        int         i = 0;
-        dht_layout_t *layout = NULL;
+    xlator_t *subvol = NULL;
+    int op_errno = -1;
+    dht_local_t *local = NULL;
+    dht_layout_t *layout = NULL;
+    int call_cnt = 0;
+    dht_conf_t *conf = 0;
+    int ret = 0;
+
+    VALIDATE_OR_GOTO(this, err);
+    VALIDATE_OR_GOTO(this->private, err);
+
+    conf = this->private;
+
+    GF_IF_NATIVE_XATTR_GOTO(conf->wild_xattr_name, key, op_errno, err);
+
+    VALIDATE_OR_GOTO(frame, err);
+
+    local = dht_local_init(frame, NULL, fd, GF_FOP_FREMOVEXATTR);
+    if (!local) {
+        op_errno = ENOMEM;
+        goto err;
+    }
+
+    subvol = local->cached_subvol;
+    if (!subvol) {
+        gf_msg_debug(this->name, 0, "no cached subvolume for inode=%s",
+                     uuid_utoa(fd->inode->gfid));
+        op_errno = EINVAL;
+        goto err;
+    }
+
+    layout = local->layout;
+    if (!local->layout) {
+        gf_msg_debug(this->name, 0, "no layout for inode=%s",
+                     uuid_utoa(fd->inode->gfid));
+        op_errno = EINVAL;
+        goto err;
+    }
+    local->xattr_req = xdata ? dict_ref(xdata) : dict_new();
+
+    local->call_cnt = call_cnt = layout->cnt;
+    local->key = gf_strdup(key);
+
+    if (IA_ISDIR(fd->inode->ia_type)) {
+        local->hashed_subvol = NULL;
+        ret = dht_dir_common_set_remove_xattr(frame, this, NULL, fd, NULL, 0,
+                                              local->xattr_req, &op_errno);
+        if (ret)
+            goto err;
 
-        local  = frame->local;
-        layout = local->layout;
+    } else {
+        local->call_cnt = 1;
+        ret = dict_set_int8(local->xattr_req, DHT_IATT_IN_XDATA_KEY, 1);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, ENOMEM, DHT_MSG_DICT_SET_FAILED,
+                   "Failed to "
+                   "set dictionary key %s for fd=%p",
+                   DHT_IATT_IN_XDATA_KEY, fd);
+        }
+
+        STACK_WIND_COOKIE(frame, dht_file_removexattr_cbk, subvol, subvol,
+                          subvol->fops->fremovexattr, fd, key,
+                          local->xattr_req);
+    }
 
-        for (i = 0; i < layout->cnt; i++)
-                subvols[i] = layout->list[i].xlator;
+    return 0;
 
-        return layout->cnt;
+err:
+    op_errno = (op_errno == -1) ? errno : op_errno;
+    DHT_STACK_UNWIND(fremovexattr, frame, -1, op_errno, NULL);
+
+    return 0;
 }
 
 int
-dht_getxattr (call_frame_t *frame, xlator_t *this,
-              loc_t *loc, const char *key, dict_t *xdata)
-#define DHT_IS_DIR(layout)  (layout->cnt > 1)
+dht_fd_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int op_ret,
+           int op_errno, fd_t *fd, dict_t *xdata)
 {
+    dht_local_t *local = NULL;
+    int this_call_cnt = 0;
+    xlator_t *prev = NULL;
 
-        xlator_t     *subvol        = NULL;
-        xlator_t     *hashed_subvol = NULL;
-        xlator_t     *cached_subvol = NULL;
-        dht_conf_t   *conf          = NULL;
-        dht_local_t  *local         = NULL;
-        dht_layout_t *layout        = NULL;
-        int           op_errno      = -1;
-        int           i             = 0;
-        int           cnt           = 0;
+    local = frame->local;
+    prev = cookie;
 
-        VALIDATE_OR_GOTO (frame, err);
-        VALIDATE_OR_GOTO (this, err);
-        VALIDATE_OR_GOTO (loc, err);
-        VALIDATE_OR_GOTO (loc->inode, err);
-        VALIDATE_OR_GOTO (this->private, err);
+    LOCK(&frame->lock);
+    {
+        if (op_ret == -1) {
+            local->op_errno = op_errno;
+            UNLOCK(&frame->lock);
+            gf_msg_debug(this->name, op_errno, "subvolume %s returned -1",
+                         prev->name);
+            goto post_unlock;
+        }
 
-        conf   = this->private;
+        local->op_ret = 0;
+    }
+    UNLOCK(&frame->lock);
+post_unlock:
+    this_call_cnt = dht_frame_return(frame);
+    if (is_last_call(this_call_cnt))
+        DHT_STACK_UNWIND(open, frame, local->op_ret, local->op_errno, local->fd,
+                         NULL);
+
+    return 0;
+}
 
-        local = dht_local_init (frame, loc, NULL, GF_FOP_GETXATTR);
-        if (!local) {
-                op_errno = ENOMEM;
+/*
+ * dht_normalize_stats -
+ */
+static void
+dht_normalize_stats(struct statvfs *buf, unsigned long bsize,
+                    unsigned long frsize)
+{
+    double factor = 0;
+
+    if (buf->f_bsize != bsize) {
+        buf->f_bsize = bsize;
+    }
+
+    if (buf->f_frsize != frsize) {
+        factor = ((double)buf->f_frsize) / frsize;
+        buf->f_frsize = frsize;
+        buf->f_blocks = (fsblkcnt_t)(factor * buf->f_blocks);
+        buf->f_bfree = (fsblkcnt_t)(factor * buf->f_bfree);
+        buf->f_bavail = (fsblkcnt_t)(factor * buf->f_bavail);
+    }
+}
 
-                goto err;
+static int
+dht_statfs_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int op_ret,
+               int op_errno, struct statvfs *statvfs, dict_t *xdata)
+{
+    gf_boolean_t event = _gf_false;
+    qdstatfs_action_t action = qdstatfs_action_OFF;
+    dht_local_t *local = NULL;
+    int this_call_cnt = 0;
+    int bsize = 0;
+    int frsize = 0;
+    GF_UNUSED int ret = 0;
+    unsigned long new_usage = 0;
+    unsigned long cur_usage = 0;
+
+    local = frame->local;
+    GF_ASSERT(local);
+
+    if (xdata)
+        ret = dict_get_int8(xdata, "quota-deem-statfs", (int8_t *)&event);
+
+    LOCK(&frame->lock);
+    {
+        if (op_ret == -1) {
+            local->op_errno = op_errno;
+            goto unlock;
         }
-
-        layout = local->layout;
-        if (!layout) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        DHT_MSG_LAYOUT_NULL,
-                        "Layout is NULL");
-                op_errno = ENOENT;
-                goto err;
+        if (!statvfs) {
+            op_errno = EINVAL;
+            local->op_ret = -1;
+            goto unlock;
         }
+        local->op_ret = 0;
 
-        if (key) {
-                local->key = gf_strdup (key);
-                if (!local->key) {
-                        op_errno = ENOMEM;
-                        goto err;
-                }
+        if (local->quota_deem_statfs) {
+            if (event == _gf_true) {
+                action = qdstatfs_action_COMPARE;
+            } else {
+                action = qdstatfs_action_NEGLECT;
+            }
+        } else {
+            if (event == _gf_true) {
+                action = qdstatfs_action_REPLACE;
+                local->quota_deem_statfs = _gf_true;
+            }
         }
 
-	if (key &&
-	    (strncmp (key, GF_XATTR_GET_REAL_FILENAME_KEY,
-		      strlen (GF_XATTR_GET_REAL_FILENAME_KEY)) == 0)
-	    && DHT_IS_DIR(layout)) {
-		dht_getxattr_get_real_filename (frame, this, loc, key, xdata);
-		return 0;
-	}
+        if (local->quota_deem_statfs) {
+            switch (action) {
+                case qdstatfs_action_NEGLECT:
+                    goto unlock;
 
-        /* for file use cached subvolume (obviously!): see if {}
-         * below
-         * for directory:
-         *  wind to all subvolumes and exclude subvolumes which
-         *  return ENOTCONN (in callback)
-         *
-         * NOTE: Don't trust inode here, as that may not be valid
-         *       (until inode_link() happens)
-         */
-        if (key && DHT_IS_DIR(layout) &&
-            (XATTR_IS_PATHINFO (key)
-             || (strcmp (key, GF_XATTR_NODE_UUID_KEY) == 0))) {
-                (void) strncpy (local->xsel, key, 256);
-                cnt = local->call_cnt = layout->cnt;
-                for (i = 0; i < cnt; i++) {
-                        subvol = layout->list[i].xlator;
-                        STACK_WIND (frame, dht_vgetxattr_dir_cbk,
-                                    subvol, subvol->fops->getxattr,
-                                    loc, key, xdata);
-                }
-                return 0;
-        }
+                case qdstatfs_action_REPLACE:
+                    local->statvfs = *statvfs;
+                    goto unlock;
 
-        /* node-uuid or pathinfo for files */
-        if (key && ((strcmp (key, GF_XATTR_NODE_UUID_KEY) == 0)
-                    || XATTR_IS_PATHINFO (key))) {
-                cached_subvol = local->cached_subvol;
-                (void) strncpy (local->xsel, key, 256);
+                case qdstatfs_action_COMPARE:
+                    new_usage = statvfs->f_blocks - statvfs->f_bfree;
+                    cur_usage = local->statvfs.f_blocks -
+                                local->statvfs.f_bfree;
 
-                local->call_cnt = 1;
-                STACK_WIND (frame, dht_vgetxattr_cbk, cached_subvol,
-                            cached_subvol->fops->getxattr, loc, key, xdata);
+                    /* Take the max of the usage from subvols */
+                    if (new_usage >= cur_usage)
+                        local->statvfs = *statvfs;
+                    goto unlock;
 
-                return 0;
+                default:
+                    break;
+            }
         }
 
-        if (key && (strcmp (key, GF_XATTR_LINKINFO_KEY) == 0)) {
+        if (local->statvfs.f_bsize != 0) {
+            bsize = max(local->statvfs.f_bsize, statvfs->f_bsize);
+            frsize = max(local->statvfs.f_frsize, statvfs->f_frsize);
+            dht_normalize_stats(&local->statvfs, bsize, frsize);
+            dht_normalize_stats(statvfs, bsize, frsize);
+        } else {
+            local->statvfs.f_bsize = statvfs->f_bsize;
+            local->statvfs.f_frsize = statvfs->f_frsize;
+        }
+
+        local->statvfs.f_blocks += statvfs->f_blocks;
+        local->statvfs.f_bfree += statvfs->f_bfree;
+        local->statvfs.f_bavail += statvfs->f_bavail;
+        local->statvfs.f_files += statvfs->f_files;
+        local->statvfs.f_ffree += statvfs->f_ffree;
+        local->statvfs.f_favail += statvfs->f_favail;
+        local->statvfs.f_fsid = statvfs->f_fsid;
+        local->statvfs.f_flag = statvfs->f_flag;
+        local->statvfs.f_namemax = statvfs->f_namemax;
+    }
+unlock:
+    UNLOCK(&frame->lock);
 
-                hashed_subvol = dht_subvol_get_hashed (this, loc);
-                if (!hashed_subvol) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                DHT_MSG_HASHED_SUBVOL_GET_FAILED,
-                                "Failed to get hashed subvol for %s",
-                                loc->path);
-                        op_errno = EINVAL;
-                        goto err;
-                }
+    this_call_cnt = dht_frame_return(frame);
+    if (is_last_call(this_call_cnt))
+        DHT_STACK_UNWIND(statfs, frame, local->op_ret, local->op_errno,
+                         &local->statvfs, xdata);
 
-                cached_subvol = dht_subvol_get_cached (this, loc->inode);
-                if (!cached_subvol) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                DHT_MSG_CACHED_SUBVOL_GET_FAILED,
-                                "Failed to get cached subvol for %s",
-                                loc->path);
-                        op_errno = EINVAL;
-                        goto err;
-                }
+    return 0;
+}
 
-                if (hashed_subvol == cached_subvol) {
-                        op_errno = ENODATA;
-                        goto err;
-                }
+int
+dht_statfs(call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xdata)
+{
+    dht_local_t *local = NULL;
+    dht_conf_t *conf = NULL;
+    int op_errno = -1;
+    int i = -1;
+    inode_t *inode = NULL;
+    inode_table_t *itable = NULL;
+    static uuid_t root_gfid = {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1};
+    loc_t newloc = {
+        0,
+    };
 
-                STACK_WIND (frame, dht_linkinfo_getxattr_cbk, hashed_subvol,
-                            hashed_subvol->fops->getxattr, loc,
-                            GF_XATTR_PATHINFO_KEY, xdata);
-                return 0;
-        }
+    VALIDATE_OR_GOTO(frame, err);
+    VALIDATE_OR_GOTO(this, err);
+    VALIDATE_OR_GOTO(loc, err);
+    VALIDATE_OR_GOTO(this->private, err);
 
-        if (key && (!strcmp (GF_XATTR_QUOTA_LIMIT_LIST, key) ||
-                    !strcmp (GF_XATTR_QUOTA_LIMIT_LIST_OBJECT, key))) {
-                /* quota hardlimit and aggregated size of a directory is stored
-                 * in inode contexts of each brick. Hence its good enough that
-                 * we send getxattr for this key to any brick.
-                 */
-                local->call_cnt = 1;
-                subvol = dht_first_up_subvol (this);
-                STACK_WIND (frame, dht_getxattr_cbk, subvol,
-                            subvol->fops->getxattr, loc, key, xdata);
-                return 0;
-        }
+    conf = this->private;
 
-        if (cluster_handle_marker_getxattr (frame, loc, key, conf->vol_uuid,
-                                            dht_getxattr_unwind,
-                                            dht_marker_populate_args) == 0)
-                return 0;
+    local = dht_local_init(frame, NULL, NULL, GF_FOP_STATFS);
+    if (!local) {
+        op_errno = ENOMEM;
+        goto err;
+    }
 
-        if (DHT_IS_DIR(layout)) {
-                cnt = local->call_cnt = layout->cnt;
-        } else {
-                cnt = local->call_cnt  = 1;
+    if (loc->inode && !IA_ISDIR(loc->inode->ia_type)) {
+        itable = loc->inode->table;
+        if (!itable) {
+            op_errno = EINVAL;
+            goto err;
         }
 
-        for (i = 0; i < cnt; i++) {
-                subvol = layout->list[i].xlator;
-                STACK_WIND (frame, dht_getxattr_cbk,
-                            subvol, subvol->fops->getxattr,
-                            loc, key, xdata);
+        loc = &local->loc2;
+
+        inode = inode_find(itable, root_gfid);
+        if (!inode) {
+            op_errno = EINVAL;
+            goto err;
         }
-        return 0;
+
+        dht_build_root_loc(inode, &newloc);
+        loc = &newloc;
+    }
+
+    local->call_cnt = conf->subvolume_cnt;
+
+    for (i = 0; i < conf->subvolume_cnt; i++) {
+        STACK_WIND(frame, dht_statfs_cbk, conf->subvolumes[i],
+                   conf->subvolumes[i]->fops->statfs, loc, xdata);
+    }
+    return 0;
 
 err:
-        op_errno = (op_errno == -1) ? errno : op_errno;
-        DHT_STACK_UNWIND (getxattr, frame, -1, op_errno, NULL, NULL);
+    op_errno = (op_errno == -1) ? errno : op_errno;
+    DHT_STACK_UNWIND(statfs, frame, -1, op_errno, NULL, NULL);
 
-        return 0;
+    return 0;
 }
-#undef DHT_IS_DIR
 
 int
-dht_fgetxattr (call_frame_t *frame, xlator_t *this,
-               fd_t *fd, const char *key, dict_t *xdata)
-{
-        xlator_t     *subvol        = NULL;
-        dht_local_t  *local         = NULL;
-        dht_layout_t *layout        = NULL;
-        int           op_errno      = -1;
-        int           i             = 0;
-        int           cnt           = 0;
-
-        VALIDATE_OR_GOTO (frame, err);
-        VALIDATE_OR_GOTO (this, err);
-        VALIDATE_OR_GOTO (fd, err);
-        VALIDATE_OR_GOTO (fd->inode, err);
-        VALIDATE_OR_GOTO (this->private, err);
-
-        local = dht_local_init (frame, NULL, fd, GF_FOP_FGETXATTR);
-        if (!local) {
-                op_errno = ENOMEM;
-
-                goto err;
-        }
-
-        layout = local->layout;
-        if (!layout) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        DHT_MSG_LAYOUT_NULL,
-                        "Layout is NULL");
-                op_errno = ENOENT;
-                goto err;
+dht_opendir(call_frame_t *frame, xlator_t *this, loc_t *loc, fd_t *fd,
+            dict_t *xdata)
+{
+    dht_local_t *local = NULL;
+    dht_conf_t *conf = NULL;
+    int op_errno = -1;
+    int i = -1;
+    int ret = 0;
+    gf_boolean_t new_xdata = _gf_false;
+    xlator_t **subvolumes = NULL;
+    int call_count = 0;
+
+    VALIDATE_OR_GOTO(frame, err);
+    VALIDATE_OR_GOTO(this, err);
+    VALIDATE_OR_GOTO(fd, err);
+    VALIDATE_OR_GOTO(this->private, err);
+
+    conf = this->private;
+
+    local = dht_local_init(frame, loc, fd, GF_FOP_OPENDIR);
+    if (!local) {
+        op_errno = ENOMEM;
+        goto err;
+    }
+    local->first_up_subvol = dht_first_up_subvol(this);
+
+    if (!xdata) {
+        xdata = dict_new();
+        if (!xdata) {
+            op_errno = ENOMEM;
+            goto err;
+        }
+        new_xdata = _gf_true;
+    }
+
+    ret = dict_set_uint32(xdata, conf->link_xattr_name, 256);
+    if (ret)
+        gf_msg(this->name, GF_LOG_WARNING, 0, DHT_MSG_DICT_SET_FAILED,
+               "Failed to set dictionary value : key = %s",
+               conf->link_xattr_name);
+
+    /* dht_readdirp will wind to all subvols so open has to be sent to
+     * all subvols whether or not conf->local_subvols is set */
+
+    call_count = local->call_cnt = conf->subvolume_cnt;
+    subvolumes = conf->subvolumes;
+
+    /* In case of parallel-readdir, the readdir-ahead will be loaded
+     * below dht, in this case, if we want to enable or disable SKIP_DIRs
+     * it has to be done in opendir, so that prefetching logic in
+     * readdir-ahead, honors it */
+    for (i = 0; i < call_count; i++) {
+        if (conf->readdir_optimize == _gf_true) {
+            if (subvolumes[i] != local->first_up_subvol) {
+                ret = dict_set_int32(xdata, GF_READDIR_SKIP_DIRS, 1);
+                if (ret)
+                    gf_msg(this->name, GF_LOG_ERROR, 0, DHT_MSG_DICT_SET_FAILED,
+                           "Failed to set dictionary"
+                           " value :key = %s, ret:%d",
+                           GF_READDIR_SKIP_DIRS, ret);
+            }
         }
 
-        if (key) {
-                local->key = gf_strdup (key);
-                if (!local->key) {
-                        op_errno = ENOMEM;
-                        goto err;
-                }
-        }
+        STACK_WIND_COOKIE(frame, dht_fd_cbk, subvolumes[i], subvolumes[i],
+                          subvolumes[i]->fops->opendir, loc, fd, xdata);
+        dict_del(xdata, GF_READDIR_SKIP_DIRS);
+    }
 
-        if ((fd->inode->ia_type == IA_IFDIR)
-	    && key
-            && (strncmp (key, GF_XATTR_LOCKINFO_KEY,
-                         strlen (GF_XATTR_LOCKINFO_KEY)) != 0)) {
-                cnt = local->call_cnt = layout->cnt;
-        } else {
-                cnt = local->call_cnt  = 1;
-        }
+    if (new_xdata)
+        dict_unref(xdata);
 
-        for (i = 0; i < cnt; i++) {
-                subvol = layout->list[i].xlator;
-                STACK_WIND (frame, dht_getxattr_cbk,
-                            subvol, subvol->fops->fgetxattr,
-                            fd, key, NULL);
-        }
-        return 0;
+    return 0;
 
 err:
-        op_errno = (op_errno == -1) ? errno : op_errno;
-        DHT_STACK_UNWIND (fgetxattr, frame, -1, op_errno, NULL, NULL);
+    op_errno = (op_errno == -1) ? errno : op_errno;
+    DHT_STACK_UNWIND(opendir, frame, -1, op_errno, NULL, NULL);
 
-        return 0;
+    return 0;
 }
 
-int
-dht_fsetxattr (call_frame_t *frame, xlator_t *this,
-               fd_t *fd, dict_t *xattr, int flags, dict_t *xdata)
+/* dht_readdirp_cbk creates a new dentry and dentry->inode is not assigned.
+   This functions assigns an inode if all of the following conditions are
+   true:
+
+   * DHT has only one child. In this case the entire layout is present on
+   this single child and hence we can set complete layout in inode.
+   * backend has complete layout and there are no anomalies in it and from
+   this information layout can be constructed and set in inode.
+*/
+
+static void
+dht_populate_inode_for_dentry(xlator_t *this, xlator_t *subvol,
+                              gf_dirent_t *entry, gf_dirent_t *orig_entry)
 {
-        xlator_t     *subvol   = NULL;
-        dht_local_t  *local    = NULL;
-        int           op_errno = EINVAL;
-        dht_conf_t   *conf     = NULL;
+    dht_layout_t *layout = NULL;
+    int ret = 0;
+    loc_t loc = {
+        0,
+    };
+
+    if (gf_uuid_is_null(orig_entry->d_stat.ia_gfid)) {
+        /* this skips the '..' entry for the root of the volume */
+        return;
+    }
 
-        VALIDATE_OR_GOTO (frame, err);
-        VALIDATE_OR_GOTO (this, err);
-        VALIDATE_OR_GOTO (fd, err);
-        VALIDATE_OR_GOTO (fd->inode, err);
-        VALIDATE_OR_GOTO (this->private, err);
+    gf_uuid_copy(loc.gfid, orig_entry->d_stat.ia_gfid);
+    loc.inode = inode_ref(orig_entry->inode);
 
-        conf = this->private;
+    if (is_revalidate(&loc)) {
+        goto out;
+    }
 
-        GF_IF_INTERNAL_XATTR_GOTO (conf->wild_xattr_name, xattr,
-                                   op_errno, err);
+    layout = dht_layout_new(this, 1);
+    if (!layout)
+        goto out;
 
-        local = dht_local_init (frame, NULL, fd, GF_FOP_FSETXATTR);
-        if (!local) {
-                op_errno = ENOMEM;
-                goto err;
+    ret = dht_layout_merge(this, layout, subvol, 0, 0, orig_entry->dict);
+    if (!ret) {
+        ret = dht_layout_normalize(this, &loc, layout);
+        if (ret == 0) {
+            dht_layout_set(this, orig_entry->inode, layout);
+            entry->inode = inode_ref(orig_entry->inode);
+            layout = NULL;
         }
+    }
 
-        subvol = local->cached_subvol;
-        if (!subvol) {
-                gf_msg_debug (this->name, 0,
-                              "no cached subvolume for fd=%p", fd);
-                op_errno = EINVAL;
-                goto err;
-        }
+    if (layout)
+        dht_layout_unref(this, layout);
 
-        local->call_cnt = 1;
+out:
+    loc_wipe(&loc);
+    return;
+}
 
-        STACK_WIND (frame, dht_err_cbk, subvol, subvol->fops->fsetxattr,
-                    fd, xattr, flags, NULL);
+/* Posix returns op_errno = ENOENT to indicate that there are no more
+ * entries
+ */
+static int
+dht_readdirp_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int op_ret,
+                 int op_errno, gf_dirent_t *orig_entries, dict_t *xdata)
+{
+    dht_local_t *local = NULL;
+    gf_dirent_t entries;
+    gf_dirent_t *orig_entry = NULL;
+    gf_dirent_t *entry = NULL;
+    xlator_t *prev = NULL;
+    xlator_t *next_subvol = NULL;
+    off_t next_offset = 0;
+    int count = 0;
+    dht_layout_t *layout = NULL;
+    dht_conf_t *conf = NULL;
+    dht_methods_t *methods = NULL;
+    xlator_t *subvol = 0;
+    xlator_t *hashed_subvol = 0;
+    int ret = 0;
+    int readdir_optimize = 0;
+    inode_table_t *itable = NULL;
+    inode_t *inode = NULL;
+    gf_boolean_t skip_hashed_check = _gf_false;
+
+    INIT_LIST_HEAD(&entries.list);
+
+    prev = cookie;
+    local = frame->local;
+    GF_VALIDATE_OR_GOTO(this->name, local->fd, unwind);
+
+    itable = local->fd->inode->table;
+
+    conf = this->private;
+    GF_VALIDATE_OR_GOTO(this->name, conf, unwind);
+
+    methods = &(conf->methods);
+
+    if (op_ret <= 0) {
+        goto done;
+    }
+
+    /* Why aren't we skipping DHT entirely in case of a single subvol?
+     * Because if this was a larger volume earlier and all but one subvol
+     * was removed, there might be stale linkto files on the subvol.
+     */
+    if (conf->subvolume_cnt == 1) {
+        /* return all directory and file entries except
+         * linkto files for a single child DHT
+         */
+        skip_hashed_check = _gf_true;
+    }
 
-        return 0;
+    if (!local->layout)
+        local->layout = dht_layout_get(this, local->fd->inode);
 
-err:
-        op_errno = (op_errno == -1) ? errno : op_errno;
-        DHT_STACK_UNWIND (fsetxattr, frame, -1, op_errno, NULL);
+    layout = local->layout;
 
-        return 0;
-}
+    /* This will skip the entries on the subvol without a layout,
+     * hence preventing the crash but rmdir might fail with
+     * "directory not empty" errors*/
 
+    if (layout == NULL)
+        goto done;
 
-static int
-dht_common_setxattr_cbk (call_frame_t *frame, void *cookie,
-                         xlator_t *this, int32_t op_ret, int32_t op_errno,
-                         dict_t *xdata)
-{
-        DHT_STACK_UNWIND (setxattr, frame, op_ret, op_errno, xdata);
+    if (conf->readdir_optimize == _gf_true)
+        readdir_optimize = 1;
 
-        return 0;
-}
+    gf_msg_debug(this->name, 0, "Processing entries from %s", prev->name);
 
-int
-dht_checking_pathinfo_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                           int op_ret, int op_errno, dict_t *xattr,
-                           dict_t *xdata)
-{
-        int           i     = -1;
-        int           ret   = -1;
-        char         *value = NULL;
-        dht_local_t  *local = NULL;
-        dht_conf_t   *conf  = NULL;
-        call_frame_t *prev  = NULL;
-        int           this_call_cnt = 0;
+    list_for_each_entry(orig_entry, (&orig_entries->list), list)
+    {
+        next_offset = orig_entry->d_off;
 
-        local = frame->local;
-        prev = cookie;
-        conf = this->private;
+        gf_msg_debug(this->name, 0, "%s: entry = %s, type = %d", prev->name,
+                     orig_entry->d_name, orig_entry->d_type);
 
-        if (op_ret == -1)
-                goto out;
+        if (IA_ISINVAL(orig_entry->d_stat.ia_type)) {
+            /*stat failed somewhere- display this entry but the data may
+             * be inaccurate.
+             */
+            gf_msg_debug(this->name, EINVAL, "Invalid stat for %s (gfid %s)",
+                         orig_entry->d_name,
+                         uuid_utoa(orig_entry->d_stat.ia_gfid));
+        }
 
+        if (check_is_linkfile(NULL, (&orig_entry->d_stat), orig_entry->dict,
+                              conf->link_xattr_name)) {
+            gf_msg_debug(this->name, 0, "%s: %s is a linkto file", prev->name,
+                         orig_entry->d_name);
+            continue;
+        }
 
-        ret = dict_get_str (xattr, GF_XATTR_PATHINFO_KEY, &value);
-        if (ret)
-                goto out;
+        if (skip_hashed_check) {
+            goto list;
+        }
 
-        if (!strcmp (value, local->key)) {
-                for (i = 0; i < conf->subvolume_cnt; i++) {
-                        if (conf->subvolumes[i] == prev->this)
-                                conf->decommissioned_bricks[i] = prev->this;
-                }
+        if (check_is_dir(NULL, (&orig_entry->d_stat), NULL)) {
+            /*Directory entries filtering :
+             * a) If rebalance is running, pick from first_up_subvol
+             * b) (rebalance not running)hashed subvolume is NULL or
+             * down then filter in first_up_subvolume. Other wise the
+             * corresponding hashed subvolume will take care of the
+             * directory entry.
+             */
+            if (readdir_optimize) {
+                if (prev == local->first_up_subvol)
+                    goto list;
+                else
+                    continue;
+            }
+
+            hashed_subvol = methods->layout_search(this, layout,
+                                                   orig_entry->d_name);
+
+            if (prev == hashed_subvol)
+                goto list;
+            if ((hashed_subvol && dht_subvol_status(conf, hashed_subvol)) ||
+                (prev != local->first_up_subvol))
+                continue;
+
+            goto list;
         }
 
-out:
-        this_call_cnt = dht_frame_return (frame);
-        if (is_last_call (this_call_cnt)) {
-                DHT_STACK_UNWIND (setxattr, frame, local->op_ret, ENOTSUP, NULL);
+    list:
+        entry = gf_dirent_for_name(orig_entry->d_name);
+        if (!entry) {
+            goto unwind;
         }
-        return 0;
 
-}
+        /* Do this if conf->search_unhashed is set to "auto" */
+        if (conf->search_unhashed == GF_DHT_LOOKUP_UNHASHED_AUTO) {
+            subvol = methods->layout_search(this, layout, orig_entry->d_name);
+            if (!subvol || (subvol != prev)) {
+                /* TODO: Count the number of entries which need
+                   linkfile to prove its existence in fs */
+                layout->search_unhashed++;
+            }
+        }
 
-int
-dht_setxattr (call_frame_t *frame, xlator_t *this,
-              loc_t *loc, dict_t *xattr, int flags, dict_t *xdata)
-{
-        xlator_t     *subvol   = NULL;
-        dht_local_t  *local    = NULL;
-        dht_conf_t   *conf     = NULL;
-        dht_methods_t *methods = NULL;
-        dht_layout_t *layout   = NULL;
-        int           i        = 0;
-        int           op_errno = EINVAL;
-        int           ret      = -1;
-        data_t       *tmp      = NULL;
-        uint32_t      dir_spread = 0;
-        char          value[4096] = {0,};
-        gf_dht_migrate_data_type_t forced_rebalance = GF_DHT_MIGRATE_DATA;
-        int           call_cnt = 0;
-
-        VALIDATE_OR_GOTO (frame, err);
-        VALIDATE_OR_GOTO (this, err);
-        VALIDATE_OR_GOTO (loc, err);
-        VALIDATE_OR_GOTO (loc->inode, err);
-
-        conf   = this->private;
-        GF_VALIDATE_OR_GOTO (this->name, conf, err);
-
-        methods = conf->methods;
-        GF_VALIDATE_OR_GOTO (this->name, conf->methods, err);
-
-        GF_IF_INTERNAL_XATTR_GOTO (conf->wild_xattr_name, xattr,
-                                   op_errno, err);
-
-        local = dht_local_init (frame, loc, NULL, GF_FOP_SETXATTR);
-        if (!local) {
-                op_errno = ENOMEM;
-                goto err;
+        entry->d_off = orig_entry->d_off;
+        entry->d_stat = orig_entry->d_stat;
+        entry->d_ino = orig_entry->d_ino;
+        entry->d_type = orig_entry->d_type;
+        entry->d_len = orig_entry->d_len;
+
+        if (orig_entry->dict)
+            entry->dict = dict_ref(orig_entry->dict);
+
+        /* making sure we set the inode ctx right with layout,
+           currently possible only for non-directories, so for
+           directories don't set entry inodes */
+        if (IA_ISDIR(entry->d_stat.ia_type)) {
+            entry->d_stat.ia_blocks = DHT_DIR_STAT_BLOCKS;
+            entry->d_stat.ia_size = DHT_DIR_STAT_SIZE;
+            if (orig_entry->inode) {
+                dht_inode_ctx_time_update(orig_entry->inode, this,
+                                          &entry->d_stat, 1);
+
+                if (conf->subvolume_cnt == 1) {
+                    dht_populate_inode_for_dentry(this, prev, entry,
+                                                  orig_entry);
+                }
+            }
+        } else {
+            if (orig_entry->dict &&
+                dict_get(orig_entry->dict, conf->link_xattr_name)) {
+                /* Strip out the S and T flags set by rebalance*/
+                DHT_STRIP_PHASE1_FLAGS(&entry->d_stat);
+            }
+
+            if (orig_entry->inode) {
+                ret = dht_layout_preset(this, prev, orig_entry->inode);
+                if (ret)
+                    gf_msg(this->name, GF_LOG_WARNING, 0,
+                           DHT_MSG_LAYOUT_SET_FAILED,
+                           "failed to link the layout "
+                           "in inode for %s",
+                           orig_entry->d_name);
+
+                entry->inode = inode_ref(orig_entry->inode);
+            } else if (itable) {
+                /*
+                 * orig_entry->inode might be null if any upper
+                 * layer xlators below client set to null, to
+                 * force a lookup on the inode even if the inode
+                 * is present in the inode table. In that case
+                 * we just update the ctx to make sure we didn't
+                 * missed anything.
+                 */
+                inode = inode_find(itable, orig_entry->d_stat.ia_gfid);
+                if (inode) {
+                    ret = dht_layout_preset(this, prev, inode);
+                    if (ret)
+                        gf_msg(this->name, GF_LOG_WARNING, 0,
+                               DHT_MSG_LAYOUT_SET_FAILED,
+                               "failed to link the layout"
+                               " in inode for %s",
+                               orig_entry->d_name);
+                    inode_unref(inode);
+                    inode = NULL;
+                }
+            }
         }
 
-        subvol = local->cached_subvol;
-        if (!subvol) {
-                gf_msg_debug (this->name, 0,
-                              "no cached subvolume for path=%s",
-                              loc->path);
-                op_errno = EINVAL;
-                goto err;
+        gf_msg_debug(this->name, 0, "%s: Adding entry = %s", prev->name,
+                     entry->d_name);
+
+        list_add_tail(&entry->list, &entries.list);
+        count++;
+    }
+
+done:
+
+    /* We need to ensure that only the last subvolume's end-of-directory
+     * notification is respected so that directory reading does not stop
+     * before all subvolumes have been read. That could happen because the
+     * posix for each subvolume sends a ENOENT on end-of-directory but in
+     * distribute we're not concerned only with a posix's view of the
+     * directory but the aggregated namespace' view of the directory.
+     * Possible values:
+     * op_ret == 0 and op_errno != 0
+     *   if op_errno != ENOENT : Error.Unwind.
+     *   if op_errno == ENOENT : There are no more entries on this subvol.
+     *                           Move to the next one.
+     * op_ret > 0 and count == 0 :
+     *    The subvol returned entries to dht but all were stripped out.
+     *    For example, if they were linkto files or dirs where
+     *    hashed_subvol != prev. Try to get some entries by winding
+     *    to the next subvol. This can be dangerous if parallel readdir
+     *    is enabled as it grows the stack.
+     *
+     * op_ret > 0 and count > 0:
+     *   We found some entries. Unwind even if the buffer is not full.
+     *
+     */
+
+    op_ret = count;
+    if (count == 0) {
+        /* non-zero next_offset means that
+         * EOF is not yet hit on the current subvol
+         */
+        if ((next_offset == 0) || (op_errno == ENOENT)) {
+            next_offset = 0;
+            next_subvol = dht_subvol_next(this, prev);
+        } else {
+            next_subvol = prev;
         }
 
-        layout = local->layout;
-        if (!layout) {
-                gf_msg_debug (this->name, 0,
-                              "no layout for path=%s", loc->path);
-                op_errno = EINVAL;
-                goto err;
+        if (!next_subvol) {
+            goto unwind;
         }
 
-        local->call_cnt = call_cnt = layout->cnt;
+        if (conf->readdir_optimize == _gf_true) {
+            if (next_subvol != local->first_up_subvol) {
+                ret = dict_set_int32(local->xattr, GF_READDIR_SKIP_DIRS, 1);
+                if (ret)
+                    gf_msg(this->name, GF_LOG_ERROR, 0, DHT_MSG_DICT_SET_FAILED,
+                           "Failed to set dictionary value"
+                           ":key = %s",
+                           GF_READDIR_SKIP_DIRS);
+            } else {
+                dict_del(local->xattr, GF_READDIR_SKIP_DIRS);
+            }
+        }
+
+        STACK_WIND_COOKIE(frame, dht_readdirp_cbk, next_subvol, next_subvol,
+                          next_subvol->fops->readdirp, local->fd, local->size,
+                          next_offset, local->xattr);
+        return 0;
+    }
 
-        tmp = dict_get (xattr, GF_XATTR_FILE_MIGRATE_KEY);
-        if (tmp) {
+unwind:
+    /* We need to ensure that only the last subvolume's end-of-directory
+     * notification is respected so that directory reading does not stop
+     * before all subvolumes have been read. That could happen because the
+     * posix for each subvolume sends a ENOENT on end-of-directory but in
+     * distribute we're not concerned only with a posix's view of the
+     * directory but the aggregated namespace' view of the directory.
+     */
+    if (op_ret < 0)
+        op_ret = 0;
+
+    if (prev != dht_last_up_subvol(this))
+        op_errno = 0;
+
+    DHT_STACK_UNWIND(readdirp, frame, op_ret, op_errno, &entries, NULL);
+
+    gf_dirent_free(&entries);
+    return 0;
+}
 
-                if (IA_ISDIR (loc->inode->ia_type)) {
-                        op_errno = ENOTSUP;
-                        goto err;
-                }
+static int
+dht_readdir_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int op_ret,
+                int op_errno, gf_dirent_t *orig_entries, dict_t *xdata)
+{
+    dht_local_t *local = NULL;
+    gf_dirent_t entries;
+    gf_dirent_t *orig_entry = NULL;
+    gf_dirent_t *entry = NULL;
+    xlator_t *prev = NULL;
+    xlator_t *next_subvol = NULL;
+    off_t next_offset = 0;
+    int count = 0;
+    dht_layout_t *layout = 0;
+    xlator_t *subvol = 0;
+    dht_conf_t *conf = NULL;
+    dht_methods_t *methods = NULL;
+    gf_boolean_t skip_hashed_check = _gf_false;
 
-                /* TODO: need to interpret the 'value' for more meaning
-                   (ie, 'target' subvolume given there, etc) */
-                memcpy (value, tmp->data, tmp->len);
-                if (strcmp (value, "force") == 0)
-                       forced_rebalance =
-                                GF_DHT_MIGRATE_DATA_EVEN_IF_LINK_EXISTS;
+    INIT_LIST_HEAD(&entries.list);
 
-                if (conf->decommission_in_progress)
-                        forced_rebalance = GF_DHT_MIGRATE_HARDLINK;
+    prev = cookie;
+    local = frame->local;
 
-                if (!loc->path) {
-                        op_errno = EINVAL;
-                        goto err;
-                }
+    conf = this->private;
+    GF_VALIDATE_OR_GOTO(this->name, conf, done);
 
-                if (!local->loc.name)
-                        local->loc.name = strrchr (local->loc.path, '/')+1;
+    methods = &(conf->methods);
 
-                if (!local->loc.parent)
-                        local->loc.parent =
-                                inode_parent(local->loc.inode, NULL, NULL);
+    if (op_ret <= 0)
+        goto done;
 
-                if ((!local->loc.name) || (!local->loc.parent)) {
-                        op_errno = EINVAL;
-                        goto err;
-                }
+    if (!local->layout)
+        local->layout = dht_layout_get(this, local->fd->inode);
 
-                methods->migration_get_dst_subvol(this, local);
+    layout = local->layout;
 
-                if (!local->rebalance.target_node) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                DHT_MSG_HASHED_SUBVOL_GET_FAILED,
-                                "Failed to get hashed subvol for %s",
-                                loc->path);
-                        op_errno = EINVAL;
-                        goto err;
-                }
+    gf_msg_debug(this->name, 0, "Processing entries from %s", prev->name);
 
-                local->rebalance.from_subvol = local->cached_subvol;
+    if (conf->subvolume_cnt == 1) {
+        /*return everything*/
+        skip_hashed_check = _gf_true;
+        count = op_ret;
+        goto done;
+    }
 
-                if (local->rebalance.target_node == local->rebalance.from_subvol) {
-                        op_errno = EEXIST;
-                        goto err;
-                }
-                if (local->rebalance.target_node) {
-                        local->flags = forced_rebalance;
+    list_for_each_entry(orig_entry, (&orig_entries->list), list)
+    {
+        next_offset = orig_entry->d_off;
 
-                        ret = dht_start_rebalance_task (this, frame);
-                        if (!ret)
-                                return 0;
+        gf_msg_debug(this->name, 0, "%s: entry = %s, type = %d", prev->name,
+                     orig_entry->d_name, orig_entry->d_type);
 
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                DHT_MSG_REBALANCE_START_FAILED,
-                                "%s: failed to create a new rebalance synctask",
-                                loc->path);
-                }
-                op_errno = EINVAL;
-                goto err;
+        subvol = methods->layout_search(this, layout, orig_entry->d_name);
 
-        }
+        if (!subvol || (subvol == prev)) {
+            entry = gf_dirent_for_name(orig_entry->d_name);
+            if (!entry) {
+                gf_msg(this->name, GF_LOG_ERROR, ENOMEM, DHT_MSG_NO_MEMORY,
+                       "Memory allocation failed ");
+                goto unwind;
+            }
 
-        tmp = dict_get (xattr, "decommission-brick");
-        if (tmp) {
-                /* This operation should happen only on '/' */
-                if (!__is_root_gfid (loc->inode->gfid)) {
-                        op_errno = ENOTSUP;
-                        goto err;
-                }
+            entry->d_off = orig_entry->d_off;
+            entry->d_ino = orig_entry->d_ino;
+            entry->d_type = orig_entry->d_type;
+            entry->d_len = orig_entry->d_len;
 
-                memcpy (value, tmp->data, ((tmp->len < 4095) ? tmp->len : 4095));
-                local->key = gf_strdup (value);
-                local->call_cnt = conf->subvolume_cnt;
+            gf_msg_debug(this->name, 0, "%s: Adding = entry %s", prev->name,
+                         entry->d_name);
 
-                for (i = 0 ; i < conf->subvolume_cnt; i++) {
-                        /* Get the pathinfo, and then compare */
-                        STACK_WIND (frame, dht_checking_pathinfo_cbk,
-                                    conf->subvolumes[i],
-                                    conf->subvolumes[i]->fops->getxattr,
-                                    loc, GF_XATTR_PATHINFO_KEY, NULL);
-                }
-                return 0;
+            list_add_tail(&entry->list, &entries.list);
+            count++;
+        }
+    }
+done:
+    op_ret = count;
+    /* We need to ensure that only the last subvolume's end-of-directory
+     * notification is respected so that directory reading does not stop
+     * before all subvolumes have been read. That could happen because the
+     * posix for each subvolume sends a ENOENT on end-of-directory but in
+     * distribute we're not concerned only with a posix's view of the
+     * directory but the aggregated namespace' view of the directory.
+     */
+    if (count == 0) {
+        if ((next_offset == 0) || (op_errno == ENOENT)) {
+            next_offset = 0;
+            next_subvol = dht_subvol_next(this, prev);
+        } else {
+            next_subvol = prev;
         }
 
-        tmp = dict_get (xattr, GF_XATTR_FIX_LAYOUT_KEY);
-        if (tmp) {
-                gf_log (this->name, GF_LOG_INFO,
-                        "fixing the layout of %s", loc->path);
+        if (!next_subvol) {
+            goto unwind;
+        }
 
-                ret = dht_fix_directory_layout (frame, dht_common_setxattr_cbk,
-                                                layout);
-                if (ret) {
-                        op_errno = ENOTCONN;
-                        goto err;
+        STACK_WIND_COOKIE(frame, dht_readdir_cbk, next_subvol, next_subvol,
+                          next_subvol->fops->readdir, local->fd, local->size,
+                          next_offset, NULL);
+        return 0;
+    }
+
+unwind:
+    /* We need to ensure that only the last subvolume's end-of-directory
+     * notification is respected so that directory reading does not stop
+     * before all subvolumes have been read. That could happen because the
+     * posix for each subvolume sends a ENOENT on end-of-directory but in
+     * distribute we're not concerned only with a posix's view of the
+     * directory but the aggregated namespace' view of the directory.
+     */
+
+    if (prev != dht_last_up_subvol(this))
+        op_errno = 0;
+
+    if (!skip_hashed_check) {
+        DHT_STACK_UNWIND(readdir, frame, op_ret, op_errno, &entries, NULL);
+        gf_dirent_free(&entries);
+
+    } else {
+        DHT_STACK_UNWIND(readdir, frame, op_ret, op_errno, orig_entries, NULL);
+    }
+    return 0;
+}
+
+static int
+dht_do_readdir(call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
+               off_t yoff, int whichop, dict_t *dict)
+{
+    dht_local_t *local = NULL;
+    int op_errno = -1;
+    xlator_t *xvol = NULL;
+    int ret = 0;
+    dht_conf_t *conf = NULL;
+
+    VALIDATE_OR_GOTO(frame, err);
+    VALIDATE_OR_GOTO(this, err);
+    VALIDATE_OR_GOTO(fd, err);
+    VALIDATE_OR_GOTO(this->private, err);
+
+    conf = this->private;
+
+    local = dht_local_init(frame, NULL, NULL, whichop);
+    if (!local) {
+        op_errno = ENOMEM;
+        goto err;
+    }
+
+    local->fd = fd_ref(fd);
+    local->size = size;
+    local->xattr_req = (dict) ? dict_ref(dict) : NULL;
+    local->first_up_subvol = dht_first_up_subvol(this);
+    local->op_ret = -1;
+
+    dht_deitransform(this, yoff, &xvol);
+
+    /* TODO: do proper readdir */
+    if (whichop == GF_FOP_READDIRP) {
+        if (dict)
+            local->xattr = dict_ref(dict);
+        else
+            local->xattr = dict_new();
+
+        if (local->xattr) {
+            ret = dict_set_uint32(local->xattr, conf->link_xattr_name, 256);
+            if (ret)
+                gf_msg(this->name, GF_LOG_WARNING, 0, DHT_MSG_DICT_SET_FAILED,
+                       "Failed to set dictionary value"
+                       " : key = %s",
+                       conf->link_xattr_name);
+
+            if (conf->readdir_optimize == _gf_true) {
+                if (xvol != local->first_up_subvol) {
+                    ret = dict_set_int32(local->xattr, GF_READDIR_SKIP_DIRS, 1);
+                    if (ret)
+                        gf_msg(this->name, GF_LOG_ERROR, 0,
+                               DHT_MSG_DICT_SET_FAILED,
+                               "Failed to set "
+                               "dictionary value: "
+                               "key = %s",
+                               GF_READDIR_SKIP_DIRS);
+                } else {
+                    dict_del(local->xattr, GF_READDIR_SKIP_DIRS);
                 }
-                return ret;
-        }
+            }
 
-        tmp = dict_get (xattr, "distribute.directory-spread-count");
-        if (tmp) {
-                /* Setxattr value is packed as 'binary', not string */
-                memcpy (value, tmp->data, ((tmp->len < 4095)?tmp->len:4095));
-                ret = gf_string2uint32 (value, &dir_spread);
-                if (!ret && ((dir_spread <= conf->subvolume_cnt) &&
-                             (dir_spread > 0))) {
-                        layout->spread_cnt = dir_spread;
-
-                        ret = dht_fix_directory_layout (frame,
-                                                        dht_common_setxattr_cbk,
-                                                        layout);
-                        if (ret) {
-                                op_errno = ENOTCONN;
-                                goto err;
-                        }
-                        return ret;
+            if (conf->subvolume_cnt == 1) {
+                ret = dict_set_uint32(local->xattr, conf->xattr_name, 4 * 4);
+                if (ret) {
+                    gf_msg(this->name, GF_LOG_WARNING, ENOMEM,
+                           DHT_MSG_DICT_SET_FAILED,
+                           "Failed to set dictionary "
+                           "value:key = %s ",
+                           conf->xattr_name);
                 }
-                gf_log (this->name, GF_LOG_ERROR,
-                        "wrong 'directory-spread-count' value (%s)", value);
-                op_errno = ENOTSUP;
-                goto err;
+            }
         }
 
-        for (i = 0; i < call_cnt; i++) {
-                STACK_WIND (frame, dht_err_cbk,
-                            layout->list[i].xlator,
-                            layout->list[i].xlator->fops->setxattr,
-                            loc, xattr, flags, xdata);
-        }
+        STACK_WIND_COOKIE(frame, dht_readdirp_cbk, xvol, xvol,
+                          xvol->fops->readdirp, fd, size, yoff, local->xattr);
+    } else {
+        STACK_WIND_COOKIE(frame, dht_readdir_cbk, xvol, xvol,
+                          xvol->fops->readdir, fd, size, yoff, local->xattr);
+    }
 
-        return 0;
+    return 0;
 
 err:
-        op_errno = (op_errno == -1) ? errno : op_errno;
-        DHT_STACK_UNWIND (setxattr, frame, -1, op_errno, NULL);
+    op_errno = (op_errno == -1) ? errno : op_errno;
+    DHT_STACK_UNWIND(readdir, frame, -1, op_errno, NULL, NULL);
 
-        return 0;
+    return 0;
 }
 
-
 int
-dht_removexattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                     int op_ret, int op_errno, dict_t *xdata)
+dht_readdir(call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
+            off_t yoff, dict_t *xdata)
 {
-        dht_local_t  *local = NULL;
-        int           this_call_cnt = 0;
-        call_frame_t *prev = NULL;
+    int op = GF_FOP_READDIR;
+    dht_conf_t *conf = NULL;
+    int i = 0;
 
-        local = frame->local;
-        prev = cookie;
-
-        LOCK (&frame->lock);
-        {
-                if (op_ret == -1) {
-                        local->op_errno = op_errno;
-                        gf_msg_debug (this->name, 0,
-                                      "subvolume %s returned -1 (%s)",
-                                      prev->this->name, strerror (op_errno));
-                        goto unlock;
-                }
+    conf = this->private;
+    if (!conf)
+        goto out;
 
-                local->op_ret = 0;
+    for (i = 0; i < conf->subvolume_cnt; i++) {
+        if (!conf->subvolume_status[i]) {
+            op = GF_FOP_READDIRP;
+            break;
         }
-unlock:
-        UNLOCK (&frame->lock);
+    }
 
-        this_call_cnt = dht_frame_return (frame);
-        if (is_last_call (this_call_cnt)) {
-                DHT_STACK_UNWIND (removexattr, frame, local->op_ret,
-                                  local->op_errno, NULL);
-        }
+    if (conf->use_readdirp)
+        op = GF_FOP_READDIRP;
 
-        return 0;
+out:
+    dht_do_readdir(frame, this, fd, size, yoff, op, 0);
+    return 0;
 }
 
-
 int
-dht_removexattr (call_frame_t *frame, xlator_t *this,
-                 loc_t *loc, const char *key, dict_t *xdata)
+dht_readdirp(call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
+             off_t yoff, dict_t *dict)
 {
-        xlator_t     *subvol = NULL;
-        int           op_errno = -1;
-        dht_local_t  *local = NULL;
-        dht_layout_t *layout = NULL;
-        int           call_cnt = 0;
-        dht_conf_t   *conf = NULL;
-        int i;
+    dht_do_readdir(frame, this, fd, size, yoff, GF_FOP_READDIRP, dict);
+    return 0;
+}
 
-        VALIDATE_OR_GOTO (this, err);
-        VALIDATE_OR_GOTO (this->private, err);
+static int
+dht_fsyncdir_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int op_ret,
+                 int op_errno, dict_t *xdata)
+{
+    dht_local_t *local = NULL;
+    int this_call_cnt = 0;
 
-        conf = this->private;
+    local = frame->local;
 
-        GF_IF_NATIVE_XATTR_GOTO (conf->wild_xattr_name, key, op_errno, err);
+    LOCK(&frame->lock);
+    {
+        if (op_ret == -1)
+            local->op_errno = op_errno;
+        else if (op_ret == 0)
+            local->op_ret = 0;
+    }
+    UNLOCK(&frame->lock);
+    this_call_cnt = dht_frame_return(frame);
+    if (is_last_call(this_call_cnt))
+        DHT_STACK_UNWIND(fsyncdir, frame, local->op_ret, local->op_errno,
+                         xdata);
+
+    return 0;
+}
 
-        VALIDATE_OR_GOTO (frame, err);
-        VALIDATE_OR_GOTO (loc, err);
-        VALIDATE_OR_GOTO (loc->inode, err);
+int
+dht_fsyncdir(call_frame_t *frame, xlator_t *this, fd_t *fd, int datasync,
+             dict_t *xdata)
+{
+    dht_local_t *local = NULL;
+    dht_conf_t *conf = NULL;
+    int op_errno = -1;
+    int i = -1;
 
-        local = dht_local_init (frame, loc, NULL, GF_FOP_REMOVEXATTR);
-        if (!local) {
-                op_errno = ENOMEM;
-                goto err;
-        }
+    VALIDATE_OR_GOTO(frame, err);
+    VALIDATE_OR_GOTO(this, err);
+    VALIDATE_OR_GOTO(fd, err);
+    VALIDATE_OR_GOTO(this->private, err);
 
-        subvol = local->cached_subvol;
-        if (!subvol) {
-                gf_msg_debug (this->name, 0,
-                              "no cached subvolume for path=%s", loc->path);
-                op_errno = EINVAL;
-                goto err;
-        }
+    conf = this->private;
 
-        layout = local->layout;
-        if (!local->layout) {
-                gf_msg_debug (this->name, 0,
-                              "no layout for path=%s", loc->path);
-                op_errno = EINVAL;
-                goto err;
-        }
+    local = dht_local_init(frame, NULL, NULL, GF_FOP_FSYNCDIR);
+    if (!local) {
+        op_errno = ENOMEM;
+        goto err;
+    }
 
-        local->call_cnt = call_cnt = layout->cnt;
-        local->key = gf_strdup (key);
+    local->fd = fd_ref(fd);
+    local->call_cnt = conf->subvolume_cnt;
 
-        for (i = 0; i < call_cnt; i++) {
-                STACK_WIND (frame, dht_removexattr_cbk,
-                            layout->list[i].xlator,
-                            layout->list[i].xlator->fops->removexattr,
-                            loc, key, NULL);
-        }
+    for (i = 0; i < conf->subvolume_cnt; i++) {
+        STACK_WIND(frame, dht_fsyncdir_cbk, conf->subvolumes[i],
+                   conf->subvolumes[i]->fops->fsyncdir, fd, datasync, xdata);
+    }
 
-        return 0;
+    return 0;
 
 err:
-        op_errno = (op_errno == -1) ? errno : op_errno;
-        DHT_STACK_UNWIND (removexattr, frame, -1, op_errno, NULL);
+    op_errno = (op_errno == -1) ? errno : op_errno;
+    DHT_STACK_UNWIND(fsyncdir, frame, -1, op_errno, NULL);
 
-        return 0;
+    return 0;
 }
 
 int
-dht_fremovexattr (call_frame_t *frame, xlator_t *this,
-                  fd_t *fd, const char *key, dict_t *xdata)
+dht_newfile_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int op_ret,
+                int op_errno, inode_t *inode, struct iatt *stbuf,
+                struct iatt *preparent, struct iatt *postparent, dict_t *xdata)
 {
-        xlator_t     *subvol = NULL;
-        int           op_errno = -1;
-        dht_local_t  *local = NULL;
-        dht_layout_t *layout = NULL;
-        int           call_cnt = 0;
-        dht_conf_t   *conf = 0;
-
-        int i;
+    xlator_t *prev = NULL;
+    int ret = -1;
+    dht_local_t *local = NULL;
+
+    if (op_ret == -1)
+        goto out;
+
+    local = frame->local;
+    if (!local) {
+        op_ret = -1;
+        op_errno = EINVAL;
+        goto out;
+    }
+
+    prev = cookie;
+
+    if (local->loc.parent) {
+        dht_inode_ctx_time_update(local->loc.parent, this, preparent, 0);
+        dht_inode_ctx_time_update(local->loc.parent, this, postparent, 1);
+    }
+
+    ret = dht_layout_preset(this, prev, inode);
+    if (ret < 0) {
+        gf_msg_debug(this->name, EINVAL,
+                     "could not set pre-set layout for subvolume %s",
+                     prev ? prev->name : NULL);
+        op_ret = -1;
+        op_errno = EINVAL;
+        goto out;
+    }
+    if (local->linked == _gf_true)
+        dht_linkfile_attr_heal(frame, this);
+out:
+    /*
+     * FIXME: ia_size and st_blocks of preparent and postparent do not have
+     * correct values. since, preparent and postparent buffers correspond
+     * to a directory these two members should have values equal to sum of
+     * corresponding values from each of the subvolume.
+     * See dht_iatt_merge for reference.
+     */
+    DHT_STRIP_PHASE1_FLAGS(stbuf);
+    dht_set_fixed_dir_stat(postparent);
+    dht_set_fixed_dir_stat(preparent);
+
+    if (local && local->lock[0].layout.parent_layout.locks) {
+        /* store op_errno for failure case*/
+        local->op_errno = op_errno;
+        local->refresh_layout_unlock(frame, this, op_ret, 1);
 
-        VALIDATE_OR_GOTO (this, err);
-        VALIDATE_OR_GOTO (this->private, err);
+        if (op_ret == 0) {
+            DHT_STACK_UNWIND(mknod, frame, op_ret, op_errno, inode, stbuf,
+                             preparent, postparent, xdata);
+        }
+    } else {
+        DHT_STACK_UNWIND(mknod, frame, op_ret, op_errno, inode, stbuf,
+                         preparent, postparent, xdata);
+    }
 
-        conf = this->private;
+    return 0;
+}
 
-        GF_IF_NATIVE_XATTR_GOTO (conf->wild_xattr_name, key, op_errno, err);
+static int
+dht_mknod_linkfile_create_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                              int32_t op_ret, int32_t op_errno, inode_t *inode,
+                              struct iatt *stbuf, struct iatt *preparent,
+                              struct iatt *postparent, dict_t *xdata)
+{
+    dht_local_t *local = NULL;
+    xlator_t *cached_subvol = NULL;
+    dht_conf_t *conf = NULL;
 
-        VALIDATE_OR_GOTO (frame, err);
+    local = frame->local;
 
-        local = dht_local_init (frame, NULL, fd, GF_FOP_FREMOVEXATTR);
-        if (!local) {
-                op_errno = ENOMEM;
-                goto err;
-        }
+    if (!local || !local->cached_subvol) {
+        op_errno = EINVAL;
+        goto err;
+    }
 
-        subvol = local->cached_subvol;
-        if (!subvol) {
-                gf_msg_debug (this->name, 0,
-                              "no cached subvolume for inode=%s",
-                              uuid_utoa (fd->inode->gfid));
-                op_errno = EINVAL;
-                goto err;
-        }
+    if (op_ret == -1) {
+        local->op_errno = op_errno;
+        goto err;
+    }
 
-        layout = local->layout;
-        if (!local->layout) {
-                gf_msg_debug (this->name, 0,
-                              "no layout for inode=%s",
-                              uuid_utoa (fd->inode->gfid));
-                op_errno = EINVAL;
-                goto err;
-        }
+    conf = this->private;
+    if (!conf) {
+        local->op_errno = EINVAL;
+        op_errno = EINVAL;
+        goto err;
+    }
 
-        local->call_cnt = call_cnt = layout->cnt;
-        local->key = gf_strdup (key);
+    cached_subvol = local->cached_subvol;
 
-        for (i = 0; i < call_cnt; i++) {
-                STACK_WIND (frame, dht_removexattr_cbk,
-                            layout->list[i].xlator,
-                            layout->list[i].xlator->fops->fremovexattr,
-                            fd, key, NULL);
-        }
+    if (local->params) {
+        dict_del(local->params, conf->link_xattr_name);
+        dict_del(local->params, GLUSTERFS_INTERNAL_FOP_KEY);
+    }
 
-        return 0;
+    STACK_WIND_COOKIE(frame, dht_newfile_cbk, (void *)cached_subvol,
+                      cached_subvol, cached_subvol->fops->mknod, &local->loc,
+                      local->mode, local->rdev, local->umask, local->params);
 
+    return 0;
 err:
-        op_errno = (op_errno == -1) ? errno : op_errno;
-        DHT_STACK_UNWIND (fremovexattr, frame, -1, op_errno, NULL);
-
-        return 0;
+    if (local && local->lock[0].layout.parent_layout.locks) {
+        local->refresh_layout_unlock(frame, this, -1, 1);
+    } else {
+        DHT_STACK_UNWIND(mknod, frame, -1, op_errno, NULL, NULL, NULL, NULL,
+                         NULL);
+    }
+    return 0;
 }
 
-
-int
-dht_fd_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-            int op_ret, int op_errno, fd_t *fd, dict_t *xdata)
+static int
+dht_mknod_wind_to_avail_subvol(call_frame_t *frame, xlator_t *this,
+                               xlator_t *subvol, loc_t *loc, dev_t rdev,
+                               mode_t mode, mode_t umask, dict_t *params)
 {
-        dht_local_t  *local = NULL;
-        int           this_call_cnt = 0;
-        call_frame_t *prev = NULL;
+    dht_local_t *local = NULL;
+    xlator_t *avail_subvol = NULL;
 
-        local = frame->local;
-        prev = cookie;
-
-        LOCK (&frame->lock);
-        {
-                if (op_ret == -1) {
-                        local->op_errno = op_errno;
-                        gf_msg_debug (this->name, 0,
-                                      "subvolume %s returned -1 (%s)",
-                                      prev->this->name, strerror (op_errno));
-                        goto unlock;
-                }
+    local = frame->local;
 
-                local->op_ret = 0;
-        }
-unlock:
-        UNLOCK (&frame->lock);
+    if (!dht_is_subvol_filled(this, subvol)) {
+        gf_msg_debug(this->name, 0, "creating %s on %s", loc->path,
+                     subvol->name);
 
-        this_call_cnt = dht_frame_return (frame);
-        if (is_last_call (this_call_cnt))
-                DHT_STACK_UNWIND (open, frame, local->op_ret, local->op_errno,
-                                  local->fd, NULL);
+        STACK_WIND_COOKIE(frame, dht_newfile_cbk, (void *)subvol, subvol,
+                          subvol->fops->mknod, loc, mode, rdev, umask, params);
+    } else {
+        avail_subvol = dht_free_disk_available_subvol(this, subvol, local);
 
-        return 0;
-}
+        if (avail_subvol != subvol) {
+            local->params = dict_ref(params);
+            local->rdev = rdev;
+            local->mode = mode;
+            local->umask = umask;
+            local->cached_subvol = avail_subvol;
+            local->hashed_subvol = subvol;
 
-/*
- * dht_normalize_stats -
- */
-static void
-dht_normalize_stats (struct statvfs *buf, unsigned long bsize,
-                     unsigned long frsize)
-{
-        double factor = 0;
+            gf_msg_debug(this->name, 0, "creating %s on %s (link at %s)",
+                         loc->path, avail_subvol->name, subvol->name);
 
-        if (buf->f_bsize != bsize) {
-                buf->f_bsize = bsize;
+            dht_linkfile_create(frame, dht_mknod_linkfile_create_cbk, this,
+                                avail_subvol, subvol, loc);
+
+            goto out;
         }
 
-        if (buf->f_frsize != frsize) {
-                factor = ((double) buf->f_frsize) / frsize;
-                buf->f_frsize = frsize;
-                buf->f_blocks = (fsblkcnt_t) (factor * buf->f_blocks);
-                buf->f_bfree  = (fsblkcnt_t) (factor * buf->f_bfree);
-                buf->f_bavail = (fsblkcnt_t) (factor * buf->f_bavail);
+        gf_msg_debug(this->name, 0, "creating %s on %s", loc->path,
+                     subvol->name);
 
-        }
+        STACK_WIND_COOKIE(frame, dht_newfile_cbk, (void *)subvol, subvol,
+                          subvol->fops->mknod, loc, mode, rdev, umask, params);
+    }
+out:
+    return 0;
 }
 
-int
-dht_statfs_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                int op_ret, int op_errno, struct statvfs *statvfs,
-                dict_t *xdata)
+static int32_t
+dht_mknod_do(call_frame_t *frame)
 {
+    dht_local_t *local = NULL;
+    dht_layout_t *refreshed = NULL;
+    xlator_t *subvol = NULL;
+    xlator_t *this = NULL;
+    dht_conf_t *conf = NULL;
+    dht_methods_t *methods = NULL;
 
-        gf_boolean_t            event              = _gf_false;
-        qdstatfs_action_t       action             = qdstatfs_action_OFF;
-        dht_local_t *           local              = NULL;
-        int                     this_call_cnt      = 0;
-        int                     bsize              = 0;
-        int                     frsize             = 0;
-        GF_UNUSED int           ret                = 0;
-        unsigned long           new_usage          = 0;
-        unsigned long           cur_usage          = 0;
+    local = frame->local;
 
-        local = frame->local;
-        GF_ASSERT (local);
+    this = THIS;
 
-        if (xdata)
-                ret = dict_get_int8 (xdata, "quota-deem-statfs",
-                                     (int8_t *)&event);
-
-        LOCK (&frame->lock);
-        {
-                if (op_ret == -1) {
-                        local->op_errno = op_errno;
-                        goto unlock;
-                }
-                if (!statvfs) {
-                        op_errno = EINVAL;
-                        local->op_ret = -1;
-                        goto unlock;
-                }
-                local->op_ret = 0;
+    conf = this->private;
 
-                switch (local->quota_deem_statfs) {
-                case _gf_true:
-                        if (event == _gf_true)
-                                action = qdstatfs_action_COMPARE;
-                        else
-                                action = qdstatfs_action_NEGLECT;
-                        break;
-
-                case _gf_false:
-                        if (event == _gf_true) {
-                                action = qdstatfs_action_REPLACE;
-                                local->quota_deem_statfs = _gf_true;
-                        }
-                        break;
+    GF_VALIDATE_OR_GOTO(this->name, conf, err);
 
-                default:
-                        gf_log (this->name, GF_LOG_ERROR, "Encountered third "
-                                "value for boolean variable %d",
-                                local->quota_deem_statfs);
-                        break;
-                }
+    methods = &(conf->methods);
 
-                if (local->quota_deem_statfs) {
-                        switch (action) {
-                        case qdstatfs_action_NEGLECT:
-                                goto unlock;
+    /* We don't need parent_loc anymore */
+    loc_wipe(&local->loc);
 
-                        case qdstatfs_action_REPLACE:
-                                local->statvfs = *statvfs;
-                                goto unlock;
+    loc_copy(&local->loc, &local->loc2);
 
-                        case qdstatfs_action_COMPARE:
-                                new_usage = statvfs->f_blocks -
-                                             statvfs->f_bfree;
-                                cur_usage = local->statvfs.f_blocks -
-                                             local->statvfs.f_bfree;
+    loc_wipe(&local->loc2);
 
-                                /* Take the max of the usage from subvols */
-                                if (new_usage >= cur_usage)
-                                        local->statvfs = *statvfs;
-                                goto unlock;
+    refreshed = local->selfheal.refreshed_layout;
 
-                        default:
-                                break;
-                        }
-                }
+    subvol = methods->layout_search(this, refreshed, local->loc.name);
 
-                if (local->statvfs.f_bsize != 0) {
-                        bsize = max(local->statvfs.f_bsize, statvfs->f_bsize);
-                        frsize = max(local->statvfs.f_frsize, statvfs->f_frsize);
-                        dht_normalize_stats(&local->statvfs, bsize, frsize);
-                        dht_normalize_stats(statvfs, bsize, frsize);
-                } else {
-                        local->statvfs.f_bsize    = statvfs->f_bsize;
-                        local->statvfs.f_frsize   = statvfs->f_frsize;
-                }
+    if (!subvol) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, DHT_MSG_HASHED_SUBVOL_GET_FAILED,
+               "no subvolume in "
+               "layout for path=%s",
+               local->loc.path);
+        local->op_errno = ENOENT;
+        goto err;
+    }
 
-                local->statvfs.f_blocks  += statvfs->f_blocks;
-                local->statvfs.f_bfree   += statvfs->f_bfree;
-                local->statvfs.f_bavail  += statvfs->f_bavail;
-                local->statvfs.f_files   += statvfs->f_files;
-                local->statvfs.f_ffree   += statvfs->f_ffree;
-                local->statvfs.f_favail  += statvfs->f_favail;
-                local->statvfs.f_fsid     = statvfs->f_fsid;
-                local->statvfs.f_flag     = statvfs->f_flag;
-                local->statvfs.f_namemax  = statvfs->f_namemax;
+    dht_mknod_wind_to_avail_subvol(frame, this, subvol, &local->loc,
+                                   local->rdev, local->mode, local->umask,
+                                   local->params);
+    return 0;
+err:
+    local->refresh_layout_unlock(frame, this, -1, 1);
 
+    return 0;
+}
 
-        }
-unlock:
-        UNLOCK (&frame->lock);
+static int32_t
+dht_mknod_unlock_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                     int32_t op_ret, int32_t op_errno, dict_t *xdata)
+{
+    DHT_STACK_DESTROY(frame);
+    return 0;
+}
 
+static int32_t
+dht_mknod_finish(call_frame_t *frame, xlator_t *this, int op_ret,
+                 int invoke_cbk)
+{
+    dht_local_t *local = NULL, *lock_local = NULL;
+    call_frame_t *lock_frame = NULL;
+    int lock_count = 0;
+
+    local = frame->local;
+    lock_count = dht_lock_count(local->lock[0].layout.parent_layout.locks,
+                                local->lock[0].layout.parent_layout.lk_count);
+    if (lock_count == 0)
+        goto done;
+
+    lock_frame = copy_frame(frame);
+    if (lock_frame == NULL) {
+        goto done;
+    }
+
+    lock_local = dht_local_init(lock_frame, &local->loc, NULL,
+                                lock_frame->root->op);
+    if (lock_local == NULL) {
+        goto done;
+    }
+
+    lock_local->lock[0]
+        .layout.parent_layout.locks = local->lock[0].layout.parent_layout.locks;
+    lock_local->lock[0].layout.parent_layout.lk_count =
+        local->lock[0].layout.parent_layout.lk_count;
+
+    local->lock[0].layout.parent_layout.locks = NULL;
+    local->lock[0].layout.parent_layout.lk_count = 0;
+
+    dht_unlock_inodelk(lock_frame,
+                       lock_local->lock[0].layout.parent_layout.locks,
+                       lock_local->lock[0].layout.parent_layout.lk_count,
+                       dht_mknod_unlock_cbk);
+    lock_frame = NULL;
 
-        this_call_cnt = dht_frame_return (frame);
-        if (is_last_call (this_call_cnt))
-                DHT_STACK_UNWIND (statfs, frame, local->op_ret, local->op_errno,
-                                  &local->statvfs, xdata);
+done:
+    if (lock_frame != NULL) {
+        DHT_STACK_DESTROY(lock_frame);
+    }
 
+    if (op_ret == 0)
         return 0;
-}
 
+    DHT_STACK_UNWIND(mknod, frame, op_ret, local->op_errno, NULL, NULL, NULL,
+                     NULL, NULL);
+    return 0;
+}
 
-int
-dht_statfs (call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xdata)
+static int32_t
+dht_mknod_lock_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                   int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
-        xlator_t     *subvol = NULL;
-        dht_local_t  *local  = NULL;
-        dht_conf_t   *conf = NULL;
-        int           op_errno = -1;
-        int           i = -1;
+    dht_local_t *local = NULL;
 
-        VALIDATE_OR_GOTO (frame, err);
-        VALIDATE_OR_GOTO (this, err);
-        VALIDATE_OR_GOTO (loc, err);
-        VALIDATE_OR_GOTO (this->private, err);
+    local = frame->local;
 
-        conf = this->private;
+    if (!local) {
+        goto err;
+    }
 
-        local = dht_local_init (frame, NULL, NULL, GF_FOP_STATFS);
-        if (!local) {
-                op_errno = ENOMEM;
-                goto err;
-        }
+    if (op_ret < 0) {
+        gf_msg("DHT", GF_LOG_ERROR, 0, DHT_MSG_INODE_LK_ERROR,
+               "mknod lock failed for file: %s", local->loc2.name);
 
-        if (!loc->inode || IA_ISDIR (loc->inode->ia_type)) {
-                local->call_cnt = conf->subvolume_cnt;
-
-                for (i = 0; i < conf->subvolume_cnt; i++) {
-                        STACK_WIND (frame, dht_statfs_cbk,
-                                    conf->subvolumes[i],
-                                    conf->subvolumes[i]->fops->statfs, loc,
-                                    xdata);
-                }
-                return 0;
-        }
+        local->op_errno = op_errno;
 
-        subvol = dht_subvol_get_cached (this, loc->inode);
-        if (!subvol) {
-                gf_msg_debug (this->name, 0,
-                              "no cached subvolume for path=%s", loc->path);
-                op_errno = EINVAL;
-                goto err;
-        }
+        goto err;
+    }
 
-        local->call_cnt = 1;
+    local->refresh_layout_unlock = dht_mknod_finish;
 
-        STACK_WIND (frame, dht_statfs_cbk,
-                    subvol, subvol->fops->statfs, loc, xdata);
+    local->refresh_layout_done = dht_mknod_do;
 
-        return 0;
+    dht_refresh_layout(frame);
 
+    return 0;
 err:
-        op_errno = (op_errno == -1) ? errno : op_errno;
-        DHT_STACK_UNWIND (statfs, frame, -1, op_errno, NULL, NULL);
-
-        return 0;
+    if (local)
+        dht_mknod_finish(frame, this, -1, 0);
+    else
+        DHT_STACK_UNWIND(mknod, frame, -1, EINVAL, NULL, NULL, NULL, NULL,
+                         NULL);
+    return 0;
 }
 
-
-int
-dht_opendir (call_frame_t *frame, xlator_t *this, loc_t *loc, fd_t *fd,
-             dict_t *xdata)
+static int32_t
+dht_mknod_lock(call_frame_t *frame, xlator_t *subvol)
 {
-        dht_local_t  *local  = NULL;
-        dht_conf_t   *conf = NULL;
-        int           op_errno = -1;
-        int           i = -1;
+    dht_local_t *local = NULL;
+    int count = 1, ret = -1;
+    dht_lock_t **lk_array = NULL;
 
-        VALIDATE_OR_GOTO (frame, err);
-        VALIDATE_OR_GOTO (this, err);
-        VALIDATE_OR_GOTO (fd, err);
-        VALIDATE_OR_GOTO (this->private, err);
+    GF_VALIDATE_OR_GOTO("dht", frame, err);
+    GF_VALIDATE_OR_GOTO(frame->this->name, frame->local, err);
 
-        conf = this->private;
+    local = frame->local;
 
-        local = dht_local_init (frame, loc, fd, GF_FOP_OPENDIR);
-        if (!local) {
-                op_errno = ENOMEM;
+    lk_array = GF_CALLOC(count, sizeof(*lk_array), gf_common_mt_pointer);
 
-                goto err;
-        }
+    if (lk_array == NULL)
+        goto err;
 
-        local->call_cnt = conf->subvolume_cnt;
+    lk_array[0] = dht_lock_new(frame->this, subvol, &local->loc, F_RDLCK,
+                               DHT_LAYOUT_HEAL_DOMAIN, NULL,
+                               IGNORE_ENOENT_ESTALE);
 
-        for (i = 0; i < conf->subvolume_cnt; i++) {
-                STACK_WIND (frame, dht_fd_cbk,
-                            conf->subvolumes[i],
-                            conf->subvolumes[i]->fops->opendir,
-                            loc, fd, xdata);
-        }
+    if (lk_array[0] == NULL)
+        goto err;
 
-        return 0;
-
-err:
-        op_errno = (op_errno == -1) ? errno : op_errno;
-        DHT_STACK_UNWIND (opendir, frame, -1, op_errno, NULL, NULL);
+    local->lock[0].layout.parent_layout.locks = lk_array;
+    local->lock[0].layout.parent_layout.lk_count = count;
 
-        return 0;
-}
+    ret = dht_blocking_inodelk(frame, lk_array, count, dht_mknod_lock_cbk);
 
-int
-dht_readdirp_cbk (call_frame_t *frame, void *cookie, xlator_t *this, int op_ret,
-                  int op_errno, gf_dirent_t *orig_entries, dict_t *xdata)
-{
-        dht_local_t  *local = NULL;
-        gf_dirent_t   entries;
-        gf_dirent_t  *orig_entry = NULL;
-        gf_dirent_t  *entry = NULL;
-        call_frame_t *prev = NULL;
-        xlator_t     *next_subvol = NULL;
-        off_t         next_offset = 0;
-        int           count = 0;
-        dht_layout_t *layout = 0;
-        dht_conf_t   *conf   = NULL;
-        dht_methods_t *methods = NULL;
-        xlator_t     *subvol = 0;
-        xlator_t     *hashed_subvol = 0;
-        int           ret    = 0;
-        int           readdir_optimize = 0;
-
-        INIT_LIST_HEAD (&entries.list);
-        prev = cookie;
-        local = frame->local;
+    if (ret < 0) {
+        local->lock[0].layout.parent_layout.locks = NULL;
+        local->lock[0].layout.parent_layout.lk_count = 0;
+        goto err;
+    }
 
-        conf  = this->private;
-        GF_VALIDATE_OR_GOTO(this->name, conf, unwind);
+    return 0;
+err:
+    if (lk_array != NULL) {
+        dht_lock_array_free(lk_array, count);
+        GF_FREE(lk_array);
+    }
 
-        methods = conf->methods;
-        GF_VALIDATE_OR_GOTO(this->name, conf->methods, done);
+    return -1;
+}
 
-        if (op_ret < 0)
-                goto done;
+static int
+dht_refresh_parent_layout_resume(call_frame_t *frame, xlator_t *this, int ret,
+                                 int invoke_cbk)
+{
+    dht_local_t *local = NULL, *parent_local = NULL;
+    call_stub_t *stub = NULL;
+    call_frame_t *parent_frame = NULL;
 
-        if (!local->layout)
-                local->layout = dht_layout_get (this, local->fd->inode);
+    local = frame->local;
 
-        layout = local->layout;
+    stub = local->stub;
+    local->stub = NULL;
 
-        if (conf->readdir_optimize == _gf_true)
-                 readdir_optimize = 1;
+    parent_frame = stub->frame;
+    parent_local = parent_frame->local;
 
-        list_for_each_entry (orig_entry, (&orig_entries->list), list) {
-                next_offset = orig_entry->d_off;
+    if (ret < 0) {
+        parent_local->op_ret = -1;
+        parent_local->op_errno = local->op_errno ? local->op_errno : EIO;
+    } else {
+        parent_local->op_ret = 0;
+    }
 
-                if (IA_ISINVAL(orig_entry->d_stat.ia_type)) {
-                        /*stat failed somewhere- ignore this entry*/
-                        continue;
-                }
+    call_resume(stub);
 
-                if (check_is_dir (NULL, (&orig_entry->d_stat), NULL)) {
+    DHT_STACK_DESTROY(frame);
 
-                /*Directory entries filtering :
-                 * a) If rebalance is running, pick from first_up_subvol
-                 * b) (rebalance not running)hashed subvolume is NULL or
-                 * down then filter in first_up_subvolume. Other wise the
-                 * corresponding hashed subvolume will take care of the
-                 * directory entry.
-                 */
+    return 0;
+}
 
-                        if (readdir_optimize) {
-                                if (prev->this == local->first_up_subvol)
-                                        goto list;
-                                else
-                                        continue;
+static int
+dht_refresh_parent_layout_done(call_frame_t *frame)
+{
+    dht_local_t *local = NULL;
+    int ret = 0;
 
-                        }
+    local = frame->local;
 
-                        hashed_subvol = methods->layout_search (this, layout,
-                                                         orig_entry->d_name);
+    if (local->op_ret < 0) {
+        ret = -1;
+        goto resume;
+    }
 
-                        if (prev->this == hashed_subvol)
-                                goto list;
-                        if ((hashed_subvol
-                                && dht_subvol_status (conf, hashed_subvol))
-                                ||(prev->this != local->first_up_subvol))
-                                continue;
+    dht_layout_set(frame->this, local->loc.inode,
+                   local->selfheal.refreshed_layout);
 
-                        goto list;
-                }
+resume:
+    dht_refresh_parent_layout_resume(frame, frame->this, ret, 1);
+    return 0;
+}
 
-                if (check_is_linkfile (NULL, (&orig_entry->d_stat),
-                                       orig_entry->dict,
-                                       conf->link_xattr_name)) {
-                        continue;
-                }
-list:
-                entry = gf_dirent_for_name (orig_entry->d_name);
-                if (!entry) {
+static int
+dht_handle_parent_layout_change(xlator_t *this, call_stub_t *stub)
+{
+    call_frame_t *refresh_frame = NULL, *frame = NULL;
+    dht_local_t *refresh_local = NULL, *local = NULL;
 
-                        goto unwind;
-                }
+    frame = stub->frame;
+    local = frame->local;
 
-                /* Do this if conf->search_unhashed is set to "auto" */
-                if (conf->search_unhashed == GF_DHT_LOOKUP_UNHASHED_AUTO) {
-                        subvol = methods->layout_search (this, layout,
-                                                         orig_entry->d_name);
-                        if (!subvol || (subvol != prev->this)) {
-                                /* TODO: Count the number of entries which need
-                                   linkfile to prove its existence in fs */
-                                layout->search_unhashed++;
-                        }
-                }
+    refresh_frame = copy_frame(frame);
+    if (!refresh_frame) {
+        gf_msg(this->name, GF_LOG_ERROR, ENOMEM, DHT_MSG_NO_MEMORY,
+               "mem allocation failed for refresh_frame");
+        return -1;
+    }
 
-                entry->d_off  = orig_entry->d_off;
-                entry->d_stat = orig_entry->d_stat;
-                entry->d_ino  = orig_entry->d_ino;
-                entry->d_type = orig_entry->d_type;
-                entry->d_len  = orig_entry->d_len;
-
-                if (orig_entry->dict)
-                        entry->dict = dict_ref (orig_entry->dict);
-
-                /* making sure we set the inode ctx right with layout,
-                   currently possible only for non-directories, so for
-                   directories don't set entry inodes */
-                if (!IA_ISDIR(entry->d_stat.ia_type) && orig_entry->inode) {
-                        ret = dht_layout_preset (this, prev->this,
-                                                 orig_entry->inode);
-                        if (ret)
-                                gf_msg (this->name, GF_LOG_WARNING, 0,
-                                        DHT_MSG_LAYOUT_SET_FAILED,
-                                        "failed to link the layout in inode");
-                        entry->inode = inode_ref (orig_entry->inode);
-                } else if (orig_entry->inode) {
-                        dht_inode_ctx_time_update (orig_entry->inode, this,
-                                                   &entry->d_stat, 1);
-                }
+    refresh_local = dht_local_init(refresh_frame, NULL, NULL, stub->fop);
+    if (!refresh_local) {
+        gf_msg(this->name, GF_LOG_ERROR, ENOMEM, DHT_MSG_NO_MEMORY,
+               "mem allocation failed for refresh_local");
+        return -1;
+    }
 
-                list_add_tail (&entry->list, &entries.list);
-                count++;
-        }
-        op_ret = count;
-        /* We need to ensure that only the last subvolume's end-of-directory
-         * notification is respected so that directory reading does not stop
-         * before all subvolumes have been read. That could happen because the
-         * posix for each subvolume sends a ENOENT on end-of-directory but in
-         * distribute we're not concerned only with a posix's view of the
-         * directory but the aggregated namespace' view of the directory.
-         */
-        if (prev->this != dht_last_up_subvol (this))
-                op_errno = 0;
+    refresh_local->loc.inode = inode_ref(local->loc.parent);
+    gf_uuid_copy(refresh_local->loc.gfid, local->loc.parent->gfid);
 
-done:
-        if (count == 0) {
-                /* non-zero next_offset means that
-                   EOF is not yet hit on the current subvol
-                */
-                if (next_offset == 0) {
-                        next_subvol = dht_subvol_next (this, prev->this);
-                } else {
-                        next_subvol = prev->this;
-                }
+    refresh_local->stub = stub;
 
-                if (!next_subvol) {
-                        goto unwind;
-                }
+    refresh_local->refresh_layout_unlock = dht_refresh_parent_layout_resume;
+    refresh_local->refresh_layout_done = dht_refresh_parent_layout_done;
 
-		if (conf->readdir_optimize == _gf_true) {
-                        if (next_subvol != local->first_up_subvol) {
-                                ret = dict_set_int32 (local->xattr,
-                                                      GF_READDIR_SKIP_DIRS, 1);
-                                if (ret)
-                                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                                DHT_MSG_DICT_SET_FAILED,
-					        "Failed to set dictionary value"
-                                                ":key = %s",
-                                                GF_READDIR_SKIP_DIRS );
-		        } else {
-                                 dict_del (local->xattr,
-                                           GF_READDIR_SKIP_DIRS);
-                        }
-                }
+    dht_refresh_layout(refresh_frame);
+    return 0;
+}
 
-                STACK_WIND (frame, dht_readdirp_cbk,
-                            next_subvol, next_subvol->fops->readdirp,
-                            local->fd, local->size, next_offset,
-                            local->xattr);
-                return 0;
-        }
+static int32_t
+dht_call_mkdir_stub(call_frame_t *frame, void *cookie, xlator_t *this,
+                    int32_t op_ret, int32_t op_errno, dict_t *xdata)
+{
+    dht_local_t *local = NULL;
+    call_stub_t *stub = NULL;
 
-unwind:
-        if (op_ret < 0)
-                op_ret = 0;
+    local = frame->local;
+    stub = local->stub;
+    local->stub = NULL;
 
-        DHT_STACK_UNWIND (readdirp, frame, op_ret, op_errno, &entries, NULL);
+    if (op_ret < 0) {
+        local->op_ret = -1;
+        local->op_errno = op_errno;
+    } else {
+        local->op_ret = 0;
+    }
 
-        gf_dirent_free (&entries);
+    call_resume(stub);
 
-        return 0;
+    return 0;
 }
 
-
-
-int
-dht_readdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                 int op_ret, int op_errno, gf_dirent_t *orig_entries,
-                 dict_t *xdata)
+static int32_t
+dht_guard_parent_layout_and_namespace(xlator_t *subvol, call_stub_t *stub)
 {
-        dht_local_t  *local = NULL;
-        gf_dirent_t   entries;
-        gf_dirent_t  *orig_entry = NULL;
-        gf_dirent_t  *entry = NULL;
-        call_frame_t *prev = NULL;
-        xlator_t     *next_subvol = NULL;
-        off_t         next_offset = 0;
-        int           count = 0;
-        dht_layout_t *layout = 0;
-        xlator_t     *subvol = 0;
-        dht_conf_t   *conf = NULL;
-        dht_methods_t *methods = NULL;
-
-        INIT_LIST_HEAD (&entries.list);
-        prev = cookie;
-        local = frame->local;
-
-        conf = this->private;
-        GF_VALIDATE_OR_GOTO (this->name, conf, done);
+    dht_local_t *local = NULL;
+    int ret = -1;
+    loc_t *loc = NULL;
+    xlator_t *hashed_subvol = NULL, *this = NULL;
+    ;
+    call_frame_t *frame = NULL;
+    char pgfid[GF_UUID_BUF_SIZE] = {0};
+    int32_t *parent_disk_layout = NULL;
+    dht_layout_t *parent_layout = NULL;
+    dht_conf_t *conf = NULL;
+
+    GF_VALIDATE_OR_GOTO("dht", stub, err);
+
+    frame = stub->frame;
+    this = frame->this;
+
+    conf = this->private;
+
+    local = frame->local;
+
+    local->stub = stub;
+
+    /* TODO: recheck whether we should lock on src or dst if we do similar
+     * stale layout checks for rename.
+     */
+    loc = &stub->args.loc;
+
+    gf_uuid_unparse(loc->parent->gfid, pgfid);
+
+    if (local->params == NULL) {
+        local->params = dict_new();
+        if (local->params == NULL) {
+            local->op_errno = ENOMEM;
+            gf_msg(this->name, GF_LOG_WARNING, local->op_errno,
+                   DHT_MSG_PARENT_LAYOUT_CHANGED,
+                   "%s (%s/%s) (path: %s): "
+                   "dict allocation failed",
+                   gf_fop_list[stub->fop], pgfid, loc->name, loc->path);
+            goto err;
+        }
+    }
+
+    hashed_subvol = dht_subvol_get_hashed(this, loc);
+    if (hashed_subvol == NULL) {
+        local->op_errno = EINVAL;
+
+        gf_msg(this->name, GF_LOG_WARNING, local->op_errno,
+               DHT_MSG_PARENT_LAYOUT_CHANGED,
+               "%s (%s/%s) (path: %s): "
+               "hashed subvolume not found",
+               gf_fop_list[stub->fop], pgfid, loc->name, loc->path);
+        goto err;
+    }
+
+    parent_layout = dht_layout_get(this, loc->parent);
+
+    ret = dht_disk_layout_extract_for_subvol(this, parent_layout, hashed_subvol,
+                                             &parent_disk_layout);
+    if (ret == -1) {
+        local->op_errno = EINVAL;
+        gf_msg(this->name, GF_LOG_WARNING, local->op_errno,
+               DHT_MSG_PARENT_LAYOUT_CHANGED,
+               "%s (%s/%s) (path: %s): "
+               "extracting in-memory layout of parent failed. ",
+               gf_fop_list[stub->fop], pgfid, loc->name, loc->path);
+        goto err;
+    }
+
+    memcpy((void *)local->parent_disk_layout, (void *)parent_disk_layout,
+           sizeof(local->parent_disk_layout));
+
+    dht_layout_unref(this, parent_layout);
+    parent_layout = NULL;
+
+    ret = dict_set_str(local->params, GF_PREOP_PARENT_KEY, conf->xattr_name);
+    if (ret < 0) {
+        local->op_errno = -ret;
+        gf_msg(this->name, GF_LOG_WARNING, local->op_errno,
+               DHT_MSG_PARENT_LAYOUT_CHANGED,
+               "%s (%s/%s) (path: %s): "
+               "setting %s key in params dictionary failed. ",
+               gf_fop_list[stub->fop], pgfid, loc->name, loc->path,
+               GF_PREOP_PARENT_KEY);
+        goto err;
+    }
+
+    ret = dict_set_bin(local->params, conf->xattr_name, parent_disk_layout,
+                       4 * 4);
+    if (ret < 0) {
+        local->op_errno = -ret;
+        gf_msg(this->name, GF_LOG_WARNING, local->op_errno,
+               DHT_MSG_PARENT_LAYOUT_CHANGED,
+               "%s (%s/%s) (path: %s): "
+               "setting parent-layout in params dictionary failed. ",
+               gf_fop_list[stub->fop], pgfid, loc->name, loc->path);
+        goto err;
+    }
+
+    parent_disk_layout = NULL;
+    local->hashed_subvol = hashed_subvol;
+
+    local->current = &local->lock[0];
+    ret = dht_protect_namespace(frame, loc, hashed_subvol, &local->current->ns,
+                                dht_call_mkdir_stub);
+    if (ret < 0)
+        goto err;
+
+    return 0;
+err:
 
-        methods = conf->methods;
-        GF_VALIDATE_OR_GOTO (this->name, conf->methods, done);
+    if (parent_disk_layout != NULL)
+        GF_FREE(parent_disk_layout);
 
-        if (op_ret < 0)
-                goto done;
+    if (parent_layout != NULL)
+        dht_layout_unref(this, parent_layout);
 
-        if (!local->layout)
-                local->layout = dht_layout_get (this, local->fd->inode);
+    return -1;
+}
 
-        layout = local->layout;
+int
+dht_mknod(call_frame_t *frame, xlator_t *this, loc_t *loc, mode_t mode,
+          dev_t rdev, mode_t umask, dict_t *params)
+{
+    xlator_t *subvol = NULL;
+    int op_errno = -1;
+    int i = 0;
+    int ret = 0;
+    dht_local_t *local = NULL;
+    dht_conf_t *conf = NULL;
+
+    VALIDATE_OR_GOTO(frame, err);
+    VALIDATE_OR_GOTO(this, err);
+    VALIDATE_OR_GOTO(loc, err);
+
+    conf = this->private;
+
+    dht_get_du_info(frame, this, loc);
+
+    local = dht_local_init(frame, loc, NULL, GF_FOP_MKNOD);
+    if (!local) {
+        op_errno = ENOMEM;
+        goto err;
+    }
+
+    subvol = dht_subvol_get_hashed(this, loc);
+    if (!subvol) {
+        gf_msg_debug(this->name, 0, "no subvolume in layout for path=%s",
+                     loc->path);
+        op_errno = EIO;
+        goto err;
+    }
+
+    /* Post remove-brick, the client layout may not be in sync with
+     * disk layout because of lack of lookup. Hence,a mknod call
+     * may fall on the decommissioned brick.  Hence, if the
+     * hashed_subvol is part of decommissioned bricks  list, do a
+     * lookup on parent dir. If a fix-layout is already done by the
+     * remove-brick process, the parent directory layout will be in
+     * sync with that of the disk. If fix-layout is still ending
+     * on the parent directory, we can let the file get created on
+     * the decommissioned brick which will be eventually migrated to
+     * non-decommissioned brick based on the new layout.
+     */
+
+    if (conf->decommission_subvols_cnt) {
+        for (i = 0; i < conf->subvolume_cnt; i++) {
+            if (conf->decommissioned_bricks[i] &&
+                conf->decommissioned_bricks[i] == subvol) {
+                gf_msg_debug(this->name, 0,
+                             "hashed subvol:%s is "
+                             "part of decommission brick list for "
+                             "file: %s",
+                             subvol->name, loc->path);
+
+                /* dht_refresh_layout needs directory info in
+                 * local->loc. Hence, storing the parent_loc in
+                 * local->loc and storing the create context in
+                 * local->loc2. We will restore this information
+                 * in dht_creation do */
+
+                ret = loc_copy(&local->loc2, &local->loc);
+                if (ret) {
+                    gf_msg(this->name, GF_LOG_ERROR, ENOMEM, DHT_MSG_NO_MEMORY,
+                           "loc_copy failed %s", loc->path);
 
-        list_for_each_entry (orig_entry, (&orig_entries->list), list) {
-                next_offset = orig_entry->d_off;
+                    goto err;
+                }
 
-                subvol = methods->layout_search (this, layout,
-                                                 orig_entry->d_name);
+                local->params = dict_ref(params);
+                local->rdev = rdev;
+                local->mode = mode;
+                local->umask = umask;
 
-                if (!subvol || (subvol == prev->this)) {
-                        entry = gf_dirent_for_name (orig_entry->d_name);
-                        if (!entry) {
-                                gf_msg (this->name, GF_LOG_ERROR, 0,
-                                        DHT_MSG_NO_MEMORY,
-                                        "Memory allocation failed ");
-                                goto unwind;
-                        }
+                loc_wipe(&local->loc);
 
-                        entry->d_off  = orig_entry->d_off;
-                        entry->d_ino  = orig_entry->d_ino;
-                        entry->d_type = orig_entry->d_type;
-                        entry->d_len  = orig_entry->d_len;
+                ret = dht_build_parent_loc(this, &local->loc, loc, &op_errno);
 
-                        list_add_tail (&entry->list, &entries.list);
-                        count++;
+                if (ret) {
+                    gf_msg(this->name, GF_LOG_ERROR, ENOMEM, DHT_MSG_LOC_FAILED,
+                           "parent loc build failed");
+                    goto err;
                 }
-        }
-        op_ret = count;
-        /* We need to ensure that only the last subvolume's end-of-directory
-         * notification is respected so that directory reading does not stop
-         * before all subvolumes have been read. That could happen because the
-         * posix for each subvolume sends a ENOENT on end-of-directory but in
-         * distribute we're not concerned only with a posix's view of the
-         * directory but the aggregated namespace' view of the directory.
-         */
-        if (prev->this != dht_last_up_subvol (this))
-                op_errno = 0;
 
-done:
-        if (count == 0) {
-                /* non-zero next_offset means that
-                   EOF is not yet hit on the current subvol
-                */
-                if (next_offset == 0) {
-                        next_subvol = dht_subvol_next (this, prev->this);
-                } else {
-                        next_subvol = prev->this;
-                }
+                ret = dht_mknod_lock(frame, subvol);
 
-                if (!next_subvol) {
-                        goto unwind;
+                if (ret < 0) {
+                    gf_msg(this->name, GF_LOG_ERROR, 0, DHT_MSG_INODE_LK_ERROR,
+                           "locking parent failed");
+                    goto err;
                 }
 
-                STACK_WIND (frame, dht_readdir_cbk,
-                            next_subvol, next_subvol->fops->readdir,
-                            local->fd, local->size, next_offset, NULL);
-                return 0;
+                goto done;
+            }
         }
+    }
 
-unwind:
-        if (op_ret < 0)
-                op_ret = 0;
+    dht_mknod_wind_to_avail_subvol(frame, this, subvol, loc, rdev, mode, umask,
+                                   params);
 
-        DHT_STACK_UNWIND (readdir, frame, op_ret, op_errno, &entries, NULL);
+done:
+    return 0;
 
-        gf_dirent_free (&entries);
+err:
+    op_errno = (op_errno == -1) ? errno : op_errno;
+    DHT_STACK_UNWIND(mknod, frame, -1, op_errno, NULL, NULL, NULL, NULL, NULL);
 
-        return 0;
+    return 0;
 }
 
-
 int
-dht_do_readdir (call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
-                off_t yoff, int whichop, dict_t *dict)
+dht_symlink(call_frame_t *frame, xlator_t *this, const char *linkname,
+            loc_t *loc, mode_t umask, dict_t *params)
 {
-        dht_local_t  *local  = NULL;
-        int           op_errno = -1;
-        xlator_t     *xvol = NULL;
-        int           ret = 0;
-        dht_conf_t   *conf = NULL;
-
-        VALIDATE_OR_GOTO (frame, err);
-        VALIDATE_OR_GOTO (this, err);
-        VALIDATE_OR_GOTO (fd, err);
-        VALIDATE_OR_GOTO (this->private, err);
+    xlator_t *subvol = NULL;
+    int op_errno = -1;
+    dht_local_t *local = NULL;
 
-        conf = this->private;
+    VALIDATE_OR_GOTO(frame, err);
+    VALIDATE_OR_GOTO(this, err);
+    VALIDATE_OR_GOTO(loc, err);
 
-        local = dht_local_init (frame, NULL, NULL, whichop);
-        if (!local) {
-                op_errno = ENOMEM;
-                goto err;
-        }
-
-        local->fd = fd_ref (fd);
-        local->size = size;
-        local->xattr_req = (dict)? dict_ref (dict) : NULL;
-        local->first_up_subvol = dht_first_up_subvol (this);
-
-        dht_deitransform (this, yoff, &xvol);
-
-        /* TODO: do proper readdir */
-        if (whichop == GF_FOP_READDIRP) {
-                if (dict)
-                        local->xattr = dict_ref (dict);
-                else
-                        local->xattr = dict_new ();
+    local = dht_local_init(frame, loc, NULL, GF_FOP_SYMLINK);
+    if (!local) {
+        op_errno = ENOMEM;
+        goto err;
+    }
 
-                if (local->xattr) {
-                        ret = dict_set_uint32 (local->xattr,
-                                               conf->link_xattr_name, 256);
-                        if (ret)
-                                gf_msg (this->name, GF_LOG_WARNING, 0,
-                                        DHT_MSG_DICT_SET_FAILED,
-                                        "Failed to set dictionary value"
-                                        " : key = %s",
-                                        conf->link_xattr_name);
+    subvol = dht_subvol_get_hashed(this, loc);
+    if (!subvol) {
+        gf_msg_debug(this->name, 0, "no subvolume in layout for path=%s",
+                     loc->path);
+        op_errno = EIO;
+        goto err;
+    }
 
-			if (conf->readdir_optimize == _gf_true) {
-                                if (xvol != local->first_up_subvol) {
-				        ret = dict_set_int32 (local->xattr,
-			                               GF_READDIR_SKIP_DIRS, 1);
-				        if (ret)
-					        gf_msg (this->name,
-                                                        GF_LOG_ERROR, 0,
-                                                        DHT_MSG_DICT_SET_FAILED,
-                                                        "Failed to set "
-                                                        "dictionary value: "
-                                                        "key = %s",
-						        GF_READDIR_SKIP_DIRS);
-                                } else {
-                                        dict_del (local->xattr,
-                                                  GF_READDIR_SKIP_DIRS);
-                                }
-			}
-                }
+    gf_msg_trace(this->name, 0, "creating %s on %s", loc->path, subvol->name);
 
-                STACK_WIND (frame, dht_readdirp_cbk, xvol, xvol->fops->readdirp,
-                            fd, size, yoff, local->xattr);
-        } else {
-                STACK_WIND (frame, dht_readdir_cbk, xvol, xvol->fops->readdir,
-                            fd, size, yoff, local->xattr);
-        }
+    STACK_WIND_COOKIE(frame, dht_newfile_cbk, (void *)subvol, subvol,
+                      subvol->fops->symlink, linkname, loc, umask, params);
 
-        return 0;
+    return 0;
 
 err:
-        op_errno = (op_errno == -1) ? errno : op_errno;
-        DHT_STACK_UNWIND (readdir, frame, -1, op_errno, NULL, NULL);
+    op_errno = (op_errno == -1) ? errno : op_errno;
+    DHT_STACK_UNWIND(link, frame, -1, op_errno, NULL, NULL, NULL, NULL, NULL);
 
-        return 0;
+    return 0;
 }
 
-
 int
-dht_readdir (call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
-             off_t yoff, dict_t *xdata)
+dht_unlink(call_frame_t *frame, xlator_t *this, loc_t *loc, int xflag,
+           dict_t *xdata)
 {
-        int          op = GF_FOP_READDIR;
-        dht_conf_t  *conf = NULL;
-        int          i = 0;
-
-        conf = this->private;
-        if (!conf)
-                goto out;
+    xlator_t *cached_subvol = NULL;
+    int op_errno = -1;
+    dht_local_t *local = NULL;
+
+    VALIDATE_OR_GOTO(frame, err);
+    VALIDATE_OR_GOTO(this, err);
+    VALIDATE_OR_GOTO(loc, err);
+
+    local = dht_local_init(frame, loc, NULL, GF_FOP_UNLINK);
+    if (!local) {
+        op_errno = ENOMEM;
+
+        goto err;
+    }
+
+    cached_subvol = local->cached_subvol;
+    if (!cached_subvol) {
+        gf_msg_debug(this->name, 0, "no cached subvolume for path=%s",
+                     loc->path);
+        op_errno = EINVAL;
+        goto err;
+    }
+
+    local->flags = xflag;
+    STACK_WIND_COOKIE(frame, dht_unlink_cbk, cached_subvol, cached_subvol,
+                      cached_subvol->fops->unlink, loc, xflag, xdata);
+
+    return 0;
+err:
+    op_errno = (op_errno == -1) ? errno : op_errno;
+    DHT_STACK_UNWIND(unlink, frame, -1, op_errno, NULL, NULL, NULL);
 
-        for (i = 0; i < conf->subvolume_cnt; i++) {
-                if (!conf->subvolume_status[i]) {
-                        op = GF_FOP_READDIRP;
-                        break;
-                }
-        }
+    return 0;
+}
 
-        if (conf->use_readdirp)
-                op = GF_FOP_READDIRP;
+static int
+dht_remove_stale_linkto_cbk(int ret, call_frame_t *sync_frame, void *data)
+{
+    DHT_STACK_DESTROY(sync_frame);
+    return 0;
+}
 
+static int
+dht_remove_stale_linkto(void *data)
+{
+    call_frame_t *frame = NULL;
+    dht_local_t *local = NULL;
+    xlator_t *this = NULL;
+    dict_t *xdata_in = NULL;
+    int ret = 0;
+
+    GF_VALIDATE_OR_GOTO("dht", data, out);
+
+    frame = data;
+    local = frame->local;
+    this = frame->this;
+    GF_VALIDATE_OR_GOTO("dht", this, out);
+    GF_VALIDATE_OR_GOTO("dht", local, out);
+    GF_VALIDATE_OR_GOTO("dht", local->link_subvol, out);
+
+    xdata_in = dict_new();
+    if (!xdata_in)
+        goto out;
+
+    ret = dht_fill_dict_to_avoid_unlink_of_migrating_file(xdata_in);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_WARNING, -ret, 0,
+               "Failed to set keys for stale linkto"
+               "deletion on path %s",
+               local->loc.path);
+        goto out;
+    }
+
+    ret = syncop_unlink(local->link_subvol, &local->loc, xdata_in, NULL);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_WARNING, -ret, 0,
+               "Removal of linkto failed"
+               " on path %s at subvol %s",
+               local->loc.path, local->link_subvol->name);
+    }
 out:
-        dht_do_readdir (frame, this, fd, size, yoff, op, 0);
-        return 0;
+    if (xdata_in)
+        dict_unref(xdata_in);
+    return ret;
 }
 
-int
-dht_readdirp (call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
-              off_t yoff, dict_t *dict)
+static int
+dht_link_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int op_ret,
+             int op_errno, inode_t *inode, struct iatt *stbuf,
+             struct iatt *preparent, struct iatt *postparent, dict_t *xdata)
 {
-        dht_do_readdir (frame, this, fd, size, yoff, GF_FOP_READDIRP, dict);
-        return 0;
-}
+    dht_local_t *local = NULL;
+    int ret = -1;
+    gf_boolean_t stbuf_merged = _gf_false;
+    xlator_t *subvol = NULL;
+    call_frame_t *cleanup_frame = NULL;
+    dht_local_t *cleanup_local = NULL;
+
+    local = frame->local;
+
+    if (op_ret == -1) {
+        /* Remove the linkto if exists */
+        if (local->linked) {
+            cleanup_frame = create_frame(this, this->ctx->pool);
+            if (cleanup_frame) {
+                cleanup_local = dht_local_init(cleanup_frame, &local->loc2,
+                                               NULL, 0);
+                if (!cleanup_local || !local->link_subvol) {
+                    DHT_STACK_DESTROY(cleanup_frame);
+                    goto out;
+                }
+                cleanup_local->link_subvol = local->link_subvol;
+                FRAME_SU_DO(cleanup_frame, dht_local_t);
+                ret = synctask_new(this->ctx->env, dht_remove_stale_linkto,
+                                   dht_remove_stale_linkto_cbk, cleanup_frame,
+                                   cleanup_frame);
+            }
+        }
+        /* No continuation on DHT inode missing errors, as we should
+         * then have a good stbuf that states P2 happened. We would
+         * get inode missing if, the file completed migrated between
+         * the lookup and the link call */
+        goto out;
+    }
+
+    /* Update parent on success, even if P1/2 checks are positive.
+     * The second call on success will further update the parent */
+    if (local->loc.parent) {
+        dht_inode_ctx_time_update(local->loc.parent, this, preparent, 0);
+        dht_inode_ctx_time_update(local->loc.parent, this, postparent, 1);
+    }
+
+    /* Update linkto attrs, if this is the first call and non-P2,
+     * if we detect P2 then we need to trust the attrs from the
+     * second call, not the first */
+    if (local->linked == _gf_true &&
+        ((local->call_cnt == 1 && !IS_DHT_MIGRATION_PHASE2(stbuf)) ||
+         (local->call_cnt != 1 && IS_DHT_MIGRATION_PHASE2(&local->stbuf)))) {
+        dht_iatt_merge(this, &local->stbuf, stbuf);
+        stbuf_merged = _gf_true;
+        dht_linkfile_attr_heal(frame, this);
+    }
+
+    /* No further P1/2 checks if we are in the second iteration of
+     * the call */
+    if (local->call_cnt != 1) {
+        goto out;
+    } else {
+        /* Preserve the return values, in case the migration decides
+         * to recreate the link on the same subvol that the current
+         * hased for the link was created on. */
+        dht_iatt_merge(this, &local->preparent, preparent);
+        dht_iatt_merge(this, &local->postparent, postparent);
+        if (!stbuf_merged) {
+            dht_iatt_merge(this, &local->stbuf, stbuf);
+            stbuf_merged = _gf_true;
+        }
+
+        local->inode = inode_ref(inode);
+    }
+
+    local->op_ret = op_ret;
+    local->op_errno = op_errno;
+    local->rebalance.target_op_fn = dht_link2;
+    dht_set_local_rebalance(this, local, stbuf, preparent, postparent, xdata);
+
+    /* Check if the rebalance phase2 is true */
+    if (IS_DHT_MIGRATION_PHASE2(stbuf)) {
+        ret = dht_inode_ctx_get_mig_info(this, local->loc.inode, NULL, &subvol);
+        if (!subvol) {
+            /* Phase 2 of migration */
+            ret = dht_rebalance_complete_check(this, frame);
+            if (!ret)
+                return 0;
+        } else {
+            dht_link2(this, subvol, frame, 0);
+            return 0;
+        }
+    }
+
+    /* Check if the rebalance phase1 is true */
+    if (IS_DHT_MIGRATION_PHASE1(stbuf)) {
+        ret = dht_inode_ctx_get_mig_info(this, local->loc.inode, NULL, &subvol);
+        if (subvol) {
+            dht_link2(this, subvol, frame, 0);
+            return 0;
+        }
+        ret = dht_rebalance_in_progress_check(this, frame);
+        if (!ret)
+            return 0;
+    }
+out:
+    DHT_STRIP_PHASE1_FLAGS(stbuf);
 
+    dht_set_fixed_dir_stat(preparent);
+    dht_set_fixed_dir_stat(postparent);
+    DHT_STACK_UNWIND(link, frame, op_ret, op_errno, inode, stbuf, preparent,
+                     postparent, NULL);
 
+    return 0;
+}
 
-int
-dht_fsyncdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                  int op_ret, int op_errno, dict_t *xdata)
+static int
+dht_link2(xlator_t *this, xlator_t *subvol, call_frame_t *frame, int ret)
 {
-        dht_local_t  *local = NULL;
-        int           this_call_cnt = 0;
+    dht_local_t *local = NULL;
+    int op_errno = EINVAL;
 
+    local = frame->local;
+    if (!local)
+        goto err;
 
-        local = frame->local;
-
-        LOCK (&frame->lock);
-        {
-                if (op_ret == -1)
-                        local->op_errno = op_errno;
+    op_errno = local->op_errno;
 
-                if (op_ret == 0)
-                        local->op_ret = 0;
-        }
-        UNLOCK (&frame->lock);
+    if (we_are_not_migrating(ret)) {
+        /* This dht xlator is not migrating the file. Unwind and
+         * pass on the original mode bits so the higher DHT layer
+         * can handle this.
+         */
+        dht_set_fixed_dir_stat(&local->preparent);
+        dht_set_fixed_dir_stat(&local->postparent);
 
-        this_call_cnt = dht_frame_return (frame);
-        if (is_last_call (this_call_cnt))
-                DHT_STACK_UNWIND (fsyncdir, frame, local->op_ret,
-                                  local->op_errno, xdata);
+        DHT_STACK_UNWIND(link, frame, local->op_ret, op_errno, local->inode,
+                         &local->stbuf, &local->preparent, &local->postparent,
+                         NULL);
+        return 0;
+    }
+
+    if (subvol == NULL) {
+        op_errno = EINVAL;
+        goto err;
+    }
+
+    /* Second call to create link file could result in EEXIST as the
+     * first call created the linkto in the currently
+     * migrating subvol, which could be the new hashed subvol */
+    if (local->link_subvol == subvol) {
+        DHT_STRIP_PHASE1_FLAGS(&local->stbuf);
+        dht_set_fixed_dir_stat(&local->preparent);
+        dht_set_fixed_dir_stat(&local->postparent);
+        DHT_STACK_UNWIND(link, frame, 0, 0, local->inode, &local->stbuf,
+                         &local->preparent, &local->postparent, NULL);
 
         return 0;
-}
+    }
 
+    local->call_cnt = 2;
 
-int
-dht_fsyncdir (call_frame_t *frame, xlator_t *this, fd_t *fd,
-              int datasync, dict_t *xdata)
-{
-        dht_local_t  *local  = NULL;
-        dht_conf_t   *conf = NULL;
-        int           op_errno = -1;
-        int           i = -1;
+    STACK_WIND(frame, dht_link_cbk, subvol, subvol->fops->link, &local->loc,
+               &local->loc2, local->xattr_req);
 
-        VALIDATE_OR_GOTO (frame, err);
-        VALIDATE_OR_GOTO (this, err);
-        VALIDATE_OR_GOTO (fd, err);
-        VALIDATE_OR_GOTO (this->private, err);
+    return 0;
+err:
+    DHT_STACK_UNWIND(link, frame, -1, op_errno, NULL, NULL, NULL, NULL, NULL);
 
-        conf = this->private;
+    return 0;
+}
 
-        local = dht_local_init (frame, NULL, NULL, GF_FOP_FSYNCDIR);
-        if (!local) {
-                op_errno = ENOMEM;
-                goto err;
-        }
+static int
+dht_link_linkfile_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                      int op_ret, int op_errno, inode_t *inode,
+                      struct iatt *stbuf, struct iatt *preparent,
+                      struct iatt *postparent, dict_t *xdata)
+{
+    dht_local_t *local = NULL;
+    xlator_t *srcvol = NULL;
 
-        local->fd = fd_ref (fd);
-        local->call_cnt = conf->subvolume_cnt;
+    if (op_ret == -1)
+        goto err;
 
-        for (i = 0; i < conf->subvolume_cnt; i++) {
-                STACK_WIND (frame, dht_fsyncdir_cbk,
-                            conf->subvolumes[i],
-                            conf->subvolumes[i]->fops->fsyncdir,
-                            fd, datasync, xdata);
-        }
+    local = frame->local;
+    srcvol = local->linkfile.srcvol;
 
-        return 0;
+    STACK_WIND(frame, dht_link_cbk, srcvol, srcvol->fops->link, &local->loc,
+               &local->loc2, local->xattr_req);
+
+    return 0;
 
 err:
-        op_errno = (op_errno == -1) ? errno : op_errno;
-        DHT_STACK_UNWIND (fsyncdir, frame, -1, op_errno, NULL);
+    DHT_STRIP_PHASE1_FLAGS(stbuf);
+    dht_set_fixed_dir_stat(preparent);
+    dht_set_fixed_dir_stat(postparent);
+    DHT_STACK_UNWIND(link, frame, op_ret, op_errno, inode, stbuf, preparent,
+                     postparent, xdata);
 
-        return 0;
+    return 0;
 }
 
-
 int
-dht_newfile_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                 int op_ret, int op_errno,
-                 inode_t *inode, struct iatt *stbuf, struct iatt *preparent,
-                 struct iatt *postparent, dict_t *xdata)
+dht_link(call_frame_t *frame, xlator_t *this, loc_t *oldloc, loc_t *newloc,
+         dict_t *xdata)
 {
-        xlator_t     *prev = NULL;
-        int           ret = -1;
-        dht_local_t  *local = NULL;
+    xlator_t *cached_subvol = NULL;
+    xlator_t *hashed_subvol = NULL;
+    int op_errno = -1;
+    int ret = -1;
+    dht_local_t *local = NULL;
+
+    VALIDATE_OR_GOTO(frame, err);
+    VALIDATE_OR_GOTO(this, err);
+    VALIDATE_OR_GOTO(oldloc, err);
+    VALIDATE_OR_GOTO(newloc, err);
+
+    local = dht_local_init(frame, oldloc, NULL, GF_FOP_LINK);
+    if (!local) {
+        op_errno = ENOMEM;
+
+        goto err;
+    }
+    local->call_cnt = 1;
+
+    cached_subvol = local->cached_subvol;
+    if (!cached_subvol) {
+        gf_msg_debug(this->name, 0, "no cached subvolume for path=%s",
+                     oldloc->path);
+        op_errno = ENOENT;
+        goto err;
+    }
+
+    hashed_subvol = dht_subvol_get_hashed(this, newloc);
+    if (!hashed_subvol) {
+        gf_msg_debug(this->name, 0, "no subvolume in layout for path=%s",
+                     newloc->path);
+        op_errno = EIO;
+        goto err;
+    }
+
+    ret = loc_copy(&local->loc2, newloc);
+    if (ret == -1) {
+        op_errno = ENOMEM;
+        goto err;
+    }
+    if (xdata)
+        local->xattr_req = dict_ref(xdata);
+
+    if (hashed_subvol != cached_subvol) {
+        gf_uuid_copy(local->gfid, oldloc->inode->gfid);
+        dht_linkfile_create(frame, dht_link_linkfile_cbk, this, cached_subvol,
+                            hashed_subvol, newloc);
+    } else {
+        STACK_WIND(frame, dht_link_cbk, cached_subvol,
+                   cached_subvol->fops->link, oldloc, newloc, xdata);
+    }
+
+    return 0;
 
+err:
+    op_errno = (op_errno == -1) ? errno : op_errno;
+    DHT_STACK_UNWIND(link, frame, -1, op_errno, NULL, NULL, NULL, NULL, NULL);
 
-        if (op_ret == -1)
-                goto out;
+    return 0;
+}
 
-        local = frame->local;
-        if (!local) {
-                op_ret = -1;
-                op_errno = EINVAL;
+int
+dht_create_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int op_ret,
+               int op_errno, fd_t *fd, inode_t *inode, struct iatt *stbuf,
+               struct iatt *preparent, struct iatt *postparent, dict_t *xdata)
+{
+    xlator_t *prev = NULL;
+    int ret = -1;
+    dht_local_t *local = NULL;
+    gf_boolean_t parent_layout_changed = _gf_false;
+    char pgfid[GF_UUID_BUF_SIZE] = {0};
+    xlator_t *subvol = NULL;
+
+    local = frame->local;
+
+    local = frame->local;
+    if (!local) {
+        op_ret = -1;
+        op_errno = EINVAL;
+        goto out;
+    }
+
+    if (op_ret == -1) {
+        local->op_errno = op_errno;
+        parent_layout_changed = (xdata &&
+                                 dict_get(xdata, GF_PREOP_CHECK_FAILED))
+                                    ? _gf_true
+                                    : _gf_false;
+
+        if (parent_layout_changed) {
+            if (local && local->lock[0].layout.parent_layout.locks) {
+                /* Returning failure as the layout could not be fixed even under
+                 * the lock */
                 goto out;
-        }
+            }
 
-        prev = cookie;
+            gf_uuid_unparse(local->loc.parent->gfid, pgfid);
+            gf_msg(this->name, GF_LOG_INFO, 0, DHT_MSG_PARENT_LAYOUT_CHANGED,
+                   "create (%s/%s) (path: %s): parent layout "
+                   "changed. Attempting a layout refresh and then a "
+                   "retry",
+                   pgfid, local->loc.name, local->loc.path);
 
-        if (local->loc.parent) {
+            /*
+              dht_refresh_layout needs directory info in local->loc.Hence,
+              storing the parent_loc in local->loc and storing the create
+              context in local->loc2. We will restore this information in
+              dht_creation_do.
+             */
 
-                dht_inode_ctx_time_update (local->loc.parent, this,
-                                           preparent, 0);
-                dht_inode_ctx_time_update (local->loc.parent, this,
-                                           postparent, 1);
-        }
+            loc_wipe(&local->loc2);
+
+            ret = loc_copy(&local->loc2, &local->loc);
+            if (ret) {
+                gf_msg(this->name, GF_LOG_ERROR, ENOMEM, DHT_MSG_NO_MEMORY,
+                       "loc_copy failed %s", local->loc.path);
 
-        ret = dht_layout_preset (this, prev, inode);
-        if (ret < 0) {
-                gf_msg_debug (this->name, 0,
-                              "could not set pre-set layout for subvolume %s",
-                              prev? prev->name: NULL);
-                op_ret   = -1;
-                op_errno = EINVAL;
                 goto out;
-        }
-        if (local->linked == _gf_true)
-                dht_linkfile_attr_heal (frame, this);
-out:
-        /*
-         * FIXME: ia_size and st_blocks of preparent and postparent do not have
-         * correct values. since, preparent and postparent buffers correspond
-         * to a directory these two members should have values equal to sum of
-         * corresponding values from each of the subvolume.
-         * See dht_iatt_merge for reference.
-         */
-        DHT_STRIP_PHASE1_FLAGS (stbuf);
-        DHT_STACK_UNWIND (mknod, frame, op_ret, op_errno, inode, stbuf,
-                          preparent, postparent, xdata);
-        return 0;
-}
+            }
 
-int
-dht_mknod_linkfile_create_cbk (call_frame_t *frame, void *cookie,
-                               xlator_t *this,
-                               int32_t op_ret, int32_t op_errno,
-                               inode_t *inode, struct iatt *stbuf,
-                               struct iatt *preparent, struct iatt *postparent,
-                               dict_t *xdata)
-{
-        dht_local_t  *local = NULL;
-        xlator_t     *cached_subvol = NULL;
+            loc_wipe(&local->loc);
 
-        if (op_ret == -1)
-                goto err;
+            ret = dht_build_parent_loc(this, &local->loc, &local->loc2,
+                                       &op_errno);
 
-        local = frame->local;
-        if (!local || !local->cached_subvol) {
-                op_errno = EINVAL;
-                goto err;
-        }
+            if (ret) {
+                gf_msg(this->name, GF_LOG_ERROR, ENOMEM, DHT_MSG_LOC_FAILED,
+                       "parent loc build failed");
+                goto out;
+            }
 
-        cached_subvol = local->cached_subvol;
+            subvol = dht_subvol_get_hashed(this, &local->loc2);
 
-        STACK_WIND_COOKIE (frame, dht_newfile_cbk, (void *)cached_subvol,
-                           cached_subvol, cached_subvol->fops->mknod,
-                           &local->loc, local->mode, local->rdev, local->umask,
-                           local->params);
+            ret = dht_create_lock(frame, subvol);
+            if (ret < 0) {
+                gf_msg(this->name, GF_LOG_ERROR, 0, DHT_MSG_INODE_LK_ERROR,
+                       "locking parent failed");
+                goto out;
+            }
 
-        return 0;
-err:
-        DHT_STACK_UNWIND (mknod, frame, -1, op_errno, NULL, NULL, NULL, NULL,
-                          NULL);
-        return 0;
-}
+            return 0;
+        }
 
-int
-dht_mknod (call_frame_t *frame, xlator_t *this,
-           loc_t *loc, mode_t mode, dev_t rdev, mode_t umask, dict_t *params)
-{
-        xlator_t    *subvol = NULL;
-        int          op_errno = -1;
-        xlator_t    *avail_subvol = NULL;
-        dht_local_t *local = NULL;
+        goto out;
+    }
 
-        VALIDATE_OR_GOTO (frame, err);
-        VALIDATE_OR_GOTO (this, err);
-        VALIDATE_OR_GOTO (loc, err);
+    prev = cookie;
 
-        dht_get_du_info (frame, this, loc);
+    if (local->loc.parent) {
+        dht_inode_ctx_time_update(local->loc.parent, this, preparent, 0);
 
-        local = dht_local_init (frame, loc, NULL, GF_FOP_MKNOD);
-        if (!local) {
-                op_errno = ENOMEM;
-                goto err;
-        }
+        dht_inode_ctx_time_update(local->loc.parent, this, postparent, 1);
+    }
 
-        subvol = dht_subvol_get_hashed (this, loc);
-        if (!subvol) {
-                gf_msg_debug (this->name, 0,
-                              "no subvolume in layout for path=%s",
-                              loc->path);
-                op_errno = ENOENT;
-                goto err;
-        }
+    ret = dht_fd_ctx_set(this, fd, prev);
+    if (ret != 0) {
+        gf_msg_debug(this->name, 0,
+                     "Possible fd leak. "
+                     "Could not set fd ctx for subvol %s",
+                     prev->name);
+    }
 
-        if (!dht_is_subvol_filled (this, subvol)) {
-                gf_msg_trace (this->name, 0,
-                              "creating %s on %s", loc->path,
-                              subvol->name);
+    ret = dht_layout_preset(this, prev, inode);
+    if (ret != 0) {
+        gf_msg_debug(this->name, 0, "could not set preset layout for subvol %s",
+                     prev->name);
+        op_ret = -1;
+        op_errno = EINVAL;
+        goto out;
+    }
 
-                STACK_WIND_COOKIE (frame, dht_newfile_cbk, (void *)subvol,
-                                   subvol, subvol->fops->mknod, loc, mode,
-                                   rdev, umask, params);
-        } else {
+    local->op_errno = op_errno;
 
-                avail_subvol = dht_free_disk_available_subvol (this, subvol,
-                                                               local);
-                if (avail_subvol != subvol) {
-                        /* Choose the minimum filled volume, and create the
-                           files there */
-
-                        local->params = dict_ref (params);
-                        local->cached_subvol = avail_subvol;
-                        local->mode = mode;
-                        local->rdev = rdev;
-                        local->umask = umask;
-                        dht_linkfile_create (frame,
-                                             dht_mknod_linkfile_create_cbk,
-                                             this, avail_subvol, subvol, loc);
-                } else {
-                        gf_msg_trace (this->name, 0,
-                                      "creating %s on %s", loc->path,
-                                      subvol->name);
-
-                        STACK_WIND_COOKIE (frame, dht_newfile_cbk,
-                                           (void *)subvol, subvol,
-                                           subvol->fops->mknod, loc, mode,
-                                           rdev, umask, params);
-                }
-        }
+    if (local->linked == _gf_true) {
+        local->stbuf = *stbuf;
+        dht_linkfile_attr_heal(frame, this);
+    }
+out:
 
-        return 0;
+    DHT_STRIP_PHASE1_FLAGS(stbuf);
+    dht_set_fixed_dir_stat(preparent);
+    dht_set_fixed_dir_stat(postparent);
 
-err:
-        op_errno = (op_errno == -1) ? errno : op_errno;
-        DHT_STACK_UNWIND (mknod, frame, -1, op_errno,
-                          NULL, NULL, NULL, NULL, NULL);
+    if (local && local->lock[0].layout.parent_layout.locks) {
+        /* store op_errno for failure case*/
+        local->op_errno = op_errno;
+        local->refresh_layout_unlock(frame, this, op_ret, 1);
 
-        return 0;
+        if (op_ret == 0) {
+            DHT_STACK_UNWIND(create, frame, op_ret, op_errno, fd, inode, stbuf,
+                             preparent, postparent, xdata);
+        }
+    } else {
+        DHT_STACK_UNWIND(create, frame, op_ret, op_errno, fd, inode, stbuf,
+                         preparent, postparent, xdata);
+    }
+    return 0;
 }
 
-
-int
-dht_symlink (call_frame_t *frame, xlator_t *this,
-             const char *linkname, loc_t *loc, mode_t umask, dict_t *params)
+static int
+dht_create_linkfile_create_cbk(call_frame_t *frame, void *cookie,
+                               xlator_t *this, int32_t op_ret, int32_t op_errno,
+                               inode_t *inode, struct iatt *stbuf,
+                               struct iatt *preparent, struct iatt *postparent,
+                               dict_t *xdata)
 {
-        xlator_t    *subvol = NULL;
-        int          op_errno = -1;
-        dht_local_t *local = NULL;
+    dht_local_t *local = NULL;
+    xlator_t *cached_subvol = NULL;
+    dht_conf_t *conf = NULL;
 
-        VALIDATE_OR_GOTO (frame, err);
-        VALIDATE_OR_GOTO (this, err);
-        VALIDATE_OR_GOTO (loc, err);
+    local = frame->local;
+    if (!local) {
+        op_errno = EINVAL;
+        goto err;
+    }
 
-        local = dht_local_init (frame, loc, NULL, GF_FOP_SYMLINK);
-        if (!local) {
-                op_errno = ENOMEM;
-                goto err;
-        }
+    if (op_ret == -1) {
+        local->op_errno = op_errno;
+        goto err;
+    }
 
-        subvol = dht_subvol_get_hashed (this, loc);
-        if (!subvol) {
-                gf_msg_debug (this->name, 0,
-                              "no subvolume in layout for path=%s",
-                              loc->path);
-                op_errno = ENOENT;
-                goto err;
-        }
+    conf = this->private;
+    if (!conf) {
+        local->op_errno = EINVAL;
+        op_errno = EINVAL;
+        goto err;
+    }
 
-        gf_msg_trace (this->name, 0,
-                      "creating %s on %s", loc->path, subvol->name);
+    cached_subvol = local->cached_subvol;
 
-        STACK_WIND_COOKIE (frame, dht_newfile_cbk, (void *)subvol, subvol,
-                           subvol->fops->symlink, linkname, loc, umask,
-                           params);
+    if (local->params) {
+        dict_del(local->params, conf->link_xattr_name);
+        dict_del(local->params, GLUSTERFS_INTERNAL_FOP_KEY);
+    }
 
-        return 0;
+    STACK_WIND_COOKIE(frame, dht_create_cbk, cached_subvol, cached_subvol,
+                      cached_subvol->fops->create, &local->loc, local->flags,
+                      local->mode, local->umask, local->fd, local->params);
 
+    return 0;
 err:
-        op_errno = (op_errno == -1) ? errno : op_errno;
-        DHT_STACK_UNWIND (link, frame, -1, op_errno,
-                          NULL, NULL, NULL, NULL, NULL);
-
-        return 0;
+    if (local && local->lock[0].layout.parent_layout.locks) {
+        local->refresh_layout_unlock(frame, this, -1, 1);
+    } else {
+        DHT_STACK_UNWIND(create, frame, -1, op_errno, NULL, NULL, NULL, NULL,
+                         NULL, NULL);
+    }
+    return 0;
 }
 
-
-int
-dht_unlink (call_frame_t *frame, xlator_t *this, loc_t *loc, int xflag,
-            dict_t *xdata)
+static int
+dht_create_wind_to_avail_subvol(call_frame_t *frame, xlator_t *this,
+                                xlator_t *subvol, loc_t *loc, int32_t flags,
+                                mode_t mode, mode_t umask, fd_t *fd,
+                                dict_t *params)
 {
-        xlator_t    *cached_subvol = NULL;
-        xlator_t    *hashed_subvol = NULL;
-        int          op_errno = -1;
-        dht_local_t *local = NULL;
+    dht_local_t *local = NULL;
+    xlator_t *avail_subvol = NULL;
 
-        VALIDATE_OR_GOTO (frame, err);
-        VALIDATE_OR_GOTO (this, err);
-        VALIDATE_OR_GOTO (loc, err);
+    local = frame->local;
 
-        local = dht_local_init (frame, loc, NULL, GF_FOP_UNLINK);
-        if (!local) {
-                op_errno = ENOMEM;
+    if (!dht_is_subvol_filled(this, subvol)) {
+        gf_msg_debug(this->name, 0, "creating %s on %s", loc->path,
+                     subvol->name);
 
-                goto err;
-        }
+        dht_set_parent_layout_in_dict(loc, this, local);
 
-        hashed_subvol = dht_subvol_get_hashed (this, loc);
-        /* Dont fail unlink if hashed_subvol is NULL which can be the result
-         * of layout anomaly */
-        if (!hashed_subvol) {
-                gf_msg_debug (this->name, 0,
-                              "no subvolume in layout for path=%s",
-                              loc->path);
-        }
+        STACK_WIND_COOKIE(frame, dht_create_cbk, subvol, subvol,
+                          subvol->fops->create, loc, flags, mode, umask, fd,
+                          params);
 
-        cached_subvol = local->cached_subvol;
-        if (!cached_subvol) {
-                gf_msg_debug (this->name, 0,
-                              "no cached subvolume for path=%s", loc->path);
-                op_errno = EINVAL;
-                goto err;
-        }
+    } else {
+        avail_subvol = dht_free_disk_available_subvol(this, subvol, local);
 
-        local->flags = xflag;
-        if (hashed_subvol && hashed_subvol != cached_subvol) {
-                STACK_WIND (frame, dht_unlink_linkfile_cbk,
-                            hashed_subvol, hashed_subvol->fops->unlink, loc,
-                            xflag, xdata);
-        } else {
-                STACK_WIND (frame, dht_unlink_cbk,
-                            cached_subvol, cached_subvol->fops->unlink, loc,
-                            xflag, xdata);
+        if (avail_subvol != subvol) {
+            local->cached_subvol = avail_subvol;
+            local->hashed_subvol = subvol;
+
+            gf_msg_debug(this->name, 0, "creating %s on %s (link at %s)",
+                         loc->path, avail_subvol->name, subvol->name);
+
+            dht_linkfile_create(frame, dht_create_linkfile_create_cbk, this,
+                                avail_subvol, subvol, loc);
+
+            goto out;
         }
 
-        return 0;
-err:
-        op_errno = (op_errno == -1) ? errno : op_errno;
-        DHT_STACK_UNWIND (unlink, frame, -1, op_errno, NULL, NULL, NULL);
+        gf_msg_debug(this->name, 0, "creating %s on %s", loc->path,
+                     subvol->name);
 
-        return 0;
+        dht_set_parent_layout_in_dict(loc, this, local);
+
+        STACK_WIND_COOKIE(frame, dht_create_cbk, subvol, subvol,
+                          subvol->fops->create, loc, flags, mode, umask, fd,
+                          params);
+    }
+out:
+    return 0;
 }
 
 int
-dht_link_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-              int op_ret, int op_errno,
-              inode_t *inode, struct iatt *stbuf, struct iatt *preparent,
-              struct iatt *postparent, dict_t *xdata)
+dht_build_parent_loc(xlator_t *this, loc_t *parent, loc_t *child,
+                     int32_t *op_errno)
 {
-        dht_local_t  *local = NULL;
-        int           ret = -1;
-        gf_boolean_t  stbuf_merged = _gf_false;
-        xlator_t     *subvol = NULL;
+    inode_table_t *table = NULL;
+    int ret = -1;
 
-        local = frame->local;
+    if (!parent || !child) {
+        if (op_errno)
+            *op_errno = EINVAL;
+        goto out;
+    }
 
-        if (op_ret == -1) {
-                /* No continuation on DHT inode missing errors, as we should
-                 * then have a good stbuf that states P2 happened. We would
-                 * get inode missing if, the file completed migrated between
-                 * the lookup and the link call */
-                goto out;
+    if (child->parent) {
+        parent->inode = inode_ref(child->parent);
+        if (!parent->inode) {
+            if (op_errno)
+                *op_errno = EINVAL;
+            goto out;
         }
 
-        /* Update parent on success, even if P1/2 checks are positve.
-         * The second call on success will further update the parent */
-        if (local->loc.parent) {
-                dht_inode_ctx_time_update (local->loc.parent, this,
-                                           preparent, 0);
-                dht_inode_ctx_time_update (local->loc.parent, this,
-                                           postparent, 1);
-        }
-
-        /* Update linkto attrs, if this is the first call and non-P2,
-         * if we detect P2 then we need to trust the attrs from the
-         * second call, not the first */
-        if (local->linked == _gf_true &&
-            ((local->call_cnt == 1 && !IS_DHT_MIGRATION_PHASE2 (stbuf))
-             || (local->call_cnt != 1 &&
-                 IS_DHT_MIGRATION_PHASE2 (&local->stbuf)))) {
-                dht_iatt_merge (this, &local->stbuf, stbuf, NULL);
-                stbuf_merged = _gf_true;
-                dht_linkfile_attr_heal (frame, this);
-        }
-
-        /* No further P1/2 checks if we are in the second iteration of
-         * the call */
-        if (local->call_cnt != 1) {
-                goto out;
-        } else {
-                /* Preserve the return values, in case the migration decides
-                 * to recreate the link on the same subvol that the current
-                 * hased for the link was created on. */
-                dht_iatt_merge (this, &local->preparent,
-                                preparent, NULL);
-                dht_iatt_merge (this, &local->postparent,
-                                postparent, NULL);
-                if (!stbuf_merged) {
-                        dht_iatt_merge (this, &local->stbuf,
-                                        stbuf, NULL);
-                        stbuf_merged = _gf_true;
-                }
+        gf_uuid_copy(parent->gfid, child->pargfid);
+
+        ret = 0;
 
-                local->inode = inode_ref (inode);
+        goto out;
+    } else {
+        if (gf_uuid_is_null(child->pargfid)) {
+            if (op_errno)
+                *op_errno = EINVAL;
+            goto out;
         }
 
-        local->op_errno = op_errno;
-        local->rebalance.target_op_fn = dht_link2;
-        /* Check if the rebalance phase2 is true */
-        if (IS_DHT_MIGRATION_PHASE2 (stbuf)) {
-                ret = dht_inode_ctx_get1 (this, local->loc.inode, &subvol);
-                if (!subvol) {
-                        /* Phase 2 of migration */
-                        ret = dht_rebalance_complete_check (this, frame);
-                        if (!ret)
-                                return 0;
-                } else {
-                        dht_link2 (this, frame, 0);
-                        return 0;
-                }
+        table = this->itable;
+
+        if (!table) {
+            if (op_errno) {
+                *op_errno = EINVAL;
+                goto out;
+            }
         }
 
-        /* Check if the rebalance phase1 is true */
-        if (IS_DHT_MIGRATION_PHASE1 (stbuf)) {
-                ret = dht_inode_ctx_get1 (this, local->loc.inode, &subvol);
-                if (subvol) {
-                        dht_link2 (this, frame, 0);
-                        return 0;
-                }
-                ret = dht_rebalance_in_progress_check (this, frame);
-                if (!ret)
-                        return 0;
+        parent->inode = inode_find(table, child->pargfid);
+
+        if (!parent->inode) {
+            if (op_errno) {
+                *op_errno = EINVAL;
+                goto out;
+            }
         }
-out:
-        DHT_STRIP_PHASE1_FLAGS (stbuf);
 
-        DHT_STACK_UNWIND (link, frame, op_ret, op_errno, inode, stbuf,
-                          preparent, postparent, NULL);
+        gf_uuid_copy(parent->gfid, child->pargfid);
 
-        return 0;
-}
+        ret = 0;
+    }
 
+out:
+    return ret;
+}
 
-int
-dht_link2 (xlator_t *this, call_frame_t *frame, int op_ret)
+static int32_t
+dht_create_do(call_frame_t *frame)
 {
-        dht_local_t *local  = NULL;
-        xlator_t    *subvol = NULL;
-        int          op_errno = EINVAL;
+    dht_local_t *local = NULL;
+    dht_layout_t *refreshed = NULL;
+    xlator_t *subvol = NULL;
+    xlator_t *this = NULL;
+    dht_conf_t *conf = NULL;
+    dht_methods_t *methods = NULL;
 
-        local = frame->local;
-        if (!local)
-                goto err;
+    local = frame->local;
 
-        op_errno = local->op_errno;
-        if (op_ret == -1)
-                goto err;
+    this = THIS;
 
-        dht_inode_ctx_get1 (this, local->loc.inode, &subvol);
-        if (!subvol) {
-                subvol = local->cached_subvol;
-                if (!subvol) {
-                        op_errno = EINVAL;
-                        goto err;
-                }
-        }
+    conf = this->private;
 
-        /* Second call to create link file could result in EEXIST as the
-         * first call created the linkto in the currently
-         * migrating subvol, which could be the new hashed subvol */
-        if (local->link_subvol == subvol) {
-                DHT_STRIP_PHASE1_FLAGS (&local->stbuf);
-                DHT_STACK_UNWIND (link, frame, 0, 0, local->inode,
-                                  &local->stbuf, &local->preparent,
-                                  &local->postparent, NULL);
+    GF_VALIDATE_OR_GOTO(this->name, conf, err);
 
-                return 0;
-        }
+    methods = &(conf->methods);
 
-        local->call_cnt = 2;
+    /* We don't need parent_loc anymore */
+    loc_wipe(&local->loc);
 
-        STACK_WIND (frame, dht_link_cbk, subvol, subvol->fops->link,
-                    &local->loc, &local->loc2, NULL);
+    loc_copy(&local->loc, &local->loc2);
 
-        return 0;
+    loc_wipe(&local->loc2);
+
+    refreshed = local->selfheal.refreshed_layout;
+
+    subvol = methods->layout_search(this, refreshed, local->loc.name);
+
+    if (!subvol) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, DHT_MSG_HASHED_SUBVOL_GET_FAILED,
+               "no subvolume in "
+               "layout for path=%s",
+               local->loc.path);
+        local->op_errno = ENOENT;
+        goto err;
+    }
+
+    dht_create_wind_to_avail_subvol(frame, this, subvol, &local->loc,
+                                    local->flags, local->mode, local->umask,
+                                    local->fd, local->params);
+    return 0;
 err:
-        DHT_STACK_UNWIND (link, frame, -1, op_errno, NULL, NULL, NULL,
-                          NULL, NULL);
+    local->refresh_layout_unlock(frame, this, -1, 1);
 
-        return 0;
+    return 0;
 }
 
-int
-dht_link_linkfile_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                       int op_ret, int op_errno,
-                       inode_t *inode, struct iatt *stbuf,
-                       struct iatt *preparent, struct iatt *postparent,
-                       dict_t *xdata)
+static int32_t
+dht_create_unlock_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                      int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
-        dht_local_t  *local = NULL;
-        xlator_t     *srcvol = NULL;
-
-        if (op_ret == -1)
-                goto err;
+    DHT_STACK_DESTROY(frame);
+    return 0;
+}
 
-        local = frame->local;
-        srcvol = local->linkfile.srcvol;
+static int32_t
+dht_create_finish(call_frame_t *frame, xlator_t *this, int op_ret,
+                  int invoke_cbk)
+{
+    dht_local_t *local = NULL, *lock_local = NULL;
+    call_frame_t *lock_frame = NULL;
+    int lock_count = 0;
+
+    local = frame->local;
+    lock_count = dht_lock_count(local->lock[0].layout.parent_layout.locks,
+                                local->lock[0].layout.parent_layout.lk_count);
+    if (lock_count == 0)
+        goto done;
+
+    lock_frame = copy_frame(frame);
+    if (lock_frame == NULL) {
+        goto done;
+    }
+
+    lock_local = dht_local_init(lock_frame, &local->loc, NULL,
+                                lock_frame->root->op);
+    if (lock_local == NULL) {
+        goto done;
+    }
+
+    lock_local->lock[0]
+        .layout.parent_layout.locks = local->lock[0].layout.parent_layout.locks;
+    lock_local->lock[0].layout.parent_layout.lk_count =
+        local->lock[0].layout.parent_layout.lk_count;
+
+    local->lock[0].layout.parent_layout.locks = NULL;
+    local->lock[0].layout.parent_layout.lk_count = 0;
+
+    dht_unlock_inodelk(lock_frame,
+                       lock_local->lock[0].layout.parent_layout.locks,
+                       lock_local->lock[0].layout.parent_layout.lk_count,
+                       dht_create_unlock_cbk);
+    lock_frame = NULL;
 
-        STACK_WIND (frame, dht_link_cbk, srcvol, srcvol->fops->link,
-                    &local->loc, &local->loc2, xdata);
+done:
+    if (lock_frame != NULL) {
+        DHT_STACK_DESTROY(lock_frame);
+    }
 
+    if (op_ret == 0)
         return 0;
 
-err:
-        DHT_STRIP_PHASE1_FLAGS (stbuf);
-        DHT_STACK_UNWIND (link, frame, op_ret, op_errno, inode, stbuf, preparent,
-                          postparent, NULL);
-
-        return 0;
+    DHT_STACK_UNWIND(create, frame, op_ret, local->op_errno, NULL, NULL, NULL,
+                     NULL, NULL, NULL);
+    return 0;
 }
 
-
-int
-dht_link (call_frame_t *frame, xlator_t *this,
-          loc_t *oldloc, loc_t *newloc, dict_t *xdata)
+static int32_t
+dht_create_lock_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                    int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
-        xlator_t    *cached_subvol = NULL;
-        xlator_t    *hashed_subvol = NULL;
-        int          op_errno = -1;
-        int          ret = -1;
-        dht_local_t *local = NULL;
+    dht_local_t *local = NULL;
 
-        VALIDATE_OR_GOTO (frame, err);
-        VALIDATE_OR_GOTO (this, err);
-        VALIDATE_OR_GOTO (oldloc, err);
-        VALIDATE_OR_GOTO (newloc, err);
+    local = frame->local;
 
-        local = dht_local_init (frame, oldloc, NULL, GF_FOP_LINK);
-        if (!local) {
-                op_errno = ENOMEM;
+    if (!local) {
+        goto err;
+    }
 
-                goto err;
-        }
-        local->call_cnt = 1;
+    if (op_ret < 0) {
+        gf_msg("DHT", GF_LOG_ERROR, 0, DHT_MSG_INODE_LK_ERROR,
+               "Create lock failed for file: %s", local->loc2.name);
 
-        cached_subvol = local->cached_subvol;
-        if (!cached_subvol) {
-                gf_msg_debug (this->name, 0,
-                              "no cached subvolume for path=%s", oldloc->path);
-                op_errno = ENOENT;
-                goto err;
-        }
+        local->op_errno = op_errno;
 
-        hashed_subvol = dht_subvol_get_hashed (this, newloc);
-        if (!hashed_subvol) {
-                gf_msg_debug (this->name, 0,
-                              "no subvolume in layout for path=%s",
-                              newloc->path);
-                op_errno = ENOENT;
-                goto err;
-        }
+        goto err;
+    }
 
-        ret = loc_copy (&local->loc2, newloc);
-        if (ret == -1) {
-                op_errno = ENOMEM;
-                goto err;
-        }
+    local->refresh_layout_unlock = dht_create_finish;
 
-        if (hashed_subvol != cached_subvol) {
-                gf_uuid_copy (local->gfid, oldloc->inode->gfid);
-                dht_linkfile_create (frame, dht_link_linkfile_cbk, this,
-                                     cached_subvol, hashed_subvol, newloc);
-        } else {
-                STACK_WIND (frame, dht_link_cbk,
-                            cached_subvol, cached_subvol->fops->link,
-                            oldloc, newloc, xdata);
-        }
+    local->refresh_layout_done = dht_create_do;
 
-        return 0;
+    dht_refresh_layout(frame);
 
+    return 0;
 err:
-        op_errno = (op_errno == -1) ? errno : op_errno;
-        DHT_STACK_UNWIND (link, frame, -1, op_errno, NULL, NULL, NULL, NULL, NULL);
-
-        return 0;
+    if (local)
+        dht_create_finish(frame, this, -1, 0);
+    else
+        DHT_STACK_UNWIND(create, frame, -1, EINVAL, NULL, NULL, NULL, NULL,
+                         NULL, NULL);
+    return 0;
 }
 
-
-int
-dht_create_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                int op_ret, int op_errno,
-                fd_t *fd, inode_t *inode, struct iatt *stbuf,
-                struct iatt *preparent, struct iatt *postparent, dict_t *xdata)
+int32_t
+dht_create_lock(call_frame_t *frame, xlator_t *subvol)
 {
-        call_frame_t *prev = NULL;
-        int           ret = -1;
-        dht_local_t  *local = NULL;
+    dht_local_t *local = NULL;
+    int count = 1, ret = -1;
+    dht_lock_t **lk_array = NULL;
 
-        if (op_ret == -1)
-                goto out;
+    GF_VALIDATE_OR_GOTO("dht", frame, err);
+    GF_VALIDATE_OR_GOTO(frame->this->name, frame->local, err);
 
-        local = frame->local;
-        if (!local) {
-                op_ret = -1;
-                op_errno = EINVAL;
-                goto out;
-        }
+    local = frame->local;
 
-        prev = cookie;
+    lk_array = GF_CALLOC(count, sizeof(*lk_array), gf_common_mt_pointer);
 
-        if (local->loc.parent) {
-                dht_inode_ctx_time_update (local->loc.parent, this,
-                                           preparent, 0);
+    if (lk_array == NULL)
+        goto err;
 
-                dht_inode_ctx_time_update (local->loc.parent, this,
-                                           postparent, 1);
-        }
+    lk_array[0] = dht_lock_new(frame->this, subvol, &local->loc, F_RDLCK,
+                               DHT_LAYOUT_HEAL_DOMAIN, NULL,
+                               IGNORE_ENOENT_ESTALE);
 
-        ret = dht_layout_preset (this, prev->this, inode);
-        if (ret != 0) {
-                gf_msg_debug (this->name, 0,
-                              "could not set preset layout for subvol %s",
-                              prev->this->name);
-                op_ret   = -1;
-                op_errno = EINVAL;
-                goto out;
-        }
-        if (local->linked == _gf_true) {
-                local->stbuf = *stbuf;
-                dht_linkfile_attr_heal (frame, this);
-        }
-out:
-        DHT_STRIP_PHASE1_FLAGS (stbuf);
-        DHT_STACK_UNWIND (create, frame, op_ret, op_errno, fd, inode, stbuf, preparent,
-                          postparent, xdata);
-        return 0;
-}
+    if (lk_array[0] == NULL)
+        goto err;
 
+    local->lock[0].layout.parent_layout.locks = lk_array;
+    local->lock[0].layout.parent_layout.lk_count = count;
 
-int
-dht_create_linkfile_create_cbk (call_frame_t *frame, void *cookie,
-                                xlator_t *this,
-                                int32_t op_ret, int32_t op_errno,
-                                inode_t *inode, struct iatt *stbuf,
-                                struct iatt *preparent, struct iatt *postparent,
-                                dict_t *xdata)
-{
-        dht_local_t  *local = NULL;
-        xlator_t     *cached_subvol = NULL;
+    ret = dht_blocking_inodelk(frame, lk_array, count, dht_create_lock_cbk);
 
-        if (op_ret == -1)
-                goto err;
+    if (ret < 0) {
+        local->lock[0].layout.parent_layout.locks = NULL;
+        local->lock[0].layout.parent_layout.lk_count = 0;
+        goto err;
+    }
 
-        local = frame->local;
-        cached_subvol = local->cached_subvol;
+    return 0;
+err:
+    if (lk_array != NULL) {
+        dht_lock_array_free(lk_array, count);
+        GF_FREE(lk_array);
+    }
 
-        STACK_WIND (frame, dht_create_cbk,
-                    cached_subvol, cached_subvol->fops->create,
-                    &local->loc, local->flags, local->mode,
-                    local->umask, local->fd, local->params);
+    return -1;
+}
+
+int
+dht_set_parent_layout_in_dict(loc_t *loc, xlator_t *this, dht_local_t *local)
+{
+    dht_conf_t *conf = this->private;
+    dht_layout_t *parent_layout = NULL;
+    int *parent_disk_layout = NULL;
+    xlator_t *hashed_subvol = NULL;
+    char pgfid[GF_UUID_BUF_SIZE] = {0};
+    int ret = 0;
+
+    gf_uuid_unparse(loc->parent->gfid, pgfid);
+
+    parent_layout = dht_layout_get(this, loc->parent);
+    hashed_subvol = dht_subvol_get_hashed(this, loc);
+
+    ret = dht_disk_layout_extract_for_subvol(this, parent_layout, hashed_subvol,
+                                             &parent_disk_layout);
+    if (ret == -1) {
+        gf_msg(this->name, GF_LOG_WARNING, local->op_errno,
+               DHT_MSG_PARENT_LAYOUT_CHANGED,
+               "%s (%s/%s) (path: %s): "
+               "extracting in-memory layout of parent failed. ",
+               gf_fop_list[local->fop], pgfid, loc->name, loc->path);
+        goto err;
+    }
+
+    ret = dict_set_str_sizen(local->params, GF_PREOP_PARENT_KEY,
+                             conf->xattr_name);
+    if (ret < 0) {
+        gf_msg(this->name, GF_LOG_WARNING, local->op_errno,
+               DHT_MSG_PARENT_LAYOUT_CHANGED,
+               "%s (%s/%s) (path: %s): "
+               "setting %s key in params dictionary failed. ",
+               gf_fop_list[local->fop], pgfid, loc->name, loc->path,
+               GF_PREOP_PARENT_KEY);
+        goto err;
+    }
+
+    ret = dict_set_bin(local->params, conf->xattr_name, parent_disk_layout,
+                       4 * 4);
+    if (ret < 0) {
+        gf_msg(this->name, GF_LOG_WARNING, local->op_errno,
+               DHT_MSG_PARENT_LAYOUT_CHANGED,
+               "%s (%s/%s) (path: %s): "
+               "setting parent-layout in params dictionary failed. ",
+               gf_fop_list[local->fop], pgfid, loc->name, loc->path);
+        goto err;
+    }
 
-        return 0;
 err:
-        DHT_STACK_UNWIND (create, frame, -1, op_errno, NULL, NULL, NULL,
-                          NULL, NULL, NULL);
-        return 0;
+    dht_layout_unref(this, parent_layout);
+    return ret;
 }
 
 int
-dht_create (call_frame_t *frame, xlator_t *this,
-            loc_t *loc, int32_t flags, mode_t mode,
-            mode_t umask, fd_t *fd, dict_t *params)
+dht_create(call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t flags,
+           mode_t mode, mode_t umask, fd_t *fd, dict_t *params)
 {
-        int          op_errno = -1;
-        xlator_t    *subvol = NULL;
-        dht_local_t *local = NULL;
-        xlator_t    *avail_subvol = NULL;
-
-        VALIDATE_OR_GOTO (frame, err);
-        VALIDATE_OR_GOTO (this, err);
-        VALIDATE_OR_GOTO (loc, err);
+    int op_errno = -1;
+    xlator_t *subvol = NULL;
+    xlator_t *hashed_subvol = NULL;
+    dht_local_t *local = NULL;
+    int i = 0;
+    dht_conf_t *conf = NULL;
+    int ret = 0;
+
+    VALIDATE_OR_GOTO(frame, err);
+    VALIDATE_OR_GOTO(this, err);
+    VALIDATE_OR_GOTO(loc, err);
+
+    conf = this->private;
+
+    dht_get_du_info(frame, this, loc);
+
+    local = dht_local_init(frame, loc, fd, GF_FOP_CREATE);
+    if (!local) {
+        op_errno = ENOMEM;
+        goto err;
+    }
+
+    local->params = dict_ref(params);
+    local->flags = flags;
+    local->mode = mode;
+    local->umask = umask;
+
+    if (dht_filter_loc_subvol_key(this, loc, &local->loc, &subvol)) {
+        gf_msg(this->name, GF_LOG_INFO, 0, DHT_MSG_SUBVOL_INFO,
+               "creating %s on %s (got create on %s)", local->loc.path,
+               subvol->name, loc->path);
+
+        /* Since lookup-optimize is enabled by default, we need
+         * to create the linkto file if required.
+         * Note this does not check for decommisioned bricks
+         * and min-free-disk limits as this is a debugging tool
+         * and not expected to be used in production.
+         */
+        hashed_subvol = dht_subvol_get_hashed(this, &local->loc);
 
-        dht_get_du_info (frame, this, loc);
+        if (hashed_subvol && (hashed_subvol != subvol)) {
+            /* Create the linkto file and then the data file */
+            local->cached_subvol = subvol;
+            local->hashed_subvol = hashed_subvol;
 
-        local = dht_local_init (frame, loc, fd, GF_FOP_CREATE);
-        if (!local) {
-                op_errno = ENOMEM;
-                goto err;
+            dht_linkfile_create(frame, dht_create_linkfile_create_cbk, this,
+                                subvol, hashed_subvol, &local->loc);
+            goto done;
         }
+        /* We either don't have a hashed subvol or the hashed subvol is
+         * the same as the one specified. No need to create the linkto
+         * file as we expect a lookup everywhere if there are problems
+         * with the parent layout
+         */
 
-        if (dht_filter_loc_subvol_key (this, loc, &local->loc,
-                                       &subvol)) {
-                gf_log (this->name, GF_LOG_INFO,
-                        "creating %s on %s (got create on %s)",
-                        local->loc.path, subvol->name, loc->path);
-                STACK_WIND (frame, dht_create_cbk,
-                            subvol, subvol->fops->create,
-                            &local->loc, flags, mode, umask, fd, params);
-                goto done;
-        }
+        dht_set_parent_layout_in_dict(loc, this, local);
+
+        STACK_WIND_COOKIE(frame, dht_create_cbk, subvol, subvol,
+                          subvol->fops->create, &local->loc, flags, mode, umask,
+                          fd, params);
+        goto done;
+    }
+
+    subvol = dht_subvol_get_hashed(this, loc);
+    if (!subvol) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, DHT_MSG_HASHED_SUBVOL_GET_FAILED,
+               "no subvolume in layout for path=%s", loc->path);
+
+        op_errno = EIO;
+        goto err;
+    }
+
+    /* Post remove-brick, the client layout may not be in sync with
+     * disk layout because of lack of lookup. Hence,a create call
+     * may fall on the decommissioned brick.  Hence, if the
+     * hashed_subvol is part of decommissioned bricks  list, do a
+     * lookup on parent dir. If a fix-layout is already done by the
+     * remove-brick process, the parent directory layout will be in
+     * sync with that of the disk. If fix-layout is still ending
+     * on the parent directory, we can let the file get created on
+     * the decommissioned brick which will be eventually migrated to
+     * non-decommissioned brick based on the new layout.
+     */
+
+    if (conf->decommission_subvols_cnt) {
+        for (i = 0; i < conf->subvolume_cnt; i++) {
+            if (conf->decommissioned_bricks[i] &&
+                conf->decommissioned_bricks[i] == subvol) {
+                gf_msg_debug(this->name, 0,
+                             "hashed subvol:%s is "
+                             "part of decommission brick list for "
+                             "file: %s",
+                             subvol->name, loc->path);
+
+                /* dht_refresh_layout needs directory info in
+                 * local->loc. Hence, storing the parent_loc in
+                 * local->loc and storing the create context in
+                 * local->loc2. We will restore this information
+                 * in dht_creation do */
+
+                ret = loc_copy(&local->loc2, &local->loc);
+                if (ret) {
+                    gf_msg(this->name, GF_LOG_ERROR, ENOMEM, DHT_MSG_NO_MEMORY,
+                           "loc_copy failed %s", loc->path);
 
-        subvol = dht_subvol_get_hashed (this, loc);
-        if (!subvol) {
-                gf_msg_debug (this->name, 0,
-                              "no subvolume in layout for path=%s",
-                              loc->path);
-                op_errno = ENOENT;
-                goto err;
-        }
+                    goto err;
+                }
 
-        if (!dht_is_subvol_filled (this, subvol)) {
-                gf_msg_trace (this->name, 0,
-                              "creating %s on %s", loc->path,
-                               subvol->name);
-                STACK_WIND (frame, dht_create_cbk,
-                            subvol, subvol->fops->create,
-                            loc, flags, mode, umask, fd, params);
-                goto done;
-        }
-        /* Choose the minimum filled volume, and create the
-           files there */
-        avail_subvol = dht_free_disk_available_subvol (this, subvol, local);
-        if (avail_subvol != subvol) {
-                local->params = dict_ref (params);
-                local->flags = flags;
-                local->mode = mode;
-                local->umask = umask;
-                local->cached_subvol = avail_subvol;
-                local->hashed_subvol = subvol;
-                gf_msg_trace (this->name, 0,
-                              "creating %s on %s (link at %s)", loc->path,
-                              avail_subvol->name, subvol->name);
-                dht_linkfile_create (frame, dht_create_linkfile_create_cbk,
-                                     this, avail_subvol, subvol, loc);
-                goto done;
-        }
-        gf_msg_trace (this->name, 0,
-                      "creating %s on %s", loc->path, subvol->name);
-        STACK_WIND (frame, dht_create_cbk,
-                    subvol, subvol->fops->create,
-                    loc, flags, mode, umask, fd, params);
-done:
-        return 0;
+                loc_wipe(&local->loc);
 
-err:
-        op_errno = (op_errno == -1) ? errno : op_errno;
-        DHT_STACK_UNWIND (create, frame, -1, op_errno, NULL, NULL, NULL,
-                          NULL, NULL, NULL);
+                ret = dht_build_parent_loc(this, &local->loc, loc, &op_errno);
 
-        return 0;
-}
+                if (ret) {
+                    gf_msg(this->name, GF_LOG_ERROR, ENOMEM, DHT_MSG_LOC_FAILED,
+                           "parent loc build failed");
+                    goto err;
+                }
 
+                ret = dht_create_lock(frame, subvol);
 
-int
-dht_mkdir_selfheal_cbk (call_frame_t *frame, void *cookie,
-                        xlator_t *this,
-                        int32_t op_ret, int32_t op_errno, dict_t *xdata)
-{
-        dht_local_t   *local = NULL;
-        dht_layout_t  *layout = NULL;
+                if (ret < 0) {
+                    gf_msg(this->name, GF_LOG_ERROR, 0, DHT_MSG_INODE_LK_ERROR,
+                           "locking parent failed");
+                    goto err;
+                }
 
-        local = frame->local;
-        layout = local->selfheal.layout;
+                goto done;
+            }
+        }
+    }
 
-        if (op_ret == 0) {
-                dht_layout_set (this, local->inode, layout);
-                if (local->loc.parent) {
-                        dht_inode_ctx_time_update (local->loc.parent, this,
-                                                   &local->preparent, 0);
+    dht_create_wind_to_avail_subvol(frame, this, subvol, loc, flags, mode,
+                                    umask, fd, params);
+done:
+    return 0;
 
-                        dht_inode_ctx_time_update (local->loc.parent, this,
-                                                   &local->postparent, 1);
-                }
-        }
+err:
 
-        DHT_STACK_UNWIND (mkdir, frame, op_ret, op_errno,
-                          local->inode, &local->stbuf, &local->preparent,
-                          &local->postparent, NULL);
+    op_errno = (op_errno == -1) ? errno : op_errno;
+    DHT_STACK_UNWIND(create, frame, -1, op_errno, NULL, NULL, NULL, NULL, NULL,
+                     NULL);
 
-        return 0;
+    return 0;
 }
 
-int
-dht_mkdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-               int op_ret, int op_errno, inode_t *inode, struct iatt *stbuf,
-               struct iatt *preparent, struct iatt *postparent, dict_t *xdata)
+static int
+dht_mkdir_selfheal_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                       int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
-        dht_local_t  *local = NULL;
-        int           this_call_cnt = 0;
-        int           ret = -1;
-        gf_boolean_t subvol_filled = _gf_false;
-        gf_boolean_t dir_exists = _gf_false;
-        call_frame_t *prev = NULL;
-        dht_layout_t *layout = NULL;
+    dht_local_t *local = NULL;
+    dht_layout_t *layout = NULL;
 
-        local = frame->local;
-        prev  = cookie;
-        layout = local->layout;
+    local = frame->local;
+    layout = local->selfheal.layout;
 
-        subvol_filled = dht_is_subvol_filled (this, prev->this);
+    FRAME_SU_UNDO(frame, dht_local_t);
+    dht_set_fixed_dir_stat(&local->preparent);
+    dht_set_fixed_dir_stat(&local->postparent);
 
-        LOCK (&frame->lock);
-        {
-                if (subvol_filled && (op_ret != -1)) {
-                        ret = dht_layout_merge (this, layout, prev->this,
-                                                -1, ENOSPC, NULL);
-                } else {
-			if (op_ret == -1 && op_errno == EEXIST) {
-				/* Very likely just a race between mkdir and
-				   self-heal (from lookup of a concurrent mkdir
-				   attempt).
-				   Ignore error for now. layout setting will
-				   anyways fail if this was a different (old)
-				   pre-existing different directory.
-				*/
-				op_ret = 0;
-                                dir_exists = _gf_true;
-                        }
-                        ret = dht_layout_merge (this, layout, prev->this,
-                                                op_ret, op_errno, NULL);
-                }
-                if (ret)
-                        gf_msg (this->name, GF_LOG_WARNING, 0,
-                                DHT_MSG_LAYOUT_MERGE_FAILED,
-                                "%s: failed to merge layouts for subvol %s",
-                                local->loc.path, prev->this->name);
-
-                if (op_ret == -1) {
-                        local->op_errno = op_errno;
-                        goto unlock;
-                }
+    if (op_ret == 0) {
+        dht_layout_set(this, local->inode, layout);
 
-                if (dir_exists)
-                        goto unlock;
+        dht_inode_ctx_time_update(local->inode, this, &local->stbuf, 1);
+        if (local->loc.parent) {
+            dht_inode_ctx_time_update(local->loc.parent, this,
+                                      &local->preparent, 0);
 
-                dht_iatt_merge (this, &local->stbuf, stbuf, prev->this);
-                dht_iatt_merge (this, &local->preparent, preparent, prev->this);
-                dht_iatt_merge (this, &local->postparent, postparent,
-                                prev->this);
+            dht_inode_ctx_time_update(local->loc.parent, this,
+                                      &local->postparent, 1);
         }
-unlock:
-        UNLOCK (&frame->lock);
+    }
 
-        this_call_cnt = dht_frame_return (frame);
-        if (is_last_call (this_call_cnt)) {
-                dht_selfheal_new_directory (frame, dht_mkdir_selfheal_cbk,
-                                            layout);
-        }
+    DHT_STACK_UNWIND(mkdir, frame, op_ret, op_errno, local->inode,
+                     &local->stbuf, &local->preparent, &local->postparent,
+                     NULL);
 
-        return 0;
+    return 0;
 }
 
-int
-dht_mkdir_hashed_cbk (call_frame_t *frame, void *cookie,
-                      xlator_t *this, int op_ret, int op_errno,
-                      inode_t *inode, struct iatt *stbuf,
-                      struct iatt *preparent, struct iatt *postparent,
-                      dict_t *xdata)
-{
-        dht_local_t  *local = NULL;
-        int           ret = -1;
-        call_frame_t *prev = NULL;
-        dht_layout_t *layout = NULL;
-        dht_conf_t   *conf = NULL;
-        int           i = 0;
-        xlator_t     *hashed_subvol = NULL;
-
-        VALIDATE_OR_GOTO (this->private, err);
-
-        local = frame->local;
-        prev  = cookie;
-        layout = local->layout;
-        conf = this->private;
-        hashed_subvol = local->hashed_subvol;
-
-        if (gf_uuid_is_null (local->loc.gfid) && !op_ret)
-                gf_uuid_copy (local->loc.gfid, stbuf->ia_gfid);
-
-        if (dht_is_subvol_filled (this, hashed_subvol))
-                ret = dht_layout_merge (this, layout, prev->this,
-                                        -1, ENOSPC, NULL);
-        else
-                ret = dht_layout_merge (this, layout, prev->this,
-                                        op_ret, op_errno, NULL);
-
-        /* TODO: we may have to return from the function
-           if layout merge fails. For now, lets just log an error */
+static int
+dht_mkdir_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int op_ret,
+              int op_errno, inode_t *inode, struct iatt *stbuf,
+              struct iatt *preparent, struct iatt *postparent, dict_t *xdata)
+{
+    dht_local_t *local = NULL;
+    int this_call_cnt = 0;
+    int ret = -1;
+    gf_boolean_t subvol_filled = _gf_false;
+    gf_boolean_t dir_exists = _gf_false;
+    xlator_t *prev = NULL;
+    dht_layout_t *layout = NULL;
+
+    local = frame->local;
+    prev = cookie;
+    layout = local->layout;
+
+    subvol_filled = dht_is_subvol_filled(this, prev);
+
+    LOCK(&frame->lock);
+    {
+        if (subvol_filled && (op_ret != -1)) {
+            ret = dht_layout_merge(this, layout, prev, -1, ENOSPC, NULL);
+        } else {
+            if (op_ret == -1 && op_errno == EEXIST) {
+                /* Very likely just a race between mkdir and
+                   self-heal (from lookup of a concurrent mkdir
+                   attempt).
+                   Ignore error for now. layout setting will
+                   anyways fail if this was a different (old)
+                   pre-existing different directory.
+                */
+                op_ret = 0;
+                dir_exists = _gf_true;
+            }
+            ret = dht_layout_merge(this, layout, prev, op_ret, op_errno, NULL);
+        }
         if (ret)
-                gf_msg (this->name, GF_LOG_WARNING, 0,
-                        DHT_MSG_LAYOUT_MERGE_FAILED,
-                        "%s: failed to merge layouts for subvol %s",
-                        local->loc.path, prev->this->name);
+            gf_msg(this->name, GF_LOG_WARNING, 0, DHT_MSG_LAYOUT_MERGE_FAILED,
+                   "%s: failed to merge layouts for subvol %s", local->loc.path,
+                   prev->name);
 
         if (op_ret == -1) {
-                local->op_errno = op_errno;
-                goto err;
+            local->op_errno = op_errno;
+            goto unlock;
         }
-        local->op_ret = 0;
 
-        dht_iatt_merge (this, &local->stbuf, stbuf, prev->this);
-        dht_iatt_merge (this, &local->preparent, preparent, prev->this);
-        dht_iatt_merge (this, &local->postparent, postparent, prev->this);
+        if (dir_exists)
+            goto unlock;
 
-        local->call_cnt = conf->subvolume_cnt - 1;
+        dht_iatt_merge(this, &local->stbuf, stbuf);
+        dht_iatt_merge(this, &local->preparent, preparent);
+        dht_iatt_merge(this, &local->postparent, postparent);
+    }
+unlock:
+    UNLOCK(&frame->lock);
 
-        if (gf_uuid_is_null (local->loc.gfid))
-                gf_uuid_copy (local->loc.gfid, stbuf->ia_gfid);
-        if (local->call_cnt == 0) {
-                dht_selfheal_directory (frame, dht_mkdir_selfheal_cbk,
-                                        &local->loc, layout);
-        }
-        for (i = 0; i < conf->subvolume_cnt; i++) {
-                if (conf->subvolumes[i] == hashed_subvol)
-                        continue;
-                STACK_WIND (frame, dht_mkdir_cbk,
-                            conf->subvolumes[i],
-                            conf->subvolumes[i]->fops->mkdir, &local->loc,
-                            local->mode, local->umask, local->params);
-        }
-        return 0;
-err:
-        DHT_STACK_UNWIND (mkdir, frame, -1, op_errno, NULL, NULL, NULL,
-                          NULL, NULL);
-        return 0;
+    this_call_cnt = dht_frame_return(frame);
+    if (is_last_call(this_call_cnt)) {
+        /*Unlock entrylk and inodelk once mkdir is done on all subvols*/
+        dht_unlock_namespace(frame, &local->lock[0]);
+        FRAME_SU_DO(frame, dht_local_t);
+        dht_selfheal_new_directory(frame, dht_mkdir_selfheal_cbk, layout);
+    }
+
+    return 0;
 }
 
+static int
+dht_mkdir_hashed_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                     int op_ret, int op_errno, inode_t *inode,
+                     struct iatt *stbuf, struct iatt *preparent,
+                     struct iatt *postparent, dict_t *xdata);
 
-int
-dht_mkdir (call_frame_t *frame, xlator_t *this,
-           loc_t *loc, mode_t mode, mode_t umask, dict_t *params)
+static int
+dht_mkdir_helper(call_frame_t *frame, xlator_t *this, loc_t *loc, mode_t mode,
+                 mode_t umask, dict_t *params)
 {
-        dht_local_t  *local  = NULL;
-        dht_conf_t   *conf = NULL;
-        int           op_errno = -1;
-        xlator_t     *hashed_subvol = NULL;
+    dht_local_t *local = NULL;
+    dht_conf_t *conf = NULL;
+    int op_errno = -1, ret = -1;
+    xlator_t *hashed_subvol = NULL;
+    int32_t *parent_disk_layout = NULL;
+    dht_layout_t *parent_layout = NULL;
+    char pgfid[GF_UUID_BUF_SIZE] = {0};
+
+    VALIDATE_OR_GOTO(frame, err);
+    VALIDATE_OR_GOTO(this, err);
+    VALIDATE_OR_GOTO(loc, err);
+    VALIDATE_OR_GOTO(loc->inode, err);
+    VALIDATE_OR_GOTO(loc->path, err);
+    VALIDATE_OR_GOTO(this->private, err);
+
+    gf_uuid_unparse(loc->parent->gfid, pgfid);
+
+    conf = this->private;
+    local = frame->local;
+
+    if (local->op_ret == -1) {
+        gf_msg(this->name, GF_LOG_WARNING, local->op_errno,
+               DHT_MSG_PARENT_LAYOUT_CHANGED,
+               "mkdir (%s/%s) (path: %s): refreshing parent layout "
+               "failed.",
+               pgfid, loc->name, loc->path);
 
+        op_errno = local->op_errno;
+        goto err;
+    }
+
+    local->op_ret = -1;
+
+    hashed_subvol = dht_subvol_get_hashed(this, loc);
+    if (hashed_subvol == NULL) {
+        gf_msg_debug(this->name, 0,
+                     "mkdir (%s/%s) (path: %s): hashed subvol not "
+                     "found",
+                     pgfid, loc->name, loc->path);
+        op_errno = ENOENT;
+        goto err;
+    }
+
+    local->hashed_subvol = hashed_subvol;
+
+    parent_layout = dht_layout_get(this, loc->parent);
+
+    ret = dht_disk_layout_extract_for_subvol(this, parent_layout, hashed_subvol,
+                                             &parent_disk_layout);
+    if (ret == -1) {
+        gf_msg(this->name, GF_LOG_WARNING, EIO, DHT_MSG_PARENT_LAYOUT_CHANGED,
+               "mkdir (%s/%s) (path: %s): "
+               "extracting in-memory layout of parent failed. ",
+               pgfid, loc->name, loc->path);
+        goto err;
+    }
+
+    if (memcmp(local->parent_disk_layout, parent_disk_layout,
+               sizeof(local->parent_disk_layout)) == 0) {
+        gf_msg(this->name, GF_LOG_WARNING, EIO, DHT_MSG_PARENT_LAYOUT_CHANGED,
+               "mkdir (%s/%s) (path: %s): loop detected. "
+               "parent layout didn't change even though "
+               "previous attempt of mkdir failed because of "
+               "in-memory layout not matching with that on disk.",
+               pgfid, loc->name, loc->path);
+        op_errno = EIO;
+        goto err;
+    }
+
+    memcpy((void *)local->parent_disk_layout, (void *)parent_disk_layout,
+           sizeof(local->parent_disk_layout));
+
+    dht_layout_unref(this, parent_layout);
+    parent_layout = NULL;
+
+    ret = dict_set_str(params, GF_PREOP_PARENT_KEY, conf->xattr_name);
+    if (ret < 0) {
+        local->op_errno = -ret;
+        gf_msg(this->name, GF_LOG_WARNING, local->op_errno,
+               DHT_MSG_PARENT_LAYOUT_CHANGED,
+               "mkdir (%s/%s) (path: %s): "
+               "setting %s key in params dictionary failed. ",
+               pgfid, loc->name, loc->path, GF_PREOP_PARENT_KEY);
+        goto err;
+    }
+
+    ret = dict_set_bin(params, conf->xattr_name, parent_disk_layout, 4 * 4);
+    if (ret < 0) {
+        local->op_errno = -ret;
+        gf_msg(this->name, GF_LOG_WARNING, local->op_errno,
+               DHT_MSG_PARENT_LAYOUT_CHANGED,
+               "setting parent-layout in params dictionary failed. "
+               "mkdir (%s/%s) (path: %s)",
+               pgfid, loc->name, loc->path);
+        goto err;
+    }
+
+    parent_disk_layout = NULL;
+
+    STACK_WIND_COOKIE(frame, dht_mkdir_hashed_cbk, hashed_subvol, hashed_subvol,
+                      hashed_subvol->fops->mkdir, loc, mode, umask, params);
+
+    return 0;
 
-        VALIDATE_OR_GOTO (frame, err);
-        VALIDATE_OR_GOTO (this, err);
-        VALIDATE_OR_GOTO (loc, err);
-        VALIDATE_OR_GOTO (loc->inode, err);
-        VALIDATE_OR_GOTO (loc->path, err);
-        VALIDATE_OR_GOTO (this->private, err);
+err:
+    dht_unlock_namespace(frame, &local->lock[0]);
 
-        conf = this->private;
+    op_errno = local ? local->op_errno : op_errno;
+    DHT_STACK_UNWIND(mkdir, frame, -1, op_errno, NULL, NULL, NULL, NULL, NULL);
 
-        dht_get_du_info (frame, this, loc);
+    if (parent_disk_layout != NULL)
+        GF_FREE(parent_disk_layout);
 
-        local = dht_local_init (frame, loc, NULL, GF_FOP_MKDIR);
-        if (!local) {
-                op_errno = ENOMEM;
-                goto err;
-        }
+    if (parent_layout != NULL)
+        dht_layout_unref(this, parent_layout);
 
-        hashed_subvol = dht_subvol_get_hashed (this, loc);
-        if (hashed_subvol == NULL) {
-                gf_msg_debug (this->name, 0,
-                              "hashed subvol not found for %s",
-                              loc->path);
-                op_errno = ENOENT;
-                goto err;
-        }
+    return 0;
+}
 
-        local->hashed_subvol = hashed_subvol;
-        local->mode = mode;
-        local->umask = umask;
-        local->params = dict_ref (params);
-        local->inode  = inode_ref (loc->inode);
+static int
+dht_mkdir_hashed_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                     int op_ret, int op_errno, inode_t *inode,
+                     struct iatt *stbuf, struct iatt *preparent,
+                     struct iatt *postparent, dict_t *xdata)
+{
+    dht_local_t *local = NULL;
+    int ret = -1;
+    xlator_t *prev = NULL;
+    dht_layout_t *layout = NULL;
+    dht_conf_t *conf = NULL;
+    int i = 0;
+    xlator_t *hashed_subvol = NULL;
+    char pgfid[GF_UUID_BUF_SIZE] = {0};
+    gf_boolean_t parent_layout_changed = _gf_false;
+    call_stub_t *stub = NULL;
+
+    local = frame->local;
+    prev = cookie;
+    layout = local->layout;
+    conf = this->private;
+    hashed_subvol = local->hashed_subvol;
+
+    gf_uuid_unparse(local->loc.parent->gfid, pgfid);
+
+    if (gf_uuid_is_null(local->loc.gfid) && !op_ret)
+        gf_uuid_copy(local->loc.gfid, stbuf->ia_gfid);
+
+    if (op_ret == -1) {
+        local->op_errno = op_errno;
 
-        local->layout = dht_layout_new (this, conf->subvolume_cnt);
-        if (!local->layout) {
-                op_errno = ENOMEM;
+        parent_layout_changed = (xdata &&
+                                 dict_get(xdata, GF_PREOP_CHECK_FAILED))
+                                    ? 1
+                                    : 0;
+        if (parent_layout_changed) {
+            gf_msg(this->name, GF_LOG_INFO, 0, DHT_MSG_PARENT_LAYOUT_CHANGED,
+                   "mkdir (%s/%s) (path: %s): parent layout "
+                   "changed. Attempting a refresh and then a "
+                   "retry",
+                   pgfid, local->loc.name, local->loc.path);
+
+            stub = fop_mkdir_stub(frame, dht_mkdir_helper, &local->loc,
+                                  local->mode, local->umask, local->params);
+            if (stub == NULL) {
                 goto err;
-        }
-
-        STACK_WIND (frame, dht_mkdir_hashed_cbk,
-                    hashed_subvol,
-                    hashed_subvol->fops->mkdir,
-                    loc, mode, umask, params);
+            }
 
+            ret = dht_handle_parent_layout_change(this, stub);
+            if (ret) {
+                goto err;
+            }
+
+            stub = NULL;
+
+            return 0;
+        }
+
+        goto err;
+    }
+
+    dict_del(local->params, GF_PREOP_PARENT_KEY);
+    dict_del(local->params, conf->xattr_name);
+
+    if (dht_is_subvol_filled(this, hashed_subvol))
+        ret = dht_layout_merge(this, layout, prev, -1, ENOSPC, NULL);
+    else
+        ret = dht_layout_merge(this, layout, prev, op_ret, op_errno, NULL);
+
+    /* TODO: we may have to return from the function
+       if layout merge fails. For now, lets just log an error */
+    if (ret)
+        gf_msg(this->name, GF_LOG_WARNING, 0, DHT_MSG_LAYOUT_MERGE_FAILED,
+               "%s: failed to merge layouts for subvol %s", local->loc.path,
+               prev->name);
+
+    local->op_ret = 0;
+
+    dht_iatt_merge(this, &local->stbuf, stbuf);
+    dht_iatt_merge(this, &local->preparent, preparent);
+    dht_iatt_merge(this, &local->postparent, postparent);
+
+    local->call_cnt = conf->subvolume_cnt - 1;
+    /* Delete internal mds xattr from params dict to avoid store
+      internal mds xattr on other subvols
+    */
+    dict_del(local->params, conf->mds_xattr_key);
+
+    if (gf_uuid_is_null(local->loc.gfid))
+        gf_uuid_copy(local->loc.gfid, stbuf->ia_gfid);
+
+    /* Set hashed subvol as a mds subvol on inode ctx */
+    /*if (!local->inode)
+            local->inode  = inode_ref (inode);
+    */
+    ret = dht_inode_ctx_mdsvol_set(local->inode, this, hashed_subvol);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, DHT_MSG_SET_INODE_CTX_FAILED,
+               "Failed to set hashed subvol for %s on inode vol is %s",
+               local->loc.path, hashed_subvol->name);
+    }
+
+    if (local->call_cnt == 0) {
+        /*Unlock namespace lock once mkdir is done on all subvols*/
+        dht_unlock_namespace(frame, &local->lock[0]);
+        FRAME_SU_DO(frame, dht_local_t);
+        dht_selfheal_directory(frame, dht_mkdir_selfheal_cbk, &local->loc,
+                               layout);
         return 0;
-
+    }
+
+    for (i = 0; i < conf->subvolume_cnt; i++) {
+        if (conf->subvolumes[i] == hashed_subvol)
+            continue;
+        STACK_WIND_COOKIE(frame, dht_mkdir_cbk, conf->subvolumes[i],
+                          conf->subvolumes[i], conf->subvolumes[i]->fops->mkdir,
+                          &local->loc, local->mode, local->umask,
+                          local->params);
+    }
+
+    return 0;
 err:
-        op_errno = (op_errno == -1) ? errno : op_errno;
-        DHT_STACK_UNWIND (mkdir, frame, -1, op_errno, NULL, NULL, NULL,
-                          NULL, NULL);
+    if (local->op_ret != 0) {
+        dht_unlock_namespace(frame, &local->lock[0]);
+    }
 
-        return 0;
+    DHT_STACK_UNWIND(mkdir, frame, -1, op_errno, NULL, NULL, NULL, NULL, NULL);
+
+    return 0;
 }
 
+static int
+dht_mkdir_guard_parent_layout_cbk(call_frame_t *frame, xlator_t *this,
+                                  loc_t *loc, mode_t mode, mode_t umask,
+                                  dict_t *params)
+{
+    dht_local_t *local = NULL;
+    dht_conf_t *conf = 0;
+    char pgfid[GF_UUID_BUF_SIZE] = {0};
+    int ret = -1;
+    int32_t zero[1] = {0};
+
+    local = frame->local;
+    conf = this->private;
+
+    gf_uuid_unparse(loc->parent->gfid, pgfid);
+
+    if (local->op_ret < 0) {
+        gf_msg(this->name, GF_LOG_WARNING, local->op_errno,
+               DHT_MSG_PARENT_LAYOUT_CHANGED,
+               "mkdir (%s/%s) (path: %s): "
+               "Acquiring lock on parent to guard against "
+               "layout-change failed.",
+               pgfid, loc->name, loc->path);
+        goto err;
+    }
+
+    local->op_ret = -1;
+    /* Add internal MDS xattr on disk for hashed subvol
+     */
+    ret = dht_dict_set_array(params, conf->mds_xattr_key, zero, 1);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_WARNING, ENOMEM, DHT_MSG_DICT_SET_FAILED,
+               "Failed to set dictionary value:key = %s for "
+               "path %s",
+               conf->mds_xattr_key, loc->path);
+    }
+
+    STACK_WIND_COOKIE(frame, dht_mkdir_hashed_cbk, local->hashed_subvol,
+                      local->hashed_subvol, local->hashed_subvol->fops->mkdir,
+                      loc, mode, umask, params);
+
+    return 0;
+err:
+    DHT_STACK_UNWIND(mkdir, frame, -1, local->op_errno, NULL, NULL, NULL, NULL,
+                     NULL);
+
+    return 0;
+}
 
 int
-dht_rmdir_selfheal_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                        int op_ret, int op_errno, dict_t *xdata)
+dht_mkdir(call_frame_t *frame, xlator_t *this, loc_t *loc, mode_t mode,
+          mode_t umask, dict_t *params)
 {
-        dht_local_t  *local = NULL;
+    dht_local_t *local = NULL;
+    dht_conf_t *conf = NULL;
+    int op_errno = EINVAL, ret = -1;
+    xlator_t *hashed_subvol = NULL;
+    char pgfid[GF_UUID_BUF_SIZE] = {0};
+    call_stub_t *stub = NULL;
+
+    VALIDATE_OR_GOTO(frame, err);
+    VALIDATE_OR_GOTO(this, err);
+    VALIDATE_OR_GOTO(loc, err);
+    VALIDATE_OR_GOTO(loc->inode, err);
+    VALIDATE_OR_GOTO(loc->path, err);
+    VALIDATE_OR_GOTO(this->private, err);
+
+    gf_uuid_unparse(loc->parent->gfid, pgfid);
+
+    conf = this->private;
+
+    if (!params || !dict_get(params, "gfid-req")) {
+        op_errno = EPERM;
+        gf_msg_callingfn(this->name, GF_LOG_WARNING, op_errno,
+                         DHT_MSG_GFID_NULL,
+                         "mkdir: %s is received "
+                         "without gfid-req %p",
+                         loc->path, params);
+        goto err;
+    }
+
+    dht_get_du_info(frame, this, loc);
+
+    local = dht_local_init(frame, loc, NULL, GF_FOP_MKDIR);
+    if (!local) {
+        op_errno = ENOMEM;
+        goto err;
+    }
+
+    hashed_subvol = dht_subvol_get_hashed(this, loc);
+    if (hashed_subvol == NULL) {
+        gf_msg_debug(this->name, 0, "hashed subvol not found for %s",
+                     loc->path);
+        local->op_errno = EIO;
+        goto err;
+    }
+
+    local->hashed_subvol = hashed_subvol;
+    local->mode = mode;
+    local->umask = umask;
+    if (params)
+        local->params = dict_ref(params);
+
+    local->inode = inode_ref(loc->inode);
+
+    local->layout = dht_layout_new(this, conf->subvolume_cnt);
+    if (!local->layout) {
+        op_errno = ENOMEM;
+        goto err;
+    }
+
+    /* set the newly created directory hash to the commit hash
+     * if the configuration option is set. If configuration option
+     * is not set, the older clients may still be connecting to the
+     * volume and hence we need to preserve the 1 in disk[0] part of the
+     * layout xattr */
+    if (conf->lookup_optimize)
+        local->layout->commit_hash = conf->vol_commit_hash;
+    else
+        local->layout->commit_hash = DHT_LAYOUT_HASH_INVALID;
+
+    stub = fop_mkdir_stub(frame, dht_mkdir_guard_parent_layout_cbk, loc, mode,
+                          umask, params);
+    if (stub == NULL) {
+        gf_msg(this->name, GF_LOG_WARNING, ENOMEM,
+               DHT_MSG_PARENT_LAYOUT_CHANGED,
+               "mkdir (%s/%s) (path: %s): "
+               "creating stub failed.",
+               pgfid, loc->name, loc->path);
+        local->op_errno = ENOMEM;
+        goto err;
+    }
+
+    ret = dht_guard_parent_layout_and_namespace(this, stub);
+    if (ret < 0) {
+        gf_msg(this->name, GF_LOG_WARNING, 0, DHT_MSG_PARENT_LAYOUT_CHANGED,
+               "mkdir (%s/%s) (path: %s) cannot wind lock request to "
+               "guard parent layout",
+               pgfid, loc->name, loc->path);
+        goto err;
+    }
+
+    return 0;
 
-        local = frame->local;
-
-        DHT_STACK_UNWIND (rmdir, frame, local->op_ret, local->op_errno,
-                          &local->preparent, &local->postparent, NULL);
+err:
+    op_errno = local ? local->op_errno : op_errno;
+    DHT_STACK_UNWIND(mkdir, frame, -1, op_errno, NULL, NULL, NULL, NULL, NULL);
 
-        return 0;
+    return 0;
 }
 
-
-int
-dht_rmdir_hashed_subvol_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-               int op_ret, int op_errno, struct iatt *preparent,
-               struct iatt *postparent, dict_t *xdata)
+static int
+dht_rmdir_selfheal_cbk(call_frame_t *heal_frame, void *cookie, xlator_t *this,
+                       int op_ret, int op_errno, dict_t *xdata)
 {
-        dht_local_t  *local = NULL;
-        dht_conf_t   *conf = NULL;
-        int           this_call_cnt = 0;
-        call_frame_t *prev = NULL;
-        char gfid[GF_UUID_BUF_SIZE] ={0};
+    dht_local_t *local = NULL;
+    dht_local_t *heal_local = NULL;
+    call_frame_t *main_frame = NULL;
 
-        local = frame->local;
-        prev  = cookie;
-        conf = this->private;
+    heal_local = heal_frame->local;
+    main_frame = heal_local->main_frame;
+    local = main_frame->local;
 
-        gf_uuid_unparse(local->loc.gfid, gfid);
+    DHT_STACK_DESTROY(heal_frame);
+    dht_set_fixed_dir_stat(&local->preparent);
+    dht_set_fixed_dir_stat(&local->postparent);
 
-        LOCK (&frame->lock);
-        {
-                if (op_ret == -1) {
-                        local->op_errno = op_errno;
-                        local->op_ret   = -1;
-                        if (conf->subvolume_cnt != 1) {
-                                if (op_errno != ENOENT && op_errno != EACCES) {
-                                        local->need_selfheal = 1;
-                                }
-                        }
+    DHT_STACK_UNWIND(rmdir, main_frame, local->op_ret, local->op_errno,
+                     &local->preparent, &local->postparent, NULL);
 
-                        gf_msg_debug (this->name, 0,
-                                      "rmdir on %s for %s failed "
-                                      "(gfid = %s) (%s)",
-                                      prev->this->name, local->loc.path,
-                                      gfid, strerror (op_errno));
-                        goto unlock;
-                }
+    return 0;
+}
 
-                dht_iatt_merge (this, &local->preparent, preparent, prev->this);
-                dht_iatt_merge (this, &local->postparent, postparent,
-                                prev->this);
+static int
+dht_rmdir_hashed_subvol_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                            int op_ret, int op_errno, struct iatt *preparent,
+                            struct iatt *postparent, dict_t *xdata)
+{
+    dht_local_t *local = NULL;
+    dht_local_t *heal_local = NULL;
+    call_frame_t *heal_frame = NULL;
+    dht_conf_t *conf = NULL;
+    int this_call_cnt = 0;
+    xlator_t *prev = NULL;
+    char gfid[GF_UUID_BUF_SIZE] = {0};
+
+    local = frame->local;
+    prev = cookie;
+    conf = this->private;
+
+    gf_uuid_unparse(local->loc.gfid, gfid);
+
+    LOCK(&frame->lock);
+    {
+        if (op_ret == -1) {
+            local->op_errno = op_errno;
+            local->op_ret = -1;
+            if (conf->subvolume_cnt != 1) {
+                if (op_errno != ENOENT && op_errno != EACCES &&
+                    op_errno != ESTALE) {
+                    local->need_selfheal = 1;
+                }
+            }
 
+            gf_msg_debug(this->name, op_errno,
+                         "rmdir on %s for %s failed "
+                         "(gfid = %s)",
+                         prev->name, local->loc.path, gfid);
+            goto unlock;
         }
+
+        dht_iatt_merge(this, &local->preparent, preparent);
+        dht_iatt_merge(this, &local->postparent, postparent);
+    }
 unlock:
-        UNLOCK (&frame->lock);
-
-        this_call_cnt = dht_frame_return (frame);
-        if (is_last_call (this_call_cnt)) {
-               if (local->need_selfheal) {
-                        local->layout =
-                                dht_layout_get (this, local->loc.inode);
-
-                        /* TODO: neater interface needed below */
-                        local->stbuf.ia_type = local->loc.inode->ia_type;
-
-                        gf_uuid_copy (local->gfid, local->loc.inode->gfid);
-                        dht_selfheal_restore (frame, dht_rmdir_selfheal_cbk,
-                                              &local->loc, local->layout);
-               } else {
-
-                        if (local->loc.parent) {
-                                dht_inode_ctx_time_update (local->loc.parent,
-                                                           this,
-                                                           &local->preparent,
-                                                           0);
-
-                                dht_inode_ctx_time_update (local->loc.parent,
-                                                           this,
-                                                           &local->postparent,
-                                                           1);
-                        }
+    UNLOCK(&frame->lock);
 
-                        DHT_STACK_UNWIND (rmdir, frame, local->op_ret,
-                                          local->op_errno, &local->preparent,
-                                          &local->postparent, NULL);
-               }
-        }
+    this_call_cnt = dht_frame_return(frame);
+    if (is_last_call(this_call_cnt)) {
+        if (local->need_selfheal) {
+            dht_rmdir_unlock(frame, this);
+            local->layout = dht_layout_get(this, local->loc.inode);
 
-        return 0;
-}
+            /* TODO: neater interface needed below */
+            local->stbuf.ia_type = local->loc.inode->ia_type;
 
+            gf_uuid_copy(local->gfid, local->loc.inode->gfid);
 
-int
-dht_rmdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-               int op_ret, int op_errno, struct iatt *preparent,
-               struct iatt *postparent, dict_t *xdata)
-{
-        dht_local_t  *local = NULL;
-        int           this_call_cnt = 0;
-        call_frame_t *prev = NULL;
-        int           done = 0;
-        char gfid[GF_UUID_BUF_SIZE] ={0};
+            /* Use a different frame or else the rmdir op_ret is
+             * overwritten by that of the selfheal */
 
-        local = frame->local;
-        prev  = cookie;
+            heal_frame = copy_frame(frame);
+
+            if (heal_frame == NULL) {
+                goto err;
+            }
 
+            heal_local = dht_local_init(heal_frame, &local->loc, NULL, 0);
+            if (!heal_local) {
+                DHT_STACK_DESTROY(heal_frame);
+                goto err;
+            }
 
-        LOCK (&frame->lock);
-        {
-                if (op_ret == -1) {
-                        if ((op_errno != ENOENT) && (op_errno != ESTALE)) {
-                                local->op_errno = op_errno;
-                                local->op_ret = -1;
+            heal_local->inode = inode_ref(local->loc.inode);
+            heal_local->main_frame = frame;
+            gf_uuid_copy(heal_local->gfid, local->loc.inode->gfid);
 
-                                if (op_errno != EACCES)
-                                        local->need_selfheal = 1;
-                        }
+            dht_selfheal_restore(heal_frame, dht_rmdir_selfheal_cbk,
+                                 &heal_local->loc, heal_local->layout);
+            return 0;
+        } else {
+            if (local->loc.parent) {
+                dht_inode_ctx_time_update(local->loc.parent, this,
+                                          &local->preparent, 0);
 
-                        gf_uuid_unparse(local->loc.gfid, gfid);
+                dht_inode_ctx_time_update(local->loc.parent, this,
+                                          &local->postparent, 1);
+            }
 
-                        gf_msg_debug (this->name, 0,
-                                      "rmdir on %s for %s failed."
-                                      "(gfid = %s) (%s)",
-                                      prev->this->name, local->loc.path,
-                                      gfid, strerror (op_errno));
-                        goto unlock;
-                }
+            dht_set_fixed_dir_stat(&local->preparent);
+            dht_set_fixed_dir_stat(&local->postparent);
 
-                /* Track if rmdir succeeded on atleast one subvol*/
-                local->fop_succeeded = 1;
-                dht_iatt_merge (this, &local->preparent, preparent, prev->this);
-                dht_iatt_merge (this, &local->postparent, postparent,
-                                prev->this);
+            dht_rmdir_unlock(frame, this);
+            DHT_STACK_UNWIND(rmdir, frame, local->op_ret, local->op_errno,
+                             &local->preparent, &local->postparent, NULL);
         }
-unlock:
-        UNLOCK (&frame->lock);
+    }
 
+    return 0;
 
-        this_call_cnt = dht_frame_return (frame);
+err:
+    DHT_STACK_UNWIND(rmdir, frame, local->op_ret, local->op_errno, NULL, NULL,
+                     NULL);
+    return 0;
+}
 
-        /* if local->hashed_subvol, we are yet to wind to hashed_subvol. */
-        if (local->hashed_subvol && (this_call_cnt == 1)) {
-                done = 1;
-        } else if (!local->hashed_subvol && !this_call_cnt) {
-                done = 1;
-        }
+static int
+dht_rmdir_unlock_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                     int32_t op_ret, int32_t op_errno, dict_t *xdata)
+{
+    DHT_STACK_DESTROY(frame);
+    return 0;
+}
 
+static int
+dht_rmdir_unlock(call_frame_t *frame, xlator_t *this)
+{
+    dht_local_t *local = NULL, *lock_local = NULL;
+    call_frame_t *lock_frame = NULL;
+    int lock_count = 0;
 
-        if (done) {
-                if (local->need_selfheal && local->fop_succeeded) {
-                        local->layout =
-                                dht_layout_get (this, local->loc.inode);
+    local = frame->local;
 
-                        /* TODO: neater interface needed below */
-                        local->stbuf.ia_type = local->loc.inode->ia_type;
+    /* Unlock entrylk */
+    dht_unlock_entrylk_wrapper(frame, &local->lock[0].ns.directory_ns);
 
-                        gf_uuid_copy (local->gfid, local->loc.inode->gfid);
-                        dht_selfheal_restore (frame, dht_rmdir_selfheal_cbk,
-                                              &local->loc, local->layout);
-                } else if (this_call_cnt) {
-                        /* If non-hashed subvol's have responded, proceed */
+    /* Unlock inodelk */
+    lock_count = dht_lock_count(local->lock[0].ns.parent_layout.locks,
+                                local->lock[0].ns.parent_layout.lk_count);
 
-                        local->need_selfheal = 0;
-                        STACK_WIND (frame, dht_rmdir_hashed_subvol_cbk,
-                                    local->hashed_subvol,
-                                    local->hashed_subvol->fops->rmdir,
-                                    &local->loc, local->flags, NULL);
-                } else if (!this_call_cnt) {
-                        /* All subvol's have responded, proceed */
+    if (lock_count == 0)
+        goto done;
 
-                        if (local->loc.parent) {
+    lock_frame = copy_frame(frame);
+    if (lock_frame == NULL)
+        goto done;
 
-                                dht_inode_ctx_time_update (local->loc.parent,
-                                                           this,
-                                                           &local->preparent,
-                                                           0);
+    lock_local = dht_local_init(lock_frame, &local->loc, NULL,
+                                lock_frame->root->op);
+    if (lock_local == NULL)
+        goto done;
 
-                                dht_inode_ctx_time_update (local->loc.parent,
-                                                           this,
-                                                           &local->postparent,
-                                                           1);
+    lock_local->lock[0].ns.parent_layout.locks = local->lock[0]
+                                                     .ns.parent_layout.locks;
+    lock_local->lock[0]
+        .ns.parent_layout.lk_count = local->lock[0].ns.parent_layout.lk_count;
 
-                        }
+    local->lock[0].ns.parent_layout.locks = NULL;
+    local->lock[0].ns.parent_layout.lk_count = 0;
+    dht_unlock_inodelk(lock_frame, lock_local->lock[0].ns.parent_layout.locks,
+                       lock_local->lock[0].ns.parent_layout.lk_count,
+                       dht_rmdir_unlock_cbk);
+    lock_frame = NULL;
 
-                        DHT_STACK_UNWIND (rmdir, frame, local->op_ret,
-                                          local->op_errno, &local->preparent,
-                                          &local->postparent, NULL);
-                }
-        }
+done:
+    if (lock_frame != NULL) {
+        DHT_STACK_DESTROY(lock_frame);
+    }
 
-        return 0;
+    return 0;
 }
 
-
-int
-dht_rmdir_do (call_frame_t *frame, xlator_t *this)
+static int
+dht_rmdir_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int op_ret,
+              int op_errno, struct iatt *preparent, struct iatt *postparent,
+              dict_t *xdata)
 {
-        dht_local_t  *local = NULL;
-        dht_conf_t   *conf = NULL;
-        int           i = 0;
-        xlator_t     *hashed_subvol = NULL;
-        char gfid[GF_UUID_BUF_SIZE] ={0};
+    dht_local_t *local = NULL;
+    int this_call_cnt = 0;
+    xlator_t *prev = NULL;
+    int done = 0;
+    char gfid[GF_UUID_BUF_SIZE] = {0};
+    dht_local_t *heal_local = NULL;
+    call_frame_t *heal_frame = NULL;
+    int ret = -1;
+
+    local = frame->local;
+    prev = cookie;
+
+    LOCK(&frame->lock);
+    {
+        if (op_ret == -1) {
+            if ((op_errno != ENOENT) && (op_errno != ESTALE)) {
+                local->op_errno = op_errno;
+                local->op_ret = -1;
 
-        VALIDATE_OR_GOTO (this->private, err);
+                if (op_errno != EACCES)
+                    local->need_selfheal = 1;
+            }
 
-        conf = this->private;
-        local = frame->local;
+            gf_uuid_unparse(local->loc.gfid, gfid);
 
-        if (local->op_ret == -1)
-                goto err;
+            gf_msg_debug(this->name, op_errno,
+                         "rmdir on %s for %s failed."
+                         "(gfid = %s)",
+                         prev->name, local->loc.path, gfid);
+            goto unlock;
+        }
 
-        local->call_cnt = conf->subvolume_cnt;
+        /* Track if rmdir succeeded on at least one subvol*/
+        local->fop_succeeded = 1;
+        dht_iatt_merge(this, &local->preparent, preparent);
+        dht_iatt_merge(this, &local->postparent, postparent);
+    }
+unlock:
+    UNLOCK(&frame->lock);
 
+    this_call_cnt = dht_frame_return(frame);
 
-        /* first remove from non-hashed_subvol */
-        hashed_subvol = dht_subvol_get_hashed (this, &local->loc);
+    /* if local->hashed_subvol, we are yet to wind to hashed_subvol. */
+    if (local->hashed_subvol && (this_call_cnt == 1)) {
+        done = 1;
+    } else if (!local->hashed_subvol && !this_call_cnt) {
+        done = 1;
+    }
 
-        if (!hashed_subvol) {
-                gf_uuid_unparse(local->loc.gfid, gfid);
+    if (done) {
+        if (local->need_selfheal && local->fop_succeeded) {
+            dht_rmdir_unlock(frame, this);
+            local->layout = dht_layout_get(this, local->loc.inode);
 
-                gf_msg (this->name, GF_LOG_WARNING, 0,
-                        DHT_MSG_HASHED_SUBVOL_GET_FAILED,
-                        "Failed to get hashed subvol for %s (gfid = %s)",
-                        local->loc.path, gfid);
-        } else {
-                local->hashed_subvol = hashed_subvol;
-        }
+            /* TODO: neater interface needed below */
+            local->stbuf.ia_type = local->loc.inode->ia_type;
+
+            gf_uuid_copy(local->gfid, local->loc.inode->gfid);
+            heal_frame = copy_frame(frame);
+            if (heal_frame == NULL) {
+                goto err;
+            }
+
+            heal_local = dht_local_init(heal_frame, &local->loc, NULL, 0);
+            if (!heal_local) {
+                DHT_STACK_DESTROY(heal_frame);
+                goto err;
+            }
+
+            heal_local->inode = inode_ref(local->loc.inode);
+            heal_local->main_frame = frame;
+            gf_uuid_copy(heal_local->gfid, local->loc.inode->gfid);
+            ret = dht_selfheal_restore(heal_frame, dht_rmdir_selfheal_cbk,
+                                       &heal_local->loc, heal_local->layout);
+            if (ret) {
+                DHT_STACK_DESTROY(heal_frame);
+                goto err;
+            }
+
+        } else if (this_call_cnt) {
+            /* If non-hashed subvol's have responded, proceed */
+            if (local->op_ret == 0) {
+                /* Delete the dir from the hashed subvol if:
+                 * The fop succeeded on at least one subvol
+                 *  and did not fail on any
+                 *  or
+                 *  The fop failed with ENOENT/ESTALE on
+                 *  all subvols */
+
+                STACK_WIND_COOKIE(frame, dht_rmdir_hashed_subvol_cbk,
+                                  local->hashed_subvol, local->hashed_subvol,
+                                  local->hashed_subvol->fops->rmdir,
+                                  &local->loc, local->flags, NULL);
+            } else {
+                /* hashed-subvol was non-NULL and rmdir failed on
+                 * all non hashed-subvols. Unwind rmdir with
+                 * local->op_ret and local->op_errno. */
+                dht_rmdir_unlock(frame, this);
+                DHT_STACK_UNWIND(rmdir, frame, local->op_ret, local->op_errno,
+                                 &local->preparent, &local->postparent, NULL);
 
-        /* When DHT has only 1 child */
-        if (conf->subvolume_cnt == 1) {
-                STACK_WIND (frame, dht_rmdir_hashed_subvol_cbk,
-                            conf->subvolumes[0],
-                            conf->subvolumes[0]->fops->rmdir,
-                            &local->loc, local->flags, NULL);
                 return 0;
-        }
+            }
+        } else if (!this_call_cnt) {
+            /* All subvol's have responded, proceed */
 
-        for (i = 0; i < conf->subvolume_cnt; i++) {
-                if (hashed_subvol &&
-                    (hashed_subvol == conf->subvolumes[i]))
-                        continue;
+            if (local->loc.parent) {
+                dht_inode_ctx_time_update(local->loc.parent, this,
+                                          &local->preparent, 0);
 
-                STACK_WIND (frame, dht_rmdir_cbk,
-                            conf->subvolumes[i],
-                            conf->subvolumes[i]->fops->rmdir,
-                            &local->loc, local->flags, NULL);
+                dht_inode_ctx_time_update(local->loc.parent, this,
+                                          &local->postparent, 1);
+            }
+
+            dht_set_fixed_dir_stat(&local->preparent);
+            dht_set_fixed_dir_stat(&local->postparent);
+
+            dht_rmdir_unlock(frame, this);
+            DHT_STACK_UNWIND(rmdir, frame, local->op_ret, local->op_errno,
+                             &local->preparent, &local->postparent, NULL);
         }
+    }
 
-        return 0;
+    return 0;
 
 err:
-        DHT_STACK_UNWIND (rmdir, frame, local->op_ret, local->op_errno,
-                          &local->preparent, &local->postparent, NULL);
-        return 0;
+    DHT_STACK_UNWIND(rmdir, frame, -1, local->op_errno, NULL, NULL, NULL);
+    return 0;
 }
 
-
-int
-dht_rmdir_linkfile_unlink_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                               int op_ret, int op_errno, struct iatt *preparent,
-                               struct iatt *postparent, dict_t *xdata)
+static int
+dht_rmdir_lock_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                   int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
-        dht_local_t    *local = NULL;
-        call_frame_t   *prev = NULL;
-        xlator_t       *src = NULL;
-        call_frame_t   *main_frame = NULL;
-        dht_local_t    *main_local = NULL;
-        int             this_call_cnt = 0;
-        char gfid[GF_UUID_BUF_SIZE] ={0};
+    dht_local_t *local = NULL;
+    dht_conf_t *conf = NULL;
+    int i = 0;
+    xlator_t *hashed_subvol;
 
+    conf = this->private;
+    local = frame->local;
 
-        local  = frame->local;
-        prev   = cookie;
-        src    = prev->this;
+    if (op_ret < 0) {
+        gf_msg(this->name, GF_LOG_WARNING, op_errno, DHT_MSG_INODE_LK_ERROR,
+               "acquiring entrylk after inodelk failed rmdir for %s)",
+               local->loc.path);
 
-        main_frame = local->main_frame;
-        main_local = main_frame->local;
+        local->op_ret = -1;
+        local->op_errno = op_errno;
+        goto err;
+    }
 
-        gf_uuid_unparse(local->loc.gfid, gfid);
+    hashed_subvol = local->hashed_subvol;
+    for (i = 0; i < conf->subvolume_cnt; i++) {
+        if (hashed_subvol && (hashed_subvol == conf->subvolumes[i]))
+            continue;
 
-        if (op_ret == 0) {
-                gf_msg_trace (this->name, 0,
-                              "Unlinked linkfile %s on %s, gfid = %s",
-                              local->loc.path, src->name, gfid);
-        } else {
-                main_local->op_ret   = -1;
-                main_local->op_errno = op_errno;
-                gf_msg_debug (this->name, 0,
-                              "Unlink of %s on %s failed. (gfid = %s) (%s)",
-                              local->loc.path, src->name, gfid,
-                              strerror (op_errno));
-        }
+        STACK_WIND_COOKIE(frame, dht_rmdir_cbk, conf->subvolumes[i],
+                          conf->subvolumes[i], conf->subvolumes[i]->fops->rmdir,
+                          &local->loc, local->flags, NULL);
+    }
 
-        this_call_cnt = dht_frame_return (main_frame);
-        if (is_last_call (this_call_cnt))
-                dht_rmdir_do (main_frame, this);
+    return 0;
 
-        DHT_STACK_DESTROY (frame);
-        return 0;
-}
+err:
+    DHT_STACK_UNWIND(rmdir, frame, local->op_ret, local->op_errno,
+                     &local->preparent, &local->postparent, NULL);
 
+    return 0;
+}
 
-int
-dht_rmdir_lookup_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                      int op_ret, int op_errno, inode_t *inode,
-                      struct iatt *stbuf, dict_t *xattr, struct iatt *parent)
+static int
+dht_rmdir_do(call_frame_t *frame, xlator_t *this)
 {
-        dht_local_t    *local = NULL;
-        call_frame_t   *prev = NULL;
-        xlator_t       *src = NULL;
-        call_frame_t   *main_frame = NULL;
-        dht_local_t    *main_local = NULL;
-        int             this_call_cnt = 0;
-        dht_conf_t     *conf = this->private;
-        char               gfid[GF_UUID_BUF_SIZE] = {0};
-
-        local = frame->local;
-        prev  = cookie;
-        src   = prev->this;
+    dht_local_t *local = NULL;
+    dht_conf_t *conf = NULL;
+    int ret = -1;
+    xlator_t *hashed_subvol = NULL;
+    char gfid[GF_UUID_BUF_SIZE] = {0};
 
-        main_frame = local->main_frame;
-        main_local = main_frame->local;
+    VALIDATE_OR_GOTO(frame->local, err);
+    local = frame->local;
+    VALIDATE_OR_GOTO(this->private, out);
+    conf = this->private;
 
-        if (op_ret != 0)
-                goto err;
+    if (local->op_ret == -1)
+        goto out;
 
-        if (!check_is_linkfile (inode, stbuf, xattr, conf->link_xattr_name)) {
-                main_local->op_ret  = -1;
-                main_local->op_errno = ENOTEMPTY;
+    local->call_cnt = conf->subvolume_cnt;
 
-                 gf_uuid_unparse(local->loc.gfid, gfid);
+    /* first remove from non-hashed_subvol */
+    hashed_subvol = dht_subvol_get_hashed(this, &local->loc);
 
-                gf_log (this->name, GF_LOG_WARNING,
-                        "%s on %s is not a linkfile (type=0%o, gfid = %s)",
-                        local->loc.path, src->name, stbuf->ia_type, gfid);
-                goto err;
-        }
+    if (!hashed_subvol) {
+        gf_uuid_unparse(local->loc.gfid, gfid);
 
-        STACK_WIND (frame, dht_rmdir_linkfile_unlink_cbk,
-                    src, src->fops->unlink, &local->loc, 0, NULL);
+        gf_msg(this->name, GF_LOG_WARNING, 0, DHT_MSG_HASHED_SUBVOL_GET_FAILED,
+               "Failed to get hashed subvol for %s (gfid = %s)",
+               local->loc.path, gfid);
+    } else {
+        local->hashed_subvol = hashed_subvol;
+    }
+
+    /* When DHT has only 1 child */
+    if (conf->subvolume_cnt == 1) {
+        STACK_WIND_COOKIE(frame, dht_rmdir_hashed_subvol_cbk,
+                          conf->subvolumes[0], conf->subvolumes[0],
+                          conf->subvolumes[0]->fops->rmdir, &local->loc,
+                          local->flags, NULL);
         return 0;
+    }
+
+    local->current = &local->lock[0];
+    ret = dht_protect_namespace(frame, &local->loc, local->hashed_subvol,
+                                &local->current->ns, dht_rmdir_lock_cbk);
+    if (ret < 0) {
+        local->op_ret = -1;
+        local->op_errno = errno ? errno : EINVAL;
+        goto out;
+    }
+
+    return 0;
+
+out:
+    dht_set_fixed_dir_stat(&local->preparent);
+    dht_set_fixed_dir_stat(&local->postparent);
+
+    DHT_STACK_UNWIND(rmdir, frame, local->op_ret, local->op_errno,
+                     &local->preparent, &local->postparent, NULL);
+    return 0;
 err:
+    DHT_STACK_UNWIND(rmdir, frame, -1, EINVAL, NULL, NULL, NULL);
+    return 0;
+}
+
+static void
+dht_rmdir_readdirp_done(call_frame_t *readdirp_frame, xlator_t *this)
+{
+    call_frame_t *main_frame = NULL;
+    dht_local_t *main_local = NULL;
+    dht_local_t *local = NULL;
+    int this_call_cnt = 0;
+
+    local = readdirp_frame->local;
+    main_frame = local->main_frame;
+    main_local = main_frame->local;
+
+    /* At least one readdirp failed.
+     * This is a bit hit or miss - if readdirp failed on more than
+     * one subvol, we don't know which error is returned.
+     */
+    if (local->op_ret == -1) {
+        main_local->op_ret = local->op_ret;
+        main_local->op_errno = local->op_errno;
+    }
+
+    this_call_cnt = dht_frame_return(main_frame);
+
+    if (is_last_call(this_call_cnt))
+        dht_rmdir_do(main_frame, this);
+
+    DHT_STACK_DESTROY(readdirp_frame);
+}
+
+/* Keep sending readdirp on the subvol until it returns no more entries
+ * It is possible that not all entries will fit in a single readdirp in
+ * which case the rmdir will keep failing with ENOTEMPTY
+ */
 
-        this_call_cnt = dht_frame_return (main_frame);
-        if (is_last_call (this_call_cnt))
-                dht_rmdir_do (main_frame, this);
+static int
+dht_rmdir_readdirp_do(call_frame_t *readdirp_frame, xlator_t *this)
+{
+    dht_local_t *local = NULL;
 
-        DHT_STACK_DESTROY (frame);
+    local = readdirp_frame->local;
+
+    if (local->op_ret == -1) {
+        /* there is no point doing another readdirp on this
+         * subvol . */
+        dht_rmdir_readdirp_done(readdirp_frame, this);
         return 0;
-}
+    }
 
+    STACK_WIND_COOKIE(readdirp_frame, dht_rmdir_readdirp_cbk,
+                      local->hashed_subvol, local->hashed_subvol,
+                      local->hashed_subvol->fops->readdirp, local->fd, 4096, 0,
+                      local->xattr);
 
-int
-dht_rmdir_cached_lookup_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                             int op_ret, int op_errno, inode_t *inode,
-                             struct iatt *stbuf, dict_t *xattr,
-                             struct iatt *parent)
-{
-        dht_local_t    *local         = NULL;
-        xlator_t       *src           = NULL;
-        call_frame_t   *main_frame    = NULL;
-        dht_local_t    *main_local    = NULL;
-        int             this_call_cnt = 0;
-        dht_conf_t     *conf          = this->private;
-        dict_t         *xattrs        = NULL;
-        int             ret           = 0;
+    return 0;
+}
 
-        local = frame->local;
-        src   = local->hashed_subvol;
+static int
+dht_rmdir_linkfile_unlink_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                              int op_ret, int op_errno, struct iatt *preparent,
+                              struct iatt *postparent, dict_t *xdata)
+{
+    dht_local_t *local = NULL;
+    xlator_t *prev = NULL;
+    xlator_t *src = NULL;
+    call_frame_t *readdirp_frame = NULL;
+    dht_local_t *readdirp_local = NULL;
+    int this_call_cnt = 0;
+    char gfid[GF_UUID_BUF_SIZE] = {0};
+
+    local = frame->local;
+    prev = cookie;
+    src = prev;
+
+    readdirp_frame = local->main_frame;
+    readdirp_local = readdirp_frame->local;
+
+    gf_uuid_unparse(local->loc.gfid, gfid);
+
+    if (op_ret == 0) {
+        gf_msg_trace(this->name, 0, "Unlinked linkfile %s on %s, gfid = %s",
+                     local->loc.path, src->name, gfid);
+    } else {
+        if (op_errno != ENOENT) {
+            readdirp_local->op_ret = -1;
+            readdirp_local->op_errno = op_errno;
+        }
+        gf_msg_debug(this->name, op_errno,
+                     "Unlink of %s on %s failed. (gfid = %s)", local->loc.path,
+                     src->name, gfid);
+    }
+
+    this_call_cnt = dht_frame_return(readdirp_frame);
+
+    if (is_last_call(this_call_cnt))
+        dht_rmdir_readdirp_do(readdirp_frame, this);
+
+    DHT_STACK_DESTROY(frame);
+    return 0;
+}
 
-        main_frame = local->main_frame;
-        main_local = main_frame->local;
+static int
+dht_rmdir_lookup_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                     int op_ret, int op_errno, inode_t *inode,
+                     struct iatt *stbuf, dict_t *xattr, struct iatt *parent)
+{
+    dht_local_t *local = NULL;
+    xlator_t *prev = NULL;
+    xlator_t *src = NULL;
+    call_frame_t *readdirp_frame = NULL;
+    dht_local_t *readdirp_local = NULL;
+    int this_call_cnt = 0;
+    dht_conf_t *conf = this->private;
+    char gfid[GF_UUID_BUF_SIZE] = {0};
+
+    local = frame->local;
+    prev = cookie;
+    src = prev;
+
+    gf_msg_debug(this->name, 0, "dht_rmdir_lookup_cbk %s", local->loc.path);
+
+    readdirp_frame = local->main_frame;
+    readdirp_local = readdirp_frame->local;
+
+    if (op_ret != 0) {
+        gf_msg(this->name, GF_LOG_WARNING, op_errno, DHT_MSG_FILE_LOOKUP_FAILED,
+               "lookup failed for %s on %s", local->loc.path, src->name);
+        goto err;
+    }
+
+    if (!check_is_linkfile(inode, stbuf, xattr, conf->link_xattr_name)) {
+        readdirp_local->op_ret = -1;
+        readdirp_local->op_errno = ENOTEMPTY;
 
-        if (op_ret == 0) {
-                main_local->op_ret  = -1;
-                main_local->op_errno = ENOTEMPTY;
+        gf_uuid_unparse(local->loc.gfid, gfid);
 
-                gf_log (this->name, GF_LOG_WARNING,
-                        "%s found on cached subvol %s",
-                        local->loc.path, src->name);
-                goto err;
-        } else if (op_errno != ENOENT) {
-                main_local->op_ret  = -1;
-                main_local->op_errno = op_errno;
-                goto err;
-        }
+        gf_msg(this->name, GF_LOG_WARNING, 0, DHT_MSG_NOT_LINK_FILE_ERROR,
+               "%s on %s is not a linkfile (type=0%o, gfid = %s)",
+               local->loc.path, src->name, stbuf->ia_type, gfid);
+        goto err;
+    }
 
-        xattrs = dict_new ();
-        if (!xattrs) {
-                gf_log (this->name, GF_LOG_ERROR, "dict_new failed");
-                goto err;
-        }
+    STACK_WIND_COOKIE(frame, dht_rmdir_linkfile_unlink_cbk, src, src,
+                      src->fops->unlink, &local->loc, 0, NULL);
+    return 0;
+err:
 
-        ret = dict_set_uint32 (xattrs, conf->link_xattr_name, 256);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        DHT_MSG_DICT_SET_FAILED,
-                        "Failed to set dictionary value: key = %s",
-                        conf->link_xattr_name);
-                if (xattrs)
-                        dict_unref (xattrs);
-                goto err;
-        }
+    this_call_cnt = dht_frame_return(readdirp_frame);
+    if (is_last_call(this_call_cnt)) {
+        dht_rmdir_readdirp_do(readdirp_frame, this);
+    }
 
-        STACK_WIND (frame, dht_rmdir_lookup_cbk,
-                    src, src->fops->lookup, &local->loc, xattrs);
-        if (xattrs)
-                dict_unref (xattrs);
+    DHT_STACK_DESTROY(frame);
+    return 0;
+}
 
-        return 0;
+static int
+dht_rmdir_cached_lookup_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                            int op_ret, int op_errno, inode_t *inode,
+                            struct iatt *stbuf, dict_t *xattr,
+                            struct iatt *parent)
+{
+    dht_local_t *local = NULL;
+    xlator_t *src = NULL;
+    call_frame_t *readdirp_frame = NULL;
+    dht_local_t *readdirp_local = NULL;
+    int this_call_cnt = 0;
+    dht_conf_t *conf = this->private;
+    dict_t *xattrs = NULL;
+    int ret = 0;
+
+    local = frame->local;
+    src = local->hashed_subvol;
+
+    /* main_frame here is the readdirp_frame */
+
+    readdirp_frame = local->main_frame;
+    readdirp_local = readdirp_frame->local;
+
+    gf_msg_debug(this->name, 0, "returning for %s ", local->loc.path);
+
+    if (op_ret == 0) {
+        readdirp_local->op_ret = -1;
+        readdirp_local->op_errno = ENOTEMPTY;
+
+        gf_msg(this->name, GF_LOG_WARNING, 0, DHT_MSG_SUBVOL_ERROR,
+               "%s found on cached subvol %s", local->loc.path, src->name);
+        goto err;
+    } else if (op_errno != ENOENT) {
+        readdirp_local->op_ret = -1;
+        readdirp_local->op_errno = op_errno;
+
+        gf_msg(this->name, GF_LOG_WARNING, op_errno, DHT_MSG_SUBVOL_ERROR,
+               "%s not found on cached subvol %s", local->loc.path, src->name);
+        goto err;
+    }
+
+    xattrs = dict_new();
+    if (!xattrs) {
+        gf_msg(this->name, GF_LOG_ERROR, ENOMEM, DHT_MSG_NO_MEMORY,
+               "dict_new failed");
+        goto err;
+    }
+
+    ret = dict_set_uint32(xattrs, conf->link_xattr_name, 256);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, DHT_MSG_DICT_SET_FAILED,
+               "Failed to set dictionary value: key = %s",
+               conf->link_xattr_name);
+        if (xattrs)
+            dict_unref(xattrs);
+        goto err;
+    }
+    STACK_WIND_COOKIE(frame, dht_rmdir_lookup_cbk, src, src, src->fops->lookup,
+                      &local->loc, xattrs);
+    if (xattrs)
+        dict_unref(xattrs);
+
+    return 0;
 err:
 
-        this_call_cnt = dht_frame_return (main_frame);
-        if (is_last_call (this_call_cnt))
-                dht_rmdir_do (main_frame, this);
+    this_call_cnt = dht_frame_return(readdirp_frame);
 
-        DHT_STACK_DESTROY (frame);
-        return 0;
+    /* Once all the lookups/unlinks etc have returned, proceed to wind
+     * readdirp on the subvol again until no entries are returned.
+     * This is required if there are more entries than can be returned
+     * in a single readdirp call.
+     */
+
+    if (is_last_call(this_call_cnt))
+        dht_rmdir_readdirp_do(readdirp_frame, this);
+
+    DHT_STACK_DESTROY(frame);
+    return 0;
 }
 
+static int
+dht_rmdir_is_subvol_empty(call_frame_t *frame, xlator_t *this,
+                          gf_dirent_t *entries, xlator_t *src)
+{
+    int ret = 0;
+    int build_ret = 0;
+    gf_dirent_t *trav = NULL;
+    call_frame_t *lookup_frame = NULL;
+    dht_local_t *lookup_local = NULL;
+    dht_local_t *local = NULL;
+    dict_t *xattrs = NULL;
+    dht_conf_t *conf = this->private;
+    xlator_t *subvol = NULL;
+    char gfid[GF_UUID_BUF_SIZE] = {0};
+    int count = 0;
+    gf_boolean_t unwind = _gf_false;
+
+    local = frame->local;
+
+    list_for_each_entry(trav, &entries->list, list)
+    {
+        if (strcmp(trav->d_name, ".") == 0)
+            continue;
+        if (strcmp(trav->d_name, "..") == 0)
+            continue;
+        if (check_is_linkfile(NULL, (&trav->d_stat), trav->dict,
+                              conf->link_xattr_name)) {
+            count++;
+            continue;
+        }
+
+        /* this entry is either a directory which is neither "." nor "..",
+           or a non directory which is not a linkfile. the directory is to
+           be treated as non-empty
+        */
+        return 0;
+    }
 
-int
-dht_rmdir_is_subvol_empty (call_frame_t *frame, xlator_t *this,
-                           gf_dirent_t *entries, xlator_t *src)
-{
-        int                 ret          = 0;
-        int                 build_ret    = 0;
-        gf_dirent_t        *trav         = NULL;
-        call_frame_t       *lookup_frame = NULL;
-        dht_local_t        *lookup_local = NULL;
-        dht_local_t        *local        = NULL;
-        dict_t             *xattrs       = NULL;
-        dht_conf_t         *conf         = this->private;
-        xlator_t           *subvol       = NULL;
-        char               gfid[GF_UUID_BUF_SIZE] = {0};
+    xattrs = dict_new();
+    if (!xattrs) {
+        gf_msg(this->name, GF_LOG_ERROR, ENOMEM, DHT_MSG_NO_MEMORY,
+               "dict_new failed");
+        return -1;
+    }
 
-        local = frame->local;
+    ret = dict_set_uint32(xattrs, conf->link_xattr_name, 256);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, DHT_MSG_DICT_SET_FAILED,
+               "Failed to set dictionary value: key = %s",
+               conf->link_xattr_name);
 
-        list_for_each_entry (trav, &entries->list, list) {
-                if (strcmp (trav->d_name, ".") == 0)
-                        continue;
-                if (strcmp (trav->d_name, "..") == 0)
-                        continue;
-                if (check_is_linkfile (NULL, (&trav->d_stat), trav->dict,
-                                              conf->link_xattr_name)) {
-                        ret++;
-                        continue;
-                }
+        if (xattrs)
+            dict_unref(xattrs);
+        return -1;
+    }
 
-                /* this entry is either a directory which is neither "." nor "..",
-                   or a non directory which is not a linkfile. the directory is to
-                   be treated as non-empty
-                */
-                return 0;
+    local->call_cnt = count;
+    ret = 0;
+
+    list_for_each_entry(trav, &entries->list, list)
+    {
+        if (strcmp(trav->d_name, ".") == 0)
+            continue;
+        if (strcmp(trav->d_name, "..") == 0)
+            continue;
+
+        lookup_frame = copy_frame(frame);
+
+        if (!lookup_frame) {
+            /* out of memory, let the rmdir fail
+               (as non-empty, unfortunately) */
+            goto err;
         }
 
-        xattrs = dict_new ();
-        if (!xattrs) {
-                gf_log (this->name, GF_LOG_ERROR, "dict_new failed");
-                return -1;
+        lookup_local = dht_local_init(lookup_frame, NULL, NULL, GF_FOP_LOOKUP);
+        if (!lookup_local) {
+            goto err;
         }
 
-        ret = dict_set_uint32 (xattrs, conf->link_xattr_name, 256);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        DHT_MSG_DICT_SET_FAILED,
-                        "Failed to set dictionary value: key = %s",
-                        conf->link_xattr_name);
-
-                if (xattrs)
-                        dict_unref (xattrs);
-                return -1;
-        }
-
-        list_for_each_entry (trav, &entries->list, list) {
-                if (strcmp (trav->d_name, ".") == 0)
-                        continue;
-                if (strcmp (trav->d_name, "..") == 0)
-                        continue;
-
-                lookup_frame = NULL;
-                lookup_local = NULL;
-
-                lookup_frame = copy_frame (frame);
-                if (!lookup_frame) {
-                        /* out of memory, let the rmdir fail
-                           (as non-empty, unfortunately) */
-                        goto err;
-                }
+        lookup_frame->local = lookup_local;
+        lookup_local->main_frame = frame;
+        lookup_local->hashed_subvol = src;
 
-                lookup_local = mem_get0 (this->local_pool);
-                if (!lookup_local) {
-                        goto err;
-                }
+        build_ret = dht_build_child_loc(this, &lookup_local->loc, &local->loc,
+                                        trav->d_name);
+        if (build_ret != 0)
+            goto err;
 
-                lookup_frame->local = lookup_local;
-                lookup_local->main_frame = frame;
-                lookup_local->hashed_subvol = src;
+        gf_uuid_copy(lookup_local->loc.gfid, trav->d_stat.ia_gfid);
 
-                build_ret = dht_build_child_loc (this, &lookup_local->loc,
-                                                 &local->loc, trav->d_name);
-                if (build_ret != 0)
-                        goto err;
+        gf_uuid_unparse(lookup_local->loc.gfid, gfid);
 
-                gf_uuid_copy (lookup_local->loc.gfid, trav->d_stat.ia_gfid);
+        gf_msg_trace(this->name, 0, "looking up %s on subvolume %s, gfid = %s",
+                     lookup_local->loc.path, src->name, gfid);
 
-                gf_uuid_unparse(lookup_local->loc.gfid, gfid);
+        subvol = dht_linkfile_subvol(this, NULL, &trav->d_stat, trav->dict);
+        if (!subvol || (subvol == src)) {
+            /* we need to delete the linkto file if it does not have a
+             * valid subvol or it points to itself.
+             */
+            gf_msg(this->name, GF_LOG_INFO, 0, DHT_MSG_INVALID_LINKFILE,
+                   "Linkfile does not have link subvolume. "
+                   "path = %s, gfid = %s",
+                   lookup_local->loc.path, gfid);
 
-                gf_msg_trace (this->name, 0,
-                              "looking up %s on subvolume %s, gfid = %s",
-                              lookup_local->loc.path, src->name, gfid);
+            gf_msg_debug(this->name, 0, "looking up %s on subvol %s, gfid = %s",
+                         lookup_local->loc.path, src->name, gfid);
 
-                LOCK (&frame->lock);
-                {
-                        local->call_cnt++;
-                }
-                UNLOCK (&frame->lock);
-
-                subvol = dht_linkfile_subvol (this, NULL, &trav->d_stat,
-                                              trav->dict);
-                if (!subvol) {
-                        gf_log (this->name, GF_LOG_INFO,
-                                "Linkfile does not have link subvolume. "
-                                "path = %s, gfid = %s",
-                                lookup_local->loc.path, gfid);
-                        STACK_WIND (lookup_frame, dht_rmdir_lookup_cbk,
-                                    src, src->fops->lookup,
-                                    &lookup_local->loc, xattrs);
-                } else {
-                        STACK_WIND (lookup_frame, dht_rmdir_cached_lookup_cbk,
-                                    subvol, subvol->fops->lookup,
-                                    &lookup_local->loc, xattrs);
-                }
-                ret++;
+            STACK_WIND_COOKIE(lookup_frame, dht_rmdir_lookup_cbk, src, src,
+                              src->fops->lookup, &lookup_local->loc, xattrs);
+        } else {
+            gf_msg_debug(this->name, 0,
+                         "Looking up linkfile target %s on "
+                         " subvol %s, gfid = %s",
+                         lookup_local->loc.path, subvol->name, gfid);
+
+            STACK_WIND(lookup_frame, dht_rmdir_cached_lookup_cbk, subvol,
+                       subvol->fops->lookup, &lookup_local->loc, xattrs);
         }
+        ret++;
 
-        if (xattrs)
-                dict_unref (xattrs);
+        lookup_frame = NULL;
+        lookup_local = NULL;
+    }
 
-        return ret;
+    if (xattrs)
+        dict_unref(xattrs);
+
+    return ret;
 err:
-        if (xattrs)
-                dict_unref (xattrs);
+    if (xattrs)
+        dict_unref(xattrs);
 
-        DHT_STACK_DESTROY (lookup_frame);
-        return 0;
+    if (lookup_frame)
+        DHT_STACK_DESTROY(lookup_frame);
+
+    /* Handle the case where the wound calls have unwound before the
+     * loop processing is done
+     */
+
+    LOCK(&frame->lock);
+    {
+        local->op_ret = -1;
+        local->op_errno = ENOTEMPTY;
+
+        local->call_cnt -= (count - ret);
+        if (!local->call_cnt)
+            unwind = _gf_true;
+    }
+    UNLOCK(&frame->lock);
+
+    if (!unwind) {
+        return ret;
+    }
+    return 0;
 }
 
+/*
+ * No more entries on this subvol. Proceed to the actual rmdir operation.
+ */
 
-int
-dht_rmdir_readdirp_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                        int op_ret, int op_errno, gf_dirent_t *entries,
-                        dict_t *xdata)
+static int
+dht_rmdir_readdirp_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                       int op_ret, int op_errno, gf_dirent_t *entries,
+                       dict_t *xdata)
 {
-        dht_local_t  *local = NULL;
-        int           this_call_cnt = -1;
-        call_frame_t *prev = NULL;
-        xlator_t     *src = NULL;
-        int           ret = 0;
+    dht_local_t *local = NULL;
+    xlator_t *prev = NULL;
+    xlator_t *src = NULL;
+    int ret = 0;
+    char *path = NULL;
+
+    local = frame->local;
+    prev = cookie;
+    src = prev;
+
+    if (op_ret > 2) {
+        /* dht_rmdir_is_subvol_empty() may free the frame,
+         * copy path for logging.
+         */
+        path = gf_strdup(local->loc.path);
 
-        local = frame->local;
-        prev  = cookie;
-        src   = prev->this;
-
-        if (op_ret > 2) {
-                ret = dht_rmdir_is_subvol_empty (frame, this, entries, src);
-
-                switch (ret) {
-                case 0: /* non linkfiles exist */
-                        gf_msg_trace (this->name, 0,
-                                      "readdir on %s for %s returned %d "
-                                      "entries", prev->this->name,
-                                      local->loc.path, op_ret);
-                        local->op_ret = -1;
-                        local->op_errno = ENOTEMPTY;
-                        break;
-                default:
-                        /* @ret number of linkfiles are getting unlinked */
-                        gf_msg_trace (this->name, 0,
-                                      "readdir on %s for %s found %d "
-                                      "linkfiles", prev->this->name,
-                                      local->loc.path, ret);
-                        break;
-                }
+        ret = dht_rmdir_is_subvol_empty(frame, this, entries, src);
+
+        switch (ret) {
+            case 0: /* non linkfiles exist */
+                gf_msg_trace(this->name, 0,
+                             "readdir on %s for %s returned %d "
+                             "entries",
+                             prev->name, local->loc.path, op_ret);
+                local->op_ret = -1;
+                local->op_errno = ENOTEMPTY;
+                break;
+            default:
+                /* @ret number of linkfiles are getting unlinked */
+                gf_msg_trace(this->name, 0,
+                             "readdir on %s for %s found %d "
+                             "linkfiles",
+                             prev->name, path, ret);
+                break;
         }
+    }
 
-        this_call_cnt = dht_frame_return (frame);
+    /* readdirp failed or no linkto files were found on this subvol */
+    if (!ret)
+        dht_rmdir_readdirp_done(frame, this);
+
+    GF_FREE(path);
+    return 0;
+}
+
+static int
+dht_rmdir_opendir_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                      int op_ret, int op_errno, fd_t *fd, dict_t *xdata)
+{
+    dht_local_t *local = NULL;
+    int this_call_cnt = -1;
+    xlator_t *prev = NULL;
+    int ret = 0;
+    dht_conf_t *conf = this->private;
+    dict_t *dict = NULL;
+    int i = 0;
+    char gfid[GF_UUID_BUF_SIZE] = {0};
+    dht_local_t *readdirp_local = NULL;
+    call_frame_t *readdirp_frame = NULL;
+    int cnt = 0;
+
+    local = frame->local;
+    prev = cookie;
+
+    this_call_cnt = dht_frame_return(frame);
+    if (op_ret == -1) {
+        gf_uuid_unparse(local->loc.gfid, gfid);
 
-        if (is_last_call (this_call_cnt)) {
-                dht_rmdir_do (frame, this);
+        gf_msg_debug(this->name, op_errno,
+                     "opendir on %s for %s failed, "
+                     "gfid = %s,",
+                     prev->name, local->loc.path, gfid);
+        if ((op_errno != ENOENT) && (op_errno != ESTALE)) {
+            local->op_ret = -1;
+            local->op_errno = op_errno;
         }
+        goto err;
+    }
 
+    if (!is_last_call(this_call_cnt))
         return 0;
-}
 
+    if (local->op_ret == -1)
+        goto err;
 
-int
-dht_rmdir_opendir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                       int op_ret, int op_errno, fd_t *fd, dict_t *xdata)
-{
-        dht_local_t  *local         = NULL;
-        int           this_call_cnt = -1;
-        call_frame_t *prev          = NULL;
-        dict_t       *dict          = NULL;
-        int           ret           = 0;
-        dht_conf_t   *conf          = this->private;
-        int           i             = 0;
-        char               gfid[GF_UUID_BUF_SIZE] = {0};
+    fd_bind(fd);
 
-        local = frame->local;
-        prev  = cookie;
+    dict = dict_new();
+    if (!dict) {
+        local->op_ret = -1;
+        local->op_errno = ENOMEM;
+        goto err;
+    }
 
+    ret = dict_set_uint32(dict, conf->link_xattr_name, 256);
+    if (ret)
+        gf_msg(this->name, GF_LOG_WARNING, 0, DHT_MSG_DICT_SET_FAILED,
+               "%s: Failed to set dictionary value:key = %s", local->loc.path,
+               conf->link_xattr_name);
 
-        this_call_cnt = dht_frame_return (frame);
-        if (op_ret == -1) {
-                gf_uuid_unparse(local->loc.gfid, gfid);
-
-                gf_msg_debug (this->name, 0,
-                              "opendir on %s for %s failed, "
-                              "gfid = %s, (%s)",
-                              prev->this->name, local->loc.path, gfid,
-                              strerror (op_errno));
-                if ((op_errno != ENOENT) && (op_errno != ESTALE)) {
-                        local->op_ret = -1;
-                        local->op_errno = op_errno;
-                }
-                goto err;
-        }
+    cnt = local->call_cnt = conf->subvolume_cnt;
 
-        if (!is_last_call (this_call_cnt))
-                return 0;
+    /* Create a separate frame per subvol as we might need
+     * to resend readdirp multiple times to get all the
+     * entries.
+     */
 
-        if (local->op_ret == -1)
-                goto err;
+    for (i = 0; i < conf->subvolume_cnt; i++) {
+        readdirp_frame = copy_frame(frame);
 
-        dict = dict_new ();
-        if (!dict) {
-                local->op_ret = -1;
-                local->op_errno = ENOMEM;
-                goto err;
+        if (!readdirp_frame) {
+            cnt--;
+            /* Reduce the local->call_cnt as well */
+            (void)dht_frame_return(frame);
+            continue;
         }
 
-        ret = dict_set_uint32 (dict, conf->link_xattr_name, 256);
-        if (ret)
-                gf_msg (this->name, GF_LOG_WARNING, 0,
-                        DHT_MSG_DICT_SET_FAILED,
-                        "%s: Failed to set dictionary value:key = %s",
-                        local->loc.path, conf->link_xattr_name);
+        readdirp_local = dht_local_init(readdirp_frame, &local->loc, local->fd,
+                                        0);
 
-        local->call_cnt = conf->subvolume_cnt;
-        for (i = 0; i < conf->subvolume_cnt; i++) {
-                STACK_WIND (frame, dht_rmdir_readdirp_cbk,
-                            conf->subvolumes[i],
-                            conf->subvolumes[i]->fops->readdirp,
-                            local->fd, 4096, 0, dict);
+        if (!readdirp_local) {
+            DHT_STACK_DESTROY(readdirp_frame);
+            cnt--;
+            /* Reduce the local->call_cnt as well */
+            dht_frame_return(frame);
+            continue;
         }
+        readdirp_local->main_frame = frame;
+        readdirp_local->op_ret = 0;
+        readdirp_local->xattr = dict_ref(dict);
+        /* overload this field to save the subvol info */
+        readdirp_local->hashed_subvol = conf->subvolumes[i];
 
-        if (dict)
-                dict_unref (dict);
+        STACK_WIND_COOKIE(readdirp_frame, dht_rmdir_readdirp_cbk,
+                          conf->subvolumes[i], conf->subvolumes[i],
+                          conf->subvolumes[i]->fops->readdirp,
+                          readdirp_local->fd, 4096, 0, readdirp_local->xattr);
+    }
 
-        return 0;
+    if (dict)
+        dict_unref(dict);
+
+    /* Could not wind readdirp to any subvol */
+
+    if (!cnt)
+        goto err;
+
+    return 0;
 
 err:
-        if (is_last_call (this_call_cnt)) {
-                dht_rmdir_do (frame, this);
-        }
+    if (is_last_call(this_call_cnt)) {
+        dht_rmdir_do(frame, this);
+    }
 
-        return 0;
+    return 0;
 }
 
-
 int
-dht_rmdir (call_frame_t *frame, xlator_t *this, loc_t *loc, int flags,
-           dict_t *xdata)
+dht_rmdir(call_frame_t *frame, xlator_t *this, loc_t *loc, int flags,
+          dict_t *xdata)
 {
-        dht_local_t  *local  = NULL;
-        dht_conf_t   *conf = NULL;
-        int           op_errno = -1;
-        int           i = -1;
+    dht_local_t *local = NULL;
+    dht_conf_t *conf = NULL;
+    int op_errno = -1;
+    int i = -1;
+    int ret = -1;
+    dict_t *xattr_req = NULL;
+
+    VALIDATE_OR_GOTO(frame, err);
+    VALIDATE_OR_GOTO(this, err);
+    VALIDATE_OR_GOTO(loc, err);
+    VALIDATE_OR_GOTO(loc->inode, err);
+    VALIDATE_OR_GOTO(loc->path, err);
+    VALIDATE_OR_GOTO(this->private, err);
+
+    conf = this->private;
+
+    local = dht_local_init(frame, loc, NULL, GF_FOP_RMDIR);
+    if (!local) {
+        op_errno = ENOMEM;
+        goto err;
+    }
+
+    local->call_cnt = conf->subvolume_cnt;
+    local->op_ret = 0;
+    local->fop_succeeded = 0;
+
+    local->flags = flags;
+
+    local->fd = fd_create(local->loc.inode, frame->root->pid);
+    if (!local->fd) {
+        op_errno = ENOMEM;
+        goto err;
+    }
+
+    if (flags) {
+        return dht_rmdir_do(frame, this);
+    }
+    if (xdata) {
+        xattr_req = dict_ref(xdata);
+    } else {
+        xattr_req = dict_new();
+    }
+    if (xattr_req) {
+        ret = dict_set_uint32(xattr_req, conf->link_xattr_name, 256);
+        /* If parallel-readdir is enabled, this is required
+         * to handle stale linkto files in the directory
+         * being deleted. If this fails, log an error but
+         * do not prevent the operation.
+         */
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, 0, "%s: failed to set key %s",
+                   loc->path, conf->link_xattr_name);
+        }
+    } else {
+        gf_msg(this->name, GF_LOG_ERROR, 0, 0, "%s: failed to set key %s",
+               loc->path, conf->link_xattr_name);
+    }
+
+    for (i = 0; i < conf->subvolume_cnt; i++) {
+        STACK_WIND_COOKIE(frame, dht_rmdir_opendir_cbk, conf->subvolumes[i],
+                          conf->subvolumes[i],
+                          conf->subvolumes[i]->fops->opendir, loc, local->fd,
+                          xattr_req);
+    }
+
+    if (xattr_req) {
+        dict_unref(xattr_req);
+    }
+    return 0;
 
-        VALIDATE_OR_GOTO (frame, err);
-        VALIDATE_OR_GOTO (this, err);
-        VALIDATE_OR_GOTO (loc, err);
-        VALIDATE_OR_GOTO (loc->inode, err);
-        VALIDATE_OR_GOTO (loc->path, err);
-        VALIDATE_OR_GOTO (this->private, err);
+err:
+    op_errno = (op_errno == -1) ? errno : op_errno;
+    DHT_STACK_UNWIND(rmdir, frame, -1, op_errno, NULL, NULL, NULL);
 
-        conf = this->private;
+    return 0;
+}
 
-        local = dht_local_init (frame, loc, NULL, GF_FOP_RMDIR);
-        if (!local) {
-                op_errno = ENOMEM;
-                goto err;
-        }
+static int
+dht_entrylk_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                int32_t op_ret, int32_t op_errno, dict_t *xdata)
 
-        local->call_cnt = conf->subvolume_cnt;
-        local->op_ret   = 0;
-        local->fop_succeeded = 0;
+{
+    DHT_STACK_UNWIND(entrylk, frame, op_ret, op_errno, xdata);
+    return 0;
+}
 
-        local->flags = flags;
+/* TODO
+ * Sending entrylk to cached subvol can result in stale lock
+ * as described in the bug 1311002.
+ */
+int
+dht_entrylk(call_frame_t *frame, xlator_t *this, const char *volume, loc_t *loc,
+            const char *basename, entrylk_cmd cmd, entrylk_type type,
+            dict_t *xdata)
+{
+    xlator_t *subvol = NULL;
+    int op_errno = -1;
+    dht_local_t *local = NULL;
+    char gfid[GF_UUID_BUF_SIZE] = {0};
+
+    VALIDATE_OR_GOTO(frame, err);
+    VALIDATE_OR_GOTO(this, err);
+    VALIDATE_OR_GOTO(loc, err);
+    VALIDATE_OR_GOTO(loc->inode, err);
+
+    local = dht_local_init(frame, loc, NULL, GF_FOP_ENTRYLK);
+    if (!local) {
+        op_errno = ENOMEM;
+        goto err;
+    }
+
+    subvol = local->cached_subvol;
+    if (!subvol) {
+        gf_uuid_unparse(loc->gfid, gfid);
 
-        local->fd = fd_create (local->loc.inode, frame->root->pid);
-        if (!local->fd) {
+        gf_msg_debug(this->name, 0,
+                     "no cached subvolume for path=%s, "
+                     "gfid = %s",
+                     loc->path, gfid);
+        op_errno = EINVAL;
+        goto err;
+    }
 
-                op_errno = ENOMEM;
-                goto err;
-        }
+    local->call_cnt = 1;
 
-        for (i = 0; i < conf->subvolume_cnt; i++) {
-                STACK_WIND (frame, dht_rmdir_opendir_cbk,
-                            conf->subvolumes[i],
-                            conf->subvolumes[i]->fops->opendir,
-                            loc, local->fd, NULL);
-        }
+    STACK_WIND(frame, dht_entrylk_cbk, subvol, subvol->fops->entrylk, volume,
+               loc, basename, cmd, type, xdata);
 
-        return 0;
+    return 0;
 
 err:
-        op_errno = (op_errno == -1) ? errno : op_errno;
-        DHT_STACK_UNWIND (rmdir, frame, -1, op_errno,
-                          NULL, NULL, NULL);
+    op_errno = (op_errno == -1) ? errno : op_errno;
+    DHT_STACK_UNWIND(entrylk, frame, -1, op_errno, NULL);
 
-        return 0;
+    return 0;
 }
 
-int
-dht_entrylk_cbk (call_frame_t *frame, void *cookie,
-                 xlator_t *this, int32_t op_ret, int32_t op_errno, dict_t *xdata)
+static int
+dht_fentrylk_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                 int32_t op_ret, int32_t op_errno, dict_t *xdata)
 
 {
-        DHT_STACK_UNWIND (entrylk, frame, op_ret, op_errno, xdata);
-        return 0;
+    DHT_STACK_UNWIND(fentrylk, frame, op_ret, op_errno, NULL);
+    return 0;
 }
 
-
 int
-dht_entrylk (call_frame_t *frame, xlator_t *this,
-             const char *volume, loc_t *loc, const char *basename,
-             entrylk_cmd cmd, entrylk_type type, dict_t *xdata)
-{
-        xlator_t     *subvol = NULL;
-        int           op_errno = -1;
-        dht_local_t  *local = NULL;
-        char          gfid[GF_UUID_BUF_SIZE] = {0};
-
-        VALIDATE_OR_GOTO (frame, err);
-        VALIDATE_OR_GOTO (this, err);
-        VALIDATE_OR_GOTO (loc, err);
-        VALIDATE_OR_GOTO (loc->inode, err);
-
-        local = dht_local_init (frame, loc, NULL, GF_FOP_ENTRYLK);
-        if (!local) {
-                op_errno = ENOMEM;
-                goto err;
-        }
-
+dht_fentrylk(call_frame_t *frame, xlator_t *this, const char *volume, fd_t *fd,
+             const char *basename, entrylk_cmd cmd, entrylk_type type,
+             dict_t *xdata)
+{
+    xlator_t *subvol = NULL;
+    int op_errno = -1;
+    char gfid[GF_UUID_BUF_SIZE] = {0};
 
-        subvol = local->cached_subvol;
-        if (!subvol) {
-                gf_uuid_unparse(loc->gfid, gfid);
+    VALIDATE_OR_GOTO(frame, err);
+    VALIDATE_OR_GOTO(this, err);
+    VALIDATE_OR_GOTO(fd, err);
+    VALIDATE_OR_GOTO(fd->inode, err);
 
-                gf_msg_debug (this->name, 0,
-                              "no cached subvolume for path=%s, "
-                              "gfid = %s", loc->path, gfid);
-                op_errno = EINVAL;
-                goto err;
-        }
+    gf_uuid_unparse(fd->inode->gfid, gfid);
 
-        local->call_cnt = 1;
+    subvol = dht_subvol_get_cached(this, fd->inode);
+    if (!subvol) {
+        gf_msg_debug(this->name, 0,
+                     "No cached subvolume for fd=%p,"
+                     " gfid = %s",
+                     fd, gfid);
+        op_errno = EINVAL;
+        goto err;
+    }
 
-        STACK_WIND (frame, dht_entrylk_cbk,
-                    subvol, subvol->fops->entrylk,
-                    volume, loc, basename, cmd, type, xdata);
+    STACK_WIND(frame, dht_fentrylk_cbk, subvol, subvol->fops->fentrylk, volume,
+               fd, basename, cmd, type, xdata);
 
-        return 0;
+    return 0;
 
 err:
-        op_errno = (op_errno == -1) ? errno : op_errno;
-        DHT_STACK_UNWIND (entrylk, frame, -1, op_errno, NULL);
+    op_errno = (op_errno == -1) ? errno : op_errno;
+    DHT_STACK_UNWIND(fentrylk, frame, -1, op_errno, NULL);
 
-        return 0;
+    return 0;
 }
 
+static int32_t
+dht_ipc_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int32_t op_ret,
+            int32_t op_errno, dict_t *xdata)
+{
+    dht_local_t *local = NULL;
+    int this_call_cnt = 0;
 
-int
-dht_fentrylk_cbk (call_frame_t *frame, void *cookie,
-                  xlator_t *this, int32_t op_ret, int32_t op_errno, dict_t *xdata)
+    GF_VALIDATE_OR_GOTO("dht", frame, out);
+    GF_VALIDATE_OR_GOTO("dht", this, out);
+    GF_VALIDATE_OR_GOTO("dht", frame->local, out);
 
-{
-        DHT_STACK_UNWIND (fentrylk, frame, op_ret, op_errno, NULL);
-        return 0;
-}
+    local = frame->local;
 
+    LOCK(&frame->lock);
+    {
+        if (op_ret < 0 && op_errno != ENOTCONN) {
+            local->op_errno = op_errno;
+            goto unlock;
+        }
+        local->op_ret = 0;
+    }
+unlock:
+    UNLOCK(&frame->lock);
 
-int
-dht_fentrylk (call_frame_t *frame, xlator_t *this,
-              const char *volume, fd_t *fd, const char *basename,
-              entrylk_cmd cmd, entrylk_type type, dict_t *xdata)
+    this_call_cnt = dht_frame_return(frame);
+    if (is_last_call(this_call_cnt)) {
+        DHT_STACK_UNWIND(ipc, frame, local->op_ret, local->op_errno, NULL);
+    }
+
+out:
+    return 0;
+}
+
+int32_t
+dht_ipc(call_frame_t *frame, xlator_t *this, int32_t op, dict_t *xdata)
 {
-        xlator_t     *subvol = NULL;
-        int           op_errno = -1;
-        char               gfid[GF_UUID_BUF_SIZE] = {0};
+    dht_local_t *local = NULL;
+    int op_errno = EINVAL;
+    dht_conf_t *conf = NULL;
+    int call_cnt = 0;
+    int i = 0;
 
+    VALIDATE_OR_GOTO(frame, err);
+    VALIDATE_OR_GOTO(this, err);
 
-        VALIDATE_OR_GOTO (frame, err);
-        VALIDATE_OR_GOTO (this, err);
-        VALIDATE_OR_GOTO (fd, err);
-        VALIDATE_OR_GOTO(fd->inode, err);
+    if (op != GF_IPC_TARGET_UPCALL)
+        goto wind_default;
 
-        gf_uuid_unparse(fd->inode->gfid, gfid);
+    VALIDATE_OR_GOTO(this->private, err);
+    conf = this->private;
 
-        subvol = dht_subvol_get_cached (this, fd->inode);
-        if (!subvol) {
-                gf_msg_debug (this->name, 0,
-                              "No cached subvolume for fd=%p,"
-                              " gfid = %s", fd, gfid);
-                op_errno = EINVAL;
-                goto err;
-        }
+    local = dht_local_init(frame, NULL, NULL, GF_FOP_IPC);
+    if (!local) {
+        op_errno = ENOMEM;
+        goto err;
+    }
 
-        STACK_WIND (frame, dht_fentrylk_cbk,
-                    subvol, subvol->fops->fentrylk,
-                    volume, fd, basename, cmd, type, xdata);
+    call_cnt = conf->subvolume_cnt;
+    local->call_cnt = call_cnt;
 
-        return 0;
+    if (xdata) {
+        if (dict_set_int8(xdata, conf->xattr_name, 0) < 0)
+            goto err;
+    }
+
+    for (i = 0; i < call_cnt; i++) {
+        STACK_WIND(frame, dht_ipc_cbk, conf->subvolumes[i],
+                   conf->subvolumes[i]->fops->ipc, op, xdata);
+    }
+
+    return 0;
 
 err:
-        op_errno = (op_errno == -1) ? errno : op_errno;
-        DHT_STACK_UNWIND (fentrylk, frame, -1, op_errno, NULL);
+    DHT_STACK_UNWIND(ipc, frame, -1, op_errno, NULL);
 
-        return 0;
-}
+    return 0;
 
+wind_default:
+    STACK_WIND(frame, default_ipc_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->ipc, op, xdata);
+    return 0;
+}
 
 int
-dht_forget (xlator_t *this, inode_t *inode)
+dht_forget(xlator_t *this, inode_t *inode)
 {
-        uint64_t        ctx_int = 0;
-        dht_inode_ctx_t *ctx    = NULL;
-        dht_layout_t *layout = NULL;
+    uint64_t ctx_int = 0;
+    dht_inode_ctx_t *ctx = NULL;
+    dht_layout_t *layout = NULL;
 
-        inode_ctx_del (inode, this, &ctx_int);
+    inode_ctx_del(inode, this, &ctx_int);
 
-        if (!ctx_int)
-                return 0;
+    if (!ctx_int)
+        return 0;
 
-        ctx = (dht_inode_ctx_t *) (long) ctx_int;
+    ctx = (dht_inode_ctx_t *)(long)ctx_int;
 
-        layout = ctx->layout;
-        ctx->layout = NULL;
-        dht_layout_unref (this, layout);
-        GF_FREE (ctx);
+    layout = ctx->layout;
+    ctx->layout = NULL;
+    dht_layout_unref(this, layout);
+    GF_FREE(ctx);
 
-        return 0;
+    return 0;
 }
 
-
 int
-dht_notify (xlator_t *this, int event, void *data, ...)
-{
-        xlator_t                *subvol = NULL;
-        int                      cnt    = -1;
-        int                      i      = -1;
-        dht_conf_t              *conf   = NULL;
-        int                      ret    = -1;
-        int                      propagate = 0;
-
-        int                      had_heard_from_all = 0;
-        int                      have_heard_from_all = 0;
-        struct timeval           time = {0,};
-        gf_defrag_info_t        *defrag = NULL;
-        dict_t                  *dict   = NULL;
-        gf_defrag_type           cmd    = 0;
-        dict_t                  *output = NULL;
-        va_list                  ap;
-        dht_methods_t           *methods = NULL;
-
-        conf = this->private;
-        GF_VALIDATE_OR_GOTO (this->name, conf, out);
-
-        methods = conf->methods;
-        GF_VALIDATE_OR_GOTO (this->name, methods, out);
-
-        /* had all subvolumes reported status once till now? */
-        had_heard_from_all = 1;
-        for (i = 0; i < conf->subvolume_cnt; i++) {
-                if (!conf->last_event[i]) {
-                        had_heard_from_all = 0;
-                }
-        }
-
-        switch (event) {
+dht_notify(xlator_t *this, int event, void *data, ...)
+{
+    xlator_t *subvol = NULL;
+    int cnt = -1;
+    int i = -1;
+    dht_conf_t *conf = NULL;
+    int ret = -1;
+    int propagate = 0;
+
+    int had_heard_from_all = 0;
+    int have_heard_from_all = 0;
+    gf_defrag_info_t *defrag = NULL;
+    dict_t *dict = NULL;
+    gf_defrag_type cmd = 0;
+    dict_t *output = NULL;
+    va_list ap;
+    struct gf_upcall *up_data = NULL;
+    struct gf_upcall_cache_invalidation *up_ci = NULL;
+
+    conf = this->private;
+    GF_VALIDATE_OR_GOTO(this->name, conf, out);
+
+    /* had all subvolumes reported status once till now? */
+    had_heard_from_all = 1;
+    for (i = 0; i < conf->subvolume_cnt; i++) {
+        if (!conf->last_event[i]) {
+            had_heard_from_all = 0;
+        }
+    }
+
+    switch (event) {
         case GF_EVENT_CHILD_UP:
-                subvol = data;
+            subvol = data;
 
-                conf->gen++;
+            conf->gen++;
 
-                for (i = 0; i < conf->subvolume_cnt; i++) {
-                        if (subvol == conf->subvolumes[i]) {
-                                cnt = i;
-                                break;
-                        }
+            for (i = 0; i < conf->subvolume_cnt; i++) {
+                if (subvol == conf->subvolumes[i]) {
+                    cnt = i;
+                    break;
                 }
+            }
 
-                if (cnt == -1) {
-                        gf_msg_debug (this->name, 0,
-                                      "got GF_EVENT_CHILD_UP bad "
-                                      "subvolume %s",
-                                      subvol->name);
-                        break;
-                }
+            if (cnt == -1) {
+                gf_msg_debug(this->name, 0,
+                             "got GF_EVENT_CHILD_UP bad "
+                             "subvolume %s",
+                             subvol->name);
+                break;
+            }
 
-                gettimeofday (&time, NULL);
-                LOCK (&conf->subvolume_lock);
-                {
-                        conf->subvolume_status[cnt] = 1;
-                        conf->last_event[cnt] = event;
-                        conf->subvol_up_time[cnt] = time.tv_sec;
-                }
-                UNLOCK (&conf->subvolume_lock);
+            LOCK(&conf->subvolume_lock);
+            {
+                conf->subvolume_status[cnt] = 1;
+                conf->last_event[cnt] = event;
+                conf->subvol_up_time[cnt] = gf_time();
+            }
+            UNLOCK(&conf->subvolume_lock);
 
-                /* one of the node came back up, do a stat update */
-                dht_get_du_info_for_subvol (this, cnt);
+            /* one of the node came back up, do a stat update */
+            dht_get_du_info_for_subvol(this, cnt);
 
-                break;
+            break;
 
-        case GF_EVENT_CHILD_MODIFIED:
-                subvol = data;
+        case GF_EVENT_SOME_DESCENDENT_UP:
+            subvol = data;
+            conf->gen++;
+            propagate = 1;
 
-                conf->gen++;
-                propagate = 1;
+            break;
 
-                break;
+        case GF_EVENT_SOME_DESCENDENT_DOWN:
+            subvol = data;
+            propagate = 1;
 
-        case GF_EVENT_CHILD_DOWN:
-                subvol = data;
-
-                if (conf->assert_no_child_down) {
-                        gf_log (this->name, GF_LOG_WARNING,
-                                "Received CHILD_DOWN. Exiting");
-                        if (conf->defrag) {
-                                gf_defrag_stop (conf->defrag,
-                                                GF_DEFRAG_STATUS_FAILED, NULL);
-                        } else {
-                                kill (getpid(), SIGTERM);
-                        }
-                }
+            break;
 
-                for (i = 0; i < conf->subvolume_cnt; i++) {
-                        if (subvol == conf->subvolumes[i]) {
-                                cnt = i;
-                                break;
-                        }
-                }
+        case GF_EVENT_CHILD_DOWN:
+            subvol = data;
 
-                if (cnt == -1) {
-                        gf_msg_debug (this->name, 0,
-                                      "got GF_EVENT_CHILD_DOWN bad "
-                                      "subvolume %s", subvol->name);
-                        break;
+            if (conf->assert_no_child_down) {
+                gf_msg(this->name, GF_LOG_WARNING, 0, DHT_MSG_CHILD_DOWN,
+                       "Received CHILD_DOWN. Exiting");
+                if (conf->defrag) {
+                    gf_defrag_stop(conf, GF_DEFRAG_STATUS_FAILED, NULL);
+                } else {
+                    kill(getpid(), SIGTERM);
                 }
+            }
 
-                LOCK (&conf->subvolume_lock);
-                {
-                        conf->subvolume_status[cnt] = 0;
-                        conf->last_event[cnt] = event;
-                        conf->subvol_up_time[cnt] = 0;
+            for (i = 0; i < conf->subvolume_cnt; i++) {
+                if (subvol == conf->subvolumes[i]) {
+                    cnt = i;
+                    break;
                 }
-                UNLOCK (&conf->subvolume_lock);
+            }
 
-                for (i = 0; i < conf->subvolume_cnt; i++)
-                        if (conf->last_event[i] != event)
-                                event = GF_EVENT_CHILD_MODIFIED;
+            if (cnt == -1) {
+                gf_msg_debug(this->name, 0,
+                             "got GF_EVENT_CHILD_DOWN bad "
+                             "subvolume %s",
+                             subvol->name);
                 break;
+            }
 
-        case GF_EVENT_CHILD_CONNECTING:
-                subvol = data;
+            LOCK(&conf->subvolume_lock);
+            {
+                conf->subvolume_status[cnt] = 0;
+                conf->last_event[cnt] = event;
+                conf->subvol_up_time[cnt] = 0;
+            }
+            UNLOCK(&conf->subvolume_lock);
 
-                for (i = 0; i < conf->subvolume_cnt; i++) {
-                        if (subvol == conf->subvolumes[i]) {
-                                cnt = i;
-                                break;
-                        }
-                }
+            for (i = 0; i < conf->subvolume_cnt; i++)
+                if (conf->last_event[i] != event)
+                    event = GF_EVENT_SOME_DESCENDENT_DOWN;
+            break;
 
-                if (cnt == -1) {
-                        gf_msg_debug (this->name, 0,
-                                      "got GF_EVENT_CHILD_CONNECTING"
-                                      " bad subvolume %s",
-                                      subvol->name);
-                        break;
-                }
+        case GF_EVENT_CHILD_CONNECTING:
+            subvol = data;
 
-                LOCK (&conf->subvolume_lock);
-                {
-                        conf->last_event[cnt] = event;
+            for (i = 0; i < conf->subvolume_cnt; i++) {
+                if (subvol == conf->subvolumes[i]) {
+                    cnt = i;
+                    break;
                 }
-                UNLOCK (&conf->subvolume_lock);
+            }
 
+            if (cnt == -1) {
+                gf_msg_debug(this->name, 0,
+                             "got GF_EVENT_CHILD_CONNECTING"
+                             " bad subvolume %s",
+                             subvol->name);
                 break;
-        case GF_EVENT_VOLUME_DEFRAG:
-        {
-                if (!conf->defrag) {
-                        return ret;
-                }
-                defrag = conf->defrag;
+            }
 
-                dict = data;
-                va_start (ap, data);
-                output = va_arg (ap, dict_t*);
+            LOCK(&conf->subvolume_lock);
+            {
+                conf->last_event[cnt] = event;
+            }
+            UNLOCK(&conf->subvolume_lock);
 
-                ret = dict_get_int32 (dict, "rebalance-command",
-                                      (int32_t*)&cmd);
-                if (ret)
-                        return ret;
-                LOCK (&defrag->lock);
-                {
-                        if (defrag->is_exiting)
-                                goto unlock;
-                        if (cmd == GF_DEFRAG_CMD_STATUS)
-                                gf_defrag_status_get (defrag, output);
-                        else if (cmd == GF_DEFRAG_CMD_STOP)
-                                gf_defrag_stop (defrag,
-                                                GF_DEFRAG_STATUS_STOPPED, output);
-                }
-unlock:
-                UNLOCK (&defrag->lock);
-                return 0;
-                break;
-        }
+            break;
+        case GF_EVENT_VOLUME_DEFRAG: {
+            if (!conf->defrag) {
+                return ret;
+            }
+            defrag = conf->defrag;
 
-        default:
-                propagate = 1;
-                break;
-        }
+            dict = data;
+            va_start(ap, data);
+            output = va_arg(ap, dict_t *);
 
+            ret = dict_get_int32(dict, "rebalance-command", (int32_t *)&cmd);
+            if (ret) {
+                va_end(ap);
+                return ret;
+            }
+            LOCK(&defrag->lock);
+            {
+                if (defrag->is_exiting)
+                    goto unlock;
+                if ((cmd == GF_DEFRAG_CMD_STATUS) ||
+                    (cmd == GF_DEFRAG_CMD_DETACH_STATUS))
+                    gf_defrag_status_get(conf, output);
+                else if (cmd == GF_DEFRAG_CMD_DETACH_START)
+                    defrag->cmd = GF_DEFRAG_CMD_DETACH_START;
+                else if (cmd == GF_DEFRAG_CMD_STOP ||
+                         cmd == GF_DEFRAG_CMD_DETACH_STOP)
+                    gf_defrag_stop(conf, GF_DEFRAG_STATUS_STOPPED, output);
+            }
+        unlock:
+            UNLOCK(&defrag->lock);
+            va_end(ap);
+            return ret;
+            break;
+        }
+        case GF_EVENT_UPCALL:
+            up_data = (struct gf_upcall *)data;
+            if (up_data->event_type != GF_UPCALL_CACHE_INVALIDATION)
+                break;
+            up_ci = (struct gf_upcall_cache_invalidation *)up_data->data;
+
+            /* Since md-cache will be aggressively filtering lookups,
+             * the stale layout issue will be more pronounced. Hence
+             * when a layout xattr is changed by the rebalance process
+             * notify all the md-cache clients to invalidate the existing
+             * stat cache and send the lookup next time*/
+            if (up_ci->dict && dict_get(up_ci->dict, conf->xattr_name))
+                up_ci->flags |= UP_EXPLICIT_LOOKUP;
+
+            /* TODO: Instead of invalidating iatt, update the new
+             * hashed/cached subvolume in dht inode_ctx */
+            if (IS_DHT_LINKFILE_MODE(&up_ci->stat))
+                up_ci->flags |= UP_EXPLICIT_LOOKUP;
+
+            propagate = 1;
+            break;
+        default:
+            propagate = 1;
+            break;
+    }
+
+    /* have all subvolumes reported status once by now? */
+    have_heard_from_all = 1;
+    for (i = 0; i < conf->subvolume_cnt; i++) {
+        if (!conf->last_event[i])
+            have_heard_from_all = 0;
+    }
+
+    /* if all subvols have reported status, no need to hide anything
+       or wait for anything else. Just propagate blindly */
+    if (have_heard_from_all) {
+        propagate = 1;
+    }
+
+    if (!had_heard_from_all && have_heard_from_all) {
+        static int run_defrag = 0;
+        /* This is the first event which completes aggregation
+           of events from all subvolumes. If at least one subvol
+           had come up, propagate CHILD_UP, but only this time
+        */
+        event = GF_EVENT_CHILD_DOWN;
 
-        /* have all subvolumes reported status once by now? */
-        have_heard_from_all = 1;
         for (i = 0; i < conf->subvolume_cnt; i++) {
-                if (!conf->last_event[i])
-                        have_heard_from_all = 0;
-        }
-
-        /* if all subvols have reported status, no need to hide anything
-           or wait for anything else. Just propagate blindly */
-        if (have_heard_from_all) {
-                propagate = 1;
-
-        }
-
-
-        if (!had_heard_from_all && have_heard_from_all) {
-                /* This is the first event which completes aggregation
-                   of events from all subvolumes. If at least one subvol
-                   had come up, propagate CHILD_UP, but only this time
-                */
-                event = GF_EVENT_CHILD_DOWN;
-
-                for (i = 0; i < conf->subvolume_cnt; i++) {
-                        if (conf->last_event[i] == GF_EVENT_CHILD_UP) {
-                                event = GF_EVENT_CHILD_UP;
-                                break;
-                        }
-
-                        if (conf->last_event[i] == GF_EVENT_CHILD_CONNECTING) {
-                                event = GF_EVENT_CHILD_CONNECTING;
-                                /* continue to check other events for CHILD_UP */
-                        }
-                }
+            if (conf->last_event[i] == GF_EVENT_CHILD_UP) {
+                event = GF_EVENT_CHILD_UP;
+                break;
+            }
 
-                /* rebalance is started with assert_no_child_down. So we do
-                 * not need to handle CHILD_DOWN event here.
-                 */
-                if (conf->defrag) {
-                        if (methods->migration_needed(this)) {
-                                ret = gf_thread_create(&conf->defrag->th,
-                                                       NULL,
-                                                       gf_defrag_start, this);
-                                if (ret) {
-                                        GF_FREE (conf->defrag);
-                                        conf->defrag = NULL;
-                                        kill (getpid(), SIGTERM);
-                                }
-                        } else {
-                                GF_FREE (conf->defrag);
-                                conf->defrag = NULL;
-                        }
-                }
+            if (conf->last_event[i] == GF_EVENT_CHILD_CONNECTING) {
+                event = GF_EVENT_CHILD_CONNECTING;
+                /* continue to check other events for CHILD_UP */
+            }
         }
 
-        ret = 0;
-        if (propagate)
-                ret = default_notify (this, event, data);
+        /* Rebalance is started with assert_no_child_down. So we do
+         * not need to handle CHILD_DOWN event here.
+         *
+         * If there is a graph switch, we should not restart the
+         * rebalance daemon. Use 'run_defrag' to indicate if the
+         * thread has already started.
+         */
+        if (conf->defrag && !run_defrag) {
+            run_defrag = 1;
+            ret = gf_thread_create(&conf->defrag->th, NULL, gf_defrag_start,
+                                   this, "dhtdg");
+            if (ret) {
+                GF_FREE(conf->defrag);
+                conf->defrag = NULL;
+                kill(getpid(), SIGTERM);
+            }
+        }
+    }
+
+    ret = 0;
+    if (propagate)
+        ret = default_notify(this, event, data);
 out:
-        return ret;
+    return ret;
 }
 
 int
-dht_inode_ctx_layout_get (inode_t *inode, xlator_t *this, dht_layout_t **layout)
+dht_inode_ctx_layout_get(inode_t *inode, xlator_t *this, dht_layout_t **layout)
 {
-        dht_inode_ctx_t         *ctx            = NULL;
-        int                      ret            = -1;
+    dht_inode_ctx_t *ctx = NULL;
+    int ret = -1;
 
-        ret = dht_inode_ctx_get (inode, this, &ctx);
+    ret = dht_inode_ctx_get(inode, this, &ctx);
 
-        if (!ret && ctx) {
-                if (ctx->layout) {
-                        if (layout)
-                                *layout = ctx->layout;
-                        ret = 0;
-                } else {
-                        ret = -1;
-                }
+    if (!ret && ctx) {
+        if (ctx->layout) {
+            if (layout)
+                *layout = ctx->layout;
+            ret = 0;
+        } else {
+            ret = -1;
         }
+    }
 
-        return ret;
+    return ret;
 }
 
 void
-dht_log_new_layout_for_dir_selfheal (xlator_t *this, loc_t *loc,
-                                 dht_layout_t *layout)
+dht_log_new_layout_for_dir_selfheal(xlator_t *this, loc_t *loc,
+                                    dht_layout_t *layout)
 {
+    char string[2048] = {0};
+    char *output_string = NULL;
+    int len = 0;
+    int off = 0;
+    int i = 0;
+    gf_loglevel_t log_level = gf_log_get_loglevel();
+    int ret = 0;
+
+    if (log_level < GF_LOG_INFO)
+        return;
 
-        char                    string[2048] = {0};
-        char                  *output_string = NULL;
-        int                              len = 0;
-        int                              off = 0;
-        int                                i = 0;
-        gf_loglevel_t             log_level = gf_log_get_loglevel();
-        int                              ret = 0;
-        int                   max_string_len = 0;
-
-        if (log_level < GF_LOG_INFO)
-                return;
+    if (!layout)
+        return;
 
-        if (!layout)
-                return;
+    if (!layout->cnt)
+        return;
 
-        if (!layout->cnt)
-                return;
+    if (!loc)
+        return;
 
-        if (!loc)
-                return;
+    if (!loc->path)
+        return;
 
-        if (!loc->path)
-                return;
+    ret = snprintf(string, sizeof(string), "Setting layout of %s with ",
+                   loc->path);
 
-        max_string_len = sizeof (string);
+    if (ret < 0)
+        return;
 
-        ret = snprintf (string, max_string_len, "Setting layout of %s with ",
-                        loc->path);
+    len += ret;
+
+    /* Calculation  of total length of the string required to calloc
+     * output_string. Log includes subvolume-name, start-range, end-range
+     * and err value.
+     *
+     * This log will help to debug cases where:
+     * a) Different processes set different layout of a directory.
+     * b) Error captured in lookup, which will be filled in layout->err
+     * (like ENOENT, ESTALE etc)
+     */
+
+    for (i = 0; i < layout->cnt; i++) {
+        ret = snprintf(string, sizeof(string),
+                       "[Subvol_name: %s, Err: %d , Start: "
+                       "0x%x, Stop: 0x%x, Hash: 0x%x], ",
+                       layout->list[i].xlator->name, layout->list[i].err,
+                       layout->list[i].start, layout->list[i].stop,
+                       layout->list[i].commit_hash);
 
         if (ret < 0)
-                return;
+            return;
 
         len += ret;
+    }
 
-       /* Calculation  of total length of the string required to calloc
-        * output_string. Log includes subvolume-name, start-range, end-range and
-        * err value.
-        *
-        * This log will help to debug cases where:
-        * a) Different processes set different layout of a directory.
-        * b) Error captured in lookup, which will be filled in layout->err
-        * (like ENOENT, ESTALE etc)
-        */
+    len++;
 
-        for (i = 0; i < layout->cnt; i++) {
+    output_string = GF_MALLOC(len + 1, gf_common_mt_char);
 
-                ret  = snprintf (string, max_string_len,
-                                 "[Subvol_name: %s, Err: %d , Start: "
-                                 "%"PRIu32 " , Stop: %"PRIu32 " ], ",
-                                 layout->list[i].xlator->name,
-                                 layout->list[i].err, layout->list[i].start,
-                                 layout->list[i].stop);
+    if (!output_string)
+        return;
 
-                if (ret < 0)
-                        return;
+    ret = snprintf(output_string, len + 1, "Setting layout of %s with ",
+                   loc->path);
 
-                len += ret;
+    if (ret < 0)
+        goto err;
 
-        }
+    off += ret;
 
-        len++;
+    for (i = 0; i < layout->cnt; i++) {
+        ret = snprintf(output_string + off, len - off,
+                       "[Subvol_name: %s, Err: %d , Start: "
+                       "0x%x, Stop: 0x%x, Hash: 0x%x], ",
+                       layout->list[i].xlator->name, layout->list[i].err,
+                       layout->list[i].start, layout->list[i].stop,
+                       layout->list[i].commit_hash);
 
-        output_string = GF_CALLOC (len, sizeof (char), gf_common_mt_char);
+        if (ret < 0)
+            goto err;
 
-        if (!output_string)
-                return;
+        off += ret;
+    }
 
-        ret = snprintf (output_string, len, "Setting layout of %s with ",
-                        loc->path);
+    gf_msg(this->name, GF_LOG_DEBUG, 0, DHT_MSG_LOG_FIXED_LAYOUT, "%s",
+           output_string);
 
-        if (ret < 0)
-                goto err;
+err:
+    GF_FREE(output_string);
+}
 
-        off += ret;
+int32_t
+dht_migration_get_dst_subvol(xlator_t *this, dht_local_t *local)
+{
+    int ret = -1;
 
+    if (!local)
+        goto out;
 
-        for (i = 0; i < layout->cnt; i++) {
+    local->rebalance.target_node = dht_subvol_get_hashed(this, &local->loc);
+
+    if (local->rebalance.target_node)
+        ret = 0;
+
+out:
+    return ret;
+}
+
+/*
+This function should not be called more then once during a FOP
+handling path. It is valid only for for ops on files
+*/
+int32_t
+dht_set_local_rebalance(xlator_t *this, dht_local_t *local, struct iatt *stbuf,
+                        struct iatt *prebuf, struct iatt *postbuf,
+                        dict_t *xdata)
+{
+    if (!local)
+        return -1;
 
-                ret  =  snprintf (output_string + off, len - off,
-                                  "[Subvol_name: %s, Err: %d , Start: "
-                                  "%"PRIu32 " , Stop: %"PRIu32 " ], ",
-                                  layout->list[i].xlator->name,
-                                  layout->list[i].err, layout->list[i].start,
-                                  layout->list[i].stop);
+    if (local->rebalance.set) {
+        gf_msg(this->name, GF_LOG_WARNING, 0, DHT_MSG_REBAL_STRUCT_SET,
+               "local->rebalance already set");
+    }
 
-                if (ret < 0)
-                        goto err;
+    if (stbuf)
+        memcpy(&local->rebalance.stbuf, stbuf, sizeof(struct iatt));
 
-                off += ret;
+    if (prebuf)
+        memcpy(&local->rebalance.prebuf, prebuf, sizeof(struct iatt));
 
-        }
+    if (postbuf)
+        memcpy(&local->rebalance.postbuf, postbuf, sizeof(struct iatt));
 
-        gf_msg (this->name, GF_LOG_INFO, 0, DHT_MSG_LOG_FIXED_LAYOUT,
-                "%s", output_string);
+    if (xdata)
+        local->rebalance.xdata = dict_ref(xdata);
+
+    local->rebalance.set = 1;
+
+    return 0;
+}
+
+int32_t
+dht_release(xlator_t *this, fd_t *fd)
+{
+    return dht_fd_ctx_destroy(this, fd);
+}
+
+static int
+dht_pt_mkdir_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int op_ret,
+                 int op_errno, inode_t *inode, struct iatt *stbuf,
+                 struct iatt *preparent, struct iatt *postparent, dict_t *xdata)
+{
+    dht_local_t *local = NULL;
+
+    local = frame->local;
+
+    if (!op_ret) {
+        dht_layout_set(this, inode, local->layout);
+    }
+
+    DHT_STACK_UNWIND(mkdir, frame, op_ret, op_errno, inode, stbuf, preparent,
+                     postparent, NULL);
+
+    return 0;
+}
+
+int32_t
+dht_pt_mkdir(call_frame_t *frame, xlator_t *this, loc_t *loc, mode_t mode,
+             mode_t umask, dict_t *xdata)
+{
+    dht_layout_t *layout = NULL;
+    dht_conf_t *conf = NULL;
+    dht_local_t *local = NULL;
+    bool free_xdata = false;
+    int ret = 0;
+    int op_errno = 0;
+    int32_t *disk_layout_p = NULL;
+
+    conf = this->private;
+
+    local = dht_local_init(frame, loc, NULL, GF_FOP_MKDIR);
+    if (!local) {
+        op_errno = ENOMEM;
+        goto err;
+    }
+
+    layout = dht_layout_new(this, conf->subvolume_cnt);
+    if (!layout)
+        goto wind;
+
+    local->layout = layout;
+
+    if (!xdata) {
+        xdata = dict_new();
+        if (!xdata)
+            goto wind;
+        free_xdata = true;
+    }
+
+    /*Set the xlator or the following will crash*/
+    layout->list[0].xlator = conf->subvolumes[0];
+
+    dht_selfheal_layout_new_directory(frame, loc, layout);
+
+    dht_disk_layout_extract(this, layout, 0, &disk_layout_p);
+
+    ret = dict_set_bin(xdata, conf->xattr_name, disk_layout_p, 4 * 4);
+    if (ret) {
+        gf_msg("dht", GF_LOG_DEBUG, EINVAL, DHT_MSG_DICT_SET_FAILED,
+               "dht layout dict set failed");
+    }
+wind:
+    STACK_WIND(frame, dht_pt_mkdir_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->mkdir, loc, mode, umask, xdata);
+    if (free_xdata)
+        dict_unref(xdata);
+    return 0;
 
 err:
-        GF_FREE (output_string);
+    op_errno = local ? local->op_errno : op_errno;
+    DHT_STACK_UNWIND(mkdir, frame, -1, op_errno, NULL, NULL, NULL, NULL, NULL);
+
+    return 0;
 }
 
-int32_t dht_migration_get_dst_subvol(xlator_t *this, dht_local_t  *local)
+static int
+dht_pt_getxattr_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                    int op_ret, int op_errno, dict_t *xattr, dict_t *xdata)
 {
-        int ret = -1;
+    dht_conf_t *conf = NULL;
 
-        if (!local)
-                goto out;
+    conf = this->private;
+    dict_del(xattr, conf->xattr_name);
+    dict_del(xattr, conf->mds_xattr_key);
+    dict_del(xattr, conf->commithash_xattr_name);
 
-        local->rebalance.target_node =
-                dht_subvol_get_hashed (this, &local->loc);
+    if (frame->root->pid >= 0) {
+        GF_REMOVE_INTERNAL_XATTR("trusted.glusterfs.quota*", xattr);
+        GF_REMOVE_INTERNAL_XATTR("trusted.pgfid*", xattr);
+    }
 
-        if (local->rebalance.target_node)
-                ret = 0;
+    DHT_STACK_UNWIND(getxattr, frame, op_ret, op_errno, xattr, xdata);
+    return 0;
+}
 
-out:
-        return ret;
+int
+dht_pt_getxattr(call_frame_t *frame, xlator_t *this, loc_t *loc,
+                const char *key, dict_t *xdata)
+{
+    STACK_WIND(frame, dht_pt_getxattr_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->getxattr, loc, key, xdata);
+    return 0;
 }
 
-int32_t dht_migration_needed(xlator_t *this)
+static int
+dht_pt_fgetxattr_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                     int op_ret, int op_errno, dict_t *xattr, dict_t *xdata)
 {
-        gf_defrag_info_t        *defrag = NULL;
-        dht_conf_t              *conf   = NULL;
-        int                      ret = 0;
+    dht_conf_t *conf = NULL;
 
-        conf = this->private;
+    conf = this->private;
+    dict_del(xattr, conf->xattr_name);
 
-        GF_VALIDATE_OR_GOTO ("dht", conf, out);
-        GF_VALIDATE_OR_GOTO ("dht", conf->defrag, out);
+    if (frame->root->pid >= 0) {
+        GF_REMOVE_INTERNAL_XATTR("trusted.glusterfs.quota*", xattr);
+        GF_REMOVE_INTERNAL_XATTR("trusted.pgfid*", xattr);
+    }
 
-        defrag = conf->defrag;
+    DHT_STACK_UNWIND(fgetxattr, frame, op_ret, op_errno, xattr, xdata);
+    return 0;
+}
 
-        if (defrag->cmd != GF_DEFRAG_CMD_START_TIER)
-                ret = 1;
+int
+dht_pt_fgetxattr(call_frame_t *frame, xlator_t *this, fd_t *fd, const char *key,
+                 dict_t *xdata)
+{
+    STACK_WIND(frame, dht_pt_fgetxattr_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->fgetxattr, fd, key, xdata);
+    return 0;
+}
 
-out:
-        return ret;
+/* The job of this function is to check if all the xlators have updated
+ * error in the layout. */
+int
+dht_dir_layout_error_check(xlator_t *this, inode_t *inode)
+{
+    dht_layout_t *layout = NULL;
+    int i = 0;
+
+    layout = dht_layout_get(this, inode);
+    for (i = 0; i < layout->cnt; i++) {
+        if (layout->list[i].err == 0) {
+            return 0;
+        }
+    }
+
+    /* Returning the first xlator error as all xlators have errors */
+    return layout->list[0].err;
 }
diff --git a/xlators/cluster/dht/src/dht-common.h b/xlators/cluster/dht/src/dht-common.h
index 3ca626feec8..fe0dc3db34a 100644
--- a/xlators/cluster/dht/src/dht-common.h
+++ b/xlators/cluster/dht/src/dht-common.h
@@ -8,99 +8,149 @@
   cases as published by the Free Software Foundation.
 */
 
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
-
 #include <regex.h>
-#include <signal.h>
 
 #include "dht-mem-types.h"
 #include "dht-messages.h"
+#include <glusterfs/call-stub.h>
 #include "libxlator.h"
-#include "syncop.h"
+#include <glusterfs/syncop.h>
+#include <glusterfs/refcount.h>
+#include <glusterfs/timer.h>
+#include "protocol-common.h"
+#include <glusterfs/glusterfs-acl.h>
 
 #ifndef _DHT_H
 #define _DHT_H
 
-#define GF_XATTR_FIX_LAYOUT_KEY     "distribute.fix.layout"
-#define GF_XATTR_FILE_MIGRATE_KEY   "trusted.distribute.migrate-data"
-#define GF_DHT_LOOKUP_UNHASHED_ON   1
+#define GF_XATTR_FIX_LAYOUT_KEY "distribute.fix.layout"
+#define GF_XATTR_FILE_MIGRATE_KEY "trusted.distribute.migrate-data"
+#define DHT_MDS_STR "mds"
+#define GF_DHT_LOOKUP_UNHASHED_OFF 0
+#define GF_DHT_LOOKUP_UNHASHED_ON 1
 #define GF_DHT_LOOKUP_UNHASHED_AUTO 2
-#define DHT_PATHINFO_HEADER         "DISTRIBUTE:"
-#define DHT_FILE_MIGRATE_DOMAIN     "dht.file.migrate"
-#define DHT_LAYOUT_HEAL_DOMAIN      "dht.layout.heal"
+#define DHT_PATHINFO_HEADER "DISTRIBUTE:"
+#define DHT_FILE_MIGRATE_DOMAIN "dht.file.migrate"
+/* Layout synchronization */
+#define DHT_LAYOUT_HEAL_DOMAIN "dht.layout.heal"
+/* Namespace synchronization */
+#define DHT_ENTRY_SYNC_DOMAIN "dht.entry.sync"
+#define DHT_LAYOUT_HASH_INVALID 1
+#define MAX_REBAL_THREADS sysconf(_SC_NPROCESSORS_ONLN)
+
+#define DHT_DIR_STAT_BLOCKS 8
+#define DHT_DIR_STAT_SIZE 4096
+
+/* Virtual xattr for subvols status */
+
+#define DHT_SUBVOL_STATUS_KEY "dht.subvol.status"
+
+/* Virtual xattrs for debugging */
 
-#include <fnmatch.h>
+#define DHT_DBG_HASHED_SUBVOL_PATTERN "dht.file.hashed-subvol.*"
+#define DHT_DBG_HASHED_SUBVOL_KEY "dht.file.hashed-subvol."
 
-typedef int (*dht_selfheal_dir_cbk_t) (call_frame_t *frame, void *cookie,
-                                       xlator_t     *this,
-                                       int32_t       op_ret, int32_t op_errno,
-                                       dict_t *xdata);
-typedef int (*dht_defrag_cbk_fn_t) (xlator_t        *this, call_frame_t *frame,
-                                    int              ret);
+/* Rebalance nodeuuid flags */
+#define REBAL_NODEUUID_MINE 0x01
 
+typedef int (*dht_selfheal_dir_cbk_t)(call_frame_t *frame, void *cookie,
+                                      xlator_t *this, int32_t op_ret,
+                                      int32_t op_errno, dict_t *xdata);
+typedef int (*dht_defrag_cbk_fn_t)(xlator_t *this, xlator_t *dst_node,
+                                   call_frame_t *frame, int ret);
+
+typedef int (*dht_refresh_layout_unlock)(call_frame_t *frame, xlator_t *this,
+                                         int op_ret, int invoke_cbk);
+
+typedef int (*dht_refresh_layout_done_handle)(call_frame_t *frame);
 
 struct dht_layout {
-        int                spread_cnt;  /* layout spread count per directory,
-                                           is controlled by 'setxattr()' with
-                                           special key */
-        int                cnt;
-        int                preset;
-        int                gen;
-        int                type;
-        int                ref; /* use with dht_conf_t->layout_lock */
-        gf_boolean_t       search_unhashed;
-        struct {
-                int        err;   /* 0 = normal
-                                     -1 = dir exists and no xattr
-                                     >0 = dir lookup failed with errno
-                                  */
-                uint32_t   start;
-                uint32_t   stop;
-                xlator_t  *xlator;
-        } list[];
+    int spread_cnt; /* layout spread count per directory,
+                       is controlled by 'setxattr()' with
+                       special key */
+    int cnt;
+    int preset;
+    /*
+     * The last *configuration* state for which this directory was known
+     * to be in balance.  The corresponding vol_commit_hash changes
+     * whenever bricks are added or removed.  This value changes when a
+     * (full) rebalance is complete.  If they match, it's safe to assume
+     * that every file is where it should be and there's no need to do
+     * lookups for files elsewhere.  If they don't, then we have to do a
+     * global lookup to be sure.
+     */
+    uint32_t commit_hash;
+    /*
+     * The *runtime* state of the volume, changes when connections to
+     * bricks are made or lost.
+     */
+    int gen;
+    int type;
+    gf_atomic_t ref; /* use with dht_conf_t->layout_lock */
+    uint32_t search_unhashed;
+    struct {
+        int err; /* 0 = normal
+                    -1 = dir exists and no xattr
+                    >0 = dir lookup failed with errno
+                 */
+        uint32_t start;
+        uint32_t stop;
+        uint32_t commit_hash;
+        xlator_t *xlator;
+    } list[];
 };
-typedef struct dht_layout  dht_layout_t;
+typedef struct dht_layout dht_layout_t;
 
 struct dht_stat_time {
-        uint32_t        atime;
-        uint32_t        atime_nsec;
-        uint32_t        ctime;
-        uint32_t        ctime_nsec;
-        uint32_t        mtime;
-        uint32_t        mtime_nsec;
+    uint32_t atime;
+    uint32_t atime_nsec;
+    uint32_t ctime;
+    uint32_t ctime_nsec;
+    uint32_t mtime;
+    uint32_t mtime_nsec;
 };
 
 typedef struct dht_stat_time dht_stat_time_t;
 
 struct dht_inode_ctx {
-        dht_layout_t    *layout;
-        dht_stat_time_t  time;
+    dht_layout_t *layout;
+    dht_stat_time_t time;
+    xlator_t *lock_subvol;
+    xlator_t *mds_subvol; /* This is only used for directories */
 };
 
 typedef struct dht_inode_ctx dht_inode_ctx_t;
 
-
 typedef enum {
-        DHT_HASH_TYPE_DM,
-        DHT_HASH_TYPE_DM_USER,
+    DHT_HASH_TYPE_DM,
+    DHT_HASH_TYPE_DM_USER,
 } dht_hashfn_type_t;
 
+typedef enum {
+    DHT_INODELK,
+    DHT_ENTRYLK,
+} dht_lock_type_t;
+
 /* rebalance related */
 struct dht_rebalance_ {
-        xlator_t            *from_subvol;
-        xlator_t            *target_node;
-        off_t                offset;
-        size_t               size;
-        int32_t              flags;
-        int                  count;
-        struct iobref       *iobref;
-        struct iovec        *vector;
-        struct iatt          stbuf;
-        dht_defrag_cbk_fn_t  target_op_fn;
-        dict_t              *xdata;
+    xlator_t *from_subvol;
+    xlator_t *target_node;
+    off_t offset;
+    size_t size;
+    int32_t flags;
+    int count;
+    struct iobref *iobref;
+    struct iovec *vector;
+    struct iatt stbuf;
+    struct iatt prebuf;
+    struct iatt postbuf;
+    dht_defrag_cbk_fn_t target_op_fn;
+    dict_t *xdata;
+    dict_t *xattr;
+    dict_t *dict;
+    struct gf_flock flock;
+    int32_t set;
+    int lock_cmd;
 };
 
 /**
@@ -108,844 +158,1227 @@ struct dht_rebalance_ {
  * events
  **/
 typedef enum {
-        qdstatfs_action_OFF = 0,
-        qdstatfs_action_REPLACE,
-        qdstatfs_action_NEGLECT,
-        qdstatfs_action_COMPARE,
+    qdstatfs_action_OFF = 0,
+    qdstatfs_action_REPLACE,
+    qdstatfs_action_NEGLECT,
+    qdstatfs_action_COMPARE,
 } qdstatfs_action_t;
 
-struct dht_skip_linkto_unlink {
+typedef enum {
+    REACTION_INVALID,
+    FAIL_ON_ANY_ERROR,
+    IGNORE_ENOENT_ESTALE,
+    IGNORE_ENOENT_ESTALE_EIO,
+} dht_reaction_type_t;
 
-        gf_boolean_t    handle_valid_link;
-        int             opend_fd_count;
-        xlator_t        *hash_links_to;
-        uuid_t          cached_gfid;
-        uuid_t          hashed_gfid;
+struct dht_skip_linkto_unlink {
+    xlator_t *hash_links_to;
+    uuid_t cached_gfid;
+    uuid_t hashed_gfid;
+    int opend_fd_count;
+    gf_boolean_t handle_valid_link;
 };
 
 typedef struct {
-        xlator_t     *xl;
-        loc_t         loc;     /* contains/points to inode to lock on. */
-        short         type;    /* read/write lock.                     */
-        char         *domain;  /* Only locks within a single domain
-                                * contend with each other
-                                */
-        gf_lkowner_t  lk_owner;
-        gf_boolean_t  locked;
+    xlator_t *xl;
+    loc_t loc;      /* contains/points to inode to lock on. */
+    char *domain;   /* Only locks within a single domain
+                     * contend with each other
+                     */
+    char *basename; /* Required for entrylk */
+    gf_boolean_t locked;
+    dht_reaction_type_t do_on_failure;
+    short type; /* read/write lock.                     */
+    gf_lkowner_t lk_owner;
 } dht_lock_t;
 
-typedef
-int (*dht_selfheal_layout_t)(call_frame_t *frame, loc_t *loc,
-                             dht_layout_t *layout);
-
-typedef
-gf_boolean_t (*dht_need_heal_t)(call_frame_t *frame, dht_layout_t **inmem,
-                                dht_layout_t **ondisk);
-
-struct dht_local {
-        int                      call_cnt;
-        loc_t                    loc;
-        loc_t                    loc2;
-        int                      op_ret;
-        int                      op_errno;
-        int                      layout_mismatch;
-        /* Use stbuf as the postbuf, when we require both
-         * pre and post attrs */
-        struct iatt              stbuf;
-        struct iatt              prebuf;
-        struct iatt              preoldparent;
-        struct iatt              postoldparent;
-        struct iatt              preparent;
-        struct iatt              postparent;
-        struct statvfs           statvfs;
-        fd_t                    *fd;
-        inode_t                 *inode;
-        dict_t                  *params;
-        dict_t                  *xattr;
-        dict_t                  *xattr_req;
-        dht_layout_t            *layout;
-        size_t                   size;
-        ino_t                    ia_ino;
-        xlator_t                *src_hashed, *src_cached;
-        xlator_t                *dst_hashed, *dst_cached;
-        xlator_t                *cached_subvol;
-        xlator_t                *hashed_subvol;
-        char                     need_selfheal;
-        int                      file_count;
-        int                      dir_count;
-        call_frame_t            *main_frame;
-        int                      fop_succeeded;
-        struct {
-                fop_mknod_cbk_t  linkfile_cbk;
-                struct iatt      stbuf;
-                loc_t            loc;
-                inode_t         *inode;
-                dict_t          *xattr;
-                xlator_t        *srcvol;
-        } linkfile;
-        struct {
-                uint32_t                hole_cnt;
-                uint32_t                overlaps_cnt;
-                uint32_t                down;
-                uint32_t                misc;
-                dht_selfheal_dir_cbk_t  dir_cbk;
-                dht_selfheal_layout_t   healer;
-                dht_need_heal_t         should_heal;
-                gf_boolean_t            force_mkdir;
-                dht_layout_t           *layout, *refreshed_layout;
-        } selfheal;
-        uint32_t                 uid;
-        uint32_t                 gid;
-
-        /* needed by nufa */
-        int32_t flags;
-        mode_t  mode;
-        dev_t   rdev;
-        mode_t  umask;
-
-        /* need for file-info */
-        char   *xattr_val;
-        char   *key;
-
-        /* which xattr request? */
-        char xsel[256];
-        int32_t alloc_len;
-
-        char   *newpath;
-
-        /* gfid related */
-        uuid_t  gfid;
-
-        /* flag used to make sure we need to return estale in
-           {lookup,revalidate}_cbk */
-        char return_estale;
-        char need_lookup_everywhere;
-
-        glusterfs_fop_t      fop;
-
-        gf_boolean_t     linked;
-        xlator_t        *link_subvol;
-
-        struct dht_rebalance_ rebalance;
-        xlator_t        *first_up_subvol;
+/* The lock structure represents inodelk. */
+typedef struct {
+    fop_inodelk_cbk_t inodelk_cbk;
+    dht_lock_t **locks;
+    int lk_count;
+    dht_reaction_type_t reaction;
 
-        gf_boolean_t     quota_deem_statfs;
+    /* whether locking failed on _any_ of the "locks" above */
+    int op_ret;
+    int op_errno;
+} dht_ilock_wrap_t;
 
-        gf_boolean_t     added_link;
-        gf_boolean_t     is_linkfile;
+/* The lock structure represents entrylk. */
+typedef struct {
+    fop_entrylk_cbk_t entrylk_cbk;
+    dht_lock_t **locks;
+    int lk_count;
+    dht_reaction_type_t reaction;
+
+    /* whether locking failed on _any_ of the "locks" above */
+    int op_ret;
+    int op_errno;
+} dht_elock_wrap_t;
+
+/* The first member of dht_dir_transaction_t should be of type dht_ilock_wrap_t.
+ * Otherwise it can result in subtle memory corruption issues as in most of the
+ * places we use lock[0].layout.my_layout or lock[0].layout.parent_layout and
+ * lock[0].ns.parent_layout (like in dht_local_wipe).
+ */
+typedef union {
+    union {
+        dht_ilock_wrap_t my_layout;
+        dht_ilock_wrap_t parent_layout;
+    } layout;
+    struct dht_namespace {
+        dht_ilock_wrap_t parent_layout;
+        dht_elock_wrap_t directory_ns;
+        fop_entrylk_cbk_t ns_cbk;
+    } ns;
+} dht_dir_transaction_t;
+
+typedef int (*dht_selfheal_layout_t)(call_frame_t *frame, loc_t *loc,
+                                     dht_layout_t *layout);
 
-        struct dht_skip_linkto_unlink  skip_unlink;
+typedef gf_boolean_t (*dht_need_heal_t)(call_frame_t *frame,
+                                        dht_layout_t **inmem,
+                                        dht_layout_t **ondisk);
 
-        struct {
-                fop_inodelk_cbk_t   inodelk_cbk;
-                dht_lock_t        **locks;
-                int                 lk_count;
-
-                /* whether locking failed on _any_ of the "locks" above */
-                int                 op_ret;
-                int                 op_errno;
-        } lock;
+struct dht_local {
+    loc_t loc;
+    loc_t loc2;
+    int call_cnt;
+    int op_ret;
+    int op_errno;
+    int layout_mismatch;
+    /* Use stbuf as the postbuf, when we require both
+     * pre and post attrs */
+    struct iatt stbuf;
+    struct iatt mds_stbuf;
+    struct iatt prebuf;
+    struct iatt preoldparent;
+    struct iatt postoldparent;
+    struct iatt preparent;
+    struct iatt postparent;
+    struct statvfs statvfs;
+    fd_t *fd;
+    inode_t *inode;
+    dict_t *params;
+    dict_t *xattr;
+    dict_t *mds_xattr;
+    dict_t *xdata; /* dict used to save xdata response by xattr fop */
+    dict_t *xattr_req;
+    dht_layout_t *layout;
+    size_t size;
+    ino_t ia_ino;
+    xlator_t *src_hashed, *src_cached;
+    xlator_t *dst_hashed, *dst_cached;
+    xlator_t *cached_subvol;
+    xlator_t *hashed_subvol;
+    xlator_t *mds_subvol; /* This is use for dir only */
+    int file_count;
+    int dir_count;
+    call_frame_t *main_frame;
+    int fop_succeeded;
+    struct {
+        fop_mknod_cbk_t linkfile_cbk;
+        struct iatt stbuf;
+        loc_t loc;
+        inode_t *inode;
+        dict_t *xattr;
+        xlator_t *srcvol;
+    } linkfile;
+    struct {
+        uint32_t hole_cnt;
+        uint32_t overlaps_cnt;
+        uint32_t down;
+        uint32_t misc;
+        dht_selfheal_dir_cbk_t dir_cbk;
+        dht_selfheal_layout_t healer;
+        dht_need_heal_t should_heal;
+        dht_layout_t *layout, *refreshed_layout;
+        uint32_t missing_cnt;
+        gf_boolean_t force_mkdir;
+    } selfheal;
+
+    dht_refresh_layout_unlock refresh_layout_unlock;
+    dht_refresh_layout_done_handle refresh_layout_done;
+
+    uint32_t uid;
+    uint32_t gid;
+    pid_t pid;
+
+    glusterfs_fop_t fop;
+
+    /* need for file-info */
+    char *xattr_val;
+    char *key;
+
+    /* needed by nufa */
+    int32_t flags;
+    mode_t mode;
+    dev_t rdev;
+    mode_t umask;
+
+    /* which xattr request? */
+    char xsel[256];
+    int32_t alloc_len;
+
+    /* gfid related */
+    uuid_t gfid;
+    uuid_t gfid_req;
+
+    xlator_t *link_subvol;
+
+    struct dht_rebalance_ rebalance;
+    xlator_t *first_up_subvol;
+
+    struct dht_skip_linkto_unlink skip_unlink;
+
+    dht_dir_transaction_t lock[2], *current;
+
+    /* inodelks during filerename for backward compatibility */
+    dht_lock_t **rename_inodelk_backward_compatible;
+
+    call_stub_t *stub;
+    int32_t parent_disk_layout[4];
+
+    /* rename rollback */
+    int *ret_cache;
+
+    loc_t loc2_copy;
+
+    int rename_inodelk_bc_count;
+    /* This is use only for directory operation */
+    int32_t valid;
+    int32_t mds_heal_fresh_lookup;
+    short lock_type;
+    char need_selfheal;
+    char need_xattr_heal;
+    char need_attrheal;
+    /* flag used to make sure we need to return estale in
+       {lookup,revalidate}_cbk */
+    char return_estale;
+    char need_lookup_everywhere;
+    /* fd open check */
+    gf_boolean_t fd_checked;
+    gf_boolean_t linked;
+    gf_boolean_t added_link;
+    gf_boolean_t is_linkfile;
+    gf_boolean_t quota_deem_statfs;
+    gf_boolean_t heal_layout;
+    gf_boolean_t locked;
+    gf_boolean_t dont_create_linkto;
+    gf_boolean_t gfid_missing;
 };
 typedef struct dht_local dht_local_t;
 
 /* du - disk-usage */
 struct dht_du {
-        double   avail_percent;
-	double   avail_inodes;
-        uint64_t avail_space;
-        uint32_t log;
-        uint32_t chunks;
+    double avail_percent;
+    double avail_inodes;
+    uint64_t avail_space;
+    uint32_t log;
+    uint32_t chunks;
+    uint32_t total_blocks;
+    uint32_t avail_blocks;
+    uint32_t frsize; /*fragment size*/
 };
 typedef struct dht_du dht_du_t;
 
 enum gf_defrag_type {
-        GF_DEFRAG_CMD_START = 1,
-        GF_DEFRAG_CMD_STOP = 1 + 1,
-        GF_DEFRAG_CMD_STATUS = 1 + 2,
-        GF_DEFRAG_CMD_START_LAYOUT_FIX = 1 + 3,
-        GF_DEFRAG_CMD_START_FORCE = 1 + 4,
-        GF_DEFRAG_CMD_START_TIER = 1 + 5,
-        GF_DEFRAG_CMD_STATUS_TIER = 1 + 6,
+    GF_DEFRAG_CMD_NONE = 0,
+    GF_DEFRAG_CMD_START = 1,
+    GF_DEFRAG_CMD_STOP = 1 + 1,
+    GF_DEFRAG_CMD_STATUS = 1 + 2,
+    GF_DEFRAG_CMD_START_LAYOUT_FIX = 1 + 3,
+    GF_DEFRAG_CMD_START_FORCE = 1 + 4,
+    GF_DEFRAG_CMD_DETACH_STATUS = 1 + 11,
+    GF_DEFRAG_CMD_DETACH_START = 1 + 13,
+    GF_DEFRAG_CMD_DETACH_COMMIT = 1 + 14,
+    GF_DEFRAG_CMD_DETACH_COMMIT_FORCE = 1 + 15,
+    GF_DEFRAG_CMD_DETACH_STOP = 1 + 16,
+    /* new labels are used so it will help
+     * while removing old labels by easily differentiating.
+     * A few labels are added so that the count remains same
+     * between this enum and the ones on the xdr file.
+     * different values for the same enum cause errors and
+     * confusion.
+     */
 };
 typedef enum gf_defrag_type gf_defrag_type;
 
 enum gf_defrag_status_t {
-        GF_DEFRAG_STATUS_NOT_STARTED,
-        GF_DEFRAG_STATUS_STARTED,
-        GF_DEFRAG_STATUS_STOPPED,
-        GF_DEFRAG_STATUS_COMPLETE,
-        GF_DEFRAG_STATUS_FAILED,
-        GF_DEFRAG_STATUS_LAYOUT_FIX_STARTED,
-        GF_DEFRAG_STATUS_LAYOUT_FIX_STOPPED,
-        GF_DEFRAG_STATUS_LAYOUT_FIX_COMPLETE,
-        GF_DEFRAG_STATUS_LAYOUT_FIX_FAILED,
+    GF_DEFRAG_STATUS_NOT_STARTED,
+    GF_DEFRAG_STATUS_STARTED,
+    GF_DEFRAG_STATUS_STOPPED,
+    GF_DEFRAG_STATUS_COMPLETE,
+    GF_DEFRAG_STATUS_FAILED,
+    GF_DEFRAG_STATUS_LAYOUT_FIX_STARTED,
+    GF_DEFRAG_STATUS_LAYOUT_FIX_STOPPED,
+    GF_DEFRAG_STATUS_LAYOUT_FIX_COMPLETE,
+    GF_DEFRAG_STATUS_LAYOUT_FIX_FAILED,
 };
 typedef enum gf_defrag_status_t gf_defrag_status_t;
 
 typedef struct gf_defrag_pattern_list gf_defrag_pattern_list_t;
 
 struct gf_defrag_pattern_list {
-        char                       path_pattern[256];
-        uint64_t                   size;
-        gf_defrag_pattern_list_t  *next;
+    char path_pattern[256];
+    uint64_t size;
+    gf_defrag_pattern_list_t *next;
 };
 
+struct dht_container {
+    union {
+        struct list_head list;
+        struct {
+            struct _gf_dirent_t *next;
+            struct _gf_dirent_t *prev;
+        };
+    };
+    gf_dirent_t *df_entry;
+    xlator_t *this;
+    loc_t *parent_loc;
+    dict_t *migrate_data;
+    int local_subvol_index;
+};
+
+typedef struct nodeuuid_info {
+    char info;   /* Set to 1 is this is my node's uuid*/
+    uuid_t uuid; /* Store the nodeuuid as well for debugging*/
+} nodeuuid_info_t;
+
+typedef struct subvol_nodeuuids_info {
+    nodeuuid_info_t *elements;
+    int count;
+} subvol_nodeuuids_info_t;
+
 struct gf_defrag_info_ {
-        uint64_t                     total_files;
-        uint64_t                     total_data;
-        uint64_t                     num_files_lookedup;
-        uint64_t                     total_failures;
-        uint64_t                     skipped;
-        gf_lock_t                    lock;
-        int                          cmd;
-        pthread_t                    th;
-        gf_defrag_status_t           defrag_status;
-        struct rpc_clnt             *rpc;
-        uint32_t                     connected;
-        uint32_t                     is_exiting;
-        pid_t                        pid;
-        inode_t                     *root_inode;
-        uuid_t                       node_uuid;
-        struct timeval               start_time;
-        gf_boolean_t                 stats;
-        gf_defrag_pattern_list_t    *defrag_pattern;
-        int                          tier_promote_frequency;
-        int                          tier_demote_frequency;
-
-        /*Data Tiering params for scanner*/
-        uint64_t                     total_files_promoted;
-        uint64_t                     total_files_demoted;
-        int                          write_freq_threshold;
-        int                          read_freq_threshold;
+    uint64_t total_files;
+    uint64_t total_data;
+    uint64_t num_files_lookedup;
+    uint64_t total_failures;
+    uint64_t skipped;
+    uint64_t num_dirs_processed;
+    uint64_t size_processed;
+    gf_lock_t lock;
+    pthread_t th;
+    struct rpc_clnt *rpc;
+    uint32_t connected;
+    uint32_t is_exiting;
+    pid_t pid;
+    int cmd;
+    inode_t *root_inode;
+    uuid_t node_uuid;
+    time_t start_time;
+    uint32_t new_commit_hash;
+    gf_defrag_status_t defrag_status;
+    gf_defrag_pattern_list_t *defrag_pattern;
+
+    pthread_cond_t parallel_migration_cond;
+    pthread_mutex_t dfq_mutex;
+    pthread_cond_t rebalance_crawler_alarm;
+    int32_t q_entry_count;
+    int32_t global_error;
+    struct dht_container *queue;
+    int32_t crawl_done;
+    int32_t abort;
+    int32_t wakeup_crawler;
+
+    /*Throttle params*/
+    /*stands for reconfigured thread count*/
+    int32_t recon_thread_count;
+    pthread_cond_t df_wakeup_thread;
+
+    /* backpointer to make it easier to write functions for rebalance */
+    xlator_t *this;
+
+    pthread_cond_t fc_wakeup_cond;
+    pthread_mutex_t fc_mutex;
+
+    /*stands for current running thread count*/
+    int32_t current_thread_count;
+
+    gf_boolean_t stats;
+    /* lock migration flag */
+    gf_boolean_t lock_migration_enabled;
 };
 
 typedef struct gf_defrag_info_ gf_defrag_info_t;
 
 struct dht_methods_s {
-        int32_t      (*migration_get_dst_subvol)(xlator_t *this,
-                                                 dht_local_t *local);
-        int32_t      (*migration_other)(xlator_t *this,
-                                        gf_defrag_info_t *defrag);
-        int32_t      (*migration_needed)(xlator_t *this);
-        xlator_t*    (*layout_search)(xlator_t *this,
-                                      dht_layout_t *layout,
-                                         const char *name);
+    int32_t (*migration_get_dst_subvol)(xlator_t *this, dht_local_t *local);
+    int32_t (*migration_other)(xlator_t *this, gf_defrag_info_t *defrag);
+    xlator_t *(*layout_search)(xlator_t *this, dht_layout_t *layout,
+                               const char *name);
 };
 
 typedef struct dht_methods_s dht_methods_t;
 
 struct dht_conf {
-        gf_lock_t      subvolume_lock;
-        int            subvolume_cnt;
-        xlator_t     **subvolumes;
-        char          *subvolume_status;
-        int           *last_event;
-        dht_layout_t **file_layouts;
-        dht_layout_t **dir_layouts;
-        gf_boolean_t   search_unhashed;
-        int            gen;
-        dht_du_t      *du_stats;
-        double         min_free_disk;
-        double         min_free_inodes;
-        char           disk_unit;
-        int32_t        refresh_interval;
-        gf_boolean_t   unhashed_sticky_bit;
-        struct timeval last_stat_fetch;
-        gf_lock_t      layout_lock;
-        dict_t        *leaf_to_subvol;
-        void          *private;     /* Can be used by wrapper xlators over
-                                       dht */
-        gf_boolean_t   use_readdirp;
-        char           vol_uuid[UUID_SIZE + 1];
-        gf_boolean_t   assert_no_child_down;
-        time_t        *subvol_up_time;
-
-        /* This is the count used as the distribute layout for a directory */
-        /* Will be a global flag to control the layout spread count */
-        uint32_t       dir_spread_cnt;
-
-        /* to keep track of nodes which are decommissioned */
-        xlator_t     **decommissioned_bricks;
-        int            decommission_in_progress;
-        int            decommission_subvols_cnt;
-
-        /* defrag related */
-        gf_defrag_info_t *defrag;
-
-        /* Request to filter directory entries in readdir request */
-
-        gf_boolean_t    readdir_optimize;
-
-        /* Support regex-based name reinterpretation. */
-        regex_t         rsync_regex;
-        gf_boolean_t    rsync_regex_valid;
-        regex_t         extra_regex;
-        gf_boolean_t    extra_regex_valid;
-
-        /* Support variable xattr names. */
-        char            *xattr_name;
-        char            *link_xattr_name;
-        char            *wild_xattr_name;
-
-        /* Support size-weighted rebalancing (heterogeneous bricks). */
-        gf_boolean_t    do_weighting;
-        gf_boolean_t    randomize_by_gfid;
-
-        dht_methods_t  *methods;
-
-        struct mem_pool *lock_pool;
+    xlator_t **subvolumes;
+    char *subvolume_status;
+    int *last_event;
+    dht_layout_t **file_layouts;
+    dht_layout_t **dir_layouts;
+    unsigned int search_unhashed;
+    int gen;
+    dht_du_t *du_stats;
+    double min_free_disk;
+    double min_free_inodes;
+    int subvolume_cnt;
+    int32_t refresh_interval;
+    gf_lock_t subvolume_lock;
+    time_t last_stat_fetch;
+    gf_lock_t layout_lock;
+    dict_t *leaf_to_subvol;
+    void *private; /* Can be used by wrapper xlators over
+                      dht */
+    time_t *subvol_up_time;
+
+    /* to keep track of nodes which are decommissioned */
+    xlator_t **decommissioned_bricks;
+    int decommission_in_progress;
+    int decommission_subvols_cnt;
+
+    /* defrag related */
+    gf_defrag_info_t *defrag;
+
+    /* Support regex-based name reinterpretation. */
+    regex_t rsync_regex;
+    regex_t extra_regex;
+
+    /* Support variable xattr names. */
+    char *xattr_name;
+    char *mds_xattr_key;
+    char *link_xattr_name;
+    char *commithash_xattr_name;
+    char *wild_xattr_name;
+
+    dht_methods_t methods;
+
+    struct mem_pool *lock_pool;
+
+    /*local subvol storage for rebalance*/
+    xlator_t **local_subvols;
+    subvol_nodeuuids_info_t *local_nodeuuids;
+    int32_t local_subvols_cnt;
+
+    int dthrottle;
+
+    /* Hard link handle requirement for migration triggered from client*/
+    synclock_t link_lock;
+
+    /* lock migration */
+    gf_lock_t lock;
+
+    /* This is the count used as the distribute layout for a directory */
+    /* Will be a global flag to control the layout spread count */
+    uint32_t dir_spread_cnt;
+
+    /*
+     * "Commit hash" for this volume topology.  Changed whenever bricks
+     * are added or removed.
+     */
+    uint32_t vol_commit_hash;
+
+    char vol_uuid[UUID_SIZE + 1];
+
+    char disk_unit;
+
+    gf_boolean_t lock_migration_enabled;
+
+    gf_boolean_t vch_forced;
+
+    gf_boolean_t use_fallocate;
+
+    gf_boolean_t force_migration;
+
+    gf_boolean_t lookup_optimize;
+
+    gf_boolean_t unhashed_sticky_bit;
+
+    gf_boolean_t assert_no_child_down;
+
+    gf_boolean_t use_readdirp;
+
+    /* Request to filter directory entries in readdir request */
+    gf_boolean_t readdir_optimize;
+
+    gf_boolean_t rsync_regex_valid;
+
+    gf_boolean_t extra_regex_valid;
+
+    /* Support size-weighted rebalancing (heterogeneous bricks). */
+    gf_boolean_t do_weighting;
+
+    gf_boolean_t randomize_by_gfid;
 };
 typedef struct dht_conf dht_conf_t;
 
+struct dht_dfoffset_ctx {
+    xlator_t *this;
+    off_t offset;
+    int32_t readdir_done;
+};
+typedef struct dht_dfoffset_ctx dht_dfoffset_ctx_t;
 
 struct dht_disk_layout {
-        uint32_t           cnt;
-        uint32_t           type;
-        struct {
-                uint32_t   start;
-                uint32_t   stop;
-        } list[1];
+    uint32_t cnt;
+    uint32_t type;
+    struct {
+        uint32_t start;
+        uint32_t stop;
+    } list[1];
 };
 typedef struct dht_disk_layout dht_disk_layout_t;
 
 typedef enum {
-        GF_DHT_MIGRATE_DATA,
-        GF_DHT_MIGRATE_DATA_EVEN_IF_LINK_EXISTS,
-        GF_DHT_MIGRATE_HARDLINK,
-        GF_DHT_MIGRATE_HARDLINK_IN_PROGRESS
+    GF_DHT_MIGRATE_DATA,
+    GF_DHT_MIGRATE_DATA_EVEN_IF_LINK_EXISTS,
+    GF_DHT_MIGRATE_HARDLINK,
+    GF_DHT_MIGRATE_HARDLINK_IN_PROGRESS
 } gf_dht_migrate_data_type_t;
 
 typedef enum {
-        GF_DHT_EQUAL_DISTRIBUTION,
-        GF_DHT_WEIGHTED_DISTRIBUTION
+    GF_DHT_EQUAL_DISTRIBUTION,
+    GF_DHT_WEIGHTED_DISTRIBUTION
 } dht_distribution_type_t;
 
+struct dir_dfmeta {
+    gf_dirent_t *equeue;
+    dht_dfoffset_ctx_t *offset_var;
+    struct list_head **head;
+    struct list_head **iterator;
+    int *fetch_entries;
+    /* fds corresponding to local subvols only */
+    fd_t **lfd;
+};
+
+typedef struct dht_migrate_info {
+    xlator_t *src_subvol;
+    xlator_t *dst_subvol;
+    GF_REF_DECL;
+} dht_migrate_info_t;
+
+typedef struct dht_fd_ctx {
+    uint64_t opened_on_dst;
+    GF_REF_DECL;
+} dht_fd_ctx_t;
+
 #define ENTRY_MISSING(op_ret, op_errno) (op_ret == -1 && op_errno == ENOENT)
 
-#define is_revalidate(loc) (dht_inode_ctx_layout_get (loc->inode, this, NULL) == 0)
+#define is_revalidate(loc)                                                     \
+    (dht_inode_ctx_layout_get((loc)->inode, this, NULL) == 0)
 
 #define is_last_call(cnt) (cnt == 0)
 
 #define DHT_MIGRATION_IN_PROGRESS 1
-#define DHT_MIGRATION_COMPLETED   2
+#define DHT_MIGRATION_COMPLETED 2
 
-#define check_is_linkfile(i,s,x,n) (IS_DHT_LINKFILE_MODE (s) && dict_get (x, n))
+#define check_is_linkfile(i, s, x, n)                                          \
+    (IS_DHT_LINKFILE_MODE(s) && dict_get(x, n))
 
-#define IS_DHT_MIGRATION_PHASE2(buf)  (                                 \
-                IA_ISREG ((buf)->ia_type) &&                            \
-                ((st_mode_from_ia ((buf)->ia_prot, (buf)->ia_type) &    \
-                  ~S_IFMT) == DHT_LINKFILE_MODE))
+#define IS_DHT_MIGRATION_PHASE2(buf)                                           \
+    (IA_ISREG((buf)->ia_type) &&                                               \
+     ((st_mode_from_ia((buf)->ia_prot, (buf)->ia_type) & ~S_IFMT) ==           \
+      DHT_LINKFILE_MODE))
 
-#define IS_DHT_MIGRATION_PHASE1(buf)  (                                 \
-                IA_ISREG ((buf)->ia_type) &&                            \
-                ((buf)->ia_prot.sticky == 1) &&                         \
-                ((buf)->ia_prot.sgid == 1))
+#define IS_DHT_MIGRATION_PHASE1(buf)                                           \
+    (IA_ISREG((buf)->ia_type) && ((buf)->ia_prot.sticky == 1) &&               \
+     ((buf)->ia_prot.sgid == 1))
 
-#define DHT_STRIP_PHASE1_FLAGS(buf)  do {                       \
-                if ((buf) && IS_DHT_MIGRATION_PHASE1(buf)) {    \
-                        (buf)->ia_prot.sticky = 0;              \
-                        (buf)->ia_prot.sgid = 0;                \
-                }                                               \
-        } while (0)
+#define DHT_STRIP_PHASE1_FLAGS(buf)                                            \
+    do {                                                                       \
+        if ((buf) && IS_DHT_MIGRATION_PHASE1(buf)) {                           \
+            (buf)->ia_prot.sticky = 0;                                         \
+            (buf)->ia_prot.sgid = 0;                                           \
+        }                                                                      \
+    } while (0)
 
 #define dht_inode_missing(op_errno) (op_errno == ENOENT || op_errno == ESTALE)
 
-#define check_is_dir(i,s,x) (IA_ISDIR(s->ia_type))
+#define check_is_dir(i, s, x) (IA_ISDIR(s->ia_type))
 
 #define layout_is_sane(layout) ((layout) && (layout->cnt > 0))
 
-#define DHT_STACK_UNWIND(fop, frame, params ...) do {           \
-                dht_local_t *__local = NULL;                    \
-                xlator_t    *__xl    = NULL;                    \
-                if (frame) {                                    \
-                        __xl         = frame->this;             \
-                        __local      = frame->local;            \
-                        frame->local = NULL;                    \
-                }                                               \
-                STACK_UNWIND_STRICT (fop, frame, params);       \
-                dht_local_wipe (__xl, __local);                 \
-        } while (0)
-
-#define DHT_STACK_DESTROY(frame) do {                   \
-                dht_local_t *__local = NULL;            \
-                xlator_t    *__xl    = NULL;            \
-                __xl                 = frame->this;     \
-                __local              = frame->local;    \
-                frame->local         = NULL;            \
-                STACK_DESTROY (frame->root);            \
-                dht_local_wipe (__xl, __local);         \
-        } while (0)
-
-#define DHT_UPDATE_TIME(ctx_sec, ctx_nsec, new_sec, new_nsec, inode, post) do {\
-                LOCK (&inode->lock);                                    \
-                {                                                       \
-                        if (ctx_sec == new_sec)                         \
-                                new_nsec = max (new_nsec, ctx_nsec);    \
-                        else if (ctx_sec > new_sec) {                   \
-                                new_sec = ctx_sec;                      \
-                                new_nsec = ctx_nsec;                    \
-                        }                                               \
-                        if (post) {                                     \
-                                ctx_sec = new_sec;                      \
-                                ctx_nsec = new_nsec;                    \
-                        }                                               \
-                }                                                       \
-                UNLOCK (&inode->lock);                                  \
-        } while (0)
-
-#define is_greater_time(a, an, b, bn) (((a) < (b)) || (((a) == (b)) && ((an) < (bn))))
-dht_layout_t                            *dht_layout_new (xlator_t *this, int cnt);
-dht_layout_t                            *dht_layout_get (xlator_t *this, inode_t *inode);
-dht_layout_t                            *dht_layout_for_subvol (xlator_t *this, xlator_t *subvol);
-xlator_t *dht_layout_search (xlator_t   *this, dht_layout_t *layout,
-                             const char *name);
-int32_t
-dht_migration_get_dst_subvol(xlator_t *this, dht_local_t  *local);
+#define we_are_not_migrating(x) ((x) == 1)
+
+#define DHT_STACK_UNWIND(fop, frame, params...)                                \
+    do {                                                                       \
+        dht_local_t *__local = NULL;                                           \
+        xlator_t *__xl = NULL;                                                 \
+        if (frame) {                                                           \
+            __xl = frame->this;                                                \
+            __local = frame->local;                                            \
+            frame->local = NULL;                                               \
+        }                                                                      \
+        STACK_UNWIND_STRICT(fop, frame, params);                               \
+        dht_local_wipe(__xl, __local);                                         \
+    } while (0)
+
+#define DHT_STACK_DESTROY(frame)                                               \
+    do {                                                                       \
+        dht_local_t *__local = NULL;                                           \
+        xlator_t *__xl = NULL;                                                 \
+        __xl = frame->this;                                                    \
+        __local = frame->local;                                                \
+        frame->local = NULL;                                                   \
+        STACK_DESTROY(frame->root);                                            \
+        dht_local_wipe(__xl, __local);                                         \
+    } while (0)
+
+#define DHT_UPDATE_TIME(ctx_sec, ctx_nsec, new_sec, new_nsec, post)            \
+    do {                                                                       \
+        if (ctx_sec == new_sec)                                                \
+            new_nsec = max(new_nsec, ctx_nsec);                                \
+        else if (ctx_sec > new_sec) {                                          \
+            new_sec = ctx_sec;                                                 \
+            new_nsec = ctx_nsec;                                               \
+        }                                                                      \
+        if (post) {                                                            \
+            ctx_sec = new_sec;                                                 \
+            ctx_nsec = new_nsec;                                               \
+        }                                                                      \
+    } while (0)
+
+#define is_greater_time(a, an, b, bn)                                          \
+    (((a) < (b)) || (((a) == (b)) && ((an) < (bn))))
+
+#define DHT_MARK_FOP_INTERNAL(xattr)                                           \
+    do {                                                                       \
+        int tmp = -1;                                                          \
+        if (!xattr) {                                                          \
+            xattr = dict_new();                                                \
+            if (!xattr)                                                        \
+                break;                                                         \
+        }                                                                      \
+        tmp = dict_set_str(xattr, GLUSTERFS_INTERNAL_FOP_KEY, "yes");          \
+        if (tmp) {                                                             \
+            gf_msg(this->name, GF_LOG_ERROR, 0, DHT_MSG_DICT_SET_FAILED,       \
+                   "Failed to set dictionary value: key = %s,"                 \
+                   " path = %s",                                               \
+                   GLUSTERFS_INTERNAL_FOP_KEY, local->loc.path);               \
+        }                                                                      \
+    } while (0)
+
+dht_layout_t *
+dht_layout_new(xlator_t *this, int cnt);
+dht_layout_t *
+dht_layout_get(xlator_t *this, inode_t *inode);
+dht_layout_t *
+dht_layout_for_subvol(xlator_t *this, xlator_t *subvol);
+xlator_t *
+dht_layout_search(xlator_t *this, dht_layout_t *layout, const char *name);
+int32_t
+dht_migration_get_dst_subvol(xlator_t *this, dht_local_t *local);
 int32_t
 dht_migration_needed(xlator_t *this);
-int                                      dht_layout_normalize (xlator_t *this, loc_t *loc, dht_layout_t *layout);
-int dht_layout_anomalies (xlator_t      *this, loc_t *loc, dht_layout_t *layout,
-                          uint32_t      *holes_p, uint32_t *overlaps_p,
-                          uint32_t      *missing_p, uint32_t *down_p,
-                          uint32_t      *misc_p, uint32_t *no_space_p);
-int dht_layout_dir_mismatch (xlator_t   *this, dht_layout_t *layout,
-                             xlator_t   *subvol, loc_t *loc, dict_t *xattr);
+int
+dht_layout_normalize(xlator_t *this, loc_t *loc, dht_layout_t *layout);
+void
+dht_layout_anomalies(xlator_t *this, loc_t *loc, dht_layout_t *layout,
+                     uint32_t *holes_p, uint32_t *overlaps_p,
+                     uint32_t *missing_p, uint32_t *down_p, uint32_t *misc_p,
+                     uint32_t *no_space_p);
+int
+dht_layout_dir_mismatch(xlator_t *this, dht_layout_t *layout, xlator_t *subvol,
+                        loc_t *loc, dict_t *xattr);
+xlator_t *
+dht_linkfile_subvol(xlator_t *this, inode_t *inode, struct iatt *buf,
+                    dict_t *xattr);
+int
+dht_linkfile_unlink(call_frame_t *frame, xlator_t *this, xlator_t *subvol,
+                    loc_t *loc);
+
+int
+dht_layouts_init(xlator_t *this, dht_conf_t *conf);
+int
+dht_layout_merge(xlator_t *this, dht_layout_t *layout, xlator_t *subvol,
+                 int op_ret, int op_errno, dict_t *xattr);
+
+int
+dht_disk_layout_extract(xlator_t *this, dht_layout_t *layout, int pos,
+                        int32_t **disk_layout_p);
+int
+dht_disk_layout_extract_for_subvol(xlator_t *this, dht_layout_t *layout,
+                                   xlator_t *subvol, int32_t **disk_layout_p);
+
+int
+dht_frame_return(call_frame_t *frame);
+
+int
+dht_deitransform(xlator_t *this, uint64_t y, xlator_t **subvol);
+
+void
+dht_local_wipe(xlator_t *this, dht_local_t *local);
+dht_local_t *
+dht_local_init(call_frame_t *frame, loc_t *loc, fd_t *fd, glusterfs_fop_t fop);
+int
+dht_iatt_merge(xlator_t *this, struct iatt *to, struct iatt *from);
+
+xlator_t *
+dht_subvol_get_hashed(xlator_t *this, loc_t *loc);
+xlator_t *
+dht_subvol_get_cached(xlator_t *this, inode_t *inode);
+xlator_t *
+dht_subvol_next(xlator_t *this, xlator_t *prev);
+xlator_t *
+dht_subvol_next_available(xlator_t *this, xlator_t *prev);
+int
+dht_subvol_cnt(xlator_t *this, xlator_t *subvol);
 
-xlator_t *dht_linkfile_subvol (xlator_t *this, inode_t *inode,
-                               struct iatt *buf, dict_t *xattr);
-int dht_linkfile_unlink (call_frame_t      *frame, xlator_t *this,
-                         xlator_t          *subvol, loc_t *loc);
+int
+dht_hash_compute(xlator_t *this, int type, const char *name, uint32_t *hash_p);
 
-int dht_layouts_init (xlator_t *this, dht_conf_t *conf);
-int dht_layout_merge (xlator_t *this, dht_layout_t *layout, xlator_t *subvol,
-                      int       op_ret, int op_errno, dict_t *xattr);
+int
+dht_linkfile_create(call_frame_t *frame, fop_mknod_cbk_t linkfile_cbk,
+                    xlator_t *this, xlator_t *tovol, xlator_t *fromvol,
+                    loc_t *loc);
+int
+dht_lookup_everywhere(call_frame_t *frame, xlator_t *this, loc_t *loc);
+int
+dht_selfheal_directory(call_frame_t *frame, dht_selfheal_dir_cbk_t cbk,
+                       loc_t *loc, dht_layout_t *layout);
+int
+dht_selfheal_new_directory(call_frame_t *frame, dht_selfheal_dir_cbk_t cbk,
+                           dht_layout_t *layout);
+int
+dht_selfheal_restore(call_frame_t *frame, dht_selfheal_dir_cbk_t cbk,
+                     loc_t *loc, dht_layout_t *layout);
+void
+dht_layout_sort_volname(dht_layout_t *layout);
 
-int dht_disk_layout_extract (xlator_t *this, dht_layout_t *layout,
-                             int       pos, int32_t **disk_layout_p);
-int dht_disk_layout_merge (xlator_t   *this, dht_layout_t *layout,
-                           int         pos, void *disk_layout_raw, int disk_layout_len);
+int
+dht_get_du_info(call_frame_t *frame, xlator_t *this, loc_t *loc);
 
+gf_boolean_t
+dht_is_subvol_filled(xlator_t *this, xlator_t *subvol);
+xlator_t *
+dht_free_disk_available_subvol(xlator_t *this, xlator_t *subvol,
+                               dht_local_t *layout);
+int
+dht_get_du_info_for_subvol(xlator_t *this, int subvol_idx);
 
-int dht_frame_return (call_frame_t *frame);
+int
+dht_layout_preset(xlator_t *this, xlator_t *subvol, inode_t *inode);
+int
+dht_layout_set(xlator_t *this, inode_t *inode, dht_layout_t *layout);
+;
+void
+dht_layout_unref(xlator_t *this, dht_layout_t *layout);
+dht_layout_t *
+dht_layout_ref(xlator_t *this, dht_layout_t *layout);
+int
+dht_layout_index_for_subvol(dht_layout_t *layout, xlator_t *subvol);
+xlator_t *
+dht_first_up_subvol(xlator_t *this);
+xlator_t *
+dht_last_up_subvol(xlator_t *this);
 
-int dht_deitransform (xlator_t *this, uint64_t y, xlator_t **subvol);
+int
+dht_build_child_loc(xlator_t *this, loc_t *child, loc_t *parent, char *name);
 
-void dht_local_wipe (xlator_t *this, dht_local_t *local);
-dht_local_t *dht_local_init (call_frame_t    *frame, loc_t *loc, fd_t *fd,
-                             glusterfs_fop_t  fop);
-int dht_iatt_merge (xlator_t                 *this, struct iatt *to, struct iatt *from,
-                    xlator_t                 *subvol);
+int
+dht_filter_loc_subvol_key(xlator_t *this, loc_t *loc, loc_t *new_loc,
+                          xlator_t **subvol);
 
-xlator_t *dht_subvol_get_hashed (xlator_t *this, loc_t *loc);
-xlator_t *dht_subvol_get_cached (xlator_t *this, inode_t *inode);
-xlator_t *dht_subvol_next (xlator_t *this, xlator_t *prev);
-xlator_t *dht_subvol_next_available (xlator_t *this, xlator_t *prev);
-int       dht_subvol_cnt (xlator_t *this, xlator_t *subvol);
+int
+dht_rename_cleanup(call_frame_t *frame);
+int
+dht_rename_link_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                    int32_t op_ret, int32_t op_errno, inode_t *inode,
+                    struct iatt *stbuf, struct iatt *preparent,
+                    struct iatt *postparent, dict_t *xdata);
 
-int dht_hash_compute (xlator_t *this, int type, const char *name, uint32_t *hash_p);
+int
+dht_update_commit_hash_for_layout(call_frame_t *frame);
+int
+dht_fix_directory_layout(call_frame_t *frame, dht_selfheal_dir_cbk_t dir_cbk,
+                         dht_layout_t *layout);
 
-int dht_linkfile_create (call_frame_t    *frame, fop_mknod_cbk_t linkfile_cbk,
-                         xlator_t        *this, xlator_t *tovol,
-                         xlator_t        *fromvol, loc_t *loc);
-int                                       dht_lookup_directory (call_frame_t *frame, xlator_t *this, loc_t *loc);
-int                                       dht_lookup_everywhere (call_frame_t *frame, xlator_t *this, loc_t *loc);
 int
-dht_selfheal_directory (call_frame_t     *frame, dht_selfheal_dir_cbk_t cbk,
-                        loc_t            *loc, dht_layout_t *layout);
+dht_init_subvolumes(xlator_t *this, dht_conf_t *conf);
 
+/* migration/rebalance */
 int
-dht_selfheal_directory_for_nameless_lookup (call_frame_t  *frame,
-                                            dht_selfheal_dir_cbk_t cbk,
-                                            loc_t  *loc, dht_layout_t *layout);
+dht_start_rebalance_task(xlator_t *this, call_frame_t *frame);
 
 int
-dht_selfheal_new_directory (call_frame_t *frame, dht_selfheal_dir_cbk_t cbk,
-                            dht_layout_t *layout);
+dht_rebalance_in_progress_check(xlator_t *this, call_frame_t *frame);
 int
-dht_selfheal_restore (call_frame_t       *frame, dht_selfheal_dir_cbk_t cbk,
-                      loc_t              *loc, dht_layout_t *layout);
+dht_rebalance_complete_check(xlator_t *this, call_frame_t *frame);
+
 int
-dht_layout_sort_volname (dht_layout_t *layout);
+dht_init_local_subvolumes(xlator_t *this, dht_conf_t *conf);
 
-int dht_get_du_info (call_frame_t *frame, xlator_t *this, loc_t *loc);
+/* FOPS */
+int32_t
+dht_lookup(call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xattr_req);
 
-gf_boolean_t dht_is_subvol_filled (xlator_t *this, xlator_t *subvol);
-xlator_t *dht_free_disk_available_subvol (xlator_t *this, xlator_t *subvol,
-                                          dht_local_t *layout);
-int       dht_get_du_info_for_subvol (xlator_t *this, int subvol_idx);
+int32_t
+dht_stat(call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xdata);
 
-int dht_layout_preset (xlator_t *this, xlator_t *subvol, inode_t *inode);
-int           dht_layout_set (xlator_t *this, inode_t *inode, dht_layout_t *layout);;
-void          dht_layout_unref (xlator_t *this, dht_layout_t *layout);
-dht_layout_t *dht_layout_ref (xlator_t *this, dht_layout_t *layout);
-xlator_t     *dht_first_up_subvol (xlator_t *this);
-xlator_t     *dht_last_up_subvol (xlator_t *this);
+int32_t
+dht_fstat(call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *xdata);
 
-int dht_build_child_loc (xlator_t *this, loc_t *child, loc_t *parent, char *name);
+int32_t
+dht_truncate(call_frame_t *frame, xlator_t *this, loc_t *loc, off_t offset,
+             dict_t *xdata);
 
-int dht_filter_loc_subvol_key (xlator_t  *this, loc_t *loc, loc_t *new_loc,
-                               xlator_t **subvol);
+int32_t
+dht_ftruncate(call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
+              dict_t *xdata);
 
-int                                     dht_rename_cleanup (call_frame_t *frame);
-int dht_rename_link_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                         int32_t           op_ret, int32_t op_errno,
-                         inode_t          *inode, struct iatt *stbuf,
-                         struct iatt      *preparent, struct iatt *postparent,
-                         dict_t *xdata);
+int32_t
+dht_access(call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t mask,
+           dict_t *xdata);
 
-int dht_fix_directory_layout (call_frame_t *frame,
-                              dht_selfheal_dir_cbk_t  dir_cbk,
-                              dht_layout_t           *layout);
+int32_t
+dht_readlink(call_frame_t *frame, xlator_t *this, loc_t *loc, size_t size,
+             dict_t *xdata);
 
-int dht_init_subvolumes (xlator_t *this, dht_conf_t *conf);
+int32_t
+dht_mknod(call_frame_t *frame, xlator_t *this, loc_t *loc, mode_t mode,
+          dev_t rdev, mode_t umask, dict_t *xdata);
 
-/* migration/rebalance */
-int dht_start_rebalance_task (xlator_t *this, call_frame_t *frame);
+int32_t
+dht_mkdir(call_frame_t *frame, xlator_t *this, loc_t *loc, mode_t mode,
+          mode_t umask, dict_t *xdata);
 
-int dht_rebalance_in_progress_check (xlator_t *this, call_frame_t *frame);
-int dht_rebalance_complete_check (xlator_t *this, call_frame_t *frame);
+int32_t
+dht_unlink(call_frame_t *frame, xlator_t *this, loc_t *loc, int xflag,
+           dict_t *xdata);
 
+int32_t
+dht_rmdir(call_frame_t *frame, xlator_t *this, loc_t *loc, int flags,
+          dict_t *xdata);
 
-/* FOPS */
-int32_t dht_lookup (call_frame_t *frame,
-                    xlator_t *this,
-                    loc_t    *loc,
-                    dict_t   *xattr_req);
-
-int32_t dht_stat (call_frame_t *frame,
-                  xlator_t *this,
-                  loc_t    *loc, dict_t *xdata);
-
-int32_t dht_fstat (call_frame_t *frame,
-                   xlator_t *this,
-                   fd_t     *fd, dict_t *xdata);
-
-int32_t dht_truncate (call_frame_t *frame,
-                      xlator_t *this,
-                      loc_t    *loc,
-                      off_t     offset, dict_t *xdata);
-
-int32_t dht_ftruncate (call_frame_t *frame,
-                       xlator_t *this,
-                       fd_t     *fd,
-                       off_t     offset, dict_t *xdata);
-
-int32_t dht_access (call_frame_t *frame,
-                    xlator_t *this,
-                    loc_t    *loc,
-                    int32_t   mask, dict_t *xdata);
-
-int32_t dht_readlink (call_frame_t *frame,
-                      xlator_t *this,
-                      loc_t    *loc,
-                      size_t    size, dict_t *xdata);
-
-int32_t dht_mknod (call_frame_t *frame, xlator_t *this, loc_t *loc,
-                   mode_t mode, dev_t rdev, mode_t umask, dict_t *xdata);
-
-int32_t dht_mkdir (call_frame_t *frame, xlator_t *this,
-                   loc_t *loc, mode_t mode, mode_t umask, dict_t *xdata);
-
-int32_t dht_unlink (call_frame_t *frame,
-                    xlator_t *this,
-                    loc_t    *loc, int xflag, dict_t *xdata);
-
-int32_t dht_rmdir (call_frame_t *frame, xlator_t *this,
-                   loc_t    *loc, int flags, dict_t *xdata);
-
-int32_t dht_symlink (call_frame_t   *frame, xlator_t *this,
-                     const char *linkpath, loc_t *loc, mode_t umask,
-                     dict_t *xdata);
-
-int32_t dht_rename (call_frame_t *frame,
-                    xlator_t *this,
-                    loc_t    *oldloc,
-                    loc_t    *newloc, dict_t *xdata);
-
-int32_t dht_link (call_frame_t *frame,
-                  xlator_t *this,
-                  loc_t    *oldloc,
-                  loc_t    *newloc, dict_t *xdata);
-
-int32_t dht_create (call_frame_t *frame, xlator_t *this,
-                    loc_t    *loc, int32_t flags, mode_t mode,
-                    mode_t umask, fd_t     *fd, dict_t *params);
-
-int32_t dht_open (call_frame_t *frame,
-                  xlator_t *this,
-                  loc_t    *loc,
-                  int32_t   flags, fd_t *fd, dict_t *xdata);
-
-int32_t dht_readv (call_frame_t *frame,
-                   xlator_t *this,
-                   fd_t     *fd,
-                   size_t    size,
-                   off_t     offset, uint32_t flags, dict_t *xdata);
-
-int32_t dht_writev (call_frame_t      *frame,
-                    xlator_t      *this,
-                    fd_t          *fd,
-                    struct iovec  *vector,
-                    int32_t        count,
-                    off_t          offset,
-                    uint32_t       flags,
-                    struct iobref *iobref, dict_t *xdata);
-
-int32_t dht_flush (call_frame_t *frame,
-                   xlator_t *this,
-                   fd_t     *fd, dict_t *xdata);
-
-int32_t dht_fsync (call_frame_t *frame,
-                   xlator_t *this,
-                   fd_t     *fd,
-                   int32_t   datasync, dict_t *xdata);
-
-int32_t dht_opendir (call_frame_t *frame,
-                     xlator_t *this,
-                     loc_t    *loc, fd_t *fd, dict_t *xdata);
-
-int32_t dht_fsyncdir (call_frame_t *frame,
-                      xlator_t *this,
-                      fd_t     *fd,
-                      int32_t   datasync, dict_t *xdata);
-
-int32_t dht_statfs (call_frame_t *frame,
-                    xlator_t *this,
-                    loc_t    *loc, dict_t *xdata);
-
-int32_t dht_setxattr (call_frame_t *frame,
-                      xlator_t *this,
-                      loc_t    *loc,
-                      dict_t   *dict,
-                      int32_t   flags, dict_t *xdata);
-
-int32_t dht_getxattr (call_frame_t   *frame,
-                      xlator_t   *this,
-                      loc_t      *loc,
-                      const char *name, dict_t *xdata);
-
-int32_t dht_fsetxattr (call_frame_t *frame,
-                       xlator_t *this,
-                       fd_t     *fd,
-                       dict_t   *dict,
-                       int32_t   flags, dict_t *xdata);
-
-int32_t dht_fgetxattr (call_frame_t   *frame,
-                       xlator_t   *this,
-                       fd_t       *fd,
-                       const char *name, dict_t *xdata);
-
-int32_t dht_removexattr (call_frame_t   *frame,
-                         xlator_t   *this,
-                         loc_t      *loc,
-                         const char *name, dict_t *xdata);
-int32_t dht_fremovexattr (call_frame_t   *frame,
-                          xlator_t   *this,
-                          fd_t      *fd,
-                          const char *name, dict_t *xdata);
-
-int32_t dht_lk (call_frame_t        *frame,
-                xlator_t        *this,
-                fd_t            *fd,
-                int32_t          cmd,
-                struct gf_flock *flock, dict_t *xdata);
-
-int32_t dht_inodelk (call_frame_t *frame, xlator_t *this,
-                     const char      *volume, loc_t *loc, int32_t cmd,
-                     struct gf_flock *flock, dict_t *xdata);
-
-int32_t dht_finodelk (call_frame_t        *frame, xlator_t *this,
-                      const char      *volume, fd_t *fd, int32_t cmd,
-                      struct gf_flock *flock, dict_t *xdata);
-
-int32_t dht_entrylk (call_frame_t    *frame, xlator_t *this,
-                     const char  *volume, loc_t *loc, const char *basename,
-                     entrylk_cmd  cmd, entrylk_type type, dict_t *xdata);
-
-int32_t dht_fentrylk (call_frame_t    *frame, xlator_t *this,
-                      const char  *volume, fd_t *fd, const char *basename,
-                      entrylk_cmd  cmd, entrylk_type type, dict_t *xdata);
-
-int32_t dht_readdir (call_frame_t  *frame,
-                     xlator_t *this,
-                     fd_t     *fd,
-                     size_t    size, off_t off, dict_t *xdata);
-
-int32_t dht_readdirp (call_frame_t *frame,
-                      xlator_t *this,
-                      fd_t     *fd,
-                      size_t    size, off_t off, dict_t *dict);
-
-int32_t dht_xattrop (call_frame_t           *frame,
-                     xlator_t           *this,
-                     loc_t              *loc,
-                     gf_xattrop_flags_t  flags,
-                     dict_t             *dict, dict_t *xdata);
-
-int32_t dht_fxattrop (call_frame_t *frame,
-                      xlator_t           *this,
-                      fd_t               *fd,
-                      gf_xattrop_flags_t  flags,
-                      dict_t             *dict, dict_t *xdata);
-
-int32_t dht_forget (xlator_t *this, inode_t *inode);
-int32_t dht_setattr (call_frame_t  *frame, xlator_t *this, loc_t *loc,
-                     struct iatt   *stbuf, int32_t valid, dict_t *xdata);
-int32_t dht_fsetattr (call_frame_t *frame, xlator_t *this, fd_t *fd,
-                      struct iatt  *stbuf, int32_t valid, dict_t *xdata);
-int32_t dht_fallocate(call_frame_t *frame, xlator_t *this, fd_t *fd,
-		      int32_t mode, off_t offset, size_t len, dict_t *xdata);
-int32_t dht_discard(call_frame_t *frame, xlator_t *this, fd_t *fd,
-		    off_t offset, size_t len, dict_t *xdata);
-int32_t dht_zerofill(call_frame_t *frame, xlator_t *this, fd_t *fd,
-                    off_t offset, off_t len, dict_t *xdata);
+int32_t
+dht_symlink(call_frame_t *frame, xlator_t *this, const char *linkpath,
+            loc_t *loc, mode_t umask, dict_t *xdata);
+
+int32_t
+dht_rename(call_frame_t *frame, xlator_t *this, loc_t *oldloc, loc_t *newloc,
+           dict_t *xdata);
+
+int32_t
+dht_link(call_frame_t *frame, xlator_t *this, loc_t *oldloc, loc_t *newloc,
+         dict_t *xdata);
+
+int32_t
+dht_create(call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t flags,
+           mode_t mode, mode_t umask, fd_t *fd, dict_t *params);
+
+int32_t
+dht_open(call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t flags,
+         fd_t *fd, dict_t *xdata);
+
+int32_t
+dht_readv(call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
+          off_t offset, uint32_t flags, dict_t *xdata);
+
+int32_t
+dht_writev(call_frame_t *frame, xlator_t *this, fd_t *fd, struct iovec *vector,
+           int32_t count, off_t offset, uint32_t flags, struct iobref *iobref,
+           dict_t *xdata);
+
+int32_t
+dht_flush(call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *xdata);
+
+int32_t
+dht_fsync(call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t datasync,
+          dict_t *xdata);
+
+int32_t
+dht_opendir(call_frame_t *frame, xlator_t *this, loc_t *loc, fd_t *fd,
+            dict_t *xdata);
+
+int32_t
+dht_fsyncdir(call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t datasync,
+             dict_t *xdata);
+
+int32_t
+dht_statfs(call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xdata);
+
+int32_t
+dht_setxattr(call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *dict,
+             int32_t flags, dict_t *xdata);
+
+int32_t
+dht_getxattr(call_frame_t *frame, xlator_t *this, loc_t *loc, const char *name,
+             dict_t *xdata);
+
+int32_t
+dht_fsetxattr(call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *dict,
+              int32_t flags, dict_t *xdata);
+
+int32_t
+dht_fgetxattr(call_frame_t *frame, xlator_t *this, fd_t *fd, const char *name,
+              dict_t *xdata);
+
+int32_t
+dht_removexattr(call_frame_t *frame, xlator_t *this, loc_t *loc,
+                const char *name, dict_t *xdata);
+int32_t
+dht_fremovexattr(call_frame_t *frame, xlator_t *this, fd_t *fd,
+                 const char *name, dict_t *xdata);
+
+int32_t
+dht_lk(call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t cmd,
+       struct gf_flock *flock, dict_t *xdata);
+
+int32_t
+dht_lease(call_frame_t *frame, xlator_t *this, loc_t *loc,
+          struct gf_lease *lease, dict_t *xdata);
+
+int32_t
+dht_inodelk(call_frame_t *frame, xlator_t *this, const char *volume, loc_t *loc,
+            int32_t cmd, struct gf_flock *flock, dict_t *xdata);
+
+int32_t
+dht_finodelk(call_frame_t *frame, xlator_t *this, const char *volume, fd_t *fd,
+             int32_t cmd, struct gf_flock *flock, dict_t *xdata);
+
+int32_t
+dht_entrylk(call_frame_t *frame, xlator_t *this, const char *volume, loc_t *loc,
+            const char *basename, entrylk_cmd cmd, entrylk_type type,
+            dict_t *xdata);
+
+int32_t
+dht_fentrylk(call_frame_t *frame, xlator_t *this, const char *volume, fd_t *fd,
+             const char *basename, entrylk_cmd cmd, entrylk_type type,
+             dict_t *xdata);
+
+int32_t
+dht_readdir(call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
+            off_t off, dict_t *xdata);
+
+int32_t
+dht_readdirp(call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
+             off_t off, dict_t *dict);
+
+int32_t
+dht_xattrop(call_frame_t *frame, xlator_t *this, loc_t *loc,
+            gf_xattrop_flags_t flags, dict_t *dict, dict_t *xdata);
+
+int32_t
+dht_fxattrop(call_frame_t *frame, xlator_t *this, fd_t *fd,
+             gf_xattrop_flags_t flags, dict_t *dict, dict_t *xdata);
+
+int32_t
+dht_forget(xlator_t *this, inode_t *inode);
+int32_t
+dht_setattr(call_frame_t *frame, xlator_t *this, loc_t *loc, struct iatt *stbuf,
+            int32_t valid, dict_t *xdata);
+int32_t
+dht_fsetattr(call_frame_t *frame, xlator_t *this, fd_t *fd, struct iatt *stbuf,
+             int32_t valid, dict_t *xdata);
+int32_t
+dht_fallocate(call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t mode,
+              off_t offset, size_t len, dict_t *xdata);
+int32_t
+dht_discard(call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
+            size_t len, dict_t *xdata);
+int32_t
+dht_zerofill(call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
+             off_t len, dict_t *xdata);
+int32_t
+dht_ipc(call_frame_t *frame, xlator_t *this, int32_t op, dict_t *xdata);
 
 int
 dht_set_subvol_range(xlator_t *this);
-int32_t dht_init (xlator_t *this);
-void    dht_fini (xlator_t *this);
-int     dht_reconfigure (xlator_t *this, dict_t *options);
-int32_t dht_notify (xlator_t *this, int32_t event, void *data, ...);
+int32_t
+dht_init(xlator_t *this);
+void
+dht_fini(xlator_t *this);
+int
+dht_reconfigure(xlator_t *this, dict_t *options);
+int32_t
+dht_notify(xlator_t *this, int32_t event, void *data, ...);
 
 /* definitions for nufa/switch */
-int dht_revalidate_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                        int op_ret, int op_errno, inode_t *inode,
-                        struct iatt *stbuf, dict_t *xattr,
-                        struct iatt *postparent);
-int dht_lookup_dir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+int
+dht_revalidate_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                   int op_ret, int op_errno, inode_t *inode, struct iatt *stbuf,
+                   dict_t *xattr, struct iatt *postparent);
+int
+dht_lookup_dir_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                   int op_ret, int op_errno, inode_t *inode, struct iatt *stbuf,
+                   dict_t *xattr, struct iatt *postparent);
+int
+dht_lookup_linkfile_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
                         int op_ret, int op_errno, inode_t *inode,
                         struct iatt *stbuf, dict_t *xattr,
                         struct iatt *postparent);
-int dht_lookup_linkfile_cbk (call_frame_t *frame, void *cookie,
-                             xlator_t *this, int op_ret, int op_errno,
-                             inode_t *inode, struct iatt *stbuf, dict_t *xattr,
-                             struct iatt *postparent);
-int dht_lookup_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                    int op_ret, int op_errno,
-                    inode_t *inode, struct iatt *stbuf, dict_t *xattr,
-                    struct iatt *postparent);
-int dht_create_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                    int op_ret, int op_errno,
-                    fd_t *fd, inode_t *inode, struct iatt *stbuf,
-                    struct iatt *preparent, struct iatt *postparent,
-                    dict_t *xdata);
-int dht_newfile_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                     int op_ret, int op_errno,
-                     inode_t *inode, struct iatt *stbuf, struct iatt *preparent,
-                     struct iatt *postparent, dict_t *xdata);
+int
+dht_lookup_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int op_ret,
+               int op_errno, inode_t *inode, struct iatt *stbuf, dict_t *xattr,
+               struct iatt *postparent);
+int
+dht_create_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int op_ret,
+               int op_errno, fd_t *fd, inode_t *inode, struct iatt *stbuf,
+               struct iatt *preparent, struct iatt *postparent, dict_t *xdata);
+int
+dht_newfile_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int op_ret,
+                int op_errno, inode_t *inode, struct iatt *stbuf,
+                struct iatt *preparent, struct iatt *postparent, dict_t *xdata);
+
+int
+dht_finodelk_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                 int32_t op_ret, int32_t op_errno, dict_t *xdata);
+
+int
+dht_getxattr_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int op_ret,
+                 int op_errno, dict_t *xattr, dict_t *xdata);
 
 int
-gf_defrag_status_get (gf_defrag_info_t *defrag, dict_t *dict);
+dht_common_xattrop_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                       int32_t op_ret, int32_t op_errno, dict_t *dict,
+                       dict_t *xdata);
+int
+gf_defrag_status_get(dht_conf_t *conf, dict_t *dict);
 
 int
-gf_defrag_stop (gf_defrag_info_t *defrag, gf_defrag_status_t status,
-                dict_t *output);
+gf_defrag_stop(dht_conf_t *conf, gf_defrag_status_t status, dict_t *output);
 
-void*
-gf_defrag_start (void *this);
+void *
+gf_defrag_start(void *this);
 
 int32_t
-gf_defrag_handle_hardlink (xlator_t *this, loc_t *loc, dict_t  *xattrs,
-                           struct iatt *stbuf);
+gf_defrag_handle_hardlink(xlator_t *this, loc_t *loc, int *fop_errno);
 int
-dht_migrate_file (xlator_t *this, loc_t *loc, xlator_t *from, xlator_t *to,
-                 int flag);
+dht_migrate_file(xlator_t *this, loc_t *loc, xlator_t *from, xlator_t *to,
+                 int flag, int *fop_errno);
 int
-dht_inode_ctx_layout_get (inode_t *inode, xlator_t *this,
-                          dht_layout_t **layout_int);
+dht_inode_ctx_layout_get(inode_t *inode, xlator_t *this,
+                         dht_layout_t **layout_int);
 int
-dht_inode_ctx_layout_set (inode_t *inode, xlator_t *this,
-                          dht_layout_t* layout_int);
+dht_inode_ctx_layout_set(inode_t *inode, xlator_t *this,
+                         dht_layout_t *layout_int);
 int
-dht_inode_ctx_time_update (inode_t *inode, xlator_t *this, struct iatt *stat,
-                           int32_t update_ctx);
-void dht_inode_ctx_time_set (inode_t *inode, xlator_t *this, struct iatt *stat);
+dht_inode_ctx_time_update(inode_t *inode, xlator_t *this, struct iatt *stat,
+                          int32_t update_ctx);
+void
+dht_inode_ctx_time_set(inode_t *inode, xlator_t *this, struct iatt *stat);
 
-int dht_inode_ctx_get (inode_t *inode, xlator_t *this, dht_inode_ctx_t **ctx);
-int dht_inode_ctx_set (inode_t *inode, xlator_t *this, dht_inode_ctx_t *ctx);
 int
-dht_dir_attr_heal (void *data);
+dht_inode_ctx_get(inode_t *inode, xlator_t *this, dht_inode_ctx_t **ctx);
 int
-dht_dir_attr_heal_done (int ret, call_frame_t *sync_frame, void *data);
+dht_inode_ctx_set(inode_t *inode, xlator_t *this, dht_inode_ctx_t *ctx);
 int
-dht_dir_has_layout (dict_t *xattr, char *name);
-gf_boolean_t
-dht_is_subvol_in_layout (dht_layout_t *layout, xlator_t *xlator);
+dht_dir_attr_heal(void *data);
+int
+dht_dir_attr_heal_done(int ret, call_frame_t *sync_frame, void *data);
 xlator_t *
-dht_subvol_with_free_space_inodes (xlator_t *this, xlator_t *subvol,
-                                   dht_layout_t *layout);
+dht_subvol_with_free_space_inodes(xlator_t *this, xlator_t *subvol,
+                                  xlator_t *ignore, dht_layout_t *layout,
+                                  uint64_t filesize);
 xlator_t *
-dht_subvol_maxspace_nonzeroinode (xlator_t *this, xlator_t *subvol,
-                                  dht_layout_t *layout);
+dht_subvol_maxspace_nonzeroinode(xlator_t *this, xlator_t *subvol,
+                                 dht_layout_t *layout);
 int
-dht_linkfile_attr_heal (call_frame_t *frame, xlator_t *this);
+dht_dir_has_layout(dict_t *xattr, char *name);
+int
+dht_linkfile_attr_heal(call_frame_t *frame, xlator_t *this);
 
-void
-dht_layout_dump (dht_layout_t  *layout, const char *prefix);
 int32_t
-dht_priv_dump (xlator_t *this);
+dht_priv_dump(xlator_t *this);
 int32_t
-dht_inodectx_dump (xlator_t *this, inode_t *inode);
+dht_inodectx_dump(xlator_t *this, inode_t *inode);
+
+gf_boolean_t
+dht_is_subvol_in_layout(dht_layout_t *layout, xlator_t *xlator);
 
 int
-dht_inode_ctx_get1 (xlator_t *this, inode_t *inode, xlator_t **subvol);
+dht_inode_ctx_get_mig_info(xlator_t *this, inode_t *inode,
+                           xlator_t **src_subvol, xlator_t **dst_subvol);
+gf_boolean_t
+dht_mig_info_is_invalid(xlator_t *current, xlator_t *src_subvol,
+                        xlator_t *dst_subvol);
 
 int
-dht_subvol_status (dht_conf_t *conf, xlator_t *subvol);
+dht_subvol_status(dht_conf_t *conf, xlator_t *subvol);
 
 void
-dht_log_new_layout_for_dir_selfheal (xlator_t *this, loc_t *loc,
-                                     dht_layout_t *layout);
+dht_log_new_layout_for_dir_selfheal(xlator_t *this, loc_t *loc,
+                                    dht_layout_t *layout);
+
 int
-dht_lookup_everywhere_done (call_frame_t *frame, xlator_t *this);
+dht_layout_sort(dht_layout_t *layout);
 
 int
-dht_fill_dict_to_avoid_unlink_of_migrating_file (dict_t *dict);
+dht_heal_full_path(void *data);
 
+int
+dht_heal_full_path_done(int op_ret, call_frame_t *frame, void *data);
 
-/* Acquire non-blocking inodelk on a list of xlators.
- *
- * @lk_array: array of lock requests lock on.
- *
- * @lk_count: number of locks in @lk_array
- *
- * @inodelk_cbk: will be called after inodelk replies are received
- *
- * @retval: -1 if stack_winding inodelk fails. 0 otherwise.
- *          inodelk_cbk is called with appropriate error on errors.
- *          On failure to acquire lock on all members of list, successful
- *          locks are unlocked before invoking cbk.
- */
+int
+dht_layout_missing_dirs(dht_layout_t *layout);
 
 int
-dht_nonblocking_inodelk (call_frame_t *frame, dht_lock_t **lk_array,
-                         int lk_count, fop_inodelk_cbk_t inodelk_cbk);
+dht_refresh_layout(call_frame_t *frame);
+
+int
+dht_build_parent_loc(xlator_t *this, loc_t *parent, loc_t *child,
+                     int32_t *op_errno);
+
+int32_t
+dht_set_local_rebalance(xlator_t *this, dht_local_t *local, struct iatt *stbuf,
+                        struct iatt *prebuf, struct iatt *postbuf,
+                        dict_t *xdata);
+void
+dht_build_root_loc(inode_t *inode, loc_t *loc);
+
+gf_boolean_t
+dht_fd_open_on_dst(xlator_t *this, fd_t *fd, xlator_t *dst);
+
+int32_t
+dht_fd_ctx_destroy(xlator_t *this, fd_t *fd);
+
+int32_t
+dht_release(xlator_t *this, fd_t *fd);
+
+int32_t
+dht_set_fixed_dir_stat(struct iatt *stat);
+
+xlator_t *
+dht_get_lock_subvolume(xlator_t *this, struct gf_flock *lock,
+                       dht_local_t *local);
+
+int
+dht_lk_inode_unref(call_frame_t *frame, int32_t op_ret);
+
+int
+dht_fd_ctx_set(xlator_t *this, fd_t *fd, xlator_t *subvol);
+
+int
+dht_check_and_open_fd_on_subvol(xlator_t *this, call_frame_t *frame);
+
+/* FD fop callbacks */
+
+int
+dht_writev_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int op_ret,
+               int op_errno, struct iatt *prebuf, struct iatt *postbuf,
+               dict_t *xdata);
+
+int
+dht_flush_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int op_ret,
+              int op_errno, dict_t *xdata);
+
+int
+dht_file_setattr_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                     int op_ret, int op_errno, struct iatt *prebuf,
+                     struct iatt *postbuf, dict_t *xdata);
+
+int
+dht_zerofill_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int op_ret,
+                 int op_errno, struct iatt *prebuf, struct iatt *postbuf,
+                 dict_t *xdata);
+
+int
+dht_discard_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int op_ret,
+                int op_errno, struct iatt *prebuf, struct iatt *postbuf,
+                dict_t *xdata);
+
+int
+dht_fallocate_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int op_ret,
+                  int op_errno, struct iatt *prebuf, struct iatt *postbuf,
+                  dict_t *xdata);
+
+int
+dht_truncate_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int op_ret,
+                 int op_errno, struct iatt *prebuf, struct iatt *postbuf,
+                 dict_t *xdata);
 
-/* same as dht_nonblocking_inodelk, but issues sequential blocking locks on
- * @lk_array directly. locks are issued on some order which remains same
- * for a list of xlators (irrespective of order of xlators within list).
- */
 int
-dht_blocking_inodelk (call_frame_t *frame, dht_lock_t **lk_array,
-                      int lk_count, fop_inodelk_cbk_t inodelk_cbk);
+dht_fsync_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int op_ret,
+              int op_errno, struct iatt *prebuf, struct iatt *postbuf,
+              dict_t *xdata);
+
+int
+dht_readv_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int op_ret,
+              int op_errno, struct iovec *vector, int count, struct iatt *stbuf,
+              struct iobref *iobref, dict_t *xdata);
+
+int
+dht_file_attr_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int op_ret,
+                  int op_errno, struct iatt *stbuf, dict_t *xdata);
+
+int
+dht_file_removexattr_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                         int op_ret, int op_errno, dict_t *xdata);
+
+int
+dht_file_setxattr_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                      int op_ret, int op_errno, dict_t *xdata);
+
+/* All custom xattr heal functions */
+int
+dht_dir_heal_xattrs(void *data);
+
+int
+dht_dir_heal_xattrs_done(int ret, call_frame_t *sync_frame, void *data);
 
 int32_t
-dht_unlock_inodelk (call_frame_t *frame, dht_lock_t **lk_array, int lk_count,
-                    fop_inodelk_cbk_t inodelk_cbk);
+dht_dict_set_array(dict_t *dict, char *key, int32_t value[], int32_t size);
+
+int
+dht_set_user_xattr(dict_t *dict, char *k, data_t *v, void *data);
+
+void
+dht_dir_set_heal_xattr(xlator_t *this, dht_local_t *local, dict_t *dst,
+                       dict_t *src, int *uret, int *uflag);
+
+int
+dht_dir_xattr_heal(xlator_t *this, dht_local_t *local, int *op_errno);
+
+int
+dht_common_mark_mdsxattr(call_frame_t *frame, int *errst, int flag);
+
+int
+dht_inode_ctx_mdsvol_get(inode_t *inode, xlator_t *this, xlator_t **mdsvol);
+
+int
+dht_selfheal_dir_setattr(call_frame_t *frame, loc_t *loc, struct iatt *stbuf,
+                         int32_t valid, dht_layout_t *layout);
+
+/* Abstract out the DHT-IATT-IN-DICT */
 
-dht_lock_t *
-dht_lock_new (xlator_t *this, xlator_t *xl, loc_t *loc, short type,
-              const char *domain);
 void
-dht_lock_array_free (dht_lock_t **lk_array, int count);
+dht_selfheal_layout_new_directory(call_frame_t *frame, loc_t *loc,
+                                  dht_layout_t *new_layout);
+
+int
+dht_pt_fgetxattr(call_frame_t *frame, xlator_t *this, fd_t *fd, const char *key,
+                 dict_t *xdata);
+
+int
+dht_pt_getxattr(call_frame_t *frame, xlator_t *this, loc_t *loc,
+                const char *key, dict_t *xdata);
 
 int32_t
-dht_lock_count (dht_lock_t **lk_array, int lk_count);
+dht_pt_mkdir(call_frame_t *frame, xlator_t *this, loc_t *loc, mode_t mode,
+             mode_t umask, dict_t *xdata);
 
 int
-dht_layout_sort (dht_layout_t *layout);
+dht_pt_rename(call_frame_t *frame, xlator_t *this, loc_t *oldloc, loc_t *newloc,
+              dict_t *xdata);
+
+int32_t
+dht_check_remote_fd_failed_error(dht_local_t *local, int op_ret, int op_errno);
 
 int
-dht_layout_missing_dirs (dht_layout_t *layout);
+dht_common_xattrop_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                       int32_t op_ret, int32_t op_errno, dict_t *dict,
+                       dict_t *xdata);
 
-#endif/* _DHT_H */
+int32_t
+dht_create_lock(call_frame_t *frame, xlator_t *subvol);
+
+int
+dht_set_parent_layout_in_dict(loc_t *loc, xlator_t *this, dht_local_t *local);
+
+int
+dht_dir_layout_error_check(xlator_t *this, inode_t *inode);
+
+int
+dht_inode_ctx_mdsvol_set(inode_t *inode, xlator_t *this, xlator_t *mds_subvol);
+#endif /* _DHT_H */
diff --git a/xlators/cluster/dht/src/dht-diskusage.c b/xlators/cluster/dht/src/dht-diskusage.c
index 621c613d08c..c0588828fdb 100644
--- a/xlators/cluster/dht/src/dht-diskusage.c
+++ b/xlators/cluster/dht/src/dht-diskusage.c
@@ -8,440 +8,480 @@
   cases as published by the Free Software Foundation.
 */
 
-
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
-
 /* TODO: add NS locking */
 
-#include "glusterfs.h"
-#include "xlator.h"
 #include "dht-common.h"
-#include "dht-messages.h"
-#include "defaults.h"
 
 #include <sys/time.h>
-
+#include <glusterfs/events.h>
 
 int
-dht_du_info_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-		 int op_ret, int op_errno, struct statvfs *statvfs,
-                 dict_t *xdata)
+dht_du_info_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int op_ret,
+                int op_errno, struct statvfs *statvfs, dict_t *xdata)
 {
-	dht_conf_t    *conf         = NULL;
-	call_frame_t  *prev          = NULL;
-	int            this_call_cnt = 0;
-	int            i = 0;
-	double         percent = 0;
-	double         percent_inodes = 0;
-	uint64_t       bytes = 0;
-        uint32_t       bpc;     /* blocks per chunk */
-        uint32_t       chunks   = 0;
-
-	conf = this->private;
-	prev = cookie;
-
-	if (op_ret == -1) {
-		gf_log (this->name, GF_LOG_WARNING,
-			"failed to get disk info from %s", prev->this->name);
-		goto out;
-	}
-
-	if (statvfs && statvfs->f_blocks) {
-		percent = (statvfs->f_bavail * 100) / statvfs->f_blocks;
-		bytes = (statvfs->f_bavail * statvfs->f_frsize);
-                /*
-                 * A 32-bit count of 1MB chunks allows a maximum brick size of
-                 * ~4PB.  It's possible that we could see a single local FS
-                 * bigger than that some day, but this code is likely to be
-                 * irrelevant by then.  Meanwhile, it's more important to keep
-                 * the chunk size small so the layout-calculation code that
-                 * uses this value can be tested on normal machines.
-                 */
-                bpc = (1 << 20) / statvfs->f_bsize;
-                chunks = (statvfs->f_blocks + bpc - 1) / bpc;
-	}
-
-	if (statvfs && statvfs->f_files) {
-		percent_inodes = (statvfs->f_ffree * 100) / statvfs->f_files;
-	} else {
-                /*
-                 * Set percent inodes to 100 for dynamically allocated inode
-                 * filesystems. The rationale is that distribute need not
-                 * worry about total inodes; rather, let the 'create()' be
-                 * scheduled on the hashed subvol regardless of the total
-                 * inodes.
-		 */
-		percent_inodes = 100;
-	}
-
-	LOCK (&conf->subvolume_lock);
-	{
-		for (i = 0; i < conf->subvolume_cnt; i++)
-			if (prev->this == conf->subvolumes[i]) {
-				conf->du_stats[i].avail_percent = percent;
-				conf->du_stats[i].avail_space   = bytes;
-				conf->du_stats[i].avail_inodes  = percent_inodes;
-                                conf->du_stats[i].chunks        = chunks;
-				gf_msg_debug (this->name, 0,
-				              "subvolume '%s': avail_percent "
-					      "is: %.2f and avail_space "
-                                              "is: %" PRIu64" and avail_inodes"
-                                              " is: %.2f",
-					      prev->this->name,
-					      conf->du_stats[i].avail_percent,
-					      conf->du_stats[i].avail_space,
-					      conf->du_stats[i].avail_inodes);
-                                break;  /* no point in looping further */
-			}
-	}
-	UNLOCK (&conf->subvolume_lock);
+    dht_conf_t *conf = NULL;
+    xlator_t *prev = NULL;
+    int this_call_cnt = 0;
+    int i = 0;
+    double percent = 0;
+    double percent_inodes = 0;
+    uint64_t bytes = 0;
+    uint32_t bpc; /* blocks per chunk */
+    uint32_t chunks = 0;
+
+    conf = this->private;
+    prev = cookie;
+
+    if (op_ret == -1 || !statvfs) {
+        gf_msg(this->name, GF_LOG_WARNING, op_errno,
+               DHT_MSG_GET_DISK_INFO_ERROR, "failed to get disk info from %s",
+               prev->name);
+        goto out;
+    }
+
+    if (statvfs->f_blocks) {
+        percent = (statvfs->f_bavail * 100) / statvfs->f_blocks;
+        bytes = (statvfs->f_bavail * statvfs->f_frsize);
+        /*
+         * A 32-bit count of 1MB chunks allows a maximum brick size of
+         * ~4PB.  It's possible that we could see a single local FS
+         * bigger than that some day, but this code is likely to be
+         * irrelevant by then.  Meanwhile, it's more important to keep
+         * the chunk size small so the layout-calculation code that
+         * uses this value can be tested on normal machines.
+         */
+        bpc = (1 << 20) / statvfs->f_bsize;
+        chunks = (statvfs->f_blocks + bpc - 1) / bpc;
+    }
+
+    if (statvfs->f_files) {
+        percent_inodes = (statvfs->f_ffree * 100) / statvfs->f_files;
+    } else {
+        /*
+         * Set percent inodes to 100 for dynamically allocated inode
+         * filesystems. The rationale is that distribute need not
+         * worry about total inodes; rather, let the 'create()' be
+         * scheduled on the hashed subvol regardless of the total
+         * inodes.
+         */
+        percent_inodes = 100;
+    }
+
+    LOCK(&conf->subvolume_lock);
+    {
+        for (i = 0; i < conf->subvolume_cnt; i++)
+            if (prev == conf->subvolumes[i]) {
+                conf->du_stats[i].avail_percent = percent;
+                conf->du_stats[i].avail_space = bytes;
+                conf->du_stats[i].avail_inodes = percent_inodes;
+                conf->du_stats[i].chunks = chunks;
+                conf->du_stats[i].total_blocks = statvfs->f_blocks;
+                conf->du_stats[i].avail_blocks = statvfs->f_bavail;
+                conf->du_stats[i].frsize = statvfs->f_frsize;
+
+                gf_msg_debug(this->name, 0,
+                             "subvolume '%s': avail_percent "
+                             "is: %.2f and avail_space "
+                             "is: %" PRIu64
+                             " and avail_inodes"
+                             " is: %.2f",
+                             prev->name, conf->du_stats[i].avail_percent,
+                             conf->du_stats[i].avail_space,
+                             conf->du_stats[i].avail_inodes);
+                break; /* no point in looping further */
+            }
+    }
+    UNLOCK(&conf->subvolume_lock);
 
 out:
-	this_call_cnt = dht_frame_return (frame);
-	if (is_last_call (this_call_cnt))
-		DHT_STACK_DESTROY (frame);
+    this_call_cnt = dht_frame_return(frame);
+    if (is_last_call(this_call_cnt))
+        DHT_STACK_DESTROY(frame);
 
-	return 0;
+    return 0;
 }
 
 int
-dht_get_du_info_for_subvol (xlator_t *this, int subvol_idx)
+dht_get_du_info_for_subvol(xlator_t *this, int subvol_idx)
 {
-	dht_conf_t    *conf         = NULL;
-	call_frame_t  *statfs_frame = NULL;
-	dht_local_t   *statfs_local = NULL;
-	call_pool_t   *pool         = NULL;
-	loc_t          tmp_loc      = {0,};
-
-	conf = this->private;
-	pool = this->ctx->pool;
-
-	statfs_frame = create_frame (this, pool);
-	if (!statfs_frame) {
-		goto err;
-	}
-
-	/* local->fop value is not used in this case */
-	statfs_local = dht_local_init (statfs_frame, NULL, NULL,
-				       GF_FOP_MAXVALUE);
-	if (!statfs_local) {
-		goto err;
-	}
-
-        /* make it root gfid, should be enough to get the proper info back */
-        tmp_loc.gfid[15] = 1;
-
-	statfs_local->call_cnt = 1;
-	STACK_WIND (statfs_frame, dht_du_info_cbk,
-		    conf->subvolumes[subvol_idx],
-		    conf->subvolumes[subvol_idx]->fops->statfs,
-		    &tmp_loc, NULL);
-
-	return 0;
+    dht_conf_t *conf = NULL;
+    call_frame_t *statfs_frame = NULL;
+    dht_local_t *statfs_local = NULL;
+    call_pool_t *pool = NULL;
+    loc_t tmp_loc = {
+        0,
+    };
+
+    conf = this->private;
+    pool = this->ctx->pool;
+
+    statfs_frame = create_frame(this, pool);
+    if (!statfs_frame) {
+        goto err;
+    }
+
+    /* local->fop value is not used in this case */
+    statfs_local = dht_local_init(statfs_frame, NULL, NULL, GF_FOP_MAXVALUE);
+    if (!statfs_local) {
+        goto err;
+    }
+
+    /* make it root gfid, should be enough to get the proper info back */
+    tmp_loc.gfid[15] = 1;
+
+    statfs_local->call_cnt = 1;
+    STACK_WIND_COOKIE(
+        statfs_frame, dht_du_info_cbk, conf->subvolumes[subvol_idx],
+        conf->subvolumes[subvol_idx],
+        conf->subvolumes[subvol_idx]->fops->statfs, &tmp_loc, NULL);
+
+    return 0;
 err:
-	if (statfs_frame)
-		DHT_STACK_DESTROY (statfs_frame);
+    if (statfs_frame)
+        DHT_STACK_DESTROY(statfs_frame);
 
-	return -1;
+    return -1;
 }
 
 int
-dht_get_du_info (call_frame_t *frame, xlator_t *this, loc_t *loc)
+dht_get_du_info(call_frame_t *frame, xlator_t *this, loc_t *loc)
 {
-	int            i            = 0;
-        int            ret          = -1;
-	dht_conf_t    *conf         = NULL;
-	call_frame_t  *statfs_frame = NULL;
-	dht_local_t   *statfs_local = NULL;
-	struct timeval tv           = {0,};
-        loc_t          tmp_loc      = {0,};
-
-	conf  = this->private;
-
-	gettimeofday (&tv, NULL);
-
-        /* make it root gfid, should be enough to get the proper
-           info back */
-        tmp_loc.gfid[15] = 1;
-
-	if (tv.tv_sec > (conf->refresh_interval
-			 + conf->last_stat_fetch.tv_sec)) {
-
-		statfs_frame = copy_frame (frame);
-		if (!statfs_frame) {
-			goto err;
-		}
-
-		/* In this case, 'local->fop' is not used */
-		statfs_local = dht_local_init (statfs_frame, loc, NULL,
-					       GF_FOP_MAXVALUE);
-		if (!statfs_local) {
-			goto err;
-		}
-
-                statfs_local->params = dict_new ();
-                if (!statfs_local->params)
-                        goto err;
-
-                ret = dict_set_int8 (statfs_local->params,
-                                     GF_INTERNAL_IGNORE_DEEM_STATFS, 1);
-                if (ret) {
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "Failed to set "
-                                GF_INTERNAL_IGNORE_DEEM_STATFS" in dict");
-                        goto err;
-                }
+    int i = 0;
+    int ret = -1;
+    dht_conf_t *conf = NULL;
+    call_frame_t *statfs_frame = NULL;
+    dht_local_t *statfs_local = NULL;
+    loc_t tmp_loc = {
+        0,
+    };
+    time_t now;
+
+    conf = this->private;
+    now = gf_time();
+    /* make it root gfid, should be enough to get the proper
+       info back */
+    tmp_loc.gfid[15] = 1;
+
+    if (now > (conf->refresh_interval + conf->last_stat_fetch)) {
+        statfs_frame = copy_frame(frame);
+        if (!statfs_frame) {
+            goto err;
+        }
+
+        /* In this case, 'local->fop' is not used */
+        statfs_local = dht_local_init(statfs_frame, loc, NULL, GF_FOP_MAXVALUE);
+        if (!statfs_local) {
+            goto err;
+        }
 
-		statfs_local->call_cnt = conf->subvolume_cnt;
-		for (i = 0; i < conf->subvolume_cnt; i++) {
-			STACK_WIND (statfs_frame, dht_du_info_cbk,
-				    conf->subvolumes[i],
-				    conf->subvolumes[i]->fops->statfs,
-				    &tmp_loc, statfs_local->params);
-		}
-
-		conf->last_stat_fetch.tv_sec = tv.tv_sec;
-	}
-	return 0;
+        statfs_local->params = dict_new();
+        if (!statfs_local->params)
+            goto err;
+
+        ret = dict_set_int8(statfs_local->params,
+                            GF_INTERNAL_IGNORE_DEEM_STATFS, 1);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, DHT_MSG_DICT_SET_FAILED,
+                   "Failed to set " GF_INTERNAL_IGNORE_DEEM_STATFS " in dict");
+            goto err;
+        }
+
+        statfs_local->call_cnt = conf->subvolume_cnt;
+        for (i = 0; i < conf->subvolume_cnt; i++) {
+            STACK_WIND_COOKIE(statfs_frame, dht_du_info_cbk,
+                              conf->subvolumes[i], conf->subvolumes[i],
+                              conf->subvolumes[i]->fops->statfs, &tmp_loc,
+                              statfs_local->params);
+        }
+
+        conf->last_stat_fetch = now;
+    }
+    return 0;
 err:
-	if (statfs_frame)
-		DHT_STACK_DESTROY (statfs_frame);
+    if (statfs_frame)
+        DHT_STACK_DESTROY(statfs_frame);
 
-	return -1;
+    return -1;
 }
 
-
 gf_boolean_t
-dht_is_subvol_filled (xlator_t *this, xlator_t *subvol)
+dht_is_subvol_filled(xlator_t *this, xlator_t *subvol)
 {
-	int         i = 0;
-	dht_conf_t *conf = NULL;
-	gf_boolean_t subvol_filled_inodes = _gf_false;
-	gf_boolean_t subvol_filled_space = _gf_false;
-	gf_boolean_t is_subvol_filled = _gf_false;
-
-	conf = this->private;
-
-	/* Check for values above specified percent or free disk */
-	LOCK (&conf->subvolume_lock);
-	{
-		for (i = 0; i < conf->subvolume_cnt; i++) {
-			if (subvol == conf->subvolumes[i]) {
-				if (conf->disk_unit == 'p') {
-					if (conf->du_stats[i].avail_percent <
-					    conf->min_free_disk) {
-						subvol_filled_space = _gf_true;
-						break;
-					}
-
-				} else {
-					if (conf->du_stats[i].avail_space <
-					    conf->min_free_disk) {
-						subvol_filled_space = _gf_true;
-						break;
-					}
-				}
-				if (conf->du_stats[i].avail_inodes <
-				    conf->min_free_inodes) {
-					subvol_filled_inodes = _gf_true;
-					break;
-				}
-			}
-		}
-	}
-	UNLOCK (&conf->subvolume_lock);
-
-	if (subvol_filled_space && conf->subvolume_status[i]) {
-		if (!(conf->du_stats[i].log++ % (GF_UNIVERSAL_ANSWER * 10))) {
-			gf_msg (this->name, GF_LOG_WARNING, 0,
-                                DHT_MSG_SUBVOL_INSUFF_SPACE,
-				"disk space on subvolume '%s' is getting "
-				"full (%.2f %%), consider adding more bricks",
-				subvol->name,
-				(100 - conf->du_stats[i].avail_percent));
-		}
-	}
-
-	if (subvol_filled_inodes && conf->subvolume_status[i]) {
-		if (!(conf->du_stats[i].log++ % (GF_UNIVERSAL_ANSWER * 10))) {
-			gf_msg (this->name, GF_LOG_CRITICAL, 0,
-                                DHT_MSG_SUBVOL_INSUFF_INODES,
-				"inodes on subvolume '%s' are at "
-				"(%.2f %%), consider adding more bricks",
-				subvol->name,
-				(100 - conf->du_stats[i].avail_inodes));
-		}
-	}
-
-	is_subvol_filled = (subvol_filled_space || subvol_filled_inodes);
-
-	return is_subvol_filled;
-}
+    int i = 0;
+    char vol_name[256];
+    dht_conf_t *conf = NULL;
+    gf_boolean_t subvol_filled_inodes = _gf_false;
+    gf_boolean_t subvol_filled_space = _gf_false;
+    gf_boolean_t is_subvol_filled = _gf_false;
+    double usage = 0;
+
+    conf = this->private;
+
+    /* Check for values above specified percent or free disk */
+    LOCK(&conf->subvolume_lock);
+    {
+        for (i = 0; i < conf->subvolume_cnt; i++) {
+            if (subvol == conf->subvolumes[i]) {
+                if (conf->disk_unit == 'p') {
+                    if (conf->du_stats[i].avail_percent < conf->min_free_disk) {
+                        subvol_filled_space = _gf_true;
+                        break;
+                    }
+
+                } else {
+                    if (conf->du_stats[i].avail_space < conf->min_free_disk) {
+                        subvol_filled_space = _gf_true;
+                        break;
+                    }
+                }
+                if (conf->du_stats[i].avail_inodes < conf->min_free_inodes) {
+                    subvol_filled_inodes = _gf_true;
+                    break;
+                }
+            }
+        }
+    }
+    UNLOCK(&conf->subvolume_lock);
 
+    if (subvol_filled_space && conf->subvolume_status[i]) {
+        if (!(conf->du_stats[i].log++ % (GF_UNIVERSAL_ANSWER * 10))) {
+            usage = 100 - conf->du_stats[i].avail_percent;
 
-/*Get the best subvolume to create the file in*/
-xlator_t *
-dht_free_disk_available_subvol (xlator_t *this, xlator_t *subvol,
-                                dht_local_t *local)
-{
-	xlator_t   *avail_subvol = NULL;
-	dht_conf_t *conf = NULL;
-        dht_layout_t *layout = NULL;
-        loc_t      *loc = NULL;
+            gf_msg(this->name, GF_LOG_WARNING, 0, DHT_MSG_SUBVOL_INSUFF_SPACE,
+                   "disk space on subvolume '%s' is getting "
+                   "full (%.2f %%), consider adding more bricks",
+                   subvol->name, usage);
 
-	conf = this->private;
-        if (!local)
-                goto out;
-        loc = &local->loc;
-        if (!local->layout) {
-                layout = dht_layout_get (this, loc->parent);
-
-                if (!layout) {
-                        gf_msg_debug (this->name, 0,
-                                      "Missing layout. path=%s,"
-                                      " parent gfid = %s", loc->path,
-                                      uuid_utoa (loc->parent->gfid));
-                        goto out;
-                }
-        } else {
-                layout = dht_layout_ref (this, local->layout);
+            (void)snprintf(vol_name, sizeof(vol_name), "%s", this->name);
+            vol_name[(strlen(this->name) - 4)] = '\0';
+
+            gf_event(EVENT_DHT_DISK_USAGE, "volume=%s;subvol=%s;usage=%.2f %%",
+                     vol_name, subvol->name, usage);
         }
+    }
+
+    if (subvol_filled_inodes && conf->subvolume_status[i]) {
+        if (!(conf->du_stats[i].log++ % (GF_UNIVERSAL_ANSWER * 10))) {
+            usage = 100 - conf->du_stats[i].avail_inodes;
+            gf_msg(this->name, GF_LOG_CRITICAL, 0, DHT_MSG_SUBVOL_INSUFF_INODES,
+                   "inodes on subvolume '%s' are at "
+                   "(%.2f %%), consider adding more bricks",
+                   subvol->name, usage);
+
+            (void)snprintf(vol_name, sizeof(vol_name), "%s", this->name);
+            vol_name[(strlen(this->name) - 4)] = '\0';
+
+            gf_event(EVENT_DHT_INODES_USAGE,
+                     "volume=%s;subvol=%s;usage=%.2f %%", vol_name,
+                     subvol->name, usage);
+        }
+    }
 
-        LOCK (&conf->subvolume_lock);
-	{
-                avail_subvol = dht_subvol_with_free_space_inodes(this, subvol,
-                                                                 layout);
-                if(!avail_subvol)
-                {
-                        avail_subvol = dht_subvol_maxspace_nonzeroinode(this,
-                                                                        subvol,
-                                                                        layout);
-                }
+    is_subvol_filled = (subvol_filled_space || subvol_filled_inodes);
+
+    return is_subvol_filled;
+}
 
-	}
-	UNLOCK (&conf->subvolume_lock);
+/*Get the best subvolume to create the file in*/
+xlator_t *
+dht_free_disk_available_subvol(xlator_t *this, xlator_t *subvol,
+                               dht_local_t *local)
+{
+    xlator_t *avail_subvol = NULL;
+    dht_conf_t *conf = NULL;
+    dht_layout_t *layout = NULL;
+    loc_t *loc = NULL;
+
+    conf = this->private;
+    if (!local)
+        goto out;
+    loc = &local->loc;
+    if (!local->layout) {
+        layout = dht_layout_get(this, loc->parent);
+
+        if (!layout) {
+            gf_msg_debug(this->name, 0,
+                         "Missing layout. path=%s,"
+                         " parent gfid = %s",
+                         loc->path, uuid_utoa(loc->parent->gfid));
+            goto out;
+        }
+    } else {
+        layout = dht_layout_ref(this, local->layout);
+    }
+
+    LOCK(&conf->subvolume_lock);
+    {
+        avail_subvol = dht_subvol_with_free_space_inodes(this, subvol, NULL,
+                                                         layout, 0);
+        if (!avail_subvol) {
+            avail_subvol = dht_subvol_maxspace_nonzeroinode(this, subvol,
+                                                            layout);
+        }
+    }
+    UNLOCK(&conf->subvolume_lock);
 out:
-	if (!avail_subvol) {
-		gf_msg_debug (this->name, 0,
-		              "No subvolume has enough free space \
+    if (!avail_subvol) {
+        gf_msg_debug(this->name, 0,
+                     "No subvolume has enough free space \
                               and/or inodes to create");
-                avail_subvol = subvol;
-	}
+        avail_subvol = subvol;
+    }
 
-        if (layout)
-                dht_layout_unref (this, layout);
-	return avail_subvol;
+    if (layout)
+        dht_layout_unref(this, layout);
+    return avail_subvol;
 }
 
-static inline
-int32_t dht_subvol_has_err (xlator_t *this, dht_layout_t *layout)
+static inline int32_t
+dht_subvol_has_err(dht_conf_t *conf, xlator_t *this, xlator_t *ignore,
+                   dht_layout_t *layout)
 {
-        int ret = -1;
-        int i   = 0;
+    int ret = -1;
+    int i = 0;
+
+    if (!this || !layout)
+        goto out;
+
+    /* this check is meant for rebalance process. The source of the file
+     * should be ignored for space check */
+    if (this == ignore) {
+        goto out;
+    }
+
+    /* check if subvol has layout errors, before selecting it */
+    for (i = 0; i < layout->cnt; i++) {
+        if (!strcmp(layout->list[i].xlator->name, this->name) &&
+            (layout->list[i].err != 0)) {
+            ret = -1;
+            goto out;
+        }
+    }
 
-        if (!this || !layout)
+    /* discard decommissioned subvol */
+    if (conf->decommission_subvols_cnt) {
+        for (i = 0; i < conf->subvolume_cnt; i++) {
+            if (conf->decommissioned_bricks[i] &&
+                conf->decommissioned_bricks[i] == this) {
+                ret = -1;
                 goto out;
-
-        /* check if subvol has layout errors, before selecting it */
-        for (i = 0; i < layout->cnt; i++) {
-                if (!strcmp (layout->list[i].xlator->name, this->name) &&
-                     (layout->list[i].err != 0)) {
-                        ret = -1;
-                        goto out;
-                }
+            }
         }
-        ret = 0;
+    }
+
+    ret = 0;
 out:
-        return ret;
+    return ret;
 }
 
 /*Get subvolume which has both space and inodes more than the min criteria*/
 xlator_t *
 dht_subvol_with_free_space_inodes(xlator_t *this, xlator_t *subvol,
-                                  dht_layout_t *layout)
+                                  xlator_t *ignore, dht_layout_t *layout,
+                                  uint64_t filesize)
 {
-        int i = 0;
-        double max = 0;
-        double max_inodes = 0;
-        int    ignore_subvol = 0;
-
-        xlator_t *avail_subvol = NULL;
-        dht_conf_t *conf = NULL;
-
-        conf = this->private;
-
-        for(i=0; i < conf->subvolume_cnt; i++) {
-                /* check if subvol has layout errors, before selecting it */
-                ignore_subvol = dht_subvol_has_err (conf->subvolumes[i],
-                                                    layout);
-                if (ignore_subvol)
-                        continue;
-
-                if ((conf->disk_unit == 'p') &&
-                    (conf->du_stats[i].avail_percent > conf->min_free_disk) &&
-                    (conf->du_stats[i].avail_inodes  > conf->min_free_inodes)) {
-                        if ((conf->du_stats[i].avail_inodes > max_inodes) ||
-                            (conf->du_stats[i].avail_percent > max)) {
-                                max = conf->du_stats[i].avail_percent;
-                                max_inodes = conf->du_stats[i].avail_inodes;
-                                avail_subvol = conf->subvolumes[i];
-                        }
-                }
+    int i = 0;
+    double max = 0;
+    double max_inodes = 0;
+    int ignore_subvol = 0;
+    uint64_t total_blocks = 0;
+    uint64_t avail_blocks = 0;
+    uint64_t frsize = 0;
+    double post_availspace = 0;
+    double post_percent = 0;
+
+    xlator_t *avail_subvol = NULL;
+    dht_conf_t *conf = NULL;
+
+    conf = this->private;
+
+    for (i = 0; i < conf->subvolume_cnt; i++) {
+        /* check if subvol has layout errors and also it is not a
+         * decommissioned brick, before selecting it */
+        ignore_subvol = dht_subvol_has_err(conf, conf->subvolumes[i], ignore,
+                                           layout);
+        if (ignore_subvol)
+            continue;
+
+        if ((conf->disk_unit == 'p') &&
+            (conf->du_stats[i].avail_percent > conf->min_free_disk) &&
+            (conf->du_stats[i].avail_inodes > conf->min_free_inodes)) {
+            if ((conf->du_stats[i].avail_inodes > max_inodes) ||
+                (conf->du_stats[i].avail_percent > max)) {
+                max = conf->du_stats[i].avail_percent;
+                max_inodes = conf->du_stats[i].avail_inodes;
+                avail_subvol = conf->subvolumes[i];
+                total_blocks = conf->du_stats[i].total_blocks;
+                avail_blocks = conf->du_stats[i].avail_blocks;
+                frsize = conf->du_stats[i].frsize;
+            }
+        }
 
-                if ((conf->disk_unit != 'p') &&
-                    (conf->du_stats[i].avail_space > conf->min_free_disk) &&
-                    (conf->du_stats[i].avail_inodes  > conf->min_free_inodes)) {
-                        if ((conf->du_stats[i].avail_inodes > max_inodes) ||
-                            (conf->du_stats[i].avail_space > max)) {
-                                max = conf->du_stats[i].avail_space;
-                                max_inodes = conf->du_stats[i].avail_inodes;
-                                avail_subvol = conf->subvolumes[i];
-                        }
-                }
+        if ((conf->disk_unit != 'p') &&
+            (conf->du_stats[i].avail_space > conf->min_free_disk) &&
+            (conf->du_stats[i].avail_inodes > conf->min_free_inodes)) {
+            if ((conf->du_stats[i].avail_inodes > max_inodes) ||
+                (conf->du_stats[i].avail_space > max)) {
+                max = conf->du_stats[i].avail_space;
+                max_inodes = conf->du_stats[i].avail_inodes;
+                avail_subvol = conf->subvolumes[i];
+            }
+        }
+    }
+
+    if (avail_subvol) {
+        if (conf->disk_unit == 'p') {
+            post_availspace = (avail_blocks * frsize) - filesize;
+            post_percent = (post_availspace * 100) / (total_blocks * frsize);
+            if (post_percent < conf->min_free_disk)
+                avail_subvol = NULL;
         }
+        if (conf->disk_unit != 'p') {
+            if ((max - filesize) < conf->min_free_disk)
+                avail_subvol = NULL;
+        }
+    }
 
-        return avail_subvol;
+    return avail_subvol;
 }
 
-
-/* Get subvol which has atleast one inode and maximum space */
+/* Get subvol which has at least one inode and maximum space */
 xlator_t *
-dht_subvol_maxspace_nonzeroinode (xlator_t *this, xlator_t *subvol,
-                                  dht_layout_t *layout)
+dht_subvol_maxspace_nonzeroinode(xlator_t *this, xlator_t *subvol,
+                                 dht_layout_t *layout)
 {
-        int         i = 0;
-        double      max = 0;
-        int         ignore_subvol = 0;
-
-        xlator_t   *avail_subvol = NULL;
-        dht_conf_t *conf = NULL;
-
-        conf = this->private;
-
-        for (i = 0; i < conf->subvolume_cnt; i++) {
-                /* check if subvol has layout errors, before selecting it */
-                ignore_subvol = dht_subvol_has_err (conf->subvolumes[i],
-                                                    layout);
-                if (ignore_subvol)
-                        continue;
-
-                if (conf->disk_unit == 'p') {
-                        if ((conf->du_stats[i].avail_percent > max)
-                            && (conf->du_stats[i].avail_inodes > 0 )) {
-                                max = conf->du_stats[i].avail_percent;
-                                avail_subvol = conf->subvolumes[i];
-                        }
-               } else {
-                         if ((conf->du_stats[i].avail_space > max)
-                            && (conf->du_stats[i].avail_inodes > 0)) {
-                                 max = conf->du_stats[i].avail_space;
-                                 avail_subvol = conf->subvolumes[i];
-                         }
-               }
+    int i = 0;
+    double max = 0;
+    int ignore_subvol = 0;
+
+    xlator_t *avail_subvol = NULL;
+    dht_conf_t *conf = NULL;
+
+    conf = this->private;
+
+    for (i = 0; i < conf->subvolume_cnt; i++) {
+        /* check if subvol has layout errors and also it is not a
+         * decommissioned brick, before selecting it*/
+
+        ignore_subvol = dht_subvol_has_err(conf, conf->subvolumes[i], NULL,
+                                           layout);
+        if (ignore_subvol)
+            continue;
+
+        if (conf->disk_unit == 'p') {
+            if ((conf->du_stats[i].avail_percent > max) &&
+                (conf->du_stats[i].avail_inodes > 0)) {
+                max = conf->du_stats[i].avail_percent;
+                avail_subvol = conf->subvolumes[i];
+            }
+        } else {
+            if ((conf->du_stats[i].avail_space > max) &&
+                (conf->du_stats[i].avail_inodes > 0)) {
+                max = conf->du_stats[i].avail_space;
+                avail_subvol = conf->subvolumes[i];
+            }
         }
+    }
 
-        return avail_subvol;
+    return avail_subvol;
 }
diff --git a/xlators/cluster/dht/src/dht-hashfn.c b/xlators/cluster/dht/src/dht-hashfn.c
index 72b3df022da..acda67c312a 100644
--- a/xlators/cluster/dht/src/dht-hashfn.c
+++ b/xlators/cluster/dht/src/dht-hashfn.c
@@ -8,104 +8,103 @@
   cases as published by the Free Software Foundation.
 */
 
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
-
-
-#include "glusterfs.h"
-#include "xlator.h"
 #include "dht-common.h"
-#include "hashfn.h"
+#include <glusterfs/hashfn.h>
 
-
-int
-dht_hash_compute_internal (int type, const char *name, uint32_t *hash_p)
+static int
+dht_hash_compute_internal(int type, const char *name, const int len,
+                          uint32_t *hash_p)
 {
-        int      ret = 0;
-        uint32_t hash = 0;
+    int ret = 0;
+    uint32_t hash = 0;
 
-        switch (type) {
+    switch (type) {
         case DHT_HASH_TYPE_DM:
         case DHT_HASH_TYPE_DM_USER:
-                hash = gf_dm_hashfn (name, strlen (name));
-                break;
+            hash = gf_dm_hashfn(name, len);
+            break;
         default:
-                ret = -1;
-                break;
-        }
+            ret = -1;
+            break;
+    }
 
-        if (ret == 0) {
-                *hash_p = hash;
-        }
+    if (ret == 0) {
+        *hash_p = hash;
+    }
 
-        return ret;
+    return ret;
 }
 
-
-static inline
-gf_boolean_t
-dht_munge_name (const char *original, char *modified, size_t len, regex_t *re)
+/* The function returns:
+ * 0  : in case no munge took place
+ * >0 : the length (inc. terminating NULL!) of the newly modified string,
+ *      if it was munged.
+ */
+static int
+dht_munge_name(const char *original, char *modified, size_t len, regex_t *re)
 {
-        regmatch_t      matches[2];
-        size_t          new_len;
-
-        if (regexec(re,original,2,matches,0) != REG_NOMATCH) {
-                if (matches[1].rm_so != -1) {
-                        new_len = matches[1].rm_eo - matches[1].rm_so;
-                        /* Equal would fail due to the NUL at the end. */
-                        if (new_len < len) {
-                                memcpy (modified,original+matches[1].rm_so,
-                                        new_len);
-                                modified[new_len] = '\0';
-                                return _gf_true;
-                        }
-                }
+    regmatch_t matches[2] = {
+        {0},
+    };
+    size_t new_len = 0;
+    int ret = 0;
+
+    ret = regexec(re, original, 2, matches, 0);
+
+    if (ret != REG_NOMATCH) {
+        if (matches[1].rm_so != -1) {
+            new_len = matches[1].rm_eo - matches[1].rm_so;
+            /* Equal would fail due to the NUL at the end. */
+            if (new_len < len) {
+                memcpy(modified, original + matches[1].rm_so, new_len);
+                modified[new_len] = '\0';
+                return new_len + 1; /* +1 for the terminating NULL */
+            }
         }
+    }
 
-        /* This is guaranteed safe because of how the dest was allocated. */
-        strcpy(modified,original);
-        return _gf_false;
+    /* This is guaranteed safe because of how the dest was allocated. */
+    strcpy(modified, original);
+    return 0;
 }
 
 int
-dht_hash_compute (xlator_t *this, int type, const char *name, uint32_t *hash_p)
+dht_hash_compute(xlator_t *this, int type, const char *name, uint32_t *hash_p)
 {
-        char            *rsync_friendly_name    = NULL;
-        dht_conf_t      *priv                   = this->private;
-        size_t           len                    = 0;
-        gf_boolean_t     munged                 = _gf_false;
-
-        /*
-         * It wouldn't be safe to use alloca in an inline function that doesn't
-         * actually get inlined, and it wouldn't be efficient to do a real
-         * allocation, so we use alloca here (if needed) and pass that to the
-         * inline.
-         */
+    char *rsync_friendly_name = NULL;
+    dht_conf_t *priv = NULL;
+    size_t len = 0;
+    int munged = 0;
 
+    priv = this->private;
+
+    if (name == NULL)
+        return -1;
+
+    len = strlen(name) + 1;
+    rsync_friendly_name = alloca(len);
+
+    LOCK(&priv->lock);
+    {
         if (priv->extra_regex_valid) {
-                len = strlen(name) + 1;
-                rsync_friendly_name = alloca(len);
-                munged = dht_munge_name (name, rsync_friendly_name, len,
-                                         &priv->extra_regex);
+            munged = dht_munge_name(name, rsync_friendly_name, len,
+                                    &priv->extra_regex);
         }
 
         if (!munged && priv->rsync_regex_valid) {
-                len = strlen(name) + 1;
-                rsync_friendly_name = alloca(len);
-                gf_msg_trace (this->name, 0, "trying regex for %s", name);
-                munged = dht_munge_name (name, rsync_friendly_name, len,
-                                         &priv->rsync_regex);
-                if (munged) {
-                        gf_msg_debug (this->name, 0,
-                                      "munged down to %s", rsync_friendly_name);
-                }
+            gf_msg_trace(this->name, 0, "trying regex for %s", name);
+            munged = dht_munge_name(name, rsync_friendly_name, len,
+                                    &priv->rsync_regex);
         }
-
-        if (!munged) {
-                rsync_friendly_name = (char *)name;
-        }
-
-        return dht_hash_compute_internal (type, rsync_friendly_name, hash_p);
+    }
+    UNLOCK(&priv->lock);
+    if (munged) {
+        gf_msg_debug(this->name, 0, "munged down to %s", rsync_friendly_name);
+        len = munged;
+    } else {
+        rsync_friendly_name = (char *)name;
+    }
+
+    return dht_hash_compute_internal(type, rsync_friendly_name, len - 1,
+                                     hash_p);
 }
diff --git a/xlators/cluster/dht/src/dht-helper.c b/xlators/cluster/dht/src/dht-helper.c
index cab66017b84..3f2fe43d5f3 100644
--- a/xlators/cluster/dht/src/dht-helper.c
+++ b/xlators/cluster/dht/src/dht-helper.c
@@ -8,1747 +8,2297 @@
   cases as published by the Free Software Foundation.
 */
 
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
-
-
-#include "glusterfs.h"
-#include "xlator.h"
 #include "dht-common.h"
-#include "dht-helper.h"
+#include "dht-lock.h"
+#include "glusterfs/compat-errno.h"  // for ENODATA on BSD
 
-static inline int
-dht_inode_ctx_set1 (xlator_t *this, inode_t *inode, xlator_t *subvol)
+static void
+dht_free_fd_ctx(dht_fd_ctx_t *fd_ctx)
 {
-        uint64_t tmp_subvol = 0;
-
-        tmp_subvol = (long)subvol;
-        return inode_ctx_set1 (inode, this, &tmp_subvol);
+    GF_FREE(fd_ctx);
 }
 
-int
-dht_inode_ctx_get1 (xlator_t *this, inode_t *inode, xlator_t **subvol)
+int32_t
+dht_fd_ctx_destroy(xlator_t *this, fd_t *fd)
 {
-        int ret = -1;
-        uint64_t tmp_subvol = 0;
-
-        ret =  inode_ctx_get1 (inode, this, &tmp_subvol);
-        if (tmp_subvol && subvol)
-                *subvol = (xlator_t *)tmp_subvol;
-
-        return ret;
+    dht_fd_ctx_t *fd_ctx = NULL;
+    uint64_t value = 0;
+    int32_t ret = -1;
+
+    GF_VALIDATE_OR_GOTO("dht", this, out);
+    GF_VALIDATE_OR_GOTO(this->name, fd, out);
+
+    ret = fd_ctx_del(fd, this, &value);
+    if (ret) {
+        goto out;
+    }
+
+    fd_ctx = (dht_fd_ctx_t *)(uintptr_t)value;
+    if (fd_ctx) {
+        GF_REF_PUT(fd_ctx);
+    }
+out:
+    return ret;
 }
 
-
-int
-dht_frame_return (call_frame_t *frame)
+static int
+__dht_fd_ctx_set(xlator_t *this, fd_t *fd, xlator_t *dst)
 {
-        dht_local_t *local = NULL;
-        int          this_call_cnt = -1;
+    dht_fd_ctx_t *fd_ctx = NULL;
+    uint64_t value = 0;
+    int ret = -1;
 
-        if (!frame)
-                return -1;
+    GF_VALIDATE_OR_GOTO("dht", this, out);
+    GF_VALIDATE_OR_GOTO(this->name, fd, out);
 
-        local = frame->local;
+    fd_ctx = GF_CALLOC(1, sizeof(*fd_ctx), gf_dht_mt_fd_ctx_t);
 
-        LOCK (&frame->lock);
-        {
-                this_call_cnt = --local->call_cnt;
-        }
-        UNLOCK (&frame->lock);
+    if (!fd_ctx) {
+        goto out;
+    }
 
-        return this_call_cnt;
-}
+    fd_ctx->opened_on_dst = (uint64_t)(uintptr_t)dst;
+    GF_REF_INIT(fd_ctx, dht_free_fd_ctx);
 
-/*
- * A slightly "updated" version of the algorithm described in the commit log
- * is used here.
- *
- * The only enhancement is that:
- *
- * - The number of bits used by the backend filesystem for HUGE d_off which
- *   is described as 63, and
- * - The number of bits used by the d_off presented by the transformation
- *   upwards which is described as 64, are both made "configurable."
- */
+    value = (uint64_t)(uintptr_t)fd_ctx;
+
+    ret = __fd_ctx_set(fd, this, value);
+    if (ret < 0) {
+        gf_smsg(this->name, GF_LOG_WARNING, 0, DHT_MSG_FD_CTX_SET_FAILED,
+                "fd=0x%p", fd, NULL);
+        GF_REF_PUT(fd_ctx);
+    }
+out:
+    return ret;
+}
 
 int
-dht_filter_loc_subvol_key (xlator_t *this, loc_t *loc, loc_t *new_loc,
-                           xlator_t **subvol)
+dht_fd_ctx_set(xlator_t *this, fd_t *fd, xlator_t *dst)
 {
-        char          *new_name  = NULL;
-        char          *new_path  = NULL;
-        xlator_list_t *trav      = NULL;
-        char           key[1024] = {0,};
-        int            ret       = 0; /* not found */
-
-        /* Why do other tasks if first required 'char' itself is not there */
-        if (!new_loc || !loc || !loc->name || !strchr (loc->name, '@'))
-                goto out;
+    dht_fd_ctx_t *fd_ctx = NULL;
+    uint64_t value = 0;
+    int ret = -1;
+
+    GF_VALIDATE_OR_GOTO("dht", this, out);
+    GF_VALIDATE_OR_GOTO(this->name, fd, out);
+
+    LOCK(&fd->lock);
+    {
+        ret = __fd_ctx_get(fd, this, &value);
+        if (ret && value) {
+            fd_ctx = (dht_fd_ctx_t *)(uintptr_t)value;
+            if (fd_ctx->opened_on_dst == (uint64_t)(uintptr_t)dst) {
+                /* This could happen due to racing
+                 * check_progress tasks*/
+                goto unlock;
+            } else {
+                /* This would be a big problem*/
+                /* Overwrite and hope for the best*/
+                fd_ctx->opened_on_dst = (uint64_t)(uintptr_t)dst;
+                UNLOCK(&fd->lock);
+                gf_smsg(this->name, GF_LOG_WARNING, 0, DHT_MSG_INVALID_VALUE,
+                        NULL);
 
-        trav = this->children;
-        while (trav) {
-                snprintf (key, 1024, "*@%s:%s", this->name, trav->xlator->name);
-                if (fnmatch (key, loc->name, FNM_NOESCAPE) == 0) {
-                        new_name = GF_CALLOC(strlen (loc->name),
-                                             sizeof (char),
-                                             gf_common_mt_char);
-                        if (!new_name)
-                                goto out;
-                        if (fnmatch (key, loc->path, FNM_NOESCAPE) == 0) {
-                                new_path = GF_CALLOC(strlen (loc->path),
-                                                     sizeof (char),
-                                                     gf_common_mt_char);
-                                if (!new_path)
-                                        goto out;
-                                strncpy (new_path, loc->path, (strlen (loc->path) -
-                                                               strlen (key) + 1));
-                        }
-                        strncpy (new_name, loc->name, (strlen (loc->name) -
-                                                       strlen (key) + 1));
-
-                        if (new_loc) {
-                                new_loc->path   = ((new_path) ? new_path:
-                                                   gf_strdup (loc->path));
-                                new_loc->name   = new_name;
-                                new_loc->inode  = inode_ref (loc->inode);
-                                new_loc->parent = inode_ref (loc->parent);
-                        }
-                        *subvol         = trav->xlator;
-                        ret = 1;  /* success */
-                        goto out;
-                }
-                trav = trav->next;
+                goto out;
+            }
         }
+        ret = __dht_fd_ctx_set(this, fd, dst);
+    }
+unlock:
+    UNLOCK(&fd->lock);
 out:
-        if (!ret) {
-                /* !success */
-                GF_FREE (new_path);
-                GF_FREE (new_name);
+    return ret;
+}
+
+static dht_fd_ctx_t *
+dht_fd_ctx_get(xlator_t *this, fd_t *fd)
+{
+    dht_fd_ctx_t *fd_ctx = NULL;
+    int ret = -1;
+    uint64_t tmp_val = 0;
+
+    GF_VALIDATE_OR_GOTO("dht", this, out);
+    GF_VALIDATE_OR_GOTO(this->name, fd, out);
+
+    LOCK(&fd->lock);
+    {
+        ret = __fd_ctx_get(fd, this, &tmp_val);
+        if ((ret < 0) || (tmp_val == 0)) {
+            goto unlock;
         }
-        return ret;
+
+        fd_ctx = (dht_fd_ctx_t *)(uintptr_t)tmp_val;
+        GF_REF_GET(fd_ctx);
+    }
+unlock:
+    UNLOCK(&fd->lock);
+
+out:
+    return fd_ctx;
 }
 
-static xlator_t *
-dht_get_subvol_from_id(xlator_t *this, int client_id)
+gf_boolean_t
+dht_fd_open_on_dst(xlator_t *this, fd_t *fd, xlator_t *dst)
 {
-        xlator_t *xl = NULL;
-        dht_conf_t *conf = NULL;
-        char sid[6] = { 0 };
+    dht_fd_ctx_t *fd_ctx = NULL;
+    gf_boolean_t opened = _gf_false;
 
-        conf = this->private;
+    fd_ctx = dht_fd_ctx_get(this, fd);
 
-        sprintf(sid, "%d", client_id);
-        if (dict_get_ptr(conf->leaf_to_subvol, sid, (void **) &xl))
-                xl = NULL;
+    if (fd_ctx) {
+        if (fd_ctx->opened_on_dst == (uint64_t)(uintptr_t)dst) {
+            opened = _gf_true;
+        }
+        GF_REF_PUT(fd_ctx);
+    }
 
-        return xl;
+    return opened;
 }
 
-int
-dht_deitransform (xlator_t *this, uint64_t y, xlator_t **subvol_p)
+void
+dht_free_mig_info(void *data)
 {
-        int         client_id = 0;
-        xlator_t   *subvol = 0;
-        dht_conf_t *conf = NULL;
+    dht_migrate_info_t *miginfo = NULL;
+
+    miginfo = data;
+    GF_FREE(miginfo);
 
-        if (!this->private)
-                return -1;
+    return;
+}
 
-        conf = this->private;
+static int
+dht_inode_ctx_set_mig_info(xlator_t *this, inode_t *inode, xlator_t *src_subvol,
+                           xlator_t *dst_subvol)
+{
+    dht_migrate_info_t *miginfo = NULL;
+    uint64_t value = 0;
+    int ret = -1;
 
-        client_id = gf_deitransform(this, y);
+    miginfo = GF_CALLOC(1, sizeof(*miginfo), gf_dht_mt_miginfo_t);
+    if (miginfo == NULL)
+        goto out;
 
-        subvol = dht_get_subvol_from_id(this, client_id);
+    miginfo->src_subvol = src_subvol;
+    miginfo->dst_subvol = dst_subvol;
+    GF_REF_INIT(miginfo, dht_free_mig_info);
 
-        if (!subvol)
-                subvol = conf->subvolumes[0];
+    value = (uint64_t)(uintptr_t)miginfo;
 
-        if (subvol_p)
-                *subvol_p = subvol;
+    ret = inode_ctx_set1(inode, this, &value);
+    if (ret < 0) {
+        GF_REF_PUT(miginfo);
+    }
 
-        return 0;
+out:
+    return ret;
 }
 
-char *
-dht_lock_asprintf (dht_lock_t *lock)
+int
+dht_inode_ctx_get_mig_info(xlator_t *this, inode_t *inode,
+                           xlator_t **src_subvol, xlator_t **dst_subvol)
 {
-        char *lk_buf                = NULL;
-        char gfid[GF_UUID_BUF_SIZE] = {0, };
+    int ret = -1;
+    uint64_t tmp_miginfo = 0;
+    dht_migrate_info_t *miginfo = NULL;
 
-        if (lock == NULL)
-                goto out;
+    LOCK(&inode->lock);
+    {
+        ret = __inode_ctx_get1(inode, this, &tmp_miginfo);
+        if ((ret < 0) || (tmp_miginfo == 0)) {
+            UNLOCK(&inode->lock);
+            goto out;
+        }
 
-        uuid_utoa_r (lock->loc.gfid, gfid);
+        miginfo = (dht_migrate_info_t *)(uintptr_t)tmp_miginfo;
+        GF_REF_GET(miginfo);
+    }
+    UNLOCK(&inode->lock);
 
-        gf_asprintf (&lk_buf, "%s:%s", lock->xl->name, gfid);
+    if (src_subvol)
+        *src_subvol = miginfo->src_subvol;
+
+    if (dst_subvol)
+        *dst_subvol = miginfo->dst_subvol;
+
+    GF_REF_PUT(miginfo);
 
 out:
-        return lk_buf;
+    return ret;
 }
 
-void
-dht_log_lk_array (char *name, gf_loglevel_t log_level, dht_lock_t **lk_array,
-                  int count)
+gf_boolean_t
+dht_mig_info_is_invalid(xlator_t *current, xlator_t *src_subvol,
+                        xlator_t *dst_subvol)
 {
-        int   i      = 0;
-        char *lk_buf = NULL;
+    /* Not set
+     */
+    if (!src_subvol || !dst_subvol)
+        return _gf_true;
+
+    /* Invalid scenarios:
+     * The src_subvol does not match the subvol on which the current op was sent
+     * so the cached subvol has changed between the last mig_info_set and now.
+     * src_subvol == dst_subvol. The file was migrated without any FOP detecting
+     * a P2 so the old dst is now the current subvol.
+     *
+     * There is still one scenario where the info could be outdated - if
+     * file has undergone multiple migrations and ends up on the same src_subvol
+     * on which the mig_info was first set.
+     */
+    if ((current == dst_subvol) || (current != src_subvol))
+        return _gf_true;
+
+    return _gf_false;
+}
 
-        if ((lk_array == NULL) || (count == 0))
-                goto out;
+/* Used to check if fd fops have the fd opened on the cached subvol
+ * This is required when:
+ * 1. an fd is opened on FILE1 on subvol1
+ * 2. the file is migrated to subvol2
+ * 3. a lookup updates the cached subvol in the inode_ctx to subvol2
+ * 4. a write comes on the fd
+ * The write is sent to subvol2 on an fd which has been opened only on fd1
+ * Since the migration phase checks don't kick in, the fop fails with EBADF
+ *
+ */
 
-        for (i = 0; i < count; i++) {
-                lk_buf = dht_lock_asprintf (lk_array[i]);
-                gf_log (name, log_level, "%d. %s", i, lk_buf);
-                GF_FREE (lk_buf);
-        }
+int
+dht_check_and_open_fd_on_subvol_complete(int ret, call_frame_t *frame,
+                                         void *data)
+{
+    glusterfs_fop_t fop = 0;
+    dht_local_t *local = NULL;
+    xlator_t *subvol = NULL;
+    xlator_t *this = NULL;
+    fd_t *fd = NULL;
+    int op_errno = -1;
+
+    local = frame->local;
+    this = frame->this;
+    fop = local->fop;
+    subvol = local->cached_subvol;
+    fd = local->fd;
+
+    if (ret) {
+        op_errno = local->op_errno;
+        goto handle_err;
+    }
+
+    switch (fop) {
+        case GF_FOP_WRITE:
+            STACK_WIND_COOKIE(frame, dht_writev_cbk, subvol, subvol,
+                              subvol->fops->writev, fd, local->rebalance.vector,
+                              local->rebalance.count, local->rebalance.offset,
+                              local->rebalance.flags, local->rebalance.iobref,
+                              local->xattr_req);
+            break;
+
+        case GF_FOP_FLUSH:
+            STACK_WIND(frame, dht_flush_cbk, subvol, subvol->fops->flush, fd,
+                       local->xattr_req);
+            break;
+
+        case GF_FOP_FSETATTR:
+            STACK_WIND_COOKIE(frame, dht_file_setattr_cbk, subvol, subvol,
+                              subvol->fops->fsetattr, fd,
+                              &local->rebalance.stbuf, local->rebalance.flags,
+                              local->xattr_req);
+            break;
+
+        case GF_FOP_ZEROFILL:
+            STACK_WIND_COOKIE(frame, dht_zerofill_cbk, subvol, subvol,
+                              subvol->fops->zerofill, fd,
+                              local->rebalance.offset, local->rebalance.size,
+                              local->xattr_req);
+
+            break;
+
+        case GF_FOP_DISCARD:
+            STACK_WIND_COOKIE(frame, dht_discard_cbk, subvol, subvol,
+                              subvol->fops->discard, local->fd,
+                              local->rebalance.offset, local->rebalance.size,
+                              local->xattr_req);
+            break;
+
+        case GF_FOP_FALLOCATE:
+            STACK_WIND_COOKIE(frame, dht_fallocate_cbk, subvol, subvol,
+                              subvol->fops->fallocate, fd,
+                              local->rebalance.flags, local->rebalance.offset,
+                              local->rebalance.size, local->xattr_req);
+            break;
+
+        case GF_FOP_FTRUNCATE:
+            STACK_WIND_COOKIE(frame, dht_truncate_cbk, subvol, subvol,
+                              subvol->fops->ftruncate, fd,
+                              local->rebalance.offset, local->xattr_req);
+            break;
+
+        case GF_FOP_FSYNC:
+            STACK_WIND_COOKIE(frame, dht_fsync_cbk, subvol, subvol,
+                              subvol->fops->fsync, local->fd,
+                              local->rebalance.flags, local->xattr_req);
+            break;
+
+        case GF_FOP_READ:
+            STACK_WIND(frame, dht_readv_cbk, subvol, subvol->fops->readv,
+                       local->fd, local->rebalance.size,
+                       local->rebalance.offset, local->rebalance.flags,
+                       local->xattr_req);
+            break;
+
+        case GF_FOP_FSTAT:
+            STACK_WIND_COOKIE(frame, dht_file_attr_cbk, subvol, subvol,
+                              subvol->fops->fstat, fd, local->xattr_req);
+            break;
+
+        case GF_FOP_FSETXATTR:
+            STACK_WIND_COOKIE(frame, dht_file_setxattr_cbk, subvol, subvol,
+                              subvol->fops->fsetxattr, local->fd,
+                              local->rebalance.xattr, local->rebalance.flags,
+                              local->xattr_req);
+            break;
+
+        case GF_FOP_FREMOVEXATTR:
+            STACK_WIND_COOKIE(frame, dht_file_removexattr_cbk, subvol, subvol,
+                              subvol->fops->fremovexattr, local->fd, local->key,
+                              local->xattr_req);
+
+            break;
+
+        case GF_FOP_FXATTROP:
+            STACK_WIND(frame, dht_common_xattrop_cbk, subvol,
+                       subvol->fops->fxattrop, local->fd,
+                       local->rebalance.flags, local->rebalance.xattr,
+                       local->xattr_req);
+            break;
+
+        case GF_FOP_FGETXATTR:
+            STACK_WIND(frame, dht_getxattr_cbk, subvol, subvol->fops->fgetxattr,
+                       local->fd, local->key, NULL);
+            break;
+
+        case GF_FOP_FINODELK:
+            STACK_WIND(frame, dht_finodelk_cbk, subvol, subvol->fops->finodelk,
+                       local->key, local->fd, local->rebalance.lock_cmd,
+                       &local->rebalance.flock, local->xattr_req);
+            break;
+        default:
+            gf_smsg(this->name, GF_LOG_ERROR, 0, DHT_MSG_UNKNOWN_FOP, "fd=%p",
+                    fd, "gfid=%s", uuid_utoa(fd->inode->gfid), "name=%s",
+                    subvol->name, NULL);
+            break;
+    }
+
+    goto out;
+
+    /* Could not open the fd on the dst. Unwind */
+
+handle_err:
+
+    switch (fop) {
+        case GF_FOP_WRITE:
+            DHT_STACK_UNWIND(writev, frame, -1, op_errno, NULL, NULL, NULL);
+            break;
+
+        case GF_FOP_FLUSH:
+            DHT_STACK_UNWIND(flush, frame, -1, op_errno, NULL);
+            break;
+
+        case GF_FOP_FSETATTR:
+            DHT_STACK_UNWIND(fsetattr, frame, -1, op_errno, NULL, NULL, NULL);
+            break;
+
+        case GF_FOP_ZEROFILL:
+            DHT_STACK_UNWIND(zerofill, frame, -1, op_errno, NULL, NULL, NULL);
+            break;
+
+        case GF_FOP_DISCARD:
+            DHT_STACK_UNWIND(discard, frame, -1, op_errno, NULL, NULL, NULL);
+            break;
+
+        case GF_FOP_FALLOCATE:
+            DHT_STACK_UNWIND(fallocate, frame, -1, op_errno, NULL, NULL, NULL);
+            break;
+
+        case GF_FOP_FTRUNCATE:
+            DHT_STACK_UNWIND(ftruncate, frame, -1, op_errno, NULL, NULL, NULL);
+            break;
+
+        case GF_FOP_FSYNC:
+            DHT_STACK_UNWIND(fsync, frame, -1, op_errno, NULL, NULL, NULL);
+            break;
+
+        case GF_FOP_READ:
+            DHT_STACK_UNWIND(readv, frame, -1, op_errno, NULL, 0, NULL, NULL,
+                             NULL);
+            break;
+
+        case GF_FOP_FSTAT:
+            DHT_STACK_UNWIND(fstat, frame, -1, op_errno, NULL, NULL);
+            break;
+
+        case GF_FOP_FSETXATTR:
+            DHT_STACK_UNWIND(fsetxattr, frame, -1, op_errno, NULL);
+            break;
+
+        case GF_FOP_FREMOVEXATTR:
+            DHT_STACK_UNWIND(fremovexattr, frame, -1, op_errno, NULL);
+            break;
+
+        case GF_FOP_FXATTROP:
+            DHT_STACK_UNWIND(fxattrop, frame, -1, op_errno, NULL, NULL);
+            break;
+
+        case GF_FOP_FGETXATTR:
+            DHT_STACK_UNWIND(fgetxattr, frame, -1, op_errno, NULL, NULL);
+            break;
+
+        case GF_FOP_FINODELK:
+            DHT_STACK_UNWIND(finodelk, frame, -1, op_errno, NULL);
+            break;
+
+        default:
+            gf_smsg(this->name, GF_LOG_ERROR, 0, DHT_MSG_UNKNOWN_FOP, "fd=%p",
+                    fd, "gfid=%s", uuid_utoa(fd->inode->gfid), "name=%s",
+                    subvol->name, NULL);
+            break;
+    }
 
 out:
-        return;
+
+    return 0;
 }
 
-void
-dht_lock_stack_destroy (call_frame_t *lock_frame)
+/* Check once again if the fd has been opened on the cached subvol.
+ * If not, open and update the fd_ctx.
+ */
+
+int
+dht_check_and_open_fd_on_subvol_task(void *data)
 {
-        dht_local_t *local = NULL;
+    loc_t loc = {
+        0,
+    };
+    int ret = -1;
+    call_frame_t *frame = NULL;
+    dht_local_t *local = NULL;
+    fd_t *fd = NULL;
+    xlator_t *this = NULL;
+    xlator_t *subvol = NULL;
+
+    frame = data;
+    local = frame->local;
+    this = THIS;
+    fd = local->fd;
+    subvol = local->cached_subvol;
+
+    local->fd_checked = _gf_true;
+
+    if (fd_is_anonymous(fd) || dht_fd_open_on_dst(this, fd, subvol)) {
+        ret = 0;
+        goto out;
+    }
 
-        local = lock_frame->local;
+    gf_msg_debug(this->name, 0, "Opening fd (%p, flags=0%o) on file %s @ %s",
+                 fd, fd->flags, uuid_utoa(fd->inode->gfid), subvol->name);
 
-        local->lock.locks = NULL;
-        local->lock.lk_count = 0;
+    loc.inode = inode_ref(fd->inode);
+    gf_uuid_copy(loc.gfid, fd->inode->gfid);
 
-        DHT_STACK_DESTROY (lock_frame);
-        return;
+    /* Open this on the dst subvol */
+
+    SYNCTASK_SETID(0, 0);
+
+    ret = syncop_open(subvol, &loc, (fd->flags & ~(O_CREAT | O_EXCL | O_TRUNC)),
+                      fd, NULL, NULL);
+
+    if (ret < 0) {
+        gf_smsg(this->name, GF_LOG_ERROR, -ret, DHT_MSG_OPEN_FD_ON_DST_FAILED,
+                "fd=%p", fd, "flags=0%o", fd->flags, "gfid=%s",
+                uuid_utoa(fd->inode->gfid), "name=%s", subvol->name, NULL);
+        /* This can happen if the cached subvol was updated in the
+         * inode_ctx and the fd was opened on the new cached suvol
+         * after this fop was wound on the old cached subvol.
+         * As we do not close the fd on the old subvol (a leak)
+         * don't treat ENOENT as an error and allow the phase1/phase2
+         * checks to handle it.
+         */
+
+        if ((-ret != ENOENT) && (-ret != ESTALE)) {
+            local->op_errno = -ret;
+            ret = -1;
+        } else {
+            ret = 0;
+        }
+
+        local->op_errno = -ret;
+        ret = -1;
+
+    } else {
+        dht_fd_ctx_set(this, fd, subvol);
+    }
+
+    SYNCTASK_SETID(frame->root->uid, frame->root->gid);
+out:
+    loc_wipe(&loc);
+
+    return ret;
 }
 
-void
-dht_lock_free (dht_lock_t *lock)
+int
+dht_check_and_open_fd_on_subvol(xlator_t *this, call_frame_t *frame)
 {
-        if (lock == NULL)
-                goto out;
+    int ret = -1;
+    dht_local_t *local = NULL;
 
-        loc_wipe (&lock->loc);
-        GF_FREE (lock->domain);
-        mem_put (lock);
+    /*
+            if (dht_fd_open_on_dst (this, fd, subvol))
+                    goto out;
+    */
+    local = frame->local;
 
-out:
-        return;
+    ret = synctask_new(this->ctx->env, dht_check_and_open_fd_on_subvol_task,
+                       dht_check_and_open_fd_on_subvol_complete, frame, frame);
+
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_ERROR, 0, DHT_MSG_SYNCTASK_CREATE_FAILED,
+                "to-check-and-open fd=%p", local->fd, NULL);
+    }
+
+    return ret;
 }
 
-void
-dht_lock_array_free (dht_lock_t **lk_array, int count)
+int
+dht_frame_return(call_frame_t *frame)
 {
-        int            i       = 0;
-        dht_lock_t    *lock    = NULL;
+    dht_local_t *local = NULL;
+    int this_call_cnt = -1;
 
-        if (lk_array == NULL)
-                goto out;
+    if (!frame)
+        return -1;
 
-        for (i = 0; i < count; i++) {
-                lock = lk_array[i];
-                lk_array[i] = NULL;
-                dht_lock_free (lock);
-        }
+    local = frame->local;
+
+    LOCK(&frame->lock);
+    {
+        this_call_cnt = --local->call_cnt;
+    }
+    UNLOCK(&frame->lock);
 
+    return this_call_cnt;
+}
+
+/*
+ * Use this function to specify which subvol you want the file created
+ * on - this need not be the hashed subvol.
+ * Format: <filename>@<this->name>:<subvol-name>
+ * Eg: file-1@vol1-dht:vol1-client-0
+ *     where vol1 is a pure distribute volume
+ *     will create file-1 on vol1-client-0
+ */
+
+int
+dht_filter_loc_subvol_key(xlator_t *this, loc_t *loc, loc_t *new_loc,
+                          xlator_t **subvol)
+{
+    char *new_name = NULL;
+    char *new_path = NULL;
+    xlator_list_t *trav = NULL;
+    char key[1024] = {
+        0,
+    };
+    int ret = 0; /* not found */
+    int keylen = 0;
+    int name_len = 0;
+    int path_len = 0;
+
+    /* Why do other tasks if first required 'char' itself is not there */
+    if (!new_loc || !loc || !loc->name || !strchr(loc->name, '@')) {
+        /* Skip the GF_FREE checks here */
+        return ret;
+    }
+
+    trav = this->children;
+    while (trav) {
+        keylen = snprintf(key, sizeof(key), "*@%s:%s", this->name,
+                          trav->xlator->name);
+        /* Ignore '*' */
+        keylen = keylen - 1;
+        if (fnmatch(key, loc->name, FNM_NOESCAPE) == 0) {
+            name_len = strlen(loc->name) - keylen;
+            new_name = GF_MALLOC(name_len + 1, gf_common_mt_char);
+            if (!new_name)
+                goto out;
+            if (fnmatch(key, loc->path, FNM_NOESCAPE) == 0) {
+                path_len = strlen(loc->path) - keylen;
+                new_path = GF_MALLOC(path_len + 1, gf_common_mt_char);
+                if (!new_path)
+                    goto out;
+                snprintf(new_path, path_len + 1, "%s", loc->path);
+            }
+            snprintf(new_name, name_len + 1, "%s", loc->name);
+
+            if (new_loc) {
+                new_loc->path = ((new_path) ? new_path : gf_strdup(loc->path));
+                new_loc->name = new_name;
+                new_loc->inode = inode_ref(loc->inode);
+                new_loc->parent = inode_ref(loc->parent);
+            }
+            *subvol = trav->xlator;
+            ret = 1; /* success */
+            goto out;
+        }
+        trav = trav->next;
+    }
 out:
-        return;
+    if (!ret) {
+        /* !success */
+        GF_FREE(new_path);
+        GF_FREE(new_name);
+    }
+    return ret;
 }
 
-dht_lock_t *
-dht_lock_new (xlator_t *this, xlator_t *xl, loc_t *loc, short type,
-              const char *domain)
+static xlator_t *
+dht_get_subvol_from_id(xlator_t *this, int client_id)
 {
-        dht_conf_t *conf = NULL;
-        dht_lock_t *lock = NULL;
+    xlator_t *xl = NULL;
+    dht_conf_t *conf = NULL;
+    char *sid = NULL;
+    int32_t ret = -1;
 
-        conf = this->private;
+    conf = this->private;
 
-        lock = mem_get0 (conf->lock_pool);
-        if (lock == NULL)
-                goto out;
+    ret = gf_asprintf(&sid, "%d", client_id);
+    if (ret == -1) {
+        gf_smsg(this->name, GF_LOG_ERROR, 0, DHT_MSG_ASPRINTF_FAILED, NULL);
+        goto out;
+    }
 
-        lock->xl = xl;
-        lock->type = type;
-        lock->domain = gf_strdup (domain);
-        if (lock->domain == NULL) {
-                dht_lock_free (lock);
-                lock = NULL;
-                goto out;
-        }
+    if (dict_get_ptr(conf->leaf_to_subvol, sid, (void **)&xl))
+        xl = NULL;
 
-        /* Fill only inode and gfid.
-           posix and protocol/server give preference to pargfid/basename over
-           gfid/inode for resolution if all the three parameters of loc_t are
-           present. I want to avoid the following hypothetical situation:
-
-           1. rebalance did a lookup on a dentry and got a gfid.
-           2. rebalance acquires lock on loc_t which was filled with gfid and
-              path (pargfid/bname) from step 1.
-           3. somebody deleted and recreated the same file
-           4. rename on the same path acquires lock on loc_t which now points
-              to a different inode (and hence gets the lock).
-           5. rebalance continues to migrate file (note that not all fops done
-              by rebalance during migration are inode/gfid based Eg., unlink)
-           6. rename continues.
-        */
-        lock->loc.inode = inode_ref (loc->inode);
-        loc_gfid (loc, lock->loc.gfid);
+    GF_FREE(sid);
 
 out:
-        return lock;
+    return xl;
 }
 
 int
-dht_local_lock_init (call_frame_t *frame, dht_lock_t **lk_array,
-                     int lk_count, fop_inodelk_cbk_t inodelk_cbk)
+dht_deitransform(xlator_t *this, uint64_t y, xlator_t **subvol_p)
 {
-        int          ret   = -1;
-        dht_local_t *local = NULL;
+    int client_id = 0;
+    xlator_t *subvol = 0;
+    dht_conf_t *conf = NULL;
 
-        local = frame->local;
+    if (!this->private)
+        return -1;
 
-        if (local == NULL) {
-                local = dht_local_init (frame, NULL, NULL, 0);
-        }
+    conf = this->private;
 
-        if (local == NULL) {
-                goto out;
-        }
+    client_id = gf_deitransform(this, y);
 
-        local->lock.inodelk_cbk = inodelk_cbk;
-        local->lock.locks = lk_array;
-        local->lock.lk_count = lk_count;
+    subvol = dht_get_subvol_from_id(this, client_id);
 
-        ret = dht_lock_order_requests (local->lock.locks,
-                                       local->lock.lk_count);
-        if (ret < 0)
-                goto out;
+    if (!subvol)
+        subvol = conf->subvolumes[0];
 
-        ret = 0;
-out:
-        return ret;
+    if (subvol_p)
+        *subvol_p = subvol;
+
+    return 0;
 }
 
 void
-dht_local_wipe (xlator_t *this, dht_local_t *local)
+dht_local_wipe(xlator_t *this, dht_local_t *local)
 {
-        if (!local)
-                return;
+    int i = 0;
 
-        loc_wipe (&local->loc);
-        loc_wipe (&local->loc2);
+    if (!local)
+        return;
 
-        if (local->xattr)
-                dict_unref (local->xattr);
+    loc_wipe(&local->loc);
+    loc_wipe(&local->loc2);
+    loc_wipe(&local->loc2_copy);
 
-        if (local->inode)
-                inode_unref (local->inode);
+    if (local->xattr)
+        dict_unref(local->xattr);
 
-        if (local->layout) {
-                dht_layout_unref (this, local->layout);
-                local->layout = NULL;
-        }
+    if (local->inode)
+        inode_unref(local->inode);
 
-        loc_wipe (&local->linkfile.loc);
+    if (local->layout) {
+        dht_layout_unref(this, local->layout);
+        local->layout = NULL;
+    }
 
-        if (local->linkfile.xattr)
-                dict_unref (local->linkfile.xattr);
+    loc_wipe(&local->linkfile.loc);
 
-        if (local->linkfile.inode)
-                inode_unref (local->linkfile.inode);
+    if (local->linkfile.xattr)
+        dict_unref(local->linkfile.xattr);
 
-        if (local->fd) {
-                fd_unref (local->fd);
-                local->fd = NULL;
-        }
+    if (local->linkfile.inode)
+        inode_unref(local->linkfile.inode);
 
-        if (local->params) {
-                dict_unref (local->params);
-                local->params = NULL;
-        }
+    if (local->fd) {
+        fd_unref(local->fd);
+        local->fd = NULL;
+    }
 
-        if (local->xattr_req)
-                dict_unref (local->xattr_req);
+    if (local->params) {
+        dict_unref(local->params);
+        local->params = NULL;
+    }
 
-        if (local->selfheal.layout) {
-                dht_layout_unref (this, local->selfheal.layout);
-                local->selfheal.layout = NULL;
-        }
+    if (local->xattr_req)
+        dict_unref(local->xattr_req);
+    if (local->mds_xattr)
+        dict_unref(local->mds_xattr);
+    if (local->xdata)
+        dict_unref(local->xdata);
 
-        dht_lock_array_free (local->lock.locks, local->lock.lk_count);
-        GF_FREE (local->lock.locks);
+    if (local->selfheal.layout) {
+        dht_layout_unref(this, local->selfheal.layout);
+        local->selfheal.layout = NULL;
+    }
 
-        GF_FREE (local->newpath);
+    if (local->selfheal.refreshed_layout) {
+        dht_layout_unref(this, local->selfheal.refreshed_layout);
+        local->selfheal.refreshed_layout = NULL;
+    }
 
-        GF_FREE (local->key);
+    for (i = 0; i < 2; i++) {
+        dht_lock_array_free(local->lock[i].ns.parent_layout.locks,
+                            local->lock[i].ns.parent_layout.lk_count);
 
-        GF_FREE (local->rebalance.vector);
+        GF_FREE(local->lock[i].ns.parent_layout.locks);
 
-        if (local->rebalance.iobref)
-                iobref_unref (local->rebalance.iobref);
+        dht_lock_array_free(local->lock[i].ns.directory_ns.locks,
+                            local->lock[i].ns.directory_ns.lk_count);
+        GF_FREE(local->lock[i].ns.directory_ns.locks);
+    }
 
-        mem_put (local);
-}
+    GF_FREE(local->key);
+
+    if (local->rebalance.xdata)
+        dict_unref(local->rebalance.xdata);
+
+    if (local->rebalance.xattr)
+        dict_unref(local->rebalance.xattr);
+
+    if (local->rebalance.dict)
+        dict_unref(local->rebalance.dict);
 
+    GF_FREE(local->rebalance.vector);
+
+    if (local->rebalance.iobref)
+        iobref_unref(local->rebalance.iobref);
+
+    if (local->stub) {
+        call_stub_destroy(local->stub);
+        local->stub = NULL;
+    }
+
+    if (local->ret_cache)
+        GF_FREE(local->ret_cache);
+
+    mem_put(local);
+}
 
 dht_local_t *
-dht_local_init (call_frame_t *frame, loc_t *loc, fd_t *fd, glusterfs_fop_t fop)
+dht_local_init(call_frame_t *frame, loc_t *loc, fd_t *fd, glusterfs_fop_t fop)
 {
-        dht_local_t *local = NULL;
-        inode_t     *inode = NULL;
-        int          ret   = 0;
+    dht_local_t *local = NULL;
+    inode_t *inode = NULL;
+    int ret = 0;
 
-        local = mem_get0 (THIS->local_pool);
-        if (!local)
-                goto out;
+    local = mem_get0(THIS->local_pool);
+    if (!local)
+        goto out;
 
-        if (loc) {
-                ret = loc_copy (&local->loc, loc);
-                if (ret)
-                        goto out;
+    if (loc) {
+        ret = loc_copy(&local->loc, loc);
+        if (ret)
+            goto out;
 
-                inode = loc->inode;
-        }
+        inode = loc->inode;
+    }
 
-        if (fd) {
-                local->fd = fd_ref (fd);
-                if (!inode)
-                        inode = fd->inode;
-        }
+    if (fd) {
+        local->fd = fd_ref(fd);
+        if (!inode)
+            inode = fd->inode;
+    }
 
-        local->op_ret   = -1;
-        local->op_errno = EUCLEAN;
-        local->fop      = fop;
+    local->op_ret = -1;
+    local->op_errno = EUCLEAN;
+    local->fop = fop;
 
-        if (inode) {
-                local->layout   = dht_layout_get (frame->this, inode);
-                local->cached_subvol = dht_subvol_get_cached (frame->this,
-                                                              inode);
-        }
+    if (inode) {
+        local->layout = dht_layout_get(frame->this, inode);
+        local->cached_subvol = dht_subvol_get_cached(frame->this, inode);
+    }
 
-        frame->local = local;
+    frame->local = local;
 
 out:
-        if (ret) {
-                if (local)
-                        mem_put (local);
-                local = NULL;
-        }
-        return local;
+    if (ret) {
+        if (local)
+            mem_put(local);
+        local = NULL;
+    }
+    return local;
 }
 
 xlator_t *
-dht_first_up_subvol (xlator_t *this)
+dht_first_up_subvol(xlator_t *this)
 {
-        dht_conf_t *conf = NULL;
-        xlator_t   *child = NULL;
-        int         i = 0;
-        time_t      time = 0;
+    dht_conf_t *conf = NULL;
+    xlator_t *child = NULL;
+    int i = 0;
+    time_t time = 0;
 
-        conf = this->private;
-        if (!conf)
-                goto out;
+    conf = this->private;
+    if (!conf)
+        goto out;
 
-        LOCK (&conf->subvolume_lock);
-        {
-                for (i = 0; i < conf->subvolume_cnt; i++) {
-                        if (conf->subvol_up_time[i]) {
-                                if (!time) {
-                                        time = conf->subvol_up_time[i];
-                                        child = conf->subvolumes[i];
-                                } else if (time > conf->subvol_up_time[i]) {
-                                        time  = conf->subvol_up_time[i];
-                                        child = conf->subvolumes[i];
-                                }
-                        }
+    LOCK(&conf->subvolume_lock);
+    {
+        for (i = 0; i < conf->subvolume_cnt; i++) {
+            if (conf->subvol_up_time[i]) {
+                if (!time) {
+                    time = conf->subvol_up_time[i];
+                    child = conf->subvolumes[i];
+                } else if (time > conf->subvol_up_time[i]) {
+                    time = conf->subvol_up_time[i];
+                    child = conf->subvolumes[i];
                 }
+            }
         }
-        UNLOCK (&conf->subvolume_lock);
+    }
+    UNLOCK(&conf->subvolume_lock);
 
 out:
-        return child;
+    return child;
 }
 
 xlator_t *
-dht_last_up_subvol (xlator_t *this)
+dht_last_up_subvol(xlator_t *this)
 {
-        dht_conf_t *conf = NULL;
-        xlator_t   *child = NULL;
-        int         i = 0;
+    dht_conf_t *conf = NULL;
+    xlator_t *child = NULL;
+    int i = 0;
 
-        conf = this->private;
-        if (!conf)
-                goto out;
+    conf = this->private;
+    if (!conf)
+        goto out;
 
-        LOCK (&conf->subvolume_lock);
-        {
-                for (i = conf->subvolume_cnt-1; i >= 0; i--) {
-                        if (conf->subvolume_status[i]) {
-                                child = conf->subvolumes[i];
-                                break;
-                        }
-                }
+    LOCK(&conf->subvolume_lock);
+    {
+        for (i = conf->subvolume_cnt - 1; i >= 0; i--) {
+            if (conf->subvolume_status[i]) {
+                child = conf->subvolumes[i];
+                break;
+            }
         }
-        UNLOCK (&conf->subvolume_lock);
+    }
+    UNLOCK(&conf->subvolume_lock);
 
 out:
-        return child;
+    return child;
 }
 
 xlator_t *
-dht_subvol_get_hashed (xlator_t *this, loc_t *loc)
+dht_subvol_get_hashed(xlator_t *this, loc_t *loc)
 {
-        dht_layout_t *layout = NULL;
-        xlator_t     *subvol = NULL;
-        dht_conf_t *conf = NULL;
-        dht_methods_t *methods = NULL;
+    dht_layout_t *layout = NULL;
+    xlator_t *subvol = NULL;
+    dht_conf_t *conf = NULL;
+    dht_methods_t *methods = NULL;
 
-        GF_VALIDATE_OR_GOTO ("dht", this, out);
-        GF_VALIDATE_OR_GOTO (this->name, loc, out);
+    GF_VALIDATE_OR_GOTO("dht", this, out);
+    GF_VALIDATE_OR_GOTO(this->name, loc, out);
 
-        conf = this->private;
-        GF_VALIDATE_OR_GOTO (this->name, conf, out);
+    conf = this->private;
+    GF_VALIDATE_OR_GOTO(this->name, conf, out);
 
-        methods = conf->methods;
-        GF_VALIDATE_OR_GOTO (this->name, conf->methods, out);
+    methods = &(conf->methods);
 
-        if (__is_root_gfid (loc->gfid)) {
-                subvol = dht_first_up_subvol (this);
-                goto out;
-        }
+    if (__is_root_gfid(loc->gfid)) {
+        subvol = dht_first_up_subvol(this);
+        goto out;
+    }
 
-        GF_VALIDATE_OR_GOTO (this->name, loc->parent, out);
-        GF_VALIDATE_OR_GOTO (this->name, loc->name, out);
+    GF_VALIDATE_OR_GOTO(this->name, loc->parent, out);
+    GF_VALIDATE_OR_GOTO(this->name, loc->name, out);
 
-        layout = dht_layout_get (this, loc->parent);
+    layout = dht_layout_get(this, loc->parent);
 
-        if (!layout) {
-                gf_msg_debug (this->name, 0,
-                              "Missing layout. path=%s, parent gfid =%s",
-                              loc->path, uuid_utoa (loc->parent->gfid));
-                goto out;
-        }
+    if (!layout) {
+        gf_msg_debug(this->name, 0, "Missing layout. path=%s, parent gfid =%s",
+                     loc->path, uuid_utoa(loc->parent->gfid));
+        goto out;
+    }
 
-        subvol = methods->layout_search (this, layout, loc->name);
+    subvol = methods->layout_search(this, layout, loc->name);
 
-        if (!subvol) {
-                gf_msg_debug (this->name, 0,
-                              "No hashed subvolume for path=%s",
-                              loc->path);
-                goto out;
-        }
+    if (!subvol) {
+        gf_msg_debug(this->name, 0, "No hashed subvolume for path=%s",
+                     loc->path);
+        goto out;
+    }
 
 out:
-        if (layout) {
-                dht_layout_unref (this, layout);
-        }
+    if (layout) {
+        dht_layout_unref(this, layout);
+    }
 
-        return subvol;
+    return subvol;
 }
 
-
 xlator_t *
-dht_subvol_get_cached (xlator_t *this, inode_t *inode)
+dht_subvol_get_cached(xlator_t *this, inode_t *inode)
 {
-        dht_layout_t *layout = NULL;
-        xlator_t     *subvol = NULL;
+    dht_layout_t *layout = NULL;
+    xlator_t *subvol = NULL;
 
-        GF_VALIDATE_OR_GOTO (this->name, this, out);
-        GF_VALIDATE_OR_GOTO (this->name, inode, out);
+    GF_VALIDATE_OR_GOTO("dht", this, out);
+    GF_VALIDATE_OR_GOTO(this->name, inode, out);
 
-        layout = dht_layout_get (this, inode);
+    layout = dht_layout_get(this, inode);
 
-        if (!layout) {
-                goto out;
-        }
+    if (!layout) {
+        goto out;
+    }
 
-        subvol = layout->list[0].xlator;
+    subvol = layout->list[0].xlator;
 
 out:
-        if (layout) {
-                dht_layout_unref (this, layout);
-        }
+    if (layout) {
+        dht_layout_unref(this, layout);
+    }
 
-        return subvol;
+    return subvol;
 }
 
-
 xlator_t *
-dht_subvol_next (xlator_t *this, xlator_t *prev)
+dht_subvol_next(xlator_t *this, xlator_t *prev)
 {
-        dht_conf_t *conf = NULL;
-        int         i = 0;
-        xlator_t   *next = NULL;
+    dht_conf_t *conf = NULL;
+    int i = 0;
+    xlator_t *next = NULL;
 
-        conf = this->private;
-        if (!conf)
-                goto out;
+    conf = this->private;
+    if (!conf)
+        goto out;
 
-        for (i = 0; i < conf->subvolume_cnt; i++) {
-                if (conf->subvolumes[i] == prev) {
-                        if ((i + 1) < conf->subvolume_cnt)
-                                next = conf->subvolumes[i + 1];
-                        break;
-                }
+    for (i = 0; i < conf->subvolume_cnt; i++) {
+        if (conf->subvolumes[i] == prev) {
+            if ((i + 1) < conf->subvolume_cnt)
+                next = conf->subvolumes[i + 1];
+            break;
         }
+    }
 
 out:
-        return next;
+    return next;
 }
 
 /* This func wraps around, if prev is actually the last subvol.
  */
 xlator_t *
-dht_subvol_next_available (xlator_t *this, xlator_t *prev)
+dht_subvol_next_available(xlator_t *this, xlator_t *prev)
 {
-        dht_conf_t *conf = NULL;
-        int         i = 0;
-        xlator_t   *next = NULL;
-
-        conf = this->private;
-        if (!conf)
-                goto out;
-
-        for (i = 0; i < conf->subvolume_cnt; i++) {
-                if (conf->subvolumes[i] == prev) {
-                        /* if prev is last in conf->subvolumes, then wrap
-                         * around.
-                         */
-                        if ((i + 1) < conf->subvolume_cnt) {
-                                next = conf->subvolumes[i + 1];
-                        } else {
-                                next = conf->subvolumes[0];
-                        }
-                        break;
-                }
-        }
+    dht_conf_t *conf = NULL;
+    int i = 0;
+    xlator_t *next = NULL;
+
+    conf = this->private;
+    if (!conf)
+        goto out;
+
+    for (i = 0; i < conf->subvolume_cnt; i++) {
+        if (conf->subvolumes[i] == prev) {
+            /* if prev is last in conf->subvolumes, then wrap
+             * around.
+             */
+            if ((i + 1) < conf->subvolume_cnt) {
+                next = conf->subvolumes[i + 1];
+            } else {
+                next = conf->subvolumes[0];
+            }
+            break;
+        }
+    }
 
 out:
-        return next;
+    return next;
 }
 int
-dht_subvol_cnt (xlator_t *this, xlator_t *subvol)
+dht_subvol_cnt(xlator_t *this, xlator_t *subvol)
 {
-        int i = 0;
-        int ret = -1;
-        dht_conf_t *conf = NULL;
+    int i = 0;
+    int ret = -1;
+    dht_conf_t *conf = NULL;
 
-        conf = this->private;
-        if (!conf)
-                goto out;
+    conf = this->private;
+    if (!conf)
+        goto out;
 
-        for (i = 0; i < conf->subvolume_cnt; i++) {
-                if (subvol == conf->subvolumes[i]) {
-                        ret = i;
-                        break;
-                }
+    for (i = 0; i < conf->subvolume_cnt; i++) {
+        if (subvol == conf->subvolumes[i]) {
+            ret = i;
+            break;
         }
+    }
 
 out:
-        return ret;
+    return ret;
 }
 
+#define set_if_greater(a, b)                                                   \
+    do {                                                                       \
+        if ((a) < (b))                                                         \
+            (a) = (b);                                                         \
+    } while (0)
 
-#define set_if_greater(a, b) do {               \
-                if ((a) < (b))                  \
-                        (a) = (b);              \
-        } while (0)
+#define set_if_greater_time(a, an, b, bn)                                      \
+    do {                                                                       \
+        if (((a) < (b)) || (((a) == (b)) && ((an) < (bn)))) {                  \
+            (a) = (b);                                                         \
+            (an) = (bn);                                                       \
+        }                                                                      \
+    } while (0)
 
+int
+dht_iatt_merge(xlator_t *this, struct iatt *to, struct iatt *from)
+{
+    if (!from || !to)
+        return 0;
 
-#define set_if_greater_time(a, an, b, bn) do {                          \
-                if (((a) < (b)) || (((a) == (b)) && ((an) < (bn)))){    \
-                        (a) = (b);                                      \
-                        (an) = (bn);                                    \
-                }                                                       \
-        } while (0)                                                     \
-
+    to->ia_dev = from->ia_dev;
+
+    gf_uuid_copy(to->ia_gfid, from->ia_gfid);
+
+    to->ia_ino = from->ia_ino;
+    to->ia_prot = from->ia_prot;
+    to->ia_type = from->ia_type;
+    to->ia_nlink = from->ia_nlink;
+    to->ia_rdev = from->ia_rdev;
+    to->ia_size += from->ia_size;
+    to->ia_blksize = from->ia_blksize;
+    to->ia_blocks += from->ia_blocks;
+
+    if (IA_ISDIR(from->ia_type)) {
+        to->ia_blocks = DHT_DIR_STAT_BLOCKS;
+        to->ia_size = DHT_DIR_STAT_SIZE;
+    }
+    set_if_greater(to->ia_uid, from->ia_uid);
+    set_if_greater(to->ia_gid, from->ia_gid);
+
+    set_if_greater_time(to->ia_atime, to->ia_atime_nsec, from->ia_atime,
+                        from->ia_atime_nsec);
+    set_if_greater_time(to->ia_mtime, to->ia_mtime_nsec, from->ia_mtime,
+                        from->ia_mtime_nsec);
+    set_if_greater_time(to->ia_ctime, to->ia_ctime_nsec, from->ia_ctime,
+                        from->ia_ctime_nsec);
+
+    return 0;
+}
 
 int
-dht_iatt_merge (xlator_t *this, struct iatt *to,
-                struct iatt *from, xlator_t *subvol)
+dht_build_child_loc(xlator_t *this, loc_t *child, loc_t *parent, char *name)
 {
-        if (!from || !to)
-                return 0;
+    if (!child) {
+        goto err;
+    }
 
-        to->ia_dev      = from->ia_dev;
+    if (strcmp(parent->path, "/") == 0)
+        gf_asprintf((char **)&child->path, "/%s", name);
+    else
+        gf_asprintf((char **)&child->path, "%s/%s", parent->path, name);
 
-        gf_uuid_copy (to->ia_gfid, from->ia_gfid);
+    if (!child->path) {
+        goto err;
+    }
 
-        to->ia_ino      = from->ia_ino;
-        to->ia_prot     = from->ia_prot;
-        to->ia_type     = from->ia_type;
-        to->ia_nlink    = from->ia_nlink;
-        to->ia_rdev     = from->ia_rdev;
-        to->ia_size    += from->ia_size;
-        to->ia_blksize  = from->ia_blksize;
-        to->ia_blocks  += from->ia_blocks;
+    child->name = strrchr(child->path, '/');
+    if (child->name)
+        child->name++;
 
-        set_if_greater (to->ia_uid, from->ia_uid);
-        set_if_greater (to->ia_gid, from->ia_gid);
+    child->parent = inode_ref(parent->inode);
+    child->inode = inode_new(parent->inode->table);
 
-        set_if_greater_time(to->ia_atime, to->ia_atime_nsec,
-                            from->ia_atime, from->ia_atime_nsec);
-        set_if_greater_time (to->ia_mtime, to->ia_mtime_nsec,
-                             from->ia_mtime, from->ia_mtime_nsec);
-        set_if_greater_time (to->ia_ctime, to->ia_ctime_nsec,
-                             from->ia_ctime, from->ia_ctime_nsec);
+    if (!child->inode) {
+        goto err;
+    }
 
-        return 0;
+    return 0;
+err:
+    if (child) {
+        loc_wipe(child);
+    }
+    return -1;
 }
 
 int
-dht_build_child_loc (xlator_t *this, loc_t *child, loc_t *parent, char *name)
+dht_init_local_subvolumes(xlator_t *this, dht_conf_t *conf)
 {
-        if (!child) {
-                goto err;
-        }
+    xlator_list_t *subvols = NULL;
+    int cnt = 0;
 
-        if (strcmp (parent->path, "/") == 0)
-                gf_asprintf ((char **)&child->path, "/%s", name);
-        else
-                gf_asprintf ((char **)&child->path, "%s/%s", parent->path, name);
-
-        if (!child->path) {
-                goto err;
-        }
+    if (!conf)
+        return -1;
 
-        child->name = strrchr (child->path, '/');
-        if (child->name)
-                child->name++;
+    for (subvols = this->children; subvols; subvols = subvols->next)
+        cnt++;
 
-        child->parent = inode_ref (parent->inode);
-        child->inode = inode_new (parent->inode->table);
+    conf->local_subvols = GF_CALLOC(cnt, sizeof(xlator_t *),
+                                    gf_dht_mt_xlator_t);
 
-        if (!child->inode) {
-                goto err;
-        }
+    /* FIX FIX : do this dynamically*/
+    conf->local_nodeuuids = GF_CALLOC(cnt, sizeof(subvol_nodeuuids_info_t),
+                                      gf_dht_nodeuuids_t);
 
-        return 0;
-err:
-        loc_wipe (child);
+    if (!conf->local_subvols || !conf->local_nodeuuids) {
         return -1;
-}
+    }
 
+    conf->local_subvols_cnt = 0;
 
+    return 0;
+}
 
 int
-dht_init_subvolumes (xlator_t *this, dht_conf_t *conf)
+dht_init_subvolumes(xlator_t *this, dht_conf_t *conf)
 {
-        xlator_list_t *subvols = NULL;
-        int            cnt = 0;
+    xlator_list_t *subvols = NULL;
+    int cnt = 0;
 
-        if (!conf)
-                return -1;
+    if (!conf)
+        return -1;
 
-        for (subvols = this->children; subvols; subvols = subvols->next)
-                cnt++;
+    for (subvols = this->children; subvols; subvols = subvols->next)
+        cnt++;
 
-        conf->subvolumes = GF_CALLOC (cnt, sizeof (xlator_t *),
-                                      gf_dht_mt_xlator_t);
-        if (!conf->subvolumes) {
-                return -1;
-        }
-        conf->subvolume_cnt = cnt;
+    conf->subvolumes = GF_CALLOC(cnt, sizeof(xlator_t *), gf_dht_mt_xlator_t);
+    if (!conf->subvolumes) {
+        return -1;
+    }
+    conf->subvolume_cnt = cnt;
+    /* Doesn't make sense to do any dht layer tasks
+       if the subvol count is 1. Set it as pass_through */
+    if (cnt == 1)
+        this->pass_through = _gf_true;
 
-        dht_set_subvol_range(this);
+    conf->local_subvols_cnt = 0;
 
-        cnt = 0;
-        for (subvols = this->children; subvols; subvols = subvols->next)
-                conf->subvolumes[cnt++] = subvols->xlator;
+    dht_set_subvol_range(this);
 
-        conf->subvolume_status = GF_CALLOC (cnt, sizeof (char),
-                                            gf_dht_mt_char);
-        if (!conf->subvolume_status) {
-                return -1;
-        }
+    cnt = 0;
+    for (subvols = this->children; subvols; subvols = subvols->next)
+        conf->subvolumes[cnt++] = subvols->xlator;
 
-        conf->last_event = GF_CALLOC (cnt, sizeof (int),
-                                      gf_dht_mt_char);
-        if (!conf->last_event) {
-                return -1;
-        }
-
-        conf->subvol_up_time = GF_CALLOC (cnt, sizeof (time_t),
-                                          gf_dht_mt_subvol_time);
-        if (!conf->subvol_up_time) {
-                return -1;
-        }
+    conf->subvolume_status = GF_CALLOC(cnt, sizeof(char), gf_dht_mt_char);
+    if (!conf->subvolume_status) {
+        return -1;
+    }
 
-        conf->du_stats = GF_CALLOC (conf->subvolume_cnt, sizeof (dht_du_t),
-                                    gf_dht_mt_dht_du_t);
-        if (!conf->du_stats) {
-                return -1;
-        }
+    conf->last_event = GF_CALLOC(cnt, sizeof(int), gf_dht_mt_char);
+    if (!conf->last_event) {
+        return -1;
+    }
 
-        conf->decommissioned_bricks = GF_CALLOC (cnt, sizeof (xlator_t *),
-                                                 gf_dht_mt_xlator_t);
-        if (!conf->decommissioned_bricks) {
-                return -1;
-        }
+    conf->subvol_up_time = GF_CALLOC(cnt, sizeof(time_t),
+                                     gf_dht_mt_subvol_time);
+    if (!conf->subvol_up_time) {
+        return -1;
+    }
 
-        return 0;
-}
+    conf->du_stats = GF_CALLOC(conf->subvolume_cnt, sizeof(dht_du_t),
+                               gf_dht_mt_dht_du_t);
+    if (!conf->du_stats) {
+        return -1;
+    }
 
+    conf->decommissioned_bricks = GF_CALLOC(cnt, sizeof(xlator_t *),
+                                            gf_dht_mt_xlator_t);
+    if (!conf->decommissioned_bricks) {
+        return -1;
+    }
 
+    return 0;
+}
 
+/*
+ op_ret values :
+  0 : Success.
+ -1 : Failure.
+  1 : File is being migrated but not by this DHT layer.
+*/
 
 static int
-dht_migration_complete_check_done (int op_ret, call_frame_t *frame, void *data)
+dht_migration_complete_check_done(int op_ret, call_frame_t *frame, void *data)
 {
-        dht_local_t *local = NULL;
+    dht_local_t *local = NULL;
+    xlator_t *subvol = NULL;
 
-        local = frame->local;
+    local = frame->local;
 
-        local->rebalance.target_op_fn (THIS, frame, op_ret);
+    if (op_ret != 0)
+        goto out;
 
-        return 0;
-}
+    if (local->cached_subvol == NULL) {
+        local->op_errno = EINVAL;
+        goto out;
+    }
 
+    subvol = local->cached_subvol;
 
-int
-dht_migration_complete_check_task (void *data)
-{
-        int           ret      = -1;
-        xlator_t     *src_node = NULL;
-        xlator_t     *dst_node = NULL;
-        dht_local_t  *local    = NULL;
-        dict_t       *dict     = NULL;
-        dht_layout_t *layout   = NULL;
-        struct iatt   stbuf    = {0,};
-        xlator_t     *this     = NULL;
-        call_frame_t *frame    = NULL;
-        loc_t         tmp_loc  = {0,};
-        char         *path     = NULL;
-        dht_conf_t   *conf     = NULL;
-        inode_t      *inode    = NULL;
-        fd_t         *iter_fd  = NULL;
-        uint64_t      tmp_subvol = 0;
-        int           open_failed = 0;
-
-        this  = THIS;
-        frame = data;
-        local = frame->local;
-        conf = this->private;
-
-        src_node = local->cached_subvol;
-
-        if (!local->loc.inode && !local->fd) {
-                local->op_errno = EINVAL;
-                goto out;
-        }
-
-        inode = (!local->fd) ? local->loc.inode : local->fd->inode;
-
-        /* getxattr on cached_subvol for 'linkto' value. Do path based getxattr
-         * as root:root. If a fd is already open, access check wont be done*/
+out:
+    local->rebalance.target_op_fn(THIS, subvol, frame, op_ret);
 
-        if (!local->loc.inode) {
-                ret = syncop_fgetxattr (src_node, local->fd, &dict,
-                                        conf->link_xattr_name, NULL, NULL);
-        } else {
-                SYNCTASK_SETID (0, 0);
-                ret = syncop_getxattr (src_node, &local->loc, &dict,
-                                       conf->link_xattr_name, NULL, NULL);
-                SYNCTASK_SETID (frame->root->uid, frame->root->gid);
-        }
+    return 0;
+}
 
-        /*
-         * temporary check related to tier promoting/demoting the file;
-         * the lower level DHT detects the migration (due to sticky
-         * bits) when it is the responsibility of the tier translator
-         * to complete the rebalance transaction. It will be corrected
-         * when rebalance and tier migration are fixed to work together.
+int
+dht_migration_complete_check_task(void *data)
+{
+    int ret = -1;
+    xlator_t *src_node = NULL;
+    xlator_t *dst_node = NULL, *linkto_target = NULL;
+    dht_local_t *local = NULL;
+    dict_t *dict = NULL;
+    struct iatt stbuf = {
+        0,
+    };
+    xlator_t *this = NULL;
+    call_frame_t *frame = NULL;
+    loc_t tmp_loc = {
+        0,
+    };
+    char *path = NULL;
+    dht_conf_t *conf = NULL;
+    inode_t *inode = NULL;
+    fd_t *iter_fd = NULL;
+    fd_t *tmp = NULL;
+    uint64_t tmp_miginfo = 0;
+    dht_migrate_info_t *miginfo = NULL;
+    gf_boolean_t skip_open = _gf_false;
+    int open_failed = 0;
+
+    this = THIS;
+    frame = data;
+    local = frame->local;
+    conf = this->private;
+
+    src_node = local->cached_subvol;
+
+    if (!local->loc.inode && !local->fd) {
+        local->op_errno = EINVAL;
+        goto out;
+    }
+
+    inode = (!local->fd) ? local->loc.inode : local->fd->inode;
+
+    /* getxattr on cached_subvol for 'linkto' value. Do path based getxattr
+     * as root:root. If a fd is already open, access check won't be done*/
+
+    if (!local->loc.inode) {
+        ret = syncop_fgetxattr(src_node, local->fd, &dict,
+                               conf->link_xattr_name, NULL, NULL);
+    } else {
+        SYNCTASK_SETID(0, 0);
+        ret = syncop_getxattr(src_node, &local->loc, &dict,
+                              conf->link_xattr_name, NULL, NULL);
+        SYNCTASK_SETID(frame->root->uid, frame->root->gid);
+    }
+
+    /*
+     * Each DHT xlator layer has its own name for the linkto xattr.
+     * If the file mode bits indicate the the file is being migrated but
+     * this layer's linkto xattr is not set, it means that another
+     * DHT layer is migrating the file. In this case, return 1 so
+     * the mode bits can be passed on to the higher layer for appropriate
+     * action.
+     */
+    if (-ret == ENODATA) {
+        /* This DHT translator is not migrating this file */
+
+        ret = inode_ctx_reset1(inode, this, &tmp_miginfo);
+        if (tmp_miginfo) {
+            /* This can be a problem if the file was
+             * migrated by two different layers. Raise
+             * a warning here.
+             */
+            gf_smsg(
+                this->name, GF_LOG_WARNING, 0, DHT_MSG_HAS_MIGINFO, "tmp=%s",
+                tmp_loc.path ? tmp_loc.path : uuid_utoa(tmp_loc.gfid), NULL);
+
+            miginfo = (void *)(uintptr_t)tmp_miginfo;
+            GF_REF_PUT(miginfo);
+        }
+        ret = 1;
+        goto out;
+    }
+
+    if (!ret)
+        linkto_target = dht_linkfile_subvol(this, NULL, NULL, dict);
+
+    if (local->loc.inode) {
+        loc_copy(&tmp_loc, &local->loc);
+    } else {
+        tmp_loc.inode = inode_ref(inode);
+        gf_uuid_copy(tmp_loc.gfid, inode->gfid);
+    }
+
+    ret = syncop_lookup(this, &tmp_loc, &stbuf, 0, 0, 0);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_ERROR, -ret, DHT_MSG_FILE_LOOKUP_FAILED,
+                "tmp=%s", tmp_loc.path ? tmp_loc.path : uuid_utoa(tmp_loc.gfid),
+                "name=%s", this->name, NULL);
+        local->op_errno = -ret;
+        ret = -1;
+        goto out;
+    }
+
+    dst_node = dht_subvol_get_cached(this, tmp_loc.inode);
+    if (linkto_target && dst_node != linkto_target) {
+        gf_smsg(this->name, GF_LOG_WARNING, 0, DHT_MSG_INVALID_LINKFILE,
+                "linkto_target_name=%s", linkto_target->name, "dst_name=%s",
+                dst_node->name, NULL);
+    }
+
+    if (gf_uuid_compare(stbuf.ia_gfid, tmp_loc.inode->gfid)) {
+        gf_smsg(this->name, GF_LOG_ERROR, 0, DHT_MSG_GFID_MISMATCH, "tmp=%s",
+                tmp_loc.path ? tmp_loc.path : uuid_utoa(tmp_loc.gfid),
+                "dst_name=%s", dst_node->name, NULL);
+        ret = -1;
+        local->op_errno = EIO;
+        goto out;
+    }
+
+    /* update local. A layout is set in inode-ctx in lookup already */
+
+    dht_layout_unref(this, local->layout);
+
+    local->layout = dht_layout_get(frame->this, inode);
+    local->cached_subvol = dst_node;
+
+    ret = 0;
+
+    /* once we detect the migration complete, the inode-ctx2 is no more
+       required.. delete the ctx and also, it means, open() already
+       done on all the fd of inode */
+    ret = inode_ctx_reset1(inode, this, &tmp_miginfo);
+    if (tmp_miginfo) {
+        miginfo = (void *)(uintptr_t)tmp_miginfo;
+        GF_REF_PUT(miginfo);
+        goto out;
+    }
+
+    /* perform 'open()' on all the fd's present on the inode */
+    if (tmp_loc.path == NULL) {
+        inode_path(inode, NULL, &path);
+        if (path)
+            tmp_loc.path = path;
+    }
+
+    LOCK(&inode->lock);
+
+    if (list_empty(&inode->fd_list))
+        goto unlock;
+
+    /* perform open as root:root. There is window between linkfile
+     * creation(root:root) and setattr with the correct uid/gid
+     */
+    SYNCTASK_SETID(0, 0);
+
+    /* It's possible that we are the last user of iter_fd after each
+     * iteration. In this case the fd_unref() of iter_fd at the end of
+     * the loop will cause the destruction of the fd. So we need to
+     * iterate the list safely because iter_fd cannot be trusted.
+     */
+    iter_fd = list_entry((&inode->fd_list)->next, typeof(*iter_fd), inode_list);
+    while (&iter_fd->inode_list != (&inode->fd_list)) {
+        if (fd_is_anonymous(iter_fd) ||
+            (dht_fd_open_on_dst(this, iter_fd, dst_node))) {
+            if (!tmp) {
+                iter_fd = list_entry(iter_fd->inode_list.next, typeof(*iter_fd),
+                                     inode_list);
+                continue;
+            }
+            skip_open = _gf_true;
+        }
+        /* We need to release the inode->lock before calling
+         * syncop_open() to avoid possible deadlocks. However this
+         * can cause the iter_fd to be released by other threads.
+         * To avoid this, we take a reference before releasing the
+         * lock.
          */
-        if (strcmp(this->parents->xlator->type, "cluster/tier") == 0) {
-                ret = 0;
-                goto out;
-        }
-
-        if (!ret)
-                dst_node = dht_linkfile_subvol (this, NULL, NULL, dict);
-
-        if (ret) {
-                if (!dht_inode_missing(-ret) || (!local->loc.inode)) {
-                        local->op_errno = -ret;
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "%s: failed to get the 'linkto' xattr %s",
-                                local->loc.path, strerror (-ret));
-                        ret = -1;
-                        goto out;
-                }
-
-                /* Need to do lookup on hashed subvol, then get the file */
-                ret = syncop_lookup (this, &local->loc, &stbuf, NULL,
-                                     NULL, NULL);
-                if (ret) {
-                        local->op_errno = -ret;
-                        ret = -1;
-                        goto out;
-                }
-
-                dst_node = dht_subvol_get_cached (this, local->loc.inode);
-        }
-
-        if (!dst_node) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "%s: failed to get the destination node",
-                        local->loc.path);
-                ret = -1;
-                local->op_errno = EINVAL;
-                goto out;
-        }
-
-        /* lookup on dst */
-        if (local->loc.inode) {
-                ret = syncop_lookup (dst_node, &local->loc, &stbuf, NULL,
-                                     NULL, NULL);
-
-                if (ret) {
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "%s: failed to lookup the file on %s",
-                                local->loc.path, dst_node->name);
-                        local->op_errno = -ret;
-                        ret = -1;
-                        goto out;
-                }
+        fd_ref(iter_fd);
 
-                if (gf_uuid_compare (stbuf.ia_gfid, local->loc.inode->gfid)) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                DHT_MSG_GFID_MISMATCH,
-                                "%s: gfid different on the target file on %s",
-                                local->loc.path, dst_node->name);
-                        ret = -1;
-                        local->op_errno = EIO;
-                        goto out;
-                }
-        }
-
-        /* update inode ctx (the layout) */
-        dht_layout_unref (this, local->layout);
-
-        ret = dht_layout_preset (this, dst_node, inode);
-        if (ret != 0) {
-                gf_msg_debug (this->name, 0,
-                              "%s: could not set preset layout "
-                              "for subvol %s", local->loc.path,
-                              dst_node->name);
-                ret   = -1;
-                local->op_errno = EINVAL;
-                goto out;
-        }
+        UNLOCK(&inode->lock);
 
-        layout = dht_layout_for_subvol (this, dst_node);
-        if (!layout) {
-                gf_log (this->name, GF_LOG_INFO,
-                        "%s: no pre-set layout for subvolume %s",
-                        local->loc.path, dst_node ? dst_node->name : "<nil>");
-                ret = -1;
-                local->op_errno = EINVAL;
-                goto out;
+        if (tmp) {
+            fd_unref(tmp);
+            tmp = NULL;
         }
+        if (skip_open)
+            goto next;
 
-        ret = dht_layout_set (this, inode, layout);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "%s: failed to set the new layout",
-                        local->loc.path);
-                local->op_errno = EINVAL;
-                goto out;
+        /* flags for open are stripped down to allow following the
+         * new location of the file, otherwise we can get EEXIST or
+         * truncate the file again as rebalance is moving the data */
+        ret = syncop_open(dst_node, &tmp_loc,
+                          (iter_fd->flags & ~(O_CREAT | O_EXCL | O_TRUNC)),
+                          iter_fd, NULL, NULL);
+        if (ret < 0) {
+            gf_smsg(this->name, GF_LOG_ERROR, -ret,
+                    DHT_MSG_OPEN_FD_ON_DST_FAILED, "id=%p", iter_fd,
+                    "flags=0%o", iter_fd->flags, "path=%s", path, "name=%s",
+                    dst_node->name, NULL);
+
+            open_failed = 1;
+            local->op_errno = -ret;
+            ret = -1;
+        } else {
+            dht_fd_ctx_set(this, iter_fd, dst_node);
         }
 
-        local->cached_subvol = dst_node;
-        ret = 0;
+    next:
+        LOCK(&inode->lock);
+        skip_open = _gf_false;
+        tmp = iter_fd;
+        iter_fd = list_entry(tmp->inode_list.next, typeof(*tmp), inode_list);
+    }
 
-        /* once we detect the migration complete, the inode-ctx2 is no more
-           required.. delete the ctx and also, it means, open() already
-           done on all the fd of inode */
-        ret = inode_ctx_reset1 (inode, this, &tmp_subvol);
-        if (tmp_subvol)
-                goto out;
+    SYNCTASK_SETID(frame->root->uid, frame->root->gid);
 
-        if (list_empty (&inode->fd_list))
-                goto out;
+    if (open_failed) {
+        ret = -1;
+        goto unlock;
+    }
+    ret = 0;
 
-        /* perform open as root:root. There is window between linkfile
-         * creation(root:root) and setattr with the correct uid/gid
-         */
-        SYNCTASK_SETID(0, 0);
+unlock:
+    UNLOCK(&inode->lock);
+    if (tmp) {
+        fd_unref(tmp);
+        tmp = NULL;
+    }
 
-        /* perform 'open()' on all the fd's present on the inode */
-        tmp_loc.inode = inode;
-        inode_path (inode, NULL, &path);
-        if (path)
-                tmp_loc.path = path;
-        list_for_each_entry (iter_fd, &inode->fd_list, inode_list) {
-                if (fd_is_anonymous (iter_fd))
-                        continue;
-
-                /* flags for open are stripped down to allow following the
-                 * new location of the file, otherwise we can get EEXIST or
-                 * truncate the file again as rebalance is moving the data */
-                ret = syncop_open (dst_node, &tmp_loc,
-                                   (iter_fd->flags &
-                                   ~(O_CREAT | O_EXCL | O_TRUNC)), iter_fd,
-                                   NULL, NULL);
-                if (ret < 0) {
-                        gf_log (this->name, GF_LOG_ERROR, "failed to open "
-                                "the fd (%p, flags=0%o) on file %s @ %s",
-                                iter_fd, iter_fd->flags, path, dst_node->name);
-                        open_failed = 1;
-                        local->op_errno = -ret;
-                        ret = -1;
-                }
-        }
-        GF_FREE (path);
-
-        SYNCTASK_SETID (frame->root->uid, frame->root->gid);
-
-        if (open_failed) {
-                ret = -1;
-                goto out;
-        }
-        ret = 0;
 out:
+    if (dict) {
+        dict_unref(dict);
+    }
 
-        return ret;
+    loc_wipe(&tmp_loc);
+
+    return ret;
 }
 
 int
-dht_rebalance_complete_check (xlator_t *this, call_frame_t *frame)
+dht_rebalance_complete_check(xlator_t *this, call_frame_t *frame)
 {
-        int         ret     = -1;
+    int ret = -1;
 
-        ret = synctask_new (this->ctx->env, dht_migration_complete_check_task,
-                            dht_migration_complete_check_done,
-                            frame, frame);
-        return ret;
+    ret = synctask_new(this->ctx->env, dht_migration_complete_check_task,
+                       dht_migration_complete_check_done, frame, frame);
+    return ret;
 }
 
 /* During 'in-progress' state, both nodes should have the file */
+/*
+ op_ret values :
+  0 : Success
+ -1 : Failure.
+  1 : File is being migrated but not by this DHT layer.
+*/
 static int
-dht_inprogress_check_done (int op_ret, call_frame_t *sync_frame, void *data)
-{
-        dht_local_t *local = NULL;
-
-        local = sync_frame->local;
-
-        local->rebalance.target_op_fn (THIS, sync_frame, op_ret);
-
-        return 0;
-}
-
-static int
-dht_rebalance_inprogress_task (void *data)
+dht_inprogress_check_done(int op_ret, call_frame_t *frame, void *data)
 {
-        int           ret      = -1;
-        xlator_t     *src_node = NULL;
-        xlator_t     *dst_node = NULL;
-        dht_local_t  *local    = NULL;
-        dict_t       *dict     = NULL;
-        call_frame_t *frame    = NULL;
-        xlator_t     *this     = NULL;
-        char         *path     = NULL;
-        struct iatt   stbuf    = {0,};
-        loc_t         tmp_loc  = {0,};
-        dht_conf_t   *conf     = NULL;
-        inode_t      *inode    = NULL;
-        fd_t         *iter_fd  = NULL;
-        int           open_failed = 0;
-
-        this  = THIS;
-        frame = data;
-        local = frame->local;
-        conf = this->private;
-
-        src_node = local->cached_subvol;
-
-        if (!local->loc.inode && !local->fd)
-                goto out;
+    dht_local_t *local = NULL;
+    xlator_t *dst_subvol = NULL, *src_subvol = NULL;
+    inode_t *inode = NULL;
 
-        inode = (!local->fd) ? local->loc.inode : local->fd->inode;
+    local = frame->local;
 
-        /* getxattr on cached_subvol for 'linkto' value. Do path based getxattr
-         * as root:root. If a fd is already open, access check wont be done*/
-        if (local->loc.inode) {
-                SYNCTASK_SETID (0, 0);
-                ret = syncop_getxattr (src_node, &local->loc, &dict,
-                                       conf->link_xattr_name, NULL, NULL);
-                SYNCTASK_SETID (frame->root->uid, frame->root->gid);
-        } else {
-                ret = syncop_fgetxattr (src_node, local->fd, &dict,
-                                        conf->link_xattr_name, NULL, NULL);
-        }
-
-        if (ret < 0) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "%s: failed to get the 'linkto' xattr %s",
-                        local->loc.path, strerror (-ret));
-                ret = -1;
-                goto out;
-        }
-
-        dst_node = dht_linkfile_subvol (this, NULL, NULL, dict);
-        if (!dst_node) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "%s: failed to get the 'linkto' xattr from dict",
-                        local->loc.path);
-                ret = -1;
-                goto out;
-        }
+    if (op_ret != 0)
+        goto out;
 
-        local->rebalance.target_node = dst_node;
-
-        if (local->loc.inode) {
-                /* lookup on dst */
-                ret = syncop_lookup (dst_node, &local->loc, &stbuf, NULL,
-                                     NULL, NULL);
-                if (ret) {
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "%s: failed to lookup the file on %s",
-                                local->loc.path, dst_node->name);
-                        ret = -1;
-                        goto out;
-                }
+    inode = local->loc.inode ? local->loc.inode : local->fd->inode;
 
-                if (gf_uuid_compare (stbuf.ia_gfid, local->loc.inode->gfid)) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                DHT_MSG_GFID_MISMATCH,
-                                "%s: gfid different on the target file on %s",
-                                local->loc.path, dst_node->name);
-                        ret = -1;
-                        goto out;
-                }
+    dht_inode_ctx_get_mig_info(THIS, inode, &src_subvol, &dst_subvol);
+    if (dht_mig_info_is_invalid(local->cached_subvol, src_subvol, dst_subvol)) {
+        dst_subvol = dht_subvol_get_cached(THIS, inode);
+        if (!dst_subvol) {
+            local->op_errno = EINVAL;
+            goto out;
         }
+    }
 
-        ret = 0;
+out:
+    local->rebalance.target_op_fn(THIS, dst_subvol, frame, op_ret);
 
-        if (list_empty (&inode->fd_list))
-                goto done;
+    return 0;
+}
 
-        /* perform open as root:root. There is window between linkfile
-         * creation(root:root) and setattr with the correct uid/gid
+static int
+dht_rebalance_inprogress_task(void *data)
+{
+    int ret = -1;
+    xlator_t *src_node = NULL;
+    xlator_t *dst_node = NULL;
+    dht_local_t *local = NULL;
+    dict_t *dict = NULL;
+    call_frame_t *frame = NULL;
+    xlator_t *this = NULL;
+    char *path = NULL;
+    struct iatt stbuf = {
+        0,
+    };
+    loc_t tmp_loc = {
+        0,
+    };
+    dht_conf_t *conf = NULL;
+    inode_t *inode = NULL;
+    fd_t *iter_fd = NULL;
+    fd_t *tmp = NULL;
+    int open_failed = 0;
+    uint64_t tmp_miginfo = 0;
+    dht_migrate_info_t *miginfo = NULL;
+    gf_boolean_t skip_open = _gf_false;
+
+    this = THIS;
+    frame = data;
+    local = frame->local;
+    conf = this->private;
+
+    src_node = local->cached_subvol;
+
+    if (!local->loc.inode && !local->fd)
+        goto out;
+
+    inode = (!local->fd) ? local->loc.inode : local->fd->inode;
+
+    /* getxattr on cached_subvol for 'linkto' value. Do path based getxattr
+     * as root:root. If a fd is already open, access check won't be done*/
+    if (local->loc.inode) {
+        SYNCTASK_SETID(0, 0);
+        ret = syncop_getxattr(src_node, &local->loc, &dict,
+                              conf->link_xattr_name, NULL, NULL);
+        SYNCTASK_SETID(frame->root->uid, frame->root->gid);
+    } else {
+        ret = syncop_fgetxattr(src_node, local->fd, &dict,
+                               conf->link_xattr_name, NULL, NULL);
+    }
+
+    /*
+     * Each DHT xlator layer has its own name for the linkto xattr.
+     * If the file mode bits indicate the the file is being migrated but
+     * this layer's linkto xattr is not present, it means that another
+     * DHT layer is migrating the file. In this case, return 1 so
+     * the mode bits can be passed on to the higher layer for appropriate
+     * action.
+     */
+
+    if (-ret == ENODATA) {
+        /* This DHT layer is not migrating this file */
+        ret = inode_ctx_reset1(inode, this, &tmp_miginfo);
+        if (tmp_miginfo) {
+            /* This can be a problem if the file was
+             * migrated by two different layers. Raise
+             * a warning here.
+             */
+            gf_smsg(
+                this->name, GF_LOG_WARNING, 0, DHT_MSG_HAS_MIGINFO, "tmp=%s",
+                tmp_loc.path ? tmp_loc.path : uuid_utoa(tmp_loc.gfid), NULL);
+            miginfo = (void *)(uintptr_t)tmp_miginfo;
+            GF_REF_PUT(miginfo);
+        }
+        ret = 1;
+        goto out;
+    }
+
+    if (ret < 0) {
+        gf_smsg(this->name, GF_LOG_ERROR, -ret, DHT_MSG_GET_XATTR_FAILED,
+                "path=%s", local->loc.path, NULL);
+        ret = -1;
+        goto out;
+    }
+
+    dst_node = dht_linkfile_subvol(this, NULL, NULL, dict);
+    if (!dst_node) {
+        gf_smsg(this->name, GF_LOG_ERROR, 0, DHT_MSG_GET_XATTR_FAILED,
+                "path=%s", local->loc.path, NULL);
+        ret = -1;
+        goto out;
+    }
+
+    local->rebalance.target_node = dst_node;
+
+    if (local->loc.inode) {
+        loc_copy(&tmp_loc, &local->loc);
+    } else {
+        tmp_loc.inode = inode_ref(inode);
+        gf_uuid_copy(tmp_loc.gfid, inode->gfid);
+    }
+
+    /* lookup on dst */
+    ret = syncop_lookup(dst_node, &tmp_loc, &stbuf, NULL, NULL, NULL);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_ERROR, -ret, DHT_MSG_FILE_LOOKUP_FAILED,
+                "tmp=%s", tmp_loc.path ? tmp_loc.path : uuid_utoa(tmp_loc.gfid),
+                "name=%s", dst_node->name, NULL);
+        ret = -1;
+        goto out;
+    }
+
+    if (gf_uuid_compare(stbuf.ia_gfid, tmp_loc.inode->gfid)) {
+        gf_smsg(this->name, GF_LOG_ERROR, 0, DHT_MSG_GFID_MISMATCH, "tmp=%s",
+                tmp_loc.path ? tmp_loc.path : uuid_utoa(tmp_loc.gfid),
+                "name=%s", dst_node->name, NULL);
+        ret = -1;
+        goto out;
+    }
+    ret = 0;
+
+    if (tmp_loc.path == NULL) {
+        inode_path(inode, NULL, &path);
+        if (path)
+            tmp_loc.path = path;
+    }
+
+    LOCK(&inode->lock);
+
+    if (list_empty(&inode->fd_list))
+        goto unlock;
+
+    /* perform open as root:root. There is window between linkfile
+     * creation(root:root) and setattr with the correct uid/gid
+     */
+    SYNCTASK_SETID(0, 0);
+
+    /* It's possible that we are the last user of iter_fd after each
+     * iteration. In this case the fd_unref() of iter_fd at the end of
+     * the loop will cause the destruction of the fd. So we need to
+     * iterate the list safely because iter_fd cannot be trusted.
+     */
+    iter_fd = list_entry((&inode->fd_list)->next, typeof(*iter_fd), inode_list);
+    while (&iter_fd->inode_list != (&inode->fd_list)) {
+        /* We need to release the inode->lock before calling
+         * syncop_open() to avoid possible deadlocks. However this
+         * can cause the iter_fd to be released by other threads.
+         * To avoid this, we take a reference before releasing the
+         * lock.
          */
-        SYNCTASK_SETID (0, 0);
 
-        tmp_loc.inode = inode;
-        inode_path (inode, NULL, &path);
-        if (path)
-                tmp_loc.path = path;
-
-        list_for_each_entry (iter_fd, &inode->fd_list, inode_list) {
-                if (fd_is_anonymous (iter_fd))
-                        continue;
-
-                /* flags for open are stripped down to allow following the
-                 * new location of the file, otherwise we can get EEXIST or
-                 * truncate the file again as rebalance is moving the data */
-                ret = syncop_open (dst_node, &tmp_loc,
-                                   (iter_fd->flags &
-                                   ~(O_CREAT | O_EXCL | O_TRUNC)), iter_fd,
-                                   NULL, NULL);
-                if (ret < 0) {
-                        gf_log (this->name, GF_LOG_ERROR, "failed to send open "
-                                "the fd (%p, flags=0%o) on file %s @ %s",
-                                iter_fd, iter_fd->flags, path, dst_node->name);
-                        ret = -1;
-                        open_failed = 1;
-                }
+        if (fd_is_anonymous(iter_fd) ||
+            (dht_fd_open_on_dst(this, iter_fd, dst_node))) {
+            if (!tmp) {
+                iter_fd = list_entry(iter_fd->inode_list.next, typeof(*iter_fd),
+                                     inode_list);
+                continue;
+            }
+            skip_open = _gf_true;
         }
-        GF_FREE (path);
 
-        SYNCTASK_SETID (frame->root->uid, frame->root->gid);
+        /* Yes, this is ugly but there isn't a cleaner way to do this
+         * the fd_ref is an atomic increment so not too bad. We want to
+         * reduce the number of inode locks and unlocks.
+         */
 
-        if (open_failed) {
-                ret = -1;
-                goto out;
-        }
+        fd_ref(iter_fd);
+        UNLOCK(&inode->lock);
 
-done:
-        ret = dht_inode_ctx_set1 (this, inode, dst_node);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "%s: failed to set inode-ctx target file at %s",
-                        local->loc.path, dst_node->name);
-                goto out;
+        if (tmp) {
+            fd_unref(tmp);
+            tmp = NULL;
         }
+        if (skip_open)
+            goto next;
 
-        ret = 0;
+        /* flags for open are stripped down to allow following the
+         * new location of the file, otherwise we can get EEXIST or
+         * truncate the file again as rebalance is moving the data */
+        ret = syncop_open(dst_node, &tmp_loc,
+                          (iter_fd->flags & ~(O_CREAT | O_EXCL | O_TRUNC)),
+                          iter_fd, NULL, NULL);
+        if (ret < 0) {
+            gf_smsg(this->name, GF_LOG_ERROR, -ret,
+                    DHT_MSG_OPEN_FD_ON_DST_FAILED, "fd=%p", iter_fd,
+                    "flags=0%o", iter_fd->flags, "path=%s", path, "name=%s",
+                    dst_node->name, NULL);
+            ret = -1;
+            open_failed = 1;
+        } else {
+            /* Potential fd leak if this fails here as it will be
+               reopened at the next Phase1/2 check */
+            dht_fd_ctx_set(this, iter_fd, dst_node);
+        }
+
+    next:
+        LOCK(&inode->lock);
+        skip_open = _gf_false;
+        tmp = iter_fd;
+        iter_fd = list_entry(tmp->inode_list.next, typeof(*tmp), inode_list);
+    }
+
+    SYNCTASK_SETID(frame->root->uid, frame->root->gid);
+
+unlock:
+    UNLOCK(&inode->lock);
+
+    if (tmp) {
+        fd_unref(tmp);
+        tmp = NULL;
+    }
+    if (open_failed) {
+        ret = -1;
+        goto out;
+    }
+
+    ret = dht_inode_ctx_set_mig_info(this, inode, src_node, dst_node);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_ERROR, 0, DHT_MSG_SET_INODE_CTX_FAILED,
+                "path=%s", local->loc.path, "name=%s", dst_node->name, NULL);
+        goto out;
+    }
+
+    ret = 0;
 out:
-        return ret;
+    if (dict) {
+        dict_unref(dict);
+    }
+
+    loc_wipe(&tmp_loc);
+    return ret;
 }
 
 int
-dht_rebalance_in_progress_check (xlator_t *this, call_frame_t *frame)
+dht_rebalance_in_progress_check(xlator_t *this, call_frame_t *frame)
 {
+    int ret = -1;
 
-        int         ret     = -1;
-
-        ret = synctask_new (this->ctx->env, dht_rebalance_inprogress_task,
-                            dht_inprogress_check_done,
-                            frame, frame);
-        return ret;
+    ret = synctask_new(this->ctx->env, dht_rebalance_inprogress_task,
+                       dht_inprogress_check_done, frame, frame);
+    return ret;
 }
 
 int
-dht_inode_ctx_layout_set (inode_t *inode, xlator_t *this,
-                          dht_layout_t *layout_int)
+dht_inode_ctx_layout_set(inode_t *inode, xlator_t *this,
+                         dht_layout_t *layout_int)
 {
-        dht_inode_ctx_t         *ctx            = NULL;
-        int                      ret            = -1;
-
-        ret = dht_inode_ctx_get (inode, this, &ctx);
-        if (!ret && ctx) {
-                ctx->layout = layout_int;
-        } else {
-                ctx = GF_CALLOC (1, sizeof (*ctx), gf_dht_mt_inode_ctx_t);
-                if (!ctx)
-                        return ret;
-                ctx->layout = layout_int;
-        }
-
-        ret = dht_inode_ctx_set (inode, this, ctx);
-
-        return ret;
+    dht_inode_ctx_t *ctx = NULL;
+    int ret = -1;
+
+    ret = dht_inode_ctx_get(inode, this, &ctx);
+    if (!ret && ctx) {
+        ctx->layout = layout_int;
+    } else {
+        ctx = GF_CALLOC(1, sizeof(*ctx), gf_dht_mt_inode_ctx_t);
+        if (!ctx)
+            return ret;
+        ctx->layout = layout_int;
+    }
+
+    ret = dht_inode_ctx_set(inode, this, ctx);
+
+    return ret;
 }
 
-
 void
-dht_inode_ctx_time_set (inode_t *inode, xlator_t *this, struct iatt *stat)
+dht_inode_ctx_time_set(inode_t *inode, xlator_t *this, struct iatt *stat)
 {
-        dht_inode_ctx_t         *ctx            = NULL;
-        dht_stat_time_t         *time           = 0;
-        int                      ret            = -1;
+    dht_inode_ctx_t *ctx = NULL;
+    dht_stat_time_t *time = 0;
+    int ret = -1;
 
-        ret = dht_inode_ctx_get (inode, this, &ctx);
+    ret = dht_inode_ctx_get(inode, this, &ctx);
 
-        if (ret)
-		return;
+    if (ret)
+        return;
 
-        time = &ctx->time;
+    time = &ctx->time;
 
-	time->mtime      = stat->ia_mtime;
-	time->mtime_nsec = stat->ia_mtime_nsec;
+    time->mtime = stat->ia_mtime;
+    time->mtime_nsec = stat->ia_mtime_nsec;
 
-	time->ctime      = stat->ia_ctime;
-	time->ctime_nsec = stat->ia_ctime_nsec;
+    time->ctime = stat->ia_ctime;
+    time->ctime_nsec = stat->ia_ctime_nsec;
 
-	time->atime      = stat->ia_atime;
-	time->atime_nsec = stat->ia_atime_nsec;
+    time->atime = stat->ia_atime;
+    time->atime_nsec = stat->ia_atime_nsec;
 
-	return;
+    return;
 }
 
-
 int
-dht_inode_ctx_time_update (inode_t *inode, xlator_t *this, struct iatt *stat,
-                           int32_t post)
+dht_inode_ctx_time_update(inode_t *inode, xlator_t *this, struct iatt *stat,
+                          int32_t post)
 {
-        dht_inode_ctx_t         *ctx            = NULL;
-        dht_stat_time_t         *time           = 0;
-        int                      ret            = -1;
-
-        GF_VALIDATE_OR_GOTO (this->name, stat, out);
-        GF_VALIDATE_OR_GOTO (this->name, inode, out);
-
-        ret = dht_inode_ctx_get (inode, this, &ctx);
-
-        if (ret) {
-                ctx = GF_CALLOC (1, sizeof (*ctx), gf_dht_mt_inode_ctx_t);
-                if (!ctx)
-                        return -1;
-        }
-
-        time = &ctx->time;
-
-        DHT_UPDATE_TIME(time->mtime, time->mtime_nsec,
-                        stat->ia_mtime, stat->ia_mtime_nsec, inode, post);
-        DHT_UPDATE_TIME(time->ctime, time->ctime_nsec,
-                        stat->ia_ctime, stat->ia_ctime_nsec, inode, post);
-        DHT_UPDATE_TIME(time->atime, time->atime_nsec,
-                        stat->ia_atime, stat->ia_atime_nsec, inode, post);
-
-        ret = dht_inode_ctx_set (inode, this, ctx);
+    dht_inode_ctx_t *ctx = NULL;
+    dht_stat_time_t *time = 0;
+    int ret = -1;
+
+    GF_VALIDATE_OR_GOTO(this->name, stat, out);
+    GF_VALIDATE_OR_GOTO(this->name, inode, out);
+
+    ret = dht_inode_ctx_get(inode, this, &ctx);
+
+    if (ret) {
+        ctx = GF_CALLOC(1, sizeof(*ctx), gf_dht_mt_inode_ctx_t);
+        if (!ctx)
+            return -1;
+    }
+
+    time = &ctx->time;
+
+    LOCK(&inode->lock);
+    {
+        DHT_UPDATE_TIME(time->mtime, time->mtime_nsec, stat->ia_mtime,
+                        stat->ia_mtime_nsec, post);
+        DHT_UPDATE_TIME(time->ctime, time->ctime_nsec, stat->ia_ctime,
+                        stat->ia_ctime_nsec, post);
+        DHT_UPDATE_TIME(time->atime, time->atime_nsec, stat->ia_atime,
+                        stat->ia_atime_nsec, post);
+    }
+    UNLOCK(&inode->lock);
+
+    ret = dht_inode_ctx_set(inode, this, ctx);
 out:
-        return 0;
+    return 0;
 }
 
 int
-dht_inode_ctx_get (inode_t *inode, xlator_t *this, dht_inode_ctx_t **ctx)
+dht_inode_ctx_get(inode_t *inode, xlator_t *this, dht_inode_ctx_t **ctx)
 {
-        int             ret     = -1;
-        uint64_t        ctx_int = 0;
-
-        GF_VALIDATE_OR_GOTO ("dht", this, out);
-        GF_VALIDATE_OR_GOTO (this->name, inode, out);
+    int ret = -1;
+    uint64_t ctx_int = 0;
 
-        ret = inode_ctx_get (inode, this, &ctx_int);
+    GF_VALIDATE_OR_GOTO("dht", this, out);
+    GF_VALIDATE_OR_GOTO(this->name, inode, out);
 
-        if (ret)
-                return ret;
+    ret = inode_ctx_get(inode, this, &ctx_int);
 
-        if (ctx)
-                *ctx = (dht_inode_ctx_t *) ctx_int;
-out:
+    if (ret)
         return ret;
-}
 
-int dht_inode_ctx_set (inode_t *inode, xlator_t *this, dht_inode_ctx_t *ctx)
-{
-        int             ret = -1;
-        uint64_t        ctx_int = 0;
-
-        GF_VALIDATE_OR_GOTO ("dht", this, out);
-        GF_VALIDATE_OR_GOTO (this->name, inode, out);
-        GF_VALIDATE_OR_GOTO (this->name, ctx, out);
-
-        ctx_int = (long)ctx;
-        ret = inode_ctx_set (inode, this, &ctx_int);
+    if (ctx)
+        *ctx = (dht_inode_ctx_t *)(uintptr_t)ctx_int;
 out:
-        return ret;
+    return ret;
 }
 
-void
-dht_set_lkowner (dht_lock_t **lk_array, int count, gf_lkowner_t *lkowner)
+int
+dht_inode_ctx_set(inode_t *inode, xlator_t *this, dht_inode_ctx_t *ctx)
 {
-        int i = 0;
-
-        if (!lk_array || !lkowner)
-                goto out;
+    int ret = -1;
+    uint64_t ctx_int = 0;
 
-        for (i = 0; i < count; i++) {
-                lk_array[i]->lk_owner = *lkowner;
-        }
+    GF_VALIDATE_OR_GOTO("dht", this, out);
+    GF_VALIDATE_OR_GOTO(this->name, inode, out);
+    GF_VALIDATE_OR_GOTO(this->name, ctx, out);
 
+    ctx_int = (long)ctx;
+    ret = inode_ctx_set(inode, this, &ctx_int);
 out:
-        return;
+    return ret;
 }
 
 int
-dht_subvol_status (dht_conf_t *conf, xlator_t *subvol)
+dht_subvol_status(dht_conf_t *conf, xlator_t *subvol)
 {
-        int i;
+    int i;
 
-        for (i=0 ; i < conf->subvolume_cnt; i++) {
-                if (conf->subvolumes[i] == subvol) {
-                        return conf->subvolume_status[i];
-                }
+    for (i = 0; i < conf->subvolume_cnt; i++) {
+        if (conf->subvolumes[i] == subvol) {
+            return conf->subvolume_status[i];
         }
-        return 0;
+    }
+    return 0;
 }
 
-void
-dht_inodelk_done (call_frame_t *lock_frame)
+inode_t *
+dht_heal_path(xlator_t *this, char *path, inode_table_t *itable)
 {
-        fop_inodelk_cbk_t  inodelk_cbk = NULL;
-        call_frame_t      *main_frame  = NULL;
-        dht_local_t       *local       = NULL;
-
-        local = lock_frame->local;
-        main_frame = local->main_frame;
-
-        local->lock.locks = NULL;
-        local->lock.lk_count = 0;
-
-        inodelk_cbk = local->lock.inodelk_cbk;
-        local->lock.inodelk_cbk = NULL;
-
-        inodelk_cbk (main_frame, NULL, main_frame->this, local->lock.op_ret,
-                     local->lock.op_errno, NULL);
-
-        dht_lock_stack_destroy (lock_frame);
-        return;
-}
-
-int
-dht_inodelk_cleanup_cbk (call_frame_t *frame, void *cookie,
-                         xlator_t *this, int32_t op_ret, int32_t op_errno,
-                         dict_t *xdata)
-{
-        dht_inodelk_done (frame);
-        return 0;
-}
-
-int32_t
-dht_lock_count (dht_lock_t **lk_array, int lk_count)
-{
-        int i = 0, locked = 0;
-
-        if ((lk_array == NULL) || (lk_count == 0))
+    int ret = -1;
+    struct iatt iatt = {
+        0,
+    };
+    inode_t *linked_inode = NULL;
+    loc_t loc = {
+        0,
+    };
+    char *bname = NULL;
+    char *save_ptr = NULL;
+    static uuid_t gfid = {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1};
+    char *tmp_path = NULL;
+
+    tmp_path = gf_strdup(path);
+    if (!tmp_path) {
+        goto out;
+    }
+
+    gf_uuid_copy(loc.pargfid, gfid);
+    loc.parent = inode_ref(itable->root);
+
+    bname = strtok_r(tmp_path, "/", &save_ptr);
+
+    /* sending a lookup on parent directory,
+     * Eg:  if  path is like /a/b/c/d/e/f/g/
+     * then we will send a lookup on a first and then b,c,d,etc
+     */
+
+    while (bname) {
+        linked_inode = NULL;
+        loc.inode = inode_grep(itable, loc.parent, bname);
+        if (loc.inode == NULL) {
+            loc.inode = inode_new(itable);
+            if (loc.inode == NULL) {
+                ret = -ENOMEM;
                 goto out;
-
-        for (i = 0; i < lk_count; i++) {
-                if (lk_array[i]->locked)
-                        locked++;
-        }
-out:
-        return locked;
-}
-
-void
-dht_inodelk_cleanup (call_frame_t *lock_frame)
-{
-        dht_lock_t  **lk_array = NULL;
-        int           lk_count = 0, lk_acquired = 0;
-        dht_local_t  *local    = NULL;
-
-        local = lock_frame->local;
-
-        lk_array = local->lock.locks;
-        lk_count = local->lock.lk_count;
-
-        lk_acquired = dht_lock_count (lk_array, lk_count);
-        if (lk_acquired != 0) {
-                dht_unlock_inodelk (lock_frame, lk_array, lk_count,
-                                    dht_inodelk_cleanup_cbk);
+            }
         } else {
-                dht_inodelk_done (lock_frame);
-        }
-
-        return;
-}
-
-int32_t
-dht_unlock_inodelk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                        int32_t op_ret, int32_t op_errno, dict_t *xdata)
-{
-        dht_local_t *local                  = NULL;
-        int          lk_index               = 0, call_cnt = 0;
-        char         gfid[GF_UUID_BUF_SIZE] = {0};
-
-        lk_index = (long) cookie;
-
-        local = frame->local;
-        if (op_ret < 0) {
-                uuid_utoa_r (local->lock.locks[lk_index]->loc.gfid,
-                             gfid);
-
-                gf_log (this->name, GF_LOG_WARNING,
-                        "unlocking failed on %s:%s (%s)",
-                        local->lock.locks[lk_index]->xl->name,
-                        gfid, strerror (op_errno));
-        } else {
-                local->lock.locks[lk_index]->locked = 0;
-        }
-
-        call_cnt = dht_frame_return (frame);
-        if (is_last_call (call_cnt)) {
-                dht_inodelk_done (frame);
-        }
-
-        return 0;
-}
-
-call_frame_t *
-dht_lock_frame (call_frame_t *parent_frame)
-{
-        call_frame_t *lock_frame = NULL;
-
-        lock_frame = copy_frame (parent_frame);
-        if (lock_frame == NULL)
+            /*
+             * Inode is already populated in the inode table.
+             * Which means we already looked up the inode and
+             * linked with a dentry. So that we will skip
+             * lookup on this entry, and proceed to next.
+             */
+            linked_inode = loc.inode;
+            bname = strtok_r(NULL, "/", &save_ptr);
+            if (!bname) {
                 goto out;
-
-        set_lk_owner_from_ptr (&lock_frame->root->lk_owner, parent_frame->root);
-
-out:
-        return lock_frame;
-}
-
-int32_t
-dht_unlock_inodelk (call_frame_t *frame, dht_lock_t **lk_array, int lk_count,
-                    fop_inodelk_cbk_t inodelk_cbk)
-{
-        dht_local_t     *local      = NULL;
-        struct gf_flock  flock      = {0,};
-        int              ret        = -1 , i = 0;
-        call_frame_t    *lock_frame = NULL;
-        int              call_cnt   = 0;
-
-        GF_VALIDATE_OR_GOTO ("dht-locks", frame, done);
-        GF_VALIDATE_OR_GOTO (frame->this->name, lk_array, done);
-        GF_VALIDATE_OR_GOTO (frame->this->name, inodelk_cbk, done);
-
-        call_cnt = dht_lock_count (lk_array, lk_count);
-        if (call_cnt == 0) {
-                ret = 0;
-                goto done;
-        }
-
-        lock_frame = dht_lock_frame (frame);
-        if (lock_frame == NULL) {
-                gf_log (frame->this->name, GF_LOG_WARNING,
-                        "cannot allocate a frame, not unlocking following "
-                        "locks:");
-
-                dht_log_lk_array (frame->this->name, GF_LOG_WARNING, lk_array,
-                                  lk_count);
-                goto done;
+            }
+            inode_unref(loc.parent);
+            loc.parent = loc.inode;
+            gf_uuid_copy(loc.pargfid, loc.inode->gfid);
+            loc.inode = NULL;
+            continue;
         }
 
-        ret = dht_local_lock_init (lock_frame, lk_array, lk_count, inodelk_cbk);
-        if (ret < 0) {
-                gf_log (frame->this->name, GF_LOG_WARNING,
-                        "storing locks in local failed, not unlocking "
-                        "following locks:");
-
-                dht_log_lk_array (frame->this->name, GF_LOG_WARNING, lk_array,
-                                  lk_count);
-
-                goto done;
-        }
-
-        local = lock_frame->local;
-        local->main_frame = frame;
-        local->call_cnt = call_cnt;
-
-        flock.l_type = F_UNLCK;
-
-        for (i = 0; i < local->lock.lk_count; i++) {
-                if (!local->lock.locks[i]->locked)
-                        continue;
+        loc.name = bname;
+        ret = loc_path(&loc, bname);
 
-                lock_frame->root->lk_owner = local->lock.locks[i]->lk_owner;
-                STACK_WIND_COOKIE (lock_frame, dht_unlock_inodelk_cbk,
-                                   (void *)(long)i,
-                                   local->lock.locks[i]->xl,
-                                   local->lock.locks[i]->xl->fops->inodelk,
-                                   local->lock.locks[i]->domain,
-                                   &local->lock.locks[i]->loc, F_SETLK,
-                                   &flock, NULL);
+        ret = syncop_lookup(this, &loc, &iatt, NULL, NULL, NULL);
+        if (ret) {
+            gf_smsg(this->name, GF_LOG_INFO, -ret, DHT_MSG_DIR_SELFHEAL_FAILED,
+                    "path=%s", path, "subvolume=%s", this->name, "bname=%s",
+                    bname, NULL);
+            goto out;
         }
 
-        return 0;
-
-done:
-        if (lock_frame)
-                dht_lock_stack_destroy (lock_frame);
-
-        /* no locks acquired, invoke inodelk_cbk */
-        if (ret == 0)
-                inodelk_cbk (frame, NULL, frame->this, 0, 0, NULL);
+        linked_inode = inode_link(loc.inode, loc.parent, bname, &iatt);
+        if (!linked_inode)
+            goto out;
 
-        return ret;
-}
-
-int32_t
-dht_nonblocking_inodelk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                             int32_t op_ret, int32_t op_errno, dict_t *xdata)
-{
-        dht_local_t *local                   = NULL;
-        int          lk_index               = 0, call_cnt = 0;
-        char          gfid[GF_UUID_BUF_SIZE] = {0};
-
-        local = frame->local;
-        lk_index = (long) cookie;
-
-        if (op_ret == -1) {
-                local->lock.op_ret = -1;
-                local->lock.op_errno = op_errno;
-
-                if (local && local->lock.locks[lk_index]) {
-                        uuid_utoa_r (local->lock.locks[lk_index]->loc.inode->gfid,
-                                     gfid);
-
-                        gf_log (this->name, GF_LOG_DEBUG,
-                                "inodelk failed on gfid: %s "
-                                "subvolume: %s (%s)", gfid,
-                                local->lock.locks[lk_index]->xl->name,
-                                strerror (op_errno));
-                }
-
-                goto out;
-        }
-
-        local->lock.locks[lk_index]->locked = _gf_true;
+        loc_wipe(&loc);
+        gf_uuid_copy(loc.pargfid, linked_inode->gfid);
+        loc.inode = NULL;
 
+        bname = strtok_r(NULL, "/", &save_ptr);
+        if (bname)
+            loc.parent = linked_inode;
+    }
 out:
-        call_cnt = dht_frame_return (frame);
-        if (is_last_call (call_cnt)) {
-                if (local->lock.op_ret < 0) {
-                        dht_inodelk_cleanup (frame);
-                        return 0;
-                }
+    inode_ref(linked_inode);
+    loc_wipe(&loc);
+    GF_FREE(tmp_path);
 
-                dht_inodelk_done (frame);
-        }
-
-        return 0;
+    return linked_inode;
 }
 
 int
-dht_nonblocking_inodelk (call_frame_t *frame, dht_lock_t **lk_array,
-                         int lk_count, fop_inodelk_cbk_t inodelk_cbk)
+dht_heal_full_path(void *data)
 {
-        struct gf_flock  flock      = {0,};
-        int              i          = 0, ret = 0;
-        dht_local_t     *local      = NULL;
-        call_frame_t    *lock_frame = NULL;
-
-        GF_VALIDATE_OR_GOTO ("dht-locks", frame, out);
-        GF_VALIDATE_OR_GOTO (frame->this->name, lk_array, out);
-        GF_VALIDATE_OR_GOTO (frame->this->name, inodelk_cbk, out);
-
-        lock_frame = dht_lock_frame (frame);
-        if (lock_frame == NULL)
-                goto out;
-
-        ret = dht_local_lock_init (lock_frame, lk_array, lk_count, inodelk_cbk);
-        if (ret < 0) {
-                goto out;
-        }
-
-        dht_set_lkowner (lk_array, lk_count, &lock_frame->root->lk_owner);
-
-        local = lock_frame->local;
-        local->main_frame = frame;
-
-        local->call_cnt = lk_count;
-
-        for (i = 0; i < local->lock.lk_count; i++) {
-                flock.l_type = local->lock.locks[i]->type;
-
-                STACK_WIND_COOKIE (lock_frame, dht_nonblocking_inodelk_cbk,
-                                   (void *) (long) i,
-                                   local->lock.locks[i]->xl,
-                                   local->lock.locks[i]->xl->fops->inodelk,
-                                   local->lock.locks[i]->domain,
-                                   &local->lock.locks[i]->loc, F_SETLK,
-                                   &flock, NULL);
+    call_frame_t *heal_frame = data;
+    dht_local_t *local = NULL;
+    loc_t loc = {
+        0,
+    };
+    dict_t *dict = NULL;
+    char *path = NULL;
+    int ret = -1;
+    xlator_t *source = NULL;
+    xlator_t *this = NULL;
+    inode_table_t *itable = NULL;
+    inode_t *inode = NULL;
+    inode_t *tmp_inode = NULL;
+
+    GF_VALIDATE_OR_GOTO("DHT", heal_frame, out);
+
+    local = heal_frame->local;
+    this = heal_frame->this;
+    source = heal_frame->cookie;
+    heal_frame->cookie = NULL;
+    gf_uuid_copy(loc.gfid, local->gfid);
+
+    if (local->loc.inode)
+        loc.inode = inode_ref(local->loc.inode);
+    else
+        goto out;
+
+    itable = loc.inode->table;
+    ret = syncop_getxattr(source, &loc, &dict, GET_ANCESTRY_PATH_KEY, NULL,
+                          NULL);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_INFO, -ret, DHT_MSG_DIR_HEAL_ABORT,
+                "subvol=%s", source->name, NULL);
+        goto out;
+    }
+
+    ret = dict_get_str(dict, GET_ANCESTRY_PATH_KEY, &path);
+    if (path) {
+        inode = dht_heal_path(this, path, itable);
+        if (inode && inode != local->inode) {
+            /*
+             * if inode returned by heal function is different
+             * from what we passed, which means a racing thread
+             * already linked a different inode for dentry.
+             * So we will update our local->inode, so that we can
+             * retrurn proper inode.
+             */
+            tmp_inode = local->inode;
+            local->inode = inode;
+            inode_unref(tmp_inode);
+            tmp_inode = NULL;
+        } else {
+            inode_unref(inode);
         }
-
-        return 0;
+    }
 
 out:
-        if (lock_frame)
-                dht_lock_stack_destroy (lock_frame);
-
-        return -1;
+    loc_wipe(&loc);
+    if (dict)
+        dict_unref(dict);
+    return 0;
 }
 
-int32_t
-dht_blocking_inodelk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                          int32_t op_ret, int32_t op_errno, dict_t *xdata)
+int
+dht_heal_full_path_done(int op_ret, call_frame_t *heal_frame, void *data)
 {
-        int          lk_index = 0;
-        dht_local_t *local    = NULL;
-
-        lk_index = (long) cookie;
-
-        local = frame->local;
-
-        if (op_ret == 0) {
-                local->lock.locks[lk_index]->locked = _gf_true;
-        } else {
-                local->lock.op_ret = -1;
-                local->lock.op_errno = op_errno;
-                goto cleanup;
-        }
-
-        if (lk_index == (local->lock.lk_count - 1)) {
-                dht_inodelk_done (frame);
-        } else {
-                dht_blocking_inodelk_rec (frame, ++lk_index);
+    call_frame_t *main_frame = NULL;
+    dht_local_t *local = NULL;
+    xlator_t *this = NULL;
+    int ret = -1;
+    int op_errno = 0;
+
+    local = heal_frame->local;
+    main_frame = local->main_frame;
+    local->main_frame = NULL;
+    this = heal_frame->this;
+
+    dht_set_fixed_dir_stat(&local->postparent);
+    if (local->need_xattr_heal) {
+        local->need_xattr_heal = 0;
+        ret = dht_dir_xattr_heal(this, local, &op_errno);
+        if (ret) {
+            gf_smsg(this->name, GF_LOG_ERROR, op_errno,
+                    DHT_MSG_DIR_XATTR_HEAL_FAILED, "path=%s", local->loc.path,
+                    NULL);
         }
+    }
 
-        return 0;
+    DHT_STACK_UNWIND(lookup, main_frame, 0, 0, local->inode, &local->stbuf,
+                     local->xattr, &local->postparent);
 
-cleanup:
-        dht_inodelk_cleanup (frame);
-
-        return 0;
+    DHT_STACK_DESTROY(heal_frame);
+    return 0;
 }
 
-void
-dht_blocking_inodelk_rec (call_frame_t *frame, int i)
+/* This function must be called inside an inode lock */
+int
+__dht_lock_subvol_set(inode_t *inode, xlator_t *this, xlator_t *lock_subvol)
 {
-        dht_local_t     *local = NULL;
-        struct gf_flock  flock = {0,};
+    dht_inode_ctx_t *ctx = NULL;
+    int ret = -1;
+    uint64_t value = 0;
 
-        local = frame->local;
+    GF_VALIDATE_OR_GOTO(this->name, inode, out);
 
-        flock.l_type = local->lock.locks[i]->type;
-
-        STACK_WIND_COOKIE (frame, dht_blocking_inodelk_cbk,
-                           (void *) (long) i,
-                           local->lock.locks[i]->xl,
-                           local->lock.locks[i]->xl->fops->inodelk,
-                           local->lock.locks[i]->domain,
-                           &local->lock.locks[i]->loc, F_SETLKW, &flock, NULL);
+    ret = __inode_ctx_get0(inode, this, &value);
+    if (ret || !value) {
+        return -1;
+    }
 
-        return;
+    ctx = (dht_inode_ctx_t *)(uintptr_t)value;
+    ctx->lock_subvol = lock_subvol;
+out:
+    return ret;
 }
 
-int
-dht_lock_request_cmp (const void *val1, const void *val2)
+xlator_t *
+dht_get_lock_subvolume(xlator_t *this, struct gf_flock *lock,
+                       dht_local_t *local)
 {
-        dht_lock_t *lock1 = NULL;
-        dht_lock_t *lock2 = NULL;
-        int         ret   = 0;
+    xlator_t *subvol = NULL;
+    inode_t *inode = NULL;
+    int32_t ret = -1;
+    uint64_t value = 0;
+    xlator_t *cached_subvol = NULL;
+    dht_inode_ctx_t *ctx = NULL;
+    char gfid[GF_UUID_BUF_SIZE] = {0};
 
-        lock1 = *(dht_lock_t **)val1;
-        lock2 = *(dht_lock_t **)val2;
+    GF_VALIDATE_OR_GOTO(this->name, lock, out);
+    GF_VALIDATE_OR_GOTO(this->name, local, out);
 
-        GF_VALIDATE_OR_GOTO ("dht-locks", lock1, out);
-        GF_VALIDATE_OR_GOTO ("dht-locks", lock2, out);
+    cached_subvol = local->cached_subvol;
 
-        ret = strcmp (lock1->xl->name, lock2->xl->name);
-
-        if (ret == 0) {
-                ret = gf_uuid_compare (lock1->loc.gfid, lock2->loc.gfid);
-        }
+    if (local->loc.inode || local->fd) {
+        inode = local->loc.inode ? local->loc.inode : local->fd->inode;
+    }
 
-out:
-        return ret;
-}
+    if (!inode)
+        goto out;
 
-int
-dht_lock_order_requests (dht_lock_t **locks, int count)
-{
-        int        ret     = -1;
-
-        if (!locks || !count)
-                goto out;
-
-        qsort (locks, count, sizeof (*locks), dht_lock_request_cmp);
-        ret = 0;
+    if (!(IA_ISDIR(inode->ia_type) || IA_ISINVAL(inode->ia_type))) {
+        /*
+         * We may get non-linked inode for directories as part
+         * of the selfheal code path. So checking  for IA_INVAL
+         * type also. This will only happen for directory.
+         */
+        subvol = local->cached_subvol;
+        goto out;
+    }
 
+    if (lock->l_type != F_UNLCK) {
+        /*
+         * inode purging might happen on NFS between a lk
+         * and unlk. Due to this lk and unlk might be sent
+         * to different subvols.
+         * So during a lock request, taking a ref on inode
+         * to prevent inode purging. inode unref will happen
+         * in unlock cbk code path.
+         */
+        inode_ref(inode);
+    }
+
+    LOCK(&inode->lock);
+    ret = __inode_ctx_get0(inode, this, &value);
+    if (!ret && value) {
+        ctx = (dht_inode_ctx_t *)(uintptr_t)value;
+        subvol = ctx->lock_subvol;
+    }
+    if (!subvol && lock->l_type != F_UNLCK && cached_subvol) {
+        ret = __dht_lock_subvol_set(inode, this, cached_subvol);
+        if (ret) {
+            gf_uuid_unparse(inode->gfid, gfid);
+            UNLOCK(&inode->lock);
+            gf_smsg(this->name, GF_LOG_WARNING, 0, DHT_MSG_SET_INODE_CTX_FAILED,
+                    "lock_subvol gfid=%s", gfid, NULL);
+            goto post_unlock;
+        }
+        subvol = cached_subvol;
+    }
+    UNLOCK(&inode->lock);
+post_unlock:
+    if (!subvol && inode && lock->l_type != F_UNLCK) {
+        inode_unref(inode);
+    }
 out:
-        return ret;
+    return subvol;
 }
 
 int
-dht_blocking_inodelk (call_frame_t *frame, dht_lock_t **lk_array,
-                      int lk_count, fop_inodelk_cbk_t inodelk_cbk)
+dht_lk_inode_unref(call_frame_t *frame, int32_t op_ret)
 {
-        int           ret        = -1;
-        call_frame_t *lock_frame = NULL;
-        dht_local_t  *local      = NULL;
-
-        GF_VALIDATE_OR_GOTO ("dht-locks", frame, out);
-        GF_VALIDATE_OR_GOTO (frame->this->name, lk_array, out);
-        GF_VALIDATE_OR_GOTO (frame->this->name, inodelk_cbk, out);
-
-        lock_frame = dht_lock_frame (frame);
-        if (lock_frame == NULL)
+    int ret = -1;
+    dht_local_t *local = NULL;
+    inode_t *inode = NULL;
+    xlator_t *this = NULL;
+    char gfid[GF_UUID_BUF_SIZE] = {0};
+
+    local = frame->local;
+    this = frame->this;
+
+    if (local->loc.inode || local->fd) {
+        inode = local->loc.inode ? local->loc.inode : local->fd->inode;
+    }
+    if (!inode) {
+        gf_smsg(this->name, GF_LOG_WARNING, 0, DHT_MSG_LOCK_INODE_UNREF_FAILED,
+                NULL);
+        goto out;
+    }
+
+    if (!(IA_ISDIR(inode->ia_type) || IA_ISINVAL(inode->ia_type))) {
+        ret = 0;
+        goto out;
+    }
+
+    switch (local->lock_type) {
+        case F_RDLCK:
+        case F_WRLCK:
+            if (op_ret) {
+                gf_uuid_unparse(inode->gfid, gfid);
+                gf_msg_debug(this->name, 0, "lock request failed for gfid %s",
+                             gfid);
+                inode_unref(inode);
                 goto out;
-
-        ret = dht_local_lock_init (lock_frame, lk_array, lk_count, inodelk_cbk);
-        if (ret < 0) {
+            }
+            break;
+
+        case F_UNLCK:
+            if (!op_ret) {
+                inode_unref(inode);
+            } else {
+                gf_uuid_unparse(inode->gfid, gfid);
+                gf_smsg(this->name, GF_LOG_WARNING, 0,
+                        DHT_MSG_LOCK_INODE_UNREF_FAILED, "gfid=%s", gfid, NULL);
                 goto out;
-        }
-
-        dht_set_lkowner (lk_array, lk_count, &lock_frame->root->lk_owner);
-
-        local = lock_frame->local;
-        local->main_frame = frame;
-
-        dht_blocking_inodelk_rec (lock_frame, 0);
-
-        return 0;
+            }
+        default:
+            break;
+    }
+    ret = 0;
 out:
-        if (lock_frame)
-                dht_lock_stack_destroy (lock_frame);
+    return ret;
+}
 
-        return -1;
+/* Code to update custom extended attributes from src dict to dst dict
+ */
+void
+dht_dir_set_heal_xattr(xlator_t *this, dht_local_t *local, dict_t *dst,
+                       dict_t *src, int *uret, int *uflag)
+{
+    int ret = -1;
+    data_t *keyval = NULL;
+    int luret = -1;
+    int luflag = -1;
+    int i = 0;
+    char **xattrs_to_heal;
+
+    if (!src || !dst) {
+        gf_smsg(this->name, GF_LOG_WARNING, EINVAL, DHT_MSG_DST_NULL_SET_FAILED,
+                "path=%s", local->loc.path, NULL);
+        return;
+    }
+    /* Check if any user xattr present in src dict and set
+       it to dst dict
+    */
+    luret = dict_foreach_fnmatch(src, "user.*", dht_set_user_xattr, dst);
+    /* Check if any other custom xattr present in src dict
+       and set it to dst dict, here index start from 1 because
+       user xattr already checked in previous statement
+    */
+
+    xattrs_to_heal = get_xattrs_to_heal();
+
+    for (i = 1; xattrs_to_heal[i]; i++) {
+        keyval = dict_get(src, xattrs_to_heal[i]);
+        if (keyval) {
+            luflag = 1;
+            ret = dict_set(dst, xattrs_to_heal[i], keyval);
+            if (ret)
+                gf_smsg(this->name, GF_LOG_WARNING, ENOMEM,
+                        DHT_MSG_DICT_SET_FAILED, "key=%s", xattrs_to_heal[i],
+                        "path=%s", local->loc.path, NULL);
+            keyval = NULL;
+        }
+    }
+    if (uret)
+        (*uret) = luret;
+    if (uflag)
+        (*uflag) = luflag;
 }
diff --git a/xlators/cluster/dht/src/dht-inode-read.c b/xlators/cluster/dht/src/dht-inode-read.c
index 78e3ef4233b..dbb8070b0da 100644
--- a/xlators/cluster/dht/src/dht-inode-read.c
+++ b/xlators/cluster/dht/src/dht-inode-read.c
@@ -8,1132 +8,1651 @@
   cases as published by the Free Software Foundation.
 */
 
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
-
 #include "dht-common.h"
 
-int dht_access2 (xlator_t *this, call_frame_t *frame, int ret);
-int dht_readv2 (xlator_t *this, call_frame_t *frame, int ret);
-int dht_attr2 (xlator_t *this, call_frame_t *frame, int ret);
-int dht_open2 (xlator_t *this, call_frame_t *frame, int ret);
-int dht_flush2 (xlator_t *this, call_frame_t *frame, int ret);
-int dht_lk2 (xlator_t *this, call_frame_t *frame, int ret);
-int dht_fsync2 (xlator_t *this, call_frame_t *frame, int ret);
-
-int
-dht_open_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-              int op_ret, int op_errno, fd_t *fd, dict_t *xdata)
+static int
+dht_access2(xlator_t *this, xlator_t *dst_node, call_frame_t *frame, int ret);
+static int
+dht_readv2(xlator_t *this, xlator_t *dst_node, call_frame_t *frame, int ret);
+static int
+dht_attr2(xlator_t *this, xlator_t *dst_node, call_frame_t *frame, int ret);
+static int
+dht_open2(xlator_t *this, xlator_t *dst_node, call_frame_t *frame, int ret);
+static int
+dht_flush2(xlator_t *this, xlator_t *dst_node, call_frame_t *frame, int ret);
+static int
+dht_lk2(xlator_t *this, xlator_t *dst_node, call_frame_t *frame, int ret);
+static int
+dht_fsync2(xlator_t *this, xlator_t *dst_node, call_frame_t *frame, int ret);
+static int
+dht_common_xattrop2(xlator_t *this, xlator_t *subvol, call_frame_t *frame,
+                    int ret);
+
+static int
+dht_open_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int op_ret,
+             int op_errno, fd_t *fd, dict_t *xdata)
 {
-        dht_local_t  *local = NULL;
-        call_frame_t *prev = NULL;
-        int           ret = 0;
-
-        local = frame->local;
-        prev = cookie;
-
-        local->op_errno = op_errno;
-        if ((op_ret == -1) && !dht_inode_missing(op_errno)) {
-                gf_msg_debug (this->name, 0,
-                              "subvolume %s returned -1 (%s)",
-                              prev->this->name, strerror (op_errno));
-                goto out;
-        }
-
-        if (!op_ret || (local->call_cnt != 1))
-                goto out;
-
-        /* rebalance would have happened */
-        local->rebalance.target_op_fn = dht_open2;
-        ret = dht_rebalance_complete_check (this, frame);
-        if (!ret)
-                return 0;
+    dht_local_t *local = NULL;
+    xlator_t *prev = NULL;
+    int ret = 0;
+
+    local = frame->local;
+    prev = cookie;
+
+    local->op_errno = op_errno;
+    if ((op_ret == -1) && !dht_inode_missing(op_errno)) {
+        gf_msg_debug(this->name, op_errno, "subvolume %s returned -1",
+                     prev->name);
+        goto out;
+    }
+
+    /* Update ctx if the fd has been opened on the target*/
+    if (!op_ret && (local->call_cnt == 1)) {
+        dht_fd_ctx_set(this, fd, prev);
+        goto out;
+    }
+
+    if (!op_ret || (local->call_cnt != 1))
+        goto out;
+
+    /* rebalance would have happened */
+    local->rebalance.target_op_fn = dht_open2;
+    ret = dht_rebalance_complete_check(this, frame);
+    if (!ret)
+        return 0;
 
 out:
-        DHT_STACK_UNWIND (open, frame, op_ret, op_errno, local->fd, xdata);
+    DHT_STACK_UNWIND(open, frame, op_ret, op_errno, local->fd, xdata);
 
-        return 0;
+    return 0;
 }
 
-int
-dht_open2 (xlator_t *this, call_frame_t *frame, int op_ret)
+static int
+dht_open2(xlator_t *this, xlator_t *subvol, call_frame_t *frame, int ret)
 {
-        dht_local_t *local  = NULL;
-        xlator_t    *subvol = NULL;
-        int          op_errno = EINVAL;
-
-        local = frame->local;
-        if (!local)
-                goto out;
+    dht_local_t *local = NULL;
+    int op_errno = EINVAL;
 
-        op_errno = ENOENT;
-        if (op_ret)
-                goto out;
+    if (!frame || !frame->local)
+        goto out;
 
-        local->call_cnt = 2;
-        subvol = local->cached_subvol;
+    local = frame->local;
+    op_errno = local->op_errno;
 
-        STACK_WIND (frame, dht_open_cbk, subvol, subvol->fops->open,
-                    &local->loc, local->rebalance.flags, local->fd,
-                    NULL);
+    if (we_are_not_migrating(ret)) {
+        /* This DHT layer is not migrating the file */
+        DHT_STACK_UNWIND(open, frame, -1, local->op_errno, NULL,
+                         local->rebalance.xdata);
         return 0;
+    }
+
+    if (subvol == NULL)
+        goto out;
+
+    local->call_cnt = 2;
+
+    STACK_WIND_COOKIE(frame, dht_open_cbk, subvol, subvol, subvol->fops->open,
+                      &local->loc, local->rebalance.flags, local->fd,
+                      local->xattr_req);
+    return 0;
 
 out:
-        DHT_STACK_UNWIND (stat, frame, -1, op_errno, NULL, NULL);
-        return 0;
+    DHT_STACK_UNWIND(open, frame, -1, op_errno, NULL, NULL);
+    return 0;
 }
 
 int
-dht_open (call_frame_t *frame, xlator_t *this,
-          loc_t *loc, int flags, fd_t *fd, dict_t *xdata)
+dht_open(call_frame_t *frame, xlator_t *this, loc_t *loc, int flags, fd_t *fd,
+         dict_t *xdata)
 {
-        xlator_t     *subvol = NULL;
-        int           op_errno = -1;
-        dht_local_t  *local = NULL;
-
-        VALIDATE_OR_GOTO (frame, err);
-        VALIDATE_OR_GOTO (this, err);
-        VALIDATE_OR_GOTO (fd, err);
-
-        local = dht_local_init (frame, loc, fd, GF_FOP_OPEN);
-        if (!local) {
-                op_errno = ENOMEM;
-                goto err;
-        }
+    xlator_t *subvol = NULL;
+    int op_errno = -1;
+    dht_local_t *local = NULL;
 
-        subvol = local->cached_subvol;
-        if (!subvol) {
-                gf_msg_debug (this->name, 0,
-                              "no cached subvolume for fd=%p", fd);
-                op_errno = EINVAL;
-                goto err;
-        }
+    VALIDATE_OR_GOTO(frame, err);
+    VALIDATE_OR_GOTO(this, err);
+    VALIDATE_OR_GOTO(fd, err);
 
-        local->rebalance.flags = flags;
-        local->call_cnt = 1;
+    local = dht_local_init(frame, loc, fd, GF_FOP_OPEN);
+    if (!local) {
+        op_errno = ENOMEM;
+        goto err;
+    }
 
-        STACK_WIND (frame, dht_open_cbk, subvol, subvol->fops->open,
-                    loc, flags, fd, xdata);
+    subvol = local->cached_subvol;
+    if (!subvol) {
+        gf_msg_debug(this->name, 0, "no cached subvolume for fd=%p", fd);
+        op_errno = EINVAL;
+        goto err;
+    }
+    if (xdata)
+        local->xattr_req = dict_ref(xdata);
 
-        return 0;
+    local->rebalance.flags = flags;
+    local->call_cnt = 1;
+
+    STACK_WIND_COOKIE(frame, dht_open_cbk, subvol, subvol, subvol->fops->open,
+                      loc, flags, fd, xdata);
+
+    return 0;
 
 err:
-        op_errno = (op_errno == -1) ? errno : op_errno;
-        DHT_STACK_UNWIND (open, frame, -1, op_errno, NULL, NULL);
+    op_errno = (op_errno == -1) ? errno : op_errno;
+    DHT_STACK_UNWIND(open, frame, -1, op_errno, NULL, NULL);
 
-        return 0;
+    return 0;
 }
 
 int
-dht_file_attr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                   int op_ret, int op_errno, struct iatt *stbuf, dict_t *xdata)
+dht_file_attr_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int op_ret,
+                  int op_errno, struct iatt *stbuf, dict_t *xdata)
 {
-        xlator_t     *subvol = 0;
-        dht_local_t  *local = NULL;
-        call_frame_t *prev = NULL;
-        int           ret = -1;
-        inode_t      *inode = NULL;
-
-        GF_VALIDATE_OR_GOTO ("dht", frame, err);
-        GF_VALIDATE_OR_GOTO ("dht", this, out);
-        GF_VALIDATE_OR_GOTO ("dht", frame->local, out);
-        GF_VALIDATE_OR_GOTO ("dht", cookie, out);
-
-        local = frame->local;
-        prev = cookie;
-
-        if ((op_ret == -1) && !dht_inode_missing(op_errno)) {
-                local->op_errno = op_errno;
-                gf_msg_debug (this->name, 0,
-                              "subvolume %s returned -1 (%s)",
-                              prev->this->name, strerror (op_errno));
-                goto out;
-        }
-
-        if (local->call_cnt != 1)
-                goto out;
+    xlator_t *subvol1 = 0;
+    xlator_t *subvol2 = 0;
+    dht_local_t *local = NULL;
+    xlator_t *prev = NULL;
+    int ret = -1;
+    inode_t *inode = NULL;
+
+    GF_VALIDATE_OR_GOTO("dht", frame, err);
+    GF_VALIDATE_OR_GOTO("dht", this, out);
+    GF_VALIDATE_OR_GOTO("dht", frame->local, out);
+    GF_VALIDATE_OR_GOTO("dht", cookie, out);
+
+    local = frame->local;
+    prev = cookie;
+
+    if ((local->fop == GF_FOP_FSTAT) &&
+        dht_check_remote_fd_failed_error(local, op_ret, op_errno)) {
+        ret = dht_check_and_open_fd_on_subvol(this, frame);
+        if (ret)
+            goto out;
+        return 0;
+    }
 
+    if ((op_ret == -1) && !dht_inode_missing(op_errno)) {
         local->op_errno = op_errno;
-        /* Check if the rebalance phase2 is true */
-        if ((op_ret == -1) || IS_DHT_MIGRATION_PHASE2 (stbuf)) {
-                inode = (local->fd) ? local->fd->inode : local->loc.inode;
-                ret = dht_inode_ctx_get1 (this, inode, &subvol);
-                if (!subvol) {
-                        /* Phase 2 of migration */
-                        local->rebalance.target_op_fn = dht_attr2;
-                        ret = dht_rebalance_complete_check (this, frame);
-                        if (!ret)
-                                return 0;
-                } else {
-                        /* value is already set in fd_ctx, that means no need
-                           to check for whether its complete or not. */
-                        dht_attr2 (this, frame, 0);
-                        return 0;
-                }
+        gf_msg_debug(this->name, op_errno, "subvolume %s returned -1",
+                     prev->name);
+        goto out;
+    }
+
+    if (local->call_cnt != 1)
+        goto out;
+
+    local->op_errno = op_errno;
+    local->op_ret = op_ret;
+
+    /* Check if the rebalance phase2 is true */
+    if ((op_ret == -1) || IS_DHT_MIGRATION_PHASE2(stbuf)) {
+        local->rebalance.target_op_fn = dht_attr2;
+        dht_set_local_rebalance(this, local, NULL, NULL, stbuf, xdata);
+        inode = (local->fd) ? local->fd->inode : local->loc.inode;
+
+        dht_inode_ctx_get_mig_info(this, inode, &subvol1, &subvol2);
+        if (dht_mig_info_is_invalid(local->cached_subvol, subvol1, subvol2)) {
+            /* Phase 2 of migration */
+            ret = dht_rebalance_complete_check(this, frame);
+            if (!ret)
+                return 0;
+        } else {
+            /* it is a non-fd op or it is an fd based Fop and
+               opened on the dst.*/
+            if (local->fd && !dht_fd_open_on_dst(this, local->fd, subvol2)) {
+                ret = dht_rebalance_complete_check(this, frame);
+                if (!ret)
+                    return 0;
+            } else {
+                dht_attr2(this, subvol2, frame, 0);
+                return 0;
+            }
         }
+    }
 
 out:
-        DHT_STRIP_PHASE1_FLAGS (stbuf);
-        DHT_STACK_UNWIND (stat, frame, op_ret, op_errno, stbuf, xdata);
+    DHT_STRIP_PHASE1_FLAGS(stbuf);
+    DHT_STACK_UNWIND(stat, frame, op_ret, op_errno, stbuf, xdata);
 err:
-        return 0;
+    return 0;
 }
 
-int
-dht_attr2 (xlator_t *this, call_frame_t *frame, int op_ret)
+static int
+dht_attr2(xlator_t *this, xlator_t *subvol, call_frame_t *frame, int ret)
 {
-        dht_local_t *local  = NULL;
-        xlator_t    *subvol = NULL;
-        int          op_errno = EINVAL;
+    dht_local_t *local = NULL;
+    int op_errno = EINVAL;
+
+    local = frame->local;
+    if (!local)
+        goto out;
+
+    op_errno = local->op_errno;
+
+    if (we_are_not_migrating(ret)) {
+        /* This dht xlator is not migrating the file. Unwind and
+         * pass on the original mode bits so the higher DHT layer
+         * can handle this.
+         */
+        DHT_STACK_UNWIND(stat, frame, local->op_ret, op_errno,
+                         &local->rebalance.postbuf, local->rebalance.xdata);
+        return 0;
+    }
 
-        local = frame->local;
-        if (!local)
-                goto out;
+    if (subvol == NULL)
+        goto out;
 
-        op_errno = local->op_errno;
-        if (op_ret == -1)
-                goto out;
+    local->call_cnt = 2;
 
-        subvol = local->cached_subvol;
-        local->call_cnt = 2;
+    if (local->fop == GF_FOP_FSTAT) {
+        STACK_WIND_COOKIE(frame, dht_file_attr_cbk, subvol, subvol,
+                          subvol->fops->fstat, local->fd, local->xattr_req);
+    } else {
+        STACK_WIND_COOKIE(frame, dht_file_attr_cbk, subvol, subvol,
+                          subvol->fops->stat, &local->loc, local->xattr_req);
+    }
 
-        if (local->fop == GF_FOP_FSTAT) {
-                STACK_WIND (frame, dht_file_attr_cbk, subvol,
-                            subvol->fops->fstat, local->fd, NULL);
-        } else {
-                STACK_WIND (frame, dht_file_attr_cbk, subvol,
-                            subvol->fops->stat, &local->loc, NULL);
-        }
-        return 0;
+    return 0;
 
 out:
-        DHT_STACK_UNWIND (stat, frame, -1, op_errno, NULL, NULL);
-        return 0;
+    DHT_STACK_UNWIND(stat, frame, -1, op_errno, NULL, NULL);
+    return 0;
+}
+
+static int
+dht_attr_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int op_ret,
+             int op_errno, struct iatt *stbuf, dict_t *xdata)
+{
+    dht_local_t *local = NULL;
+    int this_call_cnt = 0;
+    xlator_t *prev = NULL;
+    local = frame->local;
+    prev = cookie;
+
+    LOCK(&frame->lock);
+    {
+        if (op_ret == -1) {
+            local->op_errno = op_errno;
+            UNLOCK(&frame->lock);
+            gf_msg_debug(this->name, op_errno, "subvolume %s returned -1",
+                         prev->name);
+
+            goto post_unlock;
+        }
+
+        dht_iatt_merge(this, &local->stbuf, stbuf);
+
+        local->op_ret = 0;
+    }
+    UNLOCK(&frame->lock);
+post_unlock:
+    this_call_cnt = dht_frame_return(frame);
+    if (is_last_call(this_call_cnt)) {
+        DHT_STACK_UNWIND(stat, frame, local->op_ret, local->op_errno,
+                         &local->stbuf, xdata);
+    }
+
+    return 0;
 }
 
 int
-dht_attr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-              int op_ret, int op_errno, struct iatt *stbuf, dict_t *xdata)
+dht_stat(call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xdata)
 {
-        dht_local_t  *local = NULL;
-        int           this_call_cnt = 0;
-        call_frame_t *prev = NULL;
+    xlator_t *subvol = NULL;
+    int op_errno = -1;
+    dht_local_t *local = NULL;
+    dht_layout_t *layout = NULL;
+    int i = 0;
+    int call_cnt = 0;
+
+    VALIDATE_OR_GOTO(frame, err);
+    VALIDATE_OR_GOTO(this, err);
+    VALIDATE_OR_GOTO(loc, err);
+    VALIDATE_OR_GOTO(loc->inode, err);
+    VALIDATE_OR_GOTO(loc->path, err);
+
+    local = dht_local_init(frame, loc, NULL, GF_FOP_STAT);
+    if (!local) {
+        op_errno = ENOMEM;
+        goto err;
+    }
+
+    layout = local->layout;
+    if (!layout) {
+        gf_msg_debug(this->name, 0, "no layout for path=%s", loc->path);
+        op_errno = EINVAL;
+        goto err;
+    }
+    if (xdata)
+        local->xattr_req = dict_ref(xdata);
+
+    if (IA_ISREG(loc->inode->ia_type)) {
+        local->call_cnt = 1;
 
-        GF_VALIDATE_OR_GOTO ("dht", frame, err);
-        GF_VALIDATE_OR_GOTO ("dht", this, out);
-        GF_VALIDATE_OR_GOTO ("dht", frame->local, out);
-        GF_VALIDATE_OR_GOTO ("dht", cookie, out);
+        subvol = local->cached_subvol;
 
-        local = frame->local;
-        prev = cookie;
+        STACK_WIND_COOKIE(frame, dht_file_attr_cbk, subvol, subvol,
+                          subvol->fops->stat, loc, xdata);
 
-        LOCK (&frame->lock);
-        {
-                if (op_ret == -1) {
-                        local->op_errno = op_errno;
-                        gf_msg_debug (this->name, 0,
-                                      "subvolume %s returned -1 (%s)",
-                                      prev->this->name, strerror (op_errno));
+        return 0;
+    }
 
-                        goto unlock;
-                }
+    local->call_cnt = call_cnt = layout->cnt;
 
-                dht_iatt_merge (this, &local->stbuf, stbuf, prev->this);
+    for (i = 0; i < call_cnt; i++) {
+        subvol = layout->list[i].xlator;
+
+        STACK_WIND_COOKIE(frame, dht_attr_cbk, subvol, subvol,
+                          subvol->fops->stat, loc, xdata);
+    }
+
+    return 0;
 
-                local->op_ret = 0;
-        }
-unlock:
-        UNLOCK (&frame->lock);
-out:
-        this_call_cnt = dht_frame_return (frame);
-        if (is_last_call (this_call_cnt)) {
-                DHT_STACK_UNWIND (stat, frame, local->op_ret, local->op_errno,
-                                  &local->stbuf, xdata);
-        }
 err:
-        return 0;
+    op_errno = (op_errno == -1) ? errno : op_errno;
+    DHT_STACK_UNWIND(stat, frame, -1, op_errno, NULL, NULL);
+
+    return 0;
 }
 
 int
-dht_stat (call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xdata)
+dht_fstat(call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *xdata)
 {
-        xlator_t     *subvol = NULL;
-        int           op_errno = -1;
-        dht_local_t  *local = NULL;
-        dht_layout_t *layout = NULL;
-        int           i = 0;
-        int           call_cnt = 0;
-
-        VALIDATE_OR_GOTO (frame, err);
-        VALIDATE_OR_GOTO (this, err);
-        VALIDATE_OR_GOTO (loc, err);
-        VALIDATE_OR_GOTO (loc->inode, err);
-        VALIDATE_OR_GOTO (loc->path, err);
-
-
-        local = dht_local_init (frame, loc, NULL, GF_FOP_STAT);
-        if (!local) {
-                op_errno = ENOMEM;
-                goto err;
-        }
+    xlator_t *subvol = NULL;
+    int op_errno = -1;
+    dht_local_t *local = NULL;
+    dht_layout_t *layout = NULL;
+    int i = 0;
+    int call_cnt = 0;
+
+    VALIDATE_OR_GOTO(frame, err);
+    VALIDATE_OR_GOTO(this, err);
+    VALIDATE_OR_GOTO(fd, err);
+
+    local = dht_local_init(frame, NULL, fd, GF_FOP_FSTAT);
+    if (!local) {
+        op_errno = ENOMEM;
+        goto err;
+    }
+
+    layout = local->layout;
+    if (!layout) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, 0, "no layout for fd=%p", fd);
+        op_errno = EINVAL;
+        goto err;
+    }
+    if (xdata)
+        local->xattr_req = dict_ref(xdata);
+
+    if (IA_ISREG(fd->inode->ia_type)) {
+        local->call_cnt = 1;
 
-        layout = local->layout;
-        if (!layout) {
-                gf_msg_debug (this->name, 0,
-                              "no layout for path=%s", loc->path);
-                op_errno = EINVAL;
-                goto err;
-        }
+        subvol = local->cached_subvol;
 
-        if (IA_ISREG (loc->inode->ia_type)) {
-                local->call_cnt = 1;
+        STACK_WIND_COOKIE(frame, dht_file_attr_cbk, subvol, subvol,
+                          subvol->fops->fstat, fd, xdata);
+        return 0;
+    }
+
+    local->call_cnt = call_cnt = layout->cnt;
+
+    for (i = 0; i < call_cnt; i++) {
+        subvol = layout->list[i].xlator;
+        STACK_WIND_COOKIE(frame, dht_attr_cbk, subvol, subvol,
+                          subvol->fops->fstat, fd, xdata);
+    }
 
-                subvol = local->cached_subvol;
+    return 0;
+
+err:
+    op_errno = (op_errno == -1) ? errno : op_errno;
+    DHT_STACK_UNWIND(fstat, frame, -1, op_errno, NULL, NULL);
 
-                STACK_WIND (frame, dht_file_attr_cbk, subvol,
-                            subvol->fops->stat, loc, xdata);
+    return 0;
+}
 
+int
+dht_readv_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int op_ret,
+              int op_errno, struct iovec *vector, int count, struct iatt *stbuf,
+              struct iobref *iobref, dict_t *xdata)
+{
+    dht_local_t *local = NULL;
+    int ret = 0;
+    xlator_t *src_subvol = 0;
+    xlator_t *dst_subvol = 0;
+
+    local = frame->local;
+    if (!local) {
+        op_ret = -1;
+        op_errno = EINVAL;
+        goto out;
+    }
+
+    /* This is already second try, no need for re-check */
+    if (local->call_cnt != 1)
+        goto out;
+
+    if (dht_check_remote_fd_failed_error(local, op_ret, op_errno)) {
+        ret = dht_check_and_open_fd_on_subvol(this, frame);
+        if (ret)
+            goto out;
+        return 0;
+    }
+
+    if ((op_ret == -1) && !dht_inode_missing(op_errno))
+        goto out;
+
+    local->op_errno = op_errno;
+    if ((op_ret == -1) || IS_DHT_MIGRATION_PHASE2(stbuf)) {
+        local->op_ret = op_ret;
+        local->rebalance.target_op_fn = dht_readv2;
+        dht_set_local_rebalance(this, local, NULL, NULL, stbuf, xdata);
+        /* File would be migrated to other node */
+        ret = dht_inode_ctx_get_mig_info(this, local->fd->inode, &src_subvol,
+                                         &dst_subvol);
+
+        if (dht_mig_info_is_invalid(local->cached_subvol, src_subvol,
+                                    dst_subvol) ||
+            !dht_fd_open_on_dst(this, local->fd, dst_subvol)) {
+            ret = dht_rebalance_complete_check(this, frame);
+            if (!ret)
                 return 0;
+        } else {
+            /* value is already set in fd_ctx, that means no need
+               to check for whether its complete or not. */
+            dht_readv2(this, dst_subvol, frame, 0);
+            return 0;
         }
+    }
 
-        local->call_cnt = call_cnt = layout->cnt;
+out:
+    DHT_STRIP_PHASE1_FLAGS(stbuf);
 
-        for (i = 0; i < call_cnt; i++) {
-                subvol = layout->list[i].xlator;
+    DHT_STACK_UNWIND(readv, frame, op_ret, op_errno, vector, count, stbuf,
+                     iobref, xdata);
 
-                STACK_WIND (frame, dht_attr_cbk,
-                            subvol, subvol->fops->stat,
-                            loc, xdata);
-        }
+    return 0;
+}
 
+static int
+dht_readv2(xlator_t *this, xlator_t *subvol, call_frame_t *frame, int ret)
+{
+    dht_local_t *local = NULL;
+    int op_errno = EINVAL;
+
+    local = frame->local;
+    if (!local)
+        goto out;
+
+    op_errno = local->op_errno;
+
+    if (we_are_not_migrating(ret)) {
+        /* This dht xlator is not migrating the file. Unwind and
+         * pass on the original mode bits so the higher DHT layer
+         * can handle this.
+         */
+        DHT_STACK_UNWIND(readv, frame, local->op_ret, op_errno, NULL, 0,
+                         &local->rebalance.postbuf, NULL,
+                         local->rebalance.xdata);
         return 0;
+    }
 
-err:
-        op_errno = (op_errno == -1) ? errno : op_errno;
-        DHT_STACK_UNWIND (stat, frame, -1, op_errno, NULL, NULL);
+    if (subvol == NULL)
+        goto out;
 
-        return 0;
-}
+    local->call_cnt = 2;
+
+    STACK_WIND(frame, dht_readv_cbk, subvol, subvol->fops->readv, local->fd,
+               local->rebalance.size, local->rebalance.offset,
+               local->rebalance.flags, local->xattr_req);
 
+    return 0;
+
+out:
+    DHT_STACK_UNWIND(readv, frame, -1, op_errno, NULL, 0, NULL, NULL, NULL);
+    return 0;
+}
 
 int
-dht_fstat (call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *xdata)
+dht_readv(call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size, off_t off,
+          uint32_t flags, dict_t *xdata)
 {
-        xlator_t     *subvol = NULL;
-        int           op_errno = -1;
-        dht_local_t  *local = NULL;
-        dht_layout_t *layout = NULL;
-        int           i = 0;
-        int           call_cnt = 0;
-
-
-        VALIDATE_OR_GOTO (frame, err);
-        VALIDATE_OR_GOTO (this, err);
-        VALIDATE_OR_GOTO (fd, err);
-
-        local = dht_local_init (frame, NULL, fd, GF_FOP_FSTAT);
-        if (!local) {
-                op_errno = ENOMEM;
-                goto err;
-        }
+    xlator_t *subvol = NULL;
+    int op_errno = -1;
+    dht_local_t *local = NULL;
 
-        layout = local->layout;
-        if (!layout) {
-                gf_msg_debug (this->name, 0,
-                              "no layout for fd=%p", fd);
-                op_errno = EINVAL;
-                goto err;
-        }
+    VALIDATE_OR_GOTO(frame, err);
+    VALIDATE_OR_GOTO(this, err);
+    VALIDATE_OR_GOTO(fd, err);
 
-        if (IA_ISREG (fd->inode->ia_type)) {
-                local->call_cnt = 1;
+    local = dht_local_init(frame, NULL, fd, GF_FOP_READ);
+    if (!local) {
+        op_errno = ENOMEM;
+        goto err;
+    }
 
-                subvol = local->cached_subvol;
+    subvol = local->cached_subvol;
+    if (!subvol) {
+        gf_msg_debug(this->name, 0, "no cached subvolume for fd=%p", fd);
+        op_errno = EINVAL;
+        goto err;
+    }
 
-                STACK_WIND (frame, dht_file_attr_cbk, subvol,
-                            subvol->fops->fstat, fd, xdata);
-
-                return 0;
-        }
+    if (xdata)
+        local->xattr_req = dict_ref(xdata);
 
-        local->call_cnt = call_cnt = layout->cnt;
+    local->rebalance.offset = off;
+    local->rebalance.size = size;
+    local->rebalance.flags = flags;
+    local->call_cnt = 1;
 
-        for (i = 0; i < call_cnt; i++) {
-                subvol = layout->list[i].xlator;
-                STACK_WIND (frame, dht_attr_cbk,
-                            subvol, subvol->fops->fstat,
-                            fd, xdata);
-        }
+    STACK_WIND(frame, dht_readv_cbk, subvol, subvol->fops->readv, local->fd,
+               local->rebalance.size, local->rebalance.offset,
+               local->rebalance.flags, local->xattr_req);
 
-        return 0;
+    return 0;
 
 err:
-        op_errno = (op_errno == -1) ? errno : op_errno;
-        DHT_STACK_UNWIND (fstat, frame, -1, op_errno, NULL, NULL);
+    op_errno = (op_errno == -1) ? errno : op_errno;
+    DHT_STACK_UNWIND(readv, frame, -1, op_errno, NULL, 0, NULL, NULL, NULL);
 
-        return 0;
+    return 0;
 }
 
-int
-dht_readv_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-               int op_ret, int op_errno,
-               struct iovec *vector, int count, struct iatt *stbuf,
-               struct iobref *iobref, dict_t *xdata)
+static int
+dht_access_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int op_ret,
+               int op_errno, dict_t *xdata)
 {
-        dht_local_t *local      = NULL;
-        int          ret        = 0;
-        inode_t     *inode      = NULL;
-        xlator_t    *subvol = 0;
-
-        local = frame->local;
-        if (!local) {
-                op_ret = -1;
-                op_errno = EINVAL;
-                goto out;
-        }
-
-        /* This is already second try, no need for re-check */
-        if (local->call_cnt != 1)
-                goto out;
+    int ret = -1;
+    dht_local_t *local = NULL;
+    xlator_t *subvol = NULL;
+    xlator_t *prev = NULL;
+
+    local = frame->local;
+    prev = cookie;
+
+    if (!prev)
+        goto out;
+    if (local->call_cnt != 1)
+        goto out;
+    if ((op_ret == -1) &&
+        ((op_errno == ENOTCONN) || dht_inode_missing(op_errno)) &&
+        IA_ISDIR(local->loc.inode->ia_type)) {
+        subvol = dht_subvol_next_available(this, prev);
+        if (!subvol)
+            goto out;
 
-        if ((op_ret == -1) && !dht_inode_missing(op_errno))
-                goto out;
+        /* check if we are done with visiting every node */
+        if (subvol == local->cached_subvol) {
+            goto out;
+        }
 
+        STACK_WIND_COOKIE(frame, dht_access_cbk, subvol, subvol,
+                          subvol->fops->access, &local->loc,
+                          local->rebalance.flags, NULL);
+        return 0;
+    }
+    if ((op_ret == -1) && dht_inode_missing(op_errno) &&
+        !(IA_ISDIR(local->loc.inode->ia_type))) {
+        /* File would be migrated to other node */
         local->op_errno = op_errno;
-        if ((op_ret == -1) || IS_DHT_MIGRATION_PHASE2 (stbuf)) {
-                /* File would be migrated to other node */
-                ret = dht_inode_ctx_get1 (this, inode, &subvol);
-                if (!subvol) {
-                        local->rebalance.target_op_fn = dht_readv2;
-                        ret = dht_rebalance_complete_check (this, frame);
-                        if (!ret)
-                                return 0;
-                } else {
-                        /* value is already set in fd_ctx, that means no need
-                           to check for whether its complete or not. */
-                        dht_readv2 (this, frame, 0);
-                        return 0;
-                }
-        }
+        local->rebalance.target_op_fn = dht_access2;
+        ret = dht_rebalance_complete_check(frame->this, frame);
+        if (!ret)
+            return 0;
+    }
 
 out:
-        DHT_STRIP_PHASE1_FLAGS (stbuf);
-        DHT_STACK_UNWIND (readv, frame, op_ret, op_errno, vector, count, stbuf,
-                          iobref, xdata);
-
-        return 0;
+    DHT_STACK_UNWIND(access, frame, op_ret, op_errno, xdata);
+    return 0;
 }
 
-int
-dht_readv2 (xlator_t *this, call_frame_t *frame, int op_ret)
+static int
+dht_access2(xlator_t *this, xlator_t *subvol, call_frame_t *frame, int ret)
 {
-        dht_local_t *local  = NULL;
-        xlator_t    *subvol = NULL;
-        int          op_errno = EINVAL;
+    dht_local_t *local = NULL;
+    int op_errno = EINVAL;
 
-        local = frame->local;
-        if (!local)
-                goto out;
+    local = frame->local;
+    if (!local)
+        goto out;
 
-        op_errno = local->op_errno;
-        if (op_ret == -1)
-                goto out;
+    op_errno = local->op_errno;
 
-        local->call_cnt = 2;
-        subvol = local->cached_subvol;
-
-        STACK_WIND (frame, dht_readv_cbk, subvol, subvol->fops->readv,
-                    local->fd, local->rebalance.size, local->rebalance.offset,
-                    local->rebalance.flags, NULL);
+    if (we_are_not_migrating(ret)) {
+        /* This dht xlator is not migrating the file. Unwind and
+         * pass on the original mode bits so the higher DHT layer
+         * can handle this.
+         */
 
+        DHT_STACK_UNWIND(access, frame, -1, op_errno, NULL);
         return 0;
+    }
+
+    if (subvol == NULL)
+        goto out;
+
+    local->call_cnt = 2;
+
+    STACK_WIND_COOKIE(frame, dht_access_cbk, subvol, subvol,
+                      subvol->fops->access, &local->loc, local->rebalance.flags,
+                      local->xattr_req);
+
+    return 0;
 
 out:
-        DHT_STACK_UNWIND (readv, frame, -1, op_errno, NULL, 0, NULL, NULL, NULL);
-        return 0;
+    DHT_STACK_UNWIND(access, frame, -1, op_errno, NULL);
+    return 0;
 }
 
 int
-dht_readv (call_frame_t *frame, xlator_t *this,
-           fd_t *fd, size_t size, off_t off, uint32_t flags, dict_t *xdata)
+dht_access(call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t mask,
+           dict_t *xdata)
 {
-        xlator_t     *subvol = NULL;
-        int           op_errno = -1;
-        dht_local_t  *local = NULL;
-
-        VALIDATE_OR_GOTO (frame, err);
-        VALIDATE_OR_GOTO (this, err);
-        VALIDATE_OR_GOTO (fd, err);
-
-        local = dht_local_init (frame, NULL, fd, GF_FOP_READ);
-        if (!local) {
-                op_errno = ENOMEM;
-                goto err;
-        }
+    xlator_t *subvol = NULL;
+    int op_errno = -1;
+    dht_local_t *local = NULL;
+
+    VALIDATE_OR_GOTO(frame, err);
+    VALIDATE_OR_GOTO(this, err);
+    VALIDATE_OR_GOTO(loc, err);
+    VALIDATE_OR_GOTO(loc->inode, err);
+    VALIDATE_OR_GOTO(loc->path, err);
+
+    local = dht_local_init(frame, loc, NULL, GF_FOP_ACCESS);
+    if (!local) {
+        op_errno = ENOMEM;
+        goto err;
+    }
+
+    local->rebalance.flags = mask;
+    local->call_cnt = 1;
+    subvol = local->cached_subvol;
+    if (!subvol) {
+        gf_msg_debug(this->name, 0, "no cached subvolume for path=%s",
+                     loc->path);
+        op_errno = EINVAL;
+        goto err;
+    }
+    if (xdata)
+        local->xattr_req = dict_ref(xdata);
+
+    STACK_WIND_COOKIE(frame, dht_access_cbk, subvol, subvol,
+                      subvol->fops->access, loc, mask, xdata);
+
+    return 0;
 
-        subvol = local->cached_subvol;
-        if (!subvol) {
-                gf_msg_debug (this->name, 0,
-                              "no cached subvolume for fd=%p", fd);
-                op_errno = EINVAL;
-                goto err;
-        }
+err:
+    op_errno = (op_errno == -1) ? errno : op_errno;
+    DHT_STACK_UNWIND(access, frame, -1, op_errno, NULL);
 
-        local->rebalance.offset = off;
-        local->rebalance.size   = size;
-        local->rebalance.flags  = flags;
-        local->call_cnt = 1;
+    return 0;
+}
+
+int
+dht_flush_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int op_ret,
+              int op_errno, dict_t *xdata)
+{
+    dht_local_t *local = NULL;
+    xlator_t *subvol = 0;
+    int ret = 0;
+
+    local = frame->local;
 
-        STACK_WIND (frame, dht_readv_cbk,
-                    subvol, subvol->fops->readv,
-                    fd, size, off, flags, xdata);
+    local->op_errno = op_errno;
 
+    if (local->call_cnt != 1)
+        goto out;
+
+    if (dht_check_remote_fd_failed_error(local, op_ret, op_errno)) {
+        ret = dht_check_and_open_fd_on_subvol(this, frame);
+        if (ret)
+            goto out;
         return 0;
+    }
 
-err:
-        op_errno = (op_errno == -1) ? errno : op_errno;
-        DHT_STACK_UNWIND (readv, frame, -1, op_errno, NULL, 0, NULL, NULL, NULL);
+    local->rebalance.target_op_fn = dht_flush2;
 
+    local->op_ret = op_ret;
+    local->op_errno = op_errno;
+
+    /* If context is set, then send flush() it to the destination */
+    dht_inode_ctx_get_mig_info(this, local->fd->inode, NULL, &subvol);
+    if (subvol && dht_fd_open_on_dst(this, local->fd, subvol)) {
+        dht_flush2(this, subvol, frame, 0);
         return 0;
-}
+    }
 
-int
-dht_access_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                int op_ret, int op_errno, dict_t *xdata)
-{
-        int          ret = -1;
-        dht_local_t *local = NULL;
-        xlator_t    *subvol = NULL;
-        call_frame_t *prev = NULL;
-
-        local = frame->local;
-        prev = cookie;
-
-        if (!prev || !prev->this)
-                goto out;
-        if (local->call_cnt != 1)
-                goto out;
-        if ((op_ret == -1) && ((op_errno == ENOTCONN) ||
-                dht_inode_missing(op_errno)) &&
-                IA_ISDIR(local->loc.inode->ia_type)) {
-                subvol = dht_subvol_next_available (this, prev->this);
-                if (!subvol)
-                        goto out;
-
-                /* check if we are done with visiting every node */
-                if (subvol == local->cached_subvol) {
-                        goto out;
-                }
-
-                STACK_WIND (frame, dht_access_cbk, subvol, subvol->fops->access,
-                            &local->loc, local->rebalance.flags, NULL);
-                return 0;
-        }
-        if ((op_ret == -1) && dht_inode_missing(op_errno) &&
-                !(IA_ISDIR(local->loc.inode->ia_type))) {
-                /* File would be migrated to other node */
-                local->op_errno = op_errno;
-                local->rebalance.target_op_fn = dht_access2;
-                ret = dht_rebalance_complete_check (frame->this, frame);
-                if (!ret)
-                        return 0;
+    if (op_errno == EREMOTE) {
+        ret = dht_rebalance_complete_check(this, frame);
+        if (!ret) {
+            return 0;
         }
+    }
 
 out:
-        DHT_STACK_UNWIND (access, frame, op_ret, op_errno, xdata);
-        return 0;
+    DHT_STACK_UNWIND(flush, frame, op_ret, op_errno, xdata);
+
+    return 0;
 }
 
-int
-dht_access2 (xlator_t *this, call_frame_t *frame, int op_ret)
+static int
+dht_flush2(xlator_t *this, xlator_t *subvol, call_frame_t *frame, int ret)
 {
-        dht_local_t *local  = NULL;
-        xlator_t    *subvol = NULL;
-        int          op_errno = EINVAL;
+    dht_local_t *local = NULL;
+    int32_t op_errno = EINVAL;
 
-        local = frame->local;
-        if (!local)
-                goto out;
+    if ((frame == NULL) || (frame->local == NULL))
+        goto out;
 
-        op_errno = local->op_errno;
-        if (op_ret == -1)
-                goto out;
+    local = frame->local;
 
-        local->call_cnt = 2;
-        subvol = local->cached_subvol;
+    op_errno = local->op_errno;
 
-        STACK_WIND (frame, dht_access_cbk, subvol, subvol->fops->access,
-                    &local->loc, local->rebalance.flags, NULL);
+    if (subvol == NULL)
+        goto out;
 
-        return 0;
+    local->call_cnt = 2; /* This is the second attempt */
+
+    STACK_WIND(frame, dht_flush_cbk, subvol, subvol->fops->flush, local->fd,
+               local->xattr_req);
+
+    return 0;
 
 out:
-        DHT_STACK_UNWIND (access, frame, -1, op_errno, NULL);
-        return 0;
+    DHT_STACK_UNWIND(flush, frame, -1, op_errno, NULL);
+    return 0;
 }
 
-
 int
-dht_access (call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t mask,
-            dict_t *xdata)
+dht_flush(call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *xdata)
 {
-        xlator_t     *subvol = NULL;
-        int           op_errno = -1;
-        dht_local_t  *local = NULL;
-
-        VALIDATE_OR_GOTO (frame, err);
-        VALIDATE_OR_GOTO (this, err);
-        VALIDATE_OR_GOTO (loc, err);
-        VALIDATE_OR_GOTO (loc->inode, err);
-        VALIDATE_OR_GOTO (loc->path, err);
-
-        local = dht_local_init (frame, loc, NULL, GF_FOP_ACCESS);
-        if (!local) {
-                op_errno = ENOMEM;
-                goto err;
-        }
+    xlator_t *subvol = NULL;
+    int op_errno = -1;
+    dht_local_t *local = NULL;
 
-        local->rebalance.flags = mask;
-        local->call_cnt = 1;
-        subvol = local->cached_subvol;
-        if (!subvol) {
-                gf_msg_debug (this->name, 0,
-                              "no cached subvolume for path=%s", loc->path);
-                op_errno = EINVAL;
-                goto err;
-        }
+    VALIDATE_OR_GOTO(frame, err);
+    VALIDATE_OR_GOTO(this, err);
+    VALIDATE_OR_GOTO(fd, err);
 
-        STACK_WIND (frame, dht_access_cbk, subvol, subvol->fops->access,
-                    loc, mask, xdata);
+    local = dht_local_init(frame, NULL, fd, GF_FOP_FLUSH);
+    if (!local) {
+        op_errno = ENOMEM;
+        goto err;
+    }
 
-        return 0;
+    subvol = local->cached_subvol;
+    if (!subvol) {
+        gf_msg_debug(this->name, 0, "no cached subvolume for fd=%p", fd);
+        op_errno = EINVAL;
+        goto err;
+    }
+
+    if (xdata)
+        local->xattr_req = dict_ref(xdata);
+
+    local->call_cnt = 1;
+
+    STACK_WIND(frame, dht_flush_cbk, subvol, subvol->fops->flush, fd,
+               local->xattr_req);
+    return 0;
 
 err:
-        op_errno = (op_errno == -1) ? errno : op_errno;
-        DHT_STACK_UNWIND (access, frame, -1, op_errno, NULL);
+    op_errno = (op_errno == -1) ? errno : op_errno;
+    DHT_STACK_UNWIND(flush, frame, -1, op_errno, NULL);
 
-        return 0;
+    return 0;
 }
 
-
 int
-dht_flush_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-               int op_ret, int op_errno, dict_t *xdata)
+dht_fsync_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int op_ret,
+              int op_errno, struct iatt *prebuf, struct iatt *postbuf,
+              dict_t *xdata)
 {
-        dht_local_t  *local  = NULL;
-        inode_t      *inode  = NULL;
-        xlator_t     *subvol = 0;
+    dht_local_t *local = NULL;
+    xlator_t *prev = NULL;
+    int ret = -1;
+    inode_t *inode = NULL;
+    xlator_t *src_subvol = 0;
+    xlator_t *dst_subvol = 0;
+
+    local = frame->local;
+    prev = cookie;
+
+    local->op_errno = op_errno;
+
+    if (dht_check_remote_fd_failed_error(local, op_ret, op_errno)) {
+        ret = dht_check_and_open_fd_on_subvol(this, frame);
+        if (ret)
+            goto out;
+        return 0;
+    }
+
+    if (op_ret == -1 && !dht_inode_missing(op_errno)) {
+        gf_msg_debug(this->name, op_errno, "subvolume %s returned -1",
+                     prev->name);
+        goto out;
+    }
+
+    if (local->call_cnt != 1) {
+        if (local->stbuf.ia_blocks) {
+            dht_iatt_merge(this, postbuf, &local->stbuf);
+            dht_iatt_merge(this, prebuf, &local->prebuf);
+        }
+        goto out;
+    }
 
-        local = frame->local;
+    local->op_ret = op_ret;
+    inode = local->fd->inode;
 
-        local->op_errno = op_errno;
+    local->rebalance.target_op_fn = dht_fsync2;
+    dht_set_local_rebalance(this, local, NULL, prebuf, postbuf, xdata);
 
-        if (local->call_cnt != 1)
-                goto out;
+    if ((op_ret == -1) || IS_DHT_MIGRATION_PHASE2(postbuf)) {
+        ret = dht_rebalance_complete_check(this, frame);
+        if (!ret)
+            return 0;
+    }
+
+    /* Check if the rebalance phase1 is true */
+    if (IS_DHT_MIGRATION_PHASE1(postbuf)) {
+        dht_iatt_merge(this, &local->stbuf, postbuf);
+        dht_iatt_merge(this, &local->prebuf, prebuf);
+
+        dht_inode_ctx_get_mig_info(this, inode, &src_subvol, &dst_subvol);
 
-        /* If context is set, then send flush() it to the destination */
-        dht_inode_ctx_get1 (this, inode, &subvol);
-        if (subvol) {
-                dht_flush2 (this, frame, 0);
+        if (dht_mig_info_is_invalid(local->cached_subvol, src_subvol,
+                                    dst_subvol) ||
+            !dht_fd_open_on_dst(this, local->fd, dst_subvol)) {
+            ret = dht_rebalance_in_progress_check(this, frame);
+            if (!ret)
                 return 0;
+        } else {
+            dht_fsync2(this, dst_subvol, frame, 0);
+            return 0;
         }
+    }
 
 out:
-        DHT_STACK_UNWIND (flush, frame, op_ret, op_errno, xdata);
+    DHT_STRIP_PHASE1_FLAGS(postbuf);
+    DHT_STRIP_PHASE1_FLAGS(prebuf);
 
-        return 0;
+    DHT_STACK_UNWIND(fsync, frame, op_ret, op_errno, prebuf, postbuf, xdata);
+
+    return 0;
 }
 
-int
-dht_flush2 (xlator_t *this, call_frame_t *frame, int op_ret)
+static int
+dht_fsync2(xlator_t *this, xlator_t *subvol, call_frame_t *frame, int ret)
 {
-        dht_local_t  *local  = NULL;
-        xlator_t     *subvol = NULL;
-
-        local = frame->local;
+    dht_local_t *local = NULL;
+    int32_t op_errno = EINVAL;
+
+    if ((frame == NULL) || (frame->local == NULL))
+        goto out;
+
+    local = frame->local;
+    op_errno = local->op_errno;
+
+    if (we_are_not_migrating(ret)) {
+        /* This dht xlator is not migrating the file. Unwind and
+         * pass on the original mode bits so the higher DHT layer
+         * can handle this.
+         */
+        DHT_STACK_UNWIND(fsync, frame, local->op_ret, op_errno,
+                         &local->rebalance.prebuf, &local->rebalance.postbuf,
+                         local->rebalance.xdata);
+        return 0;
+    }
 
-        dht_inode_ctx_get1 (this, local->fd->inode, &subvol);
+    if (subvol == NULL)
+        goto out;
 
-        if (!subvol)
-                subvol = local->cached_subvol;
+    local->call_cnt = 2; /* This is the second attempt */
 
-        local->call_cnt = 2; /* This is the second attempt */
+    STACK_WIND_COOKIE(frame, dht_fsync_cbk, subvol, subvol, subvol->fops->fsync,
+                      local->fd, local->rebalance.flags, local->xattr_req);
 
-        STACK_WIND (frame, dht_flush_cbk,
-                    subvol, subvol->fops->flush, local->fd, NULL);
+    return 0;
 
-        return 0;
+out:
+    DHT_STACK_UNWIND(fsync, frame, -1, op_errno, NULL, NULL, NULL);
+    return 0;
 }
 
-
 int
-dht_flush (call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *xdata)
+dht_fsync(call_frame_t *frame, xlator_t *this, fd_t *fd, int datasync,
+          dict_t *xdata)
 {
-        xlator_t     *subvol = NULL;
-        int           op_errno = -1;
-        dht_local_t  *local = NULL;
-
-        VALIDATE_OR_GOTO (frame, err);
-        VALIDATE_OR_GOTO (this, err);
-        VALIDATE_OR_GOTO (fd, err);
-
-        local = dht_local_init (frame, NULL, fd, GF_FOP_FLUSH);
-        if (!local) {
-                op_errno = ENOMEM;
-                goto err;
-        }
+    xlator_t *subvol = NULL;
+    int op_errno = -1;
+    dht_local_t *local = NULL;
 
-        subvol = local->cached_subvol;
-        if (!subvol) {
-                gf_msg_debug (this->name, 0,
-                              "no cached subvolume for fd=%p", fd);
-                op_errno = EINVAL;
-                goto err;
-        }
+    VALIDATE_OR_GOTO(frame, err);
+    VALIDATE_OR_GOTO(this, err);
+    VALIDATE_OR_GOTO(fd, err);
 
-        local->call_cnt = 1;
+    local = dht_local_init(frame, NULL, fd, GF_FOP_FSYNC);
+    if (!local) {
+        op_errno = ENOMEM;
 
-        STACK_WIND (frame, dht_flush_cbk,
-                    subvol, subvol->fops->flush, fd, xdata);
+        goto err;
+    }
+    if (xdata)
+        local->xattr_req = dict_ref(xdata);
 
-        return 0;
+    local->call_cnt = 1;
+    local->rebalance.flags = datasync;
+
+    subvol = local->cached_subvol;
+
+    STACK_WIND_COOKIE(frame, dht_fsync_cbk, subvol, subvol, subvol->fops->fsync,
+                      local->fd, local->rebalance.flags, local->xattr_req);
+    return 0;
 
 err:
-        op_errno = (op_errno == -1) ? errno : op_errno;
-        DHT_STACK_UNWIND (flush, frame, -1, op_errno, NULL);
+    op_errno = (op_errno == -1) ? errno : op_errno;
+    DHT_STACK_UNWIND(fsync, frame, -1, op_errno, NULL, NULL, NULL);
 
-        return 0;
+    return 0;
 }
 
-
-int
-dht_fsync_cbk (call_frame_t *frame, void *cookie, xlator_t *this, int op_ret,
-               int op_errno, struct iatt *prebuf, struct iatt *postbuf,
-               dict_t *xdata)
+/* TODO: for 'lk()' call, we need some other special error, may be ESTALE to
+   indicate that lock migration happened on the fd, so we can consider it as
+   phase 2 of migration */
+static int
+dht_lk_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int op_ret,
+           int op_errno, struct gf_flock *flock, dict_t *xdata)
 {
-        dht_local_t  *local = NULL;
-        call_frame_t *prev = NULL;
-        int           ret = -1;
-        inode_t      *inode = NULL;
-        xlator_t     *subvol = 0;
+    dht_local_t *local = NULL;
+    int ret = -1;
+    xlator_t *subvol = NULL;
 
-        local = frame->local;
-        prev = cookie;
+    local = frame->local;
 
-        local->op_errno = op_errno;
-        if (op_ret == -1 && !dht_inode_missing(op_errno)) {
-                gf_msg_debug (this->name, 0,
-                              "subvolume %s returned -1 (%s)",
-                              prev->this->name, strerror (op_errno));
-                goto out;
-        }
+    if (!local) {
+        op_ret = -1;
+        op_errno = EINVAL;
+        goto out;
+    }
 
-        if (local->call_cnt != 1) {
-                if (local->stbuf.ia_blocks) {
-                        dht_iatt_merge (this, postbuf, &local->stbuf, NULL);
-                        dht_iatt_merge (this, prebuf, &local->prebuf, NULL);
-                }
-                goto out;
-        }
+    if (local->call_cnt != 1)
+        goto out;
 
-        local->op_errno = op_errno;
-        dht_inode_ctx_get1 (this, inode, &subvol);
-        if (!subvol) {
-                local->rebalance.target_op_fn = dht_fsync2;
-
-                /* Check if the rebalance phase1 is true */
-                if (IS_DHT_MIGRATION_PHASE1 (postbuf)) {
-                        dht_iatt_merge (this, &local->stbuf, postbuf, NULL);
-                        dht_iatt_merge (this, &local->prebuf, prebuf, NULL);
-
-                        ret = dht_rebalance_in_progress_check (this, frame);
-                }
-
-                /* Check if the rebalance phase2 is true */
-                if (IS_DHT_MIGRATION_PHASE2 (postbuf)) {
-                        ret = dht_rebalance_complete_check (this, frame);
-                }
-                if (!ret)
-                        return 0;
+    local->rebalance.target_op_fn = dht_lk2;
+
+    local->op_ret = op_ret;
+    local->op_errno = op_errno;
+
+    if (xdata)
+        local->rebalance.xdata = dict_ref(xdata);
+
+    if (op_errno == EREMOTE) {
+        dht_inode_ctx_get_mig_info(this, local->fd->inode, NULL, &subvol);
+        if (subvol && dht_fd_open_on_dst(this, local->fd, subvol)) {
+            dht_lk2(this, subvol, frame, 0);
+            return 0;
         } else {
-                dht_fsync2 (this, frame, 0);
+            ret = dht_rebalance_complete_check(this, frame);
+            if (!ret) {
                 return 0;
+            }
         }
+    }
 
 out:
-        DHT_STRIP_PHASE1_FLAGS (postbuf);
-        DHT_STRIP_PHASE1_FLAGS (prebuf);
-        DHT_STACK_UNWIND (fsync, frame, op_ret, op_errno,
-                          prebuf, postbuf, xdata);
+    dht_lk_inode_unref(frame, op_ret);
+    DHT_STACK_UNWIND(lk, frame, op_ret, op_errno, flock, xdata);
 
-        return 0;
+    return 0;
 }
 
-int
-dht_fsync2 (xlator_t *this, call_frame_t *frame, int op_ret)
+static int
+dht_lk2(xlator_t *this, xlator_t *subvol, call_frame_t *frame, int ret)
 {
-        dht_local_t  *local  = NULL;
-        xlator_t     *subvol = NULL;
+    dht_local_t *local = NULL;
+    int32_t op_errno = EINVAL;
 
-        local = frame->local;
+    if ((frame == NULL) || (frame->local == NULL))
+        goto out;
 
-        dht_inode_ctx_get1 (this, local->fd->inode, &subvol);
-        if (!subvol)
-                subvol = local->cached_subvol;
+    local = frame->local;
 
-        local->call_cnt = 2; /* This is the second attempt */
+    op_errno = local->op_errno;
 
-        STACK_WIND (frame, dht_fsync_cbk, subvol, subvol->fops->fsync,
-                    local->fd, local->rebalance.flags, NULL);
+    if (subvol == NULL)
+        goto out;
 
-        return 0;
+    local->call_cnt = 2; /* This is the second attempt */
+
+    STACK_WIND(frame, dht_lk_cbk, subvol, subvol->fops->lk, local->fd,
+               local->rebalance.lock_cmd, &local->rebalance.flock,
+               local->xattr_req);
+
+    return 0;
+
+out:
+    DHT_STACK_UNWIND(lk, frame, -1, op_errno, NULL, NULL);
+    return 0;
 }
 
 int
-dht_fsync (call_frame_t *frame, xlator_t *this, fd_t *fd, int datasync,
-           dict_t *xdata)
+dht_lk(call_frame_t *frame, xlator_t *this, fd_t *fd, int cmd,
+       struct gf_flock *flock, dict_t *xdata)
 {
-        xlator_t     *subvol = NULL;
-        int           op_errno = -1;
-        dht_local_t  *local = NULL;
+    xlator_t *lock_subvol = NULL;
+    int op_errno = -1;
+    dht_local_t *local = NULL;
+
+    VALIDATE_OR_GOTO(frame, err);
+    VALIDATE_OR_GOTO(this, err);
+    VALIDATE_OR_GOTO(fd, err);
+
+    local = dht_local_init(frame, NULL, fd, GF_FOP_LK);
+    if (!local) {
+        op_errno = ENOMEM;
+        goto err;
+    }
+
+    local->lock_type = flock->l_type;
+    lock_subvol = dht_get_lock_subvolume(this, flock, local);
+    if (!lock_subvol) {
+        gf_msg_debug(this->name, 0, "no lock subvolume for path=%p", fd);
+        op_errno = EINVAL;
+        goto err;
+    }
+
+    /*
+            local->cached_subvol = lock_subvol;
+            ret = dht_check_and_open_fd_on_subvol (this, frame);
+            if (ret)
+                    goto err;
+    */
+    if (xdata)
+        local->xattr_req = dict_ref(xdata);
+
+    local->rebalance.flock = *flock;
+    local->rebalance.lock_cmd = cmd;
+
+    local->call_cnt = 1;
+
+    STACK_WIND(frame, dht_lk_cbk, lock_subvol, lock_subvol->fops->lk, fd, cmd,
+               flock, xdata);
+
+    return 0;
+
+err:
+    op_errno = (op_errno == -1) ? errno : op_errno;
+    DHT_STACK_UNWIND(lk, frame, -1, op_errno, NULL, NULL);
 
-        VALIDATE_OR_GOTO (frame, err);
-        VALIDATE_OR_GOTO (this, err);
-        VALIDATE_OR_GOTO (fd, err);
+    return 0;
+}
 
-        local = dht_local_init (frame, NULL, fd, GF_FOP_FSYNC);
-        if (!local) {
-                op_errno = ENOMEM;
+static int
+dht_lease_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int op_ret,
+              int op_errno, struct gf_lease *lease, dict_t *xdata)
+{
+    DHT_STACK_UNWIND(lease, frame, op_ret, op_errno, lease, xdata);
 
-                goto err;
-        }
+    return 0;
+}
 
-        local->call_cnt = 1;
-        local->rebalance.flags = datasync;
+int
+dht_lease(call_frame_t *frame, xlator_t *this, loc_t *loc,
+          struct gf_lease *lease, dict_t *xdata)
+{
+    xlator_t *subvol = NULL;
+    int op_errno = -1;
 
-        subvol = local->cached_subvol;
+    VALIDATE_OR_GOTO(frame, err);
+    VALIDATE_OR_GOTO(this, err);
+    VALIDATE_OR_GOTO(loc, err);
 
-        STACK_WIND (frame, dht_fsync_cbk, subvol, subvol->fops->fsync,
-                    fd, datasync, xdata);
+    subvol = dht_subvol_get_cached(this, loc->inode);
+    if (!subvol) {
+        gf_msg_debug(this->name, 0, "no cached subvolume for path=%s",
+                     loc->path);
+        op_errno = EINVAL;
+        goto err;
+    }
 
-        return 0;
+    /* TODO: for rebalance, we need to preserve the fop arguments */
+    STACK_WIND(frame, dht_lease_cbk, subvol, subvol->fops->lease, loc, lease,
+               xdata);
+
+    return 0;
 
 err:
-        op_errno = (op_errno == -1) ? errno : op_errno;
-        DHT_STACK_UNWIND (fsync, frame, -1, op_errno, NULL, NULL, NULL);
+    op_errno = (op_errno == -1) ? errno : op_errno;
+    DHT_STACK_UNWIND(lease, frame, -1, op_errno, NULL, NULL);
 
-        return 0;
+    return 0;
 }
 
-
-/* TODO: for 'lk()' call, we need some other special error, may be ESTALE to
-   indicate that lock migration happened on the fd, so we can consider it as
-   phase 2 of migration */
-int
-dht_lk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-            int op_ret, int op_errno, struct gf_flock *flock, dict_t *xdata)
+/* Symlinks are currently not migrated, so no need for any check here */
+static int
+dht_readlink_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int op_ret,
+                 int op_errno, const char *path, struct iatt *stbuf,
+                 dict_t *xdata)
 {
-        DHT_STACK_UNWIND (lk, frame, op_ret, op_errno, flock, xdata);
+    dht_local_t *local = NULL;
 
-        return 0;
-}
+    local = frame->local;
+    if (op_ret == -1)
+        goto err;
+
+    if (!local) {
+        op_ret = -1;
+        op_errno = EINVAL;
+    }
 
+err:
+    DHT_STRIP_PHASE1_FLAGS(stbuf);
+    DHT_STACK_UNWIND(readlink, frame, op_ret, op_errno, path, stbuf, xdata);
+
+    return 0;
+}
 
 int
-dht_lk (call_frame_t *frame, xlator_t *this,
-        fd_t *fd, int cmd, struct gf_flock *flock, dict_t *xdata)
+dht_readlink(call_frame_t *frame, xlator_t *this, loc_t *loc, size_t size,
+             dict_t *xdata)
 {
-        xlator_t     *subvol = NULL;
-        int           op_errno = -1;
+    xlator_t *subvol = NULL;
+    int op_errno = -1;
+    dht_local_t *local = NULL;
+
+    VALIDATE_OR_GOTO(frame, err);
+    VALIDATE_OR_GOTO(this, err);
+    VALIDATE_OR_GOTO(loc, err);
+    VALIDATE_OR_GOTO(loc->inode, err);
+    VALIDATE_OR_GOTO(loc->path, err);
+
+    local = dht_local_init(frame, loc, NULL, GF_FOP_READLINK);
+    if (!local) {
+        op_errno = ENOMEM;
+        goto err;
+    }
+
+    subvol = local->cached_subvol;
+    if (!subvol) {
+        gf_msg_debug(this->name, 0, "no cached subvolume for path=%s",
+                     loc->path);
+        op_errno = EINVAL;
+        goto err;
+    }
+
+    STACK_WIND(frame, dht_readlink_cbk, subvol, subvol->fops->readlink, loc,
+               size, xdata);
+
+    return 0;
 
+err:
+    op_errno = (op_errno == -1) ? errno : op_errno;
+    DHT_STACK_UNWIND(readlink, frame, -1, op_errno, NULL, NULL, NULL);
 
-        VALIDATE_OR_GOTO (frame, err);
-        VALIDATE_OR_GOTO (this, err);
-        VALIDATE_OR_GOTO (fd, err);
-
-        subvol = dht_subvol_get_cached (this, fd->inode);
-        if (!subvol) {
-                gf_msg_debug (this->name, 0,
-                              "no cached subvolume for fd=%p", fd);
-                op_errno = EINVAL;
-                goto err;
-        }
+    return 0;
+}
 
-        /* TODO: for rebalance, we need to preserve the fop arguments */
-        STACK_WIND (frame, dht_lk_cbk, subvol, subvol->fops->lk, fd,
-                    cmd, flock, xdata);
+/* Get both DHT_IATT_IN_XDATA_KEY and DHT_MODE_IN_XDATA_KEY
+ * Use DHT_MODE_IN_XDATA_KEY if available, else fall back to
+ * DHT_IATT_IN_XDATA_KEY
+ * This will return a dummy iatt with only the mode and type set
+ */
+static int
+dht_read_iatt_from_xdata(dict_t *xdata, struct iatt *stbuf)
+{
+    int ret = -1;
+    int32_t mode = 0;
 
-        return 0;
+    ret = dict_get_int32(xdata, DHT_MODE_IN_XDATA_KEY, &mode);
 
-err:
-        op_errno = (op_errno == -1) ? errno : op_errno;
-        DHT_STACK_UNWIND (lk, frame, -1, op_errno, NULL, NULL);
+    if (ret) {
+        ret = dict_get_bin(xdata, DHT_IATT_IN_XDATA_KEY, (void **)&stbuf);
+    } else {
+        stbuf->ia_prot = ia_prot_from_st_mode(mode);
+        stbuf->ia_type = ia_type_from_st_mode(mode);
+    }
 
-        return 0;
+    return ret;
 }
 
-/* Symlinks are currently not migrated, so no need for any check here */
 int
-dht_readlink_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                  int op_ret, int op_errno, const char *path,
-                  struct iatt *stbuf, dict_t *xdata)
+dht_common_xattrop_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                       int32_t op_ret, int32_t op_errno, dict_t *dict,
+                       dict_t *xdata)
 {
-        dht_local_t *local = NULL;
+    dht_local_t *local = NULL;
+    call_frame_t *call_frame = NULL;
+    xlator_t *prev = NULL;
+    xlator_t *src_subvol = NULL;
+    xlator_t *dst_subvol = NULL;
+    struct iatt stbuf = {
+        0,
+    };
+    int ret = -1;
+    inode_t *inode = NULL;
+
+    local = frame->local;
+    call_frame = cookie;
+    prev = call_frame->this;
+
+    local->op_errno = op_errno;
+
+    if ((op_ret == -1) && !dht_inode_missing(op_errno)) {
+        gf_msg_debug(this->name, op_errno, "subvolume %s returned -1.",
+                     prev->name);
+        goto out;
+    }
+
+    if (local->call_cnt != 1)
+        goto out;
+
+    if (dht_check_remote_fd_failed_error(local, op_ret, op_errno)) {
+        ret = dht_check_and_open_fd_on_subvol(this, frame);
+        if (ret)
+            goto out;
+        return 0;
+    }
+
+    ret = dht_read_iatt_from_xdata(xdata, &stbuf);
 
-        local = frame->local;
-        if (op_ret == -1)
-                goto err;
+    if ((!op_ret) && (ret)) {
+        /* This is a potential problem and can cause corruption
+         * with sharding.
+         * Oh well. We tried.
+         */
+        goto out;
+    }
 
-        if (!local) {
-                op_ret = -1;
-                op_errno = EINVAL;
+    local->op_ret = op_ret;
+    local->rebalance.target_op_fn = dht_common_xattrop2;
+    if (xdata)
+        local->rebalance.xdata = dict_ref(xdata);
+
+    if (dict)
+        local->rebalance.dict = dict_ref(dict);
+
+    /* Phase 2 of migration */
+    if ((op_ret == -1) || IS_DHT_MIGRATION_PHASE2(&stbuf)) {
+        ret = dht_rebalance_complete_check(this, frame);
+        if (!ret)
+            return 0;
+    }
+
+    /* Check if the rebalance phase1 is true */
+    if (IS_DHT_MIGRATION_PHASE1(&stbuf)) {
+        inode = local->loc.inode ? local->loc.inode : local->fd->inode;
+        dht_inode_ctx_get_mig_info(this, inode, &src_subvol, &dst_subvol);
+
+        if (dht_mig_info_is_invalid(local->cached_subvol, src_subvol,
+                                    dst_subvol) ||
+            !dht_fd_open_on_dst(this, local->fd, dst_subvol)) {
+            ret = dht_rebalance_in_progress_check(this, frame);
+            if (!ret)
+                return 0;
+        } else {
+            dht_common_xattrop2(this, dst_subvol, frame, 0);
+            return 0;
         }
+    }
 
-err:
-        DHT_STRIP_PHASE1_FLAGS (stbuf);
-        DHT_STACK_UNWIND (readlink, frame, op_ret, op_errno, path, stbuf, xdata);
+out:
+    if (local->fop == GF_FOP_XATTROP) {
+        DHT_STACK_UNWIND(xattrop, frame, op_ret, op_errno, dict, xdata);
+    } else {
+        DHT_STACK_UNWIND(fxattrop, frame, op_ret, op_errno, dict, xdata);
+    }
 
-        return 0;
+    return 0;
 }
 
-
-int
-dht_readlink (call_frame_t *frame, xlator_t *this, loc_t *loc, size_t size,
-              dict_t *xdata)
+static int
+dht_common_xattrop2(xlator_t *this, xlator_t *subvol, call_frame_t *frame,
+                    int ret)
 {
-        xlator_t     *subvol = NULL;
-        int           op_errno = -1;
-        dht_local_t  *local = NULL;
-
-        VALIDATE_OR_GOTO (frame, err);
-        VALIDATE_OR_GOTO (this, err);
-        VALIDATE_OR_GOTO (loc, err);
-        VALIDATE_OR_GOTO (loc->inode, err);
-        VALIDATE_OR_GOTO (loc->path, err);
-
-        local = dht_local_init (frame, loc, NULL, GF_FOP_READLINK);
-        if (!local) {
-                op_errno = ENOMEM;
-                goto err;
+    dht_local_t *local = NULL;
+    int32_t op_errno = EINVAL;
+
+    if ((frame == NULL) || (frame->local == NULL))
+        goto out;
+
+    local = frame->local;
+    op_errno = local->op_errno;
+
+    if (we_are_not_migrating(ret)) {
+        /* This dht xlator is not migrating the file. Unwind and
+         * pass on the original mode bits so the higher DHT layer
+         * can handle this.
+         */
+        if (local->fop == GF_FOP_XATTROP) {
+            DHT_STACK_UNWIND(xattrop, frame, local->op_ret, op_errno,
+                             local->rebalance.dict, local->rebalance.xdata);
+        } else {
+            DHT_STACK_UNWIND(fxattrop, frame, local->op_ret, op_errno,
+                             local->rebalance.dict, local->rebalance.xdata);
         }
 
-        subvol = local->cached_subvol;
-        if (!subvol) {
-                gf_msg_debug (this->name, 0,
-                              "no cached subvolume for path=%s", loc->path);
-                op_errno = EINVAL;
-                goto err;
-        }
+        return 0;
+    }
 
-        STACK_WIND (frame, dht_readlink_cbk,
-                    subvol, subvol->fops->readlink,
-                    loc, size, xdata);
+    if (subvol == NULL)
+        goto out;
 
-        return 0;
+    local->call_cnt = 2; /* This is the second attempt */
 
-err:
-        op_errno = (op_errno == -1) ? errno : op_errno;
-        DHT_STACK_UNWIND (readlink, frame, -1, op_errno, NULL, NULL, NULL);
+    if (local->fop == GF_FOP_XATTROP) {
+        STACK_WIND(frame, dht_common_xattrop_cbk, subvol, subvol->fops->xattrop,
+                   &local->loc, local->rebalance.flags, local->rebalance.xattr,
+                   local->xattr_req);
+    } else {
+        STACK_WIND(frame, dht_common_xattrop_cbk, subvol,
+                   subvol->fops->fxattrop, local->fd, local->rebalance.flags,
+                   local->rebalance.xattr, local->xattr_req);
+    }
 
-        return 0;
-}
+    return 0;
 
-/* Currently no translators on top of 'distribute' will be using
- * below fops, hence not implementing 'migration' related checks
- */
+out:
 
-int
-dht_xattrop_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                 int32_t op_ret, int32_t op_errno, dict_t *dict, dict_t *xdata)
+    /* If local is unavailable we could be unwinding the wrong
+     * function here */
+
+    if (local && (local->fop == GF_FOP_XATTROP)) {
+        DHT_STACK_UNWIND(xattrop, frame, -1, op_errno, NULL, NULL);
+    } else {
+        DHT_STACK_UNWIND(fxattrop, frame, -1, op_errno, NULL, NULL);
+    }
+    return 0;
+}
+
+static int
+dht_xattrop_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                int32_t op_ret, int32_t op_errno, dict_t *dict, dict_t *xdata)
 {
-        DHT_STACK_UNWIND (xattrop, frame, op_ret, op_errno, dict, xdata);
-        return 0;
+    DHT_STACK_UNWIND(xattrop, frame, op_ret, op_errno, dict, xdata);
+    return 0;
 }
 
+/* Set both DHT_IATT_IN_XDATA_KEY and DHT_MODE_IN_XDATA_KEY
+ * Use DHT_MODE_IN_XDATA_KEY if available. Else fall back to
+ * DHT_IATT_IN_XDATA_KEY
+ */
+static int
+dht_request_iatt_in_xdata(dict_t *xattr_req)
+{
+    int ret = -1;
+
+    ret = dict_set_int8(xattr_req, DHT_MODE_IN_XDATA_KEY, 1);
+    ret = dict_set_int8(xattr_req, DHT_IATT_IN_XDATA_KEY, 1);
+
+    /* At least one call succeeded */
+    return ret;
+}
 
 int
-dht_xattrop (call_frame_t *frame, xlator_t *this, loc_t *loc,
-             gf_xattrop_flags_t flags, dict_t *dict, dict_t *xdata)
+dht_xattrop(call_frame_t *frame, xlator_t *this, loc_t *loc,
+            gf_xattrop_flags_t flags, dict_t *dict, dict_t *xdata)
 {
-        xlator_t     *subvol = NULL;
-        int           op_errno = -1;
-        dht_local_t  *local = NULL;
-
-        VALIDATE_OR_GOTO (frame, err);
-        VALIDATE_OR_GOTO (this, err);
-        VALIDATE_OR_GOTO (loc, err);
-        VALIDATE_OR_GOTO (loc->inode, err);
-        VALIDATE_OR_GOTO (loc->path, err);
-
-        local = dht_local_init (frame, loc, NULL, GF_FOP_XATTROP);
-        if (!local) {
-                op_errno = ENOMEM;
-                goto err;
-        }
+    xlator_t *subvol = NULL;
+    int op_errno = -1;
+    dht_local_t *local = NULL;
+    int ret = -1;
+
+    VALIDATE_OR_GOTO(frame, err);
+    VALIDATE_OR_GOTO(this, err);
+    VALIDATE_OR_GOTO(loc, err);
+    VALIDATE_OR_GOTO(loc->inode, err);
+
+    local = dht_local_init(frame, loc, NULL, GF_FOP_XATTROP);
+    if (!local) {
+        op_errno = ENOMEM;
+        goto err;
+    }
+
+    subvol = local->cached_subvol;
+    if (!subvol) {
+        gf_msg_debug(this->name, 0, "no cached subvolume for gfid=%s",
+                     uuid_utoa(loc->inode->gfid));
+        op_errno = EINVAL;
+        goto err;
+    }
+
+    /* Todo : Handle dirs as well. At the moment the only xlator above dht
+     * that uses xattrop is sharding and that is only for files */
+
+    if (IA_ISDIR(loc->inode->ia_type)) {
+        STACK_WIND(frame, dht_xattrop_cbk, subvol, subvol->fops->xattrop, loc,
+                   flags, dict, xdata);
+
+    } else {
+        local->xattr_req = xdata ? dict_ref(xdata) : dict_new();
+        local->call_cnt = 1;
 
-        subvol = local->cached_subvol;
-        if (!subvol) {
-                gf_msg_debug (this->name, 0,
-                              "no cached subvolume for path=%s", loc->path);
-                op_errno = EINVAL;
-                goto err;
-        }
+        local->rebalance.xattr = dict_ref(dict);
+        local->rebalance.flags = flags;
 
-        local->call_cnt = 1;
+        ret = dht_request_iatt_in_xdata(local->xattr_req);
 
-        STACK_WIND (frame,
-                    dht_xattrop_cbk,
-                    subvol, subvol->fops->xattrop,
-                    loc, flags, dict, xdata);
+        if (ret) {
+            gf_msg_debug(this->name, 0,
+                         "Failed to set dictionary key %s file=%s",
+                         DHT_IATT_IN_XDATA_KEY, loc->path);
+        }
 
-        return 0;
+        STACK_WIND(frame, dht_common_xattrop_cbk, subvol, subvol->fops->xattrop,
+                   loc, local->rebalance.flags, local->rebalance.xattr,
+                   local->xattr_req);
+    }
+
+    return 0;
 
 err:
-        op_errno = (op_errno == -1) ? errno : op_errno;
-        DHT_STACK_UNWIND (xattrop, frame, -1, op_errno, NULL, NULL);
+    op_errno = (op_errno == -1) ? errno : op_errno;
+    DHT_STACK_UNWIND(xattrop, frame, -1, op_errno, NULL, NULL);
 
-        return 0;
+    return 0;
 }
 
-
-int
-dht_fxattrop_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                  int32_t op_ret, int32_t op_errno, dict_t *dict, dict_t *xdata)
+static int
+dht_fxattrop_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                 int32_t op_ret, int32_t op_errno, dict_t *dict, dict_t *xdata)
 {
-        DHT_STACK_UNWIND (fxattrop, frame, op_ret, op_errno, dict, xdata);
-        return 0;
+    DHT_STACK_UNWIND(fxattrop, frame, op_ret, op_errno, dict, xdata);
+    return 0;
 }
 
-
 int
-dht_fxattrop (call_frame_t *frame, xlator_t *this,
-              fd_t *fd, gf_xattrop_flags_t flags, dict_t *dict, dict_t *xdata)
+dht_fxattrop(call_frame_t *frame, xlator_t *this, fd_t *fd,
+             gf_xattrop_flags_t flags, dict_t *dict, dict_t *xdata)
 {
-        xlator_t     *subvol = NULL;
-        int           op_errno = -1;
-
-        VALIDATE_OR_GOTO (frame, err);
-        VALIDATE_OR_GOTO (this, err);
-        VALIDATE_OR_GOTO (fd, err);
-
-        subvol = dht_subvol_get_cached (this, fd->inode);
-        if (!subvol) {
-                gf_msg_debug (this->name, 0,
-                              "no cached subvolume for fd=%p", fd);
-                op_errno = EINVAL;
-                goto err;
+    xlator_t *subvol = NULL;
+    int op_errno = -1;
+    dht_local_t *local = NULL;
+    int ret = -1;
+
+    VALIDATE_OR_GOTO(frame, err);
+    VALIDATE_OR_GOTO(this, err);
+    VALIDATE_OR_GOTO(fd, err);
+
+    subvol = dht_subvol_get_cached(this, fd->inode);
+    if (!subvol) {
+        gf_msg_debug(this->name, 0, "no cached subvolume for fd=%p", fd);
+        op_errno = EINVAL;
+        goto err;
+    }
+
+    local = dht_local_init(frame, NULL, fd, GF_FOP_FXATTROP);
+    if (!local) {
+        op_errno = ENOMEM;
+        goto err;
+    }
+
+    /* Todo : Handle dirs as well. At the moment the only xlator above dht
+     * that uses xattrop is sharding and that is only for files */
+
+    if (IA_ISDIR(fd->inode->ia_type)) {
+        STACK_WIND(frame, dht_fxattrop_cbk, subvol, subvol->fops->fxattrop, fd,
+                   flags, dict, xdata);
+
+    } else {
+        local->xattr_req = xdata ? dict_ref(xdata) : dict_new();
+        local->call_cnt = 1;
+
+        local->rebalance.xattr = dict_ref(dict);
+        local->rebalance.flags = flags;
+
+        ret = dht_request_iatt_in_xdata(local->xattr_req);
+
+        if (ret) {
+            gf_msg_debug(this->name, 0, "Failed to set dictionary key %s fd=%p",
+                         DHT_IATT_IN_XDATA_KEY, fd);
         }
 
-        STACK_WIND (frame,
-                    dht_fxattrop_cbk,
-                    subvol, subvol->fops->fxattrop,
-                    fd, flags, dict, xdata);
+        STACK_WIND(frame, dht_common_xattrop_cbk, subvol,
+                   subvol->fops->fxattrop, fd, local->rebalance.flags,
+                   local->rebalance.xattr, local->xattr_req);
+    }
 
-        return 0;
+    return 0;
 
 err:
-        op_errno = (op_errno == -1) ? errno : op_errno;
-        DHT_STACK_UNWIND (fxattrop, frame, -1, op_errno, NULL, NULL);
+    op_errno = (op_errno == -1) ? errno : op_errno;
+    DHT_STACK_UNWIND(fxattrop, frame, -1, op_errno, NULL, NULL);
 
-        return 0;
+    return 0;
 }
 
+/* Currently no translators on top of 'distribute' will be using
+ * below fops, hence not implementing 'migration' related checks
+ */
 
-int
-dht_inodelk_cbk (call_frame_t *frame, void *cookie,
-                 xlator_t *this, int32_t op_ret, int32_t op_errno, dict_t *xdata)
+static int
+dht_inodelk_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                int32_t op_ret, int32_t op_errno, dict_t *xdata)
 
 {
-        DHT_STACK_UNWIND (inodelk, frame, op_ret, op_errno, xdata);
-        return 0;
+    dht_lk_inode_unref(frame, op_ret);
+    DHT_STACK_UNWIND(inodelk, frame, op_ret, op_errno, xdata);
+    return 0;
 }
 
-
 int32_t
-dht_inodelk (call_frame_t *frame, xlator_t *this, const char *volume,
-             loc_t *loc, int32_t cmd, struct gf_flock *lock, dict_t *xdata)
+dht_inodelk(call_frame_t *frame, xlator_t *this, const char *volume, loc_t *loc,
+            int32_t cmd, struct gf_flock *lock, dict_t *xdata)
 {
-        xlator_t     *subvol = NULL;
-        int           op_errno = -1;
-        dht_local_t  *local = NULL;
-
+    xlator_t *lock_subvol = NULL;
+    int op_errno = -1;
+    dht_local_t *local = NULL;
 
-        VALIDATE_OR_GOTO (frame, err);
-        VALIDATE_OR_GOTO (this, err);
-        VALIDATE_OR_GOTO (loc, err);
-        VALIDATE_OR_GOTO (loc->inode, err);
+    VALIDATE_OR_GOTO(frame, err);
+    VALIDATE_OR_GOTO(this, err);
+    VALIDATE_OR_GOTO(loc, err);
+    VALIDATE_OR_GOTO(loc->inode, err);
 
-        local = dht_local_init (frame, loc, NULL, GF_FOP_INODELK);
-        if (!local) {
-                op_errno = ENOMEM;
-                goto err;
-        }
+    local = dht_local_init(frame, loc, NULL, GF_FOP_INODELK);
+    if (!local) {
+        op_errno = ENOMEM;
+        goto err;
+    }
 
-        subvol = local->cached_subvol;
-        if (!subvol) {
-                gf_msg_debug (this->name, 0,
-                              "no cached subvolume for path=%s", loc->path);
-                op_errno = EINVAL;
-                goto err;
-        }
+    local->lock_type = lock->l_type;
+    lock_subvol = dht_get_lock_subvolume(this, lock, local);
+    if (!lock_subvol) {
+        gf_msg_debug(this->name, 0, "no lock subvolume for path=%s", loc->path);
+        op_errno = EINVAL;
+        goto err;
+    }
 
-        local->call_cnt = 1;
+    local->call_cnt = 1;
 
-        STACK_WIND (frame,
-                    dht_inodelk_cbk,
-                    subvol, subvol->fops->inodelk,
-                    volume, loc, cmd, lock, xdata);
+    STACK_WIND(frame, dht_inodelk_cbk, lock_subvol, lock_subvol->fops->inodelk,
+               volume, loc, cmd, lock, xdata);
 
-        return 0;
+    return 0;
 
 err:
-        op_errno = (op_errno == -1) ? errno : op_errno;
-        DHT_STACK_UNWIND (inodelk, frame, -1, op_errno, NULL);
+    op_errno = (op_errno == -1) ? errno : op_errno;
+    DHT_STACK_UNWIND(inodelk, frame, -1, op_errno, NULL);
 
-        return 0;
+    return 0;
 }
 
-
 int
-dht_finodelk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                  int32_t op_ret, int32_t op_errno, dict_t *xdata)
+dht_finodelk_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                 int32_t op_ret, int32_t op_errno, dict_t *xdata)
 
 {
-        DHT_STACK_UNWIND (finodelk, frame, op_ret, op_errno, xdata);
-        return 0;
-}
+    dht_local_t *local = NULL;
+    int ret = 0;
 
+    GF_VALIDATE_OR_GOTO("dht", frame, out);
+    GF_VALIDATE_OR_GOTO("dht", this, out);
+    GF_VALIDATE_OR_GOTO("dht", frame->local, out);
 
-int
-dht_finodelk (call_frame_t *frame, xlator_t *this, const char *volume,
-              fd_t *fd, int32_t cmd, struct gf_flock *lock, dict_t *xdata)
-{
-        xlator_t     *subvol = NULL;
-        int           op_errno = -1;
-
-        VALIDATE_OR_GOTO (frame, err);
-        VALIDATE_OR_GOTO (this, err);
-        VALIDATE_OR_GOTO (fd, err);
-
-        subvol = dht_subvol_get_cached (this, fd->inode);
-        if (!subvol) {
-                gf_msg_debug (this->name, 0,
-                              "no cached subvolume for fd=%p", fd);
-                op_errno = EINVAL;
-                goto err;
-        }
+    local = frame->local;
 
+    if (dht_check_remote_fd_failed_error(local, op_ret, op_errno)) {
+        ret = dht_check_and_open_fd_on_subvol(this, frame);
+        if (ret)
+            goto out;
+        return 0;
+    }
 
-        STACK_WIND (frame, dht_finodelk_cbk, subvol, subvol->fops->finodelk,
-                    volume, fd, cmd, lock, xdata);
+out:
+    dht_lk_inode_unref(frame, op_ret);
+    DHT_STACK_UNWIND(finodelk, frame, op_ret, op_errno, xdata);
 
-        return 0;
+    return 0;
+}
+
+int
+dht_finodelk(call_frame_t *frame, xlator_t *this, const char *volume, fd_t *fd,
+             int32_t cmd, struct gf_flock *lock, dict_t *xdata)
+{
+    xlator_t *lock_subvol = NULL;
+    dht_local_t *local = NULL;
+    int op_errno = -1;
+
+    VALIDATE_OR_GOTO(frame, err);
+    VALIDATE_OR_GOTO(this, err);
+    VALIDATE_OR_GOTO(fd, err);
+
+    local = dht_local_init(frame, NULL, fd, GF_FOP_INODELK);
+    if (!local) {
+        op_errno = ENOMEM;
+        goto err;
+    }
+
+    local->call_cnt = 1;
+    local->lock_type = lock->l_type;
+
+    lock_subvol = dht_get_lock_subvolume(this, lock, local);
+    if (!lock_subvol) {
+        gf_msg_debug(this->name, 0, "no lock subvolume for fd=%p", fd);
+        op_errno = EINVAL;
+        goto err;
+    }
+
+    /*
+            local->cached_subvol = lock_subvol;
+            ret = dht_check_and_open_fd_on_subvol (this, frame);
+            if (ret)
+                    goto err;
+    */
+    local->rebalance.flock = *lock;
+    local->rebalance.lock_cmd = cmd;
+    local->key = gf_strdup(volume);
+
+    if (xdata)
+        local->xattr_req = dict_ref(xdata);
+
+    STACK_WIND(frame, dht_finodelk_cbk, lock_subvol,
+               lock_subvol->fops->finodelk, volume, fd, cmd, lock, xdata);
+
+    return 0;
 
 err:
-        op_errno = (op_errno == -1) ? errno : op_errno;
-        DHT_STACK_UNWIND (finodelk, frame, -1, op_errno, NULL);
+    op_errno = (op_errno == -1) ? errno : op_errno;
+    DHT_STACK_UNWIND(finodelk, frame, -1, op_errno, NULL);
 
-        return 0;
+    return 0;
 }
diff --git a/xlators/cluster/dht/src/dht-inode-write.c b/xlators/cluster/dht/src/dht-inode-write.c
index 2bfd80af3cd..2f23ce90fbd 100644
--- a/xlators/cluster/dht/src/dht-inode-write.c
+++ b/xlators/cluster/dht/src/dht-inode-write.c
@@ -8,1010 +8,1397 @@
   cases as published by the Free Software Foundation.
 */
 
-
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
-
 #include "dht-common.h"
 
-int dht_writev2 (xlator_t *this, call_frame_t *frame, int ret);
-int dht_truncate2 (xlator_t *this, call_frame_t *frame, int ret);
-int dht_setattr2 (xlator_t *this, call_frame_t *frame, int ret);
-int dht_fallocate2(xlator_t *this, call_frame_t *frame, int op_ret);
-int dht_discard2(xlator_t *this, call_frame_t *frame, int op_ret);
-int dht_zerofill2(xlator_t *this, call_frame_t *frame, int op_ret);
+static int
+dht_writev2(xlator_t *this, xlator_t *subvol, call_frame_t *frame, int ret);
+static int
+dht_truncate2(xlator_t *this, xlator_t *subvol, call_frame_t *frame, int ret);
+static int
+dht_setattr2(xlator_t *this, xlator_t *subvol, call_frame_t *frame, int ret);
+static int
+dht_fallocate2(xlator_t *this, xlator_t *subvol, call_frame_t *frame, int ret);
+static int
+dht_discard2(xlator_t *this, xlator_t *subvol, call_frame_t *frame, int ret);
+static int
+dht_zerofill2(xlator_t *this, xlator_t *subvol, call_frame_t *frame, int ret);
 
 int
-dht_writev_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                int op_ret, int op_errno, struct iatt *prebuf,
-                struct iatt *postbuf, dict_t *xdata)
+dht_writev_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int op_ret,
+               int op_errno, struct iatt *prebuf, struct iatt *postbuf,
+               dict_t *xdata)
 {
-        dht_local_t *local = NULL;
-        int          ret   = -1;
-        xlator_t    *subvol = NULL;
+    dht_local_t *local = NULL;
+    xlator_t *prev = NULL;
+    int ret = -1;
+    xlator_t *subvol1 = NULL;
+    xlator_t *subvol2 = NULL;
+
+    local = frame->local;
+    prev = cookie;
+
+    if (!local) {
+        op_ret = -1;
+        op_errno = EINVAL;
+        goto out;
+    }
+
+    /* writev fails with EBADF if dht has not yet opened the fd
+     * on the cached subvol. This could happen if the file was migrated
+     * and a lookup updated the cached subvol in the inode ctx.
+     * We only check once as this could be a valid bad fd error.
+     */
+
+    if (dht_check_remote_fd_failed_error(local, op_ret, op_errno)) {
+        ret = dht_check_and_open_fd_on_subvol(this, frame);
+        if (ret)
+            goto out;
+        return 0;
+    }
 
-        if (op_ret == -1 && !dht_inode_missing(op_errno)) {
-                goto out;
+    if (op_ret == -1 && !dht_inode_missing(op_errno)) {
+        local->op_errno = op_errno;
+        local->op_ret = -1;
+        gf_msg_debug(this->name, 0, "subvolume %s returned -1 (%s)", prev->name,
+                     strerror(op_errno));
+        goto out;
+    }
+
+    if (local->call_cnt != 1) {
+        /* preserve the modes of source */
+        if (local->stbuf.ia_blocks) {
+            dht_iatt_merge(this, postbuf, &local->stbuf);
+            dht_iatt_merge(this, prebuf, &local->prebuf);
         }
+        goto out;
+    }
 
-        local = frame->local;
-        if (!local) {
-                op_ret = -1;
-                op_errno = EINVAL;
-                goto out;
-        }
+    local->rebalance.target_op_fn = dht_writev2;
 
-        if (local->call_cnt != 1) {
-                /* preserve the modes of source */
-                if (local->stbuf.ia_blocks) {
-                        dht_iatt_merge (this, postbuf, &local->stbuf, NULL);
-                        dht_iatt_merge (this, prebuf, &local->prebuf, NULL);
-                }
+    local->op_ret = op_ret;
+    local->op_errno = op_errno;
+
+    /* We might need to pass the stbuf information to the higher DHT
+     * layer for appropriate handling.
+     */
+
+    dht_set_local_rebalance(this, local, NULL, prebuf, postbuf, xdata);
+
+    /* Phase 2 of migration */
+    if ((op_ret == -1) || IS_DHT_MIGRATION_PHASE2(postbuf)) {
+        ret = dht_rebalance_complete_check(this, frame);
+        if (!ret)
+            return 0;
+    }
+
+    /* Check if the rebalance phase1 is true */
+    if (IS_DHT_MIGRATION_PHASE1(postbuf)) {
+        if (!local->xattr_req) {
+            local->xattr_req = dict_new();
+            if (!local->xattr_req) {
+                gf_msg(this->name, GF_LOG_ERROR, DHT_MSG_NO_MEMORY, ENOMEM,
+                       "insufficient memory");
+                local->op_errno = ENOMEM;
+                local->op_ret = -1;
                 goto out;
+            }
         }
 
-        local->rebalance.target_op_fn = dht_writev2;
-
-        local->op_errno = op_errno;
-        /* Phase 2 of migration */
-        if (IS_DHT_MIGRATION_PHASE2 (postbuf)) {
-                ret = dht_rebalance_complete_check (this, frame);
-                if (!ret)
-                        return 0;
+        ret = dict_set_uint32(local->xattr_req, GF_PROTECT_FROM_EXTERNAL_WRITES,
+                              1);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, DHT_MSG_DICT_SET_FAILED, 0,
+                   "Failed to set key %s in dictionary",
+                   GF_PROTECT_FROM_EXTERNAL_WRITES);
+            local->op_errno = ENOMEM;
+            local->op_ret = -1;
+            goto out;
         }
 
-        /* Check if the rebalance phase1 is true */
-        if (IS_DHT_MIGRATION_PHASE1 (postbuf)) {
-                dht_iatt_merge (this, &local->stbuf, postbuf, NULL);
-                dht_iatt_merge (this, &local->prebuf, prebuf, NULL);
+        dht_iatt_merge(this, &local->stbuf, postbuf);
+        dht_iatt_merge(this, &local->prebuf, prebuf);
 
-                ret = dht_inode_ctx_get1 (this, local->fd->inode, &subvol);
-                if (subvol) {
-                        dht_writev2 (this, frame, 0);
-                        return 0;
-                }
-                ret = dht_rebalance_in_progress_check (this, frame);
-                if (!ret)
-                        return 0;
+        ret = dht_inode_ctx_get_mig_info(this, local->fd->inode, &subvol1,
+                                         &subvol2);
+        if (!dht_mig_info_is_invalid(local->cached_subvol, subvol1, subvol2)) {
+            if (dht_fd_open_on_dst(this, local->fd, subvol2)) {
+                dht_writev2(this, subvol2, frame, 0);
+                return 0;
+            }
         }
+        ret = dht_rebalance_in_progress_check(this, frame);
+        if (!ret)
+            return 0;
+    }
 
 out:
-        DHT_STRIP_PHASE1_FLAGS (postbuf);
-        DHT_STRIP_PHASE1_FLAGS (prebuf);
+    DHT_STRIP_PHASE1_FLAGS(postbuf);
+    DHT_STRIP_PHASE1_FLAGS(prebuf);
 
-        DHT_STACK_UNWIND (writev, frame, op_ret, op_errno, prebuf, postbuf,
-                          xdata);
+    DHT_STACK_UNWIND(writev, frame, op_ret, op_errno, prebuf, postbuf, xdata);
 
-        return 0;
+    return 0;
 }
 
-int
-dht_writev2 (xlator_t *this, call_frame_t *frame, int op_ret)
+static int
+dht_writev2(xlator_t *this, xlator_t *subvol, call_frame_t *frame, int ret)
 {
-        dht_local_t  *local  = NULL;
-        xlator_t     *subvol = NULL;
+    dht_local_t *local = NULL;
+    int32_t op_errno = EINVAL;
+
+    if ((frame == NULL) || (frame->local == NULL))
+        goto out;
+
+    local = frame->local;
+    op_errno = local->op_errno;
+
+    if (we_are_not_migrating(ret)) {
+        /* This dht xlator is not migrating the file. Unwind and
+         * pass on the original mode bits so the higher DHT layer
+         * can handle this.
+         */
+        DHT_STACK_UNWIND(writev, frame, local->op_ret, local->op_errno,
+                         &local->rebalance.prebuf, &local->rebalance.postbuf,
+                         local->rebalance.xdata);
+        return 0;
+    }
 
-        local = frame->local;
+    if (subvol == NULL)
+        goto out;
 
-        dht_inode_ctx_get1 (this, local->fd->inode, &subvol);
+    local->call_cnt = 2; /* This is the second attempt */
 
-        if (!subvol)
-                subvol = local->cached_subvol;
+    STACK_WIND_COOKIE(frame, dht_writev_cbk, subvol, subvol,
+                      subvol->fops->writev, local->fd, local->rebalance.vector,
+                      local->rebalance.count, local->rebalance.offset,
+                      local->rebalance.flags, local->rebalance.iobref,
+                      local->xattr_req);
 
-        local->call_cnt = 2; /* This is the second attempt */
+    return 0;
 
-        STACK_WIND (frame, dht_writev_cbk,
-                    subvol, subvol->fops->writev,
-                    local->fd, local->rebalance.vector, local->rebalance.count,
-                    local->rebalance.offset, local->rebalance.flags,
-                    local->rebalance.iobref, NULL);
+out:
+    DHT_STACK_UNWIND(writev, frame, -1, op_errno, NULL, NULL, NULL);
 
-        return 0;
+    return 0;
 }
 
 int
-dht_writev (call_frame_t *frame, xlator_t *this, fd_t *fd,
-            struct iovec *vector, int count, off_t off, uint32_t flags,
-            struct iobref *iobref, dict_t *xdata)
+dht_writev(call_frame_t *frame, xlator_t *this, fd_t *fd, struct iovec *vector,
+           int count, off_t off, uint32_t flags, struct iobref *iobref,
+           dict_t *xdata)
 {
-        xlator_t     *subvol = NULL;
-        int           op_errno = -1;
-        dht_local_t  *local = NULL;
-
-        VALIDATE_OR_GOTO (frame, err);
-        VALIDATE_OR_GOTO (this, err);
-        VALIDATE_OR_GOTO (fd, err);
-
-        local = dht_local_init (frame, NULL, fd, GF_FOP_WRITE);
-        if (!local) {
-
-                op_errno = ENOMEM;
-                goto err;
-        }
-
-        subvol = local->cached_subvol;
-        if (!subvol) {
-                gf_msg_debug (this->name, 0,
-                              "no cached subvolume for fd=%p", fd);
-                op_errno = EINVAL;
-                goto err;
-        }
-
+    xlator_t *subvol = NULL;
+    int op_errno = -1;
+    dht_local_t *local = NULL;
+
+    VALIDATE_OR_GOTO(frame, err);
+    VALIDATE_OR_GOTO(this, err);
+    VALIDATE_OR_GOTO(fd, err);
+
+    local = dht_local_init(frame, NULL, fd, GF_FOP_WRITE);
+    if (!local) {
+        op_errno = ENOMEM;
+        goto err;
+    }
+
+    subvol = local->cached_subvol;
+    if (!subvol) {
+        gf_msg_debug(this->name, 0, "no cached subvolume for fd=%p", fd);
+        op_errno = EINVAL;
+        goto err;
+    }
+
+    if (xdata)
+        local->xattr_req = dict_ref(xdata);
+
+    local->rebalance.vector = iov_dup(vector, count);
+    local->rebalance.offset = off;
+    local->rebalance.count = count;
+    local->rebalance.flags = flags;
+    local->rebalance.iobref = iobref_ref(iobref);
+    local->call_cnt = 1;
+
+    STACK_WIND_COOKIE(frame, dht_writev_cbk, subvol, subvol,
+                      subvol->fops->writev, fd, local->rebalance.vector,
+                      local->rebalance.count, local->rebalance.offset,
+                      local->rebalance.flags, local->rebalance.iobref,
+                      local->xattr_req);
+
+    return 0;
 
-        local->rebalance.vector = iov_dup (vector, count);
-        local->rebalance.offset = off;
-        local->rebalance.count = count;
-        local->rebalance.flags = flags;
-        local->rebalance.iobref = iobref_ref (iobref);
-        local->call_cnt = 1;
+err:
+    op_errno = (op_errno == -1) ? errno : op_errno;
+    DHT_STACK_UNWIND(writev, frame, -1, op_errno, NULL, NULL, NULL);
 
-        STACK_WIND (frame, dht_writev_cbk,
-                    subvol, subvol->fops->writev,
-                    fd, vector, count, off, flags, iobref, xdata);
+    return 0;
+}
 
+int
+dht_truncate_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int op_ret,
+                 int op_errno, struct iatt *prebuf, struct iatt *postbuf,
+                 dict_t *xdata)
+{
+    dht_local_t *local = NULL;
+    xlator_t *prev = NULL;
+    int ret = -1;
+    xlator_t *src_subvol = NULL;
+    xlator_t *dst_subvol = NULL;
+    inode_t *inode = NULL;
+
+    GF_VALIDATE_OR_GOTO("dht", frame, err);
+    GF_VALIDATE_OR_GOTO("dht", this, out);
+    GF_VALIDATE_OR_GOTO("dht", frame->local, out);
+    GF_VALIDATE_OR_GOTO("dht", cookie, out);
+
+    local = frame->local;
+    prev = cookie;
+
+    /* Needs to be checked only for ftruncate.
+     * ftruncate fails with EBADF/EINVAL if dht has not yet opened the fd
+     * on the cached subvol. This could happen if the file was migrated
+     * and a lookup updated the cached subvol in the inode ctx.
+     * We only check once as this could actually be a valid error.
+     */
+
+    if ((local->fop == GF_FOP_FTRUNCATE) &&
+        dht_check_remote_fd_failed_error(local, op_ret, op_errno)) {
+        ret = dht_check_and_open_fd_on_subvol(this, frame);
+        if (ret)
+            goto out;
         return 0;
+    }
 
-err:
-        op_errno = (op_errno == -1) ? errno : op_errno;
-        DHT_STACK_UNWIND (writev, frame, -1, op_errno, NULL, NULL, NULL);
+    if ((op_ret == -1) && !dht_inode_missing(op_errno)) {
+        local->op_errno = op_errno;
+        local->op_ret = -1;
+        gf_msg_debug(this->name, op_errno, "subvolume %s returned -1",
+                     prev->name);
 
-        return 0;
-}
+        goto out;
+    }
 
+    if (local->call_cnt != 1) {
+        if (local->stbuf.ia_blocks) {
+            dht_iatt_merge(this, postbuf, &local->stbuf);
+            dht_iatt_merge(this, prebuf, &local->prebuf);
+        }
+        goto out;
+    }
 
+    local->rebalance.target_op_fn = dht_truncate2;
 
-int
-dht_truncate_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                  int op_ret, int op_errno, struct iatt *prebuf,
-                  struct iatt *postbuf, dict_t *xdata)
-{
-        dht_local_t  *local = NULL;
-        call_frame_t *prev = NULL;
-        int           ret = -1;
-        xlator_t    *subvol = NULL;
-        inode_t      *inode = NULL;
-
-        GF_VALIDATE_OR_GOTO ("dht", frame, err);
-        GF_VALIDATE_OR_GOTO ("dht", this, out);
-        GF_VALIDATE_OR_GOTO ("dht", frame->local, out);
-        GF_VALIDATE_OR_GOTO ("dht", cookie, out);
-
-        local = frame->local;
-        prev = cookie;
-
-        if ((op_ret == -1) && !dht_inode_missing(op_errno)) {
-                local->op_errno = op_errno;
-                local->op_ret = -1;
-                gf_msg_debug (this->name, 0,
-                              "subvolume %s returned -1 (%s)",
-                              prev->this->name, strerror (op_errno));
+    local->op_ret = op_ret;
+    local->op_errno = op_errno;
 
-                goto out;
-        }
+    /* We might need to pass the stbuf information to the higher DHT
+     * layer for appropriate handling.
+     */
 
-        if (local->call_cnt != 1) {
-                if (local->stbuf.ia_blocks) {
-                        dht_iatt_merge (this, postbuf, &local->stbuf, NULL);
-                        dht_iatt_merge (this, prebuf, &local->prebuf, NULL);
-                }
-                goto out;
-        }
+    dht_set_local_rebalance(this, local, NULL, prebuf, postbuf, xdata);
 
-        local->rebalance.target_op_fn = dht_truncate2;
+    /* Phase 2 of migration */
+    if ((op_ret == -1) || IS_DHT_MIGRATION_PHASE2(postbuf)) {
+        ret = dht_rebalance_complete_check(this, frame);
+        if (!ret)
+            return 0;
+    }
 
-        local->op_errno = op_errno;
-        /* Phase 2 of migration */
-        if ((op_ret == -1) || IS_DHT_MIGRATION_PHASE2 (postbuf)) {
-                ret = dht_rebalance_complete_check (this, frame);
-                if (!ret)
-                        return 0;
-        }
+    /* Check if the rebalance phase1 is true */
+    if (IS_DHT_MIGRATION_PHASE1(postbuf)) {
+        dht_iatt_merge(this, &local->stbuf, postbuf);
+        dht_iatt_merge(this, &local->prebuf, prebuf);
 
-        /* Check if the rebalance phase1 is true */
-        if (IS_DHT_MIGRATION_PHASE1 (postbuf)) {
-                dht_iatt_merge (this, &local->stbuf, postbuf, NULL);
-                dht_iatt_merge (this, &local->prebuf, prebuf, NULL);
-                inode = (local->fd) ? local->fd->inode : local->loc.inode;
-                dht_inode_ctx_get1 (this, inode, &subvol);
-                if (subvol) {
-                        dht_truncate2 (this, frame, 0);
-                        return 0;
-                }
-                ret = dht_rebalance_in_progress_check (this, frame);
-                if (!ret)
-                        return 0;
+        inode = (local->fd) ? local->fd->inode : local->loc.inode;
+
+        dht_inode_ctx_get_mig_info(this, inode, &src_subvol, &dst_subvol);
+        if (!dht_mig_info_is_invalid(local->cached_subvol, src_subvol,
+                                     dst_subvol)) {
+            if ((!local->fd) ||
+                ((local->fd) &&
+                 dht_fd_open_on_dst(this, local->fd, dst_subvol))) {
+                dht_truncate2(this, dst_subvol, frame, 0);
+                return 0;
+            }
         }
+        ret = dht_rebalance_in_progress_check(this, frame);
+        if (!ret)
+            return 0;
+    }
 
 out:
-        DHT_STRIP_PHASE1_FLAGS (postbuf);
-        DHT_STRIP_PHASE1_FLAGS (prebuf);
-        DHT_STACK_UNWIND (truncate, frame, op_ret, op_errno,
-                          prebuf, postbuf, xdata);
+    DHT_STRIP_PHASE1_FLAGS(postbuf);
+    DHT_STRIP_PHASE1_FLAGS(prebuf);
+
+    DHT_STACK_UNWIND(truncate, frame, op_ret, op_errno, prebuf, postbuf, xdata);
 err:
-        return 0;
+    return 0;
 }
 
-
-int
-dht_truncate2 (xlator_t *this, call_frame_t *frame, int op_ret)
+static int
+dht_truncate2(xlator_t *this, xlator_t *subvol, call_frame_t *frame, int ret)
 {
-        dht_local_t  *local  = NULL;
-        xlator_t     *subvol = NULL;
-        inode_t      *inode = NULL;
+    dht_local_t *local = NULL;
+    int32_t op_errno = EINVAL;
 
-        local = frame->local;
+    if (!frame || !frame->local)
+        goto out;
 
-        inode = local->fd ? local->fd->inode : local->loc.inode;
+    local = frame->local;
+    op_errno = local->op_errno;
 
-        dht_inode_ctx_get1 (this, inode, &subvol);
-        if (!subvol)
-                subvol = local->cached_subvol;
+    /* This dht xlator is not migrating the file  */
+    if (we_are_not_migrating(ret)) {
+        DHT_STACK_UNWIND(truncate, frame, local->op_ret, local->op_errno,
+                         &local->rebalance.prebuf, &local->rebalance.postbuf,
+                         local->rebalance.xdata);
+        return 0;
+    }
 
-        local->call_cnt = 2; /* This is the second attempt */
+    if (subvol == NULL)
+        goto out;
 
-        if (local->fop == GF_FOP_TRUNCATE) {
-                STACK_WIND (frame, dht_truncate_cbk, subvol,
-                            subvol->fops->truncate, &local->loc,
-                            local->rebalance.offset, NULL);
-        } else {
-                STACK_WIND (frame, dht_truncate_cbk, subvol,
-                            subvol->fops->ftruncate, local->fd,
-                            local->rebalance.offset, NULL);
-        }
+    local->call_cnt = 2; /* This is the second attempt */
 
-        return 0;
+    if (local->fop == GF_FOP_TRUNCATE) {
+        STACK_WIND_COOKIE(frame, dht_truncate_cbk, subvol, subvol,
+                          subvol->fops->truncate, &local->loc,
+                          local->rebalance.offset, local->xattr_req);
+    } else {
+        STACK_WIND_COOKIE(frame, dht_truncate_cbk, subvol, subvol,
+                          subvol->fops->ftruncate, local->fd,
+                          local->rebalance.offset, local->xattr_req);
+    }
+
+    return 0;
+
+out:
+    DHT_STACK_UNWIND(truncate, frame, -1, op_errno, NULL, NULL, NULL);
+    return 0;
 }
 
 int
-dht_truncate (call_frame_t *frame, xlator_t *this, loc_t *loc, off_t offset,
-              dict_t *xdata)
+dht_truncate(call_frame_t *frame, xlator_t *this, loc_t *loc, off_t offset,
+             dict_t *xdata)
 {
-        xlator_t     *subvol = NULL;
-        int           op_errno = -1;
-        dht_local_t  *local = NULL;
-
-        VALIDATE_OR_GOTO (frame, err);
-        VALIDATE_OR_GOTO (this, err);
-        VALIDATE_OR_GOTO (loc, err);
-        VALIDATE_OR_GOTO (loc->inode, err);
-        VALIDATE_OR_GOTO (loc->path, err);
-
-        local = dht_local_init (frame, loc, NULL, GF_FOP_TRUNCATE);
-        if (!local) {
-                op_errno = ENOMEM;
-                goto err;
-        }
-
-        local->rebalance.offset = offset;
-        local->call_cnt = 1;
-        subvol = local->cached_subvol;
-        if (!subvol) {
-                gf_msg_debug (this->name, 0,
-                              "no cached subvolume for path=%s", loc->path);
-                op_errno = EINVAL;
-                goto err;
-        }
-
-        STACK_WIND (frame, dht_truncate_cbk,
-                    subvol, subvol->fops->truncate,
-                    loc, offset, xdata);
-
-        return 0;
+    xlator_t *subvol = NULL;
+    int op_errno = -1;
+    dht_local_t *local = NULL;
+
+    VALIDATE_OR_GOTO(frame, err);
+    VALIDATE_OR_GOTO(this, err);
+    VALIDATE_OR_GOTO(loc, err);
+    VALIDATE_OR_GOTO(loc->inode, err);
+
+    local = dht_local_init(frame, loc, NULL, GF_FOP_TRUNCATE);
+    if (!local) {
+        op_errno = ENOMEM;
+        goto err;
+    }
+
+    local->rebalance.offset = offset;
+    local->call_cnt = 1;
+    subvol = local->cached_subvol;
+    if (!subvol) {
+        gf_msg_debug(this->name, 0, "no cached subvolume for gfid=%s",
+                     uuid_utoa(loc->inode->gfid));
+        op_errno = EINVAL;
+        goto err;
+    }
+
+    if (xdata)
+        local->xattr_req = dict_ref(xdata);
+
+    STACK_WIND_COOKIE(frame, dht_truncate_cbk, subvol, subvol,
+                      subvol->fops->truncate, loc, offset, xdata);
+
+    return 0;
 
 err:
-        op_errno = (op_errno == -1) ? errno : op_errno;
-        DHT_STACK_UNWIND (truncate, frame, -1, op_errno, NULL, NULL, NULL);
+    op_errno = (op_errno == -1) ? errno : op_errno;
+    DHT_STACK_UNWIND(truncate, frame, -1, op_errno, NULL, NULL, NULL);
 
-        return 0;
+    return 0;
 }
 
 int
-dht_ftruncate (call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
-               dict_t *xdata)
+dht_ftruncate(call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
+              dict_t *xdata)
 {
-        xlator_t     *subvol = NULL;
-        int           op_errno = -1;
-        dht_local_t  *local = NULL;
-
-        VALIDATE_OR_GOTO (frame, err);
-        VALIDATE_OR_GOTO (this, err);
-        VALIDATE_OR_GOTO (fd, err);
-
-        local = dht_local_init (frame, NULL, fd, GF_FOP_FTRUNCATE);
-        if (!local) {
-                op_errno = ENOMEM;
-                goto err;
-        }
-
-        local->rebalance.offset = offset;
-        local->call_cnt = 1;
-        subvol = local->cached_subvol;
-        if (!subvol) {
-                gf_msg_debug (this->name, 0,
-                              "no cached subvolume for fd=%p", fd);
-                op_errno = EINVAL;
-                goto err;
-        }
-
-        STACK_WIND (frame, dht_truncate_cbk,
-                    subvol, subvol->fops->ftruncate,
-                    fd, offset, xdata);
-
-        return 0;
+    xlator_t *subvol = NULL;
+    int op_errno = -1;
+    dht_local_t *local = NULL;
+
+    VALIDATE_OR_GOTO(frame, err);
+    VALIDATE_OR_GOTO(this, err);
+    VALIDATE_OR_GOTO(fd, err);
+
+    local = dht_local_init(frame, NULL, fd, GF_FOP_FTRUNCATE);
+    if (!local) {
+        op_errno = ENOMEM;
+        goto err;
+    }
+
+    local->rebalance.offset = offset;
+    local->call_cnt = 1;
+    subvol = local->cached_subvol;
+    if (!subvol) {
+        gf_msg_debug(this->name, 0, "no cached subvolume for fd=%p", fd);
+        op_errno = EINVAL;
+        goto err;
+    }
+
+    if (xdata)
+        local->xattr_req = dict_ref(xdata);
+
+    STACK_WIND_COOKIE(frame, dht_truncate_cbk, subvol, subvol,
+                      subvol->fops->ftruncate, fd, local->rebalance.offset,
+                      local->xattr_req);
+    return 0;
 
 err:
-        op_errno = (op_errno == -1) ? errno : op_errno;
-        DHT_STACK_UNWIND (ftruncate, frame, -1, op_errno, NULL, NULL, NULL);
+    op_errno = (op_errno == -1) ? errno : op_errno;
+    DHT_STACK_UNWIND(ftruncate, frame, -1, op_errno, NULL, NULL, NULL);
 
-        return 0;
+    return 0;
 }
 
-
 int
-dht_fallocate_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
-                  int op_ret, int op_errno, struct iatt *prebuf,
-                  struct iatt *postbuf, dict_t *xdata)
+dht_fallocate_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int op_ret,
+                  int op_errno, struct iatt *prebuf, struct iatt *postbuf,
+                  dict_t *xdata)
 {
-        dht_local_t  *local = NULL;
-        call_frame_t *prev = NULL;
-        int           ret = -1;
-        xlator_t    *subvol = NULL;
-
-        GF_VALIDATE_OR_GOTO ("dht", frame, err);
-        GF_VALIDATE_OR_GOTO ("dht", this, out);
-        GF_VALIDATE_OR_GOTO ("dht", frame->local, out);
-        GF_VALIDATE_OR_GOTO ("dht", cookie, out);
+    dht_local_t *local = NULL;
+    xlator_t *prev = NULL;
+    int ret = -1;
+    xlator_t *src_subvol = NULL;
+    xlator_t *dst_subvol = NULL;
+
+    GF_VALIDATE_OR_GOTO("dht", frame, err);
+    GF_VALIDATE_OR_GOTO("dht", this, out);
+    GF_VALIDATE_OR_GOTO("dht", frame->local, out);
+    GF_VALIDATE_OR_GOTO("dht", cookie, out);
+
+    local = frame->local;
+    prev = cookie;
+
+    /* fallocate fails with EBADF if dht has not yet opened the fd
+     * on the cached subvol. This could happen if the file was migrated
+     * and a lookup updated the cached subvol in the inode ctx.
+     * We only check once as this could actually be a valid error.
+     */
+
+    if (dht_check_remote_fd_failed_error(local, op_ret, op_errno)) {
+        ret = dht_check_and_open_fd_on_subvol(this, frame);
+        if (ret)
+            goto out;
+        return 0;
+    }
 
-        local = frame->local;
-        prev = cookie;
+    if ((op_ret == -1) && !dht_inode_missing(op_errno)) {
+        local->op_errno = op_errno;
+        local->op_ret = -1;
+        gf_msg_debug(this->name, op_errno, "subvolume %s returned -1",
+                     prev->name);
 
-        if ((op_ret == -1) && !dht_inode_missing(op_errno)) {
-                local->op_errno = op_errno;
-                local->op_ret = -1;
-                gf_msg_debug (this->name, 0,
-                              "subvolume %s returned -1 (%s)",
-                              prev->this->name, strerror (op_errno));
+        goto out;
+    }
 
-                goto out;
+    if (local->call_cnt != 1) {
+        if (local->stbuf.ia_blocks) {
+            dht_iatt_merge(this, postbuf, &local->stbuf);
+            dht_iatt_merge(this, prebuf, &local->prebuf);
         }
+        goto out;
+    }
 
-        if (local->call_cnt != 1) {
-                if (local->stbuf.ia_blocks) {
-                        dht_iatt_merge (this, postbuf, &local->stbuf, NULL);
-                        dht_iatt_merge (this, prebuf, &local->prebuf, NULL);
-                }
-                goto out;
-        }
-        local->rebalance.target_op_fn = dht_fallocate2;
+    local->op_ret = op_ret;
+    local->op_errno = op_errno;
+    local->rebalance.target_op_fn = dht_fallocate2;
 
-        /* Phase 2 of migration */
-        if ((op_ret == -1) || IS_DHT_MIGRATION_PHASE2 (postbuf)) {
-                ret = dht_rebalance_complete_check (this, frame);
-                if (!ret)
-                        return 0;
-        }
+    dht_set_local_rebalance(this, local, NULL, prebuf, postbuf, xdata);
 
-        /* Check if the rebalance phase1 is true */
-        if (IS_DHT_MIGRATION_PHASE1 (postbuf)) {
-                dht_iatt_merge (this, &local->stbuf, postbuf, NULL);
-                dht_iatt_merge (this, &local->prebuf, prebuf, NULL);
-                dht_inode_ctx_get1 (this, local->fd->inode, &subvol);
-                if (subvol) {
-                        dht_fallocate2 (this, frame, 0);
-                        return 0;
-                }
-                ret = dht_rebalance_in_progress_check (this, frame);
-                if (!ret)
-                        return 0;
+    /* Phase 2 of migration */
+    if ((op_ret == -1) || IS_DHT_MIGRATION_PHASE2(postbuf)) {
+        ret = dht_rebalance_complete_check(this, frame);
+        if (!ret)
+            return 0;
+    }
+
+    /* Check if the rebalance phase1 is true */
+    if (IS_DHT_MIGRATION_PHASE1(postbuf)) {
+        dht_iatt_merge(this, &local->stbuf, postbuf);
+        dht_iatt_merge(this, &local->prebuf, prebuf);
+
+        dht_inode_ctx_get_mig_info(this, local->fd->inode, &src_subvol,
+                                   &dst_subvol);
+        if (!dht_mig_info_is_invalid(local->cached_subvol, src_subvol,
+                                     dst_subvol)) {
+            if (dht_fd_open_on_dst(this, local->fd, dst_subvol)) {
+                dht_fallocate2(this, dst_subvol, frame, 0);
+                return 0;
+            }
         }
+        ret = dht_rebalance_in_progress_check(this, frame);
+        if (!ret)
+            return 0;
+    }
 
 out:
-        DHT_STRIP_PHASE1_FLAGS (postbuf);
-        DHT_STRIP_PHASE1_FLAGS (prebuf);
-        DHT_STACK_UNWIND (fallocate, frame, op_ret, op_errno,
-                          prebuf, postbuf, xdata);
+    DHT_STRIP_PHASE1_FLAGS(postbuf);
+    DHT_STRIP_PHASE1_FLAGS(prebuf);
+
+    DHT_STACK_UNWIND(fallocate, frame, op_ret, op_errno, prebuf, postbuf,
+                     xdata);
 err:
-        return 0;
+    return 0;
 }
 
-int
-dht_fallocate2(xlator_t *this, call_frame_t *frame, int op_ret)
+static int
+dht_fallocate2(xlator_t *this, xlator_t *subvol, call_frame_t *frame, int ret)
 {
-        dht_local_t  *local  = NULL;
-        xlator_t     *subvol = NULL;
-
-        local = frame->local;
+    dht_local_t *local = NULL;
+    int32_t op_errno = EINVAL;
+
+    if (!frame || !frame->local)
+        goto out;
+
+    local = frame->local;
+    op_errno = local->op_errno;
+
+    if (we_are_not_migrating(ret)) {
+        /* This dht xlator is not migrating the file. Unwind and
+         * pass on the original mode bits so the higher DHT layer
+         * can handle this.
+         */
+        DHT_STACK_UNWIND(fallocate, frame, local->op_ret, local->op_errno,
+                         &local->rebalance.prebuf, &local->rebalance.postbuf,
+                         local->rebalance.xdata);
+        return 0;
+    }
 
-        dht_inode_ctx_get1 (this, local->fd->inode, &subvol);
+    if (subvol == NULL)
+        goto out;
 
-        if (!subvol)
-                subvol = local->cached_subvol;
+    local->call_cnt = 2; /* This is the second attempt */
 
-        local->call_cnt = 2; /* This is the second attempt */
+    STACK_WIND_COOKIE(frame, dht_fallocate_cbk, subvol, subvol,
+                      subvol->fops->fallocate, local->fd,
+                      local->rebalance.flags, local->rebalance.offset,
+                      local->rebalance.size, local->xattr_req);
 
-	STACK_WIND(frame, dht_fallocate_cbk, subvol, subvol->fops->fallocate,
-		   local->fd, local->rebalance.flags, local->rebalance.offset,
-		   local->rebalance.size, NULL);
+    return 0;
 
-        return 0;
+out:
+    DHT_STACK_UNWIND(fallocate, frame, -1, op_errno, NULL, NULL, NULL);
+    return 0;
 }
 
 int
 dht_fallocate(call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t mode,
-	      off_t offset, size_t len, dict_t *xdata)
+              off_t offset, size_t len, dict_t *xdata)
 {
-        xlator_t     *subvol = NULL;
-        int           op_errno = -1;
-        dht_local_t  *local = NULL;
-
-        VALIDATE_OR_GOTO (frame, err);
-        VALIDATE_OR_GOTO (this, err);
-        VALIDATE_OR_GOTO (fd, err);
-
-        local = dht_local_init (frame, NULL, fd, GF_FOP_FALLOCATE);
-        if (!local) {
-                op_errno = ENOMEM;
-                goto err;
-        }
-
-	local->rebalance.flags = mode;
-        local->rebalance.offset = offset;
-	local->rebalance.size = len;
-
-        local->call_cnt = 1;
-        subvol = local->cached_subvol;
-        if (!subvol) {
-                gf_msg_debug (this->name, 0,
-                              "no cached subvolume for fd=%p", fd);
-                op_errno = EINVAL;
-                goto err;
-        }
-
-        STACK_WIND (frame, dht_fallocate_cbk,
-                    subvol, subvol->fops->fallocate,
-                    fd, mode, offset, len, xdata);
-
-        return 0;
+    xlator_t *subvol = NULL;
+    int op_errno = -1;
+    dht_local_t *local = NULL;
+
+    VALIDATE_OR_GOTO(frame, err);
+    VALIDATE_OR_GOTO(this, err);
+    VALIDATE_OR_GOTO(fd, err);
+
+    local = dht_local_init(frame, NULL, fd, GF_FOP_FALLOCATE);
+    if (!local) {
+        op_errno = ENOMEM;
+        goto err;
+    }
+
+    local->rebalance.flags = mode;
+    local->rebalance.offset = offset;
+    local->rebalance.size = len;
+
+    local->call_cnt = 1;
+    subvol = local->cached_subvol;
+    if (!subvol) {
+        gf_msg_debug(this->name, 0, "no cached subvolume for fd=%p", fd);
+        op_errno = EINVAL;
+        goto err;
+    }
+
+    if (xdata)
+        local->xattr_req = dict_ref(xdata);
+
+    STACK_WIND_COOKIE(frame, dht_fallocate_cbk, subvol, subvol,
+                      subvol->fops->fallocate, fd, local->rebalance.flags,
+                      local->rebalance.offset, local->rebalance.size,
+                      local->xattr_req);
+
+    return 0;
 
 err:
-        op_errno = (op_errno == -1) ? errno : op_errno;
-        DHT_STACK_UNWIND (fallocate, frame, -1, op_errno, NULL, NULL, NULL);
+    op_errno = (op_errno == -1) ? errno : op_errno;
+    DHT_STACK_UNWIND(fallocate, frame, -1, op_errno, NULL, NULL, NULL);
 
-        return 0;
+    return 0;
 }
 
-
 int
-dht_discard_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
-                int op_ret, int op_errno, struct iatt *prebuf,
-                struct iatt *postbuf, dict_t *xdata)
+dht_discard_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int op_ret,
+                int op_errno, struct iatt *prebuf, struct iatt *postbuf,
+                dict_t *xdata)
 {
-        dht_local_t  *local = NULL;
-        call_frame_t *prev = NULL;
-        int           ret = -1;
-        xlator_t    *subvol = NULL;
-
-        GF_VALIDATE_OR_GOTO ("dht", frame, err);
-        GF_VALIDATE_OR_GOTO ("dht", this, out);
-        GF_VALIDATE_OR_GOTO ("dht", frame->local, out);
-        GF_VALIDATE_OR_GOTO ("dht", cookie, out);
+    dht_local_t *local = NULL;
+    xlator_t *prev = NULL;
+    int ret = -1;
+    xlator_t *src_subvol = NULL;
+    xlator_t *dst_subvol = NULL;
+
+    GF_VALIDATE_OR_GOTO("dht", frame, err);
+    GF_VALIDATE_OR_GOTO("dht", this, out);
+    GF_VALIDATE_OR_GOTO("dht", frame->local, out);
+    GF_VALIDATE_OR_GOTO("dht", cookie, out);
+
+    local = frame->local;
+    prev = cookie;
+
+    /* discard fails with EBADF if dht has not yet opened the fd
+     * on the cached subvol. This could happen if the file was migrated
+     * and a lookup updated the cached subvol in the inode ctx.
+     * We only check once as this could actually be a valid error.
+     */
+    if (dht_check_remote_fd_failed_error(local, op_ret, op_errno)) {
+        ret = dht_check_and_open_fd_on_subvol(this, frame);
+        if (ret)
+            goto out;
+        return 0;
+    }
 
-        local = frame->local;
-        prev = cookie;
+    if ((op_ret == -1) && !dht_inode_missing(op_errno)) {
+        local->op_errno = op_errno;
+        local->op_ret = -1;
+        gf_msg_debug(this->name, op_errno, "subvolume %s returned -1",
+                     prev->name);
 
-        if ((op_ret == -1) && !dht_inode_missing(op_errno)) {
-                local->op_errno = op_errno;
-                local->op_ret = -1;
-                gf_msg_debug (this->name, 0,
-                              "subvolume %s returned -1 (%s)",
-                              prev->this->name, strerror (op_errno));
+        goto out;
+    }
 
-                goto out;
+    if (local->call_cnt != 1) {
+        if (local->stbuf.ia_blocks) {
+            dht_iatt_merge(this, postbuf, &local->stbuf);
+            dht_iatt_merge(this, prebuf, &local->prebuf);
         }
+        goto out;
+    }
 
-        if (local->call_cnt != 1) {
-                if (local->stbuf.ia_blocks) {
-                        dht_iatt_merge (this, postbuf, &local->stbuf, NULL);
-                        dht_iatt_merge (this, prebuf, &local->prebuf, NULL);
-                }
-                goto out;
-        }
-        local->rebalance.target_op_fn = dht_discard2;
+    local->rebalance.target_op_fn = dht_discard2;
+    local->op_ret = op_ret;
+    local->op_errno = op_errno;
 
-        /* Phase 2 of migration */
-        if ((op_ret == -1) || IS_DHT_MIGRATION_PHASE2 (postbuf)) {
-                ret = dht_rebalance_complete_check (this, frame);
-                if (!ret)
-                        return 0;
-        }
+    dht_set_local_rebalance(this, local, NULL, prebuf, postbuf, xdata);
 
-        /* Check if the rebalance phase1 is true */
-        if (IS_DHT_MIGRATION_PHASE1 (postbuf)) {
-                dht_iatt_merge (this, &local->stbuf, postbuf, NULL);
-                dht_iatt_merge (this, &local->prebuf, prebuf, NULL);
-                dht_inode_ctx_get1 (this, local->fd->inode, &subvol);
-                if (subvol) {
-                        dht_discard2 (this, frame, 0);
-                        return 0;
-                }
-                ret = dht_rebalance_in_progress_check (this, frame);
-                if (!ret)
-                        return 0;
+    /* Phase 2 of migration */
+    if ((op_ret == -1) || IS_DHT_MIGRATION_PHASE2(postbuf)) {
+        ret = dht_rebalance_complete_check(this, frame);
+        if (!ret)
+            return 0;
+    }
+
+    /* Check if the rebalance phase1 is true */
+    if (IS_DHT_MIGRATION_PHASE1(postbuf)) {
+        dht_iatt_merge(this, &local->stbuf, postbuf);
+        dht_iatt_merge(this, &local->prebuf, prebuf);
+
+        dht_inode_ctx_get_mig_info(this, local->fd->inode, &src_subvol,
+                                   &dst_subvol);
+        if (!dht_mig_info_is_invalid(local->cached_subvol, src_subvol,
+                                     dst_subvol)) {
+            if (dht_fd_open_on_dst(this, local->fd, dst_subvol)) {
+                dht_discard2(this, dst_subvol, frame, 0);
+                return 0;
+            }
         }
+        ret = dht_rebalance_in_progress_check(this, frame);
+        if (!ret)
+            return 0;
+    }
 
 out:
-        DHT_STRIP_PHASE1_FLAGS (postbuf);
-        DHT_STRIP_PHASE1_FLAGS (prebuf);
-        DHT_STACK_UNWIND (discard, frame, op_ret, op_errno,
-                          prebuf, postbuf, xdata);
+    DHT_STRIP_PHASE1_FLAGS(postbuf);
+    DHT_STRIP_PHASE1_FLAGS(prebuf);
+
+    DHT_STACK_UNWIND(discard, frame, op_ret, op_errno, prebuf, postbuf, xdata);
 err:
-        return 0;
+    return 0;
 }
 
-int
-dht_discard2(xlator_t *this, call_frame_t *frame, int op_ret)
+static int
+dht_discard2(xlator_t *this, xlator_t *subvol, call_frame_t *frame, int ret)
 {
-        dht_local_t  *local  = NULL;
-        xlator_t     *subvol = NULL;
-
-        local = frame->local;
+    dht_local_t *local = NULL;
+    int32_t op_errno = EINVAL;
+
+    if (!frame || !frame->local)
+        goto out;
+
+    local = frame->local;
+    op_errno = local->op_errno;
+
+    if (we_are_not_migrating(ret)) {
+        /* This dht xlator is not migrating the file. Unwind and
+         * pass on the original mode bits so the higher DHT layer
+         * can handle this.
+         */
+        DHT_STACK_UNWIND(discard, frame, local->op_ret, local->op_errno,
+                         &local->rebalance.prebuf, &local->rebalance.postbuf,
+                         local->rebalance.xdata);
+        return 0;
+    }
 
-        dht_inode_ctx_get1 (this, local->fd->inode, &subvol);
+    if (subvol == NULL)
+        goto out;
 
-        if (!subvol)
-                subvol = local->cached_subvol;
+    local->call_cnt = 2; /* This is the second attempt */
 
-        local->call_cnt = 2; /* This is the second attempt */
+    STACK_WIND_COOKIE(frame, dht_discard_cbk, subvol, subvol,
+                      subvol->fops->discard, local->fd, local->rebalance.offset,
+                      local->rebalance.size, local->xattr_req);
 
-	STACK_WIND(frame, dht_discard_cbk, subvol, subvol->fops->discard,
-		   local->fd, local->rebalance.offset, local->rebalance.size,
-		   NULL);
+    return 0;
 
-        return 0;
+out:
+    DHT_STACK_UNWIND(discard, frame, -1, op_errno, NULL, NULL, NULL);
+    return 0;
 }
 
 int
 dht_discard(call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
-	    size_t len, dict_t *xdata)
+            size_t len, dict_t *xdata)
 {
-        xlator_t     *subvol = NULL;
-        int           op_errno = -1;
-        dht_local_t  *local = NULL;
-
-        VALIDATE_OR_GOTO (frame, err);
-        VALIDATE_OR_GOTO (this, err);
-        VALIDATE_OR_GOTO (fd, err);
-
-        local = dht_local_init (frame, NULL, fd, GF_FOP_DISCARD);
-        if (!local) {
-                op_errno = ENOMEM;
-                goto err;
-        }
+    xlator_t *subvol = NULL;
+    int op_errno = -1;
+    dht_local_t *local = NULL;
 
-        local->rebalance.offset = offset;
-	local->rebalance.size = len;
+    VALIDATE_OR_GOTO(frame, err);
+    VALIDATE_OR_GOTO(this, err);
+    VALIDATE_OR_GOTO(fd, err);
 
-        local->call_cnt = 1;
-        subvol = local->cached_subvol;
-        if (!subvol) {
-                gf_msg_debug (this->name, 0,
-                              "no cached subvolume for fd=%p", fd);
-                op_errno = EINVAL;
-                goto err;
-        }
+    local = dht_local_init(frame, NULL, fd, GF_FOP_DISCARD);
+    if (!local) {
+        op_errno = ENOMEM;
+        goto err;
+    }
 
-        STACK_WIND (frame, dht_discard_cbk, subvol, subvol->fops->discard,
-                    fd, offset, len, xdata);
+    local->rebalance.offset = offset;
+    local->rebalance.size = len;
 
-        return 0;
+    local->call_cnt = 1;
+    subvol = local->cached_subvol;
+    if (!subvol) {
+        gf_msg_debug(this->name, 0, "no cached subvolume for fd=%p", fd);
+        op_errno = EINVAL;
+        goto err;
+    }
+
+    if (xdata)
+        local->xattr_req = dict_ref(xdata);
+
+    STACK_WIND_COOKIE(frame, dht_discard_cbk, subvol, subvol,
+                      subvol->fops->discard, fd, local->rebalance.offset,
+                      local->rebalance.size, local->xattr_req);
+
+    return 0;
 
 err:
-        op_errno = (op_errno == -1) ? errno : op_errno;
-        DHT_STACK_UNWIND (discard, frame, -1, op_errno, NULL, NULL, NULL);
+    op_errno = (op_errno == -1) ? errno : op_errno;
+    DHT_STACK_UNWIND(discard, frame, -1, op_errno, NULL, NULL, NULL);
 
-        return 0;
+    return 0;
 }
 
 int
-dht_zerofill_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
-                int op_ret, int op_errno, struct iatt *prebuf,
-                struct iatt *postbuf, dict_t *xdata)
+dht_zerofill_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int op_ret,
+                 int op_errno, struct iatt *prebuf, struct iatt *postbuf,
+                 dict_t *xdata)
 {
-        dht_local_t  *local = NULL;
-        call_frame_t *prev  = NULL;
-        int           ret   = -1;
+    dht_local_t *local = NULL;
+    xlator_t *prev = NULL;
+    int ret = -1;
+    xlator_t *subvol1 = NULL, *subvol2 = NULL;
+
+    GF_VALIDATE_OR_GOTO("dht", frame, err);
+    GF_VALIDATE_OR_GOTO("dht", this, out);
+    GF_VALIDATE_OR_GOTO("dht", frame->local, out);
+    GF_VALIDATE_OR_GOTO("dht", cookie, out);
+
+    local = frame->local;
+    prev = cookie;
+
+    /* zerofill fails with EBADF if dht has not yet opened the fd
+     * on the cached subvol. This could happen if the file was migrated
+     * and a lookup updated the cached subvol in the inode ctx.
+     * We only check once as this could actually be a valid error.
+     */
+    if (dht_check_remote_fd_failed_error(local, op_ret, op_errno)) {
+        ret = dht_check_and_open_fd_on_subvol(this, frame);
+        if (ret)
+            goto out;
+        return 0;
+    }
 
-        GF_VALIDATE_OR_GOTO ("dht", frame, err);
-        GF_VALIDATE_OR_GOTO ("dht", this, out);
-        GF_VALIDATE_OR_GOTO ("dht", frame->local, out);
-        GF_VALIDATE_OR_GOTO ("dht", cookie, out);
+    if ((op_ret == -1) && !dht_inode_missing(op_errno)) {
+        local->op_errno = op_errno;
+        local->op_ret = -1;
+        gf_msg_debug(this->name, op_errno, "subvolume %s returned -1",
+                     prev->name);
+        goto out;
+    }
+
+    if (local->call_cnt != 1) {
+        if (local->stbuf.ia_blocks) {
+            dht_iatt_merge(this, postbuf, &local->stbuf);
+            dht_iatt_merge(this, prebuf, &local->prebuf);
+        }
+        goto out;
+    }
 
-        local = frame->local;
-        prev = cookie;
+    local->rebalance.target_op_fn = dht_zerofill2;
+    local->op_ret = op_ret;
+    local->op_errno = op_errno;
 
-        if ((op_ret == -1) && !dht_inode_missing(op_errno)) {
-                local->op_errno = op_errno;
-                local->op_ret = -1;
-                gf_msg_debug (this->name, 0,
-                              "subvolume %s returned -1 (%s)",
-                              prev->this->name, strerror (op_errno));
-                goto out;
-        }
+    dht_set_local_rebalance(this, local, NULL, prebuf, postbuf, xdata);
 
-        if (local->call_cnt != 1) {
-                if (local->stbuf.ia_blocks) {
-                        dht_iatt_merge (this, postbuf, &local->stbuf, NULL);
-                        dht_iatt_merge (this, prebuf, &local->prebuf, NULL);
-                }
-                goto out;
-        }
-        local->rebalance.target_op_fn = dht_zerofill2;
-        /* Phase 2 of migration */
-        if ((op_ret == -1) || IS_DHT_MIGRATION_PHASE2 (postbuf)) {
-                ret = dht_rebalance_complete_check (this, frame);
-                if (!ret)
-                        return 0;
+    /* Phase 2 of migration */
+    if ((op_ret == -1) || IS_DHT_MIGRATION_PHASE2(postbuf)) {
+        ret = dht_rebalance_complete_check(this, frame);
+        if (!ret)
+            return 0;
+    }
+
+    /* Check if the rebalance phase1 is true */
+    if (IS_DHT_MIGRATION_PHASE1(postbuf)) {
+        dht_iatt_merge(this, &local->stbuf, postbuf);
+        dht_iatt_merge(this, &local->prebuf, prebuf);
+
+        ret = dht_inode_ctx_get_mig_info(this, local->fd->inode, &subvol1,
+                                         &subvol2);
+        if (!dht_mig_info_is_invalid(local->cached_subvol, subvol1, subvol2)) {
+            if (dht_fd_open_on_dst(this, local->fd, subvol2)) {
+                dht_zerofill2(this, subvol2, frame, 0);
+                return 0;
+            }
         }
 
-        /* Check if the rebalance phase1 is true */
-        if (IS_DHT_MIGRATION_PHASE1 (postbuf)) {
-                dht_iatt_merge (this, &local->stbuf, postbuf, NULL);
-                dht_iatt_merge (this, &local->prebuf, prebuf, NULL);
-                ret = fd_ctx_get (local->fd, this, NULL);
-                if (!ret) {
-                        dht_zerofill2 (this, frame, 0);
-                        return 0;
-                }
-                ret = dht_rebalance_in_progress_check (this, frame);
-                if (!ret)
-                        return 0;
-        }
+        ret = dht_rebalance_in_progress_check(this, frame);
+        if (!ret)
+            return 0;
+    }
 
 out:
-        DHT_STRIP_PHASE1_FLAGS (postbuf);
-        DHT_STRIP_PHASE1_FLAGS (prebuf);
-        DHT_STACK_UNWIND (zerofill, frame, op_ret, op_errno,
-                          prebuf, postbuf, xdata);
+    DHT_STRIP_PHASE1_FLAGS(postbuf);
+    DHT_STRIP_PHASE1_FLAGS(prebuf);
+
+    DHT_STACK_UNWIND(zerofill, frame, op_ret, op_errno, prebuf, postbuf, xdata);
 err:
-        return 0;
+    return 0;
 }
 
-int
-dht_zerofill2(xlator_t *this, call_frame_t *frame, int op_ret)
+static int
+dht_zerofill2(xlator_t *this, xlator_t *subvol, call_frame_t *frame, int ret)
 {
-        dht_local_t  *local          = NULL;
-        xlator_t     *subvol         = NULL;
-        uint64_t      tmp_subvol     = 0;
-        int           ret            = -1;
-
-        local = frame->local;
+    dht_local_t *local = NULL;
+    int32_t op_errno = EINVAL;
 
-        if (local->fd)
-                ret = fd_ctx_get (local->fd, this, &tmp_subvol);
-        if (!ret)
-                subvol = (xlator_t *)(long)tmp_subvol;
+    if (!frame || !frame->local)
+        goto out;
 
-        if (!subvol)
-                subvol = local->cached_subvol;
+    local = frame->local;
 
-        local->call_cnt = 2; /* This is the second attempt */
+    op_errno = local->op_errno;
 
-        STACK_WIND(frame, dht_zerofill_cbk, subvol, subvol->fops->zerofill,
-                   local->fd, local->rebalance.offset, local->rebalance.size,
-                   NULL);
+    if (we_are_not_migrating(ret)) {
+        /* This dht xlator is not migrating the file. Unwind and
+         * pass on the original mode bits so the higher DHT layer
+         * can handle this.
+         */
+        DHT_STACK_UNWIND(zerofill, frame, local->op_ret, local->op_errno,
+                         &local->rebalance.prebuf, &local->rebalance.postbuf,
+                         local->rebalance.xdata);
 
         return 0;
+    }
+
+    if (subvol == NULL)
+        goto out;
+
+    local->call_cnt = 2; /* This is the second attempt */
+
+    STACK_WIND_COOKIE(frame, dht_zerofill_cbk, subvol, subvol,
+                      subvol->fops->zerofill, local->fd,
+                      local->rebalance.offset, local->rebalance.size,
+                      local->xattr_req);
+
+    return 0;
+
+out:
+
+    DHT_STACK_UNWIND(zerofill, frame, -1, op_errno, NULL, NULL, NULL);
+    return 0;
 }
 
 int
 dht_zerofill(call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
-            off_t len, dict_t *xdata)
+             off_t len, dict_t *xdata)
 {
-        xlator_t     *subvol       = NULL;
-        int           op_errno     = -1;
-        dht_local_t  *local        = NULL;
-
-        VALIDATE_OR_GOTO (frame, err);
-        VALIDATE_OR_GOTO (this, err);
-        VALIDATE_OR_GOTO (fd, err);
-
-        local = dht_local_init (frame, NULL, fd, GF_FOP_ZEROFILL);
-        if (!local) {
-                op_errno = ENOMEM;
-                goto err;
-        }
+    xlator_t *subvol = NULL;
+    int op_errno = -1;
+    dht_local_t *local = NULL;
 
-        local->rebalance.offset = offset;
-        local->rebalance.size = len;
+    VALIDATE_OR_GOTO(frame, err);
+    VALIDATE_OR_GOTO(this, err);
+    VALIDATE_OR_GOTO(fd, err);
 
-        local->call_cnt = 1;
-        subvol = local->cached_subvol;
-        if (!subvol) {
-                gf_msg_debug (this->name, 0,
-                              "no cached subvolume for fd=%p", fd);
-                op_errno = EINVAL;
-                goto err;
-        }
+    local = dht_local_init(frame, NULL, fd, GF_FOP_ZEROFILL);
+    if (!local) {
+        op_errno = ENOMEM;
+        goto err;
+    }
 
-        STACK_WIND (frame, dht_zerofill_cbk, subvol, subvol->fops->zerofill,
-                    fd, offset, len, xdata);
+    local->rebalance.offset = offset;
+    local->rebalance.size = len;
 
-        return 0;
+    local->call_cnt = 1;
+    subvol = local->cached_subvol;
+    if (!subvol) {
+        gf_msg_debug(this->name, 0, "no cached subvolume for fd=%p", fd);
+        op_errno = EINVAL;
+        goto err;
+    }
 
-err:
-        op_errno = (op_errno == -1) ? errno : op_errno;
-        DHT_STACK_UNWIND (zerofill, frame, -1, op_errno, NULL, NULL, NULL);
+    if (xdata)
+        local->xattr_req = dict_ref(xdata);
 
-        return 0;
-}
+    STACK_WIND_COOKIE(frame, dht_zerofill_cbk, subvol, subvol,
+                      subvol->fops->zerofill, fd, local->rebalance.offset,
+                      local->rebalance.size, local->xattr_req);
 
+    return 0;
 
+err:
+    op_errno = (op_errno == -1) ? errno : op_errno;
+    DHT_STACK_UNWIND(zerofill, frame, -1, op_errno, NULL, NULL, NULL);
+
+    return 0;
+}
 
 /* handle cases of migration here for 'setattr()' calls */
 int
-dht_file_setattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                      int op_ret, int op_errno, struct iatt *prebuf,
-                      struct iatt *postbuf, dict_t *xdata)
+dht_file_setattr_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                     int op_ret, int op_errno, struct iatt *prebuf,
+                     struct iatt *postbuf, dict_t *xdata)
 {
-        dht_local_t  *local = NULL;
-        call_frame_t *prev = NULL;
-        int           ret = -1;
+    dht_local_t *local = NULL;
+    xlator_t *prev = NULL;
+    int ret = -1;
 
-        local = frame->local;
-        prev = cookie;
+    local = frame->local;
+    prev = cookie;
 
-        local->op_errno = op_errno;
-        if ((op_ret == -1) && !dht_inode_missing(op_errno)) {
-                gf_msg_debug (this->name, 0,
-                              "subvolume %s returned -1 (%s)",
-                              prev->this->name, strerror (op_errno));
-                goto out;
-        }
+    local->op_errno = op_errno;
 
-        if (local->call_cnt != 1)
-                goto out;
+    if ((local->fop == GF_FOP_FSETATTR) &&
+        dht_check_remote_fd_failed_error(local, op_ret, op_errno)) {
+        ret = dht_check_and_open_fd_on_subvol(this, frame);
+        if (ret)
+            goto out;
+        return 0;
+    }
 
-        local->rebalance.target_op_fn = dht_setattr2;
+    if ((op_ret == -1) && !dht_inode_missing(op_errno)) {
+        gf_msg_debug(this->name, op_errno, "subvolume %s returned -1",
+                     prev->name);
+        goto out;
+    }
 
-        /* Phase 2 of migration */
-        if ((op_ret == -1) || IS_DHT_MIGRATION_PHASE2 (postbuf)) {
-                ret = dht_rebalance_complete_check (this, frame);
-                if (!ret)
-                        return 0;
-        }
+    if (local->call_cnt != 1)
+        goto out;
+
+    local->op_ret = op_ret;
+    local->op_errno = op_errno;
+
+    local->rebalance.target_op_fn = dht_setattr2;
 
-        /* At the end of the migration process, whatever 'attr' we
-           have on source file will be migrated to destination file
-           in one shot, hence we don't need to check for in progress
-           state here (ie, PHASE1) */
+    /* Phase 2 of migration */
+    if ((op_ret == -1) || IS_DHT_MIGRATION_PHASE2(postbuf)) {
+        dht_set_local_rebalance(this, local, NULL, prebuf, postbuf, xdata);
+
+        ret = dht_rebalance_complete_check(this, frame);
+        if (!ret)
+            return 0;
+    }
+
+    /* At the end of the migration process, whatever 'attr' we
+       have on source file will be migrated to destination file
+       in one shot, hence we don't need to check for in progress
+       state here (ie, PHASE1) */
 out:
-        DHT_STRIP_PHASE1_FLAGS (postbuf);
-        DHT_STRIP_PHASE1_FLAGS (prebuf);
-        DHT_STACK_UNWIND (setattr, frame, op_ret, op_errno,
-                          prebuf, postbuf, xdata);
+    DHT_STRIP_PHASE1_FLAGS(postbuf);
+    DHT_STRIP_PHASE1_FLAGS(prebuf);
 
-        return 0;
+    DHT_STACK_UNWIND(setattr, frame, op_ret, op_errno, prebuf, postbuf, xdata);
+
+    return 0;
 }
 
-int
-dht_setattr2 (xlator_t *this, call_frame_t *frame, int op_ret)
+static int
+dht_setattr2(xlator_t *this, xlator_t *subvol, call_frame_t *frame, int ret)
 {
-        dht_local_t  *local  = NULL;
-        xlator_t     *subvol = NULL;
-        inode_t      *inode = NULL;
+    dht_local_t *local = NULL;
+    int32_t op_errno = EINVAL;
+
+    if (!frame || !frame->local)
+        goto out;
+
+    local = frame->local;
+    op_errno = local->op_errno;
+
+    if (we_are_not_migrating(ret)) {
+        /* This dht xlator is not migrating the file. Unwind and
+         * pass on the original mode bits so the higher DHT layer
+         * can handle this.
+         */
+        DHT_STACK_UNWIND(setattr, frame, local->op_ret, local->op_errno,
+                         &local->rebalance.prebuf, &local->rebalance.postbuf,
+                         local->rebalance.xdata);
+        return 0;
+    }
 
-        local = frame->local;
+    if (subvol == NULL)
+        goto out;
 
-        inode = (local->fd) ? local->fd->inode : local->loc.inode;
+    local->call_cnt = 2; /* This is the second attempt */
 
-        dht_inode_ctx_get1 (this, inode, &subvol);
+    if (local->fop == GF_FOP_SETATTR) {
+        STACK_WIND_COOKIE(frame, dht_file_setattr_cbk, subvol, subvol,
+                          subvol->fops->setattr, &local->loc,
+                          &local->rebalance.stbuf, local->rebalance.flags,
+                          local->xattr_req);
+    } else {
+        STACK_WIND_COOKIE(frame, dht_file_setattr_cbk, subvol, subvol,
+                          subvol->fops->fsetattr, local->fd,
+                          &local->rebalance.stbuf, local->rebalance.flags,
+                          local->xattr_req);
+    }
 
-        if (!subvol)
-                subvol = local->cached_subvol;
+    return 0;
 
-        local->call_cnt = 2; /* This is the second attempt */
+out:
+    DHT_STACK_UNWIND(setattr, frame, -1, op_errno, NULL, NULL, NULL);
+    return 0;
+}
 
-        if (local->fop == GF_FOP_SETATTR) {
-                STACK_WIND (frame, dht_file_setattr_cbk, subvol,
-                            subvol->fops->setattr, &local->loc,
-                            &local->rebalance.stbuf, local->rebalance.flags,
-                            NULL);
-        } else {
-                STACK_WIND (frame, dht_file_setattr_cbk, subvol,
-                            subvol->fops->fsetattr, local->fd,
-                            &local->rebalance.stbuf, local->rebalance.flags,
-                            NULL);
+/* Keep the existing code same for all the cases other than regular file */
+int
+dht_setattr_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int op_ret,
+                int op_errno, struct iatt *statpre, struct iatt *statpost,
+                dict_t *xdata)
+{
+    dht_local_t *local = NULL;
+    int this_call_cnt = 0;
+    xlator_t *prev = NULL;
+
+    local = frame->local;
+    prev = cookie;
+
+    LOCK(&frame->lock);
+    {
+        if (op_ret == -1) {
+            local->op_errno = op_errno;
+            UNLOCK(&frame->lock);
+            gf_msg_debug(this->name, op_errno, "subvolume %s returned -1",
+                         prev->name);
+            goto post_unlock;
         }
 
-        return 0;
+        dht_iatt_merge(this, &local->prebuf, statpre);
+        dht_iatt_merge(this, &local->stbuf, statpost);
+
+        local->op_ret = 0;
+        local->op_errno = 0;
+    }
+    UNLOCK(&frame->lock);
+post_unlock:
+    this_call_cnt = dht_frame_return(frame);
+    if (is_last_call(this_call_cnt)) {
+        if (local->op_ret == 0)
+            dht_inode_ctx_time_set(local->loc.inode, this, &local->stbuf);
+        DHT_STACK_UNWIND(setattr, frame, local->op_ret, local->op_errno,
+                         &local->prebuf, &local->stbuf, xdata);
+    }
+
+    return 0;
 }
 
-
 /* Keep the existing code same for all the cases other than regular file */
 int
-dht_setattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                 int op_ret, int op_errno, struct iatt *statpre,
-                 struct iatt *statpost, dict_t *xdata)
+dht_non_mds_setattr_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                        int op_ret, int op_errno, struct iatt *statpre,
+                        struct iatt *statpost, dict_t *xdata)
 {
-        dht_local_t  *local = NULL;
-        int           this_call_cnt = 0;
-        call_frame_t *prev = NULL;
-
-
-        local = frame->local;
-        prev = cookie;
-
-        LOCK (&frame->lock);
-        {
-                if (op_ret == -1) {
-                        local->op_errno = op_errno;
-                        gf_msg_debug (this->name, 0,
-                                      "subvolume %s returned -1 (%s)",
-                                      prev->this->name, strerror (op_errno));
-                        goto unlock;
-                }
+    dht_local_t *local = NULL;
+    int this_call_cnt = 0;
+    xlator_t *prev = NULL;
+
+    local = frame->local;
+    prev = cookie;
+
+    if (op_ret == -1) {
+        gf_msg(this->name, op_errno, 0, 0, "subvolume %s returned -1",
+               prev->name);
+        goto post_unlock;
+    }
+
+    LOCK(&frame->lock);
+    {
+        dht_iatt_merge(this, &local->prebuf, statpre);
+        dht_iatt_merge(this, &local->stbuf, statpost);
+
+        local->op_ret = 0;
+        local->op_errno = 0;
+    }
+    UNLOCK(&frame->lock);
+post_unlock:
+    this_call_cnt = dht_frame_return(frame);
+    if (is_last_call(this_call_cnt)) {
+        dht_inode_ctx_time_set(local->loc.inode, this, &local->stbuf);
+        DHT_STACK_UNWIND(setattr, frame, 0, 0, &local->prebuf, &local->stbuf,
+                         xdata);
+    }
+
+    return 0;
+}
 
-                dht_iatt_merge (this, &local->prebuf, statpre, prev->this);
-                dht_iatt_merge (this, &local->stbuf, statpost, prev->this);
+int
+dht_mds_setattr_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                    int op_ret, int op_errno, struct iatt *statpre,
+                    struct iatt *statpost, dict_t *xdata)
 
-                local->op_ret = 0;
-        }
-unlock:
-        UNLOCK (&frame->lock);
-
-        this_call_cnt = dht_frame_return (frame);
-        if (is_last_call (this_call_cnt)) {
-		if (local->op_ret == 0)
-			dht_inode_ctx_time_set (local->loc.inode, this,
-						&local->stbuf);
-                DHT_STACK_UNWIND (setattr, frame, local->op_ret, local->op_errno,
-                                  &local->prebuf, &local->stbuf, xdata);
-	}
+{
+    dht_local_t *local = NULL;
+    dht_conf_t *conf = NULL;
+    xlator_t *prev = NULL;
+    xlator_t *mds_subvol = NULL;
+    struct iatt loc_stbuf = {
+        0,
+    };
+    int i = 0;
+
+    local = frame->local;
+    prev = cookie;
+    conf = this->private;
+    mds_subvol = local->mds_subvol;
+
+    if (op_ret == -1) {
+        local->op_ret = op_ret;
+        local->op_errno = op_errno;
+        gf_msg_debug(this->name, op_errno, "subvolume %s returned -1",
+                     prev->name);
+        goto out;
+    }
+
+    local->op_ret = 0;
+    loc_stbuf = local->stbuf;
+    dht_iatt_merge(this, &local->prebuf, statpre);
+    dht_iatt_merge(this, &local->stbuf, statpost);
+
+    local->call_cnt = conf->subvolume_cnt - 1;
+    for (i = 0; i < conf->subvolume_cnt; i++) {
+        if (mds_subvol == conf->subvolumes[i])
+            continue;
+        STACK_WIND_COOKIE(frame, dht_non_mds_setattr_cbk, conf->subvolumes[i],
+                          conf->subvolumes[i],
+                          conf->subvolumes[i]->fops->setattr, &local->loc,
+                          &loc_stbuf, local->valid, local->xattr_req);
+    }
+
+    return 0;
+out:
+    DHT_STACK_UNWIND(setattr, frame, local->op_ret, local->op_errno,
+                     &local->prebuf, &local->stbuf, xdata);
 
-        return 0;
+    return 0;
 }
 
-
 int
-dht_setattr (call_frame_t *frame, xlator_t *this, loc_t *loc,
-             struct iatt *stbuf, int32_t valid, dict_t *xdata)
+dht_setattr(call_frame_t *frame, xlator_t *this, loc_t *loc, struct iatt *stbuf,
+            int32_t valid, dict_t *xdata)
 {
-        xlator_t     *subvol = NULL;
-        dht_layout_t *layout = NULL;
-        dht_local_t  *local  = NULL;
-        int           op_errno = -1;
-        int           i = -1;
-        int           call_cnt = 0;
-
-        VALIDATE_OR_GOTO (frame, err);
-        VALIDATE_OR_GOTO (this, err);
-        VALIDATE_OR_GOTO (loc, err);
-        VALIDATE_OR_GOTO (loc->inode, err);
-        VALIDATE_OR_GOTO (loc->path, err);
-
-        local = dht_local_init (frame, loc, NULL, GF_FOP_SETATTR);
-        if (!local) {
-                op_errno = ENOMEM;
-                goto err;
-        }
+    xlator_t *subvol = NULL;
+    xlator_t *mds_subvol = NULL;
+    dht_layout_t *layout = NULL;
+    dht_local_t *local = NULL;
+    int op_errno = -1;
+    int i = -1;
+    int ret = -1;
+    int call_cnt = 0;
+    dht_conf_t *conf = NULL;
+
+    VALIDATE_OR_GOTO(frame, err);
+    VALIDATE_OR_GOTO(this, err);
+    VALIDATE_OR_GOTO(loc, err);
+    VALIDATE_OR_GOTO(loc->inode, err);
+    VALIDATE_OR_GOTO(loc->path, err);
+
+    conf = this->private;
+    local = dht_local_init(frame, loc, NULL, GF_FOP_SETATTR);
+    if (!local) {
+        op_errno = ENOMEM;
+        goto err;
+    }
+
+    layout = local->layout;
+    if (!layout) {
+        gf_msg_debug(this->name, 0, "no layout for path=%s", loc->path);
+        op_errno = EINVAL;
+        goto err;
+    }
+
+    if (!layout_is_sane(layout)) {
+        gf_msg_debug(this->name, 0, "layout is not sane for path=%s",
+                     loc->path);
+        op_errno = EINVAL;
+        goto err;
+    }
+    if (xdata)
+        local->xattr_req = dict_ref(xdata);
+
+    if (IA_ISREG(loc->inode->ia_type)) {
+        /* in the regular file _cbk(), we need to check for
+           migration possibilities */
+        local->rebalance.stbuf = *stbuf;
+        local->rebalance.flags = valid;
+        local->call_cnt = 1;
+        subvol = local->cached_subvol;
 
-        layout = local->layout;
-        if (!layout) {
-                gf_msg_debug (this->name, 0,
-                              "no layout for path=%s", loc->path);
-                op_errno = EINVAL;
-                goto err;
-        }
+        STACK_WIND_COOKIE(frame, dht_file_setattr_cbk, subvol, subvol,
+                          subvol->fops->setattr, loc, stbuf, valid, xdata);
 
-        if (!layout_is_sane (layout)) {
-                gf_msg_debug (this->name, 0,
-                              "layout is not sane for path=%s", loc->path);
-                op_errno = EINVAL;
-                goto err;
+        return 0;
+    }
+
+    local->call_cnt = call_cnt = layout->cnt;
+
+    if (IA_ISDIR(loc->inode->ia_type) && !__is_root_gfid(loc->inode->gfid) &&
+        call_cnt != 1) {
+        ret = dht_inode_ctx_mdsvol_get(loc->inode, this, &mds_subvol);
+        if (ret || !mds_subvol) {
+            gf_msg(this->name, GF_LOG_ERROR, 0,
+                   DHT_MSG_HASHED_SUBVOL_GET_FAILED,
+                   "Failed to get mds subvol for path %s", local->loc.path);
+            op_errno = EINVAL;
+            goto err;
         }
 
-        if (IA_ISREG (loc->inode->ia_type)) {
-                /* in the regular file _cbk(), we need to check for
-                   migration possibilities */
-                local->rebalance.stbuf = *stbuf;
-                local->rebalance.flags = valid;
-                local->call_cnt = 1;
-                subvol = local->cached_subvol;
-
-                STACK_WIND (frame, dht_file_setattr_cbk, subvol,
-                            subvol->fops->setattr,
-                            loc, stbuf, valid, xdata);
-
-                return 0;
+        local->mds_subvol = mds_subvol;
+        for (i = 0; i < conf->subvolume_cnt; i++) {
+            if (conf->subvolumes[i] == mds_subvol) {
+                if (!conf->subvolume_status[i]) {
+                    gf_msg(this->name, GF_LOG_WARNING, layout->list[i].err,
+                           DHT_MSG_HASHED_SUBVOL_DOWN,
+                           "MDS subvol is down for path "
+                           " %s Unable to set attr ",
+                           local->loc.path);
+                    op_errno = ENOTCONN;
+                    goto err;
+                }
+            }
         }
+        local->valid = valid;
+        local->stbuf = *stbuf;
 
-        local->call_cnt = call_cnt = layout->cnt;
-
+        STACK_WIND_COOKIE(frame, dht_mds_setattr_cbk, local->mds_subvol,
+                          local->mds_subvol, local->mds_subvol->fops->setattr,
+                          loc, stbuf, valid, xdata);
+        return 0;
+    } else {
         for (i = 0; i < call_cnt; i++) {
-                STACK_WIND (frame, dht_setattr_cbk,
-                            layout->list[i].xlator,
-                            layout->list[i].xlator->fops->setattr,
-                            loc, stbuf, valid, xdata);
+            STACK_WIND_COOKIE(frame, dht_setattr_cbk, layout->list[i].xlator,
+                              layout->list[i].xlator,
+                              layout->list[i].xlator->fops->setattr, loc, stbuf,
+                              valid, xdata);
         }
+    }
 
-        return 0;
+    return 0;
 
 err:
-        op_errno = (op_errno == -1) ? errno : op_errno;
-        DHT_STACK_UNWIND (setattr, frame, -1, op_errno, NULL, NULL, NULL);
+    op_errno = (op_errno == -1) ? errno : op_errno;
+    DHT_STACK_UNWIND(setattr, frame, -1, op_errno, NULL, NULL, NULL);
 
-        return 0;
+    return 0;
 }
 
-
 int
-dht_fsetattr (call_frame_t *frame, xlator_t *this, fd_t *fd, struct iatt *stbuf,
-              int32_t valid, dict_t *xdata)
+dht_fsetattr(call_frame_t *frame, xlator_t *this, fd_t *fd, struct iatt *stbuf,
+             int32_t valid, dict_t *xdata)
 {
-        xlator_t     *subvol = NULL;
-        dht_layout_t *layout = NULL;
-        dht_local_t  *local  = NULL;
-        int           op_errno = -1;
-        int           i = -1;
-        int           call_cnt = 0;
-
-
-        VALIDATE_OR_GOTO (frame, err);
-        VALIDATE_OR_GOTO (this, err);
-        VALIDATE_OR_GOTO (fd, err);
-
-        local = dht_local_init (frame, NULL, fd, GF_FOP_FSETATTR);
-        if (!local) {
-                op_errno = ENOMEM;
-                goto err;
-        }
-
-        layout = local->layout;
-        if (!layout) {
-                gf_msg_debug (this->name, 0,
-                              "no layout for fd=%p", fd);
-                op_errno = EINVAL;
-                goto err;
-        }
-
-        if (!layout_is_sane (layout)) {
-                gf_msg_debug (this->name, 0,
-                              "layout is not sane for fd=%p", fd);
-                op_errno = EINVAL;
-                goto err;
-        }
-
-        if (IA_ISREG (fd->inode->ia_type)) {
-                /* in the regular file _cbk(), we need to check for
-                   migration possibilities */
-                local->rebalance.stbuf = *stbuf;
-                local->rebalance.flags = valid;
-                local->call_cnt = 1;
-                subvol = local->cached_subvol;
-
-                STACK_WIND (frame, dht_file_setattr_cbk, subvol,
-                            subvol->fops->fsetattr,
-                            fd, stbuf, valid, xdata);
+    xlator_t *subvol = NULL;
+    dht_layout_t *layout = NULL;
+    dht_local_t *local = NULL;
+    int op_errno = -1;
+    int i = -1;
+    int call_cnt = 0;
+
+    VALIDATE_OR_GOTO(frame, err);
+    VALIDATE_OR_GOTO(this, err);
+    VALIDATE_OR_GOTO(fd, err);
+
+    local = dht_local_init(frame, NULL, fd, GF_FOP_FSETATTR);
+    if (!local) {
+        op_errno = ENOMEM;
+        goto err;
+    }
+
+    layout = local->layout;
+    if (!layout) {
+        gf_msg_debug(this->name, 0, "no layout for fd=%p", fd);
+        op_errno = EINVAL;
+        goto err;
+    }
+
+    if (!layout_is_sane(layout)) {
+        gf_msg_debug(this->name, 0, "layout is not sane for fd=%p", fd);
+        op_errno = EINVAL;
+        goto err;
+    }
+    if (xdata)
+        local->xattr_req = dict_ref(xdata);
+
+    if (IA_ISREG(fd->inode->ia_type)) {
+        /* in the regular file _cbk(), we need to check for
+           migration possibilities */
+        local->rebalance.stbuf = *stbuf;
+        local->rebalance.flags = valid;
+        local->call_cnt = 1;
+        subvol = local->cached_subvol;
 
-                return 0;
-        }
+        STACK_WIND_COOKIE(frame, dht_file_setattr_cbk, subvol, subvol,
+                          subvol->fops->fsetattr, fd, &local->rebalance.stbuf,
+                          local->rebalance.flags, local->xattr_req);
+        return 0;
+    }
 
-        local->call_cnt = call_cnt = layout->cnt;
+    local->call_cnt = call_cnt = layout->cnt;
 
-        for (i = 0; i < call_cnt; i++) {
-                STACK_WIND (frame, dht_setattr_cbk,
-                            layout->list[i].xlator,
-                            layout->list[i].xlator->fops->fsetattr,
-                            fd, stbuf, valid, xdata);
-        }
+    for (i = 0; i < call_cnt; i++) {
+        STACK_WIND_COOKIE(frame, dht_setattr_cbk, layout->list[i].xlator,
+                          layout->list[i].xlator,
+                          layout->list[i].xlator->fops->fsetattr, fd, stbuf,
+                          valid, xdata);
+    }
 
-        return 0;
+    return 0;
 
 err:
-        op_errno = (op_errno == -1) ? errno : op_errno;
-        DHT_STACK_UNWIND (fsetattr, frame, -1, op_errno, NULL, NULL, NULL);
+    op_errno = (op_errno == -1) ? errno : op_errno;
+    DHT_STACK_UNWIND(fsetattr, frame, -1, op_errno, NULL, NULL, NULL);
 
-        return 0;
+    return 0;
 }
diff --git a/xlators/cluster/dht/src/dht-layout.c b/xlators/cluster/dht/src/dht-layout.c
index 2ed15c5e43c..fda904c92c9 100644
--- a/xlators/cluster/dht/src/dht-layout.c
+++ b/xlators/cluster/dht/src/dht-layout.c
@@ -8,834 +8,801 @@
   cases as published by the Free Software Foundation.
 */
 
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
-
-
-#include "glusterfs.h"
-#include "xlator.h"
 #include "dht-common.h"
-#include "byte-order.h"
-#include "dht-messages.h"
+#include <glusterfs/byte-order.h>
 #include "unittest/unittest.h"
 
+#define layout_base_size (sizeof(dht_layout_t))
 
-#define layout_base_size (sizeof (dht_layout_t))
-
-#define layout_entry_size (sizeof ((dht_layout_t *)NULL)->list[0])
+#define layout_entry_size (sizeof((dht_layout_t *)NULL)->list[0])
 
 #define layout_size(cnt) (layout_base_size + (cnt * layout_entry_size))
 
 dht_layout_t *
-dht_layout_new (xlator_t *this, int cnt)
+dht_layout_new(xlator_t *this, int cnt)
 {
-        dht_layout_t *layout = NULL;
-        dht_conf_t   *conf = NULL;
+    dht_layout_t *layout = NULL;
+    dht_conf_t *conf = NULL;
 
-        REQUIRE(NULL != this);
-        REQUIRE(cnt >= 0);
+    REQUIRE(NULL != this);
+    REQUIRE(cnt >= 0);
 
-        conf = this->private;
+    conf = this->private;
 
-        layout = GF_CALLOC (1, layout_size (cnt),
-                            gf_dht_mt_dht_layout_t);
-        if (!layout) {
-                goto out;
-        }
+    layout = GF_CALLOC(1, layout_size(cnt), gf_dht_mt_dht_layout_t);
+    if (!layout) {
+        goto out;
+    }
 
-        layout->type = DHT_HASH_TYPE_DM;
-        layout->cnt = cnt;
+    layout->type = DHT_HASH_TYPE_DM;
+    layout->cnt = cnt;
 
-        if (conf) {
-                layout->spread_cnt = conf->dir_spread_cnt;
-                layout->gen = conf->gen;
-        }
+    if (conf) {
+        layout->spread_cnt = conf->dir_spread_cnt;
+        layout->gen = conf->gen;
+    }
 
-        layout->ref = 1;
+    GF_ATOMIC_INIT(layout->ref, 1);
 
-        ENSURE(NULL != layout);
-        ENSURE(layout->type == DHT_HASH_TYPE_DM);
-        ENSURE(layout->cnt == cnt);
-        ENSURE(layout->ref == 1);
+    ENSURE(NULL != layout);
+    ENSURE(layout->type == DHT_HASH_TYPE_DM);
+    ENSURE(layout->cnt == cnt);
+    ENSURE(GF_ATOMIC_GET(layout->ref) == 1);
 out:
-        return layout;
+    return layout;
 }
 
-
 dht_layout_t *
-dht_layout_get (xlator_t *this, inode_t *inode)
+dht_layout_get(xlator_t *this, inode_t *inode)
 {
-        dht_conf_t   *conf = NULL;
-        dht_layout_t *layout = NULL;
-        int           ret = 0;
-
-        conf = this->private;
-        if (!conf)
-                goto out;
-
-        LOCK (&conf->layout_lock);
-        {
-                ret = dht_inode_ctx_layout_get (inode, this, &layout);
-                if ((!ret) && layout) {
-                        layout->ref++;
-                }
-        }
-        UNLOCK (&conf->layout_lock);
-
-out:
-        return layout;
+    dht_layout_t *layout = NULL;
+    int ret = 0;
+
+    ret = dht_inode_ctx_layout_get(inode, this, &layout);
+    if ((!ret) && layout) {
+        GF_ATOMIC_INC(layout->ref);
+    }
+    return layout;
 }
 
-
 int
-dht_layout_set (xlator_t *this, inode_t *inode, dht_layout_t *layout)
+dht_layout_set(xlator_t *this, inode_t *inode, dht_layout_t *layout)
 {
-        dht_conf_t   *conf = NULL;
-        int           oldret = -1;
-        int           ret = 0;
-        dht_layout_t *old_layout;
-
-        conf = this->private;
-        if (!conf || !layout)
-                goto out;
-
-        LOCK (&conf->layout_lock);
-        {
-                oldret = dht_inode_ctx_layout_get (inode, this, &old_layout);
-                if (layout)
-                        layout->ref++;
-                dht_inode_ctx_layout_set (inode, this, layout);
-        }
-        UNLOCK (&conf->layout_lock);
-
-        if (!oldret) {
-                dht_layout_unref (this, old_layout);
-        }
+    dht_conf_t *conf = NULL;
+    int oldret = -1;
+    int ret = -1;
+    dht_layout_t *old_layout;
+
+    conf = this->private;
+    if (!conf || !layout)
+        goto out;
+
+    LOCK(&conf->layout_lock);
+    {
+        oldret = dht_inode_ctx_layout_get(inode, this, &old_layout);
+        if (layout)
+            GF_ATOMIC_INC(layout->ref);
+        ret = dht_inode_ctx_layout_set(inode, this, layout);
+    }
+    UNLOCK(&conf->layout_lock);
+
+    if (!oldret) {
+        dht_layout_unref(this, old_layout);
+    }
+    if (ret)
+        GF_ATOMIC_DEC(layout->ref);
 
 out:
-        return ret;
+    return ret;
 }
 
-
 void
-dht_layout_unref (xlator_t *this, dht_layout_t *layout)
+dht_layout_unref(xlator_t *this, dht_layout_t *layout)
 {
-        dht_conf_t  *conf = NULL;
-        int          ref = 0;
-
-        if (!layout || layout->preset || !this->private)
-                return;
+    int ref = 0;
 
-        conf = this->private;
+    if (!layout || layout->preset || !this->private)
+        return;
 
-        LOCK (&conf->layout_lock);
-        {
-                ref = --layout->ref;
-        }
-        UNLOCK (&conf->layout_lock);
+    ref = GF_ATOMIC_DEC(layout->ref);
 
-        if (!ref)
-                GF_FREE (layout);
+    if (!ref)
+        GF_FREE(layout);
 }
 
-
 dht_layout_t *
-dht_layout_ref (xlator_t *this, dht_layout_t *layout)
+dht_layout_ref(xlator_t *this, dht_layout_t *layout)
 {
-        dht_conf_t  *conf = NULL;
-
-        if (layout->preset || !this->private)
-                return layout;
+    if (layout->preset || !this->private)
+        return layout;
 
-        conf = this->private;
-        LOCK (&conf->layout_lock);
-        {
-                layout->ref++;
-        }
-        UNLOCK (&conf->layout_lock);
+    GF_ATOMIC_INC(layout->ref);
 
-        return layout;
+    return layout;
 }
 
-
 xlator_t *
-dht_layout_search (xlator_t *this, dht_layout_t *layout, const char *name)
+dht_layout_search(xlator_t *this, dht_layout_t *layout, const char *name)
 {
-        uint32_t   hash = 0;
-        xlator_t  *subvol = NULL;
-        int        i = 0;
-        int        ret = 0;
-
-        ret = dht_hash_compute (this, layout->type, name, &hash);
-        if (ret != 0) {
-                gf_log (this->name, GF_LOG_WARNING,
-                        "hash computation failed for type=%d name=%s",
-                        layout->type, name);
-                goto out;
-        }
-
-        for (i = 0; i < layout->cnt; i++) {
-                if (layout->list[i].start <= hash
-                    && layout->list[i].stop >= hash) {
-                        subvol = layout->list[i].xlator;
-                        break;
-                }
-        }
-
-        if (!subvol) {
-                gf_log (this->name, GF_LOG_WARNING,
-                        "no subvolume for hash (value) = %u", hash);
-        }
+    uint32_t hash = 0;
+    xlator_t *subvol = NULL;
+    int i = 0;
+    int ret = 0;
+
+    ret = dht_hash_compute(this, layout->type, name, &hash);
+    if (ret != 0) {
+        gf_smsg(this->name, GF_LOG_WARNING, 0, DHT_MSG_COMPUTE_HASH_FAILED,
+                "type=%d", layout->type, "name=%s", name, NULL);
+        goto out;
+    }
+
+    for (i = 0; i < layout->cnt; i++) {
+        if (layout->list[i].start <= hash && layout->list[i].stop >= hash) {
+            subvol = layout->list[i].xlator;
+            break;
+        }
+    }
+
+    if (!subvol) {
+        gf_smsg(this->name, GF_LOG_WARNING, 0, DHT_MSG_HASHED_SUBVOL_GET_FAILED,
+                "hash-value=0x%x", hash, NULL);
+    }
 
 out:
-        return subvol;
+    return subvol;
 }
 
-
 dht_layout_t *
-dht_layout_for_subvol (xlator_t *this, xlator_t *subvol)
+dht_layout_for_subvol(xlator_t *this, xlator_t *subvol)
 {
-        dht_conf_t   *conf = NULL;
-        dht_layout_t *layout = NULL;
-        int           i = 0;
-
-        conf = this->private;
-        if (!conf)
-                goto out;
-
-        for (i = 0; i < conf->subvolume_cnt; i++) {
-                if (conf->subvolumes[i] == subvol) {
-                        layout = conf->file_layouts[i];
-                        break;
-                }
+    dht_conf_t *conf = NULL;
+    dht_layout_t *layout = NULL;
+    int i = 0;
+
+    conf = this->private;
+    if (!conf)
+        goto out;
+
+    for (i = 0; i < conf->subvolume_cnt; i++) {
+        if (conf->subvolumes[i] == subvol) {
+            layout = conf->file_layouts[i];
+            break;
         }
+    }
 
 out:
-        return layout;
+    return layout;
 }
 
-
 int
-dht_layouts_init (xlator_t *this, dht_conf_t *conf)
+dht_layouts_init(xlator_t *this, dht_conf_t *conf)
 {
-        dht_layout_t *layout = NULL;
-        int           i = 0;
-        int           ret = -1;
-
-        if (!conf)
-                goto out;
-
-        conf->file_layouts = GF_CALLOC (conf->subvolume_cnt,
-                                        sizeof (dht_layout_t *),
-                                        gf_dht_mt_dht_layout_t);
-        if (!conf->file_layouts) {
-                goto out;
-        }
-
-        for (i = 0; i < conf->subvolume_cnt; i++) {
-                layout = dht_layout_new (this, 1);
+    dht_layout_t *layout = NULL;
+    int i = 0;
+    int ret = -1;
 
-                if (!layout) {
-                        goto out;
-                }
+    if (!conf)
+        goto out;
 
-                layout->preset = 1;
+    conf->file_layouts = GF_CALLOC(conf->subvolume_cnt, sizeof(dht_layout_t *),
+                                   gf_dht_mt_dht_layout_t);
+    if (!conf->file_layouts) {
+        goto out;
+    }
 
-                layout->list[0].xlator = conf->subvolumes[i];
+    for (i = 0; i < conf->subvolume_cnt; i++) {
+        layout = dht_layout_new(this, 1);
 
-                conf->file_layouts[i] = layout;
+        if (!layout) {
+            goto out;
         }
 
-        ret = 0;
+        layout->preset = 1;
+
+        layout->list[0].xlator = conf->subvolumes[i];
+
+        conf->file_layouts[i] = layout;
+    }
+
+    ret = 0;
 out:
-        return ret;
+    return ret;
 }
 
-
 int
-dht_disk_layout_extract (xlator_t *this, dht_layout_t *layout,
-                         int pos, int32_t **disk_layout_p)
+dht_disk_layout_extract(xlator_t *this, dht_layout_t *layout, int pos,
+                        int32_t **disk_layout_p)
 {
-        int      ret = -1;
-        int32_t *disk_layout = NULL;
+    int ret = -1;
+    int32_t *disk_layout = NULL;
 
-        disk_layout = GF_CALLOC (5, sizeof (int),
-                                 gf_dht_mt_int32_t);
-        if (!disk_layout) {
-                goto out;
-        }
+    disk_layout = GF_CALLOC(5, sizeof(int), gf_dht_mt_int32_t);
+    if (!disk_layout) {
+        goto out;
+    }
 
-        disk_layout[0] = hton32 (1);
-        disk_layout[1] = hton32 (layout->type);
-        disk_layout[2] = hton32 (layout->list[pos].start);
-        disk_layout[3] = hton32 (layout->list[pos].stop);
+    disk_layout[0] = hton32(layout->list[pos].commit_hash);
+    disk_layout[1] = hton32(layout->type);
+    disk_layout[2] = hton32(layout->list[pos].start);
+    disk_layout[3] = hton32(layout->list[pos].stop);
 
-        if (disk_layout_p)
-                *disk_layout_p = disk_layout;
-        else
-                GF_FREE (disk_layout);
+    if (disk_layout_p)
+        *disk_layout_p = disk_layout;
+    else
+        GF_FREE(disk_layout);
 
-        ret = 0;
+    ret = 0;
 
 out:
-        return ret;
+    return ret;
 }
 
-
 int
-dht_disk_layout_merge (xlator_t *this, dht_layout_t *layout,
-		       int pos, void *disk_layout_raw, int disk_layout_len)
+dht_disk_layout_extract_for_subvol(xlator_t *this, dht_layout_t *layout,
+                                   xlator_t *subvol, int32_t **disk_layout_p)
 {
-        int      cnt = 0;
-        int      type = 0;
-        int      start_off = 0;
-        int      stop_off = 0;
-        int      disk_layout[4];
-
-	if (!disk_layout_raw) {
-		gf_msg (this->name, GF_LOG_CRITICAL, 0,
-                        DHT_MSG_LAYOUT_MERGE_FAILED,
-                        "error no layout on disk for merge");
-		return -1;
-	}
-
-	GF_ASSERT (disk_layout_len == sizeof (disk_layout));
-
-        memcpy (disk_layout, disk_layout_raw, disk_layout_len);
-
-        cnt  = ntoh32 (disk_layout[0]);
-        if (cnt != 1) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        DHT_MSG_INVALID_DISK_LAYOUT,
-                        "Invalid disk layout: Invalid count %d", cnt);
-                return -1;
-        }
+    int i = 0;
+
+    for (i = 0; i < layout->cnt; i++) {
+        if (layout->list[i].xlator == subvol)
+            break;
+    }
+
+    if (i == layout->cnt)
+        return -1;
+
+    return dht_disk_layout_extract(this, layout, i, disk_layout_p);
+}
+
+static int
+dht_disk_layout_merge(xlator_t *this, dht_layout_t *layout, int pos,
+                      void *disk_layout_raw, int disk_layout_len)
+{
+    int type = 0;
+    int start_off = 0;
+    int stop_off = 0;
+    int commit_hash = 0;
+    int disk_layout[4];
+
+    if (!disk_layout_raw) {
+        gf_smsg(this->name, GF_LOG_CRITICAL, 0, DHT_MSG_LAYOUT_MERGE_FAILED,
+                NULL);
+        return -1;
+    }
 
-        type = ntoh32 (disk_layout[1]);
-	switch (type) {
+    GF_ASSERT(disk_layout_len == sizeof(disk_layout));
+
+    memcpy(disk_layout, disk_layout_raw, disk_layout_len);
+
+    type = ntoh32(disk_layout[1]);
+    switch (type) {
         case DHT_HASH_TYPE_DM_USER:
-                gf_msg_debug (this->name, 0, "found user-set layout");
-                layout->type = type;
-                /* Fall through. */
-	case DHT_HASH_TYPE_DM:
-		break;
+            gf_msg_debug(this->name, 0, "found user-set layout");
+            layout->type = type;
+            /* Fall through. */
+        case DHT_HASH_TYPE_DM:
+            break;
         default:
-		gf_msg (this->name, GF_LOG_CRITICAL, 0,
-                        DHT_MSG_INVALID_DISK_LAYOUT,
-			"Invalid disk layout: "
-                        "Catastrophic error layout with unknown type found %d",
-			disk_layout[1]);
-		return -1;
-	}
-
-        start_off = ntoh32 (disk_layout[2]);
-        stop_off  = ntoh32 (disk_layout[3]);
-
-        layout->list[pos].start = start_off;
-        layout->list[pos].stop  = stop_off;
-
-        gf_msg_trace (this->name, 0,
-                      "merged to layout: %u - %u (type %d) from %s",
-                      start_off, stop_off, type,
-                      layout->list[pos].xlator->name);
-
-        return 0;
-}
+            gf_smsg(this->name, GF_LOG_CRITICAL, 0, DHT_MSG_INVALID_DISK_LAYOUT,
+                    "layout=%d", disk_layout[1], NULL);
+            return -1;
+    }
+
+    commit_hash = ntoh32(disk_layout[0]);
+    start_off = ntoh32(disk_layout[2]);
+    stop_off = ntoh32(disk_layout[3]);
 
+    layout->list[pos].commit_hash = commit_hash;
+    layout->list[pos].start = start_off;
+    layout->list[pos].stop = stop_off;
+
+    gf_msg_trace(this->name, 0,
+                 "merged to layout: 0x%x - 0x%x (hash 0x%x, type %d) from %s",
+                 start_off, stop_off, commit_hash, type,
+                 layout->list[pos].xlator->name);
+
+    return 0;
+}
 
 int
-dht_layout_merge (xlator_t *this, dht_layout_t *layout, xlator_t *subvol,
-                  int op_ret, int op_errno, dict_t *xattr)
+dht_layout_merge(xlator_t *this, dht_layout_t *layout, xlator_t *subvol,
+                 int op_ret, int op_errno, dict_t *xattr)
 {
-        int         i     = 0;
-        int         ret   = -1;
-        int         err   = -1;
-        void       *disk_layout_raw = NULL;
-        int         disk_layout_len = 0;
-        dht_conf_t *conf = this->private;
-
-        if (op_ret != 0) {
-                err = op_errno;
-        }
+    int i = 0;
+    int ret = -1;
+    int err = -1;
+    void *disk_layout_raw = NULL;
+    int disk_layout_len = 0;
+    dht_conf_t *conf = this->private;
 
-        for (i = 0; i < layout->cnt; i++) {
-                if (layout->list[i].xlator == NULL) {
-                        layout->list[i].err    = err;
-                        layout->list[i].xlator = subvol;
-                        break;
-                }
-        }
+    if (op_ret != 0) {
+        err = op_errno;
+    }
 
-        if (op_ret != 0) {
-                ret = 0;
-                goto out;
-        }
+    if (!layout)
+        goto out;
 
-        if (xattr) {
-                /* during lookup and not mkdir */
-                ret = dict_get_ptr_and_len (xattr, conf->xattr_name,
-					    &disk_layout_raw, &disk_layout_len);
+    for (i = 0; i < layout->cnt; i++) {
+        if (layout->list[i].xlator == NULL) {
+            layout->list[i].err = err;
+            layout->list[i].xlator = subvol;
+            break;
         }
+    }
 
-        if (ret != 0) {
-                layout->list[i].err = 0;
-                gf_msg_trace (this->name, 0,
-                              "Missing disk layout on %s. err = %d",
-                              subvol->name, err);
-                ret = 0;
-                goto out;
-        }
+    if (op_ret != 0) {
+        ret = 0;
+        goto out;
+    }
 
-        ret = dht_disk_layout_merge (this, layout, i, disk_layout_raw,
-				     disk_layout_len);
-        if (ret != 0) {
-                gf_msg (this->name, GF_LOG_WARNING, 0,
-                        DHT_MSG_LAYOUT_MERGE_FAILED,
-                        "layout merge from subvolume %s failed",
-                        subvol->name);
-                goto out;
-        }
+    if (xattr) {
+        /* during lookup and not mkdir */
+        ret = dict_get_ptr_and_len(xattr, conf->xattr_name, &disk_layout_raw,
+                                   &disk_layout_len);
+    }
+
+    if (ret != 0) {
         layout->list[i].err = 0;
+        gf_msg_trace(this->name, 0, "Missing disk layout on %s. err = %d",
+                     subvol->name, err);
+        ret = 0;
+        goto out;
+    }
+
+    ret = dht_disk_layout_merge(this, layout, i, disk_layout_raw,
+                                disk_layout_len);
+    if (ret != 0) {
+        gf_smsg(this->name, GF_LOG_WARNING, 0, DHT_MSG_LAYOUT_MERGE_FAILED,
+                "subvolume=%s", subvol->name, NULL);
+        goto out;
+    }
+
+    if (layout->commit_hash == 0) {
+        layout->commit_hash = layout->list[i].commit_hash;
+    } else if (layout->commit_hash != layout->list[i].commit_hash) {
+        layout->commit_hash = DHT_LAYOUT_HASH_INVALID;
+    }
+
+    layout->list[i].err = 0;
 
 out:
-        return ret;
+    return ret;
 }
 
-
 void
-dht_layout_entry_swap (dht_layout_t *layout, int i, int j)
+dht_layout_entry_swap(dht_layout_t *layout, int i, int j)
 {
-        uint32_t  start_swap = 0;
-        uint32_t  stop_swap = 0;
-        xlator_t *xlator_swap = 0;
-        int       err_swap = 0;
-
-        start_swap  = layout->list[i].start;
-        stop_swap   = layout->list[i].stop;
-        xlator_swap = layout->list[i].xlator;
-        err_swap    = layout->list[i].err;
-
-        layout->list[i].start  = layout->list[j].start;
-        layout->list[i].stop   = layout->list[j].stop;
-        layout->list[i].xlator = layout->list[j].xlator;
-        layout->list[i].err    = layout->list[j].err;
-
-        layout->list[j].start  = start_swap;
-        layout->list[j].stop   = stop_swap;
-        layout->list[j].xlator = xlator_swap;
-        layout->list[j].err    = err_swap;
+    uint32_t start_swap = 0;
+    uint32_t stop_swap = 0;
+    uint32_t commit_hash_swap = 0;
+    xlator_t *xlator_swap = 0;
+    int err_swap = 0;
+
+    start_swap = layout->list[i].start;
+    stop_swap = layout->list[i].stop;
+    xlator_swap = layout->list[i].xlator;
+    err_swap = layout->list[i].err;
+    commit_hash_swap = layout->list[i].commit_hash;
+
+    layout->list[i].start = layout->list[j].start;
+    layout->list[i].stop = layout->list[j].stop;
+    layout->list[i].xlator = layout->list[j].xlator;
+    layout->list[i].err = layout->list[j].err;
+    layout->list[i].commit_hash = layout->list[j].commit_hash;
+
+    layout->list[j].start = start_swap;
+    layout->list[j].stop = stop_swap;
+    layout->list[j].xlator = xlator_swap;
+    layout->list[j].err = err_swap;
+    layout->list[j].commit_hash = commit_hash_swap;
 }
 
 void
-dht_layout_range_swap (dht_layout_t *layout, int i, int j)
+dht_layout_range_swap(dht_layout_t *layout, int i, int j)
 {
-        uint32_t  start_swap = 0;
-        uint32_t  stop_swap = 0;
+    uint32_t start_swap = 0;
+    uint32_t stop_swap = 0;
 
-        start_swap  = layout->list[i].start;
-        stop_swap   = layout->list[i].stop;
+    start_swap = layout->list[i].start;
+    stop_swap = layout->list[i].stop;
 
-        layout->list[i].start  = layout->list[j].start;
-        layout->list[i].stop   = layout->list[j].stop;
+    layout->list[i].start = layout->list[j].start;
+    layout->list[i].stop = layout->list[j].stop;
 
-        layout->list[j].start  = start_swap;
-        layout->list[j].stop   = stop_swap;
+    layout->list[j].start = start_swap;
+    layout->list[j].stop = stop_swap;
 }
-
-int64_t
-dht_layout_entry_cmp_volname (dht_layout_t *layout, int i, int j)
+static int64_t
+dht_layout_entry_cmp_volname(dht_layout_t *layout, int i, int j)
 {
-        return (strcmp (layout->list[i].xlator->name,
-                        layout->list[j].xlator->name));
+    return (strcmp(layout->list[i].xlator->name, layout->list[j].xlator->name));
 }
 
 gf_boolean_t
-dht_is_subvol_in_layout (dht_layout_t *layout, xlator_t *xlator)
+dht_is_subvol_in_layout(dht_layout_t *layout, xlator_t *xlator)
 {
-        int i = 0;
-
-        for (i = 0; i < layout->cnt; i++) {
-                /* Check if xlator is already part of layout, and layout is
-                 * non-zero. */
-                if (!strcmp (layout->list[i].xlator->name, xlator->name)) {
-                        if (layout->list[i].start != layout->list[i].stop)
-                                return _gf_true;
-                        break;
-                }
-        }
-        return _gf_false;
+    int i = 0;
+
+    for (i = 0; i < layout->cnt; i++) {
+        /* Check if xlator is already part of layout, and layout is
+         * non-zero. */
+        if (!strcmp(layout->list[i].xlator->name, xlator->name)) {
+            if (layout->list[i].start != layout->list[i].stop)
+                return _gf_true;
+            break;
+        }
+    }
+    return _gf_false;
 }
 
-int64_t
-dht_layout_entry_cmp (dht_layout_t *layout, int i, int j)
+static int64_t
+dht_layout_entry_cmp(dht_layout_t *layout, int i, int j)
 {
-        int64_t diff = 0;
+    int64_t diff = 0;
 
-        /* swap zero'ed out layouts to front, if needed */
-        if (!layout->list[j].start && !layout->list[j].stop) {
-                diff = (int64_t) layout->list[i].stop
-                       - (int64_t) layout->list[j].stop;
-                       goto out;
-        }
-        diff = (int64_t) layout->list[i].start
-                - (int64_t) layout->list[j].start;
+    /* swap zero'ed out layouts to front, if needed */
+    if (!layout->list[j].start && !layout->list[j].stop) {
+        diff = (int64_t)layout->list[i].stop - (int64_t)layout->list[j].stop;
+        goto out;
+    }
+    diff = (int64_t)layout->list[i].start - (int64_t)layout->list[j].start;
 
 out:
-        return diff;
+    return diff;
 }
 
-
 int
-dht_layout_sort (dht_layout_t *layout)
+dht_layout_sort(dht_layout_t *layout)
 {
-        int       i = 0;
-        int       j = 0;
-        int64_t   ret = 0;
+    int i = 0;
+    int j = 0;
+    int64_t ret = 0;
 
-        /* TODO: O(n^2) -- bad bad */
+    /* TODO: O(n^2) -- bad bad */
 
-        for (i = 0; i < layout->cnt - 1; i++) {
-                for (j = i + 1; j < layout->cnt; j++) {
-                        ret = dht_layout_entry_cmp (layout, i, j);
-                        if (ret > 0)
-                                dht_layout_entry_swap (layout, i, j);
-                }
+    for (i = 0; i < layout->cnt - 1; i++) {
+        for (j = i + 1; j < layout->cnt; j++) {
+            ret = dht_layout_entry_cmp(layout, i, j);
+            if (ret > 0)
+                dht_layout_entry_swap(layout, i, j);
         }
+    }
 
-        return 0;
+    return 0;
 }
 
-int
-dht_layout_sort_volname (dht_layout_t *layout)
+void
+dht_layout_sort_volname(dht_layout_t *layout)
 {
-        int       i = 0;
-        int       j = 0;
-        int64_t   ret = 0;
+    int i = 0;
+    int j = 0;
+    int64_t ret = 0;
 
-        /* TODO: O(n^2) -- bad bad */
+    /* TODO: O(n^2) -- bad bad */
 
-        for (i = 0; i < layout->cnt - 1; i++) {
-                for (j = i + 1; j < layout->cnt; j++) {
-                        ret = dht_layout_entry_cmp_volname (layout, i, j);
-                        if (ret > 0)
-                                dht_layout_entry_swap (layout, i, j);
-                }
+    for (i = 0; i < layout->cnt - 1; i++) {
+        for (j = i + 1; j < layout->cnt; j++) {
+            ret = dht_layout_entry_cmp_volname(layout, i, j);
+            if (ret > 0)
+                dht_layout_entry_swap(layout, i, j);
         }
-
-        return 0;
+    }
 }
 
-
-int
-dht_layout_anomalies (xlator_t *this, loc_t *loc, dht_layout_t *layout,
-                      uint32_t *holes_p, uint32_t *overlaps_p,
-                      uint32_t *missing_p, uint32_t *down_p, uint32_t *misc_p,
-                      uint32_t *no_space_p)
+void
+dht_layout_anomalies(xlator_t *this, loc_t *loc, dht_layout_t *layout,
+                     uint32_t *holes_p, uint32_t *overlaps_p,
+                     uint32_t *missing_p, uint32_t *down_p, uint32_t *misc_p,
+                     uint32_t *no_space_p)
 {
-        uint32_t    overlaps = 0;
-        uint32_t    missing  = 0;
-        uint32_t    down     = 0;
-        uint32_t    misc     = 0;
-        uint32_t    hole_cnt = 0;
-        uint32_t    overlap_cnt = 0;
-        int         i = 0;
-        int         ret = 0;
-        uint32_t    prev_stop = 0;
-        uint32_t    last_stop = 0;
-        char        is_virgin = 1;
-        uint32_t    no_space  = 0;
-
-        /* This function scans through the layout spread of a directory to
-           check if there are any anomalies. Prior to calling this function
-           the layout entries should be sorted in the ascending order.
-
-           If the layout entry has err != 0
-                then increment the corresponding anomaly.
-           else
-                if (start of the current layout entry > stop + 1 of previous
-                   non erroneous layout entry)
-                        then it indicates a hole in the layout
-                if (start of the current layout entry < stop + 1 of previous
-                    non erroneous layout entry)
-                         then it indicates an overlap in the layout
-        */
-        last_stop = layout->list[0].start - 1;
-        prev_stop = last_stop;
-
-        for (i = 0; i < layout->cnt; i++) {
-                switch (layout->list[i].err) {
-                case -1:
-                case ENOENT:
-                case ESTALE:
-                        missing++;
-                        continue;
-                case ENOTCONN:
-                        down++;
-                        continue;
-                case ENOSPC:
-                        no_space++;
-                        continue;
-                case 0:
-                        /* if err == 0 and start == stop, then it is a non misc++;
-                         * participating subvolume(spread-cnt). Then, do not
-                         * check for anomalies. If start != stop, then treat it
-                         * as misc err */
-                        if (layout->list[i].start == layout->list[i].stop) {
-                                continue;
-                        }
-                        break;
-                default:
-                        misc++;
-                        continue;
-                 }
-
-                is_virgin = 0;
-
-                if ((prev_stop + 1) < layout->list[i].start) {
-                        hole_cnt++;
+    uint32_t overlaps = 0;
+    uint32_t missing = 0;
+    uint32_t down = 0;
+    uint32_t misc = 0;
+    uint32_t hole_cnt = 0;
+    uint32_t overlap_cnt = 0;
+    int i = 0;
+    uint32_t prev_stop = 0;
+    uint32_t last_stop = 0;
+    char is_virgin = 1;
+    uint32_t no_space = 0;
+
+    /* This function scans through the layout spread of a directory to
+       check if there are any anomalies. Prior to calling this function
+       the layout entries should be sorted in the ascending order.
+
+       If the layout entry has err != 0
+            then increment the corresponding anomaly.
+       else
+            if (start of the current layout entry > stop + 1 of previous
+               non erroneous layout entry)
+                    then it indicates a hole in the layout
+            if (start of the current layout entry < stop + 1 of previous
+                non erroneous layout entry)
+                     then it indicates an overlap in the layout
+    */
+    last_stop = layout->list[0].start - 1;
+    prev_stop = last_stop;
+
+    for (i = 0; i < layout->cnt; i++) {
+        switch (layout->list[i].err) {
+            case -1:
+            case ENOENT:
+            case ESTALE:
+                missing++;
+                continue;
+            case ENOTCONN:
+                down++;
+                continue;
+            case ENOSPC:
+                no_space++;
+                continue;
+            case 0:
+                /* if err == 0 and start == stop, then it is a non misc++;
+                 * participating subvolume(spread-cnt). Then, do not
+                 * check for anomalies. If start != stop, then treat it
+                 * as misc err */
+                if (layout->list[i].start == layout->list[i].stop) {
+                    continue;
                 }
+                break;
+            default:
+                misc++;
+                continue;
+        }
 
-                if ((prev_stop + 1) > layout->list[i].start) {
-                        overlap_cnt++;
-                        overlaps += ((prev_stop + 1) - layout->list[i].start);
-                }
-                prev_stop = layout->list[i].stop;
+        is_virgin = 0;
+
+        if ((prev_stop + 1) < layout->list[i].start) {
+            hole_cnt++;
         }
 
-        if ((last_stop - prev_stop) || is_virgin)
-                hole_cnt++;
+        if ((prev_stop + 1) > layout->list[i].start) {
+            overlap_cnt++;
+            overlaps += ((prev_stop + 1) - layout->list[i].start);
+        }
+        prev_stop = layout->list[i].stop;
+    }
 
-        if (holes_p)
-                *holes_p = hole_cnt;
+    if ((last_stop - prev_stop) || is_virgin)
+        hole_cnt++;
 
-        if (overlaps_p)
-                *overlaps_p = overlap_cnt;
+    if (holes_p)
+        *holes_p = hole_cnt;
 
-        if (missing_p)
-                *missing_p = missing;
+    if (overlaps_p)
+        *overlaps_p = overlap_cnt;
 
-        if (down_p)
-                *down_p = down;
+    if (missing_p)
+        *missing_p = missing;
 
-        if (misc_p)
-                *misc_p = misc;
+    if (down_p)
+        *down_p = down;
 
-        if (no_space_p)
-                *no_space_p = no_space;
+    if (misc_p)
+        *misc_p = misc;
 
-        return ret;
+    if (no_space_p)
+        *no_space_p = no_space;
 }
 
-
 int
-dht_layout_missing_dirs (dht_layout_t *layout)
+dht_layout_missing_dirs(dht_layout_t *layout)
 {
-        int i = 0, missing = 0;
+    int i = 0, missing = 0;
 
-        if (layout == NULL)
-                goto out;
+    if (layout == NULL)
+        goto out;
 
-        for (i = 0; i < layout->cnt; i++) {
-                if ((layout->list[i].err == ENOENT)
-                    || ((layout->list[i].err == -1)
-                        && (layout->list[i].start == 0)
-                        && (layout->list[i].stop == 0))) {
-                        missing++;
-                }
+    for (i = 0; i < layout->cnt; i++) {
+        if ((layout->list[i].err == ENOENT) ||
+            ((layout->list[i].err == -1) && (layout->list[i].start == 0) &&
+             (layout->list[i].stop == 0))) {
+            missing++;
         }
+    }
 
 out:
-        return missing;
+    return missing;
 }
 
-
 int
-dht_layout_normalize (xlator_t *this, loc_t *loc, dht_layout_t *layout)
+dht_layout_normalize(xlator_t *this, loc_t *loc, dht_layout_t *layout)
 {
-        int          ret   = 0;
-        uint32_t     holes = 0;
-        uint32_t     overlaps = 0;
-        uint32_t     missing = 0;
-        uint32_t     down = 0;
-        uint32_t     misc = 0, missing_dirs = 0;
-        char         gfid[GF_UUID_BUF_SIZE] = {0};
-
-        ret = dht_layout_sort (layout);
-        if (ret == -1) {
-                gf_log (this->name, GF_LOG_WARNING,
-                        "sort failed?! how the ....");
-                goto out;
-        }
-
-        gf_uuid_unparse(loc->gfid, gfid);
-
-        ret = dht_layout_anomalies (this, loc, layout,
-                                    &holes, &overlaps,
-                                    &missing, &down, &misc, NULL);
-        if (ret == -1) {
-                gf_log (this->name, GF_LOG_WARNING,
-                        "Error finding anomalies in %s, gfid = %s",
-                        loc->path, gfid);
-                goto out;
-        }
-
-        if (holes || overlaps) {
-                if (missing == layout->cnt) {
-                        gf_msg_debug (this->name, 0,
-                                      "Directory %s looked up first time"
-                                      " gfid = %s", loc->path, gfid);
-                } else {
-                        gf_log (this->name, GF_LOG_INFO,
-                                "Found anomalies in %s (gfid = %s). "
-                                "Holes=%d overlaps=%d",
-                                loc->path, gfid, holes, overlaps );
-                }
-                ret = -1;
-        }
-
-        if (ret >= 0) {
-                missing_dirs = dht_layout_missing_dirs (layout);
-                /* TODO During DHT selfheal rewrite (almost) find a better place
-                 * to detect this - probably in dht_layout_anomalies()
-                 */
-                if (missing_dirs > 0)
-                        ret += missing_dirs;
-        }
+    int ret = 0;
+    uint32_t holes = 0;
+    uint32_t overlaps = 0;
+    uint32_t missing = 0;
+    uint32_t down = 0;
+    uint32_t misc = 0, missing_dirs = 0;
+    char gfid[GF_UUID_BUF_SIZE] = {0};
+
+    ret = dht_layout_sort(layout);
+    if (ret == -1) {
+        gf_smsg(this->name, GF_LOG_WARNING, 0, DHT_MSG_LAYOUT_SORT_FAILED,
+                NULL);
+        goto out;
+    }
+
+    gf_uuid_unparse(loc->gfid, gfid);
+
+    dht_layout_anomalies(this, loc, layout, &holes, &overlaps, &missing, &down,
+                         &misc, NULL);
+
+    if (holes || overlaps) {
+        if (missing == layout->cnt) {
+            gf_msg_debug(this->name, 0,
+                         "Directory %s looked up first time"
+                         " gfid = %s",
+                         loc->path, gfid);
+        } else {
+            gf_smsg(this->name, GF_LOG_INFO, 0, DHT_MSG_ANOMALIES_INFO,
+                    "path=%s", loc->path, "gfid=%s", gfid, "holes=%d", holes,
+                    "overlaps=%d", overlaps, NULL);
+        }
+        ret = -1;
+    }
+
+    if (ret >= 0) {
+        missing_dirs = dht_layout_missing_dirs(layout);
+        /* TODO During DHT selfheal rewrite (almost) find a better place
+         * to detect this - probably in dht_layout_anomalies()
+         */
+        if (missing_dirs > 0)
+            ret += missing_dirs;
+    }
 
 out:
-        return ret;
+    return ret;
 }
 
 int
-dht_dir_has_layout (dict_t *xattr, char *name)
+dht_dir_has_layout(dict_t *xattr, char *name)
 {
+    void *disk_layout_raw = NULL;
 
-        void     *disk_layout_raw = NULL;
-
-        return dict_get_ptr (xattr, name, &disk_layout_raw);
+    return dict_get_ptr(xattr, name, &disk_layout_raw);
 }
 
 int
-dht_layout_dir_mismatch (xlator_t *this, dht_layout_t *layout, xlator_t *subvol,
-                         loc_t *loc, dict_t *xattr)
+dht_layout_dir_mismatch(xlator_t *this, dht_layout_t *layout, xlator_t *subvol,
+                        loc_t *loc, dict_t *xattr)
 {
-        int         idx = 0;
-        int         pos = -1;
-        int         ret = 0;
-        int         err = 0;
-        int         dict_ret = 0;
-        int32_t     disk_layout[4];
-        void       *disk_layout_raw = NULL;
-        int32_t     count = -1;
-        uint32_t    start_off = -1;
-        uint32_t    stop_off = -1;
-        dht_conf_t *conf = this->private;
-        char        gfid[GF_UUID_BUF_SIZE] = {0};
-
-        if(loc && loc->inode)
-                gf_uuid_unparse(loc->inode->gfid, gfid);
-
-        for (idx = 0; idx < layout->cnt; idx++) {
-                if (layout->list[idx].xlator == subvol) {
-                        pos = idx;
-                        break;
-                }
-        }
-
-        if (pos == -1) {
-                gf_msg_debug (this->name, 0,
-                              "%s - no layout info for subvolume %s",
-                              loc->path, subvol->name);
-                ret = 1;
-                goto out;
-        }
+    int idx = 0;
+    int pos = -1;
+    int ret = 0;
+    int err = 0;
+    int dict_ret = 0;
+    int32_t disk_layout[4];
+    void *disk_layout_raw = NULL;
+    uint32_t start_off = -1;
+    uint32_t stop_off = -1;
+    uint32_t commit_hash = -1;
+    dht_conf_t *conf = this->private;
+    char gfid[GF_UUID_BUF_SIZE] = {0};
+
+    if (loc && loc->inode)
+        gf_uuid_unparse(loc->inode->gfid, gfid);
+
+    for (idx = 0; idx < layout->cnt; idx++) {
+        if (layout->list[idx].xlator == subvol) {
+            pos = idx;
+            break;
+        }
+    }
+
+    if (pos == -1) {
+        if (loc) {
+            gf_msg_debug(this->name, 0, "%s - no layout info for subvolume %s",
+                         loc ? loc->path : "path not found", subvol->name);
+        }
+        ret = 1;
+        goto out;
+    }
+
+    err = layout->list[pos].err;
+
+    if (!xattr) {
+        if (err == 0) {
+            if (loc) {
+                gf_smsg(this->name, GF_LOG_INFO, 0, DHT_MSG_XATTR_DICT_NULL,
+                        "path=%s", loc->path, NULL);
+            } else {
+                gf_smsg(this->name, GF_LOG_INFO, 0, DHT_MSG_XATTR_DICT_NULL,
+                        "path not found", NULL);
+            }
+            ret = -1;
+        }
+        goto out;
+    }
+
+    dict_ret = dict_get_ptr(xattr, conf->xattr_name, &disk_layout_raw);
+
+    if (dict_ret < 0) {
+        if (err == 0 && layout->list[pos].stop) {
+            if (loc) {
+                gf_smsg(this->name, GF_LOG_INFO, 0, DHT_MSG_DISK_LAYOUT_MISSING,
+                        "path=%s", loc->path, "gfid=%s", gfid, NULL);
+            } else {
+                gf_smsg(this->name, GF_LOG_INFO, 0, DHT_MSG_DISK_LAYOUT_MISSING,
+                        "path not found"
+                        "gfid=%s",
+                        gfid, NULL);
+            }
+            ret = -1;
+        }
+        goto out;
+    }
+
+    memcpy(disk_layout, disk_layout_raw, sizeof(disk_layout));
+
+    start_off = ntoh32(disk_layout[2]);
+    stop_off = ntoh32(disk_layout[3]);
+    commit_hash = ntoh32(disk_layout[0]);
+
+    if ((layout->list[pos].start != start_off) ||
+        (layout->list[pos].stop != stop_off) ||
+        (layout->list[pos].commit_hash != commit_hash)) {
+        gf_smsg(this->name, GF_LOG_INFO, 0, DHT_MSG_LAYOUT_INFO, "subvol=%s",
+                layout->list[pos].xlator->name, "inode-layout:start=0x%x",
+                layout->list[pos].start, "inode-layout:stop=0x%x",
+                layout->list[pos].stop, "layout-commit-hash=0x%x; ",
+                layout->list[pos].commit_hash, "disk-layout:start-off=0x%x",
+                start_off, "disk-layout:top-off=0x%x", stop_off,
+                "commit-hash=0x%x", commit_hash, NULL);
+        ret = 1;
+    } else {
+        ret = 0;
+    }
+out:
+    return ret;
+}
 
-        err = layout->list[pos].err;
+int
+dht_layout_preset(xlator_t *this, xlator_t *subvol, inode_t *inode)
+{
+    dht_layout_t *layout = NULL;
+    int ret = -1;
+    dht_conf_t *conf = NULL;
 
-        if (!xattr) {
-                if (err == 0) {
-                        gf_log (this->name, GF_LOG_INFO,
-                                "%s: xattr dictionary is NULL",
-                                loc->path);
-                        ret = -1;
-                }
-                goto out;
-        }
+    conf = this->private;
+    if (!conf)
+        goto out;
 
-        dict_ret = dict_get_ptr (xattr, conf->xattr_name,
-                                 &disk_layout_raw);
+    layout = dht_layout_for_subvol(this, subvol);
+    if (!layout) {
+        gf_smsg(this->name, GF_LOG_INFO, 0, DHT_MSG_SUBVOL_NO_LAYOUT_INFO,
+                "subvolume=%s", subvol ? subvol->name : "<nil>", NULL);
+        ret = -1;
+        goto out;
+    }
 
-        if (dict_ret < 0) {
-                if (err == 0 && layout->list[pos].stop) {
-                        gf_log (this->name, GF_LOG_INFO,
-                                "%s: Disk layout missing, gfid = %s",
-                                loc->path, gfid);
-                        ret = -1;
-                }
-                goto out;
-        }
+    gf_msg_debug(this->name, 0, "file = %s, subvol = %s",
+                 uuid_utoa(inode->gfid), subvol ? subvol->name : "<nil>");
 
-        memcpy (disk_layout, disk_layout_raw, sizeof (disk_layout));
+    LOCK(&conf->layout_lock);
+    {
+        dht_inode_ctx_layout_set(inode, this, layout);
+    }
 
-        count  = ntoh32 (disk_layout[0]);
-        if (count != 1) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        DHT_MSG_INVALID_DISK_LAYOUT,
-                        "Invalid disk layout: invalid count %d,"
-                        "path = %s, gfid = %s ", count, loc->path, gfid);
-                ret = -1;
-                goto out;
-        }
+    UNLOCK(&conf->layout_lock);
 
-        start_off = ntoh32 (disk_layout[2]);
-        stop_off  = ntoh32 (disk_layout[3]);
-
-        if ((layout->list[pos].start != start_off)
-            || (layout->list[pos].stop != stop_off)) {
-                gf_log (this->name, GF_LOG_INFO,
-                        "subvol: %s; inode layout - %"PRIu32" - %"PRIu32"; "
-                        "disk layout - %"PRIu32" - %"PRIu32,
-                        layout->list[pos].xlator->name,
-                        layout->list[pos].start, layout->list[pos].stop,
-                        start_off, stop_off);
-                ret = 1;
-        } else {
-                ret = 0;
-        }
+    ret = 0;
 out:
-        return ret;
+    return ret;
 }
 
-
 int
-dht_layout_preset (xlator_t *this, xlator_t *subvol, inode_t *inode)
+dht_layout_index_for_subvol(dht_layout_t *layout, xlator_t *subvol)
 {
-        dht_layout_t *layout = NULL;
-        int           ret = -1;
-        dht_conf_t   *conf = NULL;
-
-        conf = this->private;
-        if (!conf)
-                goto out;
-
-        layout = dht_layout_for_subvol (this, subvol);
-        if (!layout) {
-                gf_log (this->name, GF_LOG_INFO,
-                        "no pre-set layout for subvolume %s",
-                        subvol ? subvol->name : "<nil>");
-                ret = -1;
-                goto out;
-        }
+    int i = 0, ret = -1;
 
-        LOCK (&conf->layout_lock);
-        {
-                dht_inode_ctx_layout_set (inode, this, layout);
+    for (i = 0; i < layout->cnt; i++) {
+        if (layout->list[i].xlator == subvol) {
+            ret = i;
+            break;
         }
-        UNLOCK (&conf->layout_lock);
+    }
 
-        ret = 0;
-out:
-        return ret;
+    return ret;
 }
diff --git a/xlators/cluster/dht/src/dht-linkfile.c b/xlators/cluster/dht/src/dht-linkfile.c
index 14df3187097..89ec6cca56e 100644
--- a/xlators/cluster/dht/src/dht-linkfile.c
+++ b/xlators/cluster/dht/src/dht-linkfile.c
@@ -8,348 +8,321 @@
   cases as published by the Free Software Foundation.
 */
 
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
-
-
-#include "glusterfs.h"
-#include "xlator.h"
-#include "compat.h"
+#include <glusterfs/compat.h>
 #include "dht-common.h"
-#include "dht-messages.h"
 
-int
-dht_linkfile_lookup_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                         int op_ret, int op_errno,
-                         inode_t *inode, struct iatt *stbuf, dict_t *xattr,
-                         struct iatt *postparent)
+static int
+dht_linkfile_lookup_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                        int op_ret, int op_errno, inode_t *inode,
+                        struct iatt *stbuf, dict_t *xattr,
+                        struct iatt *postparent)
 {
-        char          is_linkfile   = 0;
-        dht_conf_t   *conf          = NULL;
-        dht_local_t  *local         = NULL;
-        call_frame_t *prev          = NULL;
-        char         gfid[GF_UUID_BUF_SIZE] = {0};
-
-        local = frame->local;
-        prev = cookie;
-        conf = this->private;
-
-        if (op_ret)
-                goto out;
-
-        gf_uuid_unparse(local->loc.gfid, gfid);
-
-        is_linkfile = check_is_linkfile (inode, stbuf, xattr,
-                                         conf->link_xattr_name);
-        if (!is_linkfile)
-                gf_log (this->name, GF_LOG_WARNING,
-                        "got non-linkfile %s:%s, gfid = %s",
-                        prev->this->name, local->loc.path, gfid);
+    char is_linkfile = 0;
+    dht_conf_t *conf = NULL;
+    dht_local_t *local = NULL;
+    xlator_t *prev = NULL;
+    char gfid[GF_UUID_BUF_SIZE] = {0};
+
+    local = frame->local;
+    prev = cookie;
+    conf = this->private;
+
+    if (op_ret)
+        goto out;
+
+    gf_uuid_unparse(local->loc.gfid, gfid);
+
+    is_linkfile = check_is_linkfile(inode, stbuf, xattr, conf->link_xattr_name);
+    if (!is_linkfile)
+        gf_smsg(this->name, GF_LOG_WARNING, 0, DHT_MSG_NOT_LINK_FILE_ERROR,
+                "name=%s", prev->name, "path=%s", local->loc.path, "gfid=%s",
+                gfid, NULL);
 out:
-        local->linkfile.linkfile_cbk (frame, cookie, this, op_ret, op_errno,
-                                      inode, stbuf, postparent, postparent,
-                                      xattr);
-        return 0;
+    local->linkfile.linkfile_cbk(frame, cookie, this, op_ret, op_errno, inode,
+                                 stbuf, postparent, postparent, xattr);
+    return 0;
 }
 
-#define is_equal(a, b) ((a) == (b))
-int
-dht_linkfile_create_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                         int op_ret, int op_errno, inode_t *inode,
-                         struct iatt *stbuf, struct iatt *preparent,
-                         struct iatt *postparent, dict_t *xdata)
+static int
+dht_linkfile_create_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                        int op_ret, int op_errno, inode_t *inode,
+                        struct iatt *stbuf, struct iatt *preparent,
+                        struct iatt *postparent, dict_t *xdata)
 {
-        dht_local_t  *local = NULL;
-        xlator_t     *subvol = NULL;
-        call_frame_t *prev = NULL;
-        dict_t       *xattrs = NULL;
-        dht_conf_t   *conf = NULL;
-        int           ret = -1;
-
-        local = frame->local;
-
-        if (!op_ret)
-                local->linked = _gf_true;
-
-        FRAME_SU_UNDO (frame, dht_local_t);
-
-        if (op_ret && (op_errno == EEXIST)) {
-                conf = this->private;
-                prev = cookie;
-                subvol = prev->this;
-                if (!subvol)
-                        goto out;
-                xattrs = dict_new ();
-                if (!xattrs)
-                        goto out;
-                ret = dict_set_uint32 (xattrs, conf->link_xattr_name, 256);
-                if (ret) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                DHT_MSG_DICT_SET_FAILED,
-                                "Failed to set dictionary value. key : %s",
-                                conf->link_xattr_name);
-                        goto out;
-                }
-
-                STACK_WIND (frame, dht_linkfile_lookup_cbk, subvol,
-                            subvol->fops->lookup, &local->loc, xattrs);
-                if (xattrs)
-                        dict_unref (xattrs);
-                return 0;
+    dht_local_t *local = NULL;
+    xlator_t *subvol = NULL;
+    dict_t *xattrs = NULL;
+    dht_conf_t *conf = NULL;
+    int ret = -1;
+
+    local = frame->local;
+
+    if (!op_ret)
+        local->linked = _gf_true;
+
+    FRAME_SU_UNDO(frame, dht_local_t);
+
+    if (op_ret && (op_errno == EEXIST)) {
+        conf = this->private;
+        subvol = cookie;
+        if (!subvol)
+            goto out;
+        xattrs = dict_new();
+        if (!xattrs)
+            goto out;
+        ret = dict_set_uint32(xattrs, conf->link_xattr_name, 256);
+        if (ret) {
+            gf_smsg(this->name, GF_LOG_ERROR, 0, DHT_MSG_DICT_SET_FAILED,
+                    "mame=%s", conf->link_xattr_name, NULL);
+            goto out;
         }
-out:
-        local->linkfile.linkfile_cbk (frame, cookie, this, op_ret, op_errno,
-                                      inode, stbuf, preparent, postparent,
-                                      xdata);
+
+        STACK_WIND_COOKIE(frame, dht_linkfile_lookup_cbk, subvol, subvol,
+                          subvol->fops->lookup, &local->linkfile.loc, xattrs);
         if (xattrs)
-                dict_unref (xattrs);
+            dict_unref(xattrs);
         return 0;
+    }
+out:
+    local->linkfile.linkfile_cbk(frame, cookie, this, op_ret, op_errno, inode,
+                                 stbuf, preparent, postparent, xdata);
+    if (xattrs)
+        dict_unref(xattrs);
+    return 0;
 }
 
-
 int
-dht_linkfile_create (call_frame_t *frame, fop_mknod_cbk_t linkfile_cbk,
-                     xlator_t *this,
-                     xlator_t *tovol, xlator_t *fromvol, loc_t *loc)
+dht_linkfile_create(call_frame_t *frame, fop_mknod_cbk_t linkfile_cbk,
+                    xlator_t *this, xlator_t *tovol, xlator_t *fromvol,
+                    loc_t *loc)
 {
-        dht_local_t *local = NULL;
-        dict_t      *dict = NULL;
-        int          need_unref = 0;
-        int          ret = 0;
-        dht_conf_t  *conf = this->private;
-        char           gfid[GF_UUID_BUF_SIZE] = {0};
-
-        local = frame->local;
-        local->linkfile.linkfile_cbk = linkfile_cbk;
-        local->linkfile.srcvol = tovol;
-
-        local->linked = _gf_false;
-
-        dict = local->params;
-        if (!dict) {
-                dict = dict_new ();
-                if (!dict)
-                        goto out;
-                need_unref = 1;
-        }
-
-
-        if (!gf_uuid_is_null (local->gfid)) {
-                gf_uuid_unparse(local->gfid, gfid);
-
-                ret = dict_set_static_bin (dict, "gfid-req", local->gfid, 16);
-                if (ret)
-                        gf_msg ("dht-linkfile", GF_LOG_INFO, 0,
-                                DHT_MSG_DICT_SET_FAILED,
-                                "%s: Failed to set dictionary value: "
-                                "key = gfid-req, gfid = %s ", loc->path, gfid);
-        } else {
-                gf_uuid_unparse(loc->gfid, gfid);
-        }
-
-        ret = dict_set_str (dict, GLUSTERFS_INTERNAL_FOP_KEY, "yes");
+    dht_local_t *local = NULL;
+    dict_t *dict = NULL;
+    int need_unref = 0;
+    int ret = 0;
+    dht_conf_t *conf = this->private;
+    char gfid[GF_UUID_BUF_SIZE] = {0};
+
+    local = frame->local;
+    local->linkfile.linkfile_cbk = linkfile_cbk;
+    local->linkfile.srcvol = tovol;
+    loc_copy(&local->linkfile.loc, loc);
+
+    local->linked = _gf_false;
+
+    dict = local->params;
+    if (!dict) {
+        dict = dict_new();
+        if (!dict)
+            goto out;
+        need_unref = 1;
+    }
+
+    if (!gf_uuid_is_null(local->gfid)) {
+        gf_uuid_unparse(local->gfid, gfid);
+
+        ret = dict_set_gfuuid(dict, "gfid-req", local->gfid, true);
         if (ret)
-                gf_msg ("dht-linkfile", GF_LOG_INFO, 0,
-                        DHT_MSG_DICT_SET_FAILED,
-                        "%s: Failed to set dictionary value: key = %s,"
-                        " gfid = %s", loc->path,
-                        GLUSTERFS_INTERNAL_FOP_KEY, gfid);
-
-        ret = dict_set_str (dict, conf->link_xattr_name, tovol->name);
-
-        if (ret < 0) {
-                gf_msg (frame->this->name, GF_LOG_INFO, 0,
-                        DHT_MSG_CREATE_LINK_FAILED,
-                        "%s: failed to initialize linkfile data, gfid = %s",
-                        loc->path, gfid);
-                goto out;
-        }
-
-        local->link_subvol = fromvol;
-        /* Always create as root:root. dht_linkfile_attr_heal fixes the
-         * ownsership */
-        FRAME_SU_DO (frame, dht_local_t);
-        STACK_WIND (frame, dht_linkfile_create_cbk,
-                    fromvol, fromvol->fops->mknod, loc,
-                    S_IFREG | DHT_LINKFILE_MODE, 0, 0, dict);
-
-        if (need_unref && dict)
-                dict_unref (dict);
-
-        return 0;
+            gf_smsg("dht-linkfile", GF_LOG_INFO, 0, DHT_MSG_DICT_SET_FAILED,
+                    "path=%s", loc->path, "gfid=%s", gfid, NULL);
+    } else {
+        gf_uuid_unparse(loc->gfid, gfid);
+    }
+
+    ret = dict_set_str(dict, GLUSTERFS_INTERNAL_FOP_KEY, "yes");
+    if (ret)
+        gf_smsg("dht-linkfile", GF_LOG_INFO, 0, DHT_MSG_DICT_SET_FAILED,
+                "path=%s", loc->path, "key=%s", GLUSTERFS_INTERNAL_FOP_KEY,
+                "gfid=%s", gfid, NULL);
+
+    ret = dict_set_str(dict, conf->link_xattr_name, tovol->name);
+
+    if (ret < 0) {
+        gf_smsg(frame->this->name, GF_LOG_INFO, 0, DHT_MSG_CREATE_LINK_FAILED,
+                "path=%s", loc->path, "gfid=%s", gfid, NULL);
+        goto out;
+    }
+
+    local->link_subvol = fromvol;
+    /* Always create as root:root. dht_linkfile_attr_heal fixes the
+     * ownsership */
+    FRAME_SU_DO(frame, dht_local_t);
+    STACK_WIND_COOKIE(frame, dht_linkfile_create_cbk, fromvol, fromvol,
+                      fromvol->fops->mknod, loc, S_IFREG | DHT_LINKFILE_MODE, 0,
+                      0, dict);
+
+    if (need_unref && dict)
+        dict_unref(dict);
+
+    return 0;
 out:
-        local->linkfile.linkfile_cbk (frame, NULL, frame->this, -1, ENOMEM,
-                                      loc->inode, NULL, NULL, NULL, NULL);
+    local->linkfile.linkfile_cbk(frame, frame->this, frame->this, -1, ENOMEM,
+                                 loc->inode, NULL, NULL, NULL, NULL);
 
-        if (need_unref && dict)
-                dict_unref (dict);
+    if (need_unref && dict)
+        dict_unref(dict);
 
-        return 0;
+    return 0;
 }
 
-
 int
-dht_linkfile_unlink_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                         int32_t op_ret, int32_t op_errno,
-                         struct iatt *preparent, struct iatt *postparent,
-                         dict_t *xdata)
+dht_linkfile_unlink_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                        int32_t op_ret, int32_t op_errno,
+                        struct iatt *preparent, struct iatt *postparent,
+                        dict_t *xdata)
 {
-        dht_local_t   *local = NULL;
-        call_frame_t  *prev = NULL;
-        xlator_t      *subvol = NULL;
-        char           gfid[GF_UUID_BUF_SIZE] = {0};
-
-        local = frame->local;
-        prev = cookie;
-        subvol = prev->this;
+    dht_local_t *local = NULL;
+    xlator_t *subvol = NULL;
+    char gfid[GF_UUID_BUF_SIZE] = {0};
 
+    local = frame->local;
+    subvol = cookie;
 
-        if (op_ret == -1) {
-
-                gf_uuid_unparse(local->loc.gfid, gfid);
-                gf_msg (this->name, GF_LOG_INFO, op_errno,
-                        DHT_MSG_UNLINK_FAILED,
-                        "Unlinking linkfile %s (gfid = %s)on "
-                        "subvolume %s failed ",
-                        local->loc.path, gfid, subvol->name);
-        }
+    if (op_ret == -1) {
+        gf_uuid_unparse(local->loc.gfid, gfid);
+        gf_smsg(this->name, GF_LOG_INFO, op_errno, DHT_MSG_UNLINK_FAILED,
+                "path=%s", local->loc.path, "gfid=%s", gfid, "subvolume=%s",
+                subvol->name, NULL);
+    }
 
-        DHT_STACK_DESTROY (frame);
+    DHT_STACK_DESTROY(frame);
 
-        return 0;
+    return 0;
 }
 
-
 int
-dht_linkfile_unlink (call_frame_t *frame, xlator_t *this,
-                     xlator_t *subvol, loc_t *loc)
+dht_linkfile_unlink(call_frame_t *frame, xlator_t *this, xlator_t *subvol,
+                    loc_t *loc)
 {
-        call_frame_t *unlink_frame = NULL;
-        dht_local_t  *unlink_local = NULL;
+    call_frame_t *unlink_frame = NULL;
+    dht_local_t *unlink_local = NULL;
 
-        unlink_frame = copy_frame (frame);
-        if (!unlink_frame) {
-                goto err;
-        }
+    unlink_frame = copy_frame(frame);
+    if (!unlink_frame) {
+        goto err;
+    }
 
-        /* Using non-fop value here, as anyways, 'local->fop' is not used in
-           this particular case */
-        unlink_local = dht_local_init (unlink_frame, loc, NULL,
-                                       GF_FOP_MAXVALUE);
-        if (!unlink_local) {
-                goto err;
-        }
+    /* Using non-fop value here, as anyways, 'local->fop' is not used in
+       this particular case */
+    unlink_local = dht_local_init(unlink_frame, loc, NULL, GF_FOP_MAXVALUE);
+    if (!unlink_local) {
+        goto err;
+    }
 
-        STACK_WIND (unlink_frame, dht_linkfile_unlink_cbk,
-                    subvol, subvol->fops->unlink,
-                    &unlink_local->loc, 0, NULL);
+    STACK_WIND_COOKIE(unlink_frame, dht_linkfile_unlink_cbk, subvol, subvol,
+                      subvol->fops->unlink, &unlink_local->loc, 0, NULL);
 
-        return 0;
+    return 0;
 err:
-        if (unlink_frame)
-                DHT_STACK_DESTROY (unlink_frame);
+    if (unlink_frame)
+        DHT_STACK_DESTROY(unlink_frame);
 
-        return -1;
+    return -1;
 }
 
-
 xlator_t *
-dht_linkfile_subvol (xlator_t *this, inode_t *inode, struct iatt *stbuf,
-                     dict_t *xattr)
+dht_linkfile_subvol(xlator_t *this, inode_t *inode, struct iatt *stbuf,
+                    dict_t *xattr)
 {
-        dht_conf_t *conf = NULL;
-        xlator_t   *subvol = NULL;
-        void       *volname = NULL;
-        int         i = 0, ret = 0;
+    dht_conf_t *conf = NULL;
+    xlator_t *subvol = NULL;
+    void *volname = NULL;
+    int i = 0, ret = 0;
 
-        conf = this->private;
+    conf = this->private;
 
-        if (!xattr)
-                goto out;
+    if (!xattr)
+        goto out;
 
-        ret = dict_get_ptr (xattr, conf->link_xattr_name, &volname);
+    ret = dict_get_ptr(xattr, conf->link_xattr_name, &volname);
 
-        if ((-1 == ret) || !volname)
-                goto out;
+    if ((-1 == ret) || !volname)
+        goto out;
 
-        for (i = 0; i < conf->subvolume_cnt; i++) {
-                if (strcmp (conf->subvolumes[i]->name, (char *)volname) == 0) {
-                        subvol = conf->subvolumes[i];
-                        break;
-                }
+    for (i = 0; i < conf->subvolume_cnt; i++) {
+        if (strcmp(conf->subvolumes[i]->name, (char *)volname) == 0) {
+            subvol = conf->subvolumes[i];
+            break;
         }
+    }
 
 out:
-        return subvol;
+    return subvol;
 }
 
-int
-dht_linkfile_setattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                          int op_ret, int op_errno, struct iatt *statpre,
-                          struct iatt *statpost, dict_t *xdata)
+static int
+dht_linkfile_setattr_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                         int op_ret, int op_errno, struct iatt *statpre,
+                         struct iatt *statpost, dict_t *xdata)
 {
-        dht_local_t *local = NULL;
-        loc_t *loc = NULL;
+    dht_local_t *local = NULL;
+    loc_t *loc = NULL;
 
-        local = frame->local;
-        loc = &local->loc;
+    local = frame->local;
+    loc = &local->loc;
 
-        if (op_ret)
-                gf_msg (this->name, GF_LOG_ERROR, op_errno,
-                        DHT_MSG_SETATTR_FAILED,
-                        "Failed to set attr uid/gid on %s"
-                        " :<gfid:%s> ",
-                        (loc->path? loc->path: "NULL"),
-                        uuid_utoa(local->gfid));
+    if (op_ret)
+        gf_smsg(this->name, GF_LOG_ERROR, op_errno, DHT_MSG_SETATTR_FAILED,
+                "path=%s", (loc->path ? loc->path : "NULL"), "gfid=%s",
+                uuid_utoa(local->gfid), NULL);
 
-        DHT_STACK_DESTROY (frame);
+    DHT_STACK_DESTROY(frame);
 
-       return 0;
+    return 0;
 }
 
 int
-dht_linkfile_attr_heal (call_frame_t *frame, xlator_t *this)
+dht_linkfile_attr_heal(call_frame_t *frame, xlator_t *this)
 {
-        int     ret = -1;
-        call_frame_t *copy = NULL;
-        dht_local_t  *local = NULL;
-        dht_local_t  *copy_local = NULL;
-        xlator_t     *subvol = NULL;
-        struct iatt   stbuf = {0,};
+    int ret = -1;
+    call_frame_t *copy = NULL;
+    dht_local_t *local = NULL;
+    dht_local_t *copy_local = NULL;
+    xlator_t *subvol = NULL;
+    struct iatt stbuf = {
+        0,
+    };
+    dict_t *xattr = NULL;
+
+    local = frame->local;
+
+    GF_VALIDATE_OR_GOTO("dht", local, out);
+    GF_VALIDATE_OR_GOTO("dht", local->link_subvol, out);
+
+    if (local->stbuf.ia_type == IA_INVAL)
+        return 0;
 
-        local = frame->local;
+    DHT_MARK_FOP_INTERNAL(xattr);
 
-        GF_VALIDATE_OR_GOTO ("dht", local, out);
-        GF_VALIDATE_OR_GOTO ("dht", local->link_subvol, out);
+    gf_uuid_copy(local->loc.gfid, local->stbuf.ia_gfid);
 
-        if (local->stbuf.ia_type == IA_INVAL)
-                return 0;
+    copy = copy_frame(frame);
 
-        gf_uuid_copy (local->loc.gfid, local->stbuf.ia_gfid);
+    if (!copy)
+        goto out;
 
-        copy = copy_frame (frame);
+    copy_local = dht_local_init(copy, &local->loc, NULL, 0);
 
-        if (!copy)
-                goto out;
+    if (!copy_local)
+        goto out;
 
-       copy_local = dht_local_init (copy, &local->loc, NULL, 0);
+    stbuf = local->stbuf;
+    subvol = local->link_subvol;
 
-       if (!copy_local)
-               goto out;
+    copy->local = copy_local;
 
-        stbuf = local->stbuf;
-        subvol = local->link_subvol;
+    FRAME_SU_DO(copy, dht_local_t);
 
-        copy->local = copy_local;
+    STACK_WIND(copy, dht_linkfile_setattr_cbk, subvol, subvol->fops->setattr,
+               &copy_local->loc, &stbuf, (GF_SET_ATTR_UID | GF_SET_ATTR_GID),
+               xattr);
+    ret = 0;
+out:
+    if ((ret < 0) && (copy))
+        DHT_STACK_DESTROY(copy);
 
-        FRAME_SU_DO (copy, dht_local_t);
+    if (xattr)
+        dict_unref(xattr);
 
-        STACK_WIND (copy, dht_linkfile_setattr_cbk, subvol,
-                    subvol->fops->setattr, &copy_local->loc,
-                    &stbuf, (GF_SET_ATTR_UID | GF_SET_ATTR_GID), NULL);
-        ret = 0;
-out:
-        return ret;
+    return ret;
 }
diff --git a/xlators/cluster/dht/src/dht-lock.c b/xlators/cluster/dht/src/dht-lock.c
new file mode 100644
index 00000000000..638821ccee5
--- /dev/null
+++ b/xlators/cluster/dht/src/dht-lock.c
@@ -0,0 +1,1392 @@
+/*
+  Copyright (c) 2016 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
+*/
+
+#include "dht-lock.h"
+
+static char *
+dht_lock_asprintf(dht_lock_t *lock)
+{
+    char *lk_buf = NULL;
+    char gfid[GF_UUID_BUF_SIZE] = {
+        0,
+    };
+
+    if (lock == NULL)
+        goto out;
+
+    uuid_utoa_r(lock->loc.gfid, gfid);
+
+    gf_asprintf(&lk_buf, "%s:%s", lock->xl->name, gfid);
+
+out:
+    return lk_buf;
+}
+
+static void
+dht_log_lk_array(char *name, gf_loglevel_t log_level, dht_lock_t **lk_array,
+                 int count)
+{
+    int i = 0;
+    char *lk_buf = NULL;
+
+    if ((lk_array == NULL) || (count == 0))
+        goto out;
+
+    for (i = 0; i < count; i++) {
+        lk_buf = dht_lock_asprintf(lk_array[i]);
+        if (!lk_buf)
+            goto out;
+
+        gf_smsg(name, log_level, 0, DHT_MSG_LK_ARRAY_INFO, "index=%d", i,
+                "lk_buf=%s", lk_buf, NULL);
+        GF_FREE(lk_buf);
+    }
+
+out:
+    return;
+}
+
+static void
+dht_lock_stack_destroy(call_frame_t *lock_frame, dht_lock_type_t lk)
+{
+    dht_local_t *local = NULL;
+
+    local = lock_frame->local;
+
+    if (lk == DHT_INODELK) {
+        local->lock[0].layout.my_layout.locks = NULL;
+        local->lock[0].layout.my_layout.lk_count = 0;
+    } else {
+        local->lock[0].ns.directory_ns.locks = NULL;
+        local->lock[0].ns.directory_ns.lk_count = 0;
+    }
+
+    DHT_STACK_DESTROY(lock_frame);
+    return;
+}
+
+static void
+dht_lock_free(dht_lock_t *lock)
+{
+    if (lock == NULL)
+        goto out;
+
+    loc_wipe(&lock->loc);
+    GF_FREE(lock->domain);
+    GF_FREE(lock->basename);
+    mem_put(lock);
+
+out:
+    return;
+}
+
+static void
+dht_set_lkowner(dht_lock_t **lk_array, int count, gf_lkowner_t *lkowner)
+{
+    int i = 0;
+
+    if (!lk_array || !lkowner)
+        goto out;
+
+    for (i = 0; i < count; i++) {
+        lk_array[i]->lk_owner = *lkowner;
+    }
+
+out:
+    return;
+}
+
+static int
+dht_lock_request_cmp(const void *val1, const void *val2)
+{
+    dht_lock_t *lock1 = NULL;
+    dht_lock_t *lock2 = NULL;
+    int ret = -1;
+
+    lock1 = *(dht_lock_t **)val1;
+    lock2 = *(dht_lock_t **)val2;
+
+    GF_VALIDATE_OR_GOTO("dht-locks", lock1, out);
+    GF_VALIDATE_OR_GOTO("dht-locks", lock2, out);
+
+    ret = strcmp(lock1->xl->name, lock2->xl->name);
+
+    if (ret == 0) {
+        ret = gf_uuid_compare(lock1->loc.gfid, lock2->loc.gfid);
+    }
+
+out:
+    return ret;
+}
+
+static int
+dht_lock_order_requests(dht_lock_t **locks, int count)
+{
+    int ret = -1;
+
+    if (!locks || !count)
+        goto out;
+
+    qsort(locks, count, sizeof(*locks), dht_lock_request_cmp);
+    ret = 0;
+
+out:
+    return ret;
+}
+
+void
+dht_lock_array_free(dht_lock_t **lk_array, int count)
+{
+    int i = 0;
+    dht_lock_t *lock = NULL;
+
+    if (lk_array == NULL)
+        goto out;
+
+    for (i = 0; i < count; i++) {
+        lock = lk_array[i];
+        lk_array[i] = NULL;
+        dht_lock_free(lock);
+    }
+
+out:
+    return;
+}
+
+int32_t
+dht_lock_count(dht_lock_t **lk_array, int lk_count)
+{
+    int i = 0, locked = 0;
+
+    if ((lk_array == NULL) || (lk_count == 0))
+        goto out;
+
+    for (i = 0; i < lk_count; i++) {
+        if (lk_array[i]->locked)
+            locked++;
+    }
+out:
+    return locked;
+}
+
+static call_frame_t *
+dht_lock_frame(call_frame_t *parent_frame)
+{
+    call_frame_t *lock_frame = NULL;
+
+    lock_frame = copy_frame(parent_frame);
+    if (lock_frame == NULL)
+        goto out;
+
+    set_lk_owner_from_ptr(&lock_frame->root->lk_owner, parent_frame->root);
+
+out:
+    return lock_frame;
+}
+
+dht_lock_t *
+dht_lock_new(xlator_t *this, xlator_t *xl, loc_t *loc, short type,
+             const char *domain, const char *basename,
+             dht_reaction_type_t do_on_failure)
+{
+    dht_conf_t *conf = NULL;
+    dht_lock_t *lock = NULL;
+
+    conf = this->private;
+
+    lock = mem_get0(conf->lock_pool);
+    if (lock == NULL)
+        goto out;
+
+    lock->xl = xl;
+    lock->type = type;
+    lock->do_on_failure = do_on_failure;
+
+    lock->domain = gf_strdup(domain);
+    if (lock->domain == NULL) {
+        dht_lock_free(lock);
+        lock = NULL;
+        goto out;
+    }
+
+    if (basename) {
+        lock->basename = gf_strdup(basename);
+        if (lock->basename == NULL) {
+            dht_lock_free(lock);
+            lock = NULL;
+            goto out;
+        }
+    }
+
+    /* Fill only inode and gfid.
+       posix and protocol/server give preference to pargfid/basename over
+       gfid/inode for resolution if all the three parameters of loc_t are
+       present. I want to avoid the following hypothetical situation:
+
+       1. rebalance did a lookup on a dentry and got a gfid.
+       2. rebalance acquires lock on loc_t which was filled with gfid and
+          path (pargfid/bname) from step 1.
+       3. somebody deleted and recreated the same file
+       4. rename on the same path acquires lock on loc_t which now points
+          to a different inode (and hence gets the lock).
+       5. rebalance continues to migrate file (note that not all fops done
+          by rebalance during migration are inode/gfid based Eg., unlink)
+       6. rename continues.
+    */
+    lock->loc.inode = inode_ref(loc->inode);
+    loc_gfid(loc, lock->loc.gfid);
+
+out:
+    return lock;
+}
+
+static int
+dht_local_entrylk_init(call_frame_t *frame, dht_lock_t **lk_array, int lk_count,
+                       fop_entrylk_cbk_t entrylk_cbk)
+{
+    int ret = -1;
+    dht_local_t *local = NULL;
+
+    local = frame->local;
+
+    if (local == NULL) {
+        local = dht_local_init(frame, NULL, NULL, 0);
+    }
+
+    if (local == NULL) {
+        goto out;
+    }
+
+    local->lock[0].ns.directory_ns.entrylk_cbk = entrylk_cbk;
+    local->lock[0].ns.directory_ns.locks = lk_array;
+    local->lock[0].ns.directory_ns.lk_count = lk_count;
+
+    ret = dht_lock_order_requests(local->lock[0].ns.directory_ns.locks,
+                                  local->lock[0].ns.directory_ns.lk_count);
+    if (ret < 0)
+        goto out;
+
+    ret = 0;
+out:
+    return ret;
+}
+
+static void
+dht_entrylk_done(call_frame_t *lock_frame)
+{
+    fop_entrylk_cbk_t entrylk_cbk = NULL;
+    call_frame_t *main_frame = NULL;
+    dht_local_t *local = NULL;
+
+    local = lock_frame->local;
+    main_frame = local->main_frame;
+
+    local->lock[0].ns.directory_ns.locks = NULL;
+    local->lock[0].ns.directory_ns.lk_count = 0;
+
+    entrylk_cbk = local->lock[0].ns.directory_ns.entrylk_cbk;
+    local->lock[0].ns.directory_ns.entrylk_cbk = NULL;
+
+    entrylk_cbk(main_frame, NULL, main_frame->this,
+                local->lock[0].ns.directory_ns.op_ret,
+                local->lock[0].ns.directory_ns.op_errno, NULL);
+
+    dht_lock_stack_destroy(lock_frame, DHT_ENTRYLK);
+    return;
+}
+
+static int32_t
+dht_unlock_entrylk_done(call_frame_t *frame, void *cookie, xlator_t *this,
+                        int32_t op_ret, int32_t op_errno, dict_t *xdata)
+{
+    dht_local_t *local = NULL;
+    char gfid[GF_UUID_BUF_SIZE] = {0};
+
+    local = frame->local;
+    gf_uuid_unparse(local->lock[0].ns.directory_ns.locks[0]->loc.inode->gfid,
+                    gfid);
+
+    if (op_ret < 0) {
+        gf_smsg(this->name, GF_LOG_WARNING, op_errno,
+                DHT_MSG_UNLOCK_GFID_FAILED, "gfid=%s", gfid,
+                "DHT_LAYOUT_HEAL_DOMAIN", NULL);
+    }
+
+    DHT_STACK_DESTROY(frame);
+    return 0;
+}
+
+static int32_t
+dht_unlock_entrylk_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                       int32_t op_ret, int32_t op_errno, dict_t *xdata)
+{
+    dht_local_t *local = NULL;
+    int lk_index = 0, call_cnt = 0;
+    char gfid[GF_UUID_BUF_SIZE] = {0};
+
+    lk_index = (long)cookie;
+
+    local = frame->local;
+
+    uuid_utoa_r(local->lock[0].ns.directory_ns.locks[lk_index]->loc.gfid, gfid);
+
+    if (op_ret < 0) {
+        gf_smsg(this->name, GF_LOG_WARNING, op_errno, DHT_MSG_UNLOCKING_FAILED,
+                "name=%s",
+                local->lock[0].ns.directory_ns.locks[lk_index]->xl->name,
+                "gfid=%s", gfid, NULL);
+    } else {
+        local->lock[0].ns.directory_ns.locks[lk_index]->locked = 0;
+    }
+
+    call_cnt = dht_frame_return(frame);
+    if (is_last_call(call_cnt)) {
+        dht_entrylk_done(frame);
+    }
+
+    return 0;
+}
+
+static int32_t
+dht_unlock_entrylk(call_frame_t *frame, dht_lock_t **lk_array, int lk_count,
+                   fop_entrylk_cbk_t entrylk_cbk)
+{
+    dht_local_t *local = NULL;
+    int ret = -1, i = 0;
+    call_frame_t *lock_frame = NULL;
+    int call_cnt = 0;
+
+    GF_VALIDATE_OR_GOTO("dht-locks", frame, done);
+    GF_VALIDATE_OR_GOTO(frame->this->name, lk_array, done);
+    GF_VALIDATE_OR_GOTO(frame->this->name, entrylk_cbk, done);
+
+    call_cnt = dht_lock_count(lk_array, lk_count);
+    if (call_cnt == 0) {
+        ret = 0;
+        goto done;
+    }
+
+    lock_frame = dht_lock_frame(frame);
+    if (lock_frame == NULL) {
+        gf_smsg(frame->this->name, GF_LOG_WARNING, 0,
+                DHT_MSG_ALLOC_FRAME_FAILED_NOT_UNLOCKING_FOLLOWING_ENTRYLKS,
+                NULL);
+
+        dht_log_lk_array(frame->this->name, GF_LOG_WARNING, lk_array, lk_count);
+        goto done;
+    }
+
+    ret = dht_local_entrylk_init(lock_frame, lk_array, lk_count, entrylk_cbk);
+    if (ret < 0) {
+        gf_smsg(frame->this->name, GF_LOG_WARNING, 0,
+                DHT_MSG_LOCAL_LOCKS_STORE_FAILED_UNLOCKING_FOLLOWING_ENTRYLK,
+                NULL);
+
+        dht_log_lk_array(frame->this->name, GF_LOG_WARNING, lk_array, lk_count);
+
+        goto done;
+    }
+
+    local = lock_frame->local;
+    local->main_frame = frame;
+    local->call_cnt = call_cnt;
+
+    for (i = 0; i < local->lock[0].ns.directory_ns.lk_count; i++) {
+        if (!local->lock[0].ns.directory_ns.locks[i]->locked)
+            continue;
+
+        lock_frame->root
+            ->lk_owner = local->lock[0].ns.directory_ns.locks[i]->lk_owner;
+        STACK_WIND_COOKIE(
+            lock_frame, dht_unlock_entrylk_cbk, (void *)(long)i,
+            local->lock[0].ns.directory_ns.locks[i]->xl,
+            local->lock[0].ns.directory_ns.locks[i]->xl->fops->entrylk,
+            local->lock[0].ns.directory_ns.locks[i]->domain,
+            &local->lock[0].ns.directory_ns.locks[i]->loc,
+            local->lock[0].ns.directory_ns.locks[i]->basename, ENTRYLK_UNLOCK,
+            ENTRYLK_WRLCK, NULL);
+        if (!--call_cnt)
+            break;
+    }
+
+    return 0;
+
+done:
+    if (lock_frame)
+        dht_lock_stack_destroy(lock_frame, DHT_ENTRYLK);
+
+    /* no locks acquired, invoke entrylk_cbk */
+    if (ret == 0)
+        entrylk_cbk(frame, NULL, frame->this, 0, 0, NULL);
+
+    return ret;
+}
+
+int32_t
+dht_unlock_entrylk_wrapper(call_frame_t *frame, dht_elock_wrap_t *entrylk)
+{
+    dht_local_t *local = NULL, *lock_local = NULL;
+    call_frame_t *lock_frame = NULL;
+    char pgfid[GF_UUID_BUF_SIZE] = {0};
+    int ret = 0;
+
+    local = frame->local;
+
+    if (!entrylk || !entrylk->locks)
+        goto out;
+
+    gf_uuid_unparse(local->loc.parent->gfid, pgfid);
+
+    lock_frame = copy_frame(frame);
+    if (lock_frame == NULL) {
+        gf_smsg(frame->this->name, GF_LOG_WARNING, ENOMEM,
+                DHT_MSG_COPY_FRAME_FAILED, "pgfid=%s", pgfid, "name=%s",
+                local->loc.name, "path=%s", local->loc.path, NULL);
+        goto done;
+    }
+
+    lock_local = dht_local_init(lock_frame, NULL, NULL, 0);
+    if (lock_local == NULL) {
+        gf_smsg(frame->this->name, GF_LOG_WARNING, ENOMEM,
+                DHT_MSG_CREATE_FAILED, "local", "pgfid=%s", pgfid, "name=%s",
+                local->loc.name, "path=%s", local->loc.path, NULL);
+        goto done;
+    }
+
+    lock_frame->local = lock_local;
+
+    lock_local->lock[0].ns.directory_ns.locks = entrylk->locks;
+    lock_local->lock[0].ns.directory_ns.lk_count = entrylk->lk_count;
+    entrylk->locks = NULL;
+    entrylk->lk_count = 0;
+
+    ret = dht_unlock_entrylk(
+        lock_frame, lock_local->lock[0].ns.directory_ns.locks,
+        lock_local->lock[0].ns.directory_ns.lk_count, dht_unlock_entrylk_done);
+    if (ret)
+        goto done;
+
+    lock_frame = NULL;
+
+done:
+    if (lock_frame != NULL) {
+        DHT_STACK_DESTROY(lock_frame);
+    }
+
+out:
+    return 0;
+}
+
+static int
+dht_entrylk_cleanup_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                        int32_t op_ret, int32_t op_errno, dict_t *xdata)
+{
+    dht_entrylk_done(frame);
+    return 0;
+}
+
+static void
+dht_entrylk_cleanup(call_frame_t *lock_frame)
+{
+    dht_lock_t **lk_array = NULL;
+    int lk_count = 0, lk_acquired = 0;
+    dht_local_t *local = NULL;
+
+    local = lock_frame->local;
+
+    lk_array = local->lock[0].ns.directory_ns.locks;
+    lk_count = local->lock[0].ns.directory_ns.lk_count;
+
+    lk_acquired = dht_lock_count(lk_array, lk_count);
+    if (lk_acquired != 0) {
+        dht_unlock_entrylk(lock_frame, lk_array, lk_count,
+                           dht_entrylk_cleanup_cbk);
+    } else {
+        dht_entrylk_done(lock_frame);
+    }
+
+    return;
+}
+
+static int32_t
+dht_blocking_entrylk_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                         int32_t op_ret, int32_t op_errno, dict_t *xdata)
+{
+    int lk_index = 0;
+    int i = 0;
+    dht_local_t *local = NULL;
+
+    lk_index = (long)cookie;
+
+    local = frame->local;
+    if (op_ret == 0) {
+        local->lock[0].ns.directory_ns.locks[lk_index]->locked = _gf_true;
+    } else {
+        switch (op_errno) {
+            case ESTALE:
+            case ENOENT:
+                if (local->lock[0]
+                        .ns.directory_ns.locks[lk_index]
+                        ->do_on_failure != IGNORE_ENOENT_ESTALE) {
+                    local->lock[0].ns.directory_ns.op_ret = -1;
+                    local->lock[0].ns.directory_ns.op_errno = op_errno;
+                    goto cleanup;
+                }
+                break;
+            default:
+                local->lock[0].ns.directory_ns.op_ret = -1;
+                local->lock[0].ns.directory_ns.op_errno = op_errno;
+                goto cleanup;
+        }
+    }
+
+    if (lk_index == (local->lock[0].ns.directory_ns.lk_count - 1)) {
+        for (i = 0; (i < local->lock[0].ns.directory_ns.lk_count) &&
+                    (!local->lock[0].ns.directory_ns.locks[i]->locked);
+             i++)
+            ;
+
+        if (i == local->lock[0].ns.directory_ns.lk_count) {
+            local->lock[0].ns.directory_ns.op_ret = -1;
+            local->lock[0].ns.directory_ns.op_errno = op_errno;
+        }
+
+        dht_entrylk_done(frame);
+    } else {
+        dht_blocking_entrylk_rec(frame, ++lk_index);
+    }
+
+    return 0;
+
+cleanup:
+    dht_entrylk_cleanup(frame);
+
+    return 0;
+}
+
+void
+dht_blocking_entrylk_rec(call_frame_t *frame, int i)
+{
+    dht_local_t *local = NULL;
+
+    local = frame->local;
+
+    STACK_WIND_COOKIE(
+        frame, dht_blocking_entrylk_cbk, (void *)(long)i,
+        local->lock[0].ns.directory_ns.locks[i]->xl,
+        local->lock[0].ns.directory_ns.locks[i]->xl->fops->entrylk,
+        local->lock[0].ns.directory_ns.locks[i]->domain,
+        &local->lock[0].ns.directory_ns.locks[i]->loc,
+        local->lock[0].ns.directory_ns.locks[i]->basename, ENTRYLK_LOCK,
+        ENTRYLK_WRLCK, NULL);
+
+    return;
+}
+
+int
+dht_blocking_entrylk(call_frame_t *frame, dht_lock_t **lk_array, int lk_count,
+                     fop_entrylk_cbk_t entrylk_cbk)
+{
+    int ret = -1;
+    call_frame_t *lock_frame = NULL;
+    dht_local_t *local = NULL;
+
+    GF_VALIDATE_OR_GOTO("dht-locks", frame, out);
+    GF_VALIDATE_OR_GOTO(frame->this->name, lk_array, out);
+    GF_VALIDATE_OR_GOTO(frame->this->name, entrylk_cbk, out);
+
+    lock_frame = dht_lock_frame(frame);
+    if (lock_frame == NULL)
+        goto out;
+
+    ret = dht_local_entrylk_init(lock_frame, lk_array, lk_count, entrylk_cbk);
+    if (ret < 0) {
+        goto out;
+    }
+
+    dht_set_lkowner(lk_array, lk_count, &lock_frame->root->lk_owner);
+
+    local = lock_frame->local;
+    local->main_frame = frame;
+
+    dht_blocking_entrylk_rec(lock_frame, 0);
+
+    return 0;
+out:
+    if (lock_frame)
+        dht_lock_stack_destroy(lock_frame, DHT_ENTRYLK);
+
+    return -1;
+}
+
+static int
+dht_local_inodelk_init(call_frame_t *frame, dht_lock_t **lk_array, int lk_count,
+                       fop_inodelk_cbk_t inodelk_cbk)
+{
+    int ret = -1;
+    dht_local_t *local = NULL;
+
+    local = frame->local;
+
+    if (local == NULL) {
+        local = dht_local_init(frame, NULL, NULL, 0);
+    }
+
+    if (local == NULL) {
+        goto out;
+    }
+
+    local->lock[0].layout.my_layout.inodelk_cbk = inodelk_cbk;
+    local->lock[0].layout.my_layout.locks = lk_array;
+    local->lock[0].layout.my_layout.lk_count = lk_count;
+
+    ret = dht_lock_order_requests(local->lock[0].layout.my_layout.locks,
+                                  local->lock[0].layout.my_layout.lk_count);
+    if (ret < 0)
+        goto out;
+
+    ret = 0;
+out:
+    return ret;
+}
+
+static void
+dht_inodelk_done(call_frame_t *lock_frame)
+{
+    fop_inodelk_cbk_t inodelk_cbk = NULL;
+    call_frame_t *main_frame = NULL;
+    dht_local_t *local = NULL;
+
+    local = lock_frame->local;
+    main_frame = local->main_frame;
+
+    local->lock[0].layout.my_layout.locks = NULL;
+    local->lock[0].layout.my_layout.lk_count = 0;
+
+    inodelk_cbk = local->lock[0].layout.my_layout.inodelk_cbk;
+    local->lock[0].layout.my_layout.inodelk_cbk = NULL;
+
+    inodelk_cbk(main_frame, NULL, main_frame->this,
+                local->lock[0].layout.my_layout.op_ret,
+                local->lock[0].layout.my_layout.op_errno, NULL);
+
+    dht_lock_stack_destroy(lock_frame, DHT_INODELK);
+    return;
+}
+
+static int32_t
+dht_unlock_inodelk_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                       int32_t op_ret, int32_t op_errno, dict_t *xdata)
+{
+    dht_local_t *local = NULL;
+    int lk_index = 0, call_cnt = 0;
+    char gfid[GF_UUID_BUF_SIZE] = {0};
+
+    lk_index = (long)cookie;
+
+    local = frame->local;
+    if (op_ret < 0) {
+        uuid_utoa_r(local->lock[0].layout.my_layout.locks[lk_index]->loc.gfid,
+                    gfid);
+
+        gf_smsg(this->name, GF_LOG_WARNING, op_errno, DHT_MSG_UNLOCKING_FAILED,
+                "name=%s",
+                local->lock[0].layout.my_layout.locks[lk_index]->xl->name,
+                "gfid=%s", gfid, NULL);
+    } else {
+        local->lock[0].layout.my_layout.locks[lk_index]->locked = 0;
+    }
+
+    call_cnt = dht_frame_return(frame);
+    if (is_last_call(call_cnt)) {
+        dht_inodelk_done(frame);
+    }
+
+    return 0;
+}
+
+static int32_t
+dht_unlock_inodelk_done(call_frame_t *frame, void *cookie, xlator_t *this,
+                        int32_t op_ret, int32_t op_errno, dict_t *xdata)
+{
+    dht_local_t *local = NULL;
+    char gfid[GF_UUID_BUF_SIZE] = {0};
+
+    local = frame->local;
+    gf_uuid_unparse(local->lock[0].layout.my_layout.locks[0]->loc.inode->gfid,
+                    gfid);
+
+    if (op_ret < 0) {
+        gf_smsg(this->name, GF_LOG_WARNING, op_errno,
+                DHT_MSG_UNLOCK_GFID_FAILED, "DHT_LAYOUT_HEAL_DOMAIN gfid=%s",
+                gfid, NULL);
+    }
+
+    DHT_STACK_DESTROY(frame);
+    return 0;
+}
+
+int32_t
+dht_unlock_inodelk(call_frame_t *frame, dht_lock_t **lk_array, int lk_count,
+                   fop_inodelk_cbk_t inodelk_cbk)
+{
+    dht_local_t *local = NULL;
+    struct gf_flock flock = {
+        0,
+    };
+    int ret = -1, i = 0;
+    call_frame_t *lock_frame = NULL;
+    int call_cnt = 0;
+
+    GF_VALIDATE_OR_GOTO("dht-locks", frame, done);
+    GF_VALIDATE_OR_GOTO(frame->this->name, lk_array, done);
+    GF_VALIDATE_OR_GOTO(frame->this->name, inodelk_cbk, done);
+
+    call_cnt = dht_lock_count(lk_array, lk_count);
+    if (call_cnt == 0) {
+        ret = 0;
+        goto done;
+    }
+
+    lock_frame = dht_lock_frame(frame);
+    if (lock_frame == NULL) {
+        gf_smsg(frame->this->name, GF_LOG_WARNING, 0,
+                DHT_MSG_ALLOC_FRAME_FAILED_NOT_UNLOCKING_FOLLOWING_ENTRYLKS,
+                NULL);
+
+        dht_log_lk_array(frame->this->name, GF_LOG_WARNING, lk_array, lk_count);
+        goto done;
+    }
+
+    ret = dht_local_inodelk_init(lock_frame, lk_array, lk_count, inodelk_cbk);
+    if (ret < 0) {
+        gf_smsg(frame->this->name, GF_LOG_WARNING, 0,
+                DHT_MSG_LOCAL_LOCKS_STORE_FAILED_UNLOCKING_FOLLOWING_ENTRYLK,
+                NULL);
+
+        dht_log_lk_array(frame->this->name, GF_LOG_WARNING, lk_array, lk_count);
+
+        goto done;
+    }
+
+    local = lock_frame->local;
+    local->main_frame = frame;
+    local->call_cnt = call_cnt;
+
+    flock.l_type = F_UNLCK;
+
+    for (i = 0; i < local->lock[0].layout.my_layout.lk_count; i++) {
+        if (!local->lock[0].layout.my_layout.locks[i]->locked)
+            continue;
+
+        lock_frame->root
+            ->lk_owner = local->lock[0].layout.my_layout.locks[i]->lk_owner;
+        STACK_WIND_COOKIE(
+            lock_frame, dht_unlock_inodelk_cbk, (void *)(long)i,
+            local->lock[0].layout.my_layout.locks[i]->xl,
+            local->lock[0].layout.my_layout.locks[i]->xl->fops->inodelk,
+            local->lock[0].layout.my_layout.locks[i]->domain,
+            &local->lock[0].layout.my_layout.locks[i]->loc, F_SETLK, &flock,
+            NULL);
+        if (!--call_cnt)
+            break;
+    }
+
+    return 0;
+
+done:
+    if (lock_frame)
+        dht_lock_stack_destroy(lock_frame, DHT_INODELK);
+
+    /* no locks acquired, invoke inodelk_cbk */
+    if (ret == 0)
+        inodelk_cbk(frame, NULL, frame->this, 0, 0, NULL);
+
+    return ret;
+}
+
+int32_t
+dht_unlock_inodelk_wrapper(call_frame_t *frame, dht_ilock_wrap_t *inodelk)
+{
+    dht_local_t *local = NULL, *lock_local = NULL;
+    call_frame_t *lock_frame = NULL;
+    char pgfid[GF_UUID_BUF_SIZE] = {0};
+    int ret = 0;
+
+    local = frame->local;
+
+    if (!inodelk || !inodelk->locks)
+        goto out;
+
+    gf_uuid_unparse(local->loc.parent->gfid, pgfid);
+
+    lock_frame = copy_frame(frame);
+    if (lock_frame == NULL) {
+        gf_smsg(frame->this->name, GF_LOG_WARNING, ENOMEM,
+                DHT_MSG_COPY_FRAME_FAILED, "pgfid=%s", pgfid, "name=%s",
+                local->loc.name, "path=%s", local->loc.path, NULL);
+        goto done;
+    }
+
+    lock_local = dht_local_init(lock_frame, NULL, NULL, 0);
+    if (lock_local == NULL) {
+        gf_smsg(frame->this->name, GF_LOG_WARNING, ENOMEM,
+                DHT_MSG_CREATE_FAILED, "local", "gfid=%s", pgfid, "name=%s",
+                local->loc.name, "path=%s", local->loc.path, NULL);
+        goto done;
+    }
+
+    lock_frame->local = lock_local;
+
+    lock_local->lock[0].layout.my_layout.locks = inodelk->locks;
+    lock_local->lock[0].layout.my_layout.lk_count = inodelk->lk_count;
+    inodelk->locks = NULL;
+    inodelk->lk_count = 0;
+
+    ret = dht_unlock_inodelk(
+        lock_frame, lock_local->lock[0].layout.my_layout.locks,
+        lock_local->lock[0].layout.my_layout.lk_count, dht_unlock_inodelk_done);
+
+    if (ret)
+        goto done;
+
+    lock_frame = NULL;
+
+done:
+    if (lock_frame != NULL) {
+        DHT_STACK_DESTROY(lock_frame);
+    }
+out:
+    return 0;
+}
+
+static int
+dht_inodelk_cleanup_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                        int32_t op_ret, int32_t op_errno, dict_t *xdata)
+{
+    dht_inodelk_done(frame);
+    return 0;
+}
+
+static void
+dht_inodelk_cleanup(call_frame_t *lock_frame)
+{
+    dht_lock_t **lk_array = NULL;
+    int lk_count = 0, lk_acquired = 0;
+    dht_local_t *local = NULL;
+
+    local = lock_frame->local;
+
+    lk_array = local->lock[0].layout.my_layout.locks;
+    lk_count = local->lock[0].layout.my_layout.lk_count;
+
+    lk_acquired = dht_lock_count(lk_array, lk_count);
+    if (lk_acquired != 0) {
+        dht_unlock_inodelk(lock_frame, lk_array, lk_count,
+                           dht_inodelk_cleanup_cbk);
+    } else {
+        dht_inodelk_done(lock_frame);
+    }
+
+    return;
+}
+
+static int32_t
+dht_nonblocking_inodelk_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                            int32_t op_ret, int32_t op_errno, dict_t *xdata)
+{
+    dht_local_t *local = NULL;
+    int lk_index = 0, call_cnt = 0;
+    char gfid[GF_UUID_BUF_SIZE] = {0};
+
+    local = frame->local;
+    lk_index = (long)cookie;
+
+    if (op_ret == -1) {
+        local->lock[0].layout.my_layout.op_ret = -1;
+        local->lock[0].layout.my_layout.op_errno = op_errno;
+
+        if (local && local->lock[0].layout.my_layout.locks[lk_index]) {
+            uuid_utoa_r(local->lock[0]
+                            .layout.my_layout.locks[lk_index]
+                            ->loc.inode->gfid,
+                        gfid);
+
+            gf_msg_debug(
+                this->name, op_errno,
+                "inodelk failed on gfid: %s "
+                "subvolume: %s",
+                gfid,
+                local->lock[0].layout.my_layout.locks[lk_index]->xl->name);
+        }
+
+        goto out;
+    }
+
+    local->lock[0].layout.my_layout.locks[lk_index]->locked = _gf_true;
+
+out:
+    call_cnt = dht_frame_return(frame);
+    if (is_last_call(call_cnt)) {
+        if (local->lock[0].layout.my_layout.op_ret < 0) {
+            dht_inodelk_cleanup(frame);
+            return 0;
+        }
+
+        dht_inodelk_done(frame);
+    }
+
+    return 0;
+}
+
+int
+dht_nonblocking_inodelk(call_frame_t *frame, dht_lock_t **lk_array,
+                        int lk_count, fop_inodelk_cbk_t inodelk_cbk)
+{
+    struct gf_flock flock = {
+        0,
+    };
+    int i = 0, ret = 0;
+    dht_local_t *local = NULL;
+    call_frame_t *lock_frame = NULL;
+
+    GF_VALIDATE_OR_GOTO("dht-locks", frame, out);
+    GF_VALIDATE_OR_GOTO(frame->this->name, lk_array, out);
+    GF_VALIDATE_OR_GOTO(frame->this->name, inodelk_cbk, out);
+
+    lock_frame = dht_lock_frame(frame);
+    if (lock_frame == NULL)
+        goto out;
+
+    ret = dht_local_inodelk_init(lock_frame, lk_array, lk_count, inodelk_cbk);
+    if (ret < 0) {
+        goto out;
+    }
+
+    dht_set_lkowner(lk_array, lk_count, &lock_frame->root->lk_owner);
+
+    local = lock_frame->local;
+    local->main_frame = frame;
+
+    local->call_cnt = lk_count;
+
+    for (i = 0; i < lk_count; i++) {
+        flock.l_type = local->lock[0].layout.my_layout.locks[i]->type;
+
+        STACK_WIND_COOKIE(
+            lock_frame, dht_nonblocking_inodelk_cbk, (void *)(long)i,
+            local->lock[0].layout.my_layout.locks[i]->xl,
+            local->lock[0].layout.my_layout.locks[i]->xl->fops->inodelk,
+            local->lock[0].layout.my_layout.locks[i]->domain,
+            &local->lock[0].layout.my_layout.locks[i]->loc, F_SETLK, &flock,
+            NULL);
+    }
+
+    return 0;
+
+out:
+    if (lock_frame)
+        dht_lock_stack_destroy(lock_frame, DHT_INODELK);
+
+    return -1;
+}
+
+static int32_t
+dht_blocking_inodelk_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                         int32_t op_ret, int32_t op_errno, dict_t *xdata)
+{
+    int lk_index = 0;
+    int i = 0;
+    dht_local_t *local = NULL;
+    char gfid[GF_UUID_BUF_SIZE] = {
+        0,
+    };
+    dht_reaction_type_t reaction = 0;
+
+    lk_index = (long)cookie;
+
+    local = frame->local;
+    if (op_ret == 0) {
+        local->lock[0].layout.my_layout.locks[lk_index]->locked = _gf_true;
+    } else {
+        switch (op_errno) {
+            case ESTALE:
+            case ENOENT:
+                reaction = local->lock[0]
+                               .layout.my_layout.locks[lk_index]
+                               ->do_on_failure;
+                if ((reaction != IGNORE_ENOENT_ESTALE) &&
+                    (reaction != IGNORE_ENOENT_ESTALE_EIO)) {
+                    gf_uuid_unparse(local->lock[0]
+                                        .layout.my_layout.locks[lk_index]
+                                        ->loc.gfid,
+                                    gfid);
+                    local->lock[0].layout.my_layout.op_ret = -1;
+                    local->lock[0].layout.my_layout.op_errno = op_errno;
+                    gf_smsg(this->name, GF_LOG_ERROR, op_errno,
+                            DHT_MSG_INODELK_FAILED, "subvol=%s",
+                            local->lock[0]
+                                .layout.my_layout.locks[lk_index]
+                                ->xl->name,
+                            "gfid=%s", gfid, NULL);
+                    goto cleanup;
+                }
+                break;
+            case EIO:
+                reaction = local->lock[0]
+                               .layout.my_layout.locks[lk_index]
+                               ->do_on_failure;
+                if (reaction != IGNORE_ENOENT_ESTALE_EIO) {
+                    gf_uuid_unparse(local->lock[0]
+                                        .layout.my_layout.locks[lk_index]
+                                        ->loc.gfid,
+                                    gfid);
+                    local->lock[0].layout.my_layout.op_ret = -1;
+                    local->lock[0].layout.my_layout.op_errno = op_errno;
+                    gf_smsg(this->name, GF_LOG_ERROR, op_errno,
+                            DHT_MSG_INODELK_FAILED, "subvol=%s",
+                            local->lock[0]
+                                .layout.my_layout.locks[lk_index]
+                                ->xl->name,
+                            "gfid=%s", gfid, NULL);
+                    goto cleanup;
+                }
+                break;
+
+            default:
+                gf_uuid_unparse(
+                    local->lock[0].layout.my_layout.locks[lk_index]->loc.gfid,
+                    gfid);
+                local->lock[0].layout.my_layout.op_ret = -1;
+                local->lock[0].layout.my_layout.op_errno = op_errno;
+                gf_smsg(
+                    this->name, GF_LOG_ERROR, op_errno, DHT_MSG_INODELK_FAILED,
+                    "subvol=%s",
+                    local->lock[0].layout.my_layout.locks[lk_index]->xl->name,
+                    "gfid=%s", gfid, NULL);
+                goto cleanup;
+        }
+    }
+
+    if (lk_index == (local->lock[0].layout.my_layout.lk_count - 1)) {
+        for (i = 0; (i < local->lock[0].layout.my_layout.lk_count) &&
+                    (!local->lock[0].layout.my_layout.locks[i]->locked);
+             i++)
+            ;
+
+        if (i == local->lock[0].layout.my_layout.lk_count) {
+            local->lock[0].layout.my_layout.op_ret = -1;
+            local->lock[0].layout.my_layout.op_errno = op_errno;
+        }
+
+        dht_inodelk_done(frame);
+    } else {
+        dht_blocking_inodelk_rec(frame, ++lk_index);
+    }
+
+    return 0;
+
+cleanup:
+    dht_inodelk_cleanup(frame);
+
+    return 0;
+}
+
+void
+dht_blocking_inodelk_rec(call_frame_t *frame, int i)
+{
+    dht_local_t *local = NULL;
+    struct gf_flock flock = {
+        0,
+    };
+
+    local = frame->local;
+
+    flock.l_type = local->lock[0].layout.my_layout.locks[i]->type;
+
+    STACK_WIND_COOKIE(
+        frame, dht_blocking_inodelk_cbk, (void *)(long)i,
+        local->lock[0].layout.my_layout.locks[i]->xl,
+        local->lock[0].layout.my_layout.locks[i]->xl->fops->inodelk,
+        local->lock[0].layout.my_layout.locks[i]->domain,
+        &local->lock[0].layout.my_layout.locks[i]->loc, F_SETLKW, &flock, NULL);
+
+    return;
+}
+
+int
+dht_blocking_inodelk(call_frame_t *frame, dht_lock_t **lk_array, int lk_count,
+                     fop_inodelk_cbk_t inodelk_cbk)
+{
+    int ret = -1;
+    call_frame_t *lock_frame = NULL;
+    dht_local_t *local = NULL;
+    dht_local_t *tmp_local = NULL;
+    char gfid[GF_UUID_BUF_SIZE] = {
+        0,
+    };
+
+    GF_VALIDATE_OR_GOTO("dht-locks", frame, out);
+    GF_VALIDATE_OR_GOTO(frame->this->name, lk_array, out);
+    GF_VALIDATE_OR_GOTO(frame->this->name, inodelk_cbk, out);
+
+    tmp_local = frame->local;
+
+    lock_frame = dht_lock_frame(frame);
+    if (lock_frame == NULL) {
+        gf_uuid_unparse(tmp_local->loc.gfid, gfid);
+        gf_smsg("dht", GF_LOG_ERROR, ENOMEM, DHT_MSG_LOCK_FRAME_FAILED,
+                "gfid=%s", gfid, "path=%s", tmp_local->loc.path, NULL);
+        goto out;
+    }
+
+    ret = dht_local_inodelk_init(lock_frame, lk_array, lk_count, inodelk_cbk);
+    if (ret < 0) {
+        gf_uuid_unparse(tmp_local->loc.gfid, gfid);
+        gf_smsg("dht", GF_LOG_ERROR, ENOMEM, DHT_MSG_LOCAL_LOCK_INIT_FAILED,
+                "gfid=%s", gfid, "path=%s", tmp_local->loc.path, NULL);
+        goto out;
+    }
+
+    dht_set_lkowner(lk_array, lk_count, &lock_frame->root->lk_owner);
+
+    local = lock_frame->local;
+    local->main_frame = frame;
+
+    dht_blocking_inodelk_rec(lock_frame, 0);
+
+    return 0;
+out:
+    if (lock_frame)
+        dht_lock_stack_destroy(lock_frame, DHT_INODELK);
+
+    return -1;
+}
+
+void
+dht_unlock_namespace(call_frame_t *frame, dht_dir_transaction_t *lock)
+{
+    GF_VALIDATE_OR_GOTO("dht-locks", frame, out);
+    GF_VALIDATE_OR_GOTO(frame->this->name, lock, out);
+
+    dht_unlock_entrylk_wrapper(frame, &lock->ns.directory_ns);
+    dht_unlock_inodelk_wrapper(frame, &lock->ns.parent_layout);
+
+out:
+    return;
+}
+
+static int32_t
+dht_protect_namespace_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                          int32_t op_ret, int32_t op_errno, dict_t *xdata)
+{
+    dht_local_t *local = NULL;
+
+    local = frame->local;
+    if (op_ret != 0)
+        dht_unlock_inodelk_wrapper(frame, &local->current->ns.parent_layout);
+
+    local->current->ns.ns_cbk(frame, cookie, this, op_ret, op_errno, xdata);
+    return 0;
+}
+
+int32_t
+dht_blocking_entrylk_after_inodelk(call_frame_t *frame, void *cookie,
+                                   xlator_t *this, int32_t op_ret,
+                                   int32_t op_errno, dict_t *xdata)
+{
+    dht_local_t *local = NULL;
+    int ret = -1;
+    loc_t *loc = NULL;
+    dht_lock_t **lk_array = NULL;
+    char pgfid[GF_UUID_BUF_SIZE] = {0};
+    int count = 0;
+    dht_elock_wrap_t *entrylk = NULL;
+
+    local = frame->local;
+    entrylk = &local->current->ns.directory_ns;
+
+    if (op_ret < 0) {
+        local->op_ret = -1;
+        local->op_errno = op_errno;
+        goto err;
+    }
+
+    loc = &entrylk->locks[0]->loc;
+    gf_uuid_unparse(loc->gfid, pgfid);
+
+    local->op_ret = 0;
+    lk_array = entrylk->locks;
+    count = entrylk->lk_count;
+
+    ret = dht_blocking_entrylk(frame, lk_array, count,
+                               dht_protect_namespace_cbk);
+
+    if (ret < 0) {
+        local->op_ret = -1;
+        local->op_errno = EIO;
+        gf_smsg(this->name, GF_LOG_WARNING, local->op_errno,
+                DHT_MSG_ENTRYLK_FAILED_AFT_INODELK, "fop=%s",
+                gf_fop_list[local->fop], "pgfid=%s", pgfid, "basename=%s",
+                entrylk->locks[0]->basename, NULL);
+        goto err;
+    }
+
+    return 0;
+
+err:
+    if (lk_array != NULL) {
+        dht_lock_array_free(lk_array, count);
+        GF_FREE(lk_array);
+        entrylk->locks = NULL;
+        entrylk->lk_count = 0;
+    }
+
+    /* Unlock inodelk. No harm calling unlock twice */
+    dht_unlock_inodelk_wrapper(frame, &local->current->ns.parent_layout);
+    /* Call ns_cbk. It will take care of unwinding */
+    local->current->ns.ns_cbk(frame, NULL, this, local->op_ret, local->op_errno,
+                              NULL);
+    return 0;
+}
+
+/* Given the loc and the subvol, this routine takes the inodelk on
+ * the parent inode and entrylk on (parent, loc->name). This routine
+ * is specific as it supports only one subvol on which it takes inodelk
+ * and then entrylk serially.
+ */
+int
+dht_protect_namespace(call_frame_t *frame, loc_t *loc, xlator_t *subvol,
+                      struct dht_namespace *ns, fop_entrylk_cbk_t ns_cbk)
+{
+    dht_ilock_wrap_t *inodelk = NULL;
+    dht_elock_wrap_t *entrylk = NULL;
+    dht_lock_t **lk_array = NULL;
+    dht_local_t *local = NULL;
+    xlator_t *this = NULL;
+    loc_t parent = {
+        0,
+    };
+    int ret = -1;
+    char pgfid[GF_UUID_BUF_SIZE] = {0};
+    int32_t op_errno = 0;
+    int count = 1;
+
+    GF_VALIDATE_OR_GOTO("dht-locks", frame, out);
+    GF_VALIDATE_OR_GOTO(frame->this->name, loc, out);
+    GF_VALIDATE_OR_GOTO(frame->this->name, loc->parent, out);
+    GF_VALIDATE_OR_GOTO(frame->this->name, subvol, out);
+
+    local = frame->local;
+    this = frame->this;
+
+    inodelk = &ns->parent_layout;
+    entrylk = &ns->directory_ns;
+
+    /* Initialize entrylk_cbk and parent loc */
+    ns->ns_cbk = ns_cbk;
+
+    ret = dht_build_parent_loc(this, &parent, loc, &op_errno);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_ERROR, op_errno, DHT_MSG_LOC_FAILED,
+                "gfid=%s", loc->gfid, "name=%s", loc->name, "path=%s",
+                loc->path, NULL);
+        goto out;
+    }
+    gf_uuid_unparse(parent.gfid, pgfid);
+
+    /* Alloc inodelk */
+    inodelk->locks = GF_CALLOC(count, sizeof(*lk_array), gf_common_mt_pointer);
+    if (inodelk->locks == NULL) {
+        local->op_errno = ENOMEM;
+        gf_smsg(this->name, GF_LOG_WARNING, local->op_errno,
+                DHT_MSG_CALLOC_FAILED, "fop=%s", gf_fop_list[local->fop],
+                "pgfid=%s", pgfid, "name=%s", loc->name, "path=%s", loc->path,
+                NULL);
+        goto out;
+    }
+
+    inodelk->locks[0] = dht_lock_new(this, subvol, &parent, F_RDLCK,
+                                     DHT_LAYOUT_HEAL_DOMAIN, NULL,
+                                     FAIL_ON_ANY_ERROR);
+    if (inodelk->locks[0] == NULL) {
+        local->op_errno = ENOMEM;
+        gf_smsg(this->name, GF_LOG_WARNING, local->op_errno,
+                DHT_MSG_LOCK_ALLOC_FAILED, "inodelk-fop=%s",
+                gf_fop_list[local->fop], "pgfid=%s", pgfid, "name=%s",
+                loc->name, "path=%s", loc->path, NULL);
+        goto err;
+    }
+    inodelk->lk_count = count;
+
+    /* Allock entrylk */
+    entrylk->locks = GF_CALLOC(count, sizeof(*lk_array), gf_common_mt_pointer);
+    if (entrylk->locks == NULL) {
+        local->op_errno = ENOMEM;
+        gf_smsg(this->name, GF_LOG_WARNING, local->op_errno,
+                DHT_MSG_CALLOC_FAILED, "entrylk-fop=%s",
+                gf_fop_list[local->fop], "pgfid=%s", pgfid, "name=%s",
+                loc->name, "path=%s", loc->path, NULL);
+
+        goto err;
+    }
+
+    entrylk->locks[0] = dht_lock_new(this, subvol, &parent, F_WRLCK,
+                                     DHT_ENTRY_SYNC_DOMAIN, loc->name,
+                                     FAIL_ON_ANY_ERROR);
+    if (entrylk->locks[0] == NULL) {
+        local->op_errno = ENOMEM;
+        gf_smsg(this->name, GF_LOG_WARNING, local->op_errno,
+                DHT_MSG_LOCK_ALLOC_FAILED, "entrylk-fop=%s",
+                gf_fop_list[local->fop], "pgfid=%s", pgfid, "name=%s",
+                loc->name, "path=%s", loc->path, NULL);
+
+        goto err;
+    }
+    entrylk->lk_count = count;
+
+    /* Take read inodelk on parent. If it is successful, take write entrylk
+     * on name in cbk.
+     */
+    lk_array = inodelk->locks;
+    ret = dht_blocking_inodelk(frame, lk_array, count,
+                               dht_blocking_entrylk_after_inodelk);
+    if (ret < 0) {
+        local->op_errno = EIO;
+        gf_smsg(this->name, GF_LOG_WARNING, local->op_errno,
+                DHT_MSG_BLOCK_INODELK_FAILED, "fop=%s", gf_fop_list[local->fop],
+                "pgfid=%s", pgfid, "name=%s", loc->name, "path=%s", loc->path,
+                NULL);
+
+        goto err;
+    }
+
+    loc_wipe(&parent);
+
+    return 0;
+err:
+    if (entrylk->locks != NULL) {
+        dht_lock_array_free(entrylk->locks, count);
+        GF_FREE(entrylk->locks);
+        entrylk->locks = NULL;
+        entrylk->lk_count = 0;
+    }
+
+    if (inodelk->locks != NULL) {
+        dht_lock_array_free(inodelk->locks, count);
+        GF_FREE(inodelk->locks);
+        inodelk->locks = NULL;
+        inodelk->lk_count = 0;
+    }
+
+    loc_wipe(&parent);
+out:
+    return -1;
+}
diff --git a/xlators/cluster/dht/src/dht-lock.h b/xlators/cluster/dht/src/dht-lock.h
new file mode 100644
index 00000000000..6485c03fb6e
--- /dev/null
+++ b/xlators/cluster/dht/src/dht-lock.h
@@ -0,0 +1,91 @@
+/*
+  Copyright (c) 2016 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
+*/
+
+#ifndef _DHT_LOCK_H
+#define _DHT_LOCK_H
+
+#include "dht-common.h"
+
+void
+dht_lock_array_free(dht_lock_t **lk_array, int count);
+
+int32_t
+dht_lock_count(dht_lock_t **lk_array, int lk_count);
+
+dht_lock_t *
+dht_lock_new(xlator_t *this, xlator_t *xl, loc_t *loc, short type,
+             const char *domain, const char *basename,
+             dht_reaction_type_t do_on_failure);
+
+int32_t
+dht_unlock_entrylk_wrapper(call_frame_t *, dht_elock_wrap_t *);
+
+void
+dht_blocking_entrylk_rec(call_frame_t *frame, int i);
+
+int
+dht_blocking_entrylk(call_frame_t *frame, dht_lock_t **lk_array, int lk_count,
+                     fop_inodelk_cbk_t entrylk_cbk);
+
+int32_t
+dht_unlock_inodelk(call_frame_t *frame, dht_lock_t **lk_array, int lk_count,
+                   fop_inodelk_cbk_t inodelk_cbk);
+
+int32_t
+dht_unlock_inodelk_wrapper(call_frame_t *, dht_ilock_wrap_t *);
+
+/* Acquire non-blocking inodelk on a list of xlators.
+ *
+ * @lk_array: array of lock requests lock on.
+ *
+ * @lk_count: number of locks in @lk_array
+ *
+ * @inodelk_cbk: will be called after inodelk replies are received
+ *
+ * @retval: -1 if stack_winding inodelk fails. 0 otherwise.
+ *          inodelk_cbk is called with appropriate error on errors.
+ *          On failure to acquire lock on all members of list, successful
+ *          locks are unlocked before invoking cbk.
+ */
+
+int
+dht_nonblocking_inodelk(call_frame_t *frame, dht_lock_t **lk_array,
+                        int lk_count, fop_inodelk_cbk_t inodelk_cbk);
+
+void
+dht_blocking_inodelk_rec(call_frame_t *frame, int i);
+
+/* same as dht_nonblocking_inodelk, but issues sequential blocking locks on
+ * @lk_array directly. locks are issued on some order which remains same
+ * for a list of xlators (irrespective of order of xlators within list).
+ */
+
+int
+dht_blocking_inodelk(call_frame_t *frame, dht_lock_t **lk_array, int lk_count,
+                     fop_inodelk_cbk_t inodelk_cbk);
+
+int32_t
+dht_blocking_entrylk_after_inodelk(call_frame_t *frame, void *cookie,
+                                   xlator_t *this, int32_t op_ret,
+                                   int32_t op_errno, dict_t *xdata);
+
+int32_t
+dht_blocking_entrylk_after_inodelk_rename(call_frame_t *frame, void *cookie,
+                                          xlator_t *this, int32_t op_ret,
+                                          int32_t op_errno, dict_t *xdata);
+
+void
+dht_unlock_namespace(call_frame_t *, dht_dir_transaction_t *);
+
+int
+dht_protect_namespace(call_frame_t *frame, loc_t *loc, xlator_t *subvol,
+                      struct dht_namespace *ns, fop_entrylk_cbk_t ns_cbk);
+
+#endif /* _DHT_LOCK_H */
diff --git a/xlators/cluster/dht/src/dht-mem-types.h b/xlators/cluster/dht/src/dht-mem-types.h
index e893eb48fd8..e3c4471334a 100644
--- a/xlators/cluster/dht/src/dht-mem-types.h
+++ b/xlators/cluster/dht/src/dht-mem-types.h
@@ -8,28 +8,31 @@
   cases as published by the Free Software Foundation.
 */
 
-
 #ifndef __DHT_MEM_TYPES_H__
 #define __DHT_MEM_TYPES_H__
 
-#include "mem-types.h"
+#include <glusterfs/mem-types.h>
 
 enum gf_dht_mem_types_ {
-        gf_dht_mt_dht_du_t = gf_common_mt_end + 1,
-        gf_dht_mt_dht_conf_t,
-        gf_dht_mt_char,
-        gf_dht_mt_int32_t,
-        gf_dht_mt_xlator_t,
-        gf_dht_mt_dht_layout_t,
-        gf_switch_mt_dht_conf_t,
-        gf_switch_mt_dht_du_t,
-        gf_switch_mt_switch_sched_array,
-        gf_switch_mt_switch_struct,
-        gf_dht_mt_subvol_time,
-        gf_dht_mt_loc_t,
-        gf_defrag_info_mt,
-        gf_dht_mt_inode_ctx_t,
-        gf_dht_mt_ctx_stat_time_t,
-        gf_dht_mt_end
+    gf_dht_mt_dht_du_t = gf_common_mt_end + 1,
+    gf_dht_mt_dht_conf_t,
+    gf_dht_mt_char,
+    gf_dht_mt_int32_t,
+    gf_dht_mt_xlator_t,
+    gf_dht_mt_dht_layout_t,
+    gf_switch_mt_switch_sched_array,
+    gf_switch_mt_switch_struct,
+    gf_dht_mt_subvol_time,
+    gf_dht_mt_loc_t,
+    gf_defrag_info_mt,
+    gf_dht_mt_inode_ctx_t,
+    gf_dht_mt_dirent_t,
+    gf_dht_mt_container_t,
+    gf_dht_mt_octx_t,
+    gf_dht_mt_miginfo_t,
+    gf_dht_mt_fd_ctx_t,
+    gf_dht_ret_cache_t,
+    gf_dht_nodeuuids_t,
+    gf_dht_mt_end
 };
 #endif
diff --git a/xlators/cluster/dht/src/dht-messages.h b/xlators/cluster/dht/src/dht-messages.h
index eb4c356e3c2..601f8dad78b 100644
--- a/xlators/cluster/dht/src/dht-messages.h
+++ b/xlators/cluster/dht/src/dht-messages.h
@@ -1,4 +1,4 @@
-/*Copyright (c) 2013 Red Hat, Inc. <http://www.redhat.com>
+/*Copyright (c) 2015 Red Hat, Inc. <http://www.redhat.com>
   This file is part of GlusterFS.
 
   This file is licensed to you under your choice of the GNU Lesser
@@ -10,456 +10,377 @@
 #ifndef _DHT_MESSAGES_H_
 #define _DHT_MESSAGES_H_
 
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
-
-#include "glfs-message-id.h"
-
-/*! \file dht-messages.h
- *  \brief DHT log-message IDs and their descriptions
- *
- */
-
-/* NOTE: Rules for message additions
- * 1) Each instance of a message is _better_ left with a unique message ID, even
- *    if the message format is the same. Reasoning is that, if the message
- *    format needs to change in one instance, the other instances are not
- *    impacted or the new change does not change the ID of the instance being
- *    modified.
- * 2) Addition of a message,
- *       - Should increment the GLFS_NUM_MESSAGES
- *       - Append to the list of messages defined, towards the end
- *       - Retain macro naming as glfs_msg_X (for redability across developers)
- * NOTE: Rules for message format modifications
- * 3) Check acorss the code if the message ID macro in question is reused
- *    anywhere. If reused then then the modifications should ensure correctness
- *    everywhere, or needs a new message ID as (1) above was not adhered to. If
- *    not used anywhere, proceed with the required modification.
- * NOTE: Rules for message deletion
- * 4) Check (3) and if used anywhere else, then cannot be deleted. If not used
- *    anywhere, then can be deleted, but will leave a hole by design, as
- *    addition rules specify modification to the end of the list and not filling
- *    holes.
- */
-
-#define GLFS_DHT_BASE                   GLFS_MSGID_COMP_DHT
-#define GLFS_DHT_NUM_MESSAGES           36
-#define GLFS_MSGID_END          (GLFS_DHT_BASE + GLFS_DHT_NUM_MESSAGES + 1)
-
-/* Messages with message IDs */
-#define glfs_msg_start_x GLFS_DHT_BASE, "Invalid: Start of messages"
-
-
-
-
-/*!
- * @messageid 109001
- * @diagnosis   Cached subvolume could not be found for the specified
- *              path
- * @recommendedaction  None
- *
- */
-
-#define DHT_MSG_CACHED_SUBVOL_GET_FAILED        (GLFS_DHT_BASE + 1)
-
-/*!
- * @messageid 109002
- * @diagnosis Linkfile creation failed
- * @recommendedaction  None
- *
- */
-
-#define DHT_MSG_CREATE_LINK_FAILED       (GLFS_DHT_BASE + 2)
-
-/*!
- * @messageid 109003
- * @diagnosis The value could not be set for the specified key in
- *       the dictionary
- *
- * @recommendedaction  None
- *
- */
-
-#define DHT_MSG_DICT_SET_FAILED       (GLFS_DHT_BASE + 3)
-
-/*!
- * @messageid 109004
- * @diagnosis Directory attributes could not be healed
- * @recommendedaction  None
- *
- */
-
-#define DHT_MSG_DIR_ATTR_HEAL_FAILED (GLFS_DHT_BASE + 4)
-
-/*!
- * @messageid 109005
- * @diagnosis Self-heal failed for the specified directory
- * @recommendedaction  Ensure that all subvolumes are online
- *              and reachable and perform a lookup operation
- *              on the directory again.
- *
- */
-
-#define DHT_MSG_DIR_SELFHEAL_FAILED  (GLFS_DHT_BASE + 5)
-
-/*!
- * @messageid 109006
- * @diagnosis The extended attributes could not be healed for
- *            the specified directory on the specified subvolume
- *
- * @recommendedaction  None
- *
- */
-
-#define DHT_MSG_DIR_SELFHEAL_XATTR_FAILED  (GLFS_DHT_BASE + 6)
-
-/*!
- * @messageid 109007
- * @diagnosis   A lookup operation found the file with the same path
- *      on multiple subvolumes.
- * @recommendedaction
- *      1. Create backups of the file on other subvolumes.
- *      2. Inspect the content of the files to identify
- *                      and retain the most appropriate file.
- *
- */
-
-#define DHT_MSG_FILE_ON_MULT_SUBVOL               (GLFS_DHT_BASE + 7)
-
-/*!
- * @messageid 109008
- * @diagnosis A path resolves to a file on one subvolume and a directory
- *             on another
- * @recommendedaction
- *              1. Create a backup of the file with a different name
- *              and delete the original file.
- *              2. In the newly created back up file, remove the "trusted.gfid"
- *                      extended attribute.
- *                - Command: setfattr -x "trusted.gfid" \<path to the newly created backup file\>
- *              3. Perform a new lookup operation on both the new and old paths.
- *              4. From the mount point, inspect both the paths and retain the
- *              relevant file or directory.
- *
- */
-
-#define DHT_MSG_FILE_TYPE_MISMATCH        (GLFS_DHT_BASE + 8)
-
-/*!
- * @messageid 109009
- * @diagnosis The GFID of the file/directory is different on different subvolumes
- * @recommendedaction  None
- *
- */
-
-#define DHT_MSG_GFID_MISMATCH        (GLFS_DHT_BASE + 9)
-
-/*!
- * @messageid 109010
- * @diagnosis The GFID of the specified file/directory is NULL.
- * @recommendedaction  None
- *
- */
-
-#define DHT_MSG_GFID_NULL        (GLFS_DHT_BASE + 10)
-
-/*!
- * @messageid 109011
- * @diagnosis The hashed subvolume could not be found for the specified
- *              file/directory
- * @recommendedaction  None
- *
- */
-
-#define DHT_MSG_HASHED_SUBVOL_GET_FAILED        (GLFS_DHT_BASE + 11)
-
-/*!
- * @messageid 109012
- * @diagnosis The Distributed Hash Table Translator could not be initiated as the
- *            system is out of memory.
- * @recommendedaction  None
- *
- */
-
-#define DHT_MSG_INIT_FAILED  (GLFS_DHT_BASE + 12)
-
-/*!
- * @messageid 109013
- * @diagnosis Invalid DHT configuration in the volfile
- * @recommendedaction  None
- *
- */
-
-#define DHT_MSG_INVALID_CONFIGURATION  (GLFS_DHT_BASE + 13)
-
-/*!
- * @messageid 109014
- * @diagnosis Invalid disk layout
- * @recommendedaction  None
- *
- */
-
-#define DHT_MSG_INVALID_DISK_LAYOUT  (GLFS_DHT_BASE + 14)
-
-/*!
- * @messageid 109015
- * @diagnosis Invalid DHT configuration option.
- * @recommendedaction
- *              1. Reset the option with a valid value using the volume
- *                set command.
- *              2. Restart the process that logged the message in the
- *                log file.
- *
- */
-
-#define DHT_MSG_INVALID_OPTION  (GLFS_DHT_BASE + 15)
-
-/*!
- * @messageid 109016
- * @diagnosis The fix layout operation failed
- * @recommendedaction  None
- *
- */
-
-#define DHT_MSG_LAYOUT_FIX_FAILED       (GLFS_DHT_BASE + 16)
-
-/*!
- * @messageid 109017
- * @diagnosis Layout merge failed
- * @recommendedaction  None
- *
- */
-
-#define DHT_MSG_LAYOUT_MERGE_FAILED       (GLFS_DHT_BASE + 17)
-
-/*!
- * @messageid 109018
- * @diagnosis The layout for the specified directory does not match
-                that on the disk.
- * @recommendedaction  None
- *
- */
-
-#define DHT_MSG_LAYOUT_MISMATCH       (GLFS_DHT_BASE + 18)
-
-/*!
- * @messageid 109019
- * @diagnosis No layout is present for the specified file/directory
- * @recommendedaction  None
- *
- */
-
-#define DHT_MSG_LAYOUT_NULL       (GLFS_DHT_BASE + 19)
-
-/*!
- * @messageid 109020
- * @diagnosis Informational message: Migration of data from the cached
- *      subvolume to the hashed subvolume is complete
- * @recommendedaction  None
- *
- */
-
-#define DHT_MSG_MIGRATE_DATA_COMPLETE     (GLFS_DHT_BASE + 20)
-
-/*!
- * @messageid 109021
- * @diagnosis Migration of data failed during the rebalance operation
- *     \n Cause: Directories could not be read to identify the files for the
- *             migration process.
- * @recommendedaction
- *             The log message would indicate the reason for the failure and
- *             the corrective action depends on the specific error that is
- *             encountered. The error is one of the standard UNIX errors.
- *
- */
-
-#define DHT_MSG_MIGRATE_DATA_FAILED     (GLFS_DHT_BASE + 21)
-
-/*!
- * @messageid 109022
- * @diagnosis Informational message: The file was migrated successfully during
- *              the rebalance operation.
- * @recommendedaction  None
- *
- */
-
-#define DHT_MSG_MIGRATE_FILE_COMPLETE     (GLFS_DHT_BASE + 22)
-
-/*!
- * @messageid 109023
- * @diagnosis File migration failed during the rebalance operation
- *            \n Cause: Rebalance moves data from the cached subvolume to
- *            the hashed subvolume. Migrating a single file is a multi-step operation
- *            which involves opening, reading, and writing the data and metadata.
- *            Any failures in this multi-step operation can result in a file
- *            migration failure.
- * @recommendedaction  The log message would indicate the reason for the failure and the
- *              corrective action depends on the specific error that is encountered.
- *              The error is one of the standard UNIX errors.
- *
- */
-
-#define DHT_MSG_MIGRATE_FILE_FAILED     (GLFS_DHT_BASE + 23)
-
-/*!
- * @messageid 109024
- * @diagnosis Out of memory
- * @recommendedaction  None
- *
- */
-
-#define DHT_MSG_NO_MEMORY        (GLFS_DHT_BASE + 24)
-
-/*!
- * @messageid 109025
- * @diagnosis  The opendir() call failed on the specified directory
- *              \n Cause: When a directory is renamed, the Distribute Hash
- *              table translator checks whether the destination directory
- *              is empty. This message indicates that the opendir() call
- *              on the destination directory has failed.
- * @recommendedaction The log message would indicate the reason for the
- *              failure and the corrective action depends on the specific
- *              error that is encountered. The error is one of the standard
- *              UNIX errors.
- *
- */
-
-#define DHT_MSG_OPENDIR_FAILED       (GLFS_DHT_BASE + 25)
-
-/*!
- * @messageid 109026
- * @diagnosis The rebalance operation failed.
- * @recommendedaction Check the log file for details about the failure.
- *     Possible causes:
- *     - A subvolume is down: Restart the rebalance operation after
- *             bringing up all subvolumes.
- *
- */
-
-#define DHT_MSG_REBALANCE_FAILED   (GLFS_DHT_BASE + 26)
-
-/*!
- * @messageid 109027
- * @diagnosis Failed to start the rebalance process.
- * @recommendedaction Check the log file for details about the failure.
- *
- */
-
-#define DHT_MSG_REBALANCE_START_FAILED     (GLFS_DHT_BASE + 27)
-
-/*!
- * @messageid 109028
- * @diagnosis Informational message that indicates the status of the
- *            rebalance operation and details as to how many files were
- *            migrated, skipped, failed etc
- * @recommendedaction  None
- *
- */
-
-#define DHT_MSG_REBALANCE_STATUS     (GLFS_DHT_BASE + 28)
-
-/*!
- * @messageid 109029
- * @diagnosis The rebalance operation was aborted by the user.
- * @recommendedaction  None
- *
- */
-
-#define DHT_MSG_REBALANCE_STOPPED     (GLFS_DHT_BASE + 29)
-
-/*!
- * @messageid 109030
- * @diagnosis The file or directory could not be renamed
- * @recommendedaction   Ensure that all the subvolumes are
- *                      online and reachable and try renaming
- *                      the file or directory again.
- *
- */
-
-#define DHT_MSG_RENAME_FAILED        (GLFS_DHT_BASE + 30)
-
-/*!
- * @messageid 109031
- * @diagnosis Attributes could not be set for the specified file or
- *             directory.
- * @recommendedaction  None
- *
- */
-
-#define DHT_MSG_SETATTR_FAILED       (GLFS_DHT_BASE + 31)
-
-/*!
- * @messageid 109032
- * @diagnosis The specified subvolume is running out of file system inodes.
-        If all subvolumes run out of inodes, then new files cannot be created.
- * @recommendedaction  Consider adding more nodes to the cluster if all subvolumes
- *        run out of inodes
- *
- */
-
-#define DHT_MSG_SUBVOL_INSUFF_INODES    (GLFS_DHT_BASE + 32)
-
-/*!
- * @messageid 109033
- * @diagnosis The specified subvolume is running out of disk space. If all
-              subvolumes run out of space, new files cannot be created.
- * @recommendedaction  Consider adding more bricks to the cluster if all subvolumes
- *              run out of disk space.
- *
- */
-
-#define DHT_MSG_SUBVOL_INSUFF_SPACE    (GLFS_DHT_BASE + 33)
-
-/*!
- * @messageid 109034
- * @diagnosis Failed to unlink the specified file/directory
- * @recommendedaction  The log message would indicate the reason
-              for the failure and the corrective action depends on
-              the specific error that is encountered.
- */
-
-#define DHT_MSG_UNLINK_FAILED     (GLFS_DHT_BASE + 34)
-
-
-
-/*!
- * @messageid 109035
- * @diagnosis The layout information could not be set in the inode
- * @recommendedaction  None
- *
- */
-
-#define DHT_MSG_LAYOUT_SET_FAILED     (GLFS_DHT_BASE + 35)
-
-/*!
- * @messageid 109036
- * @diagnosis Informational message regarding layout range distribution
- *            for a directory across subvolumes
- * @recommendedaction None
- */
-
-#define DHT_MSG_LOG_FIXED_LAYOUT     (GLFS_DHT_BASE + 36)
-
-/*
- * @messageid 109037
- * @diagnosis Informational message regarding error in tier operation
- * @recommendedaction None
- */
-
-#define DHT_MSG_LOG_TIER_ERROR     (GLFS_DHT_BASE + 37)
-
-/*
- * @messageid 109038
- * @diagnosis Informational message regarding tier operation
- * @recommendedaction None
- */
-
-#define DHT_MSG_LOG_TIER_STATUS     (GLFS_DHT_BASE + 38)
-
-
-/*------------*/
-#define glfs_msg_end_x GLFS_MSGID_END, "Invalid: End of messages"
-
+#include <glusterfs/glfs-message-id.h>
+
+/* To add new message IDs, append new identifiers at the end of the list.
+ *
+ * Never remove a message ID. If it's not used anymore, you can rename it or
+ * leave it as it is, but not delete it. This is to prevent reutilization of
+ * IDs by other messages.
+ *
+ * The component name must match one of the entries defined in
+ * glfs-message-id.h.
+ */
+
+GLFS_MSGID(
+    DHT, DHT_MSG_CACHED_SUBVOL_GET_FAILED, DHT_MSG_CREATE_LINK_FAILED,
+    DHT_MSG_DICT_SET_FAILED, DHT_MSG_DIR_ATTR_HEAL_FAILED,
+    DHT_MSG_DIR_SELFHEAL_FAILED, DHT_MSG_DIR_SELFHEAL_XATTR_FAILED,
+    DHT_MSG_FILE_ON_MULT_SUBVOL, DHT_MSG_FILE_TYPE_MISMATCH,
+    DHT_MSG_GFID_MISMATCH, DHT_MSG_GFID_NULL, DHT_MSG_HASHED_SUBVOL_GET_FAILED,
+    DHT_MSG_INIT_FAILED, DHT_MSG_INVALID_CONFIGURATION,
+    DHT_MSG_INVALID_DISK_LAYOUT, DHT_MSG_INVALID_OPTION,
+    DHT_MSG_LAYOUT_FIX_FAILED, DHT_MSG_LAYOUT_MERGE_FAILED,
+    DHT_MSG_LAYOUT_MISMATCH, DHT_MSG_LAYOUT_NULL, DHT_MSG_MIGRATE_DATA_COMPLETE,
+    DHT_MSG_MIGRATE_DATA_FAILED, DHT_MSG_MIGRATE_FILE_COMPLETE,
+    DHT_MSG_MIGRATE_FILE_FAILED, DHT_MSG_NO_MEMORY, DHT_MSG_OPENDIR_FAILED,
+    DHT_MSG_REBALANCE_FAILED, DHT_MSG_REBALANCE_START_FAILED,
+    DHT_MSG_REBALANCE_STATUS, DHT_MSG_REBALANCE_STOPPED, DHT_MSG_RENAME_FAILED,
+    DHT_MSG_SETATTR_FAILED, DHT_MSG_SUBVOL_INSUFF_INODES,
+    DHT_MSG_SUBVOL_INSUFF_SPACE, DHT_MSG_UNLINK_FAILED,
+    DHT_MSG_LAYOUT_SET_FAILED, DHT_MSG_LOG_FIXED_LAYOUT,
+    DHT_MSG_GET_XATTR_FAILED, DHT_MSG_FILE_LOOKUP_FAILED,
+    DHT_MSG_OPEN_FD_FAILED, DHT_MSG_SET_INODE_CTX_FAILED,
+    DHT_MSG_UNLOCKING_FAILED, DHT_MSG_DISK_LAYOUT_NULL, DHT_MSG_SUBVOL_INFO,
+    DHT_MSG_CHUNK_SIZE_INFO, DHT_MSG_LAYOUT_FORM_FAILED, DHT_MSG_SUBVOL_ERROR,
+    DHT_MSG_LAYOUT_SORT_FAILED, DHT_MSG_REGEX_INFO, DHT_MSG_FOPEN_FAILED,
+    DHT_MSG_SET_HOSTNAME_FAILED, DHT_MSG_BRICK_ERROR, DHT_MSG_SYNCOP_FAILED,
+    DHT_MSG_MIGRATE_INFO, DHT_MSG_SOCKET_ERROR, DHT_MSG_CREATE_FD_FAILED,
+    DHT_MSG_READDIR_ERROR, DHT_MSG_CHILD_LOC_BUILD_FAILED,
+    DHT_MSG_SET_SWITCH_PATTERN_ERROR, DHT_MSG_COMPUTE_HASH_FAILED,
+    DHT_MSG_FIND_LAYOUT_ANOMALIES_ERROR, DHT_MSG_ANOMALIES_INFO,
+    DHT_MSG_LAYOUT_INFO, DHT_MSG_INODE_LK_ERROR, DHT_MSG_RENAME_INFO,
+    DHT_MSG_DATA_NULL, DHT_MSG_AGGREGATE_QUOTA_XATTR_FAILED,
+    DHT_MSG_UNLINK_LOOKUP_INFO, DHT_MSG_LINK_FILE_LOOKUP_INFO,
+    DHT_MSG_OPERATION_NOT_SUP, DHT_MSG_NOT_LINK_FILE_ERROR, DHT_MSG_CHILD_DOWN,
+    DHT_MSG_UUID_PARSE_ERROR, DHT_MSG_GET_DISK_INFO_ERROR,
+    DHT_MSG_INVALID_VALUE, DHT_MSG_SWITCH_PATTERN_INFO,
+    DHT_MSG_SUBVOL_OP_FAILED, DHT_MSG_LAYOUT_PRESET_FAILED,
+    DHT_MSG_INVALID_LINKFILE, DHT_MSG_FIX_LAYOUT_INFO,
+    DHT_MSG_GET_HOSTNAME_FAILED, DHT_MSG_WRITE_FAILED,
+    DHT_MSG_MIGRATE_HARDLINK_FILE_FAILED, DHT_MSG_FSYNC_FAILED,
+    DHT_MSG_SUBVOL_DECOMMISSION_INFO, DHT_MSG_BRICK_QUERY_FAILED,
+    DHT_MSG_SUBVOL_NO_LAYOUT_INFO, DHT_MSG_OPEN_FD_ON_DST_FAILED,
+    DHT_MSG_SUBVOL_NOT_FOUND, DHT_MSG_FILE_LOOKUP_ON_DST_FAILED,
+    DHT_MSG_DISK_LAYOUT_MISSING, DHT_MSG_DICT_GET_FAILED,
+    DHT_MSG_REVALIDATE_CBK_INFO, DHT_MSG_UPGRADE_BRICKS, DHT_MSG_LK_ARRAY_INFO,
+    DHT_MSG_RENAME_NOT_LOCAL, DHT_MSG_RECONFIGURE_INFO,
+    DHT_MSG_INIT_LOCAL_SUBVOL_FAILED, DHT_MSG_SYS_CALL_GET_TIME_FAILED,
+    DHT_MSG_NO_DISK_USAGE_STATUS, DHT_MSG_SUBVOL_DOWN_ERROR,
+    DHT_MSG_REBAL_THROTTLE_INFO, DHT_MSG_COMMIT_HASH_INFO,
+    DHT_MSG_REBAL_STRUCT_SET, DHT_MSG_HAS_MIGINFO, DHT_MSG_SETTLE_HASH_FAILED,
+    DHT_MSG_DEFRAG_PROCESS_DIR_FAILED, DHT_MSG_FD_CTX_SET_FAILED,
+    DHT_MSG_STALE_LOOKUP, DHT_MSG_PARENT_LAYOUT_CHANGED,
+    DHT_MSG_LOCK_MIGRATION_FAILED, DHT_MSG_LOCK_INODE_UNREF_FAILED,
+    DHT_MSG_ASPRINTF_FAILED, DHT_MSG_DIR_LOOKUP_FAILED, DHT_MSG_INODELK_FAILED,
+    DHT_MSG_LOCK_FRAME_FAILED, DHT_MSG_LOCAL_LOCK_INIT_FAILED,
+    DHT_MSG_ENTRYLK_ERROR, DHT_MSG_INODELK_ERROR, DHT_MSG_LOC_FAILED,
+    DHT_MSG_UNKNOWN_FOP, DHT_MSG_MIGRATE_FILE_SKIPPED,
+    DHT_MSG_DIR_XATTR_HEAL_FAILED, DHT_MSG_HASHED_SUBVOL_DOWN,
+    DHT_MSG_NON_HASHED_SUBVOL_DOWN, DHT_MSG_SYNCTASK_CREATE_FAILED,
+    DHT_MSG_DIR_HEAL_ABORT, DHT_MSG_MIGRATE_SKIP, DHT_MSG_FD_CREATE_FAILED,
+    DHT_MSG_DICT_NEW_FAILED, DHT_MSG_FAILED_TO_OPEN, DHT_MSG_CREATE_FAILED,
+    DHT_MSG_FILE_NOT_EXIST, DHT_MSG_CHOWN_FAILED, DHT_MSG_FALLOCATE_FAILED,
+    DHT_MSG_FTRUNCATE_FAILED, DHT_MSG_STATFS_FAILED, DHT_MSG_WRITE_CROSS,
+    DHT_MSG_NEW_TARGET_FOUND, DHT_MSG_INSUFF_MEMORY, DHT_MSG_SET_XATTR_FAILED,
+    DHT_MSG_SET_MODE_FAILED, DHT_MSG_FILE_EXISTS_IN_DEST,
+    DHT_MSG_SYMLINK_FAILED, DHT_MSG_LINKFILE_DEL_FAILED, DHT_MSG_MKNOD_FAILED,
+    DHT_MSG_MIGRATE_CLEANUP_FAILED, DHT_MSG_LOCK_MIGRATE,
+    DHT_MSG_PARENT_BUILD_FAILED, DHT_MSG_HASHED_SUBVOL_NOT_FOUND,
+    DHT_MSG_ACQUIRE_ENTRYLK_FAILED, DHT_MSG_CREATE_DST_FAILED,
+    DHT_MSG_MIGRATION_EXIT, DHT_MSG_CHANGED_DST, DHT_MSG_TRACE_FAILED,
+    DHT_MSG_WRITE_LOCK_FAILED, DHT_MSG_GETACTIVELK_FAILED, DHT_MSG_STAT_FAILED,
+    DHT_MSG_UNLINK_PERFORM_FAILED, DHT_MSG_CLANUP_SOURCE_FILE_FAILED,
+    DHT_MSG_UNLOCK_FILE_FAILED, DHT_MSG_REMOVE_XATTR_FAILED,
+    DHT_MSG_DATA_MIGRATE_ABORT, DHT_MSG_DEFRAG_NULL, DHT_MSG_PARENT_NULL,
+    DHT_MSG_GFID_NOT_PRESENT, DHT_MSG_CHILD_LOC_FAILED,
+    DHT_MSG_SET_LOOKUP_FAILED, DHT_MSG_DIR_REMOVED, DHT_MSG_FIX_NOT_COMP,
+    DHT_MSG_SUBVOL_DETER_FAILED, DHT_MSG_LOCAL_SUBVOL, DHT_MSG_NODE_UUID,
+    DHT_MSG_SIZE_FILE, DHT_MSG_GET_DATA_SIZE_FAILED,
+    DHT_MSG_PTHREAD_JOIN_FAILED, DHT_MSG_COUNTER_THREAD_CREATE_FAILED,
+    DHT_MSG_MIGRATION_INIT_QUEUE_FAILED, DHT_MSG_PAUSED_TIMEOUT, DHT_MSG_WOKE,
+    DHT_MSG_ABORT_REBALANCE, DHT_MSG_CREATE_TASK_REBAL_FAILED,
+    DHT_MSG_REBAL_ESTIMATE_NOT_AVAIL, DHT_MSG_ADD_CHOICES_ERROR,
+    DHT_MSG_GET_CHOICES_ERROR, DHT_MSG_PREPARE_STATUS_ERROR,
+    DHT_MSG_SET_CHOICE_FAILED, DHT_MSG_SET_HASHED_SUBVOL_FAILED,
+    DHT_MSG_XATTR_HEAL_NOT_POSS, DHT_MSG_LINKTO_FILE_FAILED,
+    DHT_MSG_STALE_LINKFILE_DELETE, DHT_MSG_NO_SUBVOL_FOR_LINKTO,
+    DHT_MSG_SUBVOL_RETURNED, DHT_MSG_UNKNOWN_LOCAL_XSEL, DHT_MSG_GET_XATTR_ERR,
+    DHT_MSG_ALLOC_OR_FILL_FAILED, DHT_MSG_GET_REAL_NAME_FAILED,
+    DHT_MSG_COPY_UUID_FAILED, DHT_MSG_MDS_DETER_FAILED,
+    DHT_MSG_CREATE_REBAL_FAILED, DHT_MSG_LINK_LAYOUT_FAILED,
+    DHT_MSG_NO_SUBVOL_IN_LAYOUT, DHT_MSG_MEM_ALLOC_FAILED,
+    DHT_MSG_SET_IN_PARAMS_DICT_FAILED, DHT_MSG_LOC_COPY_FAILED,
+    DHT_MSG_PARENT_LOC_FAILED, DHT_MSG_CREATE_LOCK_FAILED,
+    DHT_MSG_PREV_ATTEMPT_FAILED, DHT_MSG_REFRESH_ATTEMPT,
+    DHT_MSG_ACQUIRE_LOCK_FAILED, DHT_MSG_CREATE_STUB_FAILED,
+    DHT_MSG_WIND_LOCK_REQ_FAILED, DHT_MSG_REFRESH_FAILED,
+    DHT_MSG_CACHED_SUBVOL_ERROR, DHT_MSG_NO_LINK_SUBVOL, DHT_MSG_SET_KEY_FAILED,
+    DHT_MSG_REMOVE_LINKTO_FAILED, DHT_MSG_LAYOUT_DICT_SET_FAILED,
+    DHT_MSG_XATTR_DICT_NULL, DHT_MSG_DUMMY_ALLOC_FAILED, DHT_MSG_DICT_IS_NULL,
+    DHT_MSG_LINK_INODE_FAILED, DHT_MSG_SELFHEAL_FAILED, DHT_MSG_NO_MDS_SUBVOL,
+    DHT_MSG_LIST_XATTRS_FAILED, DHT_MSG_RESET_INTER_XATTR_FAILED,
+    DHT_MSG_MDS_DOWN_UNABLE_TO_SET, DHT_MSG_WIND_UNLOCK_FAILED,
+    DHT_MSG_COMMIT_HASH_FAILED, DHT_MSG_UNLOCK_GFID_FAILED,
+    DHT_MSG_UNLOCK_FOLLOW_ENTRYLK, DHT_MSG_COPY_FRAME_FAILED,
+    DHT_MSG_UNLOCK_FOLLOW_LOCKS, DHT_MSG_ENTRYLK_FAILED_AFT_INODELK,
+    DHT_MSG_CALLOC_FAILED, DHT_MSG_LOCK_ALLOC_FAILED,
+    DHT_MSG_BLOCK_INODELK_FAILED,
+    DHT_MSG_LOCAL_LOCKS_STORE_FAILED_UNLOCKING_FOLLOWING_ENTRYLK,
+    DHT_MSG_ALLOC_FRAME_FAILED_NOT_UNLOCKING_FOLLOWING_ENTRYLKS,
+    DHT_MSG_DST_NULL_SET_FAILED);
+
+#define DHT_MSG_FD_CTX_SET_FAILED_STR "Failed to set fd ctx"
+#define DHT_MSG_INVALID_VALUE_STR "Different dst found in the fd ctx"
+#define DHT_MSG_UNKNOWN_FOP_STR "Unknown FOP on file"
+#define DHT_MSG_OPEN_FD_ON_DST_FAILED_STR "Failed to open the fd on file"
+#define DHT_MSG_SYNCTASK_CREATE_FAILED_STR "Failed to create synctask"
+#define DHT_MSG_ASPRINTF_FAILED_STR                                            \
+    "asprintf failed while fetching subvol from the id"
+#define DHT_MSG_HAS_MIGINFO_STR "Found miginfo in the inode ctx"
+#define DHT_MSG_FILE_LOOKUP_FAILED_STR "failed to lookup the file"
+#define DHT_MSG_INVALID_LINKFILE_STR                                           \
+    "linkto target is different from cached-subvol. treating as destination "  \
+    "subvol"
+#define DHT_MSG_GFID_MISMATCH_STR "gfid different on the target file"
+#define DHT_MSG_GET_XATTR_FAILED_STR "failed to get 'linkto' xattr"
+#define DHT_MSG_SET_INODE_CTX_FAILED_STR "failed to set inode-ctx target file"
+#define DHT_MSG_DIR_SELFHEAL_FAILED_STR "Healing of path failed"
+#define DHT_MSG_DIR_HEAL_ABORT_STR                                             \
+    "Failed to get path from subvol. Aborting directory healing"
+#define DHT_MSG_DIR_XATTR_HEAL_FAILED_STR "xattr heal failed for directory"
+#define DHT_MSG_LOCK_INODE_UNREF_FAILED_STR                                    \
+    "Found a NULL inode. Failed to unref the inode"
+#define DHT_MSG_DICT_SET_FAILED_STR "Failed to set dictionary value"
+#define DHT_MSG_NOT_LINK_FILE_ERROR_STR "got non-linkfile"
+#define DHT_MSG_CREATE_LINK_FAILED_STR "failed to initialize linkfile data"
+#define DHT_MSG_UNLINK_FAILED_STR "Unlinking linkfile on subvolume failed"
+#define DHT_MSG_MIGRATE_FILE_FAILED_STR "Migrate file failed"
+#define DHT_MSG_NO_MEMORY_STR "could not allocate memory for dict"
+#define DHT_MSG_SUBVOL_ERROR_STR "Failed to get linkto subvol"
+#define DHT_MSG_MIGRATE_HARDLINK_FILE_FAILED_STR "link failed on subvol"
+#define DHT_MSG_MIGRATE_FILE_SKIPPED_STR "Migration skipped"
+#define DHT_MSG_FD_CREATE_FAILED_STR "fd create failed"
+#define DHT_MSG_DICT_NEW_FAILED_STR "dict_new failed"
+#define DHT_MSG_FAILED_TO_OPEN_STR "failed to open"
+#define DHT_MSG_CREATE_FAILED_STR "failed to create"
+#define DHT_MSG_FILE_NOT_EXIST_STR "file does not exist"
+#define DHT_MSG_CHOWN_FAILED_STR "chown failed"
+#define DHT_MSG_FALLOCATE_FAILED_STR "fallocate failed"
+#define DHT_MSG_FTRUNCATE_FAILED_STR "ftruncate failed"
+#define DHT_MSG_STATFS_FAILED_STR "failed to get statfs"
+#define DHT_MSG_WRITE_CROSS_STR                                                \
+    "write will cross min-fre-disk for file on subvol. looking for new subvol"
+#define DHT_MSG_SUBVOL_INSUFF_SPACE_STR                                        \
+    "Could not find any subvol with space accommodating the file. Cosider "    \
+    "adding bricks"
+#define DHT_MSG_NEW_TARGET_FOUND_STR "New target found for file"
+#define DHT_MSG_INSUFF_MEMORY_STR "insufficient memory"
+#define DHT_MSG_SET_XATTR_FAILED_STR "failed to set xattr"
+#define DHT_MSG_SET_MODE_FAILED_STR "failed to set mode"
+#define DHT_MSG_FILE_EXISTS_IN_DEST_STR "file exists in destination"
+#define DHT_MSG_LINKFILE_DEL_FAILED_STR "failed to delete the linkfile"
+#define DHT_MSG_SYMLINK_FAILED_STR "symlink failed"
+#define DHT_MSG_MKNOD_FAILED_STR "mknod failed"
+#define DHT_MSG_SETATTR_FAILED_STR "failed to perform setattr"
+#define DHT_MSG_MIGRATE_CLEANUP_FAILED_STR                                     \
+    "Migrate file cleanup failed: failed to fstat file"
+#define DHT_MSG_LOCK_MIGRATE_STR "locks will be migrated for file"
+#define DHT_MSG_PARENT_BUILD_FAILED_STR                                        \
+    "failed to build parent loc, which is needed to acquire entrylk to "       \
+    "synchronize with renames on this path. Skipping migration"
+#define DHT_MSG_HASHED_SUBVOL_NOT_FOUND_STR                                    \
+    "cannot find hashed subvol which is needed to synchronize with renames "   \
+    "on this path. Skipping migration"
+#define DHT_MSG_ACQUIRE_ENTRYLK_FAILED_STR "failed to acquire entrylk on subvol"
+#define DHT_MSG_CREATE_DST_FAILED_STR "create dst failed for file"
+#define DHT_MSG_MIGRATION_EXIT_STR "Exiting migration"
+#define DHT_MSG_CHANGED_DST_STR "destination changed fo file"
+#define DHT_MSG_TRACE_FAILED_STR "Trace failed"
+#define DHT_MSG_WRITE_LOCK_FAILED_STR "write lock failed"
+#define DHT_MSG_GETACTIVELK_FAILED_STR "getactivelk failed for file"
+#define DHT_MSG_STAT_FAILED_STR "failed to do a stat"
+#define DHT_MSG_UNLINK_PERFORM_FAILED_STR "failed to perform unlink"
+#define DHT_MSG_MIGRATE_FILE_COMPLETE_STR "completed migration"
+#define DHT_MSG_CLANUP_SOURCE_FILE_FAILED_STR "failed to cleanup source file"
+#define DHT_MSG_UNLOCK_FILE_FAILED_STR "failed to unlock file"
+#define DHT_MSG_REMOVE_XATTR_FAILED_STR "remove xattr failed"
+#define DHT_MSG_SOCKET_ERROR_STR "Failed to unlink listener socket"
+#define DHT_MSG_HASHED_SUBVOL_GET_FAILED_STR "Failed to get hashed subvolume"
+#define DHT_MSG_CACHED_SUBVOL_GET_FAILED_STR "Failed to get cached subvolume"
+#define DHT_MSG_MIGRATE_DATA_FAILED_STR "migrate-data failed"
+#define DHT_MSG_DEFRAG_NULL_STR "defrag is NULL"
+#define DHT_MSG_DATA_MIGRATE_ABORT_STR                                         \
+    "Readdirp failed. Aborting data migration for dict"
+#define DHT_MSG_LAYOUT_FIX_FAILED_STR "fix layout failed"
+#define DHT_MSG_PARENT_NULL_STR "parent is NULL"
+#define DHT_MSG_GFID_NOT_PRESENT_STR "gfid not present"
+#define DHT_MSG_CHILD_LOC_FAILED_STR "Child loc build failed"
+#define DHT_MSG_SET_LOOKUP_FAILED_STR "Failed to set lookup"
+#define DHT_MSG_DIR_LOOKUP_FAILED_STR "lookup failed"
+#define DHT_MSG_DIR_REMOVED_STR "Dir renamed or removed. Skipping"
+#define DHT_MSG_READDIR_ERROR_STR "readdir failed, Aborting fix-layout"
+#define DHT_MSG_SETTLE_HASH_FAILED_STR "Settle hash failed"
+#define DHT_MSG_DEFRAG_PROCESS_DIR_FAILED_STR "gf_defrag_process_dir failed"
+#define DHT_MSG_FIX_NOT_COMP_STR                                               \
+    "Unable to retrieve fixlayout xattr. Assume background fix layout not "    \
+    "complete"
+#define DHT_MSG_SUBVOL_DETER_FAILED_STR                                        \
+    "local subvolume determination failed with error"
+#define DHT_MSG_LOCAL_SUBVOL_STR "local subvol"
+#define DHT_MSG_NODE_UUID_STR "node uuid"
+#define DHT_MSG_SIZE_FILE_STR "Total size files"
+#define DHT_MSG_GET_DATA_SIZE_FAILED_STR                                       \
+    "Failed to get the total data size. Unable to estimate time to complete "  \
+    "rebalance"
+#define DHT_MSG_PTHREAD_JOIN_FAILED_STR                                        \
+    "file_counter_thread: pthread_join failed"
+#define DHT_MSG_COUNTER_THREAD_CREATE_FAILED_STR                               \
+    "Failed to create the file counter thread"
+#define DHT_MSG_MIGRATION_INIT_QUEUE_FAILED_STR                                \
+    "Failed to initialise migration queue"
+#define DHT_MSG_REBALANCE_STOPPED_STR "Received stop command on rebalance"
+#define DHT_MSG_PAUSED_TIMEOUT_STR "Request pause timer timeout"
+#define DHT_MSG_WOKE_STR "woken"
+#define DHT_MSG_ABORT_REBALANCE_STR "Aborting rebalance"
+#define DHT_MSG_REBALANCE_START_FAILED_STR                                     \
+    "Failed to start rebalance: look up on / failed"
+#define DHT_MSG_CREATE_TASK_REBAL_FAILED_STR                                   \
+    "Could not create task for rebalance"
+#define DHT_MSG_REBAL_ESTIMATE_NOT_AVAIL_STR                                   \
+    "Rebalance estimates will not be available"
+#define DHT_MSG_REBALANCE_STATUS_STR "Rebalance status"
+#define DHT_MSG_DATA_NULL_STR "data value is NULL"
+#define DHT_MSG_ADD_CHOICES_ERROR_STR "Error to add choices in buffer"
+#define DHT_MSG_GET_CHOICES_ERROR_STR "Error to get choices"
+#define DHT_MSG_PREPARE_STATUS_ERROR_STR "Error to prepare status"
+#define DHT_MSG_SET_CHOICE_FAILED_STR "Failed to set full choice"
+#define DHT_MSG_AGGREGATE_QUOTA_XATTR_FAILED_STR                               \
+    "Failed to aggregate quota xattr"
+#define DHT_MSG_FILE_TYPE_MISMATCH_STR                                         \
+    "path exists as a file on one subvolume and directory on another. Please " \
+    "fix it manually"
+#define DHT_MSG_LAYOUT_SET_FAILED_STR "failed to set layout for subvolume"
+#define DHT_MSG_LAYOUT_MERGE_FAILED_STR "failed to merge layouts for subvolume"
+#define DHT_MSG_SET_HASHED_SUBVOL_FAILED_STR "Failed to set hashed subvolume"
+#define DHT_MSG_XATTR_HEAL_NOT_POSS_STR                                        \
+    "No gfid exists for path. so healing xattr is not possible"
+#define DHT_MSG_REVALIDATE_CBK_INFO_STR "Revalidate: subvolume returned -1"
+#define DHT_MSG_LAYOUT_MISMATCH_STR "Mismatching layouts"
+#define DHT_MSG_UNLINK_LOOKUP_INFO_STR "lookup_unlink retuened"
+#define DHT_MSG_LINKTO_FILE_FAILED_STR                                         \
+    "Could not unlink the linkto file as either fd is open and/or linkto "     \
+    "xattr is set"
+#define DHT_MSG_LAYOUT_PRESET_FAILED_STR                                       \
+    "Could not set pre-set layout for subvolume"
+#define DHT_MSG_FILE_ON_MULT_SUBVOL_STR                                        \
+    "multiple subvolumes have file (preferably rename the file in the "        \
+    "backend, and do a fresh lookup"
+#define DHT_MSG_STALE_LINKFILE_DELETE_STR                                      \
+    "attempting deletion of stale linkfile"
+#define DHT_MSG_LINK_FILE_LOOKUP_INFO_STR "Lookup on following linkfile"
+#define DHT_MSG_NO_SUBVOL_FOR_LINKTO_STR "No link subvolume for linkto"
+#define DHT_MSG_SUBVOL_RETURNED_STR "Subvolume returned -1"
+#define DHT_MSG_UNKNOWN_LOCAL_XSEL_STR "Unknown local->xsel"
+#define DHT_MSG_DICT_GET_FAILED_STR "Failed to get"
+#define DHT_MSG_UUID_PARSE_ERROR_STR "Failed to parse uuid"
+#define DHT_MSG_GET_XATTR_ERR_STR "getxattr err for dir"
+#define DHT_MSG_ALLOC_OR_FILL_FAILED_STR "alloc or fill failed"
+#define DHT_MSG_UPGRADE_BRICKS_STR                                             \
+    "At least one of the bricks does not support this operation. Please "      \
+    "upgrade all bricks"
+#define DHT_MSG_GET_REAL_NAME_FAILED_STR "Failed to get real filename"
+#define DHT_MSG_LAYOUT_NULL_STR "Layout is NULL"
+#define DHT_MSG_COPY_UUID_FAILED_STR "Failed to copy node uuid key"
+#define DHT_MSG_MDS_DETER_FAILED_STR                                           \
+    "Cannot determine MDS, fetching xattr randomly from a subvol"
+#define DHT_MSG_HASHED_SUBVOL_DOWN_STR                                         \
+    "MDS is down for path, so fetching xattr randomly from subvol"
+#define DHT_MSG_CREATE_REBAL_FAILED_STR                                        \
+    "failed to create a new rebalance synctask"
+#define DHT_MSG_FIX_LAYOUT_INFO_STR "fixing the layout"
+#define DHT_MSG_OPERATION_NOT_SUP_STR "wrong directory-spread-count value"
+#define DHT_MSG_LINK_LAYOUT_FAILED_STR "failed to link the layout in inode"
+#define DHT_MSG_NO_SUBVOL_IN_LAYOUT_STR "no subvolume in layout for path"
+#define DHT_MSG_INODE_LK_ERROR_STR "mknod lock failed for file"
+#define DHT_MSG_MEM_ALLOC_FAILED_STR "mem allocation failed"
+#define DHT_MSG_PARENT_LAYOUT_CHANGED_STR                                      \
+    "extracting in-memory layout of parent failed"
+#define DHT_MSG_SET_IN_PARAMS_DICT_FAILED_STR                                  \
+    "setting in params dictionary failed"
+#define DHT_MSG_LOC_COPY_FAILED_STR "loc_copy failed"
+#define DHT_MSG_LOC_FAILED_STR "parent loc build failed"
+#define DHT_MSG_PARENT_LOC_FAILED_STR "locking parent failed"
+#define DHT_MSG_CREATE_LOCK_FAILED_STR "Create lock failed"
+#define DHT_MSG_PREV_ATTEMPT_FAILED_STR                                        \
+    "mkdir loop detected. parent layout didn't change even though previous "   \
+    "attempt of mkdir failed because of in-memory layout not matching with "   \
+    "that on disk."
+#define DHT_MSG_REFRESH_ATTEMPT_STR                                            \
+    "mkdir parent layout changed. Attempting a refresh and then a retry"
+#define DHT_MSG_ACQUIRE_LOCK_FAILED_STR                                        \
+    "Acquiring lock on parent to guard against layout-change failed"
+#define DHT_MSG_CREATE_STUB_FAILED_STR "creating stub failed"
+#define DHT_MSG_WIND_LOCK_REQ_FAILED_STR                                       \
+    "cannot wind lock request to guard parent layout"
+#define DHT_MSG_REFRESH_FAILED_STR "refreshing parent layout failed."
+#define DHT_MSG_CACHED_SUBVOL_ERROR_STR "On cached subvol"
+#define DHT_MSG_NO_LINK_SUBVOL_STR "Linkfile does not have link subvolume"
+#define DHT_MSG_SET_KEY_FAILED_STR "failed to set key"
+#define DHT_MSG_CHILD_DOWN_STR "Received CHILD_DOWN. Exiting"
+#define DHT_MSG_LOG_FIXED_LAYOUT_STR "log layout fixed"
+#define DHT_MSG_REBAL_STRUCT_SET_STR "local->rebalance already set"
+#define DHT_MSG_REMOVE_LINKTO_FAILED_STR "Removal of linkto failed at subvol"
+#define DHT_MSG_LAYOUT_DICT_SET_FAILED_STR "dht layout dict set failed"
+#define DHT_MSG_SUBVOL_INFO_STR "creating subvolume"
+#define DHT_MSG_COMPUTE_HASH_FAILED_STR "hash computation failed"
+#define DHT_MSG_INVALID_DISK_LAYOUT_STR                                        \
+    "Invalid disk layout: Catastrophic error layout with unknown type found"
+#define DHT_MSG_LAYOUT_SORT_FAILED_STR "layout sort failed"
+#define DHT_MSG_ANOMALIES_INFO_STR "Found anomalies"
+#define DHT_MSG_XATTR_DICT_NULL_STR "xattr dictionary is NULL"
+#define DHT_MSG_DISK_LAYOUT_MISSING_STR "Disk layout missing"
+#define DHT_MSG_LAYOUT_INFO_STR "layout info"
+#define DHT_MSG_SUBVOL_NO_LAYOUT_INFO_STR "no pre-set layout for subvol"
+#define DHT_MSG_SELFHEAL_XATTR_FAILED_STR "layout setxattr failed"
+#define DHT_MSG_DIR_SELFHEAL_XATTR_FAILED_STR "Directory self heal xattr failed"
+#define DHT_MSG_DUMMY_ALLOC_FAILED_STR "failed to allocate dummy layout"
+#define DHT_MSG_DICT_IS_NULL_STR                                               \
+    "dict is NULL, need to make sure gfids are same"
+#define DHT_MSG_ENTRYLK_ERROR_STR "acquiring entrylk after inodelk failed"
+#define DHT_MSG_NO_DISK_USAGE_STATUS_STR "no du stats"
+#define DHT_MSG_LINK_INODE_FAILED_STR "linking inode failed"
+#define DHT_MSG_SELFHEAL_FAILED_STR "Directory selfheal failed"
+#define DHT_MSG_NO_MDS_SUBVOL_STR "No mds subvol"
+#define DHT_MSG_LIST_XATTRS_FAILED_STR "failed to list xattrs"
+#define DHT_MSG_RESET_INTER_XATTR_FAILED_STR "Failed to reset internal xattr"
+#define DHT_MSG_MDS_DOWN_UNABLE_TO_SET_STR                                     \
+    "mds subvol is down, unable to set xattr"
+#define DHT_MSG_DIR_ATTR_HEAL_FAILED_STR                                       \
+    "Directory attr heal failed. Failed to set uid/gid"
+#define DHT_MSG_WIND_UNLOCK_FAILED_STR                                         \
+    "Winding unlock failed: stale locks left on brick"
+#define DHT_MSG_COMMIT_HASH_FAILED_STR "Directory commit hash updaten failed"
+#define DHT_MSG_LK_ARRAY_INFO_STR "lk info"
+#define DHT_MSG_UNLOCK_GFID_FAILED_STR                                         \
+    "unlock failed on gfid: stale lock might be left"
+#define DHT_MSG_UNLOCKING_FAILED_STR "unlocking failed"
+#define DHT_MSG_UNLOCK_FOLLOW_ENTRYLK_STR "not unlocking following entrylks"
+#define DHT_MSG_COPY_FRAME_FAILED_STR "copy frame failed"
+#define DHT_MSG_UNLOCK_FOLLOW_LOCKS_STR "not unlocking following locks"
+#define DHT_MSG_INODELK_FAILED_STR "inodelk failed on subvol"
+#define DHT_MSG_LOCK_FRAME_FAILED_STR "memory allocation failed for lock_frame"
+#define DHT_MSG_LOCAL_LOCK_INIT_FAILED_STR "dht_local_lock_init failed"
+#define DHT_MSG_ENTRYLK_FAILED_AFT_INODELK_STR                                 \
+    "dht_blocking_entrylk failed after taking inodelk"
+#define DHT_MSG_BLOCK_INODELK_FAILED_STR "dht_blocking_inodelk failed"
+#define DHT_MSG_CALLOC_FAILED_STR "calloc failed"
+#define DHT_MSG_LOCK_ALLOC_FAILED_STR "lock allocation failed"
+#define DHT_MSG_ALLOC_FRAME_FAILED_NOT_UNLOCKING_FOLLOWING_ENTRYLKS_STR        \
+    "cannot allocate a frame, not unlocking following entrylks"
+#define DHT_MSG_LOCAL_LOCKS_STORE_FAILED_UNLOCKING_FOLLOWING_ENTRYLK_STR       \
+    "storing locks in local failed, not unlocking following entrylks"
+#define DHT_MSG_DST_NULL_SET_FAILED_STR                                        \
+    "src or dst is NULL, Failed to set dictionary value"
 
 #endif /* _DHT_MESSAGES_H_ */
diff --git a/xlators/cluster/dht/src/dht-rebalance.c b/xlators/cluster/dht/src/dht-rebalance.c
index 9f389c12213..8ba8082bd86 100644
--- a/xlators/cluster/dht/src/dht-rebalance.c
+++ b/xlators/cluster/dht/src/dht-rebalance.c
@@ -8,92 +8,143 @@
   cases as published by the Free Software Foundation.
 */
 
-
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
-
 #include "dht-common.h"
-#include "xlator.h"
-#include <signal.h>
+#include <glusterfs/syscall.h>
 #include <fnmatch.h>
 #include <signal.h>
+#include <glusterfs/events.h>
+#include "glusterfs/compat-errno.h"  // for ENODATA on BSD
+
+#define GF_DISK_SECTOR_SIZE 512
+#define DHT_REBALANCE_PID 4242        /* Change it if required */
+#define DHT_REBALANCE_BLKSIZE 1048576 /* 1 MB */
+#define MAX_MIGRATE_QUEUE_COUNT 500
+#define MIN_MIGRATE_QUEUE_COUNT 200
+#define MAX_REBAL_TYPE_SIZE 16
+#define FILE_CNT_INTERVAL 600       /* 10 mins */
+#define ESTIMATE_START_INTERVAL 600 /* 10 mins */
+#define HARDLINK_MIG_INPROGRESS -2
+#define SKIP_MIGRATION_FD_POSITIVE -3
+#ifndef MAX
+#define MAX(a, b) (((a) > (b)) ? (a) : (b))
+#endif
 
-#define GF_DISK_SECTOR_SIZE             512
-#define DHT_REBALANCE_PID               4242 /* Change it if required */
-#define DHT_REBALANCE_BLKSIZE           (128 * 1024)
+#define GF_CRAWL_INDEX_MOVE(idx, sv_cnt)                                       \
+    {                                                                          \
+        idx++;                                                                 \
+        idx %= sv_cnt;                                                         \
+    }
 
-static int
-dht_write_with_holes (xlator_t *to, fd_t *fd, struct iovec *vec, int count,
-                      int32_t size, off_t offset, struct iobref *iobref)
+uint64_t g_totalfiles = 0;
+uint64_t g_totalsize = 0;
+
+void
+gf_defrag_free_dir_dfmeta(struct dir_dfmeta *meta, int local_subvols_cnt)
 {
-        int i            = 0;
-        int ret          = -1;
-        int start_idx    = 0;
-        int tmp_offset   = 0;
-        int write_needed = 0;
-        int buf_len      = 0;
-        int size_pending = 0;
-        char *buf        = NULL;
-
-        /* loop through each vector */
-        for (i = 0; i < count; i++) {
-                buf = vec[i].iov_base;
-                buf_len = vec[i].iov_len;
-
-                for (start_idx = 0; (start_idx + GF_DISK_SECTOR_SIZE) <= buf_len;
-                     start_idx += GF_DISK_SECTOR_SIZE) {
-
-                        if (mem_0filled (buf + start_idx, GF_DISK_SECTOR_SIZE) != 0) {
-                                write_needed = 1;
-                                continue;
-                        }
+    int i = 0;
+
+    if (meta) {
+        for (i = 0; i < local_subvols_cnt; i++) {
+            if (meta->equeue)
+                gf_dirent_free(&meta->equeue[i]);
+            if (meta->lfd && meta->lfd[i])
+                fd_unref(meta->lfd[i]);
+        }
 
-                        if (write_needed) {
-                                ret = syncop_write (to, fd, (buf + tmp_offset),
-                                                    (start_idx - tmp_offset),
-                                                    (offset + tmp_offset),
-                                                    iobref, 0, NULL, NULL);
-                                /* 'path' will be logged in calling function */
-                                if (ret < 0) {
-                                        gf_log (THIS->name, GF_LOG_WARNING,
-                                                "failed to write (%s)",
-                                                strerror (-ret));
-                                        ret = -1;
-                                        goto out;
-                                }
-
-                                write_needed = 0;
-                        }
-                        tmp_offset = start_idx + GF_DISK_SECTOR_SIZE;
-                }
+        GF_FREE(meta->equeue);
+        GF_FREE(meta->head);
+        GF_FREE(meta->iterator);
+        GF_FREE(meta->offset_var);
+        GF_FREE(meta->fetch_entries);
+        GF_FREE(meta->lfd);
+        GF_FREE(meta);
+    }
+}
 
-                if ((start_idx < buf_len) || write_needed) {
-                        /* This means, last chunk is not yet written.. write it */
-                        ret = syncop_write (to, fd, (buf + tmp_offset),
-                                            (buf_len - tmp_offset),
-                                            (offset + tmp_offset), iobref, 0,
-                                            NULL, NULL);
-                        if (ret < 0) {
-                                /* 'path' will be logged in calling function */
-                                gf_log (THIS->name, GF_LOG_WARNING,
-                                        "failed to write (%s)",
-                                        strerror (-ret));
-                                ret = -1;
-                                goto out;
-                        }
-                }
+void
+gf_defrag_free_container(struct dht_container *container)
+{
+    if (container) {
+        gf_dirent_entry_free(container->df_entry);
 
-                size_pending = (size - buf_len);
-                if (!size_pending)
-                        break;
+        if (container->parent_loc) {
+            loc_wipe(container->parent_loc);
         }
 
-        ret = size;
-out:
-        return ret;
+        GF_FREE(container->parent_loc);
+
+        GF_FREE(container);
+    }
+}
 
+void
+dht_set_global_defrag_error(gf_defrag_info_t *defrag, int ret)
+{
+    LOCK(&defrag->lock);
+    {
+        defrag->global_error = ret;
+    }
+    UNLOCK(&defrag->lock);
+    return;
+}
+
+static int
+dht_send_rebalance_event(xlator_t *this, int cmd, gf_defrag_status_t status)
+{
+    int ret = -1;
+    char *volname = NULL;
+    char *tmpstr = NULL;
+    char *ptr = NULL;
+    char *suffix = "-dht";
+    int len = 0;
+
+    eventtypes_t event = EVENT_LAST;
+
+    switch (status) {
+        case GF_DEFRAG_STATUS_COMPLETE:
+            event = EVENT_VOLUME_REBALANCE_COMPLETE;
+            break;
+        case GF_DEFRAG_STATUS_FAILED:
+            event = EVENT_VOLUME_REBALANCE_FAILED;
+            break;
+        case GF_DEFRAG_STATUS_STOPPED:
+            event = EVENT_VOLUME_REBALANCE_STOP;
+            break;
+        default:
+            break;
+    }
+
+    /* DHT volume */
+    len = strlen(this->name) - strlen(suffix);
+    tmpstr = gf_strdup(this->name);
+    if (tmpstr) {
+        ptr = tmpstr + len;
+        if (!strcmp(ptr, suffix)) {
+            tmpstr[len] = '\0';
+            volname = tmpstr;
+        }
+    }
+
+    if (!volname) {
+        /* Better than nothing */
+        volname = this->name;
+    }
+
+    if (event != EVENT_LAST) {
+        gf_event(event, "volume=%s", volname);
+    }
+
+    GF_FREE(tmpstr);
+    return ret;
+}
+
+static void
+dht_strip_out_acls(dict_t *dict)
+{
+    if (dict) {
+        dict_del(dict, "trusted.SGI_ACL_FILE");
+        dict_del(dict, POSIX_ACL_ACCESS_XATTR);
+    }
 }
 
 /*
@@ -132,150 +183,316 @@ be converted to "0" in dht_migrate_file.
 */
 
 int32_t
-gf_defrag_handle_hardlink (xlator_t *this, loc_t *loc, dict_t  *xattrs,
-                           struct iatt *stbuf)
+gf_defrag_handle_hardlink(xlator_t *this, loc_t *loc, int *fop_errno)
 {
-        int32_t                 ret             = -1;
-        xlator_t               *cached_subvol   = NULL;
-        xlator_t               *hashed_subvol   = NULL;
-        xlator_t               *linkto_subvol   = NULL;
-        data_t                 *data            = NULL;
-        struct iatt             iatt            = {0,};
-        int32_t                 op_errno        = 0;
-        dht_conf_t             *conf            = NULL;
-        gf_loglevel_t          loglevel         = 0;
-        dict_t                 *link_xattr      = NULL;
-
-        GF_VALIDATE_OR_GOTO ("defrag", loc, out);
-        GF_VALIDATE_OR_GOTO ("defrag", loc->name, out);
-        GF_VALIDATE_OR_GOTO ("defrag", stbuf, out);
-        GF_VALIDATE_OR_GOTO ("defrag", this, out);
-        GF_VALIDATE_OR_GOTO ("defrag", xattrs, out);
-        GF_VALIDATE_OR_GOTO ("defrag", this->private, out);
-
-        conf = this->private;
-
-        if (gf_uuid_is_null (loc->pargfid)) {
-                gf_msg ("", GF_LOG_ERROR, 0,
-                        DHT_MSG_MIGRATE_FILE_FAILED,
-                        "Migrate file failed :"
-                        "loc->pargfid is NULL for %s", loc->path);
-                goto out;
+    int32_t ret = -1;
+    xlator_t *cached_subvol = NULL;
+    xlator_t *hashed_subvol = NULL;
+    xlator_t *linkto_subvol = NULL;
+    data_t *data = NULL;
+    struct iatt iatt = {
+        0,
+    };
+    int32_t op_errno = 0;
+    dht_conf_t *conf = NULL;
+    gf_loglevel_t loglevel = 0;
+    dict_t *link_xattr = NULL;
+    dict_t *dict = NULL;
+    dict_t *xattr_rsp = NULL;
+    struct iatt stbuf = {
+        0,
+    };
+
+    *fop_errno = EINVAL;
+
+    GF_VALIDATE_OR_GOTO("defrag", loc, out);
+    GF_VALIDATE_OR_GOTO("defrag", loc->name, out);
+    GF_VALIDATE_OR_GOTO("defrag", this, out);
+    GF_VALIDATE_OR_GOTO("defrag", this->private, out);
+
+    conf = this->private;
+
+    if (gf_uuid_is_null(loc->pargfid)) {
+        gf_msg("", GF_LOG_ERROR, 0, DHT_MSG_MIGRATE_FILE_FAILED,
+               "Migrate file failed :"
+               "loc->pargfid is NULL for %s",
+               loc->path);
+        *fop_errno = EINVAL;
+        ret = -1;
+        goto out;
+    }
+
+    if (gf_uuid_is_null(loc->gfid)) {
+        gf_msg("", GF_LOG_ERROR, 0, DHT_MSG_MIGRATE_FILE_FAILED,
+               "Migrate file failed :"
+               "loc->gfid is NULL for %s",
+               loc->path);
+        *fop_errno = EINVAL;
+        ret = -1;
+        goto out;
+    }
+
+    link_xattr = dict_new();
+    if (!link_xattr) {
+        ret = -1;
+        *fop_errno = ENOMEM;
+        goto out;
+    }
+
+    /*
+      Parallel migration can lead to migration of the hard link multiple
+      times which can lead to data loss. Hence, adding a fresh lookup to
+      decide whether migration is required or not.
+
+      Elaborating the scenario for let say 10 hardlinks [link{1..10}]:
+          Let say the first hard link "link1"  does the setxattr of the
+      new hashed subvolume info on the cached file. As there are multiple
+      threads working, we might have already all the links created on the
+      new hashed by the time we reach hardlink let say link5. Now the
+      number of links on hashed is equal to that of cached. Hence, file
+      migration will happen for link6.
+
+             Cached                                 Hashed
+      --------T link6                        rwxrwxrwx   link6
+
+      Now post above state all the link file on the cached will be zero
+      byte linkto files. Hence, if we still do migration for the following
+      files link{7..10}, we will end up migrating 0 data leading to data
+      loss.
+            Hence, a lookup can make sure whether we need to migrate the
+      file or not.
+    */
+
+    dict = dict_new();
+    if (!dict) {
+        ret = -1;
+        *fop_errno = ENOMEM;
+        gf_msg(this->name, GF_LOG_ERROR, ENOMEM, DHT_MSG_NO_MEMORY,
+               "could not allocate memory for dict");
+        goto out;
+    }
+
+    ret = dict_set_int32(dict, conf->link_xattr_name, 256);
+    if (ret) {
+        *fop_errno = ENOMEM;
+        ret = -1;
+        gf_msg(this->name, GF_LOG_ERROR, 0, DHT_MSG_MIGRATE_FILE_FAILED,
+               "Migrate file failed:"
+               "%s: failed to set 'linkto' key in dict",
+               loc->path);
+        goto out;
+    }
+
+    ret = syncop_lookup(this, loc, &stbuf, NULL, dict, &xattr_rsp);
+    if (ret) {
+        /*Ignore ENOENT and ESTALE as file might have been
+          migrated already*/
+        if (-ret == ENOENT || -ret == ESTALE) {
+            ret = -2;
+            goto out;
+        }
+        gf_msg(this->name, GF_LOG_ERROR, -ret, DHT_MSG_MIGRATE_FILE_FAILED,
+               "Migrate file failed:%s lookup failed with ret = %d", loc->path,
+               ret);
+        *fop_errno = -ret;
+        ret = -1;
+        goto out;
+    }
+
+    cached_subvol = dht_subvol_get_cached(this, loc->inode);
+    if (!cached_subvol) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, DHT_MSG_MIGRATE_FILE_FAILED,
+               "Migrate file failed :"
+               "Failed to get cached subvol"
+               " for %s on %s",
+               loc->name, this->name);
+        *fop_errno = EINVAL;
+        ret = -1;
+        goto out;
+    }
+
+    hashed_subvol = dht_subvol_get_hashed(this, loc);
+    if (!hashed_subvol) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, DHT_MSG_MIGRATE_FILE_FAILED,
+               "Migrate file failed :"
+               "Failed to get hashed subvol"
+               " for %s on %s",
+               loc->name, this->name);
+        *fop_errno = EINVAL;
+        ret = -1;
+        goto out;
+    }
+
+    /* Hardlink migration happens only with remove-brick. So this condition will
+     * be true only when the migration has happened. In case hardlinks are
+     * migrated for rebalance case, remove this check. Having this check here
+     * avoid redundant calls below*/
+    if (hashed_subvol == cached_subvol) {
+        ret = -2;
+        goto out;
+    }
+
+    gf_log(this->name, GF_LOG_INFO,
+           "Attempting to migrate hardlink %s "
+           "with gfid %s from %s -> %s",
+           loc->name, uuid_utoa(loc->gfid), cached_subvol->name,
+           hashed_subvol->name);
+
+    data = dict_get(xattr_rsp, conf->link_xattr_name);
+    /* set linkto on cached -> hashed if not present, else link it */
+    if (!data) {
+        ret = dict_set_str(link_xattr, conf->link_xattr_name,
+                           hashed_subvol->name);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, DHT_MSG_MIGRATE_FILE_FAILED,
+                   "Migrate file failed :"
+                   "Failed to set dictionary value:"
+                   " key = %s for %s",
+                   conf->link_xattr_name, loc->name);
+            *fop_errno = ENOMEM;
+            ret = -1;
+            goto out;
         }
 
-        if (gf_uuid_is_null (loc->gfid)) {
-                gf_msg ("", GF_LOG_ERROR, 0,
-                        DHT_MSG_MIGRATE_FILE_FAILED,
-                        "Migrate file failed :"
-                        "loc->gfid is NULL for %s", loc->path);
-                goto out;
+        ret = syncop_setxattr(cached_subvol, loc, link_xattr, 0, NULL, NULL);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, -ret, DHT_MSG_MIGRATE_FILE_FAILED,
+                   "Migrate file failed :"
+                   "Linkto setxattr failed %s -> %s",
+                   cached_subvol->name, loc->name);
+            *fop_errno = -ret;
+            ret = -1;
+            goto out;
         }
 
-        link_xattr = dict_new ();
-        if (!link_xattr) {
-                ret = -1;
-                errno = ENOMEM;
-                goto out;
+        gf_msg_debug(this->name, 0,
+                     "hardlink target subvol created on %s "
+                     ",cached %s, file %s",
+                     hashed_subvol->name, cached_subvol->name, loc->path);
+
+        ret = -2;
+        goto out;
+    } else {
+        linkto_subvol = dht_linkfile_subvol(this, NULL, NULL, xattr_rsp);
+        if (!linkto_subvol) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, DHT_MSG_SUBVOL_ERROR,
+                   "Failed to get "
+                   "linkto subvol for %s",
+                   loc->name);
+        } else {
+            hashed_subvol = linkto_subvol;
         }
 
-        cached_subvol = dht_subvol_get_cached (this, loc->inode);
-        if (!cached_subvol) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        DHT_MSG_MIGRATE_FILE_FAILED,
-                        "Migrate file failed :"
-                        "Failed to get cached subvol"
-                        " for %s on %s", loc->name, this->name);
+        ret = syncop_link(hashed_subvol, loc, loc, &iatt, NULL, NULL);
+        if (ret) {
+            op_errno = -ret;
+            ret = -1;
+
+            loglevel = (op_errno == EEXIST) ? GF_LOG_DEBUG : GF_LOG_ERROR;
+            gf_msg(this->name, loglevel, op_errno,
+                   DHT_MSG_MIGRATE_HARDLINK_FILE_FAILED,
+                   "link of %s -> %s"
+                   " failed on  subvol %s",
+                   loc->name, uuid_utoa(loc->gfid), hashed_subvol->name);
+            if (op_errno != EEXIST) {
+                *fop_errno = op_errno;
                 goto out;
+            }
+        } else {
+            gf_msg_debug(this->name, 0,
+                         "syncop_link successful for"
+                         " hardlink %s on subvol %s, cached %s",
+                         loc->path, hashed_subvol->name, cached_subvol->name);
         }
+    }
 
-        hashed_subvol = dht_subvol_get_hashed (this, loc);
-        if (!hashed_subvol) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        DHT_MSG_MIGRATE_FILE_FAILED,
-                        "Migrate file failed :"
-                        "Failed to get hashed subvol"
-                        " for %s on %s", loc->name, this->name);
-                goto out;
+    ret = syncop_lookup(hashed_subvol, loc, &iatt, NULL, NULL, NULL);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, -ret, DHT_MSG_MIGRATE_FILE_FAILED,
+               "Migrate file failed :Failed lookup %s on %s ", loc->name,
+               hashed_subvol->name);
+
+        *fop_errno = -ret;
+        ret = -1;
+        goto out;
+    }
+
+    /* There is a race where on the target subvol for the hardlink
+     * (note: hash subvol for the hardlink might differ from this), some
+     * other client(non-rebalance) would have created a linkto file for that
+     * hardlink as part of lookup. So let say there are 10 hardlinks, on the
+     * 5th hardlink it self the hardlinks might have migrated. Now for
+     * (6..10th) hardlinks the cached and target would be same as the file
+     * has already migrated. Hence this check is needed  */
+    if (cached_subvol == hashed_subvol) {
+        gf_msg_debug(this->name, 0,
+                     "source %s and destination %s "
+                     "for hardlink %s are same",
+                     cached_subvol->name, hashed_subvol->name, loc->path);
+        ret = -2;
+        goto out;
+    }
+
+    if (iatt.ia_nlink == stbuf.ia_nlink) {
+        ret = dht_migrate_file(this, loc, cached_subvol, hashed_subvol,
+                               GF_DHT_MIGRATE_HARDLINK_IN_PROGRESS, fop_errno);
+        if (ret) {
+            goto out;
         }
+    }
+    ret = -2;
+out:
+    if (link_xattr)
+        dict_unref(link_xattr);
 
-        gf_log (this->name, GF_LOG_INFO, "Attempting to migrate hardlink %s "
-                "with gfid %s from %s -> %s", loc->name, uuid_utoa (loc->gfid),
-                cached_subvol->name, hashed_subvol->name);
-        data = dict_get (xattrs, conf->link_xattr_name);
-        /* set linkto on cached -> hashed if not present, else link it */
-        if (!data) {
-                ret = dict_set_str (link_xattr, conf->link_xattr_name,
-                                    hashed_subvol->name);
-                if (ret) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                DHT_MSG_MIGRATE_FILE_FAILED,
-                                "Migrate file failed :"
-                                "Failed to set dictionary value:"
-                                " key = %s for %s",
-                                conf->link_xattr_name, loc->name);
-                        goto out;
-                }
+    if (xattr_rsp)
+        dict_unref(xattr_rsp);
 
-                ret = syncop_setxattr (cached_subvol, loc, link_xattr, 0, NULL,
-                                       NULL);
-                if (ret) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                DHT_MSG_MIGRATE_FILE_FAILED,
-                                "Migrate file failed :"
-                                "Linkto setxattr failed %s -> %s (%s)",
-                                cached_subvol->name,
-                                loc->name, strerror (-ret));
-                        ret = -1;
-                        goto out;
-                }
-                ret = -2;
-                goto out;
-        } else {
-                linkto_subvol = dht_linkfile_subvol (this, NULL, NULL, xattrs);
-                if (!linkto_subvol) {
-                        gf_log (this->name, GF_LOG_ERROR, "Failed to get "
-                                "linkto subvol for %s", loc->name);
-                } else {
-                        hashed_subvol = linkto_subvol;
-                }
+    if (dict)
+        dict_unref(dict);
 
-                ret = syncop_link (hashed_subvol, loc, loc, NULL, NULL);
-                if  (ret) {
-                        op_errno = -ret;
-                        ret = -1;
+    return ret;
+}
 
-                        loglevel = (op_errno == EEXIST) ? GF_LOG_DEBUG : \
-                                    GF_LOG_ERROR;
-                        gf_log (this->name, loglevel, "link of %s -> %s"
-                                " failed on  subvol %s (%s)", loc->name,
-                                uuid_utoa(loc->gfid),
-                                hashed_subvol->name, strerror (op_errno));
-                        if (op_errno != EEXIST)
-                                goto out;
-                }
-        }
-        ret = syncop_lookup (hashed_subvol, loc, &iatt, NULL, NULL, NULL);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, -ret,
-                        DHT_MSG_MIGRATE_FILE_FAILED,
-                        "Migrate file failed :Failed lookup %s on %s ",
-                        loc->name, hashed_subvol->name);
+static int
+__check_file_has_hardlink(xlator_t *this, loc_t *loc, struct iatt *stbuf,
+                          dict_t *xattrs, int flags, gf_defrag_info_t *defrag,
+                          dht_conf_t *conf, int *fop_errno)
+{
+    int ret = 0;
 
-                ret = -1;
-                goto out;
+    if (flags == GF_DHT_MIGRATE_HARDLINK_IN_PROGRESS) {
+        ret = 0;
+        return ret;
+    }
+    if (stbuf->ia_nlink > 1) {
+        /* support for decomission */
+        if (flags == GF_DHT_MIGRATE_HARDLINK) {
+            synclock_lock(&conf->link_lock);
+            ret = gf_defrag_handle_hardlink(this, loc, fop_errno);
+            synclock_unlock(&conf->link_lock);
+            /*
+            Returning zero will force the file to be remigrated.
+            Checkout gf_defrag_handle_hardlink for more information.
+            */
+            if (ret && ret != -2) {
+                gf_msg(this->name, GF_LOG_WARNING, 0,
+                       DHT_MSG_MIGRATE_FILE_FAILED,
+                       "Migrate file failed:"
+                       "%s: failed to migrate file with link",
+                       loc->path);
+            }
+        } else {
+            gf_msg(this->name, GF_LOG_WARNING, 0, DHT_MSG_MIGRATE_FILE_FAILED,
+                   "Migration skipped for:"
+                   "%s: file has hardlinks",
+                   loc->path);
+            *fop_errno = ENOTSUP;
+            ret = 1;
         }
+    }
 
-        if (iatt.ia_nlink == stbuf->ia_nlink) {
-                ret = dht_migrate_file (this, loc, cached_subvol, hashed_subvol,
-                                        GF_DHT_MIGRATE_HARDLINK_IN_PROGRESS);
-                if (ret)
-                        goto out;
-        }
-        ret = -2;
-out:
-        if (link_xattr)
-                dict_unref (link_xattr);
-        return ret;
+    return ret;
 }
 
 /*
@@ -286,593 +503,962 @@ out:
          gf_defrag_handle_hardlink for description of "returning -2")
     -1 : failure
 */
-static inline int
-__is_file_migratable (xlator_t *this, loc_t *loc,
-                      struct iatt *stbuf, dict_t *xattrs, int flags)
+static int
+__is_file_migratable(xlator_t *this, loc_t *loc, struct iatt *stbuf,
+                     dict_t *xattrs, int flags, gf_defrag_info_t *defrag,
+                     dht_conf_t *conf, int *fop_errno)
 {
-        int ret = -1;
+    int ret = -1;
+    int lock_count = 0;
+
+    if (IA_ISDIR(stbuf->ia_type)) {
+        gf_msg(this->name, GF_LOG_WARNING, 0, DHT_MSG_MIGRATE_FILE_FAILED,
+               "Migrate file failed:"
+               "%s: migrate-file called on directory",
+               loc->path);
+        *fop_errno = EISDIR;
+        ret = -1;
+        goto out;
+    }
 
-        if (IA_ISDIR (stbuf->ia_type)) {
-                gf_msg (this->name, GF_LOG_WARNING, 0,
-                        DHT_MSG_MIGRATE_FILE_FAILED,
-                        "Migrate file failed:"
-                        "%s: migrate-file called on directory", loc->path);
-                ret = -1;
-                goto out;
+    if (!conf->lock_migration_enabled) {
+        ret = dict_get_int32(xattrs, GLUSTERFS_POSIXLK_COUNT, &lock_count);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_WARNING, 0, DHT_MSG_MIGRATE_FILE_FAILED,
+                   "Migrate file failed:"
+                   "%s: Unable to get lock count for file",
+                   loc->path);
+            *fop_errno = EINVAL;
+            ret = -1;
+            goto out;
         }
 
-        if (flags == GF_DHT_MIGRATE_HARDLINK_IN_PROGRESS) {
-                ret = 0;
-                goto out;
+        if (lock_count) {
+            gf_msg(this->name, GF_LOG_WARNING, 0, DHT_MSG_MIGRATE_FILE_FAILED,
+                   "Migrate file failed: %s: File has locks."
+                   " Skipping file migration",
+                   loc->path);
+            *fop_errno = ENOTSUP;
+            ret = 1;
+            goto out;
         }
-        if (stbuf->ia_nlink > 1) {
-                /* support for decomission */
-                if (flags == GF_DHT_MIGRATE_HARDLINK) {
-                        ret = gf_defrag_handle_hardlink (this, loc,
-                                                         xattrs, stbuf);
-
-                   /*
-                   Returning zero will force the file to be remigrated.
-                   Checkout gf_defrag_handle_hardlink for more information.
-                   */
-                        if (ret && ret != -2) {
-                                gf_msg (this->name, GF_LOG_WARNING, 0,
-                                        DHT_MSG_MIGRATE_FILE_FAILED,
-                                        "Migrate file failed:"
-                                        "%s: failed to migrate file with link",
-                                        loc->path);
-                        }
-                } else {
-                        gf_msg (this->name, GF_LOG_WARNING, 0,
-                                DHT_MSG_MIGRATE_FILE_FAILED,
-                                "Migrate file failed:"
-                                "%s: file has hardlinks", loc->path);
-                        ret = -ENOTSUP;
-                }
-                goto out;
-        }
-
-        ret = 0;
+    }
 
+    /* Check if file has hardlink*/
+    ret = __check_file_has_hardlink(this, loc, stbuf, xattrs, flags, defrag,
+                                    conf, fop_errno);
 out:
-        return ret;
+    return ret;
 }
 
-static inline int
-__dht_rebalance_create_dst_file (xlator_t *to, xlator_t *from, loc_t *loc, struct iatt *stbuf,
-                                 dict_t *dict, fd_t **dst_fd, dict_t *xattr)
+static int
+__dht_rebalance_create_dst_file(xlator_t *this, xlator_t *to, xlator_t *from,
+                                loc_t *loc, struct iatt *stbuf, fd_t **dst_fd,
+                                int *fop_errno, int file_has_holes)
 {
-        xlator_t    *this = NULL;
-        int          ret  = -1;
-        fd_t        *fd   = NULL;
-        struct iatt  new_stbuf = {0,};
-        struct iatt  check_stbuf= {0,};
-        dht_conf_t  *conf = NULL;
-
-        this = THIS;
-        conf = this->private;
-
-        ret = dict_set_static_bin (dict, "gfid-req", stbuf->ia_gfid, 16);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        DHT_MSG_DICT_SET_FAILED,
-                        "%s: failed to set dictionary value: key = gfid-req",
-                        loc->path);
-                goto out;
-        }
-
-        ret = dict_set_str (dict, conf->link_xattr_name, from->name);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        DHT_MSG_DICT_SET_FAILED,
-                        "%s: failed to set dictionary value: key = %s ",
-                        loc->path, conf->link_xattr_name);
-                goto out;
+    int ret = -1;
+    int ret2 = -1;
+    fd_t *fd = NULL;
+    struct iatt new_stbuf = {
+        0,
+    };
+    struct iatt check_stbuf = {
+        0,
+    };
+    dht_conf_t *conf = NULL;
+    dict_t *dict = NULL;
+    dict_t *xdata = NULL;
+
+    conf = this->private;
+
+    dict = dict_new();
+    if (!dict) {
+        *fop_errno = ENOMEM;
+        ret = -1;
+        gf_msg(this->name, GF_LOG_ERROR, ENOMEM, DHT_MSG_NO_MEMORY,
+               "dictionary allocation failed for"
+               "path:%s",
+               loc->path);
+        goto out;
+    }
+    ret = dict_set_gfuuid(dict, "gfid-req", stbuf->ia_gfid, true);
+    if (ret) {
+        *fop_errno = ENOMEM;
+        ret = -1;
+        gf_msg(this->name, GF_LOG_ERROR, 0, DHT_MSG_DICT_SET_FAILED,
+               "%s: failed to set dictionary value: key = gfid-req", loc->path);
+        goto out;
+    }
+
+    ret = dict_set_str(dict, conf->link_xattr_name, from->name);
+    if (ret) {
+        *fop_errno = ENOMEM;
+        ret = -1;
+        gf_msg(this->name, GF_LOG_ERROR, 0, DHT_MSG_DICT_SET_FAILED,
+               "%s: failed to set dictionary value: key = %s ", loc->path,
+               conf->link_xattr_name);
+        goto out;
+    }
+
+    fd = fd_create(loc->inode, DHT_REBALANCE_PID);
+    if (!fd) {
+        *fop_errno = ENOMEM;
+        ret = -1;
+        gf_msg(this->name, GF_LOG_ERROR, ENOMEM, DHT_MSG_MIGRATE_FILE_FAILED,
+               "%s: fd create failed (destination)", loc->path);
+        goto out;
+    }
+
+    xdata = dict_new();
+    if (!xdata) {
+        *fop_errno = ENOMEM;
+        ret = -1;
+        gf_msg(this->name, GF_LOG_ERROR, ENOMEM, DHT_MSG_MIGRATE_FILE_FAILED,
+               "%s: dict_new failed)", loc->path);
+        goto out;
+    }
+
+    ret = dict_set_int32_sizen(xdata, GF_CLEAN_WRITE_PROTECTION, 1);
+    if (ret) {
+        *fop_errno = ENOMEM;
+        ret = -1;
+        gf_msg(this->name, GF_LOG_ERROR, 0, DHT_MSG_DICT_SET_FAILED,
+               "%s: failed to set dictionary value: key = %s ", loc->path,
+               GF_CLEAN_WRITE_PROTECTION);
+        goto out;
+    }
+
+    ret = syncop_lookup(to, loc, &new_stbuf, NULL, xdata, NULL);
+    if (!ret) {
+        /* File exits in the destination, check if gfid matches */
+        if (gf_uuid_compare(stbuf->ia_gfid, new_stbuf.ia_gfid) != 0) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, DHT_MSG_GFID_MISMATCH,
+                   "file %s exists in %s with different gfid", loc->path,
+                   to->name);
+            *fop_errno = EINVAL;
+            ret = -1;
+            goto out;
         }
+    }
+    if ((ret < 0) && (-ret != ENOENT)) {
+        /* File exists in destination, but not accessible */
+        gf_msg(THIS->name, GF_LOG_WARNING, -ret, DHT_MSG_MIGRATE_FILE_FAILED,
+               "%s: failed to lookup file", loc->path);
+        *fop_errno = -ret;
+        ret = -1;
+        goto out;
+    }
 
-        fd = fd_create (loc->inode, DHT_REBALANCE_PID);
-        if (!fd) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        DHT_MSG_MIGRATE_FILE_FAILED,
-                        "%s: fd create failed (destination) (%s)",
-                        loc->path, strerror (errno));
-                ret = -1;
-                goto out;
+    /* Create the destination with LINKFILE mode, and linkto xattr,
+       if the linkfile already exists, just open the file */
+    if (!ret) {
+        /*
+         * File already present, just open the file.
+         */
+        ret = syncop_open(to, loc, O_RDWR, fd, NULL, NULL);
+        if (ret < 0) {
+            gf_msg(this->name, GF_LOG_ERROR, -ret, DHT_MSG_MIGRATE_FILE_FAILED,
+                   "failed to open %s on %s", loc->path, to->name);
+            *fop_errno = -ret;
+            ret = -1;
+            goto out;
         }
-
-        ret = syncop_lookup (to, loc, &new_stbuf, NULL, NULL, NULL);
-        if (!ret) {
-                /* File exits in the destination, check if gfid matches */
-                if (gf_uuid_compare (stbuf->ia_gfid, new_stbuf.ia_gfid) != 0) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                DHT_MSG_GFID_MISMATCH,
-                                "file %s exists in %s with different gfid",
-                                loc->path, to->name);
-                        fd_unref (fd);
-                        goto out;
-                }
+    } else {
+        ret = syncop_create(to, loc, O_RDWR, DHT_LINKFILE_MODE, fd, &new_stbuf,
+                            dict, NULL);
+        if (ret < 0) {
+            gf_msg(this->name, GF_LOG_ERROR, -ret, DHT_MSG_MIGRATE_FILE_FAILED,
+                   "failed to create %s on %s", loc->path, to->name);
+            *fop_errno = -ret;
+            ret = -1;
+            goto out;
         }
-        if ((ret < 0) && (-ret != ENOENT)) {
-                /* File exists in destination, but not accessible */
-                gf_msg (THIS->name, GF_LOG_WARNING, 0,
-                        DHT_MSG_MIGRATE_FILE_FAILED,
-                        "%s: failed to lookup file (%s)",
-                        loc->path, strerror (-ret));
-                ret = -1;
-                goto out;
+    }
+
+    fd_bind(fd);
+
+    /*Reason of doing lookup after create again:
+     *In the create, there is some time-gap between opening fd at the
+     *server (posix_layer) and binding it in server (incrementing fd count),
+     *so if in that time-gap, if other process sends unlink considering it
+     *as a linkto file, because inode->fd count will be 0, so file will be
+     *unlinked at the backend. And because further operations are performed
+     *on fd, so though migration will be done but will end with no file
+     *at  the backend.
+     */
+
+    ret = syncop_lookup(to, loc, &check_stbuf, NULL, NULL, NULL);
+    if (!ret) {
+        if (gf_uuid_compare(stbuf->ia_gfid, check_stbuf.ia_gfid) != 0) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, DHT_MSG_GFID_MISMATCH,
+                   "file %s exists in %s with different gfid,"
+                   "found in lookup after create",
+                   loc->path, to->name);
+            *fop_errno = EINVAL;
+            ret = -1;
+            goto out;
         }
-
-        /* Create the destination with LINKFILE mode, and linkto xattr,
-           if the linkfile already exists, it will just open the file */
-        ret = syncop_create (to, loc, O_RDWR, DHT_LINKFILE_MODE, fd,
-                             &new_stbuf, dict, NULL);
-        if (ret < 0) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        DHT_MSG_MIGRATE_FILE_FAILED,
-                        "failed to create %s on %s (%s)",
-                        loc->path, to->name, strerror (-ret));
-                ret = -1;
-                goto out;
+    }
+
+    if (-ret == ENOENT) {
+        gf_msg(this->name, GF_LOG_ERROR, -ret, DHT_MSG_MIGRATE_FILE_FAILED,
+               "%s: file does not exist"
+               "on %s",
+               loc->path, to->name);
+        *fop_errno = -ret;
+        ret = -1;
+        goto out;
+    }
+
+    ret = syncop_fsetattr(to, fd, stbuf, (GF_SET_ATTR_UID | GF_SET_ATTR_GID),
+                          NULL, NULL, NULL, NULL);
+    if (ret < 0) {
+        *fop_errno = -ret;
+        gf_msg(this->name, GF_LOG_ERROR, -ret, DHT_MSG_MIGRATE_FILE_FAILED,
+               "chown failed for %s on %s", loc->path, to->name);
+    }
+
+    /* No need to bother about 0 byte size files */
+    if (stbuf->ia_size > 0) {
+        if (conf->use_fallocate && !file_has_holes) {
+            ret = syncop_fallocate(to, fd, 0, 0, stbuf->ia_size, NULL, NULL);
+            if (ret < 0) {
+                if (ret == -EOPNOTSUPP || ret == -EINVAL || ret == -ENOSYS) {
+                    conf->use_fallocate = _gf_false;
+                } else {
+                    gf_msg(this->name, GF_LOG_ERROR, -ret,
+                           DHT_MSG_MIGRATE_FILE_FAILED,
+                           "fallocate failed for %s on %s", loc->path,
+                           to->name);
+
+                    *fop_errno = -ret;
+
+                    /* fallocate does not release the space
+                     * in some cases
+                     */
+                    ret2 = syncop_ftruncate(to, fd, 0, NULL, NULL, NULL, NULL);
+                    if (ret2 < 0) {
+                        gf_msg(this->name, GF_LOG_WARNING, -ret2,
+                               DHT_MSG_MIGRATE_FILE_FAILED,
+                               "ftruncate failed for "
+                               "%s on %s",
+                               loc->path, to->name);
+                    }
+                    goto out;
+                }
+            }
+        } else {
+            ret = syncop_ftruncate(to, fd, stbuf->ia_size, NULL, NULL, NULL,
+                                   NULL);
+            if (ret < 0) {
+                *fop_errno = -ret;
+                gf_msg(this->name, GF_LOG_WARNING, -ret,
+                       DHT_MSG_MIGRATE_FILE_FAILED,
+                       "ftruncate failed for %s on %s", loc->path, to->name);
+            }
         }
+    }
 
+    /* success */
+    ret = 0;
 
-        /*Reason of doing lookup after create again:
-         *In the create, there is some time-gap between opening fd at the
-         *server (posix_layer) and binding it in server (incrementing fd count),
-         *so if in that time-gap, if other process sends unlink considering it
-         *as a linkto file, because inode->fd count will be 0, so file will be
-         *unlinked at the backend. And because furthur operations are performed
-         *on fd, so though migration will be done but will end with no file
-         *at  the backend.
-         */
-
-
-        ret = syncop_lookup (to, loc, &check_stbuf, NULL, NULL, NULL);
-        if (!ret) {
+    if (dst_fd)
+        *dst_fd = fd;
 
-                if (gf_uuid_compare (stbuf->ia_gfid, check_stbuf.ia_gfid) != 0) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                DHT_MSG_GFID_MISMATCH,
-                                "file %s exists in %s with different gfid,"
-                                "found in lookup after create",
-                                loc->path, to->name);
-                        ret = -1;
-                        fd_unref (fd);
-                        goto out;
-                }
-
-        }
-
-        if (-ret == ENOENT) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        DHT_MSG_MIGRATE_FILE_FAILED, "%s: file does not exists"
-                        "on %s (%s)", loc->path, to->name, strerror (-ret));
-                ret = -1;
-                fd_unref (fd);
-                goto out;
+out:
+    if (ret) {
+        if (fd) {
+            fd_unref(fd);
         }
+    }
+    if (dict)
+        dict_unref(dict);
 
-        ret = syncop_fsetxattr (to, fd, xattr, 0, NULL, NULL);
-        if (ret < 0)
-                gf_msg (this->name, GF_LOG_WARNING, 0,
-                        DHT_MSG_MIGRATE_FILE_FAILED,
-                        "%s: failed to set xattr on %s (%s)",
-                        loc->path, to->name, strerror (-ret));
-
-        ret = syncop_ftruncate (to, fd, stbuf->ia_size, NULL, NULL);
-        if (ret < 0)
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        DHT_MSG_MIGRATE_FILE_FAILED,
-                        "ftruncate failed for %s on %s (%s)",
-                        loc->path, to->name, strerror (-ret));
-
-        ret = syncop_fsetattr (to, fd, stbuf,
-                               (GF_SET_ATTR_UID | GF_SET_ATTR_GID),
-                                NULL, NULL, NULL, NULL);
-        if (ret < 0)
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        DHT_MSG_MIGRATE_FILE_FAILED,
-                        "chown failed for %s on %s (%s)",
-                        loc->path, to->name, strerror (-ret));
-
-        if (dst_fd)
-                *dst_fd = fd;
-
-        /* success */
-        ret = 0;
+    if (xdata)
+        dict_unref(xdata);
 
-out:
-        return ret;
+    return ret;
 }
 
-static inline int
-__dht_check_free_space (xlator_t *to, xlator_t *from, loc_t *loc,
-                        struct iatt *stbuf, int flag)
+static int
+__dht_check_free_space(xlator_t *this, xlator_t *to, xlator_t *from, loc_t *loc,
+                       struct iatt *stbuf, int flag, dht_conf_t *conf,
+                       gf_boolean_t *target_changed, xlator_t **new_subvol,
+                       int *fop_errno)
 {
-        struct statvfs  src_statfs = {0,};
-        struct statvfs  dst_statfs = {0,};
-        int             ret        = -1;
-        xlator_t       *this       = NULL;
-        dict_t         *xdata      = NULL;
-
-        uint64_t        src_statfs_blocks = 1;
-        uint64_t        dst_statfs_blocks = 1;
-
-        this = THIS;
-
-        xdata = dict_new ();
-        if (!xdata) {
-                errno = ENOMEM;
-                gf_log (this->name, GF_LOG_ERROR,
-                        "failed to allocate dictionary");
-                goto out;
+    struct statvfs src_statfs = {
+        0,
+    };
+    struct statvfs dst_statfs = {
+        0,
+    };
+    int ret = -1;
+    dict_t *xdata = NULL;
+    dht_layout_t *layout = NULL;
+    uint64_t src_statfs_blocks = 1;
+    uint64_t dst_statfs_blocks = 1;
+    double dst_post_availspacepercent = 0;
+    double src_post_availspacepercent = 0;
+    uint64_t file_blocks = 0;
+    uint64_t src_total_blocks = 0;
+    uint64_t dst_total_blocks = 0;
+
+    xdata = dict_new();
+    if (!xdata) {
+        *fop_errno = ENOMEM;
+        ret = -1;
+        gf_msg(this->name, GF_LOG_ERROR, ENOMEM, DHT_MSG_NO_MEMORY,
+               "failed to allocate dictionary");
+        goto out;
+    }
+
+    ret = dict_set_int8(xdata, GF_INTERNAL_IGNORE_DEEM_STATFS, 1);
+    if (ret) {
+        gf_log(this->name, GF_LOG_ERROR,
+               "Failed to set " GF_INTERNAL_IGNORE_DEEM_STATFS " in dict");
+        ret = -1;
+        *fop_errno = ENOMEM;
+        goto out;
+    }
+
+    ret = syncop_statfs(from, loc, &src_statfs, xdata, NULL);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, -ret, DHT_MSG_MIGRATE_FILE_FAILED,
+               "failed to get statfs of %s on %s", loc->path, from->name);
+        *fop_errno = -ret;
+        ret = -1;
+        goto out;
+    }
+
+    ret = syncop_statfs(to, loc, &dst_statfs, xdata, NULL);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, -ret, DHT_MSG_MIGRATE_FILE_FAILED,
+               "failed to get statfs of %s on %s", loc->path, to->name);
+        *fop_errno = -ret;
+        ret = -1;
+        goto out;
+    }
+
+    gf_msg_debug(this->name, 0,
+                 "min_free_disk - %f , block available - %" PRId64
+                 ", block size - %lu",
+                 conf->min_free_disk, dst_statfs.f_bavail, dst_statfs.f_bsize);
+
+    dst_statfs_blocks = dst_statfs.f_bavail *
+                        (dst_statfs.f_frsize / GF_DISK_SECTOR_SIZE);
+
+    src_statfs_blocks = src_statfs.f_bavail *
+                        (src_statfs.f_frsize / GF_DISK_SECTOR_SIZE);
+
+    dst_total_blocks = dst_statfs.f_blocks *
+                       (dst_statfs.f_frsize / GF_DISK_SECTOR_SIZE);
+
+    src_total_blocks = src_statfs.f_blocks *
+                       (src_statfs.f_frsize / GF_DISK_SECTOR_SIZE);
+
+    /* if force option is given, do not check for space @ dst.
+     * Check only if space is avail for the file */
+    if (flag != GF_DHT_MIGRATE_DATA)
+        goto check_avail_space;
+
+    /* Check:
+       During rebalance `migrate-data` - Destination subvol experiences
+       a `reduction` in 'blocks' of free space, at the same time source
+       subvol gains certain 'blocks' of free space. A valid check is
+       necessary here to avoid erroneous move to destination where
+       the space could be scantily available.
+       With heterogeneous brick support, an actual space comparison could
+       prevent any files being migrated to newly added bricks if they are
+       smaller then the free space available on the existing bricks.
+     */
+    if (!conf->use_fallocate) {
+        file_blocks = stbuf->ia_size + GF_DISK_SECTOR_SIZE - 1;
+        file_blocks /= GF_DISK_SECTOR_SIZE;
+
+        if (file_blocks >= dst_statfs_blocks) {
+            dst_statfs_blocks = 0;
+        } else {
+            dst_statfs_blocks -= file_blocks;
         }
+    }
+
+    src_post_availspacepercent = ((src_statfs_blocks + file_blocks) * 100) /
+                                 src_total_blocks;
+
+    dst_post_availspacepercent = (dst_statfs_blocks * 100) / dst_total_blocks;
+
+    if (dst_post_availspacepercent < src_post_availspacepercent) {
+        gf_msg(this->name, GF_LOG_WARNING, 0, DHT_MSG_MIGRATE_FILE_FAILED,
+               "data movement of file "
+               "{blocks:%" PRIu64
+               " name:(%s)} would result in "
+               "dst node (%s:%" PRIu64
+               ") having lower disk "
+               "space than the source node (%s:%" PRIu64
+               ")"
+               ".Skipping file.",
+               stbuf->ia_blocks, loc->path, to->name, dst_statfs_blocks,
+               from->name, src_statfs_blocks);
+
+        /* this is not a 'failure', but we don't want to
+           consider this as 'success' too :-/ */
+        *fop_errno = ENOSPC;
+        ret = 1;
+        goto out;
+    }
 
-        ret = dict_set_int8 (xdata, GF_INTERNAL_IGNORE_DEEM_STATFS, 1);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "Failed to set "
-                        GF_INTERNAL_IGNORE_DEEM_STATFS" in dict");
-                ret = -1;
-                goto out;
+check_avail_space:
+    if (conf->disk_unit == 'p' && dst_statfs.f_blocks) {
+        dst_post_availspacepercent = (dst_statfs_blocks * 100) /
+                                     dst_total_blocks;
+
+        gf_msg_debug(this->name, 0,
+                     "file : %s, post_availspacepercent"
+                     " : %lf f_bavail : %" PRIu64 " min-free-disk: %lf",
+                     loc->path, dst_post_availspacepercent, dst_statfs.f_bavail,
+                     conf->min_free_disk);
+
+        if (dst_post_availspacepercent < conf->min_free_disk) {
+            gf_msg(this->name, GF_LOG_WARNING, 0, 0,
+                   "Write will cross min-free-disk for "
+                   "file - %s on subvol - %s. Looking "
+                   "for new subvol",
+                   loc->path, to->name);
+
+            goto find_new_subvol;
+        } else {
+            ret = 0;
+            goto out;
         }
+    }
 
-        ret = syncop_statfs (from, loc, &src_statfs, xdata, NULL);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        DHT_MSG_MIGRATE_FILE_FAILED,
-                        "failed to get statfs of %s on %s (%s)",
-                        loc->path, from->name, strerror (-ret));
-                ret = -1;
-                goto out;
-        }
+    if (conf->disk_unit != 'p') {
+        if ((dst_statfs_blocks * GF_DISK_SECTOR_SIZE) < conf->min_free_disk) {
+            gf_msg_debug(this->name, 0,
+                         "file : %s,  destination frsize: %lu "
+                         "f_bavail : %" PRIu64 " min-free-disk: %lf",
+                         loc->path, dst_statfs.f_frsize, dst_statfs.f_bavail,
+                         conf->min_free_disk);
 
-        ret = syncop_statfs (to, loc, &dst_statfs, xdata, NULL);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        DHT_MSG_MIGRATE_FILE_FAILED,
-                        "failed to get statfs of %s on %s (%s)",
-                        loc->path, to->name, strerror (-ret));
-                ret = -1;
-                goto out;
-        }
+            gf_msg(this->name, GF_LOG_WARNING, 0, 0,
+                   "write will"
+                   " cross min-free-disk for file - %s on subvol -"
+                   " %s. looking for new subvol",
+                   loc->path, to->name);
 
-        /* if force option is given, do not check for space @ dst.
-         * Check only if space is avail for the file */
-        if (flag != GF_DHT_MIGRATE_DATA)
-                goto check_avail_space;
+            goto find_new_subvol;
 
-        /* Check:
-           During rebalance `migrate-data` - Destination subvol experiences
-           a `reduction` in 'blocks' of free space, at the same time source
-           subvol gains certain 'blocks' of free space. A valid check is
-           necessary here to avoid errorneous move to destination where
-           the space could be scantily available.
-         */
-        if (stbuf) {
-                dst_statfs_blocks = ((dst_statfs.f_bavail *
-                                      dst_statfs.f_bsize) /
-                                     GF_DISK_SECTOR_SIZE);
-                src_statfs_blocks = ((src_statfs.f_bavail *
-                                      src_statfs.f_bsize) /
-                                     GF_DISK_SECTOR_SIZE);
-                if ((dst_statfs_blocks - stbuf->ia_blocks) <
-                    (src_statfs_blocks + stbuf->ia_blocks)) {
-
-                        gf_msg (this->name, GF_LOG_WARNING, 0,
-                                DHT_MSG_MIGRATE_FILE_FAILED,
-                                "data movement attempted from node "
-                                "(%s:%"PRIu64") with higher disk space "
-                                "to a node (%s:%"PRIu64") with lesser "
-                                "disk space, file { blocks:%"PRIu64", "
-                                "name:(%s) }", from->name, src_statfs_blocks,
-                                to->name, dst_statfs_blocks,
-                                stbuf->ia_blocks, loc->path);
-
-                        /* this is not a 'failure', but we don't want to
-                           consider this as 'success' too :-/ */
-                        ret = -1;
-                        goto out;
-                }
-        }
-check_avail_space:
-        if (((dst_statfs.f_bavail * dst_statfs.f_bsize) /
-              GF_DISK_SECTOR_SIZE) < stbuf->ia_blocks) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        DHT_MSG_MIGRATE_FILE_FAILED,
-                        "data movement attempted from node (%s) to node (%s) "
-                        "which does not have required free space for (%s)",
-                        from->name, to->name, loc->path);
-                ret = -1;
-                goto out;
+        } else {
+            ret = 0;
+            goto out;
         }
+    }
 
+find_new_subvol:
+    layout = dht_layout_get(this, loc->parent);
+    if (!layout) {
+        gf_log(this->name, GF_LOG_ERROR, "Layout is NULL");
+        *fop_errno = EINVAL;
+        ret = -1;
+        goto out;
+    }
+
+    *new_subvol = dht_subvol_with_free_space_inodes(this, to, from, layout,
+                                                    stbuf->ia_size);
+    if ((!(*new_subvol)) || (*new_subvol == from)) {
+        gf_msg(this->name, GF_LOG_WARNING, 0, DHT_MSG_SUBVOL_INSUFF_SPACE,
+               "Could not find any subvol"
+               " with space accommodating the file - %s. Consider "
+               "adding bricks",
+               loc->path);
+
+        *target_changed = _gf_false;
+        *fop_errno = ENOSPC;
+        ret = -1;
+    } else {
+        gf_msg(this->name, GF_LOG_INFO, 0, 0,
+               "new target found - %s"
+               " for file - %s",
+               (*new_subvol)->name, loc->path);
+        *target_changed = _gf_true;
         ret = 0;
+    }
+
 out:
-        if (xdata)
-                dict_unref (xdata);
-        return ret;
+    if (xdata)
+        dict_unref(xdata);
+    return ret;
 }
 
-static inline int
-__dht_rebalance_migrate_data (xlator_t *from, xlator_t *to, fd_t *src, fd_t *dst,
-                             uint64_t ia_size, int hole_exists)
+static int
+__dht_rebalance_migrate_data(xlator_t *this, gf_defrag_info_t *defrag,
+                             xlator_t *from, xlator_t *to, fd_t *src, fd_t *dst,
+                             uint64_t ia_size, int hole_exists, int *fop_errno)
 {
-        int            ret    = 0;
-        int            count  = 0;
-        off_t          offset = 0;
-        struct iovec  *vector = NULL;
-        struct iobref *iobref = NULL;
-        uint64_t       total  = 0;
-        size_t         read_size = 0;
-
-        /* if file size is '0', no need to enter this loop */
-        while (total < ia_size) {
-                read_size = (((ia_size - total) > DHT_REBALANCE_BLKSIZE) ?
-                             DHT_REBALANCE_BLKSIZE : (ia_size - total));
-                ret = syncop_readv (from, src, read_size,
-                                    offset, 0, &vector, &count, &iobref, NULL,
-                                    NULL);
-                if (!ret || (ret < 0)) {
-                        break;
+    int ret = 0;
+    int count = 0;
+    off_t offset = 0;
+    off_t data_offset = 0;
+    off_t hole_offset = 0;
+    struct iovec *vector = NULL;
+    struct iobref *iobref = NULL;
+    uint64_t total = 0;
+    size_t read_size = 0;
+    size_t data_block_size = 0;
+    dict_t *xdata = NULL;
+    dht_conf_t *conf = NULL;
+
+    conf = this->private;
+
+    /* if file size is '0', no need to enter this loop */
+    while (total < ia_size) {
+        /* This is a regular file - read it sequentially */
+        if (!hole_exists) {
+            read_size = (((ia_size - total) > DHT_REBALANCE_BLKSIZE)
+                             ? DHT_REBALANCE_BLKSIZE
+                             : (ia_size - total));
+        } else {
+            /* This is a sparse file - read only the data segments in the file
+             */
+
+            /* If the previous data block is fully copied, find the next data
+             * segment
+             * starting at the offset of the last read and written byte,  */
+            if (data_block_size <= 0) {
+                ret = syncop_seek(from, src, offset, GF_SEEK_DATA, NULL,
+                                  &data_offset);
+                if (ret) {
+                    if (ret == -ENXIO)
+                        ret = 0; /* No more data segments */
+                    else
+                        *fop_errno = -ret; /* Error occurred */
+
+                    break;
                 }
 
-                if (hole_exists)
-                        ret = dht_write_with_holes (to, dst, vector, count,
-                                                    ret, offset, iobref);
-                else
-                        ret = syncop_writev (to, dst, vector, count,
-                                             offset, iobref, 0, NULL, NULL);
-                if (ret < 0) {
+                /* If the position of the current data segment is greater than
+                 * the position of the next hole, find the next hole in order to
+                 * calculate the length of the new data segment */
+                if (data_offset > hole_offset) {
+                    /* Starting at the offset of the last data segment, find the
+                     * next hole */
+                    ret = syncop_seek(from, src, data_offset, GF_SEEK_HOLE,
+                                      NULL, &hole_offset);
+                    if (ret) {
+                        /* If an error occurred here it's a real error because
+                         * if the seek for a data segment was successful then
+                         * necessarily another hole must exist (EOF is a hole)
+                         */
+                        *fop_errno = -ret;
                         break;
-                }
-                offset += ret;
-                total += ret;
+                    }
 
-                GF_FREE (vector);
-                if (iobref)
-                        iobref_unref (iobref);
-                iobref = NULL;
-                vector = NULL;
+                    /* Calculate the total size of the current data block */
+                    data_block_size = hole_offset - data_offset;
+                }
+            } else {
+                /* There is still data in the current segment, move the
+                 * data_offset to the position of the last written byte */
+                data_offset = offset;
+            }
+
+            /* Calculate how much data needs to be read and written. If the data
+             * segment's length is bigger than DHT_REBALANCE_BLKSIZE, read and
+             * write DHT_REBALANCE_BLKSIZE data length and the rest in the
+             * next iteration(s) */
+            read_size = ((data_block_size > DHT_REBALANCE_BLKSIZE)
+                             ? DHT_REBALANCE_BLKSIZE
+                             : data_block_size);
+
+            /* Calculate the remaining size of the data block - maybe there's no
+             * need to seek for data in the next iteration */
+            data_block_size -= read_size;
+
+            /* Set offset to the offset of the data segment so read and write
+             * will have the correct position */
+            offset = data_offset;
         }
-        if (iobref)
-                iobref_unref (iobref);
-        GF_FREE (vector);
-
-        if (ret >= 0)
-                ret = 0;
-        else
-                ret = -1;
-
-        return ret;
-}
 
+        ret = syncop_readv(from, src, read_size, offset, 0, &vector, &count,
+                           &iobref, NULL, NULL, NULL);
 
-static inline int
-__dht_rebalance_open_src_file (xlator_t *from, xlator_t *to, loc_t *loc,
-                               struct iatt *stbuf, fd_t **src_fd)
-{
-        int          ret  = 0;
-        fd_t        *fd   = NULL;
-        dict_t      *dict = NULL;
-        xlator_t    *this = NULL;
-        struct iatt  iatt = {0,};
-        dht_conf_t  *conf = NULL;
+        if (!ret || (ret < 0)) {
+            if (!ret) {
+                /* File was probably truncated*/
+                ret = -1;
+                *fop_errno = ENOSPC;
+            } else {
+                *fop_errno = -ret;
+            }
+            break;
+        }
 
-        this = THIS;
-        conf = this->private;
+        if (!conf->force_migration) {
+            if (!xdata) {
+                xdata = dict_new();
+                if (!xdata) {
+                    gf_msg("dht", GF_LOG_ERROR, 0, DHT_MSG_MIGRATE_FILE_FAILED,
+                           "insufficient memory");
+                    ret = -1;
+                    *fop_errno = ENOMEM;
+                    break;
+                }
 
-        fd = fd_create (loc->inode, DHT_REBALANCE_PID);
-        if (!fd) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        DHT_MSG_MIGRATE_FILE_FAILED,
-                        "%s: fd create failed (source)", loc->path);
-                ret = -1;
-                goto out;
+                /* Fail this write and abort rebalance if we
+                 * detect a write from client since migration of
+                 * this file started. This is done to avoid
+                 * potential data corruption due to out of order
+                 * writes from rebalance and client to the same
+                 * region (as compared between src and dst
+                 * files). See
+                 * https://github.com/gluster/glusterfs/issues/308
+                 * for more details.
+                 */
+                ret = dict_set_int32_sizen(xdata, GF_AVOID_OVERWRITE, 1);
+                if (ret) {
+                    gf_msg("dht", GF_LOG_ERROR, 0, ENOMEM,
+                           "failed to set dict");
+                    ret = -1;
+                    *fop_errno = ENOMEM;
+                    break;
+                }
+            }
         }
 
-        ret = syncop_open (from, loc, O_RDWR, fd, NULL, NULL);
+        ret = syncop_writev(to, dst, vector, count, offset, iobref, 0, NULL,
+                            NULL, xdata, NULL);
         if (ret < 0) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        DHT_MSG_MIGRATE_FILE_FAILED,
-                        "failed to open file %s on %s (%s)",
-                        loc->path, from->name, strerror (-ret));
-                ret = -1;
-                goto out;
+            *fop_errno = -ret;
+            break;
         }
 
+        offset += ret;
+        total += ret;
+
+        GF_FREE(vector);
+        if (iobref)
+            iobref_unref(iobref);
+        iobref = NULL;
+        vector = NULL;
+    }
+    if (iobref)
+        iobref_unref(iobref);
+    GF_FREE(vector);
+
+    if (ret >= 0)
+        ret = 0;
+    else
         ret = -1;
-        dict = dict_new ();
-        if (!dict)
-                goto out;
 
-        ret = dict_set_str (dict, conf->link_xattr_name, to->name);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "failed to set xattr in dict for %s (linkto:%s)",
-                        loc->path, to->name);
-                goto out;
-        }
+    if (xdata) {
+        dict_unref(xdata);
+    }
 
-        /* Once the migration starts, the source should have 'linkto' key set
-           to show which is the target, so other clients can work around it */
-        ret = syncop_setxattr (from, loc, dict, 0, NULL, NULL);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        DHT_MSG_MIGRATE_FILE_FAILED,
-                        "failed to set xattr on %s in %s (%s)",
-                        loc->path, from->name, strerror (-ret));
-                ret = -1;
-                goto out;
-        }
+    return ret;
+}
 
-        /* mode should be (+S+T) to indicate migration is in progress */
-        iatt.ia_prot = stbuf->ia_prot;
-        iatt.ia_type = stbuf->ia_type;
-        iatt.ia_prot.sticky = 1;
-        iatt.ia_prot.sgid = 1;
+static int
+__dht_rebalance_open_src_file(xlator_t *this, xlator_t *from, xlator_t *to,
+                              loc_t *loc, struct iatt *stbuf, fd_t **src_fd,
+                              gf_boolean_t *clean_src, int *fop_errno)
+{
+    int ret = 0;
+    fd_t *fd = NULL;
+    dict_t *dict = NULL;
+    struct iatt iatt = {
+        0,
+    };
+    dht_conf_t *conf = NULL;
+
+    conf = this->private;
+
+    *clean_src = _gf_false;
+
+    fd = fd_create(loc->inode, DHT_REBALANCE_PID);
+    if (!fd) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, DHT_MSG_MIGRATE_FILE_FAILED,
+               "%s: fd create failed (source)", loc->path);
+        *fop_errno = ENOMEM;
+        ret = -1;
+        goto out;
+    }
+
+    ret = syncop_open(from, loc, O_RDWR, fd, NULL, NULL);
+    if (ret < 0) {
+        gf_msg(this->name, GF_LOG_ERROR, -ret, DHT_MSG_MIGRATE_FILE_FAILED,
+               "failed to open file %s on %s", loc->path, from->name);
+        *fop_errno = -ret;
+        ret = -1;
+        goto out;
+    }
 
-        ret = syncop_setattr (from, loc, &iatt, GF_SET_ATTR_MODE, NULL, NULL,
-                              NULL, NULL);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        DHT_MSG_MIGRATE_FILE_FAILED,
-                        "failed to set mode on %s in %s (%s)",
-                        loc->path, from->name, strerror (-ret));
-                ret = -1;
-                goto out;
-        }
+    fd_bind(fd);
 
-        if (src_fd)
-                *src_fd = fd;
+    if (src_fd)
+        *src_fd = fd;
 
-        /* success */
-        ret = 0;
+    ret = -1;
+    dict = dict_new();
+    if (!dict) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, DHT_MSG_MIGRATE_FILE_FAILED,
+               "%s: Could not allocate memory for dict", loc->path);
+        *fop_errno = ENOMEM;
+        ret = -1;
+        goto out;
+    }
+
+    ret = dict_set_str(dict, conf->link_xattr_name, to->name);
+    if (ret) {
+        gf_log(this->name, GF_LOG_ERROR,
+               "failed to set xattr in dict for %s (linkto:%s)", loc->path,
+               to->name);
+        *fop_errno = ENOMEM;
+        ret = -1;
+        goto out;
+    }
+
+    /* Once the migration starts, the source should have 'linkto' key set
+       to show which is the target, so other clients can work around it */
+    ret = syncop_setxattr(from, loc, dict, 0, NULL, NULL);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, -ret, DHT_MSG_MIGRATE_FILE_FAILED,
+               "failed to set xattr on %s in %s", loc->path, from->name);
+        *fop_errno = -ret;
+        ret = -1;
+        goto out;
+    }
+
+    /* Reset source mode/xattr if migration fails*/
+    *clean_src = _gf_true;
+
+    /* mode should be (+S+T) to indicate migration is in progress */
+    iatt.ia_prot = stbuf->ia_prot;
+    iatt.ia_type = stbuf->ia_type;
+    iatt.ia_prot.sticky = 1;
+    iatt.ia_prot.sgid = 1;
+
+    ret = syncop_setattr(from, loc, &iatt, GF_SET_ATTR_MODE, NULL, NULL, NULL,
+                         NULL);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, -ret, DHT_MSG_MIGRATE_FILE_FAILED,
+               "failed to set mode on %s in %s", loc->path, from->name);
+        *fop_errno = -ret;
+        ret = -1;
+        goto out;
+    }
+
+    /* success */
+    ret = 0;
 out:
-        if (dict)
-                dict_unref (dict);
+    if (dict)
+        dict_unref(dict);
 
-        return ret;
+    return ret;
 }
 
 int
-migrate_special_files (xlator_t *this, xlator_t *from, xlator_t *to, loc_t *loc,
-                       struct iatt *buf)
+migrate_special_files(xlator_t *this, xlator_t *from, xlator_t *to, loc_t *loc,
+                      struct iatt *buf, int *fop_errno)
 {
-        int          ret      = -1;
-        dict_t      *rsp_dict = NULL;
-        dict_t      *dict     = NULL;
-        char        *link     = NULL;
-        struct iatt  stbuf    = {0,};
-        dht_conf_t  *conf     = this->private;
-
-        dict = dict_new ();
-        if (!dict)
-                goto out;
+    int ret = -1;
+    dict_t *rsp_dict = NULL;
+    dict_t *dict = NULL;
+    char *link = NULL;
+    struct iatt stbuf = {
+        0,
+    };
+    dht_conf_t *conf = this->private;
+
+    dict = dict_new();
+    if (!dict) {
+        *fop_errno = ENOMEM;
+        ret = -1;
+        goto out;
+    }
+    ret = dict_set_int32(dict, conf->link_xattr_name, 256);
+    if (ret) {
+        *fop_errno = ENOMEM;
+        ret = -1;
+        gf_log(this->name, GF_LOG_ERROR,
+               "%s: failed to set 'linkto' key in dict", loc->path);
+        goto out;
+    }
+
+    /* check in the destination if the file is link file */
+    ret = syncop_lookup(to, loc, &stbuf, NULL, dict, &rsp_dict);
+    if ((ret < 0) && (-ret != ENOENT)) {
+        gf_msg(this->name, GF_LOG_WARNING, -ret, DHT_MSG_MIGRATE_FILE_FAILED,
+               "%s: lookup failed", loc->path);
+        *fop_errno = -ret;
+        ret = -1;
+        goto out;
+    }
+
+    /* we no more require this key */
+    dict_del(dict, conf->link_xattr_name);
+
+    /* file exists in target node, only if it is 'linkfile' its valid,
+       otherwise, error out */
+    if (!ret) {
+        if (!check_is_linkfile(loc->inode, &stbuf, rsp_dict,
+                               conf->link_xattr_name)) {
+            gf_msg(this->name, GF_LOG_WARNING, 0, DHT_MSG_MIGRATE_FILE_FAILED,
+                   "%s: file exists in destination", loc->path);
+            *fop_errno = EINVAL;
+            ret = -1;
+            goto out;
+        }
 
-        ret = dict_set_int32 (dict, conf->link_xattr_name, 256);
+        /* as file is linkfile, delete it */
+        ret = syncop_unlink(to, loc, NULL, NULL);
         if (ret) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "%s: failed to set 'linkto' key in dict", loc->path);
-                goto out;
+            gf_msg(this->name, GF_LOG_WARNING, -ret,
+                   DHT_MSG_MIGRATE_FILE_FAILED,
+                   "%s: failed to delete the linkfile", loc->path);
+            *fop_errno = -ret;
+            ret = -1;
+            goto out;
         }
+    }
 
-        /* check in the destination if the file is link file */
-        ret = syncop_lookup (to, loc, &stbuf, NULL, dict, &rsp_dict);
-        if ((ret < 0) && (-ret != ENOENT)) {
-                gf_msg (this->name, GF_LOG_WARNING, 0,
-                        DHT_MSG_MIGRATE_FILE_FAILED,
-                        "%s: lookup failed (%s)",
-                        loc->path, strerror (-ret));
-                ret = -1;
-                goto out;
+    /* Set the gfid of the source file in dict */
+    ret = dict_set_gfuuid(dict, "gfid-req", buf->ia_gfid, true);
+    if (ret) {
+        *fop_errno = ENOMEM;
+        ret = -1;
+        gf_log(this->name, GF_LOG_ERROR,
+               "%s: failed to set gfid in dict for create", loc->path);
+        goto out;
+    }
+
+    /* Create the file in target */
+    if (IA_ISLNK(buf->ia_type)) {
+        /* Handle symlinks separately */
+        ret = syncop_readlink(from, loc, &link, buf->ia_size, NULL, NULL);
+        if (ret < 0) {
+            gf_msg(this->name, GF_LOG_WARNING, -ret,
+                   DHT_MSG_MIGRATE_FILE_FAILED,
+                   "%s: readlink on symlink failed", loc->path);
+            *fop_errno = -ret;
+            ret = -1;
+            goto out;
         }
 
-        /* we no more require this key */
-        dict_del (dict, conf->link_xattr_name);
+        ret = syncop_symlink(to, loc, link, 0, dict, NULL);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_WARNING, -ret,
+                   DHT_MSG_MIGRATE_FILE_FAILED, "%s: creating symlink failed",
+                   loc->path);
+            *fop_errno = -ret;
+            ret = -1;
+            goto out;
+        }
 
-        /* file exists in target node, only if it is 'linkfile' its valid,
-           otherwise, error out */
-        if (!ret) {
-                if (!check_is_linkfile (loc->inode, &stbuf, rsp_dict,
-                                        conf->link_xattr_name)) {
-                        gf_msg (this->name, GF_LOG_WARNING, 0,
-                                DHT_MSG_MIGRATE_FILE_FAILED,
-                                "%s: file exists in destination", loc->path);
-                        ret = -1;
-                        goto out;
-                }
+        goto done;
+    }
 
-                /* as file is linkfile, delete it */
-                ret = syncop_unlink (to, loc, NULL, NULL);
-                if (ret) {
-                        gf_msg (this->name, GF_LOG_WARNING, 0,
-                                DHT_MSG_MIGRATE_FILE_FAILED,
-                                "%s: failed to delete the linkfile (%s)",
-                                loc->path, strerror (-ret));
-                        ret = -1;
-                        goto out;
-                }
-        }
+    ret = syncop_mknod(to, loc, st_mode_from_ia(buf->ia_prot, buf->ia_type),
+                       makedev(ia_major(buf->ia_rdev), ia_minor(buf->ia_rdev)),
+                       0, dict, NULL);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_WARNING, -ret, DHT_MSG_MIGRATE_FILE_FAILED,
+               "%s: mknod failed", loc->path);
+        *fop_errno = -ret;
+        ret = -1;
+        goto out;
+    }
 
-        /* Set the gfid of the source file in dict */
-        ret = dict_set_static_bin (dict, "gfid-req", buf->ia_gfid, 16);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "%s: failed to set gfid in dict for create", loc->path);
-                goto out;
-        }
+done:
+    ret = syncop_setattr(to, loc, buf,
+                         (GF_SET_ATTR_MTIME | GF_SET_ATTR_UID |
+                          GF_SET_ATTR_GID | GF_SET_ATTR_MODE),
+                         NULL, NULL, NULL, NULL);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_WARNING, -ret, DHT_MSG_MIGRATE_FILE_FAILED,
+               "%s: failed to perform setattr on %s", loc->path, to->name);
+        *fop_errno = -ret;
+    }
+
+    ret = syncop_unlink(from, loc, NULL, NULL);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_WARNING, -ret, DHT_MSG_MIGRATE_FILE_FAILED,
+               "%s: unlink failed", loc->path);
+        *fop_errno = -ret;
+        ret = -1;
+    }
 
-        /* Create the file in target */
-        if (IA_ISLNK (buf->ia_type)) {
-                /* Handle symlinks separately */
-                ret = syncop_readlink (from, loc, &link, buf->ia_size, NULL,
-                                       NULL);
-                if (ret < 0) {
-                        gf_msg (this->name, GF_LOG_WARNING, 0,
-                                DHT_MSG_MIGRATE_FILE_FAILED,
-                                "%s: readlink on symlink failed (%s)",
-                                loc->path, strerror (-ret));
-                        ret = -1;
-                        goto out;
-                }
+out:
+    GF_FREE(link);
+    if (dict)
+        dict_unref(dict);
 
-                ret = syncop_symlink (to, loc, link, 0, dict, NULL);
-                if (ret) {
-                        gf_msg (this->name, GF_LOG_WARNING, 0,
-                                DHT_MSG_MIGRATE_FILE_FAILED,
-                                "%s: creating symlink failed (%s)",
-                                loc->path, strerror (-ret));
-                        ret = -1;
-                        goto out;
-                }
+    if (rsp_dict)
+        dict_unref(rsp_dict);
 
-                goto done;
-        }
+    return ret;
+}
 
-        ret = syncop_mknod (to, loc, st_mode_from_ia (buf->ia_prot,
-                                                      buf->ia_type),
-                            makedev (ia_major (buf->ia_rdev),
-                                     ia_minor (buf->ia_rdev)), 0, dict, NULL);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_WARNING, 0,
-                        DHT_MSG_MIGRATE_FILE_FAILED,
-                        "%s: mknod failed (%s)",
-                        loc->path, strerror (-ret));
-                ret = -1;
-                goto out;
-        }
+static int
+__dht_migration_cleanup_src_file(xlator_t *this, loc_t *loc, fd_t *fd,
+                                 xlator_t *from, ia_prot_t *src_ia_prot)
+{
+    int ret = -1;
+    dht_conf_t *conf = NULL;
+    struct iatt new_stbuf = {
+        0,
+    };
+
+    if (!this || !fd || !from || !src_ia_prot) {
+        goto out;
+    }
+
+    conf = this->private;
+
+    /*Revert source mode and xattr changes*/
+    ret = syncop_fstat(from, fd, &new_stbuf, NULL, NULL);
+    if (ret < 0) {
+        /* Failed to get the stat info */
+        gf_msg(this->name, GF_LOG_ERROR, -ret, DHT_MSG_MIGRATE_FILE_FAILED,
+               "Migrate file cleanup failed: failed to fstat "
+               "file %s on %s ",
+               loc->path, from->name);
+        ret = -1;
+        goto out;
+    }
 
-done:
-        ret = syncop_setattr (to, loc, buf,
-                              (GF_SET_ATTR_MTIME |
-                               GF_SET_ATTR_UID | GF_SET_ATTR_GID |
-                               GF_SET_ATTR_MODE), NULL, NULL, NULL, NULL);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_WARNING, 0,
-                        DHT_MSG_MIGRATE_FILE_FAILED,
-                        "%s: failed to perform setattr on %s (%s)",
-                        loc->path, to->name, strerror (-ret));
-                ret = -1;
-        }
+    /* Remove the sticky bit and sgid bit set, reset it to 0*/
+    if (!src_ia_prot->sticky)
+        new_stbuf.ia_prot.sticky = 0;
 
-        ret = syncop_unlink (from, loc, NULL, NULL);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_WARNING, 0,
-                        DHT_MSG_MIGRATE_FILE_FAILED,
-                        "%s: unlink failed (%s)",
-                        loc->path, strerror (-ret));
-                ret = -1;
-        }
+    if (!src_ia_prot->sgid)
+        new_stbuf.ia_prot.sgid = 0;
 
-out:
-        if (dict)
-                dict_unref (dict);
+    ret = syncop_fsetattr(from, fd, &new_stbuf,
+                          (GF_SET_ATTR_GID | GF_SET_ATTR_MODE), NULL, NULL,
+                          NULL, NULL);
 
-        if (rsp_dict)
-                dict_unref (rsp_dict);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_WARNING, -ret, DHT_MSG_MIGRATE_FILE_FAILED,
+               "Migrate file cleanup failed:"
+               "%s: failed to perform fsetattr on %s ",
+               loc->path, from->name);
+        ret = -1;
+        goto out;
+    }
+
+    ret = syncop_fremovexattr(from, fd, conf->link_xattr_name, 0, NULL);
+    if (ret) {
+        gf_log(this->name, GF_LOG_WARNING,
+               "%s: failed to remove linkto xattr on %s (%s)", loc->path,
+               from->name, strerror(-ret));
+        ret = -1;
+        goto out;
+    }
 
-        return ret;
+    ret = 0;
+
+out:
+    return ret;
 }
 
 /*
@@ -883,1316 +1469,3234 @@ out:
     1 : not a failure, but we can't migrate data as of now
 */
 int
-dht_migrate_file (xlator_t *this, loc_t *loc, xlator_t *from, xlator_t *to,
-                  int flag)
+dht_migrate_file(xlator_t *this, loc_t *loc, xlator_t *from, xlator_t *to,
+                 int flag, int *fop_errno)
 {
-        int             ret                  = -1;
-        struct iatt     new_stbuf            = {0,};
-        struct iatt     stbuf                = {0,};
-        struct iatt     empty_iatt           = {0,};
-        ia_prot_t       src_ia_prot          = {0,};
-        fd_t           *src_fd               = NULL;
-        fd_t           *dst_fd               = NULL;
-        dict_t         *dict                 = NULL;
-        dict_t         *xattr                = NULL;
-        dict_t         *xattr_rsp            = NULL;
-        int             file_has_holes       = 0;
-        dht_conf_t     *conf                 = this->private;
-        int             rcvd_enoent_from_src = 0;
-        struct gf_flock flock                = {0, };
-        loc_t           tmp_loc              = {0, };
-        gf_boolean_t    locked               = _gf_false;
-        int             lk_ret               = -1;
-
-        gf_log (this->name, GF_LOG_INFO, "%s: attempting to move from %s to %s",
-                loc->path, from->name, to->name);
-
-        dict = dict_new ();
-        if (!dict)
-                goto out;
+    int ret = -1;
+    struct iatt new_stbuf = {
+        0,
+    };
+    struct iatt stbuf = {
+        0,
+    };
+    struct iatt empty_iatt = {
+        0,
+    };
+    ia_prot_t src_ia_prot = {
+        0,
+    };
+    fd_t *src_fd = NULL;
+    fd_t *dst_fd = NULL;
+    dict_t *dict = NULL;
+    dict_t *xattr = NULL;
+    dict_t *xattr_rsp = NULL;
+    int file_has_holes = 0;
+    dht_conf_t *conf = this->private;
+    int rcvd_enoent_from_src = 0;
+    struct gf_flock flock = {
+        0,
+    };
+    struct gf_flock plock = {
+        0,
+    };
+    loc_t tmp_loc = {
+        0,
+    };
+    loc_t parent_loc = {
+        0,
+    };
+    gf_boolean_t inodelk_locked = _gf_false;
+    gf_boolean_t entrylk_locked = _gf_false;
+    gf_boolean_t p_locked = _gf_false;
+    int lk_ret = -1;
+    gf_defrag_info_t *defrag = NULL;
+    gf_boolean_t clean_src = _gf_false;
+    gf_boolean_t clean_dst = _gf_false;
+    int log_level = GF_LOG_INFO;
+    gf_boolean_t delete_src_linkto = _gf_true;
+    lock_migration_info_t locklist;
+    dict_t *meta_dict = NULL;
+    gf_boolean_t meta_locked = _gf_false;
+    gf_boolean_t target_changed = _gf_false;
+    xlator_t *new_target = NULL;
+    xlator_t *old_target = NULL;
+    xlator_t *hashed_subvol = NULL;
+    fd_t *linkto_fd = NULL;
+    dict_t *xdata = NULL;
+
+    if (from == to) {
+        gf_msg_debug(this->name, 0,
+                     "destination and source are same. file %s"
+                     " might have migrated already",
+                     loc->path);
+        ret = 0;
+        goto out;
+    }
 
-        ret = dict_set_int32 (dict, conf->link_xattr_name, 256);
+    gf_log(this->name, log_level, "%s: attempting to move from %s to %s",
+           loc->path, from->name, to->name);
+
+    dict = dict_new();
+    if (!dict) {
+        ret = -1;
+        *fop_errno = ENOMEM;
+        gf_msg(this->name, GF_LOG_ERROR, ENOMEM, DHT_MSG_NO_MEMORY,
+               "Could not allocate memory for dict");
+        goto out;
+    }
+    ret = dict_set_int32(dict, conf->link_xattr_name, 256);
+    if (ret) {
+        *fop_errno = ENOMEM;
+        ret = -1;
+        gf_msg(this->name, GF_LOG_ERROR, 0, DHT_MSG_MIGRATE_FILE_FAILED,
+               "Migrate file failed:"
+               "%s: failed to set 'linkto' key in dict",
+               loc->path);
+        goto out;
+    }
+
+    /* Do not migrate file in case lock migration is not enabled on the
+     * volume*/
+    if (!conf->lock_migration_enabled) {
+        ret = dict_set_int32(dict, GLUSTERFS_POSIXLK_COUNT, sizeof(int32_t));
         if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        DHT_MSG_MIGRATE_FILE_FAILED,
-                        "Migrate file failed:"
-                        "%s: failed to set 'linkto' key in dict", loc->path);
-                goto out;
+            *fop_errno = ENOMEM;
+            ret = -1;
+            gf_msg(this->name, GF_LOG_ERROR, 0, DHT_MSG_MIGRATE_FILE_FAILED,
+                   "Migrate file failed: %s: failed to "
+                   "set " GLUSTERFS_POSIXLK_COUNT " key in dict",
+                   loc->path);
+            goto out;
+        }
+    } else {
+        gf_msg(this->name, GF_LOG_INFO, 0, 0,
+               "locks will be migrated"
+               " for file: %s",
+               loc->path);
+    }
+
+    /* The file is locked to prevent a rename during a migration. Renames
+     * and migrations on the file at the same time can lead to data loss.
+     */
+
+    ret = dht_build_parent_loc(this, &parent_loc, loc, fop_errno);
+    if (ret < 0) {
+        ret = -1;
+        gf_msg(this->name, GF_LOG_WARNING, *fop_errno,
+               DHT_MSG_MIGRATE_FILE_FAILED,
+               "%s: failed to build parent loc, which is needed to "
+               "acquire entrylk to synchronize with renames on this "
+               "path. Skipping migration",
+               loc->path);
+        goto out;
+    }
+
+    hashed_subvol = dht_subvol_get_hashed(this, loc);
+    if (hashed_subvol == NULL) {
+        ret = -1;
+        gf_msg(this->name, GF_LOG_WARNING, EINVAL, DHT_MSG_MIGRATE_FILE_FAILED,
+               "%s: cannot find hashed subvol which is needed to "
+               "synchronize with renames on this path. "
+               "Skipping migration",
+               loc->path);
+        goto out;
+    }
+
+    flock.l_type = F_WRLCK;
+
+    tmp_loc.inode = inode_ref(loc->inode);
+    gf_uuid_copy(tmp_loc.gfid, loc->gfid);
+    tmp_loc.path = gf_strdup(loc->path);
+
+    /* this inodelk happens with flock.owner being zero. But to synchronize
+     * hardlink migration we need to have different lkowner for each migration
+     * Filed a bug here: https://bugzilla.redhat.com/show_bug.cgi?id=1468202 to
+     * track the fix for this. Currently synclock takes care of synchronizing
+     * hardlink migration. Once this bug is fixed we can avoid taking synclock
+     */
+    ret = syncop_inodelk(from, DHT_FILE_MIGRATE_DOMAIN, &tmp_loc, F_SETLKW,
+                         &flock, NULL, NULL);
+    if (ret < 0) {
+        *fop_errno = -ret;
+        ret = -1;
+        gf_msg(this->name, GF_LOG_WARNING, *fop_errno,
+               DHT_MSG_MIGRATE_FILE_FAILED,
+               "migrate file failed: "
+               "%s: failed to lock file on %s",
+               loc->path, from->name);
+        goto out;
+    }
+
+    inodelk_locked = _gf_true;
+
+    /* dht_rename has changed to use entrylk on hashed subvol for
+     * synchronization. So, rebalance too has to acquire an entrylk on
+     * hashed subvol.
+     */
+    ret = syncop_entrylk(hashed_subvol, DHT_ENTRY_SYNC_DOMAIN, &parent_loc,
+                         loc->name, ENTRYLK_LOCK, ENTRYLK_WRLCK, NULL, NULL);
+    if (ret < 0) {
+        *fop_errno = -ret;
+        ret = -1;
+        gf_msg(this->name, GF_LOG_WARNING, *fop_errno,
+               DHT_MSG_MIGRATE_FILE_FAILED,
+               "%s: failed to acquire entrylk on subvol %s", loc->path,
+               hashed_subvol->name);
+        goto out;
+    }
+
+    entrylk_locked = _gf_true;
+
+    /* Phase 1 - Data migration is in progress from now on */
+    ret = syncop_lookup(from, loc, &stbuf, NULL, dict, &xattr_rsp);
+    if (ret) {
+        *fop_errno = -ret;
+        ret = -1;
+        gf_msg(this->name, GF_LOG_ERROR, *fop_errno,
+               DHT_MSG_MIGRATE_FILE_FAILED,
+               "Migrate file failed:"
+               "%s: lookup failed on %s",
+               loc->path, from->name);
+        goto out;
+    }
+
+    /* preserve source mode, so set the same to the destination */
+    src_ia_prot = stbuf.ia_prot;
+
+    /* Check if file can be migrated */
+    ret = __is_file_migratable(this, loc, &stbuf, xattr_rsp, flag, defrag, conf,
+                               fop_errno);
+    if (ret) {
+        if (ret == HARDLINK_MIG_INPROGRESS)
+            ret = 0;
+        goto out;
+    }
+
+    /* Take care of the special files */
+    if (!IA_ISREG(stbuf.ia_type)) {
+        /* Special files */
+        ret = migrate_special_files(this, from, to, loc, &stbuf, fop_errno);
+        goto out;
+    }
+
+    /* Try to preserve 'holes' while migrating data */
+    if (stbuf.ia_size > (stbuf.ia_blocks * GF_DISK_SECTOR_SIZE))
+        file_has_holes = 1;
+
+    /* create the destination, with required modes/xattr */
+    ret = __dht_rebalance_create_dst_file(this, to, from, loc, &stbuf, &dst_fd,
+                                          fop_errno, file_has_holes);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, 0,
+               "Create dst failed"
+               " on - %s for file - %s",
+               to->name, loc->path);
+        goto out;
+    }
+
+    clean_dst = _gf_true;
+
+    ret = __dht_check_free_space(this, to, from, loc, &stbuf, flag, conf,
+                                 &target_changed, &new_target, fop_errno);
+    if (target_changed) {
+        /* Can't handle for hardlinks. Marking this as failure */
+        if (flag == GF_DHT_MIGRATE_HARDLINK_IN_PROGRESS || stbuf.ia_nlink > 1) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, DHT_MSG_SUBVOL_INSUFF_SPACE,
+                   "Exiting migration for"
+                   " file - %s. flag - %d, stbuf.ia_nlink - %d",
+                   loc->path, flag, stbuf.ia_nlink);
+            ret = -1;
+            goto out;
         }
 
-        flock.l_type = F_WRLCK;
+        ret = syncop_ftruncate(to, dst_fd, 0, NULL, NULL, NULL, NULL);
+        if (ret) {
+            gf_log(this->name, GF_LOG_WARNING,
+                   "%s: failed to perform truncate on %s (%s)", loc->path,
+                   to->name, strerror(-ret));
+        }
 
-        tmp_loc.inode = inode_ref (loc->inode);
-        gf_uuid_copy (tmp_loc.gfid, loc->gfid);
-        tmp_loc.path = gf_strdup(loc->path);
+        syncop_close(dst_fd);
+        dst_fd = NULL;
 
-        ret = syncop_inodelk (from, DHT_FILE_MIGRATE_DOMAIN, &tmp_loc, F_SETLKW,
-                              &flock, NULL, NULL);
-        if (ret < 0) {
-                gf_msg (this->name, GF_LOG_WARNING, 0,
-                        DHT_MSG_MIGRATE_FILE_FAILED,
-                        "migrate file failed: "
-                        "%s: failed to lock file on %s (%s)",
-                        loc->path, from->name, strerror (-ret));
-                ret = -1;
-                goto out;
-        }
+        old_target = to;
+        to = new_target;
 
-        locked = _gf_true;
+        clean_dst = _gf_false;
 
-        /* Phase 1 - Data migration is in progress from now on */
-        ret = syncop_lookup (from, loc, &stbuf, NULL, dict, &xattr_rsp);
+        /* if the file migration is successful to this new target, then
+         * update the xattr on the old destination to point the new
+         * destination. We need to do update this only post migration
+         * as in case of failure the linkto needs to point to the source
+         * subvol */
+        ret = __dht_rebalance_create_dst_file(
+            this, to, from, loc, &stbuf, &dst_fd, fop_errno, file_has_holes);
         if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        DHT_MSG_MIGRATE_FILE_FAILED,
-                        "Migrate file failed:"
-                        "%s: lookup failed on %s (%s)",
-                        loc->path, from->name, strerror (-ret));
-                ret = -1;
-                goto out;
+            gf_log(this->name, GF_LOG_ERROR,
+                   "Create dst failed"
+                   " on - %s for file - %s",
+                   to->name, loc->path);
+            goto out;
+        } else {
+            gf_msg(this->name, GF_LOG_INFO, 0, 0,
+                   "destination for file "
+                   "- %s is changed to - %s",
+                   loc->path, to->name);
+            clean_dst = _gf_true;
         }
+    }
+
+    if (ret) {
+        goto out;
+    }
+
+    /* Open the source, and also update mode/xattr */
+    ret = __dht_rebalance_open_src_file(this, from, to, loc, &stbuf, &src_fd,
+                                        &clean_src, fop_errno);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, DHT_MSG_MIGRATE_FILE_FAILED,
+               "Migrate file failed: failed to open %s on %s", loc->path,
+               from->name);
+        goto out;
+    }
+
+    /* TODO: move all xattr related operations to fd based operations */
+    ret = syncop_listxattr(from, loc, &xattr, NULL, NULL);
+    if (ret < 0) {
+        *fop_errno = -ret;
+        gf_msg(this->name, GF_LOG_WARNING, *fop_errno,
+               DHT_MSG_MIGRATE_FILE_FAILED,
+               "Migrate file failed:"
+               "%s: failed to get xattr from %s",
+               loc->path, from->name);
+        ret = -1;
+        goto out;
+    }
+
+    /* Copying posix acls to the linkto file messes up the permissions*/
+    dht_strip_out_acls(xattr);
+
+    /* Remove the linkto xattr as we don't want to overwrite the value
+     * set on the dst.
+     */
+    dict_del(xattr, conf->link_xattr_name);
+
+    /* We need to error out if this fails as having the wrong shard xattrs
+     * set on the dst could cause data corruption
+     */
+    ret = syncop_fsetxattr(to, dst_fd, xattr, 0, NULL, NULL);
+    if (ret < 0) {
+        *fop_errno = -ret;
+        gf_msg(this->name, GF_LOG_WARNING, -ret, DHT_MSG_MIGRATE_FILE_FAILED,
+               "%s: failed to set xattr on %s", loc->path, to->name);
+        ret = -1;
+        goto out;
+    }
 
+    if (xattr_rsp) {
         /* we no more require this key */
-        dict_del (dict, conf->link_xattr_name);
-
-        /* preserve source mode, so set the same to the destination */
-        src_ia_prot = stbuf.ia_prot;
+        dict_del(dict, conf->link_xattr_name);
+        dict_unref(xattr_rsp);
+    }
+
+    ret = syncop_fstat(from, src_fd, &stbuf, dict, &xattr_rsp);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, -ret, DHT_MSG_MIGRATE_FILE_FAILED,
+               "Migrate file failed:failed to lookup %s on %s ", loc->path,
+               from->name);
+        *fop_errno = -ret;
+        ret = -1;
+        goto out;
+    }
+
+    /* Check again if file has hardlink */
+    ret = __check_file_has_hardlink(this, loc, &stbuf, xattr_rsp, flag, defrag,
+                                    conf, fop_errno);
+    if (ret) {
+        if (ret == HARDLINK_MIG_INPROGRESS)
+            ret = 0;
+        goto out;
+    }
+
+    ret = __dht_rebalance_migrate_data(this, defrag, from, to, src_fd, dst_fd,
+                                       stbuf.ia_size, file_has_holes,
+                                       fop_errno);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, DHT_MSG_MIGRATE_FILE_FAILED,
+               "Migrate file failed: %s: failed to migrate data", loc->path);
 
-        /* Check if file can be migrated */
-        ret = __is_file_migratable (this, loc, &stbuf, xattr_rsp, flag);
-        if (ret) {
-                if (ret == -2)
-                        ret = 0;
-                goto out;
-        }
-        /* Take care of the special files */
-        if (!IA_ISREG (stbuf.ia_type)) {
-                /* Special files */
-                ret = migrate_special_files (this, from, to, loc, &stbuf);
-                goto out;
+        ret = -1;
+        goto out;
+    }
+
+    /* TODO: Sync the locks */
+
+    xdata = dict_new();
+    if (!xdata || dict_set_int8(xdata, "last-fsync", 1)) {
+        gf_log(this->name, GF_LOG_ERROR,
+               "%s: failed to set last-fsync flag on "
+               "%s (%s)",
+               loc->path, to->name, strerror(ENOMEM));
+    }
+
+    ret = syncop_fsync(to, dst_fd, 0, NULL, NULL, xdata, NULL);
+    if (ret) {
+        gf_log(this->name, GF_LOG_WARNING, "%s: failed to fsync on %s (%s)",
+               loc->path, to->name, strerror(-ret));
+        *fop_errno = -ret;
+    }
+
+    /* Phase 2 - Data-Migration Complete, Housekeeping updates pending */
+
+    ret = syncop_fstat(from, src_fd, &new_stbuf, NULL, NULL);
+    if (ret < 0) {
+        /* Failed to get the stat info */
+        gf_msg(this->name, GF_LOG_ERROR, -ret, DHT_MSG_MIGRATE_FILE_FAILED,
+               "Migrate file failed: failed to fstat file %s on %s ", loc->path,
+               from->name);
+        *fop_errno = -ret;
+        ret = -1;
+        goto out;
+    }
+
+    /* Lock the entire source file to prevent clients from taking a
+       lock on it as dht_lk does not handle file migration.
+
+       This still leaves a small window where conflicting locks can
+       be granted to different clients. If client1 requests a blocking
+       lock on the src file, it will be granted after the migrating
+       process releases its lock. If client2 requests a lock on the dst
+       data file, it will also be granted, but all FOPs will be redirected
+       to the dst data file.
+    */
+
+    /* Take meta lock  */
+
+    if (conf->lock_migration_enabled) {
+        meta_dict = dict_new();
+        if (!meta_dict) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, DHT_MSG_MIGRATE_FILE_FAILED,
+                   "dict_new failed");
+
+            *fop_errno = ENOMEM;
+            ret = -1;
+            goto out;
         }
 
-        /* TODO: move all xattr related operations to fd based operations */
-        ret = syncop_listxattr (from, loc, &xattr, NULL, NULL);
-        if (ret < 0) {
-                gf_msg (this->name, GF_LOG_WARNING, 0,
-                        DHT_MSG_MIGRATE_FILE_FAILED,
-                        "Migrate file failed:"
-                        "%s: failed to get xattr from %s (%s)",
-                        loc->path, from->name, strerror (-ret));
-                ret = -1;
+        ret = dict_set_str(meta_dict, GLUSTERFS_INTERNAL_FOP_KEY, "yes");
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, DHT_MSG_DICT_SET_FAILED,
+                   "Failed to set dictionary value: key = %s,"
+                   " path = %s",
+                   GLUSTERFS_INTERNAL_FOP_KEY, loc->path);
+            *fop_errno = ENOMEM;
+            ret = -1;
+            goto out;
         }
 
-        /* create the destination, with required modes/xattr */
-        ret = __dht_rebalance_create_dst_file (to, from, loc, &stbuf,
-                                               dict, &dst_fd, xattr);
-        if (ret)
-                goto out;
-
-        ret = __dht_check_free_space (to, from, loc, &stbuf, flag);
-
+        ret = dict_set_int32(meta_dict, GF_META_LOCK_KEY, 1);
         if (ret) {
-                goto out;
+            gf_msg(this->name, GF_LOG_ERROR, 0, DHT_MSG_MIGRATE_FILE_FAILED,
+                   "Trace dict_set failed");
+            *fop_errno = ENOMEM;
+            ret = -1;
+            goto out;
         }
 
-        /* Open the source, and also update mode/xattr */
-        ret = __dht_rebalance_open_src_file (from, to, loc, &stbuf, &src_fd);
+        ret = syncop_setxattr(from, loc, meta_dict, 0, NULL, NULL);
         if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        DHT_MSG_MIGRATE_FILE_FAILED,
-                        "Migrate file failed: failed to open %s on %s",
-                        loc->path, from->name);
-                goto out;
+            gf_msg(this->name, GF_LOG_ERROR, -ret, DHT_MSG_MIGRATE_FILE_FAILED,
+                   "Trace syncop_setxattr metalock failed");
+
+            *fop_errno = -ret;
+            ret = -1;
+            goto out;
+        } else {
+            meta_locked = _gf_true;
         }
+    }
 
+    if (!conf->lock_migration_enabled) {
+        plock.l_type = F_WRLCK;
+        plock.l_start = 0;
+        plock.l_len = 0;
+        plock.l_whence = SEEK_SET;
 
-        ret = syncop_fstat (from, src_fd, &stbuf, NULL, NULL);
+        ret = syncop_lk(from, src_fd, F_SETLK, &plock, NULL, NULL);
         if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, -ret,
-                        DHT_MSG_MIGRATE_FILE_FAILED,
-                        "Migrate file failed:failed to lookup %s on %s ",
-                        loc->path, from->name);
-                ret = -1;
-                goto out;
+            gf_msg(this->name, GF_LOG_ERROR, -ret, DHT_MSG_MIGRATE_FILE_FAILED,
+                   "Migrate file failed:"
+                   "%s: Failed to lock on %s",
+                   loc->path, from->name);
+            *fop_errno = -ret;
+            ret = -1;
+            goto out;
         }
 
-        /* Try to preserve 'holes' while migrating data */
-        if (stbuf.ia_size > (stbuf.ia_blocks * GF_DISK_SECTOR_SIZE))
-                file_has_holes = 1;
+        p_locked = _gf_true;
+
+    } else {
+        INIT_LIST_HEAD(&locklist.list);
 
-        /* All I/O happens in this function */
-        ret = __dht_rebalance_migrate_data (from, to, src_fd, dst_fd,
-					    stbuf.ia_size, file_has_holes);
+        ret = syncop_getactivelk(from, loc, &locklist, NULL, NULL);
+        if (ret == 0) {
+            gf_log(this->name, GF_LOG_INFO, "No active locks on:%s", loc->path);
+
+        } else if (ret > 0) {
+            ret = syncop_setactivelk(to, loc, &locklist, NULL, NULL);
+            if (ret) {
+                gf_msg(this->name, GF_LOG_ERROR, -ret,
+                       DHT_MSG_LOCK_MIGRATION_FAILED, "write lock failed on:%s",
+                       loc->path);
+
+                *fop_errno = -ret;
+                ret = -1;
+                goto metaunlock;
+            }
+        } else {
+            gf_msg(this->name, GF_LOG_ERROR, -ret,
+                   DHT_MSG_LOCK_MIGRATION_FAILED,
+                   "getactivelk failed for file: %s", loc->path);
+            *fop_errno = -ret;
+        }
+    }
+
+    /* source would have both sticky bit and sgid bit set, reset it to 0,
+       and set the source permission on destination, if it was not set
+       prior to setting rebalance-modes in source  */
+    if (!src_ia_prot.sticky)
+        new_stbuf.ia_prot.sticky = 0;
+
+    if (!src_ia_prot.sgid)
+        new_stbuf.ia_prot.sgid = 0;
+
+    /* TODO: if the source actually had sticky bit, or sgid bit set,
+       we are not handling it */
+
+    ret = syncop_fsetattr(
+        to, dst_fd, &new_stbuf,
+        (GF_SET_ATTR_UID | GF_SET_ATTR_GID | GF_SET_ATTR_MODE), NULL, NULL,
+        NULL, NULL);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_WARNING, -ret, DHT_MSG_MIGRATE_FILE_FAILED,
+               "Migrate file failed:"
+               "%s: failed to perform setattr on %s ",
+               loc->path, to->name);
+        *fop_errno = -ret;
+        ret = -1;
+        goto metaunlock;
+    }
+
+    /* Because 'futimes' is not portable */
+    ret = syncop_setattr(to, loc, &new_stbuf,
+                         (GF_SET_ATTR_MTIME | GF_SET_ATTR_ATIME), NULL, NULL,
+                         NULL, NULL);
+    if (ret) {
+        gf_log(this->name, GF_LOG_WARNING,
+               "%s: failed to perform setattr on %s ", loc->path, to->name);
+        *fop_errno = -ret;
+    }
+
+    if (target_changed) {
+        dict_del(dict, GLUSTERFS_POSIXLK_COUNT);
+        ret = dict_set_str(dict, conf->link_xattr_name, to->name);
         if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        DHT_MSG_MIGRATE_FILE_FAILED,
-                        "Migrate file failed: %s: failed to migrate data",
-                        loc->path);
-                /* reset the destination back to 0 */
-                ret = syncop_ftruncate (to, dst_fd, 0, NULL, NULL);
-                if (ret) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                DHT_MSG_MIGRATE_FILE_FAILED,
-                                "Migrate file failed: "
-                                "%s: failed to reset target size back to 0 (%s)",
-                                loc->path, strerror (-ret));
-                }
+            gf_log(this->name, GF_LOG_ERROR,
+                   "failed to set xattr in dict for %s (linkto:%s)", loc->path,
+                   to->name);
+            *fop_errno = ENOMEM;
+            ret = -1;
+            goto out;
+        }
 
+        ret = syncop_setxattr(old_target, loc, dict, 0, NULL, NULL);
+        if (ret && -ret != ESTALE && -ret != ENOENT) {
+            gf_msg(this->name, GF_LOG_ERROR, -ret, DHT_MSG_MIGRATE_FILE_FAILED,
+                   "failed to set xattr on %s in %s", loc->path,
+                   old_target->name);
+            *fop_errno = -ret;
+            ret = -1;
+            goto out;
+        } else if (-ret == ESTALE || -ret == ENOENT) {
+            /* The failure ESTALE indicates that the linkto
+             * file on the hashed subvol might have been deleted.
+             * In this case will create a linkto file with new target
+             * as linkto xattr value*/
+            linkto_fd = fd_create(loc->inode, DHT_REBALANCE_PID);
+            if (!linkto_fd) {
+                gf_msg(this->name, GF_LOG_ERROR, errno,
+                       DHT_MSG_MIGRATE_FILE_FAILED, "%s: fd create failed",
+                       loc->path);
+                *fop_errno = ENOMEM;
                 ret = -1;
                 goto out;
+            }
+            ret = syncop_create(old_target, loc, O_RDWR, DHT_LINKFILE_MODE,
+                                linkto_fd, NULL, dict, NULL);
+            if (ret != 0 && -ret != EEXIST && -ret != ESTALE) {
+                *fop_errno = -ret;
+                ret = -1;
+                gf_msg(this->name, GF_LOG_ERROR, -ret,
+                       DHT_MSG_MIGRATE_FILE_FAILED,
+                       "failed to create linkto file on %s in %s", loc->path,
+                       old_target->name);
+                goto out;
+            } else if (ret == 0) {
+                ret = syncop_fsetattr(old_target, linkto_fd, &stbuf,
+                                      (GF_SET_ATTR_UID | GF_SET_ATTR_GID), NULL,
+                                      NULL, NULL, NULL);
+                if (ret < 0) {
+                    *fop_errno = -ret;
+                    gf_msg(this->name, GF_LOG_ERROR, -ret,
+                           DHT_MSG_MIGRATE_FILE_FAILED,
+                           "chown failed for %s on %s", loc->path,
+                           old_target->name);
+                }
+            }
+        }
+    }
+
+    clean_dst = _gf_false;
+
+    /* Posix acls are not set on DHT linkto files as part of the initial
+     * initial xattrs set on the dst file, so these need
+     * to be set on the dst file after the linkto attrs are removed.
+     * TODO: Optimize this.
+     */
+    if (xattr) {
+        dict_unref(xattr);
+        xattr = NULL;
+    }
+
+    /* Set only the Posix ACLs this time */
+    ret = syncop_getxattr(from, loc, &xattr, POSIX_ACL_ACCESS_XATTR, NULL,
+                          NULL);
+    if (ret < 0) {
+        if ((-ret != ENODATA) && (-ret != ENOATTR)) {
+            gf_msg(this->name, GF_LOG_WARNING, -ret,
+                   DHT_MSG_MIGRATE_FILE_FAILED,
+                   "Migrate file failed:"
+                   "%s: failed to get xattr from %s",
+                   loc->path, from->name);
+            *fop_errno = -ret;
+        }
+    } else {
+        ret = syncop_setxattr(to, loc, xattr, 0, NULL, NULL);
+        if (ret < 0) {
+            /* Potential problem here where Posix ACLs will
+             * not be set on the target file */
+
+            gf_msg(this->name, GF_LOG_WARNING, -ret,
+                   DHT_MSG_MIGRATE_FILE_FAILED,
+                   "Migrate file failed:"
+                   "%s: failed to set xattr on %s",
+                   loc->path, to->name);
+            *fop_errno = -ret;
+        }
+    }
+
+    /* The src file is being unlinked after this so we don't need
+       to clean it up */
+    clean_src = _gf_false;
+
+    /* Make the source as a linkfile first before deleting it */
+    empty_iatt.ia_prot.sticky = 1;
+    ret = syncop_fsetattr(from, src_fd, &empty_iatt, GF_SET_ATTR_MODE, NULL,
+                          NULL, NULL, NULL);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_WARNING, -ret, DHT_MSG_MIGRATE_FILE_FAILED,
+               "Migrate file failed:"
+               "%s: failed to perform setattr on %s ",
+               loc->path, from->name);
+        *fop_errno = -ret;
+        ret = -1;
+        goto metaunlock;
+    }
+
+    /* Free up the data blocks on the source node, as the whole
+        file is migrated */
+    ret = syncop_ftruncate(from, src_fd, 0, NULL, NULL, NULL, NULL);
+    if (ret) {
+        gf_log(this->name, GF_LOG_WARNING,
+               "%s: failed to perform truncate on %s (%s)", loc->path,
+               from->name, strerror(-ret));
+        *fop_errno = -ret;
+    }
+
+    /* remove the 'linkto' xattr from the destination */
+    ret = syncop_fremovexattr(to, dst_fd, conf->link_xattr_name, 0, NULL);
+    if (ret) {
+        gf_log(this->name, GF_LOG_WARNING,
+               "%s: failed to perform removexattr on %s (%s)", loc->path,
+               to->name, strerror(-ret));
+        *fop_errno = -ret;
+    }
+
+    /* Do a stat and check the gfid before unlink */
+
+    /*
+     * Cached file changes its state from non-linkto to linkto file after
+     * migrating data. If lookup from any other mount-point is performed,
+     * converted-linkto-cached file will be treated as a stale and will be
+     * unlinked. But by this time, file is already migrated. So further
+     * failure because of ENOENT should  not be treated as error
+     */
+
+    ret = syncop_stat(from, loc, &empty_iatt, NULL, NULL);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_WARNING, -ret, DHT_MSG_MIGRATE_FILE_FAILED,
+               "%s: failed to do a stat on %s", loc->path, from->name);
+
+        if (-ret != ENOENT) {
+            *fop_errno = -ret;
+            ret = -1;
+            goto metaunlock;
         }
 
-        /* TODO: Sync the locks */
+        rcvd_enoent_from_src = 1;
+    }
 
-        ret = syncop_fsync (to, dst_fd, 0, NULL, NULL);
+    if ((gf_uuid_compare(empty_iatt.ia_gfid, loc->gfid) == 0) &&
+        (!rcvd_enoent_from_src) && delete_src_linkto) {
+        /* take out the source from namespace */
+        ret = syncop_unlink(from, loc, NULL, NULL);
         if (ret) {
-                gf_log (this->name, GF_LOG_WARNING,
-                        "%s: failed to fsync on %s (%s)",
-                        loc->path, to->name, strerror (-ret));
-                ret = -1;
+            gf_msg(this->name, GF_LOG_WARNING, -ret,
+                   DHT_MSG_MIGRATE_FILE_FAILED,
+                   "%s: failed to perform unlink on %s", loc->path, from->name);
+            *fop_errno = -ret;
+            ret = -1;
+            goto metaunlock;
         }
+    }
 
+    ret = syncop_lookup(this, loc, NULL, NULL, NULL, NULL);
+    if (ret) {
+        gf_msg_debug(this->name, -ret,
+                     "%s: failed to lookup the file on subvolumes", loc->path);
+        *fop_errno = -ret;
+    }
 
-        /* Phase 2 - Data-Migration Complete, Housekeeping updates pending */
-
-        ret = syncop_fstat (from, src_fd, &new_stbuf, NULL, NULL);
-        if (ret < 0) {
-                /* Failed to get the stat info */
-                gf_msg ( this->name, GF_LOG_ERROR, -ret,
-                        DHT_MSG_MIGRATE_FILE_FAILED,
-                        "Migrate file failed: failed to fstat file %s on %s ",
-                        loc->path, from->name);
-                ret = -1;
-                goto out;
-        }
+    gf_msg(this->name, log_level, 0, DHT_MSG_MIGRATE_FILE_COMPLETE,
+           "completed migration of %s from subvolume %s to %s", loc->path,
+           from->name, to->name);
 
-        /* source would have both sticky bit and sgid bit set, reset it to 0,
-           and set the source permission on destination, if it was not set
-           prior to setting rebalance-modes in source  */
-        if (!src_ia_prot.sticky)
-                new_stbuf.ia_prot.sticky = 0;
+    ret = 0;
 
-        if (!src_ia_prot.sgid)
-                new_stbuf.ia_prot.sgid = 0;
+metaunlock:
 
-        /* TODO: if the source actually had sticky bit, or sgid bit set,
-           we are not handling it */
+    if (conf->lock_migration_enabled && meta_locked) {
+        dict_del(meta_dict, GF_META_LOCK_KEY);
 
-        ret = syncop_fsetattr (to, dst_fd, &new_stbuf,
-                               (GF_SET_ATTR_UID | GF_SET_ATTR_GID |
-                                GF_SET_ATTR_MODE), NULL, NULL, NULL, NULL);
+        ret = dict_set_int32(meta_dict, GF_META_UNLOCK_KEY, 1);
         if (ret) {
-                gf_msg (this->name, GF_LOG_WARNING, -ret,
-                        DHT_MSG_MIGRATE_FILE_FAILED,
-                        "Migrate file failed:"
-                        "%s: failed to perform setattr on %s ",
-                        loc->path, to->name);
-                ret = -1;
-                goto out;
-        }
+            gf_msg(this->name, GF_LOG_ERROR, 0, DHT_MSG_MIGRATE_FILE_FAILED,
+                   "Trace dict_set failed");
 
-        /* Because 'futimes' is not portable */
-        ret = syncop_setattr (to, loc, &new_stbuf,
-                              (GF_SET_ATTR_MTIME | GF_SET_ATTR_ATIME),
-                              NULL, NULL, NULL, NULL);
-        if (ret) {
-                gf_log (this->name, GF_LOG_WARNING,
-                        "%s: failed to perform setattr on %s ",
-                        loc->path, to->name);
-                ret = -1;
+            *fop_errno = ENOMEM;
+            ret = -1;
+            goto out;
         }
 
-        /* Make the source as a linkfile first before deleting it */
-        empty_iatt.ia_prot.sticky = 1;
-        ret = syncop_fsetattr (from, src_fd, &empty_iatt,
-                               GF_SET_ATTR_MODE, NULL, NULL, NULL, NULL);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_WARNING, -ret,
-                        DHT_MSG_MIGRATE_FILE_FAILED,
-                        "Migrate file failed:"
-                        "%s: failed to perform setattr on %s ",
-                        loc->path, from->name);
-                ret = -1;
-                goto out;
-        }
+        if (clean_dst == _gf_false)
+            ret = dict_set_int32(meta_dict, "status", 1);
+        else
+            ret = dict_set_int32(meta_dict, "status", 0);
 
-       /* Free up the data blocks on the source node, as the whole
-           file is migrated */
-        ret = syncop_ftruncate (from, src_fd, 0, NULL, NULL);
         if (ret) {
-                gf_log (this->name, GF_LOG_WARNING,
-                        "%s: failed to perform truncate on %s (%s)",
-                        loc->path, from->name, strerror (-ret));
-                ret = -1;
-        }
+            gf_msg(this->name, GF_LOG_ERROR, 0, DHT_MSG_MIGRATE_FILE_FAILED,
+                   "Trace dict_set failed");
 
-        /* remove the 'linkto' xattr from the destination */
-        ret = syncop_fremovexattr (to, dst_fd, conf->link_xattr_name, 0, NULL);
-        if (ret) {
-                gf_log (this->name, GF_LOG_WARNING,
-                        "%s: failed to perform removexattr on %s (%s)",
-                        loc->path, to->name, strerror (-ret));
-                ret = -1;
+            *fop_errno = ENOMEM;
+            ret = -1;
+            goto out;
         }
 
-        /* Do a stat and check the gfid before unlink */
-
-        /*
-         * Cached file changes its state from non-linkto to linkto file after
-         * migrating data. If lookup from any other mount-point is performed,
-         * converted-linkto-cached file will be treated as a stale and will be
-         * unlinked. But by this time, file is already migrated. So further
-         * failure because of ENOENT should  not be treated as error
-         */
-
-        ret = syncop_stat (from, loc, &empty_iatt, NULL, NULL);
+        ret = syncop_setxattr(from, loc, meta_dict, 0, NULL, NULL);
         if (ret) {
-                gf_msg (this->name, GF_LOG_WARNING, 0,
-                        DHT_MSG_MIGRATE_FILE_FAILED,
-                        "%s: failed to do a stat on %s (%s)",
-                        loc->path, from->name, strerror (-ret));
+            gf_msg(this->name, GF_LOG_ERROR, -ret, DHT_MSG_MIGRATE_FILE_FAILED,
+                   "Trace syncop_setxattr meta unlock failed");
 
-                if (-ret != ENOENT) {
-                        ret = -1;
-                        goto out;
-                }
-
-                rcvd_enoent_from_src = 1;
+            *fop_errno = -ret;
+            ret = -1;
+            goto out;
         }
+    }
 
-        if ((gf_uuid_compare (empty_iatt.ia_gfid, loc->gfid) == 0 ) &&
-            (!rcvd_enoent_from_src)) {
-                /* take out the source from namespace */
-                ret = syncop_unlink (from, loc, NULL, NULL);
-                if (ret) {
-                        gf_msg (this->name, GF_LOG_WARNING, 0,
-                                DHT_MSG_MIGRATE_FILE_FAILED,
-                                "%s: failed to perform unlink on %s (%s)",
-                                loc->path, from->name, strerror (-ret));
-                        ret = -1;
-                        goto out;
-                }
+out:
+    if (clean_src) {
+        /* Revert source mode and xattr changes*/
+        lk_ret = __dht_migration_cleanup_src_file(this, loc, src_fd, from,
+                                                  &src_ia_prot);
+        if (lk_ret) {
+            gf_msg(this->name, GF_LOG_WARNING, 0, DHT_MSG_MIGRATE_FILE_FAILED,
+                   "%s: failed to cleanup source file on %s", loc->path,
+                   from->name);
         }
-
-        ret = syncop_lookup (this, loc, NULL, NULL, NULL, NULL);
-        if (ret) {
-                gf_msg_debug (this->name, 0,
-                              "%s: failed to lookup the file on subvolumes (%s)",
-                              loc->path, strerror (-ret));
-                ret = -1;
+    }
+
+    /* reset the destination back to 0 */
+    if (clean_dst) {
+        lk_ret = syncop_ftruncate(to, dst_fd, 0, NULL, NULL, NULL, NULL);
+        if (lk_ret) {
+            gf_msg(this->name, GF_LOG_ERROR, -lk_ret,
+                   DHT_MSG_MIGRATE_FILE_FAILED,
+                   "Migrate file failed: "
+                   "%s: failed to reset target size back to 0",
+                   loc->path);
         }
+    }
 
-        gf_msg (this->name, GF_LOG_INFO, 0,
-                DHT_MSG_MIGRATE_FILE_COMPLETE,
-                "completed migration of %s from subvolume %s to %s",
-                loc->path, from->name, to->name);
+    if (inodelk_locked) {
+        flock.l_type = F_UNLCK;
 
-        ret = 0;
-out:
-        if (locked) {
-                flock.l_type = F_UNLCK;
-
-                lk_ret = syncop_inodelk (from, DHT_FILE_MIGRATE_DOMAIN,
-                                         &tmp_loc, F_SETLK, &flock, NULL, NULL);
-                if (lk_ret < 0) {
-                        gf_msg (this->name, GF_LOG_WARNING, 0,
-                                DHT_MSG_MIGRATE_FILE_FAILED,
-                                "%s: failed to unlock file on %s (%s)",
-                                loc->path, from->name, strerror (-lk_ret));
-                }
+        lk_ret = syncop_inodelk(from, DHT_FILE_MIGRATE_DOMAIN, &tmp_loc,
+                                F_SETLK, &flock, NULL, NULL);
+        if (lk_ret < 0) {
+            gf_msg(this->name, GF_LOG_WARNING, -lk_ret,
+                   DHT_MSG_MIGRATE_FILE_FAILED,
+                   "%s: failed to unlock file on %s", loc->path, from->name);
         }
+    }
+
+    if (entrylk_locked) {
+        lk_ret = syncop_entrylk(hashed_subvol, DHT_ENTRY_SYNC_DOMAIN,
+                                &parent_loc, loc->name, ENTRYLK_UNLOCK,
+                                ENTRYLK_UNLOCK, NULL, NULL);
+        if (lk_ret < 0) {
+            gf_msg(this->name, GF_LOG_WARNING, -lk_ret,
+                   DHT_MSG_MIGRATE_FILE_FAILED,
+                   "%s: failed to unlock entrylk on %s", loc->path,
+                   hashed_subvol->name);
+        }
+    }
 
-        if (dict)
-                dict_unref (dict);
+    if (p_locked) {
+        plock.l_type = F_UNLCK;
+        lk_ret = syncop_lk(from, src_fd, F_SETLK, &plock, NULL, NULL);
 
-        if (xattr)
-                dict_unref (xattr);
-        if (xattr_rsp)
-                dict_unref (xattr_rsp);
+        if (lk_ret < 0) {
+            gf_msg(this->name, GF_LOG_WARNING, -lk_ret,
+                   DHT_MSG_MIGRATE_FILE_FAILED,
+                   "%s: failed to unlock file on %s", loc->path, from->name);
+        }
+    }
 
-        if (dst_fd)
-                syncop_close (dst_fd);
-        if (src_fd)
-                syncop_close (src_fd);
+    lk_ret = syncop_removexattr(to, loc, GF_PROTECT_FROM_EXTERNAL_WRITES, NULL,
+                                NULL);
+    if (lk_ret && (lk_ret != -ENODATA) && (lk_ret != -ENOATTR)) {
+        gf_msg(this->name, GF_LOG_WARNING, -lk_ret, 0,
+               "%s: removexattr failed key %s", loc->path,
+               GF_PROTECT_FROM_EXTERNAL_WRITES);
+    }
 
-        loc_wipe (&tmp_loc);
+    if (dict)
+        dict_unref(dict);
 
-        return ret;
-}
+    if (xattr)
+        dict_unref(xattr);
+    if (xattr_rsp)
+        dict_unref(xattr_rsp);
 
-static int
-rebalance_task (void *data)
-{
-        int           ret   = -1;
-        dht_local_t  *local = NULL;
-        call_frame_t *frame = NULL;
+    if (dst_fd)
+        syncop_close(dst_fd);
 
-        frame = data;
+    if (src_fd)
+        syncop_close(src_fd);
+    if (linkto_fd)
+        syncop_close(linkto_fd);
 
-        local = frame->local;
+    if (xdata)
+        dict_unref(xdata);
 
-        /* This function is 'synchrounous', hence if it returns,
-           we are done with the task */
-        ret = dht_migrate_file (THIS, &local->loc, local->rebalance.from_subvol,
-                                local->rebalance.target_node, local->flags);
+    loc_wipe(&tmp_loc);
+    loc_wipe(&parent_loc);
 
-        return ret;
+    return ret;
 }
 
 static int
-rebalance_task_completion (int op_ret, call_frame_t *sync_frame, void *data)
+rebalance_task(void *data)
 {
-        int           ret        = -1;
-        uint64_t      layout_int = 0;
-        dht_layout_t *layout     = 0;
-        xlator_t     *this       = NULL;
-        dht_local_t  *local      = NULL;
-        int32_t       op_errno   = EINVAL;
-
-        this = THIS;
-        local = sync_frame->local;
-
-        if (!op_ret) {
-                /* Make sure we have valid 'layout' in inode ctx
-                   after the operation */
-                ret = inode_ctx_del (local->loc.inode, this, &layout_int);
-                if (!ret && layout_int) {
-                        layout = (dht_layout_t *)(long)layout_int;
-                        dht_layout_unref (this, layout);
-                }
+    int ret = -1;
+    dht_local_t *local = NULL;
+    call_frame_t *frame = NULL;
+    int fop_errno = 0;
 
-                ret = dht_layout_preset (this, local->rebalance.target_node,
-                                         local->loc.inode);
-                if (ret)
-                        gf_log (this->name, GF_LOG_WARNING,
-                                "%s: failed to set inode ctx", local->loc.path);
-        }
+    frame = data;
 
-        if (op_ret == -1) {
-                /* Failure of migration process, mostly due to write process.
-                   as we can't preserve the exact errno, lets say there was
-                   no space to migrate-data
-                */
-                op_errno = ENOSPC;
-        }
+    local = frame->local;
 
-        if (op_ret == 1) {
-                /* migration didn't happen, but is not a failure, let the user
-                   understand that he doesn't have permission to migrate the
-                   file.
-                */
-                op_ret = -1;
-                op_errno = EPERM;
-        }
+    /* This function is 'synchrounous', hence if it returns,
+       we are done with the task */
+    ret = dht_migrate_file(THIS, &local->loc, local->rebalance.from_subvol,
+                           local->rebalance.target_node, local->flags,
+                           &fop_errno);
 
-        DHT_STACK_UNWIND (setxattr, sync_frame, op_ret, op_errno, NULL);
-        return 0;
+    return ret;
+}
+
+static int
+rebalance_task_completion(int op_ret, call_frame_t *sync_frame, void *data)
+{
+    int32_t op_errno = EINVAL;
+
+    if (op_ret == -1) {
+        /* Failure of migration process, mostly due to write process.
+           as we can't preserve the exact errno, lets say there was
+           no space to migrate-data
+        */
+        op_errno = ENOSPC;
+    } else if (op_ret == 1) {
+        /* migration didn't happen, but is not a failure, let the user
+           understand that he doesn't have permission to migrate the
+           file.
+        */
+        op_ret = -1;
+        op_errno = EPERM;
+    } else if (op_ret != 0) {
+        op_errno = -op_ret;
+        op_ret = -1;
+    }
+
+    DHT_STACK_UNWIND(setxattr, sync_frame, op_ret, op_errno, NULL);
+    return 0;
 }
 
 int
-dht_start_rebalance_task (xlator_t *this, call_frame_t *frame)
+dht_start_rebalance_task(xlator_t *this, call_frame_t *frame)
 {
-        int         ret     = -1;
+    int ret = -1;
 
-        ret = synctask_new (this->ctx->env, rebalance_task,
-                            rebalance_task_completion,
-                            frame, frame);
-        return ret;
+    ret = synctask_new(this->ctx->env, rebalance_task,
+                       rebalance_task_completion, frame, frame);
+    return ret;
 }
 
 int
-gf_listener_stop (xlator_t *this)
+gf_listener_stop(xlator_t *this)
 {
-        glusterfs_ctx_t  *ctx = NULL;
-        cmd_args_t       *cmd_args = NULL;
-        int              ret = 0;
-
-        ctx = this->ctx;
-        GF_ASSERT (ctx);
-        cmd_args = &ctx->cmd_args;
-        if (cmd_args->sock_file) {
-                ret = unlink (cmd_args->sock_file);
-                if (ret && (ENOENT == errno)) {
-                        ret = 0;
-                }
-        }
-
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "Failed to unlink listener "
-                        "socket %s, error: %s", cmd_args->sock_file,
-                        strerror (errno));
+    glusterfs_ctx_t *ctx = NULL;
+    cmd_args_t *cmd_args = NULL;
+    int ret = 0;
+
+    ctx = this->ctx;
+    GF_ASSERT(ctx);
+    cmd_args = &ctx->cmd_args;
+    if (cmd_args->sock_file) {
+        ret = sys_unlink(cmd_args->sock_file);
+        if (ret && (ENOENT == errno)) {
+            ret = 0;
         }
-        return ret;
+    }
+
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, errno, DHT_MSG_SOCKET_ERROR,
+               "Failed to unlink listener "
+               "socket %s",
+               cmd_args->sock_file);
+    }
+    return ret;
 }
 
 void
-dht_build_root_inode (xlator_t *this, inode_t **inode)
+dht_build_root_inode(xlator_t *this, inode_t **inode)
 {
-        inode_table_t    *itable        = NULL;
-        uuid_t            root_gfid     = {0, };
+    inode_table_t *itable = NULL;
+    static uuid_t root_gfid = {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1};
 
-        itable = inode_table_new (0, this);
-        if (!itable)
-                return;
+    itable = inode_table_new(0, this);
+    if (!itable)
+        return;
 
-        root_gfid[15] = 1;
-        *inode = inode_find (itable, root_gfid);
+    *inode = inode_find(itable, root_gfid);
 }
 
 void
-dht_build_root_loc (inode_t *inode, loc_t *loc)
+dht_build_root_loc(inode_t *inode, loc_t *loc)
 {
-        loc->path = "/";
-        loc->inode = inode;
-        loc->inode->ia_type = IA_IFDIR;
-        memset (loc->gfid, 0, 16);
-        loc->gfid[15] = 1;
+    loc->path = "/";
+    loc->inode = inode;
+    loc->inode->ia_type = IA_IFDIR;
+    memset(loc->gfid, 0, 16);
+    loc->gfid[15] = 1;
 }
 
-
 /* return values: 1 -> error, bug ignore and continue
                   0 -> proceed
                  -1 -> error, handle it */
 int32_t
-gf_defrag_handle_migrate_error (int32_t op_errno, gf_defrag_info_t *defrag)
+gf_defrag_handle_migrate_error(int32_t op_errno, gf_defrag_info_t *defrag)
 {
-        /* if errno is not ENOSPC or ENOTCONN, we can still continue
-           with rebalance process */
-        if ((op_errno != ENOSPC) || (op_errno != ENOTCONN))
-                return 1;
-
-        if (op_errno == ENOTCONN) {
-                /* Most probably mount point went missing (mostly due
-                   to a brick down), say rebalance failure to user,
-                   let him restart it if everything is fine */
-                defrag->defrag_status = GF_DEFRAG_STATUS_FAILED;
-                return -1;
-        }
-
-        if (op_errno == ENOSPC) {
-                /* rebalance process itself failed, may be
-                   remote brick went down, or write failed due to
-                   disk full etc etc.. */
-                defrag->defrag_status = GF_DEFRAG_STATUS_FAILED;
-                return -1;
-        }
+    int ret = 0;
+    /* if errno is not ENOTCONN, we can still continue
+       with rebalance process */
+    if (op_errno != ENOTCONN) {
+        ret = 1;
+        goto out;
+    }
+
+    if (op_errno == ENOTCONN) {
+        /* Most probably mount point went missing (mostly due
+           to a brick down), say rebalance failure to user,
+           let him restart it if everything is fine */
+        defrag->defrag_status = GF_DEFRAG_STATUS_FAILED;
+        ret = -1;
+        goto out;
+    }
 
-        return 0;
+out:
+    return ret;
 }
 
 static gf_boolean_t
-gf_defrag_pattern_match (gf_defrag_info_t *defrag, char *name, uint64_t size)
+gf_defrag_pattern_match(gf_defrag_info_t *defrag, char *name, uint64_t size)
 {
-        gf_defrag_pattern_list_t *trav = NULL;
-        gf_boolean_t               match = _gf_false;
-        gf_boolean_t               ret = _gf_false;
+    gf_defrag_pattern_list_t *trav = NULL;
+    gf_boolean_t match = _gf_false;
+    gf_boolean_t ret = _gf_false;
 
-        GF_VALIDATE_OR_GOTO ("dht", defrag, out);
+    GF_VALIDATE_OR_GOTO("dht", defrag, out);
 
-        trav = defrag->defrag_pattern;
-        while (trav) {
-                if (!fnmatch (trav->path_pattern, name, FNM_NOESCAPE)) {
-                        match = _gf_true;
-                        break;
-                }
-                trav = trav->next;
+    trav = defrag->defrag_pattern;
+    while (trav) {
+        if (!fnmatch(trav->path_pattern, name, FNM_NOESCAPE)) {
+            match = _gf_true;
+            break;
         }
+        trav = trav->next;
+    }
 
-        if ((match == _gf_true) && (size >= trav->size))
-                ret = _gf_true;
+    if ((match == _gf_true) && (size >= trav->size))
+        ret = _gf_true;
 
- out:
-        return ret;
+out:
+    return ret;
+}
+
+int
+dht_dfreaddirp_done(dht_dfoffset_ctx_t *offset_var, int cnt)
+{
+    int i;
+    int result = 1;
+
+    for (i = 0; i < cnt; i++) {
+        if (offset_var[i].readdir_done == 0) {
+            result = 0;
+            break;
+        }
+    }
+    return result;
 }
 
-/* We do a depth first traversal of directories. But before we move into
- * subdirs, we complete the data migration of those directories whose layouts
- * have been fixed
+int static gf_defrag_ctx_subvols_init(dht_dfoffset_ctx_t *offset_var,
+                                      xlator_t *this)
+{
+    int i;
+    dht_conf_t *conf = NULL;
+
+    conf = this->private;
+
+    if (!conf)
+        return -1;
+
+    for (i = 0; i < conf->local_subvols_cnt; i++) {
+        offset_var[i].this = conf->local_subvols[i];
+        offset_var[i].offset = (off_t)0;
+        offset_var[i].readdir_done = 0;
+    }
+
+    return 0;
+}
+
+static int
+dht_get_first_non_null_index(subvol_nodeuuids_info_t *entry)
+{
+    int i = 0;
+    int index = 0;
+
+    for (i = 0; i < entry->count; i++) {
+        if (!gf_uuid_is_null(entry->elements[i].uuid)) {
+            index = i;
+            goto out;
+        }
+    }
+
+    if (i == entry->count) {
+        index = -1;
+    }
+out:
+    return index;
+}
+
+/* Return value
+ * 0 : this node does not migrate the file
+ * 1 : this node migrates the file
+ *
+ * Use the hash value of the gfid to determine which node will migrate files.
+ * Using the gfid instead of the name also ensures that the same node handles
+ * all hardlinks.
  */
 
-int
-gf_defrag_migrate_data (xlator_t *this, gf_defrag_info_t *defrag, loc_t *loc,
-                        dict_t *migrate_data)
+gf_boolean_t
+gf_defrag_should_i_migrate(xlator_t *this, int local_subvol_index, uuid_t gfid)
 {
-        int                      ret            = -1;
-        loc_t                    entry_loc      = {0,};
-        fd_t                    *fd             = NULL;
-        gf_dirent_t              entries;
-        gf_dirent_t             *tmp            = NULL;
-        gf_dirent_t             *entry          = NULL;
-        gf_boolean_t             free_entries   = _gf_false;
-        off_t                    offset         = 0;
-        dict_t                  *dict           = NULL;
-        struct iatt              iatt           = {0,};
-        int32_t                  op_errno       = 0;
-        char                    *uuid_str       = NULL;
-        uuid_t                   node_uuid      = {0,};
-        struct timeval           dir_start      = {0,};
-        struct timeval           end            = {0,};
-        double                   elapsed        = {0,};
-        struct timeval           start          = {0,};
-        int                      loglevel       = GF_LOG_TRACE;
-
-        gf_log (this->name, GF_LOG_INFO, "migrate data called on %s",
-                loc->path);
-        gettimeofday (&dir_start, NULL);
-
-        fd = fd_create (loc->inode, defrag->pid);
-        if (!fd) {
-                gf_log (this->name, GF_LOG_ERROR, "Failed to create fd");
-                goto out;
+    gf_boolean_t ret = _gf_false;
+    int i = local_subvol_index;
+    char *str = NULL;
+    uint32_t hashval = 0;
+    int32_t index = 0;
+    dht_conf_t *conf = NULL;
+    char buf[UUID_CANONICAL_FORM_LEN + 1] = {
+        0,
+    };
+    subvol_nodeuuids_info_t *entry = NULL;
+
+    conf = this->private;
+
+    /* Pure distribute. A subvol in this case
+        will be handled by only one node */
+
+    entry = &(conf->local_nodeuuids[i]);
+    if (entry->count == 1) {
+        return 1;
+    }
+
+    str = uuid_utoa_r(gfid, buf);
+    if (dht_hash_compute(this, 0, str, &hashval) == 0) {
+        index = (hashval % entry->count);
+        if (entry->elements[index].info == REBAL_NODEUUID_MINE) {
+            /* Index matches this node's nodeuuid.*/
+            ret = _gf_true;
+            goto out;
         }
 
-        ret = syncop_opendir (this, loc, fd, NULL, NULL);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        DHT_MSG_MIGRATE_DATA_FAILED,
-                        "Migrate data failed: Failed to open dir %s",
-                        loc->path);
-                ret = -1;
+        /* Brick down - some other node has to migrate these files*/
+        if (gf_uuid_is_null(entry->elements[index].uuid)) {
+            /* Fall back to the first non-null index */
+            index = dht_get_first_non_null_index(entry);
+
+            if (index == -1) {
+                /* None of the bricks in the subvol are up.
+                 * CHILD_DOWN will kill the process soon */
+
+                return _gf_false;
+            }
+
+            if (entry->elements[index].info == REBAL_NODEUUID_MINE) {
+                /* Index matches this node's nodeuuid.*/
+                ret = _gf_true;
                 goto out;
+            }
         }
+    }
+out:
+    return ret;
+}
 
-        INIT_LIST_HEAD (&entries.list);
+int
+gf_defrag_migrate_single_file(void *opaque)
+{
+    xlator_t *this = NULL;
+    dht_conf_t *conf = NULL;
+    gf_defrag_info_t *defrag = NULL;
+    int ret = 0;
+    gf_dirent_t *entry = NULL;
+    struct timeval start = {
+        0,
+    };
+    loc_t entry_loc = {
+        0,
+    };
+    loc_t *loc = NULL;
+    struct iatt iatt = {
+        0,
+    };
+    dict_t *migrate_data = NULL;
+    struct timeval end = {
+        0,
+    };
+    double elapsed = {
+        0,
+    };
+    struct dht_container *rebal_entry = NULL;
+    inode_t *inode = NULL;
+    xlator_t *hashed_subvol = NULL;
+    xlator_t *cached_subvol = NULL;
+    call_frame_t *statfs_frame = NULL;
+    xlator_t *old_THIS = NULL;
+    data_t *tmp = NULL;
+    int fop_errno = 0;
+    gf_dht_migrate_data_type_t rebal_type = GF_DHT_MIGRATE_DATA;
+    char value[MAX_REBAL_TYPE_SIZE] = {
+        0,
+    };
+    struct iatt *iatt_ptr = NULL;
+    gf_boolean_t update_skippedcount = _gf_true;
+    int i = 0;
+    gf_boolean_t should_i_migrate = 0;
+
+    rebal_entry = (struct dht_container *)opaque;
+    if (!rebal_entry) {
+        gf_log("DHT", GF_LOG_ERROR, "rebal_entry is NULL");
+        ret = -1;
+        goto out;
+    }
 
-        while ((ret = syncop_readdirp (this, fd, 131072, offset, &entries,
-                                       NULL, NULL)) != 0) {
+    this = rebal_entry->this;
 
-                if (ret < 0) {
+    conf = this->private;
 
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                DHT_MSG_MIGRATE_DATA_FAILED,
-                                "%s: Migrate data failed: Readdir returned"
-                                " %s. Aborting migrate-data", loc->path,
-                                strerror(-ret));
-                        ret = -1;
-                        goto out;
-                }
+    defrag = conf->defrag;
 
-                if (list_empty (&entries.list))
-                        break;
+    loc = rebal_entry->parent_loc;
 
-                free_entries = _gf_true;
+    migrate_data = rebal_entry->migrate_data;
 
-                list_for_each_entry_safe (entry, tmp, &entries.list, list) {
+    entry = rebal_entry->df_entry;
+    iatt_ptr = &entry->d_stat;
 
-                        if (dict) {
-                                dict_unref (dict);
-                                dict = NULL;
-                        }
+    if (defrag->defrag_status != GF_DEFRAG_STATUS_STARTED) {
+        ret = -1;
+        goto out;
+    }
 
-                        if (defrag->defrag_status != GF_DEFRAG_STATUS_STARTED) {
-                                ret = 1;
-                                goto out;
-                        }
+    if (defrag->stats == _gf_true) {
+        gettimeofday(&start, NULL);
+    }
 
-                        offset = entry->d_off;
+    if (defrag->defrag_pattern &&
+        (gf_defrag_pattern_match(defrag, entry->d_name,
+                                 entry->d_stat.ia_size) == _gf_false)) {
+        gf_log(this->name, GF_LOG_ERROR, "pattern_match failed");
+        goto out;
+    }
 
-                        if (!strcmp (entry->d_name, ".") ||
-                            !strcmp (entry->d_name, ".."))
-                                continue;
+    memset(&entry_loc, 0, sizeof(entry_loc));
 
-                        if (IA_ISDIR (entry->d_stat.ia_type))
-                                continue;
+    ret = dht_build_child_loc(this, &entry_loc, loc, entry->d_name);
+    if (ret) {
+        LOCK(&defrag->lock);
+        {
+            defrag->total_failures += 1;
+        }
+        UNLOCK(&defrag->lock);
 
-                        defrag->num_files_lookedup++;
-                        if (defrag->stats == _gf_true) {
-                                gettimeofday (&start, NULL);
-                        }
+        ret = 0;
 
-                        if (defrag->defrag_pattern &&
-                            (gf_defrag_pattern_match (defrag, entry->d_name,
-                                                      entry->d_stat.ia_size)
-                             == _gf_false)) {
-                                continue;
-                        }
+        gf_log(this->name, GF_LOG_ERROR, "Child loc build failed");
 
-                        loc_wipe (&entry_loc);
-                        ret =dht_build_child_loc (this, &entry_loc, loc,
-                                                  entry->d_name);
-                        if (ret) {
-                                gf_log (this->name, GF_LOG_ERROR, "Child loc"
-                                        " build failed");
-                                goto out;
-                        }
+        goto out;
+    }
 
-                        if (gf_uuid_is_null (entry->d_stat.ia_gfid)) {
-                                gf_msg (this->name, GF_LOG_ERROR, 0,
-                                        DHT_MSG_GFID_NULL,
-                                        "%s/%s gfid not present", loc->path,
-                                         entry->d_name);
-                                continue;
-                        }
+    should_i_migrate = gf_defrag_should_i_migrate(
+        this, rebal_entry->local_subvol_index, entry->d_stat.ia_gfid);
 
-                        gf_uuid_copy (entry_loc.gfid, entry->d_stat.ia_gfid);
+    gf_uuid_copy(entry_loc.gfid, entry->d_stat.ia_gfid);
 
-                        if (gf_uuid_is_null (loc->gfid)) {
-                                gf_msg (this->name, GF_LOG_ERROR, 0,
-                                        DHT_MSG_GFID_NULL,
-                                        "%s/%s gfid not present", loc->path,
-                                         entry->d_name);
-                                continue;
-                        }
+    gf_uuid_copy(entry_loc.pargfid, loc->gfid);
 
-                        gf_uuid_copy (entry_loc.pargfid, loc->gfid);
+    ret = syncop_lookup(this, &entry_loc, &iatt, NULL, NULL, NULL);
 
-                        entry_loc.inode->ia_type = entry->d_stat.ia_type;
+    if (!should_i_migrate) {
+        /* this node isn't supposed to migrate the file. suppressing any
+         * potential error from lookup as this file is under migration by
+         * another node */
+        if (ret) {
+            gf_msg_debug(this->name, -ret,
+                         "Ignoring lookup failure: node isn't migrating %s",
+                         entry_loc.path);
+            ret = 0;
+        }
+        gf_msg_debug(this->name, 0, "Don't migrate %s ", entry_loc.path);
+        goto out;
+    }
 
-                        ret = syncop_lookup (this, &entry_loc, &iatt, NULL,
-                                             NULL, NULL);
-                        if (ret) {
-                                gf_msg (this->name, GF_LOG_ERROR, 0,
-                                        DHT_MSG_MIGRATE_FILE_FAILED,
-                                        "Migrate file failed:%s lookup failed",
-                                        entry_loc.path);
-                                ret = -1;
-                                continue;
-                        }
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, -ret, DHT_MSG_MIGRATE_FILE_FAILED,
+               "Migrate file failed: %s lookup failed", entry_loc.path);
 
-                        ret = syncop_getxattr (this, &entry_loc, &dict,
-                                               GF_XATTR_NODE_UUID_KEY, NULL,
-                                               NULL);
-                        if(ret < 0) {
-                                gf_msg (this->name, GF_LOG_ERROR, 0,
-                                        DHT_MSG_MIGRATE_FILE_FAILED,
-                                        "Migrate file failed:"
-                                        "Failed to get node-uuid for %s",
-                                        entry_loc.path);
-                                ret = -1;
-                                continue;
-                        }
+        /* Increase failure count only for remove-brick op, so that
+         * user is warned to check the removed-brick for any files left
+         * unmigrated
+         */
+        if (conf->decommission_subvols_cnt) {
+            LOCK(&defrag->lock);
+            {
+                defrag->total_failures += 1;
+            }
+            UNLOCK(&defrag->lock);
+        }
 
-                        ret = dict_get_str (dict, GF_XATTR_NODE_UUID_KEY,
-                                            &uuid_str);
-                        if(ret < 0) {
-                                gf_log (this->name, GF_LOG_ERROR, "Failed to "
-                                        "get node-uuid from dict for %s",
-                                        entry_loc.path);
-                                ret = -1;
-                                continue;
-                        }
+        ret = 0;
+        goto out;
+    }
 
-                        if (gf_uuid_parse (uuid_str, node_uuid)) {
-                                gf_log (this->name, GF_LOG_ERROR, "gf_uuid_parse "
-                                        "failed for %s", entry_loc.path);
-                                continue;
-                        }
+    iatt_ptr = &iatt;
 
-                        /* if file belongs to different node, skip migration
-                         * the other node will take responsibility of migration
-                         */
-                        if (gf_uuid_compare (node_uuid, defrag->node_uuid)) {
-                                gf_msg_trace (this->name, 0, "%s does not"
-                                              "belong to this node",
-                                              entry_loc.path);
-                                continue;
-                        }
+    hashed_subvol = dht_subvol_get_hashed(this, &entry_loc);
+    if (!hashed_subvol) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, DHT_MSG_HASHED_SUBVOL_GET_FAILED,
+               "Failed to get hashed subvol for %s", entry_loc.path);
+        ret = 0;
+        goto out;
+    }
 
-                        uuid_str = NULL;
-
-                        /* if distribute is present, it will honor this key.
-                         * -1, ENODATA is returned if distribute is not present
-                         * or file doesn't have a link-file. If file has
-                         * link-file, the path of link-file will be the value,
-                         * and also that guarantees that file has to be mostly
-                         * migrated */
-
-                        ret = syncop_getxattr (this, &entry_loc, NULL,
-                                               GF_XATTR_LINKINFO_KEY, NULL,
-                                               NULL);
-                        if (ret < 0) {
-                                if (-ret != ENODATA) {
-                                        loglevel = GF_LOG_ERROR;
-                                        defrag->total_failures += 1;
-                                } else {
-                                        loglevel = GF_LOG_TRACE;
-                                }
-                                gf_log (this->name, loglevel, "%s: failed to "
-                                        "get "GF_XATTR_LINKINFO_KEY" key - %s",
-                                        entry_loc.path, strerror (-ret));
-                                ret = -1;
-                                continue;
-                        }
+    cached_subvol = dht_subvol_get_cached(this, entry_loc.inode);
+    if (!cached_subvol) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, DHT_MSG_CACHED_SUBVOL_GET_FAILED,
+               "Failed to get cached subvol for %s", entry_loc.path);
 
-                        ret = syncop_setxattr (this, &entry_loc, migrate_data,
-                                               0, NULL, NULL);
-                        if (ret < 0) {
-                                op_errno = -ret;
-                                /* errno is overloaded. See
-                                 * rebalance_task_completion () */
-                                if (op_errno == ENOSPC) {
-                                        gf_msg_debug (this->name, 0,
-                                                      "migrate-data skipped for"
-                                                      " %s due to space "
-                                                      "constraints",
-                                                      entry_loc.path);
-                                        defrag->skipped +=1;
-                                } else{
-                                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                                DHT_MSG_MIGRATE_FILE_FAILED,
-                                                "migrate-data failed for %s",
-                                                entry_loc.path);
-                                        defrag->total_failures +=1;
-                                }
-
-                                ret = gf_defrag_handle_migrate_error (op_errno,
-                                                                      defrag);
-
-                                if (!ret)
-                                        gf_msg_debug (this->name, 0,
-                                                      "migrate-data on %s "
-                                                      "failed: %s",
-                                                      entry_loc.path,
-                                                      strerror (op_errno));
-                                else if (ret == 1)
-                                        continue;
-                                else if (ret == -1)
-                                        goto out;
-                        } else if (ret > 0) {
-                                gf_msg (this->name, GF_LOG_ERROR, 0,
-                                        DHT_MSG_MIGRATE_FILE_FAILED,
-                                        "migrate-data failed for %s",
-                                        entry_loc.path);
-                                defrag->total_failures +=1;
-                        }
+        ret = 0;
+        goto out;
+    }
 
-                        LOCK (&defrag->lock);
+    if (hashed_subvol == cached_subvol) {
+        ret = 0;
+        goto out;
+    }
+
+    inode = inode_link(entry_loc.inode, entry_loc.parent, entry->d_name, &iatt);
+    inode_unref(entry_loc.inode);
+    /* use the inode returned by inode_link */
+    entry_loc.inode = inode;
+
+    old_THIS = THIS;
+    THIS = this;
+    statfs_frame = create_frame(this, this->ctx->pool);
+    if (!statfs_frame) {
+        gf_msg(this->name, GF_LOG_ERROR, DHT_MSG_NO_MEMORY, ENOMEM,
+               "Insufficient memory. Frame creation failed");
+        ret = -1;
+        goto out;
+    }
+
+    /* async statfs information for honoring min-free-disk */
+    dht_get_du_info(statfs_frame, this, loc);
+    THIS = old_THIS;
+
+    tmp = dict_get(migrate_data, GF_XATTR_FILE_MIGRATE_KEY);
+    if (tmp) {
+        memcpy(value, tmp->data, tmp->len);
+        if (strcmp(value, "force") == 0)
+            rebal_type = GF_DHT_MIGRATE_DATA_EVEN_IF_LINK_EXISTS;
+
+        if (conf->decommission_in_progress)
+            rebal_type = GF_DHT_MIGRATE_HARDLINK;
+    }
+
+    ret = dht_migrate_file(this, &entry_loc, cached_subvol, hashed_subvol,
+                           rebal_type, &fop_errno);
+    if (ret == 1) {
+        if (fop_errno == ENOSPC) {
+            gf_msg_debug(this->name, 0,
+                         "migrate-data skipped for"
+                         " %s due to space constraints",
+                         entry_loc.path);
+
+            /* For remove-brick case if the source is not one of the
+             * removed-brick, do not mark the error as failure */
+            if (conf->decommission_subvols_cnt) {
+                for (i = 0; i < conf->subvolume_cnt; i++) {
+                    if (conf->decommissioned_bricks[i] == cached_subvol) {
+                        LOCK(&defrag->lock);
                         {
-                                defrag->total_files += 1;
-                                defrag->total_data += iatt.ia_size;
-                        }
-                        UNLOCK (&defrag->lock);
-                        if (defrag->stats == _gf_true) {
-                                gettimeofday (&end, NULL);
-                                elapsed = (end.tv_sec - start.tv_sec) * 1e6 +
-                                          (end.tv_usec - start.tv_usec);
-                                gf_log (this->name, GF_LOG_INFO, "Migration of "
-                                        "file:%s size:%"PRIu64" bytes took %.2f"
-                                        "secs", entry_loc.path, iatt.ia_size,
-                                         elapsed/1e6);
+                            defrag->total_failures += 1;
+                            update_skippedcount = _gf_false;
                         }
+                        UNLOCK(&defrag->lock);
+
+                        break;
+                    }
                 }
+            }
 
-                gf_dirent_free (&entries);
-                free_entries = _gf_false;
-                INIT_LIST_HEAD (&entries.list);
+            if (update_skippedcount) {
+                LOCK(&defrag->lock);
+                {
+                    defrag->skipped += 1;
+                }
+                UNLOCK(&defrag->lock);
+
+                gf_msg(this->name, GF_LOG_INFO, 0, DHT_MSG_MIGRATE_FILE_SKIPPED,
+                       "File migration skipped for %s.", entry_loc.path);
+            }
+
+        } else if (fop_errno == ENOTSUP) {
+            gf_msg_debug(this->name, 0,
+                         "migrate-data skipped for"
+                         " hardlink %s ",
+                         entry_loc.path);
+            LOCK(&defrag->lock);
+            {
+                defrag->skipped += 1;
+            }
+            UNLOCK(&defrag->lock);
+
+            gf_msg(this->name, GF_LOG_INFO, 0, DHT_MSG_MIGRATE_FILE_SKIPPED,
+                   "File migration skipped for %s.", entry_loc.path);
         }
 
-        gettimeofday (&end, NULL);
-        elapsed = (end.tv_sec - dir_start.tv_sec) * 1e6 +
-                  (end.tv_usec - dir_start.tv_usec);
-        gf_log (this->name, GF_LOG_INFO, "Migration operation on dir %s took "
-                "%.2f secs", loc->path, elapsed/1e6);
         ret = 0;
-out:
-        if (free_entries)
-                gf_dirent_free (&entries);
+        goto out;
+    } else if (ret < 0) {
+        if (fop_errno != EEXIST) {
+            gf_msg(this->name, GF_LOG_ERROR, fop_errno,
+                   DHT_MSG_MIGRATE_FILE_FAILED, "migrate-data failed for %s",
+                   entry_loc.path);
+
+            LOCK(&defrag->lock);
+            {
+                defrag->total_failures += 1;
+            }
+            UNLOCK(&defrag->lock);
+        }
 
-        loc_wipe (&entry_loc);
+        ret = gf_defrag_handle_migrate_error(fop_errno, defrag);
 
-        if (dict)
-                dict_unref(dict);
+        if (!ret) {
+            gf_msg(this->name, GF_LOG_ERROR, fop_errno,
+                   DHT_MSG_MIGRATE_FILE_FAILED,
+                   "migrate-data on %s failed:", entry_loc.path);
+        } else if (ret == 1) {
+            ret = 0;
+        }
 
-        if (fd)
-                fd_unref (fd);
-        return ret;
+        goto out;
+    }
+
+    LOCK(&defrag->lock);
+    {
+        defrag->total_files += 1;
+        defrag->total_data += iatt.ia_size;
+    }
+    UNLOCK(&defrag->lock);
+
+    if (defrag->stats == _gf_true) {
+        gettimeofday(&end, NULL);
+        elapsed = gf_tvdiff(&start, &end);
+        gf_log(this->name, GF_LOG_INFO,
+               "Migration of "
+               "file:%s size:%" PRIu64
+               " bytes took %.2f"
+               "secs and ret: %d",
+               entry_loc.name, iatt.ia_size, elapsed / 1e6, ret);
+    }
 
+out:
+    if (statfs_frame) {
+        STACK_DESTROY(statfs_frame->root);
+    }
+
+    if (iatt_ptr) {
+        LOCK(&defrag->lock);
+        {
+            defrag->size_processed += iatt_ptr->ia_size;
+        }
+        UNLOCK(&defrag->lock);
+    }
+    loc_wipe(&entry_loc);
+
+    return ret;
 }
 
-int
-gf_defrag_fix_layout (xlator_t *this, gf_defrag_info_t *defrag, loc_t *loc,
-                  dict_t *fix_layout, dict_t *migrate_data)
+void *
+gf_defrag_task(void *opaque)
 {
-        int                      ret            = -1;
-        loc_t                    entry_loc      = {0,};
-        fd_t                    *fd             = NULL;
-        gf_dirent_t              entries;
-        gf_dirent_t             *tmp            = NULL;
-        gf_dirent_t             *entry          = NULL;
-        gf_boolean_t             free_entries   = _gf_false;
-        off_t                    offset         = 0;
-        struct iatt              iatt           = {0,};
-        inode_t                 *linked_inode   = NULL, *inode = NULL;
-
-        ret = syncop_lookup (this, loc, &iatt, NULL, NULL, NULL);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "Lookup failed on %s",
-                        loc->path);
-                ret = -1;
-                goto out;
+    struct list_head *q_head = NULL;
+    struct dht_container *iterator = NULL;
+    gf_defrag_info_t *defrag = NULL;
+    int ret = 0;
+    pid_t pid = GF_CLIENT_PID_DEFRAG;
+
+    defrag = (gf_defrag_info_t *)opaque;
+    if (!defrag) {
+        gf_msg("dht", GF_LOG_ERROR, 0, 0, "defrag is NULL");
+        goto out;
+    }
+
+    syncopctx_setfspid(&pid);
+
+    q_head = &(defrag->queue[0].list);
+
+    /* The following while loop will dequeue one entry from the defrag->queue
+       under lock. We will update the defrag->global_error only when there
+       is an error which is critical to stop the rebalance process. The stop
+       message will be intimated to other migrator threads by setting the
+       defrag->defrag_status to GF_DEFRAG_STATUS_FAILED.
+
+       In defrag->queue, a low watermark (MIN_MIGRATE_QUEUE_COUNT) is
+       maintained so that crawler does not starve the file migration
+       workers and a high watermark (MAX_MIGRATE_QUEUE_COUNT) so that
+       crawler does not go far ahead in filling up the queue.
+     */
+
+    while (_gf_true) {
+        if (defrag->defrag_status != GF_DEFRAG_STATUS_STARTED) {
+            pthread_cond_broadcast(&defrag->rebalance_crawler_alarm);
+            pthread_cond_broadcast(&defrag->parallel_migration_cond);
+            goto out;
         }
 
-        if ((defrag->cmd != GF_DEFRAG_CMD_START_TIER) &&
-            (defrag->cmd != GF_DEFRAG_CMD_START_LAYOUT_FIX)) {
-                ret = gf_defrag_migrate_data (this, defrag, loc, migrate_data);
-                if (ret)
-                        goto out;
+        pthread_mutex_lock(&defrag->dfq_mutex);
+        {
+            /*Throttle down:
+              If the reconfigured count is less than current thread
+              count, then the current thread will sleep */
+
+            /*TODO: Need to refactor the following block to work
+             *under defrag->lock. For now access
+             * defrag->current_thread_count and rthcount under
+             * dfq_mutex lock */
+            while (!defrag->crawl_done && (defrag->recon_thread_count <
+                                           defrag->current_thread_count)) {
+                defrag->current_thread_count--;
+                gf_msg_debug("DHT", 0,
+                             "Thread sleeping. "
+                             "current thread count: %d",
+                             defrag->current_thread_count);
+
+                pthread_cond_wait(&defrag->df_wakeup_thread,
+                                  &defrag->dfq_mutex);
+
+                defrag->current_thread_count++;
+                gf_msg_debug("DHT", 0,
+                             "Thread wokeup. "
+                             "current thread count: %d",
+                             defrag->current_thread_count);
+            }
+
+            if (defrag->q_entry_count) {
+                iterator = list_entry(q_head->next, typeof(*iterator), list);
+
+                gf_msg_debug("DHT", 0,
+                             "picking entry "
+                             "%s",
+                             iterator->df_entry->d_name);
+
+                list_del_init(&(iterator->list));
+
+                defrag->q_entry_count--;
+
+                if ((defrag->q_entry_count < MIN_MIGRATE_QUEUE_COUNT) &&
+                    defrag->wakeup_crawler) {
+                    pthread_cond_broadcast(&defrag->rebalance_crawler_alarm);
+                }
+                pthread_mutex_unlock(&defrag->dfq_mutex);
+                ret = gf_defrag_migrate_single_file((void *)iterator);
+
+                /*Critical errors: ENOTCONN and ENOSPACE*/
+                if (ret) {
+                    dht_set_global_defrag_error(defrag, ret);
+
+                    defrag->defrag_status = GF_DEFRAG_STATUS_FAILED;
+
+                    pthread_cond_broadcast(&defrag->rebalance_crawler_alarm);
+
+                    pthread_cond_broadcast(&defrag->parallel_migration_cond);
+
+                    goto out;
+                }
+
+                gf_defrag_free_container(iterator);
+
+                continue;
+            } else {
+                /* defrag->crawl_done flag is set means crawling
+                 file system is done and hence a list_empty when
+                 the above flag is set indicates there are no more
+                 entries to be added to the queue and rebalance is
+                 finished */
+
+                if (!defrag->crawl_done) {
+                    defrag->current_thread_count--;
+                    gf_msg_debug("DHT", 0,
+                                 "Thread "
+                                 "sleeping while  waiting "
+                                 "for migration entries. "
+                                 "current thread  count:%d",
+                                 defrag->current_thread_count);
+
+                    pthread_cond_wait(&defrag->parallel_migration_cond,
+                                      &defrag->dfq_mutex);
+                }
+
+                if (defrag->crawl_done && !defrag->q_entry_count) {
+                    defrag->current_thread_count++;
+                    gf_msg_debug("DHT", 0, "Exiting thread");
+
+                    pthread_cond_broadcast(&defrag->parallel_migration_cond);
+                    goto unlock;
+                } else {
+                    defrag->current_thread_count++;
+                    gf_msg_debug("DHT", 0,
+                                 "Thread woke up"
+                                 " as found migrating entries. "
+                                 "current thread count:%d",
+                                 defrag->current_thread_count);
+
+                    pthread_mutex_unlock(&defrag->dfq_mutex);
+                    continue;
+                }
+            }
         }
+    unlock:
+        pthread_mutex_unlock(&defrag->dfq_mutex);
+        break;
+    }
+out:
+    return NULL;
+}
 
-        gf_msg_trace (this->name, 0, "fix layout called on %s", loc->path);
+int static gf_defrag_get_entry(xlator_t *this, int i,
+                               struct dht_container **container, loc_t *loc,
+                               dht_conf_t *conf, gf_defrag_info_t *defrag,
+                               fd_t *fd, dict_t *migrate_data,
+                               struct dir_dfmeta *dir_dfmeta, dict_t *xattr_req,
+                               int *perrno)
+{
+    int ret = 0;
+    char is_linkfile = 0;
+    gf_dirent_t *df_entry = NULL;
+    struct dht_container *tmp_container = NULL;
 
-        fd = fd_create (loc->inode, defrag->pid);
+    if (defrag->defrag_status != GF_DEFRAG_STATUS_STARTED) {
+        ret = -1;
+        goto out;
+    }
+
+    if (dir_dfmeta->offset_var[i].readdir_done == 1) {
+        ret = 0;
+        goto out;
+    }
+
+    if (dir_dfmeta->fetch_entries[i] == 1) {
         if (!fd) {
-                gf_log (this->name, GF_LOG_ERROR, "Failed to create fd");
-                ret = -1;
-                goto out;
+            dir_dfmeta->fetch_entries[i] = 0;
+            dir_dfmeta->offset_var[i].readdir_done = 1;
+            ret = 0;
+            goto out;
         }
 
-        ret = syncop_opendir (this, loc, fd, NULL, NULL);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "Failed to open dir %s",
-                        loc->path);
-                ret = -1;
-                goto out;
+        ret = syncop_readdirp(conf->local_subvols[i], fd, 131072,
+                              dir_dfmeta->offset_var[i].offset,
+                              &(dir_dfmeta->equeue[i]), xattr_req, NULL);
+        if (ret == 0) {
+            dir_dfmeta->offset_var[i].readdir_done = 1;
+            ret = 0;
+            goto out;
         }
 
-        INIT_LIST_HEAD (&entries.list);
-        while ((ret = syncop_readdirp (this, fd, 131072, offset, &entries,
-                                       NULL, NULL)) != 0)
-        {
+        if (ret < 0) {
+            gf_msg(this->name, GF_LOG_WARNING, -ret,
+                   DHT_MSG_MIGRATE_DATA_FAILED,
+                   "Readdirp failed. Aborting data migration for "
+                   "directory: %s",
+                   loc->path);
+            *perrno = -ret;
+            ret = -1;
+            goto out;
+        }
 
-                if (ret < 0) {
-                        gf_log (this->name, GF_LOG_ERROR, "Readdir returned %s"
-                                ". Aborting fix-layout",strerror(-ret));
-                        ret = -1;
-                        goto out;
-                }
+        if (list_empty(&(dir_dfmeta->equeue[i].list))) {
+            dir_dfmeta->offset_var[i].readdir_done = 1;
+            ret = 0;
+            goto out;
+        }
 
-                if (list_empty (&entries.list))
-                        break;
+        dir_dfmeta->fetch_entries[i] = 0;
+    }
 
-                free_entries = _gf_true;
+    while (1) {
+        if (defrag->defrag_status != GF_DEFRAG_STATUS_STARTED) {
+            ret = -1;
+            goto out;
+        }
 
-                list_for_each_entry_safe (entry, tmp, &entries.list, list) {
-                        if (defrag->defrag_status != GF_DEFRAG_STATUS_STARTED) {
-                                ret = 1;
-                                goto out;
-                        }
+        df_entry = list_entry(dir_dfmeta->iterator[i]->next, typeof(*df_entry),
+                              list);
 
-                        offset = entry->d_off;
+        if (&df_entry->list == dir_dfmeta->head[i]) {
+            gf_dirent_free(&(dir_dfmeta->equeue[i]));
+            INIT_LIST_HEAD(&(dir_dfmeta->equeue[i].list));
+            dir_dfmeta->fetch_entries[i] = 1;
+            dir_dfmeta->iterator[i] = dir_dfmeta->head[i];
+            ret = 0;
+            goto out;
+        }
 
-                        if (!strcmp (entry->d_name, ".") ||
-                            !strcmp (entry->d_name, ".."))
-                                continue;
+        dir_dfmeta->iterator[i] = dir_dfmeta->iterator[i]->next;
 
-                        if (!IA_ISDIR (entry->d_stat.ia_type))
-                                continue;
+        dir_dfmeta->offset_var[i].offset = df_entry->d_off;
+        if (!strcmp(df_entry->d_name, ".") || !strcmp(df_entry->d_name, ".."))
+            continue;
 
-                        loc_wipe (&entry_loc);
+        if (IA_ISDIR(df_entry->d_stat.ia_type)) {
+            defrag->size_processed += df_entry->d_stat.ia_size;
+            continue;
+        }
 
-                        ret =dht_build_child_loc (this, &entry_loc, loc,
-                                                  entry->d_name);
-                        if (ret) {
-                                gf_log (this->name, GF_LOG_ERROR, "Child loc"
-                                        " build failed");
-                                goto out;
-                        }
+        defrag->num_files_lookedup++;
 
-                        if (gf_uuid_is_null (entry->d_stat.ia_gfid)) {
-                                gf_log (this->name, GF_LOG_ERROR, "%s/%s"
-                                        " gfid not present", loc->path,
-                                         entry->d_name);
-                                continue;
-                        }
+        if (defrag->defrag_pattern &&
+            (gf_defrag_pattern_match(defrag, df_entry->d_name,
+                                     df_entry->d_stat.ia_size) == _gf_false)) {
+            defrag->size_processed += df_entry->d_stat.ia_size;
+            continue;
+        }
 
+        is_linkfile = check_is_linkfile(NULL, &df_entry->d_stat, df_entry->dict,
+                                        conf->link_xattr_name);
 
-                        gf_uuid_copy (entry_loc.gfid, entry->d_stat.ia_gfid);
+        if (is_linkfile) {
+            /* No need to add linkto file to the queue for
+               migration. Only the actual data file need to
+               be checked for migration criteria.
+            */
 
-                        /*In case the gfid stored in the inode by inode_link
-                         * and the gfid obtained in the lookup differs, then
-                         * client3_3_lookup_cbk will return ESTALE and proper
-                         * error will be captured
-                         */
+            gf_msg_debug(this->name, 0,
+                         "Skipping linkfile"
+                         " %s on subvol: %s",
+                         df_entry->d_name, conf->local_subvols[i]->name);
+            continue;
+        }
 
-                        linked_inode = inode_link (entry_loc.inode, loc->inode,
-                                                   entry->d_name,
-                                                   &entry->d_stat);
+        /*Build Container Structure */
 
-                        inode = entry_loc.inode;
-                        entry_loc.inode = linked_inode;
-                        inode_unref (inode);
+        tmp_container = GF_CALLOC(1, sizeof(struct dht_container),
+                                  gf_dht_mt_container_t);
+        if (!tmp_container) {
+            gf_log(this->name, GF_LOG_ERROR,
+                   "Failed to allocate "
+                   "memory for container");
+            ret = -1;
+            goto out;
+        }
+        tmp_container->df_entry = gf_dirent_for_name(df_entry->d_name);
+        if (!tmp_container->df_entry) {
+            gf_log(this->name, GF_LOG_ERROR,
+                   "Failed to allocate "
+                   "memory for df_entry");
+            ret = -1;
+            goto out;
+        }
 
-                        if (gf_uuid_is_null (loc->gfid)) {
-                                gf_log (this->name, GF_LOG_ERROR, "%s/%s"
-                                        " gfid not present", loc->path,
-                                         entry->d_name);
-                                continue;
-                        }
+        tmp_container->local_subvol_index = i;
 
-                        gf_uuid_copy (entry_loc.pargfid, loc->gfid);
+        tmp_container->df_entry->d_stat = df_entry->d_stat;
 
-                        ret = syncop_lookup (this, &entry_loc, &iatt, NULL,
-                                             NULL, NULL);
-                        if (ret) {
-                                gf_log (this->name, GF_LOG_ERROR, "%s"
-                                        " lookup failed", entry_loc.path);
-                                ret = -1;
-                                continue;
-                        }
+        tmp_container->df_entry->d_ino = df_entry->d_ino;
 
-                        ret = syncop_setxattr (this, &entry_loc, fix_layout,
-                                               0, NULL, NULL);
-                        if (ret) {
-                                gf_log (this->name, GF_LOG_ERROR, "Setxattr "
-                                        "failed for %s", entry_loc.path);
-                                defrag->defrag_status =
-                                GF_DEFRAG_STATUS_FAILED;
-                                defrag->total_failures ++;
-                                ret = -1;
-                                goto out;
-                        }
-                        ret = gf_defrag_fix_layout (this, defrag, &entry_loc,
-                                                    fix_layout, migrate_data);
-
-                        if (ret) {
-                                gf_msg (this->name, GF_LOG_ERROR, 0,
-                                        DHT_MSG_LAYOUT_FIX_FAILED,
-                                        "Fix layout failed for %s",
-                                        entry_loc.path);
-                                defrag->total_failures++;
-                                goto out;
-                        }
+        tmp_container->df_entry->d_type = df_entry->d_type;
 
-                }
-                gf_dirent_free (&entries);
-                free_entries = _gf_false;
-                INIT_LIST_HEAD (&entries.list);
+        tmp_container->df_entry->d_len = df_entry->d_len;
+
+        tmp_container->parent_loc = GF_CALLOC(1, sizeof(*loc), gf_dht_mt_loc_t);
+        if (!tmp_container->parent_loc) {
+            gf_log(this->name, GF_LOG_ERROR,
+                   "Failed to allocate "
+                   "memory for loc");
+            ret = -1;
+            goto out;
         }
 
-        ret = 0;
-out:
-        if (free_entries)
-                gf_dirent_free (&entries);
+        ret = loc_copy(tmp_container->parent_loc, loc);
+        if (ret) {
+            gf_log(this->name, GF_LOG_ERROR, "loc_copy failed");
+            ret = -1;
+            goto out;
+        }
 
-        loc_wipe (&entry_loc);
+        tmp_container->migrate_data = migrate_data;
 
-        if (fd)
-                fd_unref (fd);
+        tmp_container->this = this;
 
-        return ret;
+        if (df_entry->dict)
+            tmp_container->df_entry->dict = dict_ref(df_entry->dict);
+
+        /*Build Container Structure >> END*/
 
+        ret = 0;
+        goto out;
+    }
+
+out:
+    if (ret == 0) {
+        *container = tmp_container;
+    } else {
+        if (tmp_container) {
+            gf_defrag_free_container(tmp_container);
+        }
+    }
+
+    return ret;
 }
 
 int
-gf_defrag_start_crawl (void *data)
+gf_defrag_process_dir(xlator_t *this, gf_defrag_info_t *defrag, loc_t *loc,
+                      dict_t *migrate_data, int *perrno)
 {
-        xlator_t                *this   = NULL;
-        dht_conf_t              *conf   = NULL;
-        gf_defrag_info_t        *defrag = NULL;
-        int                      ret    = -1;
-        loc_t                    loc    = {0,};
-        struct iatt              iatt   = {0,};
-        struct iatt              parent = {0,};
-        dict_t                  *fix_layout = NULL;
-        dict_t                  *migrate_data = NULL;
-        dict_t                  *status = NULL;
-        glusterfs_ctx_t         *ctx = NULL;
-        dht_methods_t           *methods = NULL;
-
-        this = data;
-        if (!this)
-                goto out;
+    int ret = -1;
+    dht_conf_t *conf = NULL;
+    gf_dirent_t entries;
+    dict_t *xattr_req = NULL;
+    struct timeval dir_start = {
+        0,
+    };
+    struct timeval end = {
+        0,
+    };
+    double elapsed = {
+        0,
+    };
+    int local_subvols_cnt = 0;
+    int i = 0;
+    int j = 0;
+    struct dht_container *container = NULL;
+    int ldfq_count = 0;
+    int dfc_index = 0;
+    int throttle_up = 0;
+    struct dir_dfmeta *dir_dfmeta = NULL;
+    xlator_t *old_THIS = NULL;
+
+    gf_log(this->name, GF_LOG_INFO, "migrate data called on %s", loc->path);
+    gettimeofday(&dir_start, NULL);
+
+    conf = this->private;
+    local_subvols_cnt = conf->local_subvols_cnt;
+
+    if (!local_subvols_cnt) {
+        ret = 0;
+        goto out;
+    }
 
-        ctx = this->ctx;
-        if (!ctx)
-                goto out;
+    old_THIS = THIS;
+    THIS = this;
 
-        conf = this->private;
-        if (!conf)
-                goto out;
+    dir_dfmeta = GF_CALLOC(1, sizeof(*dir_dfmeta), gf_common_mt_pointer);
+    if (!dir_dfmeta) {
+        gf_log(this->name, GF_LOG_ERROR, "dir_dfmeta is NULL");
+        ret = -1;
+        goto out;
+    }
+
+    dir_dfmeta->lfd = GF_CALLOC(local_subvols_cnt, sizeof(fd_t *),
+                                gf_common_mt_pointer);
+    if (!dir_dfmeta->lfd) {
+        gf_smsg(this->name, GF_LOG_ERROR, ENOMEM, DHT_MSG_INSUFF_MEMORY,
+                "for dir_dfmeta", NULL);
+        ret = -1;
+        *perrno = ENOMEM;
+        goto out;
+    }
+
+    for (i = 0; i < local_subvols_cnt; i++) {
+        dir_dfmeta->lfd[i] = fd_create(loc->inode, defrag->pid);
+        if (!dir_dfmeta->lfd[i]) {
+            gf_smsg(this->name, GF_LOG_ERROR, ENOMEM, DHT_MSG_FD_CREATE_FAILED,
+                    NULL);
+            *perrno = ENOMEM;
+            ret = -1;
+            goto out;
+        }
 
-        defrag = conf->defrag;
-        if (!defrag)
+        ret = syncop_opendir(conf->local_subvols[i], loc, dir_dfmeta->lfd[i],
+                             NULL, NULL);
+        if (ret) {
+            fd_unref(dir_dfmeta->lfd[i]);
+            dir_dfmeta->lfd[i] = NULL;
+            gf_smsg(this->name, GF_LOG_WARNING, 0, DHT_MSG_FAILED_TO_OPEN,
+                    "dir: %s", loc->path, "subvol: %s",
+                    conf->local_subvols[i]->name, NULL);
+
+            if (conf->decommission_in_progress) {
+                *perrno = -ret;
+                ret = -1;
                 goto out;
+            }
+        } else {
+            fd_bind(dir_dfmeta->lfd[i]);
+        }
+    }
 
-        gettimeofday (&defrag->start_time, NULL);
-        dht_build_root_inode (this, &defrag->root_inode);
-        if (!defrag->root_inode)
-                goto out;
+    dir_dfmeta->head = GF_CALLOC(local_subvols_cnt, sizeof(*(dir_dfmeta->head)),
+                                 gf_common_mt_pointer);
+    if (!dir_dfmeta->head) {
+        gf_log(this->name, GF_LOG_ERROR, "dir_dfmeta->head is NULL");
+        ret = -1;
+        goto out;
+    }
+
+    dir_dfmeta->iterator = GF_CALLOC(local_subvols_cnt,
+                                     sizeof(*(dir_dfmeta->iterator)),
+                                     gf_common_mt_pointer);
+    if (!dir_dfmeta->iterator) {
+        gf_log(this->name, GF_LOG_ERROR, "dir_dfmeta->iterator is NULL");
+        ret = -1;
+        goto out;
+    }
 
-        dht_build_root_loc (defrag->root_inode, &loc);
+    dir_dfmeta->equeue = GF_CALLOC(local_subvols_cnt, sizeof(entries),
+                                   gf_dht_mt_dirent_t);
+    if (!dir_dfmeta->equeue) {
+        gf_log(this->name, GF_LOG_ERROR, "dir_dfmeta->equeue is NULL");
+        ret = -1;
+        goto out;
+    }
 
-        /* fix-layout on '/' first */
+    dir_dfmeta->offset_var = GF_CALLOC(
+        local_subvols_cnt, sizeof(dht_dfoffset_ctx_t), gf_dht_mt_octx_t);
+    if (!dir_dfmeta->offset_var) {
+        gf_log(this->name, GF_LOG_ERROR, "dir_dfmeta->offset_var is NULL");
+        ret = -1;
+        goto out;
+    }
+
+    ret = gf_defrag_ctx_subvols_init(dir_dfmeta->offset_var, this);
+    if (ret) {
+        gf_log(this->name, GF_LOG_ERROR,
+               "dht_dfoffset_ctx_t"
+               "initialization failed");
+        ret = -1;
+        goto out;
+    }
+
+    dir_dfmeta->fetch_entries = GF_CALLOC(local_subvols_cnt, sizeof(int),
+                                          gf_common_mt_int);
+    if (!dir_dfmeta->fetch_entries) {
+        gf_smsg(this->name, GF_LOG_ERROR, ENOMEM, DHT_MSG_INSUFF_MEMORY,
+                "for dir_dfmeta->fetch_entries", NULL);
+        ret = -1;
+        goto out;
+    }
+
+    for (i = 0; i < local_subvols_cnt; i++) {
+        INIT_LIST_HEAD(&(dir_dfmeta->equeue[i].list));
+        dir_dfmeta->head[i] = &(dir_dfmeta->equeue[i].list);
+        dir_dfmeta->iterator[i] = dir_dfmeta->head[i];
+        dir_dfmeta->fetch_entries[i] = 1;
+    }
+
+    xattr_req = dict_new();
+    if (!xattr_req) {
+        gf_log(this->name, GF_LOG_ERROR, "dict_new failed");
+        ret = -1;
+        goto out;
+    }
+
+    ret = dict_set_uint32(xattr_req, conf->link_xattr_name, 256);
+    if (ret) {
+        gf_log(this->name, GF_LOG_ERROR,
+               "failed to set dict for "
+               "key: %s",
+               conf->link_xattr_name);
+        ret = -1;
+        goto out;
+    }
 
-        ret = syncop_lookup (this, &loc, &iatt, &parent, NULL, NULL);
+    /*
+     Job: Read entries from each local subvol and store the entries
+          in equeue array of linked list. Now pick one entry from the
+          equeue array in a round robin basis and add them to defrag Queue.
+    */
 
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        DHT_MSG_REBALANCE_START_FAILED,
-                        "Failed to start rebalance: look up on / failed");
-                ret = -1;
-                goto out;
+    while (!dht_dfreaddirp_done(dir_dfmeta->offset_var, local_subvols_cnt)) {
+        pthread_mutex_lock(&defrag->dfq_mutex);
+        {
+            /*Throttle up: If reconfigured count is higher than
+              current thread count, wake up the sleeping threads
+              TODO: Need to refactor this. Instead of making the
+              thread sleep and wake, we should terminate and spawn
+              threads on-demand*/
+
+            if (defrag->recon_thread_count > defrag->current_thread_count) {
+                throttle_up = (defrag->recon_thread_count -
+                               defrag->current_thread_count);
+                for (j = 0; j < throttle_up; j++) {
+                    pthread_cond_signal(&defrag->df_wakeup_thread);
+                }
+            }
+
+            while (defrag->q_entry_count > MAX_MIGRATE_QUEUE_COUNT) {
+                defrag->wakeup_crawler = 1;
+                pthread_cond_wait(&defrag->rebalance_crawler_alarm,
+                                  &defrag->dfq_mutex);
+            }
+
+            ldfq_count = defrag->q_entry_count;
+
+            if (defrag->wakeup_crawler) {
+                defrag->wakeup_crawler = 0;
+            }
         }
+        pthread_mutex_unlock(&defrag->dfq_mutex);
+
+        while (
+            ldfq_count <= MAX_MIGRATE_QUEUE_COUNT &&
+            !dht_dfreaddirp_done(dir_dfmeta->offset_var, local_subvols_cnt)) {
+            ret = gf_defrag_get_entry(this, dfc_index, &container, loc, conf,
+                                      defrag, dir_dfmeta->lfd[dfc_index],
+                                      migrate_data, dir_dfmeta, xattr_req,
+                                      perrno);
+
+            if (defrag->defrag_status == GF_DEFRAG_STATUS_STOPPED) {
+                goto out;
+            }
+
+            if (ret) {
+                gf_log(this->name, GF_LOG_WARNING,
+                       "Found "
+                       "error from gf_defrag_get_entry");
 
-        fix_layout = dict_new ();
-        if (!fix_layout) {
                 ret = -1;
                 goto out;
+            }
+
+            /* Check if we got an entry, else we need to move the
+               index to the next subvol */
+            if (!container) {
+                GF_CRAWL_INDEX_MOVE(dfc_index, local_subvols_cnt);
+                continue;
+            }
+
+            /* Q this entry in the dfq */
+            pthread_mutex_lock(&defrag->dfq_mutex);
+            {
+                list_add_tail(&container->list, &(defrag->queue[0].list));
+                defrag->q_entry_count++;
+                ldfq_count = defrag->q_entry_count;
+
+                gf_msg_debug(this->name, 0,
+                             "added "
+                             "file:%s parent:%s to the queue ",
+                             container->df_entry->d_name,
+                             container->parent_loc->path);
+
+                pthread_cond_signal(&defrag->parallel_migration_cond);
+            }
+            pthread_mutex_unlock(&defrag->dfq_mutex);
+
+            GF_CRAWL_INDEX_MOVE(dfc_index, local_subvols_cnt);
         }
+    }
+
+    gettimeofday(&end, NULL);
+    elapsed = gf_tvdiff(&dir_start, &end);
+    gf_log(this->name, GF_LOG_INFO,
+           "Migration operation on dir %s took "
+           "%.2f secs",
+           loc->path, elapsed / 1e6);
+    ret = 0;
+out:
+    THIS = old_THIS;
+    gf_defrag_free_dir_dfmeta(dir_dfmeta, local_subvols_cnt);
+
+    if (xattr_req)
+        dict_unref(xattr_req);
+
+    /* It does not matter if it errored out - this number is
+     * used to calculate rebalance estimated time to complete.
+     * No locking required as dirs are processed by a single thread.
+     */
+    defrag->num_dirs_processed++;
+    return ret;
+}
 
-        ret = dict_set_str (fix_layout, GF_XATTR_FIX_LAYOUT_KEY, "yes");
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        DHT_MSG_REBALANCE_START_FAILED,
-                        "Failed to start rebalance:"
-                        "Failed to set dictionary value: key = %s",
-                        GF_XATTR_FIX_LAYOUT_KEY);
+int
+gf_defrag_settle_hash(xlator_t *this, gf_defrag_info_t *defrag, loc_t *loc,
+                      dict_t *fix_layout)
+{
+    int ret;
+    dht_conf_t *conf = NULL;
+    /*
+     * Now we're ready to update the directory commit hash for the volume
+     * root, so that hash miscompares and broadcast lookups can stop.
+     * However, we want to skip that if fix-layout is all we did.  In
+     * that case, we want the miscompares etc. to continue until a real
+     * rebalance is complete.
+     */
+    if (defrag->cmd == GF_DEFRAG_CMD_START_LAYOUT_FIX ||
+        defrag->cmd == GF_DEFRAG_CMD_DETACH_START) {
+        return 0;
+    }
 
-                goto out;
+    conf = this->private;
+    if (!conf) {
+        /*Uh oh
+         */
+        return -1;
+    }
+
+    if (conf->local_subvols_cnt == 0 || !conf->lookup_optimize) {
+        /* Commit hash updates are only done on local subvolumes and
+         * only when lookup optimization is needed (for older client
+         * support)
+         */
+        return 0;
+    }
+
+    ret = dict_set_uint32(fix_layout, "new-commit-hash",
+                          defrag->new_commit_hash);
+    if (ret) {
+        gf_log(this->name, GF_LOG_ERROR, "Failed to set new-commit-hash");
+        return -1;
+    }
+
+    ret = syncop_setxattr(this, loc, fix_layout, 0, NULL, NULL);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, -ret, DHT_MSG_LAYOUT_FIX_FAILED,
+               "fix layout on %s failed", loc->path);
+
+        if (-ret == ENOENT || -ret == ESTALE) {
+            /* Dir most likely is deleted */
+            return 0;
         }
 
-        ret = syncop_setxattr (this, &loc, fix_layout, 0, NULL, NULL);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        DHT_MSG_REBALANCE_FAILED,
-                        "fix layout on %s failed",
-                        loc.path);
+        return -1;
+    }
+
+    /* TBD: find more efficient solution than adding/deleting every time */
+    dict_del(fix_layout, "new-commit-hash");
+
+    return 0;
+}
+
+int
+gf_defrag_fix_layout(xlator_t *this, gf_defrag_info_t *defrag, loc_t *loc,
+                     dict_t *fix_layout, dict_t *migrate_data)
+{
+    int ret = -1;
+    loc_t entry_loc = {
+        0,
+    };
+    fd_t *fd = NULL;
+    gf_dirent_t entries;
+    gf_dirent_t *tmp = NULL;
+    gf_dirent_t *entry = NULL;
+    gf_boolean_t free_entries = _gf_false;
+    off_t offset = 0;
+    struct iatt iatt = {
+        0,
+    };
+    inode_t *linked_inode = NULL, *inode = NULL;
+    dht_conf_t *conf = NULL;
+    int perrno = 0;
+
+    conf = this->private;
+    if (!conf) {
+        ret = -1;
+        goto out;
+    }
+
+    ret = syncop_lookup(this, loc, &iatt, NULL, NULL, NULL);
+    if (ret) {
+        if (strcmp(loc->path, "/") == 0) {
+            gf_msg(this->name, GF_LOG_ERROR, -ret, DHT_MSG_DIR_LOOKUP_FAILED,
+                   "lookup failed for:%s", loc->path);
+
+            defrag->total_failures++;
+            ret = -1;
+            goto out;
+        }
+
+        if (-ret == ENOENT || -ret == ESTALE) {
+            gf_msg(this->name, GF_LOG_INFO, -ret, DHT_MSG_DIR_LOOKUP_FAILED,
+                   "Dir:%s renamed or removed. Skipping", loc->path);
+            if (conf->decommission_subvols_cnt) {
                 defrag->total_failures++;
-                ret = -1;
-                goto out;
+            }
+            ret = 0;
+            goto out;
+        } else {
+            gf_msg(this->name, GF_LOG_ERROR, -ret, DHT_MSG_DIR_LOOKUP_FAILED,
+                   "lookup failed for:%s", loc->path);
+
+            defrag->total_failures++;
+            goto out;
         }
+    }
 
-        if ((defrag->cmd != GF_DEFRAG_CMD_START_TIER) &&
-            (defrag->cmd != GF_DEFRAG_CMD_START_LAYOUT_FIX)) {
-                migrate_data = dict_new ();
-                if (!migrate_data) {
-                        ret = -1;
-                        goto out;
+    fd = fd_create(loc->inode, defrag->pid);
+    if (!fd) {
+        gf_log(this->name, GF_LOG_ERROR, "Failed to create fd");
+        ret = -1;
+        goto out;
+    }
+
+    ret = syncop_opendir(this, loc, fd, NULL, NULL);
+    if (ret) {
+        if (-ret == ENOENT || -ret == ESTALE) {
+            if (conf->decommission_subvols_cnt) {
+                defrag->total_failures++;
+            }
+            ret = 0;
+            goto out;
+        }
+
+        gf_log(this->name, GF_LOG_ERROR,
+               "Failed to open dir %s, "
+               "err:%d",
+               loc->path, -ret);
+
+        ret = -1;
+        goto out;
+    }
+
+    fd_bind(fd);
+    INIT_LIST_HEAD(&entries.list);
+
+    while ((ret = syncop_readdirp(this, fd, 131072, offset, &entries, NULL,
+                                  NULL)) != 0) {
+        if (ret < 0) {
+            if (-ret == ENOENT || -ret == ESTALE) {
+                if (conf->decommission_subvols_cnt) {
+                    defrag->total_failures++;
                 }
-                if (defrag->cmd == GF_DEFRAG_CMD_START_FORCE)
-                        ret = dict_set_str (migrate_data,
-                                            GF_XATTR_FILE_MIGRATE_KEY,
-                                            "force");
-                else
-                        ret = dict_set_str (migrate_data,
-                                            GF_XATTR_FILE_MIGRATE_KEY,
-                                            "non-force");
-                if (ret)
-                        goto out;
+                ret = 0;
+                goto out;
+            }
+
+            gf_msg(this->name, GF_LOG_ERROR, -ret, DHT_MSG_READDIR_ERROR,
+                   "readdirp failed for "
+                   "path %s. Aborting fix-layout",
+                   loc->path);
+
+            ret = -1;
+            goto out;
         }
 
-        ret = gf_defrag_fix_layout (this, defrag, &loc, fix_layout,
-                                    migrate_data);
+        if (list_empty(&entries.list))
+            break;
 
-        if (defrag->cmd == GF_DEFRAG_CMD_START_TIER) {
-                methods = conf->methods;
-                if (!methods) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                DHT_MSG_LOG_TIER_ERROR,
-                                "Methods invalid for translator.");
+        free_entries = _gf_true;
+
+        list_for_each_entry_safe(entry, tmp, &entries.list, list)
+        {
+            if (defrag->defrag_status != GF_DEFRAG_STATUS_STARTED) {
+                ret = 1;
+                goto out;
+            }
+
+            offset = entry->d_off;
+
+            if (!strcmp(entry->d_name, ".") || !strcmp(entry->d_name, ".."))
+                continue;
+            if (!IA_ISDIR(entry->d_stat.ia_type)) {
+                continue;
+            }
+            loc_wipe(&entry_loc);
+
+            ret = dht_build_child_loc(this, &entry_loc, loc, entry->d_name);
+            if (ret) {
+                gf_log(this->name, GF_LOG_ERROR,
+                       "Child loc"
+                       " build failed for entry: %s",
+                       entry->d_name);
+
+                if (conf->decommission_in_progress) {
+                    defrag->defrag_status = GF_DEFRAG_STATUS_FAILED;
+
+                    goto out;
+                } else {
+                    continue;
+                }
+            }
+
+            if (gf_uuid_is_null(entry->d_stat.ia_gfid)) {
+                gf_log(this->name, GF_LOG_ERROR,
+                       "%s/%s"
+                       " gfid not present",
+                       loc->path, entry->d_name);
+                continue;
+            }
+
+            gf_uuid_copy(entry_loc.gfid, entry->d_stat.ia_gfid);
+
+            /*In case the gfid stored in the inode by inode_link
+             * and the gfid obtained in the lookup differs, then
+             * client3_3_lookup_cbk will return ESTALE and proper
+             * error will be captured
+             */
+
+            linked_inode = inode_link(entry_loc.inode, loc->inode,
+                                      entry->d_name, &entry->d_stat);
+
+            inode = entry_loc.inode;
+            entry_loc.inode = linked_inode;
+            inode_unref(inode);
+
+            if (gf_uuid_is_null(loc->gfid)) {
+                gf_log(this->name, GF_LOG_ERROR,
+                       "%s/%s"
+                       " gfid not present",
+                       loc->path, entry->d_name);
+                continue;
+            }
+
+            gf_uuid_copy(entry_loc.pargfid, loc->gfid);
+
+            ret = syncop_lookup(this, &entry_loc, &iatt, NULL, NULL, NULL);
+            if (ret) {
+                if (-ret == ENOENT || -ret == ESTALE) {
+                    gf_msg(this->name, GF_LOG_INFO, -ret,
+                           DHT_MSG_DIR_LOOKUP_FAILED,
+                           "Dir:%s renamed or removed. "
+                           "Skipping",
+                           loc->path);
+                    ret = 0;
+                    if (conf->decommission_subvols_cnt) {
+                        defrag->total_failures++;
+                    }
+                    continue;
+                } else {
+                    gf_msg(this->name, GF_LOG_ERROR, -ret,
+                           DHT_MSG_DIR_LOOKUP_FAILED, "lookup failed for:%s",
+                           entry_loc.path);
+
+                    defrag->total_failures++;
+
+                    if (conf->decommission_in_progress) {
                         defrag->defrag_status = GF_DEFRAG_STATUS_FAILED;
                         ret = -1;
                         goto out;
+                    } else {
+                        continue;
+                    }
                 }
-                methods->migration_other(this, defrag);
+            }
+
+            /* A return value of 2 means, either process_dir or
+             * lookup of a dir failed. Hence, don't commit hash
+             * for the current directory*/
+
+            ret = gf_defrag_fix_layout(this, defrag, &entry_loc, fix_layout,
+                                       migrate_data);
+
+            if (defrag->defrag_status == GF_DEFRAG_STATUS_STOPPED ||
+                defrag->defrag_status == GF_DEFRAG_STATUS_FAILED) {
+                goto out;
+            }
+
+            if (ret) {
+                gf_msg(this->name, GF_LOG_ERROR, 0, DHT_MSG_LAYOUT_FIX_FAILED,
+                       "Fix layout failed for %s", entry_loc.path);
+
+                defrag->total_failures++;
+
+                if (conf->decommission_in_progress) {
+                    defrag->defrag_status = GF_DEFRAG_STATUS_FAILED;
+
+                    goto out;
+                } else {
+                    /* Let's not commit-hash if
+                     * gf_defrag_fix_layout failed*/
+                    continue;
+                }
+            }
         }
 
-        if ((defrag->defrag_status != GF_DEFRAG_STATUS_STOPPED) &&
-            (defrag->defrag_status != GF_DEFRAG_STATUS_FAILED)) {
-                defrag->defrag_status = GF_DEFRAG_STATUS_COMPLETE;
+        gf_dirent_free(&entries);
+        free_entries = _gf_false;
+        INIT_LIST_HEAD(&entries.list);
+    }
+
+    /* A directory layout is fixed only after its subdirs are healed to
+     * any newly added bricks. If the layout is fixed before subdirs are
+     * healed, the newly added brick will get a non-null layout.
+     * Any subdirs which hash to that layout will no longer show up
+     * in a directory listing until they are healed.
+     */
+
+    ret = syncop_setxattr(this, loc, fix_layout, 0, NULL, NULL);
+
+    /* In case of a race where the directory is deleted just before
+     * layout setxattr, the errors are updated in the layout structure.
+     * We can use this information to make a decision whether the directory
+     * is deleted entirely.
+     */
+    if (ret == 0) {
+        ret = dht_dir_layout_error_check(this, loc->inode);
+        ret = -ret;
+    }
+
+    if (ret) {
+        if (-ret == ENOENT || -ret == ESTALE) {
+            gf_msg(this->name, GF_LOG_INFO, -ret, DHT_MSG_LAYOUT_FIX_FAILED,
+                   "Setxattr failed. Dir %s "
+                   "renamed or removed",
+                   loc->path);
+            if (conf->decommission_subvols_cnt) {
+                defrag->total_failures++;
+            }
+            ret = 0;
+            goto out;
+        } else {
+            gf_msg(this->name, GF_LOG_ERROR, -ret, DHT_MSG_LAYOUT_FIX_FAILED,
+                   "Setxattr failed for %s", loc->path);
+
+            defrag->total_failures++;
+
+            if (conf->decommission_in_progress) {
+                defrag->defrag_status = GF_DEFRAG_STATUS_FAILED;
+                ret = -1;
+                goto out;
+            }
         }
+    }
 
-out:
-        LOCK (&defrag->lock);
-        {
-                status = dict_new ();
-                gf_defrag_status_get (defrag, status);
-                if (ctx && ctx->notify)
-                        ctx->notify (GF_EN_DEFRAG_STATUS, status);
-                if (status)
-                        dict_unref (status);
-                defrag->is_exiting = 1;
+    if (defrag->cmd != GF_DEFRAG_CMD_START_LAYOUT_FIX) {
+        ret = gf_defrag_process_dir(this, defrag, loc, migrate_data, &perrno);
+
+        if (ret) {
+            if (perrno == ENOENT || perrno == ESTALE) {
+                ret = 0;
+                goto out;
+            } else {
+                defrag->total_failures++;
+
+                gf_msg(this->name, GF_LOG_ERROR, 0,
+                       DHT_MSG_DEFRAG_PROCESS_DIR_FAILED,
+                       "gf_defrag_process_dir failed for "
+                       "directory: %s",
+                       loc->path);
+
+                if (conf->decommission_in_progress) {
+                    goto out;
+                }
+            }
         }
-        UNLOCK (&defrag->lock);
+    }
+
+    gf_msg_trace(this->name, 0, "fix layout called on %s", loc->path);
+
+    if (gf_defrag_settle_hash(this, defrag, loc, fix_layout) != 0) {
+        defrag->total_failures++;
+
+        gf_msg(this->name, GF_LOG_ERROR, 0, DHT_MSG_SETTLE_HASH_FAILED,
+               "Settle hash failed for %s", loc->path);
 
-        if (defrag) {
-                GF_FREE (defrag);
-                conf->defrag = NULL;
+        ret = -1;
+
+        if (conf->decommission_in_progress) {
+            defrag->defrag_status = GF_DEFRAG_STATUS_FAILED;
+            goto out;
         }
+    }
+
+    ret = 0;
+out:
+    if (free_entries)
+        gf_dirent_free(&entries);
+
+    loc_wipe(&entry_loc);
+
+    if (fd)
+        fd_unref(fd);
+
+    return ret;
+}
+
+int
+dht_init_local_subvols_and_nodeuuids(xlator_t *this, dht_conf_t *conf,
+                                     loc_t *loc)
+{
+    dict_t *dict = NULL;
+    uuid_t *uuid_ptr = NULL;
+    int ret = -1;
+    int i = 0;
+    int j = 0;
+
+    /* Find local subvolumes */
+    ret = syncop_getxattr(this, loc, &dict, GF_REBAL_FIND_LOCAL_SUBVOL, NULL,
+                          NULL);
+    if (ret && (ret != -ENODATA)) {
+        gf_msg(this->name, GF_LOG_ERROR, -ret, 0,
+               "local "
+               "subvolume determination failed with error: %d",
+               -ret);
+        ret = -1;
+        goto out;
+    }
+
+    if (!ret)
+        goto out;
+
+    ret = syncop_getxattr(this, loc, &dict, GF_REBAL_OLD_FIND_LOCAL_SUBVOL,
+                          NULL, NULL);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, -ret, 0,
+               "local "
+               "subvolume determination failed with error: %d",
+               -ret);
+        ret = -1;
+        goto out;
+    }
+    ret = 0;
 
+out:
+    if (ret) {
         return ret;
+    }
+
+    for (i = 0; i < conf->local_subvols_cnt; i++) {
+        gf_msg(this->name, GF_LOG_INFO, 0, 0,
+               "local subvol: "
+               "%s",
+               conf->local_subvols[i]->name);
+
+        for (j = 0; j < conf->local_nodeuuids[i].count; j++) {
+            uuid_ptr = &(conf->local_nodeuuids[i].elements[j].uuid);
+            gf_msg(this->name, GF_LOG_INFO, 0, 0, "node uuid : %s",
+                   uuid_utoa(*uuid_ptr));
+        }
+    }
+
+    return ret;
 }
 
+/* Functions for the rebalance estimates feature */
 
-static int
-gf_defrag_done  (int ret, call_frame_t *sync_frame, void *data)
+uint64_t
+gf_defrag_subvol_file_size(xlator_t *this, loc_t *root_loc)
+{
+    int ret = -1;
+    struct statvfs buf = {
+        0,
+    };
+
+    ret = syncop_statfs(this, root_loc, &buf, NULL, NULL);
+    if (ret) {
+        /* Aargh! */
+        return 0;
+    }
+    return ((buf.f_blocks - buf.f_bfree) * buf.f_frsize);
+}
+
+uint64_t
+gf_defrag_total_file_size(xlator_t *this, loc_t *root_loc)
 {
-        gf_listener_stop (sync_frame->this);
+    dht_conf_t *conf = NULL;
+    int i = 0;
+    uint64_t size_files = 0;
+    uint64_t total_size = 0;
 
-        STACK_DESTROY (sync_frame->root);
-        kill (getpid(), SIGTERM);
+    conf = this->private;
+    if (!conf) {
         return 0;
+    }
+
+    for (i = 0; i < conf->local_subvols_cnt; i++) {
+        size_files = gf_defrag_subvol_file_size(conf->local_subvols[i],
+                                                root_loc);
+        total_size += size_files;
+        gf_msg(this->name, GF_LOG_INFO, 0, 0,
+               "local subvol: %s,"
+               "cnt = %" PRIu64,
+               conf->local_subvols[i]->name, size_files);
+    }
+
+    gf_msg(this->name, GF_LOG_INFO, 0, 0, "Total size files = %" PRIu64,
+           total_size);
+
+    return total_size;
 }
 
-void *
-gf_defrag_start (void *data)
+static void *
+dht_file_counter_thread(void *args)
 {
-        int                      ret    = -1;
-        call_frame_t            *frame  = NULL;
-        dht_conf_t              *conf   = NULL;
-        gf_defrag_info_t        *defrag = NULL;
-        xlator_t                *this  = NULL;
-
-        this = data;
-        conf = this->private;
-        if (!conf)
-                goto out;
+    gf_defrag_info_t *defrag = NULL;
+    loc_t root_loc = {
+        0,
+    };
+    struct timespec time_to_wait = {
+        0,
+    };
+    uint64_t tmp_size = 0;
+
+    if (!args)
+        return NULL;
 
-        defrag = conf->defrag;
-        if (!defrag)
-                goto out;
+    defrag = (gf_defrag_info_t *)args;
+    dht_build_root_loc(defrag->root_inode, &root_loc);
 
-        frame = create_frame (this, this->ctx->pool);
-        if (!frame)
-                goto out;
+    while (defrag->defrag_status == GF_DEFRAG_STATUS_STARTED) {
+        timespec_now(&time_to_wait);
+        time_to_wait.tv_sec += 600;
 
-        /* If its a tiering rebalancer mark it seperately so that CTR Xlator
-         * can take appropriate action */
-        if (defrag->cmd == GF_DEFRAG_CMD_START_TIER)
-                frame->root->pid = GF_CLIENT_PID_TIER_DEFRAG;
-        else
-                frame->root->pid = GF_CLIENT_PID_DEFRAG;
+        pthread_mutex_lock(&defrag->fc_mutex);
+        pthread_cond_timedwait(&defrag->fc_wakeup_cond, &defrag->fc_mutex,
+                               &time_to_wait);
 
-        defrag->pid = frame->root->pid;
+        pthread_mutex_unlock(&defrag->fc_mutex);
 
-        defrag->defrag_status = GF_DEFRAG_STATUS_STARTED;
+        if (defrag->defrag_status != GF_DEFRAG_STATUS_STARTED)
+            break;
 
-        ret = synctask_new (this->ctx->env, gf_defrag_start_crawl,
-                            gf_defrag_done, frame, this);
+        tmp_size = gf_defrag_total_file_size(defrag->this, &root_loc);
 
-        if (ret)
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        DHT_MSG_REBALANCE_START_FAILED,
-                        "Could not create task for rebalance");
+        gf_log("dht", GF_LOG_INFO, "tmp data size =%" PRIu64, tmp_size);
+
+        if (!tmp_size) {
+            gf_msg("dht", GF_LOG_ERROR, 0, 0,
+                   "Failed to get "
+                   "the total data size. Unable to estimate "
+                   "time to complete rebalance.");
+        } else {
+            g_totalsize = tmp_size;
+            gf_msg_debug("dht", 0, "total data size =%" PRIu64, g_totalsize);
+        }
+    }
+
+    return NULL;
+}
+
+int
+gf_defrag_estimates_cleanup(xlator_t *this, gf_defrag_info_t *defrag,
+                            pthread_t filecnt_thread)
+{
+    int ret = -1;
+
+    /* Wake up the filecounter thread.
+     * By now the defrag status will no longer be
+     * GF_DEFRAG_STATUS_STARTED so the thread will exit the loop.
+     */
+    pthread_mutex_lock(&defrag->fc_mutex);
+    {
+        pthread_cond_broadcast(&defrag->fc_wakeup_cond);
+    }
+    pthread_mutex_unlock(&defrag->fc_mutex);
+
+    ret = pthread_join(filecnt_thread, NULL);
+    if (ret) {
+        gf_msg("dht", GF_LOG_ERROR, ret, 0,
+               "file_counter_thread: pthread_join failed.");
+        ret = -1;
+    }
+    return ret;
+}
+
+int
+gf_defrag_estimates_init(xlator_t *this, loc_t *loc, pthread_t *filecnt_thread)
+{
+    int ret = -1;
+    dht_conf_t *conf = NULL;
+    gf_defrag_info_t *defrag = NULL;
+
+    conf = this->private;
+    defrag = conf->defrag;
+
+    g_totalsize = gf_defrag_total_file_size(this, loc);
+    if (!g_totalsize) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, 0,
+               "Failed to get "
+               "the total data size. Unable to estimate "
+               "time to complete rebalance.");
+        goto out;
+    }
+
+    ret = gf_thread_create(filecnt_thread, NULL, dht_file_counter_thread,
+                           (void *)defrag, "dhtfcnt");
+
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, ret, 0,
+               "Failed to "
+               "create the file counter thread ");
+        ret = -1;
+        goto out;
+    }
+    ret = 0;
 out:
-        return NULL;
+    return ret;
 }
 
+/* Init and cleanup functions for parallel file migration*/
 int
-gf_defrag_status_get (gf_defrag_info_t *defrag, dict_t *dict)
+gf_defrag_parallel_migration_init(xlator_t *this, gf_defrag_info_t *defrag,
+                                  pthread_t **tid_array, int *thread_index)
 {
-        int      ret    = 0;
-        uint64_t files  = 0;
-        uint64_t size   = 0;
-        uint64_t lookup = 0;
-        uint64_t failures = 0;
-        uint64_t skipped = 0;
-        char     *status = "";
-        double   elapsed = 0;
-        struct timeval end = {0,};
-
-
-        if (!defrag)
-                goto out;
+    int ret = -1;
+    int thread_spawn_count = 0;
+    int index = 0;
+    pthread_t *tid = NULL;
 
-        ret = 0;
-        if (defrag->defrag_status == GF_DEFRAG_STATUS_NOT_STARTED)
-                goto out;
+    if (!defrag)
+        goto out;
+
+    /* Initialize global entry queue */
+    defrag->queue = GF_CALLOC(1, sizeof(struct dht_container),
+                              gf_dht_mt_container_t);
+
+    if (!defrag->queue) {
+        gf_msg(this->name, GF_LOG_ERROR, ENOMEM, 0,
+               "Failed to initialise migration queue");
+        ret = -1;
+        goto out;
+    }
+
+    INIT_LIST_HEAD(&(defrag->queue[0].list));
+
+    thread_spawn_count = MAX(MAX_REBAL_THREADS, 4);
+
+    gf_msg_debug(this->name, 0, "thread_spawn_count: %d", thread_spawn_count);
+
+    tid = GF_CALLOC(thread_spawn_count, sizeof(pthread_t),
+                    gf_common_mt_pthread_t);
+    if (!tid) {
+        gf_msg(this->name, GF_LOG_ERROR, ENOMEM, 0,
+               "Failed to create migration threads");
+        ret = -1;
+        goto out;
+    }
+    defrag->current_thread_count = thread_spawn_count;
+
+    /*Spawn Threads Here*/
+    while (index < thread_spawn_count) {
+        ret = gf_thread_create(&(tid[index]), NULL, gf_defrag_task,
+                               (void *)defrag, "dhtmig%d", (index + 1) & 0x3ff);
+        if (ret != 0) {
+            gf_msg("DHT", GF_LOG_ERROR, ret, 0, "Thread[%d] creation failed. ",
+                   index);
+            ret = -1;
+            goto out;
+        } else {
+            gf_log("DHT", GF_LOG_INFO,
+                   "Thread[%d] "
+                   "creation successful",
+                   index);
+        }
+        index++;
+    }
+
+    ret = 0;
+out:
+    *thread_index = index;
+    *tid_array = tid;
+
+    return ret;
+}
+
+int
+gf_defrag_parallel_migration_cleanup(gf_defrag_info_t *defrag,
+                                     pthread_t *tid_array, int thread_index)
+{
+    int ret = -1;
+    int i = 0;
+
+    if (!defrag)
+        goto out;
+
+    /* Wake up all migration threads */
+    pthread_mutex_lock(&defrag->dfq_mutex);
+    {
+        defrag->crawl_done = 1;
+
+        pthread_cond_broadcast(&defrag->parallel_migration_cond);
+        pthread_cond_broadcast(&defrag->df_wakeup_thread);
+    }
+    pthread_mutex_unlock(&defrag->dfq_mutex);
+
+    /*Wait for all the threads to complete their task*/
+    for (i = 0; i < thread_index; i++) {
+        pthread_join(tid_array[i], NULL);
+    }
+
+    GF_FREE(tid_array);
 
-        files  = defrag->total_files;
-        size   = defrag->total_data;
-        lookup = defrag->num_files_lookedup;
-        failures = defrag->total_failures;
-        skipped = defrag->skipped;
+    /* Cleanup the migration queue */
+    if (defrag->queue) {
+        gf_dirent_free(defrag->queue[0].df_entry);
+        INIT_LIST_HEAD(&(defrag->queue[0].list));
+    }
 
-        gettimeofday (&end, NULL);
+    GF_FREE(defrag->queue);
 
-        elapsed = end.tv_sec - defrag->start_time.tv_sec;
+    ret = 0;
+out:
+    return ret;
+}
+
+int
+gf_defrag_start_crawl(void *data)
+{
+    xlator_t *this = NULL;
+    dht_conf_t *conf = NULL;
+    gf_defrag_info_t *defrag = NULL;
+    dict_t *fix_layout = NULL;
+    dict_t *migrate_data = NULL;
+    dict_t *status = NULL;
+    glusterfs_ctx_t *ctx = NULL;
+    call_frame_t *statfs_frame = NULL;
+    xlator_t *old_THIS = NULL;
+    int ret = -1;
+    loc_t loc = {
+        0,
+    };
+    struct iatt iatt = {
+        0,
+    };
+    struct iatt parent = {
+        0,
+    };
+    int thread_index = 0;
+    pthread_t *tid = NULL;
+    pthread_t filecnt_thread;
+    gf_boolean_t fc_thread_started = _gf_false;
+
+    this = data;
+    if (!this)
+        goto exit;
+
+    ctx = this->ctx;
+    if (!ctx)
+        goto exit;
+
+    conf = this->private;
+    if (!conf)
+        goto exit;
+
+    defrag = conf->defrag;
+    if (!defrag)
+        goto exit;
+
+    defrag->start_time = gf_time();
+
+    dht_build_root_inode(this, &defrag->root_inode);
+    if (!defrag->root_inode)
+        goto out;
+
+    dht_build_root_loc(defrag->root_inode, &loc);
+
+    /* fix-layout on '/' first */
+
+    ret = syncop_lookup(this, &loc, &iatt, &parent, NULL, NULL);
+
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, -ret, DHT_MSG_REBALANCE_START_FAILED,
+               "Failed to start rebalance: look up on / failed");
+        ret = -1;
+        goto out;
+    }
+
+    old_THIS = THIS;
+    THIS = this;
 
-        if (!dict)
-                goto log;
+    statfs_frame = create_frame(this, this->ctx->pool);
+    if (!statfs_frame) {
+        gf_msg(this->name, GF_LOG_ERROR, DHT_MSG_NO_MEMORY, ENOMEM,
+               "Insufficient memory. Frame creation failed");
+        ret = -1;
+        goto out;
+    }
 
-        ret = dict_set_uint64 (dict, "files", files);
-        if (ret)
-                gf_log (THIS->name, GF_LOG_WARNING,
-                        "failed to set file count");
+    /* async statfs update for honoring min-free-disk */
+    dht_get_du_info(statfs_frame, this, &loc);
+    THIS = old_THIS;
 
-        ret = dict_set_uint64 (dict, "size", size);
-        if (ret)
-                gf_log (THIS->name, GF_LOG_WARNING,
-                        "failed to set size of xfer");
+    fix_layout = dict_new();
+    if (!fix_layout) {
+        ret = -1;
+        goto out;
+    }
+
+    /*
+     * Unfortunately, we can't do special xattrs (like fix.layout) and
+     * real ones in the same call currently, and changing it seems
+     * riskier than just doing two calls.
+     */
+
+    gf_log(this->name, GF_LOG_INFO, "%s using commit hash %u", __func__,
+           conf->vol_commit_hash);
+
+    ret = dict_set_uint32(fix_layout, conf->commithash_xattr_name,
+                          conf->vol_commit_hash);
+    if (ret) {
+        gf_log(this->name, GF_LOG_ERROR, "Failed to set %s",
+               conf->commithash_xattr_name);
+        defrag->total_failures++;
+        ret = -1;
+        goto out;
+    }
+
+    ret = syncop_setxattr(this, &loc, fix_layout, 0, NULL, NULL);
+    if (ret) {
+        gf_log(this->name, GF_LOG_ERROR,
+               "Failed to set commit hash on %s. "
+               "Rebalance cannot proceed.",
+               loc.path);
+        defrag->total_failures++;
+        ret = -1;
+        goto out;
+    }
+
+    /* We now return to our regularly scheduled program. */
+
+    ret = dict_set_str(fix_layout, GF_XATTR_FIX_LAYOUT_KEY, "yes");
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, DHT_MSG_REBALANCE_START_FAILED,
+               "Failed to start rebalance:"
+               "Failed to set dictionary value: key = %s",
+               GF_XATTR_FIX_LAYOUT_KEY);
+        defrag->total_failures++;
+        ret = -1;
+        goto out;
+    }
+
+    defrag->new_commit_hash = conf->vol_commit_hash;
 
-        ret = dict_set_uint64 (dict, "lookups", lookup);
-        if (ret)
-                gf_log (THIS->name, GF_LOG_WARNING,
-                        "failed to set lookedup file count");
+    ret = syncop_setxattr(this, &loc, fix_layout, 0, NULL, NULL);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, -ret, DHT_MSG_REBALANCE_FAILED,
+               "fix layout on %s failed", loc.path);
+        defrag->total_failures++;
+        ret = -1;
+        goto out;
+    }
 
+    if (defrag->cmd != GF_DEFRAG_CMD_START_LAYOUT_FIX) {
+        /* We need to migrate files */
 
-        ret = dict_set_int32 (dict, "status", defrag->defrag_status);
-        if (ret)
-                gf_log (THIS->name, GF_LOG_WARNING,
-                        "failed to set status");
-        if (elapsed) {
-                ret = dict_set_double (dict, "run-time", elapsed);
-                if (ret)
-                        gf_log (THIS->name, GF_LOG_WARNING,
-                                "failed to set run-time");
+        migrate_data = dict_new();
+        if (!migrate_data) {
+            defrag->total_failures++;
+            ret = -1;
+            goto out;
         }
+        ret = dict_set_str(
+            migrate_data, GF_XATTR_FILE_MIGRATE_KEY,
+            (defrag->cmd == GF_DEFRAG_CMD_START_FORCE) ? "force" : "non-force");
+        if (ret) {
+            defrag->total_failures++;
+            ret = -1;
+            goto out;
+        }
+
+        ret = dht_init_local_subvols_and_nodeuuids(this, conf, &loc);
+        if (ret) {
+            ret = -1;
+            goto out;
+        }
+
+        /* Initialise the structures required for parallel migration */
+        ret = gf_defrag_parallel_migration_init(this, defrag, &tid,
+                                                &thread_index);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, 0, "Aborting rebalance.");
+            goto out;
+        }
+
+        ret = gf_defrag_estimates_init(this, &loc, &filecnt_thread);
+        if (ret) {
+            /* Not a fatal error. Allow the rebalance to proceed*/
+            ret = 0;
+        } else {
+            fc_thread_started = _gf_true;
+        }
+    }
+
+    ret = gf_defrag_fix_layout(this, defrag, &loc, fix_layout, migrate_data);
+    if (ret) {
+        defrag->total_failures++;
+        ret = -1;
+        goto out;
+    }
+
+    if (gf_defrag_settle_hash(this, defrag, &loc, fix_layout) != 0) {
+        defrag->total_failures++;
+        ret = -1;
+        goto out;
+    }
+
+    gf_log("DHT", GF_LOG_INFO, "crawling file-system completed");
+out:
+
+    /* We are here means crawling the entire file system is done
+       or something failed. Set defrag->crawl_done flag to intimate
+       the migrator threads to exhaust the defrag->queue and terminate*/
+
+    if (ret) {
+        defrag->defrag_status = GF_DEFRAG_STATUS_FAILED;
+    }
+
+    gf_defrag_parallel_migration_cleanup(defrag, tid, thread_index);
+
+    if ((defrag->defrag_status != GF_DEFRAG_STATUS_STOPPED) &&
+        (defrag->defrag_status != GF_DEFRAG_STATUS_FAILED)) {
+        defrag->defrag_status = GF_DEFRAG_STATUS_COMPLETE;
+    }
+
+    if (fc_thread_started) {
+        gf_defrag_estimates_cleanup(this, defrag, filecnt_thread);
+    }
+
+    dht_send_rebalance_event(this, defrag->cmd, defrag->defrag_status);
+
+    status = dict_new();
+    LOCK(&defrag->lock);
+    {
+        gf_defrag_status_get(conf, status);
+        if (ctx && ctx->notify)
+            ctx->notify(GF_EN_DEFRAG_STATUS, status);
+        if (status)
+            dict_unref(status);
+        defrag->is_exiting = 1;
+    }
+    UNLOCK(&defrag->lock);
+
+    GF_FREE(defrag);
+    conf->defrag = NULL;
+
+    if (migrate_data)
+        dict_unref(migrate_data);
+
+    if (statfs_frame) {
+        STACK_DESTROY(statfs_frame->root);
+    }
+exit:
+    return ret;
+}
+
+static int
+gf_defrag_done(int ret, call_frame_t *sync_frame, void *data)
+{
+    gf_listener_stop(sync_frame->this);
+
+    STACK_DESTROY(sync_frame->root);
+    kill(getpid(), SIGTERM);
+    return 0;
+}
+
+void *
+gf_defrag_start(void *data)
+{
+    int ret = -1;
+    call_frame_t *frame = NULL;
+    dht_conf_t *conf = NULL;
+    gf_defrag_info_t *defrag = NULL;
+    xlator_t *this = NULL;
+    xlator_t *old_THIS = NULL;
+
+    this = data;
+    conf = this->private;
+    if (!conf)
+        goto out;
+
+    defrag = conf->defrag;
+    if (!defrag)
+        goto out;
+
+    frame = create_frame(this, this->ctx->pool);
+    if (!frame)
+        goto out;
+
+    frame->root->pid = GF_CLIENT_PID_DEFRAG;
+
+    defrag->pid = frame->root->pid;
+
+    defrag->defrag_status = GF_DEFRAG_STATUS_STARTED;
+
+    old_THIS = THIS;
+    THIS = this;
+    ret = synctask_new(this->ctx->env, gf_defrag_start_crawl, gf_defrag_done,
+                       frame, this);
+
+    if (ret)
+        gf_msg(this->name, GF_LOG_ERROR, 0, DHT_MSG_REBALANCE_START_FAILED,
+               "Could not create task for rebalance");
+    THIS = old_THIS;
+out:
+    return NULL;
+}
+
+uint64_t
+gf_defrag_get_estimates_based_on_size(dht_conf_t *conf)
+{
+    gf_defrag_info_t *defrag = NULL;
+    double rate_processed = 0;
+    uint64_t total_processed = 0;
+    uint64_t tmp_count = 0;
+    uint64_t time_to_complete = 0;
+    double elapsed = 0;
 
-        ret = dict_set_uint64 (dict, "failures", failures);
-        if (ret)
-                gf_log (THIS->name, GF_LOG_WARNING,
-                        "failed to set failure count");
+    defrag = conf->defrag;
+
+    if (!g_totalsize)
+        goto out;
+
+    elapsed = gf_time() - defrag->start_time;
+
+    /* Don't calculate the estimates for the first 10 minutes.
+     * It is unlikely to be accurate and estimates are not required
+     * if the process finishes in less than 10 mins.
+     */
+
+    if (elapsed < ESTIMATE_START_INTERVAL) {
+        gf_msg(THIS->name, GF_LOG_INFO, 0, 0,
+               "Rebalance estimates will not be available for the "
+               "first %d seconds.",
+               ESTIMATE_START_INTERVAL);
+
+        goto out;
+    }
+
+    total_processed = defrag->size_processed;
+
+    /* rate at which files processed */
+    rate_processed = (total_processed) / elapsed;
+
+    tmp_count = g_totalsize;
+
+    if (rate_processed) {
+        time_to_complete = (tmp_count) / rate_processed;
+
+    } else {
+        gf_msg(THIS->name, GF_LOG_ERROR, 0, 0,
+               "Unable to calculate estimated time for rebalance");
+    }
+
+    gf_log(THIS->name, GF_LOG_INFO,
+           "TIME: (size) total_processed=%" PRIu64 " tmp_cnt = %" PRIu64
+           ","
+           "rate_processed=%f, elapsed = %f",
+           total_processed, tmp_count, rate_processed, elapsed);
+
+out:
+    return time_to_complete;
+}
+
+int
+gf_defrag_status_get(dht_conf_t *conf, dict_t *dict)
+{
+    int ret = 0;
+    uint64_t files = 0;
+    uint64_t size = 0;
+    uint64_t lookup = 0;
+    uint64_t failures = 0;
+    uint64_t skipped = 0;
+    char *status = "";
+    double elapsed = 0;
+    uint64_t time_to_complete = 0;
+    uint64_t time_left = 0;
+    gf_defrag_info_t *defrag = conf->defrag;
+
+    if (!defrag)
+        goto out;
+
+    ret = 0;
+    if (defrag->defrag_status == GF_DEFRAG_STATUS_NOT_STARTED)
+        goto out;
+
+    files = defrag->total_files;
+    size = defrag->total_data;
+    lookup = defrag->num_files_lookedup;
+    failures = defrag->total_failures;
+    skipped = defrag->skipped;
+
+    elapsed = gf_time() - defrag->start_time;
+
+    /* The rebalance is still in progress */
+
+    if (defrag->defrag_status == GF_DEFRAG_STATUS_STARTED) {
+        time_to_complete = gf_defrag_get_estimates_based_on_size(conf);
+
+        if (time_to_complete && (time_to_complete > elapsed))
+            time_left = time_to_complete - elapsed;
+
+        gf_log(THIS->name, GF_LOG_INFO,
+               "TIME: Estimated total time to complete (size)= %" PRIu64
+               " seconds, seconds left = %" PRIu64 "",
+               time_to_complete, time_left);
+    }
+
+    if (!dict)
+        goto log;
+
+    ret = dict_set_uint64(dict, "files", files);
+    if (ret)
+        gf_log(THIS->name, GF_LOG_WARNING, "failed to set file count");
+
+    ret = dict_set_uint64(dict, "size", size);
+    if (ret)
+        gf_log(THIS->name, GF_LOG_WARNING, "failed to set size of xfer");
+
+    ret = dict_set_uint64(dict, "lookups", lookup);
+    if (ret)
+        gf_log(THIS->name, GF_LOG_WARNING, "failed to set lookedup file count");
+
+    ret = dict_set_int32(dict, "status", defrag->defrag_status);
+    if (ret)
+        gf_log(THIS->name, GF_LOG_WARNING, "failed to set status");
+
+    ret = dict_set_double(dict, "run-time", elapsed);
+    if (ret)
+        gf_log(THIS->name, GF_LOG_WARNING, "failed to set run-time");
+
+    ret = dict_set_uint64(dict, "failures", failures);
+    if (ret)
+        gf_log(THIS->name, GF_LOG_WARNING, "failed to set failure count");
+
+    ret = dict_set_uint64(dict, "skipped", skipped);
+    if (ret)
+        gf_log(THIS->name, GF_LOG_WARNING, "failed to set skipped file count");
+
+    ret = dict_set_uint64(dict, "time-left", time_left);
+    if (ret)
+        gf_log(THIS->name, GF_LOG_WARNING, "failed to set time-left");
 
-        ret = dict_set_uint64 (dict, "skipped", skipped);
-        if (ret)
-                gf_log (THIS->name, GF_LOG_WARNING,
-                        "failed to set skipped file count");
 log:
-        switch (defrag->defrag_status) {
+    switch (defrag->defrag_status) {
         case GF_DEFRAG_STATUS_NOT_STARTED:
-                status = "not started";
-                break;
+            status = "not started";
+            break;
         case GF_DEFRAG_STATUS_STARTED:
-                status = "in progress";
-                break;
+            status = "in progress";
+            break;
         case GF_DEFRAG_STATUS_STOPPED:
-                status = "stopped";
-                break;
+            status = "stopped";
+            break;
         case GF_DEFRAG_STATUS_COMPLETE:
-                status = "completed";
-                break;
+            status = "completed";
+            break;
         case GF_DEFRAG_STATUS_FAILED:
-                status = "failed";
-                break;
+            status = "failed";
+            break;
         default:
-                break;
-        }
-
-        gf_msg (THIS->name, GF_LOG_INFO, 0, DHT_MSG_REBALANCE_STATUS,
-                "Rebalance is %s. Time taken is %.2f secs",
-                status, elapsed);
-        gf_msg (THIS->name, GF_LOG_INFO, 0, DHT_MSG_REBALANCE_STATUS,
-                "Files migrated: %"PRIu64", size: %"
-                PRIu64", lookups: %"PRIu64", failures: %"PRIu64", skipped: "
-                "%"PRIu64, files, size, lookup, failures, skipped);
-
-
+            break;
+    }
+
+    gf_msg(THIS->name, GF_LOG_INFO, 0, DHT_MSG_REBALANCE_STATUS,
+           "Rebalance is %s. Time taken is %.2f secs", status, elapsed);
+    gf_msg(THIS->name, GF_LOG_INFO, 0, DHT_MSG_REBALANCE_STATUS,
+           "Files migrated: %" PRIu64 ", size: %" PRIu64 ", lookups: %" PRIu64
+           ", failures: %" PRIu64
+           ", skipped: "
+           "%" PRIu64,
+           files, size, lookup, failures, skipped);
 out:
-        return 0;
+    return 0;
 }
 
 int
-gf_defrag_stop (gf_defrag_info_t *defrag, gf_defrag_status_t status,
-                dict_t *output)
+gf_defrag_stop(dht_conf_t *conf, gf_defrag_status_t status, dict_t *output)
 {
-        /* TODO: set a variable 'stop_defrag' here, it should be checked
-           in defrag loop */
-        int     ret = -1;
-        GF_ASSERT (defrag);
+    /* TODO: set a variable 'stop_defrag' here, it should be checked
+       in defrag loop */
+    int ret = -1;
+    gf_defrag_info_t *defrag = conf->defrag;
 
-        if (defrag->defrag_status == GF_DEFRAG_STATUS_NOT_STARTED) {
-                goto out;
-        }
+    GF_ASSERT(defrag);
 
-        gf_msg ("", GF_LOG_INFO, 0, DHT_MSG_REBALANCE_STOPPED,
-                "Received stop command on rebalance");
-        defrag->defrag_status = status;
+    if (defrag->defrag_status == GF_DEFRAG_STATUS_NOT_STARTED) {
+        goto out;
+    }
 
-        if (output)
-                gf_defrag_status_get (defrag, output);
-        ret = 0;
+    gf_msg("", GF_LOG_INFO, 0, DHT_MSG_REBALANCE_STOPPED,
+           "Received stop command on rebalance");
+    defrag->defrag_status = status;
+
+    if (output)
+        gf_defrag_status_get(conf, output);
+    ret = 0;
 out:
-        gf_msg_debug ("", 0, "Returning %d", ret);
-        return ret;
+    gf_msg_debug("", 0, "Returning %d", ret);
+    return ret;
 }
diff --git a/xlators/cluster/dht/src/dht-rename.c b/xlators/cluster/dht/src/dht-rename.c
index 0594945203e..d9dbf50492f 100644
--- a/xlators/cluster/dht/src/dht-rename.c
+++ b/xlators/cluster/dht/src/dht-rename.c
@@ -11,1349 +11,1987 @@
 /* TODO: link(oldpath, newpath) fails if newpath already exists. DHT should
  *       delete the newpath if it gets EEXISTS from link() call.
  */
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
-
-#include "glusterfs.h"
-#include "xlator.h"
 #include "dht-common.h"
-#include "defaults.h"
+#include "dht-lock.h"
+#include <glusterfs/defaults.h>
 
+int
+dht_rename_unlock(call_frame_t *frame, xlator_t *this);
+int32_t
+dht_rename_lock_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                    int32_t op_ret, int32_t op_errno, dict_t *xdata);
 
 int
-dht_rename_dir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                    int32_t op_ret, int32_t op_errno, struct iatt *stbuf,
-                    struct iatt *preoldparent, struct iatt *postoldparent,
-                    struct iatt *prenewparent, struct iatt *postnewparent,
-                    dict_t *xdata)
+dht_rename_unlock_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                      int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
-        dht_local_t  *local = NULL;
-        int           this_call_cnt = 0;
-        call_frame_t *prev = NULL;
-        char          gfid[GF_UUID_BUF_SIZE] = {0};
+    dht_local_t *local = NULL;
 
-        local = frame->local;
-        prev = cookie;
+    local = frame->local;
 
+    dht_set_fixed_dir_stat(&local->preoldparent);
+    dht_set_fixed_dir_stat(&local->postoldparent);
+    dht_set_fixed_dir_stat(&local->preparent);
+    dht_set_fixed_dir_stat(&local->postparent);
 
-        if (op_ret == -1) {
-                /* TODO: undo the damage */
-                gf_uuid_unparse(local->loc.inode->gfid, gfid);
+    if (IA_ISREG(local->stbuf.ia_type))
+        DHT_STRIP_PHASE1_FLAGS(&local->stbuf);
 
-                gf_msg (this->name, GF_LOG_INFO, op_errno,
-                        DHT_MSG_RENAME_FAILED,
-                        "Rename %s -> %s on %s failed, (gfid = %s)",
-                        local->loc.path, local->loc2.path,
-                        prev->this->name, gfid);
+    DHT_STACK_UNWIND(rename, frame, local->op_ret, local->op_errno,
+                     &local->stbuf, &local->preoldparent, &local->postoldparent,
+                     &local->preparent, &local->postparent, local->xattr);
+    return 0;
+}
 
-                local->op_ret   = op_ret;
-                local->op_errno = op_errno;
-                goto unwind;
-        }
-        /* TODO: construct proper stbuf for dir */
-        /*
-         * FIXME: is this the correct way to build stbuf and
-         * parent bufs?
-         */
-        dht_iatt_merge (this, &local->stbuf, stbuf, prev->this);
-        dht_iatt_merge (this, &local->preoldparent, preoldparent,
-                        prev->this);
-        dht_iatt_merge (this, &local->postoldparent, postoldparent,
-                        prev->this);
-        dht_iatt_merge (this, &local->preparent, prenewparent,
-                        prev->this);
-        dht_iatt_merge (this, &local->postparent, postnewparent,
-                        prev->this);
+static void
+dht_rename_dir_unlock_src(call_frame_t *frame, xlator_t *this)
+{
+    dht_local_t *local = NULL;
 
+    local = frame->local;
+    dht_unlock_namespace(frame, &local->lock[0]);
+    return;
+}
 
-unwind:
-        this_call_cnt = dht_frame_return (frame);
-        if (is_last_call (this_call_cnt)) {
-                WIPE (&local->preoldparent);
-                WIPE (&local->postoldparent);
-                WIPE (&local->preparent);
-                WIPE (&local->postparent);
-
-                DHT_STRIP_PHASE1_FLAGS (&local->stbuf);
-                DHT_STACK_UNWIND (rename, frame, local->op_ret, local->op_errno,
-                                  &local->stbuf, &local->preoldparent,
-                                  &local->postoldparent,
-                                  &local->preparent, &local->postparent, xdata);
-        }
+static void
+dht_rename_dir_unlock_dst(call_frame_t *frame, xlator_t *this)
+{
+    dht_local_t *local = NULL;
+    int op_ret = -1;
+    char src_gfid[GF_UUID_BUF_SIZE] = {0};
+    char dst_gfid[GF_UUID_BUF_SIZE] = {0};
+
+    local = frame->local;
+
+    /* Unlock entrylk */
+    dht_unlock_entrylk_wrapper(frame, &local->lock[1].ns.directory_ns);
+
+    /* Unlock inodelk */
+    op_ret = dht_unlock_inodelk(frame, local->lock[1].ns.parent_layout.locks,
+                                local->lock[1].ns.parent_layout.lk_count,
+                                dht_rename_unlock_cbk);
+    if (op_ret < 0) {
+        uuid_utoa_r(local->loc.inode->gfid, src_gfid);
+
+        if (local->loc2.inode)
+            uuid_utoa_r(local->loc2.inode->gfid, dst_gfid);
+
+        if (IA_ISREG(local->stbuf.ia_type))
+            gf_msg(this->name, GF_LOG_WARNING, 0, DHT_MSG_UNLOCKING_FAILED,
+                   "winding unlock inodelk failed "
+                   "rename (%s:%s:%s %s:%s:%s), "
+                   "stale locks left on bricks",
+                   local->loc.path, src_gfid, local->src_cached->name,
+                   local->loc2.path, dst_gfid,
+                   local->dst_cached ? local->dst_cached->name : NULL);
+        else
+            gf_msg(this->name, GF_LOG_WARNING, 0, DHT_MSG_UNLOCKING_FAILED,
+                   "winding unlock inodelk failed "
+                   "rename (%s:%s %s:%s), "
+                   "stale locks left on bricks",
+                   local->loc.path, src_gfid, local->loc2.path, dst_gfid);
 
-        return 0;
-}
+        dht_rename_unlock_cbk(frame, NULL, this, 0, 0, NULL);
+    }
 
+    return;
+}
 
+static int
+dht_rename_dir_unlock(call_frame_t *frame, xlator_t *this)
+{
+    dht_rename_dir_unlock_src(frame, this);
+    dht_rename_dir_unlock_dst(frame, this);
+    return 0;
+}
 int
-dht_rename_hashed_dir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                           int32_t op_ret, int32_t op_errno, struct iatt *stbuf,
-                           struct iatt *preoldparent,
-                           struct iatt *postoldparent,
-                           struct iatt *prenewparent,
-                           struct iatt *postnewparent, dict_t *xdata)
+dht_rename_dir_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                   int32_t op_ret, int32_t op_errno, struct iatt *stbuf,
+                   struct iatt *preoldparent, struct iatt *postoldparent,
+                   struct iatt *prenewparent, struct iatt *postnewparent,
+                   dict_t *xdata)
 {
-        dht_conf_t   *conf = NULL;
-        dht_local_t  *local = NULL;
-        int           call_cnt = 0;
-        call_frame_t *prev = NULL;
-        int           i = 0;
-        char          gfid[GF_UUID_BUF_SIZE] = {0};
+    dht_conf_t *conf = NULL;
+    dht_local_t *local = NULL;
+    int this_call_cnt = 0;
+    xlator_t *prev = NULL;
+    int i = 0;
+    char gfid[GF_UUID_BUF_SIZE] = {0};
+    int subvol_cnt = -1;
+
+    conf = this->private;
+    local = frame->local;
+    prev = cookie;
+    subvol_cnt = dht_subvol_cnt(this, prev);
+    local->ret_cache[subvol_cnt] = op_ret;
+
+    if (op_ret == -1) {
+        gf_uuid_unparse(local->loc.inode->gfid, gfid);
 
-        conf = this->private;
-        local = frame->local;
-        prev = cookie;
+        gf_msg(this->name, GF_LOG_INFO, op_errno, DHT_MSG_RENAME_FAILED,
+               "Rename %s -> %s on %s failed, (gfid = %s)", local->loc.path,
+               local->loc2.path, prev->name, gfid);
+
+        local->op_ret = op_ret;
+        local->op_errno = op_errno;
+        goto unwind;
+    }
+    /* TODO: construct proper stbuf for dir */
+    /*
+     * FIXME: is this the correct way to build stbuf and
+     * parent bufs?
+     */
+    dht_iatt_merge(this, &local->stbuf, stbuf);
+    dht_iatt_merge(this, &local->preoldparent, preoldparent);
+    dht_iatt_merge(this, &local->postoldparent, postoldparent);
+    dht_iatt_merge(this, &local->preparent, prenewparent);
+    dht_iatt_merge(this, &local->postparent, postnewparent);
 
+unwind:
+    this_call_cnt = dht_frame_return(frame);
+    if (is_last_call(this_call_cnt)) {
+        /* We get here with local->call_cnt == 0. Which means
+         * we are the only one executing this code, there is
+         * no contention. Therefore it's safe to manipulate or
+         * deref local->call_cnt directly (without locking).
+         */
+        if (local->ret_cache[conf->subvolume_cnt] == 0) {
+            /* count errant subvols in last field of ret_cache */
+            for (i = 0; i < conf->subvolume_cnt; i++) {
+                if (local->ret_cache[i] != 0)
+                    ++local->ret_cache[conf->subvolume_cnt];
+            }
+            if (local->ret_cache[conf->subvolume_cnt]) {
+                /* undoing the damage:
+                 * for all subvolumes, where rename
+                 * succeeded, we perform the reverse operation
+                 */
+                for (i = 0; i < conf->subvolume_cnt; i++) {
+                    if (local->ret_cache[i] == 0)
+                        ++local->call_cnt;
+                }
+                for (i = 0; i < conf->subvolume_cnt; i++) {
+                    if (local->ret_cache[i])
+                        continue;
 
-        if (op_ret == -1) {
-                /* TODO: undo the damage */
+                    STACK_WIND(frame, dht_rename_dir_cbk, conf->subvolumes[i],
+                               conf->subvolumes[i]->fops->rename, &local->loc2,
+                               &local->loc, NULL);
+                }
 
-                gf_uuid_unparse(local->loc.inode->gfid, gfid);
+                return 0;
+            }
+        }
 
-                gf_msg (this->name, GF_LOG_INFO, op_errno,
-                        DHT_MSG_RENAME_FAILED,
-                        "rename %s -> %s on %s failed, (gfid = %s) ",
-                        local->loc.path, local->loc2.path,
-                        prev->this->name, gfid );
+        WIPE(&local->preoldparent);
+        WIPE(&local->postoldparent);
+        WIPE(&local->preparent);
+        WIPE(&local->postparent);
 
-                local->op_ret   = op_ret;
-                local->op_errno = op_errno;
-                goto unwind;
-        }
-        /* TODO: construct proper stbuf for dir */
-        /*
-         * FIXME: is this the correct way to build stbuf and
-         * parent bufs?
-         */
-        dht_iatt_merge (this, &local->stbuf, stbuf, prev->this);
-        dht_iatt_merge (this, &local->preoldparent, preoldparent,
-                        prev->this);
-        dht_iatt_merge (this, &local->postoldparent, postoldparent,
-                        prev->this);
-        dht_iatt_merge (this, &local->preparent, prenewparent,
-                        prev->this);
-        dht_iatt_merge (this, &local->postparent, postnewparent,
-                        prev->this);
-
-        call_cnt = local->call_cnt = conf->subvolume_cnt - 1;
-
-        if (!local->call_cnt)
-                goto unwind;
-
-        for (i = 0; i < conf->subvolume_cnt; i++) {
-                if (conf->subvolumes[i] == local->dst_hashed)
-                        continue;
-                STACK_WIND (frame, dht_rename_dir_cbk,
-                            conf->subvolumes[i],
-                            conf->subvolumes[i]->fops->rename,
-                            &local->loc, &local->loc2, NULL);
-                if (!--call_cnt)
-                        break;
-        }
+        dht_rename_dir_unlock(frame, this);
+    }
 
+    return 0;
+}
 
-        return 0;
-unwind:
-        WIPE (&local->preoldparent);
-        WIPE (&local->postoldparent);
-        WIPE (&local->preparent);
-        WIPE (&local->postparent);
+int
+dht_rename_hashed_dir_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                          int32_t op_ret, int32_t op_errno, struct iatt *stbuf,
+                          struct iatt *preoldparent, struct iatt *postoldparent,
+                          struct iatt *prenewparent, struct iatt *postnewparent,
+                          dict_t *xdata)
+{
+    dht_conf_t *conf = NULL;
+    dht_local_t *local = NULL;
+    int call_cnt = 0;
+    xlator_t *prev = NULL;
+    int i = 0;
+    char gfid[GF_UUID_BUF_SIZE] = {0};
+
+    conf = this->private;
+    local = frame->local;
+    prev = cookie;
+
+    if (op_ret == -1) {
+        gf_uuid_unparse(local->loc.inode->gfid, gfid);
 
-        DHT_STRIP_PHASE1_FLAGS (&local->stbuf);
-        DHT_STACK_UNWIND (rename, frame, local->op_ret, local->op_errno,
-                          &local->stbuf, &local->preoldparent,
-                          &local->postoldparent,
-                          &local->preparent, &local->postparent, NULL);
+        gf_msg(this->name, GF_LOG_INFO, op_errno, DHT_MSG_RENAME_FAILED,
+               "rename %s -> %s on %s failed, (gfid = %s) ", local->loc.path,
+               local->loc2.path, prev->name, gfid);
+
+        local->op_ret = op_ret;
+        local->op_errno = op_errno;
+        goto unwind;
+    }
+    /* TODO: construct proper stbuf for dir */
+    /*
+     * FIXME: is this the correct way to build stbuf and
+     * parent bufs?
+     */
+    dht_iatt_merge(this, &local->stbuf, stbuf);
+    dht_iatt_merge(this, &local->preoldparent, preoldparent);
+    dht_iatt_merge(this, &local->postoldparent, postoldparent);
+    dht_iatt_merge(this, &local->preparent, prenewparent);
+    dht_iatt_merge(this, &local->postparent, postnewparent);
+
+    call_cnt = local->call_cnt = conf->subvolume_cnt - 1;
+
+    if (!local->call_cnt)
+        goto unwind;
+
+    for (i = 0; i < conf->subvolume_cnt; i++) {
+        if (conf->subvolumes[i] == local->dst_hashed)
+            continue;
+        STACK_WIND_COOKIE(
+            frame, dht_rename_dir_cbk, conf->subvolumes[i], conf->subvolumes[i],
+            conf->subvolumes[i]->fops->rename, &local->loc, &local->loc2, NULL);
+        if (!--call_cnt)
+            break;
+    }
+
+    return 0;
+unwind:
+    WIPE(&local->preoldparent);
+    WIPE(&local->postoldparent);
+    WIPE(&local->preparent);
+    WIPE(&local->postparent);
 
-        return 0;
+    dht_rename_dir_unlock(frame, this);
+    return 0;
 }
 
-
 int
-dht_rename_dir_do (call_frame_t *frame, xlator_t *this)
+dht_rename_dir_do(call_frame_t *frame, xlator_t *this)
 {
-        dht_local_t  *local = NULL;
+    dht_local_t *local = NULL;
 
-        local = frame->local;
+    local = frame->local;
 
-        if (local->op_ret == -1)
-                goto err;
+    if (local->op_ret == -1)
+        goto err;
 
-        local->op_ret = 0;
+    local->op_ret = 0;
 
-        STACK_WIND (frame, dht_rename_hashed_dir_cbk,
-                    local->dst_hashed,
-                    local->dst_hashed->fops->rename,
-                    &local->loc, &local->loc2, NULL);
-        return 0;
+    STACK_WIND_COOKIE(frame, dht_rename_hashed_dir_cbk, local->dst_hashed,
+                      local->dst_hashed, local->dst_hashed->fops->rename,
+                      &local->loc, &local->loc2, NULL);
+    return 0;
 
 err:
-        DHT_STACK_UNWIND (rename, frame, local->op_ret, local->op_errno, NULL, NULL,
-                          NULL, NULL, NULL, NULL);
-        return 0;
+    dht_rename_dir_unlock(frame, this);
+    return 0;
 }
 
-
 int
-dht_rename_readdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                        int op_ret, int op_errno, gf_dirent_t *entries,
-                        dict_t *xdata)
+dht_rename_readdir_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                       int op_ret, int op_errno, gf_dirent_t *entries,
+                       dict_t *xdata)
 {
-        dht_local_t  *local = NULL;
-        int           this_call_cnt = -1;
-        call_frame_t *prev = NULL;
-
-        local = frame->local;
-        prev  = cookie;
-
-        if (op_ret > 2) {
-                gf_msg_trace (this->name, 0,
-                              "readdir on %s for %s returned %d entries",
-                              prev->this->name, local->loc.path, op_ret);
-                local->op_ret = -1;
-                local->op_errno = ENOTEMPTY;
-        }
+    dht_local_t *local = NULL;
+    int this_call_cnt = -1;
+    xlator_t *prev = NULL;
 
-        this_call_cnt = dht_frame_return (frame);
+    local = frame->local;
+    prev = cookie;
 
-        if (is_last_call (this_call_cnt)) {
-                dht_rename_dir_do (frame, this);
-        }
+    if (op_ret > 2) {
+        gf_msg_trace(this->name, 0, "readdir on %s for %s returned %d entries",
+                     prev->name, local->loc.path, op_ret);
+        local->op_ret = -1;
+        local->op_errno = ENOTEMPTY;
+    }
 
-        return 0;
-}
+    this_call_cnt = dht_frame_return(frame);
+
+    if (is_last_call(this_call_cnt)) {
+        dht_rename_dir_do(frame, this);
+    }
 
+    return 0;
+}
 
 int
-dht_rename_opendir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                        int op_ret, int op_errno, fd_t *fd, dict_t *xdata)
+dht_rename_opendir_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                       int op_ret, int op_errno, fd_t *fd, dict_t *xdata)
 {
-        dht_local_t  *local = NULL;
-        int           this_call_cnt = -1;
-        call_frame_t *prev = NULL;
-        char          gfid[GF_UUID_BUF_SIZE] = {0};
+    dht_local_t *local = NULL;
+    int this_call_cnt = -1;
+    xlator_t *prev = NULL;
+    char gfid[GF_UUID_BUF_SIZE] = {0};
 
-        local = frame->local;
-        prev  = cookie;
+    local = frame->local;
+    prev = cookie;
 
+    if (op_ret == -1) {
+        gf_uuid_unparse(local->loc.inode->gfid, gfid);
+        gf_msg(this->name, GF_LOG_INFO, op_errno, DHT_MSG_OPENDIR_FAILED,
+               "opendir on %s for %s failed,(gfid = %s) ", prev->name,
+               local->loc.path, gfid);
+        goto err;
+    }
 
-        if (op_ret == -1) {
+    fd_bind(fd);
+    STACK_WIND_COOKIE(frame, dht_rename_readdir_cbk, prev, prev,
+                      prev->fops->readdir, local->fd, 4096, 0, NULL);
 
-                gf_uuid_unparse(local->loc.inode->gfid, gfid);
-                gf_msg (this->name, GF_LOG_INFO, op_errno,
-                        DHT_MSG_OPENDIR_FAILED,
-                        "opendir on %s for %s failed,(gfid = %s) ",
-                        prev->this->name, local->loc.path, gfid);
-                goto err;
-        }
+    return 0;
 
-        STACK_WIND (frame, dht_rename_readdir_cbk,
-                    prev->this, prev->this->fops->readdir,
-                    local->fd, 4096, 0, NULL);
+err:
+    this_call_cnt = dht_frame_return(frame);
 
+    if (is_last_call(this_call_cnt)) {
+        dht_rename_dir_do(frame, this);
+    }
+
+    return 0;
+}
+
+int
+dht_rename_dir_lock2_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                         int32_t op_ret, int32_t op_errno, dict_t *xdata)
+{
+    dht_local_t *local = NULL;
+    char src_gfid[GF_UUID_BUF_SIZE] = {0};
+    char dst_gfid[GF_UUID_BUF_SIZE] = {0};
+    dht_conf_t *conf = NULL;
+    int i = 0;
+
+    local = frame->local;
+    conf = this->private;
+
+    if (op_ret < 0) {
+        uuid_utoa_r(local->loc.inode->gfid, src_gfid);
+
+        if (local->loc2.inode)
+            uuid_utoa_r(local->loc2.inode->gfid, dst_gfid);
+
+        gf_msg(this->name, GF_LOG_WARNING, op_errno, DHT_MSG_INODE_LK_ERROR,
+               "acquiring entrylk after inodelk failed"
+               "rename (%s:%s:%s %s:%s:%s)",
+               local->loc.path, src_gfid, local->src_cached->name,
+               local->loc2.path, dst_gfid,
+               local->dst_cached ? local->dst_cached->name : NULL);
+
+        local->op_ret = -1;
+        local->op_errno = op_errno;
+        goto err;
+    }
+
+    local->fd = fd_create(local->loc.inode, frame->root->pid);
+    if (!local->fd) {
+        op_errno = ENOMEM;
+        goto err;
+    }
+
+    local->op_ret = 0;
+
+    if (!local->dst_cached) {
+        dht_rename_dir_do(frame, this);
         return 0;
+    }
 
-err:
-        this_call_cnt = dht_frame_return (frame);
+    for (i = 0; i < conf->subvolume_cnt; i++) {
+        STACK_WIND_COOKIE(frame, dht_rename_opendir_cbk, conf->subvolumes[i],
+                          conf->subvolumes[i],
+                          conf->subvolumes[i]->fops->opendir, &local->loc2,
+                          local->fd, NULL);
+    }
 
-        if (is_last_call (this_call_cnt)) {
-                dht_rename_dir_do (frame, this);
-        }
+    return 0;
 
-        return 0;
+err:
+    /* No harm in calling an extra unlock */
+    dht_rename_dir_unlock(frame, this);
+    return 0;
 }
 
-
 int
-dht_rename_dir (call_frame_t *frame, xlator_t *this)
+dht_rename_dir_lock1_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                         int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
-        dht_conf_t  *conf = NULL;
-        dht_local_t *local = NULL;
-        int          i = 0;
-        int          op_errno = -1;
+    dht_local_t *local = NULL;
+    char src_gfid[GF_UUID_BUF_SIZE] = {0};
+    char dst_gfid[GF_UUID_BUF_SIZE] = {0};
+    int ret = 0;
+    loc_t *loc = NULL;
+    xlator_t *subvol = NULL;
+
+    local = frame->local;
+
+    if (op_ret < 0) {
+        uuid_utoa_r(local->loc.inode->gfid, src_gfid);
+
+        if (local->loc2.inode)
+            uuid_utoa_r(local->loc2.inode->gfid, dst_gfid);
+
+        gf_msg(this->name, GF_LOG_WARNING, op_errno, DHT_MSG_INODE_LK_ERROR,
+               "acquiring entrylk after inodelk failed"
+               "rename (%s:%s:%s %s:%s:%s)",
+               local->loc.path, src_gfid, local->src_cached->name,
+               local->loc2.path, dst_gfid,
+               local->dst_cached ? local->dst_cached->name : NULL);
+
+        local->op_ret = -1;
+        local->op_errno = op_errno;
+        goto err;
+    }
+
+    if (local->current == &local->lock[0]) {
+        loc = &local->loc2;
+        subvol = local->dst_hashed;
+        local->current = &local->lock[1];
+    } else {
+        loc = &local->loc;
+        subvol = local->src_hashed;
+        local->current = &local->lock[0];
+    }
+    ret = dht_protect_namespace(frame, loc, subvol, &local->current->ns,
+                                dht_rename_dir_lock2_cbk);
+    if (ret < 0) {
+        op_errno = EINVAL;
+        goto err;
+    }
+
+    return 0;
+err:
+    /* No harm in calling an extra unlock */
+    dht_rename_dir_unlock(frame, this);
+    return 0;
+}
 
+/*
+ * If the hashed subvolumes of both source and dst are the different,
+ * lock in dictionary order of hashed subvol->name. This is important
+ * in case the parent directory is the same for both src and dst to
+ * prevent inodelk deadlocks when racing with a fix-layout op on the parent.
+ *
+ * If the hashed subvols are the same, use the gfid/name to determine
+ * the order of taking locks to prevent entrylk deadlocks when the parent
+ * dirs are the same.
+ *
+ */
+static int
+dht_order_rename_lock(call_frame_t *frame, loc_t **loc, xlator_t **subvol)
+{
+    int ret = 0;
+    int op_ret = 0;
+    dht_local_t *local = NULL;
+    char *src = NULL;
+    char *dst = NULL;
+
+    local = frame->local;
+
+    if (local->src_hashed->name == local->dst_hashed->name) {
+        ret = 0;
+    } else {
+        ret = strcmp(local->src_hashed->name, local->dst_hashed->name);
+    }
+
+    if (ret == 0) {
+        /* hashed subvols are the same for src and dst */
+        /* Entrylks need to be ordered*/
+
+        src = alloca(GF_UUID_BNAME_BUF_SIZE + strlen(local->loc.name) + 1);
+        if (!src) {
+            gf_msg(frame->this->name, GF_LOG_ERROR, ENOMEM, 0,
+                   "Insufficient memory for src");
+            op_ret = -1;
+            goto out;
+        }
 
-        conf = frame->this->private;
-        local = frame->local;
+        if (!gf_uuid_is_null(local->loc.pargfid))
+            uuid_utoa_r(local->loc.pargfid, src);
+        else if (local->loc.parent)
+            uuid_utoa_r(local->loc.parent->gfid, src);
+        else
+            src[0] = '\0';
 
-        local->call_cnt = conf->subvolume_cnt;
+        strcat(src, local->loc.name);
 
-        for (i = 0; i < conf->subvolume_cnt; i++) {
-                if (!conf->subvolume_status[i]) {
-                        gf_msg (this->name, GF_LOG_INFO, 0,
-                                DHT_MSG_RENAME_FAILED,
-                                "Rename dir failed: subvolume down (%s)",
-                                conf->subvolumes[i]->name);
-                        op_errno = ENOTCONN;
-                        goto err;
-                }
+        dst = alloca(GF_UUID_BNAME_BUF_SIZE + strlen(local->loc2.name) + 1);
+        if (!dst) {
+            gf_msg(frame->this->name, GF_LOG_ERROR, ENOMEM, 0,
+                   "Insufficient memory for dst");
+            op_ret = -1;
+            goto out;
         }
 
-        local->fd = fd_create (local->loc.inode, frame->root->pid);
-        if (!local->fd) {
-                op_errno = ENOMEM;
-                goto err;
-        }
+        if (!gf_uuid_is_null(local->loc2.pargfid))
+            uuid_utoa_r(local->loc2.pargfid, dst);
+        else if (local->loc2.parent)
+            uuid_utoa_r(local->loc2.parent->gfid, dst);
+        else
+            dst[0] = '\0';
 
-        local->op_ret = 0;
+        strcat(dst, local->loc2.name);
+        ret = strcmp(src, dst);
+    }
 
-        if (!local->dst_cached) {
-                dht_rename_dir_do (frame, this);
-                return 0;
-        }
+    if (ret <= 0) {
+        /*inodelk in dictionary order of hashed subvol names*/
+        /*entrylk in dictionary order of gfid/basename */
+        local->current = &local->lock[0];
+        *loc = &local->loc;
+        *subvol = local->src_hashed;
 
-        for (i = 0; i < conf->subvolume_cnt; i++) {
-                STACK_WIND (frame, dht_rename_opendir_cbk,
-                            conf->subvolumes[i],
-                            conf->subvolumes[i]->fops->opendir,
-                            &local->loc2, local->fd, NULL);
-        }
+    } else {
+        local->current = &local->lock[1];
+        *loc = &local->loc2;
+        *subvol = local->dst_hashed;
+    }
 
-        return 0;
+    op_ret = 0;
 
-err:
-        op_errno = (op_errno == -1) ? errno : op_errno;
-        DHT_STACK_UNWIND (rename, frame, -1, op_errno, NULL, NULL, NULL, NULL,
-                          NULL, NULL);
-        return 0;
+out:
+    return op_ret;
 }
-#define DHT_MARK_FOP_INTERNAL(xattr) do {                                      \
-                int tmp = -1;                                                  \
-                if (!xattr) {                                                  \
-                        xattr = dict_new ();                                   \
-                        if (!xattr)                                            \
-                                break;                                         \
-                }                                                              \
-                tmp = dict_set_str (xattr, GLUSTERFS_INTERNAL_FOP_KEY, "yes"); \
-                if (tmp) {                                                     \
-                        gf_msg (this->name, GF_LOG_ERROR, 0,                   \
-                                DHT_MSG_DICT_SET_FAILED,                       \
-                                "Failed to set dictionary value: key = %s,"    \
-                                " path = %s", GLUSTERFS_INTERNAL_FOP_KEY,      \
-                                 local->loc.path);                             \
-                }                                                              \
-        }while (0)
-
-#define DHT_MARKER_DONT_ACCOUNT(xattr) do {                             \
-                int tmp = -1;                                                  \
-                if (!xattr) {                                                  \
-                        xattr = dict_new ();                                   \
-                        if (!xattr)                                            \
-                                break;                                         \
-                }                                                              \
-                tmp = dict_set_str (xattr, GLUSTERFS_MARKER_DONT_ACCOUNT_KEY,  \
-                                    "yes");                                    \
-                if (tmp) {                                                     \
-                        gf_msg (this->name, GF_LOG_ERROR, 0,                    \
-                                DHT_MSG_DICT_SET_FAILED,                       \
-                                "Failed to set dictionary value: key = %s,"    \
-                                " path = %s",GLUSTERFS_MARKER_DONT_ACCOUNT_KEY, \
-                                local->loc.path);                             \
-                }                                                              \
-        }while (0)
 
 int
-dht_rename_unlock_cbk (call_frame_t *frame, void *cookie,
-                       xlator_t *this, int32_t op_ret, int32_t op_errno,
-                       dict_t *xdata)
+dht_rename_dir(call_frame_t *frame, xlator_t *this)
 {
-        dht_local_t *local = NULL;
+    dht_conf_t *conf = NULL;
+    dht_local_t *local = NULL;
+    loc_t *loc = NULL;
+    xlator_t *subvol = NULL;
+    int i = 0;
+    int ret = 0;
+    int op_errno = -1;
+
+    conf = frame->this->private;
+    local = frame->local;
+
+    local->ret_cache = GF_CALLOC(conf->subvolume_cnt + 1, sizeof(int),
+                                 gf_dht_ret_cache_t);
+
+    if (local->ret_cache == NULL) {
+        op_errno = ENOMEM;
+        goto err;
+    }
+
+    local->call_cnt = conf->subvolume_cnt;
+
+    for (i = 0; i < conf->subvolume_cnt; i++) {
+        if (!conf->subvolume_status[i]) {
+            gf_msg(this->name, GF_LOG_INFO, 0, DHT_MSG_RENAME_FAILED,
+                   "Rename dir failed: subvolume down (%s)",
+                   conf->subvolumes[i]->name);
+            op_errno = ENOTCONN;
+            goto err;
+        }
+    }
+
+    /* Locks on src and dst needs to ordered which otherwise might cause
+     * deadlocks when rename (src, dst) and rename (dst, src) is done from
+     * two different clients
+     */
+    ret = dht_order_rename_lock(frame, &loc, &subvol);
+    if (ret) {
+        op_errno = ENOMEM;
+        goto err;
+    }
+
+    /* Rename must take locks on src to avoid lookup selfheal from
+     * recreating src on those subvols where the rename was successful.
+     * The locks can't be issued parallel as two different clients might
+     * attempt same rename command and be in dead lock.
+     */
+    ret = dht_protect_namespace(frame, loc, subvol, &local->current->ns,
+                                dht_rename_dir_lock1_cbk);
+    if (ret < 0) {
+        op_errno = EINVAL;
+        goto err;
+    }
+
+    return 0;
 
-        local = frame->local;
+err:
+    DHT_STACK_UNWIND(rename, frame, -1, op_errno, NULL, NULL, NULL, NULL, NULL,
+                     NULL);
+    return 0;
+}
 
-        DHT_STRIP_PHASE1_FLAGS (&local->stbuf);
-        DHT_STACK_UNWIND (rename, frame, local->op_ret, local->op_errno,
-                          &local->stbuf, &local->preoldparent,
-                          &local->postoldparent, &local->preparent,
-                          &local->postparent, NULL);
-        return 0;
+static int
+dht_rename_track_for_changelog(xlator_t *this, dict_t *xattr, loc_t *oldloc,
+                               loc_t *newloc)
+{
+    int ret = -1;
+    dht_changelog_rename_info_t *info = NULL;
+    char *name = NULL;
+    int len1 = 0;
+    int len2 = 0;
+    int size = 0;
+
+    if (!xattr || !oldloc || !newloc || !this)
+        return ret;
+
+    len1 = strlen(oldloc->name) + 1;
+    len2 = strlen(newloc->name) + 1;
+    size = sizeof(dht_changelog_rename_info_t) + len1 + len2;
+
+    info = GF_CALLOC(size, sizeof(char), gf_common_mt_char);
+    if (!info) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, DHT_MSG_DICT_SET_FAILED,
+               "Failed to calloc memory");
+        return ret;
+    }
+
+    gf_uuid_copy(info->old_pargfid, oldloc->pargfid);
+    gf_uuid_copy(info->new_pargfid, newloc->pargfid);
+
+    info->oldname_len = len1;
+    info->newname_len = len2;
+    strncpy(info->buffer, oldloc->name, len1);
+    name = info->buffer + len1;
+    strncpy(name, newloc->name, len2);
+
+    ret = dict_set_bin(xattr, DHT_CHANGELOG_RENAME_OP_KEY, info, size);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, DHT_MSG_DICT_SET_FAILED,
+               "Failed to set dictionary value: key = %s,"
+               " path = %s",
+               DHT_CHANGELOG_RENAME_OP_KEY, oldloc->name);
+        GF_FREE(info);
+    }
+
+    return ret;
 }
 
+#define DHT_MARKER_DONT_ACCOUNT(xattr)                                         \
+    do {                                                                       \
+        int tmp = -1;                                                          \
+        if (!xattr) {                                                          \
+            xattr = dict_new();                                                \
+            if (!xattr)                                                        \
+                break;                                                         \
+        }                                                                      \
+        tmp = dict_set_str(xattr, GLUSTERFS_MARKER_DONT_ACCOUNT_KEY, "yes");   \
+        if (tmp) {                                                             \
+            gf_msg(this->name, GF_LOG_ERROR, 0, DHT_MSG_DICT_SET_FAILED,       \
+                   "Failed to set dictionary value: key = %s,"                 \
+                   " path = %s",                                               \
+                   GLUSTERFS_MARKER_DONT_ACCOUNT_KEY, local->loc.path);        \
+        }                                                                      \
+    } while (0)
+
+#define DHT_CHANGELOG_TRACK_AS_RENAME(xattr, oldloc, newloc)                   \
+    do {                                                                       \
+        int tmp = -1;                                                          \
+        if (!xattr) {                                                          \
+            xattr = dict_new();                                                \
+            if (!xattr) {                                                      \
+                gf_msg(this->name, GF_LOG_ERROR, 0, DHT_MSG_DICT_SET_FAILED,   \
+                       "Failed to create dictionary to "                       \
+                       "track rename");                                        \
+                break;                                                         \
+            }                                                                  \
+        }                                                                      \
+                                                                               \
+        tmp = dht_rename_track_for_changelog(this, xattr, oldloc, newloc);     \
+                                                                               \
+        if (tmp) {                                                             \
+            gf_msg(this->name, GF_LOG_ERROR, 0, DHT_MSG_DICT_SET_FAILED,       \
+                   "Failed to set dictionary value: key = %s,"                 \
+                   " path = %s",                                               \
+                   DHT_CHANGELOG_RENAME_OP_KEY, (oldloc)->path);               \
+        }                                                                      \
+    } while (0)
+
 int
-dht_rename_unlock (call_frame_t *frame, xlator_t *this)
+dht_rename_unlock(call_frame_t *frame, xlator_t *this)
 {
-        dht_local_t *local                      = NULL;
-        int          op_ret                     = -1;
-        char         src_gfid[GF_UUID_BUF_SIZE] = {0};
-        char         dst_gfid[GF_UUID_BUF_SIZE] = {0};
-
-        local = frame->local;
-
-        op_ret = dht_unlock_inodelk (frame, local->lock.locks,
-                                     local->lock.lk_count,
-                                     dht_rename_unlock_cbk);
-        if (op_ret < 0) {
-                uuid_utoa_r (local->loc.inode->gfid, src_gfid);
-
-                if (local->loc2.inode)
-                        uuid_utoa_r (local->loc2.inode->gfid, dst_gfid);
-
-                gf_log (this->name, GF_LOG_WARNING,
-                        "winding unlock inodelk failed "
-                        "rename (%s:%s:%s %s:%s:%s), "
-                        "stale locks left on bricks",
-                        local->loc.path, src_gfid, local->src_cached->name,
-                        local->loc2.path, dst_gfid,
-                        local->dst_cached ? local->dst_cached->name : NULL);
-
-                dht_rename_unlock_cbk (frame, NULL, this, 0, 0, NULL);
-        }
-
-        return 0;
+    dht_local_t *local = NULL;
+    int op_ret = -1;
+    char src_gfid[GF_UUID_BUF_SIZE] = {0};
+    char dst_gfid[GF_UUID_BUF_SIZE] = {0};
+    dht_ilock_wrap_t inodelk_wrapper = {
+        0,
+    };
+
+    local = frame->local;
+    inodelk_wrapper.locks = local->rename_inodelk_backward_compatible;
+    inodelk_wrapper.lk_count = local->rename_inodelk_bc_count;
+
+    op_ret = dht_unlock_inodelk_wrapper(frame, &inodelk_wrapper);
+    if (op_ret < 0) {
+        uuid_utoa_r(local->loc.inode->gfid, src_gfid);
+
+        if (local->loc2.inode)
+            uuid_utoa_r(local->loc2.inode->gfid, dst_gfid);
+
+        if (IA_ISREG(local->stbuf.ia_type))
+            gf_msg(this->name, GF_LOG_WARNING, 0, DHT_MSG_UNLOCKING_FAILED,
+                   "winding unlock inodelk failed "
+                   "rename (%s:%s:%s %s:%s:%s), "
+                   "stale locks left on bricks",
+                   local->loc.path, src_gfid, local->src_cached->name,
+                   local->loc2.path, dst_gfid,
+                   local->dst_cached ? local->dst_cached->name : NULL);
+        else
+            gf_msg(this->name, GF_LOG_WARNING, 0, DHT_MSG_UNLOCKING_FAILED,
+                   "winding unlock inodelk failed "
+                   "rename (%s:%s %s:%s), "
+                   "stale locks left on bricks",
+                   local->loc.path, src_gfid, local->loc2.path, dst_gfid);
+    }
+
+    dht_unlock_namespace(frame, &local->lock[0]);
+    dht_unlock_namespace(frame, &local->lock[1]);
+
+    dht_rename_unlock_cbk(frame, NULL, this, local->op_ret, local->op_errno,
+                          NULL);
+    return 0;
 }
 
 int
-dht_rename_done (call_frame_t *frame, xlator_t *this)
+dht_rename_done(call_frame_t *frame, xlator_t *this)
 {
-        dht_local_t *local                      = NULL;
+    dht_local_t *local = NULL;
 
-        local = frame->local;
+    local = frame->local;
 
-        if (local->linked == _gf_true) {
-                local->linked = _gf_false;
-                dht_linkfile_attr_heal (frame, this);
-        }
+    if (local->linked == _gf_true) {
+        local->linked = _gf_false;
+        dht_linkfile_attr_heal(frame, this);
+    }
 
-        dht_rename_unlock (frame, this);
-        return 0;
+    dht_rename_unlock(frame, this);
+    return 0;
 }
 
 int
-dht_rename_unlink_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                       int32_t op_ret, int32_t op_errno, struct iatt *preparent,
-                       struct iatt *postparent, dict_t *xdata)
+dht_rename_unlink_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                      int32_t op_ret, int32_t op_errno, struct iatt *preparent,
+                      struct iatt *postparent, dict_t *xdata)
 {
-        dht_local_t  *local = NULL;
-        call_frame_t *prev = NULL;
-        int           this_call_cnt = 0;
+    dht_local_t *local = NULL;
+    xlator_t *prev = NULL;
+    int this_call_cnt = 0;
 
-        local = frame->local;
-        prev  = cookie;
+    local = frame->local;
+    prev = cookie;
 
-        if (!local) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "!local, should not happen");
-                goto out;
-        }
+    FRAME_SU_UNDO(frame, dht_local_t);
+    if (!local) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, DHT_MSG_INVALID_VALUE,
+               "!local, should not happen");
+        goto out;
+    }
 
-        this_call_cnt = dht_frame_return (frame);
+    this_call_cnt = dht_frame_return(frame);
 
-        if (op_ret == -1) {
-                gf_msg (this->name, GF_LOG_WARNING, op_errno,
-                        DHT_MSG_UNLINK_FAILED,
-                        "%s: Rename: unlink on %s failed ",
-                        local->loc.path, prev->this->name);
-        }
+    if (op_ret == -1) {
+        gf_msg(this->name, GF_LOG_WARNING, op_errno, DHT_MSG_UNLINK_FAILED,
+               "%s: Rename: unlink on %s failed ", local->loc.path, prev->name);
+    }
 
-        WIPE (&local->preoldparent);
-        WIPE (&local->postoldparent);
-        WIPE (&local->preparent);
-        WIPE (&local->postparent);
+    WIPE(&local->preoldparent);
+    WIPE(&local->postoldparent);
+    WIPE(&local->preparent);
+    WIPE(&local->postparent);
 
-        if (is_last_call (this_call_cnt)) {
-                dht_rename_done (frame, this);
-        }
+    if (is_last_call(this_call_cnt)) {
+        dht_rename_done(frame, this);
+    }
 
 out:
-        return 0;
+    return 0;
 }
 
-
 int
-dht_rename_cleanup (call_frame_t *frame)
+dht_rename_cleanup(call_frame_t *frame)
 {
-        dht_local_t *local                  = NULL;
-        xlator_t    *this                   = NULL;
-        xlator_t    *src_hashed             = NULL;
-        xlator_t    *src_cached             = NULL;
-        xlator_t    *dst_hashed             = NULL;
-        xlator_t    *dst_cached             = NULL;
-        int          call_cnt               = 0;
-        dict_t      *xattr                  = NULL;
-        char         gfid[GF_UUID_BUF_SIZE] = {0};
-
-        local = frame->local;
-        this  = frame->this;
-
-        src_hashed = local->src_hashed;
-        src_cached = local->src_cached;
-        dst_hashed = local->dst_hashed;
-        dst_cached = local->dst_cached;
-
-        if (src_cached == dst_cached)
-                goto nolinks;
-
-        if (local->linked && (dst_hashed != src_hashed )&&
-                (dst_hashed != src_cached)) {
-                call_cnt++;
-        }
+    dht_local_t *local = NULL;
+    xlator_t *this = NULL;
+    xlator_t *src_hashed = NULL;
+    xlator_t *src_cached = NULL;
+    xlator_t *dst_hashed = NULL;
+    xlator_t *dst_cached = NULL;
+    int call_cnt = 0;
+    dict_t *xattr = NULL;
+    char gfid[GF_UUID_BUF_SIZE] = {0};
 
-        if (local->added_link && (src_cached != dst_hashed)) {
-                call_cnt++;
-        }
+    local = frame->local;
+    this = frame->this;
 
-        local->call_cnt = call_cnt;
+    src_hashed = local->src_hashed;
+    src_cached = local->src_cached;
+    dst_hashed = local->dst_hashed;
+    dst_cached = local->dst_cached;
 
-        if (!call_cnt)
-                goto nolinks;
+    if (src_cached == dst_cached)
+        goto nolinks;
 
-        DHT_MARK_FOP_INTERNAL (xattr);
+    if (local->linked && (dst_hashed != src_hashed) &&
+        (dst_hashed != src_cached)) {
+        call_cnt++;
+    }
 
-        gf_uuid_unparse(local->loc.inode->gfid, gfid);
+    if (local->added_link && (src_cached != dst_hashed)) {
+        call_cnt++;
+    }
 
-        if (local->linked && (dst_hashed != src_hashed) &&
-                        (dst_hashed != src_cached)) {
-                dict_t *xattr_new = NULL;
+    local->call_cnt = call_cnt;
 
-                gf_msg_trace (this->name, 0,
-                              "unlinking linkfile %s @ %s => %s, (gfid = %s)",
-                              local->loc.path, dst_hashed->name,
-                              src_cached->name, gfid);
+    if (!call_cnt)
+        goto nolinks;
 
-                xattr_new = dict_copy_with_ref (xattr, NULL);
+    DHT_MARK_FOP_INTERNAL(xattr);
 
+    gf_uuid_unparse(local->loc.inode->gfid, gfid);
 
-                DHT_MARKER_DONT_ACCOUNT(xattr_new);
+    if (local->linked && (dst_hashed != src_hashed) &&
+        (dst_hashed != src_cached)) {
+        dict_t *xattr_new = NULL;
 
-                STACK_WIND (frame, dht_rename_unlink_cbk,
-                            dst_hashed, dst_hashed->fops->unlink,
-                            &local->loc, 0, xattr_new);
+        gf_msg_trace(this->name, 0,
+                     "unlinking linkfile %s @ %s => %s, (gfid = %s)",
+                     local->loc.path, dst_hashed->name, src_cached->name, gfid);
 
-                dict_unref (xattr_new);
-                xattr_new = NULL;
-        }
+        xattr_new = dict_copy_with_ref(xattr, NULL);
 
-        if (local->added_link && (src_cached != dst_hashed)) {
-                dict_t *xattr_new = NULL;
+        DHT_MARKER_DONT_ACCOUNT(xattr_new);
 
-                gf_msg_trace (this->name, 0,
-                              "unlinking link %s => %s (%s), (gfid = %s)",
-                              local->loc.path, local->loc2.path,
-                              src_cached->name, gfid);
+        FRAME_SU_DO(frame, dht_local_t);
+        STACK_WIND_COOKIE(frame, dht_rename_unlink_cbk, dst_hashed, dst_hashed,
+                          dst_hashed->fops->unlink, &local->loc, 0, xattr_new);
 
-                xattr_new = dict_copy_with_ref (xattr, NULL);
+        dict_unref(xattr_new);
+        xattr_new = NULL;
+    }
 
-                if (gf_uuid_compare (local->loc.pargfid,
-                                  local->loc2.pargfid) == 0) {
-                        DHT_MARKER_DONT_ACCOUNT(xattr_new);
-                }
+    if (local->added_link && (src_cached != dst_hashed)) {
+        dict_t *xattr_new = NULL;
+
+        gf_msg_trace(this->name, 0, "unlinking link %s => %s (%s), (gfid = %s)",
+                     local->loc.path, local->loc2.path, src_cached->name, gfid);
 
-                STACK_WIND (frame, dht_rename_unlink_cbk,
-                            src_cached, src_cached->fops->unlink,
-                            &local->loc2, 0, xattr_new);
+        xattr_new = dict_copy_with_ref(xattr, NULL);
 
-                dict_unref (xattr_new);
-                xattr_new = NULL;
+        if (gf_uuid_compare(local->loc.pargfid, local->loc2.pargfid) == 0) {
+            DHT_MARKER_DONT_ACCOUNT(xattr_new);
         }
+        /* *
+         * The link to file is created using root permission.
+         * Hence deletion should happen using root. Otherwise
+         * it will fail.
+         */
+        FRAME_SU_DO(frame, dht_local_t);
+        STACK_WIND_COOKIE(frame, dht_rename_unlink_cbk, src_cached, src_cached,
+                          src_cached->fops->unlink, &local->loc2, 0, xattr_new);
 
-        if (xattr)
-                dict_unref (xattr);
+        dict_unref(xattr_new);
+        xattr_new = NULL;
+    }
 
-        return 0;
+    if (xattr)
+        dict_unref(xattr);
+
+    return 0;
 
 nolinks:
-        WIPE (&local->preoldparent);
-        WIPE (&local->postoldparent);
-        WIPE (&local->preparent);
-        WIPE (&local->postparent);
+    WIPE(&local->preoldparent);
+    WIPE(&local->postoldparent);
+    WIPE(&local->preparent);
+    WIPE(&local->postparent);
 
-        dht_rename_unlock (frame, this);
-        return 0;
+    dht_rename_unlock(frame, this);
+    return 0;
 }
 
-
 int
-dht_rename_links_create_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                             int32_t op_ret, int32_t op_errno,
-                             inode_t *inode, struct iatt *stbuf,
-                             struct iatt *preparent, struct iatt *postparent,
-                             dict_t *xdata)
+dht_rename_unlink(call_frame_t *frame, xlator_t *this)
 {
-        call_frame_t *prev = NULL;
-        dht_local_t  *local = NULL;
+    dht_local_t *local = NULL;
+    xlator_t *src_hashed = NULL;
+    xlator_t *src_cached = NULL;
+    xlator_t *dst_hashed = NULL;
+    xlator_t *dst_cached = NULL;
+    xlator_t *rename_subvol = NULL;
+    dict_t *xattr = NULL;
 
-        prev = cookie;
-        local = frame->local;
+    local = frame->local;
 
-        if (op_ret == -1) {
-                gf_log (this->name, GF_LOG_WARNING,
-                        "link/file %s on %s failed (%s)",
-                        local->loc.path, prev->this->name, strerror (op_errno));
-        }
+    src_hashed = local->src_hashed;
+    src_cached = local->src_cached;
+    dst_hashed = local->dst_hashed;
+    dst_cached = local->dst_cached;
 
-        if (local->linked == _gf_true) {
-                local->linked = _gf_false;
-                dht_linkfile_attr_heal (frame, this);
-        }
-        DHT_STACK_DESTROY (frame);
+    local->call_cnt = 0;
 
-        return 0;
-}
+    /* NOTE: rename_subvol is the same subvolume from which dht_rename_cbk
+     * is called. since rename has already happened on rename_subvol,
+     * unlink shouldn't be sent for oldpath (either linkfile or cached-file)
+     * on rename_subvol. */
+    if (src_cached == dst_cached)
+        rename_subvol = src_cached;
+    else
+        rename_subvol = dst_hashed;
 
+    /* TODO: delete files in background */
 
-int
-dht_rename_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                int32_t op_ret, int32_t op_errno, struct iatt *stbuf,
-                struct iatt *preoldparent, struct iatt *postoldparent,
-                struct iatt *prenewparent, struct iatt *postnewparent,
-                dict_t *xdata)
-{
-        dht_local_t  *local = NULL;
-        call_frame_t *prev = NULL;
-        xlator_t     *src_hashed = NULL;
-        xlator_t     *src_cached = NULL;
-        xlator_t     *dst_hashed = NULL;
-        xlator_t     *dst_cached = NULL;
-        xlator_t     *rename_subvol = NULL;
-        call_frame_t *link_frame = NULL;
-        dht_local_t *link_local = NULL;
-        dict_t       *xattr     = NULL;
-
-        local = frame->local;
-        prev = cookie;
-
-        src_hashed = local->src_hashed;
-        src_cached = local->src_cached;
-        dst_hashed = local->dst_hashed;
-        dst_cached = local->dst_cached;
-
-        if (local->linked == _gf_true)
-                FRAME_SU_UNDO (frame, dht_local_t);
-
-        /* It is a critical failure iff we fail to rename the cached file
-         * if the rename of the linkto failed, it is not a critical failure,
-         * and we do not want to lose the created hard link for the new
-         * name as that could have been read by other clients.
-         *
-         * NOTE: If another client is attempting the same oldname -> newname
-         * rename, and finds both file names as existing, and are hard links
-         * to each other, then FUSE would send in an unlink for oldname. In
-         * this time duration if we treat the linkto as a critical error and
-         * unlink the newname we created, we would have effectively lost the
-         * file to rename operations.
-         *
-         * Repercussions of treating this as a non-critical error is that
-         * we could leave behind a stale linkto file and/or not create the new
-         * linkto file, the second case would be rectified by a subsequent
-         * lookup, the first case by a rebalance, like for all stale linkto
-         * files */
-
-        if (op_ret == -1) {
-                /* Critical failure: unable to rename the cached file */
-                if (prev->this == src_cached) {
-                        gf_msg (this->name, GF_LOG_WARNING, op_errno,
-                                DHT_MSG_RENAME_FAILED,
-                                "%s: Rename on %s failed, (gfid = %s) ",
-                                local->loc.path, prev->this->name,
-                                local->loc.inode ?
-                                uuid_utoa(local->loc.inode->gfid):"");
-                        local->op_ret   = op_ret;
-                        local->op_errno = op_errno;
-                        goto cleanup;
-                } else {
-                        /* Non-critical failure, unable to rename the linkto
-                         * file
-                         */
-                        gf_msg (this->name, GF_LOG_INFO, op_errno,
-                                DHT_MSG_RENAME_FAILED,
-                                "%s: Rename (linkto file) on %s failed, "
-                                "(gfid = %s) ",
-                                local->loc.path, prev->this->name,
-                                local->loc.inode ?
-                                uuid_utoa(local->loc.inode->gfid):"");
-                }
-        }
+    if (src_cached != dst_hashed && src_cached != dst_cached)
+        local->call_cnt++;
 
-        if ((src_cached == dst_cached) && (dst_hashed != dst_cached)) {
-                link_frame = copy_frame (frame);
-                if (!link_frame) {
-                        goto err;
-                }
+    if (src_hashed != rename_subvol && src_hashed != src_cached)
+        local->call_cnt++;
 
-                /* fop value sent as maxvalue because it is not used
-                   anywhere in this case */
-                link_local = dht_local_init (link_frame, &local->loc2, NULL,
-                                             GF_FOP_MAXVALUE);
-                if (!link_local) {
-                        goto err;
-                }
+    if (dst_cached && dst_cached != dst_hashed && dst_cached != src_cached)
+        local->call_cnt++;
 
-                if (link_local->loc.inode)
-                        inode_unref (link_local->loc.inode);
-                link_local->loc.inode = inode_ref (local->loc.inode);
-                gf_uuid_copy (link_local->gfid, local->loc.inode->gfid);
+    if (local->call_cnt == 0)
+        goto unwind;
 
-                dht_linkfile_create (link_frame, dht_rename_links_create_cbk,
-                                     this, src_cached, dst_hashed,
-                                     &link_local->loc);
-        }
+    DHT_MARK_FOP_INTERNAL(xattr);
 
-err:
-        /* Merge attrs only from src_cached. In case there of src_cached !=
-         * dst_hashed, this ignores linkfile attrs. */
-        if (prev->this == src_cached) {
-                dht_iatt_merge (this, &local->stbuf, stbuf, prev->this);
-                dht_iatt_merge (this, &local->preoldparent, preoldparent,
-                                prev->this);
-                dht_iatt_merge (this, &local->postoldparent, postoldparent,
-                                prev->this);
-                dht_iatt_merge (this, &local->preparent, prenewparent,
-                                prev->this);
-                dht_iatt_merge (this, &local->postparent, postnewparent,
-                                prev->this);
-        }
+    if (src_cached != dst_hashed && src_cached != dst_cached) {
+        dict_t *xattr_new = NULL;
 
+        xattr_new = dict_copy_with_ref(xattr, NULL);
 
-        /* NOTE: rename_subvol is the same subvolume from which dht_rename_cbk
-         *       is called. since rename has already happened on rename_subvol,
-         *       unlink should not be sent for oldpath (either linkfile or cached-file)
-         *       on rename_subvol. */
-        if (src_cached == dst_cached)
-                rename_subvol = src_cached;
-        else
-                rename_subvol = dst_hashed;
+        gf_msg_trace(this->name, 0, "deleting old src datafile %s @ %s",
+                     local->loc.path, src_cached->name);
 
-        /* TODO: delete files in background */
+        if (gf_uuid_compare(local->loc.pargfid, local->loc2.pargfid) == 0) {
+            DHT_MARKER_DONT_ACCOUNT(xattr_new);
+        }
 
-        if (src_cached != dst_hashed && src_cached != dst_cached)
-                local->call_cnt++;
+        DHT_CHANGELOG_TRACK_AS_RENAME(xattr_new, &local->loc, &local->loc2);
+        STACK_WIND_COOKIE(frame, dht_rename_unlink_cbk, src_cached, src_cached,
+                          src_cached->fops->unlink, &local->loc, 0, xattr_new);
 
-        if (src_hashed != rename_subvol && src_hashed != src_cached)
-                local->call_cnt++;
+        dict_unref(xattr_new);
+        xattr_new = NULL;
+    }
 
-        if (dst_cached && dst_cached != dst_hashed && dst_cached != src_cached)
-                local->call_cnt++;
+    if (src_hashed != rename_subvol && src_hashed != src_cached) {
+        dict_t *xattr_new = NULL;
 
-        if (local->call_cnt == 0)
-                goto unwind;
+        xattr_new = dict_copy_with_ref(xattr, NULL);
 
-        DHT_MARK_FOP_INTERNAL (xattr);
+        gf_msg_trace(this->name, 0, "deleting old src linkfile %s @ %s",
+                     local->loc.path, src_hashed->name);
 
-        if (src_cached != dst_hashed && src_cached != dst_cached) {
-                dict_t *xattr_new = NULL;
+        DHT_MARKER_DONT_ACCOUNT(xattr_new);
 
-                xattr_new = dict_copy_with_ref (xattr, NULL);
+        STACK_WIND_COOKIE(frame, dht_rename_unlink_cbk, src_hashed, src_hashed,
+                          src_hashed->fops->unlink, &local->loc, 0, xattr_new);
 
-                gf_msg_trace (this->name, 0,
-                              "deleting old src datafile %s @ %s",
-                              local->loc.path, src_cached->name);
+        dict_unref(xattr_new);
+        xattr_new = NULL;
+    }
 
-                if (gf_uuid_compare (local->loc.pargfid,
-                                  local->loc2.pargfid) == 0) {
-                        DHT_MARKER_DONT_ACCOUNT(xattr_new);
-                }
+    if (dst_cached && (dst_cached != dst_hashed) &&
+        (dst_cached != src_cached)) {
+        gf_msg_trace(this->name, 0, "deleting old dst datafile %s @ %s",
+                     local->loc2.path, dst_cached->name);
 
-                STACK_WIND (frame, dht_rename_unlink_cbk,
-                            src_cached, src_cached->fops->unlink,
-                            &local->loc, 0, xattr_new);
+        STACK_WIND_COOKIE(frame, dht_rename_unlink_cbk, dst_cached, dst_cached,
+                          dst_cached->fops->unlink, &local->loc2, 0, xattr);
+    }
+    if (xattr)
+        dict_unref(xattr);
+    return 0;
 
-                dict_unref (xattr_new);
-                xattr_new = NULL;
-        }
-
-        if (src_hashed != rename_subvol && src_hashed != src_cached) {
-                dict_t *xattr_new = NULL;
+unwind:
+    WIPE(&local->preoldparent);
+    WIPE(&local->postoldparent);
+    WIPE(&local->preparent);
+    WIPE(&local->postparent);
 
-                xattr_new = dict_copy_with_ref (xattr, NULL);
+    dht_rename_done(frame, this);
 
-                gf_msg_trace (this->name, 0,
-                              "deleting old src linkfile %s @ %s",
-                              local->loc.path, src_hashed->name);
+    return 0;
+}
 
-                DHT_MARKER_DONT_ACCOUNT(xattr_new);
+int
+dht_rename_links_create_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                            int32_t op_ret, int32_t op_errno, inode_t *inode,
+                            struct iatt *stbuf, struct iatt *preparent,
+                            struct iatt *postparent, dict_t *xdata)
+{
+    xlator_t *prev = NULL;
+    dht_local_t *local = NULL;
+    call_frame_t *main_frame = NULL;
+
+    prev = cookie;
+    local = frame->local;
+    main_frame = local->main_frame;
+
+    /* TODO: Handle this case in lookup-optimize */
+    if (op_ret == -1) {
+        gf_msg(this->name, GF_LOG_WARNING, op_errno, DHT_MSG_CREATE_LINK_FAILED,
+               "link/file %s on %s failed", local->loc.path, prev->name);
+    }
+
+    if (local->linked == _gf_true) {
+        local->linked = _gf_false;
+        dht_linkfile_attr_heal(frame, this);
+    }
+
+    dht_rename_unlink(main_frame, this);
+    DHT_STACK_DESTROY(frame);
+    return 0;
+}
 
-                STACK_WIND (frame, dht_rename_unlink_cbk,
-                            src_hashed, src_hashed->fops->unlink,
-                            &local->loc, 0, xattr_new);
+int
+dht_rename_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+               int32_t op_ret, int32_t op_errno, struct iatt *stbuf,
+               struct iatt *preoldparent, struct iatt *postoldparent,
+               struct iatt *prenewparent, struct iatt *postnewparent,
+               dict_t *xdata)
+{
+    dht_local_t *local = NULL;
+    xlator_t *prev = NULL;
+    xlator_t *src_cached = NULL;
+    xlator_t *dst_hashed = NULL;
+    xlator_t *dst_cached = NULL;
+    call_frame_t *link_frame = NULL;
+    dht_local_t *link_local = NULL;
+
+    local = frame->local;
+    prev = cookie;
+
+    src_cached = local->src_cached;
+    dst_hashed = local->dst_hashed;
+    dst_cached = local->dst_cached;
+
+    if (local->linked == _gf_true)
+        FRAME_SU_UNDO(frame, dht_local_t);
+
+    /* It is a critical failure iff we fail to rename the cached file
+     * if the rename of the linkto failed, it is not a critical failure,
+     * and we do not want to lose the created hard link for the new
+     * name as that could have been read by other clients.
+     *
+     * NOTE: If another client is attempting the same oldname -> newname
+     * rename, and finds both file names as existing, and are hard links
+     * to each other, then FUSE would send in an unlink for oldname. In
+     * this time duration if we treat the linkto as a critical error and
+     * unlink the newname we created, we would have effectively lost the
+     * file to rename operations.
+     *
+     * Repercussions of treating this as a non-critical error is that
+     * we could leave behind a stale linkto file and/or not create the new
+     * linkto file, the second case would be rectified by a subsequent
+     * lookup, the first case by a rebalance, like for all stale linkto
+     * files */
+
+    if (op_ret == -1) {
+        /* Critical failure: unable to rename the cached file */
+        if (prev == src_cached) {
+            gf_msg(this->name, GF_LOG_WARNING, op_errno, DHT_MSG_RENAME_FAILED,
+                   "%s: Rename on %s failed, (gfid = %s) ", local->loc.path,
+                   prev->name,
+                   local->loc.inode ? uuid_utoa(local->loc.inode->gfid) : "");
+            local->op_ret = op_ret;
+            local->op_errno = op_errno;
+            goto cleanup;
+        } else {
+            /* Non-critical failure, unable to rename the linkto
+             * file
+             */
+            gf_msg(this->name, GF_LOG_INFO, op_errno, DHT_MSG_RENAME_FAILED,
+                   "%s: Rename (linkto file) on %s failed, "
+                   "(gfid = %s) ",
+                   local->loc.path, prev->name,
+                   local->loc.inode ? uuid_utoa(local->loc.inode->gfid) : "");
+        }
+    }
+    if (xdata) {
+        if (!local->xattr)
+            local->xattr = dict_ref(xdata);
+        else
+            local->xattr = dict_copy_with_ref(xdata, local->xattr);
+    }
+
+    /* Merge attrs only from src_cached. In case there of src_cached !=
+     * dst_hashed, this ignores linkfile attrs. */
+    if (prev == src_cached) {
+        dht_iatt_merge(this, &local->stbuf, stbuf);
+        dht_iatt_merge(this, &local->preoldparent, preoldparent);
+        dht_iatt_merge(this, &local->postoldparent, postoldparent);
+        dht_iatt_merge(this, &local->preparent, prenewparent);
+        dht_iatt_merge(this, &local->postparent, postnewparent);
+    }
+
+    /* Create the linkto file for the dst file */
+    if ((src_cached == dst_cached) && (dst_hashed != dst_cached)) {
+        link_frame = copy_frame(frame);
+        if (!link_frame) {
+            goto unlink;
+        }
 
-                dict_unref (xattr_new);
-                xattr_new = NULL;
+        /* fop value sent as maxvalue because it is not used
+         * anywhere in this case */
+        link_local = dht_local_init(link_frame, &local->loc2, NULL,
+                                    GF_FOP_MAXVALUE);
+        if (!link_local) {
+            goto unlink;
         }
 
-        if (dst_cached
-            && (dst_cached != dst_hashed)
-            && (dst_cached != src_cached)) {
-                gf_msg_trace (this->name, 0,
-                              "deleting old dst datafile %s @ %s",
-                              local->loc2.path, dst_cached->name);
+        if (link_local->loc.inode)
+            inode_unref(link_local->loc.inode);
+        link_local->loc.inode = inode_ref(local->loc.inode);
+        link_local->main_frame = frame;
+        link_local->stbuf = local->stbuf;
+        gf_uuid_copy(link_local->gfid, local->loc.inode->gfid);
 
-                STACK_WIND (frame, dht_rename_unlink_cbk,
-                            dst_cached, dst_cached->fops->unlink,
-                            &local->loc2, 0, xattr);
-        }
-        if (xattr)
-                dict_unref (xattr);
+        dht_linkfile_create(link_frame, dht_rename_links_create_cbk, this,
+                            src_cached, dst_hashed, &link_local->loc);
         return 0;
+    }
 
-unwind:
-        WIPE (&local->preoldparent);
-        WIPE (&local->postoldparent);
-        WIPE (&local->preparent);
-        WIPE (&local->postparent);
-
-        dht_rename_done (frame, this);
+unlink:
 
-        return 0;
+    if (link_frame) {
+        DHT_STACK_DESTROY(link_frame);
+    }
+    dht_rename_unlink(frame, this);
+    return 0;
 
 cleanup:
-        dht_rename_cleanup (frame);
+    dht_rename_cleanup(frame);
 
-        return 0;
+    return 0;
 }
 
-
 int
-dht_do_rename (call_frame_t *frame)
+dht_do_rename(call_frame_t *frame)
 {
-        dht_local_t *local         = NULL;
-        xlator_t    *dst_hashed    = NULL;
-        xlator_t    *src_cached    = NULL;
-        xlator_t    *dst_cached    = NULL;
-        xlator_t    *this          = NULL;
-        xlator_t    *rename_subvol = NULL;
-        dict_t      *dict          = NULL;
-
-        local = frame->local;
-        this  = frame->this;
-
-        dst_hashed = local->dst_hashed;
-        dst_cached = local->dst_cached;
-        src_cached = local->src_cached;
-
-        if (src_cached == dst_cached)
-                rename_subvol = src_cached;
-        else
-                rename_subvol = dst_hashed;
-
-        if ((src_cached != dst_hashed) && (rename_subvol == dst_hashed)) {
-                DHT_MARKER_DONT_ACCOUNT(dict);
-        }
-
-        gf_msg_trace (this->name, 0,
-                      "renaming %s => %s (%s)",
-                      local->loc.path, local->loc2.path, rename_subvol->name);
-
-        if (local->linked == _gf_true)
-                FRAME_SU_DO (frame, dht_local_t);
-        STACK_WIND (frame, dht_rename_cbk,
-                    rename_subvol, rename_subvol->fops->rename,
-                    &local->loc, &local->loc2, dict);
-
-        return 0;
+    dht_local_t *local = NULL;
+    xlator_t *dst_hashed = NULL;
+    xlator_t *src_cached = NULL;
+    xlator_t *dst_cached = NULL;
+    xlator_t *this = NULL;
+    xlator_t *rename_subvol = NULL;
+
+    local = frame->local;
+    this = frame->this;
+
+    dst_hashed = local->dst_hashed;
+    dst_cached = local->dst_cached;
+    src_cached = local->src_cached;
+
+    if (src_cached == dst_cached)
+        rename_subvol = src_cached;
+    else
+        rename_subvol = dst_hashed;
+
+    if ((src_cached != dst_hashed) && (rename_subvol == dst_hashed)) {
+        DHT_MARKER_DONT_ACCOUNT(local->xattr_req);
+    }
+
+    if (rename_subvol == src_cached) {
+        DHT_CHANGELOG_TRACK_AS_RENAME(local->xattr_req, &local->loc,
+                                      &local->loc2);
+    }
+
+    gf_msg_trace(this->name, 0, "renaming %s => %s (%s)", local->loc.path,
+                 local->loc2.path, rename_subvol->name);
+
+    if (local->linked == _gf_true)
+        FRAME_SU_DO(frame, dht_local_t);
+    STACK_WIND_COOKIE(frame, dht_rename_cbk, rename_subvol, rename_subvol,
+                      rename_subvol->fops->rename, &local->loc, &local->loc2,
+                      local->xattr_req);
+    return 0;
 }
 
 int
-dht_rename_link_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                      int32_t op_ret, int32_t op_errno,
-                      inode_t *inode, struct iatt *stbuf,
-                      struct iatt *preparent, struct iatt *postparent,
-                      dict_t *xdata)
+dht_rename_link_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                    int32_t op_ret, int32_t op_errno, inode_t *inode,
+                    struct iatt *stbuf, struct iatt *preparent,
+                    struct iatt *postparent, dict_t *xdata)
 {
-        dht_local_t  *local = NULL;
-        call_frame_t *prev = NULL;
+    dht_local_t *local = NULL;
+    xlator_t *prev = NULL;
 
-        local = frame->local;
-        prev = cookie;
+    local = frame->local;
+    prev = cookie;
 
-        if (op_ret == -1) {
-                gf_msg_debug (this->name, 0,
-                              "link/file on %s failed (%s)",
-                              prev->this->name, strerror (op_errno));
-                local->op_ret   = -1;
-                local->op_errno = op_errno;
-                local->added_link = _gf_false;
-        } else
-                dht_iatt_merge (this, &local->stbuf, stbuf, prev->this);
+    if (op_ret == -1) {
+        gf_msg_debug(this->name, 0, "link/file on %s failed (%s)", prev->name,
+                     strerror(op_errno));
+        local->op_ret = -1;
+        local->op_errno = op_errno;
+        local->added_link = _gf_false;
+    } else
+        dht_iatt_merge(this, &local->stbuf, stbuf);
 
-        if (local->op_ret == -1)
-                goto cleanup;
+    if (local->op_ret == -1)
+        goto cleanup;
 
-        dht_do_rename (frame);
+    dht_do_rename(frame);
 
-        return 0;
+    return 0;
 
 cleanup:
-        dht_rename_cleanup (frame);
+    dht_rename_cleanup(frame);
 
-        return 0;
+    return 0;
 }
 
 int
-dht_rename_linkto_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                      int32_t op_ret, int32_t op_errno,
-                      inode_t *inode, struct iatt *stbuf,
-                      struct iatt *preparent, struct iatt *postparent,
-                      dict_t *xdata)
+dht_rename_linkto_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                      int32_t op_ret, int32_t op_errno, inode_t *inode,
+                      struct iatt *stbuf, struct iatt *preparent,
+                      struct iatt *postparent, dict_t *xdata)
 {
-        dht_local_t     *local = NULL;
-        call_frame_t    *prev = NULL;
-        xlator_t        *src_cached = NULL;
-        dict_t          *xattr = NULL;
+    dht_local_t *local = NULL;
+    xlator_t *prev = NULL;
+    xlator_t *src_cached = NULL;
+    dict_t *xattr = NULL;
 
-        DHT_MARK_FOP_INTERNAL (xattr);
+    local = frame->local;
+    DHT_MARK_FOP_INTERNAL(xattr);
+    prev = cookie;
+    src_cached = local->src_cached;
 
-        local = frame->local;
-        prev = cookie;
+    if (op_ret == -1) {
+        gf_msg_debug(this->name, 0, "link/file on %s failed (%s)", prev->name,
+                     strerror(op_errno));
+        local->op_ret = -1;
+        local->op_errno = op_errno;
+    }
 
-        src_cached = local->src_cached;
+    /* If linkto creation failed move to failure cleanup code,
+     * instead of continuing with creating the link file */
+    if (local->op_ret != 0) {
+        goto cleanup;
+    }
 
-        if (op_ret == -1) {
-                gf_msg_debug (this->name, 0,
-                              "link/file on %s failed (%s)",
-                              prev->this->name, strerror (op_errno));
-                local->op_ret = -1;
-                local->op_errno = op_errno;
-        }
+    gf_msg_trace(this->name, 0, "link %s => %s (%s)", local->loc.path,
+                 local->loc2.path, src_cached->name);
+    if (gf_uuid_compare(local->loc.pargfid, local->loc2.pargfid) == 0) {
+        DHT_MARKER_DONT_ACCOUNT(xattr);
+    }
 
-        /* If linkto creation failed move to failure cleanup code,
-        * instead of continuing with creating the link file */
-        if (local->op_ret != 0) {
-                goto cleanup;
-        }
-
-        gf_msg_trace (this->name, 0,
-                      "link %s => %s (%s)", local->loc.path,
-                      local->loc2.path, src_cached->name);
-        if (gf_uuid_compare (local->loc.pargfid,
-                                local->loc2.pargfid) == 0) {
-                DHT_MARKER_DONT_ACCOUNT(xattr);
-        }
+    local->added_link = _gf_true;
 
-        local->added_link = _gf_true;
+    STACK_WIND_COOKIE(frame, dht_rename_link_cbk, src_cached, src_cached,
+                      src_cached->fops->link, &local->loc, &local->loc2, xattr);
 
-        STACK_WIND (frame, dht_rename_link_cbk,
-                    src_cached, src_cached->fops->link,
-                    &local->loc, &local->loc2, xattr);
+    if (xattr)
+        dict_unref(xattr);
 
-        if (xattr)
-                dict_unref (xattr);
-
-        return 0;
+    return 0;
 
 cleanup:
-        dht_rename_cleanup (frame);
+    dht_rename_cleanup(frame);
 
-        if (xattr)
-                dict_unref (xattr);
+    if (xattr)
+        dict_unref(xattr);
 
-        return 0;
+    return 0;
 }
 
 int
-dht_rename_unlink_links_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                             int32_t op_ret, int32_t op_errno,
-                             struct iatt *preparent, struct iatt *postparent,
-                             dict_t *xdata)
+dht_rename_unlink_links_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                            int32_t op_ret, int32_t op_errno,
+                            struct iatt *preparent, struct iatt *postparent,
+                            dict_t *xdata)
 {
-	dht_local_t  *local = NULL;
-	call_frame_t *prev = NULL;
+    dht_local_t *local = NULL;
+    xlator_t *prev = NULL;
 
+    local = frame->local;
+    prev = cookie;
 
-	local = frame->local;
-	prev = cookie;
+    if ((op_ret == -1) && (op_errno != ENOENT)) {
+        gf_msg_debug(this->name, 0, "unlink of %s on %s failed (%s)",
+                     local->loc2.path, prev->name, strerror(op_errno));
+        local->op_ret = -1;
+        local->op_errno = op_errno;
+    }
 
-	if ((op_ret == -1) && (op_errno != ENOENT)) {
-		gf_msg_debug (this->name, 0,
-		              "unlink of %s on %s failed (%s)",
-			      local->loc2.path, prev->this->name,
-                              strerror (op_errno));
-		local->op_ret   = -1;
-		local->op_errno = op_errno;
-	}
+    if (local->op_ret == -1)
+        goto cleanup;
 
-        if (local->op_ret == -1)
-                goto cleanup;
+    dht_do_rename(frame);
 
-        dht_do_rename (frame);
-
-	return 0;
+    return 0;
 
 cleanup:
-        dht_rename_cleanup (frame);
+    dht_rename_cleanup(frame);
 
-	return 0;
+    return 0;
 }
 
-
 int
-dht_rename_create_links (call_frame_t *frame)
+dht_rename_create_links(call_frame_t *frame)
 {
-        dht_local_t *local = NULL;
-        xlator_t    *this = NULL;
-        xlator_t    *src_hashed = NULL;
-        xlator_t    *src_cached = NULL;
-        xlator_t    *dst_hashed = NULL;
-        xlator_t    *dst_cached = NULL;
-        int          call_cnt = 0;
-        dict_t      *xattr = NULL;
+    dht_local_t *local = NULL;
+    xlator_t *this = NULL;
+    xlator_t *src_hashed = NULL;
+    xlator_t *src_cached = NULL;
+    xlator_t *dst_hashed = NULL;
+    xlator_t *dst_cached = NULL;
+    int call_cnt = 0;
+    dict_t *xattr = NULL;
 
+    local = frame->local;
+    this = frame->this;
 
-        local = frame->local;
-        this  = frame->this;
+    src_hashed = local->src_hashed;
+    src_cached = local->src_cached;
+    dst_hashed = local->dst_hashed;
+    dst_cached = local->dst_cached;
 
-        src_hashed = local->src_hashed;
-        src_cached = local->src_cached;
-        dst_hashed = local->dst_hashed;
-        dst_cached = local->dst_cached;
+    DHT_MARK_FOP_INTERNAL(xattr);
 
-        DHT_MARK_FOP_INTERNAL (xattr);
+    if (src_cached == dst_cached) {
+        dict_t *xattr_new = NULL;
 
-        if (src_cached == dst_cached) {
-                dict_t *xattr_new = NULL;
+        if (dst_hashed == dst_cached)
+            goto nolinks;
 
-                if (dst_hashed == dst_cached)
-                        goto nolinks;
+        xattr_new = dict_copy_with_ref(xattr, NULL);
 
-                xattr_new = dict_copy_with_ref (xattr, NULL);
+        gf_msg_trace(this->name, 0, "unlinking dst linkfile %s @ %s",
+                     local->loc2.path, dst_hashed->name);
 
-		gf_msg_trace (this->name, 0,
-		              "unlinking dst linkfile %s @ %s",
-			      local->loc2.path, dst_hashed->name);
+        DHT_MARKER_DONT_ACCOUNT(xattr_new);
 
-                DHT_MARKER_DONT_ACCOUNT(xattr_new);
+        STACK_WIND_COOKIE(frame, dht_rename_unlink_links_cbk, dst_hashed,
+                          dst_hashed, dst_hashed->fops->unlink, &local->loc2, 0,
+                          xattr_new);
 
-		STACK_WIND (frame, dht_rename_unlink_links_cbk,
-			    dst_hashed, dst_hashed->fops->unlink,
-			    &local->loc2, 0, xattr_new);
-
-                dict_unref (xattr_new);
-                return 0;
-        }
+        dict_unref(xattr_new);
+        if (xattr)
+            dict_unref(xattr);
 
-        if (src_cached != dst_hashed) {
-                /* needed to create the link file */
-                call_cnt++;
-                if (dst_hashed != src_hashed)
-                        /* needed to create the linkto file */
-                        call_cnt ++;
+        return 0;
+    }
+
+    if (src_cached != dst_hashed) {
+        /* needed to create the link file */
+        call_cnt++;
+        if (dst_hashed != src_hashed)
+            /* needed to create the linkto file */
+            call_cnt++;
+    }
+
+    /* We should not have any failures post the link creation, as this
+     * introduces the newname into the namespace. Clients could have cached
+     * the existence of the newname and may start taking actions based on
+     * the same. Hence create the linkto first, and then attempt the link.
+     *
+     * NOTE: If another client is attempting the same oldname -> newname
+     * rename, and finds both file names as existing, and are hard links
+     * to each other, then FUSE would send in an unlink for oldname. In
+     * this time duration if we treat the linkto as a critical error and
+     * unlink the newname we created, we would have effectively lost the
+     * file to rename operations. */
+    if (dst_hashed != src_hashed && src_cached != dst_hashed) {
+        gf_msg_trace(this->name, 0, "linkfile %s @ %s => %s", local->loc.path,
+                     dst_hashed->name, src_cached->name);
+
+        memcpy(local->gfid, local->loc.inode->gfid, 16);
+        dht_linkfile_create(frame, dht_rename_linkto_cbk, this, src_cached,
+                            dst_hashed, &local->loc);
+    } else if (src_cached != dst_hashed) {
+        dict_t *xattr_new = NULL;
+
+        xattr_new = dict_copy_with_ref(xattr, NULL);
+
+        gf_msg_trace(this->name, 0, "link %s => %s (%s)", local->loc.path,
+                     local->loc2.path, src_cached->name);
+        if (gf_uuid_compare(local->loc.pargfid, local->loc2.pargfid) == 0) {
+            DHT_MARKER_DONT_ACCOUNT(xattr_new);
         }
 
-        /* We should not have any failures post the link creation, as this
-         * introduces the newname into the namespace. Clients could have cached
-         * the existence of the newname and may start taking actions based on
-         * the same. Hence create the linkto first, and then attempt the link.
-         *
-         * NOTE: If another client is attempting the same oldname -> newname
-         * rename, and finds both file names as existing, and are hard links
-         * to each other, then FUSE would send in an unlink for oldname. In
-         * this time duration if we treat the linkto as a critical error and
-         * unlink the newname we created, we would have effectively lost the
-         * file to rename operations. */
-        if (dst_hashed != src_hashed && src_cached != dst_hashed) {
-                gf_msg_trace (this->name, 0,
-                              "linkfile %s @ %s => %s",
-                              local->loc.path, dst_hashed->name,
-                              src_cached->name);
-
-                memcpy (local->gfid, local->loc.inode->gfid, 16);
-                dht_linkfile_create (frame, dht_rename_linkto_cbk, this,
-                                     src_cached, dst_hashed, &local->loc);
-        } else if (src_cached != dst_hashed) {
-                dict_t *xattr_new = NULL;
-
-                xattr_new = dict_copy_with_ref (xattr, NULL);
-
-                gf_msg_trace (this->name, 0,
-                              "link %s => %s (%s)", local->loc.path,
-                              local->loc2.path, src_cached->name);
-                if (gf_uuid_compare (local->loc.pargfid,
-                                  local->loc2.pargfid) == 0) {
-                        DHT_MARKER_DONT_ACCOUNT(xattr_new);
-                }
-
-                local->added_link = _gf_true;
+        local->added_link = _gf_true;
 
-                STACK_WIND (frame, dht_rename_link_cbk,
-                            src_cached, src_cached->fops->link,
-                            &local->loc, &local->loc2, xattr_new);
+        STACK_WIND_COOKIE(frame, dht_rename_link_cbk, src_cached, src_cached,
+                          src_cached->fops->link, &local->loc, &local->loc2,
+                          xattr_new);
 
-                dict_unref (xattr_new);
-        }
+        dict_unref(xattr_new);
+    }
 
 nolinks:
-        if (!call_cnt) {
-                /* skip to next step */
-                dht_do_rename (frame);
-        }
-        if (xattr)
-                dict_unref (xattr);
-
-        return 0;
+    if (!call_cnt) {
+        /* skip to next step */
+        dht_do_rename(frame);
+    }
+    if (xattr)
+        dict_unref(xattr);
+
+    return 0;
 }
 
 int
-dht_rename_lookup_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                       int op_ret, int op_errno,
-                       inode_t *inode, struct iatt *stbuf, dict_t *xattr,
-                       struct iatt *postparent)
+dht_rename_lookup_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                      int op_ret, int op_errno, inode_t *inode,
+                      struct iatt *stbuf, dict_t *xattr,
+                      struct iatt *postparent)
 {
-        dht_local_t *local                      = NULL;
-        int          call_cnt                   = 0;
-        dht_conf_t  *conf                       = NULL;
-
-        local = frame->local;
-        conf = this->private;
-
-        if (op_ret < 0) {
-                /* The meaning of is_linkfile is overloaded here. For locking
-                 * to work properly both rebalance and rename should acquire
-                 * lock on datafile. The reason for sending this lookup is to
-                 * find out whether we've acquired a lock on data file.
-                 * Between the lookup before rename and this rename, the
-                 * file could be migrated by a rebalance process and now this
-                 * file this might be a linkto file. We verify that by sending
-                 * this lookup. However, if this lookup fails we cannot really
-                 * say whether we've acquired lock on a datafile or linkto file.
-                 * So, we act conservatively and _assume_
-                 * that this is a linkfile and fail the rename operation.
-                 */
-                local->is_linkfile = _gf_true;
-        } else if (xattr && check_is_linkfile (inode, stbuf, xattr,
-                                               conf->link_xattr_name)) {
-                local->is_linkfile = _gf_true;
-        }
-
-        call_cnt = dht_frame_return (frame);
-        if (is_last_call (call_cnt)) {
-                if (local->is_linkfile) {
-                        local->op_ret = -1;
-                        local->op_errno = EBUSY;
-                        goto fail;
+    dht_local_t *local = NULL;
+    int call_cnt = 0;
+    dht_conf_t *conf = NULL;
+    char gfid_local[GF_UUID_BUF_SIZE] = {0};
+    char gfid_server[GF_UUID_BUF_SIZE] = {0};
+    int child_index = -1;
+    gf_boolean_t is_src = _gf_false;
+    loc_t *loc = NULL;
+
+    child_index = (long)cookie;
+
+    local = frame->local;
+    conf = this->private;
+
+    is_src = (child_index == 0);
+    if (is_src)
+        loc = &local->loc;
+    else
+        loc = &local->loc2;
+
+    if (op_ret >= 0) {
+        if (is_src)
+            local->src_cached = dht_subvol_get_cached(this, local->loc.inode);
+        else {
+            if (loc->inode)
+                gf_uuid_unparse(loc->inode->gfid, gfid_local);
+
+            gf_msg_debug(this->name, 0,
+                         "dst_cached before lookup: %s, "
+                         "(path:%s)(gfid:%s),",
+                         local->loc2.path,
+                         local->dst_cached ? local->dst_cached->name : NULL,
+                         local->dst_cached ? gfid_local : NULL);
+
+            local->dst_cached = dht_subvol_get_cached(this,
+                                                      local->loc2_copy.inode);
+
+            gf_uuid_unparse(stbuf->ia_gfid, gfid_local);
+
+            gf_msg_debug(this->name, GF_LOG_WARNING,
+                         "dst_cached after lookup: %s, "
+                         "(path:%s)(gfid:%s)",
+                         local->loc2.path,
+                         local->dst_cached ? local->dst_cached->name : NULL,
+                         local->dst_cached ? gfid_local : NULL);
+
+            if ((local->loc2.inode == NULL) ||
+                gf_uuid_compare(stbuf->ia_gfid, local->loc2.inode->gfid)) {
+                if (local->loc2.inode != NULL) {
+                    inode_unlink(local->loc2.inode, local->loc2.parent,
+                                 local->loc2.name);
+                    inode_unref(local->loc2.inode);
                 }
 
-                dht_rename_create_links (frame);
+                local->loc2.inode = inode_link(local->loc2_copy.inode,
+                                               local->loc2_copy.parent,
+                                               local->loc2_copy.name, stbuf);
+                gf_uuid_copy(local->loc2.gfid, stbuf->ia_gfid);
+            }
+        }
+    }
+
+    if (op_ret < 0) {
+        if (is_src) {
+            /* The meaning of is_linkfile is overloaded here. For locking
+             * to work properly both rebalance and rename should acquire
+             * lock on datafile. The reason for sending this lookup is to
+             * find out whether we've acquired a lock on data file.
+             * Between the lookup before rename and this rename, the
+             * file could be migrated by a rebalance process and now this
+             * file this might be a linkto file. We verify that by sending
+             * this lookup. However, if this lookup fails we cannot really
+             * say whether we've acquired lock on a datafile or linkto file.
+             * So, we act conservatively and _assume_
+             * that this is a linkfile and fail the rename operation.
+             */
+            local->is_linkfile = _gf_true;
+            local->op_errno = op_errno;
+        } else {
+            if (local->dst_cached)
+                gf_msg_debug(this->name, op_errno,
+                             "file %s (gfid:%s) was present "
+                             "(hashed-subvol=%s, "
+                             "cached-subvol=%s) before rename,"
+                             " but lookup failed",
+                             local->loc2.path,
+                             uuid_utoa(local->loc2.inode->gfid),
+                             local->dst_hashed->name, local->dst_cached->name);
+            if (dht_inode_missing(op_errno))
+                local->dst_cached = NULL;
+        }
+    } else if (is_src && xattr &&
+               check_is_linkfile(inode, stbuf, xattr, conf->link_xattr_name)) {
+        local->is_linkfile = _gf_true;
+        /* Found linkto file instead of data file, passdown ENOENT
+         * based on the above comment */
+        local->op_errno = ENOENT;
+    }
+
+    if (!local->is_linkfile && (op_ret >= 0) &&
+        gf_uuid_compare(loc->gfid, stbuf->ia_gfid)) {
+        gf_uuid_unparse(loc->gfid, gfid_local);
+        gf_uuid_unparse(stbuf->ia_gfid, gfid_server);
+
+        gf_msg(this->name, GF_LOG_WARNING, 0, DHT_MSG_GFID_MISMATCH,
+               "path:%s, received a different gfid, local_gfid= %s"
+               " server_gfid: %s",
+               local->loc.path, gfid_local, gfid_server);
+
+        /* Will passdown ENOENT anyway since the file we sent on
+         * rename is replaced with a different file */
+        local->op_errno = ENOENT;
+        /* Since local->is_linkfile is used here to detect failure,
+         * marking this to true */
+        local->is_linkfile = _gf_true;
+    }
+
+    call_cnt = dht_frame_return(frame);
+    if (is_last_call(call_cnt)) {
+        if (local->is_linkfile) {
+            local->op_ret = -1;
+            goto fail;
         }
 
-        return 0;
+        dht_rename_create_links(frame);
+    }
+
+    return 0;
 fail:
-        dht_rename_unlock (frame, this);
-        return 0;
+    dht_rename_unlock(frame, this);
+    return 0;
 }
 
-int32_t
-dht_rename_lock_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                     int32_t op_ret, int32_t op_errno, dict_t *xdata)
+int
+dht_rename_file_lock1_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                          int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
-        dht_local_t *local                      = NULL;
-        char         src_gfid[GF_UUID_BUF_SIZE] = {0};
-        char         dst_gfid[GF_UUID_BUF_SIZE] = {0};
-        dict_t      *xattr_req                  = NULL;
-        dht_conf_t  *conf                       = NULL;
-        int          i                          = 0;
-
-        local = frame->local;
-        conf = this->private;
-
-        if (op_ret < 0) {
-                uuid_utoa_r (local->loc.inode->gfid, src_gfid);
-
-                if (local->loc2.inode)
-                        uuid_utoa_r (local->loc2.inode->gfid, dst_gfid);
-
-                gf_log (this->name, GF_LOG_WARNING,
-                        "acquiring inodelk failed (%s) "
-                        "rename (%s:%s:%s %s:%s:%s), returning EBUSY",
-                        strerror (op_errno),
-                        local->loc.path, src_gfid, local->src_cached->name,
-                        local->loc2.path, dst_gfid,
-                        local->dst_cached ? local->dst_cached->name : NULL);
-
-                local->op_ret = -1;
-                local->op_errno = (op_errno == EAGAIN) ? EBUSY : op_errno;
-
-                goto done;
-        }
+    dht_local_t *local = NULL;
+    char src_gfid[GF_UUID_BUF_SIZE] = {0};
+    char dst_gfid[GF_UUID_BUF_SIZE] = {0};
+    int ret = 0;
+    loc_t *loc = NULL;
+    xlator_t *subvol = NULL;
+
+    local = frame->local;
+
+    if (op_ret < 0) {
+        uuid_utoa_r(local->loc.inode->gfid, src_gfid);
+
+        if (local->loc2.inode)
+            uuid_utoa_r(local->loc2.inode->gfid, dst_gfid);
+
+        gf_msg(this->name, GF_LOG_WARNING, op_errno, DHT_MSG_INODE_LK_ERROR,
+               "protecting namespace of %s failed"
+               "rename (%s:%s:%s %s:%s:%s)",
+               local->current == &local->lock[0] ? local->loc.path
+                                                 : local->loc2.path,
+               local->loc.path, src_gfid, local->src_hashed->name,
+               local->loc2.path, dst_gfid,
+               local->dst_hashed ? local->dst_hashed->name : NULL);
+
+        local->op_ret = -1;
+        local->op_errno = op_errno;
+        goto err;
+    }
+
+    if (local->current == &local->lock[0]) {
+        loc = &local->loc2;
+        subvol = local->dst_hashed;
+        local->current = &local->lock[1];
+    } else {
+        loc = &local->loc;
+        subvol = local->src_hashed;
+        local->current = &local->lock[0];
+    }
+
+    ret = dht_protect_namespace(frame, loc, subvol, &local->current->ns,
+                                dht_rename_lock_cbk);
+    if (ret < 0) {
+        op_errno = EINVAL;
+        goto err;
+    }
+
+    return 0;
+err:
+    /* No harm in calling an extra unlock */
+    dht_rename_unlock(frame, this);
+    return 0;
+}
 
-        xattr_req = dict_new ();
-        if (xattr_req == NULL) {
-                local->op_ret = -1;
-                local->op_errno = ENOMEM;
-                goto done;
-        }
+int32_t
+dht_rename_file_protect_namespace(call_frame_t *frame, void *cookie,
+                                  xlator_t *this, int32_t op_ret,
+                                  int32_t op_errno, dict_t *xdata)
+{
+    dht_local_t *local = NULL;
+    char src_gfid[GF_UUID_BUF_SIZE] = {0};
+    char dst_gfid[GF_UUID_BUF_SIZE] = {0};
+    int ret = 0;
+    loc_t *loc = NULL;
+    xlator_t *subvol = NULL;
+
+    local = frame->local;
+
+    if (op_ret < 0) {
+        uuid_utoa_r(local->loc.inode->gfid, src_gfid);
+
+        if (local->loc2.inode)
+            uuid_utoa_r(local->loc2.inode->gfid, dst_gfid);
+
+        gf_msg(this->name, GF_LOG_WARNING, op_errno, DHT_MSG_INODE_LK_ERROR,
+               "acquiring inodelk failed "
+               "rename (%s:%s:%s %s:%s:%s)",
+               local->loc.path, src_gfid, local->src_cached->name,
+               local->loc2.path, dst_gfid,
+               local->dst_cached ? local->dst_cached->name : NULL);
+
+        local->op_ret = -1;
+        local->op_errno = op_errno;
+
+        goto err;
+    }
+
+    /* Locks on src and dst needs to ordered which otherwise might cause
+     * deadlocks when rename (src, dst) and rename (dst, src) is done from
+     * two different clients
+     */
+    ret = dht_order_rename_lock(frame, &loc, &subvol);
+    if (ret) {
+        local->op_errno = ENOMEM;
+        goto err;
+    }
+
+    ret = dht_protect_namespace(frame, loc, subvol, &local->current->ns,
+                                dht_rename_file_lock1_cbk);
+    if (ret < 0) {
+        op_errno = EINVAL;
+        goto err;
+    }
+
+    return 0;
 
-        op_ret = dict_set_uint32 (xattr_req,
-                                  conf->link_xattr_name, 256);
-        if (op_ret < 0) {
-                local->op_ret = -1;
-                local->op_errno = -op_ret;
-                goto done;
-        }
+err:
+    /* Its fine to call unlock even when no locks are acquired, as we check
+     * for lock->locked before winding a unlock call.
+     */
+    dht_rename_unlock(frame, this);
 
-        local->call_cnt = local->lock.lk_count;
+    return 0;
+}
 
-        for (i = 0; i < local->lock.lk_count; i++) {
-                STACK_WIND (frame, dht_rename_lookup_cbk,
-                            local->lock.locks[i]->xl,
-                            local->lock.locks[i]->xl->fops->lookup,
-                            &local->lock.locks[i]->loc, xattr_req);
+int32_t
+dht_rename_lock_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                    int32_t op_ret, int32_t op_errno, dict_t *xdata)
+{
+    dht_local_t *local = NULL;
+    char src_gfid[GF_UUID_BUF_SIZE] = {0};
+    char dst_gfid[GF_UUID_BUF_SIZE] = {0};
+    dict_t *xattr_req = NULL;
+    dht_conf_t *conf = NULL;
+    int i = 0;
+    xlator_t *subvol = NULL;
+    dht_lock_t *lock = NULL;
+
+    local = frame->local;
+    conf = this->private;
+
+    if (op_ret < 0) {
+        uuid_utoa_r(local->loc.inode->gfid, src_gfid);
+
+        if (local->loc2.inode)
+            uuid_utoa_r(local->loc2.inode->gfid, dst_gfid);
+
+        gf_msg(this->name, GF_LOG_WARNING, op_errno, DHT_MSG_INODE_LK_ERROR,
+               "protecting namespace of %s failed. "
+               "rename (%s:%s:%s %s:%s:%s)",
+               local->current == &local->lock[0] ? local->loc.path
+                                                 : local->loc2.path,
+               local->loc.path, src_gfid, local->src_hashed->name,
+               local->loc2.path, dst_gfid,
+               local->dst_hashed ? local->dst_hashed->name : NULL);
+
+        local->op_ret = -1;
+        local->op_errno = op_errno;
+
+        goto done;
+    }
+
+    xattr_req = dict_new();
+    if (xattr_req == NULL) {
+        local->op_ret = -1;
+        local->op_errno = ENOMEM;
+        goto done;
+    }
+
+    op_ret = dict_set_uint32(xattr_req, conf->link_xattr_name, 256);
+    if (op_ret < 0) {
+        local->op_ret = -1;
+        local->op_errno = -op_ret;
+        goto done;
+    }
+
+    /* dst_cached might've changed. This normally happens for two reasons:
+     * 1. rebalance migrated dst
+     * 2. Another parallel rename was done overwriting dst
+     *
+     * Doing a lookup on local->loc2 when dst exists, but is associated
+     * with a different gfid will result in an ESTALE error. So, do a fresh
+     * lookup with a new inode on dst-path and handle change of dst-cached
+     * in the cbk. Also, to identify dst-cached changes we do a lookup on
+     * "this" rather than the subvol.
+     */
+    loc_copy(&local->loc2_copy, &local->loc2);
+    inode_unref(local->loc2_copy.inode);
+    local->loc2_copy.inode = inode_new(local->loc.inode->table);
+
+    /* Why not use local->lock.locks[?].loc for lookup post lock phase
+     * ---------------------------------------------------------------
+     * "layout.parent_layout.locks[?].loc" does not have the name and pargfid
+     * populated.
+     * Reason: If we had populated the name and pargfid, server might
+     * resolve to a successful lookup even if there is a file with same name
+     * with a different gfid(unlink & create) as server does name based
+     * resolution on first priority. And this can result in operating on a
+     * different inode entirely.
+     *
+     * Now consider a scenario where source file was renamed by some other
+     * client to a new name just before this lock was granted. So if a
+     * lookup would be done on local->lock[0].layout.parent_layout.locks[?].loc,
+     * server will send success even if the entry was renamed (since server will
+     * do a gfid based resolution). So once a lock is granted, make sure the
+     * file exists with the name that the client requested with.
+     * */
+
+    local->call_cnt = 2;
+    for (i = 0; i < 2; i++) {
+        if (i == 0) {
+            lock = local->rename_inodelk_backward_compatible[0];
+            if (gf_uuid_compare(local->loc.gfid, lock->loc.gfid) == 0)
+                subvol = lock->xl;
+            else {
+                lock = local->rename_inodelk_backward_compatible[1];
+                subvol = lock->xl;
+            }
+        } else {
+            subvol = this;
         }
 
-        dict_unref (xattr_req);
-        return 0;
+        STACK_WIND_COOKIE(frame, dht_rename_lookup_cbk, (void *)(long)i, subvol,
+                          subvol->fops->lookup,
+                          (i == 0) ? &local->loc : &local->loc2_copy,
+                          xattr_req);
+    }
+
+    dict_unref(xattr_req);
+    return 0;
 
 done:
-        /* Its fine to call unlock even when no locks are acquired, as we check
-         * for lock->locked before winding a unlock call.
-         */
-        dht_rename_unlock (frame, this);
+    /* Its fine to call unlock even when no locks are acquired, as we check
+     * for lock->locked before winding a unlock call.
+     */
+    dht_rename_unlock(frame, this);
 
-        if (xattr_req)
-                dict_unref (xattr_req);
+    if (xattr_req)
+        dict_unref(xattr_req);
 
-        return 0;
+    return 0;
 }
 
 int
-dht_rename_lock (call_frame_t *frame)
+dht_rename_lock(call_frame_t *frame)
 {
-        dht_local_t  *local    = NULL;
-        int           count    = 1, ret = -1;
-        dht_lock_t  **lk_array = NULL;
-
-        local = frame->local;
-
-        if (local->dst_cached)
-                count++;
-
-        lk_array = GF_CALLOC (count, sizeof (*lk_array), gf_common_mt_char);
-        if (lk_array == NULL)
-                goto err;
-
-        lk_array[0] = dht_lock_new (frame->this, local->src_cached, &local->loc,
-                                    F_WRLCK, DHT_FILE_MIGRATE_DOMAIN);
-        if (lk_array[0] == NULL)
-                goto err;
-
-        if (local->dst_cached) {
-                lk_array[1] = dht_lock_new (frame->this, local->dst_cached,
-                                            &local->loc2, F_WRLCK,
-                                            DHT_FILE_MIGRATE_DOMAIN);
-                if (lk_array[1] == NULL)
-                        goto err;
-        }
-
-        local->lock.locks = lk_array;
-        local->lock.lk_count = count;
-
-        ret = dht_nonblocking_inodelk (frame, lk_array, count,
-                                       dht_rename_lock_cbk);
-        if (ret < 0) {
-                local->lock.locks = NULL;
-                local->lock.lk_count = 0;
-                goto err;
-        }
-
-        return 0;
+    dht_local_t *local = NULL;
+    int count = 1, ret = -1;
+    dht_lock_t **lk_array = NULL;
+
+    local = frame->local;
+
+    if (local->dst_cached)
+        count++;
+
+    lk_array = GF_CALLOC(count, sizeof(*lk_array), gf_common_mt_pointer);
+    if (lk_array == NULL)
+        goto err;
+
+    lk_array[0] = dht_lock_new(frame->this, local->src_cached, &local->loc,
+                               F_WRLCK, DHT_FILE_MIGRATE_DOMAIN, NULL,
+                               FAIL_ON_ANY_ERROR);
+    if (lk_array[0] == NULL)
+        goto err;
+
+    if (local->dst_cached) {
+        /* dst might be removed by the time inodelk reaches bricks,
+         * which can result in ESTALE errors. POSIX imposes no
+         * restriction for dst to be present for renames to be
+         * successful. So, we'll ignore ESTALE errors. As far as
+         * synchronization on dst goes, we'll achieve the same by
+         * holding entrylk on parent directory of dst in the namespace
+         * of basename(dst). Also, there might not be quorum in cluster
+         * xlators like EC/disperse on errno, in which case they return
+         * EIO. For eg., in a disperse (4 + 2), 3 might return success
+         * and three might return ESTALE. Disperse, having no Quorum
+         * unwinds inodelk with EIO. So, ignore EIO too.
+         */
+        lk_array[1] = dht_lock_new(frame->this, local->dst_cached, &local->loc2,
+                                   F_WRLCK, DHT_FILE_MIGRATE_DOMAIN, NULL,
+                                   IGNORE_ENOENT_ESTALE_EIO);
+        if (lk_array[1] == NULL)
+            goto err;
+    }
+
+    local->rename_inodelk_backward_compatible = lk_array;
+    local->rename_inodelk_bc_count = count;
+
+    /* retaining inodelks for the sake of backward compatibility. Please
+     * make sure to remove this inodelk once all of 3.10, 3.12 and 3.13
+     * reach EOL. Better way of getting synchronization would be to acquire
+     * entrylks on src and dst parent directories in the namespace of
+     * basenames of src and dst
+     */
+    ret = dht_blocking_inodelk(frame, lk_array, count,
+                               dht_rename_file_protect_namespace);
+    if (ret < 0) {
+        local->rename_inodelk_backward_compatible = NULL;
+        local->rename_inodelk_bc_count = 0;
+        goto err;
+    }
+
+    return 0;
 err:
-        if (lk_array != NULL) {
-                int tmp_count = 0, i = 0;
+    if (lk_array != NULL) {
+        int tmp_count = 0, i = 0;
 
-                for (i = 0; (i < count) && (lk_array[i]); i++, tmp_count++);
+        for (i = 0; (i < count) && (lk_array[i]); i++, tmp_count++)
+            ;
 
-                dht_lock_array_free (lk_array, tmp_count);
-                GF_FREE (lk_array);
-        }
+        dht_lock_array_free(lk_array, tmp_count);
+        GF_FREE(lk_array);
+    }
 
-        return -1;
+    return -1;
 }
 
 int
-dht_rename (call_frame_t *frame, xlator_t *this,
-            loc_t *oldloc, loc_t *newloc, dict_t *xdata)
+dht_rename(call_frame_t *frame, xlator_t *this, loc_t *oldloc, loc_t *newloc,
+           dict_t *xdata)
 {
-        xlator_t    *src_cached = NULL;
-        xlator_t    *src_hashed = NULL;
-        xlator_t    *dst_cached = NULL;
-        xlator_t    *dst_hashed = NULL;
-        int          op_errno = -1;
-        int          ret = -1;
-        dht_local_t *local = NULL;
-        char         gfid[GF_UUID_BUF_SIZE] = {0};
-
-        VALIDATE_OR_GOTO (frame, err);
-        VALIDATE_OR_GOTO (this, err);
-        VALIDATE_OR_GOTO (oldloc, err);
-        VALIDATE_OR_GOTO (newloc, err);
-
-        gf_uuid_unparse(oldloc->inode->gfid, gfid);
-
-        src_hashed = dht_subvol_get_hashed (this, oldloc);
-        if (!src_hashed) {
-                gf_msg (this->name, GF_LOG_INFO, 0,
-                        DHT_MSG_RENAME_FAILED,
-                        "No hashed subvolume in layout for path=%s,"
-                        "(gfid = %s)", oldloc->path, gfid);
-                op_errno = EINVAL;
-                goto err;
-        }
-
-        src_cached = dht_subvol_get_cached (this, oldloc->inode);
-        if (!src_cached) {
-                gf_msg (this->name, GF_LOG_INFO, 0,
-                        DHT_MSG_RENAME_FAILED,
-                        "No cached subvolume for path = %s,"
-                        "(gfid = %s)", oldloc->path, gfid);
-
-                op_errno = EINVAL;
-                goto err;
-        }
-
-        dst_hashed = dht_subvol_get_hashed (this, newloc);
-        if (!dst_hashed) {
-                gf_msg (this->name, GF_LOG_INFO, 0,
-                        DHT_MSG_RENAME_FAILED,
-                        "No hashed subvolume in layout for path=%s",
-                        newloc->path);
-                op_errno = EINVAL;
-                goto err;
+    xlator_t *src_cached = NULL;
+    xlator_t *src_hashed = NULL;
+    xlator_t *dst_cached = NULL;
+    xlator_t *dst_hashed = NULL;
+    int op_errno = -1;
+    int ret = -1;
+    dht_local_t *local = NULL;
+    char gfid[GF_UUID_BUF_SIZE] = {0};
+    char newgfid[GF_UUID_BUF_SIZE] = {0};
+
+    VALIDATE_OR_GOTO(frame, err);
+    VALIDATE_OR_GOTO(this, err);
+    VALIDATE_OR_GOTO(oldloc, err);
+    VALIDATE_OR_GOTO(newloc, err);
+
+    gf_uuid_unparse(oldloc->inode->gfid, gfid);
+
+    src_hashed = dht_subvol_get_hashed(this, oldloc);
+    if (!src_hashed) {
+        gf_msg(this->name, GF_LOG_INFO, 0, DHT_MSG_RENAME_FAILED,
+               "No hashed subvolume in layout for path=%s,"
+               "(gfid = %s)",
+               oldloc->path, gfid);
+        op_errno = EINVAL;
+        goto err;
+    }
+
+    src_cached = dht_subvol_get_cached(this, oldloc->inode);
+    if (!src_cached) {
+        gf_msg(this->name, GF_LOG_INFO, 0, DHT_MSG_RENAME_FAILED,
+               "No cached subvolume for path = %s,"
+               "(gfid = %s)",
+               oldloc->path, gfid);
+
+        op_errno = EINVAL;
+        goto err;
+    }
+
+    dst_hashed = dht_subvol_get_hashed(this, newloc);
+    if (!dst_hashed) {
+        gf_msg(this->name, GF_LOG_INFO, 0, DHT_MSG_RENAME_FAILED,
+               "No hashed subvolume in layout for path=%s", newloc->path);
+        op_errno = EINVAL;
+        goto err;
+    }
+
+    if (newloc->inode)
+        dst_cached = dht_subvol_get_cached(this, newloc->inode);
+
+    local = dht_local_init(frame, oldloc, NULL, GF_FOP_RENAME);
+    if (!local) {
+        op_errno = ENOMEM;
+        goto err;
+    }
+    /* cached_subvol will be set from dht_local_init, reset it to NULL,
+       as the logic of handling rename is different  */
+    local->cached_subvol = NULL;
+
+    ret = loc_copy(&local->loc2, newloc);
+    if (ret == -1) {
+        op_errno = ENOMEM;
+        goto err;
+    }
+
+    local->src_hashed = src_hashed;
+    local->src_cached = src_cached;
+    local->dst_hashed = dst_hashed;
+    local->dst_cached = dst_cached;
+    if (xdata)
+        local->xattr_req = dict_ref(xdata);
+
+    if (newloc->inode)
+        gf_uuid_unparse(newloc->inode->gfid, newgfid);
+
+    gf_msg(this->name, GF_LOG_INFO, 0, DHT_MSG_RENAME_INFO,
+           "renaming %s (%s) (hash=%s/cache=%s) => %s (%s) "
+           "(hash=%s/cache=%s) ",
+           oldloc->path, gfid, src_hashed->name, src_cached->name, newloc->path,
+           newloc->inode ? newgfid : NULL, dst_hashed->name,
+           dst_cached ? dst_cached->name : "<nul>");
+
+    if (IA_ISDIR(oldloc->inode->ia_type)) {
+        dht_rename_dir(frame, this);
+    } else {
+        local->op_ret = 0;
+        ret = dht_rename_lock(frame);
+        if (ret < 0) {
+            op_errno = ENOMEM;
+            goto err;
         }
+    }
 
-        if (newloc->inode)
-                dst_cached = dht_subvol_get_cached (this, newloc->inode);
+    return 0;
 
-        local = dht_local_init (frame, oldloc, NULL, GF_FOP_RENAME);
-        if (!local) {
-                op_errno = ENOMEM;
-                goto err;
-        }
-        /* cached_subvol will be set from dht_local_init, reset it to NULL,
-           as the logic of handling rename is different  */
-        local->cached_subvol = NULL;
-
-        ret = loc_copy (&local->loc2, newloc);
-        if (ret == -1) {
-                op_errno = ENOMEM;
-                goto err;
-        }
+err:
+    op_errno = (op_errno == -1) ? errno : op_errno;
+    DHT_STACK_UNWIND(rename, frame, -1, op_errno, NULL, NULL, NULL, NULL, NULL,
+                     NULL);
 
-        local->src_hashed = src_hashed;
-        local->src_cached = src_cached;
-        local->dst_hashed = dst_hashed;
-        local->dst_cached = dst_cached;
+    return 0;
+}
 
-        gf_log (this->name, GF_LOG_INFO,
-                "renaming %s (hash=%s/cache=%s) => %s (hash=%s/cache=%s)",
-                oldloc->path, src_hashed->name, src_cached->name,
-                newloc->path, dst_hashed->name,
-                dst_cached ? dst_cached->name : "<nul>");
+int
+dht_pt_rename(call_frame_t *frame, xlator_t *this, loc_t *oldloc, loc_t *newloc,
+              dict_t *xdata)
+{
+    gf_boolean_t free_xdata = _gf_false;
 
-        if (IA_ISDIR (oldloc->inode->ia_type)) {
-                dht_rename_dir (frame, this);
-        } else {
-                local->op_ret = 0;
-                ret = dht_rename_lock (frame);
-                if (ret < 0)
-                        goto err;
+    /* Just a pass through */
+    if (!IA_ISDIR(oldloc->inode->ia_type)) {
+        if (!xdata) {
+            free_xdata = _gf_true;
         }
-
-        return 0;
-
-err:
-        op_errno = (op_errno == -1) ? errno : op_errno;
-        DHT_STACK_UNWIND (rename, frame, -1, op_errno, NULL, NULL, NULL, NULL,
-                          NULL, NULL);
-
-        return 0;
+        DHT_CHANGELOG_TRACK_AS_RENAME(xdata, oldloc, newloc);
+    }
+    default_rename(frame, this, oldloc, newloc, xdata);
+    if (free_xdata && xdata) {
+        dict_unref(xdata);
+        xdata = NULL;
+    }
+    return 0;
 }
diff --git a/xlators/cluster/dht/src/dht-selfheal.c b/xlators/cluster/dht/src/dht-selfheal.c
index 260c50083df..3e24065227c 100644
--- a/xlators/cluster/dht/src/dht-selfheal.c
+++ b/xlators/cluster/dht/src/dht-selfheal.c
@@ -8,1929 +8,2593 @@
   cases as published by the Free Software Foundation.
 */
 
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
-
-
-#include "glusterfs.h"
-#include "xlator.h"
-#include "dht-common.h"
-#include "dht-messages.h"
-#include "glusterfs-acl.h"
-
-#define DHT_SET_LAYOUT_RANGE(layout,i,srt,chunk,path)    do {           \
-                layout->list[i].start = srt;                            \
-                layout->list[i].stop  = srt + chunk - 1;                \
-                                                                        \
-                gf_msg_trace (this->name, 0,                            \
-                              "gave fix: %u - %u on %s for %s",         \
-                              layout->list[i].start,                    \
-                              layout->list[i].stop,                     \
-                              layout->list[i].xlator->name, path);      \
-        } while (0)
-
-#define DHT_RESET_LAYOUT_RANGE(layout)    do {                          \
-                int cnt = 0;                                            \
-                for (cnt = 0; cnt < layout->cnt; cnt++ ) {              \
-                        layout->list[cnt].start = 0;                    \
-                        layout->list[cnt].stop  = 0;                    \
-                }                                                       \
-        } while (0)
+#include "dht-lock.h"
+
+#define DHT_SET_LAYOUT_RANGE(layout, i, srt, chunk, path)                      \
+    do {                                                                       \
+        layout->list[i].start = srt;                                           \
+        layout->list[i].stop = srt + chunk - 1;                                \
+        layout->list[i].commit_hash = layout->commit_hash;                     \
+                                                                               \
+        gf_msg_trace(this->name, 0,                                            \
+                     "gave fix: 0x%x - 0x%x, with commit-hash 0x%x"            \
+                     " on %s for %s",                                          \
+                     layout->list[i].start, layout->list[i].stop,              \
+                     layout->list[i].commit_hash,                              \
+                     layout->list[i].xlator->name, path);                      \
+    } while (0)
+
+#define DHT_RESET_LAYOUT_RANGE(layout)                                         \
+    do {                                                                       \
+        int cnt = 0;                                                           \
+        for (cnt = 0; cnt < layout->cnt; cnt++) {                              \
+            layout->list[cnt].start = 0;                                       \
+            layout->list[cnt].stop = 0;                                        \
+        }                                                                      \
+    } while (0)
 
-int
-dht_selfheal_layout_lock (call_frame_t *frame, dht_layout_t *layout,
-                          gf_boolean_t newdir,
-                          dht_selfheal_layout_t healer,
-                          dht_need_heal_t should_heal);
+static int
+dht_selfheal_layout_lock(call_frame_t *frame, dht_layout_t *layout,
+                         gf_boolean_t newdir, dht_selfheal_layout_t healer,
+                         dht_need_heal_t should_heal);
 
 static uint32_t
-dht_overlap_calc (dht_layout_t *old, int o, dht_layout_t *new, int n)
+dht_overlap_calc(dht_layout_t *old, int o, dht_layout_t *new, int n)
 {
-	if (o >= old->cnt || n >= new->cnt)
-		return 0;
+    if (o >= old->cnt || n >= new->cnt)
+        return 0;
 
-	if (old->list[o].err > 0 || new->list[n].err > 0)
-		return 0;
+    if (old->list[o].err > 0 || new->list[n].err > 0)
+        return 0;
 
-        if (old->list[o].start == old->list[o].stop) {
-                return 0;
-        }
+    if (old->list[o].start == old->list[o].stop) {
+        return 0;
+    }
 
-        if (new->list[n].start == new->list[n].stop) {
-                return 0;
-        }
+    if (new->list[n].start == new->list[n].stop) {
+        return 0;
+    }
 
-	if ((old->list[o].start > new->list[n].stop) ||
-	    (old->list[o].stop < new->list[n].start))
-		return 0;
+    if ((old->list[o].start > new->list[n].stop) ||
+        (old->list[o].stop < new->list[n].start))
+        return 0;
 
-	return min (old->list[o].stop, new->list[n].stop) -
-	        max (old->list[o].start, new->list[n].start) + 1;
+    return min(old->list[o].stop, new->list[n].stop) -
+           max(old->list[o].start, new->list[n].start) + 1;
 }
 
 int
-dht_selfheal_unlock_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                         int32_t op_ret, int32_t op_errno, dict_t *xdata)
+dht_selfheal_unlock_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                        int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
-        DHT_STACK_DESTROY (frame);
-        return 0;
+    DHT_STACK_DESTROY(frame);
+    return 0;
 }
 
 int
-dht_selfheal_dir_finish (call_frame_t *frame, xlator_t *this, int ret)
+dht_selfheal_dir_finish(call_frame_t *frame, xlator_t *this, int ret,
+                        int invoke_cbk)
 {
-        dht_local_t  *local      = NULL, *lock_local = NULL;
-        call_frame_t *lock_frame = NULL;
-        int           lock_count = 0;
+    dht_local_t *local = NULL, *lock_local = NULL;
+    call_frame_t *lock_frame = NULL;
+    int lock_count = 0;
 
-        local = frame->local;
-        lock_count = dht_lock_count (local->lock.locks, local->lock.lk_count);
+    local = frame->local;
 
-        if (lock_count == 0)
-                goto done;
+    /* Unlock entrylk */
+    dht_unlock_entrylk_wrapper(frame, &local->lock[0].ns.directory_ns);
 
-        lock_frame = copy_frame (frame);
-        if (lock_frame == NULL) {
-                goto done;
-        }
+    /* Unlock inodelk */
+    lock_count = dht_lock_count(local->lock[0].ns.parent_layout.locks,
+                                local->lock[0].ns.parent_layout.lk_count);
+    if (lock_count == 0)
+        goto done;
 
-        lock_local = dht_local_init (lock_frame, &local->loc, NULL,
-                                     lock_frame->root->op);
-        if (lock_local == NULL) {
-                goto done;
-        }
+    lock_frame = copy_frame(frame);
+    if (lock_frame == NULL) {
+        goto done;
+    }
 
-        lock_local->lock.locks = local->lock.locks;
-        lock_local->lock.lk_count = local->lock.lk_count;
+    lock_local = dht_local_init(lock_frame, &local->loc, NULL,
+                                lock_frame->root->op);
+    if (lock_local == NULL) {
+        goto done;
+    }
 
-        local->lock.locks = NULL;
-        local->lock.lk_count = 0;
+    lock_local->lock[0].ns.parent_layout.locks = local->lock[0]
+                                                     .ns.parent_layout.locks;
+    lock_local->lock[0]
+        .ns.parent_layout.lk_count = local->lock[0].ns.parent_layout.lk_count;
 
-        dht_unlock_inodelk (lock_frame, lock_local->lock.locks,
-                            lock_local->lock.lk_count,
-                            dht_selfheal_unlock_cbk);
-        lock_frame = NULL;
+    local->lock[0].ns.parent_layout.locks = NULL;
+    local->lock[0].ns.parent_layout.lk_count = 0;
 
-done:
-        local->selfheal.dir_cbk (frame, NULL, frame->this, ret,
-                                 local->op_errno, NULL);
-        if (lock_frame != NULL) {
-                DHT_STACK_DESTROY (lock_frame);
-        }
+    dht_unlock_inodelk(lock_frame, lock_local->lock[0].ns.parent_layout.locks,
+                       lock_local->lock[0].ns.parent_layout.lk_count,
+                       dht_selfheal_unlock_cbk);
+    lock_frame = NULL;
 
-        return 0;
+done:
+    if (invoke_cbk)
+        local->selfheal.dir_cbk(frame, NULL, frame->this, ret, local->op_errno,
+                                NULL);
+    if (lock_frame != NULL) {
+        DHT_STACK_DESTROY(lock_frame);
+    }
+
+    return 0;
 }
 
 int
-dht_refresh_layout_done (call_frame_t *frame)
+dht_refresh_layout_done(call_frame_t *frame)
 {
-        int                    ret         = -1;
-        dht_layout_t          *refreshed   = NULL, *heal = NULL;
-        dht_local_t           *local       = NULL;
-        dht_need_heal_t        should_heal = NULL;
-        dht_selfheal_layout_t  healer      = NULL;
+    int ret = -1;
+    dht_layout_t *refreshed = NULL, *heal = NULL;
+    dht_local_t *local = NULL;
+    dht_need_heal_t should_heal = NULL;
+    dht_selfheal_layout_t healer = NULL;
 
-        local = frame->local;
+    local = frame->local;
 
-        refreshed = local->selfheal.refreshed_layout;
-        heal = local->selfheal.layout;
+    refreshed = local->selfheal.refreshed_layout;
+    heal = local->selfheal.layout;
 
-        healer = local->selfheal.healer;
-        should_heal = local->selfheal.should_heal;
+    healer = local->selfheal.healer;
+    should_heal = local->selfheal.should_heal;
 
-        ret = dht_layout_sort (refreshed);
-        if (ret == -1) {
-                gf_log (frame->this->name, GF_LOG_WARNING,
-                        "sorting the layout failed");
-                goto err;
-        }
+    ret = dht_layout_sort(refreshed);
+    if (ret == -1) {
+        gf_smsg(frame->this->name, GF_LOG_WARNING, 0,
+                DHT_MSG_LAYOUT_SORT_FAILED, NULL);
+        goto err;
+    }
 
-        if (should_heal (frame, &heal, &refreshed)) {
-                healer (frame, &local->loc, heal);
-        } else {
-                local->selfheal.layout = NULL;
-                local->selfheal.refreshed_layout = NULL;
-                local->selfheal.layout = refreshed;
+    if (should_heal(frame, &heal, &refreshed)) {
+        healer(frame, &local->loc, heal);
+    } else {
+        local->selfheal.layout = NULL;
+        local->selfheal.refreshed_layout = NULL;
+        local->selfheal.layout = refreshed;
 
-                dht_layout_unref (frame->this, heal);
+        dht_layout_unref(frame->this, heal);
 
-                dht_selfheal_dir_finish (frame, frame->this, 0);
-        }
+        dht_selfheal_dir_finish(frame, frame->this, 0, 1);
+    }
 
-        return 0;
+    return 0;
 
 err:
-        dht_selfheal_dir_finish (frame, frame->this, -1);
-        return 0;
+    dht_selfheal_dir_finish(frame, frame->this, -1, 1);
+    return 0;
 }
 
 int
-dht_refresh_layout_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                        int op_ret, int op_errno, inode_t *inode,
-                        struct iatt *stbuf, dict_t *xattr,
-                        struct iatt *postparent)
+dht_refresh_layout_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                       int op_ret, int op_errno, inode_t *inode,
+                       struct iatt *stbuf, dict_t *xattr,
+                       struct iatt *postparent)
 {
-        dht_local_t  *local         = NULL;
-        int           this_call_cnt = 0;
-        call_frame_t *prev          = NULL;
-        dht_layout_t *layout        = NULL;
-
-        GF_VALIDATE_OR_GOTO ("dht", frame, err);
-        GF_VALIDATE_OR_GOTO ("dht", this, err);
-        GF_VALIDATE_OR_GOTO ("dht", frame->local, err);
-        GF_VALIDATE_OR_GOTO ("dht", this->private, err);
-
-        local = frame->local;
-        prev  = cookie;
-
-        layout = local->selfheal.refreshed_layout;
-
-        LOCK (&frame->lock);
-        {
-                op_ret = dht_layout_merge (this, layout, prev->this,
-                                           op_ret, op_errno, xattr);
-
-                if (op_ret == -1) {
-                        local->op_errno = op_errno;
-                        gf_msg_debug (this->name, 0,
-                                      "lookup of %s on %s returned error (%s)",
-                                      local->loc.path, prev->this->name,
-                                      strerror (op_errno));
-
-                        goto unlock;
-                }
+    dht_local_t *local = NULL;
+    int this_call_cnt = 0;
+    xlator_t *prev = NULL;
+    dht_layout_t *layout = NULL;
+    char gfid[GF_UUID_BUF_SIZE] = {
+        0,
+    };
+
+    GF_VALIDATE_OR_GOTO("dht", frame, err);
+    GF_VALIDATE_OR_GOTO("dht", this, err);
+    GF_VALIDATE_OR_GOTO("dht", frame->local, err);
+    GF_VALIDATE_OR_GOTO("dht", this->private, err);
+
+    local = frame->local;
+    prev = cookie;
+
+    layout = local->selfheal.refreshed_layout;
+
+    LOCK(&frame->lock);
+    {
+        op_ret = dht_layout_merge(this, layout, prev, op_ret, op_errno, xattr);
 
-                local->op_ret = 0;
+        dht_iatt_merge(this, &local->stbuf, stbuf);
+
+        if (op_ret == -1) {
+            gf_uuid_unparse(local->loc.gfid, gfid);
+            local->op_errno = op_errno;
+            gf_smsg(this->name, GF_LOG_ERROR, op_errno,
+                    DHT_MSG_FILE_LOOKUP_FAILED, "path=%s", local->loc.path,
+                    "name=%s", prev->name, "gfid=%s", gfid, NULL);
+
+            goto unlock;
         }
-unlock:
-        UNLOCK (&frame->lock);
 
-        this_call_cnt = dht_frame_return (frame);
+        local->op_ret = 0;
+    }
+unlock:
+    UNLOCK(&frame->lock);
 
-        if (is_last_call (this_call_cnt)) {
-                if (local->op_ret == 0) {
-                        dht_refresh_layout_done (frame);
-                } else {
-                        goto err;
-                }
+    this_call_cnt = dht_frame_return(frame);
 
+    if (is_last_call(this_call_cnt)) {
+        if (local->op_ret == 0) {
+            local->refresh_layout_done(frame);
+        } else {
+            goto err;
         }
+    }
 
-        return 0;
+    return 0;
 
 err:
-        dht_selfheal_dir_finish (frame, this, -1);
-        return 0;
+    if (local) {
+        local->refresh_layout_unlock(frame, this, -1, 1);
+    }
+    return 0;
 }
 
 int
-dht_refresh_layout (call_frame_t *frame)
+dht_refresh_layout(call_frame_t *frame)
 {
-        int          call_cnt = 0;
-        int          i        = 0, ret = -1;
-        dht_conf_t  *conf     = NULL;
-        dht_local_t *local    = NULL;
-        xlator_t    *this     = NULL;
-
-        GF_VALIDATE_OR_GOTO ("dht", frame, out);
-        GF_VALIDATE_OR_GOTO ("dht", frame->local, out);
-
-        this = frame->this;
-        conf = this->private;
-        local = frame->local;
-
-        call_cnt = conf->subvolume_cnt;
-        local->call_cnt = call_cnt;
-        local->op_ret = -1;
-
-        if (local->selfheal.refreshed_layout) {
-                dht_layout_unref (this, local->selfheal.refreshed_layout);
-                local->selfheal.refreshed_layout = NULL;
-        }
-
-        local->selfheal.refreshed_layout = dht_layout_new (this,
-                                                           conf->subvolume_cnt);
-        if (!local->selfheal.refreshed_layout) {
-                goto out;
-        }
-
-        if (local->xattr != NULL) {
-                dict_del (local->xattr, conf->xattr_name);
-        }
-
+    int call_cnt = 0;
+    int i = 0, ret = -1;
+    dht_conf_t *conf = NULL;
+    dht_local_t *local = NULL;
+    xlator_t *this = NULL;
+    char gfid[GF_UUID_BUF_SIZE] = {
+        0,
+    };
+
+    GF_VALIDATE_OR_GOTO("dht", frame, out);
+    GF_VALIDATE_OR_GOTO("dht", frame->local, out);
+
+    this = frame->this;
+    conf = this->private;
+    local = frame->local;
+
+    call_cnt = conf->subvolume_cnt;
+    local->call_cnt = call_cnt;
+    local->op_ret = -1;
+
+    if (local->selfheal.refreshed_layout) {
+        dht_layout_unref(this, local->selfheal.refreshed_layout);
+        local->selfheal.refreshed_layout = NULL;
+    }
+
+    local->selfheal.refreshed_layout = dht_layout_new(this,
+                                                      conf->subvolume_cnt);
+    if (!local->selfheal.refreshed_layout) {
+        gf_uuid_unparse(local->loc.gfid, gfid);
+        gf_smsg(this->name, GF_LOG_ERROR, ENOMEM, DHT_MSG_MEM_ALLOC_FAILED,
+                "path=%s", local->loc.path, "gfid=%s", gfid, NULL);
+        goto out;
+    }
+
+    if (local->xattr != NULL) {
+        dict_del(local->xattr, conf->xattr_name);
+    }
+
+    if (local->xattr_req == NULL) {
+        gf_uuid_unparse(local->loc.gfid, gfid);
+        local->xattr_req = dict_new();
         if (local->xattr_req == NULL) {
-                local->xattr_req = dict_new ();
-                if (local->xattr_req == NULL) {
-                        goto out;
-                }
+            gf_smsg(this->name, GF_LOG_ERROR, ENOMEM, DHT_MSG_NO_MEMORY,
+                    "path=%s", local->loc.path, "gfid=%s", gfid, NULL);
+            goto out;
         }
+    }
 
-        if (dict_get (local->xattr_req, conf->xattr_name) == 0) {
-                ret = dict_set_uint32 (local->xattr_req, conf->xattr_name,
-                                       4 * 4);
-                if (ret)
-                        gf_msg (this->name, GF_LOG_WARNING, 0,
-                                DHT_MSG_DICT_SET_FAILED,
-                                "%s: Failed to set dictionary value:key = %s",
-                                local->loc.path, conf->xattr_name);
-        }
+    if (dict_get(local->xattr_req, conf->xattr_name) == 0) {
+        ret = dict_set_uint32(local->xattr_req, conf->xattr_name, 4 * 4);
+        if (ret)
+            gf_smsg(this->name, GF_LOG_WARNING, 0, DHT_MSG_DICT_SET_FAILED,
+                    "path=%s", local->loc.path, "key=%s", conf->xattr_name,
+                    NULL);
+    }
 
-        for (i = 0; i < call_cnt; i++) {
-                STACK_WIND (frame, dht_refresh_layout_cbk,
-                            conf->subvolumes[i],
-                            conf->subvolumes[i]->fops->lookup,
-                            &local->loc, local->xattr_req);
-        }
+    for (i = 0; i < call_cnt; i++) {
+        STACK_WIND_COOKIE(frame, dht_refresh_layout_cbk, conf->subvolumes[i],
+                          conf->subvolumes[i],
+                          conf->subvolumes[i]->fops->lookup, &local->loc,
+                          local->xattr_req);
+    }
 
-        return 0;
+    return 0;
 
 out:
-        dht_selfheal_dir_finish (frame, this, -1);
-        return 0;
+    if (local) {
+        local->refresh_layout_unlock(frame, this, -1, 1);
+    }
+    return 0;
 }
 
-
 int32_t
-dht_selfheal_layout_lock_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                              int32_t op_ret, int32_t op_errno, dict_t *xdata)
+dht_selfheal_layout_lock_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                             int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
-        if (op_ret < 0) {
-                goto err;
-        }
-
-        dht_refresh_layout (frame);
-        return 0;
-
-err:
-        dht_selfheal_dir_finish (frame, this, -1);
-        return 0;
-}
+    dht_local_t *local = NULL;
 
+    local = frame->local;
 
-gf_boolean_t
-dht_should_heal_layout (call_frame_t *frame, dht_layout_t **heal,
-                        dht_layout_t **ondisk)
-{
-        gf_boolean_t  fixit = _gf_true;
-        dht_local_t  *local = NULL;
-        int           ret   = -1, heal_missing_dirs = 0;
+    if (!local) {
+        goto err;
+    }
 
-        local = frame->local;
+    if (op_ret < 0) {
+        local->op_errno = op_errno;
+        goto err;
+    }
 
-        if ((heal == NULL) || (*heal == NULL) || (ondisk == NULL)
-            || (*ondisk == NULL))
-                goto out;
+    local->refresh_layout_unlock = dht_selfheal_dir_finish;
+    local->refresh_layout_done = dht_refresh_layout_done;
 
-        ret = dht_layout_anomalies (frame->this, &local->loc, *ondisk,
-                                    &local->selfheal.hole_cnt,
-                                    &local->selfheal.overlaps_cnt,
-                                    NULL, &local->selfheal.down,
-                                    &local->selfheal.misc, NULL);
+    dht_refresh_layout(frame);
+    return 0;
 
-        if (ret < 0)
-                goto out;
+err:
+    dht_selfheal_dir_finish(frame, this, -1, 1);
+    return 0;
+}
 
-        /* Directories might've been created as part of this self-heal. We've to
-         * sync non-layout xattrs and set range 0-0 on new directories
+gf_boolean_t
+dht_should_heal_layout(call_frame_t *frame, dht_layout_t **heal,
+                       dht_layout_t **ondisk)
+{
+    gf_boolean_t fixit = _gf_true;
+    dht_local_t *local = NULL;
+    int heal_missing_dirs = 0;
+
+    local = frame->local;
+
+    if ((heal == NULL) || (*heal == NULL) || (ondisk == NULL) ||
+        (*ondisk == NULL))
+        goto out;
+
+    dht_layout_anomalies(
+        frame->this, &local->loc, *ondisk, &local->selfheal.hole_cnt,
+        &local->selfheal.overlaps_cnt, &local->selfheal.missing_cnt,
+        &local->selfheal.down, &local->selfheal.misc, NULL);
+
+    /* Directories might've been created as part of this self-heal. We've to
+     * sync non-layout xattrs and set range 0-0 on new directories
+     */
+    heal_missing_dirs = local->selfheal.force_mkdir
+                            ? local->selfheal.force_mkdir
+                            : dht_layout_missing_dirs(*heal);
+
+    if ((local->selfheal.hole_cnt == 0) &&
+        (local->selfheal.overlaps_cnt == 0) && heal_missing_dirs) {
+        dht_layout_t *tmp = NULL;
+
+        /* Just added a brick and need to set 0-0 range on this brick.
+         * But ondisk layout is well-formed. So, swap layouts "heal" and
+         * "ondisk". Now "ondisk" layout will be used for healing
+         * xattrs. If there are any non-participating subvols in
+         * "ondisk" layout, dht_selfheal_dir_xattr_persubvol will set
+         * 0-0 and non-layout xattrs. This way we won't end up in
+         * "corrupting" already set and well-formed "ondisk" layout.
          */
-        heal_missing_dirs = local->selfheal.force_mkdir
-                ? local->selfheal.force_mkdir : dht_layout_missing_dirs (*heal);
-
-        if ((local->selfheal.hole_cnt == 0)
-            && (local->selfheal.overlaps_cnt == 0) && heal_missing_dirs) {
-                dht_layout_t *tmp = NULL;
-
-                /* Just added a brick and need to set 0-0 range on this brick.
-                 * But ondisk layout is well-formed. So, swap layouts "heal" and
-                 * "ondisk". Now "ondisk" layout will be used for healing
-                 * xattrs. If there are any non-participating subvols in
-                 * "ondisk" layout, dht_selfheal_dir_xattr_persubvol will set
-                 * 0-0 and non-layout xattrs. This way we won't end up in
-                 * "corrupting" already set and well-formed "ondisk" layout.
-                 */
-                tmp = *heal;
-                *heal = *ondisk;
-                *ondisk = tmp;
-
-                /* Current selfheal code, heals non-layout xattrs only after
-                 * an add-brick. In fact non-layout xattrs are considered as
-                 * secondary citizens which are healed only if layout xattrs
-                 * need to be healed. This is wrong, since for eg., quota can be
-                 * set when layout is well-formed, but a node is down. Also,
-                 * just for healing non-layout xattrs, we don't need locking.
-                 * This issue is _NOT FIXED_ by this patch.
-                 */
-        }
+        tmp = *heal;
+        *heal = *ondisk;
+        *ondisk = tmp;
+
+        /* Current selfheal code, heals non-layout xattrs only after
+         * an add-brick. In fact non-layout xattrs are considered as
+         * secondary citizens which are healed only if layout xattrs
+         * need to be healed. This is wrong, since for eg., quota can be
+         * set when layout is well-formed, but a node is down. Also,
+         * just for healing non-layout xattrs, we don't need locking.
+         * This issue is _NOT FIXED_ by this patch.
+         */
+    }
 
-        fixit = (local->selfheal.hole_cnt || local->selfheal.overlaps_cnt
-                 || heal_missing_dirs);
+    fixit = (local->selfheal.hole_cnt || local->selfheal.overlaps_cnt ||
+             heal_missing_dirs);
 
 out:
-        return fixit;
+    return fixit;
 }
 
-inline int
-dht_layout_span (dht_layout_t *layout)
+int
+dht_layout_span(dht_layout_t *layout)
 {
-        int i = 0, count = 0;
+    int i = 0, count = 0;
 
-        for (i = 0; i < layout->cnt; i++) {
-                if (layout->list[i].err)
-                        continue;
+    for (i = 0; i < layout->cnt; i++) {
+        if (layout->list[i].err)
+            continue;
 
-                if (layout->list[i].start != layout->list[i].stop)
-                        count++;
-        }
+        if (layout->list[i].start != layout->list[i].stop)
+            count++;
+    }
 
-        return count;
+    return count;
 }
 
 int
-dht_decommissioned_bricks_in_layout (xlator_t *this, dht_layout_t *layout)
+dht_decommissioned_bricks_in_layout(xlator_t *this, dht_layout_t *layout)
 {
-        dht_conf_t *conf  = NULL;
-        int         count = 0, i = 0, j = 0;
+    dht_conf_t *conf = NULL;
+    int count = 0, i = 0, j = 0;
 
-        if ((this == NULL) || (layout == NULL))
-                goto out;
+    if ((this == NULL) || (layout == NULL))
+        goto out;
 
-        conf = this->private;
+    conf = this->private;
 
-        for (i = 0; i < layout->cnt; i++) {
-                for (j = 0; j < conf->subvolume_cnt; j++) {
-                        if (conf->decommissioned_bricks[j] &&
-                            conf->decommissioned_bricks[j]
-                            == layout->list[i].xlator) {
-                                count++;
-                        }
-                }
+    for (i = 0; i < layout->cnt; i++) {
+        for (j = 0; j < conf->subvolume_cnt; j++) {
+            if (conf->decommissioned_bricks[j] &&
+                conf->decommissioned_bricks[j] == layout->list[i].xlator) {
+                count++;
+            }
         }
+    }
 
 out:
-        return count;
+    return count;
 }
 
 dht_distribution_type_t
-dht_distribution_type (xlator_t *this, dht_layout_t *layout)
+dht_distribution_type(xlator_t *this, dht_layout_t *layout)
 {
-        dht_distribution_type_t type        = GF_DHT_EQUAL_DISTRIBUTION;
-        int                     i           = 0;
-        uint32_t                start_range = 0, range = 0, diff = 0;
+    dht_distribution_type_t type = GF_DHT_EQUAL_DISTRIBUTION;
+    int i = 0;
+    uint32_t start_range = 0, range = 0, diff = 0;
 
-        if ((this == NULL) || (layout == NULL) || (layout->cnt < 1)) {
-                goto out;
-        }
+    if ((this == NULL) || (layout == NULL) || (layout->cnt < 1)) {
+        goto out;
+    }
 
-        for (i = 0; i < layout->cnt; i++) {
-                if (start_range == 0) {
-                        start_range = layout->list[i].stop
-                                - layout->list[i].start;
-                        continue;
-                }
+    for (i = 0; i < layout->cnt; i++) {
+        if (start_range == 0) {
+            start_range = layout->list[i].stop - layout->list[i].start;
+            continue;
+        }
 
-                range = layout->list[i].stop - layout->list[i].start;
-                diff = abs (range - start_range);
+        range = layout->list[i].stop - layout->list[i].start;
+        diff = (range >= start_range) ? range - start_range
+                                      : start_range - range;
 
-                if ((range != 0) && (diff > layout->cnt)) {
-                        type = GF_DHT_WEIGHTED_DISTRIBUTION;
-                        break;
-                }
+        if ((range != 0) && (diff > layout->cnt)) {
+            type = GF_DHT_WEIGHTED_DISTRIBUTION;
+            break;
         }
+    }
 
 out:
-        return type;
+    return type;
 }
 
 gf_boolean_t
-dht_should_fix_layout (call_frame_t *frame, dht_layout_t **inmem,
-                       dht_layout_t **ondisk)
+dht_should_fix_layout(call_frame_t *frame, dht_layout_t **inmem,
+                      dht_layout_t **ondisk)
 {
-        gf_boolean_t             fixit                 = _gf_true;
-        dht_local_t             *local                 = NULL;
-        int                      layout_span           = 0;
-        int                      decommissioned_bricks = 0;
-        int                      ret                   = 0;
-        dht_conf_t              *conf                  = NULL;
-        dht_distribution_type_t  inmem_dist_type       = 0;
-        dht_distribution_type_t  ondisk_dist_type      = 0;
-
-        conf = frame->this->private;
-
-        local = frame->local;
-
-        if ((inmem == NULL) || (*inmem == NULL) || (ondisk == NULL)
-            || (*ondisk == NULL))
-                goto out;
-
-        ret = dht_layout_anomalies (frame->this, &local->loc, *ondisk,
-                                    &local->selfheal.hole_cnt,
-                                    &local->selfheal.overlaps_cnt, NULL,
-                                    &local->selfheal.down,
-                                    &local->selfheal.misc, NULL);
-        if (ret < 0) {
-                fixit = _gf_false;
-                goto out;
-        }
+    gf_boolean_t fixit = _gf_true;
 
-        if (local->selfheal.down || local->selfheal.misc) {
-                fixit = _gf_false;
-                goto out;
-        }
+    dht_local_t *local = NULL;
+    int layout_span = 0;
+    int decommissioned_bricks = 0;
+    dht_conf_t *conf = NULL;
+    dht_distribution_type_t inmem_dist_type = 0;
+    dht_distribution_type_t ondisk_dist_type = 0;
 
-        if (local->selfheal.hole_cnt || local->selfheal.overlaps_cnt)
-                goto out;
+    conf = frame->this->private;
 
-        layout_span = dht_layout_span (*ondisk);
+    local = frame->local;
 
-        decommissioned_bricks
-                = dht_decommissioned_bricks_in_layout (frame->this,
-                                                       *ondisk);
-        inmem_dist_type = dht_distribution_type (frame->this, *inmem);
-        ondisk_dist_type = dht_distribution_type (frame->this, *ondisk);
+    if ((inmem == NULL) || (*inmem == NULL) || (ondisk == NULL) ||
+        (*ondisk == NULL))
+        goto out;
 
-        if ((decommissioned_bricks == 0)
-            && (layout_span == (conf->subvolume_cnt
-                                - conf->decommission_subvols_cnt))
-            && (inmem_dist_type == ondisk_dist_type))
-                fixit = _gf_false;
+    dht_layout_anomalies(frame->this, &local->loc, *ondisk,
+                         &local->selfheal.hole_cnt,
+                         &local->selfheal.overlaps_cnt, NULL,
+                         &local->selfheal.down, &local->selfheal.misc, NULL);
 
-out:
-        return fixit;
-}
+    if (local->selfheal.down || local->selfheal.misc) {
+        fixit = _gf_false;
+        goto out;
+    }
 
-int
-dht_selfheal_layout_lock (call_frame_t *frame, dht_layout_t *layout,
-                          gf_boolean_t newdir,
-                          dht_selfheal_layout_t healer,
-                          dht_need_heal_t should_heal)
-{
-        dht_local_t   *local    = NULL;
-        int            count    = 1, ret = -1, i = 0;
-        dht_lock_t   **lk_array = NULL;
-        dht_conf_t    *conf     = NULL;
-        dht_layout_t  *tmp      = NULL;
-
-        GF_VALIDATE_OR_GOTO ("dht", frame, err);
-        GF_VALIDATE_OR_GOTO (frame->this->name, frame->local, err);
-
-        local = frame->local;
-
-        conf = frame->this->private;
+    if (local->selfheal.hole_cnt || local->selfheal.overlaps_cnt)
+        goto out;
 
-        local->selfheal.healer = healer;
-        local->selfheal.should_heal = should_heal;
+    /* If commit hashes are being updated, let it through */
+    if ((*inmem)->commit_hash != (*ondisk)->commit_hash)
+        goto out;
 
-        tmp = local->selfheal.layout;
-        local->selfheal.layout = dht_layout_ref (frame->this, layout);
-        dht_layout_unref (frame->this, tmp);
+    layout_span = dht_layout_span(*ondisk);
 
-        if (!newdir) {
-                count = conf->subvolume_cnt;
+    decommissioned_bricks = dht_decommissioned_bricks_in_layout(frame->this,
+                                                                *ondisk);
+    inmem_dist_type = dht_distribution_type(frame->this, *inmem);
+    ondisk_dist_type = dht_distribution_type(frame->this, *ondisk);
 
-                lk_array = GF_CALLOC (count, sizeof (*lk_array),
-                                      gf_common_mt_char);
-                if (lk_array == NULL)
-                        goto err;
+    if ((decommissioned_bricks == 0) &&
+        (layout_span ==
+         (conf->subvolume_cnt - conf->decommission_subvols_cnt)) &&
+        (inmem_dist_type == ondisk_dist_type))
+        fixit = _gf_false;
 
-                for (i = 0; i < count; i++) {
-                        lk_array[i] = dht_lock_new (frame->this,
-                                                    conf->subvolumes[i],
-                                                    &local->loc, F_WRLCK,
-                                                    DHT_LAYOUT_HEAL_DOMAIN);
-                        if (lk_array[i] == NULL)
-                                goto err;
-                }
-        } else {
-                count = 1;
-                lk_array = GF_CALLOC (count, sizeof (*lk_array),
-                                      gf_common_mt_char);
-                if (lk_array == NULL)
-                        goto err;
-
-                lk_array[0] = dht_lock_new (frame->this, local->hashed_subvol,
-                                            &local->loc, F_WRLCK,
-                                            DHT_LAYOUT_HEAL_DOMAIN);
-                if (lk_array[0] == NULL)
-                        goto err;
-        }
+out:
 
-        local->lock.locks = lk_array;
-        local->lock.lk_count = count;
+    return fixit;
+}
 
-        ret = dht_blocking_inodelk (frame, lk_array, count,
-                                    dht_selfheal_layout_lock_cbk);
-        if (ret < 0) {
-                local->lock.locks = NULL;
-                local->lock.lk_count = 0;
+static int
+dht_selfheal_layout_lock(call_frame_t *frame, dht_layout_t *layout,
+                         gf_boolean_t newdir, dht_selfheal_layout_t healer,
+                         dht_need_heal_t should_heal)
+{
+    dht_local_t *local = NULL;
+    int count = 1, ret = -1, i = 0;
+    dht_lock_t **lk_array = NULL;
+    dht_conf_t *conf = NULL;
+    dht_layout_t *tmp = NULL;
+    char gfid[GF_UUID_BUF_SIZE] = {0};
+
+    GF_VALIDATE_OR_GOTO("dht", frame, err);
+    GF_VALIDATE_OR_GOTO(frame->this->name, frame->local, err);
+
+    local = frame->local;
+
+    conf = frame->this->private;
+
+    local->selfheal.healer = healer;
+    local->selfheal.should_heal = should_heal;
+
+    tmp = local->selfheal.layout;
+    local->selfheal.layout = dht_layout_ref(frame->this, layout);
+    dht_layout_unref(frame->this, tmp);
+
+    if (!newdir) {
+        count = conf->subvolume_cnt;
+
+        lk_array = GF_CALLOC(count, sizeof(*lk_array), gf_common_mt_char);
+        if (lk_array == NULL) {
+            gf_uuid_unparse(local->stbuf.ia_gfid, gfid);
+            gf_smsg("dht", GF_LOG_ERROR, ENOMEM, DHT_MSG_MEM_ALLOC_FAILED,
+                    "lk_array-gfid=%s", gfid, "path=%s", local->loc.path, NULL);
+            goto err;
+        }
+
+        for (i = 0; i < count; i++) {
+            lk_array[i] = dht_lock_new(
+                frame->this, conf->subvolumes[i], &local->loc, F_WRLCK,
+                DHT_LAYOUT_HEAL_DOMAIN, NULL, FAIL_ON_ANY_ERROR);
+            if (lk_array[i] == NULL) {
+                gf_uuid_unparse(local->stbuf.ia_gfid, gfid);
+                gf_smsg(THIS->name, GF_LOG_ERROR, ENOMEM,
+                        DHT_MSG_MEM_ALLOC_FAILED, "lk_array-gfid=%s", gfid,
+                        "path=%s", local->loc.path, NULL);
                 goto err;
-        }
-
-        return 0;
+            }
+        }
+    } else {
+        count = 1;
+        lk_array = GF_CALLOC(count, sizeof(*lk_array), gf_common_mt_char);
+        if (lk_array == NULL) {
+            gf_uuid_unparse(local->stbuf.ia_gfid, gfid);
+            gf_smsg(THIS->name, GF_LOG_ERROR, ENOMEM, DHT_MSG_MEM_ALLOC_FAILED,
+                    "lk_array-gfid=%s", gfid, "path=%s", local->loc.path, NULL);
+            goto err;
+        }
+
+        lk_array[0] = dht_lock_new(frame->this, local->hashed_subvol,
+                                   &local->loc, F_WRLCK, DHT_LAYOUT_HEAL_DOMAIN,
+                                   NULL, FAIL_ON_ANY_ERROR);
+        if (lk_array[0] == NULL) {
+            gf_uuid_unparse(local->stbuf.ia_gfid, gfid);
+            gf_smsg(THIS->name, GF_LOG_ERROR, ENOMEM, DHT_MSG_MEM_ALLOC_FAILED,
+                    "lk_array-gfid=%s", gfid, "path=%s", local->loc.path, NULL);
+            goto err;
+        }
+    }
+
+    local->lock[0].layout.my_layout.locks = lk_array;
+    local->lock[0].layout.my_layout.lk_count = count;
+
+    ret = dht_blocking_inodelk(frame, lk_array, count,
+                               dht_selfheal_layout_lock_cbk);
+    if (ret < 0) {
+        local->lock[0].layout.my_layout.locks = NULL;
+        local->lock[0].layout.my_layout.lk_count = 0;
+        goto err;
+    }
+
+    return 0;
 err:
-        if (lk_array != NULL) {
-                int tmp_count = 0, i = 0;
+    if (lk_array != NULL) {
+        dht_lock_array_free(lk_array, count);
+        GF_FREE(lk_array);
+    }
 
-                for (i = 0; (i < count) && (lk_array[i]); i++, tmp_count++) {
-                        ;
-                }
-
-                dht_lock_array_free (lk_array, tmp_count);
-                GF_FREE (lk_array);
-        }
-
-        return -1;
+    return -1;
 }
 
-int
-dht_selfheal_dir_xattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                            int op_ret, int op_errno, dict_t *xdata)
+static int
+dht_selfheal_dir_xattr_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                           int op_ret, int op_errno, dict_t *xdata)
 {
-        dht_local_t  *local = NULL;
-        call_frame_t *prev = NULL;
-        xlator_t     *subvol = NULL;
-        int           i = 0;
-        dht_layout_t *layout = NULL;
-        int           err = 0;
-        int           this_call_cnt = 0;
-
-        local = frame->local;
-        layout = local->selfheal.layout;
-        prev = cookie;
-        subvol = prev->this;
-
-        if (op_ret == 0)
-                err = 0;
-        else
-                err = op_errno;
-
-        for (i = 0; i < layout->cnt; i++) {
-                if (layout->list[i].xlator == subvol) {
-                        layout->list[i].err = err;
-                        break;
-                }
-        }
-
-        this_call_cnt = dht_frame_return (frame);
-
-        if (is_last_call (this_call_cnt)) {
-                dht_selfheal_dir_finish (frame, this, 0);
-        }
-
-        return 0;
+    dht_local_t *local = NULL;
+    xlator_t *subvol = NULL;
+    struct iatt *stbuf = NULL;
+    int i = 0;
+    int ret = 0;
+    dht_layout_t *layout = NULL;
+    int err = 0;
+    int this_call_cnt = 0;
+    char gfid[GF_UUID_BUF_SIZE] = {0};
+
+    local = frame->local;
+    layout = local->selfheal.layout;
+    subvol = cookie;
+
+    if (op_ret == 0) {
+        err = 0;
+    } else {
+        gf_uuid_unparse(local->loc.gfid, gfid);
+        gf_smsg(this->name, GF_LOG_ERROR, op_errno,
+                DHT_MSG_DIR_SELFHEAL_XATTR_FAILED, "name=%s", subvol->name,
+                "path=%s", local->loc.path, "gfid=%s", gfid, NULL);
+        err = op_errno;
+    }
+
+    ret = dict_get_bin(xdata, DHT_IATT_IN_XDATA_KEY, (void **)&stbuf);
+    if (ret < 0) {
+        gf_uuid_unparse(local->loc.gfid, gfid);
+        gf_msg_debug(this->name, 0,
+                     "key = %s not present in dict"
+                     ", path:%s gfid:%s",
+                     DHT_IATT_IN_XDATA_KEY, local->loc.path, gfid);
+    }
+
+    for (i = 0; i < layout->cnt; i++) {
+        if (layout->list[i].xlator == subvol) {
+            layout->list[i].err = err;
+            break;
+        }
+    }
+
+    LOCK(&frame->lock);
+    {
+        dht_iatt_merge(this, &local->stbuf, stbuf);
+    }
+    UNLOCK(&frame->lock);
+
+    this_call_cnt = dht_frame_return(frame);
+
+    if (is_last_call(this_call_cnt)) {
+        dht_selfheal_dir_finish(frame, this, 0, 1);
+    }
+
+    return 0;
 }
 
-
+/* Code is required to set user xattr to local->xattr
+ */
 int
-dht_selfheal_dir_xattr_persubvol (call_frame_t *frame, loc_t *loc,
-                                  dht_layout_t *layout, int i,
-                                  xlator_t *req_subvol)
+dht_set_user_xattr(dict_t *dict, char *k, data_t *v, void *data)
 {
-        xlator_t          *subvol = NULL;
-        dict_t            *xattr = NULL;
-        int                ret = 0;
-        xlator_t          *this = NULL;
-        int32_t           *disk_layout = NULL;
-        dht_local_t       *local = NULL;
-        dht_conf_t        *conf = NULL;
-        data_t            *data = NULL;
-        char              gfid[GF_UUID_BUF_SIZE] = {0};
-
-        local = frame->local;
-        if (req_subvol)
-                subvol = req_subvol;
-        else
-                subvol = layout->list[i].xlator;
-        this = frame->this;
-
-        GF_VALIDATE_OR_GOTO ("", this, err);
-        GF_VALIDATE_OR_GOTO (this->name, layout, err);
-        GF_VALIDATE_OR_GOTO (this->name, local, err);
-        GF_VALIDATE_OR_GOTO (this->name, subvol, err);
-        VALIDATE_OR_GOTO (this->private, err);
-
-        conf = this->private;
-
-        xattr = get_new_dict ();
-        if (!xattr) {
-                goto err;
-        }
-
-        gf_uuid_unparse(loc->inode->gfid, gfid);
-
-        ret = dht_disk_layout_extract (this, layout, i, &disk_layout);
-        if (ret == -1) {
-                gf_msg (this->name, GF_LOG_WARNING, 0,
-                        DHT_MSG_DIR_SELFHEAL_XATTR_FAILED,
-                        "Directory self heal xattr failed:"
-                        " %s: (subvol %s) Failed to extract disk layout,"
-                        " gfid = %s", loc->path, subvol->name, gfid);
-                goto err;
-        }
-
-        ret = dict_set_bin (xattr, conf->xattr_name, disk_layout, 4 * 4);
-        if (ret == -1) {
-                gf_msg (this->name, GF_LOG_WARNING, 0,
-                        DHT_MSG_DIR_SELFHEAL_XATTR_FAILED,
-                        "Directory self heal xattr failed:"
-                        "%s: (subvol %s) Failed to set xattr dictionary,"
-                        " gfid = %s", loc->path, subvol->name, gfid);
-                goto err;
-        }
-        disk_layout = NULL;
-
-        gf_msg_trace (this->name, 0,
-                      "setting hash range %u - %u (type %d) on subvolume %s"
-                      " for %s", layout->list[i].start, layout->list[i].stop,
-                      layout->type, subvol->name, loc->path);
-
-        dict_ref (xattr);
-        if (local->xattr) {
-                data = dict_get (local->xattr, QUOTA_LIMIT_KEY);
-                if (data) {
-                        ret = dict_add (xattr, QUOTA_LIMIT_KEY, data);
-                        if (ret) {
-                                gf_msg (this->name, GF_LOG_ERROR, 0,
-                                        DHT_MSG_DICT_SET_FAILED,
-                                        "%s: Failed to set dictionary value:"
-                                        " key = %s",
-                                        loc->path, QUOTA_LIMIT_KEY);
-                        }
-                }
-        }
-
-        if (!gf_uuid_is_null (local->gfid))
-                gf_uuid_copy (loc->gfid, local->gfid);
+    dict_t *set_xattr = data;
+    int ret = -1;
 
-        STACK_WIND (frame, dht_selfheal_dir_xattr_cbk,
-                    subvol, subvol->fops->setxattr,
-                    loc, xattr, 0, NULL);
-
-        dict_unref (xattr);
+    ret = dict_set(set_xattr, k, v);
+    return ret;
+}
 
-        return 0;
+static int
+dht_selfheal_dir_xattr_persubvol(call_frame_t *frame, loc_t *loc,
+                                 dht_layout_t *layout, int i,
+                                 xlator_t *req_subvol)
+{
+    xlator_t *subvol = NULL;
+    dict_t *xattr = NULL;
+    dict_t *xdata = NULL;
+    int ret = 0;
+    xlator_t *this = NULL;
+    int32_t *disk_layout = NULL;
+    dht_local_t *local = NULL;
+    dht_conf_t *conf = NULL;
+    data_t *data = NULL;
+    char gfid[GF_UUID_BUF_SIZE] = {0};
+
+    local = frame->local;
+    if (req_subvol)
+        subvol = req_subvol;
+    else
+        subvol = layout->list[i].xlator;
+    this = frame->this;
+
+    GF_VALIDATE_OR_GOTO("", this, err);
+    GF_VALIDATE_OR_GOTO(this->name, layout, err);
+    GF_VALIDATE_OR_GOTO(this->name, local, err);
+    GF_VALIDATE_OR_GOTO(this->name, subvol, err);
+    VALIDATE_OR_GOTO(this->private, err);
+
+    conf = this->private;
+
+    xattr = dict_new();
+    if (!xattr) {
+        goto err;
+    }
+
+    xdata = dict_new();
+    if (!xdata)
+        goto err;
+
+    ret = dict_set_str(xdata, GLUSTERFS_INTERNAL_FOP_KEY, "yes");
+    if (ret < 0) {
+        gf_smsg(this->name, GF_LOG_WARNING, 0, DHT_MSG_DICT_SET_FAILED,
+                "path=%s", loc->path, "key=%s", GLUSTERFS_INTERNAL_FOP_KEY,
+                "gfid=%s", gfid, NULL);
+        goto err;
+    }
+
+    ret = dict_set_int8(xdata, DHT_IATT_IN_XDATA_KEY, 1);
+    if (ret < 0) {
+        gf_smsg(this->name, GF_LOG_WARNING, 0, DHT_MSG_DICT_SET_FAILED,
+                "path=%s", loc->path, "key=%s", DHT_IATT_IN_XDATA_KEY,
+                "gfid=%s", gfid, NULL);
+        goto err;
+    }
+
+    gf_uuid_unparse(loc->inode->gfid, gfid);
+
+    ret = dht_disk_layout_extract(this, layout, i, &disk_layout);
+    if (ret == -1) {
+        gf_smsg(this->name, GF_LOG_WARNING, 0,
+                DHT_MSG_DIR_SELFHEAL_XATTR_FAILED,
+                "extract-disk-layout-failed, path=%s", loc->path, "subvol=%s",
+                subvol->name, "gfid=%s", gfid, NULL);
+        goto err;
+    }
+
+    ret = dict_set_bin(xattr, conf->xattr_name, disk_layout, 4 * 4);
+    if (ret == -1) {
+        gf_smsg(this->name, GF_LOG_WARNING, 0,
+                DHT_MSG_DIR_SELFHEAL_XATTR_FAILED, "path=%s", loc->path,
+                "subvol=%s", subvol->name,
+                "set-xattr-dictionary-failed"
+                "gfid=%s",
+                gfid, NULL);
+        goto err;
+    }
+    disk_layout = NULL;
+
+    gf_msg_trace(this->name, 0,
+                 "setting hash range 0x%x - 0x%x (type %d) on subvolume %s"
+                 " for %s",
+                 layout->list[i].start, layout->list[i].stop, layout->type,
+                 subvol->name, loc->path);
+
+    if (local->xattr) {
+        data = dict_get(local->xattr, QUOTA_LIMIT_KEY);
+        if (data) {
+            ret = dict_add(xattr, QUOTA_LIMIT_KEY, data);
+            if (ret) {
+                gf_smsg(this->name, GF_LOG_ERROR, 0, DHT_MSG_DICT_SET_FAILED,
+                        "path=%s", loc->path, "key=%s", QUOTA_LIMIT_KEY, NULL);
+            }
+        }
+        data = dict_get(local->xattr, QUOTA_LIMIT_OBJECTS_KEY);
+        if (data) {
+            ret = dict_add(xattr, QUOTA_LIMIT_OBJECTS_KEY, data);
+            if (ret) {
+                gf_smsg(this->name, GF_LOG_ERROR, 0, DHT_MSG_DICT_SET_FAILED,
+                        "path=%s", loc->path, "key=%s", QUOTA_LIMIT_OBJECTS_KEY,
+                        NULL);
+            }
+        }
+    }
+
+    if (!gf_uuid_is_null(local->gfid))
+        gf_uuid_copy(loc->gfid, local->gfid);
+
+    STACK_WIND_COOKIE(frame, dht_selfheal_dir_xattr_cbk, (void *)subvol, subvol,
+                      subvol->fops->setxattr, loc, xattr, 0, xdata);
+
+    dict_unref(xattr);
+    dict_unref(xdata);
+
+    return 0;
 
 err:
-        if (xattr)
-                dict_destroy (xattr);
+    if (xattr)
+        dict_unref(xattr);
+    if (xdata)
+        dict_unref(xdata);
 
-        GF_FREE (disk_layout);
+    GF_FREE(disk_layout);
 
-        dht_selfheal_dir_xattr_cbk (frame, subvol, frame->this,
-                                    -1, ENOMEM, NULL);
-        return 0;
+    dht_selfheal_dir_xattr_cbk(frame, (void *)subvol, frame->this, -1, ENOMEM,
+                               NULL);
+    return 0;
 }
 
-int
-dht_fix_dir_xattr (call_frame_t *frame, loc_t *loc, dht_layout_t *layout)
+static int
+dht_fix_dir_xattr(call_frame_t *frame, loc_t *loc, dht_layout_t *layout)
 {
-        dht_local_t *local = NULL;
-        int          i = 0;
-        int          count = 0;
-        xlator_t    *this = NULL;
-        dht_conf_t  *conf = NULL;
-        dht_layout_t *dummy = NULL;
-
-        local = frame->local;
-        this = frame->this;
-        conf = this->private;
+    dht_local_t *local = NULL;
+    int i = 0;
+    int count = 0;
+    xlator_t *this = NULL;
+    dht_conf_t *conf = NULL;
+    dht_layout_t *dummy = NULL;
+
+    local = frame->local;
+    this = frame->this;
+    conf = this->private;
+
+    gf_msg_debug(this->name, 0, "%s: Writing the new range for all subvolumes",
+                 loc->path);
+
+    local->call_cnt = count = conf->subvolume_cnt;
+
+    if (gf_log_get_loglevel() >= GF_LOG_DEBUG)
+        dht_log_new_layout_for_dir_selfheal(this, loc, layout);
+
+    for (i = 0; i < layout->cnt; i++) {
+        dht_selfheal_dir_xattr_persubvol(frame, loc, layout, i, NULL);
+
+        if (--count == 0)
+            goto out;
+    }
+    /* if we are here, subvolcount > layout_count. subvols-per-directory
+     * option might be set here. We need to clear out layout from the
+     * non-participating subvolumes, else it will result in overlaps */
+    dummy = dht_layout_new(this, 1);
+    if (!dummy)
+        goto out;
+    dummy->commit_hash = layout->commit_hash;
+    for (i = 0; i < conf->subvolume_cnt; i++) {
+        if (_gf_false == dht_is_subvol_in_layout(layout, conf->subvolumes[i])) {
+            dht_selfheal_dir_xattr_persubvol(frame, loc, dummy, 0,
+                                             conf->subvolumes[i]);
+            if (--count == 0)
+                break;
+        }
+    }
+
+    dht_layout_unref(this, dummy);
+out:
+    return 0;
+}
 
-        gf_msg_debug (this->name, 0,
-                      "%s: Writing the new range for all subvolumes",
-                      loc->path);
+static int
+dht_selfheal_dir_xattr(call_frame_t *frame, loc_t *loc, dht_layout_t *layout)
+{
+    dht_local_t *local = NULL;
+    int missing_xattr = 0;
+    int i = 0;
+    xlator_t *this = NULL;
+    dht_conf_t *conf = NULL;
+    dht_layout_t *dummy = NULL;
+    char gfid[GF_UUID_BUF_SIZE] = {
+        0,
+    };
+
+    local = frame->local;
+    this = frame->this;
+    conf = this->private;
+
+    for (i = 0; i < layout->cnt; i++) {
+        if (layout->list[i].err != -1 || !layout->list[i].stop) {
+            /* err != -1 would mean xattr present on the directory
+             * or the directory is non existent.
+             * !layout->list[i].stop would mean layout absent
+             */
+
+            continue;
+        }
+        missing_xattr++;
+    }
+    /* Also account for subvolumes with no-layout. Used for zero'ing out
+     * the layouts and for setting quota key's if present */
+    for (i = 0; i < conf->subvolume_cnt; i++) {
+        if (_gf_false == dht_is_subvol_in_layout(layout, conf->subvolumes[i])) {
+            missing_xattr++;
+        }
+    }
+    gf_msg_trace(this->name, 0, "%d subvolumes missing xattr for %s",
+                 missing_xattr, loc->path);
+
+    if (missing_xattr == 0) {
+        dht_selfheal_dir_finish(frame, this, 0, 1);
+        return 0;
+    }
 
-        local->call_cnt = count = conf->subvolume_cnt;
+    local->call_cnt = missing_xattr;
 
-        dht_log_new_layout_for_dir_selfheal (this, loc, layout);
+    if (gf_log_get_loglevel() >= GF_LOG_DEBUG)
+        dht_log_new_layout_for_dir_selfheal(this, loc, layout);
 
-        for (i = 0; i < layout->cnt; i++) {
-                dht_selfheal_dir_xattr_persubvol (frame, loc, layout, i, NULL);
+    for (i = 0; i < layout->cnt; i++) {
+        if (layout->list[i].err != -1 || !layout->list[i].stop)
+            continue;
 
-                if (--count == 0)
-                        goto out;
-        }
-        /* if we are here, subvolcount > layout_count. subvols-per-directory
-         * option might be set here. We need to clear out layout from the
-         * non-participating subvolumes, else it will result in overlaps */
-        dummy = dht_layout_new (this, 1);
-        if (!dummy)
-                goto out;
-        for (i = 0; i < conf->subvolume_cnt; i++) {
-                if (_gf_false ==
-                    dht_is_subvol_in_layout (layout, conf->subvolumes[i])) {
-                        dht_selfheal_dir_xattr_persubvol (frame, loc, dummy, 0,
-                                                          conf->subvolumes[i]);
-                        if (--count == 0)
-                                break;
-                }
-        }
+        dht_selfheal_dir_xattr_persubvol(frame, loc, layout, i, NULL);
 
-        dht_layout_unref (this, dummy);
+        if (--missing_xattr == 0)
+            break;
+    }
+    dummy = dht_layout_new(this, 1);
+    if (!dummy) {
+        gf_uuid_unparse(loc->gfid, gfid);
+        gf_smsg(this->name, GF_LOG_ERROR, ENOMEM, DHT_MSG_DUMMY_ALLOC_FAILED,
+                "path=%s", loc->path, "gfid=%s", gfid, NULL);
+        goto out;
+    }
+    for (i = 0; i < conf->subvolume_cnt && missing_xattr; i++) {
+        if (_gf_false == dht_is_subvol_in_layout(layout, conf->subvolumes[i])) {
+            dht_selfheal_dir_xattr_persubvol(frame, loc, dummy, 0,
+                                             conf->subvolumes[i]);
+            missing_xattr--;
+        }
+    }
+
+    dht_layout_unref(this, dummy);
 out:
-        return 0;
+    return 0;
 }
 
 int
-dht_selfheal_dir_xattr (call_frame_t *frame, loc_t *loc, dht_layout_t *layout)
+dht_selfheal_dir_setattr_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                             int op_ret, int op_errno, struct iatt *statpre,
+                             struct iatt *statpost, dict_t *xdata)
 {
-        dht_local_t *local = NULL;
-        int          missing_xattr = 0;
-        int          i = 0;
-        xlator_t    *this = NULL;
-        dht_conf_t   *conf = NULL;
-        dht_layout_t *dummy = NULL;
+    dht_local_t *local = NULL;
+    dht_layout_t *layout = NULL;
+    int this_call_cnt = 0, ret = -1;
 
-        local = frame->local;
-        this = frame->this;
-        conf = this->private;
+    local = frame->local;
+    layout = local->selfheal.layout;
 
-        for (i = 0; i < layout->cnt; i++) {
-                if (layout->list[i].err != -1 || !layout->list[i].stop) {
-                        /* err != -1 would mean xattr present on the directory
-                         * or the directory is non existent.
-                         * !layout->list[i].stop would mean layout absent
-                         */
+    this_call_cnt = dht_frame_return(frame);
 
-                        continue;
-                }
-                missing_xattr++;
-        }
-        /* Also account for subvolumes with no-layout. Used for zero'ing out
-         * the layouts and for setting quota key's if present */
-        for (i = 0; i < conf->subvolume_cnt; i++) {
-                if (_gf_false ==
-                    dht_is_subvol_in_layout (layout, conf->subvolumes[i])) {
-                        missing_xattr++;
-                }
-        }
-        gf_msg_trace (this->name, 0,
-                      "%d subvolumes missing xattr for %s",
-                      missing_xattr, loc->path);
+    if (is_last_call(this_call_cnt)) {
+        if (!local->heal_layout) {
+            gf_msg_trace(this->name, 0, "Skip heal layout for %s gfid = %s ",
+                         local->loc.path, uuid_utoa(local->gfid));
 
-        if (missing_xattr == 0) {
-                dht_selfheal_dir_finish (frame, this, 0);
-                return 0;
+            dht_selfheal_dir_finish(frame, this, 0, 1);
+            return 0;
         }
+        ret = dht_selfheal_layout_lock(frame, layout, _gf_false,
+                                       dht_selfheal_dir_xattr,
+                                       dht_should_heal_layout);
 
-        local->call_cnt = missing_xattr;
-
-        dht_log_new_layout_for_dir_selfheal (this, loc, layout);
-
-        for (i = 0; i < layout->cnt; i++) {
-                if (layout->list[i].err != -1 || !layout->list[i].stop)
-                        continue;
-
-                dht_selfheal_dir_xattr_persubvol (frame, loc, layout, i, NULL);
-
-                if (--missing_xattr == 0)
-                        break;
-        }
-        dummy = dht_layout_new (this, 1);
-        if (!dummy)
-                goto out;
-        for (i = 0; i < conf->subvolume_cnt && missing_xattr; i++) {
-                if (_gf_false ==
-                    dht_is_subvol_in_layout (layout, conf->subvolumes[i])) {
-                        dht_selfheal_dir_xattr_persubvol (frame, loc, dummy, 0,
-                                                          conf->subvolumes[i]);
-                        missing_xattr--;
-                }
+        if (ret < 0) {
+            dht_selfheal_dir_finish(frame, this, -1, 1);
         }
+    }
 
-        dht_layout_unref (this, dummy);
-out:
-        return 0;
+    return 0;
 }
 
-gf_boolean_t
-dht_is_subvol_part_of_layout (dht_layout_t *layout, xlator_t *xlator)
+int
+dht_selfheal_dir_setattr(call_frame_t *frame, loc_t *loc, struct iatt *stbuf,
+                         int32_t valid, dht_layout_t *layout)
 {
-        int               i = 0;
-        gf_boolean_t    ret = _gf_false;
-
-        for (i = 0; i < layout->cnt; i++) {
-                if (!strcmp (layout->list[i].xlator->name, xlator->name)) {
-                        ret = _gf_true;
-                        break;
+    int missing_attr = 0;
+    int i = 0, ret = -1;
+    dht_local_t *local = NULL;
+    dht_conf_t *conf = NULL;
+    xlator_t *this = NULL;
+    int cnt = 0;
+
+    local = frame->local;
+    this = frame->this;
+    conf = this->private;
+
+    /* We need to heal the attrs if:
+     * 1. Any directories were missing - the newly created dirs will need
+     *    to have the correct attrs set
+     * 2. An existing dir does not have the correct permissions -they may
+     *    have been changed when a brick was down.
+     */
+
+    for (i = 0; i < layout->cnt; i++) {
+        if (layout->list[i].err == -1)
+            missing_attr++;
+    }
+
+    if ((missing_attr == 0) && (local->need_attrheal == 0)) {
+        if (!local->heal_layout) {
+            gf_msg_trace(this->name, 0, "Skip heal layout for %s gfid = %s ",
+                         loc->path, uuid_utoa(loc->gfid));
+            dht_selfheal_dir_finish(frame, this, 0, 1);
+            return 0;
+        }
+        ret = dht_selfheal_layout_lock(frame, layout, _gf_false,
+                                       dht_selfheal_dir_xattr,
+                                       dht_should_heal_layout);
 
-                }
+        if (ret < 0) {
+            dht_selfheal_dir_finish(frame, this, -1, 1);
         }
 
-        return ret;
-}
+        return 0;
+    }
 
-int
-dht_layout_index_from_conf (dht_layout_t *layout, xlator_t *xlator)
-{
-        int i = -1;
-        int j = 0;
+    cnt = local->call_cnt = conf->subvolume_cnt;
 
-        for (j = 0; j < layout->cnt; j++) {
-                if (!strcmp (layout->list[j].xlator->name, xlator->name)) {
-                        i = j;
-                        break;
-                }
-        }
+    for (i = 0; i < cnt; i++) {
+        STACK_WIND(frame, dht_selfheal_dir_setattr_cbk, layout->list[i].xlator,
+                   layout->list[i].xlator->fops->setattr, loc, stbuf, valid,
+                   NULL);
+    }
 
-        return i;
+    return 0;
 }
 
-
 static int
-dht_selfheal_dir_xattr_for_nameless_lookup (call_frame_t *frame, loc_t *loc,
-                                            dht_layout_t  *layout)
+dht_selfheal_dir_mkdir_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                           int op_ret, int op_errno, inode_t *inode,
+                           struct iatt *stbuf, struct iatt *preparent,
+                           struct iatt *postparent, dict_t *xdata)
 {
-        dht_local_t     *local = NULL;
-        int             missing_xattr = 0;
-        int             i = 0;
-        xlator_t        *this = NULL;
-        dht_conf_t      *conf = NULL;
-        dht_layout_t    *dummy = NULL;
-        int             j = 0;
-
-        local = frame->local;
-        this = frame->this;
-        conf = this->private;
-
+    dht_local_t *local = NULL;
+    dht_layout_t *layout = NULL;
+    xlator_t *prev = NULL;
+    xlator_t *subvol = NULL;
+    int i = 0, ret = -1;
+    int this_call_cnt = 0;
+    char gfid[GF_UUID_BUF_SIZE] = {0};
+
+    local = frame->local;
+    layout = local->selfheal.layout;
+    prev = cookie;
+    subvol = prev;
+
+    if ((op_ret == 0) || ((op_ret == -1) && (op_errno == EEXIST))) {
         for (i = 0; i < layout->cnt; i++) {
-                if (layout->list[i].err != -1 || !layout->list[i].stop) {
-                        /* err != -1 would mean xattr present on the directory
-                           or the directory is non existent.
-                           !layout->list[i].stop would mean layout absent
-                        */
+            if (layout->list[i].xlator == subvol) {
+                layout->list[i].err = -1;
+                break;
+            }
+        }
+    }
+
+    if (op_ret) {
+        gf_uuid_unparse(local->loc.gfid, gfid);
+        gf_smsg(this->name,
+                ((op_errno == EEXIST) ? GF_LOG_DEBUG : GF_LOG_WARNING),
+                op_errno, DHT_MSG_DIR_SELFHEAL_FAILED, "path=%s",
+                local->loc.path, "gfid=%s", gfid, NULL);
+        goto out;
+    }
+    dht_iatt_merge(this, &local->preparent, preparent);
+    dht_iatt_merge(this, &local->postparent, postparent);
+    ret = 0;
 
-                        continue;
-                }
-                missing_xattr++;
-        }
+out:
+    this_call_cnt = dht_frame_return(frame);
 
-        /* Also account for subvolumes with no-layout. Used for zero'ing out
-           the layouts and for setting quota key's if present */
+    if (is_last_call(this_call_cnt)) {
+        dht_selfheal_dir_finish(frame, this, ret, 0);
+        dht_selfheal_dir_setattr(frame, &local->loc, &local->stbuf, 0xffffff,
+                                 layout);
+    }
 
-        /* Send  where either the subvol is not part of layout,
-         * or it is part of the layout but error is non-zero but error
-         * is not equal to -1 or ENOENT.
-         */
+    return 0;
+}
 
-        for (i = 0; i < conf->subvolume_cnt; i++) {
-                if (dht_is_subvol_part_of_layout (layout, conf->subvolumes[i])
-                    == _gf_false) {
-                        missing_xattr++;
-                        continue;
-                }
+static int
+dht_selfheal_dir_mkdir_lookup_done(call_frame_t *frame, xlator_t *this)
+{
+    dht_local_t *local = NULL;
+    int i = 0;
+    dict_t *dict = NULL;
+    dht_layout_t *layout = NULL;
+    loc_t *loc = NULL;
+    int cnt = 0;
+    int ret = -1;
+
+    VALIDATE_OR_GOTO(this->private, err);
+
+    local = frame->local;
+    layout = local->layout;
+    loc = &local->loc;
+
+    if (!gf_uuid_is_null(local->gfid)) {
+        dict = dict_new();
+        if (!dict)
+            return -1;
 
-                j = dht_layout_index_from_conf (layout, conf->subvolumes[i]);
+        ret = dict_set_gfuuid(dict, "gfid-req", local->gfid, true);
+        if (ret)
+            gf_smsg(this->name, GF_LOG_WARNING, 0, DHT_MSG_DICT_SET_FAILED,
+                    "path=%s", loc->path, "key=gfid-req", NULL);
+    } else if (local->params) {
+        /* Send the dictionary from higher layers directly */
+
+        dict = dict_ref(local->params);
+    }
+    /* Code to update all extended attributed from local->xattr
+       to dict
+    */
+    dht_dir_set_heal_xattr(this, local, dict, local->xattr, NULL, NULL);
+
+    if (!dict) {
+        gf_smsg(this->name, GF_LOG_WARNING, 0, DHT_MSG_DICT_IS_NULL, NULL);
+        dict = dict_new();
+        if (!dict)
+            return -1;
+    }
+    ret = dict_set_flag(dict, GF_INTERNAL_CTX_KEY, GF_DHT_HEAL_DIR);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_ERROR, 0, DHT_MSG_DICT_SET_FAILED, "key=%s",
+                GF_INTERNAL_CTX_KEY, "path=%s", loc->path, NULL);
+        /* We can still continue. As heal can still happen
+         * unless quota limits have reached for the dir.
+         */
+    }
 
-                if ((j != -1) && (layout->list[j].err != -1) &&
-                   (layout->list[j].err != 0) &&
-                   (layout->list[j].err != ENOENT)) {
-                        missing_xattr++;
-                }
+    cnt = layout->cnt;
+    for (i = 0; i < cnt; i++) {
+        if (layout->list[i].err == ESTALE || layout->list[i].err == ENOENT ||
+            local->selfheal.force_mkdir) {
+            gf_msg_debug(this->name, 0, "Creating directory %s on subvol %s",
+                         loc->path, layout->list[i].xlator->name);
 
+            STACK_WIND_COOKIE(
+                frame, dht_selfheal_dir_mkdir_cbk, layout->list[i].xlator,
+                layout->list[i].xlator, layout->list[i].xlator->fops->mkdir,
+                loc,
+                st_mode_from_ia(local->stbuf.ia_prot, local->stbuf.ia_type), 0,
+                dict);
         }
+    }
 
+    if (dict)
+        dict_unref(dict);
 
-        gf_log (this->name, GF_LOG_TRACE,
-                "%d subvolumes missing xattr for %s",
-                missing_xattr, loc->path);
+    return 0;
 
-        if (missing_xattr == 0) {
-                dht_selfheal_dir_finish (frame, this, 0);
-                return 0;
-        }
+err:
+    dht_selfheal_dir_finish(frame, this, -1, 1);
+    return 0;
+}
 
-        local->call_cnt = missing_xattr;
+static int
+dht_selfheal_dir_mkdir_lookup_cbk(call_frame_t *frame, void *cookie,
+                                  xlator_t *this, int op_ret, int op_errno,
+                                  inode_t *inode, struct iatt *stbuf,
+                                  dict_t *xattr, struct iatt *postparent)
+{
+    dht_local_t *local = NULL;
+    int i = 0;
+    int this_call_cnt = 0;
+    int missing_dirs = 0;
+    dht_layout_t *layout = NULL;
+    xlator_t *prev = 0;
+    loc_t *loc = NULL;
+    char gfid_local[GF_UUID_BUF_SIZE] = {0};
+    int index = -1;
+
+    VALIDATE_OR_GOTO(this->private, err);
+
+    local = frame->local;
+    layout = local->layout;
+    loc = &local->loc;
+    prev = cookie;
+
+    if (!gf_uuid_is_null(local->gfid))
+        gf_uuid_unparse(local->gfid, gfid_local);
+
+    LOCK(&frame->lock);
+    {
+        index = dht_layout_index_for_subvol(layout, prev);
+        if ((op_ret < 0) && (op_errno == ENOENT || op_errno == ESTALE)) {
+            local->selfheal.hole_cnt = !local->selfheal.hole_cnt
+                                           ? 1
+                                           : local->selfheal.hole_cnt + 1;
+            /* the status might have changed. Update the layout with the
+             * new status
+             */
+            if (index >= 0) {
+                layout->list[index].err = op_errno;
+            }
+        }
+
+        if (!op_ret) {
+            dht_iatt_merge(this, &local->stbuf, stbuf);
+            if (prev == local->mds_subvol) {
+                dict_unref(local->xattr);
+                local->xattr = dict_ref(xattr);
+            }
+            /* the status might have changed. Update the layout with the
+             * new status
+             */
+            if (index >= 0) {
+                layout->list[index].err = -1;
+            }
+        }
+    }
+    UNLOCK(&frame->lock);
+
+    this_call_cnt = dht_frame_return(frame);
+
+    if (is_last_call(this_call_cnt)) {
+        if (local->selfheal.hole_cnt == layout->cnt) {
+            gf_msg_debug(this->name, op_errno,
+                         "Lookup failed, an rmdir could have "
+                         "deleted this entry %s",
+                         loc->name);
+            local->op_errno = op_errno;
+            goto err;
+        } else {
+            for (i = 0; i < layout->cnt; i++) {
+                if (layout->list[i].err == ENOENT ||
+                    layout->list[i].err == ESTALE ||
+                    local->selfheal.force_mkdir)
+                    missing_dirs++;
+            }
+
+            if (missing_dirs == 0) {
+                dht_selfheal_dir_finish(frame, this, 0, 0);
+                dht_selfheal_dir_setattr(frame, loc, &local->stbuf, 0xffffffff,
+                                         layout);
+                return 0;
+            }
 
-        dht_log_new_layout_for_dir_selfheal (this, loc, layout);
+            local->call_cnt = missing_dirs;
+            dht_selfheal_dir_mkdir_lookup_done(frame, this);
+        }
+    }
 
-        for (i = 0; i < layout->cnt; i++) {
-                if (layout->list[i].err != -1 || !layout->list[i].stop)
-                        continue;
+    return 0;
 
-                dht_selfheal_dir_xattr_persubvol (frame, loc, layout, i, NULL);
+err:
+    dht_selfheal_dir_finish(frame, this, -1, 1);
+    return 0;
+}
 
-                if (--missing_xattr == 0)
-                        break;
+static int
+dht_selfheal_dir_mkdir_lock_cbk(call_frame_t *frame, void *cookie,
+                                xlator_t *this, int32_t op_ret,
+                                int32_t op_errno, dict_t *xdata)
+{
+    dht_local_t *local = NULL;
+    dht_conf_t *conf = NULL;
+    int i = 0;
+    int ret = -1;
+    xlator_t *mds_subvol = NULL;
+
+    VALIDATE_OR_GOTO(this->private, err);
+
+    conf = this->private;
+    local = frame->local;
+    mds_subvol = local->mds_subvol;
+
+    local->call_cnt = conf->subvolume_cnt;
+
+    if (op_ret < 0) {
+        if (op_errno == EINVAL) {
+            local->call_cnt = 1;
+            dht_selfheal_dir_mkdir_lookup_done(frame, this);
+            return 0;
+        }
+
+        gf_smsg(this->name, GF_LOG_WARNING, op_errno, DHT_MSG_ENTRYLK_ERROR,
+                "path=%s", local->loc.path, NULL);
+
+        local->op_errno = op_errno;
+        goto err;
+    }
+
+    /* After getting locks, perform lookup again to ensure that the
+       directory was not deleted by a racing rmdir
+    */
+    if (!local->xattr_req)
+        local->xattr_req = dict_new();
+
+    ret = dict_set_int32(local->xattr_req, "list-xattr", 1);
+    if (ret)
+        gf_smsg(this->name, GF_LOG_ERROR, 0, DHT_MSG_DICT_SET_FAILED, "path=%s",
+                local->loc.path, NULL);
+
+    for (i = 0; i < conf->subvolume_cnt; i++) {
+        if (mds_subvol && conf->subvolumes[i] == mds_subvol) {
+            STACK_WIND_COOKIE(frame, dht_selfheal_dir_mkdir_lookup_cbk,
+                              conf->subvolumes[i], conf->subvolumes[i],
+                              conf->subvolumes[i]->fops->lookup, &local->loc,
+                              local->xattr_req);
+        } else {
+            STACK_WIND_COOKIE(frame, dht_selfheal_dir_mkdir_lookup_cbk,
+                              conf->subvolumes[i], conf->subvolumes[i],
+                              conf->subvolumes[i]->fops->lookup, &local->loc,
+                              NULL);
         }
+    }
 
-        dummy = dht_layout_new (this, 1);
-        if (!dummy)
-                goto out;
-
-        for (i = 0; i < conf->subvolume_cnt && missing_xattr; i++) {
-              if (dht_is_subvol_part_of_layout (layout, conf->subvolumes[i])
-                  == _gf_false) {
-                        dht_selfheal_dir_xattr_persubvol (frame, loc, dummy, 0,
-                                                          conf->subvolumes[i]);
-                        missing_xattr--;
-                        continue;
-              }
-
-                j = dht_layout_index_from_conf (layout, conf->subvolumes[i]);
-
-                if ((j != -1) && (layout->list[j].err != -1) &&
-                    (layout->list[j].err != ENOENT) &&
-                    (layout->list[j].err != 0)) {
-                        dht_selfheal_dir_xattr_persubvol (frame, loc, dummy, 0,
-                                                          conf->subvolumes[i]);
-                        missing_xattr--;
-                }
-        }
-
-        dht_layout_unref (this, dummy);
-out:
-        return 0;
+    return 0;
 
+err:
+    dht_selfheal_dir_finish(frame, this, -1, 1);
+    return 0;
 }
 
-int
-dht_selfheal_dir_setattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                              int op_ret, int op_errno, struct iatt *statpre,
-                              struct iatt *statpost, dict_t *xdata)
+static int
+dht_selfheal_dir_mkdir(call_frame_t *frame, loc_t *loc, dht_layout_t *layout,
+                       int force)
 {
-        dht_local_t   *local = NULL;
-        dht_layout_t  *layout = NULL;
-        int            this_call_cnt = 0, ret = -1;
-
-        local  = frame->local;
-        layout = local->selfheal.layout;
-
-        this_call_cnt = dht_frame_return (frame);
+    int missing_dirs = 0;
+    int i = 0;
+    int op_errno = 0;
+    int ret = -1;
+    dht_local_t *local = NULL;
+    xlator_t *this = NULL;
+    dht_conf_t *conf = NULL;
+
+    local = frame->local;
+    this = frame->this;
+    conf = this->private;
+
+    local->selfheal.force_mkdir = force;
+    local->selfheal.hole_cnt = 0;
+
+    for (i = 0; i < layout->cnt; i++) {
+        if (layout->list[i].err == ENOENT || force)
+            missing_dirs++;
+    }
+
+    if (missing_dirs == 0) {
+        /* We don't need to create any directories. Proceed to heal the
+         * attrs and xattrs
+         */
+        if (!__is_root_gfid(local->stbuf.ia_gfid)) {
+            if (local->need_xattr_heal) {
+                local->need_xattr_heal = 0;
+                ret = dht_dir_xattr_heal(this, local, &op_errno);
+                if (ret) {
+                    gf_smsg(this->name, GF_LOG_ERROR, op_errno,
+                            DHT_MSG_DIR_XATTR_HEAL_FAILED, "path=%s",
+                            local->loc.path, "gfid=%s", local->gfid, NULL);
+                }
+            } else {
+                if (!gf_uuid_is_null(local->gfid))
+                    gf_uuid_copy(loc->gfid, local->gfid);
 
-        if (is_last_call (this_call_cnt)) {
-                ret = dht_selfheal_layout_lock (frame, layout, _gf_false,
-                                                dht_selfheal_dir_xattr,
-                                                dht_should_heal_layout);
+                ret = dht_common_mark_mdsxattr(frame, NULL, 0);
+                if (!ret)
+                    return 0;
 
-                if (ret < 0) {
-                        dht_selfheal_dir_finish (frame, this, -1);
-                }
+                gf_smsg(this->name, GF_LOG_INFO, 0, DHT_MSG_SET_XATTR_FAILED,
+                        "path=%s", local->loc.path, "gfid=%s", local->gfid,
+                        NULL);
+            }
         }
-
+        dht_selfheal_dir_setattr(frame, loc, &local->stbuf, 0xffffffff, layout);
         return 0;
+    }
+
+    /* MDS xattr is populated only while DHT is having more than one
+     subvol.In case of graph switch while adding more dht subvols need to
+     consider hash subvol as a MDS to avoid MDS check failure at the time
+     of running fop on directory
+    */
+    if (!dict_get(local->xattr, conf->mds_xattr_key) &&
+        (conf->subvolume_cnt > 1)) {
+        if (local->hashed_subvol == NULL) {
+            local->hashed_subvol = dht_subvol_get_hashed(this, loc);
+            if (local->hashed_subvol == NULL) {
+                local->op_errno = EINVAL;
+                gf_smsg(this->name, GF_LOG_WARNING, local->op_errno,
+                        DHT_MSG_HASHED_SUBVOL_GET_FAILED, "gfid=%s",
+                        loc->pargfid, "name=%s", loc->name, "path=%s",
+                        loc->path, NULL);
+                goto err;
+            }
+        }
+        ret = dht_inode_ctx_mdsvol_set(local->inode, this,
+                                       local->hashed_subvol);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, DHT_MSG_SET_INODE_CTX_FAILED,
+                   "Failed to set hashed subvol for %s on inode vol is %s",
+                   local->loc.path,
+                   local->hashed_subvol ? local->hashed_subvol->name : "NULL");
+            goto err;
+        }
+    }
+
+    if (local->hashed_subvol == NULL) {
+        local->hashed_subvol = dht_subvol_get_hashed(this, loc);
+        if (local->hashed_subvol == NULL) {
+            local->op_errno = EINVAL;
+            gf_smsg(this->name, GF_LOG_WARNING, local->op_errno,
+                    DHT_MSG_HASHED_SUBVOL_GET_FAILED, "gfid=%s", loc->pargfid,
+                    "name=%s", loc->name, "path=%s", loc->path, NULL);
+            goto err;
+        }
+    }
+
+    local->current = &local->lock[0];
+    ret = dht_protect_namespace(frame, loc, local->hashed_subvol,
+                                &local->current->ns,
+                                dht_selfheal_dir_mkdir_lock_cbk);
+
+    if (ret < 0)
+        goto err;
+
+    return 0;
+err:
+    return -1;
 }
 
-
-int
-dht_selfheal_dir_setattr (call_frame_t *frame, loc_t *loc, struct iatt *stbuf,
-                          int32_t valid, dht_layout_t *layout)
+static int
+dht_selfheal_layout_alloc_start(xlator_t *this, loc_t *loc,
+                                dht_layout_t *layout)
 {
-        int           missing_attr = 0;
-        int           i     = 0, ret = -1;
-        dht_local_t  *local = NULL;
-        xlator_t     *this = NULL;
-
-        local = frame->local;
-        this = frame->this;
+    int start = 0;
+    uint32_t hashval = 0;
+    int ret = 0;
+    const char *str = NULL;
+    dht_conf_t *conf = NULL;
+    char buf[UUID_CANONICAL_FORM_LEN + 1] = {
+        0,
+    };
+
+    conf = this->private;
+
+    if (conf->randomize_by_gfid) {
+        str = uuid_utoa_r(loc->gfid, buf);
+    } else {
+        str = loc->path;
+    }
+
+    ret = dht_hash_compute(this, layout->type, str, &hashval);
+    if (ret == 0) {
+        start = (hashval % layout->cnt);
+    }
+
+    return start;
+}
 
+static int
+dht_get_layout_count(xlator_t *this, dht_layout_t *layout, int new_layout)
+{
+    int i = 0;
+    int j = 0;
+    int err = 0;
+    int count = 0;
+    dht_conf_t *conf = NULL;
+
+    /* Gets in use only for replace-brick, remove-brick */
+    conf = this->private;
+    for (i = 0; i < layout->cnt; i++) {
+        for (j = 0; j < conf->subvolume_cnt; j++) {
+            if (conf->decommissioned_bricks[j] &&
+                conf->decommissioned_bricks[j] == layout->list[i].xlator) {
+                layout->list[i].err = EINVAL;
+                break;
+            }
+        }
+    }
+
+    for (i = 0; i < layout->cnt; i++) {
+        err = layout->list[i].err;
+        if (err == -1 || err == 0 || err == ENOENT) {
+            /* Take this with a pinch of salt. The behaviour seems
+             * to be slightly different when this function is
+             * invoked from mkdir codepath. For eg., err == 0 in
+             * mkdir codepath means directory created but xattr
+             * is not set yet.
+             */
+
+            /* Setting list[i].err = -1 is an indication for
+               dht_selfheal_layout_new_directory() to assign
+               a range. We set it to -1 based on any one of
+               the three criteria:
+
+               - err == -1 already, which means directory
+                 existed but layout was not set on it.
+
+               - err == 0, which means directory exists and
+                 has an old layout piece which will be
+                 overwritten now.
+
+               - err == ENOENT, which means directory does
+                 not exist (possibly racing with mkdir or
+                 finishing half done mkdir). The missing
+                 directory will be attempted to be recreated.
+            */
+            count++;
+            if (!err)
+                layout->list[i].err = -1;
+        }
+    }
+
+    /* no subvolume has enough space, but can't stop directory creation */
+    if (!count || !new_layout) {
         for (i = 0; i < layout->cnt; i++) {
-                if (layout->list[i].err == -1)
-                        missing_attr++;
-        }
-
-        if (missing_attr == 0) {
-                ret = dht_selfheal_layout_lock (frame, layout, _gf_false,
-                                                dht_selfheal_dir_xattr,
-                                                dht_should_heal_layout);
-
-                if (ret < 0) {
-                        dht_selfheal_dir_finish (frame, this, -1);
-                }
-
-                return 0;
-        }
+            err = layout->list[i].err;
+            if (err == ENOSPC) {
+                layout->list[i].err = -1;
+                count++;
+            }
+        }
+    }
+
+    /* if layout->spread_cnt is set, check if it is <= available
+     * subvolumes (down brick and decommissioned bricks are considered
+     * un-available). Else return count (available up bricks) */
+    count = ((layout->spread_cnt && (layout->spread_cnt <= count))
+                 ? layout->spread_cnt
+                 : ((count) ? count : 1));
+
+    return count;
+}
 
-        if (!gf_uuid_is_null (local->gfid))
-                gf_uuid_copy (loc->gfid, local->gfid);
+void
+dht_selfheal_layout_new_directory(call_frame_t *frame, loc_t *loc,
+                                  dht_layout_t *new_layout);
 
-        local->call_cnt = missing_attr;
-        for (i = 0; i < layout->cnt; i++) {
-                if (layout->list[i].err == -1) {
-                        gf_msg_trace (this->name, 0,
-                                      "%s: setattr on subvol %s, gfid = %s",
-                                      loc->path, layout->list[i].xlator->name,
-                                      uuid_utoa(loc->gfid));
-
-                        STACK_WIND (frame, dht_selfheal_dir_setattr_cbk,
-                                    layout->list[i].xlator,
-                                    layout->list[i].xlator->fops->setattr,
-                                    loc, stbuf, valid, NULL);
-                }
-        }
+void
+dht_layout_range_swap(dht_layout_t *layout, int i, int j);
 
-        return 0;
-}
+/*
+ * It's a bit icky using local variables in a macro, but it makes the rest
+ * of the code a lot clearer.
+ */
+#define OV_ENTRY(x, y) table[x * new->cnt + y]
 
-int
-dht_selfheal_dir_mkdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                            int op_ret, int op_errno,
-                            inode_t *inode, struct iatt *stbuf,
-                            struct iatt *preparent, struct iatt *postparent,
-                            dict_t *xdata)
+static void
+dht_selfheal_layout_maximize_overlap(call_frame_t *frame, loc_t *loc,
+                                     dht_layout_t *new, dht_layout_t *old)
 {
-        dht_local_t   *local = NULL;
-        dht_layout_t  *layout = NULL;
-        call_frame_t  *prev = NULL;
-        xlator_t      *subvol = NULL;
-        int            i = 0;
-        int            this_call_cnt = 0;
-        char           gfid[GF_UUID_BUF_SIZE] = {0};
-
-        local  = frame->local;
-        layout = local->selfheal.layout;
-        prev   = cookie;
-        subvol = prev->this;
-
-        if ((op_ret == 0) || ((op_ret == -1) && (op_errno == EEXIST))) {
-                for (i = 0; i < layout->cnt; i++) {
-                        if (layout->list[i].xlator == subvol) {
-                                layout->list[i].err = -1;
-                                break;
-                        }
+    int i = 0;
+    int j = 0;
+    uint32_t curr_overlap = 0;
+    uint32_t max_overlap = 0;
+    int max_overlap_idx = -1;
+    uint32_t overlap = 0;
+    uint32_t *table = NULL;
+
+    dht_layout_sort_volname(old);
+    /* Now both old_layout->list[] and new_layout->list[]
+       are match the same xlators/subvolumes. i.e,
+       old_layout->[i] and new_layout->[i] are referring
+       to the same subvolumes
+    */
+
+    /* Build a table of overlaps between new[i] and old[j]. */
+    table = alloca(sizeof(overlap) * old->cnt * new->cnt);
+    if (!table) {
+        return;
+    }
+    memset(table, 0, sizeof(overlap) * old->cnt * new->cnt);
+    for (i = 0; i < new->cnt; ++i) {
+        for (j = 0; j < old->cnt; ++j) {
+            OV_ENTRY(i, j) = dht_overlap_calc(old, j, new, i);
+        }
+    }
+
+    for (i = 0; i < new->cnt; i++) {
+        if (new->list[i].err > 0) {
+            /* Subvol might be marked for decommission
+               with EINVAL, or some other serious error
+               marked with positive errno.
+            */
+            continue;
+        }
+
+        max_overlap = 0;
+        max_overlap_idx = i;
+        for (j = (i + 1); j < new->cnt; ++j) {
+            if (new->list[j].err > 0) {
+                /* Subvol might be marked for decommission
+                with EINVAL, or some other serious error
+                marked with positive errno.
+                */
+                continue;
+            }
+            /* Calculate the overlap now. */
+            curr_overlap = OV_ENTRY(i, i) + OV_ENTRY(j, j);
+            /* Calculate the overlap after the proposed swap. */
+            overlap = OV_ENTRY(i, j) + OV_ENTRY(j, i);
+            /* Are we better than status quo? */
+            if (overlap > curr_overlap) {
+                overlap -= curr_overlap;
+                /* Are we better than the previous choice? */
+                if (overlap > max_overlap) {
+                    max_overlap = overlap;
+                    max_overlap_idx = j;
                 }
+            }
         }
 
-        if (op_ret) {
-
-                gf_uuid_unparse(local->loc.gfid, gfid);
-                gf_msg (this->name, ((op_errno == EEXIST) ? GF_LOG_DEBUG :
-                                     GF_LOG_WARNING),
-                        op_errno, DHT_MSG_DIR_SELFHEAL_FAILED,
-                        "Directory selfheal failed: path = %s, gfid = %s",
-                        local->loc.path, gfid );
-                goto out;
-        }
-
-        dht_iatt_merge (this, &local->stbuf, stbuf, prev->this);
-        dht_iatt_merge (this, &local->preparent, preparent, prev->this);
-        dht_iatt_merge (this, &local->postparent, postparent, prev->this);
-
-out:
-        this_call_cnt = dht_frame_return (frame);
-
-        if (is_last_call (this_call_cnt)) {
-                dht_selfheal_dir_setattr (frame, &local->loc, &local->stbuf, 0xffffff, layout);
+        if (max_overlap_idx != i) {
+            dht_layout_range_swap(new, i, max_overlap_idx);
+            /* Need to swap the table values too. */
+            for (j = 0; j < old->cnt; ++j) {
+                overlap = OV_ENTRY(i, j);
+                OV_ENTRY(i, j) = OV_ENTRY(max_overlap_idx, j);
+                OV_ENTRY(max_overlap_idx, j) = overlap;
+            }
         }
-
-        return 0;
+    }
 }
 
-void
-dht_selfheal_dir_mkdir_setacl (dict_t *xattr, dict_t *dict)
+static dht_layout_t *
+dht_fix_layout_of_directory(call_frame_t *frame, loc_t *loc,
+                            dht_layout_t *layout)
 {
-        data_t          *acl_default = NULL;
-        data_t          *acl_access = NULL;
-        xlator_t        *this = NULL;
-        int     ret = -1;
-
-        GF_ASSERT (xattr);
-        GF_ASSERT (dict);
+    int i = 0;
+    xlator_t *this = NULL;
+    dht_layout_t *new_layout = NULL;
+    dht_conf_t *priv = NULL;
+    dht_local_t *local = NULL;
+    uint32_t subvol_down = 0;
+    gf_boolean_t maximize_overlap = _gf_true;
+    char gfid[GF_UUID_BUF_SIZE] = {0};
+
+    this = frame->this;
+    priv = this->private;
+    local = frame->local;
+
+    if (layout->type == DHT_HASH_TYPE_DM_USER) {
+        gf_msg_debug(THIS->name, 0, "leaving %s alone", loc->path);
+        goto done;
+    }
+
+    new_layout = dht_layout_new(this, priv->subvolume_cnt);
+    if (!new_layout) {
+        gf_uuid_unparse(loc->gfid, gfid);
+        gf_smsg(this->name, GF_LOG_ERROR, ENOMEM, DHT_MSG_MEM_ALLOC_FAILED,
+                "new_layout, path=%s", loc->path, "gfid=%s", gfid, NULL);
+        goto done;
+    }
 
-        this = THIS;
-        GF_ASSERT (this);
+    /* If a subvolume is down, do not re-write the layout. */
+    dht_layout_anomalies(this, loc, layout, NULL, NULL, NULL, &subvol_down,
+                         NULL, NULL);
 
-        acl_default = dict_get (xattr, POSIX_ACL_DEFAULT_XATTR);
+    if (subvol_down) {
+        gf_uuid_unparse(loc->gfid, gfid);
+        gf_smsg(this->name, GF_LOG_WARNING, 0, DHT_MSG_LAYOUT_FIX_FAILED,
+                "subvol-down=%u", subvol_down, "Skipping-fix-layout", "path=%s",
+                loc->path, "gfid=%s", gfid, NULL);
+        GF_FREE(new_layout);
+        return NULL;
+    }
+
+    for (i = 0; i < new_layout->cnt; i++) {
+        if (layout->list[i].err != ENOSPC)
+            new_layout->list[i].err = layout->list[i].err;
+        else
+            new_layout->list[i].err = -1;
+
+        new_layout->list[i].xlator = layout->list[i].xlator;
+    }
+
+    new_layout->commit_hash = layout->commit_hash;
+
+    if (priv->du_stats) {
+        for (i = 0; i < priv->subvolume_cnt; ++i) {
+            gf_smsg(this->name, GF_LOG_DEBUG, 0, DHT_MSG_SUBVOL_INFO,
+                    "index=%d", i, "name=%s", priv->subvolumes[i]->name,
+                    "chunks=%u", priv->du_stats[i].chunks, "path=%s", loc->path,
+                    NULL);
+
+            /* Maximize overlap if the bricks are all the same
+             *  size.
+             * This is probably not going to be very common on
+             * live setups but will benefit our regression tests
+             */
+            if (i && (priv->du_stats[i].chunks != priv->du_stats[0].chunks)) {
+                maximize_overlap = _gf_false;
+            }
+        }
+    } else {
+        gf_smsg(this->name, GF_LOG_WARNING, 0, DHT_MSG_NO_DISK_USAGE_STATUS,
+                NULL);
+    }
+
+    /* First give it a layout as though it is a new directory. This
+       ensures rotation to kick in */
+    dht_layout_sort_volname(new_layout);
+    dht_selfheal_layout_new_directory(frame, loc, new_layout);
+
+    /* Maximize overlap if weighted-rebalance is disabled */
+    if (!priv->do_weighting)
+        maximize_overlap = _gf_true;
+
+    /* Now selectively re-assign ranges only when it helps */
+    if (maximize_overlap) {
+        dht_selfheal_layout_maximize_overlap(frame, loc, new_layout, layout);
+    }
+done:
+    if (new_layout) {
+        /* Make sure the extra 'ref' for existing layout is removed */
+        dht_layout_unref(this, local->layout);
 
-        if (!acl_default) {
-                gf_msg_debug (this->name, 0,
-                              "ACL_DEFAULT xattr not present");
-                goto cont;
-        }
-        ret = dict_set (dict, POSIX_ACL_DEFAULT_XATTR, acl_default);
-        if (ret)
-                gf_msg (this->name, GF_LOG_WARNING, 0,
-                        DHT_MSG_DICT_SET_FAILED,
-                        "Failed to set dictionary value.key = %s",
-                        POSIX_ACL_DEFAULT_XATTR);
-cont:
-        acl_access = dict_get (xattr, POSIX_ACL_ACCESS_XATTR);
-        if (!acl_access) {
-                gf_msg_debug (this->name, 0,
-                              "ACL_ACCESS xattr not present");
-                goto out;
-        }
-        ret = dict_set (dict, POSIX_ACL_ACCESS_XATTR, acl_access);
-        if (ret)
-                gf_msg (this->name, GF_LOG_WARNING, 0,
-                        DHT_MSG_DICT_SET_FAILED,
-                        "Failed to set dictionary value.key = %s",
-                        POSIX_ACL_ACCESS_XATTR);
+        local->layout = new_layout;
+    }
 
-out:
-        return;
+    return local->layout;
 }
 
-int
-dht_selfheal_dir_mkdir (call_frame_t *frame, loc_t *loc,
-                        dht_layout_t *layout, int force)
+/*
+ * Having to call this 2x for each entry in the layout is pretty horrible, but
+ * that's what all of this layout-sorting nonsense gets us.
+ */
+static uint32_t
+dht_get_chunks_from_xl(xlator_t *parent, xlator_t *child)
 {
-        int           missing_dirs = 0;
-        int           i     = 0;
-        int           ret   = -1;
-        dht_local_t  *local = NULL;
-        xlator_t     *this = NULL;
-        dict_t       *dict = NULL;
-
-        local = frame->local;
-        this = frame->this;
-
-        local->selfheal.force_mkdir = force ? _gf_true : _gf_false;
-
-        for (i = 0; i < layout->cnt; i++) {
-                if (layout->list[i].err == ENOENT || force)
-                        missing_dirs++;
-        }
+    dht_conf_t *priv = parent->private;
+    xlator_list_t *trav;
+    uint32_t index = 0;
 
-        if (missing_dirs == 0) {
-                dht_selfheal_dir_setattr (frame, loc, &local->stbuf, 0xffffffff, layout);
-                return 0;
-        }
+    if (!priv->du_stats) {
+        return 0;
+    }
 
-        local->call_cnt = missing_dirs;
-        if (!gf_uuid_is_null (local->gfid)) {
-                dict = dict_new ();
-                if (!dict)
-                        return -1;
-
-                ret = dict_set_static_bin (dict, "gfid-req", local->gfid, 16);
-                if (ret)
-                        gf_msg (this->name, GF_LOG_WARNING, 0,
-                                DHT_MSG_DICT_SET_FAILED,
-                                "%s: Failed to set dictionary value:"
-                                " key = gfid-req", loc->path);
-        } else if (local->params) {
-                /* Send the dictionary from higher layers directly */
-                dict = dict_ref (local->params);
+    for (trav = parent->children; trav; trav = trav->next) {
+        if (trav->xlator == child) {
+            return priv->du_stats[index].chunks;
         }
-        /* Set acls */
-        if (local->xattr && dict)
-                dht_selfheal_dir_mkdir_setacl (local->xattr, dict);
+        ++index;
+    }
 
-        if (!dict)
-                gf_log (this->name, GF_LOG_WARNING,
-                        "dict is NULL, need to make sure gfids are same");
-
-        for (i = 0; i < layout->cnt; i++) {
-                if (layout->list[i].err == ENOENT || force) {
-                        gf_msg_debug (this->name, 0,
-                                      "Creating directory %s on subvol %s",
-                                      loc->path, layout->list[i].xlator->name);
-
-                        STACK_WIND (frame, dht_selfheal_dir_mkdir_cbk,
-                                    layout->list[i].xlator,
-                                    layout->list[i].xlator->fops->mkdir,
-                                    loc,
-                                    st_mode_from_ia (local->stbuf.ia_prot,
-                                                     local->stbuf.ia_type),
-                                    0, dict);
-                }
-        }
-
-        if (dict)
-                dict_unref (dict);
-
-        return 0;
+    return 0;
 }
 
-int
-dht_selfheal_layout_alloc_start (xlator_t *this, loc_t *loc,
-                                 dht_layout_t *layout)
+void
+dht_selfheal_layout_new_directory(call_frame_t *frame, loc_t *loc,
+                                  dht_layout_t *layout)
 {
-        int         start                               = 0;
-        uint32_t    hashval                             = 0;
-        int         ret                                 = 0;
-        const char *str                                 = NULL;
-        dht_conf_t *conf                                = NULL;
-        char           buf[UUID_CANONICAL_FORM_LEN + 1] = {0, };
-
-        conf = this->private;
-
-        if (conf->randomize_by_gfid) {
-                str = uuid_utoa_r (loc->gfid, buf);
+    xlator_t *this = NULL;
+    double chunk = 0;
+    int i = 0;
+    uint32_t start = 0;
+    int bricks_to_use = 0;
+    int err = 0;
+    int start_subvol = 0;
+    uint32_t curr_size;
+    uint32_t range_size;
+    uint64_t total_size = 0;
+    int real_i;
+    dht_conf_t *priv;
+    gf_boolean_t weight_by_size;
+    int bricks_used = 0;
+
+    this = frame->this;
+    priv = this->private;
+    weight_by_size = priv->do_weighting;
+
+    bricks_to_use = dht_get_layout_count(this, layout, 1);
+    GF_ASSERT(bricks_to_use > 0);
+
+    bricks_used = 0;
+    for (i = 0; i < layout->cnt; ++i) {
+        err = layout->list[i].err;
+        if ((err != -1) && (err != ENOENT)) {
+            continue;
+        }
+        curr_size = dht_get_chunks_from_xl(this, layout->list[i].xlator);
+        if (!curr_size) {
+            weight_by_size = _gf_false;
+            break;
+        }
+        total_size += curr_size;
+        if (++bricks_used >= bricks_to_use) {
+            break;
+        }
+    }
+
+    if (weight_by_size && total_size) {
+        /* We know total_size is not zero. */
+        chunk = ((double)0xffffffff) / ((double)total_size);
+        gf_msg_debug(this->name, 0,
+                     "chunk size = 0xffffffff / %" PRIu64 " = %f", total_size,
+                     chunk);
+    } else {
+        weight_by_size = _gf_false;
+        chunk = ((unsigned long)0xffffffff) / bricks_to_use;
+    }
+
+    start_subvol = dht_selfheal_layout_alloc_start(this, loc, layout);
+
+    /* clear out the range, as we are re-computing here */
+    DHT_RESET_LAYOUT_RANGE(layout);
+
+    /*
+     * OK, what's this "real_i" stuff about?  This used to be two loops -
+     * from start_subvol to layout->cnt-1, then from 0 to start_subvol-1.
+     * That way is practically an open invitation to bugs when only one
+     * of the loops is updated.  Using real_i and modulo operators to make
+     * it one loop avoids this problem.  Remember, folks: it's everyone's
+     * responsibility to help stamp out copy/paste abuse.
+     */
+    bricks_used = 0;
+    for (real_i = 0; real_i < layout->cnt; real_i++) {
+        i = (real_i + start_subvol) % layout->cnt;
+        err = layout->list[i].err;
+        if ((err != -1) && (err != ENOENT)) {
+            continue;
+        }
+        if (weight_by_size) {
+            curr_size = dht_get_chunks_from_xl(this, layout->list[i].xlator);
+            if (!curr_size) {
+                continue;
+            }
         } else {
-                str = loc->path;
+            curr_size = 1;
         }
-
-        ret = dht_hash_compute (this, layout->type, str, &hashval);
-        if (ret == 0) {
-                start = (hashval % layout->cnt);
+        range_size = chunk * curr_size;
+        gf_msg_debug(this->name, 0, "assigning range size 0x%x to %s",
+                     range_size, layout->list[i].xlator->name);
+        DHT_SET_LAYOUT_RANGE(layout, i, start, range_size, loc->path);
+        if (++bricks_used >= bricks_to_use) {
+            layout->list[i].stop = 0xffffffff;
+            goto done;
         }
+        start += range_size;
+    }
 
-        return start;
+done:
+    return;
 }
 
-static inline int
-dht_get_layout_count (xlator_t *this, dht_layout_t *layout, int new_layout)
+static int
+dht_selfheal_dir_getafix(call_frame_t *frame, loc_t *loc, dht_layout_t *layout)
 {
-        int i = 0;
-        int j = 0;
-        int err = 0;
-        int count = 0;
-        dht_conf_t *conf = NULL;
-
-        /* Gets in use only for replace-brick, remove-brick */
-        conf = this->private;
-        for (i = 0; i < layout->cnt; i++) {
-                for (j = 0; j < conf->subvolume_cnt; j++) {
-                        if (conf->decommissioned_bricks[j] &&
-                            conf->decommissioned_bricks[j] == layout->list[i].xlator) {
-                                layout->list[i].err = EINVAL;
-                                break;
-                        }
-                }
-        }
-
-        for (i = 0; i < layout->cnt; i++) {
-                err = layout->list[i].err;
-                if (err == -1 || err == 0 || err == ENOENT) {
-                        /* Take this with a pinch of salt. The behaviour seems
-                         * to be slightly different when this function is
-                         * invoked from mkdir codepath. For eg., err == 0 in
-                         * mkdir codepath means directory created but xattr
-                         * is not set yet.
-                         */
-
-			/* Setting list[i].err = -1 is an indication for
-			   dht_selfheal_layout_new_directory() to assign
-			   a range. We set it to -1 based on any one of
-			   the three criteria:
-
-			   - err == -1 already, which means directory
-			     existed but layout was not set on it.
-
-			   - err == 0, which means directory exists and
-			     has an old layout piece which will be
-			     overwritten now.
-
-			   - err == ENOENT, which means directory does
-			     not exist (possibly racing with mkdir or
-			     finishing half done mkdir). The missing
-			     directory will be attempted to be recreated.
-			*/
-                        count++;
-			if (!err)
-				layout->list[i].err = -1;
-                }
-        }
+    dht_local_t *local = NULL;
+    uint32_t holes = 0;
+    int ret = -1;
+    int i = -1;
+    uint32_t overlaps = 0;
+
+    local = frame->local;
+
+    holes = local->selfheal.hole_cnt;
+    overlaps = local->selfheal.overlaps_cnt;
+
+    if (holes || overlaps) {
+        /* If the layout has anomalies which would change the hash
+         * ranges, then we need to reset the commit_hash for this
+         * directory, as the layout would change and things may not
+         * be in place as expected */
+        layout->commit_hash = DHT_LAYOUT_HASH_INVALID;
+        dht_selfheal_layout_new_directory(frame, loc, layout);
+        ret = 0;
+    }
+
+    for (i = 0; i < layout->cnt; i++) {
+        /* directory not present */
+        if (layout->list[i].err == ENOENT) {
+            ret = 0;
+            break;
+        }
+    }
+
+    /* TODO: give a fix to these non-virgins */
+
+    return ret;
+}
 
-        /* no subvolume has enough space, but can't stop directory creation */
-        if (!count || !new_layout) {
-                for (i = 0; i < layout->cnt; i++) {
-                        err = layout->list[i].err;
-                        if (err == ENOSPC) {
-                                layout->list[i].err = -1;
-                                count++;
-                        }
-                }
-        }
+int
+dht_selfheal_new_directory(call_frame_t *frame, dht_selfheal_dir_cbk_t dir_cbk,
+                           dht_layout_t *layout)
+{
+    dht_local_t *local = NULL;
+    int ret = 0;
+    inode_t *linked_inode = NULL, *inode = NULL;
+    loc_t *loc = NULL;
+    char pgfid[GF_UUID_BUF_SIZE] = {0};
+    char gfid[GF_UUID_BUF_SIZE] = {0};
+    int32_t op_errno = EIO;
 
-        /* if layout->spread_cnt is set, check if it is <= available
-         * subvolumes (down brick and decommissioned bricks are considered
-         * un-availbale). Else return count (available up bricks) */
-        count = ((layout->spread_cnt &&
-                 (layout->spread_cnt <= count)) ?
-                 layout->spread_cnt : ((count) ? count : 1));
+    local = frame->local;
 
-        return count;
-}
+    loc = &local->loc;
 
+    gf_uuid_unparse(local->stbuf.ia_gfid, gfid);
+    gf_uuid_unparse(loc->parent->gfid, pgfid);
 
-void dht_selfheal_layout_new_directory (call_frame_t *frame, loc_t *loc,
-					dht_layout_t *new_layout);
+    linked_inode = inode_link(loc->inode, loc->parent, loc->name,
+                              &local->stbuf);
+    if (!linked_inode) {
+        gf_smsg(frame->this->name, GF_LOG_WARNING, 0, DHT_MSG_LINK_INODE_FAILED,
+                "pgfid=%s", pgfid, "name=%s", loc->name, "gfid=%s", gfid, NULL);
+        ret = -1;
+        goto out;
+    }
 
-void dht_layout_entry_swap (dht_layout_t *layout, int i, int j);
-void dht_layout_range_swap (dht_layout_t *layout, int i, int j);
+    inode = loc->inode;
+    loc->inode = linked_inode;
+    inode_unref(inode);
 
-/*
- * It's a bit icky using local variables in a macro, but it makes the rest
- * of the code a lot clearer.
- */
-#define OV_ENTRY(x,y)      table[x*new->cnt+y]
+    local->selfheal.dir_cbk = dir_cbk;
+    local->selfheal.layout = dht_layout_ref(frame->this, layout);
 
-void
-dht_selfheal_layout_maximize_overlap (call_frame_t *frame, loc_t *loc,
-				      dht_layout_t *new, dht_layout_t *old)
-{
-	int           i            = 0;
-	int           j            = 0;
-	uint32_t      curr_overlap = 0;
-	uint32_t      max_overlap  = 0;
-	int           max_overlap_idx = -1;
-	uint32_t      overlap      = 0;
-        uint32_t     *table = NULL;
-
-	dht_layout_sort_volname (old);
-	/* Now both old_layout->list[] and new_layout->list[]
-	   are match the same xlators/subvolumes. i.e,
-	   old_layout->[i] and new_layout->[i] are referring
-	   to the same subvolumes
-	*/
-
-        /* Build a table of overlaps between new[i] and old[j]. */
-        table = alloca(sizeof(overlap)*old->cnt*new->cnt);
-        if (!table) {
-                return;
-        }
-        memset(table,0,sizeof(overlap)*old->cnt*new->cnt);
-        for (i = 0; i < new->cnt; ++i) {
-                for (j = 0; j < old->cnt; ++j) {
-                        OV_ENTRY(i,j) = dht_overlap_calc(old,j,new,i);
-                }
-        }
+    dht_layout_sort_volname(layout);
+    dht_selfheal_layout_new_directory(frame, &local->loc, layout);
 
-	for (i = 0; i < new->cnt; i++) {
-		if (new->list[i].err > 0) {
-			/* Subvol might be marked for decommission
-			   with EINVAL, or some other serious error
-			   marked with positive errno.
-			*/
-			continue;
-                }
+    op_errno = ENOMEM;
+    ret = dht_selfheal_layout_lock(frame, layout, _gf_true,
+                                   dht_selfheal_dir_xattr,
+                                   dht_should_heal_layout);
 
-                max_overlap = 0;
-                max_overlap_idx = i;
-                for (j = (i + 1); j < new->cnt; ++j) {
-                        if (new->list[j].err > 0) {
-			        /* Subvol might be marked for decommission
-			        with EINVAL, or some other serious error
-			        marked with positive errno.
-			        */
-			        continue;
-                        }
-                        /* Calculate the overlap now. */
-                        curr_overlap = OV_ENTRY(i,i) + OV_ENTRY(j,j);
-                        /* Calculate the overlap after the proposed swap. */
-                        overlap = OV_ENTRY(i,j) + OV_ENTRY(j,i);
-                        /* Are we better than status quo? */
-                        if (overlap > curr_overlap) {
-                                overlap -= curr_overlap;
-                                /* Are we better than the previous choice? */
-                                if (overlap > max_overlap) {
-                                        max_overlap = overlap;
-                                        max_overlap_idx = j;
-                                }
-                        }
-                }
+out:
+    if (ret < 0) {
+        dir_cbk(frame, NULL, frame->this, -1, op_errno, NULL);
+    }
 
-		if (max_overlap_idx != i) {
- 			dht_layout_range_swap (new, i, max_overlap_idx);
-                        /* Need to swap the table values too. */
-                        for (j = 0; j < old->cnt; ++j) {
-                                overlap = OV_ENTRY(i,j);
-                                OV_ENTRY(i,j) = OV_ENTRY(max_overlap_idx,j);
-                                OV_ENTRY(max_overlap_idx,j) = overlap;
-                        }
-                }
-	}
+    return 0;
 }
 
-
-dht_layout_t *
-dht_fix_layout_of_directory (call_frame_t *frame, loc_t *loc,
-                             dht_layout_t *layout)
+int
+dht_fix_directory_layout(call_frame_t *frame, dht_selfheal_dir_cbk_t dir_cbk,
+                         dht_layout_t *layout)
 {
-        int           i            = 0;
-        xlator_t     *this         = NULL;
-        dht_layout_t *new_layout   = NULL;
-        dht_conf_t   *priv         = NULL;
-        dht_local_t  *local        = NULL;
-        uint32_t      subvol_down  = 0;
-        int           ret          = 0;
-
-        this  = frame->this;
-        priv  = this->private;
-        local = frame->local;
-
-        if (layout->type == DHT_HASH_TYPE_DM_USER) {
-                gf_msg_debug (THIS->name, 0, "leaving %s alone",
-                              loc->path);
-                goto done;
-        }
+    dht_local_t *local = NULL;
+    dht_layout_t *tmp_layout = NULL;
+    int ret = 0;
 
-        new_layout = dht_layout_new (this, priv->subvolume_cnt);
-        if (!new_layout)
-                goto done;
-
-        /* If a subvolume is down, do not re-write the layout. */
-        ret = dht_layout_anomalies (this, loc, layout, NULL, NULL, NULL,
-                                    &subvol_down, NULL, NULL);
-
-        if (subvol_down || (ret == -1)) {
-                gf_msg (this->name, GF_LOG_WARNING, 0,
-                        DHT_MSG_LAYOUT_FIX_FAILED,
-                        "Layout fix failed: %u subvolume(s) are down"
-                        ". Skipping fix layout.", subvol_down);
-                GF_FREE (new_layout);
-                return NULL;
-        }
+    local = frame->local;
 
-        for (i = 0; i < new_layout->cnt; i++) {
-		if (layout->list[i].err != ENOSPC)
-			new_layout->list[i].err = layout->list[i].err;
-		else
-			new_layout->list[i].err = -1;
+    local->selfheal.dir_cbk = dir_cbk;
+    local->selfheal.layout = dht_layout_ref(frame->this, layout);
 
-		new_layout->list[i].xlator = layout->list[i].xlator;
-        }
+    /* No layout sorting required here */
+    tmp_layout = dht_fix_layout_of_directory(frame, &local->loc, layout);
+    if (!tmp_layout) {
+        return -1;
+    }
 
-        if (priv->du_stats) {
-                for (i = 0; i < priv->subvolume_cnt; ++i) {
-                        gf_log (this->name, GF_LOG_INFO,
-                                "subvolume %d (%s): %u chunks", i,
-                                priv->subvolumes[i]->name,
-                                priv->du_stats[i].chunks);
-                }
-        }
-        else {
-                gf_log (this->name, GF_LOG_WARNING, "no du stats ?!?");
-        }
+    ret = dht_selfheal_layout_lock(frame, tmp_layout, _gf_false,
+                                   dht_fix_dir_xattr, dht_should_fix_layout);
 
-	/* First give it a layout as though it is a new directory. This
-	   ensures rotation to kick in */
-        dht_layout_sort_volname (new_layout);
-	dht_selfheal_layout_new_directory (frame, loc, new_layout);
+    return ret;
+}
 
-	/* Now selectively re-assign ranges only when it helps */
-	dht_selfheal_layout_maximize_overlap (frame, loc, new_layout, layout);
+int
+dht_selfheal_directory(call_frame_t *frame, dht_selfheal_dir_cbk_t dir_cbk,
+                       loc_t *loc, dht_layout_t *layout)
+{
+    dht_local_t *local = NULL;
+    xlator_t *this = NULL;
+    uint32_t down = 0;
+    uint32_t misc = 0;
+    int ret = 0;
+    char pgfid[GF_UUID_BUF_SIZE] = {0};
+    char gfid[GF_UUID_BUF_SIZE] = {0};
+    inode_t *linked_inode = NULL, *inode = NULL;
+
+    local = frame->local;
+    this = frame->this;
+
+    local->selfheal.dir_cbk = dir_cbk;
+    local->selfheal.layout = dht_layout_ref(this, layout);
+
+    if (local->need_attrheal) {
+        if (__is_root_gfid(local->stbuf.ia_gfid)) {
+            local->stbuf.ia_gid = local->prebuf.ia_gid;
+            local->stbuf.ia_uid = local->prebuf.ia_uid;
+
+            local->stbuf.ia_ctime = local->prebuf.ia_ctime;
+            local->stbuf.ia_ctime_nsec = local->prebuf.ia_ctime_nsec;
+            local->stbuf.ia_prot = local->prebuf.ia_prot;
+
+        } else if (!IA_ISINVAL(local->mds_stbuf.ia_type)) {
+            local->stbuf = local->mds_stbuf;
+        }
+    }
+
+    if (!__is_root_gfid(local->stbuf.ia_gfid)) {
+        gf_uuid_unparse(local->stbuf.ia_gfid, gfid);
+        gf_uuid_unparse(loc->parent->gfid, pgfid);
+
+        linked_inode = inode_link(loc->inode, loc->parent, loc->name,
+                                  &local->stbuf);
+        if (!linked_inode) {
+            gf_smsg(this->name, GF_LOG_WARNING, 0, DHT_MSG_LINK_INODE_FAILED,
+                    "pgfid=%s", pgfid, "name=%s", loc->name, "gfid=%s", gfid,
+                    NULL);
+            ret = 0;
+            goto sorry_no_fix;
+        }
+
+        inode = loc->inode;
+        loc->inode = linked_inode;
+        inode_unref(inode);
+    }
+
+    if (local->need_xattr_heal && (local->mds_xattr)) {
+        dht_dir_set_heal_xattr(this, local, local->xattr, local->mds_xattr,
+                               NULL, NULL);
+        dict_unref(local->mds_xattr);
+        local->mds_xattr = NULL;
+    }
+
+    dht_layout_anomalies(this, loc, layout, &local->selfheal.hole_cnt,
+                         &local->selfheal.overlaps_cnt,
+                         &local->selfheal.missing_cnt, &local->selfheal.down,
+                         &local->selfheal.misc, NULL);
+
+    down = local->selfheal.down;
+    misc = local->selfheal.misc;
+
+    if (down) {
+        gf_smsg(this->name, GF_LOG_WARNING, 0, DHT_MSG_SELFHEAL_FAILED,
+                "path=%s", loc->path, "subvol-down=%d", down, "Not-fixing",
+                "gfid=%s", gfid, NULL);
+        ret = 0;
+        goto sorry_no_fix;
+    }
+
+    if (misc) {
+        gf_smsg(this->name, GF_LOG_WARNING, 0, DHT_MSG_SELFHEAL_FAILED,
+                "path=%s", loc->path, "misc=%d", misc, "unrecoverable-errors",
+                "gfid=%s", gfid, NULL);
+
+        ret = 0;
+        goto sorry_no_fix;
+    }
+
+    dht_layout_sort_volname(layout);
+    local->heal_layout = _gf_true;
+
+    /* Ignore return value as it can be inferred from result of
+     * dht_layout_anomalies
+     */
+    dht_selfheal_dir_getafix(frame, loc, layout);
+
+    if (!(local->selfheal.hole_cnt || local->selfheal.overlaps_cnt ||
+          local->selfheal.missing_cnt)) {
+        local->heal_layout = _gf_false;
+    }
+
+    ret = dht_selfheal_dir_mkdir(frame, loc, layout, 0);
+    if (ret < 0) {
+        ret = 0;
+        goto sorry_no_fix;
+    }
+
+    return 0;
 
-done:
-        if (new_layout) {
-                /* Now that the new layout has all the proper layout, change the
-                   inode context */
-                dht_layout_set (this, loc->inode, new_layout);
+sorry_no_fix:
+    /* TODO: need to put appropriate local->op_errno */
+    dht_selfheal_dir_finish(frame, this, ret, 1);
 
-                /* Make sure the extra 'ref' for existing layout is removed */
-                dht_layout_unref (this, local->layout);
+    return 0;
+}
 
-                local->layout = new_layout;
-        }
+int
+dht_selfheal_restore(call_frame_t *frame, dht_selfheal_dir_cbk_t dir_cbk,
+                     loc_t *loc, dht_layout_t *layout)
+{
+    int ret = 0;
+    dht_local_t *local = NULL;
 
-        return local->layout;
-}
+    local = frame->local;
 
+    local->selfheal.dir_cbk = dir_cbk;
+    local->selfheal.layout = dht_layout_ref(frame->this, layout);
 
-/*
- * Having to call this 2x for each entry in the layout is pretty horrible, but
- * that's what all of this layout-sorting nonsense gets us.
- */
-uint32_t
-dht_get_chunks_from_xl (xlator_t *parent, xlator_t *child)
-{
-        dht_conf_t      *priv   = parent->private;
-        xlator_list_t   *trav;
-        uint32_t        index   = 0;
+    ret = dht_selfheal_dir_mkdir(frame, loc, layout, 1);
 
-        if (!priv->du_stats) {
-                return 0;
-        }
+    return ret;
+}
 
-        for (trav = parent->children; trav; trav = trav->next) {
-                if (trav->xlator == child) {
-                        return priv->du_stats[index].chunks;
-                }
-                ++index;
-        }
+int
+dht_dir_heal_xattrs(void *data)
+{
+    call_frame_t *frame = NULL;
+    dht_local_t *local = NULL;
+    xlator_t *subvol = NULL;
+    xlator_t *mds_subvol = NULL;
+    xlator_t *this = NULL;
+    dht_conf_t *conf = NULL;
+    dict_t *user_xattr = NULL;
+    dict_t *internal_xattr = NULL;
+    dict_t *mds_xattr = NULL;
+    dict_t *xdata = NULL;
+    int call_cnt = 0;
+    int ret = -1;
+    int uret = 0;
+    int uflag = 0;
+    int i = 0;
+    int xattr_hashed = 0;
+    char gfid[GF_UUID_BUF_SIZE] = {0};
+    int32_t allzero[1] = {0};
+
+    GF_VALIDATE_OR_GOTO("dht", data, out);
+
+    frame = data;
+    local = frame->local;
+    this = frame->this;
+    GF_VALIDATE_OR_GOTO("dht", this, out);
+    GF_VALIDATE_OR_GOTO(this->name, local, out);
+    mds_subvol = local->mds_subvol;
+    conf = this->private;
+    GF_VALIDATE_OR_GOTO(this->name, conf, out);
+    gf_uuid_unparse(local->loc.gfid, gfid);
+
+    if (!mds_subvol) {
+        gf_smsg(this->name, GF_LOG_WARNING, 0, DHT_MSG_NO_MDS_SUBVOL, "path=%s",
+                local->loc.path, "gfid=%s", gfid, NULL);
+        goto out;
+    }
+
+    if ((local->loc.inode && gf_uuid_is_null(local->loc.inode->gfid)) ||
+        gf_uuid_is_null(local->loc.gfid)) {
+        gf_smsg(this->name, GF_LOG_WARNING, 0, DHT_MSG_GFID_NOT_PRESENT,
+                "skip-heal path=%s", local->loc.path, "gfid=%s", gfid, NULL);
+        goto out;
+    }
+
+    internal_xattr = dict_new();
+    if (!internal_xattr) {
+        gf_smsg(this->name, GF_LOG_ERROR, 0, DHT_MSG_CREATE_FAILED,
+                "dictionary", NULL);
+        goto out;
+    }
+    xdata = dict_new();
+    if (!xdata) {
+        gf_smsg(this->name, GF_LOG_ERROR, 0, DHT_MSG_CREATE_FAILED,
+                "dictionary", NULL);
+        goto out;
+    }
+
+    call_cnt = conf->subvolume_cnt;
+
+    user_xattr = dict_new();
+    if (!user_xattr) {
+        gf_smsg(this->name, GF_LOG_ERROR, 0, DHT_MSG_CREATE_FAILED,
+                "dictionary", NULL);
+        goto out;
+    }
+
+    ret = syncop_listxattr(local->mds_subvol, &local->loc, &mds_xattr, NULL,
+                           NULL);
+    if (ret < 0) {
+        gf_smsg(this->name, GF_LOG_ERROR, -ret, DHT_MSG_LIST_XATTRS_FAILED,
+                "path=%s", local->loc.path, "name=%s", local->mds_subvol->name,
+                NULL);
+    }
+
+    if (!mds_xattr)
+        goto out;
+
+    dht_dir_set_heal_xattr(this, local, user_xattr, mds_xattr, &uret, &uflag);
+
+    /* To set quota related xattr need to set GLUSTERFS_INTERNAL_FOP_KEY
+     * key value to 1
+     */
+    if (dict_get(user_xattr, QUOTA_LIMIT_KEY) ||
+        dict_get(user_xattr, QUOTA_LIMIT_OBJECTS_KEY)) {
+        ret = dict_set_int32(xdata, GLUSTERFS_INTERNAL_FOP_KEY, 1);
+        if (ret) {
+            gf_smsg(this->name, GF_LOG_ERROR, 0, DHT_MSG_DICT_SET_FAILED,
+                    "key=%s", GLUSTERFS_INTERNAL_FOP_KEY, "path=%s",
+                    local->loc.path, NULL);
+            goto out;
+        }
+    }
+    if (uret <= 0 && !uflag)
+        goto out;
+
+    for (i = 0; i < call_cnt; i++) {
+        subvol = conf->subvolumes[i];
+        if (subvol == mds_subvol)
+            continue;
+        if (uret || uflag) {
+            /* Custom xattr heal is required - let posix handle it */
+            ret = dict_set_int8(xdata, "sync_backend_xattrs", _gf_true);
+            if (ret) {
+                gf_smsg(this->name, GF_LOG_WARNING, 0, DHT_MSG_DICT_SET_FAILED,
+                        "path=%s", local->loc.path, "key=%s",
+                        "sync_backend_xattrs", NULL);
+                goto out;
+            }
+
+            ret = syncop_setxattr(subvol, &local->loc, user_xattr, 0, xdata,
+                                  NULL);
+            if (ret) {
+                xattr_hashed = 1;
+                gf_smsg(this->name, GF_LOG_ERROR, -ret,
+                        DHT_MSG_DIR_XATTR_HEAL_FAILED,
+                        "set-user-xattr-failed path=%s", local->loc.path,
+                        "subvol=%s", subvol->name, "gfid=%s", gfid, NULL);
+            } else {
+                dict_del(xdata, "sync_backend_xattrs");
+            }
+        }
+    }
+    /* After heal all custom xattr reset internal MDS xattr to 0 */
+    if (!xattr_hashed) {
+        ret = dht_dict_set_array(internal_xattr, conf->mds_xattr_key, allzero,
+                                 1);
+        if (ret) {
+            gf_smsg(this->name, GF_LOG_WARNING, ENOMEM, DHT_MSG_DICT_SET_FAILED,
+                    "key=%s", conf->mds_xattr_key, "path=%s", local->loc.path,
+                    NULL);
+            goto out;
+        }
+        ret = syncop_setxattr(mds_subvol, &local->loc, internal_xattr, 0, NULL,
+                              NULL);
+        if (ret) {
+            gf_smsg(this->name, GF_LOG_ERROR, -ret,
+                    DHT_MSG_DIR_XATTR_HEAL_FAILED, "path=%s", local->loc.path,
+                    "subvol=%s", mds_subvol->name, "gfid=%s", gfid, NULL);
+        }
+    }
 
-        return 0;
+out:
+    if (user_xattr)
+        dict_unref(user_xattr);
+    if (mds_xattr)
+        dict_unref(mds_xattr);
+    if (internal_xattr)
+        dict_unref(internal_xattr);
+    if (xdata)
+        dict_unref(xdata);
+    return 0;
 }
 
+int
+dht_dir_heal_xattrs_done(int ret, call_frame_t *sync_frame, void *data)
+{
+    DHT_STACK_DESTROY(sync_frame);
+    return 0;
+}
 
-void
-dht_selfheal_layout_new_directory (call_frame_t *frame, loc_t *loc,
-                                   dht_layout_t *layout)
+int
+dht_dir_attr_heal(void *data)
 {
-        xlator_t    *this = NULL;
-        uint32_t     chunk = 0;
-        int          i = 0;
-        uint32_t     start = 0;
-        int          bricks_to_use = 0;
-        int          err = 0;
-        int          start_subvol = 0;
-        uint32_t     curr_size;
-        uint32_t     total_size = 0;
-        int          real_i;
-        dht_conf_t   *priv;
-        gf_boolean_t weight_by_size;
-        int          bricks_used = 0;
-
-        this = frame->this;
-        priv = this->private;
-        weight_by_size = priv->do_weighting;
-
-        bricks_to_use = dht_get_layout_count (this, layout, 1);
-        GF_ASSERT (bricks_to_use > 0);
-
-        bricks_used = 0;
-        for (i = 0; i < layout->cnt; ++i) {
-                err = layout->list[i].err;
-                if ((err != -1) && (err != ENOENT)) {
-                        continue;
-                }
-                curr_size = dht_get_chunks_from_xl (this,
-                                                    layout->list[i].xlator);
-                if (!curr_size) {
-                        weight_by_size = _gf_false;
-                        break;
-                }
-                total_size += curr_size;
-                if (++bricks_used >= bricks_to_use) {
-                        break;
+    call_frame_t *frame = NULL;
+    dht_local_t *local = NULL;
+    xlator_t *subvol = NULL;
+    xlator_t *mds_subvol = NULL;
+    xlator_t *this = NULL;
+    dht_conf_t *conf = NULL;
+    int call_cnt = 0;
+    int ret = -1;
+    int i = 0;
+    char gfid[GF_UUID_BUF_SIZE] = {0};
+
+    GF_VALIDATE_OR_GOTO("dht", data, out);
+
+    frame = data;
+    local = frame->local;
+    this = frame->this;
+    GF_VALIDATE_OR_GOTO("dht", this, out);
+    GF_VALIDATE_OR_GOTO("dht", local, out);
+    conf = this->private;
+    GF_VALIDATE_OR_GOTO("dht", conf, out);
+
+    mds_subvol = local->mds_subvol;
+    call_cnt = conf->subvolume_cnt;
+
+    if (!__is_root_gfid(local->stbuf.ia_gfid) && (!mds_subvol)) {
+        gf_smsg(this->name, GF_LOG_WARNING, 0, DHT_MSG_NO_MDS_SUBVOL, "path=%s",
+                local->loc.path, "gfid=%s", gfid, NULL);
+        goto out;
+    }
+
+    if (!__is_root_gfid(local->stbuf.ia_gfid)) {
+        for (i = 0; i < conf->subvolume_cnt; i++) {
+            if (conf->subvolumes[i] == mds_subvol) {
+                if (!conf->subvolume_status[i]) {
+                    gf_smsg(this->name, GF_LOG_ERROR, 0,
+                            DHT_MSG_MDS_DOWN_UNABLE_TO_SET, "path=%s",
+                            local->loc.path, "gfid=%s", gfid, NULL);
+                    goto out;
                 }
+            }
+        }
+    }
+
+    for (i = 0; i < call_cnt; i++) {
+        subvol = conf->subvolumes[i];
+        if (!subvol || subvol == mds_subvol)
+            continue;
+        if (__is_root_gfid(local->stbuf.ia_gfid)) {
+            ret = syncop_setattr(
+                subvol, &local->loc, &local->stbuf,
+                (GF_SET_ATTR_UID | GF_SET_ATTR_GID | GF_SET_ATTR_MODE), NULL,
+                NULL, NULL, NULL);
+        } else {
+            ret = syncop_setattr(
+                subvol, &local->loc, &local->mds_stbuf,
+                (GF_SET_ATTR_UID | GF_SET_ATTR_GID | GF_SET_ATTR_MODE), NULL,
+                NULL, NULL, NULL);
         }
 
-        if (weight_by_size && total_size) {
-                /* We know total_size is not zero. */
-                chunk = ((unsigned long) 0xffffffff) / total_size;
-                gf_log (this->name, GF_LOG_INFO,
-                        "chunk size = 0xffffffff / %u = 0x%x",
-                        total_size, chunk);
-        }
-        else {
-                weight_by_size = _gf_false;
-                chunk = ((unsigned long) 0xffffffff) / bricks_to_use;
-        }
-
-        start_subvol = dht_selfheal_layout_alloc_start (this, loc, layout);
-
-        /* clear out the range, as we are re-computing here */
-        DHT_RESET_LAYOUT_RANGE (layout);
+        if (ret) {
+            gf_uuid_unparse(local->loc.gfid, gfid);
 
-        /*
-         * OK, what's this "real_i" stuff about?  This used to be two loops -
-         * from start_subvol to layout->cnt-1, then from 0 to start_subvol-1.
-         * That way is practically an open invitation to bugs when only one
-         * of the loops is updated.  Using real_i and modulo operators to make
-         * it one loop avoids this problem.  Remember, folks: it's everyone's
-         * responsibility to help stamp out copy/paste abuse.
-         */
-        bricks_used = 0;
-        for (real_i = 0; real_i < layout->cnt; real_i++) {
-                i = (real_i + start_subvol) % layout->cnt;
-                err = layout->list[i].err;
-                if ((err != -1) && (err != ENOENT)) {
-                        continue;
-                }
-                if (weight_by_size) {
-                        curr_size = dht_get_chunks_from_xl (this,
-                                layout->list[i].xlator);
-                        if (!curr_size) {
-                                continue;
-                        }
-                }
-                else {
-                        curr_size = 1;
-                }
-                gf_log (this->name, GF_LOG_INFO,
-                        "assigning range size 0x%x to %s", chunk * curr_size,
-                        layout->list[i].xlator->name);
-                DHT_SET_LAYOUT_RANGE(layout, i, start, chunk * curr_size,
-                                     loc->path);
-                if (++bricks_used >= bricks_to_use) {
-                        layout->list[i].stop = 0xffffffff;
-                        goto done;
-                }
-                start += (chunk * curr_size);
+            gf_smsg(this->name, GF_LOG_ERROR, -ret,
+                    DHT_MSG_DIR_ATTR_HEAL_FAILED, "path=%s", local->loc.path,
+                    "subvol=%s", subvol->name, "gfid=%s", gfid, NULL);
         }
-
-done:
-        return;
+    }
+out:
+    return 0;
 }
 
 int
-dht_selfheal_dir_getafix (call_frame_t *frame, loc_t *loc,
-                          dht_layout_t *layout)
+dht_dir_attr_heal_done(int ret, call_frame_t *sync_frame, void *data)
 {
-        dht_local_t *local = NULL;
-        uint32_t     holes = 0;
-        int          ret = -1;
-        int          i = -1;
-        uint32_t     overlaps = 0;
+    DHT_STACK_DESTROY(sync_frame);
+    return 0;
+}
 
-        local = frame->local;
+/* EXIT: dht_update_commit_hash_for_layout */
+static int
+dht_update_commit_hash_for_layout_done(call_frame_t *frame, void *cookie,
+                                       xlator_t *this, int32_t op_ret,
+                                       int32_t op_errno, dict_t *xdata)
+{
+    dht_local_t *local = NULL;
 
-        holes = local->selfheal.hole_cnt;
-        overlaps = local->selfheal.overlaps_cnt;
+    local = frame->local;
 
-        if (holes || overlaps) {
-                dht_selfheal_layout_new_directory (frame, loc, layout);
-                ret = 0;
-        }
+    /* preserve oldest error */
+    if (op_ret && !local->op_ret) {
+        local->op_ret = op_ret;
+        local->op_errno = op_errno;
+    }
 
-        for (i = 0; i < layout->cnt; i++) {
-                /* directory not present */
-                if (layout->list[i].err == ENOENT) {
-                        ret = 0;
-                        break;
-                }
-        }
-
-        /* TODO: give a fix to these non-virgins */
+    DHT_STACK_UNWIND(setxattr, frame, local->op_ret, local->op_errno, NULL);
 
-        return ret;
+    return 0;
 }
 
-int
-dht_selfheal_new_directory (call_frame_t *frame,
-                            dht_selfheal_dir_cbk_t dir_cbk,
-                            dht_layout_t *layout)
+static int
+dht_update_commit_hash_for_layout_unlock(call_frame_t *frame, xlator_t *this)
 {
-        dht_local_t *local = NULL;
-        int          ret   = 0;
-
-        local = frame->local;
+    dht_local_t *local = NULL;
+    int ret = 0;
 
-        local->selfheal.dir_cbk = dir_cbk;
-        local->selfheal.layout = dht_layout_ref (frame->this, layout);
+    local = frame->local;
 
-        dht_layout_sort_volname (layout);
-        dht_selfheal_layout_new_directory (frame, &local->loc, layout);
+    ret = dht_unlock_inodelk(frame, local->lock[0].layout.my_layout.locks,
+                             local->lock[0].layout.my_layout.lk_count,
+                             dht_update_commit_hash_for_layout_done);
+    if (ret < 0) {
+        /* preserve oldest error, just ... */
+        if (!local->op_ret) {
+            local->op_errno = errno;
+            local->op_ret = -1;
+        }
 
-        ret = dht_selfheal_layout_lock (frame, layout, _gf_true,
-                                        dht_selfheal_dir_xattr,
-                                        dht_should_heal_layout);
+        gf_smsg(this->name, GF_LOG_WARNING, errno, DHT_MSG_WIND_UNLOCK_FAILED,
+                "path=%s", local->loc.path, NULL);
 
-        if (ret < 0) {
-                dir_cbk (frame, NULL, frame->this, -1, ENOMEM, NULL);
-        }
+        dht_update_commit_hash_for_layout_done(frame, NULL, this, 0, 0, NULL);
+    }
 
-        return 0;
+    return 0;
 }
 
-int
-dht_fix_directory_layout (call_frame_t *frame,
-                          dht_selfheal_dir_cbk_t dir_cbk,
-                          dht_layout_t *layout)
+static int
+dht_update_commit_hash_for_layout_cbk(call_frame_t *frame, void *cookie,
+                                      xlator_t *this, int op_ret, int op_errno,
+                                      dict_t *xdata)
 {
-        dht_local_t  *local      = NULL;
-        dht_layout_t *tmp_layout = NULL;
-        int           ret        = 0;
+    dht_local_t *local = NULL;
+    int this_call_cnt = 0;
 
-        local = frame->local;
+    local = frame->local;
 
-        local->selfheal.dir_cbk = dir_cbk;
-        local->selfheal.layout = dht_layout_ref (frame->this, layout);
+    LOCK(&frame->lock);
+    /* store first failure, just because */
+    if (op_ret && !local->op_ret) {
+        local->op_ret = op_ret;
+        local->op_errno = op_errno;
+    }
+    UNLOCK(&frame->lock);
 
-        /* No layout sorting required here */
-        tmp_layout = dht_fix_layout_of_directory (frame, &local->loc, layout);
-        if (!tmp_layout) {
-                return -1;
-        }
+    this_call_cnt = dht_frame_return(frame);
 
-        ret = dht_selfheal_layout_lock (frame, tmp_layout, _gf_false,
-                                        dht_fix_dir_xattr,
-                                        dht_should_fix_layout);
+    if (is_last_call(this_call_cnt)) {
+        dht_update_commit_hash_for_layout_unlock(frame, this);
+    }
 
-        return ret;
+    return 0;
 }
 
-
-int
-dht_selfheal_directory (call_frame_t *frame, dht_selfheal_dir_cbk_t dir_cbk,
-                        loc_t *loc, dht_layout_t *layout)
+static int
+dht_update_commit_hash_for_layout_resume(call_frame_t *frame, void *cookie,
+                                         xlator_t *this, int32_t op_ret,
+                                         int32_t op_errno, dict_t *xdata)
 {
-        dht_local_t *local    = NULL;
-        uint32_t     down     = 0;
-        uint32_t     misc     = 0;
-        int          ret      = 0;
-        xlator_t    *this     = NULL;
-        char         gfid[GF_UUID_BUF_SIZE] = {0};
-
-        local = frame->local;
-        this = frame->this;
-
-        gf_uuid_unparse(loc->gfid, gfid);
-
-        dht_layout_anomalies (this, loc, layout,
-                              &local->selfheal.hole_cnt,
-                              &local->selfheal.overlaps_cnt,
-                              NULL, &local->selfheal.down,
-                              &local->selfheal.misc, NULL);
-
-        down     = local->selfheal.down;
-        misc     = local->selfheal.misc;
-
-        local->selfheal.dir_cbk = dir_cbk;
-        local->selfheal.layout = dht_layout_ref (this, layout);
-
-        if (down) {
-                gf_msg (this->name, GF_LOG_WARNING, 0,
-                        DHT_MSG_DIR_SELFHEAL_FAILED,
-                        "Directory selfheal failed: %d subvolumes down."
-                        "Not fixing. path = %s, gfid = %s",
-                        down, loc->path, gfid);
-                ret = 0;
-                goto sorry_no_fix;
-        }
+    dht_local_t *local = NULL;
+    int count = 1, ret = -1, i = 0, j = 0;
+    dht_conf_t *conf = NULL;
+    dht_layout_t *layout = NULL;
+    int32_t *disk_layout = NULL;
+    dict_t **xattr = NULL;
+
+    local = frame->local;
+    conf = frame->this->private;
+    count = conf->local_subvols_cnt;
+    layout = local->layout;
+
+    if (op_ret < 0) {
+        goto err_done;
+    }
+
+    /* We precreate the xattr list as we cannot change call count post the
+     * first wind as we may never continue from there. So we finish prep
+     * work before winding the setxattrs */
+    xattr = GF_CALLOC(count, sizeof(*xattr), gf_common_mt_char);
+    if (!xattr) {
+        local->op_errno = errno;
+
+        gf_smsg(this->name, GF_LOG_WARNING, errno, DHT_MSG_COMMIT_HASH_FAILED,
+                "allocation-failed path=%s", local->loc.path, NULL);
+
+        goto err;
+    }
+
+    for (i = 0; i < count; i++) {
+        /* find the layout index for the subvolume */
+        ret = dht_layout_index_for_subvol(layout, conf->local_subvols[i]);
+        if (ret < 0) {
+            local->op_errno = ENOENT;
 
-        if (misc) {
-                gf_msg (this->name, GF_LOG_WARNING, 0,
-                        DHT_MSG_DIR_SELFHEAL_FAILED,
-                        "Directory selfheal failed : %d subvolumes "
-                        "have unrecoverable errors. path = %s, gfid = %s",
-                        misc, loc->path, gfid);
+            gf_smsg(this->name, GF_LOG_WARNING, 0, DHT_MSG_COMMIT_HASH_FAILED,
+                    "path=%s", local->loc.path, "subvol=%s",
+                    conf->local_subvols[i]->name, "find-disk-layout-failed",
+                    NULL);
 
-                ret = 0;
-                goto sorry_no_fix;
+            goto err;
         }
+        j = ret;
 
-        dht_layout_sort_volname (layout);
-        ret = dht_selfheal_dir_getafix (frame, loc, layout);
+        /* update the commit hash for the layout */
+        layout->list[j].commit_hash = layout->commit_hash;
 
+        /* extract the current layout */
+        ret = dht_disk_layout_extract(this, layout, j, &disk_layout);
         if (ret == -1) {
-                gf_msg (this->name, GF_LOG_WARNING, 0,
-                        DHT_MSG_DIR_SELFHEAL_FAILED,
-                        "Directory selfheal failed: "
-                        "Unable to form layout for directory %s",
-                        loc->path);
-                goto sorry_no_fix;
-        }
+            local->op_errno = errno;
 
-        dht_selfheal_dir_mkdir (frame, loc, layout, 0);
+            gf_smsg(this->name, GF_LOG_WARNING, errno,
+                    DHT_MSG_COMMIT_HASH_FAILED, "path=%s", local->loc.path,
+                    "subvol=%s", conf->local_subvols[i]->name,
+                    "extract-disk-layout-failed", NULL);
 
-        return 0;
+            goto err;
+        }
 
-sorry_no_fix:
-        /* TODO: need to put appropriate local->op_errno */
-        dht_selfheal_dir_finish (frame, this, ret);
+        xattr[i] = dict_new();
+        if (!xattr[i]) {
+            local->op_errno = errno;
 
-        return 0;
-}
+            gf_smsg(this->name, GF_LOG_WARNING, errno,
+                    DHT_MSG_COMMIT_HASH_FAILED, "path=%s Allocation-failed",
+                    local->loc.path, NULL);
 
-int
-dht_selfheal_directory_for_nameless_lookup (call_frame_t *frame,
-                                            dht_selfheal_dir_cbk_t dir_cbk,
-                                            loc_t *loc, dht_layout_t *layout)
-{
-        dht_local_t     *local  = NULL;
-        uint32_t        down    = 0;
-        uint32_t        misc    = 0;
-        int             ret     = 0;
-        xlator_t        *this   = NULL;
-
-        local = frame->local;
-        this = frame->this;
-        dht_layout_anomalies (this, loc, layout,
-                              &local->selfheal.hole_cnt,
-                              &local->selfheal.overlaps_cnt,
-                              NULL, &local->selfheal.down,
-                              &local->selfheal.misc, NULL);
-
-        down     = local->selfheal.down;
-        misc     = local->selfheal.misc;
-
-        local->selfheal.dir_cbk = dir_cbk;
-        local->selfheal.layout = dht_layout_ref (this, layout);
-
-        if (down) {
-                gf_log (this->name, GF_LOG_WARNING,
-                        "%d subvolumes down -- not fixing", down);
-                ret = 0;
-                goto sorry_no_fix;
+            goto err;
         }
 
-        if (misc) {
-                gf_log (this->name, GF_LOG_WARNING,
-                        "%d subvolumes have unrecoverable errors", misc);
-                ret = 0;
-                goto sorry_no_fix;
-        }
+        ret = dict_set_bin(xattr[i], conf->xattr_name, disk_layout, 4 * 4);
+        if (ret != 0) {
+            local->op_errno = ENOMEM;
 
-        dht_layout_sort_volname (layout);
-        ret = dht_selfheal_dir_getafix (frame, loc, layout);
+            gf_smsg(this->name, GF_LOG_WARNING, 0,
+                    DHT_MSG_DIR_SELFHEAL_XATTR_FAILED, "path=%s",
+                    local->loc.path, "subvol=%s", conf->local_subvols[i]->name,
+                    "set-xattr-failed", NULL);
 
-        if (ret == -1) {
-                gf_log (this->name, GF_LOG_WARNING,
-                        "not able to form layout for the directory");
-                goto sorry_no_fix;
+            goto err;
         }
+        disk_layout = NULL;
 
-        ret = dht_selfheal_layout_lock (frame, layout, _gf_false,
-                                     dht_selfheal_dir_xattr_for_nameless_lookup,
-                                        dht_should_heal_layout);
-
-        if (ret < 0) {
-                goto sorry_no_fix;
+        gf_msg_trace(this->name, 0,
+                     "setting commit hash %u on subvolume %s"
+                     " for %s",
+                     layout->list[j].commit_hash, conf->local_subvols[i]->name,
+                     local->loc.path);
+    }
+
+    /* wind the setting of the commit hash across the local subvols */
+    local->call_cnt = count;
+    local->op_ret = 0;
+    local->op_errno = 0;
+    for (i = 0; i < count; i++) {
+        STACK_WIND(frame, dht_update_commit_hash_for_layout_cbk,
+                   conf->local_subvols[i],
+                   conf->local_subvols[i]->fops->setxattr, &local->loc,
+                   xattr[i], 0, NULL);
+    }
+    for (i = 0; i < count; i++)
+        dict_unref(xattr[i]);
+    GF_FREE(xattr);
+
+    return 0;
+err:
+    if (xattr) {
+        for (i = 0; i < count; i++) {
+            if (xattr[i])
+                dict_unref(xattr[i]);
         }
 
-        return 0;
+        GF_FREE(xattr);
+    }
 
-sorry_no_fix:
-        /* TODO: need to put appropriate local->op_errno */
-        dht_selfheal_dir_finish (frame, this, ret);
+    GF_FREE(disk_layout);
 
-        return 0;
+    local->op_ret = -1;
 
+    dht_update_commit_hash_for_layout_unlock(frame, this);
 
-}
-
-int
-dht_selfheal_restore (call_frame_t *frame, dht_selfheal_dir_cbk_t dir_cbk,
-                      loc_t *loc, dht_layout_t *layout)
-{
-        int          ret = 0;
-        dht_local_t *local    = NULL;
+    return 0;
+err_done:
+    local->op_ret = -1;
 
-        local = frame->local;
+    dht_update_commit_hash_for_layout_done(frame, NULL, this, 0, 0, NULL);
 
-        local->selfheal.dir_cbk = dir_cbk;
-        local->selfheal.layout = dht_layout_ref (frame->this, layout);
-
-        ret = dht_selfheal_dir_mkdir (frame, loc, layout, 1);
-
-        return ret;
+    return 0;
 }
 
+/* ENTER: dht_update_commit_hash_for_layout (see EXIT above)
+ * This function is invoked from rebalance only.
+ * As a result, the check here is simple enough to see if defrag is present
+ * in the conf, as other data would be populated appropriately if so.
+ * If ever this was to be used in other code paths, checks would need to
+ * change.
+ *
+ * Functional details:
+ *  - Lock the inodes on the subvols that we want the commit hash updated
+ *  - Update each layout with the inode layout, modified to take in the new
+ *    commit hash.
+ *  - Unlock and return.
+ */
 int
-dht_dir_attr_heal (void *data)
+dht_update_commit_hash_for_layout(call_frame_t *frame)
 {
-        call_frame_t    *frame = NULL;
-        dht_local_t     *local = NULL;
-        xlator_t        *subvol = NULL;
-        xlator_t        *this  = NULL;
-        dht_conf_t      *conf  = NULL;
-        int              call_cnt = 0;
-        int              ret   = -1;
-        int              i     = 0;
-        char         gfid[GF_UUID_BUF_SIZE] = {0};
-
-
-        GF_VALIDATE_OR_GOTO ("dht", data, out);
-
-        frame = data;
-        local = frame->local;
-        this = frame->this;
-        GF_VALIDATE_OR_GOTO ("dht", this, out);
-        GF_VALIDATE_OR_GOTO ("dht", local, out);
-        conf = this->private;
-        GF_VALIDATE_OR_GOTO ("dht", conf, out);
-
-        call_cnt = conf->subvolume_cnt;
-
-        for (i = 0; i < call_cnt; i++) {
-                subvol = conf->subvolumes[i];
-                if (!subvol || (subvol == dht_first_up_subvol (this)))
-                        continue;
-                ret = syncop_setattr (subvol, &local->loc, &local->stbuf,
-                                      (GF_SET_ATTR_UID | GF_SET_ATTR_GID),
-                                      NULL, NULL, NULL, NULL);
-                if (ret) {
-                        gf_uuid_unparse(local->loc.gfid, gfid);
-
-                        gf_msg ("dht", GF_LOG_ERROR, -ret,
-                                DHT_MSG_DIR_ATTR_HEAL_FAILED,
-                                "Directory attr heal failed. Failed to set"
-                                " uid/gid on path %s on subvol %s, gfid = %s ",
-                                local->loc.path, subvol->name, gfid);
-                }
-        }
-out:
-        return 0;
-}
+    dht_local_t *local = NULL;
+    int count = 1, ret = -1, i = 0;
+    dht_lock_t **lk_array = NULL;
+    dht_conf_t *conf = NULL;
+
+    GF_VALIDATE_OR_GOTO("dht", frame, err);
+    GF_VALIDATE_OR_GOTO(frame->this->name, frame->local, err);
+
+    local = frame->local;
+    conf = frame->this->private;
+
+    if (!conf->defrag)
+        goto err;
+
+    count = conf->local_subvols_cnt;
+    lk_array = GF_CALLOC(count, sizeof(*lk_array), gf_common_mt_char);
+    if (lk_array == NULL)
+        goto err;
+
+    for (i = 0; i < count; i++) {
+        lk_array[i] = dht_lock_new(frame->this, conf->local_subvols[i],
+                                   &local->loc, F_WRLCK, DHT_LAYOUT_HEAL_DOMAIN,
+                                   NULL, FAIL_ON_ANY_ERROR);
+        if (lk_array[i] == NULL)
+            goto err;
+    }
+
+    local->lock[0].layout.my_layout.locks = lk_array;
+    local->lock[0].layout.my_layout.lk_count = count;
+
+    ret = dht_blocking_inodelk(frame, lk_array, count,
+                               dht_update_commit_hash_for_layout_resume);
+    if (ret < 0) {
+        local->lock[0].layout.my_layout.locks = NULL;
+        local->lock[0].layout.my_layout.lk_count = 0;
+        goto err;
+    }
+
+    return 0;
+err:
+    if (lk_array != NULL) {
+        dht_lock_array_free(lk_array, count);
+        GF_FREE(lk_array);
+    }
 
-int
-dht_dir_attr_heal_done (int ret, call_frame_t *sync_frame, void *data)
-{
-        DHT_STACK_DESTROY (sync_frame);
-        return 0;
+    return -1;
 }
diff --git a/xlators/cluster/dht/src/dht-shared.c b/xlators/cluster/dht/src/dht-shared.c
index fc281b80287..bb72b0ffbb5 100644
--- a/xlators/cluster/dht/src/dht-shared.c
+++ b/xlators/cluster/dht/src/dht-shared.c
@@ -8,18 +8,14 @@
   cases as published by the Free Software Foundation.
 */
 
-
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
-
 /* TODO: add NS locking */
-
-#include "statedump.h"
+#include <glusterfs/statedump.h>
 #include "dht-common.h"
 #include "dht-messages.h"
 
+#ifndef MAX
+#define MAX(a, b) (((a) > (b)) ? (a) : (b))
+#endif
 
 /* TODO:
    - use volumename in xattr instead of "dht"
@@ -27,874 +23,1082 @@
    - handle all cases in self heal layout reconstruction
    - complete linkfile selfheal
 */
-struct volume_options options[];
 
-extern dht_methods_t dht_methods;
-
-void
-dht_layout_dump (dht_layout_t  *layout, const char *prefix)
+static void
+dht_layout_dump(dht_layout_t *layout, const char *prefix)
 {
-
-        char    key[GF_DUMP_MAX_BUF_LEN];
-        int     i = 0;
-
-        if (!layout)
-                goto out;
-        if (!prefix)
-                goto out;
-
-        gf_proc_dump_build_key(key, prefix, "cnt");
-        gf_proc_dump_write(key, "%d", layout->cnt);
-        gf_proc_dump_build_key(key, prefix, "preset");
-        gf_proc_dump_write(key, "%d", layout->preset);
-        gf_proc_dump_build_key(key, prefix, "gen");
-        gf_proc_dump_write(key, "%d", layout->gen);
-        if (layout->type != IA_INVAL) {
-                gf_proc_dump_build_key(key, prefix, "inode type");
-                gf_proc_dump_write(key, "%d", layout->type);
-        }
-
-        if  (!IA_ISDIR (layout->type))
-                goto out;
-
-        for (i = 0; i < layout->cnt; i++) {
-                gf_proc_dump_build_key(key, prefix,"list[%d].err", i);
-                gf_proc_dump_write(key, "%d", layout->list[i].err);
-                gf_proc_dump_build_key(key, prefix,"list[%d].start", i);
-                gf_proc_dump_write(key, "%u", layout->list[i].start);
-                gf_proc_dump_build_key(key, prefix,"list[%d].stop", i);
-                gf_proc_dump_write(key, "%u", layout->list[i].stop);
-                if (layout->list[i].xlator) {
-                        gf_proc_dump_build_key(key, prefix,
-                                               "list[%d].xlator.type", i);
-                        gf_proc_dump_write(key, "%s",
-                                           layout->list[i].xlator->type);
-                        gf_proc_dump_build_key(key, prefix,
-                                               "list[%d].xlator.name", i);
-                        gf_proc_dump_write(key, "%s",
-                                           layout->list[i].xlator->name);
-                }
+    char key[GF_DUMP_MAX_BUF_LEN];
+    int i = 0;
+
+    if (!layout)
+        goto out;
+
+    gf_proc_dump_build_key(key, prefix, "cnt");
+    gf_proc_dump_write(key, "%d", layout->cnt);
+    gf_proc_dump_build_key(key, prefix, "preset");
+    gf_proc_dump_write(key, "%d", layout->preset);
+    gf_proc_dump_build_key(key, prefix, "gen");
+    gf_proc_dump_write(key, "%d", layout->gen);
+    if (layout->type != IA_INVAL) {
+        gf_proc_dump_build_key(key, prefix, "inode type");
+        gf_proc_dump_write(key, "%d", layout->type);
+    }
+
+    if (!IA_ISDIR(layout->type))
+        goto out;
+
+    for (i = 0; i < layout->cnt; i++) {
+        gf_proc_dump_build_key(key, prefix, "list[%d].err", i);
+        gf_proc_dump_write(key, "%d", layout->list[i].err);
+        gf_proc_dump_build_key(key, prefix, "list[%d].start", i);
+        gf_proc_dump_write(key, "0x%x", layout->list[i].start);
+        gf_proc_dump_build_key(key, prefix, "list[%d].stop", i);
+        gf_proc_dump_write(key, "0x%x", layout->list[i].stop);
+        if (layout->list[i].xlator) {
+            gf_proc_dump_build_key(key, prefix, "list[%d].xlator.type", i);
+            gf_proc_dump_write(key, "%s", layout->list[i].xlator->type);
+            gf_proc_dump_build_key(key, prefix, "list[%d].xlator.name", i);
+            gf_proc_dump_write(key, "%s", layout->list[i].xlator->name);
         }
+    }
 
 out:
-        return;
+    return;
 }
 
-
 int32_t
-dht_priv_dump (xlator_t *this)
+dht_priv_dump(xlator_t *this)
 {
-        char            key_prefix[GF_DUMP_MAX_BUF_LEN];
-        char            key[GF_DUMP_MAX_BUF_LEN];
-        int             i = 0;
-        dht_conf_t      *conf = NULL;
-        int             ret = -1;
+    char key_prefix[GF_DUMP_MAX_BUF_LEN];
+    char key[GF_DUMP_MAX_BUF_LEN];
+    int i = 0;
+    dht_conf_t *conf = NULL;
+    int ret = -1;
 
-        if (!this)
-                goto out;
+    if (!this)
+        goto out;
 
-        conf = this->private;
-        if (!conf)
-                goto out;
+    conf = this->private;
+    if (!conf)
+        goto out;
 
-        ret = TRY_LOCK(&conf->subvolume_lock);
-        if (ret != 0) {
-                return ret;
+    ret = TRY_LOCK(&conf->subvolume_lock);
+    if (ret != 0) {
+        return ret;
+    }
+
+    gf_proc_dump_add_section("xlator.cluster.dht.%s.priv", this->name);
+    gf_proc_dump_build_key(key_prefix, "xlator.cluster.dht", "%s.priv",
+                           this->name);
+    gf_proc_dump_write("subvol_cnt", "%d", conf->subvolume_cnt);
+    for (i = 0; i < conf->subvolume_cnt; i++) {
+        snprintf(key, sizeof(key), "subvolumes[%d]", i);
+        gf_proc_dump_write(key, "%s.%s", conf->subvolumes[i]->type,
+                           conf->subvolumes[i]->name);
+        if (conf->file_layouts && conf->file_layouts[i]) {
+            snprintf(key, sizeof(key), "file_layouts[%d]", i);
+            dht_layout_dump(conf->file_layouts[i], key);
         }
-
-        gf_proc_dump_add_section("xlator.cluster.dht.%s.priv", this->name);
-        gf_proc_dump_build_key(key_prefix,"xlator.cluster.dht","%s.priv",
-                               this->name);
-        gf_proc_dump_write("subvol_cnt","%d", conf->subvolume_cnt);
+        if (conf->dir_layouts && conf->dir_layouts[i]) {
+            snprintf(key, sizeof(key), "dir_layouts[%d]", i);
+            dht_layout_dump(conf->dir_layouts[i], key);
+        }
+        if (conf->subvolume_status) {
+            snprintf(key, sizeof(key), "subvolume_status[%d]", i);
+            gf_proc_dump_write(key, "%d", (int)conf->subvolume_status[i]);
+        }
+    }
+
+    gf_proc_dump_write("search_unhashed", "%d", conf->search_unhashed);
+    gf_proc_dump_write("gen", "%d", conf->gen);
+    gf_proc_dump_write("min_free_disk", "%lf", conf->min_free_disk);
+    gf_proc_dump_write("min_free_inodes", "%lf", conf->min_free_inodes);
+    gf_proc_dump_write("disk_unit", "%c", conf->disk_unit);
+    gf_proc_dump_write("refresh_interval", "%d", conf->refresh_interval);
+    gf_proc_dump_write("unhashed_sticky_bit", "%d", conf->unhashed_sticky_bit);
+    gf_proc_dump_write("use-readdirp", "%d", conf->use_readdirp);
+
+    if (conf->du_stats && conf->subvolume_status) {
         for (i = 0; i < conf->subvolume_cnt; i++) {
-                snprintf (key, sizeof (key), "subvolumes[%d]", i);
-                gf_proc_dump_write(key, "%s.%s", conf->subvolumes[i]->type,
-                                   conf->subvolumes[i]->name);
-                if (conf->file_layouts && conf->file_layouts[i]){
-                        snprintf (key, sizeof (key), "file_layouts[%d]", i);
-                        dht_layout_dump(conf->file_layouts[i], key);
-                }
-                if (conf->dir_layouts && conf->dir_layouts[i]) {
-                        snprintf (key, sizeof (key), "dir_layouts[%d]", i);
-                        dht_layout_dump(conf->dir_layouts[i], key);
-                }
-                if (conf->subvolume_status) {
-
-                        snprintf (key, sizeof (key), "subvolume_status[%d]", i);
-                        gf_proc_dump_write(key, "%d",
-                                           (int)conf->subvolume_status[i]);
-                }
+            if (!conf->subvolume_status[i])
+                continue;
 
-        }
+            snprintf(key, sizeof(key), "subvolumes[%d]", i);
+            gf_proc_dump_write(key, "%s", conf->subvolumes[i]->name);
+
+            snprintf(key, sizeof(key), "du_stats[%d].avail_percent", i);
+            gf_proc_dump_write(key, "%lf", conf->du_stats[i].avail_percent);
 
-        gf_proc_dump_write("search_unhashed", "%d", conf->search_unhashed);
-        gf_proc_dump_write("gen", "%d", conf->gen);
-        gf_proc_dump_write("min_free_disk", "%lf", conf->min_free_disk);
-	gf_proc_dump_write("min_free_inodes", "%lf", conf->min_free_inodes);
-        gf_proc_dump_write("disk_unit", "%c", conf->disk_unit);
-        gf_proc_dump_write("refresh_interval", "%d", conf->refresh_interval);
-        gf_proc_dump_write("unhashed_sticky_bit", "%d", conf->unhashed_sticky_bit);
-
-        if (conf->du_stats) {
-                for (i = 0; i < conf->subvolume_cnt; i++) {
-                        if (!conf->subvolume_status[i])
-                                continue;
-
-                        snprintf (key, sizeof (key), "subvolumes[%d]", i);
-                        gf_proc_dump_write (key, "%s",
-                                            conf->subvolumes[i]->name);
-
-                        snprintf (key, sizeof (key),
-                                  "du_stats[%d].avail_percent", i);
-                        gf_proc_dump_write (key, "%lf",
-                                            conf->du_stats[i].avail_percent);
-
-                        snprintf (key, sizeof (key), "du_stats[%d].avail_space",
-                                  i);
-                        gf_proc_dump_write (key, "%lu",
-                                            conf->du_stats[i].avail_space);
-
-                        snprintf (key, sizeof (key),
-                                  "du_stats[%d].avail_inodes", i);
-                        gf_proc_dump_write (key, "%lf",
-                                            conf->du_stats[i].avail_inodes);
-
-                        snprintf (key, sizeof (key), "du_stats[%d].log", i);
-                        gf_proc_dump_write (key, "%lu",
-                                            conf->du_stats[i].log);
-                }
+            snprintf(key, sizeof(key), "du_stats[%d].avail_space", i);
+            gf_proc_dump_write(key, "%" PRIu64, conf->du_stats[i].avail_space);
+
+            snprintf(key, sizeof(key), "du_stats[%d].avail_inodes", i);
+            gf_proc_dump_write(key, "%lf", conf->du_stats[i].avail_inodes);
+
+            snprintf(key, sizeof(key), "du_stats[%d].log", i);
+            gf_proc_dump_write(key, "%" PRIu32, conf->du_stats[i].log);
         }
+    }
 
-        if (conf->last_stat_fetch.tv_sec)
-                gf_proc_dump_write("last_stat_fetch", "%s",
-                                    ctime(&conf->last_stat_fetch.tv_sec));
+    if (conf->last_stat_fetch)
+        gf_proc_dump_write("last_stat_fetch", "%s",
+                           ctime(&conf->last_stat_fetch));
 
-        UNLOCK(&conf->subvolume_lock);
+    UNLOCK(&conf->subvolume_lock);
 
 out:
-        return ret;
+    return ret;
 }
 
 int32_t
-dht_inodectx_dump (xlator_t *this, inode_t *inode)
+dht_inodectx_dump(xlator_t *this, inode_t *inode)
 {
-        int             ret = -1;
-        dht_layout_t    *layout = NULL;
+    int ret = -1;
+    dht_layout_t *layout = NULL;
 
-        if (!this)
-                goto out;
-        if (!inode)
-                goto out;
+    if (!this)
+        goto out;
+    if (!inode)
+        goto out;
 
-        ret = dht_inode_ctx_layout_get (inode, this, &layout);
+    ret = dht_inode_ctx_layout_get(inode, this, &layout);
 
-        if ((ret != 0) || !layout)
-                return ret;
+    if ((ret != 0) || !layout)
+        return ret;
 
-        gf_proc_dump_add_section("xlator.cluster.dht.%s.inode", this->name);
-        dht_layout_dump(layout, "layout");
+    gf_proc_dump_add_section("xlator.cluster.dht.%s.inode", this->name);
+    dht_layout_dump(layout, "layout");
 
 out:
-        return ret;
+    return ret;
 }
 
 void
-dht_fini (xlator_t *this)
+dht_fini(xlator_t *this)
 {
-        int         i = 0;
-        dht_conf_t *conf = NULL;
+    int i = 0;
+    dht_conf_t *conf = NULL;
+
+    GF_VALIDATE_OR_GOTO("dht", this, out);
+
+    conf = this->private;
+    this->private = NULL;
+    if (conf) {
+        if (conf->file_layouts) {
+            for (i = 0; i < conf->subvolume_cnt; i++) {
+                GF_FREE(conf->file_layouts[i]);
+            }
+            GF_FREE(conf->file_layouts);
+        }
 
-        GF_VALIDATE_OR_GOTO ("dht", this, out);
+        dict_unref(conf->leaf_to_subvol);
 
-        conf = this->private;
-        this->private = NULL;
-        if (conf) {
-                if (conf->file_layouts) {
-                        for (i = 0; i < conf->subvolume_cnt; i++) {
-                                GF_FREE (conf->file_layouts[i]);
-                        }
-                        GF_FREE (conf->file_layouts);
-                }
+        /* allocated in dht_init_subvolumes() */
+        GF_FREE(conf->subvolumes);
+        GF_FREE(conf->subvolume_status);
+        GF_FREE(conf->last_event);
+        GF_FREE(conf->subvol_up_time);
+        GF_FREE(conf->du_stats);
+        GF_FREE(conf->decommissioned_bricks);
 
-                dict_destroy(conf->leaf_to_subvol);
+        /* allocated in dht_init() */
+        GF_FREE(conf->mds_xattr_key);
+        GF_FREE(conf->link_xattr_name);
+        GF_FREE(conf->commithash_xattr_name);
+        GF_FREE(conf->wild_xattr_name);
 
-                GF_FREE (conf->subvolumes);
+        /* allocated in dht_init_regex() */
+        if (conf->rsync_regex_valid)
+            regfree(&conf->rsync_regex);
+        if (conf->extra_regex_valid)
+            regfree(&conf->extra_regex);
 
-                GF_FREE (conf->subvolume_status);
+        synclock_destroy(&conf->link_lock);
 
-                if (conf->lock_pool)
-                        mem_pool_destroy (conf->lock_pool);
+        if (conf->lock_pool)
+            mem_pool_destroy(conf->lock_pool);
 
-                GF_FREE (conf);
-        }
+        GF_FREE(conf);
+    }
 out:
-        return;
+    return;
 }
 
 int32_t
-mem_acct_init (xlator_t *this)
+mem_acct_init(xlator_t *this)
 {
-        int     ret = -1;
+    int ret = -1;
 
-        GF_VALIDATE_OR_GOTO ("dht", this, out);
+    GF_VALIDATE_OR_GOTO("dht", this, out);
 
-        ret = xlator_mem_acct_init (this, gf_dht_mt_end + 1);
+    ret = xlator_mem_acct_init(this, gf_dht_mt_end + 1);
 
-        if (ret != 0) {
-                gf_log (this->name, GF_LOG_ERROR, "Memory accounting init"
-                        "failed");
-                return ret;
-        }
-out:
+    if (ret != 0) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, DHT_MSG_NO_MEMORY,
+               "Memory accounting init failed");
         return ret;
+    }
+out:
+    return ret;
 }
 
-
-int
-dht_parse_decommissioned_bricks (xlator_t *this, dht_conf_t *conf,
-                                 const char *bricks)
+static int
+dht_parse_decommissioned_bricks(xlator_t *this, dht_conf_t *conf,
+                                const char *bricks)
 {
-        int         i  = 0;
-        int         ret  = -1;
-        char       *tmpstr = NULL;
-        char       *dup_brick = NULL;
-        char       *node = NULL;
-
-        if (!conf || !bricks)
-                goto out;
-
-        dup_brick = gf_strdup (bricks);
-        node = strtok_r (dup_brick, ",", &tmpstr);
-        while (node) {
-                for (i = 0; i < conf->subvolume_cnt; i++) {
-                        if (!strcmp (conf->subvolumes[i]->name, node)) {
-                                conf->decommissioned_bricks[i] =
-                                        conf->subvolumes[i];
-                                        conf->decommission_subvols_cnt++;
-                                gf_log (this->name, GF_LOG_INFO,
-                                        "decommissioning subvolume %s",
-                                        conf->subvolumes[i]->name);
-                                break;
-                        }
-                }
-                if (i == conf->subvolume_cnt) {
-                        /* Wrong node given. */
-                        goto out;
-                }
-                node = strtok_r (NULL, ",", &tmpstr);
+    int i = 0;
+    int ret = -1;
+    char *tmpstr = NULL;
+    char *dup_brick = NULL;
+    char *node = NULL;
+
+    if (!conf || !bricks)
+        goto out;
+
+    dup_brick = gf_strdup(bricks);
+    if (dup_brick == NULL) {
+        goto out;
+    }
+
+    node = strtok_r(dup_brick, ",", &tmpstr);
+    while (node) {
+        for (i = 0; i < conf->subvolume_cnt; i++) {
+            if (!strcmp(conf->subvolumes[i]->name, node)) {
+                conf->decommissioned_bricks[i] = conf->subvolumes[i];
+                conf->decommission_subvols_cnt++;
+                gf_msg(this->name, GF_LOG_INFO, 0,
+                       DHT_MSG_SUBVOL_DECOMMISSION_INFO,
+                       "decommissioning subvolume %s",
+                       conf->subvolumes[i]->name);
+                break;
+            }
         }
+        if (i == conf->subvolume_cnt) {
+            /* Wrong node given. */
+            goto out;
+        }
+        node = strtok_r(NULL, ",", &tmpstr);
+    }
 
-        ret = 0;
-        conf->decommission_in_progress = 1;
+    ret = 0;
+    conf->decommission_in_progress = 1;
 out:
-        GF_FREE (dup_brick);
+    GF_FREE(dup_brick);
 
-        return ret;
+    return ret;
 }
 
-int
-dht_decommissioned_remove (xlator_t *this, dht_conf_t *conf)
+static void
+dht_decommissioned_remove(xlator_t *this, dht_conf_t *conf)
 {
-        int         i  = 0;
-        int         ret  = -1;
-
-        if (!conf)
-                goto out;
+    int i = 0;
 
-        for (i = 0; i < conf->subvolume_cnt; i++) {
-                if (conf->decommissioned_bricks[i]) {
-                        conf->decommissioned_bricks[i] = NULL;
-                        conf->decommission_subvols_cnt--;
-                }
+    for (i = 0; i < conf->subvolume_cnt; i++) {
+        if (conf->decommissioned_bricks[i]) {
+            conf->decommissioned_bricks[i] = NULL;
+            conf->decommission_subvols_cnt--;
         }
-
-        ret = 0;
-out:
-
-        return ret;
+    }
 }
-void
-dht_init_regex (xlator_t *this, dict_t *odict, char *name,
-                regex_t *re, gf_boolean_t *re_valid)
+
+static void
+dht_init_regex(xlator_t *this, dict_t *odict, char *name, regex_t *re,
+               gf_boolean_t *re_valid, dht_conf_t *conf)
 {
-        char    *temp_str;
+    char *temp_str = NULL;
 
-        if (dict_get_str (odict, name, &temp_str) != 0) {
-                if (strcmp(name,"rsync-hash-regex")) {
-                        return;
-                }
-                temp_str = "^\\.(.+)\\.[^.]+$";
+    if (dict_get_str(odict, name, &temp_str) != 0) {
+        if (strcmp(name, "rsync-hash-regex")) {
+            return;
         }
+        temp_str = "^\\.(.+)\\.[^.]+$";
+    }
 
+    LOCK(&conf->lock);
+    {
         if (*re_valid) {
-                regfree(re);
-                *re_valid = _gf_false;
+            regfree(re);
+            *re_valid = _gf_false;
         }
 
-        if (!strcmp(temp_str,"none")) {
-                return;
+        if (!strcmp(temp_str, "none")) {
+            goto unlock;
         }
 
-        if (regcomp(re,temp_str,REG_EXTENDED) == 0) {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "using regex %s = %s", name, temp_str);
-                *re_valid = _gf_true;
-        }
-        else {
-                gf_log (this->name, GF_LOG_WARNING,
-                        "compiling regex %s failed", temp_str);
+        if (regcomp(re, temp_str, REG_EXTENDED) == 0) {
+            gf_msg_debug(this->name, 0, "using regex %s = %s", name, temp_str);
+            *re_valid = _gf_true;
+        } else {
+            gf_msg(this->name, GF_LOG_WARNING, 0, DHT_MSG_REGEX_INFO,
+                   "compiling regex %s failed", temp_str);
         }
+    }
+unlock:
+    UNLOCK(&conf->lock);
 }
 
 int
 dht_set_subvol_range(xlator_t *this)
 {
-        int ret = -1;
-        dht_conf_t *conf = NULL;
+    int ret = -1;
+    dht_conf_t *conf = NULL;
 
-        conf = this->private;
+    conf = this->private;
 
-        if (!conf)
-                goto out;
+    if (!conf)
+        goto out;
 
-        conf->leaf_to_subvol = dict_new();
-        if (!conf->leaf_to_subvol)
-                goto out;
+    conf->leaf_to_subvol = dict_new();
+    if (!conf->leaf_to_subvol)
+        goto out;
 
-        ret = glusterfs_reachable_leaves(this, conf->leaf_to_subvol);
+    ret = glusterfs_reachable_leaves(this, conf->leaf_to_subvol);
 
 out:
-        return ret;
+    return ret;
+}
+
+static int
+dht_configure_throttle(xlator_t *this, dht_conf_t *conf, char *temp_str)
+{
+    int rebal_thread_count = 0;
+    int ret = 0;
+
+    pthread_mutex_lock(&conf->defrag->dfq_mutex);
+    {
+        if (!strcasecmp(temp_str, "lazy")) {
+            conf->defrag->recon_thread_count = 1;
+        } else if (!strcasecmp(temp_str, "normal")) {
+            conf->defrag->recon_thread_count = 2;
+        } else if (!strcasecmp(temp_str, "aggressive")) {
+            conf->defrag->recon_thread_count = MAX(MAX_REBAL_THREADS - 4, 4);
+        } else if ((gf_string2int(temp_str, &rebal_thread_count) == 0)) {
+            if ((rebal_thread_count > 0) &&
+                (rebal_thread_count <= MAX_REBAL_THREADS)) {
+                conf->defrag->recon_thread_count = rebal_thread_count;
+                pthread_mutex_unlock(&conf->defrag->dfq_mutex);
+                gf_msg(this->name, GF_LOG_INFO, 0, 0,
+                       "rebal thread count configured to %d",
+                       rebal_thread_count);
+                goto out;
+            } else {
+                pthread_mutex_unlock(&conf->defrag->dfq_mutex);
+                gf_msg(this->name, GF_LOG_ERROR, 0, DHT_MSG_INVALID_OPTION,
+                       "Invalid option: Reconfigure: "
+                       "rebal-throttle should be "
+                       "within range of 0 and maximum number of"
+                       " cores available");
+                ret = -1;
+                goto out;
+            }
+        } else {
+            gf_msg(this->name, GF_LOG_ERROR, 0, DHT_MSG_INVALID_OPTION,
+                   "Invalid option: Reconfigure: "
+                   "rebal-throttle should be {lazy|normal|aggressive}"
+                   " or a number up to the number of cores available,"
+                   " not (%s), defaulting to (%d)",
+                   temp_str, conf->dthrottle);
+            ret = -1;
+        }
+    }
+    pthread_mutex_unlock(&conf->defrag->dfq_mutex);
+
+out:
+    return ret;
 }
 
 int
-dht_reconfigure (xlator_t *this, dict_t *options)
+dht_reconfigure(xlator_t *this, dict_t *options)
 {
-        dht_conf_t      *conf = NULL;
-        char            *temp_str = NULL;
-        gf_boolean_t     search_unhashed;
-        int              ret = -1;
-
-        GF_VALIDATE_OR_GOTO ("dht", this, out);
-        GF_VALIDATE_OR_GOTO ("dht", options, out);
-
-        conf = this->private;
-        if (!conf)
-                return 0;
-
-        if (dict_get_str (options, "lookup-unhashed", &temp_str) == 0) {
-                /* If option is not "auto", other options _should_ be boolean*/
-                if (strcasecmp (temp_str, "auto")) {
-                        if (!gf_string2boolean (temp_str, &search_unhashed)) {
-                                gf_msg_debug(this->name, 0, "Reconfigure: "
-                                             "lookup-unhashed reconfigured(%s)",
-                                             temp_str);
-                                conf->search_unhashed = search_unhashed;
-                        } else {
-                                gf_msg(this->name, GF_LOG_ERROR, 0,
-                                       DHT_MSG_INVALID_OPTION,
-                                       "Invalid option: Reconfigure: "
-                                       "lookup-unhashed should be boolean,"
-                                       " not (%s), defaulting to (%d)",
-                                       temp_str, conf->search_unhashed);
-                                ret = -1;
-                                goto out;
-                        }
-                } else {
-                        gf_msg_debug(this->name, 0, "Reconfigure:"
-                                     " lookup-unhashed reconfigured auto ");
-                        conf->search_unhashed = GF_DHT_LOOKUP_UNHASHED_AUTO;
-                }
+    dht_conf_t *conf = NULL;
+    char *temp_str = NULL;
+    gf_boolean_t search_unhashed;
+    int ret = -1;
+
+    GF_VALIDATE_OR_GOTO("dht", this, out);
+    GF_VALIDATE_OR_GOTO("dht", options, out);
+
+    conf = this->private;
+    if (!conf)
+        return 0;
+
+    if (dict_get_str(options, "lookup-unhashed", &temp_str) == 0) {
+        /* If option is not "auto", other options _should_ be boolean*/
+        if (strcasecmp(temp_str, "auto")) {
+            if (!gf_string2boolean(temp_str, &search_unhashed)) {
+                gf_msg_debug(this->name, 0,
+                             "Reconfigure: "
+                             "lookup-unhashed reconfigured(%s)",
+                             temp_str);
+                conf->search_unhashed = search_unhashed;
+            } else {
+                gf_msg(this->name, GF_LOG_ERROR, 0, DHT_MSG_INVALID_OPTION,
+                       "Invalid option: Reconfigure: "
+                       "lookup-unhashed should be boolean,"
+                       " not (%s), defaulting to (%d)",
+                       temp_str, conf->search_unhashed);
+                ret = -1;
+                goto out;
+            }
+        } else {
+            gf_msg_debug(this->name, 0,
+                         "Reconfigure:"
+                         " lookup-unhashed reconfigured auto ");
+            conf->search_unhashed = GF_DHT_LOOKUP_UNHASHED_AUTO;
         }
+    }
+
+    GF_OPTION_RECONF("lookup-optimize", conf->lookup_optimize, options, bool,
+                     out);
+
+    GF_OPTION_RECONF("min-free-disk", conf->min_free_disk, options,
+                     percent_or_size, out);
+    /* option can be any one of percent or bytes */
+    conf->disk_unit = 0;
+    if (conf->min_free_disk < 100.0)
+        conf->disk_unit = 'p';
 
-	GF_OPTION_RECONF ("min-free-disk", conf->min_free_disk, options,
-                          percent_or_size, out);
-        /* option can be any one of percent or bytes */
-        conf->disk_unit = 0;
-        if (conf->min_free_disk < 100.0)
-                conf->disk_unit = 'p';
+    GF_OPTION_RECONF("min-free-inodes", conf->min_free_inodes, options, percent,
+                     out);
 
-	GF_OPTION_RECONF ("min-free-inodes", conf->min_free_inodes, options,
-                          percent, out);
+    GF_OPTION_RECONF("directory-layout-spread", conf->dir_spread_cnt, options,
+                     uint32, out);
 
-        GF_OPTION_RECONF ("directory-layout-spread", conf->dir_spread_cnt,
-                          options, uint32, out);
+    GF_OPTION_RECONF("readdir-optimize", conf->readdir_optimize, options, bool,
+                     out);
+    GF_OPTION_RECONF("randomize-hash-range-by-gfid", conf->randomize_by_gfid,
+                     options, bool, out);
 
-        GF_OPTION_RECONF ("readdir-optimize", conf->readdir_optimize, options,
-                          bool, out);
-        GF_OPTION_RECONF ("randomize-hash-range-by-gfid",
-                          conf->randomize_by_gfid,
-                          options, bool, out);
+    GF_OPTION_RECONF("lock-migration", conf->lock_migration_enabled, options,
+                     bool, out);
 
-        if (conf->defrag) {
-                GF_OPTION_RECONF ("rebalance-stats", conf->defrag->stats,
-                                  options, bool, out);
+    GF_OPTION_RECONF("force-migration", conf->force_migration, options, bool,
+                     out);
+
+    if (conf->defrag) {
+        if (dict_get_str(options, "rebal-throttle", &temp_str) == 0) {
+            ret = dht_configure_throttle(this, conf, temp_str);
+            if (ret == -1)
+                goto out;
         }
+    }
+
+    if (conf->defrag) {
+        conf->defrag->lock_migration_enabled = conf->lock_migration_enabled;
+    }
+
+    if (conf->defrag) {
+        GF_OPTION_RECONF("rebalance-stats", conf->defrag->stats, options, bool,
+                         out);
+    }
+
+    if (dict_get_str(options, "decommissioned-bricks", &temp_str) == 0) {
+        ret = dht_parse_decommissioned_bricks(this, conf, temp_str);
+        if (ret == -1)
+            goto out;
+    } else {
+        dht_decommissioned_remove(this, conf);
+    }
+
+    dht_init_regex(this, options, "rsync-hash-regex", &conf->rsync_regex,
+                   &conf->rsync_regex_valid, conf);
+    dht_init_regex(this, options, "extra-hash-regex", &conf->extra_regex,
+                   &conf->extra_regex_valid, conf);
+
+    GF_OPTION_RECONF("weighted-rebalance", conf->do_weighting, options, bool,
+                     out);
+
+    GF_OPTION_RECONF("use-readdirp", conf->use_readdirp, options, bool, out);
+    ret = 0;
+out:
+    return ret;
+}
 
-        if (dict_get_str (options, "decommissioned-bricks", &temp_str) == 0) {
-                ret = dht_parse_decommissioned_bricks (this, conf, temp_str);
-                if (ret == -1)
-                        goto out;
-        } else {
-                ret = dht_decommissioned_remove (this, conf);
-                if (ret == -1)
-                        goto out;
+static int
+gf_defrag_pattern_list_fill(xlator_t *this, gf_defrag_info_t *defrag,
+                            char *data)
+{
+    int ret = -1;
+    char *tmp_str = NULL;
+    char *tmp_str1 = NULL;
+    char *dup_str = NULL;
+    char *num = NULL;
+    char *pattern_str = NULL;
+    char *pattern = NULL;
+    gf_defrag_pattern_list_t *temp_list = NULL;
+    gf_defrag_pattern_list_t *pattern_list = NULL;
+
+    if (!this || !defrag || !data)
+        goto out;
+
+    /* Get the pattern for pattern list. "pattern:<optional-size>"
+     * eg: *avi, *pdf:10MB, *:1TB
+     */
+    pattern_str = strtok_r(data, ",", &tmp_str);
+    while (pattern_str) {
+        dup_str = gf_strdup(pattern_str);
+        if (!dup_str)
+            goto out;
+        pattern_list = GF_CALLOC(1, sizeof(gf_defrag_pattern_list_t), 1);
+        if (!pattern_list) {
+            goto out;
+        }
+        pattern = strtok_r(dup_str, ":", &tmp_str1);
+        num = strtok_r(NULL, ":", &tmp_str1);
+        if (!pattern)
+            goto out;
+        if (!num) {
+            if (gf_string2bytesize_uint64(pattern, &pattern_list->size) == 0) {
+                pattern = "*";
+            }
+        } else if (gf_string2bytesize_uint64(num, &pattern_list->size) != 0) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, DHT_MSG_INVALID_OPTION,
+                   "Invalid option. Defrag pattern:"
+                   " Invalid number format \"%s\"",
+                   num);
+            goto out;
         }
+        memcpy(pattern_list->path_pattern, pattern, strlen(dup_str));
 
-        dht_init_regex (this, options, "rsync-hash-regex",
-                        &conf->rsync_regex, &conf->rsync_regex_valid);
-        dht_init_regex (this, options, "extra-hash-regex",
-                        &conf->extra_regex, &conf->extra_regex_valid);
+        if (!defrag->defrag_pattern)
+            temp_list = NULL;
+        else
+            temp_list = defrag->defrag_pattern;
 
-        GF_OPTION_RECONF ("weighted-rebalance", conf->do_weighting, options,
-                          bool, out);
+        pattern_list->next = temp_list;
 
-        ret = 0;
+        defrag->defrag_pattern = pattern_list;
+        pattern_list = NULL;
+
+        GF_FREE(dup_str);
+        dup_str = NULL;
+
+        pattern_str = strtok_r(NULL, ",", &tmp_str);
+    }
+
+    ret = 0;
 out:
-        return ret;
+    if (ret)
+        GF_FREE(pattern_list);
+    GF_FREE(dup_str);
+
+    return ret;
 }
 
 static int
-gf_defrag_pattern_list_fill (xlator_t *this, gf_defrag_info_t *defrag, char *data)
+dht_init_methods(xlator_t *this)
 {
-        int                    ret = -1;
-        char                  *tmp_str = NULL;
-        char                  *tmp_str1 = NULL;
-        char                  *dup_str = NULL;
-        char                  *num = NULL;
-        char                  *pattern_str = NULL;
-        char                  *pattern = NULL;
-        gf_defrag_pattern_list_t *temp_list = NULL;
-        gf_defrag_pattern_list_t *pattern_list = NULL;
-
-        if (!this || !defrag || !data)
-                goto out;
+    int ret = -1;
+    dht_conf_t *conf = NULL;
+    dht_methods_t *methods = NULL;
 
-        /* Get the pattern for pattern list. "pattern:<optional-size>"
-         * eg: *avi, *pdf:10MB, *:1TB
-         */
-        pattern_str = strtok_r (data, ",", &tmp_str);
-        while (pattern_str) {
-                dup_str = gf_strdup (pattern_str);
-                pattern_list = GF_CALLOC (1, sizeof (gf_defrag_pattern_list_t),
-                                        1);
-                if (!pattern_list) {
-                        goto out;
-                }
-                pattern = strtok_r (dup_str, ":", &tmp_str1);
-                num = strtok_r (NULL, ":", &tmp_str1);
-                if (!pattern)
-                        goto out;
-                if (!num) {
-                        if (gf_string2bytesize_uint64(pattern, &pattern_list->size)
-                             == 0) {
-                                pattern = "*";
-                        }
-                } else if (gf_string2bytesize_uint64 (num, &pattern_list->size) != 0) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                DHT_MSG_INVALID_OPTION,
-                                "Invalid option. Defrag pattern:"
-                                " Invalid number format \"%s\"", num);
-                        goto out;
-                }
-                memcpy (pattern_list->path_pattern, pattern, strlen (dup_str));
-
-                if (!defrag->defrag_pattern)
-                        temp_list = NULL;
-                else
-                        temp_list = defrag->defrag_pattern;
-
-                pattern_list->next = temp_list;
-
-                defrag->defrag_pattern = pattern_list;
-                pattern_list = NULL;
-
-                GF_FREE (dup_str);
-                dup_str = NULL;
-
-                pattern_str = strtok_r (NULL, ",", &tmp_str);
-        }
+    GF_VALIDATE_OR_GOTO("dht", this, err);
 
-        ret = 0;
-out:
-        if (ret)
-                GF_FREE (pattern_list);
-        GF_FREE (dup_str);
+    conf = this->private;
+    methods = &(conf->methods);
 
-        return ret;
+    methods->migration_get_dst_subvol = dht_migration_get_dst_subvol;
+    methods->migration_other = NULL;
+    methods->layout_search = dht_layout_search;
+
+    ret = 0;
+err:
+    return ret;
 }
 
 int
-dht_init (xlator_t *this)
+dht_init(xlator_t *this)
 {
-        dht_conf_t                      *conf           = NULL;
-        char                            *temp_str       = NULL;
-        int                              ret            = -1;
-        int                              i              = 0;
-        gf_defrag_info_t                *defrag         = NULL;
-        int                              cmd            = 0;
-        char                            *node_uuid      = NULL;
-
-
-        GF_VALIDATE_OR_GOTO ("dht", this, err);
-
-        if (!this->children) {
-                gf_msg (this->name, GF_LOG_CRITICAL, 0,
-                        DHT_MSG_INVALID_CONFIGURATION,
-                        "Distribute needs more than one subvolume");
-                return -1;
-        }
-
-        if (!this->parents) {
-                gf_msg (this->name, GF_LOG_WARNING, 0,
-                        DHT_MSG_INVALID_CONFIGURATION,
-                        "dangling volume. check volfile");
-        }
+    dht_conf_t *conf = NULL;
+    char *temp_str = NULL;
+    int ret = -1;
+    int i = 0;
+    gf_defrag_info_t *defrag = NULL;
+    int cmd = 0;
+    char *node_uuid = NULL;
+    uint32_t commit_hash = 0;
+
+    GF_VALIDATE_OR_GOTO("dht", this, err);
+
+    if (!this->children) {
+        gf_msg(this->name, GF_LOG_CRITICAL, 0, DHT_MSG_INVALID_CONFIGURATION,
+               "Distribute needs more than one subvolume");
+        return -1;
+    }
 
-        conf = GF_CALLOC (1, sizeof (*conf), gf_dht_mt_dht_conf_t);
-        if (!conf) {
-                goto err;
-        }
+    if (!this->parents) {
+        gf_msg(this->name, GF_LOG_WARNING, 0, DHT_MSG_INVALID_CONFIGURATION,
+               "dangling volume. check volfile");
+    }
 
-        ret = dict_get_int32 (this->options, "rebalance-cmd", &cmd);
+    conf = GF_CALLOC(1, sizeof(*conf), gf_dht_mt_dht_conf_t);
+    if (!conf) {
+        goto err;
+    }
 
-        if (cmd) {
-                defrag = GF_CALLOC (1, sizeof (gf_defrag_info_t),
-                                    gf_defrag_info_mt);
+    LOCK_INIT(&conf->subvolume_lock);
+    LOCK_INIT(&conf->layout_lock);
+    LOCK_INIT(&conf->lock);
+    synclock_init(&conf->link_lock, SYNC_LOCK_DEFAULT);
 
-                GF_VALIDATE_OR_GOTO (this->name, defrag, err);
+    /* We get the commit-hash to set only for rebalance process */
+    if (dict_get_uint32(this->options, "commit-hash", &commit_hash) == 0) {
+        gf_msg(this->name, GF_LOG_INFO, 0, DHT_MSG_COMMIT_HASH_INFO,
+               "%s using commit hash %u", __func__, commit_hash);
+        conf->vol_commit_hash = commit_hash;
+        conf->vch_forced = _gf_true;
+    }
 
-                LOCK_INIT (&defrag->lock);
+    ret = dict_get_int32(this->options, "rebalance-cmd", &cmd);
 
-                defrag->is_exiting = 0;
+    if (cmd) {
+        defrag = GF_CALLOC(1, sizeof(gf_defrag_info_t), gf_defrag_info_mt);
 
-                conf->defrag = defrag;
+        GF_VALIDATE_OR_GOTO(this->name, defrag, err);
 
-                ret = dict_get_str (this->options, "node-uuid", &node_uuid);
-                if (ret) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                DHT_MSG_INVALID_CONFIGURATION,
-                                "Invalid volume configuration: "
-                                "node-uuid not specified");
-                        goto err;
-                }
+        LOCK_INIT(&defrag->lock);
 
-                if (gf_uuid_parse (node_uuid, defrag->node_uuid)) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                DHT_MSG_INVALID_OPTION, "Invalid option:"
-                                " Cannot parse glusterd node uuid");
-                        goto err;
-                }
+        defrag->is_exiting = 0;
 
-                defrag->cmd = cmd;
+        conf->defrag = defrag;
+        defrag->this = this;
 
-                defrag->stats = _gf_false;
+        ret = dict_get_str(this->options, "node-uuid", &node_uuid);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, DHT_MSG_INVALID_CONFIGURATION,
+                   "Invalid volume configuration: "
+                   "node-uuid not specified");
+            goto err;
         }
 
-        conf->search_unhashed = GF_DHT_LOOKUP_UNHASHED_ON;
-        if (dict_get_str (this->options, "lookup-unhashed", &temp_str) == 0) {
-                /* If option is not "auto", other options _should_ be boolean */
-                if (strcasecmp (temp_str, "auto")) {
-                        ret = gf_string2boolean (temp_str,
-                                                 &conf->search_unhashed);
-                        if (ret == -1)
-                                goto err;
-                }
-                else
-                        conf->search_unhashed = GF_DHT_LOOKUP_UNHASHED_AUTO;
+        if (gf_uuid_parse(node_uuid, defrag->node_uuid)) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, DHT_MSG_INVALID_OPTION,
+                   "Invalid option:"
+                   " Cannot parse glusterd node uuid");
+            goto err;
         }
 
-        GF_OPTION_INIT ("unhashed-sticky-bit", conf->unhashed_sticky_bit, bool,
-                        err);
+        defrag->cmd = cmd;
 
-        GF_OPTION_INIT ("use-readdirp", conf->use_readdirp, bool, err);
+        defrag->stats = _gf_false;
 
-	GF_OPTION_INIT ("min-free-disk", conf->min_free_disk, percent_or_size,
-			err);
+        defrag->queue = NULL;
 
-        GF_OPTION_INIT ("min-free-inodes", conf->min_free_inodes, percent,
-                        err);
+        defrag->crawl_done = 0;
 
-        conf->dir_spread_cnt = conf->subvolume_cnt;
-        GF_OPTION_INIT ("directory-layout-spread", conf->dir_spread_cnt,
-                        uint32, err);
+        defrag->global_error = 0;
 
-        GF_OPTION_INIT ("assert-no-child-down", conf->assert_no_child_down,
-                        bool, err);
+        defrag->q_entry_count = 0;
 
-        GF_OPTION_INIT ("readdir-optimize", conf->readdir_optimize, bool, err);
+        defrag->wakeup_crawler = 0;
 
-        if (defrag) {
-                GF_OPTION_INIT ("rebalance-stats", defrag->stats, bool, err);
-                if (dict_get_str (this->options, "rebalance-filter", &temp_str)
-                    == 0) {
-                        if (gf_defrag_pattern_list_fill (this, defrag, temp_str)
-                            == -1) {
-                                gf_msg (this->name, GF_LOG_ERROR, 0,
-                                        DHT_MSG_INVALID_OPTION,
-                                        "Invalid option:"
-                                        " Cannot parse rebalance-filter (%s)",
-                                        temp_str);
+        pthread_mutex_init(&defrag->dfq_mutex, 0);
+        pthread_cond_init(&defrag->parallel_migration_cond, 0);
+        pthread_cond_init(&defrag->rebalance_crawler_alarm, 0);
+        pthread_cond_init(&defrag->df_wakeup_thread, 0);
 
-                                goto err;
-                        }
-                }
-        }
+        pthread_mutex_init(&defrag->fc_mutex, 0);
+        pthread_cond_init(&defrag->fc_wakeup_cond, 0);
 
-        /* option can be any one of percent or bytes */
-        conf->disk_unit = 0;
-        if (conf->min_free_disk < 100)
-                conf->disk_unit = 'p';
+        defrag->global_error = 0;
+    }
 
-        ret = dht_init_subvolumes (this, conf);
-        if (ret == -1) {
+    conf->use_fallocate = 1;
+
+    conf->search_unhashed = GF_DHT_LOOKUP_UNHASHED_ON;
+    if (dict_get_str(this->options, "lookup-unhashed", &temp_str) == 0) {
+        /* If option is not "auto", other options _should_ be boolean */
+        if (strcasecmp(temp_str, "auto")) {
+            gf_boolean_t search_unhashed_bool;
+            ret = gf_string2boolean(temp_str, &search_unhashed_bool);
+            if (ret == -1) {
                 goto err;
+            }
+            conf->search_unhashed = search_unhashed_bool
+                                        ? GF_DHT_LOOKUP_UNHASHED_ON
+                                        : GF_DHT_LOOKUP_UNHASHED_OFF;
+        } else {
+            conf->search_unhashed = GF_DHT_LOOKUP_UNHASHED_AUTO;
         }
+    }
 
-        if (dict_get_str (this->options, "decommissioned-bricks", &temp_str) == 0) {
-                ret = dht_parse_decommissioned_bricks (this, conf, temp_str);
-                if (ret == -1)
-                        goto err;
-        }
+    GF_OPTION_INIT("lookup-optimize", conf->lookup_optimize, bool, err);
 
-        dht_init_regex (this, this->options, "rsync-hash-regex",
-                        &conf->rsync_regex, &conf->rsync_regex_valid);
-        dht_init_regex (this, this->options, "extra-hash-regex",
-                        &conf->extra_regex, &conf->extra_regex_valid);
+    GF_OPTION_INIT("unhashed-sticky-bit", conf->unhashed_sticky_bit, bool, err);
 
-        ret = dht_layouts_init (this, conf);
-        if (ret == -1) {
-                goto err;
-        }
+    GF_OPTION_INIT("use-readdirp", conf->use_readdirp, bool, err);
 
-        LOCK_INIT (&conf->subvolume_lock);
-        LOCK_INIT (&conf->layout_lock);
+    GF_OPTION_INIT("min-free-disk", conf->min_free_disk, percent_or_size, err);
 
-        conf->gen = 1;
+    GF_OPTION_INIT("min-free-inodes", conf->min_free_inodes, percent, err);
 
-        this->local_pool = mem_pool_new (dht_local_t, 512);
-        if (!this->local_pool) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        DHT_MSG_INIT_FAILED,
-                        " DHT initialisation failed. "
-                        "failed to create local_t's memory pool");
-                goto err;
-        }
+    conf->dir_spread_cnt = conf->subvolume_cnt;
+    GF_OPTION_INIT("directory-layout-spread", conf->dir_spread_cnt, uint32,
+                   err);
 
-        GF_OPTION_INIT ("randomize-hash-range-by-gfid",
-                        conf->randomize_by_gfid, bool, err);
+    GF_OPTION_INIT("assert-no-child-down", conf->assert_no_child_down, bool,
+                   err);
 
-        GF_OPTION_INIT ("xattr-name", conf->xattr_name, str, err);
-        gf_asprintf (&conf->link_xattr_name, "%s."DHT_LINKFILE_STR,
-                     conf->xattr_name);
-        gf_asprintf (&conf->wild_xattr_name, "%s*", conf->xattr_name);
-        if (!conf->link_xattr_name || !conf->wild_xattr_name) {
-                goto err;
-        }
+    GF_OPTION_INIT("readdir-optimize", conf->readdir_optimize, bool, err);
+
+    GF_OPTION_INIT("lock-migration", conf->lock_migration_enabled, bool, err);
 
-        GF_OPTION_INIT ("weighted-rebalance", conf->do_weighting, bool, err);
+    GF_OPTION_INIT("force-migration", conf->force_migration, bool, err);
 
-        conf->lock_pool = mem_pool_new (dht_lock_t, 512);
-        if (!conf->lock_pool) {
-                gf_msg (this->name, GF_LOG_ERROR, 0, DHT_MSG_INIT_FAILED,
-                        "failed to create lock mem_pool, failing "
-                        "initialization");
+    if (defrag) {
+        defrag->lock_migration_enabled = conf->lock_migration_enabled;
+
+        GF_OPTION_INIT("rebalance-stats", defrag->stats, bool, err);
+        if (dict_get_str(this->options, "rebalance-filter", &temp_str) == 0) {
+            if (gf_defrag_pattern_list_fill(this, defrag, temp_str) == -1) {
+                gf_msg(this->name, GF_LOG_ERROR, 0, DHT_MSG_INVALID_OPTION,
+                       "Invalid option:"
+                       " Cannot parse rebalance-filter (%s)",
+                       temp_str);
+
+                goto err;
+            }
+        }
+    }
+
+    /* option can be any one of percent or bytes */
+    conf->disk_unit = 0;
+    if (conf->min_free_disk < 100)
+        conf->disk_unit = 'p';
+
+    ret = dht_init_subvolumes(this, conf);
+    if (ret == -1) {
+        goto err;
+    }
+
+    if (cmd) {
+        ret = dht_init_local_subvolumes(this, conf);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, 0,
+                   DHT_MSG_INIT_LOCAL_SUBVOL_FAILED,
+                   "dht_init_local_subvolumes failed");
+            goto err;
+        }
+    }
+
+    if (dict_get_str(this->options, "decommissioned-bricks", &temp_str) == 0) {
+        ret = dht_parse_decommissioned_bricks(this, conf, temp_str);
+        if (ret == -1)
+            goto err;
+    }
+
+    dht_init_regex(this, this->options, "rsync-hash-regex", &conf->rsync_regex,
+                   &conf->rsync_regex_valid, conf);
+    dht_init_regex(this, this->options, "extra-hash-regex", &conf->extra_regex,
+                   &conf->extra_regex_valid, conf);
+
+    ret = dht_layouts_init(this, conf);
+    if (ret == -1) {
+        goto err;
+    }
+
+    conf->gen = 1;
+
+    this->local_pool = mem_pool_new(dht_local_t, 512);
+    if (!this->local_pool) {
+        gf_msg(this->name, GF_LOG_ERROR, ENOMEM, DHT_MSG_NO_MEMORY,
+               " DHT initialisation failed. "
+               "failed to create local_t's memory pool");
+        goto err;
+    }
+
+    GF_OPTION_INIT("randomize-hash-range-by-gfid", conf->randomize_by_gfid,
+                   bool, err);
+
+    if (defrag) {
+        GF_OPTION_INIT("rebal-throttle", temp_str, str, err);
+        if (temp_str) {
+            ret = dht_configure_throttle(this, conf, temp_str);
+            if (ret == -1)
                 goto err;
         }
+    }
 
-        this->private = conf;
+    GF_OPTION_INIT("xattr-name", conf->xattr_name, str, err);
+    gf_asprintf(&conf->mds_xattr_key, "%s." DHT_MDS_STR, conf->xattr_name);
+    gf_asprintf(&conf->link_xattr_name, "%s." DHT_LINKFILE_STR,
+                conf->xattr_name);
+    gf_asprintf(&conf->commithash_xattr_name, "%s." DHT_COMMITHASH_STR,
+                conf->xattr_name);
+    gf_asprintf(&conf->wild_xattr_name, "%s*", conf->xattr_name);
+    if (!conf->link_xattr_name || !conf->wild_xattr_name) {
+        goto err;
+    }
 
-        if (dht_set_subvol_range(this))
-                goto err;
+    GF_OPTION_INIT("weighted-rebalance", conf->do_weighting, bool, err);
 
-        conf->methods = &dht_methods;
+    conf->lock_pool = mem_pool_new(dht_lock_t, 512);
+    if (!conf->lock_pool) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, DHT_MSG_INIT_FAILED,
+               "failed to create lock mem_pool, failing "
+               "initialization");
+        goto err;
+    }
 
-        return 0;
+    this->private = conf;
+
+    if (dht_set_subvol_range(this))
+        goto err;
+
+    if (dht_init_methods(this))
+        goto err;
+
+    return 0;
 
 err:
-        if (conf) {
-                if (conf->file_layouts) {
-                        for (i = 0; i < conf->subvolume_cnt; i++) {
-                                GF_FREE (conf->file_layouts[i]);
-                        }
-                        GF_FREE (conf->file_layouts);
-                }
+    if (conf) {
+        if (conf->file_layouts) {
+            for (i = 0; i < conf->subvolume_cnt; i++) {
+                GF_FREE(conf->file_layouts[i]);
+            }
+            GF_FREE(conf->file_layouts);
+        }
 
-                GF_FREE (conf->subvolumes);
+        GF_FREE(conf->subvolumes);
 
-                GF_FREE (conf->subvolume_status);
+        GF_FREE(conf->subvolume_status);
 
-                GF_FREE (conf->du_stats);
+        GF_FREE(conf->du_stats);
 
-                GF_FREE (conf->defrag);
+        GF_FREE(conf->defrag);
 
-                GF_FREE (conf->xattr_name);
-                GF_FREE (conf->link_xattr_name);
-                GF_FREE (conf->wild_xattr_name);
+        GF_FREE(conf->xattr_name);
+        GF_FREE(conf->link_xattr_name);
+        GF_FREE(conf->wild_xattr_name);
+        GF_FREE(conf->mds_xattr_key);
 
-                if (conf->lock_pool)
-                        mem_pool_destroy (conf->lock_pool);
+        if (conf->lock_pool)
+            mem_pool_destroy(conf->lock_pool);
 
-                GF_FREE (conf);
-        }
+        GF_FREE(conf);
+    }
 
-        return -1;
+    return -1;
 }
 
-
-struct volume_options options[] = {
-        { .key  = {"lookup-unhashed"},
-          .value = {"auto", "yes", "no", "enable", "disable", "1", "0",
-                    "on", "off"},
-          .type = GF_OPTION_TYPE_STR,
-          .default_value = "on",
-          .description = "This option if set to ON, does a lookup through "
-          "all the sub-volumes, in case a lookup didn't return any result "
-          "from the hash subvolume. If set to OFF, it does not do a lookup "
-          "on the remaining subvolumes."
-        },
-        { .key  = {"min-free-disk"},
-          .type = GF_OPTION_TYPE_PERCENT_OR_SIZET,
-          .default_value = "10%",
-          .description = "Percentage/Size of disk space, after which the "
-          "process starts balancing out the cluster, and logs will appear "
-          "in log files",
-        },
-	{ .key  = {"min-free-inodes"},
-          .type = GF_OPTION_TYPE_PERCENT,
-          .default_value = "5%",
-          .description = "after system has only N% of inodes, warnings "
-          "starts to appear in log files",
-        },
-        { .key = {"unhashed-sticky-bit"},
-          .type = GF_OPTION_TYPE_BOOL,
-          .default_value = "off",
-        },
-        { .key = {"use-readdirp"},
-          .type = GF_OPTION_TYPE_BOOL,
-          .default_value = "on",
-          .description = "This option if set to ON, forces the use of "
-          "readdirp, and hence also displays the stats of the files."
-        },
-        { .key = {"assert-no-child-down"},
-          .type = GF_OPTION_TYPE_BOOL,
-          .default_value = "off",
-          .description = "This option if set to ON, in the event of "
-          "CHILD_DOWN, will call exit."
-        },
-        { .key  = {"directory-layout-spread"},
-          .type = GF_OPTION_TYPE_INT,
-          .min  = 1,
-          .validate = GF_OPT_VALIDATE_MIN,
-          .description = "Specifies the directory layout spread. Takes number "
-                         "of subvolumes as default value."
-        },
-        { .key  = {"decommissioned-bricks"},
-          .type = GF_OPTION_TYPE_ANY,
-          .description = "This option if set to ON, decommissions "
-          "the brick, so that no new data is allowed to be created "
-          "on that brick."
-        },
-        { .key  = {"rebalance-cmd"},
-          .type = GF_OPTION_TYPE_INT,
-        },
-        { .key = {"node-uuid"},
-          .type = GF_OPTION_TYPE_STR,
-        },
-        { .key = {"rebalance-stats"},
-          .type = GF_OPTION_TYPE_BOOL,
-          .default_value = "off",
-          .description = "This option if set to ON displays and logs the "
-          " time taken for migration of each file, during the rebalance "
-          "process. If set to OFF, the rebalance logs will only display the "
-          "time spent in each directory."
-        },
-        { .key = {"readdir-optimize"},
-          .type = GF_OPTION_TYPE_BOOL,
-          .default_value = "off",
-          .description = "This option if set to ON enables the optimization "
-          "that allows DHT to requests non-first subvolumes to filter out "
-          "directory entries."
-        },
-        { .key = {"rsync-hash-regex"},
-          .type = GF_OPTION_TYPE_STR,
-          /* Setting a default here doesn't work.  See dht_init_regex. */
-          .description = "Regular expression for stripping temporary-file "
-          "suffix and prefix used by rsync, to prevent relocation when the "
-          "file is renamed."
-        },
-        { .key = {"extra-hash-regex"},
-          .type = GF_OPTION_TYPE_STR,
-          /* Setting a default here doesn't work.  See dht_init_regex. */
-          .description = "Regular expression for stripping temporary-file "
-          "suffix and prefix used by an application, to prevent relocation when "
-          "the file is renamed."
-        },
-        { .key = {"rebalance-filter"},
-          .type = GF_OPTION_TYPE_STR,
-        },
-
-        { .key = {"xattr-name"},
-          .type = GF_OPTION_TYPE_STR,
-          .default_value = "trusted.glusterfs.dht",
-          .description = "Base for extended attributes used by this "
-          "translator instance, to avoid conflicts with others above or "
-          "below it."
-        },
-
-        { .key = {"weighted-rebalance"},
-          .type = GF_OPTION_TYPE_BOOL,
-          .default_value = "on",
-          .description = "When enabled, files will be allocated to bricks "
-          "with a probability proportional to their size.  Otherwise, all "
-          "bricks will have the same probability (legacy behavior)."
-        },
-
-        /* NUFA option */
-        { .key  = {"local-volume-name"},
-          .type = GF_OPTION_TYPE_XLATOR
-        },
-
-        /* tier options */
-        { .key  = {"tier-promote-frequency"},
-          .type = GF_OPTION_TYPE_INT,
-          .default_value = "120",
-          .description = "Frequency to promote files to fast tier"
-        },
-
-        { .key  = {"tier-demote-frequency"},
-          .type = GF_OPTION_TYPE_INT,
-          .default_value = "120",
-          .description = "Frequency to demote files to slow tier"
-        },
-
-        { .key  = {"write-freq-threshold"},
-          .type = GF_OPTION_TYPE_INT,
-          .default_value = "0",
-          .description = "Defines the write fequency "
-                        "that would be considered hot"
-        },
-
-        { .key  = {"read-freq-threshold"},
-          .type = GF_OPTION_TYPE_INT,
-          .default_value = "0",
-          .description = "Defines the read fequency "
-                        "that would be considered hot"
-        },
-
-        /* switch option */
-        { .key  = {"pattern.switch.case"},
-          .type = GF_OPTION_TYPE_ANY
-        },
-
-        { .key =  {"randomize-hash-range-by-gfid"},
-          .type = GF_OPTION_TYPE_BOOL,
-          .default_value = "off",
-          .description = "Use gfid of directory to determine the subvolume "
-          "from which hash ranges are allocated starting with 0. "
-          "Note that we still use a directory/file's name to determine the "
-          "subvolume to which it hashes"
-        },
-
-        { .key  = {NULL} },
+struct volume_options dht_options[] = {
+    {
+        .key = {"lookup-unhashed"},
+        .value = {"auto", "yes", "no", "enable", "disable", "1", "0", "on",
+                  "off"},
+        .type = GF_OPTION_TYPE_STR,
+        .default_value = "on",
+        .description =
+            "This option if set to ON, does a lookup through "
+            "all the sub-volumes, in case a lookup didn't return any result "
+            "from the hash subvolume. If set to OFF, it does not do a lookup "
+            "on the remaining subvolumes.",
+        .op_version = {1},
+        .flags = OPT_FLAG_CLIENT_OPT | OPT_FLAG_SETTABLE,
+        .level = OPT_STATUS_BASIC,
+    },
+    {.key = {"lookup-optimize"},
+     .type = GF_OPTION_TYPE_BOOL,
+     .default_value = "on",
+     .description =
+         "This option if set to ON enables the optimization "
+         "of -ve lookups, by not doing a lookup on non-hashed subvolumes for "
+         "files, in case the hashed subvolume does not return any result. "
+         "This option disregards the lookup-unhashed setting, when enabled.",
+     .op_version = {GD_OP_VERSION_3_7_2},
+     .level = OPT_STATUS_ADVANCED,
+     .flags = OPT_FLAG_CLIENT_OPT | OPT_FLAG_SETTABLE | OPT_FLAG_DOC},
+    {.key = {"min-free-disk"},
+     .type = GF_OPTION_TYPE_PERCENT_OR_SIZET,
+     .default_value = "10%",
+     .description =
+         "Percentage/Size of disk space, after which the "
+         "process starts balancing out the cluster, and logs will appear "
+         "in log files",
+     .op_version = {1},
+     .level = OPT_STATUS_BASIC,
+     .flags = OPT_FLAG_CLIENT_OPT | OPT_FLAG_SETTABLE | OPT_FLAG_DOC},
+    {.key = {"min-free-inodes"},
+     .type = GF_OPTION_TYPE_PERCENT,
+     .default_value = "5%",
+     .description = "after system has only N% of inodes, warnings "
+                    "starts to appear in log files",
+     .op_version = {1},
+     .level = OPT_STATUS_BASIC,
+     .flags = OPT_FLAG_CLIENT_OPT | OPT_FLAG_SETTABLE | OPT_FLAG_DOC},
+    {
+        .key = {"unhashed-sticky-bit"},
+        .type = GF_OPTION_TYPE_BOOL,
+        .default_value = "off",
+    },
+    {.key = {"use-readdirp"},
+     .type = GF_OPTION_TYPE_BOOL,
+     .default_value = "on",
+     .description = "This option if set to ON, forces the use of "
+                    "readdirp, and hence also displays the stats of the files.",
+     .level = OPT_STATUS_ADVANCED,
+     .flags = OPT_FLAG_CLIENT_OPT | OPT_FLAG_SETTABLE | OPT_FLAG_DOC},
+    {.key = {"assert-no-child-down"},
+     .type = GF_OPTION_TYPE_BOOL,
+     .default_value = "off",
+     .description = "This option if set to ON, in the event of "
+                    "CHILD_DOWN, will call exit."},
+    {
+        .key = {"directory-layout-spread"},
+        .type = GF_OPTION_TYPE_INT,
+        .min = 1,
+        .validate = GF_OPT_VALIDATE_MIN,
+        .description = "Specifies the directory layout spread. Takes number "
+                       "of subvolumes as default value.",
+
+        .op_version = {2},
+    },
+    {
+        .key = {"decommissioned-bricks"},
+        .type = GF_OPTION_TYPE_ANY,
+        .description =
+            "This option if set to ON, decommissions "
+            "the brick, so that no new data is allowed to be created "
+            "on that brick.",
+        .level = OPT_STATUS_ADVANCED,
+    },
+    {
+        .key = {"rebalance-cmd"},
+        .type = GF_OPTION_TYPE_INT,
+    },
+    {
+        .key = {"commit-hash"},
+        .type = GF_OPTION_TYPE_INT,
+    },
+    {
+        .key = {"node-uuid"},
+        .type = GF_OPTION_TYPE_STR,
+    },
+    {
+        .key = {"rebalance-stats"},
+        .type = GF_OPTION_TYPE_BOOL,
+        .default_value = "off",
+        .description =
+            "This option if set to ON displays and logs the "
+            " time taken for migration of each file, during the rebalance "
+            "process. If set to OFF, the rebalance logs will only display the "
+            "time spent in each directory.",
+        .op_version = {2},
+        .level = OPT_STATUS_BASIC,
+    },
+    {.key = {"readdir-optimize"},
+     .type = GF_OPTION_TYPE_BOOL,
+     .default_value = "off",
+     .description =
+         "This option if set to ON enables the optimization "
+         "that allows DHT to requests non-first subvolumes to filter out "
+         "directory entries.",
+     .op_version = {1},
+     .level = OPT_STATUS_ADVANCED,
+     .flags = OPT_FLAG_CLIENT_OPT | OPT_FLAG_SETTABLE | OPT_FLAG_DOC},
+    {.key = {"rsync-hash-regex"},
+     .type = GF_OPTION_TYPE_STR,
+     /* Setting a default here doesn't work.  See dht_init_regex. */
+     .description =
+         "Regular expression for stripping temporary-file "
+         "suffix and prefix used by rsync, to prevent relocation when the "
+         "file is renamed.",
+     .op_version = {3},
+     .level = OPT_STATUS_BASIC,
+     .flags = OPT_FLAG_CLIENT_OPT | OPT_FLAG_SETTABLE | OPT_FLAG_DOC},
+    {.key = {"extra-hash-regex"},
+     .type = GF_OPTION_TYPE_STR,
+     /* Setting a default here doesn't work.  See dht_init_regex. */
+     .description =
+         "Regular expression for stripping temporary-file "
+         "suffix and prefix used by an application, to prevent relocation when "
+         "the file is renamed.",
+     .op_version = {3},
+     .level = OPT_STATUS_BASIC,
+     .flags = OPT_FLAG_CLIENT_OPT | OPT_FLAG_SETTABLE | OPT_FLAG_DOC},
+    {
+        .key = {"rebalance-filter"},
+        .type = GF_OPTION_TYPE_STR,
+    },
+
+    {
+        .key = {"xattr-name"},
+        .type = GF_OPTION_TYPE_STR,
+        .default_value = "trusted.glusterfs.dht",
+        .description =
+            "Base for extended attributes used by this "
+            "translator instance, to avoid conflicts with others above or "
+            "below it.",
+        .op_version = {3},
+    },
+
+    {.key = {"weighted-rebalance"},
+     .type = GF_OPTION_TYPE_BOOL,
+     .default_value = "on",
+     .description =
+         "When enabled, files will be allocated to bricks "
+         "with a probability proportional to their size.  Otherwise, all "
+         "bricks will have the same probability (legacy behavior).",
+     .op_version = {GD_OP_VERSION_3_6_0},
+     .level = OPT_STATUS_BASIC,
+     .flags = OPT_FLAG_CLIENT_OPT | OPT_FLAG_SETTABLE | OPT_FLAG_DOC},
+
+    /* NUFA option */
+    {.key = {"local-volume-name"}, .type = GF_OPTION_TYPE_XLATOR},
+
+    /* switch option */
+    {.key = {"pattern.switch.case"}, .type = GF_OPTION_TYPE_ANY},
+
+    {
+        .key = {"randomize-hash-range-by-gfid"},
+        .type = GF_OPTION_TYPE_BOOL,
+        .default_value = "off",
+        .description =
+            "Use gfid of directory to determine the subvolume "
+            "from which hash ranges are allocated starting with 0. "
+            "Note that we still use a directory/file's name to determine the "
+            "subvolume to which it hashes",
+        .op_version = {GD_OP_VERSION_3_6_0},
+    },
+
+    {.key = {"rebal-throttle"},
+     .type = GF_OPTION_TYPE_STR,
+     .default_value = "normal",
+     .description = " Sets the maximum number of parallel file migrations "
+                    "allowed on a node during the rebalance operation. The"
+                    " default value is normal and allows a max of "
+                    "[($(processing units) - 4) / 2), 2]  files to be "
+                    "migrated at a time. Lazy will allow only one file to "
+                    "be migrated at a time and aggressive will allow "
+                    "max of [($(processing units) - 4) / 2), 4]",
+     .op_version = {GD_OP_VERSION_3_7_0},
+     .level = OPT_STATUS_BASIC,
+     .flags = OPT_FLAG_CLIENT_OPT | OPT_FLAG_SETTABLE | OPT_FLAG_DOC
+
+    },
+
+    {.key = {"lock-migration"},
+     .type = GF_OPTION_TYPE_BOOL,
+     .default_value = "off",
+     .description = " If enabled this feature will migrate the posix locks"
+                    " associated with a file during rebalance",
+     .op_version = {GD_OP_VERSION_3_8_0},
+     .level = OPT_STATUS_ADVANCED,
+     .flags = OPT_FLAG_CLIENT_OPT | OPT_FLAG_SETTABLE | OPT_FLAG_DOC},
+
+    {.key = {"force-migration"},
+     .type = GF_OPTION_TYPE_BOOL,
+     .default_value = "off",
+     .description = "If disabled, rebalance will not migrate files that "
+                    "are being written to by an application",
+     .op_version = {GD_OP_VERSION_4_0_0},
+     .level = OPT_STATUS_ADVANCED,
+     .flags = OPT_FLAG_CLIENT_OPT | OPT_FLAG_SETTABLE | OPT_FLAG_DOC},
+
+    {.key = {NULL}},
 };
+
+#define NUM_DHT_OPTIONS (sizeof(dht_options) / sizeof(dht_options[0]))
+
+extern struct volume_options options[NUM_DHT_OPTIONS]
+    __attribute__((alias("dht_options")));
diff --git a/xlators/cluster/dht/src/dht.c b/xlators/cluster/dht/src/dht.c
index 3934df5ec64..53de8292704 100644
--- a/xlators/cluster/dht/src/dht.c
+++ b/xlators/cluster/dht/src/dht.c
@@ -8,87 +8,116 @@
   cases as published by the Free Software Foundation.
 */
 
-
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
-
-#include "statedump.h"
 #include "dht-common.h"
 
-dht_methods_t dht_methods = {
-        .migration_get_dst_subvol = dht_migration_get_dst_subvol,
-        .migration_needed = dht_migration_needed,
-        .layout_search   = dht_layout_search,
-};
+struct xlator_fops dht_pt_fops = {
+    /* we need to keep mkdir to make sure we
+       have layout on new directory */
+    .mkdir = dht_pt_mkdir,
+    .getxattr = dht_pt_getxattr,
+    .fgetxattr = dht_pt_fgetxattr,
 
-class_methods_t class_methods = {
-        .init           = dht_init,
-        .fini           = dht_fini,
-        .reconfigure    = dht_reconfigure,
-        .notify         = dht_notify
+    /* required to trace fop properly in changelog */
+    .rename = dht_pt_rename,
+
+    /* FIXME: commenting the '.lookup()' below made some of
+       the failing tests to pass. I would remove the below
+       line, but keeping it here as a reminder for people
+       to check for issues if they find concerns with DHT
+       pass-through logic  */
+    /*
+      .lookup = dht_lookup,
+      .readdir = dht_readdir,
+      .readdirp = dht_readdirp,
+    */
+    /* Keeping above as commented, mainly to support the
+       usecase of a gluster volume getting to 1x(anytype),
+       due to remove-brick (shrinking) exercise. In that case,
+       we would need above fops to be available, so we can
+       handle the case of dangling linkto files (if any) */
 };
 
 struct xlator_fops fops = {
-        .lookup      = dht_lookup,
-        .mknod       = dht_mknod,
-        .create      = dht_create,
+    .ipc = dht_ipc,
+    .lookup = dht_lookup,
+    .mknod = dht_mknod,
+    .create = dht_create,
 
-        .open        = dht_open,
-        .statfs      = dht_statfs,
-        .opendir     = dht_opendir,
-        .readdir     = dht_readdir,
-        .readdirp    = dht_readdirp,
-        .fsyncdir    = dht_fsyncdir,
-        .symlink     = dht_symlink,
-        .unlink      = dht_unlink,
-        .link        = dht_link,
-        .mkdir       = dht_mkdir,
-        .rmdir       = dht_rmdir,
-        .rename      = dht_rename,
-        .entrylk     = dht_entrylk,
-        .fentrylk    = dht_fentrylk,
+    .open = dht_open,
+    .statfs = dht_statfs,
+    .opendir = dht_opendir,
+    .readdir = dht_readdir,
+    .readdirp = dht_readdirp,
+    .fsyncdir = dht_fsyncdir,
+    .symlink = dht_symlink,
+    .unlink = dht_unlink,
+    .link = dht_link,
+    .mkdir = dht_mkdir,
+    .rmdir = dht_rmdir,
+    .rename = dht_rename,
+    .entrylk = dht_entrylk,
+    .fentrylk = dht_fentrylk,
 
-        /* Inode read operations */
-        .stat        = dht_stat,
-        .fstat       = dht_fstat,
-        .access      = dht_access,
-        .readlink    = dht_readlink,
-        .getxattr    = dht_getxattr,
-        .fgetxattr    = dht_fgetxattr,
-        .readv       = dht_readv,
-        .flush       = dht_flush,
-        .fsync       = dht_fsync,
-        .inodelk     = dht_inodelk,
-        .finodelk    = dht_finodelk,
-        .lk          = dht_lk,
+    /* Inode read operations */
+    .stat = dht_stat,
+    .fstat = dht_fstat,
+    .access = dht_access,
+    .readlink = dht_readlink,
+    .getxattr = dht_getxattr,
+    .fgetxattr = dht_fgetxattr,
+    .readv = dht_readv,
+    .flush = dht_flush,
+    .fsync = dht_fsync,
+    .inodelk = dht_inodelk,
+    .finodelk = dht_finodelk,
+    .lk = dht_lk,
+    .lease = dht_lease,
 
-        /* Inode write operations */
-        .fremovexattr = dht_fremovexattr,
-        .removexattr = dht_removexattr,
-        .setxattr    = dht_setxattr,
-        .fsetxattr   = dht_fsetxattr,
-        .truncate    = dht_truncate,
-        .ftruncate   = dht_ftruncate,
-        .writev      = dht_writev,
-        .xattrop     = dht_xattrop,
-        .fxattrop    = dht_fxattrop,
-        .setattr     = dht_setattr,
-        .fsetattr    = dht_fsetattr,
-	.fallocate   = dht_fallocate,
-	.discard     = dht_discard,
-        .zerofill    = dht_zerofill,
+    /* Inode write operations */
+    .fremovexattr = dht_fremovexattr,
+    .removexattr = dht_removexattr,
+    .setxattr = dht_setxattr,
+    .fsetxattr = dht_fsetxattr,
+    .truncate = dht_truncate,
+    .ftruncate = dht_ftruncate,
+    .writev = dht_writev,
+    .xattrop = dht_xattrop,
+    .fxattrop = dht_fxattrop,
+    .setattr = dht_setattr,
+    .fsetattr = dht_fsetattr,
+    .fallocate = dht_fallocate,
+    .discard = dht_discard,
+    .zerofill = dht_zerofill,
 };
 
 struct xlator_dumpops dumpops = {
-        .priv = dht_priv_dump,
-        .inodectx = dht_inodectx_dump,
+    .priv = dht_priv_dump,
+    .inodectx = dht_inodectx_dump,
 };
 
-
 struct xlator_cbks cbks = {
-//      .release    = dht_release,
-//      .releasedir = dht_releasedir,
-        .forget     = dht_forget
+    .release = dht_release,
+    //      .releasedir = dht_releasedir,
+    .forget = dht_forget,
+};
+
+extern int32_t
+mem_acct_init(xlator_t *this);
+
+extern struct volume_options dht_options[];
+
+xlator_api_t xlator_api = {
+    .init = dht_init,
+    .fini = dht_fini,
+    .notify = dht_notify,
+    .reconfigure = dht_reconfigure,
+    .mem_acct_init = mem_acct_init,
+    .op_version = {1}, /* Present from the initial version */
+    .dumpops = &dumpops,
+    .fops = &fops,
+    .cbks = &cbks,
+    .options = dht_options,
+    .identifier = "distribute",
+    .pass_through_fops = &dht_pt_fops,
+    .category = GF_MAINTAINED,
 };
diff --git a/xlators/cluster/dht/src/nufa.c b/xlators/cluster/dht/src/nufa.c
index 72d6d9c10e5..3648a564840 100644
--- a/xlators/cluster/dht/src/nufa.c
+++ b/xlators/cluster/dht/src/nufa.c
@@ -8,674 +8,650 @@
   cases as published by the Free Software Foundation.
 */
 
-
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
-
 #include "dht-common.h"
 
 /* TODO: all 'TODO's in dht.c holds good */
 
-extern struct volume_options options[];
+extern struct volume_options dht_options[];
 
 int
-nufa_local_lookup_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                       int op_ret, int op_errno,
-                       inode_t *inode, struct iatt *stbuf, dict_t *xattr,
-                       struct iatt *postparent)
+nufa_local_lookup_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                      int op_ret, int op_errno, inode_t *inode,
+                      struct iatt *stbuf, dict_t *xattr,
+                      struct iatt *postparent)
 {
-        xlator_t     *subvol      = NULL;
-        char          is_linkfile = 0;
-        char          is_dir      = 0;
-        dht_conf_t   *conf        = NULL;
-        dht_local_t  *local       = NULL;
-        loc_t        *loc         = NULL;
-        int           i           = 0;
-        call_frame_t *prev        = NULL;
-        int           call_cnt    = 0;
-        int           ret         = 0;
-
-        conf  = this->private;
-
-        prev  = cookie;
-        local = frame->local;
-        loc   = &local->loc;
-
-        if (ENTRY_MISSING (op_ret, op_errno)) {
-                if (conf->search_unhashed) {
-                        local->op_errno = ENOENT;
-                        dht_lookup_everywhere (frame, this, loc);
-                        return 0;
-                }
+    xlator_t *subvol = NULL;
+    char is_linkfile = 0;
+    char is_dir = 0;
+    dht_conf_t *conf = NULL;
+    dht_local_t *local = NULL;
+    loc_t *loc = NULL;
+    int i = 0;
+    xlator_t *prev = NULL;
+    int call_cnt = 0;
+    int ret = 0;
+
+    conf = this->private;
+
+    prev = cookie;
+    local = frame->local;
+    loc = &local->loc;
+
+    if (ENTRY_MISSING(op_ret, op_errno)) {
+        if (conf->search_unhashed) {
+            local->op_errno = ENOENT;
+            dht_lookup_everywhere(frame, this, loc);
+            return 0;
         }
-
-        if (op_ret == -1)
-                goto out;
-
-        is_linkfile = check_is_linkfile (inode, stbuf, xattr,
-                                         conf->link_xattr_name);
-        is_dir      = check_is_dir (inode, stbuf, xattr);
-
-        if (!is_dir && !is_linkfile) {
-                /* non-directory and not a linkfile */
-                ret = dht_layout_preset (this, prev->this, inode);
-                if (ret < 0) {
-                        gf_msg_debug (this->name, 0,
-                                      "could not set pre-set layout for subvol"
-                                      " %s", prev->this->name);
-                        op_ret   = -1;
-                        op_errno = EINVAL;
-                        goto err;
-                }
-
-                goto out;
+    }
+
+    if (op_ret == -1)
+        goto out;
+
+    is_linkfile = check_is_linkfile(inode, stbuf, xattr, conf->link_xattr_name);
+    is_dir = check_is_dir(inode, stbuf, xattr);
+
+    if (!is_dir && !is_linkfile) {
+        /* non-directory and not a linkfile */
+        ret = dht_layout_preset(this, prev, inode);
+        if (ret < 0) {
+            gf_msg_debug(this->name, 0,
+                         "could not set pre-set layout for subvol"
+                         " %s",
+                         prev->name);
+            op_ret = -1;
+            op_errno = EINVAL;
+            goto err;
         }
 
-        if (is_dir) {
-                call_cnt        = conf->subvolume_cnt;
-                local->call_cnt = call_cnt;
-
-                local->inode = inode_ref (inode);
-                local->xattr = dict_ref (xattr);
-
-                local->op_ret = 0;
-                local->op_errno = 0;
-
-                local->layout = dht_layout_new (this, conf->subvolume_cnt);
-                if (!local->layout) {
-                        op_ret   = -1;
-                        op_errno = ENOMEM;
-                        goto err;
-                }
-
-                for (i = 0; i < call_cnt; i++) {
-                        STACK_WIND (frame, dht_lookup_dir_cbk,
-                                    conf->subvolumes[i],
-                                    conf->subvolumes[i]->fops->lookup,
-                                    &local->loc, local->xattr_req);
-                }
-        }
+        goto out;
+    }
+
+    if (is_dir) {
+        call_cnt = conf->subvolume_cnt;
+        local->call_cnt = call_cnt;
 
-        if (is_linkfile) {
-                subvol = dht_linkfile_subvol (this, inode, stbuf, xattr);
+        local->inode = inode_ref(inode);
+        local->xattr = dict_ref(xattr);
 
-                if (!subvol) {
-                        gf_msg_debug (this->name, 0,
-                                      "linkfile has no link subvolume. path=%s",
-                                      loc->path);
-                        dht_lookup_everywhere (frame, this, loc);
-                        return 0;
-                }
+        local->op_ret = 0;
+        local->op_errno = 0;
 
-                STACK_WIND (frame, dht_lookup_linkfile_cbk,
-                            subvol, subvol->fops->lookup,
-                            &local->loc, local->xattr_req);
+        local->layout = dht_layout_new(this, conf->subvolume_cnt);
+        if (!local->layout) {
+            op_ret = -1;
+            op_errno = ENOMEM;
+            goto err;
         }
 
-        return 0;
+        for (i = 0; i < call_cnt; i++) {
+            STACK_WIND_COOKIE(frame, dht_lookup_dir_cbk, conf->subvolumes[i],
+                              conf->subvolumes[i],
+                              conf->subvolumes[i]->fops->lookup, &local->loc,
+                              local->xattr_req);
+        }
+    }
 
-out:
-        if (!local->hashed_subvol) {
-                gf_msg_debug (this->name, 0,
-                              "no subvolume in layout for path=%s",
-                              local->loc.path);
-                local->op_errno = ENOENT;
-                dht_lookup_everywhere (frame, this, loc);
-                return 0;
+    if (is_linkfile) {
+        subvol = dht_linkfile_subvol(this, inode, stbuf, xattr);
+
+        if (!subvol) {
+            gf_msg_debug(this->name, 0,
+                         "linkfile has no link subvolume. path=%s", loc->path);
+            dht_lookup_everywhere(frame, this, loc);
+            return 0;
         }
 
-        STACK_WIND (frame, dht_lookup_cbk,
-                    local->hashed_subvol, local->hashed_subvol->fops->lookup,
-                    &local->loc, local->xattr_req);
+        STACK_WIND_COOKIE(frame, dht_lookup_linkfile_cbk, subvol, subvol,
+                          subvol->fops->lookup, &local->loc, local->xattr_req);
+    }
+
+    return 0;
 
+out:
+    if (!local->hashed_subvol) {
+        gf_msg_debug(this->name, 0, "no subvolume in layout for path=%s",
+                     local->loc.path);
+        local->op_errno = ENOENT;
+        dht_lookup_everywhere(frame, this, loc);
         return 0;
+    }
+
+    STACK_WIND_COOKIE(frame, dht_lookup_cbk, local->hashed_subvol,
+                      local->hashed_subvol, local->hashed_subvol->fops->lookup,
+                      &local->loc, local->xattr_req);
+
+    return 0;
 
 err:
-        DHT_STACK_UNWIND (lookup, frame, op_ret, op_errno,
-                          inode, stbuf, xattr, postparent);
-        return 0;
+    DHT_STACK_UNWIND(lookup, frame, op_ret, op_errno, inode, stbuf, xattr,
+                     postparent);
+    return 0;
 }
 
 int
-nufa_lookup (call_frame_t *frame, xlator_t *this,
-             loc_t *loc, dict_t *xattr_req)
+nufa_lookup(call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xattr_req)
 {
-        xlator_t     *hashed_subvol = NULL;
-        xlator_t     *subvol = NULL;
-        dht_local_t  *local  = NULL;
-        dht_conf_t   *conf = NULL;
-        int           ret    = -1;
-        int           op_errno = -1;
-        dht_layout_t *layout = NULL;
-        int           i = 0;
-        int           call_cnt = 0;
-
-
-        VALIDATE_OR_GOTO (frame, err);
-        VALIDATE_OR_GOTO (this, err);
-        VALIDATE_OR_GOTO (loc, err);
-        VALIDATE_OR_GOTO (loc->inode, err);
-        VALIDATE_OR_GOTO (loc->path, err);
-
-        conf = this->private;
-
-        local = dht_local_init (frame, loc, NULL, GF_FOP_LOOKUP);
-        if (!local) {
-                op_errno = ENOMEM;
-                goto err;
+    xlator_t *hashed_subvol = NULL;
+    xlator_t *subvol = NULL;
+    dht_local_t *local = NULL;
+    dht_conf_t *conf = NULL;
+    int ret = -1;
+    int op_errno = -1;
+    dht_layout_t *layout = NULL;
+    int i = 0;
+    int call_cnt = 0;
+
+    VALIDATE_OR_GOTO(frame, err);
+    VALIDATE_OR_GOTO(this, err);
+    VALIDATE_OR_GOTO(loc, err);
+    VALIDATE_OR_GOTO(loc->inode, err);
+    VALIDATE_OR_GOTO(loc->path, err);
+
+    conf = this->private;
+
+    local = dht_local_init(frame, loc, NULL, GF_FOP_LOOKUP);
+    if (!local) {
+        op_errno = ENOMEM;
+        goto err;
+    }
+
+    if (xattr_req) {
+        local->xattr_req = dict_ref(xattr_req);
+    } else {
+        local->xattr_req = dict_new();
+    }
+
+    hashed_subvol = dht_subvol_get_hashed(this, &local->loc);
+
+    local->hashed_subvol = hashed_subvol;
+
+    if (is_revalidate(loc)) {
+        layout = local->layout;
+        if (!layout) {
+            gf_msg_debug(this->name, 0,
+                         "revalidate lookup without cache. "
+                         "path=%s",
+                         loc->path);
+            op_errno = EINVAL;
+            goto err;
         }
 
-        if (xattr_req) {
-                local->xattr_req = dict_ref (xattr_req);
-        } else {
-                local->xattr_req = dict_new ();
+        if (layout->gen && (layout->gen < conf->gen)) {
+            gf_msg_debug(this->name, 0, "incomplete layout failure for path=%s",
+                         loc->path);
+            dht_layout_unref(this, local->layout);
+            goto do_fresh_lookup;
         }
 
-        hashed_subvol = dht_subvol_get_hashed (this, &local->loc);
-
-        local->hashed_subvol = hashed_subvol;
-
-        if (is_revalidate (loc)) {
-                layout = local->layout;
-                if (!layout) {
-                        gf_msg_debug (this->name, 0,
-                                      "revalidate lookup without cache. "
-                                      "path=%s", loc->path);
-                        op_errno = EINVAL;
-                        goto err;
-                }
-
-                if (layout->gen && (layout->gen < conf->gen)) {
-                        gf_msg_debug (this->name, 0,
-                                      "incomplete layout failure for path=%s",
-                                      loc->path);
-                        dht_layout_unref (this, local->layout);
-                        goto do_fresh_lookup;
-                }
-
-                local->inode = inode_ref (loc->inode);
-
-                local->call_cnt = layout->cnt;
-                call_cnt = local->call_cnt;
-
-                /* NOTE: we don't require 'trusted.glusterfs.dht.linkto' attribute,
-                 *       revalidates directly go to the cached-subvolume.
-                 */
-                ret = dict_set_uint32 (local->xattr_req,
-                                       conf->xattr_name, 4 * 4);
-                if (ret < 0) {
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "Failed to set dict value.");
-                        op_errno = -1;
-                        goto err;
-                }
-
-                for (i = 0; i < layout->cnt; i++) {
-                        subvol = layout->list[i].xlator;
-
-                        STACK_WIND (frame, dht_revalidate_cbk,
-                                    subvol, subvol->fops->lookup,
-                                    loc, local->xattr_req);
-
-                        if (!--call_cnt)
-                                break;
-                }
-        } else {
-        do_fresh_lookup:
-                ret = dict_set_uint32 (local->xattr_req,
-                                       conf->xattr_name, 4 * 4);
-                if (ret < 0) {
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "Failed to set dict value.");
-                        op_errno = -1;
-                        goto err;
-                }
-
-                ret = dict_set_uint32 (local->xattr_req,
-                                       conf->link_xattr_name, 256);
-                if (ret < 0) {
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "Failed to set dict value.");
-                        op_errno = -1;
-                        goto err;
-                }
-
-                /* Send it to only local volume */
-                STACK_WIND (frame, nufa_local_lookup_cbk,
-                            (xlator_t *)conf->private,
-                            ((xlator_t *)conf->private)->fops->lookup,
-                            loc, local->xattr_req);
-        }
+        local->inode = inode_ref(loc->inode);
 
-        return 0;
+        local->call_cnt = layout->cnt;
+        call_cnt = local->call_cnt;
 
-err:
-        op_errno = (op_errno == -1) ? errno : op_errno;
-        DHT_STACK_UNWIND (lookup, frame, -1, op_errno, NULL, NULL, NULL,
-                          NULL);
-        return 0;
-}
+        /* NOTE: we don't require 'trusted.glusterfs.dht.linkto' attribute,
+         *       revalidates directly go to the cached-subvolume.
+         */
+        ret = dict_set_uint32(local->xattr_req, conf->xattr_name, 4 * 4);
+        if (ret < 0) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, DHT_MSG_DICT_SET_FAILED,
+                   "Failed to set dict value.");
+            op_errno = -1;
+            goto err;
+        }
 
-int
-nufa_create_linkfile_create_cbk (call_frame_t *frame, void *cookie,
-                                 xlator_t *this, int op_ret, int op_errno,
-                                 inode_t *inode, struct iatt *stbuf,
-                                 struct iatt *preparent,
-                                 struct iatt *postparent, dict_t *xdata)
-{
-        dht_local_t  *local = NULL;
+        for (i = 0; i < layout->cnt; i++) {
+            subvol = layout->list[i].xlator;
+
+            STACK_WIND_COOKIE(frame, dht_revalidate_cbk, subvol, subvol,
+                              subvol->fops->lookup, loc, local->xattr_req);
 
-        local = frame->local;
+            if (!--call_cnt)
+                break;
+        }
+    } else {
+    do_fresh_lookup:
+        ret = dict_set_uint32(local->xattr_req, conf->xattr_name, 4 * 4);
+        if (ret < 0) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, DHT_MSG_DICT_SET_FAILED,
+                   "Failed to set dict value.");
+            op_errno = -1;
+            goto err;
+        }
 
-        if (op_ret == -1)
-                goto err;
+        ret = dict_set_uint32(local->xattr_req, conf->link_xattr_name, 256);
+        if (ret < 0) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, DHT_MSG_DICT_SET_FAILED,
+                   "Failed to set dict value.");
+            op_errno = -1;
+            goto err;
+        }
 
-        STACK_WIND (frame, dht_create_cbk,
-                    local->cached_subvol, local->cached_subvol->fops->create,
-                    &local->loc, local->flags, local->mode, local->umask,
-                    local->fd, local->params);
+        /* Send it to only local volume */
+        STACK_WIND_COOKIE(
+            frame, nufa_local_lookup_cbk, ((xlator_t *)conf->private),
+            ((xlator_t *)conf->private),
+            ((xlator_t *)conf->private)->fops->lookup, loc, local->xattr_req);
+    }
 
-        return 0;
+    return 0;
 
 err:
-        DHT_STACK_UNWIND (create, frame, -1, op_errno,
-                          NULL, NULL, NULL, NULL, NULL, NULL);
-        return 0;
+    op_errno = (op_errno == -1) ? errno : op_errno;
+    DHT_STACK_UNWIND(lookup, frame, -1, op_errno, NULL, NULL, NULL, NULL);
+    return 0;
 }
 
 int
-nufa_create (call_frame_t *frame, xlator_t *this,
-             loc_t *loc, int32_t flags, mode_t mode,
-             mode_t umask, fd_t *fd, dict_t *params)
+nufa_create_linkfile_create_cbk(call_frame_t *frame, void *cookie,
+                                xlator_t *this, int op_ret, int op_errno,
+                                inode_t *inode, struct iatt *stbuf,
+                                struct iatt *preparent, struct iatt *postparent,
+                                dict_t *xdata)
 {
-        dht_local_t *local = NULL;
-        dht_conf_t  *conf  = NULL;
-        xlator_t    *subvol = NULL;
-        xlator_t    *avail_subvol = NULL;
-        int          op_errno = -1;
-
-        VALIDATE_OR_GOTO (frame, err);
-        VALIDATE_OR_GOTO (this, err);
-        VALIDATE_OR_GOTO (loc, err);
+    dht_local_t *local = NULL;
 
-        conf  = this->private;
+    local = frame->local;
 
-        dht_get_du_info (frame, this, loc);
+    if (op_ret == -1)
+        goto err;
 
-        local = dht_local_init (frame, loc, fd, GF_FOP_CREATE);
-        if (!local) {
-                op_errno = ENOMEM;
-                goto err;
-        }
+    STACK_WIND_COOKIE(frame, dht_create_cbk, local->cached_subvol,
+                      local->cached_subvol, local->cached_subvol->fops->create,
+                      &local->loc, local->flags, local->mode, local->umask,
+                      local->fd, local->params);
 
-        subvol = dht_subvol_get_hashed (this, loc);
-        if (!subvol) {
-                gf_msg_debug (this->name, 0,
-                              "no subvolume in layout for path=%s",
-                              loc->path);
-                op_errno = ENOENT;
-                goto err;
-        }
+    return 0;
 
-        avail_subvol = conf->private;
-        if (dht_is_subvol_filled (this, (xlator_t *)conf->private)) {
-                avail_subvol =
-                        dht_free_disk_available_subvol (this,
-                                                        (xlator_t *)conf->private,
-                                                        local);
-        }
+err:
+    DHT_STACK_UNWIND(create, frame, -1, op_errno, NULL, NULL, NULL, NULL, NULL,
+                     NULL);
+    return 0;
+}
 
-        if (subvol != avail_subvol) {
-                /* create a link file instead of actual file */
-                local->params = dict_ref (params);
-                local->mode = mode;
-                local->flags = flags;
-                local->umask = umask;
-                local->cached_subvol = avail_subvol;
-                dht_linkfile_create (frame, nufa_create_linkfile_create_cbk,
-                                     this, avail_subvol, subvol, loc);
-                return 0;
-        }
+int
+nufa_create(call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t flags,
+            mode_t mode, mode_t umask, fd_t *fd, dict_t *params)
+{
+    dht_local_t *local = NULL;
+    dht_conf_t *conf = NULL;
+    xlator_t *subvol = NULL;
+    xlator_t *avail_subvol = NULL;
+    int op_errno = -1;
+
+    VALIDATE_OR_GOTO(frame, err);
+    VALIDATE_OR_GOTO(this, err);
+    VALIDATE_OR_GOTO(loc, err);
+
+    conf = this->private;
+
+    dht_get_du_info(frame, this, loc);
+
+    local = dht_local_init(frame, loc, fd, GF_FOP_CREATE);
+    if (!local) {
+        op_errno = ENOMEM;
+        goto err;
+    }
+
+    subvol = dht_subvol_get_hashed(this, loc);
+    if (!subvol) {
+        gf_msg_debug(this->name, 0, "no subvolume in layout for path=%s",
+                     loc->path);
+        op_errno = ENOENT;
+        goto err;
+    }
+
+    avail_subvol = conf->private;
+    if (dht_is_subvol_filled(this, (xlator_t *)conf->private)) {
+        avail_subvol = dht_free_disk_available_subvol(
+            this, (xlator_t *)conf->private, local);
+    }
+
+    if (subvol != avail_subvol) {
+        /* create a link file instead of actual file */
+        local->params = dict_ref(params);
+        local->mode = mode;
+        local->flags = flags;
+        local->umask = umask;
+        local->cached_subvol = avail_subvol;
+        dht_linkfile_create(frame, nufa_create_linkfile_create_cbk, this,
+                            avail_subvol, subvol, loc);
+        return 0;
+    }
 
-        gf_msg_trace (this->name, 0,
-                      "creating %s on %s", loc->path, subvol->name);
+    gf_msg_trace(this->name, 0, "creating %s on %s", loc->path, subvol->name);
 
-        STACK_WIND (frame, dht_create_cbk,
-                    subvol, subvol->fops->create,
-                    loc, flags, mode, umask, fd, params);
+    STACK_WIND_COOKIE(frame, dht_create_cbk, subvol, subvol,
+                      subvol->fops->create, loc, flags, mode, umask, fd,
+                      params);
 
-        return 0;
+    return 0;
 
 err:
-        op_errno = (op_errno == -1) ? errno : op_errno;
-        DHT_STACK_UNWIND (create, frame, -1, op_errno,
-                          NULL, NULL, NULL, NULL, NULL, NULL);
+    op_errno = (op_errno == -1) ? errno : op_errno;
+    DHT_STACK_UNWIND(create, frame, -1, op_errno, NULL, NULL, NULL, NULL, NULL,
+                     NULL);
 
-        return 0;
+    return 0;
 }
 
 int
-nufa_mknod_linkfile_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                         int op_ret, int op_errno, inode_t *inode,
-                         struct iatt *stbuf, struct iatt *preparent,
-                         struct iatt *postparent, dict_t *xdata)
+nufa_mknod_linkfile_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                        int op_ret, int op_errno, inode_t *inode,
+                        struct iatt *stbuf, struct iatt *preparent,
+                        struct iatt *postparent, dict_t *xdata)
 {
-        dht_local_t  *local = NULL;
+    dht_local_t *local = NULL;
 
-        local = frame->local;
-        if (!local || !local->cached_subvol) {
-                op_errno = EINVAL;
-                op_ret = -1;
-                goto err;
-        }
+    local = frame->local;
+    if (!local || !local->cached_subvol) {
+        op_errno = EINVAL;
+        op_ret = -1;
+        goto err;
+    }
 
-        if (op_ret >= 0) {
-                STACK_WIND_COOKIE (frame, dht_newfile_cbk,
-                            (void *)local->cached_subvol, local->cached_subvol,
-                            local->cached_subvol->fops->mknod,
-                            &local->loc, local->mode, local->rdev,
-                            local->umask, local->params);
+    if (op_ret >= 0) {
+        STACK_WIND_COOKIE(
+            frame, dht_newfile_cbk, (void *)local->cached_subvol,
+            local->cached_subvol, local->cached_subvol->fops->mknod,
+            &local->loc, local->mode, local->rdev, local->umask, local->params);
 
-                return 0;
-        }
+        return 0;
+    }
 err:
-        WIPE (postparent);
-        WIPE (preparent);
+    WIPE(postparent);
+    WIPE(preparent);
 
-        DHT_STACK_UNWIND (link, frame, op_ret, op_errno,
-                          inode, stbuf, preparent, postparent, xdata);
-        return 0;
+    DHT_STACK_UNWIND(link, frame, op_ret, op_errno, inode, stbuf, preparent,
+                     postparent, xdata);
+    return 0;
 }
 
-
 int
-nufa_mknod (call_frame_t *frame, xlator_t *this,
-            loc_t *loc, mode_t mode, dev_t rdev, mode_t umask, dict_t *params)
+nufa_mknod(call_frame_t *frame, xlator_t *this, loc_t *loc, mode_t mode,
+           dev_t rdev, mode_t umask, dict_t *params)
 {
-        dht_local_t *local = NULL;
-        dht_conf_t  *conf  = NULL;
-        xlator_t    *subvol = NULL;
-        xlator_t    *avail_subvol = NULL;
-        int          op_errno = -1;
-
-        VALIDATE_OR_GOTO (frame, err);
-        VALIDATE_OR_GOTO (this, err);
-        VALIDATE_OR_GOTO (loc, err);
-
-        conf  = this->private;
-
-        dht_get_du_info (frame, this, loc);
-
-        local = dht_local_init (frame, loc, NULL, GF_FOP_MKNOD);
-        if (!local) {
-                op_errno = ENOMEM;
-                goto err;
-        }
-
-        subvol = dht_subvol_get_hashed (this, loc);
-        if (!subvol) {
-                gf_msg_debug (this->name, 0,
-                              "no subvolume in layout for path=%s",
-                              loc->path);
-                op_errno = ENOENT;
-                goto err;
-        }
-
-        /* Consider the disksize in consideration */
-        avail_subvol = conf->private;
-        if (dht_is_subvol_filled (this, (xlator_t *)conf->private)) {
-                avail_subvol =
-                        dht_free_disk_available_subvol (this,
-                                                        (xlator_t *)conf->private,
-                                                        local);
-        }
-
-        if (avail_subvol != subvol) {
-                /* Create linkfile first */
-
-                local->params = dict_ref (params);
-                local->mode = mode;
-                local->umask = umask;
-                local->rdev = rdev;
-                local->cached_subvol = avail_subvol;
-
-                dht_linkfile_create (frame, nufa_mknod_linkfile_cbk, this,
-                                     avail_subvol, subvol, loc);
-                return 0;
-        }
+    dht_local_t *local = NULL;
+    dht_conf_t *conf = NULL;
+    xlator_t *subvol = NULL;
+    xlator_t *avail_subvol = NULL;
+    int op_errno = -1;
+
+    VALIDATE_OR_GOTO(frame, err);
+    VALIDATE_OR_GOTO(this, err);
+    VALIDATE_OR_GOTO(loc, err);
+
+    conf = this->private;
+
+    dht_get_du_info(frame, this, loc);
+
+    local = dht_local_init(frame, loc, NULL, GF_FOP_MKNOD);
+    if (!local) {
+        op_errno = ENOMEM;
+        goto err;
+    }
+
+    subvol = dht_subvol_get_hashed(this, loc);
+    if (!subvol) {
+        gf_msg_debug(this->name, 0, "no subvolume in layout for path=%s",
+                     loc->path);
+        op_errno = ENOENT;
+        goto err;
+    }
+
+    /* Consider the disksize in consideration */
+    avail_subvol = conf->private;
+    if (dht_is_subvol_filled(this, (xlator_t *)conf->private)) {
+        avail_subvol = dht_free_disk_available_subvol(
+            this, (xlator_t *)conf->private, local);
+    }
+
+    if (avail_subvol != subvol) {
+        /* Create linkfile first */
+
+        local->params = dict_ref(params);
+        local->mode = mode;
+        local->umask = umask;
+        local->rdev = rdev;
+        local->cached_subvol = avail_subvol;
+
+        dht_linkfile_create(frame, nufa_mknod_linkfile_cbk, this, avail_subvol,
+                            subvol, loc);
+        return 0;
+    }
 
-        gf_msg_trace (this->name, 0,
-                      "creating %s on %s", loc->path, subvol->name);
+    gf_msg_trace(this->name, 0, "creating %s on %s", loc->path, subvol->name);
 
-        STACK_WIND_COOKIE (frame, dht_newfile_cbk, (void *)subvol, subvol,
-                           subvol->fops->mknod, loc, mode, rdev, umask,
-                           params);
+    STACK_WIND_COOKIE(frame, dht_newfile_cbk, (void *)subvol, subvol,
+                      subvol->fops->mknod, loc, mode, rdev, umask, params);
 
-        return 0;
+    return 0;
 
 err:
-        op_errno = (op_errno == -1) ? errno : op_errno;
-        DHT_STACK_UNWIND (mknod, frame, -1, op_errno,
-                          NULL, NULL, NULL, NULL, NULL);
+    op_errno = (op_errno == -1) ? errno : op_errno;
+    DHT_STACK_UNWIND(mknod, frame, -1, op_errno, NULL, NULL, NULL, NULL, NULL);
 
-        return 0;
+    return 0;
 }
 
-
 gf_boolean_t
-same_first_part (char *str1, char term1, char *str2, char term2)
+same_first_part(char *str1, char term1, char *str2, char term2)
 {
-        gf_boolean_t    ended1;
-        gf_boolean_t    ended2;
-
-        for (;;) {
-                ended1 = ((*str1 == '\0') || (*str1 == term1));
-                ended2 = ((*str2 == '\0') || (*str2 == term2));
-                if (ended1 && ended2) {
-                        return _gf_true;
-                }
-                if (ended1 || ended2 || (*str1 != *str2)) {
-                        return _gf_false;
-                }
-                ++str1;
-                ++str2;
+    gf_boolean_t ended1;
+    gf_boolean_t ended2;
+
+    for (;;) {
+        ended1 = ((*str1 == '\0') || (*str1 == term1));
+        ended2 = ((*str2 == '\0') || (*str2 == term2));
+        if (ended1 && ended2) {
+            return _gf_true;
         }
+        if (ended1 || ended2 || (*str1 != *str2)) {
+            return _gf_false;
+        }
+        ++str1;
+        ++str2;
+    }
 }
 
 typedef struct nufa_args {
-        xlator_t    *this;
-        char        *volname;
-        gf_boolean_t addr_match;
+    xlator_t *this;
+    char *volname;
+    gf_boolean_t addr_match;
 } nufa_args_t;
 
 static void
-nufa_find_local_brick (xlator_t *xl, void *data)
+nufa_find_local_brick(xlator_t *xl, void *data)
 {
-        nufa_args_t     *args = data;
-        xlator_t        *this = args->this;
-        char            *local_volname = args->volname;
-        gf_boolean_t    addr_match = args->addr_match;
-        char            *brick_host = NULL;
-        dht_conf_t      *conf = this->private;
-        int             ret = -1;
-
-        /*This means a local subvol was already found. We pick the first brick
-         * that is local*/
-        if (conf->private)
-                return;
-
-        if (strcmp (xl->name, local_volname) == 0) {
-                conf->private = xl;
-                gf_log (this->name, GF_LOG_INFO, "Using specified subvol %s",
-                        local_volname);
-                return;
-        }
-
-        if (!addr_match)
-                return;
-
-        ret = dict_get_str (xl->options, "remote-host", &brick_host);
-        if ((ret == 0) &&
-            (gf_is_same_address (local_volname, brick_host) ||
-             gf_is_local_addr (brick_host))) {
-                conf->private = xl;
-                gf_log (this->name, GF_LOG_INFO, "Using the first local "
-                        "subvol %s", xl->name);
-                return;
-        }
-
+    nufa_args_t *args = data;
+    xlator_t *this = args->this;
+    char *local_volname = args->volname;
+    gf_boolean_t addr_match = args->addr_match;
+    char *brick_host = NULL;
+    dht_conf_t *conf = this->private;
+    int ret = -1;
+
+    /*This means a local subvol was already found. We pick the first brick
+     * that is local*/
+    if (conf->private)
+        return;
+
+    if (strcmp(xl->name, local_volname) == 0) {
+        conf->private = xl;
+        gf_msg(this->name, GF_LOG_INFO, 0, DHT_MSG_SUBVOL_INFO,
+               "Using specified subvol %s", local_volname);
+        return;
+    }
+
+    if (!addr_match)
+        return;
+
+    ret = dict_get_str(xl->options, "remote-host", &brick_host);
+    if ((ret == 0) && (gf_is_same_address(local_volname, brick_host) ||
+                       gf_is_local_addr(brick_host))) {
+        conf->private = xl;
+        gf_msg(this->name, GF_LOG_INFO, 0, DHT_MSG_SUBVOL_INFO,
+               "Using the first local "
+               "subvol %s",
+               xl->name);
+        return;
+    }
 }
 
 static void
-nufa_to_dht (xlator_t *this)
+nufa_to_dht(xlator_t *this)
 {
-        GF_ASSERT (this);
-        GF_ASSERT (this->fops);
+    GF_ASSERT(this);
+    GF_ASSERT(this->fops);
 
-        this->fops->lookup = dht_lookup;
-        this->fops->create = dht_create;
-        this->fops->mknod  = dht_mknod;
+    this->fops->lookup = dht_lookup;
+    this->fops->create = dht_create;
+    this->fops->mknod = dht_mknod;
 }
 
 int
-nufa_find_local_subvol (xlator_t *this,
-                        void (*fn) (xlator_t *each, void* data), void *data)
+nufa_find_local_subvol(xlator_t *this, void (*fn)(xlator_t *each, void *data),
+                       void *data)
 {
-        int             ret = -1;
-        dht_conf_t      *conf = this->private;
-        xlator_list_t   *trav = NULL;
-        xlator_t        *parent = NULL;
-        xlator_t        *candidate = NULL;
-
-        xlator_foreach_depth_first (this, fn, data);
-        if (!conf->private) {
-                gf_log (this->name, GF_LOG_ERROR, "Couldn't find a local "
-                        "brick");
-                return -1;
+    int ret = -1;
+    dht_conf_t *conf = this->private;
+    xlator_list_t *trav = NULL;
+    xlator_t *parent = NULL;
+    xlator_t *candidate = NULL;
+
+    xlator_foreach_depth_first(this, fn, data);
+    if (!conf->private) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, DHT_MSG_BRICK_ERROR,
+               "Couldn't find a local "
+               "brick");
+        return -1;
+    }
+
+    candidate = conf->private;
+    trav = candidate->parents;
+    while (trav) {
+        parent = trav->xlator;
+        if (strcmp(parent->type, "cluster/nufa") == 0) {
+            gf_msg(this->name, GF_LOG_INFO, 0, DHT_MSG_SUBVOL_INFO,
+                   "Found local subvol, "
+                   "%s",
+                   candidate->name);
+            ret = 0;
+            conf->private = candidate;
+            break;
         }
 
-        candidate = conf->private;
-        trav = candidate->parents;
-        while (trav) {
-
-                parent = trav->xlator;
-                if (strcmp (parent->type, "cluster/nufa") == 0) {
-                        gf_log (this->name, GF_LOG_INFO, "Found local subvol, "
-                                "%s", candidate->name);
-                        ret = 0;
-                        conf->private = candidate;
-                        break;
-                }
-
-                candidate = parent;
-                trav = parent->parents;
-        }
+        candidate = parent;
+        trav = parent->parents;
+    }
 
-        return ret;
+    return ret;
 }
 
 int
-nufa_init (xlator_t *this)
+nufa_init(xlator_t *this)
 {
-        data_t        *data = NULL;
-        char          *local_volname = NULL;
-        int            ret = -1;
-        char           my_hostname[256];
-        gf_boolean_t   addr_match = _gf_false;
-        nufa_args_t    args = {0, };
-
-        ret = dht_init(this);
-        if (ret) {
-                return ret;
-        }
-
-        if ((data = dict_get (this->options, "local-volume-name"))) {
-                local_volname = data->data;
-
-        } else {
-                addr_match = _gf_true;
-                local_volname = "localhost";
-                ret = gethostname (my_hostname, 256);
-                if (ret == 0)
-                        local_volname = my_hostname;
-
-                else
-                        gf_log (this->name, GF_LOG_WARNING,
-                                "could not find hostname (%s)",
-                                strerror (errno));
-
-        }
-
-        args.this = this;
-        args.volname = local_volname;
-        args.addr_match = addr_match;
-        ret = nufa_find_local_subvol (this, nufa_find_local_brick, &args);
-        if (ret) {
-                gf_log (this->name, GF_LOG_INFO,
-                        "Unable to find local subvolume, switching "
-                        "to dht mode");
-                nufa_to_dht (this);
-        }
-        return 0;
+    data_t *data = NULL;
+    char *local_volname = NULL;
+    int ret = -1;
+    char my_hostname[256];
+    gf_boolean_t addr_match = _gf_false;
+    nufa_args_t args = {
+        0,
+    };
+
+    ret = dht_init(this);
+    if (ret) {
+        return ret;
+    }
+
+    if ((data = dict_get(this->options, "local-volume-name"))) {
+        local_volname = data->data;
+
+    } else {
+        addr_match = _gf_true;
+        local_volname = "localhost";
+        ret = gethostname(my_hostname, 256);
+        if (ret == 0)
+            local_volname = my_hostname;
+
+        else
+            gf_msg(this->name, GF_LOG_WARNING, errno,
+                   DHT_MSG_GET_HOSTNAME_FAILED, "could not find hostname");
+    }
+
+    args.this = this;
+    args.volname = local_volname;
+    args.addr_match = addr_match;
+    ret = nufa_find_local_subvol(this, nufa_find_local_brick, &args);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_INFO, 0, DHT_MSG_SUBVOL_INFO,
+               "Unable to find local subvolume, switching "
+               "to dht mode");
+        nufa_to_dht(this);
+    }
+    return 0;
 }
 
 dht_methods_t dht_methods = {
-        .migration_get_dst_subvol = dht_migration_get_dst_subvol,
-        .migration_needed = dht_migration_needed,
-        .layout_search   = dht_layout_search,
-};
-
-class_methods_t class_methods = {
-        .init           = nufa_init,
-        .fini           = dht_fini,
-        .reconfigure    = dht_reconfigure,
-        .notify         = dht_notify
+    .migration_get_dst_subvol = dht_migration_get_dst_subvol,
+    .layout_search = dht_layout_search,
 };
 
-
 struct xlator_fops fops = {
-        .lookup      = nufa_lookup,
-        .create      = nufa_create,
-        .mknod       = nufa_mknod,
-
-        .stat        = dht_stat,
-        .fstat       = dht_fstat,
-        .truncate    = dht_truncate,
-        .ftruncate   = dht_ftruncate,
-        .access      = dht_access,
-        .readlink    = dht_readlink,
-        .setxattr    = dht_setxattr,
-        .getxattr    = dht_getxattr,
-        .removexattr = dht_removexattr,
-        .open        = dht_open,
-        .readv       = dht_readv,
-        .writev      = dht_writev,
-        .flush       = dht_flush,
-        .fsync       = dht_fsync,
-        .statfs      = dht_statfs,
-        .lk          = dht_lk,
-        .opendir     = dht_opendir,
-        .readdir     = dht_readdir,
-        .readdirp    = dht_readdirp,
-        .fsyncdir    = dht_fsyncdir,
-        .symlink     = dht_symlink,
-        .unlink      = dht_unlink,
-        .link        = dht_link,
-        .mkdir       = dht_mkdir,
-        .rmdir       = dht_rmdir,
-        .rename      = dht_rename,
-        .inodelk     = dht_inodelk,
-        .finodelk    = dht_finodelk,
-        .entrylk     = dht_entrylk,
-        .fentrylk    = dht_fentrylk,
-        .xattrop     = dht_xattrop,
-        .fxattrop    = dht_fxattrop,
-        .setattr     = dht_setattr,
+    .lookup = nufa_lookup,
+    .create = nufa_create,
+    .mknod = nufa_mknod,
+
+    .stat = dht_stat,
+    .fstat = dht_fstat,
+    .truncate = dht_truncate,
+    .ftruncate = dht_ftruncate,
+    .access = dht_access,
+    .readlink = dht_readlink,
+    .setxattr = dht_setxattr,
+    .getxattr = dht_getxattr,
+    .removexattr = dht_removexattr,
+    .open = dht_open,
+    .readv = dht_readv,
+    .writev = dht_writev,
+    .flush = dht_flush,
+    .fsync = dht_fsync,
+    .statfs = dht_statfs,
+    .lk = dht_lk,
+    .opendir = dht_opendir,
+    .readdir = dht_readdir,
+    .readdirp = dht_readdirp,
+    .fsyncdir = dht_fsyncdir,
+    .symlink = dht_symlink,
+    .unlink = dht_unlink,
+    .link = dht_link,
+    .mkdir = dht_mkdir,
+    .rmdir = dht_rmdir,
+    .rename = dht_rename,
+    .inodelk = dht_inodelk,
+    .finodelk = dht_finodelk,
+    .entrylk = dht_entrylk,
+    .fentrylk = dht_fentrylk,
+    .xattrop = dht_xattrop,
+    .fxattrop = dht_fxattrop,
+    .setattr = dht_setattr,
 };
 
-
-struct xlator_cbks cbks = {
-        .forget     = dht_forget
+struct xlator_cbks cbks = {.forget = dht_forget};
+extern int32_t
+mem_acct_init(xlator_t *this);
+
+xlator_api_t xlator_api = {
+    .init = nufa_init,
+    .fini = dht_fini,
+    .notify = dht_notify,
+    .reconfigure = dht_reconfigure,
+    .mem_acct_init = mem_acct_init,
+    .op_version = {1}, /* Present from the initial version */
+    .fops = &fops,
+    .cbks = &cbks,
+    .options = dht_options,
+    .identifier = "nufa",
+    .category = GF_TECH_PREVIEW,
 };
diff --git a/xlators/cluster/dht/src/switch.c b/xlators/cluster/dht/src/switch.c
index 0071dfa265d..207d109a025 100644
--- a/xlators/cluster/dht/src/switch.c
+++ b/xlators/cluster/dht/src/switch.c
@@ -8,12 +8,6 @@
   cases as published by the Free Software Foundation.
 */
 
-
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
-
 #include "dht-common.h"
 #include "dht-mem-types.h"
 
@@ -22,885 +16,876 @@
 #include <fnmatch.h>
 #include <string.h>
 
-extern struct volume_options options[];
+extern struct volume_options dht_options[];
 
 struct switch_sched_array {
-        xlator_t *xl;
-        int32_t   eligible;
-        int32_t   considered;
+    xlator_t *xl;
+    int32_t eligible;
+    int32_t considered;
 };
 
 /* Select one of this struct based on the path's pattern match */
 struct switch_struct {
-        struct switch_struct      *next;
-        struct switch_sched_array *array;
-        int32_t                    node_index; /* Index of the node in
-                                                  this pattern. */
-        int32_t                    num_child;  /* Total num of child nodes
-                                                  with this pattern. */
-        char                       path_pattern[256];
+    struct switch_struct *next;
+    struct switch_sched_array *array;
+    int32_t node_index; /* Index of the node in
+                           this pattern. */
+    int32_t num_child;  /* Total num of child nodes
+                           with this pattern. */
+    char path_pattern[256];
 };
 
 /* TODO: all 'TODO's in dht.c holds good */
 /* This function should return child node as '*:subvolumes' is inserterd */
 
 static int32_t
-gf_switch_valid_child (xlator_t *this, const char *child)
+gf_switch_valid_child(xlator_t *this, const char *child)
 {
-        xlator_list_t *children = NULL;
-        int32_t        ret = 0;
-
-        children = this->children;
-        while (children) {
-                if (!strcmp (child, children->xlator->name)) {
-                        ret = 1;
-                        break;
-                }
-                children = children->next;
+    xlator_list_t *children = NULL;
+    int32_t ret = 0;
+
+    children = this->children;
+    while (children) {
+        if (!strcmp(child, children->xlator->name)) {
+            ret = 1;
+            break;
         }
+        children = children->next;
+    }
 
-        return ret;
+    return ret;
 }
 
 static xlator_t *
-get_switch_matching_subvol (const char *path, dht_conf_t *conf,
-                            xlator_t *hashed_subvol)
+get_switch_matching_subvol(const char *path, dht_conf_t *conf,
+                           xlator_t *hashed_subvol)
 {
-        struct switch_struct *cond      = NULL;
-        struct switch_struct *trav      = NULL;
-        char                 *pathname  = NULL;
-        int                   idx       = 0;
-        xlator_t             *subvol    = NULL;
-
-        cond = conf->private;
-        subvol = hashed_subvol;
-        if (!cond)
-                goto out;
-
-        pathname = gf_strdup (path);
-        if (!pathname)
-                goto out;
-
-        trav = cond;
-        while (trav) {
-                if (fnmatch (trav->path_pattern,
-                             pathname, FNM_NOESCAPE) == 0) {
-                        for (idx = 0; idx < trav->num_child; idx++) {
-                                if (trav->array[idx].xl == hashed_subvol)
-                                        goto out;
-                        }
-                        idx = trav->node_index++;
-                        trav->node_index %= trav->num_child;
-                        subvol = trav->array[idx].xl;
-                        goto out;
-                }
-                trav = trav->next;
+    struct switch_struct *cond = NULL;
+    struct switch_struct *trav = NULL;
+    char *pathname = NULL;
+    int idx = 0;
+    xlator_t *subvol = NULL;
+
+    cond = conf->private;
+    subvol = hashed_subvol;
+    if (!cond)
+        goto out;
+
+    pathname = gf_strdup(path);
+    if (!pathname)
+        goto out;
+
+    trav = cond;
+    while (trav) {
+        if (fnmatch(trav->path_pattern, pathname, FNM_NOESCAPE) == 0) {
+            for (idx = 0; idx < trav->num_child; idx++) {
+                if (trav->array[idx].xl == hashed_subvol)
+                    goto out;
+            }
+            idx = trav->node_index++;
+            trav->node_index %= trav->num_child;
+            subvol = trav->array[idx].xl;
+            goto out;
         }
+        trav = trav->next;
+    }
 out:
-        GF_FREE (pathname);
+    GF_FREE(pathname);
 
-        return subvol;
+    return subvol;
 }
 
-
 int
-switch_local_lookup_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                         int op_ret, int op_errno,
-                         inode_t *inode, struct iatt *stbuf, dict_t *xattr,
-                         struct iatt *postparent)
+switch_local_lookup_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                        int op_ret, int op_errno, inode_t *inode,
+                        struct iatt *stbuf, dict_t *xattr,
+                        struct iatt *postparent)
 {
-        xlator_t     *subvol      = NULL;
-        char          is_linkfile = 0;
-        char          is_dir      = 0;
-        dht_conf_t   *conf        = NULL;
-        dht_local_t  *local       = NULL;
-        loc_t        *loc         = NULL;
-        int           i           = 0;
-        call_frame_t *prev        = NULL;
-        int           call_cnt    = 0;
-        int           ret         = 0;
-
-        conf  = this->private;
-
-        prev  = cookie;
-        local = frame->local;
-        loc   = &local->loc;
-
-        if (ENTRY_MISSING (op_ret, op_errno)) {
-                if (conf->search_unhashed) {
-                        local->op_errno = ENOENT;
-                        dht_lookup_everywhere (frame, this, loc);
-                        return 0;
-                }
+    xlator_t *subvol = NULL;
+    char is_linkfile = 0;
+    char is_dir = 0;
+    dht_conf_t *conf = NULL;
+    dht_local_t *local = NULL;
+    loc_t *loc = NULL;
+    int i = 0;
+    xlator_t *prev = NULL;
+    int call_cnt = 0;
+    int ret = 0;
+
+    conf = this->private;
+
+    prev = cookie;
+    local = frame->local;
+    loc = &local->loc;
+
+    if (ENTRY_MISSING(op_ret, op_errno)) {
+        if (conf->search_unhashed) {
+            local->op_errno = ENOENT;
+            dht_lookup_everywhere(frame, this, loc);
+            return 0;
         }
-
-        if (op_ret == -1)
-                goto out;
-
-        is_linkfile = check_is_linkfile (inode, stbuf, xattr,
-                                         conf->link_xattr_name);
-        is_dir      = check_is_dir (inode, stbuf, xattr);
-
-        if (!is_dir && !is_linkfile) {
-                /* non-directory and not a linkfile */
-
-                ret = dht_layout_preset (this, prev->this, inode);
-                if (ret < 0) {
-                        gf_msg_debug (this->name, 0,
-                                      "could not set pre-set layout "
-                                      "for subvol %s",
-                                      prev->this->name);
-                        op_ret   = -1;
-                        op_errno = EINVAL;
-                        goto err;
-                }
-
-                goto out;
+    }
+
+    if (op_ret == -1)
+        goto out;
+
+    is_linkfile = check_is_linkfile(inode, stbuf, xattr, conf->link_xattr_name);
+    is_dir = check_is_dir(inode, stbuf, xattr);
+
+    if (!is_dir && !is_linkfile) {
+        /* non-directory and not a linkfile */
+
+        ret = dht_layout_preset(this, prev, inode);
+        if (ret < 0) {
+            gf_msg_debug(this->name, 0,
+                         "could not set pre-set layout "
+                         "for subvol %s",
+                         prev->name);
+            op_ret = -1;
+            op_errno = EINVAL;
+            goto err;
         }
 
-        if (is_dir) {
-                call_cnt        = conf->subvolume_cnt;
-                local->call_cnt = call_cnt;
+        goto out;
+    }
 
-                local->inode = inode_ref (inode);
-                local->xattr = dict_ref (xattr);
+    if (is_dir) {
+        call_cnt = conf->subvolume_cnt;
+        local->call_cnt = call_cnt;
 
-                local->op_ret = 0;
-                local->op_errno = 0;
+        local->inode = inode_ref(inode);
+        local->xattr = dict_ref(xattr);
 
-                local->layout = dht_layout_new (this, conf->subvolume_cnt);
-                if (!local->layout) {
-                        op_ret   = -1;
-                        op_errno = ENOMEM;
-                        gf_msg_debug (this->name, 0,
-                                      "memory allocation failed :(");
-                        goto err;
-                }
+        local->op_ret = 0;
+        local->op_errno = 0;
 
-                for (i = 0; i < call_cnt; i++) {
-                        STACK_WIND (frame, dht_lookup_dir_cbk,
-                                    conf->subvolumes[i],
-                                    conf->subvolumes[i]->fops->lookup,
-                                    &local->loc, local->xattr_req);
-                }
+        local->layout = dht_layout_new(this, conf->subvolume_cnt);
+        if (!local->layout) {
+            op_ret = -1;
+            op_errno = ENOMEM;
+            gf_msg_debug(this->name, 0, "memory allocation failed :(");
+            goto err;
         }
 
-        if (is_linkfile) {
-                subvol = dht_linkfile_subvol (this, inode, stbuf, xattr);
+        for (i = 0; i < call_cnt; i++) {
+            STACK_WIND_COOKIE(frame, dht_lookup_dir_cbk, conf->subvolumes[i],
+                              conf->subvolumes[i],
+                              conf->subvolumes[i]->fops->lookup, &local->loc,
+                              local->xattr_req);
+        }
+    }
 
-                if (!subvol) {
-                        gf_msg_debug (this->name, 0,
-                                      "linkfile has no link subvolume.path=%s",
-                                      loc->path);
-                        dht_lookup_everywhere (frame, this, loc);
-                        return 0;
-                }
+    if (is_linkfile) {
+        subvol = dht_linkfile_subvol(this, inode, stbuf, xattr);
 
-                STACK_WIND (frame, dht_lookup_linkfile_cbk,
-                            subvol, subvol->fops->lookup,
-                            &local->loc, local->xattr_req);
+        if (!subvol) {
+            gf_msg_debug(this->name, 0,
+                         "linkfile has no link subvolume.path=%s", loc->path);
+            dht_lookup_everywhere(frame, this, loc);
+            return 0;
         }
 
-        return 0;
+        STACK_WIND_COOKIE(frame, dht_lookup_linkfile_cbk, subvol, subvol,
+                          subvol->fops->lookup, &local->loc, local->xattr_req);
+    }
+
+    return 0;
 
 out:
-        if (!local->hashed_subvol) {
-                gf_msg_debug (this->name, 0,
-                              "no subvolume in layout for path=%s",
-                              local->loc.path);
-                local->op_errno = ENOENT;
-                dht_lookup_everywhere (frame, this, loc);
-                return 0;
-        }
+    if (!local->hashed_subvol) {
+        gf_msg_debug(this->name, 0, "no subvolume in layout for path=%s",
+                     local->loc.path);
+        local->op_errno = ENOENT;
+        dht_lookup_everywhere(frame, this, loc);
+        return 0;
+    }
 
-        STACK_WIND (frame, dht_lookup_cbk,
-                    local->hashed_subvol, local->hashed_subvol->fops->lookup,
-                    &local->loc, local->xattr_req);
+    STACK_WIND_COOKIE(frame, dht_lookup_cbk, local->hashed_subvol,
+                      local->hashed_subvol, local->hashed_subvol->fops->lookup,
+                      &local->loc, local->xattr_req);
 
-        return 0;
+    return 0;
 
 err:
-        DHT_STACK_UNWIND (lookup, frame, op_ret, op_errno,
-                          inode, stbuf, xattr, NULL);
-        return 0;
+    DHT_STACK_UNWIND(lookup, frame, op_ret, op_errno, inode, stbuf, xattr,
+                     NULL);
+    return 0;
 }
 
 int
-switch_lookup (call_frame_t *frame, xlator_t *this,
-               loc_t *loc, dict_t *xattr_req)
+switch_lookup(call_frame_t *frame, xlator_t *this, loc_t *loc,
+              dict_t *xattr_req)
 {
-        xlator_t     *hashed_subvol = NULL;
-        xlator_t     *cached_subvol = NULL;
-        xlator_t     *subvol = NULL;
-        dht_local_t  *local  = NULL;
-        dht_conf_t   *conf = NULL;
-        int           ret    = -1;
-        int           op_errno = -1;
-        dht_layout_t *layout = NULL;
-        int           i = 0;
-        int           call_cnt = 0;
-
-
-        VALIDATE_OR_GOTO (frame, err);
-        VALIDATE_OR_GOTO (this, err);
-        VALIDATE_OR_GOTO (loc, err);
-        VALIDATE_OR_GOTO (loc->inode, err);
-        VALIDATE_OR_GOTO (loc->path, err);
-
-        conf = this->private;
-
-        local = dht_local_init (frame, loc, NULL, GF_FOP_LOOKUP);
-        if (!local) {
-                op_errno = ENOMEM;
-                goto err;
+    xlator_t *hashed_subvol = NULL;
+    xlator_t *cached_subvol = NULL;
+    xlator_t *subvol = NULL;
+    dht_local_t *local = NULL;
+    dht_conf_t *conf = NULL;
+    int ret = -1;
+    int op_errno = -1;
+    dht_layout_t *layout = NULL;
+    int i = 0;
+    int call_cnt = 0;
+
+    VALIDATE_OR_GOTO(frame, err);
+    VALIDATE_OR_GOTO(this, err);
+    VALIDATE_OR_GOTO(loc, err);
+    VALIDATE_OR_GOTO(loc->inode, err);
+    VALIDATE_OR_GOTO(loc->path, err);
+
+    conf = this->private;
+
+    local = dht_local_init(frame, loc, NULL, GF_FOP_LOOKUP);
+    if (!local) {
+        op_errno = ENOMEM;
+        goto err;
+    }
+
+    if (xattr_req) {
+        local->xattr_req = dict_ref(xattr_req);
+    } else {
+        local->xattr_req = dict_new();
+    }
+
+    hashed_subvol = dht_subvol_get_hashed(this, &local->loc);
+    cached_subvol = local->cached_subvol;
+
+    local->hashed_subvol = hashed_subvol;
+
+    if (is_revalidate(loc)) {
+        layout = local->layout;
+        if (!layout) {
+            gf_msg_debug(this->name, 0,
+                         "revalidate lookup without cache. path=%s", loc->path);
+            op_errno = EINVAL;
+            goto err;
         }
 
-        if (xattr_req) {
-                local->xattr_req = dict_ref (xattr_req);
-        } else {
-                local->xattr_req = dict_new ();
+        if (layout->gen && (layout->gen < conf->gen)) {
+            gf_msg_debug(this->name, 0, "incomplete layout failure for path=%s",
+                         loc->path);
+            dht_layout_unref(this, local->layout);
+            goto do_fresh_lookup;
         }
 
-        hashed_subvol = dht_subvol_get_hashed (this, &local->loc);
-        cached_subvol = local->cached_subvol;
+        local->inode = inode_ref(loc->inode);
 
-        local->hashed_subvol = hashed_subvol;
+        local->call_cnt = layout->cnt;
+        call_cnt = local->call_cnt;
 
-        if (is_revalidate (loc)) {
-                layout = local->layout;
-                if (!layout) {
-                        gf_msg_debug(this->name, 0,
-                                     "revalidate lookup without cache. path=%s",
-                                     loc->path);
-                        op_errno = EINVAL;
-                        goto err;
-                }
-
-                if (layout->gen && (layout->gen < conf->gen)) {
-                        gf_msg_debug (this->name, 0,
-                                      "incomplete layout failure for path=%s",
-                                      loc->path);
-                        dht_layout_unref (this, local->layout);
-                        goto do_fresh_lookup;
-                }
-
-                local->inode    = inode_ref (loc->inode);
-
-                local->call_cnt = layout->cnt;
-                call_cnt = local->call_cnt;
+        /* NOTE: we don't require 'trusted.glusterfs.dht.linkto'
+         * attribute, revalidates directly go to the cached-subvolume.
+         */
+        ret = dict_set_uint32(local->xattr_req, conf->xattr_name, 4 * 4);
+        if (ret < 0)
+            gf_msg(this->name, GF_LOG_WARNING, 0, DHT_MSG_DICT_SET_FAILED,
+                   "failed to set dict value for %s", conf->xattr_name);
 
-                /* NOTE: we don't require 'trusted.glusterfs.dht.linkto'
-                 * attribute, revalidates directly go to the cached-subvolume.
-                 */
-                ret = dict_set_uint32 (local->xattr_req,
-                                       conf->xattr_name, 4 * 4);
-                if (ret < 0)
-                        gf_log (this->name, GF_LOG_WARNING,
-                                "failed to set dict value for %s",
-                                conf->xattr_name);
+        for (i = 0; i < layout->cnt; i++) {
+            subvol = layout->list[i].xlator;
 
-                for (i = 0; i < layout->cnt; i++) {
-                        subvol = layout->list[i].xlator;
+            STACK_WIND_COOKIE(frame, dht_revalidate_cbk, subvol, subvol,
+                              subvol->fops->lookup, loc, local->xattr_req);
 
-                        STACK_WIND (frame, dht_revalidate_cbk,
-                                    subvol, subvol->fops->lookup,
-                                    loc, local->xattr_req);
+            if (!--call_cnt)
+                break;
+        }
+    } else {
+    do_fresh_lookup:
+        ret = dict_set_uint32(local->xattr_req, conf->xattr_name, 4 * 4);
+        if (ret < 0)
+            gf_msg(this->name, GF_LOG_WARNING, 0, DHT_MSG_DICT_SET_FAILED,
+                   "failed to set dict value for %s", conf->xattr_name);
+
+        ret = dict_set_uint32(local->xattr_req, conf->link_xattr_name, 256);
+        if (ret < 0)
+            gf_msg(this->name, GF_LOG_WARNING, EINVAL, DHT_MSG_DICT_SET_FAILED,
+                   "failed to set dict value for %s", conf->link_xattr_name);
+
+        if (!hashed_subvol) {
+            gf_msg_debug(this->name, 0,
+                         "no subvolume in layout for path=%s, "
+                         "checking on all the subvols to see if "
+                         "it is a directory",
+                         loc->path);
+            call_cnt = conf->subvolume_cnt;
+            local->call_cnt = call_cnt;
+
+            local->layout = dht_layout_new(this, conf->subvolume_cnt);
+            if (!local->layout) {
+                op_errno = ENOMEM;
+                goto err;
+            }
+
+            for (i = 0; i < call_cnt; i++) {
+                STACK_WIND_COOKIE(frame, dht_lookup_dir_cbk,
+                                  conf->subvolumes[i], conf->subvolumes[i],
+                                  conf->subvolumes[i]->fops->lookup,
+                                  &local->loc, local->xattr_req);
+            }
+            return 0;
+        }
 
-                        if (!--call_cnt)
-                                break;
-                }
+        /*  */
+        cached_subvol = get_switch_matching_subvol(loc->path, conf,
+                                                   hashed_subvol);
+        if (cached_subvol == hashed_subvol) {
+            STACK_WIND_COOKIE(frame, dht_lookup_cbk, hashed_subvol,
+                              hashed_subvol, hashed_subvol->fops->lookup, loc,
+                              local->xattr_req);
         } else {
-        do_fresh_lookup:
-                ret = dict_set_uint32 (local->xattr_req,
-                                       conf->xattr_name, 4 * 4);
-                if (ret < 0)
-                        gf_log (this->name, GF_LOG_WARNING,
-                                "failed to set dict value for %s",
-                                conf->xattr_name);
-
-                ret = dict_set_uint32 (local->xattr_req,
-                                       conf->link_xattr_name, 256);
-                if (ret < 0)
-                        gf_log (this->name, GF_LOG_WARNING,
-                                "failed to set dict value for %s",
-                                conf->link_xattr_name);
-
-                if (!hashed_subvol) {
-                        gf_msg_debug (this->name, 0,
-                                      "no subvolume in layout for path=%s, "
-                                      "checking on all the subvols to see if "
-                                      "it is a directory", loc->path);
-                        call_cnt        = conf->subvolume_cnt;
-                        local->call_cnt = call_cnt;
-
-                        local->layout = dht_layout_new (this,
-                                                        conf->subvolume_cnt);
-                        if (!local->layout) {
-                                op_errno = ENOMEM;
-                                goto err;
-                        }
-
-                        for (i = 0; i < call_cnt; i++) {
-                                STACK_WIND (frame, dht_lookup_dir_cbk,
-                                            conf->subvolumes[i],
-                                            conf->subvolumes[i]->fops->lookup,
-                                            &local->loc, local->xattr_req);
-                        }
-                        return 0;
-                }
-
-                /*  */
-                cached_subvol = get_switch_matching_subvol (loc->path, conf,
-                                                            hashed_subvol);
-                if (cached_subvol == hashed_subvol) {
-                        STACK_WIND (frame, dht_lookup_cbk,
-                                    hashed_subvol,
-                                    hashed_subvol->fops->lookup,
-                                    loc, local->xattr_req);
-                } else {
-                        STACK_WIND (frame, switch_local_lookup_cbk,
-                                    cached_subvol,
-                                    cached_subvol->fops->lookup,
-                                    loc, local->xattr_req);
-                }
+            STACK_WIND_COOKIE(frame, switch_local_lookup_cbk, cached_subvol,
+                              cached_subvol, cached_subvol->fops->lookup, loc,
+                              local->xattr_req);
         }
+    }
 
-        return 0;
+    return 0;
 
 err:
-        op_errno = (op_errno == -1) ? errno : op_errno;
-        DHT_STACK_UNWIND (lookup, frame, -1, op_errno,
-                          NULL, NULL, NULL, NULL);
-        return 0;
+    op_errno = (op_errno == -1) ? errno : op_errno;
+    DHT_STACK_UNWIND(lookup, frame, -1, op_errno, NULL, NULL, NULL, NULL);
+    return 0;
 }
 
 int
-switch_create_linkfile_create_cbk (call_frame_t *frame, void *cookie,
-                                   xlator_t *this, int op_ret, int op_errno,
-                                   inode_t *inode, struct iatt *stbuf,
-                                   struct iatt *preparent,
-                                   struct iatt *postparent, dict_t *xdata)
+switch_create_linkfile_create_cbk(call_frame_t *frame, void *cookie,
+                                  xlator_t *this, int op_ret, int op_errno,
+                                  inode_t *inode, struct iatt *stbuf,
+                                  struct iatt *preparent,
+                                  struct iatt *postparent, dict_t *xdata)
 {
-        dht_local_t  *local = NULL;
+    dht_local_t *local = NULL;
 
-        local = frame->local;
+    local = frame->local;
 
-        if (op_ret == -1)
-                goto err;
+    if (op_ret == -1)
+        goto err;
 
-        STACK_WIND (frame, dht_create_cbk,
-                    local->cached_subvol, local->cached_subvol->fops->create,
-                    &local->loc, local->flags, local->mode, local->umask,
-                    local->fd, local->params);
+    STACK_WIND_COOKIE(frame, dht_create_cbk, local->cached_subvol,
+                      local->cached_subvol, local->cached_subvol->fops->create,
+                      &local->loc, local->flags, local->mode, local->umask,
+                      local->fd, local->params);
 
-        return 0;
+    return 0;
 
 err:
-        DHT_STACK_UNWIND (create, frame, -1, op_errno,
-                          NULL, NULL, NULL, NULL, NULL, NULL);
-        return 0;
+    DHT_STACK_UNWIND(create, frame, -1, op_errno, NULL, NULL, NULL, NULL, NULL,
+                     NULL);
+    return 0;
 }
 
 int
-switch_create (call_frame_t *frame, xlator_t *this,
-               loc_t *loc, int32_t flags, mode_t mode,
-               mode_t umask, fd_t *fd, dict_t *params)
+switch_create(call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t flags,
+              mode_t mode, mode_t umask, fd_t *fd, dict_t *params)
 {
-        dht_local_t *local = NULL;
-        dht_conf_t  *conf  = NULL;
-        xlator_t    *subvol = NULL;
-        xlator_t    *avail_subvol = NULL;
-        int          op_errno = -1;
-
-        VALIDATE_OR_GOTO (frame, err);
-        VALIDATE_OR_GOTO (this, err);
-        VALIDATE_OR_GOTO (loc, err);
-
-        conf  = this->private;
-
-        dht_get_du_info (frame, this, loc);
-
-        local = dht_local_init (frame, loc, fd, GF_FOP_CREATE);
-        if (!local) {
-                op_errno = ENOMEM;
-                goto err;
-        }
-
-        subvol = dht_subvol_get_hashed (this, loc);
-        if (!subvol) {
-                gf_msg_debug (this->name, 0,
-                              "no subvolume in layout for path=%s",
-                              loc->path);
-                op_errno = ENOENT;
-                goto err;
-        }
-
-        avail_subvol = get_switch_matching_subvol (loc->path, conf, subvol);
-        if (dht_is_subvol_filled (this, avail_subvol)) {
-                avail_subvol =
-                        dht_free_disk_available_subvol (this, avail_subvol,
-                                                        local);
-        }
-
-        if (subvol != avail_subvol) {
-                /* create a link file instead of actual file */
-                local->mode = mode;
-                local->flags = flags;
-                local->umask = umask;
-                local->cached_subvol = avail_subvol;
-                dht_linkfile_create (frame, switch_create_linkfile_create_cbk,
-                                     this, avail_subvol, subvol, loc);
-                return 0;
-        }
+    dht_local_t *local = NULL;
+    dht_conf_t *conf = NULL;
+    xlator_t *subvol = NULL;
+    xlator_t *avail_subvol = NULL;
+    int op_errno = -1;
+
+    VALIDATE_OR_GOTO(frame, err);
+    VALIDATE_OR_GOTO(this, err);
+    VALIDATE_OR_GOTO(loc, err);
+
+    conf = this->private;
+
+    dht_get_du_info(frame, this, loc);
+
+    local = dht_local_init(frame, loc, fd, GF_FOP_CREATE);
+    if (!local) {
+        op_errno = ENOMEM;
+        goto err;
+    }
+
+    subvol = dht_subvol_get_hashed(this, loc);
+    if (!subvol) {
+        gf_msg_debug(this->name, 0, "no subvolume in layout for path=%s",
+                     loc->path);
+        op_errno = ENOENT;
+        goto err;
+    }
+
+    avail_subvol = get_switch_matching_subvol(loc->path, conf, subvol);
+    if (dht_is_subvol_filled(this, avail_subvol)) {
+        avail_subvol = dht_free_disk_available_subvol(this, avail_subvol,
+                                                      local);
+    }
+
+    if (subvol != avail_subvol) {
+        /* create a link file instead of actual file */
+        local->mode = mode;
+        local->flags = flags;
+        local->umask = umask;
+        local->cached_subvol = avail_subvol;
+        dht_linkfile_create(frame, switch_create_linkfile_create_cbk, this,
+                            avail_subvol, subvol, loc);
+        return 0;
+    }
 
-        gf_msg_trace (this->name, 0,
-                      "creating %s on %s", loc->path, subvol->name);
+    gf_msg_trace(this->name, 0, "creating %s on %s", loc->path, subvol->name);
 
-        STACK_WIND (frame, dht_create_cbk,
-                    subvol, subvol->fops->create,
-                    loc, flags, mode, umask, fd, params);
+    STACK_WIND_COOKIE(frame, dht_create_cbk, subvol, subvol,
+                      subvol->fops->create, loc, flags, mode, umask, fd,
+                      params);
 
-        return 0;
+    return 0;
 
 err:
-        op_errno = (op_errno == -1) ? errno : op_errno;
-        DHT_STACK_UNWIND (create, frame, -1, op_errno,
-                          NULL, NULL, NULL, NULL, NULL, NULL);
+    op_errno = (op_errno == -1) ? errno : op_errno;
+    DHT_STACK_UNWIND(create, frame, -1, op_errno, NULL, NULL, NULL, NULL, NULL,
+                     NULL);
 
-        return 0;
+    return 0;
 }
 
 int
-switch_mknod_linkfile_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                           int op_ret, int op_errno, inode_t *inode,
-                           struct iatt *stbuf, struct iatt *preparent,
-                           struct iatt *postparent, dict_t *xdata)
+switch_mknod_linkfile_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                          int op_ret, int op_errno, inode_t *inode,
+                          struct iatt *stbuf, struct iatt *preparent,
+                          struct iatt *postparent, dict_t *xdata)
 {
-        dht_local_t  *local = NULL;
+    dht_local_t *local = NULL;
 
-        local = frame->local;
-        if (!local || !local->cached_subvol) {
-                op_errno = EINVAL;
-                op_ret = -1;
-                goto err;
-        }
+    local = frame->local;
+    if (!local || !local->cached_subvol) {
+        op_errno = EINVAL;
+        op_ret = -1;
+        goto err;
+    }
 
-        if (op_ret >= 0) {
-                STACK_WIND_COOKIE (frame, dht_newfile_cbk,
-                            (void *)local->cached_subvol, local->cached_subvol,
-                            local->cached_subvol->fops->mknod,
-                            &local->loc, local->mode, local->rdev,
-                            local->umask, local->params);
+    if (op_ret >= 0) {
+        STACK_WIND_COOKIE(
+            frame, dht_newfile_cbk, (void *)local->cached_subvol,
+            local->cached_subvol, local->cached_subvol->fops->mknod,
+            &local->loc, local->mode, local->rdev, local->umask, local->params);
 
-                return 0;
-        }
-err:
-        DHT_STACK_UNWIND (link, frame, op_ret, op_errno,
-                          inode, stbuf, preparent, postparent, xdata);
         return 0;
+    }
+err:
+    DHT_STACK_UNWIND(link, frame, op_ret, op_errno, inode, stbuf, preparent,
+                     postparent, xdata);
+    return 0;
 }
 
-
 int
-switch_mknod (call_frame_t *frame, xlator_t *this, loc_t *loc, mode_t mode,
-              dev_t rdev, mode_t umask, dict_t *params)
+switch_mknod(call_frame_t *frame, xlator_t *this, loc_t *loc, mode_t mode,
+             dev_t rdev, mode_t umask, dict_t *params)
 {
-        dht_local_t *local = NULL;
-        dht_conf_t  *conf  = NULL;
-        xlator_t    *subvol = NULL;
-        xlator_t    *avail_subvol = NULL;
-        int          op_errno = -1;
-
-        VALIDATE_OR_GOTO (frame, err);
-        VALIDATE_OR_GOTO (this, err);
-        VALIDATE_OR_GOTO (loc, err);
-
-        conf  = this->private;
-
-        dht_get_du_info (frame, this, loc);
-
-        local = dht_local_init (frame, loc, NULL, GF_FOP_MKNOD);
-        if (!local) {
-                op_errno = ENOMEM;
-                goto err;
-        }
-
-        subvol = dht_subvol_get_hashed (this, loc);
-        if (!subvol) {
-                gf_msg_debug (this->name, 0,
-                              "no subvolume in layout for path=%s",
-                              loc->path);
-                op_errno = ENOENT;
-                goto err;
-        }
-
-        /* Consider the disksize in consideration */
-        avail_subvol = get_switch_matching_subvol (loc->path, conf, subvol);
-        if (dht_is_subvol_filled (this, avail_subvol)) {
-                avail_subvol =
-                        dht_free_disk_available_subvol (this, avail_subvol,
-                                                        local);
-        }
-
-        if (avail_subvol != subvol) {
-                /* Create linkfile first */
-
-                local->params = dict_ref (params);
-                local->mode = mode;
-                local->umask = umask;
-                local->rdev = rdev;
-                local->cached_subvol = avail_subvol;
-
-                dht_linkfile_create (frame, switch_mknod_linkfile_cbk,
-                                     this, avail_subvol, subvol, loc);
-                return 0;
-        }
+    dht_local_t *local = NULL;
+    dht_conf_t *conf = NULL;
+    xlator_t *subvol = NULL;
+    xlator_t *avail_subvol = NULL;
+    int op_errno = -1;
+
+    VALIDATE_OR_GOTO(frame, err);
+    VALIDATE_OR_GOTO(this, err);
+    VALIDATE_OR_GOTO(loc, err);
+
+    conf = this->private;
+
+    dht_get_du_info(frame, this, loc);
+
+    local = dht_local_init(frame, loc, NULL, GF_FOP_MKNOD);
+    if (!local) {
+        op_errno = ENOMEM;
+        goto err;
+    }
+
+    subvol = dht_subvol_get_hashed(this, loc);
+    if (!subvol) {
+        gf_msg_debug(this->name, 0, "no subvolume in layout for path=%s",
+                     loc->path);
+        op_errno = ENOENT;
+        goto err;
+    }
+
+    /* Consider the disksize in consideration */
+    avail_subvol = get_switch_matching_subvol(loc->path, conf, subvol);
+    if (dht_is_subvol_filled(this, avail_subvol)) {
+        avail_subvol = dht_free_disk_available_subvol(this, avail_subvol,
+                                                      local);
+    }
+
+    if (avail_subvol != subvol) {
+        /* Create linkfile first */
+
+        local->params = dict_ref(params);
+        local->mode = mode;
+        local->umask = umask;
+        local->rdev = rdev;
+        local->cached_subvol = avail_subvol;
+
+        dht_linkfile_create(frame, switch_mknod_linkfile_cbk, this,
+                            avail_subvol, subvol, loc);
+        return 0;
+    }
 
-        gf_msg_trace (this->name, 0,
-                      "creating %s on %s", loc->path, subvol->name);
+    gf_msg_trace(this->name, 0, "creating %s on %s", loc->path, subvol->name);
 
-        STACK_WIND_COOKIE (frame, dht_newfile_cbk, (void *)subvol, subvol,
-                           subvol->fops->mknod, loc, mode, rdev, umask,
-                           params);
+    STACK_WIND_COOKIE(frame, dht_newfile_cbk, (void *)subvol, subvol,
+                      subvol->fops->mknod, loc, mode, rdev, umask, params);
 
-        return 0;
+    return 0;
 
 err:
-        op_errno = (op_errno == -1) ? errno : op_errno;
-        DHT_STACK_UNWIND (mknod, frame, -1, op_errno,
-                          NULL, NULL, NULL, NULL, NULL);
+    op_errno = (op_errno == -1) ? errno : op_errno;
+    DHT_STACK_UNWIND(mknod, frame, -1, op_errno, NULL, NULL, NULL, NULL, NULL);
 
-        return 0;
+    return 0;
 }
 
-
 void
-switch_fini (xlator_t *this)
+switch_fini(xlator_t *this)
 {
-        dht_conf_t           *conf = NULL;
-        struct switch_struct *trav = NULL;
-        struct switch_struct *prev = NULL;
-
-        conf = this->private;
-
-        if (conf) {
-                trav = (struct switch_struct *)conf->private;
-                conf->private = NULL;
-                while (trav) {
-                        GF_FREE (trav->array);
-                        prev = trav;
-                        trav = trav->next;
-                        GF_FREE (prev);
-                }
+    dht_conf_t *conf = NULL;
+    struct switch_struct *trav = NULL;
+    struct switch_struct *prev = NULL;
+
+    conf = this->private;
+
+    if (conf) {
+        trav = (struct switch_struct *)conf->private;
+        conf->private = NULL;
+        while (trav) {
+            GF_FREE(trav->array);
+            prev = trav;
+            trav = trav->next;
+            GF_FREE(prev);
         }
+    }
 
-        dht_fini(this);
+    dht_fini(this);
 }
 
 int
-set_switch_pattern (xlator_t *this, dht_conf_t *conf,
-                    const char *pattern_str)
+set_switch_pattern(xlator_t *this, dht_conf_t *conf, const char *pattern_str)
 {
-        int                         flag = 0;
-        int                         idx = 0;
-        int                         index = 0;
-        int                         child_count = 0;
-        char                       *tmp = NULL;
-        char                       *tmp1 = NULL;
-        char                       *child = NULL;
-        char                       *tmp_str = NULL;
-        char                       *tmp_str1 = NULL;
-        char                       *dup_str = NULL;
-        char                       *dup_childs = NULL;
-        char                       *switch_str = NULL;
-        char                       *pattern = NULL;
-        char                       *childs = NULL;
-        char                       *option_string = NULL;
-        struct switch_struct        *switch_buf = NULL;
-        struct switch_struct        *switch_opt = NULL;
-        struct switch_struct        *trav = NULL;
-        struct switch_sched_array  *switch_buf_array = NULL;
-        xlator_list_t              *trav_xl = NULL;
-
-        trav_xl = this->children;
-        while (trav_xl) {
-                index++;
-                trav_xl = trav_xl->next;
+    int flag = 0;
+    int idx = 0;
+    int index = 0;
+    int child_count = 0;
+    char *tmp = NULL;
+    char *tmp1 = NULL;
+    char *child = NULL;
+    char *tmp_str = NULL;
+    char *tmp_str1 = NULL;
+    char *dup_str = NULL;
+    char *dup_childs = NULL;
+    char *switch_str = NULL;
+    char *pattern = NULL;
+    char *childs = NULL;
+    char *option_string = NULL;
+    size_t pattern_length;
+    struct switch_struct *switch_buf = NULL;
+    struct switch_struct *switch_opt = NULL;
+    struct switch_struct *trav = NULL;
+    struct switch_sched_array *switch_buf_array = NULL;
+    xlator_list_t *trav_xl = NULL;
+
+    trav_xl = this->children;
+    while (trav_xl) {
+        index++;
+        trav_xl = trav_xl->next;
+    }
+    child_count = index;
+    switch_buf_array = GF_CALLOC((index + 1), sizeof(struct switch_sched_array),
+                                 gf_switch_mt_switch_sched_array);
+    if (!switch_buf_array)
+        goto err;
+
+    trav_xl = this->children;
+    index = 0;
+
+    while (trav_xl) {
+        switch_buf_array[index].xl = trav_xl->xlator;
+        switch_buf_array[index].eligible = 1;
+        trav_xl = trav_xl->next;
+        index++;
+    }
+
+    /*  *jpg:child1,child2;*mpg:child3;*:child4,child5,child6 */
+
+    /* Get the pattern for considering switch case.
+       "option block-size *avi:10MB" etc */
+    option_string = gf_strdup(pattern_str);
+    if (option_string == NULL) {
+        goto err;
+    }
+    switch_str = strtok_r(option_string, ";", &tmp_str);
+    while (switch_str) {
+        dup_str = gf_strdup(switch_str);
+        if (dup_str == NULL) {
+            goto err;
         }
-        child_count = index;
-        switch_buf_array = GF_CALLOC ((index + 1),
-                                      sizeof (struct switch_sched_array),
-                                      gf_switch_mt_switch_sched_array);
-        if (!switch_buf_array)
-                goto err;
-
-        trav_xl = this->children;
-        index = 0;
-
-        while (trav_xl) {
-                switch_buf_array[index].xl = trav_xl->xlator;
-                switch_buf_array[index].eligible = 1;
-                trav_xl = trav_xl->next;
-                index++;
+        switch_opt = GF_CALLOC(1, sizeof(struct switch_struct),
+                               gf_switch_mt_switch_struct);
+        if (!switch_opt) {
+            GF_FREE(dup_str);
+            goto err;
         }
 
-        /*  *jpg:child1,child2;*mpg:child3;*:child4,child5,child6 */
-
-        /* Get the pattern for considering switch case.
-           "option block-size *avi:10MB" etc */
-        option_string = gf_strdup (pattern_str);
-        switch_str = strtok_r (option_string, ";", &tmp_str);
-        while (switch_str) {
-                dup_str = gf_strdup (switch_str);
-                switch_opt = GF_CALLOC (1, sizeof (struct switch_struct),
-                                        gf_switch_mt_switch_struct);
-                if (!switch_opt) {
-                        GF_FREE (dup_str);
-                        goto err;
-                }
+        pattern = strtok_r(dup_str, ":", &tmp_str1);
+        childs = strtok_r(NULL, ":", &tmp_str1);
+        if (strncmp(pattern, "*", 2) == 0) {
+            gf_msg("switch", GF_LOG_INFO, 0, DHT_MSG_SWITCH_PATTERN_INFO,
+                   "'*' pattern will be taken by default "
+                   "for all the unconfigured child nodes,"
+                   " hence neglecting current option");
+            switch_str = strtok_r(NULL, ";", &tmp_str);
+            GF_FREE(switch_opt);
+            switch_opt = NULL;
+            GF_FREE(dup_str);
+            continue;
+        }
+        GF_FREE(dup_str);
+
+        pattern_length = strlen(pattern);
+        if (pattern_length >= (sizeof(switch_opt->path_pattern))) {
+            gf_msg(this->name, GF_LOG_ERROR, 0,
+                   DHT_MSG_SET_SWITCH_PATTERN_ERROR, "Pattern (%s) too long",
+                   pattern);
+            goto err;
+        }
+        memcpy(switch_opt->path_pattern, pattern, pattern_length);
+        switch_opt->path_pattern[pattern_length] = '\0';
 
-                pattern = strtok_r (dup_str, ":", &tmp_str1);
-                childs = strtok_r (NULL, ":", &tmp_str1);
-                if (strncmp (pattern, "*", 2) == 0) {
-                        gf_log ("switch", GF_LOG_INFO,
-                                "'*' pattern will be taken by default "
-                                "for all the unconfigured child nodes,"
-                                " hence neglecting current option");
-                        switch_str = strtok_r (NULL, ";", &tmp_str);
-                        GF_FREE (switch_opt);
-                        GF_FREE (dup_str);
-                        continue;
-                }
-                GF_FREE (dup_str);
-                memcpy (switch_opt->path_pattern, pattern, strlen (pattern));
-                if (childs) {
-                        dup_childs = gf_strdup (childs);
-                        child = strtok_r (dup_childs, ",", &tmp);
-                        while (child) {
-                                if (gf_switch_valid_child (this, child)) {
-                                        idx++;
-                                        child = strtok_r (NULL, ",", &tmp);
-                                } else {
-                                        gf_log (this->name, GF_LOG_ERROR,
-                                                "%s is not a subvolume of %s. "
-                                                "pattern can only be scheduled "
-                                                "only to a subvolume of %s",
-                                                child, this->name, this->name);
-                                        goto err;
-                                }
-                        }
-                        GF_FREE (dup_childs);
-                        child = strtok_r (childs, ",", &tmp1);
-                        switch_opt->num_child = idx;
-                        switch_opt->array = GF_CALLOC (1, (idx *
-                                                           sizeof (struct switch_sched_array)),
-                                                       gf_switch_mt_switch_sched_array);
-                        if (!switch_opt->array)
-                                goto err;
-                        idx = 0;
-                        while (child) {
-                                for (index = 0; index < child_count; index++) {
-                                        if (strcmp (switch_buf_array[index].xl->name,
-                                                    child) == 0) {
-                                                gf_msg_debug ("switch", 0,
-                                                              "'%s' pattern will be "
-                                                              "scheduled to \"%s\"",
-                                                              switch_opt->path_pattern, child);
-                                                /*
-                                                  if (switch_buf_array[index-1].considered) {
-                                                  gf_msg_debug ("switch", 0,
-                                                  "ambiguity found, exiting");
-                                                  return -1;
-                                                  }
-                                                */
-                                                switch_opt->array[idx].xl = switch_buf_array[index].xl;
-                                                switch_buf_array[index].considered = 1;
-                                                idx++;
-                                                break;
-                                        }
-                                }
-                                child = strtok_r (NULL, ",", &tmp1);
-                        }
+        if (childs) {
+            dup_childs = gf_strdup(childs);
+            if (dup_childs == NULL) {
+                goto err;
+            }
+            child = strtok_r(dup_childs, ",", &tmp);
+            while (child) {
+                if (gf_switch_valid_child(this, child)) {
+                    idx++;
+                    child = strtok_r(NULL, ",", &tmp);
                 } else {
-                        /* error */
-                        gf_log ("switch", GF_LOG_ERROR,
-                                "Check \"scheduler.switch.case\" "
-                                "option in unify volume. Exiting");
-                        goto err;
+                    gf_msg(this->name, GF_LOG_ERROR, 0, DHT_MSG_SUBVOL_ERROR,
+                           "%s is not a subvolume of %s. "
+                           "pattern can only be scheduled "
+                           "only to a subvolume of %s",
+                           child, this->name, this->name);
+                    GF_FREE(dup_childs);
+                    goto err;
                 }
-
-                /* Link it to the main structure */
-                if (switch_buf) {
-                        /* there are already few entries */
-                        trav = switch_buf;
-                        while (trav->next)
-                                trav = trav->next;
-                        trav->next = switch_opt;
-                } else {
-                        /* First entry */
-                        switch_buf = switch_opt;
+            }
+            GF_FREE(dup_childs);
+            child = strtok_r(childs, ",", &tmp1);
+            switch_opt->num_child = idx;
+            switch_opt->array = GF_CALLOC(
+                1, (idx * sizeof(struct switch_sched_array)),
+                gf_switch_mt_switch_sched_array);
+            if (!switch_opt->array)
+                goto err;
+            idx = 0;
+            while (child) {
+                for (index = 0; index < child_count; index++) {
+                    if (strcmp(switch_buf_array[index].xl->name, child) == 0) {
+                        gf_msg_debug("switch", 0,
+                                     "'%s' pattern will be "
+                                     "scheduled to \"%s\"",
+                                     switch_opt->path_pattern, child);
+                        /*
+                          if (switch_buf_array[index-1].considered) {
+                          gf_msg_debug ("switch", 0,
+                          "ambiguity found, exiting");
+                          return -1;
+                          }
+                        */
+                        switch_opt->array[idx].xl = switch_buf_array[index].xl;
+                        switch_buf_array[index].considered = 1;
+                        idx++;
+                        break;
+                    }
                 }
-                switch_opt = NULL;
-                switch_str = strtok_r (NULL, ";", &tmp_str);
+                child = strtok_r(NULL, ",", &tmp1);
+            }
+        } else {
+            /* error */
+            gf_msg("switch", GF_LOG_ERROR, 0, DHT_MSG_SET_SWITCH_PATTERN_ERROR,
+                   "Check \"scheduler.switch.case\" "
+                   "option in unify volume. Exiting");
+            goto err;
         }
 
-        /* Now, all the pattern based considerations done, so for all the
-         * remaining pattern, '*' to all the remaining child nodes
-         */
-        {
-                for (index=0; index < child_count; index++) {
-                        /* check for considered flag */
-                        if (switch_buf_array[index].considered)
-                                continue;
-                        flag++;
-                }
-                if (!flag) {
-                        gf_log ("switch", GF_LOG_ERROR,
-                                "No nodes left for pattern '*'. Exiting");
-                        goto err;
-                }
-                switch_opt = GF_CALLOC (1, sizeof (struct switch_struct),
-                                        gf_switch_mt_switch_struct);
-                if (!switch_opt)
-                        goto err;
-
-                /* Add the '*' pattern to the array */
-                memcpy (switch_opt->path_pattern, "*", 2);
-                switch_opt->num_child = flag;
-                switch_opt->array =
-                        GF_CALLOC (1,
-                                   flag * sizeof (struct switch_sched_array),
-                                   gf_switch_mt_switch_sched_array);
-                if (!switch_opt->array)
-                        goto err;
-                flag = 0;
-                for (index=0; index < child_count; index++) {
-                        /* check for considered flag */
-                        if (switch_buf_array[index].considered)
-                                continue;
-                        gf_msg_debug ("switch", 0, "'%s'"
-                                      " pattern will be scheduled to \"%s\"",
-                                      switch_opt->path_pattern,
-                                      switch_buf_array[index].xl->name);
-
-                        switch_opt->array[flag].xl =
-                                switch_buf_array[index].xl;
-                        switch_buf_array[index].considered = 1;
-                        flag++;
-                }
-                if (switch_buf) {
-                        /* there are already few entries */
-                        trav = switch_buf;
-                        while (trav->next)
-                                trav = trav->next;
-                        trav->next = switch_opt;
-                } else {
-                        /* First entry */
-                        switch_buf = switch_opt;
-                }
-                switch_opt = NULL;
+        /* Link it to the main structure */
+        if (switch_buf) {
+            /* there are already few entries */
+            trav = switch_buf;
+            while (trav->next)
+                trav = trav->next;
+            trav->next = switch_opt;
+        } else {
+            /* First entry */
+            switch_buf = switch_opt;
+        }
+        switch_opt = NULL;
+        switch_str = strtok_r(NULL, ";", &tmp_str);
+    }
+
+    /* Now, all the pattern based considerations done, so for all the
+     * remaining pattern, '*' to all the remaining child nodes
+     */
+    {
+        for (index = 0; index < child_count; index++) {
+            /* check for considered flag */
+            if (switch_buf_array[index].considered)
+                continue;
+            flag++;
+        }
+        if (!flag) {
+            gf_msg("switch", GF_LOG_ERROR, 0, DHT_MSG_SET_SWITCH_PATTERN_ERROR,
+                   "No nodes left for pattern '*'. Exiting");
+            goto err;
+        }
+        switch_opt = GF_CALLOC(1, sizeof(struct switch_struct),
+                               gf_switch_mt_switch_struct);
+        if (!switch_opt)
+            goto err;
+
+        /* Add the '*' pattern to the array */
+        memcpy(switch_opt->path_pattern, "*", 2);
+        switch_opt->num_child = flag;
+        switch_opt->array = GF_CALLOC(1,
+                                      flag * sizeof(struct switch_sched_array),
+                                      gf_switch_mt_switch_sched_array);
+        if (!switch_opt->array)
+            goto err;
+        flag = 0;
+        for (index = 0; index < child_count; index++) {
+            /* check for considered flag */
+            if (switch_buf_array[index].considered)
+                continue;
+            gf_msg_debug("switch", 0,
+                         "'%s'"
+                         " pattern will be scheduled to \"%s\"",
+                         switch_opt->path_pattern,
+                         switch_buf_array[index].xl->name);
+
+            switch_opt->array[flag].xl = switch_buf_array[index].xl;
+            switch_buf_array[index].considered = 1;
+            flag++;
+        }
+        if (switch_buf) {
+            /* there are already few entries */
+            trav = switch_buf;
+            while (trav->next)
+                trav = trav->next;
+            trav->next = switch_opt;
+        } else {
+            /* First entry */
+            switch_buf = switch_opt;
         }
-        /* */
-        conf->private = switch_buf;
+        switch_opt = NULL;
+    }
+    /* */
+    conf->private = switch_buf;
 
-        return 0;
+    GF_FREE(option_string);
+    return 0;
 err:
-        GF_FREE (switch_buf_array);
-        GF_FREE (switch_opt);
+    GF_FREE(switch_buf_array);
+    GF_FREE(switch_opt);
+    GF_FREE(option_string);
 
-        if (switch_buf) {
-                trav = switch_buf;
-                while (trav) {
-                        GF_FREE (trav->array);
-                        switch_opt = trav;
-                        trav = trav->next;
-                        GF_FREE (switch_opt);
-                }
+    if (switch_buf) {
+        trav = switch_buf;
+        while (trav) {
+            GF_FREE(trav->array);
+            switch_opt = trav;
+            trav = trav->next;
+            GF_FREE(switch_opt);
         }
-        return -1;
+    }
+    return -1;
 }
 
-
 int32_t
-switch_init (xlator_t *this)
+switch_init(xlator_t *this)
 {
-        dht_conf_t            *conf = NULL;
-        data_t                *data = NULL;
-        int                    ret = -1;
+    dht_conf_t *conf = NULL;
+    data_t *data = NULL;
+    int ret = -1;
+
+    ret = dht_init(this);
+    if (ret) {
+        return ret;
+    }
+    conf = this->private;
 
-        ret = dht_init(this);
+    data = dict_get(this->options, "pattern.switch.case");
+    if (data) {
+        /* TODO: */
+        ret = set_switch_pattern(this, conf, data->data);
         if (ret) {
-                return ret;
-        }
-        conf = this->private;
-
-        data = dict_get (this->options, "pattern.switch.case");
-        if (data) {
-                /* TODO: */
-                ret = set_switch_pattern (this, conf, data->data);
-                if (ret) {
-                        goto err;
-                }
+            goto err;
         }
+    }
 
-        this->private = conf;
-        return 0;
+    this->private = conf;
+    return 0;
 
 err:
-        dht_fini(this);
-        return -1;
+    dht_fini(this);
+    return -1;
 }
 
-
-class_methods_t class_methods = {
-        .init           = switch_init,
-        .fini           = switch_fini,
-        .reconfigure    = dht_reconfigure,
-        .notify         = dht_notify
-};
-
-
 struct xlator_fops fops = {
-        .lookup      = switch_lookup,
-        .create      = switch_create,
-        .mknod       = switch_mknod,
-
-        .stat        = dht_stat,
-        .fstat       = dht_fstat,
-        .truncate    = dht_truncate,
-        .ftruncate   = dht_ftruncate,
-        .access      = dht_access,
-        .readlink    = dht_readlink,
-        .setxattr    = dht_setxattr,
-        .getxattr    = dht_getxattr,
-        .removexattr = dht_removexattr,
-        .open        = dht_open,
-        .readv       = dht_readv,
-        .writev      = dht_writev,
-        .flush       = dht_flush,
-        .fsync       = dht_fsync,
-        .statfs      = dht_statfs,
-        .lk          = dht_lk,
-        .opendir     = dht_opendir,
-        .readdir     = dht_readdir,
-        .readdirp    = dht_readdirp,
-        .fsyncdir    = dht_fsyncdir,
-        .symlink     = dht_symlink,
-        .unlink      = dht_unlink,
-        .link        = dht_link,
-        .mkdir       = dht_mkdir,
-        .rmdir       = dht_rmdir,
-        .rename      = dht_rename,
-        .inodelk     = dht_inodelk,
-        .finodelk    = dht_finodelk,
-        .entrylk     = dht_entrylk,
-        .fentrylk    = dht_fentrylk,
-        .xattrop     = dht_xattrop,
-        .fxattrop    = dht_fxattrop,
-        .setattr     = dht_setattr,
+    .lookup = switch_lookup,
+    .create = switch_create,
+    .mknod = switch_mknod,
+
+    .stat = dht_stat,
+    .fstat = dht_fstat,
+    .truncate = dht_truncate,
+    .ftruncate = dht_ftruncate,
+    .access = dht_access,
+    .readlink = dht_readlink,
+    .setxattr = dht_setxattr,
+    .getxattr = dht_getxattr,
+    .removexattr = dht_removexattr,
+    .open = dht_open,
+    .readv = dht_readv,
+    .writev = dht_writev,
+    .flush = dht_flush,
+    .fsync = dht_fsync,
+    .statfs = dht_statfs,
+    .lk = dht_lk,
+    .opendir = dht_opendir,
+    .readdir = dht_readdir,
+    .readdirp = dht_readdirp,
+    .fsyncdir = dht_fsyncdir,
+    .symlink = dht_symlink,
+    .unlink = dht_unlink,
+    .link = dht_link,
+    .mkdir = dht_mkdir,
+    .rmdir = dht_rmdir,
+    .rename = dht_rename,
+    .inodelk = dht_inodelk,
+    .finodelk = dht_finodelk,
+    .entrylk = dht_entrylk,
+    .fentrylk = dht_fentrylk,
+    .xattrop = dht_xattrop,
+    .fxattrop = dht_fxattrop,
+    .setattr = dht_setattr,
 };
 
-
-struct xlator_cbks cbks = {
-        .forget     = dht_forget
+struct xlator_cbks cbks = {.forget = dht_forget};
+extern int32_t
+mem_acct_init(xlator_t *this);
+
+xlator_api_t xlator_api = {
+    .init = switch_init,
+    .fini = switch_fini,
+    .notify = dht_notify,
+    .reconfigure = dht_reconfigure,
+    .mem_acct_init = mem_acct_init,
+    .op_version = {1}, /* Present from the initial version */
+    .fops = &fops,
+    .cbks = &cbks,
+    .options = dht_options,
+    .identifier = "switch",
+    .category = GF_TECH_PREVIEW,
 };
diff --git a/xlators/cluster/dht/src/tier.c b/xlators/cluster/dht/src/tier.c
deleted file mode 100644
index cf481c5e7f2..00000000000
--- a/xlators/cluster/dht/src/tier.c
+++ /dev/null
@@ -1,1198 +0,0 @@
-/*
-  Copyright (c) 2015 Red Hat, Inc. <http://www.redhat.com>
-  This file is part of GlusterFS.
-
-  This file is licensed to you under your choice of the GNU Lesser
-  General Public License, version 3 or any later version (LGPLv3 or
-  later), or the GNU General Public License, version 2 (GPLv2), in all
-  cases as published by the Free Software Foundation.
-*/
-
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
-#include <dlfcn.h>
-
-#include "dht-common.h"
-#include "tier.h"
-
-/*Hard coded DB info*/
-static gfdb_db_type_t dht_tier_db_type = GFDB_SQLITE3;
-/*Hard coded DB info*/
-
-/*Mutex for updating the data movement stats*/
-static pthread_mutex_t dm_stat_mutex = PTHREAD_MUTEX_INITIALIZER;
-
-static char *promotion_qfile;
-static char *demotion_qfile;
-
-static void *libhandle;
-static gfdb_methods_t gfdb_methods;
-
-#define DB_QUERY_RECORD_SIZE 4096
-
-
-
-static int
-tier_parse_query_str (char *query_record_str,
-                      char *gfid, char *link_buffer, ssize_t *link_size)
-{
-        char *token_str = NULL;
-        char *delimiter = "|";
-        char *saveptr = NULL;
-        int ret = -1;
-
-        GF_VALIDATE_OR_GOTO ("tier", query_record_str, out);
-        GF_VALIDATE_OR_GOTO ("tier", gfid, out);
-        GF_VALIDATE_OR_GOTO ("tier", link_buffer, out);
-        GF_VALIDATE_OR_GOTO ("tier", link_size, out);
-
-        token_str = strtok_r (query_record_str, delimiter, &saveptr);
-        if (!token_str)
-                goto out;
-
-        strcpy (gfid, token_str);
-
-
-        token_str = strtok_r (NULL, delimiter, &saveptr);
-        if (!token_str)
-                goto out;
-
-        strcpy (link_buffer, token_str);
-
-        token_str = strtok_r (NULL, delimiter, &saveptr);
-        if (!token_str)
-                goto out;
-
-        *link_size = atoi (token_str);
-
-        ret = 0;
-out:
-        return ret;
-}
-
-static int
-tier_check_same_node (xlator_t *this, loc_t *loc, gf_defrag_info_t *defrag)
-{
-        int     ret            = -1;
-        dict_t *dict           = NULL;
-        char   *uuid_str       = NULL;
-        uuid_t  node_uuid      = {0,};
-
-        GF_VALIDATE_OR_GOTO ("tier", this, out);
-        GF_VALIDATE_OR_GOTO (this->name, loc, out);
-        GF_VALIDATE_OR_GOTO (this->name, defrag, out);
-
-        if (syncop_getxattr (this, loc, &dict, GF_XATTR_NODE_UUID_KEY,
-                             NULL, NULL)) {
-                gf_msg (this->name, GF_LOG_ERROR, 0, DHT_MSG_LOG_TIER_ERROR,
-                        "Unable to get NODE_UUID_KEY %s %s\n",
-                        loc->name, loc->path);
-                goto out;
-        }
-
-        if (dict_get_str (dict, GF_XATTR_NODE_UUID_KEY, &uuid_str) < 0) {
-                gf_msg (this->name, GF_LOG_ERROR, 0, DHT_MSG_LOG_TIER_ERROR,
-                        "Failed to get node-uuid for %s", loc->path);
-                goto out;
-        }
-
-        if (gf_uuid_parse (uuid_str, node_uuid)) {
-                gf_msg (this->name, GF_LOG_INFO, 0, DHT_MSG_LOG_TIER_ERROR,
-                        "uuid_parse failed for %s", loc->path);
-                goto out;
-        }
-
-        if (gf_uuid_compare (node_uuid, defrag->node_uuid)) {
-                gf_msg (this->name, GF_LOG_INFO, 0, DHT_MSG_LOG_TIER_STATUS,
-                        "%s does not belong to this node", loc->path);
-                goto out;
-        }
-
-        ret = 0;
-out:
-        if (dict)
-                dict_unref(dict);
-
-        return ret;
-}
-
-static int
-tier_migrate_using_query_file (void *_args)
-{
-        int ret                                 = -1;
-        char gfid_str[UUID_CANONICAL_FORM_LEN+1] = "";
-        char query_record_str[4096]             = "";
-        query_cbk_args_t *query_cbk_args       = (query_cbk_args_t *) _args;
-        xlator_t *this                          = NULL;
-        gf_defrag_info_t *defrag                = NULL;
-        char *token_str                         = NULL;
-        char *delimiter                         = "::";
-        char *link_buffer                       = NULL;
-        gfdb_query_record_t *query_record       = NULL;
-        gfdb_link_info_t *link_info             = NULL;
-        struct iatt par_stbuf                   = {0,};
-        struct iatt current                     = {0,};
-        loc_t p_loc                             = {0,};
-        loc_t loc                               = {0,};
-        dict_t *migrate_data                    = NULL;
-        inode_t *linked_inode                   = NULL;
-        int per_file_status                     = 0;
-        int per_link_status                     = 0;
-        int total_status                        = 0;
-        FILE *queryFILE                         = NULL;
-        char *link_str                          = NULL;
-        xlator_t *src_subvol                    = NULL;
-        dht_conf_t   *conf                      = NULL;
-
-        GF_VALIDATE_OR_GOTO ("tier", query_cbk_args, out);
-        GF_VALIDATE_OR_GOTO ("tier", query_cbk_args->this, out);
-        this = query_cbk_args->this;
-        GF_VALIDATE_OR_GOTO (this->name, query_cbk_args->defrag, out);
-        GF_VALIDATE_OR_GOTO (this->name, query_cbk_args->queryFILE, out);
-        GF_VALIDATE_OR_GOTO (this->name, this->private, out);
-
-        conf = this->private;
-
-        defrag = query_cbk_args->defrag;
-
-        queryFILE = query_cbk_args->queryFILE;
-
-        query_record = gfdb_query_record_init();
-        if (!query_record) {
-                goto out;
-        }
-
-        query_record->_link_info_str = GF_CALLOC (1, DB_QUERY_RECORD_SIZE,
-                                                  gf_common_mt_char);
-        if (!query_record->_link_info_str) {
-                goto out;
-        }
-        link_buffer = query_record->_link_info_str;
-
-        link_info = gfdb_link_info_init ();
-
-        migrate_data = dict_new ();
-        if (!migrate_data)
-                goto out;
-
-        /* Per file */
-        while (fscanf (queryFILE, "%s", query_record_str) != EOF) {
-
-                per_file_status      = 0;
-                per_link_status      = 0;
-
-                memset (gfid_str, 0, UUID_CANONICAL_FORM_LEN+1);
-                memset (query_record->_link_info_str, 0, DB_QUERY_RECORD_SIZE);
-
-                if (tier_parse_query_str (query_record_str, gfid_str,
-                                          link_buffer,
-                                          &query_record->link_info_size)) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                DHT_MSG_LOG_TIER_ERROR,
-                                "failed parsing %s\n", query_record_str);
-                        continue;
-                }
-
-                gf_uuid_parse (gfid_str, query_record->gfid);
-
-                if (dict_get(migrate_data, GF_XATTR_FILE_MIGRATE_KEY))
-                        dict_del(migrate_data, GF_XATTR_FILE_MIGRATE_KEY);
-
-                if (dict_get(migrate_data, "from.migrator"))
-                        dict_del(migrate_data, "from.migrator");
-
-                token_str = strtok (link_buffer, delimiter);
-                if (token_str != NULL) {
-                        per_file_status =
-                                dict_set_str (migrate_data,
-                                              GF_XATTR_FILE_MIGRATE_KEY,
-                                              "force");
-                        if (per_file_status) {
-                                goto per_file_out;
-                        }
-
-                        /* Flag to suggest the xattr call is from migrator */
-                        per_file_status = dict_set_str (migrate_data,
-                                "from.migrator", "yes");
-                        if (per_file_status) {
-                                goto per_file_out;
-                        }
-                }
-                per_link_status = 0;
-                /* Per link of file */
-                while (token_str != NULL) {
-
-                        link_str = gf_strdup (token_str);
-
-                        if (!link_info) {
-                                per_link_status = -1;
-                                goto per_file_out;
-                        }
-
-                        memset (link_info, 0, sizeof(gfdb_link_info_t));
-
-                        ret = str_to_link_info (link_str, link_info);
-                        if (ret) {
-                                gf_msg (this->name, GF_LOG_ERROR, 0,
-                                        DHT_MSG_LOG_TIER_ERROR,
-                                        "failed parsing %s\n", link_str);
-                                per_link_status = -1;
-                                goto error;
-                        }
-
-                        gf_uuid_copy (p_loc.gfid, link_info->pargfid);
-
-                        p_loc.inode = inode_new (defrag->root_inode->table);
-                        if (!p_loc.inode) {
-                                gf_msg (this->name, GF_LOG_ERROR, 0,
-                                        DHT_MSG_LOG_TIER_ERROR,
-                                        "failed parsing %s\n", link_str);
-                                per_link_status = -1;
-                                goto error;
-                        }
-
-                        ret = syncop_lookup (this, &p_loc, &par_stbuf, NULL,
-                                             NULL, NULL);
-                        if (ret) {
-                                gf_msg (this->name, GF_LOG_ERROR, 0,
-                                        DHT_MSG_LOG_TIER_ERROR,
-                                        " ERROR in parent lookup\n");
-                                per_link_status = -1;
-                                goto error;
-                        }
-                        linked_inode = inode_link (p_loc.inode, NULL, NULL,
-                                                        &par_stbuf);
-                        inode_unref (p_loc.inode);
-                        p_loc.inode = linked_inode;
-
-                        gf_uuid_copy (loc.gfid, query_record->gfid);
-                        loc.inode = inode_new (defrag->root_inode->table);
-                        gf_uuid_copy (loc.pargfid, link_info->pargfid);
-                        loc.parent = inode_ref(p_loc.inode);
-
-                        loc.name = gf_strdup (link_info->file_name);
-                        if (!loc.name) {
-                                gf_msg (this->name, GF_LOG_ERROR, 0,
-                                        DHT_MSG_LOG_TIER_ERROR, "ERROR in "
-                                        "memory allocation\n");
-                                per_link_status = -1;
-                                goto error;
-                        }
-
-                        loc.path = gf_strdup (link_info->file_path);
-                        if (!loc.path) {
-                                gf_msg (this->name, GF_LOG_ERROR, 0,
-                                        DHT_MSG_LOG_TIER_ERROR, "ERROR in "
-                                        "memory allocation\n");
-                                per_link_status = -1;
-                                goto error;
-                        }
-
-                        gf_uuid_copy (loc.parent->gfid, link_info->pargfid);
-
-                        ret = syncop_lookup (this, &loc, &current, NULL,
-                                             NULL, NULL);
-                        if (ret) {
-                                gf_msg (this->name, GF_LOG_ERROR, 0,
-                                        DHT_MSG_LOG_TIER_ERROR, "ERROR in "
-                                        "current lookup\n");
-                                per_link_status = -1;
-                                goto error;
-                        }
-                        linked_inode = inode_link (loc.inode, NULL, NULL,
-                                                        &current);
-                        inode_unref (loc.inode);
-                        loc.inode = linked_inode;
-
-                        /*
-                         * Do not promote/demote if file already is where it
-                         * should be. This shall become a skipped count.
-                         */
-                        src_subvol = dht_subvol_get_cached(this, loc.inode);
-
-                        if (query_cbk_args->is_promotion &&
-                             src_subvol == conf->subvolumes[1]) {
-                                per_link_status = -1;
-                                goto error;
-                        }
-
-                        if (!query_cbk_args->is_promotion &&
-                            src_subvol == conf->subvolumes[0]) {
-                                per_link_status = -1;
-                                goto error;
-                        }
-
-                        gf_msg (this->name, GF_LOG_INFO, 0,
-                                DHT_MSG_LOG_TIER_STATUS, "Tier %d"
-                                " src_subvol %s file %s",
-                                query_cbk_args->is_promotion,
-                                src_subvol->name,
-                                loc.name);
-
-                        if (tier_check_same_node (this, &loc, defrag)) {
-                                per_link_status = -1;
-                                goto error;
-                        }
-
-                        ret = syncop_setxattr (this, &loc, migrate_data, 0,
-                                               NULL, NULL);
-                        if (ret) {
-                                gf_msg (this->name, GF_LOG_ERROR, 0,
-                                        DHT_MSG_LOG_TIER_ERROR, "ERROR %d in "
-                                        "current migration %s %s\n", ret,
-                                        loc.name,
-                                        loc.path);
-                                per_link_status = -1;
-                                goto error;
-                        }
-error:
-
-                        loc_wipe(&loc);
-                        loc_wipe(&p_loc);
-
-                        token_str = NULL;
-                        token_str = strtok (NULL, delimiter);
-                        GF_FREE (link_str);
-                }
-                per_file_status = per_link_status;
-per_file_out:
-                if (per_file_status) {
-                        pthread_mutex_lock (&dm_stat_mutex);
-                        defrag->total_failures++;
-                        pthread_mutex_unlock (&dm_stat_mutex);
-                } else {
-                        pthread_mutex_lock (&dm_stat_mutex);
-                        defrag->total_files++;
-                        pthread_mutex_unlock (&dm_stat_mutex);
-                }
-                total_status = total_status + per_file_status;
-                per_link_status = 0;
-                per_file_status = 0;
-                query_record_str[0] = '\0';
-        }
-
-out:
-        if (link_buffer)
-                GF_FREE (link_buffer);
-        gfdb_link_info_fini (&link_info);
-        if (migrate_data)
-                dict_unref (migrate_data);
-        gfdb_query_record_fini (&query_record);
-        return total_status;
-}
-
-
-/*This is the call back function per record/file from data base*/
-static int
-tier_gf_query_callback (gfdb_query_record_t *gfdb_query_record,
-                        void *_args) {
-        int ret = -1;
-        char gfid_str[UUID_CANONICAL_FORM_LEN] = "";
-        query_cbk_args_t *query_cbk_args = _args;
-
-        GF_VALIDATE_OR_GOTO ("tier", query_cbk_args, out);
-        GF_VALIDATE_OR_GOTO ("tier", query_cbk_args->defrag, out);
-        GF_VALIDATE_OR_GOTO ("tier", query_cbk_args->queryFILE, out);
-
-        gf_uuid_unparse (gfdb_query_record->gfid, gfid_str);
-        fprintf (query_cbk_args->queryFILE, "%s|%s|%ld\n", gfid_str,
-                 gfdb_query_record->_link_info_str,
-                 gfdb_query_record->link_info_size);
-
-        pthread_mutex_lock (&dm_stat_mutex);
-        query_cbk_args->defrag->num_files_lookedup++;
-        pthread_mutex_unlock (&dm_stat_mutex);
-
-        ret = 0;
-out:
-        return ret;
-}
-
-/*This is the call back function for each brick from hot/cold bricklist
- * It picks up each bricks db and queries for eligible files for migration.
- * The list of eligible files are populated in appropriate query files*/
-static int
-tier_process_brick_cbk (dict_t *brick_dict, char *key, data_t *value,
-                        void *args) {
-        int ret                                         = -1;
-        char *db_path                                   = NULL;
-        query_cbk_args_t *query_cbk_args              	= NULL;
-        xlator_t *this                                  = NULL;
-        gfdb_conn_node_t *conn_node                   	= NULL;
-        dict_t *params_dict                             = NULL;
-        _gfdb_brick_dict_info_t *gfdb_brick_dict_info   = args;
-
-        /*Init of all the essentials*/
-        GF_VALIDATE_OR_GOTO ("tier", gfdb_brick_dict_info , out);
-        query_cbk_args = gfdb_brick_dict_info->_query_cbk_args;
-
-        GF_VALIDATE_OR_GOTO ("tier", query_cbk_args->this, out);
-        this = query_cbk_args->this;
-
-        GF_VALIDATE_OR_GOTO (this->name,
-                             gfdb_brick_dict_info->_query_cbk_args, out);
-
-        GF_VALIDATE_OR_GOTO (this->name, value, out);
-        db_path = data_to_str(value);
-
-        /*Preparing DB parameters before init_db i.e getting db connection*/
-        params_dict = dict_new ();
-        if (!params_dict) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        DHT_MSG_LOG_TIER_ERROR,
-                        "DB Params cannot initialized!");
-                goto out;
-        }
-        SET_DB_PARAM_TO_DICT(this->name, params_dict, gfdb_methods.dbpath,
-                                db_path, ret, out);
-
-        /*Get the db connection*/
-        conn_node = gfdb_methods.init_db((void *)params_dict, dht_tier_db_type);
-        if (!conn_node) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        DHT_MSG_LOG_TIER_ERROR,
-                        "FATAL: Failed initializing db operations");
-                         goto out;
-        }
-
-        /*Query for eligible files from db*/
-        query_cbk_args->queryFILE = fopen(GET_QFILE_PATH
-                                (gfdb_brick_dict_info->_gfdb_promote), "a+");
-        if (!query_cbk_args->queryFILE) {
-                gf_msg (this->name, GF_LOG_ERROR, 0, DHT_MSG_LOG_TIER_ERROR,
-                                "Failed to open query file %s:%s",
-                                GET_QFILE_PATH
-                        (gfdb_brick_dict_info->_gfdb_promote),
-                        strerror(errno));
-                goto out;
-        }
-        if (!gfdb_brick_dict_info->_gfdb_promote) {
-                if (query_cbk_args->defrag->write_freq_threshold == 0 &&
-                        query_cbk_args->defrag->read_freq_threshold == 0) {
-                                ret = gfdb_methods.find_unchanged_for_time (
-                                        conn_node,
-                                        tier_gf_query_callback,
-                                        (void *)query_cbk_args,
-                                        gfdb_brick_dict_info->time_stamp);
-                } else {
-                                ret = gfdb_methods.find_unchanged_for_time_freq (
-                                        conn_node,
-                                        tier_gf_query_callback,
-                                        (void *)query_cbk_args,
-                                        gfdb_brick_dict_info->time_stamp,
-                                        query_cbk_args->defrag->
-                                                        write_freq_threshold,
-                                        query_cbk_args->defrag->
-                                                        read_freq_threshold,
-                                        _gf_false);
-                }
-        } else {
-                if (query_cbk_args->defrag->write_freq_threshold == 0 &&
-                        query_cbk_args->defrag->read_freq_threshold == 0) {
-                        ret = gfdb_methods.find_recently_changed_files (
-                                conn_node,
-                                tier_gf_query_callback,
-                                (void *)query_cbk_args,
-                                gfdb_brick_dict_info->time_stamp);
-                } else {
-                        ret = gfdb_methods.find_recently_changed_files_freq (
-                                conn_node,
-                                tier_gf_query_callback,
-                                (void *)query_cbk_args,
-                                gfdb_brick_dict_info->time_stamp,
-                                query_cbk_args->defrag->
-                                write_freq_threshold,
-                                query_cbk_args->defrag->read_freq_threshold,
-                                _gf_false);
-                }
-        }
-        if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                                DHT_MSG_LOG_TIER_ERROR,
-                                "FATAL: query from db failed");
-                        goto out;
-                }
-        ret = 0;
-out:
-        if (query_cbk_args->queryFILE) {
-                fclose (query_cbk_args->queryFILE);
-                query_cbk_args->queryFILE = NULL;
-        }
-        gfdb_methods.fini_db (conn_node);
-        return ret;
-}
-
-inline int
-tier_build_migration_qfile (demotion_args_t *args,
-                            query_cbk_args_t *query_cbk_args,
-                            gf_boolean_t is_promotion)
-{
-        gfdb_time_t                     current_time;
-        _gfdb_brick_dict_info_t         gfdb_brick_dict_info;
-        gfdb_time_t                     time_in_past;
-        int                             ret = -1;
-
-        /*
-         *  The first time this function is called, query file will
-         *  not exist on a given instance of running the migration daemon.
-         * The remove call is optimistic and it is legal if it fails.
-         */
-
-        ret = remove (GET_QFILE_PATH (is_promotion));
-        if (ret == -1) {
-                gf_msg (args->this->name, GF_LOG_INFO, 0,
-                        DHT_MSG_LOG_TIER_STATUS,
-                        "Failed to remove %s",
-                        GET_QFILE_PATH (is_promotion));
-        }
-
-        time_in_past.tv_sec = args->freq_time;
-        time_in_past.tv_usec = 0;
-
-        ret = gettimeofday (&current_time, NULL);
-        if (ret == -1) {
-                gf_log (args->this->name, GF_LOG_ERROR,
-                        "Failed to get current timen");
-                goto out;
-        }
-        time_in_past.tv_sec = current_time.tv_sec - time_in_past.tv_sec;
-        time_in_past.tv_usec = current_time.tv_usec - time_in_past.tv_usec;
-        gfdb_brick_dict_info.time_stamp = &time_in_past;
-        gfdb_brick_dict_info._gfdb_promote = is_promotion;
-        gfdb_brick_dict_info._query_cbk_args = query_cbk_args;
-        ret = dict_foreach (args->brick_list, tier_process_brick_cbk,
-                            &gfdb_brick_dict_info);
-        if (ret) {
-                gf_log (args->this->name, GF_LOG_ERROR,
-                        "Brick query failedn");
-                goto out;
-        }
-out:
-        return ret;
-}
-
-inline int
-tier_migrate_files_using_qfile (demotion_args_t *comp,
-                                query_cbk_args_t *query_cbk_args,
-                                char *qfile)
-{
-        char renamed_file[PATH_MAX] = "";
-        int ret = -1;
-
-        query_cbk_args->queryFILE = fopen (qfile, "r");
-        if (!query_cbk_args->queryFILE) {
-                gf_log ("tier", GF_LOG_ERROR,
-                        "Failed opening %s for migration", qfile);
-                goto out;
-        }
-        ret = tier_migrate_using_query_file ((void *)query_cbk_args);
-        fclose (query_cbk_args->queryFILE);
-        query_cbk_args->queryFILE = NULL;
-        if (ret) {
-                sprintf (renamed_file, "%s.err", qfile);
-                rename (qfile, renamed_file);
-        }
-out:
-        return ret;
-}
-
-/*Demotion Thread*/
-static void *
-tier_demote (void *args)
-{
-        int ret = -1;
-        query_cbk_args_t query_cbk_args;
-        demotion_args_t *demotion_args = args;
-
-        GF_VALIDATE_OR_GOTO ("tier", demotion_args, out);
-        GF_VALIDATE_OR_GOTO ("tier", demotion_args->this, out);
-        GF_VALIDATE_OR_GOTO (demotion_args->this->name,
-                             demotion_args->brick_list, out);
-        GF_VALIDATE_OR_GOTO (demotion_args->this->name,
-                             demotion_args->defrag, out);
-
-        query_cbk_args.this = demotion_args->this;
-        query_cbk_args.defrag = demotion_args->defrag;
-        query_cbk_args.is_promotion = 0;
-
-        /*Build the query file using bricklist*/
-        ret = tier_build_migration_qfile(demotion_args, &query_cbk_args,
-                                    _gf_false);
-        if (ret)
-                goto out;
-
-        /* Migrate files using the query file */
-        ret = tier_migrate_files_using_qfile (args,
-                                              &query_cbk_args, demotion_qfile);
-        if (ret)
-                goto out;
-
-out:
-        demotion_args->return_value = ret;
-        return NULL;
-}
-
-
-/*Promotion Thread*/
-static void
-*tier_promote (void *args)
-{
-        int ret = -1;
-        query_cbk_args_t query_cbk_args;
-        promotion_args_t *promotion_args = args;
-
-        GF_VALIDATE_OR_GOTO ("tier", promotion_args->this, out);
-        GF_VALIDATE_OR_GOTO (promotion_args->this->name,
-                             promotion_args->brick_list, out);
-        GF_VALIDATE_OR_GOTO (promotion_args->this->name,
-                             promotion_args->defrag, out);
-
-        query_cbk_args.this = promotion_args->this;
-        query_cbk_args.defrag = promotion_args->defrag;
-        query_cbk_args.is_promotion = 1;
-
-        /*Build the query file using bricklist*/
-        ret = tier_build_migration_qfile(promotion_args, &query_cbk_args,
-                                         _gf_true);
-        if (ret)
-                goto out;
-
-        /* Migrate files using the query file */
-        ret = tier_migrate_files_using_qfile (args,
-                                              &query_cbk_args,
-                                              promotion_qfile);
-        if (ret)
-                goto out;
-
-out:
-        promotion_args->return_value = ret;
-        return NULL;
-}
-
-static int
-tier_get_bricklist (xlator_t *xl, dict_t *bricklist)
-{
-        xlator_list_t  *child = NULL;
-        char           *rv        = NULL;
-        char           *rh        = NULL;
-        char           localhost[256] = {0};
-        char           *db_path = "";
-        char           *brickname = NULL;
-        char            db_name[PATH_MAX] = "";
-        int             ret = 0;
-
-        GF_VALIDATE_OR_GOTO ("tier", xl, out);
-        GF_VALIDATE_OR_GOTO ("tier", bricklist, out);
-
-        gethostname (localhost, sizeof (localhost));
-
-        /*
-         * This function obtains remote subvolumes and filters out only
-         * those running on the same node as the tier daemon.
-         */
-        if (strcmp(xl->type, "protocol/client") == 0) {
-                ret = dict_get_str(xl->options, "remote-host", &rh);
-                if (ret < 0)
-                        goto out;
-
-               if (gf_is_local_addr (rh)) {
-
-                        ret = dict_get_str(xl->options, "remote-subvolume",
-                                           &rv);
-                        if (ret < 0)
-                                goto out;
-                        brickname = strrchr(rv, '/') + 1;
-                        snprintf(db_name, sizeof(db_name), "%s.db",
-                                 brickname);
-                        db_path = GF_CALLOC (PATH_MAX, 1, gf_common_mt_char);
-                        if (!db_path) {
-                                gf_msg ("tier", GF_LOG_ERROR, 0,
-                                        DHT_MSG_LOG_TIER_STATUS,
-                                        "Failed to allocate memory for bricklist");
-                                goto out;
-                        }
-
-                        sprintf(db_path, "%s/%s/%s", rv,
-                                GF_HIDDEN_PATH,
-                                db_name);
-                        if (dict_add_dynstr_with_alloc(bricklist, "brick",
-                                                       db_path))
-                                goto out;
-
-                        ret = 0;
-                        goto out;
-                }
-        }
-
-        for (child = xl->children; child; child = child->next) {
-                ret = tier_get_bricklist(child->xlator, bricklist);
-        }
-out:
-        return ret;
-}
-
-int
-tier_start (xlator_t *this, gf_defrag_info_t *defrag)
-{
-        dict_t       *bricklist_cold = NULL;
-        dict_t       *bricklist_hot = NULL;
-        dht_conf_t   *conf     = NULL;
-        int tick = 0;
-        int next_demote = 0;
-        int next_promote = 0;
-        int freq_promote = 0;
-        int freq_demote = 0;
-        promotion_args_t promotion_args = { 0 };
-        demotion_args_t demotion_args = { 0 };
-        int ret_promotion = 0;
-        int ret_demotion = 0;
-        int ret = 0;
-        pthread_t promote_thread;
-        pthread_t demote_thread;
-
-        conf   = this->private;
-
-        bricklist_cold = dict_new();
-        if (!bricklist_cold)
-                return -1;
-
-        bricklist_hot = dict_new();
-        if (!bricklist_hot)
-                return -1;
-
-        tier_get_bricklist (conf->subvolumes[0], bricklist_cold);
-        tier_get_bricklist (conf->subvolumes[1], bricklist_hot);
-
-        freq_promote = defrag->tier_promote_frequency;
-        freq_demote  = defrag->tier_demote_frequency;
-
-        next_promote = defrag->tier_promote_frequency % TIMER_SECS;
-        next_demote  = defrag->tier_demote_frequency % TIMER_SECS;
-
-
-        gf_msg (this->name, GF_LOG_INFO, 0,
-                DHT_MSG_LOG_TIER_STATUS, "Begin run tier promote %d demote %d",
-                next_promote, next_demote);
-
-        defrag->defrag_status = GF_DEFRAG_STATUS_STARTED;
-
-        while (1) {
-
-                sleep(1);
-
-                ret_promotion = -1;
-                ret_demotion = -1;
-
-                if (defrag->defrag_status != GF_DEFRAG_STATUS_STARTED) {
-                        ret = 1;
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                DHT_MSG_LOG_TIER_ERROR,
-                                "defrag->defrag_status != "
-                                "GF_DEFRAG_STATUS_STARTED");
-                        goto out;
-                }
-
-                tick = (tick + 1) % TIMER_SECS;
-                if ((next_demote != tick) && (next_promote != tick))
-                        continue;
-
-                if (next_demote >= tick) {
-                        demotion_args.this = this;
-                        demotion_args.brick_list = bricklist_hot;
-                        demotion_args.defrag = defrag;
-                        demotion_args.freq_time = freq_demote;
-                        ret_demotion = pthread_create (&demote_thread, NULL,
-                                        &tier_demote, &demotion_args);
-                        if (ret_demotion) {
-                                gf_msg (this->name, GF_LOG_ERROR, 0,
-                                        DHT_MSG_LOG_TIER_ERROR,
-                                        "Failed starting Demotion thread!");
-                        }
-                        freq_demote = defrag->tier_demote_frequency;
-                        next_demote = (tick + freq_demote) % TIMER_SECS;
-                }
-
-                if (next_promote >= tick) {
-                        promotion_args.this = this;
-                        promotion_args.brick_list = bricklist_cold;
-                        promotion_args.defrag = defrag;
-                        promotion_args.freq_time = freq_promote;
-                        ret_promotion = pthread_create (&promote_thread, NULL,
-                                                &tier_promote, &promotion_args);
-                        if (ret_promotion) {
-                                gf_msg (this->name, GF_LOG_ERROR, 0,
-                                        DHT_MSG_LOG_TIER_ERROR,
-                                        "Failed starting Promotion thread!");
-                        }
-                        freq_promote = defrag->tier_promote_frequency;
-                        next_promote = (tick + freq_promote) % TIMER_SECS;
-                }
-
-                if (ret_demotion == 0) {
-                        pthread_join (demote_thread, NULL);
-                        if (demotion_args.return_value) {
-                                gf_msg (this->name, GF_LOG_ERROR, 0,
-                                        DHT_MSG_LOG_TIER_ERROR,
-                                        "Demotion failed!");
-                        }
-                        ret_demotion = demotion_args.return_value;
-                }
-
-                if (ret_promotion == 0) {
-                        pthread_join (promote_thread, NULL);
-                        if (promotion_args.return_value) {
-                                gf_msg (this->name, GF_LOG_ERROR, 0,
-                                        DHT_MSG_LOG_TIER_ERROR,
-                                        "Promotion failed!");
-                        }
-                        ret_promotion = promotion_args.return_value;
-                }
-
-                /*Collect previous and current cummulative status */
-                ret = ret | ret_demotion | ret_promotion;
-
-                /*reseting promotion and demotion arguments for next iteration*/
-                memset (&demotion_args, 0, sizeof(demotion_args_t));
-                memset (&promotion_args, 0, sizeof(promotion_args_t));
-
-        }
-
-        ret = 0;
-out:
-
-        dict_unref(bricklist_cold);
-        dict_unref(bricklist_hot);
-
-        return ret;
-}
-
-int32_t
-tier_migration_needed (xlator_t *this)
-{
-        gf_defrag_info_t        *defrag = NULL;
-        dht_conf_t              *conf   = NULL;
-        int                      ret = 0;
-
-        conf = this->private;
-
-        GF_VALIDATE_OR_GOTO (this->name, conf, out);
-        GF_VALIDATE_OR_GOTO (this->name, conf->defrag, out);
-
-        defrag = conf->defrag;
-
-        if (defrag->cmd == GF_DEFRAG_CMD_START_TIER)
-                ret = 1;
-out:
-        return ret;
-}
-
-int32_t
-tier_migration_get_dst (xlator_t *this, dht_local_t *local)
-{
-        dht_conf_t              *conf   = NULL;
-        int32_t                  ret = -1;
-
-        GF_VALIDATE_OR_GOTO("tier", this, out);
-        GF_VALIDATE_OR_GOTO(this->name, this->private, out);
-
-        conf = this->private;
-        if (!conf)
-                goto out;
-
-        if (conf->subvolumes[0] == local->cached_subvol)
-                local->rebalance.target_node =
-                        conf->subvolumes[1];
-        else
-                local->rebalance.target_node =
-                        conf->subvolumes[0];
-
-        if (local->rebalance.target_node)
-                ret = 0;
-
-out:
-        return ret;
-}
-
-xlator_t *
-tier_search (xlator_t *this, dht_layout_t *layout, const char *name)
-{
-        xlator_t  *subvol = NULL;
-        void      *value;
-        int        search_first_subvol = 0;
-
-        GF_VALIDATE_OR_GOTO("tier", this, out);
-        GF_VALIDATE_OR_GOTO(this->name, layout, out);
-        GF_VALIDATE_OR_GOTO(this->name, name, out);
-
-        if (!dict_get_ptr (this->options, "rule", &value) &&
-            !strcmp(layout->list[0].xlator->name, value)) {
-                search_first_subvol = 1;
-        }
-
-        if (search_first_subvol)
-                subvol = layout->list[0].xlator;
-        else
-                subvol = layout->list[1].xlator;
-
-out:
-        return subvol;
-}
-
-
-dht_methods_t tier_methods = {
-        .migration_get_dst_subvol = tier_migration_get_dst,
-        .migration_other = tier_start,
-        .migration_needed = tier_migration_needed,
-        .layout_search   = tier_search,
-};
-
-static int
-tier_load_externals (xlator_t *this)
-{
-        int               ret            = -1;
-        char *libpathfull = (LIBDIR "/libgfdb.so." LIBGFDB_VERSION);
-        get_gfdb_methods_t get_gfdb_methods;
-
-        GF_VALIDATE_OR_GOTO("this", this, out);
-
-        libhandle = dlopen (libpathfull, RTLD_NOW);
-        if (!libhandle) {
-                gf_msg(this->name, GF_LOG_ERROR, 0,
-                       DHT_MSG_LOG_TIER_ERROR,
-                       "Error loading libgfdb.so %s\n", dlerror());
-                ret = -1;
-                goto out;
-        }
-
-        get_gfdb_methods = dlsym (libhandle, "get_gfdb_methods");
-        if (!get_gfdb_methods) {
-                gf_msg(this->name, GF_LOG_ERROR, 0,
-                       DHT_MSG_LOG_TIER_ERROR,
-                       "Error loading get_gfdb_methods()");
-                ret = -1;
-                goto out;
-        }
-
-        get_gfdb_methods (&gfdb_methods);
-
-        ret = 0;
-
-out:
-        if (ret && libhandle)
-                dlclose (libhandle);
-
-        return ret;
-}
-
-int
-tier_init (xlator_t *this)
-{
-        int               ret            = -1;
-        int               freq           = 0;
-        dht_conf_t       *conf           = NULL;
-        gf_defrag_info_t *defrag         = NULL;
-
-        ret = dht_init(this);
-        if (ret) {
-                gf_msg(this->name, GF_LOG_ERROR, 0,
-                       DHT_MSG_LOG_TIER_ERROR,
-                       "dht_init failed");
-                goto out;
-        }
-
-        conf = this->private;
-
-        conf->methods = &tier_methods;
-
-        if (conf->subvolume_cnt != 2) {
-                gf_msg(this->name, GF_LOG_ERROR, 0,
-                       DHT_MSG_LOG_TIER_ERROR,
-                       "Invalid number of subvolumes %d", conf->subvolume_cnt);
-                goto out;
-        }
-
-        /* if instatiated from client side initialization is complete. */
-        if (!conf->defrag) {
-                ret = 0;
-                goto out;
-        }
-
-        /* if instatiated from server side, load db libraries */
-        ret = tier_load_externals(this);
-        if (ret) {
-                gf_msg(this->name, GF_LOG_ERROR, 0,
-                       DHT_MSG_LOG_TIER_ERROR,
-                       "Could not load externals. Aborting");
-                goto out;
-        }
-
-        defrag = conf->defrag;
-
-        ret = dict_get_int32 (this->options,
-                              "tier-promote-frequency", &freq);
-        if (ret) {
-                freq = DEFAULT_PROMOTE_FREQ_SEC;
-        }
-
-        defrag->tier_promote_frequency = freq;
-
-        ret = dict_get_int32 (this->options,
-                              "tier-demote-frequency", &freq);
-        if (ret) {
-                freq = DEFAULT_DEMOTE_FREQ_SEC;
-        }
-
-        defrag->tier_demote_frequency = freq;
-
-        ret = dict_get_int32 (this->options,
-                              "write-freq-threshold", &freq);
-        if (ret) {
-                freq = DEFAULT_WRITE_FREQ_SEC;
-        }
-
-        defrag->write_freq_threshold = freq;
-
-        ret = dict_get_int32 (this->options,
-                              "read-freq-threshold", &freq);
-        if (ret) {
-                freq = DEFAULT_READ_FREQ_SEC;
-        }
-
-        defrag->read_freq_threshold = freq;
-
-        ret = gf_asprintf(&promotion_qfile, "%s/%s-%d",
-                          DEFAULT_VAR_RUN_DIRECTORY,
-                          PROMOTION_QFILE,
-                          getpid());
-        if (ret < 0)
-                goto out;
-
-        ret = gf_asprintf(&demotion_qfile, "%s/%s-%d",
-                          DEFAULT_VAR_RUN_DIRECTORY,
-                          DEMOTION_QFILE,
-                          getpid());
-        if (ret < 0) {
-                GF_FREE(promotion_qfile);
-                goto out;
-        }
-
-        gf_msg(this->name, GF_LOG_INFO, 0,
-               DHT_MSG_LOG_TIER_STATUS,
-               "Promote/demote frequency %d/%d "
-               "Write/Read freq thresholds %d/%d",
-               defrag->tier_promote_frequency,
-               defrag->tier_demote_frequency,
-               defrag->write_freq_threshold,
-               defrag->read_freq_threshold);
-
-        gf_msg(this->name, GF_LOG_INFO, 0,
-               DHT_MSG_LOG_TIER_STATUS,
-               "Promote file %s demote file %s",
-               promotion_qfile, demotion_qfile);
-
-        ret = 0;
-
-out:
-
-        return ret;
-}
-
-
-int
-tier_reconfigure (xlator_t *this, dict_t *options)
-{
-        dht_conf_t       *conf           = NULL;
-        gf_defrag_info_t *defrag         = NULL;
-
-        conf = this->private;
-
-        if (conf->defrag) {
-                defrag = conf->defrag;
-                GF_OPTION_RECONF ("tier-promote-frequency",
-                                  defrag->tier_promote_frequency, options,
-                                  int32, out);
-
-                GF_OPTION_RECONF ("tier-demote-frequency",
-                                  defrag->tier_demote_frequency, options,
-                                  int32, out);
-
-                GF_OPTION_RECONF ("write-freq-threshold",
-                                  defrag->write_freq_threshold, options,
-                                  int32, out);
-
-                GF_OPTION_RECONF ("read-freq-threshold",
-                                  defrag->read_freq_threshold, options,
-                                  int32, out);
-        }
-
-out:
-        return dht_reconfigure (this, options);
-}
-
-void
-tier_fini (xlator_t *this)
-{
-        if (libhandle)
-                dlclose(libhandle);
-
-        GF_FREE(demotion_qfile);
-        GF_FREE(promotion_qfile);
-
-        dht_fini(this);
-}
-
-class_methods_t class_methods = {
-        .init           = tier_init,
-        .fini           = tier_fini,
-        .reconfigure    = tier_reconfigure,
-        .notify         = dht_notify
-};
-
-
-struct xlator_fops fops = {
-        .lookup      = dht_lookup,
-        .create      = dht_create,
-        .mknod       = dht_mknod,
-
-        .stat        = dht_stat,
-        .fstat       = dht_fstat,
-        .truncate    = dht_truncate,
-        .ftruncate   = dht_ftruncate,
-        .access      = dht_access,
-        .readlink    = dht_readlink,
-        .setxattr    = dht_setxattr,
-        .getxattr    = dht_getxattr,
-        .removexattr = dht_removexattr,
-        .open        = dht_open,
-        .readv       = dht_readv,
-        .writev      = dht_writev,
-        .flush       = dht_flush,
-        .fsync       = dht_fsync,
-        .statfs      = dht_statfs,
-        .lk          = dht_lk,
-        .opendir     = dht_opendir,
-        .readdir     = dht_readdir,
-        .readdirp    = dht_readdirp,
-        .fsyncdir    = dht_fsyncdir,
-        .symlink     = dht_symlink,
-        .unlink      = dht_unlink,
-        .link        = dht_link,
-        .mkdir       = dht_mkdir,
-        .rmdir       = dht_rmdir,
-        .rename      = dht_rename,
-        .inodelk     = dht_inodelk,
-        .finodelk    = dht_finodelk,
-        .entrylk     = dht_entrylk,
-        .fentrylk    = dht_fentrylk,
-        .xattrop     = dht_xattrop,
-        .fxattrop    = dht_fxattrop,
-        .setattr     = dht_setattr,
-};
-
-
-struct xlator_cbks cbks = {
-        .forget     = dht_forget
-};
-
diff --git a/xlators/cluster/dht/src/tier.h b/xlators/cluster/dht/src/tier.h
deleted file mode 100644
index 7d4ea6b7ebe..00000000000
--- a/xlators/cluster/dht/src/tier.h
+++ /dev/null
@@ -1,73 +0,0 @@
-/*
-  Copyright (c) 2015 Red Hat, Inc. <http://www.redhat.com>
-  This file is part of GlusterFS.
-
-  This file is licensed to you under your choice of the GNU Lesser
-  General Public License, version 3 or any later version (LGPLv3 or
-  later), or the GNU General Public License, version 2 (GPLv2), in all
-  cases as published by the Free Software Foundation.
-*/
-
-#ifndef _TIER_H_
-#define _TIER_H_
-
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
-
-
-/******************************************************************************/
-/* This is from dht-rebalancer.c as we dont have dht-rebalancer.h */
-#include "dht-common.h"
-#include "xlator.h"
-#include <signal.h>
-#include <fnmatch.h>
-#include <signal.h>
-
-#define DEFAULT_PROMOTE_FREQ_SEC 120
-#define DEFAULT_DEMOTE_FREQ_SEC  120
-#define DEFAULT_WRITE_FREQ_SEC 120
-#define DEFAULT_READ_FREQ_SEC 120
-
-/*
- * Size of timer wheel. We would not promote or demote less
- * frequently than this number.
- */
-#define TIMER_SECS 3600
-
-#include "gfdb_data_store.h"
-#include <ctype.h>
-#include <sys/stat.h>
-
-#define PROMOTION_QFILE "promotequeryfile"
-#define DEMOTION_QFILE "demotequeryfile"
-
-#define GET_QFILE_PATH(is_promotion)\
-        (is_promotion) ? promotion_qfile : demotion_qfile
-
-typedef struct _query_cbk_args {
-        xlator_t *this;
-        gf_defrag_info_t *defrag;
-        FILE *queryFILE;
-        int is_promotion;
-} query_cbk_args_t;
-
-int
-gf_run_tier(xlator_t *this, gf_defrag_info_t *defrag);
-
-typedef struct _gfdb_brick_dict_info {
-        gfdb_time_t           *time_stamp;
-        gf_boolean_t            _gfdb_promote;
-        query_cbk_args_t       *_query_cbk_args;
-} _gfdb_brick_dict_info_t;
-
-typedef struct _dm_thread_args {
-        xlator_t                *this;
-        gf_defrag_info_t        *defrag;
-        dict_t                  *brick_list;
-        int                     freq_time;
-        int                     return_value;
-} promotion_args_t, demotion_args_t;
-
-#endif
diff --git a/xlators/cluster/dht/src/unittest/dht_layout_mock.c b/xlators/cluster/dht/src/unittest/dht_layout_mock.c
index faf871d49be..771452963d1 100644
--- a/xlators/cluster/dht/src/unittest/dht_layout_mock.c
+++ b/xlators/cluster/dht/src/unittest/dht_layout_mock.c
@@ -7,71 +7,67 @@
   later), or the GNU General Public License, version 2 (GPLv2), in all
   cases as published by the Free Software Foundation.
 */
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
-
-#include "glusterfs.h"
-#include "xlator.h"
+#include <glusterfs/glusterfs.h>
+#include <glusterfs/xlator.h>
 #include "dht-common.h"
-#include "byte-order.h"
+#include <glusterfs/byte-order.h>
 
 int
-dht_hash_compute (xlator_t *this, int type, const char *name, uint32_t *hash_p)
+dht_hash_compute(xlator_t *this, int type, const char *name, uint32_t *hash_p)
 {
     return 0;
 }
 
 int
-dht_inode_ctx_layout_get (inode_t *inode, xlator_t *this, dht_layout_t **layout)
+dht_inode_ctx_layout_get(inode_t *inode, xlator_t *this, dht_layout_t **layout)
 {
     return 0;
 }
 
 int
-dht_inode_ctx_layout_set (inode_t *inode, xlator_t *this,
-                          dht_layout_t *layout_int)
+dht_inode_ctx_layout_set(inode_t *inode, xlator_t *this,
+                         dht_layout_t *layout_int)
 {
     return 0;
 }
 
 int
-dict_get_ptr (dict_t *this, char *key, void **ptr)
+dict_get_ptr(dict_t *this, char *key, void **ptr)
 {
     return 0;
 }
 
 int
-dict_get_ptr_and_len (dict_t *this, char *key, void **ptr, int *len)
+dict_get_ptr_and_len(dict_t *this, char *key, void **ptr, int *len)
 {
     return 0;
 }
 
-int _gf_log (const char *domain, const char *file,
-             const char *function, int32_t line, gf_loglevel_t level,
-             const char *fmt, ...)
+int
+_gf_log(const char *domain, const char *file, const char *function,
+        int32_t line, gf_loglevel_t level, const char *fmt, ...)
 {
     return 0;
 }
 
-int _gf_log_callingfn (const char *domain, const char *file,
-                       const char *function, int32_t line, gf_loglevel_t level,
-                       const char *fmt, ...)
+int
+_gf_log_callingfn(const char *domain, const char *file, const char *function,
+                  int32_t line, gf_loglevel_t level, const char *fmt, ...)
 {
     return 0;
 }
 
-void gf_uuid_unparse(const uuid_t uu, char *out)
+void
+gf_uuid_unparse(const uuid_t uu, char *out)
 {
     // could call a will-return function here
     // to place the correct data in *out
 }
 
 int
-_gf_msg (const char *domain, const char *file, const char *function,
-         int32_t line, gf_loglevel_t level, int errnum, int trace,
-         uint64_t msgid, const char *fmt, ...)
+_gf_msg(const char *domain, const char *file, const char *function,
+        int32_t line, gf_loglevel_t level, int errnum, int trace,
+        uint64_t msgid, const char *fmt, ...)
 {
     return 0;
 }
diff --git a/xlators/cluster/dht/src/unittest/dht_layout_unittest.c b/xlators/cluster/dht/src/unittest/dht_layout_unittest.c
index 3702af366f9..c94a1d0a2e1 100644
--- a/xlators/cluster/dht/src/unittest/dht_layout_unittest.c
+++ b/xlators/cluster/dht/src/unittest/dht_layout_unittest.c
@@ -9,8 +9,8 @@
 */
 
 #include "dht-common.h"
-#include "logging.h"
-#include "xlator.h"
+#include <glusterfs/logging.h>
+#include <glusterfs/xlator.h>
 
 #include <inttypes.h>
 #include <stdarg.h>
@@ -33,16 +33,17 @@ helper_xlator_init(uint32_t num_types)
 
     xl = test_calloc(1, sizeof(xlator_t));
     assert_non_null(xl);
-    xl->mem_acct.num_types = num_types;
-    xl->mem_acct.rec = test_calloc(num_types, sizeof(struct mem_acct_rec));
-    assert_non_null(xl->mem_acct.rec);
+    xl->mem_acct->num_types = num_types;
+    xl->mem_acct = test_calloc(sizeof(struct mem_acct) +
+                               sizeof(struct mem_acct_rec) + num_types);
+    assert_non_null(xl->mem_acct);
 
     xl->ctx = test_calloc(1, sizeof(glusterfs_ctx_t));
     assert_non_null(xl->ctx);
 
     for (i = 0; i < num_types; i++) {
-            ret = LOCK_INIT(&(xl->mem_acct.rec[i].lock));
-            assert_false(ret);
+        ret = LOCK_INIT(&(xl->mem_acct.rec[i].lock));
+        assert_false(ret);
     }
 
     ENSURE(num_types == xl->mem_acct.num_types);
@@ -57,8 +58,8 @@ helper_xlator_destroy(xlator_t *xl)
     int i, ret;
 
     for (i = 0; i < xl->mem_acct.num_types; i++) {
-            ret = LOCK_DESTROY(&(xl->mem_acct.rec[i].lock));
-            assert_int_equal(ret, 0);
+        ret = LOCK_DESTROY(&(xl->mem_acct.rec[i].lock));
+        assert_int_equal(ret, 0);
     }
 
     free(xl->mem_acct.rec);
@@ -75,7 +76,7 @@ test_dht_layout_new(void **state)
 {
     xlator_t *xl;
     dht_layout_t *layout;
-    dht_conf_t   *conf;
+    dht_conf_t *conf;
     int cnt;
 
     expect_assert_failure(dht_layout_new(NULL, 0));
@@ -89,7 +90,7 @@ test_dht_layout_new(void **state)
     assert_non_null(layout);
     assert_int_equal(layout->type, DHT_HASH_TYPE_DM);
     assert_int_equal(layout->cnt, cnt);
-    assert_int_equal(layout->ref, 1);
+    assert_int_equal(GF_ATOMIC_GET(layout->ref), 1);
     assert_int_equal(layout->gen, 0);
     assert_int_equal(layout->spread_cnt, 0);
     free(layout);
@@ -106,7 +107,7 @@ test_dht_layout_new(void **state)
     assert_non_null(layout);
     assert_int_equal(layout->type, DHT_HASH_TYPE_DM);
     assert_int_equal(layout->cnt, cnt);
-    assert_int_equal(layout->ref, 1);
+    assert_int_equal(GF_ATOMIC_GET(layout->ref), 1);
     assert_int_equal(layout->gen, conf->gen);
     assert_int_equal(layout->spread_cnt, conf->dir_spread_cnt);
     free(layout);
@@ -115,7 +116,9 @@ test_dht_layout_new(void **state)
     helper_xlator_destroy(xl);
 }
 
-int main(void) {
+int
+main(void)
+{
     const struct CMUnitTest xlator_dht_layout_tests[] = {
         unit_test(test_dht_layout_new),
     };
diff --git a/xlators/cluster/ec/src/Makefile.am b/xlators/cluster/ec/src/Makefile.am
index 12d87f99e4d..406a636bbc2 100644
--- a/xlators/cluster/ec/src/Makefile.am
+++ b/xlators/cluster/ec/src/Makefile.am
@@ -12,8 +12,11 @@ ec_sources += ec-dir-write.c
 ec_sources += ec-inode-read.c
 ec_sources += ec-inode-write.c
 ec_sources += ec-combine.c
-ec_sources += ec-gf.c
 ec_sources += ec-method.c
+ec_sources += ec-galois.c
+ec_sources += ec-code.c
+ec_sources += ec-code-c.c
+ec_sources += ec-gf8.c
 ec_sources += ec-heal.c
 ec_sources += ec-heald.c
 
@@ -24,22 +27,50 @@ ec_headers += ec-data.h
 ec_headers += ec-fops.h
 ec_headers += ec-common.h
 ec_headers += ec-combine.h
-ec_headers += ec-gf.h
 ec_headers += ec-method.h
+ec_headers += ec-galois.h
+ec_headers += ec-code.h
+ec_headers += ec-code-c.h
+ec_headers += ec-gf8.h
 ec_headers += ec-heald.h
+ec_headers += ec-messages.h
+ec_headers += ec-types.h
+
+if ENABLE_EC_DYNAMIC_INTEL
+  ec_sources += ec-code-intel.c
+  ec_headers += ec-code-intel.h
+endif
+
+if ENABLE_EC_DYNAMIC_X64
+  ec_sources += ec-code-x64.c
+  ec_headers += ec-code-x64.h
+endif
+
+if ENABLE_EC_DYNAMIC_SSE
+  ec_sources += ec-code-sse.c
+  ec_headers += ec-code-sse.h
+endif
+
+if ENABLE_EC_DYNAMIC_AVX
+  ec_sources += ec-code-avx.c
+  ec_headers += ec-code-avx.h
+endif
 
 ec_ext_sources = $(top_builddir)/xlators/lib/src/libxlator.c
 
 ec_ext_headers = $(top_builddir)/xlators/lib/src/libxlator.h
 
-ec_la_LDFLAGS = -module -avoid-version
+ec_la_LDFLAGS = -module $(GF_XLATOR_DEFAULT_LDFLAGS)
 ec_la_SOURCES = $(ec_sources) $(ec_headers) $(ec_ext_sources) $(ec_ext_headers)
 ec_la_LIBADD = $(top_builddir)/libglusterfs/src/libglusterfs.la
 
-AM_CPPFLAGS  = $(GF_CPPFLAGS)
+AM_CPPFLAGS = $(GF_CPPFLAGS)
 AM_CPPFLAGS += -I$(top_srcdir)/libglusterfs/src
 AM_CPPFLAGS += -I$(top_srcdir)/xlators/lib/src
 AM_CPPFLAGS += -I$(top_srcdir)/rpc/rpc-lib/src
+AM_CPPFLAGS += -I$(top_srcdir)/rpc/xdr/src
+AM_CPPFLAGS += -I$(top_builddir)/rpc/xdr/src
+AM_CPPFLAGS += -DGLUSTERFS_LIBEXECDIR=\"$(GLUSTERFS_LIBEXECDIR)\"
 
 AM_CFLAGS = -Wall $(GF_CFLAGS)
 
diff --git a/xlators/cluster/ec/src/ec-code-avx.c b/xlators/cluster/ec/src/ec-code-avx.c
new file mode 100644
index 00000000000..70afaa00f54
--- /dev/null
+++ b/xlators/cluster/ec/src/ec-code-avx.c
@@ -0,0 +1,109 @@
+/*
+  Copyright (c) 2015 DataLab, s.l. <http://www.datalab.es>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
+*/
+
+#include <errno.h>
+
+#include "ec-code-intel.h"
+
+static void
+ec_code_avx_prolog(ec_code_builder_t *builder)
+{
+    builder->loop = builder->address;
+}
+
+static void
+ec_code_avx_epilog(ec_code_builder_t *builder)
+{
+    ec_code_intel_op_add_i2r(builder, 32, REG_DX);
+    ec_code_intel_op_add_i2r(builder, 32, REG_DI);
+    ec_code_intel_op_test_i2r(builder, builder->width - 1, REG_DX);
+    ec_code_intel_op_jne(builder, builder->loop);
+
+    ec_code_intel_op_ret(builder, 0);
+}
+
+static void
+ec_code_avx_load(ec_code_builder_t *builder, uint32_t dst, uint32_t idx,
+                 uint32_t bit)
+{
+    if (builder->linear) {
+        ec_code_intel_op_mov_m2avx(
+            builder, REG_SI, REG_DX, 1,
+            idx * builder->width * builder->bits + bit * builder->width, dst);
+    } else {
+        if (builder->base != idx) {
+            ec_code_intel_op_mov_m2r(builder, REG_SI, REG_NULL, 0, idx * 8,
+                                     REG_AX);
+            builder->base = idx;
+        }
+        ec_code_intel_op_mov_m2avx(builder, REG_AX, REG_DX, 1,
+                                   bit * builder->width, dst);
+    }
+}
+
+static void
+ec_code_avx_store(ec_code_builder_t *builder, uint32_t src, uint32_t bit)
+{
+    ec_code_intel_op_mov_avx2m(builder, src, REG_DI, REG_NULL, 0,
+                               bit * builder->width);
+}
+
+static void
+ec_code_avx_copy(ec_code_builder_t *builder, uint32_t dst, uint32_t src)
+{
+    ec_code_intel_op_mov_avx2avx(builder, src, dst);
+}
+
+static void
+ec_code_avx_xor2(ec_code_builder_t *builder, uint32_t dst, uint32_t src)
+{
+    ec_code_intel_op_xor_avx2avx(builder, src, dst);
+}
+
+static void
+ec_code_avx_xor3(ec_code_builder_t *builder, uint32_t dst, uint32_t src1,
+                 uint32_t src2)
+{
+    ec_code_intel_op_mov_avx2avx(builder, src1, dst);
+    ec_code_intel_op_xor_avx2avx(builder, src2, dst);
+}
+
+static void
+ec_code_avx_xorm(ec_code_builder_t *builder, uint32_t dst, uint32_t idx,
+                 uint32_t bit)
+{
+    if (builder->linear) {
+        ec_code_intel_op_xor_m2avx(
+            builder, REG_SI, REG_DX, 1,
+            idx * builder->width * builder->bits + bit * builder->width, dst);
+    } else {
+        if (builder->base != idx) {
+            ec_code_intel_op_mov_m2r(builder, REG_SI, REG_NULL, 0, idx * 8,
+                                     REG_AX);
+            builder->base = idx;
+        }
+        ec_code_intel_op_xor_m2avx(builder, REG_AX, REG_DX, 1,
+                                   bit * builder->width, dst);
+    }
+}
+
+static char *ec_code_avx_needed_flags[] = {"avx2", NULL};
+
+ec_code_gen_t ec_code_gen_avx = {.name = "avx",
+                                 .flags = ec_code_avx_needed_flags,
+                                 .width = 32,
+                                 .prolog = ec_code_avx_prolog,
+                                 .epilog = ec_code_avx_epilog,
+                                 .load = ec_code_avx_load,
+                                 .store = ec_code_avx_store,
+                                 .copy = ec_code_avx_copy,
+                                 .xor2 = ec_code_avx_xor2,
+                                 .xor3 = ec_code_avx_xor3,
+                                 .xorm = ec_code_avx_xorm};
diff --git a/xlators/cluster/dht/src/dht-helper.h b/xlators/cluster/ec/src/ec-code-avx.h
index e3ab9c4d93b..fdca4ad2c8f 100644
--- a/xlators/cluster/dht/src/dht-helper.h
+++ b/xlators/cluster/ec/src/ec-code-avx.h
@@ -1,5 +1,5 @@
 /*
-  Copyright (c) 2008-2014 Red Hat, Inc. <http://www.redhat.com>
+  Copyright (c) 2015 DataLab, s.l. <http://www.datalab.es>
   This file is part of GlusterFS.
 
   This file is licensed to you under your choice of the GNU Lesser
@@ -7,13 +7,12 @@
   later), or the GNU General Public License, version 2 (GPLv2), in all
   cases as published by the Free Software Foundation.
 */
-#ifndef _DHT_HELPER_H
-#define _DHT_HELPER_H
 
-int
-dht_lock_order_requests (dht_lock_t **lk_array, int count);
+#ifndef __EC_CODE_AVX_H__
+#define __EC_CODE_AVX_H__
 
-void
-dht_blocking_inodelk_rec (call_frame_t *frame, int i);
+#include "ec-code.h"
 
-#endif
+extern ec_code_gen_t ec_code_gen_avx;
+
+#endif /* __EC_CODE_AVX_H__ */
diff --git a/xlators/cluster/ec/src/ec-code-c.c b/xlators/cluster/ec/src/ec-code-c.c
new file mode 100644
index 00000000000..acdc665c2cf
--- /dev/null
+++ b/xlators/cluster/ec/src/ec-code-c.c
@@ -0,0 +1,11679 @@
+/*
+  Copyright (c) 2015 DataLab, s.l. <http://www.datalab.es>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
+*/
+
+#include <inttypes.h>
+#include <string.h>
+
+#include "ec-method.h"
+#include "ec-code-c.h"
+
+#define WIDTH (EC_METHOD_WORD_SIZE / sizeof(uint64_t))
+
+static void
+gf8_muladd_00(void *out, void *in)
+{
+    memcpy(out, in, EC_METHOD_WORD_SIZE * 8);
+}
+
+static void
+gf8_muladd_01(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        out_ptr[0] ^= in_ptr[0];
+        out_ptr[WIDTH] ^= in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] ^= in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] ^= in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] ^= in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] ^= in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] ^= in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] ^= in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_02(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        out0 = in7;
+        out1 = in0;
+        out7 = in6;
+        out5 = in4;
+        out6 = in5;
+        out3 = in2 ^ in7;
+        out4 = in3 ^ in7;
+        out2 = in1 ^ in7;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_03(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        out0 = in0 ^ in7;
+        tmp0 = in2 ^ in7;
+        out1 = in0 ^ in1;
+        out7 = in6 ^ in7;
+        out5 = in4 ^ in5;
+        out6 = in5 ^ in6;
+        out4 = in3 ^ in4 ^ in7;
+        out2 = tmp0 ^ in1;
+        out3 = tmp0 ^ in3;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_04(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        out0 = in6;
+        out1 = in7;
+        out7 = in5;
+        out6 = in4;
+        tmp0 = in6 ^ in7;
+        out2 = in0 ^ in6;
+        out5 = in3 ^ in7;
+        out3 = tmp0 ^ in1;
+        out4 = tmp0 ^ in2;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_05(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        out0 = in0 ^ in6;
+        out1 = in1 ^ in7;
+        out7 = in5 ^ in7;
+        out6 = in4 ^ in6;
+        out2 = out0 ^ in2;
+        out3 = out1 ^ in3 ^ in6;
+        out5 = out7 ^ in3;
+        out4 = out6 ^ in2 ^ in7;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_06(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        out0 = in6 ^ in7;
+        tmp0 = in1 ^ in6;
+        out1 = in0 ^ in7;
+        out7 = in5 ^ in6;
+        out6 = in4 ^ in5;
+        out4 = in2 ^ in3 ^ in6;
+        out5 = in3 ^ in4 ^ in7;
+        out3 = tmp0 ^ in2;
+        out2 = tmp0 ^ out1;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_07(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0, tmp1, tmp2, tmp3;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        tmp0 = in2 ^ in6;
+        tmp1 = in5 ^ in6;
+        tmp2 = in0 ^ in7;
+        tmp3 = tmp0 ^ in3;
+        out6 = tmp1 ^ in4;
+        out7 = tmp1 ^ in7;
+        out0 = tmp2 ^ in6;
+        out1 = tmp2 ^ in1;
+        out3 = tmp3 ^ in1;
+        out4 = tmp3 ^ in4;
+        out5 = out4 ^ out7 ^ in2;
+        out2 = tmp0 ^ out1;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_08(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        out0 = in5;
+        out1 = in6;
+        out7 = in4;
+        out6 = in3 ^ in7;
+        out3 = in0 ^ in5 ^ in6;
+        out5 = in2 ^ in6 ^ in7;
+        out2 = in5 ^ in7;
+        out4 = out2 ^ in1 ^ in6;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_09(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        out0 = in0 ^ in5;
+        tmp0 = in3 ^ in6;
+        out1 = in1 ^ in6;
+        out7 = in4 ^ in7;
+        out2 = in2 ^ in5 ^ in7;
+        out3 = tmp0 ^ out0;
+        out6 = tmp0 ^ in7;
+        out4 = out1 ^ out7 ^ in5;
+        out5 = out2 ^ in6;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_0A(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        out0 = in5 ^ in7;
+        out1 = in0 ^ in6;
+        out7 = in4 ^ in6;
+        out2 = in1 ^ in5;
+        out6 = out0 ^ in3;
+        out3 = out0 ^ out1 ^ in2;
+        out5 = out7 ^ in2 ^ in7;
+        out4 = out2 ^ in3 ^ in6;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_0B(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0, tmp1, tmp2;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        tmp0 = in2 ^ in5;
+        tmp1 = in0 ^ in6;
+        tmp2 = in4 ^ in7;
+        out0 = in0 ^ in5 ^ in7;
+        out2 = tmp0 ^ in1;
+        out1 = tmp1 ^ in1;
+        out6 = tmp1 ^ out0 ^ in3;
+        out7 = tmp2 ^ in6;
+        out4 = tmp2 ^ out6 ^ in1;
+        out3 = out6 ^ in0 ^ in2;
+        out5 = tmp0 ^ out7;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_0C(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0, tmp1;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        out0 = in5 ^ in6;
+        out1 = in6 ^ in7;
+        out7 = in4 ^ in5;
+        tmp0 = in1 ^ in5;
+        tmp1 = in0 ^ in7;
+        out5 = in2 ^ in3 ^ in6;
+        out6 = in3 ^ in4 ^ in7;
+        out2 = tmp1 ^ out0;
+        out4 = tmp0 ^ in2;
+        out3 = tmp0 ^ tmp1;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_0D(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0, tmp1, tmp2;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        tmp0 = in4 ^ in5;
+        tmp1 = in5 ^ in6;
+        out1 = in1 ^ in6 ^ in7;
+        out7 = tmp0 ^ in7;
+        out4 = tmp0 ^ in1 ^ in2;
+        out0 = tmp1 ^ in0;
+        tmp2 = tmp1 ^ in3;
+        out6 = tmp2 ^ out7;
+        out2 = out0 ^ in2 ^ in7;
+        out3 = out0 ^ out1 ^ in3;
+        out5 = tmp2 ^ in2;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_0E(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0, tmp1, tmp2, tmp3;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        tmp0 = in0 ^ in1;
+        tmp1 = in2 ^ in5;
+        tmp2 = in5 ^ in6;
+        out1 = in0 ^ in6 ^ in7;
+        out3 = tmp0 ^ tmp1;
+        out2 = tmp0 ^ tmp2;
+        tmp3 = tmp1 ^ in3;
+        out7 = tmp2 ^ in4;
+        out0 = tmp2 ^ in7;
+        out4 = tmp3 ^ in1 ^ in7;
+        out5 = tmp3 ^ out7;
+        out6 = out0 ^ out5 ^ in2;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_0F(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0, tmp1, tmp2, tmp3;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        tmp0 = in6 ^ in7;
+        tmp1 = tmp0 ^ in1;
+        tmp2 = tmp0 ^ in5;
+        out1 = tmp1 ^ in0;
+        out7 = tmp2 ^ in4;
+        out0 = tmp2 ^ in0;
+        out6 = out7 ^ in3;
+        out5 = out6 ^ in2 ^ in7;
+        tmp3 = tmp1 ^ out0 ^ in2;
+        out4 = tmp1 ^ out5;
+        out2 = tmp3 ^ in6;
+        out3 = tmp3 ^ in3;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_10(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0, tmp1;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        out0 = in4;
+        out1 = in5;
+        out7 = in3 ^ in7;
+        tmp0 = in6 ^ in7;
+        out2 = in4 ^ in6;
+        tmp1 = out2 ^ in5;
+        out6 = tmp0 ^ in2;
+        out3 = tmp0 ^ tmp1;
+        out5 = out2 ^ out3 ^ in1;
+        out4 = tmp1 ^ in0;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_11(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        out7 = in3;
+        out0 = in0 ^ in4;
+        out1 = in1 ^ in5;
+        out6 = in2 ^ in7;
+        out4 = in0 ^ in5 ^ in6;
+        out5 = in1 ^ in6 ^ in7;
+        out2 = in2 ^ in4 ^ in6;
+        out3 = in3 ^ in4 ^ in5 ^ in7;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_12(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0, tmp1;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        out0 = in4 ^ in7;
+        out1 = in0 ^ in5;
+        out3 = in2 ^ in4 ^ in5;
+        tmp0 = out0 ^ in6;
+        out2 = tmp0 ^ in1;
+        tmp1 = tmp0 ^ in3;
+        out6 = tmp0 ^ out3;
+        out5 = out2 ^ in5;
+        out7 = tmp1 ^ in4;
+        out4 = tmp1 ^ out1;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_13(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0, tmp1;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        out7 = in3 ^ in6;
+        tmp0 = in0 ^ in5;
+        tmp1 = in4 ^ in7;
+        out6 = in2 ^ in5 ^ in7;
+        out4 = tmp0 ^ out7 ^ in7;
+        out1 = tmp0 ^ in1;
+        out0 = tmp1 ^ in0;
+        out5 = tmp1 ^ in1 ^ in6;
+        out3 = tmp1 ^ out6 ^ in3;
+        out2 = out5 ^ in2;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_14(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0, tmp1;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        out0 = in4 ^ in6;
+        out1 = in5 ^ in7;
+        out2 = in0 ^ in4;
+        tmp0 = out0 ^ in5;
+        out7 = out1 ^ in3;
+        tmp1 = out1 ^ in2;
+        out3 = tmp0 ^ in1;
+        out6 = tmp0 ^ tmp1;
+        out4 = tmp1 ^ out2;
+        out5 = out3 ^ in3 ^ in4;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_15(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        out7 = in3 ^ in5;
+        tmp0 = in0 ^ in4;
+        out1 = in1 ^ in5 ^ in7;
+        out5 = in1 ^ in3 ^ in6;
+        out0 = tmp0 ^ in6;
+        out2 = tmp0 ^ in2;
+        out3 = out5 ^ in4 ^ in5;
+        out6 = out2 ^ in0 ^ in7;
+        out4 = tmp0 ^ out6 ^ in5;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_16(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0, tmp1, tmp2, tmp3;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        tmp0 = in0 ^ in5;
+        tmp1 = in4 ^ in7;
+        tmp2 = in2 ^ in3 ^ in4;
+        out1 = tmp0 ^ in7;
+        out4 = tmp0 ^ tmp2;
+        out0 = tmp1 ^ in6;
+        tmp3 = tmp1 ^ in1;
+        out6 = out0 ^ in2 ^ in5;
+        out2 = tmp3 ^ in0;
+        out3 = out6 ^ in1;
+        out7 = tmp2 ^ out6;
+        out5 = tmp3 ^ out7;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_17(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0, tmp1, tmp2, tmp3;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        tmp0 = in2 ^ in5;
+        tmp1 = in3 ^ in6;
+        tmp2 = tmp0 ^ in4;
+        out4 = tmp0 ^ in0 ^ in3;
+        out7 = tmp1 ^ in5;
+        tmp3 = tmp1 ^ in1;
+        out6 = tmp2 ^ in7;
+        out5 = tmp3 ^ in4;
+        out3 = tmp3 ^ out6;
+        out0 = out3 ^ out4 ^ in1;
+        out2 = out3 ^ out7 ^ in0;
+        out1 = tmp2 ^ out2;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_18(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0, tmp1;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        out0 = in4 ^ in5;
+        out1 = in5 ^ in6;
+        tmp0 = in4 ^ in7;
+        out5 = in1 ^ in2 ^ in5;
+        out6 = in2 ^ in3 ^ in6;
+        out2 = tmp0 ^ out1;
+        out7 = tmp0 ^ in3;
+        tmp1 = tmp0 ^ in0;
+        out3 = tmp1 ^ in6;
+        out4 = tmp1 ^ in1;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_19(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0, tmp1;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        out5 = in1 ^ in2;
+        out7 = in3 ^ in4;
+        tmp0 = in0 ^ in7;
+        out6 = in2 ^ in3;
+        out1 = in1 ^ in5 ^ in6;
+        out0 = in0 ^ in4 ^ in5;
+        out4 = tmp0 ^ in1;
+        tmp1 = tmp0 ^ in6;
+        out2 = tmp1 ^ out0 ^ in2;
+        out3 = tmp1 ^ out7;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_1A(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0, tmp1, tmp2, tmp3;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        tmp0 = in4 ^ in5;
+        tmp1 = in5 ^ in6;
+        tmp2 = tmp0 ^ in1;
+        out0 = tmp0 ^ in7;
+        out1 = tmp1 ^ in0;
+        tmp3 = tmp1 ^ in3;
+        out5 = tmp2 ^ in2;
+        out2 = tmp2 ^ in6;
+        out7 = tmp3 ^ out0;
+        out6 = tmp3 ^ in2;
+        out4 = tmp3 ^ out2 ^ in0;
+        out3 = tmp0 ^ out1 ^ in2;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_1B(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0, tmp1, tmp2, tmp3, tmp4;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        tmp0 = in2 ^ in4;
+        tmp1 = in2 ^ in5;
+        tmp2 = in3 ^ in6;
+        out5 = tmp0 ^ in1;
+        tmp3 = tmp0 ^ in0;
+        out6 = tmp1 ^ in3;
+        out0 = tmp1 ^ tmp3 ^ in7;
+        out7 = tmp2 ^ in4;
+        tmp4 = out5 ^ in6;
+        out3 = tmp2 ^ tmp3;
+        out2 = tmp4 ^ in5;
+        out4 = tmp4 ^ out3;
+        out1 = tmp3 ^ out2;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_1C(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0, tmp1, tmp2, tmp3, tmp4;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        tmp0 = in2 ^ in3;
+        tmp1 = in4 ^ in6;
+        tmp2 = in5 ^ in7;
+        out6 = tmp0 ^ tmp1;
+        out0 = tmp1 ^ in5;
+        out1 = tmp2 ^ in6;
+        tmp3 = tmp2 ^ in1;
+        tmp4 = tmp2 ^ in4;
+        out2 = tmp4 ^ in0;
+        out7 = tmp4 ^ in3;
+        out5 = tmp0 ^ tmp3;
+        out3 = tmp3 ^ out2;
+        out4 = out3 ^ in2 ^ in6;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_1D(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0, tmp1, tmp2, tmp3, tmp4;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        tmp0 = in1 ^ in3;
+        tmp1 = in0 ^ in4;
+        tmp2 = in3 ^ in4;
+        tmp3 = in2 ^ in7;
+        out3 = tmp0 ^ tmp1;
+        out5 = tmp0 ^ tmp3;
+        tmp4 = tmp1 ^ in5;
+        out6 = tmp2 ^ in2;
+        out7 = tmp2 ^ in5;
+        out2 = tmp3 ^ tmp4;
+        out4 = out3 ^ out6 ^ in6;
+        out0 = tmp4 ^ in6;
+        out1 = out2 ^ out4 ^ in4;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_1E(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0, tmp1, tmp2, tmp3;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        tmp0 = in0 ^ in4;
+        tmp1 = in2 ^ in7;
+        tmp2 = tmp0 ^ in1;
+        out3 = tmp1 ^ tmp2;
+        out2 = tmp2 ^ in5;
+        out4 = out3 ^ in3 ^ in6;
+        tmp3 = out4 ^ in7;
+        out6 = tmp3 ^ out2 ^ in4;
+        out7 = tmp1 ^ out6;
+        out0 = out7 ^ in3;
+        out1 = tmp0 ^ out0;
+        out5 = tmp3 ^ out1;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_1F(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0, tmp1;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        tmp0 = in4 ^ in6;
+        tmp1 = tmp0 ^ in5;
+        out7 = tmp1 ^ in3;
+        out0 = tmp1 ^ in0 ^ in7;
+        out6 = out7 ^ in2 ^ in6;
+        out1 = out0 ^ in1 ^ in4;
+        out4 = out0 ^ out6 ^ in1;
+        out3 = tmp0 ^ out4;
+        out2 = out4 ^ out7 ^ in7;
+        out5 = out3 ^ in0;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_20(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0, tmp1;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        out1 = in4;
+        out0 = in3 ^ in7;
+        tmp0 = in3 ^ in4;
+        tmp1 = in6 ^ in7;
+        out2 = out0 ^ in5;
+        out4 = tmp0 ^ in5;
+        out3 = tmp0 ^ tmp1;
+        out7 = tmp1 ^ in2;
+        out6 = tmp1 ^ in1 ^ in5;
+        out5 = out2 ^ out3 ^ in0;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_21(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        out1 = in1 ^ in4;
+        tmp0 = in4 ^ in6;
+        out4 = in3 ^ in5;
+        out7 = in2 ^ in6;
+        out0 = in0 ^ in3 ^ in7;
+        out6 = in1 ^ in5 ^ in7;
+        out3 = tmp0 ^ in7;
+        out5 = tmp0 ^ in0;
+        out2 = out4 ^ in2 ^ in7;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_22(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        out0 = in3;
+        out1 = in0 ^ in4;
+        out7 = in2 ^ in7;
+        out4 = in4 ^ in5 ^ in7;
+        out5 = in0 ^ in5 ^ in6;
+        out6 = in1 ^ in6 ^ in7;
+        out3 = in2 ^ in3 ^ in4 ^ in6;
+        out2 = in1 ^ in3 ^ in5;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_23(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        out7 = in2;
+        out0 = in0 ^ in3;
+        out4 = in5 ^ in7;
+        out5 = in0 ^ in6;
+        out6 = in1 ^ in7;
+        out3 = in2 ^ in4 ^ in6;
+        out1 = in0 ^ in1 ^ in4;
+        out2 = out4 ^ out6 ^ in2 ^ in3;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_24(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0, tmp1;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        out1 = in4 ^ in7;
+        tmp0 = in3 ^ in4;
+        out0 = in3 ^ in6 ^ in7;
+        out3 = tmp0 ^ in1;
+        tmp1 = out0 ^ in5;
+        out6 = tmp1 ^ out3;
+        out2 = tmp1 ^ in0;
+        out7 = tmp1 ^ in2 ^ in3;
+        out5 = out2 ^ in4;
+        out4 = tmp0 ^ out7;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_25(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        out3 = in1 ^ in4;
+        tmp0 = in2 ^ in5;
+        out1 = out3 ^ in7;
+        out7 = tmp0 ^ in6;
+        out6 = out1 ^ in5;
+        out4 = out7 ^ in3 ^ in7;
+        out2 = out4 ^ in0;
+        out0 = tmp0 ^ out2;
+        out5 = out0 ^ in4;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_26(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0, tmp1, tmp2;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        out0 = in3 ^ in6;
+        tmp0 = in4 ^ in7;
+        out7 = in2 ^ in5 ^ in7;
+        tmp1 = out0 ^ in0 ^ in5;
+        out1 = tmp0 ^ in0;
+        tmp2 = tmp0 ^ in6;
+        out2 = tmp1 ^ in1;
+        out5 = tmp1 ^ in7;
+        out6 = tmp2 ^ in1;
+        out4 = tmp2 ^ out7;
+        out3 = out0 ^ out6 ^ in2;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_27(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        out7 = in2 ^ in5;
+        out0 = in0 ^ in3 ^ in6;
+        out6 = in1 ^ in4 ^ in7;
+        out4 = out7 ^ in6;
+        out2 = out0 ^ out7 ^ in1;
+        out5 = out0 ^ in7;
+        out1 = out6 ^ in0;
+        out3 = out6 ^ in2;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_28(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0, tmp1, tmp2;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        out2 = in3;
+        out1 = in4 ^ in6;
+        out0 = in3 ^ in5 ^ in7;
+        tmp0 = out1 ^ in7;
+        tmp1 = out0 ^ in4;
+        out7 = tmp0 ^ in2;
+        tmp2 = tmp0 ^ in1;
+        out3 = tmp1 ^ in0;
+        out6 = tmp1 ^ tmp2;
+        out4 = tmp2 ^ in3;
+        out5 = out3 ^ in2 ^ in3;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_29(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0, tmp1, tmp2;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        out2 = in2 ^ in3;
+        tmp0 = in1 ^ in3;
+        tmp1 = in4 ^ in6;
+        tmp2 = in0 ^ in4 ^ in7;
+        out6 = tmp0 ^ in5;
+        out4 = tmp0 ^ in6 ^ in7;
+        out1 = tmp1 ^ in1;
+        out7 = tmp1 ^ in2;
+        out3 = tmp2 ^ in5;
+        out5 = tmp2 ^ in2;
+        out0 = out3 ^ in3 ^ in4;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_2A(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0, tmp1;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        out0 = in3 ^ in5;
+        tmp0 = in1 ^ in3;
+        tmp1 = in0 ^ in4;
+        out7 = in2 ^ in4 ^ in7;
+        out3 = tmp1 ^ out0 ^ in2;
+        out2 = tmp0 ^ in7;
+        out6 = tmp0 ^ in6;
+        out1 = tmp1 ^ in6;
+        out5 = tmp1 ^ out7 ^ in5;
+        out4 = out1 ^ in0 ^ in1;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_2B(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0, tmp1;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        out4 = in1 ^ in6;
+        out7 = in2 ^ in4;
+        tmp0 = in0 ^ in5;
+        tmp1 = in2 ^ in7;
+        out6 = in1 ^ in3;
+        out1 = out4 ^ in0 ^ in4;
+        out3 = tmp0 ^ out7;
+        out0 = tmp0 ^ in3;
+        out5 = tmp1 ^ in0;
+        out2 = tmp1 ^ out6;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_2C(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0, tmp1, tmp2, tmp3;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        tmp0 = in2 ^ in5;
+        tmp1 = in2 ^ in3 ^ in4;
+        tmp2 = tmp0 ^ in6;
+        out4 = tmp1 ^ in1;
+        out5 = tmp1 ^ in0 ^ in5;
+        tmp3 = tmp2 ^ in4;
+        out6 = tmp2 ^ out4;
+        out7 = tmp3 ^ in7;
+        out2 = tmp3 ^ out5;
+        out3 = out6 ^ in0;
+        out0 = tmp1 ^ out7;
+        out1 = tmp0 ^ out7;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_2D(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0, tmp1, tmp2, tmp3;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        tmp0 = in2 ^ in3;
+        out4 = tmp0 ^ in1;
+        tmp1 = tmp0 ^ in0;
+        out2 = tmp1 ^ in6;
+        out5 = tmp1 ^ in4;
+        tmp2 = out2 ^ in2;
+        tmp3 = tmp2 ^ in5;
+        out0 = tmp3 ^ in7;
+        out7 = tmp3 ^ out5;
+        out6 = out4 ^ out7 ^ in6;
+        out3 = tmp2 ^ out6;
+        out1 = out0 ^ out6 ^ in0;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_2E(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0, tmp1, tmp2;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        tmp0 = in4 ^ in7;
+        out0 = in3 ^ in5 ^ in6;
+        tmp1 = tmp0 ^ in0;
+        tmp2 = tmp0 ^ in2;
+        out1 = tmp1 ^ in6;
+        out4 = tmp2 ^ in1;
+        out7 = tmp2 ^ in5;
+        out3 = out0 ^ out4 ^ in0;
+        out2 = out3 ^ out7 ^ in7;
+        out6 = tmp1 ^ out2;
+        out5 = tmp1 ^ out7 ^ in3;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_2F(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0, tmp1, tmp2;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        tmp0 = in0 ^ in3;
+        tmp1 = in2 ^ in5;
+        out4 = in1 ^ in2 ^ in7;
+        out6 = in1 ^ in3 ^ in4;
+        out5 = tmp0 ^ in2;
+        tmp2 = tmp0 ^ in6;
+        out7 = tmp1 ^ in4;
+        out0 = tmp2 ^ in5;
+        out2 = tmp2 ^ out4;
+        out1 = tmp2 ^ out6 ^ in7;
+        out3 = tmp1 ^ out1;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_30(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0, tmp1;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        out1 = in4 ^ in5;
+        tmp0 = in3 ^ in6;
+        tmp1 = in4 ^ in7;
+        out6 = in1 ^ in2 ^ in5;
+        out3 = tmp0 ^ in5;
+        out4 = tmp0 ^ in0;
+        out7 = tmp0 ^ in2;
+        out0 = tmp1 ^ in3;
+        out2 = tmp1 ^ out3;
+        out5 = tmp1 ^ in0 ^ in1;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_31(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0, tmp1, tmp2;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        out3 = in5 ^ in6;
+        tmp0 = in4 ^ in5;
+        tmp1 = in0 ^ in3 ^ in4;
+        tmp2 = out3 ^ in2;
+        out1 = tmp0 ^ in1;
+        out0 = tmp1 ^ in7;
+        out4 = tmp1 ^ in6;
+        out6 = tmp2 ^ in1;
+        out2 = tmp2 ^ out0 ^ in0;
+        out5 = out1 ^ in0 ^ in7;
+        out7 = tmp0 ^ out2;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_32(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0, tmp1;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        out0 = in3 ^ in4;
+        out7 = in2 ^ in3;
+        tmp0 = in5 ^ in6;
+        tmp1 = in0 ^ in7;
+        out6 = in1 ^ in2;
+        out1 = in0 ^ in4 ^ in5;
+        out2 = tmp0 ^ out0 ^ in1;
+        out3 = tmp0 ^ out7 ^ in7;
+        out4 = tmp1 ^ in6;
+        out5 = tmp1 ^ in1;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_33(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0, tmp1, tmp2, tmp3, tmp4;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        tmp0 = in2 ^ in3;
+        tmp1 = in0 ^ in4;
+        tmp2 = in1 ^ in5;
+        out6 = in1 ^ in2 ^ in6;
+        out7 = tmp0 ^ in7;
+        out0 = tmp1 ^ in3;
+        out1 = tmp1 ^ tmp2;
+        tmp3 = tmp2 ^ in7;
+        tmp4 = tmp2 ^ in4 ^ in6;
+        out5 = tmp3 ^ in0;
+        out3 = tmp3 ^ out6;
+        out4 = tmp4 ^ out5;
+        out2 = tmp0 ^ tmp4;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_34(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0, tmp1, tmp2, tmp3, tmp4;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        tmp0 = in3 ^ in4;
+        tmp1 = in4 ^ in5;
+        tmp2 = tmp0 ^ in1;
+        tmp3 = tmp0 ^ in6;
+        out1 = tmp1 ^ in7;
+        tmp4 = tmp1 ^ in2;
+        out5 = tmp2 ^ in0;
+        out3 = tmp2 ^ out1;
+        out0 = tmp3 ^ in7;
+        out7 = tmp3 ^ tmp4;
+        out6 = tmp4 ^ in1;
+        out2 = out3 ^ out5 ^ in3;
+        out4 = tmp4 ^ out2;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_35(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0, tmp1, tmp2;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        tmp0 = in2 ^ in6;
+        tmp1 = in5 ^ in7;
+        out7 = tmp0 ^ tmp1 ^ in3;
+        out3 = tmp1 ^ in1;
+        out1 = out3 ^ in4;
+        tmp2 = out1 ^ in7;
+        out5 = tmp2 ^ in0 ^ in3;
+        out6 = tmp0 ^ tmp2;
+        out0 = out3 ^ out5 ^ in6;
+        out4 = tmp0 ^ out0;
+        out2 = out4 ^ in5;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_36(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0, tmp1;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        out4 = in0 ^ in2;
+        tmp0 = in1 ^ in3;
+        out0 = in3 ^ in4 ^ in6;
+        out6 = in1 ^ in2 ^ in4;
+        out5 = tmp0 ^ in0;
+        tmp1 = out5 ^ in5;
+        out2 = tmp1 ^ in4;
+        out3 = tmp1 ^ out4;
+        out1 = tmp0 ^ out2 ^ in7;
+        out7 = out3 ^ in1;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_37(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0, tmp1, tmp2, tmp3;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        tmp0 = in1 ^ in2;
+        tmp1 = in2 ^ in4;
+        tmp2 = tmp0 ^ in6;
+        out3 = tmp0 ^ in5;
+        out4 = tmp1 ^ in0;
+        out6 = tmp2 ^ in4;
+        out1 = out3 ^ out4 ^ in7;
+        tmp3 = out4 ^ in1 ^ in3;
+        out7 = tmp3 ^ out1;
+        out2 = tmp3 ^ in5;
+        out5 = tmp1 ^ out2;
+        out0 = tmp2 ^ tmp3;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_38(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0, tmp1, tmp2;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        out3 = in0 ^ in3;
+        tmp0 = in3 ^ in4;
+        tmp1 = in5 ^ in7;
+        tmp2 = out3 ^ in1;
+        out2 = tmp0 ^ in6;
+        out0 = tmp0 ^ tmp1;
+        out4 = tmp1 ^ tmp2;
+        out7 = out2 ^ in2;
+        out1 = out2 ^ in3 ^ in5;
+        out6 = out4 ^ in0 ^ in2;
+        out5 = tmp2 ^ out7;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_39(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0, tmp1, tmp2;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        out3 = in0;
+        tmp0 = in1 ^ in5;
+        tmp1 = tmp0 ^ in4;
+        out1 = tmp1 ^ in6;
+        out5 = out1 ^ in0 ^ in2;
+        tmp2 = tmp0 ^ out5;
+        out2 = tmp2 ^ in0 ^ in3;
+        out7 = out2 ^ in7;
+        out6 = tmp1 ^ out7;
+        out4 = tmp2 ^ out6;
+        out0 = out4 ^ in1;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_3A(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0, tmp1, tmp2, tmp3, tmp4, tmp5;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        tmp0 = in0 ^ in1;
+        tmp1 = in0 ^ in2;
+        tmp2 = in3 ^ in4;
+        tmp3 = in1 ^ in6;
+        tmp4 = in3 ^ in7;
+        out4 = tmp0 ^ in5;
+        out5 = tmp1 ^ tmp3;
+        out3 = tmp1 ^ tmp4;
+        out0 = tmp2 ^ in5;
+        out7 = tmp2 ^ in2;
+        tmp5 = tmp3 ^ in4;
+        out2 = tmp4 ^ tmp5;
+        out1 = tmp5 ^ out4;
+        out6 = tmp0 ^ out3;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_3B(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0, tmp1, tmp2;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        tmp0 = in1 ^ in6;
+        tmp1 = in2 ^ in7;
+        tmp2 = tmp0 ^ in3;
+        out3 = tmp1 ^ in0;
+        out6 = tmp1 ^ tmp2;
+        out2 = out6 ^ in4;
+        out7 = tmp0 ^ out2;
+        out0 = out3 ^ out7 ^ in5;
+        out5 = out0 ^ out2 ^ in7;
+        out1 = tmp2 ^ out0;
+        out4 = out1 ^ in6;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_3C(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0, tmp1, tmp2;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        tmp0 = in0 ^ in3;
+        tmp1 = in2 ^ in7;
+        tmp2 = in1 ^ in6 ^ in7;
+        out2 = tmp0 ^ in4;
+        out3 = tmp0 ^ tmp2;
+        out4 = tmp1 ^ out3 ^ in5;
+        out5 = tmp2 ^ out2 ^ in2;
+        out1 = out4 ^ out5 ^ in6;
+        out0 = out1 ^ in3;
+        out7 = tmp1 ^ out0;
+        out6 = tmp2 ^ out7;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_3D(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0, tmp1, tmp2;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        tmp0 = in0 ^ in2;
+        tmp1 = tmp0 ^ in3;
+        out2 = tmp1 ^ in4;
+        tmp2 = out2 ^ in5;
+        out4 = tmp2 ^ in1 ^ in6;
+        out5 = out4 ^ in7;
+        out6 = out5 ^ in0;
+        out7 = out6 ^ in1;
+        out0 = tmp0 ^ out7;
+        out1 = tmp1 ^ out5;
+        out3 = tmp2 ^ out6;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_3E(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0, tmp1;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        tmp0 = in3 ^ in5;
+        tmp1 = tmp0 ^ in4;
+        out0 = tmp1 ^ in6;
+        out7 = tmp1 ^ in2;
+        out6 = out7 ^ in1 ^ in5 ^ in7;
+        out2 = out6 ^ in0 ^ in2;
+        out4 = out0 ^ out6 ^ in0;
+        out5 = tmp0 ^ out4;
+        out3 = out5 ^ in7;
+        out1 = out3 ^ out6 ^ in5;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_3F(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0, tmp1, tmp2;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        tmp0 = in0 ^ in1;
+        out3 = tmp0 ^ in2 ^ in6;
+        tmp1 = out3 ^ in5 ^ in7;
+        out4 = tmp1 ^ in4;
+        out5 = tmp1 ^ in3;
+        out1 = out4 ^ in2;
+        out7 = out1 ^ out3 ^ in3;
+        out2 = tmp0 ^ out7 ^ in5;
+        tmp2 = out2 ^ in0;
+        out6 = tmp2 ^ in6;
+        out0 = tmp1 ^ tmp2;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_40(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0, tmp1;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        out1 = in3 ^ in7;
+        tmp0 = in3 ^ in4;
+        tmp1 = in6 ^ in7;
+        out4 = tmp0 ^ in2;
+        out5 = tmp0 ^ in5;
+        out0 = tmp1 ^ in2;
+        out7 = tmp1 ^ in1 ^ in5;
+        out2 = out0 ^ in4;
+        out3 = out2 ^ out5 ^ in7;
+        out6 = out3 ^ out4 ^ in0;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_41(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0, tmp1;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        out4 = in2 ^ in3;
+        tmp0 = in5 ^ in6;
+        tmp1 = in6 ^ in7;
+        out5 = in3 ^ in4;
+        out1 = in1 ^ in3 ^ in7;
+        out6 = in0 ^ in4 ^ in5;
+        out3 = tmp0 ^ in2;
+        out7 = tmp0 ^ in1;
+        out2 = tmp1 ^ in4;
+        out0 = tmp1 ^ in0 ^ in2;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_42(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        out0 = in2 ^ in6;
+        out5 = in3 ^ in5;
+        out1 = in0 ^ in3 ^ in7;
+        out7 = in1 ^ in5 ^ in7;
+        out4 = in2 ^ in4 ^ in7;
+        out6 = in0 ^ in4 ^ in6;
+        out2 = out0 ^ in1 ^ in4;
+        out3 = out5 ^ in6 ^ in7;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_43(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        out5 = in3;
+        out7 = in1 ^ in5;
+        out4 = in2 ^ in7;
+        out6 = in0 ^ in4;
+        out0 = in0 ^ in2 ^ in6;
+        out3 = in5 ^ in6 ^ in7;
+        out2 = in1 ^ in4 ^ in6;
+        out1 = in0 ^ in1 ^ in3 ^ in7;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_44(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        out1 = in3;
+        out0 = in2 ^ in7;
+        tmp0 = in4 ^ in7;
+        out7 = in1 ^ in6 ^ in7;
+        out6 = in0 ^ in5 ^ in6;
+        out4 = tmp0 ^ in3 ^ in6;
+        out3 = out0 ^ in1 ^ in3 ^ in5;
+        out2 = out0 ^ in0 ^ in4;
+        out5 = tmp0 ^ in5;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_45(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        out1 = in1 ^ in3;
+        out7 = in1 ^ in6;
+        out5 = in4 ^ in7;
+        out6 = in0 ^ in5;
+        out0 = in0 ^ in2 ^ in7;
+        out4 = in3 ^ in6 ^ in7;
+        out2 = out5 ^ in0;
+        out3 = out0 ^ out6 ^ in1;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_46(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        out0 = in2;
+        out1 = in0 ^ in3;
+        out7 = in1 ^ in7;
+        out4 = in4 ^ in6;
+        out5 = in5 ^ in7;
+        out6 = in0 ^ in6;
+        out3 = in1 ^ in3 ^ in5;
+        out2 = out4 ^ out6 ^ in1 ^ in2;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_47(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        out4 = in6;
+        out7 = in1;
+        out5 = in7;
+        out6 = in0;
+        tmp0 = in0 ^ in1;
+        out3 = in1 ^ in5;
+        out0 = in0 ^ in2;
+        out1 = tmp0 ^ in3;
+        out2 = tmp0 ^ in4;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_48(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0, tmp1;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        tmp0 = in2 ^ in3;
+        out1 = in3 ^ in6 ^ in7;
+        out3 = tmp0 ^ in0;
+        out0 = tmp0 ^ out1 ^ in5;
+        tmp1 = out0 ^ in4;
+        out2 = tmp1 ^ in7;
+        out5 = tmp1 ^ in3;
+        out4 = out5 ^ in1;
+        out7 = tmp0 ^ out4;
+        out6 = tmp1 ^ out3;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_49(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0, tmp1;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        out3 = in0 ^ in2;
+        tmp0 = in2 ^ in5;
+        out2 = in4 ^ in5 ^ in6;
+        tmp1 = tmp0 ^ out2 ^ in3;
+        out7 = out2 ^ in1;
+        out5 = tmp1 ^ in7;
+        out4 = out5 ^ out7 ^ in6;
+        out1 = tmp0 ^ out4;
+        out6 = out1 ^ out7 ^ in0;
+        out0 = tmp1 ^ out6;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_4A(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0, tmp1;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        tmp0 = in2 ^ in6;
+        tmp1 = in3 ^ in7;
+        out0 = tmp0 ^ in5;
+        out3 = tmp1 ^ in0;
+        out5 = tmp1 ^ out0;
+        out4 = out0 ^ in1 ^ in4;
+        out1 = out3 ^ in6;
+        out2 = out4 ^ in7;
+        out6 = out1 ^ in4;
+        out7 = tmp0 ^ out2;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_4B(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0, tmp1, tmp2, tmp3;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        out3 = in0 ^ in7;
+        tmp0 = in1 ^ in5;
+        tmp1 = in2 ^ in6;
+        tmp2 = out3 ^ in3;
+        out7 = tmp0 ^ in4;
+        out4 = tmp0 ^ tmp1;
+        tmp3 = tmp1 ^ in0;
+        out6 = tmp2 ^ in4;
+        out5 = tmp2 ^ tmp3;
+        out1 = tmp2 ^ in1 ^ in6;
+        out2 = out7 ^ in6 ^ in7;
+        out0 = tmp3 ^ in5;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_4C(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0, tmp1, tmp2;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        out1 = in3 ^ in6;
+        tmp0 = in2 ^ in5;
+        tmp1 = out1 ^ in5 ^ in7;
+        out0 = tmp0 ^ in7;
+        tmp2 = tmp0 ^ in4;
+        out6 = tmp1 ^ in0;
+        out2 = tmp2 ^ in0;
+        out5 = tmp2 ^ in6;
+        out3 = tmp0 ^ out6 ^ in1;
+        out7 = out0 ^ out5 ^ in1;
+        out4 = tmp1 ^ out7;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_4D(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0, tmp1, tmp2;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        tmp0 = in0 ^ in5;
+        tmp1 = in1 ^ in6;
+        out4 = in1 ^ in3 ^ in5;
+        tmp2 = tmp0 ^ in7;
+        out2 = tmp0 ^ in4;
+        out1 = tmp1 ^ in3;
+        out7 = tmp1 ^ in4;
+        out0 = tmp2 ^ in2;
+        out6 = tmp2 ^ in3;
+        out5 = out7 ^ in1 ^ in2;
+        out3 = tmp1 ^ out0 ^ in5;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_4E(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        out0 = in2 ^ in5;
+        out7 = in1 ^ in4 ^ in7;
+        out1 = in0 ^ in3 ^ in6;
+        out5 = out0 ^ in6;
+        out4 = out7 ^ in5;
+        out3 = out1 ^ in1;
+        out6 = out1 ^ in7;
+        out2 = out4 ^ in0 ^ in2;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_4F(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        out5 = in2 ^ in6;
+        out7 = in1 ^ in4;
+        out3 = in0 ^ in1 ^ in6;
+        out4 = in1 ^ in5 ^ in7;
+        out0 = in0 ^ in2 ^ in5;
+        out6 = in0 ^ in3 ^ in7;
+        out1 = out3 ^ in3;
+        out2 = out4 ^ in0 ^ in4;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_50(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0, tmp1, tmp2;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        out2 = in2 ^ in7;
+        tmp0 = in3 ^ in5;
+        out0 = out2 ^ in4 ^ in6;
+        out1 = tmp0 ^ in7;
+        tmp1 = tmp0 ^ in6;
+        out3 = out0 ^ in3;
+        out7 = tmp1 ^ in1;
+        tmp2 = tmp1 ^ in0;
+        out5 = out3 ^ in1 ^ in2;
+        out4 = tmp2 ^ in2;
+        out6 = tmp2 ^ out3;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_51(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        out2 = in7;
+        out3 = in2 ^ in4 ^ in6 ^ in7;
+        out0 = out3 ^ in0;
+        out6 = out0 ^ in5;
+        out4 = out6 ^ in3 ^ in7;
+        out1 = out0 ^ out4 ^ in1;
+        out7 = out1 ^ in6;
+        out5 = out7 ^ in4;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_52(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0, tmp1, tmp2, tmp3;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        out2 = in1 ^ in2;
+        tmp0 = in2 ^ in4;
+        tmp1 = in3 ^ in5;
+        tmp2 = in3 ^ in6;
+        tmp3 = in0 ^ in7;
+        out0 = tmp0 ^ in6;
+        out6 = tmp0 ^ tmp3;
+        out7 = tmp1 ^ in1;
+        out1 = tmp1 ^ tmp3;
+        out3 = tmp2 ^ in4;
+        out5 = tmp2 ^ in1 ^ in7;
+        out4 = tmp2 ^ out1 ^ in2;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_53(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        out2 = in1;
+        out3 = in4 ^ in6;
+        out0 = out3 ^ in0 ^ in2;
+        out6 = out0 ^ in7;
+        out4 = out6 ^ in5;
+        out7 = out0 ^ out4 ^ in1 ^ in3;
+        out1 = out7 ^ in0;
+        out5 = out7 ^ in6;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_54(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0, tmp1, tmp2, tmp3;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        out1 = in3 ^ in5;
+        tmp0 = in1 ^ in3;
+        tmp1 = in2 ^ in4;
+        tmp2 = in0 ^ in7;
+        out5 = in1 ^ in4 ^ in6;
+        out4 = tmp2 ^ out1;
+        out7 = tmp0 ^ in6;
+        out3 = tmp0 ^ tmp1;
+        out0 = tmp1 ^ in7;
+        tmp3 = tmp2 ^ in2;
+        out2 = tmp3 ^ in6;
+        out6 = tmp3 ^ in5;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_55(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0, tmp1, tmp2;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        tmp0 = in1 ^ in3;
+        tmp1 = in1 ^ in4;
+        tmp2 = in6 ^ in7;
+        out7 = tmp0 ^ tmp2;
+        out1 = tmp0 ^ in5;
+        out3 = tmp1 ^ in2;
+        out5 = tmp1 ^ in5 ^ in6;
+        out2 = tmp2 ^ in0;
+        out4 = out5 ^ out7 ^ in0;
+        out6 = out2 ^ in2 ^ in5;
+        out0 = out5 ^ out6 ^ in1;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_56(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        out0 = in2 ^ in4;
+        tmp0 = in0 ^ in2;
+        out4 = in0 ^ in5;
+        out7 = in1 ^ in3;
+        out5 = in1 ^ in6;
+        out6 = tmp0 ^ in7;
+        out2 = tmp0 ^ out5;
+        out1 = out4 ^ in3;
+        out3 = out7 ^ in4 ^ in7;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_57(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0, tmp1;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        tmp0 = in0 ^ in5;
+        tmp1 = in1 ^ in7;
+        out0 = in0 ^ in2 ^ in4;
+        out5 = in1 ^ in5 ^ in6;
+        out4 = tmp0 ^ in4;
+        out1 = tmp0 ^ in1 ^ in3;
+        out2 = tmp0 ^ out5;
+        out3 = tmp1 ^ in4;
+        out7 = tmp1 ^ in3;
+        out6 = tmp1 ^ out2 ^ in2;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_58(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0, tmp1;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        out2 = in2 ^ in5;
+        tmp0 = in2 ^ in3 ^ in4;
+        out5 = tmp0 ^ in1;
+        out6 = tmp0 ^ in0 ^ in5;
+        out3 = out6 ^ in7;
+        tmp1 = out2 ^ out5;
+        out7 = tmp1 ^ in6;
+        out4 = tmp1 ^ out3 ^ in3;
+        out0 = out4 ^ out7 ^ in0;
+        out1 = tmp0 ^ out0;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_59(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0, tmp1;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        out2 = in5;
+        tmp0 = in0 ^ in5 ^ in7;
+        out3 = tmp0 ^ in2 ^ in4;
+        out0 = out3 ^ in6;
+        tmp1 = out0 ^ in7;
+        out6 = tmp1 ^ in3;
+        out5 = out6 ^ in0 ^ in1 ^ in6;
+        out4 = tmp0 ^ out5;
+        out1 = tmp1 ^ out4;
+        out7 = out1 ^ in4;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_5A(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0, tmp1, tmp2;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        tmp0 = in1 ^ in2;
+        tmp1 = in2 ^ in5;
+        out5 = tmp0 ^ in3;
+        out4 = tmp0 ^ in0;
+        tmp2 = tmp1 ^ in4;
+        out2 = tmp1 ^ in1 ^ in7;
+        out7 = tmp2 ^ out5;
+        out6 = out4 ^ out7 ^ in5;
+        out0 = tmp2 ^ in6;
+        out1 = out0 ^ out6 ^ in7;
+        out3 = tmp1 ^ out6;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_5B(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0, tmp1, tmp2, tmp3, tmp4;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        tmp0 = in2 ^ in3;
+        tmp1 = in0 ^ in4;
+        tmp2 = in1 ^ in5;
+        out5 = tmp0 ^ tmp2;
+        tmp3 = tmp1 ^ in6;
+        out3 = tmp1 ^ in5;
+        out2 = tmp2 ^ in7;
+        tmp4 = out3 ^ in2;
+        out7 = out2 ^ in3 ^ in4;
+        out0 = tmp4 ^ in6;
+        out6 = tmp0 ^ tmp3;
+        out4 = tmp2 ^ tmp4;
+        out1 = tmp3 ^ out7;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_5C(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0, tmp1, tmp2;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        tmp0 = in3 ^ in6;
+        tmp1 = in0 ^ in2 ^ in5;
+        out1 = tmp0 ^ in5;
+        tmp2 = tmp0 ^ in1;
+        out2 = tmp1 ^ in6;
+        out6 = tmp1 ^ in3;
+        out4 = tmp2 ^ in0;
+        out7 = tmp2 ^ in4;
+        out3 = tmp1 ^ out7;
+        out0 = out3 ^ out4 ^ in7;
+        out5 = out0 ^ in1 ^ in5;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_5D(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0, tmp1, tmp2, tmp3, tmp4;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        tmp0 = in0 ^ in1;
+        tmp1 = in0 ^ in6;
+        out2 = tmp1 ^ in5;
+        tmp2 = out2 ^ in3;
+        out6 = tmp2 ^ in2;
+        out1 = tmp0 ^ tmp2;
+        tmp3 = out1 ^ in4 ^ in5;
+        out4 = tmp3 ^ in0;
+        out7 = tmp3 ^ in7;
+        tmp4 = out4 ^ out6;
+        out5 = tmp4 ^ in7;
+        out0 = tmp0 ^ out5;
+        out3 = tmp1 ^ tmp4;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_5E(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0, tmp1, tmp2, tmp3, tmp4;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        tmp0 = in2 ^ in5;
+        tmp1 = in3 ^ in5;
+        tmp2 = in1 ^ in7;
+        out7 = in1 ^ in3 ^ in4;
+        out0 = tmp0 ^ in4;
+        tmp3 = tmp1 ^ in0;
+        out5 = tmp2 ^ in2;
+        out1 = tmp3 ^ in6;
+        out6 = tmp0 ^ tmp3;
+        tmp4 = tmp2 ^ out1;
+        out3 = tmp4 ^ in4;
+        out4 = tmp1 ^ tmp4;
+        out2 = tmp0 ^ out4;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_5F(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0, tmp1, tmp2, tmp3;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        tmp0 = in1 ^ in5;
+        tmp1 = in0 ^ in6;
+        tmp2 = tmp0 ^ in7;
+        tmp3 = tmp1 ^ in3;
+        out2 = tmp1 ^ tmp2;
+        out5 = tmp2 ^ in2;
+        out6 = tmp3 ^ in2;
+        out3 = out2 ^ in4;
+        out4 = out3 ^ in5;
+        out1 = tmp0 ^ tmp3;
+        out7 = tmp3 ^ out4;
+        out0 = out4 ^ out5 ^ in6;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_60(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0, tmp1;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        out4 = in2 ^ in5;
+        tmp0 = in3 ^ in6;
+        out1 = in3 ^ in4 ^ in7;
+        out7 = out4 ^ in1;
+        tmp1 = out4 ^ in4;
+        out0 = tmp0 ^ in2;
+        out5 = tmp0 ^ in0;
+        out2 = tmp0 ^ tmp1;
+        out3 = tmp1 ^ in7;
+        out6 = out3 ^ out7 ^ in0;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_61(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0, tmp1;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        tmp0 = in2 ^ in5;
+        out4 = tmp0 ^ in4;
+        tmp1 = out4 ^ in3;
+        out3 = tmp1 ^ in7;
+        out2 = tmp1 ^ in2 ^ in6;
+        out1 = tmp0 ^ out3 ^ in1;
+        out0 = out2 ^ out4 ^ in0;
+        out7 = tmp1 ^ out1;
+        out6 = out0 ^ out1 ^ in2;
+        out5 = tmp0 ^ out0;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_62(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0, tmp1, tmp2, tmp3;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        out3 = in4 ^ in5;
+        tmp0 = in0 ^ in3 ^ in4;
+        out1 = tmp0 ^ in7;
+        out5 = tmp0 ^ in6;
+        tmp1 = out1 ^ in0;
+        tmp2 = tmp1 ^ out3;
+        out4 = tmp2 ^ in2;
+        tmp3 = tmp2 ^ in1;
+        out0 = out4 ^ in5 ^ in6;
+        out7 = tmp3 ^ out0;
+        out6 = tmp0 ^ tmp3;
+        out2 = tmp1 ^ out7;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_63(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0, tmp1, tmp2, tmp3, tmp4;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        tmp0 = in3 ^ in4;
+        tmp1 = in1 ^ in7;
+        out3 = tmp0 ^ in5;
+        tmp2 = out3 ^ in6;
+        out4 = out3 ^ in2 ^ in7;
+        out5 = tmp2 ^ in0;
+        tmp3 = out5 ^ in3;
+        out0 = tmp3 ^ out4;
+        out2 = tmp1 ^ tmp2;
+        out6 = tmp1 ^ tmp3;
+        tmp4 = tmp0 ^ out2;
+        out1 = tmp4 ^ out5;
+        out7 = tmp4 ^ in2;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_64(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0, tmp1;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        out0 = in2 ^ in3;
+        out1 = in3 ^ in4;
+        out7 = in1 ^ in2;
+        tmp0 = in4 ^ in5;
+        tmp1 = in0 ^ in7;
+        out4 = in5 ^ in6 ^ in7;
+        out2 = tmp0 ^ out0 ^ in0;
+        out3 = tmp0 ^ out7 ^ in6;
+        out5 = tmp1 ^ in6;
+        out6 = tmp1 ^ in1;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_65(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0, tmp1, tmp2, tmp3;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        tmp0 = in0 ^ in3;
+        tmp1 = in4 ^ in5;
+        tmp2 = in6 ^ in7;
+        out7 = in1 ^ in2 ^ in7;
+        out1 = in1 ^ in3 ^ in4;
+        out0 = tmp0 ^ in2;
+        out2 = tmp0 ^ tmp1;
+        out4 = tmp1 ^ tmp2;
+        tmp3 = tmp2 ^ in0;
+        out3 = out4 ^ out7 ^ in3;
+        out5 = tmp3 ^ in5;
+        out6 = tmp3 ^ in1;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_66(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0, tmp1, tmp2, tmp3, tmp4;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        tmp0 = in1 ^ in2;
+        tmp1 = in2 ^ in3;
+        tmp2 = in0 ^ in4;
+        out7 = tmp0 ^ in6;
+        out0 = tmp1 ^ in7;
+        out1 = tmp2 ^ in3;
+        tmp3 = tmp2 ^ in6;
+        tmp4 = out1 ^ in5;
+        out5 = tmp3 ^ in7;
+        out4 = tmp3 ^ tmp4;
+        out2 = tmp0 ^ tmp4 ^ in7;
+        out6 = tmp1 ^ out2 ^ in4;
+        out3 = tmp3 ^ out6;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_67(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0, tmp1, tmp2, tmp3;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        tmp0 = in0 ^ in3;
+        tmp1 = tmp0 ^ in1;
+        tmp2 = tmp0 ^ in7;
+        out1 = tmp1 ^ in4;
+        out0 = tmp2 ^ in2;
+        tmp3 = out1 ^ in7;
+        out2 = tmp3 ^ in5;
+        out3 = out2 ^ in0 ^ in6;
+        out7 = tmp1 ^ out0 ^ in6;
+        out5 = tmp1 ^ out3;
+        out4 = tmp2 ^ out5;
+        out6 = tmp3 ^ out4;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_68(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0, tmp1, tmp2, tmp3;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        tmp0 = in3 ^ in4;
+        tmp1 = in2 ^ in3 ^ in5;
+        tmp2 = tmp0 ^ in1;
+        tmp3 = tmp0 ^ in6;
+        out0 = tmp1 ^ in6;
+        out6 = tmp2 ^ in0;
+        out7 = tmp1 ^ tmp2;
+        out1 = tmp3 ^ in7;
+        out2 = out1 ^ in2;
+        out4 = tmp2 ^ out2;
+        out3 = out4 ^ out6 ^ in3;
+        out5 = tmp3 ^ out3;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_69(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        tmp0 = in6 ^ in7;
+        out2 = tmp0 ^ in3 ^ in4;
+        out1 = out2 ^ in1;
+        out3 = out2 ^ in0 ^ in2;
+        out4 = out1 ^ in2 ^ in3;
+        out6 = out1 ^ in0 ^ in7;
+        out7 = out4 ^ in5 ^ in6;
+        out5 = out4 ^ out6 ^ in5;
+        out0 = tmp0 ^ out5;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_6A(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0, tmp1, tmp2;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        tmp0 = in2 ^ in6;
+        out3 = in0 ^ in4 ^ in6;
+        tmp1 = tmp0 ^ in3;
+        out4 = tmp1 ^ in1;
+        tmp2 = tmp1 ^ in7;
+        out2 = out4 ^ in4;
+        out0 = tmp2 ^ in5;
+        out5 = tmp2 ^ out3;
+        out7 = out2 ^ in3 ^ in5;
+        out1 = tmp0 ^ out5;
+        out6 = tmp1 ^ out7 ^ in0;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_6B(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0, tmp1;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        tmp0 = in4 ^ in6;
+        out2 = tmp0 ^ in1 ^ in3;
+        out4 = out2 ^ in2;
+        tmp1 = out2 ^ in0;
+        out7 = out4 ^ in3 ^ in5 ^ in7;
+        out1 = tmp1 ^ in7;
+        out3 = tmp1 ^ in1;
+        out6 = tmp1 ^ in5;
+        out0 = tmp1 ^ out7 ^ in6;
+        out5 = tmp0 ^ out0;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_6C(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0, tmp1;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        out4 = in1;
+        tmp0 = in2 ^ in3;
+        out5 = in0 ^ in2;
+        out1 = in3 ^ in4 ^ in6;
+        tmp1 = out5 ^ in1;
+        out0 = tmp0 ^ in5;
+        out6 = tmp0 ^ tmp1;
+        out3 = tmp1 ^ in4;
+        out7 = out3 ^ in0;
+        out2 = out6 ^ out7 ^ in7;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_6D(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0, tmp1;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        out4 = in1 ^ in4;
+        tmp0 = in0 ^ in2;
+        tmp1 = out4 ^ in3;
+        out7 = out4 ^ in2 ^ in7;
+        out5 = tmp0 ^ in5;
+        out3 = tmp0 ^ tmp1;
+        out1 = tmp1 ^ in6;
+        out0 = out5 ^ in3;
+        out2 = out3 ^ out7 ^ in4;
+        out6 = out1 ^ in0 ^ in4;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_6E(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0, tmp1, tmp2;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        tmp0 = in1 ^ in3;
+        tmp1 = in0 ^ in4;
+        out4 = tmp0 ^ in7;
+        out6 = tmp0 ^ in0 ^ in5;
+        out5 = tmp1 ^ in2;
+        tmp2 = tmp1 ^ in3;
+        out3 = tmp2 ^ out4;
+        out1 = tmp2 ^ in6;
+        out2 = tmp0 ^ out5;
+        out0 = out2 ^ out3 ^ in5;
+        out7 = out1 ^ out2 ^ in4;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_6F(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0, tmp1, tmp2;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        tmp0 = in3 ^ in7;
+        tmp1 = tmp0 ^ in4;
+        tmp2 = tmp0 ^ in0 ^ in2;
+        out4 = tmp1 ^ in1;
+        out0 = tmp2 ^ in5;
+        out3 = out4 ^ in0;
+        out2 = out3 ^ in7;
+        out1 = out2 ^ in6;
+        out6 = out1 ^ in4 ^ in5;
+        out7 = tmp2 ^ out1;
+        out5 = tmp1 ^ out0;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_70(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0, tmp1, tmp2;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        out3 = in2;
+        tmp0 = in2 ^ in4;
+        out2 = in2 ^ in3 ^ in5;
+        tmp1 = tmp0 ^ in6;
+        tmp2 = out2 ^ in7;
+        out0 = tmp1 ^ in3;
+        out4 = tmp1 ^ in0;
+        out7 = tmp2 ^ in1;
+        out6 = out4 ^ in1;
+        out5 = out7 ^ in0 ^ in2;
+        out1 = tmp0 ^ tmp2;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_71(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0, tmp1, tmp2;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        out2 = in3 ^ in5;
+        out3 = in2 ^ in3;
+        tmp0 = in0 ^ in2;
+        tmp1 = out2 ^ in1;
+        out4 = tmp0 ^ in6;
+        tmp2 = tmp0 ^ in1;
+        out7 = tmp1 ^ in2;
+        out1 = tmp1 ^ in4 ^ in7;
+        out0 = out4 ^ in3 ^ in4;
+        out6 = tmp2 ^ in4;
+        out5 = tmp2 ^ out3 ^ in7;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_72(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0, tmp1, tmp2;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        out3 = in7;
+        tmp0 = in0 ^ in4;
+        tmp1 = tmp0 ^ in3 ^ in7;
+        out1 = tmp1 ^ in5;
+        out5 = out1 ^ in1;
+        tmp2 = tmp0 ^ out5;
+        out2 = tmp2 ^ in2;
+        out7 = out2 ^ in6;
+        out6 = tmp1 ^ out7;
+        out4 = tmp2 ^ out6;
+        out0 = out4 ^ in0;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_73(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        out3 = in3 ^ in7;
+        out2 = out3 ^ in1 ^ in5;
+        out1 = out2 ^ in0 ^ in4;
+        out5 = out1 ^ in5;
+        out6 = out1 ^ out3 ^ in2;
+        out0 = out2 ^ out6 ^ in6;
+        out7 = out0 ^ out1 ^ in3;
+        out4 = out0 ^ in4;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_74(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0, tmp1;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        tmp0 = in3 ^ in4;
+        tmp1 = in1 ^ in2 ^ in6;
+        out4 = in0 ^ in4 ^ in7;
+        out5 = in0 ^ in1 ^ in5;
+        out0 = tmp0 ^ in2;
+        out1 = tmp0 ^ in5;
+        out3 = tmp1 ^ in7;
+        out6 = tmp1 ^ in0;
+        out2 = tmp1 ^ out5 ^ in3;
+        out7 = out3 ^ in3 ^ in6;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_75(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0, tmp1, tmp2;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        out4 = in0 ^ in7;
+        tmp0 = in1 ^ in3;
+        out5 = in0 ^ in1;
+        out7 = tmp0 ^ in2;
+        tmp1 = tmp0 ^ in4;
+        out6 = out5 ^ in2;
+        tmp2 = out7 ^ in6;
+        out1 = tmp1 ^ in5;
+        out0 = tmp1 ^ out6;
+        out3 = tmp2 ^ in7;
+        out2 = tmp2 ^ out6 ^ in5;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_76(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0, tmp1, tmp2, tmp3;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        out3 = in1 ^ in6;
+        tmp0 = in0 ^ in5;
+        tmp1 = in3 ^ in7;
+        tmp2 = tmp0 ^ in4;
+        tmp3 = tmp1 ^ in2;
+        out5 = tmp2 ^ in1;
+        out1 = tmp2 ^ in3;
+        out0 = tmp3 ^ in4;
+        out4 = out1 ^ in5;
+        out7 = tmp3 ^ out3;
+        out2 = tmp0 ^ out7;
+        out6 = tmp1 ^ out2;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_77(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0, tmp1, tmp2, tmp3;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        out4 = in0 ^ in3;
+        tmp0 = in1 ^ in4;
+        tmp1 = in1 ^ in6;
+        tmp2 = out4 ^ in5;
+        out5 = tmp0 ^ in0;
+        out1 = tmp0 ^ tmp2;
+        out3 = tmp1 ^ in3;
+        out2 = tmp1 ^ tmp2 ^ in7;
+        out7 = out3 ^ in2;
+        tmp3 = out7 ^ in6;
+        out6 = tmp2 ^ tmp3;
+        out0 = tmp3 ^ out5 ^ in7;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_78(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0, tmp1, tmp2;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        tmp0 = in0 ^ in3;
+        tmp1 = in2 ^ in7;
+        tmp2 = in0 ^ in5 ^ in6;
+        out2 = tmp1 ^ in3;
+        out3 = tmp2 ^ in2;
+        out5 = out3 ^ in1 ^ in3;
+        out0 = tmp0 ^ out3 ^ in4;
+        out1 = tmp1 ^ out0;
+        out4 = out1 ^ out5 ^ in5;
+        out7 = tmp0 ^ out4;
+        out6 = tmp2 ^ out7;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_79(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0, tmp1, tmp2, tmp3;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        out2 = in3 ^ in7;
+        tmp0 = in3 ^ in4;
+        tmp1 = in1 ^ in5;
+        tmp2 = tmp1 ^ in2;
+        out4 = tmp2 ^ in0 ^ in7;
+        tmp3 = out4 ^ in5;
+        out5 = tmp3 ^ out2 ^ in6;
+        out7 = tmp0 ^ tmp2;
+        out6 = tmp0 ^ tmp3;
+        out3 = tmp1 ^ out5;
+        out0 = out3 ^ in4;
+        out1 = tmp3 ^ out0;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_7A(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0, tmp1;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        tmp0 = in1 ^ in2;
+        out2 = tmp0 ^ in3;
+        tmp1 = out2 ^ in4;
+        out4 = tmp1 ^ in0 ^ in5;
+        out5 = out4 ^ in6;
+        out6 = out5 ^ in7;
+        out7 = out6 ^ in0;
+        out0 = out7 ^ in1;
+        out1 = tmp0 ^ out6;
+        out3 = tmp1 ^ out6;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_7B(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0, tmp1, tmp2;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        out2 = in1 ^ in3;
+        tmp0 = in0 ^ in5;
+        out4 = tmp0 ^ out2 ^ in2;
+        tmp1 = out4 ^ in4;
+        out6 = tmp1 ^ in7;
+        out5 = tmp1 ^ in5 ^ in6;
+        out0 = out6 ^ in1 ^ in6;
+        tmp2 = out0 ^ in2;
+        out1 = tmp2 ^ in1;
+        out3 = tmp2 ^ in4;
+        out7 = tmp0 ^ out5;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_7C(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0, tmp1;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        tmp0 = in3 ^ in5;
+        tmp1 = tmp0 ^ in4;
+        out0 = tmp1 ^ in2;
+        out1 = tmp1 ^ in6;
+        out7 = out0 ^ in1 ^ in5 ^ in7;
+        out5 = out1 ^ out7 ^ in0;
+        out3 = out5 ^ in6;
+        out6 = tmp0 ^ out5;
+        out2 = out6 ^ in1;
+        out4 = out2 ^ out7 ^ in5;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_7D(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0, tmp1, tmp2, tmp3;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        tmp0 = in1 ^ in2;
+        tmp1 = tmp0 ^ in3;
+        tmp2 = tmp0 ^ in6;
+        out7 = tmp1 ^ in4;
+        tmp3 = tmp2 ^ in0;
+        out5 = tmp3 ^ in7;
+        out4 = tmp3 ^ in2 ^ in5;
+        out2 = tmp1 ^ out5;
+        out6 = tmp2 ^ out2;
+        out0 = out4 ^ out7 ^ in6;
+        out1 = tmp3 ^ out0;
+        out3 = out6 ^ in5;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_7E(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0, tmp1, tmp2;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        tmp0 = in3 ^ in4;
+        tmp1 = in0 ^ in5;
+        out1 = tmp0 ^ tmp1 ^ in6;
+        out3 = tmp1 ^ in1;
+        out4 = out1 ^ in1 ^ in7;
+        tmp2 = out4 ^ in3;
+        out5 = tmp2 ^ in2;
+        out6 = tmp0 ^ out5;
+        out7 = tmp1 ^ out4 ^ in2;
+        out2 = out6 ^ in5 ^ in7;
+        out0 = tmp2 ^ out2;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_7F(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0, tmp1, tmp2, tmp3;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        tmp0 = in2 ^ in7;
+        tmp1 = tmp0 ^ in3 ^ in5;
+        tmp2 = tmp1 ^ in0;
+        out0 = tmp2 ^ in4;
+        out6 = tmp2 ^ in1;
+        out3 = tmp0 ^ out6;
+        tmp3 = out3 ^ in6;
+        out1 = tmp3 ^ in4;
+        out2 = tmp3 ^ in5;
+        out4 = tmp3 ^ in7;
+        out5 = tmp1 ^ out1;
+        out7 = out0 ^ out4 ^ in3;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_80(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0, tmp1, tmp2;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        tmp0 = in2 ^ in3;
+        tmp1 = in4 ^ in5;
+        out1 = in2 ^ in6 ^ in7;
+        out5 = tmp0 ^ in4;
+        tmp2 = tmp0 ^ in1;
+        out6 = tmp1 ^ in3;
+        out7 = tmp1 ^ in0 ^ in6;
+        out4 = tmp2 ^ in7;
+        out3 = tmp2 ^ out6;
+        out2 = out3 ^ out5 ^ in6;
+        out0 = out2 ^ in3 ^ in7;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_81(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0, tmp1;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        tmp0 = in4 ^ in6;
+        tmp1 = tmp0 ^ in3;
+        out6 = tmp1 ^ in5;
+        out5 = out6 ^ in2 ^ in6;
+        out3 = out5 ^ in1;
+        out2 = tmp0 ^ out3;
+        out1 = out3 ^ out6 ^ in7;
+        out4 = tmp1 ^ out1;
+        out7 = out2 ^ out4 ^ in0;
+        out0 = out7 ^ in1 ^ in4;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_82(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        out4 = in1 ^ in2;
+        tmp0 = in6 ^ in7;
+        out5 = in2 ^ in3;
+        out6 = in3 ^ in4;
+        out7 = in0 ^ in4 ^ in5;
+        out0 = in1 ^ in5 ^ in6;
+        out1 = tmp0 ^ in0 ^ in2;
+        out2 = tmp0 ^ in3 ^ in5;
+        out3 = tmp0 ^ out0 ^ in4;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_83(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0, tmp1, tmp2, tmp3, tmp4;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        tmp0 = in0 ^ in1;
+        tmp1 = in2 ^ in5;
+        tmp2 = in3 ^ in6;
+        out4 = in1 ^ in2 ^ in4;
+        out0 = tmp0 ^ in5 ^ in6;
+        out5 = tmp1 ^ in3;
+        tmp3 = tmp1 ^ in7;
+        out6 = tmp2 ^ in4;
+        out2 = tmp2 ^ tmp3;
+        tmp4 = tmp3 ^ out4;
+        out1 = tmp3 ^ out0;
+        out3 = tmp4 ^ in3;
+        out7 = tmp0 ^ tmp4;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_84(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        out1 = in2 ^ in6;
+        out6 = in3 ^ in5;
+        out0 = in1 ^ in5 ^ in7;
+        out7 = in0 ^ in4 ^ in6;
+        out4 = in1 ^ in3 ^ in6;
+        out5 = in2 ^ in4 ^ in7;
+        out2 = out6 ^ in0 ^ in1;
+        out3 = out5 ^ in5 ^ in6;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_85(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0, tmp1, tmp2, tmp3;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        tmp0 = in1 ^ in6;
+        tmp1 = in3 ^ in6;
+        tmp2 = tmp0 ^ in4;
+        out1 = tmp0 ^ in2;
+        out6 = tmp1 ^ in5;
+        out4 = tmp2 ^ in3;
+        tmp3 = out1 ^ out6;
+        out2 = tmp3 ^ in0;
+        out3 = tmp2 ^ tmp3 ^ in7;
+        out7 = out2 ^ out3 ^ in1;
+        out5 = tmp1 ^ out3;
+        out0 = tmp2 ^ out7 ^ in5;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_86(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        out6 = in3;
+        out7 = in0 ^ in4;
+        out0 = in1 ^ in5;
+        out5 = in2 ^ in7;
+        out3 = in4 ^ in5 ^ in6;
+        out1 = in0 ^ in2 ^ in6;
+        out4 = in1 ^ in6 ^ in7;
+        out2 = in0 ^ in3 ^ in5 ^ in7;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_87(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0, tmp1;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        out6 = in3 ^ in6;
+        tmp0 = in0 ^ in1;
+        out7 = in0 ^ in4 ^ in7;
+        out5 = in2 ^ in5 ^ in7;
+        out3 = out6 ^ in4 ^ in5;
+        out0 = tmp0 ^ in5;
+        tmp1 = tmp0 ^ in6;
+        out2 = out5 ^ in0 ^ in3;
+        out1 = tmp1 ^ in2;
+        out4 = tmp1 ^ out7;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_88(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0, tmp1;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        out1 = in2 ^ in7;
+        tmp0 = in5 ^ in6;
+        out0 = in1 ^ in6 ^ in7;
+        out6 = in4 ^ in5 ^ in7;
+        out3 = out0 ^ out1 ^ in0 ^ in4;
+        out7 = tmp0 ^ in0;
+        tmp1 = tmp0 ^ in3;
+        out2 = out0 ^ in3;
+        out4 = tmp1 ^ in2;
+        out5 = tmp1 ^ out6;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_89(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0, tmp1, tmp2;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        tmp0 = in0 ^ in7;
+        tmp1 = in2 ^ in7;
+        tmp2 = tmp0 ^ in6;
+        out1 = tmp1 ^ in1;
+        out7 = tmp2 ^ in5;
+        out0 = tmp2 ^ in1;
+        out2 = out1 ^ in3 ^ in6;
+        out6 = out7 ^ in0 ^ in4;
+        out5 = out6 ^ in3;
+        out3 = tmp0 ^ out2 ^ in4;
+        out4 = tmp1 ^ out5;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_8A(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        out0 = in1 ^ in6;
+        out7 = in0 ^ in5;
+        out2 = in3 ^ in6;
+        out6 = in4 ^ in7;
+        out1 = in0 ^ in2 ^ in7;
+        out3 = out0 ^ out6 ^ in0;
+        out4 = out1 ^ out7 ^ in6;
+        out5 = out2 ^ in7;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_8B(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0, tmp1, tmp2, tmp3, tmp4;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        tmp0 = in0 ^ in1;
+        tmp1 = in3 ^ in6;
+        tmp2 = in5 ^ in7;
+        tmp3 = tmp0 ^ in7;
+        out0 = tmp0 ^ in6;
+        out2 = tmp1 ^ in2;
+        out5 = tmp1 ^ tmp2;
+        out7 = tmp2 ^ in0;
+        tmp4 = tmp3 ^ in4;
+        out1 = tmp3 ^ in2;
+        out6 = tmp4 ^ out0;
+        out4 = out6 ^ in2 ^ in5;
+        out3 = tmp1 ^ tmp4;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_8C(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        out1 = in2;
+        out0 = in1 ^ in7;
+        out7 = in0 ^ in6;
+        out5 = in4 ^ in6;
+        out6 = in5 ^ in7;
+        out2 = out0 ^ in0 ^ in3;
+        out3 = out5 ^ out7 ^ in2 ^ in7;
+        out4 = out6 ^ in3;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_8D(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        out1 = in1 ^ in2;
+        tmp0 = in6 ^ in7;
+        out0 = in0 ^ in1 ^ in7;
+        out5 = in4 ^ in5 ^ in6;
+        out6 = tmp0 ^ in5;
+        out7 = tmp0 ^ in0;
+        out4 = tmp0 ^ out5 ^ in3;
+        out2 = out0 ^ in2 ^ in3;
+        out3 = out2 ^ in1 ^ in4;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_8E(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        out0 = in1;
+        out4 = in5;
+        out7 = in0;
+        out5 = in6;
+        out6 = in7;
+        out3 = in0 ^ in4;
+        out1 = in0 ^ in2;
+        out2 = in0 ^ in3;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_8F(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        out0 = in0 ^ in1;
+        tmp0 = in0 ^ in3;
+        out4 = in4 ^ in5;
+        out7 = in0 ^ in7;
+        out5 = in5 ^ in6;
+        out6 = in6 ^ in7;
+        out1 = out0 ^ in2;
+        out2 = tmp0 ^ in2;
+        out3 = tmp0 ^ in4;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_90(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0, tmp1, tmp2;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        tmp0 = in1 ^ in2;
+        tmp1 = in2 ^ in6 ^ in7;
+        out3 = tmp0 ^ in7;
+        out1 = tmp1 ^ in5;
+        tmp2 = out1 ^ in4;
+        out6 = tmp2 ^ in3;
+        out5 = out6 ^ in1;
+        out4 = out5 ^ in0;
+        out0 = tmp0 ^ tmp2;
+        out7 = tmp0 ^ out4;
+        out2 = tmp1 ^ out5;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_91(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0, tmp1, tmp2, tmp3;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        tmp0 = in2 ^ in4;
+        tmp1 = tmp0 ^ in3 ^ in5;
+        out2 = tmp1 ^ in1;
+        out6 = tmp1 ^ in7;
+        tmp2 = out2 ^ in5 ^ in7;
+        out3 = tmp2 ^ in4;
+        out5 = tmp2 ^ in6;
+        out1 = tmp1 ^ out5 ^ in2;
+        tmp3 = out1 ^ in0;
+        out4 = tmp3 ^ in3;
+        out0 = tmp0 ^ tmp3;
+        out7 = tmp2 ^ tmp3;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_92(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0, tmp1;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        out3 = in1;
+        tmp0 = in4 ^ in5;
+        tmp1 = tmp0 ^ in1;
+        out2 = tmp0 ^ in3 ^ in7;
+        out0 = tmp1 ^ in6;
+        out7 = out2 ^ in0;
+        out4 = out0 ^ in0 ^ in2;
+        out5 = out4 ^ out7 ^ in5;
+        out6 = tmp1 ^ out5;
+        out1 = out6 ^ out7 ^ in7;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_93(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0, tmp1, tmp2;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        out3 = in1 ^ in3;
+        tmp0 = in2 ^ in7;
+        tmp1 = out3 ^ in6;
+        tmp2 = tmp0 ^ in4;
+        out5 = tmp0 ^ tmp1;
+        out6 = tmp2 ^ in3;
+        out2 = out6 ^ in5;
+        out0 = out2 ^ out5 ^ in0;
+        out7 = tmp1 ^ out0;
+        out1 = tmp2 ^ out0;
+        out4 = out1 ^ in7;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_94(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        out3 = in2 ^ in6;
+        tmp0 = in1 ^ in4 ^ in5;
+        out1 = out3 ^ in5;
+        out5 = tmp0 ^ out3;
+        out0 = tmp0 ^ in7;
+        out4 = tmp0 ^ in0 ^ in3;
+        out6 = out1 ^ in3 ^ in7;
+        out2 = out4 ^ in6;
+        out7 = out0 ^ out2 ^ in4;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_95(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0, tmp1, tmp2, tmp3, tmp4, tmp5;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        tmp0 = in2 ^ in3;
+        out3 = tmp0 ^ in6;
+        tmp1 = tmp0 ^ in7;
+        tmp2 = out3 ^ in0;
+        out6 = tmp1 ^ in5;
+        tmp3 = tmp2 ^ in4;
+        out7 = tmp3 ^ in2;
+        tmp4 = tmp3 ^ in5;
+        out2 = tmp4 ^ in1;
+        tmp5 = out2 ^ in6;
+        out0 = tmp1 ^ tmp5;
+        out1 = tmp5 ^ out7;
+        out4 = tmp2 ^ out1;
+        out5 = tmp4 ^ out4;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_96(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0, tmp1, tmp2;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        out3 = in6 ^ in7;
+        tmp0 = in1 ^ in5;
+        tmp1 = in5 ^ in6;
+        out6 = out3 ^ in2 ^ in3;
+        out0 = tmp0 ^ in4;
+        tmp2 = tmp1 ^ in2;
+        out4 = out0 ^ in0 ^ in7;
+        out1 = tmp2 ^ in0;
+        out5 = tmp2 ^ in1;
+        out7 = tmp0 ^ out4 ^ in3;
+        out2 = tmp1 ^ out7;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_97(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0, tmp1, tmp2, tmp3;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        tmp0 = in0 ^ in4;
+        tmp1 = in2 ^ in6;
+        out3 = in3 ^ in6 ^ in7;
+        out7 = tmp0 ^ in3;
+        tmp2 = tmp0 ^ in5;
+        out5 = tmp1 ^ in1;
+        out6 = tmp1 ^ out3;
+        out0 = tmp2 ^ in1;
+        out2 = tmp2 ^ out3 ^ in2;
+        tmp3 = out0 ^ in4;
+        out4 = tmp3 ^ in7;
+        out1 = tmp1 ^ tmp3;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_98(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0, tmp1;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        tmp0 = in5 ^ in7;
+        tmp1 = in1 ^ in4 ^ in7;
+        out1 = tmp0 ^ in2;
+        out0 = tmp1 ^ in6;
+        out2 = tmp1 ^ in3;
+        out6 = out0 ^ out1 ^ in1;
+        out5 = tmp0 ^ out2;
+        out3 = tmp1 ^ out6 ^ in0;
+        out7 = out0 ^ out5 ^ in0;
+        out4 = out6 ^ out7 ^ in7;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_99(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0, tmp1, tmp2;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        tmp0 = in0 ^ in3;
+        out5 = in1 ^ in3 ^ in4;
+        out6 = in2 ^ in4 ^ in5;
+        out4 = tmp0 ^ in2;
+        tmp1 = tmp0 ^ in6;
+        tmp2 = out5 ^ in7;
+        out7 = tmp1 ^ in5;
+        out0 = tmp1 ^ tmp2;
+        out2 = tmp2 ^ in2;
+        out3 = out0 ^ out6 ^ in3;
+        out1 = tmp1 ^ out3;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_9A(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0, tmp1, tmp2;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        out2 = in3 ^ in4;
+        tmp0 = in0 ^ in5;
+        tmp1 = in1 ^ in6;
+        out5 = in1 ^ in3 ^ in5;
+        tmp2 = tmp0 ^ in7;
+        out3 = tmp0 ^ tmp1;
+        out0 = tmp1 ^ in4;
+        out7 = tmp2 ^ in3;
+        out1 = tmp2 ^ in2;
+        out6 = out0 ^ in1 ^ in2;
+        out4 = out1 ^ in4 ^ in5;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_9B(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        out5 = in1 ^ in3;
+        tmp0 = in3 ^ in5;
+        out6 = in2 ^ in4;
+        out4 = in0 ^ in2 ^ in7;
+        out7 = tmp0 ^ in0;
+        out2 = out6 ^ in3;
+        out1 = out4 ^ in1 ^ in5;
+        out3 = out7 ^ in1 ^ in6;
+        out0 = tmp0 ^ out3 ^ in4;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_9C(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        out1 = in2 ^ in5;
+        tmp0 = in0 ^ in3 ^ in6;
+        out3 = out1 ^ in0;
+        out6 = out1 ^ in6;
+        out7 = tmp0 ^ in7;
+        out4 = out7 ^ in4;
+        out2 = out4 ^ in1;
+        out0 = tmp0 ^ out2;
+        out5 = out0 ^ in5;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_9D(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        out6 = in2 ^ in5;
+        tmp0 = in0 ^ in3;
+        out5 = in1 ^ in4 ^ in7;
+        out1 = out6 ^ in1;
+        out3 = tmp0 ^ out6;
+        out7 = tmp0 ^ in6;
+        out0 = out5 ^ in0;
+        out4 = out7 ^ in7;
+        out2 = out5 ^ out7 ^ in2;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_9E(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        out0 = in1 ^ in4;
+        tmp0 = in0 ^ in5;
+        out6 = in2 ^ in6;
+        out7 = in0 ^ in3 ^ in7;
+        out4 = in0 ^ in4 ^ in6;
+        out5 = in1 ^ in5 ^ in7;
+        out1 = tmp0 ^ in2;
+        out3 = tmp0 ^ in7;
+        out2 = out4 ^ in3;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_9F(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        out6 = in2;
+        out7 = in0 ^ in3;
+        tmp0 = in0 ^ in1;
+        out4 = in0 ^ in6;
+        out5 = in1 ^ in7;
+        out1 = tmp0 ^ in2 ^ in5;
+        out2 = out7 ^ in2 ^ in4 ^ in6;
+        out3 = out7 ^ in5 ^ in7;
+        out0 = tmp0 ^ in4;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_A0(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0, tmp1, tmp2, tmp3;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        tmp0 = in1 ^ in6;
+        out2 = tmp0 ^ in7;
+        tmp1 = tmp0 ^ in5;
+        out6 = out2 ^ in3 ^ in4;
+        out0 = tmp1 ^ in3;
+        tmp2 = out0 ^ in2;
+        out3 = tmp2 ^ in7;
+        tmp3 = tmp2 ^ in1;
+        out5 = tmp3 ^ in0;
+        out4 = tmp3 ^ out6;
+        out7 = out5 ^ out6 ^ in1;
+        out1 = tmp1 ^ out4;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_A1(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0, tmp1, tmp2;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        tmp0 = in2 ^ in5;
+        tmp1 = tmp0 ^ in1;
+        tmp2 = tmp0 ^ in4;
+        out4 = tmp1 ^ in7;
+        out7 = tmp2 ^ in0;
+        out6 = tmp2 ^ out4 ^ in3;
+        out3 = out4 ^ in6;
+        out2 = out3 ^ in5;
+        out1 = out2 ^ in4;
+        out5 = out1 ^ out6 ^ in0;
+        out0 = tmp1 ^ out5;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_A2(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        out2 = in6;
+        tmp0 = in1 ^ in3 ^ in5;
+        out3 = tmp0 ^ in6;
+        out4 = tmp0 ^ in2 ^ in4;
+        out0 = out3 ^ in7;
+        out6 = out0 ^ in4;
+        out1 = out0 ^ out4 ^ in0;
+        out7 = out1 ^ in5;
+        out5 = out7 ^ in3 ^ in7;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_A3(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0, tmp1;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        out2 = in2 ^ in6;
+        out3 = in1 ^ in5 ^ in6;
+        tmp0 = out2 ^ in0;
+        out4 = out2 ^ out3 ^ in3;
+        tmp1 = tmp0 ^ in4;
+        out0 = tmp0 ^ out4 ^ in7;
+        out5 = tmp1 ^ in3;
+        out7 = tmp1 ^ in5;
+        out1 = tmp1 ^ in1 ^ in7;
+        out6 = tmp1 ^ out0 ^ in2;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_A4(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0, tmp1, tmp2, tmp3, tmp4;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        tmp0 = in1 ^ in3;
+        tmp1 = in2 ^ in4;
+        tmp2 = in2 ^ in5;
+        tmp3 = in0 ^ in7;
+        out0 = tmp0 ^ in5;
+        out6 = tmp0 ^ in6 ^ in7;
+        out1 = tmp1 ^ in6;
+        out7 = tmp1 ^ tmp3;
+        out3 = tmp2 ^ in3;
+        tmp4 = tmp2 ^ out1;
+        out2 = tmp3 ^ in1;
+        out5 = tmp4 ^ out7;
+        out4 = tmp4 ^ in1;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_A5(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0, tmp1, tmp2;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        out3 = in2 ^ in5;
+        tmp0 = in1 ^ in6;
+        tmp1 = in0 ^ in1;
+        tmp2 = in2 ^ in4;
+        out6 = in1 ^ in3 ^ in7;
+        out4 = tmp0 ^ in5;
+        out1 = tmp0 ^ tmp2;
+        out0 = tmp1 ^ in3 ^ in5;
+        out2 = tmp1 ^ in2 ^ in7;
+        out7 = tmp2 ^ in0;
+        out5 = tmp0 ^ out2;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_A6(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        out2 = in0;
+        out3 = in3 ^ in5 ^ in7;
+        out1 = in0 ^ in2 ^ in4 ^ in6;
+        out0 = out3 ^ in1;
+        out7 = out1 ^ in7;
+        out6 = out0 ^ in6;
+        out5 = out7 ^ in5;
+        out4 = out6 ^ in4;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_A7(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        out2 = in0 ^ in2;
+        out3 = in5 ^ in7;
+        out7 = out2 ^ in4 ^ in6;
+        out6 = out3 ^ in1 ^ in3;
+        out1 = out7 ^ in1;
+        out5 = out7 ^ in7;
+        out0 = out6 ^ in0;
+        out4 = out6 ^ in6;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_A8(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0, tmp1, tmp2;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        tmp0 = in2 ^ in4;
+        tmp1 = in1 ^ in6;
+        tmp2 = in0 ^ in2 ^ in7;
+        out1 = tmp0 ^ in7;
+        out4 = tmp0 ^ in6;
+        out0 = tmp1 ^ in3;
+        out2 = tmp1 ^ in5;
+        out6 = tmp1 ^ in4;
+        out7 = tmp2 ^ in5;
+        out3 = tmp2 ^ out0 ^ in6;
+        out5 = out7 ^ in2 ^ in3;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_A9(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        out4 = in2 ^ in6;
+        out6 = in1 ^ in4;
+        out7 = in0 ^ in2 ^ in5;
+        out5 = in0 ^ in3 ^ in7;
+        out2 = out4 ^ in1 ^ in5;
+        out1 = out6 ^ in2 ^ in7;
+        out0 = out2 ^ out7 ^ in3;
+        out3 = out1 ^ in0 ^ in4;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_AA(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0, tmp1, tmp2;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        tmp0 = in0 ^ in2;
+        tmp1 = in1 ^ in3;
+        tmp2 = in6 ^ in7;
+        out1 = tmp0 ^ in4 ^ in7;
+        out3 = tmp1 ^ in0;
+        out0 = tmp1 ^ tmp2;
+        out2 = tmp2 ^ in5;
+        out7 = tmp0 ^ out2;
+        out6 = out1 ^ out7 ^ in1;
+        out5 = out0 ^ out6 ^ in0;
+        out4 = out5 ^ out7 ^ in7;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_AB(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0, tmp1;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        out3 = in0 ^ in1;
+        tmp0 = in1 ^ in4;
+        tmp1 = in0 ^ in7;
+        out6 = tmp0 ^ in5;
+        out1 = tmp0 ^ tmp1 ^ in2;
+        out5 = tmp1 ^ in3 ^ in4;
+        out0 = tmp0 ^ out5 ^ in6;
+        out4 = out0 ^ out3 ^ in2;
+        out2 = out4 ^ in3 ^ in5;
+        out7 = tmp1 ^ out2;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_AC(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        out0 = in1 ^ in3;
+        out1 = in2 ^ in4;
+        tmp0 = in0 ^ in2;
+        out4 = in4 ^ in7;
+        out5 = in0 ^ in5;
+        out6 = in1 ^ in6;
+        out7 = tmp0 ^ in7;
+        out3 = tmp0 ^ in3 ^ in6;
+        out2 = out5 ^ in1;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_AD(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        out4 = in7;
+        out5 = in0;
+        out6 = in1;
+        out7 = in0 ^ in2;
+        out0 = in0 ^ in1 ^ in3;
+        out2 = out7 ^ in1 ^ in5;
+        out1 = in1 ^ in2 ^ in4;
+        out3 = out7 ^ in6;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_AE(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0, tmp1, tmp2;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        out4 = in3 ^ in4;
+        tmp0 = in0 ^ in4;
+        tmp1 = in0 ^ in7;
+        out0 = in1 ^ in3 ^ in7;
+        out1 = tmp0 ^ in2;
+        out5 = tmp0 ^ in5;
+        tmp2 = tmp1 ^ in6;
+        out2 = tmp1 ^ in5;
+        out3 = tmp2 ^ in3;
+        out7 = tmp2 ^ in2;
+        out6 = tmp2 ^ out2 ^ in1;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_AF(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        out4 = in3;
+        tmp0 = in0 ^ in7;
+        out5 = in0 ^ in4;
+        out6 = in1 ^ in5;
+        out7 = in0 ^ in2 ^ in6;
+        out0 = tmp0 ^ in1 ^ in3;
+        out3 = tmp0 ^ in6;
+        out2 = tmp0 ^ in2 ^ in5;
+        out1 = out5 ^ in1 ^ in2;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_B0(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0, tmp1, tmp2, tmp3;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        tmp0 = in1 ^ in4;
+        tmp1 = in3 ^ in6;
+        out2 = tmp0 ^ in7;
+        tmp2 = tmp0 ^ tmp1;
+        out0 = tmp2 ^ in5;
+        out3 = tmp2 ^ in2;
+        out6 = out3 ^ in6;
+        tmp3 = out6 ^ in0 ^ in1;
+        out7 = tmp3 ^ in5;
+        out5 = tmp3 ^ out2;
+        out1 = out0 ^ out5 ^ in0;
+        out4 = tmp1 ^ out5;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_B1(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0, tmp1;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        tmp0 = in1 ^ in4;
+        out2 = tmp0 ^ in2 ^ in7;
+        tmp1 = out2 ^ in6;
+        out1 = tmp1 ^ in5;
+        out3 = tmp1 ^ in7;
+        out4 = tmp1 ^ in0;
+        out6 = out3 ^ in3;
+        out0 = out6 ^ in0 ^ in2 ^ in5;
+        out5 = tmp1 ^ out0 ^ in1;
+        out7 = tmp0 ^ out5;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_B2(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0, tmp1, tmp2, tmp3, tmp4;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        out2 = in4;
+        tmp0 = in4 ^ in7;
+        tmp1 = in1 ^ in3 ^ in6;
+        out3 = tmp0 ^ tmp1;
+        tmp2 = tmp1 ^ in0;
+        out0 = out3 ^ in5;
+        out4 = tmp2 ^ in2;
+        tmp3 = out4 ^ in6;
+        out5 = tmp0 ^ tmp3;
+        out1 = tmp3 ^ out0;
+        tmp4 = out1 ^ in7;
+        out7 = tmp4 ^ in3;
+        out6 = tmp2 ^ tmp4;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_B3(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0, tmp1, tmp2;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        out2 = in2 ^ in4;
+        tmp0 = in0 ^ in5;
+        tmp1 = in1 ^ in6;
+        out3 = tmp1 ^ in4 ^ in7;
+        tmp2 = tmp0 ^ out3;
+        out0 = tmp2 ^ in3;
+        out1 = tmp2 ^ in2;
+        out5 = out0 ^ in2 ^ in6;
+        out7 = tmp1 ^ out5;
+        out4 = out7 ^ in1 ^ in5 ^ in7;
+        out6 = tmp0 ^ out4;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_B4(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        out4 = in0 ^ in1;
+        out5 = out4 ^ in2;
+        tmp0 = out4 ^ in4;
+        out6 = out5 ^ in0 ^ in3;
+        out7 = tmp0 ^ out6;
+        out2 = tmp0 ^ in6 ^ in7;
+        out3 = out7 ^ in0 ^ in7;
+        out0 = out5 ^ out7 ^ in5;
+        out1 = out0 ^ out6 ^ in6;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_B5(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0, tmp1, tmp2;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        tmp0 = in0 ^ in1;
+        tmp1 = in2 ^ in4;
+        out4 = tmp0 ^ in4;
+        out3 = tmp1 ^ in7;
+        tmp2 = out4 ^ in5;
+        out7 = out3 ^ in0 ^ in3;
+        out0 = tmp2 ^ in3;
+        out2 = tmp0 ^ out3 ^ in6;
+        out5 = tmp1 ^ tmp2;
+        out6 = out2 ^ out7 ^ in2;
+        out1 = tmp0 ^ out0 ^ out6;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_B6(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0, tmp1, tmp2, tmp3;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        out3 = in3 ^ in4;
+        tmp0 = in1 ^ in2;
+        tmp1 = in0 ^ in4;
+        tmp2 = in3 ^ in5;
+        tmp3 = out3 ^ in1 ^ in7;
+        out5 = tmp0 ^ tmp1;
+        out6 = tmp0 ^ tmp2;
+        out2 = tmp1 ^ in6;
+        out4 = tmp1 ^ tmp3;
+        out0 = tmp3 ^ in5;
+        out1 = out2 ^ in2 ^ in5;
+        out7 = tmp2 ^ out1;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_B7(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0, tmp1;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        out3 = in4;
+        tmp0 = in0 ^ in4;
+        out2 = tmp0 ^ in2 ^ in6;
+        tmp1 = out2 ^ in7;
+        out1 = out2 ^ in1 ^ in5;
+        out7 = tmp1 ^ in3;
+        out5 = out1 ^ in6;
+        out6 = tmp0 ^ out1 ^ in3;
+        out0 = tmp1 ^ out6;
+        out4 = out0 ^ in5;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_B8(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0, tmp1, tmp2;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        tmp0 = in1 ^ in4;
+        tmp1 = in2 ^ in5;
+        out2 = tmp0 ^ in5;
+        out4 = tmp1 ^ in0;
+        tmp2 = tmp1 ^ in7;
+        out6 = tmp2 ^ out2;
+        out7 = out4 ^ in3;
+        out1 = tmp2 ^ in4;
+        out3 = tmp0 ^ out7;
+        out0 = out3 ^ out4 ^ in6;
+        out5 = out0 ^ in0 ^ in4;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_B9(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0, tmp1, tmp2;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        tmp0 = in0 ^ in2;
+        tmp1 = in4 ^ in5;
+        out4 = tmp0 ^ tmp1;
+        tmp2 = tmp0 ^ in3 ^ in7;
+        out3 = out4 ^ in1;
+        out7 = tmp2 ^ in5;
+        out2 = out3 ^ in0;
+        out1 = out2 ^ in7;
+        out6 = out1 ^ in5 ^ in6;
+        out0 = tmp2 ^ out6;
+        out5 = tmp1 ^ out0;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_BA(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0, tmp1, tmp2;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        tmp0 = in5 ^ in7;
+        out2 = tmp0 ^ in4;
+        tmp1 = out2 ^ in2;
+        out1 = tmp1 ^ in0;
+        out6 = tmp1 ^ in1;
+        out4 = out1 ^ in3 ^ in4;
+        tmp2 = out4 ^ out6;
+        out7 = out4 ^ in6 ^ in7;
+        out5 = tmp2 ^ in6;
+        out3 = tmp0 ^ tmp2;
+        out0 = out6 ^ out7 ^ in0;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_BB(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0, tmp1;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        out2 = in2 ^ in4 ^ in5 ^ in7;
+        tmp0 = out2 ^ in1;
+        out4 = out2 ^ in0 ^ in3;
+        out1 = tmp0 ^ in0;
+        out6 = tmp0 ^ in6;
+        out3 = out1 ^ in2;
+        tmp1 = out4 ^ out6 ^ in4;
+        out0 = tmp1 ^ in7;
+        out5 = tmp1 ^ in5;
+        out7 = tmp0 ^ tmp1;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_BC(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0, tmp1, tmp2;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        tmp0 = in0 ^ in2;
+        tmp1 = in2 ^ in4;
+        out0 = in1 ^ in3 ^ in4;
+        out6 = in1 ^ in2 ^ in7;
+        out7 = tmp0 ^ in3;
+        out5 = tmp0 ^ out6 ^ in6;
+        out1 = tmp1 ^ in5;
+        tmp2 = out1 ^ out5 ^ in1;
+        out3 = tmp2 ^ in3;
+        out4 = tmp1 ^ tmp2;
+        out2 = tmp2 ^ out6;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_BD(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0, tmp1, tmp2;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        tmp0 = in0 ^ in3;
+        tmp1 = in1 ^ in4;
+        out0 = tmp0 ^ tmp1;
+        out7 = tmp0 ^ in2 ^ in7;
+        out1 = tmp1 ^ in2 ^ in5;
+        tmp2 = out1 ^ in0;
+        out2 = tmp2 ^ in6;
+        out3 = out2 ^ in1 ^ in7;
+        out4 = out3 ^ in2;
+        out5 = tmp1 ^ out4;
+        out6 = tmp2 ^ out4;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_BE(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        tmp0 = in0 ^ in3 ^ in6;
+        out4 = tmp0 ^ in5;
+        out7 = tmp0 ^ in2;
+        out3 = out4 ^ in4;
+        out1 = out3 ^ out7 ^ in0;
+        out2 = out3 ^ in3 ^ in7;
+        out0 = out2 ^ out4 ^ in1;
+        out5 = tmp0 ^ out0;
+        out6 = out1 ^ out5 ^ in6;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_BF(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0, tmp1, tmp2, tmp3;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        tmp0 = in0 ^ in4;
+        out3 = tmp0 ^ in5 ^ in6;
+        out4 = out3 ^ in3;
+        tmp1 = out3 ^ in7;
+        out2 = tmp1 ^ in2;
+        out5 = tmp1 ^ in1;
+        tmp2 = out2 ^ in5;
+        out7 = tmp2 ^ in3 ^ in4;
+        tmp3 = tmp0 ^ out5;
+        out0 = tmp3 ^ out4;
+        out1 = tmp2 ^ tmp3;
+        out6 = tmp3 ^ in2;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_C0(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0, tmp1;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        out5 = in2 ^ in5;
+        tmp0 = in1 ^ in4;
+        tmp1 = in3 ^ in6;
+        out0 = out5 ^ in1;
+        out4 = tmp0 ^ in7;
+        out3 = tmp0 ^ tmp1;
+        out1 = tmp1 ^ in2;
+        out6 = tmp1 ^ in0;
+        out7 = out4 ^ in0;
+        out2 = out4 ^ out5 ^ in3;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_C1(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0, tmp1;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        out5 = in2;
+        tmp0 = in0 ^ in1;
+        out4 = in1 ^ in7;
+        out6 = in0 ^ in3;
+        out3 = in1 ^ in4 ^ in6;
+        tmp1 = tmp0 ^ in2;
+        out7 = tmp0 ^ in4;
+        out0 = tmp1 ^ in5;
+        out1 = tmp1 ^ out6 ^ in6;
+        out2 = out6 ^ out7 ^ in5 ^ in7;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_C2(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0, tmp1;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        out4 = in1 ^ in3 ^ in4;
+        tmp0 = in0 ^ in3 ^ in6;
+        out5 = in2 ^ in4 ^ in5;
+        tmp1 = out4 ^ in7;
+        out1 = tmp0 ^ in2;
+        out6 = tmp0 ^ in5;
+        out2 = out5 ^ in3;
+        out7 = tmp0 ^ tmp1;
+        out3 = tmp1 ^ in2 ^ in6;
+        out0 = tmp1 ^ out2;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_C3(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0, tmp1, tmp2;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        out4 = in1 ^ in3;
+        tmp0 = in0 ^ in2;
+        tmp1 = in3 ^ in5;
+        out5 = in2 ^ in4;
+        tmp2 = tmp0 ^ out4;
+        out2 = tmp1 ^ in4;
+        out6 = tmp1 ^ in0;
+        out0 = tmp1 ^ tmp2 ^ in7;
+        out1 = tmp2 ^ in6;
+        out7 = out1 ^ out5 ^ in3;
+        out3 = tmp0 ^ out7 ^ in7;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_C4(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0, tmp1;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        tmp0 = in3 ^ in7;
+        out3 = tmp0 ^ in4;
+        tmp1 = tmp0 ^ in2;
+        out1 = tmp1 ^ in6;
+        out5 = tmp1 ^ in5;
+        out4 = out1 ^ out3 ^ in1;
+        out0 = out4 ^ in4 ^ in5;
+        out2 = out0 ^ out3 ^ in0;
+        out7 = out1 ^ out2 ^ in7;
+        out6 = tmp1 ^ out0 ^ out7;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_C5(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        out3 = in4 ^ in7;
+        tmp0 = in3 ^ in7;
+        out4 = in1 ^ in2 ^ in6;
+        out6 = in0 ^ in3 ^ in4;
+        out5 = tmp0 ^ in2;
+        out1 = tmp0 ^ out4;
+        out0 = out4 ^ in0 ^ in5;
+        out2 = out0 ^ out5 ^ in4;
+        out7 = tmp0 ^ out2 ^ in6;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_C6(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0, tmp1, tmp2, tmp3, tmp4, tmp5;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        tmp0 = in5 ^ in6;
+        tmp1 = in1 ^ in7;
+        tmp2 = tmp0 ^ in0;
+        tmp3 = tmp0 ^ tmp1;
+        tmp4 = tmp2 ^ in4;
+        out0 = tmp3 ^ in2;
+        out6 = tmp4 ^ in3;
+        out2 = out6 ^ in2;
+        out7 = tmp1 ^ tmp4;
+        out3 = tmp2 ^ out2;
+        tmp5 = out3 ^ in5;
+        out5 = tmp5 ^ in7;
+        out4 = tmp3 ^ tmp5;
+        out1 = tmp4 ^ out5;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_C7(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0, tmp1;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        out3 = in2 ^ in4;
+        tmp0 = in3 ^ in5;
+        tmp1 = out3 ^ in7;
+        out6 = tmp0 ^ in0 ^ in4;
+        out5 = tmp1 ^ in3;
+        out2 = out6 ^ in6;
+        out7 = out2 ^ in1 ^ in3;
+        out0 = tmp1 ^ out7;
+        out1 = tmp0 ^ out0;
+        out4 = out1 ^ in0;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_C8(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0, tmp1;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        out0 = in1 ^ in2;
+        out1 = in2 ^ in3;
+        tmp0 = in5 ^ in6;
+        tmp1 = in0 ^ in7;
+        out2 = out1 ^ in1 ^ in4;
+        out4 = tmp0 ^ in4;
+        out5 = tmp0 ^ in7;
+        out6 = tmp1 ^ in6;
+        out7 = tmp1 ^ in1;
+        out3 = out2 ^ in0 ^ in2 ^ in5;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_C9(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        out4 = in5 ^ in6;
+        out7 = in0 ^ in1;
+        tmp0 = in1 ^ in3;
+        out5 = in6 ^ in7;
+        out6 = in0 ^ in7;
+        out0 = out7 ^ in2;
+        out3 = out7 ^ in4 ^ in5;
+        out1 = tmp0 ^ in2;
+        out2 = tmp0 ^ in4;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_CA(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0, tmp1, tmp2, tmp3;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        tmp0 = in0 ^ in7;
+        tmp1 = in2 ^ in7;
+        tmp2 = tmp0 ^ in6;
+        out0 = tmp1 ^ in1;
+        tmp3 = tmp1 ^ in3;
+        out6 = tmp2 ^ in5;
+        out7 = tmp2 ^ in1;
+        out2 = tmp3 ^ in4;
+        out5 = out6 ^ in0 ^ in4;
+        out4 = out5 ^ in3;
+        out1 = tmp0 ^ tmp3;
+        out3 = tmp3 ^ out5 ^ out7;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_CB(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0, tmp1, tmp2;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        tmp0 = in4 ^ in7;
+        tmp1 = in5 ^ in7;
+        out7 = in0 ^ in1 ^ in6;
+        out5 = tmp0 ^ in6;
+        out2 = tmp0 ^ in3;
+        out6 = tmp1 ^ in0;
+        out4 = tmp1 ^ in3 ^ in6;
+        tmp2 = out5 ^ out7 ^ in2;
+        out1 = tmp2 ^ out2;
+        out0 = tmp2 ^ in4;
+        out3 = tmp2 ^ in5;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_CC(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0, tmp1, tmp2, tmp3;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        tmp0 = in3 ^ in5;
+        tmp1 = in1 ^ in6;
+        out1 = in2 ^ in3 ^ in7;
+        out5 = tmp0 ^ in6;
+        out0 = tmp1 ^ in2;
+        tmp2 = out5 ^ in0 ^ in7;
+        out3 = tmp2 ^ in4;
+        out6 = tmp0 ^ out3;
+        out7 = tmp1 ^ tmp2 ^ in3;
+        tmp3 = out1 ^ out6;
+        out4 = tmp2 ^ tmp3;
+        out2 = tmp3 ^ in1;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_CD(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0, tmp1;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        out5 = in3 ^ in6;
+        tmp0 = in0 ^ in1;
+        tmp1 = in2 ^ in7;
+        out6 = in0 ^ in4 ^ in7;
+        out2 = tmp0 ^ out5 ^ in4;
+        out7 = tmp0 ^ in5;
+        out0 = tmp0 ^ in2 ^ in6;
+        out4 = tmp1 ^ in5;
+        out1 = tmp1 ^ in1 ^ in3;
+        out3 = out6 ^ in5 ^ in6;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_CE(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0, tmp1, tmp2;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        tmp0 = in2 ^ in5;
+        tmp1 = tmp0 ^ in3;
+        out4 = tmp1 ^ in4;
+        tmp2 = out4 ^ in6;
+        out3 = tmp2 ^ in0;
+        out5 = tmp2 ^ in2;
+        out2 = out3 ^ in5 ^ in7;
+        out6 = tmp1 ^ out2;
+        out7 = out2 ^ out4 ^ in1;
+        out1 = tmp2 ^ out6;
+        out0 = tmp0 ^ out7 ^ in0;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_CF(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0, tmp1, tmp2;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        tmp0 = in3 ^ in6;
+        tmp1 = in0 ^ in1 ^ in5;
+        out4 = in2 ^ in3 ^ in5;
+        out5 = tmp0 ^ in4;
+        out7 = tmp1 ^ in6;
+        out1 = tmp1 ^ out4 ^ in7;
+        tmp2 = out5 ^ in0;
+        out2 = tmp2 ^ in7;
+        out3 = tmp2 ^ out4;
+        out6 = tmp0 ^ out2 ^ in5;
+        out0 = tmp0 ^ out1;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_D0(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0, tmp1, tmp2, tmp3, tmp4;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        tmp0 = in0 ^ in3;
+        tmp1 = in1 ^ in4;
+        tmp2 = in2 ^ in5;
+        out7 = tmp0 ^ tmp1;
+        out0 = tmp1 ^ tmp2;
+        tmp3 = tmp2 ^ in3;
+        out1 = tmp3 ^ in6;
+        tmp4 = out1 ^ in1;
+        out2 = tmp4 ^ in7;
+        out3 = out2 ^ in2;
+        out4 = tmp0 ^ out3;
+        out5 = tmp3 ^ out3;
+        out6 = tmp4 ^ out4;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_D1(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0, tmp1, tmp2;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        tmp0 = in3 ^ in5 ^ in6;
+        tmp1 = tmp0 ^ in1;
+        out1 = tmp1 ^ in2;
+        out2 = tmp1 ^ in7;
+        out3 = out2 ^ in3;
+        out5 = out3 ^ in2;
+        tmp2 = out3 ^ in0;
+        out4 = tmp2 ^ in4;
+        out7 = tmp0 ^ out4;
+        out6 = tmp2 ^ out1 ^ in6;
+        out0 = out2 ^ out6 ^ in4;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_D2(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        tmp0 = in5 ^ in6;
+        out2 = tmp0 ^ in2 ^ in3;
+        out1 = out2 ^ in0;
+        out3 = out2 ^ in1;
+        out4 = out1 ^ in1 ^ in2;
+        out6 = out1 ^ in6 ^ in7;
+        out7 = out4 ^ in4 ^ in5;
+        out5 = out4 ^ out6 ^ in4;
+        out0 = tmp0 ^ out5;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_D3(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0, tmp1, tmp2, tmp3;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        out2 = in3 ^ in5 ^ in6;
+        tmp0 = out2 ^ in2;
+        tmp1 = tmp0 ^ in1;
+        out1 = tmp1 ^ in0;
+        out3 = tmp1 ^ in3;
+        out4 = out1 ^ in2 ^ in4;
+        tmp2 = out4 ^ in5;
+        out7 = tmp2 ^ in7;
+        out0 = tmp0 ^ out7;
+        tmp3 = out0 ^ in0;
+        out5 = tmp3 ^ in6;
+        out6 = tmp2 ^ tmp3;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_D4(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0, tmp1, tmp2;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        out3 = in3 ^ in5;
+        tmp0 = in1 ^ in5;
+        tmp1 = tmp0 ^ in2;
+        out4 = tmp1 ^ in0;
+        tmp2 = tmp1 ^ in6;
+        out2 = out4 ^ in3 ^ in7;
+        out0 = tmp2 ^ in4;
+        out5 = tmp2 ^ out3;
+        out1 = tmp0 ^ out5 ^ in7;
+        out6 = tmp0 ^ out2 ^ in4;
+        out7 = tmp1 ^ out6 ^ in7;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_D5(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0, tmp1, tmp2;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        out3 = in5;
+        tmp0 = in0 ^ in4;
+        tmp1 = tmp0 ^ in1 ^ in5;
+        out4 = tmp1 ^ in2;
+        out0 = out4 ^ in6;
+        tmp2 = tmp0 ^ out0;
+        out5 = tmp2 ^ in3;
+        out1 = out5 ^ in7;
+        out6 = tmp1 ^ out1;
+        out7 = tmp2 ^ out6;
+        out2 = out7 ^ in4;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_D6(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0, tmp1;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        tmp0 = in1 ^ in2 ^ in4 ^ in6;
+        out5 = tmp0 ^ in3;
+        out0 = tmp0 ^ in5 ^ in7;
+        out3 = out0 ^ out5 ^ in2;
+        tmp1 = out3 ^ in0;
+        out1 = tmp1 ^ in6;
+        out2 = tmp1 ^ in7;
+        out4 = tmp1 ^ in1;
+        out6 = tmp1 ^ in4;
+        out7 = tmp0 ^ out2;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_D7(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0, tmp1, tmp2;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        tmp0 = in0 ^ in3;
+        out3 = in2 ^ in5 ^ in7;
+        out2 = tmp0 ^ in5;
+        tmp1 = tmp0 ^ out3 ^ in1;
+        out1 = tmp1 ^ in6;
+        out4 = tmp1 ^ in4;
+        tmp2 = out1 ^ in4;
+        out6 = tmp2 ^ in1;
+        out7 = tmp2 ^ in2;
+        out0 = tmp2 ^ in3;
+        out5 = tmp2 ^ in0 ^ in7;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_D8(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0, tmp1;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        out4 = in0;
+        out5 = in1;
+        tmp0 = in1 ^ in2;
+        out6 = in0 ^ in2;
+        out0 = tmp0 ^ in4;
+        tmp1 = tmp0 ^ in3;
+        out7 = tmp1 ^ out6;
+        out2 = tmp1 ^ in6;
+        out3 = out7 ^ in7;
+        out1 = tmp1 ^ in1 ^ in5;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_D9(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        out4 = in0 ^ in4;
+        out5 = in1 ^ in5;
+        out2 = in1 ^ in3 ^ in6;
+        out3 = in0 ^ in1 ^ in7;
+        out6 = in0 ^ in2 ^ in6;
+        out0 = out4 ^ in1 ^ in2;
+        out1 = out5 ^ in2 ^ in3;
+        out7 = out3 ^ in3;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_DA(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0, tmp1;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        out5 = in1 ^ in4;
+        tmp0 = in2 ^ in7;
+        tmp1 = in0 ^ in2 ^ in3;
+        out0 = tmp0 ^ out5;
+        out4 = tmp0 ^ tmp1;
+        out2 = tmp0 ^ in3 ^ in6;
+        out1 = tmp1 ^ in5;
+        out3 = tmp1 ^ in1;
+        out6 = out1 ^ in3;
+        out7 = out3 ^ in2 ^ in6;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_DB(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0, tmp1, tmp2, tmp3, tmp4;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        tmp0 = in0 ^ in1;
+        tmp1 = in1 ^ in5;
+        tmp2 = in3 ^ in7;
+        out3 = tmp0 ^ in2;
+        out5 = tmp1 ^ in4;
+        out6 = tmp1 ^ out3 ^ in6;
+        out2 = tmp2 ^ in6;
+        tmp3 = tmp2 ^ in4;
+        tmp4 = out3 ^ in3;
+        out4 = tmp3 ^ in0;
+        out1 = tmp4 ^ in5;
+        out0 = tmp3 ^ tmp4;
+        out7 = tmp0 ^ out2;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_DC(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0, tmp1, tmp2, tmp3;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        tmp0 = in0 ^ in2;
+        tmp1 = in0 ^ in3;
+        out6 = tmp0 ^ in4;
+        tmp2 = tmp0 ^ in7;
+        out3 = tmp1 ^ in6;
+        tmp3 = tmp1 ^ in1;
+        out1 = tmp1 ^ tmp2 ^ in5;
+        out4 = tmp2 ^ in6;
+        out2 = tmp3 ^ in2;
+        out7 = tmp3 ^ in5;
+        out5 = tmp2 ^ out2;
+        out0 = out2 ^ out3 ^ in4;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_DD(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        out3 = in0 ^ in6;
+        out2 = in0 ^ in1 ^ in3;
+        out6 = out3 ^ in2 ^ in4;
+        out7 = out2 ^ in5 ^ in7;
+        out0 = out6 ^ in1;
+        out4 = out6 ^ in7;
+        out5 = out7 ^ in0;
+        out1 = out5 ^ in2;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_DE(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0, tmp1;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        tmp0 = in2 ^ in3 ^ in6;
+        tmp1 = in3 ^ in4 ^ in7;
+        out4 = tmp0 ^ in0;
+        out5 = tmp1 ^ in1;
+        out3 = out4 ^ in7;
+        out2 = out3 ^ in6;
+        out1 = out2 ^ in5;
+        out6 = tmp1 ^ out1;
+        out0 = tmp0 ^ out5;
+        out7 = out0 ^ out1 ^ in4;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_DF(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0, tmp1, tmp2;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        out2 = in0 ^ in3 ^ in7;
+        tmp0 = out2 ^ in1 ^ in5;
+        out1 = tmp0 ^ in2;
+        out7 = tmp0 ^ in6;
+        out5 = tmp0 ^ in0 ^ in4;
+        tmp1 = out1 ^ out5 ^ in6;
+        out4 = tmp1 ^ in3;
+        out6 = tmp1 ^ in5;
+        tmp2 = tmp1 ^ in7;
+        out0 = tmp2 ^ in1;
+        out3 = tmp2 ^ in4;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_E0(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0, tmp1;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        out3 = in1 ^ in7;
+        tmp0 = in2 ^ in4;
+        out4 = out3 ^ in3 ^ in5;
+        out2 = tmp0 ^ in1;
+        tmp1 = tmp0 ^ in6;
+        out0 = out4 ^ in2;
+        out6 = out4 ^ in0;
+        out1 = tmp1 ^ in3;
+        out5 = tmp1 ^ in0;
+        out7 = out5 ^ in1;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_E1(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0, tmp1, tmp2, tmp3;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        out2 = in1 ^ in4;
+        tmp0 = in1 ^ in7;
+        out3 = tmp0 ^ in3;
+        tmp1 = out3 ^ in5;
+        out4 = tmp1 ^ in4;
+        tmp2 = tmp1 ^ in0;
+        out0 = tmp2 ^ in2;
+        out6 = tmp2 ^ in6;
+        tmp3 = out0 ^ out4 ^ in6;
+        out5 = tmp3 ^ in5;
+        out7 = tmp0 ^ tmp3;
+        out1 = tmp2 ^ out5 ^ in7;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_E2(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        out3 = in1 ^ in2;
+        out4 = in1 ^ in5;
+        out2 = in2 ^ in4 ^ in7;
+        out5 = in0 ^ in2 ^ in6;
+        out0 = out3 ^ in3 ^ in5;
+        out7 = out3 ^ in0 ^ in4;
+        out6 = out2 ^ out7 ^ in3;
+        out1 = out5 ^ in3 ^ in4;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_E3(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0, tmp1, tmp2, tmp3, tmp4;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        out2 = in4 ^ in7;
+        tmp0 = in1 ^ in3;
+        out3 = tmp0 ^ in2;
+        tmp1 = out3 ^ in0;
+        out0 = tmp1 ^ in5;
+        tmp2 = tmp1 ^ in4;
+        out1 = tmp2 ^ in6;
+        tmp3 = tmp2 ^ in3;
+        out7 = tmp3 ^ in7;
+        out6 = out1 ^ out2 ^ in2;
+        tmp4 = tmp0 ^ out0;
+        out5 = tmp4 ^ in6;
+        out4 = tmp3 ^ tmp4;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_E4(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0, tmp1, tmp2;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        out3 = in6;
+        tmp0 = in0 ^ in4;
+        tmp1 = tmp0 ^ in2 ^ in6;
+        out2 = tmp1 ^ in1;
+        out7 = out2 ^ in5;
+        tmp2 = tmp0 ^ out7;
+        out4 = tmp2 ^ in3;
+        out0 = out4 ^ in7;
+        out6 = tmp1 ^ out0;
+        out5 = tmp2 ^ out6;
+        out1 = out5 ^ in0;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_E5(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0, tmp1, tmp2;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        out3 = in3 ^ in6;
+        tmp0 = in0 ^ in1;
+        tmp1 = in5 ^ in7;
+        out2 = tmp0 ^ in4 ^ in6;
+        tmp2 = tmp1 ^ out2;
+        out6 = tmp2 ^ in3;
+        out7 = tmp2 ^ in2;
+        out0 = out6 ^ in2 ^ in4;
+        out5 = out6 ^ in1 ^ in2;
+        out1 = tmp0 ^ out5 ^ in5;
+        out4 = tmp1 ^ out1;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_E6(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        out3 = in2 ^ in6 ^ in7;
+        out2 = out3 ^ in0 ^ in4;
+        out4 = out3 ^ in1 ^ in5;
+        out1 = out2 ^ in3;
+        out7 = out2 ^ out4 ^ in2;
+        out0 = out4 ^ in3 ^ in7;
+        out5 = out1 ^ in4;
+        out6 = out0 ^ out2 ^ in5;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_E7(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0, tmp1, tmp2, tmp3, tmp4;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        tmp0 = in2 ^ in3;
+        out3 = tmp0 ^ in6 ^ in7;
+        tmp1 = out3 ^ in0;
+        out5 = tmp1 ^ in5;
+        tmp2 = tmp1 ^ in4;
+        tmp3 = out5 ^ in7;
+        out1 = tmp2 ^ in1;
+        out0 = tmp3 ^ in1;
+        out6 = out1 ^ in2;
+        out2 = tmp0 ^ tmp2;
+        tmp4 = tmp3 ^ out6;
+        out4 = tmp4 ^ in6;
+        out7 = tmp4 ^ in0;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_E8(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0, tmp1, tmp2, tmp3;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        out4 = in3 ^ in6;
+        tmp0 = in4 ^ in7;
+        out1 = in2 ^ in3 ^ in4;
+        out5 = tmp0 ^ in0;
+        tmp1 = tmp0 ^ in1;
+        tmp2 = tmp1 ^ in5;
+        out0 = tmp1 ^ out1;
+        out2 = tmp2 ^ in2;
+        out6 = tmp2 ^ out5;
+        tmp3 = out6 ^ in6;
+        out3 = tmp3 ^ in7;
+        out7 = tmp3 ^ in2 ^ in5;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_E9(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0, tmp1, tmp2;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        tmp0 = in0 ^ in1;
+        tmp1 = in3 ^ in6;
+        tmp2 = tmp0 ^ in6;
+        out4 = tmp1 ^ in4;
+        out6 = tmp2 ^ in5;
+        out7 = tmp2 ^ in2 ^ in7;
+        out3 = out6 ^ in3 ^ in7;
+        out0 = tmp1 ^ out7;
+        out2 = out3 ^ out4 ^ in0;
+        out5 = tmp0 ^ out2;
+        out1 = out0 ^ out5 ^ in5;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_EA(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        out4 = in6 ^ in7;
+        out5 = in0 ^ in7;
+        out6 = in0 ^ in1;
+        out0 = in1 ^ in2 ^ in3;
+        out2 = in2 ^ in4 ^ in5;
+        out7 = out6 ^ in2;
+        out1 = out0 ^ out6 ^ in4;
+        out3 = out7 ^ in5 ^ in6;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_EB(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0, tmp1;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        out2 = in4 ^ in5;
+        tmp0 = in0 ^ in1;
+        out4 = in4 ^ in6 ^ in7;
+        out5 = in0 ^ in5 ^ in7;
+        out6 = tmp0 ^ in6;
+        tmp1 = tmp0 ^ in2;
+        out0 = tmp1 ^ in3;
+        out7 = tmp1 ^ in7;
+        out1 = out0 ^ in4;
+        out3 = out0 ^ in5 ^ in6;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_EC(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        out3 = in0 ^ in5;
+        out4 = in2 ^ in3 ^ in7;
+        out5 = in0 ^ in3 ^ in4;
+        out6 = out3 ^ in1 ^ in4;
+        out1 = out4 ^ in4;
+        out0 = out4 ^ in1 ^ in6;
+        out2 = out0 ^ out5 ^ in5;
+        out7 = out2 ^ in4 ^ in7;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_ED(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0, tmp1;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        tmp0 = in2 ^ in4;
+        tmp1 = in3 ^ in5;
+        out4 = tmp0 ^ in3 ^ in7;
+        out3 = tmp1 ^ in0;
+        out1 = out4 ^ in1;
+        out5 = out3 ^ in4;
+        out7 = out1 ^ out5 ^ in6;
+        out2 = tmp0 ^ out7;
+        out0 = tmp1 ^ out7;
+        out6 = out2 ^ in7;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_EE(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0, tmp1, tmp2, tmp3;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        out4 = in2;
+        tmp0 = in0 ^ in1;
+        out5 = in0 ^ in3;
+        tmp1 = tmp0 ^ in2;
+        out6 = tmp0 ^ in4;
+        tmp2 = tmp1 ^ out5;
+        out7 = tmp1 ^ in5;
+        out1 = tmp2 ^ out6 ^ in7;
+        out0 = tmp2 ^ in6;
+        tmp3 = out7 ^ in1;
+        out3 = tmp3 ^ in7;
+        out2 = tmp3 ^ in4 ^ in6;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_EF(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0, tmp1;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        out4 = in2 ^ in4;
+        tmp0 = in0 ^ in5;
+        tmp1 = in4 ^ in6;
+        out5 = tmp0 ^ in3;
+        out2 = tmp0 ^ tmp1;
+        out6 = tmp1 ^ in0 ^ in1;
+        out3 = out5 ^ in2 ^ in7;
+        out7 = out3 ^ in1 ^ in3;
+        out0 = out4 ^ out6 ^ in3;
+        out1 = tmp1 ^ out0 ^ in7;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_F0(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0, tmp1, tmp2, tmp3;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        tmp0 = in1 ^ in2;
+        tmp1 = in4 ^ in5;
+        out2 = tmp0 ^ in6;
+        out3 = tmp1 ^ in1;
+        tmp2 = tmp1 ^ in7;
+        out1 = out2 ^ out3 ^ in3;
+        tmp3 = tmp0 ^ tmp2;
+        out0 = tmp3 ^ in3;
+        out5 = tmp3 ^ in0;
+        out4 = out1 ^ out5 ^ in4;
+        out7 = out4 ^ in2;
+        out6 = tmp2 ^ out7;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_F1(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0, tmp1, tmp2, tmp3;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        out2 = in1 ^ in6;
+        tmp0 = in3 ^ in5;
+        out3 = tmp0 ^ in1 ^ in4;
+        tmp1 = out3 ^ in2;
+        out1 = tmp1 ^ in6;
+        tmp2 = tmp1 ^ in0;
+        tmp3 = out1 ^ in5;
+        out0 = tmp2 ^ in7;
+        out6 = tmp2 ^ in4;
+        out7 = tmp3 ^ in0;
+        out5 = tmp0 ^ out0;
+        out4 = tmp3 ^ out5 ^ in1;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_F2(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0, tmp1, tmp2, tmp3;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        tmp0 = in4 ^ in5;
+        out2 = in2 ^ in6 ^ in7;
+        tmp1 = tmp0 ^ in1;
+        tmp2 = tmp1 ^ in2;
+        out0 = tmp2 ^ in3;
+        out3 = tmp2 ^ in7;
+        out5 = out3 ^ in0 ^ in4;
+        tmp3 = tmp0 ^ out5;
+        out7 = tmp3 ^ in3;
+        out4 = tmp3 ^ out2;
+        out1 = out0 ^ out4 ^ in4;
+        out6 = tmp1 ^ out1;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_F3(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0, tmp1;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        out2 = in6 ^ in7;
+        tmp0 = in0 ^ in1;
+        out4 = tmp0 ^ in6;
+        tmp1 = tmp0 ^ in2;
+        out5 = tmp1 ^ in7;
+        out6 = tmp1 ^ in3;
+        out7 = out6 ^ in4;
+        out0 = out7 ^ in5;
+        out1 = out0 ^ in6;
+        out3 = out0 ^ in0 ^ in7;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_F4(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        out2 = in0 ^ in1 ^ in2;
+        tmp0 = out2 ^ in3;
+        out4 = tmp0 ^ in4;
+        out5 = out4 ^ in5;
+        out6 = out5 ^ in6;
+        out7 = out6 ^ in7;
+        out0 = out7 ^ in0;
+        out1 = out0 ^ in1;
+        out3 = tmp0 ^ out7;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_F5(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        out2 = in0 ^ in1;
+        tmp0 = out2 ^ in2;
+        out4 = tmp0 ^ in3;
+        out5 = out4 ^ in4;
+        out6 = out5 ^ in5;
+        out7 = out6 ^ in6;
+        out0 = out7 ^ in7;
+        out1 = out0 ^ in0;
+        out3 = tmp0 ^ out0;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_F6(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0, tmp1;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        tmp0 = in0 ^ in7;
+        out2 = tmp0 ^ in2;
+        out4 = out2 ^ in1 ^ in4;
+        out7 = out4 ^ in3 ^ in5;
+        out5 = out7 ^ in4 ^ in7;
+        out0 = tmp0 ^ out7 ^ in6;
+        tmp1 = out0 ^ in1;
+        out6 = out0 ^ in0 ^ in5;
+        out3 = tmp1 ^ in3;
+        out1 = tmp0 ^ tmp1;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_F7(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        out2 = in0 ^ in7;
+        tmp0 = out2 ^ in1;
+        out4 = tmp0 ^ in2;
+        out5 = out4 ^ in3 ^ in7;
+        out6 = out5 ^ in4;
+        out7 = out6 ^ in5;
+        out0 = out7 ^ in6;
+        out1 = out0 ^ in7;
+        out3 = tmp0 ^ out1;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_F8(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0, tmp1, tmp2;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        tmp0 = in0 ^ in4;
+        tmp1 = in3 ^ in5;
+        tmp2 = tmp0 ^ in6;
+        out4 = tmp0 ^ tmp1;
+        out1 = tmp1 ^ in2 ^ in4;
+        out3 = tmp2 ^ in1;
+        out5 = out3 ^ in5;
+        out7 = out1 ^ out5 ^ in7;
+        out6 = tmp1 ^ out7;
+        out0 = tmp2 ^ out7;
+        out2 = out6 ^ in0;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_F9(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0, tmp1, tmp2, tmp3, tmp4;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        tmp0 = in3 ^ in5;
+        tmp1 = in0 ^ in6;
+        out4 = tmp0 ^ in0;
+        tmp2 = tmp1 ^ in4;
+        tmp3 = tmp1 ^ in2;
+        out5 = tmp2 ^ in1;
+        out3 = out5 ^ in3;
+        tmp4 = tmp3 ^ out3;
+        out1 = tmp4 ^ in5;
+        out0 = tmp4 ^ in0 ^ in7;
+        out6 = tmp0 ^ out0 ^ in4;
+        out7 = tmp2 ^ tmp4;
+        out2 = tmp3 ^ out6;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_FA(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0, tmp1, tmp2, tmp3;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        tmp0 = in0 ^ in1;
+        tmp1 = tmp0 ^ in2;
+        tmp2 = tmp0 ^ in5;
+        tmp3 = tmp1 ^ in7;
+        out5 = tmp2 ^ in6;
+        out6 = tmp3 ^ in6;
+        out7 = tmp3 ^ in3;
+        out3 = out6 ^ in4;
+        out2 = tmp1 ^ out5;
+        out4 = out2 ^ out3 ^ in1;
+        out0 = out4 ^ out7 ^ in5;
+        out1 = tmp2 ^ out0;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_FB(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0, tmp1;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        out2 = in5 ^ in6;
+        tmp0 = in0 ^ in1;
+        out4 = in0 ^ in5 ^ in7;
+        out5 = tmp0 ^ in6;
+        tmp1 = tmp0 ^ in2;
+        out6 = tmp1 ^ in7;
+        out7 = tmp1 ^ in3;
+        out0 = out7 ^ in4;
+        out1 = out0 ^ in5;
+        out3 = out0 ^ in6 ^ in7;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_FC(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0, tmp1, tmp2, tmp3;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        tmp0 = in1 ^ in2;
+        tmp1 = in0 ^ in7;
+        out2 = tmp0 ^ tmp1 ^ in5;
+        out3 = tmp1 ^ in4;
+        tmp2 = out2 ^ in6;
+        out6 = tmp2 ^ in4;
+        out7 = tmp2 ^ in3;
+        out4 = out6 ^ in1 ^ in3;
+        tmp3 = out4 ^ in0;
+        out1 = tmp3 ^ in6;
+        out0 = tmp3 ^ in1 ^ in5;
+        out5 = tmp0 ^ out4;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_FD(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0, tmp1, tmp2, tmp3;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        tmp0 = in0 ^ in5;
+        tmp1 = in1 ^ in7;
+        out2 = tmp0 ^ tmp1;
+        out6 = out2 ^ in2 ^ in4;
+        tmp2 = out6 ^ in0;
+        out1 = tmp2 ^ in3;
+        out0 = tmp0 ^ out1 ^ in6;
+        out5 = out0 ^ in2;
+        tmp3 = out5 ^ in1;
+        out3 = tmp3 ^ in6;
+        out7 = tmp2 ^ tmp3;
+        out4 = tmp1 ^ out7;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_FE(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0, tmp1, tmp2, tmp3, tmp4;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        tmp0 = in0 ^ in2;
+        out2 = tmp0 ^ in5;
+        out3 = tmp0 ^ in4;
+        tmp1 = out3 ^ in6;
+        out4 = tmp1 ^ in5;
+        tmp2 = tmp1 ^ in1;
+        out6 = tmp2 ^ in7;
+        tmp3 = tmp2 ^ in0;
+        out0 = tmp3 ^ in3;
+        tmp4 = out0 ^ out4 ^ in7;
+        out5 = tmp4 ^ in6;
+        out7 = tmp4 ^ in2;
+        out1 = tmp3 ^ out5;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void
+gf8_muladd_FF(void *out, void *in)
+{
+    unsigned int i;
+    uint64_t *in_ptr = (uint64_t *)in;
+    uint64_t *out_ptr = (uint64_t *)out;
+
+    for (i = 0; i < WIDTH; i++) {
+        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
+        uint64_t tmp0, tmp1, tmp2, tmp3;
+
+        uint64_t in0 = out_ptr[0];
+        uint64_t in1 = out_ptr[WIDTH];
+        uint64_t in2 = out_ptr[WIDTH * 2];
+        uint64_t in3 = out_ptr[WIDTH * 3];
+        uint64_t in4 = out_ptr[WIDTH * 4];
+        uint64_t in5 = out_ptr[WIDTH * 5];
+        uint64_t in6 = out_ptr[WIDTH * 6];
+        uint64_t in7 = out_ptr[WIDTH * 7];
+
+        out2 = in0 ^ in5;
+        tmp0 = in4 ^ in7;
+        tmp1 = out2 ^ in2;
+        out4 = tmp1 ^ in6;
+        out7 = tmp1 ^ in1 ^ in3;
+        out1 = tmp0 ^ out7;
+        tmp2 = out1 ^ in5;
+        out6 = tmp2 ^ in3;
+        tmp3 = tmp2 ^ in7;
+        out0 = tmp3 ^ in6;
+        out3 = tmp3 ^ in1;
+        out5 = tmp0 ^ out0 ^ in2;
+
+        out_ptr[0] = out0 ^ in_ptr[0];
+        out_ptr[WIDTH] = out1 ^ in_ptr[WIDTH];
+        out_ptr[WIDTH * 2] = out2 ^ in_ptr[WIDTH * 2];
+        out_ptr[WIDTH * 3] = out3 ^ in_ptr[WIDTH * 3];
+        out_ptr[WIDTH * 4] = out4 ^ in_ptr[WIDTH * 4];
+        out_ptr[WIDTH * 5] = out5 ^ in_ptr[WIDTH * 5];
+        out_ptr[WIDTH * 6] = out6 ^ in_ptr[WIDTH * 6];
+        out_ptr[WIDTH * 7] = out7 ^ in_ptr[WIDTH * 7];
+
+        in_ptr++;
+        out_ptr++;
+    }
+}
+
+static void (*gf8_muladd[])(void *out, void *in) = {
+    gf8_muladd_00, gf8_muladd_01, gf8_muladd_02, gf8_muladd_03, gf8_muladd_04,
+    gf8_muladd_05, gf8_muladd_06, gf8_muladd_07, gf8_muladd_08, gf8_muladd_09,
+    gf8_muladd_0A, gf8_muladd_0B, gf8_muladd_0C, gf8_muladd_0D, gf8_muladd_0E,
+    gf8_muladd_0F, gf8_muladd_10, gf8_muladd_11, gf8_muladd_12, gf8_muladd_13,
+    gf8_muladd_14, gf8_muladd_15, gf8_muladd_16, gf8_muladd_17, gf8_muladd_18,
+    gf8_muladd_19, gf8_muladd_1A, gf8_muladd_1B, gf8_muladd_1C, gf8_muladd_1D,
+    gf8_muladd_1E, gf8_muladd_1F, gf8_muladd_20, gf8_muladd_21, gf8_muladd_22,
+    gf8_muladd_23, gf8_muladd_24, gf8_muladd_25, gf8_muladd_26, gf8_muladd_27,
+    gf8_muladd_28, gf8_muladd_29, gf8_muladd_2A, gf8_muladd_2B, gf8_muladd_2C,
+    gf8_muladd_2D, gf8_muladd_2E, gf8_muladd_2F, gf8_muladd_30, gf8_muladd_31,
+    gf8_muladd_32, gf8_muladd_33, gf8_muladd_34, gf8_muladd_35, gf8_muladd_36,
+    gf8_muladd_37, gf8_muladd_38, gf8_muladd_39, gf8_muladd_3A, gf8_muladd_3B,
+    gf8_muladd_3C, gf8_muladd_3D, gf8_muladd_3E, gf8_muladd_3F, gf8_muladd_40,
+    gf8_muladd_41, gf8_muladd_42, gf8_muladd_43, gf8_muladd_44, gf8_muladd_45,
+    gf8_muladd_46, gf8_muladd_47, gf8_muladd_48, gf8_muladd_49, gf8_muladd_4A,
+    gf8_muladd_4B, gf8_muladd_4C, gf8_muladd_4D, gf8_muladd_4E, gf8_muladd_4F,
+    gf8_muladd_50, gf8_muladd_51, gf8_muladd_52, gf8_muladd_53, gf8_muladd_54,
+    gf8_muladd_55, gf8_muladd_56, gf8_muladd_57, gf8_muladd_58, gf8_muladd_59,
+    gf8_muladd_5A, gf8_muladd_5B, gf8_muladd_5C, gf8_muladd_5D, gf8_muladd_5E,
+    gf8_muladd_5F, gf8_muladd_60, gf8_muladd_61, gf8_muladd_62, gf8_muladd_63,
+    gf8_muladd_64, gf8_muladd_65, gf8_muladd_66, gf8_muladd_67, gf8_muladd_68,
+    gf8_muladd_69, gf8_muladd_6A, gf8_muladd_6B, gf8_muladd_6C, gf8_muladd_6D,
+    gf8_muladd_6E, gf8_muladd_6F, gf8_muladd_70, gf8_muladd_71, gf8_muladd_72,
+    gf8_muladd_73, gf8_muladd_74, gf8_muladd_75, gf8_muladd_76, gf8_muladd_77,
+    gf8_muladd_78, gf8_muladd_79, gf8_muladd_7A, gf8_muladd_7B, gf8_muladd_7C,
+    gf8_muladd_7D, gf8_muladd_7E, gf8_muladd_7F, gf8_muladd_80, gf8_muladd_81,
+    gf8_muladd_82, gf8_muladd_83, gf8_muladd_84, gf8_muladd_85, gf8_muladd_86,
+    gf8_muladd_87, gf8_muladd_88, gf8_muladd_89, gf8_muladd_8A, gf8_muladd_8B,
+    gf8_muladd_8C, gf8_muladd_8D, gf8_muladd_8E, gf8_muladd_8F, gf8_muladd_90,
+    gf8_muladd_91, gf8_muladd_92, gf8_muladd_93, gf8_muladd_94, gf8_muladd_95,
+    gf8_muladd_96, gf8_muladd_97, gf8_muladd_98, gf8_muladd_99, gf8_muladd_9A,
+    gf8_muladd_9B, gf8_muladd_9C, gf8_muladd_9D, gf8_muladd_9E, gf8_muladd_9F,
+    gf8_muladd_A0, gf8_muladd_A1, gf8_muladd_A2, gf8_muladd_A3, gf8_muladd_A4,
+    gf8_muladd_A5, gf8_muladd_A6, gf8_muladd_A7, gf8_muladd_A8, gf8_muladd_A9,
+    gf8_muladd_AA, gf8_muladd_AB, gf8_muladd_AC, gf8_muladd_AD, gf8_muladd_AE,
+    gf8_muladd_AF, gf8_muladd_B0, gf8_muladd_B1, gf8_muladd_B2, gf8_muladd_B3,
+    gf8_muladd_B4, gf8_muladd_B5, gf8_muladd_B6, gf8_muladd_B7, gf8_muladd_B8,
+    gf8_muladd_B9, gf8_muladd_BA, gf8_muladd_BB, gf8_muladd_BC, gf8_muladd_BD,
+    gf8_muladd_BE, gf8_muladd_BF, gf8_muladd_C0, gf8_muladd_C1, gf8_muladd_C2,
+    gf8_muladd_C3, gf8_muladd_C4, gf8_muladd_C5, gf8_muladd_C6, gf8_muladd_C7,
+    gf8_muladd_C8, gf8_muladd_C9, gf8_muladd_CA, gf8_muladd_CB, gf8_muladd_CC,
+    gf8_muladd_CD, gf8_muladd_CE, gf8_muladd_CF, gf8_muladd_D0, gf8_muladd_D1,
+    gf8_muladd_D2, gf8_muladd_D3, gf8_muladd_D4, gf8_muladd_D5, gf8_muladd_D6,
+    gf8_muladd_D7, gf8_muladd_D8, gf8_muladd_D9, gf8_muladd_DA, gf8_muladd_DB,
+    gf8_muladd_DC, gf8_muladd_DD, gf8_muladd_DE, gf8_muladd_DF, gf8_muladd_E0,
+    gf8_muladd_E1, gf8_muladd_E2, gf8_muladd_E3, gf8_muladd_E4, gf8_muladd_E5,
+    gf8_muladd_E6, gf8_muladd_E7, gf8_muladd_E8, gf8_muladd_E9, gf8_muladd_EA,
+    gf8_muladd_EB, gf8_muladd_EC, gf8_muladd_ED, gf8_muladd_EE, gf8_muladd_EF,
+    gf8_muladd_F0, gf8_muladd_F1, gf8_muladd_F2, gf8_muladd_F3, gf8_muladd_F4,
+    gf8_muladd_F5, gf8_muladd_F6, gf8_muladd_F7, gf8_muladd_F8, gf8_muladd_F9,
+    gf8_muladd_FA, gf8_muladd_FB, gf8_muladd_FC, gf8_muladd_FD, gf8_muladd_FE,
+    gf8_muladd_FF};
+
+static uint64_t zero[EC_METHOD_WORD_SIZE * 8] = {
+    0,
+};
+
+void
+ec_code_c_prepare(ec_gf_t *gf, uint32_t *values, uint32_t count)
+{
+    uint32_t i, last, tmp;
+
+    last = 1;
+    for (i = count; i > 0; i--) {
+        if (values[i - 1] != 0) {
+            tmp = values[i - 1];
+            values[i - 1] = ec_gf_div(gf, tmp, last);
+            last = tmp;
+        }
+    }
+}
+
+void
+ec_code_c_linear(void *dst, void *src, uint64_t offset, uint32_t *values,
+                 uint32_t count)
+{
+    src += offset;
+    gf8_muladd_00(dst, src);
+    while (--count > 0) {
+        src += EC_METHOD_CHUNK_SIZE;
+        gf8_muladd[*values](dst, src);
+        values++;
+    }
+}
+
+void
+ec_code_c_interleaved(void *dst, void **src, uint64_t offset, uint32_t *values,
+                      uint32_t count)
+{
+    uint32_t i, last, tmp;
+
+    i = 0;
+    while ((last = *values++) == 0) {
+        i++;
+    }
+    gf8_muladd_00(dst, src[i++] + offset);
+    while (i < count) {
+        tmp = *values++;
+        if (tmp != 0) {
+            gf8_muladd[last](dst, src[i] + offset);
+            last = tmp;
+        }
+        i++;
+    }
+    gf8_muladd[last](dst, zero);
+}
diff --git a/xlators/cluster/ec/src/ec-code-c.h b/xlators/cluster/ec/src/ec-code-c.h
new file mode 100644
index 00000000000..42b5a064eb8
--- /dev/null
+++ b/xlators/cluster/ec/src/ec-code-c.h
@@ -0,0 +1,27 @@
+/*
+  Copyright (c) 2015 DataLab, s.l. <http://www.datalab.es>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
+*/
+
+#ifndef __EC_CODE_C_H__
+#define __EC_CODE_C_H__
+
+#include "ec-types.h"
+
+void
+ec_code_c_prepare(ec_gf_t *gf, uint32_t *values, uint32_t count);
+
+void
+ec_code_c_linear(void *dst, void *src, uint64_t offset, uint32_t *values,
+                 uint32_t count);
+
+void
+ec_code_c_interleaved(void *dst, void **src, uint64_t offset, uint32_t *values,
+                      uint32_t count);
+
+#endif /* __EC_CODE_C_H__ */
diff --git a/xlators/cluster/ec/src/ec-code-intel.c b/xlators/cluster/ec/src/ec-code-intel.c
new file mode 100644
index 00000000000..f1c4e13e321
--- /dev/null
+++ b/xlators/cluster/ec/src/ec-code-intel.c
@@ -0,0 +1,594 @@
+/*
+  Copyright (c) 2015 DataLab, s.l. <http://www.datalab.es>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
+*/
+
+#include <inttypes.h>
+#include <string.h>
+#include <errno.h>
+
+#include "ec-code-intel.h"
+
+static void
+ec_code_intel_init(ec_code_intel_t *intel)
+{
+    memset(intel, 0, sizeof(ec_code_intel_t));
+}
+
+static void
+ec_code_intel_prefix(ec_code_intel_t *intel, uint8_t prefix)
+{
+    intel->prefix.data[intel->prefix.bytes++] = prefix;
+}
+
+static void
+ec_code_intel_rex(ec_code_intel_t *intel, gf_boolean_t w)
+{
+    gf_boolean_t present = _gf_false;
+
+    if (w) {
+        intel->rex.w = 1;
+        present = _gf_true;
+    }
+    if (intel->modrm.present) {
+        if (intel->modrm.reg > 7) {
+            intel->modrm.reg &= 7;
+            intel->rex.r = 1;
+            present = _gf_true;
+        }
+        if (intel->sib.present) {
+            if (intel->sib.index > 7) {
+                intel->sib.index &= 7;
+                intel->rex.x = 1;
+                present = _gf_true;
+            }
+            if (intel->sib.base > 7) {
+                intel->sib.base &= 7;
+                intel->rex.b = 1;
+                present = _gf_true;
+            }
+        } else if (intel->modrm.rm > 7) {
+            intel->modrm.rm &= 7;
+            intel->rex.b = 1;
+            present = _gf_true;
+        }
+    } else if (intel->reg > 7) {
+        intel->reg &= 7;
+        intel->rex.b = 1;
+        present = _gf_true;
+    }
+    intel->rex.present = present;
+}
+
+static void
+ec_code_intel_vex(ec_code_intel_t *intel, gf_boolean_t w, gf_boolean_t l,
+                  ec_code_vex_opcode_t opcode, ec_code_vex_prefix_t prefix,
+                  uint32_t reg)
+{
+    ec_code_intel_rex(intel, w);
+    if (((intel->rex.w == 1) || (intel->rex.x == 0) || (intel->rex.b == 0)) ||
+        ((opcode != VEX_OPCODE_NONE) && (opcode != VEX_OPCODE_0F))) {
+        intel->rex.present = _gf_false;
+
+        intel->vex.bytes = 3;
+        intel->vex.data[0] = 0xC4;
+        intel->vex.data[1] = ((intel->rex.r << 7) | (intel->rex.x << 6) |
+                              (intel->rex.b << 5) | opcode) ^
+                             0xE0;
+        intel->vex.data[2] = (intel->rex.w << 7) | ((~reg & 0x0F) << 3) |
+                             (l ? 0x04 : 0x00) | prefix;
+    } else {
+        intel->vex.bytes = 2;
+        intel->vex.data[0] = 0xC5;
+        intel->vex.data[1] = (intel->rex.r << 7) | ((~reg & 0x0F) << 3) |
+                             (l ? 0x04 : 0x00) | prefix;
+    }
+}
+
+static void
+ec_code_intel_modrm_reg(ec_code_intel_t *intel, uint32_t rm, uint32_t reg)
+{
+    intel->modrm.present = _gf_true;
+    intel->modrm.mod = 3;
+    intel->modrm.rm = rm;
+    intel->modrm.reg = reg;
+}
+
+static void
+ec_code_intel_modrm_mem(ec_code_intel_t *intel, uint32_t reg,
+                        ec_code_intel_reg_t base, ec_code_intel_reg_t index,
+                        uint32_t scale, int32_t offset)
+{
+    if (index == REG_SP) {
+        intel->invalid = _gf_true;
+        return;
+    }
+    if ((index != REG_NULL) && (scale != 1) && (scale != 2) && (scale != 4) &&
+        (scale != 8)) {
+        intel->invalid = _gf_true;
+        return;
+    }
+    scale >>= 1;
+    if (scale == 4) {
+        scale = 3;
+    }
+
+    intel->modrm.present = _gf_true;
+    intel->modrm.reg = reg;
+
+    intel->offset.value = offset;
+    if ((offset == 0) && (base != REG_BP)) {
+        intel->modrm.mod = 0;
+        intel->offset.bytes = 0;
+    } else if ((offset >= -128) && (offset <= 127)) {
+        intel->modrm.mod = 1;
+        intel->offset.bytes = 1;
+    } else {
+        intel->modrm.mod = 2;
+        intel->offset.bytes = 4;
+    }
+
+    intel->modrm.rm = base;
+    if ((index != REG_NULL) || (base == REG_SP)) {
+        intel->modrm.rm = 4;
+        intel->sib.present = _gf_true;
+        intel->sib.index = index;
+        if (index == REG_NULL) {
+            intel->sib.index = 4;
+        }
+        intel->sib.scale = scale;
+        intel->sib.base = base;
+        if (base == REG_NULL) {
+            intel->sib.base = 5;
+            intel->modrm.mod = 0;
+            intel->offset.bytes = 4;
+        }
+    } else if (base == REG_NULL) {
+        intel->modrm.mod = 0;
+        intel->modrm.rm = 5;
+        intel->offset.bytes = 4;
+    }
+}
+
+static void
+ec_code_intel_op_1(ec_code_intel_t *intel, uint8_t opcode, uint32_t reg)
+{
+    intel->reg = reg;
+    intel->opcode.bytes = 1;
+    intel->opcode.data[0] = opcode;
+}
+
+static void
+ec_code_intel_op_2(ec_code_intel_t *intel, uint8_t opcode1, uint8_t opcode2,
+                   uint32_t reg)
+{
+    intel->reg = reg;
+    intel->opcode.bytes = 2;
+    intel->opcode.data[0] = opcode1;
+    intel->opcode.data[1] = opcode2;
+}
+
+static void
+ec_code_intel_immediate_1(ec_code_intel_t *intel, uint32_t value)
+{
+    intel->immediate.bytes = 1;
+    intel->immediate.value = value;
+}
+
+static void
+ec_code_intel_immediate_2(ec_code_intel_t *intel, uint32_t value)
+{
+    intel->immediate.bytes = 2;
+    intel->immediate.value = value;
+}
+
+static void
+ec_code_intel_immediate_4(ec_code_intel_t *intel, uint32_t value)
+{
+    intel->immediate.bytes = 4;
+    intel->immediate.value = value;
+}
+
+static void
+ec_code_intel_emit(ec_code_builder_t *builder, ec_code_intel_t *intel)
+{
+    uint8_t insn[15];
+    uint32_t i, count;
+
+    if (intel->invalid) {
+        ec_code_error(builder, EINVAL);
+        return;
+    }
+
+    count = 0;
+    for (i = 0; i < intel->prefix.bytes; i++) {
+        insn[count++] = intel->prefix.data[i];
+    }
+    for (i = 0; i < intel->vex.bytes; i++) {
+        insn[count++] = intel->vex.data[i];
+    }
+    if (intel->rex.present) {
+        insn[count++] = 0x40 | (intel->rex.w << 3) | (intel->rex.r << 2) |
+                        (intel->rex.x << 1) | (intel->rex.b << 0);
+    }
+    for (i = 0; i < intel->opcode.bytes; i++) {
+        insn[count++] = intel->opcode.data[i];
+    }
+    if (intel->modrm.present) {
+        insn[count++] = (intel->modrm.mod << 6) | (intel->modrm.reg << 3) |
+                        (intel->modrm.rm << 0);
+        if (intel->sib.present) {
+            insn[count++] = (intel->sib.scale << 6) | (intel->sib.index << 3) |
+                            (intel->sib.base << 0);
+        }
+    }
+    for (i = 0; i < intel->offset.bytes; i++) {
+        insn[count++] = intel->offset.data[i];
+    }
+    for (i = 0; i < intel->immediate.bytes; i++) {
+        insn[count++] = intel->immediate.data[i];
+    }
+
+    ec_code_emit(builder, insn, count);
+}
+
+void
+ec_code_intel_op_push_r(ec_code_builder_t *builder, ec_code_intel_reg_t reg)
+{
+    ec_code_intel_t intel;
+
+    ec_code_intel_init(&intel);
+
+    ec_code_intel_op_1(&intel, 0x50 | (reg & 7), reg);
+    ec_code_intel_rex(&intel, _gf_false);
+
+    ec_code_intel_emit(builder, &intel);
+}
+
+void
+ec_code_intel_op_pop_r(ec_code_builder_t *builder, ec_code_intel_reg_t reg)
+{
+    ec_code_intel_t intel;
+
+    ec_code_intel_init(&intel);
+
+    ec_code_intel_op_1(&intel, 0x58 | (reg & 7), reg);
+    ec_code_intel_rex(&intel, _gf_false);
+
+    ec_code_intel_emit(builder, &intel);
+}
+
+void
+ec_code_intel_op_ret(ec_code_builder_t *builder, uint32_t size)
+{
+    ec_code_intel_t intel;
+
+    ec_code_intel_init(&intel);
+
+    if (size == 0) {
+        ec_code_intel_op_1(&intel, 0xC3, 0);
+    } else {
+        ec_code_intel_immediate_2(&intel, size);
+        ec_code_intel_op_1(&intel, 0xC2, 0);
+    }
+    ec_code_intel_rex(&intel, _gf_false);
+
+    ec_code_intel_emit(builder, &intel);
+}
+
+void
+ec_code_intel_op_mov_r2r(ec_code_builder_t *builder, ec_code_intel_reg_t src,
+                         ec_code_intel_reg_t dst)
+{
+    ec_code_intel_t intel;
+
+    ec_code_intel_init(&intel);
+
+    ec_code_intel_modrm_reg(&intel, dst, src);
+    ec_code_intel_op_1(&intel, 0x89, 0);
+    ec_code_intel_rex(&intel, _gf_true);
+
+    ec_code_intel_emit(builder, &intel);
+}
+
+void
+ec_code_intel_op_mov_r2m(ec_code_builder_t *builder, ec_code_intel_reg_t src,
+                         ec_code_intel_reg_t base, ec_code_intel_reg_t index,
+                         uint32_t scale, int32_t offset)
+{
+    ec_code_intel_t intel;
+
+    ec_code_intel_init(&intel);
+
+    ec_code_intel_modrm_mem(&intel, src, base, index, scale, offset);
+    ec_code_intel_op_1(&intel, 0x89, 0);
+    ec_code_intel_rex(&intel, _gf_true);
+
+    ec_code_intel_emit(builder, &intel);
+}
+
+void
+ec_code_intel_op_mov_m2r(ec_code_builder_t *builder, ec_code_intel_reg_t base,
+                         ec_code_intel_reg_t index, uint32_t scale,
+                         int32_t offset, ec_code_intel_reg_t dst)
+{
+    ec_code_intel_t intel;
+
+    ec_code_intel_init(&intel);
+
+    ec_code_intel_modrm_mem(&intel, dst, base, index, scale, offset);
+    ec_code_intel_op_1(&intel, 0x8B, 0);
+    ec_code_intel_rex(&intel, _gf_true);
+
+    ec_code_intel_emit(builder, &intel);
+}
+
+void
+ec_code_intel_op_xor_r2r(ec_code_builder_t *builder, ec_code_intel_reg_t src,
+                         ec_code_intel_reg_t dst)
+{
+    ec_code_intel_t intel;
+
+    ec_code_intel_init(&intel);
+
+    ec_code_intel_modrm_reg(&intel, dst, src);
+    ec_code_intel_op_1(&intel, 0x31, 0);
+    ec_code_intel_rex(&intel, _gf_true);
+
+    ec_code_intel_emit(builder, &intel);
+}
+
+void
+ec_code_intel_op_xor_m2r(ec_code_builder_t *builder, ec_code_intel_reg_t base,
+                         ec_code_intel_reg_t index, uint32_t scale,
+                         int32_t offset, ec_code_intel_reg_t dst)
+{
+    ec_code_intel_t intel;
+
+    ec_code_intel_init(&intel);
+
+    ec_code_intel_modrm_mem(&intel, dst, base, index, scale, offset);
+    ec_code_intel_op_1(&intel, 0x33, 0);
+    ec_code_intel_rex(&intel, _gf_true);
+
+    ec_code_intel_emit(builder, &intel);
+}
+
+void
+ec_code_intel_op_add_i2r(ec_code_builder_t *builder, int32_t value,
+                         ec_code_intel_reg_t reg)
+{
+    ec_code_intel_t intel;
+
+    ec_code_intel_init(&intel);
+
+    if ((value >= -128) && (value < 128)) {
+        ec_code_intel_modrm_reg(&intel, reg, 0);
+        ec_code_intel_op_1(&intel, 0x83, 0);
+        ec_code_intel_immediate_1(&intel, value);
+    } else {
+        if (reg == REG_AX) {
+            ec_code_intel_op_1(&intel, 0x05, reg);
+        } else {
+            ec_code_intel_modrm_reg(&intel, reg, 0);
+            ec_code_intel_op_1(&intel, 0x81, 0);
+        }
+        ec_code_intel_immediate_4(&intel, value);
+    }
+    ec_code_intel_rex(&intel, _gf_true);
+
+    ec_code_intel_emit(builder, &intel);
+}
+
+void
+ec_code_intel_op_test_i2r(ec_code_builder_t *builder, uint32_t value,
+                          ec_code_intel_reg_t reg)
+{
+    ec_code_intel_t intel;
+
+    ec_code_intel_init(&intel);
+
+    if (reg == REG_AX) {
+        ec_code_intel_op_1(&intel, 0xA9, reg);
+    } else {
+        ec_code_intel_modrm_reg(&intel, reg, 0);
+        ec_code_intel_op_1(&intel, 0xF7, 0);
+    }
+    ec_code_intel_immediate_4(&intel, value);
+    ec_code_intel_rex(&intel, _gf_true);
+
+    ec_code_intel_emit(builder, &intel);
+}
+
+void
+ec_code_intel_op_jne(ec_code_builder_t *builder, uint32_t address)
+{
+    ec_code_intel_t intel;
+    int32_t rel;
+
+    ec_code_intel_init(&intel);
+
+    rel = address - builder->address - 2;
+    if ((rel >= -128) && (rel < 128)) {
+        ec_code_intel_op_1(&intel, 0x75, 0);
+        ec_code_intel_immediate_1(&intel, rel);
+    } else {
+        rel -= 4;
+        ec_code_intel_op_2(&intel, 0x0F, 0x85, 0);
+        ec_code_intel_immediate_4(&intel, rel);
+    }
+    ec_code_intel_rex(&intel, _gf_false);
+
+    ec_code_intel_emit(builder, &intel);
+}
+
+void
+ec_code_intel_op_mov_sse2sse(ec_code_builder_t *builder, uint32_t src,
+                             uint32_t dst)
+{
+    ec_code_intel_t intel;
+
+    ec_code_intel_init(&intel);
+
+    ec_code_intel_prefix(&intel, 0x66);
+    ec_code_intel_modrm_reg(&intel, src, dst);
+    ec_code_intel_op_2(&intel, 0x0F, 0x6F, 0);
+    ec_code_intel_rex(&intel, _gf_false);
+
+    ec_code_intel_emit(builder, &intel);
+}
+
+void
+ec_code_intel_op_mov_sse2m(ec_code_builder_t *builder, uint32_t src,
+                           ec_code_intel_reg_t base, ec_code_intel_reg_t index,
+                           uint32_t scale, int32_t offset)
+{
+    ec_code_intel_t intel;
+
+    ec_code_intel_init(&intel);
+
+    ec_code_intel_prefix(&intel, 0x66);
+    ec_code_intel_modrm_mem(&intel, src, base, index, scale, offset);
+    ec_code_intel_op_2(&intel, 0x0F, 0x7F, 0);
+    ec_code_intel_rex(&intel, _gf_false);
+
+    ec_code_intel_emit(builder, &intel);
+}
+
+void
+ec_code_intel_op_mov_m2sse(ec_code_builder_t *builder, ec_code_intel_reg_t base,
+                           ec_code_intel_reg_t index, uint32_t scale,
+                           int32_t offset, uint32_t dst)
+{
+    ec_code_intel_t intel;
+
+    ec_code_intel_init(&intel);
+
+    ec_code_intel_prefix(&intel, 0x66);
+    ec_code_intel_modrm_mem(&intel, dst, base, index, scale, offset);
+    ec_code_intel_op_2(&intel, 0x0F, 0x6F, 0);
+    ec_code_intel_rex(&intel, _gf_false);
+
+    ec_code_intel_emit(builder, &intel);
+}
+
+void
+ec_code_intel_op_xor_sse2sse(ec_code_builder_t *builder, uint32_t src,
+                             uint32_t dst)
+{
+    ec_code_intel_t intel;
+
+    ec_code_intel_init(&intel);
+
+    ec_code_intel_prefix(&intel, 0x66);
+    ec_code_intel_modrm_reg(&intel, src, dst);
+    ec_code_intel_op_2(&intel, 0x0F, 0xEF, 0);
+    ec_code_intel_rex(&intel, _gf_false);
+
+    ec_code_intel_emit(builder, &intel);
+}
+
+void
+ec_code_intel_op_xor_m2sse(ec_code_builder_t *builder, ec_code_intel_reg_t base,
+                           ec_code_intel_reg_t index, uint32_t scale,
+                           int32_t offset, uint32_t dst)
+{
+    ec_code_intel_t intel;
+
+    ec_code_intel_init(&intel);
+
+    ec_code_intel_prefix(&intel, 0x66);
+    ec_code_intel_modrm_mem(&intel, dst, base, index, scale, offset);
+    ec_code_intel_op_2(&intel, 0x0F, 0xEF, 0);
+    ec_code_intel_rex(&intel, _gf_false);
+
+    ec_code_intel_emit(builder, &intel);
+}
+
+void
+ec_code_intel_op_mov_avx2avx(ec_code_builder_t *builder, uint32_t src,
+                             uint32_t dst)
+{
+    ec_code_intel_t intel;
+
+    ec_code_intel_init(&intel);
+
+    ec_code_intel_modrm_reg(&intel, src, dst);
+    ec_code_intel_op_1(&intel, 0x6F, 0);
+    ec_code_intel_vex(&intel, _gf_false, _gf_true, VEX_OPCODE_0F, VEX_PREFIX_66,
+                      VEX_REG_NONE);
+
+    ec_code_intel_emit(builder, &intel);
+}
+
+void
+ec_code_intel_op_mov_avx2m(ec_code_builder_t *builder, uint32_t src,
+                           ec_code_intel_reg_t base, ec_code_intel_reg_t index,
+                           uint32_t scale, int32_t offset)
+{
+    ec_code_intel_t intel;
+
+    ec_code_intel_init(&intel);
+
+    ec_code_intel_modrm_mem(&intel, src, base, index, scale, offset);
+    ec_code_intel_op_1(&intel, 0x7F, 0);
+    ec_code_intel_vex(&intel, _gf_false, _gf_true, VEX_OPCODE_0F, VEX_PREFIX_66,
+                      VEX_REG_NONE);
+
+    ec_code_intel_emit(builder, &intel);
+}
+
+void
+ec_code_intel_op_mov_m2avx(ec_code_builder_t *builder, ec_code_intel_reg_t base,
+                           ec_code_intel_reg_t index, uint32_t scale,
+                           int32_t offset, uint32_t dst)
+{
+    ec_code_intel_t intel;
+
+    ec_code_intel_init(&intel);
+
+    ec_code_intel_modrm_mem(&intel, dst, base, index, scale, offset);
+    ec_code_intel_op_1(&intel, 0x6F, 0);
+    ec_code_intel_vex(&intel, _gf_false, _gf_true, VEX_OPCODE_0F, VEX_PREFIX_66,
+                      VEX_REG_NONE);
+
+    ec_code_intel_emit(builder, &intel);
+}
+
+void
+ec_code_intel_op_xor_avx2avx(ec_code_builder_t *builder, uint32_t src,
+                             uint32_t dst)
+{
+    ec_code_intel_t intel;
+
+    ec_code_intel_init(&intel);
+
+    ec_code_intel_modrm_reg(&intel, src, dst);
+    ec_code_intel_op_1(&intel, 0xEF, 0);
+    ec_code_intel_vex(&intel, _gf_false, _gf_true, VEX_OPCODE_0F, VEX_PREFIX_66,
+                      dst);
+
+    ec_code_intel_emit(builder, &intel);
+}
+
+void
+ec_code_intel_op_xor_m2avx(ec_code_builder_t *builder, ec_code_intel_reg_t base,
+                           ec_code_intel_reg_t index, uint32_t scale,
+                           int32_t offset, uint32_t dst)
+{
+    ec_code_intel_t intel;
+
+    ec_code_intel_init(&intel);
+
+    ec_code_intel_modrm_mem(&intel, dst, base, index, scale, offset);
+    ec_code_intel_op_1(&intel, 0xEF, 0);
+    ec_code_intel_vex(&intel, _gf_false, _gf_true, VEX_OPCODE_0F, VEX_PREFIX_66,
+                      dst);
+
+    ec_code_intel_emit(builder, &intel);
+}
diff --git a/xlators/cluster/ec/src/ec-code-intel.h b/xlators/cluster/ec/src/ec-code-intel.h
new file mode 100644
index 00000000000..3fa4a174765
--- /dev/null
+++ b/xlators/cluster/ec/src/ec-code-intel.h
@@ -0,0 +1,191 @@
+/*
+  Copyright (c) 2015 DataLab, s.l. <http://www.datalab.es>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
+*/
+
+#ifndef __EC_CODE_INTEL_H__
+#define __EC_CODE_INTEL_H__
+
+#include "ec-code.h"
+
+#define VEX_REG_NONE 0
+
+enum _ec_code_intel_reg;
+typedef enum _ec_code_intel_reg ec_code_intel_reg_t;
+
+enum _ec_code_vex_prefix;
+typedef enum _ec_code_vex_prefix ec_code_vex_prefix_t;
+
+enum _ec_code_vex_opcode;
+typedef enum _ec_code_vex_opcode ec_code_vex_opcode_t;
+
+struct _ec_code_intel_buffer;
+typedef struct _ec_code_intel_buffer ec_code_intel_buffer_t;
+
+struct _ec_code_intel_sib;
+typedef struct _ec_code_intel_sib ec_code_intel_sib_t;
+
+struct _ec_code_intel_modrm;
+typedef struct _ec_code_intel_modrm ec_code_intel_modrm_t;
+
+struct _ec_code_intel_rex;
+typedef struct _ec_code_intel_rex ec_code_intel_rex_t;
+
+struct _ec_code_intel;
+typedef struct _ec_code_intel ec_code_intel_t;
+
+enum _ec_code_intel_reg {
+    REG_NULL = -1,
+    REG_AX,
+    REG_CX,
+    REG_DX,
+    REG_BX,
+    REG_SP,
+    REG_BP,
+    REG_SI,
+    REG_DI,
+    REG_8,
+    REG_9,
+    REG_10,
+    REG_11,
+    REG_12,
+    REG_13,
+    REG_14,
+    REG_15
+};
+
+enum _ec_code_vex_prefix {
+    VEX_PREFIX_NONE = 0,
+    VEX_PREFIX_66,
+    VEX_PREFIX_F3,
+    VEX_PREFIX_F2
+};
+
+enum _ec_code_vex_opcode {
+    VEX_OPCODE_NONE = 0,
+    VEX_OPCODE_0F,
+    VEX_OPCODE_0F_38,
+    VEX_OPCODE_0F_3A
+};
+
+struct _ec_code_intel_buffer {
+    uint32_t bytes;
+    union {
+        uint8_t data[4];
+        uint32_t value;
+    };
+};
+
+struct _ec_code_intel_sib {
+    gf_boolean_t present;
+    uint32_t base;
+    uint32_t index;
+    uint32_t scale;
+};
+
+struct _ec_code_intel_modrm {
+    gf_boolean_t present;
+    uint32_t mod;
+    uint32_t rm;
+    uint32_t reg;
+};
+
+struct _ec_code_intel_rex {
+    gf_boolean_t present;
+    uint32_t w;
+    uint32_t r;
+    uint32_t x;
+    uint32_t b;
+};
+
+struct _ec_code_intel {
+    gf_boolean_t invalid;
+    ec_code_intel_buffer_t prefix;
+    ec_code_intel_buffer_t opcode;
+    ec_code_intel_buffer_t offset;
+    ec_code_intel_buffer_t immediate;
+    ec_code_intel_buffer_t vex;
+    ec_code_intel_rex_t rex;
+    ec_code_intel_modrm_t modrm;
+    ec_code_intel_sib_t sib;
+    uint32_t reg;
+};
+
+void
+ec_code_intel_op_push_r(ec_code_builder_t *builder, ec_code_intel_reg_t reg);
+void
+ec_code_intel_op_pop_r(ec_code_builder_t *builder, ec_code_intel_reg_t reg);
+void
+ec_code_intel_op_ret(ec_code_builder_t *builder, uint32_t size);
+
+void
+ec_code_intel_op_mov_r2r(ec_code_builder_t *builder, ec_code_intel_reg_t src,
+                         ec_code_intel_reg_t dst);
+void
+ec_code_intel_op_mov_r2m(ec_code_builder_t *builder, ec_code_intel_reg_t src,
+                         ec_code_intel_reg_t base, ec_code_intel_reg_t index,
+                         uint32_t scale, int32_t offset);
+void
+ec_code_intel_op_mov_m2r(ec_code_builder_t *builder, ec_code_intel_reg_t base,
+                         ec_code_intel_reg_t index, uint32_t scale,
+                         int32_t offset, ec_code_intel_reg_t dst);
+void
+ec_code_intel_op_xor_r2r(ec_code_builder_t *builder, ec_code_intel_reg_t src,
+                         ec_code_intel_reg_t dst);
+void
+ec_code_intel_op_xor_m2r(ec_code_builder_t *builder, ec_code_intel_reg_t base,
+                         ec_code_intel_reg_t index, uint32_t scale,
+                         int32_t offset, ec_code_intel_reg_t dst);
+void
+ec_code_intel_op_add_i2r(ec_code_builder_t *builder, int32_t value,
+                         ec_code_intel_reg_t reg);
+void
+ec_code_intel_op_test_i2r(ec_code_builder_t *builder, uint32_t value,
+                          ec_code_intel_reg_t reg);
+void
+ec_code_intel_op_jne(ec_code_builder_t *builder, uint32_t address);
+
+void
+ec_code_intel_op_mov_sse2sse(ec_code_builder_t *builder, uint32_t src,
+                             uint32_t dst);
+void
+ec_code_intel_op_mov_sse2m(ec_code_builder_t *builder, uint32_t src,
+                           ec_code_intel_reg_t base, ec_code_intel_reg_t index,
+                           uint32_t scale, int32_t offset);
+void
+ec_code_intel_op_mov_m2sse(ec_code_builder_t *builder, ec_code_intel_reg_t base,
+                           ec_code_intel_reg_t index, uint32_t scale,
+                           int32_t offset, uint32_t dst);
+void
+ec_code_intel_op_xor_sse2sse(ec_code_builder_t *builder, uint32_t src,
+                             uint32_t dst);
+void
+ec_code_intel_op_xor_m2sse(ec_code_builder_t *builder, ec_code_intel_reg_t base,
+                           ec_code_intel_reg_t index, uint32_t scale,
+                           int32_t offset, uint32_t dst);
+
+void
+ec_code_intel_op_mov_avx2avx(ec_code_builder_t *builder, uint32_t src,
+                             uint32_t dst);
+void
+ec_code_intel_op_mov_avx2m(ec_code_builder_t *builder, uint32_t src,
+                           ec_code_intel_reg_t base, ec_code_intel_reg_t index,
+                           uint32_t scale, int32_t offset);
+void
+ec_code_intel_op_mov_m2avx(ec_code_builder_t *builder, ec_code_intel_reg_t base,
+                           ec_code_intel_reg_t index, uint32_t scale,
+                           int32_t offset, uint32_t dst);
+void
+ec_code_intel_op_xor_avx2avx(ec_code_builder_t *builder, uint32_t src,
+                             uint32_t dst);
+void
+ec_code_intel_op_xor_m2avx(ec_code_builder_t *builder, ec_code_intel_reg_t base,
+                           ec_code_intel_reg_t index, uint32_t scale,
+                           int32_t offset, uint32_t dst);
+
+#endif /* __EC_CODE_INTEL_H__ */
diff --git a/xlators/cluster/ec/src/ec-code-sse.c b/xlators/cluster/ec/src/ec-code-sse.c
new file mode 100644
index 00000000000..e11e7ff8400
--- /dev/null
+++ b/xlators/cluster/ec/src/ec-code-sse.c
@@ -0,0 +1,101 @@
+/*
+  Copyright (c) 2015 DataLab, s.l. <http://www.datalab.es>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
+*/
+
+#include <errno.h>
+
+#include "ec-code-intel.h"
+
+static void
+ec_code_sse_prolog(ec_code_builder_t *builder)
+{
+    builder->loop = builder->address;
+}
+
+static void
+ec_code_sse_epilog(ec_code_builder_t *builder)
+{
+    ec_code_intel_op_add_i2r(builder, 16, REG_DX);
+    ec_code_intel_op_add_i2r(builder, 16, REG_DI);
+    ec_code_intel_op_test_i2r(builder, builder->width - 1, REG_DX);
+    ec_code_intel_op_jne(builder, builder->loop);
+
+    ec_code_intel_op_ret(builder, 0);
+}
+
+static void
+ec_code_sse_load(ec_code_builder_t *builder, uint32_t dst, uint32_t idx,
+                 uint32_t bit)
+{
+    if (builder->linear) {
+        ec_code_intel_op_mov_m2sse(
+            builder, REG_SI, REG_DX, 1,
+            idx * builder->width * builder->bits + bit * builder->width, dst);
+    } else {
+        if (builder->base != idx) {
+            ec_code_intel_op_mov_m2r(builder, REG_SI, REG_NULL, 0, idx * 8,
+                                     REG_AX);
+            builder->base = idx;
+        }
+        ec_code_intel_op_mov_m2sse(builder, REG_AX, REG_DX, 1,
+                                   bit * builder->width, dst);
+    }
+}
+
+static void
+ec_code_sse_store(ec_code_builder_t *builder, uint32_t src, uint32_t bit)
+{
+    ec_code_intel_op_mov_sse2m(builder, src, REG_DI, REG_NULL, 0,
+                               bit * builder->width);
+}
+
+static void
+ec_code_sse_copy(ec_code_builder_t *builder, uint32_t dst, uint32_t src)
+{
+    ec_code_intel_op_mov_sse2sse(builder, src, dst);
+}
+
+static void
+ec_code_sse_xor2(ec_code_builder_t *builder, uint32_t dst, uint32_t src)
+{
+    ec_code_intel_op_xor_sse2sse(builder, src, dst);
+}
+
+static void
+ec_code_sse_xorm(ec_code_builder_t *builder, uint32_t dst, uint32_t idx,
+                 uint32_t bit)
+{
+    if (builder->linear) {
+        ec_code_intel_op_xor_m2sse(
+            builder, REG_SI, REG_DX, 1,
+            idx * builder->width * builder->bits + bit * builder->width, dst);
+    } else {
+        if (builder->base != idx) {
+            ec_code_intel_op_mov_m2r(builder, REG_SI, REG_NULL, 0, idx * 8,
+                                     REG_AX);
+            builder->base = idx;
+        }
+        ec_code_intel_op_xor_m2sse(builder, REG_AX, REG_DX, 1,
+                                   bit * builder->width, dst);
+    }
+}
+
+static char *ec_code_sse_needed_flags[] = {"sse2", NULL};
+
+ec_code_gen_t ec_code_gen_sse = {.name = "sse",
+                                 .flags = ec_code_sse_needed_flags,
+                                 .width = 16,
+                                 .prolog = ec_code_sse_prolog,
+                                 .epilog = ec_code_sse_epilog,
+                                 .load = ec_code_sse_load,
+                                 .store = ec_code_sse_store,
+                                 .copy = ec_code_sse_copy,
+                                 .xor2 = ec_code_sse_xor2,
+                                 .xor3 = NULL,
+                                 .xorm = ec_code_sse_xorm};
diff --git a/xlators/features/changetimerecorder/src/ctr_mem_types.h b/xlators/cluster/ec/src/ec-code-sse.h
index 48387597814..f1acbcf894b 100644
--- a/xlators/features/changetimerecorder/src/ctr_mem_types.h
+++ b/xlators/cluster/ec/src/ec-code-sse.h
@@ -1,5 +1,5 @@
 /*
-  Copyright (c) 2008-2015 Red Hat, Inc. <http://www.redhat.com>
+  Copyright (c) 2015 DataLab, s.l. <http://www.datalab.es>
   This file is part of GlusterFS.
 
   This file is licensed to you under your choice of the GNU Lesser
@@ -8,15 +8,11 @@
   cases as published by the Free Software Foundation.
 */
 
+#ifndef __EC_CODE_SSE_H__
+#define __EC_CODE_SSE_H__
 
-#ifndef __CTR_MEM_TYPES_H__
-#define __CTR_MEM_TYPES_H__
+#include "ec-code.h"
 
-#include "gfdb_mem-types.h"
-
-enum gf_ctr_mem_types_ {
-        gf_ctr_mt_private_t = gfdb_mt_end + 1,
-        gf_ctr_mt_end
-};
-#endif
+extern ec_code_gen_t ec_code_gen_sse;
 
+#endif /* __EC_CODE_SSE_H__ */
diff --git a/xlators/cluster/ec/src/ec-code-x64.c b/xlators/cluster/ec/src/ec-code-x64.c
new file mode 100644
index 00000000000..26565b4493f
--- /dev/null
+++ b/xlators/cluster/ec/src/ec-code-x64.c
@@ -0,0 +1,144 @@
+/*
+  Copyright (c) 2015 DataLab, s.l. <http://www.datalab.es>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
+*/
+
+#include <errno.h>
+
+#include "ec-code-intel.h"
+
+static ec_code_intel_reg_t ec_code_x64_regmap[] = {
+    REG_AX, REG_CX, REG_BP, REG_8,  REG_9, REG_10,
+    REG_11, REG_12, REG_13, REG_14, REG_15};
+
+static void
+ec_code_x64_prolog(ec_code_builder_t *builder)
+{
+    uint32_t i;
+
+    ec_code_intel_op_push_r(builder, REG_BP);
+    if (!builder->linear) {
+        ec_code_intel_op_push_r(builder, REG_BX);
+    }
+    if (builder->regs > 11) {
+        ec_code_error(builder, EINVAL);
+        return;
+    }
+    for (i = 7; i < builder->regs; i++) {
+        ec_code_intel_op_push_r(builder, ec_code_x64_regmap[i]);
+    }
+
+    builder->loop = builder->address;
+}
+
+static void
+ec_code_x64_epilog(ec_code_builder_t *builder)
+{
+    uint32_t i;
+
+    ec_code_intel_op_add_i2r(builder, 8, REG_DX);
+    ec_code_intel_op_add_i2r(builder, 8, REG_DI);
+    ec_code_intel_op_test_i2r(builder, builder->width - 1, REG_DX);
+    ec_code_intel_op_jne(builder, builder->loop);
+
+    if (builder->regs > 11) {
+        ec_code_error(builder, EINVAL);
+        return;
+    }
+    for (i = builder->regs; i > 7; i--) {
+        ec_code_intel_op_pop_r(builder, ec_code_x64_regmap[i - 1]);
+    }
+    if (!builder->linear) {
+        ec_code_intel_op_pop_r(builder, REG_BX);
+    }
+    ec_code_intel_op_pop_r(builder, REG_BP);
+    ec_code_intel_op_ret(builder, 0);
+}
+
+static void
+ec_code_x64_load(ec_code_builder_t *builder, uint32_t dst, uint32_t idx,
+                 uint32_t bit)
+{
+    dst = ec_code_x64_regmap[dst];
+
+    if (builder->linear) {
+        ec_code_intel_op_mov_m2r(
+            builder, REG_SI, REG_DX, 1,
+            idx * builder->width * builder->bits + bit * builder->width, dst);
+    } else {
+        if (builder->base != idx) {
+            ec_code_intel_op_mov_m2r(builder, REG_SI, REG_NULL, 0, idx * 8,
+                                     REG_BX);
+            builder->base = idx;
+        }
+        ec_code_intel_op_mov_m2r(builder, REG_BX, REG_DX, 1,
+                                 bit * builder->width, dst);
+    }
+}
+
+static void
+ec_code_x64_store(ec_code_builder_t *builder, uint32_t src, uint32_t bit)
+{
+    src = ec_code_x64_regmap[src];
+
+    ec_code_intel_op_mov_r2m(builder, src, REG_DI, REG_NULL, 0,
+                             bit * builder->width);
+}
+
+static void
+ec_code_x64_copy(ec_code_builder_t *builder, uint32_t dst, uint32_t src)
+{
+    dst = ec_code_x64_regmap[dst];
+    src = ec_code_x64_regmap[src];
+
+    ec_code_intel_op_mov_r2r(builder, src, dst);
+}
+
+static void
+ec_code_x64_xor2(ec_code_builder_t *builder, uint32_t dst, uint32_t src)
+{
+    dst = ec_code_x64_regmap[dst];
+    src = ec_code_x64_regmap[src];
+
+    ec_code_intel_op_xor_r2r(builder, src, dst);
+}
+
+static void
+ec_code_x64_xorm(ec_code_builder_t *builder, uint32_t dst, uint32_t idx,
+                 uint32_t bit)
+{
+    dst = ec_code_x64_regmap[dst];
+
+    if (builder->linear) {
+        ec_code_intel_op_xor_m2r(
+            builder, REG_SI, REG_DX, 1,
+            idx * builder->width * builder->bits + bit * builder->width, dst);
+    } else {
+        if (builder->base != idx) {
+            ec_code_intel_op_mov_m2r(builder, REG_SI, REG_NULL, 0, idx * 8,
+                                     REG_BX);
+            builder->base = idx;
+        }
+        ec_code_intel_op_xor_m2r(builder, REG_BX, REG_DX, 1,
+                                 bit * builder->width, dst);
+    }
+}
+
+static char *ec_code_x64_needed_flags[] = {NULL};
+
+ec_code_gen_t ec_code_gen_x64 = {.name = "x64",
+                                 .flags = ec_code_x64_needed_flags,
+                                 .width = sizeof(uint64_t),
+                                 .prolog = ec_code_x64_prolog,
+                                 .epilog = ec_code_x64_epilog,
+                                 .load = ec_code_x64_load,
+                                 .store = ec_code_x64_store,
+                                 .copy = ec_code_x64_copy,
+                                 .xor2 = ec_code_x64_xor2,
+                                 .xor3 = NULL,
+                                 .xorm = ec_code_x64_xorm};
diff --git a/xlators/cluster/ec/src/ec-code-x64.h b/xlators/cluster/ec/src/ec-code-x64.h
new file mode 100644
index 00000000000..bd8174e4bf5
--- /dev/null
+++ b/xlators/cluster/ec/src/ec-code-x64.h
@@ -0,0 +1,18 @@
+/*
+  Copyright (c) 2015 DataLab, s.l. <http://www.datalab.es>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
+*/
+
+#ifndef __EC_CODE_X64_H__
+#define __EC_CODE_X64_H__
+
+#include "ec-code.h"
+
+extern ec_code_gen_t ec_code_gen_x64;
+
+#endif /* __EC_CODE_X64_H__ */
diff --git a/xlators/cluster/ec/src/ec-code.c b/xlators/cluster/ec/src/ec-code.c
new file mode 100644
index 00000000000..03162ae05a9
--- /dev/null
+++ b/xlators/cluster/ec/src/ec-code.c
@@ -0,0 +1,1060 @@
+/*
+  Copyright (c) 2015 DataLab, s.l. <http://www.datalab.es>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
+*/
+
+#include <string.h>
+#include <sys/mman.h>
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <ctype.h>
+
+#include <glusterfs/syscall.h>
+
+#include "ec-mem-types.h"
+#include "ec-code.h"
+#include "ec-messages.h"
+#include "ec-code-c.h"
+#include "ec-helpers.h"
+
+#ifdef USE_EC_DYNAMIC_X64
+#include "ec-code-x64.h"
+#endif
+
+#ifdef USE_EC_DYNAMIC_SSE
+#include "ec-code-sse.h"
+#endif
+
+#ifdef USE_EC_DYNAMIC_AVX
+#include "ec-code-avx.h"
+#endif
+
+#define EC_CODE_SIZE (1024 * 64)
+#define EC_CODE_ALIGN 4096
+
+#define EC_CODE_CHUNK_MIN_SIZE 512
+
+#define EC_PROC_BUFFER_SIZE 4096
+
+#define PROC_CPUINFO "/proc/cpuinfo"
+
+struct _ec_code_proc;
+typedef struct _ec_code_proc ec_code_proc_t;
+
+struct _ec_code_proc {
+    int32_t fd;
+    gf_boolean_t eof;
+    gf_boolean_t error;
+    gf_boolean_t skip;
+    ssize_t size;
+    ssize_t pos;
+    char buffer[EC_PROC_BUFFER_SIZE];
+};
+
+static ec_code_gen_t *ec_code_gen_table[] = {
+#ifdef USE_EC_DYNAMIC_AVX
+    &ec_code_gen_avx,
+#endif
+#ifdef USE_EC_DYNAMIC_SSE
+    &ec_code_gen_sse,
+#endif
+#ifdef USE_EC_DYNAMIC_X64
+    &ec_code_gen_x64,
+#endif
+    NULL};
+
+static void
+ec_code_arg_set(ec_code_arg_t *arg, uint32_t value)
+{
+    arg->value = value;
+}
+
+static void
+ec_code_arg_assign(ec_code_builder_t *builder, ec_code_op_t *op,
+                   ec_code_arg_t *arg, uint32_t reg)
+{
+    arg->value = reg;
+
+    if (builder->regs <= reg) {
+        builder->regs = reg + 1;
+    }
+}
+
+static void
+ec_code_arg_use(ec_code_builder_t *builder, ec_code_op_t *op,
+                ec_code_arg_t *arg, uint32_t reg)
+{
+    arg->value = reg;
+}
+
+static void
+ec_code_arg_update(ec_code_builder_t *builder, ec_code_op_t *op,
+                   ec_code_arg_t *arg, uint32_t reg)
+{
+    arg->value = reg;
+}
+
+static ec_code_op_t *
+ec_code_op_next(ec_code_builder_t *builder)
+{
+    ec_code_op_t *op;
+
+    op = &builder->ops[builder->count++];
+    memset(op, 0, sizeof(ec_code_op_t));
+
+    return op;
+}
+
+static void
+ec_code_load(ec_code_builder_t *builder, uint32_t bit, uint32_t offset)
+{
+    ec_code_op_t *op;
+
+    op = ec_code_op_next(builder);
+
+    op->op = EC_GF_OP_LOAD;
+    ec_code_arg_assign(builder, op, &op->arg1, builder->map[bit]);
+    ec_code_arg_set(&op->arg2, offset);
+    ec_code_arg_set(&op->arg3, bit);
+}
+
+static void
+ec_code_store(ec_code_builder_t *builder, uint32_t reg, uint32_t bit)
+{
+    ec_code_op_t *op;
+
+    op = ec_code_op_next(builder);
+
+    op->op = EC_GF_OP_STORE;
+    ec_code_arg_use(builder, op, &op->arg1, builder->map[reg]);
+    ec_code_arg_set(&op->arg2, 0);
+    ec_code_arg_set(&op->arg3, bit);
+}
+
+static void
+ec_code_copy(ec_code_builder_t *builder, uint32_t dst, uint32_t src)
+{
+    ec_code_op_t *op;
+
+    op = ec_code_op_next(builder);
+
+    op->op = EC_GF_OP_COPY;
+    ec_code_arg_assign(builder, op, &op->arg1, builder->map[dst]);
+    ec_code_arg_use(builder, op, &op->arg2, builder->map[src]);
+    ec_code_arg_set(&op->arg3, 0);
+}
+
+static void
+ec_code_xor2(ec_code_builder_t *builder, uint32_t dst, uint32_t src)
+{
+    ec_code_op_t *op;
+
+    op = ec_code_op_next(builder);
+
+    op->op = EC_GF_OP_XOR2;
+    ec_code_arg_update(builder, op, &op->arg1, builder->map[dst]);
+    ec_code_arg_use(builder, op, &op->arg2, builder->map[src]);
+    ec_code_arg_set(&op->arg3, 0);
+}
+
+static void
+ec_code_xor3(ec_code_builder_t *builder, uint32_t dst, uint32_t src1,
+             uint32_t src2)
+{
+    ec_code_op_t *op;
+
+    if (builder->code->gen->xor3 == NULL) {
+        ec_code_copy(builder, dst, src1);
+        ec_code_xor2(builder, dst, src2);
+
+        return;
+    }
+
+    op = ec_code_op_next(builder);
+
+    op->op = EC_GF_OP_XOR3;
+    ec_code_arg_assign(builder, op, &op->arg1, builder->map[dst]);
+    ec_code_arg_use(builder, op, &op->arg2, builder->map[src1]);
+    ec_code_arg_use(builder, op, &op->arg3, builder->map[src2]);
+}
+
+static void
+ec_code_xorm(ec_code_builder_t *builder, uint32_t bit, uint32_t offset)
+{
+    ec_code_op_t *op;
+
+    op = ec_code_op_next(builder);
+
+    op->op = EC_GF_OP_XORM;
+    ec_code_arg_update(builder, op, &op->arg1, builder->map[bit]);
+    ec_code_arg_set(&op->arg2, offset);
+    ec_code_arg_set(&op->arg3, bit);
+}
+
+static void
+ec_code_dup(ec_code_builder_t *builder, ec_gf_op_t *op)
+{
+    switch (op->op) {
+        case EC_GF_OP_COPY:
+            ec_code_copy(builder, op->arg1, op->arg2);
+            break;
+        case EC_GF_OP_XOR2:
+            ec_code_xor2(builder, op->arg1, op->arg2);
+            break;
+        case EC_GF_OP_XOR3:
+            ec_code_xor3(builder, op->arg1, op->arg2, op->arg3);
+            break;
+        default:
+            break;
+    }
+}
+
+static void
+ec_code_gf_load(ec_code_builder_t *builder, uint32_t offset)
+{
+    uint32_t i;
+
+    for (i = 0; i < builder->code->gf->bits; i++) {
+        ec_code_load(builder, i, offset);
+    }
+}
+
+static void
+ec_code_gf_load_xor(ec_code_builder_t *builder, uint32_t offset)
+{
+    uint32_t i;
+
+    for (i = 0; i < builder->code->gf->bits; i++) {
+        ec_code_xorm(builder, i, offset);
+    }
+}
+
+static void
+ec_code_gf_store(ec_code_builder_t *builder)
+{
+    uint32_t i;
+
+    for (i = 0; i < builder->code->gf->bits; i++) {
+        ec_code_store(builder, i, i);
+    }
+}
+
+static void
+ec_code_gf_clear(ec_code_builder_t *builder)
+{
+    uint32_t i;
+
+    ec_code_xor2(builder, 0, 0);
+    for (i = 0; i < builder->code->gf->bits; i++) {
+        ec_code_store(builder, 0, i);
+    }
+}
+
+static void
+ec_code_gf_mul(ec_code_builder_t *builder, uint32_t value)
+{
+    ec_gf_mul_t *mul;
+    ec_gf_op_t *op;
+    uint32_t map[EC_GF_MAX_REGS];
+    int32_t i;
+
+    mul = builder->code->gf->table[value];
+    for (op = mul->ops; op->op != EC_GF_OP_END; op++) {
+        ec_code_dup(builder, op);
+    }
+
+    for (i = 0; i < mul->regs; i++) {
+        map[i] = builder->map[mul->map[i]];
+    }
+    memcpy(builder->map, map, sizeof(uint32_t) * mul->regs);
+}
+
+static ec_code_builder_t *
+ec_code_prepare(ec_code_t *code, uint32_t count, uint32_t width,
+                gf_boolean_t linear)
+{
+    ec_code_builder_t *builder;
+    uint32_t i;
+
+    count *= code->gf->bits + code->gf->max_ops;
+    count += code->gf->bits;
+    builder = GF_MALLOC(
+        sizeof(ec_code_builder_t) + sizeof(ec_code_op_t) * count,
+        ec_mt_ec_code_builder_t);
+    if (builder == NULL) {
+        return EC_ERR(ENOMEM);
+    }
+
+    builder->address = 0;
+    builder->code = code;
+    builder->size = 0;
+    builder->count = 0;
+    builder->regs = 0;
+    builder->error = 0;
+    builder->bits = code->gf->bits;
+    builder->width = width;
+    builder->data = NULL;
+    builder->linear = linear;
+    builder->base = -1;
+
+    for (i = 0; i < EC_GF_MAX_REGS; i++) {
+        builder->map[i] = i;
+    }
+
+    return builder;
+}
+
+static size_t
+ec_code_space_size(void)
+{
+    return (sizeof(ec_code_space_t) + 15) & ~15;
+}
+
+static size_t
+ec_code_chunk_size(void)
+{
+    return (sizeof(ec_code_chunk_t) + 15) & ~15;
+}
+
+static ec_code_chunk_t *
+ec_code_chunk_from_space(ec_code_space_t *space)
+{
+    return (ec_code_chunk_t *)((uintptr_t)space + ec_code_space_size());
+}
+
+static void *
+ec_code_to_executable(ec_code_space_t *space, void *addr)
+{
+    return (void *)((uintptr_t)addr - (uintptr_t)space +
+                    (uintptr_t)space->exec);
+}
+
+static void *
+ec_code_from_executable(ec_code_space_t *space, void *addr)
+{
+    return (void *)((uintptr_t)addr - (uintptr_t)space->exec +
+                    (uintptr_t)space);
+}
+
+static void *
+ec_code_func_from_chunk(ec_code_chunk_t *chunk, void **exec)
+{
+    void *addr;
+
+    addr = (void *)((uintptr_t)chunk + ec_code_chunk_size());
+
+    *exec = ec_code_to_executable(chunk->space, addr);
+
+    return addr;
+}
+
+static ec_code_chunk_t *
+ec_code_chunk_from_func(ec_code_func_linear_t func)
+{
+    ec_code_chunk_t *chunk;
+
+    chunk = (ec_code_chunk_t *)((uintptr_t)func - ec_code_chunk_size());
+
+    return ec_code_from_executable(chunk->space, chunk);
+}
+
+static ec_code_chunk_t *
+ec_code_chunk_split(ec_code_chunk_t *chunk, size_t size)
+{
+    ec_code_chunk_t *extra;
+    ssize_t avail;
+
+    avail = chunk->size - size - ec_code_chunk_size();
+    if (avail > 0) {
+        extra = (ec_code_chunk_t *)((uintptr_t)chunk + chunk->size - avail);
+        extra->space = chunk->space;
+        extra->size = avail;
+        list_add(&extra->list, &chunk->list);
+        chunk->size = size;
+    }
+    list_del_init(&chunk->list);
+
+    return chunk;
+}
+
+static gf_boolean_t
+ec_code_chunk_touch(ec_code_chunk_t *prev, ec_code_chunk_t *next)
+{
+    uintptr_t end;
+
+    end = (uintptr_t)prev + ec_code_chunk_size() + prev->size;
+    return (end == (uintptr_t)next);
+}
+
+static ec_code_space_t *
+ec_code_space_create(ec_code_t *code, size_t size)
+{
+    char path[] = GLUSTERFS_LIBEXECDIR "/ec-code-dynamic.XXXXXX";
+    ec_code_space_t *space;
+    void *exec;
+    int32_t fd, err;
+
+    /* We need to create memory areas to store the generated dynamic code.
+     * Obviously these areas need to be written to be able to create the
+     * code and they also need to be executable to execute it.
+     *
+     * However it's a bad practice to have a memory region that is both
+     * writable *and* executable. In fact, selinux forbids this and causes
+     * attempts to do so to fail (unless specifically configured).
+     *
+     * To solve the problem we'll use two distinct memory areas mapped to
+     * the same physical storage. One of the memory areas will have write
+     * permission, and the other will have execute permission. Both areas
+     * will have the same contents. The physical storage will be a regular
+     * file that will be mmapped to both areas.
+     */
+
+    /* We need to create a temporary file as the backend storage for the
+     * memory mapped areas. */
+    /* coverity[secure_temp] mkstemp uses 0600 as the mode and is safe */
+    fd = mkstemp(path);
+    if (fd < 0) {
+        err = errno;
+        gf_msg(THIS->name, GF_LOG_ERROR, err, EC_MSG_DYN_CREATE_FAILED,
+               "Unable to create a temporary file for the ec dynamic "
+               "code");
+        space = EC_ERR(err);
+        goto done;
+    }
+    /* Once created we don't need to keep it in the file system. It will
+     * still exist until we close the last file descriptor or unmap the
+     * memory areas bound to the file. */
+    sys_unlink(path);
+
+    size = (size + EC_CODE_ALIGN - 1) & ~(EC_CODE_ALIGN - 1);
+    if (sys_ftruncate(fd, size) < 0) {
+        err = errno;
+        gf_msg(THIS->name, GF_LOG_ERROR, err, EC_MSG_DYN_CREATE_FAILED,
+               "Unable to resize the file for the ec dynamic code");
+        space = EC_ERR(err);
+        goto done_close;
+    }
+
+    /* This creates an executable memory area to be able to run the
+     * generated fragments of code. */
+    exec = mmap(NULL, size, PROT_READ | PROT_EXEC, MAP_SHARED, fd, 0);
+    if (exec == MAP_FAILED) {
+        err = errno;
+        gf_msg(THIS->name, GF_LOG_ERROR, err, EC_MSG_DYN_CREATE_FAILED,
+               "Unable to map the executable area for the ec dynamic "
+               "code");
+        space = EC_ERR(err);
+        goto done_close;
+    }
+    /* It's not important to check the return value of mlock(). If it fails
+     * everything will continue to work normally. */
+    mlock(exec, size);
+
+    /* This maps a read/write memory area to be able to create the dynamici
+     * code. */
+    space = mmap(NULL, size, PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0);
+    if (space == MAP_FAILED) {
+        err = errno;
+        gf_msg(THIS->name, GF_LOG_ERROR, err, EC_MSG_DYN_CREATE_FAILED,
+               "Unable to map the writable area for the ec dynamic "
+               "code");
+        space = EC_ERR(err);
+
+        munmap(exec, size);
+
+        goto done_close;
+    }
+
+    space->exec = exec;
+    space->size = size;
+    space->code = code;
+    list_add_tail(&space->list, &code->spaces);
+    INIT_LIST_HEAD(&space->chunks);
+
+done_close:
+    /* If everything has succeeded, we already have the memory areas
+     * mapped. We don't need the file descriptor anymore because the
+     * backend storage will be there until the mmap()'d regions are
+     * unmapped. */
+    sys_close(fd);
+done:
+    return space;
+}
+
+static void
+ec_code_space_destroy(ec_code_space_t *space)
+{
+    list_del_init(&space->list);
+
+    munmap(space->exec, space->size);
+    munmap(space, space->size);
+}
+
+static void
+ec_code_chunk_merge(ec_code_chunk_t *chunk)
+{
+    ec_code_chunk_t *item, *tmp;
+
+    list_for_each_entry_safe(item, tmp, &chunk->space->chunks, list)
+    {
+        if ((uintptr_t)item > (uintptr_t)chunk) {
+            list_add_tail(&chunk->list, &item->list);
+            if (ec_code_chunk_touch(chunk, item)) {
+                chunk->size += item->size + ec_code_chunk_size();
+                list_del_init(&item->list);
+            }
+
+            goto check;
+        }
+        if (ec_code_chunk_touch(item, chunk)) {
+            item->size += chunk->size + ec_code_chunk_size();
+            list_del_init(&item->list);
+            chunk = item;
+        }
+    }
+    list_add_tail(&chunk->list, &chunk->space->chunks);
+
+check:
+    if (chunk->size ==
+        chunk->space->size - ec_code_space_size() - ec_code_chunk_size()) {
+        ec_code_space_destroy(chunk->space);
+    }
+}
+
+static ec_code_chunk_t *
+ec_code_space_alloc(ec_code_t *code, size_t size)
+{
+    ec_code_space_t *space;
+    ec_code_chunk_t *chunk;
+    size_t map_size;
+
+    /* To minimize fragmentation, we only allocate chunks of sizes multiples
+     * of EC_CODE_CHUNK_MIN_SIZE. */
+    size = ((size + ec_code_chunk_size() + EC_CODE_CHUNK_MIN_SIZE - 1) &
+            ~(EC_CODE_CHUNK_MIN_SIZE - 1)) -
+           ec_code_chunk_size();
+    list_for_each_entry(space, &code->spaces, list)
+    {
+        list_for_each_entry(chunk, &space->chunks, list)
+        {
+            if (chunk->size >= size) {
+                goto out;
+            }
+        }
+    }
+
+    map_size = EC_CODE_SIZE - ec_code_space_size() - ec_code_chunk_size();
+    if (map_size < size) {
+        map_size = size;
+    }
+    space = ec_code_space_create(code, map_size);
+    if (EC_IS_ERR(space)) {
+        return (ec_code_chunk_t *)space;
+    }
+
+    chunk = ec_code_chunk_from_space(space);
+    chunk->size = map_size - ec_code_space_size() - ec_code_chunk_size();
+    list_add(&chunk->list, &space->chunks);
+
+out:
+    chunk->space = space;
+
+    return ec_code_chunk_split(chunk, size);
+}
+
+static ec_code_chunk_t *
+ec_code_alloc(ec_code_t *code, uint32_t size)
+{
+    ec_code_chunk_t *chunk;
+
+    LOCK(&code->lock);
+
+    chunk = ec_code_space_alloc(code, size);
+
+    UNLOCK(&code->lock);
+
+    return chunk;
+}
+
+static void
+ec_code_free(ec_code_chunk_t *chunk)
+{
+    gf_lock_t *lock;
+
+    lock = &chunk->space->code->lock;
+    LOCK(lock);
+
+    ec_code_chunk_merge(chunk);
+
+    UNLOCK(lock);
+}
+
+static int32_t
+ec_code_write(ec_code_builder_t *builder)
+{
+    ec_code_gen_t *gen;
+    ec_code_op_t *op;
+    uint32_t i;
+
+    builder->error = 0;
+    builder->size = 0;
+    builder->address = 0;
+    builder->base = -1;
+
+    gen = builder->code->gen;
+    gen->prolog(builder);
+    for (i = 0; i < builder->count; i++) {
+        op = &builder->ops[i];
+        switch (op->op) {
+            case EC_GF_OP_LOAD:
+                gen->load(builder, op->arg1.value, op->arg2.value,
+                          op->arg3.value);
+                break;
+            case EC_GF_OP_STORE:
+                gen->store(builder, op->arg1.value, op->arg3.value);
+                break;
+            case EC_GF_OP_COPY:
+                gen->copy(builder, op->arg1.value, op->arg2.value);
+                break;
+            case EC_GF_OP_XOR2:
+                gen->xor2(builder, op->arg1.value, op->arg2.value);
+                break;
+            case EC_GF_OP_XOR3:
+                gen->xor3(builder, op->arg1.value, op->arg2.value,
+                          op->arg3.value);
+                break;
+            case EC_GF_OP_XORM:
+                gen->xorm(builder, op->arg1.value, op->arg2.value,
+                          op->arg3.value);
+                break;
+            default:
+                break;
+        }
+    }
+    gen->epilog(builder);
+
+    return builder->error;
+}
+
+static void *
+ec_code_compile(ec_code_builder_t *builder)
+{
+    ec_code_chunk_t *chunk;
+    void *func;
+    int32_t err;
+
+    err = ec_code_write(builder);
+    if (err != 0) {
+        return EC_ERR(err);
+    }
+
+    chunk = ec_code_alloc(builder->code, builder->size);
+    if (EC_IS_ERR(chunk)) {
+        return chunk;
+    }
+    builder->data = ec_code_func_from_chunk(chunk, &func);
+
+    err = ec_code_write(builder);
+    if (err != 0) {
+        ec_code_free(chunk);
+
+        return EC_ERR(err);
+    }
+
+    GF_FREE(builder);
+
+    return func;
+}
+
+ec_code_t *
+ec_code_create(ec_gf_t *gf, ec_code_gen_t *gen)
+{
+    ec_code_t *code;
+
+    code = GF_MALLOC(sizeof(ec_code_t), ec_mt_ec_code_t);
+    if (code == NULL) {
+        return EC_ERR(ENOMEM);
+    }
+    memset(code, 0, sizeof(ec_code_t));
+    INIT_LIST_HEAD(&code->spaces);
+    LOCK_INIT(&code->lock);
+
+    code->gf = gf;
+    code->gen = gen;
+
+    return code;
+}
+
+void
+ec_code_destroy(ec_code_t *code)
+{
+    if (!list_empty(&code->spaces)) {
+    }
+
+    LOCK_DESTROY(&code->lock);
+
+    GF_FREE(code);
+}
+
+static uint32_t
+ec_code_value_next(uint32_t *values, uint32_t count, uint32_t *offset)
+{
+    uint32_t i, next;
+
+    next = 0;
+    for (i = *offset + 1; i < count; i++) {
+        next = values[i];
+        if (next != 0) {
+            break;
+        }
+    }
+    *offset = i;
+
+    return next;
+}
+
+static void *
+ec_code_build_dynamic(ec_code_t *code, uint32_t width, uint32_t *values,
+                      uint32_t count, gf_boolean_t linear)
+{
+    ec_code_builder_t *builder;
+    uint32_t offset, val, next;
+
+    builder = ec_code_prepare(code, count, width, linear);
+    if (EC_IS_ERR(builder)) {
+        return builder;
+    }
+
+    offset = -1;
+    next = ec_code_value_next(values, count, &offset);
+    if (next != 0) {
+        ec_code_gf_load(builder, offset);
+        do {
+            val = next;
+            next = ec_code_value_next(values, count, &offset);
+            if (next != 0) {
+                ec_code_gf_mul(builder, ec_gf_div(code->gf, val, next));
+                ec_code_gf_load_xor(builder, offset);
+            }
+        } while (next != 0);
+        ec_code_gf_mul(builder, val);
+        ec_code_gf_store(builder);
+    } else {
+        ec_code_gf_clear(builder);
+    }
+
+    return ec_code_compile(builder);
+}
+
+static void *
+ec_code_build(ec_code_t *code, uint32_t width, uint32_t *values, uint32_t count,
+              gf_boolean_t linear)
+{
+    void *func;
+
+    if (code->gen != NULL) {
+        func = ec_code_build_dynamic(code, width, values, count, linear);
+        if (!EC_IS_ERR(func)) {
+            return func;
+        }
+
+        gf_msg_debug(THIS->name, GF_LOG_DEBUG,
+                     "Unable to generate dynamic code. Falling back "
+                     "to precompiled code");
+
+        /* The dynamic code generation shouldn't fail in normal
+         * conditions, but if it fails at some point, it's very
+         * probable that it will fail again, so we completely disable
+         * dynamic code generation. */
+        code->gen = NULL;
+    }
+
+    ec_code_c_prepare(code->gf, values, count);
+
+    if (linear) {
+        return ec_code_c_linear;
+    }
+
+    return ec_code_c_interleaved;
+}
+
+ec_code_func_linear_t
+ec_code_build_linear(ec_code_t *code, uint32_t width, uint32_t *values,
+                     uint32_t count)
+{
+    return (ec_code_func_linear_t)ec_code_build(code, width, values, count,
+                                                _gf_true);
+}
+
+ec_code_func_interleaved_t
+ec_code_build_interleaved(ec_code_t *code, uint32_t width, uint32_t *values,
+                          uint32_t count)
+{
+    return (ec_code_func_interleaved_t)ec_code_build(code, width, values, count,
+                                                     _gf_false);
+}
+
+void
+ec_code_release(ec_code_t *code, ec_code_func_t *func)
+{
+    if ((func->linear != ec_code_c_linear) &&
+        (func->interleaved != ec_code_c_interleaved)) {
+        ec_code_free(ec_code_chunk_from_func(func->linear));
+    }
+}
+
+void
+ec_code_error(ec_code_builder_t *builder, int32_t error)
+{
+    if (builder->error == 0) {
+        gf_msg(THIS->name, GF_LOG_ERROR, error, EC_MSG_DYN_CODEGEN_FAILED,
+               "Failed to generate dynamic code");
+        builder->error = error;
+    }
+}
+
+void
+ec_code_emit(ec_code_builder_t *builder, uint8_t *bytes, uint32_t count)
+{
+    if (builder->error != 0) {
+        return;
+    }
+
+    if (builder->data != NULL) {
+        memcpy(builder->data + builder->size, bytes, count);
+    }
+
+    builder->size += count;
+    builder->address += count;
+}
+
+static char *
+ec_code_proc_trim_left(char *text, ssize_t *length)
+{
+    ssize_t len;
+
+    for (len = *length; (len > 0) && isspace(*text); len--) {
+        text++;
+    }
+    *length = len;
+
+    return text;
+}
+
+static char *
+ec_code_proc_trim_right(char *text, ssize_t *length, char sep)
+{
+    char *last;
+    ssize_t len;
+
+    len = *length;
+
+    last = text;
+    for (len = *length; (len > 0) && (*text != sep); len--) {
+        if (!isspace(*text)) {
+            last = text + 1;
+        }
+        text++;
+    }
+    *last = 0;
+    *length = len;
+
+    return text;
+}
+
+static char *
+ec_code_proc_line_parse(ec_code_proc_t *file, ssize_t *length)
+{
+    char *text, *end;
+    ssize_t len;
+
+    len = file->size - file->pos;
+    text = ec_code_proc_trim_left(file->buffer + file->pos, &len);
+    end = ec_code_proc_trim_right(text, &len, '\n');
+    if (len == 0) {
+        if (!file->eof) {
+            if (text == file->buffer) {
+                file->size = file->pos = 0;
+                file->skip = _gf_true;
+            } else {
+                file->size = file->pos = end - text;
+                memmove(file->buffer, text, file->pos + 1);
+            }
+            len = sys_read(file->fd, file->buffer + file->pos,
+                           sizeof(file->buffer) - file->pos - 1);
+            if (len > 0) {
+                file->size += len;
+            }
+            file->error = len < 0;
+            file->eof = len <= 0;
+
+            return NULL;
+        }
+        file->size = file->pos = 0;
+    } else {
+        file->pos = end - file->buffer + 1;
+    }
+
+    *length = end - text;
+
+    if (file->skip) {
+        file->skip = _gf_false;
+        text = NULL;
+    }
+
+    return text;
+}
+
+static char *
+ec_code_proc_line(ec_code_proc_t *file, ssize_t *length)
+{
+    char *text;
+
+    text = NULL;
+    while (!file->eof) {
+        text = ec_code_proc_line_parse(file, length);
+        if (text != NULL) {
+            break;
+        }
+    }
+
+    return text;
+}
+
+static char *
+ec_code_proc_split(char *text, ssize_t *length, char sep)
+{
+    text = ec_code_proc_trim_right(text, length, sep);
+    if (*length == 0) {
+        return NULL;
+    }
+    (*length)--;
+    text++;
+
+    return ec_code_proc_trim_left(text, length);
+}
+
+static uint32_t
+ec_code_cpu_check(uint32_t idx, char *list, uint32_t count)
+{
+    ec_code_gen_t *gen;
+    char **ptr;
+    char *table[count + 1];
+    uint32_t i;
+
+    for (i = 0; i < count; i++) {
+        table[i] = list;
+        list += strlen(list) + 1;
+    }
+
+    gen = ec_code_gen_table[idx];
+    while (gen != NULL) {
+        for (ptr = gen->flags; *ptr != NULL; ptr++) {
+            for (i = 0; i < count; i++) {
+                if (strcmp(*ptr, table[i]) == 0) {
+                    break;
+                }
+            }
+            if (i >= count) {
+                gen = ec_code_gen_table[++idx];
+                break;
+            }
+        }
+        if (*ptr == NULL) {
+            break;
+        }
+    }
+
+    return idx;
+}
+
+ec_code_gen_t *
+ec_code_detect(xlator_t *xl, const char *def)
+{
+    ec_code_proc_t file;
+    ec_code_gen_t *gen = NULL;
+    char *line, *data, *list;
+    ssize_t length;
+    uint32_t count, base, select;
+
+    if (strcmp(def, "none") == 0) {
+        gf_msg(xl->name, GF_LOG_INFO, 0, EC_MSG_EXTENSION_NONE,
+               "Not using any cpu extensions");
+
+        return NULL;
+    }
+
+    file.fd = sys_open(PROC_CPUINFO, O_RDONLY, 0);
+    if (file.fd < 0) {
+        goto out;
+    }
+    file.size = file.pos = 0;
+    file.eof = file.error = file.skip = _gf_false;
+
+    select = 0;
+    if (strcmp(def, "auto") != 0) {
+        while (ec_code_gen_table[select] != NULL) {
+            if (strcmp(ec_code_gen_table[select]->name, def) == 0) {
+                break;
+            }
+            select++;
+        }
+        if (ec_code_gen_table[select] == NULL) {
+            gf_msg(xl->name, GF_LOG_WARNING, EINVAL, EC_MSG_EXTENSION_UNKNOWN,
+                   "CPU extension '%s' is not known. Not using any cpu "
+                   "extensions",
+                   def);
+
+            return NULL;
+        }
+    } else {
+        def = NULL;
+    }
+
+    while ((line = ec_code_proc_line(&file, &length)) != NULL) {
+        data = ec_code_proc_split(line, &length, ':');
+        if ((data != NULL) && (strcmp(line, "flags") == 0)) {
+            list = data;
+            count = 0;
+            while ((data != NULL) && (*data != 0)) {
+                count++;
+                data = ec_code_proc_split(data, &length, ' ');
+            }
+            base = select;
+            select = ec_code_cpu_check(select, list, count);
+            if ((base != select) && (def != NULL)) {
+                gf_msg(xl->name, GF_LOG_WARNING, ENOTSUP,
+                       EC_MSG_EXTENSION_UNSUPPORTED,
+                       "CPU extension '%s' is not supported", def);
+                def = NULL;
+            }
+        }
+    }
+
+    if (file.error) {
+        gf_msg(xl->name, GF_LOG_WARNING, 0, EC_MSG_EXTENSION_FAILED,
+               "Unable to determine supported CPU extensions. Not using any "
+               "cpu extensions");
+
+        gen = NULL;
+    } else {
+        gen = ec_code_gen_table[select];
+        if (gen == NULL) {
+            gf_msg(xl->name, GF_LOG_INFO, 0, EC_MSG_EXTENSION_NONE,
+                   "Not using any cpu extensions");
+        } else {
+            gf_msg(xl->name, GF_LOG_INFO, 0, EC_MSG_EXTENSION,
+                   "Using '%s' CPU extensions", gen->name);
+        }
+    }
+
+    sys_close(file.fd);
+
+out:
+    return gen;
+}
diff --git a/xlators/cluster/ec/src/ec-code.h b/xlators/cluster/ec/src/ec-code.h
new file mode 100644
index 00000000000..75fb35d93e3
--- /dev/null
+++ b/xlators/cluster/ec/src/ec-code.h
@@ -0,0 +1,44 @@
+/*
+  Copyright (c) 2015 DataLab, s.l. <http://www.datalab.es>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
+*/
+
+#ifndef __EC_CODE_H__
+#define __EC_CODE_H__
+
+#include <glusterfs/xlator.h>
+#include <glusterfs/list.h>
+
+#include "ec-types.h"
+#include "ec-galois.h"
+
+ec_code_gen_t *
+ec_code_detect(xlator_t *xl, const char *def);
+
+ec_code_t *
+ec_code_create(ec_gf_t *gf, ec_code_gen_t *gen);
+
+void
+ec_code_destroy(ec_code_t *code);
+
+ec_code_func_linear_t
+ec_code_build_linear(ec_code_t *code, uint32_t width, uint32_t *values,
+                     uint32_t count);
+ec_code_func_interleaved_t
+ec_code_build_interleaved(ec_code_t *code, uint32_t width, uint32_t *values,
+                          uint32_t count);
+void
+ec_code_release(ec_code_t *code, ec_code_func_t *func);
+
+void
+ec_code_error(ec_code_builder_t *builder, int32_t error);
+
+void
+ec_code_emit(ec_code_builder_t *builder, uint8_t *bytes, uint32_t count);
+
+#endif /* __EC_CODE_H__ */
diff --git a/xlators/cluster/ec/src/ec-combine.c b/xlators/cluster/ec/src/ec-combine.c
index 5f5d97275be..703a30e2485 100644
--- a/xlators/cluster/ec/src/ec-combine.c
+++ b/xlators/cluster/ec/src/ec-combine.c
@@ -11,56 +11,57 @@
 #include <fnmatch.h>
 
 #include "libxlator.h"
-#include "byte-order.h"
+#include <glusterfs/byte-order.h>
 
-#include "ec-data.h"
+#include "ec-types.h"
 #include "ec-helpers.h"
 #include "ec-common.h"
 #include "ec-combine.h"
+#include "ec-messages.h"
+#include <glusterfs/quota-common-utils.h>
 
 #define EC_QUOTA_PREFIX "trusted.glusterfs.quota."
 
+#define EC_MISSING_DATA ((data_t *)1ULL)
+
 struct _ec_dict_info;
 typedef struct _ec_dict_info ec_dict_info_t;
 
 struct _ec_dict_combine;
 typedef struct _ec_dict_combine ec_dict_combine_t;
 
-struct _ec_dict_info
-{
-    dict_t * dict;
-    int32_t  count;
+struct _ec_dict_info {
+    dict_t *dict;
+    int32_t count;
 };
 
-struct _ec_dict_combine
-{
-    ec_cbk_data_t * cbk;
-    int32_t         which;
+struct _ec_dict_combine {
+    ec_cbk_data_t *cbk;
+    int32_t which;
 };
 
 int32_t
-ec_combine_write (ec_fop_data_t *fop, ec_cbk_data_t *dst,
-                  ec_cbk_data_t *src)
+ec_combine_write(ec_fop_data_t *fop, ec_cbk_data_t *dst, ec_cbk_data_t *src)
 {
-        int     valid = 0;
+    int valid = 0;
 
-        if (!fop || !dst || !src)
-                return 0;
+    if (!fop || !dst || !src)
+        return 0;
 
-        switch (fop->id) {
+    switch (fop->id) {
         case GF_FOP_REMOVEXATTR:
         case GF_FOP_FREMOVEXATTR:
         case GF_FOP_SETXATTR:
         case GF_FOP_FSETXATTR:
-                return 1;
+            return 1;
 
         case GF_FOP_SYMLINK:
         case GF_FOP_LINK:
         case GF_FOP_CREATE:
         case GF_FOP_MKNOD:
         case GF_FOP_MKDIR:
-                valid = 3;
-                break;
+            valid = 3;
+            break;
         case GF_FOP_UNLINK:
         case GF_FOP_RMDIR:
         case GF_FOP_SETATTR:
@@ -71,75 +72,129 @@ ec_combine_write (ec_fop_data_t *fop, ec_cbk_data_t *dst,
         case GF_FOP_FALLOCATE:
         case GF_FOP_DISCARD:
         case GF_FOP_ZEROFILL:
-                valid = 2;
-                break;
+            valid = 2;
+            break;
         case GF_FOP_RENAME:
-                valid = 5;
-                break;
+            valid = 5;
+            break;
         default:
-                gf_log_callingfn (fop->xl->name, GF_LOG_WARNING, "Invalid fop "
-                                  "%d", fop->id);
-                return 0;
-                break;
-        }
-
-        if (!ec_iatt_combine(dst->iatt, src->iatt, valid)) {
-                gf_log(fop->xl->name, GF_LOG_NOTICE, "Mismatching iatt in "
-                       "answers of '%s'", gf_fop_list[fop->id]);
+            gf_msg_callingfn(fop->xl->name, GF_LOG_WARNING, EINVAL,
+                             EC_MSG_INVALID_FOP, "Invalid fop %d", fop->id);
+            return 0;
+            break;
+    }
 
-                return 0;
-        }
-        return 1;
+    if (!ec_iatt_combine(fop, dst->iatt, src->iatt, valid)) {
+        gf_msg(fop->xl->name, GF_LOG_NOTICE, 0, EC_MSG_IATT_MISMATCH,
+               "Mismatching iatt in "
+               "answers of '%s'",
+               gf_fop_list[fop->id]);
+        return 0;
+    }
+    return 1;
 }
 
-void ec_iatt_time_merge(uint32_t * dst_sec, uint32_t * dst_nsec,
-                        uint32_t src_sec, uint32_t src_nsec)
+void
+ec_iatt_time_merge(int64_t *dst_sec, uint32_t *dst_nsec, int64_t src_sec,
+                   uint32_t src_nsec)
 {
     if ((*dst_sec < src_sec) ||
-        ((*dst_sec == src_sec) && (*dst_nsec < src_nsec)))
-    {
+        ((*dst_sec == src_sec) && (*dst_nsec < src_nsec))) {
         *dst_sec = src_sec;
         *dst_nsec = src_nsec;
     }
 }
 
-int32_t ec_iatt_combine(struct iatt * dst, struct iatt * src, int32_t count)
+static gf_boolean_t
+ec_iatt_is_trusted(ec_fop_data_t *fop, struct iatt *iatt)
 {
+    uint64_t ino;
     int32_t i;
 
-    for (i = 0; i < count; i++)
-    {
+    /* Only the top level fop will have fop->locks filled. */
+    while (fop->parent != NULL) {
+        fop = fop->parent;
+    }
+
+    /* Lookups are special requests always done without locks taken but they
+     * require to be able to identify differences between bricks. Special
+     * handling of these differences is already done in lookup specific code
+     * so we shouldn't ignore any difference here and consider all iatt
+     * structures as trusted. */
+    if (fop->id == GF_FOP_LOOKUP) {
+        return _gf_true;
+    }
+
+    /* Check if the iatt references an inode locked by the current fop */
+    for (i = 0; i < fop->lock_count; i++) {
+        ino = gfid_to_ino(fop->locks[i].lock->loc.inode->gfid);
+        if (iatt->ia_ino == ino) {
+            return _gf_true;
+        }
+    }
+
+    return _gf_false;
+}
+
+int32_t
+ec_iatt_combine(ec_fop_data_t *fop, struct iatt *dst, struct iatt *src,
+                int32_t count)
+{
+    int32_t i;
+    gf_boolean_t failed = _gf_false;
+
+    for (i = 0; i < count; i++) {
+        /* Check for basic fields. These fields must be equal always, even if
+         * the inode is not locked because in these cases the parent inode
+         * will be locked and differences in these fields require changes in
+         * the parent directory. */
         if ((dst[i].ia_ino != src[i].ia_ino) ||
-            (dst[i].ia_uid != src[i].ia_uid) ||
-            (dst[i].ia_gid != src[i].ia_gid) ||
             (((dst[i].ia_type == IA_IFBLK) || (dst[i].ia_type == IA_IFCHR)) &&
              (dst[i].ia_rdev != src[i].ia_rdev)) ||
-            ((dst[i].ia_type == IA_IFREG) &&
-             (dst[i].ia_size != src[i].ia_size)) ||
-            (st_mode_from_ia(dst[i].ia_prot, dst[i].ia_type) !=
-             st_mode_from_ia(src[i].ia_prot, src[i].ia_type)) ||
-            (gf_uuid_compare(dst[i].ia_gfid, src[i].ia_gfid) != 0))
-        {
-            gf_log(THIS->name, GF_LOG_WARNING,
-                   "Failed to combine iatt (inode: %lu-%lu, links: %u-%u, "
-                   "uid: %u-%u, gid: %u-%u, rdev: %lu-%lu, size: %lu-%lu, "
-                   "mode: %o-%o)",
+            (gf_uuid_compare(dst[i].ia_gfid, src[i].ia_gfid) != 0)) {
+            failed = _gf_true;
+        }
+        /* Check for not so stable fields. These fields can change if the
+         * inode is not locked. */
+        if (!failed && ((dst[i].ia_uid != src[i].ia_uid) ||
+                        (dst[i].ia_gid != src[i].ia_gid) ||
+                        (st_mode_from_ia(dst[i].ia_prot, dst[i].ia_type) !=
+                         st_mode_from_ia(src[i].ia_prot, src[i].ia_type)))) {
+            if (ec_iatt_is_trusted(fop, dst)) {
+                /* If the iatt contains information from an inode that is
+                 * locked, these differences are real problems, so we need to
+                 * report them. Otherwise we ignore them and don't care which
+                 * data is returned. */
+                failed = _gf_true;
+            } else {
+                gf_msg_debug(fop->xl->name, 0,
+                             "Ignoring iatt differences because inode is not "
+                             "locked");
+            }
+        }
+        if (failed) {
+            gf_msg(fop->xl->name, GF_LOG_WARNING, 0, EC_MSG_IATT_COMBINE_FAIL,
+                   "Failed to combine iatt (inode: %" PRIu64 "-%" PRIu64
+                   ", "
+                   "links: %u-%u, uid: %u-%u, gid: %u-%u, "
+                   "rdev: %" PRIu64 "-%" PRIu64 ", size: %" PRIu64 "-%" PRIu64
+                   ", "
+                   "mode: %o-%o), %s",
                    dst[i].ia_ino, src[i].ia_ino, dst[i].ia_nlink,
-                   src[i].ia_nlink, dst[i].ia_uid, src[i].ia_uid,
-                   dst[i].ia_gid, src[i].ia_gid, dst[i].ia_rdev,
-                   src[i].ia_rdev, dst[i].ia_size, src[i].ia_size,
+                   src[i].ia_nlink, dst[i].ia_uid, src[i].ia_uid, dst[i].ia_gid,
+                   src[i].ia_gid, dst[i].ia_rdev, src[i].ia_rdev,
+                   dst[i].ia_size, src[i].ia_size,
                    st_mode_from_ia(dst[i].ia_prot, dst[i].ia_type),
-                   st_mode_from_ia(src[i].ia_prot, dst[i].ia_type));
+                   st_mode_from_ia(src[i].ia_prot, dst[i].ia_type),
+                   ec_msg_str(fop));
 
             return 0;
         }
     }
 
-    while (count-- > 0)
-    {
+    while (count-- > 0) {
         dst[count].ia_blocks += src[count].ia_blocks;
-        if (dst[count].ia_blksize < src[count].ia_blksize)
-        {
+        if (dst[count].ia_blksize < src[count].ia_blksize) {
             dst[count].ia_blksize = src[count].ia_blksize;
         }
 
@@ -154,293 +209,273 @@ int32_t ec_iatt_combine(struct iatt * dst, struct iatt * src, int32_t count)
     return 1;
 }
 
-void ec_iatt_rebuild(ec_t * ec, struct iatt * iatt, int32_t count,
-                     int32_t answers)
+void
+ec_iatt_rebuild(ec_t *ec, struct iatt *iatt, int32_t count, int32_t answers)
 {
     uint64_t blocks;
 
-    while (count-- > 0)
-    {
+    while (count-- > 0) {
         blocks = iatt[count].ia_blocks * ec->fragments + answers - 1;
         blocks /= answers;
         iatt[count].ia_blocks = blocks;
     }
 }
 
-int32_t ec_dict_data_compare(dict_t * dict, char * key, data_t * value,
-                             void * arg)
+gf_boolean_t
+ec_xattr_match(dict_t *dict, char *key, data_t *value, void *arg)
 {
-    ec_dict_info_t * info = arg;
-    data_t * data;
-
-    data = dict_get(info->dict, key);
-    if (data == NULL)
-    {
-        gf_log("ec", GF_LOG_DEBUG, "key '%s' found only on one dict", key);
-
-        return -1;
+    if ((fnmatch(GF_XATTR_STIME_PATTERN, key, 0) == 0) ||
+        (strcmp(key, GET_LINK_COUNT) == 0) ||
+        (strcmp(key, GLUSTERFS_INODELK_COUNT) == 0) ||
+        (strcmp(key, GLUSTERFS_ENTRYLK_COUNT) == 0) ||
+        (strcmp(key, GLUSTERFS_OPEN_FD_COUNT) == 0)) {
+        return _gf_false;
     }
 
-    info->count--;
+    return _gf_true;
+}
 
+gf_boolean_t
+ec_value_ignore(char *key)
+{
     if ((strcmp(key, GF_CONTENT_KEY) == 0) ||
         (strcmp(key, GF_XATTR_PATHINFO_KEY) == 0) ||
         (strcmp(key, GF_XATTR_USER_PATHINFO_KEY) == 0) ||
         (strcmp(key, GF_XATTR_LOCKINFO_KEY) == 0) ||
         (strcmp(key, GLUSTERFS_OPEN_FD_COUNT) == 0) ||
-        (strncmp(key, GF_XATTR_CLRLK_CMD, strlen(GF_XATTR_CLRLK_CMD)) == 0) ||
-        (strncmp(key, EC_QUOTA_PREFIX, strlen(EC_QUOTA_PREFIX)) == 0) ||
-        (fnmatch(GF_XATTR_STIME_PATTERN, key, 0) == 0) ||
+        (strcmp(key, GLUSTERFS_INODELK_COUNT) == 0) ||
+        (strcmp(key, GLUSTERFS_ENTRYLK_COUNT) == 0) ||
+        (strncmp(key, GF_XATTR_CLRLK_CMD, SLEN(GF_XATTR_CLRLK_CMD)) == 0) ||
+        (strcmp(key, DHT_IATT_IN_XDATA_KEY) == 0) ||
+        (strncmp(key, EC_QUOTA_PREFIX, SLEN(EC_QUOTA_PREFIX)) == 0) ||
         (fnmatch(MARKER_XATTR_PREFIX ".*." XTIME, key, 0) == 0) ||
         (fnmatch(GF_XATTR_MARKER_KEY ".*", key, 0) == 0) ||
-        (XATTR_IS_NODE_UUID(key)))
-    {
-        return 0;
-    }
-
-    if ((data->len != value->len) ||
-        (memcmp(data->data, value->data, data->len) != 0))
-    {
-        gf_log("ec", GF_LOG_DEBUG, "key '%s' is different (size: %u, %u)",
-               key, data->len, value->len);
-
-        return -1;
+        (XATTR_IS_NODE_UUID(key))) {
+        return _gf_true;
     }
 
-    return 0;
+    return _gf_false;
 }
 
-int32_t ec_dict_data_show(dict_t * dict, char * key, data_t * value,
-                          void * arg)
+int32_t
+ec_dict_compare(dict_t *dict1, dict_t *dict2)
 {
-    if (dict_get(arg, key) == NULL)
-    {
-        gf_log("ec", GF_LOG_DEBUG, "key '%s' found only on one dict", key);
-    }
-
+    if (are_dicts_equal(dict1, dict2, ec_xattr_match, ec_value_ignore))
+        return 1;
     return 0;
 }
 
-int32_t ec_dict_compare(dict_t * dict1, dict_t * dict2)
+static uint32_t
+ec_dict_list(data_t **list, ec_cbk_data_t *cbk, int32_t which, char *key,
+             gf_boolean_t global)
 {
-    ec_dict_info_t info;
-    dict_t * dict;
-
-    if (dict1 != NULL)
-    {
-        info.dict = dict1;
-        info.count = dict1->count;
-        dict = dict2;
-    }
-    else if (dict2 != NULL)
-    {
-        info.dict = dict2;
-        info.count = dict2->count;
-        dict = dict1;
-    }
-    else
-    {
-        return 1;
-    }
+    ec_t *ec = cbk->fop->xl->private;
+    ec_cbk_data_t *ans = NULL;
+    dict_t *dict = NULL;
+    data_t *data;
+    uint32_t count;
+    int32_t i;
 
-    if (dict != NULL)
-    {
-        if (dict_foreach(dict, ec_dict_data_compare, &info) != 0)
-        {
-            return 0;
+    for (i = 0; i < ec->nodes; i++) {
+        /* We initialize the list with EC_MISSING_DATA if we are
+         * returning a global list or the current subvolume belongs
+         * to the group of the accepted answer. Note that if some
+         * subvolume is known to be down before issuing the request,
+         * we won't have any answer from it, so we set here the
+         * appropriate default value. */
+        if (global || ((cbk->mask & (1ULL << i)) != 0)) {
+            list[i] = EC_MISSING_DATA;
+        } else {
+            list[i] = NULL;
         }
     }
 
-    if (info.count != 0)
-    {
-        dict_foreach(info.dict, ec_dict_data_show, dict);
-    }
-
-    return (info.count == 0);
-}
-
-int32_t ec_dict_list(data_t ** list, int32_t * count, ec_cbk_data_t * cbk,
-                     int32_t which, char * key)
-{
-    ec_cbk_data_t * ans;
-    dict_t * dict;
-    int32_t i, max;
-
-    max = *count;
-    i = 0;
-    for (ans = cbk; ans != NULL; ans = ans->next)
+    count = 0;
+    list_for_each_entry(ans, &cbk->fop->answer_list, answer_list)
     {
-        if (i >= max)
-        {
-            gf_log(cbk->fop->xl->name, GF_LOG_ERROR, "Unexpected number of "
-                                                     "dictionaries");
-
-            return 0;
-        }
-
-        dict = (which == EC_COMBINE_XDATA) ? ans->xdata : ans->dict;
-        list[i] = dict_get(dict, key);
-        if (list[i] == NULL)
-        {
-            gf_log(cbk->fop->xl->name, GF_LOG_ERROR, "Unexpected missing "
-                                                     "dictionary entry");
-
-            return 0;
+        if (global || ((cbk->mask & ans->mask) != 0)) {
+            dict = (which == EC_COMBINE_XDATA) ? ans->xdata : ans->dict;
+            data = dict_get(dict, key);
+            if (data != NULL) {
+                list[ans->idx] = data;
+                count++;
+            }
         }
-
-        i++;
     }
 
-    *count = i;
-
-    return 1;
+    return count;
 }
 
-char * ec_concat_prepare(xlator_t * xl, char ** sep, char ** post,
-                         const char * fmt, va_list args)
+int32_t
+ec_concat_prepare(xlator_t *xl, char **str, char **sep, char **post,
+                  const char *fmt, va_list args)
 {
-    char * str, * tmp;
+    char *tmp;
     int32_t len;
 
-    len = gf_vasprintf(&str, fmt, args);
-    if (len < 0)
-    {
-        return NULL;
+    len = gf_vasprintf(str, fmt, args);
+    if (len < 0) {
+        return -ENOMEM;
     }
 
-    tmp = strchr(str, '{');
-    if (tmp == NULL)
-    {
+    tmp = strchr(*str, '{');
+    if (tmp == NULL) {
         goto out;
     }
     *tmp++ = 0;
     *sep = tmp;
     tmp = strchr(tmp, '}');
-    if (tmp == NULL)
-    {
+    if (tmp == NULL) {
         goto out;
     }
     *tmp++ = 0;
     *post = tmp;
 
-    return str;
+    return 0;
 
 out:
-    gf_log(xl->name, GF_LOG_ERROR, "Invalid concat format");
+    gf_msg(xl->name, GF_LOG_ERROR, EINVAL, EC_MSG_INVALID_FORMAT,
+           "Invalid concat format");
 
-    GF_FREE(str);
+    GF_FREE(*str);
 
-    return NULL;
+    return -EINVAL;
 }
 
-int32_t ec_dict_data_concat(const char * fmt, ec_cbk_data_t * cbk,
-                            int32_t which, char * key, ...)
+static int32_t
+ec_dict_data_concat(ec_cbk_data_t *cbk, int32_t which, char *key, char *new_key,
+                    const char *def, gf_boolean_t global, const char *fmt, ...)
 {
-    data_t * data[cbk->count];
-    char * str = NULL, * pre = NULL, * sep, * post;
-    dict_t * dict;
+    ec_t *ec = cbk->fop->xl->private;
+    data_t *data[ec->nodes];
+    char *str = NULL, *pre = NULL, *sep, *post;
+    dict_t *dict;
     va_list args;
-    int32_t i, num, len, prelen, postlen, seplen, tmp;
-    int32_t ret = -1;
+    int32_t i, num, len, deflen, prelen, postlen, seplen, tmp;
+    int32_t err;
 
-    num = cbk->count;
-    if (!ec_dict_list(data, &num, cbk, which, key))
-    {
-        return -1;
-    }
+    ec_dict_list(data, cbk, which, key, global);
 
-    va_start(args, key);
-    pre = ec_concat_prepare(cbk->fop->xl, &sep, &post, fmt, args);
+    va_start(args, fmt);
+    err = ec_concat_prepare(cbk->fop->xl, &pre, &sep, &post, fmt, args);
     va_end(args);
 
-    if (pre == NULL)
-    {
-        return -1;
+    if (err != 0) {
+        return err;
     }
 
     prelen = strlen(pre);
     seplen = strlen(sep);
     postlen = strlen(post);
 
-    len = prelen + (num - 1) * seplen + postlen + 1;
-    for (i = 0; i < num; i++)
-    {
-        len += data[i]->len - 1;
+    deflen = 0;
+    if (def != NULL) {
+        deflen = strlen(def);
     }
 
+    len = prelen + postlen + 1;
+    num = -1;
+    for (i = 0; i < ec->nodes; i++) {
+        if (data[i] == NULL) {
+            continue;
+        }
+        if (data[i] == EC_MISSING_DATA) {
+            if (def == NULL) {
+                continue;
+            }
+            len += deflen;
+        } else {
+            len += data[i]->len - 1;
+        }
+        if (num >= 0) {
+            len += seplen;
+        }
+        num++;
+    }
+
+    err = -ENOMEM;
+
     str = GF_MALLOC(len, gf_common_mt_char);
-    if (str == NULL)
-    {
+    if (str == NULL) {
         goto out;
     }
 
     memcpy(str, pre, prelen);
     len = prelen;
-    for (i = 0; i < num; i++)
-    {
-        if (i > 0) {
+    for (i = 0; i < ec->nodes; i++) {
+        if (data[i] == NULL) {
+            continue;
+        }
+        if (data[i] == EC_MISSING_DATA) {
+            if (deflen == 0) {
+                continue;
+            }
+            tmp = deflen;
+            memcpy(str + len, def, tmp);
+        } else {
+            tmp = data[i]->len - 1;
+            memcpy(str + len, data[i]->data, tmp);
+        }
+        len += tmp;
+        if (i < num) {
             memcpy(str + len, sep, seplen);
             len += seplen;
         }
-        tmp = data[i]->len - 1;
-        memcpy(str + len, data[i]->data, tmp);
-        len += tmp;
     }
     memcpy(str + len, post, postlen + 1);
 
     dict = (which == EC_COMBINE_XDATA) ? cbk->xdata : cbk->dict;
-    if (dict_set_dynstr(dict, key, str) != 0)
-    {
+    if (new_key) {
+        key = new_key;
+    }
+    err = dict_set_dynstr(dict, key, str);
+    if (err != 0) {
         goto out;
     }
 
     str = NULL;
 
-    ret = 0;
-
 out:
     GF_FREE(str);
     GF_FREE(pre);
 
-    return ret;
+    return err;
 }
 
-int32_t ec_dict_data_merge(ec_cbk_data_t * cbk, int32_t which, char * key)
+int32_t
+ec_dict_data_merge(ec_cbk_data_t *cbk, int32_t which, char *key)
 {
-    data_t * data[cbk->count];
-    dict_t * dict, * lockinfo, * tmp;
-    char * ptr = NULL;
-    int32_t i, num, len;
-    int32_t ret = -1;
-
-    num = cbk->count;
-    if (!ec_dict_list(data, &num, cbk, which, key))
-    {
-        return -1;
-    }
+    ec_t *ec = cbk->fop->xl->private;
+    data_t *data[ec->nodes];
+    dict_t *dict, *lockinfo, *tmp = NULL;
+    char *ptr = NULL;
+    int32_t i, len;
+    int32_t err;
+
+    ec_dict_list(data, cbk, which, key, _gf_false);
 
     lockinfo = dict_new();
-    if (lockinfo == NULL)
-    {
-        return -1;
+    if (lockinfo == NULL) {
+        return -ENOMEM;
     }
 
-    if (dict_unserialize(data[0]->data, data[0]->len, &lockinfo) != 0)
-    {
-        goto out;
-    }
+    for (i = 0; i < ec->nodes; i++) {
+        if ((data[i] == NULL) || (data[i] == EC_MISSING_DATA)) {
+            continue;
+        }
 
-    for (i = 1; i < num; i++)
-    {
         tmp = dict_new();
-        if (tmp == NULL)
-        {
+        if (tmp == NULL) {
+            err = -ENOMEM;
+
             goto out;
         }
-        if ((dict_unserialize(data[i]->data, data[i]->len, &tmp) != 0) ||
-            (dict_copy(tmp, lockinfo) == NULL))
-        {
-            dict_unref(tmp);
+        err = dict_unserialize(data[i]->data, data[i]->len, &tmp);
+        if (err != 0) {
+            goto out;
+        }
+        if (dict_copy(tmp, lockinfo) == NULL) {
+            err = -ENOMEM;
 
             goto out;
         }
@@ -448,154 +483,190 @@ int32_t ec_dict_data_merge(ec_cbk_data_t * cbk, int32_t which, char * key)
         dict_unref(tmp);
     }
 
-    len = dict_serialized_length(lockinfo);
-    if (len < 0)
-    {
-        goto out;
-    }
-    ptr = GF_MALLOC(len, gf_common_mt_char);
-    if (ptr == NULL)
-    {
-        goto out;
-    }
-    if (dict_serialize(lockinfo, ptr) != 0)
-    {
+    tmp = NULL;
+
+    err = dict_allocate_and_serialize(lockinfo, (char **)&ptr,
+                                      (unsigned int *)&len);
+    if (err != 0) {
         goto out;
     }
+
     dict = (which == EC_COMBINE_XDATA) ? cbk->xdata : cbk->dict;
-    if (dict_set_dynptr(dict, key, ptr, len) != 0)
-    {
+    err = dict_set_dynptr(dict, key, ptr, len);
+    if (err != 0) {
         goto out;
     }
 
     ptr = NULL;
 
-    ret = 0;
-
 out:
     GF_FREE(ptr);
     dict_unref(lockinfo);
+    if (tmp != NULL) {
+        dict_unref(tmp);
+    }
 
-    return ret;
+    return err;
 }
 
-int32_t ec_dict_data_uuid(ec_cbk_data_t * cbk, int32_t which, char * key)
+int32_t
+ec_dict_data_uuid(ec_cbk_data_t *cbk, int32_t which, char *key)
 {
-    ec_cbk_data_t * ans, * min;
-    dict_t * src, * dst;
-    data_t * data;
+    ec_cbk_data_t *ans, *min;
+    dict_t *src, *dst;
+    data_t *data;
 
     min = cbk;
-    for (ans = cbk->next; ans != NULL; ans = ans->next)
-    {
-        if (ans->idx < min->idx)
-        {
+    for (ans = cbk->next; ans != NULL; ans = ans->next) {
+        if (ans->idx < min->idx) {
             min = ans;
         }
     }
 
-    if (min != cbk)
-    {
+    if (min != cbk) {
         src = (which == EC_COMBINE_XDATA) ? min->xdata : min->dict;
         dst = (which == EC_COMBINE_XDATA) ? cbk->xdata : cbk->dict;
 
         data = dict_get(src, key);
-        if (data == NULL)
-        {
-            return -1;
+        if (data == NULL) {
+            return -ENOENT;
         }
-        if (dict_set(dst, key, data) != 0)
-        {
-            return -1;
+        if (dict_set(dst, key, data) != 0) {
+            return -ENOMEM;
         }
     }
 
     return 0;
 }
 
-int32_t ec_dict_data_max32(ec_cbk_data_t *cbk, int32_t which, char *key)
+int32_t
+ec_dict_data_iatt(ec_cbk_data_t *cbk, int32_t which, char *key)
 {
-    data_t * data[cbk->count];
-    dict_t * dict;
-    int32_t i, num;
-    uint32_t max, tmp;
+    ec_t *ec = cbk->fop->xl->private;
+    data_t *data[ec->nodes];
+    dict_t *dict;
+    struct iatt *stbuf, *tmp;
+    int32_t i, ret;
 
-    num = cbk->count;
-    if (!ec_dict_list(data, &num, cbk, which, key))
-    {
-        return -1;
-    }
+    ec_dict_list(data, cbk, which, key, _gf_false);
 
-    if (num <= 1)
-    {
-        return 0;
+    stbuf = NULL;
+    for (i = 0; i < ec->nodes; i++) {
+        if ((data[i] == NULL) || (data[i] == EC_MISSING_DATA)) {
+            continue;
+        }
+        tmp = data_to_iatt(data[i], key);
+        if (tmp == NULL) {
+            ret = -EINVAL;
+            goto out;
+        }
+        if (stbuf == NULL) {
+            stbuf = GF_MALLOC(sizeof(struct iatt), gf_common_mt_char);
+            if (stbuf == NULL) {
+                ret = -ENOMEM;
+                goto out;
+            }
+            *stbuf = *tmp;
+        } else {
+            if (!ec_iatt_combine(cbk->fop, stbuf, tmp, 1)) {
+                ret = -EINVAL;
+                goto out;
+            }
+        }
     }
 
-    max = data_to_uint32(data[0]);
-    for (i = 1; i < num; i++)
-    {
-        tmp = data_to_uint32(data[i]);
-        if (max < tmp)
-        {
-            max = tmp;
+    if ((stbuf != NULL) && (stbuf->ia_type == IA_IFREG)) {
+        ec_iatt_rebuild(ec, stbuf, 1, cbk->count);
+        /* TODO: not sure if an iatt could come in xdata from a fop that takes
+         *       no locks. */
+        if (!ec_get_inode_size(cbk->fop, cbk->fop->locks[0].lock->loc.inode,
+                               &stbuf->ia_size)) {
+            ret = -EINVAL;
+            goto out;
         }
     }
 
     dict = (which == EC_COMBINE_XDATA) ? cbk->xdata : cbk->dict;
-    if (dict_set_uint32(dict, key, max) != 0)
-    {
-        return -1;
+    ret = dict_set_iatt(dict, key, stbuf, false);
+    if (ret >= 0) {
+        stbuf = NULL;
     }
 
-    return 0;
+out:
+    GF_FREE(stbuf);
+
+    return ret;
 }
 
-int32_t ec_dict_data_max64(ec_cbk_data_t *cbk, int32_t which, char *key)
+int32_t
+ec_dict_data_max32(ec_cbk_data_t *cbk, int32_t which, char *key)
 {
-    data_t *data[cbk->count];
+    ec_t *ec = cbk->fop->xl->private;
+    data_t *data[ec->nodes];
     dict_t *dict;
-    int32_t i, num;
-    uint64_t max, tmp;
+    int32_t i;
+    uint32_t max, tmp;
 
-    num = cbk->count;
-    if (!ec_dict_list(data, &num, cbk, which, key)) {
-        return -1;
-    }
+    ec_dict_list(data, cbk, which, key, _gf_false);
 
-    if (num <= 1) {
-        return 0;
-    }
+    max = 0;
+    for (i = 0; i < ec->nodes; i++) {
+        if ((data[i] == NULL) || (data[i] == EC_MISSING_DATA)) {
+            continue;
+        }
 
-    max = data_to_uint64(data[0]);
-    for (i = 1; i < num; i++) {
-        tmp = data_to_uint64(data[i]);
+        tmp = data_to_uint32(data[i]);
         if (max < tmp) {
             max = tmp;
         }
     }
 
     dict = (which == EC_COMBINE_XDATA) ? cbk->xdata : cbk->dict;
-    if (dict_set_uint64(dict, key, max) != 0) {
-        return -1;
-    }
-
-    return 0;
+    return dict_set_uint32(dict, key, max);
 }
 
-int32_t ec_dict_data_quota(ec_cbk_data_t *cbk, int32_t which, char *key)
+int32_t
+ec_dict_data_max64(ec_cbk_data_t *cbk, int32_t which, char *key)
 {
-    data_t *data[cbk->count];
+    ec_t *ec = cbk->fop->xl->private;
+    data_t *data[ec->nodes];
     dict_t *dict;
-    ec_t *ec;
-    int32_t i, num;
+    int32_t i;
     uint64_t max, tmp;
 
-    num = cbk->count;
-    if (!ec_dict_list(data, &num, cbk, which, key)) {
-        return -1;
+    ec_dict_list(data, cbk, which, key, _gf_false);
+
+    max = 0;
+    for (i = 0; i < ec->nodes; i++) {
+        if ((data[i] == NULL) || (data[i] == EC_MISSING_DATA)) {
+            continue;
+        }
+
+        tmp = data_to_uint64(data[i]);
+        if (max < tmp) {
+            max = tmp;
+        }
     }
 
-    if (num == 0) {
+    dict = (which == EC_COMBINE_XDATA) ? cbk->xdata : cbk->dict;
+    return dict_set_uint64(dict, key, max);
+}
+
+int32_t
+ec_dict_data_quota(ec_cbk_data_t *cbk, int32_t which, char *key)
+{
+    ec_t *ec = cbk->fop->xl->private;
+    data_t *data[ec->nodes];
+    dict_t *dict = NULL;
+    int32_t i = 0;
+    quota_meta_t size = {
+        0,
+    };
+    quota_meta_t max_size = {
+        0,
+    };
+
+    if (ec_dict_list(data, cbk, which, key, _gf_false) == 0) {
         return 0;
     }
 
@@ -604,77 +675,83 @@ int32_t ec_dict_data_quota(ec_cbk_data_t *cbk, int32_t which, char *key)
      * bricks and we can receive slightly different values. If that's the
      * case, we take the maximum of all received values.
      */
-    max = ntoh64(*(uint64_t *)data_to_ptr(data[0]));
-    for (i = 1; i < num; i++) {
-        tmp = ntoh64(*(uint64_t *)data_to_ptr(data[i]));
-        if (max < tmp) {
-            max = tmp;
+    for (i = 0; i < ec->nodes; i++) {
+        if ((data[i] == NULL) || (data[i] == EC_MISSING_DATA) ||
+            (quota_data_to_meta(data[i], &size) < 0)) {
+            continue;
         }
+
+        if (size.size > max_size.size)
+            max_size.size = size.size;
+        if (size.file_count > max_size.file_count)
+            max_size.file_count = size.file_count;
+        if (size.dir_count > max_size.dir_count)
+            max_size.dir_count = size.dir_count;
     }
 
-    ec = cbk->fop->xl->private;
-    max *= ec->fragments;
+    max_size.size *= ec->fragments;
 
     dict = (which == EC_COMBINE_XDATA) ? cbk->xdata : cbk->dict;
-    if (ec_dict_set_number(dict, key, max) != 0) {
-        return -1;
-    }
-
-    return 0;
+    return quota_dict_set_meta(dict, key, &max_size, IA_IFDIR);
 }
 
-int32_t ec_dict_data_stime(ec_cbk_data_t * cbk, int32_t which, char * key)
+int32_t
+ec_dict_data_stime(ec_cbk_data_t *cbk, int32_t which, char *key)
 {
-    data_t * data[cbk->count];
-    dict_t * dict;
-    int32_t i, num;
+    ec_t *ec = cbk->fop->xl->private;
+    data_t *data[ec->nodes];
+    dict_t *dict;
+    int32_t i, err;
 
-    num = cbk->count;
-    if (!ec_dict_list(data, &num, cbk, which, key))
-    {
-        return -1;
-    }
+    ec_dict_list(data, cbk, which, key, _gf_false);
 
     dict = (which == EC_COMBINE_XDATA) ? cbk->xdata : cbk->dict;
-    for (i = 1; i < num; i++)
-    {
-        if (gf_get_max_stime(cbk->fop->xl, dict, key, data[i]) != 0)
-        {
-            gf_log(cbk->fop->xl->name, GF_LOG_ERROR, "STIME combination "
-                                                     "failed");
+    for (i = 0; i < ec->nodes; i++) {
+        if ((data[i] == NULL) || (data[i] == EC_MISSING_DATA)) {
+            continue;
+        }
+        err = gf_get_max_stime(cbk->fop->xl, dict, key, data[i]);
+        if (err != 0) {
+            gf_msg(cbk->fop->xl->name, GF_LOG_ERROR, -err,
+                   EC_MSG_STIME_COMBINE_FAIL, "STIME combination failed");
 
-            return -1;
+            return err;
         }
     }
 
     return 0;
 }
 
-int32_t ec_dict_data_combine(dict_t * dict, char * key, data_t * value,
-                             void * arg)
+int32_t
+ec_dict_data_combine(dict_t *dict, char *key, data_t *value, void *arg)
 {
-    ec_dict_combine_t * data = arg;
+    ec_dict_combine_t *data = arg;
 
     if ((strcmp(key, GF_XATTR_PATHINFO_KEY) == 0) ||
-        (strcmp(key, GF_XATTR_USER_PATHINFO_KEY) == 0))
-    {
-        return ec_dict_data_concat("(<EC:%s> { })", data->cbk, data->which,
-                                   key, data->cbk->fop->xl->name);
+        (strcmp(key, GF_XATTR_USER_PATHINFO_KEY) == 0)) {
+        return ec_dict_data_concat(data->cbk, data->which, key, NULL, NULL,
+                                   _gf_false, _gf_false, "(<EC:%s> { })",
+                                   data->cbk->fop->xl->name);
     }
 
-    if (strncmp(key, GF_XATTR_CLRLK_CMD, strlen(GF_XATTR_CLRLK_CMD)) == 0)
-    {
-        return ec_dict_data_concat("{\n}", data->cbk, data->which, key);
+    if (strncmp(key, GF_XATTR_CLRLK_CMD, SLEN(GF_XATTR_CLRLK_CMD)) == 0) {
+        return ec_dict_data_concat(data->cbk, data->which, key, NULL, NULL,
+                                   _gf_false, "{\n}");
     }
 
-    if (strncmp(key, GF_XATTR_LOCKINFO_KEY,
-                strlen(GF_XATTR_LOCKINFO_KEY)) == 0)
-    {
+    if (strncmp(key, GF_XATTR_LOCKINFO_KEY, SLEN(GF_XATTR_LOCKINFO_KEY)) == 0) {
         return ec_dict_data_merge(data->cbk, data->which, key);
     }
 
-    if (strcmp(key, GLUSTERFS_OPEN_FD_COUNT) == 0)
-    {
+    if (strcmp(key, GET_LINK_COUNT) == 0) {
+        return ec_dict_data_max32(data->cbk, data->which, key);
+    }
+
+    if (strcmp(key, GLUSTERFS_OPEN_FD_COUNT) == 0) {
+        return ec_dict_data_max32(data->cbk, data->which, key);
+    }
+    if ((strcmp(key, GLUSTERFS_INODELK_COUNT) == 0) ||
+        (strcmp(key, GLUSTERFS_ENTRYLK_COUNT) == 0)) {
         return ec_dict_data_max32(data->cbk, data->which, key);
     }
 
@@ -682,17 +759,22 @@ int32_t ec_dict_data_combine(dict_t * dict, char * key, data_t * value,
         return ec_dict_data_quota(data->cbk, data->which, key);
     }
     /* Ignore all other quota attributes */
-    if (strncmp(key, EC_QUOTA_PREFIX, strlen(EC_QUOTA_PREFIX)) == 0) {
+    if (strncmp(key, EC_QUOTA_PREFIX, SLEN(EC_QUOTA_PREFIX)) == 0) {
         return 0;
     }
 
-    if (XATTR_IS_NODE_UUID(key))
-    {
-        return ec_dict_data_uuid(data->cbk, data->which, key);
+    if (XATTR_IS_NODE_UUID(key)) {
+        if (data->cbk->fop->int32) {
+            /* List of node uuid is requested */
+            return ec_dict_data_concat(data->cbk, data->which, key,
+                                       GF_XATTR_LIST_NODE_UUIDS_KEY, UUID0_STR,
+                                       _gf_true, "{ }");
+        } else {
+            return ec_dict_data_uuid(data->cbk, data->which, key);
+        }
     }
 
-    if (fnmatch(GF_XATTR_STIME_PATTERN, key, FNM_NOESCAPE) == 0)
-    {
+    if (fnmatch(GF_XATTR_STIME_PATTERN, key, FNM_NOESCAPE) == 0) {
         return ec_dict_data_stime(data->cbk, data->which, key);
     }
 
@@ -700,71 +782,74 @@ int32_t ec_dict_data_combine(dict_t * dict, char * key, data_t * value,
         return ec_dict_data_max64(data->cbk, data->which, key);
     }
 
+    if (strcmp(key, GF_PRESTAT) == 0 || strcmp(key, GF_POSTSTAT) == 0) {
+        return ec_dict_data_iatt(data->cbk, data->which, key);
+    }
+
     return 0;
 }
 
-int32_t ec_dict_combine(ec_cbk_data_t * cbk, int32_t which)
+int32_t
+ec_dict_combine(ec_cbk_data_t *cbk, int32_t which)
 {
-    dict_t * dict;
+    dict_t *dict = NULL;
     ec_dict_combine_t data;
+    int32_t err = 0;
 
     data.cbk = cbk;
     data.which = which;
 
     dict = (which == EC_COMBINE_XDATA) ? cbk->xdata : cbk->dict;
-    if ((dict != NULL) &&
-        (dict_foreach(dict, ec_dict_data_combine, &data) != 0))
-    {
-        gf_log(cbk->fop->xl->name, GF_LOG_ERROR, "Dictionary combination "
-                                                 "failed");
+    if (dict != NULL) {
+        err = dict_foreach(dict, ec_dict_data_combine, &data);
+        if (err != 0) {
+            gf_msg(cbk->fop->xl->name, GF_LOG_ERROR, -err,
+                   EC_MSG_DICT_COMBINE_FAIL, "Dictionary combination failed");
 
-        return 0;
+            return err;
+        }
     }
 
-    return 1;
+    return 0;
 }
 
-int32_t ec_vector_compare(struct iovec * dst_vector, int32_t dst_count,
-                          struct iovec * src_vector, int32_t src_count)
+int32_t
+ec_vector_compare(struct iovec *dst_vector, int32_t dst_count,
+                  struct iovec *src_vector, int32_t src_count)
 {
     int32_t dst_size = 0, src_size = 0;
 
-    if (dst_count > 0)
-    {
+    if (dst_count > 0) {
         dst_size = iov_length(dst_vector, dst_count);
     }
-    if (src_count > 0)
-    {
+    if (src_count > 0) {
         src_size = iov_length(src_vector, src_count);
     }
 
     return (dst_size == src_size);
 }
 
-int32_t ec_flock_compare(struct gf_flock * dst, struct gf_flock * src)
+int32_t
+ec_flock_compare(struct gf_flock *dst, struct gf_flock *src)
 {
-    if ((dst->l_type != src->l_type) ||
-        (dst->l_whence != src->l_whence) ||
-        (dst->l_start != src->l_start) ||
-        (dst->l_len != src->l_len) ||
+    if ((dst->l_type != src->l_type) || (dst->l_whence != src->l_whence) ||
+        (dst->l_start != src->l_start) || (dst->l_len != src->l_len) ||
         (dst->l_pid != src->l_pid) ||
-        !is_same_lkowner(&dst->l_owner, &src->l_owner))
-    {
+        !is_same_lkowner(&dst->l_owner, &src->l_owner)) {
         return 0;
     }
 
     return 1;
 }
 
-void ec_statvfs_combine(struct statvfs * dst, struct statvfs * src)
+void
+ec_statvfs_combine(struct statvfs *dst, struct statvfs *src)
 {
-    if (dst->f_bsize < src->f_bsize)
-    {
+    if (dst->f_bsize < src->f_bsize) {
         dst->f_bsize = src->f_bsize;
     }
 
-    if (dst->f_frsize < src->f_frsize)
-    {
+    if (dst->f_frsize < src->f_frsize) {
         dst->f_blocks *= dst->f_frsize;
         dst->f_blocks /= src->f_frsize;
 
@@ -775,9 +860,7 @@ void ec_statvfs_combine(struct statvfs * dst, struct statvfs * src)
         dst->f_bavail /= src->f_frsize;
 
         dst->f_frsize = src->f_frsize;
-    }
-    else if (dst->f_frsize > src->f_frsize)
-    {
+    } else if (dst->f_frsize > src->f_frsize) {
         src->f_blocks *= src->f_frsize;
         src->f_blocks /= dst->f_frsize;
 
@@ -787,111 +870,102 @@ void ec_statvfs_combine(struct statvfs * dst, struct statvfs * src)
         src->f_bavail *= src->f_frsize;
         src->f_bavail /= dst->f_frsize;
     }
-    if (dst->f_blocks > src->f_blocks)
-    {
+    if (dst->f_blocks > src->f_blocks) {
         dst->f_blocks = src->f_blocks;
     }
-    if (dst->f_bfree > src->f_bfree)
-    {
+    if (dst->f_bfree > src->f_bfree) {
         dst->f_bfree = src->f_bfree;
     }
-    if (dst->f_bavail > src->f_bavail)
-    {
+    if (dst->f_bavail > src->f_bavail) {
         dst->f_bavail = src->f_bavail;
     }
 
-    if (dst->f_files < src->f_files)
-    {
+    if (dst->f_files < src->f_files) {
         dst->f_files = src->f_files;
     }
-    if (dst->f_ffree > src->f_ffree)
-    {
+    if (dst->f_ffree > src->f_ffree) {
         dst->f_ffree = src->f_ffree;
     }
-    if (dst->f_favail > src->f_favail)
-    {
+    if (dst->f_favail > src->f_favail) {
         dst->f_favail = src->f_favail;
     }
-    if (dst->f_namemax > src->f_namemax)
-    {
+    if (dst->f_namemax > src->f_namemax) {
         dst->f_namemax = src->f_namemax;
     }
 
-    if (dst->f_flag != src->f_flag)
-    {
-        gf_log(THIS->name, GF_LOG_DEBUG, "Mismatching file system flags "
-                                         "(%lX, %lX)",
-               dst->f_flag, src->f_flag);
+    if (dst->f_flag != src->f_flag) {
+        gf_msg_debug(THIS->name, 0,
+                     "Mismatching file system flags "
+                     "(%lX, %lX)",
+                     dst->f_flag, src->f_flag);
     }
     dst->f_flag &= src->f_flag;
 }
 
-int32_t ec_combine_check(ec_cbk_data_t * dst, ec_cbk_data_t * src,
-                         ec_combine_f combine)
+int32_t
+ec_combine_check(ec_cbk_data_t *dst, ec_cbk_data_t *src, ec_combine_f combine)
 {
-    ec_fop_data_t * fop = dst->fop;
+    ec_fop_data_t *fop = dst->fop;
 
-    if (dst->op_ret != src->op_ret)
-    {
-        gf_log(fop->xl->name, GF_LOG_DEBUG, "Mismatching return code in "
-                                            "answers of '%s': %d <-> %d",
-               ec_fop_name(fop->id), dst->op_ret, src->op_ret);
+    if (dst->op_ret != src->op_ret) {
+        gf_msg_debug(fop->xl->name, 0,
+                     "Mismatching return code in "
+                     "answers of '%s': %d <-> %d",
+                     ec_fop_name(fop->id), dst->op_ret, src->op_ret);
 
         return 0;
     }
-    if (dst->op_ret < 0)
-    {
-        if (dst->op_errno != src->op_errno)
-        {
-            gf_log(fop->xl->name, GF_LOG_DEBUG, "Mismatching errno code in "
-                                                "answers of '%s': %d <-> %d",
-                   ec_fop_name(fop->id), dst->op_errno, src->op_errno);
+    if (dst->op_ret < 0) {
+        if (dst->op_errno != src->op_errno) {
+            gf_msg_debug(fop->xl->name, 0,
+                         "Mismatching errno code in "
+                         "answers of '%s': %d <-> %d",
+                         ec_fop_name(fop->id), dst->op_errno, src->op_errno);
 
             return 0;
         }
     }
 
-    if (!ec_dict_compare(dst->xdata, src->xdata))
-    {
-        gf_log(fop->xl->name, GF_LOG_WARNING, "Mismatching xdata in answers "
-                                              "of '%s'",
+    if (!ec_dict_compare(dst->xdata, src->xdata)) {
+        gf_msg(fop->xl->name, GF_LOG_DEBUG, 0, EC_MSG_XDATA_MISMATCH,
+               "Mismatching xdata in answers "
+               "of '%s'",
                ec_fop_name(fop->id));
 
         return 0;
     }
 
-    if ((dst->op_ret >= 0) && (combine != NULL))
-    {
+    if ((dst->op_ret >= 0) && (combine != NULL)) {
         return combine(fop, dst, src);
     }
 
     return 1;
 }
 
-void ec_combine (ec_cbk_data_t *newcbk, ec_combine_f combine)
+void
+ec_combine(ec_cbk_data_t *newcbk, ec_combine_f combine)
 {
     ec_fop_data_t *fop = newcbk->fop;
     ec_cbk_data_t *cbk = NULL, *tmp = NULL;
     struct list_head *item = NULL;
-    int32_t needed = 0, resume = 0;
+    int32_t needed = 0;
     char str[32];
 
     LOCK(&fop->lock);
 
+    fop->received |= newcbk->mask;
+
     item = fop->cbk_list.prev;
     list_for_each_entry(cbk, &fop->cbk_list, list)
     {
-        if (ec_combine_check(newcbk, cbk, combine))
-        {
+        if (ec_combine_check(newcbk, cbk, combine)) {
             newcbk->count += cbk->count;
             newcbk->mask |= cbk->mask;
 
             item = cbk->list.prev;
-            while (item != &fop->cbk_list)
-            {
+            while (item != &fop->cbk_list) {
                 tmp = list_entry(item, ec_cbk_data_t, list);
-                if (tmp->count >= newcbk->count)
-                {
+                if (tmp->count >= newcbk->count) {
                     break;
                 }
                 item = item->prev;
@@ -908,22 +982,14 @@ void ec_combine (ec_cbk_data_t *newcbk, ec_combine_f combine)
     ec_trace("ANSWER", fop, "combine=%s[%d]",
              ec_bin(str, sizeof(str), newcbk->mask, 0), newcbk->count);
 
-    if ((newcbk->count == fop->expected) && (fop->answer == NULL)) {
-        fop->answer = newcbk;
-
-        resume = 1;
-    }
-
     cbk = list_entry(fop->cbk_list.next, ec_cbk_data_t, list);
-    needed = fop->minimum - cbk->count - fop->winds + 1;
+    if ((fop->mask ^ fop->remaining) == fop->received) {
+        needed = fop->minimum - cbk->count;
+    }
 
     UNLOCK(&fop->lock);
 
     if (needed > 0) {
         ec_dispatch_next(fop, newcbk->idx);
-    } else if (resume) {
-        ec_update_bad(fop, newcbk->mask);
-
-        ec_resume(fop, 0);
     }
 }
diff --git a/xlators/cluster/ec/src/ec-combine.h b/xlators/cluster/ec/src/ec-combine.h
index cae2bb9274f..1010cc3be26 100644
--- a/xlators/cluster/ec/src/ec-combine.h
+++ b/xlators/cluster/ec/src/ec-combine.h
@@ -11,27 +11,34 @@
 #ifndef __EC_COMBINE_H__
 #define __EC_COMBINE_H__
 
-#define EC_COMBINE_DICT  0
+#define EC_COMBINE_DICT 0
 #define EC_COMBINE_XDATA 1
 
-typedef int32_t (* ec_combine_f)(ec_fop_data_t * fop, ec_cbk_data_t * dst,
-                                 ec_cbk_data_t * src);
+typedef int32_t (*ec_combine_f)(ec_fop_data_t *fop, ec_cbk_data_t *dst,
+                                ec_cbk_data_t *src);
 
-void ec_iatt_rebuild(ec_t * ec, struct iatt * iatt, int32_t count,
-                     int32_t answers);
+void
+ec_iatt_rebuild(ec_t *ec, struct iatt *iatt, int32_t count, int32_t answers);
 
-int32_t ec_iatt_combine(struct iatt * dst, struct iatt * src, int32_t count);
-int32_t ec_dict_compare(dict_t * dict1, dict_t * dict2);
-int32_t ec_vector_compare(struct iovec * dst_vector, int32_t dst_count,
-                          struct iovec * src_vector, int32_t src_count);
-int32_t ec_flock_compare(struct gf_flock * dst, struct gf_flock * src);
-void ec_statvfs_combine(struct statvfs * dst, struct statvfs * src);
+int32_t
+ec_iatt_combine(ec_fop_data_t *fop, struct iatt *dst, struct iatt *src,
+                int32_t count);
+int32_t
+ec_dict_compare(dict_t *dict1, dict_t *dict2);
+int32_t
+ec_vector_compare(struct iovec *dst_vector, int32_t dst_count,
+                  struct iovec *src_vector, int32_t src_count);
+int32_t
+ec_flock_compare(struct gf_flock *dst, struct gf_flock *src);
+void
+ec_statvfs_combine(struct statvfs *dst, struct statvfs *src);
 
-int32_t ec_dict_combine(ec_cbk_data_t * cbk, int32_t which);
+int32_t
+ec_dict_combine(ec_cbk_data_t *cbk, int32_t which);
 
-void ec_combine(ec_cbk_data_t * cbk, ec_combine_f combine);
+void
+ec_combine(ec_cbk_data_t *cbk, ec_combine_f combine);
 
 int32_t
-ec_combine_write (ec_fop_data_t *fop, ec_cbk_data_t *dst,
-                  ec_cbk_data_t *src);
+ec_combine_write(ec_fop_data_t *fop, ec_cbk_data_t *dst, ec_cbk_data_t *src);
 #endif /* __EC_COMBINE_H__ */
diff --git a/xlators/cluster/ec/src/ec-common.c b/xlators/cluster/ec/src/ec-common.c
index 7354277da1b..b955efd8c2d 100644
--- a/xlators/cluster/ec/src/ec-common.c
+++ b/xlators/cluster/ec/src/ec-common.c
@@ -8,222 +8,382 @@
   cases as published by the Free Software Foundation.
 */
 
-#include "byte-order.h"
+#include <glusterfs/byte-order.h>
+#include <glusterfs/hashfn.h>
 
 #include "ec-mem-types.h"
-#include "ec-data.h"
+#include "ec-types.h"
 #include "ec-helpers.h"
 #include "ec-combine.h"
 #include "ec-common.h"
 #include "ec-fops.h"
 #include "ec-method.h"
 #include "ec.h"
+#include "ec-messages.h"
 
-int32_t ec_child_valid(ec_t * ec, ec_fop_data_t * fop, int32_t idx)
+#define EC_INVALID_INDEX UINT32_MAX
+
+void
+ec_update_fd_status(fd_t *fd, xlator_t *xl, int idx, int32_t ret_status)
 {
-    return (idx < ec->nodes) && (((fop->remaining >> idx) & 1) == 1);
+    ec_fd_t *fd_ctx;
+
+    if (fd == NULL)
+        return;
+
+    LOCK(&fd->lock);
+    {
+        fd_ctx = __ec_fd_get(fd, xl);
+        if (fd_ctx) {
+            if (ret_status >= 0)
+                fd_ctx->fd_status[idx] = EC_FD_OPENED;
+            else
+                fd_ctx->fd_status[idx] = EC_FD_NOT_OPENED;
+        }
+    }
+    UNLOCK(&fd->lock);
 }
 
-int32_t ec_child_next(ec_t * ec, ec_fop_data_t * fop, int32_t idx)
+static uintptr_t
+ec_fd_ctx_need_open(fd_t *fd, xlator_t *this, uintptr_t mask)
 {
-    while (!ec_child_valid(ec, fop, idx))
+    int i = 0;
+    int count = 0;
+    ec_t *ec = NULL;
+    ec_fd_t *fd_ctx = NULL;
+    uintptr_t need_open = 0;
+
+    ec = this->private;
+
+    fd_ctx = ec_fd_get(fd, this);
+    if (!fd_ctx)
+        return count;
+
+    LOCK(&fd->lock);
     {
-        if (++idx >= ec->nodes)
-        {
-            idx = 0;
-        }
-        if (idx == fop->first)
-        {
-            return -1;
+        for (i = 0; i < ec->nodes; i++) {
+            if ((fd_ctx->fd_status[i] == EC_FD_NOT_OPENED) &&
+                ((ec->xl_up & (1 << i)) != 0) && ((mask & (1 << i)) != 0)) {
+                fd_ctx->fd_status[i] = EC_FD_OPENING;
+                need_open |= (1 << i);
+                count++;
+            }
         }
     }
+    UNLOCK(&fd->lock);
 
-    return idx;
+    /* If fd needs to open on minimum number of nodes
+     * then ignore fixing the fd as it has been
+     * requested from heal operation.
+     */
+    if (count >= ec->fragments) {
+        need_open = 0;
+    }
+
+    return need_open;
 }
 
-uintptr_t ec_inode_good(inode_t * inode, xlator_t * xl)
+static gf_boolean_t
+ec_is_fd_fixable(fd_t *fd)
 {
-    ec_inode_t * ctx;
-    uintptr_t bad = 0;
+    if (!fd || !fd->inode)
+        return _gf_false;
+    else if (fd_is_anonymous(fd))
+        return _gf_false;
+    else if (gf_uuid_is_null(fd->inode->gfid))
+        return _gf_false;
 
-    ctx = ec_inode_get(inode, xl);
-    if (ctx != NULL)
-    {
-        bad = ctx->bad;
+    return _gf_true;
+}
+
+static void
+ec_fix_open(ec_fop_data_t *fop, uintptr_t mask)
+{
+    uintptr_t need_open = 0;
+    int ret = 0;
+    int32_t flags = 0;
+    loc_t loc = {
+        0,
+    };
+
+    if (!ec_is_fd_fixable(fop->fd))
+        goto out;
+
+    /* Evaluate how many remote fd's to be opened */
+    need_open = ec_fd_ctx_need_open(fop->fd, fop->xl, mask);
+    if (need_open == 0) {
+        goto out;
+    }
+
+    loc.inode = inode_ref(fop->fd->inode);
+    gf_uuid_copy(loc.gfid, fop->fd->inode->gfid);
+    ret = loc_path(&loc, NULL);
+    if (ret < 0) {
+        goto out;
     }
 
-    return ~bad;
+    flags = fop->fd->flags & (~(O_TRUNC | O_APPEND | O_CREAT | O_EXCL));
+    if (IA_IFDIR == fop->fd->inode->ia_type) {
+        ec_opendir(fop->frame, fop->xl, need_open,
+                   EC_MINIMUM_ONE | EC_FOP_NO_PROPAGATE_ERROR, NULL, NULL,
+                   &fop->loc[0], fop->fd, NULL);
+    } else {
+        ec_open(fop->frame, fop->xl, need_open,
+                EC_MINIMUM_ONE | EC_FOP_NO_PROPAGATE_ERROR, NULL, NULL, &loc,
+                flags, fop->fd, NULL);
+    }
+
+out:
+    loc_wipe(&loc);
 }
 
-uintptr_t ec_fd_good(fd_t * fd, xlator_t * xl)
+static off_t
+ec_range_end_get(off_t fl_start, uint64_t fl_size)
 {
-    ec_fd_t * ctx;
-    uintptr_t bad = 0;
-
-    ctx = ec_fd_get(fd, xl);
-    if (ctx != NULL)
-    {
-        bad = ctx->bad;
+    if (fl_size > 0) {
+        if (fl_size >= EC_RANGE_FULL) {
+            /* Infinity */
+            fl_start = LLONG_MAX;
+        } else {
+            fl_start += fl_size - 1;
+            if (fl_start < 0) {
+                /* Overflow */
+                fl_start = LLONG_MAX;
+            }
+        }
     }
 
-    return ~bad;
+    return fl_start;
 }
 
-uintptr_t ec_update_inode(ec_fop_data_t * fop, inode_t * inode, uintptr_t good,
-                          uintptr_t bad)
+static gf_boolean_t
+ec_is_range_conflict(ec_lock_link_t *l1, ec_lock_link_t *l2)
 {
-    ec_inode_t * ctx = NULL;
+    return ((l1->fl_end >= l2->fl_start) && (l2->fl_end >= l1->fl_start));
+}
 
-    if (inode != NULL)
-    {
-        LOCK(&inode->lock);
+static gf_boolean_t
+ec_lock_conflict(ec_lock_link_t *l1, ec_lock_link_t *l2)
+{
+    ec_t *ec = l1->fop->xl->private;
 
-        ctx = __ec_inode_get(inode, fop->xl);
-        if (ctx != NULL)
-        {
-            ctx->bad &= ~good;
-            bad |= ctx->bad;
-            ctx->bad = bad;
-        }
+    /* Fops like access/stat won't have to worry what the other fops are
+     * modifying as the fop is wound only to one brick. So it can be
+     * executed in parallel*/
+    if (l1->fop->minimum == EC_MINIMUM_ONE ||
+        l2->fop->minimum == EC_MINIMUM_ONE)
+        return _gf_false;
 
-        UNLOCK(&inode->lock);
+    if ((l1->fop->flags & EC_FLAG_LOCK_SHARED) &&
+        (l2->fop->flags & EC_FLAG_LOCK_SHARED))
+        return _gf_false;
+
+    if (!ec->parallel_writes) {
+        return _gf_true;
     }
 
-    return bad;
+    return ec_is_range_conflict(l1, l2);
 }
 
-uintptr_t ec_update_fd(ec_fop_data_t * fop, fd_t * fd, uintptr_t good,
-                       uintptr_t bad)
+uint32_t
+ec_select_first_by_read_policy(ec_t *ec, ec_fop_data_t *fop)
 {
-    ec_fd_t * ctx = NULL;
+    if (ec->read_policy == EC_ROUND_ROBIN) {
+        return ec->idx;
+    } else if (ec->read_policy == EC_GFID_HASH) {
+        if (fop->use_fd) {
+            return SuperFastHash((char *)fop->fd->inode->gfid,
+                                 sizeof(fop->fd->inode->gfid)) %
+                   ec->nodes;
+        } else {
+            if (gf_uuid_is_null(fop->loc[0].gfid))
+                loc_gfid(&fop->loc[0], fop->loc[0].gfid);
+            return SuperFastHash((char *)fop->loc[0].gfid,
+                                 sizeof(fop->loc[0].gfid)) %
+                   ec->nodes;
+        }
+    }
+    return 0;
+}
 
-    LOCK(&fd->lock);
+static gf_boolean_t
+ec_child_valid(ec_t *ec, ec_fop_data_t *fop, uint32_t idx)
+{
+    return (idx < ec->nodes) && (((fop->remaining >> idx) & 1) == 1);
+}
 
-    ctx = __ec_fd_get(fd, fop->xl);
-    if (ctx != NULL)
-    {
-        ctx->bad &= ~good;
-        bad |= ctx->bad;
-        ctx->bad = bad;
+static uint32_t
+ec_child_next(ec_t *ec, ec_fop_data_t *fop, uint32_t idx)
+{
+    while (!ec_child_valid(ec, fop, idx)) {
+        if (++idx >= ec->nodes) {
+            idx = 0;
+        }
+        if (idx == fop->first) {
+            return EC_INVALID_INDEX;
+        }
     }
 
-    UNLOCK(&fd->lock);
-
-    return bad;
+    return idx;
 }
 
-int32_t ec_heal_report(call_frame_t * frame, void * cookie, xlator_t * this,
-                       int32_t op_ret, int32_t op_errno, uintptr_t mask,
-                       uintptr_t good, uintptr_t bad, dict_t * xdata)
+int32_t
+ec_heal_report(call_frame_t *frame, void *cookie, xlator_t *this,
+               int32_t op_ret, int32_t op_errno, uintptr_t mask, uintptr_t good,
+               uintptr_t bad, uint32_t pending, dict_t *xdata)
 {
     if (op_ret < 0) {
-        gf_log(this->name, GF_LOG_WARNING, "Heal failed (error %d)",
-               op_errno);
+        gf_msg(this->name, GF_LOG_DEBUG, op_errno, EC_MSG_HEAL_FAIL,
+               "Heal failed");
     } else {
         if ((mask & ~good) != 0) {
-            gf_log(this->name, GF_LOG_INFO, "Heal succeeded on %d/%d "
-                                            "subvolumes",
-                   ec_bits_count(mask & ~(good | bad)),
-                   ec_bits_count(mask & ~good));
+            gf_msg(this->name, GF_LOG_DEBUG, 0, EC_MSG_HEAL_SUCCESS,
+                   "Heal succeeded on %d/%d "
+                   "subvolumes",
+                   gf_bits_count(mask & ~(good | bad)),
+                   gf_bits_count(mask & ~good));
         }
     }
 
     return 0;
 }
 
-int32_t ec_fop_needs_heal(ec_fop_data_t *fop)
+static uintptr_t
+ec_fop_needs_name_heal(ec_fop_data_t *fop)
+{
+    ec_t *ec = NULL;
+    ec_cbk_data_t *cbk = NULL;
+    ec_cbk_data_t *enoent_cbk = NULL;
+
+    ec = fop->xl->private;
+    if (fop->id != GF_FOP_LOOKUP)
+        return 0;
+
+    if (!fop->loc[0].name || strlen(fop->loc[0].name) == 0)
+        return 0;
+
+    list_for_each_entry(cbk, &fop->cbk_list, list)
+    {
+        if (cbk->op_ret < 0 && cbk->op_errno == ENOENT) {
+            enoent_cbk = cbk;
+            break;
+        }
+    }
+
+    if (!enoent_cbk)
+        return 0;
+
+    return ec->xl_up & ~enoent_cbk->mask;
+}
+
+int32_t
+ec_fop_needs_heal(ec_fop_data_t *fop)
 {
     ec_t *ec = fop->xl->private;
 
+    if (fop->lock_count == 0) {
+        /*
+         * if fop->lock_count is zero that means it saw version mismatch
+         * without any locks so it can't be trusted. If we launch a heal
+         * based on this it will lead to INODELKs which will affect I/O
+         * performance. Considering self-heal-daemon and operations on
+         * the inode from client which take locks can still trigger the
+         * heal we can choose to not attempt a heal when fop->lock_count
+         * is zero.
+         */
+        return 0;
+    }
     return (ec->xl_up & ~(fop->remaining | fop->good)) != 0;
 }
 
-void ec_check_status(ec_fop_data_t * fop)
+void
+ec_check_status(ec_fop_data_t *fop)
 {
-    ec_t * ec = fop->xl->private;
+    ec_t *ec = fop->xl->private;
     int32_t partial = 0;
+    char str1[32], str2[32], str3[32], str4[32], str5[32];
 
-    if (fop->answer->op_ret >= 0) {
-        if ((fop->id == GF_FOP_LOOKUP) ||
-            (fop->id == GF_FOP_STAT) || (fop->id == GF_FOP_FSTAT)) {
+    if (!ec_fop_needs_name_heal(fop) && !ec_fop_needs_heal(fop)) {
+        return;
+    }
+
+    if (fop->answer && fop->answer->op_ret >= 0) {
+        if ((fop->id == GF_FOP_LOOKUP) || (fop->id == GF_FOP_STAT) ||
+            (fop->id == GF_FOP_FSTAT)) {
             partial = fop->answer->iatt[0].ia_type == IA_IFDIR;
         } else if (fop->id == GF_FOP_OPENDIR) {
             partial = 1;
         }
     }
 
-    if (!ec_fop_needs_heal(fop)) {
-        return;
-    }
-
-    gf_log(fop->xl->name, GF_LOG_WARNING, "Operation failed on some "
-                                          "subvolumes (up=%lX, mask=%lX, "
-                                          "remaining=%lX, good=%lX, bad=%lX)",
-           ec->xl_up, fop->mask, fop->remaining, fop->good, fop->bad);
-
-    if (fop->use_fd)
-    {
+    gf_msg(
+        fop->xl->name, GF_LOG_WARNING, 0, EC_MSG_OP_FAIL_ON_SUBVOLS,
+        "Operation failed on %d of %d subvolumes.(up=%s, mask=%s, "
+        "remaining=%s, good=%s, bad=%s,"
+        "(Least significant bit represents first client/brick of subvol), %s)",
+        gf_bits_count(ec->xl_up & ~(fop->remaining | fop->good)), ec->nodes,
+        ec_bin(str1, sizeof(str1), ec->xl_up, ec->nodes),
+        ec_bin(str2, sizeof(str2), fop->mask, ec->nodes),
+        ec_bin(str3, sizeof(str3), fop->remaining, ec->nodes),
+        ec_bin(str4, sizeof(str4), fop->good, ec->nodes),
+        ec_bin(str5, sizeof(str5), ec->xl_up & ~(fop->remaining | fop->good),
+               ec->nodes),
+        ec_msg_str(fop));
+    if (fop->use_fd) {
         if (fop->fd != NULL) {
             ec_fheal(NULL, fop->xl, -1, EC_MINIMUM_ONE, ec_heal_report, NULL,
                      fop->fd, partial, NULL);
         }
-    }
-    else
-    {
+    } else {
         ec_heal(NULL, fop->xl, -1, EC_MINIMUM_ONE, ec_heal_report, NULL,
                 &fop->loc[0], partial, NULL);
 
-        if (fop->loc[1].inode != NULL)
-        {
+        if (fop->loc[1].inode != NULL) {
             ec_heal(NULL, fop->xl, -1, EC_MINIMUM_ONE, ec_heal_report, NULL,
                     &fop->loc[1], partial, NULL);
         }
     }
 }
 
-void ec_update_bad(ec_fop_data_t * fop, uintptr_t good)
+void
+ec_update_good(ec_fop_data_t *fop, uintptr_t good)
 {
-    ec_t *ec = fop->xl->private;
-    uintptr_t bad;
-
-    bad = ec->xl_up & ~(fop->remaining | good);
-    fop->bad |= bad;
-    fop->good |= good;
-
-    if (fop->parent == NULL)
-    {
-        if ((fop->flags & EC_FLAG_UPDATE_LOC_PARENT) != 0)
-        {
-            ec_update_inode(fop, fop->loc[0].parent, good, bad);
-        }
-        if ((fop->flags & EC_FLAG_UPDATE_LOC_INODE) != 0)
-        {
-            ec_update_inode(fop, fop->loc[0].inode, good, bad);
-        }
-        ec_update_inode(fop, fop->loc[1].inode, good, bad);
-        if ((fop->flags & EC_FLAG_UPDATE_FD_INODE) != 0)
-        {
-            ec_update_inode(fop, fop->fd->inode, good, bad);
-        }
-        if ((fop->flags & EC_FLAG_UPDATE_FD) != 0)
-        {
-            ec_update_fd(fop, fop->fd, good, bad);
-        }
+    fop->good = good;
 
+    /* Fops that are executed only on one brick do not have enough information
+     * to decide if healing is needed or not. */
+    if ((fop->expected != 1) && (fop->parent == NULL)) {
         ec_check_status(fop);
     }
 }
 
+void
+ec_lock_update_good(ec_lock_t *lock, ec_fop_data_t *fop)
+{
+    /* Fops that are executed only on one brick do not have enough information
+     * to update the global mask of good bricks. */
+    if (fop->expected == 1) {
+        return;
+    }
+
+    /* When updating the good mask of the lock, we only take into consideration
+     * those bits corresponding to the bricks where the fop has been executed.
+     * Bad bricks are removed from good_mask, but once marked as bad it's never
+     * set to good until the lock is released and reacquired */
 
-void __ec_fop_set_error(ec_fop_data_t * fop, int32_t error)
+    lock->good_mask &= fop->good | fop->remaining;
+}
+
+void
+__ec_fop_set_error(ec_fop_data_t *fop, int32_t error)
 {
-    if ((error != 0) && (fop->error == 0))
-    {
+    if ((error != 0) && (fop->error == 0)) {
         fop->error = error;
     }
 }
 
-void ec_fop_set_error(ec_fop_data_t * fop, int32_t error)
+void
+ec_fop_set_error(ec_fop_data_t *fop, int32_t error)
 {
     LOCK(&fop->lock);
 
@@ -232,17 +392,63 @@ void ec_fop_set_error(ec_fop_data_t * fop, int32_t error)
     UNLOCK(&fop->lock);
 }
 
-void ec_sleep(ec_fop_data_t *fop)
+gf_boolean_t
+ec_cbk_set_error(ec_cbk_data_t *cbk, int32_t error, gf_boolean_t ro)
+{
+    if ((error != 0) && (cbk->op_ret >= 0)) {
+        /* If cbk->op_errno was 0, it means that the fop succeeded and this
+         * error has happened while processing the answer. If the operation was
+         * read-only, there's no problem (i.e. we simply return the generated
+         * error code). However if it caused a modification, we must return EIO
+         * to indicate that the operation has been partially executed. */
+        cbk->op_errno = ro ? error : EIO;
+        cbk->op_ret = -1;
+
+        ec_fop_set_error(cbk->fop, cbk->op_errno);
+    }
+
+    return (cbk->op_ret < 0);
+}
+
+ec_cbk_data_t *
+ec_fop_prepare_answer(ec_fop_data_t *fop, gf_boolean_t ro)
+{
+    ec_cbk_data_t *cbk;
+    int32_t err;
+
+    cbk = fop->answer;
+    if (cbk == NULL) {
+        ec_fop_set_error(fop, EIO);
+
+        return NULL;
+    }
+
+    if (cbk->op_ret < 0) {
+        ec_fop_set_error(fop, cbk->op_errno);
+    }
+
+    err = ec_dict_combine(cbk, EC_COMBINE_XDATA);
+    if (ec_cbk_set_error(cbk, -err, ro)) {
+        return NULL;
+    }
+
+    return cbk;
+}
+
+void
+ec_sleep(ec_fop_data_t *fop)
 {
     LOCK(&fop->lock);
 
+    GF_ASSERT(fop->refs > 0);
     fop->refs++;
     fop->jobs++;
 
     UNLOCK(&fop->lock);
 }
 
-int32_t ec_check_complete(ec_fop_data_t * fop, ec_resume_f resume)
+int32_t
+ec_check_complete(ec_fop_data_t *fop, ec_resume_f resume)
 {
     int32_t error = -1;
 
@@ -250,14 +456,11 @@ int32_t ec_check_complete(ec_fop_data_t * fop, ec_resume_f resume)
 
     GF_ASSERT(fop->resume == NULL);
 
-    if (fop->jobs != 0)
-    {
+    if (--fop->jobs != 0) {
         ec_trace("WAIT", fop, "resume=%p", resume);
 
         fop->resume = resume;
-    }
-    else
-    {
+    } else {
         error = fop->error;
         fop->error = 0;
     }
@@ -267,22 +470,8 @@ int32_t ec_check_complete(ec_fop_data_t * fop, ec_resume_f resume)
     return error;
 }
 
-void ec_wait_winds(ec_fop_data_t * fop)
-{
-    LOCK(&fop->lock);
-
-    if (fop->winds > 0)
-    {
-        fop->jobs++;
-        fop->refs++;
-
-        fop->flags |= EC_FLAG_WAITING_WINDS;
-    }
-
-    UNLOCK(&fop->lock);
-}
-
-void ec_resume(ec_fop_data_t * fop, int32_t error)
+void
+ec_resume(ec_fop_data_t *fop, int32_t error)
 {
     ec_resume_f resume = NULL;
 
@@ -290,16 +479,13 @@ void ec_resume(ec_fop_data_t * fop, int32_t error)
 
     __ec_fop_set_error(fop, error);
 
-    if (--fop->jobs == 0)
-    {
+    if (--fop->jobs == 0) {
         resume = fop->resume;
         fop->resume = NULL;
-        if (resume != NULL)
-        {
+        if (resume != NULL) {
             ec_trace("RESUME", fop, "error=%d", error);
 
-            if (fop->error != 0)
-            {
+            if (fop->error != 0) {
                 error = fop->error;
             }
             fop->error = 0;
@@ -308,31 +494,50 @@ void ec_resume(ec_fop_data_t * fop, int32_t error)
 
     UNLOCK(&fop->lock);
 
-    if (resume != NULL)
-    {
+    if (resume != NULL) {
         resume(fop, error);
     }
 
     ec_fop_data_release(fop);
 }
 
-void ec_resume_parent(ec_fop_data_t * fop, int32_t error)
+void
+ec_resume_parent(ec_fop_data_t *fop)
 {
-    ec_fop_data_t * parent;
+    ec_fop_data_t *parent;
+    int32_t error = 0;
 
     parent = fop->parent;
-    if (parent != NULL)
-    {
+    if (parent != NULL) {
+        if ((fop->fop_flags & EC_FOP_NO_PROPAGATE_ERROR) == 0) {
+            error = fop->error;
+        }
         ec_trace("RESUME_PARENT", fop, "error=%u", error);
         fop->parent = NULL;
         ec_resume(parent, error);
     }
 }
 
-void ec_complete(ec_fop_data_t * fop)
+gf_boolean_t
+ec_is_recoverable_error(int32_t op_errno)
 {
-    ec_cbk_data_t * cbk = NULL;
+    switch (op_errno) {
+        case ENOTCONN:
+        case ESTALE:
+        case ENOENT:
+        case EBADFD: /*Opened fd but brick is disconnected*/
+        case EIO:    /*Backend-fs crash like XFS/ext4 etc*/
+            return _gf_true;
+    }
+    return _gf_false;
+}
+
+void
+ec_complete(ec_fop_data_t *fop)
+{
+    ec_cbk_data_t *cbk = NULL;
     int32_t resume = 0, update = 0;
+    int healing_count = 0;
 
     LOCK(&fop->lock);
 
@@ -342,8 +547,10 @@ void ec_complete(ec_fop_data_t * fop)
         if (fop->answer == NULL) {
             if (!list_empty(&fop->cbk_list)) {
                 cbk = list_entry(fop->cbk_list.next, ec_cbk_data_t, list);
-                if ((cbk->count >= fop->minimum) &&
-                    ((cbk->op_ret >= 0) || (cbk->op_errno != ENOTCONN))) {
+                healing_count = gf_bits_count(cbk->mask & fop->healing);
+                /* fop shouldn't be treated as success if it is not
+                 * successful on at least fop->minimum good copies*/
+                if ((cbk->count - healing_count) >= fop->minimum) {
                     fop->answer = cbk;
 
                     update = 1;
@@ -352,69 +559,144 @@ void ec_complete(ec_fop_data_t * fop)
 
             resume = 1;
         }
-        else if ((fop->flags & EC_FLAG_WAITING_WINDS) != 0)
-        {
-            resume = 1;
-        }
     }
 
     UNLOCK(&fop->lock);
 
-    /* ec_update_bad() locks inode->lock. This may cause deadlocks with
-       fop->lock when used in another order. Since ec_update_bad() will not
+    /* ec_update_good() locks inode->lock. This may cause deadlocks with
+       fop->lock when used in another order. Since ec_update_good() will not
        be called more than once for each fop, it can be called from outside
        the fop->lock locked region. */
     if (update) {
-        ec_update_bad(fop, cbk->mask);
+        ec_update_good(fop, cbk->mask);
     }
 
-    if (resume)
-    {
+    if (resume) {
         ec_resume(fop, 0);
     }
 
     ec_fop_data_release(fop);
 }
 
-int32_t ec_child_select(ec_fop_data_t * fop)
+/* There could be already granted locks sitting on the bricks, unlock for which
+ * must be wound at all costs*/
+static gf_boolean_t
+ec_must_wind(ec_fop_data_t *fop)
 {
-    ec_t * ec = fop->xl->private;
-    uintptr_t mask = 0;
-    int32_t first = 0, num = 0;
+    if ((fop->id == GF_FOP_INODELK) || (fop->id == GF_FOP_FINODELK) ||
+        (fop->id == GF_FOP_LK)) {
+        if (fop->flock.l_type == F_UNLCK)
+            return _gf_true;
+    } else if ((fop->id == GF_FOP_ENTRYLK) || (fop->id == GF_FOP_FENTRYLK)) {
+        if (fop->entrylk_cmd == ENTRYLK_UNLOCK)
+            return _gf_true;
+    }
 
-    fop->mask &= ec->node_mask;
+    return _gf_false;
+}
 
-    mask = ec->xl_up;
-    if (fop->parent == NULL)
-    {
-        if (fop->loc[0].inode != NULL) {
-            mask &= ec_inode_good(fop->loc[0].inode, fop->xl);
-        }
-        if (fop->loc[1].inode != NULL) {
-            mask &= ec_inode_good(fop->loc[1].inode, fop->xl);
+static gf_boolean_t
+ec_internal_op(ec_fop_data_t *fop)
+{
+    if (ec_must_wind(fop))
+        return _gf_true;
+    if (fop->id == GF_FOP_XATTROP)
+        return _gf_true;
+    if (fop->id == GF_FOP_FXATTROP)
+        return _gf_true;
+    if (fop->id == GF_FOP_OPEN)
+        return _gf_true;
+    return _gf_false;
+}
+
+char *
+ec_msg_str(ec_fop_data_t *fop)
+{
+    loc_t *loc1 = NULL;
+    loc_t *loc2 = NULL;
+    char gfid1[64] = {0};
+    char gfid2[64] = {0};
+    ec_fop_data_t *parent = fop->parent;
+
+    if (fop->errstr)
+        return fop->errstr;
+    if (!fop->use_fd) {
+        loc1 = &fop->loc[0];
+        loc2 = &fop->loc[1];
+
+        if (fop->id == GF_FOP_RENAME) {
+            gf_asprintf(&fop->errstr,
+                        "FOP : '%s' failed on '%s' and '%s' with gfids "
+                        "%s and %s respectively. Parent FOP: %s",
+                        ec_fop_name(fop->id), loc1->path, loc2->path,
+                        uuid_utoa_r(loc1->gfid, gfid1),
+                        uuid_utoa_r(loc2->gfid, gfid2),
+                        parent ? ec_fop_name(parent->id) : "No Parent");
+        } else {
+            gf_asprintf(
+                &fop->errstr,
+                "FOP : '%s' failed on '%s' with gfid %s. Parent FOP: %s",
+                ec_fop_name(fop->id), loc1->path,
+                uuid_utoa_r(loc1->gfid, gfid1),
+                parent ? ec_fop_name(parent->id) : "No Parent");
         }
-        if (fop->fd != NULL) {
-            if (fop->fd->inode != NULL) {
-                mask &= ec_inode_good(fop->fd->inode, fop->xl);
-            }
-            mask &= ec_fd_good(fop->fd, fop->xl);
+    } else {
+        gf_asprintf(
+            &fop->errstr, "FOP : '%s' failed on gfid %s. Parent FOP: %s",
+            ec_fop_name(fop->id), uuid_utoa_r(fop->fd->inode->gfid, gfid1),
+            parent ? ec_fop_name(parent->id) : "No Parent");
+    }
+    return fop->errstr;
+}
+
+static void
+ec_log_insufficient_vol(ec_fop_data_t *fop, int32_t have, uint32_t need,
+                        int32_t loglevel)
+{
+    ec_t *ec = fop->xl->private;
+    char str1[32], str2[32], str3[32];
+
+    gf_msg(ec->xl->name, loglevel, 0, EC_MSG_CHILDS_INSUFFICIENT,
+           "Insufficient available children for this request: "
+           "Have : %d, Need : %u : Child UP : %s "
+           "Mask: %s, Healing : %s : %s ",
+           have, need, ec_bin(str1, sizeof(str1), ec->xl_up, ec->nodes),
+           ec_bin(str2, sizeof(str2), fop->mask, ec->nodes),
+           ec_bin(str3, sizeof(str3), fop->healing, ec->nodes),
+           ec_msg_str(fop));
+}
+
+static int32_t
+ec_child_select(ec_fop_data_t *fop)
+{
+    ec_t *ec = fop->xl->private;
+    int32_t first = 0, num = 0;
+
+    ec_fop_cleanup(fop);
+
+    fop->mask &= ec->node_mask;
+    /* Wind the fop on same subvols as parent for any internal extra fops like
+     * head/tail read in case of writev fop. Unlocks shouldn't do this because
+     * unlock should go on all subvols where lock is performed*/
+    if (fop->parent && !ec_internal_op(fop)) {
+        fop->mask &= (fop->parent->mask & ~fop->parent->healing);
+        if (ec_is_data_fop(fop->id)) {
+            fop->healing |= fop->parent->healing;
         }
     }
-    if ((fop->mask & ~mask) != 0)
-    {
-        gf_log(fop->xl->name, GF_LOG_WARNING, "Executing operation with "
-                                              "some subvolumes unavailable "
-                                              "(%lX)", fop->mask & ~mask);
 
-        fop->mask &= mask;
+    if ((fop->mask & ~ec->xl_up) != 0) {
+        gf_msg(fop->xl->name, GF_LOG_WARNING, 0, EC_MSG_OP_EXEC_UNAVAIL,
+               "Executing operation with "
+               "some subvolumes unavailable. (%" PRIXPTR "). %s ",
+               fop->mask & ~ec->xl_up, ec_msg_str(fop));
+        fop->mask &= ec->xl_up;
     }
 
-    switch (fop->minimum)
-    {
+    switch (fop->minimum) {
         case EC_MINIMUM_ALL:
-            fop->minimum = ec_bits_count(fop->mask);
-            if (fop->minimum >= ec->fragments)
-            {
+            fop->minimum = gf_bits_count(fop->mask);
+            if (fop->minimum >= ec->fragments) {
                 break;
             }
         case EC_MINIMUM_MIN:
@@ -424,41 +706,51 @@ int32_t ec_child_select(ec_fop_data_t * fop)
             fop->minimum = 1;
     }
 
-    first = ec->idx;
-    if (++first >= ec->nodes)
-    {
-        first = 0;
+    if (ec->read_policy == EC_ROUND_ROBIN) {
+        first = ec->idx;
+        if (++first >= ec->nodes) {
+            first = 0;
+        }
+        ec->idx = first;
     }
-    ec->idx = first;
 
+    num = gf_bits_count(fop->mask);
+    /*Unconditionally wind on healing subvolumes*/
+    fop->mask |= fop->healing;
     fop->remaining = fop->mask;
+    fop->received = 0;
 
     ec_trace("SELECT", fop, "");
 
-    num = ec_bits_count(fop->mask);
-    if ((num < fop->minimum) && (num < ec->fragments))
-    {
-        gf_log(ec->xl->name, GF_LOG_ERROR, "Insufficient available childs "
-                                           "for this request (have %d, need "
-                                           "%d)", num, fop->minimum);
-
+    if ((num < fop->minimum) && (num < ec->fragments)) {
+        ec_log_insufficient_vol(fop, num, fop->minimum, GF_LOG_ERROR);
         return 0;
     }
 
-    ec_sleep(fop);
+    if (!fop->parent && fop->lock_count &&
+        (fop->locks[0].update[EC_DATA_TXN] ||
+         fop->locks[0].update[EC_METADATA_TXN])) {
+        if (ec->quorum_count && (num < ec->quorum_count)) {
+            ec_log_insufficient_vol(fop, num, ec->quorum_count, GF_LOG_ERROR);
+            return 0;
+        }
+    }
 
     return 1;
 }
 
-int32_t ec_dispatch_next(ec_fop_data_t * fop, int32_t idx)
+void
+ec_dispatch_next(ec_fop_data_t *fop, uint32_t idx)
 {
-    ec_t * ec = fop->xl->private;
+    uint32_t i = EC_INVALID_INDEX;
+    ec_t *ec = fop->xl->private;
 
     LOCK(&fop->lock);
 
-    idx = ec_child_next(ec, fop, idx);
-    if (idx >= 0)
-    {
+    i = ec_child_next(ec, fop, idx);
+    if (i < EC_MAX_NODES) {
+        idx = i;
+
         fop->remaining ^= 1ULL << idx;
 
         ec_trace("EXECUTE", fop, "idx=%d", idx);
@@ -469,20 +761,18 @@ int32_t ec_dispatch_next(ec_fop_data_t * fop, int32_t idx)
 
     UNLOCK(&fop->lock);
 
-    if (idx >= 0)
-    {
+    if (i < EC_MAX_NODES) {
         fop->wind(ec, fop, idx);
     }
-
-    return idx;
 }
 
-void ec_dispatch_mask(ec_fop_data_t * fop, uintptr_t mask)
+void
+ec_dispatch_mask(ec_fop_data_t *fop, uintptr_t mask)
 {
-    ec_t * ec = fop->xl->private;
+    ec_t *ec = fop->xl->private;
     int32_t count, idx;
 
-    count = ec_bits_count(mask);
+    count = gf_bits_count(mask);
 
     LOCK(&fop->lock);
 
@@ -496,10 +786,8 @@ void ec_dispatch_mask(ec_fop_data_t * fop, uintptr_t mask)
     UNLOCK(&fop->lock);
 
     idx = 0;
-    while (mask != 0)
-    {
-        if ((mask & 1) != 0)
-        {
+    while (mask != 0) {
+        if ((mask & 1) != 0) {
             fop->wind(ec, fop, idx);
         }
         idx++;
@@ -507,1022 +795,2042 @@ void ec_dispatch_mask(ec_fop_data_t * fop, uintptr_t mask)
     }
 }
 
-void ec_dispatch_start(ec_fop_data_t * fop)
+void
+ec_dispatch_start(ec_fop_data_t *fop)
 {
     fop->answer = NULL;
     fop->good = 0;
-    fop->bad = 0;
 
     INIT_LIST_HEAD(&fop->cbk_list);
 
-    if (fop->lock_count > 0)
-    {
+    if (fop->lock_count > 0) {
         ec_owner_copy(fop->frame, &fop->req_frame->root->lk_owner);
     }
 }
 
-void ec_dispatch_one(ec_fop_data_t * fop)
+void
+ec_dispatch_one(ec_fop_data_t *fop)
 {
-    ec_t * ec = fop->xl->private;
-
     ec_dispatch_start(fop);
 
-    if (ec_child_select(fop))
-    {
+    if (ec_child_select(fop)) {
+        ec_sleep(fop);
+
         fop->expected = 1;
-        fop->first = ec->idx;
+        fop->first = ec_select_first_by_read_policy(fop->xl->private, fop);
 
         ec_dispatch_next(fop, fop->first);
     }
 }
 
-int32_t ec_dispatch_one_retry(ec_fop_data_t * fop, int32_t idx, int32_t op_ret,
-                              int32_t op_errno)
+gf_boolean_t
+ec_dispatch_one_retry(ec_fop_data_t *fop, ec_cbk_data_t **cbk)
 {
-    if ((op_ret < 0) && (op_errno == ENOTCONN))
-    {
-        return (ec_dispatch_next(fop, idx) >= 0);
+    ec_cbk_data_t *tmp;
+
+    tmp = ec_fop_prepare_answer(fop, _gf_true);
+    if (cbk != NULL) {
+        *cbk = tmp;
+    }
+    if ((tmp != NULL) && (tmp->op_ret < 0) &&
+        ec_is_recoverable_error(tmp->op_errno)) {
+        GF_ASSERT(fop->mask & (1ULL << tmp->idx));
+        fop->mask ^= (1ULL << tmp->idx);
+        if (fop->mask) {
+            return _gf_true;
+        }
     }
 
-    return 0;
+    return _gf_false;
 }
 
-void ec_dispatch_inc(ec_fop_data_t * fop)
+void
+ec_dispatch_inc(ec_fop_data_t *fop)
 {
     ec_dispatch_start(fop);
 
-    if (ec_child_select(fop))
-    {
-        fop->expected = ec_bits_count(fop->remaining);
+    if (ec_child_select(fop)) {
+        ec_sleep(fop);
+
+        fop->expected = gf_bits_count(fop->remaining);
         fop->first = 0;
 
         ec_dispatch_next(fop, 0);
     }
 }
 
-void ec_dispatch_all(ec_fop_data_t * fop)
+void
+ec_dispatch_all(ec_fop_data_t *fop)
 {
     ec_dispatch_start(fop);
 
-    if (ec_child_select(fop))
-    {
-        fop->expected = ec_bits_count(fop->remaining);
+    if (ec_child_select(fop)) {
+        ec_sleep(fop);
+
+        fop->expected = gf_bits_count(fop->remaining);
         fop->first = 0;
 
         ec_dispatch_mask(fop, fop->remaining);
     }
 }
 
-void ec_dispatch_min(ec_fop_data_t * fop)
+void
+ec_dispatch_min(ec_fop_data_t *fop)
 {
-    ec_t * ec = fop->xl->private;
+    ec_t *ec = fop->xl->private;
     uintptr_t mask;
-    int32_t idx, count;
+    uint32_t idx;
+    int32_t count;
 
     ec_dispatch_start(fop);
 
-    if (ec_child_select(fop))
-    {
+    if (ec_child_select(fop)) {
+        ec_sleep(fop);
+
         fop->expected = count = ec->fragments;
-        fop->first = ec->idx;
+        fop->first = ec_select_first_by_read_policy(fop->xl->private, fop);
         idx = fop->first - 1;
         mask = 0;
-        while (count-- > 0)
-        {
+        while (count-- > 0) {
             idx = ec_child_next(ec, fop, idx + 1);
-            mask |= 1ULL << idx;
+            if (idx < EC_MAX_NODES)
+                mask |= 1ULL << idx;
         }
 
         ec_dispatch_mask(fop, mask);
     }
 }
 
-ec_lock_t * ec_lock_allocate(xlator_t * xl, int32_t kind, loc_t * loc)
+void
+ec_succeed_all(ec_fop_data_t *fop)
 {
-    ec_t * ec = xl->private;
-    ec_lock_t * lock;
+    ec_dispatch_start(fop);
+
+    if (ec_child_select(fop)) {
+        fop->expected = gf_bits_count(fop->remaining);
+        fop->first = 0;
+
+        /* Simulate a successful execution on all bricks */
+        ec_trace("SUCCEED", fop, "");
+
+        fop->good = fop->remaining;
+        fop->remaining = 0;
+    }
+}
+
+ec_lock_t *
+ec_lock_allocate(ec_fop_data_t *fop, loc_t *loc)
+{
+    ec_t *ec = fop->xl->private;
+    ec_lock_t *lock;
+    int32_t err;
 
     if ((loc->inode == NULL) ||
-        (gf_uuid_is_null(loc->gfid) && gf_uuid_is_null(loc->inode->gfid)))
-    {
-        gf_log(xl->name, GF_LOG_ERROR, "Trying to lock based on an invalid "
-                                       "inode");
+        (gf_uuid_is_null(loc->gfid) && gf_uuid_is_null(loc->inode->gfid))) {
+        gf_msg(fop->xl->name, GF_LOG_ERROR, EINVAL, EC_MSG_INVALID_INODE,
+               "Trying to lock based on an invalid "
+               "inode");
+
+        __ec_fop_set_error(fop, EINVAL);
 
         return NULL;
     }
 
     lock = mem_get0(ec->lock_pool);
-    if (lock != NULL)
-    {
-        lock->kind = kind;
-        lock->good_mask = -1ULL;
+    if (lock != NULL) {
+        lock->good_mask = UINTPTR_MAX;
+        INIT_LIST_HEAD(&lock->owners);
         INIT_LIST_HEAD(&lock->waiting);
-        if (ec_loc_from_loc(xl, &lock->loc, loc) != 0)
-        {
+        INIT_LIST_HEAD(&lock->frozen);
+        err = ec_loc_from_loc(fop->xl, &lock->loc, loc);
+        if (err != 0) {
             mem_put(lock);
             lock = NULL;
+
+            __ec_fop_set_error(fop, -err);
         }
     }
 
     return lock;
 }
 
-void ec_lock_destroy(ec_lock_t * lock)
+void
+ec_lock_destroy(ec_lock_t *lock)
 {
     loc_wipe(&lock->loc);
+    if (lock->fd != NULL) {
+        fd_unref(lock->fd);
+    }
 
     mem_put(lock);
 }
 
-int32_t ec_lock_compare(ec_lock_t * lock1, ec_lock_t * lock2)
+int32_t
+ec_lock_compare(ec_lock_t *lock1, ec_lock_t *lock2)
 {
     return gf_uuid_compare(lock1->loc.gfid, lock2->loc.gfid);
 }
 
-ec_lock_link_t *ec_lock_insert(ec_fop_data_t *fop, ec_lock_t *lock,
-                               int32_t update)
+static void
+ec_lock_insert(ec_fop_data_t *fop, ec_lock_t *lock, uint32_t flags, loc_t *base,
+               off_t fl_start, uint64_t fl_size)
 {
-    ec_lock_t *new_lock, *tmp;
-    ec_lock_link_t *link = NULL;
-    int32_t tmp_update;
+    ec_lock_link_t *link;
 
-    new_lock = lock;
+    /* This check is only prepared for up to 2 locks per fop. If more locks
+     * are needed this must be changed. */
     if ((fop->lock_count > 0) &&
-        (ec_lock_compare(fop->locks[0].lock, new_lock) > 0))
-    {
-        tmp = fop->locks[0].lock;
-        fop->locks[0].lock = new_lock;
-        new_lock = tmp;
-
-        tmp_update = fop->locks_update;
-        fop->locks_update = update;
-        update = tmp_update;
+        (ec_lock_compare(fop->locks[0].lock, lock) < 0)) {
+        fop->first_lock = fop->lock_count;
+    } else {
+        /* When the first lock is added to the current fop, request lock
+         * counts from locks xlator to be able to determine if there is
+         * contention and release the lock sooner. */
+        if (fop->xdata == NULL) {
+            fop->xdata = dict_new();
+            if (fop->xdata == NULL) {
+                ec_fop_set_error(fop, ENOMEM);
+                return;
+            }
+        }
+        if (dict_set_str(fop->xdata, GLUSTERFS_INODELK_DOM_COUNT,
+                         fop->xl->name) != 0) {
+            ec_fop_set_error(fop, ENOMEM);
+            return;
+        }
     }
-    fop->locks[fop->lock_count].lock = new_lock;
-    fop->locks[fop->lock_count].fop = fop;
 
-    fop->locks_update |= update << fop->lock_count;
+    link = &fop->locks[fop->lock_count++];
 
-    fop->lock_count++;
+    link->lock = lock;
+    link->fop = fop;
+    link->update[EC_DATA_TXN] = (flags & EC_UPDATE_DATA) != 0;
+    link->update[EC_METADATA_TXN] = (flags & EC_UPDATE_META) != 0;
+    link->base = base;
+    link->fl_start = fl_start;
+    link->fl_end = ec_range_end_get(fl_start, fl_size);
 
-    if (lock->timer != NULL) {
-        link = lock->timer->data;
-        ec_trace("UNLOCK_CANCELLED", link->fop, "lock=%p", lock);
-        gf_timer_call_cancel(fop->xl->ctx, lock->timer);
-        lock->timer = NULL;
-    } else {
-        lock->refs++;
-    }
-
-    return link;
+    lock->refs_pending++;
 }
 
-void ec_lock_prepare_entry(ec_fop_data_t *fop, loc_t *loc, int32_t update)
+static void
+ec_lock_prepare_inode_internal(ec_fop_data_t *fop, loc_t *loc, uint32_t flags,
+                               loc_t *base, off_t fl_start, uint64_t fl_size)
 {
-    ec_lock_t * lock = NULL;
-    ec_inode_t * ctx = NULL;
-    ec_lock_link_t *link = NULL;
-    loc_t tmp;
+    ec_lock_t *lock = NULL;
+    ec_inode_t *ctx;
 
-    if ((fop->parent != NULL) || (fop->error != 0))
-    {
+    if ((fop->parent != NULL) || (fop->error != 0) || (loc->inode == NULL)) {
         return;
     }
 
-    /* update is only 0 for 'opendir', which needs to lock the entry pointed
-     * by loc instead of its parent.
-     */
-    if (update)
-    {
-        if (ec_loc_parent(fop->xl, loc, &tmp) != 0) {
-            ec_fop_set_error(fop, EIO);
+    LOCK(&loc->inode->lock);
 
-            return;
-        }
+    ctx = __ec_inode_get(loc->inode, fop->xl);
+    if (ctx == NULL) {
+        __ec_fop_set_error(fop, ENOMEM);
+
+        goto unlock;
+    }
+
+    if (ctx->inode_lock != NULL) {
+        lock = ctx->inode_lock;
 
         /* If there's another lock, make sure that it's not the same. Otherwise
          * do not insert it.
          *
          * This can only happen on renames where source and target names are
          * in the same directory. */
-        if ((fop->lock_count > 0) &&
-            (fop->locks[0].lock->loc.inode == tmp.inode)) {
-            goto wipe;
-        }
-    } else {
-        if (ec_loc_from_loc(fop->xl, &tmp, loc) != 0) {
-            ec_fop_set_error(fop, EIO);
+        if ((fop->lock_count > 0) && (fop->locks[0].lock == lock)) {
+            /* Combine data/meta updates */
+            fop->locks[0].update[EC_DATA_TXN] |= (flags & EC_UPDATE_DATA) != 0;
+            fop->locks[0].update[EC_METADATA_TXN] |= (flags & EC_UPDATE_META) !=
+                                                     0;
+
+            /* Only one base inode is allowed per fop, so there shouldn't be
+             * overwrites here. */
+            if (base != NULL) {
+                fop->locks[0].base = base;
+            }
 
-            return;
+            goto update_query;
         }
-    }
 
-    LOCK(&tmp.inode->lock);
-
-    ctx = __ec_inode_get(tmp.inode, fop->xl);
-    if (ctx == NULL)
-    {
-        __ec_fop_set_error(fop, EIO);
-
-        goto unlock;
-    }
-
-    if (ctx->entry_lock != NULL)
-    {
-        lock = ctx->entry_lock;
-        ec_trace("LOCK_ENTRYLK", fop, "lock=%p, inode=%p, path=%s"
-                                      "Lock already acquired",
-                 lock, tmp.inode, tmp.path);
+        ec_trace("LOCK_INODELK", fop,
+                 "lock=%p, inode=%p. Lock already "
+                 "acquired",
+                 lock, loc->inode);
 
         goto insert;
     }
 
-    lock = ec_lock_allocate(fop->xl, EC_LOCK_ENTRY, &tmp);
-    if (lock == NULL)
-    {
-        __ec_fop_set_error(fop, EIO);
-
+    lock = ec_lock_allocate(fop, loc);
+    if (lock == NULL) {
         goto unlock;
     }
 
     ec_trace("LOCK_CREATE", fop, "lock=%p", lock);
 
-    lock->type = ENTRYLK_WRLCK;
+    lock->flock.l_type = F_WRLCK;
+    lock->flock.l_whence = SEEK_SET;
 
-    lock->plock = &ctx->entry_lock;
-    ctx->entry_lock = lock;
+    lock->ctx = ctx;
+    ctx->inode_lock = lock;
 
 insert:
-    link = ec_lock_insert(fop, lock, update);
-
+    ec_lock_insert(fop, lock, flags, base, fl_start, fl_size);
+update_query:
+    lock->query |= (flags & EC_QUERY_INFO) != 0;
 unlock:
-    UNLOCK(&tmp.inode->lock);
-
-wipe:
-    loc_wipe(&tmp);
+    UNLOCK(&loc->inode->lock);
+}
 
-    if (link != NULL) {
-        ec_resume(link->fop, 0);
-    }
+void
+ec_lock_prepare_inode(ec_fop_data_t *fop, loc_t *loc, uint32_t flags,
+                      off_t fl_start, uint64_t fl_size)
+{
+    ec_lock_prepare_inode_internal(fop, loc, flags, NULL, fl_start, fl_size);
 }
 
-void ec_lock_prepare_inode(ec_fop_data_t *fop, loc_t *loc, int32_t update)
+void
+ec_lock_prepare_parent_inode(ec_fop_data_t *fop, loc_t *loc, loc_t *base,
+                             uint32_t flags)
 {
-    ec_lock_link_t *link = NULL;
-    ec_lock_t * lock;
-    ec_inode_t * ctx;
+    loc_t tmp;
+    int32_t err;
 
-    if ((fop->parent != NULL) || (fop->error != 0) || (loc->inode == NULL))
-    {
+    if (fop->error != 0) {
         return;
     }
 
-    LOCK(&loc->inode->lock);
+    err = ec_loc_parent(fop->xl, loc, &tmp);
+    if (err != 0) {
+        ec_fop_set_error(fop, -err);
 
-    ctx = __ec_inode_get(loc->inode, fop->xl);
-    if (ctx == NULL)
-    {
-        __ec_fop_set_error(fop, EIO);
+        return;
+    }
 
-        goto unlock;
+    if ((flags & EC_INODE_SIZE) != 0) {
+        flags ^= EC_INODE_SIZE;
+    } else {
+        base = NULL;
     }
 
-    if (ctx->inode_lock != NULL)
-    {
-        lock = ctx->inode_lock;
-        ec_trace("LOCK_INODELK", fop, "lock=%p, inode=%p. Lock already "
-                                      "acquired", lock, loc->inode);
+    ec_lock_prepare_inode_internal(fop, &tmp, flags, base, 0, EC_RANGE_FULL);
 
-        goto insert;
+    loc_wipe(&tmp);
+}
+
+void
+ec_lock_prepare_fd(ec_fop_data_t *fop, fd_t *fd, uint32_t flags, off_t fl_start,
+                   uint64_t fl_size)
+{
+    loc_t loc;
+    int32_t err;
+
+    if (fop->error != 0) {
+        return;
     }
 
-    lock = ec_lock_allocate(fop->xl, EC_LOCK_INODE, loc);
-    if (lock == NULL)
-    {
-        __ec_fop_set_error(fop, EIO);
+    err = ec_loc_from_fd(fop->xl, &loc, fd);
+    if (err != 0) {
+        ec_fop_set_error(fop, -err);
 
-        goto unlock;
+        return;
     }
 
-    ec_trace("LOCK_CREATE", fop, "lock=%p", lock);
+    ec_lock_prepare_inode_internal(fop, &loc, flags, NULL, fl_start, fl_size);
 
-    lock->flock.l_type = F_WRLCK;
-    lock->flock.l_whence = SEEK_SET;
+    loc_wipe(&loc);
+}
 
-    lock->plock = &ctx->inode_lock;
-    ctx->inode_lock = lock;
+gf_boolean_t
+ec_config_check(xlator_t *xl, ec_config_t *config)
+{
+    ec_t *ec;
 
-insert:
-    link = ec_lock_insert(fop, lock, update);
+    ec = xl->private;
+    if ((config->version != EC_CONFIG_VERSION) ||
+        (config->algorithm != EC_CONFIG_ALGORITHM) ||
+        (config->gf_word_size != EC_GF_BITS) || (config->bricks != ec->nodes) ||
+        (config->redundancy != ec->redundancy) ||
+        (config->chunk_size != EC_METHOD_CHUNK_SIZE)) {
+        uint32_t data_bricks;
 
-unlock:
-    UNLOCK(&loc->inode->lock);
+        /* This combination of version/algorithm requires the following
+           values. Incorrect values for these fields are a sign of
+           corruption:
 
-    if (link != NULL) {
-        ec_resume(link->fop, 0);
+             redundancy > 0
+             redundancy * 2 < bricks
+             gf_word_size must be a power of 2
+             chunk_size (in bits) must be a multiple of gf_word_size *
+                 (bricks - redundancy) */
+
+        data_bricks = config->bricks - config->redundancy;
+        if ((config->redundancy < 1) ||
+            (config->redundancy * 2 >= config->bricks) ||
+            !ec_is_power_of_2(config->gf_word_size) ||
+            ((config->chunk_size * 8) % (config->gf_word_size * data_bricks) !=
+             0)) {
+            gf_msg(xl->name, GF_LOG_ERROR, EINVAL, EC_MSG_INVALID_CONFIG,
+                   "Invalid or corrupted config");
+        } else {
+            gf_msg(xl->name, GF_LOG_ERROR, EINVAL, EC_MSG_INVALID_CONFIG,
+                   "Unsupported config "
+                   "(V=%u, A=%u, W=%u, "
+                   "N=%u, R=%u, S=%u)",
+                   config->version, config->algorithm, config->gf_word_size,
+                   config->bricks, config->redundancy, config->chunk_size);
+        }
+
+        return _gf_false;
     }
+
+    return _gf_true;
 }
 
-void ec_lock_prepare_fd(ec_fop_data_t *fop, fd_t *fd, int32_t update)
+gf_boolean_t
+ec_set_dirty_flag(ec_lock_link_t *link, ec_inode_t *ctx, uint64_t *dirty)
 {
-    loc_t loc;
+    gf_boolean_t set_dirty = _gf_false;
 
-    if ((fop->parent != NULL) || (fop->error != 0))
-    {
-        return;
+    if (link->update[EC_DATA_TXN] && !ctx->dirty[EC_DATA_TXN]) {
+        if (!link->optimistic_changelog)
+            dirty[EC_DATA_TXN] = 1;
     }
 
-    if (ec_loc_from_fd(fop->xl, &loc, fd) == 0)
-    {
-        ec_lock_prepare_inode(fop, &loc, update);
-
-        loc_wipe(&loc);
+    if (link->update[EC_METADATA_TXN] && !ctx->dirty[EC_METADATA_TXN]) {
+        if (!link->optimistic_changelog)
+            dirty[EC_METADATA_TXN] = 1;
     }
-    else
-    {
-        ec_fop_set_error(fop, EIO);
+
+    if (dirty[EC_METADATA_TXN] || dirty[EC_DATA_TXN]) {
+        set_dirty = _gf_true;
     }
+
+    return set_dirty;
 }
 
-int32_t ec_locked(call_frame_t * frame, void * cookie, xlator_t * this,
-                  int32_t op_ret, int32_t op_errno, dict_t * xdata)
+int32_t
+ec_prepare_update_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                      int32_t op_ret, int32_t op_errno, dict_t *dict,
+                      dict_t *xdata)
 {
-    ec_fop_data_t * fop = cookie;
-    ec_lock_t * lock = NULL;
-
-    if (op_ret >= 0)
-    {
-        lock = fop->data;
-        lock->mask = fop->good;
-        lock->acquired = 1;
+    struct list_head list;
+    ec_fop_data_t *fop = cookie, *parent, *tmp;
+    ec_lock_link_t *parent_link = fop->data;
+    ec_lock_link_t *link = NULL;
+    ec_lock_t *lock = NULL;
+    ec_inode_t *ctx;
+    gf_boolean_t release = _gf_false;
+    uint64_t provided_flags = 0;
+    uint64_t dirty[EC_VERSION_SIZE] = {0, 0};
+    lock = parent_link->lock;
+    parent = parent_link->fop;
+    ctx = lock->ctx;
 
-        fop->parent->mask &= fop->good;
-        fop->parent->locked++;
+    INIT_LIST_HEAD(&list);
+    provided_flags = EC_PROVIDED_FLAGS(parent_link->waiting_flags);
 
-        ec_trace("LOCKED", fop->parent, "lock=%p", lock);
+    LOCK(&lock->loc.inode->lock);
 
-        ec_lock(fop->parent);
-    }
-    else
+    list_for_each_entry(link, &lock->owners, owner_list)
     {
-        gf_log(this->name, GF_LOG_WARNING, "Failed to complete preop lock");
+        if ((link->waiting_flags & provided_flags) != 0) {
+            link->waiting_flags ^= (link->waiting_flags & provided_flags);
+            if (EC_NEEDED_FLAGS(link->waiting_flags) == 0)
+                list_add_tail(&link->fop->cbk_list, &list);
+        }
     }
+    if (op_ret < 0) {
+        gf_msg(this->name, GF_LOG_WARNING, op_errno, EC_MSG_SIZE_VERS_GET_FAIL,
+               "Failed to get size and version :  %s", ec_msg_str(fop));
 
-    return 0;
-}
+        goto unlock;
+    }
 
-void ec_lock(ec_fop_data_t * fop)
-{
-    ec_lock_t * lock;
+    if (EC_FLAGS_HAVE(provided_flags, EC_FLAG_XATTROP)) {
+        op_errno = -ec_dict_del_array(dict, EC_XATTR_VERSION, ctx->pre_version,
+                                      EC_VERSION_SIZE);
+        if (op_errno != 0) {
+            gf_msg(this->name, GF_LOG_ERROR, op_errno,
+                   EC_MSG_VER_XATTR_GET_FAIL, "Unable to get version xattr. %s",
+                   ec_msg_str(fop));
+            goto unlock;
+        }
+        ctx->post_version[0] += ctx->pre_version[0];
+        ctx->post_version[1] += ctx->pre_version[1];
+
+        ctx->have_version = _gf_true;
+
+        if (lock->loc.inode->ia_type == IA_IFREG ||
+            lock->loc.inode->ia_type == IA_INVAL) {
+            op_errno = -ec_dict_del_number(dict, EC_XATTR_SIZE, &ctx->pre_size);
+            if (op_errno != 0) {
+                if (lock->loc.inode->ia_type == IA_IFREG) {
+                    gf_msg(this->name, GF_LOG_ERROR, op_errno,
+                           EC_MSG_SIZE_XATTR_GET_FAIL,
+                           "Unable to get size xattr. %s", ec_msg_str(fop));
+                    goto unlock;
+                }
+            } else {
+                ctx->post_size = ctx->pre_size;
 
-    while (fop->locked < fop->lock_count)
-    {
-        lock = fop->locks[fop->locked].lock;
+                ctx->have_size = _gf_true;
+            }
 
-        LOCK(&lock->loc.inode->lock);
+            op_errno = -ec_dict_del_config(dict, EC_XATTR_CONFIG, &ctx->config);
+            if (op_errno != 0) {
+                if ((lock->loc.inode->ia_type == IA_IFREG) ||
+                    (op_errno != ENODATA)) {
+                    gf_msg(this->name, GF_LOG_ERROR, op_errno,
+                           EC_MSG_CONFIG_XATTR_GET_FAIL,
+                           "Unable to get config xattr. %s", ec_msg_str(fop));
 
-        if (lock->owner != NULL)
-        {
-            ec_trace("LOCK_WAIT", fop, "lock=%p", lock);
+                    goto unlock;
+                }
+            } else {
+                if (!ec_config_check(parent->xl, &ctx->config)) {
+                    gf_msg(this->name, GF_LOG_ERROR, EINVAL,
+                           EC_MSG_CONFIG_XATTR_INVALID, "Invalid config xattr");
 
-            list_add_tail(&fop->locks[fop->locked].wait_list, &lock->waiting);
+                    op_errno = EINVAL;
 
-            ec_sleep(fop);
+                    goto unlock;
+                }
+                ctx->have_config = _gf_true;
+            }
+        }
+        ctx->have_info = _gf_true;
+    }
 
-            UNLOCK(&lock->loc.inode->lock);
+    ec_set_dirty_flag(fop->data, ctx, dirty);
+    if (dirty[EC_METADATA_TXN] &&
+        (EC_FLAGS_HAVE(provided_flags, EC_FLAG_METADATA_DIRTY))) {
+        GF_ASSERT(!ctx->dirty[EC_METADATA_TXN]);
+        ctx->dirty[EC_METADATA_TXN] = 1;
+    }
 
-            break;
+    if (dirty[EC_DATA_TXN] &&
+        (EC_FLAGS_HAVE(provided_flags, EC_FLAG_DATA_DIRTY))) {
+        GF_ASSERT(!ctx->dirty[EC_DATA_TXN]);
+        ctx->dirty[EC_DATA_TXN] = 1;
+    }
+    op_errno = 0;
+unlock:
+
+    lock->waiting_flags ^= provided_flags;
+
+    if (op_errno == 0) {
+        /* If the fop fails on any of the good bricks, it is important to mark
+         * it dirty and update versions right away if dirty was not set before.
+         */
+        if (lock->good_mask & ~(fop->good | fop->remaining)) {
+            release = _gf_true;
         }
-        lock->owner = fop;
 
-        UNLOCK(&lock->loc.inode->lock);
+        if (parent_link->update[0] && !parent_link->dirty[0]) {
+            lock->release |= release;
+        }
 
-        if (!lock->acquired)
-        {
-            ec_owner_set(fop->frame, lock);
+        if (parent_link->update[1] && !parent_link->dirty[1]) {
+            lock->release |= release;
+        }
 
-            if (lock->kind == EC_LOCK_ENTRY)
-            {
-                ec_trace("LOCK_ACQUIRE", fop, "lock=%p, inode=%p, path=%s",
-                         lock, lock->loc.inode, lock->loc.path);
+        /* We don't allow the main fop to be executed on bricks that have not
+         * succeeded the initial xattrop. */
+        ec_lock_update_good(lock, fop);
 
-                ec_entrylk(fop->frame, fop->xl, -1, EC_MINIMUM_ALL, ec_locked,
-                           lock, fop->xl->name, &lock->loc, NULL,
-                           ENTRYLK_LOCK, lock->type, NULL);
-            }
-            else
-            {
-                ec_trace("LOCK_ACQUIRE", fop, "lock=%p, inode=%p", lock,
-                         lock->loc.inode);
+        /*As of now only data healing marks bricks as healing*/
+        lock->healing |= fop->healing;
+    }
 
-                ec_inodelk(fop->frame, fop->xl, -1, EC_MINIMUM_ALL, ec_locked,
-                           lock, fop->xl->name, &lock->loc, F_SETLKW,
-                           &lock->flock, NULL);
-            }
+    UNLOCK(&lock->loc.inode->lock);
 
-            break;
-        }
+    while (!list_empty(&list)) {
+        tmp = list_entry(list.next, ec_fop_data_t, cbk_list);
+        list_del_init(&tmp->cbk_list);
 
-        ec_trace("LOCK_REUSE", fop, "lock=%p", lock);
+        if (op_errno == 0) {
+            tmp->mask &= fop->good;
 
-        if (lock->have_size)
-        {
-            fop->pre_size = fop->post_size = lock->size;
-            fop->have_size = 1;
+            /*As of now only data healing marks bricks as healing*/
+            if (ec_is_data_fop(tmp->id)) {
+                tmp->healing |= fop->healing;
+            }
         }
-        fop->mask &= lock->good_mask;
 
-        fop->locked++;
+        ec_resume(tmp, op_errno);
     }
+
+    return 0;
 }
 
-gf_boolean_t
-ec_config_check (ec_fop_data_t *fop, dict_t *xdata)
+static gf_boolean_t
+ec_set_needed_flag(ec_lock_t *lock, ec_lock_link_t *link, uint64_t flag)
 {
-    ec_t *ec;
+    uint64_t current;
 
-    if (ec_dict_del_config(xdata, EC_XATTR_CONFIG, &fop->config) < 0) {
-        gf_log(fop->xl->name, GF_LOG_ERROR, "Failed to get a valid "
-                                            "config");
+    link->waiting_flags |= EC_FLAG_NEEDS(flag);
 
-        ec_fop_set_error(fop, EIO);
+    current = EC_NEEDED_FLAGS(lock->waiting_flags);
+    if (!EC_FLAGS_HAVE(current, flag)) {
+        lock->waiting_flags |= EC_FLAG_NEEDS(flag);
+        link->waiting_flags |= EC_FLAG_PROVIDES(flag);
 
-        return _gf_false;
+        return _gf_true;
     }
 
-    ec = fop->xl->private;
-    if ((fop->config.version != EC_CONFIG_VERSION) ||
-        (fop->config.algorithm != EC_CONFIG_ALGORITHM) ||
-        (fop->config.gf_word_size != EC_GF_BITS) ||
-        (fop->config.bricks != ec->nodes) ||
-        (fop->config.redundancy != ec->redundancy) ||
-        (fop->config.chunk_size != EC_METHOD_CHUNK_SIZE)) {
-        uint32_t data_bricks;
+    return _gf_false;
+}
 
-        /* This combination of version/algorithm requires the following
-           values. Incorrect values for these fields are a sign of
-           corruption:
+static uint64_t
+ec_set_xattrop_flags_and_params(ec_lock_t *lock, ec_lock_link_t *link,
+                                uint64_t *dirty)
+{
+    uint64_t oldflags = 0;
+    uint64_t newflags = 0;
+    ec_inode_t *ctx = lock->ctx;
 
-             redundancy > 0
-             redundancy * 2 < bricks
-             gf_word_size must be a power of 2
-             chunk_size (in bits) must be a multiple of gf_word_size *
-                 (bricks - redundancy) */
+    oldflags = EC_NEEDED_FLAGS(lock->waiting_flags);
 
-        data_bricks = fop->config.bricks - fop->config.redundancy;
-        if ((fop->config.redundancy < 1) ||
-            (fop->config.redundancy * 2 >= fop->config.bricks) ||
-            !ec_is_power_of_2(fop->config.gf_word_size) ||
-            ((fop->config.chunk_size * 8) % (fop->config.gf_word_size *
-                                             data_bricks) != 0)) {
-            gf_log(fop->xl->name, GF_LOG_ERROR, "Invalid or corrupted config");
-        } else {
-            gf_log(fop->xl->name, GF_LOG_ERROR, "Unsupported config "
-                                                "(V=%u, A=%u, W=%u, "
-                                                "N=%u, R=%u, S=%u)",
-                   fop->config.version, fop->config.algorithm,
-                   fop->config.gf_word_size, fop->config.bricks,
-                   fop->config.redundancy, fop->config.chunk_size);
-        }
+    if (lock->query && !ctx->have_info) {
+        ec_set_needed_flag(lock, link, EC_FLAG_XATTROP);
+    }
 
-        ec_fop_set_error(fop, EIO);
+    if (dirty[EC_DATA_TXN]) {
+        if (!ec_set_needed_flag(lock, link, EC_FLAG_DATA_DIRTY)) {
+            dirty[EC_DATA_TXN] = 0;
+        }
+    }
 
-        return _gf_false;
+    if (dirty[EC_METADATA_TXN]) {
+        if (!ec_set_needed_flag(lock, link, EC_FLAG_METADATA_DIRTY)) {
+            dirty[EC_METADATA_TXN] = 0;
+        }
     }
+    newflags = EC_NEEDED_FLAGS(lock->waiting_flags);
 
-    return _gf_true;
+    return oldflags ^ newflags;
 }
 
-int32_t ec_get_size_version_set(call_frame_t * frame, void * cookie,
-                                xlator_t * this, int32_t op_ret,
-                                int32_t op_errno, inode_t * inode,
-                                struct iatt * buf, dict_t * xdata,
-                                struct iatt * postparent)
+void
+ec_get_size_version(ec_lock_link_t *link)
 {
-    ec_fop_data_t * fop = cookie;
-    ec_inode_t * ctx;
-    ec_lock_t *lock = NULL;
+    loc_t loc;
+    ec_lock_t *lock;
+    ec_inode_t *ctx;
+    ec_fop_data_t *fop;
+    dict_t *dict = NULL;
+    dict_t *xdata = NULL;
+    ec_t *ec = NULL;
+    int32_t error = 0;
+    gf_boolean_t set_dirty = _gf_false;
+    uint64_t allzero[EC_VERSION_SIZE] = {0, 0};
+    uint64_t dirty[EC_VERSION_SIZE] = {0, 0};
+    lock = link->lock;
+    ctx = lock->ctx;
+    fop = link->fop;
+    ec = fop->xl->private;
+    uint64_t changed_flags = 0;
 
-    if (op_ret >= 0)
-    {
-        if ((buf->ia_type == IA_IFREG) && !ec_config_check(fop, xdata)) {
-            return 0;
+    if (ec->optimistic_changelog && !(ec->node_mask & ~link->lock->good_mask) &&
+        !ec_is_data_fop(fop->id))
+        link->optimistic_changelog = _gf_true;
+
+    memset(&loc, 0, sizeof(loc));
+
+    LOCK(&lock->loc.inode->lock);
+
+    set_dirty = ec_set_dirty_flag(link, ctx, dirty);
+
+    /* If ec metadata has already been retrieved, do not try again. */
+    if (ctx->have_info) {
+        if (ec_is_data_fop(fop->id)) {
+            fop->healing |= lock->healing;
         }
+        if (!set_dirty)
+            goto unlock;
+    }
 
-        LOCK(&inode->lock);
+    /* Determine if there's something we need to retrieve for the current
+     * operation. */
+    if (!set_dirty && !lock->query && (lock->loc.inode->ia_type != IA_IFREG) &&
+        (lock->loc.inode->ia_type != IA_INVAL)) {
+        goto unlock;
+    }
 
-        ctx = __ec_inode_get(inode, this);
-        if (ctx != NULL) {
-            if (ctx->inode_lock != NULL) {
-                lock = ctx->inode_lock;
-                lock->version = fop->answer->version;
+    changed_flags = ec_set_xattrop_flags_and_params(lock, link, dirty);
+    if (link->waiting_flags) {
+        /* This fop needs to wait until all its flags are cleared which
+         * potentially can be cleared by other xattrops that are already
+         * wound*/
+        ec_sleep(fop);
+    } else {
+        GF_ASSERT(!changed_flags);
+    }
 
-                if (buf->ia_type == IA_IFREG) {
-                    lock->have_size = 1;
-                    lock->size = buf->ia_size;
-                }
+unlock:
+    UNLOCK(&lock->loc.inode->lock);
+
+    if (!changed_flags)
+        goto out;
+
+    dict = dict_new();
+    if (dict == NULL) {
+        error = -ENOMEM;
+        goto out;
+    }
+
+    if (EC_FLAGS_HAVE(changed_flags, EC_FLAG_XATTROP)) {
+        /* Once we know that an xattrop will be needed,
+         * we try to get all available information in a
+         * single call. */
+        error = ec_dict_set_array(dict, EC_XATTR_VERSION, allzero,
+                                  EC_VERSION_SIZE);
+        if (error != 0) {
+            goto out;
+        }
+
+        if (lock->loc.inode->ia_type == IA_IFREG ||
+            lock->loc.inode->ia_type == IA_INVAL) {
+            error = ec_dict_set_number(dict, EC_XATTR_SIZE, 0);
+            if (error == 0) {
+                error = ec_dict_set_number(dict, EC_XATTR_CONFIG, 0);
             }
-            if (ctx->entry_lock != NULL) {
-                lock = ctx->entry_lock;
-                lock->version = fop->answer->version;
+            if (error != 0) {
+                goto out;
             }
-        }
 
-        UNLOCK(&inode->lock);
+            xdata = dict_new();
+            if (xdata == NULL || dict_set_int32(xdata, GF_GET_SIZE, 1)) {
+                error = -ENOMEM;
+                goto out;
+            }
+        }
+    }
 
-        if (lock != NULL)
-        {
-            // Only update parent mask if the lookup has been made with
-            // inode locked.
-            fop->parent->mask &= fop->good;
+    if (memcmp(allzero, dirty, sizeof(allzero))) {
+        error = ec_dict_set_array(dict, EC_XATTR_DIRTY, dirty, EC_VERSION_SIZE);
+        if (error != 0) {
+            goto out;
         }
+    }
 
-        if (buf->ia_type == IA_IFREG) {
-            fop->parent->pre_size = fop->parent->post_size = buf->ia_size;
-            fop->parent->have_size = 1;
+    fop->frame->root->uid = 0;
+    fop->frame->root->gid = 0;
+
+    /* For normal fops, ec_[f]xattrop() must succeed on at least
+     * EC_MINIMUM_MIN bricks, however when this is called as part of a
+     * self-heal operation the mask of target bricks (fop->mask) could
+     * contain less than EC_MINIMUM_MIN bricks, causing the xattrop to
+     * always fail. Thus we always use the same minimum used for the main
+     * fop.
+     */
+    if (lock->fd == NULL) {
+        error = ec_loc_from_loc(fop->xl, &loc, &lock->loc);
+        if (error != 0) {
+            goto out;
+        }
+        if (gf_uuid_is_null(loc.pargfid)) {
+            if (loc.parent != NULL) {
+                inode_unref(loc.parent);
+                loc.parent = NULL;
+            }
+            GF_FREE((char *)loc.path);
+            loc.path = NULL;
+            loc.name = NULL;
         }
+
+        ec_xattrop(fop->frame, fop->xl, fop->mask, fop->minimum,
+                   ec_prepare_update_cbk, link, &loc, GF_XATTROP_ADD_ARRAY64,
+                   dict, xdata);
+    } else {
+        ec_fxattrop(fop->frame, fop->xl, fop->mask, fop->minimum,
+                    ec_prepare_update_cbk, link, lock->fd,
+                    GF_XATTROP_ADD_ARRAY64, dict, xdata);
+    }
+
+    error = 0;
+
+out:
+    fop->frame->root->uid = fop->uid;
+    fop->frame->root->gid = fop->gid;
+
+    loc_wipe(&loc);
+
+    if (dict != NULL) {
+        dict_unref(dict);
+    }
+
+    if (xdata != NULL) {
+        dict_unref(xdata);
+    }
+
+    if (error != 0) {
+        ec_fop_set_error(fop, -error);
+    }
+}
+
+gf_boolean_t
+__ec_get_inode_size(ec_fop_data_t *fop, inode_t *inode, uint64_t *size)
+{
+    ec_inode_t *ctx;
+    gf_boolean_t found = _gf_false;
+
+    ctx = __ec_inode_get(inode, fop->xl);
+    if (ctx == NULL) {
+        goto out;
     }
-    else
+
+    if (ctx->have_size) {
+        *size = ctx->post_size;
+        found = _gf_true;
+    }
+
+out:
+    return found;
+}
+
+gf_boolean_t
+ec_get_inode_size(ec_fop_data_t *fop, inode_t *inode, uint64_t *size)
+{
+    gf_boolean_t found = _gf_false;
+
+    LOCK(&inode->lock);
     {
-        gf_log(this->name, GF_LOG_WARNING, "Failed to get size and version "
-                                           "(error %d)", op_errno);
-        ec_fop_set_error(fop, op_errno);
+        found = __ec_get_inode_size(fop, inode, size);
     }
+    UNLOCK(&inode->lock);
 
-    return 0;
+    return found;
 }
 
-int32_t ec_prepare_update_cbk(call_frame_t *frame, void *cookie,
-                              xlator_t *this, int32_t op_ret, int32_t op_errno,
-                              dict_t *dict, dict_t *xdata)
+gf_boolean_t
+__ec_set_inode_size(ec_fop_data_t *fop, inode_t *inode, uint64_t size)
 {
-    ec_fop_data_t *fop = cookie, *parent;
-    ec_lock_t *lock = NULL;
+    ec_inode_t *ctx;
+    gf_boolean_t found = _gf_false;
 
-    if (op_ret >= 0) {
-        parent = fop->parent;
-        while ((parent != NULL) && (parent->locks[0].lock == NULL)) {
-            parent = parent->parent;
-        }
-        if (parent == NULL) {
-            return 0;
-        }
+    ctx = __ec_inode_get(inode, fop->xl);
+    if (ctx == NULL) {
+        goto out;
+    }
 
-        lock = parent->locks[0].lock;
-        lock->is_dirty = _gf_true;
+    /* Normal fops always have ctx->have_size set. However self-heal calls this
+     * to prepare the inode, so ctx->have_size will be false. In this case we
+     * prepare both pre_size and post_size, and set have_size and have_info to
+     * true. */
+    if (!ctx->have_size) {
+        ctx->pre_size = size;
+        ctx->have_size = ctx->have_info = _gf_true;
+    }
+    ctx->post_size = size;
 
-        if (!ec_config_check(fop, dict)) {
-            return 0;
-        }
+    found = _gf_true;
 
-        LOCK(&lock->loc.inode->lock);
+out:
+    return found;
+}
 
-        if ((ec_dict_del_number(dict, EC_XATTR_VERSION, &lock->version) != 0) ||
-            (ec_dict_del_number(dict, EC_XATTR_SIZE, &lock->size) != 0)) {
-            UNLOCK(&lock->loc.inode->lock);
+gf_boolean_t
+ec_set_inode_size(ec_fop_data_t *fop, inode_t *inode, uint64_t size)
+{
+    gf_boolean_t found = _gf_false;
 
-            ec_fop_set_error(fop, EIO);
+    LOCK(&inode->lock);
+    {
+        found = __ec_set_inode_size(fop, inode, size);
+    }
+    UNLOCK(&inode->lock);
 
-            return 0;
-        }
+    return found;
+}
 
-        lock->have_size = 1;
+static void
+ec_release_stripe_cache(ec_inode_t *ctx)
+{
+    ec_stripe_list_t *stripe_cache = NULL;
+    ec_stripe_t *stripe = NULL;
 
-        UNLOCK(&lock->loc.inode->lock);
+    stripe_cache = &ctx->stripe_cache;
+    while (!list_empty(&stripe_cache->lru)) {
+        stripe = list_first_entry(&stripe_cache->lru, ec_stripe_t, lru);
+        list_del(&stripe->lru);
+        GF_FREE(stripe);
+    }
+    stripe_cache->count = 0;
+    stripe_cache->max = 0;
+}
 
-        fop->parent->mask &= fop->good;
+void
+ec_clear_inode_info(ec_fop_data_t *fop, inode_t *inode)
+{
+    ec_inode_t *ctx;
+
+    LOCK(&inode->lock);
+
+    ctx = __ec_inode_get(inode, fop->xl);
+    if (ctx == NULL) {
+        goto unlock;
+    }
+
+    ec_release_stripe_cache(ctx);
+    ctx->have_info = _gf_false;
+    ctx->have_config = _gf_false;
+    ctx->have_version = _gf_false;
+    ctx->have_size = _gf_false;
 
-        fop->parent->pre_size = fop->parent->post_size = lock->size;
-        fop->parent->have_size = 1;
+    memset(&ctx->config, 0, sizeof(ctx->config));
+    memset(ctx->pre_version, 0, sizeof(ctx->pre_version));
+    memset(ctx->post_version, 0, sizeof(ctx->post_version));
+    ctx->pre_size = ctx->post_size = 0;
+    memset(ctx->dirty, 0, sizeof(ctx->dirty));
+
+unlock:
+    UNLOCK(&inode->lock);
+}
+
+int32_t
+ec_get_real_size_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                     int32_t op_ret, int32_t op_errno, inode_t *inode,
+                     struct iatt *buf, dict_t *xdata, struct iatt *postparent)
+{
+    ec_fop_data_t *fop = cookie;
+    ec_lock_link_t *link;
+
+    if (op_ret >= 0) {
+        link = fop->data;
+        link->size = buf->ia_size;
     } else {
-        gf_log(this->name, GF_LOG_WARNING,
-               "Failed to get size and version (error %d: %s)", op_errno,
-               strerror (op_errno));
-        ec_fop_set_error(fop, op_errno);
+        /* Prevent failure of parent fop. */
+        fop->error = 0;
     }
 
     return 0;
 }
 
-void ec_get_size_version(ec_fop_data_t * fop)
+/* This function is used to get the trusted.ec.size xattr from a file when
+ * no lock is needed on the inode. This is only required to maintain iatt
+ * structs on fops that manipulate directory entries but do not operate
+ * directly on the inode, like link, rename, ...
+ *
+ * Any error processing this request is ignored. In the worst case, an invalid
+ * or not up to date value in the iatt could cause some cache invalidation.
+ */
+void
+ec_get_real_size(ec_lock_link_t *link)
 {
-    loc_t loc;
-    dict_t * xdata;
-    uid_t uid;
-    gid_t gid;
-    int32_t error = ENOMEM;
+    ec_fop_data_t *fop;
+    dict_t *xdata;
 
-    if (fop->have_size)
-    {
+    if (link->base == NULL || link->base->inode == NULL) {
         return;
     }
 
-    if ((fop->parent != NULL) && fop->parent->have_size)
-    {
-        fop->pre_size = fop->parent->pre_size;
-        fop->post_size = fop->parent->post_size;
+    if (link->base->inode->ia_type != IA_IFREG) {
+        return;
+    }
 
-        fop->have_size = 1;
+    fop = link->fop;
 
+    if (ec_get_inode_size(fop, link->base->inode, &link->size)) {
         return;
     }
 
-    memset(&loc, 0, sizeof(loc));
-
     xdata = dict_new();
-    if (xdata == NULL)
-    {
-        goto out;
+    if (xdata == NULL) {
+        return;
     }
-    if ((dict_set_uint64(xdata, EC_XATTR_VERSION, 0) != 0) ||
-        (dict_set_uint64(xdata, EC_XATTR_SIZE, 0) != 0) ||
-        (dict_set_uint64(xdata, EC_XATTR_CONFIG, 0) != 0) ||
-        (dict_set_uint64(xdata, EC_XATTR_DIRTY, 0) != 0))
-    {
+    if (ec_dict_set_number(xdata, EC_XATTR_SIZE, 0) != 0) {
         goto out;
     }
 
-    uid = fop->frame->root->uid;
-    gid = fop->frame->root->gid;
+    /* Send a simple lookup. A single answer is considered ok since this value
+     * is only used to return an iatt struct related to an inode that is not
+     * locked and have not suffered any operation. */
+    ec_lookup(fop->frame, fop->xl, fop->mask, 1, ec_get_real_size_cbk, link,
+              link->base, xdata);
 
-    fop->frame->root->uid = 0;
-    fop->frame->root->gid = 0;
+out:
+    if (xdata != NULL) {
+        dict_unref(xdata);
+    }
+}
+
+static void
+ec_lock_update_fd(ec_lock_t *lock, ec_fop_data_t *fop)
+{
+    /* If the fop has an fd available, attach it to the lock structure to be
+     * able to do fxattrop calls instead of xattrop. */
+    if (fop->use_fd && (lock->fd == NULL)) {
+        lock->fd = __fd_ref(fop->fd);
+    }
+}
 
-    error = EIO;
+static gf_boolean_t
+ec_link_has_lock_conflict(ec_lock_link_t *link, gf_boolean_t waitlist_check)
+{
+    ec_lock_link_t *trav_link = NULL;
 
-    if (!fop->use_fd)
+    list_for_each_entry(trav_link, &link->lock->owners, owner_list)
     {
-        if (ec_loc_from_loc(fop->xl, &loc, &fop->loc[0]) != 0)
-        {
-            goto out;
+        if (ec_lock_conflict(trav_link, link))
+            return _gf_true;
+    }
+
+    if (!waitlist_check)
+        return _gf_false;
+
+    list_for_each_entry(trav_link, &link->lock->waiting, wait_list)
+    {
+        if (ec_lock_conflict(trav_link, link))
+            return _gf_true;
+    }
+
+    return _gf_false;
+}
+
+static void
+ec_lock_wake_shared(ec_lock_t *lock, struct list_head *list)
+{
+    ec_fop_data_t *fop;
+    ec_lock_link_t *link;
+    gf_boolean_t conflict = _gf_false;
+
+    while (!conflict && !list_empty(&lock->waiting)) {
+        link = list_entry(lock->waiting.next, ec_lock_link_t, wait_list);
+        fop = link->fop;
+
+        /* If lock is not acquired, at most one fop can be assigned as owner.
+         * The following fops will need to wait in the lock->waiting queue
+         * until the lock has been fully acquired. */
+        conflict = !lock->acquired;
+
+        /* If the fop is not shareable, only this fop can be assigned as owner.
+         * Other fops will need to wait until this one finishes. */
+        if (ec_link_has_lock_conflict(link, _gf_false)) {
+            conflict = _gf_true;
         }
-        if (gf_uuid_is_null(loc.pargfid))
-        {
-            if (loc.parent != NULL)
-            {
-                inode_unref(loc.parent);
-                loc.parent = NULL;
-            }
-            GF_FREE((char *)loc.path);
-            loc.path = NULL;
-            loc.name = NULL;
+
+        /* If only one fop is allowed, it can be assigned as the owner of the
+         * lock only if there weren't any other owner. */
+        if (conflict && !list_empty(&lock->owners)) {
+            break;
         }
-    } else if (ec_loc_from_fd(fop->xl, &loc, fop->fd) != 0) {
-        goto out;
+
+        list_move_tail(&link->wait_list, list);
+
+        list_add_tail(&link->owner_list, &lock->owners);
+        lock->refs_owners++;
+
+        ec_lock_update_fd(lock, fop);
     }
+}
 
-    /* For normal fops, ec_lookup() must succeed on at least EC_MINIMUM_MIN
-     * bricks, however when this is called as part of a self-heal operation
-     * the mask of target bricks (fop->mask) could contain less than
-     * EC_MINIMUM_MIN bricks, causing the lookup to always fail. Thus we
-     * always use the same minimum used for the main fop.
-     */
-    ec_lookup(fop->frame, fop->xl, fop->mask, fop->minimum,
-              ec_get_size_version_set, NULL, &loc, xdata);
+static void
+ec_lock_apply(ec_lock_link_t *link)
+{
+    ec_fop_data_t *fop = link->fop;
 
-    fop->frame->root->uid = uid;
-    fop->frame->root->gid = gid;
+    fop->mask &= link->lock->good_mask;
+    fop->locked++;
 
-    error = 0;
+    ec_get_size_version(link);
+    ec_get_real_size(link);
+}
 
-out:
-    loc_wipe(&loc);
+gf_boolean_t
+ec_lock_acquire(ec_lock_link_t *link);
 
-    if (xdata != NULL)
-    {
-        dict_unref(xdata);
-    }
+static void
+ec_lock_resume_shared(struct list_head *list)
+{
+    ec_lock_link_t *link;
 
-    if (error != 0) {
-        ec_fop_set_error(fop, error);
+    while (!list_empty(list)) {
+        link = list_entry(list->next, ec_lock_link_t, wait_list);
+        list_del_init(&link->wait_list);
+
+        if (link->lock->acquired) {
+            ec_lock_apply(link);
+            ec_lock(link->fop);
+        } else {
+            GF_ASSERT(list_empty(list));
+
+            ec_lock_acquire(link);
+        }
+
+        ec_resume(link->fop, 0);
     }
 }
 
-void ec_prepare_update(ec_fop_data_t *fop)
+void
+ec_lock_acquired(ec_lock_link_t *link)
 {
-    loc_t loc;
-    dict_t *xdata;
-    ec_fop_data_t *tmp;
+    struct list_head list;
     ec_lock_t *lock;
-    uid_t uid;
-    gid_t gid;
-    int32_t error = ENOMEM;
+    ec_fop_data_t *fop;
+
+    lock = link->lock;
+    fop = link->fop;
 
-    tmp = fop;
-    while ((tmp != NULL) && (tmp->locks[0].lock == NULL)) {
-        tmp = tmp->parent;
+    ec_trace("LOCKED", fop, "lock=%p", lock);
+
+    INIT_LIST_HEAD(&list);
+
+    LOCK(&lock->loc.inode->lock);
+
+    lock->acquired = _gf_true;
+    if (lock->contention) {
+        lock->release = _gf_true;
+        lock->contention = _gf_false;
     }
-    if ((tmp != NULL) && tmp->locks[0].lock->is_dirty) {
-        lock = tmp->locks[0].lock;
 
-        fop->pre_size = fop->post_size = lock->size;
-        fop->have_size = 1;
+    ec_lock_update_fd(lock, fop);
+    ec_lock_wake_shared(lock, &list);
 
-        return;
+    UNLOCK(&lock->loc.inode->lock);
+
+    ec_lock_apply(link);
+
+    if (fop->use_fd &&
+        (link->update[EC_DATA_TXN] || link->update[EC_METADATA_TXN])) {
+        /* Try to reopen closed fd's only if lock has succeeded. */
+        ec_fix_open(fop, lock->mask);
     }
 
-    memset(&loc, 0, sizeof(loc));
+    ec_lock_resume_shared(&list);
+}
 
-    xdata = dict_new();
-    if (xdata == NULL) {
-        goto out;
+int32_t
+ec_locked(call_frame_t *frame, void *cookie, xlator_t *this, int32_t op_ret,
+          int32_t op_errno, dict_t *xdata)
+{
+    ec_fop_data_t *fop = cookie;
+    ec_lock_link_t *link = NULL;
+    ec_lock_t *lock = NULL;
+
+    link = fop->data;
+    lock = link->lock;
+    if (op_ret >= 0) {
+        lock->mask = lock->good_mask = fop->good;
+        lock->healing = 0;
+
+        ec_lock_acquired(link);
+        ec_lock(fop->parent);
+    } else {
+        LOCK(&lock->loc.inode->lock);
+        {
+            lock->contention = _gf_false;
+        }
+        UNLOCK(&lock->loc.inode->lock);
+        gf_msg(this->name, GF_LOG_WARNING, op_errno, EC_MSG_PREOP_LOCK_FAILED,
+               "Failed to complete preop lock");
     }
-    if ((ec_dict_set_number(xdata, EC_XATTR_VERSION, 0) != 0) ||
-        (ec_dict_set_number(xdata, EC_XATTR_SIZE, 0) != 0) ||
-        (ec_dict_set_number(xdata, EC_XATTR_CONFIG, 0) != 0) ||
-        (ec_dict_set_number(xdata, EC_XATTR_DIRTY, 1) != 0)) {
-        goto out;
+
+    return 0;
+}
+
+gf_boolean_t
+ec_lock_acquire(ec_lock_link_t *link)
+{
+    ec_lock_t *lock;
+    ec_fop_data_t *fop;
+    gf_lkowner_t lk_owner;
+
+    lock = link->lock;
+    fop = link->fop;
+
+    if (!lock->acquired) {
+        set_lk_owner_from_ptr(&lk_owner, lock);
+
+        ec_trace("LOCK_ACQUIRE", fop, "lock=%p, inode=%p", lock,
+                 lock->loc.inode);
+
+        lock->flock.l_type = F_WRLCK;
+        ec_inodelk(fop->frame, fop->xl, &lk_owner, -1, EC_MINIMUM_ALL,
+                   ec_locked, link, fop->xl->name, &lock->loc, F_SETLKW,
+                   &lock->flock, NULL);
+
+        return _gf_false;
     }
 
-    uid = fop->frame->root->uid;
-    gid = fop->frame->root->gid;
+    ec_trace("LOCK_REUSE", fop, "lock=%p", lock);
 
-    fop->frame->root->uid = 0;
-    fop->frame->root->gid = 0;
+    ec_lock_acquired(link);
 
-    error = EIO;
+    return _gf_true;
+}
 
-    if (!fop->use_fd) {
-        if (ec_loc_from_loc(fop->xl, &loc, &fop->loc[0]) != 0) {
-            goto out;
-        }
+static ec_lock_link_t *
+ec_lock_timer_cancel(xlator_t *xl, ec_lock_t *lock)
+{
+    ec_lock_link_t *timer_link;
 
-        ec_xattrop(fop->frame, fop->xl, fop->mask, fop->minimum,
-                   ec_prepare_update_cbk, NULL, &loc, GF_XATTROP_ADD_ARRAY64,
-                   xdata, NULL);
+    /* If we don't have any timer, there's nothing to cancel. */
+    if (lock->timer == NULL) {
+        return NULL;
+    }
+
+    /* We are trying to access a lock that has an unlock timer active.
+     * This means that the lock must be idle, i.e. no fop can be in the
+     * owner, waiting or frozen lists. It also means that the lock cannot
+     * have been marked as being released (this is done without timers).
+     * There should only be one owner reference, but it's possible that
+     * some fops are being prepared to use this lock. */
+    GF_ASSERT((lock->refs_owners == 1) && list_empty(&lock->owners) &&
+              list_empty(&lock->waiting));
+
+    /* We take the timer_link before cancelling the timer, since a
+     * successful cancellation will destroy it. It must not be NULL
+     * because it references the fop responsible for the delayed unlock
+     * that we are currently trying to cancel. */
+    timer_link = lock->timer->data;
+    GF_ASSERT(timer_link != NULL);
+
+    if (gf_timer_call_cancel(xl->ctx, lock->timer) < 0) {
+        /* It's too late to avoid the execution of the timer callback.
+         * Since we need to be sure that the callback has access to all
+         * needed resources, we cannot resume the execution of the
+         * timer fop now. This will be done in the callback. */
+        timer_link = NULL;
     } else {
-        ec_fxattrop(fop->frame, fop->xl, fop->mask, fop->minimum,
-                   ec_prepare_update_cbk, NULL, fop->fd,
-                   GF_XATTROP_ADD_ARRAY64, xdata, NULL);
+        /* The timer has been cancelled. The fop referenced by
+         * timer_link holds the last reference. The caller is
+         * responsible to release it when not needed anymore. */
+        ec_trace("UNLOCK_CANCELLED", timer_link->fop, "lock=%p", lock);
+    }
+
+    /* We have two options here:
+     *
+     * 1. The timer has been successfully cancelled.
+     *
+     *    This is the easiest case and we can continue with the currently
+     *    acquired lock.
+     *
+     * 2. The timer callback has already been fired.
+     *
+     *    In this case we have not been able to cancel the timer before
+     *    the timer callback has been fired, but we also know that
+     *    lock->timer != NULL. This means that the timer callback is still
+     *    trying to acquire the inode mutex that we currently own. We are
+     *    safe until we release it. In this case we can safely clear
+     *    lock->timer. This will cause that the timer callback does nothing
+     *    once it acquires the mutex.
+     */
+    lock->timer = NULL;
+
+    return timer_link;
+}
+
+static gf_boolean_t
+ec_lock_assign_owner(ec_lock_link_t *link)
+{
+    ec_fop_data_t *fop;
+    ec_lock_t *lock;
+    ec_lock_link_t *timer_link = NULL;
+    gf_boolean_t assigned = _gf_false;
+
+    /* The link cannot be in any list because we have just finished preparing
+     * it. */
+    GF_ASSERT(list_empty(&link->wait_list));
+
+    fop = link->fop;
+    lock = link->lock;
+
+    LOCK(&lock->loc.inode->lock);
+
+    /* Since the link has just been prepared but it's not active yet, the
+     * refs_pending must be one at least (the ref owned by this link). */
+    GF_ASSERT(lock->refs_pending > 0);
+    /* The link is not pending any more. It will be assigned to the owner,
+     * waiting or frozen list. */
+    lock->refs_pending--;
+
+    if (lock->release) {
+        ec_trace("LOCK_QUEUE_FREEZE", fop, "lock=%p", lock);
+
+        /* When lock->release is set, we'll unlock the lock as soon as
+         * possible, meaning that we won't use a timer. */
+        GF_ASSERT(lock->timer == NULL);
+
+        /* The lock is marked to be released. We can still have owners and fops
+         * in the waiting ilist f they have been added before the lock has been
+         * marked to be released. However new fops are put into the frozen list
+         * to wait for the next unlock/lock cycle. */
+        list_add_tail(&link->wait_list, &lock->frozen);
+
+        goto unlock;
     }
 
-    fop->frame->root->uid = uid;
-    fop->frame->root->gid = gid;
+    /* The lock is not marked to be released, so the frozen list should be
+     * empty. */
+    GF_ASSERT(list_empty(&lock->frozen));
 
-    error = 0;
+    timer_link = ec_lock_timer_cancel(fop->xl, lock);
 
-out:
-    loc_wipe(&loc);
+    if (!list_empty(&lock->owners)) {
+        /* There are other owners of this lock. We can only take ownership if
+         * the lock is already acquired and doesn't have conflict with existing
+         * owners, or waiters(to prevent starvation).
+         * Otherwise we need to wait.
+         */
+        if (!lock->acquired || ec_link_has_lock_conflict(link, _gf_true)) {
+            ec_trace("LOCK_QUEUE_WAIT", fop, "lock=%p", lock);
 
-    if (xdata != NULL) {
-        dict_unref(xdata);
+            list_add_tail(&link->wait_list, &lock->waiting);
+
+            goto unlock;
+        }
     }
 
-    if (error != 0) {
-        ec_fop_set_error(fop, error);
+    list_add_tail(&link->owner_list, &lock->owners);
+
+    /* If timer_link is not NULL, it means that we have inherited the owner
+     * reference assigned to the timer fop. In this case we simply reuse it.
+     * Otherwise we need to increase the number of owners. */
+    if (timer_link == NULL) {
+        lock->refs_owners++;
     }
+
+    assigned = _gf_true;
+
+unlock:
+    if (!assigned) {
+        /* We have not been able to take ownership of this lock. The fop must
+         * be put to sleep. */
+        ec_sleep(fop);
+    }
+
+    UNLOCK(&lock->loc.inode->lock);
+
+    /* If we have cancelled the timer, we need to resume the fop that was
+     * waiting for it. */
+    if (timer_link != NULL) {
+        ec_resume(timer_link->fop, 0);
+    }
+
+    return assigned;
 }
 
-int32_t ec_unlocked(call_frame_t *frame, void *cookie, xlator_t *this,
-                    int32_t op_ret, int32_t op_errno, dict_t *xdata)
+static void
+ec_lock_next_owner(ec_lock_link_t *link, ec_cbk_data_t *cbk,
+                   gf_boolean_t release)
 {
-    ec_fop_data_t *fop = cookie;
+    struct list_head list;
+    ec_lock_t *lock = link->lock;
+    ec_fop_data_t *fop = link->fop;
+    ec_inode_t *ctx = lock->ctx;
 
-    if (op_ret < 0) {
-        gf_log(this->name, GF_LOG_WARNING, "entry/inode unlocking failed (%s)",
-               ec_fop_name(fop->parent->id));
-    } else {
-        ec_trace("UNLOCKED", fop->parent, "lock=%p", fop->data);
+    INIT_LIST_HEAD(&list);
+
+    LOCK(&lock->loc.inode->lock);
+
+    ec_trace("LOCK_DONE", fop, "lock=%p", lock);
+
+    /* Current link must belong to the owner list of the lock. We don't
+     * decrement lock->refs_owners here because the inode mutex is released
+     * before ec_unlock() is called and we need to know when the last owner
+     * unlocks the lock to do proper cleanup. lock->refs_owners is used for
+     * this task. */
+    GF_ASSERT((lock->refs_owners > 0) && !list_empty(&link->owner_list));
+    list_del_init(&link->owner_list);
+
+    lock->release |= release;
+
+    if ((fop->error == 0) && (cbk != NULL) && (cbk->op_ret >= 0)) {
+        if (link->update[0]) {
+            ctx->post_version[0]++;
+        }
+        if (link->update[1]) {
+            ctx->post_version[1]++;
+        }
+        /* If the fop fails on any of the good bricks, it is important to mark
+         * it dirty and update versions right away. */
+        if (link->update[0] || link->update[1]) {
+            if (lock->good_mask & ~(fop->good | fop->remaining)) {
+                lock->release = _gf_true;
+            }
+        }
     }
 
-    return 0;
+    if (fop->healing) {
+        lock->healing = fop->healing & (fop->good | fop->remaining);
+    }
+    ec_lock_update_good(lock, fop);
+
+    ec_lock_wake_shared(lock, &list);
+
+    UNLOCK(&lock->loc.inode->lock);
+
+    ec_lock_resume_shared(&list);
 }
 
-void ec_unlock_lock(ec_fop_data_t *fop, ec_lock_t *lock)
+void
+ec_lock(ec_fop_data_t *fop)
 {
-    if ((lock->mask != 0) && lock->acquired) {
-        ec_owner_set(fop->frame, lock);
+    ec_lock_link_t *link;
 
-        switch (lock->kind) {
-        case EC_LOCK_ENTRY:
-            ec_trace("UNLOCK_ENTRYLK", fop, "lock=%p, inode=%p, path=%s", lock,
-                     lock->loc.inode, lock->loc.path);
+    /* There is a chance that ec_resume is called on fop even before ec_sleep.
+     * Which can result in refs == 0 for fop leading to use after free in this
+     * function when it calls ec_sleep so do ec_sleep at start and ec_resume at
+     * the end of this function.*/
+    ec_sleep(fop);
 
-            ec_entrylk(fop->frame, fop->xl, lock->mask, EC_MINIMUM_ALL,
-                       ec_unlocked, lock, fop->xl->name, &lock->loc, NULL,
-                       ENTRYLK_UNLOCK, lock->type, NULL);
+    while (fop->locked < fop->lock_count) {
+        /* Since there are only up to 2 locks per fop, this xor will change
+         * the order of the locks if fop->first_lock is 1. */
+        link = &fop->locks[fop->locked ^ fop->first_lock];
 
+        if (!ec_lock_assign_owner(link) || !ec_lock_acquire(link)) {
             break;
+        }
+    }
+
+    ec_resume(fop, 0);
+}
 
-        case EC_LOCK_INODE:
-            lock->flock.l_type = F_UNLCK;
-            ec_trace("UNLOCK_INODELK", fop, "lock=%p, inode=%p", lock,
-                     lock->loc.inode);
+void
+ec_lock_unfreeze(ec_lock_link_t *link)
+{
+    struct list_head list;
+    ec_lock_t *lock;
+    gf_boolean_t destroy = _gf_false;
 
-            ec_inodelk(fop->frame, fop->xl, lock->mask, EC_MINIMUM_ALL,
-                       ec_unlocked, lock, fop->xl->name, &lock->loc, F_SETLK,
-                       &lock->flock, NULL);
+    lock = link->lock;
 
-            break;
+    INIT_LIST_HEAD(&list);
 
-        default:
-            gf_log(fop->xl->name, GF_LOG_ERROR, "Invalid lock type");
-        }
+    LOCK(&lock->loc.inode->lock);
+
+    /* The lock must be marked to be released here, since we have just released
+     * it and any attempt to assign it to more fops must have added them to the
+     * frozen list. We can only have one active reference here: the one that
+     * is processing this unfreeze. */
+    GF_ASSERT(lock->release && (lock->refs_owners == 1));
+    lock->release = _gf_false;
+    lock->refs_owners = 0;
+
+    lock->acquired = _gf_false;
+
+    /* We are unfreezing a lock. This means that the lock has already been
+     * released. In this state it shouldn't have a pending timer nor have any
+     * owner, and the waiting list should be empty. Only the frozen list can
+     * contain some fop. */
+    GF_ASSERT((lock->timer == NULL) && list_empty(&lock->waiting) &&
+              list_empty(&lock->owners));
+
+    /* We move all frozen fops to the waiting list. */
+    list_splice_init(&lock->frozen, &lock->waiting);
+
+    /* If we don't have any fop waiting nor there are any prepared fops using
+     * this lock, we can finally dispose it. */
+    destroy = list_empty(&lock->waiting) && (lock->refs_pending == 0);
+    if (destroy) {
+        ec_trace("LOCK_DESTROY", link->fop, "lock=%p", lock);
+
+        lock->ctx->inode_lock = NULL;
+    } else {
+        ec_trace("LOCK_UNFREEZE", link->fop, "lock=%p", lock);
+
+        ec_lock_wake_shared(lock, &list);
     }
 
-    ec_trace("LOCK_DESTROY", fop, "lock=%p", lock);
+    UNLOCK(&lock->loc.inode->lock);
+
+    ec_lock_resume_shared(&list);
 
-    ec_lock_destroy(lock);
+    if (destroy) {
+        ec_lock_destroy(lock);
+    }
 }
 
-int32_t ec_update_size_version_done(call_frame_t * frame, void * cookie,
-                                    xlator_t * this, int32_t op_ret,
-                                    int32_t op_errno, dict_t * xattr,
-                                    dict_t * xdata)
+int32_t
+ec_unlocked(call_frame_t *frame, void *cookie, xlator_t *this, int32_t op_ret,
+            int32_t op_errno, dict_t *xdata)
 {
-    ec_fop_data_t * fop = cookie;
+    ec_fop_data_t *fop = cookie;
+    ec_lock_link_t *link = fop->data;
 
-    if (op_ret < 0)
-    {
-        gf_log(fop->xl->name, GF_LOG_ERROR, "Failed to update version and "
-                                            "size (error %d)", op_errno);
-    }
-    else
-    {
-        fop->parent->mask &= fop->good;
+    if (op_ret < 0) {
+        gf_msg(this->name, GF_LOG_WARNING, op_errno, EC_MSG_UNLOCK_FAILED,
+               "entry/inode unlocking failed :(%s)", ec_msg_str(link->fop));
+    } else {
+        ec_trace("UNLOCKED", link->fop, "lock=%p", link->lock);
     }
 
-    if (fop->data != NULL) {
-        ec_unlock_lock(fop->parent, fop->data);
-    }
+    ec_lock_unfreeze(link);
 
     return 0;
 }
 
-void ec_update_size_version(ec_fop_data_t *fop, loc_t *loc, uint64_t version,
-                            uint64_t size, gf_boolean_t dirty, ec_lock_t *lock)
+void
+ec_unlock_lock(ec_lock_link_t *link)
 {
-    dict_t * dict;
-    uid_t uid;
-    gid_t gid;
+    ec_lock_t *lock;
+    ec_fop_data_t *fop;
+    gf_lkowner_t lk_owner;
+
+    lock = link->lock;
+    fop = link->fop;
 
-    if (fop->parent != NULL)
+    lock->unlock_now = _gf_false;
+    ec_clear_inode_info(fop, lock->loc.inode);
+
+    if ((lock->mask != 0) && lock->acquired) {
+        set_lk_owner_from_ptr(&lk_owner, lock);
+        lock->flock.l_type = F_UNLCK;
+        ec_trace("UNLOCK_INODELK", fop, "lock=%p, inode=%p", lock,
+                 lock->loc.inode);
+
+        ec_inodelk(fop->frame, fop->xl, &lk_owner, lock->mask, EC_MINIMUM_ONE,
+                   ec_unlocked, link, fop->xl->name, &lock->loc, F_SETLK,
+                   &lock->flock, NULL);
+    } else {
+        ec_lock_unfreeze(link);
+    }
+}
+
+void
+ec_inode_bad_inc(inode_t *inode, xlator_t *xl)
+{
+    ec_inode_t *ctx = NULL;
+
+    LOCK(&inode->lock);
     {
-        fop->parent->post_size = fop->post_size;
+        ctx = __ec_inode_get(inode, xl);
+        if (ctx == NULL) {
+            goto unlock;
+        }
+        ctx->bad_version++;
+    }
+unlock:
+    UNLOCK(&inode->lock);
+}
 
-        return;
+int32_t
+ec_update_size_version_done(call_frame_t *frame, void *cookie, xlator_t *this,
+                            int32_t op_ret, int32_t op_errno, dict_t *xattr,
+                            dict_t *xdata)
+{
+    ec_fop_data_t *fop = cookie;
+    ec_lock_link_t *link;
+    ec_lock_t *lock;
+    ec_inode_t *ctx;
+
+    link = fop->data;
+    lock = link->lock;
+    ctx = lock->ctx;
+
+    if (op_ret < 0) {
+        if (link->lock->fd == NULL) {
+            ec_inode_bad_inc(link->lock->loc.inode, this);
+        } else {
+            ec_inode_bad_inc(link->lock->fd->inode, this);
+        }
+
+        gf_msg(fop->xl->name, fop_log_level(fop->id, op_errno), op_errno,
+               EC_MSG_SIZE_VERS_UPDATE_FAIL,
+               "Failed to update version and size. %s", ec_msg_str(fop));
+    } else {
+        fop->parent->good &= fop->good;
+
+        ec_lock_update_good(lock, fop);
+
+        if (ec_dict_del_array(xattr, EC_XATTR_VERSION, ctx->post_version,
+                              EC_VERSION_SIZE) == 0) {
+            ctx->pre_version[0] = ctx->post_version[0];
+            ctx->pre_version[1] = ctx->post_version[1];
+
+            ctx->have_version = _gf_true;
+        }
+        if (ec_dict_del_number(xattr, EC_XATTR_SIZE, &ctx->post_size) == 0) {
+            ctx->pre_size = ctx->post_size;
+
+            ctx->have_size = _gf_true;
+        }
+        if ((ec_dict_del_config(xdata, EC_XATTR_CONFIG, &ctx->config) == 0) &&
+            ec_config_check(fop->xl, &ctx->config)) {
+            ctx->have_config = _gf_true;
+        }
+
+        ctx->have_info = _gf_true;
+    }
+    /* If we are here because of fop's and other than unlock request,
+     * that means we are still holding a lock. That make sure
+     * lock->unlock_now can not be modified.
+     */
+    if (lock->unlock_now) {
+        ec_unlock_lock(fop->data);
     }
 
-    ec_trace("UPDATE", fop, "version=%ld, size=%ld, dirty=%u", version, size,
-             dirty);
+    return 0;
+}
+
+void
+ec_update_size_version(ec_lock_link_t *link, uint64_t *version, uint64_t size,
+                       uint64_t *dirty)
+{
+    ec_fop_data_t *fop;
+    ec_lock_t *lock;
+    ec_inode_t *ctx;
+    dict_t *dict = NULL;
+    uintptr_t update_on = 0;
+    int32_t err = -ENOMEM;
+
+    fop = link->fop;
+    lock = link->lock;
+    ctx = lock->ctx;
+
+    ec_trace("UPDATE", fop, "version=%ld/%ld, size=%ld, dirty=%ld/%ld",
+             version[0], version[1], size, dirty[0], dirty[1]);
 
     dict = dict_new();
-    if (dict == NULL)
-    {
+    if (dict == NULL) {
         goto out;
     }
 
-    if (version != 0) {
-        if (ec_dict_set_number(dict, EC_XATTR_VERSION, version) != 0) {
+    /* If we don't have version information or it has been modified, we
+     * update it. */
+    if (!ctx->have_version || (version[0] != 0) || (version[1] != 0)) {
+        err = ec_dict_set_array(dict, EC_XATTR_VERSION, version,
+                                EC_VERSION_SIZE);
+        if (err != 0) {
             goto out;
         }
     }
+
     if (size != 0) {
-        if (ec_dict_set_number(dict, EC_XATTR_SIZE, size) != 0) {
+        /* If size has been changed, we should already
+         * know the previous size of the file. */
+        GF_ASSERT(ctx->have_size);
+
+        err = ec_dict_set_number(dict, EC_XATTR_SIZE, size);
+        if (err != 0) {
             goto out;
         }
     }
-    if (dirty) {
-        if (ec_dict_set_number(dict, EC_XATTR_DIRTY, -1) != 0) {
+
+    if (dirty[0] || dirty[1]) {
+        err = ec_dict_set_array(dict, EC_XATTR_DIRTY, dirty, EC_VERSION_SIZE);
+        if (err != 0) {
             goto out;
         }
     }
 
-    uid = fop->frame->root->uid;
-    gid = fop->frame->root->gid;
+    /* If config information is not known, we request it now. */
+    if ((lock->loc.inode->ia_type == IA_IFREG) && !ctx->have_config) {
+        /* A failure requesting this xattr is ignored because it's not
+         * absolutely required right now. */
+        (void)ec_dict_set_number(dict, EC_XATTR_CONFIG, 0);
+    }
 
     fop->frame->root->uid = 0;
     fop->frame->root->gid = 0;
 
-    if (fop->use_fd) {
-            ec_fxattrop(fop->frame, fop->xl, fop->mask, EC_MINIMUM_MIN,
-                       ec_update_size_version_done, lock, fop->fd,
-                       GF_XATTROP_ADD_ARRAY64, dict, NULL);
+    update_on = lock->good_mask | lock->healing;
+
+    if (link->lock->fd == NULL) {
+        ec_xattrop(fop->frame, fop->xl, update_on, EC_MINIMUM_MIN,
+                   ec_update_size_version_done, link, &link->lock->loc,
+                   GF_XATTROP_ADD_ARRAY64, dict, NULL);
     } else {
-            ec_xattrop(fop->frame, fop->xl, fop->mask, EC_MINIMUM_MIN,
-                       ec_update_size_version_done, lock, loc,
-                       GF_XATTROP_ADD_ARRAY64, dict, NULL);
+        ec_fxattrop(fop->frame, fop->xl, update_on, EC_MINIMUM_MIN,
+                    ec_update_size_version_done, link, link->lock->fd,
+                    GF_XATTROP_ADD_ARRAY64, dict, NULL);
     }
 
-    fop->frame->root->uid = uid;
-    fop->frame->root->gid = gid;
+    fop->frame->root->uid = fop->uid;
+    fop->frame->root->gid = fop->gid;
 
     dict_unref(dict);
 
     return;
 
 out:
-    if (dict != NULL)
-    {
+    if (dict != NULL) {
         dict_unref(dict);
     }
 
-    ec_fop_set_error(fop, EIO);
+    ec_fop_set_error(fop, -err);
 
-    gf_log(fop->xl->name, GF_LOG_ERROR, "Unable to update version and size");
+    gf_msg(fop->xl->name, GF_LOG_ERROR, -err, EC_MSG_SIZE_VERS_UPDATE_FAIL,
+           "Unable to update version and size. %s", ec_msg_str(fop));
+
+    if (lock->unlock_now) {
+        ec_unlock_lock(fop->data);
+    }
 }
 
-void ec_unlock_now(ec_fop_data_t *fop, ec_lock_t *lock)
+gf_boolean_t
+ec_update_info(ec_lock_link_t *link)
 {
-    ec_trace("UNLOCK_NOW", fop, "lock=%p", lock);
+    ec_lock_t *lock;
+    ec_inode_t *ctx;
+    uint64_t version[2] = {0, 0};
+    uint64_t dirty[2] = {0, 0};
+    uint64_t size;
+    ec_t *ec = NULL;
+    uintptr_t mask;
 
-    if ((lock->version_delta != 0) || lock->is_dirty) {
-        ec_update_size_version(fop, &lock->loc, lock->version_delta,
-                               lock->size_delta, lock->is_dirty, lock);
-    } else {
-        ec_unlock_lock(fop, lock);
+    lock = link->lock;
+    ctx = lock->ctx;
+    ec = link->fop->xl->private;
+
+    /* pre_version[*] will be 0 if have_version is false */
+    version[EC_DATA_TXN] = ctx->post_version[EC_DATA_TXN] -
+                           ctx->pre_version[EC_DATA_TXN];
+    version[EC_METADATA_TXN] = ctx->post_version[EC_METADATA_TXN] -
+                               ctx->pre_version[EC_METADATA_TXN];
+
+    size = ctx->post_size - ctx->pre_size;
+    /* If we set the dirty flag for update fop, we have to unset it.
+     * If fop has failed on some bricks, leave the dirty as marked. */
+
+    if (lock->unlock_now) {
+        if (version[EC_DATA_TXN]) {
+            /*A data fop will have difference in post and pre version
+             *and for data fop we send writes on healing bricks also */
+            mask = lock->good_mask | lock->healing;
+        } else {
+            mask = lock->good_mask;
+        }
+        /* Ensure that nodes are up while doing final
+         * metadata update.*/
+        if (!(ec->node_mask & ~(mask)) && !(ec->node_mask & ~ec->xl_up)) {
+            if (ctx->dirty[EC_DATA_TXN] != 0) {
+                dirty[EC_DATA_TXN] = -1;
+            }
+            if (ctx->dirty[EC_METADATA_TXN] != 0) {
+                dirty[EC_METADATA_TXN] = -1;
+            }
+            /*If everything is fine and we already
+             *have version xattr set on entry, there
+             *is no need to update version again*/
+            if (ctx->pre_version[EC_DATA_TXN]) {
+                version[EC_DATA_TXN] = 0;
+            }
+            if (ctx->pre_version[EC_METADATA_TXN]) {
+                version[EC_METADATA_TXN] = 0;
+            }
+        } else {
+            link->optimistic_changelog = _gf_false;
+            ec_set_dirty_flag(link, ctx, dirty);
+        }
+        memset(ctx->dirty, 0, sizeof(ctx->dirty));
     }
 
-    ec_resume(fop, 0);
+    if ((version[EC_DATA_TXN] != 0) || (version[EC_METADATA_TXN] != 0) ||
+        (dirty[EC_DATA_TXN] != 0) || (dirty[EC_METADATA_TXN] != 0)) {
+        ec_update_size_version(link, version, size, dirty);
+        return _gf_true;
+    }
+
+    return _gf_false;
 }
 
-void ec_unlock_timer_cbk(void *data)
+void
+ec_unlock_now(ec_lock_link_t *link)
 {
-    ec_lock_link_t *link = data;
-    ec_lock_t *lock = link->lock;
-    ec_fop_data_t *fop = NULL;
+    ec_lock_t *lock;
+    lock = link->lock;
 
-    LOCK(&lock->loc.inode->lock);
+    ec_trace("UNLOCK_NOW", link->fop, "lock=%p", link->lock);
+    /*At this point, lock is not being used by any fop and
+     *can not be reused by any fop as it is going to be released.
+     *lock->unlock_now can not be modified at any other place.
+     */
+    lock->unlock_now = _gf_true;
 
-    if (lock->timer != NULL) {
-        fop = link->fop;
+    if (!ec_update_info(link)) {
+        ec_unlock_lock(link);
+    }
+
+    ec_resume(link->fop, 0);
+}
+
+void
+ec_lock_release(ec_t *ec, inode_t *inode)
+{
+    ec_lock_t *lock;
+    ec_inode_t *ctx;
+    ec_lock_link_t *timer_link = NULL;
+
+    LOCK(&inode->lock);
+
+    ctx = __ec_inode_get(inode, ec->xl);
+    if (ctx == NULL) {
+        goto done;
+    }
+    lock = ctx->inode_lock;
+    if ((lock == NULL) || lock->release) {
+        goto done;
+    }
+
+    gf_msg_debug(ec->xl->name, 0, "Releasing inode %p due to lock contention",
+                 inode);
+
+    if (!lock->acquired) {
+        /* This happens if some bricks already got the lock while inodelk is in
+         * progress.  Set release to true after lock is acquired*/
+        lock->contention = _gf_true;
+        goto done;
+    }
+
+    /* The lock is not marked to be released, so the frozen list should be
+     * empty. */
+    GF_ASSERT(list_empty(&lock->frozen));
+
+    timer_link = ec_lock_timer_cancel(ec->xl, lock);
 
-        ec_trace("UNLOCK_DELAYED", fop, "lock=%p", lock);
+    /* We mark the lock to be released as soon as possible. */
+    lock->release = _gf_true;
 
-        GF_ASSERT(lock->refs == 1);
+done:
+    UNLOCK(&inode->lock);
 
-        gf_timer_call_cancel(fop->xl->ctx, lock->timer);
+    /* If we have cancelled the timer, we need to start the unlock of the
+     * inode. If there was a timer but we have been unable to cancel it
+     * because it was just triggered, the timer callback will take care
+     * of releasing the inode. */
+    if (timer_link != NULL) {
+        ec_unlock_now(timer_link);
+    }
+}
+
+void
+ec_unlock_timer_add(ec_lock_link_t *link);
+
+void
+ec_unlock_timer_del(ec_lock_link_t *link)
+{
+    ec_lock_t *lock;
+    inode_t *inode;
+    gf_boolean_t now = _gf_false;
+
+    /* If we are here, it means that the timer has expired before having
+     * been cancelled. This guarantees that 'link' is still valid because
+     * the fop that contains it must be pending (if timer cancellation in
+     * ec_lock_assign_owner() fails, the fop is left sleeping).
+     *
+     * At the same time, the fop still has a reference to the lock, so
+     * it must also be valid.
+     */
+    lock = link->lock;
+
+    /* 'lock' must have a valid inode since it can only be destroyed
+     * when the lock itself is destroyed, but we have a reference to the
+     * lock to avoid this.
+     */
+    inode = lock->loc.inode;
+
+    LOCK(&inode->lock);
+
+    if (lock->timer != NULL) {
+        ec_trace("UNLOCK_DELAYED", link->fop, "lock=%p", lock);
+
+        /* The unlock timer has expired without anyone cancelling it.
+         * This means that it shouldn't have any owner, and the waiting
+         * and frozen lists should be empty.  It must have only one
+         * owner reference, but there can be fops being prepared
+         * though.
+         * */
+        GF_ASSERT(!lock->release && (lock->refs_owners == 1) &&
+                  list_empty(&lock->owners) && list_empty(&lock->waiting) &&
+                  list_empty(&lock->frozen));
+
+        gf_timer_call_cancel(link->fop->xl->ctx, lock->timer);
         lock->timer = NULL;
-        *lock->plock = NULL;
+
+        /* Any fop being processed from now on, will need to wait
+         * until the next unlock/lock cycle. */
+        lock->release = now = _gf_true;
     }
 
-    UNLOCK(&lock->loc.inode->lock);
+    UNLOCK(&inode->lock);
 
-    if (fop != NULL) {
-        ec_unlock_now(fop, lock);
+    if (now) {
+        ec_unlock_now(link);
+    } else {
+        /* The timer has been cancelled just after firing it but before
+         * getting here. This means that another fop has used the lock
+         * and everything should be handled as if this callback were
+         * have not been executed. However we still have an owner
+         * reference.
+         *
+         * We need to release our reference. If this is not the last
+         * reference (the most common case because another fop has
+         * taken another ref) we only need to decrement the counter.
+         * Otherwise we have been delayed enough so that the other fop
+         * has had time to acquire the reference, do its operation and
+         * release it. At the time of releasing it, the fop did found
+         * that the ref counter was > 1 (our reference), so the delayed
+         * unlock timer wasn't started. We need to start it again if we
+         * are the last reference.
+         *
+         * ec_unlock_timer_add() handles both cases.
+         */
+        ec_unlock_timer_add(link);
+
+        /* We need to resume the fop that was waiting for the delayed
+         * unlock.
+         */
+        ec_resume(link->fop, 0);
+    }
+}
+
+void
+ec_unlock_timer_cbk(void *data)
+{
+    ec_unlock_timer_del(data);
+}
+
+static gf_boolean_t
+ec_eager_lock_used(ec_t *ec, ec_fop_data_t *fop)
+{
+    /* Fops with no locks at this point mean that they are sent as sub-fops
+     * of other higher level fops. In this case we simply assume that the
+     * parent fop will take correct care of the eager lock. */
+    if (fop->lock_count == 0) {
+        return _gf_true;
+    }
+
+    /* We may have more than one lock, but this only happens in the rename
+     * fop, and both locks will reference an inode of the same type (a
+     * directory in this case), so we only need to check the first lock. */
+    if (fop->locks[0].lock->loc.inode->ia_type == IA_IFREG) {
+        return ec->eager_lock;
+    }
+
+    return ec->other_eager_lock;
+}
+
+static uint32_t
+ec_eager_lock_timeout(ec_t *ec, ec_lock_t *lock)
+{
+    if (lock->loc.inode->ia_type == IA_IFREG) {
+        return ec->eager_lock_timeout;
     }
+
+    return ec->other_eager_lock_timeout;
 }
 
-void ec_unlock_timer_add(ec_lock_link_t *link)
+static gf_boolean_t
+ec_lock_delay_create(ec_lock_link_t *link)
 {
     struct timespec delay;
     ec_fop_data_t *fop = link->fop;
     ec_lock_t *lock = link->lock;
-    int32_t refs = 1;
+
+    delay.tv_sec = ec_eager_lock_timeout(fop->xl->private, lock);
+    delay.tv_nsec = 0;
+    lock->timer = gf_timer_call_after(fop->xl->ctx, delay, ec_unlock_timer_cbk,
+                                      link);
+    if (lock->timer == NULL) {
+        gf_msg(fop->xl->name, GF_LOG_WARNING, ENOMEM,
+               EC_MSG_UNLOCK_DELAY_FAILED, "Unable to delay an unlock");
+
+        return _gf_false;
+    }
+
+    return _gf_true;
+}
+
+void
+ec_unlock_timer_add(ec_lock_link_t *link)
+{
+    ec_fop_data_t *fop = link->fop;
+    ec_lock_t *lock = link->lock;
+    gf_boolean_t now = _gf_false;
 
     LOCK(&lock->loc.inode->lock);
 
-    GF_ASSERT(lock->timer == NULL);
+    /* We are trying to unlock the lock. We can have multiple scenarios here,
+     * but all of them need to have lock->timer == NULL:
+     *
+     * 1. There are other owners currently running that can call ec_unlock().
+     *
+     *    None of them can have started the timer until the last one. But this
+     *    call should be the consequence of this lastest one.
+     *
+     * 2. There are fops in the waiting or frozen lists.
+     *
+     *    These fops cannot call ec_unlock(). So we should be here.
+     *
+     * We must reach here with at least one owner reference.
+     */
+    GF_ASSERT((lock->timer == NULL) && (lock->refs_owners > 0));
+
+    /* If the fop detects that a heal is needed, we mark the lock to be
+     * released as soon as possible. */
+    lock->release |= ec_fop_needs_heal(fop);
 
-    if (lock->refs != 1) {
+    if (lock->refs_owners > 1) {
         ec_trace("UNLOCK_SKIP", fop, "lock=%p", lock);
 
-        lock->refs--;
+        /* If there are other owners we cannot do anything else with the lock.
+         * Note that the current fop has already been removed from the owners
+         * list in ec_lock_reuse(). */
+        lock->refs_owners--;
 
         UNLOCK(&lock->loc.inode->lock);
     } else if (lock->acquired) {
-        delay.tv_sec = 1;
-        delay.tv_nsec = 0;
+        /* There are no other owners and the lock is acquired. If there were
+         * fops waiting, at least one of them should have been promoted to an
+         * owner, so the waiting list should be empty. */
+        GF_ASSERT(list_empty(&lock->owners) && list_empty(&lock->waiting));
 
+        ec_t *ec = fop->xl->private;
+
+        /* If everything goes as expected this fop will be put to sleep until
+         * the timer callback is executed. */
         ec_sleep(fop);
 
-        /* If healing is needed, do not delay lock release to let self-heal
-         * start working as soon as possible. */
-        if (!ec_fop_needs_heal(fop)) {
-            ec_trace("UNLOCK_DELAY", fop, "lock=%p", lock);
-
-            delay.tv_sec = 1;
-            delay.tv_nsec = 0;
-            lock->timer = gf_timer_call_after(fop->xl->ctx, delay,
-                                              ec_unlock_timer_cbk, link);
-            if (lock->timer == NULL) {
-                gf_log(fop->xl->name, GF_LOG_WARNING, "Unable to delay an "
-                                                      "unlock");
-
-                *lock->plock = NULL;
-                refs = 0;
+        /* If the lock needs to be released, or ec is shutting down, do not
+         * delay lock release. */
+        if (!lock->release && !ec->shutdown) {
+            ec_trace("UNLOCK_DELAY", fop, "lock=%p, release=%d", lock,
+                     lock->release);
+
+            if (!ec_lock_delay_create(link)) {
+                /* We are unable to create a new timer. We immediately release
+                 * the lock. */
+                lock->release = now = _gf_true;
             }
+
         } else {
-            ec_trace("UNLOCK_FORCE", fop, "lock=%p", lock);
-            *lock->plock = NULL;
-            refs = 0;
+            ec_trace("UNLOCK_FORCE", fop, "lock=%p, release=%d", lock,
+                     lock->release);
+            lock->release = now = _gf_true;
         }
 
         UNLOCK(&lock->loc.inode->lock);
 
-        if (refs == 0) {
-            ec_unlock_now(fop, lock);
+        if (now) {
+            ec_unlock_now(link);
         }
     } else {
-        *lock->plock = NULL;
+        /* There are no owners and the lock is not acquired. This can only
+         * happen if a lock attempt has failed and we get to the unlock step
+         * of the fop. As in the previous case, the waiting list must be
+         * empty. */
+        GF_ASSERT(list_empty(&lock->owners) && list_empty(&lock->waiting));
+
+        /* We need to mark the lock to be released to correctly handle fops
+         * that may get in after we release the inode mutex but before
+         * ec_lock_unfreeze() is processed. */
+        lock->release = _gf_true;
 
         UNLOCK(&lock->loc.inode->lock);
 
-        ec_lock_destroy(lock);
+        ec_lock_unfreeze(link);
     }
 }
 
-void ec_unlock(ec_fop_data_t *fop)
+void
+ec_unlock(ec_fop_data_t *fop)
 {
     int32_t i;
 
@@ -1531,103 +2839,155 @@ void ec_unlock(ec_fop_data_t *fop)
     }
 }
 
-void ec_flush_size_version(ec_fop_data_t * fop)
+void
+ec_flush_size_version(ec_fop_data_t *fop)
 {
-    ec_lock_t * lock;
-    uint64_t version, delta;
-
     GF_ASSERT(fop->lock_count == 1);
+    ec_update_info(&fop->locks[0]);
+}
 
-    lock = fop->locks[0].lock;
-
-    LOCK(&lock->loc.inode->lock);
-
-    GF_ASSERT(lock->owner == fop);
-
-    version = lock->version_delta;
-    delta = lock->size_delta;
-    lock->version_delta = 0;
-    lock->size_delta = 0;
-
-    UNLOCK(&lock->loc.inode->lock);
+static void
+ec_update_stripe(ec_t *ec, ec_stripe_list_t *stripe_cache, ec_stripe_t *stripe,
+                 ec_fop_data_t *fop)
+{
+    off_t base;
+
+    /* On write fops, we only update existing fragments if the write has
+     * succeeded. Otherwise, we remove them from the cache. */
+    if ((fop->id == GF_FOP_WRITE) && (fop->answer != NULL) &&
+        (fop->answer->op_ret >= 0)) {
+        base = stripe->frag_offset - fop->frag_range.first;
+        base *= ec->fragments;
+
+        /* We check if the stripe offset falls inside the real region
+         * modified by the write fop (a write request is allowed,
+         * though uncommon, to write less bytes than requested). The
+         * current write fop implementation doesn't allow partial
+         * writes of fragments, so if there's no error, we are sure
+         * that a full stripe has been completely modified or not
+         * touched at all. The value of op_ret may not be a multiple
+         * of the stripe size because it depends on the requested
+         * size by the user, so we update the stripe if the write has
+         * modified at least one byte (meaning ec has written the full
+         * stripe). */
+        if (base < fop->answer->op_ret + fop->head) {
+            memcpy(stripe->data, fop->vector[0].iov_base + base,
+                   ec->stripe_size);
+            list_move_tail(&stripe->lru, &stripe_cache->lru);
+
+            GF_ATOMIC_INC(ec->stats.stripe_cache.updates);
+        }
+    } else {
+        stripe->frag_offset = -1;
+        list_move(&stripe->lru, &stripe_cache->lru);
 
-    if (version > 0)
-    {
-        ec_update_size_version(fop, &lock->loc, version, delta, _gf_false,
-                               NULL);
+        GF_ATOMIC_INC(ec->stats.stripe_cache.invals);
     }
 }
 
-void ec_lock_reuse(ec_fop_data_t *fop)
+static void
+ec_update_cached_stripes(ec_fop_data_t *fop)
 {
-    ec_fop_data_t * wait_fop;
-    ec_lock_t * lock;
-    ec_lock_link_t * link;
-    int32_t i;
-
-    for (i = 0; i < fop->lock_count; i++)
-    {
-        wait_fop = NULL;
-
-        lock = fop->locks[i].lock;
-
-        LOCK(&lock->loc.inode->lock);
+    uint64_t first;
+    uint64_t last;
+    ec_stripe_t *stripe = NULL;
+    ec_inode_t *ctx = NULL;
+    ec_stripe_list_t *stripe_cache = NULL;
+    inode_t *inode = NULL;
+    struct list_head *temp;
+    struct list_head sentinel;
+
+    first = fop->frag_range.first;
+    /* 'last' represents the first stripe not touched by the operation */
+    last = fop->frag_range.last;
+
+    /* If there are no modified stripes, we don't need to do anything
+     * else. */
+    if (last <= first) {
+        return;
+    }
 
-        ec_trace("LOCK_DONE", fop, "lock=%p", lock);
+    if (!fop->use_fd) {
+        inode = fop->loc[0].inode;
+    } else {
+        inode = fop->fd->inode;
+    }
 
-        GF_ASSERT(lock->owner == fop);
-        lock->owner = NULL;
+    LOCK(&inode->lock);
 
-        if (((fop->locks_update >> i) & 1) != 0) {
-            if (fop->error == 0)
-            {
-                lock->version_delta++;
-                lock->size_delta += fop->post_size - fop->pre_size;
-                if (fop->have_size) {
-                    lock->size = fop->post_size;
-                    lock->have_size = 1;
-                }
-            }
+    ctx = __ec_inode_get(inode, fop->xl);
+    if (ctx == NULL) {
+        goto out;
+    }
+    stripe_cache = &ctx->stripe_cache;
+
+    /* Since we'll be moving elements of the list to the tail, we might
+     * end in an infinite loop. To avoid it, we insert a sentinel element
+     * into the list, so that it will be used to detect when we have
+     * traversed all existing elements once. */
+    list_add_tail(&sentinel, &stripe_cache->lru);
+    temp = stripe_cache->lru.next;
+    while (temp != &sentinel) {
+        stripe = list_entry(temp, ec_stripe_t, lru);
+        temp = temp->next;
+        if ((first <= stripe->frag_offset) && (stripe->frag_offset < last)) {
+            ec_update_stripe(fop->xl->private, stripe_cache, stripe, fop);
         }
+    }
+    list_del(&sentinel);
 
-        lock->good_mask &= fop->mask;
-
-        if (!list_empty(&lock->waiting))
-        {
-            link = list_entry(lock->waiting.next, ec_lock_link_t, wait_list);
-            list_del_init(&link->wait_list);
-
-            wait_fop = link->fop;
+out:
+    UNLOCK(&inode->lock);
+}
 
-            if (lock->kind == EC_LOCK_INODE)
-            {
-                wait_fop->pre_size = wait_fop->post_size = fop->post_size;
-                wait_fop->have_size = fop->have_size;
+void
+ec_lock_reuse(ec_fop_data_t *fop)
+{
+    ec_cbk_data_t *cbk;
+    ec_t *ec = NULL;
+    int32_t i, count;
+    gf_boolean_t release = _gf_false;
+    ec = fop->xl->private;
+    cbk = fop->answer;
+
+    if (ec_eager_lock_used(ec, fop) && cbk != NULL) {
+        if (cbk->xdata != NULL) {
+            if ((dict_get_int32(cbk->xdata, GLUSTERFS_INODELK_COUNT, &count) ==
+                 0) &&
+                (count > 1)) {
+                release = _gf_true;
+            }
+            if (release) {
+                gf_msg_debug(fop->xl->name, 0, "Lock contention detected");
             }
-            wait_fop->mask &= fop->mask;
         }
+    } else {
+        /* If eager lock is disabled or if we haven't get
+         * an answer with enough quorum, we always release
+         * the lock. */
+        release = _gf_true;
+    }
+    ec_update_cached_stripes(fop);
 
-        UNLOCK(&lock->loc.inode->lock);
-
-        if (wait_fop != NULL)
-        {
-            ec_lock(wait_fop);
-
-            ec_resume(wait_fop, 0);
-        }
+    for (i = 0; i < fop->lock_count; i++) {
+        ec_lock_next_owner(&fop->locks[i], cbk, release);
     }
 }
 
-void __ec_manager(ec_fop_data_t * fop, int32_t error)
+void
+__ec_manager(ec_fop_data_t *fop, int32_t error)
 {
     ec_t *ec = fop->xl->private;
 
     do {
         ec_trace("MANAGER", fop, "error=%d", error);
 
-        if (ec->xl_up_count < ec->fragments) {
-            error = ENOTCONN;
+        if (!ec_must_wind(fop)) {
+            if (ec->xl_up_count < ec->fragments) {
+                error = ENOTCONN;
+            }
         }
+
         if (error != 0) {
             fop->error = error;
             fop->state = -fop->state;
@@ -1639,22 +2999,44 @@ void __ec_manager(ec_fop_data_t * fop, int32_t error)
             break;
         }
 
+        /* At each state, fop must not be used anywhere else and there
+         * shouldn't be any pending subfop going on. */
+        GF_ASSERT(fop->jobs == 0);
+
+        /* While the manager is running we need to avoid that subfops launched
+         * from it could finish and call ec_resume() before the fop->handler
+         * has completed. This could lead to the same manager being executed
+         * by two threads concurrently. ec_check_complete() will take care of
+         * this reference. */
+        fop->jobs = 1;
+
         fop->state = fop->handler(fop, fop->state);
+        GF_ASSERT(fop->state >= 0);
 
         error = ec_check_complete(fop, __ec_manager);
     } while (error >= 0);
 }
 
-void ec_manager(ec_fop_data_t * fop, int32_t error)
+void
+ec_manager(ec_fop_data_t *fop, int32_t error)
 {
     GF_ASSERT(fop->jobs == 0);
     GF_ASSERT(fop->winds == 0);
     GF_ASSERT(fop->error == 0);
 
-    if (fop->state == EC_STATE_START)
-    {
+    if (fop->state == EC_STATE_START) {
         fop->state = EC_STATE_INIT;
     }
 
     __ec_manager(fop, error);
 }
+
+gf_boolean_t
+__ec_is_last_fop(ec_t *ec)
+{
+    if ((list_empty(&ec->pending_fops)) &&
+        (GF_ATOMIC_GET(ec->async_fop_count) == 0)) {
+        return _gf_true;
+    }
+    return _gf_false;
+}
diff --git a/xlators/cluster/ec/src/ec-common.h b/xlators/cluster/ec/src/ec-common.h
index aaae16e71c3..51493612ac6 100644
--- a/xlators/cluster/ec/src/ec-common.h
+++ b/xlators/cluster/ec/src/ec-common.h
@@ -11,93 +11,224 @@
 #ifndef __EC_COMMON_H__
 #define __EC_COMMON_H__
 
-#include "xlator.h"
-
+#include "glusterfs/compat-errno.h"  // for ENODATA on BSD
 #include "ec-data.h"
 
+typedef enum { EC_DATA_TXN, EC_METADATA_TXN } ec_txn_t;
+
+#define EC_FOP_HEAL -1
+#define EC_FOP_FHEAL -2
+
 #define EC_CONFIG_VERSION 0
 
 #define EC_CONFIG_ALGORITHM 0
 
-#define EC_FLAG_UPDATE_LOC_PARENT 0x0001
-#define EC_FLAG_UPDATE_LOC_INODE  0x0002
-#define EC_FLAG_UPDATE_FD         0x0004
-#define EC_FLAG_UPDATE_FD_INODE   0x0008
-
-#define EC_FLAG_WAITING_WINDS     0x0010
-
-#define EC_MINIMUM_ONE   -1
-#define EC_MINIMUM_MIN   -2
-#define EC_MINIMUM_ALL   -3
-
-#define EC_LOCK_ENTRY   0
-#define EC_LOCK_INODE   1
-
-#define EC_STATE_START                        0
-#define EC_STATE_END                          0
-#define EC_STATE_INIT                         1
-#define EC_STATE_LOCK                         2
-#define EC_STATE_GET_SIZE_AND_VERSION         3
-#define EC_STATE_DISPATCH                     4
-#define EC_STATE_PREPARE_ANSWER               5
-#define EC_STATE_REPORT                       6
-#define EC_STATE_LOCK_REUSE                   7
-#define EC_STATE_UNLOCK                       8
-
-#define EC_STATE_DELAYED_START              100
-
-#define EC_STATE_HEAL_ENTRY_LOOKUP          200
-#define EC_STATE_HEAL_ENTRY_PREPARE         201
-#define EC_STATE_HEAL_PRE_INODELK_LOCK      202
-#define EC_STATE_HEAL_PRE_INODE_LOOKUP      203
-#define EC_STATE_HEAL_XATTRIBUTES_REMOVE    204
-#define EC_STATE_HEAL_XATTRIBUTES_SET       205
-#define EC_STATE_HEAL_ATTRIBUTES            206
-#define EC_STATE_HEAL_OPEN                  207
-#define EC_STATE_HEAL_REOPEN_FD             208
-#define EC_STATE_HEAL_UNLOCK                209
-#define EC_STATE_HEAL_UNLOCK_ENTRY          210
-#define EC_STATE_HEAL_DATA_LOCK             211
-#define EC_STATE_HEAL_DATA_COPY             212
-#define EC_STATE_HEAL_DATA_UNLOCK           213
-#define EC_STATE_HEAL_POST_INODELK_LOCK     214
-#define EC_STATE_HEAL_POST_INODE_LOOKUP     215
-#define EC_STATE_HEAL_SETATTR               216
-#define EC_STATE_HEAL_POST_INODELK_UNLOCK   217
-#define EC_STATE_HEAL_DISPATCH              218
-
-int32_t ec_dispatch_one_retry(ec_fop_data_t * fop, int32_t idx, int32_t op_ret,
-                              int32_t op_errno);
-int32_t ec_dispatch_next(ec_fop_data_t * fop, int32_t idx);
-
-void ec_complete(ec_fop_data_t * fop);
-
-void ec_update_bad(ec_fop_data_t * fop, uintptr_t good);
-
-void ec_fop_set_error(ec_fop_data_t * fop, int32_t error);
-
-void ec_lock_prepare_inode(ec_fop_data_t *fop, loc_t *loc, int32_t update);
-void ec_lock_prepare_entry(ec_fop_data_t *fop, loc_t *loc, int32_t update);
-void ec_lock_prepare_fd(ec_fop_data_t *fop, fd_t *fd, int32_t update);
-void ec_lock(ec_fop_data_t * fop);
-void ec_lock_reuse(ec_fop_data_t *fop);
-void ec_unlock(ec_fop_data_t * fop);
-
-void ec_get_size_version(ec_fop_data_t * fop);
-void ec_prepare_update(ec_fop_data_t *fop);
-void ec_flush_size_version(ec_fop_data_t * fop);
-
-void ec_dispatch_all(ec_fop_data_t * fop);
-void ec_dispatch_inc(ec_fop_data_t * fop);
-void ec_dispatch_min(ec_fop_data_t * fop);
-void ec_dispatch_one(ec_fop_data_t * fop);
-
-void ec_wait_winds(ec_fop_data_t * fop);
-
-void ec_sleep(ec_fop_data_t *fop);
-void ec_resume(ec_fop_data_t * fop, int32_t error);
-void ec_resume_parent(ec_fop_data_t * fop, int32_t error);
-
-void ec_manager(ec_fop_data_t * fop, int32_t error);
-
+#define EC_FLAG_LOCK_SHARED 0x0001
+
+#define QUORUM_CBK(fn, fop, frame, cookie, this, op_ret, op_errno, params...)  \
+    do {                                                                       \
+        ec_t *__ec = fop->xl->private;                                         \
+        int32_t __op_ret = 0;                                                  \
+        int32_t __op_errno = 0;                                                \
+        int32_t __success_count = gf_bits_count(fop->good);                    \
+                                                                               \
+        __op_ret = op_ret;                                                     \
+        __op_errno = op_errno;                                                 \
+        if (!fop->parent && frame &&                                           \
+            (GF_CLIENT_PID_SELF_HEALD != frame->root->pid) &&                  \
+            __ec->quorum_count && (__success_count < __ec->quorum_count) &&    \
+            op_ret >= 0) {                                                     \
+            __op_ret = -1;                                                     \
+            __op_errno = EIO;                                                  \
+            gf_msg(__ec->xl->name, GF_LOG_ERROR, 0,                            \
+                   EC_MSG_CHILDS_INSUFFICIENT,                                 \
+                   "Insufficient available children for this request "         \
+                   "(have %d, need %d). %s",                                   \
+                   __success_count, __ec->quorum_count, ec_msg_str(fop));      \
+        }                                                                      \
+        fn(frame, cookie, this, __op_ret, __op_errno, params);                 \
+    } while (0)
+
+enum _ec_xattrop_flags {
+    EC_FLAG_XATTROP,
+    EC_FLAG_DATA_DIRTY,
+    EC_FLAG_METADATA_DIRTY,
+
+    /* Add any new flag here, before EC_FLAG_MAX. The maximum number of
+     * flags that can be defined is 16. */
+
+    EC_FLAG_MAX
+};
+
+/* We keep two sets of flags. One to determine what's really providing the
+ * current xattrop and the other to know what the parent fop of the xattrop
+ * needs to proceed. It might happen that a fop needs some information that
+ * is being already requested by a previous fop. The two sets are stored
+ * contiguously. */
+
+#define EC_FLAG_NEEDS(_flag) (1 << (_flag))
+#define EC_FLAG_PROVIDES(_flag) (1 << ((_flag) + EC_FLAG_MAX))
+
+#define EC_NEEDED_FLAGS(_flags) ((_flags) & ((1 << EC_FLAG_MAX) - 1))
+
+#define EC_PROVIDED_FLAGS(_flags) EC_NEEDED_FLAGS((_flags) >> EC_FLAG_MAX)
+
+#define EC_FLAGS_HAVE(_flags, _flag) (((_flags) & (1 << (_flag))) != 0)
+
+#define EC_SELFHEAL_BIT 62
+
+#define EC_MINIMUM_ONE (1 << 6)
+#define EC_MINIMUM_MIN (2 << 6)
+#define EC_MINIMUM_ALL (3 << 6)
+#define EC_FOP_NO_PROPAGATE_ERROR (1 << 8)
+#define EC_FOP_MINIMUM(_flags) ((_flags)&255)
+#define EC_FOP_FLAGS(_flags) ((_flags) & ~255)
+
+#define EC_UPDATE_DATA 1
+#define EC_UPDATE_META 2
+#define EC_QUERY_INFO 4
+#define EC_INODE_SIZE 8
+
+#define EC_STATE_START 0
+#define EC_STATE_END 0
+#define EC_STATE_INIT 1
+#define EC_STATE_LOCK 2
+#define EC_STATE_DISPATCH 3
+#define EC_STATE_PREPARE_ANSWER 4
+#define EC_STATE_REPORT 5
+#define EC_STATE_LOCK_REUSE 6
+#define EC_STATE_UNLOCK 7
+
+#define EC_STATE_DELAYED_START 100
+
+#define EC_STATE_HEAL_ENTRY_LOOKUP 200
+#define EC_STATE_HEAL_ENTRY_PREPARE 201
+#define EC_STATE_HEAL_PRE_INODELK_LOCK 202
+#define EC_STATE_HEAL_PRE_INODE_LOOKUP 203
+#define EC_STATE_HEAL_XATTRIBUTES_REMOVE 204
+#define EC_STATE_HEAL_XATTRIBUTES_SET 205
+#define EC_STATE_HEAL_ATTRIBUTES 206
+#define EC_STATE_HEAL_OPEN 207
+#define EC_STATE_HEAL_REOPEN_FD 208
+#define EC_STATE_HEAL_UNLOCK 209
+#define EC_STATE_HEAL_UNLOCK_ENTRY 210
+#define EC_STATE_HEAL_DATA_LOCK 211
+#define EC_STATE_HEAL_DATA_COPY 212
+#define EC_STATE_HEAL_DATA_UNLOCK 213
+#define EC_STATE_HEAL_POST_INODELK_LOCK 214
+#define EC_STATE_HEAL_POST_INODE_LOOKUP 215
+#define EC_STATE_HEAL_SETATTR 216
+#define EC_STATE_HEAL_POST_INODELK_UNLOCK 217
+#define EC_STATE_HEAL_DISPATCH 218
+
+/* Value to cover the full range of a file */
+#define EC_RANGE_FULL ((uint64_t)LLONG_MAX + 1)
+
+gf_boolean_t
+ec_dispatch_one_retry(ec_fop_data_t *fop, ec_cbk_data_t **cbk);
+void
+ec_dispatch_next(ec_fop_data_t *fop, uint32_t idx);
+
+void
+ec_complete(ec_fop_data_t *fop);
+
+void
+ec_update_good(ec_fop_data_t *fop, uintptr_t good);
+
+void
+ec_fop_set_error(ec_fop_data_t *fop, int32_t error);
+
+void
+__ec_fop_set_error(ec_fop_data_t *fop, int32_t error);
+
+ec_cbk_data_t *
+ec_fop_prepare_answer(ec_fop_data_t *fop, gf_boolean_t ro);
+
+gf_boolean_t
+ec_cbk_set_error(ec_cbk_data_t *cbk, int32_t error, gf_boolean_t ro);
+
+void
+ec_lock_prepare_inode(ec_fop_data_t *fop, loc_t *loc, uint32_t flags,
+                      off_t fl_start, uint64_t fl_size);
+void
+ec_lock_prepare_parent_inode(ec_fop_data_t *fop, loc_t *loc, loc_t *base,
+                             uint32_t flags);
+void
+ec_lock_prepare_fd(ec_fop_data_t *fop, fd_t *fd, uint32_t flags, off_t fl_start,
+                   uint64_t fl_size);
+void
+ec_lock(ec_fop_data_t *fop);
+void
+ec_lock_reuse(ec_fop_data_t *fop);
+void
+ec_unlock(ec_fop_data_t *fop);
+void
+ec_lock_release(ec_t *ec, inode_t *inode);
+
+gf_boolean_t
+ec_get_inode_size(ec_fop_data_t *fop, inode_t *inode, uint64_t *size);
+gf_boolean_t
+__ec_get_inode_size(ec_fop_data_t *fop, inode_t *inode, uint64_t *size);
+gf_boolean_t
+ec_set_inode_size(ec_fop_data_t *fop, inode_t *inode, uint64_t size);
+gf_boolean_t
+__ec_set_inode_size(ec_fop_data_t *fop, inode_t *inode, uint64_t size);
+void
+ec_clear_inode_info(ec_fop_data_t *fop, inode_t *inode);
+
+void
+ec_flush_size_version(ec_fop_data_t *fop);
+
+void
+ec_dispatch_all(ec_fop_data_t *fop);
+void
+ec_dispatch_inc(ec_fop_data_t *fop);
+void
+ec_dispatch_min(ec_fop_data_t *fop);
+void
+ec_dispatch_one(ec_fop_data_t *fop);
+
+void
+ec_succeed_all(ec_fop_data_t *fop);
+
+void
+ec_sleep(ec_fop_data_t *fop);
+void
+ec_resume(ec_fop_data_t *fop, int32_t error);
+void
+ec_resume_parent(ec_fop_data_t *fop);
+
+void
+ec_manager(ec_fop_data_t *fop, int32_t error);
+gf_boolean_t
+ec_is_recoverable_error(int32_t op_errno);
+void
+ec_handle_healers_done(ec_fop_data_t *fop);
+
+int32_t
+ec_heal_inspect(call_frame_t *frame, ec_t *ec, inode_t *inode,
+                unsigned char *locked_on, gf_boolean_t self_locked,
+                gf_boolean_t thorough, ec_heal_need_t *need_heal);
+int32_t
+ec_get_heal_info(xlator_t *this, loc_t *loc, dict_t **dict);
+
+int32_t
+ec_lock_unlocked(call_frame_t *frame, void *cookie, xlator_t *this,
+                 int32_t op_ret, int32_t op_errno, dict_t *xdata);
+
+void
+ec_update_fd_status(fd_t *fd, xlator_t *xl, int child_index,
+                    int32_t ret_status);
+gf_boolean_t
+ec_is_entry_healing(ec_fop_data_t *fop);
+void
+ec_set_entry_healing(ec_fop_data_t *fop);
+void
+ec_reset_entry_healing(ec_fop_data_t *fop);
+char *
+ec_msg_str(ec_fop_data_t *fop);
+gf_boolean_t
+__ec_is_last_fop(ec_t *ec);
+void
+ec_lock_update_good(ec_lock_t *lock, ec_fop_data_t *fop);
 #endif /* __EC_COMMON_H__ */
diff --git a/xlators/cluster/ec/src/ec-data.c b/xlators/cluster/ec/src/ec-data.c
index fb47aea90a8..06388833546 100644
--- a/xlators/cluster/ec/src/ec-data.c
+++ b/xlators/cluster/ec/src/ec-data.c
@@ -8,49 +8,49 @@
   cases as published by the Free Software Foundation.
 */
 
-#include "ec-mem-types.h"
 #include "ec-helpers.h"
 #include "ec-common.h"
 #include "ec-data.h"
+#include "ec-messages.h"
 
-ec_cbk_data_t * ec_cbk_data_allocate(call_frame_t * frame, xlator_t * this,
-                                     ec_fop_data_t * fop, int32_t id,
-                                     int32_t idx, int32_t op_ret,
-                                     int32_t op_errno)
+ec_cbk_data_t *
+ec_cbk_data_allocate(call_frame_t *frame, xlator_t *this, ec_fop_data_t *fop,
+                     int32_t id, int32_t idx, int32_t op_ret, int32_t op_errno)
 {
-    ec_cbk_data_t * cbk;
-    ec_t * ec = this->private;
+    ec_cbk_data_t *cbk;
+    ec_t *ec = this->private;
 
-    if (fop->xl != this)
-    {
-        gf_log(this->name, GF_LOG_ERROR, "Mismatching xlators between request "
-                                         "and answer (req=%s, ans=%s).",
-                                         fop->xl->name, this->name);
+    if (fop->xl != this) {
+        gf_msg(this->name, GF_LOG_ERROR, EINVAL, EC_MSG_XLATOR_MISMATCH,
+               "Mismatching xlators between request "
+               "and answer (req=%s, ans=%s).",
+               fop->xl->name, this->name);
 
         return NULL;
     }
-    if (fop->frame != frame)
-    {
-        gf_log(this->name, GF_LOG_ERROR, "Mismatching frames between request "
-                                         "and answer (req=%p, ans=%p).",
-                                         fop->frame, frame);
+    if (fop->frame != frame) {
+        gf_msg(this->name, GF_LOG_ERROR, EINVAL, EC_MSG_FRAME_MISMATCH,
+               "Mismatching frames between request "
+               "and answer (req=%p, ans=%p).",
+               fop->frame, frame);
 
         return NULL;
     }
-    if (fop->id != id)
-    {
-        gf_log(this->name, GF_LOG_ERROR, "Mismatching fops between request "
-                                         "and answer (req=%d, ans=%d).",
-                                         fop->id, id);
+    if (fop->id != id) {
+        gf_msg(this->name, GF_LOG_ERROR, EINVAL, EC_MSG_FOP_MISMATCH,
+               "Mismatching fops between request "
+               "and answer (req=%d, ans=%d).",
+               fop->id, id);
 
         return NULL;
     }
 
     cbk = mem_get0(ec->cbk_pool);
-    if (cbk == NULL)
-    {
-        gf_log(this->name, GF_LOG_ERROR, "Failed to allocate memory for an "
-                                         "answer.");
+    if (cbk == NULL) {
+        gf_msg(this->name, GF_LOG_ERROR, ENOMEM, EC_MSG_NO_MEMORY,
+               "Failed to allocate memory for an "
+               "answer.");
+        return NULL;
     }
 
     cbk->fop = fop;
@@ -59,6 +59,7 @@ ec_cbk_data_t * ec_cbk_data_allocate(call_frame_t * frame, xlator_t * this,
     cbk->count = 1;
     cbk->op_ret = op_ret;
     cbk->op_errno = op_errno;
+    INIT_LIST_HEAD(&cbk->entries.list);
 
     LOCK(&fop->lock);
 
@@ -69,51 +70,58 @@ ec_cbk_data_t * ec_cbk_data_allocate(call_frame_t * frame, xlator_t * this,
     return cbk;
 }
 
-void ec_cbk_data_destroy(ec_cbk_data_t * cbk)
+void
+ec_cbk_data_destroy(ec_cbk_data_t *cbk)
 {
-    if (cbk->xdata != NULL)
-    {
+    if (cbk->xdata != NULL) {
         dict_unref(cbk->xdata);
     }
-    if (cbk->dict != NULL)
-    {
+    if (cbk->dict != NULL) {
         dict_unref(cbk->dict);
     }
-    if (cbk->inode != NULL)
-    {
+    if (cbk->inode != NULL) {
         inode_unref(cbk->inode);
     }
-    if (cbk->fd != NULL)
-    {
+    if (cbk->fd != NULL) {
         fd_unref(cbk->fd);
     }
-    if (cbk->buffers != NULL)
-    {
+    if (cbk->buffers != NULL) {
         iobref_unref(cbk->buffers);
     }
     GF_FREE(cbk->vector);
+    gf_dirent_free(&cbk->entries);
+    GF_FREE(cbk->str);
 
     mem_put(cbk);
 }
 
-ec_fop_data_t * ec_fop_data_allocate(call_frame_t * frame, xlator_t * this,
-                                     int32_t id, uint32_t flags,
-                                     uintptr_t target, int32_t minimum,
-                                     ec_wind_f wind, ec_handler_f handler,
-                                     ec_cbk_t cbks, void * data)
+ec_fop_data_t *
+ec_fop_data_allocate(call_frame_t *frame, xlator_t *this, int32_t id,
+                     uint32_t flags, uintptr_t target, uint32_t fop_flags,
+                     ec_wind_f wind, ec_handler_f handler, ec_cbk_t cbks,
+                     void *data)
 {
-    ec_fop_data_t * fop, * parent;
-    ec_t * ec = this->private;
+    ec_fop_data_t *fop, *parent;
+    ec_t *ec = this->private;
 
     fop = mem_get0(ec->fop_pool);
-    if (fop == NULL)
-    {
-        gf_log(this->name, GF_LOG_ERROR, "Failed to allocate memory for a "
-                                         "request.");
+    if (fop == NULL) {
+        gf_msg(this->name, GF_LOG_ERROR, ENOMEM, EC_MSG_NO_MEMORY,
+               "Failed to allocate memory for a "
+               "request.");
 
         return NULL;
     }
 
+    INIT_LIST_HEAD(&fop->cbk_list);
+    INIT_LIST_HEAD(&fop->healer);
+    INIT_LIST_HEAD(&fop->answer_list);
+    INIT_LIST_HEAD(&fop->pending_list);
+    INIT_LIST_HEAD(&fop->locks[0].owner_list);
+    INIT_LIST_HEAD(&fop->locks[0].wait_list);
+    INIT_LIST_HEAD(&fop->locks[1].owner_list);
+    INIT_LIST_HEAD(&fop->locks[1].wait_list);
+
     fop->xl = this;
     fop->req_frame = frame;
 
@@ -124,18 +132,15 @@ ec_fop_data_t * ec_fop_data_allocate(call_frame_t * frame, xlator_t * this,
      * TODO: minimize usage of private frames. Reuse req_frame as much as
      *       possible.
      */
-    if (frame != NULL)
-    {
+    if (frame != NULL) {
         fop->frame = copy_frame(frame);
-    }
-    else
-    {
+    } else {
         fop->frame = create_frame(this, this->ctx->pool);
     }
-    if (fop->frame == NULL)
-    {
-        gf_log(this->name, GF_LOG_ERROR, "Failed to create a private frame "
-                                         "for a request");
+    if (fop->frame == NULL) {
+        gf_msg(this->name, GF_LOG_ERROR, ENOMEM, EC_MSG_NO_MEMORY,
+               "Failed to create a private frame "
+               "for a request");
 
         mem_put(fop);
 
@@ -145,41 +150,42 @@ ec_fop_data_t * ec_fop_data_allocate(call_frame_t * frame, xlator_t * this,
     fop->refs = 1;
 
     fop->flags = flags;
-    fop->minimum = minimum;
+    fop->minimum = EC_FOP_MINIMUM(fop_flags);
+    fop->fop_flags = EC_FOP_FLAGS(fop_flags);
     fop->mask = target;
 
-    INIT_LIST_HEAD(&fop->cbk_list);
-    INIT_LIST_HEAD(&fop->answer_list);
-
     fop->wind = wind;
     fop->handler = handler;
     fop->cbks = cbks;
     fop->data = data;
 
+    fop->uid = fop->frame->root->uid;
+    fop->gid = fop->frame->root->gid;
+
     LOCK_INIT(&fop->lock);
 
     fop->frame->local = fop;
 
-    if (frame != NULL)
-    {
+    if (frame != NULL) {
         parent = frame->local;
-        if (parent != NULL)
-        {
-            LOCK(&parent->lock);
-
-            parent->jobs++;
-            parent->refs++;
-
-            UNLOCK(&parent->lock);
+        if (parent != NULL) {
+            ec_sleep(parent);
         }
 
         fop->parent = parent;
     }
 
+    LOCK(&ec->lock);
+
+    list_add_tail(&fop->pending_list, &ec->pending_fops);
+
+    UNLOCK(&ec->lock);
+
     return fop;
 }
 
-void ec_fop_data_acquire(ec_fop_data_t * fop)
+void
+ec_fop_data_acquire(ec_fop_data_t *fop)
 {
     LOCK(&fop->lock);
 
@@ -190,44 +196,74 @@ void ec_fop_data_acquire(ec_fop_data_t * fop)
     UNLOCK(&fop->lock);
 }
 
-void ec_fop_data_release(ec_fop_data_t * fop)
+static void
+ec_handle_last_pending_fop_completion(ec_fop_data_t *fop, gf_boolean_t *notify)
+{
+    ec_t *ec = fop->xl->private;
+
+    *notify = _gf_false;
+
+    if (!list_empty(&fop->pending_list)) {
+        LOCK(&ec->lock);
+        {
+            list_del_init(&fop->pending_list);
+            *notify = __ec_is_last_fop(ec);
+        }
+        UNLOCK(&ec->lock);
+    }
+}
+
+void
+ec_fop_cleanup(ec_fop_data_t *fop)
+{
+    ec_cbk_data_t *cbk, *tmp;
+
+    list_for_each_entry_safe(cbk, tmp, &fop->answer_list, answer_list)
+    {
+        list_del_init(&cbk->answer_list);
+
+        ec_cbk_data_destroy(cbk);
+    }
+    INIT_LIST_HEAD(&fop->cbk_list);
+
+    fop->answer = NULL;
+}
+
+void
+ec_fop_data_release(ec_fop_data_t *fop)
 {
-    ec_cbk_data_t * cbk, * tmp;
+    ec_t *ec = NULL;
     int32_t refs;
+    gf_boolean_t notify = _gf_false;
 
     LOCK(&fop->lock);
 
     ec_trace("RELEASE", fop, "");
 
+    GF_ASSERT(fop->refs > 0);
     refs = --fop->refs;
 
     UNLOCK(&fop->lock);
 
-    if (refs == 0)
-    {
+    if (refs == 0) {
         fop->frame->local = NULL;
         STACK_DESTROY(fop->frame->root);
 
         LOCK_DESTROY(&fop->lock);
 
-        if (fop->xdata != NULL)
-        {
+        if (fop->xdata != NULL) {
             dict_unref(fop->xdata);
         }
-        if (fop->dict != NULL)
-        {
+        if (fop->dict != NULL) {
             dict_unref(fop->dict);
         }
-        if (fop->inode != NULL)
-        {
+        if (fop->inode != NULL) {
             inode_unref(fop->inode);
         }
-        if (fop->fd != NULL)
-        {
+        if (fop->fd != NULL) {
             fd_unref(fop->fd);
         }
-        if (fop->buffers != NULL)
-        {
+        if (fop->buffers != NULL) {
             iobref_unref(fop->buffers);
         }
         GF_FREE(fop->vector);
@@ -235,16 +271,18 @@ void ec_fop_data_release(ec_fop_data_t * fop)
         GF_FREE(fop->str[1]);
         loc_wipe(&fop->loc[0]);
         loc_wipe(&fop->loc[1]);
+        GF_FREE(fop->errstr);
 
-        ec_resume_parent(fop, fop->error);
+        ec_resume_parent(fop);
 
-        list_for_each_entry_safe(cbk, tmp, &fop->answer_list, answer_list)
-        {
-            list_del_init(&cbk->answer_list);
-
-            ec_cbk_data_destroy(cbk);
-        }
+        ec_fop_cleanup(fop);
 
+        ec = fop->xl->private;
+        ec_handle_last_pending_fop_completion(fop, &notify);
+        ec_handle_healers_done(fop);
         mem_put(fop);
+        if (notify) {
+            ec_pending_fops_completed(ec);
+        }
     }
 }
diff --git a/xlators/cluster/ec/src/ec-data.h b/xlators/cluster/ec/src/ec-data.h
index ac59a6b2e14..c8a74ffe1ed 100644
--- a/xlators/cluster/ec/src/ec-data.h
+++ b/xlators/cluster/ec/src/ec-data.h
@@ -11,288 +11,25 @@
 #ifndef __EC_DATA_H__
 #define __EC_DATA_H__
 
-#include "xlator.h"
-
-#include "ec.h"
-
-struct _ec_config;
-typedef struct _ec_config ec_config_t;
-
-struct _ec_fd;
-typedef struct _ec_fd ec_fd_t;
-
-struct _ec_inode;
-typedef struct _ec_inode ec_inode_t;
-
-union _ec_cbk;
-typedef union _ec_cbk ec_cbk_t;
-
-struct _ec_lock;
-typedef struct _ec_lock ec_lock_t;
-
-struct _ec_lock_link;
-typedef struct _ec_lock_link ec_lock_link_t;
-
-struct _ec_fop_data;
-typedef struct _ec_fop_data ec_fop_data_t;
-
-struct _ec_cbk_data;
-typedef struct _ec_cbk_data ec_cbk_data_t;
-
-struct _ec_heal;
-typedef struct _ec_heal ec_heal_t;
-
-typedef void (* ec_wind_f)(ec_t *, ec_fop_data_t *, int32_t);
-typedef int32_t (* ec_handler_f)(ec_fop_data_t *, int32_t);
-typedef void (* ec_resume_f)(ec_fop_data_t *, int32_t);
-
-struct _ec_config
-{
-    uint32_t version;
-    uint8_t  algorithm;
-    uint8_t  gf_word_size;
-    uint8_t  bricks;
-    uint8_t  redundancy;
-    uint32_t chunk_size;
-};
-
-struct _ec_fd
-{
-    uintptr_t bad;
-    loc_t     loc;
-    uintptr_t open;
-    int32_t   flags;
-};
-
-struct _ec_inode
-{
-    uintptr_t         bad;
-    ec_lock_t        *entry_lock;
-    ec_lock_t        *inode_lock;
-    struct list_head  heal;
-
-};
-
-typedef int32_t (* fop_heal_cbk_t)(call_frame_t *, void * cookie, xlator_t *,
-                                   int32_t, int32_t, uintptr_t, uintptr_t,
-                                   uintptr_t, dict_t *);
-typedef int32_t (* fop_fheal_cbk_t)(call_frame_t *, void * cookie, xlator_t *,
-                                    int32_t, int32_t, uintptr_t, uintptr_t,
-                                    uintptr_t, dict_t *);
-
-
-union _ec_cbk
-{
-    fop_access_cbk_t       access;
-    fop_create_cbk_t       create;
-    fop_discard_cbk_t      discard;
-    fop_entrylk_cbk_t      entrylk;
-    fop_fentrylk_cbk_t     fentrylk;
-    fop_fallocate_cbk_t    fallocate;
-    fop_flush_cbk_t        flush;
-    fop_fsync_cbk_t        fsync;
-    fop_fsyncdir_cbk_t     fsyncdir;
-    fop_getxattr_cbk_t     getxattr;
-    fop_fgetxattr_cbk_t    fgetxattr;
-    fop_heal_cbk_t         heal;
-    fop_fheal_cbk_t        fheal;
-    fop_inodelk_cbk_t      inodelk;
-    fop_finodelk_cbk_t     finodelk;
-    fop_link_cbk_t         link;
-    fop_lk_cbk_t           lk;
-    fop_lookup_cbk_t       lookup;
-    fop_mkdir_cbk_t        mkdir;
-    fop_mknod_cbk_t        mknod;
-    fop_open_cbk_t         open;
-    fop_opendir_cbk_t      opendir;
-    fop_readdir_cbk_t      readdir;
-    fop_readdirp_cbk_t     readdirp;
-    fop_readlink_cbk_t     readlink;
-    fop_readv_cbk_t        readv;
-    fop_removexattr_cbk_t  removexattr;
-    fop_fremovexattr_cbk_t fremovexattr;
-    fop_rename_cbk_t       rename;
-    fop_rmdir_cbk_t        rmdir;
-    fop_setattr_cbk_t      setattr;
-    fop_fsetattr_cbk_t     fsetattr;
-    fop_setxattr_cbk_t     setxattr;
-    fop_fsetxattr_cbk_t    fsetxattr;
-    fop_stat_cbk_t         stat;
-    fop_fstat_cbk_t        fstat;
-    fop_statfs_cbk_t       statfs;
-    fop_symlink_cbk_t      symlink;
-    fop_truncate_cbk_t     truncate;
-    fop_ftruncate_cbk_t    ftruncate;
-    fop_unlink_cbk_t       unlink;
-    fop_writev_cbk_t       writev;
-    fop_xattrop_cbk_t      xattrop;
-    fop_fxattrop_cbk_t     fxattrop;
-    fop_zerofill_cbk_t     zerofill;
-};
-
-struct _ec_lock
-{
-    ec_lock_t        **plock;
-    gf_timer_t        *timer;
-    struct list_head   waiting;
-    uintptr_t          mask;
-    uintptr_t          good_mask;
-    int32_t            kind;
-    int32_t            refs;
-    int32_t            acquired;
-    int32_t            have_size;
-    uint64_t           size;
-    uint64_t           size_delta;
-    uint64_t           version;
-    uint64_t           version_delta;
-    ec_fop_data_t     *owner;
-    loc_t              loc;
-    union
-    {
-        entrylk_type     type;
-        struct gf_flock  flock;
-    };
-    gf_boolean_t       is_dirty;
-};
-
-struct _ec_lock_link
-{
-    ec_lock_t *      lock;
-    ec_fop_data_t *  fop;
-    struct list_head wait_list;
-};
-
-struct _ec_fop_data
-{
-    int32_t            id;
-    int32_t            refs;
-    int32_t            state;
-    int32_t            minimum;
-    int32_t            expected;
-    int32_t            winds;
-    int32_t            jobs;
-    int32_t            error;
-    ec_fop_data_t *    parent;
-    xlator_t *         xl;
-    call_frame_t *     req_frame;   // frame of the calling xlator
-    call_frame_t *     frame;       // frame used by this fop
-    struct list_head   cbk_list;    // sorted list of groups of answers
-    struct list_head   answer_list; // list of answers
-    ec_cbk_data_t *    answer;      // accepted answer
-    int32_t            lock_count;
-    int32_t            locked;
-    ec_lock_link_t     locks[2];
-    int32_t            locks_update;
-    int32_t            have_size;
-    uint64_t           pre_size;
-    uint64_t           post_size;
-    gf_lock_t          lock;
-    ec_config_t        config;
-
-    uint32_t           flags;
-    uint32_t           first;
-    uintptr_t          mask;
-    uintptr_t          remaining;
-    uintptr_t          good;
-    uintptr_t          bad;
-
-    ec_wind_f          wind;
-    ec_handler_f       handler;
-    ec_resume_f        resume;
-    ec_cbk_t           cbks;
-    void *             data;
-    ec_heal_t         *heal;
-
-    uint64_t           user_size;
-    uint32_t           head;
-
-    int32_t            use_fd;
-
-    dict_t *           xdata;
-    dict_t *           dict;
-    int32_t            int32;
-    uint32_t           uint32;
-    uint64_t           size;
-    off_t              offset;
-    mode_t             mode[2];
-    entrylk_cmd        entrylk_cmd;
-    entrylk_type       entrylk_type;
-    gf_xattrop_flags_t xattrop_flags;
-    dev_t              dev;
-    inode_t *          inode;
-    fd_t *             fd;
-    struct iatt        iatt;
-    char *             str[2];
-    loc_t              loc[2];
-    struct gf_flock    flock;
-    struct iovec *     vector;
-    struct iobref *    buffers;
-};
-
-struct _ec_cbk_data
-{
-    struct list_head list;        // item in the sorted list of groups
-    struct list_head answer_list; // item in the list of answers
-    ec_fop_data_t *  fop;
-    ec_cbk_data_t *  next;        // next answer in the same group
-    int32_t          idx;
-    int32_t          op_ret;
-    int32_t          op_errno;
-    int32_t          count;
-    uintptr_t        mask;
-
-    dict_t *         xdata;
-    dict_t *         dict;
-    int32_t          int32;
-    uintptr_t        uintptr[3];
-    uint64_t         size;
-    uint64_t         version;
-    inode_t *        inode;
-    fd_t *           fd;
-    struct statvfs   statvfs;
-    struct iatt      iatt[5];
-    struct gf_flock  flock;
-    struct iovec *   vector;
-    struct iobref *  buffers;
-    gf_boolean_t     dirty;
-};
-
-struct _ec_heal
-{
-    struct list_head  list;
-    gf_lock_t         lock;
-    xlator_t         *xl;
-    ec_fop_data_t    *fop;
-    void             *data;
-    ec_fop_data_t    *lookup;
-    loc_t             loc;
-    struct iatt       iatt;
-    char             *symlink;
-    fd_t             *fd;
-    int32_t           partial;
-    int32_t           done;
-    gf_boolean_t      nameheal;
-    uintptr_t         available;
-    uintptr_t         good;
-    uintptr_t         bad;
-    uintptr_t         open;
-    uintptr_t         fixed;
-    uint64_t          offset;
-    uint64_t          size;
-    uint64_t          version;
-    uint64_t          raw_size;
-};
-
-ec_cbk_data_t * ec_cbk_data_allocate(call_frame_t * frame, xlator_t * this,
-                                     ec_fop_data_t * fop, int32_t id,
-                                     int32_t idx, int32_t op_ret,
-                                     int32_t op_errno);
-ec_fop_data_t * ec_fop_data_allocate(call_frame_t * frame, xlator_t * this,
-                                     int32_t id, uint32_t flags,
-                                     uintptr_t target, int32_t minimum,
-                                     ec_wind_f wind, ec_handler_f handler,
-                                     ec_cbk_t cbks, void * data);
-void ec_fop_data_acquire(ec_fop_data_t * fop);
-void ec_fop_data_release(ec_fop_data_t * fop);
+#include "ec-types.h"
+
+ec_cbk_data_t *
+ec_cbk_data_allocate(call_frame_t *frame, xlator_t *this, ec_fop_data_t *fop,
+                     int32_t id, int32_t idx, int32_t op_ret, int32_t op_errno);
+ec_fop_data_t *
+ec_fop_data_allocate(call_frame_t *frame, xlator_t *this, int32_t id,
+                     uint32_t flags, uintptr_t target, uint32_t fop_flags,
+                     ec_wind_f wind, ec_handler_f handler, ec_cbk_t cbks,
+                     void *data);
+void
+ec_fop_data_acquire(ec_fop_data_t *fop);
+void
+ec_fop_data_release(ec_fop_data_t *fop);
+
+void
+ec_fop_cleanup(ec_fop_data_t *fop);
+
+void
+ec_pending_fops_completed(ec_t *ec);
 
 #endif /* __EC_DATA_H__ */
diff --git a/xlators/cluster/ec/src/ec-dir-read.c b/xlators/cluster/ec/src/ec-dir-read.c
index 17e1a3d124e..f71dcfac293 100644
--- a/xlators/cluster/ec/src/ec-dir-read.c
+++ b/xlators/cluster/ec/src/ec-dir-read.c
@@ -8,24 +8,26 @@
   cases as published by the Free Software Foundation.
 */
 
-#include "xlator.h"
-#include "defaults.h"
-
+#include "ec.h"
+#include "ec-messages.h"
 #include "ec-helpers.h"
 #include "ec-common.h"
 #include "ec-combine.h"
-#include "ec-method.h"
 #include "ec-fops.h"
 
-/* FOP: opendir */
+/****************************************************************
+ *
+ * File Operation: opendir
+ *
+ ***************************************************************/
 
-int32_t ec_combine_opendir(ec_fop_data_t * fop, ec_cbk_data_t * dst,
-                           ec_cbk_data_t * src)
+int32_t
+ec_combine_opendir(ec_fop_data_t *fop, ec_cbk_data_t *dst, ec_cbk_data_t *src)
 {
-    if (dst->fd != src->fd)
-    {
-        gf_log(fop->xl->name, GF_LOG_NOTICE, "Mismatching fd in answers "
-                                             "of 'GF_FOP_OPENDIR': %p <-> %p",
+    if (dst->fd != src->fd) {
+        gf_msg(fop->xl->name, GF_LOG_NOTICE, 0, EC_MSG_FD_MISMATCH,
+               "Mismatching fd in answers "
+               "of 'GF_FOP_OPENDIR': %p <-> %p",
                dst->fd, src->fd);
 
         return 0;
@@ -34,12 +36,12 @@ int32_t ec_combine_opendir(ec_fop_data_t * fop, ec_cbk_data_t * dst,
     return 1;
 }
 
-int32_t ec_opendir_cbk(call_frame_t * frame, void * cookie, xlator_t * this,
-                       int32_t op_ret, int32_t op_errno, fd_t * fd,
-                       dict_t * xdata)
+int32_t
+ec_opendir_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+               int32_t op_ret, int32_t op_errno, fd_t *fd, dict_t *xdata)
 {
-    ec_fop_data_t * fop = NULL;
-    ec_cbk_data_t * cbk = NULL;
+    ec_fop_data_t *fop = NULL;
+    ec_cbk_data_t *cbk = NULL;
     int32_t idx = (int32_t)(uintptr_t)cookie;
 
     VALIDATE_OR_GOTO(this, out);
@@ -49,52 +51,51 @@ int32_t ec_opendir_cbk(call_frame_t * frame, void * cookie, xlator_t * this,
 
     fop = frame->local;
 
-    ec_trace("CBK", fop, "idx=%d, frame=%p, op_ret=%d, op_errno=%d", idx,
-             frame, op_ret, op_errno);
+    ec_trace("CBK", fop, "idx=%d, frame=%p, op_ret=%d, op_errno=%d", idx, frame,
+             op_ret, op_errno);
 
     cbk = ec_cbk_data_allocate(frame, this, fop, GF_FOP_OPENDIR, idx, op_ret,
                                op_errno);
-    if (cbk != NULL)
-    {
-        if (op_ret >= 0)
-        {
-            if (fd != NULL)
-            {
+    if (cbk != NULL) {
+        if (op_ret >= 0) {
+            if (fd != NULL) {
                 cbk->fd = fd_ref(fd);
-                if (cbk->fd == NULL)
-                {
-                    gf_log(this->name, GF_LOG_ERROR, "Failed to reference a "
-                                                     "file descriptor.");
+                if (cbk->fd == NULL) {
+                    gf_msg(this->name, GF_LOG_ERROR, 0,
+                           EC_MSG_FILE_DESC_REF_FAIL,
+                           "Failed to reference a "
+                           "file descriptor.");
 
                     goto out;
                 }
             }
         }
-        if (xdata != NULL)
-        {
+        if (xdata != NULL) {
             cbk->xdata = dict_ref(xdata);
-            if (cbk->xdata == NULL)
-            {
-                gf_log(this->name, GF_LOG_ERROR, "Failed to reference a "
-                                                 "dictionary.");
+            if (cbk->xdata == NULL) {
+                gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_DICT_REF_FAIL,
+                       "Failed to reference a "
+                       "dictionary.");
 
                 goto out;
             }
         }
 
         ec_combine(cbk, ec_combine_opendir);
+
+        ec_update_fd_status(fd, this, idx, op_ret);
     }
 
 out:
-    if (fop != NULL)
-    {
+    if (fop != NULL) {
         ec_complete(fop);
     }
 
     return 0;
 }
 
-void ec_wind_opendir(ec_t * ec, ec_fop_data_t * fop, int32_t idx)
+void
+ec_wind_opendir(ec_t *ec, ec_fop_data_t *fop, int32_t idx)
 {
     ec_trace("WIND", fop, "idx=%d", idx);
 
@@ -103,39 +104,45 @@ void ec_wind_opendir(ec_t * ec, ec_fop_data_t * fop, int32_t idx)
                       &fop->loc[0], fop->fd, fop->xdata);
 }
 
-int32_t ec_manager_opendir(ec_fop_data_t * fop, int32_t state)
+int32_t
+ec_manager_opendir(ec_fop_data_t *fop, int32_t state)
 {
-    ec_cbk_data_t * cbk;
+    ec_cbk_data_t *cbk;
     ec_fd_t *ctx;
+    int32_t err;
 
-    switch (state)
-    {
+    switch (state) {
         case EC_STATE_INIT:
             LOCK(&fop->fd->lock);
 
             ctx = __ec_fd_get(fop->fd, fop->xl);
-            if ((ctx == NULL) ||
-                (ec_loc_from_loc(fop->xl, &ctx->loc, &fop->loc[0])) != 0) {
+            if (ctx == NULL) {
                 UNLOCK(&fop->fd->lock);
 
-                fop->error = EIO;
+                fop->error = ENOMEM;
 
                 return EC_STATE_REPORT;
             }
+            if (!ctx->loc.inode) {
+                err = ec_loc_from_loc(fop->xl, &ctx->loc, &fop->loc[0]);
+                if (err != 0) {
+                    UNLOCK(&fop->fd->lock);
+
+                    fop->error = -err;
+
+                    return EC_STATE_REPORT;
+                }
+            }
 
             UNLOCK(&fop->fd->lock);
 
             /* Fall through */
 
         case EC_STATE_LOCK:
-            ec_lock_prepare_entry(fop, &fop->loc[0], 0);
+            ec_lock_prepare_inode(fop, &fop->loc[0], EC_QUERY_INFO, 0,
+                                  EC_RANGE_FULL);
             ec_lock(fop);
 
-            return EC_STATE_GET_SIZE_AND_VERSION;
-
-        case EC_STATE_GET_SIZE_AND_VERSION:
-            ec_get_size_version(fop);
-
             return EC_STATE_DISPATCH;
 
         case EC_STATE_DISPATCH:
@@ -144,38 +151,20 @@ int32_t ec_manager_opendir(ec_fop_data_t * fop, int32_t state)
             return EC_STATE_PREPARE_ANSWER;
 
         case EC_STATE_PREPARE_ANSWER:
-            cbk = fop->answer;
-            if (cbk != NULL)
-            {
-                if (!ec_dict_combine(cbk, EC_COMBINE_XDATA))
-                {
-                    if (cbk->op_ret >= 0)
-                    {
-                        cbk->op_ret = -1;
-                        cbk->op_errno = EIO;
-                    }
+            cbk = ec_fop_prepare_answer(fop, _gf_true);
+            if (cbk != NULL) {
+                /* Save which subvolumes successfully opened the directory.
+                 * If ctx->open is 0, it means that readdir cannot be
+                 * processed in this directory.
+                 */
+                LOCK(&fop->fd->lock);
+
+                ctx = __ec_fd_get(fop->fd, fop->xl);
+                if (ctx != NULL) {
+                    ctx->open |= cbk->mask;
                 }
-                if (cbk->op_ret >= 0) {
-                    /* Save which subvolumes successfully opened the directory.
-                     * If ctx->open is 0, it means that readdir cannot be
-                     * processed in this directory.
-                     */
-                    LOCK(&fop->fd->lock);
-
-                    ctx = __ec_fd_get(fop->fd, fop->xl);
-                    if (ctx != NULL) {
-                        ctx->open |= cbk->mask;
-                    }
 
-                    UNLOCK(&fop->fd->lock);
-                }
-                if (cbk->op_ret < 0) {
-                    ec_fop_set_error(fop, cbk->op_errno);
-                }
-            }
-            else
-            {
-                ec_fop_set_error(fop, EIO);
+                UNLOCK(&fop->fd->lock);
             }
 
             return EC_STATE_REPORT;
@@ -185,8 +174,7 @@ int32_t ec_manager_opendir(ec_fop_data_t * fop, int32_t state)
 
             GF_ASSERT(cbk != NULL);
 
-            if (fop->cbks.opendir != NULL)
-            {
+            if (fop->cbks.opendir != NULL) {
                 fop->cbks.opendir(fop->req_frame, fop, fop->xl, cbk->op_ret,
                                   cbk->op_errno, cbk->fd, cbk->xdata);
             }
@@ -195,14 +183,12 @@ int32_t ec_manager_opendir(ec_fop_data_t * fop, int32_t state)
 
         case -EC_STATE_INIT:
         case -EC_STATE_LOCK:
-        case -EC_STATE_GET_SIZE_AND_VERSION:
         case -EC_STATE_DISPATCH:
         case -EC_STATE_PREPARE_ANSWER:
         case -EC_STATE_REPORT:
             GF_ASSERT(fop->error != 0);
 
-            if (fop->cbks.opendir != NULL)
-            {
+            if (fop->cbks.opendir != NULL) {
                 fop->cbks.opendir(fop->req_frame, fop, fop->xl, -1, fop->error,
                                   NULL, NULL);
             }
@@ -222,62 +208,59 @@ int32_t ec_manager_opendir(ec_fop_data_t * fop, int32_t state)
             return EC_STATE_END;
 
         default:
-            gf_log(fop->xl->name, GF_LOG_ERROR, "Unhandled state %d for %s",
-                   state, ec_fop_name(fop->id));
+            gf_msg(fop->xl->name, GF_LOG_ERROR, EINVAL, EC_MSG_UNHANDLED_STATE,
+                   "Unhandled state %d for %s", state, ec_fop_name(fop->id));
 
             return EC_STATE_END;
     }
 }
 
-void ec_opendir(call_frame_t * frame, xlator_t * this, uintptr_t target,
-                int32_t minimum, fop_opendir_cbk_t func, void * data,
-                loc_t * loc, fd_t * fd, dict_t * xdata)
+void
+ec_opendir(call_frame_t *frame, xlator_t *this, uintptr_t target,
+           uint32_t fop_flags, fop_opendir_cbk_t func, void *data, loc_t *loc,
+           fd_t *fd, dict_t *xdata)
 {
-    ec_cbk_t callback = { .opendir = func };
-    ec_fop_data_t * fop = NULL;
-    int32_t error = EIO;
+    ec_cbk_t callback = {.opendir = func};
+    ec_fop_data_t *fop = NULL;
+    int32_t error = ENOMEM;
 
-    gf_log("ec", GF_LOG_TRACE, "EC(OPENDIR) %p", frame);
+    gf_msg_trace("ec", 0, "EC(OPENDIR) %p", frame);
 
     VALIDATE_OR_GOTO(this, out);
     GF_VALIDATE_OR_GOTO(this->name, frame, out);
     GF_VALIDATE_OR_GOTO(this->name, this->private, out);
 
-    fop = ec_fop_data_allocate(frame, this, GF_FOP_OPENDIR, EC_FLAG_UPDATE_FD,
-                               target, minimum, ec_wind_opendir,
+    fop = ec_fop_data_allocate(frame, this, GF_FOP_OPENDIR, EC_FLAG_LOCK_SHARED,
+                               target, fop_flags, ec_wind_opendir,
                                ec_manager_opendir, callback, data);
-    if (fop == NULL)
-    {
+    if (fop == NULL) {
         goto out;
     }
 
-    if (loc != NULL)
-    {
-        if (loc_copy(&fop->loc[0], loc) != 0)
-        {
-            gf_log(this->name, GF_LOG_ERROR, "Failed to copy a location.");
+    if (loc != NULL) {
+        if (loc_copy(&fop->loc[0], loc) != 0) {
+            gf_msg(this->name, GF_LOG_ERROR, ENOMEM, EC_MSG_LOC_COPY_FAIL,
+                   "Failed to copy a location.");
 
             goto out;
         }
     }
-    if (fd != NULL)
-    {
+    if (fd != NULL) {
         fop->fd = fd_ref(fd);
-        if (fop->fd == NULL)
-        {
-            gf_log(this->name, GF_LOG_ERROR, "Failed to reference a "
-                                             "file descriptor.");
+        if (fop->fd == NULL) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_FILE_DESC_REF_FAIL,
+                   "Failed to reference a "
+                   "file descriptor.");
 
             goto out;
         }
     }
-    if (xdata != NULL)
-    {
+    if (xdata != NULL) {
         fop->xdata = dict_ref(xdata);
-        if (fop->xdata == NULL)
-        {
-            gf_log(this->name, GF_LOG_ERROR, "Failed to reference a "
-                                             "dictionary.");
+        if (fop->xdata == NULL) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_DICT_REF_FAIL,
+                   "Failed to reference a "
+                   "dictionary.");
 
             goto out;
         }
@@ -286,46 +269,72 @@ void ec_opendir(call_frame_t * frame, xlator_t * this, uintptr_t target,
     error = 0;
 
 out:
-    if (fop != NULL)
-    {
+    if (fop != NULL) {
         ec_manager(fop, error);
+    } else {
+        func(frame, NULL, this, -1, error, NULL, NULL);
     }
-    else
-    {
-        func(frame, NULL, this, -1, EIO, NULL, NULL);
+}
+
+/* Returns -1 if client_id is invalid else index of child subvol in xl_list */
+int
+ec_deitransform(xlator_t *this, off_t offset)
+{
+    int idx = -1;
+    int client_id = -1;
+    ec_t *ec = this->private;
+    char id[32] = {0};
+    int err;
+
+    client_id = gf_deitransform(this, offset);
+    sprintf(id, "%d", client_id);
+    err = dict_get_int32(ec->leaf_to_subvolid, id, &idx);
+    if (err < 0) {
+        idx = err;
+        goto out;
     }
+
+out:
+    if (idx < 0) {
+        gf_msg(this->name, GF_LOG_ERROR, EINVAL, EC_MSG_INVALID_REQUEST,
+               "Invalid index %d in readdirp request", client_id);
+        idx = -EINVAL;
+    }
+    return idx;
 }
 
 /* FOP: readdir */
 
-void ec_adjust_readdir(ec_t * ec, int32_t idx, gf_dirent_t * entries)
+void
+ec_adjust_readdirp(ec_t *ec, int32_t idx, gf_dirent_t *entries)
 {
-    gf_dirent_t * entry;
+    gf_dirent_t *entry;
 
     list_for_each_entry(entry, &entries->list, list)
     {
-        if (entry->d_stat.ia_type == IA_IFREG)
-        {
+        if (!entry->inode)
+            continue;
+
+        if (entry->d_stat.ia_type == IA_IFREG) {
             if ((entry->dict == NULL) ||
                 (ec_dict_del_number(entry->dict, EC_XATTR_SIZE,
-                                    &entry->d_stat.ia_size) != 0))
-            {
-                gf_log(ec->xl->name, GF_LOG_WARNING, "Unable to get exact "
-                                                     "file size.");
-
-                entry->d_stat.ia_size *= ec->fragments;
+                                    &entry->d_stat.ia_size) != 0)) {
+                inode_unref(entry->inode);
+                entry->inode = NULL;
+            } else {
+                ec_iatt_rebuild(ec, &entry->d_stat, 1, 1);
             }
-
-            ec_iatt_rebuild(ec, &entry->d_stat, 1, 1);
         }
     }
 }
 
-int32_t ec_readdir_cbk(call_frame_t * frame, void * cookie, xlator_t * this,
-                       int32_t op_ret, int32_t op_errno, gf_dirent_t * entries,
-                       dict_t * xdata)
+int32_t
+ec_common_readdir_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                      int32_t op_ret, int32_t op_errno, gf_dirent_t *entries,
+                      dict_t *xdata)
 {
-    ec_fop_data_t * fop = NULL;
+    ec_fop_data_t *fop = NULL;
+    ec_cbk_data_t *cbk = NULL;
     int32_t idx = (int32_t)(uintptr_t)cookie;
 
     VALIDATE_OR_GOTO(this, out);
@@ -335,155 +344,199 @@ int32_t ec_readdir_cbk(call_frame_t * frame, void * cookie, xlator_t * this,
 
     fop = frame->local;
 
-    ec_trace("CBK", fop, "idx=%d, frame=%p, op_ret=%d, op_errno=%d", idx,
-             frame, op_ret, op_errno);
+    ec_trace("CBK", fop, "idx=%d, frame=%p, op_ret=%d, op_errno=%d", idx, frame,
+             op_ret, op_errno);
 
-    if (op_ret > 0)
-    {
-        ec_adjust_readdir(fop->xl->private, idx, entries);
-    }
-
-    if (!ec_dispatch_one_retry(fop, idx, op_ret, op_errno))
-    {
-        if (fop->cbks.readdir != NULL)
-        {
-            fop->cbks.readdir(fop->req_frame, fop, this, op_ret, op_errno,
-                              entries, xdata);
-        }
+    cbk = ec_cbk_data_allocate(frame, this, fop, fop->id, idx, op_ret,
+                               op_errno);
+    if (cbk) {
+        if (xdata)
+            cbk->xdata = dict_ref(xdata);
+        if (cbk->op_ret >= 0)
+            list_splice_init(&entries->list, &cbk->entries.list);
+        ec_combine(cbk, NULL);
     }
 
 out:
-    if (fop != NULL)
-    {
+    if (fop != NULL) {
         ec_complete(fop);
     }
 
     return 0;
 }
 
-void ec_wind_readdir(ec_t * ec, ec_fop_data_t * fop, int32_t idx)
+void
+ec_wind_readdir(ec_t *ec, ec_fop_data_t *fop, int32_t idx)
 {
     ec_trace("WIND", fop, "idx=%d", idx);
 
-    STACK_WIND_COOKIE(fop->frame, ec_readdir_cbk, (void *)(uintptr_t)idx,
+    STACK_WIND_COOKIE(fop->frame, ec_common_readdir_cbk, (void *)(uintptr_t)idx,
                       ec->xl_list[idx], ec->xl_list[idx]->fops->readdir,
                       fop->fd, fop->size, fop->offset, fop->xdata);
 }
 
-int32_t ec_manager_readdir(ec_fop_data_t * fop, int32_t state)
+int32_t
+ec_manager_readdir(ec_fop_data_t *fop, int32_t state)
 {
-    ec_fd_t *ctx;
+    ec_fd_t *ctx = NULL;
+    ec_cbk_data_t *cbk = NULL;
 
-    switch (state)
-    {
+    switch (state) {
         case EC_STATE_INIT:
             /* Return error if opendir has not been successfully called on
              * any subvolume. */
             ctx = ec_fd_get(fop->fd, fop->xl);
-            if ((ctx == NULL) || (ctx->open == 0)) {
-                fop->error = EINVAL;
+            if (ctx == NULL) {
+                fop->error = ENOMEM;
+            } else if (ctx->open == 0) {
+                fop->error = EBADFD;
+            }
 
+            if (fop->error) {
+                gf_msg(fop->xl->name, GF_LOG_ERROR, fop->error,
+                       EC_MSG_INVALID_REQUEST, "EC is not winding readdir: %s",
+                       ec_msg_str(fop));
                 return EC_STATE_REPORT;
             }
 
-            if (fop->xdata == NULL)
-            {
-                fop->xdata = dict_new();
-                if (fop->xdata == NULL)
-                {
-                    gf_log(fop->xl->name, GF_LOG_ERROR, "Unable to prepare "
-                                                        "readdirp request");
+            if (fop->id == GF_FOP_READDIRP) {
+                int32_t err;
 
-                    fop->error = EIO;
+                if (fop->xdata == NULL) {
+                    fop->xdata = dict_new();
+                    if (fop->xdata == NULL) {
+                        fop->error = ENOMEM;
 
-                    return EC_STATE_REPORT;
+                        return EC_STATE_REPORT;
+                    }
                 }
-            }
-            if (dict_set_uint64(fop->xdata, EC_XATTR_SIZE, 0) != 0)
-            {
-                gf_log(fop->xl->name, GF_LOG_ERROR, "Unable to prepare "
-                                                    "readdirp request");
 
-                fop->error = EIO;
+                err = dict_set_uint64(fop->xdata, EC_XATTR_SIZE, 0);
+                if (err != 0) {
+                    fop->error = -err;
 
-                return EC_STATE_REPORT;
+                    return EC_STATE_REPORT;
+                }
             }
 
-            if (fop->offset != 0)
-            {
+            if (fop->offset != 0) {
+                /* Non-zero offset is irrecoverable error as the offset may not
+                 * be valid on other bricks*/
                 int32_t idx = -1;
-                ec_t    *ec = fop->xl->private;
-
-                idx = gf_deitransform(fop->xl, fop->offset);
-
-                if ((idx < 0) || (idx >= ec->nodes)) {
 
-                        gf_log(fop->xl->name, GF_LOG_ERROR,
-                               "Invalid index %d in readdirp request", idx);
+                idx = ec_deitransform(fop->xl, fop->offset);
 
-                        fop->error = EIO;
-
-                        return EC_STATE_REPORT;
+                if (idx < 0) {
+                    fop->error = -idx;
+                    return EC_STATE_REPORT;
                 }
                 fop->mask &= 1ULL << idx;
+            } else {
+                ec_lock_prepare_fd(fop, fop->fd, EC_QUERY_INFO, 0,
+                                   EC_RANGE_FULL);
+                ec_lock(fop);
             }
 
-        /* Fall through */
+            return EC_STATE_DISPATCH;
 
         case EC_STATE_DISPATCH:
             ec_dispatch_one(fop);
 
+            return EC_STATE_PREPARE_ANSWER;
+
+        case EC_STATE_PREPARE_ANSWER:
+            if (ec_dispatch_one_retry(fop, &cbk)) {
+                return EC_STATE_DISPATCH;
+            }
+
+            if ((cbk != NULL) && (cbk->op_ret > 0) &&
+                (fop->id == GF_FOP_READDIRP)) {
+                ec_adjust_readdirp(fop->xl->private, cbk->idx, &cbk->entries);
+            }
+
             return EC_STATE_REPORT;
 
+        case EC_STATE_REPORT:
+            cbk = fop->answer;
+            GF_ASSERT(cbk);
+            if (fop->id == GF_FOP_READDIR) {
+                if (fop->cbks.readdir != NULL) {
+                    fop->cbks.readdir(fop->req_frame, fop, fop->xl, cbk->op_ret,
+                                      cbk->op_errno, &cbk->entries, cbk->xdata);
+                }
+            } else {
+                if (fop->cbks.readdirp != NULL) {
+                    fop->cbks.readdirp(fop->req_frame, fop, fop->xl,
+                                       cbk->op_ret, cbk->op_errno,
+                                       &cbk->entries, cbk->xdata);
+                }
+            }
+            if (fop->offset == 0)
+                return EC_STATE_LOCK_REUSE;
+            else
+                return EC_STATE_END;
+
         case -EC_STATE_INIT:
+        case -EC_STATE_LOCK:
+        case -EC_STATE_DISPATCH:
+        case -EC_STATE_PREPARE_ANSWER:
         case -EC_STATE_REPORT:
-            if (fop->id == GF_FOP_READDIR)
-            {
-                if (fop->cbks.readdir != NULL)
-                {
+            if (fop->id == GF_FOP_READDIR) {
+                if (fop->cbks.readdir != NULL) {
                     fop->cbks.readdir(fop->req_frame, fop, fop->xl, -1,
                                       fop->error, NULL, NULL);
                 }
-            }
-            else
-            {
-                if (fop->cbks.readdirp != NULL)
-                {
+            } else {
+                if (fop->cbks.readdirp != NULL) {
                     fop->cbks.readdirp(fop->req_frame, fop, fop->xl, -1,
                                        fop->error, NULL, NULL);
                 }
             }
+            if (fop->offset == 0)
+                return EC_STATE_LOCK_REUSE;
+            else
+                return EC_STATE_END;
 
-        case EC_STATE_REPORT:
-            return EC_STATE_END;
+        case -EC_STATE_LOCK_REUSE:
+        case EC_STATE_LOCK_REUSE:
+            GF_ASSERT(fop->offset == 0);
+            ec_lock_reuse(fop);
+
+            return EC_STATE_UNLOCK;
+
+        case -EC_STATE_UNLOCK:
+        case EC_STATE_UNLOCK:
+            GF_ASSERT(fop->offset == 0);
+            ec_unlock(fop);
 
+            return EC_STATE_END;
         default:
-            gf_log(fop->xl->name, GF_LOG_ERROR, "Unhandled state %d for %s",
-                   state, ec_fop_name(fop->id));
+            gf_msg(fop->xl->name, GF_LOG_ERROR, EINVAL, EC_MSG_UNHANDLED_STATE,
+                   "Unhandled state %d for %s", state, ec_fop_name(fop->id));
 
             return EC_STATE_END;
     }
 }
 
-void ec_readdir(call_frame_t * frame, xlator_t * this, uintptr_t target,
-                int32_t minimum, fop_readdir_cbk_t func, void * data,
-                fd_t * fd, size_t size, off_t offset, dict_t * xdata)
+void
+ec_readdir(call_frame_t *frame, xlator_t *this, uintptr_t target,
+           uint32_t fop_flags, fop_readdir_cbk_t func, void *data, fd_t *fd,
+           size_t size, off_t offset, dict_t *xdata)
 {
-    ec_cbk_t callback = { .readdir = func };
-    ec_fop_data_t * fop = NULL;
-    int32_t error = EIO;
+    ec_cbk_t callback = {.readdir = func};
+    ec_fop_data_t *fop = NULL;
+    int32_t error = ENOMEM;
 
-    gf_log("ec", GF_LOG_TRACE, "EC(READDIR) %p", frame);
+    gf_msg_trace("ec", 0, "EC(READDIR) %p", frame);
 
     VALIDATE_OR_GOTO(this, out);
     GF_VALIDATE_OR_GOTO(this->name, frame, out);
     GF_VALIDATE_OR_GOTO(this->name, this->private, out);
 
-    fop = ec_fop_data_allocate(frame, this, GF_FOP_READDIR, 0, target, minimum,
-                               ec_wind_readdir, ec_manager_readdir, callback,
-                               data);
-    if (fop == NULL)
-    {
+    fop = ec_fop_data_allocate(frame, this, GF_FOP_READDIR, EC_FLAG_LOCK_SHARED,
+                               target, fop_flags, ec_wind_readdir,
+                               ec_manager_readdir, callback, data);
+    if (fop == NULL) {
         goto out;
     }
 
@@ -492,24 +545,22 @@ void ec_readdir(call_frame_t * frame, xlator_t * this, uintptr_t target,
     fop->size = size;
     fop->offset = offset;
 
-    if (fd != NULL)
-    {
+    if (fd != NULL) {
         fop->fd = fd_ref(fd);
-        if (fop->fd == NULL)
-        {
-            gf_log(this->name, GF_LOG_ERROR, "Failed to reference a "
-                                             "file descriptor.");
+        if (fop->fd == NULL) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_FILE_DESC_REF_FAIL,
+                   "Failed to reference a "
+                   "file descriptor.");
 
             goto out;
         }
     }
-    if (xdata != NULL)
-    {
+    if (xdata != NULL) {
         fop->xdata = dict_ref(xdata);
-        if (fop->xdata == NULL)
-        {
-            gf_log(this->name, GF_LOG_ERROR, "Failed to reference a "
-                                             "dictionary.");
+        if (fop->xdata == NULL) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_DICT_REF_FAIL,
+                   "Failed to reference a "
+                   "dictionary.");
 
             goto out;
         }
@@ -518,86 +569,44 @@ void ec_readdir(call_frame_t * frame, xlator_t * this, uintptr_t target,
     error = 0;
 
 out:
-    if (fop != NULL)
-    {
+    if (fop != NULL) {
         ec_manager(fop, error);
-    }
-    else
-    {
-        func(frame, NULL, this, -1, EIO, NULL, NULL);
+    } else {
+        func(frame, NULL, this, -1, error, NULL, NULL);
     }
 }
 
 /* FOP: readdirp */
 
-int32_t ec_readdirp_cbk(call_frame_t * frame, void * cookie, xlator_t * this,
-                        int32_t op_ret, int32_t op_errno,
-                        gf_dirent_t * entries, dict_t * xdata)
-{
-    ec_fop_data_t * fop = NULL;
-    int32_t idx = (int32_t)(uintptr_t)cookie;
-
-    VALIDATE_OR_GOTO(this, out);
-    GF_VALIDATE_OR_GOTO(this->name, frame, out);
-    GF_VALIDATE_OR_GOTO(this->name, frame->local, out);
-    GF_VALIDATE_OR_GOTO(this->name, this->private, out);
-
-    fop = frame->local;
-
-    ec_trace("CBK", fop, "idx=%d, frame=%p, op_ret=%d, op_errno=%d", idx,
-             frame, op_ret, op_errno);
-
-    if (op_ret > 0)
-    {
-        ec_adjust_readdir(fop->xl->private, idx, entries);
-    }
-
-    if (!ec_dispatch_one_retry(fop, idx, op_ret, op_errno))
-    {
-        if (fop->cbks.readdirp != NULL)
-        {
-            fop->cbks.readdirp(fop->req_frame, fop, this, op_ret, op_errno,
-                               entries, xdata);
-        }
-    }
-
-out:
-    if (fop != NULL)
-    {
-        ec_complete(fop);
-    }
-
-    return 0;
-}
-
-void ec_wind_readdirp(ec_t * ec, ec_fop_data_t * fop, int32_t idx)
+void
+ec_wind_readdirp(ec_t *ec, ec_fop_data_t *fop, int32_t idx)
 {
     ec_trace("WIND", fop, "idx=%d", idx);
 
-    STACK_WIND_COOKIE(fop->frame, ec_readdirp_cbk, (void *)(uintptr_t)idx,
+    STACK_WIND_COOKIE(fop->frame, ec_common_readdir_cbk, (void *)(uintptr_t)idx,
                       ec->xl_list[idx], ec->xl_list[idx]->fops->readdirp,
                       fop->fd, fop->size, fop->offset, fop->xdata);
 }
 
-void ec_readdirp(call_frame_t * frame, xlator_t * this, uintptr_t target,
-                 int32_t minimum, fop_readdirp_cbk_t func, void * data,
-                 fd_t * fd, size_t size, off_t offset, dict_t * xdata)
+void
+ec_readdirp(call_frame_t *frame, xlator_t *this, uintptr_t target,
+            uint32_t fop_flags, fop_readdirp_cbk_t func, void *data, fd_t *fd,
+            size_t size, off_t offset, dict_t *xdata)
 {
-    ec_cbk_t callback = { .readdirp = func };
-    ec_fop_data_t * fop = NULL;
-    int32_t error = EIO;
+    ec_cbk_t callback = {.readdirp = func};
+    ec_fop_data_t *fop = NULL;
+    int32_t error = ENOMEM;
 
-    gf_log("ec", GF_LOG_TRACE, "EC(READDIRP) %p", frame);
+    gf_msg_trace("ec", 0, "EC(READDIRP) %p", frame);
 
     VALIDATE_OR_GOTO(this, out);
     GF_VALIDATE_OR_GOTO(this->name, frame, out);
     GF_VALIDATE_OR_GOTO(this->name, this->private, out);
 
-    fop = ec_fop_data_allocate(frame, this, GF_FOP_READDIRP, 0, target,
-                               minimum, ec_wind_readdirp, ec_manager_readdir,
-                               callback, data);
-    if (fop == NULL)
-    {
+    fop = ec_fop_data_allocate(
+        frame, this, GF_FOP_READDIRP, EC_FLAG_LOCK_SHARED, target, fop_flags,
+        ec_wind_readdirp, ec_manager_readdir, callback, data);
+    if (fop == NULL) {
         goto out;
     }
 
@@ -606,24 +615,22 @@ void ec_readdirp(call_frame_t * frame, xlator_t * this, uintptr_t target,
     fop->size = size;
     fop->offset = offset;
 
-    if (fd != NULL)
-    {
+    if (fd != NULL) {
         fop->fd = fd_ref(fd);
-        if (fop->fd == NULL)
-        {
-            gf_log(this->name, GF_LOG_ERROR, "Failed to reference a "
-                                             "file descriptor.");
+        if (fop->fd == NULL) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_FILE_DESC_REF_FAIL,
+                   "Failed to reference a "
+                   "file descriptor.");
 
             goto out;
         }
     }
-    if (xdata != NULL)
-    {
+    if (xdata != NULL) {
         fop->xdata = dict_ref(xdata);
-        if (fop->xdata == NULL)
-        {
-            gf_log(this->name, GF_LOG_ERROR, "Failed to reference a "
-                                             "dictionary.");
+        if (fop->xdata == NULL) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_DICT_REF_FAIL,
+                   "Failed to reference a "
+                   "dictionary.");
 
             goto out;
         }
@@ -632,12 +639,9 @@ void ec_readdirp(call_frame_t * frame, xlator_t * this, uintptr_t target,
     error = 0;
 
 out:
-    if (fop != NULL)
-    {
+    if (fop != NULL) {
         ec_manager(fop, error);
-    }
-    else
-    {
-        func(frame, NULL, this, -1, EIO, NULL, NULL);
+    } else {
+        func(frame, NULL, this, -1, error, NULL, NULL);
     }
 }
diff --git a/xlators/cluster/ec/src/ec-dir-write.c b/xlators/cluster/ec/src/ec-dir-write.c
index 866f5120bf6..53d27d895c3 100644
--- a/xlators/cluster/ec/src/ec-dir-write.c
+++ b/xlators/cluster/ec/src/ec-dir-write.c
@@ -8,9 +8,8 @@
   cases as published by the Free Software Foundation.
 */
 
-#include "xlator.h"
-#include "defaults.h"
-
+#include "ec.h"
+#include "ec-messages.h"
 #include "ec-helpers.h"
 #include "ec-common.h"
 #include "ec-combine.h"
@@ -18,75 +17,75 @@
 #include "ec-fops.h"
 
 int
-ec_dir_write_cbk (call_frame_t *frame, xlator_t *this,
-                  void *cookie, int op_ret, int op_errno,
-                  struct iatt *poststat, struct iatt *preparent,
-                  struct iatt *postparent, struct iatt *preparent2,
-                  struct iatt *postparent2, dict_t *xdata)
+ec_dir_write_cbk(call_frame_t *frame, xlator_t *this, void *cookie, int op_ret,
+                 int op_errno, struct iatt *poststat, struct iatt *preparent,
+                 struct iatt *postparent, struct iatt *preparent2,
+                 struct iatt *postparent2, dict_t *xdata)
 {
-        ec_fop_data_t *fop = NULL;
-        ec_cbk_data_t *cbk = NULL;
-        int           i    = 0;
-        int           idx  = 0;
+    ec_fop_data_t *fop = NULL;
+    ec_cbk_data_t *cbk = NULL;
+    int i = 0;
+    int idx = 0;
 
-        VALIDATE_OR_GOTO (this, out);
-        GF_VALIDATE_OR_GOTO (this->name, frame, out);
-        GF_VALIDATE_OR_GOTO (this->name, frame->local, out);
-        GF_VALIDATE_OR_GOTO (this->name, this->private, out);
+    VALIDATE_OR_GOTO(this, out);
+    GF_VALIDATE_OR_GOTO(this->name, frame, out);
+    GF_VALIDATE_OR_GOTO(this->name, frame->local, out);
+    GF_VALIDATE_OR_GOTO(this->name, this->private, out);
 
-        fop = frame->local;
-        idx = (long) cookie;
+    fop = frame->local;
+    idx = (long)cookie;
 
-        ec_trace("CBK", fop, "idx=%d, frame=%p, op_ret=%d, op_errno=%d", idx,
-                 frame, op_ret, op_errno);
+    ec_trace("CBK", fop, "idx=%d, frame=%p, op_ret=%d, op_errno=%d", idx, frame,
+             op_ret, op_errno);
 
-        cbk = ec_cbk_data_allocate (frame, this, fop, fop->id, idx, op_ret,
-                                    op_errno);
-        if (!cbk)
-                goto out;
+    cbk = ec_cbk_data_allocate(frame, this, fop, fop->id, idx, op_ret,
+                               op_errno);
+    if (!cbk)
+        goto out;
 
-        if (op_ret < 0)
-                goto out;
+    if (xdata)
+        cbk->xdata = dict_ref(xdata);
 
-        if (xdata)
-                cbk->xdata = dict_ref (xdata);
+    if (op_ret < 0)
+        goto out;
 
-        if (poststat)
-                cbk->iatt[i++] = *poststat;
+    if (poststat)
+        cbk->iatt[i++] = *poststat;
 
-        if (preparent)
-                cbk->iatt[i++] = *preparent;
+    if (preparent)
+        cbk->iatt[i++] = *preparent;
 
-        if (postparent)
-                cbk->iatt[i++] = *postparent;
+    if (postparent)
+        cbk->iatt[i++] = *postparent;
 
-        if (preparent2)
-                cbk->iatt[i++] = *preparent2;
+    if (preparent2)
+        cbk->iatt[i++] = *preparent2;
 
-        if (postparent2)
-                cbk->iatt[i++] = *postparent2;
+    if (postparent2)
+        cbk->iatt[i++] = *postparent2;
 
 out:
-        if (cbk)
-                ec_combine (cbk, ec_combine_write);
-        if (fop)
-                ec_complete (fop);
-        return 0;
+    if (cbk)
+        ec_combine(cbk, ec_combine_write);
+
+    if (fop)
+        ec_complete(fop);
+    return 0;
 }
 
 /* FOP: create */
 
-int32_t ec_create_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
-                      int32_t op_ret, int32_t op_errno, fd_t *fd,
-                      inode_t *inode, struct iatt *buf,
-                      struct iatt *preparent, struct iatt *postparent,
-                      dict_t *xdata)
+int32_t
+ec_create_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int32_t op_ret,
+              int32_t op_errno, fd_t *fd, inode_t *inode, struct iatt *buf,
+              struct iatt *preparent, struct iatt *postparent, dict_t *xdata)
 {
-        return ec_dir_write_cbk (frame, this, cookie, op_ret, op_errno,
-                                 buf, preparent, postparent, NULL, NULL, xdata);
+    return ec_dir_write_cbk(frame, this, cookie, op_ret, op_errno, buf,
+                            preparent, postparent, NULL, NULL, xdata);
 }
 
-void ec_wind_create(ec_t * ec, ec_fop_data_t * fop, int32_t idx)
+void
+ec_wind_create(ec_t *ec, ec_fop_data_t *fop, int32_t idx)
 {
     ec_trace("WIND", fop, "idx=%d", idx);
 
@@ -96,25 +95,33 @@ void ec_wind_create(ec_t * ec, ec_fop_data_t * fop, int32_t idx)
                       fop->fd, fop->xdata);
 }
 
-int32_t ec_manager_create(ec_fop_data_t * fop, int32_t state)
+int32_t
+ec_manager_create(ec_fop_data_t *fop, int32_t state)
 {
+    ec_config_t config;
+    ec_t *ec;
+    ec_cbk_data_t *cbk;
+    ec_fd_t *ctx;
+    uint64_t version[2] = {0, 0};
+    int32_t err;
 
-
-    ec_t * ec;
-    ec_cbk_data_t * cbk;
-    ec_fd_t * ctx;
-
-    switch (state)
-    {
+    switch (state) {
         case EC_STATE_INIT:
             LOCK(&fop->fd->lock);
 
             ctx = __ec_fd_get(fop->fd, fop->xl);
-            if ((ctx == NULL) ||
-                (ec_loc_from_loc(fop->xl, &ctx->loc, &fop->loc[0])) != 0) {
+            if (ctx == NULL) {
                 UNLOCK(&fop->fd->lock);
 
-                fop->error = EIO;
+                fop->error = ENOMEM;
+
+                return EC_STATE_REPORT;
+            }
+            err = ec_loc_from_loc(fop->xl, &ctx->loc, &fop->loc[0]);
+            if (err != 0) {
+                UNLOCK(&fop->fd->lock);
+
+                fop->error = -err;
 
                 return EC_STATE_REPORT;
             }
@@ -123,12 +130,10 @@ int32_t ec_manager_create(ec_fop_data_t * fop, int32_t state)
 
             UNLOCK(&fop->fd->lock);
 
-            if (fop->xdata == NULL)
-            {
+            if (fop->xdata == NULL) {
                 fop->xdata = dict_new();
-                if (fop->xdata == NULL)
-                {
-                    fop->error = EIO;
+                if (fop->xdata == NULL) {
+                    fop->error = ENOMEM;
 
                     return EC_STATE_REPORT;
                 }
@@ -136,29 +141,29 @@ int32_t ec_manager_create(ec_fop_data_t * fop, int32_t state)
 
             ec = fop->xl->private;
 
-            fop->config.version = EC_CONFIG_VERSION;
-            fop->config.algorithm = EC_CONFIG_ALGORITHM;
-            fop->config.gf_word_size = EC_GF_BITS;
-            fop->config.bricks = ec->nodes;
-            fop->config.redundancy = ec->redundancy;
-            fop->config.chunk_size = EC_METHOD_CHUNK_SIZE;
+            config.version = EC_CONFIG_VERSION;
+            config.algorithm = EC_CONFIG_ALGORITHM;
+            config.gf_word_size = EC_GF_BITS;
+            config.bricks = ec->nodes;
+            config.redundancy = ec->redundancy;
+            config.chunk_size = EC_METHOD_CHUNK_SIZE;
 
-            if (ec_dict_set_config(fop->xdata, EC_XATTR_CONFIG,
-                                   &fop->config) < 0)
-            {
-                fop->error = EIO;
+            err = ec_dict_set_config(fop->xdata, EC_XATTR_CONFIG, &config);
+            if (err != 0) {
+                fop->error = -err;
 
                 return EC_STATE_REPORT;
             }
-
-            if (ec_dict_set_number(fop->xdata, EC_XATTR_VERSION, 0) != 0) {
-                fop->error = EIO;
+            err = ec_dict_set_array(fop->xdata, EC_XATTR_VERSION, version,
+                                    EC_VERSION_SIZE);
+            if (err != 0) {
+                fop->error = -err;
 
                 return EC_STATE_REPORT;
             }
-
-            if (ec_dict_set_number(fop->xdata, EC_XATTR_SIZE, 0) != 0) {
-                fop->error = EIO;
+            err = ec_dict_set_number(fop->xdata, EC_XATTR_SIZE, 0);
+            if (err != 0) {
+                fop->error = -err;
 
                 return EC_STATE_REPORT;
             }
@@ -167,10 +172,11 @@ int32_t ec_manager_create(ec_fop_data_t * fop, int32_t state)
              * need to remove O_APPEND from flags (if present) */
             fop->int32 &= ~O_APPEND;
 
-        /* Fall through */
+            /* Fall through */
 
         case EC_STATE_LOCK:
-            ec_lock_prepare_entry(fop, &fop->loc[0], 1);
+            ec_lock_prepare_parent_inode(fop, &fop->loc[0], NULL,
+                                         EC_UPDATE_DATA | EC_UPDATE_META);
             ec_lock(fop);
 
             return EC_STATE_DISPATCH;
@@ -181,44 +187,25 @@ int32_t ec_manager_create(ec_fop_data_t * fop, int32_t state)
             return EC_STATE_PREPARE_ANSWER;
 
         case EC_STATE_PREPARE_ANSWER:
-            cbk = fop->answer;
-            if (cbk != NULL)
-            {
-                if (!ec_dict_combine(cbk, EC_COMBINE_XDATA))
-                {
-                    if (cbk->op_ret >= 0)
-                    {
-                        cbk->op_ret = -1;
-                        cbk->op_errno = EIO;
-                    }
-                }
-                if (cbk->op_ret >= 0) {
-                    ec_iatt_rebuild(fop->xl->private, cbk->iatt, 3,
-                                    cbk->count);
-
-                    if (ec_loc_update(fop->xl, &fop->loc[0], cbk->inode,
-                                      &cbk->iatt[0]) != 0) {
-                        cbk->op_ret = -1;
-                        cbk->op_errno = EIO;
-                    } else {
-                        LOCK(&fop->fd->lock);
-
-                        ctx = __ec_fd_get(fop->fd, fop->xl);
-                        if (ctx != NULL) {
-                            ctx->open |= cbk->mask;
-                        }
-
-                        UNLOCK(&fop->fd->lock);
+            cbk = ec_fop_prepare_answer(fop, _gf_false);
+            if (cbk != NULL) {
+                int32_t err;
+
+                ec_iatt_rebuild(fop->xl->private, cbk->iatt, 3, cbk->count);
+
+                err = ec_loc_update(fop->xl, &fop->loc[0], cbk->inode,
+                                    &cbk->iatt[0]);
+                if (!ec_cbk_set_error(cbk, -err, _gf_false)) {
+                    LOCK(&fop->fd->lock);
+
+                    ctx = __ec_fd_get(fop->fd, fop->xl);
+                    if (ctx != NULL) {
+                        ctx->open |= cbk->mask;
                     }
-                }
-                if (cbk->op_ret < 0) {
-                    ec_fop_set_error(fop, cbk->op_errno);
+
+                    UNLOCK(&fop->fd->lock);
                 }
             }
-            else
-            {
-                ec_fop_set_error(fop, EIO);
-            }
 
             return EC_STATE_REPORT;
 
@@ -227,12 +214,11 @@ int32_t ec_manager_create(ec_fop_data_t * fop, int32_t state)
 
             GF_ASSERT(cbk != NULL);
 
-            if (fop->cbks.create != NULL)
-            {
-                fop->cbks.create (fop->req_frame, fop, fop->xl, cbk->op_ret,
-                                  cbk->op_errno, fop->fd, fop->loc[0].inode,
-                                  &cbk->iatt[0], &cbk->iatt[1], &cbk->iatt[2],
-                                  cbk->xdata);
+            if (fop->cbks.create != NULL) {
+                QUORUM_CBK(fop->cbks.create, fop, fop->req_frame, fop, fop->xl,
+                           cbk->op_ret, cbk->op_errno, fop->fd,
+                           fop->loc[0].inode, &cbk->iatt[0], &cbk->iatt[1],
+                           &cbk->iatt[2], cbk->xdata);
             }
 
             return EC_STATE_LOCK_REUSE;
@@ -244,8 +230,7 @@ int32_t ec_manager_create(ec_fop_data_t * fop, int32_t state)
         case -EC_STATE_REPORT:
             GF_ASSERT(fop->error != 0);
 
-            if (fop->cbks.create != NULL)
-            {
+            if (fop->cbks.create != NULL) {
                 fop->cbks.create(fop->req_frame, fop, fop->xl, -1, fop->error,
                                  NULL, NULL, NULL, NULL, NULL, NULL);
             }
@@ -265,35 +250,32 @@ int32_t ec_manager_create(ec_fop_data_t * fop, int32_t state)
             return EC_STATE_END;
 
         default:
-            gf_log(fop->xl->name, GF_LOG_ERROR, "Unhandled state %d for %s",
-                   state, ec_fop_name(fop->id));
+            gf_msg(fop->xl->name, GF_LOG_ERROR, EINVAL, EC_MSG_UNHANDLED_STATE,
+                   "Unhandled state %d for %s", state, ec_fop_name(fop->id));
 
             return EC_STATE_END;
     }
 }
 
-void ec_create(call_frame_t * frame, xlator_t * this, uintptr_t target,
-               int32_t minimum, fop_create_cbk_t func, void * data,
-               loc_t * loc, int32_t flags, mode_t mode, mode_t umask,
-               fd_t * fd, dict_t * xdata)
+void
+ec_create(call_frame_t *frame, xlator_t *this, uintptr_t target,
+          uint32_t fop_flags, fop_create_cbk_t func, void *data, loc_t *loc,
+          int32_t flags, mode_t mode, mode_t umask, fd_t *fd, dict_t *xdata)
 {
-    ec_cbk_t callback = { .create = func };
-    ec_fop_data_t * fop = NULL;
-    int32_t error = EIO;
+    ec_cbk_t callback = {.create = func};
+    ec_fop_data_t *fop = NULL;
+    int32_t error = ENOMEM;
 
-    gf_log("ec", GF_LOG_TRACE, "EC(CREATE) %p", frame);
+    gf_msg_trace("ec", 0, "EC(CREATE) %p", frame);
 
     VALIDATE_OR_GOTO(this, out);
     GF_VALIDATE_OR_GOTO(this->name, frame, out);
     GF_VALIDATE_OR_GOTO(this->name, this->private, out);
 
-    fop = ec_fop_data_allocate(frame, this, GF_FOP_CREATE,
-                               EC_FLAG_UPDATE_LOC_PARENT |
-                               EC_FLAG_UPDATE_FD_INODE, target, minimum,
+    fop = ec_fop_data_allocate(frame, this, GF_FOP_CREATE, 0, target, fop_flags,
                                ec_wind_create, ec_manager_create, callback,
                                data);
-    if (fop == NULL)
-    {
+    if (fop == NULL) {
         goto out;
     }
 
@@ -301,33 +283,30 @@ void ec_create(call_frame_t * frame, xlator_t * this, uintptr_t target,
     fop->mode[0] = mode;
     fop->mode[1] = umask;
 
-    if (loc != NULL)
-    {
-        if (loc_copy(&fop->loc[0], loc) != 0)
-        {
-            gf_log(this->name, GF_LOG_ERROR, "Failed to copy a location.");
+    if (loc != NULL) {
+        if (loc_copy(&fop->loc[0], loc) != 0) {
+            gf_msg(this->name, GF_LOG_ERROR, ENOMEM, EC_MSG_LOC_COPY_FAIL,
+                   "Failed to copy a location.");
 
             goto out;
         }
     }
-    if (fd != NULL)
-    {
+    if (fd != NULL) {
         fop->fd = fd_ref(fd);
-        if (fop->fd == NULL)
-        {
-            gf_log(this->name, GF_LOG_ERROR, "Failed to reference a "
-                                             "file descriptor.");
+        if (fop->fd == NULL) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_FILE_DESC_REF_FAIL,
+                   "Failed to reference a "
+                   "file descriptor.");
 
             goto out;
         }
     }
-    if (xdata != NULL)
-    {
-        fop->xdata = dict_ref(xdata);
-        if (fop->xdata == NULL)
-        {
-            gf_log(this->name, GF_LOG_ERROR, "Failed to reference a "
-                                             "dictionary.");
+    if (xdata != NULL) {
+        fop->xdata = dict_copy_with_ref(xdata, NULL);
+        if (fop->xdata == NULL) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_DICT_REF_FAIL,
+                   "Failed to reference a "
+                   "dictionary.");
 
             goto out;
         }
@@ -336,28 +315,26 @@ void ec_create(call_frame_t * frame, xlator_t * this, uintptr_t target,
     error = 0;
 
 out:
-    if (fop != NULL)
-    {
+    if (fop != NULL) {
         ec_manager(fop, error);
-    }
-    else
-    {
-        func(frame, NULL, this, -1, EIO, NULL, NULL, NULL, NULL, NULL, NULL);
+    } else {
+        func(frame, NULL, this, -1, error, NULL, NULL, NULL, NULL, NULL, NULL);
     }
 }
 
 /* FOP: link */
 
-int32_t ec_link_cbk(call_frame_t * frame, void * cookie, xlator_t * this,
-                    int32_t op_ret, int32_t op_errno, inode_t * inode,
-                    struct iatt * buf, struct iatt * preparent,
-                    struct iatt * postparent, dict_t * xdata)
+int32_t
+ec_link_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int32_t op_ret,
+            int32_t op_errno, inode_t *inode, struct iatt *buf,
+            struct iatt *preparent, struct iatt *postparent, dict_t *xdata)
 {
-        return ec_dir_write_cbk (frame, this, cookie, op_ret, op_errno,
-                                 buf, preparent, postparent, NULL, NULL, xdata);
+    return ec_dir_write_cbk(frame, this, cookie, op_ret, op_errno, buf,
+                            preparent, postparent, NULL, NULL, xdata);
 }
 
-void ec_wind_link(ec_t * ec, ec_fop_data_t * fop, int32_t idx)
+void
+ec_wind_link(ec_t *ec, ec_fop_data_t *fop, int32_t idx)
 {
     ec_trace("WIND", fop, "idx=%d", idx);
 
@@ -366,25 +343,19 @@ void ec_wind_link(ec_t * ec, ec_fop_data_t * fop, int32_t idx)
                       &fop->loc[0], &fop->loc[1], fop->xdata);
 }
 
-int32_t ec_manager_link(ec_fop_data_t * fop, int32_t state)
+int32_t
+ec_manager_link(ec_fop_data_t *fop, int32_t state)
 {
-    ec_cbk_data_t * cbk;
+    ec_cbk_data_t *cbk;
 
-    switch (state)
-    {
+    switch (state) {
         case EC_STATE_INIT:
         case EC_STATE_LOCK:
-            // Parent entry of fop->loc[0] should be locked, but I don't
-            // receive enough information to do it (fop->loc[0].parent is
-            // NULL).
-            ec_lock_prepare_entry(fop, &fop->loc[1], 1);
+            ec_lock_prepare_parent_inode(
+                fop, &fop->loc[1], &fop->loc[0],
+                EC_UPDATE_DATA | EC_UPDATE_META | EC_INODE_SIZE);
             ec_lock(fop);
 
-            return EC_STATE_GET_SIZE_AND_VERSION;
-
-        case EC_STATE_GET_SIZE_AND_VERSION:
-            ec_get_size_version(fop);
-
             return EC_STATE_DISPATCH;
 
         case EC_STATE_DISPATCH:
@@ -393,37 +364,19 @@ int32_t ec_manager_link(ec_fop_data_t * fop, int32_t state)
             return EC_STATE_PREPARE_ANSWER;
 
         case EC_STATE_PREPARE_ANSWER:
-            cbk = fop->answer;
-            if (cbk != NULL)
-            {
-                if (!ec_dict_combine(cbk, EC_COMBINE_XDATA))
-                {
-                    if (cbk->op_ret >= 0)
-                    {
-                        cbk->op_ret = -1;
-                        cbk->op_errno = EIO;
-                    }
-                }
-                if (cbk->op_ret >= 0) {
-                    ec_iatt_rebuild(fop->xl->private, cbk->iatt, 3,
-                                    cbk->count);
-                    if (cbk->iatt[0].ia_type == IA_IFREG) {
-                        cbk->iatt[0].ia_size = fop->pre_size;
-                    }
+            cbk = ec_fop_prepare_answer(fop, _gf_false);
+            if (cbk != NULL) {
+                int32_t err;
 
-                    if (ec_loc_update(fop->xl, &fop->loc[0], cbk->inode,
-                                      &cbk->iatt[0]) != 0) {
-                        cbk->op_ret = -1;
-                        cbk->op_errno = EIO;
-                    }
-                }
-                if (cbk->op_ret < 0) {
-                    ec_fop_set_error(fop, cbk->op_errno);
+                ec_iatt_rebuild(fop->xl->private, cbk->iatt, 3, cbk->count);
+
+                if (cbk->iatt[0].ia_type == IA_IFREG) {
+                    cbk->iatt[0].ia_size = fop->locks[0].size;
                 }
-            }
-            else
-            {
-                ec_fop_set_error(fop, EIO);
+
+                err = ec_loc_update(fop->xl, &fop->loc[0], cbk->inode,
+                                    &cbk->iatt[0]);
+                ec_cbk_set_error(cbk, -err, _gf_false);
             }
 
             return EC_STATE_REPORT;
@@ -433,25 +386,23 @@ int32_t ec_manager_link(ec_fop_data_t * fop, int32_t state)
 
             GF_ASSERT(cbk != NULL);
 
-            if (fop->cbks.link != NULL)
-            {
-                fop->cbks.link(fop->req_frame, fop, fop->xl, cbk->op_ret,
-                               cbk->op_errno, fop->loc[0].inode, &cbk->iatt[0],
-                               &cbk->iatt[1], &cbk->iatt[2], cbk->xdata);
+            if (fop->cbks.link != NULL) {
+                QUORUM_CBK(fop->cbks.link, fop, fop->req_frame, fop, fop->xl,
+                           cbk->op_ret, cbk->op_errno, fop->loc[0].inode,
+                           &cbk->iatt[0], &cbk->iatt[1], &cbk->iatt[2],
+                           cbk->xdata);
             }
 
             return EC_STATE_LOCK_REUSE;
 
         case -EC_STATE_INIT:
         case -EC_STATE_LOCK:
-        case -EC_STATE_GET_SIZE_AND_VERSION:
         case -EC_STATE_DISPATCH:
         case -EC_STATE_PREPARE_ANSWER:
         case -EC_STATE_REPORT:
             GF_ASSERT(fop->error != 0);
 
-            if (fop->cbks.link != NULL)
-            {
+            if (fop->cbks.link != NULL) {
                 fop->cbks.link(fop->req_frame, fop, fop->xl, -1, fop->error,
                                NULL, NULL, NULL, NULL, NULL);
             }
@@ -471,59 +422,56 @@ int32_t ec_manager_link(ec_fop_data_t * fop, int32_t state)
             return EC_STATE_END;
 
         default:
-            gf_log(fop->xl->name, GF_LOG_ERROR, "Unhandled state %d for %s",
-                   state, ec_fop_name(fop->id));
+            gf_msg(fop->xl->name, GF_LOG_ERROR, EINVAL, EC_MSG_UNHANDLED_STATE,
+                   "Unhandled state %d for %s", state, ec_fop_name(fop->id));
 
             return EC_STATE_END;
     }
 }
 
-void ec_link(call_frame_t * frame, xlator_t * this, uintptr_t target,
-             int32_t minimum, fop_link_cbk_t func, void * data, loc_t * oldloc,
-             loc_t * newloc, dict_t * xdata)
+void
+ec_link(call_frame_t *frame, xlator_t *this, uintptr_t target,
+        uint32_t fop_flags, fop_link_cbk_t func, void *data, loc_t *oldloc,
+        loc_t *newloc, dict_t *xdata)
 {
-    ec_cbk_t callback = { .link = func };
-    ec_fop_data_t * fop = NULL;
-    int32_t error = EIO;
+    ec_cbk_t callback = {.link = func};
+    ec_fop_data_t *fop = NULL;
+    int32_t error = ENOMEM;
 
-    gf_log("ec", GF_LOG_TRACE, "EC(LINK) %p", frame);
+    gf_msg_trace("ec", 0, "EC(LINK) %p", frame);
 
     VALIDATE_OR_GOTO(this, out);
     GF_VALIDATE_OR_GOTO(this->name, frame, out);
     GF_VALIDATE_OR_GOTO(this->name, this->private, out);
 
-    fop = ec_fop_data_allocate(frame, this, GF_FOP_LINK, 0, target, minimum,
+    fop = ec_fop_data_allocate(frame, this, GF_FOP_LINK, 0, target, fop_flags,
                                ec_wind_link, ec_manager_link, callback, data);
-    if (fop == NULL)
-    {
+    if (fop == NULL) {
         goto out;
     }
 
-    if (oldloc != NULL)
-    {
-        if (loc_copy(&fop->loc[0], oldloc) != 0)
-        {
-            gf_log(this->name, GF_LOG_ERROR, "Failed to copy a location.");
+    if (oldloc != NULL) {
+        if (loc_copy(&fop->loc[0], oldloc) != 0) {
+            gf_msg(this->name, GF_LOG_ERROR, ENOMEM, EC_MSG_LOC_COPY_FAIL,
+                   "Failed to copy a location.");
 
             goto out;
         }
     }
-    if (newloc != NULL)
-    {
-        if (loc_copy(&fop->loc[1], newloc) != 0)
-        {
-            gf_log(this->name, GF_LOG_ERROR, "Failed to copy a location.");
+    if (newloc != NULL) {
+        if (loc_copy(&fop->loc[1], newloc) != 0) {
+            gf_msg(this->name, GF_LOG_ERROR, ENOMEM, EC_MSG_LOC_COPY_FAIL,
+                   "Failed to copy a location.");
 
             goto out;
         }
     }
-    if (xdata != NULL)
-    {
-        fop->xdata = dict_ref(xdata);
-        if (fop->xdata == NULL)
-        {
-            gf_log(this->name, GF_LOG_ERROR, "Failed to reference a "
-                                             "dictionary.");
+    if (xdata != NULL) {
+        fop->xdata = dict_copy_with_ref(xdata, NULL);
+        if (fop->xdata == NULL) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_DICT_REF_FAIL,
+                   "Failed to reference a "
+                   "dictionary.");
 
             goto out;
         }
@@ -532,28 +480,26 @@ void ec_link(call_frame_t * frame, xlator_t * this, uintptr_t target,
     error = 0;
 
 out:
-    if (fop != NULL)
-    {
+    if (fop != NULL) {
         ec_manager(fop, error);
-    }
-    else
-    {
-        func(frame, NULL, this, -1, EIO, NULL, NULL, NULL, NULL, NULL);
+    } else {
+        func(frame, NULL, this, -1, error, NULL, NULL, NULL, NULL, NULL);
     }
 }
 
 /* FOP: mkdir */
 
-int32_t ec_mkdir_cbk(call_frame_t * frame, void * cookie, xlator_t * this,
-                     int32_t op_ret, int32_t op_errno, inode_t * inode,
-                     struct iatt * buf, struct iatt * preparent,
-                     struct iatt * postparent, dict_t * xdata)
+int32_t
+ec_mkdir_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int32_t op_ret,
+             int32_t op_errno, inode_t *inode, struct iatt *buf,
+             struct iatt *preparent, struct iatt *postparent, dict_t *xdata)
 {
-        return ec_dir_write_cbk (frame, this, cookie, op_ret, op_errno,
-                                 buf, preparent, postparent, NULL, NULL, xdata);
+    return ec_dir_write_cbk(frame, this, cookie, op_ret, op_errno, buf,
+                            preparent, postparent, NULL, NULL, xdata);
 }
 
-void ec_wind_mkdir(ec_t * ec, ec_fop_data_t * fop, int32_t idx)
+void
+ec_wind_mkdir(ec_t *ec, ec_fop_data_t *fop, int32_t idx)
 {
     ec_trace("WIND", fop, "idx=%d", idx);
 
@@ -562,32 +508,36 @@ void ec_wind_mkdir(ec_t * ec, ec_fop_data_t * fop, int32_t idx)
                       &fop->loc[0], fop->mode[0], fop->mode[1], fop->xdata);
 }
 
-int32_t ec_manager_mkdir(ec_fop_data_t * fop, int32_t state)
+int32_t
+ec_manager_mkdir(ec_fop_data_t *fop, int32_t state)
 {
-    ec_cbk_data_t * cbk;
+    ec_cbk_data_t *cbk;
+    uint64_t version[2] = {0, 0};
+    int32_t err;
 
-    switch (state)
-    {
+    switch (state) {
         case EC_STATE_INIT:
             if (fop->xdata == NULL) {
                 fop->xdata = dict_new();
                 if (fop->xdata == NULL) {
-                    fop->error = EIO;
+                    fop->error = ENOMEM;
 
                     return EC_STATE_REPORT;
                 }
             }
 
-            if (ec_dict_set_number(fop->xdata, EC_XATTR_VERSION, 0) != 0) {
-                fop->error = EIO;
-
+            err = ec_dict_set_array(fop->xdata, EC_XATTR_VERSION, version,
+                                    EC_VERSION_SIZE);
+            if (err != 0) {
+                fop->error = -err;
                 return EC_STATE_REPORT;
             }
 
-        /* Fall through */
+            /* Fall through */
 
         case EC_STATE_LOCK:
-            ec_lock_prepare_entry(fop, &fop->loc[0], 1);
+            ec_lock_prepare_parent_inode(fop, &fop->loc[0], NULL,
+                                         EC_UPDATE_DATA | EC_UPDATE_META);
             ec_lock(fop);
 
             return EC_STATE_DISPATCH;
@@ -598,34 +548,15 @@ int32_t ec_manager_mkdir(ec_fop_data_t * fop, int32_t state)
             return EC_STATE_PREPARE_ANSWER;
 
         case EC_STATE_PREPARE_ANSWER:
-            cbk = fop->answer;
-            if (cbk != NULL)
-            {
-                if (!ec_dict_combine(cbk, EC_COMBINE_XDATA))
-                {
-                    if (cbk->op_ret >= 0)
-                    {
-                        cbk->op_ret = -1;
-                        cbk->op_errno = EIO;
-                    }
-                }
-                if (cbk->op_ret >= 0) {
-                    ec_iatt_rebuild(fop->xl->private, cbk->iatt, 3,
-                                    cbk->count);
-
-                    if (ec_loc_update(fop->xl, &fop->loc[0], cbk->inode,
-                                      &cbk->iatt[0]) != 0) {
-                        cbk->op_ret = -1;
-                        cbk->op_errno = EIO;
-                    }
-                }
-                if (cbk->op_ret < 0) {
-                    ec_fop_set_error(fop, cbk->op_errno);
-                }
-            }
-            else
-            {
-                ec_fop_set_error(fop, EIO);
+            cbk = ec_fop_prepare_answer(fop, _gf_false);
+            if (cbk != NULL) {
+                int32_t err;
+
+                ec_iatt_rebuild(fop->xl->private, cbk->iatt, 3, cbk->count);
+
+                err = ec_loc_update(fop->xl, &fop->loc[0], cbk->inode,
+                                    &cbk->iatt[0]);
+                ec_cbk_set_error(cbk, -err, _gf_false);
             }
 
             return EC_STATE_REPORT;
@@ -635,11 +566,11 @@ int32_t ec_manager_mkdir(ec_fop_data_t * fop, int32_t state)
 
             GF_ASSERT(cbk != NULL);
 
-            if (fop->cbks.mkdir != NULL)
-            {
-                fop->cbks.mkdir(fop->req_frame, fop, fop->xl, cbk->op_ret,
-                                cbk->op_errno, fop->loc[0].inode, &cbk->iatt[0],
-                                &cbk->iatt[1], &cbk->iatt[2], cbk->xdata);
+            if (fop->cbks.mkdir != NULL) {
+                QUORUM_CBK(fop->cbks.mkdir, fop, fop->req_frame, fop, fop->xl,
+                           cbk->op_ret, cbk->op_errno, fop->loc[0].inode,
+                           &cbk->iatt[0], &cbk->iatt[1], &cbk->iatt[2],
+                           cbk->xdata);
             }
 
             return EC_STATE_LOCK_REUSE;
@@ -649,12 +580,13 @@ int32_t ec_manager_mkdir(ec_fop_data_t * fop, int32_t state)
         case -EC_STATE_DISPATCH:
         case -EC_STATE_PREPARE_ANSWER:
         case -EC_STATE_REPORT:
+            cbk = fop->answer;
             GF_ASSERT(fop->error != 0);
 
-            if (fop->cbks.mkdir != NULL)
-            {
+            if (fop->cbks.mkdir != NULL) {
                 fop->cbks.mkdir(fop->req_frame, fop, fop->xl, -1, fop->error,
-                                NULL, NULL, NULL, NULL, NULL);
+                                NULL, NULL, NULL, NULL,
+                                ((cbk) ? cbk->xdata : NULL));
             }
 
             return EC_STATE_LOCK_REUSE;
@@ -672,55 +604,51 @@ int32_t ec_manager_mkdir(ec_fop_data_t * fop, int32_t state)
             return EC_STATE_END;
 
         default:
-            gf_log(fop->xl->name, GF_LOG_ERROR, "Unhandled state %d for %s",
-                   state, ec_fop_name(fop->id));
+            gf_msg(fop->xl->name, GF_LOG_ERROR, EINVAL, EC_MSG_UNHANDLED_STATE,
+                   "Unhandled state %d for %s", state, ec_fop_name(fop->id));
 
             return EC_STATE_END;
     }
 }
 
-void ec_mkdir(call_frame_t * frame, xlator_t * this, uintptr_t target,
-              int32_t minimum, fop_mkdir_cbk_t func, void * data, loc_t * loc,
-              mode_t mode, mode_t umask, dict_t * xdata)
+void
+ec_mkdir(call_frame_t *frame, xlator_t *this, uintptr_t target,
+         uint32_t fop_flags, fop_mkdir_cbk_t func, void *data, loc_t *loc,
+         mode_t mode, mode_t umask, dict_t *xdata)
 {
-    ec_cbk_t callback = { .mkdir = func };
-    ec_fop_data_t * fop = NULL;
-    int32_t error = EIO;
+    ec_cbk_t callback = {.mkdir = func};
+    ec_fop_data_t *fop = NULL;
+    int32_t error = ENOMEM;
 
-    gf_log("ec", GF_LOG_TRACE, "EC(MKDIR) %p", frame);
+    gf_msg_trace("ec", 0, "EC(MKDIR) %p", frame);
 
     VALIDATE_OR_GOTO(this, out);
     GF_VALIDATE_OR_GOTO(this->name, frame, out);
     GF_VALIDATE_OR_GOTO(this->name, this->private, out);
 
-    fop = ec_fop_data_allocate(frame, this, GF_FOP_MKDIR,
-                               EC_FLAG_UPDATE_LOC_PARENT, target, minimum,
-                               ec_wind_mkdir, ec_manager_mkdir, callback,
-                               data);
-    if (fop == NULL)
-    {
+    fop = ec_fop_data_allocate(frame, this, GF_FOP_MKDIR, 0, target, fop_flags,
+                               ec_wind_mkdir, ec_manager_mkdir, callback, data);
+    if (fop == NULL) {
         goto out;
     }
 
     fop->mode[0] = mode;
     fop->mode[1] = umask;
 
-    if (loc != NULL)
-    {
-        if (loc_copy(&fop->loc[0], loc) != 0)
-        {
-            gf_log(this->name, GF_LOG_ERROR, "Failed to copy a location.");
+    if (loc != NULL) {
+        if (loc_copy(&fop->loc[0], loc) != 0) {
+            gf_msg(this->name, GF_LOG_ERROR, ENOMEM, EC_MSG_LOC_COPY_FAIL,
+                   "Failed to copy a location.");
 
             goto out;
         }
     }
-    if (xdata != NULL)
-    {
-        fop->xdata = dict_ref(xdata);
-        if (fop->xdata == NULL)
-        {
-            gf_log(this->name, GF_LOG_ERROR, "Failed to reference a "
-                                             "dictionary.");
+    if (xdata != NULL) {
+        fop->xdata = dict_copy_with_ref(xdata, NULL);
+        if (fop->xdata == NULL) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_DICT_REF_FAIL,
+                   "Failed to reference a "
+                   "dictionary.");
 
             goto out;
         }
@@ -729,28 +657,26 @@ void ec_mkdir(call_frame_t * frame, xlator_t * this, uintptr_t target,
     error = 0;
 
 out:
-    if (fop != NULL)
-    {
+    if (fop != NULL) {
         ec_manager(fop, error);
-    }
-    else
-    {
-        func(frame, NULL, this, -1, EIO, NULL, NULL, NULL, NULL, NULL);
+    } else {
+        func(frame, NULL, this, -1, error, NULL, NULL, NULL, NULL, NULL);
     }
 }
 
 /* FOP: mknod */
 
-int32_t ec_mknod_cbk(call_frame_t * frame, void * cookie, xlator_t * this,
-                     int32_t op_ret, int32_t op_errno, inode_t * inode,
-                     struct iatt * buf, struct iatt * preparent,
-                     struct iatt * postparent, dict_t * xdata)
+int32_t
+ec_mknod_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int32_t op_ret,
+             int32_t op_errno, inode_t *inode, struct iatt *buf,
+             struct iatt *preparent, struct iatt *postparent, dict_t *xdata)
 {
-        return ec_dir_write_cbk (frame, this, cookie, op_ret, op_errno,
-                                 buf, preparent, postparent, NULL, NULL, xdata);
+    return ec_dir_write_cbk(frame, this, cookie, op_ret, op_errno, buf,
+                            preparent, postparent, NULL, NULL, xdata);
 }
 
-void ec_wind_mknod(ec_t * ec, ec_fop_data_t * fop, int32_t idx)
+void
+ec_wind_mknod(ec_t *ec, ec_fop_data_t *fop, int32_t idx)
 {
     ec_trace("WIND", fop, "idx=%d", idx);
 
@@ -760,19 +686,23 @@ void ec_wind_mknod(ec_t * ec, ec_fop_data_t * fop, int32_t idx)
                       fop->xdata);
 }
 
-int32_t ec_manager_mknod(ec_fop_data_t * fop, int32_t state)
+int32_t
+ec_manager_mknod(ec_fop_data_t *fop, int32_t state)
 {
+    ec_config_t config;
     ec_t *ec;
-    ec_cbk_data_t * cbk;
+    ec_cbk_data_t *cbk;
+    uint64_t version[2] = {0, 0};
 
-    switch (state)
-    {
+    switch (state) {
         case EC_STATE_INIT:
             if (S_ISREG(fop->mode[0])) {
+                int32_t err;
+
                 if (fop->xdata == NULL) {
                     fop->xdata = dict_new();
                     if (fop->xdata == NULL) {
-                        fop->error = EIO;
+                        fop->error = ENOMEM;
 
                         return EC_STATE_REPORT;
                     }
@@ -780,37 +710,39 @@ int32_t ec_manager_mknod(ec_fop_data_t * fop, int32_t state)
 
                 ec = fop->xl->private;
 
-                fop->config.version = EC_CONFIG_VERSION;
-                fop->config.algorithm = EC_CONFIG_ALGORITHM;
-                fop->config.gf_word_size = EC_GF_BITS;
-                fop->config.bricks = ec->nodes;
-                fop->config.redundancy = ec->redundancy;
-                fop->config.chunk_size = EC_METHOD_CHUNK_SIZE;
+                config.version = EC_CONFIG_VERSION;
+                config.algorithm = EC_CONFIG_ALGORITHM;
+                config.gf_word_size = EC_GF_BITS;
+                config.bricks = ec->nodes;
+                config.redundancy = ec->redundancy;
+                config.chunk_size = EC_METHOD_CHUNK_SIZE;
 
-                if (ec_dict_set_config(fop->xdata, EC_XATTR_CONFIG,
-                                       &fop->config) < 0) {
-                    fop->error = EIO;
+                err = ec_dict_set_config(fop->xdata, EC_XATTR_CONFIG, &config);
+                if (err != 0) {
+                    fop->error = -err;
 
                     return EC_STATE_REPORT;
                 }
-
-                if (ec_dict_set_number(fop->xdata, EC_XATTR_VERSION, 0) != 0) {
-                    fop->error = EIO;
+                err = ec_dict_set_array(fop->xdata, EC_XATTR_VERSION, version,
+                                        EC_VERSION_SIZE);
+                if (err != 0) {
+                    fop->error = -err;
 
                     return EC_STATE_REPORT;
                 }
-
-                if (ec_dict_set_number(fop->xdata, EC_XATTR_SIZE, 0) != 0) {
-                    fop->error = EIO;
+                err = ec_dict_set_number(fop->xdata, EC_XATTR_SIZE, 0);
+                if (err != 0) {
+                    fop->error = -err;
 
                     return EC_STATE_REPORT;
                 }
             }
 
-        /* Fall through */
+            /* Fall through */
 
         case EC_STATE_LOCK:
-            ec_lock_prepare_entry(fop, &fop->loc[0], 1);
+            ec_lock_prepare_parent_inode(fop, &fop->loc[0], NULL,
+                                         EC_UPDATE_DATA | EC_UPDATE_META);
             ec_lock(fop);
 
             return EC_STATE_DISPATCH;
@@ -821,34 +753,15 @@ int32_t ec_manager_mknod(ec_fop_data_t * fop, int32_t state)
             return EC_STATE_PREPARE_ANSWER;
 
         case EC_STATE_PREPARE_ANSWER:
-            cbk = fop->answer;
-            if (cbk != NULL)
-            {
-                if (!ec_dict_combine(cbk, EC_COMBINE_XDATA))
-                {
-                    if (cbk->op_ret >= 0)
-                    {
-                        cbk->op_ret = -1;
-                        cbk->op_errno = EIO;
-                    }
-                }
-                if (cbk->op_ret >= 0) {
-                    ec_iatt_rebuild(fop->xl->private, cbk->iatt, 3,
-                                    cbk->count);
-
-                    if (ec_loc_update(fop->xl, &fop->loc[0], cbk->inode,
-                                      &cbk->iatt[0]) != 0) {
-                        cbk->op_ret = -1;
-                        cbk->op_errno = EIO;
-                    }
-                }
-                if (cbk->op_ret < 0) {
-                    ec_fop_set_error(fop, cbk->op_errno);
-                }
-            }
-            else
-            {
-                ec_fop_set_error(fop, EIO);
+            cbk = ec_fop_prepare_answer(fop, _gf_false);
+            if (cbk != NULL) {
+                int32_t err;
+
+                ec_iatt_rebuild(fop->xl->private, cbk->iatt, 3, cbk->count);
+
+                err = ec_loc_update(fop->xl, &fop->loc[0], cbk->inode,
+                                    &cbk->iatt[0]);
+                ec_cbk_set_error(cbk, -err, _gf_false);
             }
 
             return EC_STATE_REPORT;
@@ -858,11 +771,11 @@ int32_t ec_manager_mknod(ec_fop_data_t * fop, int32_t state)
 
             GF_ASSERT(cbk != NULL);
 
-            if (fop->cbks.mknod != NULL)
-            {
-                fop->cbks.mknod(fop->req_frame, fop, fop->xl, cbk->op_ret,
-                                cbk->op_errno, fop->loc[0].inode, &cbk->iatt[0],
-                                &cbk->iatt[1], &cbk->iatt[2], cbk->xdata);
+            if (fop->cbks.mknod != NULL) {
+                QUORUM_CBK(fop->cbks.mknod, fop, fop->req_frame, fop, fop->xl,
+                           cbk->op_ret, cbk->op_errno, fop->loc[0].inode,
+                           &cbk->iatt[0], &cbk->iatt[1], &cbk->iatt[2],
+                           cbk->xdata);
             }
 
             return EC_STATE_LOCK_REUSE;
@@ -874,8 +787,7 @@ int32_t ec_manager_mknod(ec_fop_data_t * fop, int32_t state)
         case -EC_STATE_REPORT:
             GF_ASSERT(fop->error != 0);
 
-            if (fop->cbks.mknod != NULL)
-            {
+            if (fop->cbks.mknod != NULL) {
                 fop->cbks.mknod(fop->req_frame, fop, fop->xl, -1, fop->error,
                                 NULL, NULL, NULL, NULL, NULL);
             }
@@ -895,33 +807,31 @@ int32_t ec_manager_mknod(ec_fop_data_t * fop, int32_t state)
             return EC_STATE_END;
 
         default:
-            gf_log(fop->xl->name, GF_LOG_ERROR, "Unhandled state %d for %s",
-                   state, ec_fop_name(fop->id));
+            gf_msg(fop->xl->name, GF_LOG_ERROR, EINVAL, EC_MSG_UNHANDLED_STATE,
+                   "Unhandled state %d for %s", state, ec_fop_name(fop->id));
 
             return EC_STATE_END;
     }
 }
 
-void ec_mknod(call_frame_t * frame, xlator_t * this, uintptr_t target,
-              int32_t minimum, fop_mknod_cbk_t func, void * data, loc_t * loc,
-              mode_t mode, dev_t rdev, mode_t umask, dict_t * xdata)
+void
+ec_mknod(call_frame_t *frame, xlator_t *this, uintptr_t target,
+         uint32_t fop_flags, fop_mknod_cbk_t func, void *data, loc_t *loc,
+         mode_t mode, dev_t rdev, mode_t umask, dict_t *xdata)
 {
-    ec_cbk_t callback = { .mknod = func };
-    ec_fop_data_t * fop = NULL;
-    int32_t error = EIO;
+    ec_cbk_t callback = {.mknod = func};
+    ec_fop_data_t *fop = NULL;
+    int32_t error = ENOMEM;
 
-    gf_log("ec", GF_LOG_TRACE, "EC(MKNOD) %p", frame);
+    gf_msg_trace("ec", 0, "EC(MKNOD) %p", frame);
 
     VALIDATE_OR_GOTO(this, out);
     GF_VALIDATE_OR_GOTO(this->name, frame, out);
     GF_VALIDATE_OR_GOTO(this->name, this->private, out);
 
-    fop = ec_fop_data_allocate(frame, this, GF_FOP_MKNOD,
-                               EC_FLAG_UPDATE_LOC_PARENT, target, minimum,
-                               ec_wind_mknod, ec_manager_mknod, callback,
-                               data);
-    if (fop == NULL)
-    {
+    fop = ec_fop_data_allocate(frame, this, GF_FOP_MKNOD, 0, target, fop_flags,
+                               ec_wind_mknod, ec_manager_mknod, callback, data);
+    if (fop == NULL) {
         goto out;
     }
 
@@ -929,22 +839,20 @@ void ec_mknod(call_frame_t * frame, xlator_t * this, uintptr_t target,
     fop->dev = rdev;
     fop->mode[1] = umask;
 
-    if (loc != NULL)
-    {
-        if (loc_copy(&fop->loc[0], loc) != 0)
-        {
-            gf_log(this->name, GF_LOG_ERROR, "Failed to copy a location.");
+    if (loc != NULL) {
+        if (loc_copy(&fop->loc[0], loc) != 0) {
+            gf_msg(this->name, GF_LOG_ERROR, ENOMEM, EC_MSG_LOC_COPY_FAIL,
+                   "Failed to copy a location.");
 
             goto out;
         }
     }
-    if (xdata != NULL)
-    {
-        fop->xdata = dict_ref(xdata);
-        if (fop->xdata == NULL)
-        {
-            gf_log(this->name, GF_LOG_ERROR, "Failed to reference a "
-                                             "dictionary.");
+    if (xdata != NULL) {
+        fop->xdata = dict_copy_with_ref(xdata, NULL);
+        if (fop->xdata == NULL) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_DICT_REF_FAIL,
+                   "Failed to reference a "
+                   "dictionary.");
 
             goto out;
         }
@@ -953,30 +861,28 @@ void ec_mknod(call_frame_t * frame, xlator_t * this, uintptr_t target,
     error = 0;
 
 out:
-    if (fop != NULL)
-    {
+    if (fop != NULL) {
         ec_manager(fop, error);
-    }
-    else
-    {
-        func(frame, NULL, this, -1, EIO, NULL, NULL, NULL, NULL, NULL);
+    } else {
+        func(frame, NULL, this, -1, error, NULL, NULL, NULL, NULL, NULL);
     }
 }
 
 /* FOP: rename */
 
-int32_t ec_rename_cbk(call_frame_t * frame, void * cookie, xlator_t * this,
-                      int32_t op_ret, int32_t op_errno, struct iatt * buf,
-                      struct iatt * preoldparent, struct iatt * postoldparent,
-                      struct iatt * prenewparent, struct iatt * postnewparent,
-                      dict_t * xdata)
+int32_t
+ec_rename_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int32_t op_ret,
+              int32_t op_errno, struct iatt *buf, struct iatt *preoldparent,
+              struct iatt *postoldparent, struct iatt *prenewparent,
+              struct iatt *postnewparent, dict_t *xdata)
 {
-        return ec_dir_write_cbk (frame, this, cookie, op_ret, op_errno,
-                                 buf, preoldparent, postoldparent, prenewparent,
-                                 postnewparent, xdata);
+    return ec_dir_write_cbk(frame, this, cookie, op_ret, op_errno, buf,
+                            preoldparent, postoldparent, prenewparent,
+                            postnewparent, xdata);
 }
 
-void ec_wind_rename(ec_t * ec, ec_fop_data_t * fop, int32_t idx)
+void
+ec_wind_rename(ec_t *ec, ec_fop_data_t *fop, int32_t idx)
 {
     ec_trace("WIND", fop, "idx=%d", idx);
 
@@ -985,23 +891,21 @@ void ec_wind_rename(ec_t * ec, ec_fop_data_t * fop, int32_t idx)
                       &fop->loc[0], &fop->loc[1], fop->xdata);
 }
 
-int32_t ec_manager_rename(ec_fop_data_t * fop, int32_t state)
+int32_t
+ec_manager_rename(ec_fop_data_t *fop, int32_t state)
 {
-    ec_cbk_data_t * cbk;
+    ec_cbk_data_t *cbk;
 
-    switch (state)
-    {
+    switch (state) {
         case EC_STATE_INIT:
         case EC_STATE_LOCK:
-            ec_lock_prepare_entry(fop, &fop->loc[0], 1);
-            ec_lock_prepare_entry(fop, &fop->loc[1], 1);
+            ec_lock_prepare_parent_inode(
+                fop, &fop->loc[0], &fop->loc[0],
+                EC_UPDATE_DATA | EC_UPDATE_META | EC_INODE_SIZE);
+            ec_lock_prepare_parent_inode(fop, &fop->loc[1], NULL,
+                                         EC_UPDATE_DATA | EC_UPDATE_META);
             ec_lock(fop);
 
-            return EC_STATE_GET_SIZE_AND_VERSION;
-
-        case EC_STATE_GET_SIZE_AND_VERSION:
-            ec_get_size_version(fop);
-
             return EC_STATE_DISPATCH;
 
         case EC_STATE_DISPATCH:
@@ -1010,36 +914,14 @@ int32_t ec_manager_rename(ec_fop_data_t * fop, int32_t state)
             return EC_STATE_PREPARE_ANSWER;
 
         case EC_STATE_PREPARE_ANSWER:
-            cbk = fop->answer;
-            if (cbk != NULL)
-            {
-                if (!ec_dict_combine(cbk, EC_COMBINE_XDATA))
-                {
-                    if (cbk->op_ret >= 0)
-                    {
-                        cbk->op_ret = -1;
-                        cbk->op_errno = EIO;
-                    }
-                }
-                if (cbk->op_ret < 0)
-                {
-                    ec_fop_set_error(fop, cbk->op_errno);
-                }
-                else
-                {
-                    ec_iatt_rebuild(fop->xl->private, cbk->iatt, 5,
-                                    cbk->count);
-
-                    if (cbk->iatt[0].ia_type == IA_IFREG)
-                    {
-                        cbk->iatt[0].ia_size = fop->pre_size;
-                    }
+            cbk = ec_fop_prepare_answer(fop, _gf_false);
+            if (cbk != NULL) {
+                ec_iatt_rebuild(fop->xl->private, cbk->iatt, 5, cbk->count);
+
+                if (cbk->iatt[0].ia_type == IA_IFREG) {
+                    cbk->iatt[0].ia_size = fop->locks[0].size;
                 }
             }
-            else
-            {
-                ec_fop_set_error(fop, EIO);
-            }
 
             return EC_STATE_REPORT;
 
@@ -1048,26 +930,23 @@ int32_t ec_manager_rename(ec_fop_data_t * fop, int32_t state)
 
             GF_ASSERT(cbk != NULL);
 
-            if (fop->cbks.rename != NULL)
-            {
-                fop->cbks.rename(fop->req_frame, fop, fop->xl, cbk->op_ret,
-                                 cbk->op_errno, &cbk->iatt[0], &cbk->iatt[1],
-                                 &cbk->iatt[2], &cbk->iatt[3], &cbk->iatt[4],
-                                 cbk->xdata);
+            if (fop->cbks.rename != NULL) {
+                QUORUM_CBK(fop->cbks.rename, fop, fop->req_frame, fop, fop->xl,
+                           cbk->op_ret, cbk->op_errno, &cbk->iatt[0],
+                           &cbk->iatt[1], &cbk->iatt[2], &cbk->iatt[3],
+                           &cbk->iatt[4], cbk->xdata);
             }
 
             return EC_STATE_LOCK_REUSE;
 
         case -EC_STATE_INIT:
         case -EC_STATE_LOCK:
-        case -EC_STATE_GET_SIZE_AND_VERSION:
         case -EC_STATE_DISPATCH:
         case -EC_STATE_PREPARE_ANSWER:
         case -EC_STATE_REPORT:
             GF_ASSERT(fop->error != 0);
 
-            if (fop->cbks.rename != NULL)
-            {
+            if (fop->cbks.rename != NULL) {
                 fop->cbks.rename(fop->req_frame, fop, fop->xl, -1, fop->error,
                                  NULL, NULL, NULL, NULL, NULL, NULL);
             }
@@ -1087,61 +966,57 @@ int32_t ec_manager_rename(ec_fop_data_t * fop, int32_t state)
             return EC_STATE_END;
 
         default:
-            gf_log(fop->xl->name, GF_LOG_ERROR, "Unhandled state %d for %s",
-                   state, ec_fop_name(fop->id));
+            gf_msg(fop->xl->name, GF_LOG_ERROR, EINVAL, EC_MSG_UNHANDLED_STATE,
+                   "Unhandled state %d for %s", state, ec_fop_name(fop->id));
 
             return EC_STATE_END;
     }
 }
 
-void ec_rename(call_frame_t * frame, xlator_t * this, uintptr_t target,
-               int32_t minimum, fop_rename_cbk_t func, void * data,
-               loc_t * oldloc, loc_t * newloc, dict_t * xdata)
+void
+ec_rename(call_frame_t *frame, xlator_t *this, uintptr_t target,
+          uint32_t fop_flags, fop_rename_cbk_t func, void *data, loc_t *oldloc,
+          loc_t *newloc, dict_t *xdata)
 {
-    ec_cbk_t callback = { .rename = func };
-    ec_fop_data_t * fop = NULL;
-    int32_t error = EIO;
+    ec_cbk_t callback = {.rename = func};
+    ec_fop_data_t *fop = NULL;
+    int32_t error = ENOMEM;
 
-    gf_log("ec", GF_LOG_TRACE, "EC(RENAME) %p", frame);
+    gf_msg_trace("ec", 0, "EC(RENAME) %p", frame);
 
     VALIDATE_OR_GOTO(this, out);
     GF_VALIDATE_OR_GOTO(this->name, frame, out);
     GF_VALIDATE_OR_GOTO(this->name, this->private, out);
 
-    fop = ec_fop_data_allocate(frame, this, GF_FOP_RENAME,
-                               EC_FLAG_UPDATE_LOC_PARENT, target, minimum,
+    fop = ec_fop_data_allocate(frame, this, GF_FOP_RENAME, 0, target, fop_flags,
                                ec_wind_rename, ec_manager_rename, callback,
                                data);
-    if (fop == NULL)
-    {
+    if (fop == NULL) {
         goto out;
     }
 
-    if (oldloc != NULL)
-    {
-        if (loc_copy(&fop->loc[0], oldloc) != 0)
-        {
-            gf_log(this->name, GF_LOG_ERROR, "Failed to copy a location.");
+    if (oldloc != NULL) {
+        if (loc_copy(&fop->loc[0], oldloc) != 0) {
+            gf_msg(this->name, GF_LOG_ERROR, ENOMEM, EC_MSG_LOC_COPY_FAIL,
+                   "Failed to copy a location.");
 
             goto out;
         }
     }
-    if (newloc != NULL)
-    {
-        if (loc_copy(&fop->loc[1], newloc) != 0)
-        {
-            gf_log(this->name, GF_LOG_ERROR, "Failed to copy a location.");
+    if (newloc != NULL) {
+        if (loc_copy(&fop->loc[1], newloc) != 0) {
+            gf_msg(this->name, GF_LOG_ERROR, ENOMEM, EC_MSG_LOC_COPY_FAIL,
+                   "Failed to copy a location.");
 
             goto out;
         }
     }
-    if (xdata != NULL)
-    {
-        fop->xdata = dict_ref(xdata);
-        if (fop->xdata == NULL)
-        {
-            gf_log(this->name, GF_LOG_ERROR, "Failed to reference a "
-                                             "dictionary.");
+    if (xdata != NULL) {
+        fop->xdata = dict_copy_with_ref(xdata, NULL);
+        if (fop->xdata == NULL) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_DICT_REF_FAIL,
+                   "Failed to reference a "
+                   "dictionary.");
 
             goto out;
         }
@@ -1150,27 +1025,26 @@ void ec_rename(call_frame_t * frame, xlator_t * this, uintptr_t target,
     error = 0;
 
 out:
-    if (fop != NULL)
-    {
+    if (fop != NULL) {
         ec_manager(fop, error);
-    }
-    else
-    {
-        func(frame, NULL, this, -1, EIO, NULL, NULL, NULL, NULL, NULL, NULL);
+    } else {
+        func(frame, NULL, this, -1, error, NULL, NULL, NULL, NULL, NULL, NULL);
     }
 }
 
 /* FOP: rmdir */
 
-int32_t ec_rmdir_cbk(call_frame_t * frame, void * cookie, xlator_t * this,
-                     int32_t op_ret, int32_t op_errno, struct iatt * preparent,
-                     struct iatt * postparent, dict_t * xdata)
+int32_t
+ec_rmdir_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int32_t op_ret,
+             int32_t op_errno, struct iatt *preparent, struct iatt *postparent,
+             dict_t *xdata)
 {
-        return ec_dir_write_cbk (frame, this, cookie, op_ret, op_errno, NULL,
-                                 preparent, postparent, NULL, NULL, xdata);
+    return ec_dir_write_cbk(frame, this, cookie, op_ret, op_errno, NULL,
+                            preparent, postparent, NULL, NULL, xdata);
 }
 
-void ec_wind_rmdir(ec_t * ec, ec_fop_data_t * fop, int32_t idx)
+void
+ec_wind_rmdir(ec_t *ec, ec_fop_data_t *fop, int32_t idx)
 {
     ec_trace("WIND", fop, "idx=%d", idx);
 
@@ -1179,15 +1053,16 @@ void ec_wind_rmdir(ec_t * ec, ec_fop_data_t * fop, int32_t idx)
                       &fop->loc[0], fop->int32, fop->xdata);
 }
 
-int32_t ec_manager_rmdir(ec_fop_data_t * fop, int32_t state)
+int32_t
+ec_manager_rmdir(ec_fop_data_t *fop, int32_t state)
 {
-    ec_cbk_data_t * cbk;
+    ec_cbk_data_t *cbk;
 
-    switch (state)
-    {
+    switch (state) {
         case EC_STATE_INIT:
         case EC_STATE_LOCK:
-            ec_lock_prepare_entry(fop, &fop->loc[0], 1);
+            ec_lock_prepare_parent_inode(fop, &fop->loc[0], NULL,
+                                         EC_UPDATE_DATA | EC_UPDATE_META);
             ec_lock(fop);
 
             return EC_STATE_DISPATCH;
@@ -1198,26 +1073,7 @@ int32_t ec_manager_rmdir(ec_fop_data_t * fop, int32_t state)
             return EC_STATE_PREPARE_ANSWER;
 
         case EC_STATE_PREPARE_ANSWER:
-            cbk = fop->answer;
-            if (cbk != NULL)
-            {
-                if (!ec_dict_combine(cbk, EC_COMBINE_XDATA))
-                {
-                    if (cbk->op_ret >= 0)
-                    {
-                        cbk->op_ret = -1;
-                        cbk->op_errno = EIO;
-                    }
-                }
-                if (cbk->op_ret < 0)
-                {
-                    ec_fop_set_error(fop, cbk->op_errno);
-                }
-            }
-            else
-            {
-                ec_fop_set_error(fop, EIO);
-            }
+            ec_fop_prepare_answer(fop, _gf_false);
 
             return EC_STATE_REPORT;
 
@@ -1226,11 +1082,10 @@ int32_t ec_manager_rmdir(ec_fop_data_t * fop, int32_t state)
 
             GF_ASSERT(cbk != NULL);
 
-            if (fop->cbks.rmdir != NULL)
-            {
-                fop->cbks.rmdir(fop->req_frame, fop, fop->xl, cbk->op_ret,
-                                cbk->op_errno, &cbk->iatt[0], &cbk->iatt[1],
-                                cbk->xdata);
+            if (fop->cbks.rmdir != NULL) {
+                QUORUM_CBK(fop->cbks.rmdir, fop, fop->req_frame, fop, fop->xl,
+                           cbk->op_ret, cbk->op_errno, &cbk->iatt[0],
+                           &cbk->iatt[1], cbk->xdata);
             }
 
             return EC_STATE_LOCK_REUSE;
@@ -1242,8 +1097,7 @@ int32_t ec_manager_rmdir(ec_fop_data_t * fop, int32_t state)
         case -EC_STATE_REPORT:
             GF_ASSERT(fop->error != 0);
 
-            if (fop->cbks.rmdir != NULL)
-            {
+            if (fop->cbks.rmdir != NULL) {
                 fop->cbks.rmdir(fop->req_frame, fop, fop->xl, -1, fop->error,
                                 NULL, NULL, NULL);
             }
@@ -1263,54 +1117,50 @@ int32_t ec_manager_rmdir(ec_fop_data_t * fop, int32_t state)
             return EC_STATE_END;
 
         default:
-            gf_log(fop->xl->name, GF_LOG_ERROR, "Unhandled state %d for %s",
-                   state, ec_fop_name(fop->id));
+            gf_msg(fop->xl->name, GF_LOG_ERROR, EINVAL, EC_MSG_UNHANDLED_STATE,
+                   "Unhandled state %d for %s", state, ec_fop_name(fop->id));
 
             return EC_STATE_END;
     }
 }
 
-void ec_rmdir(call_frame_t * frame, xlator_t * this, uintptr_t target,
-              int32_t minimum, fop_rmdir_cbk_t func, void * data, loc_t * loc,
-              int xflags, dict_t * xdata)
+void
+ec_rmdir(call_frame_t *frame, xlator_t *this, uintptr_t target,
+         uint32_t fop_flags, fop_rmdir_cbk_t func, void *data, loc_t *loc,
+         int xflags, dict_t *xdata)
 {
-    ec_cbk_t callback = { .rmdir = func };
-    ec_fop_data_t * fop = NULL;
-    int32_t error = EIO;
+    ec_cbk_t callback = {.rmdir = func};
+    ec_fop_data_t *fop = NULL;
+    int32_t error = ENOMEM;
 
-    gf_log("ec", GF_LOG_TRACE, "EC(RMDIR) %p", frame);
+    gf_msg_trace("ec", 0, "EC(RMDIR) %p", frame);
 
     VALIDATE_OR_GOTO(this, out);
     GF_VALIDATE_OR_GOTO(this->name, frame, out);
     GF_VALIDATE_OR_GOTO(this->name, this->private, out);
 
-    fop = ec_fop_data_allocate(frame, this, GF_FOP_RMDIR,
-                               EC_FLAG_UPDATE_LOC_PARENT, target, minimum,
-                               ec_wind_rmdir, ec_manager_rmdir, callback,
-                               data);
-    if (fop == NULL)
-    {
+    fop = ec_fop_data_allocate(frame, this, GF_FOP_RMDIR, 0, target, fop_flags,
+                               ec_wind_rmdir, ec_manager_rmdir, callback, data);
+    if (fop == NULL) {
         goto out;
     }
 
     fop->int32 = xflags;
 
-    if (loc != NULL)
-    {
-        if (loc_copy(&fop->loc[0], loc) != 0)
-        {
-            gf_log(this->name, GF_LOG_ERROR, "Failed to copy a location.");
+    if (loc != NULL) {
+        if (loc_copy(&fop->loc[0], loc) != 0) {
+            gf_msg(this->name, GF_LOG_ERROR, ENOMEM, EC_MSG_LOC_COPY_FAIL,
+                   "Failed to copy a location.");
 
             goto out;
         }
     }
-    if (xdata != NULL)
-    {
-        fop->xdata = dict_ref(xdata);
-        if (fop->xdata == NULL)
-        {
-            gf_log(this->name, GF_LOG_ERROR, "Failed to reference a "
-                                             "dictionary.");
+    if (xdata != NULL) {
+        fop->xdata = dict_copy_with_ref(xdata, NULL);
+        if (fop->xdata == NULL) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_DICT_REF_FAIL,
+                   "Failed to reference a "
+                   "dictionary.");
 
             goto out;
         }
@@ -1319,28 +1169,27 @@ void ec_rmdir(call_frame_t * frame, xlator_t * this, uintptr_t target,
     error = 0;
 
 out:
-    if (fop != NULL)
-    {
+    if (fop != NULL) {
         ec_manager(fop, error);
-    }
-    else
-    {
-        func(frame, NULL, this, -1, EIO, NULL, NULL, NULL);
+    } else {
+        func(frame, NULL, this, -1, error, NULL, NULL, NULL);
     }
 }
 
 /* FOP: symlink */
 
-int32_t ec_symlink_cbk(call_frame_t * frame, void * cookie, xlator_t * this,
-                       int32_t op_ret, int32_t op_errno, inode_t * inode,
-                       struct iatt * buf, struct iatt * preparent,
-                       struct iatt * postparent, dict_t * xdata)
+int32_t
+ec_symlink_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+               int32_t op_ret, int32_t op_errno, inode_t *inode,
+               struct iatt *buf, struct iatt *preparent,
+               struct iatt *postparent, dict_t *xdata)
 {
-        return ec_dir_write_cbk (frame, this, cookie, op_ret, op_errno,
-                                 buf, preparent, postparent, NULL, NULL, xdata);
+    return ec_dir_write_cbk(frame, this, cookie, op_ret, op_errno, buf,
+                            preparent, postparent, NULL, NULL, xdata);
 }
 
-void ec_wind_symlink(ec_t * ec, ec_fop_data_t * fop, int32_t idx)
+void
+ec_wind_symlink(ec_t *ec, ec_fop_data_t *fop, int32_t idx)
 {
     ec_trace("WIND", fop, "idx=%d", idx);
 
@@ -1349,15 +1198,16 @@ void ec_wind_symlink(ec_t * ec, ec_fop_data_t * fop, int32_t idx)
                       fop->str[0], &fop->loc[0], fop->mode[0], fop->xdata);
 }
 
-int32_t ec_manager_symlink(ec_fop_data_t * fop, int32_t state)
+int32_t
+ec_manager_symlink(ec_fop_data_t *fop, int32_t state)
 {
-    ec_cbk_data_t * cbk;
+    ec_cbk_data_t *cbk;
 
-    switch (state)
-    {
+    switch (state) {
         case EC_STATE_INIT:
         case EC_STATE_LOCK:
-            ec_lock_prepare_entry(fop, &fop->loc[0], 1);
+            ec_lock_prepare_parent_inode(fop, &fop->loc[0], NULL,
+                                         EC_UPDATE_DATA | EC_UPDATE_META);
             ec_lock(fop);
 
             return EC_STATE_DISPATCH;
@@ -1368,34 +1218,15 @@ int32_t ec_manager_symlink(ec_fop_data_t * fop, int32_t state)
             return EC_STATE_PREPARE_ANSWER;
 
         case EC_STATE_PREPARE_ANSWER:
-            cbk = fop->answer;
-            if (cbk != NULL)
-            {
-                if (!ec_dict_combine(cbk, EC_COMBINE_XDATA))
-                {
-                    if (cbk->op_ret >= 0)
-                    {
-                        cbk->op_ret = -1;
-                        cbk->op_errno = EIO;
-                    }
-                }
-                if (cbk->op_ret >= 0) {
-                    ec_iatt_rebuild(fop->xl->private, cbk->iatt, 3,
-                                    cbk->count);
-
-                    if (ec_loc_update(fop->xl, &fop->loc[0], cbk->inode,
-                                      &cbk->iatt[0]) != 0) {
-                        cbk->op_ret = -1;
-                        cbk->op_errno = EIO;
-                    }
-                }
-                if (cbk->op_ret < 0) {
-                    ec_fop_set_error(fop, cbk->op_errno);
-                }
-            }
-            else
-            {
-                ec_fop_set_error(fop, EIO);
+            cbk = ec_fop_prepare_answer(fop, _gf_false);
+            if (cbk != NULL) {
+                int32_t err;
+
+                ec_iatt_rebuild(fop->xl->private, cbk->iatt, 3, cbk->count);
+
+                err = ec_loc_update(fop->xl, &fop->loc[0], cbk->inode,
+                                    &cbk->iatt[0]);
+                ec_cbk_set_error(cbk, -err, _gf_false);
             }
 
             return EC_STATE_REPORT;
@@ -1405,12 +1236,11 @@ int32_t ec_manager_symlink(ec_fop_data_t * fop, int32_t state)
 
             GF_ASSERT(cbk != NULL);
 
-            if (fop->cbks.symlink != NULL)
-            {
-                fop->cbks.symlink(fop->req_frame, fop, fop->xl, cbk->op_ret,
-                                  cbk->op_errno, fop->loc[0].inode,
-                                  &cbk->iatt[0], &cbk->iatt[1], &cbk->iatt[2],
-                                  cbk->xdata);
+            if (fop->cbks.symlink != NULL) {
+                QUORUM_CBK(fop->cbks.symlink, fop, fop->req_frame, fop, fop->xl,
+                           cbk->op_ret, cbk->op_errno, fop->loc[0].inode,
+                           &cbk->iatt[0], &cbk->iatt[1], &cbk->iatt[2],
+                           cbk->xdata);
             }
 
             return EC_STATE_LOCK_REUSE;
@@ -1422,8 +1252,7 @@ int32_t ec_manager_symlink(ec_fop_data_t * fop, int32_t state)
         case -EC_STATE_REPORT:
             GF_ASSERT(fop->error != 0);
 
-            if (fop->cbks.symlink != NULL)
-            {
+            if (fop->cbks.symlink != NULL) {
                 fop->cbks.symlink(fop->req_frame, fop, fop->xl, -1, fop->error,
                                   NULL, NULL, NULL, NULL, NULL);
             }
@@ -1443,65 +1272,60 @@ int32_t ec_manager_symlink(ec_fop_data_t * fop, int32_t state)
             return EC_STATE_END;
 
         default:
-            gf_log(fop->xl->name, GF_LOG_ERROR, "Unhandled state %d for %s",
-                   state, ec_fop_name(fop->id));
+            gf_msg(fop->xl->name, GF_LOG_ERROR, EINVAL, EC_MSG_UNHANDLED_STATE,
+                   "Unhandled state %d for %s", state, ec_fop_name(fop->id));
 
             return EC_STATE_END;
     }
 }
 
-void ec_symlink(call_frame_t * frame, xlator_t * this, uintptr_t target,
-                int32_t minimum, fop_symlink_cbk_t func, void * data,
-                const char * linkname, loc_t * loc, mode_t umask,
-                dict_t * xdata)
+void
+ec_symlink(call_frame_t *frame, xlator_t *this, uintptr_t target,
+           uint32_t fop_flags, fop_symlink_cbk_t func, void *data,
+           const char *linkname, loc_t *loc, mode_t umask, dict_t *xdata)
 {
-    ec_cbk_t callback = { .symlink = func };
-    ec_fop_data_t * fop = NULL;
-    int32_t error = EIO;
+    ec_cbk_t callback = {.symlink = func};
+    ec_fop_data_t *fop = NULL;
+    int32_t error = ENOMEM;
 
-    gf_log("ec", GF_LOG_TRACE, "EC(SYMLINK) %p", frame);
+    gf_msg_trace("ec", 0, "EC(SYMLINK) %p", frame);
 
     VALIDATE_OR_GOTO(this, out);
     GF_VALIDATE_OR_GOTO(this->name, frame, out);
     GF_VALIDATE_OR_GOTO(this->name, this->private, out);
 
-    fop = ec_fop_data_allocate(frame, this, GF_FOP_SYMLINK,
-                               EC_FLAG_UPDATE_LOC_PARENT, target, minimum,
-                               ec_wind_symlink, ec_manager_symlink, callback,
-                               data);
-    if (fop == NULL)
-    {
+    fop = ec_fop_data_allocate(frame, this, GF_FOP_SYMLINK, 0, target,
+                               fop_flags, ec_wind_symlink, ec_manager_symlink,
+                               callback, data);
+    if (fop == NULL) {
         goto out;
     }
 
     fop->mode[0] = umask;
 
-    if (linkname != NULL)
-    {
+    if (linkname != NULL) {
         fop->str[0] = gf_strdup(linkname);
-        if (fop->str[0] == NULL)
-        {
-            gf_log(this->name, GF_LOG_ERROR, "Failed to duplicate a string.");
+        if (fop->str[0] == NULL) {
+            gf_msg(this->name, GF_LOG_ERROR, ENOMEM, EC_MSG_NO_MEMORY,
+                   "Failed to duplicate a string.");
 
             goto out;
         }
     }
-    if (loc != NULL)
-    {
-        if (loc_copy(&fop->loc[0], loc) != 0)
-        {
-            gf_log(this->name, GF_LOG_ERROR, "Failed to copy a location.");
+    if (loc != NULL) {
+        if (loc_copy(&fop->loc[0], loc) != 0) {
+            gf_msg(this->name, GF_LOG_ERROR, ENOMEM, EC_MSG_LOC_COPY_FAIL,
+                   "Failed to copy a location.");
 
             goto out;
         }
     }
-    if (xdata != NULL)
-    {
-        fop->xdata = dict_ref(xdata);
-        if (fop->xdata == NULL)
-        {
-            gf_log(this->name, GF_LOG_ERROR, "Failed to reference a "
-                                             "dictionary.");
+    if (xdata != NULL) {
+        fop->xdata = dict_copy_with_ref(xdata, NULL);
+        if (fop->xdata == NULL) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_DICT_REF_FAIL,
+                   "Failed to reference a "
+                   "dictionary.");
 
             goto out;
         }
@@ -1510,28 +1334,26 @@ void ec_symlink(call_frame_t * frame, xlator_t * this, uintptr_t target,
     error = 0;
 
 out:
-    if (fop != NULL)
-    {
+    if (fop != NULL) {
         ec_manager(fop, error);
-    }
-    else
-    {
-        func(frame, NULL, this, -1, EIO, NULL, NULL, NULL, NULL, NULL);
+    } else {
+        func(frame, NULL, this, -1, error, NULL, NULL, NULL, NULL, NULL);
     }
 }
 
 /* FOP: unlink */
 
-int32_t ec_unlink_cbk(call_frame_t * frame, void * cookie, xlator_t * this,
-                      int32_t op_ret, int32_t op_errno,
-                      struct iatt * preparent, struct iatt * postparent,
-                      dict_t * xdata)
+int32_t
+ec_unlink_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int32_t op_ret,
+              int32_t op_errno, struct iatt *preparent, struct iatt *postparent,
+              dict_t *xdata)
 {
-        return ec_dir_write_cbk (frame, this, cookie, op_ret, op_errno, NULL,
-                                 preparent, postparent, NULL, NULL, xdata);
+    return ec_dir_write_cbk(frame, this, cookie, op_ret, op_errno, NULL,
+                            preparent, postparent, NULL, NULL, xdata);
 }
 
-void ec_wind_unlink(ec_t * ec, ec_fop_data_t * fop, int32_t idx)
+void
+ec_wind_unlink(ec_t *ec, ec_fop_data_t *fop, int32_t idx)
 {
     ec_trace("WIND", fop, "idx=%d", idx);
 
@@ -1540,22 +1362,18 @@ void ec_wind_unlink(ec_t * ec, ec_fop_data_t * fop, int32_t idx)
                       &fop->loc[0], fop->int32, fop->xdata);
 }
 
-int32_t ec_manager_unlink(ec_fop_data_t * fop, int32_t state)
+int32_t
+ec_manager_unlink(ec_fop_data_t *fop, int32_t state)
 {
-    ec_cbk_data_t * cbk;
+    ec_cbk_data_t *cbk;
 
-    switch (state)
-    {
+    switch (state) {
         case EC_STATE_INIT:
         case EC_STATE_LOCK:
-            ec_lock_prepare_entry(fop, &fop->loc[0], 1);
+            ec_lock_prepare_parent_inode(fop, &fop->loc[0], NULL,
+                                         EC_UPDATE_DATA | EC_UPDATE_META);
             ec_lock(fop);
 
-            return EC_STATE_GET_SIZE_AND_VERSION;
-
-        case EC_STATE_GET_SIZE_AND_VERSION:
-            ec_get_size_version(fop);
-
             return EC_STATE_DISPATCH;
 
         case EC_STATE_DISPATCH:
@@ -1564,26 +1382,7 @@ int32_t ec_manager_unlink(ec_fop_data_t * fop, int32_t state)
             return EC_STATE_PREPARE_ANSWER;
 
         case EC_STATE_PREPARE_ANSWER:
-            cbk = fop->answer;
-            if (cbk != NULL)
-            {
-                if (!ec_dict_combine(cbk, EC_COMBINE_XDATA))
-                {
-                    if (cbk->op_ret >= 0)
-                    {
-                        cbk->op_ret = -1;
-                        cbk->op_errno = EIO;
-                    }
-                }
-                if (cbk->op_ret < 0)
-                {
-                    ec_fop_set_error(fop, cbk->op_errno);
-                }
-            }
-            else
-            {
-                ec_fop_set_error(fop, EIO);
-            }
+            ec_fop_prepare_answer(fop, _gf_false);
 
             return EC_STATE_REPORT;
 
@@ -1592,25 +1391,22 @@ int32_t ec_manager_unlink(ec_fop_data_t * fop, int32_t state)
 
             GF_ASSERT(cbk != NULL);
 
-            if (fop->cbks.unlink != NULL)
-            {
-                fop->cbks.unlink(fop->req_frame, fop, fop->xl, cbk->op_ret,
-                                 cbk->op_errno, &cbk->iatt[0], &cbk->iatt[1],
-                                 cbk->xdata);
+            if (fop->cbks.unlink != NULL) {
+                QUORUM_CBK(fop->cbks.unlink, fop, fop->req_frame, fop, fop->xl,
+                           cbk->op_ret, cbk->op_errno, &cbk->iatt[0],
+                           &cbk->iatt[1], cbk->xdata);
             }
 
             return EC_STATE_LOCK_REUSE;
 
         case -EC_STATE_INIT:
         case -EC_STATE_LOCK:
-        case -EC_STATE_GET_SIZE_AND_VERSION:
         case -EC_STATE_DISPATCH:
         case -EC_STATE_PREPARE_ANSWER:
         case -EC_STATE_REPORT:
             GF_ASSERT(fop->error != 0);
 
-            if (fop->cbks.unlink != NULL)
-            {
+            if (fop->cbks.unlink != NULL) {
                 fop->cbks.unlink(fop->req_frame, fop, fop->xl, -1, fop->error,
                                  NULL, NULL, NULL);
             }
@@ -1630,54 +1426,51 @@ int32_t ec_manager_unlink(ec_fop_data_t * fop, int32_t state)
             return EC_STATE_END;
 
         default:
-            gf_log(fop->xl->name, GF_LOG_ERROR, "Unhandled state %d for %s",
-                   state, ec_fop_name(fop->id));
+            gf_msg(fop->xl->name, GF_LOG_ERROR, EINVAL, EC_MSG_UNHANDLED_STATE,
+                   "Unhandled state %d for %s", state, ec_fop_name(fop->id));
 
             return EC_STATE_END;
     }
 }
 
-void ec_unlink(call_frame_t * frame, xlator_t * this, uintptr_t target,
-               int32_t minimum, fop_unlink_cbk_t func, void * data,
-               loc_t * loc, int xflags, dict_t * xdata)
+void
+ec_unlink(call_frame_t *frame, xlator_t *this, uintptr_t target,
+          uint32_t fop_flags, fop_unlink_cbk_t func, void *data, loc_t *loc,
+          int xflags, dict_t *xdata)
 {
-    ec_cbk_t callback = { .unlink = func };
-    ec_fop_data_t * fop = NULL;
-    int32_t error = EIO;
+    ec_cbk_t callback = {.unlink = func};
+    ec_fop_data_t *fop = NULL;
+    int32_t error = ENOMEM;
 
-    gf_log("ec", GF_LOG_TRACE, "EC(UNLINK) %p", frame);
+    gf_msg_trace("ec", 0, "EC(UNLINK) %p", frame);
 
     VALIDATE_OR_GOTO(this, out);
     GF_VALIDATE_OR_GOTO(this->name, frame, out);
     GF_VALIDATE_OR_GOTO(this->name, this->private, out);
 
-    fop = ec_fop_data_allocate(frame, this, GF_FOP_UNLINK,
-                               EC_FLAG_UPDATE_LOC_PARENT, target, minimum,
+    fop = ec_fop_data_allocate(frame, this, GF_FOP_UNLINK, 0, target, fop_flags,
                                ec_wind_unlink, ec_manager_unlink, callback,
                                data);
-    if (fop == NULL)
-    {
+    if (fop == NULL) {
         goto out;
     }
 
     fop->int32 = xflags;
 
-    if (loc != NULL)
-    {
-        if (loc_copy(&fop->loc[0], loc) != 0)
-        {
-            gf_log(this->name, GF_LOG_ERROR, "Failed to copy a location.");
+    if (loc != NULL) {
+        if (loc_copy(&fop->loc[0], loc) != 0) {
+            gf_msg(this->name, GF_LOG_ERROR, ENOMEM, EC_MSG_LOC_COPY_FAIL,
+                   "Failed to copy a location.");
 
             goto out;
         }
     }
-    if (xdata != NULL)
-    {
-        fop->xdata = dict_ref(xdata);
-        if (fop->xdata == NULL)
-        {
-            gf_log(this->name, GF_LOG_ERROR, "Failed to reference a "
-                                             "dictionary.");
+    if (xdata != NULL) {
+        fop->xdata = dict_copy_with_ref(xdata, NULL);
+        if (fop->xdata == NULL) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_DICT_REF_FAIL,
+                   "Failed to reference a "
+                   "dictionary.");
 
             goto out;
         }
@@ -1686,12 +1479,9 @@ void ec_unlink(call_frame_t * frame, xlator_t * this, uintptr_t target,
     error = 0;
 
 out:
-    if (fop != NULL)
-    {
+    if (fop != NULL) {
         ec_manager(fop, error);
-    }
-    else
-    {
-        func(frame, NULL, this, -1, EIO, NULL, NULL, NULL);
+    } else {
+        func(frame, NULL, this, -1, error, NULL, NULL, NULL);
     }
 }
diff --git a/xlators/cluster/ec/src/ec-fops.h b/xlators/cluster/ec/src/ec-fops.h
index d6b9770f720..07edf8a7fec 100644
--- a/xlators/cluster/ec/src/ec-fops.h
+++ b/xlators/cluster/ec/src/ec-fops.h
@@ -11,191 +11,244 @@
 #ifndef __EC_FOPS_H__
 #define __EC_FOPS_H__
 
-#include "xlator.h"
+#include <glusterfs/xlator.h>
 
-#include "ec-data.h"
+#include "ec-types.h"
 #include "ec-common.h"
 
-#define EC_FOP_HEAL     -1
-#define EC_FOP_FHEAL    -2
-
-void ec_access(call_frame_t * frame, xlator_t * this, uintptr_t target,
-               int32_t minimum, fop_access_cbk_t func, void *data, loc_t * loc,
-               int32_t mask, dict_t * xdata);
-
-void ec_create(call_frame_t * frame, xlator_t * this, uintptr_t target,
-               int32_t minimum, fop_create_cbk_t func, void *data, loc_t * loc,
-               int32_t flags, mode_t mode, mode_t umask, fd_t * fd,
-               dict_t * xdata);
-
-void ec_entrylk(call_frame_t * frame, xlator_t * this, uintptr_t target,
-                int32_t minimum, fop_entrylk_cbk_t func, void *data,
-                const char * volume, loc_t * loc, const char * basename,
-                entrylk_cmd cmd, entrylk_type type, dict_t * xdata);
-
-void ec_fentrylk(call_frame_t * frame, xlator_t * this, uintptr_t target,
-                 int32_t minimum, fop_fentrylk_cbk_t func, void *data,
-                 const char * volume, fd_t * fd, const char * basename,
-                 entrylk_cmd cmd, entrylk_type type, dict_t * xdata);
-
-void ec_flush(call_frame_t * frame, xlator_t * this, uintptr_t target,
-              int32_t minimum, fop_flush_cbk_t func, void *data, fd_t * fd,
-              dict_t * xdata);
-
-void ec_fsync(call_frame_t * frame, xlator_t * this, uintptr_t target,
-              int32_t minimum, fop_fsync_cbk_t func, void *data, fd_t * fd,
-              int32_t datasync, dict_t * xdata);
-
-void ec_fsyncdir(call_frame_t * frame, xlator_t * this, uintptr_t target,
-                 int32_t minimum, fop_fsyncdir_cbk_t func, void *data,
-                 fd_t * fd, int32_t datasync, dict_t * xdata);
-
-void ec_getxattr(call_frame_t * frame, xlator_t * this, uintptr_t target,
-                 int32_t minimum, fop_getxattr_cbk_t func, void *data,
-                 loc_t * loc, const char * name, dict_t * xdata);
-
-void ec_fgetxattr(call_frame_t * frame, xlator_t * this, uintptr_t target,
-                  int32_t minimum, fop_fgetxattr_cbk_t func, void *data,
-                  fd_t * fd, const char * name, dict_t * xdata);
-
-void ec_heal(call_frame_t * frame, xlator_t * this, uintptr_t target,
-             int32_t minimum, fop_heal_cbk_t func, void *data, loc_t * loc,
-             int32_t partial, dict_t *xdata);
-
-void ec_fheal(call_frame_t * frame, xlator_t * this, uintptr_t target,
-              int32_t minimum, fop_fheal_cbk_t func, void *data, fd_t * fd,
-              int32_t partial, dict_t *xdata);
-
-void ec_inodelk(call_frame_t * frame, xlator_t * this, uintptr_t target,
-                int32_t minimum, fop_inodelk_cbk_t func, void *data,
-                const char * volume, loc_t * loc, int32_t cmd,
-                struct gf_flock * flock, dict_t * xdata);
-
-void ec_finodelk(call_frame_t * frame, xlator_t * this, uintptr_t target,
-                 int32_t minimum, fop_finodelk_cbk_t func, void *data,
-                 const char * volume, fd_t * fd, int32_t cmd,
-                 struct gf_flock * flock, dict_t * xdata);
-
-void ec_link(call_frame_t * frame, xlator_t * this, uintptr_t target,
-             int32_t minimum, fop_link_cbk_t func, void *data, loc_t * oldloc,
-             loc_t * newloc, dict_t * xdata);
-
-void ec_lk(call_frame_t * frame, xlator_t * this, uintptr_t target,
-           int32_t minimum, fop_lk_cbk_t func, void *data, fd_t * fd,
-           int32_t cmd, struct gf_flock * flock, dict_t * xdata);
-
-void ec_lookup(call_frame_t * frame, xlator_t * this, uintptr_t target,
-               int32_t minimum, fop_lookup_cbk_t func, void *data, loc_t * loc,
-               dict_t * xdata);
-
-void ec_mkdir(call_frame_t * frame, xlator_t * this, uintptr_t target,
-              int32_t minimum, fop_mkdir_cbk_t func, void *data, loc_t * loc,
-              mode_t mode, mode_t umask, dict_t * xdata);
-
-void ec_mknod(call_frame_t * frame, xlator_t * this, uintptr_t target,
-              int32_t minimum, fop_mknod_cbk_t func, void *data, loc_t * loc,
-              mode_t mode, dev_t rdev, mode_t umask, dict_t * xdata);
-
-void ec_open(call_frame_t * frame, xlator_t * this, uintptr_t target,
-             int32_t minimum, fop_open_cbk_t func, void *data, loc_t * loc,
-             int32_t flags, fd_t * fd, dict_t * xdata);
-
-void ec_opendir(call_frame_t * frame, xlator_t * this, uintptr_t target,
-                int32_t minimum, fop_opendir_cbk_t func, void *data,
-                loc_t * loc, fd_t * fd, dict_t * xdata);
-
-void ec_readdir(call_frame_t * frame, xlator_t * this, uintptr_t target,
-                int32_t minimum, fop_readdir_cbk_t func, void *data, fd_t * fd,
-                size_t size, off_t offset, dict_t * xdata);
-
-void ec_readdirp(call_frame_t * frame, xlator_t * this, uintptr_t target,
-                 int32_t minimum, fop_readdirp_cbk_t func, void *data,
-                 fd_t * fd, size_t size, off_t offset, dict_t * xdata);
-
-void ec_readlink(call_frame_t * frame, xlator_t * this, uintptr_t target,
-                 int32_t minimum, fop_readlink_cbk_t func, void *data,
-                 loc_t * loc, size_t size, dict_t * xdata);
-
-void ec_readv(call_frame_t * frame, xlator_t * this, uintptr_t target,
-              int32_t minimum, fop_readv_cbk_t func, void *data, fd_t * fd,
-              size_t size, off_t offset, uint32_t flags, dict_t * xdata);
-
-void ec_removexattr(call_frame_t * frame, xlator_t * this, uintptr_t target,
-                    int32_t minimum, fop_removexattr_cbk_t func, void *data,
-                    loc_t * loc, const char * name, dict_t * xdata);
-
-void ec_fremovexattr(call_frame_t * frame, xlator_t * this, uintptr_t target,
-                     int32_t minimum, fop_fremovexattr_cbk_t func, void *data,
-                     fd_t * fd, const char * name, dict_t * xdata);
-
-void ec_rename(call_frame_t * frame, xlator_t * this, uintptr_t target,
-               int32_t minimum, fop_rename_cbk_t func, void *data,
-               loc_t * oldloc, loc_t * newloc, dict_t * xdata);
-
-void ec_rmdir(call_frame_t * frame, xlator_t * this, uintptr_t target,
-              int32_t minimum, fop_rmdir_cbk_t func, void *data, loc_t * loc,
-              int xflags, dict_t * xdata);
-
-void ec_setattr(call_frame_t * frame, xlator_t * this, uintptr_t target,
-                int32_t minimum, fop_setattr_cbk_t func, void *data,
-                loc_t * loc, struct iatt * stbuf, int32_t valid,
-                dict_t * xdata);
-
-void ec_fsetattr(call_frame_t * frame, xlator_t * this, uintptr_t target,
-                 int32_t minimum, fop_fsetattr_cbk_t func, void *data,
-                 fd_t * fd, struct iatt * stbuf, int32_t valid,
-                 dict_t * xdata);
-
-void ec_setxattr(call_frame_t * frame, xlator_t * this, uintptr_t target,
-                 int32_t minimum, fop_setxattr_cbk_t func, void *data,
-                 loc_t * loc, dict_t * dict, int32_t flags, dict_t * xdata);
-
-void ec_fsetxattr(call_frame_t * frame, xlator_t * this, uintptr_t target,
-                  int32_t minimum, fop_fsetxattr_cbk_t func, void *data,
-                  fd_t * fd, dict_t * dict, int32_t flags, dict_t * xdata);
-
-void ec_stat(call_frame_t * frame, xlator_t * this, uintptr_t target,
-             int32_t minimum, fop_stat_cbk_t func, void *data, loc_t * loc,
-             dict_t * xdata);
-
-void ec_fstat(call_frame_t * frame, xlator_t * this, uintptr_t target,
-              int32_t minimum, fop_fstat_cbk_t func, void *data, fd_t * fd,
-              dict_t * xdata);
-
-void ec_statfs(call_frame_t * frame, xlator_t * this, uintptr_t target,
-               int32_t minimum, fop_statfs_cbk_t func, void *data, loc_t * loc,
-               dict_t * xdata);
-
-void ec_symlink(call_frame_t * frame, xlator_t * this, uintptr_t target,
-                int32_t minimum, fop_symlink_cbk_t func, void *data,
-                const char * linkname, loc_t * loc, mode_t umask,
-                dict_t * xdata);
-
-void ec_truncate(call_frame_t * frame, xlator_t * this, uintptr_t target,
-                 int32_t minimum, fop_truncate_cbk_t func, void *data,
-                 loc_t * loc, off_t offset, dict_t * xdata);
-
-void ec_ftruncate(call_frame_t * frame, xlator_t * this, uintptr_t target,
-                  int32_t minimum, fop_ftruncate_cbk_t func, void *data,
-                  fd_t * fd, off_t offset, dict_t * xdata);
-
-void ec_unlink(call_frame_t * frame, xlator_t * this, uintptr_t target,
-               int32_t minimum, fop_unlink_cbk_t func, void *data, loc_t * loc,
-               int xflags, dict_t * xdata);
-
-void ec_writev(call_frame_t * frame, xlator_t * this, uintptr_t target,
-               int32_t minimum, fop_writev_cbk_t func, void *data, fd_t * fd,
-               struct iovec * vector, int32_t count, off_t offset,
-               uint32_t flags, struct iobref * iobref, dict_t * xdata);
-
-void ec_xattrop(call_frame_t * frame, xlator_t * this, uintptr_t target,
-                int32_t minimum, fop_xattrop_cbk_t func, void *data,
-                loc_t * loc, gf_xattrop_flags_t optype, dict_t * xattr,
-                dict_t * xdata);
-
-void ec_fxattrop(call_frame_t * frame, xlator_t * this, uintptr_t target,
-                 int32_t minimum, fop_fxattrop_cbk_t func, void *data,
-                 fd_t * fd, gf_xattrop_flags_t optype, dict_t * xattr,
-                 dict_t * xdata);
+void
+ec_access(call_frame_t *frame, xlator_t *this, uintptr_t target,
+          uint32_t fop_flags, fop_access_cbk_t func, void *data, loc_t *loc,
+          int32_t mask, dict_t *xdata);
+
+void
+ec_create(call_frame_t *frame, xlator_t *this, uintptr_t target,
+          uint32_t fop_flags, fop_create_cbk_t func, void *data, loc_t *loc,
+          int32_t flags, mode_t mode, mode_t umask, fd_t *fd, dict_t *xdata);
+
+void
+ec_entrylk(call_frame_t *frame, xlator_t *this, uintptr_t target,
+           uint32_t fop_flags, fop_entrylk_cbk_t func, void *data,
+           const char *volume, loc_t *loc, const char *basename,
+           entrylk_cmd cmd, entrylk_type type, dict_t *xdata);
+
+void
+ec_fentrylk(call_frame_t *frame, xlator_t *this, uintptr_t target,
+            uint32_t fop_flags, fop_fentrylk_cbk_t func, void *data,
+            const char *volume, fd_t *fd, const char *basename, entrylk_cmd cmd,
+            entrylk_type type, dict_t *xdata);
+
+void
+ec_flush(call_frame_t *frame, xlator_t *this, uintptr_t target,
+         uint32_t fop_flags, fop_flush_cbk_t func, void *data, fd_t *fd,
+         dict_t *xdata);
+
+void
+ec_fsync(call_frame_t *frame, xlator_t *this, uintptr_t target,
+         uint32_t fop_flags, fop_fsync_cbk_t func, void *data, fd_t *fd,
+         int32_t datasync, dict_t *xdata);
+
+void
+ec_fsyncdir(call_frame_t *frame, xlator_t *this, uintptr_t target,
+            uint32_t fop_flags, fop_fsyncdir_cbk_t func, void *data, fd_t *fd,
+            int32_t datasync, dict_t *xdata);
+
+void
+ec_getxattr(call_frame_t *frame, xlator_t *this, uintptr_t target,
+            uint32_t fop_flags, fop_getxattr_cbk_t func, void *data, loc_t *loc,
+            const char *name, dict_t *xdata);
+
+void
+ec_fgetxattr(call_frame_t *frame, xlator_t *this, uintptr_t target,
+             uint32_t fop_flags, fop_fgetxattr_cbk_t func, void *data, fd_t *fd,
+             const char *name, dict_t *xdata);
+
+void
+ec_heal(call_frame_t *frame, xlator_t *this, uintptr_t target,
+        uint32_t fop_flags, fop_heal_cbk_t func, void *data, loc_t *loc,
+        int32_t partial, dict_t *xdata);
+
+void
+ec_fheal(call_frame_t *frame, xlator_t *this, uintptr_t target,
+         uint32_t fop_flags, fop_fheal_cbk_t func, void *data, fd_t *fd,
+         int32_t partial, dict_t *xdata);
+
+void
+ec_inodelk(call_frame_t *frame, xlator_t *this, gf_lkowner_t *owner,
+           uintptr_t target, uint32_t fop_flags, fop_inodelk_cbk_t func,
+           void *data, const char *volume, loc_t *loc, int32_t cmd,
+           struct gf_flock *flock, dict_t *xdata);
+
+void
+ec_finodelk(call_frame_t *frame, xlator_t *this, gf_lkowner_t *owner,
+            uintptr_t target, uint32_t fop_flags, fop_finodelk_cbk_t func,
+            void *data, const char *volume, fd_t *fd, int32_t cmd,
+            struct gf_flock *flock, dict_t *xdata);
+
+void
+ec_link(call_frame_t *frame, xlator_t *this, uintptr_t target,
+        uint32_t fop_flags, fop_link_cbk_t func, void *data, loc_t *oldloc,
+        loc_t *newloc, dict_t *xdata);
+
+void
+ec_lk(call_frame_t *frame, xlator_t *this, uintptr_t target, uint32_t fop_flags,
+      fop_lk_cbk_t func, void *data, fd_t *fd, int32_t cmd,
+      struct gf_flock *flock, dict_t *xdata);
+
+void
+ec_lookup(call_frame_t *frame, xlator_t *this, uintptr_t target,
+          uint32_t fop_flags, fop_lookup_cbk_t func, void *data, loc_t *loc,
+          dict_t *xdata);
+
+void
+ec_mkdir(call_frame_t *frame, xlator_t *this, uintptr_t target,
+         uint32_t fop_flags, fop_mkdir_cbk_t func, void *data, loc_t *loc,
+         mode_t mode, mode_t umask, dict_t *xdata);
+
+void
+ec_mknod(call_frame_t *frame, xlator_t *this, uintptr_t target,
+         uint32_t fop_flags, fop_mknod_cbk_t func, void *data, loc_t *loc,
+         mode_t mode, dev_t rdev, mode_t umask, dict_t *xdata);
+
+void
+ec_open(call_frame_t *frame, xlator_t *this, uintptr_t target,
+        uint32_t fop_flags, fop_open_cbk_t func, void *data, loc_t *loc,
+        int32_t flags, fd_t *fd, dict_t *xdata);
+
+void
+ec_opendir(call_frame_t *frame, xlator_t *this, uintptr_t target,
+           uint32_t fop_flags, fop_opendir_cbk_t func, void *data, loc_t *loc,
+           fd_t *fd, dict_t *xdata);
+
+void
+ec_readdir(call_frame_t *frame, xlator_t *this, uintptr_t target,
+           uint32_t fop_flags, fop_readdir_cbk_t func, void *data, fd_t *fd,
+           size_t size, off_t offset, dict_t *xdata);
+
+void
+ec_readdirp(call_frame_t *frame, xlator_t *this, uintptr_t target,
+            uint32_t fop_flags, fop_readdirp_cbk_t func, void *data, fd_t *fd,
+            size_t size, off_t offset, dict_t *xdata);
+
+void
+ec_readlink(call_frame_t *frame, xlator_t *this, uintptr_t target,
+            uint32_t fop_flags, fop_readlink_cbk_t func, void *data, loc_t *loc,
+            size_t size, dict_t *xdata);
+
+void
+ec_readv(call_frame_t *frame, xlator_t *this, uintptr_t target,
+         uint32_t fop_flags, fop_readv_cbk_t func, void *data, fd_t *fd,
+         size_t size, off_t offset, uint32_t flags, dict_t *xdata);
+
+void
+ec_removexattr(call_frame_t *frame, xlator_t *this, uintptr_t target,
+               uint32_t fop_flags, fop_removexattr_cbk_t func, void *data,
+               loc_t *loc, const char *name, dict_t *xdata);
+
+void
+ec_fremovexattr(call_frame_t *frame, xlator_t *this, uintptr_t target,
+                uint32_t fop_flags, fop_fremovexattr_cbk_t func, void *data,
+                fd_t *fd, const char *name, dict_t *xdata);
+
+void
+ec_rename(call_frame_t *frame, xlator_t *this, uintptr_t target,
+          uint32_t fop_flags, fop_rename_cbk_t func, void *data, loc_t *oldloc,
+          loc_t *newloc, dict_t *xdata);
+
+void
+ec_rmdir(call_frame_t *frame, xlator_t *this, uintptr_t target,
+         uint32_t fop_flags, fop_rmdir_cbk_t func, void *data, loc_t *loc,
+         int xflags, dict_t *xdata);
+
+void
+ec_setattr(call_frame_t *frame, xlator_t *this, uintptr_t target,
+           uint32_t fop_flags, fop_setattr_cbk_t func, void *data, loc_t *loc,
+           struct iatt *stbuf, int32_t valid, dict_t *xdata);
+
+void
+ec_fsetattr(call_frame_t *frame, xlator_t *this, uintptr_t target,
+            uint32_t fop_flags, fop_fsetattr_cbk_t func, void *data, fd_t *fd,
+            struct iatt *stbuf, int32_t valid, dict_t *xdata);
+
+void
+ec_setxattr(call_frame_t *frame, xlator_t *this, uintptr_t target,
+            uint32_t fop_flags, fop_setxattr_cbk_t func, void *data, loc_t *loc,
+            dict_t *dict, int32_t flags, dict_t *xdata);
+
+void
+ec_fsetxattr(call_frame_t *frame, xlator_t *this, uintptr_t target,
+             uint32_t fop_flags, fop_fsetxattr_cbk_t func, void *data, fd_t *fd,
+             dict_t *dict, int32_t flags, dict_t *xdata);
+
+void
+ec_stat(call_frame_t *frame, xlator_t *this, uintptr_t target,
+        uint32_t fop_flags, fop_stat_cbk_t func, void *data, loc_t *loc,
+        dict_t *xdata);
+
+void
+ec_fstat(call_frame_t *frame, xlator_t *this, uintptr_t target,
+         uint32_t fop_flags, fop_fstat_cbk_t func, void *data, fd_t *fd,
+         dict_t *xdata);
+
+void
+ec_statfs(call_frame_t *frame, xlator_t *this, uintptr_t target,
+          uint32_t fop_flags, fop_statfs_cbk_t func, void *data, loc_t *loc,
+          dict_t *xdata);
+
+void
+ec_symlink(call_frame_t *frame, xlator_t *this, uintptr_t target,
+           uint32_t fop_flags, fop_symlink_cbk_t func, void *data,
+           const char *linkname, loc_t *loc, mode_t umask, dict_t *xdata);
+
+void
+ec_fallocate(call_frame_t *frame, xlator_t *this, uintptr_t target,
+             uint32_t fop_flags, fop_fallocate_cbk_t func, void *data, fd_t *fd,
+             int32_t mode, off_t offset, size_t len, dict_t *xdata);
+
+void
+ec_discard(call_frame_t *frame, xlator_t *this, uintptr_t target,
+           uint32_t fop_flags, fop_discard_cbk_t func, void *data, fd_t *fd,
+           off_t offset, size_t len, dict_t *xdata);
+
+void
+ec_truncate(call_frame_t *frame, xlator_t *this, uintptr_t target,
+            uint32_t fop_flags, fop_truncate_cbk_t func, void *data, loc_t *loc,
+            off_t offset, dict_t *xdata);
+
+void
+ec_ftruncate(call_frame_t *frame, xlator_t *this, uintptr_t target,
+             uint32_t fop_flags, fop_ftruncate_cbk_t func, void *data, fd_t *fd,
+             off_t offset, dict_t *xdata);
+
+void
+ec_unlink(call_frame_t *frame, xlator_t *this, uintptr_t target,
+          uint32_t fop_flags, fop_unlink_cbk_t func, void *data, loc_t *loc,
+          int xflags, dict_t *xdata);
+
+void
+ec_writev(call_frame_t *frame, xlator_t *this, uintptr_t target,
+          uint32_t fop_flags, fop_writev_cbk_t func, void *data, fd_t *fd,
+          struct iovec *vector, int32_t count, off_t offset, uint32_t flags,
+          struct iobref *iobref, dict_t *xdata);
+
+void
+ec_xattrop(call_frame_t *frame, xlator_t *this, uintptr_t target,
+           uint32_t fop_flags, fop_xattrop_cbk_t func, void *data, loc_t *loc,
+           gf_xattrop_flags_t optype, dict_t *xattr, dict_t *xdata);
+
+void
+ec_fxattrop(call_frame_t *frame, xlator_t *this, uintptr_t target,
+            uint32_t fop_flags, fop_fxattrop_cbk_t func, void *data, fd_t *fd,
+            gf_xattrop_flags_t optype, dict_t *xattr, dict_t *xdata);
+
+void
+ec_seek(call_frame_t *frame, xlator_t *this, uintptr_t target,
+        uint32_t fop_flags, fop_seek_cbk_t func, void *data, fd_t *fd,
+        off_t offset, gf_seek_what_t what, dict_t *xdata);
+
+void
+ec_ipc(call_frame_t *frame, xlator_t *this, uintptr_t target,
+       uint32_t fop_flags, fop_ipc_cbk_t func, void *data, int32_t op,
+       dict_t *xdata);
 
 #endif /* __EC_FOPS_H__ */
diff --git a/xlators/cluster/ec/src/ec-galois.c b/xlators/cluster/ec/src/ec-galois.c
new file mode 100644
index 00000000000..6e4990c71f5
--- /dev/null
+++ b/xlators/cluster/ec/src/ec-galois.c
@@ -0,0 +1,183 @@
+/*
+  Copyright (c) 2015 DataLab, s.l. <http://www.datalab.es>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
+*/
+
+#include <string.h>
+
+#include "ec-mem-types.h"
+#include "ec-gf8.h"
+#include "ec-helpers.h"
+
+static ec_gf_t *
+ec_gf_alloc(uint32_t bits, uint32_t mod)
+{
+    ec_gf_t *gf;
+
+    gf = GF_MALLOC(sizeof(ec_gf_t), ec_mt_ec_gf_t);
+    if (gf == NULL) {
+        goto failed;
+    }
+
+    gf->bits = bits;
+    gf->size = 1 << bits;
+    gf->mod = mod;
+
+    gf->log = GF_MALLOC(sizeof(uint32_t) * (gf->size * 2 - 1),
+                        gf_common_mt_int);
+    if (gf->log == NULL) {
+        goto failed_gf;
+    }
+    gf->pow = GF_MALLOC(sizeof(uint32_t) * (gf->size * 2 - 1),
+                        gf_common_mt_int);
+    if (gf->pow == NULL) {
+        goto failed_log;
+    }
+
+    return gf;
+
+failed_log:
+    GF_FREE(gf->log);
+failed_gf:
+    GF_FREE(gf);
+failed:
+    return EC_ERR(ENOMEM);
+}
+
+static void
+ec_gf_init_tables(ec_gf_t *gf)
+{
+    uint32_t i, tmp;
+
+    memset(gf->log, -1, sizeof(uint32_t) * gf->size);
+
+    gf->pow[0] = 1;
+    gf->log[0] = gf->size;
+    gf->log[1] = 0;
+    for (i = 1; i < gf->size; i++) {
+        tmp = gf->pow[i - 1] << 1;
+        if (tmp >= gf->size) {
+            tmp ^= gf->mod;
+        }
+        gf->pow[i + gf->size - 1] = gf->pow[i] = tmp;
+        gf->log[tmp + gf->size - 1] = gf->log[tmp] = i;
+    }
+}
+
+ec_gf_t *
+ec_gf_prepare(uint32_t bits, uint32_t mod)
+{
+    ec_gf_mul_t **tbl;
+    ec_gf_t *gf;
+    uint32_t i, j;
+
+    if (bits != 8) {
+        return EC_ERR(EINVAL);
+    }
+
+    tbl = ec_gf8_mul;
+    if (mod == 0) {
+        mod = 0x11d;
+    }
+
+    gf = ec_gf_alloc(bits, mod);
+    if (EC_IS_ERR(gf)) {
+        return gf;
+    }
+    ec_gf_init_tables(gf);
+
+    gf->table = tbl;
+    gf->min_ops = bits * bits;
+    gf->max_ops = 0;
+    gf->avg_ops = 0;
+    for (i = 1; i < gf->size; i++) {
+        for (j = 0; tbl[i]->ops[j].op != EC_GF_OP_END; j++) {
+        }
+        if (gf->max_ops < j) {
+            gf->max_ops = j;
+        }
+        if (gf->min_ops > j) {
+            gf->min_ops = j;
+        }
+        gf->avg_ops += j;
+    }
+    gf->avg_ops /= gf->size;
+
+    return gf;
+}
+
+void
+ec_gf_destroy(ec_gf_t *gf)
+{
+    GF_FREE(gf->pow);
+    GF_FREE(gf->log);
+    GF_FREE(gf);
+}
+
+uint32_t
+ec_gf_add(ec_gf_t *gf, uint32_t a, uint32_t b)
+{
+    if ((a >= gf->size) || (b >= gf->size)) {
+        return gf->size;
+    }
+
+    return a ^ b;
+}
+
+uint32_t
+ec_gf_mul(ec_gf_t *gf, uint32_t a, uint32_t b)
+{
+    if ((a >= gf->size) || (b >= gf->size)) {
+        return gf->size;
+    }
+
+    if ((a != 0) && (b != 0)) {
+        return gf->pow[gf->log[a] + gf->log[b]];
+    }
+
+    return 0;
+}
+
+uint32_t
+ec_gf_div(ec_gf_t *gf, uint32_t a, uint32_t b)
+{
+    if ((a >= gf->size) || (b >= gf->size)) {
+        return gf->size;
+    }
+
+    if (b != 0) {
+        if (a != 0) {
+            return gf->pow[gf->size - 1 + gf->log[a] - gf->log[b]];
+        }
+
+        return 0;
+    }
+
+    return gf->size;
+}
+
+uint32_t
+ec_gf_exp(ec_gf_t *gf, uint32_t a, uint32_t b)
+{
+    uint32_t r;
+
+    if ((a >= gf->size) || ((a == 0) && (b == 0))) {
+        return gf->size;
+    }
+
+    r = 1;
+    while (b != 0) {
+        if ((b & 1) != 0) {
+            r = ec_gf_mul(gf, r, a);
+        }
+        a = ec_gf_mul(gf, a, a);
+        b >>= 1;
+    }
+
+    return r;
+}
diff --git a/xlators/cluster/ec/src/ec-galois.h b/xlators/cluster/ec/src/ec-galois.h
new file mode 100644
index 00000000000..ed55d53e419
--- /dev/null
+++ b/xlators/cluster/ec/src/ec-galois.h
@@ -0,0 +1,32 @@
+/*
+  Copyright (c) 2015 DataLab, s.l. <http://www.datalab.es>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
+*/
+
+#ifndef __EC_GALOIS_H__
+#define __EC_GALOIS_H__
+
+#include <inttypes.h>
+
+#include "ec-types.h"
+
+ec_gf_t *
+ec_gf_prepare(uint32_t bits, uint32_t mod);
+void
+ec_gf_destroy(ec_gf_t *gf);
+
+uint32_t
+ec_gf_add(ec_gf_t *gf, uint32_t a, uint32_t b);
+uint32_t
+ec_gf_mul(ec_gf_t *gf, uint32_t a, uint32_t b);
+uint32_t
+ec_gf_div(ec_gf_t *gf, uint32_t a, uint32_t b);
+uint32_t
+ec_gf_exp(ec_gf_t *gf, uint32_t a, uint32_t b);
+
+#endif /* __EC_GALOIS_H__ */
diff --git a/xlators/cluster/ec/src/ec-generic.c b/xlators/cluster/ec/src/ec-generic.c
index 73df0d89db8..884deb93669 100644
--- a/xlators/cluster/ec/src/ec-generic.c
+++ b/xlators/cluster/ec/src/ec-generic.c
@@ -8,22 +8,23 @@
   cases as published by the Free Software Foundation.
 */
 
-#include "xlator.h"
-#include "defaults.h"
+#include <glusterfs/byte-order.h>
 
+#include "ec.h"
+#include "ec-messages.h"
 #include "ec-helpers.h"
 #include "ec-common.h"
 #include "ec-combine.h"
-#include "ec-method.h"
 #include "ec-fops.h"
 
 /* FOP: flush */
 
-int32_t ec_flush_cbk(call_frame_t * frame, void * cookie, xlator_t * this,
-                     int32_t op_ret, int32_t op_errno, dict_t * xdata)
+int32_t
+ec_flush_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int32_t op_ret,
+             int32_t op_errno, dict_t *xdata)
 {
-    ec_fop_data_t * fop = NULL;
-    ec_cbk_data_t * cbk = NULL;
+    ec_fop_data_t *fop = NULL;
+    ec_cbk_data_t *cbk = NULL;
     int32_t idx = (int32_t)(uintptr_t)cookie;
 
     VALIDATE_OR_GOTO(this, out);
@@ -33,20 +34,18 @@ int32_t ec_flush_cbk(call_frame_t * frame, void * cookie, xlator_t * this,
 
     fop = frame->local;
 
-    ec_trace("CBK", fop, "idx=%d, frame=%p, op_ret=%d, op_errno=%d", idx,
-             frame, op_ret, op_errno);
+    ec_trace("CBK", fop, "idx=%d, frame=%p, op_ret=%d, op_errno=%d", idx, frame,
+             op_ret, op_errno);
 
     cbk = ec_cbk_data_allocate(frame, this, fop, GF_FOP_FLUSH, idx, op_ret,
                                op_errno);
-    if (cbk != NULL)
-    {
-        if (xdata != NULL)
-        {
+    if (cbk != NULL) {
+        if (xdata != NULL) {
             cbk->xdata = dict_ref(xdata);
-            if (cbk->xdata == NULL)
-            {
-                gf_log(this->name, GF_LOG_ERROR, "Failed to reference a "
-                                                 "dictionary.");
+            if (cbk->xdata == NULL) {
+                gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_DICT_REF_FAIL,
+                       "Failed to reference a "
+                       "dictionary.");
 
                 goto out;
             }
@@ -56,15 +55,15 @@ int32_t ec_flush_cbk(call_frame_t * frame, void * cookie, xlator_t * this,
     }
 
 out:
-    if (fop != NULL)
-    {
+    if (fop != NULL) {
         ec_complete(fop);
     }
 
     return 0;
 }
 
-void ec_wind_flush(ec_t * ec, ec_fop_data_t * fop, int32_t idx)
+void
+ec_wind_flush(ec_t *ec, ec_fop_data_t *fop, int32_t idx)
 {
     ec_trace("WIND", fop, "idx=%d", idx);
 
@@ -73,15 +72,15 @@ void ec_wind_flush(ec_t * ec, ec_fop_data_t * fop, int32_t idx)
                       fop->xdata);
 }
 
-int32_t ec_manager_flush(ec_fop_data_t * fop, int32_t state)
+int32_t
+ec_manager_flush(ec_fop_data_t *fop, int32_t state)
 {
-    ec_cbk_data_t * cbk;
+    ec_cbk_data_t *cbk;
 
-    switch (state)
-    {
+    switch (state) {
         case EC_STATE_INIT:
         case EC_STATE_LOCK:
-            ec_lock_prepare_fd(fop, fop->fd, 0);
+            ec_lock_prepare_fd(fop, fop->fd, 0, 0, EC_RANGE_FULL);
             ec_lock(fop);
 
             return EC_STATE_DISPATCH;
@@ -97,26 +96,7 @@ int32_t ec_manager_flush(ec_fop_data_t * fop, int32_t state)
             return EC_STATE_PREPARE_ANSWER;
 
         case EC_STATE_PREPARE_ANSWER:
-            cbk = fop->answer;
-            if (cbk != NULL)
-            {
-                if (!ec_dict_combine(cbk, EC_COMBINE_XDATA))
-                {
-                    if (cbk->op_ret >= 0)
-                    {
-                        cbk->op_ret = -1;
-                        cbk->op_errno = EIO;
-                    }
-                }
-                if (cbk->op_ret < 0)
-                {
-                    ec_fop_set_error(fop, cbk->op_errno);
-                }
-            }
-            else
-            {
-                ec_fop_set_error(fop, EIO);
-            }
+            ec_fop_prepare_answer(fop, _gf_false);
 
             return EC_STATE_REPORT;
 
@@ -125,8 +105,7 @@ int32_t ec_manager_flush(ec_fop_data_t * fop, int32_t state)
 
             GF_ASSERT(cbk != NULL);
 
-            if (fop->cbks.flush != NULL)
-            {
+            if (fop->cbks.flush != NULL) {
                 fop->cbks.flush(fop->req_frame, fop, fop->xl, cbk->op_ret,
                                 cbk->op_errno, cbk->xdata);
             }
@@ -135,13 +114,13 @@ int32_t ec_manager_flush(ec_fop_data_t * fop, int32_t state)
 
         case -EC_STATE_INIT:
         case -EC_STATE_LOCK:
+        case -EC_STATE_DELAYED_START:
         case -EC_STATE_DISPATCH:
         case -EC_STATE_PREPARE_ANSWER:
         case -EC_STATE_REPORT:
             GF_ASSERT(fop->error != 0);
 
-            if (fop->cbks.flush != NULL)
-            {
+            if (fop->cbks.flush != NULL) {
                 fop->cbks.flush(fop->req_frame, fop, fop->xl, -1, fop->error,
                                 NULL);
             }
@@ -161,55 +140,93 @@ int32_t ec_manager_flush(ec_fop_data_t * fop, int32_t state)
             return EC_STATE_END;
 
         default:
-            gf_log(fop->xl->name, GF_LOG_ERROR, "Unhandled state %d for %s",
-                   state, ec_fop_name(fop->id));
+            gf_msg(fop->xl->name, GF_LOG_ERROR, EINVAL, EC_MSG_UNHANDLED_STATE,
+                   "Unhandled state %d for %s", state, ec_fop_name(fop->id));
 
             return EC_STATE_END;
     }
 }
 
-void ec_flush(call_frame_t * frame, xlator_t * this, uintptr_t target,
-              int32_t minimum, fop_flush_cbk_t func, void * data, fd_t * fd,
-              dict_t * xdata)
+static int32_t
+ec_validate_fd(fd_t *fd, xlator_t *xl)
+{
+    uint64_t iversion = 0;
+    uint64_t fversion = 0;
+    ec_inode_t *inode_ctx = NULL;
+    ec_fd_t *fd_ctx = NULL;
+
+    LOCK(&fd->lock);
+    {
+        fd_ctx = __ec_fd_get(fd, xl);
+        if (fd_ctx) {
+            fversion = fd_ctx->bad_version;
+        }
+    }
+    UNLOCK(&fd->lock);
+
+    LOCK(&fd->inode->lock);
+    {
+        inode_ctx = __ec_inode_get(fd->inode, xl);
+        if (inode_ctx) {
+            iversion = inode_ctx->bad_version;
+        }
+    }
+    UNLOCK(&fd->inode->lock);
+    if (fversion < iversion) {
+        return EBADF;
+    }
+    return 0;
+}
+
+void
+ec_flush(call_frame_t *frame, xlator_t *this, uintptr_t target,
+         uint32_t fop_flags, fop_flush_cbk_t func, void *data, fd_t *fd,
+         dict_t *xdata)
 {
-    ec_cbk_t callback = { .flush = func };
-    ec_fop_data_t * fop = NULL;
-    int32_t error = EIO;
+    ec_cbk_t callback = {.flush = func};
+    ec_fop_data_t *fop = NULL;
+    int32_t error = ENOMEM;
 
-    gf_log("ec", GF_LOG_TRACE, "EC(FLUSH) %p", frame);
+    gf_msg_trace("ec", 0, "EC(FLUSH) %p", frame);
 
     VALIDATE_OR_GOTO(this, out);
     GF_VALIDATE_OR_GOTO(this->name, frame, out);
     GF_VALIDATE_OR_GOTO(this->name, this->private, out);
 
-    fop = ec_fop_data_allocate(frame, this, GF_FOP_FLUSH, EC_FLAG_UPDATE_FD,
-                               target, minimum, ec_wind_flush,
-                               ec_manager_flush, callback, data);
-    if (fop == NULL)
-    {
+    if (fd) {
+        error = ec_validate_fd(fd, this);
+        if (error) {
+            gf_msg(this->name, GF_LOG_ERROR, EBADF, EC_MSG_FD_BAD,
+                   "Failing %s on %s", gf_fop_list[GF_FOP_FLUSH],
+                   fd->inode ? uuid_utoa(fd->inode->gfid) : "");
+            goto out;
+        }
+    }
+
+    fop = ec_fop_data_allocate(frame, this, GF_FOP_FLUSH, 0, target, fop_flags,
+                               ec_wind_flush, ec_manager_flush, callback, data);
+    if (fop == NULL) {
         goto out;
     }
 
     fop->use_fd = 1;
 
-    if (fd != NULL)
-    {
+    if (fd != NULL) {
         fop->fd = fd_ref(fd);
-        if (fop->fd == NULL)
-        {
-            gf_log(this->name, GF_LOG_ERROR, "Failed to reference a "
-                                             "file descriptor.");
+        if (fop->fd == NULL) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_FILE_DESC_REF_FAIL,
+                   "Failed to reference a "
+                   "file descriptor.");
 
             goto out;
         }
     }
-    if (xdata != NULL)
-    {
+    if (xdata != NULL) {
         fop->xdata = dict_ref(xdata);
-        if (fop->xdata == NULL)
-        {
-            gf_log(this->name, GF_LOG_ERROR, "Failed to reference a "
-                                             "dictionary.");
+        if (fop->xdata == NULL) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_DICT_REF_FAIL,
+                   "Failed to reference a "
+                   "dictionary.");
 
             goto out;
         }
@@ -218,25 +235,22 @@ void ec_flush(call_frame_t * frame, xlator_t * this, uintptr_t target,
     error = 0;
 
 out:
-    if (fop != NULL)
-    {
+    if (fop != NULL) {
         ec_manager(fop, error);
-    }
-    else
-    {
-        func(frame, NULL, this, -1, EIO, NULL);
+    } else {
+        func(frame, NULL, this, -1, error, NULL);
     }
 }
 
 /* FOP: fsync */
 
-int32_t ec_combine_fsync(ec_fop_data_t * fop, ec_cbk_data_t * dst,
-                         ec_cbk_data_t * src)
+int32_t
+ec_combine_fsync(ec_fop_data_t *fop, ec_cbk_data_t *dst, ec_cbk_data_t *src)
 {
-    if (!ec_iatt_combine(dst->iatt, src->iatt, 2))
-    {
-        gf_log(fop->xl->name, GF_LOG_NOTICE, "Mismatching iatt in "
-                                             "answers of 'GF_FOP_FSYNC'");
+    if (!ec_iatt_combine(fop, dst->iatt, src->iatt, 2)) {
+        gf_msg(fop->xl->name, GF_LOG_NOTICE, 0, EC_MSG_IATT_MISMATCH,
+               "Mismatching iatt in "
+               "answers of 'GF_FOP_FSYNC'");
 
         return 0;
     }
@@ -244,12 +258,13 @@ int32_t ec_combine_fsync(ec_fop_data_t * fop, ec_cbk_data_t * dst,
     return 1;
 }
 
-int32_t ec_fsync_cbk(call_frame_t * frame, void * cookie, xlator_t * this,
-                     int32_t op_ret, int32_t op_errno, struct iatt * prebuf,
-                     struct iatt * postbuf, dict_t * xdata)
+int32_t
+ec_fsync_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int32_t op_ret,
+             int32_t op_errno, struct iatt *prebuf, struct iatt *postbuf,
+             dict_t *xdata)
 {
-    ec_fop_data_t * fop = NULL;
-    ec_cbk_data_t * cbk = NULL;
+    ec_fop_data_t *fop = NULL;
+    ec_cbk_data_t *cbk = NULL;
     int32_t idx = (int32_t)(uintptr_t)cookie;
 
     VALIDATE_OR_GOTO(this, out);
@@ -259,31 +274,26 @@ int32_t ec_fsync_cbk(call_frame_t * frame, void * cookie, xlator_t * this,
 
     fop = frame->local;
 
-    ec_trace("CBK", fop, "idx=%d, frame=%p, op_ret=%d, op_errno=%d", idx,
-             frame, op_ret, op_errno);
+    ec_trace("CBK", fop, "idx=%d, frame=%p, op_ret=%d, op_errno=%d", idx, frame,
+             op_ret, op_errno);
 
     cbk = ec_cbk_data_allocate(frame, this, fop, GF_FOP_FSYNC, idx, op_ret,
                                op_errno);
-    if (cbk != NULL)
-    {
-        if (op_ret >= 0)
-        {
-            if (prebuf != NULL)
-            {
+    if (cbk != NULL) {
+        if (op_ret >= 0) {
+            if (prebuf != NULL) {
                 cbk->iatt[0] = *prebuf;
             }
-            if (postbuf != NULL)
-            {
+            if (postbuf != NULL) {
                 cbk->iatt[1] = *postbuf;
             }
         }
-        if (xdata != NULL)
-        {
+        if (xdata != NULL) {
             cbk->xdata = dict_ref(xdata);
-            if (cbk->xdata == NULL)
-            {
-                gf_log(this->name, GF_LOG_ERROR, "Failed to reference a "
-                                                 "dictionary.");
+            if (cbk->xdata == NULL) {
+                gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_DICT_REF_FAIL,
+                       "Failed to reference a "
+                       "dictionary.");
 
                 goto out;
             }
@@ -293,15 +303,15 @@ int32_t ec_fsync_cbk(call_frame_t * frame, void * cookie, xlator_t * this,
     }
 
 out:
-    if (fop != NULL)
-    {
+    if (fop != NULL) {
         ec_complete(fop);
     }
 
     return 0;
 }
 
-void ec_wind_fsync(ec_t * ec, ec_fop_data_t * fop, int32_t idx)
+void
+ec_wind_fsync(ec_t *ec, ec_fop_data_t *fop, int32_t idx)
 {
     ec_trace("WIND", fop, "idx=%d", idx);
 
@@ -310,22 +320,17 @@ void ec_wind_fsync(ec_t * ec, ec_fop_data_t * fop, int32_t idx)
                       fop->int32, fop->xdata);
 }
 
-int32_t ec_manager_fsync(ec_fop_data_t * fop, int32_t state)
+int32_t
+ec_manager_fsync(ec_fop_data_t *fop, int32_t state)
 {
-    ec_cbk_data_t * cbk;
+    ec_cbk_data_t *cbk;
 
-    switch (state)
-    {
+    switch (state) {
         case EC_STATE_INIT:
         case EC_STATE_LOCK:
-            ec_lock_prepare_fd(fop, fop->fd, 0);
+            ec_lock_prepare_fd(fop, fop->fd, EC_QUERY_INFO, 0, EC_RANGE_FULL);
             ec_lock(fop);
 
-            return EC_STATE_GET_SIZE_AND_VERSION;
-
-        case EC_STATE_GET_SIZE_AND_VERSION:
-            ec_get_size_version(fop);
-
             return EC_STATE_DISPATCH;
 
         case EC_STATE_DISPATCH:
@@ -339,33 +344,14 @@ int32_t ec_manager_fsync(ec_fop_data_t * fop, int32_t state)
             return EC_STATE_PREPARE_ANSWER;
 
         case EC_STATE_PREPARE_ANSWER:
-            cbk = fop->answer;
-            if (cbk != NULL)
-            {
-                if (!ec_dict_combine(cbk, EC_COMBINE_XDATA))
-                {
-                    if (cbk->op_ret >= 0)
-                    {
-                        cbk->op_ret = -1;
-                        cbk->op_errno = EIO;
-                    }
-                }
-                if (cbk->op_ret < 0)
-                {
-                    ec_fop_set_error(fop, cbk->op_errno);
-                }
-                else
-                {
-                    ec_iatt_rebuild(fop->xl->private, cbk->iatt, 2,
-                                    cbk->count);
-
-                    cbk->iatt[0].ia_size = fop->pre_size;
-                    cbk->iatt[1].ia_size = fop->post_size;
-                }
-            }
-            else
-            {
-                ec_fop_set_error(fop, EIO);
+            cbk = ec_fop_prepare_answer(fop, _gf_false);
+            if (cbk != NULL) {
+                ec_iatt_rebuild(fop->xl->private, cbk->iatt, 2, cbk->count);
+
+                /* This shouldn't fail because we have the inode locked. */
+                GF_ASSERT(ec_get_inode_size(fop, fop->fd->inode,
+                                            &cbk->iatt[0].ia_size));
+                cbk->iatt[1].ia_size = cbk->iatt[0].ia_size;
             }
 
             return EC_STATE_REPORT;
@@ -375,8 +361,7 @@ int32_t ec_manager_fsync(ec_fop_data_t * fop, int32_t state)
 
             GF_ASSERT(cbk != NULL);
 
-            if (fop->cbks.fsync != NULL)
-            {
+            if (fop->cbks.fsync != NULL) {
                 fop->cbks.fsync(fop->req_frame, fop, fop->xl, cbk->op_ret,
                                 cbk->op_errno, &cbk->iatt[0], &cbk->iatt[1],
                                 cbk->xdata);
@@ -386,14 +371,13 @@ int32_t ec_manager_fsync(ec_fop_data_t * fop, int32_t state)
 
         case -EC_STATE_INIT:
         case -EC_STATE_LOCK:
-        case -EC_STATE_GET_SIZE_AND_VERSION:
         case -EC_STATE_DISPATCH:
         case -EC_STATE_PREPARE_ANSWER:
         case -EC_STATE_REPORT:
+        case -EC_STATE_DELAYED_START:
             GF_ASSERT(fop->error != 0);
 
-            if (fop->cbks.fsync != NULL)
-            {
+            if (fop->cbks.fsync != NULL) {
                 fop->cbks.fsync(fop->req_frame, fop, fop->xl, -1, fop->error,
                                 NULL, NULL, NULL);
             }
@@ -413,32 +397,41 @@ int32_t ec_manager_fsync(ec_fop_data_t * fop, int32_t state)
             return EC_STATE_END;
 
         default:
-            gf_log(fop->xl->name, GF_LOG_ERROR, "Unhandled state %d for %s",
-                   state, ec_fop_name(fop->id));
+            gf_msg(fop->xl->name, GF_LOG_ERROR, EINVAL, EC_MSG_UNHANDLED_STATE,
+                   "Unhandled state %d for %s", state, ec_fop_name(fop->id));
 
             return EC_STATE_END;
     }
 }
 
-void ec_fsync(call_frame_t * frame, xlator_t * this, uintptr_t target,
-              int32_t minimum, fop_fsync_cbk_t func, void * data, fd_t * fd,
-              int32_t datasync, dict_t * xdata)
+void
+ec_fsync(call_frame_t *frame, xlator_t *this, uintptr_t target,
+         uint32_t fop_flags, fop_fsync_cbk_t func, void *data, fd_t *fd,
+         int32_t datasync, dict_t *xdata)
 {
-    ec_cbk_t callback = { .fsync = func };
-    ec_fop_data_t * fop = NULL;
-    int32_t error = EIO;
+    ec_cbk_t callback = {.fsync = func};
+    ec_fop_data_t *fop = NULL;
+    int32_t error = ENOMEM;
 
-    gf_log("ec", GF_LOG_TRACE, "EC(FSYNC) %p", frame);
+    gf_msg_trace("ec", 0, "EC(FSYNC) %p", frame);
 
     VALIDATE_OR_GOTO(this, out);
     GF_VALIDATE_OR_GOTO(this->name, frame, out);
     GF_VALIDATE_OR_GOTO(this->name, this->private, out);
 
-    fop = ec_fop_data_allocate(frame, this, GF_FOP_FSYNC, EC_FLAG_UPDATE_FD,
-                               target, minimum, ec_wind_fsync,
-                               ec_manager_fsync, callback, data);
-    if (fop == NULL)
-    {
+    if (fd) {
+        error = ec_validate_fd(fd, this);
+        if (error) {
+            gf_msg(this->name, GF_LOG_ERROR, EBADF, EC_MSG_FD_BAD,
+                   "Failing %s on %s", gf_fop_list[GF_FOP_FSYNC],
+                   fd->inode ? uuid_utoa(fd->inode->gfid) : "");
+            goto out;
+        }
+    }
+
+    fop = ec_fop_data_allocate(frame, this, GF_FOP_FSYNC, 0, target, fop_flags,
+                               ec_wind_fsync, ec_manager_fsync, callback, data);
+    if (fop == NULL) {
         goto out;
     }
 
@@ -446,24 +439,22 @@ void ec_fsync(call_frame_t * frame, xlator_t * this, uintptr_t target,
 
     fop->int32 = datasync;
 
-    if (fd != NULL)
-    {
+    if (fd != NULL) {
         fop->fd = fd_ref(fd);
-        if (fop->fd == NULL)
-        {
-            gf_log(this->name, GF_LOG_ERROR, "Failed to reference a "
-                                             "file descriptor.");
+        if (fop->fd == NULL) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_FILE_DESC_REF_FAIL,
+                   "Failed to reference a "
+                   "file descriptor.");
 
             goto out;
         }
     }
-    if (xdata != NULL)
-    {
+    if (xdata != NULL) {
         fop->xdata = dict_ref(xdata);
-        if (fop->xdata == NULL)
-        {
-            gf_log(this->name, GF_LOG_ERROR, "Failed to reference a "
-                                             "dictionary.");
+        if (fop->xdata == NULL) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_DICT_REF_FAIL,
+                   "Failed to reference a "
+                   "dictionary.");
 
             goto out;
         }
@@ -472,23 +463,21 @@ void ec_fsync(call_frame_t * frame, xlator_t * this, uintptr_t target,
     error = 0;
 
 out:
-    if (fop != NULL)
-    {
+    if (fop != NULL) {
         ec_manager(fop, error);
-    }
-    else
-    {
-        func(frame, NULL, this, -1, EIO, NULL, NULL, NULL);
+    } else {
+        func(frame, NULL, this, -1, error, NULL, NULL, NULL);
     }
 }
 
 /* FOP: fsyncdir */
 
-int32_t ec_fsyncdir_cbk(call_frame_t * frame, void * cookie, xlator_t * this,
-                        int32_t op_ret, int32_t op_errno, dict_t * xdata)
+int32_t
+ec_fsyncdir_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
-    ec_fop_data_t * fop = NULL;
-    ec_cbk_data_t * cbk = NULL;
+    ec_fop_data_t *fop = NULL;
+    ec_cbk_data_t *cbk = NULL;
     int32_t idx = (int32_t)(uintptr_t)cookie;
 
     VALIDATE_OR_GOTO(this, out);
@@ -498,20 +487,18 @@ int32_t ec_fsyncdir_cbk(call_frame_t * frame, void * cookie, xlator_t * this,
 
     fop = frame->local;
 
-    ec_trace("CBK", fop, "idx=%d, frame=%p, op_ret=%d, op_errno=%d", idx,
-             frame, op_ret, op_errno);
+    ec_trace("CBK", fop, "idx=%d, frame=%p, op_ret=%d, op_errno=%d", idx, frame,
+             op_ret, op_errno);
 
     cbk = ec_cbk_data_allocate(frame, this, fop, GF_FOP_FSYNCDIR, idx, op_ret,
                                op_errno);
-    if (cbk != NULL)
-    {
-        if (xdata != NULL)
-        {
+    if (cbk != NULL) {
+        if (xdata != NULL) {
             cbk->xdata = dict_ref(xdata);
-            if (cbk->xdata == NULL)
-            {
-                gf_log(this->name, GF_LOG_ERROR, "Failed to reference a "
-                                                 "dictionary.");
+            if (cbk->xdata == NULL) {
+                gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_DICT_REF_FAIL,
+                       "Failed to reference a "
+                       "dictionary.");
 
                 goto out;
             }
@@ -521,15 +508,15 @@ int32_t ec_fsyncdir_cbk(call_frame_t * frame, void * cookie, xlator_t * this,
     }
 
 out:
-    if (fop != NULL)
-    {
+    if (fop != NULL) {
         ec_complete(fop);
     }
 
     return 0;
 }
 
-void ec_wind_fsyncdir(ec_t * ec, ec_fop_data_t * fop, int32_t idx)
+void
+ec_wind_fsyncdir(ec_t *ec, ec_fop_data_t *fop, int32_t idx)
 {
     ec_trace("WIND", fop, "idx=%d", idx);
 
@@ -538,15 +525,15 @@ void ec_wind_fsyncdir(ec_t * ec, ec_fop_data_t * fop, int32_t idx)
                       fop->fd, fop->int32, fop->xdata);
 }
 
-int32_t ec_manager_fsyncdir(ec_fop_data_t * fop, int32_t state)
+int32_t
+ec_manager_fsyncdir(ec_fop_data_t *fop, int32_t state)
 {
-    ec_cbk_data_t * cbk;
+    ec_cbk_data_t *cbk;
 
-    switch (state)
-    {
+    switch (state) {
         case EC_STATE_INIT:
         case EC_STATE_LOCK:
-            ec_lock_prepare_fd(fop, fop->fd, 0);
+            ec_lock_prepare_fd(fop, fop->fd, 0, 0, EC_RANGE_FULL);
             ec_lock(fop);
 
             return EC_STATE_DISPATCH;
@@ -562,26 +549,7 @@ int32_t ec_manager_fsyncdir(ec_fop_data_t * fop, int32_t state)
             return EC_STATE_PREPARE_ANSWER;
 
         case EC_STATE_PREPARE_ANSWER:
-            cbk = fop->answer;
-            if (cbk != NULL)
-            {
-                if (!ec_dict_combine(cbk, EC_COMBINE_XDATA))
-                {
-                    if (cbk->op_ret >= 0)
-                    {
-                        cbk->op_ret = -1;
-                        cbk->op_errno = EIO;
-                    }
-                }
-                if (cbk->op_ret < 0)
-                {
-                    ec_fop_set_error(fop, cbk->op_errno);
-                }
-            }
-            else
-            {
-                ec_fop_set_error(fop, EIO);
-            }
+            ec_fop_prepare_answer(fop, _gf_false);
 
             return EC_STATE_REPORT;
 
@@ -590,8 +558,7 @@ int32_t ec_manager_fsyncdir(ec_fop_data_t * fop, int32_t state)
 
             GF_ASSERT(cbk != NULL);
 
-            if (fop->cbks.fsyncdir != NULL)
-            {
+            if (fop->cbks.fsyncdir != NULL) {
                 fop->cbks.fsyncdir(fop->req_frame, fop, fop->xl, cbk->op_ret,
                                    cbk->op_errno, cbk->xdata);
             }
@@ -603,12 +570,12 @@ int32_t ec_manager_fsyncdir(ec_fop_data_t * fop, int32_t state)
         case -EC_STATE_DISPATCH:
         case -EC_STATE_PREPARE_ANSWER:
         case -EC_STATE_REPORT:
+        case -EC_STATE_DELAYED_START:
             GF_ASSERT(fop->error != 0);
 
-            if (fop->cbks.fsyncdir != NULL)
-            {
-                fop->cbks.fsyncdir(fop->req_frame, fop, fop->xl, -1,
-                                   fop->error, NULL);
+            if (fop->cbks.fsyncdir != NULL) {
+                fop->cbks.fsyncdir(fop->req_frame, fop, fop->xl, -1, fop->error,
+                                   NULL);
             }
 
             return EC_STATE_LOCK_REUSE;
@@ -626,32 +593,32 @@ int32_t ec_manager_fsyncdir(ec_fop_data_t * fop, int32_t state)
             return EC_STATE_END;
 
         default:
-            gf_log(fop->xl->name, GF_LOG_ERROR, "Unhandled state %d for %s",
-                   state, ec_fop_name(fop->id));
+            gf_msg(fop->xl->name, GF_LOG_ERROR, EINVAL, EC_MSG_UNHANDLED_STATE,
+                   "Unhandled state %d for %s", state, ec_fop_name(fop->id));
 
             return EC_STATE_END;
     }
 }
 
-void ec_fsyncdir(call_frame_t * frame, xlator_t * this, uintptr_t target,
-                 int32_t minimum, fop_fsyncdir_cbk_t func, void * data,
-                 fd_t * fd, int32_t datasync, dict_t * xdata)
+void
+ec_fsyncdir(call_frame_t *frame, xlator_t *this, uintptr_t target,
+            uint32_t fop_flags, fop_fsyncdir_cbk_t func, void *data, fd_t *fd,
+            int32_t datasync, dict_t *xdata)
 {
-    ec_cbk_t callback = { .fsyncdir = func };
-    ec_fop_data_t * fop = NULL;
-    int32_t error = EIO;
+    ec_cbk_t callback = {.fsyncdir = func};
+    ec_fop_data_t *fop = NULL;
+    int32_t error = ENOMEM;
 
-    gf_log("ec", GF_LOG_TRACE, "EC(FSYNCDIR) %p", frame);
+    gf_msg_trace("ec", 0, "EC(FSYNCDIR) %p", frame);
 
     VALIDATE_OR_GOTO(this, out);
     GF_VALIDATE_OR_GOTO(this->name, frame, out);
     GF_VALIDATE_OR_GOTO(this->name, this->private, out);
 
-    fop = ec_fop_data_allocate(frame, this, GF_FOP_FSYNCDIR, EC_FLAG_UPDATE_FD,
-                               target, minimum, ec_wind_fsyncdir,
-                               ec_manager_fsyncdir, callback, data);
-    if (fop == NULL)
-    {
+    fop = ec_fop_data_allocate(frame, this, GF_FOP_FSYNCDIR, 0, target,
+                               fop_flags, ec_wind_fsyncdir, ec_manager_fsyncdir,
+                               callback, data);
+    if (fop == NULL) {
         goto out;
     }
 
@@ -659,24 +626,22 @@ void ec_fsyncdir(call_frame_t * frame, xlator_t * this, uintptr_t target,
 
     fop->int32 = datasync;
 
-    if (fd != NULL)
-    {
+    if (fd != NULL) {
         fop->fd = fd_ref(fd);
-        if (fop->fd == NULL)
-        {
-            gf_log(this->name, GF_LOG_ERROR, "Failed to reference a "
-                                             "file descriptor.");
+        if (fop->fd == NULL) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_FILE_DESC_REF_FAIL,
+                   "Failed to reference a "
+                   "file descriptor.");
 
             goto out;
         }
     }
-    if (xdata != NULL)
-    {
+    if (xdata != NULL) {
         fop->xdata = dict_ref(xdata);
-        if (fop->xdata == NULL)
-        {
-            gf_log(this->name, GF_LOG_ERROR, "Failed to reference a "
-                                             "dictionary.");
+        if (fop->xdata == NULL) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_DICT_REF_FAIL,
+                   "Failed to reference a "
+                   "dictionary.");
 
             goto out;
         }
@@ -685,139 +650,66 @@ void ec_fsyncdir(call_frame_t * frame, xlator_t * this, uintptr_t target,
     error = 0;
 
 out:
-    if (fop != NULL)
-    {
+    if (fop != NULL) {
         ec_manager(fop, error);
-    }
-    else
-    {
-        func(frame, NULL, this, -1, EIO, NULL);
+    } else {
+        func(frame, NULL, this, -1, error, NULL);
     }
 }
 
 /* FOP: lookup */
 
-void ec_lookup_rebuild(ec_t * ec, ec_fop_data_t * fop, ec_cbk_data_t * cbk)
+void
+ec_lookup_rebuild(ec_t *ec, ec_fop_data_t *fop, ec_cbk_data_t *cbk)
 {
-    ec_cbk_data_t * ans = NULL;
-    ec_inode_t * ctx = NULL;
-    ec_lock_t * lock = NULL;
-    data_t * data = NULL;
-    uint8_t * buff = NULL;
+    ec_inode_t *ctx = NULL;
     uint64_t size = 0;
-    int32_t i = 0, have_size = 0;
+    int32_t have_size = 0, err;
 
-    if (cbk->op_ret < 0)
-    {
+    if (cbk->op_ret < 0) {
         return;
     }
 
-    ec_dict_del_number(cbk->xdata, EC_XATTR_VERSION, &cbk->version);
-
-    if (ec_loc_update(fop->xl, &fop->loc[0], cbk->inode, &cbk->iatt[0]) != 0) {
-        cbk->op_ret = -1;
-        cbk->op_errno = EIO;
+    ec_dict_del_array(cbk->xdata, EC_XATTR_VERSION, cbk->version,
+                      EC_VERSION_SIZE);
 
+    err = ec_loc_update(fop->xl, &fop->loc[0], cbk->inode, &cbk->iatt[0]);
+    if (ec_cbk_set_error(cbk, -err, _gf_true)) {
         return;
     }
 
     LOCK(&cbk->inode->lock);
 
     ctx = __ec_inode_get(cbk->inode, fop->xl);
-    if ((ctx != NULL) && (ctx->inode_lock != NULL))
-    {
-        lock = ctx->inode_lock;
-        cbk->version = lock->version;
-        if (lock->have_size)
-        {
-            size = lock->size;
+    if (ctx != NULL) {
+        if (ctx->have_version) {
+            cbk->version[0] = ctx->post_version[0];
+            cbk->version[1] = ctx->post_version[1];
+        }
+        if (ctx->have_size) {
+            size = ctx->post_size;
             have_size = 1;
         }
     }
 
     UNLOCK(&cbk->inode->lock);
 
-    if (cbk->iatt[0].ia_type == IA_IFREG)
-    {
-        uint8_t * blocks[cbk->count];
-        uint32_t values[cbk->count];
-
+    if (cbk->iatt[0].ia_type == IA_IFREG) {
         cbk->size = cbk->iatt[0].ia_size;
         ec_dict_del_number(cbk->xdata, EC_XATTR_SIZE, &cbk->iatt[0].ia_size);
-        if (have_size)
-        {
+        if (have_size) {
             cbk->iatt[0].ia_size = size;
         }
-
-        size = UINT64_MAX;
-        for (i = 0, ans = cbk; (ans != NULL) && (i < ec->fragments);
-             ans = ans->next)
-        {
-            if (!ans->dirty) {
-                data = dict_get(ans->xdata, GF_CONTENT_KEY);
-                if (data != NULL)
-                {
-                    values[i] = ans->idx;
-                    blocks[i] = (uint8_t *)data->data;
-                    if (size > data->len) {
-                        size = data->len;
-                    }
-                    i++;
-                }
-            }
-        }
-
-        if (i >= ec->fragments)
-        {
-            size -= size % ec->fragment_size;
-            if (size > 0)
-            {
-                buff = GF_MALLOC(size * ec->fragments, gf_common_mt_char);
-                if (buff != NULL)
-                {
-                    size = ec_method_decode(size, ec->fragments, values,
-                                            blocks, buff);
-                    if (size > fop->size)
-                    {
-                        size = fop->size;
-                    }
-                    if (size > cbk->iatt[0].ia_size)
-                    {
-                        size = cbk->iatt[0].ia_size;
-                    }
-
-                    if (dict_set_bin(cbk->xdata, GF_CONTENT_KEY, buff,
-                                     size) != 0)
-                    {
-                        GF_FREE(buff);
-                        buff = NULL;
-                        gf_log(fop->xl->name, GF_LOG_WARNING, "Lookup "
-                                                              "read-ahead "
-                                                              "failed");
-                    }
-                }
-                else
-                {
-                    gf_log(fop->xl->name, GF_LOG_WARNING, "Lookup read-ahead "
-                                                          "failed");
-                }
-            }
-        }
-
-        if (buff == NULL)
-        {
-            dict_del(cbk->xdata, GF_CONTENT_KEY);
-        }
     }
 }
 
-int32_t ec_combine_lookup(ec_fop_data_t * fop, ec_cbk_data_t * dst,
-                          ec_cbk_data_t * src)
+int32_t
+ec_combine_lookup(ec_fop_data_t *fop, ec_cbk_data_t *dst, ec_cbk_data_t *src)
 {
-    if (!ec_iatt_combine(dst->iatt, src->iatt, 2))
-    {
-        gf_log(fop->xl->name, GF_LOG_NOTICE, "Mismatching iatt in "
-                                             "answers of 'GF_FOP_LOOKUP'");
+    if (!ec_iatt_combine(fop, dst->iatt, src->iatt, 2)) {
+        gf_msg(fop->xl->name, GF_LOG_DEBUG, 0, EC_MSG_IATT_MISMATCH,
+               "Mismatching iatt in "
+               "answers of 'GF_FOP_LOOKUP'");
 
         return 0;
     }
@@ -825,14 +717,15 @@ int32_t ec_combine_lookup(ec_fop_data_t * fop, ec_cbk_data_t * dst,
     return 1;
 }
 
-int32_t ec_lookup_cbk(call_frame_t * frame, void * cookie, xlator_t * this,
-                      int32_t op_ret, int32_t op_errno, inode_t * inode,
-                      struct iatt * buf, dict_t * xdata,
-                      struct iatt * postparent)
+int32_t
+ec_lookup_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int32_t op_ret,
+              int32_t op_errno, inode_t *inode, struct iatt *buf, dict_t *xdata,
+              struct iatt *postparent)
 {
-    ec_fop_data_t * fop = NULL;
-    ec_cbk_data_t * cbk = NULL;
+    ec_fop_data_t *fop = NULL;
+    ec_cbk_data_t *cbk = NULL;
     int32_t idx = (int32_t)(uintptr_t)cookie;
+    uint64_t dirty[2] = {0};
 
     VALIDATE_OR_GOTO(this, out);
     GF_VALIDATE_OR_GOTO(this->name, frame, out);
@@ -841,65 +734,54 @@ int32_t ec_lookup_cbk(call_frame_t * frame, void * cookie, xlator_t * this,
 
     fop = frame->local;
 
-    ec_trace("CBK", fop, "idx=%d, frame=%p, op_ret=%d, op_errno=%d", idx,
-             frame, op_ret, op_errno);
+    ec_trace("CBK", fop, "idx=%d, frame=%p, op_ret=%d, op_errno=%d", idx, frame,
+             op_ret, op_errno);
 
     cbk = ec_cbk_data_allocate(frame, this, fop, GF_FOP_LOOKUP, idx, op_ret,
                                op_errno);
-    if (cbk != NULL)
-    {
-        if (op_ret >= 0)
-        {
-            if (inode != NULL)
-            {
+    if (cbk != NULL) {
+        if (op_ret >= 0) {
+            if (inode != NULL) {
                 cbk->inode = inode_ref(inode);
-                if (cbk->inode == NULL)
-                {
-                    gf_log(this->name, GF_LOG_ERROR,
+                if (cbk->inode == NULL) {
+                    gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_INODE_REF_FAIL,
                            "Failed to reference an inode.");
 
                     goto out;
                 }
             }
-            if (buf != NULL)
-            {
+            if (buf != NULL) {
                 cbk->iatt[0] = *buf;
             }
-            if (postparent != NULL)
-            {
+            if (postparent != NULL) {
                 cbk->iatt[1] = *postparent;
             }
         }
-        if (xdata != NULL)
-        {
-            uint64_t dirty;
-
+        if (xdata != NULL) {
             cbk->xdata = dict_ref(xdata);
-            if (cbk->xdata == NULL)
-            {
-                gf_log(this->name, GF_LOG_ERROR, "Failed to reference a "
-                                                 "dictionary.");
+            if (cbk->xdata == NULL) {
+                gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_DICT_REF_FAIL,
+                       "Failed to reference a "
+                       "dictionary.");
 
                 goto out;
             }
-            if (ec_dict_del_number(cbk->xdata, EC_XATTR_DIRTY, &dirty) == 0) {
-                cbk->dirty = dirty != 0;
-            }
+            ec_dict_del_array(xdata, EC_XATTR_DIRTY, dirty, EC_VERSION_SIZE);
         }
 
         ec_combine(cbk, ec_combine_lookup);
     }
 
 out:
-    if (fop != NULL)
-    {
+    if (fop != NULL) {
         ec_complete(fop);
     }
 
     return 0;
 }
 
-void ec_wind_lookup(ec_t * ec, ec_fop_data_t * fop, int32_t idx)
+void
+ec_wind_lookup(ec_t *ec, ec_fop_data_t *fop, int32_t idx)
 {
     ec_trace("WIND", fop, "idx=%d", idx);
 
@@ -908,54 +790,49 @@ void ec_wind_lookup(ec_t * ec, ec_fop_data_t * fop, int32_t idx)
                       &fop->loc[0], fop->xdata);
 }
 
-int32_t ec_manager_lookup(ec_fop_data_t * fop, int32_t state)
+int32_t
+ec_manager_lookup(ec_fop_data_t *fop, int32_t state)
 {
-    ec_cbk_data_t * cbk;
+    ec_cbk_data_t *cbk;
+    int32_t err;
 
-    switch (state)
-    {
+    switch (state) {
         case EC_STATE_INIT:
-            if (fop->xdata == NULL)
-            {
+            if (fop->xdata == NULL) {
                 fop->xdata = dict_new();
-                if (fop->xdata == NULL)
-                {
-                    gf_log(fop->xl->name, GF_LOG_ERROR, "Unable to prepare "
-                                                        "lookup request");
+                if (fop->xdata == NULL) {
+                    gf_msg(fop->xl->name, GF_LOG_ERROR, ENOMEM,
+                           EC_MSG_LOOKUP_REQ_PREP_FAIL,
+                           "Unable to prepare "
+                           "lookup request");
 
-                    fop->error = EIO;
+                    fop->error = ENOMEM;
 
                     return EC_STATE_REPORT;
                 }
+            } else {
+                /*TODO: To be handled once we have 'syndromes' */
+                dict_del(fop->xdata, GF_CONTENT_KEY);
             }
-            else
-            {
-                uint64_t size;
-
-                if (dict_get_uint64(fop->xdata, GF_CONTENT_KEY, &size) == 0)
-                {
-                    fop->size = size;
-                    size = ec_adjust_size(fop->xl->private, size, 1);
-                    if (dict_set_uint64(fop->xdata, GF_CONTENT_KEY, size) != 0)
-                    {
-                        gf_log("ec", GF_LOG_DEBUG, "Unable to update lookup "
-                                                   "content size");
-                    }
-                }
+            err = dict_set_uint64(fop->xdata, EC_XATTR_SIZE, 0);
+            if (err == 0) {
+                err = dict_set_uint64(fop->xdata, EC_XATTR_VERSION, 0);
             }
-            if ((dict_set_uint64(fop->xdata, EC_XATTR_SIZE, 0) != 0) ||
-                (dict_set_uint64(fop->xdata, EC_XATTR_VERSION, 0) != 0) ||
-                (dict_set_uint64(fop->xdata, EC_XATTR_DIRTY, 0) != 0))
-            {
-                gf_log(fop->xl->name, GF_LOG_ERROR, "Unable to prepare lookup "
-                                                    "request");
+            if (err == 0) {
+                err = dict_set_uint64(fop->xdata, EC_XATTR_DIRTY, 0);
+            }
+            if (err != 0) {
+                gf_msg(fop->xl->name, GF_LOG_ERROR, -err,
+                       EC_MSG_LOOKUP_REQ_PREP_FAIL,
+                       "Unable to prepare lookup "
+                       "request");
 
-                fop->error = EIO;
+                fop->error = -err;
 
                 return EC_STATE_REPORT;
             }
 
-        /* Fall through */
+            /* Fall through */
 
         case EC_STATE_DISPATCH:
             ec_dispatch_all(fop);
@@ -963,32 +840,23 @@ int32_t ec_manager_lookup(ec_fop_data_t * fop, int32_t state)
             return EC_STATE_PREPARE_ANSWER;
 
         case EC_STATE_PREPARE_ANSWER:
-            cbk = fop->answer;
-            if (cbk != NULL)
-            {
-                if (!ec_dict_combine(cbk, EC_COMBINE_XDATA))
-                {
-                    if (cbk->op_ret >= 0)
-                    {
-                        cbk->op_ret = -1;
-                        cbk->op_errno = EIO;
-                    }
-                }
-                if (cbk->op_ret < 0)
-                {
-                    ec_fop_set_error(fop, cbk->op_errno);
-                }
-                else
-                {
-                    ec_iatt_rebuild(fop->xl->private, cbk->iatt, 2,
-                                    cbk->count);
-
-                    ec_lookup_rebuild(fop->xl->private, fop, cbk);
-                }
+            /*
+             * Lookup happens without any lock, so there is a chance that it
+             * will have answers before modification happened and after
+             * modification happened in the same response. So choose the next
+             * best answer when the answers don't match for EC_MINIMUM_MIN
+             */
+
+            if (!fop->answer && !list_empty(&fop->cbk_list)) {
+                fop->answer = list_entry(fop->cbk_list.next, ec_cbk_data_t,
+                                         list);
             }
-            else
-            {
-                ec_fop_set_error(fop, EIO);
+
+            cbk = ec_fop_prepare_answer(fop, _gf_true);
+            if (cbk != NULL) {
+                ec_iatt_rebuild(fop->xl->private, cbk->iatt, 2, cbk->count);
+
+                ec_lookup_rebuild(fop->xl->private, fop, cbk);
             }
 
             return EC_STATE_REPORT;
@@ -998,8 +866,7 @@ int32_t ec_manager_lookup(ec_fop_data_t * fop, int32_t state)
 
             GF_ASSERT(cbk != NULL);
 
-            if (fop->cbks.lookup != NULL)
-            {
+            if (fop->cbks.lookup != NULL) {
                 fop->cbks.lookup(fop->req_frame, fop, fop->xl, cbk->op_ret,
                                  cbk->op_errno, cbk->inode, &cbk->iatt[0],
                                  cbk->xdata, &cbk->iatt[1]);
@@ -1013,8 +880,7 @@ int32_t ec_manager_lookup(ec_fop_data_t * fop, int32_t state)
         case -EC_STATE_REPORT:
             GF_ASSERT(fop->error != 0);
 
-            if (fop->cbks.lookup != NULL)
-            {
+            if (fop->cbks.lookup != NULL) {
                 fop->cbks.lookup(fop->req_frame, fop, fop->xl, -1, fop->error,
                                  NULL, NULL, NULL, NULL);
             }
@@ -1022,85 +888,77 @@ int32_t ec_manager_lookup(ec_fop_data_t * fop, int32_t state)
             return EC_STATE_END;
 
         default:
-            gf_log(fop->xl->name, GF_LOG_ERROR, "Unhandled state %d for %s",
-                   state, ec_fop_name(fop->id));
+            gf_msg(fop->xl->name, GF_LOG_ERROR, EINVAL, EC_MSG_UNHANDLED_STATE,
+                   "Unhandled state %d for %s", state, ec_fop_name(fop->id));
 
             return EC_STATE_END;
     }
 }
 
-void ec_lookup(call_frame_t * frame, xlator_t * this, uintptr_t target,
-               int32_t minimum, fop_lookup_cbk_t func, void * data,
-               loc_t * loc, dict_t * xdata)
+void
+ec_lookup(call_frame_t *frame, xlator_t *this, uintptr_t target,
+          uint32_t fop_flags, fop_lookup_cbk_t func, void *data, loc_t *loc,
+          dict_t *xdata)
 {
-    ec_cbk_t callback = { .lookup = func };
-    ec_fop_data_t * fop = NULL;
-    int32_t error = EIO;
+    ec_cbk_t callback = {.lookup = func};
+    ec_fop_data_t *fop = NULL;
+    int32_t error = ENOMEM;
 
-    gf_log("ec", GF_LOG_TRACE, "EC(LOOKUP) %p", frame);
+    gf_msg_trace("ec", 0, "EC(LOOKUP) %p", frame);
 
     VALIDATE_OR_GOTO(this, out);
     GF_VALIDATE_OR_GOTO(this->name, frame, out);
     GF_VALIDATE_OR_GOTO(this->name, this->private, out);
 
-    fop = ec_fop_data_allocate(frame, this, GF_FOP_LOOKUP, 0, target, minimum,
-                               ec_wind_lookup, ec_manager_lookup, callback,
-                               data);
-    if (fop == NULL)
-    {
+    fop = ec_fop_data_allocate(frame, this, GF_FOP_LOOKUP, EC_FLAG_LOCK_SHARED,
+                               target, fop_flags, ec_wind_lookup,
+                               ec_manager_lookup, callback, data);
+    if (fop == NULL) {
         goto out;
     }
 
-    if (loc != NULL)
-    {
-        if (loc_copy(&fop->loc[0], loc) != 0)
-        {
-            gf_log(this->name, GF_LOG_ERROR, "Failed to copy a location.");
+    if (loc != NULL) {
+        if (loc_copy(&fop->loc[0], loc) != 0) {
+            gf_msg(this->name, GF_LOG_ERROR, ENOMEM, EC_MSG_LOC_COPY_FAIL,
+                   "Failed to copy a location.");
 
             goto out;
         }
     }
-    if (xdata != NULL)
-    {
-        fop->xdata = dict_ref(xdata);
+    if (xdata != NULL) {
+        fop->xdata = dict_copy_with_ref(xdata, NULL);
+        /* Do not log failures here as a memory problem would have already
+         * been logged by the corresponding alloc functions */
         if (fop->xdata == NULL)
-        {
-            gf_log(this->name, GF_LOG_ERROR, "Failed to reference a "
-                                             "dictionary.");
-
             goto out;
-        }
     }
 
     error = 0;
 
 out:
-    if (fop != NULL)
-    {
+    if (fop != NULL) {
         ec_manager(fop, error);
-    }
-    else
-    {
-        func(frame, NULL, this, -1, EIO, NULL, NULL, NULL, NULL);
+    } else {
+        func(frame, NULL, this, -1, error, NULL, NULL, NULL, NULL);
     }
 }
 
 /* FOP: statfs */
 
-int32_t ec_combine_statfs(ec_fop_data_t * fop, ec_cbk_data_t * dst,
-                          ec_cbk_data_t * src)
+int32_t
+ec_combine_statfs(ec_fop_data_t *fop, ec_cbk_data_t *dst, ec_cbk_data_t *src)
 {
     ec_statvfs_combine(&dst->statvfs, &src->statvfs);
 
     return 1;
 }
 
-int32_t ec_statfs_cbk(call_frame_t * frame, void * cookie, xlator_t * this,
-                      int32_t op_ret, int32_t op_errno, struct statvfs * buf,
-                      dict_t * xdata)
+int32_t
+ec_statfs_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int32_t op_ret,
+              int32_t op_errno, struct statvfs *buf, dict_t *xdata)
 {
-    ec_fop_data_t * fop = NULL;
-    ec_cbk_data_t * cbk = NULL;
+    ec_fop_data_t *fop = NULL;
+    ec_cbk_data_t *cbk = NULL;
     int32_t idx = (int32_t)(uintptr_t)cookie;
 
     VALIDATE_OR_GOTO(this, out);
@@ -1110,27 +968,23 @@ int32_t ec_statfs_cbk(call_frame_t * frame, void * cookie, xlator_t * this,
 
     fop = frame->local;
 
-    ec_trace("CBK", fop, "idx=%d, frame=%p, op_ret=%d, op_errno=%d", idx,
-             frame, op_ret, op_errno);
+    ec_trace("CBK", fop, "idx=%d, frame=%p, op_ret=%d, op_errno=%d", idx, frame,
+             op_ret, op_errno);
 
     cbk = ec_cbk_data_allocate(frame, this, fop, GF_FOP_STATFS, idx, op_ret,
                                op_errno);
-    if (cbk != NULL)
-    {
-        if (op_ret >= 0)
-        {
-            if (buf != NULL)
-            {
+    if (cbk != NULL) {
+        if (op_ret >= 0) {
+            if (buf != NULL) {
                 cbk->statvfs = *buf;
             }
         }
-        if (xdata != NULL)
-        {
+        if (xdata != NULL) {
             cbk->xdata = dict_ref(xdata);
-            if (cbk->xdata == NULL)
-            {
-                gf_log(this->name, GF_LOG_ERROR, "Failed to reference a "
-                                                 "dictionary.");
+            if (cbk->xdata == NULL) {
+                gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_DICT_REF_FAIL,
+                       "Failed to reference a "
+                       "dictionary.");
 
                 goto out;
             }
@@ -1140,15 +994,15 @@ int32_t ec_statfs_cbk(call_frame_t * frame, void * cookie, xlator_t * this,
     }
 
 out:
-    if (fop != NULL)
-    {
+    if (fop != NULL) {
         ec_complete(fop);
     }
 
     return 0;
 }
 
-void ec_wind_statfs(ec_t * ec, ec_fop_data_t * fop, int32_t idx)
+void
+ec_wind_statfs(ec_t *ec, ec_fop_data_t *fop, int32_t idx)
 {
     ec_trace("WIND", fop, "idx=%d", idx);
 
@@ -1157,12 +1011,14 @@ void ec_wind_statfs(ec_t * ec, ec_fop_data_t * fop, int32_t idx)
                       &fop->loc[0], fop->xdata);
 }
 
-int32_t ec_manager_statfs(ec_fop_data_t * fop, int32_t state)
+int32_t
+ec_manager_statfs(ec_fop_data_t *fop, int32_t state)
 {
-    ec_cbk_data_t * cbk;
+    ec_cbk_data_t *cbk = NULL;
+    gf_boolean_t deem_statfs_enabled = _gf_false;
+    int32_t err = 0;
 
-    switch (state)
-    {
+    switch (state) {
         case EC_STATE_INIT:
         case EC_STATE_DISPATCH:
             ec_dispatch_all(fop);
@@ -1170,34 +1026,24 @@ int32_t ec_manager_statfs(ec_fop_data_t * fop, int32_t state)
             return EC_STATE_PREPARE_ANSWER;
 
         case EC_STATE_PREPARE_ANSWER:
-            cbk = fop->answer;
-            if (cbk != NULL)
-            {
-                if (!ec_dict_combine(cbk, EC_COMBINE_XDATA))
-                {
-                    if (cbk->op_ret >= 0)
-                    {
-                        cbk->op_ret = -1;
-                        cbk->op_errno = EIO;
+            cbk = ec_fop_prepare_answer(fop, _gf_true);
+            if (cbk != NULL) {
+                ec_t *ec = fop->xl->private;
+
+                if (cbk->xdata) {
+                    err = dict_get_int8(cbk->xdata, "quota-deem-statfs",
+                                        (int8_t *)&deem_statfs_enabled);
+                    if (err != -ENOENT) {
+                        ec_cbk_set_error(cbk, -err, _gf_true);
                     }
                 }
-                if (cbk->op_ret < 0)
-                {
-                    ec_fop_set_error(fop, cbk->op_errno);
-                }
-                else
-                {
-                    ec_t * ec = fop->xl->private;
 
+                if (err != 0 || deem_statfs_enabled == _gf_false) {
                     cbk->statvfs.f_blocks *= ec->fragments;
                     cbk->statvfs.f_bfree *= ec->fragments;
                     cbk->statvfs.f_bavail *= ec->fragments;
                 }
             }
-            else
-            {
-                ec_fop_set_error(fop, EIO);
-            }
 
             return EC_STATE_REPORT;
 
@@ -1206,8 +1052,7 @@ int32_t ec_manager_statfs(ec_fop_data_t * fop, int32_t state)
 
             GF_ASSERT(cbk != NULL);
 
-            if (fop->cbks.statfs != NULL)
-            {
+            if (fop->cbks.statfs != NULL) {
                 fop->cbks.statfs(fop->req_frame, fop, fop->xl, cbk->op_ret,
                                  cbk->op_errno, &cbk->statvfs, cbk->xdata);
             }
@@ -1220,8 +1065,7 @@ int32_t ec_manager_statfs(ec_fop_data_t * fop, int32_t state)
         case -EC_STATE_REPORT:
             GF_ASSERT(fop->error != 0);
 
-            if (fop->cbks.statfs != NULL)
-            {
+            if (fop->cbks.statfs != NULL) {
                 fop->cbks.statfs(fop->req_frame, fop, fop->xl, -1, fop->error,
                                  NULL, NULL);
             }
@@ -1229,51 +1073,49 @@ int32_t ec_manager_statfs(ec_fop_data_t * fop, int32_t state)
             return EC_STATE_END;
 
         default:
-            gf_log(fop->xl->name, GF_LOG_ERROR, "Unhandled state %d for %s",
-                   state, ec_fop_name(fop->id));
+            gf_msg(fop->xl->name, GF_LOG_ERROR, EINVAL, EC_MSG_UNHANDLED_STATE,
+                   "Unhandled state %d for %s", state, ec_fop_name(fop->id));
 
             return EC_STATE_END;
     }
 }
 
-void ec_statfs(call_frame_t * frame, xlator_t * this, uintptr_t target,
-               int32_t minimum, fop_statfs_cbk_t func, void * data,
-               loc_t * loc, dict_t * xdata)
+void
+ec_statfs(call_frame_t *frame, xlator_t *this, uintptr_t target,
+          uint32_t fop_flags, fop_statfs_cbk_t func, void *data, loc_t *loc,
+          dict_t *xdata)
 {
-    ec_cbk_t callback = { .statfs = func };
-    ec_fop_data_t * fop = NULL;
-    int32_t error = EIO;
+    ec_cbk_t callback = {.statfs = func};
+    ec_fop_data_t *fop = NULL;
+    int32_t error = ENOMEM;
 
-    gf_log("ec", GF_LOG_TRACE, "EC(STATFS) %p", frame);
+    gf_msg_trace("ec", 0, "EC(STATFS) %p", frame);
 
     VALIDATE_OR_GOTO(this, out);
     GF_VALIDATE_OR_GOTO(this->name, frame, out);
     GF_VALIDATE_OR_GOTO(this->name, this->private, out);
 
-    fop = ec_fop_data_allocate(frame, this, GF_FOP_STATFS, 0, target, minimum,
-                               ec_wind_statfs, ec_manager_statfs, callback,
-                               data);
-    if (fop == NULL)
-    {
+    fop = ec_fop_data_allocate(frame, this, GF_FOP_STATFS, EC_FLAG_LOCK_SHARED,
+                               target, fop_flags, ec_wind_statfs,
+                               ec_manager_statfs, callback, data);
+    if (fop == NULL) {
         goto out;
     }
 
-    if (loc != NULL)
-    {
-        if (loc_copy(&fop->loc[0], loc) != 0)
-        {
-            gf_log(this->name, GF_LOG_ERROR, "Failed to copy a location.");
+    if (loc != NULL) {
+        if (loc_copy(&fop->loc[0], loc) != 0) {
+            gf_msg(this->name, GF_LOG_ERROR, ENOMEM, EC_MSG_LOC_COPY_FAIL,
+                   "Failed to copy a location.");
 
             goto out;
         }
     }
-    if (xdata != NULL)
-    {
+    if (xdata != NULL) {
         fop->xdata = dict_ref(xdata);
-        if (fop->xdata == NULL)
-        {
-            gf_log(this->name, GF_LOG_ERROR, "Failed to reference a "
-                                             "dictionary.");
+        if (fop->xdata == NULL) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_DICT_REF_FAIL,
+                   "Failed to reference a "
+                   "dictionary.");
 
             goto out;
         }
@@ -1282,25 +1124,22 @@ void ec_statfs(call_frame_t * frame, xlator_t * this, uintptr_t target,
     error = 0;
 
 out:
-    if (fop != NULL)
-    {
+    if (fop != NULL) {
         ec_manager(fop, error);
-    }
-    else
-    {
-        func(frame, NULL, this, -1, EIO, NULL, NULL);
+    } else {
+        func(frame, NULL, this, -1, error, NULL, NULL);
     }
 }
 
 /* FOP: xattrop */
 
-int32_t ec_combine_xattrop(ec_fop_data_t * fop, ec_cbk_data_t * dst,
-                           ec_cbk_data_t * src)
+int32_t
+ec_combine_xattrop(ec_fop_data_t *fop, ec_cbk_data_t *dst, ec_cbk_data_t *src)
 {
-    if (!ec_dict_compare(dst->dict, src->dict))
-    {
-        gf_log(fop->xl->name, GF_LOG_NOTICE, "Mismatching dictionary in "
-                                             "answers of 'GF_FOP_XATTROP'");
+    if (!ec_dict_compare(dst->dict, src->dict)) {
+        gf_msg(fop->xl->name, GF_LOG_DEBUG, 0, EC_MSG_DICT_MISMATCH,
+               "Mismatching dictionary in "
+               "answers of 'GF_FOP_XATTROP'");
 
         return 0;
     }
@@ -1308,12 +1147,16 @@ int32_t ec_combine_xattrop(ec_fop_data_t * fop, ec_cbk_data_t * dst,
     return 1;
 }
 
-int32_t ec_xattrop_cbk(call_frame_t * frame, void * cookie, xlator_t * this,
-                       int32_t op_ret, int32_t op_errno, dict_t * xattr,
-                       dict_t * xdata)
+int32_t
+ec_xattrop_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+               int32_t op_ret, int32_t op_errno, dict_t *xattr, dict_t *xdata)
 {
-    ec_fop_data_t * fop = NULL;
-    ec_cbk_data_t * cbk = NULL;
+    ec_fop_data_t *fop = NULL;
+    ec_lock_link_t *link = NULL;
+    ec_cbk_data_t *cbk = NULL;
+    uint64_t dirty[2] = {0};
+    data_t *data;
+    uint64_t *version;
     int32_t idx = (int32_t)(uintptr_t)cookie;
 
     VALIDATE_OR_GOTO(this, out);
@@ -1323,57 +1166,53 @@ int32_t ec_xattrop_cbk(call_frame_t * frame, void * cookie, xlator_t * this,
 
     fop = frame->local;
 
-    ec_trace("CBK", fop, "idx=%d, frame=%p, op_ret=%d, op_errno=%d", idx,
-             frame, op_ret, op_errno);
+    ec_trace("CBK", fop, "idx=%d, frame=%p, op_ret=%d, op_errno=%d", idx, frame,
+             op_ret, op_errno);
 
-    cbk = ec_cbk_data_allocate(frame, this, fop, GF_FOP_XATTROP, idx, op_ret,
+    cbk = ec_cbk_data_allocate(frame, this, fop, fop->id, idx, op_ret,
                                op_errno);
-    if (cbk != NULL)
-    {
-        if (op_ret >= 0)
-        {
-            if (xattr != NULL)
-            {
-                cbk->dict = dict_ref(xattr);
-                if (cbk->dict == NULL)
-                {
-                    gf_log(this->name, GF_LOG_ERROR, "Failed to reference a "
-                                                     "dictionary.");
+    if (!cbk)
+        goto out;
 
-                    goto out;
-                }
-            }
-        }
-        if (xdata != NULL)
-        {
-            uint64_t dirty;
+    if (op_ret >= 0) {
+        cbk->dict = dict_ref(xattr);
 
-            cbk->xdata = dict_ref(xdata);
-            if (cbk->xdata == NULL)
-            {
-                gf_log(this->name, GF_LOG_ERROR, "Failed to reference a "
-                                                 "dictionary.");
+        data = dict_get(cbk->dict, EC_XATTR_VERSION);
+        if ((data != NULL) && (data->len >= sizeof(uint64_t))) {
+            version = (uint64_t *)data->data;
 
-                goto out;
-            }
-            if (ec_dict_del_number(cbk->xdata, EC_XATTR_DIRTY, &dirty) == 0) {
-                cbk->dirty = dirty != 0;
+            if (((ntoh64(version[0]) >> EC_SELFHEAL_BIT) & 1) != 0) {
+                LOCK(&fop->lock);
+
+                fop->healing |= 1ULL << idx;
+
+                UNLOCK(&fop->lock);
             }
         }
 
-        ec_combine(cbk, ec_combine_xattrop);
+        ec_dict_del_array(xattr, EC_XATTR_DIRTY, dirty, EC_VERSION_SIZE);
+        link = fop->data;
+        if (link) {
+            /*Keep a note of if the dirty is already set or not*/
+            link->dirty[0] |= (dirty[0] != 0);
+            link->dirty[1] |= (dirty[1] != 0);
+        }
     }
 
+    if (xdata)
+        cbk->xdata = dict_ref(xdata);
+
+    ec_combine(cbk, ec_combine_xattrop);
+
 out:
-    if (fop != NULL)
-    {
+    if (fop)
         ec_complete(fop);
-    }
 
     return 0;
 }
 
-void ec_wind_xattrop(ec_t * ec, ec_fop_data_t * fop, int32_t idx)
+void
+ec_wind_xattrop(ec_t *ec, ec_fop_data_t *fop, int32_t idx)
 {
     ec_trace("WIND", fop, "idx=%d", idx);
 
@@ -1382,21 +1221,20 @@ void ec_wind_xattrop(ec_t * ec, ec_fop_data_t * fop, int32_t idx)
                       &fop->loc[0], fop->xattrop_flags, fop->dict, fop->xdata);
 }
 
-int32_t ec_manager_xattrop(ec_fop_data_t * fop, int32_t state)
+int32_t
+ec_manager_xattrop(ec_fop_data_t *fop, int32_t state)
 {
-    ec_cbk_data_t * cbk;
+    ec_cbk_data_t *cbk;
 
-    switch (state)
-    {
+    switch (state) {
         case EC_STATE_INIT:
         case EC_STATE_LOCK:
-            if (fop->fd == NULL)
-            {
-                ec_lock_prepare_inode(fop, &fop->loc[0], 1);
-            }
-            else
-            {
-                ec_lock_prepare_fd(fop, fop->fd, 1);
+            if (fop->fd == NULL) {
+                ec_lock_prepare_inode(fop, &fop->loc[0], EC_UPDATE_META, 0,
+                                      EC_RANGE_FULL);
+            } else {
+                ec_lock_prepare_fd(fop, fop->fd, EC_UPDATE_META, 0,
+                                   EC_RANGE_FULL);
             }
             ec_lock(fop);
 
@@ -1408,27 +1246,12 @@ int32_t ec_manager_xattrop(ec_fop_data_t * fop, int32_t state)
             return EC_STATE_PREPARE_ANSWER;
 
         case EC_STATE_PREPARE_ANSWER:
-            cbk = fop->answer;
-            if (cbk != NULL)
-            {
-                if (!ec_dict_combine(cbk, EC_COMBINE_XDATA) ||
-                    ((cbk->op_ret >= 0) && !ec_dict_combine(cbk,
-                                                            EC_COMBINE_DICT)))
-                {
-                    if (cbk->op_ret >= 0)
-                    {
-                        cbk->op_ret = -1;
-                        cbk->op_errno = EIO;
-                    }
-                }
-                if (cbk->op_ret < 0)
-                {
-                    ec_fop_set_error(fop, cbk->op_errno);
-                }
-            }
-            else
-            {
-                ec_fop_set_error(fop, EIO);
+            cbk = ec_fop_prepare_answer(fop, _gf_false);
+            if (cbk != NULL) {
+                int32_t err;
+
+                err = ec_dict_combine(cbk, EC_COMBINE_DICT);
+                ec_cbk_set_error(cbk, -err, _gf_false);
             }
 
             return EC_STATE_REPORT;
@@ -1438,19 +1261,13 @@ int32_t ec_manager_xattrop(ec_fop_data_t * fop, int32_t state)
 
             GF_ASSERT(cbk != NULL);
 
-            if (fop->id == GF_FOP_XATTROP)
-            {
-                if (fop->cbks.xattrop != NULL)
-                {
-                    fop->cbks.xattrop(fop->req_frame, fop, fop->xl,
-                                      cbk->op_ret, cbk->op_errno, cbk->dict,
-                                      cbk->xdata);
+            if (fop->id == GF_FOP_XATTROP) {
+                if (fop->cbks.xattrop != NULL) {
+                    fop->cbks.xattrop(fop->req_frame, fop, fop->xl, cbk->op_ret,
+                                      cbk->op_errno, cbk->dict, cbk->xdata);
                 }
-            }
-            else
-            {
-                if (fop->cbks.fxattrop != NULL)
-                {
+            } else {
+                if (fop->cbks.fxattrop != NULL) {
                     fop->cbks.fxattrop(fop->req_frame, fop, fop->xl,
                                        cbk->op_ret, cbk->op_errno, cbk->dict,
                                        cbk->xdata);
@@ -1466,18 +1283,13 @@ int32_t ec_manager_xattrop(ec_fop_data_t * fop, int32_t state)
         case -EC_STATE_REPORT:
             GF_ASSERT(fop->error != 0);
 
-            if (fop->id == GF_FOP_XATTROP)
-            {
-                if (fop->cbks.xattrop != NULL)
-                {
+            if (fop->id == GF_FOP_XATTROP) {
+                if (fop->cbks.xattrop != NULL) {
                     fop->cbks.xattrop(fop->req_frame, fop, fop->xl, -1,
                                       fop->error, NULL, NULL);
                 }
-            }
-            else
-            {
-                if (fop->cbks.fxattrop != NULL)
-                {
+            } else {
+                if (fop->cbks.fxattrop != NULL) {
                     fop->cbks.fxattrop(fop->req_frame, fop, fop->xl, -1,
                                        fop->error, NULL, NULL);
                 }
@@ -1498,66 +1310,61 @@ int32_t ec_manager_xattrop(ec_fop_data_t * fop, int32_t state)
             return EC_STATE_END;
 
         default:
-            gf_log(fop->xl->name, GF_LOG_ERROR, "Unhandled state %d for %s",
-                   state, ec_fop_name(fop->id));
+            gf_msg(fop->xl->name, GF_LOG_ERROR, EINVAL, EC_MSG_UNHANDLED_STATE,
+                   "Unhandled state %d for %s", state, ec_fop_name(fop->id));
 
             return EC_STATE_END;
     }
 }
 
-void ec_xattrop(call_frame_t * frame, xlator_t * this, uintptr_t target,
-                int32_t minimum, fop_xattrop_cbk_t func, void * data,
-                loc_t * loc, gf_xattrop_flags_t optype, dict_t * xattr,
-                dict_t * xdata)
+void
+ec_xattrop(call_frame_t *frame, xlator_t *this, uintptr_t target,
+           uint32_t fop_flags, fop_xattrop_cbk_t func, void *data, loc_t *loc,
+           gf_xattrop_flags_t optype, dict_t *xattr, dict_t *xdata)
 {
-    ec_cbk_t callback = { .xattrop = func };
-    ec_fop_data_t * fop = NULL;
-    int32_t error = EIO;
+    ec_cbk_t callback = {.xattrop = func};
+    ec_fop_data_t *fop = NULL;
+    int32_t error = ENOMEM;
 
-    gf_log("ec", GF_LOG_TRACE, "EC(XATTROP) %p", frame);
+    gf_msg_trace("ec", 0, "EC(XATTROP) %p", frame);
 
     VALIDATE_OR_GOTO(this, out);
     GF_VALIDATE_OR_GOTO(this->name, frame, out);
     GF_VALIDATE_OR_GOTO(this->name, this->private, out);
 
-    fop = ec_fop_data_allocate(frame, this, GF_FOP_XATTROP,
-                               EC_FLAG_UPDATE_LOC_INODE, target, minimum,
-                               ec_wind_xattrop, ec_manager_xattrop, callback,
-                               data);
-    if (fop == NULL)
-    {
+    fop = ec_fop_data_allocate(frame, this, GF_FOP_XATTROP, 0, target,
+                               fop_flags, ec_wind_xattrop, ec_manager_xattrop,
+                               callback, data);
+    if (fop == NULL) {
         goto out;
     }
 
     fop->xattrop_flags = optype;
 
-    if (loc != NULL)
-    {
-        if (loc_copy(&fop->loc[0], loc) != 0)
-        {
-            gf_log(this->name, GF_LOG_ERROR, "Failed to copy a location.");
+    if (loc != NULL) {
+        if (loc_copy(&fop->loc[0], loc) != 0) {
+            gf_msg(this->name, GF_LOG_ERROR, ENOMEM, EC_MSG_LOC_COPY_FAIL,
+                   "Failed to copy a location.");
 
             goto out;
         }
     }
-    if (xattr != NULL)
-    {
+    if (xattr != NULL) {
         fop->dict = dict_ref(xattr);
-        if (fop->dict == NULL)
-        {
-            gf_log(this->name, GF_LOG_ERROR, "Failed to reference a "
-                                             "dictionary.");
+        if (fop->dict == NULL) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_DICT_REF_FAIL,
+                   "Failed to reference a "
+                   "dictionary.");
 
             goto out;
         }
     }
-    if (xdata != NULL)
-    {
+    if (xdata != NULL) {
         fop->xdata = dict_ref(xdata);
-        if (fop->xdata == NULL)
-        {
-            gf_log(this->name, GF_LOG_ERROR, "Failed to reference a "
-                                             "dictionary.");
+        if (fop->xdata == NULL) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_DICT_REF_FAIL,
+                   "Failed to reference a "
+                   "dictionary.");
 
             goto out;
         }
@@ -1566,24 +1373,98 @@ void ec_xattrop(call_frame_t * frame, xlator_t * this, uintptr_t target,
     error = 0;
 
 out:
-    if (fop != NULL)
-    {
+    if (fop != NULL) {
         ec_manager(fop, error);
+    } else {
+        func(frame, NULL, this, -1, error, NULL, NULL);
     }
-    else
-    {
-        func(frame, NULL, this, -1, EIO, NULL, NULL);
+}
+
+void
+ec_wind_fxattrop(ec_t *ec, ec_fop_data_t *fop, int32_t idx)
+{
+    ec_trace("WIND", fop, "idx=%d", idx);
+
+    STACK_WIND_COOKIE(fop->frame, ec_xattrop_cbk, (void *)(uintptr_t)idx,
+                      ec->xl_list[idx], ec->xl_list[idx]->fops->fxattrop,
+                      fop->fd, fop->xattrop_flags, fop->dict, fop->xdata);
+}
+
+void
+ec_fxattrop(call_frame_t *frame, xlator_t *this, uintptr_t target,
+            uint32_t fop_flags, fop_fxattrop_cbk_t func, void *data, fd_t *fd,
+            gf_xattrop_flags_t optype, dict_t *xattr, dict_t *xdata)
+{
+    ec_cbk_t callback = {.fxattrop = func};
+    ec_fop_data_t *fop = NULL;
+    int32_t error = ENOMEM;
+
+    gf_msg_trace("ec", 0, "EC(FXATTROP) %p", frame);
+
+    VALIDATE_OR_GOTO(this, out);
+    GF_VALIDATE_OR_GOTO(this->name, frame, out);
+    GF_VALIDATE_OR_GOTO(this->name, this->private, out);
+
+    fop = ec_fop_data_allocate(frame, this, GF_FOP_FXATTROP, 0, target,
+                               fop_flags, ec_wind_fxattrop, ec_manager_xattrop,
+                               callback, data);
+    if (fop == NULL) {
+        goto out;
+    }
+
+    fop->use_fd = 1;
+
+    fop->xattrop_flags = optype;
+
+    if (fd != NULL) {
+        fop->fd = fd_ref(fd);
+        if (fop->fd == NULL) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_FILE_DESC_REF_FAIL,
+                   "Failed to reference a "
+                   "file descriptor.");
+
+            goto out;
+        }
+    }
+    if (xattr != NULL) {
+        fop->dict = dict_ref(xattr);
+        if (fop->dict == NULL) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_DICT_REF_FAIL,
+                   "Failed to reference a "
+                   "dictionary.");
+
+            goto out;
+        }
+    }
+    if (xdata != NULL) {
+        fop->xdata = dict_ref(xdata);
+        if (fop->xdata == NULL) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_DICT_REF_FAIL,
+                   "Failed to reference a "
+                   "dictionary.");
+
+            goto out;
+        }
+    }
+
+    error = 0;
+
+out:
+    if (fop != NULL) {
+        ec_manager(fop, error);
+    } else {
+        func(frame, NULL, this, -1, error, NULL, NULL);
     }
 }
 
-/* FOP: fxattrop */
+/* FOP: IPC */
 
-int32_t ec_fxattrop_cbk(call_frame_t * frame, void * cookie, xlator_t * this,
-                        int32_t op_ret, int32_t op_errno, dict_t * xattr,
-                        dict_t * xdata)
+int32_t
+ec_ipc_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int32_t op_ret,
+           int32_t op_errno, dict_t *xdata)
 {
-    ec_fop_data_t * fop = NULL;
-    ec_cbk_data_t * cbk = NULL;
+    ec_fop_data_t *fop = NULL;
+    ec_cbk_data_t *cbk = NULL;
     int32_t idx = (int32_t)(uintptr_t)cookie;
 
     VALIDATE_OR_GOTO(this, out);
@@ -1593,131 +1474,118 @@ int32_t ec_fxattrop_cbk(call_frame_t * frame, void * cookie, xlator_t * this,
 
     fop = frame->local;
 
-    ec_trace("CBK", fop, "idx=%d, frame=%p, op_ret=%d, op_errno=%d", idx,
-             frame, op_ret, op_errno);
+    ec_trace("CBK", fop, "idx=%d, frame=%p, op_ret=%d, op_errno=%d", idx, frame,
+             op_ret, op_errno);
 
-    cbk = ec_cbk_data_allocate(frame, this, fop, GF_FOP_FXATTROP, idx, op_ret,
+    cbk = ec_cbk_data_allocate(frame, this, fop, GF_FOP_IPC, idx, op_ret,
                                op_errno);
-    if (cbk != NULL)
-    {
-        if (op_ret >= 0)
-        {
-            if (xattr != NULL)
-            {
-                cbk->dict = dict_ref(xattr);
-                if (cbk->dict == NULL)
-                {
-                    gf_log(this->name, GF_LOG_ERROR, "Failed to reference a "
-                                                     "dictionary.");
 
-                    goto out;
-                }
-            }
-        }
-        if (xdata != NULL)
-        {
+    if (cbk != NULL) {
+        if (xdata != NULL) {
             cbk->xdata = dict_ref(xdata);
-            if (cbk->xdata == NULL)
-            {
-                gf_log(this->name, GF_LOG_ERROR, "Failed to reference a "
-                                                 "dictionary.");
-
-                goto out;
-            }
         }
 
-        ec_combine(cbk, ec_combine_xattrop);
+        ec_combine(cbk, NULL);
     }
 
 out:
-    if (fop != NULL)
-    {
+    if (fop != NULL) {
         ec_complete(fop);
     }
 
     return 0;
 }
 
-void ec_wind_fxattrop(ec_t * ec, ec_fop_data_t * fop, int32_t idx)
+void
+ec_wind_ipc(ec_t *ec, ec_fop_data_t *fop, int32_t idx)
 {
     ec_trace("WIND", fop, "idx=%d", idx);
 
-    STACK_WIND_COOKIE(fop->frame, ec_fxattrop_cbk, (void *)(uintptr_t)idx,
-                      ec->xl_list[idx], ec->xl_list[idx]->fops->fxattrop,
-                      fop->fd, fop->xattrop_flags, fop->dict, fop->xdata);
+    STACK_WIND_COOKIE(fop->frame, ec_ipc_cbk, (void *)(uintptr_t)idx,
+                      ec->xl_list[idx], ec->xl_list[idx]->fops->ipc, fop->int32,
+                      fop->xdata);
 }
 
-void ec_fxattrop(call_frame_t * frame, xlator_t * this, uintptr_t target,
-                 int32_t minimum, fop_fxattrop_cbk_t func, void * data,
-                 fd_t * fd, gf_xattrop_flags_t optype, dict_t * xattr,
-                 dict_t * xdata)
+int32_t
+ec_manager_ipc(ec_fop_data_t *fop, int32_t state)
 {
-    ec_cbk_t callback = { .fxattrop = func };
-    ec_fop_data_t * fop = NULL;
-    int32_t error = EIO;
+    ec_cbk_data_t *cbk;
 
-    gf_log("ec", GF_LOG_TRACE, "EC(FXATTROP) %p", frame);
+    switch (state) {
+        case EC_STATE_INIT:
+        case EC_STATE_DISPATCH:
+            ec_dispatch_all(fop);
 
-    VALIDATE_OR_GOTO(this, out);
-    GF_VALIDATE_OR_GOTO(this->name, frame, out);
-    GF_VALIDATE_OR_GOTO(this->name, this->private, out);
+            return EC_STATE_PREPARE_ANSWER;
 
-    fop = ec_fop_data_allocate(frame, this, GF_FOP_FXATTROP,
-                               EC_FLAG_UPDATE_FD_INODE, target, minimum,
-                               ec_wind_fxattrop, ec_manager_xattrop, callback,
-                               data);
-    if (fop == NULL)
-    {
-        goto out;
-    }
+        case EC_STATE_PREPARE_ANSWER:
+            ec_fop_prepare_answer(fop, _gf_true);
 
-    fop->use_fd = 1;
+            return EC_STATE_REPORT;
 
-    fop->xattrop_flags = optype;
+        case EC_STATE_REPORT:
+            cbk = fop->answer;
 
-    if (fd != NULL)
-    {
-        fop->fd = fd_ref(fd);
-        if (fop->fd == NULL)
-        {
-            gf_log(this->name, GF_LOG_ERROR, "Failed to reference a "
-                                             "file descriptor.");
+            GF_ASSERT(cbk != NULL);
+            if (fop->cbks.ipc != NULL) {
+                fop->cbks.ipc(fop->req_frame, fop, fop->xl, cbk->op_ret,
+                              cbk->op_errno, cbk->xdata);
+            }
 
-            goto out;
-        }
+            return EC_STATE_END;
+
+        case -EC_STATE_INIT:
+        case -EC_STATE_DISPATCH:
+        case -EC_STATE_PREPARE_ANSWER:
+        case -EC_STATE_REPORT:
+            GF_ASSERT(fop->error != 0);
+
+            if (fop->cbks.ipc != NULL) {
+                fop->cbks.ipc(fop->req_frame, fop, fop->xl, -1, fop->error,
+                              NULL);
+            }
+
+            return EC_STATE_END;
+
+        default:
+            gf_msg(fop->xl->name, GF_LOG_ERROR, EINVAL, EC_MSG_UNHANDLED_STATE,
+                   "Unhandled state %d for %s", state, ec_fop_name(fop->id));
+
+            return EC_STATE_END;
     }
-    if (xattr != NULL)
-    {
-        fop->dict = dict_ref(xattr);
-        if (fop->dict == NULL)
-        {
-            gf_log(this->name, GF_LOG_ERROR, "Failed to reference a "
-                                             "dictionary.");
+}
 
-            goto out;
-        }
+void
+ec_ipc(call_frame_t *frame, xlator_t *this, uintptr_t target,
+       uint32_t fop_flags, fop_ipc_cbk_t func, void *data, int32_t op,
+       dict_t *xdata)
+{
+    ec_cbk_t callback = {.ipc = func};
+    ec_fop_data_t *fop = NULL;
+    int32_t error = ENOMEM;
+
+    gf_msg_trace("ec", 0, "EC(IPC) %p", frame);
+
+    VALIDATE_OR_GOTO(this, out);
+    GF_VALIDATE_OR_GOTO(this->name, frame, out);
+    GF_VALIDATE_OR_GOTO(this->name, this->private, out);
+
+    fop = ec_fop_data_allocate(frame, this, GF_FOP_IPC, 0, target, fop_flags,
+                               ec_wind_ipc, ec_manager_ipc, callback, data);
+    if (fop == NULL) {
+        goto out;
     }
-    if (xdata != NULL)
-    {
+    if (xdata != NULL) {
         fop->xdata = dict_ref(xdata);
-        if (fop->xdata == NULL)
-        {
-            gf_log(this->name, GF_LOG_ERROR, "Failed to reference a "
-                                             "dictionary.");
-
-            goto out;
-        }
     }
+    fop->int32 = op;
 
     error = 0;
 
 out:
-    if (fop != NULL)
-    {
+    if (fop != NULL) {
         ec_manager(fop, error);
-    }
-    else
-    {
-        func(frame, NULL, this, -1, EIO, NULL, NULL);
+    } else {
+        func(frame, NULL, this, -1, error, NULL);
     }
 }
diff --git a/xlators/cluster/ec/src/ec-gf.c b/xlators/cluster/ec/src/ec-gf.c
deleted file mode 100644
index 1ae8928f20b..00000000000
--- a/xlators/cluster/ec/src/ec-gf.c
+++ /dev/null
@@ -1,11635 +0,0 @@
-/*
-  Copyright (c) 2012-2014 DataLab, s.l. <http://www.datalab.es>
-  This file is part of GlusterFS.
-
-  This file is licensed to you under your choice of the GNU Lesser
-  General Public License, version 3 or any later version (LGPLv3 or
-  later), or the GNU General Public License, version 2 (GPLv2), in all
-  cases as published by the Free Software Foundation.
-*/
-
-#include <inttypes.h>
-#include <string.h>
-
-#include "ec-gf.h"
-
-static void gf8_muladd_00(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    memcpy(out, in, sizeof(uint64_t) * 8 * width);
-}
-
-static void gf8_muladd_01(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        out_ptr[0] ^= in_ptr[0];
-        out_ptr[width] ^= in_ptr[width];
-        out_ptr[width * 2] ^= in_ptr[width * 2];
-        out_ptr[width * 3] ^= in_ptr[width * 3];
-        out_ptr[width * 4] ^= in_ptr[width * 4];
-        out_ptr[width * 5] ^= in_ptr[width * 5];
-        out_ptr[width * 6] ^= in_ptr[width * 6];
-        out_ptr[width * 7] ^= in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_02(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        out0 = in7;
-        out1 = in0;
-        out7 = in6;
-        out5 = in4;
-        out6 = in5;
-        out3 = in2 ^ in7;
-        out4 = in3 ^ in7;
-        out2 = in1 ^ in7;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_03(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        out0 = in0 ^ in7;
-        tmp0 = in2 ^ in7;
-        out1 = in0 ^ in1;
-        out7 = in6 ^ in7;
-        out5 = in4 ^ in5;
-        out6 = in5 ^ in6;
-        out4 = in3 ^ in4 ^ in7;
-        out2 = tmp0 ^ in1;
-        out3 = tmp0 ^ in3;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_04(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        out0 = in6;
-        out1 = in7;
-        out7 = in5;
-        out6 = in4;
-        tmp0 = in6 ^ in7;
-        out2 = in0 ^ in6;
-        out5 = in3 ^ in7;
-        out3 = tmp0 ^ in1;
-        out4 = tmp0 ^ in2;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_05(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        out0 = in0 ^ in6;
-        out1 = in1 ^ in7;
-        out7 = in5 ^ in7;
-        out6 = in4 ^ in6;
-        out2 = out0 ^ in2;
-        out3 = out1 ^ in3 ^ in6;
-        out5 = out7 ^ in3;
-        out4 = out6 ^ in2 ^ in7;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_06(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        out0 = in6 ^ in7;
-        tmp0 = in1 ^ in6;
-        out1 = in0 ^ in7;
-        out7 = in5 ^ in6;
-        out6 = in4 ^ in5;
-        out4 = in2 ^ in3 ^ in6;
-        out5 = in3 ^ in4 ^ in7;
-        out3 = tmp0 ^ in2;
-        out2 = tmp0 ^ out1;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_07(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0, tmp1, tmp2, tmp3;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        tmp0 = in2 ^ in6;
-        tmp1 = in5 ^ in6;
-        tmp2 = in0 ^ in7;
-        tmp3 = tmp0 ^ in3;
-        out6 = tmp1 ^ in4;
-        out7 = tmp1 ^ in7;
-        out0 = tmp2 ^ in6;
-        out1 = tmp2 ^ in1;
-        out3 = tmp3 ^ in1;
-        out4 = tmp3 ^ in4;
-        out5 = out4 ^ out7 ^ in2;
-        out2 = tmp0 ^ out1;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_08(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        out0 = in5;
-        out1 = in6;
-        out7 = in4;
-        out6 = in3 ^ in7;
-        out3 = in0 ^ in5 ^ in6;
-        out5 = in2 ^ in6 ^ in7;
-        out2 = in5 ^ in7;
-        out4 = out2 ^ in1 ^ in6;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_09(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        out0 = in0 ^ in5;
-        tmp0 = in3 ^ in6;
-        out1 = in1 ^ in6;
-        out7 = in4 ^ in7;
-        out2 = in2 ^ in5 ^ in7;
-        out3 = tmp0 ^ out0;
-        out6 = tmp0 ^ in7;
-        out4 = out1 ^ out7 ^ in5;
-        out5 = out2 ^ in6;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_0A(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        out0 = in5 ^ in7;
-        out1 = in0 ^ in6;
-        out7 = in4 ^ in6;
-        out2 = in1 ^ in5;
-        out6 = out0 ^ in3;
-        out3 = out0 ^ out1 ^ in2;
-        out5 = out7 ^ in2 ^ in7;
-        out4 = out2 ^ in3 ^ in6;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_0B(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0, tmp1, tmp2;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        tmp0 = in2 ^ in5;
-        tmp1 = in0 ^ in6;
-        tmp2 = in4 ^ in7;
-        out0 = in0 ^ in5 ^ in7;
-        out2 = tmp0 ^ in1;
-        out1 = tmp1 ^ in1;
-        out6 = tmp1 ^ out0 ^ in3;
-        out7 = tmp2 ^ in6;
-        out4 = tmp2 ^ out6 ^ in1;
-        out3 = out6 ^ in0 ^ in2;
-        out5 = tmp0 ^ out7;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_0C(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0, tmp1;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        out0 = in5 ^ in6;
-        out1 = in6 ^ in7;
-        out7 = in4 ^ in5;
-        tmp0 = in1 ^ in5;
-        tmp1 = in0 ^ in7;
-        out5 = in2 ^ in3 ^ in6;
-        out6 = in3 ^ in4 ^ in7;
-        out2 = tmp1 ^ out0;
-        out4 = tmp0 ^ in2;
-        out3 = tmp0 ^ tmp1;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_0D(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0, tmp1, tmp2;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        tmp0 = in4 ^ in5;
-        tmp1 = in5 ^ in6;
-        out1 = in1 ^ in6 ^ in7;
-        out7 = tmp0 ^ in7;
-        out4 = tmp0 ^ in1 ^ in2;
-        out0 = tmp1 ^ in0;
-        tmp2 = tmp1 ^ in3;
-        out6 = tmp2 ^ out7;
-        out2 = out0 ^ in2 ^ in7;
-        out3 = out0 ^ out1 ^ in3;
-        out5 = tmp2 ^ in2;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_0E(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0, tmp1, tmp2, tmp3;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        tmp0 = in0 ^ in1;
-        tmp1 = in2 ^ in5;
-        tmp2 = in5 ^ in6;
-        out1 = in0 ^ in6 ^ in7;
-        out3 = tmp0 ^ tmp1;
-        out2 = tmp0 ^ tmp2;
-        tmp3 = tmp1 ^ in3;
-        out7 = tmp2 ^ in4;
-        out0 = tmp2 ^ in7;
-        out4 = tmp3 ^ in1 ^ in7;
-        out5 = tmp3 ^ out7;
-        out6 = out0 ^ out5 ^ in2;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_0F(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0, tmp1, tmp2, tmp3;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        tmp0 = in6 ^ in7;
-        tmp1 = tmp0 ^ in1;
-        tmp2 = tmp0 ^ in5;
-        out1 = tmp1 ^ in0;
-        out7 = tmp2 ^ in4;
-        out0 = tmp2 ^ in0;
-        out6 = out7 ^ in3;
-        out5 = out6 ^ in2 ^ in7;
-        tmp3 = tmp1 ^ out0 ^ in2;
-        out4 = tmp1 ^ out5;
-        out2 = tmp3 ^ in6;
-        out3 = tmp3 ^ in3;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_10(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0, tmp1;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        out0 = in4;
-        out1 = in5;
-        out7 = in3 ^ in7;
-        tmp0 = in6 ^ in7;
-        out2 = in4 ^ in6;
-        tmp1 = out2 ^ in5;
-        out6 = tmp0 ^ in2;
-        out3 = tmp0 ^ tmp1;
-        out5 = out2 ^ out3 ^ in1;
-        out4 = tmp1 ^ in0;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_11(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        out7 = in3;
-        out0 = in0 ^ in4;
-        out1 = in1 ^ in5;
-        out6 = in2 ^ in7;
-        out4 = in0 ^ in5 ^ in6;
-        out5 = in1 ^ in6 ^ in7;
-        out2 = in2 ^ in4 ^ in6;
-        out3 = in3 ^ in4 ^ in5 ^ in7;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_12(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0, tmp1;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        out0 = in4 ^ in7;
-        out1 = in0 ^ in5;
-        out3 = in2 ^ in4 ^ in5;
-        tmp0 = out0 ^ in6;
-        out2 = tmp0 ^ in1;
-        tmp1 = tmp0 ^ in3;
-        out6 = tmp0 ^ out3;
-        out5 = out2 ^ in5;
-        out7 = tmp1 ^ in4;
-        out4 = tmp1 ^ out1;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_13(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0, tmp1;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        out7 = in3 ^ in6;
-        tmp0 = in0 ^ in5;
-        tmp1 = in4 ^ in7;
-        out6 = in2 ^ in5 ^ in7;
-        out4 = tmp0 ^ out7 ^ in7;
-        out1 = tmp0 ^ in1;
-        out0 = tmp1 ^ in0;
-        out5 = tmp1 ^ in1 ^ in6;
-        out3 = tmp1 ^ out6 ^ in3;
-        out2 = out5 ^ in2;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_14(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0, tmp1;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        out0 = in4 ^ in6;
-        out1 = in5 ^ in7;
-        out2 = in0 ^ in4;
-        tmp0 = out0 ^ in5;
-        out7 = out1 ^ in3;
-        tmp1 = out1 ^ in2;
-        out3 = tmp0 ^ in1;
-        out6 = tmp0 ^ tmp1;
-        out4 = tmp1 ^ out2;
-        out5 = out3 ^ in3 ^ in4;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_15(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        out7 = in3 ^ in5;
-        tmp0 = in0 ^ in4;
-        out1 = in1 ^ in5 ^ in7;
-        out5 = in1 ^ in3 ^ in6;
-        out0 = tmp0 ^ in6;
-        out2 = tmp0 ^ in2;
-        out3 = out5 ^ in4 ^ in5;
-        out6 = out2 ^ in0 ^ in7;
-        out4 = tmp0 ^ out6 ^ in5;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_16(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0, tmp1, tmp2, tmp3;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        tmp0 = in0 ^ in5;
-        tmp1 = in4 ^ in7;
-        tmp2 = in2 ^ in3 ^ in4;
-        out1 = tmp0 ^ in7;
-        out4 = tmp0 ^ tmp2;
-        out0 = tmp1 ^ in6;
-        tmp3 = tmp1 ^ in1;
-        out6 = out0 ^ in2 ^ in5;
-        out2 = tmp3 ^ in0;
-        out3 = out6 ^ in1;
-        out7 = tmp2 ^ out6;
-        out5 = tmp3 ^ out7;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_17(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0, tmp1, tmp2, tmp3;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        tmp0 = in2 ^ in5;
-        tmp1 = in3 ^ in6;
-        tmp2 = tmp0 ^ in4;
-        out4 = tmp0 ^ in0 ^ in3;
-        out7 = tmp1 ^ in5;
-        tmp3 = tmp1 ^ in1;
-        out6 = tmp2 ^ in7;
-        out5 = tmp3 ^ in4;
-        out3 = tmp3 ^ out6;
-        out0 = out3 ^ out4 ^ in1;
-        out2 = out3 ^ out7 ^ in0;
-        out1 = tmp2 ^ out2;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_18(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0, tmp1;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        out0 = in4 ^ in5;
-        out1 = in5 ^ in6;
-        tmp0 = in4 ^ in7;
-        out5 = in1 ^ in2 ^ in5;
-        out6 = in2 ^ in3 ^ in6;
-        out2 = tmp0 ^ out1;
-        out7 = tmp0 ^ in3;
-        tmp1 = tmp0 ^ in0;
-        out3 = tmp1 ^ in6;
-        out4 = tmp1 ^ in1;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_19(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0, tmp1;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        out5 = in1 ^ in2;
-        out7 = in3 ^ in4;
-        tmp0 = in0 ^ in7;
-        out6 = in2 ^ in3;
-        out1 = in1 ^ in5 ^ in6;
-        out0 = in0 ^ in4 ^ in5;
-        out4 = tmp0 ^ in1;
-        tmp1 = tmp0 ^ in6;
-        out2 = tmp1 ^ out0 ^ in2;
-        out3 = tmp1 ^ out7;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_1A(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0, tmp1, tmp2, tmp3;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        tmp0 = in4 ^ in5;
-        tmp1 = in5 ^ in6;
-        tmp2 = tmp0 ^ in1;
-        out0 = tmp0 ^ in7;
-        out1 = tmp1 ^ in0;
-        tmp3 = tmp1 ^ in3;
-        out5 = tmp2 ^ in2;
-        out2 = tmp2 ^ in6;
-        out7 = tmp3 ^ out0;
-        out6 = tmp3 ^ in2;
-        out4 = tmp3 ^ out2 ^ in0;
-        out3 = tmp0 ^ out1 ^ in2;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_1B(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0, tmp1, tmp2, tmp3, tmp4;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        tmp0 = in2 ^ in4;
-        tmp1 = in2 ^ in5;
-        tmp2 = in3 ^ in6;
-        out5 = tmp0 ^ in1;
-        tmp3 = tmp0 ^ in0;
-        out6 = tmp1 ^ in3;
-        out0 = tmp1 ^ tmp3 ^ in7;
-        out7 = tmp2 ^ in4;
-        tmp4 = out5 ^ in6;
-        out3 = tmp2 ^ tmp3;
-        out2 = tmp4 ^ in5;
-        out4 = tmp4 ^ out3;
-        out1 = tmp3 ^ out2;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_1C(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0, tmp1, tmp2, tmp3, tmp4;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        tmp0 = in2 ^ in3;
-        tmp1 = in4 ^ in6;
-        tmp2 = in5 ^ in7;
-        out6 = tmp0 ^ tmp1;
-        out0 = tmp1 ^ in5;
-        out1 = tmp2 ^ in6;
-        tmp3 = tmp2 ^ in1;
-        tmp4 = tmp2 ^ in4;
-        out2 = tmp4 ^ in0;
-        out7 = tmp4 ^ in3;
-        out5 = tmp0 ^ tmp3;
-        out3 = tmp3 ^ out2;
-        out4 = out3 ^ in2 ^ in6;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_1D(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0, tmp1, tmp2, tmp3, tmp4;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        tmp0 = in1 ^ in3;
-        tmp1 = in0 ^ in4;
-        tmp2 = in3 ^ in4;
-        tmp3 = in2 ^ in7;
-        out3 = tmp0 ^ tmp1;
-        out5 = tmp0 ^ tmp3;
-        tmp4 = tmp1 ^ in5;
-        out6 = tmp2 ^ in2;
-        out7 = tmp2 ^ in5;
-        out2 = tmp3 ^ tmp4;
-        out4 = out3 ^ out6 ^ in6;
-        out0 = tmp4 ^ in6;
-        out1 = out2 ^ out4 ^ in4;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_1E(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0, tmp1, tmp2, tmp3;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        tmp0 = in0 ^ in4;
-        tmp1 = in2 ^ in7;
-        tmp2 = tmp0 ^ in1;
-        out3 = tmp1 ^ tmp2;
-        out2 = tmp2 ^ in5;
-        out4 = out3 ^ in3 ^ in6;
-        tmp3 = out4 ^ in7;
-        out6 = tmp3 ^ out2 ^ in4;
-        out7 = tmp1 ^ out6;
-        out0 = out7 ^ in3;
-        out1 = tmp0 ^ out0;
-        out5 = tmp3 ^ out1;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_1F(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0, tmp1;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        tmp0 = in4 ^ in6;
-        tmp1 = tmp0 ^ in5;
-        out7 = tmp1 ^ in3;
-        out0 = tmp1 ^ in0 ^ in7;
-        out6 = out7 ^ in2 ^ in6;
-        out1 = out0 ^ in1 ^ in4;
-        out4 = out0 ^ out6 ^ in1;
-        out3 = tmp0 ^ out4;
-        out2 = out4 ^ out7 ^ in7;
-        out5 = out3 ^ in0;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_20(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0, tmp1;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        out1 = in4;
-        out0 = in3 ^ in7;
-        tmp0 = in3 ^ in4;
-        tmp1 = in6 ^ in7;
-        out2 = out0 ^ in5;
-        out4 = tmp0 ^ in5;
-        out3 = tmp0 ^ tmp1;
-        out7 = tmp1 ^ in2;
-        out6 = tmp1 ^ in1 ^ in5;
-        out5 = out2 ^ out3 ^ in0;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_21(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        out1 = in1 ^ in4;
-        tmp0 = in4 ^ in6;
-        out4 = in3 ^ in5;
-        out7 = in2 ^ in6;
-        out0 = in0 ^ in3 ^ in7;
-        out6 = in1 ^ in5 ^ in7;
-        out3 = tmp0 ^ in7;
-        out5 = tmp0 ^ in0;
-        out2 = out4 ^ in2 ^ in7;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_22(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        out0 = in3;
-        out1 = in0 ^ in4;
-        out7 = in2 ^ in7;
-        out4 = in4 ^ in5 ^ in7;
-        out5 = in0 ^ in5 ^ in6;
-        out6 = in1 ^ in6 ^ in7;
-        out3 = in2 ^ in3 ^ in4 ^ in6;
-        out2 = in1 ^ in3 ^ in5;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_23(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        out7 = in2;
-        out0 = in0 ^ in3;
-        out4 = in5 ^ in7;
-        out5 = in0 ^ in6;
-        out6 = in1 ^ in7;
-        out3 = in2 ^ in4 ^ in6;
-        out1 = in0 ^ in1 ^ in4;
-        out2 = out4 ^ out6 ^ in2 ^ in3;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_24(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0, tmp1;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        out1 = in4 ^ in7;
-        tmp0 = in3 ^ in4;
-        out0 = in3 ^ in6 ^ in7;
-        out3 = tmp0 ^ in1;
-        tmp1 = out0 ^ in5;
-        out6 = tmp1 ^ out3;
-        out2 = tmp1 ^ in0;
-        out7 = tmp1 ^ in2 ^ in3;
-        out5 = out2 ^ in4;
-        out4 = tmp0 ^ out7;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_25(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        out3 = in1 ^ in4;
-        tmp0 = in2 ^ in5;
-        out1 = out3 ^ in7;
-        out7 = tmp0 ^ in6;
-        out6 = out1 ^ in5;
-        out4 = out7 ^ in3 ^ in7;
-        out2 = out4 ^ in0;
-        out0 = tmp0 ^ out2;
-        out5 = out0 ^ in4;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_26(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0, tmp1, tmp2;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        out0 = in3 ^ in6;
-        tmp0 = in4 ^ in7;
-        out7 = in2 ^ in5 ^ in7;
-        tmp1 = out0 ^ in0 ^ in5;
-        out1 = tmp0 ^ in0;
-        tmp2 = tmp0 ^ in6;
-        out2 = tmp1 ^ in1;
-        out5 = tmp1 ^ in7;
-        out6 = tmp2 ^ in1;
-        out4 = tmp2 ^ out7;
-        out3 = out0 ^ out6 ^ in2;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_27(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        out7 = in2 ^ in5;
-        out0 = in0 ^ in3 ^ in6;
-        out6 = in1 ^ in4 ^ in7;
-        out4 = out7 ^ in6;
-        out2 = out0 ^ out7 ^ in1;
-        out5 = out0 ^ in7;
-        out1 = out6 ^ in0;
-        out3 = out6 ^ in2;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_28(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0, tmp1, tmp2;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        out2 = in3;
-        out1 = in4 ^ in6;
-        out0 = in3 ^ in5 ^ in7;
-        tmp0 = out1 ^ in7;
-        tmp1 = out0 ^ in4;
-        out7 = tmp0 ^ in2;
-        tmp2 = tmp0 ^ in1;
-        out3 = tmp1 ^ in0;
-        out6 = tmp1 ^ tmp2;
-        out4 = tmp2 ^ in3;
-        out5 = out3 ^ in2 ^ in3;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_29(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0, tmp1, tmp2;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        out2 = in2 ^ in3;
-        tmp0 = in1 ^ in3;
-        tmp1 = in4 ^ in6;
-        tmp2 = in0 ^ in4 ^ in7;
-        out6 = tmp0 ^ in5;
-        out4 = tmp0 ^ in6 ^ in7;
-        out1 = tmp1 ^ in1;
-        out7 = tmp1 ^ in2;
-        out3 = tmp2 ^ in5;
-        out5 = tmp2 ^ in2;
-        out0 = out3 ^ in3 ^ in4;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_2A(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0, tmp1;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        out0 = in3 ^ in5;
-        tmp0 = in1 ^ in3;
-        tmp1 = in0 ^ in4;
-        out7 = in2 ^ in4 ^ in7;
-        out3 = tmp1 ^ out0 ^ in2;
-        out2 = tmp0 ^ in7;
-        out6 = tmp0 ^ in6;
-        out1 = tmp1 ^ in6;
-        out5 = tmp1 ^ out7 ^ in5;
-        out4 = out1 ^ in0 ^ in1;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_2B(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0, tmp1;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        out4 = in1 ^ in6;
-        out7 = in2 ^ in4;
-        tmp0 = in0 ^ in5;
-        tmp1 = in2 ^ in7;
-        out6 = in1 ^ in3;
-        out1 = out4 ^ in0 ^ in4;
-        out3 = tmp0 ^ out7;
-        out0 = tmp0 ^ in3;
-        out5 = tmp1 ^ in0;
-        out2 = tmp1 ^ out6;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_2C(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0, tmp1, tmp2, tmp3;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        tmp0 = in2 ^ in5;
-        tmp1 = in2 ^ in3 ^ in4;
-        tmp2 = tmp0 ^ in6;
-        out4 = tmp1 ^ in1;
-        out5 = tmp1 ^ in0 ^ in5;
-        tmp3 = tmp2 ^ in4;
-        out6 = tmp2 ^ out4;
-        out7 = tmp3 ^ in7;
-        out2 = tmp3 ^ out5;
-        out3 = out6 ^ in0;
-        out0 = tmp1 ^ out7;
-        out1 = tmp0 ^ out7;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_2D(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0, tmp1, tmp2, tmp3;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        tmp0 = in2 ^ in3;
-        out4 = tmp0 ^ in1;
-        tmp1 = tmp0 ^ in0;
-        out2 = tmp1 ^ in6;
-        out5 = tmp1 ^ in4;
-        tmp2 = out2 ^ in2;
-        tmp3 = tmp2 ^ in5;
-        out0 = tmp3 ^ in7;
-        out7 = tmp3 ^ out5;
-        out6 = out4 ^ out7 ^ in6;
-        out3 = tmp2 ^ out6;
-        out1 = out0 ^ out6 ^ in0;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_2E(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0, tmp1, tmp2;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        tmp0 = in4 ^ in7;
-        out0 = in3 ^ in5 ^ in6;
-        tmp1 = tmp0 ^ in0;
-        tmp2 = tmp0 ^ in2;
-        out1 = tmp1 ^ in6;
-        out4 = tmp2 ^ in1;
-        out7 = tmp2 ^ in5;
-        out3 = out0 ^ out4 ^ in0;
-        out2 = out3 ^ out7 ^ in7;
-        out6 = tmp1 ^ out2;
-        out5 = tmp1 ^ out7 ^ in3;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_2F(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0, tmp1, tmp2;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        tmp0 = in0 ^ in3;
-        tmp1 = in2 ^ in5;
-        out4 = in1 ^ in2 ^ in7;
-        out6 = in1 ^ in3 ^ in4;
-        out5 = tmp0 ^ in2;
-        tmp2 = tmp0 ^ in6;
-        out7 = tmp1 ^ in4;
-        out0 = tmp2 ^ in5;
-        out2 = tmp2 ^ out4;
-        out1 = tmp2 ^ out6 ^ in7;
-        out3 = tmp1 ^ out1;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_30(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0, tmp1;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        out1 = in4 ^ in5;
-        tmp0 = in3 ^ in6;
-        tmp1 = in4 ^ in7;
-        out6 = in1 ^ in2 ^ in5;
-        out3 = tmp0 ^ in5;
-        out4 = tmp0 ^ in0;
-        out7 = tmp0 ^ in2;
-        out0 = tmp1 ^ in3;
-        out2 = tmp1 ^ out3;
-        out5 = tmp1 ^ in0 ^ in1;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_31(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0, tmp1, tmp2;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        out3 = in5 ^ in6;
-        tmp0 = in4 ^ in5;
-        tmp1 = in0 ^ in3 ^ in4;
-        tmp2 = out3 ^ in2;
-        out1 = tmp0 ^ in1;
-        out0 = tmp1 ^ in7;
-        out4 = tmp1 ^ in6;
-        out6 = tmp2 ^ in1;
-        out2 = tmp2 ^ out0 ^ in0;
-        out5 = out1 ^ in0 ^ in7;
-        out7 = tmp0 ^ out2;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_32(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0, tmp1;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        out0 = in3 ^ in4;
-        out7 = in2 ^ in3;
-        tmp0 = in5 ^ in6;
-        tmp1 = in0 ^ in7;
-        out6 = in1 ^ in2;
-        out1 = in0 ^ in4 ^ in5;
-        out2 = tmp0 ^ out0 ^ in1;
-        out3 = tmp0 ^ out7 ^ in7;
-        out4 = tmp1 ^ in6;
-        out5 = tmp1 ^ in1;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_33(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0, tmp1, tmp2, tmp3, tmp4;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        tmp0 = in2 ^ in3;
-        tmp1 = in0 ^ in4;
-        tmp2 = in1 ^ in5;
-        out6 = in1 ^ in2 ^ in6;
-        out7 = tmp0 ^ in7;
-        out0 = tmp1 ^ in3;
-        out1 = tmp1 ^ tmp2;
-        tmp3 = tmp2 ^ in7;
-        tmp4 = tmp2 ^ in4 ^ in6;
-        out5 = tmp3 ^ in0;
-        out3 = tmp3 ^ out6;
-        out4 = tmp4 ^ out5;
-        out2 = tmp0 ^ tmp4;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_34(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0, tmp1, tmp2, tmp3, tmp4;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        tmp0 = in3 ^ in4;
-        tmp1 = in4 ^ in5;
-        tmp2 = tmp0 ^ in1;
-        tmp3 = tmp0 ^ in6;
-        out1 = tmp1 ^ in7;
-        tmp4 = tmp1 ^ in2;
-        out5 = tmp2 ^ in0;
-        out3 = tmp2 ^ out1;
-        out0 = tmp3 ^ in7;
-        out7 = tmp3 ^ tmp4;
-        out6 = tmp4 ^ in1;
-        out2 = out3 ^ out5 ^ in3;
-        out4 = tmp4 ^ out2;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_35(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0, tmp1, tmp2;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        tmp0 = in2 ^ in6;
-        tmp1 = in5 ^ in7;
-        out7 = tmp0 ^ tmp1 ^ in3;
-        out3 = tmp1 ^ in1;
-        out1 = out3 ^ in4;
-        tmp2 = out1 ^ in7;
-        out5 = tmp2 ^ in0 ^ in3;
-        out6 = tmp0 ^ tmp2;
-        out0 = out3 ^ out5 ^ in6;
-        out4 = tmp0 ^ out0;
-        out2 = out4 ^ in5;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_36(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0, tmp1;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        out4 = in0 ^ in2;
-        tmp0 = in1 ^ in3;
-        out0 = in3 ^ in4 ^ in6;
-        out6 = in1 ^ in2 ^ in4;
-        out5 = tmp0 ^ in0;
-        tmp1 = out5 ^ in5;
-        out2 = tmp1 ^ in4;
-        out3 = tmp1 ^ out4;
-        out1 = tmp0 ^ out2 ^ in7;
-        out7 = out3 ^ in1;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_37(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0, tmp1, tmp2, tmp3;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        tmp0 = in1 ^ in2;
-        tmp1 = in2 ^ in4;
-        tmp2 = tmp0 ^ in6;
-        out3 = tmp0 ^ in5;
-        out4 = tmp1 ^ in0;
-        out6 = tmp2 ^ in4;
-        out1 = out3 ^ out4 ^ in7;
-        tmp3 = out4 ^ in1 ^ in3;
-        out7 = tmp3 ^ out1;
-        out2 = tmp3 ^ in5;
-        out5 = tmp1 ^ out2;
-        out0 = tmp2 ^ tmp3;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_38(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0, tmp1, tmp2;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        out3 = in0 ^ in3;
-        tmp0 = in3 ^ in4;
-        tmp1 = in5 ^ in7;
-        tmp2 = out3 ^ in1;
-        out2 = tmp0 ^ in6;
-        out0 = tmp0 ^ tmp1;
-        out4 = tmp1 ^ tmp2;
-        out7 = out2 ^ in2;
-        out1 = out2 ^ in3 ^ in5;
-        out6 = out4 ^ in0 ^ in2;
-        out5 = tmp2 ^ out7;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_39(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0, tmp1, tmp2;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        out3 = in0;
-        tmp0 = in1 ^ in5;
-        tmp1 = tmp0 ^ in4;
-        out1 = tmp1 ^ in6;
-        out5 = out1 ^ in0 ^ in2;
-        tmp2 = tmp0 ^ out5;
-        out2 = tmp2 ^ in0 ^ in3;
-        out7 = out2 ^ in7;
-        out6 = tmp1 ^ out7;
-        out4 = tmp2 ^ out6;
-        out0 = out4 ^ in1;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_3A(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0, tmp1, tmp2, tmp3, tmp4, tmp5;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        tmp0 = in0 ^ in1;
-        tmp1 = in0 ^ in2;
-        tmp2 = in3 ^ in4;
-        tmp3 = in1 ^ in6;
-        tmp4 = in3 ^ in7;
-        out4 = tmp0 ^ in5;
-        out5 = tmp1 ^ tmp3;
-        out3 = tmp1 ^ tmp4;
-        out0 = tmp2 ^ in5;
-        out7 = tmp2 ^ in2;
-        tmp5 = tmp3 ^ in4;
-        out2 = tmp4 ^ tmp5;
-        out1 = tmp5 ^ out4;
-        out6 = tmp0 ^ out3;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_3B(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0, tmp1, tmp2;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        tmp0 = in1 ^ in6;
-        tmp1 = in2 ^ in7;
-        tmp2 = tmp0 ^ in3;
-        out3 = tmp1 ^ in0;
-        out6 = tmp1 ^ tmp2;
-        out2 = out6 ^ in4;
-        out7 = tmp0 ^ out2;
-        out0 = out3 ^ out7 ^ in5;
-        out5 = out0 ^ out2 ^ in7;
-        out1 = tmp2 ^ out0;
-        out4 = out1 ^ in6;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_3C(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0, tmp1, tmp2;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        tmp0 = in0 ^ in3;
-        tmp1 = in2 ^ in7;
-        tmp2 = in1 ^ in6 ^ in7;
-        out2 = tmp0 ^ in4;
-        out3 = tmp0 ^ tmp2;
-        out4 = tmp1 ^ out3 ^ in5;
-        out5 = tmp2 ^ out2 ^ in2;
-        out1 = out4 ^ out5 ^ in6;
-        out0 = out1 ^ in3;
-        out7 = tmp1 ^ out0;
-        out6 = tmp2 ^ out7;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_3D(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0, tmp1, tmp2;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        tmp0 = in0 ^ in2;
-        tmp1 = tmp0 ^ in3;
-        out2 = tmp1 ^ in4;
-        tmp2 = out2 ^ in5;
-        out4 = tmp2 ^ in1 ^ in6;
-        out5 = out4 ^ in7;
-        out6 = out5 ^ in0;
-        out7 = out6 ^ in1;
-        out0 = tmp0 ^ out7;
-        out1 = tmp1 ^ out5;
-        out3 = tmp2 ^ out6;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_3E(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0, tmp1;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        tmp0 = in3 ^ in5;
-        tmp1 = tmp0 ^ in4;
-        out0 = tmp1 ^ in6;
-        out7 = tmp1 ^ in2;
-        out6 = out7 ^ in1 ^ in5 ^ in7;
-        out2 = out6 ^ in0 ^ in2;
-        out4 = out0 ^ out6 ^ in0;
-        out5 = tmp0 ^ out4;
-        out3 = out5 ^ in7;
-        out1 = out3 ^ out6 ^ in5;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_3F(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0, tmp1, tmp2;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        tmp0 = in0 ^ in1;
-        out3 = tmp0 ^ in2 ^ in6;
-        tmp1 = out3 ^ in5 ^ in7;
-        out4 = tmp1 ^ in4;
-        out5 = tmp1 ^ in3;
-        out1 = out4 ^ in2;
-        out7 = out1 ^ out3 ^ in3;
-        out2 = tmp0 ^ out7 ^ in5;
-        tmp2 = out2 ^ in0;
-        out6 = tmp2 ^ in6;
-        out0 = tmp1 ^ tmp2;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_40(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0, tmp1;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        out1 = in3 ^ in7;
-        tmp0 = in3 ^ in4;
-        tmp1 = in6 ^ in7;
-        out4 = tmp0 ^ in2;
-        out5 = tmp0 ^ in5;
-        out0 = tmp1 ^ in2;
-        out7 = tmp1 ^ in1 ^ in5;
-        out2 = out0 ^ in4;
-        out3 = out2 ^ out5 ^ in7;
-        out6 = out3 ^ out4 ^ in0;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_41(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0, tmp1;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        out4 = in2 ^ in3;
-        tmp0 = in5 ^ in6;
-        tmp1 = in6 ^ in7;
-        out5 = in3 ^ in4;
-        out1 = in1 ^ in3 ^ in7;
-        out6 = in0 ^ in4 ^ in5;
-        out3 = tmp0 ^ in2;
-        out7 = tmp0 ^ in1;
-        out2 = tmp1 ^ in4;
-        out0 = tmp1 ^ in0 ^ in2;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_42(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        out0 = in2 ^ in6;
-        out5 = in3 ^ in5;
-        out1 = in0 ^ in3 ^ in7;
-        out7 = in1 ^ in5 ^ in7;
-        out4 = in2 ^ in4 ^ in7;
-        out6 = in0 ^ in4 ^ in6;
-        out2 = out0 ^ in1 ^ in4;
-        out3 = out5 ^ in6 ^ in7;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_43(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        out5 = in3;
-        out7 = in1 ^ in5;
-        out4 = in2 ^ in7;
-        out6 = in0 ^ in4;
-        out0 = in0 ^ in2 ^ in6;
-        out3 = in5 ^ in6 ^ in7;
-        out2 = in1 ^ in4 ^ in6;
-        out1 = in0 ^ in1 ^ in3 ^ in7;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_44(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        out1 = in3;
-        out0 = in2 ^ in7;
-        tmp0 = in4 ^ in7;
-        out7 = in1 ^ in6 ^ in7;
-        out6 = in0 ^ in5 ^ in6;
-        out4 = tmp0 ^ in3 ^ in6;
-        out3 = out0 ^ in1 ^ in3 ^ in5;
-        out2 = out0 ^ in0 ^ in4;
-        out5 = tmp0 ^ in5;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_45(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        out1 = in1 ^ in3;
-        out7 = in1 ^ in6;
-        out5 = in4 ^ in7;
-        out6 = in0 ^ in5;
-        out0 = in0 ^ in2 ^ in7;
-        out4 = in3 ^ in6 ^ in7;
-        out2 = out5 ^ in0;
-        out3 = out0 ^ out6 ^ in1;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_46(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        out0 = in2;
-        out1 = in0 ^ in3;
-        out7 = in1 ^ in7;
-        out4 = in4 ^ in6;
-        out5 = in5 ^ in7;
-        out6 = in0 ^ in6;
-        out3 = in1 ^ in3 ^ in5;
-        out2 = out4 ^ out6 ^ in1 ^ in2;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_47(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        out4 = in6;
-        out7 = in1;
-        out5 = in7;
-        out6 = in0;
-        tmp0 = in0 ^ in1;
-        out3 = in1 ^ in5;
-        out0 = in0 ^ in2;
-        out1 = tmp0 ^ in3;
-        out2 = tmp0 ^ in4;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_48(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0, tmp1;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        tmp0 = in2 ^ in3;
-        out1 = in3 ^ in6 ^ in7;
-        out3 = tmp0 ^ in0;
-        out0 = tmp0 ^ out1 ^ in5;
-        tmp1 = out0 ^ in4;
-        out2 = tmp1 ^ in7;
-        out5 = tmp1 ^ in3;
-        out4 = out5 ^ in1;
-        out7 = tmp0 ^ out4;
-        out6 = tmp1 ^ out3;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_49(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0, tmp1;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        out3 = in0 ^ in2;
-        tmp0 = in2 ^ in5;
-        out2 = in4 ^ in5 ^ in6;
-        tmp1 = tmp0 ^ out2 ^ in3;
-        out7 = out2 ^ in1;
-        out5 = tmp1 ^ in7;
-        out4 = out5 ^ out7 ^ in6;
-        out1 = tmp0 ^ out4;
-        out6 = out1 ^ out7 ^ in0;
-        out0 = tmp1 ^ out6;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_4A(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0, tmp1;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        tmp0 = in2 ^ in6;
-        tmp1 = in3 ^ in7;
-        out0 = tmp0 ^ in5;
-        out3 = tmp1 ^ in0;
-        out5 = tmp1 ^ out0;
-        out4 = out0 ^ in1 ^ in4;
-        out1 = out3 ^ in6;
-        out2 = out4 ^ in7;
-        out6 = out1 ^ in4;
-        out7 = tmp0 ^ out2;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_4B(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0, tmp1, tmp2, tmp3;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        out3 = in0 ^ in7;
-        tmp0 = in1 ^ in5;
-        tmp1 = in2 ^ in6;
-        tmp2 = out3 ^ in3;
-        out7 = tmp0 ^ in4;
-        out4 = tmp0 ^ tmp1;
-        tmp3 = tmp1 ^ in0;
-        out6 = tmp2 ^ in4;
-        out5 = tmp2 ^ tmp3;
-        out1 = tmp2 ^ in1 ^ in6;
-        out2 = out7 ^ in6 ^ in7;
-        out0 = tmp3 ^ in5;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_4C(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0, tmp1, tmp2;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        out1 = in3 ^ in6;
-        tmp0 = in2 ^ in5;
-        tmp1 = out1 ^ in5 ^ in7;
-        out0 = tmp0 ^ in7;
-        tmp2 = tmp0 ^ in4;
-        out6 = tmp1 ^ in0;
-        out2 = tmp2 ^ in0;
-        out5 = tmp2 ^ in6;
-        out3 = tmp0 ^ out6 ^ in1;
-        out7 = out0 ^ out5 ^ in1;
-        out4 = tmp1 ^ out7;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_4D(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0, tmp1, tmp2;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        tmp0 = in0 ^ in5;
-        tmp1 = in1 ^ in6;
-        out4 = in1 ^ in3 ^ in5;
-        tmp2 = tmp0 ^ in7;
-        out2 = tmp0 ^ in4;
-        out1 = tmp1 ^ in3;
-        out7 = tmp1 ^ in4;
-        out0 = tmp2 ^ in2;
-        out6 = tmp2 ^ in3;
-        out5 = out7 ^ in1 ^ in2;
-        out3 = tmp1 ^ out0 ^ in5;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_4E(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        out0 = in2 ^ in5;
-        out7 = in1 ^ in4 ^ in7;
-        out1 = in0 ^ in3 ^ in6;
-        out5 = out0 ^ in6;
-        out4 = out7 ^ in5;
-        out3 = out1 ^ in1;
-        out6 = out1 ^ in7;
-        out2 = out4 ^ in0 ^ in2;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_4F(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        out5 = in2 ^ in6;
-        out7 = in1 ^ in4;
-        out3 = in0 ^ in1 ^ in6;
-        out4 = in1 ^ in5 ^ in7;
-        out0 = in0 ^ in2 ^ in5;
-        out6 = in0 ^ in3 ^ in7;
-        out1 = out3 ^ in3;
-        out2 = out4 ^ in0 ^ in4;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_50(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0, tmp1, tmp2;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        out2 = in2 ^ in7;
-        tmp0 = in3 ^ in5;
-        out0 = out2 ^ in4 ^ in6;
-        out1 = tmp0 ^ in7;
-        tmp1 = tmp0 ^ in6;
-        out3 = out0 ^ in3;
-        out7 = tmp1 ^ in1;
-        tmp2 = tmp1 ^ in0;
-        out5 = out3 ^ in1 ^ in2;
-        out4 = tmp2 ^ in2;
-        out6 = tmp2 ^ out3;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_51(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        out2 = in7;
-        out3 = in2 ^ in4 ^ in6 ^ in7;
-        out0 = out3 ^ in0;
-        out6 = out0 ^ in5;
-        out4 = out6 ^ in3 ^ in7;
-        out1 = out0 ^ out4 ^ in1;
-        out7 = out1 ^ in6;
-        out5 = out7 ^ in4;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_52(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0, tmp1, tmp2, tmp3;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        out2 = in1 ^ in2;
-        tmp0 = in2 ^ in4;
-        tmp1 = in3 ^ in5;
-        tmp2 = in3 ^ in6;
-        tmp3 = in0 ^ in7;
-        out0 = tmp0 ^ in6;
-        out6 = tmp0 ^ tmp3;
-        out7 = tmp1 ^ in1;
-        out1 = tmp1 ^ tmp3;
-        out3 = tmp2 ^ in4;
-        out5 = tmp2 ^ in1 ^ in7;
-        out4 = tmp2 ^ out1 ^ in2;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_53(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        out2 = in1;
-        out3 = in4 ^ in6;
-        out0 = out3 ^ in0 ^ in2;
-        out6 = out0 ^ in7;
-        out4 = out6 ^ in5;
-        out7 = out0 ^ out4 ^ in1 ^ in3;
-        out1 = out7 ^ in0;
-        out5 = out7 ^ in6;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_54(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0, tmp1, tmp2, tmp3;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        out1 = in3 ^ in5;
-        tmp0 = in1 ^ in3;
-        tmp1 = in2 ^ in4;
-        tmp2 = in0 ^ in7;
-        out5 = in1 ^ in4 ^ in6;
-        out4 = tmp2 ^ out1;
-        out7 = tmp0 ^ in6;
-        out3 = tmp0 ^ tmp1;
-        out0 = tmp1 ^ in7;
-        tmp3 = tmp2 ^ in2;
-        out2 = tmp3 ^ in6;
-        out6 = tmp3 ^ in5;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_55(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0, tmp1, tmp2;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        tmp0 = in1 ^ in3;
-        tmp1 = in1 ^ in4;
-        tmp2 = in6 ^ in7;
-        out7 = tmp0 ^ tmp2;
-        out1 = tmp0 ^ in5;
-        out3 = tmp1 ^ in2;
-        out5 = tmp1 ^ in5 ^ in6;
-        out2 = tmp2 ^ in0;
-        out4 = out5 ^ out7 ^ in0;
-        out6 = out2 ^ in2 ^ in5;
-        out0 = out5 ^ out6 ^ in1;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_56(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        out0 = in2 ^ in4;
-        tmp0 = in0 ^ in2;
-        out4 = in0 ^ in5;
-        out7 = in1 ^ in3;
-        out5 = in1 ^ in6;
-        out6 = tmp0 ^ in7;
-        out2 = tmp0 ^ out5;
-        out1 = out4 ^ in3;
-        out3 = out7 ^ in4 ^ in7;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_57(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0, tmp1;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        tmp0 = in0 ^ in5;
-        tmp1 = in1 ^ in7;
-        out0 = in0 ^ in2 ^ in4;
-        out5 = in1 ^ in5 ^ in6;
-        out4 = tmp0 ^ in4;
-        out1 = tmp0 ^ in1 ^ in3;
-        out2 = tmp0 ^ out5;
-        out3 = tmp1 ^ in4;
-        out7 = tmp1 ^ in3;
-        out6 = tmp1 ^ out2 ^ in2;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_58(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0, tmp1;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        out2 = in2 ^ in5;
-        tmp0 = in2 ^ in3 ^ in4;
-        out5 = tmp0 ^ in1;
-        out6 = tmp0 ^ in0 ^ in5;
-        out3 = out6 ^ in7;
-        tmp1 = out2 ^ out5;
-        out7 = tmp1 ^ in6;
-        out4 = tmp1 ^ out3 ^ in3;
-        out0 = out4 ^ out7 ^ in0;
-        out1 = tmp0 ^ out0;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_59(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0, tmp1;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        out2 = in5;
-        tmp0 = in0 ^ in5 ^ in7;
-        out3 = tmp0 ^ in2 ^ in4;
-        out0 = out3 ^ in6;
-        tmp1 = out0 ^ in7;
-        out6 = tmp1 ^ in3;
-        out5 = out6 ^ in0 ^ in1 ^ in6;
-        out4 = tmp0 ^ out5;
-        out1 = tmp1 ^ out4;
-        out7 = out1 ^ in4;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_5A(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0, tmp1, tmp2;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        tmp0 = in1 ^ in2;
-        tmp1 = in2 ^ in5;
-        out5 = tmp0 ^ in3;
-        out4 = tmp0 ^ in0;
-        tmp2 = tmp1 ^ in4;
-        out2 = tmp1 ^ in1 ^ in7;
-        out7 = tmp2 ^ out5;
-        out6 = out4 ^ out7 ^ in5;
-        out0 = tmp2 ^ in6;
-        out1 = out0 ^ out6 ^ in7;
-        out3 = tmp1 ^ out6;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_5B(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0, tmp1, tmp2, tmp3, tmp4;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        tmp0 = in2 ^ in3;
-        tmp1 = in0 ^ in4;
-        tmp2 = in1 ^ in5;
-        out5 = tmp0 ^ tmp2;
-        tmp3 = tmp1 ^ in6;
-        out3 = tmp1 ^ in5;
-        out2 = tmp2 ^ in7;
-        tmp4 = out3 ^ in2;
-        out7 = out2 ^ in3 ^ in4;
-        out0 = tmp4 ^ in6;
-        out6 = tmp0 ^ tmp3;
-        out4 = tmp2 ^ tmp4;
-        out1 = tmp3 ^ out7;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_5C(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0, tmp1, tmp2;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        tmp0 = in3 ^ in6;
-        tmp1 = in0 ^ in2 ^ in5;
-        out1 = tmp0 ^ in5;
-        tmp2 = tmp0 ^ in1;
-        out2 = tmp1 ^ in6;
-        out6 = tmp1 ^ in3;
-        out4 = tmp2 ^ in0;
-        out7 = tmp2 ^ in4;
-        out3 = tmp1 ^ out7;
-        out0 = out3 ^ out4 ^ in7;
-        out5 = out0 ^ in1 ^ in5;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_5D(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0, tmp1, tmp2, tmp3, tmp4;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        tmp0 = in0 ^ in1;
-        tmp1 = in0 ^ in6;
-        out2 = tmp1 ^ in5;
-        tmp2 = out2 ^ in3;
-        out6 = tmp2 ^ in2;
-        out1 = tmp0 ^ tmp2;
-        tmp3 = out1 ^ in4 ^ in5;
-        out4 = tmp3 ^ in0;
-        out7 = tmp3 ^ in7;
-        tmp4 = out4 ^ out6;
-        out5 = tmp4 ^ in7;
-        out0 = tmp0 ^ out5;
-        out3 = tmp1 ^ tmp4;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_5E(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0, tmp1, tmp2, tmp3, tmp4;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        tmp0 = in2 ^ in5;
-        tmp1 = in3 ^ in5;
-        tmp2 = in1 ^ in7;
-        out7 = in1 ^ in3 ^ in4;
-        out0 = tmp0 ^ in4;
-        tmp3 = tmp1 ^ in0;
-        out5 = tmp2 ^ in2;
-        out1 = tmp3 ^ in6;
-        out6 = tmp0 ^ tmp3;
-        tmp4 = tmp2 ^ out1;
-        out3 = tmp4 ^ in4;
-        out4 = tmp1 ^ tmp4;
-        out2 = tmp0 ^ out4;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_5F(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0, tmp1, tmp2, tmp3;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        tmp0 = in1 ^ in5;
-        tmp1 = in0 ^ in6;
-        tmp2 = tmp0 ^ in7;
-        tmp3 = tmp1 ^ in3;
-        out2 = tmp1 ^ tmp2;
-        out5 = tmp2 ^ in2;
-        out6 = tmp3 ^ in2;
-        out3 = out2 ^ in4;
-        out4 = out3 ^ in5;
-        out1 = tmp0 ^ tmp3;
-        out7 = tmp3 ^ out4;
-        out0 = out4 ^ out5 ^ in6;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_60(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0, tmp1;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        out4 = in2 ^ in5;
-        tmp0 = in3 ^ in6;
-        out1 = in3 ^ in4 ^ in7;
-        out7 = out4 ^ in1;
-        tmp1 = out4 ^ in4;
-        out0 = tmp0 ^ in2;
-        out5 = tmp0 ^ in0;
-        out2 = tmp0 ^ tmp1;
-        out3 = tmp1 ^ in7;
-        out6 = out3 ^ out7 ^ in0;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_61(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0, tmp1;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        tmp0 = in2 ^ in5;
-        out4 = tmp0 ^ in4;
-        tmp1 = out4 ^ in3;
-        out3 = tmp1 ^ in7;
-        out2 = tmp1 ^ in2 ^ in6;
-        out1 = tmp0 ^ out3 ^ in1;
-        out0 = out2 ^ out4 ^ in0;
-        out7 = tmp1 ^ out1;
-        out6 = out0 ^ out1 ^ in2;
-        out5 = tmp0 ^ out0;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_62(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0, tmp1, tmp2, tmp3;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        out3 = in4 ^ in5;
-        tmp0 = in0 ^ in3 ^ in4;
-        out1 = tmp0 ^ in7;
-        out5 = tmp0 ^ in6;
-        tmp1 = out1 ^ in0;
-        tmp2 = tmp1 ^ out3;
-        out4 = tmp2 ^ in2;
-        tmp3 = tmp2 ^ in1;
-        out0 = out4 ^ in5 ^ in6;
-        out7 = tmp3 ^ out0;
-        out6 = tmp0 ^ tmp3;
-        out2 = tmp1 ^ out7;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_63(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0, tmp1, tmp2, tmp3, tmp4;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        tmp0 = in3 ^ in4;
-        tmp1 = in1 ^ in7;
-        out3 = tmp0 ^ in5;
-        tmp2 = out3 ^ in6;
-        out4 = out3 ^ in2 ^ in7;
-        out5 = tmp2 ^ in0;
-        tmp3 = out5 ^ in3;
-        out0 = tmp3 ^ out4;
-        out2 = tmp1 ^ tmp2;
-        out6 = tmp1 ^ tmp3;
-        tmp4 = tmp0 ^ out2;
-        out1 = tmp4 ^ out5;
-        out7 = tmp4 ^ in2;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_64(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0, tmp1;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        out0 = in2 ^ in3;
-        out1 = in3 ^ in4;
-        out7 = in1 ^ in2;
-        tmp0 = in4 ^ in5;
-        tmp1 = in0 ^ in7;
-        out4 = in5 ^ in6 ^ in7;
-        out2 = tmp0 ^ out0 ^ in0;
-        out3 = tmp0 ^ out7 ^ in6;
-        out5 = tmp1 ^ in6;
-        out6 = tmp1 ^ in1;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_65(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0, tmp1, tmp2, tmp3;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        tmp0 = in0 ^ in3;
-        tmp1 = in4 ^ in5;
-        tmp2 = in6 ^ in7;
-        out7 = in1 ^ in2 ^ in7;
-        out1 = in1 ^ in3 ^ in4;
-        out0 = tmp0 ^ in2;
-        out2 = tmp0 ^ tmp1;
-        out4 = tmp1 ^ tmp2;
-        tmp3 = tmp2 ^ in0;
-        out3 = out4 ^ out7 ^ in3;
-        out5 = tmp3 ^ in5;
-        out6 = tmp3 ^ in1;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_66(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0, tmp1, tmp2, tmp3, tmp4;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        tmp0 = in1 ^ in2;
-        tmp1 = in2 ^ in3;
-        tmp2 = in0 ^ in4;
-        out7 = tmp0 ^ in6;
-        out0 = tmp1 ^ in7;
-        out1 = tmp2 ^ in3;
-        tmp3 = tmp2 ^ in6;
-        tmp4 = out1 ^ in5;
-        out5 = tmp3 ^ in7;
-        out4 = tmp3 ^ tmp4;
-        out2 = tmp0 ^ tmp4 ^ in7;
-        out6 = tmp1 ^ out2 ^ in4;
-        out3 = tmp3 ^ out6;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_67(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0, tmp1, tmp2, tmp3;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        tmp0 = in0 ^ in3;
-        tmp1 = tmp0 ^ in1;
-        tmp2 = tmp0 ^ in7;
-        out1 = tmp1 ^ in4;
-        out0 = tmp2 ^ in2;
-        tmp3 = out1 ^ in7;
-        out2 = tmp3 ^ in5;
-        out3 = out2 ^ in0 ^ in6;
-        out7 = tmp1 ^ out0 ^ in6;
-        out5 = tmp1 ^ out3;
-        out4 = tmp2 ^ out5;
-        out6 = tmp3 ^ out4;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_68(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0, tmp1, tmp2, tmp3;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        tmp0 = in3 ^ in4;
-        tmp1 = in2 ^ in3 ^ in5;
-        tmp2 = tmp0 ^ in1;
-        tmp3 = tmp0 ^ in6;
-        out0 = tmp1 ^ in6;
-        out6 = tmp2 ^ in0;
-        out7 = tmp1 ^ tmp2;
-        out1 = tmp3 ^ in7;
-        out2 = out1 ^ in2;
-        out4 = tmp2 ^ out2;
-        out3 = out4 ^ out6 ^ in3;
-        out5 = tmp3 ^ out3;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_69(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        tmp0 = in6 ^ in7;
-        out2 = tmp0 ^ in3 ^ in4;
-        out1 = out2 ^ in1;
-        out3 = out2 ^ in0 ^ in2;
-        out4 = out1 ^ in2 ^ in3;
-        out6 = out1 ^ in0 ^ in7;
-        out7 = out4 ^ in5 ^ in6;
-        out5 = out4 ^ out6 ^ in5;
-        out0 = tmp0 ^ out5;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_6A(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0, tmp1, tmp2;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        tmp0 = in2 ^ in6;
-        out3 = in0 ^ in4 ^ in6;
-        tmp1 = tmp0 ^ in3;
-        out4 = tmp1 ^ in1;
-        tmp2 = tmp1 ^ in7;
-        out2 = out4 ^ in4;
-        out0 = tmp2 ^ in5;
-        out5 = tmp2 ^ out3;
-        out7 = out2 ^ in3 ^ in5;
-        out1 = tmp0 ^ out5;
-        out6 = tmp1 ^ out7 ^ in0;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_6B(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0, tmp1;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        tmp0 = in4 ^ in6;
-        out2 = tmp0 ^ in1 ^ in3;
-        out4 = out2 ^ in2;
-        tmp1 = out2 ^ in0;
-        out7 = out4 ^ in3 ^ in5 ^ in7;
-        out1 = tmp1 ^ in7;
-        out3 = tmp1 ^ in1;
-        out6 = tmp1 ^ in5;
-        out0 = tmp1 ^ out7 ^ in6;
-        out5 = tmp0 ^ out0;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_6C(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0, tmp1;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        out4 = in1;
-        tmp0 = in2 ^ in3;
-        out5 = in0 ^ in2;
-        out1 = in3 ^ in4 ^ in6;
-        tmp1 = out5 ^ in1;
-        out0 = tmp0 ^ in5;
-        out6 = tmp0 ^ tmp1;
-        out3 = tmp1 ^ in4;
-        out7 = out3 ^ in0;
-        out2 = out6 ^ out7 ^ in7;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_6D(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0, tmp1;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        out4 = in1 ^ in4;
-        tmp0 = in0 ^ in2;
-        tmp1 = out4 ^ in3;
-        out7 = out4 ^ in2 ^ in7;
-        out5 = tmp0 ^ in5;
-        out3 = tmp0 ^ tmp1;
-        out1 = tmp1 ^ in6;
-        out0 = out5 ^ in3;
-        out2 = out3 ^ out7 ^ in4;
-        out6 = out1 ^ in0 ^ in4;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_6E(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0, tmp1, tmp2;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        tmp0 = in1 ^ in3;
-        tmp1 = in0 ^ in4;
-        out4 = tmp0 ^ in7;
-        out6 = tmp0 ^ in0 ^ in5;
-        out5 = tmp1 ^ in2;
-        tmp2 = tmp1 ^ in3;
-        out3 = tmp2 ^ out4;
-        out1 = tmp2 ^ in6;
-        out2 = tmp0 ^ out5;
-        out0 = out2 ^ out3 ^ in5;
-        out7 = out1 ^ out2 ^ in4;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_6F(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0, tmp1, tmp2;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        tmp0 = in3 ^ in7;
-        tmp1 = tmp0 ^ in4;
-        tmp2 = tmp0 ^ in0 ^ in2;
-        out4 = tmp1 ^ in1;
-        out0 = tmp2 ^ in5;
-        out3 = out4 ^ in0;
-        out2 = out3 ^ in7;
-        out1 = out2 ^ in6;
-        out6 = out1 ^ in4 ^ in5;
-        out7 = tmp2 ^ out1;
-        out5 = tmp1 ^ out0;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_70(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0, tmp1, tmp2;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        out3 = in2;
-        tmp0 = in2 ^ in4;
-        out2 = in2 ^ in3 ^ in5;
-        tmp1 = tmp0 ^ in6;
-        tmp2 = out2 ^ in7;
-        out0 = tmp1 ^ in3;
-        out4 = tmp1 ^ in0;
-        out7 = tmp2 ^ in1;
-        out6 = out4 ^ in1;
-        out5 = out7 ^ in0 ^ in2;
-        out1 = tmp0 ^ tmp2;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_71(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0, tmp1, tmp2;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        out2 = in3 ^ in5;
-        out3 = in2 ^ in3;
-        tmp0 = in0 ^ in2;
-        tmp1 = out2 ^ in1;
-        out4 = tmp0 ^ in6;
-        tmp2 = tmp0 ^ in1;
-        out7 = tmp1 ^ in2;
-        out1 = tmp1 ^ in4 ^ in7;
-        out0 = out4 ^ in3 ^ in4;
-        out6 = tmp2 ^ in4;
-        out5 = tmp2 ^ out3 ^ in7;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_72(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0, tmp1, tmp2;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        out3 = in7;
-        tmp0 = in0 ^ in4;
-        tmp1 = tmp0 ^ in3 ^ in7;
-        out1 = tmp1 ^ in5;
-        out5 = out1 ^ in1;
-        tmp2 = tmp0 ^ out5;
-        out2 = tmp2 ^ in2;
-        out7 = out2 ^ in6;
-        out6 = tmp1 ^ out7;
-        out4 = tmp2 ^ out6;
-        out0 = out4 ^ in0;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_73(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        out3 = in3 ^ in7;
-        out2 = out3 ^ in1 ^ in5;
-        out1 = out2 ^ in0 ^ in4;
-        out5 = out1 ^ in5;
-        out6 = out1 ^ out3 ^ in2;
-        out0 = out2 ^ out6 ^ in6;
-        out7 = out0 ^ out1 ^ in3;
-        out4 = out0 ^ in4;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_74(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0, tmp1;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        tmp0 = in3 ^ in4;
-        tmp1 = in1 ^ in2 ^ in6;
-        out4 = in0 ^ in4 ^ in7;
-        out5 = in0 ^ in1 ^ in5;
-        out0 = tmp0 ^ in2;
-        out1 = tmp0 ^ in5;
-        out3 = tmp1 ^ in7;
-        out6 = tmp1 ^ in0;
-        out2 = tmp1 ^ out5 ^ in3;
-        out7 = out3 ^ in3 ^ in6;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_75(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0, tmp1, tmp2;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        out4 = in0 ^ in7;
-        tmp0 = in1 ^ in3;
-        out5 = in0 ^ in1;
-        out7 = tmp0 ^ in2;
-        tmp1 = tmp0 ^ in4;
-        out6 = out5 ^ in2;
-        tmp2 = out7 ^ in6;
-        out1 = tmp1 ^ in5;
-        out0 = tmp1 ^ out6;
-        out3 = tmp2 ^ in7;
-        out2 = tmp2 ^ out6 ^ in5;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_76(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0, tmp1, tmp2, tmp3;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        out3 = in1 ^ in6;
-        tmp0 = in0 ^ in5;
-        tmp1 = in3 ^ in7;
-        tmp2 = tmp0 ^ in4;
-        tmp3 = tmp1 ^ in2;
-        out5 = tmp2 ^ in1;
-        out1 = tmp2 ^ in3;
-        out0 = tmp3 ^ in4;
-        out4 = out1 ^ in5;
-        out7 = tmp3 ^ out3;
-        out2 = tmp0 ^ out7;
-        out6 = tmp1 ^ out2;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_77(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0, tmp1, tmp2, tmp3;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        out4 = in0 ^ in3;
-        tmp0 = in1 ^ in4;
-        tmp1 = in1 ^ in6;
-        tmp2 = out4 ^ in5;
-        out5 = tmp0 ^ in0;
-        out1 = tmp0 ^ tmp2;
-        out3 = tmp1 ^ in3;
-        out2 = tmp1 ^ tmp2 ^ in7;
-        out7 = out3 ^ in2;
-        tmp3 = out7 ^ in6;
-        out6 = tmp2 ^ tmp3;
-        out0 = tmp3 ^ out5 ^ in7;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_78(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0, tmp1, tmp2;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        tmp0 = in0 ^ in3;
-        tmp1 = in2 ^ in7;
-        tmp2 = in0 ^ in5 ^ in6;
-        out2 = tmp1 ^ in3;
-        out3 = tmp2 ^ in2;
-        out5 = out3 ^ in1 ^ in3;
-        out0 = tmp0 ^ out3 ^ in4;
-        out1 = tmp1 ^ out0;
-        out4 = out1 ^ out5 ^ in5;
-        out7 = tmp0 ^ out4;
-        out6 = tmp2 ^ out7;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_79(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0, tmp1, tmp2, tmp3;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        out2 = in3 ^ in7;
-        tmp0 = in3 ^ in4;
-        tmp1 = in1 ^ in5;
-        tmp2 = tmp1 ^ in2;
-        out4 = tmp2 ^ in0 ^ in7;
-        tmp3 = out4 ^ in5;
-        out5 = tmp3 ^ out2 ^ in6;
-        out7 = tmp0 ^ tmp2;
-        out6 = tmp0 ^ tmp3;
-        out3 = tmp1 ^ out5;
-        out0 = out3 ^ in4;
-        out1 = tmp3 ^ out0;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_7A(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0, tmp1;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        tmp0 = in1 ^ in2;
-        out2 = tmp0 ^ in3;
-        tmp1 = out2 ^ in4;
-        out4 = tmp1 ^ in0 ^ in5;
-        out5 = out4 ^ in6;
-        out6 = out5 ^ in7;
-        out7 = out6 ^ in0;
-        out0 = out7 ^ in1;
-        out1 = tmp0 ^ out6;
-        out3 = tmp1 ^ out6;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_7B(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0, tmp1, tmp2;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        out2 = in1 ^ in3;
-        tmp0 = in0 ^ in5;
-        out4 = tmp0 ^ out2 ^ in2;
-        tmp1 = out4 ^ in4;
-        out6 = tmp1 ^ in7;
-        out5 = tmp1 ^ in5 ^ in6;
-        out0 = out6 ^ in1 ^ in6;
-        tmp2 = out0 ^ in2;
-        out1 = tmp2 ^ in1;
-        out3 = tmp2 ^ in4;
-        out7 = tmp0 ^ out5;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_7C(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0, tmp1;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        tmp0 = in3 ^ in5;
-        tmp1 = tmp0 ^ in4;
-        out0 = tmp1 ^ in2;
-        out1 = tmp1 ^ in6;
-        out7 = out0 ^ in1 ^ in5 ^ in7;
-        out5 = out1 ^ out7 ^ in0;
-        out3 = out5 ^ in6;
-        out6 = tmp0 ^ out5;
-        out2 = out6 ^ in1;
-        out4 = out2 ^ out7 ^ in5;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_7D(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0, tmp1, tmp2, tmp3;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        tmp0 = in1 ^ in2;
-        tmp1 = tmp0 ^ in3;
-        tmp2 = tmp0 ^ in6;
-        out7 = tmp1 ^ in4;
-        tmp3 = tmp2 ^ in0;
-        out5 = tmp3 ^ in7;
-        out4 = tmp3 ^ in2 ^ in5;
-        out2 = tmp1 ^ out5;
-        out6 = tmp2 ^ out2;
-        out0 = out4 ^ out7 ^ in6;
-        out1 = tmp3 ^ out0;
-        out3 = out6 ^ in5;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_7E(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0, tmp1, tmp2;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        tmp0 = in3 ^ in4;
-        tmp1 = in0 ^ in5;
-        out1 = tmp0 ^ tmp1 ^ in6;
-        out3 = tmp1 ^ in1;
-        out4 = out1 ^ in1 ^ in7;
-        tmp2 = out4 ^ in3;
-        out5 = tmp2 ^ in2;
-        out6 = tmp0 ^ out5;
-        out7 = tmp1 ^ out4 ^ in2;
-        out2 = out6 ^ in5 ^ in7;
-        out0 = tmp2 ^ out2;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_7F(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0, tmp1, tmp2, tmp3;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        tmp0 = in2 ^ in7;
-        tmp1 = tmp0 ^ in3 ^ in5;
-        tmp2 = tmp1 ^ in0;
-        out0 = tmp2 ^ in4;
-        out6 = tmp2 ^ in1;
-        out3 = tmp0 ^ out6;
-        tmp3 = out3 ^ in6;
-        out1 = tmp3 ^ in4;
-        out2 = tmp3 ^ in5;
-        out4 = tmp3 ^ in7;
-        out5 = tmp1 ^ out1;
-        out7 = out0 ^ out4 ^ in3;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_80(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0, tmp1, tmp2;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        tmp0 = in2 ^ in3;
-        tmp1 = in4 ^ in5;
-        out1 = in2 ^ in6 ^ in7;
-        out5 = tmp0 ^ in4;
-        tmp2 = tmp0 ^ in1;
-        out6 = tmp1 ^ in3;
-        out7 = tmp1 ^ in0 ^ in6;
-        out4 = tmp2 ^ in7;
-        out3 = tmp2 ^ out6;
-        out2 = out3 ^ out5 ^ in6;
-        out0 = out2 ^ in3 ^ in7;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_81(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0, tmp1;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        tmp0 = in4 ^ in6;
-        tmp1 = tmp0 ^ in3;
-        out6 = tmp1 ^ in5;
-        out5 = out6 ^ in2 ^ in6;
-        out3 = out5 ^ in1;
-        out2 = tmp0 ^ out3;
-        out1 = out3 ^ out6 ^ in7;
-        out4 = tmp1 ^ out1;
-        out7 = out2 ^ out4 ^ in0;
-        out0 = out7 ^ in1 ^ in4;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_82(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        out4 = in1 ^ in2;
-        tmp0 = in6 ^ in7;
-        out5 = in2 ^ in3;
-        out6 = in3 ^ in4;
-        out7 = in0 ^ in4 ^ in5;
-        out0 = in1 ^ in5 ^ in6;
-        out1 = tmp0 ^ in0 ^ in2;
-        out2 = tmp0 ^ in3 ^ in5;
-        out3 = tmp0 ^ out0 ^ in4;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_83(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0, tmp1, tmp2, tmp3, tmp4;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        tmp0 = in0 ^ in1;
-        tmp1 = in2 ^ in5;
-        tmp2 = in3 ^ in6;
-        out4 = in1 ^ in2 ^ in4;
-        out0 = tmp0 ^ in5 ^ in6;
-        out5 = tmp1 ^ in3;
-        tmp3 = tmp1 ^ in7;
-        out6 = tmp2 ^ in4;
-        out2 = tmp2 ^ tmp3;
-        tmp4 = tmp3 ^ out4;
-        out1 = tmp3 ^ out0;
-        out3 = tmp4 ^ in3;
-        out7 = tmp0 ^ tmp4;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_84(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        out1 = in2 ^ in6;
-        out6 = in3 ^ in5;
-        out0 = in1 ^ in5 ^ in7;
-        out7 = in0 ^ in4 ^ in6;
-        out4 = in1 ^ in3 ^ in6;
-        out5 = in2 ^ in4 ^ in7;
-        out2 = out6 ^ in0 ^ in1;
-        out3 = out5 ^ in5 ^ in6;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_85(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0, tmp1, tmp2, tmp3;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        tmp0 = in1 ^ in6;
-        tmp1 = in3 ^ in6;
-        tmp2 = tmp0 ^ in4;
-        out1 = tmp0 ^ in2;
-        out6 = tmp1 ^ in5;
-        out4 = tmp2 ^ in3;
-        tmp3 = out1 ^ out6;
-        out2 = tmp3 ^ in0;
-        out3 = tmp2 ^ tmp3 ^ in7;
-        out7 = out2 ^ out3 ^ in1;
-        out5 = tmp1 ^ out3;
-        out0 = tmp2 ^ out7 ^ in5;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_86(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        out6 = in3;
-        out7 = in0 ^ in4;
-        out0 = in1 ^ in5;
-        out5 = in2 ^ in7;
-        out3 = in4 ^ in5 ^ in6;
-        out1 = in0 ^ in2 ^ in6;
-        out4 = in1 ^ in6 ^ in7;
-        out2 = in0 ^ in3 ^ in5 ^ in7;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_87(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0, tmp1;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        out6 = in3 ^ in6;
-        tmp0 = in0 ^ in1;
-        out7 = in0 ^ in4 ^ in7;
-        out5 = in2 ^ in5 ^ in7;
-        out3 = out6 ^ in4 ^ in5;
-        out0 = tmp0 ^ in5;
-        tmp1 = tmp0 ^ in6;
-        out2 = out5 ^ in0 ^ in3;
-        out1 = tmp1 ^ in2;
-        out4 = tmp1 ^ out7;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_88(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0, tmp1;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        out1 = in2 ^ in7;
-        tmp0 = in5 ^ in6;
-        out0 = in1 ^ in6 ^ in7;
-        out6 = in4 ^ in5 ^ in7;
-        out3 = out0 ^ out1 ^ in0 ^ in4;
-        out7 = tmp0 ^ in0;
-        tmp1 = tmp0 ^ in3;
-        out2 = out0 ^ in3;
-        out4 = tmp1 ^ in2;
-        out5 = tmp1 ^ out6;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_89(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0, tmp1, tmp2;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        tmp0 = in0 ^ in7;
-        tmp1 = in2 ^ in7;
-        tmp2 = tmp0 ^ in6;
-        out1 = tmp1 ^ in1;
-        out7 = tmp2 ^ in5;
-        out0 = tmp2 ^ in1;
-        out2 = out1 ^ in3 ^ in6;
-        out6 = out7 ^ in0 ^ in4;
-        out5 = out6 ^ in3;
-        out3 = tmp0 ^ out2 ^ in4;
-        out4 = tmp1 ^ out5;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_8A(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        out0 = in1 ^ in6;
-        out7 = in0 ^ in5;
-        out2 = in3 ^ in6;
-        out6 = in4 ^ in7;
-        out1 = in0 ^ in2 ^ in7;
-        out3 = out0 ^ out6 ^ in0;
-        out4 = out1 ^ out7 ^ in6;
-        out5 = out2 ^ in7;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_8B(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0, tmp1, tmp2, tmp3, tmp4;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        tmp0 = in0 ^ in1;
-        tmp1 = in3 ^ in6;
-        tmp2 = in5 ^ in7;
-        tmp3 = tmp0 ^ in7;
-        out0 = tmp0 ^ in6;
-        out2 = tmp1 ^ in2;
-        out5 = tmp1 ^ tmp2;
-        out7 = tmp2 ^ in0;
-        tmp4 = tmp3 ^ in4;
-        out1 = tmp3 ^ in2;
-        out6 = tmp4 ^ out0;
-        out4 = out6 ^ in2 ^ in5;
-        out3 = tmp1 ^ tmp4;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_8C(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        out1 = in2;
-        out0 = in1 ^ in7;
-        out7 = in0 ^ in6;
-        out5 = in4 ^ in6;
-        out6 = in5 ^ in7;
-        out2 = out0 ^ in0 ^ in3;
-        out3 = out5 ^ out7 ^ in2 ^ in7;
-        out4 = out6 ^ in3;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_8D(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        out1 = in1 ^ in2;
-        tmp0 = in6 ^ in7;
-        out0 = in0 ^ in1 ^ in7;
-        out5 = in4 ^ in5 ^ in6;
-        out6 = tmp0 ^ in5;
-        out7 = tmp0 ^ in0;
-        out4 = tmp0 ^ out5 ^ in3;
-        out2 = out0 ^ in2 ^ in3;
-        out3 = out2 ^ in1 ^ in4;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_8E(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        out0 = in1;
-        out4 = in5;
-        out7 = in0;
-        out5 = in6;
-        out6 = in7;
-        out3 = in0 ^ in4;
-        out1 = in0 ^ in2;
-        out2 = in0 ^ in3;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_8F(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        out0 = in0 ^ in1;
-        tmp0 = in0 ^ in3;
-        out4 = in4 ^ in5;
-        out7 = in0 ^ in7;
-        out5 = in5 ^ in6;
-        out6 = in6 ^ in7;
-        out1 = out0 ^ in2;
-        out2 = tmp0 ^ in2;
-        out3 = tmp0 ^ in4;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_90(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0, tmp1, tmp2;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        tmp0 = in1 ^ in2;
-        tmp1 = in2 ^ in6 ^ in7;
-        out3 = tmp0 ^ in7;
-        out1 = tmp1 ^ in5;
-        tmp2 = out1 ^ in4;
-        out6 = tmp2 ^ in3;
-        out5 = out6 ^ in1;
-        out4 = out5 ^ in0;
-        out0 = tmp0 ^ tmp2;
-        out7 = tmp0 ^ out4;
-        out2 = tmp1 ^ out5;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_91(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0, tmp1, tmp2, tmp3;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        tmp0 = in2 ^ in4;
-        tmp1 = tmp0 ^ in3 ^ in5;
-        out2 = tmp1 ^ in1;
-        out6 = tmp1 ^ in7;
-        tmp2 = out2 ^ in5 ^ in7;
-        out3 = tmp2 ^ in4;
-        out5 = tmp2 ^ in6;
-        out1 = tmp1 ^ out5 ^ in2;
-        tmp3 = out1 ^ in0;
-        out4 = tmp3 ^ in3;
-        out0 = tmp0 ^ tmp3;
-        out7 = tmp2 ^ tmp3;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_92(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0, tmp1;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        out3 = in1;
-        tmp0 = in4 ^ in5;
-        tmp1 = tmp0 ^ in1;
-        out2 = tmp0 ^ in3 ^ in7;
-        out0 = tmp1 ^ in6;
-        out7 = out2 ^ in0;
-        out4 = out0 ^ in0 ^ in2;
-        out5 = out4 ^ out7 ^ in5;
-        out6 = tmp1 ^ out5;
-        out1 = out6 ^ out7 ^ in7;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_93(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0, tmp1, tmp2;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        out3 = in1 ^ in3;
-        tmp0 = in2 ^ in7;
-        tmp1 = out3 ^ in6;
-        tmp2 = tmp0 ^ in4;
-        out5 = tmp0 ^ tmp1;
-        out6 = tmp2 ^ in3;
-        out2 = out6 ^ in5;
-        out0 = out2 ^ out5 ^ in0;
-        out7 = tmp1 ^ out0;
-        out1 = tmp2 ^ out0;
-        out4 = out1 ^ in7;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_94(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        out3 = in2 ^ in6;
-        tmp0 = in1 ^ in4 ^ in5;
-        out1 = out3 ^ in5;
-        out5 = tmp0 ^ out3;
-        out0 = tmp0 ^ in7;
-        out4 = tmp0 ^ in0 ^ in3;
-        out6 = out1 ^ in3 ^ in7;
-        out2 = out4 ^ in6;
-        out7 = out0 ^ out2 ^ in4;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_95(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0, tmp1, tmp2, tmp3, tmp4, tmp5;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        tmp0 = in2 ^ in3;
-        out3 = tmp0 ^ in6;
-        tmp1 = tmp0 ^ in7;
-        tmp2 = out3 ^ in0;
-        out6 = tmp1 ^ in5;
-        tmp3 = tmp2 ^ in4;
-        out7 = tmp3 ^ in2;
-        tmp4 = tmp3 ^ in5;
-        out2 = tmp4 ^ in1;
-        tmp5 = out2 ^ in6;
-        out0 = tmp1 ^ tmp5;
-        out1 = tmp5 ^ out7;
-        out4 = tmp2 ^ out1;
-        out5 = tmp4 ^ out4;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_96(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0, tmp1, tmp2;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        out3 = in6 ^ in7;
-        tmp0 = in1 ^ in5;
-        tmp1 = in5 ^ in6;
-        out6 = out3 ^ in2 ^ in3;
-        out0 = tmp0 ^ in4;
-        tmp2 = tmp1 ^ in2;
-        out4 = out0 ^ in0 ^ in7;
-        out1 = tmp2 ^ in0;
-        out5 = tmp2 ^ in1;
-        out7 = tmp0 ^ out4 ^ in3;
-        out2 = tmp1 ^ out7;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_97(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0, tmp1, tmp2, tmp3;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        tmp0 = in0 ^ in4;
-        tmp1 = in2 ^ in6;
-        out3 = in3 ^ in6 ^ in7;
-        out7 = tmp0 ^ in3;
-        tmp2 = tmp0 ^ in5;
-        out5 = tmp1 ^ in1;
-        out6 = tmp1 ^ out3;
-        out0 = tmp2 ^ in1;
-        out2 = tmp2 ^ out3 ^ in2;
-        tmp3 = out0 ^ in4;
-        out4 = tmp3 ^ in7;
-        out1 = tmp1 ^ tmp3;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_98(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0, tmp1;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        tmp0 = in5 ^ in7;
-        tmp1 = in1 ^ in4 ^ in7;
-        out1 = tmp0 ^ in2;
-        out0 = tmp1 ^ in6;
-        out2 = tmp1 ^ in3;
-        out6 = out0 ^ out1 ^ in1;
-        out5 = tmp0 ^ out2;
-        out3 = tmp1 ^ out6 ^ in0;
-        out7 = out0 ^ out5 ^ in0;
-        out4 = out6 ^ out7 ^ in7;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_99(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0, tmp1, tmp2;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        tmp0 = in0 ^ in3;
-        out5 = in1 ^ in3 ^ in4;
-        out6 = in2 ^ in4 ^ in5;
-        out4 = tmp0 ^ in2;
-        tmp1 = tmp0 ^ in6;
-        tmp2 = out5 ^ in7;
-        out7 = tmp1 ^ in5;
-        out0 = tmp1 ^ tmp2;
-        out2 = tmp2 ^ in2;
-        out3 = out0 ^ out6 ^ in3;
-        out1 = tmp1 ^ out3;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_9A(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0, tmp1, tmp2;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        out2 = in3 ^ in4;
-        tmp0 = in0 ^ in5;
-        tmp1 = in1 ^ in6;
-        out5 = in1 ^ in3 ^ in5;
-        tmp2 = tmp0 ^ in7;
-        out3 = tmp0 ^ tmp1;
-        out0 = tmp1 ^ in4;
-        out7 = tmp2 ^ in3;
-        out1 = tmp2 ^ in2;
-        out6 = out0 ^ in1 ^ in2;
-        out4 = out1 ^ in4 ^ in5;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_9B(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        out5 = in1 ^ in3;
-        tmp0 = in3 ^ in5;
-        out6 = in2 ^ in4;
-        out4 = in0 ^ in2 ^ in7;
-        out7 = tmp0 ^ in0;
-        out2 = out6 ^ in3;
-        out1 = out4 ^ in1 ^ in5;
-        out3 = out7 ^ in1 ^ in6;
-        out0 = tmp0 ^ out3 ^ in4;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_9C(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        out1 = in2 ^ in5;
-        tmp0 = in0 ^ in3 ^ in6;
-        out3 = out1 ^ in0;
-        out6 = out1 ^ in6;
-        out7 = tmp0 ^ in7;
-        out4 = out7 ^ in4;
-        out2 = out4 ^ in1;
-        out0 = tmp0 ^ out2;
-        out5 = out0 ^ in5;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_9D(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        out6 = in2 ^ in5;
-        tmp0 = in0 ^ in3;
-        out5 = in1 ^ in4 ^ in7;
-        out1 = out6 ^ in1;
-        out3 = tmp0 ^ out6;
-        out7 = tmp0 ^ in6;
-        out0 = out5 ^ in0;
-        out4 = out7 ^ in7;
-        out2 = out5 ^ out7 ^ in2;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_9E(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        out0 = in1 ^ in4;
-        tmp0 = in0 ^ in5;
-        out6 = in2 ^ in6;
-        out7 = in0 ^ in3 ^ in7;
-        out4 = in0 ^ in4 ^ in6;
-        out5 = in1 ^ in5 ^ in7;
-        out1 = tmp0 ^ in2;
-        out3 = tmp0 ^ in7;
-        out2 = out4 ^ in3;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_9F(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        out6 = in2;
-        out7 = in0 ^ in3;
-        tmp0 = in0 ^ in1;
-        out4 = in0 ^ in6;
-        out5 = in1 ^ in7;
-        out1 = tmp0 ^ in2 ^ in5;
-        out2 = out7 ^ in2 ^ in4 ^ in6;
-        out3 = out7 ^ in5 ^ in7;
-        out0 = tmp0 ^ in4;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_A0(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0, tmp1, tmp2, tmp3;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        tmp0 = in1 ^ in6;
-        out2 = tmp0 ^ in7;
-        tmp1 = tmp0 ^ in5;
-        out6 = out2 ^ in3 ^ in4;
-        out0 = tmp1 ^ in3;
-        tmp2 = out0 ^ in2;
-        out3 = tmp2 ^ in7;
-        tmp3 = tmp2 ^ in1;
-        out5 = tmp3 ^ in0;
-        out4 = tmp3 ^ out6;
-        out7 = out5 ^ out6 ^ in1;
-        out1 = tmp1 ^ out4;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_A1(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0, tmp1, tmp2;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        tmp0 = in2 ^ in5;
-        tmp1 = tmp0 ^ in1;
-        tmp2 = tmp0 ^ in4;
-        out4 = tmp1 ^ in7;
-        out7 = tmp2 ^ in0;
-        out6 = tmp2 ^ out4 ^ in3;
-        out3 = out4 ^ in6;
-        out2 = out3 ^ in5;
-        out1 = out2 ^ in4;
-        out5 = out1 ^ out6 ^ in0;
-        out0 = tmp1 ^ out5;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_A2(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        out2 = in6;
-        tmp0 = in1 ^ in3 ^ in5;
-        out3 = tmp0 ^ in6;
-        out4 = tmp0 ^ in2 ^ in4;
-        out0 = out3 ^ in7;
-        out6 = out0 ^ in4;
-        out1 = out0 ^ out4 ^ in0;
-        out7 = out1 ^ in5;
-        out5 = out7 ^ in3 ^ in7;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_A3(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0, tmp1;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        out2 = in2 ^ in6;
-        out3 = in1 ^ in5 ^ in6;
-        tmp0 = out2 ^ in0;
-        out4 = out2 ^ out3 ^ in3;
-        tmp1 = tmp0 ^ in4;
-        out0 = tmp0 ^ out4 ^ in7;
-        out5 = tmp1 ^ in3;
-        out7 = tmp1 ^ in5;
-        out1 = tmp1 ^ in1 ^ in7;
-        out6 = tmp1 ^ out0 ^ in2;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_A4(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0, tmp1, tmp2, tmp3, tmp4;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        tmp0 = in1 ^ in3;
-        tmp1 = in2 ^ in4;
-        tmp2 = in2 ^ in5;
-        tmp3 = in0 ^ in7;
-        out0 = tmp0 ^ in5;
-        out6 = tmp0 ^ in6 ^ in7;
-        out1 = tmp1 ^ in6;
-        out7 = tmp1 ^ tmp3;
-        out3 = tmp2 ^ in3;
-        tmp4 = tmp2 ^ out1;
-        out2 = tmp3 ^ in1;
-        out5 = tmp4 ^ out7;
-        out4 = tmp4 ^ in1;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_A5(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0, tmp1, tmp2;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        out3 = in2 ^ in5;
-        tmp0 = in1 ^ in6;
-        tmp1 = in0 ^ in1;
-        tmp2 = in2 ^ in4;
-        out6 = in1 ^ in3 ^ in7;
-        out4 = tmp0 ^ in5;
-        out1 = tmp0 ^ tmp2;
-        out0 = tmp1 ^ in3 ^ in5;
-        out2 = tmp1 ^ in2 ^ in7;
-        out7 = tmp2 ^ in0;
-        out5 = tmp0 ^ out2;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_A6(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        out2 = in0;
-        out3 = in3 ^ in5 ^ in7;
-        out1 = in0 ^ in2 ^ in4 ^ in6;
-        out0 = out3 ^ in1;
-        out7 = out1 ^ in7;
-        out6 = out0 ^ in6;
-        out5 = out7 ^ in5;
-        out4 = out6 ^ in4;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_A7(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        out2 = in0 ^ in2;
-        out3 = in5 ^ in7;
-        out7 = out2 ^ in4 ^ in6;
-        out6 = out3 ^ in1 ^ in3;
-        out1 = out7 ^ in1;
-        out5 = out7 ^ in7;
-        out0 = out6 ^ in0;
-        out4 = out6 ^ in6;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_A8(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0, tmp1, tmp2;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        tmp0 = in2 ^ in4;
-        tmp1 = in1 ^ in6;
-        tmp2 = in0 ^ in2 ^ in7;
-        out1 = tmp0 ^ in7;
-        out4 = tmp0 ^ in6;
-        out0 = tmp1 ^ in3;
-        out2 = tmp1 ^ in5;
-        out6 = tmp1 ^ in4;
-        out7 = tmp2 ^ in5;
-        out3 = tmp2 ^ out0 ^ in6;
-        out5 = out7 ^ in2 ^ in3;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_A9(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        out4 = in2 ^ in6;
-        out6 = in1 ^ in4;
-        out7 = in0 ^ in2 ^ in5;
-        out5 = in0 ^ in3 ^ in7;
-        out2 = out4 ^ in1 ^ in5;
-        out1 = out6 ^ in2 ^ in7;
-        out0 = out2 ^ out7 ^ in3;
-        out3 = out1 ^ in0 ^ in4;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_AA(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0, tmp1, tmp2;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        tmp0 = in0 ^ in2;
-        tmp1 = in1 ^ in3;
-        tmp2 = in6 ^ in7;
-        out1 = tmp0 ^ in4 ^ in7;
-        out3 = tmp1 ^ in0;
-        out0 = tmp1 ^ tmp2;
-        out2 = tmp2 ^ in5;
-        out7 = tmp0 ^ out2;
-        out6 = out1 ^ out7 ^ in1;
-        out5 = out0 ^ out6 ^ in0;
-        out4 = out5 ^ out7 ^ in7;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_AB(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0, tmp1;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        out3 = in0 ^ in1;
-        tmp0 = in1 ^ in4;
-        tmp1 = in0 ^ in7;
-        out6 = tmp0 ^ in5;
-        out1 = tmp0 ^ tmp1 ^ in2;
-        out5 = tmp1 ^ in3 ^ in4;
-        out0 = tmp0 ^ out5 ^ in6;
-        out4 = out0 ^ out3 ^ in2;
-        out2 = out4 ^ in3 ^ in5;
-        out7 = tmp1 ^ out2;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_AC(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        out0 = in1 ^ in3;
-        out1 = in2 ^ in4;
-        tmp0 = in0 ^ in2;
-        out4 = in4 ^ in7;
-        out5 = in0 ^ in5;
-        out6 = in1 ^ in6;
-        out7 = tmp0 ^ in7;
-        out3 = tmp0 ^ in3 ^ in6;
-        out2 = out5 ^ in1;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_AD(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        out4 = in7;
-        out5 = in0;
-        out6 = in1;
-        out7 = in0 ^ in2;
-        out0 = in0 ^ in1 ^ in3;
-        out2 = out7 ^ in1 ^ in5;
-        out1 = in1 ^ in2 ^ in4;
-        out3 = out7 ^ in6;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_AE(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0, tmp1, tmp2;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        out4 = in3 ^ in4;
-        tmp0 = in0 ^ in4;
-        tmp1 = in0 ^ in7;
-        out0 = in1 ^ in3 ^ in7;
-        out1 = tmp0 ^ in2;
-        out5 = tmp0 ^ in5;
-        tmp2 = tmp1 ^ in6;
-        out2 = tmp1 ^ in5;
-        out3 = tmp2 ^ in3;
-        out7 = tmp2 ^ in2;
-        out6 = tmp2 ^ out2 ^ in1;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_AF(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        out4 = in3;
-        tmp0 = in0 ^ in7;
-        out5 = in0 ^ in4;
-        out6 = in1 ^ in5;
-        out7 = in0 ^ in2 ^ in6;
-        out0 = tmp0 ^ in1 ^ in3;
-        out3 = tmp0 ^ in6;
-        out2 = tmp0 ^ in2 ^ in5;
-        out1 = out5 ^ in1 ^ in2;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_B0(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0, tmp1, tmp2, tmp3;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        tmp0 = in1 ^ in4;
-        tmp1 = in3 ^ in6;
-        out2 = tmp0 ^ in7;
-        tmp2 = tmp0 ^ tmp1;
-        out0 = tmp2 ^ in5;
-        out3 = tmp2 ^ in2;
-        out6 = out3 ^ in6;
-        tmp3 = out6 ^ in0 ^ in1;
-        out7 = tmp3 ^ in5;
-        out5 = tmp3 ^ out2;
-        out1 = out0 ^ out5 ^ in0;
-        out4 = tmp1 ^ out5;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_B1(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0, tmp1;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        tmp0 = in1 ^ in4;
-        out2 = tmp0 ^ in2 ^ in7;
-        tmp1 = out2 ^ in6;
-        out1 = tmp1 ^ in5;
-        out3 = tmp1 ^ in7;
-        out4 = tmp1 ^ in0;
-        out6 = out3 ^ in3;
-        out0 = out6 ^ in0 ^ in2 ^ in5;
-        out5 = tmp1 ^ out0 ^ in1;
-        out7 = tmp0 ^ out5;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_B2(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0, tmp1, tmp2, tmp3, tmp4;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        out2 = in4;
-        tmp0 = in4 ^ in7;
-        tmp1 = in1 ^ in3 ^ in6;
-        out3 = tmp0 ^ tmp1;
-        tmp2 = tmp1 ^ in0;
-        out0 = out3 ^ in5;
-        out4 = tmp2 ^ in2;
-        tmp3 = out4 ^ in6;
-        out5 = tmp0 ^ tmp3;
-        out1 = tmp3 ^ out0;
-        tmp4 = out1 ^ in7;
-        out7 = tmp4 ^ in3;
-        out6 = tmp2 ^ tmp4;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_B3(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0, tmp1, tmp2;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        out2 = in2 ^ in4;
-        tmp0 = in0 ^ in5;
-        tmp1 = in1 ^ in6;
-        out3 = tmp1 ^ in4 ^ in7;
-        tmp2 = tmp0 ^ out3;
-        out0 = tmp2 ^ in3;
-        out1 = tmp2 ^ in2;
-        out5 = out0 ^ in2 ^ in6;
-        out7 = tmp1 ^ out5;
-        out4 = out7 ^ in1 ^ in5 ^ in7;
-        out6 = tmp0 ^ out4;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_B4(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        out4 = in0 ^ in1;
-        out5 = out4 ^ in2;
-        tmp0 = out4 ^ in4;
-        out6 = out5 ^ in0 ^ in3;
-        out7 = tmp0 ^ out6;
-        out2 = tmp0 ^ in6 ^ in7;
-        out3 = out7 ^ in0 ^ in7;
-        out0 = out5 ^ out7 ^ in5;
-        out1 = out0 ^ out6 ^ in6;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_B5(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0, tmp1, tmp2;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        tmp0 = in0 ^ in1;
-        tmp1 = in2 ^ in4;
-        out4 = tmp0 ^ in4;
-        out3 = tmp1 ^ in7;
-        tmp2 = out4 ^ in5;
-        out7 = out3 ^ in0 ^ in3;
-        out0 = tmp2 ^ in3;
-        out2 = tmp0 ^ out3 ^ in6;
-        out5 = tmp1 ^ tmp2;
-        out6 = out2 ^ out7 ^ in2;
-        out1 = tmp0 ^ out0 ^ out6;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_B6(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0, tmp1, tmp2, tmp3;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        out3 = in3 ^ in4;
-        tmp0 = in1 ^ in2;
-        tmp1 = in0 ^ in4;
-        tmp2 = in3 ^ in5;
-        tmp3 = out3 ^ in1 ^ in7;
-        out5 = tmp0 ^ tmp1;
-        out6 = tmp0 ^ tmp2;
-        out2 = tmp1 ^ in6;
-        out4 = tmp1 ^ tmp3;
-        out0 = tmp3 ^ in5;
-        out1 = out2 ^ in2 ^ in5;
-        out7 = tmp2 ^ out1;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_B7(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0, tmp1;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        out3 = in4;
-        tmp0 = in0 ^ in4;
-        out2 = tmp0 ^ in2 ^ in6;
-        tmp1 = out2 ^ in7;
-        out1 = out2 ^ in1 ^ in5;
-        out7 = tmp1 ^ in3;
-        out5 = out1 ^ in6;
-        out6 = tmp0 ^ out1 ^ in3;
-        out0 = tmp1 ^ out6;
-        out4 = out0 ^ in5;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_B8(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0, tmp1, tmp2;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        tmp0 = in1 ^ in4;
-        tmp1 = in2 ^ in5;
-        out2 = tmp0 ^ in5;
-        out4 = tmp1 ^ in0;
-        tmp2 = tmp1 ^ in7;
-        out6 = tmp2 ^ out2;
-        out7 = out4 ^ in3;
-        out1 = tmp2 ^ in4;
-        out3 = tmp0 ^ out7;
-        out0 = out3 ^ out4 ^ in6;
-        out5 = out0 ^ in0 ^ in4;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_B9(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0, tmp1, tmp2;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        tmp0 = in0 ^ in2;
-        tmp1 = in4 ^ in5;
-        out4 = tmp0 ^ tmp1;
-        tmp2 = tmp0 ^ in3 ^ in7;
-        out3 = out4 ^ in1;
-        out7 = tmp2 ^ in5;
-        out2 = out3 ^ in0;
-        out1 = out2 ^ in7;
-        out6 = out1 ^ in5 ^ in6;
-        out0 = tmp2 ^ out6;
-        out5 = tmp1 ^ out0;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_BA(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0, tmp1, tmp2;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        tmp0 = in5 ^ in7;
-        out2 = tmp0 ^ in4;
-        tmp1 = out2 ^ in2;
-        out1 = tmp1 ^ in0;
-        out6 = tmp1 ^ in1;
-        out4 = out1 ^ in3 ^ in4;
-        tmp2 = out4 ^ out6;
-        out7 = out4 ^ in6 ^ in7;
-        out5 = tmp2 ^ in6;
-        out3 = tmp0 ^ tmp2;
-        out0 = out6 ^ out7 ^ in0;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_BB(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0, tmp1;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        out2 = in2 ^ in4 ^ in5 ^ in7;
-        tmp0 = out2 ^ in1;
-        out4 = out2 ^ in0 ^ in3;
-        out1 = tmp0 ^ in0;
-        out6 = tmp0 ^ in6;
-        out3 = out1 ^ in2;
-        tmp1 = out4 ^ out6 ^ in4;
-        out0 = tmp1 ^ in7;
-        out5 = tmp1 ^ in5;
-        out7 = tmp0 ^ tmp1;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_BC(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0, tmp1, tmp2;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        tmp0 = in0 ^ in2;
-        tmp1 = in2 ^ in4;
-        out0 = in1 ^ in3 ^ in4;
-        out6 = in1 ^ in2 ^ in7;
-        out7 = tmp0 ^ in3;
-        out5 = tmp0 ^ out6 ^ in6;
-        out1 = tmp1 ^ in5;
-        tmp2 = out1 ^ out5 ^ in1;
-        out3 = tmp2 ^ in3;
-        out4 = tmp1 ^ tmp2;
-        out2 = tmp2 ^ out6;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_BD(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0, tmp1, tmp2;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        tmp0 = in0 ^ in3;
-        tmp1 = in1 ^ in4;
-        out0 = tmp0 ^ tmp1;
-        out7 = tmp0 ^ in2 ^ in7;
-        out1 = tmp1 ^ in2 ^ in5;
-        tmp2 = out1 ^ in0;
-        out2 = tmp2 ^ in6;
-        out3 = out2 ^ in1 ^ in7;
-        out4 = out3 ^ in2;
-        out5 = tmp1 ^ out4;
-        out6 = tmp2 ^ out4;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_BE(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        tmp0 = in0 ^ in3 ^ in6;
-        out4 = tmp0 ^ in5;
-        out7 = tmp0 ^ in2;
-        out3 = out4 ^ in4;
-        out1 = out3 ^ out7 ^ in0;
-        out2 = out3 ^ in3 ^ in7;
-        out0 = out2 ^ out4 ^ in1;
-        out5 = tmp0 ^ out0;
-        out6 = out1 ^ out5 ^ in6;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_BF(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0, tmp1, tmp2, tmp3;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        tmp0 = in0 ^ in4;
-        out3 = tmp0 ^ in5 ^ in6;
-        out4 = out3 ^ in3;
-        tmp1 = out3 ^ in7;
-        out2 = tmp1 ^ in2;
-        out5 = tmp1 ^ in1;
-        tmp2 = out2 ^ in5;
-        out7 = tmp2 ^ in3 ^ in4;
-        tmp3 = tmp0 ^ out5;
-        out0 = tmp3 ^ out4;
-        out1 = tmp2 ^ tmp3;
-        out6 = tmp3 ^ in2;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_C0(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0, tmp1;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        out5 = in2 ^ in5;
-        tmp0 = in1 ^ in4;
-        tmp1 = in3 ^ in6;
-        out0 = out5 ^ in1;
-        out4 = tmp0 ^ in7;
-        out3 = tmp0 ^ tmp1;
-        out1 = tmp1 ^ in2;
-        out6 = tmp1 ^ in0;
-        out7 = out4 ^ in0;
-        out2 = out4 ^ out5 ^ in3;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_C1(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0, tmp1;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        out5 = in2;
-        tmp0 = in0 ^ in1;
-        out4 = in1 ^ in7;
-        out6 = in0 ^ in3;
-        out3 = in1 ^ in4 ^ in6;
-        tmp1 = tmp0 ^ in2;
-        out7 = tmp0 ^ in4;
-        out0 = tmp1 ^ in5;
-        out1 = tmp1 ^ out6 ^ in6;
-        out2 = out6 ^ out7 ^ in5 ^ in7;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_C2(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0, tmp1;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        out4 = in1 ^ in3 ^ in4;
-        tmp0 = in0 ^ in3 ^ in6;
-        out5 = in2 ^ in4 ^ in5;
-        tmp1 = out4 ^ in7;
-        out1 = tmp0 ^ in2;
-        out6 = tmp0 ^ in5;
-        out2 = out5 ^ in3;
-        out7 = tmp0 ^ tmp1;
-        out3 = tmp1 ^ in2 ^ in6;
-        out0 = tmp1 ^ out2;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_C3(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0, tmp1, tmp2;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        out4 = in1 ^ in3;
-        tmp0 = in0 ^ in2;
-        tmp1 = in3 ^ in5;
-        out5 = in2 ^ in4;
-        tmp2 = tmp0 ^ out4;
-        out2 = tmp1 ^ in4;
-        out6 = tmp1 ^ in0;
-        out0 = tmp1 ^ tmp2 ^ in7;
-        out1 = tmp2 ^ in6;
-        out7 = out1 ^ out5 ^ in3;
-        out3 = tmp0 ^ out7 ^ in7;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_C4(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0, tmp1;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        tmp0 = in3 ^ in7;
-        out3 = tmp0 ^ in4;
-        tmp1 = tmp0 ^ in2;
-        out1 = tmp1 ^ in6;
-        out5 = tmp1 ^ in5;
-        out4 = out1 ^ out3 ^ in1;
-        out0 = out4 ^ in4 ^ in5;
-        out2 = out0 ^ out3 ^ in0;
-        out7 = out1 ^ out2 ^ in7;
-        out6 = tmp1 ^ out0 ^ out7;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_C5(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        out3 = in4 ^ in7;
-        tmp0 = in3 ^ in7;
-        out4 = in1 ^ in2 ^ in6;
-        out6 = in0 ^ in3 ^ in4;
-        out5 = tmp0 ^ in2;
-        out1 = tmp0 ^ out4;
-        out0 = out4 ^ in0 ^ in5;
-        out2 = out0 ^ out5 ^ in4;
-        out7 = tmp0 ^ out2 ^ in6;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_C6(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0, tmp1, tmp2, tmp3, tmp4, tmp5;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        tmp0 = in5 ^ in6;
-        tmp1 = in1 ^ in7;
-        tmp2 = tmp0 ^ in0;
-        tmp3 = tmp0 ^ tmp1;
-        tmp4 = tmp2 ^ in4;
-        out0 = tmp3 ^ in2;
-        out6 = tmp4 ^ in3;
-        out2 = out6 ^ in2;
-        out7 = tmp1 ^ tmp4;
-        out3 = tmp2 ^ out2;
-        tmp5 = out3 ^ in5;
-        out5 = tmp5 ^ in7;
-        out4 = tmp3 ^ tmp5;
-        out1 = tmp4 ^ out5;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_C7(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0, tmp1;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        out3 = in2 ^ in4;
-        tmp0 = in3 ^ in5;
-        tmp1 = out3 ^ in7;
-        out6 = tmp0 ^ in0 ^ in4;
-        out5 = tmp1 ^ in3;
-        out2 = out6 ^ in6;
-        out7 = out2 ^ in1 ^ in3;
-        out0 = tmp1 ^ out7;
-        out1 = tmp0 ^ out0;
-        out4 = out1 ^ in0;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_C8(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0, tmp1;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        out0 = in1 ^ in2;
-        out1 = in2 ^ in3;
-        tmp0 = in5 ^ in6;
-        tmp1 = in0 ^ in7;
-        out2 = out1 ^ in1 ^ in4;
-        out4 = tmp0 ^ in4;
-        out5 = tmp0 ^ in7;
-        out6 = tmp1 ^ in6;
-        out7 = tmp1 ^ in1;
-        out3 = out2 ^ in0 ^ in2 ^ in5;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_C9(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        out4 = in5 ^ in6;
-        out7 = in0 ^ in1;
-        tmp0 = in1 ^ in3;
-        out5 = in6 ^ in7;
-        out6 = in0 ^ in7;
-        out0 = out7 ^ in2;
-        out3 = out7 ^ in4 ^ in5;
-        out1 = tmp0 ^ in2;
-        out2 = tmp0 ^ in4;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_CA(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0, tmp1, tmp2, tmp3;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        tmp0 = in0 ^ in7;
-        tmp1 = in2 ^ in7;
-        tmp2 = tmp0 ^ in6;
-        out0 = tmp1 ^ in1;
-        tmp3 = tmp1 ^ in3;
-        out6 = tmp2 ^ in5;
-        out7 = tmp2 ^ in1;
-        out2 = tmp3 ^ in4;
-        out5 = out6 ^ in0 ^ in4;
-        out4 = out5 ^ in3;
-        out1 = tmp0 ^ tmp3;
-        out3 = tmp3 ^ out5 ^ out7;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_CB(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0, tmp1, tmp2;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        tmp0 = in4 ^ in7;
-        tmp1 = in5 ^ in7;
-        out7 = in0 ^ in1 ^ in6;
-        out5 = tmp0 ^ in6;
-        out2 = tmp0 ^ in3;
-        out6 = tmp1 ^ in0;
-        out4 = tmp1 ^ in3 ^ in6;
-        tmp2 = out5 ^ out7 ^ in2;
-        out1 = tmp2 ^ out2;
-        out0 = tmp2 ^ in4;
-        out3 = tmp2 ^ in5;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_CC(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0, tmp1, tmp2, tmp3;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        tmp0 = in3 ^ in5;
-        tmp1 = in1 ^ in6;
-        out1 = in2 ^ in3 ^ in7;
-        out5 = tmp0 ^ in6;
-        out0 = tmp1 ^ in2;
-        tmp2 = out5 ^ in0 ^ in7;
-        out3 = tmp2 ^ in4;
-        out6 = tmp0 ^ out3;
-        out7 = tmp1 ^ tmp2 ^ in3;
-        tmp3 = out1 ^ out6;
-        out4 = tmp2 ^ tmp3;
-        out2 = tmp3 ^ in1;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_CD(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0, tmp1;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        out5 = in3 ^ in6;
-        tmp0 = in0 ^ in1;
-        tmp1 = in2 ^ in7;
-        out6 = in0 ^ in4 ^ in7;
-        out2 = tmp0 ^ out5 ^ in4;
-        out7 = tmp0 ^ in5;
-        out0 = tmp0 ^ in2 ^ in6;
-        out4 = tmp1 ^ in5;
-        out1 = tmp1 ^ in1 ^ in3;
-        out3 = out6 ^ in5 ^ in6;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_CE(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0, tmp1, tmp2;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        tmp0 = in2 ^ in5;
-        tmp1 = tmp0 ^ in3;
-        out4 = tmp1 ^ in4;
-        tmp2 = out4 ^ in6;
-        out3 = tmp2 ^ in0;
-        out5 = tmp2 ^ in2;
-        out2 = out3 ^ in5 ^ in7;
-        out6 = tmp1 ^ out2;
-        out7 = out2 ^ out4 ^ in1;
-        out1 = tmp2 ^ out6;
-        out0 = tmp0 ^ out7 ^ in0;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_CF(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0, tmp1, tmp2;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        tmp0 = in3 ^ in6;
-        tmp1 = in0 ^ in1 ^ in5;
-        out4 = in2 ^ in3 ^ in5;
-        out5 = tmp0 ^ in4;
-        out7 = tmp1 ^ in6;
-        out1 = tmp1 ^ out4 ^ in7;
-        tmp2 = out5 ^ in0;
-        out2 = tmp2 ^ in7;
-        out3 = tmp2 ^ out4;
-        out6 = tmp0 ^ out2 ^ in5;
-        out0 = tmp0 ^ out1;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_D0(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0, tmp1, tmp2, tmp3, tmp4;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        tmp0 = in0 ^ in3;
-        tmp1 = in1 ^ in4;
-        tmp2 = in2 ^ in5;
-        out7 = tmp0 ^ tmp1;
-        out0 = tmp1 ^ tmp2;
-        tmp3 = tmp2 ^ in3;
-        out1 = tmp3 ^ in6;
-        tmp4 = out1 ^ in1;
-        out2 = tmp4 ^ in7;
-        out3 = out2 ^ in2;
-        out4 = tmp0 ^ out3;
-        out5 = tmp3 ^ out3;
-        out6 = tmp4 ^ out4;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_D1(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0, tmp1, tmp2;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        tmp0 = in3 ^ in5 ^ in6;
-        tmp1 = tmp0 ^ in1;
-        out1 = tmp1 ^ in2;
-        out2 = tmp1 ^ in7;
-        out3 = out2 ^ in3;
-        out5 = out3 ^ in2;
-        tmp2 = out3 ^ in0;
-        out4 = tmp2 ^ in4;
-        out7 = tmp0 ^ out4;
-        out6 = tmp2 ^ out1 ^ in6;
-        out0 = out2 ^ out6 ^ in4;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_D2(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        tmp0 = in5 ^ in6;
-        out2 = tmp0 ^ in2 ^ in3;
-        out1 = out2 ^ in0;
-        out3 = out2 ^ in1;
-        out4 = out1 ^ in1 ^ in2;
-        out6 = out1 ^ in6 ^ in7;
-        out7 = out4 ^ in4 ^ in5;
-        out5 = out4 ^ out6 ^ in4;
-        out0 = tmp0 ^ out5;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_D3(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0, tmp1, tmp2, tmp3;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        out2 = in3 ^ in5 ^ in6;
-        tmp0 = out2 ^ in2;
-        tmp1 = tmp0 ^ in1;
-        out1 = tmp1 ^ in0;
-        out3 = tmp1 ^ in3;
-        out4 = out1 ^ in2 ^ in4;
-        tmp2 = out4 ^ in5;
-        out7 = tmp2 ^ in7;
-        out0 = tmp0 ^ out7;
-        tmp3 = out0 ^ in0;
-        out5 = tmp3 ^ in6;
-        out6 = tmp2 ^ tmp3;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_D4(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0, tmp1, tmp2;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        out3 = in3 ^ in5;
-        tmp0 = in1 ^ in5;
-        tmp1 = tmp0 ^ in2;
-        out4 = tmp1 ^ in0;
-        tmp2 = tmp1 ^ in6;
-        out2 = out4 ^ in3 ^ in7;
-        out0 = tmp2 ^ in4;
-        out5 = tmp2 ^ out3;
-        out1 = tmp0 ^ out5 ^ in7;
-        out6 = tmp0 ^ out2 ^ in4;
-        out7 = tmp1 ^ out6 ^ in7;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_D5(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0, tmp1, tmp2;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        out3 = in5;
-        tmp0 = in0 ^ in4;
-        tmp1 = tmp0 ^ in1 ^ in5;
-        out4 = tmp1 ^ in2;
-        out0 = out4 ^ in6;
-        tmp2 = tmp0 ^ out0;
-        out5 = tmp2 ^ in3;
-        out1 = out5 ^ in7;
-        out6 = tmp1 ^ out1;
-        out7 = tmp2 ^ out6;
-        out2 = out7 ^ in4;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_D6(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0, tmp1;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        tmp0 = in1 ^ in2 ^ in4 ^ in6;
-        out5 = tmp0 ^ in3;
-        out0 = tmp0 ^ in5 ^ in7;
-        out3 = out0 ^ out5 ^ in2;
-        tmp1 = out3 ^ in0;
-        out1 = tmp1 ^ in6;
-        out2 = tmp1 ^ in7;
-        out4 = tmp1 ^ in1;
-        out6 = tmp1 ^ in4;
-        out7 = tmp0 ^ out2;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_D7(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0, tmp1, tmp2;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        tmp0 = in0 ^ in3;
-        out3 = in2 ^ in5 ^ in7;
-        out2 = tmp0 ^ in5;
-        tmp1 = tmp0 ^ out3 ^ in1;
-        out1 = tmp1 ^ in6;
-        out4 = tmp1 ^ in4;
-        tmp2 = out1 ^ in4;
-        out6 = tmp2 ^ in1;
-        out7 = tmp2 ^ in2;
-        out0 = tmp2 ^ in3;
-        out5 = tmp2 ^ in0 ^ in7;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_D8(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0, tmp1;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        out4 = in0;
-        out5 = in1;
-        tmp0 = in1 ^ in2;
-        out6 = in0 ^ in2;
-        out0 = tmp0 ^ in4;
-        tmp1 = tmp0 ^ in3;
-        out7 = tmp1 ^ out6;
-        out2 = tmp1 ^ in6;
-        out3 = out7 ^ in7;
-        out1 = tmp1 ^ in1 ^ in5;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_D9(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        out4 = in0 ^ in4;
-        out5 = in1 ^ in5;
-        out2 = in1 ^ in3 ^ in6;
-        out3 = in0 ^ in1 ^ in7;
-        out6 = in0 ^ in2 ^ in6;
-        out0 = out4 ^ in1 ^ in2;
-        out1 = out5 ^ in2 ^ in3;
-        out7 = out3 ^ in3;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_DA(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0, tmp1;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        out5 = in1 ^ in4;
-        tmp0 = in2 ^ in7;
-        tmp1 = in0 ^ in2 ^ in3;
-        out0 = tmp0 ^ out5;
-        out4 = tmp0 ^ tmp1;
-        out2 = tmp0 ^ in3 ^ in6;
-        out1 = tmp1 ^ in5;
-        out3 = tmp1 ^ in1;
-        out6 = out1 ^ in3;
-        out7 = out3 ^ in2 ^ in6;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_DB(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0, tmp1, tmp2, tmp3, tmp4;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        tmp0 = in0 ^ in1;
-        tmp1 = in1 ^ in5;
-        tmp2 = in3 ^ in7;
-        out3 = tmp0 ^ in2;
-        out5 = tmp1 ^ in4;
-        out6 = tmp1 ^ out3 ^ in6;
-        out2 = tmp2 ^ in6;
-        tmp3 = tmp2 ^ in4;
-        tmp4 = out3 ^ in3;
-        out4 = tmp3 ^ in0;
-        out1 = tmp4 ^ in5;
-        out0 = tmp3 ^ tmp4;
-        out7 = tmp0 ^ out2;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_DC(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0, tmp1, tmp2, tmp3;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        tmp0 = in0 ^ in2;
-        tmp1 = in0 ^ in3;
-        out6 = tmp0 ^ in4;
-        tmp2 = tmp0 ^ in7;
-        out3 = tmp1 ^ in6;
-        tmp3 = tmp1 ^ in1;
-        out1 = tmp1 ^ tmp2 ^ in5;
-        out4 = tmp2 ^ in6;
-        out2 = tmp3 ^ in2;
-        out7 = tmp3 ^ in5;
-        out5 = tmp2 ^ out2;
-        out0 = out2 ^ out3 ^ in4;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_DD(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        out3 = in0 ^ in6;
-        out2 = in0 ^ in1 ^ in3;
-        out6 = out3 ^ in2 ^ in4;
-        out7 = out2 ^ in5 ^ in7;
-        out0 = out6 ^ in1;
-        out4 = out6 ^ in7;
-        out5 = out7 ^ in0;
-        out1 = out5 ^ in2;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_DE(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0, tmp1;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        tmp0 = in2 ^ in3 ^ in6;
-        tmp1 = in3 ^ in4 ^ in7;
-        out4 = tmp0 ^ in0;
-        out5 = tmp1 ^ in1;
-        out3 = out4 ^ in7;
-        out2 = out3 ^ in6;
-        out1 = out2 ^ in5;
-        out6 = tmp1 ^ out1;
-        out0 = tmp0 ^ out5;
-        out7 = out0 ^ out1 ^ in4;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_DF(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0, tmp1, tmp2;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        out2 = in0 ^ in3 ^ in7;
-        tmp0 = out2 ^ in1 ^ in5;
-        out1 = tmp0 ^ in2;
-        out7 = tmp0 ^ in6;
-        out5 = tmp0 ^ in0 ^ in4;
-        tmp1 = out1 ^ out5 ^ in6;
-        out4 = tmp1 ^ in3;
-        out6 = tmp1 ^ in5;
-        tmp2 = tmp1 ^ in7;
-        out0 = tmp2 ^ in1;
-        out3 = tmp2 ^ in4;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_E0(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0, tmp1;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        out3 = in1 ^ in7;
-        tmp0 = in2 ^ in4;
-        out4 = out3 ^ in3 ^ in5;
-        out2 = tmp0 ^ in1;
-        tmp1 = tmp0 ^ in6;
-        out0 = out4 ^ in2;
-        out6 = out4 ^ in0;
-        out1 = tmp1 ^ in3;
-        out5 = tmp1 ^ in0;
-        out7 = out5 ^ in1;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_E1(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0, tmp1, tmp2, tmp3;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        out2 = in1 ^ in4;
-        tmp0 = in1 ^ in7;
-        out3 = tmp0 ^ in3;
-        tmp1 = out3 ^ in5;
-        out4 = tmp1 ^ in4;
-        tmp2 = tmp1 ^ in0;
-        out0 = tmp2 ^ in2;
-        out6 = tmp2 ^ in6;
-        tmp3 = out0 ^ out4 ^ in6;
-        out5 = tmp3 ^ in5;
-        out7 = tmp0 ^ tmp3;
-        out1 = tmp2 ^ out5 ^ in7;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_E2(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        out3 = in1 ^ in2;
-        out4 = in1 ^ in5;
-        out2 = in2 ^ in4 ^ in7;
-        out5 = in0 ^ in2 ^ in6;
-        out0 = out3 ^ in3 ^ in5;
-        out7 = out3 ^ in0 ^ in4;
-        out6 = out2 ^ out7 ^ in3;
-        out1 = out5 ^ in3 ^ in4;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_E3(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0, tmp1, tmp2, tmp3, tmp4;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        out2 = in4 ^ in7;
-        tmp0 = in1 ^ in3;
-        out3 = tmp0 ^ in2;
-        tmp1 = out3 ^ in0;
-        out0 = tmp1 ^ in5;
-        tmp2 = tmp1 ^ in4;
-        out1 = tmp2 ^ in6;
-        tmp3 = tmp2 ^ in3;
-        out7 = tmp3 ^ in7;
-        out6 = out1 ^ out2 ^ in2;
-        tmp4 = tmp0 ^ out0;
-        out5 = tmp4 ^ in6;
-        out4 = tmp3 ^ tmp4;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_E4(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0, tmp1, tmp2;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        out3 = in6;
-        tmp0 = in0 ^ in4;
-        tmp1 = tmp0 ^ in2 ^ in6;
-        out2 = tmp1 ^ in1;
-        out7 = out2 ^ in5;
-        tmp2 = tmp0 ^ out7;
-        out4 = tmp2 ^ in3;
-        out0 = out4 ^ in7;
-        out6 = tmp1 ^ out0;
-        out5 = tmp2 ^ out6;
-        out1 = out5 ^ in0;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_E5(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0, tmp1, tmp2;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        out3 = in3 ^ in6;
-        tmp0 = in0 ^ in1;
-        tmp1 = in5 ^ in7;
-        out2 = tmp0 ^ in4 ^ in6;
-        tmp2 = tmp1 ^ out2;
-        out6 = tmp2 ^ in3;
-        out7 = tmp2 ^ in2;
-        out0 = out6 ^ in2 ^ in4;
-        out5 = out6 ^ in1 ^ in2;
-        out1 = tmp0 ^ out5 ^ in5;
-        out4 = tmp1 ^ out1;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_E6(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        out3 = in2 ^ in6 ^ in7;
-        out2 = out3 ^ in0 ^ in4;
-        out4 = out3 ^ in1 ^ in5;
-        out1 = out2 ^ in3;
-        out7 = out2 ^ out4 ^ in2;
-        out0 = out4 ^ in3 ^ in7;
-        out5 = out1 ^ in4;
-        out6 = out0 ^ out2 ^ in5;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_E7(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0, tmp1, tmp2, tmp3, tmp4;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        tmp0 = in2 ^ in3;
-        out3 = tmp0 ^ in6 ^ in7;
-        tmp1 = out3 ^ in0;
-        out5 = tmp1 ^ in5;
-        tmp2 = tmp1 ^ in4;
-        tmp3 = out5 ^ in7;
-        out1 = tmp2 ^ in1;
-        out0 = tmp3 ^ in1;
-        out6 = out1 ^ in2;
-        out2 = tmp0 ^ tmp2;
-        tmp4 = tmp3 ^ out6;
-        out4 = tmp4 ^ in6;
-        out7 = tmp4 ^ in0;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_E8(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0, tmp1, tmp2, tmp3;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        out4 = in3 ^ in6;
-        tmp0 = in4 ^ in7;
-        out1 = in2 ^ in3 ^ in4;
-        out5 = tmp0 ^ in0;
-        tmp1 = tmp0 ^ in1;
-        tmp2 = tmp1 ^ in5;
-        out0 = tmp1 ^ out1;
-        out2 = tmp2 ^ in2;
-        out6 = tmp2 ^ out5;
-        tmp3 = out6 ^ in6;
-        out3 = tmp3 ^ in7;
-        out7 = tmp3 ^ in2 ^ in5;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_E9(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0, tmp1, tmp2;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        tmp0 = in0 ^ in1;
-        tmp1 = in3 ^ in6;
-        tmp2 = tmp0 ^ in6;
-        out4 = tmp1 ^ in4;
-        out6 = tmp2 ^ in5;
-        out7 = tmp2 ^ in2 ^ in7;
-        out3 = out6 ^ in3 ^ in7;
-        out0 = tmp1 ^ out7;
-        out2 = out3 ^ out4 ^ in0;
-        out5 = tmp0 ^ out2;
-        out1 = out0 ^ out5 ^ in5;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_EA(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        out4 = in6 ^ in7;
-        out5 = in0 ^ in7;
-        out6 = in0 ^ in1;
-        out0 = in1 ^ in2 ^ in3;
-        out2 = in2 ^ in4 ^ in5;
-        out7 = out6 ^ in2;
-        out1 = out0 ^ out6 ^ in4;
-        out3 = out7 ^ in5 ^ in6;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_EB(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0, tmp1;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        out2 = in4 ^ in5;
-        tmp0 = in0 ^ in1;
-        out4 = in4 ^ in6 ^ in7;
-        out5 = in0 ^ in5 ^ in7;
-        out6 = tmp0 ^ in6;
-        tmp1 = tmp0 ^ in2;
-        out0 = tmp1 ^ in3;
-        out7 = tmp1 ^ in7;
-        out1 = out0 ^ in4;
-        out3 = out0 ^ in5 ^ in6;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_EC(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        out3 = in0 ^ in5;
-        out4 = in2 ^ in3 ^ in7;
-        out5 = in0 ^ in3 ^ in4;
-        out6 = out3 ^ in1 ^ in4;
-        out1 = out4 ^ in4;
-        out0 = out4 ^ in1 ^ in6;
-        out2 = out0 ^ out5 ^ in5;
-        out7 = out2 ^ in4 ^ in7;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_ED(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0, tmp1;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        tmp0 = in2 ^ in4;
-        tmp1 = in3 ^ in5;
-        out4 = tmp0 ^ in3 ^ in7;
-        out3 = tmp1 ^ in0;
-        out1 = out4 ^ in1;
-        out5 = out3 ^ in4;
-        out7 = out1 ^ out5 ^ in6;
-        out2 = tmp0 ^ out7;
-        out0 = tmp1 ^ out7;
-        out6 = out2 ^ in7;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_EE(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0, tmp1, tmp2, tmp3;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        out4 = in2;
-        tmp0 = in0 ^ in1;
-        out5 = in0 ^ in3;
-        tmp1 = tmp0 ^ in2;
-        out6 = tmp0 ^ in4;
-        tmp2 = tmp1 ^ out5;
-        out7 = tmp1 ^ in5;
-        out1 = tmp2 ^ out6 ^ in7;
-        out0 = tmp2 ^ in6;
-        tmp3 = out7 ^ in1;
-        out3 = tmp3 ^ in7;
-        out2 = tmp3 ^ in4 ^ in6;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_EF(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0, tmp1;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        out4 = in2 ^ in4;
-        tmp0 = in0 ^ in5;
-        tmp1 = in4 ^ in6;
-        out5 = tmp0 ^ in3;
-        out2 = tmp0 ^ tmp1;
-        out6 = tmp1 ^ in0 ^ in1;
-        out3 = out5 ^ in2 ^ in7;
-        out7 = out3 ^ in1 ^ in3;
-        out0 = out4 ^ out6 ^ in3;
-        out1 = tmp1 ^ out0 ^ in7;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_F0(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0, tmp1, tmp2, tmp3;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        tmp0 = in1 ^ in2;
-        tmp1 = in4 ^ in5;
-        out2 = tmp0 ^ in6;
-        out3 = tmp1 ^ in1;
-        tmp2 = tmp1 ^ in7;
-        out1 = out2 ^ out3 ^ in3;
-        tmp3 = tmp0 ^ tmp2;
-        out0 = tmp3 ^ in3;
-        out5 = tmp3 ^ in0;
-        out4 = out1 ^ out5 ^ in4;
-        out7 = out4 ^ in2;
-        out6 = tmp2 ^ out7;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_F1(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0, tmp1, tmp2, tmp3;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        out2 = in1 ^ in6;
-        tmp0 = in3 ^ in5;
-        out3 = tmp0 ^ in1 ^ in4;
-        tmp1 = out3 ^ in2;
-        out1 = tmp1 ^ in6;
-        tmp2 = tmp1 ^ in0;
-        tmp3 = out1 ^ in5;
-        out0 = tmp2 ^ in7;
-        out6 = tmp2 ^ in4;
-        out7 = tmp3 ^ in0;
-        out5 = tmp0 ^ out0;
-        out4 = tmp3 ^ out5 ^ in1;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_F2(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0, tmp1, tmp2, tmp3;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        tmp0 = in4 ^ in5;
-        out2 = in2 ^ in6 ^ in7;
-        tmp1 = tmp0 ^ in1;
-        tmp2 = tmp1 ^ in2;
-        out0 = tmp2 ^ in3;
-        out3 = tmp2 ^ in7;
-        out5 = out3 ^ in0 ^ in4;
-        tmp3 = tmp0 ^ out5;
-        out7 = tmp3 ^ in3;
-        out4 = tmp3 ^ out2;
-        out1 = out0 ^ out4 ^ in4;
-        out6 = tmp1 ^ out1;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_F3(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0, tmp1;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        out2 = in6 ^ in7;
-        tmp0 = in0 ^ in1;
-        out4 = tmp0 ^ in6;
-        tmp1 = tmp0 ^ in2;
-        out5 = tmp1 ^ in7;
-        out6 = tmp1 ^ in3;
-        out7 = out6 ^ in4;
-        out0 = out7 ^ in5;
-        out1 = out0 ^ in6;
-        out3 = out0 ^ in0 ^ in7;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_F4(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        out2 = in0 ^ in1 ^ in2;
-        tmp0 = out2 ^ in3;
-        out4 = tmp0 ^ in4;
-        out5 = out4 ^ in5;
-        out6 = out5 ^ in6;
-        out7 = out6 ^ in7;
-        out0 = out7 ^ in0;
-        out1 = out0 ^ in1;
-        out3 = tmp0 ^ out7;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_F5(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        out2 = in0 ^ in1;
-        tmp0 = out2 ^ in2;
-        out4 = tmp0 ^ in3;
-        out5 = out4 ^ in4;
-        out6 = out5 ^ in5;
-        out7 = out6 ^ in6;
-        out0 = out7 ^ in7;
-        out1 = out0 ^ in0;
-        out3 = tmp0 ^ out0;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_F6(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0, tmp1;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        tmp0 = in0 ^ in7;
-        out2 = tmp0 ^ in2;
-        out4 = out2 ^ in1 ^ in4;
-        out7 = out4 ^ in3 ^ in5;
-        out5 = out7 ^ in4 ^ in7;
-        out0 = tmp0 ^ out7 ^ in6;
-        tmp1 = out0 ^ in1;
-        out6 = out0 ^ in0 ^ in5;
-        out3 = tmp1 ^ in3;
-        out1 = tmp0 ^ tmp1;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_F7(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        out2 = in0 ^ in7;
-        tmp0 = out2 ^ in1;
-        out4 = tmp0 ^ in2;
-        out5 = out4 ^ in3 ^ in7;
-        out6 = out5 ^ in4;
-        out7 = out6 ^ in5;
-        out0 = out7 ^ in6;
-        out1 = out0 ^ in7;
-        out3 = tmp0 ^ out1;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_F8(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0, tmp1, tmp2;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        tmp0 = in0 ^ in4;
-        tmp1 = in3 ^ in5;
-        tmp2 = tmp0 ^ in6;
-        out4 = tmp0 ^ tmp1;
-        out1 = tmp1 ^ in2 ^ in4;
-        out3 = tmp2 ^ in1;
-        out5 = out3 ^ in5;
-        out7 = out1 ^ out5 ^ in7;
-        out6 = tmp1 ^ out7;
-        out0 = tmp2 ^ out7;
-        out2 = out6 ^ in0;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_F9(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0, tmp1, tmp2, tmp3, tmp4;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        tmp0 = in3 ^ in5;
-        tmp1 = in0 ^ in6;
-        out4 = tmp0 ^ in0;
-        tmp2 = tmp1 ^ in4;
-        tmp3 = tmp1 ^ in2;
-        out5 = tmp2 ^ in1;
-        out3 = out5 ^ in3;
-        tmp4 = tmp3 ^ out3;
-        out1 = tmp4 ^ in5;
-        out0 = tmp4 ^ in0 ^ in7;
-        out6 = tmp0 ^ out0 ^ in4;
-        out7 = tmp2 ^ tmp4;
-        out2 = tmp3 ^ out6;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_FA(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0, tmp1, tmp2, tmp3;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        tmp0 = in0 ^ in1;
-        tmp1 = tmp0 ^ in2;
-        tmp2 = tmp0 ^ in5;
-        tmp3 = tmp1 ^ in7;
-        out5 = tmp2 ^ in6;
-        out6 = tmp3 ^ in6;
-        out7 = tmp3 ^ in3;
-        out3 = out6 ^ in4;
-        out2 = tmp1 ^ out5;
-        out4 = out2 ^ out3 ^ in1;
-        out0 = out4 ^ out7 ^ in5;
-        out1 = tmp2 ^ out0;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_FB(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0, tmp1;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        out2 = in5 ^ in6;
-        tmp0 = in0 ^ in1;
-        out4 = in0 ^ in5 ^ in7;
-        out5 = tmp0 ^ in6;
-        tmp1 = tmp0 ^ in2;
-        out6 = tmp1 ^ in7;
-        out7 = tmp1 ^ in3;
-        out0 = out7 ^ in4;
-        out1 = out0 ^ in5;
-        out3 = out0 ^ in6 ^ in7;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_FC(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0, tmp1, tmp2, tmp3;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        tmp0 = in1 ^ in2;
-        tmp1 = in0 ^ in7;
-        out2 = tmp0 ^ tmp1 ^ in5;
-        out3 = tmp1 ^ in4;
-        tmp2 = out2 ^ in6;
-        out6 = tmp2 ^ in4;
-        out7 = tmp2 ^ in3;
-        out4 = out6 ^ in1 ^ in3;
-        tmp3 = out4 ^ in0;
-        out1 = tmp3 ^ in6;
-        out0 = tmp3 ^ in1 ^ in5;
-        out5 = tmp0 ^ out4;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_FD(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0, tmp1, tmp2, tmp3;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        tmp0 = in0 ^ in5;
-        tmp1 = in1 ^ in7;
-        out2 = tmp0 ^ tmp1;
-        out6 = out2 ^ in2 ^ in4;
-        tmp2 = out6 ^ in0;
-        out1 = tmp2 ^ in3;
-        out0 = tmp0 ^ out1 ^ in6;
-        out5 = out0 ^ in2;
-        tmp3 = out5 ^ in1;
-        out3 = tmp3 ^ in6;
-        out7 = tmp2 ^ tmp3;
-        out4 = tmp1 ^ out7;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_FE(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0, tmp1, tmp2, tmp3, tmp4;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        tmp0 = in0 ^ in2;
-        out2 = tmp0 ^ in5;
-        out3 = tmp0 ^ in4;
-        tmp1 = out3 ^ in6;
-        out4 = tmp1 ^ in5;
-        tmp2 = tmp1 ^ in1;
-        out6 = tmp2 ^ in7;
-        tmp3 = tmp2 ^ in0;
-        out0 = tmp3 ^ in3;
-        tmp4 = out0 ^ out4 ^ in7;
-        out5 = tmp4 ^ in6;
-        out7 = tmp4 ^ in2;
-        out1 = tmp3 ^ out5;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-static void gf8_muladd_FF(uint8_t * out, uint8_t * in, unsigned int width)
-{
-    unsigned int i;
-    uint64_t * in_ptr = (uint64_t *)in;
-    uint64_t * out_ptr = (uint64_t *)out;
-
-    for (i = 0; i < width; i++)
-    {
-        uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
-        uint64_t tmp0, tmp1, tmp2, tmp3;
-
-        uint64_t in0 = out_ptr[0];
-        uint64_t in1 = out_ptr[width];
-        uint64_t in2 = out_ptr[width * 2];
-        uint64_t in3 = out_ptr[width * 3];
-        uint64_t in4 = out_ptr[width * 4];
-        uint64_t in5 = out_ptr[width * 5];
-        uint64_t in6 = out_ptr[width * 6];
-        uint64_t in7 = out_ptr[width * 7];
-
-        out2 = in0 ^ in5;
-        tmp0 = in4 ^ in7;
-        tmp1 = out2 ^ in2;
-        out4 = tmp1 ^ in6;
-        out7 = tmp1 ^ in1 ^ in3;
-        out1 = tmp0 ^ out7;
-        tmp2 = out1 ^ in5;
-        out6 = tmp2 ^ in3;
-        tmp3 = tmp2 ^ in7;
-        out0 = tmp3 ^ in6;
-        out3 = tmp3 ^ in1;
-        out5 = tmp0 ^ out0 ^ in2;
-
-        out_ptr[0] = out0 ^ in_ptr[0];
-        out_ptr[width] = out1 ^ in_ptr[width];
-        out_ptr[width * 2] = out2 ^ in_ptr[width * 2];
-        out_ptr[width * 3] = out3 ^ in_ptr[width * 3];
-        out_ptr[width * 4] = out4 ^ in_ptr[width * 4];
-        out_ptr[width * 5] = out5 ^ in_ptr[width * 5];
-        out_ptr[width * 6] = out6 ^ in_ptr[width * 6];
-        out_ptr[width * 7] = out7 ^ in_ptr[width * 7];
-
-        in_ptr++;
-        out_ptr++;
-    }
-}
-
-void (* ec_gf_muladd[])(uint8_t * out, uint8_t * in, unsigned int width) =
-{
-    gf8_muladd_00, gf8_muladd_01, gf8_muladd_02, gf8_muladd_03,
-    gf8_muladd_04, gf8_muladd_05, gf8_muladd_06, gf8_muladd_07,
-    gf8_muladd_08, gf8_muladd_09, gf8_muladd_0A, gf8_muladd_0B,
-    gf8_muladd_0C, gf8_muladd_0D, gf8_muladd_0E, gf8_muladd_0F,
-    gf8_muladd_10, gf8_muladd_11, gf8_muladd_12, gf8_muladd_13,
-    gf8_muladd_14, gf8_muladd_15, gf8_muladd_16, gf8_muladd_17,
-    gf8_muladd_18, gf8_muladd_19, gf8_muladd_1A, gf8_muladd_1B,
-    gf8_muladd_1C, gf8_muladd_1D, gf8_muladd_1E, gf8_muladd_1F,
-    gf8_muladd_20, gf8_muladd_21, gf8_muladd_22, gf8_muladd_23,
-    gf8_muladd_24, gf8_muladd_25, gf8_muladd_26, gf8_muladd_27,
-    gf8_muladd_28, gf8_muladd_29, gf8_muladd_2A, gf8_muladd_2B,
-    gf8_muladd_2C, gf8_muladd_2D, gf8_muladd_2E, gf8_muladd_2F,
-    gf8_muladd_30, gf8_muladd_31, gf8_muladd_32, gf8_muladd_33,
-    gf8_muladd_34, gf8_muladd_35, gf8_muladd_36, gf8_muladd_37,
-    gf8_muladd_38, gf8_muladd_39, gf8_muladd_3A, gf8_muladd_3B,
-    gf8_muladd_3C, gf8_muladd_3D, gf8_muladd_3E, gf8_muladd_3F,
-    gf8_muladd_40, gf8_muladd_41, gf8_muladd_42, gf8_muladd_43,
-    gf8_muladd_44, gf8_muladd_45, gf8_muladd_46, gf8_muladd_47,
-    gf8_muladd_48, gf8_muladd_49, gf8_muladd_4A, gf8_muladd_4B,
-    gf8_muladd_4C, gf8_muladd_4D, gf8_muladd_4E, gf8_muladd_4F,
-    gf8_muladd_50, gf8_muladd_51, gf8_muladd_52, gf8_muladd_53,
-    gf8_muladd_54, gf8_muladd_55, gf8_muladd_56, gf8_muladd_57,
-    gf8_muladd_58, gf8_muladd_59, gf8_muladd_5A, gf8_muladd_5B,
-    gf8_muladd_5C, gf8_muladd_5D, gf8_muladd_5E, gf8_muladd_5F,
-    gf8_muladd_60, gf8_muladd_61, gf8_muladd_62, gf8_muladd_63,
-    gf8_muladd_64, gf8_muladd_65, gf8_muladd_66, gf8_muladd_67,
-    gf8_muladd_68, gf8_muladd_69, gf8_muladd_6A, gf8_muladd_6B,
-    gf8_muladd_6C, gf8_muladd_6D, gf8_muladd_6E, gf8_muladd_6F,
-    gf8_muladd_70, gf8_muladd_71, gf8_muladd_72, gf8_muladd_73,
-    gf8_muladd_74, gf8_muladd_75, gf8_muladd_76, gf8_muladd_77,
-    gf8_muladd_78, gf8_muladd_79, gf8_muladd_7A, gf8_muladd_7B,
-    gf8_muladd_7C, gf8_muladd_7D, gf8_muladd_7E, gf8_muladd_7F,
-    gf8_muladd_80, gf8_muladd_81, gf8_muladd_82, gf8_muladd_83,
-    gf8_muladd_84, gf8_muladd_85, gf8_muladd_86, gf8_muladd_87,
-    gf8_muladd_88, gf8_muladd_89, gf8_muladd_8A, gf8_muladd_8B,
-    gf8_muladd_8C, gf8_muladd_8D, gf8_muladd_8E, gf8_muladd_8F,
-    gf8_muladd_90, gf8_muladd_91, gf8_muladd_92, gf8_muladd_93,
-    gf8_muladd_94, gf8_muladd_95, gf8_muladd_96, gf8_muladd_97,
-    gf8_muladd_98, gf8_muladd_99, gf8_muladd_9A, gf8_muladd_9B,
-    gf8_muladd_9C, gf8_muladd_9D, gf8_muladd_9E, gf8_muladd_9F,
-    gf8_muladd_A0, gf8_muladd_A1, gf8_muladd_A2, gf8_muladd_A3,
-    gf8_muladd_A4, gf8_muladd_A5, gf8_muladd_A6, gf8_muladd_A7,
-    gf8_muladd_A8, gf8_muladd_A9, gf8_muladd_AA, gf8_muladd_AB,
-    gf8_muladd_AC, gf8_muladd_AD, gf8_muladd_AE, gf8_muladd_AF,
-    gf8_muladd_B0, gf8_muladd_B1, gf8_muladd_B2, gf8_muladd_B3,
-    gf8_muladd_B4, gf8_muladd_B5, gf8_muladd_B6, gf8_muladd_B7,
-    gf8_muladd_B8, gf8_muladd_B9, gf8_muladd_BA, gf8_muladd_BB,
-    gf8_muladd_BC, gf8_muladd_BD, gf8_muladd_BE, gf8_muladd_BF,
-    gf8_muladd_C0, gf8_muladd_C1, gf8_muladd_C2, gf8_muladd_C3,
-    gf8_muladd_C4, gf8_muladd_C5, gf8_muladd_C6, gf8_muladd_C7,
-    gf8_muladd_C8, gf8_muladd_C9, gf8_muladd_CA, gf8_muladd_CB,
-    gf8_muladd_CC, gf8_muladd_CD, gf8_muladd_CE, gf8_muladd_CF,
-    gf8_muladd_D0, gf8_muladd_D1, gf8_muladd_D2, gf8_muladd_D3,
-    gf8_muladd_D4, gf8_muladd_D5, gf8_muladd_D6, gf8_muladd_D7,
-    gf8_muladd_D8, gf8_muladd_D9, gf8_muladd_DA, gf8_muladd_DB,
-    gf8_muladd_DC, gf8_muladd_DD, gf8_muladd_DE, gf8_muladd_DF,
-    gf8_muladd_E0, gf8_muladd_E1, gf8_muladd_E2, gf8_muladd_E3,
-    gf8_muladd_E4, gf8_muladd_E5, gf8_muladd_E6, gf8_muladd_E7,
-    gf8_muladd_E8, gf8_muladd_E9, gf8_muladd_EA, gf8_muladd_EB,
-    gf8_muladd_EC, gf8_muladd_ED, gf8_muladd_EE, gf8_muladd_EF,
-    gf8_muladd_F0, gf8_muladd_F1, gf8_muladd_F2, gf8_muladd_F3,
-    gf8_muladd_F4, gf8_muladd_F5, gf8_muladd_F6, gf8_muladd_F7,
-    gf8_muladd_F8, gf8_muladd_F9, gf8_muladd_FA, gf8_muladd_FB,
-    gf8_muladd_FC, gf8_muladd_FD, gf8_muladd_FE, gf8_muladd_FF
-};
diff --git a/xlators/cluster/ec/src/ec-gf8.c b/xlators/cluster/ec/src/ec-gf8.c
new file mode 100644
index 00000000000..039adae5929
--- /dev/null
+++ b/xlators/cluster/ec/src/ec-gf8.c
@@ -0,0 +1,5882 @@
+/*
+  Copyright (c) 2015 DataLab, s.l. <http://www.datalab.es>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
+*/
+
+#include "ec-gf8.h"
+
+static ec_gf_op_t ec_gf8_mul_00_ops[] = {{EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_00 = {0,
+                                    {
+                                        0,
+                                    },
+                                    ec_gf8_mul_00_ops};
+
+static ec_gf_op_t ec_gf8_mul_01_ops[] = {{EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_01 = {8,
+                                    {
+                                        0,
+                                        1,
+                                        2,
+                                        3,
+                                        4,
+                                        5,
+                                        6,
+                                        7,
+                                    },
+                                    ec_gf8_mul_01_ops};
+
+static ec_gf_op_t ec_gf8_mul_02_ops[] = {{EC_GF_OP_XOR2, 1, 7, 0},
+                                         {EC_GF_OP_XOR2, 2, 7, 0},
+                                         {EC_GF_OP_XOR2, 3, 7, 0},
+                                         {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_02 = {8,
+                                    {
+                                        7,
+                                        0,
+                                        1,
+                                        2,
+                                        3,
+                                        4,
+                                        5,
+                                        6,
+                                    },
+                                    ec_gf8_mul_02_ops};
+
+static ec_gf_op_t ec_gf8_mul_03_ops[] = {
+    {EC_GF_OP_XOR2, 3, 7, 0}, {EC_GF_OP_COPY, 8, 3, 0},
+    {EC_GF_OP_XOR2, 3, 2, 0}, {EC_GF_OP_XOR2, 2, 1, 0},
+    {EC_GF_OP_XOR2, 1, 0, 0}, {EC_GF_OP_XOR2, 2, 7, 0},
+    {EC_GF_OP_XOR2, 0, 7, 0}, {EC_GF_OP_XOR2, 7, 6, 0},
+    {EC_GF_OP_XOR2, 6, 5, 0}, {EC_GF_OP_XOR2, 5, 4, 0},
+    {EC_GF_OP_XOR2, 4, 8, 0}, {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_03 = {9,
+                                    {
+                                        0,
+                                        1,
+                                        2,
+                                        3,
+                                        4,
+                                        5,
+                                        6,
+                                        7,
+                                        8,
+                                    },
+                                    ec_gf8_mul_03_ops};
+
+static ec_gf_op_t ec_gf8_mul_04_ops[] = {
+    {EC_GF_OP_XOR3, 8, 6, 7}, {EC_GF_OP_XOR2, 2, 8, 0},
+    {EC_GF_OP_XOR2, 3, 7, 0}, {EC_GF_OP_XOR2, 1, 8, 0},
+    {EC_GF_OP_XOR2, 0, 6, 0}, {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_04 = {9,
+                                    {
+                                        6,
+                                        7,
+                                        0,
+                                        1,
+                                        2,
+                                        3,
+                                        4,
+                                        5,
+                                        8,
+                                    },
+                                    ec_gf8_mul_04_ops};
+
+static ec_gf_op_t ec_gf8_mul_05_ops[] = {
+    {EC_GF_OP_XOR2, 4, 6, 0}, {EC_GF_OP_XOR2, 1, 7, 0},
+    {EC_GF_OP_XOR2, 5, 7, 0}, {EC_GF_OP_XOR2, 0, 6, 0},
+    {EC_GF_OP_XOR2, 7, 4, 0}, {EC_GF_OP_XOR2, 6, 3, 0},
+    {EC_GF_OP_XOR2, 7, 2, 0}, {EC_GF_OP_XOR2, 6, 1, 0},
+    {EC_GF_OP_XOR2, 3, 5, 0}, {EC_GF_OP_XOR2, 2, 0, 0},
+    {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_05 = {8,
+                                    {
+                                        0,
+                                        1,
+                                        2,
+                                        6,
+                                        7,
+                                        3,
+                                        4,
+                                        5,
+                                    },
+                                    ec_gf8_mul_05_ops};
+
+static ec_gf_op_t ec_gf8_mul_06_ops[] = {
+    {EC_GF_OP_XOR2, 2, 6, 0}, {EC_GF_OP_COPY, 8, 2, 0},
+    {EC_GF_OP_XOR2, 8, 3, 0}, {EC_GF_OP_XOR2, 2, 1, 0},
+    {EC_GF_OP_XOR2, 3, 4, 0}, {EC_GF_OP_XOR2, 1, 0, 0},
+    {EC_GF_OP_XOR2, 3, 7, 0}, {EC_GF_OP_XOR2, 0, 7, 0},
+    {EC_GF_OP_XOR2, 7, 6, 0}, {EC_GF_OP_XOR2, 4, 5, 0},
+    {EC_GF_OP_XOR2, 1, 7, 0}, {EC_GF_OP_XOR2, 5, 6, 0},
+    {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_06 = {9,
+                                    {
+                                        7,
+                                        0,
+                                        1,
+                                        2,
+                                        8,
+                                        3,
+                                        4,
+                                        5,
+                                        6,
+                                    },
+                                    ec_gf8_mul_06_ops};
+
+static ec_gf_op_t ec_gf8_mul_07_ops[] = {
+    {EC_GF_OP_XOR2, 5, 6, 0}, {EC_GF_OP_XOR2, 0, 7, 0},
+    {EC_GF_OP_XOR2, 3, 6, 0}, {EC_GF_OP_XOR2, 7, 5, 0},
+    {EC_GF_OP_XOR2, 6, 0, 0}, {EC_GF_OP_XOR2, 0, 1, 0},
+    {EC_GF_OP_XOR2, 5, 4, 0}, {EC_GF_OP_XOR2, 4, 3, 0},
+    {EC_GF_OP_XOR2, 1, 2, 0}, {EC_GF_OP_XOR2, 2, 4, 0},
+    {EC_GF_OP_XOR2, 3, 1, 0}, {EC_GF_OP_XOR2, 4, 7, 0},
+    {EC_GF_OP_XOR2, 1, 6, 0}, {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_07 = {8,
+                                    {
+                                        6,
+                                        0,
+                                        1,
+                                        3,
+                                        2,
+                                        4,
+                                        5,
+                                        7,
+                                    },
+                                    ec_gf8_mul_07_ops};
+
+static ec_gf_op_t ec_gf8_mul_08_ops[] = {
+    {EC_GF_OP_XOR2, 0, 5, 0}, {EC_GF_OP_XOR2, 1, 5, 0},
+    {EC_GF_OP_XOR2, 0, 6, 0}, {EC_GF_OP_XOR3, 8, 6, 7},
+    {EC_GF_OP_XOR2, 1, 8, 0}, {EC_GF_OP_XOR2, 3, 7, 0},
+    {EC_GF_OP_XOR2, 7, 5, 0}, {EC_GF_OP_XOR2, 2, 8, 0},
+    {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_08 = {9,
+                                    {
+                                        5,
+                                        6,
+                                        7,
+                                        0,
+                                        1,
+                                        2,
+                                        3,
+                                        4,
+                                        8,
+                                    },
+                                    ec_gf8_mul_08_ops};
+
+static ec_gf_op_t ec_gf8_mul_09_ops[] = {
+    {EC_GF_OP_XOR2, 2, 4, 0}, {EC_GF_OP_XOR2, 4, 7, 0},
+    {EC_GF_OP_XOR2, 0, 5, 0}, {EC_GF_OP_XOR2, 5, 4, 0},
+    {EC_GF_OP_XOR2, 3, 6, 0}, {EC_GF_OP_XOR2, 2, 5, 0},
+    {EC_GF_OP_XOR2, 1, 6, 0}, {EC_GF_OP_XOR2, 7, 3, 0},
+    {EC_GF_OP_XOR2, 6, 2, 0}, {EC_GF_OP_XOR2, 5, 1, 0},
+    {EC_GF_OP_XOR2, 3, 0, 0}, {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_09 = {8,
+                                    {
+                                        0,
+                                        1,
+                                        2,
+                                        3,
+                                        5,
+                                        6,
+                                        7,
+                                        4,
+                                    },
+                                    ec_gf8_mul_09_ops};
+
+static ec_gf_op_t ec_gf8_mul_0A_ops[] = {
+    {EC_GF_OP_XOR2, 1, 5, 0}, {EC_GF_OP_XOR2, 4, 6, 0},
+    {EC_GF_OP_XOR2, 5, 7, 0}, {EC_GF_OP_XOR2, 7, 4, 0},
+    {EC_GF_OP_XOR2, 0, 6, 0}, {EC_GF_OP_XOR2, 7, 2, 0},
+    {EC_GF_OP_XOR2, 6, 1, 0}, {EC_GF_OP_XOR2, 2, 0, 0},
+    {EC_GF_OP_XOR2, 6, 3, 0}, {EC_GF_OP_XOR2, 2, 5, 0},
+    {EC_GF_OP_XOR2, 3, 5, 0}, {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_0A = {8,
+                                    {
+                                        5,
+                                        0,
+                                        1,
+                                        2,
+                                        6,
+                                        7,
+                                        3,
+                                        4,
+                                    },
+                                    ec_gf8_mul_0A_ops};
+
+static ec_gf_op_t ec_gf8_mul_0B_ops[] = {
+    {EC_GF_OP_XOR2, 4, 7, 0}, {EC_GF_OP_XOR2, 7, 0, 0},
+    {EC_GF_OP_XOR2, 7, 5, 0}, {EC_GF_OP_XOR2, 3, 7, 0},
+    {EC_GF_OP_COPY, 9, 3, 0}, {EC_GF_OP_XOR2, 5, 2, 0},
+    {EC_GF_OP_XOR2, 3, 6, 0}, {EC_GF_OP_COPY, 8, 5, 0},
+    {EC_GF_OP_XOR2, 0, 3, 0}, {EC_GF_OP_XOR2, 5, 1, 0},
+    {EC_GF_OP_XOR2, 6, 4, 0}, {EC_GF_OP_XOR2, 1, 0, 0},
+    {EC_GF_OP_XOR2, 2, 3, 0}, {EC_GF_OP_XOR2, 4, 1, 0},
+    {EC_GF_OP_XOR3, 3, 8, 6}, {EC_GF_OP_XOR2, 1, 9, 0},
+    {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_0B = {10,
+                                    {
+                                        7,
+                                        1,
+                                        5,
+                                        2,
+                                        4,
+                                        3,
+                                        0,
+                                        6,
+                                        8,
+                                        9,
+                                    },
+                                    ec_gf8_mul_0B_ops};
+
+static ec_gf_op_t ec_gf8_mul_0C_ops[] = {
+    {EC_GF_OP_XOR2, 1, 5, 0}, {EC_GF_OP_COPY, 8, 1, 0},
+    {EC_GF_OP_XOR2, 8, 2, 0}, {EC_GF_OP_XOR2, 2, 3, 0},
+    {EC_GF_OP_XOR2, 3, 4, 0}, {EC_GF_OP_XOR2, 0, 7, 0},
+    {EC_GF_OP_XOR2, 4, 5, 0}, {EC_GF_OP_XOR2, 3, 7, 0},
+    {EC_GF_OP_XOR2, 5, 6, 0}, {EC_GF_OP_XOR2, 1, 0, 0},
+    {EC_GF_OP_XOR2, 2, 6, 0}, {EC_GF_OP_XOR2, 7, 6, 0},
+    {EC_GF_OP_XOR2, 0, 5, 0}, {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_0C = {9,
+                                    {
+                                        5,
+                                        7,
+                                        0,
+                                        1,
+                                        8,
+                                        2,
+                                        3,
+                                        4,
+                                        6,
+                                    },
+                                    ec_gf8_mul_0C_ops};
+
+static ec_gf_op_t ec_gf8_mul_0D_ops[] = {
+    {EC_GF_OP_XOR2, 5, 7, 0}, {EC_GF_OP_XOR2, 4, 5, 0},
+    {EC_GF_OP_XOR2, 5, 0, 0}, {EC_GF_OP_XOR2, 5, 1, 0},
+    {EC_GF_OP_XOR2, 1, 7, 0}, {EC_GF_OP_XOR2, 0, 3, 0},
+    {EC_GF_OP_XOR2, 6, 1, 0}, {EC_GF_OP_XOR2, 3, 5, 0},
+    {EC_GF_OP_XOR3, 8, 2, 4}, {EC_GF_OP_XOR2, 5, 6, 0},
+    {EC_GF_OP_XOR2, 2, 5, 0}, {EC_GF_OP_XOR2, 1, 8, 0},
+    {EC_GF_OP_XOR2, 0, 2, 0}, {EC_GF_OP_XOR2, 7, 2, 0},
+    {EC_GF_OP_XOR3, 2, 8, 0}, {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_0D = {9,
+                                    {
+                                        5,
+                                        6,
+                                        7,
+                                        3,
+                                        1,
+                                        0,
+                                        2,
+                                        4,
+                                        8,
+                                    },
+                                    ec_gf8_mul_0D_ops};
+
+static ec_gf_op_t ec_gf8_mul_0E_ops[] = {
+    {EC_GF_OP_XOR2, 0, 5, 0}, {EC_GF_OP_XOR2, 5, 6, 0},
+    {EC_GF_OP_XOR2, 1, 0, 0}, {EC_GF_OP_XOR2, 7, 5, 0},
+    {EC_GF_OP_XOR2, 3, 6, 0}, {EC_GF_OP_XOR2, 0, 7, 0},
+    {EC_GF_OP_XOR2, 5, 4, 0}, {EC_GF_OP_XOR2, 6, 1, 0},
+    {EC_GF_OP_XOR2, 4, 3, 0}, {EC_GF_OP_XOR2, 1, 2, 0},
+    {EC_GF_OP_XOR2, 3, 0, 0}, {EC_GF_OP_XOR2, 2, 4, 0},
+    {EC_GF_OP_XOR2, 3, 1, 0}, {EC_GF_OP_XOR2, 4, 7, 0},
+    {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_0E = {8,
+                                    {
+                                        7,
+                                        0,
+                                        6,
+                                        1,
+                                        3,
+                                        2,
+                                        4,
+                                        5,
+                                    },
+                                    ec_gf8_mul_0E_ops};
+
+static ec_gf_op_t ec_gf8_mul_0F_ops[] = {
+    {EC_GF_OP_XOR2, 2, 7, 0}, {EC_GF_OP_XOR2, 7, 1, 0},
+    {EC_GF_OP_XOR2, 7, 6, 0}, {EC_GF_OP_XOR2, 4, 0, 0},
+    {EC_GF_OP_XOR2, 0, 7, 0}, {EC_GF_OP_XOR2, 5, 0, 0},
+    {EC_GF_OP_XOR2, 6, 3, 0}, {EC_GF_OP_XOR2, 1, 5, 0},
+    {EC_GF_OP_XOR2, 5, 2, 0}, {EC_GF_OP_XOR2, 4, 1, 0},
+    {EC_GF_OP_XOR2, 3, 4, 0}, {EC_GF_OP_XOR2, 6, 5, 0},
+    {EC_GF_OP_XOR2, 2, 3, 0}, {EC_GF_OP_XOR2, 7, 2, 0},
+    {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_0F = {8,
+                                    {
+                                        1,
+                                        0,
+                                        5,
+                                        6,
+                                        7,
+                                        2,
+                                        3,
+                                        4,
+                                    },
+                                    ec_gf8_mul_0F_ops};
+
+static ec_gf_op_t ec_gf8_mul_10_ops[] = {
+    {EC_GF_OP_XOR2, 3, 7, 0}, {EC_GF_OP_XOR2, 7, 6, 0},
+    {EC_GF_OP_XOR2, 2, 7, 0}, {EC_GF_OP_XOR2, 7, 5, 0},
+    {EC_GF_OP_XOR2, 6, 4, 0}, {EC_GF_OP_XOR2, 0, 5, 0},
+    {EC_GF_OP_XOR2, 1, 7, 0}, {EC_GF_OP_XOR2, 0, 6, 0},
+    {EC_GF_OP_XOR2, 7, 6, 0}, {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_10 = {8,
+                                    {
+                                        4,
+                                        5,
+                                        6,
+                                        7,
+                                        0,
+                                        1,
+                                        2,
+                                        3,
+                                    },
+                                    ec_gf8_mul_10_ops};
+
+static ec_gf_op_t ec_gf8_mul_11_ops[] = {
+    {EC_GF_OP_XOR2, 1, 5, 0}, {EC_GF_OP_XOR2, 5, 6, 0},
+    {EC_GF_OP_XOR2, 6, 4, 0}, {EC_GF_OP_XOR2, 4, 0, 0},
+    {EC_GF_OP_XOR2, 0, 5, 0}, {EC_GF_OP_XOR2, 5, 7, 0},
+    {EC_GF_OP_XOR2, 7, 2, 0}, {EC_GF_OP_XOR2, 2, 6, 0},
+    {EC_GF_OP_XOR2, 6, 3, 0}, {EC_GF_OP_XOR2, 6, 5, 0},
+    {EC_GF_OP_XOR2, 5, 1, 0}, {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_11 = {8,
+                                    {
+                                        4,
+                                        1,
+                                        2,
+                                        6,
+                                        0,
+                                        5,
+                                        7,
+                                        3,
+                                    },
+                                    ec_gf8_mul_11_ops};
+
+static ec_gf_op_t ec_gf8_mul_12_ops[] = {
+    {EC_GF_OP_XOR2, 7, 4, 0}, {EC_GF_OP_XOR2, 6, 7, 0},
+    {EC_GF_OP_XOR2, 2, 4, 0}, {EC_GF_OP_XOR2, 3, 6, 0},
+    {EC_GF_OP_XOR2, 4, 3, 0}, {EC_GF_OP_XOR2, 1, 6, 0},
+    {EC_GF_OP_XOR2, 2, 5, 0}, {EC_GF_OP_XOR2, 0, 5, 0},
+    {EC_GF_OP_XOR2, 6, 2, 0}, {EC_GF_OP_XOR2, 5, 1, 0},
+    {EC_GF_OP_XOR2, 3, 0, 0}, {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_12 = {8,
+                                    {
+                                        7,
+                                        0,
+                                        1,
+                                        2,
+                                        3,
+                                        5,
+                                        6,
+                                        4,
+                                    },
+                                    ec_gf8_mul_12_ops};
+
+static ec_gf_op_t ec_gf8_mul_13_ops[] = {
+    {EC_GF_OP_XOR2, 4, 7, 0}, {EC_GF_OP_XOR2, 7, 5, 0},
+    {EC_GF_OP_XOR2, 3, 6, 0}, {EC_GF_OP_XOR2, 5, 1, 0},
+    {EC_GF_OP_XOR2, 6, 4, 0}, {EC_GF_OP_XOR2, 7, 2, 0},
+    {EC_GF_OP_XOR2, 4, 0, 0}, {EC_GF_OP_XOR2, 5, 0, 0},
+    {EC_GF_OP_XOR2, 1, 6, 0}, {EC_GF_OP_XOR3, 8, 3, 7},
+    {EC_GF_OP_XOR2, 0, 2, 0}, {EC_GF_OP_XOR2, 6, 8, 0},
+    {EC_GF_OP_XOR2, 2, 1, 0}, {EC_GF_OP_XOR2, 0, 8, 0},
+    {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_13 = {9,
+                                    {
+                                        4,
+                                        5,
+                                        2,
+                                        6,
+                                        0,
+                                        1,
+                                        7,
+                                        3,
+                                        8,
+                                    },
+                                    ec_gf8_mul_13_ops};
+
+static ec_gf_op_t ec_gf8_mul_14_ops[] = {
+    {EC_GF_OP_XOR2, 6, 4, 0}, {EC_GF_OP_XOR2, 7, 5, 0},
+    {EC_GF_OP_XOR2, 5, 6, 0}, {EC_GF_OP_XOR2, 2, 7, 0},
+    {EC_GF_OP_XOR2, 0, 4, 0}, {EC_GF_OP_XOR2, 1, 5, 0},
+    {EC_GF_OP_XOR2, 5, 2, 0}, {EC_GF_OP_XOR2, 4, 3, 0},
+    {EC_GF_OP_XOR2, 2, 0, 0}, {EC_GF_OP_XOR2, 3, 7, 0},
+    {EC_GF_OP_XOR2, 4, 1, 0}, {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_14 = {8,
+                                    {
+                                        6,
+                                        7,
+                                        0,
+                                        1,
+                                        2,
+                                        4,
+                                        5,
+                                        3,
+                                    },
+                                    ec_gf8_mul_14_ops};
+
+static ec_gf_op_t ec_gf8_mul_15_ops[] = {
+    {EC_GF_OP_COPY, 8, 0, 0}, {EC_GF_OP_XOR2, 0, 4, 0},
+    {EC_GF_OP_XOR2, 2, 0, 0}, {EC_GF_OP_XOR2, 0, 6, 0},
+    {EC_GF_OP_XOR2, 6, 1, 0}, {EC_GF_OP_XOR2, 1, 5, 0},
+    {EC_GF_OP_XOR2, 4, 5, 0}, {EC_GF_OP_XOR2, 1, 7, 0},
+    {EC_GF_OP_XOR2, 6, 3, 0}, {EC_GF_OP_XOR2, 7, 2, 0},
+    {EC_GF_OP_XOR2, 3, 5, 0}, {EC_GF_OP_XOR3, 5, 8, 7},
+    {EC_GF_OP_XOR2, 7, 4, 0}, {EC_GF_OP_XOR2, 4, 6, 0},
+    {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_15 = {9,
+                                    {
+                                        0,
+                                        1,
+                                        2,
+                                        4,
+                                        7,
+                                        6,
+                                        5,
+                                        3,
+                                        8,
+                                    },
+                                    ec_gf8_mul_15_ops};
+
+static ec_gf_op_t ec_gf8_mul_16_ops[] = {
+    {EC_GF_OP_XOR2, 4, 7, 0}, {EC_GF_OP_XOR2, 3, 6, 0},
+    {EC_GF_OP_XOR2, 7, 5, 0}, {EC_GF_OP_XOR2, 6, 4, 0},
+    {EC_GF_OP_XOR2, 7, 0, 0}, {EC_GF_OP_XOR2, 2, 6, 0},
+    {EC_GF_OP_XOR2, 3, 7, 0}, {EC_GF_OP_XOR2, 4, 1, 0},
+    {EC_GF_OP_XOR2, 5, 2, 0}, {EC_GF_OP_XOR2, 4, 0, 0},
+    {EC_GF_OP_XOR2, 2, 3, 0}, {EC_GF_OP_XOR2, 0, 3, 0},
+    {EC_GF_OP_XOR2, 1, 5, 0}, {EC_GF_OP_XOR2, 3, 4, 0},
+    {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_16 = {8,
+                                    {
+                                        6,
+                                        7,
+                                        4,
+                                        1,
+                                        2,
+                                        3,
+                                        5,
+                                        0,
+                                    },
+                                    ec_gf8_mul_16_ops};
+
+static ec_gf_op_t ec_gf8_mul_17_ops[] = {
+    {EC_GF_OP_XOR2, 3, 5, 0}, {EC_GF_OP_XOR2, 6, 3, 0},
+    {EC_GF_OP_XOR2, 3, 0, 0}, {EC_GF_OP_XOR2, 7, 5, 0},
+    {EC_GF_OP_XOR2, 3, 2, 0}, {EC_GF_OP_XOR2, 2, 7, 0},
+    {EC_GF_OP_XOR2, 4, 2, 0}, {EC_GF_OP_XOR2, 0, 1, 0},
+    {EC_GF_OP_XOR2, 5, 4, 0}, {EC_GF_OP_XOR2, 7, 0, 0},
+    {EC_GF_OP_XOR2, 0, 5, 0}, {EC_GF_OP_XOR2, 5, 6, 0},
+    {EC_GF_OP_XOR2, 1, 5, 0}, {EC_GF_OP_XOR2, 5, 3, 0},
+    {EC_GF_OP_XOR2, 2, 1, 0}, {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_17 = {8,
+                                    {
+                                        5,
+                                        7,
+                                        0,
+                                        1,
+                                        3,
+                                        2,
+                                        4,
+                                        6,
+                                    },
+                                    ec_gf8_mul_17_ops};
+
+static ec_gf_op_t ec_gf8_mul_18_ops[] = {
+    {EC_GF_OP_XOR2, 7, 4, 0}, {EC_GF_OP_XOR2, 0, 7, 0},
+    {EC_GF_OP_COPY, 8, 0, 0}, {EC_GF_OP_XOR2, 0, 1, 0},
+    {EC_GF_OP_XOR2, 1, 2, 0}, {EC_GF_OP_XOR2, 4, 5, 0},
+    {EC_GF_OP_XOR2, 2, 3, 0}, {EC_GF_OP_XOR2, 1, 5, 0},
+    {EC_GF_OP_XOR2, 3, 7, 0}, {EC_GF_OP_XOR2, 5, 6, 0},
+    {EC_GF_OP_XOR2, 2, 6, 0}, {EC_GF_OP_XOR2, 7, 5, 0},
+    {EC_GF_OP_XOR2, 6, 8, 0}, {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_18 = {9,
+                                    {
+                                        4,
+                                        5,
+                                        7,
+                                        6,
+                                        0,
+                                        1,
+                                        2,
+                                        3,
+                                        8,
+                                    },
+                                    ec_gf8_mul_18_ops};
+
+static ec_gf_op_t ec_gf8_mul_19_ops[] = {
+    {EC_GF_OP_XOR2, 7, 0, 0}, {EC_GF_OP_XOR2, 0, 4, 0},
+    {EC_GF_OP_XOR2, 6, 1, 0}, {EC_GF_OP_XOR2, 7, 1, 0},
+    {EC_GF_OP_XOR2, 4, 3, 0}, {EC_GF_OP_XOR2, 0, 5, 0},
+    {EC_GF_OP_XOR2, 5, 6, 0}, {EC_GF_OP_XOR2, 3, 2, 0},
+    {EC_GF_OP_XOR2, 1, 2, 0}, {EC_GF_OP_XOR2, 6, 7, 0},
+    {EC_GF_OP_XOR2, 2, 0, 0}, {EC_GF_OP_XOR2, 2, 6, 0},
+    {EC_GF_OP_XOR2, 6, 4, 0}, {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_19 = {8,
+                                    {
+                                        0,
+                                        5,
+                                        2,
+                                        6,
+                                        7,
+                                        1,
+                                        3,
+                                        4,
+                                    },
+                                    ec_gf8_mul_19_ops};
+
+static ec_gf_op_t ec_gf8_mul_1A_ops[] = {
+    {EC_GF_OP_XOR2, 6, 5, 0}, {EC_GF_OP_XOR2, 5, 4, 0},
+    {EC_GF_OP_XOR2, 4, 0, 0}, {EC_GF_OP_XOR2, 7, 5, 0},
+    {EC_GF_OP_XOR2, 0, 6, 0}, {EC_GF_OP_XOR2, 4, 1, 0},
+    {EC_GF_OP_XOR2, 6, 3, 0}, {EC_GF_OP_XOR2, 3, 4, 0},
+    {EC_GF_OP_XOR2, 5, 2, 0}, {EC_GF_OP_XOR2, 4, 0, 0},
+    {EC_GF_OP_XOR2, 2, 6, 0}, {EC_GF_OP_XOR2, 1, 5, 0},
+    {EC_GF_OP_XOR2, 6, 7, 0}, {EC_GF_OP_XOR2, 5, 0, 0},
+    {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_1A = {8,
+                                    {
+                                        7,
+                                        0,
+                                        4,
+                                        5,
+                                        3,
+                                        1,
+                                        2,
+                                        6,
+                                    },
+                                    ec_gf8_mul_1A_ops};
+
+static ec_gf_op_t ec_gf8_mul_1B_ops[] = {
+    {EC_GF_OP_XOR2, 1, 0, 0}, {EC_GF_OP_XOR2, 0, 2, 0},
+    {EC_GF_OP_XOR2, 2, 5, 0}, {EC_GF_OP_XOR2, 7, 2, 0},
+    {EC_GF_OP_XOR2, 2, 3, 0}, {EC_GF_OP_XOR2, 4, 0, 0},
+    {EC_GF_OP_XOR2, 3, 1, 0}, {EC_GF_OP_XOR2, 1, 4, 0},
+    {EC_GF_OP_XOR2, 7, 4, 0}, {EC_GF_OP_XOR2, 6, 1, 0},
+    {EC_GF_OP_XOR2, 5, 6, 0}, {EC_GF_OP_XOR2, 6, 3, 0},
+    {EC_GF_OP_XOR2, 4, 5, 0}, {EC_GF_OP_XOR2, 0, 6, 0},
+    {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_1B = {8,
+                                    {
+                                        7,
+                                        4,
+                                        5,
+                                        6,
+                                        3,
+                                        1,
+                                        2,
+                                        0,
+                                    },
+                                    ec_gf8_mul_1B_ops};
+
+static ec_gf_op_t ec_gf8_mul_1C_ops[] = {
+    {EC_GF_OP_XOR2, 6, 4, 0}, {EC_GF_OP_XOR2, 5, 6, 0},
+    {EC_GF_OP_XOR2, 6, 3, 0}, {EC_GF_OP_XOR2, 3, 0, 0},
+    {EC_GF_OP_XOR2, 0, 1, 0}, {EC_GF_OP_XOR2, 2, 6, 0},
+    {EC_GF_OP_XOR2, 0, 4, 0}, {EC_GF_OP_XOR2, 7, 5, 0},
+    {EC_GF_OP_XOR2, 1, 2, 0}, {EC_GF_OP_XOR2, 4, 7, 0},
+    {EC_GF_OP_XOR2, 7, 1, 0}, {EC_GF_OP_XOR2, 6, 4, 0},
+    {EC_GF_OP_XOR2, 1, 3, 0}, {EC_GF_OP_XOR2, 3, 6, 0},
+    {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_1C = {8,
+                                    {
+                                        5,
+                                        4,
+                                        3,
+                                        0,
+                                        1,
+                                        7,
+                                        2,
+                                        6,
+                                    },
+                                    ec_gf8_mul_1C_ops};
+
+static ec_gf_op_t ec_gf8_mul_1D_ops[] = {
+    {EC_GF_OP_XOR2, 7, 1, 0}, {EC_GF_OP_XOR2, 1, 0, 0},
+    {EC_GF_OP_XOR2, 3, 2, 0}, {EC_GF_OP_XOR2, 0, 4, 0},
+    {EC_GF_OP_XOR2, 2, 1, 0}, {EC_GF_OP_XOR2, 4, 3, 0},
+    {EC_GF_OP_XOR2, 3, 7, 0}, {EC_GF_OP_XOR3, 8, 4, 2},
+    {EC_GF_OP_XOR2, 2, 6, 0}, {EC_GF_OP_XOR2, 6, 5, 0},
+    {EC_GF_OP_XOR2, 5, 8, 0}, {EC_GF_OP_XOR2, 7, 6, 0},
+    {EC_GF_OP_XOR2, 0, 6, 0}, {EC_GF_OP_XOR2, 1, 5, 0},
+    {EC_GF_OP_XOR2, 5, 3, 0}, {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_1D = {9,
+                                    {
+                                        0,
+                                        7,
+                                        5,
+                                        8,
+                                        2,
+                                        3,
+                                        4,
+                                        1,
+                                        6,
+                                    },
+                                    ec_gf8_mul_1D_ops};
+
+static ec_gf_op_t ec_gf8_mul_1E_ops[] = {
+    {EC_GF_OP_XOR2, 4, 0, 0}, {EC_GF_OP_XOR2, 0, 5, 0},
+    {EC_GF_OP_XOR2, 0, 6, 0}, {EC_GF_OP_XOR2, 1, 4, 0},
+    {EC_GF_OP_XOR2, 5, 1, 0}, {EC_GF_OP_XOR2, 2, 7, 0},
+    {EC_GF_OP_XOR2, 7, 0, 0}, {EC_GF_OP_XOR2, 1, 2, 0},
+    {EC_GF_OP_XOR2, 6, 3, 0}, {EC_GF_OP_XOR2, 4, 7, 0},
+    {EC_GF_OP_XOR2, 6, 1, 0}, {EC_GF_OP_XOR2, 3, 4, 0},
+    {EC_GF_OP_XOR2, 0, 6, 0}, {EC_GF_OP_XOR2, 2, 3, 0},
+    {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_1E = {8,
+                                    {
+                                        4,
+                                        7,
+                                        5,
+                                        1,
+                                        6,
+                                        0,
+                                        2,
+                                        3,
+                                    },
+                                    ec_gf8_mul_1E_ops};
+
+static ec_gf_op_t ec_gf8_mul_1F_ops[] = {
+    {EC_GF_OP_XOR2, 5, 2, 0}, {EC_GF_OP_XOR2, 5, 4, 0},
+    {EC_GF_OP_XOR2, 3, 5, 0}, {EC_GF_OP_XOR2, 5, 0, 0},
+    {EC_GF_OP_XOR2, 5, 1, 0}, {EC_GF_OP_XOR2, 6, 3, 0},
+    {EC_GF_OP_XOR2, 7, 5, 0}, {EC_GF_OP_XOR2, 2, 6, 0},
+    {EC_GF_OP_XOR2, 7, 2, 0}, {EC_GF_OP_XOR3, 8, 3, 7},
+    {EC_GF_OP_XOR2, 4, 8, 0}, {EC_GF_OP_XOR2, 1, 8, 0},
+    {EC_GF_OP_XOR2, 6, 4, 0}, {EC_GF_OP_XOR2, 0, 6, 0},
+    {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_1F = {9,
+                                    {
+                                        1,
+                                        4,
+                                        5,
+                                        6,
+                                        7,
+                                        0,
+                                        3,
+                                        2,
+                                        8,
+                                    },
+                                    ec_gf8_mul_1F_ops};
+
+static ec_gf_op_t ec_gf8_mul_20_ops[] = {
+    {EC_GF_OP_XOR2, 6, 7, 0}, {EC_GF_OP_XOR2, 1, 5, 0},
+    {EC_GF_OP_XOR2, 7, 3, 0}, {EC_GF_OP_XOR2, 1, 6, 0},
+    {EC_GF_OP_XOR2, 3, 4, 0}, {EC_GF_OP_XOR2, 2, 6, 0},
+    {EC_GF_OP_XOR2, 6, 3, 0}, {EC_GF_OP_XOR2, 3, 5, 0},
+    {EC_GF_OP_XOR2, 5, 7, 0}, {EC_GF_OP_XOR2, 0, 5, 0},
+    {EC_GF_OP_XOR2, 0, 6, 0}, {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_20 = {8,
+                                    {
+                                        7,
+                                        4,
+                                        5,
+                                        6,
+                                        3,
+                                        0,
+                                        1,
+                                        2,
+                                    },
+                                    ec_gf8_mul_20_ops};
+
+static ec_gf_op_t ec_gf8_mul_21_ops[] = {
+    {EC_GF_OP_COPY, 9, 0, 0}, {EC_GF_OP_XOR2, 0, 3, 0},
+    {EC_GF_OP_XOR2, 5, 3, 0}, {EC_GF_OP_XOR2, 3, 1, 0},
+    {EC_GF_OP_XOR2, 1, 4, 0}, {EC_GF_OP_XOR2, 4, 6, 0},
+    {EC_GF_OP_XOR3, 8, 7, 5}, {EC_GF_OP_XOR2, 0, 7, 0},
+    {EC_GF_OP_XOR2, 6, 2, 0}, {EC_GF_OP_XOR2, 7, 4, 0},
+    {EC_GF_OP_XOR2, 3, 8, 0}, {EC_GF_OP_XOR2, 2, 8, 0},
+    {EC_GF_OP_XOR2, 4, 9, 0}, {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_21 = {10,
+                                    {
+                                        0,
+                                        1,
+                                        2,
+                                        7,
+                                        5,
+                                        4,
+                                        3,
+                                        6,
+                                        8,
+                                        9,
+                                    },
+                                    ec_gf8_mul_21_ops};
+
+static ec_gf_op_t ec_gf8_mul_22_ops[] = {
+    {EC_GF_OP_XOR2, 0, 4, 0}, {EC_GF_OP_XOR2, 4, 6, 0},
+    {EC_GF_OP_XOR2, 6, 7, 0}, {EC_GF_OP_XOR2, 7, 2, 0},
+    {EC_GF_OP_XOR2, 2, 3, 0}, {EC_GF_OP_XOR2, 2, 4, 0},
+    {EC_GF_OP_XOR2, 4, 5, 0}, {EC_GF_OP_XOR2, 5, 3, 0},
+    {EC_GF_OP_XOR2, 5, 1, 0}, {EC_GF_OP_XOR2, 1, 6, 0},
+    {EC_GF_OP_XOR2, 6, 4, 0}, {EC_GF_OP_XOR2, 4, 0, 0},
+    {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_22 = {8,
+                                    {
+                                        3,
+                                        0,
+                                        5,
+                                        2,
+                                        6,
+                                        4,
+                                        1,
+                                        7,
+                                    },
+                                    ec_gf8_mul_22_ops};
+
+static ec_gf_op_t ec_gf8_mul_23_ops[] = {
+    {EC_GF_OP_COPY, 8, 2, 0}, {EC_GF_OP_XOR2, 2, 4, 0},
+    {EC_GF_OP_XOR2, 2, 6, 0}, {EC_GF_OP_XOR2, 4, 0, 0},
+    {EC_GF_OP_XOR2, 6, 0, 0}, {EC_GF_OP_XOR2, 0, 3, 0},
+    {EC_GF_OP_XOR2, 3, 5, 0}, {EC_GF_OP_XOR2, 3, 8, 0},
+    {EC_GF_OP_XOR2, 4, 1, 0}, {EC_GF_OP_XOR2, 5, 7, 0},
+    {EC_GF_OP_XOR2, 3, 1, 0}, {EC_GF_OP_XOR2, 1, 7, 0},
+    {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_23 = {9,
+                                    {
+                                        0,
+                                        4,
+                                        3,
+                                        2,
+                                        5,
+                                        6,
+                                        1,
+                                        8,
+                                        7,
+                                    },
+                                    ec_gf8_mul_23_ops};
+
+static ec_gf_op_t ec_gf8_mul_24_ops[] = {
+    {EC_GF_OP_XOR2, 6, 3, 0}, {EC_GF_OP_XOR2, 6, 7, 0},
+    {EC_GF_OP_XOR2, 5, 6, 0}, {EC_GF_OP_XOR2, 2, 0, 0},
+    {EC_GF_OP_XOR2, 0, 5, 0}, {EC_GF_OP_XOR2, 7, 4, 0},
+    {EC_GF_OP_XOR2, 3, 4, 0}, {EC_GF_OP_XOR2, 4, 0, 0},
+    {EC_GF_OP_XOR2, 1, 3, 0}, {EC_GF_OP_XOR2, 2, 4, 0},
+    {EC_GF_OP_XOR2, 5, 1, 0}, {EC_GF_OP_XOR2, 3, 2, 0},
+    {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_24 = {8,
+                                    {
+                                        6,
+                                        7,
+                                        0,
+                                        1,
+                                        2,
+                                        4,
+                                        5,
+                                        3,
+                                    },
+                                    ec_gf8_mul_24_ops};
+
+static ec_gf_op_t ec_gf8_mul_25_ops[] = {
+    {EC_GF_OP_XOR2, 2, 5, 0}, {EC_GF_OP_XOR2, 6, 2, 0},
+    {EC_GF_OP_XOR2, 3, 7, 0}, {EC_GF_OP_XOR2, 3, 6, 0},
+    {EC_GF_OP_XOR2, 0, 3, 0}, {EC_GF_OP_XOR2, 1, 4, 0},
+    {EC_GF_OP_XOR2, 2, 0, 0}, {EC_GF_OP_XOR2, 7, 1, 0},
+    {EC_GF_OP_XOR2, 4, 2, 0}, {EC_GF_OP_XOR2, 5, 7, 0},
+    {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_25 = {8,
+                                    {
+                                        2,
+                                        7,
+                                        0,
+                                        1,
+                                        3,
+                                        4,
+                                        5,
+                                        6,
+                                    },
+                                    ec_gf8_mul_25_ops};
+
+static ec_gf_op_t ec_gf8_mul_26_ops[] = {
+    {EC_GF_OP_XOR2, 1, 0, 0}, {EC_GF_OP_XOR2, 0, 7, 0},
+    {EC_GF_OP_XOR2, 4, 0, 0}, {EC_GF_OP_XOR2, 3, 6, 0},
+    {EC_GF_OP_XOR2, 6, 4, 0}, {EC_GF_OP_XOR2, 2, 5, 0},
+    {EC_GF_OP_XOR2, 7, 2, 0}, {EC_GF_OP_XOR2, 5, 3, 0},
+    {EC_GF_OP_XOR2, 2, 6, 0}, {EC_GF_OP_XOR2, 6, 1, 0},
+    {EC_GF_OP_XOR2, 1, 5, 0}, {EC_GF_OP_XOR2, 5, 0, 0},
+    {EC_GF_OP_XOR2, 0, 2, 0}, {EC_GF_OP_XOR2, 2, 1, 0},
+    {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_26 = {8,
+                                    {
+                                        3,
+                                        4,
+                                        1,
+                                        2,
+                                        0,
+                                        5,
+                                        6,
+                                        7,
+                                    },
+                                    ec_gf8_mul_26_ops};
+
+static ec_gf_op_t ec_gf8_mul_27_ops[] = {
+    {EC_GF_OP_XOR2, 3, 0, 0}, {EC_GF_OP_XOR2, 4, 1, 0},
+    {EC_GF_OP_XOR2, 5, 2, 0}, {EC_GF_OP_XOR2, 4, 7, 0},
+    {EC_GF_OP_XOR2, 1, 5, 0}, {EC_GF_OP_XOR2, 3, 6, 0},
+    {EC_GF_OP_XOR2, 2, 4, 0}, {EC_GF_OP_XOR2, 0, 4, 0},
+    {EC_GF_OP_XOR2, 6, 5, 0}, {EC_GF_OP_XOR2, 1, 3, 0},
+    {EC_GF_OP_XOR2, 7, 3, 0}, {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_27 = {8,
+                                    {
+                                        3,
+                                        0,
+                                        1,
+                                        2,
+                                        6,
+                                        7,
+                                        4,
+                                        5,
+                                    },
+                                    ec_gf8_mul_27_ops};
+
+static ec_gf_op_t ec_gf8_mul_28_ops[] = {
+    {EC_GF_OP_XOR2, 6, 4, 0}, {EC_GF_OP_XOR2, 5, 7, 0},
+    {EC_GF_OP_XOR2, 7, 6, 0}, {EC_GF_OP_XOR2, 4, 5, 0},
+    {EC_GF_OP_XOR2, 1, 3, 0}, {EC_GF_OP_XOR2, 5, 3, 0},
+    {EC_GF_OP_XOR2, 1, 7, 0}, {EC_GF_OP_XOR2, 0, 4, 0},
+    {EC_GF_OP_XOR2, 4, 1, 0}, {EC_GF_OP_XOR2, 7, 2, 0},
+    {EC_GF_OP_XOR2, 2, 0, 0}, {EC_GF_OP_XOR2, 0, 3, 0},
+    {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_28 = {8,
+                                    {
+                                        5,
+                                        6,
+                                        3,
+                                        0,
+                                        1,
+                                        2,
+                                        4,
+                                        7,
+                                    },
+                                    ec_gf8_mul_28_ops};
+
+static ec_gf_op_t ec_gf8_mul_29_ops[] = {
+    {EC_GF_OP_XOR2, 6, 4, 0}, {EC_GF_OP_XOR2, 5, 3, 0},
+    {EC_GF_OP_XOR2, 4, 3, 0}, {EC_GF_OP_XOR2, 3, 2, 0},
+    {EC_GF_OP_XOR2, 7, 4, 0}, {EC_GF_OP_XOR2, 2, 6, 0},
+    {EC_GF_OP_XOR2, 6, 1, 0}, {EC_GF_OP_XOR2, 1, 5, 0},
+    {EC_GF_OP_XOR2, 0, 7, 0}, {EC_GF_OP_XOR2, 5, 0, 0},
+    {EC_GF_OP_XOR2, 7, 6, 0}, {EC_GF_OP_XOR2, 0, 3, 0},
+    {EC_GF_OP_XOR2, 4, 5, 0}, {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_29 = {8,
+                                    {
+                                        4,
+                                        6,
+                                        3,
+                                        5,
+                                        7,
+                                        0,
+                                        1,
+                                        2,
+                                    },
+                                    ec_gf8_mul_29_ops};
+
+static ec_gf_op_t ec_gf8_mul_2A_ops[] = {
+    {EC_GF_OP_COPY, 8, 1, 0}, {EC_GF_OP_XOR2, 8, 0, 0},
+    {EC_GF_OP_XOR2, 4, 0, 0}, {EC_GF_OP_XOR2, 0, 2, 0},
+    {EC_GF_OP_XOR2, 1, 3, 0}, {EC_GF_OP_XOR2, 3, 5, 0},
+    {EC_GF_OP_XOR2, 0, 7, 0}, {EC_GF_OP_XOR2, 2, 3, 0},
+    {EC_GF_OP_XOR2, 7, 1, 0}, {EC_GF_OP_XOR2, 5, 0, 0},
+    {EC_GF_OP_XOR2, 2, 4, 0}, {EC_GF_OP_XOR2, 0, 4, 0},
+    {EC_GF_OP_XOR2, 1, 6, 0}, {EC_GF_OP_XOR2, 4, 6, 0},
+    {EC_GF_OP_XOR3, 6, 8, 4}, {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_2A = {9,
+                                    {
+                                        3,
+                                        4,
+                                        7,
+                                        2,
+                                        6,
+                                        5,
+                                        1,
+                                        0,
+                                        8,
+                                    },
+                                    ec_gf8_mul_2A_ops};
+
+static ec_gf_op_t ec_gf8_mul_2B_ops[] = {
+    {EC_GF_OP_XOR2, 7, 2, 0}, {EC_GF_OP_XOR2, 2, 4, 0},
+    {EC_GF_OP_XOR2, 6, 1, 0}, {EC_GF_OP_XOR2, 4, 0, 0},
+    {EC_GF_OP_XOR2, 5, 0, 0}, {EC_GF_OP_XOR2, 4, 6, 0},
+    {EC_GF_OP_XOR2, 1, 3, 0}, {EC_GF_OP_XOR2, 0, 7, 0},
+    {EC_GF_OP_XOR2, 3, 5, 0}, {EC_GF_OP_XOR2, 7, 1, 0},
+    {EC_GF_OP_XOR2, 5, 2, 0}, {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_2B = {8,
+                                    {
+                                        3,
+                                        4,
+                                        7,
+                                        5,
+                                        6,
+                                        0,
+                                        1,
+                                        2,
+                                    },
+                                    ec_gf8_mul_2B_ops};
+
+static ec_gf_op_t ec_gf8_mul_2C_ops[] = {
+    {EC_GF_OP_XOR2, 3, 4, 0}, {EC_GF_OP_XOR2, 4, 7, 0},
+    {EC_GF_OP_XOR2, 2, 1, 0}, {EC_GF_OP_XOR2, 6, 4, 0},
+    {EC_GF_OP_XOR2, 2, 3, 0}, {EC_GF_OP_XOR2, 3, 6, 0},
+    {EC_GF_OP_XOR2, 5, 3, 0}, {EC_GF_OP_XOR2, 1, 5, 0},
+    {EC_GF_OP_XOR2, 3, 0, 0}, {EC_GF_OP_XOR2, 4, 1, 0},
+    {EC_GF_OP_XOR2, 7, 3, 0}, {EC_GF_OP_XOR2, 1, 2, 0},
+    {EC_GF_OP_XOR2, 0, 4, 0}, {EC_GF_OP_XOR2, 3, 1, 0},
+    {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_2C = {8,
+                                    {
+                                        5,
+                                        6,
+                                        7,
+                                        0,
+                                        2,
+                                        3,
+                                        4,
+                                        1,
+                                    },
+                                    ec_gf8_mul_2C_ops};
+
+static ec_gf_op_t ec_gf8_mul_2D_ops[] = {
+    {EC_GF_OP_XOR2, 3, 2, 0}, {EC_GF_OP_XOR2, 1, 3, 0},
+    {EC_GF_OP_XOR2, 3, 0, 0}, {EC_GF_OP_XOR2, 4, 3, 0},
+    {EC_GF_OP_XOR2, 3, 6, 0}, {EC_GF_OP_XOR2, 6, 1, 0},
+    {EC_GF_OP_XOR2, 2, 3, 0}, {EC_GF_OP_XOR3, 8, 4, 6},
+    {EC_GF_OP_XOR2, 5, 8, 0}, {EC_GF_OP_XOR2, 7, 8, 0},
+    {EC_GF_OP_XOR2, 2, 5, 0}, {EC_GF_OP_XOR2, 0, 7, 0},
+    {EC_GF_OP_XOR2, 6, 2, 0}, {EC_GF_OP_XOR2, 7, 2, 0},
+    {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_2D = {9,
+                                    {
+                                        7,
+                                        0,
+                                        3,
+                                        5,
+                                        1,
+                                        4,
+                                        2,
+                                        6,
+                                        8,
+                                    },
+                                    ec_gf8_mul_2D_ops};
+
+static ec_gf_op_t ec_gf8_mul_2E_ops[] = {
+    {EC_GF_OP_XOR2, 4, 1, 0}, {EC_GF_OP_XOR2, 0, 1, 0},
+    {EC_GF_OP_COPY, 8, 4, 0}, {EC_GF_OP_XOR2, 3, 6, 0},
+    {EC_GF_OP_XOR2, 1, 5, 0}, {EC_GF_OP_XOR2, 4, 3, 0},
+    {EC_GF_OP_XOR2, 8, 7, 0}, {EC_GF_OP_XOR2, 5, 3, 0},
+    {EC_GF_OP_XOR2, 2, 8, 0}, {EC_GF_OP_XOR2, 3, 0, 0},
+    {EC_GF_OP_XOR2, 6, 8, 0}, {EC_GF_OP_XOR2, 1, 2, 0},
+    {EC_GF_OP_XOR2, 7, 3, 0}, {EC_GF_OP_XOR2, 0, 6, 0},
+    {EC_GF_OP_XOR2, 3, 1, 0}, {EC_GF_OP_XOR2, 6, 3, 0},
+    {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_2E = {9,
+                                    {
+                                        5,
+                                        0,
+                                        7,
+                                        3,
+                                        2,
+                                        6,
+                                        4,
+                                        1,
+                                        8,
+                                    },
+                                    ec_gf8_mul_2E_ops};
+
+static ec_gf_op_t ec_gf8_mul_2F_ops[] = {
+    {EC_GF_OP_XOR2, 0, 2, 0}, {EC_GF_OP_XOR2, 0, 3, 0},
+    {EC_GF_OP_XOR2, 7, 1, 0}, {EC_GF_OP_XOR2, 6, 0, 0},
+    {EC_GF_OP_XOR2, 7, 2, 0}, {EC_GF_OP_XOR2, 3, 4, 0},
+    {EC_GF_OP_XOR3, 8, 7, 6}, {EC_GF_OP_XOR2, 1, 3, 0},
+    {EC_GF_OP_XOR2, 5, 2, 0}, {EC_GF_OP_XOR2, 3, 8, 0},
+    {EC_GF_OP_XOR2, 2, 8, 0}, {EC_GF_OP_XOR2, 4, 5, 0},
+    {EC_GF_OP_XOR2, 6, 5, 0}, {EC_GF_OP_XOR2, 5, 3, 0},
+    {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_2F = {9,
+                                    {
+                                        6,
+                                        3,
+                                        2,
+                                        5,
+                                        7,
+                                        0,
+                                        1,
+                                        4,
+                                        8,
+                                    },
+                                    ec_gf8_mul_2F_ops};
+
+static ec_gf_op_t ec_gf8_mul_30_ops[] = {
+    {EC_GF_OP_COPY, 8, 0, 0}, {EC_GF_OP_XOR2, 8, 1, 0},
+    {EC_GF_OP_XOR2, 1, 2, 0}, {EC_GF_OP_XOR2, 7, 4, 0},
+    {EC_GF_OP_XOR2, 6, 3, 0}, {EC_GF_OP_XOR2, 3, 7, 0},
+    {EC_GF_OP_XOR2, 0, 6, 0}, {EC_GF_OP_XOR2, 1, 5, 0},
+    {EC_GF_OP_XOR2, 4, 5, 0}, {EC_GF_OP_XOR2, 2, 6, 0},
+    {EC_GF_OP_XOR2, 5, 6, 0}, {EC_GF_OP_XOR3, 6, 8, 7},
+    {EC_GF_OP_XOR2, 7, 5, 0}, {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_30 = {9,
+                                    {
+                                        3,
+                                        4,
+                                        7,
+                                        5,
+                                        0,
+                                        6,
+                                        1,
+                                        2,
+                                        8,
+                                    },
+                                    ec_gf8_mul_30_ops};
+
+static ec_gf_op_t ec_gf8_mul_31_ops[] = {
+    {EC_GF_OP_XOR2, 3, 0, 0}, {EC_GF_OP_XOR2, 3, 4, 0},
+    {EC_GF_OP_XOR2, 4, 5, 0}, {EC_GF_OP_XOR2, 2, 1, 0},
+    {EC_GF_OP_XOR2, 1, 4, 0}, {EC_GF_OP_XOR2, 5, 6, 0},
+    {EC_GF_OP_XOR2, 6, 3, 0}, {EC_GF_OP_XOR2, 0, 7, 0},
+    {EC_GF_OP_XOR2, 2, 5, 0}, {EC_GF_OP_XOR2, 7, 3, 0},
+    {EC_GF_OP_XOR2, 0, 1, 0}, {EC_GF_OP_XOR2, 3, 2, 0},
+    {EC_GF_OP_XOR2, 3, 0, 0}, {EC_GF_OP_XOR2, 4, 3, 0},
+    {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_31 = {8,
+                                    {
+                                        7,
+                                        1,
+                                        4,
+                                        5,
+                                        6,
+                                        0,
+                                        2,
+                                        3,
+                                    },
+                                    ec_gf8_mul_31_ops};
+
+static ec_gf_op_t ec_gf8_mul_32_ops[] = {
+    {EC_GF_OP_XOR2, 6, 5, 0}, {EC_GF_OP_XOR2, 5, 0, 0},
+    {EC_GF_OP_XOR2, 0, 1, 0}, {EC_GF_OP_XOR2, 0, 7, 0},
+    {EC_GF_OP_XOR2, 7, 6, 0}, {EC_GF_OP_XOR2, 6, 1, 0},
+    {EC_GF_OP_XOR2, 1, 2, 0}, {EC_GF_OP_XOR2, 2, 3, 0},
+    {EC_GF_OP_XOR2, 3, 4, 0}, {EC_GF_OP_XOR2, 4, 5, 0},
+    {EC_GF_OP_XOR2, 6, 3, 0}, {EC_GF_OP_XOR2, 5, 7, 0},
+    {EC_GF_OP_XOR2, 7, 2, 0}, {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_32 = {8,
+                                    {
+                                        3,
+                                        4,
+                                        6,
+                                        7,
+                                        5,
+                                        0,
+                                        1,
+                                        2,
+                                    },
+                                    ec_gf8_mul_32_ops};
+
+static ec_gf_op_t ec_gf8_mul_33_ops[] = {
+    {EC_GF_OP_XOR2, 3, 2, 0}, {EC_GF_OP_XOR2, 2, 1, 0},
+    {EC_GF_OP_XOR2, 1, 0, 0}, {EC_GF_OP_XOR2, 1, 5, 0},
+    {EC_GF_OP_XOR2, 4, 1, 0}, {EC_GF_OP_XOR2, 6, 2, 0},
+    {EC_GF_OP_XOR2, 1, 7, 0}, {EC_GF_OP_XOR2, 2, 4, 0},
+    {EC_GF_OP_XOR2, 7, 3, 0}, {EC_GF_OP_XOR2, 0, 6, 0},
+    {EC_GF_OP_XOR2, 3, 2, 0}, {EC_GF_OP_XOR2, 5, 3, 0},
+    {EC_GF_OP_XOR2, 3, 0, 0}, {EC_GF_OP_XOR2, 0, 1, 0},
+    {EC_GF_OP_XOR2, 2, 0, 0}, {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_33 = {8,
+                                    {
+                                        5,
+                                        4,
+                                        3,
+                                        0,
+                                        2,
+                                        1,
+                                        6,
+                                        7,
+                                    },
+                                    ec_gf8_mul_33_ops};
+
+static ec_gf_op_t ec_gf8_mul_34_ops[] = {
+    {EC_GF_OP_XOR2, 5, 4, 0}, {EC_GF_OP_XOR2, 4, 3, 0},
+    {EC_GF_OP_XOR2, 2, 5, 0}, {EC_GF_OP_XOR2, 6, 4, 0},
+    {EC_GF_OP_XOR2, 5, 7, 0}, {EC_GF_OP_XOR2, 4, 0, 0},
+    {EC_GF_OP_XOR2, 7, 6, 0}, {EC_GF_OP_XOR2, 0, 5, 0},
+    {EC_GF_OP_XOR2, 6, 2, 0}, {EC_GF_OP_XOR2, 4, 1, 0},
+    {EC_GF_OP_XOR2, 3, 0, 0}, {EC_GF_OP_XOR2, 1, 2, 0},
+    {EC_GF_OP_XOR2, 0, 4, 0}, {EC_GF_OP_XOR2, 2, 3, 0},
+    {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_34 = {8,
+                                    {
+                                        7,
+                                        5,
+                                        3,
+                                        0,
+                                        2,
+                                        4,
+                                        1,
+                                        6,
+                                    },
+                                    ec_gf8_mul_34_ops};
+
+static ec_gf_op_t ec_gf8_mul_35_ops[] = {
+    {EC_GF_OP_XOR2, 0, 1, 0}, {EC_GF_OP_XOR2, 1, 4, 0},
+    {EC_GF_OP_XOR2, 7, 1, 0}, {EC_GF_OP_XOR2, 3, 7, 0},
+    {EC_GF_OP_XOR2, 0, 3, 0}, {EC_GF_OP_XOR2, 7, 5, 0},
+    {EC_GF_OP_XOR2, 2, 0, 0}, {EC_GF_OP_XOR2, 4, 7, 0},
+    {EC_GF_OP_XOR2, 5, 2, 0}, {EC_GF_OP_XOR2, 6, 0, 0},
+    {EC_GF_OP_XOR2, 1, 5, 0}, {EC_GF_OP_XOR2, 0, 4, 0},
+    {EC_GF_OP_XOR2, 1, 6, 0}, {EC_GF_OP_XOR2, 3, 1, 0},
+    {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_35 = {8,
+                                    {
+                                        6,
+                                        7,
+                                        5,
+                                        4,
+                                        2,
+                                        0,
+                                        1,
+                                        3,
+                                    },
+                                    ec_gf8_mul_35_ops};
+
+static ec_gf_op_t ec_gf8_mul_36_ops[] = {
+    {EC_GF_OP_XOR2, 5, 2, 0}, {EC_GF_OP_XOR2, 2, 0, 0},
+    {EC_GF_OP_XOR2, 6, 4, 0}, {EC_GF_OP_XOR2, 4, 2, 0},
+    {EC_GF_OP_XOR2, 7, 4, 0}, {EC_GF_OP_XOR2, 0, 1, 0},
+    {EC_GF_OP_XOR2, 6, 3, 0}, {EC_GF_OP_XOR2, 7, 5, 0},
+    {EC_GF_OP_XOR2, 5, 3, 0}, {EC_GF_OP_XOR2, 3, 0, 0},
+    {EC_GF_OP_XOR2, 1, 5, 0}, {EC_GF_OP_XOR2, 0, 4, 0},
+    {EC_GF_OP_XOR2, 4, 1, 0}, {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_36 = {8,
+                                    {
+                                        6,
+                                        7,
+                                        4,
+                                        1,
+                                        2,
+                                        3,
+                                        0,
+                                        5,
+                                    },
+                                    ec_gf8_mul_36_ops};
+
+static ec_gf_op_t ec_gf8_mul_37_ops[] = {
+    {EC_GF_OP_XOR2, 1, 2, 0}, {EC_GF_OP_XOR2, 0, 2, 0},
+    {EC_GF_OP_XOR2, 6, 1, 0}, {EC_GF_OP_XOR2, 0, 4, 0},
+    {EC_GF_OP_XOR2, 1, 5, 0}, {EC_GF_OP_XOR3, 8, 0, 1},
+    {EC_GF_OP_XOR2, 3, 8, 0}, {EC_GF_OP_XOR2, 7, 8, 0},
+    {EC_GF_OP_XOR2, 2, 3, 0}, {EC_GF_OP_XOR2, 3, 4, 0},
+    {EC_GF_OP_XOR2, 5, 2, 0}, {EC_GF_OP_XOR2, 4, 6, 0},
+    {EC_GF_OP_XOR2, 6, 5, 0}, {EC_GF_OP_XOR2, 5, 7, 0},
+    {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_37 = {9,
+                                    {
+                                        6,
+                                        7,
+                                        2,
+                                        1,
+                                        0,
+                                        3,
+                                        4,
+                                        5,
+                                        8,
+                                    },
+                                    ec_gf8_mul_37_ops};
+
+static ec_gf_op_t ec_gf8_mul_38_ops[] = {
+    {EC_GF_OP_XOR2, 6, 4, 0}, {EC_GF_OP_XOR2, 5, 6, 0},
+    {EC_GF_OP_XOR2, 6, 3, 0}, {EC_GF_OP_XOR2, 3, 0, 0},
+    {EC_GF_OP_XOR2, 2, 6, 0}, {EC_GF_OP_XOR2, 7, 5, 0},
+    {EC_GF_OP_XOR2, 0, 1, 0}, {EC_GF_OP_XOR3, 8, 6, 7},
+    {EC_GF_OP_XOR2, 1, 2, 0}, {EC_GF_OP_XOR2, 7, 1, 0},
+    {EC_GF_OP_XOR2, 0, 8, 0}, {EC_GF_OP_XOR2, 4, 8, 0},
+    {EC_GF_OP_XOR2, 1, 3, 0}, {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_38 = {9,
+                                    {
+                                        4,
+                                        5,
+                                        6,
+                                        3,
+                                        0,
+                                        1,
+                                        7,
+                                        2,
+                                        8,
+                                    },
+                                    ec_gf8_mul_38_ops};
+
+static ec_gf_op_t ec_gf8_mul_39_ops[] = {
+    {EC_GF_OP_XOR2, 5, 1, 0}, {EC_GF_OP_XOR2, 4, 5, 0},
+    {EC_GF_OP_XOR2, 6, 4, 0}, {EC_GF_OP_XOR2, 2, 0, 0},
+    {EC_GF_OP_XOR2, 2, 6, 0}, {EC_GF_OP_XOR2, 3, 0, 0},
+    {EC_GF_OP_XOR2, 5, 2, 0}, {EC_GF_OP_XOR2, 3, 5, 0},
+    {EC_GF_OP_XOR2, 7, 3, 0}, {EC_GF_OP_XOR2, 4, 7, 0},
+    {EC_GF_OP_XOR2, 5, 4, 0}, {EC_GF_OP_XOR2, 1, 5, 0},
+    {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_39 = {8,
+                                    {
+                                        1,
+                                        6,
+                                        3,
+                                        0,
+                                        5,
+                                        2,
+                                        4,
+                                        7,
+                                    },
+                                    ec_gf8_mul_39_ops};
+
+static ec_gf_op_t ec_gf8_mul_3A_ops[] = {
+    {EC_GF_OP_XOR2, 6, 1, 0}, {EC_GF_OP_XOR2, 7, 3, 0},
+    {EC_GF_OP_XOR2, 1, 0, 0}, {EC_GF_OP_XOR2, 3, 4, 0},
+    {EC_GF_OP_XOR2, 0, 2, 0}, {EC_GF_OP_XOR2, 4, 6, 0},
+    {EC_GF_OP_XOR2, 2, 3, 0}, {EC_GF_OP_XOR2, 6, 0, 0},
+    {EC_GF_OP_XOR2, 3, 5, 0}, {EC_GF_OP_XOR2, 0, 7, 0},
+    {EC_GF_OP_XOR2, 5, 1, 0}, {EC_GF_OP_XOR2, 7, 4, 0},
+    {EC_GF_OP_XOR2, 1, 0, 0}, {EC_GF_OP_XOR2, 4, 5, 0},
+    {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_3A = {8,
+                                    {
+                                        3,
+                                        4,
+                                        7,
+                                        0,
+                                        5,
+                                        6,
+                                        1,
+                                        2,
+                                    },
+                                    ec_gf8_mul_3A_ops};
+
+static ec_gf_op_t ec_gf8_mul_3B_ops[] = {
+    {EC_GF_OP_XOR2, 2, 0, 0}, {EC_GF_OP_XOR2, 0, 4, 0},
+    {EC_GF_OP_XOR2, 3, 0, 0}, {EC_GF_OP_XOR2, 7, 2, 0},
+    {EC_GF_OP_XOR3, 8, 7, 3}, {EC_GF_OP_XOR2, 1, 6, 0},
+    {EC_GF_OP_XOR2, 3, 5, 0}, {EC_GF_OP_XOR2, 5, 1, 0},
+    {EC_GF_OP_XOR2, 1, 8, 0}, {EC_GF_OP_XOR2, 0, 5, 0},
+    {EC_GF_OP_XOR2, 2, 5, 0}, {EC_GF_OP_XOR2, 4, 1, 0},
+    {EC_GF_OP_XOR2, 6, 0, 0}, {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_3B = {9,
+                                    {
+                                        3,
+                                        0,
+                                        1,
+                                        7,
+                                        6,
+                                        2,
+                                        4,
+                                        8,
+                                        5,
+                                    },
+                                    ec_gf8_mul_3B_ops};
+
+static ec_gf_op_t ec_gf8_mul_3C_ops[] = {
+    {EC_GF_OP_XOR2, 7, 5, 0}, {EC_GF_OP_XOR2, 7, 4, 0},
+    {EC_GF_OP_XOR2, 1, 0, 0}, {EC_GF_OP_XOR2, 6, 7, 0},
+    {EC_GF_OP_XOR2, 0, 3, 0}, {EC_GF_OP_XOR2, 3, 6, 0},
+    {EC_GF_OP_XOR2, 5, 3, 0}, {EC_GF_OP_XOR2, 1, 5, 0},
+    {EC_GF_OP_XOR2, 2, 1, 0}, {EC_GF_OP_XOR2, 1, 4, 0},
+    {EC_GF_OP_XOR2, 7, 2, 0}, {EC_GF_OP_XOR2, 4, 0, 0},
+    {EC_GF_OP_XOR2, 5, 7, 0}, {EC_GF_OP_XOR2, 0, 5, 0},
+    {EC_GF_OP_XOR2, 5, 1, 0}, {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_3C = {8,
+                                    {
+                                        3,
+                                        6,
+                                        4,
+                                        1,
+                                        7,
+                                        2,
+                                        0,
+                                        5,
+                                    },
+                                    ec_gf8_mul_3C_ops};
+
+static ec_gf_op_t ec_gf8_mul_3D_ops[] = {
+    {EC_GF_OP_XOR2, 2, 0, 0}, {EC_GF_OP_XOR2, 3, 2, 0},
+    {EC_GF_OP_XOR2, 5, 1, 0}, {EC_GF_OP_XOR2, 4, 3, 0},
+    {EC_GF_OP_XOR2, 5, 4, 0}, {EC_GF_OP_XOR2, 6, 5, 0},
+    {EC_GF_OP_XOR2, 7, 6, 0}, {EC_GF_OP_XOR2, 0, 7, 0},
+    {EC_GF_OP_XOR2, 3, 7, 0}, {EC_GF_OP_XOR2, 1, 0, 0},
+    {EC_GF_OP_XOR2, 2, 1, 0}, {EC_GF_OP_XOR2, 5, 1, 0},
+    {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_3D = {8,
+                                    {
+                                        2,
+                                        3,
+                                        4,
+                                        5,
+                                        6,
+                                        7,
+                                        0,
+                                        1,
+                                    },
+                                    ec_gf8_mul_3D_ops};
+
+static ec_gf_op_t ec_gf8_mul_3E_ops[] = {
+    {EC_GF_OP_XOR2, 3, 5, 0}, {EC_GF_OP_XOR2, 4, 3, 0},
+    {EC_GF_OP_XOR2, 6, 4, 0}, {EC_GF_OP_XOR2, 4, 2, 0},
+    {EC_GF_OP_XOR2, 1, 7, 0}, {EC_GF_OP_XOR2, 1, 4, 0},
+    {EC_GF_OP_XOR2, 5, 1, 0}, {EC_GF_OP_XOR2, 0, 5, 0},
+    {EC_GF_OP_XOR2, 2, 0, 0}, {EC_GF_OP_XOR2, 0, 6, 0},
+    {EC_GF_OP_XOR2, 3, 0, 0}, {EC_GF_OP_XOR2, 7, 3, 0},
+    {EC_GF_OP_XOR2, 1, 7, 0}, {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_3E = {8,
+                                    {
+                                        6,
+                                        1,
+                                        2,
+                                        7,
+                                        0,
+                                        3,
+                                        5,
+                                        4,
+                                    },
+                                    ec_gf8_mul_3E_ops};
+
+static ec_gf_op_t ec_gf8_mul_3F_ops[] = {
+    {EC_GF_OP_COPY, 8, 0, 0},  {EC_GF_OP_XOR2, 0, 1, 0},
+    {EC_GF_OP_COPY, 10, 4, 0}, {EC_GF_OP_XOR2, 0, 6, 0},
+    {EC_GF_OP_XOR2, 4, 5, 0},  {EC_GF_OP_XOR2, 4, 0, 0},
+    {EC_GF_OP_XOR2, 1, 3, 0},  {EC_GF_OP_COPY, 9, 2, 0},
+    {EC_GF_OP_XOR2, 1, 4, 0},  {EC_GF_OP_XOR2, 2, 0, 0},
+    {EC_GF_OP_XOR2, 7, 4, 0},  {EC_GF_OP_XOR3, 4, 9, 7},
+    {EC_GF_OP_XOR2, 3, 4, 0},  {EC_GF_OP_XOR2, 5, 3, 0},
+    {EC_GF_OP_XOR2, 0, 3, 0},  {EC_GF_OP_XOR2, 6, 5, 0},
+    {EC_GF_OP_XOR2, 3, 10, 0}, {EC_GF_OP_XOR2, 5, 8, 0},
+    {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_3F = {11,
+                                    {
+                                        1,
+                                        7,
+                                        6,
+                                        2,
+                                        4,
+                                        3,
+                                        5,
+                                        0,
+                                        8,
+                                        9,
+                                        10,
+                                    },
+                                    ec_gf8_mul_3F_ops};
+
+static ec_gf_op_t ec_gf8_mul_40_ops[] = {
+    {EC_GF_OP_XOR2, 1, 2, 0}, {EC_GF_OP_XOR2, 2, 3, 0},
+    {EC_GF_OP_XOR2, 2, 4, 0}, {EC_GF_OP_XOR2, 4, 5, 0},
+    {EC_GF_OP_XOR2, 6, 4, 0}, {EC_GF_OP_XOR2, 0, 6, 0},
+    {EC_GF_OP_XOR2, 7, 3, 0}, {EC_GF_OP_XOR2, 6, 2, 0},
+    {EC_GF_OP_XOR2, 3, 4, 0}, {EC_GF_OP_XOR3, 8, 7, 6},
+    {EC_GF_OP_XOR2, 1, 8, 0}, {EC_GF_OP_XOR2, 5, 8, 0},
+    {EC_GF_OP_XOR2, 4, 8, 0}, {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_40 = {9,
+                                    {
+                                        5,
+                                        7,
+                                        4,
+                                        6,
+                                        2,
+                                        3,
+                                        0,
+                                        1,
+                                        8,
+                                    },
+                                    ec_gf8_mul_40_ops};
+
+static ec_gf_op_t ec_gf8_mul_41_ops[] = {
+    {EC_GF_OP_COPY, 8, 0, 0}, {EC_GF_OP_XOR2, 8, 4, 0},
+    {EC_GF_OP_XOR2, 8, 5, 0}, {EC_GF_OP_XOR2, 5, 6, 0},
+    {EC_GF_OP_XOR2, 0, 2, 0}, {EC_GF_OP_XOR2, 6, 7, 0},
+    {EC_GF_OP_XOR2, 0, 6, 0}, {EC_GF_OP_XOR2, 7, 1, 0},
+    {EC_GF_OP_XOR2, 6, 4, 0}, {EC_GF_OP_XOR2, 1, 5, 0},
+    {EC_GF_OP_XOR2, 7, 3, 0}, {EC_GF_OP_XOR2, 4, 3, 0},
+    {EC_GF_OP_XOR2, 5, 2, 0}, {EC_GF_OP_XOR2, 3, 2, 0},
+    {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_41 = {9,
+                                    {
+                                        0,
+                                        7,
+                                        6,
+                                        5,
+                                        3,
+                                        4,
+                                        8,
+                                        1,
+                                        2,
+                                    },
+                                    ec_gf8_mul_41_ops};
+
+static ec_gf_op_t ec_gf8_mul_42_ops[] = {
+    {EC_GF_OP_COPY, 8, 0, 0}, {EC_GF_OP_XOR2, 0, 2, 0},
+    {EC_GF_OP_XOR2, 8, 3, 0}, {EC_GF_OP_XOR2, 2, 6, 0},
+    {EC_GF_OP_XOR2, 3, 5, 0}, {EC_GF_OP_XOR2, 4, 2, 0},
+    {EC_GF_OP_XOR2, 5, 1, 0}, {EC_GF_OP_XOR2, 0, 4, 0},
+    {EC_GF_OP_XOR2, 6, 7, 0}, {EC_GF_OP_XOR2, 1, 4, 0},
+    {EC_GF_OP_XOR2, 5, 7, 0}, {EC_GF_OP_XOR2, 4, 6, 0},
+    {EC_GF_OP_XOR2, 7, 8, 0}, {EC_GF_OP_XOR2, 6, 3, 0},
+    {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_42 = {9,
+                                    {
+                                        2,
+                                        7,
+                                        1,
+                                        6,
+                                        4,
+                                        3,
+                                        0,
+                                        5,
+                                        8,
+                                    },
+                                    ec_gf8_mul_42_ops};
+
+static ec_gf_op_t ec_gf8_mul_43_ops[] = {
+    {EC_GF_OP_XOR2, 5, 1, 0}, {EC_GF_OP_XOR2, 1, 6, 0},
+    {EC_GF_OP_XOR2, 6, 0, 0}, {EC_GF_OP_XOR2, 0, 4, 0},
+    {EC_GF_OP_XOR2, 4, 1, 0}, {EC_GF_OP_XOR2, 1, 7, 0},
+    {EC_GF_OP_XOR2, 7, 2, 0}, {EC_GF_OP_XOR2, 2, 6, 0},
+    {EC_GF_OP_XOR2, 6, 3, 0}, {EC_GF_OP_XOR2, 6, 1, 0},
+    {EC_GF_OP_XOR2, 1, 5, 0}, {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_43 = {8,
+                                    {
+                                        2,
+                                        6,
+                                        4,
+                                        1,
+                                        7,
+                                        3,
+                                        0,
+                                        5,
+                                    },
+                                    ec_gf8_mul_43_ops};
+
+static ec_gf_op_t ec_gf8_mul_44_ops[] = {
+    {EC_GF_OP_XOR2, 1, 6, 0}, {EC_GF_OP_XOR2, 4, 7, 0},
+    {EC_GF_OP_XOR2, 6, 5, 0}, {EC_GF_OP_XOR2, 5, 4, 0},
+    {EC_GF_OP_XOR2, 4, 0, 0}, {EC_GF_OP_XOR2, 4, 2, 0},
+    {EC_GF_OP_XOR2, 0, 6, 0}, {EC_GF_OP_XOR2, 2, 7, 0},
+    {EC_GF_OP_XOR2, 6, 3, 0}, {EC_GF_OP_XOR2, 7, 1, 0},
+    {EC_GF_OP_XOR2, 1, 2, 0}, {EC_GF_OP_XOR2, 1, 6, 0},
+    {EC_GF_OP_XOR2, 6, 5, 0}, {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_44 = {8,
+                                    {
+                                        2,
+                                        3,
+                                        4,
+                                        1,
+                                        6,
+                                        5,
+                                        0,
+                                        7,
+                                    },
+                                    ec_gf8_mul_44_ops};
+
+static ec_gf_op_t ec_gf8_mul_45_ops[] = {
+    {EC_GF_OP_XOR2, 2, 0, 0}, {EC_GF_OP_XOR2, 4, 7, 0},
+    {EC_GF_OP_XOR2, 2, 7, 0}, {EC_GF_OP_XOR2, 7, 3, 0},
+    {EC_GF_OP_XOR2, 7, 6, 0}, {EC_GF_OP_XOR2, 5, 0, 0},
+    {EC_GF_OP_XOR2, 6, 1, 0}, {EC_GF_OP_XOR2, 3, 1, 0},
+    {EC_GF_OP_XOR2, 0, 4, 0}, {EC_GF_OP_XOR2, 1, 5, 0},
+    {EC_GF_OP_XOR2, 1, 2, 0}, {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_45 = {8,
+                                    {
+                                        2,
+                                        3,
+                                        0,
+                                        1,
+                                        7,
+                                        4,
+                                        5,
+                                        6,
+                                    },
+                                    ec_gf8_mul_45_ops};
+
+static ec_gf_op_t ec_gf8_mul_46_ops[] = {
+    {EC_GF_OP_XOR3, 8, 2, 4}, {EC_GF_OP_XOR2, 4, 6, 0},
+    {EC_GF_OP_XOR2, 8, 0, 0}, {EC_GF_OP_XOR2, 6, 0, 0},
+    {EC_GF_OP_XOR2, 0, 3, 0}, {EC_GF_OP_XOR2, 3, 1, 0},
+    {EC_GF_OP_XOR2, 3, 5, 0}, {EC_GF_OP_XOR2, 5, 7, 0},
+    {EC_GF_OP_XOR2, 7, 1, 0}, {EC_GF_OP_XOR2, 1, 8, 0},
+    {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_46 = {9,
+                                    {
+                                        2,
+                                        0,
+                                        1,
+                                        3,
+                                        4,
+                                        5,
+                                        6,
+                                        7,
+                                        8,
+                                    },
+                                    ec_gf8_mul_46_ops};
+
+static ec_gf_op_t ec_gf8_mul_47_ops[] = {
+    {EC_GF_OP_XOR3, 8, 0, 1}, {EC_GF_OP_XOR2, 2, 0, 0},
+    {EC_GF_OP_XOR2, 3, 8, 0}, {EC_GF_OP_XOR2, 5, 1, 0},
+    {EC_GF_OP_XOR2, 4, 8, 0}, {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_47 = {9,
+                                    {
+                                        2,
+                                        3,
+                                        4,
+                                        5,
+                                        6,
+                                        7,
+                                        0,
+                                        1,
+                                        8,
+                                    },
+                                    ec_gf8_mul_47_ops};
+
+static ec_gf_op_t ec_gf8_mul_48_ops[] = {
+    {EC_GF_OP_XOR2, 5, 2, 0}, {EC_GF_OP_XOR2, 5, 4, 0},
+    {EC_GF_OP_XOR2, 6, 5, 0}, {EC_GF_OP_XOR2, 7, 6, 0},
+    {EC_GF_OP_XOR2, 2, 3, 0}, {EC_GF_OP_XOR2, 4, 7, 0},
+    {EC_GF_OP_XOR2, 3, 7, 0}, {EC_GF_OP_XOR2, 1, 3, 0},
+    {EC_GF_OP_XOR2, 0, 2, 0}, {EC_GF_OP_XOR2, 5, 3, 0},
+    {EC_GF_OP_XOR2, 2, 1, 0}, {EC_GF_OP_XOR2, 7, 0, 0},
+    {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_48 = {8,
+                                    {
+                                        4,
+                                        5,
+                                        6,
+                                        0,
+                                        1,
+                                        3,
+                                        7,
+                                        2,
+                                    },
+                                    ec_gf8_mul_48_ops};
+
+static ec_gf_op_t ec_gf8_mul_49_ops[] = {
+    {EC_GF_OP_XOR2, 3, 0, 0}, {EC_GF_OP_XOR2, 0, 2, 0},
+    {EC_GF_OP_XOR2, 6, 5, 0}, {EC_GF_OP_XOR3, 8, 0, 6},
+    {EC_GF_OP_XOR2, 3, 1, 0}, {EC_GF_OP_XOR2, 7, 8, 0},
+    {EC_GF_OP_XOR2, 1, 4, 0}, {EC_GF_OP_XOR2, 3, 7, 0},
+    {EC_GF_OP_XOR2, 4, 6, 0}, {EC_GF_OP_XOR2, 5, 3, 0},
+    {EC_GF_OP_XOR2, 6, 1, 0}, {EC_GF_OP_XOR2, 2, 5, 0},
+    {EC_GF_OP_XOR2, 5, 1, 0}, {EC_GF_OP_XOR3, 1, 8, 5},
+    {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_49 = {9,
+                                    {
+                                        7,
+                                        2,
+                                        4,
+                                        0,
+                                        3,
+                                        5,
+                                        1,
+                                        6,
+                                        8,
+                                    },
+                                    ec_gf8_mul_49_ops};
+
+static ec_gf_op_t ec_gf8_mul_4A_ops[] = {
+    {EC_GF_OP_XOR2, 2, 6, 0}, {EC_GF_OP_XOR2, 1, 4, 0},
+    {EC_GF_OP_XOR2, 3, 7, 0}, {EC_GF_OP_XOR2, 5, 2, 0},
+    {EC_GF_OP_XOR2, 1, 5, 0}, {EC_GF_OP_XOR2, 0, 3, 0},
+    {EC_GF_OP_XOR2, 7, 1, 0}, {EC_GF_OP_XOR2, 6, 0, 0},
+    {EC_GF_OP_XOR2, 3, 5, 0}, {EC_GF_OP_XOR2, 2, 7, 0},
+    {EC_GF_OP_XOR2, 4, 6, 0}, {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_4A = {8,
+                                    {
+                                        5,
+                                        6,
+                                        7,
+                                        0,
+                                        1,
+                                        3,
+                                        4,
+                                        2,
+                                    },
+                                    ec_gf8_mul_4A_ops};
+
+static ec_gf_op_t ec_gf8_mul_4B_ops[] = {
+    {EC_GF_OP_XOR2, 6, 7, 0}, {EC_GF_OP_XOR2, 7, 0, 0},
+    {EC_GF_OP_XOR2, 0, 1, 0}, {EC_GF_OP_XOR2, 1, 4, 0},
+    {EC_GF_OP_XOR3, 8, 3, 7}, {EC_GF_OP_XOR2, 3, 6, 0},
+    {EC_GF_OP_XOR2, 1, 5, 0}, {EC_GF_OP_XOR2, 4, 8, 0},
+    {EC_GF_OP_XOR2, 2, 3, 0}, {EC_GF_OP_XOR2, 5, 8, 0},
+    {EC_GF_OP_XOR2, 3, 0, 0}, {EC_GF_OP_XOR2, 6, 1, 0},
+    {EC_GF_OP_XOR2, 5, 2, 0}, {EC_GF_OP_XOR2, 0, 5, 0},
+    {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_4B = {9,
+                                    {
+                                        5,
+                                        3,
+                                        6,
+                                        7,
+                                        0,
+                                        2,
+                                        4,
+                                        1,
+                                        8,
+                                    },
+                                    ec_gf8_mul_4B_ops};
+
+static ec_gf_op_t ec_gf8_mul_4C_ops[] = {
+    {EC_GF_OP_XOR2, 5, 2, 0}, {EC_GF_OP_XOR2, 2, 0, 0},
+    {EC_GF_OP_XOR2, 3, 6, 0}, {EC_GF_OP_XOR2, 2, 3, 0},
+    {EC_GF_OP_XOR2, 4, 5, 0}, {EC_GF_OP_XOR2, 1, 2, 0},
+    {EC_GF_OP_XOR2, 5, 7, 0}, {EC_GF_OP_XOR2, 6, 4, 0},
+    {EC_GF_OP_XOR2, 7, 1, 0}, {EC_GF_OP_XOR2, 4, 0, 0},
+    {EC_GF_OP_XOR2, 1, 6, 0}, {EC_GF_OP_XOR2, 2, 5, 0},
+    {EC_GF_OP_XOR2, 0, 1, 0}, {EC_GF_OP_XOR2, 1, 2, 0},
+    {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_4C = {8,
+                                    {
+                                        5,
+                                        3,
+                                        4,
+                                        7,
+                                        0,
+                                        6,
+                                        2,
+                                        1,
+                                    },
+                                    ec_gf8_mul_4C_ops};
+
+static ec_gf_op_t ec_gf8_mul_4D_ops[] = {
+    {EC_GF_OP_COPY, 8, 3, 0}, {EC_GF_OP_XOR2, 3, 4, 0},
+    {EC_GF_OP_XOR2, 4, 6, 0}, {EC_GF_OP_XOR2, 0, 1, 0},
+    {EC_GF_OP_XOR2, 1, 4, 0}, {EC_GF_OP_XOR3, 9, 3, 1},
+    {EC_GF_OP_XOR2, 5, 9, 0}, {EC_GF_OP_XOR2, 4, 2, 0},
+    {EC_GF_OP_XOR2, 6, 5, 0}, {EC_GF_OP_XOR2, 0, 6, 0},
+    {EC_GF_OP_XOR2, 7, 0, 0}, {EC_GF_OP_XOR2, 3, 0, 0},
+    {EC_GF_OP_XOR2, 2, 7, 0}, {EC_GF_OP_XOR3, 0, 8, 2},
+    {EC_GF_OP_XOR2, 5, 2, 0}, {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_4D = {10,
+                                    {
+                                        0,
+                                        9,
+                                        3,
+                                        5,
+                                        6,
+                                        4,
+                                        7,
+                                        1,
+                                        2,
+                                        8,
+                                    },
+                                    ec_gf8_mul_4D_ops};
+
+static ec_gf_op_t ec_gf8_mul_4E_ops[] = {
+    {EC_GF_OP_XOR2, 3, 0, 0}, {EC_GF_OP_XOR2, 4, 1, 0},
+    {EC_GF_OP_XOR2, 0, 2, 0}, {EC_GF_OP_XOR2, 4, 7, 0},
+    {EC_GF_OP_XOR2, 2, 5, 0}, {EC_GF_OP_XOR2, 5, 4, 0},
+    {EC_GF_OP_XOR2, 3, 6, 0}, {EC_GF_OP_XOR2, 0, 5, 0},
+    {EC_GF_OP_XOR2, 6, 2, 0}, {EC_GF_OP_XOR2, 7, 3, 0},
+    {EC_GF_OP_XOR2, 1, 3, 0}, {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_4E = {8,
+                                    {
+                                        2,
+                                        3,
+                                        0,
+                                        1,
+                                        5,
+                                        6,
+                                        7,
+                                        4,
+                                    },
+                                    ec_gf8_mul_4E_ops};
+
+static ec_gf_op_t ec_gf8_mul_4F_ops[] = {
+    {EC_GF_OP_XOR2, 4, 1, 0}, {EC_GF_OP_XOR2, 1, 0, 0},
+    {EC_GF_OP_XOR2, 7, 0, 0}, {EC_GF_OP_XOR2, 0, 2, 0},
+    {EC_GF_OP_XOR2, 0, 5, 0}, {EC_GF_OP_XOR2, 2, 6, 0},
+    {EC_GF_OP_XOR2, 5, 7, 0}, {EC_GF_OP_XOR2, 6, 1, 0},
+    {EC_GF_OP_XOR2, 7, 3, 0}, {EC_GF_OP_XOR2, 1, 5, 0},
+    {EC_GF_OP_XOR2, 3, 6, 0}, {EC_GF_OP_XOR2, 5, 4, 0},
+    {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_4F = {8,
+                                    {
+                                        0,
+                                        3,
+                                        5,
+                                        6,
+                                        1,
+                                        2,
+                                        7,
+                                        4,
+                                    },
+                                    ec_gf8_mul_4F_ops};
+
+static ec_gf_op_t ec_gf8_mul_50_ops[] = {
+    {EC_GF_OP_XOR2, 5, 3, 0}, {EC_GF_OP_XOR2, 4, 6, 0},
+    {EC_GF_OP_XOR2, 6, 5, 0}, {EC_GF_OP_XOR2, 5, 7, 0},
+    {EC_GF_OP_XOR2, 7, 2, 0}, {EC_GF_OP_XOR2, 0, 2, 0},
+    {EC_GF_OP_XOR2, 4, 7, 0}, {EC_GF_OP_XOR2, 0, 6, 0},
+    {EC_GF_OP_XOR2, 3, 4, 0}, {EC_GF_OP_XOR2, 6, 1, 0},
+    {EC_GF_OP_XOR2, 2, 3, 0}, {EC_GF_OP_XOR2, 1, 2, 0},
+    {EC_GF_OP_XOR2, 2, 0, 0}, {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_50 = {8,
+                                    {
+                                        4,
+                                        5,
+                                        7,
+                                        3,
+                                        0,
+                                        1,
+                                        2,
+                                        6,
+                                    },
+                                    ec_gf8_mul_50_ops};
+
+static ec_gf_op_t ec_gf8_mul_51_ops[] = {
+    {EC_GF_OP_XOR2, 2, 1, 0}, {EC_GF_OP_XOR2, 3, 5, 0},
+    {EC_GF_OP_XOR2, 2, 3, 0}, {EC_GF_OP_XOR2, 3, 7, 0},
+    {EC_GF_OP_XOR2, 1, 3, 0}, {EC_GF_OP_XOR2, 6, 1, 0},
+    {EC_GF_OP_XOR2, 4, 6, 0}, {EC_GF_OP_XOR2, 2, 4, 0},
+    {EC_GF_OP_XOR2, 0, 2, 0}, {EC_GF_OP_XOR2, 5, 0, 0},
+    {EC_GF_OP_XOR2, 3, 0, 0}, {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_51 = {8,
+                                    {
+                                        0,
+                                        1,
+                                        7,
+                                        2,
+                                        3,
+                                        4,
+                                        5,
+                                        6,
+                                    },
+                                    ec_gf8_mul_51_ops};
+
+static ec_gf_op_t ec_gf8_mul_52_ops[] = {
+    {EC_GF_OP_XOR2, 0, 2, 0}, {EC_GF_OP_XOR2, 0, 7, 0},
+    {EC_GF_OP_COPY, 8, 0, 0}, {EC_GF_OP_XOR2, 0, 4, 0},
+    {EC_GF_OP_XOR2, 4, 6, 0}, {EC_GF_OP_COPY, 9, 4, 0},
+    {EC_GF_OP_XOR2, 4, 3, 0}, {EC_GF_OP_XOR2, 3, 1, 0},
+    {EC_GF_OP_XOR2, 5, 3, 0}, {EC_GF_OP_XOR2, 6, 3, 0},
+    {EC_GF_OP_XOR2, 1, 2, 0}, {EC_GF_OP_XOR3, 3, 5, 8},
+    {EC_GF_OP_XOR2, 7, 6, 0}, {EC_GF_OP_XOR2, 2, 9, 0},
+    {EC_GF_OP_XOR2, 6, 3, 0}, {EC_GF_OP_XOR2, 3, 1, 0},
+    {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_52 = {10,
+                                    {
+                                        2,
+                                        3,
+                                        1,
+                                        4,
+                                        6,
+                                        7,
+                                        0,
+                                        5,
+                                        8,
+                                        9,
+                                    },
+                                    ec_gf8_mul_52_ops};
+
+static ec_gf_op_t ec_gf8_mul_53_ops[] = {
+    {EC_GF_OP_XOR2, 2, 0, 0}, {EC_GF_OP_XOR2, 3, 1, 0},
+    {EC_GF_OP_XOR2, 4, 6, 0}, {EC_GF_OP_XOR2, 2, 4, 0},
+    {EC_GF_OP_XOR2, 5, 7, 0}, {EC_GF_OP_XOR2, 3, 5, 0},
+    {EC_GF_OP_XOR2, 7, 2, 0}, {EC_GF_OP_XOR2, 5, 2, 0},
+    {EC_GF_OP_XOR2, 6, 3, 0}, {EC_GF_OP_XOR2, 0, 3, 0},
+    {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_53 = {8,
+                                    {
+                                        2,
+                                        0,
+                                        1,
+                                        4,
+                                        5,
+                                        6,
+                                        7,
+                                        3,
+                                    },
+                                    ec_gf8_mul_53_ops};
+
+static ec_gf_op_t ec_gf8_mul_54_ops[] = {
+    {EC_GF_OP_XOR2, 7, 2, 0}, {EC_GF_OP_XOR2, 0, 7, 0},
+    {EC_GF_OP_XOR2, 7, 4, 0}, {EC_GF_OP_XOR2, 4, 1, 0},
+    {EC_GF_OP_XOR2, 2, 3, 0}, {EC_GF_OP_XOR2, 1, 3, 0},
+    {EC_GF_OP_XOR2, 3, 5, 0}, {EC_GF_OP_XOR2, 1, 6, 0},
+    {EC_GF_OP_XOR2, 5, 0, 0}, {EC_GF_OP_XOR2, 0, 6, 0},
+    {EC_GF_OP_XOR2, 6, 4, 0}, {EC_GF_OP_XOR2, 4, 2, 0},
+    {EC_GF_OP_XOR2, 2, 5, 0}, {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_54 = {8,
+                                    {
+                                        7,
+                                        3,
+                                        0,
+                                        4,
+                                        2,
+                                        6,
+                                        5,
+                                        1,
+                                    },
+                                    ec_gf8_mul_54_ops};
+
+static ec_gf_op_t ec_gf8_mul_55_ops[] = {
+    {EC_GF_OP_XOR2, 4, 2, 0}, {EC_GF_OP_XOR2, 7, 0, 0},
+    {EC_GF_OP_XOR2, 4, 1, 0}, {EC_GF_OP_XOR2, 3, 1, 0},
+    {EC_GF_OP_XOR2, 6, 7, 0}, {EC_GF_OP_XOR2, 2, 5, 0},
+    {EC_GF_OP_XOR2, 7, 4, 0}, {EC_GF_OP_XOR2, 5, 3, 0},
+    {EC_GF_OP_XOR2, 2, 6, 0}, {EC_GF_OP_XOR2, 1, 7, 0},
+    {EC_GF_OP_XOR2, 3, 6, 0}, {EC_GF_OP_XOR2, 7, 2, 0},
+    {EC_GF_OP_XOR2, 0, 3, 0}, {EC_GF_OP_XOR2, 3, 7, 0},
+    {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_55 = {8,
+                                    {
+                                        1,
+                                        5,
+                                        6,
+                                        4,
+                                        3,
+                                        7,
+                                        2,
+                                        0,
+                                    },
+                                    ec_gf8_mul_55_ops};
+
+static ec_gf_op_t ec_gf8_mul_56_ops[] = {
+    {EC_GF_OP_XOR2, 5, 0, 0}, {EC_GF_OP_XOR2, 6, 1, 0},
+    {EC_GF_OP_XOR2, 0, 2, 0}, {EC_GF_OP_XOR2, 1, 3, 0},
+    {EC_GF_OP_XOR2, 2, 4, 0}, {EC_GF_OP_XOR2, 3, 5, 0},
+    {EC_GF_OP_XOR2, 4, 1, 0}, {EC_GF_OP_XOR2, 4, 7, 0},
+    {EC_GF_OP_XOR2, 7, 0, 0}, {EC_GF_OP_XOR2, 0, 6, 0},
+    {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_56 = {8,
+                                    {
+                                        2,
+                                        3,
+                                        0,
+                                        4,
+                                        5,
+                                        6,
+                                        7,
+                                        1,
+                                    },
+                                    ec_gf8_mul_56_ops};
+
+static ec_gf_op_t ec_gf8_mul_57_ops[] = {
+    {EC_GF_OP_XOR2, 2, 0, 0}, {EC_GF_OP_XOR2, 5, 0, 0},
+    {EC_GF_OP_XOR2, 0, 1, 0}, {EC_GF_OP_XOR2, 0, 6, 0},
+    {EC_GF_OP_XOR2, 3, 1, 0}, {EC_GF_OP_XOR2, 6, 7, 0},
+    {EC_GF_OP_XOR2, 1, 4, 0}, {EC_GF_OP_XOR2, 6, 2, 0},
+    {EC_GF_OP_XOR2, 1, 7, 0}, {EC_GF_OP_XOR2, 2, 4, 0},
+    {EC_GF_OP_XOR2, 7, 3, 0}, {EC_GF_OP_XOR2, 4, 5, 0},
+    {EC_GF_OP_XOR2, 3, 5, 0}, {EC_GF_OP_XOR2, 5, 0, 0},
+    {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_57 = {8,
+                                    {
+                                        2,
+                                        3,
+                                        0,
+                                        1,
+                                        4,
+                                        5,
+                                        6,
+                                        7,
+                                    },
+                                    ec_gf8_mul_57_ops};
+
+static ec_gf_op_t ec_gf8_mul_58_ops[] = {
+    {EC_GF_OP_XOR2, 3, 2, 0}, {EC_GF_OP_XOR2, 2, 5, 0},
+    {EC_GF_OP_XOR2, 4, 3, 0}, {EC_GF_OP_XOR2, 5, 0, 0},
+    {EC_GF_OP_XOR2, 0, 1, 0}, {EC_GF_OP_XOR2, 5, 4, 0},
+    {EC_GF_OP_XOR2, 3, 7, 0}, {EC_GF_OP_XOR2, 1, 4, 0},
+    {EC_GF_OP_XOR2, 7, 5, 0}, {EC_GF_OP_XOR2, 0, 3, 0},
+    {EC_GF_OP_XOR2, 6, 2, 0}, {EC_GF_OP_XOR2, 3, 6, 0},
+    {EC_GF_OP_XOR2, 6, 1, 0}, {EC_GF_OP_XOR2, 4, 3, 0},
+    {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_58 = {8,
+                                    {
+                                        4,
+                                        3,
+                                        2,
+                                        7,
+                                        0,
+                                        1,
+                                        5,
+                                        6,
+                                    },
+                                    ec_gf8_mul_58_ops};
+
+static ec_gf_op_t ec_gf8_mul_59_ops[] = {
+    {EC_GF_OP_XOR2, 0, 1, 0}, {EC_GF_OP_XOR2, 1, 3, 0},
+    {EC_GF_OP_XOR2, 1, 5, 0}, {EC_GF_OP_XOR2, 2, 4, 0},
+    {EC_GF_OP_XOR2, 0, 6, 0}, {EC_GF_OP_XOR2, 2, 1, 0},
+    {EC_GF_OP_XOR2, 0, 2, 0}, {EC_GF_OP_XOR2, 3, 0, 0},
+    {EC_GF_OP_XOR2, 7, 3, 0}, {EC_GF_OP_XOR2, 6, 7, 0},
+    {EC_GF_OP_XOR2, 1, 6, 0}, {EC_GF_OP_XOR2, 3, 1, 0},
+    {EC_GF_OP_XOR2, 4, 3, 0}, {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_59 = {8,
+                                    {
+                                        7,
+                                        3,
+                                        5,
+                                        6,
+                                        1,
+                                        2,
+                                        0,
+                                        4,
+                                    },
+                                    ec_gf8_mul_59_ops};
+
+static ec_gf_op_t ec_gf8_mul_5A_ops[] = {
+    {EC_GF_OP_XOR2, 5, 2, 0}, {EC_GF_OP_XOR2, 2, 1, 0},
+    {EC_GF_OP_XOR2, 4, 5, 0}, {EC_GF_OP_XOR2, 3, 2, 0},
+    {EC_GF_OP_XOR2, 6, 4, 0}, {EC_GF_OP_XOR2, 4, 3, 0},
+    {EC_GF_OP_XOR2, 2, 0, 0}, {EC_GF_OP_XOR2, 0, 4, 0},
+    {EC_GF_OP_XOR2, 1, 0, 0}, {EC_GF_OP_XOR2, 5, 1, 0},
+    {EC_GF_OP_XOR2, 7, 5, 0}, {EC_GF_OP_XOR2, 0, 7, 0},
+    {EC_GF_OP_XOR2, 7, 6, 0}, {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_5A = {8,
+                                    {
+                                        6,
+                                        7,
+                                        0,
+                                        1,
+                                        2,
+                                        3,
+                                        5,
+                                        4,
+                                    },
+                                    ec_gf8_mul_5A_ops};
+
+static ec_gf_op_t ec_gf8_mul_5B_ops[] = {
+    {EC_GF_OP_XOR2, 0, 4, 0}, {EC_GF_OP_XOR2, 1, 5, 0},
+    {EC_GF_OP_XOR2, 4, 3, 0}, {EC_GF_OP_XOR2, 5, 0, 0},
+    {EC_GF_OP_XOR2, 3, 2, 0}, {EC_GF_OP_XOR2, 2, 5, 0},
+    {EC_GF_OP_XOR2, 0, 6, 0}, {EC_GF_OP_XOR2, 6, 2, 0},
+    {EC_GF_OP_XOR2, 7, 1, 0}, {EC_GF_OP_XOR2, 2, 1, 0},
+    {EC_GF_OP_XOR2, 1, 3, 0}, {EC_GF_OP_XOR2, 4, 7, 0},
+    {EC_GF_OP_XOR2, 3, 0, 0}, {EC_GF_OP_XOR2, 0, 4, 0},
+    {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_5B = {8,
+                                    {
+                                        6,
+                                        0,
+                                        7,
+                                        5,
+                                        2,
+                                        1,
+                                        3,
+                                        4,
+                                    },
+                                    ec_gf8_mul_5B_ops};
+
+static ec_gf_op_t ec_gf8_mul_5C_ops[] = {
+    {EC_GF_OP_COPY, 8, 3, 0}, {EC_GF_OP_XOR2, 3, 6, 0},
+    {EC_GF_OP_XOR2, 4, 1, 0}, {EC_GF_OP_XOR2, 5, 3, 0},
+    {EC_GF_OP_XOR2, 1, 0, 0}, {EC_GF_OP_XOR2, 0, 5, 0},
+    {EC_GF_OP_XOR2, 1, 3, 0}, {EC_GF_OP_XOR2, 2, 0, 0},
+    {EC_GF_OP_XOR2, 3, 4, 0}, {EC_GF_OP_XOR2, 4, 2, 0},
+    {EC_GF_OP_XOR2, 6, 2, 0}, {EC_GF_OP_XOR2, 7, 4, 0},
+    {EC_GF_OP_XOR2, 2, 8, 0}, {EC_GF_OP_XOR2, 0, 7, 0},
+    {EC_GF_OP_XOR2, 7, 1, 0}, {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_5C = {9,
+                                    {
+                                        7,
+                                        5,
+                                        2,
+                                        4,
+                                        1,
+                                        0,
+                                        6,
+                                        3,
+                                        8,
+                                    },
+                                    ec_gf8_mul_5C_ops};
+
+static ec_gf_op_t ec_gf8_mul_5D_ops[] = {
+    {EC_GF_OP_XOR2, 4, 1, 0}, {EC_GF_OP_XOR2, 1, 0, 0},
+    {EC_GF_OP_XOR2, 6, 0, 0}, {EC_GF_OP_XOR2, 4, 3, 0},
+    {EC_GF_OP_XOR2, 5, 6, 0}, {EC_GF_OP_XOR2, 6, 4, 0},
+    {EC_GF_OP_XOR2, 3, 5, 0}, {EC_GF_OP_XOR2, 7, 6, 0},
+    {EC_GF_OP_XOR2, 2, 3, 0}, {EC_GF_OP_XOR2, 0, 7, 0},
+    {EC_GF_OP_XOR2, 3, 1, 0}, {EC_GF_OP_XOR2, 7, 2, 0},
+    {EC_GF_OP_XOR2, 4, 2, 0}, {EC_GF_OP_XOR2, 1, 7, 0},
+    {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_5D = {8,
+                                    {
+                                        1,
+                                        3,
+                                        5,
+                                        4,
+                                        6,
+                                        7,
+                                        2,
+                                        0,
+                                    },
+                                    ec_gf8_mul_5D_ops};
+
+static ec_gf_op_t ec_gf8_mul_5E_ops[] = {
+    {EC_GF_OP_XOR2, 6, 0, 0}, {EC_GF_OP_XOR2, 6, 5, 0},
+    {EC_GF_OP_XOR2, 1, 2, 0}, {EC_GF_OP_XOR2, 5, 2, 0},
+    {EC_GF_OP_XOR2, 2, 3, 0}, {EC_GF_OP_XOR2, 7, 1, 0},
+    {EC_GF_OP_XOR2, 3, 6, 0}, {EC_GF_OP_XOR2, 0, 2, 0},
+    {EC_GF_OP_XOR2, 6, 7, 0}, {EC_GF_OP_XOR2, 2, 4, 0},
+    {EC_GF_OP_XOR2, 4, 5, 0}, {EC_GF_OP_XOR2, 1, 2, 0},
+    {EC_GF_OP_XOR2, 5, 6, 0}, {EC_GF_OP_XOR2, 2, 6, 0},
+    {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_5E = {8,
+                                    {
+                                        4,
+                                        3,
+                                        6,
+                                        2,
+                                        5,
+                                        7,
+                                        0,
+                                        1,
+                                    },
+                                    ec_gf8_mul_5E_ops};
+
+static ec_gf_op_t ec_gf8_mul_5F_ops[] = {
+    {EC_GF_OP_XOR2, 0, 3, 0}, {EC_GF_OP_XOR2, 1, 5, 0},
+    {EC_GF_OP_XOR2, 0, 6, 0}, {EC_GF_OP_XOR2, 1, 0, 0},
+    {EC_GF_OP_XOR2, 7, 1, 0}, {EC_GF_OP_XOR2, 3, 7, 0},
+    {EC_GF_OP_XOR2, 2, 0, 0}, {EC_GF_OP_XOR2, 4, 3, 0},
+    {EC_GF_OP_XOR2, 7, 2, 0}, {EC_GF_OP_XOR2, 5, 4, 0},
+    {EC_GF_OP_XOR2, 6, 5, 0}, {EC_GF_OP_XOR2, 0, 5, 0},
+    {EC_GF_OP_XOR2, 6, 7, 0}, {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_5F = {8,
+                                    {
+                                        6,
+                                        1,
+                                        3,
+                                        4,
+                                        5,
+                                        7,
+                                        2,
+                                        0,
+                                    },
+                                    ec_gf8_mul_5F_ops};
+
+static ec_gf_op_t ec_gf8_mul_60_ops[] = {
+    {EC_GF_OP_XOR2, 5, 2, 0}, {EC_GF_OP_XOR2, 7, 4, 0},
+    {EC_GF_OP_XOR2, 4, 5, 0}, {EC_GF_OP_XOR2, 6, 3, 0},
+    {EC_GF_OP_XOR2, 2, 6, 0}, {EC_GF_OP_XOR2, 4, 6, 0},
+    {EC_GF_OP_XOR2, 3, 7, 0}, {EC_GF_OP_XOR2, 6, 0, 0},
+    {EC_GF_OP_XOR2, 0, 1, 0}, {EC_GF_OP_XOR2, 0, 7, 0},
+    {EC_GF_OP_XOR2, 1, 5, 0}, {EC_GF_OP_XOR2, 7, 5, 0},
+    {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_60 = {8,
+                                    {
+                                        2,
+                                        3,
+                                        4,
+                                        7,
+                                        5,
+                                        6,
+                                        0,
+                                        1,
+                                    },
+                                    ec_gf8_mul_60_ops};
+
+static ec_gf_op_t ec_gf8_mul_61_ops[] = {
+    {EC_GF_OP_XOR2, 1, 2, 0}, {EC_GF_OP_XOR2, 6, 2, 0},
+    {EC_GF_OP_XOR2, 2, 5, 0}, {EC_GF_OP_XOR2, 4, 2, 0},
+    {EC_GF_OP_XOR2, 0, 3, 0}, {EC_GF_OP_XOR2, 3, 4, 0},
+    {EC_GF_OP_XOR2, 0, 6, 0}, {EC_GF_OP_XOR2, 7, 3, 0},
+    {EC_GF_OP_XOR2, 2, 0, 0}, {EC_GF_OP_XOR2, 6, 3, 0},
+    {EC_GF_OP_XOR2, 1, 7, 0}, {EC_GF_OP_XOR2, 5, 1, 0},
+    {EC_GF_OP_XOR2, 1, 2, 0}, {EC_GF_OP_XOR2, 3, 5, 0},
+    {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_61 = {8,
+                                    {
+                                        0,
+                                        5,
+                                        6,
+                                        7,
+                                        4,
+                                        2,
+                                        1,
+                                        3,
+                                    },
+                                    ec_gf8_mul_61_ops};
+
+static ec_gf_op_t ec_gf8_mul_62_ops[] = {
+    {EC_GF_OP_XOR2, 3, 7, 0}, {EC_GF_OP_XOR2, 1, 2, 0},
+    {EC_GF_OP_XOR2, 2, 3, 0}, {EC_GF_OP_XOR2, 3, 4, 0},
+    {EC_GF_OP_XOR2, 4, 5, 0}, {EC_GF_OP_XOR2, 0, 3, 0},
+    {EC_GF_OP_XOR2, 5, 2, 0}, {EC_GF_OP_XOR2, 7, 0, 0},
+    {EC_GF_OP_XOR2, 2, 6, 0}, {EC_GF_OP_XOR2, 1, 5, 0},
+    {EC_GF_OP_XOR2, 6, 7, 0}, {EC_GF_OP_XOR2, 7, 1, 0},
+    {EC_GF_OP_XOR2, 1, 2, 0}, {EC_GF_OP_XOR2, 3, 1, 0},
+    {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_62 = {8,
+                                    {
+                                        2,
+                                        0,
+                                        3,
+                                        4,
+                                        5,
+                                        6,
+                                        7,
+                                        1,
+                                    },
+                                    ec_gf8_mul_62_ops};
+
+static ec_gf_op_t ec_gf8_mul_63_ops[] = {
+    {EC_GF_OP_XOR2, 4, 3, 0}, {EC_GF_OP_XOR2, 5, 4, 0},
+    {EC_GF_OP_XOR2, 6, 5, 0}, {EC_GF_OP_XOR2, 1, 7, 0},
+    {EC_GF_OP_XOR2, 0, 6, 0}, {EC_GF_OP_XOR2, 6, 1, 0},
+    {EC_GF_OP_XOR2, 7, 2, 0}, {EC_GF_OP_XOR2, 3, 0, 0},
+    {EC_GF_OP_XOR2, 4, 6, 0}, {EC_GF_OP_XOR2, 7, 5, 0},
+    {EC_GF_OP_XOR2, 1, 3, 0}, {EC_GF_OP_XOR2, 2, 4, 0},
+    {EC_GF_OP_XOR2, 3, 7, 0}, {EC_GF_OP_XOR2, 4, 0, 0},
+    {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_63 = {8,
+                                    {
+                                        3,
+                                        4,
+                                        6,
+                                        5,
+                                        7,
+                                        0,
+                                        1,
+                                        2,
+                                    },
+                                    ec_gf8_mul_63_ops};
+
+static ec_gf_op_t ec_gf8_mul_64_ops[] = {
+    {EC_GF_OP_COPY, 8, 1, 0}, {EC_GF_OP_XOR2, 8, 0, 0},
+    {EC_GF_OP_XOR2, 1, 2, 0}, {EC_GF_OP_XOR2, 8, 7, 0},
+    {EC_GF_OP_XOR2, 2, 3, 0}, {EC_GF_OP_XOR2, 3, 4, 0},
+    {EC_GF_OP_XOR2, 7, 6, 0}, {EC_GF_OP_XOR2, 4, 5, 0},
+    {EC_GF_OP_XOR2, 6, 1, 0}, {EC_GF_OP_XOR2, 5, 7, 0},
+    {EC_GF_OP_XOR2, 6, 4, 0}, {EC_GF_OP_XOR2, 7, 0, 0},
+    {EC_GF_OP_XOR2, 4, 2, 0}, {EC_GF_OP_XOR2, 4, 0, 0},
+    {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_64 = {9,
+                                    {
+                                        2,
+                                        3,
+                                        4,
+                                        6,
+                                        5,
+                                        7,
+                                        8,
+                                        1,
+                                        0,
+                                    },
+                                    ec_gf8_mul_64_ops};
+
+static ec_gf_op_t ec_gf8_mul_65_ops[] = {
+    {EC_GF_OP_XOR2, 6, 7, 0}, {EC_GF_OP_XOR2, 7, 2, 0},
+    {EC_GF_OP_XOR2, 5, 6, 0}, {EC_GF_OP_XOR2, 2, 0, 0},
+    {EC_GF_OP_XOR2, 4, 5, 0}, {EC_GF_OP_XOR2, 2, 3, 0},
+    {EC_GF_OP_XOR2, 6, 1, 0}, {EC_GF_OP_XOR2, 3, 4, 0},
+    {EC_GF_OP_XOR2, 7, 1, 0}, {EC_GF_OP_XOR2, 6, 0, 0},
+    {EC_GF_OP_XOR2, 1, 3, 0}, {EC_GF_OP_XOR2, 0, 5, 0},
+    {EC_GF_OP_XOR2, 3, 7, 0}, {EC_GF_OP_XOR2, 5, 1, 0},
+    {EC_GF_OP_XOR2, 1, 6, 0}, {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_65 = {8,
+                                    {
+                                        2,
+                                        5,
+                                        1,
+                                        3,
+                                        4,
+                                        0,
+                                        6,
+                                        7,
+                                    },
+                                    ec_gf8_mul_65_ops};
+
+static ec_gf_op_t ec_gf8_mul_66_ops[] = {
+    {EC_GF_OP_XOR2, 4, 0, 0}, {EC_GF_OP_XOR2, 0, 1, 0},
+    {EC_GF_OP_XOR2, 1, 2, 0}, {EC_GF_OP_XOR2, 2, 3, 0},
+    {EC_GF_OP_XOR2, 3, 4, 0}, {EC_GF_OP_XOR2, 5, 7, 0},
+    {EC_GF_OP_XOR2, 2, 7, 0}, {EC_GF_OP_XOR2, 0, 5, 0},
+    {EC_GF_OP_XOR2, 4, 6, 0}, {EC_GF_OP_XOR2, 5, 3, 0},
+    {EC_GF_OP_XOR2, 6, 1, 0}, {EC_GF_OP_XOR2, 7, 4, 0},
+    {EC_GF_OP_XOR2, 1, 5, 0}, {EC_GF_OP_XOR2, 4, 0, 0},
+    {EC_GF_OP_XOR2, 5, 7, 0}, {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_66 = {8,
+                                    {
+                                        2,
+                                        3,
+                                        1,
+                                        4,
+                                        5,
+                                        7,
+                                        0,
+                                        6,
+                                    },
+                                    ec_gf8_mul_66_ops};
+
+static ec_gf_op_t ec_gf8_mul_67_ops[] = {
+    {EC_GF_OP_XOR2, 3, 0, 0}, {EC_GF_OP_XOR2, 3, 1, 0},
+    {EC_GF_OP_XOR2, 1, 4, 0}, {EC_GF_OP_XOR2, 4, 3, 0},
+    {EC_GF_OP_XOR2, 7, 4, 0}, {EC_GF_OP_XOR2, 5, 7, 0},
+    {EC_GF_OP_XOR2, 0, 5, 0}, {EC_GF_OP_XOR2, 6, 0, 0},
+    {EC_GF_OP_XOR2, 3, 6, 0}, {EC_GF_OP_XOR2, 1, 3, 0},
+    {EC_GF_OP_XOR2, 7, 1, 0}, {EC_GF_OP_XOR2, 2, 7, 0},
+    {EC_GF_OP_XOR2, 0, 2, 0}, {EC_GF_OP_XOR2, 2, 3, 0},
+    {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_67 = {8,
+                                    {
+                                        2,
+                                        4,
+                                        5,
+                                        6,
+                                        7,
+                                        3,
+                                        1,
+                                        0,
+                                    },
+                                    ec_gf8_mul_67_ops};
+
+static ec_gf_op_t ec_gf8_mul_68_ops[] = {
+    {EC_GF_OP_XOR2, 5, 2, 0}, {EC_GF_OP_XOR2, 5, 4, 0},
+    {EC_GF_OP_XOR2, 4, 3, 0}, {EC_GF_OP_XOR2, 6, 4, 0},
+    {EC_GF_OP_XOR2, 7, 6, 0}, {EC_GF_OP_XOR2, 4, 0, 0},
+    {EC_GF_OP_XOR2, 2, 7, 0}, {EC_GF_OP_XOR2, 4, 1, 0},
+    {EC_GF_OP_XOR2, 0, 2, 0}, {EC_GF_OP_XOR2, 1, 5, 0},
+    {EC_GF_OP_XOR2, 3, 0, 0}, {EC_GF_OP_XOR2, 5, 6, 0},
+    {EC_GF_OP_XOR2, 0, 4, 0}, {EC_GF_OP_XOR2, 6, 3, 0},
+    {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_68 = {8,
+                                    {
+                                        5,
+                                        7,
+                                        2,
+                                        3,
+                                        0,
+                                        6,
+                                        4,
+                                        1,
+                                    },
+                                    ec_gf8_mul_68_ops};
+
+static ec_gf_op_t ec_gf8_mul_69_ops[] = {
+    {EC_GF_OP_XOR2, 4, 6, 0}, {EC_GF_OP_XOR2, 4, 7, 0},
+    {EC_GF_OP_XOR2, 3, 4, 0}, {EC_GF_OP_XOR2, 2, 1, 0},
+    {EC_GF_OP_XOR2, 1, 3, 0}, {EC_GF_OP_XOR2, 4, 2, 0},
+    {EC_GF_OP_XOR2, 0, 1, 0}, {EC_GF_OP_XOR2, 5, 4, 0},
+    {EC_GF_OP_XOR2, 7, 0, 0}, {EC_GF_OP_XOR2, 6, 5, 0},
+    {EC_GF_OP_XOR2, 2, 0, 0}, {EC_GF_OP_XOR2, 5, 7, 0},
+    {EC_GF_OP_XOR2, 0, 6, 0}, {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_69 = {8,
+                                    {
+                                        0,
+                                        1,
+                                        3,
+                                        2,
+                                        4,
+                                        5,
+                                        7,
+                                        6,
+                                    },
+                                    ec_gf8_mul_69_ops};
+
+static ec_gf_op_t ec_gf8_mul_6A_ops[] = {
+    {EC_GF_OP_XOR2, 7, 3, 0}, {EC_GF_OP_XOR2, 3, 5, 0},
+    {EC_GF_OP_XOR2, 0, 4, 0}, {EC_GF_OP_XOR2, 2, 6, 0},
+    {EC_GF_OP_XOR2, 6, 0, 0}, {EC_GF_OP_XOR2, 5, 2, 0},
+    {EC_GF_OP_XOR2, 1, 3, 0}, {EC_GF_OP_XOR2, 0, 1, 0},
+    {EC_GF_OP_XOR2, 1, 5, 0}, {EC_GF_OP_XOR2, 5, 7, 0},
+    {EC_GF_OP_XOR2, 4, 1, 0}, {EC_GF_OP_XOR2, 7, 6, 0},
+    {EC_GF_OP_XOR2, 3, 4, 0}, {EC_GF_OP_XOR2, 2, 7, 0},
+    {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_6A = {8,
+                                    {
+                                        5,
+                                        7,
+                                        4,
+                                        6,
+                                        1,
+                                        2,
+                                        0,
+                                        3,
+                                    },
+                                    ec_gf8_mul_6A_ops};
+
+static ec_gf_op_t ec_gf8_mul_6B_ops[] = {
+    {EC_GF_OP_COPY, 8, 1, 0}, {EC_GF_OP_XOR2, 1, 2, 0},
+    {EC_GF_OP_XOR2, 1, 6, 0}, {EC_GF_OP_XOR2, 3, 4, 0},
+    {EC_GF_OP_XOR2, 3, 1, 0}, {EC_GF_OP_XOR2, 2, 3, 0},
+    {EC_GF_OP_XOR2, 0, 2, 0}, {EC_GF_OP_XOR2, 1, 5, 0},
+    {EC_GF_OP_XOR2, 7, 0, 0}, {EC_GF_OP_XOR2, 5, 0, 0},
+    {EC_GF_OP_XOR2, 1, 7, 0}, {EC_GF_OP_XOR2, 4, 1, 0},
+    {EC_GF_OP_XOR2, 6, 4, 0}, {EC_GF_OP_XOR2, 4, 0, 0},
+    {EC_GF_OP_XOR2, 0, 8, 0}, {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_6B = {9,
+                                    {
+                                        6,
+                                        7,
+                                        2,
+                                        0,
+                                        3,
+                                        1,
+                                        5,
+                                        4,
+                                        8,
+                                    },
+                                    ec_gf8_mul_6B_ops};
+
+static ec_gf_op_t ec_gf8_mul_6C_ops[] = {
+    {EC_GF_OP_XOR2, 5, 2, 0}, {EC_GF_OP_XOR2, 4, 3, 0},
+    {EC_GF_OP_XOR2, 5, 3, 0}, {EC_GF_OP_XOR2, 2, 0, 0},
+    {EC_GF_OP_XOR2, 6, 4, 0}, {EC_GF_OP_XOR2, 4, 2, 0},
+    {EC_GF_OP_XOR2, 3, 1, 0}, {EC_GF_OP_XOR2, 3, 0, 0},
+    {EC_GF_OP_XOR2, 7, 4, 0}, {EC_GF_OP_XOR2, 4, 3, 0},
+    {EC_GF_OP_XOR2, 0, 4, 0}, {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_6C = {8,
+                                    {
+                                        5,
+                                        6,
+                                        7,
+                                        0,
+                                        1,
+                                        2,
+                                        3,
+                                        4,
+                                    },
+                                    ec_gf8_mul_6C_ops};
+
+static ec_gf_op_t ec_gf8_mul_6D_ops[] = {
+    {EC_GF_OP_XOR2, 4, 1, 0}, {EC_GF_OP_XOR2, 1, 0, 0},
+    {EC_GF_OP_XOR2, 7, 4, 0}, {EC_GF_OP_XOR2, 0, 2, 0},
+    {EC_GF_OP_XOR2, 1, 3, 0}, {EC_GF_OP_XOR2, 2, 7, 0},
+    {EC_GF_OP_XOR3, 8, 3, 4}, {EC_GF_OP_XOR2, 7, 1, 0},
+    {EC_GF_OP_XOR2, 5, 0, 0}, {EC_GF_OP_XOR2, 1, 6, 0},
+    {EC_GF_OP_XOR2, 0, 8, 0}, {EC_GF_OP_XOR2, 3, 5, 0},
+    {EC_GF_OP_XOR2, 6, 8, 0}, {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_6D = {9,
+                                    {
+                                        3,
+                                        6,
+                                        7,
+                                        0,
+                                        4,
+                                        5,
+                                        1,
+                                        2,
+                                        8,
+                                    },
+                                    ec_gf8_mul_6D_ops};
+
+static ec_gf_op_t ec_gf8_mul_6E_ops[] = {
+    {EC_GF_OP_XOR2, 3, 1, 0}, {EC_GF_OP_XOR2, 1, 2, 0},
+    {EC_GF_OP_XOR2, 2, 0, 0}, {EC_GF_OP_XOR2, 7, 3, 0},
+    {EC_GF_OP_XOR2, 0, 5, 0}, {EC_GF_OP_XOR2, 6, 1, 0},
+    {EC_GF_OP_XOR2, 0, 3, 0}, {EC_GF_OP_XOR2, 2, 4, 0},
+    {EC_GF_OP_XOR2, 1, 7, 0}, {EC_GF_OP_XOR2, 4, 6, 0},
+    {EC_GF_OP_XOR2, 3, 2, 0}, {EC_GF_OP_XOR2, 5, 1, 0},
+    {EC_GF_OP_XOR2, 6, 3, 0}, {EC_GF_OP_XOR2, 1, 3, 0},
+    {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_6E = {8,
+                                    {
+                                        5,
+                                        6,
+                                        3,
+                                        1,
+                                        7,
+                                        2,
+                                        0,
+                                        4,
+                                    },
+                                    ec_gf8_mul_6E_ops};
+
+static ec_gf_op_t ec_gf8_mul_6F_ops[] = {
+    {EC_GF_OP_COPY, 8, 0, 0}, {EC_GF_OP_XOR2, 0, 1, 0},
+    {EC_GF_OP_XOR2, 0, 4, 0}, {EC_GF_OP_XOR2, 3, 0, 0},
+    {EC_GF_OP_XOR2, 2, 5, 0}, {EC_GF_OP_XOR2, 7, 3, 0},
+    {EC_GF_OP_XOR2, 2, 0, 0}, {EC_GF_OP_XOR2, 6, 3, 0},
+    {EC_GF_OP_XOR3, 0, 8, 7}, {EC_GF_OP_XOR2, 1, 2, 0},
+    {EC_GF_OP_XOR2, 5, 6, 0}, {EC_GF_OP_XOR2, 2, 0, 0},
+    {EC_GF_OP_XOR2, 4, 5, 0}, {EC_GF_OP_XOR2, 5, 2, 0},
+    {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_6F = {9,
+                                    {
+                                        2,
+                                        6,
+                                        3,
+                                        7,
+                                        0,
+                                        1,
+                                        4,
+                                        5,
+                                        8,
+                                    },
+                                    ec_gf8_mul_6F_ops};
+
+static ec_gf_op_t ec_gf8_mul_70_ops[] = {
+    {EC_GF_OP_XOR2, 3, 2, 0}, {EC_GF_OP_XOR2, 5, 3, 0},
+    {EC_GF_OP_XOR2, 6, 4, 0}, {EC_GF_OP_XOR2, 4, 2, 0},
+    {EC_GF_OP_XOR2, 7, 5, 0}, {EC_GF_OP_XOR2, 0, 2, 0},
+    {EC_GF_OP_XOR2, 3, 6, 0}, {EC_GF_OP_XOR2, 4, 7, 0},
+    {EC_GF_OP_XOR2, 6, 0, 0}, {EC_GF_OP_XOR2, 7, 1, 0},
+    {EC_GF_OP_XOR2, 1, 6, 0}, {EC_GF_OP_XOR2, 0, 7, 0},
+    {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_70 = {8,
+                                    {
+                                        3,
+                                        4,
+                                        5,
+                                        2,
+                                        6,
+                                        0,
+                                        1,
+                                        7,
+                                    },
+                                    ec_gf8_mul_70_ops};
+
+static ec_gf_op_t ec_gf8_mul_71_ops[] = {
+    {EC_GF_OP_XOR2, 6, 0, 0}, {EC_GF_OP_XOR2, 0, 1, 0},
+    {EC_GF_OP_XOR2, 1, 5, 0}, {EC_GF_OP_XOR2, 0, 4, 0},
+    {EC_GF_OP_XOR2, 5, 3, 0}, {EC_GF_OP_XOR2, 4, 3, 0},
+    {EC_GF_OP_XOR2, 6, 2, 0}, {EC_GF_OP_XOR2, 3, 2, 0},
+    {EC_GF_OP_XOR2, 7, 4, 0}, {EC_GF_OP_XOR2, 2, 0, 0},
+    {EC_GF_OP_XOR2, 4, 6, 0}, {EC_GF_OP_XOR2, 0, 7, 0},
+    {EC_GF_OP_XOR2, 7, 1, 0}, {EC_GF_OP_XOR2, 1, 3, 0},
+    {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_71 = {8,
+                                    {
+                                        4,
+                                        7,
+                                        5,
+                                        3,
+                                        6,
+                                        0,
+                                        2,
+                                        1,
+                                    },
+                                    ec_gf8_mul_71_ops};
+
+static ec_gf_op_t ec_gf8_mul_72_ops[] = {
+    {EC_GF_OP_XOR2, 4, 0, 0}, {EC_GF_OP_XOR2, 3, 7, 0},
+    {EC_GF_OP_XOR2, 3, 4, 0}, {EC_GF_OP_XOR2, 5, 3, 0},
+    {EC_GF_OP_XOR2, 1, 5, 0}, {EC_GF_OP_XOR2, 4, 1, 0},
+    {EC_GF_OP_XOR2, 2, 4, 0}, {EC_GF_OP_XOR2, 6, 2, 0},
+    {EC_GF_OP_XOR2, 3, 6, 0}, {EC_GF_OP_XOR2, 4, 3, 0},
+    {EC_GF_OP_XOR2, 0, 4, 0}, {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_72 = {8,
+                                    {
+                                        0,
+                                        5,
+                                        2,
+                                        7,
+                                        4,
+                                        1,
+                                        3,
+                                        6,
+                                    },
+                                    ec_gf8_mul_72_ops};
+
+static ec_gf_op_t ec_gf8_mul_73_ops[] = {
+    {EC_GF_OP_XOR2, 1, 5, 0}, {EC_GF_OP_XOR2, 2, 1, 0},
+    {EC_GF_OP_XOR2, 7, 3, 0}, {EC_GF_OP_XOR2, 1, 7, 0},
+    {EC_GF_OP_XOR2, 0, 4, 0}, {EC_GF_OP_XOR2, 6, 2, 0},
+    {EC_GF_OP_XOR2, 2, 0, 0}, {EC_GF_OP_XOR2, 3, 6, 0},
+    {EC_GF_OP_XOR2, 0, 1, 0}, {EC_GF_OP_XOR2, 6, 0, 0},
+    {EC_GF_OP_XOR2, 5, 0, 0}, {EC_GF_OP_XOR2, 4, 6, 0},
+    {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_73 = {8,
+                                    {
+                                        6,
+                                        0,
+                                        1,
+                                        7,
+                                        4,
+                                        5,
+                                        2,
+                                        3,
+                                    },
+                                    ec_gf8_mul_73_ops};
+
+static ec_gf_op_t ec_gf8_mul_74_ops[] = {
+    {EC_GF_OP_XOR2, 3, 2, 0}, {EC_GF_OP_XOR2, 2, 5, 0},
+    {EC_GF_OP_XOR2, 5, 0, 0}, {EC_GF_OP_XOR2, 5, 1, 0},
+    {EC_GF_OP_XOR2, 1, 3, 0}, {EC_GF_OP_XOR2, 0, 7, 0},
+    {EC_GF_OP_XOR2, 6, 5, 0}, {EC_GF_OP_XOR2, 7, 1, 0},
+    {EC_GF_OP_XOR2, 1, 6, 0}, {EC_GF_OP_XOR2, 3, 4, 0},
+    {EC_GF_OP_XOR2, 6, 2, 0}, {EC_GF_OP_XOR2, 4, 0, 0},
+    {EC_GF_OP_XOR2, 2, 3, 0}, {EC_GF_OP_XOR2, 0, 6, 0},
+    {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_74 = {8,
+                                    {
+                                        3,
+                                        2,
+                                        1,
+                                        0,
+                                        4,
+                                        5,
+                                        6,
+                                        7,
+                                    },
+                                    ec_gf8_mul_74_ops};
+
+static ec_gf_op_t ec_gf8_mul_75_ops[] = {
+    {EC_GF_OP_XOR2, 2, 0, 0}, {EC_GF_OP_XOR2, 3, 2, 0},
+    {EC_GF_OP_XOR2, 2, 1, 0}, {EC_GF_OP_XOR2, 1, 0, 0},
+    {EC_GF_OP_XOR2, 4, 3, 0}, {EC_GF_OP_XOR2, 3, 1, 0},
+    {EC_GF_OP_XOR2, 0, 7, 0}, {EC_GF_OP_XOR2, 6, 3, 0},
+    {EC_GF_OP_XOR2, 5, 2, 0}, {EC_GF_OP_XOR2, 7, 6, 0},
+    {EC_GF_OP_XOR2, 6, 5, 0}, {EC_GF_OP_XOR2, 5, 4, 0},
+    {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_75 = {8,
+                                    {
+                                        4,
+                                        5,
+                                        6,
+                                        7,
+                                        0,
+                                        1,
+                                        2,
+                                        3,
+                                    },
+                                    ec_gf8_mul_75_ops};
+
+static ec_gf_op_t ec_gf8_mul_76_ops[] = {
+    {EC_GF_OP_XOR2, 7, 3, 0}, {EC_GF_OP_XOR2, 6, 1, 0},
+    {EC_GF_OP_XOR2, 2, 7, 0}, {EC_GF_OP_XOR3, 8, 6, 2},
+    {EC_GF_OP_XOR2, 0, 5, 0}, {EC_GF_OP_XOR2, 2, 4, 0},
+    {EC_GF_OP_XOR2, 4, 0, 0}, {EC_GF_OP_XOR2, 0, 8, 0},
+    {EC_GF_OP_XOR2, 3, 4, 0}, {EC_GF_OP_XOR2, 1, 4, 0},
+    {EC_GF_OP_XOR2, 7, 0, 0}, {EC_GF_OP_XOR2, 5, 3, 0},
+    {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_76 = {9,
+                                    {
+                                        2,
+                                        3,
+                                        0,
+                                        6,
+                                        5,
+                                        1,
+                                        7,
+                                        8,
+                                        4,
+                                    },
+                                    ec_gf8_mul_76_ops};
+
+static ec_gf_op_t ec_gf8_mul_77_ops[] = {
+    {EC_GF_OP_XOR2, 7, 6, 0}, {EC_GF_OP_XOR2, 6, 1, 0},
+    {EC_GF_OP_XOR2, 1, 0, 0}, {EC_GF_OP_XOR2, 5, 1, 0},
+    {EC_GF_OP_XOR2, 6, 3, 0}, {EC_GF_OP_XOR2, 0, 3, 0},
+    {EC_GF_OP_XOR2, 1, 4, 0}, {EC_GF_OP_XOR2, 3, 5, 0},
+    {EC_GF_OP_XOR2, 4, 3, 0}, {EC_GF_OP_XOR2, 5, 2, 0},
+    {EC_GF_OP_XOR2, 3, 7, 0}, {EC_GF_OP_XOR2, 2, 6, 0},
+    {EC_GF_OP_XOR2, 7, 1, 0}, {EC_GF_OP_XOR2, 7, 2, 0},
+    {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_77 = {8,
+                                    {
+                                        7,
+                                        4,
+                                        3,
+                                        6,
+                                        0,
+                                        1,
+                                        5,
+                                        2,
+                                    },
+                                    ec_gf8_mul_77_ops};
+
+static ec_gf_op_t ec_gf8_mul_78_ops[] = {
+    {EC_GF_OP_XOR2, 6, 5, 0}, {EC_GF_OP_XOR2, 6, 0, 0},
+    {EC_GF_OP_XOR2, 1, 0, 0}, {EC_GF_OP_XOR2, 7, 2, 0},
+    {EC_GF_OP_XOR2, 2, 6, 0}, {EC_GF_OP_XOR2, 0, 3, 0},
+    {EC_GF_OP_XOR2, 3, 7, 0}, {EC_GF_OP_XOR3, 8, 0, 2},
+    {EC_GF_OP_XOR2, 4, 8, 0}, {EC_GF_OP_XOR2, 1, 8, 0},
+    {EC_GF_OP_XOR2, 7, 4, 0}, {EC_GF_OP_XOR2, 5, 7, 0},
+    {EC_GF_OP_XOR2, 5, 1, 0}, {EC_GF_OP_XOR2, 0, 5, 0},
+    {EC_GF_OP_XOR2, 6, 0, 0}, {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_78 = {9,
+                                    {
+                                        4,
+                                        7,
+                                        3,
+                                        2,
+                                        5,
+                                        1,
+                                        6,
+                                        0,
+                                        8,
+                                    },
+                                    ec_gf8_mul_78_ops};
+
+static ec_gf_op_t ec_gf8_mul_79_ops[] = {
+    {EC_GF_OP_XOR2, 7, 3, 0}, {EC_GF_OP_XOR3, 8, 4, 7},
+    {EC_GF_OP_XOR2, 0, 8, 0}, {EC_GF_OP_XOR2, 2, 1, 0},
+    {EC_GF_OP_XOR2, 6, 8, 0}, {EC_GF_OP_XOR2, 2, 0, 0},
+    {EC_GF_OP_XOR2, 1, 6, 0}, {EC_GF_OP_XOR2, 3, 2, 0},
+    {EC_GF_OP_XOR2, 5, 1, 0}, {EC_GF_OP_XOR2, 6, 3, 0},
+    {EC_GF_OP_XOR2, 3, 5, 0}, {EC_GF_OP_XOR2, 4, 3, 0},
+    {EC_GF_OP_XOR2, 1, 4, 0}, {EC_GF_OP_XOR2, 0, 1, 0},
+    {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_79 = {9,
+                                    {
+                                        4,
+                                        5,
+                                        7,
+                                        3,
+                                        1,
+                                        6,
+                                        2,
+                                        0,
+                                        8,
+                                    },
+                                    ec_gf8_mul_79_ops};
+
+static ec_gf_op_t ec_gf8_mul_7A_ops[] = {
+    {EC_GF_OP_XOR2, 2, 1, 0}, {EC_GF_OP_XOR2, 3, 2, 0},
+    {EC_GF_OP_XOR2, 4, 0, 0}, {EC_GF_OP_XOR2, 4, 3, 0},
+    {EC_GF_OP_XOR2, 5, 4, 0}, {EC_GF_OP_XOR2, 6, 5, 0},
+    {EC_GF_OP_XOR2, 7, 6, 0}, {EC_GF_OP_XOR2, 0, 7, 0},
+    {EC_GF_OP_XOR2, 2, 7, 0}, {EC_GF_OP_XOR2, 1, 0, 0},
+    {EC_GF_OP_XOR2, 4, 0, 0}, {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_7A = {8,
+                                    {
+                                        1,
+                                        2,
+                                        3,
+                                        4,
+                                        5,
+                                        6,
+                                        7,
+                                        0,
+                                    },
+                                    ec_gf8_mul_7A_ops};
+
+static ec_gf_op_t ec_gf8_mul_7B_ops[] = {
+    {EC_GF_OP_XOR2, 3, 1, 0}, {EC_GF_OP_XOR3, 8, 5, 3},
+    {EC_GF_OP_XOR2, 8, 0, 0}, {EC_GF_OP_COPY, 9, 4, 0},
+    {EC_GF_OP_XOR2, 8, 2, 0}, {EC_GF_OP_XOR2, 4, 6, 0},
+    {EC_GF_OP_XOR2, 4, 8, 0}, {EC_GF_OP_XOR2, 7, 4, 0},
+    {EC_GF_OP_XOR2, 5, 4, 0}, {EC_GF_OP_XOR2, 0, 4, 0},
+    {EC_GF_OP_XOR2, 2, 7, 0}, {EC_GF_OP_XOR3, 4, 1, 9},
+    {EC_GF_OP_XOR2, 6, 7, 0}, {EC_GF_OP_XOR2, 1, 7, 0},
+    {EC_GF_OP_XOR2, 4, 2, 0}, {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_7B = {10,
+                                    {
+                                        1,
+                                        2,
+                                        3,
+                                        4,
+                                        8,
+                                        5,
+                                        6,
+                                        0,
+                                        7,
+                                        9,
+                                    },
+                                    ec_gf8_mul_7B_ops};
+
+static ec_gf_op_t ec_gf8_mul_7C_ops[] = {
+    {EC_GF_OP_XOR2, 5, 3, 0}, {EC_GF_OP_XOR2, 4, 5, 0},
+    {EC_GF_OP_XOR2, 2, 4, 0}, {EC_GF_OP_XOR2, 4, 6, 0},
+    {EC_GF_OP_XOR2, 3, 1, 0}, {EC_GF_OP_XOR2, 0, 4, 0},
+    {EC_GF_OP_XOR2, 7, 2, 0}, {EC_GF_OP_XOR2, 3, 0, 0},
+    {EC_GF_OP_XOR2, 7, 3, 0}, {EC_GF_OP_XOR2, 5, 7, 0},
+    {EC_GF_OP_XOR2, 1, 7, 0}, {EC_GF_OP_XOR2, 6, 5, 0},
+    {EC_GF_OP_XOR2, 0, 5, 0}, {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_7C = {8,
+                                    {
+                                        2,
+                                        4,
+                                        1,
+                                        6,
+                                        3,
+                                        5,
+                                        7,
+                                        0,
+                                    },
+                                    ec_gf8_mul_7C_ops};
+
+static ec_gf_op_t ec_gf8_mul_7D_ops[] = {
+    {EC_GF_OP_XOR2, 2, 1, 0}, {EC_GF_OP_XOR2, 3, 2, 0},
+    {EC_GF_OP_XOR2, 2, 6, 0}, {EC_GF_OP_XOR2, 1, 0, 0},
+    {EC_GF_OP_XOR2, 0, 2, 0}, {EC_GF_OP_XOR2, 1, 5, 0},
+    {EC_GF_OP_XOR2, 7, 0, 0}, {EC_GF_OP_XOR2, 6, 1, 0},
+    {EC_GF_OP_XOR2, 4, 3, 0}, {EC_GF_OP_XOR2, 3, 7, 0},
+    {EC_GF_OP_XOR2, 1, 4, 0}, {EC_GF_OP_XOR2, 2, 3, 0},
+    {EC_GF_OP_XOR2, 0, 1, 0}, {EC_GF_OP_XOR2, 5, 2, 0},
+    {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_7D = {8,
+                                    {
+                                        1,
+                                        0,
+                                        3,
+                                        5,
+                                        6,
+                                        7,
+                                        2,
+                                        4,
+                                    },
+                                    ec_gf8_mul_7D_ops};
+
+static ec_gf_op_t ec_gf8_mul_7E_ops[] = {
+    {EC_GF_OP_XOR2, 0, 5, 0}, {EC_GF_OP_COPY, 8, 0, 0},
+    {EC_GF_OP_XOR2, 0, 1, 0}, {EC_GF_OP_XOR2, 6, 3, 0},
+    {EC_GF_OP_XOR2, 6, 0, 0}, {EC_GF_OP_XOR2, 5, 6, 0},
+    {EC_GF_OP_XOR2, 6, 4, 0}, {EC_GF_OP_XOR2, 7, 6, 0},
+    {EC_GF_OP_XOR2, 1, 6, 0}, {EC_GF_OP_XOR3, 6, 2, 7},
+    {EC_GF_OP_XOR2, 3, 6, 0}, {EC_GF_OP_XOR2, 2, 5, 0},
+    {EC_GF_OP_XOR2, 4, 6, 0}, {EC_GF_OP_XOR2, 5, 3, 0},
+    {EC_GF_OP_XOR2, 6, 8, 0}, {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_7E = {9,
+                                    {
+                                        5,
+                                        1,
+                                        2,
+                                        0,
+                                        7,
+                                        3,
+                                        4,
+                                        6,
+                                        8,
+                                    },
+                                    ec_gf8_mul_7E_ops};
+
+static ec_gf_op_t ec_gf8_mul_7F_ops[] = {
+    {EC_GF_OP_COPY, 8, 0, 0}, {EC_GF_OP_XOR2, 0, 1, 0},
+    {EC_GF_OP_XOR2, 0, 3, 0}, {EC_GF_OP_XOR2, 5, 0, 0},
+    {EC_GF_OP_XOR3, 9, 7, 5}, {EC_GF_OP_XOR2, 2, 9, 0},
+    {EC_GF_OP_XOR2, 1, 2, 0}, {EC_GF_OP_XOR2, 0, 6, 0},
+    {EC_GF_OP_XOR2, 4, 1, 0}, {EC_GF_OP_XOR2, 6, 9, 0},
+    {EC_GF_OP_XOR3, 9, 6, 4}, {EC_GF_OP_XOR2, 7, 9, 0},
+    {EC_GF_OP_XOR2, 3, 9, 0}, {EC_GF_OP_XOR2, 1, 7, 0},
+    {EC_GF_OP_XOR2, 7, 8, 0}, {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_7F = {10,
+                                    {
+                                        4,
+                                        1,
+                                        0,
+                                        5,
+                                        6,
+                                        7,
+                                        2,
+                                        3,
+                                        8,
+                                        9,
+                                    },
+                                    ec_gf8_mul_7F_ops};
+
+static ec_gf_op_t ec_gf8_mul_80_ops[] = {
+    {EC_GF_OP_XOR2, 7, 3, 0}, {EC_GF_OP_XOR2, 3, 4, 0},
+    {EC_GF_OP_XOR2, 1, 2, 0}, {EC_GF_OP_XOR2, 2, 3, 0},
+    {EC_GF_OP_XOR2, 3, 5, 0}, {EC_GF_OP_XOR2, 5, 1, 0},
+    {EC_GF_OP_XOR2, 0, 1, 0}, {EC_GF_OP_XOR2, 4, 5, 0},
+    {EC_GF_OP_XOR2, 1, 7, 0}, {EC_GF_OP_XOR2, 6, 4, 0},
+    {EC_GF_OP_XOR2, 0, 6, 0}, {EC_GF_OP_XOR2, 6, 2, 0},
+    {EC_GF_OP_XOR2, 7, 6, 0}, {EC_GF_OP_XOR2, 5, 7, 0},
+    {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_80 = {8,
+                                    {
+                                        7,
+                                        5,
+                                        6,
+                                        4,
+                                        1,
+                                        2,
+                                        3,
+                                        0,
+                                    },
+                                    ec_gf8_mul_80_ops};
+
+static ec_gf_op_t ec_gf8_mul_81_ops[] = {
+    {EC_GF_OP_XOR2, 3, 5, 0}, {EC_GF_OP_XOR2, 4, 6, 0},
+    {EC_GF_OP_XOR2, 3, 4, 0}, {EC_GF_OP_XOR2, 2, 3, 0},
+    {EC_GF_OP_XOR2, 6, 2, 0}, {EC_GF_OP_XOR2, 1, 6, 0},
+    {EC_GF_OP_XOR2, 7, 1, 0}, {EC_GF_OP_XOR2, 4, 1, 0},
+    {EC_GF_OP_XOR2, 5, 7, 0}, {EC_GF_OP_XOR2, 0, 5, 0},
+    {EC_GF_OP_XOR2, 7, 3, 0}, {EC_GF_OP_XOR2, 2, 0, 0},
+    {EC_GF_OP_XOR2, 0, 4, 0}, {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_81 = {8,
+                                    {
+                                        2,
+                                        7,
+                                        4,
+                                        1,
+                                        5,
+                                        6,
+                                        3,
+                                        0,
+                                    },
+                                    ec_gf8_mul_81_ops};
+
+static ec_gf_op_t ec_gf8_mul_82_ops[] = {
+    {EC_GF_OP_XOR2, 7, 6, 0}, {EC_GF_OP_XOR2, 6, 1, 0},
+    {EC_GF_OP_COPY, 8, 6, 0}, {EC_GF_OP_XOR2, 6, 5, 0},
+    {EC_GF_OP_XOR2, 5, 4, 0}, {EC_GF_OP_XOR2, 4, 3, 0},
+    {EC_GF_OP_XOR2, 1, 2, 0}, {EC_GF_OP_XOR2, 3, 2, 0},
+    {EC_GF_OP_XOR2, 2, 0, 0}, {EC_GF_OP_XOR2, 2, 7, 0},
+    {EC_GF_OP_XOR2, 0, 5, 0}, {EC_GF_OP_XOR2, 7, 5, 0},
+    {EC_GF_OP_XOR3, 5, 8, 7}, {EC_GF_OP_XOR2, 7, 4, 0},
+    {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_82 = {9,
+                                    {
+                                        6,
+                                        2,
+                                        7,
+                                        5,
+                                        1,
+                                        3,
+                                        4,
+                                        0,
+                                        8,
+                                    },
+                                    ec_gf8_mul_82_ops};
+
+static ec_gf_op_t ec_gf8_mul_83_ops[] = {
+    {EC_GF_OP_XOR2, 0, 1, 0}, {EC_GF_OP_XOR2, 1, 2, 0},
+    {EC_GF_OP_XOR2, 2, 3, 0}, {EC_GF_OP_XOR2, 0, 3, 0},
+    {EC_GF_OP_XOR2, 2, 5, 0}, {EC_GF_OP_XOR2, 7, 2, 0},
+    {EC_GF_OP_XOR2, 5, 0, 0}, {EC_GF_OP_XOR2, 3, 6, 0},
+    {EC_GF_OP_XOR2, 1, 4, 0}, {EC_GF_OP_XOR2, 6, 7, 0},
+    {EC_GF_OP_XOR2, 4, 3, 0}, {EC_GF_OP_XOR2, 7, 1, 0},
+    {EC_GF_OP_XOR2, 3, 5, 0}, {EC_GF_OP_XOR2, 0, 7, 0},
+    {EC_GF_OP_XOR2, 5, 6, 0}, {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_83 = {8,
+                                    {
+                                        3,
+                                        5,
+                                        6,
+                                        7,
+                                        1,
+                                        2,
+                                        4,
+                                        0,
+                                    },
+                                    ec_gf8_mul_83_ops};
+
+static ec_gf_op_t ec_gf8_mul_84_ops[] = {
+    {EC_GF_OP_XOR2, 3, 5, 0}, {EC_GF_OP_XOR2, 7, 5, 0},
+    {EC_GF_OP_XOR2, 5, 6, 0}, {EC_GF_OP_XOR2, 6, 2, 0},
+    {EC_GF_OP_XOR2, 4, 6, 0}, {EC_GF_OP_XOR2, 2, 0, 0},
+    {EC_GF_OP_XOR2, 2, 4, 0}, {EC_GF_OP_XOR2, 4, 7, 0},
+    {EC_GF_OP_XOR2, 7, 1, 0}, {EC_GF_OP_XOR2, 1, 3, 0},
+    {EC_GF_OP_XOR2, 0, 1, 0}, {EC_GF_OP_XOR2, 1, 5, 0},
+    {EC_GF_OP_XOR2, 5, 4, 0}, {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_84 = {8,
+                                    {
+                                        7,
+                                        6,
+                                        0,
+                                        4,
+                                        1,
+                                        5,
+                                        3,
+                                        2,
+                                    },
+                                    ec_gf8_mul_84_ops};
+
+static ec_gf_op_t ec_gf8_mul_85_ops[] = {
+    {EC_GF_OP_XOR2, 3, 6, 0}, {EC_GF_OP_XOR2, 7, 5, 0},
+    {EC_GF_OP_XOR2, 6, 2, 0}, {EC_GF_OP_XOR2, 5, 3, 0},
+    {EC_GF_OP_XOR2, 2, 7, 0}, {EC_GF_OP_XOR2, 4, 2, 0},
+    {EC_GF_OP_XOR2, 7, 0, 0}, {EC_GF_OP_XOR2, 3, 4, 0},
+    {EC_GF_OP_XOR2, 6, 1, 0}, {EC_GF_OP_XOR2, 7, 1, 0},
+    {EC_GF_OP_XOR2, 0, 6, 0}, {EC_GF_OP_XOR2, 1, 3, 0},
+    {EC_GF_OP_XOR2, 0, 5, 0}, {EC_GF_OP_XOR2, 2, 1, 0},
+    {EC_GF_OP_XOR2, 1, 0, 0}, {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_85 = {8,
+                                    {
+                                        7,
+                                        6,
+                                        0,
+                                        3,
+                                        2,
+                                        4,
+                                        5,
+                                        1,
+                                    },
+                                    ec_gf8_mul_85_ops};
+
+static ec_gf_op_t ec_gf8_mul_86_ops[] = {
+    {EC_GF_OP_XOR2, 1, 5, 0}, {EC_GF_OP_XOR2, 5, 6, 0},
+    {EC_GF_OP_XOR2, 6, 0, 0}, {EC_GF_OP_XOR2, 0, 4, 0},
+    {EC_GF_OP_XOR2, 4, 5, 0}, {EC_GF_OP_XOR2, 5, 7, 0},
+    {EC_GF_OP_XOR2, 7, 2, 0}, {EC_GF_OP_XOR2, 2, 6, 0},
+    {EC_GF_OP_XOR2, 6, 3, 0}, {EC_GF_OP_XOR2, 6, 5, 0},
+    {EC_GF_OP_XOR2, 5, 1, 0}, {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_86 = {8,
+                                    {
+                                        1,
+                                        2,
+                                        6,
+                                        4,
+                                        5,
+                                        7,
+                                        3,
+                                        0,
+                                    },
+                                    ec_gf8_mul_86_ops};
+
+static ec_gf_op_t ec_gf8_mul_87_ops[] = {
+    {EC_GF_OP_XOR2, 1, 0, 0}, {EC_GF_OP_COPY, 8, 1, 0},
+    {EC_GF_OP_XOR2, 8, 6, 0}, {EC_GF_OP_XOR2, 6, 3, 0},
+    {EC_GF_OP_XOR2, 3, 0, 0}, {EC_GF_OP_XOR2, 0, 4, 0},
+    {EC_GF_OP_XOR2, 1, 5, 0}, {EC_GF_OP_XOR2, 4, 5, 0},
+    {EC_GF_OP_XOR2, 0, 7, 0}, {EC_GF_OP_XOR2, 7, 5, 0},
+    {EC_GF_OP_XOR2, 4, 6, 0}, {EC_GF_OP_XOR3, 5, 8, 0},
+    {EC_GF_OP_XOR2, 7, 2, 0}, {EC_GF_OP_XOR2, 2, 8, 0},
+    {EC_GF_OP_XOR2, 3, 7, 0}, {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_87 = {9,
+                                    {
+                                        1,
+                                        2,
+                                        3,
+                                        4,
+                                        5,
+                                        7,
+                                        6,
+                                        0,
+                                        8,
+                                    },
+                                    ec_gf8_mul_87_ops};
+
+static ec_gf_op_t ec_gf8_mul_88_ops[] = {
+    {EC_GF_OP_XOR2, 5, 6, 0}, {EC_GF_OP_XOR2, 6, 7, 0},
+    {EC_GF_OP_XOR2, 4, 6, 0}, {EC_GF_OP_XOR2, 6, 1, 0},
+    {EC_GF_OP_XOR2, 7, 2, 0}, {EC_GF_OP_XOR2, 1, 0, 0},
+    {EC_GF_OP_XOR2, 2, 3, 0}, {EC_GF_OP_XOR2, 1, 7, 0},
+    {EC_GF_OP_XOR2, 0, 5, 0}, {EC_GF_OP_XOR2, 2, 5, 0},
+    {EC_GF_OP_XOR2, 1, 4, 0}, {EC_GF_OP_XOR2, 5, 4, 0},
+    {EC_GF_OP_XOR2, 4, 3, 0}, {EC_GF_OP_XOR2, 3, 6, 0},
+    {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_88 = {8,
+                                    {
+                                        6,
+                                        7,
+                                        3,
+                                        1,
+                                        2,
+                                        4,
+                                        5,
+                                        0,
+                                    },
+                                    ec_gf8_mul_88_ops};
+
+static ec_gf_op_t ec_gf8_mul_89_ops[] = {
+    {EC_GF_OP_XOR2, 7, 2, 0}, {EC_GF_OP_XOR2, 5, 1, 0},
+    {EC_GF_OP_XOR2, 1, 7, 0}, {EC_GF_OP_XOR2, 2, 0, 0},
+    {EC_GF_OP_XOR2, 5, 0, 0}, {EC_GF_OP_XOR2, 6, 1, 0},
+    {EC_GF_OP_XOR2, 2, 6, 0}, {EC_GF_OP_XOR3, 8, 5, 2},
+    {EC_GF_OP_XOR2, 4, 8, 0}, {EC_GF_OP_XOR2, 6, 3, 0},
+    {EC_GF_OP_XOR2, 0, 8, 0}, {EC_GF_OP_XOR2, 3, 4, 0},
+    {EC_GF_OP_XOR2, 7, 3, 0}, {EC_GF_OP_XOR2, 5, 7, 0},
+    {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_89 = {9,
+                                    {
+                                        2,
+                                        1,
+                                        6,
+                                        5,
+                                        7,
+                                        3,
+                                        4,
+                                        0,
+                                        8,
+                                    },
+                                    ec_gf8_mul_89_ops};
+
+static ec_gf_op_t ec_gf8_mul_8A_ops[] = {
+    {EC_GF_OP_XOR2, 2, 0, 0}, {EC_GF_OP_XOR2, 5, 0, 0},
+    {EC_GF_OP_XOR2, 1, 6, 0}, {EC_GF_OP_XOR2, 3, 6, 0},
+    {EC_GF_OP_XOR2, 0, 1, 0}, {EC_GF_OP_XOR2, 6, 5, 0},
+    {EC_GF_OP_XOR2, 2, 7, 0}, {EC_GF_OP_XOR2, 4, 7, 0},
+    {EC_GF_OP_XOR2, 6, 2, 0}, {EC_GF_OP_XOR2, 7, 3, 0},
+    {EC_GF_OP_XOR2, 0, 4, 0}, {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_8A = {8,
+                                    {
+                                        1,
+                                        2,
+                                        3,
+                                        0,
+                                        6,
+                                        7,
+                                        4,
+                                        5,
+                                    },
+                                    ec_gf8_mul_8A_ops};
+
+static ec_gf_op_t ec_gf8_mul_8B_ops[] = {
+    {EC_GF_OP_XOR2, 1, 0, 0}, {EC_GF_OP_XOR2, 3, 6, 0},
+    {EC_GF_OP_XOR2, 6, 1, 0}, {EC_GF_OP_XOR2, 1, 7, 0},
+    {EC_GF_OP_XOR2, 7, 5, 0}, {EC_GF_OP_XOR2, 4, 1, 0},
+    {EC_GF_OP_XOR2, 5, 2, 0}, {EC_GF_OP_XOR2, 0, 7, 0},
+    {EC_GF_OP_XOR2, 1, 2, 0}, {EC_GF_OP_XOR2, 7, 3, 0},
+    {EC_GF_OP_XOR2, 2, 3, 0}, {EC_GF_OP_XOR2, 3, 4, 0},
+    {EC_GF_OP_XOR2, 4, 6, 0}, {EC_GF_OP_XOR2, 5, 4, 0},
+    {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_8B = {8,
+                                    {
+                                        6,
+                                        1,
+                                        2,
+                                        3,
+                                        5,
+                                        7,
+                                        4,
+                                        0,
+                                    },
+                                    ec_gf8_mul_8B_ops};
+
+static ec_gf_op_t ec_gf8_mul_8C_ops[] = {
+    {EC_GF_OP_XOR2, 1, 7, 0}, {EC_GF_OP_XOR2, 5, 7, 0},
+    {EC_GF_OP_XOR2, 7, 4, 0}, {EC_GF_OP_XOR2, 7, 2, 0},
+    {EC_GF_OP_XOR2, 4, 6, 0}, {EC_GF_OP_XOR2, 7, 0, 0},
+    {EC_GF_OP_XOR2, 6, 0, 0}, {EC_GF_OP_XOR2, 0, 3, 0},
+    {EC_GF_OP_XOR2, 3, 5, 0}, {EC_GF_OP_XOR2, 0, 1, 0},
+    {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_8C = {8,
+                                    {
+                                        1,
+                                        2,
+                                        0,
+                                        7,
+                                        3,
+                                        4,
+                                        5,
+                                        6,
+                                    },
+                                    ec_gf8_mul_8C_ops};
+
+static ec_gf_op_t ec_gf8_mul_8D_ops[] = {
+    {EC_GF_OP_XOR2, 7, 0, 0}, {EC_GF_OP_XOR2, 0, 5, 0},
+    {EC_GF_OP_XOR2, 5, 4, 0}, {EC_GF_OP_XOR2, 5, 6, 0},
+    {EC_GF_OP_XOR2, 3, 7, 0}, {EC_GF_OP_XOR2, 6, 7, 0},
+    {EC_GF_OP_XOR2, 7, 1, 0}, {EC_GF_OP_XOR2, 4, 3, 0},
+    {EC_GF_OP_XOR2, 1, 2, 0}, {EC_GF_OP_XOR2, 2, 4, 0},
+    {EC_GF_OP_XOR2, 3, 1, 0}, {EC_GF_OP_XOR2, 4, 0, 0},
+    {EC_GF_OP_XOR2, 0, 6, 0}, {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_8D = {8,
+                                    {
+                                        7,
+                                        1,
+                                        3,
+                                        2,
+                                        4,
+                                        5,
+                                        0,
+                                        6,
+                                    },
+                                    ec_gf8_mul_8D_ops};
+
+static ec_gf_op_t ec_gf8_mul_8E_ops[] = {{EC_GF_OP_XOR2, 2, 0, 0},
+                                         {EC_GF_OP_XOR2, 3, 0, 0},
+                                         {EC_GF_OP_XOR2, 4, 0, 0},
+                                         {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_8E = {8,
+                                    {
+                                        1,
+                                        2,
+                                        3,
+                                        4,
+                                        5,
+                                        6,
+                                        7,
+                                        0,
+                                    },
+                                    ec_gf8_mul_8E_ops};
+
+static ec_gf_op_t ec_gf8_mul_8F_ops[] = {
+    {EC_GF_OP_XOR2, 1, 0, 0}, {EC_GF_OP_XOR2, 3, 0, 0},
+    {EC_GF_OP_XOR2, 0, 7, 0}, {EC_GF_OP_XOR2, 7, 6, 0},
+    {EC_GF_OP_XOR2, 6, 5, 0}, {EC_GF_OP_XOR2, 5, 4, 0},
+    {EC_GF_OP_XOR2, 4, 3, 0}, {EC_GF_OP_XOR2, 3, 2, 0},
+    {EC_GF_OP_XOR2, 2, 1, 0}, {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_8F = {8,
+                                    {
+                                        1,
+                                        2,
+                                        3,
+                                        4,
+                                        5,
+                                        6,
+                                        7,
+                                        0,
+                                    },
+                                    ec_gf8_mul_8F_ops};
+
+static ec_gf_op_t ec_gf8_mul_90_ops[] = {
+    {EC_GF_OP_XOR2, 2, 1, 0}, {EC_GF_OP_XOR2, 7, 2, 0},
+    {EC_GF_OP_XOR2, 6, 7, 0}, {EC_GF_OP_XOR2, 5, 1, 0},
+    {EC_GF_OP_XOR2, 5, 6, 0}, {EC_GF_OP_XOR2, 4, 5, 0},
+    {EC_GF_OP_XOR2, 3, 4, 0}, {EC_GF_OP_XOR2, 4, 2, 0},
+    {EC_GF_OP_XOR2, 1, 3, 0}, {EC_GF_OP_XOR2, 6, 3, 0},
+    {EC_GF_OP_XOR2, 0, 1, 0}, {EC_GF_OP_XOR2, 2, 0, 0},
+    {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_90 = {8,
+                                    {
+                                        4,
+                                        5,
+                                        6,
+                                        7,
+                                        0,
+                                        1,
+                                        3,
+                                        2,
+                                    },
+                                    ec_gf8_mul_90_ops};
+
+static ec_gf_op_t ec_gf8_mul_91_ops[] = {
+    {EC_GF_OP_XOR2, 0, 3, 0}, {EC_GF_OP_XOR2, 3, 4, 0},
+    {EC_GF_OP_COPY, 9, 1, 0}, {EC_GF_OP_COPY, 8, 3, 0},
+    {EC_GF_OP_XOR2, 3, 5, 0}, {EC_GF_OP_XOR2, 1, 2, 0},
+    {EC_GF_OP_XOR2, 1, 3, 0}, {EC_GF_OP_XOR2, 7, 1, 0},
+    {EC_GF_OP_XOR2, 5, 7, 0}, {EC_GF_OP_XOR2, 7, 9, 0},
+    {EC_GF_OP_XOR2, 6, 5, 0}, {EC_GF_OP_XOR2, 4, 5, 0},
+    {EC_GF_OP_XOR2, 3, 6, 0}, {EC_GF_OP_XOR2, 0, 3, 0},
+    {EC_GF_OP_XOR3, 5, 8, 0}, {EC_GF_OP_XOR2, 2, 5, 0},
+    {EC_GF_OP_XOR2, 5, 4, 0}, {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_91 = {10,
+                                    {
+                                        2,
+                                        3,
+                                        1,
+                                        4,
+                                        0,
+                                        6,
+                                        7,
+                                        5,
+                                        8,
+                                        9,
+                                    },
+                                    ec_gf8_mul_91_ops};
+
+static ec_gf_op_t ec_gf8_mul_92_ops[] = {
+    {EC_GF_OP_XOR2, 4, 1, 0}, {EC_GF_OP_XOR2, 5, 4, 0},
+    {EC_GF_OP_XOR2, 6, 5, 0}, {EC_GF_OP_XOR2, 2, 0, 0},
+    {EC_GF_OP_XOR2, 2, 6, 0}, {EC_GF_OP_XOR2, 3, 1, 0},
+    {EC_GF_OP_XOR2, 4, 2, 0}, {EC_GF_OP_XOR2, 3, 0, 0},
+    {EC_GF_OP_XOR2, 7, 4, 0}, {EC_GF_OP_XOR2, 3, 7, 0},
+    {EC_GF_OP_XOR2, 5, 3, 0}, {EC_GF_OP_XOR2, 4, 5, 0},
+    {EC_GF_OP_XOR2, 0, 4, 0}, {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_92 = {8,
+                                    {
+                                        6,
+                                        7,
+                                        0,
+                                        1,
+                                        2,
+                                        3,
+                                        5,
+                                        4,
+                                    },
+                                    ec_gf8_mul_92_ops};
+
+static ec_gf_op_t ec_gf8_mul_93_ops[] = {
+    {EC_GF_OP_XOR2, 2, 7, 0}, {EC_GF_OP_XOR2, 4, 2, 0},
+    {EC_GF_OP_XOR2, 1, 3, 0}, {EC_GF_OP_XOR2, 3, 4, 0},
+    {EC_GF_OP_XOR2, 0, 2, 0}, {EC_GF_OP_XOR2, 5, 3, 0},
+    {EC_GF_OP_XOR2, 6, 1, 0}, {EC_GF_OP_XOR2, 0, 5, 0},
+    {EC_GF_OP_XOR2, 2, 6, 0}, {EC_GF_OP_XOR2, 6, 0, 0},
+    {EC_GF_OP_XOR2, 4, 6, 0}, {EC_GF_OP_XOR2, 7, 4, 0},
+    {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_93 = {8,
+                                    {
+                                        6,
+                                        4,
+                                        5,
+                                        1,
+                                        7,
+                                        2,
+                                        3,
+                                        0,
+                                    },
+                                    ec_gf8_mul_93_ops};
+
+static ec_gf_op_t ec_gf8_mul_94_ops[] = {
+    {EC_GF_OP_XOR2, 0, 2, 0}, {EC_GF_OP_XOR2, 2, 6, 0},
+    {EC_GF_OP_XOR2, 7, 2, 0}, {EC_GF_OP_XOR2, 3, 7, 0},
+    {EC_GF_OP_XOR2, 0, 3, 0}, {EC_GF_OP_XOR2, 3, 5, 0},
+    {EC_GF_OP_XOR2, 1, 4, 0}, {EC_GF_OP_XOR2, 5, 2, 0},
+    {EC_GF_OP_XOR2, 4, 0, 0}, {EC_GF_OP_XOR2, 1, 5, 0},
+    {EC_GF_OP_XOR2, 7, 1, 0}, {EC_GF_OP_XOR2, 0, 7, 0},
+    {EC_GF_OP_XOR2, 6, 0, 0}, {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_94 = {8,
+                                    {
+                                        7,
+                                        5,
+                                        0,
+                                        2,
+                                        6,
+                                        1,
+                                        3,
+                                        4,
+                                    },
+                                    ec_gf8_mul_94_ops};
+
+static ec_gf_op_t ec_gf8_mul_95_ops[] = {
+    {EC_GF_OP_XOR2, 3, 2, 0}, {EC_GF_OP_XOR2, 7, 3, 0},
+    {EC_GF_OP_XOR2, 3, 6, 0}, {EC_GF_OP_XOR2, 0, 3, 0},
+    {EC_GF_OP_XOR2, 4, 0, 0}, {EC_GF_OP_XOR2, 2, 4, 0},
+    {EC_GF_OP_XOR2, 4, 5, 0}, {EC_GF_OP_XOR2, 1, 4, 0},
+    {EC_GF_OP_XOR2, 5, 7, 0}, {EC_GF_OP_XOR2, 6, 1, 0},
+    {EC_GF_OP_XOR2, 7, 6, 0}, {EC_GF_OP_XOR2, 6, 2, 0},
+    {EC_GF_OP_XOR2, 0, 6, 0}, {EC_GF_OP_XOR2, 4, 0, 0},
+    {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_95 = {8,
+                                    {
+                                        7,
+                                        6,
+                                        1,
+                                        3,
+                                        0,
+                                        4,
+                                        5,
+                                        2,
+                                    },
+                                    ec_gf8_mul_95_ops};
+
+static ec_gf_op_t ec_gf8_mul_96_ops[] = {
+    {EC_GF_OP_XOR2, 5, 1, 0}, {EC_GF_OP_XOR2, 4, 5, 0},
+    {EC_GF_OP_XOR2, 5, 6, 0}, {EC_GF_OP_XOR2, 6, 7, 0},
+    {EC_GF_OP_XOR3, 8, 0, 4}, {EC_GF_OP_XOR2, 3, 6, 0},
+    {EC_GF_OP_XOR2, 7, 8, 0}, {EC_GF_OP_XOR2, 0, 1, 0},
+    {EC_GF_OP_XOR2, 8, 3, 0}, {EC_GF_OP_XOR2, 3, 2, 0},
+    {EC_GF_OP_XOR2, 1, 8, 0}, {EC_GF_OP_XOR2, 2, 5, 0},
+    {EC_GF_OP_XOR2, 5, 8, 0}, {EC_GF_OP_XOR2, 0, 2, 0},
+    {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_96 = {9,
+                                    {
+                                        4,
+                                        0,
+                                        1,
+                                        6,
+                                        7,
+                                        2,
+                                        3,
+                                        5,
+                                        8,
+                                    },
+                                    ec_gf8_mul_96_ops};
+
+static ec_gf_op_t ec_gf8_mul_97_ops[] = {
+    {EC_GF_OP_XOR2, 5, 0, 0}, {EC_GF_OP_COPY, 8, 2, 0},
+    {EC_GF_OP_XOR2, 0, 3, 0}, {EC_GF_OP_XOR2, 8, 6, 0},
+    {EC_GF_OP_XOR2, 5, 1, 0}, {EC_GF_OP_XOR2, 3, 7, 0},
+    {EC_GF_OP_XOR2, 1, 8, 0}, {EC_GF_OP_XOR2, 7, 5, 0},
+    {EC_GF_OP_XOR2, 2, 3, 0}, {EC_GF_OP_XOR2, 6, 3, 0},
+    {EC_GF_OP_XOR2, 0, 4, 0}, {EC_GF_OP_XOR2, 3, 1, 0},
+    {EC_GF_OP_XOR2, 4, 5, 0}, {EC_GF_OP_XOR2, 5, 8, 0},
+    {EC_GF_OP_XOR2, 3, 4, 0}, {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_97 = {9,
+                                    {
+                                        4,
+                                        5,
+                                        3,
+                                        6,
+                                        7,
+                                        1,
+                                        2,
+                                        0,
+                                        8,
+                                    },
+                                    ec_gf8_mul_97_ops};
+
+static ec_gf_op_t ec_gf8_mul_98_ops[] = {
+    {EC_GF_OP_XOR2, 5, 7, 0}, {EC_GF_OP_XOR2, 4, 1, 0},
+    {EC_GF_OP_XOR2, 2, 5, 0}, {EC_GF_OP_XOR2, 4, 7, 0},
+    {EC_GF_OP_XOR2, 1, 2, 0}, {EC_GF_OP_XOR2, 3, 4, 0},
+    {EC_GF_OP_XOR2, 0, 1, 0}, {EC_GF_OP_XOR2, 4, 6, 0},
+    {EC_GF_OP_XOR2, 5, 3, 0}, {EC_GF_OP_XOR2, 6, 0, 0},
+    {EC_GF_OP_XOR2, 1, 4, 0}, {EC_GF_OP_XOR2, 0, 5, 0},
+    {EC_GF_OP_XOR2, 7, 0, 0}, {EC_GF_OP_XOR2, 0, 1, 0},
+    {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_98 = {8,
+                                    {
+                                        4,
+                                        2,
+                                        3,
+                                        6,
+                                        7,
+                                        5,
+                                        1,
+                                        0,
+                                    },
+                                    ec_gf8_mul_98_ops};
+
+static ec_gf_op_t ec_gf8_mul_99_ops[] = {
+    {EC_GF_OP_XOR2, 0, 2, 0}, {EC_GF_OP_XOR2, 7, 1, 0},
+    {EC_GF_OP_XOR2, 0, 3, 0}, {EC_GF_OP_XOR2, 7, 2, 0},
+    {EC_GF_OP_XOR2, 3, 4, 0}, {EC_GF_OP_XOR2, 2, 5, 0},
+    {EC_GF_OP_XOR2, 1, 3, 0}, {EC_GF_OP_XOR2, 5, 7, 0},
+    {EC_GF_OP_XOR2, 4, 2, 0}, {EC_GF_OP_XOR2, 3, 7, 0},
+    {EC_GF_OP_XOR2, 6, 0, 0}, {EC_GF_OP_XOR2, 2, 6, 0},
+    {EC_GF_OP_XOR2, 6, 3, 0}, {EC_GF_OP_XOR2, 7, 2, 0},
+    {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_99 = {8,
+                                    {
+                                        6,
+                                        5,
+                                        3,
+                                        7,
+                                        0,
+                                        1,
+                                        4,
+                                        2,
+                                    },
+                                    ec_gf8_mul_99_ops};
+
+static ec_gf_op_t ec_gf8_mul_9A_ops[] = {
+    {EC_GF_OP_XOR2, 5, 4, 0}, {EC_GF_OP_XOR2, 6, 4, 0},
+    {EC_GF_OP_XOR2, 4, 3, 0}, {EC_GF_OP_XOR2, 3, 2, 0},
+    {EC_GF_OP_XOR2, 2, 6, 0}, {EC_GF_OP_XOR2, 6, 1, 0},
+    {EC_GF_OP_XOR2, 1, 0, 0}, {EC_GF_OP_XOR2, 0, 5, 0},
+    {EC_GF_OP_XOR3, 8, 4, 0}, {EC_GF_OP_XOR2, 0, 6, 0},
+    {EC_GF_OP_XOR2, 7, 8, 0}, {EC_GF_OP_XOR2, 1, 8, 0},
+    {EC_GF_OP_XOR2, 3, 7, 0}, {EC_GF_OP_XOR2, 5, 3, 0},
+    {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_9A = {9,
+                                    {
+                                        6,
+                                        3,
+                                        4,
+                                        0,
+                                        5,
+                                        1,
+                                        2,
+                                        7,
+                                        8,
+                                    },
+                                    ec_gf8_mul_9A_ops};
+
+static ec_gf_op_t ec_gf8_mul_9B_ops[] = {
+    {EC_GF_OP_XOR2, 6, 0, 0}, {EC_GF_OP_XOR2, 7, 2, 0},
+    {EC_GF_OP_COPY, 9, 5, 0}, {EC_GF_OP_XOR2, 7, 0, 0},
+    {EC_GF_OP_XOR2, 2, 4, 0}, {EC_GF_OP_XOR2, 6, 1, 0},
+    {EC_GF_OP_XOR2, 5, 1, 0}, {EC_GF_OP_XOR2, 4, 6, 0},
+    {EC_GF_OP_XOR3, 8, 3, 2}, {EC_GF_OP_XOR2, 1, 3, 0},
+    {EC_GF_OP_XOR2, 5, 7, 0}, {EC_GF_OP_XOR2, 3, 9, 0},
+    {EC_GF_OP_XOR2, 0, 3, 0}, {EC_GF_OP_XOR2, 6, 3, 0},
+    {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_9B = {10,
+                                    {
+                                        4,
+                                        5,
+                                        8,
+                                        6,
+                                        7,
+                                        1,
+                                        2,
+                                        0,
+                                        3,
+                                        9,
+                                    },
+                                    ec_gf8_mul_9B_ops};
+
+static ec_gf_op_t ec_gf8_mul_9C_ops[] = {
+    {EC_GF_OP_XOR2, 3, 0, 0}, {EC_GF_OP_XOR2, 3, 6, 0},
+    {EC_GF_OP_XOR2, 7, 3, 0}, {EC_GF_OP_XOR2, 4, 7, 0},
+    {EC_GF_OP_XOR2, 1, 4, 0}, {EC_GF_OP_XOR2, 3, 1, 0},
+    {EC_GF_OP_XOR2, 2, 5, 0}, {EC_GF_OP_XOR2, 5, 3, 0},
+    {EC_GF_OP_XOR2, 6, 2, 0}, {EC_GF_OP_XOR2, 0, 2, 0},
+    {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_9C = {8,
+                                    {
+                                        3,
+                                        2,
+                                        1,
+                                        0,
+                                        4,
+                                        5,
+                                        6,
+                                        7,
+                                    },
+                                    ec_gf8_mul_9C_ops};
+
+static ec_gf_op_t ec_gf8_mul_9D_ops[] = {
+    {EC_GF_OP_XOR2, 3, 0, 0}, {EC_GF_OP_XOR2, 4, 1, 0},
+    {EC_GF_OP_XOR2, 6, 3, 0}, {EC_GF_OP_XOR2, 5, 2, 0},
+    {EC_GF_OP_XOR2, 2, 6, 0}, {EC_GF_OP_XOR2, 4, 7, 0},
+    {EC_GF_OP_XOR2, 1, 5, 0}, {EC_GF_OP_XOR2, 3, 5, 0},
+    {EC_GF_OP_XOR2, 7, 6, 0}, {EC_GF_OP_XOR2, 0, 4, 0},
+    {EC_GF_OP_XOR2, 2, 4, 0}, {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_9D = {8,
+                                    {
+                                        0,
+                                        1,
+                                        2,
+                                        3,
+                                        7,
+                                        4,
+                                        5,
+                                        6,
+                                    },
+                                    ec_gf8_mul_9D_ops};
+
+static ec_gf_op_t ec_gf8_mul_9E_ops[] = {
+    {EC_GF_OP_XOR2, 7, 0, 0}, {EC_GF_OP_COPY, 8, 7, 0},
+    {EC_GF_OP_XOR2, 8, 5, 0}, {EC_GF_OP_XOR2, 5, 2, 0},
+    {EC_GF_OP_XOR2, 2, 6, 0}, {EC_GF_OP_XOR2, 5, 0, 0},
+    {EC_GF_OP_XOR2, 6, 4, 0}, {EC_GF_OP_XOR2, 6, 0, 0},
+    {EC_GF_OP_XOR2, 7, 3, 0}, {EC_GF_OP_XOR2, 0, 1, 0},
+    {EC_GF_OP_XOR2, 4, 1, 0}, {EC_GF_OP_XOR2, 3, 6, 0},
+    {EC_GF_OP_XOR2, 0, 8, 0}, {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_9E = {9,
+                                    {
+                                        4,
+                                        5,
+                                        3,
+                                        8,
+                                        6,
+                                        0,
+                                        2,
+                                        7,
+                                        1,
+                                    },
+                                    ec_gf8_mul_9E_ops};
+
+static ec_gf_op_t ec_gf8_mul_9F_ops[] = {
+    {EC_GF_OP_XOR3, 8, 1, 2}, {EC_GF_OP_XOR2, 8, 3, 0},
+    {EC_GF_OP_XOR2, 3, 0, 0}, {EC_GF_OP_XOR2, 4, 0, 0},
+    {EC_GF_OP_XOR2, 4, 1, 0}, {EC_GF_OP_XOR2, 5, 3, 0},
+    {EC_GF_OP_XOR2, 0, 6, 0}, {EC_GF_OP_XOR2, 1, 7, 0},
+    {EC_GF_OP_XOR2, 6, 4, 0}, {EC_GF_OP_XOR2, 7, 5, 0},
+    {EC_GF_OP_XOR2, 6, 8, 0}, {EC_GF_OP_XOR2, 5, 8, 0},
+    {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_9F = {9,
+                                    {
+                                        4,
+                                        5,
+                                        6,
+                                        7,
+                                        0,
+                                        1,
+                                        2,
+                                        3,
+                                        8,
+                                    },
+                                    ec_gf8_mul_9F_ops};
+
+static ec_gf_op_t ec_gf8_mul_A0_ops[] = {
+    {EC_GF_OP_XOR2, 6, 1, 0}, {EC_GF_OP_XOR2, 5, 6, 0},
+    {EC_GF_OP_XOR2, 6, 7, 0}, {EC_GF_OP_XOR2, 0, 1, 0},
+    {EC_GF_OP_XOR2, 4, 6, 0}, {EC_GF_OP_XOR2, 1, 2, 0},
+    {EC_GF_OP_XOR2, 1, 4, 0}, {EC_GF_OP_XOR2, 4, 3, 0},
+    {EC_GF_OP_XOR2, 3, 5, 0}, {EC_GF_OP_XOR2, 2, 3, 0},
+    {EC_GF_OP_XOR2, 5, 1, 0}, {EC_GF_OP_XOR2, 7, 2, 0},
+    {EC_GF_OP_XOR2, 2, 0, 0}, {EC_GF_OP_XOR2, 0, 5, 0},
+    {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_A0 = {8,
+                                    {
+                                        3,
+                                        1,
+                                        6,
+                                        7,
+                                        5,
+                                        2,
+                                        4,
+                                        0,
+                                    },
+                                    ec_gf8_mul_A0_ops};
+
+static ec_gf_op_t ec_gf8_mul_A1_ops[] = {
+    {EC_GF_OP_XOR2, 2, 6, 0}, {EC_GF_OP_XOR2, 1, 7, 0},
+    {EC_GF_OP_XOR2, 1, 2, 0}, {EC_GF_OP_XOR2, 5, 1, 0},
+    {EC_GF_OP_XOR2, 0, 3, 0}, {EC_GF_OP_XOR2, 6, 5, 0},
+    {EC_GF_OP_XOR2, 4, 1, 0}, {EC_GF_OP_XOR2, 0, 2, 0},
+    {EC_GF_OP_XOR2, 3, 4, 0}, {EC_GF_OP_XOR3, 8, 0, 6},
+    {EC_GF_OP_XOR2, 2, 3, 0}, {EC_GF_OP_XOR2, 7, 8, 0},
+    {EC_GF_OP_XOR2, 3, 8, 0}, {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_A1 = {9,
+                                    {
+                                        7,
+                                        4,
+                                        1,
+                                        5,
+                                        6,
+                                        0,
+                                        2,
+                                        3,
+                                        8,
+                                    },
+                                    ec_gf8_mul_A1_ops};
+
+static ec_gf_op_t ec_gf8_mul_A2_ops[] = {
+    {EC_GF_OP_XOR2, 2, 1, 0}, {EC_GF_OP_XOR2, 2, 4, 0},
+    {EC_GF_OP_XOR2, 3, 5, 0}, {EC_GF_OP_XOR2, 1, 6, 0},
+    {EC_GF_OP_XOR2, 2, 3, 0}, {EC_GF_OP_XOR2, 3, 1, 0},
+    {EC_GF_OP_XOR2, 0, 2, 0}, {EC_GF_OP_XOR2, 7, 3, 0},
+    {EC_GF_OP_XOR2, 1, 0, 0}, {EC_GF_OP_XOR2, 0, 7, 0},
+    {EC_GF_OP_XOR2, 4, 7, 0}, {EC_GF_OP_XOR2, 5, 0, 0},
+    {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_A2 = {8,
+                                    {
+                                        7,
+                                        0,
+                                        6,
+                                        3,
+                                        2,
+                                        1,
+                                        4,
+                                        5,
+                                    },
+                                    ec_gf8_mul_A2_ops};
+
+static ec_gf_op_t ec_gf8_mul_A3_ops[] = {
+    {EC_GF_OP_COPY, 8, 2, 0}, {EC_GF_OP_XOR2, 2, 6, 0},
+    {EC_GF_OP_XOR2, 0, 2, 0}, {EC_GF_OP_XOR2, 4, 0, 0},
+    {EC_GF_OP_XOR2, 1, 5, 0}, {EC_GF_OP_XOR2, 5, 4, 0},
+    {EC_GF_OP_XOR2, 6, 1, 0}, {EC_GF_OP_XOR2, 4, 3, 0},
+    {EC_GF_OP_XOR2, 7, 1, 0}, {EC_GF_OP_XOR2, 3, 8, 0},
+    {EC_GF_OP_XOR2, 0, 7, 0}, {EC_GF_OP_XOR2, 1, 3, 0},
+    {EC_GF_OP_XOR2, 7, 5, 0}, {EC_GF_OP_XOR2, 3, 0, 0},
+    {EC_GF_OP_XOR2, 0, 4, 0}, {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_A3 = {9,
+                                    {
+                                        3,
+                                        7,
+                                        2,
+                                        6,
+                                        1,
+                                        4,
+                                        0,
+                                        5,
+                                        8,
+                                    },
+                                    ec_gf8_mul_A3_ops};
+
+static ec_gf_op_t ec_gf8_mul_A4_ops[] = {
+    {EC_GF_OP_XOR2, 0, 1, 0}, {EC_GF_OP_XOR2, 4, 2, 0},
+    {EC_GF_OP_XOR2, 5, 3, 0}, {EC_GF_OP_XOR2, 7, 0, 0},
+    {EC_GF_OP_XOR2, 2, 5, 0}, {EC_GF_OP_XOR2, 6, 4, 0},
+    {EC_GF_OP_XOR2, 5, 1, 0}, {EC_GF_OP_XOR2, 4, 7, 0},
+    {EC_GF_OP_XOR2, 3, 6, 0}, {EC_GF_OP_XOR2, 1, 4, 0},
+    {EC_GF_OP_XOR2, 3, 1, 0}, {EC_GF_OP_XOR2, 0, 3, 0},
+    {EC_GF_OP_XOR2, 3, 2, 0}, {EC_GF_OP_XOR2, 4, 3, 0},
+    {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_A4 = {8,
+                                    {
+                                        5,
+                                        6,
+                                        7,
+                                        2,
+                                        4,
+                                        3,
+                                        0,
+                                        1,
+                                    },
+                                    ec_gf8_mul_A4_ops};
+
+static ec_gf_op_t ec_gf8_mul_A5_ops[] = {
+    {EC_GF_OP_XOR2, 7, 1, 0}, {EC_GF_OP_XOR2, 1, 5, 0},
+    {EC_GF_OP_XOR2, 7, 0, 0}, {EC_GF_OP_XOR2, 3, 0, 0},
+    {EC_GF_OP_XOR2, 6, 1, 0}, {EC_GF_OP_XOR2, 5, 2, 0},
+    {EC_GF_OP_XOR2, 0, 2, 0}, {EC_GF_OP_XOR2, 1, 3, 0},
+    {EC_GF_OP_XOR3, 8, 5, 6}, {EC_GF_OP_XOR2, 2, 7, 0},
+    {EC_GF_OP_XOR2, 0, 4, 0}, {EC_GF_OP_XOR2, 3, 7, 0},
+    {EC_GF_OP_XOR2, 4, 8, 0}, {EC_GF_OP_XOR2, 7, 8, 0},
+    {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_A5 = {9,
+                                    {
+                                        1,
+                                        4,
+                                        2,
+                                        5,
+                                        6,
+                                        7,
+                                        3,
+                                        0,
+                                        8,
+                                    },
+                                    ec_gf8_mul_A5_ops};
+
+static ec_gf_op_t ec_gf8_mul_A6_ops[] = {
+    {EC_GF_OP_XOR2, 2, 0, 0}, {EC_GF_OP_XOR2, 4, 6, 0},
+    {EC_GF_OP_XOR2, 3, 5, 0}, {EC_GF_OP_XOR2, 2, 4, 0},
+    {EC_GF_OP_XOR2, 3, 7, 0}, {EC_GF_OP_XOR2, 7, 2, 0},
+    {EC_GF_OP_XOR2, 1, 3, 0}, {EC_GF_OP_XOR2, 5, 7, 0},
+    {EC_GF_OP_XOR2, 6, 1, 0}, {EC_GF_OP_XOR2, 4, 1, 0},
+    {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_A6 = {8,
+                                    {
+                                        1,
+                                        2,
+                                        0,
+                                        3,
+                                        4,
+                                        5,
+                                        6,
+                                        7,
+                                    },
+                                    ec_gf8_mul_A6_ops};
+
+static ec_gf_op_t ec_gf8_mul_A7_ops[] = {
+    {EC_GF_OP_XOR2, 2, 0, 0}, {EC_GF_OP_XOR2, 3, 1, 0},
+    {EC_GF_OP_XOR2, 5, 7, 0}, {EC_GF_OP_XOR2, 4, 2, 0},
+    {EC_GF_OP_XOR2, 3, 5, 0}, {EC_GF_OP_XOR2, 4, 6, 0},
+    {EC_GF_OP_XOR2, 0, 3, 0}, {EC_GF_OP_XOR2, 6, 3, 0},
+    {EC_GF_OP_XOR2, 1, 4, 0}, {EC_GF_OP_XOR2, 7, 4, 0},
+    {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_A7 = {8,
+                                    {
+                                        0,
+                                        1,
+                                        2,
+                                        5,
+                                        6,
+                                        7,
+                                        3,
+                                        4,
+                                    },
+                                    ec_gf8_mul_A7_ops};
+
+static ec_gf_op_t ec_gf8_mul_A8_ops[] = {
+    {EC_GF_OP_XOR2, 0, 2, 0}, {EC_GF_OP_XOR2, 0, 7, 0},
+    {EC_GF_OP_COPY, 8, 0, 0}, {EC_GF_OP_XOR2, 8, 1, 0},
+    {EC_GF_OP_XOR2, 1, 6, 0}, {EC_GF_OP_COPY, 9, 4, 0},
+    {EC_GF_OP_XOR2, 0, 5, 0}, {EC_GF_OP_XOR2, 4, 1, 0},
+    {EC_GF_OP_XOR2, 5, 1, 0}, {EC_GF_OP_XOR2, 8, 3, 0},
+    {EC_GF_OP_XOR2, 1, 3, 0}, {EC_GF_OP_XOR2, 3, 2, 0},
+    {EC_GF_OP_XOR2, 2, 9, 0}, {EC_GF_OP_XOR2, 3, 0, 0},
+    {EC_GF_OP_XOR2, 7, 2, 0}, {EC_GF_OP_XOR2, 6, 2, 0},
+    {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_A8 = {10,
+                                    {
+                                        1,
+                                        7,
+                                        5,
+                                        8,
+                                        6,
+                                        3,
+                                        4,
+                                        0,
+                                        2,
+                                        9,
+                                    },
+                                    ec_gf8_mul_A8_ops};
+
+static ec_gf_op_t ec_gf8_mul_A9_ops[] = {
+    {EC_GF_OP_XOR2, 4, 1, 0}, {EC_GF_OP_XOR2, 1, 0, 0},
+    {EC_GF_OP_XOR2, 5, 0, 0}, {EC_GF_OP_XOR2, 0, 3, 0},
+    {EC_GF_OP_XOR2, 0, 7, 0}, {EC_GF_OP_XOR2, 5, 2, 0},
+    {EC_GF_OP_XOR2, 7, 2, 0}, {EC_GF_OP_XOR2, 2, 6, 0},
+    {EC_GF_OP_XOR2, 6, 1, 0}, {EC_GF_OP_XOR2, 1, 7, 0},
+    {EC_GF_OP_XOR2, 3, 6, 0}, {EC_GF_OP_XOR2, 7, 4, 0},
+    {EC_GF_OP_XOR2, 6, 5, 0}, {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_A9 = {8,
+                                    {
+                                        3,
+                                        7,
+                                        6,
+                                        1,
+                                        2,
+                                        0,
+                                        4,
+                                        5,
+                                    },
+                                    ec_gf8_mul_A9_ops};
+
+static ec_gf_op_t ec_gf8_mul_AA_ops[] = {
+    {EC_GF_OP_XOR2, 5, 6, 0}, {EC_GF_OP_XOR2, 3, 0, 0},
+    {EC_GF_OP_XOR2, 4, 5, 0}, {EC_GF_OP_XOR2, 6, 3, 0},
+    {EC_GF_OP_XOR2, 5, 7, 0}, {EC_GF_OP_XOR2, 2, 0, 0},
+    {EC_GF_OP_XOR2, 7, 6, 0}, {EC_GF_OP_XOR2, 2, 5, 0},
+    {EC_GF_OP_XOR2, 0, 1, 0}, {EC_GF_OP_XOR2, 3, 1, 0},
+    {EC_GF_OP_XOR2, 0, 7, 0}, {EC_GF_OP_XOR2, 1, 4, 0},
+    {EC_GF_OP_XOR2, 7, 4, 0}, {EC_GF_OP_XOR2, 4, 2, 0},
+    {EC_GF_OP_XOR2, 6, 4, 0}, {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_AA = {8,
+                                    {
+                                        0,
+                                        4,
+                                        5,
+                                        3,
+                                        6,
+                                        7,
+                                        1,
+                                        2,
+                                    },
+                                    ec_gf8_mul_AA_ops};
+
+static ec_gf_op_t ec_gf8_mul_AB_ops[] = {
+    {EC_GF_OP_COPY, 8, 0, 0}, {EC_GF_OP_XOR2, 0, 1, 0},
+    {EC_GF_OP_XOR2, 1, 4, 0}, {EC_GF_OP_XOR2, 1, 5, 0},
+    {EC_GF_OP_XOR2, 6, 1, 0}, {EC_GF_OP_XOR2, 2, 0, 0},
+    {EC_GF_OP_COPY, 9, 6, 0}, {EC_GF_OP_XOR2, 6, 2, 0},
+    {EC_GF_OP_XOR2, 4, 6, 0}, {EC_GF_OP_XOR2, 7, 4, 0},
+    {EC_GF_OP_XOR2, 8, 7, 0}, {EC_GF_OP_XOR2, 3, 8, 0},
+    {EC_GF_OP_XOR2, 5, 3, 0}, {EC_GF_OP_XOR2, 6, 3, 0},
+    {EC_GF_OP_XOR2, 2, 5, 0}, {EC_GF_OP_XOR3, 3, 9, 7},
+    {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_AB = {10,
+                                    {
+                                        2,
+                                        3,
+                                        8,
+                                        0,
+                                        5,
+                                        6,
+                                        1,
+                                        4,
+                                        7,
+                                        9,
+                                    },
+                                    ec_gf8_mul_AB_ops};
+
+static ec_gf_op_t ec_gf8_mul_AC_ops[] = {
+    {EC_GF_OP_XOR2, 5, 0, 0}, {EC_GF_OP_XOR2, 0, 2, 0},
+    {EC_GF_OP_XOR2, 2, 4, 0}, {EC_GF_OP_XOR2, 4, 7, 0},
+    {EC_GF_OP_XOR2, 7, 0, 0}, {EC_GF_OP_XOR2, 0, 3, 0},
+    {EC_GF_OP_XOR2, 0, 6, 0}, {EC_GF_OP_XOR2, 3, 1, 0},
+    {EC_GF_OP_XOR2, 6, 1, 0}, {EC_GF_OP_XOR2, 1, 5, 0},
+    {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_AC = {8,
+                                    {
+                                        3,
+                                        2,
+                                        1,
+                                        0,
+                                        4,
+                                        5,
+                                        6,
+                                        7,
+                                    },
+                                    ec_gf8_mul_AC_ops};
+
+static ec_gf_op_t ec_gf8_mul_AD_ops[] = {
+    {EC_GF_OP_XOR3, 8, 1, 2}, {EC_GF_OP_XOR2, 2, 0, 0},
+    {EC_GF_OP_XOR2, 3, 1, 0}, {EC_GF_OP_XOR2, 5, 0, 0},
+    {EC_GF_OP_XOR2, 3, 0, 0}, {EC_GF_OP_XOR2, 4, 8, 0},
+    {EC_GF_OP_XOR2, 5, 8, 0}, {EC_GF_OP_XOR2, 6, 2, 0},
+    {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_AD = {9,
+                                    {
+                                        3,
+                                        4,
+                                        5,
+                                        6,
+                                        7,
+                                        0,
+                                        1,
+                                        2,
+                                        8,
+                                    },
+                                    ec_gf8_mul_AD_ops};
+
+static ec_gf_op_t ec_gf8_mul_AE_ops[] = {
+    {EC_GF_OP_XOR2, 6, 5, 0}, {EC_GF_OP_XOR2, 5, 0, 0},
+    {EC_GF_OP_COPY, 8, 5, 0}, {EC_GF_OP_XOR2, 5, 7, 0},
+    {EC_GF_OP_XOR2, 7, 1, 0}, {EC_GF_OP_XOR2, 0, 2, 0},
+    {EC_GF_OP_XOR2, 1, 6, 0}, {EC_GF_OP_XOR2, 7, 3, 0},
+    {EC_GF_OP_XOR2, 6, 5, 0}, {EC_GF_OP_XOR2, 2, 6, 0},
+    {EC_GF_OP_XOR2, 0, 4, 0}, {EC_GF_OP_XOR2, 6, 3, 0},
+    {EC_GF_OP_XOR2, 3, 4, 0}, {EC_GF_OP_XOR2, 4, 8, 0},
+    {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_AE = {9,
+                                    {
+                                        7,
+                                        0,
+                                        5,
+                                        6,
+                                        3,
+                                        4,
+                                        1,
+                                        2,
+                                        8,
+                                    },
+                                    ec_gf8_mul_AE_ops};
+
+static ec_gf_op_t ec_gf8_mul_AF_ops[] = {
+    {EC_GF_OP_XOR2, 4, 0, 0}, {EC_GF_OP_XOR2, 6, 0, 0},
+    {EC_GF_OP_XOR2, 0, 1, 0}, {EC_GF_OP_XOR2, 0, 7, 0},
+    {EC_GF_OP_XOR2, 5, 1, 0}, {EC_GF_OP_XOR2, 7, 6, 0},
+    {EC_GF_OP_XOR2, 1, 2, 0}, {EC_GF_OP_XOR2, 6, 2, 0},
+    {EC_GF_OP_XOR2, 2, 5, 0}, {EC_GF_OP_XOR2, 1, 4, 0},
+    {EC_GF_OP_XOR2, 2, 0, 0}, {EC_GF_OP_XOR2, 0, 3, 0},
+    {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_AF = {8,
+                                    {
+                                        0,
+                                        1,
+                                        2,
+                                        7,
+                                        3,
+                                        4,
+                                        5,
+                                        6,
+                                    },
+                                    ec_gf8_mul_AF_ops};
+
+static ec_gf_op_t ec_gf8_mul_B0_ops[] = {
+    {EC_GF_OP_XOR2, 4, 1, 0}, {EC_GF_OP_XOR2, 3, 6, 0},
+    {EC_GF_OP_XOR2, 7, 4, 0}, {EC_GF_OP_XOR2, 4, 3, 0},
+    {EC_GF_OP_XOR2, 2, 4, 0}, {EC_GF_OP_XOR2, 1, 0, 0},
+    {EC_GF_OP_XOR2, 4, 5, 0}, {EC_GF_OP_XOR2, 6, 2, 0},
+    {EC_GF_OP_XOR2, 1, 6, 0}, {EC_GF_OP_XOR2, 0, 4, 0},
+    {EC_GF_OP_XOR2, 5, 1, 0}, {EC_GF_OP_XOR2, 1, 7, 0},
+    {EC_GF_OP_XOR2, 0, 1, 0}, {EC_GF_OP_XOR2, 3, 1, 0},
+    {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_B0 = {8,
+                                    {
+                                        4,
+                                        0,
+                                        7,
+                                        2,
+                                        3,
+                                        1,
+                                        6,
+                                        5,
+                                    },
+                                    ec_gf8_mul_B0_ops};
+
+static ec_gf_op_t ec_gf8_mul_B1_ops[] = {
+    {EC_GF_OP_XOR2, 4, 1, 0}, {EC_GF_OP_COPY, 8, 4, 0},
+    {EC_GF_OP_XOR2, 2, 7, 0}, {EC_GF_OP_XOR2, 4, 2, 0},
+    {EC_GF_OP_XOR2, 2, 3, 0}, {EC_GF_OP_XOR2, 6, 4, 0},
+    {EC_GF_OP_XOR2, 1, 0, 0}, {EC_GF_OP_XOR2, 2, 5, 0},
+    {EC_GF_OP_XOR2, 0, 6, 0}, {EC_GF_OP_XOR2, 7, 6, 0},
+    {EC_GF_OP_XOR2, 1, 2, 0}, {EC_GF_OP_XOR2, 6, 5, 0},
+    {EC_GF_OP_XOR2, 3, 7, 0}, {EC_GF_OP_XOR2, 2, 0, 0},
+    {EC_GF_OP_XOR3, 5, 8, 1}, {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_B1 = {9,
+                                    {
+                                        2,
+                                        6,
+                                        4,
+                                        7,
+                                        0,
+                                        1,
+                                        3,
+                                        5,
+                                        8,
+                                    },
+                                    ec_gf8_mul_B1_ops};
+
+static ec_gf_op_t ec_gf8_mul_B2_ops[] = {
+    {EC_GF_OP_XOR2, 1, 3, 0}, {EC_GF_OP_XOR2, 2, 1, 0},
+    {EC_GF_OP_XOR2, 0, 2, 0}, {EC_GF_OP_XOR2, 6, 0, 0},
+    {EC_GF_OP_XOR2, 1, 6, 0}, {EC_GF_OP_XOR3, 8, 4, 5},
+    {EC_GF_OP_XOR2, 2, 8, 0}, {EC_GF_OP_XOR2, 8, 1, 0},
+    {EC_GF_OP_XOR2, 7, 8, 0}, {EC_GF_OP_XOR2, 3, 8, 0},
+    {EC_GF_OP_XOR2, 0, 7, 0}, {EC_GF_OP_XOR2, 5, 0, 0},
+    {EC_GF_OP_XOR2, 1, 5, 0}, {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_B2 = {9,
+                                    {
+                                        0,
+                                        7,
+                                        4,
+                                        5,
+                                        6,
+                                        1,
+                                        2,
+                                        3,
+                                        8,
+                                    },
+                                    ec_gf8_mul_B2_ops};
+
+static ec_gf_op_t ec_gf8_mul_B3_ops[] = {
+    {EC_GF_OP_XOR2, 5, 0, 0}, {EC_GF_OP_COPY, 9, 5, 0},
+    {EC_GF_OP_XOR2, 4, 2, 0}, {EC_GF_OP_XOR3, 8, 6, 4},
+    {EC_GF_OP_XOR2, 5, 3, 0}, {EC_GF_OP_XOR2, 8, 5, 0},
+    {EC_GF_OP_XOR2, 0, 1, 0}, {EC_GF_OP_XOR2, 7, 8, 0},
+    {EC_GF_OP_XOR2, 0, 8, 0}, {EC_GF_OP_XOR2, 1, 7, 0},
+    {EC_GF_OP_XOR2, 2, 1, 0}, {EC_GF_OP_XOR2, 6, 1, 0},
+    {EC_GF_OP_XOR2, 3, 1, 0}, {EC_GF_OP_XOR2, 5, 2, 0},
+    {EC_GF_OP_XOR3, 1, 9, 0}, {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_B3 = {10,
+                                    {
+                                        2,
+                                        3,
+                                        4,
+                                        5,
+                                        1,
+                                        6,
+                                        0,
+                                        7,
+                                        8,
+                                        9,
+                                    },
+                                    ec_gf8_mul_B3_ops};
+
+static ec_gf_op_t ec_gf8_mul_B4_ops[] = {
+    {EC_GF_OP_XOR2, 3, 1, 0}, {EC_GF_OP_XOR2, 4, 3, 0},
+    {EC_GF_OP_XOR2, 1, 0, 0}, {EC_GF_OP_XOR2, 3, 2, 0},
+    {EC_GF_OP_XOR2, 2, 1, 0}, {EC_GF_OP_XOR2, 5, 4, 0},
+    {EC_GF_OP_XOR2, 4, 2, 0}, {EC_GF_OP_XOR2, 6, 3, 0},
+    {EC_GF_OP_XOR2, 7, 4, 0}, {EC_GF_OP_XOR2, 0, 7, 0},
+    {EC_GF_OP_XOR2, 7, 6, 0}, {EC_GF_OP_XOR2, 6, 5, 0},
+    {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_B4 = {8,
+                                    {
+                                        5,
+                                        6,
+                                        7,
+                                        0,
+                                        1,
+                                        2,
+                                        3,
+                                        4,
+                                    },
+                                    ec_gf8_mul_B4_ops};
+
+static ec_gf_op_t ec_gf8_mul_B5_ops[] = {
+    {EC_GF_OP_XOR2, 1, 0, 0}, {EC_GF_OP_XOR2, 0, 2, 0},
+    {EC_GF_OP_XOR2, 0, 3, 0}, {EC_GF_OP_XOR2, 6, 0, 0},
+    {EC_GF_OP_COPY, 8, 6, 0}, {EC_GF_OP_XOR2, 6, 1, 0},
+    {EC_GF_OP_XOR2, 1, 4, 0}, {EC_GF_OP_XOR2, 4, 2, 0},
+    {EC_GF_OP_XOR2, 5, 1, 0}, {EC_GF_OP_XOR2, 7, 4, 0},
+    {EC_GF_OP_XOR2, 3, 5, 0}, {EC_GF_OP_XOR2, 0, 7, 0},
+    {EC_GF_OP_XOR2, 5, 4, 0}, {EC_GF_OP_XOR2, 2, 0, 0},
+    {EC_GF_OP_XOR3, 4, 8, 3}, {EC_GF_OP_XOR2, 0, 6, 0},
+    {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_B5 = {9,
+                                    {
+                                        3,
+                                        4,
+                                        0,
+                                        7,
+                                        1,
+                                        5,
+                                        6,
+                                        2,
+                                        8,
+                                    },
+                                    ec_gf8_mul_B5_ops};
+
+static ec_gf_op_t ec_gf8_mul_B6_ops[] = {
+    {EC_GF_OP_XOR2, 7, 1, 0}, {EC_GF_OP_XOR2, 0, 4, 0},
+    {EC_GF_OP_XOR2, 1, 2, 0}, {EC_GF_OP_XOR2, 4, 3, 0},
+    {EC_GF_OP_XOR2, 2, 3, 0}, {EC_GF_OP_XOR2, 7, 4, 0},
+    {EC_GF_OP_XOR2, 3, 5, 0}, {EC_GF_OP_XOR2, 5, 7, 0},
+    {EC_GF_OP_XOR2, 6, 0, 0}, {EC_GF_OP_XOR2, 7, 0, 0},
+    {EC_GF_OP_XOR2, 0, 1, 0}, {EC_GF_OP_XOR2, 2, 6, 0},
+    {EC_GF_OP_XOR2, 1, 3, 0}, {EC_GF_OP_XOR2, 3, 2, 0},
+    {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_B6 = {8,
+                                    {
+                                        5,
+                                        3,
+                                        6,
+                                        4,
+                                        7,
+                                        0,
+                                        1,
+                                        2,
+                                    },
+                                    ec_gf8_mul_B6_ops};
+
+static ec_gf_op_t ec_gf8_mul_B7_ops[] = {
+    {EC_GF_OP_XOR2, 2, 1, 0}, {EC_GF_OP_XOR2, 0, 4, 0},
+    {EC_GF_OP_XOR2, 2, 6, 0}, {EC_GF_OP_XOR2, 0, 2, 0},
+    {EC_GF_OP_XOR2, 1, 0, 0}, {EC_GF_OP_XOR2, 0, 5, 0},
+    {EC_GF_OP_XOR2, 7, 1, 0}, {EC_GF_OP_XOR2, 3, 7, 0},
+    {EC_GF_OP_XOR2, 6, 0, 0}, {EC_GF_OP_XOR2, 2, 3, 0},
+    {EC_GF_OP_XOR2, 5, 2, 0}, {EC_GF_OP_XOR2, 7, 5, 0},
+    {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_B7 = {8,
+                                    {
+                                        5,
+                                        0,
+                                        1,
+                                        4,
+                                        2,
+                                        6,
+                                        7,
+                                        3,
+                                    },
+                                    ec_gf8_mul_B7_ops};
+
+static ec_gf_op_t ec_gf8_mul_B8_ops[] = {
+    {EC_GF_OP_XOR2, 2, 5, 0}, {EC_GF_OP_XOR2, 7, 2, 0},
+    {EC_GF_OP_XOR2, 2, 0, 0}, {EC_GF_OP_XOR2, 0, 1, 0},
+    {EC_GF_OP_XOR2, 6, 3, 0}, {EC_GF_OP_XOR2, 3, 2, 0},
+    {EC_GF_OP_XOR2, 1, 4, 0}, {EC_GF_OP_XOR2, 0, 6, 0},
+    {EC_GF_OP_XOR2, 4, 7, 0}, {EC_GF_OP_XOR2, 5, 1, 0},
+    {EC_GF_OP_XOR2, 6, 1, 0}, {EC_GF_OP_XOR2, 7, 5, 0},
+    {EC_GF_OP_XOR2, 1, 3, 0}, {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_B8 = {8,
+                                    {
+                                        6,
+                                        4,
+                                        5,
+                                        1,
+                                        2,
+                                        0,
+                                        7,
+                                        3,
+                                    },
+                                    ec_gf8_mul_B8_ops};
+
+static ec_gf_op_t ec_gf8_mul_B9_ops[] = {
+    {EC_GF_OP_COPY, 8, 0, 0}, {EC_GF_OP_XOR2, 0, 1, 0},
+    {EC_GF_OP_XOR2, 2, 5, 0}, {EC_GF_OP_XOR2, 0, 4, 0},
+    {EC_GF_OP_XOR2, 2, 0, 0}, {EC_GF_OP_XOR2, 3, 0, 0},
+    {EC_GF_OP_XOR3, 0, 8, 2}, {EC_GF_OP_XOR2, 6, 3, 0},
+    {EC_GF_OP_XOR2, 1, 2, 0}, {EC_GF_OP_XOR2, 7, 0, 0},
+    {EC_GF_OP_XOR2, 5, 6, 0}, {EC_GF_OP_XOR2, 3, 7, 0},
+    {EC_GF_OP_XOR2, 4, 5, 0}, {EC_GF_OP_XOR2, 5, 3, 0},
+    {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_B9 = {9,
+                                    {
+                                        6,
+                                        7,
+                                        0,
+                                        2,
+                                        1,
+                                        4,
+                                        5,
+                                        3,
+                                        8,
+                                    },
+                                    ec_gf8_mul_B9_ops};
+
+static ec_gf_op_t ec_gf8_mul_BA_ops[] = {
+    {EC_GF_OP_XOR2, 2, 0, 0}, {EC_GF_OP_XOR2, 5, 7, 0},
+    {EC_GF_OP_XOR2, 4, 5, 0}, {EC_GF_OP_XOR2, 3, 2, 0},
+    {EC_GF_OP_XOR2, 2, 4, 0}, {EC_GF_OP_XOR2, 5, 3, 0},
+    {EC_GF_OP_XOR2, 1, 2, 0}, {EC_GF_OP_XOR2, 6, 5, 0},
+    {EC_GF_OP_XOR2, 0, 1, 0}, {EC_GF_OP_XOR2, 7, 6, 0},
+    {EC_GF_OP_XOR2, 3, 0, 0}, {EC_GF_OP_XOR2, 6, 0, 0},
+    {EC_GF_OP_XOR2, 1, 7, 0}, {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_BA = {8,
+                                    {
+                                        1,
+                                        2,
+                                        4,
+                                        3,
+                                        5,
+                                        6,
+                                        0,
+                                        7,
+                                    },
+                                    ec_gf8_mul_BA_ops};
+
+static ec_gf_op_t ec_gf8_mul_BB_ops[] = {
+    {EC_GF_OP_XOR2, 3, 6, 0}, {EC_GF_OP_COPY, 8, 3, 0},
+    {EC_GF_OP_XOR2, 1, 0, 0}, {EC_GF_OP_XOR2, 3, 1, 0},
+    {EC_GF_OP_XOR2, 4, 3, 0}, {EC_GF_OP_XOR2, 8, 5, 0},
+    {EC_GF_OP_XOR2, 7, 4, 0}, {EC_GF_OP_XOR2, 5, 4, 0},
+    {EC_GF_OP_XOR2, 8, 7, 0}, {EC_GF_OP_XOR2, 2, 8, 0},
+    {EC_GF_OP_XOR2, 0, 2, 0}, {EC_GF_OP_XOR2, 1, 2, 0},
+    {EC_GF_OP_XOR2, 6, 0, 0}, {EC_GF_OP_XOR2, 4, 0, 0},
+    {EC_GF_OP_XOR2, 3, 6, 0}, {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_BB = {9,
+                                    {
+                                        7,
+                                        2,
+                                        1,
+                                        8,
+                                        3,
+                                        5,
+                                        6,
+                                        4,
+                                        0,
+                                    },
+                                    ec_gf8_mul_BB_ops};
+
+static ec_gf_op_t ec_gf8_mul_BC_ops[] = {
+    {EC_GF_OP_COPY, 8, 1, 0}, {EC_GF_OP_XOR2, 8, 2, 0},
+    {EC_GF_OP_XOR2, 6, 0, 0}, {EC_GF_OP_XOR2, 2, 3, 0},
+    {EC_GF_OP_XOR2, 6, 7, 0}, {EC_GF_OP_XOR2, 0, 2, 0},
+    {EC_GF_OP_XOR2, 4, 2, 0}, {EC_GF_OP_XOR2, 1, 6, 0},
+    {EC_GF_OP_XOR2, 7, 8, 0}, {EC_GF_OP_XOR3, 2, 8, 4},
+    {EC_GF_OP_XOR2, 5, 6, 0}, {EC_GF_OP_XOR2, 4, 5, 0},
+    {EC_GF_OP_XOR2, 3, 4, 0}, {EC_GF_OP_XOR2, 6, 3, 0},
+    {EC_GF_OP_XOR2, 3, 7, 0}, {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_BC = {9,
+                                    {
+                                        2,
+                                        6,
+                                        3,
+                                        4,
+                                        5,
+                                        1,
+                                        7,
+                                        0,
+                                        8,
+                                    },
+                                    ec_gf8_mul_BC_ops};
+
+static ec_gf_op_t ec_gf8_mul_BD_ops[] = {
+    {EC_GF_OP_XOR2, 3, 0, 0}, {EC_GF_OP_XOR2, 3, 1, 0},
+    {EC_GF_OP_XOR2, 1, 2, 0}, {EC_GF_OP_XOR2, 7, 1, 0},
+    {EC_GF_OP_XOR2, 1, 4, 0}, {EC_GF_OP_XOR2, 4, 3, 0},
+    {EC_GF_OP_XOR2, 5, 1, 0}, {EC_GF_OP_XOR2, 3, 7, 0},
+    {EC_GF_OP_XOR2, 0, 6, 0}, {EC_GF_OP_XOR2, 0, 5, 0},
+    {EC_GF_OP_XOR2, 6, 7, 0}, {EC_GF_OP_XOR2, 7, 0, 0},
+    {EC_GF_OP_XOR2, 2, 7, 0}, {EC_GF_OP_XOR2, 1, 2, 0},
+    {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_BD = {8,
+                                    {
+                                        4,
+                                        5,
+                                        0,
+                                        2,
+                                        7,
+                                        1,
+                                        6,
+                                        3,
+                                    },
+                                    ec_gf8_mul_BD_ops};
+
+static ec_gf_op_t ec_gf8_mul_BE_ops[] = {
+    {EC_GF_OP_XOR2, 0, 3, 0}, {EC_GF_OP_XOR2, 0, 6, 0},
+    {EC_GF_OP_XOR2, 1, 5, 0}, {EC_GF_OP_XOR2, 5, 0, 0},
+    {EC_GF_OP_XOR2, 4, 5, 0}, {EC_GF_OP_XOR2, 3, 4, 0},
+    {EC_GF_OP_XOR2, 7, 3, 0}, {EC_GF_OP_XOR2, 3, 2, 0},
+    {EC_GF_OP_XOR2, 1, 7, 0}, {EC_GF_OP_XOR2, 2, 0, 0},
+    {EC_GF_OP_XOR2, 6, 3, 0}, {EC_GF_OP_XOR2, 0, 1, 0},
+    {EC_GF_OP_XOR2, 3, 1, 0}, {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_BE = {8,
+                                    {
+                                        0,
+                                        6,
+                                        7,
+                                        4,
+                                        5,
+                                        1,
+                                        3,
+                                        2,
+                                    },
+                                    ec_gf8_mul_BE_ops};
+
+static ec_gf_op_t ec_gf8_mul_BF_ops[] = {
+    {EC_GF_OP_XOR2, 7, 1, 0}, {EC_GF_OP_XOR2, 6, 7, 0},
+    {EC_GF_OP_XOR2, 5, 6, 0}, {EC_GF_OP_XOR2, 4, 5, 0},
+    {EC_GF_OP_XOR2, 0, 4, 0}, {EC_GF_OP_XOR2, 1, 2, 0},
+    {EC_GF_OP_XOR2, 7, 0, 0}, {EC_GF_OP_XOR2, 2, 5, 0},
+    {EC_GF_OP_XOR2, 1, 0, 0}, {EC_GF_OP_XOR2, 6, 1, 0},
+    {EC_GF_OP_XOR2, 4, 6, 0}, {EC_GF_OP_XOR2, 4, 3, 0},
+    {EC_GF_OP_XOR2, 3, 7, 0}, {EC_GF_OP_XOR2, 5, 3, 0},
+    {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_BF = {8,
+                                    {
+                                        5,
+                                        6,
+                                        1,
+                                        7,
+                                        3,
+                                        0,
+                                        2,
+                                        4,
+                                    },
+                                    ec_gf8_mul_BF_ops};
+
+static ec_gf_op_t ec_gf8_mul_C0_ops[] = {
+    {EC_GF_OP_XOR2, 4, 1, 0}, {EC_GF_OP_XOR2, 6, 3, 0},
+    {EC_GF_OP_XOR2, 7, 4, 0}, {EC_GF_OP_XOR2, 4, 6, 0},
+    {EC_GF_OP_XOR2, 5, 2, 0}, {EC_GF_OP_XOR2, 2, 6, 0},
+    {EC_GF_OP_XOR2, 3, 5, 0}, {EC_GF_OP_XOR2, 6, 0, 0},
+    {EC_GF_OP_XOR2, 1, 5, 0}, {EC_GF_OP_XOR2, 3, 7, 0},
+    {EC_GF_OP_XOR2, 0, 7, 0}, {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_C0 = {8,
+                                    {
+                                        1,
+                                        2,
+                                        3,
+                                        4,
+                                        7,
+                                        5,
+                                        6,
+                                        0,
+                                    },
+                                    ec_gf8_mul_C0_ops};
+
+static ec_gf_op_t ec_gf8_mul_C1_ops[] = {
+    {EC_GF_OP_XOR3, 8, 1, 2}, {EC_GF_OP_XOR2, 8, 3, 0},
+    {EC_GF_OP_XOR2, 4, 1, 0}, {EC_GF_OP_XOR2, 3, 0, 0},
+    {EC_GF_OP_XOR2, 0, 4, 0}, {EC_GF_OP_XOR2, 1, 7, 0},
+    {EC_GF_OP_XOR2, 5, 3, 0}, {EC_GF_OP_XOR2, 7, 0, 0},
+    {EC_GF_OP_XOR2, 4, 6, 0}, {EC_GF_OP_XOR2, 7, 5, 0},
+    {EC_GF_OP_XOR2, 6, 8, 0}, {EC_GF_OP_XOR2, 5, 8, 0},
+    {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_C1 = {9,
+                                    {
+                                        5,
+                                        6,
+                                        7,
+                                        4,
+                                        1,
+                                        2,
+                                        3,
+                                        0,
+                                        8,
+                                    },
+                                    ec_gf8_mul_C1_ops};
+
+static ec_gf_op_t ec_gf8_mul_C2_ops[] = {
+    {EC_GF_OP_XOR2, 0, 2, 0}, {EC_GF_OP_XOR2, 1, 3, 0},
+    {EC_GF_OP_XOR2, 0, 3, 0}, {EC_GF_OP_XOR2, 1, 4, 0},
+    {EC_GF_OP_XOR2, 6, 0, 0}, {EC_GF_OP_XOR2, 4, 2, 0},
+    {EC_GF_OP_XOR2, 2, 6, 0}, {EC_GF_OP_XOR2, 4, 5, 0},
+    {EC_GF_OP_XOR2, 5, 2, 0}, {EC_GF_OP_XOR2, 7, 1, 0},
+    {EC_GF_OP_XOR2, 3, 4, 0}, {EC_GF_OP_XOR2, 2, 7, 0},
+    {EC_GF_OP_XOR2, 7, 3, 0}, {EC_GF_OP_XOR2, 0, 2, 0},
+    {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_C2 = {8,
+                                    {
+                                        7,
+                                        6,
+                                        3,
+                                        0,
+                                        1,
+                                        4,
+                                        5,
+                                        2,
+                                    },
+                                    ec_gf8_mul_C2_ops};
+
+static ec_gf_op_t ec_gf8_mul_C3_ops[] = {
+    {EC_GF_OP_COPY, 8, 0, 0}, {EC_GF_OP_XOR2, 0, 1, 0},
+    {EC_GF_OP_XOR2, 0, 2, 0}, {EC_GF_OP_XOR2, 6, 0, 0},
+    {EC_GF_OP_XOR2, 2, 4, 0}, {EC_GF_OP_XOR2, 7, 0, 0},
+    {EC_GF_OP_XOR3, 0, 2, 6}, {EC_GF_OP_XOR2, 6, 3, 0},
+    {EC_GF_OP_XOR3, 9, 1, 0}, {EC_GF_OP_XOR2, 1, 3, 0},
+    {EC_GF_OP_XOR2, 3, 5, 0}, {EC_GF_OP_XOR2, 5, 7, 0},
+    {EC_GF_OP_XOR2, 4, 3, 0}, {EC_GF_OP_XOR2, 7, 9, 0},
+    {EC_GF_OP_XOR2, 3, 8, 0}, {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_C3 = {10,
+                                    {
+                                        5,
+                                        6,
+                                        4,
+                                        7,
+                                        1,
+                                        2,
+                                        3,
+                                        0,
+                                        8,
+                                        9,
+                                    },
+                                    ec_gf8_mul_C3_ops};
+
+static ec_gf_op_t ec_gf8_mul_C4_ops[] = {
+    {EC_GF_OP_XOR2, 3, 7, 0}, {EC_GF_OP_XOR2, 2, 3, 0},
+    {EC_GF_OP_XOR2, 3, 4, 0}, {EC_GF_OP_XOR2, 4, 5, 0},
+    {EC_GF_OP_XOR2, 5, 2, 0}, {EC_GF_OP_XOR2, 1, 0, 0},
+    {EC_GF_OP_XOR2, 2, 6, 0}, {EC_GF_OP_XOR2, 1, 4, 0},
+    {EC_GF_OP_XOR2, 6, 7, 0}, {EC_GF_OP_XOR2, 0, 3, 0},
+    {EC_GF_OP_XOR2, 7, 1, 0}, {EC_GF_OP_XOR2, 1, 2, 0},
+    {EC_GF_OP_XOR2, 6, 0, 0}, {EC_GF_OP_XOR2, 0, 1, 0},
+    {EC_GF_OP_XOR2, 4, 0, 0}, {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_C4 = {8,
+                                    {
+                                        0,
+                                        2,
+                                        1,
+                                        3,
+                                        4,
+                                        5,
+                                        6,
+                                        7,
+                                    },
+                                    ec_gf8_mul_C4_ops};
+
+static ec_gf_op_t ec_gf8_mul_C5_ops[] = {
+    {EC_GF_OP_XOR2, 0, 4, 0}, {EC_GF_OP_XOR2, 5, 0, 0},
+    {EC_GF_OP_XOR2, 6, 1, 0}, {EC_GF_OP_XOR2, 0, 3, 0},
+    {EC_GF_OP_XOR2, 1, 5, 0}, {EC_GF_OP_XOR2, 6, 2, 0},
+    {EC_GF_OP_XOR2, 3, 7, 0}, {EC_GF_OP_XOR2, 5, 6, 0},
+    {EC_GF_OP_XOR2, 7, 4, 0}, {EC_GF_OP_XOR2, 2, 3, 0},
+    {EC_GF_OP_XOR2, 4, 5, 0}, {EC_GF_OP_XOR2, 3, 6, 0},
+    {EC_GF_OP_XOR2, 5, 2, 0}, {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_C5 = {8,
+                                    {
+                                        4,
+                                        3,
+                                        5,
+                                        7,
+                                        6,
+                                        2,
+                                        0,
+                                        1,
+                                    },
+                                    ec_gf8_mul_C5_ops};
+
+static ec_gf_op_t ec_gf8_mul_C6_ops[] = {
+    {EC_GF_OP_XOR2, 4, 3, 0}, {EC_GF_OP_COPY, 8, 4, 0},
+    {EC_GF_OP_XOR2, 4, 2, 0}, {EC_GF_OP_XOR3, 9, 5, 4},
+    {EC_GF_OP_XOR2, 6, 9, 0}, {EC_GF_OP_XOR2, 0, 6, 0},
+    {EC_GF_OP_XOR2, 1, 7, 0}, {EC_GF_OP_XOR2, 2, 0, 0},
+    {EC_GF_OP_XOR2, 7, 9, 0}, {EC_GF_OP_XOR2, 6, 1, 0},
+    {EC_GF_OP_XOR2, 3, 2, 0}, {EC_GF_OP_XOR2, 5, 6, 0},
+    {EC_GF_OP_XOR2, 1, 3, 0}, {EC_GF_OP_XOR2, 6, 8, 0},
+    {EC_GF_OP_XOR2, 3, 7, 0}, {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_C6 = {10,
+                                    {
+                                        6,
+                                        3,
+                                        0,
+                                        4,
+                                        5,
+                                        7,
+                                        2,
+                                        1,
+                                        8,
+                                        9,
+                                    },
+                                    ec_gf8_mul_C6_ops};
+
+static ec_gf_op_t ec_gf8_mul_C7_ops[] = {
+    {EC_GF_OP_XOR2, 5, 0, 0}, {EC_GF_OP_XOR2, 5, 3, 0},
+    {EC_GF_OP_XOR2, 2, 4, 0}, {EC_GF_OP_XOR2, 4, 5, 0},
+    {EC_GF_OP_XOR2, 1, 3, 0}, {EC_GF_OP_XOR2, 6, 4, 0},
+    {EC_GF_OP_XOR2, 7, 2, 0}, {EC_GF_OP_XOR2, 1, 6, 0},
+    {EC_GF_OP_XOR2, 3, 7, 0}, {EC_GF_OP_XOR2, 7, 1, 0},
+    {EC_GF_OP_XOR2, 5, 7, 0}, {EC_GF_OP_XOR2, 0, 5, 0},
+    {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_C7 = {8,
+                                    {
+                                        7,
+                                        0,
+                                        6,
+                                        2,
+                                        5,
+                                        3,
+                                        4,
+                                        1,
+                                    },
+                                    ec_gf8_mul_C7_ops};
+
+static ec_gf_op_t ec_gf8_mul_C8_ops[] = {
+    {EC_GF_OP_XOR2, 6, 5, 0}, {EC_GF_OP_XOR2, 5, 0, 0},
+    {EC_GF_OP_XOR2, 0, 1, 0}, {EC_GF_OP_XOR2, 0, 7, 0},
+    {EC_GF_OP_XOR2, 7, 6, 0}, {EC_GF_OP_XOR2, 6, 4, 0},
+    {EC_GF_OP_XOR2, 4, 3, 0}, {EC_GF_OP_XOR2, 4, 1, 0},
+    {EC_GF_OP_XOR2, 3, 2, 0}, {EC_GF_OP_XOR2, 1, 2, 0},
+    {EC_GF_OP_XOR2, 2, 4, 0}, {EC_GF_OP_XOR2, 4, 5, 0},
+    {EC_GF_OP_XOR2, 5, 7, 0}, {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_C8 = {8,
+                                    {
+                                        1,
+                                        3,
+                                        2,
+                                        4,
+                                        6,
+                                        7,
+                                        5,
+                                        0,
+                                    },
+                                    ec_gf8_mul_C8_ops};
+
+static ec_gf_op_t ec_gf8_mul_C9_ops[] = {
+    {EC_GF_OP_XOR2, 1, 0, 0}, {EC_GF_OP_XOR2, 3, 0, 0},
+    {EC_GF_OP_XOR2, 0, 7, 0}, {EC_GF_OP_XOR2, 7, 6, 0},
+    {EC_GF_OP_XOR2, 4, 1, 0}, {EC_GF_OP_XOR2, 6, 5, 0},
+    {EC_GF_OP_XOR2, 5, 4, 0}, {EC_GF_OP_XOR2, 2, 1, 0},
+    {EC_GF_OP_XOR2, 4, 3, 0}, {EC_GF_OP_XOR2, 3, 2, 0},
+    {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_C9 = {8,
+                                    {
+                                        2,
+                                        3,
+                                        4,
+                                        5,
+                                        6,
+                                        7,
+                                        0,
+                                        1,
+                                    },
+                                    ec_gf8_mul_C9_ops};
+
+static ec_gf_op_t ec_gf8_mul_CA_ops[] = {
+    {EC_GF_OP_XOR2, 6, 7, 0}, {EC_GF_OP_XOR2, 7, 2, 0},
+    {EC_GF_OP_XOR2, 5, 6, 0}, {EC_GF_OP_XOR2, 2, 0, 0},
+    {EC_GF_OP_XOR2, 4, 5, 0}, {EC_GF_OP_XOR2, 2, 3, 0},
+    {EC_GF_OP_XOR2, 6, 1, 0}, {EC_GF_OP_XOR2, 3, 4, 0},
+    {EC_GF_OP_XOR2, 1, 7, 0}, {EC_GF_OP_XOR2, 6, 0, 0},
+    {EC_GF_OP_XOR2, 7, 3, 0}, {EC_GF_OP_XOR2, 0, 5, 0},
+    {EC_GF_OP_XOR2, 5, 7, 0}, {EC_GF_OP_XOR2, 7, 6, 0},
+    {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_CA = {8,
+                                    {
+                                        1,
+                                        2,
+                                        5,
+                                        7,
+                                        3,
+                                        4,
+                                        0,
+                                        6,
+                                    },
+                                    ec_gf8_mul_CA_ops};
+
+static ec_gf_op_t ec_gf8_mul_CB_ops[] = {
+    {EC_GF_OP_XOR2, 1, 0, 0}, {EC_GF_OP_XOR2, 2, 1, 0},
+    {EC_GF_OP_XOR2, 1, 6, 0}, {EC_GF_OP_XOR2, 6, 3, 0},
+    {EC_GF_OP_XOR2, 3, 2, 0}, {EC_GF_OP_XOR2, 2, 7, 0},
+    {EC_GF_OP_XOR2, 4, 2, 0}, {EC_GF_OP_XOR2, 7, 5, 0},
+    {EC_GF_OP_XOR2, 5, 4, 0}, {EC_GF_OP_XOR2, 0, 7, 0},
+    {EC_GF_OP_XOR2, 4, 3, 0}, {EC_GF_OP_XOR2, 7, 6, 0},
+    {EC_GF_OP_XOR2, 6, 4, 0}, {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_CB = {8,
+                                    {
+                                        2,
+                                        3,
+                                        4,
+                                        5,
+                                        7,
+                                        6,
+                                        0,
+                                        1,
+                                    },
+                                    ec_gf8_mul_CB_ops};
+
+static ec_gf_op_t ec_gf8_mul_CC_ops[] = {
+    {EC_GF_OP_XOR2, 7, 2, 0}, {EC_GF_OP_XOR2, 4, 7, 0},
+    {EC_GF_OP_XOR2, 0, 2, 0}, {EC_GF_OP_XOR2, 7, 3, 0},
+    {EC_GF_OP_XOR2, 2, 1, 0}, {EC_GF_OP_XOR2, 3, 5, 0},
+    {EC_GF_OP_XOR2, 1, 7, 0}, {EC_GF_OP_XOR2, 2, 6, 0},
+    {EC_GF_OP_XOR2, 5, 4, 0}, {EC_GF_OP_XOR2, 6, 3, 0},
+    {EC_GF_OP_XOR2, 4, 6, 0}, {EC_GF_OP_XOR2, 0, 4, 0},
+    {EC_GF_OP_XOR2, 3, 0, 0}, {EC_GF_OP_XOR2, 1, 3, 0},
+    {EC_GF_OP_XOR2, 4, 1, 0}, {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_CC = {8,
+                                    {
+                                        2,
+                                        7,
+                                        1,
+                                        0,
+                                        5,
+                                        6,
+                                        3,
+                                        4,
+                                    },
+                                    ec_gf8_mul_CC_ops};
+
+static ec_gf_op_t ec_gf8_mul_CD_ops[] = {
+    {EC_GF_OP_XOR2, 4, 0, 0}, {EC_GF_OP_XOR2, 3, 6, 0},
+    {EC_GF_OP_XOR2, 0, 1, 0}, {EC_GF_OP_XOR2, 6, 2, 0},
+    {EC_GF_OP_XOR2, 1, 3, 0}, {EC_GF_OP_XOR2, 2, 5, 0},
+    {EC_GF_OP_XOR2, 1, 4, 0}, {EC_GF_OP_XOR2, 5, 0, 0},
+    {EC_GF_OP_XOR2, 4, 7, 0}, {EC_GF_OP_XOR2, 0, 6, 0},
+    {EC_GF_OP_XOR2, 7, 2, 0}, {EC_GF_OP_XOR2, 6, 4, 0},
+    {EC_GF_OP_XOR2, 2, 6, 0}, {EC_GF_OP_XOR2, 6, 1, 0},
+    {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_CD = {8,
+                                    {
+                                        0,
+                                        6,
+                                        1,
+                                        2,
+                                        7,
+                                        3,
+                                        4,
+                                        5,
+                                    },
+                                    ec_gf8_mul_CD_ops};
+
+static ec_gf_op_t ec_gf8_mul_CE_ops[] = {
+    {EC_GF_OP_XOR2, 0, 2, 0}, {EC_GF_OP_XOR2, 3, 5, 0},
+    {EC_GF_OP_XOR2, 5, 0, 0}, {EC_GF_OP_XOR2, 7, 5, 0},
+    {EC_GF_OP_COPY, 8, 7, 0}, {EC_GF_OP_XOR2, 7, 3, 0},
+    {EC_GF_OP_XOR2, 3, 4, 0}, {EC_GF_OP_XOR2, 6, 3, 0},
+    {EC_GF_OP_XOR2, 2, 3, 0}, {EC_GF_OP_XOR3, 3, 6, 8},
+    {EC_GF_OP_XOR2, 0, 6, 0}, {EC_GF_OP_XOR3, 8, 2, 3},
+    {EC_GF_OP_XOR2, 1, 8, 0}, {EC_GF_OP_XOR2, 4, 8, 0},
+    {EC_GF_OP_XOR2, 5, 1, 0}, {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_CE = {9,
+                                    {
+                                        5,
+                                        7,
+                                        3,
+                                        0,
+                                        2,
+                                        6,
+                                        4,
+                                        1,
+                                        8,
+                                    },
+                                    ec_gf8_mul_CE_ops};
+
+static ec_gf_op_t ec_gf8_mul_CF_ops[] = {
+    {EC_GF_OP_XOR2, 1, 2, 0}, {EC_GF_OP_XOR2, 5, 3, 0},
+    {EC_GF_OP_XOR2, 3, 6, 0}, {EC_GF_OP_XOR2, 2, 5, 0},
+    {EC_GF_OP_XOR2, 1, 4, 0}, {EC_GF_OP_XOR2, 4, 3, 0},
+    {EC_GF_OP_XOR2, 0, 4, 0}, {EC_GF_OP_XOR2, 7, 0, 0},
+    {EC_GF_OP_XOR2, 0, 2, 0}, {EC_GF_OP_XOR2, 6, 7, 0},
+    {EC_GF_OP_XOR2, 5, 6, 0}, {EC_GF_OP_XOR2, 6, 1, 0},
+    {EC_GF_OP_XOR2, 1, 0, 0}, {EC_GF_OP_XOR2, 3, 6, 0},
+    {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_CF = {8,
+                                    {
+                                        3,
+                                        6,
+                                        7,
+                                        0,
+                                        2,
+                                        4,
+                                        5,
+                                        1,
+                                    },
+                                    ec_gf8_mul_CF_ops};
+
+static ec_gf_op_t ec_gf8_mul_D0_ops[] = {
+    {EC_GF_OP_XOR2, 5, 2, 0}, {EC_GF_OP_XOR2, 0, 3, 0},
+    {EC_GF_OP_XOR2, 3, 5, 0}, {EC_GF_OP_XOR2, 6, 3, 0},
+    {EC_GF_OP_XOR2, 4, 1, 0}, {EC_GF_OP_XOR2, 1, 6, 0},
+    {EC_GF_OP_XOR2, 5, 4, 0}, {EC_GF_OP_XOR2, 7, 1, 0},
+    {EC_GF_OP_XOR2, 4, 0, 0}, {EC_GF_OP_XOR2, 2, 7, 0},
+    {EC_GF_OP_XOR2, 0, 2, 0}, {EC_GF_OP_XOR2, 3, 2, 0},
+    {EC_GF_OP_XOR2, 1, 0, 0}, {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_D0 = {8,
+                                    {
+                                        5,
+                                        6,
+                                        7,
+                                        2,
+                                        0,
+                                        3,
+                                        1,
+                                        4,
+                                    },
+                                    ec_gf8_mul_D0_ops};
+
+static ec_gf_op_t ec_gf8_mul_D1_ops[] = {
+    {EC_GF_OP_XOR2, 1, 2, 0}, {EC_GF_OP_XOR2, 0, 4, 0},
+    {EC_GF_OP_XOR2, 5, 1, 0}, {EC_GF_OP_XOR2, 4, 6, 0},
+    {EC_GF_OP_XOR2, 6, 5, 0}, {EC_GF_OP_XOR2, 5, 0, 0},
+    {EC_GF_OP_XOR2, 7, 6, 0}, {EC_GF_OP_XOR2, 6, 3, 0},
+    {EC_GF_OP_XOR2, 2, 7, 0}, {EC_GF_OP_XOR2, 0, 2, 0},
+    {EC_GF_OP_XOR2, 3, 2, 0}, {EC_GF_OP_XOR3, 8, 6, 0},
+    {EC_GF_OP_XOR2, 4, 8, 0}, {EC_GF_OP_XOR2, 1, 8, 0},
+    {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_D1 = {9,
+                                    {
+                                        5,
+                                        6,
+                                        3,
+                                        2,
+                                        0,
+                                        7,
+                                        4,
+                                        1,
+                                        8,
+                                    },
+                                    ec_gf8_mul_D1_ops};
+
+static ec_gf_op_t ec_gf8_mul_D2_ops[] = {
+    {EC_GF_OP_XOR2, 3, 5, 0}, {EC_GF_OP_XOR2, 3, 6, 0},
+    {EC_GF_OP_XOR2, 2, 3, 0}, {EC_GF_OP_XOR2, 3, 0, 0},
+    {EC_GF_OP_XOR2, 0, 2, 0}, {EC_GF_OP_XOR2, 3, 1, 0},
+    {EC_GF_OP_XOR2, 7, 0, 0}, {EC_GF_OP_XOR2, 4, 3, 0},
+    {EC_GF_OP_XOR2, 1, 2, 0}, {EC_GF_OP_XOR2, 6, 7, 0},
+    {EC_GF_OP_XOR2, 5, 4, 0}, {EC_GF_OP_XOR2, 4, 6, 0},
+    {EC_GF_OP_XOR2, 7, 5, 0}, {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_D2 = {8,
+                                    {
+                                        7,
+                                        0,
+                                        2,
+                                        1,
+                                        3,
+                                        4,
+                                        6,
+                                        5,
+                                    },
+                                    ec_gf8_mul_D2_ops};
+
+static ec_gf_op_t ec_gf8_mul_D3_ops[] = {
+    {EC_GF_OP_XOR2, 4, 7, 0}, {EC_GF_OP_COPY, 8, 4, 0},
+    {EC_GF_OP_XOR2, 2, 1, 0}, {EC_GF_OP_XOR2, 4, 2, 0},
+    {EC_GF_OP_XOR2, 5, 4, 0}, {EC_GF_OP_XOR2, 6, 5, 0},
+    {EC_GF_OP_XOR2, 8, 6, 0}, {EC_GF_OP_XOR2, 3, 8, 0},
+    {EC_GF_OP_XOR2, 2, 3, 0}, {EC_GF_OP_XOR2, 3, 0, 0},
+    {EC_GF_OP_XOR2, 1, 3, 0}, {EC_GF_OP_XOR2, 0, 5, 0},
+    {EC_GF_OP_XOR2, 7, 1, 0}, {EC_GF_OP_XOR2, 1, 5, 0},
+    {EC_GF_OP_XOR2, 4, 7, 0}, {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_D3 = {9,
+                                    {
+                                        0,
+                                        3,
+                                        2,
+                                        8,
+                                        4,
+                                        6,
+                                        7,
+                                        1,
+                                        5,
+                                    },
+                                    ec_gf8_mul_D3_ops};
+
+static ec_gf_op_t ec_gf8_mul_D4_ops[] = {
+    {EC_GF_OP_XOR2, 1, 5, 0}, {EC_GF_OP_COPY, 8, 1, 0},
+    {EC_GF_OP_XOR2, 1, 2, 0}, {EC_GF_OP_XOR2, 0, 1, 0},
+    {EC_GF_OP_XOR2, 5, 3, 0}, {EC_GF_OP_XOR2, 6, 1, 0},
+    {EC_GF_OP_XOR2, 3, 0, 0}, {EC_GF_OP_XOR3, 1, 7, 8},
+    {EC_GF_OP_XOR2, 7, 3, 0}, {EC_GF_OP_XOR2, 3, 4, 0},
+    {EC_GF_OP_XOR2, 4, 6, 0}, {EC_GF_OP_XOR2, 2, 3, 0},
+    {EC_GF_OP_XOR2, 6, 5, 0}, {EC_GF_OP_XOR2, 3, 1, 0},
+    {EC_GF_OP_XOR2, 1, 6, 0}, {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_D4 = {9,
+                                    {
+                                        4,
+                                        1,
+                                        7,
+                                        5,
+                                        0,
+                                        6,
+                                        3,
+                                        2,
+                                        8,
+                                    },
+                                    ec_gf8_mul_D4_ops};
+
+static ec_gf_op_t ec_gf8_mul_D5_ops[] = {
+    {EC_GF_OP_XOR2, 0, 4, 0}, {EC_GF_OP_XOR2, 1, 5, 0},
+    {EC_GF_OP_XOR2, 1, 0, 0}, {EC_GF_OP_XOR2, 2, 1, 0},
+    {EC_GF_OP_XOR2, 6, 2, 0}, {EC_GF_OP_XOR2, 0, 6, 0},
+    {EC_GF_OP_XOR2, 3, 0, 0}, {EC_GF_OP_XOR2, 7, 3, 0},
+    {EC_GF_OP_XOR2, 1, 7, 0}, {EC_GF_OP_XOR2, 0, 1, 0},
+    {EC_GF_OP_XOR2, 4, 0, 0}, {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_D5 = {8,
+                                    {
+                                        6,
+                                        7,
+                                        4,
+                                        5,
+                                        2,
+                                        3,
+                                        1,
+                                        0,
+                                    },
+                                    ec_gf8_mul_D5_ops};
+
+static ec_gf_op_t ec_gf8_mul_D6_ops[] = {
+    {EC_GF_OP_COPY, 8, 0, 0}, {EC_GF_OP_XOR2, 0, 2, 0},
+    {EC_GF_OP_XOR2, 2, 1, 0}, {EC_GF_OP_XOR2, 2, 4, 0},
+    {EC_GF_OP_XOR2, 2, 6, 0}, {EC_GF_OP_XOR2, 3, 2, 0},
+    {EC_GF_OP_XOR2, 0, 3, 0}, {EC_GF_OP_XOR2, 5, 0, 0},
+    {EC_GF_OP_XOR2, 2, 5, 0}, {EC_GF_OP_XOR2, 7, 2, 0},
+    {EC_GF_OP_XOR2, 1, 7, 0}, {EC_GF_OP_XOR2, 4, 7, 0},
+    {EC_GF_OP_XOR2, 6, 7, 0}, {EC_GF_OP_XOR2, 0, 7, 0},
+    {EC_GF_OP_XOR2, 7, 8, 0}, {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_D6 = {9,
+                                    {
+                                        0,
+                                        6,
+                                        2,
+                                        7,
+                                        1,
+                                        3,
+                                        4,
+                                        5,
+                                        8,
+                                    },
+                                    ec_gf8_mul_D6_ops};
+
+static ec_gf_op_t ec_gf8_mul_D7_ops[] = {
+    {EC_GF_OP_XOR2, 7, 2, 0}, {EC_GF_OP_XOR2, 5, 7, 0},
+    {EC_GF_OP_XOR2, 7, 0, 0}, {EC_GF_OP_XOR2, 0, 1, 0},
+    {EC_GF_OP_XOR3, 8, 3, 5}, {EC_GF_OP_XOR2, 0, 4, 0},
+    {EC_GF_OP_XOR2, 0, 8, 0}, {EC_GF_OP_XOR2, 6, 0, 0},
+    {EC_GF_OP_XOR2, 2, 6, 0}, {EC_GF_OP_XOR2, 1, 6, 0},
+    {EC_GF_OP_XOR2, 4, 6, 0}, {EC_GF_OP_XOR2, 3, 6, 0},
+    {EC_GF_OP_XOR3, 6, 7, 8}, {EC_GF_OP_XOR2, 7, 2, 0},
+    {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_D7 = {9,
+                                    {
+                                        3,
+                                        4,
+                                        6,
+                                        5,
+                                        0,
+                                        7,
+                                        1,
+                                        2,
+                                        8,
+                                    },
+                                    ec_gf8_mul_D7_ops};
+
+static ec_gf_op_t ec_gf8_mul_D8_ops[] = {
+    {EC_GF_OP_XOR2, 3, 2, 0}, {EC_GF_OP_XOR2, 4, 1, 0},
+    {EC_GF_OP_XOR2, 5, 3, 0}, {EC_GF_OP_XOR2, 4, 2, 0},
+    {EC_GF_OP_XOR2, 3, 1, 0}, {EC_GF_OP_XOR2, 2, 0, 0},
+    {EC_GF_OP_XOR2, 6, 3, 0}, {EC_GF_OP_XOR2, 3, 2, 0},
+    {EC_GF_OP_XOR2, 7, 3, 0}, {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_D8 = {8,
+                                    {
+                                        4,
+                                        5,
+                                        6,
+                                        7,
+                                        0,
+                                        1,
+                                        2,
+                                        3,
+                                    },
+                                    ec_gf8_mul_D8_ops};
+
+static ec_gf_op_t ec_gf8_mul_D9_ops[] = {
+    {EC_GF_OP_XOR2, 4, 0, 0}, {EC_GF_OP_XOR2, 0, 1, 0},
+    {EC_GF_OP_XOR2, 5, 1, 0}, {EC_GF_OP_XOR2, 1, 2, 0},
+    {EC_GF_OP_XOR2, 6, 1, 0}, {EC_GF_OP_XOR2, 7, 0, 0},
+    {EC_GF_OP_XOR2, 1, 4, 0}, {EC_GF_OP_XOR2, 2, 3, 0},
+    {EC_GF_OP_XOR2, 0, 6, 0}, {EC_GF_OP_XOR2, 3, 7, 0},
+    {EC_GF_OP_XOR2, 6, 2, 0}, {EC_GF_OP_XOR2, 2, 5, 0},
+    {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_D9 = {8,
+                                    {
+                                        1,
+                                        2,
+                                        6,
+                                        7,
+                                        4,
+                                        5,
+                                        0,
+                                        3,
+                                    },
+                                    ec_gf8_mul_D9_ops};
+
+static ec_gf_op_t ec_gf8_mul_DA_ops[] = {
+    {EC_GF_OP_XOR2, 6, 2, 0}, {EC_GF_OP_XOR2, 2, 7, 0},
+    {EC_GF_OP_XOR2, 7, 3, 0}, {EC_GF_OP_XOR2, 0, 7, 0},
+    {EC_GF_OP_XOR3, 8, 2, 0}, {EC_GF_OP_XOR2, 7, 6, 0},
+    {EC_GF_OP_XOR2, 4, 1, 0}, {EC_GF_OP_XOR2, 1, 8, 0},
+    {EC_GF_OP_XOR2, 5, 8, 0}, {EC_GF_OP_XOR2, 2, 4, 0},
+    {EC_GF_OP_XOR2, 6, 1, 0}, {EC_GF_OP_XOR2, 3, 5, 0},
+    {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_DA = {9,
+                                    {
+                                        2,
+                                        5,
+                                        7,
+                                        1,
+                                        0,
+                                        4,
+                                        3,
+                                        6,
+                                        8,
+                                    },
+                                    ec_gf8_mul_DA_ops};
+
+static ec_gf_op_t ec_gf8_mul_DB_ops[] = {
+    {EC_GF_OP_COPY, 8, 0, 0}, {EC_GF_OP_XOR2, 0, 1, 0},
+    {EC_GF_OP_XOR2, 2, 0, 0}, {EC_GF_OP_XOR2, 5, 2, 0},
+    {EC_GF_OP_XOR2, 1, 5, 0}, {EC_GF_OP_XOR2, 8, 4, 0},
+    {EC_GF_OP_XOR2, 5, 3, 0}, {EC_GF_OP_XOR2, 4, 2, 0},
+    {EC_GF_OP_XOR2, 3, 7, 0}, {EC_GF_OP_XOR2, 7, 4, 0},
+    {EC_GF_OP_XOR2, 4, 1, 0}, {EC_GF_OP_XOR2, 1, 6, 0},
+    {EC_GF_OP_XOR2, 6, 3, 0}, {EC_GF_OP_XOR2, 3, 8, 0},
+    {EC_GF_OP_XOR2, 0, 6, 0}, {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_DB = {9,
+                                    {
+                                        7,
+                                        5,
+                                        6,
+                                        2,
+                                        3,
+                                        4,
+                                        1,
+                                        0,
+                                        8,
+                                    },
+                                    ec_gf8_mul_DB_ops};
+
+static ec_gf_op_t ec_gf8_mul_DC_ops[] = {
+    {EC_GF_OP_XOR2, 7, 3, 0}, {EC_GF_OP_XOR2, 3, 0, 0},
+    {EC_GF_OP_XOR2, 4, 2, 0}, {EC_GF_OP_XOR2, 0, 4, 0},
+    {EC_GF_OP_XOR2, 3, 1, 0}, {EC_GF_OP_XOR2, 6, 1, 0},
+    {EC_GF_OP_XOR2, 1, 7, 0}, {EC_GF_OP_XOR2, 4, 6, 0},
+    {EC_GF_OP_XOR2, 7, 2, 0}, {EC_GF_OP_XOR2, 6, 3, 0},
+    {EC_GF_OP_XOR2, 2, 3, 0}, {EC_GF_OP_XOR2, 3, 5, 0},
+    {EC_GF_OP_XOR2, 5, 7, 0}, {EC_GF_OP_XOR2, 7, 6, 0},
+    {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_DC = {8,
+                                    {
+                                        4,
+                                        5,
+                                        2,
+                                        6,
+                                        7,
+                                        1,
+                                        0,
+                                        3,
+                                    },
+                                    ec_gf8_mul_DC_ops};
+
+static ec_gf_op_t ec_gf8_mul_DD_ops[] = {
+    {EC_GF_OP_XOR2, 3, 1, 0}, {EC_GF_OP_XOR2, 3, 0, 0},
+    {EC_GF_OP_XOR2, 5, 7, 0}, {EC_GF_OP_XOR2, 5, 3, 0},
+    {EC_GF_OP_XOR2, 4, 2, 0}, {EC_GF_OP_XOR2, 6, 0, 0},
+    {EC_GF_OP_XOR2, 0, 5, 0}, {EC_GF_OP_XOR2, 4, 6, 0},
+    {EC_GF_OP_XOR2, 2, 0, 0}, {EC_GF_OP_XOR2, 1, 4, 0},
+    {EC_GF_OP_XOR2, 7, 4, 0}, {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_DD = {8,
+                                    {
+                                        1,
+                                        2,
+                                        3,
+                                        6,
+                                        7,
+                                        0,
+                                        4,
+                                        5,
+                                    },
+                                    ec_gf8_mul_DD_ops};
+
+static ec_gf_op_t ec_gf8_mul_DE_ops[] = {
+    {EC_GF_OP_XOR2, 2, 0, 0}, {EC_GF_OP_XOR2, 3, 7, 0},
+    {EC_GF_OP_XOR2, 2, 3, 0}, {EC_GF_OP_XOR2, 6, 2, 0},
+    {EC_GF_OP_XOR2, 1, 4, 0}, {EC_GF_OP_XOR2, 7, 6, 0},
+    {EC_GF_OP_XOR2, 5, 2, 0}, {EC_GF_OP_XOR2, 1, 3, 0},
+    {EC_GF_OP_XOR2, 0, 7, 0}, {EC_GF_OP_XOR2, 4, 5, 0},
+    {EC_GF_OP_XOR2, 0, 1, 0}, {EC_GF_OP_XOR2, 3, 4, 0},
+    {EC_GF_OP_XOR2, 4, 0, 0}, {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_DE = {8,
+                                    {
+                                        0,
+                                        5,
+                                        2,
+                                        6,
+                                        7,
+                                        1,
+                                        3,
+                                        4,
+                                    },
+                                    ec_gf8_mul_DE_ops};
+
+static ec_gf_op_t ec_gf8_mul_DF_ops[] = {
+    {EC_GF_OP_COPY, 8, 0, 0}, {EC_GF_OP_XOR2, 0, 2, 0},
+    {EC_GF_OP_XOR2, 0, 4, 0}, {EC_GF_OP_XOR2, 8, 3, 0},
+    {EC_GF_OP_COPY, 9, 0, 0}, {EC_GF_OP_XOR2, 0, 6, 0},
+    {EC_GF_OP_XOR2, 8, 7, 0}, {EC_GF_OP_XOR2, 3, 0, 0},
+    {EC_GF_OP_XOR2, 7, 0, 0}, {EC_GF_OP_XOR2, 0, 5, 0},
+    {EC_GF_OP_XOR2, 4, 7, 0}, {EC_GF_OP_XOR2, 5, 1, 0},
+    {EC_GF_OP_XOR2, 7, 1, 0}, {EC_GF_OP_XOR2, 5, 8, 0},
+    {EC_GF_OP_XOR2, 2, 5, 0}, {EC_GF_OP_XOR2, 6, 5, 0},
+    {EC_GF_OP_XOR3, 1, 9, 2}, {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_DF = {10,
+                                    {
+                                        7,
+                                        2,
+                                        8,
+                                        4,
+                                        3,
+                                        1,
+                                        0,
+                                        6,
+                                        5,
+                                        9,
+                                    },
+                                    ec_gf8_mul_DF_ops};
+
+static ec_gf_op_t ec_gf8_mul_E0_ops[] = {
+    {EC_GF_OP_XOR2, 4, 2, 0}, {EC_GF_OP_XOR2, 6, 4, 0},
+    {EC_GF_OP_XOR2, 5, 3, 0}, {EC_GF_OP_XOR2, 3, 6, 0},
+    {EC_GF_OP_XOR2, 4, 1, 0}, {EC_GF_OP_XOR2, 7, 1, 0},
+    {EC_GF_OP_XOR2, 5, 7, 0}, {EC_GF_OP_XOR2, 6, 0, 0},
+    {EC_GF_OP_XOR2, 2, 5, 0}, {EC_GF_OP_XOR2, 0, 5, 0},
+    {EC_GF_OP_XOR2, 1, 6, 0}, {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_E0 = {8,
+                                    {
+                                        2,
+                                        3,
+                                        4,
+                                        7,
+                                        5,
+                                        6,
+                                        0,
+                                        1,
+                                    },
+                                    ec_gf8_mul_E0_ops};
+
+static ec_gf_op_t ec_gf8_mul_E1_ops[] = {
+    {EC_GF_OP_COPY, 8, 1, 0}, {EC_GF_OP_XOR2, 8, 7, 0},
+    {EC_GF_OP_XOR2, 3, 8, 0}, {EC_GF_OP_XOR3, 9, 5, 3},
+    {EC_GF_OP_XOR2, 0, 9, 0}, {EC_GF_OP_XOR2, 1, 4, 0},
+    {EC_GF_OP_XOR2, 7, 0, 0}, {EC_GF_OP_XOR2, 6, 0, 0},
+    {EC_GF_OP_XOR2, 4, 9, 0}, {EC_GF_OP_XOR2, 0, 2, 0},
+    {EC_GF_OP_XOR2, 2, 4, 0}, {EC_GF_OP_XOR2, 2, 6, 0},
+    {EC_GF_OP_XOR2, 5, 2, 0}, {EC_GF_OP_XOR2, 2, 8, 0},
+    {EC_GF_OP_XOR2, 7, 5, 0}, {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_E1 = {10,
+                                    {
+                                        0,
+                                        7,
+                                        1,
+                                        3,
+                                        4,
+                                        5,
+                                        6,
+                                        2,
+                                        8,
+                                        9,
+                                    },
+                                    ec_gf8_mul_E1_ops};
+
+static ec_gf_op_t ec_gf8_mul_E2_ops[] = {
+    {EC_GF_OP_XOR2, 6, 0, 0}, {EC_GF_OP_XOR2, 5, 1, 0},
+    {EC_GF_OP_XOR2, 6, 2, 0}, {EC_GF_OP_XOR2, 1, 2, 0},
+    {EC_GF_OP_XOR2, 2, 3, 0}, {EC_GF_OP_XOR2, 0, 1, 0},
+    {EC_GF_OP_XOR2, 3, 4, 0}, {EC_GF_OP_XOR2, 7, 2, 0},
+    {EC_GF_OP_XOR2, 4, 0, 0}, {EC_GF_OP_XOR2, 2, 5, 0},
+    {EC_GF_OP_XOR2, 0, 7, 0}, {EC_GF_OP_XOR2, 7, 3, 0},
+    {EC_GF_OP_XOR2, 3, 6, 0}, {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_E2 = {8,
+                                    {
+                                        2,
+                                        3,
+                                        7,
+                                        1,
+                                        5,
+                                        6,
+                                        0,
+                                        4,
+                                    },
+                                    ec_gf8_mul_E2_ops};
+
+static ec_gf_op_t ec_gf8_mul_E3_ops[] = {
+    {EC_GF_OP_XOR2, 7, 4, 0}, {EC_GF_OP_XOR2, 3, 1, 0},
+    {EC_GF_OP_XOR3, 8, 2, 7}, {EC_GF_OP_XOR2, 4, 5, 0},
+    {EC_GF_OP_XOR2, 2, 3, 0}, {EC_GF_OP_XOR2, 5, 0, 0},
+    {EC_GF_OP_XOR2, 5, 2, 0}, {EC_GF_OP_XOR2, 0, 1, 0},
+    {EC_GF_OP_XOR2, 6, 5, 0}, {EC_GF_OP_XOR2, 1, 4, 0},
+    {EC_GF_OP_XOR2, 0, 8, 0}, {EC_GF_OP_XOR2, 4, 6, 0},
+    {EC_GF_OP_XOR2, 3, 6, 0}, {EC_GF_OP_XOR3, 6, 8, 4},
+    {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_E3 = {9,
+                                    {
+                                        5,
+                                        4,
+                                        7,
+                                        2,
+                                        1,
+                                        3,
+                                        6,
+                                        0,
+                                        8,
+                                    },
+                                    ec_gf8_mul_E3_ops};
+
+static ec_gf_op_t ec_gf8_mul_E4_ops[] = {
+    {EC_GF_OP_XOR2, 4, 0, 0}, {EC_GF_OP_XOR2, 2, 6, 0},
+    {EC_GF_OP_XOR2, 2, 4, 0}, {EC_GF_OP_XOR2, 1, 2, 0},
+    {EC_GF_OP_XOR2, 5, 1, 0}, {EC_GF_OP_XOR2, 4, 5, 0},
+    {EC_GF_OP_XOR2, 3, 4, 0}, {EC_GF_OP_XOR2, 7, 3, 0},
+    {EC_GF_OP_XOR2, 2, 7, 0}, {EC_GF_OP_XOR2, 4, 2, 0},
+    {EC_GF_OP_XOR2, 0, 4, 0}, {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_E4 = {8,
+                                    {
+                                        7,
+                                        0,
+                                        1,
+                                        6,
+                                        3,
+                                        4,
+                                        2,
+                                        5,
+                                    },
+                                    ec_gf8_mul_E4_ops};
+
+static ec_gf_op_t ec_gf8_mul_E5_ops[] = {
+    {EC_GF_OP_XOR2, 7, 5, 0}, {EC_GF_OP_XOR2, 5, 0, 0},
+    {EC_GF_OP_XOR2, 0, 1, 0}, {EC_GF_OP_XOR2, 6, 3, 0},
+    {EC_GF_OP_XOR2, 0, 6, 0}, {EC_GF_OP_XOR2, 0, 4, 0},
+    {EC_GF_OP_COPY, 8, 0, 0}, {EC_GF_OP_XOR2, 0, 7, 0},
+    {EC_GF_OP_XOR2, 2, 0, 0}, {EC_GF_OP_XOR2, 5, 2, 0},
+    {EC_GF_OP_XOR2, 1, 2, 0}, {EC_GF_OP_XOR2, 4, 2, 0},
+    {EC_GF_OP_XOR2, 7, 5, 0}, {EC_GF_OP_XOR2, 2, 3, 0},
+    {EC_GF_OP_XOR2, 3, 8, 0}, {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_E5 = {9,
+                                    {
+                                        4,
+                                        5,
+                                        3,
+                                        6,
+                                        7,
+                                        1,
+                                        0,
+                                        2,
+                                        8,
+                                    },
+                                    ec_gf8_mul_E5_ops};
+
+static ec_gf_op_t ec_gf8_mul_E6_ops[] = {
+    {EC_GF_OP_XOR2, 6, 2, 0}, {EC_GF_OP_XOR2, 1, 6, 0},
+    {EC_GF_OP_XOR2, 6, 7, 0}, {EC_GF_OP_XOR2, 0, 3, 0},
+    {EC_GF_OP_XOR2, 5, 1, 0}, {EC_GF_OP_XOR2, 0, 6, 0},
+    {EC_GF_OP_XOR2, 7, 5, 0}, {EC_GF_OP_XOR2, 4, 0, 0},
+    {EC_GF_OP_XOR2, 5, 3, 0}, {EC_GF_OP_XOR2, 3, 4, 0},
+    {EC_GF_OP_XOR2, 1, 4, 0}, {EC_GF_OP_XOR2, 2, 3, 0},
+    {EC_GF_OP_XOR2, 2, 7, 0}, {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_E6 = {8,
+                                    {
+                                        5,
+                                        4,
+                                        3,
+                                        6,
+                                        7,
+                                        0,
+                                        1,
+                                        2,
+                                    },
+                                    ec_gf8_mul_E6_ops};
+
+static ec_gf_op_t ec_gf8_mul_E7_ops[] = {
+    {EC_GF_OP_COPY, 8, 6, 0}, {EC_GF_OP_XOR2, 3, 2, 0},
+    {EC_GF_OP_XOR2, 6, 7, 0}, {EC_GF_OP_XOR2, 6, 3, 0},
+    {EC_GF_OP_XOR3, 9, 0, 6}, {EC_GF_OP_XOR2, 4, 9, 0},
+    {EC_GF_OP_XOR2, 5, 9, 0}, {EC_GF_OP_XOR2, 3, 4, 0},
+    {EC_GF_OP_XOR2, 7, 5, 0}, {EC_GF_OP_XOR2, 4, 1, 0},
+    {EC_GF_OP_XOR2, 2, 4, 0}, {EC_GF_OP_XOR2, 1, 7, 0},
+    {EC_GF_OP_XOR2, 7, 2, 0}, {EC_GF_OP_XOR2, 0, 7, 0},
+    {EC_GF_OP_XOR2, 7, 8, 0}, {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_E7 = {10,
+                                    {
+                                        1,
+                                        4,
+                                        3,
+                                        6,
+                                        7,
+                                        5,
+                                        2,
+                                        0,
+                                        8,
+                                        9,
+                                    },
+                                    ec_gf8_mul_E7_ops};
+
+static ec_gf_op_t ec_gf8_mul_E8_ops[] = {
+    {EC_GF_OP_XOR2, 1, 0, 0}, {EC_GF_OP_XOR2, 0, 4, 0},
+    {EC_GF_OP_XOR2, 4, 2, 0}, {EC_GF_OP_XOR2, 2, 5, 0},
+    {EC_GF_OP_XOR2, 4, 3, 0}, {EC_GF_OP_XOR2, 5, 1, 0},
+    {EC_GF_OP_XOR2, 3, 6, 0}, {EC_GF_OP_XOR2, 6, 5, 0},
+    {EC_GF_OP_XOR2, 0, 7, 0}, {EC_GF_OP_XOR2, 7, 6, 0},
+    {EC_GF_OP_XOR2, 1, 0, 0}, {EC_GF_OP_XOR2, 6, 2, 0},
+    {EC_GF_OP_XOR2, 2, 1, 0}, {EC_GF_OP_XOR2, 1, 4, 0},
+    {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_E8 = {8,
+                                    {
+                                        1,
+                                        4,
+                                        2,
+                                        7,
+                                        3,
+                                        0,
+                                        5,
+                                        6,
+                                    },
+                                    ec_gf8_mul_E8_ops};
+
+static ec_gf_op_t ec_gf8_mul_E9_ops[] = {
+    {EC_GF_OP_XOR2, 1, 0, 0}, {EC_GF_OP_COPY, 8, 1, 0},
+    {EC_GF_OP_XOR2, 1, 6, 0}, {EC_GF_OP_XOR2, 6, 3, 0},
+    {EC_GF_OP_XOR2, 4, 6, 0}, {EC_GF_OP_XOR2, 2, 1, 0},
+    {EC_GF_OP_XOR2, 3, 7, 0}, {EC_GF_OP_XOR2, 0, 4, 0},
+    {EC_GF_OP_XOR2, 7, 2, 0}, {EC_GF_OP_XOR2, 5, 1, 0},
+    {EC_GF_OP_XOR2, 2, 0, 0}, {EC_GF_OP_XOR2, 6, 7, 0},
+    {EC_GF_OP_XOR2, 3, 5, 0}, {EC_GF_OP_XOR2, 0, 3, 0},
+    {EC_GF_OP_XOR3, 1, 8, 0}, {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_E9 = {9,
+                                    {
+                                        6,
+                                        2,
+                                        0,
+                                        3,
+                                        4,
+                                        1,
+                                        5,
+                                        7,
+                                        8,
+                                    },
+                                    ec_gf8_mul_E9_ops};
+
+static ec_gf_op_t ec_gf8_mul_EA_ops[] = {
+    {EC_GF_OP_XOR2, 2, 1, 0}, {EC_GF_OP_XOR2, 3, 2, 0},
+    {EC_GF_OP_XOR2, 1, 0, 0}, {EC_GF_OP_XOR2, 2, 0, 0},
+    {EC_GF_OP_XOR2, 0, 7, 0}, {EC_GF_OP_XOR2, 5, 2, 0},
+    {EC_GF_OP_XOR2, 7, 6, 0}, {EC_GF_OP_XOR2, 4, 1, 0},
+    {EC_GF_OP_XOR2, 6, 5, 0}, {EC_GF_OP_XOR2, 5, 4, 0},
+    {EC_GF_OP_XOR2, 4, 3, 0}, {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_EA = {8,
+                                    {
+                                        3,
+                                        4,
+                                        5,
+                                        6,
+                                        7,
+                                        0,
+                                        1,
+                                        2,
+                                    },
+                                    ec_gf8_mul_EA_ops};
+
+static ec_gf_op_t ec_gf8_mul_EB_ops[] = {
+    {EC_GF_OP_XOR2, 1, 0, 0}, {EC_GF_OP_XOR2, 2, 1, 0},
+    {EC_GF_OP_XOR2, 3, 2, 0}, {EC_GF_OP_XOR2, 2, 7, 0},
+    {EC_GF_OP_XOR2, 7, 5, 0}, {EC_GF_OP_XOR2, 5, 4, 0},
+    {EC_GF_OP_XOR2, 0, 7, 0}, {EC_GF_OP_XOR2, 1, 6, 0},
+    {EC_GF_OP_XOR2, 4, 3, 0}, {EC_GF_OP_XOR2, 6, 5, 0},
+    {EC_GF_OP_XOR2, 7, 6, 0}, {EC_GF_OP_XOR2, 6, 4, 0},
+    {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_EB = {8,
+                                    {
+                                        3,
+                                        4,
+                                        5,
+                                        6,
+                                        7,
+                                        0,
+                                        1,
+                                        2,
+                                    },
+                                    ec_gf8_mul_EB_ops};
+
+static ec_gf_op_t ec_gf8_mul_EC_ops[] = {
+    {EC_GF_OP_XOR2, 0, 5, 0}, {EC_GF_OP_XOR2, 6, 1, 0},
+    {EC_GF_OP_XOR3, 8, 4, 0}, {EC_GF_OP_XOR2, 1, 8, 0},
+    {EC_GF_OP_XOR2, 7, 3, 0}, {EC_GF_OP_XOR2, 6, 2, 0},
+    {EC_GF_OP_XOR2, 3, 8, 0}, {EC_GF_OP_XOR2, 2, 7, 0},
+    {EC_GF_OP_XOR2, 7, 6, 0}, {EC_GF_OP_XOR2, 5, 3, 0},
+    {EC_GF_OP_XOR2, 4, 2, 0}, {EC_GF_OP_XOR2, 6, 0, 0},
+    {EC_GF_OP_XOR2, 3, 7, 0}, {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_EC = {9,
+                                    {
+                                        7,
+                                        4,
+                                        3,
+                                        0,
+                                        2,
+                                        5,
+                                        1,
+                                        6,
+                                        8,
+                                    },
+                                    ec_gf8_mul_EC_ops};
+
+static ec_gf_op_t ec_gf8_mul_ED_ops[] = {
+    {EC_GF_OP_XOR2, 5, 3, 0}, {EC_GF_OP_XOR2, 0, 5, 0},
+    {EC_GF_OP_XOR2, 2, 4, 0}, {EC_GF_OP_XOR2, 4, 0, 0},
+    {EC_GF_OP_XOR2, 3, 1, 0}, {EC_GF_OP_XOR2, 6, 4, 0},
+    {EC_GF_OP_XOR2, 3, 6, 0}, {EC_GF_OP_XOR2, 7, 3, 0},
+    {EC_GF_OP_XOR2, 2, 7, 0}, {EC_GF_OP_XOR2, 6, 2, 0},
+    {EC_GF_OP_XOR2, 5, 2, 0}, {EC_GF_OP_XOR2, 1, 6, 0},
+    {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_ED = {8,
+                                    {
+                                        5,
+                                        6,
+                                        7,
+                                        0,
+                                        1,
+                                        4,
+                                        3,
+                                        2,
+                                    },
+                                    ec_gf8_mul_ED_ops};
+
+static ec_gf_op_t ec_gf8_mul_EE_ops[] = {
+    {EC_GF_OP_XOR2, 5, 3, 0}, {EC_GF_OP_XOR2, 3, 0, 0},
+    {EC_GF_OP_XOR2, 0, 1, 0}, {EC_GF_OP_XOR3, 8, 2, 3},
+    {EC_GF_OP_XOR2, 0, 4, 0}, {EC_GF_OP_XOR2, 4, 8, 0},
+    {EC_GF_OP_XOR2, 6, 4, 0}, {EC_GF_OP_XOR2, 8, 5, 0},
+    {EC_GF_OP_XOR2, 4, 7, 0}, {EC_GF_OP_XOR2, 5, 6, 0},
+    {EC_GF_OP_XOR2, 1, 8, 0}, {EC_GF_OP_XOR2, 7, 8, 0},
+    {EC_GF_OP_XOR2, 6, 0, 0}, {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_EE = {9,
+                                    {
+                                        6,
+                                        4,
+                                        5,
+                                        7,
+                                        2,
+                                        3,
+                                        0,
+                                        1,
+                                        8,
+                                    },
+                                    ec_gf8_mul_EE_ops};
+
+static ec_gf_op_t ec_gf8_mul_EF_ops[] = {
+    {EC_GF_OP_XOR2, 5, 1, 0}, {EC_GF_OP_XOR2, 1, 3, 0},
+    {EC_GF_OP_XOR2, 0, 1, 0}, {EC_GF_OP_COPY, 8, 0, 0},
+    {EC_GF_OP_XOR2, 8, 2, 0}, {EC_GF_OP_XOR2, 0, 5, 0},
+    {EC_GF_OP_XOR2, 2, 4, 0}, {EC_GF_OP_XOR2, 7, 8, 0},
+    {EC_GF_OP_XOR2, 3, 2, 0}, {EC_GF_OP_XOR2, 6, 8, 0},
+    {EC_GF_OP_XOR2, 4, 7, 0}, {EC_GF_OP_XOR2, 3, 6, 0},
+    {EC_GF_OP_XOR2, 7, 5, 0}, {EC_GF_OP_XOR2, 5, 3, 0},
+    {EC_GF_OP_XOR2, 1, 7, 0}, {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_EF = {9,
+                                    {
+                                        6,
+                                        4,
+                                        5,
+                                        7,
+                                        2,
+                                        0,
+                                        3,
+                                        1,
+                                        8,
+                                    },
+                                    ec_gf8_mul_EF_ops};
+
+static ec_gf_op_t ec_gf8_mul_F0_ops[] = {
+    {EC_GF_OP_XOR2, 6, 1, 0}, {EC_GF_OP_XOR2, 6, 2, 0},
+    {EC_GF_OP_XOR2, 3, 0, 0}, {EC_GF_OP_XOR3, 8, 3, 6},
+    {EC_GF_OP_XOR2, 5, 8, 0}, {EC_GF_OP_XOR2, 8, 4, 0},
+    {EC_GF_OP_XOR2, 1, 5, 0}, {EC_GF_OP_XOR2, 7, 8, 0},
+    {EC_GF_OP_XOR2, 0, 1, 0}, {EC_GF_OP_XOR2, 2, 7, 0},
+    {EC_GF_OP_XOR2, 4, 0, 0}, {EC_GF_OP_XOR2, 1, 8, 0},
+    {EC_GF_OP_XOR2, 0, 2, 0}, {EC_GF_OP_XOR2, 3, 0, 0},
+    {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_F0 = {9,
+                                    {
+                                        3,
+                                        4,
+                                        6,
+                                        1,
+                                        2,
+                                        0,
+                                        5,
+                                        7,
+                                        8,
+                                    },
+                                    ec_gf8_mul_F0_ops};
+
+static ec_gf_op_t ec_gf8_mul_F1_ops[] = {
+    {EC_GF_OP_XOR2, 3, 1, 0}, {EC_GF_OP_XOR2, 3, 5, 0},
+    {EC_GF_OP_COPY, 8, 3, 0}, {EC_GF_OP_XOR2, 3, 4, 0},
+    {EC_GF_OP_XOR2, 2, 3, 0}, {EC_GF_OP_COPY, 9, 2, 0},
+    {EC_GF_OP_XOR2, 2, 6, 0}, {EC_GF_OP_XOR2, 9, 0, 0},
+    {EC_GF_OP_XOR2, 6, 1, 0}, {EC_GF_OP_XOR2, 5, 2, 0},
+    {EC_GF_OP_XOR2, 7, 9, 0}, {EC_GF_OP_XOR2, 4, 9, 0},
+    {EC_GF_OP_XOR2, 0, 5, 0}, {EC_GF_OP_XOR3, 9, 8, 7},
+    {EC_GF_OP_XOR2, 1, 9, 0}, {EC_GF_OP_XOR2, 5, 9, 0},
+    {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_F1 = {10,
+                                    {
+                                        7,
+                                        2,
+                                        6,
+                                        3,
+                                        5,
+                                        1,
+                                        4,
+                                        0,
+                                        8,
+                                        9,
+                                    },
+                                    ec_gf8_mul_F1_ops};
+
+static ec_gf_op_t ec_gf8_mul_F2_ops[] = {
+    {EC_GF_OP_XOR2, 0, 1, 0}, {EC_GF_OP_XOR2, 5, 4, 0},
+    {EC_GF_OP_XOR2, 1, 5, 0}, {EC_GF_OP_XOR2, 7, 2, 0},
+    {EC_GF_OP_XOR2, 0, 6, 0}, {EC_GF_OP_XOR2, 6, 7, 0},
+    {EC_GF_OP_XOR2, 4, 0, 0}, {EC_GF_OP_XOR2, 2, 3, 0},
+    {EC_GF_OP_XOR2, 7, 1, 0}, {EC_GF_OP_XOR3, 8, 6, 4},
+    {EC_GF_OP_XOR2, 1, 2, 0}, {EC_GF_OP_XOR2, 2, 0, 0},
+    {EC_GF_OP_XOR2, 3, 8, 0}, {EC_GF_OP_XOR2, 5, 8, 0},
+    {EC_GF_OP_XOR2, 0, 1, 0}, {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_F2 = {9,
+                                    {
+                                        1,
+                                        0,
+                                        6,
+                                        7,
+                                        4,
+                                        5,
+                                        2,
+                                        3,
+                                        8,
+                                    },
+                                    ec_gf8_mul_F2_ops};
+
+static ec_gf_op_t ec_gf8_mul_F3_ops[] = {
+    {EC_GF_OP_XOR2, 1, 0, 0}, {EC_GF_OP_XOR2, 2, 1, 0},
+    {EC_GF_OP_XOR2, 3, 2, 0}, {EC_GF_OP_XOR2, 4, 3, 0},
+    {EC_GF_OP_XOR2, 2, 7, 0}, {EC_GF_OP_XOR2, 0, 7, 0},
+    {EC_GF_OP_XOR2, 5, 4, 0}, {EC_GF_OP_XOR2, 1, 6, 0},
+    {EC_GF_OP_XOR2, 7, 6, 0}, {EC_GF_OP_XOR2, 0, 5, 0},
+    {EC_GF_OP_XOR2, 6, 5, 0}, {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_F3 = {8,
+                                    {
+                                        5,
+                                        6,
+                                        7,
+                                        0,
+                                        1,
+                                        2,
+                                        3,
+                                        4,
+                                    },
+                                    ec_gf8_mul_F3_ops};
+
+static ec_gf_op_t ec_gf8_mul_F4_ops[] = {
+    {EC_GF_OP_XOR2, 1, 0, 0}, {EC_GF_OP_XOR2, 2, 1, 0},
+    {EC_GF_OP_XOR2, 3, 2, 0}, {EC_GF_OP_XOR2, 4, 3, 0},
+    {EC_GF_OP_XOR2, 5, 4, 0}, {EC_GF_OP_XOR2, 6, 5, 0},
+    {EC_GF_OP_XOR2, 7, 6, 0}, {EC_GF_OP_XOR2, 0, 7, 0},
+    {EC_GF_OP_XOR2, 1, 7, 0}, {EC_GF_OP_XOR2, 3, 7, 0},
+    {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_F4 = {8,
+                                    {
+                                        0,
+                                        1,
+                                        2,
+                                        3,
+                                        4,
+                                        5,
+                                        6,
+                                        7,
+                                    },
+                                    ec_gf8_mul_F4_ops};
+
+static ec_gf_op_t ec_gf8_mul_F5_ops[] = {
+    {EC_GF_OP_XOR2, 1, 0, 0}, {EC_GF_OP_XOR2, 2, 1, 0},
+    {EC_GF_OP_XOR2, 3, 2, 0}, {EC_GF_OP_XOR2, 4, 3, 0},
+    {EC_GF_OP_XOR2, 5, 4, 0}, {EC_GF_OP_XOR2, 6, 5, 0},
+    {EC_GF_OP_XOR2, 7, 6, 0}, {EC_GF_OP_XOR2, 0, 7, 0},
+    {EC_GF_OP_XOR2, 2, 7, 0}, {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_F5 = {8,
+                                    {
+                                        7,
+                                        0,
+                                        1,
+                                        2,
+                                        3,
+                                        4,
+                                        5,
+                                        6,
+                                    },
+                                    ec_gf8_mul_F5_ops};
+
+static ec_gf_op_t ec_gf8_mul_F6_ops[] = {
+    {EC_GF_OP_XOR2, 3, 1, 0}, {EC_GF_OP_COPY, 8, 3, 0},
+    {EC_GF_OP_XOR2, 3, 5, 0}, {EC_GF_OP_XOR2, 2, 0, 0},
+    {EC_GF_OP_COPY, 9, 3, 0}, {EC_GF_OP_XOR2, 3, 2, 0},
+    {EC_GF_OP_XOR2, 2, 7, 0}, {EC_GF_OP_XOR2, 4, 2, 0},
+    {EC_GF_OP_XOR2, 9, 4, 0}, {EC_GF_OP_XOR2, 4, 1, 0},
+    {EC_GF_OP_XOR2, 6, 9, 0}, {EC_GF_OP_XOR2, 7, 6, 0},
+    {EC_GF_OP_XOR2, 0, 7, 0}, {EC_GF_OP_XOR2, 5, 7, 0},
+    {EC_GF_OP_XOR2, 6, 1, 0}, {EC_GF_OP_XOR3, 7, 8, 0},
+    {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_F6 = {10,
+                                    {
+                                        0,
+                                        6,
+                                        2,
+                                        7,
+                                        4,
+                                        3,
+                                        5,
+                                        9,
+                                        1,
+                                        8,
+                                    },
+                                    ec_gf8_mul_F6_ops};
+
+static ec_gf_op_t ec_gf8_mul_F7_ops[] = {
+    {EC_GF_OP_XOR2, 1, 0, 0}, {EC_GF_OP_XOR2, 2, 1, 0},
+    {EC_GF_OP_XOR2, 3, 2, 0}, {EC_GF_OP_XOR2, 4, 3, 0},
+    {EC_GF_OP_XOR2, 5, 4, 0}, {EC_GF_OP_XOR2, 6, 5, 0},
+    {EC_GF_OP_XOR2, 0, 7, 0}, {EC_GF_OP_XOR2, 2, 7, 0},
+    {EC_GF_OP_XOR2, 1, 6, 0}, {EC_GF_OP_XOR2, 7, 6, 0},
+    {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_F7 = {8,
+                                    {
+                                        6,
+                                        7,
+                                        0,
+                                        1,
+                                        2,
+                                        3,
+                                        4,
+                                        5,
+                                    },
+                                    ec_gf8_mul_F7_ops};
+
+static ec_gf_op_t ec_gf8_mul_F8_ops[] = {
+    {EC_GF_OP_XOR2, 4, 0, 0}, {EC_GF_OP_XOR2, 3, 5, 0},
+    {EC_GF_OP_XOR2, 6, 4, 0}, {EC_GF_OP_XOR2, 2, 0, 0},
+    {EC_GF_OP_XOR2, 4, 3, 0}, {EC_GF_OP_XOR2, 1, 6, 0},
+    {EC_GF_OP_XOR2, 2, 4, 0}, {EC_GF_OP_XOR2, 5, 1, 0},
+    {EC_GF_OP_XOR2, 7, 2, 0}, {EC_GF_OP_XOR2, 7, 5, 0},
+    {EC_GF_OP_XOR2, 3, 7, 0}, {EC_GF_OP_XOR2, 6, 7, 0},
+    {EC_GF_OP_XOR2, 0, 3, 0}, {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_F8 = {8,
+                                    {
+                                        6,
+                                        2,
+                                        0,
+                                        1,
+                                        4,
+                                        5,
+                                        3,
+                                        7,
+                                    },
+                                    ec_gf8_mul_F8_ops};
+
+static ec_gf_op_t ec_gf8_mul_F9_ops[] = {
+    {EC_GF_OP_XOR2, 1, 5, 0}, {EC_GF_OP_XOR2, 5, 3, 0},
+    {EC_GF_OP_XOR2, 6, 1, 0}, {EC_GF_OP_XOR2, 0, 5, 0},
+    {EC_GF_OP_XOR2, 7, 6, 0}, {EC_GF_OP_XOR2, 6, 0, 0},
+    {EC_GF_OP_XOR2, 2, 6, 0}, {EC_GF_OP_XOR2, 6, 4, 0},
+    {EC_GF_OP_XOR2, 1, 2, 0}, {EC_GF_OP_XOR2, 3, 6, 0},
+    {EC_GF_OP_XOR3, 8, 7, 1}, {EC_GF_OP_XOR2, 1, 3, 0},
+    {EC_GF_OP_XOR2, 4, 8, 0}, {EC_GF_OP_XOR2, 5, 8, 0},
+    {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_F9 = {9,
+                                    {
+                                        4,
+                                        1,
+                                        7,
+                                        6,
+                                        0,
+                                        3,
+                                        5,
+                                        2,
+                                        8,
+                                    },
+                                    ec_gf8_mul_F9_ops};
+
+static ec_gf_op_t ec_gf8_mul_FA_ops[] = {
+    {EC_GF_OP_XOR2, 1, 0, 0}, {EC_GF_OP_XOR2, 0, 4, 0},
+    {EC_GF_OP_XOR2, 2, 1, 0}, {EC_GF_OP_XOR2, 0, 7, 0},
+    {EC_GF_OP_XOR2, 7, 2, 0}, {EC_GF_OP_XOR2, 1, 5, 0},
+    {EC_GF_OP_XOR2, 3, 7, 0}, {EC_GF_OP_XOR2, 5, 0, 0},
+    {EC_GF_OP_XOR2, 7, 6, 0}, {EC_GF_OP_XOR2, 0, 3, 0},
+    {EC_GF_OP_XOR2, 6, 1, 0}, {EC_GF_OP_XOR2, 4, 7, 0},
+    {EC_GF_OP_XOR2, 1, 0, 0}, {EC_GF_OP_XOR2, 2, 6, 0},
+    {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_FA = {8,
+                                    {
+                                        0,
+                                        1,
+                                        2,
+                                        4,
+                                        5,
+                                        6,
+                                        7,
+                                        3,
+                                    },
+                                    ec_gf8_mul_FA_ops};
+
+static ec_gf_op_t ec_gf8_mul_FB_ops[] = {
+    {EC_GF_OP_XOR2, 1, 0, 0}, {EC_GF_OP_XOR2, 2, 1, 0},
+    {EC_GF_OP_XOR2, 0, 5, 0}, {EC_GF_OP_XOR2, 3, 2, 0},
+    {EC_GF_OP_XOR2, 0, 7, 0}, {EC_GF_OP_XOR2, 2, 7, 0},
+    {EC_GF_OP_XOR2, 1, 6, 0}, {EC_GF_OP_XOR2, 7, 6, 0},
+    {EC_GF_OP_XOR2, 4, 3, 0}, {EC_GF_OP_XOR2, 6, 5, 0},
+    {EC_GF_OP_XOR2, 7, 4, 0}, {EC_GF_OP_XOR2, 5, 4, 0},
+    {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_FB = {8,
+                                    {
+                                        4,
+                                        5,
+                                        6,
+                                        7,
+                                        0,
+                                        1,
+                                        2,
+                                        3,
+                                    },
+                                    ec_gf8_mul_FB_ops};
+
+static ec_gf_op_t ec_gf8_mul_FC_ops[] = {
+    {EC_GF_OP_XOR2, 7, 0, 0}, {EC_GF_OP_XOR2, 7, 4, 0},
+    {EC_GF_OP_XOR2, 5, 1, 0}, {EC_GF_OP_COPY, 9, 3, 0},
+    {EC_GF_OP_XOR3, 8, 5, 7}, {EC_GF_OP_XOR2, 3, 6, 0},
+    {EC_GF_OP_XOR2, 8, 3, 0}, {EC_GF_OP_XOR2, 2, 8, 0},
+    {EC_GF_OP_XOR2, 1, 2, 0}, {EC_GF_OP_XOR2, 4, 2, 0},
+    {EC_GF_OP_XOR2, 0, 1, 0}, {EC_GF_OP_XOR2, 3, 4, 0},
+    {EC_GF_OP_XOR2, 5, 0, 0}, {EC_GF_OP_XOR2, 6, 0, 0},
+    {EC_GF_OP_XOR3, 0, 9, 2}, {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_FC = {10,
+                                    {
+                                        5,
+                                        6,
+                                        3,
+                                        7,
+                                        1,
+                                        8,
+                                        0,
+                                        4,
+                                        2,
+                                        9,
+                                    },
+                                    ec_gf8_mul_FC_ops};
+
+static ec_gf_op_t ec_gf8_mul_FD_ops[] = {
+    {EC_GF_OP_XOR2, 7, 1, 0}, {EC_GF_OP_COPY, 8, 7, 0},
+    {EC_GF_OP_XOR2, 5, 0, 0}, {EC_GF_OP_XOR2, 7, 5, 0},
+    {EC_GF_OP_XOR2, 4, 2, 0}, {EC_GF_OP_XOR2, 4, 7, 0},
+    {EC_GF_OP_XOR2, 5, 6, 0}, {EC_GF_OP_XOR2, 0, 4, 0},
+    {EC_GF_OP_XOR2, 3, 0, 0}, {EC_GF_OP_XOR2, 5, 3, 0},
+    {EC_GF_OP_XOR2, 2, 5, 0}, {EC_GF_OP_XOR2, 1, 2, 0},
+    {EC_GF_OP_XOR2, 0, 1, 0}, {EC_GF_OP_XOR2, 6, 1, 0},
+    {EC_GF_OP_XOR3, 1, 8, 0}, {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_FD = {9,
+                                    {
+                                        5,
+                                        3,
+                                        7,
+                                        6,
+                                        1,
+                                        2,
+                                        4,
+                                        0,
+                                        8,
+                                    },
+                                    ec_gf8_mul_FD_ops};
+
+static ec_gf_op_t ec_gf8_mul_FE_ops[] = {
+    {EC_GF_OP_XOR2, 2, 0, 0}, {EC_GF_OP_COPY, 8, 2, 0},
+    {EC_GF_OP_XOR2, 2, 4, 0}, {EC_GF_OP_XOR2, 6, 2, 0},
+    {EC_GF_OP_XOR2, 8, 5, 0}, {EC_GF_OP_XOR2, 5, 6, 0},
+    {EC_GF_OP_XOR2, 6, 1, 0}, {EC_GF_OP_XOR2, 0, 6, 0},
+    {EC_GF_OP_XOR2, 6, 7, 0}, {EC_GF_OP_XOR2, 7, 3, 0},
+    {EC_GF_OP_XOR2, 7, 8, 0}, {EC_GF_OP_XOR2, 3, 0, 0},
+    {EC_GF_OP_XOR2, 4, 7, 0}, {EC_GF_OP_XOR2, 1, 7, 0},
+    {EC_GF_OP_XOR2, 0, 4, 0}, {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_FE = {9,
+                                    {
+                                        3,
+                                        4,
+                                        8,
+                                        2,
+                                        5,
+                                        0,
+                                        6,
+                                        1,
+                                        7,
+                                    },
+                                    ec_gf8_mul_FE_ops};
+
+static ec_gf_op_t ec_gf8_mul_FF_ops[] = {
+    {EC_GF_OP_XOR2, 4, 7, 0}, {EC_GF_OP_COPY, 9, 0, 0},
+    {EC_GF_OP_COPY, 8, 4, 0}, {EC_GF_OP_XOR2, 9, 1, 0},
+    {EC_GF_OP_XOR2, 4, 2, 0}, {EC_GF_OP_XOR2, 9, 4, 0},
+    {EC_GF_OP_XOR2, 0, 5, 0}, {EC_GF_OP_XOR2, 2, 0, 0},
+    {EC_GF_OP_XOR2, 3, 9, 0}, {EC_GF_OP_XOR2, 7, 3, 0},
+    {EC_GF_OP_XOR2, 2, 6, 0}, {EC_GF_OP_XOR2, 5, 3, 0},
+    {EC_GF_OP_XOR2, 6, 7, 0}, {EC_GF_OP_XOR2, 1, 7, 0},
+    {EC_GF_OP_XOR3, 3, 8, 5}, {EC_GF_OP_XOR2, 4, 6, 0},
+    {EC_GF_OP_END, 0, 0, 0}};
+
+static ec_gf_mul_t ec_gf8_mul_FF = {10,
+                                    {
+                                        6,
+                                        5,
+                                        0,
+                                        1,
+                                        2,
+                                        4,
+                                        9,
+                                        3,
+                                        7,
+                                        8,
+                                    },
+                                    ec_gf8_mul_FF_ops};
+
+ec_gf_mul_t *ec_gf8_mul[] = {
+    &ec_gf8_mul_00, &ec_gf8_mul_01, &ec_gf8_mul_02, &ec_gf8_mul_03,
+    &ec_gf8_mul_04, &ec_gf8_mul_05, &ec_gf8_mul_06, &ec_gf8_mul_07,
+    &ec_gf8_mul_08, &ec_gf8_mul_09, &ec_gf8_mul_0A, &ec_gf8_mul_0B,
+    &ec_gf8_mul_0C, &ec_gf8_mul_0D, &ec_gf8_mul_0E, &ec_gf8_mul_0F,
+    &ec_gf8_mul_10, &ec_gf8_mul_11, &ec_gf8_mul_12, &ec_gf8_mul_13,
+    &ec_gf8_mul_14, &ec_gf8_mul_15, &ec_gf8_mul_16, &ec_gf8_mul_17,
+    &ec_gf8_mul_18, &ec_gf8_mul_19, &ec_gf8_mul_1A, &ec_gf8_mul_1B,
+    &ec_gf8_mul_1C, &ec_gf8_mul_1D, &ec_gf8_mul_1E, &ec_gf8_mul_1F,
+    &ec_gf8_mul_20, &ec_gf8_mul_21, &ec_gf8_mul_22, &ec_gf8_mul_23,
+    &ec_gf8_mul_24, &ec_gf8_mul_25, &ec_gf8_mul_26, &ec_gf8_mul_27,
+    &ec_gf8_mul_28, &ec_gf8_mul_29, &ec_gf8_mul_2A, &ec_gf8_mul_2B,
+    &ec_gf8_mul_2C, &ec_gf8_mul_2D, &ec_gf8_mul_2E, &ec_gf8_mul_2F,
+    &ec_gf8_mul_30, &ec_gf8_mul_31, &ec_gf8_mul_32, &ec_gf8_mul_33,
+    &ec_gf8_mul_34, &ec_gf8_mul_35, &ec_gf8_mul_36, &ec_gf8_mul_37,
+    &ec_gf8_mul_38, &ec_gf8_mul_39, &ec_gf8_mul_3A, &ec_gf8_mul_3B,
+    &ec_gf8_mul_3C, &ec_gf8_mul_3D, &ec_gf8_mul_3E, &ec_gf8_mul_3F,
+    &ec_gf8_mul_40, &ec_gf8_mul_41, &ec_gf8_mul_42, &ec_gf8_mul_43,
+    &ec_gf8_mul_44, &ec_gf8_mul_45, &ec_gf8_mul_46, &ec_gf8_mul_47,
+    &ec_gf8_mul_48, &ec_gf8_mul_49, &ec_gf8_mul_4A, &ec_gf8_mul_4B,
+    &ec_gf8_mul_4C, &ec_gf8_mul_4D, &ec_gf8_mul_4E, &ec_gf8_mul_4F,
+    &ec_gf8_mul_50, &ec_gf8_mul_51, &ec_gf8_mul_52, &ec_gf8_mul_53,
+    &ec_gf8_mul_54, &ec_gf8_mul_55, &ec_gf8_mul_56, &ec_gf8_mul_57,
+    &ec_gf8_mul_58, &ec_gf8_mul_59, &ec_gf8_mul_5A, &ec_gf8_mul_5B,
+    &ec_gf8_mul_5C, &ec_gf8_mul_5D, &ec_gf8_mul_5E, &ec_gf8_mul_5F,
+    &ec_gf8_mul_60, &ec_gf8_mul_61, &ec_gf8_mul_62, &ec_gf8_mul_63,
+    &ec_gf8_mul_64, &ec_gf8_mul_65, &ec_gf8_mul_66, &ec_gf8_mul_67,
+    &ec_gf8_mul_68, &ec_gf8_mul_69, &ec_gf8_mul_6A, &ec_gf8_mul_6B,
+    &ec_gf8_mul_6C, &ec_gf8_mul_6D, &ec_gf8_mul_6E, &ec_gf8_mul_6F,
+    &ec_gf8_mul_70, &ec_gf8_mul_71, &ec_gf8_mul_72, &ec_gf8_mul_73,
+    &ec_gf8_mul_74, &ec_gf8_mul_75, &ec_gf8_mul_76, &ec_gf8_mul_77,
+    &ec_gf8_mul_78, &ec_gf8_mul_79, &ec_gf8_mul_7A, &ec_gf8_mul_7B,
+    &ec_gf8_mul_7C, &ec_gf8_mul_7D, &ec_gf8_mul_7E, &ec_gf8_mul_7F,
+    &ec_gf8_mul_80, &ec_gf8_mul_81, &ec_gf8_mul_82, &ec_gf8_mul_83,
+    &ec_gf8_mul_84, &ec_gf8_mul_85, &ec_gf8_mul_86, &ec_gf8_mul_87,
+    &ec_gf8_mul_88, &ec_gf8_mul_89, &ec_gf8_mul_8A, &ec_gf8_mul_8B,
+    &ec_gf8_mul_8C, &ec_gf8_mul_8D, &ec_gf8_mul_8E, &ec_gf8_mul_8F,
+    &ec_gf8_mul_90, &ec_gf8_mul_91, &ec_gf8_mul_92, &ec_gf8_mul_93,
+    &ec_gf8_mul_94, &ec_gf8_mul_95, &ec_gf8_mul_96, &ec_gf8_mul_97,
+    &ec_gf8_mul_98, &ec_gf8_mul_99, &ec_gf8_mul_9A, &ec_gf8_mul_9B,
+    &ec_gf8_mul_9C, &ec_gf8_mul_9D, &ec_gf8_mul_9E, &ec_gf8_mul_9F,
+    &ec_gf8_mul_A0, &ec_gf8_mul_A1, &ec_gf8_mul_A2, &ec_gf8_mul_A3,
+    &ec_gf8_mul_A4, &ec_gf8_mul_A5, &ec_gf8_mul_A6, &ec_gf8_mul_A7,
+    &ec_gf8_mul_A8, &ec_gf8_mul_A9, &ec_gf8_mul_AA, &ec_gf8_mul_AB,
+    &ec_gf8_mul_AC, &ec_gf8_mul_AD, &ec_gf8_mul_AE, &ec_gf8_mul_AF,
+    &ec_gf8_mul_B0, &ec_gf8_mul_B1, &ec_gf8_mul_B2, &ec_gf8_mul_B3,
+    &ec_gf8_mul_B4, &ec_gf8_mul_B5, &ec_gf8_mul_B6, &ec_gf8_mul_B7,
+    &ec_gf8_mul_B8, &ec_gf8_mul_B9, &ec_gf8_mul_BA, &ec_gf8_mul_BB,
+    &ec_gf8_mul_BC, &ec_gf8_mul_BD, &ec_gf8_mul_BE, &ec_gf8_mul_BF,
+    &ec_gf8_mul_C0, &ec_gf8_mul_C1, &ec_gf8_mul_C2, &ec_gf8_mul_C3,
+    &ec_gf8_mul_C4, &ec_gf8_mul_C5, &ec_gf8_mul_C6, &ec_gf8_mul_C7,
+    &ec_gf8_mul_C8, &ec_gf8_mul_C9, &ec_gf8_mul_CA, &ec_gf8_mul_CB,
+    &ec_gf8_mul_CC, &ec_gf8_mul_CD, &ec_gf8_mul_CE, &ec_gf8_mul_CF,
+    &ec_gf8_mul_D0, &ec_gf8_mul_D1, &ec_gf8_mul_D2, &ec_gf8_mul_D3,
+    &ec_gf8_mul_D4, &ec_gf8_mul_D5, &ec_gf8_mul_D6, &ec_gf8_mul_D7,
+    &ec_gf8_mul_D8, &ec_gf8_mul_D9, &ec_gf8_mul_DA, &ec_gf8_mul_DB,
+    &ec_gf8_mul_DC, &ec_gf8_mul_DD, &ec_gf8_mul_DE, &ec_gf8_mul_DF,
+    &ec_gf8_mul_E0, &ec_gf8_mul_E1, &ec_gf8_mul_E2, &ec_gf8_mul_E3,
+    &ec_gf8_mul_E4, &ec_gf8_mul_E5, &ec_gf8_mul_E6, &ec_gf8_mul_E7,
+    &ec_gf8_mul_E8, &ec_gf8_mul_E9, &ec_gf8_mul_EA, &ec_gf8_mul_EB,
+    &ec_gf8_mul_EC, &ec_gf8_mul_ED, &ec_gf8_mul_EE, &ec_gf8_mul_EF,
+    &ec_gf8_mul_F0, &ec_gf8_mul_F1, &ec_gf8_mul_F2, &ec_gf8_mul_F3,
+    &ec_gf8_mul_F4, &ec_gf8_mul_F5, &ec_gf8_mul_F6, &ec_gf8_mul_F7,
+    &ec_gf8_mul_F8, &ec_gf8_mul_F9, &ec_gf8_mul_FA, &ec_gf8_mul_FB,
+    &ec_gf8_mul_FC, &ec_gf8_mul_FD, &ec_gf8_mul_FE, &ec_gf8_mul_FF};
diff --git a/xlators/cluster/ec/src/ec-gf.h b/xlators/cluster/ec/src/ec-gf8.h
index 23bca91e3b5..4aca91127fc 100644
--- a/xlators/cluster/ec/src/ec-gf.h
+++ b/xlators/cluster/ec/src/ec-gf8.h
@@ -1,5 +1,5 @@
 /*
-  Copyright (c) 2012-2014 DataLab, s.l. <http://www.datalab.es>
+  Copyright (c) 2015 DataLab, s.l. <http://www.datalab.es>
   This file is part of GlusterFS.
 
   This file is licensed to you under your choice of the GNU Lesser
@@ -11,13 +11,8 @@
 #ifndef __EC_GF8_H__
 #define __EC_GF8_H__
 
-#define EC_GF_BITS 8
-#define EC_GF_MOD 0x11D
+#include "ec-galois.h"
 
-#define EC_GF_SIZE (1 << EC_GF_BITS)
-#define EC_GF_WORD_SIZE sizeof(uint64_t)
-
-extern void (* ec_gf_muladd[])(uint8_t * out, uint8_t * in,
-                               unsigned int width);
+extern ec_gf_mul_t *ec_gf8_mul[];
 
 #endif /* __EC_GF8_H__ */
diff --git a/xlators/cluster/ec/src/ec-heal.c b/xlators/cluster/ec/src/ec-heal.c
index d82592d666e..7d991f04aac 100644
--- a/xlators/cluster/ec/src/ec-heal.c
+++ b/xlators/cluster/ec/src/ec-heal.c
@@ -8,158 +8,164 @@
   cases as published by the Free Software Foundation.
 */
 
-#include "xlator.h"
-#include "defaults.h"
-#include "compat-errno.h"
-
+#include <glusterfs/defaults.h>
+#include <glusterfs/compat-errno.h>
+#include <glusterfs/byte-order.h>
+#include <glusterfs/syncop.h>
+#include <glusterfs/syncop-utils.h>
+#include <glusterfs/cluster-syncop.h>
+
+#include "ec.h"
+#include "ec-types.h"
+#include "ec-messages.h"
 #include "ec-helpers.h"
 #include "ec-common.h"
 #include "ec-combine.h"
 #include "ec-method.h"
 #include "ec-fops.h"
-
-#include "ec-mem-types.h"
-#include "ec-data.h"
-
-static char *ec_ignore_xattrs[] = {
-        GF_SELINUX_XATTR_KEY,
-        QUOTA_SIZE_KEY,
-        NULL
+#include "ec-heald.h"
+
+#define EC_COUNT(array, max)                                                   \
+    ({                                                                         \
+        int __i;                                                               \
+        int __res = 0;                                                         \
+        for (__i = 0; __i < max; __i++)                                        \
+            if (array[__i])                                                    \
+                __res++;                                                       \
+        __res;                                                                 \
+    })
+#define EC_INTERSECT(dst, src1, src2, max)                                     \
+    ({                                                                         \
+        int __i;                                                               \
+        for (__i = 0; __i < max; __i++)                                        \
+            dst[__i] = src1[__i] && src2[__i];                                 \
+    })
+#define EC_ADJUST_SOURCE(source, sources, max)                                 \
+    ({                                                                         \
+        int __i;                                                               \
+        if (sources[source] == 0) {                                            \
+            source = -1;                                                       \
+            for (__i = 0; __i < max; __i++)                                    \
+                if (sources[__i])                                              \
+                    source = __i;                                              \
+        }                                                                      \
+    })
+#define IA_EQUAL(f, s, field)                                                  \
+    (memcmp(&(f.ia_##field), &(s.ia_##field), sizeof(s.ia_##field)) == 0)
+#define EC_REPLIES_ALLOC(replies, numsubvols)                                  \
+    do {                                                                       \
+        int __i = 0;                                                           \
+        replies = alloca0(numsubvols * sizeof(*replies));                      \
+        for (__i = 0; __i < numsubvols; __i++)                                 \
+            INIT_LIST_HEAD(&replies[__i].entries.list);                        \
+    } while (0)
+
+struct ec_name_data {
+    call_frame_t *frame;
+    unsigned char *participants;
+    unsigned char *failed_on;
+    unsigned char *gfidless;
+    unsigned char *enoent;
+    unsigned char *same;
+    char *name;
+    inode_t *parent;
+    default_args_cbk_t *replies;
+    uint32_t heal_pending;
 };
 
+static char *ec_ignore_xattrs[] = {GF_SELINUX_XATTR_KEY, QUOTA_SIZE_KEY, NULL};
+
 static gf_boolean_t
-ec_ignorable_key_match (dict_t *dict, char *key, data_t *val, void *mdata)
+ec_ignorable_key_match(dict_t *dict, char *key, data_t *val, void *mdata)
 {
-        int i = 0;
+    int i = 0;
 
-        if (!key)
-                goto out;
+    if (!key)
+        goto out;
 
-        for (i = 0; ec_ignore_xattrs[i]; i++) {
-                if (!strcmp (key, ec_ignore_xattrs[i]))
-                       return _gf_true;
-        }
+    if (strncmp(key, EC_XATTR_PREFIX, SLEN(EC_XATTR_PREFIX)) == 0)
+        return _gf_true;
+
+    for (i = 0; ec_ignore_xattrs[i]; i++) {
+        if (!strcmp(key, ec_ignore_xattrs[i]))
+            return _gf_true;
+    }
 
 out:
-        return _gf_false;
+    return _gf_false;
 }
 
-/* FOP: heal */
-
-void ec_heal_exclude(ec_heal_t * heal, uintptr_t mask)
+static gf_boolean_t
+ec_sh_key_match(dict_t *dict, char *key, data_t *val, void *mdata)
 {
-    LOCK(&heal->lock);
-
-    heal->bad &= ~mask;
-
-    UNLOCK(&heal->lock);
+    return !ec_ignorable_key_match(dict, key, val, mdata);
 }
+/* FOP: heal */
 
-void ec_heal_lookup_resume(ec_fop_data_t * fop)
+void
+ec_set_entry_healing(ec_fop_data_t *fop)
 {
-    ec_heal_t * heal = fop->data;
-    ec_cbk_data_t * cbk;
-    uintptr_t good = 0, bad = 0;
+    ec_inode_t *ctx = NULL;
+    loc_t *loc = NULL;
 
-    if (heal->lookup != NULL)
-    {
-        ec_fop_data_release(heal->lookup);
-    }
-    ec_fop_data_acquire(fop);
+    if (!fop)
+        return;
 
-    list_for_each_entry(cbk, &fop->cbk_list, list)
+    loc = &fop->loc[0];
+    LOCK(&loc->inode->lock);
     {
-        if ((cbk->op_ret < 0) && (cbk->op_errno == ENOTCONN))
-        {
-            continue;
-        }
-
-        if (cbk == fop->answer)
-        {
-            if (cbk->op_ret >= 0)
-            {
-                heal->iatt = cbk->iatt[0];
-                heal->version = cbk->version;
-                heal->raw_size = cbk->size;
-                heal->fop->pre_size = cbk->iatt[0].ia_size;
-                heal->fop->post_size = cbk->iatt[0].ia_size;
-                heal->fop->have_size = 1;
-
-                if (ec_loc_update(heal->xl, &heal->loc, cbk->inode,
-                                  &cbk->iatt[0]) != 0)
-                {
-                    fop->answer = NULL;
-                    fop->error = EIO;
-
-                    bad |= cbk->mask;
-
-                    continue;
-                }
-            }
-
-            good |= cbk->mask;
-        }
-        else
-        {
-            bad |= cbk->mask;
+        ctx = __ec_inode_get(loc->inode, fop->xl);
+        if (ctx) {
+            ctx->heal_count += 1;
         }
     }
-
-    /* Heal lookups are not executed concurrently with anything else. So, when
-     * a lookup finishes, it's safe to access heal->good and heal->bad without
-     * acquiring any lock.
-     */
-    heal->good = good;
-    heal->bad = bad;
-
-    heal->lookup = fop;
-
-    ec_resume_parent(fop, fop->answer != NULL ? 0 : fop->error);
+    UNLOCK(&loc->inode->lock);
 }
 
-int32_t ec_heal_entry_lookup_cbk(call_frame_t * frame, void * cookie,
-                                 xlator_t * this, int32_t op_ret,
-                                 int32_t op_errno, inode_t * inode,
-                                 struct iatt * buf, dict_t * xdata,
-                                 struct iatt * postparent)
-{
-    ec_heal_lookup_resume(cookie);
-
-    return 0;
-}
-
-int32_t ec_heal_inode_lookup_cbk(call_frame_t * frame, void * cookie,
-                                 xlator_t * this, int32_t op_ret,
-                                 int32_t op_errno, inode_t * inode,
-                                 struct iatt * buf, dict_t * xdata,
-                                 struct iatt * postparent)
+void
+ec_reset_entry_healing(ec_fop_data_t *fop)
 {
-    ec_heal_lookup_resume(cookie);
+    ec_inode_t *ctx = NULL;
+    loc_t *loc = NULL;
+    int32_t heal_count = 0;
+    if (!fop)
+        return;
 
-    return 0;
+    loc = &fop->loc[0];
+    LOCK(&loc->inode->lock);
+    {
+        ctx = __ec_inode_get(loc->inode, fop->xl);
+        if (ctx) {
+            ctx->heal_count += -1;
+            heal_count = ctx->heal_count;
+        }
+    }
+    UNLOCK(&loc->inode->lock);
+    GF_ASSERT(heal_count >= 0);
 }
 
-uintptr_t ec_heal_check(ec_fop_data_t * fop, uintptr_t * pgood)
+uintptr_t
+ec_heal_check(ec_fop_data_t *fop, uintptr_t *pgood)
 {
-    ec_cbk_data_t * cbk;
-    uintptr_t mask[2] = { 0, 0 };
+    ec_cbk_data_t *cbk;
+    uintptr_t mask[2] = {0, 0};
 
     list_for_each_entry(cbk, &fop->cbk_list, list)
     {
         mask[cbk->op_ret >= 0] |= cbk->mask;
     }
 
-    if (pgood != NULL)
-    {
+    if (pgood != NULL) {
         *pgood = mask[1];
     }
 
     return mask[0];
 }
 
-void ec_heal_update(ec_fop_data_t * fop, int32_t is_open)
+void
+ec_heal_update(ec_fop_data_t *fop, int32_t is_open)
 {
-    ec_heal_t * heal = fop->data;
+    ec_heal_t *heal = fop->data;
     uintptr_t good, bad;
 
     bad = ec_heal_check(fop, &good);
@@ -167,8 +173,7 @@ void ec_heal_update(ec_fop_data_t * fop, int32_t is_open)
     LOCK(&heal->lock);
 
     heal->bad &= ~bad;
-    if (is_open)
-    {
+    if (is_open) {
         heal->open |= good;
     }
 
@@ -177,9 +182,10 @@ void ec_heal_update(ec_fop_data_t * fop, int32_t is_open)
     fop->error = 0;
 }
 
-void ec_heal_avoid(ec_fop_data_t * fop)
+void
+ec_heal_avoid(ec_fop_data_t *fop)
 {
-    ec_heal_t * heal = fop->data;
+    ec_heal_t *heal = fop->data;
     uintptr_t bad;
 
     bad = ec_heal_check(fop, NULL);
@@ -191,1452 +197,3171 @@ void ec_heal_avoid(ec_fop_data_t * fop)
     UNLOCK(&heal->lock);
 }
 
-int32_t ec_heal_mkdir_cbk(call_frame_t * frame, void * cookie, xlator_t * this,
-                          int32_t op_ret, int32_t op_errno, inode_t * inode,
-                          struct iatt * buf, struct iatt * preparent,
-                          struct iatt * postparent, dict_t * xdata)
+int32_t
+ec_heal_lock_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                 int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
-    ec_heal_update(cookie, 0);
+    ec_fop_data_t *fop = cookie;
+    ec_heal_t *heal = fop->data;
+
+    if (op_ret >= 0) {
+        GF_ASSERT(
+            ec_set_inode_size(heal->fop, heal->fd->inode, heal->total_size));
+    }
 
     return 0;
 }
 
-int32_t ec_heal_mknod_cbk(call_frame_t * frame, void * cookie, xlator_t * this,
-                          int32_t op_ret, int32_t op_errno, inode_t * inode,
-                          struct iatt * buf, struct iatt * preparent,
-                          struct iatt * postparent, dict_t * xdata)
+void
+ec_heal_lock(ec_heal_t *heal, int32_t type, fd_t *fd, loc_t *loc, off_t offset,
+             size_t size)
 {
-    ec_heal_update(cookie, 0);
+    struct gf_flock flock;
+    fop_inodelk_cbk_t cbk = NULL;
 
-    return 0;
+    flock.l_type = type;
+    flock.l_whence = SEEK_SET;
+    flock.l_start = offset;
+    flock.l_len = size;
+    flock.l_pid = 0;
+    flock.l_owner.len = 0;
+
+    if (type == F_UNLCK) {
+        /* Remove inode size information before unlocking it. */
+        if (fd == NULL) {
+            ec_clear_inode_info(heal->fop, heal->loc.inode);
+        } else {
+            ec_clear_inode_info(heal->fop, heal->fd->inode);
+        }
+        cbk = ec_lock_unlocked;
+    } else {
+        /* Otherwise use the callback to update size information. */
+        cbk = ec_heal_lock_cbk;
+    }
+
+    if (fd != NULL) {
+        ec_finodelk(heal->fop->frame, heal->xl,
+                    &heal->fop->frame->root->lk_owner, heal->fop->mask,
+                    EC_MINIMUM_ALL, cbk, heal, heal->xl->name, fd, F_SETLKW,
+                    &flock, NULL);
+    } else {
+        ec_inodelk(heal->fop->frame, heal->xl,
+                   &heal->fop->frame->root->lk_owner, heal->fop->mask,
+                   EC_MINIMUM_ALL, cbk, heal, heal->xl->name, loc, F_SETLKW,
+                   &flock, NULL);
+    }
 }
 
-int32_t ec_heal_symlink_cbk(call_frame_t * frame, void * cookie,
-                            xlator_t * this, int32_t op_ret, int32_t op_errno,
-                            inode_t * inode, struct iatt * buf,
-                            struct iatt * preparent, struct iatt * postparent,
-                            dict_t * xdata)
+void
+ec_heal_inodelk(ec_heal_t *heal, int32_t type, int32_t use_fd, off_t offset,
+                size_t size)
 {
-    ec_heal_update(cookie, 0);
-
-    return 0;
+    ec_heal_lock(heal, type, use_fd ? heal->fd : NULL, &heal->loc, offset,
+                 size);
 }
 
-int32_t ec_heal_create_cbk(call_frame_t * frame, void * cookie,
-                           xlator_t * this, int32_t op_ret, int32_t op_errno,
-                           fd_t * fd, inode_t * inode, struct iatt * buf,
-                           struct iatt * preparent, struct iatt * postparent,
-                           dict_t * xdata)
+int32_t
+ec_heal_xattr_clean(dict_t *dict, char *key, data_t *data, void *arg)
 {
-    ec_heal_update(cookie, 1);
+    dict_t *base = arg;
+
+    if (ec_ignorable_key_match(NULL, key, NULL, NULL)) {
+        dict_del(dict, key);
+        return 0;
+    }
+
+    if (dict_get(base, key) != NULL)
+        dict_del(dict, key);
 
     return 0;
 }
 
-int32_t ec_heal_setattr_cbk(call_frame_t * frame, void * cookie,
-                            xlator_t * this, int32_t op_ret, int32_t op_errno,
-                            struct iatt * preop_stbuf,
-                            struct iatt * postop_stbuf,
-                            dict_t * xdata)
+/********************************************************************
+ * ec_wind_xattrop_parallel:
+ *              Helper function to update the extended attributes
+ *    in parallel.
+ *
+ *******************************************************************/
+void
+ec_wind_xattrop_parallel(call_frame_t *frame, xlator_t *subvol, int child_index,
+                         loc_t *loc, gf_xattrop_flags_t flags, dict_t **dict,
+                         dict_t *xdata)
 {
-    ec_heal_update(cookie, 0);
-
-    return 0;
+    gf_msg_debug("EC", 0, "WIND: on child %d ", child_index);
+    STACK_WIND_COOKIE(
+        frame, cluster_xattrop_cbk, (void *)(uintptr_t)child_index, subvol,
+        subvol->fops->xattrop, loc, flags, dict[child_index], xdata);
 }
 
-int32_t ec_heal_setxattr_cbk(call_frame_t * frame, void * cookie,
-                             xlator_t * this, int32_t op_ret, int32_t op_errno,
-                             dict_t * xdata)
+int32_t
+ec_heal_writev_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                   int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
+                   struct iatt *postbuf, dict_t *xdata)
 {
+    ec_fop_data_t *fop = cookie;
+    ec_heal_t *heal = fop->data;
+
+    ec_trace("WRITE_CBK", cookie, "ret=%d, errno=%d", op_ret, op_errno);
+
+    gf_msg_debug(fop->xl->name, 0,
+                 "%s: write op_ret %d, op_errno %s"
+                 " at %" PRIu64,
+                 uuid_utoa(heal->fd->inode->gfid), op_ret, strerror(op_errno),
+                 heal->offset);
+
     ec_heal_update(cookie, 0);
 
     return 0;
 }
 
-int32_t ec_heal_removexattr_cbk(call_frame_t * frame, void * cookie,
-                                xlator_t * this, int32_t op_ret,
-                                int32_t op_errno, dict_t * xdata)
+int32_t
+ec_heal_readv_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                  int32_t op_ret, int32_t op_errno, struct iovec *vector,
+                  int32_t count, struct iatt *stbuf, struct iobref *iobref,
+                  dict_t *xdata)
 {
-    ec_heal_update(cookie, 0);
+    ec_fop_data_t *fop = cookie;
+    ec_heal_t *heal = fop->data;
+
+    ec_trace("READ_CBK", fop, "ret=%d, errno=%d", op_ret, op_errno);
+
+    ec_heal_avoid(fop);
+
+    if (op_ret > 0) {
+        gf_msg_debug(fop->xl->name, 0,
+                     "%s: read succeeded, proceeding "
+                     "to write at %" PRIu64,
+                     uuid_utoa(heal->fd->inode->gfid), heal->offset);
+        ec_writev(heal->fop->frame, heal->xl, heal->bad, EC_MINIMUM_ONE,
+                  ec_heal_writev_cbk, heal, heal->fd, vector, count,
+                  heal->offset, 0, iobref, NULL);
+    } else {
+        if (op_ret < 0) {
+            gf_msg_debug(fop->xl->name, 0,
+                         "%s: read failed %s, failing "
+                         "to heal block at %" PRIu64,
+                         uuid_utoa(heal->fd->inode->gfid), strerror(op_errno),
+                         heal->offset);
+            heal->bad = 0;
+        }
+        heal->done = 1;
+    }
 
     return 0;
 }
 
-int32_t ec_heal_link_cbk(call_frame_t * frame, void * cookie, xlator_t * this,
-                         int32_t op_ret, int32_t op_errno, inode_t * inode,
-                         struct iatt * buf, struct iatt * preparent,
-                         struct iatt * postparent, dict_t * xdata)
+void
+ec_heal_data_block(ec_heal_t *heal)
 {
-    ec_fop_data_t * fop = cookie;
-    ec_heal_t * heal = fop->data;
-    uintptr_t good, bad;
+    ec_trace("DATA", heal->fop, "good=%lX, bad=%lX", heal->good, heal->bad);
 
-    bad = ec_heal_check(fop, &good);
-    ec_heal_exclude(heal, good);
+    if ((heal->good != 0) && (heal->bad != 0) &&
+        (heal->iatt.ia_type == IA_IFREG)) {
+        ec_readv(heal->fop->frame, heal->xl, heal->good, EC_MINIMUM_MIN,
+                 ec_heal_readv_cbk, heal, heal->fd, heal->size, heal->offset, 0,
+                 NULL);
+    }
+}
 
-    if (bad != 0)
-    {
-        fop->error = 0;
+/* FOP: fheal */
 
-        xdata = fop->xdata;
-        fop = fop->parent;
+void
+ec_fheal(call_frame_t *frame, xlator_t *this, uintptr_t target,
+         uint32_t fop_flags, fop_fheal_cbk_t func, void *data, fd_t *fd,
+         int32_t partial, dict_t *xdata)
+{
+    ec_fd_t *ctx = ec_fd_get(fd, this);
 
-        ec_create(fop->frame, fop->xl, bad, EC_MINIMUM_ONE,
-                  ec_heal_create_cbk, heal, &heal->loc, 0,
-                  st_mode_from_ia(heal->iatt.ia_prot, IA_INVAL),
-                  0, heal->fd, xdata);
+    if (ctx != NULL) {
+        gf_msg_trace("ec", 0, "FHEAL ctx: flags=%X, open=%" PRIXPTR, ctx->flags,
+                     ctx->open);
+        ec_heal(frame, this, target, fop_flags, func, data, &ctx->loc, partial,
+                xdata);
     }
-
-    return 0;
 }
 
-int32_t ec_heal_target_open_cbk(call_frame_t * frame, void * cookie,
-                                xlator_t * this, int32_t op_ret,
-                                int32_t op_errno, fd_t * fd, dict_t * xdata)
+/* Common heal code */
+void
+ec_mask_to_char_array(uintptr_t mask, unsigned char *array, int numsubvols)
 {
-    ec_heal_update(cookie, 1);
+    int i = 0;
 
-    return 0;
+    for (i = 0; i < numsubvols; i++)
+        array[i] = ((mask >> i) & 1);
 }
 
-int32_t ec_heal_source_open_cbk(call_frame_t * frame, void * cookie,
-                                xlator_t * this, int32_t op_ret,
-                                int32_t op_errno, fd_t * fd, dict_t * xdata)
+uintptr_t
+ec_char_array_to_mask(unsigned char *array, int numsubvols)
 {
-    ec_heal_avoid(cookie);
+    int i = 0;
+    uintptr_t mask = 0;
 
-    return 0;
+    if (array == NULL)
+        goto out;
+
+    for (i = 0; i < numsubvols; i++)
+        if (array[i])
+            mask |= (1ULL << i);
+out:
+    return mask;
 }
 
-int32_t ec_heal_reopen_cbk(call_frame_t * frame, void * cookie,
-                           xlator_t * this, int32_t op_ret, int32_t op_errno,
-                           fd_t * fd, dict_t * xdata)
+int
+ec_heal_entry_find_direction(ec_t *ec, default_args_cbk_t *replies,
+                             uint64_t *versions, uint64_t *dirty,
+                             unsigned char *sources,
+                             unsigned char *healed_sinks)
 {
-    ec_fop_data_t * fop = cookie;
-    ec_fd_t * ctx;
-    uintptr_t good;
+    uint64_t xattr[EC_VERSION_SIZE] = {0};
+    int source = -1;
+    uint64_t max_version = 0;
+    int ret = 0;
+    int i = 0;
+
+    for (i = 0; i < ec->nodes; i++) {
+        if (!replies[i].valid)
+            continue;
 
-    ec_heal_check(fop, &good);
+        if (replies[i].op_ret == -1)
+            continue;
 
-    if (good != 0)
-    {
-        LOCK(&fd->lock);
+        if (source == -1)
+            source = i;
 
-        ctx = __ec_fd_get(fd, fop->xl);
-        if (ctx != NULL) {
-            ctx->bad &= ~good;
-            ctx->open |= good;
+        ret = ec_dict_get_array(replies[i].xdata, EC_XATTR_VERSION, xattr,
+                                EC_VERSION_SIZE);
+        if (ret == 0) {
+            versions[i] = xattr[EC_DATA_TXN];
+            if (max_version < versions[i]) {
+                max_version = versions[i];
+                source = i;
+            }
         }
 
-        UNLOCK(&fd->lock);
+        memset(xattr, 0, sizeof(xattr));
+        ret = ec_dict_get_array(replies[i].xdata, EC_XATTR_DIRTY, xattr,
+                                EC_VERSION_SIZE);
+        if (ret == 0) {
+            dirty[i] = xattr[EC_DATA_TXN];
+        }
     }
 
-    return 0;
-}
+    if (source < 0)
+        goto out;
 
-int32_t ec_heal_create(ec_heal_t * heal, uintptr_t mask, int32_t try_link)
-{
-    loc_t loc;
-    dict_t * xdata;
+    for (i = 0; i < ec->nodes; i++) {
+        if (!replies[i].valid)
+            continue;
 
-    xdata = dict_new();
-    if (xdata == NULL)
-    {
-        return ENOMEM;
+        if (replies[i].op_ret == -1)
+            continue;
+
+        if (versions[i] == versions[source])
+            sources[i] = 1;
+        else
+            healed_sinks[i] = 1;
     }
 
-    if (dict_set_static_bin(xdata, "gfid-req", heal->iatt.ia_gfid,
-                            sizeof(uuid_t)) != 0)
-    {
-        dict_unref(xdata);
+out:
+    return source;
+}
 
-        return ENOMEM;
+int
+ec_adjust_versions(call_frame_t *frame, ec_t *ec, ec_txn_t type, inode_t *inode,
+                   int source, unsigned char *sources,
+                   unsigned char *healed_sinks, uint64_t *versions,
+                   uint64_t *dirty)
+{
+    int i = 0;
+    int ret = 0;
+    int call_count = 0;
+    dict_t **xattr = NULL;
+    int op_ret = 0;
+    loc_t loc = {0};
+    gf_boolean_t erase_dirty = _gf_false;
+    uint64_t *versions_xattr = NULL;
+    uint64_t *dirty_xattr = NULL;
+    uint64_t allzero[2] = {0};
+    unsigned char *on = NULL;
+    unsigned char *output = NULL;
+    default_args_cbk_t *replies = NULL;
+
+    /* Allocate the required memory */
+    loc.inode = inode_ref(inode);
+    gf_uuid_copy(loc.gfid, inode->gfid);
+    on = alloca0(ec->nodes);
+    output = alloca0(ec->nodes);
+    EC_REPLIES_ALLOC(replies, ec->nodes);
+    xattr = GF_CALLOC(ec->nodes, sizeof(*xattr), gf_common_mt_pointer);
+    if (!xattr) {
+        op_ret = -ENOMEM;
+        goto out;
+    }
+    for (i = 0; i < ec->nodes; i++) {
+        xattr[i] = dict_new();
+        if (!xattr[i]) {
+            op_ret = -ENOMEM;
+            goto out;
+        }
     }
 
-    if ((heal->iatt.ia_type == IA_IFREG) && try_link)
-    {
-        memset(&loc, 0, sizeof(loc));
-        loc.inode = heal->loc.inode;
-        gf_uuid_copy(loc.gfid, heal->iatt.ia_gfid);
+    /* dirty xattr represents if the file/dir needs heal. Unless all the
+     * copies are healed, don't erase it */
+    if (EC_COUNT(sources, ec->nodes) + EC_COUNT(healed_sinks, ec->nodes) ==
+        ec->nodes)
+        erase_dirty = _gf_true;
+    else
+        op_ret = -ENOTCONN;
 
-        ec_link(heal->fop->frame, heal->xl, mask, EC_MINIMUM_ONE,
-                ec_heal_link_cbk, heal, &loc, &heal->loc, xdata);
+    /* Populate the xattr array */
+    for (i = 0; i < ec->nodes; i++) {
+        if (!sources[i] && !healed_sinks[i])
+            continue;
+        versions_xattr = GF_CALLOC(EC_VERSION_SIZE, sizeof(*versions_xattr),
+                                   gf_common_mt_pointer);
+        if (!versions_xattr) {
+            op_ret = -ENOMEM;
+            continue;
+        }
 
-        dict_unref(xdata);
+        versions_xattr[type] = hton64(versions[source] - versions[i]);
+        ret = dict_set_bin(xattr[i], EC_XATTR_VERSION, versions_xattr,
+                           (sizeof(*versions_xattr) * EC_VERSION_SIZE));
+        if (ret < 0) {
+            op_ret = -ENOMEM;
+            continue;
+        }
 
-        return 0;
-    }
+        if (erase_dirty) {
+            dirty_xattr = GF_CALLOC(EC_VERSION_SIZE, sizeof(*dirty_xattr),
+                                    gf_common_mt_pointer);
+            if (!dirty_xattr) {
+                op_ret = -ENOMEM;
+                continue;
+            }
 
-    switch (heal->iatt.ia_type)
-    {
-        case IA_IFDIR:
-            ec_mkdir(heal->fop->frame, heal->xl, mask, EC_MINIMUM_ONE,
-                     ec_heal_mkdir_cbk, heal, &heal->loc,
-                     st_mode_from_ia(heal->iatt.ia_prot, IA_INVAL),
-                     0, xdata);
+            dirty_xattr[type] = hton64(-dirty[i]);
+            ret = dict_set_bin(xattr[i], EC_XATTR_DIRTY, dirty_xattr,
+                               (sizeof(*dirty_xattr) * EC_VERSION_SIZE));
+            if (ret < 0) {
+                op_ret = -ENOMEM;
+                continue;
+            }
+        }
 
-            break;
+        if (memcmp(versions_xattr, allzero,
+                   (sizeof(*versions_xattr) * EC_VERSION_SIZE)) == 0) {
+            if (!erase_dirty) {
+                continue;
+            }
 
-        case IA_IFLNK:
-            ec_symlink(heal->fop->frame, heal->xl, mask, EC_MINIMUM_ONE,
-                       ec_heal_symlink_cbk, heal, heal->symlink, &heal->loc,
-                       0, xdata);
+            if (memcmp(dirty_xattr, allzero,
+                       (sizeof(*dirty_xattr) * EC_VERSION_SIZE)) == 0) {
+                continue;
+            }
+        }
 
-            break;
+        on[i] = 1;
+        call_count++;
+    }
 
-        case IA_IFREG:
-            ec_create(heal->fop->frame, heal->xl, mask, EC_MINIMUM_ONE,
-                      ec_heal_create_cbk, heal, &heal->loc, 0,
-                      st_mode_from_ia(heal->iatt.ia_prot, IA_INVAL),
-                      0, heal->fd, xdata);
+    /* Update the bricks with xattr */
+    if (call_count) {
+        PARALLEL_FOP_ONLIST(ec->xl_list, on, ec->nodes, replies, frame,
+                            ec_wind_xattrop_parallel, &loc,
+                            GF_XATTROP_ADD_ARRAY64, xattr, NULL);
+        ret = cluster_fop_success_fill(replies, ec->nodes, output);
+    }
 
-            break;
+    if (ret < call_count) {
+        op_ret = -ENOTCONN;
+        goto out;
+    }
 
-        default:
-            ec_mknod(heal->fop->frame, heal->xl, mask, EC_MINIMUM_ONE,
-                     ec_heal_mknod_cbk, heal, &heal->loc,
-                     st_mode_from_ia(heal->iatt.ia_prot, IA_INVAL),
-                     heal->iatt.ia_rdev, 0, xdata);
+out:
+    /* Cleanup */
+    if (xattr) {
+        for (i = 0; i < ec->nodes; i++) {
+            if (xattr[i])
+                dict_unref(xattr[i]);
+        }
+        GF_FREE(xattr);
+    }
+    cluster_replies_wipe(replies, ec->nodes);
+    loc_wipe(&loc);
+    return op_ret;
+}
 
-            break;
+int
+ec_heal_metadata_find_direction(ec_t *ec, default_args_cbk_t *replies,
+                                uint64_t *versions, uint64_t *dirty,
+                                unsigned char *sources,
+                                unsigned char *healed_sinks)
+{
+    uint64_t xattr[EC_VERSION_SIZE] = {0};
+    uint64_t max_version = 0;
+    int same_count = 0;
+    int max_same_count = 0;
+    int same_source = -1;
+    int ret = 0;
+    int i = 0;
+    int j = 0;
+    int *groups = NULL;
+    struct iatt source_ia = {0};
+    struct iatt child_ia = {0};
+
+    groups = alloca0(ec->nodes * sizeof(*groups));
+    for (i = 0; i < ec->nodes; i++)
+        groups[i] = -1;
+
+    for (i = 0; i < ec->nodes; i++) {
+        if (!replies[i].valid)
+            continue;
+        if (replies[i].op_ret < 0)
+            continue;
+        ret = ec_dict_get_array(replies[i].xdata, EC_XATTR_VERSION, xattr,
+                                EC_VERSION_SIZE);
+        if (ret == 0) {
+            versions[i] = xattr[EC_METADATA_TXN];
+        }
+
+        memset(xattr, 0, sizeof(xattr));
+        ret = ec_dict_get_array(replies[i].xdata, EC_XATTR_DIRTY, xattr,
+                                EC_VERSION_SIZE);
+        if (ret == 0) {
+            dirty[i] = xattr[EC_METADATA_TXN];
+        }
+        if (groups[i] >= 0) /*Already part of group*/
+            continue;
+        groups[i] = i;
+        same_count = 1;
+        source_ia = replies[i].stat;
+        for (j = i + 1; j < ec->nodes; j++) {
+            if (!replies[j].valid || replies[j].op_ret < 0)
+                continue;
+            child_ia = replies[j].stat;
+            if (!IA_EQUAL(source_ia, child_ia, gfid) ||
+                !IA_EQUAL(source_ia, child_ia, type) ||
+                !IA_EQUAL(source_ia, child_ia, prot) ||
+                !IA_EQUAL(source_ia, child_ia, uid) ||
+                !IA_EQUAL(source_ia, child_ia, gid))
+                continue;
+            if (!are_dicts_equal(replies[i].xdata, replies[j].xdata,
+                                 ec_sh_key_match, NULL))
+                continue;
+            groups[j] = i;
+            same_count++;
+        }
+
+        if (max_same_count < same_count) {
+            max_same_count = same_count;
+            same_source = i;
+        }
     }
 
-    dict_unref(xdata);
+    if (max_same_count < ec->fragments) {
+        ret = -EIO;
+        goto out;
+    }
 
-    return 0;
+    for (i = 0; i < ec->nodes; i++) {
+        if (groups[i] == groups[same_source])
+            sources[i] = 1;
+        else if (replies[i].valid && replies[i].op_ret >= 0)
+            healed_sinks[i] = 1;
+    }
+    for (i = 0; i < ec->nodes; i++) {
+        if (sources[i] && (versions[i] > max_version)) {
+            same_source = i;
+            max_version = versions[i];
+        }
+    }
+    ret = same_source;
+out:
+    return ret;
 }
 
-int32_t ec_heal_parent_cbk(call_frame_t *frame, void *cookie, xlator_t *xl,
-                           int32_t op_ret, int32_t op_errno, uintptr_t mask,
-                           uintptr_t good, uintptr_t bad, dict_t *xdata)
+int
+__ec_heal_metadata_prepare(call_frame_t *frame, ec_t *ec, inode_t *inode,
+                           unsigned char *locked_on,
+                           default_args_cbk_t *replies, uint64_t *versions,
+                           uint64_t *dirty, unsigned char *sources,
+                           unsigned char *healed_sinks)
 {
-    ec_fop_data_t *fop = cookie;
-    ec_heal_t *heal = fop->data;
+    loc_t loc = {0};
+    unsigned char *output = NULL;
+    unsigned char *lookup_on = NULL;
+    int ret = 0;
+    int source = 0;
+    default_args_cbk_t *greplies = NULL;
+    int i = 0;
+    EC_REPLIES_ALLOC(greplies, ec->nodes);
+
+    loc.inode = inode_ref(inode);
+    gf_uuid_copy(loc.gfid, inode->gfid);
+    output = alloca0(ec->nodes);
+    lookup_on = alloca0(ec->nodes);
+    ret = cluster_lookup(ec->xl_list, locked_on, ec->nodes, replies, output,
+                         frame, ec->xl, &loc, NULL);
+    if (ret <= ec->fragments) {
+        ret = -ENOTCONN;
+        goto out;
+    }
 
-    /* Even if parent self-heal has failed, we try to heal the current entry */
-    ec_heal_create(heal, fop->mask, 0);
+    memcpy(lookup_on, output, ec->nodes);
+    /*Use getxattr to get the filtered xattrs which filter internal xattrs*/
+    ret = cluster_getxattr(ec->xl_list, lookup_on, ec->nodes, greplies, output,
+                           frame, ec->xl, &loc, NULL, NULL);
+    for (i = 0; i < ec->nodes; i++) {
+        if (lookup_on[i] && !output[i]) {
+            replies[i].valid = 0;
+            continue;
+        }
+        if (replies[i].xdata) {
+            dict_unref(replies[i].xdata);
+            replies[i].xdata = NULL;
+            if (greplies[i].xattr)
+                replies[i].xdata = dict_ref(greplies[i].xattr);
+        }
+    }
 
-    return 0;
+    source = ec_heal_metadata_find_direction(ec, replies, versions, dirty,
+                                             sources, healed_sinks);
+    if (source < 0) {
+        ret = -EIO;
+        goto out;
+    }
+    ret = source;
+out:
+    cluster_replies_wipe(greplies, ec->nodes);
+    loc_wipe(&loc);
+    return ret;
 }
 
-void ec_heal_parent(ec_heal_t *heal, uintptr_t mask)
+/* Metadata heal */
+int
+__ec_removexattr_sinks(call_frame_t *frame, ec_t *ec, inode_t *inode,
+                       int source, unsigned char *sources,
+                       unsigned char *healed_sinks, default_args_cbk_t *replies)
 {
-    loc_t parent;
-    int32_t healing = 0;
+    int i = 0;
+    int ret = 0;
+    loc_t loc = {0};
 
-    /* First we try to do a partial heal of the parent directory to avoid
-     * ENOENT/ENOTDIR errors caused by missing parents */
-    if (ec_loc_parent(heal->xl, &heal->loc, &parent) == 0) {
-        if (!__is_root_gfid(parent.gfid)) {
-            ec_heal(heal->fop->frame, heal->xl, mask, EC_MINIMUM_ONE,
-                    ec_heal_parent_cbk, heal, &parent, 1, NULL);
+    loc.inode = inode_ref(inode);
+    gf_uuid_copy(loc.gfid, inode->gfid);
 
-            healing = 1;
+    for (i = 0; i < ec->nodes; i++) {
+        if (i == source)
+            continue;
+        if (!sources[i] && !healed_sinks[i])
+            continue;
+        ret = dict_foreach(replies[i].xdata, ec_heal_xattr_clean,
+                           replies[source].xdata);
+        if (ret < 0) {
+            sources[i] = 0;
+            healed_sinks[i] = 0;
+            continue;
+        }
+
+        if (replies[i].xdata->count == 0) {
+            continue;
+        } else if (sources[i]) {
+            /* This can happen if setxattr/removexattr succeeds on
+             * the bricks but fails to update the version. This
+             * will make sure that the xattrs are made equal after
+             * heal*/
+            sources[i] = 0;
+            healed_sinks[i] = 1;
         }
-        loc_wipe(&parent);
-    }
 
-    if (!healing) {
-        ec_heal_create(heal, mask, 0);
+        ret = syncop_removexattr(ec->xl_list[i], &loc, "", replies[i].xdata,
+                                 NULL);
+        if (ret < 0)
+            healed_sinks[i] = 0;
     }
+
+    loc_wipe(&loc);
+    if (EC_COUNT(healed_sinks, ec->nodes) == 0)
+        return -ENOTCONN;
+    return 0;
 }
 
-void ec_heal_recreate(ec_fop_data_t * fop)
+int
+__ec_heal_metadata(call_frame_t *frame, ec_t *ec, inode_t *inode,
+                   unsigned char *locked_on, unsigned char *sources,
+                   unsigned char *healed_sinks)
 {
-    ec_cbk_data_t * cbk;
-    ec_heal_t * heal = fop->data;
-    uintptr_t mask = 0;
+    loc_t loc = {0};
+    int ret = 0;
+    int source = 0;
+    default_args_cbk_t *replies = NULL;
+    default_args_cbk_t *sreplies = NULL;
+    uint64_t *versions = NULL;
+    uint64_t *dirty = NULL;
+    unsigned char *output = NULL;
+    dict_t *source_dict = NULL;
+    struct iatt source_buf = {0};
+
+    EC_REPLIES_ALLOC(replies, ec->nodes);
+    EC_REPLIES_ALLOC(sreplies, ec->nodes);
+
+    loc.inode = inode_ref(inode);
+    gf_uuid_copy(loc.gfid, inode->gfid);
+    output = alloca0(ec->nodes);
+    versions = alloca0(ec->nodes * sizeof(*versions));
+    dirty = alloca0(ec->nodes * sizeof(*dirty));
+    source = __ec_heal_metadata_prepare(frame, ec, inode, locked_on, replies,
+                                        versions, dirty, sources, healed_sinks);
+    if (source < 0) {
+        ret = -EIO;
+        goto out;
+    }
 
-    if (heal->iatt.ia_type == IA_INVAL)
-    {
-        return;
+    if ((EC_COUNT(sources, ec->nodes) == ec->nodes) ||
+        (EC_COUNT(healed_sinks, ec->nodes) == 0)) {
+        ret = 0;
+        goto erase_dirty;
     }
 
-    list_for_each_entry(cbk, &fop->cbk_list, list)
-    {
-        if ((cbk->op_ret >= 0) || (cbk->op_errno == ENOENT) ||
-            (cbk->op_errno == ENOTDIR))
-        {
-            mask |= cbk->mask;
-        }
+    source_buf = replies[source].stat;
+    ret = cluster_setattr(ec->xl_list, healed_sinks, ec->nodes, sreplies,
+                          output, frame, ec->xl, &loc, &source_buf,
+                          GF_SET_ATTR_MODE | GF_SET_ATTR_UID | GF_SET_ATTR_GID,
+                          NULL);
+    /*In case the operation fails on some of the subvols*/
+    memcpy(healed_sinks, output, ec->nodes);
+    if (EC_COUNT(healed_sinks, ec->nodes) == 0) {
+        ret = -ENOTCONN;
+        goto out;
     }
 
-    if (mask != 0)
-    {
-        ec_heal_parent(heal, mask);
+    ret = __ec_removexattr_sinks(frame, ec, inode, source, sources,
+                                 healed_sinks, replies);
+    if (ret < 0)
+        goto out;
+
+    source_dict = dict_ref(replies[source].xdata);
+    if (dict_foreach_match(source_dict, ec_ignorable_key_match, NULL,
+                           dict_remove_foreach_fn, NULL) == -1) {
+        ret = -ENOMEM;
+        goto out;
     }
-}
 
-int32_t ec_heal_rmdir_cbk(call_frame_t * frame, void * cookie, xlator_t * this,
-                          int32_t op_ret, int32_t op_errno,
-                          struct iatt * preparent, struct iatt * postparent,
-                          dict_t * xdata)
-{
-    ec_heal_update(cookie, 0);
-    ec_heal_recreate(cookie);
+    ret = cluster_setxattr(ec->xl_list, healed_sinks, ec->nodes, replies,
+                           output, frame, ec->xl, &loc, source_dict, 0, NULL);
 
-    return 0;
+    EC_INTERSECT(healed_sinks, healed_sinks, output, ec->nodes);
+    if (EC_COUNT(healed_sinks, ec->nodes) == 0) {
+        ret = -ENOTCONN;
+        goto out;
+    }
+
+erase_dirty:
+    ret = ec_adjust_versions(frame, ec, EC_METADATA_TXN, inode, source, sources,
+                             healed_sinks, versions, dirty);
+out:
+    if (source_dict)
+        dict_unref(source_dict);
+
+    loc_wipe(&loc);
+    cluster_replies_wipe(replies, ec->nodes);
+    cluster_replies_wipe(sreplies, ec->nodes);
+    return ret;
 }
 
-int32_t ec_heal_unlink_cbk(call_frame_t * frame, void * cookie,
-                           xlator_t * this, int32_t op_ret, int32_t op_errno,
-                           struct iatt * preparent, struct iatt * postparent,
-                           dict_t * xdata)
+int
+ec_heal_metadata(call_frame_t *frame, ec_t *ec, inode_t *inode,
+                 unsigned char *sources, unsigned char *healed_sinks)
 {
-    ec_heal_update(cookie, 0);
-    ec_heal_recreate(cookie);
-
-    return 0;
+    unsigned char *locked_on = NULL;
+    unsigned char *up_subvols = NULL;
+    unsigned char *output = NULL;
+    int ret = 0;
+    default_args_cbk_t *replies = NULL;
+
+    EC_REPLIES_ALLOC(replies, ec->nodes);
+    locked_on = alloca0(ec->nodes);
+    output = alloca0(ec->nodes);
+    up_subvols = alloca0(ec->nodes);
+    ec_mask_to_char_array(ec->xl_up, up_subvols, ec->nodes);
+    ret = cluster_inodelk(ec->xl_list, up_subvols, ec->nodes, replies,
+                          locked_on, frame, ec->xl, ec->xl->name, inode, 0, 0);
+    {
+        if (ret <= ec->fragments) {
+            gf_msg_debug(ec->xl->name, 0,
+                         "%s: Skipping heal "
+                         "as only %d number of subvolumes could "
+                         "be locked",
+                         uuid_utoa(inode->gfid), ret);
+            ret = -ENOTCONN;
+            goto unlock;
+        }
+        ret = __ec_heal_metadata(frame, ec, inode, locked_on, sources,
+                                 healed_sinks);
+    }
+unlock:
+    cluster_uninodelk(ec->xl_list, locked_on, ec->nodes, replies, output, frame,
+                      ec->xl, ec->xl->name, inode, 0, 0);
+    cluster_replies_wipe(replies, ec->nodes);
+    return ret;
 }
 
-int32_t
-ec_heal_init (ec_fop_data_t * fop)
+/*entry heal*/
+int
+__ec_heal_entry_prepare(call_frame_t *frame, ec_t *ec, inode_t *inode,
+                        unsigned char *locked_on, uint64_t *versions,
+                        uint64_t *dirty, unsigned char *sources,
+                        unsigned char *healed_sinks)
 {
-    ec_t * ec = fop->xl->private;
-    struct iobuf_pool * pool;
-    inode_t * inode;
-    ec_inode_t * ctx;
-    ec_heal_t * heal = NULL;
-    int32_t error = 0;
+    loc_t loc = {0};
+    int source = 0;
+    int ret = 0;
+    default_args_cbk_t *replies = NULL;
+    unsigned char *output = NULL;
+    dict_t *xdata = NULL;
 
-    inode = fop->loc[0].inode;
-    if (inode == NULL)
-    {
-        gf_log(fop->xl->name, GF_LOG_WARNING, "Unable to start inode healing "
-                                              "because there is not enough "
-                                              "information");
+    EC_REPLIES_ALLOC(replies, ec->nodes);
 
-        return ENODATA;
+    loc.inode = inode_ref(inode);
+    gf_uuid_copy(loc.gfid, inode->gfid);
+    xdata = dict_new();
+    if (!xdata) {
+        ret = -ENOMEM;
+        goto out;
     }
 
-    heal = GF_MALLOC(sizeof(ec_heal_t), ec_mt_ec_heal_t);
-    if (heal == NULL)
-    {
-        return ENOMEM;
+    if (dict_set_uint64(xdata, EC_XATTR_VERSION, 0) ||
+        dict_set_uint64(xdata, EC_XATTR_DIRTY, 0)) {
+        ret = -ENOMEM;
+        goto out;
     }
 
-    memset(heal, 0, sizeof(ec_heal_t));
+    output = alloca0(ec->nodes);
+    ret = cluster_lookup(ec->xl_list, locked_on, ec->nodes, replies, output,
+                         frame, ec->xl, &loc, xdata);
+    if (ret <= ec->fragments) {
+        ret = -ENOTCONN;
+        goto out;
+    }
 
-    if (ec_loc_from_loc(fop->xl, &heal->loc, &fop->loc[0]) != 0) {
-        error = ENOMEM;
+    source = ec_heal_entry_find_direction(ec, replies, versions, dirty, sources,
+                                          healed_sinks);
+    if (source < 0) {
+        ret = -EIO;
+        goto out;
+    }
+    ret = source;
+out:
+    if (xdata)
+        dict_unref(xdata);
+    loc_wipe(&loc);
+    cluster_replies_wipe(replies, ec->nodes);
+    return ret;
+}
+int32_t
+ec_set_new_entry_dirty(ec_t *ec, loc_t *loc, struct iatt *ia,
+                       call_frame_t *frame, xlator_t *this, unsigned char *on)
+{
+    dict_t *xattr = NULL;
+    int32_t ret = -1;
+    default_args_cbk_t *replies = NULL;
+    unsigned char *output = NULL;
+    uint64_t dirty[EC_VERSION_SIZE] = {1, 1};
+    loc_t newloc = {0};
+
+    /*Symlinks don't have any data to be healed*/
+    if (ia->ia_type == IA_IFLNK)
+        dirty[EC_DATA_TXN] = 0;
+
+    newloc.inode = inode_ref(loc->inode);
+    gf_uuid_copy(newloc.gfid, ia->ia_gfid);
+    EC_REPLIES_ALLOC(replies, ec->nodes);
+    output = alloca0(ec->nodes);
+    xattr = dict_new();
+    if (!xattr) {
+        ret = -ENOMEM;
         goto out;
     }
 
-    LOCK_INIT(&heal->lock);
+    ret = ec_dict_set_array(xattr, EC_XATTR_DIRTY, dirty, EC_VERSION_SIZE);
+    if (ret)
+        goto out;
 
-    heal->xl = fop->xl;
-    heal->fop = fop;
-    pool = fop->xl->ctx->iobuf_pool;
-    heal->size = iobpool_default_pagesize(pool) * ec->fragments;
-    heal->partial = fop->int32;
-    fop->heal = heal;
+    ret = cluster_xattrop(ec->xl_list, on, ec->nodes, replies, output, frame,
+                          ec->xl, &newloc, GF_XATTROP_ADD_ARRAY64, xattr, NULL);
 
-    LOCK(&inode->lock);
+    if (ret < ec->fragments) {
+        ret = -ENOTCONN;
+        goto out;
+    }
 
-    ctx = __ec_inode_get(inode, fop->xl);
-    if (ctx == NULL)
-    {
-        error = EIO;
+out:
+    if (xattr)
+        dict_unref(xattr);
+    cluster_replies_wipe(replies, ec->nodes);
+    loc_wipe(&newloc);
+    return ret;
+}
 
-        goto unlock;
+/*Name heal*/
+int
+ec_delete_stale_name(dict_t *gfid_db, char *key, data_t *d, void *data)
+{
+    struct ec_name_data *name_data = data;
+    struct iatt *ia = NULL;
+    ec_t *ec = NULL;
+    loc_t loc = {0};
+    unsigned char *same = data_to_bin(d);
+    default_args_cbk_t *replies = NULL;
+    unsigned char *output = NULL;
+    int ret = 0;
+    int estale_count = 0;
+    int i = 0;
+    call_frame_t *frame = name_data->frame;
+    uuid_t gfid;
+
+    ec = name_data->frame->this->private;
+    EC_REPLIES_ALLOC(replies, ec->nodes);
+    if (EC_COUNT(same, ec->nodes) >= ec->fragments) {
+        ret = 0;
+        goto out;
     }
 
-    if (list_empty(&ctx->heal)) {
-        gf_log("ec", GF_LOG_INFO, "Healing '%s', gfid %s", heal->loc.path,
-               uuid_utoa(heal->loc.gfid));
-    } else {
-        error = EEXIST;
+    loc.parent = inode_ref(name_data->parent);
+    loc.inode = inode_new(name_data->parent->table);
+    if (!loc.inode) {
+        ret = -ENOMEM;
+        goto out;
     }
 
-    list_add_tail(&heal->list, &ctx->heal);
-    heal = NULL;
+    gf_uuid_parse(key, gfid);
+    gf_uuid_copy(loc.pargfid, name_data->parent->gfid);
+    loc.name = name_data->name;
+    output = alloca0(ec->nodes);
+    ret = cluster_lookup(ec->xl_list, name_data->participants, ec->nodes,
+                         replies, output, name_data->frame, ec->xl, &loc, NULL);
 
-unlock:
-    UNLOCK(&inode->lock);
+    for (i = 0; i < ec->nodes; i++) {
+        if (!replies[i].valid)
+            continue;
+        if (replies[i].op_ret == -1) {
+            if (replies[i].op_errno == ESTALE || replies[i].op_errno == ENOENT)
+                estale_count++;
+            else
+                name_data->participants[i] = 0;
+        } else if (gf_uuid_compare(gfid, replies[i].stat.ia_gfid)) {
+            estale_count++;
+            gf_msg_debug(ec->xl->name, 0, "%s/%s: different gfid as %s",
+                         uuid_utoa(name_data->parent->gfid), name_data->name,
+                         key);
+        }
+    }
 
-    if (error == EEXIST) {
-        LOCK(&fop->lock);
+    if (estale_count <= ec->redundancy) {
+        /* We have at least ec->fragments number of fragments, so the
+         * file is recoverable, so don't delete it*/
 
-        fop->jobs++;
-        fop->refs++;
+        /* Please note that the lookup call above could fail with
+         * ENOTCONN on all subvoumes and still this branch will be
+         * true, but in those cases conservatively we decide to not
+         * delete the file until we are sure*/
+        ret = 0;
+        goto out;
+    }
 
-        UNLOCK(&fop->lock);
+    /*Noway to recover, delete the name*/
+    loc_wipe(&loc);
+    loc.parent = inode_ref(name_data->parent);
+    gf_uuid_copy(loc.pargfid, loc.parent->gfid);
+    loc.name = name_data->name;
+    for (i = 0; i < ec->nodes; i++) {
+        if (same[i] && replies[i].valid && (replies[i].op_ret == 0)) {
+            ia = &replies[i].stat;
+            break;
+        }
+    }
 
-        error = 0;
+    if (!ia) {
+        ret = -ENOTCONN;
+        goto out;
     }
 
+    if (IA_ISDIR(ia->ia_type)) {
+        ret = cluster_rmdir(ec->xl_list, same, ec->nodes, replies, output,
+                            frame, ec->xl, &loc, 1, NULL);
+        gf_msg_debug(ec->xl->name, 0,
+                     "cluster rmdir succeeded on %d "
+                     "nodes",
+                     ret);
+    } else {
+        ret = cluster_unlink(ec->xl_list, same, ec->nodes, replies, output,
+                             frame, ec->xl, &loc, 0, NULL);
+        gf_msg_debug(ec->xl->name, 0,
+                     "cluster unlink succeeded on %d "
+                     "nodes",
+                     ret);
+    }
+
+    for (i = 0; i < ec->nodes; i++) {
+        if (output[i]) {
+            same[i] = 0;
+            name_data->enoent[i] = 1;
+        } else {
+            /*op failed*/
+            if (same[i])
+                name_data->participants[i] = 0;
+        }
+    }
+    ret = 0;
+    /*This will help in making decisions about creating names*/
+    dict_del(gfid_db, key);
 out:
-    GF_FREE(heal);
+    if (ret < 0) {
+        gf_msg_debug(ec->xl->name, 0, "%s/%s: heal failed %s",
+                     uuid_utoa(name_data->parent->gfid), name_data->name,
+                     strerror(-ret));
+    }
+    cluster_replies_wipe(replies, ec->nodes);
+    loc_wipe(&loc);
+    return ret;
+}
 
-    return error;
+int
+ec_delete_stale_names(call_frame_t *frame, ec_t *ec, inode_t *parent,
+                      char *name, default_args_cbk_t *replies, dict_t *gfid_db,
+                      unsigned char *enoent, unsigned char *gfidless,
+                      unsigned char *participants)
+{
+    struct ec_name_data name_data = {0};
+
+    name_data.enoent = enoent;
+    name_data.gfidless = gfidless;
+    name_data.participants = participants;
+    name_data.name = name;
+    name_data.parent = parent;
+    name_data.frame = frame;
+    name_data.replies = replies;
+    return dict_foreach(gfid_db, ec_delete_stale_name, &name_data);
 }
 
-void ec_heal_entrylk(ec_heal_t * heal, entrylk_cmd cmd)
+int
+_assign_same(dict_t *dict, char *key, data_t *value, void *data)
 {
-    loc_t loc;
+    struct ec_name_data *name_data = data;
 
-    if (ec_loc_parent(heal->xl, &heal->loc, &loc) != 0) {
-        ec_fop_set_error(heal->fop, EIO);
+    name_data->same = data_to_bin(value);
+    return 0;
+}
 
-        return;
+int
+ec_create_name(call_frame_t *frame, ec_t *ec, inode_t *parent, char *name,
+               default_args_cbk_t *lookup_replies, dict_t *gfid_db,
+               unsigned char *enoent, unsigned char *participants)
+{
+    int ret = 0;
+    int i = 0;
+    struct ec_name_data name_data = {0};
+    struct iatt *ia = NULL;
+    unsigned char *output = 0;
+    unsigned char *output1 = 0;
+    unsigned char *on = NULL;
+    default_args_cbk_t *replies = NULL;
+    loc_t loc = {0};
+    loc_t srcloc = {0};
+    unsigned char *link = NULL;
+    unsigned char *create = NULL;
+    dict_t *xdata = NULL;
+    char *linkname = NULL;
+    ec_config_t config;
+
+    /* There should be just one gfid key */
+    EC_REPLIES_ALLOC(replies, ec->nodes);
+    if (gfid_db->count != 1) {
+        ret = -EINVAL;
+        goto out;
     }
 
-    ec_entrylk(heal->fop->frame, heal->xl, -1, EC_MINIMUM_ALL, NULL, NULL,
-               heal->xl->name, &loc, NULL, cmd, ENTRYLK_WRLCK, NULL);
+    ret = dict_foreach(gfid_db, _assign_same, &name_data);
+    if (ret < 0)
+        goto out;
+    /*There should at least be one valid success reply with gfid*/
+    for (i = 0; i < ec->nodes; i++)
+        if (name_data.same[i])
+            break;
 
-    loc_wipe(&loc);
-}
+    if (i == ec->nodes) {
+        ret = -EINVAL;
+        goto out;
+    }
 
-void ec_heal_inodelk(ec_heal_t * heal, int32_t type, int32_t use_fd,
-                     off_t offset, size_t size)
-{
-    struct gf_flock flock;
+    ia = &lookup_replies[i].stat;
+    xdata = dict_new();
+    loc.parent = inode_ref(parent);
+    gf_uuid_copy(loc.pargfid, parent->gfid);
+    loc.inode = inode_new(parent->table);
+    if (loc.inode)
+        srcloc.inode = inode_ref(loc.inode);
+    gf_uuid_copy(srcloc.gfid, ia->ia_gfid);
+    if (!loc.inode || !xdata ||
+        dict_set_static_bin(xdata, "gfid-req", ia->ia_gfid,
+                            sizeof(ia->ia_gfid))) {
+        ret = -ENOMEM;
+        goto out;
+    }
+    loc.name = name;
+    link = alloca0(ec->nodes);
+    create = alloca0(ec->nodes);
+    on = alloca0(ec->nodes);
+    output = alloca0(ec->nodes);
+    output1 = alloca0(ec->nodes);
 
-    flock.l_type = type;
-    flock.l_whence = SEEK_SET;
-    flock.l_start = offset;
-    flock.l_len = size;
-    flock.l_pid = 0;
-    flock.l_owner.len = 0;
+    for (i = 0; i < ec->nodes; i++) {
+        if (!lookup_replies[i].valid)
+            continue;
+        if (lookup_replies[i].op_ret)
+            continue;
+        on[i] = 1;
+    }
+    switch (ia->ia_type) {
+        case IA_IFDIR:
+            ec_set_new_entry_dirty(ec, &loc, ia, frame, ec->xl, on);
+            (void)cluster_mkdir(
+                ec->xl_list, enoent, ec->nodes, replies, output, frame, ec->xl,
+                &loc, st_mode_from_ia(ia->ia_prot, ia->ia_type), 0, xdata);
+            break;
 
-    if (use_fd)
-    {
-        ec_finodelk(heal->fop->frame, heal->xl, heal->fop->mask,
-                    EC_MINIMUM_ALL, NULL, NULL, heal->xl->name, heal->fd,
-                    F_SETLKW, &flock, NULL);
+        case IA_IFLNK:
+            /*Check for hard links and create/link*/
+            ret = cluster_lookup(ec->xl_list, enoent, ec->nodes, replies,
+                                 output, frame, ec->xl, &srcloc, NULL);
+            for (i = 0; i < ec->nodes; i++) {
+                if (output[i]) {
+                    link[i] = 1;
+                } else {
+                    if (replies[i].op_errno == ENOENT ||
+                        replies[i].op_errno == ESTALE) {
+                        create[i] = 1;
+                    }
+                }
+            }
+
+            if (EC_COUNT(link, ec->nodes)) {
+                cluster_link(ec->xl_list, link, ec->nodes, replies, output1,
+                             frame, ec->xl, &srcloc, &loc, NULL);
+            }
+
+            if (EC_COUNT(create, ec->nodes)) {
+                cluster_readlink(ec->xl_list, name_data.same, ec->nodes,
+                                 replies, output, frame, ec->xl, &srcloc, 4096,
+                                 NULL);
+                if (EC_COUNT(output, ec->nodes) == 0) {
+                    ret = -ENOTCONN;
+                    goto out;
+                }
+
+                for (i = 0; i < ec->nodes; i++) {
+                    if (output[i])
+                        break;
+                }
+                linkname = alloca0(strlen(replies[i].buf) + 1);
+                strcpy(linkname, replies[i].buf);
+                ec_set_new_entry_dirty(ec, &loc, ia, frame, ec->xl, on);
+                cluster_symlink(ec->xl_list, create, ec->nodes, replies, output,
+                                frame, ec->xl, linkname, &loc, 0, xdata);
+            }
+            for (i = 0; i < ec->nodes; i++)
+                if (output1[i])
+                    output[i] = 1;
+            break;
+        case IA_IFREG:
+            ec_set_new_entry_dirty(ec, &loc, ia, frame, ec->xl, on);
+            config.version = EC_CONFIG_VERSION;
+            config.algorithm = EC_CONFIG_ALGORITHM;
+            config.gf_word_size = EC_GF_BITS;
+            config.bricks = ec->nodes;
+            config.redundancy = ec->redundancy;
+            config.chunk_size = EC_METHOD_CHUNK_SIZE;
+
+            ret = ec_dict_set_config(xdata, EC_XATTR_CONFIG, &config);
+            if (ret != 0) {
+                goto out;
+            }
+
+            /* Fall through */
+
+        default:
+            ret = dict_set_int32(xdata, GLUSTERFS_INTERNAL_FOP_KEY, 1);
+            if (ret)
+                goto out;
+            ret = cluster_mknod(
+                ec->xl_list, enoent, ec->nodes, replies, output, frame, ec->xl,
+                &loc, st_mode_from_ia(ia->ia_prot, ia->ia_type),
+                makedev(ia_major(ia->ia_rdev), ia_minor(ia->ia_rdev)), 0,
+                xdata);
+            break;
     }
-    else
-    {
-        ec_inodelk(heal->fop->frame, heal->xl, heal->fop->mask, EC_MINIMUM_ALL,
-                   NULL, NULL, heal->xl->name, &heal->loc, F_SETLKW, &flock,
-                   NULL);
+
+    for (i = 0; i < ec->nodes; i++) {
+        if (enoent[i] && !output[i])
+            participants[i] = 0;
     }
+
+    ret = 0;
+out:
+    if (ret < 0)
+        gf_msg_debug(ec->xl->name, 0, "%s/%s: heal failed %s",
+                     uuid_utoa(parent->gfid), name, strerror(-ret));
+    cluster_replies_wipe(replies, ec->nodes);
+    loc_wipe(&loc);
+    loc_wipe(&srcloc);
+    if (xdata)
+        dict_unref(xdata);
+    return ret;
 }
 
-void ec_heal_lookup(ec_heal_t *heal, uintptr_t mask)
+int
+__ec_heal_name(call_frame_t *frame, ec_t *ec, inode_t *parent, char *name,
+               unsigned char *participants)
 {
-    dict_t * xdata;
-    int32_t error = ENOMEM;
-
+    unsigned char *output = NULL;
+    unsigned char *enoent = NULL;
+    default_args_cbk_t *replies = NULL;
+    dict_t *xdata = NULL;
+    dict_t *gfid_db = NULL;
+    int ret = 0;
+    loc_t loc = {0};
+    int i = 0;
+    struct iatt *ia = NULL;
+    char gfid[64] = {0};
+    unsigned char *same = NULL;
+    unsigned char *gfidless = NULL;
+
+    EC_REPLIES_ALLOC(replies, ec->nodes);
+    loc.parent = inode_ref(parent);
+    loc.inode = inode_new(parent->table);
+    gf_uuid_copy(loc.pargfid, parent->gfid);
+    loc.name = name;
     xdata = dict_new();
-    if (xdata == NULL)
-    {
+    gfid_db = dict_new();
+    if (!xdata || !gfid_db || !loc.inode) {
+        ret = -ENOMEM;
         goto out;
     }
-    if (dict_set_uint64(xdata, "list-xattr", 0) != 0)
-    {
+
+    ret = dict_set_int32(xdata, GF_GFIDLESS_LOOKUP, 1);
+    if (ret) {
+        ret = -ENOMEM;
         goto out;
     }
 
-    ec_lookup(heal->fop->frame, heal->xl, mask, EC_MINIMUM_MIN,
-              ec_heal_inode_lookup_cbk, heal, &heal->loc, xdata);
+    output = alloca0(ec->nodes);
+    gfidless = alloca0(ec->nodes);
+    enoent = alloca0(ec->nodes);
+    ret = cluster_lookup(ec->xl_list, participants, ec->nodes, replies, output,
+                         frame, ec->xl, &loc, NULL);
+    for (i = 0; i < ec->nodes; i++) {
+        if (!replies[i].valid)
+            continue;
 
-    error = 0;
+        if (replies[i].op_ret == -1) {
+            /*If ESTALE comes here, that means parent dir is not
+             * present, nothing to do there, so reset participants
+             * for that brick*/
+            if (replies[i].op_errno == ENOENT)
+                enoent[i] = 1;
+            else
+                participants[i] = 0;
+            continue;
+        }
+        ia = &replies[i].stat;
+        if (gf_uuid_is_null(ia->ia_gfid)) {
+            if (IA_ISDIR(ia->ia_type) || ia->ia_size == 0)
+                gfidless[i] = 1;
+            else
+                participants[i] = 0;
+        } else {
+            uuid_utoa_r(ia->ia_gfid, gfid);
+            ret = dict_get_bin(gfid_db, gfid, (void **)&same);
+            if (ret < 0) {
+                same = alloca0(ec->nodes);
+            }
+            same[i] = 1;
+            if (ret < 0) {
+                ret = dict_set_static_bin(gfid_db, gfid, same, ec->nodes);
+            }
+            if (ret < 0)
+                goto out;
+        }
+    }
 
-out:
-    if (xdata != NULL)
-    {
-        dict_unref(xdata);
+    ret = ec_delete_stale_names(frame, ec, parent, name, replies, gfid_db,
+                                enoent, gfidless, participants);
+
+    if (gfid_db->count == 0) {
+        /* All entries seem to be stale entries and deleted,
+         * nothing more to do.*/
+        goto out;
+    }
+
+    if (gfid_db->count > 1) {
+        gf_msg(ec->xl->name, GF_LOG_INFO, 0, EC_MSG_HEAL_FAIL,
+               "%s/%s: Not able to heal", uuid_utoa(parent->gfid), name);
+        memset(participants, 0, ec->nodes);
+        goto out;
+    }
+
+    EC_INTERSECT(enoent, enoent, participants, ec->nodes);
+    if (EC_COUNT(enoent, ec->nodes) == 0) {
+        ret = 0;
+        goto out;
     }
 
-    ec_fop_set_error(heal->fop, error);
+    ret = ec_create_name(frame, ec, parent, name, replies, gfid_db, enoent,
+                         participants);
+    if (ret >= 0) {
+        /* If ec_create_name() succeeded we return 1 to indicate that a new
+         * file has been created and it will need to be healed. */
+        ret = 1;
+    }
+out:
+    cluster_replies_wipe(replies, ec->nodes);
+    loc_wipe(&loc);
+    if (xdata)
+        dict_unref(xdata);
+    if (gfid_db)
+        dict_unref(gfid_db);
+    return ret;
 }
 
-void ec_heal_remove(ec_heal_t * heal, ec_cbk_data_t * cbk)
+int
+ec_heal_name(call_frame_t *frame, ec_t *ec, inode_t *parent, char *name,
+             unsigned char *participants)
 {
-    if (cbk->iatt[0].ia_type == IA_IFDIR)
-    {
-        // TODO: Remove directory recursively ?
-        ec_rmdir(heal->fop->frame, heal->xl, cbk->mask, EC_MINIMUM_ONE,
-                 ec_heal_rmdir_cbk, heal, &heal->loc, 0, NULL);
+    int ret = 0;
+    default_args_cbk_t *replies = NULL;
+    unsigned char *output = NULL;
+    unsigned char *locked_on = NULL;
+    loc_t loc = {0};
+
+    loc.parent = inode_ref(parent);
+    loc.name = name;
+    loc.inode = inode_new(parent->table);
+    if (!loc.inode) {
+        ret = -ENOMEM;
+        goto out;
     }
-    else
+
+    EC_REPLIES_ALLOC(replies, ec->nodes);
+    output = alloca0(ec->nodes);
+    locked_on = alloca0(ec->nodes);
+    ret = cluster_inodelk(ec->xl_list, participants, ec->nodes, replies,
+                          locked_on, frame, ec->xl, ec->xl->name, parent, 0, 0);
     {
-        ec_unlink(heal->fop->frame, heal->xl, cbk->mask, EC_MINIMUM_ONE,
-                  ec_heal_unlink_cbk, heal, &heal->loc, 0, NULL);
+        if (ret <= ec->fragments) {
+            gf_msg_debug(ec->xl->name, 0,
+                         "%s/%s: Skipping "
+                         "heal as only %d number of subvolumes could "
+                         "be locked",
+                         uuid_utoa(parent->gfid), name, ret);
+            ret = -ENOTCONN;
+            goto unlock;
+        }
+        EC_INTERSECT(participants, participants, locked_on, ec->nodes);
+        ret = __ec_heal_name(frame, ec, parent, name, participants);
     }
+unlock:
+    cluster_uninodelk(ec->xl_list, locked_on, ec->nodes, replies, output, frame,
+                      ec->xl, ec->xl->name, parent, 0, 0);
+out:
+    cluster_replies_wipe(replies, ec->nodes);
+    loc_wipe(&loc);
+    return ret;
 }
 
-void ec_heal_remove_others(ec_heal_t * heal)
+int
+ec_name_heal_handler(xlator_t *subvol, gf_dirent_t *entry, loc_t *parent,
+                     void *data)
 {
-    struct list_head * item;
-    ec_cbk_data_t * cbk;
-
-    item = heal->lookup->cbk_list.next;
-    do
-    {
-        item = item->next;
-        cbk = list_entry(item, ec_cbk_data_t, list);
+    struct ec_name_data *name_data = data;
+    xlator_t *this = THIS;
+    ec_t *ec = this->private;
+    unsigned char *name_on = alloca0(ec->nodes);
+    int i = 0;
+    int ret = 0;
+
+    if (ec->shutdown) {
+        gf_msg_debug(this->name, 0,
+                     "Cancelling directory heal "
+                     "because EC is stopping.");
+        return -ENOTCONN;
+    }
+
+    memcpy(name_on, name_data->participants, ec->nodes);
+    ret = ec_heal_name(name_data->frame, ec, parent->inode, entry->d_name,
+                       name_on);
+
+    if (ret < 0) {
+        memset(name_on, 0, ec->nodes);
+    } else {
+        name_data->heal_pending += ret;
+    }
 
-        if (cbk->op_ret < 0)
-        {
-            if ((cbk->op_errno != ENOENT) && (cbk->op_errno != ENOTDIR) &&
-                (cbk->op_errno != ESTALE))
-            {
-                gf_log(heal->xl->name, GF_LOG_WARNING, "Don't know how to "
-                                                       "remove inode with "
-                                                       "error %d",
-                       cbk->op_errno);
-            }
+    for (i = 0; i < ec->nodes; i++)
+        if (name_data->participants[i] && !name_on[i])
+            name_data->failed_on[i] = 1;
 
-            ec_heal_exclude(heal, cbk->mask);
+    return 0;
+}
 
+int
+ec_heal_names(call_frame_t *frame, ec_t *ec, inode_t *inode,
+              unsigned char *participants, uint32_t *pending)
+{
+    int i = 0;
+    int j = 0;
+    loc_t loc = {0};
+    struct ec_name_data name_data = {0};
+    int ret = 0;
+
+    loc.inode = inode_ref(inode);
+    gf_uuid_copy(loc.gfid, inode->gfid);
+    name_data.frame = frame;
+    name_data.participants = participants;
+    name_data.failed_on = alloca0(ec->nodes);
+    name_data.heal_pending = 0;
+
+    for (i = 0; i < ec->nodes; i++) {
+        if (!participants[i])
             continue;
+        ret = syncop_dir_scan(ec->xl_list[i], &loc, GF_CLIENT_PID_SELF_HEALD,
+                              &name_data, ec_name_heal_handler);
+        if (ret < 0) {
+            break;
         }
+        for (j = 0; j < ec->nodes; j++)
+            if (name_data.failed_on[j])
+                participants[j] = 0;
 
-        ec_heal_remove(heal, cbk);
-    } while (item->next != &heal->lookup->cbk_list);
+        if (EC_COUNT(participants, ec->nodes) <= ec->fragments) {
+            ret = -ENOTCONN;
+            break;
+        }
+    }
+    *pending += name_data.heal_pending;
+
+    loc_wipe(&loc);
+    return ret;
 }
 
-void ec_heal_prepare_others(ec_heal_t * heal)
+int
+__ec_heal_entry(call_frame_t *frame, ec_t *ec, inode_t *inode,
+                unsigned char *heal_on, unsigned char *sources,
+                unsigned char *healed_sinks, uint32_t *pending)
 {
-    struct list_head * item;
-    ec_cbk_data_t * cbk;
-
-    item = heal->lookup->cbk_list.next;
-    while (item->next != &heal->lookup->cbk_list)
+    unsigned char *locked_on = NULL;
+    unsigned char *output = NULL;
+    uint64_t *versions = NULL;
+    uint64_t *dirty = NULL;
+    unsigned char *participants = NULL;
+    default_args_cbk_t *replies = NULL;
+    int ret = 0;
+    int source = 0;
+    int i = 0;
+
+    locked_on = alloca0(ec->nodes);
+    output = alloca0(ec->nodes);
+    versions = alloca0(ec->nodes * sizeof(*versions));
+    dirty = alloca0(ec->nodes * sizeof(*dirty));
+
+    EC_REPLIES_ALLOC(replies, ec->nodes);
+    ret = cluster_inodelk(ec->xl_list, heal_on, ec->nodes, replies, locked_on,
+                          frame, ec->xl, ec->xl->name, inode, 0, 0);
     {
-        item = item->next;
-        cbk = list_entry(item, ec_cbk_data_t, list);
+        if (ret <= ec->fragments) {
+            gf_msg_debug(ec->xl->name, 0,
+                         "%s: Skipping heal "
+                         "as only %d number of subvolumes could "
+                         "be locked",
+                         uuid_utoa(inode->gfid), ret);
+            ret = -ENOTCONN;
+            goto unlock;
+        }
+        ret = __ec_heal_entry_prepare(frame, ec, inode, locked_on, versions,
+                                      dirty, sources, healed_sinks);
+        source = ret;
+    }
+unlock:
+    cluster_uninodelk(ec->xl_list, locked_on, ec->nodes, replies, output, frame,
+                      ec->xl, ec->xl->name, inode, 0, 0);
+    if (ret < 0)
+        goto out;
 
-        if (cbk->op_ret < 0)
-        {
-            if ((cbk->op_errno == ENOENT) || (cbk->op_errno == ESTALE))
-            {
-                ec_heal_create(heal, cbk->mask, 1);
-            }
-            else
-            {
-                gf_log(heal->xl->name, GF_LOG_ERROR, "Don't know how to "
-                                                     "heal error %d",
-                       cbk->op_errno);
+    participants = alloca0(ec->nodes);
+    for (i = 0; i < ec->nodes; i++) {
+        if (sources[i] || healed_sinks[i])
+            participants[i] = 1;
+    }
+    ret = ec_heal_names(frame, ec, inode, participants, pending);
 
-                ec_heal_exclude(heal, cbk->mask);
-            }
-        }
-        else
-        {
-            if ((heal->iatt.ia_type != cbk->iatt[0].ia_type) ||
-                (gf_uuid_compare(heal->iatt.ia_gfid, cbk->iatt[0].ia_gfid) != 0))
-            {
-                ec_heal_remove(heal, cbk);
-            }
+    if (EC_COUNT(participants, ec->nodes) <= ec->fragments)
+        goto out;
+
+    for (i = 0; i < ec->nodes; i++) {
+        if (!participants[i]) {
+            sources[i] = 0;
+            healed_sinks[i] = 0;
         }
     }
+
+    ec_adjust_versions(frame, ec, EC_DATA_TXN, inode, source, sources,
+                       healed_sinks, versions, dirty);
+out:
+    cluster_replies_wipe(replies, ec->nodes);
+    return ret;
 }
 
-int32_t ec_heal_readlink_cbk(call_frame_t * frame, void * cookie,
-                             xlator_t * this, int32_t op_ret, int32_t op_errno,
-                             const char * path, struct iatt * buf,
-                             dict_t * xdata)
+int
+ec_heal_entry(call_frame_t *frame, ec_t *ec, inode_t *inode,
+              unsigned char *sources, unsigned char *healed_sinks,
+              uint32_t *pending)
 {
-    ec_fop_data_t * fop = cookie;
-    ec_heal_t * heal = fop->data;
-
-    if (op_ret >= 0)
+    unsigned char *locked_on = NULL;
+    unsigned char *up_subvols = NULL;
+    unsigned char *output = NULL;
+    char selfheal_domain[1024] = {0};
+    int ret = 0;
+    default_args_cbk_t *replies = NULL;
+
+    EC_REPLIES_ALLOC(replies, ec->nodes);
+    locked_on = alloca0(ec->nodes);
+    output = alloca0(ec->nodes);
+    up_subvols = alloca0(ec->nodes);
+
+    sprintf(selfheal_domain, "%s:self-heal", ec->xl->name);
+    ec_mask_to_char_array(ec->xl_up, up_subvols, ec->nodes);
+    /*If other processes are already doing the heal, don't block*/
+    ret = cluster_tiebreaker_inodelk(ec->xl_list, up_subvols, ec->nodes,
+                                     replies, locked_on, frame, ec->xl,
+                                     selfheal_domain, inode, 0, 0);
     {
-        heal->symlink = gf_strdup(path);
-        if (heal->symlink != NULL)
-        {
-            ec_heal_prepare_others(heal);
-        }
-        else
-        {
-            ec_fop_set_error(fop, EIO);
+        if (ret <= ec->fragments) {
+            gf_msg_debug(ec->xl->name, 0,
+                         "%s: Skipping heal "
+                         "as only %d number of subvolumes could "
+                         "be locked",
+                         uuid_utoa(inode->gfid), ret);
+            ret = -ENOTCONN;
+            goto unlock;
         }
+        ret = __ec_heal_entry(frame, ec, inode, locked_on, sources,
+                              healed_sinks, pending);
     }
-
-    return 0;
+unlock:
+    cluster_uninodelk(ec->xl_list, locked_on, ec->nodes, replies, output, frame,
+                      ec->xl, selfheal_domain, inode, 0, 0);
+    cluster_replies_wipe(replies, ec->nodes);
+    return ret;
 }
 
-ec_cbk_data_t * ec_heal_lookup_check(ec_heal_t * heal, uintptr_t * pgood,
-                                     uintptr_t * pbad)
+/*Find direction for data heal and heal info*/
+int
+ec_heal_data_find_direction(ec_t *ec, default_args_cbk_t *replies,
+                            uint64_t *data_versions, uint64_t *dirty,
+                            uint64_t *size, unsigned char *sources,
+                            unsigned char *healed_sinks,
+                            gf_boolean_t check_ondisksize, int which)
 {
-    ec_fop_data_t * fop = heal->lookup;
-    ec_cbk_data_t * cbk = NULL, * ans = NULL;
-    uintptr_t good = 0, bad = 0;
+    uint64_t xattr[EC_VERSION_SIZE] = {0};
+    char version_size[128] = {0};
+    dict_t *version_size_db = NULL;
+    unsigned char *same = NULL;
+    int max_same_count = 0;
+    int source = 0;
+    int i = 0;
+    int ret = 0;
+    dict_t *dict = NULL;
+    uint64_t source_size = 0;
+
+    version_size_db = dict_new();
+    if (!version_size_db) {
+        ret = -ENOMEM;
+        goto out;
+    }
 
-    list_for_each_entry(ans, &fop->cbk_list, list)
-    {
-        if ((ans->op_ret < 0) && (ans->op_errno == ENOTCONN))
-        {
+    for (i = 0; i < ec->nodes; i++) {
+        if (!replies[i].valid)
+            continue;
+        if (replies[i].op_ret < 0)
             continue;
+        dict = (which == EC_COMBINE_XDATA) ? replies[i].xdata
+                                           : replies[i].xattr;
+
+        ret = ec_dict_get_array(dict, EC_XATTR_VERSION, xattr, EC_VERSION_SIZE);
+        if (ret == 0) {
+            data_versions[i] = xattr[EC_DATA_TXN];
         }
 
-        if (ans == fop->answer)
-        {
-            good |= ans->mask;
-            cbk = ans;
+        memset(xattr, 0, sizeof(xattr));
+        ret = ec_dict_get_array(dict, EC_XATTR_DIRTY, xattr, EC_VERSION_SIZE);
+        if (ret == 0) {
+            dirty[i] = xattr[EC_DATA_TXN];
         }
-        else
-        {
-            bad |= ans->mask;
+        ret = ec_dict_del_number(dict, EC_XATTR_SIZE, &size[i]);
+        /*Build a db of same metadata and data version and size*/
+        snprintf(version_size, sizeof(version_size), "%" PRIu64 "-%" PRIu64,
+                 data_versions[i], size[i]);
+
+        ret = dict_get_bin(version_size_db, version_size, (void **)&same);
+        if (ret < 0) {
+            same = alloca0(ec->nodes);
         }
-    }
-
-    *pgood = good;
-    *pbad = bad;
-
-    return cbk;
-}
-
-void ec_heal_prepare(ec_heal_t * heal)
-{
-    ec_cbk_data_t * cbk;
-    int32_t error = ENOMEM;
 
-    heal->available = heal->good;
+        same[i] = 1;
+        if (max_same_count < EC_COUNT(same, ec->nodes)) {
+            max_same_count = EC_COUNT(same, ec->nodes);
+            source = i;
+        }
 
-    cbk = heal->lookup->answer;
-    if (cbk->op_ret < 0)
-    {
-        if ((cbk->op_errno == ENOENT) || (cbk->op_errno == ENOTDIR))
-        {
-            ec_heal_remove_others(heal);
+        if (ret < 0) {
+            ret = dict_set_static_bin(version_size_db, version_size, same,
+                                      ec->nodes);
         }
-        else
-        {
-            gf_log(heal->xl->name, GF_LOG_ERROR, "Don't know how to heal "
-                                                 "error %d",
-                   cbk->op_errno);
+
+        if (ret < 0) {
+            ret = -ENOMEM;
+            goto out;
         }
     }
-    else
-    {
-        if (heal->iatt.ia_type == IA_IFREG)
-        {
-            heal->fd = fd_create(heal->loc.inode, heal->fop->frame->root->pid);
-            if (heal->fd == NULL)
-            {
-                gf_log(heal->xl->name, GF_LOG_ERROR, "Unable to create a new "
-                                                     "file descriptor");
+    /* If we don't have ec->fragments number of same version,size it is not
+     * recoverable*/
+    if (max_same_count < ec->fragments) {
+        ret = -EIO;
+        goto out;
+    } else {
+        snprintf(version_size, sizeof(version_size), "%" PRIu64 "-%" PRIu64,
+                 data_versions[source], size[source]);
 
-                goto out;
-            }
+        ret = dict_get_bin(version_size_db, version_size, (void **)&same);
+        if (ret < 0)
+            goto out;
+        memcpy(sources, same, ec->nodes);
+        for (i = 0; i < ec->nodes; i++) {
+            if (replies[i].valid && (replies[i].op_ret == 0) && !sources[i])
+                healed_sinks[i] = 1;
         }
+    }
 
-        if (heal->iatt.ia_type == IA_IFLNK)
-        {
-            ec_readlink(heal->fop->frame, heal->xl, cbk->mask, EC_MINIMUM_ONE,
-                        ec_heal_readlink_cbk, heal, &heal->loc,
-                        heal->iatt.ia_size, NULL);
+    /* There could be files with versions, size same but on disk ia_size
+     * could be different because of disk crashes, mark them as sinks as
+     * well*/
+
+    if (check_ondisksize) {
+        source_size = size[source];
+        ec_adjust_size_up(ec, &source_size, _gf_true);
+
+        for (i = 0; i < ec->nodes; i++) {
+            if (sources[i]) {
+                if (replies[i].stat.ia_size != source_size) {
+                    sources[i] = 0;
+                    healed_sinks[i] = 1;
+                    max_same_count--;
+                } else {
+                    source = i;
+                }
+            }
         }
-        else
-        {
-            ec_heal_prepare_others(heal);
+        if (max_same_count < ec->fragments) {
+            ret = -EIO;
+            goto out;
         }
     }
 
-    error = 0;
-
+    ret = source;
 out:
-    ec_fop_set_error(heal->fop, error);
+    if (version_size_db)
+        dict_unref(version_size_db);
+    return ret;
 }
 
-int32_t ec_heal_open_others(ec_heal_t * heal)
+int
+__ec_heal_data_prepare(call_frame_t *frame, ec_t *ec, fd_t *fd,
+                       unsigned char *locked_on, uint64_t *versions,
+                       uint64_t *dirty, uint64_t *size, unsigned char *sources,
+                       unsigned char *healed_sinks, unsigned char *trim,
+                       struct iatt *stbuf)
 {
-    struct list_head * item;
-    ec_cbk_data_t * cbk;
-    uintptr_t mask = 0, open = heal->open;
+    default_args_cbk_t *replies = NULL;
+    default_args_cbk_t *fstat_replies = NULL;
+    unsigned char *output = NULL;
+    unsigned char *fstat_output = NULL;
+    dict_t *xattrs = NULL;
+    uint64_t zero_array[2] = {0};
+    int source = 0;
+    int ret = 0;
+    uint64_t zero_value = 0;
+    int i = 0;
+
+    EC_REPLIES_ALLOC(replies, ec->nodes);
+    EC_REPLIES_ALLOC(fstat_replies, ec->nodes);
+    output = alloca0(ec->nodes);
+    fstat_output = alloca0(ec->nodes);
+    xattrs = dict_new();
+    if (!xattrs ||
+        dict_set_static_bin(xattrs, EC_XATTR_VERSION, zero_array,
+                            sizeof(zero_array)) ||
+        dict_set_static_bin(xattrs, EC_XATTR_DIRTY, zero_array,
+                            sizeof(zero_array)) ||
+        dict_set_static_bin(xattrs, EC_XATTR_SIZE, &zero_value,
+                            sizeof(zero_value))) {
+        ret = -ENOMEM;
+        goto out;
+    }
 
-    item = heal->lookup->cbk_list.next;
-    while (item->next != &heal->lookup->cbk_list)
-    {
-        item = item->next;
-        cbk = list_entry(item, ec_cbk_data_t, list);
+    ret = cluster_fxattrop(ec->xl_list, locked_on, ec->nodes, replies, output,
+                           frame, ec->xl, fd, GF_XATTROP_ADD_ARRAY64, xattrs,
+                           NULL);
 
-        if ((cbk->op_ret < 0) || (cbk->iatt[0].ia_type != IA_IFREG) ||
-            (gf_uuid_compare(heal->iatt.ia_gfid, cbk->iatt[0].ia_gfid) != 0))
-        {
-            ec_heal_exclude(heal, cbk->mask);
-        }
-        else
-        {
-            mask |= cbk->mask & ~heal->open;
-        }
-    }
+    ret = cluster_fstat(ec->xl_list, locked_on, ec->nodes, fstat_replies,
+                        fstat_output, frame, ec->xl, fd, NULL);
 
-    if (mask != 0)
-    {
-        ec_open(heal->fop->frame, heal->xl, mask, EC_MINIMUM_ONE,
-                ec_heal_target_open_cbk, heal, &heal->loc, O_RDWR | O_TRUNC,
-                heal->fd, NULL);
+    for (i = 0; i < ec->nodes; i++) {
+        output[i] = output[i] && fstat_output[i];
+        replies[i].valid = output[i];
+        if (output[i])
+            replies[i].stat = fstat_replies[i].stat;
+    }
 
-        open |= mask;
+    if (EC_COUNT(output, ec->nodes) <= ec->fragments) {
+        ret = -ENOTCONN;
+        goto out;
     }
 
-    return (open != 0);
-}
+    source = ec_heal_data_find_direction(ec, replies, versions, dirty, size,
+                                         sources, healed_sinks, _gf_true,
+                                         EC_COMBINE_DICT);
+    ret = source;
+    if (ret < 0)
+        goto out;
 
-uintptr_t ec_heal_needs_data_rebuild(ec_heal_t *heal)
-{
-    ec_fop_data_t *fop = heal->lookup;
-    ec_cbk_data_t *cbk = NULL;
-    uintptr_t bad = 0;
+    if (stbuf)
+        *stbuf = replies[source].stat;
 
-    if ((heal->fop->error != 0) || (heal->good == 0) ||
-        (heal->iatt.ia_type != IA_IFREG)) {
-        return 0;
+    for (i = 0; i < ec->nodes; i++) {
+        if (healed_sinks[i]) {
+            if (replies[i].stat.ia_size)
+                trim[i] = 1;
+        }
     }
 
-    list_for_each_entry(cbk, &fop->cbk_list, list) {
-        if ((cbk->op_ret >= 0) &&
-            ((cbk->size != heal->raw_size) ||
-             (cbk->version != heal->version))) {
-            bad |= cbk->mask;
-        }
+    if (EC_COUNT(sources, ec->nodes) < ec->fragments) {
+        ret = -ENOTCONN;
+        goto out;
     }
 
-    return bad;
+    ret = source;
+out:
+    if (xattrs)
+        dict_unref(xattrs);
+    cluster_replies_wipe(replies, ec->nodes);
+    cluster_replies_wipe(fstat_replies, ec->nodes);
+    if (ret < 0) {
+        gf_msg_debug(ec->xl->name, 0, "%s: heal failed %s",
+                     uuid_utoa(fd->inode->gfid), strerror(-ret));
+    } else {
+        gf_msg_debug(ec->xl->name, 0,
+                     "%s: sources: %d, sinks: "
+                     "%d",
+                     uuid_utoa(fd->inode->gfid), EC_COUNT(sources, ec->nodes),
+                     EC_COUNT(healed_sinks, ec->nodes));
+    }
+    return ret;
 }
 
-void ec_heal_setxattr_others(ec_heal_t * heal)
+int
+__ec_heal_mark_sinks(call_frame_t *frame, ec_t *ec, fd_t *fd,
+                     uint64_t *versions, unsigned char *healed_sinks)
 {
-    ec_cbk_data_t * cbk;
-    dict_t * xdata;
-    int32_t error = ENOMEM;
+    int i = 0;
+    int ret = 0;
+    unsigned char *mark = NULL;
+    dict_t *xattrs = NULL;
+    default_args_cbk_t *replies = NULL;
+    unsigned char *output = NULL;
+    uint64_t versions_xattr[2] = {0};
+
+    EC_REPLIES_ALLOC(replies, ec->nodes);
+    xattrs = dict_new();
+    if (!xattrs) {
+        ret = -ENOMEM;
+        goto out;
+    }
 
-    if ((heal->good != 0) && (heal->bad != 0))
-    {
-        cbk = heal->lookup->answer;
-        xdata = cbk->xdata;
+    mark = alloca0(ec->nodes);
+    for (i = 0; i < ec->nodes; i++) {
+        if (!healed_sinks[i])
+            continue;
+        if ((versions[i] >> EC_SELFHEAL_BIT) & 1)
+            continue;
+        mark[i] = 1;
+    }
 
-        if (dict_foreach_match (xdata, ec_ignorable_key_match, NULL,
-                                dict_remove_foreach_fn, NULL) == -1)
-                goto out;
+    if (EC_COUNT(mark, ec->nodes) == 0)
+        return 0;
 
-        if ((cbk->iatt[0].ia_type == IA_IFREG) ||
-            (cbk->iatt[0].ia_type == IA_IFDIR))
-        {
-            if (ec_dict_set_number(xdata, EC_XATTR_VERSION, cbk->version) != 0)
-            {
-                goto out;
-            }
-            if (cbk->iatt[0].ia_type == IA_IFREG)
-            {
-                uint64_t dirty;
-
-                dirty = ec_heal_needs_data_rebuild(heal) != 0;
-                if ((ec_dict_set_number(xdata, EC_XATTR_SIZE,
-                                        cbk->iatt[0].ia_size) != 0) ||
-                    (ec_dict_set_number(xdata, EC_XATTR_DIRTY, dirty) != 0)) {
-                    goto out;
-                }
-            }
-        }
+    versions_xattr[EC_DATA_TXN] = hton64(1ULL << EC_SELFHEAL_BIT);
+    if (dict_set_static_bin(xattrs, EC_XATTR_VERSION, versions_xattr,
+                            sizeof(versions_xattr))) {
+        ret = -ENOMEM;
+        goto out;
+    }
 
-        ec_setxattr(heal->fop->frame, heal->xl, heal->bad, EC_MINIMUM_ONE,
-                    ec_heal_setxattr_cbk, heal, &heal->loc, xdata, 0, NULL);
+    output = alloca0(ec->nodes);
+    ret = cluster_fxattrop(ec->xl_list, mark, ec->nodes, replies, output, frame,
+                           ec->xl, fd, GF_XATTROP_ADD_ARRAY64, xattrs, NULL);
+    for (i = 0; i < ec->nodes; i++) {
+        if (!output[i]) {
+            if (mark[i])
+                healed_sinks[i] = 0;
+            continue;
+        }
+        versions[i] |= (1ULL << EC_SELFHEAL_BIT);
     }
 
-    error = 0;
+    if (EC_COUNT(healed_sinks, ec->nodes) == 0) {
+        ret = -ENOTCONN;
+        goto out;
+    }
+    ret = 0;
 
 out:
-    ec_fop_set_error(heal->fop, error);
+    cluster_replies_wipe(replies, ec->nodes);
+    if (xattrs)
+        dict_unref(xattrs);
+    if (ret < 0)
+        gf_msg_debug(ec->xl->name, 0, "%s: heal failed %s",
+                     uuid_utoa(fd->inode->gfid), strerror(-ret));
+    return ret;
 }
 
 int32_t
-ec_heal_xattr_clean (dict_t *dict, char *key, data_t *data,
-                     void *arg)
+ec_manager_heal_block(ec_fop_data_t *fop, int32_t state)
 {
-        dict_t *base = arg;
+    ec_heal_t *heal = fop->data;
+    heal->fop = fop;
 
-        if (ec_ignorable_key_match (NULL, key, NULL, NULL)) {
-                dict_del (dict, key);
-                return 0;
-        }
+    switch (state) {
+        case EC_STATE_INIT:
+            ec_owner_set(fop->frame, fop->frame->root);
 
-        if (dict_get (base, key) != NULL)
-                dict_del (dict, key);
+            ec_heal_inodelk(heal, F_WRLCK, 1, 0, 0);
 
-        return 0;
-}
+            return EC_STATE_HEAL_DATA_COPY;
 
-void ec_heal_removexattr_others(ec_heal_t * heal)
-{
-    struct list_head * item;
-    ec_cbk_data_t * cbk;
-    dict_t * xdata;
+        case EC_STATE_HEAL_DATA_COPY:
+            gf_msg_debug(fop->xl->name, 0, "%s: read/write starting",
+                         uuid_utoa(heal->fd->inode->gfid));
+            ec_heal_data_block(heal);
 
-    if ((heal->good == 0) || (heal->bad == 0))
-    {
-        return;
-    }
+            return EC_STATE_HEAL_DATA_UNLOCK;
 
-    xdata = heal->lookup->answer->xdata;
-    item = heal->lookup->cbk_list.next;
-    while (item->next != &heal->lookup->cbk_list)
-    {
-        item = item->next;
-        cbk = list_entry(item, ec_cbk_data_t, list);
+        case -EC_STATE_HEAL_DATA_COPY:
+        case -EC_STATE_HEAL_DATA_UNLOCK:
+        case EC_STATE_HEAL_DATA_UNLOCK:
+            ec_heal_inodelk(heal, F_UNLCK, 1, 0, 0);
 
-        if (cbk->op_ret >= 0)
-        {
-            if (dict_foreach(cbk->xdata, ec_heal_xattr_clean, xdata) == 0)
-            {
-                ec_removexattr(heal->fop->frame, heal->xl, cbk->mask,
-                               EC_MINIMUM_ONE, ec_heal_removexattr_cbk, heal,
-                               &heal->loc, "", cbk->xdata);
+            return EC_STATE_REPORT;
+
+        case EC_STATE_REPORT:
+            if (fop->cbks.heal) {
+                fop->cbks.heal(fop->req_frame, fop->data, fop->xl, 0, 0,
+                               (heal->good | heal->bad), heal->good, heal->bad,
+                               0, NULL);
             }
-        }
-    }
-}
 
-void ec_heal_attr(ec_heal_t * heal)
-{
-    if ((heal->good != 0) && (heal->bad != 0))
-    {
-        ec_setattr(heal->fop->frame, heal->xl, heal->bad, EC_MINIMUM_ONE,
-                   ec_heal_setattr_cbk, heal, &heal->loc, &heal->iatt,
-                   GF_SET_ATTR_MODE | GF_SET_ATTR_UID | GF_SET_ATTR_GID |
-                   GF_SET_ATTR_ATIME | GF_SET_ATTR_MTIME, NULL);
+            return EC_STATE_END;
+        case -EC_STATE_REPORT:
+            if (fop->cbks.heal) {
+                fop->cbks.heal(fop->req_frame, fop->data, fop->xl, -1,
+                               fop->error, 0, 0, 0, 0, NULL);
+            }
+
+            return EC_STATE_END;
+        default:
+            gf_msg(fop->xl->name, GF_LOG_ERROR, 0, EC_MSG_UNHANDLED_STATE,
+                   "Unhandled state %d for %s", state, ec_fop_name(fop->id));
+
+            return EC_STATE_END;
     }
 }
 
+/*Takes lock */
 void
-ec_heal_open (ec_heal_t * heal)
+ec_heal_block(call_frame_t *frame, xlator_t *this, uintptr_t target,
+              uint32_t fop_flags, fop_heal_cbk_t func, ec_heal_t *heal)
 {
-    heal->bad = ec_heal_needs_data_rebuild(heal);
-    if (heal->bad == 0) {
-        return;
-    }
+    ec_cbk_t callback = {.heal = func};
+    ec_fop_data_t *fop = NULL;
+    int32_t error = ENOMEM;
 
-    if (!heal->fd) {
-            /* name-less loc heal */
-            heal->fd = fd_create (heal->loc.inode,
-                                  heal->fop->frame->root->pid);
-    }
+    gf_msg_trace("ec", 0, "EC(HEAL) %p", frame);
+
+    VALIDATE_OR_GOTO(this, out);
+    GF_VALIDATE_OR_GOTO(this->name, this->private, out);
+
+    fop = ec_fop_data_allocate(frame, this, EC_FOP_HEAL, 0, target, fop_flags,
+                               NULL, ec_manager_heal_block, callback, heal);
+    if (fop == NULL)
+        goto out;
 
-    if (!heal->fd) {
-            ec_fop_set_error(heal->fop, ENOMEM);
-            return;
+    error = 0;
+
+out:
+    if (fop != NULL) {
+        ec_manager(fop, error);
+    } else {
+        func(frame, heal, this, -1, error, 0, 0, 0, 0, NULL);
     }
+}
 
-    if (ec_heal_open_others(heal))
-    {
-        ec_open(heal->fop->frame, heal->xl, heal->good, EC_MINIMUM_MIN,
-                ec_heal_source_open_cbk, heal, &heal->loc, O_RDONLY, heal->fd,
-                NULL);
+int32_t
+ec_heal_block_done(call_frame_t *frame, void *cookie, xlator_t *this,
+                   int32_t op_ret, int32_t op_errno, uintptr_t mask,
+                   uintptr_t good, uintptr_t bad, uint32_t pending,
+                   dict_t *xdata)
+{
+    ec_heal_t *heal = cookie;
+
+    if (heal->fop) {
+        heal->fop->heal = NULL;
     }
+    heal->fop = NULL;
+    heal->error = op_ret < 0 ? op_errno : 0;
+    syncbarrier_wake(heal->data);
+    return 0;
 }
 
-void ec_heal_reopen_fd(ec_heal_t * heal)
+int
+ec_sync_heal_block(call_frame_t *frame, xlator_t *this, ec_heal_t *heal)
 {
-    inode_t * inode;
-    fd_t * fd;
-    ec_fd_t *ctx_fd;
-    ec_inode_t *ctx_inode;
-    uintptr_t mask;
-    int32_t flags;
+    ec_heal_block(frame, this, heal->bad | heal->good, EC_MINIMUM_ONE,
+                  ec_heal_block_done, heal);
+    syncbarrier_wait(heal->data, 1);
+    if (heal->error != 0) {
+        return -heal->error;
+    }
+    if (heal->bad == 0)
+        return -ENOTCONN;
+    return 0;
+}
 
-    inode = heal->loc.inode;
+int
+ec_rebuild_data(call_frame_t *frame, ec_t *ec, fd_t *fd, uint64_t size,
+                unsigned char *sources, unsigned char *healed_sinks)
+{
+    ec_heal_t *heal = NULL;
+    int ret = 0;
+    syncbarrier_t barrier;
+
+    if (syncbarrier_init(&barrier))
+        return -ENOMEM;
+
+    heal = alloca0(sizeof(*heal));
+    heal->fd = fd_ref(fd);
+    heal->xl = ec->xl;
+    heal->data = &barrier;
+    ec_adjust_size_up(ec, &size, _gf_false);
+    heal->total_size = size;
+    heal->size = (128 * GF_UNIT_KB * (ec->self_heal_window_size));
+    /* We need to adjust the size to a multiple of the stripe size of the
+     * volume. Otherwise writes would need to fill gaps (head and/or tail)
+     * with existent data from the bad bricks. This could be garbage on a
+     * damaged file or it could fail if there aren't enough bricks. */
+    heal->size -= heal->size % ec->stripe_size;
+    heal->bad = ec_char_array_to_mask(healed_sinks, ec->nodes);
+    heal->good = ec_char_array_to_mask(sources, ec->nodes);
+    heal->iatt.ia_type = IA_IFREG;
+    LOCK_INIT(&heal->lock);
 
-    LOCK(&inode->lock);
+    for (heal->offset = 0; (heal->offset < size) && !heal->done;
+         heal->offset += heal->size) {
+        /* We immediately abort any heal if a shutdown request has been
+         * received to avoid delays. The healing of this file will be
+         * restarted by another SHD or other client that accesses the
+         * file. */
+        if (ec->shutdown) {
+            gf_msg_debug(ec->xl->name, 0,
+                         "Cancelling heal because "
+                         "EC is stopping.");
+            ret = -ENOTCONN;
+            break;
+        }
 
-    ctx_inode = __ec_inode_get(inode, heal->xl);
-    if (ctx_inode != NULL) {
-        ctx_inode->bad &= ~(heal->good | heal->bad);
+        gf_msg_debug(ec->xl->name, 0,
+                     "%s: sources: %d, sinks: "
+                     "%d, offset: %" PRIu64 " bsize: %" PRIu64,
+                     uuid_utoa(fd->inode->gfid), EC_COUNT(sources, ec->nodes),
+                     EC_COUNT(healed_sinks, ec->nodes), heal->offset,
+                     heal->size);
+        ret = ec_sync_heal_block(frame, ec->xl, heal);
+        if (ret < 0)
+            break;
     }
+    memset(healed_sinks, 0, ec->nodes);
+    ec_mask_to_char_array(heal->bad, healed_sinks, ec->nodes);
+    fd_unref(heal->fd);
+    LOCK_DESTROY(&heal->lock);
+    syncbarrier_destroy(heal->data);
+    if (ret < 0)
+        gf_msg_debug(ec->xl->name, 0, "%s: heal failed %s",
+                     uuid_utoa(fd->inode->gfid), strerror(-ret));
+    return ret;
+}
 
-    list_for_each_entry(fd, &inode->fd_list, inode_list)
-    {
-        ctx_fd = ec_fd_get(fd, heal->xl);
-        if (ctx_fd != NULL) {
-            mask = heal->bad & ~ctx_fd->open;
-            if (mask != 0)
-            {
-                UNLOCK(&inode->lock);
-
-                if (heal->iatt.ia_type == IA_IFDIR)
-                {
-                    ec_opendir(heal->fop->frame, heal->xl, mask,
-                               EC_MINIMUM_ONE, ec_heal_reopen_cbk, NULL,
-                               &heal->loc, fd, NULL);
-                }
-                else
-                {
-                    flags = ctx_fd->flags & ~(O_TRUNC | O_APPEND);
+int
+__ec_heal_trim_sinks(call_frame_t *frame, ec_t *ec, fd_t *fd,
+                     unsigned char *healed_sinks, unsigned char *trim,
+                     uint64_t size)
+{
+    default_args_cbk_t *replies = NULL;
+    unsigned char *output = NULL;
+    int ret = 0;
+    int i = 0;
+    off_t trim_offset = 0;
 
-                    ec_open(heal->fop->frame, heal->xl, mask, EC_MINIMUM_ONE,
-                            ec_heal_reopen_cbk, NULL, &heal->loc, flags, fd,
-                            NULL);
-                }
+    EC_REPLIES_ALLOC(replies, ec->nodes);
+    output = alloca0(ec->nodes);
 
-                LOCK(&inode->lock);
-            }
-        }
+    if (EC_COUNT(trim, ec->nodes) == 0) {
+        ret = 0;
+        goto out;
+    }
+    trim_offset = size;
+    ec_adjust_offset_up(ec, &trim_offset, _gf_true);
+    ret = cluster_ftruncate(ec->xl_list, trim, ec->nodes, replies, output,
+                            frame, ec->xl, fd, trim_offset, NULL);
+    for (i = 0; i < ec->nodes; i++) {
+        if (!output[i] && trim[i])
+            healed_sinks[i] = 0;
     }
 
-    UNLOCK(&inode->lock);
+    if (EC_COUNT(healed_sinks, ec->nodes) == 0) {
+        ret = -ENOTCONN;
+        goto out;
+    }
+
+out:
+    cluster_replies_wipe(replies, ec->nodes);
+    if (ret < 0)
+        gf_msg_debug(ec->xl->name, 0, "%s: heal failed %s",
+                     uuid_utoa(fd->inode->gfid), strerror(-ret));
+    return ret;
 }
 
-int32_t ec_heal_writev_cbk(call_frame_t * frame, void * cookie,
-                           xlator_t * this, int32_t op_ret, int32_t op_errno,
-                           struct iatt * prebuf, struct iatt * postbuf,
-                           dict_t * xdata)
+int
+ec_data_undo_pending(call_frame_t *frame, ec_t *ec, fd_t *fd, dict_t *xattr,
+                     uint64_t *versions, uint64_t *dirty, uint64_t *size,
+                     int source, gf_boolean_t erase_dirty, int idx)
 {
-    ec_trace("WRITE_CBK", cookie, "ret=%d, errno=%d", op_ret, op_errno);
+    uint64_t versions_xattr[2] = {0};
+    uint64_t dirty_xattr[2] = {0};
+    uint64_t allzero[2] = {0};
+    uint64_t size_xattr = 0;
+    int ret = 0;
+
+    versions_xattr[EC_DATA_TXN] = hton64(versions[source] - versions[idx]);
+    ret = dict_set_static_bin(xattr, EC_XATTR_VERSION, versions_xattr,
+                              sizeof(versions_xattr));
+    if (ret < 0)
+        goto out;
 
-    ec_heal_update(cookie, 0);
+    size_xattr = hton64(size[source] - size[idx]);
+    ret = dict_set_static_bin(xattr, EC_XATTR_SIZE, &size_xattr,
+                              sizeof(size_xattr));
+    if (ret < 0)
+        goto out;
 
-    return 0;
+    if (erase_dirty) {
+        dirty_xattr[EC_DATA_TXN] = hton64(-dirty[idx]);
+        ret = dict_set_static_bin(xattr, EC_XATTR_DIRTY, dirty_xattr,
+                                  sizeof(dirty_xattr));
+        if (ret < 0)
+            goto out;
+    }
+
+    if ((memcmp(versions_xattr, allzero, sizeof(allzero)) == 0) &&
+        (memcmp(dirty_xattr, allzero, sizeof(allzero)) == 0) &&
+        (size_xattr == 0)) {
+        ret = 0;
+        goto out;
+    }
+
+    ret = syncop_fxattrop(ec->xl_list[idx], fd, GF_XATTROP_ADD_ARRAY64, xattr,
+                          NULL, NULL, NULL);
+out:
+    return ret;
 }
 
-int32_t ec_heal_readv_cbk(call_frame_t * frame, void * cookie, xlator_t * this,
-                          int32_t op_ret, int32_t op_errno,
-                          struct iovec * vector, int32_t count,
-                          struct iatt * stbuf, struct iobref * iobref,
-                          dict_t * xdata)
+int
+__ec_fd_data_adjust_versions(call_frame_t *frame, ec_t *ec, fd_t *fd,
+                             unsigned char *sources,
+                             unsigned char *healed_sinks, uint64_t *versions,
+                             uint64_t *dirty, uint64_t *size)
 {
-    ec_fop_data_t * fop = cookie;
-    ec_heal_t * heal = fop->data;
+    dict_t *xattr = NULL;
+    int i = 0;
+    int ret = 0;
+    int op_ret = 0;
+    int source = -1;
+    gf_boolean_t erase_dirty = _gf_false;
+
+    xattr = dict_new();
+    if (!xattr) {
+        op_ret = -ENOMEM;
+        goto out;
+    }
 
-    ec_trace("READ_CBK", fop, "ret=%d, errno=%d", op_ret, op_errno);
+    /* dirty xattr represents if the file needs heal. Unless all the
+     * copies are healed, don't erase it */
+    if (EC_COUNT(sources, ec->nodes) + EC_COUNT(healed_sinks, ec->nodes) ==
+        ec->nodes)
+        erase_dirty = _gf_true;
 
-    ec_heal_avoid(fop);
+    for (i = 0; i < ec->nodes; i++) {
+        if (sources[i]) {
+            source = i;
+            break;
+        }
+    }
 
-    if (op_ret > 0)
-    {
-        ec_writev(heal->fop->frame, heal->xl, heal->bad, EC_MINIMUM_ONE,
-                  ec_heal_writev_cbk, heal, heal->fd, vector, count,
-                  heal->offset, 0, iobref, NULL);
+    if (source == -1) {
+        op_ret = -ENOTCONN;
+        goto out;
     }
-    else
-    {
-        heal->done = 1;
+
+    for (i = 0; i < ec->nodes; i++) {
+        if (healed_sinks[i]) {
+            ret = ec_data_undo_pending(frame, ec, fd, xattr, versions, dirty,
+                                       size, source, erase_dirty, i);
+            if (ret < 0)
+                goto out;
+        }
     }
 
-    return 0;
+    if (!erase_dirty)
+        goto out;
+
+    for (i = 0; i < ec->nodes; i++) {
+        if (sources[i]) {
+            ret = ec_data_undo_pending(frame, ec, fd, xattr, versions, dirty,
+                                       size, source, erase_dirty, i);
+            if (ret < 0)
+                continue;
+        }
+    }
+out:
+    if (xattr)
+        dict_unref(xattr);
+    return op_ret;
 }
 
-void ec_heal_data(ec_heal_t * heal)
+int
+ec_restore_time_and_adjust_versions(call_frame_t *frame, ec_t *ec, fd_t *fd,
+                                    unsigned char *sources,
+                                    unsigned char *healed_sinks,
+                                    uint64_t *versions, uint64_t *dirty,
+                                    uint64_t *size)
 {
-    ec_trace("DATA", heal->fop, "good=%lX, bad=%lX", heal->good, heal->bad);
-
-    if ((heal->good != 0) && (heal->bad != 0) &&
-        (heal->iatt.ia_type == IA_IFREG))
+    unsigned char *locked_on = NULL;
+    unsigned char *participants = NULL;
+    unsigned char *output = NULL;
+    default_args_cbk_t *replies = NULL;
+    unsigned char *postsh_sources = NULL;
+    unsigned char *postsh_healed_sinks = NULL;
+    unsigned char *postsh_trim = NULL;
+    uint64_t *postsh_versions = NULL;
+    uint64_t *postsh_dirty = NULL;
+    uint64_t *postsh_size = NULL;
+    int ret = 0;
+    int i = 0;
+    struct iatt source_buf = {0};
+    loc_t loc = {0};
+
+    locked_on = alloca0(ec->nodes);
+    output = alloca0(ec->nodes);
+    participants = alloca0(ec->nodes);
+    postsh_sources = alloca0(ec->nodes);
+    postsh_healed_sinks = alloca0(ec->nodes);
+    postsh_trim = alloca0(ec->nodes);
+    postsh_versions = alloca0(ec->nodes * sizeof(*postsh_versions));
+    postsh_dirty = alloca0(ec->nodes * sizeof(*postsh_dirty));
+    postsh_size = alloca0(ec->nodes * sizeof(*postsh_size));
+
+    for (i = 0; i < ec->nodes; i++) {
+        if (healed_sinks[i] || sources[i])
+            participants[i] = 1;
+    }
+
+    EC_REPLIES_ALLOC(replies, ec->nodes);
+    ret = cluster_inodelk(ec->xl_list, participants, ec->nodes, replies,
+                          locked_on, frame, ec->xl, ec->xl->name, fd->inode, 0,
+                          0);
     {
-        ec_readv(heal->fop->frame, heal->xl, heal->good, EC_MINIMUM_MIN,
-                 ec_heal_readv_cbk, heal, heal->fd, heal->size, heal->offset,
-                 0, NULL);
+        if (ret <= ec->fragments) {
+            gf_msg_debug(ec->xl->name, 0,
+                         "%s: Skipping heal "
+                         "as only %d number of subvolumes could "
+                         "be locked",
+                         uuid_utoa(fd->inode->gfid), ret);
+            ret = -ENOTCONN;
+            goto unlock;
+        }
+
+        ret = __ec_heal_data_prepare(frame, ec, fd, locked_on, postsh_versions,
+                                     postsh_dirty, postsh_size, postsh_sources,
+                                     postsh_healed_sinks, postsh_trim,
+                                     &source_buf);
+        if (ret < 0)
+            goto unlock;
+
+        loc.inode = inode_ref(fd->inode);
+        gf_uuid_copy(loc.gfid, fd->inode->gfid);
+        ret = cluster_setattr(
+            ec->xl_list, healed_sinks, ec->nodes, replies, output, frame,
+            ec->xl, &loc, &source_buf,
+            GF_SET_ATTR_ATIME | GF_SET_ATTR_MTIME | GF_SET_ATTR_CTIME, NULL);
+        EC_INTERSECT(healed_sinks, healed_sinks, output, ec->nodes);
+        if (EC_COUNT(healed_sinks, ec->nodes) == 0) {
+            ret = -ENOTCONN;
+            goto unlock;
+        }
+        ret = __ec_fd_data_adjust_versions(frame, ec, fd, sources, healed_sinks,
+                                           versions, dirty, size);
     }
+unlock:
+    cluster_uninodelk(ec->xl_list, locked_on, ec->nodes, replies, output, frame,
+                      ec->xl, ec->xl->name, fd->inode, 0, 0);
+    cluster_replies_wipe(replies, ec->nodes);
+    loc_wipe(&loc);
+    return ret;
 }
 
-void ec_heal_update_dirty(ec_heal_t *heal, uintptr_t mask)
+int
+__ec_heal_data(call_frame_t *frame, ec_t *ec, fd_t *fd, unsigned char *heal_on,
+               unsigned char *sources, unsigned char *healed_sinks)
 {
-    dict_t *dict;
+    unsigned char *locked_on = NULL;
+    unsigned char *output = NULL;
+    uint64_t *versions = NULL;
+    uint64_t *dirty = NULL;
+    uint64_t *size = NULL;
+    unsigned char *trim = NULL;
+    default_args_cbk_t *replies = NULL;
+    int ret = 0;
+    int source = 0;
+
+    locked_on = alloca0(ec->nodes);
+    output = alloca0(ec->nodes);
+    trim = alloca0(ec->nodes);
+    versions = alloca0(ec->nodes * sizeof(*versions));
+    dirty = alloca0(ec->nodes * sizeof(*dirty));
+    size = alloca0(ec->nodes * sizeof(*size));
+
+    EC_REPLIES_ALLOC(replies, ec->nodes);
+    ret = cluster_inodelk(ec->xl_list, heal_on, ec->nodes, replies, locked_on,
+                          frame, ec->xl, ec->xl->name, fd->inode, 0, 0);
+    {
+        if (ret <= ec->fragments) {
+            gf_msg_debug(ec->xl->name, 0,
+                         "%s: Skipping heal "
+                         "as only %d number of subvolumes could "
+                         "be locked",
+                         uuid_utoa(fd->inode->gfid), ret);
+            ret = -ENOTCONN;
+            goto unlock;
+        }
 
-    dict = dict_new();
-    if (dict == NULL) {
-        ec_fop_set_error(heal->fop, EIO);
+        ret = __ec_heal_data_prepare(frame, ec, fd, locked_on, versions, dirty,
+                                     size, sources, healed_sinks, trim, NULL);
+        if (ret < 0)
+            goto unlock;
 
-        return;
-    }
+        if (EC_COUNT(healed_sinks, ec->nodes) == 0) {
+            ret = __ec_fd_data_adjust_versions(
+                frame, ec, fd, sources, healed_sinks, versions, dirty, size);
+            goto unlock;
+        }
 
-    if (ec_dict_set_number(dict, EC_XATTR_DIRTY, -1) != 0) {
-        dict_unref(dict);
-        ec_fop_set_error(heal->fop, EIO);
+        source = ret;
+        ret = __ec_heal_mark_sinks(frame, ec, fd, versions, healed_sinks);
+        if (ret < 0)
+            goto unlock;
 
-        return;
+        ret = __ec_heal_trim_sinks(frame, ec, fd, healed_sinks, trim,
+                                   size[source]);
     }
+unlock:
+    cluster_uninodelk(ec->xl_list, locked_on, ec->nodes, replies, output, frame,
+                      ec->xl, ec->xl->name, fd->inode, 0, 0);
+    if (ret < 0)
+        goto out;
 
-    ec_fxattrop(heal->fop->frame, heal->xl, mask, EC_MINIMUM_ONE, NULL, NULL,
-                heal->fd, GF_XATTROP_ADD_ARRAY64, dict, NULL);
+    if (EC_COUNT(healed_sinks, ec->nodes) == 0)
+        goto out;
+
+    gf_msg_debug(ec->xl->name, 0,
+                 "%s: sources: %d, sinks: "
+                 "%d",
+                 uuid_utoa(fd->inode->gfid), EC_COUNT(sources, ec->nodes),
+                 EC_COUNT(healed_sinks, ec->nodes));
+
+    ret = ec_rebuild_data(frame, ec, fd, size[source], sources, healed_sinks);
+    if (ret < 0)
+        goto out;
 
-    dict_unref(dict);
+    ret = ec_restore_time_and_adjust_versions(
+        frame, ec, fd, sources, healed_sinks, versions, dirty, size);
+out:
+    cluster_replies_wipe(replies, ec->nodes);
+    return ret;
 }
 
-void ec_heal_dispatch(ec_heal_t *heal)
+int
+ec_heal_data(call_frame_t *frame, ec_t *ec, gf_boolean_t block, inode_t *inode,
+             unsigned char *sources, unsigned char *healed_sinks)
 {
-    ec_fop_data_t *fop;
-    ec_cbk_data_t *cbk;
-    inode_t *inode;
-    ec_inode_t *ctx;
-    ec_heal_t *next = NULL;
-    struct list_head list;
-    int32_t error;
-
-    inode = heal->loc.inode;
-
-    INIT_LIST_HEAD(&list);
-
-    LOCK(&inode->lock);
-
-    /* A heal object not belonging to any list means that it has not been fully
-     * executed. It got its information from a previous heal that was executing
-     * when this heal started. */
-    if (!list_empty(&heal->list)) {
-        list_del_init(&heal->list);
-        ctx = __ec_inode_get(inode, heal->xl);
-        if (ctx != NULL) {
-            ctx->bad &= ~heal->good;
-
-            if (heal->partial) {
-                /* Collect all partial heal requests. All of them will receive
-                 * the same answer. 'next' will contain a pointer to the first
-                 * full request (if any) after this partial heal request.*/
-                while (!list_empty(&ctx->heal)) {
-                    next = list_entry(ctx->heal.next, ec_heal_t, list);
-                    if (!next->partial) {
-                        break;
-                    }
-                    list_move_tail(&next->list, &list);
-                }
-                if (list_empty(&ctx->heal)) {
-                    next = NULL;
-                }
-            } else {
-                /* This is a full heal request, so take all received heal
-                 * requests to answer them now. */
-                list_splice_init(&ctx->heal, &list);
-            }
-        }
+    unsigned char *locked_on = NULL;
+    unsigned char *up_subvols = NULL;
+    unsigned char *output = NULL;
+    default_args_cbk_t *replies = NULL;
+    fd_t *fd = NULL;
+    loc_t loc = {0};
+    char selfheal_domain[1024] = {0};
+    int ret = 0;
+
+    EC_REPLIES_ALLOC(replies, ec->nodes);
+
+    locked_on = alloca0(ec->nodes);
+    output = alloca0(ec->nodes);
+    up_subvols = alloca0(ec->nodes);
+    loc.inode = inode_ref(inode);
+    gf_uuid_copy(loc.gfid, inode->gfid);
+
+    fd = fd_create(inode, 0);
+    if (!fd) {
+        ret = -ENOMEM;
+        goto out;
     }
 
-    UNLOCK(&inode->lock);
-
-    fop = heal->fop;
-    error = fop->error;
-
-    cbk = ec_cbk_data_allocate(fop->frame, heal->xl, fop, fop->id, 0,
-                               error == 0 ? 0 : -1, error);
-    if (cbk != NULL) {
-        cbk->uintptr[0] = heal->available;
-        cbk->uintptr[1] = heal->good;
-        cbk->uintptr[2] = heal->fixed;
+    ec_mask_to_char_array(ec->xl_up, up_subvols, ec->nodes);
 
-        ec_combine(cbk, NULL);
-
-        fop->answer = cbk;
-    } else if (error == 0) {
-        error = ENOMEM;
+    ret = cluster_open(ec->xl_list, up_subvols, ec->nodes, replies, output,
+                       frame, ec->xl, &loc, O_RDWR | O_LARGEFILE, fd, NULL);
+    if (ret <= ec->fragments) {
+        ret = -ENOTCONN;
+        goto out;
     }
 
-    if (heal->lookup != NULL)
-    {
-        ec_fop_data_release(heal->lookup);
+    fd_bind(fd);
+    sprintf(selfheal_domain, "%s:self-heal", ec->xl->name);
+    /*If other processes are already doing the heal, don't block*/
+    if (block) {
+        ret = cluster_inodelk(ec->xl_list, output, ec->nodes, replies,
+                              locked_on, frame, ec->xl, selfheal_domain, inode,
+                              0, 0);
+    } else {
+        ret = cluster_tiebreaker_inodelk(ec->xl_list, output, ec->nodes,
+                                         replies, locked_on, frame, ec->xl,
+                                         selfheal_domain, inode, 0, 0);
     }
-    if (heal->fd != NULL)
     {
-        fd_unref(heal->fd);
+        if (ret <= ec->fragments) {
+            gf_msg_debug(ec->xl->name, 0,
+                         "%s: Skipping heal "
+                         "as only %d number of subvolumes could "
+                         "be locked",
+                         uuid_utoa(inode->gfid), ret);
+            ret = -ENOTCONN;
+            goto unlock;
+        }
+        ret = __ec_heal_data(frame, ec, fd, locked_on, sources, healed_sinks);
     }
-    GF_FREE(heal->symlink);
-    loc_wipe(&heal->loc);
-
-    LOCK_DESTROY(&heal->lock);
+unlock:
+    cluster_uninodelk(ec->xl_list, locked_on, ec->nodes, replies, output, frame,
+                      ec->xl, selfheal_domain, inode, 0, 0);
+out:
+    if (fd)
+        fd_unref(fd);
+    loc_wipe(&loc);
+    cluster_replies_wipe(replies, ec->nodes);
+    return ret;
+}
 
-    GF_FREE(heal);
+int
+ec_heal_purge_stale_index(call_frame_t *frame, ec_t *ec, inode_t *inode)
+{
+    int i = 0;
+    int ret = 0;
+    dict_t **xattr = NULL;
+    loc_t loc = {0};
+    uint64_t dirty_xattr[EC_VERSION_SIZE] = {0};
+    unsigned char *on = NULL;
+    default_args_cbk_t *replies = NULL;
+    dict_t *dict = NULL;
+
+    /* Allocate the required memory */
+    loc.inode = inode_ref(inode);
+    gf_uuid_copy(loc.gfid, inode->gfid);
+    on = alloca0(ec->nodes);
+    EC_REPLIES_ALLOC(replies, ec->nodes);
+    xattr = GF_CALLOC(ec->nodes, sizeof(*xattr), gf_common_mt_pointer);
+    if (!xattr) {
+        ret = -ENOMEM;
+        goto out;
+    }
+    dict = dict_new();
+    if (!dict) {
+        ret = -ENOMEM;
+        goto out;
+    }
+    for (i = 0; i < ec->nodes; i++) {
+        xattr[i] = dict;
+        on[i] = 1;
+    }
+    ret = dict_set_static_bin(dict, EC_XATTR_DIRTY, dirty_xattr,
+                              (sizeof(*dirty_xattr) * EC_VERSION_SIZE));
+    if (ret < 0) {
+        ret = -ENOMEM;
+        goto out;
+    }
+    PARALLEL_FOP_ONLIST(ec->xl_list, on, ec->nodes, replies, frame,
+                        ec_wind_xattrop_parallel, &loc, GF_XATTROP_ADD_ARRAY64,
+                        xattr, NULL);
+out:
+    if (dict) {
+        dict_unref(dict);
+    }
+    if (xattr) {
+        GF_FREE(xattr);
+    }
+    cluster_replies_wipe(replies, ec->nodes);
+    loc_wipe(&loc);
+    return ret;
+}
 
-    ec_fop_set_error(fop, error);
+void
+ec_heal_do(xlator_t *this, void *data, loc_t *loc, int32_t partial)
+{
+    call_frame_t *frame = NULL;
+    unsigned char *participants = NULL;
+    unsigned char *msources = NULL;
+    unsigned char *mhealed_sinks = NULL;
+    unsigned char *sources = NULL;
+    unsigned char *healed_sinks = NULL;
+    ec_t *ec = NULL;
+    int ret = 0;
+    int op_ret = 0;
+    int op_errno = 0;
+    intptr_t mgood = 0;
+    intptr_t mbad = 0;
+    intptr_t good = 0;
+    intptr_t bad = 0;
+    uint32_t pending = 0;
+    ec_fop_data_t *fop = data;
+    gf_boolean_t blocking = _gf_false;
+    ec_heal_need_t need_heal = EC_HEAL_NONEED;
+    unsigned char *up_subvols = NULL;
+    char up_bricks[32];
+
+    ec = this->private;
+
+    /* If it is heal request from getxattr, complete the heal and then
+     * unwind, if it is ec_heal with NULL as frame then no need to block
+     * the heal as the caller doesn't care about its completion. In case
+     * of heald whichever gets tiebreaking inodelk will take care of the
+     * heal, so no need to block*/
+    if (fop->req_frame && !ec->shd.iamshd)
+        blocking = _gf_true;
+
+    frame = create_frame(this, this->ctx->pool);
+    if (!frame)
+        goto out;
 
-    /* Resume all pending heal requests, setting the same data obtained by
-     * this heal execution. */
-    while (!list_empty(&list)) {
-        heal = list_entry(list.next, ec_heal_t, list);
-        list_del_init(&heal->list);
+    ec_owner_set(frame, frame->root);
+    /*Do heal as root*/
+    frame->root->uid = 0;
+    frame->root->gid = 0;
+    /*Mark the fops as internal*/
+    frame->root->pid = GF_CLIENT_PID_SELF_HEALD;
+    participants = alloca0(ec->nodes);
+    ec_mask_to_char_array(ec->xl_up, participants, ec->nodes);
+
+    up_subvols = alloca0(ec->nodes);
+    ec_mask_to_char_array(ec->xl_up, up_subvols, ec->nodes);
+
+    if (loc->name && strlen(loc->name)) {
+        ret = ec_heal_name(frame, ec, loc->parent, (char *)loc->name,
+                           participants);
+        if (ret >= 0) {
+            gf_msg_debug(this->name, 0,
+                         "%s: name heal "
+                         "successful on %" PRIXPTR,
+                         loc->path,
+                         ec_char_array_to_mask(participants, ec->nodes));
+        } else {
+            gf_msg_debug(
+                this->name, 0,
+                "%s: name heal "
+                "failed. ret = %d, subvolumes up = %s",
+                loc->path, ret,
+                ec_bin(up_bricks, sizeof(up_bricks), ec->xl_up, ec->nodes));
+        }
+    }
 
-        heal->available = cbk->uintptr[0];
-        heal->good = cbk->uintptr[1];
-        heal->fixed = cbk->uintptr[2];
+    /* Mount triggers heal only when it detects that it must need heal, shd
+     * triggers heals periodically which need not be thorough*/
+    if (ec->shd.iamshd && (ret <= 0)) {
+        ec_heal_inspect(frame, ec, loc->inode, up_subvols, _gf_false, _gf_false,
+                        &need_heal);
+
+        if (need_heal == EC_HEAL_PURGE_INDEX) {
+            gf_msg(ec->xl->name, GF_LOG_INFO, 0, EC_MSG_HEAL_FAIL,
+                   "Index entry needs to be purged for: %s ",
+                   uuid_utoa(loc->gfid));
+            /* We need to send zero-xattrop so that stale index entry could be
+             * removed. We need not take lock on this entry to do so as
+             * xattrop on a brick is atomic. */
+            ec_heal_purge_stale_index(frame, ec, loc->inode);
+            goto out;
+        } else if (need_heal == EC_HEAL_NONEED) {
+            gf_msg(ec->xl->name, GF_LOG_DEBUG, 0, EC_MSG_HEAL_FAIL,
+                   "Heal is not required for : %s ", uuid_utoa(loc->gfid));
+            goto out;
+        }
+    }
 
-        /* Setting 'done' to 1 avoids executing all heal logic and directly
-         * reports the result to the caller. */
-        heal->done = 1;
+    sources = alloca0(ec->nodes);
+    healed_sinks = alloca0(ec->nodes);
+    if (IA_ISREG(loc->inode->ia_type)) {
+        ret = ec_heal_data(frame, ec, blocking, loc->inode, sources,
+                           healed_sinks);
+    } else if (IA_ISDIR(loc->inode->ia_type) && !partial) {
+        ret = ec_heal_entry(frame, ec, loc->inode, sources, healed_sinks,
+                            &pending);
+    } else {
+        ret = 0;
+        memcpy(sources, participants, ec->nodes);
+        memcpy(healed_sinks, participants, ec->nodes);
+    }
 
-        ec_resume(heal->fop, error);
+    if (ret == 0) {
+        good = ec_char_array_to_mask(sources, ec->nodes);
+        bad = ec_char_array_to_mask(healed_sinks, ec->nodes);
+    } else {
+        op_ret = -1;
+        op_errno = -ret;
+    }
+    msources = alloca0(ec->nodes);
+    mhealed_sinks = alloca0(ec->nodes);
+    ret = ec_heal_metadata(frame, ec, loc->inode, msources, mhealed_sinks);
+    if (ret == 0) {
+        mgood = ec_char_array_to_mask(msources, ec->nodes);
+        mbad = ec_char_array_to_mask(mhealed_sinks, ec->nodes);
+    } else {
+        op_ret = -1;
+        op_errno = -ret;
     }
 
-    /* If there is a pending full request, resume it. */
-    if (next != NULL) {
-        ec_resume(next->fop, 0);
+out:
+    ec_reset_entry_healing(fop);
+    if (fop->cbks.heal) {
+        fop->cbks.heal(fop->req_frame, fop->data, fop->xl, op_ret, op_errno,
+                       ec_char_array_to_mask(participants, ec->nodes),
+                       mgood & good, mbad & bad, pending, NULL);
     }
+    if (frame)
+        STACK_DESTROY(frame->root);
+
+    return;
 }
 
-void ec_wind_heal(ec_t * ec, ec_fop_data_t * fop, int32_t idx)
+int
+ec_synctask_heal_wrap(void *opaque)
 {
-    ec_cbk_data_t * cbk;
-    ec_heal_t *heal = fop->heal;
+    ec_fop_data_t *fop = opaque;
+    ec_heal_do(fop->xl, fop, &fop->loc[0], fop->int32);
+    return 0;
+}
 
-    ec_trace("WIND", fop, "idx=%d", idx);
+int
+ec_heal_done(int ret, call_frame_t *heal, void *opaque)
+{
+    if (opaque)
+        ec_fop_data_release(opaque);
+    return 0;
+}
 
-    cbk = ec_cbk_data_allocate(fop->frame, fop->xl, fop, EC_FOP_HEAL, idx,
-                               fop->error == 0 ? 0 : -1, fop->error);
-    if (cbk != NULL)
-    {
-        cbk->uintptr[0] = heal->available;
-        cbk->uintptr[1] = heal->good;
-        cbk->uintptr[2] = heal->bad;
+ec_fop_data_t *
+__ec_dequeue_heals(ec_t *ec)
+{
+    ec_fop_data_t *fop = NULL;
+
+    if (list_empty(&ec->heal_waiting))
+        goto none;
+
+    if ((ec->background_heals > 0) && (ec->healers >= ec->background_heals))
+        goto none;
+
+    fop = list_entry(ec->heal_waiting.next, ec_fop_data_t, healer);
+    ec->heal_waiters--;
+    list_del_init(&fop->healer);
+    list_add(&fop->healer, &ec->healing);
+    ec->healers++;
+    return fop;
+none:
+    gf_msg_debug(ec->xl->name, 0, "Num healers: %d, Num Waiters: %d",
+                 ec->healers, ec->heal_waiters);
+    return NULL;
+}
 
-        ec_combine(cbk, NULL);
+void
+ec_heal_fail(ec_t *ec, ec_fop_data_t *fop)
+{
+    if (fop->cbks.heal) {
+        fop->cbks.heal(fop->req_frame, fop->data, ec->xl, -1, fop->error, 0, 0,
+                       0, 0, NULL);
     }
-
-    ec_complete(fop);
+    ec_fop_data_release(fop);
 }
 
-int32_t
-ec_manager_heal (ec_fop_data_t * fop, int32_t state)
+void
+ec_launch_heal(ec_t *ec, ec_fop_data_t *fop)
 {
-    ec_cbk_data_t * cbk;
-    ec_heal_t *heal = fop->heal;
+    int ret = 0;
+    call_frame_t *frame = NULL;
 
-    switch (state)
-    {
-        case EC_STATE_INIT:
-            ec_owner_set(fop->frame, fop->frame->root);
+    frame = create_frame(ec->xl, ec->xl->ctx->pool);
+    if (!frame) {
+        ret = -1;
+        goto out;
+    }
 
-            fop->error = ec_heal_init(fop);
-            if (fop->error != 0)
-            {
-                return EC_STATE_REPORT;
-            }
+    ec_owner_set(frame, frame->root);
+    /*Do heal as root*/
+    frame->root->uid = 0;
+    frame->root->gid = 0;
+    /*Mark the fops as internal*/
+    frame->root->pid = GF_CLIENT_PID_SELF_HEALD;
 
-            heal = fop->heal;
-            /* root loc doesn't have pargfid/parent */
-            if (loc_is_root (&heal->loc) ||
-                !gf_uuid_is_null(heal->loc.pargfid) || heal->loc.parent) {
-                    heal->nameheal = _gf_true;
-                    return EC_STATE_DISPATCH;
-            } else {
-                    /* No need to perform 'name' heal.*/
-                    return EC_STATE_HEAL_PRE_INODELK_LOCK;
-            }
+    ret = synctask_new(ec->xl->ctx->env, ec_synctask_heal_wrap, ec_heal_done,
+                       frame, fop);
+out:
+    if (ret < 0) {
+        ec_fop_set_error(fop, ENOMEM);
+        ec_heal_fail(ec, fop);
+    }
 
-        case EC_STATE_DISPATCH:
-            if (heal->done) {
-                return EC_STATE_HEAL_DISPATCH;
-            }
+    if (frame)
+        STACK_DESTROY(frame->root);
+}
 
-            ec_heal_entrylk(heal, ENTRYLK_LOCK);
+void
+ec_handle_healers_done(ec_fop_data_t *fop)
+{
+    ec_t *ec = fop->xl->private;
+    ec_fop_data_t *heal_fop = NULL;
 
-            return EC_STATE_HEAL_ENTRY_LOOKUP;
+    if (list_empty(&fop->healer))
+        return;
 
-        case EC_STATE_HEAL_ENTRY_LOOKUP:
-            ec_lookup(fop->frame, heal->xl, fop->mask, EC_MINIMUM_MIN,
-                      ec_heal_entry_lookup_cbk, heal, &heal->loc, NULL);
+    LOCK(&ec->lock);
 
-            return EC_STATE_HEAL_ENTRY_PREPARE;
+    list_del_init(&fop->healer);
 
-        case EC_STATE_HEAL_ENTRY_PREPARE:
-            if (!heal->partial || (heal->iatt.ia_type == IA_IFDIR)) {
-                ec_heal_prepare(heal);
-            }
+    do {
+        ec->healers--;
+        heal_fop = __ec_dequeue_heals(ec);
 
-            if (heal->partial) {
-                return EC_STATE_HEAL_UNLOCK_ENTRY;
-            }
+        if ((heal_fop != NULL) && ec->shutdown) {
+            /* This will prevent ec_handle_healers_done() to be
+             * called recursively. That would be problematic if
+             * the queue is too big. */
+            list_del_init(&heal_fop->healer);
 
-            return EC_STATE_HEAL_PRE_INODELK_LOCK;
+            UNLOCK(&ec->lock);
 
-        case EC_STATE_HEAL_PRE_INODELK_LOCK:
-            if (heal->done)
-                    return EC_STATE_HEAL_DISPATCH;
+            ec_fop_set_error(fop, ENOTCONN);
+            ec_heal_fail(ec, heal_fop);
 
-            /* Only heal data/metadata if enough information is supplied. */
-            if (gf_uuid_is_null(heal->loc.gfid))
-            {
-                ec_heal_entrylk(heal, ENTRYLK_UNLOCK);
+            LOCK(&ec->lock);
+        }
+    } while ((heal_fop != NULL) && ec->shutdown);
 
-                return EC_STATE_HEAL_DISPATCH;
-            }
+    UNLOCK(&ec->lock);
 
-            ec_heal_inodelk(heal, F_WRLCK, 0, 0, 0);
+    if (heal_fop)
+        ec_launch_heal(ec, heal_fop);
+}
 
-            return EC_STATE_HEAL_PRE_INODE_LOOKUP;
+gf_boolean_t
+ec_is_entry_healing(ec_fop_data_t *fop)
+{
+    ec_inode_t *ctx = NULL;
+    int32_t heal_count = 0;
+    loc_t *loc = NULL;
 
-        case EC_STATE_HEAL_PRE_INODE_LOOKUP:
-            ec_heal_lookup(heal, heal->fop->mask);
+    loc = &fop->loc[0];
 
-            return EC_STATE_HEAL_XATTRIBUTES_REMOVE;
+    LOCK(&loc->inode->lock);
+    {
+        ctx = __ec_inode_get(loc->inode, fop->xl);
+        if (ctx) {
+            heal_count = ctx->heal_count;
+        }
+    }
+    UNLOCK(&loc->inode->lock);
+    GF_ASSERT(heal_count >= 0);
+    return heal_count;
+}
 
-        case EC_STATE_HEAL_XATTRIBUTES_REMOVE:
-            ec_heal_removexattr_others(heal);
+void
+ec_heal_throttle(xlator_t *this, ec_fop_data_t *fop)
+{
+    gf_boolean_t can_heal = _gf_true;
+    ec_t *ec = this->private;
+    ec_fop_data_t *fop_rel = NULL;
 
-            return EC_STATE_HEAL_XATTRIBUTES_SET;
+    if (fop->req_frame == NULL) {
+        LOCK(&ec->lock);
+        {
+            if ((ec->background_heals > 0) &&
+                (ec->heal_wait_qlen + ec->background_heals) >
+                    (ec->heal_waiters + ec->healers)) {
+                if (!ec_is_entry_healing(fop)) {
+                    list_add_tail(&fop->healer, &ec->heal_waiting);
+                    ec->heal_waiters++;
+                    ec_set_entry_healing(fop);
+                } else {
+                    fop_rel = fop;
+                }
+                fop = __ec_dequeue_heals(ec);
+            } else {
+                can_heal = _gf_false;
+            }
+        }
+        UNLOCK(&ec->lock);
+    }
 
-        case EC_STATE_HEAL_XATTRIBUTES_SET:
-            ec_heal_setxattr_others(heal);
+    if (can_heal) {
+        if (fop) {
+            if (fop->req_frame != NULL) {
+                ec_set_entry_healing(fop);
+            }
+            ec_launch_heal(ec, fop);
+        }
+    } else {
+        gf_msg_debug(this->name, 0,
+                     "Max number of heals are "
+                     "pending, background self-heal rejected");
+        ec_fop_set_error(fop, EBUSY);
+        ec_heal_fail(ec, fop);
+    }
+    if (fop_rel) {
+        ec_heal_done(0, NULL, fop_rel);
+    }
+}
 
-            return EC_STATE_HEAL_ATTRIBUTES;
+void
+ec_heal(call_frame_t *frame, xlator_t *this, uintptr_t target,
+        uint32_t fop_flags, fop_heal_cbk_t func, void *data, loc_t *loc,
+        int32_t partial, dict_t *xdata)
+{
+    ec_cbk_t callback = {.heal = func};
+    ec_fop_data_t *fop = NULL;
+    int32_t err = EINVAL;
 
-        case EC_STATE_HEAL_ATTRIBUTES:
-            ec_heal_attr(heal);
+    gf_msg_trace("ec", 0, "EC(HEAL) %p", frame);
 
-            return EC_STATE_HEAL_OPEN;
+    VALIDATE_OR_GOTO(this, fail);
+    GF_VALIDATE_OR_GOTO(this->name, this->private, fail);
 
-        case EC_STATE_HEAL_OPEN:
-            ec_heal_open(heal);
+    if (!loc || !loc->inode || gf_uuid_is_null(loc->inode->gfid))
+        goto fail;
 
-            return EC_STATE_HEAL_REOPEN_FD;
+    if (frame && frame->local)
+        goto fail;
+    fop = ec_fop_data_allocate(frame, this, EC_FOP_HEAL, 0, target, fop_flags,
+                               NULL, NULL, callback, data);
 
-        case EC_STATE_HEAL_REOPEN_FD:
-            ec_heal_reopen_fd(heal);
+    err = ENOMEM;
 
-            return EC_STATE_HEAL_UNLOCK;
+    if (fop == NULL)
+        goto fail;
 
-        case -EC_STATE_HEAL_XATTRIBUTES_REMOVE:
-        case -EC_STATE_HEAL_XATTRIBUTES_SET:
-        case -EC_STATE_HEAL_ATTRIBUTES:
-        case -EC_STATE_HEAL_OPEN:
-        case -EC_STATE_HEAL_REOPEN_FD:
-        case -EC_STATE_HEAL_UNLOCK:
-        case EC_STATE_HEAL_UNLOCK:
-            ec_heal_inodelk(heal, F_UNLCK, 0, 0, 0);
+    fop->int32 = partial;
 
-        /* Fall through */
+    if (loc) {
+        if (loc_copy(&fop->loc[0], loc) != 0)
+            goto fail;
+    }
 
-        case -EC_STATE_HEAL_ENTRY_PREPARE:
-        case -EC_STATE_HEAL_PRE_INODELK_LOCK:
-        case -EC_STATE_HEAL_PRE_INODE_LOOKUP:
-        case -EC_STATE_HEAL_UNLOCK_ENTRY:
-        case EC_STATE_HEAL_UNLOCK_ENTRY:
-            if (heal->nameheal)
-                    ec_heal_entrylk(heal, ENTRYLK_UNLOCK);
+    if (xdata)
+        fop->xdata = dict_ref(xdata);
 
-            heal->bad = ec_heal_needs_data_rebuild(heal);
-            if (heal->bad != 0)
-            {
-                return EC_STATE_HEAL_DATA_LOCK;
-            }
+    ec_heal_throttle(this, fop);
 
-            return EC_STATE_HEAL_DISPATCH;
+    return;
 
-        case EC_STATE_HEAL_DATA_LOCK:
-            if (heal->done)
-            {
-                return EC_STATE_HEAL_POST_INODELK_LOCK;
-            }
+fail:
+    if (fop)
+        ec_fop_data_release(fop);
+    if (func)
+        func(frame, data, this, -1, err, 0, 0, 0, 0, NULL);
+}
 
-            ec_heal_inodelk(heal, F_WRLCK, 1, heal->offset, heal->size);
+int
+ec_replace_heal_done(int ret, call_frame_t *heal, void *opaque)
+{
+    ec_t *ec = opaque;
+    gf_boolean_t last_fop = _gf_false;
 
-            return EC_STATE_HEAL_DATA_COPY;
+    if (GF_ATOMIC_DEC(ec->async_fop_count) == 0) {
+        LOCK(&ec->lock);
+        {
+            last_fop = __ec_is_last_fop(ec);
+        }
+        UNLOCK(&ec->lock);
+    }
+    gf_msg_debug(ec->xl->name, 0, "getxattr on bricks is done ret %d", ret);
 
-        case EC_STATE_HEAL_DATA_COPY:
-            ec_heal_data(heal);
+    if (last_fop)
+        ec_pending_fops_completed(ec);
 
-            return EC_STATE_HEAL_DATA_UNLOCK;
+    return 0;
+}
 
-        case -EC_STATE_HEAL_DATA_COPY:
-        case -EC_STATE_HEAL_DATA_UNLOCK:
-        case EC_STATE_HEAL_DATA_UNLOCK:
-            ec_heal_inodelk(heal, F_UNLCK, 1, heal->offset, heal->size);
+int32_t
+ec_replace_heal(ec_t *ec, inode_t *inode)
+{
+    loc_t loc = {0};
+    int ret = 0;
+
+    loc.inode = inode_ref(inode);
+    gf_uuid_copy(loc.gfid, inode->gfid);
+    ret = syncop_getxattr(ec->xl, &loc, NULL, EC_XATTR_HEAL, NULL, NULL);
+    if (ret < 0)
+        gf_msg_debug(ec->xl->name, 0, "Heal failed for replace brick ret = %d",
+                     ret);
+
+    /* Once the root inode has been checked, it might have triggered a
+     * self-heal on it after a replace brick command or for some other
+     * reason. It can also happen that the volume already had damaged
+     * files in the index, even if the heal on the root directory failed.
+     * In both cases we need to wake all index healers to continue
+     * healing remaining entries that are marked as dirty. */
+    ec_shd_index_healer_wake(ec);
 
-            heal->offset += heal->size;
+    loc_wipe(&loc);
+    return ret;
+}
 
-            return EC_STATE_HEAL_DATA_LOCK;
+int32_t
+ec_replace_brick_heal_wrap(void *opaque)
+{
+    ec_t *ec = opaque;
+    inode_table_t *itable = NULL;
+    int32_t ret = -1;
 
-        case EC_STATE_HEAL_POST_INODELK_LOCK:
-            ec_heal_inodelk(heal, F_WRLCK, 1, 0, 0);
+    if (ec->xl->itable)
+        itable = ec->xl->itable;
+    else
+        goto out;
 
-            return EC_STATE_HEAL_POST_INODE_LOOKUP;
+    if (xlator_is_cleanup_starting(ec->xl))
+        goto out;
 
-        case EC_STATE_HEAL_POST_INODE_LOOKUP:
-            heal->fixed = heal->bad;
-            ec_heal_update_dirty(heal, heal->bad);
-            ec_heal_lookup(heal, heal->good);
+    ret = ec_replace_heal(ec, itable->root);
+out:
+    return ret;
+}
 
-            return EC_STATE_HEAL_SETATTR;
+int32_t
+ec_launch_replace_heal(ec_t *ec)
+{
+    int ret = -1;
 
-        case EC_STATE_HEAL_SETATTR:
-            ec_setattr(heal->fop->frame, heal->xl, heal->fixed, EC_MINIMUM_ONE,
-                       ec_heal_setattr_cbk, heal, &heal->loc, &heal->iatt,
-                       GF_SET_ATTR_MODE | GF_SET_ATTR_UID | GF_SET_ATTR_GID |
-                       GF_SET_ATTR_ATIME | GF_SET_ATTR_MTIME, NULL);
+    ret = synctask_new(ec->xl->ctx->env, ec_replace_brick_heal_wrap,
+                       ec_replace_heal_done, NULL, ec);
 
-            return EC_STATE_HEAL_POST_INODELK_UNLOCK;
+    if (ret < 0) {
+        gf_msg_debug(ec->xl->name, 0, "Heal failed for replace brick ret = %d",
+                     ret);
+        ec_replace_heal_done(-1, NULL, ec);
+    }
 
-        case -EC_STATE_HEAL_SETATTR:
-        case -EC_STATE_HEAL_POST_INODELK_UNLOCK:
-        case EC_STATE_HEAL_POST_INODELK_UNLOCK:
-            ec_heal_inodelk(heal, F_UNLCK, 1, 0, 0);
+    return ret;
+}
 
-            return EC_STATE_HEAL_DISPATCH;
-
-        case -EC_STATE_HEAL_POST_INODELK_LOCK:
-        case -EC_STATE_HEAL_POST_INODE_LOOKUP:
-        case -EC_STATE_HEAL_ENTRY_LOOKUP:
-        case -EC_STATE_HEAL_DATA_LOCK:
-        case -EC_STATE_HEAL_DISPATCH:
-        case EC_STATE_HEAL_DISPATCH:
-            ec_heal_dispatch(heal);
-
-            return EC_STATE_PREPARE_ANSWER;
-
-        case EC_STATE_PREPARE_ANSWER:
-            cbk = fop->answer;
-            if (cbk != NULL)
-            {
-                if (!ec_dict_combine(cbk, EC_COMBINE_XDATA))
-                {
-                    if (cbk->op_ret >= 0)
-                    {
-                        cbk->op_ret = -1;
-                        cbk->op_errno = EIO;
-                    }
-                }
-                if (cbk->op_ret < 0)
-                {
-                    ec_fop_set_error(fop, cbk->op_errno);
-                }
-            }
-            else
-            {
-                ec_fop_set_error(fop, EIO);
-            }
+int32_t
+ec_set_heal_info(dict_t **dict_rsp, char *status)
+{
+    dict_t *dict = NULL;
+    int ret = 0;
 
-            return EC_STATE_REPORT;
+    dict = dict_new();
+    if (!dict) {
+        ret = -ENOMEM;
+        goto out;
+    }
+    ret = dict_set_str(dict, "heal-info", status);
+    if (ret) {
+        gf_msg(THIS->name, GF_LOG_WARNING, -ret, EC_MSG_HEAL_FAIL,
+               "Failed to set heal-info key to "
+               "%s",
+               status);
+        dict_unref(dict);
+        dict = NULL;
+    }
+    *dict_rsp = dict;
+out:
+    return ret;
+}
 
-        case EC_STATE_REPORT:
-            cbk = fop->answer;
-
-            GF_ASSERT(cbk != NULL);
-
-            if (fop->id == EC_FOP_HEAL)
-            {
-                if (fop->cbks.heal != NULL)
-                {
-                    fop->cbks.heal(fop->req_frame, fop, fop->xl, cbk->op_ret,
-                                   cbk->op_errno, cbk->uintptr[0],
-                                   cbk->uintptr[1], cbk->uintptr[2],
-                                   cbk->xdata);
-                }
-            }
-            else
-            {
-                if (fop->cbks.fheal != NULL)
-                {
-                    fop->cbks.fheal(fop->req_frame, fop, fop->xl, cbk->op_ret,
-                                    cbk->op_errno, cbk->uintptr[0],
-                                    cbk->uintptr[1], cbk->uintptr[2],
-                                    cbk->xdata);
+static int32_t
+_need_heal_calculate(ec_t *ec, uint64_t *dirty, unsigned char *sources,
+                     gf_boolean_t self_locked, int32_t lock_count,
+                     ec_heal_need_t *need_heal, uint64_t *versions)
+{
+    int i = 0;
+    int source_count = 0;
+
+    source_count = EC_COUNT(sources, ec->nodes);
+    if (source_count == ec->nodes) {
+        *need_heal = EC_HEAL_NONEED;
+        if (self_locked || lock_count == 0) {
+            for (i = 0; i < ec->nodes; i++) {
+                if (dirty[i] || (versions[i] != versions[0])) {
+                    *need_heal = EC_HEAL_MUST;
+                    goto out;
                 }
             }
-
-            return EC_STATE_END;
-
-        case -EC_STATE_INIT:
-        case -EC_STATE_DISPATCH:
-        case -EC_STATE_PREPARE_ANSWER:
-        case -EC_STATE_REPORT:
-            GF_ASSERT(fop->error != 0);
-
-            if (fop->id == EC_FOP_HEAL)
-            {
-                if (fop->cbks.heal != NULL)
-                {
-                    fop->cbks.heal(fop->req_frame, fop, fop->xl, -1,
-                                   fop->error, 0, 0, 0, NULL);
+            /* If lock count is 0, all dirty flags are 0 and all the
+             * versions are macthing then why are we here. It looks
+             * like something went wrong while removing the index entries
+             * after completing a successful heal or fop. In this case
+             * we need to remove this index entry to avoid triggering heal
+             * in a loop and causing lookups again and again*/
+            *need_heal = EC_HEAL_PURGE_INDEX;
+        } else {
+            for (i = 0; i < ec->nodes; i++) {
+                /* Since each lock can only increment the dirty
+                 * count once, if dirty is > 1 it means that
+                 * another operation has left the dirty count
+                 * set and this indicates a problem in the
+                 * inode.*/
+                if (dirty[i] > 1) {
+                    *need_heal = EC_HEAL_MUST;
+                    goto out;
                 }
-            }
-            else
-            {
-                if (fop->cbks.fheal != NULL)
-                {
-                    fop->cbks.fheal(fop->req_frame, fop, fop->xl, -1,
-                                    fop->error, 0, 0, 0, NULL);
+                if (dirty[i] != dirty[0] || (versions[i] != versions[0])) {
+                    *need_heal = EC_HEAL_MAYBE;
                 }
             }
+        }
+    } else {
+        *need_heal = EC_HEAL_MUST;
+    }
 
-            return EC_STATE_END;
+out:
+    return source_count;
+}
 
-        default:
-            gf_log(fop->xl->name, GF_LOG_ERROR, "Unhandled state %d for %s",
-                   state, ec_fop_name(fop->id));
+static int32_t
+ec_need_metadata_heal(ec_t *ec, inode_t *inode, default_args_cbk_t *replies,
+                      int32_t lock_count, gf_boolean_t self_locked,
+                      gf_boolean_t thorough, ec_heal_need_t *need_heal)
+{
+    uint64_t *dirty = NULL;
+    unsigned char *sources = NULL;
+    unsigned char *healed_sinks = NULL;
+    uint64_t *meta_versions = NULL;
+    int ret = 0;
+
+    sources = alloca0(ec->nodes);
+    healed_sinks = alloca0(ec->nodes);
+    dirty = alloca0(ec->nodes * sizeof(*dirty));
+    meta_versions = alloca0(ec->nodes * sizeof(*meta_versions));
+    ret = ec_heal_metadata_find_direction(ec, replies, meta_versions, dirty,
+                                          sources, healed_sinks);
+    if (ret < 0 && ret != -EIO) {
+        goto out;
+    }
 
-            return EC_STATE_END;
+    ret = _need_heal_calculate(ec, dirty, sources, self_locked, lock_count,
+                               need_heal, meta_versions);
+out:
+    return ret;
+}
+
+static int32_t
+ec_need_data_heal(ec_t *ec, inode_t *inode, default_args_cbk_t *replies,
+                  int32_t lock_count, gf_boolean_t self_locked,
+                  gf_boolean_t thorough, ec_heal_need_t *need_heal)
+{
+    uint64_t *dirty = NULL;
+    unsigned char *sources = NULL;
+    unsigned char *healed_sinks = NULL;
+    uint64_t *data_versions = NULL;
+    uint64_t *size = NULL;
+    int ret = 0;
+
+    sources = alloca0(ec->nodes);
+    healed_sinks = alloca0(ec->nodes);
+    dirty = alloca0(ec->nodes * sizeof(*dirty));
+    data_versions = alloca0(ec->nodes * sizeof(*data_versions));
+    size = alloca0(ec->nodes * sizeof(*size));
+
+    /* When dd is going on and heal info is called there is a very good
+     * chance for on disk sizes to mismatch even though nothing is wrong
+     * we don't need ondisk size check there. But if the file is either
+     * self-locked or the caller wants a thorough check then make sure to
+     * perform on disk check also. */
+    ret = ec_heal_data_find_direction(
+        ec, replies, data_versions, dirty, size, sources, healed_sinks,
+        self_locked || thorough, EC_COMBINE_XDATA);
+    if (ret < 0 && ret != -EIO) {
+        goto out;
     }
+
+    ret = _need_heal_calculate(ec, dirty, sources, self_locked, lock_count,
+                               need_heal, data_versions);
+out:
+    return ret;
 }
 
-void ec_heal(call_frame_t * frame, xlator_t * this, uintptr_t target,
-             int32_t minimum, fop_heal_cbk_t func, void * data, loc_t * loc,
-             int32_t partial, dict_t *xdata)
+static int32_t
+ec_need_entry_heal(ec_t *ec, inode_t *inode, default_args_cbk_t *replies,
+                   int32_t lock_count, gf_boolean_t self_locked,
+                   gf_boolean_t thorough, ec_heal_need_t *need_heal)
 {
-    ec_cbk_t callback = { .heal = func };
-    ec_fop_data_t * fop = NULL;
-    int32_t error = EIO;
+    uint64_t *dirty = NULL;
+    unsigned char *sources = NULL;
+    unsigned char *healed_sinks = NULL;
+    uint64_t *data_versions = NULL;
+    int ret = 0;
+
+    sources = alloca0(ec->nodes);
+    healed_sinks = alloca0(ec->nodes);
+    dirty = alloca0(ec->nodes * sizeof(*dirty));
+    data_versions = alloca0(ec->nodes * sizeof(*data_versions));
+
+    ret = ec_heal_entry_find_direction(ec, replies, data_versions, dirty,
+                                       sources, healed_sinks);
+    if (ret < 0 && ret != -EIO) {
+        goto out;
+    }
 
-    gf_log("ec", GF_LOG_TRACE, "EC(HEAL) %p", frame);
+    ret = _need_heal_calculate(ec, dirty, sources, self_locked, lock_count,
+                               need_heal, data_versions);
+out:
+    return ret;
+}
 
-    VALIDATE_OR_GOTO(this, out);
-    GF_VALIDATE_OR_GOTO(this->name, this->private, out);
+static int32_t
+ec_need_heal(ec_t *ec, inode_t *inode, default_args_cbk_t *replies,
+             int32_t lock_count, gf_boolean_t self_locked,
+             gf_boolean_t thorough, ec_heal_need_t *need_heal)
+{
+    int ret = 0;
 
-    fop = ec_fop_data_allocate(frame, this, EC_FOP_HEAL,
-                               EC_FLAG_UPDATE_LOC_INODE, target, minimum,
-                               ec_wind_heal, ec_manager_heal, callback, data);
-    if (fop == NULL)
-    {
+    ret = ec_need_metadata_heal(ec, inode, replies, lock_count, self_locked,
+                                thorough, need_heal);
+    if (ret < 0)
         goto out;
+
+    if (*need_heal == EC_HEAL_MUST)
+        goto out;
+
+    if (inode->ia_type == IA_IFREG) {
+        ret = ec_need_data_heal(ec, inode, replies, lock_count, self_locked,
+                                thorough, need_heal);
+    } else if (inode->ia_type == IA_IFDIR) {
+        ret = ec_need_entry_heal(ec, inode, replies, lock_count, self_locked,
+                                 thorough, need_heal);
     }
 
-    fop->int32 = partial;
+out:
+    return ret;
+}
 
-    if (loc != NULL)
-    {
-        if (loc_copy(&fop->loc[0], loc) != 0)
-        {
-            gf_log(this->name, GF_LOG_ERROR, "Failed to copy a location.");
+int32_t
+ec_heal_inspect(call_frame_t *frame, ec_t *ec, inode_t *inode,
+                unsigned char *locked_on, gf_boolean_t self_locked,
+                gf_boolean_t thorough, ec_heal_need_t *need_heal)
+{
+    loc_t loc = {0};
+    int i = 0;
+    int ret = 0;
+    dict_t *xdata = NULL;
+    uint64_t zero_array[2] = {0};
+    uint64_t zero_value = 0;
+    unsigned char *output = NULL;
+    default_args_cbk_t *replies = NULL;
+    int32_t lock_count = 0;
+
+    EC_REPLIES_ALLOC(replies, ec->nodes);
+    output = alloca0(ec->nodes);
+
+    loc.inode = inode_ref(inode);
+    gf_uuid_copy(loc.gfid, inode->gfid);
 
-            goto out;
-        }
+    xdata = dict_new();
+    if (!xdata ||
+        dict_set_static_bin(xdata, EC_XATTR_VERSION, zero_array,
+                            sizeof(zero_array)) ||
+        dict_set_static_bin(xdata, EC_XATTR_DIRTY, zero_array,
+                            sizeof(zero_array)) ||
+        dict_set_static_bin(xdata, EC_XATTR_SIZE, &zero_value,
+                            sizeof(zero_value))) {
+        ret = -ENOMEM;
+        goto out;
     }
-    if (xdata != NULL)
-    {
-        fop->xdata = dict_ref(xdata);
-        if (fop->xdata == NULL)
-        {
-            gf_log(this->name, GF_LOG_ERROR, "Failed to reference a "
-                                             "dictionary.");
 
+    if (!self_locked) {
+        ret = dict_set_str(xdata, GLUSTERFS_INODELK_DOM_COUNT, ec->xl->name);
+        if (ret) {
+            ret = -ENOMEM;
             goto out;
         }
     }
 
-    error = 0;
+    ret = cluster_lookup(ec->xl_list, locked_on, ec->nodes, replies, output,
+                         frame, ec->xl, &loc, xdata);
 
-out:
-    if (fop != NULL)
-    {
-        ec_manager(fop, error);
+    if (ret != ec->nodes) {
+        ret = ec->nodes;
+        *need_heal = EC_HEAL_MUST;
+        goto out;
     }
-    else
-    {
-        func(frame, NULL, this, -1, EIO, 0, 0, 0, NULL);
+
+    if (self_locked)
+        goto need_heal;
+
+    for (i = 0; i < ec->nodes; i++) {
+        if (!output[i] || !replies[i].xdata) {
+            continue;
+        }
+        if ((dict_get_int32(replies[i].xdata, GLUSTERFS_INODELK_COUNT,
+                            &lock_count) == 0) &&
+            lock_count > 0) {
+            break;
+        }
+    }
+need_heal:
+    ret = ec_need_heal(ec, inode, replies, lock_count, self_locked, thorough,
+                       need_heal);
+out:
+    cluster_replies_wipe(replies, ec->nodes);
+    loc_wipe(&loc);
+    if (xdata) {
+        dict_unref(xdata);
     }
+    return ret;
 }
 
-/* FOP: fheal */
+int32_t
+ec_heal_locked_inspect(call_frame_t *frame, ec_t *ec, inode_t *inode,
+                       ec_heal_need_t *need_heal)
+{
+    unsigned char *locked_on = NULL;
+    unsigned char *up_subvols = NULL;
+    unsigned char *output = NULL;
+    default_args_cbk_t *replies = NULL;
+    int ret = 0;
+
+    EC_REPLIES_ALLOC(replies, ec->nodes);
+    locked_on = alloca0(ec->nodes);
+    output = alloca0(ec->nodes);
+    up_subvols = alloca0(ec->nodes);
+    ec_mask_to_char_array(ec->xl_up, up_subvols, ec->nodes);
+
+    ret = cluster_inodelk(ec->xl_list, up_subvols, ec->nodes, replies,
+                          locked_on, frame, ec->xl, ec->xl->name, inode, 0, 0);
+    if (ret != ec->nodes) {
+        *need_heal = EC_HEAL_MUST;
+        goto unlock;
+    }
+    ret = ec_heal_inspect(frame, ec, inode, locked_on, _gf_true, _gf_true,
+                          need_heal);
+unlock:
+    cluster_uninodelk(ec->xl_list, locked_on, ec->nodes, replies, output, frame,
+                      ec->xl, ec->xl->name, inode, 0, 0);
+    cluster_replies_wipe(replies, ec->nodes);
+    return ret;
+}
 
-void ec_wind_fheal(ec_t * ec, ec_fop_data_t * fop, int32_t idx)
+int32_t
+ec_get_heal_info(xlator_t *this, loc_t *entry_loc, dict_t **dict_rsp)
 {
-    ec_cbk_data_t * cbk;
-    ec_heal_t *heal = fop->heal;
+    int ret = -ENOMEM;
+    ec_heal_need_t need_heal = EC_HEAL_NONEED;
+    call_frame_t *frame = NULL;
+    ec_t *ec = NULL;
+    unsigned char *up_subvols = NULL;
+    loc_t loc = {
+        0,
+    };
 
-    ec_trace("WIND", fop, "idx=%d", idx);
+    VALIDATE_OR_GOTO(this, out);
+    GF_VALIDATE_OR_GOTO(this->name, entry_loc, out);
 
-    cbk = ec_cbk_data_allocate(fop->frame, fop->xl, fop, EC_FOP_FHEAL, idx,
-                               fop->error == 0 ? 0 : -1, fop->error);
-    if (cbk != NULL)
-    {
-        cbk->uintptr[0] = heal->available;
-        cbk->uintptr[1] = heal->good;
-        cbk->uintptr[2] = heal->bad;
+    ec = this->private;
+    up_subvols = alloca0(ec->nodes);
+    ec_mask_to_char_array(ec->xl_up, up_subvols, ec->nodes);
 
-        ec_combine(cbk, NULL);
+    if (EC_COUNT(up_subvols, ec->nodes) != ec->nodes) {
+        need_heal = EC_HEAL_MUST;
+        goto set_heal;
     }
+    frame = create_frame(this, this->ctx->pool);
+    if (!frame) {
+        goto out;
+    }
+    ec_owner_set(frame, frame->root);
+    frame->root->uid = 0;
+    frame->root->gid = 0;
+    frame->root->pid = GF_CLIENT_PID_SELF_HEALD;
 
-    ec_complete(fop);
-}
-
-void ec_fheal(call_frame_t * frame, xlator_t * this, uintptr_t target,
-              int32_t minimum, fop_fheal_cbk_t func, void * data, fd_t * fd,
-              int32_t partial, dict_t *xdata)
-{
-    ec_fd_t * ctx = ec_fd_get(fd, this);
+    if (loc_copy(&loc, entry_loc) != 0) {
+        gf_msg(this->name, GF_LOG_ERROR, ENOMEM, EC_MSG_LOC_COPY_FAIL,
+               "Failed to copy a location.");
+        goto out;
+    }
+    if (!loc.inode) {
+        ret = syncop_inode_find(this, this, loc.gfid, &loc.inode, NULL, NULL);
+        if (ret < 0)
+            goto out;
+    }
 
-    if (ctx != NULL)
-    {
-        gf_log("ec", GF_LOG_DEBUG, "FHEAL ctx: flags=%X, open=%lX, bad=%lX",
-               ctx->flags, ctx->open, ctx->bad);
-        ec_heal(frame, this, target, minimum, func, data, &ctx->loc, partial,
-                xdata);
+    ret = ec_heal_inspect(frame, ec, loc.inode, up_subvols, _gf_false,
+                          _gf_false, &need_heal);
+    if (ret == ec->nodes && need_heal != EC_HEAL_MAYBE) {
+        goto set_heal;
+    }
+    need_heal = EC_HEAL_NONEED;
+    ret = ec_heal_locked_inspect(frame, ec, loc.inode, &need_heal);
+    if (ret < 0)
+        goto out;
+set_heal:
+    if (need_heal == EC_HEAL_MUST) {
+        ret = ec_set_heal_info(dict_rsp, "heal");
+    } else {
+        ret = ec_set_heal_info(dict_rsp, "no-heal");
     }
+out:
+    if (frame) {
+        STACK_DESTROY(frame->root);
+    }
+    loc_wipe(&loc);
+    return ret;
 }
diff --git a/xlators/cluster/ec/src/ec-heald.c b/xlators/cluster/ec/src/ec-heald.c
index 53b3996590c..5c1586bc9c5 100644
--- a/xlators/cluster/ec/src/ec-heald.c
+++ b/xlators/cluster/ec/src/ec-heald.c
@@ -8,592 +8,674 @@
   cases as published by the Free Software Foundation.
 */
 
-#include "xlator.h"
-#include "defaults.h"
-#include "compat-errno.h"
+#include <glusterfs/defaults.h>
+#include <glusterfs/compat-errno.h>
 #include "ec.h"
+#include "ec-messages.h"
 #include "ec-heald.h"
 #include "ec-mem-types.h"
-#include "syncop.h"
-#include "syncop-utils.h"
+#include <glusterfs/syncop.h>
+#include <glusterfs/syncop-utils.h>
 #include "protocol-common.h"
 
-#define SHD_INODE_LRU_LIMIT          2048
-#define ASSERT_LOCAL(this, healer)				        \
-        do {                                                            \
-                if (!ec_shd_is_subvol_local (this, healer->subvol)) {	\
-                        healer->local = _gf_false;			\
-                        if (safe_break (healer)) {			\
-                                break;					\
-                        } else {					\
-                                continue;				\
-                        }						\
-                } else {						\
-                        healer->local = _gf_true;			\
-                }                                                       \
-        } while (0);
-
-
-#define NTH_INDEX_HEALER(this, n) (&((((ec_t *)this->private))->shd.index_healers[n]))
-#define NTH_FULL_HEALER(this, n) (&((((ec_t *)this->private))->shd.full_healers[n]))
+#define NTH_INDEX_HEALER(this, n)                                              \
+    (&((((ec_t *)this->private))->shd.index_healers[n]))
+#define NTH_FULL_HEALER(this, n)                                               \
+    (&((((ec_t *)this->private))->shd.full_healers[n]))
 
 gf_boolean_t
-ec_shd_is_subvol_local (xlator_t *this, int subvol)
+ec_shd_is_subvol_local(xlator_t *this, int subvol)
 {
-        ec_t *ec = NULL;
-        gf_boolean_t is_local = _gf_false;
-        loc_t loc = {0, };
-
-        ec = this->private;
-        loc.inode = this->itable->root;
-        syncop_is_subvol_local (ec->xl_list[subvol], &loc, &is_local);
-        return is_local;
+    ec_t *ec = NULL;
+    gf_boolean_t is_local = _gf_false;
+    loc_t loc = {
+        0,
+    };
+
+    ec = this->private;
+    loc.inode = this->itable->root;
+    syncop_is_subvol_local(ec->xl_list[subvol], &loc, &is_local);
+    return is_local;
 }
 
 char *
-ec_subvol_name (xlator_t *this, int subvol)
+ec_subvol_name(xlator_t *this, int subvol)
 {
-        ec_t *ec = NULL;
+    ec_t *ec = NULL;
 
-        ec = this->private;
-        if (subvol < 0 || subvol > ec->nodes)
-                return NULL;
+    ec = this->private;
+    if (subvol < 0 || subvol > ec->nodes)
+        return NULL;
 
-        return ec->xl_list[subvol]->name;
+    return ec->xl_list[subvol]->name;
 }
 
 int
-__ec_shd_healer_wait (struct subvol_healer *healer)
+__ec_shd_healer_wait(struct subvol_healer *healer)
 {
-        ec_t *ec = NULL;
-        struct timespec wait_till = {0, };
-        int ret = 0;
+    ec_t *ec = NULL;
+    struct timespec wait_till = {
+        0,
+    };
+    int ret = 0;
 
-        ec = healer->this->private;
+    ec = healer->this->private;
 
 disabled_loop:
-        wait_till.tv_sec = time (NULL) + 60;
-
-        while (!healer->rerun) {
-                ret = pthread_cond_timedwait (&healer->cond,
-                                              &healer->mutex,
-                                              &wait_till);
-                if (ret == ETIMEDOUT)
-                        break;
-        }
+    wait_till.tv_sec = gf_time() + ec->shd.timeout;
 
-        ret = healer->rerun;
-        healer->rerun = 0;
+    while (!healer->rerun) {
+        ret = pthread_cond_timedwait(&healer->cond, &healer->mutex, &wait_till);
+        if (ret == ETIMEDOUT)
+            break;
+    }
 
-        if (!ec->shd.enabled || !ec->up)
-                goto disabled_loop;
+    if (ec->shutdown) {
+        healer->running = _gf_false;
+        return -1;
+    }
 
-        return ret;
-}
+    ret = healer->rerun;
+    healer->rerun = 0;
 
+    if (!ec->shd.enabled || !ec->up)
+        goto disabled_loop;
+
+    return ret;
+}
 
 int
-ec_shd_healer_wait (struct subvol_healer *healer)
+ec_shd_healer_wait(struct subvol_healer *healer)
 {
-        int ret = 0;
+    int ret = 0;
 
-        pthread_mutex_lock (&healer->mutex);
-        {
-                ret = __ec_shd_healer_wait (healer);
-        }
-        pthread_mutex_unlock (&healer->mutex);
+    pthread_mutex_lock(&healer->mutex);
+    {
+        ret = __ec_shd_healer_wait(healer);
+    }
+    pthread_mutex_unlock(&healer->mutex);
 
-        return ret;
+    return ret;
 }
 
-
-gf_boolean_t
-safe_break (struct subvol_healer *healer)
+int
+ec_shd_index_inode(xlator_t *this, xlator_t *subvol, inode_t **inode)
 {
-        gf_boolean_t ret = _gf_false;
+    loc_t rootloc = {
+        0,
+    };
+    int ret = 0;
+    dict_t *xattr = NULL;
+    void *index_gfid = NULL;
+
+    *inode = NULL;
+    rootloc.inode = inode_ref(this->itable->root);
+    gf_uuid_copy(rootloc.gfid, rootloc.inode->gfid);
+
+    ret = syncop_getxattr(subvol, &rootloc, &xattr, GF_XATTROP_INDEX_GFID, NULL,
+                          NULL);
+    if (ret < 0)
+        goto out;
+    if (!xattr) {
+        ret = -EINVAL;
+        goto out;
+    }
+
+    ret = dict_get_ptr(xattr, GF_XATTROP_INDEX_GFID, &index_gfid);
+    if (ret)
+        goto out;
+
+    gf_msg_debug(this->name, 0, "index-dir gfid for %s: %s", subvol->name,
+                 uuid_utoa(index_gfid));
+
+    ret = syncop_inode_find(this, subvol, index_gfid, inode, NULL, NULL);
 
-        pthread_mutex_lock (&healer->mutex);
-        {
-                if (healer->rerun)
-                        goto unlock;
+out:
+    loc_wipe(&rootloc);
 
-                healer->running = _gf_false;
-                ret = _gf_true;
-        }
-unlock:
-        pthread_mutex_unlock (&healer->mutex);
+    if (xattr)
+        dict_unref(xattr);
 
-        return ret;
+    return ret;
 }
 
-
-inode_t *
-ec_shd_inode_find (xlator_t *this, xlator_t *subvol, uuid_t gfid)
+int
+ec_shd_index_purge(xlator_t *subvol, inode_t *inode, char *name)
 {
-        inode_t     *inode = NULL;
-        int         ret    = 0;
-        loc_t       loc    = {0, };
-        struct iatt iatt   = {0, };
-
-        inode = inode_find (this->itable, gfid);
-        if (inode) {
-                inode_lookup (inode);
-                goto out;
-        }
+    loc_t loc = {
+        0,
+    };
+    int ret = 0;
 
-        loc.inode = inode_new (this->itable);
-        if (!loc.inode)
-                goto out;
-        gf_uuid_copy (loc.gfid, gfid);
+    loc.parent = inode_ref(inode);
+    loc.name = name;
 
-        ret = syncop_lookup (subvol, &loc, &iatt, NULL, NULL, NULL);
-        if (ret < 0)
-                goto out;
+    ret = syncop_unlink(subvol, &loc, NULL, NULL);
 
-        inode = inode_link (loc.inode, NULL, NULL, &iatt);
-        if (inode)
-                inode_lookup (inode);
-out:
-        loc_wipe (&loc);
-        return inode;
+    loc_wipe(&loc);
+    return ret;
 }
 
+static gf_boolean_t
+ec_is_heal_completed(char *status)
+{
+    char *bad_pos = NULL;
+    char *zero_pos = NULL;
+
+    if (!status) {
+        return _gf_false;
+    }
+
+    /*Logic:
+     * Status will be of the form Good: <binary>, Bad: <binary>
+     * If heal completes, if we do strchr for '0' it should be present after
+     * 'Bad:' i.e. strRchr for ':'
+     * */
+
+    zero_pos = strchr(status, '0');
+    bad_pos = strrchr(status, ':');
+    if (!zero_pos || !bad_pos) {
+        /*malformed status*/
+        return _gf_false;
+    }
+
+    if (zero_pos > bad_pos) {
+        return _gf_true;
+    }
+
+    return _gf_false;
+}
 
-inode_t*
-ec_shd_index_inode (xlator_t *this, xlator_t *subvol)
+int
+ec_shd_selfheal(struct subvol_healer *healer, int child, loc_t *loc,
+                gf_boolean_t full)
 {
-        loc_t   rootloc     = {0, };
-        inode_t *inode      = NULL;
-        int     ret         = 0;
-        dict_t  *xattr      = NULL;
-        void    *index_gfid = NULL;
-
-        rootloc.inode = inode_ref (this->itable->root);
-        gf_uuid_copy (rootloc.gfid, rootloc.inode->gfid);
-
-        ret = syncop_getxattr (subvol, &rootloc, &xattr,
-                               GF_XATTROP_INDEX_GFID, NULL, NULL);
-        if (ret || !xattr) {
-                errno = -ret;
-                goto out;
+    dict_t *xdata = NULL;
+    dict_t *dict = NULL;
+    uint32_t count;
+    int32_t ret;
+    char *heal_status = NULL;
+    ec_t *ec = healer->this->private;
+
+    GF_ATOMIC_INC(ec->stats.shd.attempted);
+    ret = syncop_getxattr(healer->this, loc, &dict, EC_XATTR_HEAL, NULL,
+                          &xdata);
+    if (ret == 0) {
+        if (dict && (dict_get_str(dict, EC_XATTR_HEAL, &heal_status) == 0)) {
+            if (ec_is_heal_completed(heal_status)) {
+                GF_ATOMIC_INC(ec->stats.shd.completed);
+            }
         }
+    }
+
+    if (!full && (loc->inode->ia_type == IA_IFDIR)) {
+        /* If we have just healed a directory, it's possible that
+         * other index entries have appeared to be healed. */
+        if ((xdata != NULL) &&
+            (dict_get_uint32(xdata, EC_XATTR_HEAL_NEW, &count) == 0) &&
+            (count > 0)) {
+            /* Force a rerun of the index healer. */
+            gf_msg_debug(healer->this->name, 0, "%d more entries to heal",
+                         count);
+
+            healer->rerun = _gf_true;
+        }
+    }
 
-        ret = dict_get_ptr (xattr, GF_XATTROP_INDEX_GFID, &index_gfid);
-        if (ret)
-                goto out;
-
-        gf_log (this->name, GF_LOG_DEBUG, "index-dir gfid for %s: %s",
-                subvol->name, uuid_utoa (index_gfid));
-
-        inode = ec_shd_inode_find (this, subvol, index_gfid);
-
-out:
-        loc_wipe (&rootloc);
+    if (xdata != NULL) {
+        dict_unref(xdata);
+    }
 
-        if (xattr)
-                dict_unref (xattr);
+    if (dict) {
+        dict_unref(dict);
+    }
 
-        return inode;
+    return ret;
 }
 
 int
-ec_shd_index_purge (xlator_t *subvol, inode_t *inode, char *name)
+ec_shd_index_heal(xlator_t *subvol, gf_dirent_t *entry, loc_t *parent,
+                  void *data)
 {
-        loc_t loc = {0, };
-        int ret = 0;
-
-        loc.parent = inode_ref (inode);
-        loc.name = name;
+    struct subvol_healer *healer = data;
+    ec_t *ec = NULL;
+    loc_t loc = {0};
+    int ret = 0;
+
+    ec = healer->this->private;
+    if (ec->xl_up_count <= ec->fragments) {
+        return -ENOTCONN;
+    }
+    if (!ec->shd.enabled)
+        return -EBUSY;
+
+    gf_msg_debug(healer->this->name, 0, "got entry: %s", entry->d_name);
+
+    ret = gf_uuid_parse(entry->d_name, loc.gfid);
+    if (ret)
+        return 0;
 
-        ret = syncop_unlink (subvol, &loc, NULL, NULL);
+    /* If this fails with ENOENT/ESTALE index is stale */
+    ret = syncop_gfid_to_path(healer->this->itable, subvol, loc.gfid,
+                              (char **)&loc.path);
+    if (ret < 0)
+        goto out;
 
-        loc_wipe (&loc);
-        return ret;
-}
+    ret = syncop_inode_find(healer->this, healer->this, loc.gfid, &loc.inode,
+                            NULL, NULL);
+    if (ret < 0)
+        goto out;
 
-int
-ec_shd_selfheal (struct subvol_healer *healer, int child, loc_t *loc)
-{
-        return syncop_getxattr (healer->this, loc, NULL, EC_XATTR_HEAL, NULL,
-                                NULL);
+    ec_shd_selfheal(healer, healer->subvol, &loc, _gf_false);
+out:
+    if (ret == -ENOENT || ret == -ESTALE) {
+        gf_msg(healer->this->name, GF_LOG_DEBUG, 0, EC_MSG_HEAL_FAIL,
+               "Purging index for gfid %s:", uuid_utoa(loc.gfid));
+        ec_shd_index_purge(subvol, parent->inode, entry->d_name);
+    }
+    loc_wipe(&loc);
+
+    return 0;
 }
 
-
 int
-ec_shd_index_heal (xlator_t *subvol, gf_dirent_t *entry, loc_t *parent,
-                   void *data)
+ec_shd_index_sweep(struct subvol_healer *healer)
 {
-        struct subvol_healer *healer = data;
-        ec_t                 *ec = NULL;
-        loc_t                loc = {0};
-        int                  ret     = 0;
-
-        ec = healer->this->private;
-        if (!ec->shd.enabled)
-                return -EBUSY;
-
-        gf_log (healer->this->name, GF_LOG_DEBUG, "got entry: %s",
-                entry->d_name);
-
-        ret = gf_uuid_parse (entry->d_name, loc.gfid);
-        if (ret)
-                return 0;
-
-        /* If this fails with ENOENT/ESTALE index is stale */
-        ret = syncop_gfid_to_path (healer->this->itable, subvol, loc.gfid,
-                                   (char **)&loc.path);
-        if (ret == -ENOENT || ret == -ESTALE) {
-                ec_shd_index_purge (subvol, parent->inode, entry->d_name);
-                goto out;
-        }
-
-        loc.inode = ec_shd_inode_find (healer->this, healer->this, loc.gfid);
-        if (!loc.inode)
-                goto out;
-
-        ec_shd_selfheal (healer, healer->subvol, &loc);
-
+    loc_t loc = {0};
+    ec_t *ec = NULL;
+    int ret = 0;
+    xlator_t *subvol = NULL;
+    dict_t *xdata = NULL;
+
+    ec = healer->this->private;
+    subvol = ec->xl_list[healer->subvol];
+
+    ret = ec_shd_index_inode(healer->this, subvol, &loc.inode);
+    if (ret < 0) {
+        gf_msg(healer->this->name, GF_LOG_WARNING, errno,
+               EC_MSG_INDEX_DIR_GET_FAIL, "unable to get index-dir on %s",
+               subvol->name);
+        goto out;
+    }
+
+    xdata = dict_new();
+    if (!xdata || dict_set_int32(xdata, "get-gfid-type", 1)) {
+        ret = -ENOMEM;
+        goto out;
+    }
+
+    _mask_cancellation();
+    ret = syncop_mt_dir_scan(NULL, subvol, &loc, GF_CLIENT_PID_SELF_HEALD,
+                             healer, ec_shd_index_heal, xdata,
+                             ec->shd.max_threads, ec->shd.wait_qlength);
+    _unmask_cancellation();
 out:
-        loc_wipe (&loc);
+    if (xdata)
+        dict_unref(xdata);
+    loc_wipe(&loc);
 
-        return 0;
+    return ret;
 }
 
 int
-ec_shd_index_sweep (struct subvol_healer *healer)
+ec_shd_full_heal(xlator_t *subvol, gf_dirent_t *entry, loc_t *parent,
+                 void *data)
 {
-        loc_t         loc     = {0};
-        ec_t          *ec     = NULL;
-        int           ret     = 0;
-        xlator_t      *subvol = NULL;
-
-        ec = healer->this->private;
-        subvol = ec->xl_list[healer->subvol];
-
-        loc.inode = ec_shd_index_inode (healer->this, subvol);
-        if (!loc.inode) {
-                gf_log (healer->this->name, GF_LOG_WARNING,
-                        "unable to get index-dir on %s", subvol->name);
-                return -errno;
-        }
-
-        ret = syncop_dir_scan (subvol, &loc, GF_CLIENT_PID_AFR_SELF_HEALD,
-                               healer, ec_shd_index_heal);
+    struct subvol_healer *healer = data;
+    xlator_t *this = healer->this;
+    ec_t *ec = NULL;
+    loc_t loc = {0};
+    int ret = 0;
+
+    ec = this->private;
+
+    if (this->cleanup_starting) {
+        return -ENOTCONN;
+    }
+
+    if (ec->xl_up_count <= ec->fragments) {
+        return -ENOTCONN;
+    }
+    if (!ec->shd.enabled)
+        return -EBUSY;
+
+    if (gf_uuid_is_null(entry->d_stat.ia_gfid)) {
+        /* It's possible that an entry has been removed just after
+         * being seen in a directory but before getting its stat info.
+         * In this case we'll receive a NULL gfid here. Since the file
+         * doesn't exist anymore, we can safely ignore it. */
+        return 0;
+    }
 
-        inode_forget (loc.inode, 1);
-        loc_wipe (&loc);
+    loc.parent = inode_ref(parent->inode);
+    loc.name = entry->d_name;
+    gf_uuid_copy(loc.gfid, entry->d_stat.ia_gfid);
 
-        return ret;
-}
+    /* If this fails with ENOENT/ESTALE index is stale */
+    ret = syncop_gfid_to_path(this->itable, subvol, loc.gfid,
+                              (char **)&loc.path);
+    if (ret < 0)
+        goto out;
 
-int
-ec_shd_full_heal (xlator_t *subvol, gf_dirent_t *entry, loc_t *parent,
-                   void *data)
-{
-        struct subvol_healer *healer = data;
-        xlator_t             *this   = healer->this;
-        ec_t                 *ec     = NULL;
-        loc_t                loc    = {0};
-        int                  ret    = 0;
-
-        ec = this->private;
-        if (!ec->shd.enabled)
-                return -EBUSY;
-
-        loc.parent = inode_ref (parent->inode);
-        loc.name   = entry->d_name;
-        gf_uuid_copy (loc.gfid, entry->d_stat.ia_gfid);
-
-        /* If this fails with ENOENT/ESTALE index is stale */
-        ret = syncop_gfid_to_path (this->itable, subvol, loc.gfid,
-                                   (char **)&loc.path);
-        if (ret < 0)
-                goto out;
-
-        loc.inode = ec_shd_inode_find (this, this, loc.gfid);
-        if (!loc.inode) {
-                ret = -EINVAL;
-                goto out;
-        }
+    ret = syncop_inode_find(this, this, loc.gfid, &loc.inode, NULL, NULL);
+    if (ret < 0)
+        goto out;
 
-        ec_shd_selfheal (healer, healer->subvol, &loc);
+    ec_shd_selfheal(healer, healer->subvol, &loc, _gf_true);
 
-        loc_wipe (&loc);
-        ret = 0;
+    ret = 0;
 
 out:
-        return ret;
+    loc_wipe(&loc);
+    return ret;
 }
 
 int
-ec_shd_full_sweep (struct subvol_healer *healer, inode_t *inode)
+ec_shd_full_sweep(struct subvol_healer *healer, inode_t *inode)
 {
-        ec_t           *ec  = NULL;
-        loc_t          loc  = {0};
-
-        ec = healer->this->private;
-        loc.inode = inode;
-        return syncop_ftw (ec->xl_list[healer->subvol], &loc,
-                           GF_CLIENT_PID_AFR_SELF_HEALD, healer,
-                           ec_shd_full_heal);
+    ec_t *ec = NULL;
+    loc_t loc = {0};
+    int ret = -1;
+
+    ec = healer->this->private;
+    loc.inode = inode;
+    _mask_cancellation();
+    ret = syncop_ftw(ec->xl_list[healer->subvol], &loc,
+                     GF_CLIENT_PID_SELF_HEALD, healer, ec_shd_full_heal);
+    _unmask_cancellation();
+    return ret;
 }
 
-
 void *
-ec_shd_index_healer (void *data)
+ec_shd_index_healer(void *data)
 {
-        struct subvol_healer *healer = NULL;
-        xlator_t *this = NULL;
-
-        healer = data;
-        THIS = this = healer->this;
-
-        for (;;) {
-                ec_shd_healer_wait (healer);
-
-                ASSERT_LOCAL(this, healer);
-
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "starting index sweep on subvol %s",
-                        ec_subvol_name (this, healer->subvol));
-
-                ec_shd_index_sweep (healer);
-
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "finished index sweep on subvol %s",
-                        ec_subvol_name (this, healer->subvol));
+    struct subvol_healer *healer = NULL;
+    xlator_t *this = NULL;
+    int run = 0;
+
+    healer = data;
+    THIS = this = healer->this;
+    ec_t *ec = this->private;
+
+    for (;;) {
+        run = ec_shd_healer_wait(healer);
+        if (run == -1)
+            break;
+
+        if (ec->xl_up_count > ec->fragments) {
+            gf_msg_debug(this->name, 0, "starting index sweep on subvol %s",
+                         ec_subvol_name(this, healer->subvol));
+            ec_shd_index_sweep(healer);
         }
+        gf_msg_debug(this->name, 0, "finished index sweep on subvol %s",
+                     ec_subvol_name(this, healer->subvol));
+    }
 
-        return NULL;
+    return NULL;
 }
 
-
 void *
-ec_shd_full_healer (void *data)
+ec_shd_full_healer(void *data)
 {
-        struct subvol_healer *healer = NULL;
-        xlator_t *this = NULL;
-
-        int run = 0;
-
-        healer = data;
-        THIS = this = healer->this;
-
-        for (;;) {
-                pthread_mutex_lock (&healer->mutex);
-                {
-                        run = __ec_shd_healer_wait (healer);
-                        if (!run)
-                                healer->running = _gf_false;
-                }
-                pthread_mutex_unlock (&healer->mutex);
-
-                if (!run)
-                        break;
-
-                ASSERT_LOCAL(this, healer);
-
-                gf_log (this->name, GF_LOG_INFO,
-                        "starting full sweep on subvol %s",
-                        ec_subvol_name (this, healer->subvol));
+    struct subvol_healer *healer = NULL;
+    xlator_t *this = NULL;
+    loc_t rootloc = {0};
+
+    int run = 0;
+
+    healer = data;
+    THIS = this = healer->this;
+    ec_t *ec = this->private;
+
+    rootloc.inode = this->itable->root;
+    for (;;) {
+        run = ec_shd_healer_wait(healer);
+        if (run < 0) {
+            break;
+        } else if (run == 0) {
+            continue;
+        }
 
-                ec_shd_full_sweep (healer, this->itable->root);
+        if (ec->xl_up_count > ec->fragments) {
+            gf_msg(this->name, GF_LOG_INFO, 0, EC_MSG_FULL_SWEEP_START,
+                   "starting full sweep on subvol %s",
+                   ec_subvol_name(this, healer->subvol));
 
-                gf_log (this->name, GF_LOG_INFO,
-                        "finished full sweep on subvol %s",
-                        ec_subvol_name (this, healer->subvol));
+            ec_shd_selfheal(healer, healer->subvol, &rootloc, _gf_true);
+            ec_shd_full_sweep(healer, this->itable->root);
         }
 
-        return NULL;
-}
+        gf_msg(this->name, GF_LOG_INFO, 0, EC_MSG_FULL_SWEEP_STOP,
+               "finished full sweep on subvol %s",
+               ec_subvol_name(this, healer->subvol));
+    }
 
+    return NULL;
+}
 
 int
-ec_shd_healer_init (xlator_t *this, struct subvol_healer *healer)
+ec_shd_healer_init(xlator_t *this, struct subvol_healer *healer)
 {
-        int ret = 0;
+    int ret = 0;
 
-        ret = pthread_mutex_init (&healer->mutex, NULL);
-        if (ret)
-                goto out;
+    ret = pthread_mutex_init(&healer->mutex, NULL);
+    if (ret)
+        goto out;
 
-        ret = pthread_cond_init (&healer->cond, NULL);
-        if (ret)
-                goto out;
+    ret = pthread_cond_init(&healer->cond, NULL);
+    if (ret)
+        goto out;
 
-        healer->this = this;
-        healer->running = _gf_false;
-        healer->rerun = _gf_false;
-        healer->local = _gf_false;
+    healer->this = this;
+    healer->running = _gf_false;
+    healer->rerun = _gf_false;
 out:
-        return ret;
+    return ret;
 }
 
-
 int
-ec_shd_healer_spawn (xlator_t *this, struct subvol_healer *healer,
-                      void *(threadfn)(void *))
+ec_shd_healer_spawn(xlator_t *this, struct subvol_healer *healer,
+                    void *(threadfn)(void *))
 {
-        int ret = 0;
-
-        pthread_mutex_lock (&healer->mutex);
-        {
-                if (healer->running) {
-                        pthread_cond_signal (&healer->cond);
-                } else {
-                        ret = gf_thread_create (&healer->thread, NULL,
-                                                threadfn, healer);
-                        if (ret)
-                                goto unlock;
-                        healer->running = 1;
-                }
-
-                healer->rerun = 1;
+    int ret = 0;
+
+    pthread_mutex_lock(&healer->mutex);
+    {
+        if (healer->running) {
+            pthread_cond_signal(&healer->cond);
+        } else {
+            ret = gf_thread_create(&healer->thread, NULL, threadfn, healer,
+                                   "ecshd");
+            if (ret)
+                goto unlock;
+            healer->running = 1;
         }
+
+        healer->rerun = 1;
+    }
 unlock:
-        pthread_mutex_unlock (&healer->mutex);
+    pthread_mutex_unlock(&healer->mutex);
 
-        return ret;
+    return ret;
 }
 
 int
-ec_shd_full_healer_spawn (xlator_t *this, int subvol)
+ec_shd_full_healer_spawn(xlator_t *this, int subvol)
 {
-        return ec_shd_healer_spawn (this, NTH_FULL_HEALER (this, subvol),
-                                    ec_shd_full_healer);
-}
+    if (xlator_is_cleanup_starting(this))
+        return -1;
 
+    return ec_shd_healer_spawn(this, NTH_FULL_HEALER(this, subvol),
+                               ec_shd_full_healer);
+}
 
 int
-ec_shd_index_healer_spawn (xlator_t *this, int subvol)
+ec_shd_index_healer_spawn(xlator_t *this, int subvol)
 {
-        return ec_shd_healer_spawn (this, NTH_INDEX_HEALER (this, subvol),
-                                    ec_shd_index_healer);
+    if (xlator_is_cleanup_starting(this))
+        return -1;
+
+    return ec_shd_healer_spawn(this, NTH_INDEX_HEALER(this, subvol),
+                               ec_shd_index_healer);
 }
 
 void
-ec_selfheal_childup (ec_t *ec, int child)
+ec_shd_index_healer_wake(ec_t *ec)
 {
-        if (!ec->shd.iamshd)
-                return;
-        ec_shd_index_healer_spawn (ec->xl, child);
+    int32_t i;
+
+    for (i = 0; i < ec->nodes; i++) {
+        if (((ec->xl_up >> i) & 1) != 0) {
+            ec_shd_index_healer_spawn(ec->xl, i);
+        }
+    }
 }
 
 int
-ec_selfheal_daemon_init (xlator_t *this)
+ec_selfheal_daemon_init(xlator_t *this)
 {
-        ec_t    *ec = NULL;
-        ec_self_heald_t *shd = NULL;
-        int ret = -1;
-        int i = 0;
-
-        ec = this->private;
-        shd = &ec->shd;
-
-        this->itable = inode_table_new (SHD_INODE_LRU_LIMIT, this);
-        if (!this->itable)
-                goto out;
-
-        shd->index_healers = GF_CALLOC (sizeof(*shd->index_healers),
-                                        ec->nodes,
-                                        ec_mt_subvol_healer_t);
-        if (!shd->index_healers)
-                goto out;
-
-        for (i = 0; i < ec->nodes; i++) {
-                shd->index_healers[i].subvol = i;
-                ret = ec_shd_healer_init (this, &shd->index_healers[i]);
-                if (ret)
-                        goto out;
-        }
+    ec_t *ec = NULL;
+    ec_self_heald_t *shd = NULL;
+    int ret = -1;
+    int i = 0;
+
+    ec = this->private;
+    shd = &ec->shd;
+
+    shd->index_healers = GF_CALLOC(sizeof(*shd->index_healers), ec->nodes,
+                                   ec_mt_subvol_healer_t);
+    if (!shd->index_healers)
+        goto out;
+
+    for (i = 0; i < ec->nodes; i++) {
+        shd->index_healers[i].subvol = i;
+        ret = ec_shd_healer_init(this, &shd->index_healers[i]);
+        if (ret)
+            goto out;
+    }
 
-        shd->full_healers = GF_CALLOC (sizeof(*shd->full_healers),
-                                       ec->nodes,
-                                       ec_mt_subvol_healer_t);
-        if (!shd->full_healers)
-                goto out;
-
-        for (i = 0; i < ec->nodes; i++) {
-                shd->full_healers[i].subvol = i;
-                ret = ec_shd_healer_init (this, &shd->full_healers[i]);
-                if (ret)
-                        goto out;
-        }
+    shd->full_healers = GF_CALLOC(sizeof(*shd->full_healers), ec->nodes,
+                                  ec_mt_subvol_healer_t);
+    if (!shd->full_healers)
+        goto out;
 
-        ret = 0;
+    for (i = 0; i < ec->nodes; i++) {
+        shd->full_healers[i].subvol = i;
+        ret = ec_shd_healer_init(this, &shd->full_healers[i]);
+        if (ret)
+            goto out;
+    }
+
+    ret = 0;
 out:
-        return ret;
+    return ret;
 }
 
-
 int
-ec_heal_op (xlator_t *this, dict_t *output, gf_xl_afr_op_t op, int xl_id)
+ec_heal_op(xlator_t *this, dict_t *output, gf_xl_afr_op_t op, int xl_id)
 {
-        char key[64] = {0};
-        int op_ret = 0;
-        ec_t *ec = NULL;
-        int     i = 0;
-        GF_UNUSED int     ret = 0;
-
-        ec = this->private;
-
-        for (i = 0; i < ec->nodes; i++) {
-                snprintf (key, sizeof (key), "%d-%d-status", xl_id, i);
-
-                op_ret = -1;
-                if (((ec->xl_up >> i) & 1) == 0) {
-                        ret = dict_set_str (output, key, "Brick is not connected");
-                } else if (!ec->up) {
-                        ret = dict_set_str (output, key,
-                                            "Disperse subvolume is not up");
-                } else if (!ec_shd_is_subvol_local (this, i)) {
-                        ret = dict_set_str (output, key, "Brick is remote");
-                } else {
-                        ret = dict_set_str (output, key, "Started self-heal");
-                        if (op == GF_SHD_OP_HEAL_FULL) {
-                                ec_shd_full_healer_spawn (this, i);
-                        } else if (op == GF_SHD_OP_HEAL_INDEX) {
-                                ec_shd_index_healer_spawn (this, i);
-                        }
-                        op_ret = 0;
-                }
+    char key[64] = {0};
+    int op_ret = 0;
+    ec_t *ec = NULL;
+    int i = 0;
+    GF_UNUSED int ret = 0;
+
+    ec = this->private;
+
+    op_ret = -1;
+    for (i = 0; i < ec->nodes; i++) {
+        snprintf(key, sizeof(key), "%d-%d-status", xl_id, i);
+
+        if (((ec->xl_up >> i) & 1) == 0) {
+            ret = dict_set_str(output, key, "Brick is not connected");
+        } else if (!ec->up) {
+            ret = dict_set_str(output, key, "Disperse subvolume is not up");
+        } else if (!ec_shd_is_subvol_local(this, i)) {
+            ret = dict_set_str(output, key, "Brick is remote");
+        } else {
+            ret = dict_set_str(output, key, "Started self-heal");
+            if (op == GF_SHD_OP_HEAL_FULL) {
+                ec_shd_full_healer_spawn(this, i);
+            } else if (op == GF_SHD_OP_HEAL_INDEX) {
+                ec_shd_index_healer_spawn(this, i);
+            }
+            op_ret = 0;
         }
-        return op_ret;
+    }
+    return op_ret;
 }
 
 int
-ec_xl_op (xlator_t *this, dict_t *input, dict_t *output)
+ec_xl_op(xlator_t *this, dict_t *input, dict_t *output)
 {
-        gf_xl_afr_op_t op     = GF_SHD_OP_INVALID;
-        int            ret    = 0;
-        int            xl_id  = 0;
+    gf_xl_afr_op_t op = GF_SHD_OP_INVALID;
+    int ret = 0;
+    int xl_id = 0;
 
-        ret = dict_get_int32 (input, "xl-op", (int32_t *)&op);
-        if (ret)
-                goto out;
+    ret = dict_get_int32(input, "xl-op", (int32_t *)&op);
+    if (ret)
+        goto out;
 
-        ret = dict_get_int32 (input, this->name, &xl_id);
-        if (ret)
-                goto out;
+    ret = dict_get_int32(input, this->name, &xl_id);
+    if (ret)
+        goto out;
 
-        ret = dict_set_int32 (output, this->name, xl_id);
-        if (ret)
-                goto out;
+    ret = dict_set_int32(output, this->name, xl_id);
+    if (ret)
+        goto out;
 
-        switch (op) {
+    switch (op) {
         case GF_SHD_OP_HEAL_FULL:
-                ret = ec_heal_op (this, output, op, xl_id);
-                break;
+            ret = ec_heal_op(this, output, op, xl_id);
+            break;
 
         case GF_SHD_OP_HEAL_INDEX:
-                ret = ec_heal_op (this, output, op, xl_id);
-                break;
+            ret = ec_heal_op(this, output, op, xl_id);
+            break;
 
         default:
-                ret = -1;
-                break;
-        }
+            ret = -1;
+            break;
+    }
 out:
-        dict_del (output, this->name);
-        return ret;
+    dict_del(output, this->name);
+    return ret;
+}
+
+void
+ec_destroy_healer_object(xlator_t *this, struct subvol_healer *healer)
+{
+    if (!healer)
+        return;
+
+    pthread_cond_destroy(&healer->cond);
+    pthread_mutex_destroy(&healer->mutex);
+}
+
+void
+ec_selfheal_daemon_fini(xlator_t *this)
+{
+    struct subvol_healer *healer = NULL;
+    ec_self_heald_t *shd = NULL;
+    ec_t *priv = NULL;
+    int i = 0;
+
+    priv = this->private;
+    if (!priv)
+        return;
+
+    shd = &priv->shd;
+    if (!shd->iamshd)
+        return;
+
+    for (i = 0; i < priv->nodes; i++) {
+        healer = &shd->index_healers[i];
+        ec_destroy_healer_object(this, healer);
+
+        healer = &shd->full_healers[i];
+        ec_destroy_healer_object(this, healer);
+    }
+
+    GF_FREE(shd->index_healers);
+    GF_FREE(shd->full_healers);
 }
diff --git a/xlators/cluster/ec/src/ec-heald.h b/xlators/cluster/ec/src/ec-heald.h
index 0f27a8ec776..6c7da4edc10 100644
--- a/xlators/cluster/ec/src/ec-heald.h
+++ b/xlators/cluster/ec/src/ec-heald.h
@@ -11,37 +11,20 @@
 #ifndef __EC_HEALD_H__
 #define __EC_HEALD_H__
 
-#include "xlator.h"
-
-struct _ec;
-typedef struct _ec ec_t;
-
-struct subvol_healer {
-        xlator_t        *this;
-        int              subvol;
-        gf_boolean_t     local;
-        gf_boolean_t     running;
-        gf_boolean_t     rerun;
-        pthread_mutex_t  mutex;
-        pthread_cond_t   cond;
-        pthread_t        thread;
-};
-
-struct _ec_self_heald;
-typedef struct _ec_self_heald ec_self_heald_t;
-
-struct _ec_self_heald {
-        gf_boolean_t            iamshd;
-        gf_boolean_t            enabled;
-        int                     timeout;
-        struct subvol_healer   *index_healers;
-        struct subvol_healer   *full_healers;
-};
+#include "ec-types.h"           // for ec_t
+#include "glusterfs/dict.h"     // for dict_t
+#include "glusterfs/globals.h"  // for xlator_t
 
 int
-ec_xl_op (xlator_t *this, dict_t *input, dict_t *output);
+ec_xl_op(xlator_t *this, dict_t *input, dict_t *output);
 
 int
-ec_selfheal_daemon_init (xlator_t *this);
-void ec_selfheal_childup (ec_t *ec, int child);
+ec_selfheal_daemon_init(xlator_t *this);
+
+void
+ec_shd_index_healer_wake(ec_t *ec);
+
+void
+ec_selfheal_daemon_fini(xlator_t *this);
+
 #endif /* __EC_HEALD_H__ */
diff --git a/xlators/cluster/ec/src/ec-helpers.c b/xlators/cluster/ec/src/ec-helpers.c
index 65deba63959..48f54475e01 100644
--- a/xlators/cluster/ec/src/ec-helpers.c
+++ b/xlators/cluster/ec/src/ec-helpers.c
@@ -10,35 +10,29 @@
 
 #include <libgen.h>
 
-#include "byte-order.h"
+#include <glusterfs/byte-order.h>
 
+#include "ec.h"
 #include "ec-mem-types.h"
+#include "ec-messages.h"
 #include "ec-fops.h"
+#include "ec-method.h"
 #include "ec-helpers.h"
 
-#ifndef ffsll
-#define ffsll(x) __builtin_ffsll(x)
-#endif
+static const char *ec_fop_list[] = {[-EC_FOP_HEAL] = "HEAL"};
 
-static const char * ec_fop_list[] =
-{
-    [-EC_FOP_HEAL] = "HEAL"
-};
-
-const char * ec_bin(char * str, size_t size, uint64_t value, int32_t digits)
+const char *
+ec_bin(char *str, size_t size, uint64_t value, int32_t digits)
 {
     str += size;
 
-    if (size-- < 1)
-    {
+    if (size-- < 1) {
         goto failed;
     }
     *--str = 0;
 
-    while ((value != 0) || (digits > 0))
-    {
-        if (size-- < 1)
-        {
+    while ((value != 0) || (digits > 0)) {
+        if (size-- < 1) {
             goto failed;
         }
         *--str = '0' + (value & 1);
@@ -52,21 +46,22 @@ failed:
     return "<buffer too small>";
 }
 
-const char * ec_fop_name(int32_t id)
+const char *
+ec_fop_name(int32_t id)
 {
-    if (id >= 0)
-    {
+    if (id >= 0) {
         return gf_fop_list[id];
     }
 
     return ec_fop_list[-id];
 }
 
-void ec_trace(const char * event, ec_fop_data_t * fop, const char * fmt, ...)
+void
+ec_trace(const char *event, ec_fop_data_t *fop, const char *fmt, ...)
 {
     char str1[32], str2[32], str3[32];
-    char * msg;
-    ec_t * ec = fop->xl->private;
+    char *msg;
+    ec_t *ec = fop->xl->private;
     va_list args;
     int32_t ret;
 
@@ -74,45 +69,28 @@ void ec_trace(const char * event, ec_fop_data_t * fop, const char * fmt, ...)
     ret = vasprintf(&msg, fmt, args);
     va_end(args);
 
-    if (ret < 0)
-    {
+    if (ret < 0) {
         msg = "<memory allocation error>";
     }
 
-    gf_log("ec", GF_LOG_TRACE, "%s(%s) %p(%p) [refs=%d, winds=%d, jobs=%d] "
-                               "frame=%p/%p, min/exp=%d/%d, err=%d state=%d "
-                               "{%s:%s:%s} %s",
-           event, ec_fop_name(fop->id), fop, fop->parent, fop->refs,
-           fop->winds, fop->jobs, fop->req_frame, fop->frame, fop->minimum,
-           fop->expected, fop->error, fop->state,
-           ec_bin(str1, sizeof(str1), fop->mask, ec->nodes),
-           ec_bin(str2, sizeof(str2), fop->remaining, ec->nodes),
-           ec_bin(str3, sizeof(str3), fop->bad, ec->nodes), msg);
+    gf_msg_trace("ec", 0,
+                 "%s(%s) %p(%p) [refs=%d, winds=%d, jobs=%d] "
+                 "frame=%p/%p, min/exp=%d/%d, err=%d state=%d "
+                 "{%s:%s:%s} %s",
+                 event, ec_fop_name(fop->id), fop, fop->parent, fop->refs,
+                 fop->winds, fop->jobs, fop->req_frame, fop->frame,
+                 fop->minimum, fop->expected, fop->error, fop->state,
+                 ec_bin(str1, sizeof(str1), fop->mask, ec->nodes),
+                 ec_bin(str2, sizeof(str2), fop->remaining, ec->nodes),
+                 ec_bin(str3, sizeof(str3), fop->good, ec->nodes), msg);
 
-    if (ret >= 0)
-    {
+    if (ret >= 0) {
         free(msg);
     }
 }
 
-int32_t ec_bits_count(uint64_t n)
-{
-    n -= (n >> 1) & 0x5555555555555555ULL;
-    n = ((n >> 2) & 0x3333333333333333ULL) + (n & 0x3333333333333333ULL);
-    n = (n + (n >> 4)) & 0x0F0F0F0F0F0F0F0FULL;
-    n += n >> 8;
-    n += n >> 16;
-    n += n >> 32;
-
-    return n & 0xFF;
-}
-
-int32_t ec_bits_index(uint64_t n)
-{
-    return ffsll(n) - 1;
-}
-
-int32_t ec_bits_consume(uint64_t * n)
+int32_t
+ec_bits_consume(uint64_t *n)
 {
     uint64_t tmp;
 
@@ -120,24 +98,21 @@ int32_t ec_bits_consume(uint64_t * n)
     tmp &= -tmp;
     *n ^= tmp;
 
-    return ffsll(tmp) - 1;
+    return gf_bits_index(tmp);
 }
 
-size_t ec_iov_copy_to(void * dst, struct iovec * vector, int32_t count,
-                      off_t offset, size_t size)
+size_t
+ec_iov_copy_to(void *dst, struct iovec *vector, int32_t count, off_t offset,
+               size_t size)
 {
     int32_t i = 0;
     size_t total = 0, len = 0;
 
-    while (i < count)
-    {
-        if (offset < vector[i].iov_len)
-        {
-            while ((i < count) && (size > 0))
-            {
+    while (i < count) {
+        if (offset < vector[i].iov_len) {
+            while ((i < count) && (size > 0)) {
                 len = size;
-                if (len > vector[i].iov_len - offset)
-                {
+                if (len > vector[i].iov_len - offset) {
                     len = vector[i].iov_len - offset;
                 }
                 memcpy(dst, vector[i++].iov_base + offset, len);
@@ -157,30 +132,161 @@ size_t ec_iov_copy_to(void * dst, struct iovec * vector, int32_t count,
     return total;
 }
 
-int32_t ec_dict_set_number(dict_t * dict, char * key, uint64_t value)
+int32_t
+ec_buffer_alloc(xlator_t *xl, size_t size, struct iobref **piobref, void **ptr)
 {
-    uint64_t * ptr;
+    struct iobref *iobref = NULL;
+    struct iobuf *iobuf = NULL;
+    int32_t ret = -ENOMEM;
+
+    iobuf = iobuf_get_page_aligned(xl->ctx->iobuf_pool, size,
+                                   EC_METHOD_WORD_SIZE);
+    if (iobuf == NULL) {
+        goto out;
+    }
+
+    iobref = *piobref;
+    if (iobref == NULL) {
+        iobref = iobref_new();
+        if (iobref == NULL) {
+            goto out;
+        }
+    }
+
+    ret = iobref_add(iobref, iobuf);
+    if (ret != 0) {
+        if (iobref != *piobref) {
+            iobref_unref(iobref);
+        }
+        iobref = NULL;
+
+        goto out;
+    }
+
+    GF_ASSERT(EC_ALIGN_CHECK(iobuf->ptr, EC_METHOD_WORD_SIZE));
+
+    *ptr = iobuf->ptr;
+
+out:
+    if (iobuf != NULL) {
+        iobuf_unref(iobuf);
+    }
+
+    if (iobref != NULL) {
+        *piobref = iobref;
+    }
+
+    return ret;
+}
+
+int32_t
+ec_dict_set_array(dict_t *dict, char *key, uint64_t value[], int32_t size)
+{
+    int ret = -1;
+    uint64_t *ptr = NULL;
+    int32_t vindex;
+
+    if (value == NULL) {
+        return -EINVAL;
+    }
+
+    ptr = GF_MALLOC(sizeof(uint64_t) * size, gf_common_mt_char);
+    if (ptr == NULL) {
+        return -ENOMEM;
+    }
+    for (vindex = 0; vindex < size; vindex++) {
+        ptr[vindex] = hton64(value[vindex]);
+    }
+    ret = dict_set_bin(dict, key, ptr, sizeof(uint64_t) * size);
+    if (ret)
+        GF_FREE(ptr);
+    return ret;
+}
+
+int32_t
+ec_dict_get_array(dict_t *dict, char *key, uint64_t value[], int32_t size)
+{
+    void *ptr;
+    int32_t len;
+    int32_t vindex;
+    int32_t old_size = 0;
+    int32_t err;
+
+    if (dict == NULL) {
+        return -EINVAL;
+    }
+    err = dict_get_ptr_and_len(dict, key, &ptr, &len);
+    if (err != 0) {
+        return err;
+    }
+
+    if (len > (size * sizeof(uint64_t)) || (len % sizeof(uint64_t))) {
+        return -EINVAL;
+    }
+
+    /* 3.6 version ec would have stored version in 64 bit. In that case treat
+     * metadata versions same as data*/
+    old_size = min(size, len / sizeof(uint64_t));
+    for (vindex = 0; vindex < old_size; vindex++) {
+        value[vindex] = ntoh64(*((uint64_t *)ptr + vindex));
+    }
+
+    if (old_size < size) {
+        for (vindex = old_size; vindex < size; vindex++) {
+            value[vindex] = value[old_size - 1];
+        }
+    }
+
+    return 0;
+}
+
+int32_t
+ec_dict_del_array(dict_t *dict, char *key, uint64_t value[], int32_t size)
+{
+    int ret = 0;
+
+    ret = ec_dict_get_array(dict, key, value, size);
+    if (ret == 0)
+        dict_del(dict, key);
+
+    return ret;
+}
+
+int32_t
+ec_dict_set_number(dict_t *dict, char *key, uint64_t value)
+{
+    int ret = -1;
+    uint64_t *ptr;
 
     ptr = GF_MALLOC(sizeof(value), gf_common_mt_char);
-    if (ptr == NULL)
-    {
-        return -1;
+    if (ptr == NULL) {
+        return -ENOMEM;
     }
 
     *ptr = hton64(value);
 
-    return dict_set_bin(dict, key, ptr, sizeof(value));
+    ret = dict_set_bin(dict, key, ptr, sizeof(value));
+    if (ret)
+        GF_FREE(ptr);
+
+    return ret;
 }
 
-int32_t ec_dict_del_number(dict_t * dict, char * key, uint64_t * value)
+int32_t
+ec_dict_del_number(dict_t *dict, char *key, uint64_t *value)
 {
-    void * ptr;
-    int32_t len;
+    void *ptr;
+    int32_t len, err;
 
-    if ((dict == NULL) || (dict_get_ptr_and_len(dict, key, &ptr, &len) != 0) ||
-        (len != sizeof(uint64_t)))
-    {
-        return -1;
+    if (dict == NULL) {
+        return -EINVAL;
+    }
+    err = dict_get_ptr_and_len(dict, key, &ptr, &len);
+    if (err != 0) {
+        return err;
+    }
+    if (len != sizeof(uint64_t)) {
+        return -EINVAL;
     }
 
     *value = ntoh64(*(uint64_t *)ptr);
@@ -190,22 +296,24 @@ int32_t ec_dict_del_number(dict_t * dict, char * key, uint64_t * value)
     return 0;
 }
 
-int32_t ec_dict_set_config(dict_t * dict, char * key, ec_config_t * config)
+int32_t
+ec_dict_set_config(dict_t *dict, char *key, ec_config_t *config)
 {
-    uint64_t * ptr, data;
+    int ret = -1;
+    uint64_t *ptr, data;
 
-    if (config->version > EC_CONFIG_VERSION)
-    {
-        gf_log("ec", GF_LOG_ERROR, "Trying to store an unsupported config "
-                                   "version (%u)", config->version);
+    if (config->version > EC_CONFIG_VERSION) {
+        gf_msg("ec", GF_LOG_ERROR, EINVAL, EC_MSG_UNSUPPORTED_VERSION,
+               "Trying to store an unsupported config "
+               "version (%u)",
+               config->version);
 
-        return -1;
+        return -EINVAL;
     }
 
     ptr = GF_MALLOC(sizeof(uint64_t), gf_common_mt_char);
-    if (ptr == NULL)
-    {
-        return -1;
+    if (ptr == NULL) {
+        return -ENOMEM;
     }
 
     data = ((uint64_t)config->version) << 56;
@@ -217,30 +325,51 @@ int32_t ec_dict_set_config(dict_t * dict, char * key, ec_config_t * config)
 
     *ptr = hton64(data);
 
-    return dict_set_bin(dict, key, ptr, sizeof(uint64_t));
+    ret = dict_set_bin(dict, key, ptr, sizeof(uint64_t));
+    if (ret)
+        GF_FREE(ptr);
+
+    return ret;
 }
 
-int32_t ec_dict_del_config(dict_t * dict, char * key, ec_config_t * config)
+int32_t
+ec_dict_del_config(dict_t *dict, char *key, ec_config_t *config)
 {
-    void * ptr;
+    void *ptr;
     uint64_t data;
-    int32_t len;
+    int32_t len, err;
 
-    if ((dict == NULL) || (dict_get_ptr_and_len(dict, key, &ptr, &len) != 0) ||
-        (len != sizeof(uint64_t)))
-    {
-        return -1;
+    if (dict == NULL) {
+        return -EINVAL;
+    }
+    err = dict_get_ptr_and_len(dict, key, &ptr, &len);
+    if (err != 0) {
+        return err;
+    }
+    if (len != sizeof(uint64_t)) {
+        return -EINVAL;
     }
 
     data = ntoh64(*(uint64_t *)ptr);
+    /* Currently we need to get the config xattr for entries of type IA_INVAL.
+     * These entries can later become IA_DIR entries (after inode_link()),
+     * which don't have a config xattr. However, since the xattr is requested
+     * using an xattrop() fop, it will always return a config full of 0's
+     * instead of saying that it doesn't exist.
+     *
+     * We need to filter out this case and consider that a config xattr == 0 is
+     * the same as a non-existent xattr. Otherwise ec_config_check() will fail.
+     */
+    if (data == 0) {
+        return -ENODATA;
+    }
 
     config->version = (data >> 56) & 0xff;
-    if (config->version > EC_CONFIG_VERSION)
-    {
-        gf_log("ec", GF_LOG_ERROR, "Found an unsupported config version (%u)",
-               config->version);
+    if (config->version > EC_CONFIG_VERSION) {
+        gf_msg("ec", GF_LOG_ERROR, EINVAL, EC_MSG_UNSUPPORTED_VERSION,
+               "Found an unsupported config version (%u)", config->version);
 
-        return -1;
+        return -EINVAL;
     }
 
     config->algorithm = (data >> 48) & 0xff;
@@ -254,43 +383,43 @@ int32_t ec_dict_del_config(dict_t * dict, char * key, ec_config_t * config)
     return 0;
 }
 
-int32_t ec_loc_gfid_check(xlator_t * xl, uuid_t dst, uuid_t src)
+gf_boolean_t
+ec_loc_gfid_check(xlator_t *xl, uuid_t dst, uuid_t src)
 {
-    if (gf_uuid_is_null(src))
-    {
-        return 1;
+    if (gf_uuid_is_null(src)) {
+        return _gf_true;
     }
 
-    if (gf_uuid_is_null(dst))
-    {
+    if (gf_uuid_is_null(dst)) {
         gf_uuid_copy(dst, src);
 
-        return 1;
+        return _gf_true;
     }
 
-    if (gf_uuid_compare(dst, src) != 0)
-    {
-        gf_log(xl->name, GF_LOG_WARNING, "Mismatching GFID's in loc");
+    if (gf_uuid_compare(dst, src) != 0) {
+        gf_msg(xl->name, GF_LOG_WARNING, 0, EC_MSG_GFID_MISMATCH,
+               "Mismatching GFID's in loc");
 
-        return 0;
+        return _gf_false;
     }
 
-    return 1;
+    return _gf_true;
 }
 
-int32_t ec_loc_setup_inode(xlator_t *xl, loc_t *loc)
+int32_t
+ec_loc_setup_inode(xlator_t *xl, inode_table_t *table, loc_t *loc)
 {
-    int32_t ret = -1;
+    int32_t ret = -EINVAL;
 
     if (loc->inode != NULL) {
         if (!ec_loc_gfid_check(xl, loc->gfid, loc->inode->gfid)) {
             goto out;
         }
-    } else if (loc->parent != NULL) {
+    } else if (table != NULL) {
         if (!gf_uuid_is_null(loc->gfid)) {
-            loc->inode = inode_find(loc->parent->table, loc->gfid);
-        } else if (loc->path && strchr (loc->path, '/')) {
-            loc->inode = inode_resolve(loc->parent->table, (char *)loc->path);
+            loc->inode = inode_find(table, loc->gfid);
+        } else if (loc->path && strchr(loc->path, '/')) {
+            loc->inode = inode_resolve(table, (char *)loc->path);
         }
     }
 
@@ -300,49 +429,65 @@ out:
     return ret;
 }
 
-int32_t ec_loc_setup_parent(xlator_t *xl, loc_t *loc)
+int32_t
+ec_loc_setup_parent(xlator_t *xl, inode_table_t *table, loc_t *loc)
 {
     char *path, *parent;
-    int32_t ret = -1;
+    int32_t ret = -EINVAL;
 
     if (loc->parent != NULL) {
         if (!ec_loc_gfid_check(xl, loc->pargfid, loc->parent->gfid)) {
             goto out;
         }
-    } else if (loc->inode != NULL) {
+    } else if (table != NULL) {
         if (!gf_uuid_is_null(loc->pargfid)) {
-            loc->parent = inode_find(loc->inode->table, loc->pargfid);
-        } else if (loc->path && strchr (loc->path, '/')) {
+            loc->parent = inode_find(table, loc->pargfid);
+        } else if (loc->path && strchr(loc->path, '/')) {
             path = gf_strdup(loc->path);
             if (path == NULL) {
-                gf_log(xl->name, GF_LOG_ERROR, "Unable to duplicate path '%s'",
-                       loc->path);
+                gf_msg(xl->name, GF_LOG_ERROR, ENOMEM, EC_MSG_NO_MEMORY,
+                       "Unable to duplicate path '%s'", loc->path);
+
+                ret = -ENOMEM;
 
                 goto out;
             }
             parent = dirname(path);
-            loc->parent = inode_resolve(loc->inode->table, parent);
+            loc->parent = inode_resolve(table, parent);
+            if (loc->parent != NULL) {
+                gf_uuid_copy(loc->pargfid, loc->parent->gfid);
+            }
             GF_FREE(path);
         }
     }
 
+    /* If 'pargfid' has not been determined, clear 'name' to avoid resolutions
+       based on <gfid:pargfid>/name. */
+    if (gf_uuid_is_null(loc->pargfid)) {
+        loc->name = NULL;
+    }
+
     ret = 0;
 
 out:
     return ret;
 }
 
-int32_t ec_loc_setup_path(xlator_t *xl, loc_t *loc)
+int32_t
+ec_loc_setup_path(xlator_t *xl, loc_t *loc)
 {
-    uuid_t root = {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1};
+    static uuid_t root = {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1};
     char *name;
-    int32_t ret = -1;
+    int32_t ret = -EINVAL;
 
     if (loc->path != NULL) {
         name = strrchr(loc->path, '/');
         if (name == NULL) {
-                ret = 0; /*<gfid:gfid-str>*/
-                goto out;
+            /* Allow gfid paths: <gfid:...> */
+            if (strncmp(loc->path, "<gfid:", 6) == 0) {
+                ret = 0;
+            }
+            goto out;
         }
         if (name == loc->path) {
             if (name[1] == 0) {
@@ -359,8 +504,8 @@ int32_t ec_loc_setup_path(xlator_t *xl, loc_t *loc)
 
         if (loc->name != NULL) {
             if (strcmp(loc->name, name) != 0) {
-                gf_log(xl->name, GF_LOG_ERROR, "Invalid name '%s' in loc",
-                       loc->name);
+                gf_msg(xl->name, GF_LOG_ERROR, EINVAL, EC_MSG_INVALID_LOC_NAME,
+                       "Invalid name '%s' in loc", loc->name);
 
                 goto out;
             }
@@ -375,45 +520,58 @@ out:
     return ret;
 }
 
-int32_t ec_loc_parent(xlator_t *xl, loc_t *loc, loc_t *parent)
+int32_t
+ec_loc_parent(xlator_t *xl, loc_t *loc, loc_t *parent)
 {
+    inode_table_t *table = NULL;
     char *str = NULL;
-    int32_t ret = -1;
+    int32_t ret = -ENOMEM;
 
     memset(parent, 0, sizeof(loc_t));
 
     if (loc->parent != NULL) {
+        table = loc->parent->table;
         parent->inode = inode_ref(loc->parent);
+    } else if (loc->inode != NULL) {
+        table = loc->inode->table;
     }
     if (!gf_uuid_is_null(loc->pargfid)) {
         gf_uuid_copy(parent->gfid, loc->pargfid);
     }
-    if (loc->path && strchr (loc->path, '/')) {
+    if (loc->path && strchr(loc->path, '/')) {
         str = gf_strdup(loc->path);
         if (str == NULL) {
-            gf_log(xl->name, GF_LOG_ERROR, "Unable to duplicate path '%s'",
-                   loc->path);
+            gf_msg(xl->name, GF_LOG_ERROR, ENOMEM, EC_MSG_NO_MEMORY,
+                   "Unable to duplicate path '%s'", loc->path);
 
             goto out;
         }
         parent->path = gf_strdup(dirname(str));
         if (parent->path == NULL) {
-            gf_log(xl->name, GF_LOG_ERROR, "Unable to duplicate path '%s'",
-                   dirname(str));
+            gf_msg(xl->name, GF_LOG_ERROR, ENOMEM, EC_MSG_NO_MEMORY,
+                   "Unable to duplicate path '%s'", dirname(str));
 
             goto out;
         }
     }
 
-    if ((ec_loc_setup_path(xl, parent) != 0) ||
-        (ec_loc_setup_inode(xl, parent) != 0) ||
-        (ec_loc_setup_parent(xl, parent) != 0)) {
+    ret = ec_loc_setup_path(xl, parent);
+    if (ret == 0) {
+        ret = ec_loc_setup_inode(xl, table, parent);
+    }
+    if (ret == 0) {
+        ret = ec_loc_setup_parent(xl, table, parent);
+    }
+    if (ret != 0) {
         goto out;
     }
 
     if ((parent->inode == NULL) && (parent->path == NULL) &&
         gf_uuid_is_null(parent->gfid)) {
-        gf_log(xl->name, GF_LOG_ERROR, "Parent inode missing for loc_t");
+        gf_msg(xl->name, GF_LOG_ERROR, EINVAL, EC_MSG_LOC_PARENT_INODE_MISSING,
+               "Parent inode missing for loc_t");
+
+        ret = -EINVAL;
 
         goto out;
     }
@@ -423,25 +581,32 @@ int32_t ec_loc_parent(xlator_t *xl, loc_t *loc, loc_t *parent)
 out:
     GF_FREE(str);
 
-    if (ret != 0)
-    {
+    if (ret != 0) {
         loc_wipe(parent);
     }
 
     return ret;
 }
 
-int32_t ec_loc_update(xlator_t *xl, loc_t *loc, inode_t *inode,
-                      struct iatt *iatt)
+int32_t
+ec_loc_update(xlator_t *xl, loc_t *loc, inode_t *inode, struct iatt *iatt)
 {
-    int32_t ret = -1;
-
-    if ((inode != NULL) && (loc->inode != inode)) {
-        if (loc->inode != NULL) {
-            inode_unref(loc->inode);
+    inode_table_t *table = NULL;
+    int32_t ret = -EINVAL;
+
+    if (inode != NULL) {
+        table = inode->table;
+        if (loc->inode != inode) {
+            if (loc->inode != NULL) {
+                inode_unref(loc->inode);
+            }
+            loc->inode = inode_ref(inode);
+            gf_uuid_copy(loc->gfid, inode->gfid);
         }
-        loc->inode = inode_ref(inode);
-        gf_uuid_copy(loc->gfid, inode->gfid);
+    } else if (loc->inode != NULL) {
+        table = loc->inode->table;
+    } else if (loc->parent != NULL) {
+        table = loc->parent->table;
     }
 
     if (iatt != NULL) {
@@ -450,22 +615,26 @@ int32_t ec_loc_update(xlator_t *xl, loc_t *loc, inode_t *inode,
         }
     }
 
-    if ((ec_loc_setup_path(xl, loc) != 0) ||
-        (ec_loc_setup_inode(xl, loc) != 0) ||
-        (ec_loc_setup_parent(xl, loc) != 0)) {
+    ret = ec_loc_setup_path(xl, loc);
+    if (ret == 0) {
+        ret = ec_loc_setup_inode(xl, table, loc);
+    }
+    if (ret == 0) {
+        ret = ec_loc_setup_parent(xl, table, loc);
+    }
+    if (ret != 0) {
         goto out;
     }
 
-    ret = 0;
-
 out:
     return ret;
 }
 
-int32_t ec_loc_from_fd(xlator_t * xl, loc_t * loc, fd_t * fd)
+int32_t
+ec_loc_from_fd(xlator_t *xl, loc_t *loc, fd_t *fd)
 {
-    ec_fd_t * ctx;
-    int32_t ret = -1;
+    ec_fd_t *ctx;
+    int32_t ret = -ENOMEM;
 
     memset(loc, 0, sizeof(*loc));
 
@@ -476,12 +645,11 @@ int32_t ec_loc_from_fd(xlator_t * xl, loc_t * loc, fd_t * fd)
         }
     }
 
-    if (ec_loc_update(xl, loc, fd->inode, NULL) != 0) {
+    ret = ec_loc_update(xl, loc, fd->inode, NULL);
+    if (ret != 0) {
         goto out;
     }
 
-    ret = 0;
-
 out:
     if (ret != 0) {
         loc_wipe(loc);
@@ -490,9 +658,10 @@ out:
     return ret;
 }
 
-int32_t ec_loc_from_loc(xlator_t * xl, loc_t * dst, loc_t * src)
+int32_t
+ec_loc_from_loc(xlator_t *xl, loc_t *dst, loc_t *src)
 {
-    int32_t ret = -1;
+    int32_t ret = -ENOMEM;
 
     memset(dst, 0, sizeof(*dst));
 
@@ -500,12 +669,11 @@ int32_t ec_loc_from_loc(xlator_t * xl, loc_t * dst, loc_t * src)
         goto out;
     }
 
-    if (ec_loc_update(xl, dst, NULL, NULL) != 0) {
+    ret = ec_loc_update(xl, dst, NULL, NULL);
+    if (ret != 0) {
         goto out;
     }
 
-    ret = 0;
-
 out:
     if (ret != 0) {
         loc_wipe(dst);
@@ -514,50 +682,62 @@ out:
     return ret;
 }
 
-void ec_owner_set(call_frame_t * frame, void * owner)
+void
+ec_owner_set(call_frame_t *frame, void *owner)
 {
     set_lk_owner_from_ptr(&frame->root->lk_owner, owner);
 }
 
-void ec_owner_copy(call_frame_t * frame, gf_lkowner_t * owner)
+void
+ec_owner_copy(call_frame_t *frame, gf_lkowner_t *owner)
 {
-    frame->root->lk_owner.len = owner->len;
-    memcpy(frame->root->lk_owner.data, owner->data, owner->len);
+    lk_owner_copy(&frame->root->lk_owner, owner);
 }
 
-ec_inode_t * __ec_inode_get(inode_t * inode, xlator_t * xl)
+static void
+ec_stripe_cache_init(ec_t *ec, ec_inode_t *ctx)
 {
-    ec_inode_t * ctx = NULL;
+    ec_stripe_list_t *stripe_cache = NULL;
+
+    stripe_cache = &(ctx->stripe_cache);
+    if (stripe_cache->max == 0) {
+        stripe_cache->max = ec->stripe_cache;
+    }
+}
+
+ec_inode_t *
+__ec_inode_get(inode_t *inode, xlator_t *xl)
+{
+    ec_inode_t *ctx = NULL;
     uint64_t value = 0;
 
-    if ((__inode_ctx_get(inode, xl, &value) != 0) || (value == 0))
-    {
+    if ((__inode_ctx_get(inode, xl, &value) != 0) || (value == 0)) {
         ctx = GF_MALLOC(sizeof(*ctx), ec_mt_ec_inode_t);
-        if (ctx != NULL)
-        {
+        if (ctx != NULL) {
             memset(ctx, 0, sizeof(*ctx));
             INIT_LIST_HEAD(&ctx->heal);
-
+            INIT_LIST_HEAD(&ctx->stripe_cache.lru);
+            ctx->heal_count = 0;
             value = (uint64_t)(uintptr_t)ctx;
-            if (__inode_ctx_set(inode, xl, &value) != 0)
-            {
+            if (__inode_ctx_set(inode, xl, &value) != 0) {
                 GF_FREE(ctx);
 
                 return NULL;
             }
         }
-    }
-    else
-    {
+    } else {
         ctx = (ec_inode_t *)(uintptr_t)value;
     }
+    if (ctx)
+        ec_stripe_cache_init(xl->private, ctx);
 
     return ctx;
 }
 
-ec_inode_t * ec_inode_get(inode_t * inode, xlator_t * xl)
+ec_inode_t *
+ec_inode_get(inode_t *inode, xlator_t *xl)
 {
-    ec_inode_t * ctx = NULL;
+    ec_inode_t *ctx = NULL;
 
     LOCK(&inode->lock);
 
@@ -568,34 +748,47 @@ ec_inode_t * ec_inode_get(inode_t * inode, xlator_t * xl)
     return ctx;
 }
 
-ec_fd_t * __ec_fd_get(fd_t * fd, xlator_t * xl)
+ec_fd_t *
+__ec_fd_get(fd_t *fd, xlator_t *xl)
 {
-    ec_fd_t * ctx = NULL;
+    int i = 0;
+    ec_fd_t *ctx = NULL;
+    ec_inode_t *ictx = NULL;
     uint64_t value = 0;
+    ec_t *ec = xl->private;
 
-    if ((__fd_ctx_get(fd, xl, &value) != 0) || (value == 0))
-    {
-        ctx = GF_MALLOC(sizeof(*ctx), ec_mt_ec_fd_t);
-        if (ctx != NULL)
-        {
+    if ((__fd_ctx_get(fd, xl, &value) != 0) || (value == 0)) {
+        ctx = GF_MALLOC(sizeof(*ctx) + (sizeof(ec_fd_status_t) * ec->nodes),
+                        ec_mt_ec_fd_t);
+        if (ctx != NULL) {
             memset(ctx, 0, sizeof(*ctx));
 
+            for (i = 0; i < ec->nodes; i++) {
+                if (fd_is_anonymous(fd)) {
+                    ctx->fd_status[i] = EC_FD_OPENED;
+                } else {
+                    ctx->fd_status[i] = EC_FD_NOT_OPENED;
+                }
+            }
+
             value = (uint64_t)(uintptr_t)ctx;
-            if (__fd_ctx_set(fd, xl, value) != 0)
-            {
+            if (__fd_ctx_set(fd, xl, value) != 0) {
                 GF_FREE(ctx);
-
                 return NULL;
             }
+            /* Only refering bad-version so no need for lock
+             * */
+            ictx = __ec_inode_get(fd->inode, xl);
+            if (ictx) {
+                ctx->bad_version = ictx->bad_version;
+            }
         }
-    }
-    else
-    {
+    } else {
         ctx = (ec_fd_t *)(uintptr_t)value;
     }
 
     /* Treat anonymous fd specially */
-    if (fd->anonymous) {
+    if (fd->anonymous && ctx) {
         /* Mark the fd open for all subvolumes. */
         ctx->open = -1;
         /* Try to populate ctx->loc with fd->inode information. */
@@ -605,9 +798,10 @@ ec_fd_t * __ec_fd_get(fd_t * fd, xlator_t * xl)
     return ctx;
 }
 
-ec_fd_t * ec_fd_get(fd_t * fd, xlator_t * xl)
+ec_fd_t *
+ec_fd_get(fd_t *fd, xlator_t *xl)
 {
-    ec_fd_t * ctx = NULL;
+    ec_fd_t *ctx = NULL;
 
     LOCK(&fd->lock);
 
@@ -618,48 +812,56 @@ ec_fd_t * ec_fd_get(fd_t * fd, xlator_t * xl)
     return ctx;
 }
 
-uint32_t ec_adjust_offset(ec_t * ec, off_t * offset, int32_t scale)
+gf_boolean_t
+ec_is_internal_xattr(dict_t *dict, char *key, data_t *value, void *data)
 {
-    off_t head, tmp;
-
-    tmp = *offset;
-    head = tmp % ec->stripe_size;
-    tmp -= head;
-    if (scale)
-    {
-        tmp /= ec->fragments;
-    }
-
-    *offset = tmp;
+    if (key && (strncmp(key, EC_XATTR_PREFIX, SLEN(EC_XATTR_PREFIX)) == 0))
+        return _gf_true;
 
-    return head;
+    return _gf_false;
 }
 
-uint64_t ec_adjust_size(ec_t * ec, uint64_t size, int32_t scale)
+void
+ec_filter_internal_xattrs(dict_t *xattr)
 {
-    size += ec->stripe_size - 1;
-    size -= size % ec->stripe_size;
-    if (scale)
-    {
-        size /= ec->fragments;
-    }
-
-    return size;
+    dict_foreach_match(xattr, ec_is_internal_xattr, NULL,
+                       dict_remove_foreach_fn, NULL);
 }
 
 gf_boolean_t
-ec_is_internal_xattr (dict_t *dict, char *key, data_t *value, void *data)
+ec_is_data_fop(glusterfs_fop_t fop)
 {
-        if (key &&
-            (strncmp (key, EC_XATTR_PREFIX, strlen (EC_XATTR_PREFIX)) == 0))
-                return _gf_true;
-
-        return _gf_false;
+    switch (fop) {
+        case GF_FOP_WRITE:
+        case GF_FOP_TRUNCATE:
+        case GF_FOP_FTRUNCATE:
+        case GF_FOP_FALLOCATE:
+        case GF_FOP_DISCARD:
+        case GF_FOP_ZEROFILL:
+            return _gf_true;
+        default:
+            return _gf_false;
+    }
+    return _gf_false;
 }
-
-void
-ec_filter_internal_xattrs (dict_t *xattr)
+/*
+gf_boolean_t
+ec_is_metadata_fop (int32_t lock_kind, glusterfs_fop_t fop)
 {
-        dict_foreach_match (xattr, ec_is_internal_xattr, NULL,
-                            dict_remove_foreach_fn, NULL);
-}
+        if (lock_kind == EC_LOCK_ENTRY) {
+                return _gf_false;
+        }
+
+        switch (fop) {
+        case GF_FOP_SETATTR:
+        case GF_FOP_FSETATTR:
+        case GF_FOP_SETXATTR:
+        case GF_FOP_FSETXATTR:
+        case GF_FOP_REMOVEXATTR:
+        case GF_FOP_FREMOVEXATTR:
+                return _gf_true;
+        default:
+                return _gf_false;
+        }
+        return _gf_false;
+}*/
diff --git a/xlators/cluster/ec/src/ec-helpers.h b/xlators/cluster/ec/src/ec-helpers.h
index 11d2707b3c0..015db514e05 100644
--- a/xlators/cluster/ec/src/ec-helpers.h
+++ b/xlators/cluster/ec/src/ec-helpers.h
@@ -11,48 +11,190 @@
 #ifndef __EC_HELPERS_H__
 #define __EC_HELPERS_H__
 
-#include "ec-data.h"
+#include "ec-types.h"
 
-const char * ec_bin(char * str, size_t size, uint64_t value, int32_t digits);
-const char * ec_fop_name(int32_t id);
-void ec_trace(const char * event, ec_fop_data_t * fop, const char * fmt, ...);
-int32_t ec_bits_count(uint64_t n);
-int32_t ec_bits_index(uint64_t n);
-int32_t ec_bits_consume(uint64_t * n);
-size_t ec_iov_copy_to(void * dst, struct iovec * vector, int32_t count,
-                      off_t offset, size_t size);
+#define EC_ERR(_x) ((void *)-(intptr_t)(_x))
+#define EC_IS_ERR(_x) (((uintptr_t)(_x) & ~0xfffULL) == ~0xfffULL)
+#define EC_GET_ERR(_x) ((int32_t)(intptr_t)(_x))
 
-int32_t ec_dict_set_number(dict_t * dict, char * key, uint64_t value);
-int32_t ec_dict_del_number(dict_t * dict, char * key, uint64_t * value);
-int32_t ec_dict_set_config(dict_t * dict, char * key, ec_config_t * config);
-int32_t ec_dict_del_config(dict_t * dict, char * key, ec_config_t * config);
+#define EC_ALIGN_CHECK(_ptr, _align) ((((uintptr_t)(_ptr)) & ((_align)-1)) == 0)
 
-int32_t ec_loc_parent(xlator_t *xl, loc_t *loc, loc_t *parent);
-int32_t ec_loc_update(xlator_t *xl, loc_t *loc, inode_t *inode,
-                      struct iatt *iatt);
+const char *
+ec_bin(char *str, size_t size, uint64_t value, int32_t digits);
+const char *
+ec_fop_name(int32_t id);
+void
+ec_trace(const char *event, ec_fop_data_t *fop, const char *fmt, ...);
+int32_t
+ec_bits_consume(uint64_t *n);
+size_t
+ec_iov_copy_to(void *dst, struct iovec *vector, int32_t count, off_t offset,
+               size_t size);
+int32_t
+ec_buffer_alloc(xlator_t *xl, size_t size, struct iobref **piobref, void **ptr);
+int32_t
+ec_dict_set_array(dict_t *dict, char *key, uint64_t *value, int32_t size);
+int32_t
+ec_dict_get_array(dict_t *dict, char *key, uint64_t value[], int32_t size);
+
+int32_t
+ec_dict_del_array(dict_t *dict, char *key, uint64_t *value, int32_t size);
+int32_t
+ec_dict_set_number(dict_t *dict, char *key, uint64_t value);
+int32_t
+ec_dict_del_number(dict_t *dict, char *key, uint64_t *value);
+int32_t
+ec_dict_set_config(dict_t *dict, char *key, ec_config_t *config);
+int32_t
+ec_dict_del_config(dict_t *dict, char *key, ec_config_t *config);
+
+int32_t
+ec_loc_parent(xlator_t *xl, loc_t *loc, loc_t *parent);
+int32_t
+ec_loc_update(xlator_t *xl, loc_t *loc, inode_t *inode, struct iatt *iatt);
+
+int32_t
+ec_loc_from_fd(xlator_t *xl, loc_t *loc, fd_t *fd);
+int32_t
+ec_loc_from_loc(xlator_t *xl, loc_t *dst, loc_t *src);
+
+void
+ec_owner_set(call_frame_t *frame, void *owner);
+void
+ec_owner_copy(call_frame_t *frame, gf_lkowner_t *owner);
+
+ec_inode_t *
+__ec_inode_get(inode_t *inode, xlator_t *xl);
+ec_inode_t *
+ec_inode_get(inode_t *inode, xlator_t *xl);
+ec_fd_t *
+__ec_fd_get(fd_t *fd, xlator_t *xl);
+ec_fd_t *
+ec_fd_get(fd_t *fd, xlator_t *xl);
+
+static inline uint32_t
+ec_adjust_size_down(ec_t *ec, uint64_t *value, gf_boolean_t scale)
+{
+    uint64_t head, tmp;
+
+    tmp = *value;
+    head = tmp % ec->stripe_size;
+    tmp -= head;
 
-int32_t ec_loc_from_fd(xlator_t * xl, loc_t * loc, fd_t * fd);
-int32_t ec_loc_from_loc(xlator_t * xl, loc_t * dst, loc_t * src);
+    if (scale) {
+        tmp /= ec->fragments;
+    }
 
-void ec_owner_set(call_frame_t * frame, void * owner);
-void ec_owner_copy(call_frame_t * frame, gf_lkowner_t * owner);
+    *value = tmp;
 
-ec_inode_t * __ec_inode_get(inode_t * inode, xlator_t * xl);
-ec_inode_t * ec_inode_get(inode_t * inode, xlator_t * xl);
-ec_fd_t * __ec_fd_get(fd_t * fd, xlator_t * xl);
-ec_fd_t * ec_fd_get(fd_t * fd, xlator_t * xl);
+    return (uint32_t)head;
+}
+
+/* This function can cause an overflow if the passed value is too near to the
+ * uint64_t limit. If this happens, it returns the tail in negative form and
+ * the value is set to UINT64_MAX. */
+static inline int32_t
+ec_adjust_size_up(ec_t *ec, uint64_t *value, gf_boolean_t scale)
+{
+    uint64_t tmp;
+    int32_t tail;
 
-uint32_t ec_adjust_offset(ec_t * ec, off_t * offset, int32_t scale);
-uint64_t ec_adjust_size(ec_t * ec, uint64_t size, int32_t scale);
+    tmp = *value;
+    /* We first adjust the value down. This never causes overflow. */
+    tail = ec_adjust_size_down(ec, &tmp, scale);
+
+    /* If the value was already aligned, tail will be 0 and nothing else
+     * needs to be done. */
+    if (tail != 0) {
+        /* Otherwise, we need to compute the real tail and adjust the
+         * returned value to the next stripe. */
+        tail = ec->stripe_size - tail;
+        if (scale) {
+            tmp += ec->fragment_size;
+        } else {
+            tmp += ec->stripe_size;
+            /* If no scaling is requested there's a possibility of
+             * overflow. */
+            if (tmp < ec->stripe_size) {
+                tmp = UINT64_MAX;
+                tail = -tail;
+            }
+        }
+    }
+
+    *value = tmp;
+
+    return tail;
+}
 
-static inline int32_t ec_is_power_of_2(uint32_t value)
+/* This function is equivalent to ec_adjust_size_down() but with a potentially
+ * different parameter size (off_t vs uint64_t). */
+static inline uint32_t
+ec_adjust_offset_down(ec_t *ec, off_t *value, gf_boolean_t scale)
+{
+    off_t head, tmp;
+
+    tmp = *value;
+    head = tmp % ec->stripe_size;
+    tmp -= head;
+
+    if (scale) {
+        tmp /= ec->fragments;
+    }
+
+    *value = tmp;
+
+    return (uint32_t)head;
+}
+
+/* This function is equivalent to ec_adjust_size_up() but with a potentially
+ * different parameter size (off_t vs uint64_t). */
+static inline int32_t
+ec_adjust_offset_up(ec_t *ec, off_t *value, gf_boolean_t scale)
+{
+    uint64_t tail, tmp;
+
+    /* An offset is a signed type that can only have positive values, so
+     * we take advantage of this to avoid overflows. We simply convert it
+     * to an unsigned integer and operate normally. This won't cause an
+     * overflow. Overflow is only checked when converting back to an
+     * off_t. */
+    tmp = *value;
+    tail = ec->stripe_size;
+    tail -= (tmp + tail - 1) % tail + 1;
+    tmp += tail;
+    if (scale) {
+        /* If we are scaling, we'll never get an overflow. */
+        tmp /= ec->fragments;
+    } else {
+        /* Check if there has been an overflow. */
+        if ((off_t)tmp < 0) {
+            tmp = GF_OFF_MAX;
+            tail = -tail;
+        }
+    }
+
+    *value = (off_t)tmp;
+
+    return (int32_t)tail;
+}
+
+static inline int32_t
+ec_is_power_of_2(uint32_t value)
 {
     return (value != 0) && ((value & (value - 1)) == 0);
 }
 
 gf_boolean_t
-ec_is_internal_xattr (dict_t *dict, char *key, data_t *value, void *data);
+ec_is_internal_xattr(dict_t *dict, char *key, data_t *value, void *data);
 
 void
-ec_filter_internal_xattrs (dict_t *xattr);
+ec_filter_internal_xattrs(dict_t *xattr);
+
+gf_boolean_t
+ec_is_data_fop(glusterfs_fop_t fop);
+
+int32_t
+ec_launch_replace_heal(ec_t *ec);
+
 #endif /* __EC_HELPERS_H__ */
diff --git a/xlators/cluster/ec/src/ec-inode-read.c b/xlators/cluster/ec/src/ec-inode-read.c
index d126d48eb12..dad5f4d7018 100644
--- a/xlators/cluster/ec/src/ec-inode-read.c
+++ b/xlators/cluster/ec/src/ec-inode-read.c
@@ -8,9 +8,8 @@
   cases as published by the Free Software Foundation.
 */
 
-#include "xlator.h"
-#include "defaults.h"
-
+#include "ec.h"
+#include "ec-messages.h"
 #include "ec-helpers.h"
 #include "ec-common.h"
 #include "ec-combine.h"
@@ -19,10 +18,12 @@
 
 /* FOP: access */
 
-int32_t ec_access_cbk(call_frame_t * frame, void * cookie, xlator_t * this,
-                      int32_t op_ret, int32_t op_errno, dict_t * xdata)
+int32_t
+ec_access_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int32_t op_ret,
+              int32_t op_errno, dict_t *xdata)
 {
-    ec_fop_data_t * fop = NULL;
+    ec_fop_data_t *fop = NULL;
+    ec_cbk_data_t *cbk = NULL;
     int32_t idx = (int32_t)(uintptr_t)cookie;
 
     VALIDATE_OR_GOTO(this, out);
@@ -32,28 +33,27 @@ int32_t ec_access_cbk(call_frame_t * frame, void * cookie, xlator_t * this,
 
     fop = frame->local;
 
-    ec_trace("CBK", fop, "idx=%d, frame=%p, op_ret=%d, op_errno=%d", idx,
-             frame, op_ret, op_errno);
+    ec_trace("CBK", fop, "idx=%d, frame=%p, op_ret=%d, op_errno=%d", idx, frame,
+             op_ret, op_errno);
 
-    if (!ec_dispatch_one_retry(fop, idx, op_ret, op_errno))
-    {
-        if (fop->cbks.access != NULL)
-        {
-            fop->cbks.access(fop->req_frame, fop, this, op_ret, op_errno,
-                             xdata);
-        }
+    cbk = ec_cbk_data_allocate(frame, this, fop, GF_FOP_ACCESS, idx, op_ret,
+                               op_errno);
+    if (cbk) {
+        if (xdata)
+            cbk->xdata = dict_ref(xdata);
+        ec_combine(cbk, NULL);
     }
 
 out:
-    if (fop != NULL)
-    {
+    if (fop != NULL) {
         ec_complete(fop);
     }
 
     return 0;
 }
 
-void ec_wind_access(ec_t * ec, ec_fop_data_t * fop, int32_t idx)
+void
+ec_wind_access(ec_t *ec, ec_fop_data_t *fop, int32_t idx)
 {
     ec_trace("WIND", fop, "idx=%d", idx);
 
@@ -62,75 +62,112 @@ void ec_wind_access(ec_t * ec, ec_fop_data_t * fop, int32_t idx)
                       &fop->loc[0], fop->int32, fop->xdata);
 }
 
-int32_t ec_manager_access(ec_fop_data_t * fop, int32_t state)
+int32_t
+ec_manager_access(ec_fop_data_t *fop, int32_t state)
 {
-    switch (state)
-    {
+    ec_cbk_data_t *cbk = NULL;
+
+    switch (state) {
         case EC_STATE_INIT:
+        case EC_STATE_LOCK:
+            ec_lock_prepare_inode(fop, &fop->loc[0], EC_QUERY_INFO, 0,
+                                  EC_RANGE_FULL);
+            ec_lock(fop);
+
+            return EC_STATE_DISPATCH;
+
         case EC_STATE_DISPATCH:
             ec_dispatch_one(fop);
 
+            return EC_STATE_PREPARE_ANSWER;
+
+        case EC_STATE_PREPARE_ANSWER:
+            if (ec_dispatch_one_retry(fop, NULL)) {
+                return EC_STATE_DISPATCH;
+            }
+
             return EC_STATE_REPORT;
 
+        case EC_STATE_REPORT:
+            cbk = fop->answer;
+            GF_ASSERT(cbk);
+            if (fop->cbks.access != NULL) {
+                if (cbk) {
+                    fop->cbks.access(fop->req_frame, fop, fop->xl, cbk->op_ret,
+                                     cbk->op_errno, cbk->xdata);
+                }
+            }
+            return EC_STATE_LOCK_REUSE;
+
         case -EC_STATE_INIT:
+        case -EC_STATE_LOCK:
+        case -EC_STATE_DISPATCH:
+        case -EC_STATE_PREPARE_ANSWER:
         case -EC_STATE_REPORT:
-            if (fop->cbks.access != NULL)
-            {
+            if (fop->cbks.access != NULL) {
                 fop->cbks.access(fop->req_frame, fop, fop->xl, -1, fop->error,
                                  NULL);
             }
+            return EC_STATE_LOCK_REUSE;
+
+        case -EC_STATE_LOCK_REUSE:
+        case EC_STATE_LOCK_REUSE:
+            ec_lock_reuse(fop);
+
+            return EC_STATE_UNLOCK;
+
+        case -EC_STATE_UNLOCK:
+        case EC_STATE_UNLOCK:
+            ec_unlock(fop);
 
-        case EC_STATE_REPORT:
             return EC_STATE_END;
 
         default:
-            gf_log(fop->xl->name, GF_LOG_ERROR, "Unhandled state %d for %s",
-                   state, ec_fop_name(fop->id));
+            gf_msg(fop->xl->name, GF_LOG_ERROR, EINVAL, EC_MSG_UNHANDLED_STATE,
+                   "Unhandled state %d for %s", state, ec_fop_name(fop->id));
 
             return EC_STATE_END;
     }
 }
 
-void ec_access(call_frame_t * frame, xlator_t * this, uintptr_t target,
-               int32_t minimum, fop_access_cbk_t func, void * data,
-               loc_t * loc, int32_t mask, dict_t * xdata)
+void
+ec_access(call_frame_t *frame, xlator_t *this, uintptr_t target,
+          uint32_t fop_flags, fop_access_cbk_t func, void *data, loc_t *loc,
+          int32_t mask, dict_t *xdata)
 {
-    ec_cbk_t callback = { .access = func };
-    ec_fop_data_t * fop = NULL;
-    int32_t error = EIO;
+    ec_cbk_t callback = {.access = func};
+    ec_fop_data_t *fop = NULL;
+    int32_t error = ENOMEM;
 
-    gf_log("ec", GF_LOG_TRACE, "EC(ACCESS) %p", frame);
+    gf_msg_trace("ec", 0, "EC(ACCESS) %p", frame);
 
     VALIDATE_OR_GOTO(this, out);
     GF_VALIDATE_OR_GOTO(this->name, frame, out);
     GF_VALIDATE_OR_GOTO(this->name, this->private, out);
 
-    fop = ec_fop_data_allocate(frame, this, GF_FOP_ACCESS, 0, target, minimum,
-                               ec_wind_access, ec_manager_access, callback,
-                               data);
-    if (fop == NULL)
-    {
+    fop = ec_fop_data_allocate(frame, this, GF_FOP_ACCESS, EC_FLAG_LOCK_SHARED,
+                               target, fop_flags, ec_wind_access,
+                               ec_manager_access, callback, data);
+    if (fop == NULL) {
         goto out;
     }
 
     fop->int32 = mask;
 
-    if (loc != NULL)
-    {
-        if (loc_copy(&fop->loc[0], loc) != 0)
-        {
-            gf_log(this->name, GF_LOG_ERROR, "Failed to copy a location.");
+    if (loc != NULL) {
+        if (loc_copy(&fop->loc[0], loc) != 0) {
+            gf_msg(this->name, GF_LOG_ERROR, ENOMEM, EC_MSG_LOC_COPY_FAIL,
+                   "Failed to copy a location.");
 
             goto out;
         }
     }
-    if (xdata != NULL)
-    {
+    if (xdata != NULL) {
         fop->xdata = dict_ref(xdata);
-        if (fop->xdata == NULL)
-        {
-            gf_log(this->name, GF_LOG_ERROR, "Failed to reference a "
-                                             "dictionary.");
+        if (fop->xdata == NULL) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_DICT_REF_FAIL,
+                   "Failed to reference a "
+                   "dictionary.");
 
             goto out;
         }
@@ -139,25 +176,22 @@ void ec_access(call_frame_t * frame, xlator_t * this, uintptr_t target,
     error = 0;
 
 out:
-    if (fop != NULL)
-    {
+    if (fop != NULL) {
         ec_manager(fop, error);
-    }
-    else
-    {
-        func(frame, NULL, this, -1, EIO, NULL);
+    } else {
+        func(frame, NULL, this, -1, error, NULL);
     }
 }
 
 /* FOP: getxattr */
 
-int32_t ec_combine_getxattr(ec_fop_data_t * fop, ec_cbk_data_t * dst,
-                            ec_cbk_data_t * src)
+int32_t
+ec_combine_getxattr(ec_fop_data_t *fop, ec_cbk_data_t *dst, ec_cbk_data_t *src)
 {
-    if (!ec_dict_compare(dst->dict, src->dict))
-    {
-        gf_log(fop->xl->name, GF_LOG_NOTICE, "Mismatching dictionary in "
-                                             "answers of 'GF_FOP_GETXATTR'");
+    if (!ec_dict_compare(dst->dict, src->dict)) {
+        gf_msg(fop->xl->name, GF_LOG_NOTICE, 0, EC_MSG_DICT_MISMATCH,
+               "Mismatching dictionary in "
+               "answers of 'GF_FOP_GETXATTR'");
 
         return 0;
     }
@@ -165,12 +199,12 @@ int32_t ec_combine_getxattr(ec_fop_data_t * fop, ec_cbk_data_t * dst,
     return 1;
 }
 
-int32_t ec_getxattr_cbk(call_frame_t * frame, void * cookie, xlator_t * this,
-                        int32_t op_ret, int32_t op_errno, dict_t * dict,
-                        dict_t * xdata)
+int32_t
+ec_getxattr_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                int32_t op_ret, int32_t op_errno, dict_t *dict, dict_t *xdata)
 {
-    ec_fop_data_t * fop = NULL;
-    ec_cbk_data_t * cbk = NULL;
+    ec_fop_data_t *fop = NULL;
+    ec_cbk_data_t *cbk = NULL;
     int32_t idx = (int32_t)(uintptr_t)cookie;
 
     VALIDATE_OR_GOTO(this, out);
@@ -180,34 +214,30 @@ int32_t ec_getxattr_cbk(call_frame_t * frame, void * cookie, xlator_t * this,
 
     fop = frame->local;
 
-    ec_trace("CBK", fop, "idx=%d, frame=%p, op_ret=%d, op_errno=%d", idx,
-             frame, op_ret, op_errno);
+    ec_trace("CBK", fop, "idx=%d, frame=%p, op_ret=%d, op_errno=%d", idx, frame,
+             op_ret, op_errno);
 
     cbk = ec_cbk_data_allocate(frame, this, fop, GF_FOP_GETXATTR, idx, op_ret,
                                op_errno);
-    if (cbk != NULL)
-    {
-        if (op_ret >= 0)
-        {
-            if (dict != NULL)
-            {
+    if (cbk != NULL) {
+        if (op_ret >= 0) {
+            if (dict != NULL) {
                 cbk->dict = dict_ref(dict);
-                if (cbk->dict == NULL)
-                {
-                    gf_log(this->name, GF_LOG_ERROR, "Failed to reference a "
-                                                     "dictionary.");
+                if (cbk->dict == NULL) {
+                    gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_DICT_REF_FAIL,
+                           "Failed to reference a "
+                           "dictionary.");
 
                     goto out;
                 }
             }
         }
-        if (xdata != NULL)
-        {
+        if (xdata != NULL) {
             cbk->xdata = dict_ref(xdata);
-            if (cbk->xdata == NULL)
-            {
-                gf_log(this->name, GF_LOG_ERROR, "Failed to reference a "
-                                                 "dictionary.");
+            if (cbk->xdata == NULL) {
+                gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_DICT_REF_FAIL,
+                       "Failed to reference a "
+                       "dictionary.");
 
                 goto out;
             }
@@ -217,15 +247,15 @@ int32_t ec_getxattr_cbk(call_frame_t * frame, void * cookie, xlator_t * this,
     }
 
 out:
-    if (fop != NULL)
-    {
+    if (fop != NULL) {
         ec_complete(fop);
     }
 
     return 0;
 }
 
-void ec_wind_getxattr(ec_t * ec, ec_fop_data_t * fop, int32_t idx)
+void
+ec_wind_getxattr(ec_t *ec, ec_fop_data_t *fop, int32_t idx)
 {
     ec_trace("WIND", fop, "idx=%d", idx);
 
@@ -234,23 +264,44 @@ void ec_wind_getxattr(ec_t * ec, ec_fop_data_t * fop, int32_t idx)
                       &fop->loc[0], fop->str[0], fop->xdata);
 }
 
-int32_t ec_manager_getxattr(ec_fop_data_t * fop, int32_t state)
+void
+ec_handle_special_xattrs(ec_fop_data_t *fop)
 {
-    ec_cbk_data_t * cbk;
+    ec_cbk_data_t *cbk = NULL;
+    /* Stime may not be available on all the bricks, so even if some of the
+     * subvols succeed the operation, treat it as answer.*/
+    if (fop->str[0] && fnmatch(GF_XATTR_STIME_PATTERN, fop->str[0], 0) == 0) {
+        if (!fop->answer || (fop->answer->op_ret < 0)) {
+            list_for_each_entry(cbk, &fop->cbk_list, list)
+            {
+                if (cbk->op_ret >= 0) {
+                    fop->answer = cbk;
+                    break;
+                }
+            }
+        }
+    }
+}
+
+int32_t
+ec_manager_getxattr(ec_fop_data_t *fop, int32_t state)
+{
+    ec_cbk_data_t *cbk;
 
-    switch (state)
-    {
+    switch (state) {
         case EC_STATE_INIT:
         case EC_STATE_LOCK:
             /* clear-locks commands must be done without any locks acquired
                to avoid interferences. */
             if ((fop->str[0] == NULL) ||
                 (strncmp(fop->str[0], GF_XATTR_CLRLK_CMD,
-                         strlen(GF_XATTR_CLRLK_CMD)) != 0)) {
+                         SLEN(GF_XATTR_CLRLK_CMD)) != 0)) {
                 if (fop->fd == NULL) {
-                    ec_lock_prepare_inode(fop, &fop->loc[0], 0);
+                    ec_lock_prepare_inode(fop, &fop->loc[0], EC_QUERY_INFO, 0,
+                                          EC_RANGE_FULL);
                 } else {
-                    ec_lock_prepare_fd(fop, fop->fd, 0);
+                    ec_lock_prepare_fd(fop, fop->fd, EC_QUERY_INFO, 0,
+                                       EC_RANGE_FULL);
                 }
                 ec_lock(fop);
             }
@@ -258,41 +309,35 @@ int32_t ec_manager_getxattr(ec_fop_data_t * fop, int32_t state)
             return EC_STATE_DISPATCH;
 
         case EC_STATE_DISPATCH:
-            ec_dispatch_all(fop);
+            if (fop->minimum == EC_MINIMUM_ALL) {
+                ec_dispatch_all(fop);
+            } else {
+                ec_dispatch_one(fop);
+            }
 
             return EC_STATE_PREPARE_ANSWER;
 
         case EC_STATE_PREPARE_ANSWER:
-            cbk = fop->answer;
-            if (cbk != NULL)
-            {
-                if (!ec_dict_combine(cbk, EC_COMBINE_XDATA) ||
-                    ((cbk->op_ret >= 0) && !ec_dict_combine(cbk,
-                                                            EC_COMBINE_DICT)))
-                {
-                    if (cbk->op_ret >= 0)
-                    {
-                        cbk->op_ret = -1;
-                        cbk->op_errno = EIO;
-                    }
+            ec_handle_special_xattrs(fop);
+            if (fop->minimum == EC_MINIMUM_ALL) {
+                cbk = ec_fop_prepare_answer(fop, _gf_true);
+            } else {
+                if (ec_dispatch_one_retry(fop, &cbk)) {
+                    return EC_STATE_DISPATCH;
                 }
-                if (cbk->op_ret < 0)
-                {
-                    ec_fop_set_error(fop, cbk->op_errno);
-                }
-                else
-                {
+            }
+            if (cbk != NULL) {
+                int32_t err;
+
+                err = ec_dict_combine(cbk, EC_COMBINE_DICT);
+                if (!ec_cbk_set_error(cbk, -err, _gf_true)) {
                     if (cbk->xdata != NULL)
-                            ec_filter_internal_xattrs (cbk->xdata);
+                        ec_filter_internal_xattrs(cbk->xdata);
 
                     if (cbk->dict != NULL)
-                            ec_filter_internal_xattrs (cbk->dict);
+                        ec_filter_internal_xattrs(cbk->dict);
                 }
             }
-            else
-            {
-                ec_fop_set_error(fop, EIO);
-            }
 
             return EC_STATE_REPORT;
 
@@ -301,8 +346,7 @@ int32_t ec_manager_getxattr(ec_fop_data_t * fop, int32_t state)
 
             GF_ASSERT(cbk != NULL);
 
-            if (fop->cbks.getxattr != NULL)
-            {
+            if (fop->cbks.getxattr != NULL) {
                 fop->cbks.getxattr(fop->req_frame, fop, fop->xl, cbk->op_ret,
                                    cbk->op_errno, cbk->dict, cbk->xdata);
             }
@@ -316,10 +360,9 @@ int32_t ec_manager_getxattr(ec_fop_data_t * fop, int32_t state)
         case -EC_STATE_REPORT:
             GF_ASSERT(fop->error != 0);
 
-            if (fop->cbks.getxattr != NULL)
-            {
-                fop->cbks.getxattr(fop->req_frame, fop, fop->xl, -1,
-                                   fop->error, NULL, NULL);
+            if (fop->cbks.getxattr != NULL) {
+                fop->cbks.getxattr(fop->req_frame, fop, fop->xl, -1, fop->error,
+                                   NULL, NULL);
             }
 
             return EC_STATE_LOCK_REUSE;
@@ -337,24 +380,44 @@ int32_t ec_manager_getxattr(ec_fop_data_t * fop, int32_t state)
             return EC_STATE_END;
 
         default:
-            gf_log(fop->xl->name, GF_LOG_ERROR, "Unhandled state %d for %s",
-                   state, ec_fop_name(fop->id));
+            gf_msg(fop->xl->name, GF_LOG_ERROR, EINVAL, EC_MSG_UNHANDLED_STATE,
+                   "Unhandled state %d for %s", state, ec_fop_name(fop->id));
 
             return EC_STATE_END;
     }
 }
 
-int32_t ec_getxattr_heal_cbk(call_frame_t *frame, void *cookie, xlator_t *xl,
-                             int32_t op_ret, int32_t op_errno, uintptr_t mask,
-                             uintptr_t good, uintptr_t bad, dict_t *xdata)
+int32_t
+ec_getxattr_heal_cbk(call_frame_t *frame, void *cookie, xlator_t *xl,
+                     int32_t op_ret, int32_t op_errno, uintptr_t mask,
+                     uintptr_t good, uintptr_t bad, uint32_t pending,
+                     dict_t *xdata)
 {
-    ec_fop_data_t *fop = cookie;
-    fop_getxattr_cbk_t func = fop->data;
+    fop_getxattr_cbk_t func = cookie;
     ec_t *ec = xl->private;
     dict_t *dict = NULL;
     char *str;
     char bin1[65], bin2[65];
 
+    /* We try to return the 'pending' information in xdata, but if this cannot
+     * be set, we will ignore it silently. We prefer to report the success or
+     * failure of the heal itself. */
+    if (xdata == NULL) {
+        xdata = dict_new();
+    } else {
+        dict_ref(xdata);
+    }
+    if (xdata != NULL) {
+        if (dict_set_uint32(xdata, EC_XATTR_HEAL_NEW, pending) != 0) {
+            /* dict_set_uint32() is marked as 'warn_unused_result' and gcc
+             * enforces to check the result in this case. However we don't
+             * really care if it succeeded or not. We'll just do the same.
+             *
+             * This empty 'if' avoids the warning, and it will be removed by
+             * the optimizer. */
+        }
+    }
+
     if (op_ret >= 0) {
         dict = dict_new();
         if (dict == NULL) {
@@ -374,7 +437,7 @@ int32_t ec_getxattr_heal_cbk(call_frame_t *frame, void *cookie, xlator_t *xl,
                 goto out;
             }
 
-            if (dict_set_str(dict, EC_XATTR_HEAL, str) != 0) {
+            if (dict_set_dynstr(dict, EC_XATTR_HEAL, str) != 0) {
                 GF_FREE(str);
                 dict_unref(dict);
                 dict = NULL;
@@ -388,25 +451,28 @@ int32_t ec_getxattr_heal_cbk(call_frame_t *frame, void *cookie, xlator_t *xl,
     }
 
 out:
-    func(frame, NULL, xl, op_ret, op_errno, dict, NULL);
+    func(frame, NULL, xl, op_ret, op_errno, dict, xdata);
 
     if (dict != NULL) {
         dict_unref(dict);
     }
+    if (xdata != NULL) {
+        dict_unref(xdata);
+    }
 
     return 0;
 }
 
 void
-ec_getxattr (call_frame_t *frame, xlator_t *this, uintptr_t target,
-             int32_t minimum, fop_getxattr_cbk_t func, void *data,
-             loc_t *loc, const char *name, dict_t *xdata)
+ec_getxattr(call_frame_t *frame, xlator_t *this, uintptr_t target,
+            uint32_t fop_flags, fop_getxattr_cbk_t func, void *data, loc_t *loc,
+            const char *name, dict_t *xdata)
 {
-    ec_cbk_t callback = { .getxattr = func };
-    ec_fop_data_t * fop = NULL;
-    int32_t error = EIO;
+    ec_cbk_t callback = {.getxattr = func};
+    ec_fop_data_t *fop = NULL;
+    int32_t error = ENOMEM;
 
-    gf_log("ec", GF_LOG_TRACE, "EC(GETXATTR) %p", frame);
+    gf_msg_trace("ec", 0, "EC(GETXATTR) %p", frame);
 
     VALIDATE_OR_GOTO(this, out);
     GF_VALIDATE_OR_GOTO(this->name, frame, out);
@@ -414,47 +480,49 @@ ec_getxattr (call_frame_t *frame, xlator_t *this, uintptr_t target,
 
     /* Special handling of an explicit self-heal request */
     if ((name != NULL) && (strcmp(name, EC_XATTR_HEAL) == 0)) {
-        ec_heal(frame, this, target, EC_MINIMUM_ONE, ec_getxattr_heal_cbk,
-                func, loc, 0, NULL);
+        ec_heal(frame, this, target, EC_MINIMUM_ONE, ec_getxattr_heal_cbk, func,
+                loc, 0, NULL);
 
         return;
     }
 
-    fop = ec_fop_data_allocate(frame, this, GF_FOP_GETXATTR,
-                               EC_FLAG_UPDATE_LOC_INODE, target, minimum,
-                               ec_wind_getxattr, ec_manager_getxattr, callback,
-                               data);
-    if (fop == NULL)
-    {
+    fop = ec_fop_data_allocate(
+        frame, this, GF_FOP_GETXATTR, EC_FLAG_LOCK_SHARED, target, fop_flags,
+        ec_wind_getxattr, ec_manager_getxattr, callback, data);
+    if (fop == NULL) {
         goto out;
     }
 
-    if (loc != NULL)
-    {
-        if (loc_copy(&fop->loc[0], loc) != 0)
-        {
-            gf_log(this->name, GF_LOG_ERROR, "Failed to copy a location.");
+    if (loc != NULL) {
+        if (loc_copy(&fop->loc[0], loc) != 0) {
+            gf_msg(this->name, GF_LOG_ERROR, ENOMEM, EC_MSG_LOC_COPY_FAIL,
+                   "Failed to copy a location.");
 
             goto out;
         }
     }
-    if (name != NULL)
-    {
-        fop->str[0] = gf_strdup(name);
-        if (fop->str[0] == NULL)
-        {
-            gf_log(this->name, GF_LOG_ERROR, "Failed to duplicate a string.");
+    if (name != NULL) {
+        /* In case of list-node-uuids xattr, set flag to indicate
+         * the same and use node-uuid xattr for winding fop */
+        if (XATTR_IS_NODE_UUID_LIST(name)) {
+            fop->int32 = 1;
+            fop->str[0] = gf_strdup(GF_XATTR_NODE_UUID_KEY);
+        } else {
+            fop->str[0] = gf_strdup(name);
+        }
+        if (fop->str[0] == NULL) {
+            gf_msg(this->name, GF_LOG_ERROR, ENOMEM, EC_MSG_NO_MEMORY,
+                   "Failed to duplicate a string.");
 
             goto out;
         }
     }
-    if (xdata != NULL)
-    {
+    if (xdata != NULL) {
         fop->xdata = dict_ref(xdata);
-        if (fop->xdata == NULL)
-        {
-            gf_log(this->name, GF_LOG_ERROR, "Failed to reference a "
-                                             "dictionary.");
+        if (fop->xdata == NULL) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_DICT_REF_FAIL,
+                   "Failed to reference a "
+                   "dictionary.");
 
             goto out;
         }
@@ -464,20 +532,20 @@ ec_getxattr (call_frame_t *frame, xlator_t *this, uintptr_t target,
 
 out:
     if (fop != NULL) {
-        ec_manager (fop, error);
+        ec_manager(fop, error);
     } else {
-        func (frame, NULL, this, -1, error, NULL, NULL);
+        func(frame, NULL, this, -1, error, NULL, NULL);
     }
 }
 
 /* FOP: fgetxattr */
 
-int32_t ec_fgetxattr_cbk(call_frame_t * frame, void * cookie, xlator_t * this,
-                         int32_t op_ret, int32_t op_errno, dict_t * dict,
-                         dict_t * xdata)
+int32_t
+ec_fgetxattr_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                 int32_t op_ret, int32_t op_errno, dict_t *dict, dict_t *xdata)
 {
-    ec_fop_data_t * fop = NULL;
-    ec_cbk_data_t * cbk = NULL;
+    ec_fop_data_t *fop = NULL;
+    ec_cbk_data_t *cbk = NULL;
     int32_t idx = (int32_t)(uintptr_t)cookie;
 
     VALIDATE_OR_GOTO(this, out);
@@ -487,34 +555,30 @@ int32_t ec_fgetxattr_cbk(call_frame_t * frame, void * cookie, xlator_t * this,
 
     fop = frame->local;
 
-    ec_trace("CBK", fop, "idx=%d, frame=%p, op_ret=%d, op_errno=%d", idx,
-             frame, op_ret, op_errno);
+    ec_trace("CBK", fop, "idx=%d, frame=%p, op_ret=%d, op_errno=%d", idx, frame,
+             op_ret, op_errno);
 
     cbk = ec_cbk_data_allocate(frame, this, fop, GF_FOP_FGETXATTR, idx, op_ret,
                                op_errno);
-    if (cbk != NULL)
-    {
-        if (op_ret >= 0)
-        {
-            if (dict != NULL)
-            {
+    if (cbk != NULL) {
+        if (op_ret >= 0) {
+            if (dict != NULL) {
                 cbk->dict = dict_ref(dict);
-                if (cbk->dict == NULL)
-                {
-                    gf_log(this->name, GF_LOG_ERROR, "Failed to reference a "
-                                                     "dictionary.");
+                if (cbk->dict == NULL) {
+                    gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_DICT_REF_FAIL,
+                           "Failed to reference a "
+                           "dictionary.");
 
                     goto out;
                 }
             }
         }
-        if (xdata != NULL)
-        {
+        if (xdata != NULL) {
             cbk->xdata = dict_ref(xdata);
-            if (cbk->xdata == NULL)
-            {
-                gf_log(this->name, GF_LOG_ERROR, "Failed to reference a "
-                                                 "dictionary.");
+            if (cbk->xdata == NULL) {
+                gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_DICT_REF_FAIL,
+                       "Failed to reference a "
+                       "dictionary.");
 
                 goto out;
             }
@@ -524,8 +588,7 @@ int32_t ec_fgetxattr_cbk(call_frame_t * frame, void * cookie, xlator_t * this,
     }
 
 out:
-    if (fop != NULL)
-    {
+    if (fop != NULL) {
         ec_complete(fop);
     }
 
@@ -533,7 +596,7 @@ out:
 }
 
 void
-ec_wind_fgetxattr (ec_t *ec, ec_fop_data_t *fop, int32_t idx)
+ec_wind_fgetxattr(ec_t *ec, ec_fop_data_t *fop, int32_t idx)
 {
     ec_trace("WIND", fop, "idx=%d", idx);
 
@@ -543,59 +606,54 @@ ec_wind_fgetxattr (ec_t *ec, ec_fop_data_t *fop, int32_t idx)
 }
 
 void
-ec_fgetxattr (call_frame_t *frame, xlator_t *this, uintptr_t target,
-              int32_t minimum, fop_fgetxattr_cbk_t func, void *data,
-              fd_t *fd, const char *name, dict_t *xdata)
+ec_fgetxattr(call_frame_t *frame, xlator_t *this, uintptr_t target,
+             uint32_t fop_flags, fop_fgetxattr_cbk_t func, void *data, fd_t *fd,
+             const char *name, dict_t *xdata)
 {
-    ec_cbk_t callback = { .fgetxattr = func };
-    ec_fop_data_t * fop = NULL;
-    int32_t error = EIO;
+    ec_cbk_t callback = {.fgetxattr = func};
+    ec_fop_data_t *fop = NULL;
+    int32_t error = ENOMEM;
 
-    gf_log("ec", GF_LOG_TRACE, "EC(FGETXATTR) %p", frame);
+    gf_msg_trace("ec", 0, "EC(FGETXATTR) %p", frame);
 
     VALIDATE_OR_GOTO(this, out);
     GF_VALIDATE_OR_GOTO(this->name, frame, out);
     GF_VALIDATE_OR_GOTO(this->name, this->private, out);
 
-    fop = ec_fop_data_allocate(frame, this, GF_FOP_FGETXATTR,
-                               EC_FLAG_UPDATE_FD_INODE, target, minimum,
-                               ec_wind_fgetxattr, ec_manager_getxattr,
-                               callback, data);
-    if (fop == NULL)
-    {
+    fop = ec_fop_data_allocate(
+        frame, this, GF_FOP_FGETXATTR, EC_FLAG_LOCK_SHARED, target, fop_flags,
+        ec_wind_fgetxattr, ec_manager_getxattr, callback, data);
+    if (fop == NULL) {
         goto out;
     }
 
     fop->use_fd = 1;
 
-    if (fd != NULL)
-    {
+    if (fd != NULL) {
         fop->fd = fd_ref(fd);
-        if (fop->fd == NULL)
-        {
-            gf_log(this->name, GF_LOG_ERROR, "Failed to reference a "
-                                             "file descriptor.");
+        if (fop->fd == NULL) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_FILE_DESC_REF_FAIL,
+                   "Failed to reference a "
+                   "file descriptor.");
 
             goto out;
         }
     }
-    if (name != NULL)
-    {
+    if (name != NULL) {
         fop->str[0] = gf_strdup(name);
-        if (fop->str[0] == NULL)
-        {
-            gf_log(this->name, GF_LOG_ERROR, "Failed to duplicate a string.");
+        if (fop->str[0] == NULL) {
+            gf_msg(this->name, GF_LOG_ERROR, ENOMEM, EC_MSG_NO_MEMORY,
+                   "Failed to duplicate a string.");
 
             goto out;
         }
     }
-    if (xdata != NULL)
-    {
+    if (xdata != NULL) {
         fop->xdata = dict_ref(xdata);
-        if (fop->xdata == NULL)
-        {
-            gf_log(this->name, GF_LOG_ERROR, "Failed to reference a "
-                                             "dictionary.");
+        if (fop->xdata == NULL) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_DICT_REF_FAIL,
+                   "Failed to reference a "
+                   "dictionary.");
 
             goto out;
         }
@@ -605,21 +663,21 @@ ec_fgetxattr (call_frame_t *frame, xlator_t *this, uintptr_t target,
 
 out:
     if (fop != NULL) {
-        ec_manager (fop, error);
+        ec_manager(fop, error);
     } else {
-        func (frame, NULL, this, -1, error, NULL, NULL);
+        func(frame, NULL, this, -1, error, NULL, NULL);
     }
 }
 
 /* FOP: open */
 
-int32_t ec_combine_open(ec_fop_data_t * fop, ec_cbk_data_t * dst,
-                        ec_cbk_data_t * src)
+int32_t
+ec_combine_open(ec_fop_data_t *fop, ec_cbk_data_t *dst, ec_cbk_data_t *src)
 {
-    if (dst->fd != src->fd)
-    {
-        gf_log(fop->xl->name, GF_LOG_NOTICE, "Mismatching fd in answers "
-                                             "of 'GF_FOP_OPEN': %p <-> %p",
+    if (dst->fd != src->fd) {
+        gf_msg(fop->xl->name, GF_LOG_NOTICE, 0, EC_MSG_FD_MISMATCH,
+               "Mismatching fd in answers "
+               "of 'GF_FOP_OPEN': %p <-> %p",
                dst->fd, src->fd);
 
         return 0;
@@ -628,12 +686,12 @@ int32_t ec_combine_open(ec_fop_data_t * fop, ec_cbk_data_t * dst,
     return 1;
 }
 
-int32_t ec_open_cbk(call_frame_t * frame, void * cookie, xlator_t * this,
-                    int32_t op_ret, int32_t op_errno, fd_t * fd,
-                    dict_t * xdata)
+int32_t
+ec_open_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int32_t op_ret,
+            int32_t op_errno, fd_t *fd, dict_t *xdata)
 {
-    ec_fop_data_t * fop = NULL;
-    ec_cbk_data_t * cbk = NULL;
+    ec_fop_data_t *fop = NULL;
+    ec_cbk_data_t *cbk = NULL;
     int32_t idx = (int32_t)(uintptr_t)cookie;
 
     VALIDATE_OR_GOTO(this, out);
@@ -643,52 +701,51 @@ int32_t ec_open_cbk(call_frame_t * frame, void * cookie, xlator_t * this,
 
     fop = frame->local;
 
-    ec_trace("CBK", fop, "idx=%d, frame=%p, op_ret=%d, op_errno=%d", idx,
-             frame, op_ret, op_errno);
+    ec_trace("CBK", fop, "idx=%d, frame=%p, op_ret=%d, op_errno=%d", idx, frame,
+             op_ret, op_errno);
 
     cbk = ec_cbk_data_allocate(frame, this, fop, GF_FOP_OPEN, idx, op_ret,
                                op_errno);
-    if (cbk != NULL)
-    {
-        if (op_ret >= 0)
-        {
-            if (fd != NULL)
-            {
+    if (cbk != NULL) {
+        if (op_ret >= 0) {
+            if (fd != NULL) {
                 cbk->fd = fd_ref(fd);
-                if (cbk->fd == NULL)
-                {
-                    gf_log(this->name, GF_LOG_ERROR, "Failed to reference a "
-                                                     "file descriptor.");
+                if (cbk->fd == NULL) {
+                    gf_msg(this->name, GF_LOG_ERROR, 0,
+                           EC_MSG_FILE_DESC_REF_FAIL,
+                           "Failed to reference a "
+                           "file descriptor.");
 
                     goto out;
                 }
             }
         }
-        if (xdata != NULL)
-        {
+        if (xdata != NULL) {
             cbk->xdata = dict_ref(xdata);
-            if (cbk->xdata == NULL)
-            {
-                gf_log(this->name, GF_LOG_ERROR, "Failed to reference a "
-                                                 "dictionary.");
+            if (cbk->xdata == NULL) {
+                gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_DICT_REF_FAIL,
+                       "Failed to reference a "
+                       "dictionary.");
 
                 goto out;
             }
         }
 
         ec_combine(cbk, ec_combine_open);
+
+        ec_update_fd_status(fd, this, idx, op_ret);
     }
 
 out:
-    if (fop != NULL)
-    {
+    if (fop != NULL) {
         ec_complete(fop);
     }
 
     return 0;
 }
 
-void ec_wind_open(ec_t * ec, ec_fop_data_t * fop, int32_t idx)
+void
+ec_wind_open(ec_t *ec, ec_fop_data_t *fop, int32_t idx)
 {
     ec_trace("WIND", fop, "idx=%d", idx);
 
@@ -697,10 +754,10 @@ void ec_wind_open(ec_t * ec, ec_fop_data_t * fop, int32_t idx)
                       &fop->loc[0], fop->int32, fop->fd, fop->xdata);
 }
 
-int32_t ec_open_truncate_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
-                             int32_t op_ret, int32_t op_errno,
-                             struct iatt *prebuf, struct iatt *postbuf,
-                             dict_t *xdata)
+int32_t
+ec_open_truncate_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                     int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
+                     struct iatt *postbuf, dict_t *xdata)
 {
     ec_fop_data_t *fop = cookie;
     int32_t error = 0;
@@ -717,25 +774,35 @@ int32_t ec_open_truncate_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
     return 0;
 }
 
-int32_t ec_manager_open(ec_fop_data_t * fop, int32_t state)
+int32_t
+ec_manager_open(ec_fop_data_t *fop, int32_t state)
 {
-    ec_cbk_data_t * cbk;
-    ec_fd_t * ctx;
+    ec_cbk_data_t *cbk;
+    ec_fd_t *ctx;
+    int32_t err;
 
-    switch (state)
-    {
+    switch (state) {
         case EC_STATE_INIT:
             LOCK(&fop->fd->lock);
 
             ctx = __ec_fd_get(fop->fd, fop->xl);
-            if ((ctx == NULL) ||
-                (ec_loc_from_loc(fop->xl, &ctx->loc, &fop->loc[0])) != 0) {
+            if (ctx == NULL) {
                 UNLOCK(&fop->fd->lock);
 
-                fop->error = EIO;
+                fop->error = ENOMEM;
 
                 return EC_STATE_REPORT;
             }
+            if (!ctx->loc.inode) {
+                err = ec_loc_from_loc(fop->xl, &ctx->loc, &fop->loc[0]);
+                if (err != 0) {
+                    UNLOCK(&fop->fd->lock);
+
+                    fop->error = -err;
+
+                    return EC_STATE_REPORT;
+                }
+            }
 
             ctx->flags = fop->int32;
 
@@ -750,7 +817,7 @@ int32_t ec_manager_open(ec_fop_data_t * fop, int32_t state)
             fop->uint32 = fop->int32 & O_TRUNC;
             fop->int32 &= ~(O_APPEND | O_TRUNC);
 
-        /* Fall through */
+            /* Fall through */
 
         case EC_STATE_DISPATCH:
             ec_dispatch_all(fop);
@@ -758,51 +825,33 @@ int32_t ec_manager_open(ec_fop_data_t * fop, int32_t state)
             return EC_STATE_PREPARE_ANSWER;
 
         case EC_STATE_PREPARE_ANSWER:
-            cbk = fop->answer;
-            if (cbk != NULL)
-            {
-                if (!ec_dict_combine(cbk, EC_COMBINE_XDATA))
-                {
-                    if (cbk->op_ret >= 0)
-                    {
-                        cbk->op_ret = -1;
-                        cbk->op_errno = EIO;
+            cbk = ec_fop_prepare_answer(fop, _gf_true);
+            if (cbk != NULL) {
+                int32_t err;
+
+                err = ec_loc_update(fop->xl, &fop->loc[0], cbk->fd->inode,
+                                    NULL);
+                if (!ec_cbk_set_error(cbk, -err, _gf_true)) {
+                    LOCK(&fop->fd->lock);
+
+                    ctx = __ec_fd_get(fop->fd, fop->xl);
+                    if (ctx != NULL) {
+                        ctx->open |= cbk->mask;
                     }
-                }
-                if (cbk->op_ret >= 0) {
-                    if (ec_loc_update(fop->xl, &fop->loc[0], cbk->fd->inode,
-                                      NULL) != 0) {
-                        cbk->op_ret = -1;
-                        cbk->op_errno = EIO;
-                    } else {
-                        LOCK(&fop->fd->lock);
-
-                        ctx = __ec_fd_get(fop->fd, fop->xl);
-                        if (ctx != NULL) {
-                            ctx->open |= cbk->mask;
-                        }
-
-                        UNLOCK(&fop->fd->lock);
-
-                        /* If O_TRUNC was specified, call ftruncate to
-                           effectively trunc the file with appropriate locks
-                           acquired. We don't use ctx->flags because self-heal
-                           can use the same fd with different flags. */
-                        if (fop->uint32 != 0) {
-                            ec_sleep(fop);
-                            ec_ftruncate(fop->req_frame, fop->xl, cbk->mask,
-                                         fop->minimum, ec_open_truncate_cbk,
-                                         fop, cbk->fd, 0, NULL);
-                        }
+
+                    UNLOCK(&fop->fd->lock);
+
+                    /* If O_TRUNC was specified, call ftruncate to
+                       effectively trunc the file with appropriate locks
+                       acquired. We don't use ctx->flags because self-heal
+                       can use the same fd with different flags. */
+                    if (fop->uint32 != 0) {
+                        ec_sleep(fop);
+                        ec_ftruncate(fop->req_frame, fop->xl, cbk->mask,
+                                     fop->minimum, ec_open_truncate_cbk, fop,
+                                     cbk->fd, 0, NULL);
                     }
                 }
-                if (cbk->op_ret < 0) {
-                    ec_fop_set_error(fop, cbk->op_errno);
-                }
-            }
-            else
-            {
-                ec_fop_set_error(fop, EIO);
             }
 
             return EC_STATE_REPORT;
@@ -812,8 +861,7 @@ int32_t ec_manager_open(ec_fop_data_t * fop, int32_t state)
 
             GF_ASSERT(cbk != NULL);
 
-            if (fop->cbks.open != NULL)
-            {
+            if (fop->cbks.open != NULL) {
                 fop->cbks.open(fop->req_frame, fop, fop->xl, cbk->op_ret,
                                cbk->op_errno, cbk->fd, cbk->xdata);
             }
@@ -826,8 +874,7 @@ int32_t ec_manager_open(ec_fop_data_t * fop, int32_t state)
         case -EC_STATE_REPORT:
             GF_ASSERT(fop->error != 0);
 
-            if (fop->cbks.open != NULL)
-            {
+            if (fop->cbks.open != NULL) {
                 fop->cbks.open(fop->req_frame, fop, fop->xl, -1, fop->error,
                                NULL, NULL);
             }
@@ -835,64 +882,61 @@ int32_t ec_manager_open(ec_fop_data_t * fop, int32_t state)
             return EC_STATE_END;
 
         default:
-            gf_log(fop->xl->name, GF_LOG_ERROR, "Unhandled state %d for %s",
-                   state, ec_fop_name(fop->id));
+            gf_msg(fop->xl->name, GF_LOG_ERROR, EINVAL, EC_MSG_UNHANDLED_STATE,
+                   "Unhandled state %d for %s", state, ec_fop_name(fop->id));
 
             return EC_STATE_END;
     }
 }
 
-void ec_open(call_frame_t * frame, xlator_t * this, uintptr_t target,
-             int32_t minimum, fop_open_cbk_t func, void * data, loc_t * loc,
-             int32_t flags, fd_t * fd, dict_t * xdata)
+void
+ec_open(call_frame_t *frame, xlator_t *this, uintptr_t target,
+        uint32_t fop_flags, fop_open_cbk_t func, void *data, loc_t *loc,
+        int32_t flags, fd_t *fd, dict_t *xdata)
 {
-    ec_cbk_t callback = { .open = func };
-    ec_fop_data_t * fop = NULL;
-    int32_t error = EIO;
+    ec_cbk_t callback = {.open = func};
+    ec_fop_data_t *fop = NULL;
+    int32_t error = ENOMEM;
 
-    gf_log("ec", GF_LOG_TRACE, "EC(OPEN) %p", frame);
+    gf_msg_trace("ec", 0, "EC(OPEN) %p", frame);
 
     VALIDATE_OR_GOTO(this, out);
     GF_VALIDATE_OR_GOTO(this->name, frame, out);
     GF_VALIDATE_OR_GOTO(this->name, this->private, out);
 
-    fop = ec_fop_data_allocate(frame, this, GF_FOP_OPEN, EC_FLAG_UPDATE_FD,
-                               target, minimum, ec_wind_open, ec_manager_open,
+    fop = ec_fop_data_allocate(frame, this, GF_FOP_OPEN, EC_FLAG_LOCK_SHARED,
+                               target, fop_flags, ec_wind_open, ec_manager_open,
                                callback, data);
-    if (fop == NULL)
-    {
+    if (fop == NULL) {
         goto out;
     }
 
     fop->int32 = flags;
 
-    if (loc != NULL)
-    {
-        if (loc_copy(&fop->loc[0], loc) != 0)
-        {
-            gf_log(this->name, GF_LOG_ERROR, "Failed to copy a location.");
+    if (loc != NULL) {
+        if (loc_copy(&fop->loc[0], loc) != 0) {
+            gf_msg(this->name, GF_LOG_ERROR, ENOMEM, EC_MSG_LOC_COPY_FAIL,
+                   "Failed to copy a location.");
 
             goto out;
         }
     }
-    if (fd != NULL)
-    {
+    if (fd != NULL) {
         fop->fd = fd_ref(fd);
-        if (fop->fd == NULL)
-        {
-            gf_log(this->name, GF_LOG_ERROR, "Failed to reference a "
-                                             "file descriptor.");
+        if (fop->fd == NULL) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_FILE_DESC_REF_FAIL,
+                   "Failed to reference a "
+                   "file descriptor.");
 
             goto out;
         }
     }
-    if (xdata != NULL)
-    {
+    if (xdata != NULL) {
         fop->xdata = dict_ref(xdata);
-        if (fop->xdata == NULL)
-        {
-            gf_log(this->name, GF_LOG_ERROR, "Failed to reference a "
-                                             "dictionary.");
+        if (fop->xdata == NULL) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_DICT_REF_FAIL,
+                   "Failed to reference a "
+                   "dictionary.");
 
             goto out;
         }
@@ -901,25 +945,22 @@ void ec_open(call_frame_t * frame, xlator_t * this, uintptr_t target,
     error = 0;
 
 out:
-    if (fop != NULL)
-    {
+    if (fop != NULL) {
         ec_manager(fop, error);
-    }
-    else
-    {
-        func(frame, NULL, this, -1, EIO, NULL, NULL);
+    } else {
+        func(frame, NULL, this, -1, error, NULL, NULL);
     }
 }
 
 /* FOP: readlink */
 
-int32_t ec_combine_readlink(ec_fop_data_t * fop, ec_cbk_data_t * dst,
-                            ec_cbk_data_t * src)
+int32_t
+ec_combine_readlink(ec_fop_data_t *fop, ec_cbk_data_t *dst, ec_cbk_data_t *src)
 {
-    if (!ec_iatt_combine(dst->iatt, src->iatt, 1))
-    {
-        gf_log(fop->xl->name, GF_LOG_NOTICE, "Mismatching iatt in "
-                                             "answers of 'GF_FOP_READLINK'");
+    if (!ec_iatt_combine(fop, dst->iatt, src->iatt, 1)) {
+        gf_msg(fop->xl->name, GF_LOG_NOTICE, 0, EC_MSG_IATT_MISMATCH,
+               "Mismatching iatt in "
+               "answers of 'GF_FOP_READLINK'");
 
         return 0;
     }
@@ -927,11 +968,13 @@ int32_t ec_combine_readlink(ec_fop_data_t * fop, ec_cbk_data_t * dst,
     return 1;
 }
 
-int32_t ec_readlink_cbk(call_frame_t * frame, void * cookie, xlator_t * this,
-                        int32_t op_ret, int32_t op_errno, const char * path,
-                        struct iatt * buf, dict_t * xdata)
+int32_t
+ec_readlink_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                int32_t op_ret, int32_t op_errno, const char *path,
+                struct iatt *buf, dict_t *xdata)
 {
-    ec_fop_data_t * fop = NULL;
+    ec_fop_data_t *fop = NULL;
+    ec_cbk_data_t *cbk = NULL;
     int32_t idx = (int32_t)(uintptr_t)cookie;
 
     VALIDATE_OR_GOTO(this, out);
@@ -941,33 +984,34 @@ int32_t ec_readlink_cbk(call_frame_t * frame, void * cookie, xlator_t * this,
 
     fop = frame->local;
 
-    ec_trace("CBK", fop, "idx=%d, frame=%p, op_ret=%d, op_errno=%d", idx,
-             frame, op_ret, op_errno);
+    ec_trace("CBK", fop, "idx=%d, frame=%p, op_ret=%d, op_errno=%d", idx, frame,
+             op_ret, op_errno);
 
-    if (op_ret > 0)
-    {
-        ec_iatt_rebuild(fop->xl->private, buf, 1, 1);
-    }
+    cbk = ec_cbk_data_allocate(frame, this, fop, fop->id, idx, op_ret,
+                               op_errno);
+    if (cbk) {
+        if (xdata)
+            cbk->xdata = dict_ref(xdata);
 
-    if (!ec_dispatch_one_retry(fop, idx, op_ret, op_errno))
-    {
-        if (fop->cbks.readlink != NULL)
-        {
-            fop->cbks.readlink(fop->req_frame, fop, this, op_ret, op_errno,
-                               path, buf, xdata);
+        if (cbk->op_ret >= 0) {
+            cbk->iatt[0] = *buf;
+            cbk->str = gf_strdup(path);
+            if (!cbk->str) {
+                ec_cbk_set_error(cbk, ENOMEM, _gf_true);
+            }
         }
+        ec_combine(cbk, NULL);
     }
 
 out:
     if (fop != NULL)
-    {
         ec_complete(fop);
-    }
 
     return 0;
 }
 
-void ec_wind_readlink(ec_t * ec, ec_fop_data_t * fop, int32_t idx)
+void
+ec_wind_readlink(ec_t *ec, ec_fop_data_t *fop, int32_t idx)
 {
     ec_trace("WIND", fop, "idx=%d", idx);
 
@@ -976,75 +1020,114 @@ void ec_wind_readlink(ec_t * ec, ec_fop_data_t * fop, int32_t idx)
                       &fop->loc[0], fop->size, fop->xdata);
 }
 
-int32_t ec_manager_readlink(ec_fop_data_t * fop, int32_t state)
+int32_t
+ec_manager_readlink(ec_fop_data_t *fop, int32_t state)
 {
-    switch (state)
-    {
+    ec_cbk_data_t *cbk = NULL;
+
+    switch (state) {
         case EC_STATE_INIT:
+        case EC_STATE_LOCK:
+            ec_lock_prepare_inode(fop, &fop->loc[0], EC_QUERY_INFO, 0,
+                                  EC_RANGE_FULL);
+            ec_lock(fop);
+            return EC_STATE_DISPATCH;
+
         case EC_STATE_DISPATCH:
             ec_dispatch_one(fop);
 
+            return EC_STATE_PREPARE_ANSWER;
+
+        case EC_STATE_PREPARE_ANSWER:
+            if (ec_dispatch_one_retry(fop, &cbk)) {
+                return EC_STATE_DISPATCH;
+            }
+
+            if ((cbk != NULL) && (cbk->op_ret >= 0)) {
+                ec_iatt_rebuild(fop->xl->private, &cbk->iatt[0], 1, 1);
+            }
+
             return EC_STATE_REPORT;
 
+        case EC_STATE_REPORT:
+            cbk = fop->answer;
+            GF_ASSERT(cbk);
+            if (fop->cbks.readlink != NULL) {
+                fop->cbks.readlink(fop->req_frame, fop, fop->xl, cbk->op_ret,
+                                   cbk->op_errno, cbk->str, &cbk->iatt[0],
+                                   cbk->xdata);
+            }
+
+            return EC_STATE_LOCK_REUSE;
+
         case -EC_STATE_INIT:
+        case -EC_STATE_LOCK:
+        case -EC_STATE_DISPATCH:
+        case -EC_STATE_PREPARE_ANSWER:
         case -EC_STATE_REPORT:
-            if (fop->cbks.readlink != NULL)
-            {
-                fop->cbks.readlink(fop->req_frame, fop, fop->xl, -1,
-                                   fop->error, NULL, NULL, NULL);
+            if (fop->cbks.readlink != NULL) {
+                fop->cbks.readlink(fop->req_frame, fop, fop->xl, -1, fop->error,
+                                   NULL, NULL, NULL);
             }
+            return EC_STATE_LOCK_REUSE;
 
-        case EC_STATE_REPORT:
-            return EC_STATE_END;
+        case -EC_STATE_LOCK_REUSE:
+        case EC_STATE_LOCK_REUSE:
+            ec_lock_reuse(fop);
 
+            return EC_STATE_UNLOCK;
+
+        case -EC_STATE_UNLOCK:
+        case EC_STATE_UNLOCK:
+            ec_unlock(fop);
+
+            return EC_STATE_END;
         default:
-            gf_log(fop->xl->name, GF_LOG_ERROR, "Unhandled state %d for %s",
-                   state, ec_fop_name(fop->id));
+            gf_msg(fop->xl->name, GF_LOG_ERROR, EINVAL, EC_MSG_UNHANDLED_STATE,
+                   "Unhandled state %d for %s", state, ec_fop_name(fop->id));
 
             return EC_STATE_END;
     }
 }
 
-void ec_readlink(call_frame_t * frame, xlator_t * this, uintptr_t target,
-                 int32_t minimum, fop_readlink_cbk_t func, void * data,
-                 loc_t * loc, size_t size, dict_t * xdata)
+void
+ec_readlink(call_frame_t *frame, xlator_t *this, uintptr_t target,
+            uint32_t fop_flags, fop_readlink_cbk_t func, void *data, loc_t *loc,
+            size_t size, dict_t *xdata)
 {
-    ec_cbk_t callback = { .readlink = func };
-    ec_fop_data_t * fop = NULL;
-    int32_t error = EIO;
+    ec_cbk_t callback = {.readlink = func};
+    ec_fop_data_t *fop = NULL;
+    int32_t error = ENOMEM;
 
-    gf_log("ec", GF_LOG_TRACE, "EC(READLINK) %p", frame);
+    gf_msg_trace("ec", 0, "EC(READLINK) %p", frame);
 
     VALIDATE_OR_GOTO(this, out);
     GF_VALIDATE_OR_GOTO(this->name, frame, out);
     GF_VALIDATE_OR_GOTO(this->name, this->private, out);
 
-    fop = ec_fop_data_allocate(frame, this, GF_FOP_READLINK, 0, target,
-                               minimum, ec_wind_readlink, ec_manager_readlink,
-                               callback, data);
-    if (fop == NULL)
-    {
+    fop = ec_fop_data_allocate(
+        frame, this, GF_FOP_READLINK, EC_FLAG_LOCK_SHARED, target, fop_flags,
+        ec_wind_readlink, ec_manager_readlink, callback, data);
+    if (fop == NULL) {
         goto out;
     }
 
     fop->size = size;
 
-    if (loc != NULL)
-    {
-        if (loc_copy(&fop->loc[0], loc) != 0)
-        {
-            gf_log(this->name, GF_LOG_ERROR, "Failed to copy a location.");
+    if (loc != NULL) {
+        if (loc_copy(&fop->loc[0], loc) != 0) {
+            gf_msg(this->name, GF_LOG_ERROR, ENOMEM, EC_MSG_LOC_COPY_FAIL,
+                   "Failed to copy a location.");
 
             goto out;
         }
     }
-    if (xdata != NULL)
-    {
+    if (xdata != NULL) {
         fop->xdata = dict_ref(xdata);
-        if (fop->xdata == NULL)
-        {
-            gf_log(this->name, GF_LOG_ERROR, "Failed to reference a "
-                                             "dictionary.");
+        if (fop->xdata == NULL) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_DICT_REF_FAIL,
+                   "Failed to reference a "
+                   "dictionary.");
 
             goto out;
         }
@@ -1053,95 +1136,82 @@ void ec_readlink(call_frame_t * frame, xlator_t * this, uintptr_t target,
     error = 0;
 
 out:
-    if (fop != NULL)
-    {
+    if (fop != NULL) {
         ec_manager(fop, error);
-    }
-    else
-    {
-        func(frame, NULL, this, -1, EIO, NULL, NULL, NULL);
+    } else {
+        func(frame, NULL, this, -1, error, NULL, NULL, NULL);
     }
 }
 
 /* FOP: readv */
 
-int32_t ec_readv_rebuild(ec_t * ec, ec_fop_data_t * fop, ec_cbk_data_t * cbk)
+int32_t
+ec_readv_rebuild(ec_t *ec, ec_fop_data_t *fop, ec_cbk_data_t *cbk)
 {
-    ec_cbk_data_t * ans = NULL;
-    struct iobref * iobref = NULL;
-    struct iobuf * iobuf = NULL;
-    uint8_t * buff = NULL, * ptr;
-    size_t fsize = 0, size = 0, max = 0;
-    int32_t i = 0;
-
-    if (cbk->op_ret < 0)
-    {
+    struct iovec vector[1];
+    ec_cbk_data_t *ans = NULL;
+    struct iobref *iobref = NULL;
+    void *ptr;
+    uint64_t fsize = 0, size = 0, max = 0;
+    int32_t pos, err = -ENOMEM;
+
+    if (cbk->op_ret < 0) {
+        err = -cbk->op_errno;
+
         goto out;
     }
 
-    cbk->iatt[0].ia_size = fop->pre_size;
+    /* This shouldn't fail because we have the inode locked. */
+    GF_ASSERT(ec_get_inode_size(fop, fop->fd->inode, &cbk->iatt[0].ia_size));
 
-    if (cbk->op_ret > 0)
-    {
-        struct iovec vector[1];
-        uint8_t * blocks[cbk->count];
+    if (cbk->op_ret > 0) {
+        void *blocks[cbk->count];
         uint32_t values[cbk->count];
 
         fsize = cbk->op_ret;
         size = fsize * ec->fragments;
-        buff = GF_MALLOC(size, gf_common_mt_char);
-        if (buff == NULL)
-        {
-            goto out;
-        }
-        ptr = buff;
-        for (i = 0, ans = cbk; ans != NULL; i++, ans = ans->next)
-        {
-            values[i] = ans->idx;
-            blocks[i] = ptr;
-            ptr += ec_iov_copy_to(ptr, ans->vector, ans->int32, 0, fsize);
+        for (ans = cbk; ans != NULL; ans = ans->next) {
+            pos = gf_bits_count(cbk->mask & ((1 << ans->idx) - 1));
+            values[pos] = ans->idx + 1;
+            blocks[pos] = ans->vector[0].iov_base;
+            if ((ans->int32 != 1) ||
+                !EC_ALIGN_CHECK(blocks[pos], EC_METHOD_WORD_SIZE)) {
+                if (iobref == NULL) {
+                    err = ec_buffer_alloc(ec->xl, size, &iobref, &ptr);
+                    if (err != 0) {
+                        goto out;
+                    }
+                }
+                ec_iov_copy_to(ptr, ans->vector, ans->int32, 0, fsize);
+                blocks[pos] = ptr;
+                ptr += fsize;
+            }
         }
 
-        iobref = iobref_new();
-        if (iobref == NULL)
-        {
+        err = ec_buffer_alloc(ec->xl, size, &iobref, &ptr);
+        if (err != 0) {
             goto out;
         }
-        iobuf = iobuf_get2(fop->xl->ctx->iobuf_pool, size);
-        if (iobuf == NULL)
-        {
-            goto out;
-        }
-        if (iobref_add(iobref, iobuf) != 0)
-        {
+
+        err = ec_method_decode(&ec->matrix, fsize, cbk->mask, values, blocks,
+                               ptr);
+        if (err != 0) {
             goto out;
         }
 
-        vector[0].iov_base = iobuf->ptr;
-        vector[0].iov_len = ec_method_decode(fsize, ec->fragments, values,
-                                             blocks, iobuf->ptr);
-
-        iobuf_unref(iobuf);
-
-        GF_FREE(buff);
-        buff = NULL;
-
-        vector[0].iov_base += fop->head;
-        vector[0].iov_len -= fop->head;
+        vector[0].iov_base = ptr + fop->head;
+        vector[0].iov_len = size - fop->head;
 
         max = fop->offset * ec->fragments + size;
-        if (max > cbk->iatt[0].ia_size)
-        {
+        if (max > cbk->iatt[0].ia_size) {
             max = cbk->iatt[0].ia_size;
         }
         max -= fop->offset * ec->fragments + fop->head;
-        if (max > fop->user_size)
-        {
+        if (max > fop->user_size) {
             max = fop->user_size;
         }
         size -= fop->head;
-        if (size > max)
-        {
+        if (size > max) {
             vector[0].iov_len -= size - max;
             size = max;
         }
@@ -1154,50 +1224,36 @@ int32_t ec_readv_rebuild(ec_t * ec, ec_fop_data_t * fop, ec_cbk_data_t * cbk)
 
         GF_FREE(cbk->vector);
         cbk->vector = iov_dup(vector, 1);
-        if (cbk->vector == NULL)
-        {
-            cbk->op_ret = -1;
-            cbk->op_errno = EIO;
-
-            return 0;
+        if (cbk->vector == NULL) {
+            return -ENOMEM;
         }
     }
 
-    return 1;
+    return 0;
 
 out:
-    if (iobuf != NULL)
-    {
-        iobuf_unref(iobuf);
-    }
-    if (iobref != NULL)
-    {
+    if (iobref != NULL) {
         iobref_unref(iobref);
     }
-    GF_FREE(buff);
 
-    return 0;
+    return err;
 }
 
-int32_t ec_combine_readv(ec_fop_data_t * fop, ec_cbk_data_t * dst,
-                         ec_cbk_data_t * src)
+int32_t
+ec_combine_readv(ec_fop_data_t *fop, ec_cbk_data_t *dst, ec_cbk_data_t *src)
 {
-    if (src->dirty) {
-        return 0;
-    }
-
-    if (!ec_vector_compare(dst->vector, dst->int32, src->vector, src->int32))
-    {
-        gf_log(fop->xl->name, GF_LOG_NOTICE, "Mismatching vector in "
-                                             "answers of 'GF_FOP_READ'");
+    if (!ec_vector_compare(dst->vector, dst->int32, src->vector, src->int32)) {
+        gf_msg(fop->xl->name, GF_LOG_NOTICE, 0, EC_MSG_VECTOR_MISMATCH,
+               "Mismatching vector in "
+               "answers of 'GF_FOP_READ'");
 
         return 0;
     }
 
-    if (!ec_iatt_combine(dst->iatt, src->iatt, 1))
-    {
-        gf_log(fop->xl->name, GF_LOG_NOTICE, "Mismatching iatt in "
-                                             "answers of 'GF_FOP_READ'");
+    if (!ec_iatt_combine(fop, dst->iatt, src->iatt, 1)) {
+        gf_msg(fop->xl->name, GF_LOG_NOTICE, 0, EC_MSG_IATT_MISMATCH,
+               "Mismatching iatt in "
+               "answers of 'GF_FOP_READ'");
 
         return 0;
     }
@@ -1205,14 +1261,14 @@ int32_t ec_combine_readv(ec_fop_data_t * fop, ec_cbk_data_t * dst,
     return 1;
 }
 
-int32_t ec_readv_cbk(call_frame_t * frame, void * cookie, xlator_t * this,
-                     int32_t op_ret, int32_t op_errno, struct iovec * vector,
-                     int32_t count, struct iatt * stbuf,
-                     struct iobref * iobref, dict_t * xdata)
+int32_t
+ec_readv_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int32_t op_ret,
+             int32_t op_errno, struct iovec *vector, int32_t count,
+             struct iatt *stbuf, struct iobref *iobref, dict_t *xdata)
 {
-    ec_fop_data_t * fop = NULL;
-    ec_cbk_data_t * cbk = NULL;
-    ec_t * ec = this->private;
+    ec_fop_data_t *fop = NULL;
+    ec_cbk_data_t *cbk = NULL;
+    ec_t *ec = this->private;
     int32_t idx = (int32_t)(uintptr_t)cookie;
 
     VALIDATE_OR_GOTO(this, out);
@@ -1222,76 +1278,68 @@ int32_t ec_readv_cbk(call_frame_t * frame, void * cookie, xlator_t * this,
 
     fop = frame->local;
 
-    ec_trace("CBK", fop, "idx=%d, frame=%p, op_ret=%d, op_errno=%d", idx,
-             frame, op_ret, op_errno);
+    ec_trace("CBK", fop, "idx=%d, frame=%p, op_ret=%d, op_errno=%d", idx, frame,
+             op_ret, op_errno);
 
     cbk = ec_cbk_data_allocate(frame, this, fop, GF_FOP_READ, idx, op_ret,
                                op_errno);
-    if (cbk != NULL)
-    {
-        if (op_ret >= 0)
-        {
+    if (cbk != NULL) {
+        if (op_ret >= 0) {
             cbk->int32 = count;
 
-            if (count > 0)
-            {
+            if (count > 0) {
                 cbk->vector = iov_dup(vector, count);
-                if (cbk->vector == NULL)
-                {
-                    gf_log(this->name, GF_LOG_ERROR, "Failed to duplicate a "
-                                                     "vector list.");
+                if (cbk->vector == NULL) {
+                    gf_msg(this->name, GF_LOG_ERROR, ENOMEM, EC_MSG_NO_MEMORY,
+                           "Failed to duplicate a "
+                           "vector list.");
 
                     goto out;
                 }
                 cbk->int32 = count;
             }
-            if (stbuf != NULL)
-            {
+            if (stbuf != NULL) {
                 cbk->iatt[0] = *stbuf;
             }
-            if (iobref != NULL)
-            {
+            if (iobref != NULL) {
                 cbk->buffers = iobref_ref(iobref);
-                if (cbk->buffers == NULL)
-                {
-                    gf_log(this->name, GF_LOG_ERROR, "Failed to reference a "
-                                                     "buffer.");
+                if (cbk->buffers == NULL) {
+                    gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_BUF_REF_FAIL,
+                           "Failed to reference a "
+                           "buffer.");
 
                     goto out;
                 }
             }
         }
-        if (xdata != NULL)
-        {
+        if (xdata != NULL) {
             cbk->xdata = dict_ref(xdata);
-            if (cbk->xdata == NULL)
-            {
-                gf_log(this->name, GF_LOG_ERROR, "Failed to reference a "
-                                                 "dictionary.");
+            if (cbk->xdata == NULL) {
+                gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_DICT_REF_FAIL,
+                       "Failed to reference a "
+                       "dictionary.");
 
                 goto out;
             }
         }
 
-        if ((op_ret > 0) && ((op_ret % ec->fragment_size) != 0))
-        {
-            cbk->op_ret = -1;
-            cbk->op_errno = EIO;
+        if ((op_ret > 0) && ((op_ret % ec->fragment_size) != 0)) {
+            ec_cbk_set_error(cbk, EIO, _gf_true);
         }
 
         ec_combine(cbk, ec_combine_readv);
     }
 
 out:
-    if (fop != NULL)
-    {
+    if (fop != NULL) {
         ec_complete(fop);
     }
 
     return 0;
 }
 
-void ec_wind_readv(ec_t * ec, ec_fop_data_t * fop, int32_t idx)
+void
+ec_wind_readv(ec_t *ec, ec_fop_data_t *fop, int32_t idx)
 {
     ec_trace("WIND", fop, "idx=%d", idx);
 
@@ -1300,67 +1348,49 @@ void ec_wind_readv(ec_t * ec, ec_fop_data_t * fop, int32_t idx)
                       fop->size, fop->offset, fop->uint32, fop->xdata);
 }
 
-int32_t ec_manager_readv(ec_fop_data_t * fop, int32_t state)
+int32_t
+ec_manager_readv(ec_fop_data_t *fop, int32_t state)
 {
-    ec_cbk_data_t * cbk;
+    ec_cbk_data_t *cbk;
+    ec_t *ec = fop->xl->private;
 
-    switch (state)
-    {
+    switch (state) {
         case EC_STATE_INIT:
             fop->user_size = fop->size;
-            fop->head = ec_adjust_offset(fop->xl->private, &fop->offset, 1);
-            fop->size = ec_adjust_size(fop->xl->private, fop->size + fop->head,
-                                       1);
+            fop->head = ec_adjust_offset_down(fop->xl->private, &fop->offset,
+                                              _gf_true);
+            fop->size += fop->head;
+            ec_adjust_size_up(fop->xl->private, &fop->size, _gf_true);
 
-        /* Fall through */
+            /* Fall through */
 
         case EC_STATE_LOCK:
-            ec_lock_prepare_fd(fop, fop->fd, 0);
+            ec_lock_prepare_fd(fop, fop->fd, EC_QUERY_INFO, fop->offset,
+                               fop->size);
             ec_lock(fop);
 
-            return EC_STATE_GET_SIZE_AND_VERSION;
-
-        case EC_STATE_GET_SIZE_AND_VERSION:
-            ec_get_size_version(fop);
-
             return EC_STATE_DISPATCH;
 
         case EC_STATE_DISPATCH:
+            if (ec->read_mask) {
+                fop->mask &= ec->read_mask;
+            }
             ec_dispatch_min(fop);
 
             return EC_STATE_PREPARE_ANSWER;
 
         case EC_STATE_PREPARE_ANSWER:
-            cbk = fop->answer;
-            if (cbk != NULL)
-            {
-                if (!ec_dict_combine(cbk, EC_COMBINE_XDATA))
-                {
-                    if (cbk->op_ret >= 0)
-                    {
-                        cbk->op_ret = -1;
-                        cbk->op_errno = EIO;
-                    }
-                }
-                if (cbk->op_ret < 0)
-                {
-                    ec_fop_set_error(fop, cbk->op_errno);
-                }
-                else
-                {
-                    ec_iatt_rebuild(fop->xl->private, cbk->iatt, 1,
-                                    cbk->count);
-
-                    if (!ec_readv_rebuild(fop->xl->private, fop, cbk))
-                    {
-                        ec_fop_set_error(fop, EIO);
-                    }
+            cbk = ec_fop_prepare_answer(fop, _gf_true);
+            if (cbk != NULL) {
+                int32_t err;
+
+                ec_iatt_rebuild(fop->xl->private, cbk->iatt, 1, cbk->count);
+
+                err = ec_readv_rebuild(fop->xl->private, fop, cbk);
+                if (err != 0) {
+                    ec_cbk_set_error(cbk, -err, _gf_true);
                 }
             }
-            else
-            {
-                ec_fop_set_error(fop, EIO);
-            }
 
             return EC_STATE_REPORT;
 
@@ -1369,8 +1399,7 @@ int32_t ec_manager_readv(ec_fop_data_t * fop, int32_t state)
 
             GF_ASSERT(cbk != NULL);
 
-            if (fop->cbks.readv != NULL)
-            {
+            if (fop->cbks.readv != NULL) {
                 fop->cbks.readv(fop->req_frame, fop, fop->xl, cbk->op_ret,
                                 cbk->op_errno, cbk->vector, cbk->int32,
                                 &cbk->iatt[0], cbk->buffers, cbk->xdata);
@@ -1380,14 +1409,12 @@ int32_t ec_manager_readv(ec_fop_data_t * fop, int32_t state)
 
         case -EC_STATE_INIT:
         case -EC_STATE_LOCK:
-        case -EC_STATE_GET_SIZE_AND_VERSION:
         case -EC_STATE_DISPATCH:
         case -EC_STATE_PREPARE_ANSWER:
         case -EC_STATE_REPORT:
             GF_ASSERT(fop->error != 0);
 
-            if (fop->cbks.readv != NULL)
-            {
+            if (fop->cbks.readv != NULL) {
                 fop->cbks.readv(fop->req_frame, fop, fop->xl, -1, fop->error,
                                 NULL, 0, NULL, NULL, NULL);
             }
@@ -1407,32 +1434,32 @@ int32_t ec_manager_readv(ec_fop_data_t * fop, int32_t state)
             return EC_STATE_END;
 
         default:
-            gf_log(fop->xl->name, GF_LOG_ERROR, "Unhandled state %d for %s",
-                   state, ec_fop_name(fop->id));
+            gf_msg(fop->xl->name, GF_LOG_ERROR, EINVAL, EC_MSG_UNHANDLED_STATE,
+                   "Unhandled state %d for %s", state, ec_fop_name(fop->id));
 
             return EC_STATE_END;
     }
 }
 
-void ec_readv(call_frame_t * frame, xlator_t * this, uintptr_t target,
-              int32_t minimum, fop_readv_cbk_t func, void * data, fd_t * fd,
-              size_t size, off_t offset, uint32_t flags, dict_t * xdata)
+void
+ec_readv(call_frame_t *frame, xlator_t *this, uintptr_t target,
+         uint32_t fop_flags, fop_readv_cbk_t func, void *data, fd_t *fd,
+         size_t size, off_t offset, uint32_t flags, dict_t *xdata)
 {
-    ec_cbk_t callback = { .readv = func };
-    ec_fop_data_t * fop = NULL;
-    int32_t error = EIO;
+    ec_cbk_t callback = {.readv = func};
+    ec_fop_data_t *fop = NULL;
+    int32_t error = ENOMEM;
 
-    gf_log("ec", GF_LOG_TRACE, "EC(READ) %p", frame);
+    gf_msg_trace("ec", 0, "EC(READ) %p", frame);
 
     VALIDATE_OR_GOTO(this, out);
     GF_VALIDATE_OR_GOTO(this->name, frame, out);
     GF_VALIDATE_OR_GOTO(this->name, this->private, out);
 
-    fop = ec_fop_data_allocate(frame, this, GF_FOP_READ, EC_FLAG_UPDATE_FD,
-                               target, minimum, ec_wind_readv,
+    fop = ec_fop_data_allocate(frame, this, GF_FOP_READ, EC_FLAG_LOCK_SHARED,
+                               target, fop_flags, ec_wind_readv,
                                ec_manager_readv, callback, data);
-    if (fop == NULL)
-    {
+    if (fop == NULL) {
         goto out;
     }
 
@@ -1442,24 +1469,22 @@ void ec_readv(call_frame_t * frame, xlator_t * this, uintptr_t target,
     fop->offset = offset;
     fop->uint32 = flags;
 
-    if (fd != NULL)
-    {
+    if (fd != NULL) {
         fop->fd = fd_ref(fd);
-        if (fop->fd == NULL)
-        {
-            gf_log(this->name, GF_LOG_ERROR, "Failed to reference a "
-                                             "file descriptor.");
+        if (fop->fd == NULL) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_FILE_DESC_REF_FAIL,
+                   "Failed to reference a "
+                   "file descriptor.");
 
             goto out;
         }
     }
-    if (xdata != NULL)
-    {
+    if (xdata != NULL) {
         fop->xdata = dict_ref(xdata);
-        if (fop->xdata == NULL)
-        {
-            gf_log(this->name, GF_LOG_ERROR, "Failed to reference a "
-                                             "dictionary.");
+        if (fop->xdata == NULL) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_DICT_REF_FAIL,
+                   "Failed to reference a "
+                   "dictionary.");
 
             goto out;
         }
@@ -1468,25 +1493,227 @@ void ec_readv(call_frame_t * frame, xlator_t * this, uintptr_t target,
     error = 0;
 
 out:
-    if (fop != NULL)
-    {
+    if (fop != NULL) {
         ec_manager(fop, error);
+    } else {
+        func(frame, NULL, this, -1, error, NULL, 0, NULL, NULL, NULL);
+    }
+}
+
+/* FOP: seek */
+
+int32_t
+ec_seek_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int32_t op_ret,
+            int32_t op_errno, off_t offset, dict_t *xdata)
+{
+    ec_fop_data_t *fop = NULL;
+    ec_cbk_data_t *cbk = NULL;
+    ec_t *ec = this->private;
+    int32_t idx = (int32_t)(uintptr_t)cookie;
+
+    VALIDATE_OR_GOTO(this, out);
+    GF_VALIDATE_OR_GOTO(this->name, frame, out);
+    GF_VALIDATE_OR_GOTO(this->name, frame->local, out);
+    GF_VALIDATE_OR_GOTO(this->name, this->private, out);
+
+    fop = frame->local;
+
+    ec_trace("CBK", fop, "idx=%d, frame=%p, op_ret=%d, op_errno=%d", idx, frame,
+             op_ret, op_errno);
+
+    cbk = ec_cbk_data_allocate(frame, this, fop, GF_FOP_SEEK, idx, op_ret,
+                               op_errno);
+    if (cbk != NULL) {
+        if (op_ret >= 0) {
+            cbk->offset = offset;
+        }
+        if (xdata != NULL) {
+            cbk->xdata = dict_ref(xdata);
+        }
+
+        if ((op_ret > 0) && ((cbk->offset % ec->fragment_size) != 0)) {
+            cbk->op_ret = -1;
+            cbk->op_errno = EIO;
+        }
+
+        ec_combine(cbk, NULL);
     }
-    else
-    {
-        func(frame, NULL, this, -1, EIO, NULL, 0, NULL, NULL, NULL);
+
+out:
+    if (fop != NULL) {
+        ec_complete(fop);
+    }
+
+    return 0;
+}
+
+void
+ec_wind_seek(ec_t *ec, ec_fop_data_t *fop, int32_t idx)
+{
+    ec_trace("WIND", fop, "idx=%d", idx);
+
+    STACK_WIND_COOKIE(fop->frame, ec_seek_cbk, (void *)(uintptr_t)idx,
+                      ec->xl_list[idx], ec->xl_list[idx]->fops->seek, fop->fd,
+                      fop->offset, fop->seek, fop->xdata);
+}
+
+int32_t
+ec_manager_seek(ec_fop_data_t *fop, int32_t state)
+{
+    ec_cbk_data_t *cbk;
+    uint64_t size;
+
+    switch (state) {
+        case EC_STATE_INIT:
+            fop->user_size = fop->offset;
+            fop->head = ec_adjust_offset_down(fop->xl->private, &fop->offset,
+                                              _gf_true);
+
+            /* Fall through */
+
+        case EC_STATE_LOCK:
+            ec_lock_prepare_fd(fop, fop->fd, EC_QUERY_INFO, fop->offset,
+                               EC_RANGE_FULL);
+            ec_lock(fop);
+
+            return EC_STATE_DISPATCH;
+
+        case EC_STATE_DISPATCH:
+            /* This shouldn't fail because we have the inode locked. */
+            GF_ASSERT(
+                ec_get_inode_size(fop, fop->locks[0].lock->loc.inode, &size));
+
+            if (fop->user_size >= size) {
+                ec_fop_set_error(fop, ENXIO);
+
+                return EC_STATE_REPORT;
+            }
+
+            ec_dispatch_one(fop);
+
+            return EC_STATE_PREPARE_ANSWER;
+
+        case EC_STATE_PREPARE_ANSWER:
+            if (ec_dispatch_one_retry(fop, &cbk)) {
+                return EC_STATE_DISPATCH;
+            }
+            if ((cbk != NULL) && (cbk->op_ret >= 0)) {
+                ec_t *ec = fop->xl->private;
+
+                /* This shouldn't fail because we have the inode locked. */
+                GF_ASSERT(ec_get_inode_size(fop, fop->locks[0].lock->loc.inode,
+                                            &size));
+
+                cbk->offset *= ec->fragments;
+                if (cbk->offset < fop->user_size) {
+                    cbk->offset = fop->user_size;
+                }
+                if (cbk->offset > size) {
+                    cbk->offset = size;
+                }
+            }
+
+            return EC_STATE_REPORT;
+
+        case EC_STATE_REPORT:
+            cbk = fop->answer;
+
+            GF_ASSERT(cbk != NULL);
+
+            if (fop->cbks.seek != NULL) {
+                fop->cbks.seek(fop->req_frame, fop, fop->xl, cbk->op_ret,
+                               cbk->op_errno, cbk->offset, cbk->xdata);
+            }
+
+            return EC_STATE_LOCK_REUSE;
+
+        case -EC_STATE_INIT:
+        case -EC_STATE_LOCK:
+        case -EC_STATE_DISPATCH:
+        case -EC_STATE_PREPARE_ANSWER:
+        case -EC_STATE_REPORT:
+            GF_ASSERT(fop->error != 0);
+
+            if (fop->cbks.seek != NULL) {
+                fop->cbks.seek(fop->req_frame, fop, fop->xl, -1, fop->error, 0,
+                               NULL);
+            }
+
+            return EC_STATE_LOCK_REUSE;
+
+        case -EC_STATE_LOCK_REUSE:
+        case EC_STATE_LOCK_REUSE:
+            ec_lock_reuse(fop);
+
+            return EC_STATE_UNLOCK;
+
+        case -EC_STATE_UNLOCK:
+        case EC_STATE_UNLOCK:
+            ec_unlock(fop);
+
+            return EC_STATE_END;
+
+        default:
+            gf_msg(fop->xl->name, GF_LOG_ERROR, 0, EC_MSG_UNHANDLED_STATE,
+                   "Unhandled state %d for %s", state, ec_fop_name(fop->id));
+
+            return EC_STATE_END;
+    }
+}
+
+void
+ec_seek(call_frame_t *frame, xlator_t *this, uintptr_t target,
+        uint32_t fop_flags, fop_seek_cbk_t func, void *data, fd_t *fd,
+        off_t offset, gf_seek_what_t what, dict_t *xdata)
+{
+    ec_cbk_t callback = {.seek = func};
+    ec_fop_data_t *fop = NULL;
+    int32_t error = EIO;
+
+    gf_msg_trace("ec", 0, "EC(SEEK) %p", frame);
+
+    VALIDATE_OR_GOTO(this, out);
+    GF_VALIDATE_OR_GOTO(this->name, frame, out);
+    GF_VALIDATE_OR_GOTO(this->name, this->private, out);
+
+    fop = ec_fop_data_allocate(frame, this, GF_FOP_SEEK, EC_FLAG_LOCK_SHARED,
+                               target, fop_flags, ec_wind_seek, ec_manager_seek,
+                               callback, data);
+    if (fop == NULL) {
+        goto out;
+    }
+
+    fop->use_fd = 1;
+
+    fop->offset = offset;
+    fop->seek = what;
+
+    if (fd != NULL) {
+        fop->fd = fd_ref(fd);
+    }
+    if (xdata != NULL) {
+        fop->xdata = dict_ref(xdata);
+    }
+
+    error = 0;
+
+out:
+    if (fop != NULL) {
+        ec_manager(fop, error);
+    } else {
+        func(frame, NULL, this, -1, EIO, 0, NULL);
     }
 }
 
 /* FOP: stat */
 
-int32_t ec_combine_stat(ec_fop_data_t * fop, ec_cbk_data_t * dst,
-                        ec_cbk_data_t * src)
+int32_t
+ec_combine_stat(ec_fop_data_t *fop, ec_cbk_data_t *dst, ec_cbk_data_t *src)
 {
-    if (!ec_iatt_combine(dst->iatt, src->iatt, 1))
-    {
-        gf_log(fop->xl->name, GF_LOG_NOTICE, "Mismatching iatt in "
-                                             "answers of 'GF_FOP_STAT'");
+    if (!ec_iatt_combine(fop, dst->iatt, src->iatt, 1)) {
+        gf_msg(fop->xl->name, GF_LOG_NOTICE, 0, EC_MSG_IATT_MISMATCH,
+               "Mismatching iatt in "
+               "answers of 'GF_FOP_STAT'");
 
         return 0;
     }
@@ -1494,12 +1721,12 @@ int32_t ec_combine_stat(ec_fop_data_t * fop, ec_cbk_data_t * dst,
     return 1;
 }
 
-int32_t ec_stat_cbk(call_frame_t * frame, void * cookie, xlator_t * this,
-                    int32_t op_ret, int32_t op_errno, struct iatt * buf,
-                    dict_t * xdata)
+int32_t
+ec_stat_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int32_t op_ret,
+            int32_t op_errno, struct iatt *buf, dict_t *xdata)
 {
-    ec_fop_data_t * fop = NULL;
-    ec_cbk_data_t * cbk = NULL;
+    ec_fop_data_t *fop = NULL;
+    ec_cbk_data_t *cbk = NULL;
     int32_t idx = (int32_t)(uintptr_t)cookie;
 
     VALIDATE_OR_GOTO(this, out);
@@ -1509,27 +1736,23 @@ int32_t ec_stat_cbk(call_frame_t * frame, void * cookie, xlator_t * this,
 
     fop = frame->local;
 
-    ec_trace("CBK", fop, "idx=%d, frame=%p, op_ret=%d, op_errno=%d", idx,
-             frame, op_ret, op_errno);
+    ec_trace("CBK", fop, "idx=%d, frame=%p, op_ret=%d, op_errno=%d", idx, frame,
+             op_ret, op_errno);
 
     cbk = ec_cbk_data_allocate(frame, this, fop, GF_FOP_STAT, idx, op_ret,
                                op_errno);
-    if (cbk != NULL)
-    {
-        if (op_ret >= 0)
-        {
-            if (buf != NULL)
-            {
+    if (cbk != NULL) {
+        if (op_ret >= 0) {
+            if (buf != NULL) {
                 cbk->iatt[0] = *buf;
             }
         }
-        if (xdata != NULL)
-        {
+        if (xdata != NULL) {
             cbk->xdata = dict_ref(xdata);
-            if (cbk->xdata == NULL)
-            {
-                gf_log(this->name, GF_LOG_ERROR, "Failed to reference a "
-                                                 "dictionary.");
+            if (cbk->xdata == NULL) {
+                gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_DICT_REF_FAIL,
+                       "Failed to reference a "
+                       "dictionary.");
 
                 goto out;
             }
@@ -1539,15 +1762,15 @@ int32_t ec_stat_cbk(call_frame_t * frame, void * cookie, xlator_t * this,
     }
 
 out:
-    if (fop != NULL)
-    {
+    if (fop != NULL) {
         ec_complete(fop);
     }
 
     return 0;
 }
 
-void ec_wind_stat(ec_t * ec, ec_fop_data_t * fop, int32_t idx)
+void
+ec_wind_stat(ec_t *ec, ec_fop_data_t *fop, int32_t idx)
 {
     ec_trace("WIND", fop, "idx=%d", idx);
 
@@ -1556,29 +1779,23 @@ void ec_wind_stat(ec_t * ec, ec_fop_data_t * fop, int32_t idx)
                       &fop->loc[0], fop->xdata);
 }
 
-int32_t ec_manager_stat(ec_fop_data_t * fop, int32_t state)
+int32_t
+ec_manager_stat(ec_fop_data_t *fop, int32_t state)
 {
-    ec_cbk_data_t * cbk;
+    ec_cbk_data_t *cbk;
 
-    switch (state)
-    {
+    switch (state) {
         case EC_STATE_INIT:
         case EC_STATE_LOCK:
-            if (fop->fd == NULL)
-            {
-                ec_lock_prepare_inode(fop, &fop->loc[0], 0);
-            }
-            else
-            {
-                ec_lock_prepare_fd(fop, fop->fd, 0);
+            if (fop->fd == NULL) {
+                ec_lock_prepare_inode(fop, &fop->loc[0], EC_QUERY_INFO, 0,
+                                      EC_RANGE_FULL);
+            } else {
+                ec_lock_prepare_fd(fop, fop->fd, EC_QUERY_INFO, 0,
+                                   EC_RANGE_FULL);
             }
             ec_lock(fop);
 
-            return EC_STATE_GET_SIZE_AND_VERSION;
-
-        case EC_STATE_GET_SIZE_AND_VERSION:
-            ec_get_size_version(fop);
-
             return EC_STATE_DISPATCH;
 
         case EC_STATE_DISPATCH:
@@ -1587,33 +1804,18 @@ int32_t ec_manager_stat(ec_fop_data_t * fop, int32_t state)
             return EC_STATE_PREPARE_ANSWER;
 
         case EC_STATE_PREPARE_ANSWER:
-            cbk = fop->answer;
-            if (cbk != NULL)
-            {
-                if (!ec_dict_combine(cbk, EC_COMBINE_XDATA))
-                {
-                    if (cbk->op_ret >= 0)
-                    {
-                        cbk->op_ret = -1;
-                        cbk->op_errno = EIO;
-                    }
-                }
-                if (cbk->op_ret < 0)
-                {
-                    ec_fop_set_error(fop, cbk->op_errno);
-                }
-                else
-                {
-                    ec_iatt_rebuild(fop->xl->private, cbk->iatt, 1,
-                                    cbk->count);
+            cbk = ec_fop_prepare_answer(fop, _gf_true);
 
-                    cbk->iatt[0].ia_size = fop->pre_size;
+            if (cbk != NULL) {
+                if (cbk->iatt[0].ia_type == IA_IFREG) {
+                    ec_iatt_rebuild(fop->xl->private, cbk->iatt, 1, cbk->count);
+
+                    /* This shouldn't fail because we have the inode locked. */
+                    GF_ASSERT(ec_get_inode_size(fop,
+                                                fop->locks[0].lock->loc.inode,
+                                                &cbk->iatt[0].ia_size));
                 }
             }
-            else
-            {
-                ec_fop_set_error(fop, EIO);
-            }
 
             return EC_STATE_REPORT;
 
@@ -1622,18 +1824,13 @@ int32_t ec_manager_stat(ec_fop_data_t * fop, int32_t state)
 
             GF_ASSERT(cbk != NULL);
 
-            if (fop->id == GF_FOP_STAT)
-            {
-                if (fop->cbks.stat != NULL)
-                {
+            if (fop->id == GF_FOP_STAT) {
+                if (fop->cbks.stat != NULL) {
                     fop->cbks.stat(fop->req_frame, fop, fop->xl, cbk->op_ret,
                                    cbk->op_errno, &cbk->iatt[0], cbk->xdata);
                 }
-            }
-            else
-            {
-                if (fop->cbks.fstat != NULL)
-                {
+            } else {
+                if (fop->cbks.fstat != NULL) {
                     fop->cbks.fstat(fop->req_frame, fop, fop->xl, cbk->op_ret,
                                     cbk->op_errno, &cbk->iatt[0], cbk->xdata);
                 }
@@ -1643,24 +1840,18 @@ int32_t ec_manager_stat(ec_fop_data_t * fop, int32_t state)
 
         case -EC_STATE_INIT:
         case -EC_STATE_LOCK:
-        case -EC_STATE_GET_SIZE_AND_VERSION:
         case -EC_STATE_DISPATCH:
         case -EC_STATE_PREPARE_ANSWER:
         case -EC_STATE_REPORT:
             GF_ASSERT(fop->error != 0);
 
-            if (fop->id == GF_FOP_STAT)
-            {
-                if (fop->cbks.stat != NULL)
-                {
-                    fop->cbks.stat(fop->req_frame, fop, fop->xl, -1,
-                                   fop->error, NULL, NULL);
+            if (fop->id == GF_FOP_STAT) {
+                if (fop->cbks.stat != NULL) {
+                    fop->cbks.stat(fop->req_frame, fop, fop->xl, -1, fop->error,
+                                   NULL, NULL);
                 }
-            }
-            else
-            {
-                if (fop->cbks.fstat != NULL)
-                {
+            } else {
+                if (fop->cbks.fstat != NULL) {
                     fop->cbks.fstat(fop->req_frame, fop, fop->xl, -1,
                                     fop->error, NULL, NULL);
                 }
@@ -1681,51 +1872,49 @@ int32_t ec_manager_stat(ec_fop_data_t * fop, int32_t state)
             return EC_STATE_END;
 
         default:
-            gf_log(fop->xl->name, GF_LOG_ERROR, "Unhandled state %d for %s",
-                   state, ec_fop_name(fop->id));
+            gf_msg(fop->xl->name, GF_LOG_ERROR, EINVAL, EC_MSG_UNHANDLED_STATE,
+                   "Unhandled state %d for %s", state, ec_fop_name(fop->id));
 
             return EC_STATE_END;
     }
 }
 
-void ec_stat(call_frame_t * frame, xlator_t * this, uintptr_t target,
-             int32_t minimum, fop_stat_cbk_t func, void * data, loc_t * loc,
-             dict_t * xdata)
+void
+ec_stat(call_frame_t *frame, xlator_t *this, uintptr_t target,
+        uint32_t fop_flags, fop_stat_cbk_t func, void *data, loc_t *loc,
+        dict_t *xdata)
 {
-    ec_cbk_t callback = { .stat = func };
-    ec_fop_data_t * fop = NULL;
-    int32_t error = EIO;
+    ec_cbk_t callback = {.stat = func};
+    ec_fop_data_t *fop = NULL;
+    int32_t error = ENOMEM;
 
-    gf_log("ec", GF_LOG_TRACE, "EC(STAT) %p", frame);
+    gf_msg_trace("ec", 0, "EC(STAT) %p", frame);
 
     VALIDATE_OR_GOTO(this, out);
     GF_VALIDATE_OR_GOTO(this->name, frame, out);
     GF_VALIDATE_OR_GOTO(this->name, this->private, out);
 
-    fop = ec_fop_data_allocate(frame, this, GF_FOP_STAT,
-                               EC_FLAG_UPDATE_LOC_INODE, target, minimum,
-                               ec_wind_stat, ec_manager_stat, callback, data);
-    if (fop == NULL)
-    {
+    fop = ec_fop_data_allocate(frame, this, GF_FOP_STAT, EC_FLAG_LOCK_SHARED,
+                               target, fop_flags, ec_wind_stat, ec_manager_stat,
+                               callback, data);
+    if (fop == NULL) {
         goto out;
     }
 
-    if (loc != NULL)
-    {
-        if (loc_copy(&fop->loc[0], loc) != 0)
-        {
-            gf_log(this->name, GF_LOG_ERROR, "Failed to copy a location.");
+    if (loc != NULL) {
+        if (loc_copy(&fop->loc[0], loc) != 0) {
+            gf_msg(this->name, GF_LOG_ERROR, ENOMEM, EC_MSG_LOC_COPY_FAIL,
+                   "Failed to copy a location.");
 
             goto out;
         }
     }
-    if (xdata != NULL)
-    {
+    if (xdata != NULL) {
         fop->xdata = dict_ref(xdata);
-        if (fop->xdata == NULL)
-        {
-            gf_log(this->name, GF_LOG_ERROR, "Failed to reference a "
-                                             "dictionary.");
+        if (fop->xdata == NULL) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_DICT_REF_FAIL,
+                   "Failed to reference a "
+                   "dictionary.");
 
             goto out;
         }
@@ -1734,24 +1923,21 @@ void ec_stat(call_frame_t * frame, xlator_t * this, uintptr_t target,
     error = 0;
 
 out:
-    if (fop != NULL)
-    {
+    if (fop != NULL) {
         ec_manager(fop, error);
-    }
-    else
-    {
-        func(frame, NULL, this, -1, EIO, NULL, NULL);
+    } else {
+        func(frame, NULL, this, -1, error, NULL, NULL);
     }
 }
 
 /* FOP: fstat */
 
-int32_t ec_fstat_cbk(call_frame_t * frame, void * cookie, xlator_t * this,
-                     int32_t op_ret, int32_t op_errno, struct iatt * buf,
-                     dict_t * xdata)
+int32_t
+ec_fstat_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int32_t op_ret,
+             int32_t op_errno, struct iatt *buf, dict_t *xdata)
 {
-    ec_fop_data_t * fop = NULL;
-    ec_cbk_data_t * cbk = NULL;
+    ec_fop_data_t *fop = NULL;
+    ec_cbk_data_t *cbk = NULL;
     int32_t idx = (int32_t)(uintptr_t)cookie;
 
     VALIDATE_OR_GOTO(this, out);
@@ -1761,27 +1947,23 @@ int32_t ec_fstat_cbk(call_frame_t * frame, void * cookie, xlator_t * this,
 
     fop = frame->local;
 
-    ec_trace("CBK", fop, "idx=%d, frame=%p, op_ret=%d, op_errno=%d", idx,
-             frame, op_ret, op_errno);
+    ec_trace("CBK", fop, "idx=%d, frame=%p, op_ret=%d, op_errno=%d", idx, frame,
+             op_ret, op_errno);
 
     cbk = ec_cbk_data_allocate(frame, this, fop, GF_FOP_FSTAT, idx, op_ret,
                                op_errno);
-    if (cbk != NULL)
-    {
-        if (op_ret >= 0)
-        {
-            if (buf != NULL)
-            {
+    if (cbk != NULL) {
+        if (op_ret >= 0) {
+            if (buf != NULL) {
                 cbk->iatt[0] = *buf;
             }
         }
-        if (xdata != NULL)
-        {
+        if (xdata != NULL) {
             cbk->xdata = dict_ref(xdata);
-            if (cbk->xdata == NULL)
-            {
-                gf_log(this->name, GF_LOG_ERROR, "Failed to reference a "
-                                                 "dictionary.");
+            if (cbk->xdata == NULL) {
+                gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_DICT_REF_FAIL,
+                       "Failed to reference a "
+                       "dictionary.");
 
                 goto out;
             }
@@ -1791,15 +1973,15 @@ int32_t ec_fstat_cbk(call_frame_t * frame, void * cookie, xlator_t * this,
     }
 
 out:
-    if (fop != NULL)
-    {
+    if (fop != NULL) {
         ec_complete(fop);
     }
 
     return 0;
 }
 
-void ec_wind_fstat(ec_t * ec, ec_fop_data_t * fop, int32_t idx)
+void
+ec_wind_fstat(ec_t *ec, ec_fop_data_t *fop, int32_t idx)
 {
     ec_trace("WIND", fop, "idx=%d", idx);
 
@@ -1808,48 +1990,46 @@ void ec_wind_fstat(ec_t * ec, ec_fop_data_t * fop, int32_t idx)
                       fop->xdata);
 }
 
-void ec_fstat(call_frame_t * frame, xlator_t * this, uintptr_t target,
-              int32_t minimum, fop_fstat_cbk_t func, void * data, fd_t * fd,
-              dict_t * xdata)
+void
+ec_fstat(call_frame_t *frame, xlator_t *this, uintptr_t target,
+         uint32_t fop_flags, fop_fstat_cbk_t func, void *data, fd_t *fd,
+         dict_t *xdata)
 {
-    ec_cbk_t callback = { .fstat = func };
-    ec_fop_data_t * fop = NULL;
-    int32_t error = EIO;
+    ec_cbk_t callback = {.fstat = func};
+    ec_fop_data_t *fop = NULL;
+    int32_t error = ENOMEM;
 
-    gf_log("ec", GF_LOG_TRACE, "EC(FSTAT) %p", frame);
+    gf_msg_trace("ec", 0, "EC(FSTAT) %p", frame);
 
     VALIDATE_OR_GOTO(this, out);
     GF_VALIDATE_OR_GOTO(this->name, frame, out);
     GF_VALIDATE_OR_GOTO(this->name, this->private, out);
 
-    fop = ec_fop_data_allocate(frame, this, GF_FOP_FSTAT,
-                               EC_FLAG_UPDATE_FD_INODE, target, minimum,
-                               ec_wind_fstat, ec_manager_stat, callback, data);
-    if (fop == NULL)
-    {
+    fop = ec_fop_data_allocate(frame, this, GF_FOP_FSTAT, EC_FLAG_LOCK_SHARED,
+                               target, fop_flags, ec_wind_fstat,
+                               ec_manager_stat, callback, data);
+    if (fop == NULL) {
         goto out;
     }
 
     fop->use_fd = 1;
 
-    if (fd != NULL)
-    {
+    if (fd != NULL) {
         fop->fd = fd_ref(fd);
-        if (fop->fd == NULL)
-        {
-            gf_log(this->name, GF_LOG_ERROR, "Failed to reference a "
-                                             "file descriptor.");
+        if (fop->fd == NULL) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_FILE_DESC_REF_FAIL,
+                   "Failed to reference a "
+                   "file descriptor.");
 
             goto out;
         }
     }
-    if (xdata != NULL)
-    {
+    if (xdata != NULL) {
         fop->xdata = dict_ref(xdata);
-        if (fop->xdata == NULL)
-        {
-            gf_log(this->name, GF_LOG_ERROR, "Failed to reference a "
-                                             "dictionary.");
+        if (fop->xdata == NULL) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_DICT_REF_FAIL,
+                   "Failed to reference a "
+                   "dictionary.");
 
             goto out;
         }
@@ -1858,12 +2038,9 @@ void ec_fstat(call_frame_t * frame, xlator_t * this, uintptr_t target,
     error = 0;
 
 out:
-    if (fop != NULL)
-    {
+    if (fop != NULL) {
         ec_manager(fop, error);
-    }
-    else
-    {
-        func(frame, NULL, this, -1, EIO, NULL, NULL);
+    } else {
+        func(frame, NULL, this, -1, error, NULL, NULL);
     }
 }
diff --git a/xlators/cluster/ec/src/ec-inode-write.c b/xlators/cluster/ec/src/ec-inode-write.c
index 6b485a26fbc..9b5fe2a7fdc 100644
--- a/xlators/cluster/ec/src/ec-inode-write.c
+++ b/xlators/cluster/ec/src/ec-inode-write.c
@@ -8,72 +8,163 @@
   cases as published by the Free Software Foundation.
 */
 
-#include "xlator.h"
-#include "defaults.h"
-
+#include "ec-messages.h"
 #include "ec-helpers.h"
 #include "ec-common.h"
 #include "ec-combine.h"
 #include "ec-method.h"
 #include "ec-fops.h"
+#include "ec-mem-types.h"
+
+int32_t
+ec_update_writev_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                     int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
+                     struct iatt *postbuf, dict_t *xdata)
+{
+    ec_fop_data_t *fop = cookie;
+    ec_cbk_data_t *cbk = NULL;
+    ec_fop_data_t *parent = fop->parent;
+    int i = 0;
+
+    ec_trace("UPDATE_WRITEV_CBK", cookie, "ret=%d, errno=%d, parent-fop=%s",
+             op_ret, op_errno, ec_fop_name(parent->id));
+
+    if (op_ret < 0) {
+        ec_fop_set_error(parent, op_errno);
+        goto out;
+    }
+    cbk = ec_cbk_data_allocate(parent->frame, this, parent, parent->id, 0,
+                               op_ret, op_errno);
+    if (!cbk) {
+        ec_fop_set_error(parent, ENOMEM);
+        goto out;
+    }
+
+    if (xdata)
+        cbk->xdata = dict_ref(xdata);
+
+    if (prebuf)
+        cbk->iatt[i++] = *prebuf;
+
+    if (postbuf)
+        cbk->iatt[i++] = *postbuf;
+
+    LOCK(&parent->lock);
+    {
+        parent->good &= fop->good;
+
+        if (gf_bits_count(parent->good) < parent->minimum) {
+            __ec_fop_set_error(parent, EIO);
+        } else if (fop->error == 0 && parent->answer == NULL) {
+            parent->answer = cbk;
+        }
+    }
+    UNLOCK(&parent->lock);
+out:
+    return 0;
+}
+
+static int32_t
+ec_update_write(ec_fop_data_t *fop, uintptr_t mask, off_t offset, uint64_t size)
+{
+    struct iobref *iobref = NULL;
+    struct iobuf *iobuf = NULL;
+    struct iovec vector;
+    int32_t err = -ENOMEM;
+
+    iobref = iobref_new();
+    if (iobref == NULL) {
+        goto out;
+    }
+    iobuf = iobuf_get(fop->xl->ctx->iobuf_pool);
+    if (iobuf == NULL) {
+        goto out;
+    }
+    err = iobref_add(iobref, iobuf);
+    if (err != 0) {
+        goto out;
+    }
+
+    if (fop->locks[0].lock)
+        ec_lock_update_good(fop->locks[0].lock, fop);
+    vector.iov_base = iobuf->ptr;
+    vector.iov_len = size;
+    memset(vector.iov_base, 0, vector.iov_len);
+
+    ec_writev(fop->frame, fop->xl, mask, fop->minimum, ec_update_writev_cbk,
+              NULL, fop->fd, &vector, 1, offset, 0, iobref, NULL);
+
+    err = 0;
+
+out:
+    if (iobuf != NULL) {
+        iobuf_unref(iobuf);
+    }
+    if (iobref != NULL) {
+        iobref_unref(iobref);
+    }
+
+    return err;
+}
 
 int
-ec_inode_write_cbk (call_frame_t *frame, xlator_t *this, void *cookie,
-                    int op_ret, int op_errno, struct iatt *prestat,
-                    struct iatt *poststat, dict_t *xdata)
+ec_inode_write_cbk(call_frame_t *frame, xlator_t *this, void *cookie,
+                   int op_ret, int op_errno, struct iatt *prestat,
+                   struct iatt *poststat, dict_t *xdata)
 {
-        ec_fop_data_t *fop = NULL;
-        ec_cbk_data_t *cbk = NULL;
-        int           i    = 0;
-        int           idx  = 0;
+    ec_fop_data_t *fop = NULL;
+    ec_cbk_data_t *cbk = NULL;
+    int i = 0;
+    int idx = 0;
 
-        VALIDATE_OR_GOTO (this, out);
-        GF_VALIDATE_OR_GOTO (this->name, frame, out);
-        GF_VALIDATE_OR_GOTO (this->name, frame->local, out);
-        GF_VALIDATE_OR_GOTO (this->name, this->private, out);
+    VALIDATE_OR_GOTO(this, out);
+    GF_VALIDATE_OR_GOTO(this->name, frame, out);
+    GF_VALIDATE_OR_GOTO(this->name, frame->local, out);
+    GF_VALIDATE_OR_GOTO(this->name, this->private, out);
 
-        fop = frame->local;
-        idx = (int32_t)(uintptr_t) cookie;
+    fop = frame->local;
+    idx = (int32_t)(uintptr_t)cookie;
 
-        ec_trace("CBK", fop, "idx=%d, frame=%p, op_ret=%d, op_errno=%d", idx,
-                 frame, op_ret, op_errno);
+    ec_trace("CBK", fop, "idx=%d, frame=%p, op_ret=%d, op_errno=%d", idx, frame,
+             op_ret, op_errno);
 
-        cbk = ec_cbk_data_allocate (frame, this, fop, fop->id, idx, op_ret,
-                                    op_errno);
-        if (!cbk)
-                goto out;
+    cbk = ec_cbk_data_allocate(frame, this, fop, fop->id, idx, op_ret,
+                               op_errno);
+    if (!cbk)
+        goto out;
 
-        if (op_ret < 0)
-                goto out;
+    if (op_ret < 0)
+        goto out;
 
-        if (xdata)
-                cbk->xdata = dict_ref (xdata);
+    if (xdata)
+        cbk->xdata = dict_ref(xdata);
 
-        if (prestat)
-                cbk->iatt[i++] = *prestat;
+    if (prestat)
+        cbk->iatt[i++] = *prestat;
 
-        if (poststat)
-                cbk->iatt[i++] = *poststat;
+    if (poststat)
+        cbk->iatt[i++] = *poststat;
 
 out:
-        if (cbk)
-                ec_combine (cbk, ec_combine_write);
+    if (cbk)
+        ec_combine(cbk, ec_combine_write);
 
-        if (fop)
-                ec_complete (fop);
-        return 0;
+    if (fop)
+        ec_complete(fop);
+    return 0;
 }
 /* FOP: removexattr */
 
-int32_t ec_removexattr_cbk (call_frame_t *frame, void *cookie,
-                            xlator_t *this, int32_t op_ret, int32_t op_errno,
-                            dict_t *xdata)
+int32_t
+ec_removexattr_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                   int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
-        return ec_inode_write_cbk (frame, this, cookie, op_ret, op_errno,
-                                   NULL, NULL, xdata);
+    return ec_inode_write_cbk(frame, this, cookie, op_ret, op_errno, NULL, NULL,
+                              xdata);
 }
 
-void ec_wind_removexattr(ec_t * ec, ec_fop_data_t * fop, int32_t idx)
+void
+ec_wind_removexattr(ec_t *ec, ec_fop_data_t *fop, int32_t idx)
 {
     ec_trace("WIND", fop, "idx=%d", idx);
 
@@ -83,170 +174,154 @@ void ec_wind_removexattr(ec_t * ec, ec_fop_data_t * fop, int32_t idx)
 }
 
 void
-ec_xattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this, int32_t op_ret,
-              int32_t op_errno, dict_t *xdata)
+ec_xattr_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int32_t op_ret,
+             int32_t op_errno, dict_t *xdata)
 {
-        ec_fop_data_t *fop = cookie;
-        switch (fop->id) {
+    ec_fop_data_t *fop = cookie;
+    switch (fop->id) {
         case GF_FOP_SETXATTR:
-                if (fop->cbks.setxattr) {
-                        fop->cbks.setxattr (frame, cookie, this, op_ret,
-                                            op_errno, xdata);
-                }
-                break;
+            if (fop->cbks.setxattr) {
+                QUORUM_CBK(fop->cbks.setxattr, fop, frame, cookie, this, op_ret,
+                           op_errno, xdata);
+            }
+            break;
         case GF_FOP_REMOVEXATTR:
-                if (fop->cbks.removexattr) {
-                        fop->cbks.removexattr (frame, cookie, this, op_ret,
-                                               op_errno, xdata);
-                }
-                break;
+            if (fop->cbks.removexattr) {
+                QUORUM_CBK(fop->cbks.removexattr, fop, frame, cookie, this,
+                           op_ret, op_errno, xdata);
+            }
+            break;
         case GF_FOP_FSETXATTR:
-                if (fop->cbks.fsetxattr) {
-                        fop->cbks.fsetxattr (frame, cookie, this, op_ret,
-                                             op_errno, xdata);
-                }
-                break;
+            if (fop->cbks.fsetxattr) {
+                QUORUM_CBK(fop->cbks.fsetxattr, fop, frame, cookie, this,
+                           op_ret, op_errno, xdata);
+            }
+            break;
         case GF_FOP_FREMOVEXATTR:
-                if (fop->cbks.fremovexattr) {
-                        fop->cbks.fremovexattr (frame, cookie, this, op_ret,
-                                                op_errno, xdata);
-                }
-                break;
-        }
+            if (fop->cbks.fremovexattr) {
+                QUORUM_CBK(fop->cbks.fremovexattr, fop, frame, cookie, this,
+                           op_ret, op_errno, xdata);
+            }
+            break;
+    }
 }
 
 int32_t
-ec_manager_xattr (ec_fop_data_t *fop, int32_t state)
+ec_manager_xattr(ec_fop_data_t *fop, int32_t state)
 {
-    ec_cbk_data_t * cbk;
+    ec_cbk_data_t *cbk;
 
-        switch (state) {
+    switch (state) {
         case EC_STATE_INIT:
         case EC_STATE_LOCK:
-                if (fop->fd == NULL)
-                    ec_lock_prepare_inode(fop, &fop->loc[0], 1);
-                else
-                    ec_lock_prepare_fd(fop, fop->fd, 1);
-
-                ec_lock(fop);
+            if (fop->fd == NULL) {
+                ec_lock_prepare_inode(fop, &fop->loc[0],
+                                      EC_UPDATE_META | EC_QUERY_INFO, 0,
+                                      EC_RANGE_FULL);
+            } else {
+                ec_lock_prepare_fd(fop, fop->fd, EC_UPDATE_META | EC_QUERY_INFO,
+                                   0, EC_RANGE_FULL);
+            }
+            ec_lock(fop);
 
-                return EC_STATE_DISPATCH;
+            return EC_STATE_DISPATCH;
 
         case EC_STATE_DISPATCH:
-                ec_dispatch_all(fop);
+            ec_dispatch_all(fop);
 
-                return EC_STATE_PREPARE_ANSWER;
+            return EC_STATE_PREPARE_ANSWER;
 
         case EC_STATE_PREPARE_ANSWER:
-                cbk = fop->answer;
-                if (cbk) {
-                        if (!ec_dict_combine (cbk, EC_COMBINE_XDATA)) {
-                                if (cbk->op_ret >= 0) {
-                                    cbk->op_ret = -1;
-                                    cbk->op_errno = EIO;
-                                }
-                        }
-
-                        if (cbk->op_ret < 0)
-                            ec_fop_set_error(fop, cbk->op_errno);
-                } else {
-                    ec_fop_set_error(fop, EIO);
-                }
+            ec_fop_prepare_answer(fop, _gf_false);
 
-                return EC_STATE_REPORT;
+            return EC_STATE_REPORT;
 
         case EC_STATE_REPORT:
-                cbk = fop->answer;
+            cbk = fop->answer;
 
-                GF_ASSERT(cbk != NULL);
+            GF_ASSERT(cbk != NULL);
 
-                ec_xattr_cbk (fop->req_frame, fop, fop->xl, cbk->op_ret,
-                              cbk->op_errno, cbk->xdata);
+            ec_xattr_cbk(fop->req_frame, fop, fop->xl, cbk->op_ret,
+                         cbk->op_errno, cbk->xdata);
 
-                    return EC_STATE_LOCK_REUSE;
+            return EC_STATE_LOCK_REUSE;
 
         case -EC_STATE_INIT:
         case -EC_STATE_LOCK:
         case -EC_STATE_DISPATCH:
         case -EC_STATE_PREPARE_ANSWER:
         case -EC_STATE_REPORT:
-                GF_ASSERT(fop->error != 0);
+            GF_ASSERT(fop->error != 0);
 
-                ec_xattr_cbk (fop->req_frame, fop, fop->xl, -1, fop->error,
-                              NULL);
+            ec_xattr_cbk(fop->req_frame, fop, fop->xl, -1, fop->error, NULL);
 
-                return EC_STATE_LOCK_REUSE;
+            return EC_STATE_LOCK_REUSE;
 
         case -EC_STATE_LOCK_REUSE:
         case EC_STATE_LOCK_REUSE:
-                ec_lock_reuse(fop);
+            ec_lock_reuse(fop);
 
-                return EC_STATE_UNLOCK;
+            return EC_STATE_UNLOCK;
 
         case -EC_STATE_UNLOCK:
         case EC_STATE_UNLOCK:
-                ec_unlock(fop);
+            ec_unlock(fop);
 
-                return EC_STATE_END;
+            return EC_STATE_END;
 
         default:
-                gf_log(fop->xl->name, GF_LOG_ERROR, "Unhandled state %d for %s",
-                       state, ec_fop_name(fop->id));
+            gf_msg(fop->xl->name, GF_LOG_ERROR, EINVAL, EC_MSG_UNHANDLED_STATE,
+                   "Unhandled state %d for %s", state, ec_fop_name(fop->id));
 
-                return EC_STATE_END;
-        }
+            return EC_STATE_END;
+    }
 }
 
 void
-ec_removexattr (call_frame_t *frame, xlator_t *this, uintptr_t target,
-                int32_t minimum, fop_removexattr_cbk_t func, void *data,
-                loc_t *loc, const char *name, dict_t *xdata)
+ec_removexattr(call_frame_t *frame, xlator_t *this, uintptr_t target,
+               uint32_t fop_flags, fop_removexattr_cbk_t func, void *data,
+               loc_t *loc, const char *name, dict_t *xdata)
 {
-    ec_cbk_t callback = { .removexattr = func };
-    ec_fop_data_t * fop = NULL;
-    int32_t error = EIO;
+    ec_cbk_t callback = {.removexattr = func};
+    ec_fop_data_t *fop = NULL;
+    int32_t error = ENOMEM;
 
-    gf_log("ec", GF_LOG_TRACE, "EC(REMOVEXATTR) %p", frame);
+    gf_msg_trace("ec", 0, "EC(REMOVEXATTR) %p", frame);
 
-    VALIDATE_OR_GOTO (this, out);
-    GF_VALIDATE_OR_GOTO (this->name, frame, out);
-    GF_VALIDATE_OR_GOTO (this->name, this->private, out);
+    VALIDATE_OR_GOTO(this, out);
+    GF_VALIDATE_OR_GOTO(this->name, frame, out);
+    GF_VALIDATE_OR_GOTO(this->name, this->private, out);
 
-    fop = ec_fop_data_allocate(frame, this, GF_FOP_REMOVEXATTR,
-                               EC_FLAG_UPDATE_LOC_INODE, target, minimum,
-                               ec_wind_removexattr, ec_manager_xattr,
+    fop = ec_fop_data_allocate(frame, this, GF_FOP_REMOVEXATTR, 0, target,
+                               fop_flags, ec_wind_removexattr, ec_manager_xattr,
                                callback, data);
-    if (fop == NULL)
-    {
+    if (fop == NULL) {
         goto out;
     }
 
-    if (loc != NULL)
-    {
-        if (loc_copy(&fop->loc[0], loc) != 0)
-        {
-            gf_log(this->name, GF_LOG_ERROR, "Failed to copy a location.");
+    if (loc != NULL) {
+        if (loc_copy(&fop->loc[0], loc) != 0) {
+            gf_msg(this->name, GF_LOG_ERROR, ENOMEM, EC_MSG_LOC_COPY_FAIL,
+                   "Failed to copy a location.");
 
             goto out;
         }
     }
-    if (name != NULL)
-    {
+    if (name != NULL) {
         fop->str[0] = gf_strdup(name);
-        if (fop->str[0] == NULL)
-        {
-            gf_log(this->name, GF_LOG_ERROR, "Failed to duplicate a string.");
+        if (fop->str[0] == NULL) {
+            gf_msg(this->name, GF_LOG_ERROR, ENOMEM, EC_MSG_NO_MEMORY,
+                   "Failed to duplicate a string.");
 
             goto out;
         }
     }
-    if (xdata != NULL)
-    {
-        fop->xdata = dict_ref(xdata);
-        if (fop->xdata == NULL)
-        {
-            gf_log(this->name, GF_LOG_ERROR, "Failed to reference a "
-                                             "dictionary.");
+    if (xdata != NULL) {
+        fop->xdata = dict_copy_with_ref(xdata, NULL);
+        if (fop->xdata == NULL) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_DICT_REF_FAIL,
+                   "Failed to reference a "
+                   "dictionary.");
 
             goto out;
         }
@@ -256,23 +331,24 @@ ec_removexattr (call_frame_t *frame, xlator_t *this, uintptr_t target,
 
 out:
     if (fop != NULL) {
-        ec_manager (fop, error);
+        ec_manager(fop, error);
     } else {
-        func (frame, NULL, this, -1, error, NULL);
+        func(frame, NULL, this, -1, error, NULL);
     }
 }
 
 /* FOP: fremovexattr */
 
-int32_t ec_fremovexattr_cbk (call_frame_t *frame, void *cookie,
-                             xlator_t *this, int32_t op_ret, int32_t op_errno,
-                             dict_t *xdata)
+int32_t
+ec_fremovexattr_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                    int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
-        return ec_inode_write_cbk (frame, this, cookie, op_ret, op_errno,
-                                   NULL, NULL, xdata);
+    return ec_inode_write_cbk(frame, this, cookie, op_ret, op_errno, NULL, NULL,
+                              xdata);
 }
 
-void ec_wind_fremovexattr(ec_t * ec, ec_fop_data_t * fop, int32_t idx)
+void
+ec_wind_fremovexattr(ec_t *ec, ec_fop_data_t *fop, int32_t idx)
 {
     ec_trace("WIND", fop, "idx=%d", idx);
 
@@ -282,59 +358,54 @@ void ec_wind_fremovexattr(ec_t * ec, ec_fop_data_t * fop, int32_t idx)
 }
 
 void
-ec_fremovexattr (call_frame_t *frame, xlator_t *this, uintptr_t target,
-                 int32_t minimum, fop_fremovexattr_cbk_t func, void *data,
-                 fd_t *fd, const char *name, dict_t *xdata)
+ec_fremovexattr(call_frame_t *frame, xlator_t *this, uintptr_t target,
+                uint32_t fop_flags, fop_fremovexattr_cbk_t func, void *data,
+                fd_t *fd, const char *name, dict_t *xdata)
 {
-    ec_cbk_t callback = { .fremovexattr = func };
-    ec_fop_data_t * fop = NULL;
-    int32_t error = EIO;
+    ec_cbk_t callback = {.fremovexattr = func};
+    ec_fop_data_t *fop = NULL;
+    int32_t error = ENOMEM;
 
-    gf_log("ec", GF_LOG_TRACE, "EC(FREMOVEXATTR) %p", frame);
+    gf_msg_trace("ec", 0, "EC(FREMOVEXATTR) %p", frame);
 
-    VALIDATE_OR_GOTO (this, out);
-    GF_VALIDATE_OR_GOTO (this->name, frame, out);
-    GF_VALIDATE_OR_GOTO (this->name, this->private, out);
+    VALIDATE_OR_GOTO(this, out);
+    GF_VALIDATE_OR_GOTO(this->name, frame, out);
+    GF_VALIDATE_OR_GOTO(this->name, this->private, out);
 
-    fop = ec_fop_data_allocate(frame, this, GF_FOP_FREMOVEXATTR,
-                               EC_FLAG_UPDATE_FD_INODE, target, minimum,
-                               ec_wind_fremovexattr, ec_manager_xattr,
-                               callback, data);
-    if (fop == NULL)
-    {
+    fop = ec_fop_data_allocate(frame, this, GF_FOP_FREMOVEXATTR, 0, target,
+                               fop_flags, ec_wind_fremovexattr,
+                               ec_manager_xattr, callback, data);
+    if (fop == NULL) {
         goto out;
     }
 
     fop->use_fd = 1;
 
-    if (fd != NULL)
-    {
+    if (fd != NULL) {
         fop->fd = fd_ref(fd);
-        if (fop->fd == NULL)
-        {
-            gf_log(this->name, GF_LOG_ERROR, "Failed to reference a "
-                                             "file descriptor.");
+        if (fop->fd == NULL) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_FILE_DESC_REF_FAIL,
+                   "Failed to reference a "
+                   "file descriptor.");
 
             goto out;
         }
     }
-    if (name != NULL)
-    {
+    if (name != NULL) {
         fop->str[0] = gf_strdup(name);
-        if (fop->str[0] == NULL)
-        {
-            gf_log(this->name, GF_LOG_ERROR, "Failed to duplicate a string.");
+        if (fop->str[0] == NULL) {
+            gf_msg(this->name, GF_LOG_ERROR, ENOMEM, EC_MSG_NO_MEMORY,
+                   "Failed to duplicate a string.");
 
             goto out;
         }
     }
-    if (xdata != NULL)
-    {
-        fop->xdata = dict_ref(xdata);
-        if (fop->xdata == NULL)
-        {
-            gf_log(this->name, GF_LOG_ERROR, "Failed to reference a "
-                                             "dictionary.");
+    if (xdata != NULL) {
+        fop->xdata = dict_copy_with_ref(xdata, NULL);
+        if (fop->xdata == NULL) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_DICT_REF_FAIL,
+                   "Failed to reference a "
+                   "dictionary.");
 
             goto out;
         }
@@ -344,24 +415,25 @@ ec_fremovexattr (call_frame_t *frame, xlator_t *this, uintptr_t target,
 
 out:
     if (fop != NULL) {
-        ec_manager (fop, error);
+        ec_manager(fop, error);
     } else {
-        func (frame, NULL, this, -1, error, NULL);
+        func(frame, NULL, this, -1, error, NULL);
     }
 }
 
 /* FOP: setattr */
 
-int32_t ec_setattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                        int32_t op_ret, int32_t op_errno,
-                        struct iatt *prestat, struct iatt *poststat,
-                        dict_t *xdata)
+int32_t
+ec_setattr_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+               int32_t op_ret, int32_t op_errno, struct iatt *prestat,
+               struct iatt *poststat, dict_t *xdata)
 {
-        return ec_inode_write_cbk (frame, this, cookie, op_ret, op_errno,
-                                   prestat, poststat, xdata);
+    return ec_inode_write_cbk(frame, this, cookie, op_ret, op_errno, prestat,
+                              poststat, xdata);
 }
 
-void ec_wind_setattr(ec_t * ec, ec_fop_data_t * fop, int32_t idx)
+void
+ec_wind_setattr(ec_t *ec, ec_fop_data_t *fop, int32_t idx)
 {
     ec_trace("WIND", fop, "idx=%d", idx);
 
@@ -370,29 +442,24 @@ void ec_wind_setattr(ec_t * ec, ec_fop_data_t * fop, int32_t idx)
                       &fop->loc[0], &fop->iatt, fop->int32, fop->xdata);
 }
 
-int32_t ec_manager_setattr(ec_fop_data_t * fop, int32_t state)
+int32_t
+ec_manager_setattr(ec_fop_data_t *fop, int32_t state)
 {
-    ec_cbk_data_t * cbk;
+    ec_cbk_data_t *cbk;
 
-    switch (state)
-    {
+    switch (state) {
         case EC_STATE_INIT:
         case EC_STATE_LOCK:
-            if (fop->fd == NULL)
-            {
-                ec_lock_prepare_inode(fop, &fop->loc[0], 1);
-            }
-            else
-            {
-                ec_lock_prepare_fd(fop, fop->fd, 1);
+            if (fop->fd == NULL) {
+                ec_lock_prepare_inode(fop, &fop->loc[0],
+                                      EC_UPDATE_META | EC_QUERY_INFO, 0,
+                                      EC_RANGE_FULL);
+            } else {
+                ec_lock_prepare_fd(fop, fop->fd, EC_UPDATE_META | EC_QUERY_INFO,
+                                   0, EC_RANGE_FULL);
             }
             ec_lock(fop);
 
-            return EC_STATE_GET_SIZE_AND_VERSION;
-
-        case EC_STATE_GET_SIZE_AND_VERSION:
-            ec_get_size_version(fop);
-
             return EC_STATE_DISPATCH;
 
         case EC_STATE_DISPATCH:
@@ -401,34 +468,18 @@ int32_t ec_manager_setattr(ec_fop_data_t * fop, int32_t state)
             return EC_STATE_PREPARE_ANSWER;
 
         case EC_STATE_PREPARE_ANSWER:
-            cbk = fop->answer;
-            if (cbk != NULL)
-            {
-                if (!ec_dict_combine(cbk, EC_COMBINE_XDATA))
-                {
-                    if (cbk->op_ret >= 0)
-                    {
-                        cbk->op_ret = -1;
-                        cbk->op_errno = EIO;
-                    }
-                }
-                if (cbk->op_ret < 0)
-                {
-                    ec_fop_set_error(fop, cbk->op_errno);
-                }
-                else
-                {
-                    ec_iatt_rebuild(fop->xl->private, cbk->iatt, 2,
-                                    cbk->count);
-
-                    cbk->iatt[0].ia_size = fop->pre_size;
-                    cbk->iatt[1].ia_size = fop->pre_size;
+            cbk = ec_fop_prepare_answer(fop, _gf_false);
+            if (cbk != NULL) {
+                if (cbk->iatt[0].ia_type == IA_IFREG) {
+                    ec_iatt_rebuild(fop->xl->private, cbk->iatt, 2, cbk->count);
+
+                    /* This shouldn't fail because we have the inode locked. */
+                    GF_ASSERT(ec_get_inode_size(fop,
+                                                fop->locks[0].lock->loc.inode,
+                                                &cbk->iatt[0].ia_size));
+                    cbk->iatt[1].ia_size = cbk->iatt[0].ia_size;
                 }
             }
-            else
-            {
-                ec_fop_set_error(fop, EIO);
-            }
 
             return EC_STATE_REPORT;
 
@@ -437,24 +488,17 @@ int32_t ec_manager_setattr(ec_fop_data_t * fop, int32_t state)
 
             GF_ASSERT(cbk != NULL);
 
-            if (fop->id == GF_FOP_SETATTR)
-            {
-                if (fop->cbks.setattr != NULL)
-                {
-                    fop->cbks.setattr(fop->req_frame, fop, fop->xl,
-                                      cbk->op_ret, cbk->op_errno,
-                                      &cbk->iatt[0], &cbk->iatt[1],
-                                      cbk->xdata);
+            if (fop->id == GF_FOP_SETATTR) {
+                if (fop->cbks.setattr != NULL) {
+                    QUORUM_CBK(fop->cbks.setattr, fop, fop->req_frame, fop,
+                               fop->xl, cbk->op_ret, cbk->op_errno,
+                               &cbk->iatt[0], &cbk->iatt[1], cbk->xdata);
                 }
-            }
-            else
-            {
-                if (fop->cbks.fsetattr != NULL)
-                {
-                    fop->cbks.fsetattr(fop->req_frame, fop, fop->xl,
-                                       cbk->op_ret, cbk->op_errno,
-                                       &cbk->iatt[0], &cbk->iatt[1],
-                                       cbk->xdata);
+            } else {
+                if (fop->cbks.fsetattr != NULL) {
+                    QUORUM_CBK(fop->cbks.fsetattr, fop, fop->req_frame, fop,
+                               fop->xl, cbk->op_ret, cbk->op_errno,
+                               &cbk->iatt[0], &cbk->iatt[1], cbk->xdata);
                 }
             }
 
@@ -462,24 +506,18 @@ int32_t ec_manager_setattr(ec_fop_data_t * fop, int32_t state)
 
         case -EC_STATE_INIT:
         case -EC_STATE_LOCK:
-        case -EC_STATE_GET_SIZE_AND_VERSION:
         case -EC_STATE_DISPATCH:
         case -EC_STATE_PREPARE_ANSWER:
         case -EC_STATE_REPORT:
             GF_ASSERT(fop->error != 0);
 
-            if (fop->id == GF_FOP_SETATTR)
-            {
-                if (fop->cbks.setattr != NULL)
-                {
+            if (fop->id == GF_FOP_SETATTR) {
+                if (fop->cbks.setattr != NULL) {
                     fop->cbks.setattr(fop->req_frame, fop, fop->xl, -1,
                                       fop->error, NULL, NULL, NULL);
                 }
-            }
-            else
-            {
-                if (fop->cbks.fsetattr != NULL)
-                {
+            } else {
+                if (fop->cbks.fsetattr != NULL) {
                     fop->cbks.fsetattr(fop->req_frame, fop, fop->xl, -1,
                                        fop->error, NULL, NULL, NULL);
                 }
@@ -500,59 +538,54 @@ int32_t ec_manager_setattr(ec_fop_data_t * fop, int32_t state)
             return EC_STATE_END;
 
         default:
-            gf_log(fop->xl->name, GF_LOG_ERROR, "Unhandled state %d for %s",
-                   state, ec_fop_name(fop->id));
+            gf_msg(fop->xl->name, GF_LOG_ERROR, EINVAL, EC_MSG_UNHANDLED_STATE,
+                   "Unhandled state %d for %s", state, ec_fop_name(fop->id));
 
             return EC_STATE_END;
     }
 }
 
-void ec_setattr(call_frame_t * frame, xlator_t * this, uintptr_t target,
-                int32_t minimum, fop_setattr_cbk_t func, void * data,
-                loc_t * loc, struct iatt * stbuf, int32_t valid,
-                dict_t * xdata)
+void
+ec_setattr(call_frame_t *frame, xlator_t *this, uintptr_t target,
+           uint32_t fop_flags, fop_setattr_cbk_t func, void *data, loc_t *loc,
+           struct iatt *stbuf, int32_t valid, dict_t *xdata)
 {
-    ec_cbk_t callback = { .setattr = func };
-    ec_fop_data_t * fop = NULL;
-    int32_t error = EIO;
+    ec_cbk_t callback = {.setattr = func};
+    ec_fop_data_t *fop = NULL;
+    int32_t error = ENOMEM;
 
-    gf_log("ec", GF_LOG_TRACE, "EC(SETATTR) %p", frame);
+    gf_msg_trace("ec", 0, "EC(SETATTR) %p", frame);
 
     VALIDATE_OR_GOTO(this, out);
     GF_VALIDATE_OR_GOTO(this->name, frame, out);
     GF_VALIDATE_OR_GOTO(this->name, this->private, out);
 
-    fop = ec_fop_data_allocate(frame, this, GF_FOP_SETATTR,
-                               EC_FLAG_UPDATE_LOC_INODE, target, minimum,
-                               ec_wind_setattr, ec_manager_setattr, callback,
-                               data);
-    if (fop == NULL)
-    {
+    fop = ec_fop_data_allocate(frame, this, GF_FOP_SETATTR, 0, target,
+                               fop_flags, ec_wind_setattr, ec_manager_setattr,
+                               callback, data);
+    if (fop == NULL) {
         goto out;
     }
 
     fop->int32 = valid;
 
-    if (loc != NULL)
-    {
-        if (loc_copy(&fop->loc[0], loc) != 0)
-        {
-            gf_log(this->name, GF_LOG_ERROR, "Failed to copy a location.");
+    if (loc != NULL) {
+        if (loc_copy(&fop->loc[0], loc) != 0) {
+            gf_msg(this->name, GF_LOG_ERROR, ENOMEM, EC_MSG_LOC_COPY_FAIL,
+                   "Failed to copy a location.");
 
             goto out;
         }
     }
-    if (stbuf != NULL)
-    {
+    if (stbuf != NULL) {
         fop->iatt = *stbuf;
     }
-    if (xdata != NULL)
-    {
-        fop->xdata = dict_ref(xdata);
-        if (fop->xdata == NULL)
-        {
-            gf_log(this->name, GF_LOG_ERROR, "Failed to reference a "
-                                             "dictionary.");
+    if (xdata != NULL) {
+        fop->xdata = dict_copy_with_ref(xdata, NULL);
+        if (fop->xdata == NULL) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_DICT_REF_FAIL,
+                   "Failed to reference a "
+                   "dictionary.");
 
             goto out;
         }
@@ -561,28 +594,26 @@ void ec_setattr(call_frame_t * frame, xlator_t * this, uintptr_t target,
     error = 0;
 
 out:
-    if (fop != NULL)
-    {
+    if (fop != NULL) {
         ec_manager(fop, error);
-    }
-    else
-    {
-        func(frame, NULL, this, -1, EIO, NULL, NULL, NULL);
+    } else {
+        func(frame, NULL, this, -1, error, NULL, NULL, NULL);
     }
 }
 
 /* FOP: fsetattr */
 
-int32_t ec_fsetattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                         int32_t op_ret, int32_t op_errno,
-                         struct iatt *prestat, struct iatt *poststat,
-                         dict_t *xdata)
+int32_t
+ec_fsetattr_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                int32_t op_ret, int32_t op_errno, struct iatt *prestat,
+                struct iatt *poststat, dict_t *xdata)
 {
-        return ec_inode_write_cbk (frame, this, cookie, op_ret, op_errno,
-                                   prestat, poststat, xdata);
+    return ec_inode_write_cbk(frame, this, cookie, op_ret, op_errno, prestat,
+                              poststat, xdata);
 }
 
-void ec_wind_fsetattr(ec_t * ec, ec_fop_data_t * fop, int32_t idx)
+void
+ec_wind_fsetattr(ec_t *ec, ec_fop_data_t *fop, int32_t idx)
 {
     ec_trace("WIND", fop, "idx=%d", idx);
 
@@ -591,26 +622,25 @@ void ec_wind_fsetattr(ec_t * ec, ec_fop_data_t * fop, int32_t idx)
                       fop->fd, &fop->iatt, fop->int32, fop->xdata);
 }
 
-void ec_fsetattr(call_frame_t * frame, xlator_t * this, uintptr_t target,
-                 int32_t minimum, fop_fsetattr_cbk_t func, void * data,
-                 fd_t * fd, struct iatt * stbuf, int32_t valid, dict_t * xdata)
+void
+ec_fsetattr(call_frame_t *frame, xlator_t *this, uintptr_t target,
+            uint32_t fop_flags, fop_fsetattr_cbk_t func, void *data, fd_t *fd,
+            struct iatt *stbuf, int32_t valid, dict_t *xdata)
 {
-    ec_cbk_t callback = { .fsetattr = func };
-    ec_fop_data_t * fop = NULL;
-    int32_t error = EIO;
+    ec_cbk_t callback = {.fsetattr = func};
+    ec_fop_data_t *fop = NULL;
+    int32_t error = ENOMEM;
 
-    gf_log("ec", GF_LOG_TRACE, "EC(FSETATTR) %p", frame);
+    gf_msg_trace("ec", 0, "EC(FSETATTR) %p", frame);
 
     VALIDATE_OR_GOTO(this, out);
     GF_VALIDATE_OR_GOTO(this->name, frame, out);
     GF_VALIDATE_OR_GOTO(this->name, this->private, out);
 
-    fop = ec_fop_data_allocate(frame, this, GF_FOP_FSETATTR,
-                               EC_FLAG_UPDATE_FD_INODE, target, minimum,
-                               ec_wind_fsetattr, ec_manager_setattr, callback,
-                               data);
-    if (fop == NULL)
-    {
+    fop = ec_fop_data_allocate(frame, this, GF_FOP_FSETATTR, 0, target,
+                               fop_flags, ec_wind_fsetattr, ec_manager_setattr,
+                               callback, data);
+    if (fop == NULL) {
         goto out;
     }
 
@@ -618,28 +648,25 @@ void ec_fsetattr(call_frame_t * frame, xlator_t * this, uintptr_t target,
 
     fop->int32 = valid;
 
-    if (fd != NULL)
-    {
+    if (fd != NULL) {
         fop->fd = fd_ref(fd);
-        if (fop->fd == NULL)
-        {
-            gf_log(this->name, GF_LOG_ERROR, "Failed to reference a "
-                                             "file descriptor.");
+        if (fop->fd == NULL) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_FILE_DESC_REF_FAIL,
+                   "Failed to reference a "
+                   "file descriptor.");
 
             goto out;
         }
     }
-    if (stbuf != NULL)
-    {
+    if (stbuf != NULL) {
         fop->iatt = *stbuf;
     }
-    if (xdata != NULL)
-    {
-        fop->xdata = dict_ref(xdata);
-        if (fop->xdata == NULL)
-        {
-            gf_log(this->name, GF_LOG_ERROR, "Failed to reference a "
-                                             "dictionary.");
+    if (xdata != NULL) {
+        fop->xdata = dict_copy_with_ref(xdata, NULL);
+        if (fop->xdata == NULL) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_DICT_REF_FAIL,
+                   "Failed to reference a "
+                   "dictionary.");
 
             goto out;
         }
@@ -648,26 +675,25 @@ void ec_fsetattr(call_frame_t * frame, xlator_t * this, uintptr_t target,
     error = 0;
 
 out:
-    if (fop != NULL)
-    {
+    if (fop != NULL) {
         ec_manager(fop, error);
-    }
-    else
-    {
-        func(frame, NULL, this, -1, EIO, NULL, NULL, NULL);
+    } else {
+        func(frame, NULL, this, -1, error, NULL, NULL, NULL);
     }
 }
 
 /* FOP: setxattr */
 
-int32_t ec_setxattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                         int32_t op_ret, int32_t op_errno, dict_t *xdata)
+int32_t
+ec_setxattr_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
-        return ec_inode_write_cbk (frame, this, cookie, op_ret, op_errno,
-                                   NULL, NULL, xdata);
+    return ec_inode_write_cbk(frame, this, cookie, op_ret, op_errno, NULL, NULL,
+                              xdata);
 }
 
-void ec_wind_setxattr(ec_t * ec, ec_fop_data_t * fop, int32_t idx)
+void
+ec_wind_setxattr(ec_t *ec, ec_fop_data_t *fop, int32_t idx)
 {
     ec_trace("WIND", fop, "idx=%d", idx);
 
@@ -677,58 +703,53 @@ void ec_wind_setxattr(ec_t * ec, ec_fop_data_t * fop, int32_t idx)
 }
 
 void
-ec_setxattr (call_frame_t *frame, xlator_t *this, uintptr_t target,
-             int32_t minimum, fop_setxattr_cbk_t func, void *data,
-             loc_t *loc, dict_t *dict, int32_t flags, dict_t *xdata)
+ec_setxattr(call_frame_t *frame, xlator_t *this, uintptr_t target,
+            uint32_t fop_flags, fop_setxattr_cbk_t func, void *data, loc_t *loc,
+            dict_t *dict, int32_t flags, dict_t *xdata)
 {
-    ec_cbk_t callback = { .setxattr = func };
-    ec_fop_data_t * fop = NULL;
-    int32_t error = EIO;
+    ec_cbk_t callback = {.setxattr = func};
+    ec_fop_data_t *fop = NULL;
+    int32_t error = ENOMEM;
 
-    gf_log("ec", GF_LOG_TRACE, "EC(SETXATTR) %p", frame);
+    gf_msg_trace("ec", 0, "EC(SETXATTR) %p", frame);
 
     VALIDATE_OR_GOTO(this, out);
     GF_VALIDATE_OR_GOTO(this->name, frame, out);
     GF_VALIDATE_OR_GOTO(this->name, this->private, out);
 
-    fop = ec_fop_data_allocate(frame, this, GF_FOP_SETXATTR,
-                               EC_FLAG_UPDATE_LOC_INODE, target, minimum,
-                               ec_wind_setxattr, ec_manager_xattr, callback,
-                               data);
-    if (fop == NULL)
-    {
+    fop = ec_fop_data_allocate(frame, this, GF_FOP_SETXATTR, 0, target,
+                               fop_flags, ec_wind_setxattr, ec_manager_xattr,
+                               callback, data);
+    if (fop == NULL) {
         goto out;
     }
 
     fop->int32 = flags;
 
-    if (loc != NULL)
-    {
-        if (loc_copy(&fop->loc[0], loc) != 0)
-        {
-            gf_log(this->name, GF_LOG_ERROR, "Failed to copy a location.");
+    if (loc != NULL) {
+        if (loc_copy(&fop->loc[0], loc) != 0) {
+            gf_msg(this->name, GF_LOG_ERROR, ENOMEM, EC_MSG_LOC_COPY_FAIL,
+                   "Failed to copy a location.");
 
             goto out;
         }
     }
-    if (dict != NULL)
-    {
-        fop->dict = dict_ref(dict);
-        if (fop->dict == NULL)
-        {
-            gf_log(this->name, GF_LOG_ERROR, "Failed to reference a "
-                                             "dictionary.");
+    if (dict != NULL) {
+        fop->dict = dict_copy_with_ref(dict, NULL);
+        if (fop->dict == NULL) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_DICT_REF_FAIL,
+                   "Failed to reference a "
+                   "dictionary.");
 
             goto out;
         }
     }
-    if (xdata != NULL)
-    {
-        fop->xdata = dict_ref(xdata);
-        if (fop->xdata == NULL)
-        {
-            gf_log(this->name, GF_LOG_ERROR, "Failed to reference a "
-                                             "dictionary.");
+    if (xdata != NULL) {
+        fop->xdata = dict_copy_with_ref(xdata, NULL);
+        if (fop->xdata == NULL) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_DICT_REF_FAIL,
+                   "Failed to reference a "
+                   "dictionary.");
 
             goto out;
         }
@@ -738,43 +759,41 @@ ec_setxattr (call_frame_t *frame, xlator_t *this, uintptr_t target,
 
 out:
     if (fop != NULL) {
-        ec_manager (fop, error);
+        ec_manager(fop, error);
     } else {
-        func (frame, NULL, this, -1, error, NULL);
+        func(frame, NULL, this, -1, error, NULL);
     }
 }
 
 /* FOP: fsetxattr */
 
 int32_t
-ec_fsetxattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                  int32_t op_ret, int32_t op_errno, dict_t *xdata)
+ec_fsetxattr_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                 int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
-    ec_fop_data_t * fop = NULL;
-    ec_cbk_data_t * cbk = NULL;
+    ec_fop_data_t *fop = NULL;
+    ec_cbk_data_t *cbk = NULL;
     int32_t idx = (int32_t)(uintptr_t)cookie;
 
-    VALIDATE_OR_GOTO (this, out);
-    GF_VALIDATE_OR_GOTO (this->name, frame, out);
-    GF_VALIDATE_OR_GOTO (this->name, frame->local, out);
-    GF_VALIDATE_OR_GOTO (this->name, this->private, out);
+    VALIDATE_OR_GOTO(this, out);
+    GF_VALIDATE_OR_GOTO(this->name, frame, out);
+    GF_VALIDATE_OR_GOTO(this->name, frame->local, out);
+    GF_VALIDATE_OR_GOTO(this->name, this->private, out);
 
     fop = frame->local;
 
-    ec_trace("CBK", fop, "idx=%d, frame=%p, op_ret=%d, op_errno=%d", idx,
-             frame, op_ret, op_errno);
+    ec_trace("CBK", fop, "idx=%d, frame=%p, op_ret=%d, op_errno=%d", idx, frame,
+             op_ret, op_errno);
 
     cbk = ec_cbk_data_allocate(frame, this, fop, GF_FOP_FSETXATTR, idx, op_ret,
                                op_errno);
-    if (cbk != NULL)
-    {
-        if (xdata != NULL)
-        {
+    if (cbk != NULL) {
+        if (xdata != NULL) {
             cbk->xdata = dict_ref(xdata);
-            if (cbk->xdata == NULL)
-            {
-                gf_log(this->name, GF_LOG_ERROR, "Failed to reference a "
-                                                 "dictionary.");
+            if (cbk->xdata == NULL) {
+                gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_DICT_REF_FAIL,
+                       "Failed to reference a "
+                       "dictionary.");
 
                 goto out;
             }
@@ -784,15 +803,15 @@ ec_fsetxattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
     }
 
 out:
-    if (fop != NULL)
-    {
+    if (fop != NULL) {
         ec_complete(fop);
     }
 
     return 0;
 }
 
-void ec_wind_fsetxattr(ec_t * ec, ec_fop_data_t * fop, int32_t idx)
+void
+ec_wind_fsetxattr(ec_t *ec, ec_fop_data_t *fop, int32_t idx)
 {
     ec_trace("WIND", fop, "idx=%d", idx);
 
@@ -802,26 +821,24 @@ void ec_wind_fsetxattr(ec_t * ec, ec_fop_data_t * fop, int32_t idx)
 }
 
 void
-ec_fsetxattr (call_frame_t *frame, xlator_t *this, uintptr_t target,
-              int32_t minimum, fop_fsetxattr_cbk_t func, void *data,
-              fd_t *fd, dict_t *dict, int32_t flags, dict_t *xdata)
+ec_fsetxattr(call_frame_t *frame, xlator_t *this, uintptr_t target,
+             uint32_t fop_flags, fop_fsetxattr_cbk_t func, void *data, fd_t *fd,
+             dict_t *dict, int32_t flags, dict_t *xdata)
 {
-    ec_cbk_t callback = { .fsetxattr = func };
-    ec_fop_data_t * fop = NULL;
-    int32_t error = EIO;
+    ec_cbk_t callback = {.fsetxattr = func};
+    ec_fop_data_t *fop = NULL;
+    int32_t error = ENOMEM;
 
-    gf_log("ec", GF_LOG_TRACE, "EC(FSETXATTR) %p", frame);
+    gf_msg_trace("ec", 0, "EC(FSETXATTR) %p", frame);
 
     VALIDATE_OR_GOTO(this, out);
     GF_VALIDATE_OR_GOTO(this->name, frame, out);
     GF_VALIDATE_OR_GOTO(this->name, this->private, out);
 
-    fop = ec_fop_data_allocate(frame, this, GF_FOP_FSETXATTR,
-                               EC_FLAG_UPDATE_FD_INODE, target, minimum,
-                               ec_wind_fsetxattr, ec_manager_xattr,
+    fop = ec_fop_data_allocate(frame, this, GF_FOP_FSETXATTR, 0, target,
+                               fop_flags, ec_wind_fsetxattr, ec_manager_xattr,
                                callback, data);
-    if (fop == NULL)
-    {
+    if (fop == NULL) {
         goto out;
     }
 
@@ -829,35 +846,32 @@ ec_fsetxattr (call_frame_t *frame, xlator_t *this, uintptr_t target,
 
     fop->int32 = flags;
 
-    if (fd != NULL)
-    {
+    if (fd != NULL) {
         fop->fd = fd_ref(fd);
-        if (fop->fd == NULL)
-        {
-            gf_log(this->name, GF_LOG_ERROR, "Failed to reference a "
-                                             "file descriptor.");
+        if (fop->fd == NULL) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_FILE_DESC_REF_FAIL,
+                   "Failed to reference a "
+                   "file descriptor.");
 
             goto out;
         }
     }
-    if (dict != NULL)
-    {
-        fop->dict = dict_ref(dict);
-        if (fop->dict == NULL)
-        {
-            gf_log(this->name, GF_LOG_ERROR, "Failed to reference a "
-                                             "dictionary.");
+    if (dict != NULL) {
+        fop->dict = dict_copy_with_ref(dict, NULL);
+        if (fop->dict == NULL) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_DICT_REF_FAIL,
+                   "Failed to reference a "
+                   "dictionary.");
 
             goto out;
         }
     }
-    if (xdata != NULL)
-    {
-        fop->xdata = dict_ref(xdata);
-        if (fop->xdata == NULL)
-        {
-            gf_log(this->name, GF_LOG_ERROR, "Failed to reference a "
-                                             "dictionary.");
+    if (xdata != NULL) {
+        fop->xdata = dict_copy_with_ref(xdata, NULL);
+        if (fop->xdata == NULL) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_DICT_REF_FAIL,
+                   "Failed to reference a "
+                   "dictionary.");
 
             goto out;
         }
@@ -867,110 +881,515 @@ ec_fsetxattr (call_frame_t *frame, xlator_t *this, uintptr_t target,
 
 out:
     if (fop != NULL) {
-        ec_manager (fop, error);
+        ec_manager(fop, error);
     } else {
-        func (frame, NULL, this, -1, error, NULL);
+        func(frame, NULL, this, -1, error, NULL);
     }
 }
 
-/* FOP: truncate */
+/*********************************************************************
+ *
+ * File Operation : fallocate
+ *
+ *********************************************************************/
 
-int32_t ec_truncate_write(ec_fop_data_t * fop, uintptr_t mask)
+int32_t
+ec_fallocate_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                 int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
+                 struct iatt *postbuf, dict_t *xdata)
 {
-    ec_t * ec = fop->xl->private;
-    struct iobref * iobref = NULL;
-    struct iobuf * iobuf = NULL;
-    struct iovec vector;
-    int32_t ret = 0;
+    return ec_inode_write_cbk(frame, this, cookie, op_ret, op_errno, prebuf,
+                              postbuf, xdata);
+}
 
-    iobref = iobref_new();
-    if (iobref == NULL)
-    {
-        goto out;
-    }
-    iobuf = iobuf_get(fop->xl->ctx->iobuf_pool);
-    if (iobuf == NULL)
-    {
-        goto out;
+void
+ec_wind_fallocate(ec_t *ec, ec_fop_data_t *fop, int32_t idx)
+{
+    ec_trace("WIND", fop, "idx=%d", idx);
+
+    STACK_WIND_COOKIE(fop->frame, ec_fallocate_cbk, (void *)(uintptr_t)idx,
+                      ec->xl_list[idx], ec->xl_list[idx]->fops->fallocate,
+                      fop->fd, fop->int32, fop->offset, fop->size, fop->xdata);
+}
+
+int32_t
+ec_manager_fallocate(ec_fop_data_t *fop, int32_t state)
+{
+    ec_cbk_data_t *cbk = NULL;
+
+    switch (state) {
+        case EC_STATE_INIT:
+            if (fop->size == 0) {
+                ec_fop_set_error(fop, EINVAL);
+                return EC_STATE_REPORT;
+            }
+            if (fop->int32 &
+                (FALLOC_FL_COLLAPSE_RANGE | FALLOC_FL_INSERT_RANGE |
+                 FALLOC_FL_ZERO_RANGE | FALLOC_FL_PUNCH_HOLE)) {
+                ec_fop_set_error(fop, ENOTSUP);
+                return EC_STATE_REPORT;
+            }
+            fop->user_size = fop->offset + fop->size;
+            fop->head = ec_adjust_offset_down(fop->xl->private, &fop->offset,
+                                              _gf_true);
+            fop->size += fop->head;
+            ec_adjust_size_up(fop->xl->private, &fop->size, _gf_true);
+
+            /* Fall through */
+
+        case EC_STATE_LOCK:
+            ec_lock_prepare_fd(fop, fop->fd,
+                               EC_UPDATE_DATA | EC_UPDATE_META | EC_QUERY_INFO,
+                               fop->offset, fop->size);
+            ec_lock(fop);
+
+            return EC_STATE_DISPATCH;
+
+        case EC_STATE_DISPATCH:
+
+            ec_dispatch_all(fop);
+
+            return EC_STATE_PREPARE_ANSWER;
+
+        case EC_STATE_PREPARE_ANSWER:
+            cbk = ec_fop_prepare_answer(fop, _gf_false);
+            if (cbk != NULL) {
+                ec_iatt_rebuild(fop->xl->private, cbk->iatt, 2, cbk->count);
+
+                /* This shouldn't fail because we have the inode locked. */
+                LOCK(&fop->locks[0].lock->loc.inode->lock);
+                {
+                    GF_ASSERT(__ec_get_inode_size(fop,
+                                                  fop->locks[0].lock->loc.inode,
+                                                  &cbk->iatt[0].ia_size));
+
+                    /*If mode has FALLOC_FL_KEEP_SIZE keep the size */
+                    if (fop->int32 & FALLOC_FL_KEEP_SIZE) {
+                        cbk->iatt[1].ia_size = cbk->iatt[0].ia_size;
+                    } else if (fop->user_size > cbk->iatt[0].ia_size) {
+                        cbk->iatt[1].ia_size = fop->user_size;
+
+                        /* This shouldn't fail because we have the inode
+                         * locked. */
+                        GF_ASSERT(__ec_set_inode_size(
+                            fop, fop->locks[0].lock->loc.inode,
+                            cbk->iatt[1].ia_size));
+                    } else {
+                        cbk->iatt[1].ia_size = cbk->iatt[0].ia_size;
+                    }
+                }
+                UNLOCK(&fop->locks[0].lock->loc.inode->lock);
+            }
+
+            return EC_STATE_REPORT;
+
+        case EC_STATE_REPORT:
+            cbk = fop->answer;
+
+            GF_ASSERT(cbk != NULL);
+
+            if (fop->cbks.fallocate != NULL) {
+                QUORUM_CBK(fop->cbks.fallocate, fop, fop->req_frame, fop,
+                           fop->xl, cbk->op_ret, cbk->op_errno, &cbk->iatt[0],
+                           &cbk->iatt[1], cbk->xdata);
+            }
+
+            return EC_STATE_LOCK_REUSE;
+
+        case -EC_STATE_INIT:
+        case -EC_STATE_LOCK:
+        case -EC_STATE_DISPATCH:
+        case -EC_STATE_PREPARE_ANSWER:
+        case -EC_STATE_REPORT:
+            GF_ASSERT(fop->error != 0);
+
+            if (fop->cbks.fallocate != NULL) {
+                fop->cbks.fallocate(fop->req_frame, fop, fop->xl, -1,
+                                    fop->error, NULL, NULL, NULL);
+            }
+
+            return EC_STATE_LOCK_REUSE;
+
+        case -EC_STATE_LOCK_REUSE:
+        case EC_STATE_LOCK_REUSE:
+            ec_lock_reuse(fop);
+
+            return EC_STATE_UNLOCK;
+
+        case -EC_STATE_UNLOCK:
+        case EC_STATE_UNLOCK:
+            ec_unlock(fop);
+
+            return EC_STATE_END;
+
+        default:
+            gf_msg(fop->xl->name, GF_LOG_ERROR, EINVAL, EC_MSG_UNHANDLED_STATE,
+                   "Unhandled state %d for %s", state, ec_fop_name(fop->id));
+
+            return EC_STATE_END;
     }
-    if (iobref_add(iobref, iobuf) != 0)
-    {
+}
+
+void
+ec_fallocate(call_frame_t *frame, xlator_t *this, uintptr_t target,
+             uint32_t fop_flags, fop_fallocate_cbk_t func, void *data, fd_t *fd,
+             int32_t mode, off_t offset, size_t len, dict_t *xdata)
+{
+    ec_cbk_t callback = {.fallocate = func};
+    ec_fop_data_t *fop = NULL;
+    int32_t error = ENOMEM;
+
+    gf_msg_trace("ec", 0, "EC(FALLOCATE) %p", frame);
+
+    VALIDATE_OR_GOTO(this, out);
+    GF_VALIDATE_OR_GOTO(this->name, frame, out);
+    GF_VALIDATE_OR_GOTO(this->name, this->private, out);
+
+    fop = ec_fop_data_allocate(frame, this, GF_FOP_FALLOCATE, 0, target,
+                               fop_flags, ec_wind_fallocate,
+                               ec_manager_fallocate, callback, data);
+    if (fop == NULL) {
         goto out;
     }
 
-    vector.iov_base = iobuf->ptr;
-    vector.iov_len = fop->offset * ec->fragments - fop->user_size;
-    memset(vector.iov_base, 0, vector.iov_len);
+    fop->use_fd = 1;
+    fop->int32 = mode;
+    fop->offset = offset;
+    fop->size = len;
 
-    iobuf_unref (iobuf);
-    iobuf = NULL;
+    if (fd != NULL) {
+        fop->fd = fd_ref(fd);
+        if (fop->fd == NULL) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_FILE_DESC_REF_FAIL,
+                   "Failed to reference a "
+                   "file descriptor.");
+            goto out;
+        }
+    }
 
-    ec_writev(fop->frame, fop->xl, mask, fop->minimum, NULL, NULL, fop->fd,
-              &vector, 1, fop->user_size, 0, iobref, NULL);
+    if (xdata != NULL) {
+        fop->xdata = dict_ref(xdata);
+        if (fop->xdata == NULL) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_DICT_REF_FAIL,
+                   "Failed to reference a "
+                   "dictionary.");
+            goto out;
+        }
+    }
 
-    ret = 1;
+    error = 0;
 
 out:
-    if (iobuf != NULL)
-    {
-        iobuf_unref(iobuf);
+    if (fop != NULL) {
+        ec_manager(fop, error);
+    } else {
+        func(frame, NULL, this, -1, error, NULL, NULL, NULL);
     }
-    if (iobref != NULL)
-    {
-        iobref_unref(iobref);
+}
+
+/*********************************************************************
+ *
+ * File Operation : Discard
+ *
+ *********************************************************************/
+void
+ec_update_discard_write(ec_fop_data_t *fop, uintptr_t mask)
+{
+    ec_t *ec = fop->xl->private;
+    off_t off_head = 0;
+    off_t off_tail = 0;
+    uint64_t size_head = 0;
+    uint64_t size_tail = 0;
+    int error = 0;
+
+    off_head = fop->offset * ec->fragments - fop->int32;
+    if (fop->size == 0) {
+        error = ec_update_write(fop, mask, off_head, fop->user_size);
+    } else {
+        size_head = fop->int32;
+        size_tail = (off_head + fop->user_size) % ec->stripe_size;
+        off_tail = off_head + fop->user_size - size_tail;
+        if (size_head) {
+            error = ec_update_write(fop, mask, off_head, size_head);
+            if (error) {
+                goto out;
+            }
+        }
+        if (size_tail) {
+            error = ec_update_write(fop, mask, off_tail, size_tail);
+        }
     }
+out:
+    if (error)
+        ec_fop_set_error(fop, -error);
+}
+
+void
+ec_discard_adjust_offset_size(ec_fop_data_t *fop)
+{
+    ec_t *ec = fop->xl->private;
 
-    return ret;
+    fop->user_size = fop->size;
+    /* If discard length covers at least a fragment on brick, we will
+     * perform discard operation(when fop->size is non-zero) else we just
+     * write zeros.
+     */
+    fop->int32 = ec_adjust_offset_up(ec, &fop->offset, _gf_true);
+    fop->frag_range.first = fop->offset;
+    if (fop->size < fop->int32) {
+        fop->size = 0;
+    } else {
+        fop->size -= fop->int32;
+        ec_adjust_size_down(ec, &fop->size, _gf_true);
+    }
+    fop->frag_range.last = fop->offset + fop->size;
 }
 
-int32_t ec_truncate_open_cbk(call_frame_t * frame, void * cookie,
-                             xlator_t * this, int32_t op_ret, int32_t op_errno,
-                             fd_t * fd, dict_t * xdata)
+int32_t
+ec_discard_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+               int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
+               struct iatt *postbuf, dict_t *xdata)
 {
-    ec_fop_data_t * fop = cookie;
+    return ec_inode_write_cbk(frame, this, cookie, op_ret, op_errno, prebuf,
+                              postbuf, xdata);
+}
 
-    if (op_ret >= 0)
-    {
-        if (!ec_truncate_write(fop->parent, fop->answer->mask))
-        {
-            fop->error = EIO;
+void
+ec_wind_discard(ec_t *ec, ec_fop_data_t *fop, int32_t idx)
+{
+    ec_trace("WIND", fop, "idx=%d", idx);
+
+    STACK_WIND_COOKIE(fop->frame, ec_discard_cbk, (void *)(uintptr_t)idx,
+                      ec->xl_list[idx], ec->xl_list[idx]->fops->discard,
+                      fop->fd, fop->offset, fop->size, fop->xdata);
+}
+
+int32_t
+ec_manager_discard(ec_fop_data_t *fop, int32_t state)
+{
+    ec_cbk_data_t *cbk = NULL;
+    off_t fl_start = 0;
+    uint64_t fl_size = 0;
+
+    switch (state) {
+        case EC_STATE_INIT:
+            if ((fop->size <= 0) || (fop->offset < 0)) {
+                ec_fop_set_error(fop, EINVAL);
+                return EC_STATE_REPORT;
+            }
+            /* Because of the head/tail writes, "discard" happens on the
+             * remaining regions, but we need to compute region including
+             * head/tail writes so compute them separately*/
+            fl_start = fop->offset;
+            fl_size = fop->size;
+            fl_size += ec_adjust_offset_down(fop->xl->private, &fl_start,
+                                             _gf_true);
+            ec_adjust_size_up(fop->xl->private, &fl_size, _gf_true);
+
+            ec_discard_adjust_offset_size(fop);
+
+            /* Fall through */
+
+        case EC_STATE_LOCK:
+            ec_lock_prepare_fd(fop, fop->fd,
+                               EC_UPDATE_DATA | EC_UPDATE_META | EC_QUERY_INFO,
+                               fl_start, fl_size);
+            ec_lock(fop);
+
+            return EC_STATE_DISPATCH;
+
+        case EC_STATE_DISPATCH:
+
+            /* Dispatch discard fop only if we have whole fragment
+             * to deallocate */
+            if (fop->size) {
+                ec_dispatch_all(fop);
+                return EC_STATE_DELAYED_START;
+            } else {
+                /* Assume discard to have succeeded on all bricks */
+                ec_succeed_all(fop);
+            }
+
+            /* Fall through */
+
+        case EC_STATE_DELAYED_START:
+
+            if (fop->size) {
+                if (fop->answer && fop->answer->op_ret == 0)
+                    ec_update_discard_write(fop, fop->answer->mask);
+            } else {
+                ec_update_discard_write(fop, fop->mask);
+            }
+
+            return EC_STATE_PREPARE_ANSWER;
+
+        case EC_STATE_PREPARE_ANSWER:
+            cbk = ec_fop_prepare_answer(fop, _gf_false);
+            if (cbk != NULL) {
+                ec_iatt_rebuild(fop->xl->private, cbk->iatt, 2, cbk->count);
+
+                /* This shouldn't fail because we have the inode locked. */
+                GF_ASSERT(ec_get_inode_size(fop, fop->locks[0].lock->loc.inode,
+                                            &cbk->iatt[0].ia_size));
+
+                cbk->iatt[1].ia_size = cbk->iatt[0].ia_size;
+            }
+            return EC_STATE_REPORT;
+
+        case EC_STATE_REPORT:
+            cbk = fop->answer;
+
+            GF_ASSERT(cbk != NULL);
+
+            if (fop->cbks.discard != NULL) {
+                QUORUM_CBK(fop->cbks.discard, fop, fop->req_frame, fop, fop->xl,
+                           cbk->op_ret, cbk->op_errno, &cbk->iatt[0],
+                           &cbk->iatt[1], cbk->xdata);
+            }
+
+            return EC_STATE_LOCK_REUSE;
+
+        case -EC_STATE_INIT:
+        case -EC_STATE_LOCK:
+        case -EC_STATE_DISPATCH:
+        case -EC_STATE_DELAYED_START:
+        case -EC_STATE_PREPARE_ANSWER:
+        case -EC_STATE_REPORT:
+            GF_ASSERT(fop->error != 0);
+
+            if (fop->cbks.discard != NULL) {
+                fop->cbks.discard(fop->req_frame, fop, fop->xl, -1, fop->error,
+                                  NULL, NULL, NULL);
+            }
+
+            return EC_STATE_LOCK_REUSE;
+
+        case -EC_STATE_LOCK_REUSE:
+        case EC_STATE_LOCK_REUSE:
+            ec_lock_reuse(fop);
+
+            return EC_STATE_UNLOCK;
+
+        case -EC_STATE_UNLOCK:
+        case EC_STATE_UNLOCK:
+            ec_unlock(fop);
+
+            return EC_STATE_END;
+
+        default:
+            gf_msg(fop->xl->name, GF_LOG_ERROR, EINVAL, EC_MSG_UNHANDLED_STATE,
+                   "Unhandled state %d for %s", state, ec_fop_name(fop->id));
+
+            return EC_STATE_END;
+    }
+}
+
+void
+ec_discard(call_frame_t *frame, xlator_t *this, uintptr_t target,
+           uint32_t fop_flags, fop_discard_cbk_t func, void *data, fd_t *fd,
+           off_t offset, size_t len, dict_t *xdata)
+{
+    ec_cbk_t callback = {.discard = func};
+    ec_fop_data_t *fop = NULL;
+    int32_t error = ENOMEM;
+
+    gf_msg_trace("ec", 0, "EC(DISCARD) %p", frame);
+
+    VALIDATE_OR_GOTO(this, out);
+    GF_VALIDATE_OR_GOTO(this->name, frame, out);
+    GF_VALIDATE_OR_GOTO(this->name, this->private, out);
+
+    fop = ec_fop_data_allocate(frame, this, GF_FOP_DISCARD, 0, target,
+                               fop_flags, ec_wind_discard, ec_manager_discard,
+                               callback, data);
+    if (fop == NULL) {
+        goto out;
+    }
+
+    fop->use_fd = 1;
+    fop->offset = offset;
+    fop->size = len;
+
+    if (fd != NULL) {
+        fop->fd = fd_ref(fd);
+    }
+
+    if (xdata != NULL) {
+        fop->xdata = dict_ref(xdata);
+    }
+
+    error = 0;
+
+out:
+    if (fop != NULL) {
+        ec_manager(fop, error);
+    } else {
+        func(frame, NULL, this, -1, error, NULL, NULL, NULL);
+    }
+}
+
+/*********************************************************************
+ *
+ * File Operation : truncate
+ *
+ *********************************************************************/
+
+int32_t
+ec_update_truncate_write(ec_fop_data_t *fop, uintptr_t mask)
+{
+    ec_t *ec = fop->xl->private;
+    uint64_t size = fop->offset * ec->fragments - fop->user_size;
+    return ec_update_write(fop, mask, fop->user_size, size);
+}
+
+int32_t
+ec_truncate_open_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                     int32_t op_ret, int32_t op_errno, fd_t *fd, dict_t *xdata)
+{
+    ec_fop_data_t *fop = cookie;
+    int32_t err;
+
+    fop->parent->good &= fop->good;
+    if (op_ret >= 0) {
+        fd_bind(fd);
+        err = ec_update_truncate_write(fop->parent, fop->answer->mask);
+        if (err != 0) {
+            ec_fop_set_error(fop->parent, -err);
         }
     }
 
     return 0;
 }
 
-int32_t ec_truncate_clean(ec_fop_data_t * fop)
+int32_t
+ec_truncate_clean(ec_fop_data_t *fop)
 {
-    if (fop->fd == NULL)
-    {
+    if (fop->fd == NULL) {
         fop->fd = fd_create(fop->loc[0].inode, fop->frame->root->pid);
-        if (fop->fd == NULL)
-        {
-            return 0;
+        if (fop->fd == NULL) {
+            return -ENOMEM;
         }
 
         ec_open(fop->frame, fop->xl, fop->answer->mask, fop->minimum,
-                ec_truncate_open_cbk, fop, &fop->loc[0], O_RDWR, fop->fd,
-                NULL);
+                ec_truncate_open_cbk, fop, &fop->loc[0], O_RDWR, fop->fd, NULL);
 
-        return 1;
-    }
-    else
-    {
-        return ec_truncate_write(fop, fop->answer->mask);
+        return 0;
+    } else {
+        return ec_update_truncate_write(fop, fop->answer->mask);
     }
 }
 
-int32_t ec_truncate_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                         int32_t op_ret, int32_t op_errno, struct iatt *prestat,
-                         struct iatt *poststat, dict_t *xdata)
+int32_t
+ec_truncate_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                int32_t op_ret, int32_t op_errno, struct iatt *prestat,
+                struct iatt *poststat, dict_t *xdata)
 {
-        return ec_inode_write_cbk (frame, this, cookie, op_ret, op_errno,
-                                   prestat, poststat, xdata);
+    return ec_inode_write_cbk(frame, this, cookie, op_ret, op_errno, prestat,
+                              poststat, xdata);
 }
 
-void ec_wind_truncate(ec_t * ec, ec_fop_data_t * fop, int32_t idx)
+void
+ec_wind_truncate(ec_t *ec, ec_fop_data_t *fop, int32_t idx)
 {
     ec_trace("WIND", fop, "idx=%d", idx);
 
@@ -979,34 +1398,38 @@ void ec_wind_truncate(ec_t * ec, ec_fop_data_t * fop, int32_t idx)
                       &fop->loc[0], fop->offset, fop->xdata);
 }
 
-int32_t ec_manager_truncate(ec_fop_data_t * fop, int32_t state)
+int32_t
+ec_manager_truncate(ec_fop_data_t *fop, int32_t state)
 {
-    ec_cbk_data_t * cbk;
+    ec_cbk_data_t *cbk;
+    off_t offset_down;
 
-    switch (state)
-    {
+    switch (state) {
         case EC_STATE_INIT:
             fop->user_size = fop->offset;
-            fop->offset = ec_adjust_size(fop->xl->private, fop->offset, 1);
+            ec_adjust_offset_up(fop->xl->private, &fop->offset, _gf_true);
+            fop->frag_range.first = fop->offset;
+            fop->frag_range.last = UINT64_MAX;
 
-        /* Fall through */
+            /* Fall through */
 
         case EC_STATE_LOCK:
-            if (fop->id == GF_FOP_TRUNCATE)
-            {
-                ec_lock_prepare_inode(fop, &fop->loc[0], 1);
-            }
-            else
-            {
-                ec_lock_prepare_fd(fop, fop->fd, 1);
+            offset_down = fop->user_size;
+            ec_adjust_offset_down(fop->xl->private, &offset_down, _gf_true);
+
+            if (fop->id == GF_FOP_TRUNCATE) {
+                ec_lock_prepare_inode(
+                    fop, &fop->loc[0],
+                    EC_UPDATE_DATA | EC_UPDATE_META | EC_QUERY_INFO,
+                    offset_down, EC_RANGE_FULL);
+            } else {
+                ec_lock_prepare_fd(
+                    fop, fop->fd,
+                    EC_UPDATE_DATA | EC_UPDATE_META | EC_QUERY_INFO,
+                    offset_down, EC_RANGE_FULL);
             }
             ec_lock(fop);
 
-            return EC_STATE_GET_SIZE_AND_VERSION;
-
-        case EC_STATE_GET_SIZE_AND_VERSION:
-            ec_prepare_update(fop);
-
             return EC_STATE_DISPATCH;
 
         case EC_STATE_DISPATCH:
@@ -1015,43 +1438,30 @@ int32_t ec_manager_truncate(ec_fop_data_t * fop, int32_t state)
             return EC_STATE_PREPARE_ANSWER;
 
         case EC_STATE_PREPARE_ANSWER:
-            cbk = fop->answer;
-            if (cbk != NULL)
-            {
-                if (!ec_dict_combine(cbk, EC_COMBINE_XDATA))
-                {
-                    if (cbk->op_ret >= 0)
-                    {
-                        cbk->op_ret = -1;
-                        cbk->op_errno = EIO;
-                    }
-                }
-                if (cbk->op_ret < 0)
-                {
-                    ec_fop_set_error(fop, cbk->op_errno);
-                }
-                else
-                {
-                    ec_iatt_rebuild(fop->xl->private, cbk->iatt, 2,
-                                    cbk->count);
-
-                    cbk->iatt[0].ia_size = fop->pre_size;
-                    cbk->iatt[1].ia_size = fop->user_size;
-                    fop->post_size = fop->user_size;
-                    if ((fop->pre_size > fop->post_size) &&
-                        (fop->user_size != fop->offset))
-                    {
-                        if (!ec_truncate_clean(fop))
-                        {
-                            ec_fop_set_error(fop, EIO);
-                        }
+            cbk = ec_fop_prepare_answer(fop, _gf_false);
+            if (cbk != NULL) {
+                int32_t err;
+
+                ec_iatt_rebuild(fop->xl->private, cbk->iatt, 2, cbk->count);
+
+                /* This shouldn't fail because we have the inode locked. */
+                /* Inode size doesn't need to be updated under locks, because
+                 * conflicting operations won't be in-flight
+                 */
+                GF_ASSERT(ec_get_inode_size(fop, fop->locks[0].lock->loc.inode,
+                                            &cbk->iatt[0].ia_size));
+                cbk->iatt[1].ia_size = fop->user_size;
+                /* This shouldn't fail because we have the inode locked. */
+                GF_ASSERT(ec_set_inode_size(fop, fop->locks[0].lock->loc.inode,
+                                            fop->user_size));
+                if ((cbk->iatt[0].ia_size > cbk->iatt[1].ia_size) &&
+                    (fop->user_size != fop->offset)) {
+                    err = ec_truncate_clean(fop);
+                    if (err != 0) {
+                        ec_cbk_set_error(cbk, -err, _gf_false);
                     }
                 }
             }
-            else
-            {
-                ec_fop_set_error(fop, EIO);
-            }
 
             return EC_STATE_REPORT;
 
@@ -1060,24 +1470,17 @@ int32_t ec_manager_truncate(ec_fop_data_t * fop, int32_t state)
 
             GF_ASSERT(cbk != NULL);
 
-            if (fop->id == GF_FOP_TRUNCATE)
-            {
-                if (fop->cbks.truncate != NULL)
-                {
-                    fop->cbks.truncate(fop->req_frame, fop, fop->xl,
-                                       cbk->op_ret, cbk->op_errno,
-                                       &cbk->iatt[0], &cbk->iatt[1],
-                                       cbk->xdata);
+            if (fop->id == GF_FOP_TRUNCATE) {
+                if (fop->cbks.truncate != NULL) {
+                    QUORUM_CBK(fop->cbks.truncate, fop, fop->req_frame, fop,
+                               fop->xl, cbk->op_ret, cbk->op_errno,
+                               &cbk->iatt[0], &cbk->iatt[1], cbk->xdata);
                 }
-            }
-            else
-            {
-                if (fop->cbks.ftruncate != NULL)
-                {
-                    fop->cbks.ftruncate(fop->req_frame, fop, fop->xl,
-                                        cbk->op_ret, cbk->op_errno,
-                                        &cbk->iatt[0], &cbk->iatt[1],
-                                        cbk->xdata);
+            } else {
+                if (fop->cbks.ftruncate != NULL) {
+                    QUORUM_CBK(fop->cbks.ftruncate, fop, fop->req_frame, fop,
+                               fop->xl, cbk->op_ret, cbk->op_errno,
+                               &cbk->iatt[0], &cbk->iatt[1], cbk->xdata);
                 }
             }
 
@@ -1085,24 +1488,18 @@ int32_t ec_manager_truncate(ec_fop_data_t * fop, int32_t state)
 
         case -EC_STATE_INIT:
         case -EC_STATE_LOCK:
-        case -EC_STATE_GET_SIZE_AND_VERSION:
         case -EC_STATE_DISPATCH:
         case -EC_STATE_PREPARE_ANSWER:
         case -EC_STATE_REPORT:
             GF_ASSERT(fop->error != 0);
 
-            if (fop->id == GF_FOP_TRUNCATE)
-            {
-                if (fop->cbks.truncate != NULL)
-                {
+            if (fop->id == GF_FOP_TRUNCATE) {
+                if (fop->cbks.truncate != NULL) {
                     fop->cbks.truncate(fop->req_frame, fop, fop->xl, -1,
                                        fop->error, NULL, NULL, NULL);
                 }
-            }
-            else
-            {
-                if (fop->cbks.ftruncate != NULL)
-                {
+            } else {
+                if (fop->cbks.ftruncate != NULL) {
                     fop->cbks.ftruncate(fop->req_frame, fop, fop->xl, -1,
                                         fop->error, NULL, NULL, NULL);
                 }
@@ -1123,54 +1520,51 @@ int32_t ec_manager_truncate(ec_fop_data_t * fop, int32_t state)
             return EC_STATE_END;
 
         default:
-            gf_log(fop->xl->name, GF_LOG_ERROR, "Unhandled state %d for %s",
-                   state, ec_fop_name(fop->id));
+            gf_msg(fop->xl->name, GF_LOG_ERROR, EINVAL, EC_MSG_UNHANDLED_STATE,
+                   "Unhandled state %d for %s", state, ec_fop_name(fop->id));
 
             return EC_STATE_END;
     }
 }
 
-void ec_truncate(call_frame_t * frame, xlator_t * this, uintptr_t target,
-                 int32_t minimum, fop_truncate_cbk_t func, void * data,
-                 loc_t * loc, off_t offset, dict_t * xdata)
+void
+ec_truncate(call_frame_t *frame, xlator_t *this, uintptr_t target,
+            uint32_t fop_flags, fop_truncate_cbk_t func, void *data, loc_t *loc,
+            off_t offset, dict_t *xdata)
 {
-    ec_cbk_t callback = { .truncate = func };
-    ec_fop_data_t * fop = NULL;
-    int32_t error = EIO;
+    ec_cbk_t callback = {.truncate = func};
+    ec_fop_data_t *fop = NULL;
+    int32_t error = ENOMEM;
 
-    gf_log("ec", GF_LOG_TRACE, "EC(TRUNCATE) %p", frame);
+    gf_msg_trace("ec", 0, "EC(TRUNCATE) %p", frame);
 
     VALIDATE_OR_GOTO(this, out);
     GF_VALIDATE_OR_GOTO(this->name, frame, out);
     GF_VALIDATE_OR_GOTO(this->name, this->private, out);
 
-    fop = ec_fop_data_allocate(frame, this, GF_FOP_TRUNCATE,
-                               EC_FLAG_UPDATE_LOC_INODE, target, minimum,
-                               ec_wind_truncate, ec_manager_truncate, callback,
-                               data);
-    if (fop == NULL)
-    {
+    fop = ec_fop_data_allocate(frame, this, GF_FOP_TRUNCATE, 0, target,
+                               fop_flags, ec_wind_truncate, ec_manager_truncate,
+                               callback, data);
+    if (fop == NULL) {
         goto out;
     }
 
     fop->offset = offset;
 
-    if (loc != NULL)
-    {
-        if (loc_copy(&fop->loc[0], loc) != 0)
-        {
-            gf_log(this->name, GF_LOG_ERROR, "Failed to copy a location.");
+    if (loc != NULL) {
+        if (loc_copy(&fop->loc[0], loc) != 0) {
+            gf_msg(this->name, GF_LOG_ERROR, ENOMEM, EC_MSG_LOC_COPY_FAIL,
+                   "Failed to copy a location.");
 
             goto out;
         }
     }
-    if (xdata != NULL)
-    {
-        fop->xdata = dict_ref(xdata);
-        if (fop->xdata == NULL)
-        {
-            gf_log(this->name, GF_LOG_ERROR, "Failed to reference a "
-                                             "dictionary.");
+    if (xdata != NULL) {
+        fop->xdata = dict_copy_with_ref(xdata, NULL);
+        if (fop->xdata == NULL) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_DICT_REF_FAIL,
+                   "Failed to reference a "
+                   "dictionary.");
 
             goto out;
         }
@@ -1179,28 +1573,26 @@ void ec_truncate(call_frame_t * frame, xlator_t * this, uintptr_t target,
     error = 0;
 
 out:
-    if (fop != NULL)
-    {
+    if (fop != NULL) {
         ec_manager(fop, error);
-    }
-    else
-    {
-        func(frame, NULL, this, -1, EIO, NULL, NULL, NULL);
+    } else {
+        func(frame, NULL, this, -1, error, NULL, NULL, NULL);
     }
 }
 
 /* FOP: ftruncate */
 
-int32_t ec_ftruncate_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                          int32_t op_ret, int32_t op_errno,
-                          struct iatt *prestat, struct iatt *poststat,
-                          dict_t *xdata)
+int32_t
+ec_ftruncate_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                 int32_t op_ret, int32_t op_errno, struct iatt *prestat,
+                 struct iatt *poststat, dict_t *xdata)
 {
-        return ec_inode_write_cbk (frame, this, cookie, op_ret, op_errno,
-                                   prestat, poststat, xdata);
+    return ec_inode_write_cbk(frame, this, cookie, op_ret, op_errno, prestat,
+                              poststat, xdata);
 }
 
-void ec_wind_ftruncate(ec_t * ec, ec_fop_data_t * fop, int32_t idx)
+void
+ec_wind_ftruncate(ec_t *ec, ec_fop_data_t *fop, int32_t idx)
 {
     ec_trace("WIND", fop, "idx=%d", idx);
 
@@ -1209,26 +1601,25 @@ void ec_wind_ftruncate(ec_t * ec, ec_fop_data_t * fop, int32_t idx)
                       fop->fd, fop->offset, fop->xdata);
 }
 
-void ec_ftruncate(call_frame_t * frame, xlator_t * this, uintptr_t target,
-                  int32_t minimum, fop_ftruncate_cbk_t func, void * data,
-                  fd_t * fd, off_t offset, dict_t * xdata)
+void
+ec_ftruncate(call_frame_t *frame, xlator_t *this, uintptr_t target,
+             uint32_t fop_flags, fop_ftruncate_cbk_t func, void *data, fd_t *fd,
+             off_t offset, dict_t *xdata)
 {
-    ec_cbk_t callback = { .ftruncate = func };
-    ec_fop_data_t * fop = NULL;
-    int32_t error = EIO;
+    ec_cbk_t callback = {.ftruncate = func};
+    ec_fop_data_t *fop = NULL;
+    int32_t error = ENOMEM;
 
-    gf_log("ec", GF_LOG_TRACE, "EC(FTRUNCATE) %p", frame);
+    gf_msg_trace("ec", 0, "EC(FTRUNCATE) %p", frame);
 
     VALIDATE_OR_GOTO(this, out);
     GF_VALIDATE_OR_GOTO(this->name, frame, out);
     GF_VALIDATE_OR_GOTO(this->name, this->private, out);
 
-    fop = ec_fop_data_allocate(frame, this, GF_FOP_FTRUNCATE,
-                               EC_FLAG_UPDATE_FD_INODE, target, minimum,
-                               ec_wind_ftruncate, ec_manager_truncate,
-                               callback, data);
-    if (fop == NULL)
-    {
+    fop = ec_fop_data_allocate(frame, this, GF_FOP_FTRUNCATE, 0, target,
+                               fop_flags, ec_wind_ftruncate,
+                               ec_manager_truncate, callback, data);
+    if (fop == NULL) {
         goto out;
     }
 
@@ -1236,24 +1627,22 @@ void ec_ftruncate(call_frame_t * frame, xlator_t * this, uintptr_t target,
 
     fop->offset = offset;
 
-    if (fd != NULL)
-    {
+    if (fd != NULL) {
         fop->fd = fd_ref(fd);
-        if (fop->fd == NULL)
-        {
-            gf_log(this->name, GF_LOG_ERROR, "Failed to reference a "
-                                             "file descriptor.");
+        if (fop->fd == NULL) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_FILE_DESC_REF_FAIL,
+                   "Failed to reference a "
+                   "file descriptor.");
 
             goto out;
         }
     }
-    if (xdata != NULL)
-    {
-        fop->xdata = dict_ref(xdata);
-        if (fop->xdata == NULL)
-        {
-            gf_log(this->name, GF_LOG_ERROR, "Failed to reference a "
-                                             "dictionary.");
+    if (xdata != NULL) {
+        fop->xdata = dict_copy_with_ref(xdata, NULL);
+        if (fop->xdata == NULL) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_DICT_REF_FAIL,
+                   "Failed to reference a "
+                   "dictionary.");
 
             goto out;
         }
@@ -1262,35 +1651,100 @@ void ec_ftruncate(call_frame_t * frame, xlator_t * this, uintptr_t target,
     error = 0;
 
 out:
-    if (fop != NULL)
-    {
+    if (fop != NULL) {
         ec_manager(fop, error);
-    }
-    else
-    {
-        func(frame, NULL, this, -1, EIO, NULL, NULL, NULL);
+    } else {
+        func(frame, NULL, this, -1, error, NULL, NULL, NULL);
     }
 }
 
 /* FOP: writev */
+static ec_stripe_t *
+ec_allocate_stripe(ec_t *ec, ec_stripe_list_t *stripe_cache)
+{
+    ec_stripe_t *stripe = NULL;
 
-int32_t ec_writev_merge_tail(call_frame_t * frame, void * cookie,
-                             xlator_t * this, int32_t op_ret, int32_t op_errno,
-                             struct iovec * vector, int32_t count,
-                             struct iatt * stbuf, struct iobref * iobref,
-                             dict_t * xdata)
+    if (stripe_cache->count >= stripe_cache->max) {
+        GF_ASSERT(!list_empty(&stripe_cache->lru));
+        stripe = list_first_entry(&stripe_cache->lru, ec_stripe_t, lru);
+        list_move_tail(&stripe->lru, &stripe_cache->lru);
+        GF_ATOMIC_INC(ec->stats.stripe_cache.evicts);
+    } else {
+        stripe = GF_MALLOC(sizeof(ec_stripe_t) + ec->stripe_size,
+                           ec_mt_ec_stripe_t);
+        if (stripe != NULL) {
+            stripe_cache->count++;
+            list_add_tail(&stripe->lru, &stripe_cache->lru);
+            GF_ATOMIC_INC(ec->stats.stripe_cache.allocs);
+        } else {
+            GF_ATOMIC_INC(ec->stats.stripe_cache.errors);
+        }
+    }
+
+    return stripe;
+}
+
+static void
+ec_write_stripe_data(ec_t *ec, ec_fop_data_t *fop, ec_stripe_t *stripe)
 {
-    ec_t * ec = this->private;
-    ec_fop_data_t * fop = frame->local;
-    size_t size, base, tmp;
+    off_t base;
 
-    if (op_ret >= 0)
-    {
+    base = fop->size - ec->stripe_size;
+    memcpy(stripe->data, fop->vector[0].iov_base + base, ec->stripe_size);
+    stripe->frag_offset = fop->frag_range.last - ec->fragment_size;
+}
+
+static void
+ec_add_stripe_in_cache(ec_t *ec, ec_fop_data_t *fop)
+{
+    ec_inode_t *ctx = NULL;
+    ec_stripe_t *stripe = NULL;
+    ec_stripe_list_t *stripe_cache = NULL;
+    gf_boolean_t failed = _gf_true;
+
+    LOCK(&fop->fd->inode->lock);
+
+    ctx = __ec_inode_get(fop->fd->inode, fop->xl);
+    if (ctx == NULL) {
+        goto out;
+    }
+
+    stripe_cache = &ctx->stripe_cache;
+    if (stripe_cache->max > 0) {
+        stripe = ec_allocate_stripe(ec, stripe_cache);
+        if (stripe == NULL) {
+            goto out;
+        }
+
+        ec_write_stripe_data(ec, fop, stripe);
+    }
+
+    failed = _gf_false;
+
+out:
+    UNLOCK(&fop->fd->inode->lock);
+
+    if (failed) {
+        gf_msg(ec->xl->name, GF_LOG_DEBUG, ENOMEM, EC_MSG_FILE_DESC_REF_FAIL,
+               "Failed to create and add stripe in cache");
+    }
+}
+
+int32_t
+ec_writev_merge_tail(call_frame_t *frame, void *cookie, xlator_t *this,
+                     int32_t op_ret, int32_t op_errno, struct iovec *vector,
+                     int32_t count, struct iatt *stbuf, struct iobref *iobref,
+                     dict_t *xdata)
+{
+    ec_t *ec = this->private;
+    ec_fop_data_t *fop = frame->local;
+    uint64_t size, base, tmp;
+
+    if (op_ret >= 0) {
         tmp = 0;
         size = fop->size - fop->user_size - fop->head;
         base = ec->stripe_size - size;
-        if (op_ret > base)
-        {
+        if (op_ret > base) {
             tmp = min(op_ret - base, size);
             ec_iov_copy_to(fop->vector[0].iov_base + fop->size - size, vector,
                            count, base, tmp);
@@ -1298,46 +1752,44 @@ int32_t ec_writev_merge_tail(call_frame_t * frame, void * cookie,
             size -= tmp;
         }
 
-        if (size > 0)
-        {
+        if (size > 0) {
             memset(fop->vector[0].iov_base + fop->size - size, 0, size);
         }
-    }
 
+        if (ec->stripe_cache) {
+            ec_add_stripe_in_cache(ec, fop);
+        }
+    }
     return 0;
 }
 
-int32_t ec_writev_merge_head(call_frame_t * frame, void * cookie,
-                             xlator_t * this, int32_t op_ret, int32_t op_errno,
-                             struct iovec * vector, int32_t count,
-                             struct iatt * stbuf, struct iobref * iobref,
-                             dict_t * xdata)
+int32_t
+ec_writev_merge_head(call_frame_t *frame, void *cookie, xlator_t *this,
+                     int32_t op_ret, int32_t op_errno, struct iovec *vector,
+                     int32_t count, struct iatt *stbuf, struct iobref *iobref,
+                     dict_t *xdata)
 {
-    ec_t * ec = this->private;
-    ec_fop_data_t * fop = frame->local;
-    size_t size, base;
+    ec_t *ec = this->private;
+    ec_fop_data_t *fop = frame->local;
+    uint64_t size, base;
 
-    if (op_ret >= 0)
-    {
+    if (op_ret >= 0) {
         size = fop->head;
         base = 0;
 
-        if (op_ret > 0)
-        {
+        if (op_ret > 0) {
             base = min(op_ret, size);
             ec_iov_copy_to(fop->vector[0].iov_base, vector, count, 0, base);
 
             size -= base;
         }
 
-        if (size > 0)
-        {
+        if (size > 0) {
             memset(fop->vector[0].iov_base + base, 0, size);
         }
 
         size = fop->size - fop->user_size - fop->head;
-        if ((size > 0) && (fop->size == ec->stripe_size))
-        {
+        if ((size > 0) && (fop->size == ec->stripe_size)) {
             ec_writev_merge_tail(frame, cookie, this, op_ret, op_errno, vector,
                                  count, stbuf, iobref, xdata);
         }
@@ -1346,198 +1798,375 @@ int32_t ec_writev_merge_head(call_frame_t * frame, void * cookie,
     return 0;
 }
 
-void ec_writev_start(ec_fop_data_t *fop)
+static int
+ec_make_internal_fop_xdata(dict_t **xdata)
 {
-    ec_t *ec = fop->xl->private;
-    struct iobref *iobref = NULL;
-    struct iobuf *iobuf = NULL;
-    void *ptr = NULL;
-    ec_fd_t *ctx;
-    fd_t *fd;
-    size_t tail;
-    uid_t uid;
-    gid_t gid;
+    dict_t *dict = NULL;
 
-    fd = fd_anonymous(fop->fd->inode);
-    if (fd == NULL) {
-        ec_fop_set_error(fop, EIO);
+    if (*xdata)
+        return 0;
 
-        return;
-    }
+    dict = dict_new();
+    if (!dict)
+        goto out;
 
-    uid = fop->frame->root->uid;
-    fop->frame->root->uid = 0;
-    gid = fop->frame->root->gid;
-    fop->frame->root->gid = 0;
+    if (dict_set_str(dict, GLUSTERFS_INTERNAL_FOP_KEY, "yes"))
+        goto out;
 
-    ctx = ec_fd_get(fop->fd, fop->xl);
-    if (ctx != NULL) {
-        if ((ctx->flags & O_APPEND) != 0) {
-            fop->offset = fop->pre_size;
+    *xdata = dict;
+    return 0;
+out:
+    if (dict)
+        dict_unref(dict);
+    return -1;
+}
+
+static int32_t
+ec_writev_prepare_buffers(ec_t *ec, ec_fop_data_t *fop)
+{
+    struct iobref *iobref = NULL;
+    struct iovec *iov;
+    void *ptr;
+    int32_t err;
+
+    fop->user_size = iov_length(fop->vector, fop->int32);
+    fop->head = ec_adjust_offset_down(ec, &fop->offset, _gf_false);
+    fop->frag_range.first = fop->offset / ec->fragments;
+    fop->size = fop->user_size + fop->head;
+    ec_adjust_size_up(ec, &fop->size, _gf_false);
+    fop->frag_range.last = fop->frag_range.first + fop->size / ec->fragments;
+
+    if ((fop->int32 != 1) || (fop->head != 0) || (fop->size > fop->user_size) ||
+        !EC_ALIGN_CHECK(fop->vector[0].iov_base, EC_METHOD_WORD_SIZE)) {
+        err = ec_buffer_alloc(ec->xl, fop->size, &iobref, &ptr);
+        if (err != 0) {
+            goto out;
         }
+
+        ec_iov_copy_to(ptr + fop->head, fop->vector, fop->int32, 0,
+                       fop->user_size);
+
+        fop->vector[0].iov_base = ptr;
+        fop->vector[0].iov_len = fop->size;
+
+        iobref_unref(fop->buffers);
+        fop->buffers = iobref;
     }
 
-    fop->user_size = iov_length(fop->vector, fop->int32);
-    fop->head = ec_adjust_offset(ec, &fop->offset, 0);
-    fop->size = ec_adjust_size(ec, fop->user_size + fop->head, 0);
+    if (fop->int32 != 2) {
+        iov = GF_MALLOC(VECTORSIZE(2), gf_common_mt_iovec);
+        if (iov == NULL) {
+            err = -ENOMEM;
 
-    iobref = iobref_new();
-    if (iobref == NULL) {
-        goto out;
+            goto out;
+        }
+        iov[0].iov_base = fop->vector[0].iov_base;
+        iov[0].iov_len = fop->vector[0].iov_len;
+
+        GF_FREE(fop->vector);
+        fop->vector = iov;
     }
-    iobuf = iobuf_get2(fop->xl->ctx->iobuf_pool, fop->size);
-    if (iobuf == NULL) {
+
+    fop->vector[1].iov_len = fop->size / ec->fragments;
+    err = ec_buffer_alloc(ec->xl, fop->vector[1].iov_len * ec->nodes,
+                          &fop->buffers, &fop->vector[1].iov_base);
+    if (err != 0) {
         goto out;
     }
-    if (iobref_add(iobref, iobuf) != 0) {
-        goto out;
+
+    err = 0;
+
+out:
+    return err;
+}
+
+static void
+ec_merge_stripe_head_locked(ec_t *ec, ec_fop_data_t *fop, ec_stripe_t *stripe)
+{
+    uint32_t head, size;
+
+    head = fop->head;
+    memcpy(fop->vector[0].iov_base, stripe->data, head);
+
+    size = ec->stripe_size - head;
+    if (size > fop->user_size) {
+        head += fop->user_size;
+        size = ec->stripe_size - head;
+        memcpy(fop->vector[0].iov_base + head, stripe->data + head, size);
     }
+}
 
-    ptr = iobuf->ptr + fop->head;
-    ec_iov_copy_to(ptr, fop->vector, fop->int32, 0, fop->user_size);
+static void
+ec_merge_stripe_tail_locked(ec_t *ec, ec_fop_data_t *fop, ec_stripe_t *stripe)
+{
+    uint32_t head, tail;
+    off_t offset;
 
-    fop->vector[0].iov_base = iobuf->ptr;
-    fop->vector[0].iov_len = fop->size;
+    offset = fop->user_size + fop->head;
+    tail = fop->size - offset;
+    head = ec->stripe_size - tail;
 
-    iobuf_unref(iobuf);
+    memcpy(fop->vector[0].iov_base + offset, stripe->data + head, tail);
+}
 
-    iobref_unref(fop->buffers);
-    fop->buffers = iobref;
+static ec_stripe_t *
+ec_get_stripe_from_cache_locked(ec_t *ec, ec_fop_data_t *fop,
+                                uint64_t frag_offset)
+{
+    ec_inode_t *ctx = NULL;
+    ec_stripe_t *stripe = NULL;
+    ec_stripe_list_t *stripe_cache = NULL;
 
-    if (fop->head > 0)
-    {
-        ec_readv(fop->frame, fop->xl, -1, EC_MINIMUM_MIN, ec_writev_merge_head,
-                 NULL, fd, ec->stripe_size, fop->offset, 0, NULL);
+    ctx = __ec_inode_get(fop->fd->inode, fop->xl);
+    if (ctx == NULL) {
+        GF_ATOMIC_INC(ec->stats.stripe_cache.errors);
+        return NULL;
     }
-    tail = fop->size - fop->user_size - fop->head;
-    if ((tail > 0) && ((fop->head == 0) || (fop->size > ec->stripe_size)))
+
+    stripe_cache = &ctx->stripe_cache;
+    list_for_each_entry(stripe, &stripe_cache->lru, lru)
     {
-        if (fop->pre_size > fop->offset + fop->head + fop->user_size)
-        {
-            ec_readv(fop->frame, fop->xl, -1, EC_MINIMUM_MIN,
-                     ec_writev_merge_tail, NULL, fd, ec->stripe_size,
-                     fop->offset + fop->size - ec->stripe_size, 0, NULL);
-        }
-        else
-        {
-            memset(fop->vector[0].iov_base + fop->size - tail, 0, tail);
+        if (stripe->frag_offset == frag_offset) {
+            list_move_tail(&stripe->lru, &stripe_cache->lru);
+            GF_ATOMIC_INC(ec->stats.stripe_cache.hits);
+            return stripe;
         }
     }
 
-    fop->frame->root->uid = uid;
-    fop->frame->root->gid = gid;
+    GF_ATOMIC_INC(ec->stats.stripe_cache.misses);
 
-    fd_unref(fd);
+    return NULL;
+}
 
-    return;
+static gf_boolean_t
+ec_get_and_merge_stripe(ec_t *ec, ec_fop_data_t *fop, ec_stripe_part_t which)
+{
+    uint64_t frag_offset;
+    ec_stripe_t *stripe = NULL;
+    gf_boolean_t found = _gf_false;
 
-out:
-    if (iobuf != NULL) {
-        iobuf_unref(iobuf);
-    }
-    if (iobref != NULL) {
-        iobref_unref(iobref);
+    if (!ec->stripe_cache) {
+        return found;
     }
 
-    fop->frame->root->uid = uid;
-    fop->frame->root->gid = gid;
+    LOCK(&fop->fd->inode->lock);
+    if (which == EC_STRIPE_HEAD) {
+        frag_offset = fop->frag_range.first;
+        stripe = ec_get_stripe_from_cache_locked(ec, fop, frag_offset);
+        if (stripe) {
+            ec_merge_stripe_head_locked(ec, fop, stripe);
+            found = _gf_true;
+        }
+    }
 
-    fd_unref(fd);
+    if (which == EC_STRIPE_TAIL) {
+        frag_offset = fop->frag_range.last - ec->fragment_size;
+        stripe = ec_get_stripe_from_cache_locked(ec, fop, frag_offset);
+        if (stripe) {
+            ec_merge_stripe_tail_locked(ec, fop, stripe);
+            found = _gf_true;
+        }
+    }
+    UNLOCK(&fop->fd->inode->lock);
 
-    ec_fop_set_error(fop, EIO);
+    return found;
 }
 
-int32_t ec_writev_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                       int32_t op_ret, int32_t op_errno, struct iatt *prestat,
-                       struct iatt *poststat, dict_t *xdata)
+static uintptr_t
+ec_get_lock_good_mask(inode_t *inode, xlator_t *xl)
 {
-        ec_t    *ec = NULL;
-        if (this && this->private) {
-                ec = this->private;
-                if ((op_ret > 0) && ((op_ret % ec->fragment_size) != 0)) {
-                        op_ret = -1;
-                        op_errno = EIO;
-                }
-        }
-        return ec_inode_write_cbk (frame, this, cookie, op_ret, op_errno,
-                                   prestat, poststat, xdata);
+    ec_lock_t *lock = NULL;
+    ec_inode_t *ictx = NULL;
+    LOCK(&inode->lock);
+    {
+        ictx = __ec_inode_get(inode, xl);
+        if (ictx)
+            lock = ictx->inode_lock;
+    }
+    UNLOCK(&inode->lock);
+    if (lock)
+        return lock->good_mask;
+    return 0;
 }
 
-void ec_wind_writev(ec_t * ec, ec_fop_data_t * fop, int32_t idx)
+void
+ec_writev_start(ec_fop_data_t *fop)
 {
-    ec_trace("WIND", fop, "idx=%d", idx);
+    ec_t *ec = fop->xl->private;
+    ec_fd_t *ctx;
+    fd_t *fd;
+    dict_t *xdata = NULL;
+    uint64_t tail, current;
+    int32_t err = -ENOMEM;
+    gf_boolean_t found_stripe = _gf_false;
 
-    struct iovec vector[1];
-    struct iobref * iobref = NULL;
-    struct iobuf * iobuf = NULL;
-    ssize_t size = 0, bufsize = 0;
+    /* This shouldn't fail because we have the inode locked. */
+    GF_ASSERT(ec_get_inode_size(fop, fop->fd->inode, &current));
 
-    iobref = iobref_new();
-    if (iobref == NULL)
-    {
-        goto out;
+    fd = fd_anonymous(fop->fd->inode);
+    if (fd == NULL) {
+        goto failed;
     }
 
-    size = fop->vector[0].iov_len;
-    bufsize = size / ec->fragments;
+    fop->frame->root->uid = 0;
+    fop->frame->root->gid = 0;
 
-    iobuf = iobuf_get2(fop->xl->ctx->iobuf_pool, bufsize);
-    if (iobuf == NULL)
-    {
-        goto out;
+    ctx = ec_fd_get(fop->fd, fop->xl);
+    if (ctx != NULL) {
+        if ((ctx->flags & O_APPEND) != 0) {
+            /* Appending writes take full locks so size won't change because
+             * of any parallel operations
+             */
+            fop->offset = current;
+        }
     }
-    if (iobref_add(iobref, iobuf) != 0)
-    {
-        goto out;
+
+    err = ec_writev_prepare_buffers(ec, fop);
+    if (err != 0) {
+        goto failed_fd;
+    }
+    tail = fop->size - fop->user_size - fop->head;
+    if (fop->head > 0) {
+        if (current > fop->offset) {
+            found_stripe = ec_get_and_merge_stripe(ec, fop, EC_STRIPE_HEAD);
+            if (!found_stripe) {
+                if (ec_make_internal_fop_xdata(&xdata)) {
+                    err = -ENOMEM;
+                    goto failed_xdata;
+                }
+                ec_readv(fop->frame, fop->xl,
+                         ec_get_lock_good_mask(fop->fd->inode, fop->xl),
+                         EC_MINIMUM_MIN, ec_writev_merge_head, NULL, fd,
+                         ec->stripe_size, fop->offset, 0, xdata);
+            }
+        } else {
+            memset(fop->vector[0].iov_base, 0, fop->head);
+            memset(fop->vector[0].iov_base + fop->size - tail, 0, tail);
+            if (ec->stripe_cache && (fop->size <= ec->stripe_size)) {
+                ec_add_stripe_in_cache(ec, fop);
+            }
+        }
+    }
+
+    if ((tail > 0) && ((fop->head == 0) || (fop->size > ec->stripe_size))) {
+        /* Current locking scheme will make sure the 'current' below will
+         * never decrease while the fop is in progress, so the checks will
+         * work as expected
+         */
+        if (current > fop->offset + fop->head + fop->user_size) {
+            found_stripe = ec_get_and_merge_stripe(ec, fop, EC_STRIPE_TAIL);
+            if (!found_stripe) {
+                if (ec_make_internal_fop_xdata(&xdata)) {
+                    err = -ENOMEM;
+                    goto failed_xdata;
+                }
+                ec_readv(fop->frame, fop->xl,
+                         ec_get_lock_good_mask(fop->fd->inode, fop->xl),
+                         EC_MINIMUM_MIN, ec_writev_merge_tail, NULL, fd,
+                         ec->stripe_size,
+                         fop->offset + fop->size - ec->stripe_size, 0, xdata);
+            }
+        } else {
+            memset(fop->vector[0].iov_base + fop->size - tail, 0, tail);
+            if (ec->stripe_cache) {
+                ec_add_stripe_in_cache(ec, fop);
+            }
+        }
     }
 
-    ec_method_encode(size, ec->fragments, idx, fop->vector[0].iov_base,
-                     iobuf->ptr);
+    err = 0;
 
-    vector[0].iov_base = iobuf->ptr;
-    vector[0].iov_len = bufsize;
+failed_xdata:
+    if (xdata) {
+        dict_unref(xdata);
+    }
+failed_fd:
+    fd_unref(fd);
+failed:
+    ec_fop_set_error(fop, -err);
+}
 
-    iobuf_unref(iobuf);
+int32_t
+ec_writev_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int32_t op_ret,
+              int32_t op_errno, struct iatt *prestat, struct iatt *poststat,
+              dict_t *xdata)
+{
+    ec_t *ec = NULL;
+    if (this && this->private) {
+        ec = this->private;
+        if ((op_ret > 0) && ((op_ret % ec->fragment_size) != 0)) {
+            op_ret = -1;
+            op_errno = EIO;
+        }
+    }
+    return ec_inode_write_cbk(frame, this, cookie, op_ret, op_errno, prestat,
+                              poststat, xdata);
+}
 
-    STACK_WIND_COOKIE(fop->frame, ec_writev_cbk, (void *)(uintptr_t)idx,
-                      ec->xl_list[idx], ec->xl_list[idx]->fops->writev,
-                      fop->fd, vector, 1, fop->offset / ec->fragments,
-                      fop->uint32, iobref, fop->xdata);
+void
+ec_wind_writev(ec_t *ec, ec_fop_data_t *fop, int32_t idx)
+{
+    ec_trace("WIND", fop, "idx=%d", idx);
 
-    iobref_unref(iobref);
+    struct iovec vector[1];
+    size_t size;
 
-    return;
+    size = fop->vector[1].iov_len;
 
-out:
-    if (iobuf != NULL)
-    {
-        iobuf_unref(iobuf);
-    }
-    if (iobref != NULL)
-    {
-        iobref_unref(iobref);
-    }
+    vector[0].iov_base = fop->vector[1].iov_base + idx * size;
+    vector[0].iov_len = size;
 
-    ec_writev_cbk(fop->frame, (void *)(uintptr_t)idx, fop->xl, -1, EIO, NULL,
-                  NULL, NULL);
+    STACK_WIND_COOKIE(fop->frame, ec_writev_cbk, (void *)(uintptr_t)idx,
+                      ec->xl_list[idx], ec->xl_list[idx]->fops->writev, fop->fd,
+                      vector, 1, fop->offset / ec->fragments, fop->uint32,
+                      fop->buffers, fop->xdata);
 }
 
-int32_t ec_manager_writev(ec_fop_data_t * fop, int32_t state)
+static void
+ec_writev_encode(ec_fop_data_t *fop)
 {
-    ec_cbk_data_t * cbk;
+    ec_t *ec = fop->xl->private;
+    void *blocks[ec->nodes];
+    uint32_t i;
 
-    switch (state)
-    {
+    blocks[0] = fop->vector[1].iov_base;
+    for (i = 1; i < ec->nodes; i++) {
+        blocks[i] = blocks[i - 1] + fop->vector[1].iov_len;
+    }
+    ec_method_encode(&ec->matrix, fop->vector[0].iov_len,
+                     fop->vector[0].iov_base, blocks);
+}
+
+int32_t
+ec_manager_writev(ec_fop_data_t *fop, int32_t state)
+{
+    ec_cbk_data_t *cbk;
+    ec_fd_t *ctx = NULL;
+    ec_t *ec = fop->xl->private;
+    off_t fl_start = 0;
+    uint64_t fl_size = LONG_MAX;
+
+    switch (state) {
         case EC_STATE_INIT:
         case EC_STATE_LOCK:
-            ec_lock_prepare_fd(fop, fop->fd, 1);
+            ctx = ec_fd_get(fop->fd, fop->xl);
+            if (ctx != NULL) {
+                if ((ctx->flags & O_APPEND) == 0) {
+                    off_t user_size = 0;
+                    off_t head = 0;
+
+                    fl_start = fop->offset;
+                    user_size = iov_length(fop->vector, fop->int32);
+                    head = ec_adjust_offset_down(ec, &fl_start, _gf_true);
+                    fl_size = user_size + head;
+                    ec_adjust_size_up(ec, &fl_size, _gf_true);
+                }
+            }
+            ec_lock_prepare_fd(fop, fop->fd,
+                               EC_UPDATE_DATA | EC_UPDATE_META | EC_QUERY_INFO,
+                               fl_start, fl_size);
             ec_lock(fop);
 
-            return EC_STATE_GET_SIZE_AND_VERSION;
-
-        case EC_STATE_GET_SIZE_AND_VERSION:
-            ec_prepare_update(fop);
-
             return EC_STATE_DISPATCH;
 
         case EC_STATE_DISPATCH:
@@ -1546,62 +2175,60 @@ int32_t ec_manager_writev(ec_fop_data_t * fop, int32_t state)
             return EC_STATE_DELAYED_START;
 
         case EC_STATE_DELAYED_START:
+            /* Restore uid, gid if they were changed to do some partial
+             * reads. */
+            fop->frame->root->uid = fop->uid;
+            fop->frame->root->gid = fop->gid;
+
+            ec_writev_encode(fop);
+
             ec_dispatch_all(fop);
 
             return EC_STATE_PREPARE_ANSWER;
 
         case EC_STATE_PREPARE_ANSWER:
-            cbk = fop->answer;
-            if (cbk != NULL)
-            {
-                if (!ec_dict_combine(cbk, EC_COMBINE_XDATA))
-                {
-                    if (cbk->op_ret >= 0)
-                    {
-                        cbk->op_ret = -1;
-                        cbk->op_errno = EIO;
-                    }
-                }
-                if (cbk->op_ret < 0)
-                {
-                    ec_fop_set_error(fop, cbk->op_errno);
-                }
-                else
-                {
-                    ec_t * ec = fop->xl->private;
-                    size_t size;
+            cbk = ec_fop_prepare_answer(fop, _gf_false);
+            if (cbk != NULL) {
+                ec_t *ec = fop->xl->private;
+                uint64_t size;
 
-                    ec_iatt_rebuild(fop->xl->private, cbk->iatt, 2,
-                                    cbk->count);
+                ec_iatt_rebuild(fop->xl->private, cbk->iatt, 2, cbk->count);
 
+                /* This shouldn't fail because we have the inode locked. */
+                LOCK(&fop->fd->inode->lock);
+                {
+                    GF_ASSERT(__ec_get_inode_size(fop, fop->fd->inode,
+                                                  &cbk->iatt[0].ia_size));
+                    cbk->iatt[1].ia_size = cbk->iatt[0].ia_size;
                     size = fop->offset + fop->head + fop->user_size;
-                    if (size > fop->pre_size)
-                    {
-                        fop->post_size = size;
+                    if (size > cbk->iatt[0].ia_size) {
+                        /* Only update inode size if this is a top level fop.
+                         * Otherwise this is an internal write and the top
+                         * level fop should take care of the real inode size.
+                         */
+                        if (fop->parent == NULL) {
+                            /* This shouldn't fail because we have the inode
+                             * locked. */
+                            GF_ASSERT(
+                                __ec_set_inode_size(fop, fop->fd->inode, size));
+                        }
+                        cbk->iatt[1].ia_size = size;
                     }
+                }
+                UNLOCK(&fop->fd->inode->lock);
 
-                    cbk->iatt[0].ia_size = fop->pre_size;
-                    cbk->iatt[1].ia_size = fop->post_size;
-
+                if (fop->error == 0) {
                     cbk->op_ret *= ec->fragments;
-                    if (cbk->op_ret < fop->head)
-                    {
+                    if (cbk->op_ret < fop->head) {
                         cbk->op_ret = 0;
-                    }
-                    else
-                    {
+                    } else {
                         cbk->op_ret -= fop->head;
                     }
-                    if (cbk->op_ret > fop->user_size)
-                    {
+                    if (cbk->op_ret > fop->user_size) {
                         cbk->op_ret = fop->user_size;
                     }
                 }
             }
-            else
-            {
-                ec_fop_set_error(fop, EIO);
-            }
 
             return EC_STATE_REPORT;
 
@@ -1610,25 +2237,30 @@ int32_t ec_manager_writev(ec_fop_data_t * fop, int32_t state)
 
             GF_ASSERT(cbk != NULL);
 
-            if (fop->cbks.writev != NULL)
-            {
-                fop->cbks.writev(fop->req_frame, fop, fop->xl, cbk->op_ret,
-                                 cbk->op_errno, &cbk->iatt[0], &cbk->iatt[1],
-                                 cbk->xdata);
+            if (fop->cbks.writev != NULL) {
+                QUORUM_CBK(fop->cbks.writev, fop, fop->req_frame, fop, fop->xl,
+                           cbk->op_ret, cbk->op_errno, &cbk->iatt[0],
+                           &cbk->iatt[1], cbk->xdata);
             }
 
             return EC_STATE_LOCK_REUSE;
 
+        case -EC_STATE_DELAYED_START:
+            /* We have failed while doing partial reads. We need to restore
+             * original uid, gid. */
+            fop->frame->root->uid = fop->uid;
+            fop->frame->root->gid = fop->gid;
+
+            /* Fall through */
+
         case -EC_STATE_INIT:
         case -EC_STATE_LOCK:
-        case -EC_STATE_GET_SIZE_AND_VERSION:
         case -EC_STATE_DISPATCH:
         case -EC_STATE_PREPARE_ANSWER:
         case -EC_STATE_REPORT:
             GF_ASSERT(fop->error != 0);
 
-            if (fop->cbks.writev != NULL)
-            {
+            if (fop->cbks.writev != NULL) {
                 fop->cbks.writev(fop->req_frame, fop, fop->xl, -1, fop->error,
                                  NULL, NULL, NULL);
             }
@@ -1648,34 +2280,33 @@ int32_t ec_manager_writev(ec_fop_data_t * fop, int32_t state)
             return EC_STATE_END;
 
         default:
-            gf_log(fop->xl->name, GF_LOG_ERROR, "Unhandled state %d for %s",
-                   state, ec_fop_name(fop->id));
+            gf_msg(fop->xl->name, GF_LOG_ERROR, EINVAL, EC_MSG_UNHANDLED_STATE,
+                   "Unhandled state %d for %s", state, ec_fop_name(fop->id));
 
             return EC_STATE_END;
     }
 }
 
-void ec_writev(call_frame_t * frame, xlator_t * this, uintptr_t target,
-               int32_t minimum, fop_writev_cbk_t func, void * data, fd_t * fd,
-               struct iovec * vector, int32_t count, off_t offset,
-               uint32_t flags, struct iobref * iobref, dict_t * xdata)
+void
+ec_writev(call_frame_t *frame, xlator_t *this, uintptr_t target,
+          uint32_t fop_flags, fop_writev_cbk_t func, void *data, fd_t *fd,
+          struct iovec *vector, int32_t count, off_t offset, uint32_t flags,
+          struct iobref *iobref, dict_t *xdata)
 {
-    ec_cbk_t callback = { .writev = func };
-    ec_fop_data_t * fop = NULL;
-    int32_t error = EIO;
+    ec_cbk_t callback = {.writev = func};
+    ec_fop_data_t *fop = NULL;
+    int32_t error = ENOMEM;
 
-    gf_log("ec", GF_LOG_TRACE, "EC(WRITE) %p", frame);
+    gf_msg_trace("ec", 0, "EC(WRITE) %p", frame);
 
     VALIDATE_OR_GOTO(this, out);
     GF_VALIDATE_OR_GOTO(this->name, frame, out);
     GF_VALIDATE_OR_GOTO(this->name, this->private, out);
 
-    fop = ec_fop_data_allocate(frame, this, GF_FOP_WRITE,
-                               EC_FLAG_UPDATE_FD_INODE, target, minimum,
+    fop = ec_fop_data_allocate(frame, this, GF_FOP_WRITE, 0, target, fop_flags,
                                ec_wind_writev, ec_manager_writev, callback,
                                data);
-    if (fop == NULL)
-    {
+    if (fop == NULL) {
         goto out;
     }
 
@@ -1685,47 +2316,43 @@ void ec_writev(call_frame_t * frame, xlator_t * this, uintptr_t target,
 
     fop->use_fd = 1;
 
-    if (fd != NULL)
-    {
+    if (fd != NULL) {
         fop->fd = fd_ref(fd);
-        if (fop->fd == NULL)
-        {
-            gf_log(this->name, GF_LOG_ERROR, "Failed to reference a "
-                                             "file descriptor.");
+        if (fop->fd == NULL) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_FILE_DESC_REF_FAIL,
+                   "Failed to reference a "
+                   "file descriptor.");
 
             goto out;
         }
     }
-    if (count > 0)
-    {
+    if (count > 0) {
         fop->vector = iov_dup(vector, count);
-        if (fop->vector == NULL)
-        {
-            gf_log(this->name, GF_LOG_ERROR, "Failed to duplicate a "
-                                             "vector list.");
+        if (fop->vector == NULL) {
+            gf_msg(this->name, GF_LOG_ERROR, ENOMEM, EC_MSG_NO_MEMORY,
+                   "Failed to duplicate a "
+                   "vector list.");
 
             goto out;
         }
         fop->int32 = count;
     }
-    if (iobref != NULL)
-    {
+    if (iobref != NULL) {
         fop->buffers = iobref_ref(iobref);
-        if (fop->buffers == NULL)
-        {
-            gf_log(this->name, GF_LOG_ERROR, "Failed to reference a "
-                                             "buffer.");
+        if (fop->buffers == NULL) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_BUF_REF_FAIL,
+                   "Failed to reference a "
+                   "buffer.");
 
             goto out;
         }
     }
-    if (xdata != NULL)
-    {
-        fop->xdata = dict_ref(xdata);
-        if (fop->xdata == NULL)
-        {
-            gf_log(this->name, GF_LOG_ERROR, "Failed to reference a "
-                                             "dictionary.");
+    if (xdata != NULL) {
+        fop->xdata = dict_copy_with_ref(xdata, NULL);
+        if (fop->xdata == NULL) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_DICT_REF_FAIL,
+                   "Failed to reference a "
+                   "dictionary.");
 
             goto out;
         }
@@ -1734,12 +2361,9 @@ void ec_writev(call_frame_t * frame, xlator_t * this, uintptr_t target,
     error = 0;
 
 out:
-    if (fop != NULL)
-    {
+    if (fop != NULL) {
         ec_manager(fop, error);
-    }
-    else
-    {
-        func(frame, NULL, this, -1, EIO, NULL, NULL, NULL);
+    } else {
+        func(frame, NULL, this, -1, error, NULL, NULL, NULL);
     }
 }
diff --git a/xlators/cluster/ec/src/ec-locks.c b/xlators/cluster/ec/src/ec-locks.c
index 2e301631b3f..601960d6154 100644
--- a/xlators/cluster/ec/src/ec-locks.c
+++ b/xlators/cluster/ec/src/ec-locks.c
@@ -8,71 +8,110 @@
   cases as published by the Free Software Foundation.
 */
 
-#include "xlator.h"
-#include "defaults.h"
-
 #include "ec-helpers.h"
 #include "ec-common.h"
 #include "ec-combine.h"
-#include "ec-method.h"
 #include "ec-fops.h"
+#include "ec-messages.h"
 
 #define EC_LOCK_MODE_NONE 0
-#define EC_LOCK_MODE_INC  1
-#define EC_LOCK_MODE_ALL  2
+#define EC_LOCK_MODE_INC 1
+#define EC_LOCK_MODE_ALL 2
 
-int32_t ec_lock_check(ec_fop_data_t * fop, ec_cbk_data_t ** cbk,
-                      uintptr_t * mask)
+int32_t
+ec_lock_check(ec_fop_data_t *fop, uintptr_t *mask)
 {
-    ec_t * ec = fop->xl->private;
-    ec_cbk_data_t * ans = NULL;
-    uintptr_t locked = 0, notlocked = 0;
+    ec_t *ec = fop->xl->private;
+    ec_cbk_data_t *ans = NULL;
+    ec_cbk_data_t *cbk = NULL;
+    uintptr_t locked = 0;
+    int32_t good = 0;
+    int32_t eagain = 0;
+    int32_t estale = 0;
     int32_t error = -1;
 
+    /* There are some errors that we'll handle in an special way while trying
+     * to acquire a lock.
+     *
+     *   EAGAIN:  If it's found during a parallel non-blocking lock request, we
+     *            consider that there's contention on the inode, so we consider
+     *            the acquisition a failure and try again with a sequential
+     *            blocking lock request. This will ensure that we get a lock on
+     *            as many bricks as possible (ignoring EAGAIN here would cause
+     *            unnecessary triggers of self-healing).
+     *
+     *            If it's found during a sequential blocking lock request, it's
+     *            considered an error. Lock will only succeed if there are
+     *            enough other bricks locked.
+     *
+     *   ESTALE:  This can appear during parallel or sequential lock request if
+     *            the inode has just been unlinked. We consider this error is
+     *            not recoverable, but we also don't consider it as fatal. So,
+     *            if it happens during parallel lock, we won't attempt a
+     *            sequential one unless there are EAGAIN errors on other
+     *            bricks (and are enough to form a quorum), but if we reach
+     *            quorum counting the ESTALE bricks, we consider the whole
+     *            result of the operation is ESTALE instead of EIO.
+     */
+
     list_for_each_entry(ans, &fop->cbk_list, list)
     {
-        if (ans->op_ret >= 0)
-        {
-            if (locked != 0)
-            {
+        if (ans->op_ret >= 0) {
+            if (locked != 0) {
                 error = EIO;
             }
             locked |= ans->mask;
-            *cbk = ans;
-        }
-        else if (ans->op_errno == EAGAIN)
-        {
-            notlocked |= ans->mask;
+            good = ans->count;
+            cbk = ans;
+        } else if (ans->op_errno == ESTALE) {
+            estale += ans->count;
+        } else if ((ans->op_errno == EAGAIN) &&
+                   (fop->uint32 != EC_LOCK_MODE_INC)) {
+            eagain += ans->count;
         }
     }
 
-    if (error == -1)
-    {
-        if (ec_bits_count(locked | notlocked) >= ec->fragments)
-        {
-            if (notlocked == 0)
-            {
-                fop->answer = *cbk;
+    if (error == -1) {
+        /* If we have enough quorum with succeeded and EAGAIN answers, we
+         * ignore for now any ESTALE answer. If there are EAGAIN answers,
+         * we retry with a sequential blocking lock request if needed.
+         * Otherwise we succeed. */
+        if ((good + eagain) >= ec->fragments) {
+            if (eagain == 0) {
+                if (fop->answer == NULL) {
+                    fop->answer = cbk;
+                }
 
-                ec_update_bad(fop, locked);
+                ec_update_good(fop, locked);
 
                 error = 0;
-            }
-            else
-            {
-                if (fop->uint32 == EC_LOCK_MODE_NONE)
-                {
-                    error = EAGAIN;
-                }
-                else
-                {
-                    fop->uint32 = EC_LOCK_MODE_INC;
+            } else {
+                switch (fop->uint32) {
+                    case EC_LOCK_MODE_NONE:
+                        error = EAGAIN;
+                        break;
+                    case EC_LOCK_MODE_ALL:
+                        fop->uint32 = EC_LOCK_MODE_INC;
+                        break;
+                    default:
+                        /* This shouldn't happen because eagain cannot be > 0
+                         * when fop->uint32 is EC_LOCK_MODE_INC. */
+                        error = EIO;
+                        break;
                 }
             }
-        }
-        else
-        {
-            error = EIO;
+        } else {
+            /* We have been unable to find enough candidates that will be able
+             * to take the lock. If we have quorum on some answer, we return
+             * it. Otherwise we check if ESTALE answers allow us to reach
+             * quorum. If so, we return ESTALE. */
+            if (fop->answer && fop->answer->op_ret < 0) {
+                error = fop->answer->op_errno;
+            } else if ((good + eagain + estale) >= ec->fragments) {
+                error = ESTALE;
+            } else {
+                error = EIO;
+            }
         }
     }
 
@@ -81,47 +120,26 @@ int32_t ec_lock_check(ec_fop_data_t * fop, ec_cbk_data_t ** cbk,
     return error;
 }
 
-uintptr_t ec_lock_handler(ec_fop_data_t * fop, ec_cbk_data_t * cbk,
-                          ec_combine_f combine)
+int32_t
+ec_lock_unlocked(call_frame_t *frame, void *cookie, xlator_t *this,
+                 int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
-    uintptr_t mask = 0;
-
-    if (fop->uint32 == EC_LOCK_MODE_INC)
-    {
-        if (cbk->op_ret < 0)
-        {
-            if (cbk->op_errno != ENOTCONN)
-            {
-                mask = fop->mask & ~fop->remaining & ~cbk->mask;
-                fop->remaining = 0;
-            }
-        }
-    }
-
-    ec_combine(cbk, combine);
-
-    return mask;
-}
-
-int32_t ec_lock_unlocked(call_frame_t * frame, void * cookie,
-                         xlator_t * this, int32_t op_ret, int32_t op_errno,
-                         dict_t * xdata)
-{
-    if (op_ret < 0)
-    {
-        gf_log(this->name, GF_LOG_WARNING, "Failed to unlock an entry/inode");
+    if (op_ret < 0) {
+        gf_msg(this->name, GF_LOG_WARNING, op_errno, EC_MSG_UNLOCK_FAILED,
+               "Failed to unlock an entry/inode");
     }
 
     return 0;
 }
 
-int32_t ec_lock_lk_unlocked(call_frame_t * frame, void * cookie,
-                            xlator_t * this, int32_t op_ret, int32_t op_errno,
-                            struct gf_flock * flock, dict_t * xdata)
+int32_t
+ec_lock_lk_unlocked(call_frame_t *frame, void *cookie, xlator_t *this,
+                    int32_t op_ret, int32_t op_errno, struct gf_flock *flock,
+                    dict_t *xdata)
 {
-    if (op_ret < 0)
-    {
-        gf_log(this->name, GF_LOG_WARNING, "Failed to unlock an lk");
+    if (op_ret < 0) {
+        gf_msg(this->name, GF_LOG_WARNING, op_errno, EC_MSG_LK_UNLOCK_FAILED,
+               "Failed to unlock an lk");
     }
 
     return 0;
@@ -129,11 +147,12 @@ int32_t ec_lock_lk_unlocked(call_frame_t * frame, void * cookie,
 
 /* FOP: entrylk */
 
-int32_t ec_entrylk_cbk(call_frame_t * frame, void * cookie, xlator_t * this,
-                       int32_t op_ret, int32_t op_errno, dict_t * xdata)
+int32_t
+ec_entrylk_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+               int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
-    ec_fop_data_t * fop = NULL;
-    ec_cbk_data_t * cbk = NULL;
+    ec_fop_data_t *fop = NULL;
+    ec_cbk_data_t *cbk = NULL;
     int32_t idx = (int32_t)(uintptr_t)cookie;
 
     VALIDATE_OR_GOTO(this, out);
@@ -143,46 +162,36 @@ int32_t ec_entrylk_cbk(call_frame_t * frame, void * cookie, xlator_t * this,
 
     fop = frame->local;
 
-    ec_trace("CBK", fop, "idx=%d, frame=%p, op_ret=%d, op_errno=%d", idx,
-             frame, op_ret, op_errno);
+    ec_trace("CBK", fop, "idx=%d, frame=%p, op_ret=%d, op_errno=%d", idx, frame,
+             op_ret, op_errno);
 
     cbk = ec_cbk_data_allocate(frame, this, fop, GF_FOP_ENTRYLK, idx, op_ret,
                                op_errno);
-    if (cbk != NULL)
-    {
-        uintptr_t mask;
-
-        if (xdata != NULL)
-        {
+    if (cbk != NULL) {
+        if (xdata != NULL) {
             cbk->xdata = dict_ref(xdata);
-            if (cbk->xdata == NULL)
-            {
-                gf_log(this->name, GF_LOG_ERROR, "Failed to reference a "
-                                                 "dictionary.");
+            if (cbk->xdata == NULL) {
+                gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_DICT_REF_FAIL,
+                       "Failed to reference a "
+                       "dictionary.");
 
                 goto out;
             }
         }
 
-        mask = ec_lock_handler(fop, cbk, NULL);
-        if (mask != 0)
-        {
-            ec_entrylk(fop->req_frame, fop->xl, mask, 1, ec_lock_unlocked,
-                       NULL, fop->str[0], &fop->loc[0], fop->str[1],
-                       ENTRYLK_UNLOCK, fop->entrylk_type, fop->xdata);
-        }
+        ec_combine(cbk, NULL);
     }
 
 out:
-    if (fop != NULL)
-    {
+    if (fop != NULL) {
         ec_complete(fop);
     }
 
     return 0;
 }
 
-void ec_wind_entrylk(ec_t * ec, ec_fop_data_t * fop, int32_t idx)
+void
+ec_wind_entrylk(ec_t *ec, ec_fop_data_t *fop, int32_t idx)
 {
     ec_trace("WIND", fop, "idx=%d", idx);
 
@@ -192,20 +201,19 @@ void ec_wind_entrylk(ec_t * ec, ec_fop_data_t * fop, int32_t idx)
                       fop->entrylk_type, fop->xdata);
 }
 
-int32_t ec_manager_entrylk(ec_fop_data_t * fop, int32_t state)
+int32_t
+ec_manager_entrylk(ec_fop_data_t *fop, int32_t state)
 {
-    ec_cbk_data_t * cbk;
+    ec_cbk_data_t *cbk;
 
-    switch (state)
-    {
+    switch (state) {
         case EC_STATE_INIT:
-            if (fop->entrylk_cmd == ENTRYLK_LOCK)
-            {
+            if (fop->entrylk_cmd == ENTRYLK_LOCK) {
                 fop->uint32 = EC_LOCK_MODE_ALL;
                 fop->entrylk_cmd = ENTRYLK_LOCK_NB;
             }
 
-        /* Fall through */
+            /* Fall through */
 
         case EC_STATE_DISPATCH:
             ec_dispatch_all(fop);
@@ -213,46 +221,37 @@ int32_t ec_manager_entrylk(ec_fop_data_t * fop, int32_t state)
             return EC_STATE_PREPARE_ANSWER;
 
         case EC_STATE_PREPARE_ANSWER:
-            cbk = fop->answer;
-            if ((cbk == NULL) ||
-                ((cbk->op_ret < 0) && (cbk->op_errno == EAGAIN)))
-            {
+        case -EC_STATE_PREPARE_ANSWER:
+            if (fop->entrylk_cmd != ENTRYLK_UNLOCK) {
                 uintptr_t mask;
 
-                fop->error = ec_lock_check(fop, &cbk, &mask);
-                if (fop->error != 0)
-                {
-                    if (mask != 0)
-                    {
-                        if (fop->id == GF_FOP_ENTRYLK)
-                        {
-                            ec_entrylk(fop->req_frame, fop->xl, mask, 1,
-                                       ec_lock_unlocked, NULL, fop->str[0],
-                                       &fop->loc[0], fop->str[1],
-                                       ENTRYLK_UNLOCK, fop->entrylk_type,
-                                       fop->xdata);
-                        }
-                        else
-                        {
-                            ec_fentrylk(fop->req_frame, fop->xl, mask, 1,
+                ec_fop_set_error(fop, ec_lock_check(fop, &mask));
+                if (fop->error != 0) {
+                    if (mask != 0) {
+                        if (fop->id == GF_FOP_ENTRYLK) {
+                            ec_entrylk(
+                                fop->frame, fop->xl, mask, 1, ec_lock_unlocked,
+                                NULL, fop->str[0], &fop->loc[0], fop->str[1],
+                                ENTRYLK_UNLOCK, fop->entrylk_type, fop->xdata);
+                        } else {
+                            ec_fentrylk(fop->frame, fop->xl, mask, 1,
                                         ec_lock_unlocked, NULL, fop->str[0],
                                         fop->fd, fop->str[1], ENTRYLK_UNLOCK,
                                         fop->entrylk_type, fop->xdata);
                         }
                     }
-                    if (fop->error > 0)
-                    {
-                        return EC_STATE_REPORT;
-                    }
+                    if (fop->error < 0) {
+                        fop->error = 0;
 
-                    fop->error = 0;
+                        fop->entrylk_cmd = ENTRYLK_LOCK;
 
-                    fop->entrylk_cmd = ENTRYLK_LOCK;
+                        ec_dispatch_inc(fop);
 
-                    ec_dispatch_inc(fop);
-
-                    return EC_STATE_PREPARE_ANSWER;
+                        return EC_STATE_PREPARE_ANSWER;
+                    }
                 }
+            } else {
+                ec_fop_prepare_answer(fop, _gf_true);
             }
 
             return EC_STATE_REPORT;
@@ -262,125 +261,104 @@ int32_t ec_manager_entrylk(ec_fop_data_t * fop, int32_t state)
 
             GF_ASSERT(cbk != NULL);
 
-            if (fop->id == GF_FOP_ENTRYLK)
-            {
-                if (fop->cbks.entrylk != NULL)
-                {
-                    fop->cbks.entrylk(fop->req_frame, fop, fop->xl,
-                                      cbk->op_ret, cbk->op_errno, cbk->xdata);
+            if (fop->id == GF_FOP_ENTRYLK) {
+                if (fop->cbks.entrylk != NULL) {
+                    fop->cbks.entrylk(fop->req_frame, fop, fop->xl, cbk->op_ret,
+                                      cbk->op_errno, cbk->xdata);
                 }
-            }
-            else
-            {
-                if (fop->cbks.fentrylk != NULL)
-                {
+            } else {
+                if (fop->cbks.fentrylk != NULL) {
                     fop->cbks.fentrylk(fop->req_frame, fop, fop->xl,
                                        cbk->op_ret, cbk->op_errno, cbk->xdata);
                 }
             }
 
-            ec_wait_winds(fop);
-
             return EC_STATE_END;
 
         case -EC_STATE_INIT:
         case -EC_STATE_DISPATCH:
-        case -EC_STATE_PREPARE_ANSWER:
         case -EC_STATE_REPORT:
             GF_ASSERT(fop->error != 0);
 
-            if (fop->id == GF_FOP_ENTRYLK)
-            {
-                if (fop->cbks.entrylk != NULL)
-                {
+            if (fop->id == GF_FOP_ENTRYLK) {
+                if (fop->cbks.entrylk != NULL) {
                     fop->cbks.entrylk(fop->req_frame, fop, fop->xl, -1,
                                       fop->error, NULL);
                 }
-            }
-            else
-            {
-                if (fop->cbks.fentrylk != NULL)
-                {
+            } else {
+                if (fop->cbks.fentrylk != NULL) {
                     fop->cbks.fentrylk(fop->req_frame, fop, fop->xl, -1,
                                        fop->error, NULL);
                 }
             }
 
-            ec_wait_winds(fop);
-
             return EC_STATE_END;
 
         default:
-            gf_log(fop->xl->name, GF_LOG_ERROR, "Unhandled state %d for %s",
-                   state, ec_fop_name(fop->id));
+            gf_msg(fop->xl->name, GF_LOG_ERROR, EINVAL, EC_MSG_UNHANDLED_STATE,
+                   "Unhandled state %d for %s", state, ec_fop_name(fop->id));
 
             return EC_STATE_END;
     }
 }
 
-void ec_entrylk(call_frame_t * frame, xlator_t * this, uintptr_t target,
-                int32_t minimum, fop_entrylk_cbk_t func, void * data,
-                const char * volume, loc_t * loc, const char * basename,
-                entrylk_cmd cmd, entrylk_type type, dict_t * xdata)
+void
+ec_entrylk(call_frame_t *frame, xlator_t *this, uintptr_t target,
+           uint32_t fop_flags, fop_entrylk_cbk_t func, void *data,
+           const char *volume, loc_t *loc, const char *basename,
+           entrylk_cmd cmd, entrylk_type type, dict_t *xdata)
 {
-    ec_cbk_t callback = { .entrylk = func };
-    ec_fop_data_t * fop = NULL;
-    int32_t error = EIO;
+    ec_cbk_t callback = {.entrylk = func};
+    ec_fop_data_t *fop = NULL;
+    int32_t error = ENOMEM;
 
-    gf_log("ec", GF_LOG_TRACE, "EC(ENTRYLK) %p", frame);
+    gf_msg_trace("ec", 0, "EC(ENTRYLK) %p", frame);
 
-    VALIDATE_OR_GOTO(this, out);
     GF_VALIDATE_OR_GOTO(this->name, frame, out);
     GF_VALIDATE_OR_GOTO(this->name, this->private, out);
 
-    fop = ec_fop_data_allocate(frame, this, GF_FOP_ENTRYLK,
-                               EC_FLAG_UPDATE_LOC_INODE, target, minimum,
-                               ec_wind_entrylk, ec_manager_entrylk, callback,
-                               data);
-    if (fop == NULL)
-    {
+    fop = ec_fop_data_allocate(frame, this, GF_FOP_ENTRYLK, 0, target,
+                               fop_flags, ec_wind_entrylk, ec_manager_entrylk,
+                               callback, data);
+    if (fop == NULL) {
         goto out;
     }
 
     fop->entrylk_cmd = cmd;
     fop->entrylk_type = type;
 
-    if (volume != NULL)
-    {
+    if (volume != NULL) {
         fop->str[0] = gf_strdup(volume);
-        if (fop->str[0] == NULL)
-        {
-            gf_log(this->name, GF_LOG_ERROR, "Failed to duplicate a string.");
+        if (fop->str[0] == NULL) {
+            gf_msg(this->name, GF_LOG_ERROR, ENOMEM, EC_MSG_NO_MEMORY,
+                   "Failed to duplicate a string.");
 
             goto out;
         }
     }
-    if (loc != NULL)
-    {
-        if (loc_copy(&fop->loc[0], loc) != 0)
-        {
-            gf_log(this->name, GF_LOG_ERROR, "Failed to copy a location.");
+    if (loc != NULL) {
+        if (loc_copy(&fop->loc[0], loc) != 0) {
+            gf_msg(this->name, GF_LOG_ERROR, ENOMEM, EC_MSG_LOC_COPY_FAIL,
+                   "Failed to copy a location.");
 
             goto out;
         }
     }
-    if (basename != NULL)
-    {
+    if (basename != NULL) {
         fop->str[1] = gf_strdup(basename);
-        if (fop->str[1] == NULL)
-        {
-            gf_log(this->name, GF_LOG_ERROR, "Failed to duplicate a string.");
+        if (fop->str[1] == NULL) {
+            gf_msg(this->name, GF_LOG_ERROR, ENOMEM, EC_MSG_NO_MEMORY,
+                   "Failed to duplicate a string.");
 
             goto out;
         }
     }
-    if (xdata != NULL)
-    {
+    if (xdata != NULL) {
         fop->xdata = dict_ref(xdata);
-        if (fop->xdata == NULL)
-        {
-            gf_log(this->name, GF_LOG_ERROR, "Failed to reference a "
-                                             "dictionary.");
+        if (fop->xdata == NULL) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_DICT_REF_FAIL,
+                   "Failed to reference a "
+                   "dictionary.");
 
             goto out;
         }
@@ -389,23 +367,21 @@ void ec_entrylk(call_frame_t * frame, xlator_t * this, uintptr_t target,
     error = 0;
 
 out:
-    if (fop != NULL)
-    {
+    if (fop != NULL) {
         ec_manager(fop, error);
-    }
-    else
-    {
-        func(frame, NULL, this, -1, EIO, NULL);
+    } else {
+        func(frame, NULL, this, -1, error, NULL);
     }
 }
 
 /* FOP: fentrylk */
 
-int32_t ec_fentrylk_cbk(call_frame_t * frame, void * cookie, xlator_t * this,
-                        int32_t op_ret, int32_t op_errno, dict_t * xdata)
+int32_t
+ec_fentrylk_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
-    ec_fop_data_t * fop = NULL;
-    ec_cbk_data_t * cbk = NULL;
+    ec_fop_data_t *fop = NULL;
+    ec_cbk_data_t *cbk = NULL;
     int32_t idx = (int32_t)(uintptr_t)cookie;
 
     VALIDATE_OR_GOTO(this, out);
@@ -415,46 +391,36 @@ int32_t ec_fentrylk_cbk(call_frame_t * frame, void * cookie, xlator_t * this,
 
     fop = frame->local;
 
-    ec_trace("CBK", fop, "idx=%d, frame=%p, op_ret=%d, op_errno=%d", idx,
-             frame, op_ret, op_errno);
+    ec_trace("CBK", fop, "idx=%d, frame=%p, op_ret=%d, op_errno=%d", idx, frame,
+             op_ret, op_errno);
 
     cbk = ec_cbk_data_allocate(frame, this, fop, GF_FOP_FENTRYLK, idx, op_ret,
                                op_errno);
-    if (cbk != NULL)
-    {
-        uintptr_t mask;
-
-        if (xdata != NULL)
-        {
+    if (cbk != NULL) {
+        if (xdata != NULL) {
             cbk->xdata = dict_ref(xdata);
-            if (cbk->xdata == NULL)
-            {
-                gf_log(this->name, GF_LOG_ERROR, "Failed to reference a "
-                                                 "dictionary.");
+            if (cbk->xdata == NULL) {
+                gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_DICT_REF_FAIL,
+                       "Failed to reference a "
+                       "dictionary.");
 
                 goto out;
             }
         }
 
-        mask = ec_lock_handler(fop, cbk, NULL);
-        if (mask != 0)
-        {
-            ec_fentrylk(fop->req_frame, fop->xl, mask, 1, ec_lock_unlocked,
-                        NULL, fop->str[0], fop->fd, fop->str[1],
-                        ENTRYLK_UNLOCK, fop->entrylk_type, fop->xdata);
-        }
+        ec_combine(cbk, NULL);
     }
 
 out:
-    if (fop != NULL)
-    {
+    if (fop != NULL) {
         ec_complete(fop);
     }
 
     return 0;
 }
 
-void ec_wind_fentrylk(ec_t * ec, ec_fop_data_t * fop, int32_t idx)
+void
+ec_wind_fentrylk(ec_t *ec, ec_fop_data_t *fop, int32_t idx)
 {
     ec_trace("WIND", fop, "idx=%d", idx);
 
@@ -464,27 +430,25 @@ void ec_wind_fentrylk(ec_t * ec, ec_fop_data_t * fop, int32_t idx)
                       fop->entrylk_type, fop->xdata);
 }
 
-void ec_fentrylk(call_frame_t * frame, xlator_t * this, uintptr_t target,
-                 int32_t minimum, fop_fentrylk_cbk_t func, void * data,
-                 const char * volume, fd_t * fd, const char * basename,
-                 entrylk_cmd cmd, entrylk_type type, dict_t * xdata)
+void
+ec_fentrylk(call_frame_t *frame, xlator_t *this, uintptr_t target,
+            uint32_t fop_flags, fop_fentrylk_cbk_t func, void *data,
+            const char *volume, fd_t *fd, const char *basename, entrylk_cmd cmd,
+            entrylk_type type, dict_t *xdata)
 {
-    ec_cbk_t callback = { .fentrylk = func };
-    ec_fop_data_t * fop = NULL;
-    int32_t error = EIO;
+    ec_cbk_t callback = {.fentrylk = func};
+    ec_fop_data_t *fop = NULL;
+    int32_t error = ENOMEM;
 
-    gf_log("ec", GF_LOG_TRACE, "EC(FENTRYLK) %p", frame);
+    gf_msg_trace("ec", 0, "EC(FENTRYLK) %p", frame);
 
-    VALIDATE_OR_GOTO(this, out);
     GF_VALIDATE_OR_GOTO(this->name, frame, out);
     GF_VALIDATE_OR_GOTO(this->name, this->private, out);
 
-    fop = ec_fop_data_allocate(frame, this, GF_FOP_FENTRYLK,
-                               EC_FLAG_UPDATE_FD_INODE, target, minimum,
-                               ec_wind_fentrylk, ec_manager_entrylk, callback,
-                               data);
-    if (fop == NULL)
-    {
+    fop = ec_fop_data_allocate(frame, this, GF_FOP_FENTRYLK, 0, target,
+                               fop_flags, ec_wind_fentrylk, ec_manager_entrylk,
+                               callback, data);
+    if (fop == NULL) {
         goto out;
     }
 
@@ -493,44 +457,40 @@ void ec_fentrylk(call_frame_t * frame, xlator_t * this, uintptr_t target,
     fop->entrylk_cmd = cmd;
     fop->entrylk_type = type;
 
-    if (volume != NULL)
-    {
+    if (volume != NULL) {
         fop->str[0] = gf_strdup(volume);
-        if (fop->str[0] == NULL)
-        {
-            gf_log(this->name, GF_LOG_ERROR, "Failed to duplicate a string.");
+        if (fop->str[0] == NULL) {
+            gf_msg(this->name, GF_LOG_ERROR, ENOMEM, EC_MSG_NO_MEMORY,
+                   "Failed to duplicate a string.");
 
             goto out;
         }
     }
-    if (fd != NULL)
-    {
+    if (fd != NULL) {
         fop->fd = fd_ref(fd);
-        if (fop->fd == NULL)
-        {
-            gf_log(this->name, GF_LOG_ERROR, "Failed to reference a "
-                                             "file descriptor.");
+        if (fop->fd == NULL) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_FILE_DESC_REF_FAIL,
+                   "Failed to reference a "
+                   "file descriptor.");
 
             goto out;
         }
     }
-    if (basename != NULL)
-    {
+    if (basename != NULL) {
         fop->str[1] = gf_strdup(basename);
-        if (fop->str[1] == NULL)
-        {
-            gf_log(this->name, GF_LOG_ERROR, "Failed to duplicate a string.");
+        if (fop->str[1] == NULL) {
+            gf_msg(this->name, GF_LOG_ERROR, ENOMEM, EC_MSG_NO_MEMORY,
+                   "Failed to duplicate a string.");
 
             goto out;
         }
     }
-    if (xdata != NULL)
-    {
+    if (xdata != NULL) {
         fop->xdata = dict_ref(xdata);
-        if (fop->xdata == NULL)
-        {
-            gf_log(this->name, GF_LOG_ERROR, "Failed to reference a "
-                                             "dictionary.");
+        if (fop->xdata == NULL) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_DICT_REF_FAIL,
+                   "Failed to reference a "
+                   "dictionary.");
 
             goto out;
         }
@@ -539,23 +499,21 @@ void ec_fentrylk(call_frame_t * frame, xlator_t * this, uintptr_t target,
     error = 0;
 
 out:
-    if (fop != NULL)
-    {
+    if (fop != NULL) {
         ec_manager(fop, error);
-    }
-    else
-    {
-        func(frame, NULL, this, -1, EIO, NULL);
+    } else {
+        func(frame, NULL, this, -1, error, NULL);
     }
 }
 
 /* FOP: inodelk */
 
-int32_t ec_inodelk_cbk(call_frame_t * frame, void * cookie, xlator_t * this,
-                       int32_t op_ret, int32_t op_errno, dict_t * xdata)
+int32_t
+ec_inodelk_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+               int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
-    ec_fop_data_t * fop = NULL;
-    ec_cbk_data_t * cbk = NULL;
+    ec_fop_data_t *fop = NULL;
+    ec_cbk_data_t *cbk = NULL;
     int32_t idx = (int32_t)(uintptr_t)cookie;
 
     VALIDATE_OR_GOTO(this, out);
@@ -565,56 +523,36 @@ int32_t ec_inodelk_cbk(call_frame_t * frame, void * cookie, xlator_t * this,
 
     fop = frame->local;
 
-    ec_trace("CBK", fop, "idx=%d, frame=%p, op_ret=%d, op_errno=%d", idx,
-             frame, op_ret, op_errno);
+    ec_trace("CBK", fop, "idx=%d, frame=%p, op_ret=%d, op_errno=%d", idx, frame,
+             op_ret, op_errno);
 
     cbk = ec_cbk_data_allocate(frame, this, fop, GF_FOP_INODELK, idx, op_ret,
                                op_errno);
-    if (cbk != NULL)
-    {
-        uintptr_t mask;
-
-        if (xdata != NULL)
-        {
+    if (cbk != NULL) {
+        if (xdata != NULL) {
             cbk->xdata = dict_ref(xdata);
-            if (cbk->xdata == NULL)
-            {
-                gf_log(this->name, GF_LOG_ERROR, "Failed to reference a "
-                                                 "dictionary.");
+            if (cbk->xdata == NULL) {
+                gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_DICT_REF_FAIL,
+                       "Failed to reference a "
+                       "dictionary.");
 
                 goto out;
             }
         }
 
-        mask = ec_lock_handler(fop, cbk, NULL);
-        if (mask != 0)
-        {
-            ec_t * ec = fop->xl->private;
-            struct gf_flock flock;
-
-            flock.l_type = F_UNLCK;
-            flock.l_whence = fop->flock.l_whence;
-            flock.l_start = fop->flock.l_start * ec->fragments;
-            flock.l_len = fop->flock.l_len * ec->fragments;
-            flock.l_pid = 0;
-            flock.l_owner.len = 0;
-
-            ec_inodelk(fop->req_frame, fop->xl, mask, 1, ec_lock_unlocked,
-                       NULL, fop->str[0], &fop->loc[0], F_SETLK, &flock,
-                       fop->xdata);
-        }
+        ec_combine(cbk, NULL);
     }
 
 out:
-    if (fop != NULL)
-    {
+    if (fop != NULL) {
         ec_complete(fop);
     }
 
     return 0;
 }
 
-void ec_wind_inodelk(ec_t * ec, ec_fop_data_t * fop, int32_t idx)
+void
+ec_wind_inodelk(ec_t *ec, ec_fop_data_t *fop, int32_t idx)
 {
     ec_trace("WIND", fop, "idx=%d", idx);
 
@@ -624,24 +562,22 @@ void ec_wind_inodelk(ec_t * ec, ec_fop_data_t * fop, int32_t idx)
                       fop->xdata);
 }
 
-int32_t ec_manager_inodelk(ec_fop_data_t * fop, int32_t state)
+int32_t
+ec_manager_inodelk(ec_fop_data_t *fop, int32_t state)
 {
-    ec_cbk_data_t * cbk;
+    ec_cbk_data_t *cbk;
 
-    switch (state)
-    {
+    switch (state) {
         case EC_STATE_INIT:
-            fop->flock.l_len += ec_adjust_offset(fop->xl->private,
-                                                 &fop->flock.l_start, 1);
-            fop->flock.l_len = ec_adjust_size(fop->xl->private,
-                                              fop->flock.l_len, 1);
-            if ((fop->int32 == F_SETLKW) && (fop->flock.l_type != F_UNLCK))
-            {
+            fop->flock.l_len += ec_adjust_offset_down(
+                fop->xl->private, &fop->flock.l_start, _gf_true);
+            ec_adjust_offset_up(fop->xl->private, &fop->flock.l_len, _gf_true);
+            if ((fop->int32 == F_SETLKW) && (fop->flock.l_type != F_UNLCK)) {
                 fop->uint32 = EC_LOCK_MODE_ALL;
                 fop->int32 = F_SETLK;
             }
 
-        /* Fall through */
+            /* Fall through */
 
         case EC_STATE_DISPATCH:
             ec_dispatch_all(fop);
@@ -649,18 +585,14 @@ int32_t ec_manager_inodelk(ec_fop_data_t * fop, int32_t state)
             return EC_STATE_PREPARE_ANSWER;
 
         case EC_STATE_PREPARE_ANSWER:
-            cbk = fop->answer;
-            if ((cbk == NULL) ||
-                ((cbk->op_ret < 0) && (cbk->op_errno == EAGAIN)))
-            {
+        case -EC_STATE_PREPARE_ANSWER:
+            if (fop->flock.l_type != F_UNLCK) {
                 uintptr_t mask;
 
-                fop->error = ec_lock_check(fop, &cbk, &mask);
-                if (fop->error != 0)
-                {
-                    if (mask != 0)
-                    {
-                        ec_t * ec = fop->xl->private;
+                ec_fop_set_error(fop, ec_lock_check(fop, &mask));
+                if (fop->error != 0) {
+                    if (mask != 0) {
+                        ec_t *ec = fop->xl->private;
                         struct gf_flock flock;
 
                         flock.l_type = F_UNLCK;
@@ -670,33 +602,31 @@ int32_t ec_manager_inodelk(ec_fop_data_t * fop, int32_t state)
                         flock.l_pid = 0;
                         flock.l_owner.len = 0;
 
-                        if (fop->id == GF_FOP_INODELK)
-                        {
-                            ec_inodelk(fop->req_frame, fop->xl, mask, 1,
+                        if (fop->id == GF_FOP_INODELK) {
+                            ec_inodelk(fop->frame, fop->xl,
+                                       &fop->frame->root->lk_owner, mask, 1,
                                        ec_lock_unlocked, NULL, fop->str[0],
                                        &fop->loc[0], F_SETLK, &flock,
                                        fop->xdata);
-                        }
-                        else
-                        {
-                            ec_finodelk(fop->req_frame, fop->xl, mask, 1,
+                        } else {
+                            ec_finodelk(fop->frame, fop->xl,
+                                        &fop->frame->root->lk_owner, mask, 1,
                                         ec_lock_unlocked, NULL, fop->str[0],
                                         fop->fd, F_SETLK, &flock, fop->xdata);
                         }
                     }
-                    if (fop->error > 0)
-                    {
-                        return EC_STATE_REPORT;
-                    }
+                    if (fop->error < 0) {
+                        fop->error = 0;
 
-                    fop->error = 0;
+                        fop->int32 = F_SETLKW;
 
-                    fop->int32 = F_SETLKW;
+                        ec_dispatch_inc(fop);
 
-                    ec_dispatch_inc(fop);
-
-                    return EC_STATE_PREPARE_ANSWER;
+                        return EC_STATE_PREPARE_ANSWER;
+                    }
                 }
+            } else {
+                ec_fop_prepare_answer(fop, _gf_true);
             }
 
             return EC_STATE_REPORT;
@@ -706,128 +636,108 @@ int32_t ec_manager_inodelk(ec_fop_data_t * fop, int32_t state)
 
             GF_ASSERT(cbk != NULL);
 
-            if (fop->id == GF_FOP_INODELK)
-            {
-                if (fop->cbks.inodelk != NULL)
-                {
-                    fop->cbks.inodelk(fop->req_frame, fop, fop->xl,
-                                      cbk->op_ret, cbk->op_errno, cbk->xdata);
+            if (fop->id == GF_FOP_INODELK) {
+                if (fop->cbks.inodelk != NULL) {
+                    fop->cbks.inodelk(fop->req_frame, fop, fop->xl, cbk->op_ret,
+                                      cbk->op_errno, cbk->xdata);
                 }
-            }
-            else
-            {
-                if (fop->cbks.finodelk != NULL)
-                {
+            } else {
+                if (fop->cbks.finodelk != NULL) {
                     fop->cbks.finodelk(fop->req_frame, fop, fop->xl,
                                        cbk->op_ret, cbk->op_errno, cbk->xdata);
                 }
             }
 
-            ec_wait_winds(fop);
-
             return EC_STATE_END;
 
         case -EC_STATE_INIT:
         case -EC_STATE_DISPATCH:
-        case -EC_STATE_PREPARE_ANSWER:
         case -EC_STATE_REPORT:
             GF_ASSERT(fop->error != 0);
 
-            if (fop->id == GF_FOP_INODELK)
-            {
-                if (fop->cbks.inodelk != NULL)
-                {
+            if (fop->id == GF_FOP_INODELK) {
+                if (fop->cbks.inodelk != NULL) {
                     fop->cbks.inodelk(fop->req_frame, fop, fop->xl, -1,
                                       fop->error, NULL);
                 }
-            }
-            else
-            {
-                if (fop->cbks.finodelk != NULL)
-                {
+            } else {
+                if (fop->cbks.finodelk != NULL) {
                     fop->cbks.finodelk(fop->req_frame, fop, fop->xl, -1,
                                        fop->error, NULL);
                 }
             }
 
-            ec_wait_winds(fop);
-
             return EC_STATE_END;
 
         default:
-            gf_log(fop->xl->name, GF_LOG_ERROR, "Unhandled state %d for %s",
-                   state, ec_fop_name(fop->id));
+            gf_msg(fop->xl->name, GF_LOG_ERROR, EINVAL, EC_MSG_UNHANDLED_STATE,
+                   "Unhandled state %d for %s", state, ec_fop_name(fop->id));
 
             return EC_STATE_END;
     }
 }
 
-void ec_inodelk(call_frame_t * frame, xlator_t * this, uintptr_t target,
-                int32_t minimum, fop_inodelk_cbk_t func, void * data,
-                const char * volume, loc_t * loc, int32_t cmd,
-                struct gf_flock * flock, dict_t * xdata)
+void
+ec_inodelk(call_frame_t *frame, xlator_t *this, gf_lkowner_t *owner,
+           uintptr_t target, uint32_t fop_flags, fop_inodelk_cbk_t func,
+           void *data, const char *volume, loc_t *loc, int32_t cmd,
+           struct gf_flock *flock, dict_t *xdata)
 {
-    ec_cbk_t callback = { .inodelk = func };
-    ec_fop_data_t * fop = NULL;
-    int32_t error = EIO;
+    ec_cbk_t callback = {.inodelk = func};
+    ec_fop_data_t *fop = NULL;
+    int32_t error = ENOMEM;
 
-    gf_log("ec", GF_LOG_TRACE, "EC(INODELK) %p", frame);
+    gf_msg_trace("ec", 0, "EC(INODELK) %p", frame);
 
     VALIDATE_OR_GOTO(this, out);
     GF_VALIDATE_OR_GOTO(this->name, frame, out);
     GF_VALIDATE_OR_GOTO(this->name, this->private, out);
 
-    fop = ec_fop_data_allocate(frame, this, GF_FOP_INODELK,
-                               EC_FLAG_UPDATE_LOC_INODE, target, minimum,
-                               ec_wind_inodelk, ec_manager_inodelk, callback,
-                               data);
-    if (fop == NULL)
-    {
+    fop = ec_fop_data_allocate(frame, this, GF_FOP_INODELK, 0, target,
+                               fop_flags, ec_wind_inodelk, ec_manager_inodelk,
+                               callback, data);
+    if (fop == NULL) {
         goto out;
     }
 
     fop->int32 = cmd;
+    ec_owner_copy(fop->frame, owner);
 
-    if (volume != NULL)
-    {
+    if (volume != NULL) {
         fop->str[0] = gf_strdup(volume);
-        if (fop->str[0] == NULL)
-        {
-            gf_log(this->name, GF_LOG_ERROR, "Failed to duplicate a string.");
+        if (fop->str[0] == NULL) {
+            gf_msg(this->name, GF_LOG_ERROR, ENOMEM, EC_MSG_NO_MEMORY,
+                   "Failed to duplicate a string.");
 
             goto out;
         }
     }
-    if (loc != NULL)
-    {
-        if (loc_copy(&fop->loc[0], loc) != 0)
-        {
-            gf_log(this->name, GF_LOG_ERROR, "Failed to copy a location.");
+    if (loc != NULL) {
+        if (loc_copy(&fop->loc[0], loc) != 0) {
+            gf_msg(this->name, GF_LOG_ERROR, ENOMEM, EC_MSG_LOC_COPY_FAIL,
+                   "Failed to copy a location.");
 
             goto out;
         }
     }
-    if (flock != NULL)
-    {
+    if (flock != NULL) {
         fop->flock.l_type = flock->l_type;
         fop->flock.l_whence = flock->l_whence;
         fop->flock.l_start = flock->l_start;
         fop->flock.l_len = flock->l_len;
         fop->flock.l_pid = flock->l_pid;
         fop->flock.l_owner.len = flock->l_owner.len;
-        if (flock->l_owner.len > 0)
-        {
+        if (flock->l_owner.len > 0) {
             memcpy(fop->flock.l_owner.data, flock->l_owner.data,
                    flock->l_owner.len);
         }
     }
-    if (xdata != NULL)
-    {
+    if (xdata != NULL) {
         fop->xdata = dict_ref(xdata);
-        if (fop->xdata == NULL)
-        {
-            gf_log(this->name, GF_LOG_ERROR, "Failed to reference a "
-                                             "dictionary.");
+        if (fop->xdata == NULL) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_DICT_REF_FAIL,
+                   "Failed to reference a "
+                   "dictionary.");
 
             goto out;
         }
@@ -836,23 +746,21 @@ void ec_inodelk(call_frame_t * frame, xlator_t * this, uintptr_t target,
     error = 0;
 
 out:
-    if (fop != NULL)
-    {
+    if (fop != NULL) {
         ec_manager(fop, error);
-    }
-    else
-    {
-        func(frame, NULL, this, -1, EIO, NULL);
+    } else {
+        func(frame, NULL, this, -1, error, NULL);
     }
 }
 
 /* FOP: finodelk */
 
-int32_t ec_finodelk_cbk(call_frame_t * frame, void * cookie, xlator_t * this,
-                        int32_t op_ret, int32_t op_errno, dict_t * xdata)
+int32_t
+ec_finodelk_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
-    ec_fop_data_t * fop = NULL;
-    ec_cbk_data_t * cbk = NULL;
+    ec_fop_data_t *fop = NULL;
+    ec_cbk_data_t *cbk = NULL;
     int32_t idx = (int32_t)(uintptr_t)cookie;
 
     VALIDATE_OR_GOTO(this, out);
@@ -862,56 +770,36 @@ int32_t ec_finodelk_cbk(call_frame_t * frame, void * cookie, xlator_t * this,
 
     fop = frame->local;
 
-    ec_trace("CBK", fop, "idx=%d, frame=%p, op_ret=%d, op_errno=%d", idx,
-             frame, op_ret, op_errno);
+    ec_trace("CBK", fop, "idx=%d, frame=%p, op_ret=%d, op_errno=%d", idx, frame,
+             op_ret, op_errno);
 
     cbk = ec_cbk_data_allocate(frame, this, fop, GF_FOP_FINODELK, idx, op_ret,
                                op_errno);
-    if (cbk != NULL)
-    {
-        uintptr_t mask;
-
-        if (xdata != NULL)
-        {
+    if (cbk != NULL) {
+        if (xdata != NULL) {
             cbk->xdata = dict_ref(xdata);
-            if (cbk->xdata == NULL)
-            {
-                gf_log(this->name, GF_LOG_ERROR, "Failed to reference a "
-                                                 "dictionary.");
+            if (cbk->xdata == NULL) {
+                gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_DICT_REF_FAIL,
+                       "Failed to reference a "
+                       "dictionary.");
 
                 goto out;
             }
         }
 
-        mask = ec_lock_handler(fop, cbk, NULL);
-        if (mask != 0)
-        {
-            ec_t * ec = fop->xl->private;
-            struct gf_flock flock;
-
-            flock.l_type = F_UNLCK;
-            flock.l_whence = fop->flock.l_whence;
-            flock.l_start = fop->flock.l_start * ec->fragments;
-            flock.l_len = fop->flock.l_len * ec->fragments;
-            flock.l_pid = 0;
-            flock.l_owner.len = 0;
-
-            ec_finodelk(fop->req_frame, fop->xl, mask, 1, ec_lock_unlocked,
-                        NULL, fop->str[0], fop->fd, F_SETLK, &flock,
-                        fop->xdata);
-        }
+        ec_combine(cbk, NULL);
     }
 
 out:
-    if (fop != NULL)
-    {
+    if (fop != NULL) {
         ec_complete(fop);
     }
 
     return 0;
 }
 
-void ec_wind_finodelk(ec_t * ec, ec_fop_data_t * fop, int32_t idx)
+void
+ec_wind_finodelk(ec_t *ec, ec_fop_data_t *fop, int32_t idx)
 {
     ec_trace("WIND", fop, "idx=%d", idx);
 
@@ -921,76 +809,71 @@ void ec_wind_finodelk(ec_t * ec, ec_fop_data_t * fop, int32_t idx)
                       fop->xdata);
 }
 
-void ec_finodelk(call_frame_t * frame, xlator_t * this, uintptr_t target,
-                 int32_t minimum, fop_finodelk_cbk_t func, void * data,
-                 const char * volume, fd_t * fd, int32_t cmd,
-                 struct gf_flock * flock, dict_t * xdata)
+void
+ec_finodelk(call_frame_t *frame, xlator_t *this, gf_lkowner_t *owner,
+            uintptr_t target, uint32_t fop_flags, fop_finodelk_cbk_t func,
+            void *data, const char *volume, fd_t *fd, int32_t cmd,
+            struct gf_flock *flock, dict_t *xdata)
 {
-    ec_cbk_t callback = { .finodelk = func };
-    ec_fop_data_t * fop = NULL;
-    int32_t error = EIO;
+    ec_cbk_t callback = {.finodelk = func};
+    ec_fop_data_t *fop = NULL;
+    int32_t error = ENOMEM;
 
-    gf_log("ec", GF_LOG_TRACE, "EC(FINODELK) %p", frame);
+    gf_msg_trace("ec", 0, "EC(FINODELK) %p", frame);
 
     VALIDATE_OR_GOTO(this, out);
     GF_VALIDATE_OR_GOTO(this->name, frame, out);
     GF_VALIDATE_OR_GOTO(this->name, this->private, out);
 
-    fop = ec_fop_data_allocate(frame, this, GF_FOP_FINODELK,
-                               EC_FLAG_UPDATE_FD_INODE, target, minimum,
-                               ec_wind_finodelk, ec_manager_inodelk, callback,
-                               data);
-    if (fop == NULL)
-    {
+    fop = ec_fop_data_allocate(frame, this, GF_FOP_FINODELK, 0, target,
+                               fop_flags, ec_wind_finodelk, ec_manager_inodelk,
+                               callback, data);
+    if (fop == NULL) {
         goto out;
     }
 
     fop->use_fd = 1;
 
     fop->int32 = cmd;
+    ec_owner_copy(fop->frame, owner);
 
-    if (volume != NULL)
-    {
+    if (volume != NULL) {
         fop->str[0] = gf_strdup(volume);
-        if (fop->str[0] == NULL)
-        {
-            gf_log(this->name, GF_LOG_ERROR, "Failed to duplicate a string.");
+        if (fop->str[0] == NULL) {
+            gf_msg(this->name, GF_LOG_ERROR, ENOMEM, EC_MSG_NO_MEMORY,
+                   "Failed to duplicate a string.");
 
             goto out;
         }
     }
-    if (fd != NULL)
-    {
+    if (fd != NULL) {
         fop->fd = fd_ref(fd);
-        if (fop->fd == NULL)
-        {
-            gf_log(this->name, GF_LOG_ERROR, "Failed to reference a "
-                                             "file descriptor.");
+        if (fop->fd == NULL) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_DICT_REF_FAIL,
+                   "Failed to reference a "
+                   "file descriptor.");
 
             goto out;
         }
     }
-    if (flock != NULL)
-    {
+    if (flock != NULL) {
         fop->flock.l_type = flock->l_type;
         fop->flock.l_whence = flock->l_whence;
         fop->flock.l_start = flock->l_start;
         fop->flock.l_len = flock->l_len;
         fop->flock.l_pid = flock->l_pid;
         fop->flock.l_owner.len = flock->l_owner.len;
-        if (flock->l_owner.len > 0)
-        {
+        if (flock->l_owner.len > 0) {
             memcpy(fop->flock.l_owner.data, flock->l_owner.data,
                    flock->l_owner.len);
         }
     }
-    if (xdata != NULL)
-    {
+    if (xdata != NULL) {
         fop->xdata = dict_ref(xdata);
-        if (fop->xdata == NULL)
-        {
-            gf_log(this->name, GF_LOG_ERROR, "Failed to reference a "
-                                             "dictionary.");
+        if (fop->xdata == NULL) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_DICT_REF_FAIL,
+                   "Failed to reference a "
+                   "dictionary.");
 
             goto out;
         }
@@ -999,25 +882,22 @@ void ec_finodelk(call_frame_t * frame, xlator_t * this, uintptr_t target,
     error = 0;
 
 out:
-    if (fop != NULL)
-    {
+    if (fop != NULL) {
         ec_manager(fop, error);
-    }
-    else
-    {
-        func(frame, NULL, this, -1, EIO, NULL);
+    } else {
+        func(frame, NULL, this, -1, error, NULL);
     }
 }
 
 /* FOP: lk */
 
-int32_t ec_combine_lk(ec_fop_data_t * fop, ec_cbk_data_t * dst,
-                      ec_cbk_data_t * src)
+int32_t
+ec_combine_lk(ec_fop_data_t *fop, ec_cbk_data_t *dst, ec_cbk_data_t *src)
 {
-    if (!ec_flock_compare(&dst->flock, &src->flock))
-    {
-        gf_log(fop->xl->name, GF_LOG_NOTICE, "Mismatching lock in "
-                                             "answers of 'GF_FOP_LK'");
+    if (!ec_flock_compare(&dst->flock, &src->flock)) {
+        gf_msg(fop->xl->name, GF_LOG_NOTICE, 0, EC_MSG_LOCK_MISMATCH,
+               "Mismatching lock in "
+               "answers of 'GF_FOP_LK'");
 
         return 0;
     }
@@ -1025,12 +905,12 @@ int32_t ec_combine_lk(ec_fop_data_t * fop, ec_cbk_data_t * dst,
     return 1;
 }
 
-int32_t ec_lk_cbk(call_frame_t * frame, void * cookie, xlator_t * this,
-                  int32_t op_ret, int32_t op_errno, struct gf_flock * flock,
-                  dict_t * xdata)
+int32_t
+ec_lk_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int32_t op_ret,
+          int32_t op_errno, struct gf_flock *flock, dict_t *xdata)
 {
-    ec_fop_data_t * fop = NULL;
-    ec_cbk_data_t * cbk = NULL;
+    ec_fop_data_t *fop = NULL;
+    ec_cbk_data_t *cbk = NULL;
     int32_t idx = (int32_t)(uintptr_t)cookie;
 
     VALIDATE_OR_GOTO(this, out);
@@ -1040,72 +920,50 @@ int32_t ec_lk_cbk(call_frame_t * frame, void * cookie, xlator_t * this,
 
     fop = frame->local;
 
-    ec_trace("CBK", fop, "idx=%d, frame=%p, op_ret=%d, op_errno=%d", idx,
-             frame, op_ret, op_errno);
+    ec_trace("CBK", fop, "idx=%d, frame=%p, op_ret=%d, op_errno=%d", idx, frame,
+             op_ret, op_errno);
 
     cbk = ec_cbk_data_allocate(frame, this, fop, GF_FOP_LK, idx, op_ret,
                                op_errno);
-    if (cbk != NULL)
-    {
-        uintptr_t mask;
-
-        if (op_ret >= 0)
-        {
-            if (flock != NULL)
-            {
+    if (cbk != NULL) {
+        if (op_ret >= 0) {
+            if (flock != NULL) {
                 cbk->flock.l_type = flock->l_type;
                 cbk->flock.l_whence = flock->l_whence;
                 cbk->flock.l_start = flock->l_start;
                 cbk->flock.l_len = flock->l_len;
                 cbk->flock.l_pid = flock->l_pid;
                 cbk->flock.l_owner.len = flock->l_owner.len;
-                if (flock->l_owner.len > 0)
-                {
+                if (flock->l_owner.len > 0) {
                     memcpy(cbk->flock.l_owner.data, flock->l_owner.data,
                            flock->l_owner.len);
                 }
             }
         }
-        if (xdata != NULL)
-        {
+        if (xdata != NULL) {
             cbk->xdata = dict_ref(xdata);
-            if (cbk->xdata == NULL)
-            {
-                gf_log(this->name, GF_LOG_ERROR, "Failed to reference a "
-                                                 "dictionary.");
+            if (cbk->xdata == NULL) {
+                gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_DICT_REF_FAIL,
+                       "Failed to reference a "
+                       "dictionary.");
 
                 goto out;
             }
         }
 
-        mask = ec_lock_handler(fop, cbk, ec_combine_lk);
-        if (mask != 0)
-        {
-            ec_t * ec = fop->xl->private;
-            struct gf_flock flock;
-
-            flock.l_type = F_UNLCK;
-            flock.l_whence = fop->flock.l_whence;
-            flock.l_start = fop->flock.l_start * ec->fragments;
-            flock.l_len = fop->flock.l_len * ec->fragments;
-            flock.l_pid = 0;
-            flock.l_owner.len = 0;
-
-            ec_lk(fop->req_frame, fop->xl, mask, 1, ec_lock_lk_unlocked, NULL,
-                  fop->fd, F_SETLK, &flock, fop->xdata);
-        }
+        ec_combine(cbk, ec_combine_lk);
     }
 
 out:
-    if (fop != NULL)
-    {
+    if (fop != NULL) {
         ec_complete(fop);
     }
 
     return 0;
 }
 
-void ec_wind_lk(ec_t * ec, ec_fop_data_t * fop, int32_t idx)
+void
+ec_wind_lk(ec_t *ec, ec_fop_data_t *fop, int32_t idx)
 {
     ec_trace("WIND", fop, "idx=%d", idx);
 
@@ -1114,24 +972,19 @@ void ec_wind_lk(ec_t * ec, ec_fop_data_t * fop, int32_t idx)
                       fop->int32, &fop->flock, fop->xdata);
 }
 
-int32_t ec_manager_lk(ec_fop_data_t * fop, int32_t state)
+int32_t
+ec_manager_lk(ec_fop_data_t *fop, int32_t state)
 {
-    ec_cbk_data_t * cbk;
+    ec_cbk_data_t *cbk;
 
-    switch (state)
-    {
+    switch (state) {
         case EC_STATE_INIT:
-            fop->flock.l_len += ec_adjust_offset(fop->xl->private,
-                                                 &fop->flock.l_start, 1);
-            fop->flock.l_len = ec_adjust_size(fop->xl->private,
-                                              fop->flock.l_len, 1);
-            if ((fop->int32 == F_SETLKW) && (fop->flock.l_type != F_UNLCK))
-            {
+            if ((fop->int32 == F_SETLKW) && (fop->flock.l_type != F_UNLCK)) {
                 fop->uint32 = EC_LOCK_MODE_ALL;
                 fop->int32 = F_SETLK;
             }
 
-        /* Fall through */
+            /* Fall through */
 
         case EC_STATE_DISPATCH:
             ec_dispatch_all(fop);
@@ -1139,44 +992,38 @@ int32_t ec_manager_lk(ec_fop_data_t * fop, int32_t state)
             return EC_STATE_PREPARE_ANSWER;
 
         case EC_STATE_PREPARE_ANSWER:
-            cbk = fop->answer;
-            if ((cbk == NULL) ||
-                ((cbk->op_ret < 0) && (cbk->op_errno == EAGAIN)))
-            {
+        case -EC_STATE_PREPARE_ANSWER:
+            if (fop->flock.l_type != F_UNLCK) {
                 uintptr_t mask;
 
-                fop->error = ec_lock_check(fop, &cbk, &mask);
-                if (fop->error != 0)
-                {
-                    if (mask != 0)
-                    {
-                        ec_t * ec = fop->xl->private;
-                        struct gf_flock flock;
+                ec_fop_set_error(fop, ec_lock_check(fop, &mask));
+                if (fop->error != 0) {
+                    if (mask != 0) {
+                        struct gf_flock flock = {0};
 
                         flock.l_type = F_UNLCK;
                         flock.l_whence = fop->flock.l_whence;
-                        flock.l_start = fop->flock.l_start * ec->fragments;
-                        flock.l_len = fop->flock.l_len * ec->fragments;
-                        flock.l_pid = 0;
-                        flock.l_owner.len = 0;
+                        flock.l_start = fop->flock.l_start;
+                        flock.l_len = fop->flock.l_len;
+                        flock.l_pid = fop->flock.l_pid;
+                        lk_owner_copy(&flock.l_owner, &fop->flock.l_owner);
 
-                        ec_lk(fop->req_frame, fop->xl, mask, 1,
-                              ec_lock_lk_unlocked, NULL, fop->fd, F_SETLK,
-                              &flock, fop->xdata);
-                    }
-                    if (fop->error > 0)
-                    {
-                        return EC_STATE_REPORT;
+                        ec_lk(fop->frame, fop->xl, mask, 1, ec_lock_lk_unlocked,
+                              NULL, fop->fd, F_SETLK, &flock, fop->xdata);
                     }
 
-                    fop->error = 0;
+                    if (fop->error < 0) {
+                        fop->error = 0;
 
-                    fop->int32 = F_SETLKW;
+                        fop->int32 = F_SETLKW;
 
-                    ec_dispatch_inc(fop);
+                        ec_dispatch_inc(fop);
 
-                    return EC_STATE_PREPARE_ANSWER;
+                        return EC_STATE_PREPARE_ANSWER;
+                    }
                 }
+            } else {
+                ec_fop_prepare_answer(fop, _gf_true);
             }
 
             return EC_STATE_REPORT;
@@ -1186,59 +1033,50 @@ int32_t ec_manager_lk(ec_fop_data_t * fop, int32_t state)
 
             GF_ASSERT(cbk != NULL);
 
-            if (fop->cbks.lk != NULL)
-            {
+            if (fop->cbks.lk != NULL) {
                 fop->cbks.lk(fop->req_frame, fop, fop->xl, cbk->op_ret,
                              cbk->op_errno, &cbk->flock, cbk->xdata);
             }
 
-            ec_wait_winds(fop);
-
             return EC_STATE_END;
 
         case -EC_STATE_INIT:
         case -EC_STATE_DISPATCH:
-        case -EC_STATE_PREPARE_ANSWER:
         case -EC_STATE_REPORT:
             GF_ASSERT(fop->error != 0);
 
-            if (fop->cbks.lk != NULL)
-            {
-                fop->cbks.lk(fop->req_frame, fop, fop->xl, -1, fop->error,
-                             NULL, NULL);
+            if (fop->cbks.lk != NULL) {
+                fop->cbks.lk(fop->req_frame, fop, fop->xl, -1, fop->error, NULL,
+                             NULL);
             }
 
-            ec_wait_winds(fop);
-
             return EC_STATE_END;
 
         default:
-            gf_log(fop->xl->name, GF_LOG_ERROR, "Unhandled state %d for %s",
-                   state, ec_fop_name(fop->id));
+            gf_msg(fop->xl->name, GF_LOG_ERROR, EINVAL, EC_MSG_UNHANDLED_STATE,
+                   "Unhandled state %d for %s", state, ec_fop_name(fop->id));
 
             return EC_STATE_END;
     }
 }
 
-void ec_lk(call_frame_t * frame, xlator_t * this, uintptr_t target,
-           int32_t minimum, fop_lk_cbk_t func, void * data, fd_t * fd,
-           int32_t cmd, struct gf_flock * flock, dict_t * xdata)
+void
+ec_lk(call_frame_t *frame, xlator_t *this, uintptr_t target, uint32_t fop_flags,
+      fop_lk_cbk_t func, void *data, fd_t *fd, int32_t cmd,
+      struct gf_flock *flock, dict_t *xdata)
 {
-    ec_cbk_t callback = { .lk = func };
-    ec_fop_data_t * fop = NULL;
-    int32_t error = EIO;
+    ec_cbk_t callback = {.lk = func};
+    ec_fop_data_t *fop = NULL;
+    int32_t error = ENOMEM;
 
-    gf_log("ec", GF_LOG_TRACE, "EC(LK) %p", frame);
+    gf_msg_trace("ec", 0, "EC(LK) %p", frame);
 
-    VALIDATE_OR_GOTO(this, out);
     GF_VALIDATE_OR_GOTO(this->name, frame, out);
     GF_VALIDATE_OR_GOTO(this->name, this->private, out);
 
-    fop = ec_fop_data_allocate(frame, this, GF_FOP_LK, EC_FLAG_UPDATE_FD_INODE,
-                               target, minimum, ec_wind_lk, ec_manager_lk,
-                               callback, data);
-    if (fop == NULL)
-    {
+    fop = ec_fop_data_allocate(frame, this, GF_FOP_LK, 0, target, fop_flags,
+                               ec_wind_lk, ec_manager_lk, callback, data);
+    if (fop == NULL) {
         goto out;
     }
 
@@ -1246,38 +1084,34 @@ void ec_lk(call_frame_t * frame, xlator_t * this, uintptr_t target,
 
     fop->int32 = cmd;
 
-    if (fd != NULL)
-    {
+    if (fd != NULL) {
         fop->fd = fd_ref(fd);
-        if (fop->fd == NULL)
-        {
-            gf_log(this->name, GF_LOG_ERROR, "Failed to reference a "
-                                             "file descriptor.");
+        if (fop->fd == NULL) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_FILE_DESC_REF_FAIL,
+                   "Failed to reference a "
+                   "file descriptor.");
 
             goto out;
         }
     }
-    if (flock != NULL)
-    {
+    if (flock != NULL) {
         fop->flock.l_type = flock->l_type;
         fop->flock.l_whence = flock->l_whence;
         fop->flock.l_start = flock->l_start;
         fop->flock.l_len = flock->l_len;
         fop->flock.l_pid = flock->l_pid;
         fop->flock.l_owner.len = flock->l_owner.len;
-        if (flock->l_owner.len > 0)
-        {
+        if (flock->l_owner.len > 0) {
             memcpy(fop->flock.l_owner.data, flock->l_owner.data,
                    flock->l_owner.len);
         }
     }
-    if (xdata != NULL)
-    {
+    if (xdata != NULL) {
         fop->xdata = dict_ref(xdata);
-        if (fop->xdata == NULL)
-        {
-            gf_log(this->name, GF_LOG_ERROR, "Failed to reference a "
-                                             "dictionary.");
+        if (fop->xdata == NULL) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_DICT_REF_FAIL,
+                   "Failed to reference a "
+                   "dictionary.");
 
             goto out;
         }
@@ -1286,12 +1120,9 @@ void ec_lk(call_frame_t * frame, xlator_t * this, uintptr_t target,
     error = 0;
 
 out:
-    if (fop != NULL)
-    {
+    if (fop != NULL) {
         ec_manager(fop, error);
-    }
-    else
-    {
-        func(frame, NULL, this, -1, EIO, NULL, NULL);
+    } else {
+        func(frame, NULL, this, -1, error, NULL, NULL);
     }
 }
diff --git a/xlators/cluster/ec/src/ec-mem-types.h b/xlators/cluster/ec/src/ec-mem-types.h
index df65a031590..3252c4c1c58 100644
--- a/xlators/cluster/ec/src/ec-mem-types.h
+++ b/xlators/cluster/ec/src/ec-mem-types.h
@@ -11,16 +11,19 @@
 #ifndef __EC_MEM_TYPES_H__
 #define __EC_MEM_TYPES_H__
 
-#include "mem-types.h"
+#include <glusterfs/mem-types.h>
 
-enum gf_ec_mem_types_
-{
+enum gf_ec_mem_types_ {
     ec_mt_ec_t = gf_common_mt_end + 1,
     ec_mt_xlator_t,
     ec_mt_ec_inode_t,
     ec_mt_ec_fd_t,
-    ec_mt_ec_heal_t,
     ec_mt_subvol_healer_t,
+    ec_mt_ec_gf_t,
+    ec_mt_ec_code_t,
+    ec_mt_ec_code_builder_t,
+    ec_mt_ec_matrix_t,
+    ec_mt_ec_stripe_t,
     ec_mt_end
 };
 
diff --git a/xlators/cluster/ec/src/ec-messages.h b/xlators/cluster/ec/src/ec-messages.h
new file mode 100644
index 00000000000..72e98f11286
--- /dev/null
+++ b/xlators/cluster/ec/src/ec-messages.h
@@ -0,0 +1,61 @@
+/*
+  Copyright (c) 2015 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
+*/
+
+#ifndef _EC_MESSAGES_H_
+#define _EC_MESSAGES_H_
+
+#include <glusterfs/glfs-message-id.h>
+
+/* To add new message IDs, append new identifiers at the end of the list.
+ *
+ * Never remove a message ID. If it's not used anymore, you can rename it or
+ * leave it as it is, but not delete it. This is to prevent reutilization of
+ * IDs by other messages.
+ *
+ * The component name must match one of the entries defined in
+ * glfs-message-id.h.
+ */
+
+GLFS_MSGID(EC, EC_MSG_INVALID_CONFIG, EC_MSG_HEAL_FAIL,
+           EC_MSG_DICT_COMBINE_FAIL, EC_MSG_STIME_COMBINE_FAIL,
+           EC_MSG_INVALID_DICT_NUMS, EC_MSG_IATT_COMBINE_FAIL,
+           EC_MSG_INVALID_FORMAT, EC_MSG_DICT_GET_FAILED,
+           EC_MSG_UNHANDLED_STATE, EC_MSG_FILE_DESC_REF_FAIL,
+           EC_MSG_LOC_COPY_FAIL, EC_MSG_BUF_REF_FAIL, EC_MSG_DICT_REF_FAIL,
+           EC_MSG_LK_UNLOCK_FAILED, EC_MSG_UNLOCK_FAILED,
+           EC_MSG_LOC_PARENT_INODE_MISSING, EC_MSG_INVALID_LOC_NAME,
+           EC_MSG_NO_MEMORY, EC_MSG_GFID_MISMATCH, EC_MSG_UNSUPPORTED_VERSION,
+           EC_MSG_FD_CREATE_FAIL, EC_MSG_READDIRP_REQ_PREP_FAIL,
+           EC_MSG_LOOKUP_REQ_PREP_FAIL, EC_MSG_INODE_REF_FAIL,
+           EC_MSG_LOOKUP_READAHEAD_FAIL, EC_MSG_FRAME_MISMATCH,
+           EC_MSG_XLATOR_MISMATCH, EC_MSG_VECTOR_MISMATCH, EC_MSG_IATT_MISMATCH,
+           EC_MSG_FD_MISMATCH, EC_MSG_DICT_MISMATCH, EC_MSG_INDEX_DIR_GET_FAIL,
+           EC_MSG_PREOP_LOCK_FAILED, EC_MSG_CHILDS_INSUFFICIENT,
+           EC_MSG_OP_EXEC_UNAVAIL, EC_MSG_UNLOCK_DELAY_FAILED,
+           EC_MSG_SIZE_VERS_UPDATE_FAIL, EC_MSG_INVALID_REQUEST,
+           EC_MSG_INVALID_LOCK_TYPE, EC_MSG_SIZE_VERS_GET_FAIL,
+           EC_MSG_FILE_SIZE_GET_FAIL, EC_MSG_FOP_MISMATCH,
+           EC_MSG_SUBVOL_ID_DICT_SET_FAIL, EC_MSG_SUBVOL_BUILD_FAIL,
+           EC_MSG_XLATOR_INIT_FAIL, EC_MSG_NO_PARENTS, EC_MSG_TIMER_CREATE_FAIL,
+           EC_MSG_TOO_MANY_SUBVOLS, EC_MSG_DATA_UNAVAILABLE,
+           EC_MSG_INODE_REMOVE_FAIL, EC_MSG_INVALID_REDUNDANCY,
+           EC_MSG_XLATOR_PARSE_OPT_FAIL, EC_MSG_OP_FAIL_ON_SUBVOLS,
+           EC_MSG_INVALID_INODE, EC_MSG_LOCK_MISMATCH, EC_MSG_XDATA_MISMATCH,
+           EC_MSG_HEALING_INFO, EC_MSG_HEAL_SUCCESS, EC_MSG_FULL_SWEEP_START,
+           EC_MSG_FULL_SWEEP_STOP, EC_MSG_INVALID_FOP, EC_MSG_EC_UP,
+           EC_MSG_EC_DOWN, EC_MSG_SIZE_XATTR_GET_FAIL,
+           EC_MSG_VER_XATTR_GET_FAIL, EC_MSG_CONFIG_XATTR_GET_FAIL,
+           EC_MSG_CONFIG_XATTR_INVALID, EC_MSG_EXTENSION, EC_MSG_EXTENSION_NONE,
+           EC_MSG_EXTENSION_UNKNOWN, EC_MSG_EXTENSION_UNSUPPORTED,
+           EC_MSG_EXTENSION_FAILED, EC_MSG_NO_GF, EC_MSG_MATRIX_FAILED,
+           EC_MSG_DYN_CREATE_FAILED, EC_MSG_DYN_CODEGEN_FAILED,
+           EC_MSG_THREAD_CLEANUP_FAILED, EC_MSG_FD_BAD);
+
+#endif /* !_EC_MESSAGES_H_ */
diff --git a/xlators/cluster/ec/src/ec-method.c b/xlators/cluster/ec/src/ec-method.c
index faab0115cdd..55faed0b193 100644
--- a/xlators/cluster/ec/src/ec-method.c
+++ b/xlators/cluster/ec/src/ec-method.c
@@ -1,5 +1,5 @@
 /*
-  Copyright (c) 2012-2014 DataLab, s.l. <http://www.datalab.es>
+  Copyright (c) 2012-2015 DataLab, s.l. <http://www.datalab.es>
   This file is part of GlusterFS.
 
   This file is licensed to you under your choice of the GNU Lesser
@@ -11,149 +11,423 @@
 #include <string.h>
 #include <inttypes.h>
 
-#include "ec-gf.h"
+#include "ec-types.h"
+#include "ec-mem-types.h"
+#include "ec-galois.h"
+#include "ec-code.h"
 #include "ec-method.h"
+#include "ec-helpers.h"
 
-static uint32_t GfPow[EC_GF_SIZE << 1];
-static uint32_t GfLog[EC_GF_SIZE << 1];
+static void
+ec_method_matrix_normal(ec_gf_t *gf, uint32_t *matrix, uint32_t columns,
+                        uint32_t *values, uint32_t count)
+{
+    uint32_t i, j, v, tmp;
+
+    columns--;
+    for (i = 0; i < count; i++) {
+        v = *values++;
+        *matrix++ = tmp = ec_gf_exp(gf, v, columns);
+        for (j = 0; j < columns; j++) {
+            *matrix++ = tmp = ec_gf_div(gf, tmp, v);
+        }
+    }
+}
+
+static void
+ec_method_matrix_inverse(ec_gf_t *gf, uint32_t *matrix, uint32_t *values,
+                         uint32_t count)
+{
+    uint32_t a[count];
+    uint32_t i, j, p, last, tmp;
+
+    last = count - 1;
+    for (i = 0; i < last; i++) {
+        a[i] = 1;
+    }
+    a[i] = values[0];
+    for (i = last; i > 0; i--) {
+        for (j = i - 1; j < last; j++) {
+            a[j] = a[j + 1] ^ ec_gf_mul(gf, values[i], a[j]);
+        }
+        a[j] = ec_gf_mul(gf, values[i], a[j]);
+    }
+    for (i = 0; i < count; i++) {
+        p = a[0];
+        matrix += count;
+        *matrix = tmp = p ^ values[i];
+        for (j = 1; j < last; j++) {
+            matrix += count;
+            *matrix = tmp = a[j] ^ ec_gf_mul(gf, values[i], tmp);
+            p = tmp ^ ec_gf_mul(gf, values[i], p);
+        }
+        for (j = 0; j < last; j++) {
+            *matrix = ec_gf_div(gf, *matrix, p);
+            matrix -= count;
+        }
+        *matrix = ec_gf_div(gf, 1, p);
+        matrix++;
+    }
+}
 
-void ec_method_initialize(void)
+static void
+ec_method_matrix_init(ec_matrix_list_t *list, ec_matrix_t *matrix,
+                      uintptr_t mask, uint32_t *rows, gf_boolean_t inverse)
 {
     uint32_t i;
 
-    GfPow[0] = 1;
-    GfLog[0] = EC_GF_SIZE;
-    for (i = 1; i < EC_GF_SIZE; i++)
-    {
-        GfPow[i] = GfPow[i - 1] << 1;
-        if (GfPow[i] >= EC_GF_SIZE)
-        {
-            GfPow[i] ^= EC_GF_MOD;
+    matrix->refs = 1;
+    matrix->mask = mask;
+    matrix->code = list->code;
+    matrix->columns = list->columns;
+    INIT_LIST_HEAD(&matrix->lru);
+
+    if (inverse) {
+        matrix->rows = list->columns;
+        ec_method_matrix_inverse(matrix->code->gf, matrix->values, rows,
+                                 matrix->rows);
+        for (i = 0; i < matrix->rows; i++) {
+            matrix->row_data[i].values = matrix->values + i * matrix->columns;
+            matrix->row_data[i].func.interleaved = ec_code_build_interleaved(
+                matrix->code, EC_METHOD_WORD_SIZE, matrix->row_data[i].values,
+                matrix->columns);
+        }
+    } else {
+        matrix->rows = list->rows;
+        ec_method_matrix_normal(matrix->code->gf, matrix->values,
+                                matrix->columns, rows, matrix->rows);
+        for (i = 0; i < matrix->rows; i++) {
+            matrix->row_data[i].values = matrix->values + i * matrix->columns;
+            matrix->row_data[i].func.linear = ec_code_build_linear(
+                matrix->code, EC_METHOD_WORD_SIZE, matrix->row_data[i].values,
+                matrix->columns);
         }
-        GfPow[i + EC_GF_SIZE - 1] = GfPow[i];
-        GfLog[GfPow[i] + EC_GF_SIZE - 1] = GfLog[GfPow[i]] = i;
     }
 }
 
-static uint32_t ec_method_mul(uint32_t a, uint32_t b)
+static void
+ec_method_matrix_release(ec_matrix_t *matrix)
 {
-    if (a && b)
-    {
-        return GfPow[GfLog[a] + GfLog[b]];
+    uint32_t i;
+
+    for (i = 0; i < matrix->rows; i++) {
+        if (matrix->row_data[i].func.linear != NULL) {
+            ec_code_release(matrix->code, &matrix->row_data[i].func);
+            matrix->row_data[i].func.linear = NULL;
+        }
     }
-    return 0;
 }
 
-static uint32_t ec_method_div(uint32_t a, uint32_t b)
+static void
+ec_method_matrix_destroy(ec_matrix_list_t *list, ec_matrix_t *matrix)
+{
+    list_del_init(&matrix->lru);
+
+    ec_method_matrix_release(matrix);
+
+    mem_put(matrix);
+
+    list->count--;
+}
+
+static void
+ec_method_matrix_unref(ec_matrix_list_t *list, ec_matrix_t *matrix)
 {
-    if (b)
-    {
-        if (a)
-        {
-            return GfPow[EC_GF_SIZE - 1 + GfLog[a] - GfLog[b]];
+    if (--matrix->refs == 0) {
+        list_add_tail(&matrix->lru, &list->lru);
+        if (list->count > list->max) {
+            matrix = list_first_entry(&list->lru, ec_matrix_t, lru);
+            ec_method_matrix_destroy(list, matrix);
         }
-        return 0;
     }
-    return EC_GF_SIZE;
 }
 
-size_t ec_method_encode(size_t size, uint32_t columns, uint32_t row,
-                        uint8_t * in, uint8_t * out)
+static ec_matrix_t *
+ec_method_matrix_lookup(ec_matrix_list_t *list, uintptr_t mask, uint32_t *pos)
 {
-    uint32_t i, j;
+    ec_matrix_t *matrix;
+    uint32_t i, j, k;
 
-    size /= EC_METHOD_CHUNK_SIZE * columns;
-    row++;
-    for (j = 0; j < size; j++)
-    {
-        ec_gf_muladd[0](out, in, EC_METHOD_WIDTH);
-        in += EC_METHOD_CHUNK_SIZE;
-        for (i = 1; i < columns; i++)
-        {
-            ec_gf_muladd[row](out, in, EC_METHOD_WIDTH);
-            in += EC_METHOD_CHUNK_SIZE;
+    i = 0;
+    j = list->count;
+    while (i < j) {
+        k = (i + j) >> 1;
+        matrix = list->objects[k];
+        if (matrix->mask == mask) {
+            *pos = k;
+            return matrix;
+        }
+        if (matrix->mask < mask) {
+            i = k + 1;
+        } else {
+            j = k;
         }
-        out += EC_METHOD_CHUNK_SIZE;
     }
+    *pos = i;
 
-    return size * EC_METHOD_CHUNK_SIZE;
+    return NULL;
 }
 
-size_t ec_method_decode(size_t size, uint32_t columns, uint32_t * rows,
-                        uint8_t ** in, uint8_t * out)
+static void
+ec_method_matrix_remove(ec_matrix_list_t *list, uintptr_t mask)
 {
-    uint32_t i, j, k, off, last, value;
-    uint32_t f;
-    uint8_t inv[EC_METHOD_MAX_FRAGMENTS][EC_METHOD_MAX_FRAGMENTS + 1];
-    uint8_t mtx[EC_METHOD_MAX_FRAGMENTS][EC_METHOD_MAX_FRAGMENTS];
-    uint8_t dummy[EC_METHOD_CHUNK_SIZE];
+    uint32_t pos;
 
-    size /= EC_METHOD_CHUNK_SIZE;
+    if (ec_method_matrix_lookup(list, mask, &pos) != NULL) {
+        list->count--;
+        if (pos < list->count) {
+            memmove(list->objects + pos, list->objects + pos + 1,
+                    sizeof(ec_matrix_t *) * (list->count - pos));
+        }
+    }
+}
+
+static void
+ec_method_matrix_insert(ec_matrix_list_t *list, ec_matrix_t *matrix)
+{
+    uint32_t pos;
 
-    memset(inv, 0, sizeof(inv));
-    memset(mtx, 0, sizeof(mtx));
-    memset(dummy, 0, sizeof(dummy));
-    for (i = 0; i < columns; i++)
-    {
-        inv[i][i] = 1;
-        inv[i][columns] = 1;
+    GF_ASSERT(ec_method_matrix_lookup(list, matrix->mask, &pos) == NULL);
+
+    if (pos < list->count) {
+        memmove(list->objects + pos + 1, list->objects + pos,
+                sizeof(ec_matrix_t *) * (list->count - pos));
     }
-    for (i = 0; i < columns; i++)
-    {
-        mtx[i][columns - 1] = 1;
-        for (j = columns - 1; j > 0; j--)
-        {
-            mtx[i][j - 1] = ec_method_mul(mtx[i][j], rows[i] + 1);
-        }
+    list->objects[pos] = matrix;
+    list->count++;
+}
+
+static ec_matrix_t *
+ec_method_matrix_get(ec_matrix_list_t *list, uintptr_t mask, uint32_t *rows)
+{
+    ec_matrix_t *matrix;
+    uint32_t pos;
+
+    LOCK(&list->lock);
+
+    matrix = ec_method_matrix_lookup(list, mask, &pos);
+    if (matrix != NULL) {
+        list_del_init(&matrix->lru);
+        matrix->refs++;
+
+        goto out;
     }
 
-    for (i = 0; i < columns; i++)
-    {
-        f = mtx[i][i];
-        for (j = 0; j < columns; j++)
-        {
-            mtx[i][j] = ec_method_div(mtx[i][j], f);
-            inv[i][j] = ec_method_div(inv[i][j], f);
+    if ((list->count >= list->max) && !list_empty(&list->lru)) {
+        matrix = list_first_entry(&list->lru, ec_matrix_t, lru);
+        list_del_init(&matrix->lru);
+
+        ec_method_matrix_remove(list, matrix->mask);
+
+        ec_method_matrix_release(matrix);
+    } else {
+        matrix = mem_get0(list->pool);
+        if (matrix == NULL) {
+            matrix = EC_ERR(ENOMEM);
+            goto out;
         }
-        for (j = 0; j < columns; j++)
-        {
-            if (i != j)
-            {
-                f = mtx[j][i];
-                for (k = 0; k < columns; k++)
-                {
-                    mtx[j][k] ^= ec_method_mul(mtx[i][k], f);
-                    inv[j][k] ^= ec_method_mul(inv[i][k], f);
-                }
-            }
+        matrix->values = (uint32_t *)((uintptr_t)matrix + sizeof(ec_matrix_t) +
+                                      sizeof(ec_matrix_row_t) * list->columns);
+    }
+
+    ec_method_matrix_init(list, matrix, mask, rows, _gf_true);
+
+    if (list->count < list->max) {
+        ec_method_matrix_insert(list, matrix);
+    } else {
+        matrix->mask = 0;
+    }
+
+out:
+    UNLOCK(&list->lock);
+
+    return matrix;
+}
+
+static void
+ec_method_matrix_put(ec_matrix_list_t *list, ec_matrix_t *matrix)
+{
+    LOCK(&list->lock);
+
+    ec_method_matrix_unref(list, matrix);
+
+    UNLOCK(&list->lock);
+}
+
+static int32_t
+ec_method_setup(xlator_t *xl, ec_matrix_list_t *list, const char *gen)
+{
+    ec_matrix_t *matrix;
+    uint32_t values[list->rows];
+    uint32_t i;
+    int32_t err;
+
+    matrix = GF_MALLOC(sizeof(ec_matrix_t) +
+                           sizeof(ec_matrix_row_t) * list->rows +
+                           sizeof(uint32_t) * list->columns * list->rows,
+                       ec_mt_ec_matrix_t);
+    if (matrix == NULL) {
+        err = -ENOMEM;
+        goto failed;
+    }
+    memset(matrix, 0, sizeof(ec_matrix_t));
+    matrix->values = (uint32_t *)((uintptr_t)matrix + sizeof(ec_matrix_t) +
+                                  sizeof(ec_matrix_row_t) * list->rows);
+
+    list->code = ec_code_create(list->gf, ec_code_detect(xl, gen));
+    if (EC_IS_ERR(list->code)) {
+        err = EC_GET_ERR(list->code);
+        list->code = NULL;
+        goto failed_matrix;
+    }
+
+    for (i = 0; i < list->rows; i++) {
+        values[i] = i + 1;
+    }
+    ec_method_matrix_init(list, matrix, 0, values, _gf_false);
+
+    list->encode = matrix;
+
+    return 0;
+
+failed_matrix:
+    GF_FREE(matrix);
+failed:
+    return err;
+}
+
+int32_t
+ec_method_init(xlator_t *xl, ec_matrix_list_t *list, uint32_t columns,
+               uint32_t rows, uint32_t max, const char *gen)
+{
+    list->columns = columns;
+    list->rows = rows;
+    list->max = max;
+    list->stripe = EC_METHOD_CHUNK_SIZE * list->columns;
+    INIT_LIST_HEAD(&list->lru);
+    int32_t err;
+
+    list->pool = mem_pool_new_fn(xl->ctx,
+                                 sizeof(ec_matrix_t) +
+                                     sizeof(ec_matrix_row_t) * columns +
+                                     sizeof(uint32_t) * columns * columns,
+                                 128, "ec_matrix_t");
+    if (list->pool == NULL) {
+        err = -ENOMEM;
+        goto failed;
+    }
+
+    list->objects = GF_MALLOC(sizeof(ec_matrix_t *) * max, ec_mt_ec_matrix_t);
+    if (list->objects == NULL) {
+        err = -ENOMEM;
+        goto failed_pool;
+    }
+
+    list->gf = ec_gf_prepare(EC_GF_BITS, EC_GF_MOD);
+    if (EC_IS_ERR(list->gf)) {
+        err = EC_GET_ERR(list->gf);
+        goto failed_objects;
+    }
+
+    err = ec_method_setup(xl, list, gen);
+    if (err != 0) {
+        goto failed_gf;
+    }
+
+    LOCK_INIT(&list->lock);
+
+    return 0;
+
+failed_gf:
+    ec_gf_destroy(list->gf);
+failed_objects:
+    GF_FREE(list->objects);
+failed_pool:
+    mem_pool_destroy(list->pool);
+failed:
+    list->pool = NULL;
+    list->objects = NULL;
+    list->gf = NULL;
+
+    return err;
+}
+
+void
+ec_method_fini(ec_matrix_list_t *list)
+{
+    ec_matrix_t *matrix;
+
+    if (list->encode == NULL) {
+        return;
+    }
+
+    while (!list_empty(&list->lru)) {
+        matrix = list_first_entry(&list->lru, ec_matrix_t, lru);
+        ec_method_matrix_destroy(list, matrix);
+    }
+
+    GF_ASSERT(list->count == 0);
+
+    if (list->pool) /*Init was successful*/
+        LOCK_DESTROY(&list->lock);
+
+    ec_method_matrix_release(list->encode);
+    GF_FREE(list->encode);
+
+    ec_code_destroy(list->code);
+    ec_gf_destroy(list->gf);
+    GF_FREE(list->objects);
+
+    if (list->pool)
+        mem_pool_destroy(list->pool);
+}
+
+int32_t
+ec_method_update(xlator_t *xl, ec_matrix_list_t *list, const char *gen)
+{
+    /* TODO: Allow changing code generator */
+
+    return 0;
+}
+
+void
+ec_method_encode(ec_matrix_list_t *list, uint64_t size, void *in, void **out)
+{
+    ec_matrix_t *matrix;
+    uint64_t pos;
+    uint32_t i;
+
+    matrix = list->encode;
+    for (pos = 0; pos < size; pos += list->stripe) {
+        for (i = 0; i < matrix->rows; i++) {
+            matrix->row_data[i].func.linear(
+                out[i], in, pos, matrix->row_data[i].values, list->columns);
+            out[i] += EC_METHOD_CHUNK_SIZE;
         }
     }
-    off = 0;
-    for (f = 0; f < size; f++)
-    {
-        for (i = 0; i < columns; i++)
-        {
-            last = 0;
-            j = 0;
-            do
-            {
-                while (inv[i][j] == 0)
-                {
-                    j++;
-                }
-                if (j < columns)
-                {
-                    value = ec_method_div(last, inv[i][j]);
-                    last = inv[i][j];
-                    ec_gf_muladd[value](out, in[j] + off, EC_METHOD_WIDTH);
-                    j++;
-                }
-            } while (j < columns);
-            ec_gf_muladd[last](out, dummy, EC_METHOD_WIDTH);
+}
+
+int32_t
+ec_method_decode(ec_matrix_list_t *list, uint64_t size, uintptr_t mask,
+                 uint32_t *rows, void **in, void *out)
+{
+    ec_matrix_t *matrix;
+    uint64_t pos;
+    uint32_t i;
+
+    matrix = ec_method_matrix_get(list, mask, rows);
+    if (EC_IS_ERR(matrix)) {
+        return EC_GET_ERR(matrix);
+    }
+    for (pos = 0; pos < size; pos += EC_METHOD_CHUNK_SIZE) {
+        for (i = 0; i < matrix->rows; i++) {
+            matrix->row_data[i].func.interleaved(
+                out, in, pos, matrix->row_data[i].values, list->columns);
             out += EC_METHOD_CHUNK_SIZE;
         }
-        off += EC_METHOD_CHUNK_SIZE;
     }
 
-    return size * EC_METHOD_CHUNK_SIZE * columns;
+    ec_method_matrix_put(list, matrix);
+
+    return 0;
 }
diff --git a/xlators/cluster/ec/src/ec-method.h b/xlators/cluster/ec/src/ec-method.h
index 29b46e10443..f91233b2f88 100644
--- a/xlators/cluster/ec/src/ec-method.h
+++ b/xlators/cluster/ec/src/ec-method.h
@@ -1,5 +1,5 @@
 /*
-  Copyright (c) 2012-2014 DataLab, s.l. <http://www.datalab.es>
+  Copyright (c) 2012-2015 DataLab, s.l. <http://www.datalab.es>
   This file is part of GlusterFS.
 
   This file is licensed to you under your choice of the GNU Lesser
@@ -11,22 +11,38 @@
 #ifndef __EC_METHOD_H__
 #define __EC_METHOD_H__
 
-#include "ec-gf.h"
+#include "ec-types.h"
+#include "ec-galois.h"
+
+#define EC_GF_BITS 8
+#define EC_GF_MOD 0x11D
+
+#define EC_GF_SIZE (1 << EC_GF_BITS)
 
 /* Determines the maximum size of the matrix used to encode/decode data */
 #define EC_METHOD_MAX_FRAGMENTS 16
 /* Determines the maximum number of usable elements in the Galois Field */
-#define EC_METHOD_MAX_NODES     (EC_GF_SIZE - 1)
+#define EC_METHOD_MAX_NODES (EC_GF_SIZE - 1)
 
 #define EC_METHOD_WORD_SIZE 64
 
 #define EC_METHOD_CHUNK_SIZE (EC_METHOD_WORD_SIZE * EC_GF_BITS)
-#define EC_METHOD_WIDTH (EC_METHOD_WORD_SIZE / EC_GF_WORD_SIZE)
 
-void ec_method_initialize(void);
-size_t ec_method_encode(size_t size, uint32_t columns, uint32_t row,
-                        uint8_t * in, uint8_t * out);
-size_t ec_method_decode(size_t size, uint32_t columns, uint32_t * rows,
-                        uint8_t ** in, uint8_t * out);
+int32_t
+ec_method_init(xlator_t *xl, ec_matrix_list_t *list, uint32_t columns,
+               uint32_t rows, uint32_t max, const char *gen);
+
+void
+ec_method_fini(ec_matrix_list_t *list);
+
+int32_t
+ec_method_update(xlator_t *xl, ec_matrix_list_t *list, const char *gen);
+
+void
+ec_method_encode(ec_matrix_list_t *list, uint64_t size, void *in, void **out);
+
+int32_t
+ec_method_decode(ec_matrix_list_t *list, uint64_t size, uintptr_t mask,
+                 uint32_t *rows, void **in, void *out);
 
 #endif /* __EC_METHOD_H__ */
diff --git a/xlators/cluster/ec/src/ec-types.h b/xlators/cluster/ec/src/ec-types.h
new file mode 100644
index 00000000000..de9b89bb2c9
--- /dev/null
+++ b/xlators/cluster/ec/src/ec-types.h
@@ -0,0 +1,690 @@
+/*
+  Copyright (c) 2015 DataLab, s.l. <http://www.datalab.es>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
+*/
+
+#ifndef __EC_TYPES_H__
+#define __EC_TYPES_H__
+
+#include <glusterfs/xlator.h>
+#include <glusterfs/timer.h>
+#include "libxlator.h"
+#include <glusterfs/atomic.h>
+
+#define EC_GF_MAX_REGS 16
+
+enum _ec_heal_need;
+typedef enum _ec_heal_need ec_heal_need_t;
+
+enum _ec_stripe_part;
+typedef enum _ec_stripe_part ec_stripe_part_t;
+
+enum _ec_read_policy;
+typedef enum _ec_read_policy ec_read_policy_t;
+
+struct _ec_config;
+typedef struct _ec_config ec_config_t;
+
+struct _ec_fd;
+typedef struct _ec_fd ec_fd_t;
+
+struct _ec_fragment_range;
+typedef struct _ec_fragment_range ec_fragment_range_t;
+
+struct _ec_inode;
+typedef struct _ec_inode ec_inode_t;
+
+union _ec_cbk;
+typedef union _ec_cbk ec_cbk_t;
+
+struct _ec_lock;
+typedef struct _ec_lock ec_lock_t;
+
+struct _ec_lock_link;
+typedef struct _ec_lock_link ec_lock_link_t;
+
+struct _ec_fop_data;
+typedef struct _ec_fop_data ec_fop_data_t;
+
+struct _ec_cbk_data;
+typedef struct _ec_cbk_data ec_cbk_data_t;
+
+enum _ec_gf_opcode;
+typedef enum _ec_gf_opcode ec_gf_opcode_t;
+
+struct _ec_gf_op;
+typedef struct _ec_gf_op ec_gf_op_t;
+
+struct _ec_gf_mul;
+typedef struct _ec_gf_mul ec_gf_mul_t;
+
+struct _ec_gf;
+typedef struct _ec_gf ec_gf_t;
+
+struct _ec_code_gen;
+typedef struct _ec_code_gen ec_code_gen_t;
+
+struct _ec_code;
+typedef struct _ec_code ec_code_t;
+
+struct _ec_code_arg;
+typedef struct _ec_code_arg ec_code_arg_t;
+
+struct _ec_code_op;
+typedef struct _ec_code_op ec_code_op_t;
+
+struct _ec_code_builder;
+typedef struct _ec_code_builder ec_code_builder_t;
+
+struct _ec_code_chunk;
+typedef struct _ec_code_chunk ec_code_chunk_t;
+
+struct _ec_stripe;
+typedef struct _ec_stripe ec_stripe_t;
+
+struct _ec_stripe_list;
+typedef struct _ec_stripe_list ec_stripe_list_t;
+
+struct _ec_code_space;
+typedef struct _ec_code_space ec_code_space_t;
+
+typedef void (*ec_code_func_linear_t)(void *dst, void *src, uint64_t offset,
+                                      uint32_t *values, uint32_t count);
+
+typedef void (*ec_code_func_interleaved_t)(void *dst, void **src,
+                                           uint64_t offset, uint32_t *values,
+                                           uint32_t count);
+
+union _ec_code_func;
+typedef union _ec_code_func ec_code_func_t;
+
+struct _ec_matrix_row;
+typedef struct _ec_matrix_row ec_matrix_row_t;
+
+struct _ec_matrix;
+typedef struct _ec_matrix ec_matrix_t;
+
+struct _ec_matrix_list;
+typedef struct _ec_matrix_list ec_matrix_list_t;
+
+struct _ec_heal;
+typedef struct _ec_heal ec_heal_t;
+
+struct _ec_self_heald;
+typedef struct _ec_self_heald ec_self_heald_t;
+
+struct _ec_statistics;
+typedef struct _ec_statistics ec_statistics_t;
+
+struct _ec;
+typedef struct _ec ec_t;
+
+typedef void (*ec_wind_f)(ec_t *, ec_fop_data_t *, int32_t);
+typedef int32_t (*ec_handler_f)(ec_fop_data_t *, int32_t);
+typedef void (*ec_resume_f)(ec_fop_data_t *, int32_t);
+
+enum _ec_read_policy { EC_ROUND_ROBIN, EC_GFID_HASH, EC_READ_POLICY_MAX };
+
+enum _ec_heal_need {
+    EC_HEAL_NONEED,
+    EC_HEAL_MAYBE,
+    EC_HEAL_MUST,
+    EC_HEAL_PURGE_INDEX
+};
+
+enum _ec_stripe_part { EC_STRIPE_HEAD, EC_STRIPE_TAIL };
+
+/* Enumartions to indicate FD status. */
+typedef enum { EC_FD_NOT_OPENED, EC_FD_OPENED, EC_FD_OPENING } ec_fd_status_t;
+
+struct _ec_config {
+    uint32_t version;
+    uint8_t algorithm;
+    uint8_t gf_word_size;
+    uint8_t bricks;
+    uint8_t redundancy;
+    uint32_t chunk_size;
+};
+
+struct _ec_fd {
+    loc_t loc;
+    uintptr_t open;
+    int32_t flags;
+    uint64_t bad_version;
+    ec_fd_status_t fd_status[0];
+};
+
+struct _ec_stripe {
+    struct list_head lru; /* LRU list member */
+    uint64_t frag_offset; /* Fragment offset of this stripe */
+    char data[];          /* Contents of the stripe */
+};
+
+struct _ec_stripe_list {
+    struct list_head lru;
+    uint32_t count;
+    uint32_t max;
+};
+
+struct _ec_inode {
+    ec_lock_t *inode_lock;
+    gf_boolean_t have_info;
+    gf_boolean_t have_config;
+    gf_boolean_t have_version;
+    gf_boolean_t have_size;
+    int32_t heal_count;
+    ec_config_t config;
+    uint64_t pre_version[2];
+    uint64_t post_version[2];
+    uint64_t pre_size;
+    uint64_t post_size;
+    uint64_t dirty[2];
+    struct list_head heal;
+    ec_stripe_list_t stripe_cache;
+    uint64_t bad_version;
+};
+
+typedef int32_t (*fop_heal_cbk_t)(call_frame_t *, void *, xlator_t *, int32_t,
+                                  int32_t, uintptr_t, uintptr_t, uintptr_t,
+                                  uint32_t, dict_t *);
+typedef int32_t (*fop_fheal_cbk_t)(call_frame_t *, void *, xlator_t *, int32_t,
+                                   int32_t, uintptr_t, uintptr_t, uintptr_t,
+                                   uint32_t, dict_t *);
+
+union _ec_cbk {
+    fop_access_cbk_t access;
+    fop_create_cbk_t create;
+    fop_discard_cbk_t discard;
+    fop_entrylk_cbk_t entrylk;
+    fop_fentrylk_cbk_t fentrylk;
+    fop_fallocate_cbk_t fallocate;
+    fop_flush_cbk_t flush;
+    fop_fsync_cbk_t fsync;
+    fop_fsyncdir_cbk_t fsyncdir;
+    fop_getxattr_cbk_t getxattr;
+    fop_fgetxattr_cbk_t fgetxattr;
+    fop_heal_cbk_t heal;
+    fop_fheal_cbk_t fheal;
+    fop_inodelk_cbk_t inodelk;
+    fop_finodelk_cbk_t finodelk;
+    fop_link_cbk_t link;
+    fop_lk_cbk_t lk;
+    fop_lookup_cbk_t lookup;
+    fop_mkdir_cbk_t mkdir;
+    fop_mknod_cbk_t mknod;
+    fop_open_cbk_t open;
+    fop_opendir_cbk_t opendir;
+    fop_readdir_cbk_t readdir;
+    fop_readdirp_cbk_t readdirp;
+    fop_readlink_cbk_t readlink;
+    fop_readv_cbk_t readv;
+    fop_removexattr_cbk_t removexattr;
+    fop_fremovexattr_cbk_t fremovexattr;
+    fop_rename_cbk_t rename;
+    fop_rmdir_cbk_t rmdir;
+    fop_setattr_cbk_t setattr;
+    fop_fsetattr_cbk_t fsetattr;
+    fop_setxattr_cbk_t setxattr;
+    fop_fsetxattr_cbk_t fsetxattr;
+    fop_stat_cbk_t stat;
+    fop_fstat_cbk_t fstat;
+    fop_statfs_cbk_t statfs;
+    fop_symlink_cbk_t symlink;
+    fop_truncate_cbk_t truncate;
+    fop_ftruncate_cbk_t ftruncate;
+    fop_unlink_cbk_t unlink;
+    fop_writev_cbk_t writev;
+    fop_xattrop_cbk_t xattrop;
+    fop_fxattrop_cbk_t fxattrop;
+    fop_zerofill_cbk_t zerofill;
+    fop_seek_cbk_t seek;
+    fop_ipc_cbk_t ipc;
+};
+
+struct _ec_lock {
+    ec_inode_t *ctx;
+    gf_timer_t *timer;
+
+    /* List of owners of this lock. All fops added to this list are running
+     * concurrently. */
+    struct list_head owners;
+
+    /* List of fops waiting to be an owner of the lock. Fops are added to this
+     * list when the current owner has an incompatible access (conflicting lock)
+     * or the lock is not acquired yet. */
+    struct list_head waiting;
+
+    /* List of fops that will wait until the next unlock/lock cycle. This
+     * happens when the currently acquired lock is decided to be released as
+     * soon as possible. In this case, all frozen fops will be continued only
+     * after the lock is reacquired. */
+    struct list_head frozen;
+
+    uintptr_t mask;
+    uintptr_t good_mask;
+    uintptr_t healing;
+    uint32_t refs_owners;   /* Refs for fops owning the lock */
+    uint32_t refs_pending;  /* Refs assigned to fops being prepared */
+    uint32_t waiting_flags; /*Track xattrop/dirty marking*/
+    gf_boolean_t acquired;
+    gf_boolean_t contention;
+    gf_boolean_t unlock_now;
+    gf_boolean_t release;
+    gf_boolean_t query;
+    fd_t *fd;
+    loc_t loc;
+    union {
+        entrylk_type type;
+        struct gf_flock flock;
+    };
+};
+
+struct _ec_lock_link {
+    ec_lock_t *lock;
+    ec_fop_data_t *fop;
+    struct list_head owner_list;
+    struct list_head wait_list;
+    gf_boolean_t update[2];
+    gf_boolean_t dirty[2];
+    gf_boolean_t optimistic_changelog;
+    loc_t *base;
+    uint64_t size;
+    uint32_t waiting_flags;
+    off_t fl_start;
+    off_t fl_end;
+};
+
+/* This structure keeps a range of fragment offsets affected by a fop. Since
+ * real file offsets can be difficult to handle correctly because of overflows,
+ * we use the 'scaled' offset, which corresponds to the offset of the fragment
+ * seen by the bricks, which is always smaller and cannot overflow. */
+struct _ec_fragment_range {
+    uint64_t first; /* Address of the first affected fragment as seen by the
+                       bricks (offset on brick) */
+    uint64_t last;  /* Address of the first non affected fragment as seen by
+                       the bricks (offset on brick) */
+};
+
+/* EC xlator data structure to collect all the data required to perform
+ * the file operation.*/
+struct _ec_fop_data {
+    int32_t id; /* ID of the file operation */
+    int32_t refs;
+    int32_t state;
+    uint32_t minimum; /* Minimum number of successful
+                         operation required to conclude a
+                         fop as successful */
+    int32_t expected;
+    int32_t winds;
+    int32_t jobs;
+    int32_t error;
+    ec_fop_data_t *parent;
+    xlator_t *xl;                  /* points to EC xlator */
+    call_frame_t *req_frame;       /* frame of the calling xlator */
+    call_frame_t *frame;           /* frame used by this fop */
+    struct list_head cbk_list;     /* sorted list of groups of answers */
+    struct list_head answer_list;  /* list of answers */
+    struct list_head pending_list; /* member of ec_t.pending_fops */
+    ec_cbk_data_t *answer;         /* accepted answer */
+    int32_t lock_count;
+    int32_t locked;
+    gf_lock_t lock;
+    ec_lock_link_t locks[2];
+    int32_t first_lock;
+
+    uint32_t fop_flags; /* Flags passed by the caller. */
+    uint32_t flags;     /* Internal flags. */
+    uint32_t first;
+    uintptr_t mask;
+    uintptr_t healing; /*Dispatch is done but call is successful only
+                         if fop->minimum number of subvolumes succeed
+                         which are not healing*/
+    uintptr_t remaining;
+    uintptr_t received; /* Mask of responses */
+    uintptr_t good;
+
+    uid_t uid;
+    gid_t gid;
+
+    ec_wind_f wind;       /* Function to wind to */
+    ec_handler_f handler; /* FOP manager function */
+    ec_resume_f resume;
+    ec_cbk_t cbks; /* Callback function for this FOP */
+    void *data;
+    ec_heal_t *heal;
+    struct list_head healer;
+
+    uint64_t user_size;
+    uint32_t head;
+
+    int32_t use_fd; /* Indicates whether this FOP uses FD or
+                       not */
+
+    dict_t *xdata;
+    dict_t *dict;
+    int32_t int32;
+    uint32_t uint32;
+    uint64_t size;
+    off_t offset;
+    mode_t mode[2];
+    entrylk_cmd entrylk_cmd;
+    entrylk_type entrylk_type;
+    gf_xattrop_flags_t xattrop_flags;
+    dev_t dev;
+    inode_t *inode;
+    fd_t *fd; /* FD of the file on which FOP is
+                 being carried upon */
+    struct iatt iatt;
+    char *str[2];
+    loc_t loc[2]; /* Holds the location details for
+                     the file */
+    struct gf_flock flock;
+    struct iovec *vector;
+    struct iobref *buffers;
+    gf_seek_what_t seek;
+    ec_fragment_range_t frag_range; /* This will hold the range of stripes
+                                        affected by the fop. */
+    char *errstr;                   /*String of fop name, path and gfid
+                                     to be used in gf_msg. */
+};
+
+struct _ec_cbk_data {
+    struct list_head list;        /* item in the sorted list of groups */
+    struct list_head answer_list; /* item in the list of answers */
+    ec_fop_data_t *fop;
+    ec_cbk_data_t *next; /* next answer in the same group */
+    uint32_t idx;
+    int32_t op_ret;
+    int32_t op_errno;
+    int32_t count;
+    uintptr_t mask;
+
+    dict_t *xdata;
+    dict_t *dict;
+    int32_t int32;
+    uintptr_t uintptr[3];
+    uint64_t size;
+    uint64_t version[2];
+    inode_t *inode;
+    fd_t *fd;
+    struct statvfs statvfs;
+    struct iatt iatt[5];
+    struct gf_flock flock;
+    struct iovec *vector;
+    struct iobref *buffers;
+    char *str;
+    gf_dirent_t entries;
+    off_t offset;
+    gf_seek_what_t what;
+};
+
+enum _ec_gf_opcode {
+    EC_GF_OP_LOAD,
+    EC_GF_OP_STORE,
+    EC_GF_OP_COPY,
+    EC_GF_OP_XOR2,
+    EC_GF_OP_XOR3,
+    EC_GF_OP_XORM,
+    EC_GF_OP_END
+};
+
+struct _ec_gf_op {
+    ec_gf_opcode_t op;
+    uint32_t arg1;
+    uint32_t arg2;
+    uint32_t arg3;
+};
+
+struct _ec_gf_mul {
+    uint32_t regs;
+    uint32_t map[EC_GF_MAX_REGS];
+    ec_gf_op_t *ops;
+};
+
+struct _ec_gf {
+    uint32_t bits;
+    uint32_t size;
+    uint32_t mod;
+    uint32_t min_ops;
+    uint32_t max_ops;
+    uint32_t avg_ops;
+    uint32_t *log;
+    uint32_t *pow;
+    ec_gf_mul_t **table;
+};
+
+struct _ec_code_gen {
+    char *name;
+    char **flags;
+    uint32_t width;
+
+    void (*prolog)(ec_code_builder_t *builder);
+    void (*epilog)(ec_code_builder_t *builder);
+    void (*load)(ec_code_builder_t *builder, uint32_t reg, uint32_t offset,
+                 uint32_t bit);
+    void (*store)(ec_code_builder_t *builder, uint32_t reg, uint32_t bit);
+    void (*copy)(ec_code_builder_t *builder, uint32_t dst, uint32_t src);
+    void (*xor2)(ec_code_builder_t *builder, uint32_t dst, uint32_t src);
+    void (*xor3)(ec_code_builder_t *builder, uint32_t dst, uint32_t src1,
+                 uint32_t src2);
+    void (*xorm)(ec_code_builder_t *builder, uint32_t dst, uint32_t offset,
+                 uint32_t bit);
+};
+
+struct _ec_code {
+    gf_lock_t lock;
+    struct list_head spaces;
+    ec_gf_t *gf;
+    ec_code_gen_t *gen;
+};
+
+struct _ec_code_arg {
+    uint32_t value;
+};
+
+struct _ec_code_op {
+    ec_gf_opcode_t op;
+    ec_code_arg_t arg1;
+    ec_code_arg_t arg2;
+    ec_code_arg_t arg3;
+};
+
+struct _ec_code_builder {
+    ec_code_t *code;
+    uint64_t address;
+    uint8_t *data;
+    uint32_t size;
+    int32_t error;
+    uint32_t regs;
+    uint32_t bits;
+    uint32_t width;
+    uint32_t count;
+    uint32_t base;
+    uint32_t map[EC_GF_MAX_REGS];
+    gf_boolean_t linear;
+    uint64_t loop;
+    ec_code_op_t ops[0];
+};
+
+struct _ec_code_chunk {
+    struct list_head list;
+    size_t size;
+    ec_code_space_t *space;
+};
+
+struct _ec_code_space {
+    struct list_head list;
+    struct list_head chunks;
+    ec_code_t *code;
+    void *exec;
+    size_t size;
+};
+
+union _ec_code_func {
+    ec_code_func_linear_t linear;
+    ec_code_func_interleaved_t interleaved;
+};
+
+struct _ec_matrix_row {
+    ec_code_func_t func;
+    uint32_t *values;
+};
+
+struct _ec_matrix {
+    struct list_head lru;
+    uint32_t refs;
+    uint32_t columns;
+    uint32_t rows;
+    uintptr_t mask;
+    ec_code_t *code;
+    uint32_t *values;
+    ec_matrix_row_t row_data[0];
+};
+
+struct _ec_matrix_list {
+    struct list_head lru;
+    gf_lock_t lock;
+    uint32_t columns;
+    uint32_t rows;
+    uint32_t max;
+    uint32_t count;
+    uint32_t stripe;
+    struct mem_pool *pool;
+    ec_gf_t *gf;
+    ec_code_t *code;
+    ec_matrix_t *encode;
+    ec_matrix_t **objects;
+};
+
+struct _ec_heal {
+    struct list_head list;
+    gf_lock_t lock;
+    xlator_t *xl;
+    ec_fop_data_t *fop;
+    void *data;
+    ec_fop_data_t *lookup;
+    loc_t loc;
+    struct iatt iatt;
+    char *symlink;
+    fd_t *fd;
+    int32_t partial;
+    int32_t done;
+    int32_t error;
+    gf_boolean_t nameheal;
+    uintptr_t available;
+    uintptr_t good;
+    uintptr_t bad;
+    uintptr_t open;
+    uintptr_t fixed;
+    uint64_t offset;
+    uint64_t size;
+    uint64_t total_size;
+    uint64_t version[2];
+    uint64_t raw_size;
+};
+
+struct subvol_healer {
+    xlator_t *this;
+    int subvol;
+    gf_boolean_t running;
+    gf_boolean_t rerun;
+    pthread_mutex_t mutex;
+    pthread_cond_t cond;
+    pthread_t thread;
+};
+
+struct _ec_self_heald {
+    gf_boolean_t iamshd;
+    gf_boolean_t enabled;
+    int timeout;
+    uint32_t max_threads;
+    uint32_t wait_qlength;
+    struct subvol_healer *index_healers;
+    struct subvol_healer *full_healers;
+};
+
+struct _ec_statistics {
+    struct {
+        gf_atomic_t hits;    /* Cache hits. */
+        gf_atomic_t misses;  /* Cache misses. */
+        gf_atomic_t updates; /* Number of times an existing stripe has
+                                been updated with new content. */
+        gf_atomic_t invals;  /* Number of times an existing stripe has
+                                been invalidated because of truncates
+                                or discards. */
+        gf_atomic_t evicts;  /* Number of times that an existing entry
+                                has been evicted to make room for newer
+                                entries. */
+        gf_atomic_t allocs;  /* Number of memory allocations made to
+                                store stripes. */
+        gf_atomic_t errors;  /* Number of errors that have caused extra
+                                requests. (Basically memory allocation
+                                errors). */
+    } stripe_cache;
+    struct {
+        gf_atomic_t attempted; /*Number of heals attempted on
+                                files/directories*/
+        gf_atomic_t completed; /*Number of heals complted on files/directories*/
+    } shd;
+};
+
+struct _ec {
+    xlator_t *xl;
+    int32_t healers;
+    int32_t heal_waiters;
+    int32_t nodes; /* Total number of bricks(n) */
+    int32_t bits_for_nodes;
+    int32_t fragments;      /* Data bricks(k) */
+    int32_t redundancy;     /* Redundant bricks(m) */
+    uint32_t fragment_size; /* Size of fragment/chunk on a
+                               brick. */
+    uint32_t stripe_size;   /* (fragment_size * fragments)
+                               maximum size of user data
+                               stored in one stripe. */
+    int32_t up;             /* Represents whether EC volume is
+                               up or not. */
+    uint32_t idx;
+    uint32_t xl_up_count;     /* Number of UP bricks. */
+    uintptr_t xl_up;          /* Bit flag representing UP
+                                 bricks */
+    uint32_t xl_notify_count; /* Number of notifications. */
+    uintptr_t xl_notify;      /* Bit flag representing
+                                 notification for bricks. */
+    uintptr_t node_mask;
+    uintptr_t read_mask;         /*Stores user defined read-mask*/
+    gf_atomic_t async_fop_count; /* Number of on going asynchronous fops. */
+    xlator_t **xl_list;
+    gf_lock_t lock;
+    gf_timer_t *timer;
+    gf_boolean_t shutdown;
+    gf_boolean_t eager_lock;
+    gf_boolean_t other_eager_lock;
+    gf_boolean_t optimistic_changelog;
+    gf_boolean_t parallel_writes;
+    uint32_t stripe_cache;
+    uint32_t quorum_count;
+    uint32_t background_heals;
+    uint32_t heal_wait_qlen;
+    uint32_t self_heal_window_size; /* max size of read/writes */
+    uint32_t eager_lock_timeout;
+    uint32_t other_eager_lock_timeout;
+    struct list_head pending_fops;
+    struct list_head heal_waiting;
+    struct list_head healing;
+    struct mem_pool *fop_pool;
+    struct mem_pool *cbk_pool;
+    struct mem_pool *lock_pool;
+    ec_self_heald_t shd;
+    char vol_uuid[UUID_SIZE + 1];
+    dict_t *leaf_to_subvolid;
+    ec_read_policy_t read_policy;
+    ec_matrix_list_t matrix;
+    ec_statistics_t stats;
+};
+
+#endif /* __EC_TYPES_H__ */
diff --git a/xlators/cluster/ec/src/ec.c b/xlators/cluster/ec/src/ec.c
index fd5123d61da..7344be4968d 100644
--- a/xlators/cluster/ec/src/ec.c
+++ b/xlators/cluster/ec/src/ec.c
@@ -1,5 +1,5 @@
 /*
-  Copyright (c) 2012-2014 DataLab, s.l. <http://www.datalab.es>
+  Copyright (c) 2012-2015 DataLab, s.l. <http://www.datalab.es>
   This file is part of GlusterFS.
 
   This file is licensed to you under your choice of the GNU Lesser
@@ -8,59 +8,63 @@
   cases as published by the Free Software Foundation.
 */
 
-#include "defaults.h"
-#include "statedump.h"
-#include "compat-errno.h"
+#include <glusterfs/defaults.h>
+#include <glusterfs/statedump.h>
+#include <glusterfs/compat-errno.h>
+#include <glusterfs/upcall-utils.h>
 
+#include "ec.h"
+#include "ec-messages.h"
 #include "ec-mem-types.h"
+#include "ec-types.h"
 #include "ec-helpers.h"
 #include "ec-common.h"
 #include "ec-fops.h"
 #include "ec-method.h"
-#include "ec.h"
+#include "ec-code.h"
 #include "ec-heald.h"
+#include <glusterfs/events.h>
 
-#define EC_MAX_FRAGMENTS EC_METHOD_MAX_FRAGMENTS
-/* The maximum number of nodes is derived from the maximum allowed fragments
- * using the rule that redundancy cannot be equal or greater than the number
- * of fragments.
- */
-#define EC_MAX_NODES min(EC_MAX_FRAGMENTS * 2 - 1, EC_METHOD_MAX_NODES)
+static char *ec_read_policies[EC_READ_POLICY_MAX + 1] = {
+    [EC_ROUND_ROBIN] = "round-robin",
+    [EC_GFID_HASH] = "gfid-hash",
+    [EC_READ_POLICY_MAX] = NULL};
 
 #define EC_INTERNAL_XATTR_OR_GOTO(name, xattr, op_errno, label)                \
-        do {                                                                   \
-                if (ec_is_internal_xattr (NULL, (char *)name, NULL, NULL)) {   \
-                        op_errno = EPERM;                                      \
-                        goto label;                                            \
-                }                                                              \
-                if (name && (strlen (name) == 0) && xattr) {                   \
-                        /* Bulk [f]removexattr/[f]setxattr */                  \
-                        GF_IF_INTERNAL_XATTR_GOTO (EC_XATTR_PREFIX"*", xattr,  \
-                                                   op_errno, label);           \
-                }                                                              \
-        } while (0)
-
-int32_t ec_parse_options(xlator_t * this)
-{
-    ec_t * ec = this->private;
+    do {                                                                       \
+        if (ec_is_internal_xattr(NULL, (char *)name, NULL, NULL)) {            \
+            op_errno = EPERM;                                                  \
+            goto label;                                                        \
+        }                                                                      \
+        if (name && (strlen(name) == 0) && xattr) {                            \
+            /* Bulk [f]removexattr/[f]setxattr */                              \
+            GF_IF_INTERNAL_XATTR_GOTO(EC_XATTR_PREFIX "*", xattr, op_errno,    \
+                                      label);                                  \
+        }                                                                      \
+    } while (0)
+
+int32_t
+ec_parse_options(xlator_t *this)
+{
+    ec_t *ec = this->private;
     int32_t error = EINVAL;
     uintptr_t mask;
 
     GF_OPTION_INIT("redundancy", ec->redundancy, int32, out);
     ec->fragments = ec->nodes - ec->redundancy;
     if ((ec->redundancy < 1) || (ec->redundancy >= ec->fragments) ||
-        (ec->fragments > EC_MAX_FRAGMENTS))
-    {
-        gf_log(this->name, GF_LOG_ERROR, "Invalid redundancy (must be between "
-                                         "1 and %d)", (ec->nodes - 1) / 2);
+        (ec->fragments > EC_MAX_FRAGMENTS)) {
+        gf_msg(this->name, GF_LOG_ERROR, EINVAL, EC_MSG_INVALID_REDUNDANCY,
+               "Invalid redundancy (must be between "
+               "1 and %d)",
+               (ec->nodes - 1) / 2);
 
         goto out;
     }
 
     ec->bits_for_nodes = 1;
     mask = 2;
-    while (ec->nodes > mask)
-    {
+    while (ec->nodes > mask) {
         ec->bits_for_nodes++;
         mask <<= 1;
     }
@@ -68,9 +72,10 @@ int32_t ec_parse_options(xlator_t * this)
     ec->fragment_size = EC_METHOD_CHUNK_SIZE;
     ec->stripe_size = ec->fragment_size * ec->fragments;
 
-    gf_log("ec", GF_LOG_DEBUG, "Initialized with: nodes=%u, fragments=%u, "
-                               "stripe_size=%u, node_mask=%lX",
-           ec->nodes, ec->fragments, ec->stripe_size, ec->node_mask);
+    gf_msg_debug("ec", 0,
+                 "Initialized with: nodes=%u, fragments=%u, "
+                 "stripe_size=%u, node_mask=%" PRIxFAST32,
+                 ec->nodes, ec->fragments, ec->stripe_size, ec->node_mask);
 
     error = 0;
 
@@ -78,28 +83,28 @@ out:
     return error;
 }
 
-int32_t ec_prepare_childs(xlator_t * this)
+int32_t
+ec_prepare_childs(xlator_t *this)
 {
-    ec_t * ec = this->private;
-    xlator_list_t * child = NULL;
+    ec_t *ec = this->private;
+    xlator_list_t *child = NULL;
     int32_t count = 0;
 
-    for (child = this->children; child != NULL; child = child->next)
-    {
+    for (child = this->children; child != NULL; child = child->next) {
         count++;
     }
-    if (count > EC_MAX_NODES)
-    {
-        gf_log(this->name, GF_LOG_ERROR, "Too many subvolumes");
+    if (count > EC_MAX_NODES) {
+        gf_msg(this->name, GF_LOG_ERROR, EINVAL, EC_MSG_TOO_MANY_SUBVOLS,
+               "Too many subvolumes");
 
         return EINVAL;
     }
     ec->nodes = count;
 
     ec->xl_list = GF_CALLOC(count, sizeof(ec->xl_list[0]), ec_mt_xlator_t);
-    if (ec->xl_list == NULL)
-    {
-        gf_log(this->name, GF_LOG_ERROR, "Allocation of xlator list failed");
+    if (ec->xl_list == NULL) {
+        gf_msg(this->name, GF_LOG_ERROR, ENOMEM, EC_MSG_NO_MEMORY,
+               "Allocation of xlator list failed");
 
         return ENOMEM;
     }
@@ -107,24 +112,51 @@ int32_t ec_prepare_childs(xlator_t * this)
     ec->xl_up_count = 0;
 
     count = 0;
-    for (child = this->children; child != NULL; child = child->next)
-    {
+    for (child = this->children; child != NULL; child = child->next) {
         ec->xl_list[count++] = child->xlator;
     }
 
     return 0;
 }
 
-void __ec_destroy_private(xlator_t * this)
+/* This function transforms the subvol to subvol-id*/
+static int
+_subvol_to_subvolid(dict_t *this, char *key, data_t *value, void *data)
 {
-    ec_t * ec = this->private;
+    ec_t *ec = data;
+    xlator_t *subvol = NULL;
+    int i = 0;
+    int ret = -1;
 
-    if (ec != NULL)
-    {
+    subvol = data_to_ptr(value);
+    for (i = 0; i < ec->nodes; i++) {
+        if (ec->xl_list[i] == subvol) {
+            ret = dict_set_int32(this, key, i);
+            /* -1 stops dict_foreach and returns -1*/
+            if (ret < 0)
+                ret = -1;
+            goto out;
+        }
+    }
+out:
+    return ret;
+}
+
+int
+ec_subvol_to_subvol_id_transform(ec_t *ec, dict_t *leaf_to_subvolid)
+{
+    return dict_foreach(leaf_to_subvolid, _subvol_to_subvolid, ec);
+}
+
+void
+__ec_destroy_private(xlator_t *this)
+{
+    ec_t *ec = this->private;
+
+    if (ec != NULL) {
         LOCK(&ec->lock);
 
-        if (ec->timer != NULL)
-        {
+        if (ec->timer != NULL) {
             gf_timer_call_cancel(this->ctx, ec->timer);
             ec->timer = NULL;
         }
@@ -142,39 +174,41 @@ void __ec_destroy_private(xlator_t * this)
         sleep(2);
 
         this->private = NULL;
-        if (ec->xl_list != NULL)
-        {
+        if (ec->xl_list != NULL) {
             GF_FREE(ec->xl_list);
             ec->xl_list = NULL;
         }
 
-        if (ec->fop_pool != NULL)
-        {
+        if (ec->fop_pool != NULL) {
             mem_pool_destroy(ec->fop_pool);
         }
 
-        if (ec->cbk_pool != NULL)
-        {
+        if (ec->cbk_pool != NULL) {
             mem_pool_destroy(ec->cbk_pool);
         }
 
-        if (ec->lock_pool != NULL)
-        {
+        if (ec->lock_pool != NULL) {
             mem_pool_destroy(ec->lock_pool);
         }
 
         LOCK_DESTROY(&ec->lock);
 
+        if (ec->leaf_to_subvolid)
+            dict_unref(ec->leaf_to_subvolid);
+
+        ec_method_fini(&ec->matrix);
+
         GF_FREE(ec);
     }
 }
 
-int32_t mem_acct_init(xlator_t * this)
+int32_t
+mem_acct_init(xlator_t *this)
 {
-    if (xlator_mem_acct_init(this, ec_mt_end + 1) != 0)
-    {
-        gf_log(this->name, GF_LOG_ERROR, "Memory accounting initialization "
-                                         "failed.");
+    if (xlator_mem_acct_init(this, ec_mt_end + 1) != 0) {
+        gf_msg(this->name, GF_LOG_ERROR, ENOMEM, EC_MSG_NO_MEMORY,
+               "Memory accounting initialization "
+               "failed.");
 
         return -1;
     }
@@ -182,261 +216,583 @@ int32_t mem_acct_init(xlator_t * this)
     return 0;
 }
 
-int32_t
-reconfigure (xlator_t *this, dict_t *options)
+void
+ec_configure_background_heal_opts(ec_t *ec, int background_heals,
+                                  int heal_wait_qlen)
 {
-        ec_t *ec = this->private;
+    if (background_heals == 0) {
+        ec->heal_wait_qlen = 0;
+    } else {
+        ec->heal_wait_qlen = heal_wait_qlen;
+    }
+    ec->background_heals = background_heals;
+}
 
-        GF_OPTION_RECONF ("self-heal-daemon", ec->shd.enabled, options, bool, failed);
-        GF_OPTION_RECONF ("iam-self-heal-daemon", ec->shd.iamshd, options,
-                          bool, failed);
+int
+ec_assign_read_policy(ec_t *ec, char *read_policy)
+{
+    int read_policy_idx = -1;
 
-        return 0;
-failed:
+    read_policy_idx = gf_get_index_by_elem(ec_read_policies, read_policy);
+    if (read_policy_idx < 0 || read_policy_idx >= EC_READ_POLICY_MAX)
         return -1;
+
+    ec->read_policy = read_policy_idx;
+    return 0;
 }
 
-glusterfs_event_t
-ec_get_event_from_state (ec_t *ec)
-{
-        int               down_count = 0;
+int32_t
+reconfigure(xlator_t *this, dict_t *options)
+{
+    ec_t *ec = this->private;
+    char *read_policy = NULL;
+    char *extensions = NULL;
+    uint32_t heal_wait_qlen = 0;
+    uint32_t background_heals = 0;
+    int32_t ret = -1;
+    int32_t err;
+
+    GF_OPTION_RECONF("cpu-extensions", extensions, options, str, failed);
+
+    GF_OPTION_RECONF("self-heal-daemon", ec->shd.enabled, options, bool,
+                     failed);
+    GF_OPTION_RECONF("iam-self-heal-daemon", ec->shd.iamshd, options, bool,
+                     failed);
+    GF_OPTION_RECONF("eager-lock", ec->eager_lock, options, bool, failed);
+    GF_OPTION_RECONF("other-eager-lock", ec->other_eager_lock, options, bool,
+                     failed);
+    GF_OPTION_RECONF("eager-lock-timeout", ec->eager_lock_timeout, options,
+                     uint32, failed);
+    GF_OPTION_RECONF("other-eager-lock-timeout", ec->other_eager_lock_timeout,
+                     options, uint32, failed);
+    GF_OPTION_RECONF("background-heals", background_heals, options, uint32,
+                     failed);
+    GF_OPTION_RECONF("heal-wait-qlength", heal_wait_qlen, options, uint32,
+                     failed);
+    GF_OPTION_RECONF("self-heal-window-size", ec->self_heal_window_size,
+                     options, uint32, failed);
+    GF_OPTION_RECONF("heal-timeout", ec->shd.timeout, options, int32, failed);
+    ec_configure_background_heal_opts(ec, background_heals, heal_wait_qlen);
+    GF_OPTION_RECONF("shd-max-threads", ec->shd.max_threads, options, uint32,
+                     failed);
+    GF_OPTION_RECONF("shd-wait-qlength", ec->shd.wait_qlength, options, uint32,
+                     failed);
+
+    GF_OPTION_RECONF("read-policy", read_policy, options, str, failed);
+
+    GF_OPTION_RECONF("optimistic-change-log", ec->optimistic_changelog, options,
+                     bool, failed);
+    GF_OPTION_RECONF("parallel-writes", ec->parallel_writes, options, bool,
+                     failed);
+    GF_OPTION_RECONF("stripe-cache", ec->stripe_cache, options, uint32, failed);
+    GF_OPTION_RECONF("quorum-count", ec->quorum_count, options, uint32, failed);
+    ret = 0;
+    if (ec_assign_read_policy(ec, read_policy)) {
+        ret = -1;
+    }
 
-        if (ec->xl_up_count >= ec->fragments) {
-                /* If ec is up but some subvolumes are yet to notify, give
-                 * grace time for other subvols to notify to prevent start of
-                 * I/O which may result in self-heals */
-                if (ec->timer && ec->xl_notify_count < ec->nodes)
-                        return GF_EVENT_MAXVAL;
+    err = ec_method_update(this, &ec->matrix, extensions);
+    if (err != 0) {
+        ret = -1;
+    }
 
-                return GF_EVENT_CHILD_UP;
-        } else {
-                down_count = ec->xl_notify_count - ec->xl_up_count;
-                if (down_count > ec->redundancy)
-                        return GF_EVENT_CHILD_DOWN;
-        }
+failed:
+    return ret;
+}
+
+glusterfs_event_t
+ec_get_event_from_state(ec_t *ec)
+{
+    int down_count = 0;
+
+    if (ec->xl_up_count >= ec->fragments) {
+        /* If ec is up but some subvolumes are yet to notify, give
+         * grace time for other subvols to notify to prevent start of
+         * I/O which may result in self-heals */
+        if (ec->xl_notify_count < ec->nodes)
+            return GF_EVENT_MAXVAL;
+
+        return GF_EVENT_CHILD_UP;
+    } else {
+        down_count = ec->xl_notify_count - ec->xl_up_count;
+        if (down_count > ec->redundancy)
+            return GF_EVENT_CHILD_DOWN;
+    }
 
-        return GF_EVENT_MAXVAL;
+    return GF_EVENT_MAXVAL;
 }
 
 void
-ec_up (xlator_t *this, ec_t *ec)
+ec_up(xlator_t *this, ec_t *ec)
 {
-        if (ec->timer != NULL) {
-                gf_timer_call_cancel (this->ctx, ec->timer);
-                ec->timer = NULL;
-        }
+    char str1[32], str2[32];
+
+    if (ec->timer != NULL) {
+        gf_timer_call_cancel(this->ctx, ec->timer);
+        ec->timer = NULL;
+    }
+
+    ec->up = 1;
+    gf_msg(this->name, GF_LOG_INFO, 0, EC_MSG_EC_UP,
+           "Going UP : Child UP = %s Child Notify = %s",
+           ec_bin(str1, sizeof(str1), ec->xl_up, ec->nodes),
+           ec_bin(str2, sizeof(str2), ec->xl_notify, ec->nodes));
 
-        ec->up = 1;
-        gf_log(this->name, GF_LOG_INFO, "Going UP");
+    gf_event(EVENT_EC_MIN_BRICKS_UP, "subvol=%s", this->name);
 }
 
 void
-ec_down (xlator_t *this, ec_t *ec)
+ec_down(xlator_t *this, ec_t *ec)
 {
-        if (ec->timer != NULL) {
-                gf_timer_call_cancel(this->ctx, ec->timer);
-                ec->timer = NULL;
-        }
+    char str1[32], str2[32];
+
+    if (ec->timer != NULL) {
+        gf_timer_call_cancel(this->ctx, ec->timer);
+        ec->timer = NULL;
+    }
 
-        ec->up = 0;
-        gf_log(this->name, GF_LOG_INFO, "Going DOWN");
+    ec->up = 0;
+    gf_msg(this->name, GF_LOG_INFO, 0, EC_MSG_EC_DOWN,
+           "Going DOWN : Child UP = %s Child Notify = %s",
+           ec_bin(str1, sizeof(str1), ec->xl_up, ec->nodes),
+           ec_bin(str2, sizeof(str2), ec->xl_notify, ec->nodes));
+
+    gf_event(EVENT_EC_MIN_BRICKS_NOT_UP, "subvol=%s", this->name);
 }
 
 void
-ec_notify_cbk (void *data)
+ec_notify_cbk(void *data)
 {
-        ec_t *ec = data;
-        glusterfs_event_t event = GF_EVENT_MAXVAL;
+    ec_t *ec = data;
+    glusterfs_event_t event = GF_EVENT_MAXVAL;
+    gf_boolean_t propagate = _gf_false;
+    gf_boolean_t launch_heal = _gf_false;
 
-        LOCK(&ec->lock);
-        {
-                if (!ec->timer) {
-                        /*
-                         * Either child_up/child_down is already sent to parent
-                         * This is a spurious wake up.
-                         */
-                        goto unlock;
-                }
-
-                gf_timer_call_cancel (ec->xl->ctx, ec->timer);
-                ec->timer = NULL;
-
-                event = ec_get_event_from_state (ec);
-                /* If event is still MAXVAL then enough subvolumes didn't
-                 * notify, treat it as CHILD_DOWN. */
-                if (event == GF_EVENT_MAXVAL) {
-                        event = GF_EVENT_CHILD_DOWN;
-                        ec->xl_notify = (1ULL << ec->nodes) - 1ULL;
-                        ec->xl_notify_count = ec->nodes;
-                } else if (event == GF_EVENT_CHILD_UP) {
-                        /* Rest of the bricks are still not coming up,
-                         * notify that ec is up. Files/directories will be
-                         * healed as in when they come up. */
-                        ec_up (ec->xl, ec);
-                }
-
-                /* CHILD_DOWN should not come here as no grace period is given
-                 * for notifying CHILD_DOWN. */
-
-                default_notify (ec->xl, event, NULL);
+    LOCK(&ec->lock);
+    {
+        if (!ec->timer) {
+            /*
+             * Either child_up/child_down is already sent to parent
+             * This is a spurious wake up.
+             */
+            goto unlock;
         }
+
+        gf_timer_call_cancel(ec->xl->ctx, ec->timer);
+        ec->timer = NULL;
+
+        /* The timeout has expired, so any subvolume that has not
+         * already reported its state, will be considered to be down.
+         * We mark as if all bricks had reported. */
+        ec->xl_notify = (1ULL << ec->nodes) - 1ULL;
+        ec->xl_notify_count = ec->nodes;
+
+        /* Since we have marked all subvolumes as notified, it's
+         * guaranteed that ec_get_event_from_state() will return
+         * CHILD_UP or CHILD_DOWN, but not MAXVAL. */
+        event = ec_get_event_from_state(ec);
+        if (event == GF_EVENT_CHILD_UP) {
+            /* We are ready to bring the volume up. If there are
+             * still bricks DOWN, they will be healed when they
+             * come up. */
+            ec_up(ec->xl, ec);
+
+            if (ec->shd.iamshd && !ec->shutdown) {
+                launch_heal = _gf_true;
+                GF_ATOMIC_INC(ec->async_fop_count);
+            }
+        }
+
+        propagate = _gf_true;
+    }
 unlock:
-        UNLOCK(&ec->lock);
+    UNLOCK(&ec->lock);
+
+    if (launch_heal) {
+        /* We have just brought the volume UP, so we trigger
+         * a self-heal check on the root directory. */
+        ec_launch_replace_heal(ec);
+    }
+    if (propagate) {
+        default_notify(ec->xl, event, NULL);
+    }
 }
 
 void
-ec_launch_notify_timer (xlator_t *this, ec_t *ec)
+ec_launch_notify_timer(xlator_t *this, ec_t *ec)
+{
+    struct timespec delay = {
+        0,
+    };
+
+    gf_msg_debug(this->name, 0, "Initiating child-down timer");
+    delay.tv_sec = 10;
+    delay.tv_nsec = 0;
+    ec->timer = gf_timer_call_after(this->ctx, delay, ec_notify_cbk, ec);
+    if (ec->timer == NULL) {
+        gf_msg(this->name, GF_LOG_ERROR, ENOMEM, EC_MSG_TIMER_CREATE_FAIL,
+               "Cannot create timer "
+               "for delayed initialization");
+    }
+}
+
+gf_boolean_t
+ec_disable_delays(ec_t *ec)
 {
-        struct timespec delay = {0, };
+    ec->shutdown = _gf_true;
 
-        gf_log (this->name, GF_LOG_DEBUG, "Initiating child-down timer");
-        delay.tv_sec = 10;
-        delay.tv_nsec = 0;
-        ec->timer = gf_timer_call_after (this->ctx, delay, ec_notify_cbk, ec);
-        if (ec->timer == NULL) {
-                gf_log(this->name, GF_LOG_ERROR, "Cannot create timer "
-                       "for delayed initialization");
-        }
+    return __ec_is_last_fop(ec);
 }
 
 void
-ec_handle_up (xlator_t *this, ec_t *ec, int32_t idx)
+ec_cleanup_healer_object(ec_t *ec)
 {
-        if (((ec->xl_notify >> idx) & 1) == 0) {
-                ec->xl_notify |= 1ULL << idx;
-                ec->xl_notify_count++;
+    struct subvol_healer *healer = NULL;
+    ec_self_heald_t *shd = NULL;
+    void *res = NULL;
+    int i = 0;
+    gf_boolean_t is_join = _gf_false;
+
+    shd = &ec->shd;
+    if (!shd->iamshd)
+        return;
+
+    for (i = 0; i < ec->nodes; i++) {
+        healer = &shd->index_healers[i];
+        pthread_mutex_lock(&healer->mutex);
+        {
+            healer->rerun = 1;
+            if (healer->running) {
+                pthread_cond_signal(&healer->cond);
+                is_join = _gf_true;
+            }
+        }
+        pthread_mutex_unlock(&healer->mutex);
+        if (is_join) {
+            pthread_join(healer->thread, &res);
+            is_join = _gf_false;
         }
 
-        if (((ec->xl_up >> idx) & 1) == 0) { /* Duplicate event */
-                ec->xl_up |= 1ULL << idx;
-                ec->xl_up_count++;
+        healer = &shd->full_healers[i];
+        pthread_mutex_lock(&healer->mutex);
+        {
+            healer->rerun = 1;
+            if (healer->running) {
+                pthread_cond_signal(&healer->cond);
+                is_join = _gf_true;
+            }
+        }
+        pthread_mutex_unlock(&healer->mutex);
+        if (is_join) {
+            pthread_join(healer->thread, &res);
+            is_join = _gf_false;
         }
+    }
 }
-
 void
-ec_handle_down (xlator_t *this, ec_t *ec, int32_t idx)
+ec_pending_fops_completed(ec_t *ec)
 {
-        if (((ec->xl_notify >> idx) & 1) == 0) {
-                ec->xl_notify |= 1ULL << idx;
-                ec->xl_notify_count++;
-        }
+    if (ec->shutdown) {
+        default_notify(ec->xl, GF_EVENT_PARENT_DOWN, NULL);
+    }
+}
 
-        if (((ec->xl_up >> idx) & 1) != 0) { /* Duplicate event */
-                gf_log(this->name, GF_LOG_DEBUG, "Child %d is DOWN", idx);
+static gf_boolean_t
+ec_set_up_state(ec_t *ec, uintptr_t index_mask, uintptr_t new_state)
+{
+    uintptr_t current_state = 0;
 
-                ec->xl_up ^= 1ULL << idx;
-                ec->xl_up_count--;
-        }
+    if (xlator_is_cleanup_starting(ec->xl))
+        return _gf_false;
+
+    if ((ec->xl_notify & index_mask) == 0) {
+        ec->xl_notify |= index_mask;
+        ec->xl_notify_count++;
+    }
+    current_state = ec->xl_up & index_mask;
+    if (current_state != new_state) {
+        ec->xl_up ^= index_mask;
+        ec->xl_up_count += (current_state ? -1 : 1);
+
+        return _gf_true;
+    }
+
+    return _gf_false;
+}
+
+static gf_boolean_t
+ec_upcall(ec_t *ec, struct gf_upcall *upcall)
+{
+    struct gf_upcall_cache_invalidation *ci = NULL;
+    struct gf_upcall_inodelk_contention *lc = NULL;
+    inode_t *inode;
+    inode_table_t *table;
+
+    switch (upcall->event_type) {
+        case GF_UPCALL_CACHE_INVALIDATION:
+            ci = upcall->data;
+            ci->flags |= UP_INVAL_ATTR;
+            return _gf_true;
+
+        case GF_UPCALL_INODELK_CONTENTION:
+            lc = upcall->data;
+            if (strcmp(lc->domain, ec->xl->name) != 0) {
+                /* The lock is not owned by EC, ignore it. */
+                return _gf_true;
+            }
+            table = ((xlator_t *)ec->xl->graph->top)->itable;
+            if (table == NULL) {
+                /* Self-heal daemon doesn't have an inode table on the top
+                 * xlator because it doesn't need it. In this case we should
+                 * use the inode table managed by EC itself where all inodes
+                 * being healed should be present. However self-heal doesn't
+                 * use eager-locking and inodelk's are already released as
+                 * soon as possible. In this case we can safely ignore these
+                 * notifications. */
+                return _gf_false;
+            }
+            inode = inode_find(table, upcall->gfid);
+            /* If inode is not found, it means that it's already released,
+             * so we can ignore it. Probably it has been released and
+             * destroyed while the contention notification was being sent.
+             */
+            if (inode != NULL) {
+                ec_lock_release(ec, inode);
+                inode_unref(inode);
+            }
+
+            return _gf_false;
+
+        default:
+            return _gf_true;
+    }
 }
 
 int32_t
-ec_notify (xlator_t *this, int32_t event, void *data, void *data2)
-{
-        ec_t             *ec        = this->private;
-        int32_t           idx       = 0;
-        int32_t           error     = 0;
-        glusterfs_event_t old_event = GF_EVENT_MAXVAL;
-        glusterfs_event_t new_event = GF_EVENT_MAXVAL;
-        dict_t            *input    = NULL;
-        dict_t            *output   = NULL;
-
-        if (event == GF_EVENT_TRANSLATOR_OP) {
-                if (!ec->up) {
-                        error = -1;
-                        goto out;
-                } else {
-                        input = data;
-                        output = data2;
-                        error = ec_xl_op (this, input, output);
-                }
-                goto out;
-        }
+ec_notify(xlator_t *this, int32_t event, void *data, void *data2)
+{
+    ec_t *ec = this->private;
+    int32_t idx = 0;
+    int32_t error = 0;
+    glusterfs_event_t old_event = GF_EVENT_MAXVAL;
+    dict_t *input = NULL;
+    dict_t *output = NULL;
+    gf_boolean_t propagate = _gf_true;
+    gf_boolean_t needs_shd_check = _gf_false;
+    int32_t orig_event = event;
+    uintptr_t mask = 0;
+
+    gf_msg_trace(this->name, 0, "NOTIFY(%d): %p, %p", event, data, data2);
+
+    if (event == GF_EVENT_UPCALL) {
+        propagate = ec_upcall(ec, data);
+        goto done;
+    }
 
-        for (idx = 0; idx < ec->nodes; idx++) {
-                if (ec->xl_list[idx] == data) {
-                        if (event == GF_EVENT_CHILD_UP)
-                                ec_selfheal_childup (ec, idx);
-                        break;
-                }
+    if (event == GF_EVENT_TRANSLATOR_OP) {
+        if (!ec->up) {
+            error = -1;
+        } else {
+            input = data;
+            output = data2;
+            error = ec_xl_op(this, input, output);
         }
+        goto out;
+    }
 
-        LOCK (&ec->lock);
-
-        if (event == GF_EVENT_PARENT_UP) {
-                /*
-                 * Start a timer which sends appropriate event to parent
-                 * xlator to prevent the 'mount' syscall from hanging.
-                 */
-                ec_launch_notify_timer (this, ec);
-                goto unlock;
+    for (idx = 0; idx < ec->nodes; idx++) {
+        if (ec->xl_list[idx] == data) {
+            break;
         }
+    }
 
-        gf_log (this->name, GF_LOG_TRACE, "NOTIFY(%d): %p, %d",
-                event, data, idx);
+    LOCK(&ec->lock);
 
-        if (idx < ec->nodes) { /* CHILD_* events */
+    if (event == GF_EVENT_PARENT_UP) {
+        /*
+         * Start a timer which sends appropriate event to parent
+         * xlator to prevent the 'mount' syscall from hanging.
+         */
+        ec_launch_notify_timer(this, ec);
+        goto unlock;
+    } else if (event == GF_EVENT_PARENT_DOWN) {
+        /* If there aren't pending fops running after we have waken up
+         * them, we immediately propagate the notification. */
+        propagate = ec_disable_delays(ec);
+        ec_cleanup_healer_object(ec);
+        goto unlock;
+    }
 
-                old_event = ec_get_event_from_state (ec);
+    if (idx < ec->nodes) { /* CHILD_* events */
+        old_event = ec_get_event_from_state(ec);
+
+        mask = 1ULL << idx;
+        if (event == GF_EVENT_CHILD_UP) {
+            /* We need to trigger a selfheal if a brick changes
+             * to UP state. */
+            if (ec_set_up_state(ec, mask, mask) && ec->shd.iamshd &&
+                !ec->shutdown) {
+                needs_shd_check = _gf_true;
+            }
+        } else if (event == GF_EVENT_CHILD_DOWN) {
+            ec_set_up_state(ec, mask, 0);
+        }
 
-                if (event == GF_EVENT_CHILD_UP) {
-                        ec_handle_up (this, ec, idx);
-                } else if (event == GF_EVENT_CHILD_DOWN) {
-                        ec_handle_down (this, ec, idx);
-                }
+        event = ec_get_event_from_state(ec);
 
-                new_event = ec_get_event_from_state (ec);
+        if (event == GF_EVENT_CHILD_UP) {
+            if (!ec->up) {
+                ec_up(this, ec);
+            }
+        } else {
+            /* If the volume is not UP, it's irrelevant if one
+             * brick has come up. We cannot heal anything. */
+            needs_shd_check = _gf_false;
 
-                if (new_event == GF_EVENT_CHILD_UP && !ec->up) {
-                        ec_up (this, ec);
-                } else if (new_event == GF_EVENT_CHILD_DOWN && ec->up) {
-                        ec_down (this, ec);
-                }
+            if ((event == GF_EVENT_CHILD_DOWN) && ec->up) {
+                ec_down(this, ec);
+            }
+        }
 
-                if ((new_event == old_event) && (new_event != GF_EVENT_MAXVAL))
-                        new_event = GF_EVENT_CHILD_MODIFIED;
+        if (event != GF_EVENT_MAXVAL) {
+            if (event == old_event) {
+                if (orig_event == GF_EVENT_CHILD_UP)
+                    event = GF_EVENT_SOME_DESCENDENT_UP;
+                else /* orig_event has to be GF_EVENT_CHILD_DOWN */
+                    event = GF_EVENT_SOME_DESCENDENT_DOWN;
+            }
+        } else {
+            propagate = _gf_false;
+            needs_shd_check = _gf_false;
+        }
 
-                event = GF_EVENT_MAXVAL;/* Take care of notifying inside lock */
-                if (new_event != GF_EVENT_MAXVAL)
-                        error = default_notify (this, new_event, data);
+        if (needs_shd_check) {
+            GF_ATOMIC_INC(ec->async_fop_count);
         }
-    unlock:
-            UNLOCK (&ec->lock);
+    }
+unlock:
+    UNLOCK(&ec->lock);
+
+done:
+    if (needs_shd_check) {
+        ec_launch_replace_heal(ec);
+    }
+    if (propagate) {
+        error = default_notify(this, event, data);
+    }
 
-            if (event != GF_EVENT_MAXVAL)
-                    return default_notify (this, event, data);
 out:
-            return error;
+    return error;
 }
 
 int32_t
-notify (xlator_t *this, int32_t event, void *data, ...)
+notify(xlator_t *this, int32_t event, void *data, ...)
 {
-        int ret = -1;
-        va_list         ap;
-        void *data2 = NULL;
+    int ret = -1;
+    va_list ap;
+    void *data2 = NULL;
 
-        va_start (ap, data);
-        data2 = va_arg (ap, dict_t*);
-        va_end (ap);
-        ret = ec_notify (this, event, data, data2);
+    va_start(ap, data);
+    data2 = va_arg(ap, dict_t *);
+    va_end(ap);
+    ret = ec_notify(this, event, data, data2);
 
-        return ret;
+    return ret;
+}
+
+static void
+ec_statistics_init(ec_t *ec)
+{
+    GF_ATOMIC_INIT(ec->stats.stripe_cache.hits, 0);
+    GF_ATOMIC_INIT(ec->stats.stripe_cache.misses, 0);
+    GF_ATOMIC_INIT(ec->stats.stripe_cache.updates, 0);
+    GF_ATOMIC_INIT(ec->stats.stripe_cache.invals, 0);
+    GF_ATOMIC_INIT(ec->stats.stripe_cache.evicts, 0);
+    GF_ATOMIC_INIT(ec->stats.stripe_cache.allocs, 0);
+    GF_ATOMIC_INIT(ec->stats.stripe_cache.errors, 0);
+    GF_ATOMIC_INIT(ec->stats.shd.attempted, 0);
+    GF_ATOMIC_INIT(ec->stats.shd.completed, 0);
+}
+
+static int
+ec_assign_read_mask(ec_t *ec, char *read_mask_str)
+{
+    char *mask = NULL;
+    char *maskptr = NULL;
+    char *saveptr = NULL;
+    char *id_str = NULL;
+    int id = 0;
+    int ret = 0;
+    uintptr_t read_mask = 0;
+
+    if (!read_mask_str) {
+        ec->read_mask = 0;
+        ret = 0;
+        goto out;
+    }
+
+    mask = gf_strdup(read_mask_str);
+    if (!mask) {
+        ret = -1;
+        goto out;
+    }
+    maskptr = mask;
+
+    for (;;) {
+        id_str = strtok_r(maskptr, ":", &saveptr);
+        if (id_str == NULL)
+            break;
+        if (gf_string2int(id_str, &id)) {
+            gf_msg(ec->xl->name, GF_LOG_ERROR, 0, EC_MSG_XLATOR_INIT_FAIL,
+                   "In read-mask \"%s\" id %s is not a valid integer",
+                   read_mask_str, id_str);
+            ret = -1;
+            goto out;
+        }
+
+        if ((id < 0) || (id >= ec->nodes)) {
+            gf_msg(ec->xl->name, GF_LOG_ERROR, 0, EC_MSG_XLATOR_INIT_FAIL,
+                   "In read-mask \"%s\" id %d is not in range [0 - %d]",
+                   read_mask_str, id, ec->nodes - 1);
+            ret = -1;
+            goto out;
+        }
+        read_mask |= (1UL << id);
+        maskptr = NULL;
+    }
+
+    if (gf_bits_count(read_mask) < ec->fragments) {
+        gf_msg(ec->xl->name, GF_LOG_ERROR, 0, EC_MSG_XLATOR_INIT_FAIL,
+               "read-mask \"%s\" should contain at least %d ids", read_mask_str,
+               ec->fragments);
+        ret = -1;
+        goto out;
+    }
+    ec->read_mask = read_mask;
+    ret = 0;
+out:
+    GF_FREE(mask);
+    return ret;
 }
 
 int32_t
-init (xlator_t *this)
+init(xlator_t *this)
 {
     ec_t *ec = NULL;
-
-    if (this->parents == NULL)
-    {
-        gf_log(this->name, GF_LOG_WARNING, "Volume does not have parents.");
+    char *read_policy = NULL;
+    char *extensions = NULL;
+    int32_t err;
+    char *read_mask_str = NULL;
+
+    if (this->parents == NULL) {
+        gf_msg(this->name, GF_LOG_WARNING, 0, EC_MSG_NO_PARENTS,
+               "Volume does not have parents.");
     }
 
     ec = GF_MALLOC(sizeof(*ec), ec_mt_ec_t);
-    if (ec == NULL)
-    {
-        gf_log(this->name, GF_LOG_ERROR, "Failed to allocate private memory.");
+    if (ec == NULL) {
+        gf_msg(this->name, GF_LOG_ERROR, ENOMEM, EC_MSG_NO_MEMORY,
+               "Failed to allocate private memory.");
 
         return -1;
     }
@@ -447,38 +803,104 @@ init (xlator_t *this)
     ec->xl = this;
     LOCK_INIT(&ec->lock);
 
+    GF_ATOMIC_INIT(ec->async_fop_count, 0);
+    INIT_LIST_HEAD(&ec->pending_fops);
+    INIT_LIST_HEAD(&ec->heal_waiting);
+    INIT_LIST_HEAD(&ec->healing);
+
     ec->fop_pool = mem_pool_new(ec_fop_data_t, 1024);
     ec->cbk_pool = mem_pool_new(ec_cbk_data_t, 4096);
     ec->lock_pool = mem_pool_new(ec_lock_t, 1024);
     if ((ec->fop_pool == NULL) || (ec->cbk_pool == NULL) ||
-        (ec->lock_pool == NULL))
-    {
-        gf_log(this->name, GF_LOG_ERROR, "Failed to create memory pools.");
+        (ec->lock_pool == NULL)) {
+        gf_msg(this->name, GF_LOG_ERROR, ENOMEM, EC_MSG_NO_MEMORY,
+               "Failed to create memory pools.");
 
         goto failed;
     }
 
-    if (ec_prepare_childs(this) != 0)
-    {
-        gf_log(this->name, GF_LOG_ERROR, "Failed to initialize xlator");
+    if (ec_prepare_childs(this) != 0) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_XLATOR_INIT_FAIL,
+               "Failed to initialize xlator");
 
         goto failed;
     }
 
-    if (ec_parse_options(this) != 0)
-    {
-        gf_log(this->name, GF_LOG_ERROR, "Failed to parse xlator options");
+    if (ec_parse_options(this) != 0) {
+        gf_msg(this->name, GF_LOG_ERROR, EINVAL, EC_MSG_XLATOR_PARSE_OPT_FAIL,
+               "Failed to parse xlator options");
 
         goto failed;
     }
 
-    ec_method_initialize();
-    GF_OPTION_INIT ("self-heal-daemon", ec->shd.enabled, bool, failed);
-    GF_OPTION_INIT ("iam-self-heal-daemon", ec->shd.iamshd, bool, failed);
+    GF_OPTION_INIT("cpu-extensions", extensions, str, failed);
+
+    err = ec_method_init(this, &ec->matrix, ec->fragments, ec->nodes,
+                         ec->nodes * 2, extensions);
+    if (err != 0) {
+        gf_msg(this->name, GF_LOG_ERROR, -err, EC_MSG_MATRIX_FAILED,
+               "Failed to initialize matrix management");
+
+        goto failed;
+    }
+
+    GF_OPTION_INIT("self-heal-daemon", ec->shd.enabled, bool, failed);
+    GF_OPTION_INIT("iam-self-heal-daemon", ec->shd.iamshd, bool, failed);
+    GF_OPTION_INIT("eager-lock", ec->eager_lock, bool, failed);
+    GF_OPTION_INIT("other-eager-lock", ec->other_eager_lock, bool, failed);
+    GF_OPTION_INIT("eager-lock-timeout", ec->eager_lock_timeout, uint32,
+                   failed);
+    GF_OPTION_INIT("other-eager-lock-timeout", ec->other_eager_lock_timeout,
+                   uint32, failed);
+    GF_OPTION_INIT("background-heals", ec->background_heals, uint32, failed);
+    GF_OPTION_INIT("heal-wait-qlength", ec->heal_wait_qlen, uint32, failed);
+    GF_OPTION_INIT("self-heal-window-size", ec->self_heal_window_size, uint32,
+                   failed);
+    ec_configure_background_heal_opts(ec, ec->background_heals,
+                                      ec->heal_wait_qlen);
+    GF_OPTION_INIT("read-policy", read_policy, str, failed);
+    if (ec_assign_read_policy(ec, read_policy))
+        goto failed;
+
+    GF_OPTION_INIT("heal-timeout", ec->shd.timeout, int32, failed);
+    GF_OPTION_INIT("shd-max-threads", ec->shd.max_threads, uint32, failed);
+    GF_OPTION_INIT("shd-wait-qlength", ec->shd.wait_qlength, uint32, failed);
+    GF_OPTION_INIT("optimistic-change-log", ec->optimistic_changelog, bool,
+                   failed);
+    GF_OPTION_INIT("parallel-writes", ec->parallel_writes, bool, failed);
+    GF_OPTION_INIT("stripe-cache", ec->stripe_cache, uint32, failed);
+    GF_OPTION_INIT("quorum-count", ec->quorum_count, uint32, failed);
+    GF_OPTION_INIT("ec-read-mask", read_mask_str, str, failed);
+
+    if (ec_assign_read_mask(ec, read_mask_str))
+        goto failed;
+
+    this->itable = inode_table_new(EC_SHD_INODE_LRU_LIMIT, this);
+    if (!this->itable)
+        goto failed;
 
     if (ec->shd.iamshd)
-            ec_selfheal_daemon_init (this);
-    gf_log(this->name, GF_LOG_DEBUG, "Disperse translator initialized.");
+        ec_selfheal_daemon_init(this);
+    gf_msg_debug(this->name, 0, "Disperse translator initialized.");
+
+    ec->leaf_to_subvolid = dict_new();
+    if (!ec->leaf_to_subvolid)
+        goto failed;
+    if (glusterfs_reachable_leaves(this, ec->leaf_to_subvolid)) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_SUBVOL_BUILD_FAIL,
+               "Failed to build subvol "
+               "dictionary");
+        goto failed;
+    }
+
+    if (ec_subvol_to_subvol_id_transform(ec, ec->leaf_to_subvolid) < 0) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, EC_MSG_SUBVOL_ID_DICT_SET_FAIL,
+               "Failed to build subvol-id "
+               "dictionary");
+        goto failed;
+    }
+
+    ec_statistics_init(ec);
 
     return 0;
 
@@ -488,13 +910,16 @@ failed:
     return -1;
 }
 
-void fini(xlator_t * this)
+void
+fini(xlator_t *this)
 {
+    ec_selfheal_daemon_fini(this);
     __ec_destroy_private(this);
 }
 
-int32_t ec_gf_access(call_frame_t * frame, xlator_t * this, loc_t * loc,
-                     int32_t mask, dict_t * xdata)
+int32_t
+ec_gf_access(call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t mask,
+             dict_t *xdata)
 {
     ec_access(frame, this, -1, EC_MINIMUM_ONE, default_access_cbk, NULL, loc,
               mask, xdata);
@@ -502,9 +927,9 @@ int32_t ec_gf_access(call_frame_t * frame, xlator_t * this, loc_t * loc,
     return 0;
 }
 
-int32_t ec_gf_create(call_frame_t * frame, xlator_t * this, loc_t * loc,
-                     int32_t flags, mode_t mode, mode_t umask, fd_t * fd,
-                     dict_t * xdata)
+int32_t
+ec_gf_create(call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t flags,
+             mode_t mode, mode_t umask, fd_t *fd, dict_t *xdata)
 {
     ec_create(frame, this, -1, EC_MINIMUM_MIN, default_create_cbk, NULL, loc,
               flags, mode, umask, fd, xdata);
@@ -512,45 +937,58 @@ int32_t ec_gf_create(call_frame_t * frame, xlator_t * this, loc_t * loc,
     return 0;
 }
 
-int32_t ec_gf_discard(call_frame_t * frame, xlator_t * this, fd_t * fd,
-                      off_t offset, size_t len, dict_t * xdata)
+int32_t
+ec_gf_discard(call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
+              size_t len, dict_t *xdata)
 {
-    default_discard_failure_cbk(frame, ENOTSUP);
+    ec_discard(frame, this, -1, EC_MINIMUM_MIN, default_discard_cbk, NULL, fd,
+               offset, len, xdata);
 
     return 0;
 }
 
-int32_t ec_gf_entrylk(call_frame_t * frame, xlator_t * this,
-                      const char * volume, loc_t * loc, const char * basename,
-                      entrylk_cmd cmd, entrylk_type type, dict_t * xdata)
+int32_t
+ec_gf_entrylk(call_frame_t *frame, xlator_t *this, const char *volume,
+              loc_t *loc, const char *basename, entrylk_cmd cmd,
+              entrylk_type type, dict_t *xdata)
 {
-    ec_entrylk(frame, this, -1, EC_MINIMUM_ALL, default_entrylk_cbk, NULL,
-               volume, loc, basename, cmd, type, xdata);
+    uint32_t fop_flags = EC_MINIMUM_ALL;
+
+    if (cmd == ENTRYLK_UNLOCK)
+        fop_flags = EC_MINIMUM_ONE;
+    ec_entrylk(frame, this, -1, fop_flags, default_entrylk_cbk, NULL, volume,
+               loc, basename, cmd, type, xdata);
 
     return 0;
 }
 
-int32_t ec_gf_fentrylk(call_frame_t * frame, xlator_t * this,
-                       const char * volume, fd_t * fd, const char * basename,
-                       entrylk_cmd cmd, entrylk_type type, dict_t * xdata)
+int32_t
+ec_gf_fentrylk(call_frame_t *frame, xlator_t *this, const char *volume,
+               fd_t *fd, const char *basename, entrylk_cmd cmd,
+               entrylk_type type, dict_t *xdata)
 {
-    ec_fentrylk(frame, this, -1, EC_MINIMUM_ALL, default_fentrylk_cbk, NULL,
-                volume, fd, basename, cmd, type, xdata);
+    uint32_t fop_flags = EC_MINIMUM_ALL;
+
+    if (cmd == ENTRYLK_UNLOCK)
+        fop_flags = EC_MINIMUM_ONE;
+    ec_fentrylk(frame, this, -1, fop_flags, default_fentrylk_cbk, NULL, volume,
+                fd, basename, cmd, type, xdata);
 
     return 0;
 }
 
-int32_t ec_gf_fallocate(call_frame_t * frame, xlator_t * this, fd_t * fd,
-                        int32_t keep_size, off_t offset, size_t len,
-                        dict_t * xdata)
+int32_t
+ec_gf_fallocate(call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t mode,
+                off_t offset, size_t len, dict_t *xdata)
 {
-    default_fallocate_failure_cbk(frame, ENOTSUP);
+    ec_fallocate(frame, this, -1, EC_MINIMUM_MIN, default_fallocate_cbk, NULL,
+                 fd, mode, offset, len, xdata);
 
     return 0;
 }
 
-int32_t ec_gf_flush(call_frame_t * frame, xlator_t * this, fd_t * fd,
-                    dict_t * xdata)
+int32_t
+ec_gf_flush(call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *xdata)
 {
     ec_flush(frame, this, -1, EC_MINIMUM_MIN, default_flush_cbk, NULL, fd,
              xdata);
@@ -558,8 +996,9 @@ int32_t ec_gf_flush(call_frame_t * frame, xlator_t * this, fd_t * fd,
     return 0;
 }
 
-int32_t ec_gf_fsync(call_frame_t * frame, xlator_t * this, fd_t * fd,
-                    int32_t datasync, dict_t * xdata)
+int32_t
+ec_gf_fsync(call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t datasync,
+            dict_t *xdata)
 {
     ec_fsync(frame, this, -1, EC_MINIMUM_MIN, default_fsync_cbk, NULL, fd,
              datasync, xdata);
@@ -567,124 +1006,139 @@ int32_t ec_gf_fsync(call_frame_t * frame, xlator_t * this, fd_t * fd,
     return 0;
 }
 
-int32_t ec_gf_fsyncdir(call_frame_t * frame, xlator_t * this, fd_t * fd,
-                       int32_t datasync, dict_t * xdata)
+int32_t
+ec_gf_fsyncdir(call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t datasync,
+               dict_t *xdata)
 {
-    ec_fsyncdir(frame, this, -1, EC_MINIMUM_MIN, default_fsyncdir_cbk, NULL,
-                fd, datasync, xdata);
+    ec_fsyncdir(frame, this, -1, EC_MINIMUM_MIN, default_fsyncdir_cbk, NULL, fd,
+                datasync, xdata);
 
     return 0;
 }
 
 int
-ec_marker_populate_args (call_frame_t *frame, int type, int *gauge,
-                         xlator_t **subvols)
+ec_marker_populate_args(call_frame_t *frame, int type, int *gauge,
+                        xlator_t **subvols)
 {
-        xlator_t *this = frame->this;
-        ec_t *ec = this->private;
+    xlator_t *this = frame->this;
+    ec_t *ec = this->private;
 
-        memcpy (subvols, ec->xl_list, sizeof (*subvols) * ec->nodes);
+    memcpy(subvols, ec->xl_list, sizeof(*subvols) * ec->nodes);
 
-        if (type == MARKER_XTIME_TYPE) {
-                /*Don't error out on ENOENT/ENOTCONN */
-                gauge[MCNT_NOTFOUND] = 0;
-                gauge[MCNT_ENOTCONN] = 0;
-        }
+    if (type == MARKER_XTIME_TYPE) {
+        /*Don't error out on ENOENT/ENOTCONN */
+        gauge[MCNT_NOTFOUND] = 0;
+        gauge[MCNT_ENOTCONN] = 0;
+    }
 
-        return ec->nodes;
+    return ec->nodes;
 }
 
 int32_t
-ec_handle_heal_commands (call_frame_t *frame, xlator_t *this, loc_t *loc,
-                         const char *name, dict_t *xdata)
+ec_handle_heal_commands(call_frame_t *frame, xlator_t *this, loc_t *loc,
+                        const char *name, dict_t *xdata)
 {
-        dict_t  *dict_rsp = NULL;
-        int     op_ret = -1;
-        int     op_errno = ENOMEM;
+    dict_t *dict_rsp = NULL;
+    int op_ret = -1;
+    int op_errno = ENOMEM;
 
-        if (!name || strcmp (name, GF_HEAL_INFO))
-                return -1;
+    if (!name || strcmp(name, GF_HEAL_INFO))
+        return -1;
 
-        dict_rsp = dict_new ();
-        if (dict_rsp == NULL)
-                goto out;
+    op_errno = -ec_get_heal_info(this, loc, &dict_rsp);
+    if (op_errno <= 0) {
+        op_errno = op_ret = 0;
+    }
 
-        if (dict_set_str (dict_rsp, "heal-info", "heal") == 0)
-                op_ret = 0;
-out:
-        STACK_UNWIND_STRICT (getxattr, frame, op_ret, op_errno, dict_rsp, NULL);
-        if (dict_rsp)
-                dict_unref (dict_rsp);
-        return 0;
+    STACK_UNWIND_STRICT(getxattr, frame, op_ret, op_errno, dict_rsp, NULL);
+    if (dict_rsp)
+        dict_unref(dict_rsp);
+    return 0;
 }
 
 int32_t
-ec_gf_getxattr (call_frame_t *frame, xlator_t *this, loc_t *loc,
-                const char *name, dict_t *xdata)
+ec_gf_getxattr(call_frame_t *frame, xlator_t *this, loc_t *loc,
+               const char *name, dict_t *xdata)
 {
-        int     error = 0;
-        ec_t    *ec = this->private;
+    int error = 0;
+    ec_t *ec = this->private;
+    int32_t fop_flags = EC_MINIMUM_ONE;
 
-        if (name && strcmp (name, EC_XATTR_HEAL) != 0) {
-                EC_INTERNAL_XATTR_OR_GOTO(name, NULL, error, out);
-        }
+    if (name && strcmp(name, EC_XATTR_HEAL) != 0) {
+        EC_INTERNAL_XATTR_OR_GOTO(name, NULL, error, out);
+    }
 
-        if (ec_handle_heal_commands (frame, this, loc, name, xdata) == 0)
-                return 0;
+    if (ec_handle_heal_commands(frame, this, loc, name, xdata) == 0)
+        return 0;
+
+    if (cluster_handle_marker_getxattr(frame, loc, name, ec->vol_uuid, NULL,
+                                       ec_marker_populate_args) == 0)
+        return 0;
 
-        if (cluster_handle_marker_getxattr (frame, loc, name, ec->vol_uuid,
-                                            NULL, ec_marker_populate_args) == 0)
-                return 0;
+    if (name && ((fnmatch(GF_XATTR_STIME_PATTERN, name, 0) == 0) ||
+                 XATTR_IS_NODE_UUID(name) || XATTR_IS_NODE_UUID_LIST(name))) {
+        fop_flags = EC_MINIMUM_ALL;
+    }
 
-        ec_getxattr (frame, this, -1, EC_MINIMUM_MIN, default_getxattr_cbk,
-                     NULL, loc, name, xdata);
+    ec_getxattr(frame, this, -1, fop_flags, default_getxattr_cbk, NULL, loc,
+                name, xdata);
 
-        return 0;
+    return 0;
 out:
-        error = ENODATA;
-        STACK_UNWIND_STRICT (getxattr, frame, -1, error, NULL, NULL);
-        return 0;
+    error = ENODATA;
+    STACK_UNWIND_STRICT(getxattr, frame, -1, error, NULL, NULL);
+    return 0;
 }
 
 int32_t
-ec_gf_fgetxattr (call_frame_t *frame, xlator_t *this, fd_t *fd,
-                 const char *name, dict_t *xdata)
+ec_gf_fgetxattr(call_frame_t *frame, xlator_t *this, fd_t *fd, const char *name,
+                dict_t *xdata)
 {
-        int     error = 0;
+    int error = 0;
 
-        EC_INTERNAL_XATTR_OR_GOTO(name, NULL, error, out);
+    EC_INTERNAL_XATTR_OR_GOTO(name, NULL, error, out);
 
-        ec_fgetxattr (frame, this, -1, EC_MINIMUM_MIN, default_fgetxattr_cbk,
-                      NULL, fd, name, xdata);
-        return 0;
+    ec_fgetxattr(frame, this, -1, EC_MINIMUM_ONE, default_fgetxattr_cbk, NULL,
+                 fd, name, xdata);
+    return 0;
 out:
-        error = ENODATA;
-        STACK_UNWIND_STRICT (fgetxattr, frame, -1, error, NULL, NULL);
-        return 0;
+    error = ENODATA;
+    STACK_UNWIND_STRICT(fgetxattr, frame, -1, error, NULL, NULL);
+    return 0;
 }
 
-int32_t ec_gf_inodelk(call_frame_t * frame, xlator_t * this,
-                      const char * volume, loc_t * loc, int32_t cmd,
-                      struct gf_flock * flock, dict_t * xdata)
+int32_t
+ec_gf_inodelk(call_frame_t *frame, xlator_t *this, const char *volume,
+              loc_t *loc, int32_t cmd, struct gf_flock *flock, dict_t *xdata)
 {
-    ec_inodelk(frame, this, -1, EC_MINIMUM_ALL, default_inodelk_cbk, NULL,
-               volume, loc, cmd, flock, xdata);
+    int32_t fop_flags = EC_MINIMUM_ALL;
+
+    if (flock->l_type == F_UNLCK)
+        fop_flags = EC_MINIMUM_ONE;
+
+    ec_inodelk(frame, this, &frame->root->lk_owner, -1, fop_flags,
+               default_inodelk_cbk, NULL, volume, loc, cmd, flock, xdata);
 
     return 0;
 }
 
-int32_t ec_gf_finodelk(call_frame_t * frame, xlator_t * this,
-                       const char * volume, fd_t * fd, int32_t cmd,
-                       struct gf_flock * flock, dict_t * xdata)
+int32_t
+ec_gf_finodelk(call_frame_t *frame, xlator_t *this, const char *volume,
+               fd_t *fd, int32_t cmd, struct gf_flock *flock, dict_t *xdata)
 {
-    ec_finodelk(frame, this, -1, EC_MINIMUM_ALL, default_finodelk_cbk, NULL,
-                volume, fd, cmd, flock, xdata);
+    int32_t fop_flags = EC_MINIMUM_ALL;
+
+    if (flock->l_type == F_UNLCK)
+        fop_flags = EC_MINIMUM_ONE;
+    ec_finodelk(frame, this, &frame->root->lk_owner, -1, fop_flags,
+                default_finodelk_cbk, NULL, volume, fd, cmd, flock, xdata);
 
     return 0;
 }
 
-int32_t ec_gf_link(call_frame_t * frame, xlator_t * this, loc_t * oldloc,
-                   loc_t * newloc, dict_t * xdata)
+int32_t
+ec_gf_link(call_frame_t *frame, xlator_t *this, loc_t *oldloc, loc_t *newloc,
+           dict_t *xdata)
 {
     ec_link(frame, this, -1, EC_MINIMUM_MIN, default_link_cbk, NULL, oldloc,
             newloc, xdata);
@@ -692,17 +1146,22 @@ int32_t ec_gf_link(call_frame_t * frame, xlator_t * this, loc_t * oldloc,
     return 0;
 }
 
-int32_t ec_gf_lk(call_frame_t * frame, xlator_t * this, fd_t * fd,
-                 int32_t cmd, struct gf_flock * flock, dict_t * xdata)
+int32_t
+ec_gf_lk(call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t cmd,
+         struct gf_flock *flock, dict_t *xdata)
 {
-    ec_lk(frame, this, -1, EC_MINIMUM_ALL, default_lk_cbk, NULL, fd, cmd,
-          flock, xdata);
+    int32_t fop_flags = EC_MINIMUM_ALL;
+
+    if (flock->l_type == F_UNLCK)
+        fop_flags = EC_MINIMUM_ONE;
+    ec_lk(frame, this, -1, fop_flags, default_lk_cbk, NULL, fd, cmd, flock,
+          xdata);
 
     return 0;
 }
 
-int32_t ec_gf_lookup(call_frame_t * frame, xlator_t * this, loc_t * loc,
-                     dict_t * xdata)
+int32_t
+ec_gf_lookup(call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xdata)
 {
     ec_lookup(frame, this, -1, EC_MINIMUM_MIN, default_lookup_cbk, NULL, loc,
               xdata);
@@ -710,8 +1169,9 @@ int32_t ec_gf_lookup(call_frame_t * frame, xlator_t * this, loc_t * loc,
     return 0;
 }
 
-int32_t ec_gf_mkdir(call_frame_t * frame, xlator_t * this, loc_t * loc,
-                    mode_t mode, mode_t umask, dict_t * xdata)
+int32_t
+ec_gf_mkdir(call_frame_t *frame, xlator_t *this, loc_t *loc, mode_t mode,
+            mode_t umask, dict_t *xdata)
 {
     ec_mkdir(frame, this, -1, EC_MINIMUM_MIN, default_mkdir_cbk, NULL, loc,
              mode, umask, xdata);
@@ -719,8 +1179,9 @@ int32_t ec_gf_mkdir(call_frame_t * frame, xlator_t * this, loc_t * loc,
     return 0;
 }
 
-int32_t ec_gf_mknod(call_frame_t * frame, xlator_t * this, loc_t * loc,
-                    mode_t mode, dev_t rdev, mode_t umask, dict_t * xdata)
+int32_t
+ec_gf_mknod(call_frame_t *frame, xlator_t *this, loc_t *loc, mode_t mode,
+            dev_t rdev, mode_t umask, dict_t *xdata)
 {
     ec_mknod(frame, this, -1, EC_MINIMUM_MIN, default_mknod_cbk, NULL, loc,
              mode, rdev, umask, xdata);
@@ -728,17 +1189,19 @@ int32_t ec_gf_mknod(call_frame_t * frame, xlator_t * this, loc_t * loc,
     return 0;
 }
 
-int32_t ec_gf_open(call_frame_t * frame, xlator_t * this, loc_t * loc,
-                   int32_t flags, fd_t * fd, dict_t * xdata)
+int32_t
+ec_gf_open(call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t flags,
+           fd_t *fd, dict_t *xdata)
 {
-    ec_open(frame, this, -1, EC_MINIMUM_MIN, default_open_cbk, NULL, loc,
-            flags, fd, xdata);
+    ec_open(frame, this, -1, EC_MINIMUM_MIN, default_open_cbk, NULL, loc, flags,
+            fd, xdata);
 
     return 0;
 }
 
-int32_t ec_gf_opendir(call_frame_t * frame, xlator_t * this, loc_t * loc,
-                      fd_t * fd, dict_t * xdata)
+int32_t
+ec_gf_opendir(call_frame_t *frame, xlator_t *this, loc_t *loc, fd_t *fd,
+              dict_t *xdata)
 {
     ec_opendir(frame, this, -1, EC_MINIMUM_MIN, default_opendir_cbk, NULL, loc,
                fd, xdata);
@@ -746,8 +1209,9 @@ int32_t ec_gf_opendir(call_frame_t * frame, xlator_t * this, loc_t * loc,
     return 0;
 }
 
-int32_t ec_gf_readdir(call_frame_t * frame, xlator_t * this, fd_t * fd,
-                      size_t size, off_t offset, dict_t * xdata)
+int32_t
+ec_gf_readdir(call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
+              off_t offset, dict_t *xdata)
 {
     ec_readdir(frame, this, -1, EC_MINIMUM_ONE, default_readdir_cbk, NULL, fd,
                size, offset, xdata);
@@ -755,17 +1219,19 @@ int32_t ec_gf_readdir(call_frame_t * frame, xlator_t * this, fd_t * fd,
     return 0;
 }
 
-int32_t ec_gf_readdirp(call_frame_t * frame, xlator_t * this, fd_t * fd,
-                       size_t size, off_t offset, dict_t * xdata)
+int32_t
+ec_gf_readdirp(call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
+               off_t offset, dict_t *xdata)
 {
-    ec_readdirp(frame, this, -1, EC_MINIMUM_ONE, default_readdirp_cbk, NULL,
-                fd, size, offset, xdata);
+    ec_readdirp(frame, this, -1, EC_MINIMUM_ONE, default_readdirp_cbk, NULL, fd,
+                size, offset, xdata);
 
     return 0;
 }
 
-int32_t ec_gf_readlink(call_frame_t * frame, xlator_t * this, loc_t * loc,
-                       size_t size, dict_t * xdata)
+int32_t
+ec_gf_readlink(call_frame_t *frame, xlator_t *this, loc_t *loc, size_t size,
+               dict_t *xdata)
 {
     ec_readlink(frame, this, -1, EC_MINIMUM_ONE, default_readlink_cbk, NULL,
                 loc, size, xdata);
@@ -773,60 +1239,63 @@ int32_t ec_gf_readlink(call_frame_t * frame, xlator_t * this, loc_t * loc,
     return 0;
 }
 
-int32_t ec_gf_readv(call_frame_t * frame, xlator_t * this, fd_t * fd,
-                    size_t size, off_t offset, uint32_t flags, dict_t * xdata)
+int32_t
+ec_gf_readv(call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
+            off_t offset, uint32_t flags, dict_t *xdata)
 {
-    ec_readv(frame, this, -1, EC_MINIMUM_MIN, default_readv_cbk, NULL, fd,
-             size, offset, flags, xdata);
+    ec_readv(frame, this, -1, EC_MINIMUM_MIN, default_readv_cbk, NULL, fd, size,
+             offset, flags, xdata);
 
     return 0;
 }
 
 int32_t
-ec_gf_removexattr (call_frame_t *frame, xlator_t *this, loc_t *loc,
-                   const char *name, dict_t *xdata)
+ec_gf_removexattr(call_frame_t *frame, xlator_t *this, loc_t *loc,
+                  const char *name, dict_t *xdata)
 {
-        int     error  = 0;
+    int error = 0;
 
-        EC_INTERNAL_XATTR_OR_GOTO (name, xdata, error, out);
+    EC_INTERNAL_XATTR_OR_GOTO(name, xdata, error, out);
 
-        ec_removexattr (frame, this, -1, EC_MINIMUM_MIN,
-                        default_removexattr_cbk, NULL, loc, name, xdata);
+    ec_removexattr(frame, this, -1, EC_MINIMUM_MIN, default_removexattr_cbk,
+                   NULL, loc, name, xdata);
 
-        return 0;
+    return 0;
 out:
-        STACK_UNWIND_STRICT (removexattr, frame, -1, error, NULL);
-        return 0;
+    STACK_UNWIND_STRICT(removexattr, frame, -1, error, NULL);
+    return 0;
 }
 
 int32_t
-ec_gf_fremovexattr (call_frame_t *frame, xlator_t *this, fd_t *fd,
-                    const char *name, dict_t *xdata)
+ec_gf_fremovexattr(call_frame_t *frame, xlator_t *this, fd_t *fd,
+                   const char *name, dict_t *xdata)
 {
-        int     error  = 0;
+    int error = 0;
 
-        EC_INTERNAL_XATTR_OR_GOTO (name, xdata, error, out);
+    EC_INTERNAL_XATTR_OR_GOTO(name, xdata, error, out);
 
-        ec_fremovexattr (frame, this, -1, EC_MINIMUM_MIN,
-                         default_fremovexattr_cbk, NULL, fd, name, xdata);
+    ec_fremovexattr(frame, this, -1, EC_MINIMUM_MIN, default_fremovexattr_cbk,
+                    NULL, fd, name, xdata);
 
-        return 0;
+    return 0;
 out:
-        STACK_UNWIND_STRICT (fremovexattr, frame, -1, error, NULL);
-        return 0;
+    STACK_UNWIND_STRICT(fremovexattr, frame, -1, error, NULL);
+    return 0;
 }
 
-int32_t ec_gf_rename(call_frame_t * frame, xlator_t * this, loc_t * oldloc,
-                     loc_t * newloc, dict_t * xdata)
+int32_t
+ec_gf_rename(call_frame_t *frame, xlator_t *this, loc_t *oldloc, loc_t *newloc,
+             dict_t *xdata)
 {
-    ec_rename(frame, this, -1, EC_MINIMUM_MIN, default_rename_cbk, NULL,
-              oldloc, newloc, xdata);
+    ec_rename(frame, this, -1, EC_MINIMUM_MIN, default_rename_cbk, NULL, oldloc,
+              newloc, xdata);
 
     return 0;
 }
 
-int32_t ec_gf_rmdir(call_frame_t * frame, xlator_t * this, loc_t * loc,
-                    int xflags, dict_t * xdata)
+int32_t
+ec_gf_rmdir(call_frame_t *frame, xlator_t *this, loc_t *loc, int xflags,
+            dict_t *xdata)
 {
     ec_rmdir(frame, this, -1, EC_MINIMUM_MIN, default_rmdir_cbk, NULL, loc,
              xflags, xdata);
@@ -834,8 +1303,9 @@ int32_t ec_gf_rmdir(call_frame_t * frame, xlator_t * this, loc_t * loc,
     return 0;
 }
 
-int32_t ec_gf_setattr(call_frame_t * frame, xlator_t * this, loc_t * loc,
-                      struct iatt * stbuf, int32_t valid, dict_t * xdata)
+int32_t
+ec_gf_setattr(call_frame_t *frame, xlator_t *this, loc_t *loc,
+              struct iatt *stbuf, int32_t valid, dict_t *xdata)
 {
     ec_setattr(frame, this, -1, EC_MINIMUM_MIN, default_setattr_cbk, NULL, loc,
                stbuf, valid, xdata);
@@ -843,51 +1313,52 @@ int32_t ec_gf_setattr(call_frame_t * frame, xlator_t * this, loc_t * loc,
     return 0;
 }
 
-int32_t ec_gf_fsetattr(call_frame_t * frame, xlator_t * this, fd_t * fd,
-                       struct iatt * stbuf, int32_t valid, dict_t * xdata)
+int32_t
+ec_gf_fsetattr(call_frame_t *frame, xlator_t *this, fd_t *fd,
+               struct iatt *stbuf, int32_t valid, dict_t *xdata)
 {
-    ec_fsetattr(frame, this, -1, EC_MINIMUM_MIN, default_fsetattr_cbk, NULL,
-                fd, stbuf, valid, xdata);
+    ec_fsetattr(frame, this, -1, EC_MINIMUM_MIN, default_fsetattr_cbk, NULL, fd,
+                stbuf, valid, xdata);
 
     return 0;
 }
 
 int32_t
-ec_gf_setxattr (call_frame_t *frame, xlator_t *this, loc_t *loc,
-                dict_t *dict, int32_t flags, dict_t *xdata)
+ec_gf_setxattr(call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *dict,
+               int32_t flags, dict_t *xdata)
 {
-        int     error = 0;
+    int error = 0;
 
-        EC_INTERNAL_XATTR_OR_GOTO ("", dict, error, out);
+    EC_INTERNAL_XATTR_OR_GOTO("", dict, error, out);
 
-        ec_setxattr (frame, this, -1, EC_MINIMUM_MIN, default_setxattr_cbk,
-                     NULL, loc, dict, flags, xdata);
+    ec_setxattr(frame, this, -1, EC_MINIMUM_MIN, default_setxattr_cbk, NULL,
+                loc, dict, flags, xdata);
 
-        return 0;
+    return 0;
 out:
-        STACK_UNWIND_STRICT (setxattr, frame, -1, error, NULL);
-        return 0;
+    STACK_UNWIND_STRICT(setxattr, frame, -1, error, NULL);
+    return 0;
 }
 
 int32_t
-ec_gf_fsetxattr (call_frame_t *frame, xlator_t *this, fd_t *fd,
-                 dict_t *dict, int32_t flags, dict_t *xdata)
+ec_gf_fsetxattr(call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *dict,
+                int32_t flags, dict_t *xdata)
 {
-        int     error = 0;
+    int error = 0;
 
-        EC_INTERNAL_XATTR_OR_GOTO ("", dict, error, out);
+    EC_INTERNAL_XATTR_OR_GOTO("", dict, error, out);
 
-        ec_fsetxattr (frame, this, -1, EC_MINIMUM_MIN, default_fsetxattr_cbk,
-                      NULL, fd, dict, flags, xdata);
+    ec_fsetxattr(frame, this, -1, EC_MINIMUM_MIN, default_fsetxattr_cbk, NULL,
+                 fd, dict, flags, xdata);
 
-        return 0;
+    return 0;
 out:
-        STACK_UNWIND_STRICT (fsetxattr, frame, -1, error, NULL);
-        return 0;
+    STACK_UNWIND_STRICT(fsetxattr, frame, -1, error, NULL);
+    return 0;
 }
 
-int32_t ec_gf_stat(call_frame_t * frame, xlator_t * this, loc_t * loc,
-                   dict_t * xdata)
+int32_t
+ec_gf_stat(call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xdata)
 {
     ec_stat(frame, this, -1, EC_MINIMUM_MIN, default_stat_cbk, NULL, loc,
             xdata);
@@ -895,8 +1366,8 @@ int32_t ec_gf_stat(call_frame_t * frame, xlator_t * this, loc_t * loc,
     return 0;
 }
 
-int32_t ec_gf_fstat(call_frame_t * frame, xlator_t * this, fd_t * fd,
-                    dict_t * xdata)
+int32_t
+ec_gf_fstat(call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *xdata)
 {
     ec_fstat(frame, this, -1, EC_MINIMUM_MIN, default_fstat_cbk, NULL, fd,
              xdata);
@@ -904,8 +1375,8 @@ int32_t ec_gf_fstat(call_frame_t * frame, xlator_t * this, fd_t * fd,
     return 0;
 }
 
-int32_t ec_gf_statfs(call_frame_t * frame, xlator_t * this, loc_t * loc,
-                     dict_t * xdata)
+int32_t
+ec_gf_statfs(call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xdata)
 {
     ec_statfs(frame, this, -1, EC_MINIMUM_MIN, default_statfs_cbk, NULL, loc,
               xdata);
@@ -913,9 +1384,9 @@ int32_t ec_gf_statfs(call_frame_t * frame, xlator_t * this, loc_t * loc,
     return 0;
 }
 
-int32_t ec_gf_symlink(call_frame_t * frame, xlator_t * this,
-                      const char * linkname, loc_t * loc, mode_t umask,
-                      dict_t * xdata)
+int32_t
+ec_gf_symlink(call_frame_t *frame, xlator_t *this, const char *linkname,
+              loc_t *loc, mode_t umask, dict_t *xdata)
 {
     ec_symlink(frame, this, -1, EC_MINIMUM_MIN, default_symlink_cbk, NULL,
                linkname, loc, umask, xdata);
@@ -923,8 +1394,9 @@ int32_t ec_gf_symlink(call_frame_t * frame, xlator_t * this,
     return 0;
 }
 
-int32_t ec_gf_truncate(call_frame_t * frame, xlator_t * this, loc_t * loc,
-                       off_t offset, dict_t * xdata)
+int32_t
+ec_gf_truncate(call_frame_t *frame, xlator_t *this, loc_t *loc, off_t offset,
+               dict_t *xdata)
 {
     ec_truncate(frame, this, -1, EC_MINIMUM_MIN, default_truncate_cbk, NULL,
                 loc, offset, xdata);
@@ -932,8 +1404,9 @@ int32_t ec_gf_truncate(call_frame_t * frame, xlator_t * this, loc_t * loc,
     return 0;
 }
 
-int32_t ec_gf_ftruncate(call_frame_t * frame, xlator_t * this, fd_t * fd,
-                        off_t offset, dict_t * xdata)
+int32_t
+ec_gf_ftruncate(call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
+                dict_t *xdata)
 {
     ec_ftruncate(frame, this, -1, EC_MINIMUM_MIN, default_ftruncate_cbk, NULL,
                  fd, offset, xdata);
@@ -941,8 +1414,9 @@ int32_t ec_gf_ftruncate(call_frame_t * frame, xlator_t * this, fd_t * fd,
     return 0;
 }
 
-int32_t ec_gf_unlink(call_frame_t * frame, xlator_t * this, loc_t * loc,
-                     int xflags, dict_t * xdata)
+int32_t
+ec_gf_unlink(call_frame_t *frame, xlator_t *this, loc_t *loc, int xflags,
+             dict_t *xdata)
 {
     ec_unlink(frame, this, -1, EC_MINIMUM_MIN, default_unlink_cbk, NULL, loc,
               xflags, xdata);
@@ -950,9 +1424,10 @@ int32_t ec_gf_unlink(call_frame_t * frame, xlator_t * this, loc_t * loc,
     return 0;
 }
 
-int32_t ec_gf_writev(call_frame_t * frame, xlator_t * this, fd_t * fd,
-                     struct iovec * vector, int32_t count, off_t offset,
-                     uint32_t flags, struct iobref * iobref, dict_t * xdata)
+int32_t
+ec_gf_writev(call_frame_t *frame, xlator_t *this, fd_t *fd,
+             struct iovec *vector, int32_t count, off_t offset, uint32_t flags,
+             struct iobref *iobref, dict_t *xdata)
 {
     ec_writev(frame, this, -1, EC_MINIMUM_MIN, default_writev_cbk, NULL, fd,
               vector, count, offset, flags, iobref, xdata);
@@ -960,9 +1435,9 @@ int32_t ec_gf_writev(call_frame_t * frame, xlator_t * this, fd_t * fd,
     return 0;
 }
 
-int32_t ec_gf_xattrop(call_frame_t * frame, xlator_t * this, loc_t * loc,
-                      gf_xattrop_flags_t optype, dict_t * xattr,
-                      dict_t * xdata)
+int32_t
+ec_gf_xattrop(call_frame_t *frame, xlator_t *this, loc_t *loc,
+              gf_xattrop_flags_t optype, dict_t *xattr, dict_t *xdata)
 {
     ec_xattrop(frame, this, -1, EC_MINIMUM_MIN, default_xattrop_cbk, NULL, loc,
                optype, xattr, xdata);
@@ -970,70 +1445,94 @@ int32_t ec_gf_xattrop(call_frame_t * frame, xlator_t * this, loc_t * loc,
     return 0;
 }
 
-int32_t ec_gf_fxattrop(call_frame_t * frame, xlator_t * this, fd_t * fd,
-                       gf_xattrop_flags_t optype, dict_t * xattr,
-                       dict_t * xdata)
+int32_t
+ec_gf_fxattrop(call_frame_t *frame, xlator_t *this, fd_t *fd,
+               gf_xattrop_flags_t optype, dict_t *xattr, dict_t *xdata)
 {
-    ec_fxattrop(frame, this, -1, EC_MINIMUM_MIN, default_fxattrop_cbk, NULL,
-                fd, optype, xattr, xdata);
+    ec_fxattrop(frame, this, -1, EC_MINIMUM_MIN, default_fxattrop_cbk, NULL, fd,
+                optype, xattr, xdata);
 
     return 0;
 }
 
-int32_t ec_gf_zerofill(call_frame_t * frame, xlator_t * this, fd_t * fd,
-                       off_t offset, off_t len, dict_t * xdata)
+int32_t
+ec_gf_zerofill(call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
+               off_t len, dict_t *xdata)
 {
     default_zerofill_failure_cbk(frame, ENOTSUP);
 
     return 0;
 }
 
-int32_t ec_gf_forget(xlator_t * this, inode_t * inode)
+int32_t
+ec_gf_seek(call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
+           gf_seek_what_t what, dict_t *xdata)
+{
+    ec_seek(frame, this, -1, EC_MINIMUM_ONE, default_seek_cbk, NULL, fd, offset,
+            what, xdata);
+
+    return 0;
+}
+
+int32_t
+ec_gf_ipc(call_frame_t *frame, xlator_t *this, int32_t op, dict_t *xdata)
+{
+    ec_ipc(frame, this, -1, EC_MINIMUM_MIN, default_ipc_cbk, NULL, op, xdata);
+    return 0;
+}
+
+int32_t
+ec_gf_forget(xlator_t *this, inode_t *inode)
 {
     uint64_t value = 0;
-    ec_inode_t * ctx = NULL;
+    ec_inode_t *ctx = NULL;
 
-    if ((inode_ctx_del(inode, this, &value) == 0) && (value != 0))
-    {
+    if ((inode_ctx_del(inode, this, &value) == 0) && (value != 0)) {
         ctx = (ec_inode_t *)(uintptr_t)value;
+        /* We can only forget an inode if it has been unlocked, so the stripe
+         * cache should also be empty. */
+        GF_ASSERT(list_empty(&ctx->stripe_cache.lru));
         GF_FREE(ctx);
     }
 
     return 0;
 }
 
-void ec_gf_release_fd(xlator_t * this, fd_t * fd)
+void
+ec_gf_release_fd(xlator_t *this, fd_t *fd)
 {
     uint64_t value = 0;
-    ec_fd_t * ctx = NULL;
+    ec_fd_t *ctx = NULL;
 
-    if ((fd_ctx_del(fd, this, &value) == 0) && (value != 0))
-    {
+    if ((fd_ctx_del(fd, this, &value) == 0) && (value != 0)) {
         ctx = (ec_fd_t *)(uintptr_t)value;
         loc_wipe(&ctx->loc);
         GF_FREE(ctx);
     }
 }
 
-int32_t ec_gf_release(xlator_t * this, fd_t * fd)
+int32_t
+ec_gf_release(xlator_t *this, fd_t *fd)
 {
     ec_gf_release_fd(this, fd);
 
     return 0;
 }
 
-int32_t ec_gf_releasedir(xlator_t * this, fd_t * fd)
+int32_t
+ec_gf_releasedir(xlator_t *this, fd_t *fd)
 {
     ec_gf_release_fd(this, fd);
 
     return 0;
 }
 
-int32_t ec_dump_private(xlator_t *this)
+int32_t
+ec_dump_private(xlator_t *this)
 {
     ec_t *ec = NULL;
-    char  key_prefix[GF_DUMP_MAX_BUF_LEN];
-    char  tmp[65];
+    char key_prefix[GF_DUMP_MAX_BUF_LEN];
+    char tmp[65];
 
     GF_ASSERT(this);
 
@@ -1041,7 +1540,8 @@ int32_t ec_dump_private(xlator_t *this)
     GF_ASSERT(ec);
 
     snprintf(key_prefix, GF_DUMP_MAX_BUF_LEN, "%s.%s", this->type, this->name);
-    gf_proc_dump_add_section(key_prefix);
+    gf_proc_dump_add_section("%s", key_prefix);
+    gf_proc_dump_write("up", "%u", ec->up);
     gf_proc_dump_write("nodes", "%u", ec->nodes);
     gf_proc_dump_write("redundancy", "%u", ec->redundancy);
     gf_proc_dump_write("fragment_size", "%u", ec->fragment_size);
@@ -1049,88 +1549,325 @@ int32_t ec_dump_private(xlator_t *this)
     gf_proc_dump_write("childs_up", "%u", ec->xl_up_count);
     gf_proc_dump_write("childs_up_mask", "%s",
                        ec_bin(tmp, sizeof(tmp), ec->xl_up, ec->nodes));
+    if (ec->read_mask) {
+        gf_proc_dump_write("read-mask", "%s",
+                           ec_bin(tmp, sizeof(tmp), ec->read_mask, ec->nodes));
+    }
+    gf_proc_dump_write("background-heals", "%d", ec->background_heals);
+    gf_proc_dump_write("heal-wait-qlength", "%d", ec->heal_wait_qlen);
+    gf_proc_dump_write("self-heal-window-size", "%" PRIu32,
+                       ec->self_heal_window_size);
+    gf_proc_dump_write("healers", "%d", ec->healers);
+    gf_proc_dump_write("heal-waiters", "%d", ec->heal_waiters);
+    gf_proc_dump_write("read-policy", "%s", ec_read_policies[ec->read_policy]);
+    gf_proc_dump_write("parallel-writes", "%d", ec->parallel_writes);
+    gf_proc_dump_write("quorum-count", "%u", ec->quorum_count);
+
+    snprintf(key_prefix, GF_DUMP_MAX_BUF_LEN, "%s.%s.stats.stripe_cache",
+             this->type, this->name);
+    gf_proc_dump_add_section("%s", key_prefix);
+
+    gf_proc_dump_write("hits", "%" GF_PRI_ATOMIC,
+                       GF_ATOMIC_GET(ec->stats.stripe_cache.hits));
+    gf_proc_dump_write("misses", "%" GF_PRI_ATOMIC,
+                       GF_ATOMIC_GET(ec->stats.stripe_cache.misses));
+    gf_proc_dump_write("updates", "%" GF_PRI_ATOMIC,
+                       GF_ATOMIC_GET(ec->stats.stripe_cache.updates));
+    gf_proc_dump_write("invalidations", "%" GF_PRI_ATOMIC,
+                       GF_ATOMIC_GET(ec->stats.stripe_cache.invals));
+    gf_proc_dump_write("evicts", "%" GF_PRI_ATOMIC,
+                       GF_ATOMIC_GET(ec->stats.stripe_cache.evicts));
+    gf_proc_dump_write("allocations", "%" GF_PRI_ATOMIC,
+                       GF_ATOMIC_GET(ec->stats.stripe_cache.allocs));
+    gf_proc_dump_write("errors", "%" GF_PRI_ATOMIC,
+                       GF_ATOMIC_GET(ec->stats.stripe_cache.errors));
+    gf_proc_dump_write("heals-attempted", "%" GF_PRI_ATOMIC,
+                       GF_ATOMIC_GET(ec->stats.shd.attempted));
+    gf_proc_dump_write("heals-completed", "%" GF_PRI_ATOMIC,
+                       GF_ATOMIC_GET(ec->stats.shd.completed));
 
     return 0;
 }
 
-struct xlator_fops fops =
-{
-    .lookup       = ec_gf_lookup,
-    .stat         = ec_gf_stat,
-    .fstat        = ec_gf_fstat,
-    .truncate     = ec_gf_truncate,
-    .ftruncate    = ec_gf_ftruncate,
-    .access       = ec_gf_access,
-    .readlink     = ec_gf_readlink,
-    .mknod        = ec_gf_mknod,
-    .mkdir        = ec_gf_mkdir,
-    .unlink       = ec_gf_unlink,
-    .rmdir        = ec_gf_rmdir,
-    .symlink      = ec_gf_symlink,
-    .rename       = ec_gf_rename,
-    .link         = ec_gf_link,
-    .create       = ec_gf_create,
-    .open         = ec_gf_open,
-    .readv        = ec_gf_readv,
-    .writev       = ec_gf_writev,
-    .flush        = ec_gf_flush,
-    .fsync        = ec_gf_fsync,
-    .opendir      = ec_gf_opendir,
-    .readdir      = ec_gf_readdir,
-    .readdirp     = ec_gf_readdirp,
-    .fsyncdir     = ec_gf_fsyncdir,
-    .statfs       = ec_gf_statfs,
-    .setxattr     = ec_gf_setxattr,
-    .getxattr     = ec_gf_getxattr,
-    .fsetxattr    = ec_gf_fsetxattr,
-    .fgetxattr    = ec_gf_fgetxattr,
-    .removexattr  = ec_gf_removexattr,
-    .fremovexattr = ec_gf_fremovexattr,
-    .lk           = ec_gf_lk,
-    .inodelk      = ec_gf_inodelk,
-    .finodelk     = ec_gf_finodelk,
-    .entrylk      = ec_gf_entrylk,
-    .fentrylk     = ec_gf_fentrylk,
-    .xattrop      = ec_gf_xattrop,
-    .fxattrop     = ec_gf_fxattrop,
-    .setattr      = ec_gf_setattr,
-    .fsetattr     = ec_gf_fsetattr,
-    .fallocate    = ec_gf_fallocate,
-    .discard      = ec_gf_discard,
-    .zerofill     = ec_gf_zerofill
-};
-
-struct xlator_cbks cbks =
-{
-    .forget            = ec_gf_forget,
-    .release           = ec_gf_release,
-    .releasedir        = ec_gf_releasedir
-};
-
-struct xlator_dumpops dumpops = {
-    .priv = ec_dump_private
-};
-
-struct volume_options options[] =
-{
+struct xlator_fops fops = {.lookup = ec_gf_lookup,
+                           .stat = ec_gf_stat,
+                           .fstat = ec_gf_fstat,
+                           .truncate = ec_gf_truncate,
+                           .ftruncate = ec_gf_ftruncate,
+                           .access = ec_gf_access,
+                           .readlink = ec_gf_readlink,
+                           .mknod = ec_gf_mknod,
+                           .mkdir = ec_gf_mkdir,
+                           .unlink = ec_gf_unlink,
+                           .rmdir = ec_gf_rmdir,
+                           .symlink = ec_gf_symlink,
+                           .rename = ec_gf_rename,
+                           .link = ec_gf_link,
+                           .create = ec_gf_create,
+                           .open = ec_gf_open,
+                           .readv = ec_gf_readv,
+                           .writev = ec_gf_writev,
+                           .flush = ec_gf_flush,
+                           .fsync = ec_gf_fsync,
+                           .opendir = ec_gf_opendir,
+                           .readdir = ec_gf_readdir,
+                           .readdirp = ec_gf_readdirp,
+                           .fsyncdir = ec_gf_fsyncdir,
+                           .statfs = ec_gf_statfs,
+                           .setxattr = ec_gf_setxattr,
+                           .getxattr = ec_gf_getxattr,
+                           .fsetxattr = ec_gf_fsetxattr,
+                           .fgetxattr = ec_gf_fgetxattr,
+                           .removexattr = ec_gf_removexattr,
+                           .fremovexattr = ec_gf_fremovexattr,
+                           .lk = ec_gf_lk,
+                           .inodelk = ec_gf_inodelk,
+                           .finodelk = ec_gf_finodelk,
+                           .entrylk = ec_gf_entrylk,
+                           .fentrylk = ec_gf_fentrylk,
+                           .xattrop = ec_gf_xattrop,
+                           .fxattrop = ec_gf_fxattrop,
+                           .setattr = ec_gf_setattr,
+                           .fsetattr = ec_gf_fsetattr,
+                           .fallocate = ec_gf_fallocate,
+                           .discard = ec_gf_discard,
+                           .zerofill = ec_gf_zerofill,
+                           .seek = ec_gf_seek,
+                           .ipc = ec_gf_ipc};
+
+struct xlator_cbks cbks = {.forget = ec_gf_forget,
+                           .release = ec_gf_release,
+                           .releasedir = ec_gf_releasedir};
+
+struct xlator_dumpops dumpops = {.priv = ec_dump_private};
+
+struct volume_options options[] = {
+    {.key = {"redundancy"},
+     .type = GF_OPTION_TYPE_INT,
+     .default_value = "{{ volume.redundancy }}",
+     .description = "Maximum number of bricks that can fail "
+                    "simultaneously without losing data."},
     {
-        .key = { "redundancy" },
-        .type = GF_OPTION_TYPE_INT,
-        .description = "Maximum number of bricks that can fail "
-                       "simultaneously without losing data."
-    },
-    {
-        .key = { "self-heal-daemon" },
+        .key = {"self-heal-daemon"},
         .type = GF_OPTION_TYPE_BOOL,
         .description = "self-heal daemon enable/disable",
         .default_value = "enable",
+        .op_version = {GD_OP_VERSION_3_7_0},
+        .flags = OPT_FLAG_SETTABLE | OPT_FLAG_DOC,
+        .tags = {"disperse"},
+    },
+    {.key = {"iam-self-heal-daemon"},
+     .type = GF_OPTION_TYPE_BOOL,
+     .default_value = "off",
+     .description = "This option differentiates if the disperse "
+                    "translator is running as part of self-heal-daemon "
+                    "or not."},
+    {.key = {"eager-lock"},
+     .type = GF_OPTION_TYPE_BOOL,
+     .default_value = "on",
+     .op_version = {GD_OP_VERSION_3_7_10},
+     .flags = OPT_FLAG_SETTABLE | OPT_FLAG_CLIENT_OPT | OPT_FLAG_DOC,
+     .tags = {"disperse"},
+     .description = "Enable/Disable eager lock for regular files on a "
+                    "disperse volume. If a fop takes a lock and completes "
+                    "its operation, it waits for next 1 second before "
+                    "releasing the lock, to see if the lock can be reused "
+                    "for next fop from the same client. If ec finds any lock "
+                    "contention within 1 second it releases the lock "
+                    "immediately before time expires. This improves the "
+                    "performance of file operations. However, as it takes "
+                    "lock on first brick, for few operations like read, "
+                    "discovery of lock contention might take long time and "
+                    "can actually degrade the performance. If eager lock is "
+                    "disabled, lock will be released as soon as fop "
+                    "completes."},
+    {.key = {"other-eager-lock"},
+     .type = GF_OPTION_TYPE_BOOL,
+     .default_value = "on",
+     .op_version = {GD_OP_VERSION_3_13_0},
+     .flags = OPT_FLAG_SETTABLE | OPT_FLAG_CLIENT_OPT | OPT_FLAG_DOC,
+     .tags = {"disperse"},
+     .description = "It's equivalent to the eager-lock option but for non "
+                    "regular files."},
+    {.key = {"eager-lock-timeout"},
+     .type = GF_OPTION_TYPE_INT,
+     .min = 1,
+     .max = 60,
+     .default_value = "1",
+     .op_version = {GD_OP_VERSION_4_0_0},
+     .flags = OPT_FLAG_SETTABLE | OPT_FLAG_CLIENT_OPT | OPT_FLAG_DOC,
+     .tags = {"disperse", "locks", "timeout"},
+     .description = "Maximum time (in seconds) that a lock on an inode is "
+                    "kept held if no new operations on the inode are "
+                    "received."},
+    {.key = {"other-eager-lock-timeout"},
+     .type = GF_OPTION_TYPE_INT,
+     .min = 1,
+     .max = 60,
+     .default_value = "1",
+     .op_version = {GD_OP_VERSION_4_0_0},
+     .flags = OPT_FLAG_SETTABLE | OPT_FLAG_CLIENT_OPT | OPT_FLAG_DOC,
+     .tags = {"disperse", "locks", "timeout"},
+     .description = "It's equivalent to eager-lock-timeout option but for "
+                    "non regular files."},
+    {
+        .key = {"background-heals"},
+        .type = GF_OPTION_TYPE_INT,
+        .min = 0, /*Disabling background heals*/
+        .max = 256,
+        .default_value = "8",
+        .op_version = {GD_OP_VERSION_3_7_3},
+        .flags = OPT_FLAG_SETTABLE | OPT_FLAG_CLIENT_OPT | OPT_FLAG_DOC,
+        .tags = {"disperse"},
+        .description = "This option can be used to control number of parallel"
+                       " heals",
+    },
+    {
+        .key = {"heal-wait-qlength"},
+        .type = GF_OPTION_TYPE_INT,
+        .min = 0,
+        .max =
+            65536, /*Around 100MB as of now with sizeof(ec_fop_data_t) at 1800*/
+        .default_value = "128",
+        .op_version = {GD_OP_VERSION_3_7_3},
+        .flags = OPT_FLAG_SETTABLE | OPT_FLAG_CLIENT_OPT | OPT_FLAG_DOC,
+        .tags = {"disperse"},
+        .description = "This option can be used to control number of heals"
+                       " that can wait",
+    },
+    {.key = {"heal-timeout"},
+     .type = GF_OPTION_TYPE_INT,
+     .min = 60,
+     .max = INT_MAX,
+     .default_value = "600",
+     .op_version = {GD_OP_VERSION_3_7_3},
+     .flags = OPT_FLAG_SETTABLE,
+     .tags = {"disperse"},
+     .description = "time interval for checking the need to self-heal "
+                    "in self-heal-daemon"},
+    {
+        .key = {"read-policy"},
+        .type = GF_OPTION_TYPE_STR,
+        .value = {"round-robin", "gfid-hash"},
+        .default_value = "gfid-hash",
+        .op_version = {GD_OP_VERSION_3_7_6},
+        .flags = OPT_FLAG_SETTABLE | OPT_FLAG_CLIENT_OPT | OPT_FLAG_DOC,
+        .tags = {"disperse"},
+        .description =
+            "inode-read fops happen only on 'k' number of bricks in"
+            " n=k+m disperse subvolume. 'round-robin' selects the read"
+            " subvolume using round-robin algo. 'gfid-hash' selects read"
+            " subvolume based on hash of the gfid of that file/directory.",
     },
-    { .key = {"iam-self-heal-daemon"},
-      .type = GF_OPTION_TYPE_BOOL,
-      .default_value = "off",
-      .description = "This option differentiates if the disperse "
-                     "translator is running as part of self-heal-daemon "
-                     "or not."
+    {.key = {"shd-max-threads"},
+     .type = GF_OPTION_TYPE_INT,
+     .min = 1,
+     .max = 64,
+     .default_value = "1",
+     .op_version = {GD_OP_VERSION_3_9_0},
+     .flags = OPT_FLAG_SETTABLE | OPT_FLAG_DOC,
+     .tags = {"disperse"},
+     .description = "Maximum number of parallel heals SHD can do per local "
+                    "brick.  This can substantially lower heal times, "
+                    "but can also crush your bricks if you don't have "
+                    "the storage hardware to support this."},
+    {.key = {"shd-wait-qlength"},
+     .type = GF_OPTION_TYPE_INT,
+     .min = 1,
+     .max = 65536,
+     .default_value = "1024",
+     .op_version = {GD_OP_VERSION_3_9_0},
+     .flags = OPT_FLAG_SETTABLE | OPT_FLAG_DOC,
+     .tags = {"disperse"},
+     .description = "This option can be used to control number of heals"
+                    " that can wait in SHD per subvolume"},
+    {.key = {"cpu-extensions"},
+     .type = GF_OPTION_TYPE_STR,
+     .value = {"none", "auto", "x64", "sse", "avx"},
+     .default_value = "auto",
+     .op_version = {GD_OP_VERSION_3_9_0},
+     .flags = OPT_FLAG_SETTABLE | OPT_FLAG_CLIENT_OPT | OPT_FLAG_DOC,
+     .tags = {"disperse"},
+     .description = "force the cpu extensions to be used to accelerate the "
+                    "galois field computations."},
+    {.key = {"self-heal-window-size"},
+     .type = GF_OPTION_TYPE_INT,
+     .min = 1,
+     .max = 1024,
+     .default_value = "1",
+     .op_version = {GD_OP_VERSION_3_11_0},
+     .flags = OPT_FLAG_SETTABLE | OPT_FLAG_CLIENT_OPT | OPT_FLAG_DOC,
+     .tags = {"disperse"},
+     .description = "Maximum number blocks(128KB) per file for which "
+                    "self-heal process would be applied simultaneously."},
+    {.key = {"optimistic-change-log"},
+     .type = GF_OPTION_TYPE_BOOL,
+     .default_value = "on",
+     .op_version = {GD_OP_VERSION_3_10_1},
+     .flags = OPT_FLAG_SETTABLE | OPT_FLAG_CLIENT_OPT,
+     .tags = {"disperse"},
+     .description = "Set/Unset dirty flag for every update fop at the start"
+                    "of the fop. If OFF, this option impacts performance of"
+                    "entry  operations or metadata operations as it will"
+                    "set dirty flag at the start and unset it at the end of"
+                    "ALL update fop. If ON and all the bricks are good,"
+                    "dirty flag will be set at the start only for file fops"
+                    "For metadata and entry fops dirty flag will not be set"
+                    "at the start, if all the bricks are good. This does"
+                    "not impact performance for metadata operations and"
+                    "entry operation but has a very small window to miss"
+                    "marking entry as dirty in case it is required to be"
+                    "healed"},
+    {.key = {"parallel-writes"},
+     .type = GF_OPTION_TYPE_BOOL,
+     .default_value = "on",
+     .description = "This controls if writes can be wound in parallel as long"
+                    "as it doesn't modify same stripes"},
+    {.key = {"stripe-cache"},
+     .type = GF_OPTION_TYPE_INT,
+     .min = 0, /*Disabling stripe_cache*/
+     .max = EC_STRIPE_CACHE_MAX_SIZE,
+     .default_value = "4",
+     .description = "This option will keep the last stripe of write fop"
+                    "in memory. If next write falls in this stripe, we need"
+                    "not to read it again from backend and we can save READ"
+                    "fop going over the network. This will improve performance,"
+                    "specially for sequential writes. However, this will also"
+                    "lead to extra memory consumption, maximum "
+                    "(cache size * stripe size) Bytes per open file."},
+    {
+        .key = {"quorum-count"},
+        .type = GF_OPTION_TYPE_INT,
+        .default_value = "0",
+        .description =
+            "This option can be used to define how many successes on"
+            "the bricks constitute a success to the application. This"
+            " count should be in the range"
+            "[disperse-data-count,  disperse-count] (inclusive)",
+    },
+    {
+        .key = {"ec-read-mask"},
+        .type = GF_OPTION_TYPE_STR,
+        .default_value = NULL,
+        .description = "This option can be used to choose which bricks can be"
+                       " used for reading data/metadata of a file/directory",
     },
-    { }
+    {
+        .key = {NULL},
+    },
+};
+
+xlator_api_t xlator_api = {
+    .init = init,
+    .fini = fini,
+    .notify = notify,
+    .reconfigure = reconfigure,
+    .mem_acct_init = mem_acct_init,
+    .op_version = {1},
+    .dumpops = &dumpops,
+    .fops = &fops,
+    .cbks = &cbks,
+    .options = options,
+    .identifier = "disperse",
+    .category = GF_MAINTAINED,
 };
diff --git a/xlators/cluster/ec/src/ec.h b/xlators/cluster/ec/src/ec.h
index d565bf8fcb4..6f6de6d5981 100644
--- a/xlators/cluster/ec/src/ec.h
+++ b/xlators/cluster/ec/src/ec.h
@@ -11,41 +11,24 @@
 #ifndef __EC_H__
 #define __EC_H__
 
-#include "xlator.h"
-#include "timer.h"
-#include "ec-heald.h"
-#include "libxlator.h"
+#include "ec-method.h"
 
-#define EC_XATTR_PREFIX  "trusted.ec."
-#define EC_XATTR_CONFIG  EC_XATTR_PREFIX"config"
-#define EC_XATTR_SIZE    EC_XATTR_PREFIX"size"
-#define EC_XATTR_VERSION EC_XATTR_PREFIX"version"
-#define EC_XATTR_HEAL    EC_XATTR_PREFIX"heal"
-#define EC_XATTR_DIRTY   EC_XATTR_PREFIX"dirty"
+#define EC_XATTR_PREFIX "trusted.ec."
+#define EC_XATTR_CONFIG EC_XATTR_PREFIX "config"
+#define EC_XATTR_SIZE EC_XATTR_PREFIX "size"
+#define EC_XATTR_VERSION EC_XATTR_PREFIX "version"
+#define EC_XATTR_HEAL EC_XATTR_PREFIX "heal"
+#define EC_XATTR_HEAL_NEW EC_XATTR_PREFIX "heal-new"
+#define EC_XATTR_DIRTY EC_XATTR_PREFIX "dirty"
+#define EC_STRIPE_CACHE_MAX_SIZE 10
+#define EC_VERSION_SIZE 2
+#define EC_SHD_INODE_LRU_LIMIT 10
+
+#define EC_MAX_FRAGMENTS EC_METHOD_MAX_FRAGMENTS
+/* The maximum number of nodes is derived from the maximum allowed fragments
+ * using the rule that redundancy cannot be equal or greater than the number
+ * of fragments.
+ */
+#define EC_MAX_NODES min(EC_MAX_FRAGMENTS * 2 - 1, EC_METHOD_MAX_NODES)
 
-struct _ec
-{
-    xlator_t *        xl;
-    int32_t           nodes;
-    int32_t           bits_for_nodes;
-    int32_t           fragments;
-    int32_t           redundancy;
-    uint32_t          fragment_size;
-    uint32_t          stripe_size;
-    int32_t           up;
-    uint32_t          idx;
-    uint32_t          xl_up_count;
-    uintptr_t         xl_up;
-    uint32_t          xl_notify_count;
-    uintptr_t         xl_notify;
-    uintptr_t         node_mask;
-    xlator_t **       xl_list;
-    gf_lock_t         lock;
-    gf_timer_t *      timer;
-    struct mem_pool * fop_pool;
-    struct mem_pool * cbk_pool;
-    struct mem_pool * lock_pool;
-    ec_self_heald_t   shd;
-    char              vol_uuid[UUID_SIZE + 1];
-};
 #endif /* __EC_H__ */
diff --git a/xlators/cluster/ha/src/Makefile.am b/xlators/cluster/ha/src/Makefile.am
deleted file mode 100644
index 5c1364b7f9d..00000000000
--- a/xlators/cluster/ha/src/Makefile.am
+++ /dev/null
@@ -1,16 +0,0 @@
-xlator_LTLIBRARIES = ha.la
-xlatordir = $(libdir)/glusterfs/$(PACKAGE_VERSION)/xlator/testing/cluster
-
-ha_la_LDFLAGS = -module -avoid-version 
-
-ha_la_SOURCES = ha-helpers.c ha.c
-ha_la_LIBADD = $(top_builddir)/libglusterfs/src/libglusterfs.la
-
-noinst_HEADERS = ha.h
-
-AM_CPPFLAGS = $(GF_CPPFLAGS) -I$(top_srcdir)/libglusterfs/src
-
-AM_CFLAGS = -Wall $(GF_CFLAGS)
-
-CLEANFILES = 
-
diff --git a/xlators/cluster/ha/src/ha-helpers.c b/xlators/cluster/ha/src/ha-helpers.c
deleted file mode 100644
index 19be1ed2773..00000000000
--- a/xlators/cluster/ha/src/ha-helpers.c
+++ /dev/null
@@ -1,194 +0,0 @@
-/*
-   Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
-   This file is part of GlusterFS.
-
-   This file is licensed to you under your choice of the GNU Lesser
-   General Public License, version 3 or any later version (LGPLv3 or
-   later), or the GNU General Public License, version 2 (GPLv2), in all
-   cases as published by the Free Software Foundation.
-*/
-#include "xlator.h"
-#include "call-stub.h"
-#include "defaults.h"
-#include "dict.h"
-#include "compat-errno.h"
-#include "ha.h"
-
-#define HA_TRANSPORT_NOTCONN(_ret, _errno, _fd) \
-	((_ret == -1) && (_fd ? (_errno == EBADFD):(_errno == ENOTCONN)))
-
-int ha_alloc_init_fd (call_frame_t *frame, fd_t *fd)
-{
-	ha_local_t *local = NULL;
-	int i = -1;
-	ha_private_t *pvt = NULL;
-	int child_count = 0;
-	int ret = -1;
-	hafd_t *hafdp = NULL;
-	xlator_t *this = NULL;
-	uint64_t tmp_hafdp = 0;
-
-	this = frame->this;
-	local = frame->local;
-	pvt = this->private;
-	child_count = pvt->child_count;
-
-	if (local == NULL) {
-		ret = fd_ctx_get (fd, this, &tmp_hafdp);
-		if (ret < 0) {
-			goto out;
-		}
-		hafdp = (hafd_t *)(long)tmp_hafdp;
-                local = frame->local = GF_CALLOC (1, sizeof (*local), 
-                                                  gf_ha_mt_ha_local_t);
-		if (local == NULL) {
-			ret = -ENOMEM;
-			goto out;
-		}
-                local->state = GF_CALLOC (1, child_count, 
-                                          gf_ha_mt_child_count);
-		if (local->state == NULL) {
-			ret = -ENOMEM;
-			goto out;
-		}
-
-		/* take care of the preferred subvolume */
-		if (pvt->pref_subvol == -1)
-			local->active = hafdp->active;
-		else
-			local->active = pvt->pref_subvol;
-
-		LOCK (&hafdp->lock);
-		memcpy (local->state, hafdp->fdstate, child_count);
-		UNLOCK (&hafdp->lock);
-
-		/* in case the preferred subvolume is down */
-		if ((local->active != -1) && (local->state[local->active] == 0))
-			local->active = -1;
-
-		for (i = 0; i < child_count; i++) {
-			if (local->state[i]) {
-				if (local->active == -1)
-					local->active = i;
-				local->tries++;
-			}
-		}
-		if (local->active == -1) {
-			ret = -ENOTCONN;
-			goto out;
-		}
-		local->fd = fd_ref (fd);
-	}
-	ret = 0;
-out:
-	return ret;
-}
-
-int ha_handle_cbk (call_frame_t *frame, void *cookie, int op_ret, int op_errno) 
-{
-	xlator_t *xl = NULL;
-	ha_private_t *pvt = NULL;
-	xlator_t **children = NULL;
-	int prev_child = -1;
-	hafd_t *hafdp = NULL;
-	int ret = -1;
-	call_stub_t *stub = NULL;
-	ha_local_t *local = NULL;
-	uint64_t tmp_hafdp = 0;
-
-	xl = frame->this;
-	pvt = xl->private;
-	children = pvt->children;
-	prev_child = (long) cookie;
-	local = frame->local;
-
-	if (op_ret == -1) {
-		gf_log (xl->name, GF_LOG_ERROR ,"(child=%s) (op_ret=%d op_errno=%s)",
-			children[prev_child]->name, op_ret, strerror (op_errno));
-	}
-
-	if (HA_TRANSPORT_NOTCONN (op_ret, op_errno, (local->fd))) {
-		ret = 0;
-		if (local->fd) {
-			ret = fd_ctx_get (local->fd, xl, &tmp_hafdp);
-		}
-		hafdp = (hafd_t *)(long)tmp_hafdp;		
-		if (ret == 0) {
-			if (local->fd) {
-				LOCK(&hafdp->lock);
-				hafdp->fdstate[prev_child] = 0;
-				UNLOCK(&hafdp->lock);
-			}
-			local->tries--;
-			if (local->tries != 0) {
-				while (1) {
-					local->active = (local->active + 1) % pvt->child_count;
-					if (local->state[local->active])
-						break;
-				}
-				stub = local->stub;
-				local->stub = NULL;
-				call_resume (stub);
-				return -1;
-			}
-		}
-	}
-	if (local->stub) {
-		call_stub_destroy (local->stub);
-                local->stub = NULL;
-        }
-
-	if (local->fd) {
-		GF_FREE (local->state);
-                local->state = NULL;
-
-		fd_unref (local->fd);
-                local->fd = NULL;
-	}
-	return 0;
-}
-
-int ha_alloc_init_inode (call_frame_t *frame, inode_t *inode)
-{
-	int i = -1;
-	ha_private_t *pvt = NULL;
-	xlator_t *xl = NULL;
-	int ret = -1;
-	ha_local_t *local = NULL;
-	uint64_t tmp_state = 0;
-
-	xl = frame->this;
-	pvt = xl->private;
-	local = frame->local;
-
-	if (local == NULL) {
-                local = frame->local = GF_CALLOC (1, sizeof (*local), 
-                                                  gf_ha_mt_ha_local_t);
-		if (local == NULL) {
-			ret = -ENOMEM;
-			goto out;
-		}
-		local->active = pvt->pref_subvol;
-		ret = inode_ctx_get (inode, xl, &tmp_state);
-		if (ret < 0) {
-			goto out;
-		}
-		local->state = (char *)(long)tmp_state;
-		if (local->active != -1 && local->state[local->active] == 0)
-			local->active = -1;
-		for (i = 0; i < pvt->child_count; i++) {
-			if (local->state[i]) {
-				if (local->active == -1)
-					local->active = i;
-				local->tries++;
-			}
-		}
-		if (local->active == -1) {
-			ret = -ENOTCONN;
-			goto out;
-		}
-	}
-	ret = 0;
-out:
-	return ret;
-}
diff --git a/xlators/cluster/ha/src/ha-mem-types.h b/xlators/cluster/ha/src/ha-mem-types.h
deleted file mode 100644
index e5e97d237dc..00000000000
--- a/xlators/cluster/ha/src/ha-mem-types.h
+++ /dev/null
@@ -1,26 +0,0 @@
-/*
-   Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
-   This file is part of GlusterFS.
-
-   This file is licensed to you under your choice of the GNU Lesser
-   General Public License, version 3 or any later version (LGPLv3 or
-   later), or the GNU General Public License, version 2 (GPLv2), in all
-   cases as published by the Free Software Foundation.
-*/
-
-#ifndef __HA_MEM_TYPES_H__
-#define __HA_MEM_TYPES_H__
-
-#include "mem-types.h"
-
-enum gf_ha_mem_types_ {
-        gf_ha_mt_ha_local_t = gf_common_mt_end + 1,
-        gf_ha_mt_hafd_t,
-        gf_ha_mt_char,
-        gf_ha_mt_child_count,
-        gf_ha_mt_xlator_t,
-        gf_ha_mt_ha_private_t,
-        gf_ha_mt_end
-};
-#endif
-
diff --git a/xlators/cluster/ha/src/ha.c b/xlators/cluster/ha/src/ha.c
deleted file mode 100644
index 742ec46f107..00000000000
--- a/xlators/cluster/ha/src/ha.c
+++ /dev/null
@@ -1,4013 +0,0 @@
-/*
-   Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
-   This file is part of GlusterFS.
-
-   This file is licensed to you under your choice of the GNU Lesser
-   General Public License, version 3 or any later version (LGPLv3 or
-   later), or the GNU General Public License, version 2 (GPLv2), in all
-   cases as published by the Free Software Foundation.
-*/
-/* generate errors randomly, code is simple now, better alogorithm
- * can be written to decide what error to be returned and when
- */
-
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
-
-#include "xlator.h"
-#include "call-stub.h"
-#include "defaults.h"
-#include "dict.h"
-#include "compat-errno.h"
-#include "ha.h"
-
-/*
- * TODO:
- * - dbench fails if ha over server side afr
- * - lock calls - lock on all subvols.
- * - support preferred-subvolume option. code already there.
- * - do not alloc the call-stub in case only one subvol is up.
- */
-
-void
-ha_local_wipe (ha_local_t *local)
-{
-        if (local->stub) {
-                call_stub_destroy (local->stub);
-                local->stub = NULL;
-        }
-
-        if (local->state) {
-                GF_FREE (local->state);
-                local->state = NULL;
-        }
-
-        if (local->dict) {
-                dict_unref (local->dict);
-                local->dict = NULL;
-        }
-
-        loc_wipe (&local->loc);
-
-        if (local->fd) {
-                fd_unref (local->fd);
-                local->fd = NULL;
-        }
-
-        if (local->inode) {
-                inode_unref (local->inode);
-                local->inode = NULL;
-        }
-
-        GF_FREE (local);
-        return;
-}
-
-
-int
-ha_forget (xlator_t *this,
-	   inode_t *inode)
-{
-	uint64_t stateino = 0;
-	char *state = NULL;
-	if (!inode_ctx_del (inode, this, &stateino)) {
-		state =  ((char *)(long)stateino);
-		GF_FREE (state);
-	}
-
-	return 0;
-
-}
-
-int32_t 
-ha_lookup_cbk (call_frame_t *frame,
-	       void *cookie,
-	       xlator_t *this,
-	       int32_t op_ret,
-	       int32_t op_errno,
-	       inode_t *inode,
-	       struct iatt *buf,
-               dict_t *dict,
-               struct iatt *postparent)
-{
-	ha_local_t *local = NULL;
-	ha_private_t *pvt = NULL;
-	int child_count = 0, i = 0, callcnt = 0;
-	char *state = NULL;
-	call_frame_t *prev_frame = NULL;
-	xlator_t **children = NULL;
-	uint64_t tmp_state = 0;
-
-	local = frame->local;
-	pvt = this->private;
-	child_count = pvt->child_count;
-	prev_frame = cookie;
-	children = pvt->children;
-
-	for (i = 0; i < child_count; i++) {
-		if (pvt->children[i] == prev_frame->this)
-			break;
-	}
-	if ((op_ret == -1) && (op_errno != ENOENT)) {
-		gf_log (this->name, GF_LOG_ERROR, "(child=%s) (op_ret=%d op_errno=%s)", 
-			  children[i]->name, op_ret, strerror (op_errno));
-	}
-	inode_ctx_get (local->inode, this, &tmp_state);
-	state = (char *)(long)tmp_state;
-
-	LOCK (&frame->lock);
-	if (local->revalidate == 1) {
-		if ((!op_ret) != state[i]) {
-			local->revalidate_error = 1;
-			gf_log (this->name, GF_LOG_DEBUG, "revalidate error on %s", 
-				pvt->children[i]->name);
-		}
-	} else {
-		if (op_ret == 0) {
-			state[i] = 1;
-		}
-	}
-	if (local->op_ret == -1 && op_ret == 0) {
-		local->op_ret = 0;
-		local->buf = *buf;
-                local->postparent = *postparent;
-		if (dict)
-			local->dict = dict_ref (dict);
-	}
-	if (op_ret == -1 && op_ret != ENOTCONN)
-		local->op_errno = op_errno;
-	callcnt = --local->call_count;
-	UNLOCK (&frame->lock);
-
-	if (callcnt == 0) {
-		dict_t *ctx = local->dict;
-		inode_t *inode = local->inode;
-		if (local->revalidate_error == 1) {
-			local->op_ret = -1;
-			local->op_errno = EIO;
-			gf_log (this->name, GF_LOG_DEBUG, "revalidate error, returning EIO");
-		}
-		STACK_UNWIND (frame,
-			      local->op_ret,
-			      local->op_errno,
-			      inode,
-			      &local->buf,
-			      ctx,
-                              &local->postparent);
-		if (inode)
-			inode_unref (inode);
-		if (ctx)
-			dict_unref (ctx);
-	}
-	return 0;
-}
-
-int32_t
-ha_lookup (call_frame_t *frame,
-	   xlator_t *this,
-	   loc_t *loc,
-	   dict_t *xattr_req)
-{
-	ha_local_t *local = NULL;
-	ha_private_t *pvt = NULL;
-	int child_count = 0, i = 0;
-	char *state = NULL;
-	xlator_t **children = NULL;
-	int ret = -1;
-        int32_t op_errno = EINVAL;
-
-	local = frame->local;
-	pvt = this->private;
-	child_count = pvt->child_count;
-	children = pvt->children;
-
-	frame->local = local = GF_CALLOC (1, sizeof (*local), 
-                                          gf_ha_mt_ha_local_t);
-        if (!local) {
-                gf_log (this->name, GF_LOG_ERROR, "out of memory");
-                op_errno = ENOMEM;
-                goto unwind;
-        }
-
-	child_count = pvt->child_count;
-	local->inode = inode_ref (loc->inode);
-
-	ret = inode_ctx_get (loc->inode, this, NULL);
-	if (ret) {
-		state = GF_CALLOC (1, child_count, gf_ha_mt_child_count);
-                if (state == NULL) {
-                        gf_log (this->name, GF_LOG_ERROR, "out of memory");
-                        op_errno = ENOMEM;
-                        goto unwind;
-                }
-
-		inode_ctx_put (loc->inode, this, (uint64_t)(long)state);
-	} else
-		local->revalidate = 1;
-
-	local->op_ret = -1;
-	local->op_errno = ENOTCONN;
-	local->call_count = child_count;
-
-	for (i = 0; i < child_count; i++) {
-		STACK_WIND (frame,
-			    ha_lookup_cbk,
-			    children[i],
-			    children[i]->fops->lookup,
-			    loc,
-			    xattr_req);
-	}
-	return 0;
-
-unwind:
-        local = frame->local;
-        frame->local = NULL;
-	STACK_UNWIND (frame, -1, op_errno, NULL, NULL, NULL, NULL);
-
-        ha_local_wipe (local);
-        return 0;
-}
-
- int32_t
-ha_stat_cbk (call_frame_t *frame,
-	     void *cookie,
-	     xlator_t *this,
-	     int32_t op_ret,
-	     int32_t op_errno,
-	     struct iatt *buf)
-{
-	int ret = -1;
-
-	ret = ha_handle_cbk (frame, cookie, op_ret, op_errno);
-
-	if (ret == 0) {
-		STACK_UNWIND (frame,
-			      op_ret,
-			      op_errno,
-			      buf);
-	}
-	return 0;
-}
-
-int32_t
-ha_stat (call_frame_t *frame,
-	 xlator_t *this,
-	 loc_t *loc)
-{
-	ha_local_t *local = NULL;
-	int op_errno = ENOTCONN;
-
-	op_errno = ha_alloc_init_inode (frame, loc->inode);
-	if (op_errno < 0) {
-		op_errno = -op_errno;
-		goto err;
-	}
-	local = frame->local;
-	local->stub = fop_stat_stub (frame, ha_stat, loc);
-
-	STACK_WIND_COOKIE (frame,
-			   ha_stat_cbk,
-			   (void *)(long)local->active,
-			   HA_ACTIVE_CHILD(this, local),
-			   HA_ACTIVE_CHILD(this, local)->fops->stat,
-			   loc);
-	return 0;
-err:
-	STACK_UNWIND (frame, -1, op_errno, NULL);
-	return 0;	
-}
-
-int32_t
-ha_setattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                int32_t op_ret, int32_t op_errno, struct iatt *statpre,
-                struct iatt *statpost)
-{
-	int ret = -1;
-
-	ret = ha_handle_cbk (frame, cookie, op_ret, op_errno);
-
-	if (ret == 0) {
-		STACK_UNWIND (frame, op_ret, op_errno, statpre, statpost);
-	}
-	return 0;
-}
-
-
-int32_t
-ha_setattr (call_frame_t *frame, xlator_t *this, loc_t *loc, struct iatt *stbuf,
-            int32_t valid)
-{
-	ha_local_t *local = NULL;
-	int op_errno = 0;
-
-	op_errno = ha_alloc_init_inode (frame, loc->inode);
-	if (op_errno < 0) {
-		op_errno = -op_errno;
-		goto err;
-	}
-	local = frame->local;
-	local->stub = fop_setattr_stub (frame, ha_setattr, loc, stbuf, valid);
-
-	STACK_WIND_COOKIE (frame,
-			   ha_setattr_cbk,
-			   (void *)(long)local->active,
-			   HA_ACTIVE_CHILD(this, local),
-			   HA_ACTIVE_CHILD(this, local)->fops->setattr,
-			   loc, stbuf, valid);
-	return 0;
-err:
-	STACK_UNWIND (frame, -1, op_errno, NULL, NULL);
-	return 0;
-}
-
-
-int32_t
-ha_fsetattr (call_frame_t *frame, xlator_t *this, fd_t *fd, struct iatt *stbuf,
-             int32_t valid)
-{
-	ha_local_t *local = NULL;
-	int op_errno = 0;
-
-	op_errno = ha_alloc_init_fd (frame, fd);
-	if (op_errno < 0) {
-		op_errno = -op_errno;
-		goto err;
-	}
-	local = frame->local;
-	local->stub = fop_fsetattr_stub (frame, ha_fsetattr, fd, stbuf, valid);
-
-	STACK_WIND_COOKIE (frame,	      
-			   ha_setattr_cbk,
-			   (void *)(long)local->active,
-			   HA_ACTIVE_CHILD(this, local),
-			   HA_ACTIVE_CHILD(this, local)->fops->fsetattr,
-			   fd, stbuf, valid);
-	return 0;
-err:
-	STACK_UNWIND (frame, -1, op_errno, NULL, NULL);
-	return 0;
-}
-
-
-int32_t
-ha_truncate_cbk (call_frame_t *frame,
-		 void *cookie,
-		 xlator_t *this,
-		 int32_t op_ret,
-		 int32_t op_errno,
-		 struct iatt *prebuf,
-                 struct iatt *postbuf)
-{
-	int ret = -1;
-
-	ret = ha_handle_cbk (frame, cookie, op_ret, op_errno);
-	if (ret == 0) {
-		STACK_UNWIND (frame,
-			      op_ret,
-			      op_errno,
-			      prebuf,
-                              postbuf);
-	}
-	return 0;
-}
-
-int32_t
-ha_truncate (call_frame_t *frame,
-	     xlator_t *this,
-	     loc_t *loc,
-	     off_t offset)
-{
-	ha_local_t *local = NULL;
-	int op_errno = 0;
-
-	op_errno = ha_alloc_init_inode (frame, loc->inode);
-	if (op_errno < 0) {
-		op_errno = -op_errno;
-		goto err;
-	}
-	local = frame->local;
-	local->stub = fop_truncate_stub (frame, ha_truncate, loc, offset);
-        if (!local->stub) {
-                op_errno = ENOMEM;
-                gf_log (this->name, GF_LOG_ERROR, "out of memory");
-                goto err;
-        }
-
-	STACK_WIND_COOKIE (frame,
-			   ha_truncate_cbk,
-			   (void *)(long)local->active,
-			   HA_ACTIVE_CHILD(this, local),
-			   HA_ACTIVE_CHILD(this, local)->fops->truncate,
-			   loc,
-			   offset);
-	return 0;
-err:
-	STACK_UNWIND (frame, -1, op_errno, NULL, NULL);
-	return 0;
-}
-
- int32_t
-ha_ftruncate_cbk (call_frame_t *frame,
-		  void *cookie,
-		  xlator_t *this,
-		  int32_t op_ret,
-		  int32_t op_errno,
-		  struct iatt *prebuf,
-                  struct iatt *postbuf)
-{
-	int ret = -1;
-
-	ret = ha_handle_cbk (frame, cookie, op_ret, op_errno);
-
-	if (ret == 0) {
-		STACK_UNWIND (frame,
-			      op_ret,
-			      op_errno,
-			      prebuf, postbuf);
-	}
-	return 0;
-}
-
-int32_t
-ha_ftruncate (call_frame_t *frame,
-	      xlator_t *this,
-	      fd_t *fd,
-	      off_t offset)
-{
-	ha_local_t *local = NULL;
-	int op_errno = 0;
-
-	op_errno = ha_alloc_init_fd (frame, fd);
-	if (op_errno < 0) {
-		op_errno = -op_errno;
-		goto err;
-	}
-	local = frame->local;
-	local->stub = fop_ftruncate_stub (frame, ha_ftruncate, fd, offset);
-        if (!local->stub) {
-                op_errno = ENOMEM;
-                gf_log (this->name, GF_LOG_ERROR, "out of memory");
-                goto err;
-        }
-
-	STACK_WIND_COOKIE (frame,
-			   ha_ftruncate_cbk,
-			   (void *)(long)local->active,
-			   HA_ACTIVE_CHILD(this, local),
-			   HA_ACTIVE_CHILD(this, local)->fops->ftruncate,
-			   fd,
-			   offset);
-	return 0;
-err:
-        local = frame->local;
-        frame->local = NULL;
-
-	STACK_UNWIND (frame, -1, op_errno, NULL, NULL);
-
-        ha_local_wipe (local);
-	return 0;
-}
-
-int32_t
-ha_access_cbk (call_frame_t *frame,
-	       void *cookie,
-	       xlator_t *this,
-	       int32_t op_ret,
-	       int32_t op_errno)
-{
-	int ret = -1;
-
-	ret = ha_handle_cbk (frame, cookie, op_ret, op_errno);
-
-	if (ret == 0) {
-		STACK_UNWIND (frame,
-			      op_ret,
-			      op_errno);
-	}
-	return 0;
-}
-
-int32_t
-ha_access (call_frame_t *frame,
-	   xlator_t *this,
-	   loc_t *loc,
-	   int32_t mask)
-{
-	ha_local_t *local = NULL;
-	int op_errno = 0;
-
-	op_errno = ha_alloc_init_inode (frame, loc->inode);
-	if (op_errno < 0) {
-		op_errno = -op_errno;
-		goto err;
-	}
-	local = frame->local;
-	local->stub = fop_access_stub (frame, ha_access, loc, mask);
-        if (!local->stub) {
-                op_errno = ENOMEM;
-                gf_log (this->name, GF_LOG_ERROR, "out of memory");
-                goto err;
-        }
-
-	STACK_WIND_COOKIE (frame,
-			   ha_access_cbk,
-			   (void *)(long)local->active,
-			   HA_ACTIVE_CHILD(this, local),
-			   HA_ACTIVE_CHILD(this, local)->fops->access,
-			   loc,
-			   mask);
-	return 0;
-err:
-        local = frame->local;
-        frame->local = NULL;
-
-	STACK_UNWIND (frame, -1, op_errno);
-
-        ha_local_wipe (local);
-	return 0;
-}
-
-
- int32_t
-ha_readlink_cbk (call_frame_t *frame,
-		 void *cookie,
-		 xlator_t *this,
-		 int32_t op_ret,
-		 int32_t op_errno,
-		 const char *path,
-                 struct iatt *sbuf)
-{
-	int ret = -1;
-
-	ret = ha_handle_cbk (frame, cookie, op_ret, op_errno);
-
-	if (ret == 0) {
-		STACK_UNWIND (frame,
-			      op_ret,
-			      op_errno,
-			      path,
-                              sbuf);
-	}
-	return 0;
-}
-
-int32_t
-ha_readlink (call_frame_t *frame,
-	     xlator_t *this,
-	     loc_t *loc,
-	     size_t size)
-{
-	ha_local_t *local = frame->local;
-	int op_errno = 0;
-
-	op_errno = ha_alloc_init_inode (frame, loc->inode);
-	if (op_errno < 0) {
-		op_errno = -op_errno;
-		goto err;
-	}
-	local = frame->local;
-	local->stub = fop_readlink_stub (frame, ha_readlink, loc, size);
-
-	STACK_WIND_COOKIE (frame,
-			   ha_readlink_cbk,
-			   (void *)(long)local->active,
-			   HA_ACTIVE_CHILD(this, local),
-			   HA_ACTIVE_CHILD(this, local)->fops->readlink,
-			   loc,
-			   size);
-	return 0;
-err:
-	STACK_UNWIND (frame, -1, op_errno, NULL, NULL);
-	return 0;
-}
-
-int
-ha_mknod_lookup_cbk (call_frame_t *frame,
-		     void *cookie,
-		     xlator_t *this,
-		     int32_t op_ret,
-		     int32_t op_errno,
-		     inode_t *inode,
-		     struct iatt *buf,
-                     dict_t *dict,
-                     struct iatt *postparent)
-{
-	ha_local_t *local = NULL;
-	ha_private_t *pvt = NULL;
-	char *stateino = NULL;
-	int child_count = 0, i = 0, cnt = 0, ret = 0;
-	call_frame_t *prev_frame = NULL;
-	xlator_t **children = NULL;
-	uint64_t tmp_stateino = 0;
-
-	local = frame->local;
-	pvt = this->private;
-	child_count = pvt->child_count;
-	prev_frame = cookie;
-	children = pvt->children;
-
-	for (i = 0; i < child_count; i++)
-		if (prev_frame->this == children[i])
-			break;
-
-	if (op_ret == -1) {
-		gf_log (this->name, GF_LOG_ERROR, 
-			"(path=%s) (op_ret=%d op_errno=%d)", 
-			local->stub->args.mknod.loc.path, op_ret, op_errno);
-	}
-	ret = inode_ctx_get (local->stub->args.mknod.loc.inode, 
-			     this, &tmp_stateino);
-	stateino = (char *)(long)tmp_stateino;
-	if (ret != 0) {
-		gf_log (this->name, GF_LOG_ERROR, 
-			"unwind(-1), inode_ctx_get() error");
-		/* It is difficult to handle this error at this stage
-		 * as we still expect more cbks, we can't return as
-		 * of now
-		 */
-	} else if (op_ret == 0) {
-		stateino[i] = 1;
-	}
-	LOCK (&frame->lock);
-	cnt = --local->call_count;
-	UNLOCK (&frame->lock);
-
-	if (cnt == 0) {
-		call_stub_t *stub = local->stub;
-		GF_FREE (local->state);
-		STACK_UNWIND (frame,
-			      local->op_ret,
-			      local->op_errno,
-			      local->stub->args.mknod.loc.inode,
-			      &local->buf, &local->preparent,
-                              &local->postparent);
-		call_stub_destroy (stub);
-	}
-	return 0;
-}
-
-int32_t
-ha_mknod_cbk (call_frame_t *frame,
-	      void *cookie,
-	      xlator_t *this,
-	      int32_t op_ret,
-	      int32_t op_errno,
-	      inode_t *inode,
-              struct iatt *buf,
-              struct iatt *preparent,
-              struct iatt *postparent)
-{
-	ha_local_t *local = NULL;
-	ha_private_t *pvt = NULL;
-	char *stateino = NULL;
-	int child_count = 0, i = 0, cnt = 0, ret = 0;
-	call_frame_t *prev_frame = NULL;
-	xlator_t **children = NULL;
-	uint64_t tmp_stateino = 0;
-
-	local = frame->local;
-	pvt = this->private;
-	child_count = pvt->child_count;
-	prev_frame = cookie;
-	children = pvt->children;
-
-	for (i = 0; i < child_count; i++)
-		if (prev_frame->this == children[i])
-			break;
-
-	if (op_ret == -1) {
-		local->op_errno = op_errno;
-		gf_log (this->name, GF_LOG_ERROR, "(path=%s) (op_ret=%d op_errno=%d)", local->stub->args.mknod.loc.path, op_ret, op_errno);
-	}
-
-	ret = inode_ctx_get (local->stub->args.mknod.loc.inode, 
-			     this, &tmp_stateino);
-	stateino = (char *)(long)tmp_stateino;
-
-	if (ret != 0) {
-		gf_log (this->name, GF_LOG_ERROR, "inode_ctx_get() error");
-		/* FIXME: handle the case */
-	}
-	if (op_ret == 0) {
-		stateino[i] = 1;
-		local->op_ret = 0;
-		local->first_success = 1;
-		local->buf = *buf;
-                local->preparent = *preparent;
-                local->postparent = *postparent;
-	}
-	cnt = --local->call_count;
-	for (i = local->active + 1; i < child_count; i++) {
-		if (local->state[i])
-			break;
-	}
-
-	if (cnt == 0 || i == child_count) {
-		call_stub_t *stub = local->stub;
-		GF_FREE (local->state);
-		stub = local->stub;
-		STACK_UNWIND (frame, local->op_ret, local->op_errno,
-                              local->stub->args.mknod.loc.inode, &local->buf,
-                              &local->preparent, &local->postparent);
-		call_stub_destroy (stub);
-		return 0;
-	}
-
-	local->active = i;
-
-	if (local->first_success == 0) {
-		STACK_WIND (frame,
-			    ha_mknod_cbk,
-			    children[i],
-			    children[i]->fops->mknod,
-			    &local->stub->args.mknod.loc,
-			    local->stub->args.mknod.mode,
-			    local->stub->args.mknod.rdev);
-		return 0;
-	}
-	cnt = local->call_count;
-
-	for (; i < child_count; i++) {
-		if (local->state[i]) {
-			STACK_WIND (frame,
-				    ha_mknod_lookup_cbk,
-				    children[i],
-				    children[i]->fops->lookup,
-				    &local->stub->args.mknod.loc,
-				    0);
-			if (--cnt == 0)
-				break;
-		}
-	}
-	return 0;
-}
-
-int32_t
-ha_mknod (call_frame_t *frame,
-	  xlator_t *this,
-	  loc_t *loc,
-	  mode_t mode,
-	  dev_t rdev)
-{
-	ha_local_t *local = NULL;
-	ha_private_t *pvt = NULL;
-	int child_count = 0, i = 0;
-	char *stateino = NULL;
-        int32_t op_errno = EINVAL;
-
-	local = frame->local;
-	pvt = this->private;
-	child_count = pvt->child_count;
-
-	frame->local = local = GF_CALLOC (1, sizeof (*local), 
-                                          gf_ha_mt_ha_local_t);
-        if (!local) {
-                gf_log (this->name, GF_LOG_ERROR, "out of memory");
-                op_errno = ENOMEM;
-                goto err;
-        }
-
-	local->stub = fop_mknod_stub (frame, ha_mknod, loc, mode, rdev);
-        if (!local->stub) {
-                gf_log (this->name, GF_LOG_ERROR, "out of memory");
-                op_errno = ENOMEM;
-                goto err;
-        }
-
-	local->op_ret = -1;
-	local->op_errno = ENOTCONN;
-	local->state = GF_CALLOC (1, child_count, gf_ha_mt_char);
-        if (!local->state) {
-                gf_log (this->name, GF_LOG_ERROR, "out of memory");
-                op_errno = ENOMEM;
-                goto err;
-        }
-
-	memcpy (local->state, pvt->state, child_count);
-	local->active = -1;
-
-	stateino = GF_CALLOC (1, child_count, gf_ha_mt_char);
-        if (!stateino) {
-                gf_log (this->name, GF_LOG_ERROR, "out of memory");
-                op_errno = ENOMEM;
-                goto err;
-        }
-
-	inode_ctx_put (loc->inode, this, (uint64_t)(long)stateino);
-
-	for (i = 0; i < child_count; i++) {
-		if (local->state[i]) {
-			local->call_count++;
-			if (local->active == -1) 
-				local->active = i;
-		}
-	}
-
-	STACK_WIND (frame,
-		    ha_mknod_cbk,
-		    HA_ACTIVE_CHILD(this, local),
-		    HA_ACTIVE_CHILD(this, local)->fops->mknod,
-		    loc, mode, rdev);
-	return 0;
-
-err:
-        local = frame->local;
-        frame->local = NULL;
-
-	STACK_UNWIND (frame, -1, op_errno, NULL, NULL, NULL, NULL);
-        ha_local_wipe (local);
-        return 0;
-}
-
-
-int
-ha_mkdir_lookup_cbk (call_frame_t *frame,
-		     void *cookie,
-		     xlator_t *this,
-		     int32_t op_ret,
-		     int32_t op_errno,
-		     inode_t *inode,
-		     struct iatt *buf,
-                     dict_t *dict,
-                     struct iatt *postparent)
-{
-	ha_local_t *local = NULL;
-	ha_private_t *pvt = NULL;
-	char *stateino = NULL;
-	int child_count = 0, i = 0, cnt = 0;
-	call_frame_t *prev_frame = NULL;
-	xlator_t **children = NULL;
-	uint64_t tmp_stateino = 0;
-
-	local = frame->local;
-	pvt = this->private;
-	child_count = pvt->child_count;
-	prev_frame = cookie;
-	children = pvt->children;
-
-	for (i = 0; i < child_count; i++)
-		if (prev_frame->this == children[i])
-			break;
-
-	if (op_ret == -1) {
-		gf_log (this->name, GF_LOG_ERROR, "(path=%s) (op_ret=%d op_errno=%d)", local->stub->args.mkdir.loc.path, op_ret, op_errno);
-	}
-	inode_ctx_get (local->stub->args.mkdir.loc.inode, 
-		       this, &tmp_stateino);  
-	stateino = (char *)(long)tmp_stateino;
-
-	if (op_ret == 0)
-		stateino[i] = 1;
-
-	LOCK (&frame->lock);
-	cnt = --local->call_count;
-	UNLOCK (&frame->lock);
-
-	if (cnt == 0) {
-		call_stub_t *stub = local->stub;
-		GF_FREE (local->state);
-		STACK_UNWIND (frame,
-			      local->op_ret,
-			      local->op_errno,
-			      local->stub->args.mkdir.loc.inode, &local->buf,
-			      &local->preparent, &local->postparent);
-		call_stub_destroy (stub);
-	}
-	return 0;
-}
-
-int32_t
-ha_mkdir_cbk (call_frame_t *frame,
-	      void *cookie,
-	      xlator_t *this,
-	      int32_t op_ret,
-	      int32_t op_errno,
-	      inode_t *inode,
-              struct iatt *buf,
-              struct iatt *preparent,
-              struct iatt *postparent)
-{
-	ha_local_t *local = NULL;
-	ha_private_t *pvt = NULL;
-	char *stateino = NULL;
-	int child_count = 0, i = 0, cnt = 0;
-	call_frame_t *prev_frame = NULL;
-	xlator_t **children = NULL;
-	uint64_t tmp_stateino = 0;
-
-	local = frame->local;
-	pvt = this->private;
-	child_count = pvt->child_count;
-	prev_frame = cookie;
-	children = pvt->children;
-	
-	for (i = 0; i < child_count; i++)
-		if (prev_frame->this == children[i])
-			break;
-
-	if (op_ret == -1) {
-		local->op_errno = op_errno;
-		gf_log (this->name, GF_LOG_ERROR, "(path=%s) (op_ret=%d op_errno=%d)", local->stub->args.mkdir.loc.path, op_ret, op_errno);
-	}
-
-	inode_ctx_get (local->stub->args.mkdir.loc.inode, 
-		       this, &tmp_stateino);
-	stateino = (char *)(long)tmp_stateino;
-
-	if (op_ret == 0) {
-		stateino[i] = 1;
-		local->op_ret = 0;
-		local->first_success = 1;
-		local->buf = *buf;
-                local->preparent = *preparent;
-                local->postparent = *postparent;
-	}
-	cnt = --local->call_count;
-	for (i = local->active + 1; i < child_count; i++) {
-		if (local->state[i])
-			break;
-	}
-
-	if (cnt == 0 || i == child_count) {
-		call_stub_t *stub = local->stub;
-		GF_FREE (local->state);
-		stub = local->stub;
-		STACK_UNWIND (frame, local->op_ret, local->op_errno,
-                              local->stub->args.mkdir.loc.inode, &local->buf,
-                              &local->preparent, &local->postparent);
-		call_stub_destroy (stub);
-		return 0;
-	}
-
-	local->active = i;
-
-	if (local->first_success == 0) {
-		STACK_WIND (frame,
-			    ha_mkdir_cbk,
-			    children[i],
-			    children[i]->fops->mkdir,
-			    &local->stub->args.mkdir.loc,
-			    local->stub->args.mkdir.mode);
-		return 0;
-	}
-	cnt = local->call_count;
-
-	for (; i < child_count; i++) {
-		if (local->state[i]) {
-			STACK_WIND (frame,
-				    ha_mkdir_lookup_cbk,
-				    children[i],
-				    children[i]->fops->lookup,
-				    &local->stub->args.mkdir.loc,
-				    0);
-			if (--cnt == 0)
-				break;
-		}
-	}
-	return 0;
-}
-
-int32_t
-ha_mkdir (call_frame_t *frame,
-	  xlator_t *this,
-	  loc_t *loc,
-	  mode_t mode)
-{
-	ha_local_t *local = NULL;
-	ha_private_t *pvt = NULL;
-	int child_count = 0, i = 0;
-	char *stateino = NULL;
-        int32_t op_errno = EINVAL;
-
-	local = frame->local;
-	pvt = this->private;
-	child_count = pvt->child_count;
-
-	frame->local = local = GF_CALLOC (1, sizeof (*local), 
-                                          gf_ha_mt_ha_local_t);
-        if (!frame->local) {
-                gf_log (this->name, GF_LOG_ERROR, "out of memory");
-                op_errno = ENOMEM;
-                goto err;
-        }
-
-	local->stub = fop_mkdir_stub (frame, ha_mkdir, loc, mode);
-        if (!local->stub) {
-                gf_log (this->name, GF_LOG_ERROR, "out of memory");
-                op_errno = ENOMEM;
-                goto err;
-        }
-
-	local->op_ret = -1;
-	local->op_errno = ENOTCONN;
-	local->state = GF_CALLOC (1, child_count, gf_ha_mt_char);
-        if (!local->state) {
-                gf_log (this->name, GF_LOG_ERROR, "out of memory");
-                op_errno = ENOMEM;
-                goto err;
-        }
-
-	memcpy (local->state, pvt->state, child_count);
-	local->active = -1;
-
-	stateino = GF_CALLOC (1, child_count, gf_ha_mt_char);
-        if (!stateino) {
-                gf_log (this->name, GF_LOG_ERROR, "out of memory");
-                op_errno = ENOMEM;
-                goto err;
-        }
-
-	inode_ctx_put (loc->inode, this, (uint64_t)(long)stateino);
-	for (i = 0; i < child_count; i++) {
-		if (local->state[i]) {
-			local->call_count++;
-			if (local->active == -1)
-				local->active = i;
-		}
-	}
-
-	STACK_WIND (frame,
-		    ha_mkdir_cbk,
-		    HA_ACTIVE_CHILD(this, local),
-		    HA_ACTIVE_CHILD(this, local)->fops->mkdir,
-		    loc, mode);
-	return 0;
-err:
-        local = frame->local;
-        frame->local = NULL;
-
-	STACK_UNWIND (frame, -1, op_errno, NULL, NULL, NULL, NULL);
-        ha_local_wipe (local);
-        return 0;
-}
-
- int32_t
-ha_unlink_cbk (call_frame_t *frame,
-	       void *cookie,
-	       xlator_t *this,
-	       int32_t op_ret,
-	       int32_t op_errno,
-               struct iatt *preparent,
-               struct iatt *postparent)
-{
-	int ret = -1;
-
-	ret = ha_handle_cbk (frame, cookie, op_ret, op_errno);
-	if (ret == 0) {
-		STACK_UNWIND (frame, op_ret, op_errno, preparent, postparent);
-	}
-	return 0;
-}
-
-int32_t
-ha_unlink (call_frame_t *frame,
-	   xlator_t *this,
-	   loc_t *loc)
-{
-	ha_local_t *local = NULL;
-	int op_errno = 0;
-
-	op_errno = ha_alloc_init_inode (frame, loc->inode);
-
-	if (op_errno < 0) {
-		op_errno = -op_errno;
-		goto err;
-	}
-	local = frame->local;
-	local->stub = fop_unlink_stub (frame, ha_unlink, loc);
-        if (!local->stub) {
-                gf_log (this->name, GF_LOG_ERROR, "out of memory");
-                op_errno = ENOMEM;
-                goto err;
-        }
-
-	STACK_WIND_COOKIE (frame,
-			   ha_unlink_cbk,
-			   (void *)(long)local->active,
-			   HA_ACTIVE_CHILD(this, local),
-			   HA_ACTIVE_CHILD(this, local)->fops->unlink,
-			   loc);
-	return 0;
-err:
-	STACK_UNWIND (frame, -1, op_errno, NULL, NULL);
-	return 0;
-}
-
- int32_t
-ha_rmdir_cbk (call_frame_t *frame,
-	      void *cookie,
-	      xlator_t *this,
-	      int32_t op_ret,
-	      int32_t op_errno,
-              struct iatt *preparent,
-              struct iatt *postparent)
-{
-	int ret = -1;
-
-	ret = ha_handle_cbk (frame, cookie, op_ret, op_errno);
-
-	if (ret == 0) {
-		STACK_UNWIND (frame,
-			      op_ret,
-			      op_errno,
-                              preparent,
-                              postparent);
-	}
-	return 0;
-}
-
-int32_t
-ha_rmdir (call_frame_t *frame,
-	  xlator_t *this,
-	  loc_t *loc)
-{
-	ha_local_t *local = frame->local;
-	int op_errno = 0;
-
-	op_errno = ha_alloc_init_inode (frame, loc->inode);
-	if (op_errno < 0) {
-		op_errno = -op_errno;
-		goto err;
-	}
-	local = frame->local;
-	local->stub = fop_rmdir_stub (frame, ha_rmdir, loc);
-        if (!local->stub) {
-                op_errno = ENOMEM;
-                gf_log (this->name, GF_LOG_ERROR, "out of memory");
-                goto err;
-        }
-
-	STACK_WIND_COOKIE (frame,
-			   ha_rmdir_cbk,
-			   (void *)(long)local->active,
-			   HA_ACTIVE_CHILD(this, local),
-			   HA_ACTIVE_CHILD(this, local)->fops->rmdir,
-			   loc);
-	return 0;
-err:
-	STACK_UNWIND (frame, -1, op_errno, NULL, NULL);
-	return 0;
-}
-
-
-int
-ha_symlink_lookup_cbk (call_frame_t *frame,
-		       void *cookie,
-		       xlator_t *this,
-		       int32_t op_ret,
-		       int32_t op_errno,
-		       inode_t *inode,
-		       struct iatt *buf,
-                       dict_t *dict,
-                       struct iatt *postparent)
-{
-	ha_local_t *local = NULL;
-	ha_private_t *pvt = NULL;
-	char *stateino = NULL;
-	int child_count = 0, i = 0, cnt = 0;
-	call_frame_t *prev_frame = NULL;
-	xlator_t **children = NULL;
-	uint64_t tmp_stateino = 0;
-
-	local = frame->local;
-	pvt = this->private;
-	child_count = pvt->child_count;
-	prev_frame = cookie;
-	children = pvt->children;
-
-	for (i = 0; i < child_count; i++)
-		if (prev_frame->this == children[i])
-			break;
-
-	if (op_ret == -1) {
-		gf_log (this->name, GF_LOG_ERROR, "(path=%s) (op_ret=%d op_errno=%d)", local->stub->args.symlink.loc.path, op_ret, op_errno);
-	}
-	inode_ctx_get (local->stub->args.symlink.loc.inode,
-		       this, &tmp_stateino);  
-	stateino = (char *)(long)tmp_stateino;
-
-	if (op_ret == 0)
-		stateino[i] = 1;
-
-	LOCK (&frame->lock);
-	cnt = --local->call_count;
-	UNLOCK (&frame->lock);
-
-	if (cnt == 0) {
-		call_stub_t *stub = local->stub;
-		GF_FREE (local->state);
-		STACK_UNWIND (frame,
-			      local->op_ret,
-			      local->op_errno,
-			      local->stub->args.symlink.loc.inode, &local->buf,
-                              &local->preparent, &local->postparent);
-		call_stub_destroy (stub);
-	}
-	return 0;
-}
-
-int32_t
-ha_symlink_cbk (call_frame_t *frame,
-		void *cookie,
-		xlator_t *this,
-		int32_t op_ret,
-		int32_t op_errno,
-		inode_t *inode,
-                struct iatt *buf,
-                struct iatt *preparent,
-                struct iatt *postparent)
-{
-	ha_local_t *local = NULL;
-	ha_private_t *pvt = NULL;
-	char *stateino = NULL;
-	int child_count = 0, i = 0, cnt = 0;
-	call_frame_t *prev_frame = NULL;
-	xlator_t **children = NULL;
-	uint64_t tmp_stateino = 0;
-
-	local = frame->local;
-	pvt = this->private;
-	child_count = pvt->child_count;
-	prev_frame = cookie;
-	children = pvt->children;
-
-	for (i = 0; i < child_count; i++)
-		if (prev_frame->this == children[i])
-			break;
-
-	if (op_ret == -1) {
-		local->op_errno = op_errno;
-		gf_log (this->name, GF_LOG_ERROR, "(path=%s) (op_ret=%d op_errno=%d)", local->stub->args.symlink.loc.path, op_ret, op_errno);
-	}
-	inode_ctx_get (local->stub->args.symlink.loc.inode, 
-		       this, &tmp_stateino);
-	stateino = (char *)(long)tmp_stateino;
-
-	if (op_ret == 0) {
-		stateino[i] = 1;
-		local->op_ret = 0;
-		local->first_success = 1;
-		local->buf = *buf;
-                local->preparent = *preparent;
-                local->postparent = *postparent;
-	}
-	cnt = --local->call_count;
-	for (i = local->active + 1; i < child_count; i++) {
-		if (local->state[i])
-			break;
-	}
-
-	if (cnt == 0 || i == child_count) {
-		call_stub_t *stub = local->stub;
-		GF_FREE (local->state);
-		stub = local->stub;
-		STACK_UNWIND (frame, local->op_ret, local->op_errno, 
-			      local->stub->args.symlink.loc.inode, &local->buf,
-                              &local->preparent, &local->postparent);
-		call_stub_destroy (stub);
-		return 0;
-	}
-
-	local->active = i;
-
-	if (local->first_success == 0) {
-		STACK_WIND (frame,
-			    ha_symlink_cbk,
-			    children[i],
-			    children[i]->fops->symlink,
-			    local->stub->args.symlink.linkname,
-			    &local->stub->args.symlink.loc);
-		return 0;
-	}
-	cnt = local->call_count;
-
-	for (; i < child_count; i++) {
-		if (local->state[i]) {
-			STACK_WIND (frame,
-				    ha_symlink_lookup_cbk,
-				    children[i],
-				    children[i]->fops->lookup,
-				    &local->stub->args.symlink.loc,
-				    0);
-			if (--cnt == 0)
-				break;
-		}
-	}
-	return 0;
-}
-
-int32_t
-ha_symlink (call_frame_t *frame,
-	    xlator_t *this,
-	    const char *linkname,
-	    loc_t *loc)
-{
-	ha_local_t *local = NULL;
-	ha_private_t *pvt = NULL;
-	int child_count = 0, i = 0;
-	char *stateino = NULL;
-        int32_t op_errno = EINVAL;
-
-	local = frame->local;
-	pvt = this->private;
-	child_count = pvt->child_count;
-
-	frame->local = local = GF_CALLOC (1, sizeof (*local), 
-                                          gf_ha_mt_ha_local_t);
-        if (!local) {
-                op_errno = ENOMEM;
-                gf_log (this->name, GF_LOG_ERROR, "out of memory");
-                goto err;
-        }
-
-	local->stub = fop_symlink_stub (frame, ha_symlink, linkname, loc);
-        if (!local->stub) {
-                op_errno = ENOMEM;
-                gf_log (this->name, GF_LOG_ERROR, "out of memory");
-                goto err;
-        }
-
-	local->op_ret = -1;
-	local->op_errno = ENOTCONN;
-	local->state = GF_CALLOC (1, child_count, gf_ha_mt_char);
-        if (!local->state) {
-                op_errno = ENOMEM;
-                gf_log (this->name, GF_LOG_ERROR, "out of memory");
-                goto err;
-        }
-
-	memcpy (local->state, pvt->state, child_count);
-	local->active = -1;
-
-	stateino = GF_CALLOC (1, child_count, gf_ha_mt_char);
-        if (!stateino) {
-                op_errno = ENOMEM;
-                gf_log (this->name, GF_LOG_ERROR, "out of memory");
-                goto err;
-        }
-
-	inode_ctx_put (loc->inode, this, (uint64_t)(long)stateino);
-
-	for (i = 0; i < child_count; i++) {
-		if (local->state[i]) {
-			local->call_count++;
-			if (local->active == -1) {
-				local->active = i;
-			}
-		}
-	}
-
-	STACK_WIND (frame,
-		    ha_symlink_cbk,
-		    HA_ACTIVE_CHILD(this, local),
-		    HA_ACTIVE_CHILD(this, local)->fops->symlink,
-		    linkname, loc);
-	return 0;
-err:
-        local = frame->local;
-        frame->local = NULL;
-        STACK_UNWIND (frame, -1, op_errno, NULL, NULL, NULL, NULL);
-        ha_local_wipe (local);
-        return 0;
-}
-
- int32_t
-ha_rename_cbk (call_frame_t *frame,
-	       void *cookie,
-	       xlator_t *this,
-	       int32_t op_ret,
-	       int32_t op_errno,
-	       struct iatt *buf,
-               struct iatt *preoldparent,
-               struct iatt *postoldparent,
-               struct iatt *prenewparent,
-               struct iatt *postnewparent)
-{
-	int ret = -1;
-
-	ret = ha_handle_cbk (frame, cookie, op_ret, op_errno);
-
-	if (ret == 0) {
-		STACK_UNWIND (frame, op_ret, op_errno, buf, preoldparent,
-                              postoldparent, prenewparent, postnewparent);
-	}
-	return 0;
-}
-
-int32_t
-ha_rename (call_frame_t *frame,
-	   xlator_t *this,
-	   loc_t *oldloc,
-	   loc_t *newloc)
-{
-	ha_local_t *local = NULL;
-	int op_errno = 0;
-
-	op_errno = ha_alloc_init_inode (frame, oldloc->inode);
-	if (op_errno < 0) {
-		op_errno = -op_errno;
-		goto err;
-	}
-	local = frame->local;
-	local->stub = fop_rename_stub (frame, ha_rename, oldloc, newloc);
-        if (!local->stub) {
-                gf_log (this->name, GF_LOG_ERROR, "out of memory");
-                op_errno = ENOMEM;
-                goto err;
-        }
-
-	STACK_WIND_COOKIE (frame,
-			   ha_rename_cbk,
-			   (void *)(long)local->active,
-			   HA_ACTIVE_CHILD(this, local),
-			   HA_ACTIVE_CHILD(this, local)->fops->rename,
-			   oldloc, newloc);
-	return 0;
-err:
-	STACK_UNWIND (frame, -1, op_errno, NULL, NULL, NULL, NULL, NULL);
-	return 0;
-}
-
-int
-ha_link_lookup_cbk (call_frame_t *frame,
-		    void *cookie,
-		    xlator_t *this,
-		    int32_t op_ret,
-		    int32_t op_errno,
-		    inode_t *inode,
-		    struct iatt *buf,
-                    dict_t *dict,
-                    struct iatt *postparent)
-{
-	ha_local_t *local = NULL;
-	ha_private_t *pvt = NULL;
-	char *stateino = NULL;
-	int child_count = 0, i = 0, cnt = 0;
-	call_frame_t *prev_frame = NULL;
-	xlator_t **children = NULL;
-	uint64_t tmp_stateino = 0;
-
-	local = frame->local;
-	pvt = this->private;
-	child_count = pvt->child_count;
-	prev_frame = cookie;
-	children = pvt->children;
-
-	for (i = 0; i < child_count; i++)
-		if (prev_frame->this == children[i])
-			break;
-
-	if (op_ret == -1) {
-		gf_log (this->name, GF_LOG_ERROR, "(path=%s) (op_ret=%d op_errno=%d)", local->stub->args.link.newloc.path, op_ret, op_errno);
-	}
-	inode_ctx_get (local->stub->args.link.newloc.inode, 
-		       this, &tmp_stateino);  
-	stateino = (char *)(long)tmp_stateino;
-
-	if (op_ret == 0)
-		stateino[i] = 1;
-
-	LOCK (&frame->lock);
-	cnt = --local->call_count;
-	UNLOCK (&frame->lock);
-
-	if (cnt == 0) {
-		call_stub_t *stub = local->stub;
-		GF_FREE (local->state);
-		STACK_UNWIND (frame,
-			      local->op_ret,
-			      local->op_errno,
-			      local->stub->args.link.oldloc.inode, &local->buf,
-                              &local->preparent, &local->postparent);
-		call_stub_destroy (stub);
-	}
-	return 0;
-}
-
-int32_t
-ha_link_cbk (call_frame_t *frame,
-	     void *cookie,
-	     xlator_t *this,
-	     int32_t op_ret,
-	     int32_t op_errno,
-	     inode_t *inode,
-             struct iatt *buf,
-             struct iatt *preparent,
-             struct iatt *postparent)
-{
-	ha_local_t *local = NULL;
-	ha_private_t *pvt = NULL;
-	char *stateino = NULL;
-	int child_count = 0, i = 0, cnt = 0;
-	call_frame_t *prev_frame = NULL;
-	xlator_t **children = NULL;
-	uint64_t tmp_stateino = 0;
-
-	local = frame->local;
-	pvt = this->private;
-	child_count = pvt->child_count;
-	prev_frame = cookie;
-	children = pvt->children;
-
-	for (i = 0; i < child_count; i++)
-		if (prev_frame->this == children[i])
-			break;
-
-	if (op_ret == -1) {
-		local->op_errno = op_errno;
-		gf_log (this->name, GF_LOG_ERROR, "(path=%s) (op_ret=%d op_errno=%d)", local->stub->args.link.newloc.path, op_ret, op_errno);
-	}
-	inode_ctx_get (local->stub->args.link.newloc.inode, 
-		       this, &tmp_stateino);
-	stateino = (char *)(long)tmp_stateino;
-
-	if (op_ret == 0) {
-		stateino[i] = 1;
-		local->op_ret = 0;
-		local->first_success = 1;
-		local->buf = *buf;
-                local->preparent = *preparent;
-                local->postparent = *postparent;
-	}
-	cnt = --local->call_count;
-	for (i = local->active + 1; i < child_count; i++) {
-		if (local->state[i])
-			break;
-	}
-
-	if (cnt == 0 || i == child_count) {
-		call_stub_t *stub = local->stub;
-		GF_FREE (local->state);
-		stub = local->stub;
-		STACK_UNWIND (frame, local->op_ret, local->op_errno,
-                              local->stub->args.link.oldloc.inode, &local->buf,
-                              &local->preparent, &local->postparent);
-		call_stub_destroy (stub);
-		return 0;
-	}
-
-	local->active = i;
-
-	if (local->first_success == 0) {
-		STACK_WIND (frame,
-			    ha_link_cbk,
-			    children[i],
-			    children[i]->fops->link,
-			    &local->stub->args.link.oldloc,
-			    &local->stub->args.link.newloc);
-		return 0;
-	}
-	cnt = local->call_count;
-
-	for (; i < child_count; i++) {
-		if (local->state[i]) {
-			STACK_WIND (frame,
-				    ha_link_lookup_cbk,
-				    children[i],
-				    children[i]->fops->lookup,
-				    &local->stub->args.link.newloc,
-				    0);
-			if (--cnt == 0)
-				break;
-		}
-	}
-	return 0;
-}
-
-int32_t
-ha_link (call_frame_t *frame,
-	 xlator_t *this,
-	 loc_t *oldloc,
-	 loc_t *newloc)
-{
-	ha_local_t *local = NULL;
-	ha_private_t *pvt = NULL;
-	int child_count = 0, i = 0;
-	char *stateino = NULL;
-	int32_t ret = 0, op_errno = 0;
-	uint64_t tmp_stateino = 0;
-
-	ret = inode_ctx_get (newloc->inode, this, &tmp_stateino);
-	if (ret != 0) {
-		gf_log (this->name, GF_LOG_ERROR, "dict_ptr_error()");
-	}
-	stateino = (char *)(long)tmp_stateino;
-
-	if (stateino == NULL) {
-		gf_log (this->name, GF_LOG_ERROR, 
-			"newloc->inode's ctx is NULL, returning EINVAL");
-		STACK_UNWIND (frame, -1, EINVAL, oldloc->inode, NULL, NULL,
-                              NULL);
-		return 0;
-	}
-
-	local = frame->local;
-	pvt = this->private;
-	child_count = pvt->child_count;
-
-	frame->local = local = GF_CALLOC (1, sizeof (*local), 
-                                          gf_ha_mt_ha_local_t);
-        if (!frame->local) {
-                gf_log (this->name, GF_LOG_ERROR, "out of memory");
-                op_errno = ENOMEM;
-                goto err;
-        }
-
-	local->stub = fop_link_stub (frame, ha_link, oldloc, newloc);
-        if (!local->stub) {
-                gf_log (this->name, GF_LOG_ERROR, "out of memory");
-                op_errno = ENOMEM;
-                goto err;
-        }
-
-	local->op_ret = -1;
-	local->op_errno = ENOTCONN;
-	local->state = GF_CALLOC (1, child_count, gf_ha_mt_char);
-        if (!local->state) {
-                gf_log (this->name, GF_LOG_ERROR, "out of memory");
-                op_errno = ENOMEM;
-                goto err;
-        }
-
-	memcpy (local->state, pvt->state, child_count);
-	local->active = -1;
-
-	for (i = 0; i < child_count; i++) {
-		if (local->state[i]) {
-			local->call_count++;
-			if (local->active == -1)
-				local->active = i;
-		}
-	}
-
-	STACK_WIND (frame,
-		    ha_link_cbk,
-		    HA_ACTIVE_CHILD(this, local),
-		    HA_ACTIVE_CHILD(this, local)->fops->link,
-		    oldloc,
-		    newloc);
-	return 0;
-err:
-        local = frame->local;
-        frame->local = NULL;
-        STACK_UNWIND (frame, -1, op_errno, NULL, NULL, NULL, NULL);
-        return 0;
-}
-
-int32_t
-ha_create_cbk (call_frame_t *frame,
-	       void *cookie,
-	       xlator_t *this,
-	       int32_t op_ret,
-	       int32_t op_errno,
-	       fd_t *fd,
-	       inode_t *inode,
-	       struct iatt *buf,
-               struct iatt *preparent,
-               struct iatt *postparent)
-{
-	ha_local_t *local = NULL;
-	ha_private_t *pvt = NULL;
-	int i, child_count = 0, cnt = 0, ret = 0;
-	char *stateino = NULL;
-	hafd_t *hafdp = NULL;
-	call_frame_t *prev_frame = NULL;
-	xlator_t **children = NULL;
-	uint64_t tmp_stateino = 0;
-	uint64_t tmp_hafdp = 0;
-
-	local = frame->local;
-	pvt = this->private;
-	child_count = pvt->child_count;
-	prev_frame = cookie;
-	children = pvt->children;
-
-	ret = inode_ctx_get (local->stub->args.create.loc.inode, 
-			     this, &tmp_stateino);
-	stateino = (char *)(long)tmp_stateino;
-
-	if (ret != 0) {
-		gf_log (this->name, GF_LOG_ERROR, "dict_to_ptr() error");
-		/* FIXME: handle */
-	}
-	ret = fd_ctx_get (local->stub->args.create.fd, this, &tmp_hafdp);
-	if (ret != 0) {
-		gf_log (this->name, GF_LOG_ERROR, "dict_to_ptr() error");
-		/* FIXME: handle */
-	}
-	hafdp = (hafd_t *)(long)tmp_hafdp;
-
-	for (i = 0; i < child_count; i++) {
-		if (prev_frame->this == children[i])
-			break;
-	}
-
-	if (op_ret == -1) {
-		local->op_errno = op_errno;
-		gf_log (this->name, GF_LOG_ERROR, "(path=%s) (op_ret=%d op_errno=%d)", local->stub->args.create.loc.path, op_ret, op_errno);
-	}
-	if (op_ret != -1) {
-		stateino[i] = 1;
-		hafdp->fdstate[i] = 1;
-		if (local->op_ret == -1) {
-			local->op_ret = 0;
-			local->buf = *buf;
-                        local->preparent = *preparent;
-                        local->postparent = *postparent;
-			local->first_success = 1;
-		}
-		local->stub->args.create.flags &= (~O_EXCL);
-	}
-	LOCK (&frame->lock);
-	cnt = --local->call_count;
-	UNLOCK (&frame->lock);
-
-	for (i = local->active + 1; i < child_count; i++) {
-		if (local->state[i])
-			break;
-	}
-
-	if (cnt == 0 || i == child_count) {
-		char *state = local->state;
-		call_stub_t *stub = local->stub;
-		STACK_UNWIND (frame, local->op_ret, local->op_errno,
-			      stub->args.create.fd,
-			      stub->args.create.loc.inode, &local->buf,
-                              &local->preparent, &local->postparent);
-		GF_FREE (state);
-		call_stub_destroy (stub);
-		return 0;
-	}
-	local->active = i;
-	cnt = local->call_count;
-	for (; i < child_count; i++) {
-		if (local->state[i]) {
-			STACK_WIND (frame,
-				    ha_create_cbk,
-				    children[i],
-				    children[i]->fops->create,
-				    &local->stub->args.create.loc,
-				    local->stub->args.create.flags,
-				    local->stub->args.create.mode,
-				    local->stub->args.create.fd);
-			if ((local->first_success == 0) || (cnt == 0))
-				break;
-		}
-	}
-	return 0;
-}
-
-int32_t
-ha_create (call_frame_t *frame,
-	   xlator_t *this,
-	   loc_t *loc,
-	   int32_t flags,
-	   mode_t mode, fd_t *fd)
-{
-	ha_local_t *local = NULL;
-	ha_private_t *pvt = NULL;
-	int i, child_count = 0;
-	char *stateino = NULL;
-	xlator_t **children = NULL;
-	hafd_t *hafdp = NULL;
-        int32_t op_errno = EINVAL;
-
-	local = frame->local;
-	pvt = this->private;
-	child_count = pvt->child_count;
-	children = pvt->children;
-
-	if (local == NULL) {
-                frame->local = local = GF_CALLOC (1, sizeof (*local), 
-                                                  gf_ha_mt_ha_local_t);
-                if (!local) {
-                        op_errno = ENOMEM;
-                        gf_log (this->name, GF_LOG_ERROR, "out of memory");
-                        goto err;
-                }
-
-		local->stub = fop_create_stub (frame, ha_create, loc, flags, mode, fd);
-                if (!local->stub) {
-                        op_errno = ENOMEM;
-                        gf_log (this->name, GF_LOG_ERROR, "out of memory");
-                        goto err;
-                }
-
-                local->state = GF_CALLOC (1, child_count, gf_ha_mt_char);
-                if (!local->state) {
-                        op_errno = ENOMEM;
-                        gf_log (this->name, GF_LOG_ERROR, "out of memory");
-                        goto err;
-                }
-
-		local->active = -1;
-		local->op_ret = -1;
-		local->op_errno = ENOTCONN;
-		memcpy (local->state, pvt->state, child_count);
-
-		for (i = 0; i < pvt->child_count; i++) {
-			if (local->state[i]) {
-				local->call_count++;
-				if (local->active == -1)
-					local->active = i;
-			}
-		}
-		/* FIXME handle active -1 */
-		stateino = GF_CALLOC (1, child_count, gf_ha_mt_char);
-                if (!stateino) {
-                        op_errno = ENOMEM;
-                        gf_log (this->name, GF_LOG_ERROR, "out of memory");
-                        goto err;
-                }
-
-		hafdp = GF_CALLOC (1, sizeof (*hafdp), gf_ha_mt_hafd_t);
-                if (!hafdp) {
-                        op_errno = ENOMEM;
-                        gf_log (this->name, GF_LOG_ERROR, "out of memory");
-                        goto err;
-                }
-
-		hafdp->fdstate = GF_CALLOC (1, child_count, gf_ha_mt_char);
-                if (!hafdp->fdstate) {
-                        op_errno = ENOMEM;
-                        gf_log (this->name, GF_LOG_ERROR, "out of memory");
-                        goto err;
-                }
-
-		hafdp->path = gf_strdup(loc->path);
-                if (!hafdp->path) {
-                        op_errno = ENOMEM;
-                        gf_log (this->name, GF_LOG_ERROR, "out of memory");
-                        goto err;
-                }
-
-		LOCK_INIT (&hafdp->lock);
-		fd_ctx_set (fd, this, (uint64_t)(long)hafdp);
-		inode_ctx_put (loc->inode, this, (uint64_t)(long)stateino);
-	}
-
-	STACK_WIND (frame,
-		    ha_create_cbk,
-		    children[local->active],
-		    children[local->active]->fops->create,
-		    loc, flags, mode, fd);
-	return 0;
-err:
-        local = frame->local;
-        frame->local = NULL;
-        STACK_UNWIND (frame, -1, op_errno, NULL, NULL, NULL, NULL, NULL);
-        ha_local_wipe (local);
-
-        if (stateino) {
-                GF_FREE (stateino);
-                stateino = NULL;
-        }
-
-        if (hafdp) {
-                GF_FREE (hafdp->fdstate);
-
-                GF_FREE (hafdp->path);
-
-                GF_FREE (hafdp);
-        }
-
-        return 0;
-}
-
- int32_t
-ha_open_cbk (call_frame_t *frame,
-	     void *cookie,
-	     xlator_t *this,
-	     int32_t op_ret,
-	     int32_t op_errno,
-	     fd_t *fd)
-{
-	ha_local_t *local = NULL;
-	ha_private_t *pvt = NULL;
-	xlator_t **children = NULL;
-	int i = 0, child_count = 0, callcnt = 0, ret = 0;
-	call_frame_t *prev_frame = NULL;
-	hafd_t *hafdp = NULL;
-	uint64_t tmp_hafdp = 0;
-
-	local = frame->local;
-	pvt = this->private;
-	children = pvt->children;
-	child_count = pvt->child_count;
-	prev_frame = cookie;
-
-	ret = fd_ctx_get (local->fd, this, &tmp_hafdp);
-	if (ret != 0) {
-		gf_log (this->name, GF_LOG_ERROR, "dict_ptr_error()");
-	}
-	hafdp = (hafd_t *)(long)tmp_hafdp;
-
-	for (i = 0; i < child_count; i++)
-		if (children[i] == prev_frame->this)
-			break;
-	LOCK (&frame->lock);
-	if (op_ret != -1) {
-		hafdp->fdstate[i] = 1;
-		local->op_ret = 0;
-	}
-	if (op_ret == -1 && op_errno != ENOTCONN)
-		local->op_errno = op_errno;
-	callcnt = --local->call_count;
-	UNLOCK (&frame->lock);
-
-	if (callcnt == 0) {
-		STACK_UNWIND (frame,
-			      local->op_ret,
-			      local->op_errno,
-			      local->fd);
-	}
-	return 0;
-}
-
-int32_t
-ha_open (call_frame_t *frame,
-	 xlator_t *this,
-	 loc_t *loc,
-	 int32_t flags, fd_t *fd, int wbflags)
-{
-	ha_local_t *local = NULL;
-	ha_private_t *pvt = NULL;
-	char *stateino = NULL;
-	xlator_t **children = NULL;
-	int cnt = 0, i, child_count = 0, ret = 0;
-	hafd_t *hafdp = NULL;
-	uint64_t tmp_stateino = 0;
-        int32_t op_errno = ENOMEM;
-
-	local = frame->local;
-	pvt = this->private;
-	children = pvt->children;
-	child_count = pvt->child_count;
-
-
-	frame->local = local = GF_CALLOC (1, sizeof (*local), 
-                                          gf_ha_mt_ha_local_t);
-        if (!local) {
-                op_errno = ENOMEM;
-                gf_log (this->name, GF_LOG_ERROR, "out of memory");
-                goto err;
-        }
-                
-	local->op_ret = -1;
-	local->op_errno = ENOTCONN;
-	local->fd = fd;
-
-	hafdp = GF_CALLOC (1, sizeof (*hafdp), gf_ha_mt_hafd_t);
-        if (!hafdp) {
-                op_errno = ENOMEM;
-                gf_log (this->name, GF_LOG_ERROR, "out of memory");
-                goto err;
-        }
-
-	hafdp->fdstate = GF_CALLOC (1, child_count, gf_ha_mt_char);
-        if (!hafdp->fdstate) {
-                op_errno = ENOMEM;
-                gf_log (this->name, GF_LOG_ERROR, "out of memory");
-                goto err;
-        }
-
-	hafdp->path = gf_strdup (loc->path);
-        if (!hafdp->path) {
-                op_errno = ENOMEM;
-                gf_log (this->name, GF_LOG_ERROR, "out of memory");
-                goto err;
-        }
-
-	hafdp->active = -1;
-	if (pvt->pref_subvol == -1) {
-		hafdp->active = fd->inode->ino % child_count;
-	}
-
-	LOCK_INIT (&hafdp->lock);
-	fd_ctx_set (fd, this, (uint64_t)(long)hafdp);
-	ret = inode_ctx_get (loc->inode, this, &tmp_stateino);
-	stateino = (char *)(long)tmp_stateino;
-
-	for (i = 0; i < child_count; i++)
-		if (stateino[i])
-			cnt++;
-	local->call_count = cnt;
-	for (i = 0; i < child_count; i++) {
-		if (stateino[i]) {
-			STACK_WIND (frame,
-				    ha_open_cbk,
-				    children[i],
-				    children[i]->fops->open,
-				    loc, flags, fd, wbflags);
-			if (--cnt == 0)
-				break;
-		}
-	}
-	return 0;
-err:
-        local = frame->local;
-        frame->local = NULL;
-
-	STACK_UNWIND (frame, -1, op_errno, fd);
-        if (hafdp) {
-                if (hafdp->fdstate) {
-                        GF_FREE (hafdp->fdstate);
-                        hafdp->fdstate = NULL;
-                }
-
-                if (hafdp->path) {
-                        GF_FREE (hafdp->path);
-                        hafdp->path = NULL;
-                }
-
-                GF_FREE (hafdp);
-        }
-
-        ha_local_wipe (local);
-        return 0;
-}
-
- int32_t
-ha_readv_cbk (call_frame_t *frame,
-	      void *cookie,
-	      xlator_t *this,
-	      int32_t op_ret,
-	      int32_t op_errno,
-	      struct iovec *vector,
-	      int32_t count,
-	      struct iatt *stbuf,
-              struct iobref *iobref)
-{
-	int ret = 0;
-
-	ret = ha_handle_cbk (frame, cookie, op_ret, op_errno);
-
-	if (ret == 0) {
-		STACK_UNWIND (frame,
-			      op_ret,
-			      op_errno,
-			      vector,
-			      count,
-			      stbuf,
-                              iobref);
-	}
-	return 0;
-}
-
-int32_t
-ha_readv (call_frame_t *frame,
-	  xlator_t *this,
-	  fd_t *fd,
-	  size_t size,
-	  off_t offset)
-{
-	ha_local_t *local = NULL;
-	int op_errno = 0;
-
-	op_errno = ha_alloc_init_fd (frame, fd);
-	if (op_errno < 0) {
-		op_errno = -op_errno;
-		goto err;
-	}
-	local = frame->local;
-	local->stub = fop_readv_stub (frame, ha_readv, fd, size, offset);
-        if (!local->stub) {
-                op_errno = ENOMEM;
-                gf_log (this->name, GF_LOG_ERROR, "out of memory");
-                goto err;
-        }
-
-	STACK_WIND_COOKIE (frame,
-			   ha_readv_cbk,
-			   (void *)(long)local->active,
-			   HA_ACTIVE_CHILD(this, local),
-			   HA_ACTIVE_CHILD(this, local)->fops->readv,
-			   fd,
-			   size,
-			   offset);
-	return 0;
-err:
-        local = frame->local;
-        frame->local = NULL;
-
-	STACK_UNWIND (frame, -1, op_errno, NULL, 0, NULL, NULL);
-
-        ha_local_wipe (local);
-	return 0;
-}
-
- int32_t
-ha_writev_cbk (call_frame_t *frame,
-	       void *cookie,
-	       xlator_t *this,
-	       int32_t op_ret,
-	       int32_t op_errno,
-               struct iatt *prebuf,
-	       struct iatt *postbuf)
-{
-	int ret = 0;
-	ret = ha_handle_cbk (frame, cookie, op_ret, op_errno);
-
-	if (ret == 0) {
-		STACK_UNWIND (frame,
-			      op_ret,
-			      op_errno,
-                              prebuf,
-			      postbuf);
-	}
-	return 0;
-}
-
-int32_t
-ha_writev (call_frame_t *frame,
-	   xlator_t *this,
-	   fd_t *fd,
-	   struct iovec *vector,
-	   int32_t count,
-	   off_t off,
-           struct iobref *iobref)
-{
-	ha_local_t *local = NULL;
-	int op_errno = 0;
-
-	op_errno = ha_alloc_init_fd (frame, fd);
-	if (op_errno < 0) {
-		op_errno = -op_errno;
-		goto err;
-	}
-	local = frame->local;
-	local->stub = fop_writev_stub (frame, ha_writev, fd, vector, count, off,
-                                       iobref);
-        if (!local->stub) {
-                op_errno = ENOMEM;
-                gf_log (this->name, GF_LOG_ERROR, "out of memory");
-                goto err;
-        }
-
-	STACK_WIND_COOKIE (frame,
-			   ha_writev_cbk,
-			   (void *)(long)local->active,
-			   HA_ACTIVE_CHILD(this, local),
-			   HA_ACTIVE_CHILD(this, local)->fops->writev,
-			   fd,
-			   vector,
-			   count,
-			   off,
-                           iobref);
-	return 0;
-err:
-        local = frame->local;
-        frame->local = NULL;
-
-	STACK_UNWIND (frame, -1, op_errno, NULL, NULL);
-
-        ha_local_wipe (local);
-	return 0;	
-}
-
- int32_t
-ha_flush_cbk (call_frame_t *frame,
-	      void *cookie,
-	      xlator_t *this,
-	      int32_t op_ret,
-	      int32_t op_errno)
-{
-	int ret = 0;
-	ret = ha_handle_cbk (frame, cookie, op_ret, op_errno);
-
-	if (ret == 0) {
-		STACK_UNWIND (frame,
-			      op_ret,
-			      op_errno);
-	}
-	return 0;
-}
-
-int32_t
-ha_flush (call_frame_t *frame,
-	  xlator_t *this,
-	  fd_t *fd)
-{
-	ha_local_t *local = NULL;
-	int op_errno = 0;
-
-	op_errno = ha_alloc_init_fd (frame, fd);
-	if (op_errno < 0) {
-		op_errno = -op_errno;
-		goto err;
-	}
-	local = frame->local;
-	local->stub = fop_flush_stub (frame, ha_flush, fd);
-        if (!local->stub) {
-                op_errno = ENOMEM;
-                gf_log (this->name, GF_LOG_ERROR, "out of memory");
-                goto err;
-        }
-                
-	STACK_WIND_COOKIE (frame,
-			   ha_flush_cbk,
-			   (void *)(long)local->active,
-			   HA_ACTIVE_CHILD(this, local),
-			   HA_ACTIVE_CHILD(this, local)->fops->flush,
-			   fd);
-	return 0;
-err:
-        local = frame->local;
-        frame->local = NULL;
-
-	STACK_UNWIND (frame, -1, op_errno);
-
-        ha_local_wipe (local);
-	return 0;
-}
-
-
- int32_t
-ha_fsync_cbk (call_frame_t *frame,
-	      void *cookie,
-	      xlator_t *this,
-	      int32_t op_ret,
-	      int32_t op_errno,
-              struct iatt *prebuf,
-              struct iatt *postbuf)
-{
-	int ret = 0;
-	ret = ha_handle_cbk (frame, cookie, op_ret, op_errno);
-
-	if (ret == 0) {
-		STACK_UNWIND (frame,
-			      op_ret,
-			      op_errno);
-	}
-	return 0;
-}
-
-int32_t
-ha_fsync (call_frame_t *frame,
-	  xlator_t *this,
-	  fd_t *fd,
-	  int32_t flags)
-{
-	ha_local_t *local = NULL;
-	int op_errno = 0;
-
-	op_errno = ha_alloc_init_fd (frame, fd);
-	if (op_errno < 0) {
-		op_errno = -op_errno;
-		goto err;
-	}
-	local = frame->local;
-	local->stub = fop_fsync_stub (frame, ha_fsync, fd, flags);
-        if (!local->stub) {
-                gf_log (this->name, GF_LOG_ERROR, "out of memory");
-                op_errno = ENOMEM;
-                goto err;
-        }
-
-	STACK_WIND_COOKIE (frame,
-			   ha_fsync_cbk,
-			   (void *)(long)local->active,
-			   HA_ACTIVE_CHILD(this, local),
-			   HA_ACTIVE_CHILD(this, local)->fops->fsync,
-			   fd,
-			   flags);
-	return 0;
-err:
-        local = frame->local;
-        frame->local = NULL;
-
-	STACK_UNWIND (frame, -1, op_errno);
-
-        ha_local_wipe (local);
-	return 0;
-}
-
- int32_t
-ha_fstat_cbk (call_frame_t *frame,
-	      void *cookie,
-	      xlator_t *this,
-	      int32_t op_ret,
-	      int32_t op_errno,
-	      struct iatt *buf)
-{
-	int ret = 0;
-
-	ret = ha_handle_cbk (frame, cookie, op_ret, op_errno);
-
-	if (ret == 0) {
-		STACK_UNWIND (frame,
-			      op_ret,
-			      op_errno,
-			      buf);
-	}
-	return 0;
-}
-
-int32_t
-ha_fstat (call_frame_t *frame,
-	  xlator_t *this,
-	  fd_t *fd)
-{
-	ha_local_t *local = NULL;
-	int op_errno = 0;
-
-	op_errno = ha_alloc_init_fd (frame, fd);
-
-	if (op_errno < 0) {
-		op_errno = -op_errno;
-		goto err;
-	}
-	local = frame->local;
-	local->stub = fop_fstat_stub (frame, ha_fstat, fd);
-        if (!local->stub) {
-                op_errno = ENOMEM;
-                gf_log (this->name, GF_LOG_ERROR, "out of memory");
-                goto err;
-        }
-
-	STACK_WIND_COOKIE (frame,
-			   ha_fstat_cbk,
-			   (void *)(long)local->active,
-			   HA_ACTIVE_CHILD(this, local),
-			   HA_ACTIVE_CHILD(this, local)->fops->fstat,
-			   fd);
-	return 0;
-err:
-        local = frame->local;
-        frame->local = NULL;
-
-	STACK_UNWIND (frame, -1, op_errno, NULL);
-
-        ha_local_wipe (local);
-	return 0;
-}
-
-int32_t
-ha_opendir_cbk (call_frame_t *frame,
-		void *cookie,
-		xlator_t *this,
-		int32_t op_ret,
-		int32_t op_errno,
-		fd_t *fd)
-{
-	ha_local_t *local = NULL;
-	ha_private_t *pvt = NULL;
-	xlator_t **children = NULL;
-	int i = 0, child_count = 0, callcnt = 0, ret = 0;
-	call_frame_t *prev_frame = NULL;
-	hafd_t *hafdp = NULL;
-	uint64_t tmp_hafdp = 0;
-
-	local = frame->local;
-	pvt = this->private;
-	children = pvt->children;
-	child_count = pvt->child_count;
-	prev_frame = cookie;
-
-	ret = fd_ctx_get (local->fd, this, &tmp_hafdp);
-	if (ret != 0) {
-		gf_log (this->name, GF_LOG_ERROR, "dict_ptr_error()");
-	}
-	hafdp = (hafd_t *)(long)tmp_hafdp;
-
-	for (i = 0; i < child_count; i++)
-		if (children[i] == prev_frame->this)
-			break;
-	LOCK (&frame->lock);
-	if (op_ret != -1) {
-		hafdp->fdstate[i] = 1;
-		local->op_ret = 0;
-	}
-	if (op_ret == -1 && op_errno != ENOTCONN)
-		local->op_errno = op_errno;
-	callcnt = --local->call_count;
-	UNLOCK (&frame->lock);
-
-	if (callcnt == 0) {
-		STACK_UNWIND (frame,
-			      local->op_ret,
-			      local->op_errno,
-			      local->fd);
-	}
-	return 0;
-}
-
-int32_t
-ha_opendir (call_frame_t *frame,
-	    xlator_t *this,
-	    loc_t *loc, fd_t *fd)
-{
-	ha_local_t *local = NULL;
-	ha_private_t *pvt = NULL;
-	char *stateino = NULL;
-	xlator_t **children = NULL;
-	int cnt = 0, i, child_count = 0, ret = 0;
-	hafd_t *hafdp = NULL;
-	uint64_t tmp_stateino = 0;
-        int32_t op_errno = EINVAL;
-
-	local = frame->local;
-	pvt = this->private;
-	children = pvt->children;
-	child_count = pvt->child_count;
-
-	frame->local = local = GF_CALLOC (1, sizeof (*local), 
-                                          gf_ha_mt_ha_local_t);
-        if (!local) {
-                op_errno = ENOMEM;
-                gf_log (this->name, GF_LOG_ERROR, "out of memory");
-                goto err;
-        }
-
-	local->op_ret = -1;
-	local->op_errno = ENOTCONN;
-	local->fd = fd;
-
-	hafdp = GF_CALLOC (1, sizeof (*hafdp), gf_ha_mt_hafd_t);
-        if (!hafdp) {
-                op_errno = ENOMEM;
-                gf_log (this->name, GF_LOG_ERROR, "out of memory");
-                goto err;
-        }
-
-	hafdp->fdstate = GF_CALLOC (1, child_count, gf_ha_mt_char);
-        if (!hafdp->fdstate) {
-                op_errno = ENOMEM;
-                gf_log (this->name, GF_LOG_ERROR, "out of memory");
-                goto err;
-        }
-
-	hafdp->path = gf_strdup (loc->path);
-        if (!hafdp->path) {
-                op_errno = ENOMEM;
-                gf_log (this->name, GF_LOG_ERROR, "out of memory");
-                goto err;
-        }
-
-	LOCK_INIT (&hafdp->lock);
-	fd_ctx_set (fd, this, (uint64_t)(long)hafdp);
-	ret = inode_ctx_get (loc->inode, this, &tmp_stateino);
-	stateino = (char *)(long)tmp_stateino;
-	
-	if (ret != 0) {
-		gf_log (this->name, GF_LOG_ERROR, "inode_ctx_get() error");
-	}
-	for (i = 0; i < child_count; i++)
-		if (stateino[i])
-			cnt++;
-	local->call_count = cnt;
-	for (i = 0; i < child_count; i++) {
-		if (stateino[i]) {
-			STACK_WIND (frame,
-				    ha_opendir_cbk,
-				    children[i],
-				    children[i]->fops->opendir,
-				    loc, fd);
-			if (--cnt == 0)
-				break;
-		}
-	}
-	return 0;
-err:
-        local = frame->local;
-        frame->local = NULL;
-
-	STACK_UNWIND (frame, -1, op_errno, NULL);
-        ha_local_wipe (local);
-        if (hafdp) {
-                if (hafdp->fdstate) {
-                        GF_FREE (hafdp->fdstate);
-                        hafdp->fdstate = NULL;
-                }
-
-                if (hafdp->path) {
-                        GF_FREE (hafdp->path);
-                        hafdp->path = NULL;
-                }
-
-                GF_FREE (hafdp);
-        }
-        return 0;
-}
-
- int32_t
-ha_getdents_cbk (call_frame_t *frame,
-		 void *cookie,
-		 xlator_t *this,
-		 int32_t op_ret,
-		 int32_t op_errno,
-		 dir_entry_t *entries,
-		 int32_t count)
-{
-	int ret = 0;
-
-	ret = ha_handle_cbk (frame, cookie, op_ret, op_errno);
-	if (ret == 0) {
-		STACK_UNWIND (frame,
-			      op_ret,
-			      op_errno,
-			      entries,
-			      count);
-	}
-	return 0;
-}
-
-int32_t
-ha_getdents (call_frame_t *frame,
-	     xlator_t *this,
-	     fd_t *fd,
-	     size_t size,
-	     off_t offset,
-	     int32_t flag)
-{
-	ha_local_t *local = NULL;
-	int op_errno = 0;
-
-	op_errno = ha_alloc_init_fd (frame, fd);
-	if (op_errno < 0) {
-		op_errno = -op_errno;
-		goto err;
-	}
-	local = frame->local;
-	local->stub = fop_getdents_stub (frame, ha_getdents, fd, size, offset,
-                                         flag);
-        if (!local->stub) {
-                op_errno = ENOMEM;
-                gf_log (this->name, GF_LOG_ERROR, "out of memory");
-                goto err;
-        }
-
-	STACK_WIND_COOKIE (frame,
-			   ha_getdents_cbk,
-			   (void *)(long)local->active,
-			   HA_ACTIVE_CHILD(this, local),
-			   HA_ACTIVE_CHILD(this, local)->fops->getdents,
-			   fd,
-			   size,
-			   offset,
-			   flag);
-	return 0;
-err:
-        local = frame->local;
-        frame->local = NULL;
-
-	STACK_UNWIND (frame, -1, op_errno, NULL, 0);
-
-        ha_local_wipe (local);
-	return 0;
-}
-
- int32_t
-ha_setdents_cbk (call_frame_t *frame,
-		 void *cookie,
-		 xlator_t *this,
-		 int32_t op_ret,
-		 int32_t op_errno)
-{
-	int ret = 0;
-
-	ret = ha_handle_cbk (frame, cookie, op_ret, op_errno);
-
-	if (ret == 0) {
-		STACK_UNWIND (frame,
-			      op_ret,
-			      op_errno);
-	}
-	return 0;
-}
-
-int32_t
-ha_setdents (call_frame_t *frame,
-	     xlator_t *this,
-	     fd_t *fd,
-	     int32_t flags,
-	     dir_entry_t *entries,
-	     int32_t count)
-{
-	ha_local_t *local = NULL;
-	int op_errno = 0;
-
-	op_errno = ha_alloc_init_fd (frame, fd);
-	if (op_errno < 0) {
-		op_errno = -op_errno;
-		goto err;
-	}
-	local = frame->local;
-
-	local->stub = fop_setdents_stub (frame, ha_setdents, fd, flags, entries,
-                                         count);
-        if (!local->stub) {
-                op_errno = ENOMEM;
-                gf_log (this->name, GF_LOG_ERROR, "out of memory");
-                goto err;
-        }
-
-	STACK_WIND_COOKIE (frame,
-			   ha_setdents_cbk,
-			   (void *)(long)local->active,
-			   HA_ACTIVE_CHILD(this, local),
-			   HA_ACTIVE_CHILD(this, local)->fops->setdents,
-			   fd,
-			   flags,
-			   entries,
-			   count);
-	return 0;
-err:
-        local = frame->local;
-        frame->local = NULL;
-
-	STACK_UNWIND (frame, -1, op_errno);
-
-        ha_local_wipe (local);
-	return 0;
-}
-
- int32_t
-ha_fsyncdir_cbk (call_frame_t *frame,
-		 void *cookie,
-		 xlator_t *this,
-		 int32_t op_ret,
-		 int32_t op_errno)
-{
-	int ret = 0;
-
-	ret = ha_handle_cbk (frame, cookie, op_ret, op_errno);
-	if (ret == 0) {
-		STACK_UNWIND (frame,
-			      op_ret,
-			      op_errno);
-	}
-	return 0;
-}
-
-int32_t
-ha_fsyncdir (call_frame_t *frame,
-	     xlator_t *this,
-	     fd_t *fd,
-	     int32_t flags)
-{
-	ha_local_t *local = NULL;
-	int op_errno = 0;
-
-	op_errno = ha_alloc_init_fd (frame, fd);
-	if (op_errno < 0) {
-		op_errno = -op_errno;
-		goto err;
-	}
-	local = frame->local;
-	local->stub = fop_fsyncdir_stub (frame, ha_fsyncdir, fd, flags);
-        if (!local->stub) {
-                op_errno = ENOMEM;
-                gf_log (this->name, GF_LOG_ERROR, "out of memory");
-                goto err;
-        }
-
-	STACK_WIND_COOKIE (frame,
-			   ha_fsyncdir_cbk,
-			   (void *)(long)local->active,
-			   HA_ACTIVE_CHILD(this, local),
-			   HA_ACTIVE_CHILD(this, local)->fops->fsyncdir,
-			   fd,
-			   flags);
-	return 0;
-err:
-        local = frame->local;
-        frame->local = NULL;
- 
-	STACK_UNWIND (frame, -1, op_errno, NULL, NULL);
-
-        ha_local_wipe (local);
-	return 0;
-}
-
-
- int32_t
-ha_statfs_cbk (call_frame_t *frame,
-	       void *cookie,
-	       xlator_t *this,
-	       int32_t op_ret,
-	       int32_t op_errno,
-	       struct statvfs *buf)
-{
-        ha_local_t   *local = NULL;
-        ha_private_t *priv  = NULL;
-
-        GF_ASSERT (this);
-
-        local = frame->local;
-        priv = this->private;
-        GF_ASSERT (priv);
-
-        if (-1 == op_ret) {
-                local->active = (local->active + 1) % priv->child_count;
-                local->tries--;
-                if (!local->tries)
-                        goto out;
-                
-                STACK_WIND (frame, ha_statfs_cbk,
-                            HA_ACTIVE_CHILD(this, local),
-                            HA_ACTIVE_CHILD(this, local)->fops->statfs, 
-                            &local->loc);
-                return 0;
-        }
-
- out:
-        loc_wipe (&local->loc);
-        STACK_UNWIND (frame, op_ret, op_errno, buf);
-	return 0;
-}
-
-int32_t
-ha_statfs (call_frame_t *frame,
-	   xlator_t *this,
-	   loc_t *loc)
-{
-        ha_private_t *priv = NULL;
-	ha_local_t   *local = NULL;
-	int           op_errno = 0;
-
-        /* The normal way of handling failover doesn't work here
-         * as loc->inode may be null in this case.
-         */
-        local = GF_CALLOC (1, sizeof (*local), 
-                           gf_ha_mt_ha_local_t);
-        if (!local) {
-                op_errno = ENOMEM;
-                goto err;
-        }
-        priv = this->private;
-        frame->local  = local;
-        local->active = priv->pref_subvol;
-        if (-1 == local->active)
-                local->active = 0;
-        local->tries  = priv->child_count;
-        loc_copy (&local->loc, loc);
-
-	STACK_WIND (frame, ha_statfs_cbk, HA_ACTIVE_CHILD(this, local),
-                    HA_ACTIVE_CHILD(this, local)->fops->statfs, loc);
-	return 0;
-err:
-	STACK_UNWIND (frame, -1, op_errno, NULL);
-	return 0;
-}
-
- int32_t
-ha_setxattr_cbk (call_frame_t *frame,
-		 void *cookie,
-		 xlator_t *this,
-		 int32_t op_ret,
-		 int32_t op_errno)
-{
-	int ret = -1;
-
-	ret = ha_handle_cbk (frame, cookie, op_ret, op_errno);
-
-	if (ret == 0) {
-		STACK_UNWIND (frame,
-			      op_ret,
-			      op_errno);
-	}
-	return 0;
-}
-
-int32_t
-ha_setxattr (call_frame_t *frame,
-	     xlator_t *this,
-	     loc_t *loc,
-	     dict_t *dict,
-	     int32_t flags)
-{
-	ha_local_t *local = NULL;
-	int op_errno = 0;
-
-	op_errno = ha_alloc_init_inode (frame, loc->inode);
-	if (op_errno < 0) {
-		op_errno = -op_errno;
-		goto err;
-	}
-	local = frame->local;
-	local->stub = fop_setxattr_stub (frame, ha_setxattr, loc, dict, flags);
-        if (!local->stub) {
-                op_errno = ENOMEM;
-                gf_log (this->name, GF_LOG_ERROR, "out of memory");
-                goto err;
-        }
-
-	STACK_WIND_COOKIE (frame,
-			   ha_setxattr_cbk,
-			   (void *)(long)local->active,
-			   HA_ACTIVE_CHILD(this, local),
-			   HA_ACTIVE_CHILD(this, local)->fops->setxattr,
-			   loc,
-			   dict,
-			   flags);
-	return 0;
-err:
-        local = frame->local;
-        frame->local = NULL;
-
-	STACK_UNWIND (frame, -1, op_errno);
-
-        ha_local_wipe (local);
-	return 0;
-}
-
- int32_t
-ha_getxattr_cbk (call_frame_t *frame,
-		 void *cookie,
-		 xlator_t *this,
-		 int32_t op_ret,
-		 int32_t op_errno,
-		 dict_t *dict)
-{
-	int ret = -1;
-
-	ret = ha_handle_cbk (frame, cookie, op_ret, op_errno);
-
-	if (ret == 0) {
-		STACK_UNWIND (frame,
-			      op_ret,
-			      op_errno,
-			      dict);
-	}
-	return 0;
-}
-
-int32_t
-ha_getxattr (call_frame_t *frame,
-	     xlator_t *this,
-	     loc_t *loc,
-	     const char *name)
-{
-	ha_local_t *local = NULL;
-	int op_errno = 0;
-
-	op_errno = ha_alloc_init_inode (frame, loc->inode);
-	if (op_errno < 0) {
-		op_errno = -op_errno;
-		goto err;
-	}
-	local = frame->local;
-	local->stub = fop_getxattr_stub (frame, ha_getxattr, loc, name);
-        if (!local->stub) {
-                op_errno = ENOMEM;
-                gf_log (this->name, GF_LOG_ERROR, "out of memory");
-                goto err;
-        }
-
-	STACK_WIND_COOKIE (frame,
-			   ha_getxattr_cbk,
-			   (void *)(long)local->active,
-			   HA_ACTIVE_CHILD(this, local),
-			   HA_ACTIVE_CHILD(this, local)->fops->getxattr,
-			   loc,
-			   name);
-	return 0;
-err:
-        local = frame->local;
-        frame->local = NULL;
-
-	STACK_UNWIND (frame, -1, op_errno, NULL);
-
-        ha_local_wipe (local);
-	return 0;
-}
-
-int32_t
-ha_xattrop_cbk (call_frame_t *frame,
-		void *cookie,
-		xlator_t *this,
-		int32_t op_ret,
-		int32_t op_errno,
-		dict_t *dict)
-{
-	int ret = -1;
-	ret = ha_handle_cbk (frame, cookie, op_ret, op_errno);
-	if (ret == 0) {
-		STACK_UNWIND (frame, op_ret, op_errno, dict);
-	}
-	return 0;
-}
-
-
-int32_t
-ha_xattrop (call_frame_t *frame,
-	    xlator_t *this,
-	    loc_t *loc,
-	    gf_xattrop_flags_t flags,
-	    dict_t *dict)
-{
-	ha_local_t *local = NULL;
-	int op_errno = 0;
-
-	op_errno = ha_alloc_init_inode (frame, loc->inode);
-	if (op_errno < 0) {
-		op_errno = -op_errno;
-		goto err;
-	}
-	local = frame->local;
-
-	local->stub = fop_xattrop_stub (frame, ha_xattrop, loc, flags, dict);
-        if (!local->stub) {
-                op_errno = ENOMEM;
-                gf_log (this->name, GF_LOG_ERROR, "out of memory");
-                goto err;
-        }
-
-	STACK_WIND_COOKIE (frame,
-			   ha_xattrop_cbk,
-			   (void *)(long)local->active,
-			   HA_ACTIVE_CHILD(this, local),
-			   HA_ACTIVE_CHILD(this, local)->fops->xattrop,
-			   loc,
-			   flags,
-			   dict);
-	return 0;
-err:
-        local = frame->local;
-        frame->local = NULL;
-
-	STACK_UNWIND (frame, -1, op_errno, dict);
-
-        ha_local_wipe (local);
-	return 0;
-}
-
-int32_t
-ha_fxattrop_cbk (call_frame_t *frame,
-		 void *cookie,
-		 xlator_t *this,
-		 int32_t op_ret,
-		 int32_t op_errno,
-		 dict_t *dict)
-{
-	int ret = -1;
-	ret = ha_handle_cbk (frame, cookie, op_ret, op_errno);
-	if (ret == 0)
-		STACK_UNWIND (frame, op_ret, op_errno, dict);
-	return 0;
-}
-
-int32_t
-ha_fxattrop (call_frame_t *frame,
-	     xlator_t *this,
-	     fd_t *fd,
-	     gf_xattrop_flags_t flags,
-	     dict_t *dict)
-{
-	ha_local_t *local = NULL;
-	int op_errno = 0;
-
-	op_errno = ha_alloc_init_fd (frame, fd);
-	if (op_errno < 0) {
-		op_errno = -op_errno;
-		goto err;
-	}
-	local = frame->local;
-	local->stub = fop_fxattrop_stub (frame, ha_fxattrop, fd, flags, dict);
-        if (!local->stub) {
-                op_errno = ENOMEM;
-                gf_log (this->name, GF_LOG_ERROR, "out of memory");
-                goto err;
-        }
-
-	STACK_WIND_COOKIE (frame,
-			   ha_fxattrop_cbk,
-			   (void *)(long)local->active,
-			   HA_ACTIVE_CHILD(this, local),
-			   HA_ACTIVE_CHILD(this, local)->fops->fxattrop,
-			   fd,
-			   flags,
-			   dict);
-	return 0;
-err:
-        local = frame->local;
-        frame->local = NULL;
-
-	STACK_UNWIND (frame, -1, op_errno, dict);
-
-        ha_local_wipe (local);
-	return 0;
-}
-
- int32_t
-ha_removexattr_cbk (call_frame_t *frame,
-		    void *cookie,
-		    xlator_t *this,
-		    int32_t op_ret,
-		    int32_t op_errno)
-{
-	int ret = -1;
-	ret = ha_handle_cbk (frame, cookie, op_ret, op_errno);
-	if (ret == 0) {
-		STACK_UNWIND (frame,
-			      op_ret,
-			      op_errno);
-	}
-	return 0;
-}
-
-int32_t
-ha_removexattr (call_frame_t *frame,
-		xlator_t *this,
-		loc_t *loc,
-		const char *name)
-{
-	ha_local_t *local = NULL;
-	int op_errno = 0;
-
-	op_errno = ha_alloc_init_inode (frame, loc->inode);
-	if (op_errno < 0) {
-		op_errno = -op_errno;
-		goto err;
-	}
-	local = frame->local;
-	
-	local->stub = fop_removexattr_stub (frame, ha_removexattr, loc, name);
-        if (!local->stub) {
-                op_errno = ENOMEM;
-                gf_log (this->name, GF_LOG_ERROR, "out of memory");
-                goto err;
-        }
-
-	STACK_WIND_COOKIE (frame,
-			   ha_removexattr_cbk,
-			   (void *)(long)local->active,
-			   HA_ACTIVE_CHILD(this, local),
-			   HA_ACTIVE_CHILD(this, local)->fops->removexattr,
-			   loc,
-			   name);
-	return 0;
-err:
-        local = frame->local;
-        frame->local = NULL;
-
-	STACK_UNWIND (frame, -1, op_errno);
-
-        ha_local_wipe (local);
-	return 0;
-}
-
-int32_t
-ha_lk_setlk_unlck_cbk (call_frame_t *frame,
-		       void *cookie,
-		       xlator_t *this,
-		       int32_t op_ret,
-		       int32_t op_errno,
-		       struct gf_flock *lock)
-{
-	ha_local_t *local = NULL;
-	int cnt = 0;
-	call_stub_t *stub = NULL;
-
-	local = frame->local;
-
-	LOCK (&frame->lock);
-	cnt = --local->call_count;
-	if (op_ret == 0)
-		local->op_ret = 0;
-	UNLOCK (&frame->lock);
-
-	if (cnt == 0) {
-		stub = local->stub;
-		GF_FREE (local->state);
-		if (stub->args.lk.lock.l_type == F_UNLCK) {
-			STACK_UNWIND (frame, local->op_ret, local->op_errno, &stub->args.lk.lock);
-		} else {
-			STACK_UNWIND (frame, -1, EIO, NULL);
-		}
-		call_stub_destroy (stub);
-	}
-	return 0;
-}
-
-int32_t
-ha_lk_setlk_cbk (call_frame_t *frame,
-		 void *cookie,
-		 xlator_t *this,
-		 int32_t op_ret,
-		 int32_t op_errno,
-		 struct gf_flock *lock)
-{
-	ha_local_t *local = NULL;
-	ha_private_t *pvt = NULL;
-	xlator_t **children = NULL;
-	int i = 0, cnt = 0, j = 0;
-	int child_count = 0;
-	call_frame_t *prev_frame = NULL;
-	char *state = NULL;
-
-	local = frame->local;
-	pvt = this->private;
-	children = pvt->children;
-	child_count = pvt->child_count;
-	prev_frame = cookie;
-	state = local->state;
-
-	if (op_ret == 0)
-		local->op_ret = 0;
-
-	if ((op_ret == 0) || (op_ret == -1 && op_errno == ENOTCONN)) {
-		for (i = 0; i < child_count; i++) {
-			if (prev_frame->this == cookie)
-				break;
-		}
-		i++;
-		for (; i < child_count; i++) {
-			if (local->state[i])
-				break;
-		}
-		if (i == child_count) {
-			call_stub_t *stub = local->stub;
-			GF_FREE (local->state);
-			STACK_UNWIND (frame, 0, op_errno, &stub->args.lk.lock);
-			call_stub_destroy (stub);
-			return 0;
-		}
-		STACK_WIND (frame,
-			    ha_lk_setlk_cbk,
-			    children[i],
-			    children[i]->fops->lk,
-			    local->stub->args.lk.fd,
-			    local->stub->args.lk.cmd,
-			    &local->stub->args.lk.lock);
-		return 0;
-	} else {
-		for (i = 0; i < child_count; i++) {
-			if (prev_frame->this == cookie)
-				break;
-		}
-		cnt = 0;
-		for (j = 0; j < i; j++) {
-			if (state[i])
-				cnt++;
-		}
-		if (cnt) {
-			struct gf_flock lock;
-			lock = local->stub->args.lk.lock;
-			for (i = 0; i < child_count; i++) {
-				if (state[i]) {
-					STACK_WIND (frame,
-						    ha_lk_setlk_unlck_cbk,
-						    children[i],
-						    children[i]->fops->lk,
-						    local->stub->args.lk.fd,
-						    local->stub->args.lk.cmd,
-						    &lock);
-					if (--cnt == 0)
-						break;
-				}
-			}
-			return 0;
-		} else {
-			GF_FREE (local->state);
-			call_stub_destroy (local->stub);
-			STACK_UNWIND (frame,
-				      op_ret,
-				      op_errno,
-				      lock);
-			return 0;
-		}
-	}
-}
-
-int32_t
-ha_lk_getlk_cbk (call_frame_t *frame,
-		 void *cookie,
-		 xlator_t *this,
-		 int32_t op_ret,
-		 int32_t op_errno,
-		 struct gf_flock *lock)
-{
-	ha_local_t *local = NULL;
-	ha_private_t *pvt = NULL;
-	fd_t *fd = NULL;
-	int child_count = 0, i = 0;
-	xlator_t **children = NULL;
-	call_frame_t *prev_frame = NULL;
-
-	local = frame->local;
-	pvt = this->private;
-	fd = local->stub->args.lk.fd;
-	child_count = pvt->child_count;
-	children = pvt->children;
-	prev_frame = cookie;
-
-	if (op_ret == 0) {
-		GF_FREE (local->state);
-		call_stub_destroy (local->stub);
-		STACK_UNWIND (frame, 0, 0, lock);
-		return 0;
-	}
-
-	for (i = 0; i < child_count; i++) {
-		if (prev_frame->this == children[i])
-			break;
-	}
-
-	for (; i < child_count; i++) {
-		if (local->state[i])
-			break;
-	}
-
-	if (i == child_count) {
-		GF_FREE (local->state);
-		call_stub_destroy (local->stub);
-		STACK_UNWIND (frame, op_ret, op_errno, lock);
-		return 0;
-	}
-
-	STACK_WIND (frame,
-		    ha_lk_getlk_cbk,
-		    children[i],
-		    children[i]->fops->lk,
-		    fd,
-		    local->stub->args.lk.cmd,
-		    &local->stub->args.lk.lock);
-	return 0;
-}
-
-int32_t
-ha_lk (call_frame_t *frame,
-       xlator_t *this,
-       fd_t *fd,
-       int32_t cmd,
-       struct gf_flock *lock)
-{
-	ha_local_t *local = NULL;
-	ha_private_t *pvt = NULL;
-	hafd_t *hafdp = NULL;
-	char *state = NULL;
-	int child_count = 0, i = 0, cnt = 0, ret = 0;
-	xlator_t **children = NULL;
-	uint64_t tmp_hafdp = 0;
-        int32_t op_errno = EINVAL;
-
-	local = frame->local;
-	pvt = this->private;
-	child_count = pvt->child_count;
-	children = pvt->children;
-	ret = fd_ctx_get (fd, this, &tmp_hafdp);
-	if (ret < 0)
-		gf_log (this->name, GF_LOG_ERROR, "fd_ctx_get failed");
-
-	if (local == NULL) {
-                local = frame->local = GF_CALLOC (1, sizeof (*local),
-                                                  gf_ha_mt_ha_local_t);
-                if (!local) {
-                        op_errno = ENOMEM;
-                        gf_log (this->name, GF_LOG_ERROR, "out of memory");
-                        goto err;
-                }
-                        
-		local->active = -1;
-		local->op_ret = -1;
-		local->op_errno = ENOTCONN;
-	}
-	hafdp = (hafd_t *)(long)tmp_hafdp;
-
-	if (local->active == -1) {
-                op_errno = ENOTCONN;
-                goto err;
-	}
-
-	local->stub = fop_lk_stub (frame, ha_lk, fd, cmd, lock);
-        if (!local->stub) {
-                op_errno = ENOMEM;
-                gf_log (this->name, GF_LOG_ERROR, "out of memory");
-                goto err;
-        }
-
-	local->state = GF_CALLOC (1, child_count, gf_ha_mt_char);
-        if (!local->state) {
-                op_errno = ENOMEM;
-                gf_log (this->name, GF_LOG_ERROR, "out of memory");
-                goto err;
-        }
-
-	state = hafdp->fdstate;
-	LOCK (&hafdp->lock);
-	memcpy (local->state, state, child_count);
-	UNLOCK (&hafdp->lock);
-	if (cmd == F_GETLK) {
-		for (i = 0; i < child_count; i++) {
-			if (local->state[i])
-				break;
-		}
-		STACK_WIND (frame,
-			    ha_lk_getlk_cbk,
-			    children[i],
-			    children[i]->fops->lk,
-			    fd,
-			    cmd,
-			    lock);
-	} else if (cmd == F_SETLK && lock->l_type == F_UNLCK) {
-		for (i = 0; i < child_count; i++) {
-			if (local->state[i])
-				local->call_count++;
-		}
-		cnt = local->call_count;
-		for (i = 0; i < child_count; i++) {
-			if (local->state[i]) {
-				STACK_WIND (frame,
-					    ha_lk_setlk_unlck_cbk,
-					    children[i],
-					    children[i]->fops->lk,
-					    fd, cmd, lock);
-				if (--cnt == 0)
-					break;
-			}
-		}
-	} else {
-		for (i = 0; i < child_count; i++) {
-			if (local->state[i])
-				break;
-		}
-		STACK_WIND (frame,
-			    ha_lk_setlk_cbk,
-			    children[i],
-			    children[i]->fops->lk,
-			    fd,
-			    cmd,
-			    lock);
-	}
-	return 0;
-err:
-        local = frame->local;
-        frame->local = NULL;
-
-        STACK_UNWIND (frame, -1, op_errno, NULL);
-
-        ha_local_wipe (local);
-        return 0;
-}
-
- int32_t
-ha_inode_entry_lk_cbk (call_frame_t *frame,
-		       void *cookie,
-		       xlator_t *this,
-		       int32_t op_ret,
-		       int32_t op_errno)
-{
-	int ret = -1;
-
-	ret = ha_handle_cbk (frame, cookie, op_ret, op_errno);
-	if (ret == 0) {
-		STACK_UNWIND (frame,
-			      op_ret,
-			      op_errno);
-	}
-	return 0;
-}
-
-int32_t
-ha_inodelk (call_frame_t *frame,
-	    xlator_t *this,
-            const char *volume,
-	    loc_t *loc,
-	    int32_t cmd,
-	    struct gf_flock *lock)
-{
-	ha_local_t *local = NULL;
-	int op_errno = 0;
-
-	op_errno = ha_alloc_init_inode (frame, loc->inode);
-	if (op_errno < 0) {
-		op_errno = -op_errno;
-		goto err;
-	}
-	local = frame->local;
-	local->stub = fop_inodelk_stub (frame, ha_inodelk, volume,
-                                        loc, cmd, lock);
-	STACK_WIND_COOKIE (frame,
-			   ha_inode_entry_lk_cbk,
-			   (void *)(long)local->active,
-			   HA_ACTIVE_CHILD(this, local),
-			   HA_ACTIVE_CHILD(this, local)->fops->inodelk,
-			   volume,
-                           loc,
-			   cmd,
-			   lock);
-	return 0;
-err:
-	STACK_UNWIND (frame, -1, op_errno);
-	return 0;
-}
-
-int32_t
-ha_entrylk (call_frame_t *frame,
-	    xlator_t *this,
-            const char *volume,
-	    loc_t *loc,
-	    const char *basename,
-	    entrylk_cmd cmd,
-	    entrylk_type type)
-{
-	ha_local_t *local = NULL;
-	int op_errno = 0;
-
-	op_errno = ha_alloc_init_inode (frame, loc->inode);
-	if (op_errno < 0) {
-		op_errno = -op_errno;
-		goto err;
-	}
-	local = frame->local;
-	local->stub = fop_entrylk_stub (frame, ha_entrylk, volume,
-                                        loc, basename, cmd, type);
-	STACK_WIND_COOKIE (frame,
-			   ha_inode_entry_lk_cbk,
-			   (void *)(long)local->active,
-			   HA_ACTIVE_CHILD(this, local),
-			   HA_ACTIVE_CHILD(this, local)->fops->entrylk,
-			   volume, loc, basename, cmd, type);
-	return 0;
-err:
-	STACK_UNWIND (frame, -1, op_errno);
-	return 0;
-}
-
- int32_t
-ha_checksum_cbk (call_frame_t *frame,
-		 void *cookie,
-		 xlator_t *this,
-		 int32_t op_ret,
-		 int32_t op_errno,
-		 uint8_t *file_checksum,
-		 uint8_t *dir_checksum)
-{
-	int ret = -1;
-
-	ret = ha_handle_cbk (frame, cookie, op_ret, op_errno);
-	if (ret == 0) {
-		STACK_UNWIND (frame,
-			      op_ret,
-			      op_errno,
-			      file_checksum,
-			      dir_checksum);
-	}
-	return 0;
-}
-
-int32_t
-ha_checksum (call_frame_t *frame,
-	     xlator_t *this,
-	     loc_t *loc,
-	     int32_t flag)
-{
-	int op_errno = 0;
-	ha_local_t *local = NULL;
-
-	op_errno = ha_alloc_init_inode (frame, loc->inode);
-	if (op_errno < 0) {
-		op_errno = -op_errno;
-		goto err;
-	}
-	local = frame->local;
-	local->stub = fop_checksum_stub (frame, ha_checksum, loc, flag);
-        if (!local->stub) {
-                op_errno = ENOMEM;
-                gf_log (this->name, GF_LOG_ERROR, "out of memory");
-                goto err;
-        }
-
-	STACK_WIND_COOKIE (frame,
-			   ha_checksum_cbk,
-			   (void *)(long)local->active,
-			   HA_ACTIVE_CHILD(this, local),
-			   HA_ACTIVE_CHILD(this, local)->fops->checksum,
-			   loc,
-			   flag);
-	return 0;
-err:
-        local = frame->local;
-        frame->local = NULL;
-
-	STACK_UNWIND (frame, -1, op_errno, NULL, NULL);
-
-        ha_local_wipe (local);
-	return 0;
-}
-
-int32_t
-ha_common_readdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                       int32_t op_ret, int32_t op_errno, gf_dirent_t *entries)
-{
-	int ret = 0;
-
-	ret = ha_handle_cbk (frame, cookie, op_ret, op_errno);
-	if (ret == 0)
-		STACK_UNWIND (frame, op_ret, op_errno, entries);
-	return 0;
-}
-
-int32_t
-ha_readdir (call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
-            off_t off);
-
-int32_t
-ha_readdirp (call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
-             off_t off);
-int32_t
-ha_do_readdir (call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
-               off_t off, int whichop)
-{
-	ha_local_t *local = NULL;
-	int op_errno = 0;
-
-	op_errno = ha_alloc_init_fd (frame, fd);
-	if (op_errno < 0) {
-		op_errno = -op_errno;
-		goto err;
-	}
-	local = frame->local;
-        if (whichop == GF_FOP_READDIR)
-                local->stub = fop_readdir_stub (frame, ha_readdir, fd, size,
-                                                off);
-        else
-                local->stub = fop_readdirp_stub (frame, ha_readdirp, fd, size,
-                                                 off);
-        if (!local->stub) {
-                op_errno = ENOMEM; 
-                gf_log (this->name, GF_LOG_ERROR, "out of memory");
-                goto err;
-        }
-
-        if (whichop == GF_FOP_READDIR)
-                STACK_WIND_COOKIE (frame, ha_common_readdir_cbk,
-                                   (void *)(long)local->active,
-                                   HA_ACTIVE_CHILD(this, local),
-                                   HA_ACTIVE_CHILD(this, local)->fops->readdir,
-                                   fd, size, off);
-        else
-                STACK_WIND_COOKIE (frame, ha_common_readdir_cbk,
-                                   (void *)(long)local->active,
-                                   HA_ACTIVE_CHILD(this, local),
-                                   HA_ACTIVE_CHILD(this, local)->fops->readdirp,
-                                   fd, size, off);
-	return 0;
-err:
-        local = frame->local;
-        frame->local = NULL;
-
-	STACK_UNWIND (frame, -1, ENOTCONN, NULL);
-
-        ha_local_wipe (local);
-	return 0;
-}
-
-
-int32_t
-ha_readdir (call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
-            off_t off)
-{
-        ha_do_readdir (frame, this, fd, size, off, GF_FOP_READDIR);
-        return 0;
-}
-
-int32_t
-ha_readdirp (call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
-             off_t off)
-{
-        ha_do_readdir (frame, this, fd, size, off, GF_FOP_READDIRP);
-        return 0;
-}
-
-/* Management operations */
-
- int32_t
-ha_stats_cbk (call_frame_t *frame,
-	      void *cookie,
-	      xlator_t *this,
-	      int32_t op_ret,
-	      int32_t op_errno,
-	      struct xlator_stats *stats)
-{
-	ha_local_t *local = NULL;
-	ha_private_t *pvt = NULL;
-	call_frame_t *prev_frame = NULL;
-	xlator_t **children = NULL;
-	int i = 0;
-
-	local = frame->local;
-	pvt = this->private;
-	prev_frame = cookie;
-	children = pvt->children;
-
-	if (op_ret == -1 && op_errno == ENOTCONN) {
-		for (i = 0; i < pvt->child_count; i++) {
-			if (prev_frame->this == children[i])
-				break;
-		}
-		i++;
-		for (; i < pvt->child_count; i++) {
-			if (pvt->state[i])
-				break;
-		}
-
-		if (i == pvt->child_count) {
-			STACK_UNWIND (frame, -1, ENOTCONN, NULL);
-			return 0;
-		}
-		STACK_WIND (frame,
-			    ha_stats_cbk,
-			    children[i],
-			    children[i]->mops->stats,
-			    local->flags);
-		return 0;
-	}
-
-	STACK_UNWIND (frame,
-		      op_ret,
-		      op_errno,
-		      stats);
-	return 0;
-}
-
-int32_t
-ha_stats (call_frame_t *frame,
-	  xlator_t *this,
-	  int32_t flags)
-{
-	ha_local_t *local = NULL;
-	ha_private_t *pvt = NULL;
-	xlator_t **children = NULL;
-	int i = 0;
-        int32_t op_errno = EINVAL;
-
-	local = frame->local = GF_CALLOC (1, sizeof (*local),
-                                          gf_ha_mt_ha_local_t);
-        if (!local) {
-                op_errno = ENOMEM;
-                gf_log (this->name, GF_LOG_ERROR, "out of memory");
-                goto err;
-        }
-
-	pvt = this->private;
-	children = pvt->children;
-	for (i = 0; i < pvt->child_count; i++) {
-		if (pvt->state[i])
-			break;
-	}
-
-	if (i == pvt->child_count) {
-                op_errno = ENOTCONN;
-                goto err;
-	}
-	local->flags = flags;
-
-	STACK_WIND (frame,
-		    ha_stats_cbk,
-		    children[i],
-		    children[i]->mops->stats,
-		    flags);
-	return 0;
-
-err:
-        local = frame->local;
-        frame->local = NULL;
-
-        STACK_UNWIND (frame, -1, ENOTCONN, NULL);
-
-        ha_local_wipe (local);
-        return 0;
-
-}
-
-
-int32_t
-ha_getspec_cbk (call_frame_t *frame,
-		void *cookie,
-		xlator_t *this,
-		int32_t op_ret,
-		int32_t op_errno,
-		char *spec_data)
-{
-	ha_local_t *local = NULL;
-	ha_private_t *pvt = NULL;
-	call_frame_t *prev_frame = NULL;
-	xlator_t **children = NULL;
-	int i = 0;
-
-	local = frame->local;
-	pvt = this->private;
-	prev_frame = cookie;
-	children = pvt->children;
-
-	if (op_ret == -1 && op_errno == ENOTCONN) {
-		for (i = 0; i < pvt->child_count; i++) {
-			if (prev_frame->this == children[i])
-				break;
-		}
-		i++;
-		for (; i < pvt->child_count; i++) {
-			if (pvt->state[i])
-				break;
-		}
-
-		if (i == pvt->child_count) {
-			STACK_UNWIND (frame, -1, ENOTCONN, NULL);
-			return 0;
-		}
-		STACK_WIND (frame,
-			    ha_getspec_cbk,
-			    children[i],
-			    children[i]->mops->getspec,
-			    local->pattern,
-			    local->flags);
-		return 0;
-	}
-
-	STACK_UNWIND (frame,
-		      op_ret,
-		      op_errno,
-		      spec_data);
-	return 0;
-}
-
-int32_t
-ha_getspec (call_frame_t *frame,
-	    xlator_t *this,
-	    const char *key,
-	    int32_t flags)
-{
-	ha_local_t *local = NULL;
-	ha_private_t *pvt = NULL;
-	xlator_t **children = NULL;
-	int i = 0;
-        int32_t op_errno = EINVAL;
-
-	local = frame->local = GF_CALLOC (1, sizeof (*local), 
-                                          gf_ha_mt_ha_local_t);
-        if (!local) {
-                op_errno = ENOMEM;
-                gf_log (this->name, GF_LOG_ERROR, "out of memory");
-                goto err;
-        }
-
-	pvt = this->private;
-	children = pvt->children;
-
-	for (i = 0; i < pvt->child_count; i++) {
-		if (pvt->state[i])
-			break;
-	}
-
-	if (i == pvt->child_count) {
-                op_errno = ENOTCONN;
-                goto err;
-	}
-	local->flags = flags;
-	local->pattern = (char *)key;
-
-	STACK_WIND (frame,
-		    ha_getspec_cbk,
-		    children[i],
-		    children[i]->mops->getspec,
-		    key, flags);
-	return 0;
-err:
-        local = frame->local;
-        frame->local = NULL;
-
-        STACK_UNWIND (frame, -1, ENOTCONN, NULL);
-
-        ha_local_wipe (local);
-        return 0;
-
-}
-
-int32_t
-ha_closedir (xlator_t *this,
-	     fd_t *fd)
-{
-	hafd_t *hafdp = NULL;
-	int op_errno = 0;
-	uint64_t tmp_hafdp = 0;
-
-	op_errno = fd_ctx_del (fd, this, &tmp_hafdp);
-	if (op_errno != 0) {
-		gf_log (this->name, GF_LOG_ERROR, "fd_ctx_del() error");
-		return 0;
-	}
-	hafdp = (hafd_t *)(long)tmp_hafdp;
-
-	GF_FREE (hafdp->fdstate);
-	GF_FREE (hafdp->path);
-	LOCK_DESTROY (&hafdp->lock);
-	return 0;
-}
-
-int32_t
-ha_close (xlator_t *this,
-	  fd_t *fd)
-{
-	hafd_t *hafdp = NULL;
-	int op_errno = 0;
-	uint64_t tmp_hafdp = 0;
-
-	op_errno = fd_ctx_del (fd, this, &tmp_hafdp);
-	if (op_errno != 0) {
-		gf_log (this->name, GF_LOG_ERROR, "fd_ctx_del() error");
-		return 0;
-	}
-	hafdp = (hafd_t *)(long)tmp_hafdp;
-
-	GF_FREE (hafdp->fdstate);
-	GF_FREE (hafdp->path);
-	LOCK_DESTROY (&hafdp->lock);
-	return 0;
-}
-
-/* notify */
-int32_t
-notify (xlator_t *this,
-	int32_t event,
-	void *data,
-	...)
-{
-	ha_private_t *pvt = NULL;
-	int32_t i = 0, upcnt = 0;
-
-	pvt = this->private;
-	if (pvt == NULL) {
-		gf_log (this->name, GF_LOG_DEBUG, "got notify before init()");
-		return 0;
-	}
-
-	switch (event)
-	{
-	case GF_EVENT_CHILD_DOWN:
-	{
-		for (i = 0; i < pvt->child_count; i++) {
-			if (data == pvt->children[i])
-				break;
-		}
-	        gf_log (this->name, GF_LOG_DEBUG, "GF_EVENT_CHILD_DOWN from %s", pvt->children[i]->name);
-		pvt->state[i] = 0;
-		for (i = 0; i < pvt->child_count; i++) {
-			if (pvt->state[i])
-				break;
-		}
-		if (i == pvt->child_count) {
-			default_notify (this, event, data);
-		}
-	}
-	break;
-	case GF_EVENT_CHILD_UP:
-	{
-		for (i = 0; i < pvt->child_count; i++) {
-			if (data == pvt->children[i])
-				break;
-		}
-
-		gf_log (this->name, GF_LOG_DEBUG, "GF_EVENT_CHILD_UP from %s", pvt->children[i]->name);
-
-		pvt->state[i] = 1;
-
-		for (i = 0; i < pvt->child_count; i++) {
-			if (pvt->state[i])
-				upcnt++;
-		}
-
-		if (upcnt == 1) {
-			default_notify (this, event, data);
-		}
-	}
-	break;
-
-	default:
-	{
-		default_notify (this, event, data);
-	}
-	}
-
-	return 0;
-}
-
-int32_t
-mem_acct_init (xlator_t *this)
-{
-        int     ret = -1;
-
-        if (!this)
-                return ret;
-
-        ret = xlator_mem_acct_init (this, gf_ha_mt_end + 1);
-        
-        if (ret != 0) {
-                gf_log (this->name, GF_LOG_ERROR, "Memory accounting init"
-                                "failed");
-                return ret;
-        }
-
-        return ret;
-}
-
-int
-init (xlator_t *this)
-{
-	ha_private_t *pvt = NULL;
-	xlator_list_t *trav = NULL;
-	int count = 0, ret = 0;
-
-
-	if (!this->children) {
-		gf_log (this->name,GF_LOG_ERROR, 
-			"FATAL: ha should have one or more child defined");
-		return -1;
-	}
-
-	if (!this->parents) {
-		gf_log (this->name, GF_LOG_WARNING,
-			"dangling volume. check volfile ");
-	}
-  
-	trav = this->children;
-	pvt = GF_CALLOC (1, sizeof (ha_private_t), gf_ha_mt_ha_private_t);
-
-	ret = dict_get_int32 (this->options, "preferred-subvolume", 
-			      &pvt->pref_subvol);
-	if (ret < 0) {
-		pvt->pref_subvol = -1;
-	}
-
-	trav = this->children;
-	while (trav) {
-		count++;
-		trav = trav->next;
-	}
-
-	pvt->child_count = count;
-	pvt->children = GF_CALLOC (count, sizeof (xlator_t*), 
-                                   gf_ha_mt_xlator_t);
-
-	trav = this->children;
-	count = 0;
-	while (trav) {
-		pvt->children[count] = trav->xlator;
-		count++;
-		trav = trav->next;
-	}
-
-	pvt->state = GF_CALLOC (1, count, gf_ha_mt_char);
-	this->private = pvt;
-	return 0;
-}
-
-void
-fini (xlator_t *this)
-{
-	ha_private_t *priv = NULL;
-	priv = this->private;
-	GF_FREE (priv);
-	return;
-}
-
-
-struct xlator_fops fops = {
-	.lookup      = ha_lookup,
-	.stat        = ha_stat,
-	.readlink    = ha_readlink,
-	.mknod       = ha_mknod,
-	.mkdir       = ha_mkdir,
-	.unlink      = ha_unlink,
-	.rmdir       = ha_rmdir,
-	.symlink     = ha_symlink,
-	.rename      = ha_rename,
-	.link        = ha_link,
-	.truncate    = ha_truncate,
-	.create      = ha_create,
-	.open        = ha_open,
-	.readv       = ha_readv,
-	.writev      = ha_writev,
-	.statfs      = ha_statfs,
-	.flush       = ha_flush,
-	.fsync       = ha_fsync,
-	.setxattr    = ha_setxattr,
-	.getxattr    = ha_getxattr,
-	.removexattr = ha_removexattr,
-	.opendir     = ha_opendir,
-	.readdir     = ha_readdir,
-	.readdirp    = ha_readdirp,
-	.getdents    = ha_getdents,
-	.fsyncdir    = ha_fsyncdir,
-	.access      = ha_access,
-	.ftruncate   = ha_ftruncate,
-	.fstat       = ha_fstat,
-	.lk          = ha_lk,
-	.setdents    = ha_setdents,
-	.lookup_cbk  = ha_lookup_cbk,
-	.checksum    = ha_checksum,
-	.xattrop     = ha_xattrop,
-	.fxattrop    = ha_fxattrop,
-        .setattr     = ha_setattr,
-        .fsetattr    = ha_fsetattr,
-};
-
-struct xlator_cbks cbks = {
-	.release    = ha_close,
-	.releasedir = ha_closedir,
-	.forget     = ha_forget,
-};
diff --git a/xlators/cluster/ha/src/ha.h b/xlators/cluster/ha/src/ha.h
deleted file mode 100644
index e2ed7eaa68a..00000000000
--- a/xlators/cluster/ha/src/ha.h
+++ /dev/null
@@ -1,53 +0,0 @@
-/*
-   Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
-   This file is part of GlusterFS.
-
-   This file is licensed to you under your choice of the GNU Lesser
-   General Public License, version 3 or any later version (LGPLv3 or
-   later), or the GNU General Public License, version 2 (GPLv2), in all
-   cases as published by the Free Software Foundation.
-*/
-#ifndef __HA_H_
-#define __HA_H_
-
-#include "ha-mem-types.h"
-
-typedef struct {
-	call_stub_t *stub;
-	int32_t op_ret, op_errno;
-	int32_t active, tries, revalidate, revalidate_error;
-	int32_t call_count;
-	char *state, *pattern;
-	dict_t *dict;
-	loc_t loc;
-	struct iatt buf;
-        struct iatt postparent;
-        struct iatt preparent;
-	fd_t *fd;
-	inode_t *inode;
-	int32_t flags;
-	int32_t first_success;
-} ha_local_t;
-
-typedef struct {
-	char *state;
-	xlator_t **children;
-	int child_count, pref_subvol;
-} ha_private_t;
-
-typedef struct {
-	char *fdstate;
-	char *path;
-	gf_lock_t lock;
-	int active;
-} hafd_t;
-
-#define HA_ACTIVE_CHILD(this, local) (((ha_private_t *)this->private)->children[local->active])
-
-extern int ha_alloc_init_fd (call_frame_t *frame, fd_t *fd);
-
-extern int ha_handle_cbk (call_frame_t *frame, void *cookie, int op_ret, int op_errno) ;
-
-extern int ha_alloc_init_inode (call_frame_t *frame, inode_t *inode);
-
-#endif
diff --git a/xlators/cluster/map/src/Makefile.am b/xlators/cluster/map/src/Makefile.am
deleted file mode 100644
index a278b05e2d1..00000000000
--- a/xlators/cluster/map/src/Makefile.am
+++ /dev/null
@@ -1,16 +0,0 @@
-xlator_LTLIBRARIES = map.la
-xlatordir = $(libdir)/glusterfs/$(PACKAGE_VERSION)/xlator/testing/cluster
-
-map_la_LDFLAGS = -module -avoid-version 
-
-map_la_SOURCES = map.c map-helper.c
-map_la_LIBADD = $(top_builddir)/libglusterfs/src/libglusterfs.la
-
-noinst_HEADERS = map.h
-
-AM_CPPFLAGS = $(GF_CPPFLAGS) -I$(top_srcdir)/libglusterfs/src
-
-AM_CFLAGS = -Wall $(GF_CFLAGS)
-
-CLEANFILES = 
-
diff --git a/xlators/cluster/map/src/map-helper.c b/xlators/cluster/map/src/map-helper.c
deleted file mode 100644
index 851397b68d3..00000000000
--- a/xlators/cluster/map/src/map-helper.c
+++ /dev/null
@@ -1,348 +0,0 @@
-/*
-   Copyright (c) 2009-2012 Red Hat, Inc. <http://www.redhat.com>
-   This file is part of GlusterFS.
-
-   This file is licensed to you under your choice of the GNU Lesser
-   General Public License, version 3 or any later version (LGPLv3 or
-   later), or the GNU General Public License, version 2 (GPLv2), in all
-   cases as published by the Free Software Foundation.
-*/
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
-
-#include "xlator.h"
-#include "map.h"
-
-
-xlator_t *
-map_subvol_next (xlator_t *this, xlator_t *prev)
-{
-	map_private_t *priv = NULL;
-	xlator_t      *next = NULL;
-	int            i = 0;
-
-	priv = this->private;
-
-	for (i = 0; i < priv->child_count; i++) {
-		if (priv->xlarray[i].xl == prev) {
-			if ((i + 1) < priv->child_count)
-				next = priv->xlarray[i + 1].xl;
-			break;
-		}
-	}
-
-	return next;
-}
-
-int
-map_subvol_cnt (xlator_t *this, xlator_t *subvol)
-{
-	int i = 0;
-	int ret = -1;
-	map_private_t *priv = NULL;
-
-	priv = this->private;
-
-	for (i = 0; i < priv->child_count; i++) {
-		if (subvol == priv->xlarray[i].xl) {
-			ret = i;
-			break;
-		}
-	}
-
-	return ret;
-}
-
-int
-map_itransform (xlator_t *this, xlator_t *subvol, uint64_t x, uint64_t *y_p)
-{
-	map_private_t *priv = NULL;
-	int         cnt = 0;
-	int         max = 0;
-	uint64_t    y = 0;
-
-	if (x == ((uint64_t) -1)) {
-		y = (uint64_t) -1;
-		goto out;
-	}
-
-	priv = this->private;
-
-	max = priv->child_count;
-	cnt = map_subvol_cnt (this, subvol);
-
-	y = ((x * max) + cnt);
-
-out:
-	if (y_p)
-		*y_p = y;
-
-	return 0;
-}
-
-
-int
-map_deitransform (xlator_t *this, uint64_t y, xlator_t **subvol_p,
-		  uint64_t *x_p)
-{
-	int         cnt = 0;
-	int         max = 0;
-	uint64_t    x = 0;
-	xlator_t   *subvol = 0;
-	map_private_t *priv = NULL;
-
-	priv = this->private;
-	max = priv->child_count;
-
-	cnt = y % max;
-	x   = y / max;
-
-	subvol = priv->xlarray[cnt].xl;
-
-	if (subvol_p)
-		*subvol_p = subvol;
-
-	if (x_p)
-		*x_p = x;
-
-	return 0;
-}
-
-
-xlator_t *
-get_mapping_subvol_from_path (xlator_t *this, const char *path) 
-{
-	map_private_t      *priv = NULL;
-	struct map_pattern *map = NULL;
-
-	/* To make sure we handle '/' properly */
-	if (!strcmp (path, "/"))
-		return NULL;
-
-	priv = this->private;
-
-	map = priv->map;
-	while (map) {
-		if (!strncmp (map->directory, path, map->dir_len)) {
-			if ((path[map->dir_len] == '/') ||
-			    (path[map->dir_len] == '\0')) {
-				return map->xl;
-			}
-		}
-		
-		map = map->next;
-	}
-
-	return priv->default_xl;
-}
-
-xlator_t *
-get_mapping_subvol_from_ctx (xlator_t *this, inode_t *inode)
-{
-	uint64_t subvol = 0;
-	int      ret    = -1;
-
-	ret = inode_ctx_get (inode, this, &subvol);
-	if (ret != 0) 
-		return NULL;
-
-	return (xlator_t *)(long)subvol;
-}
-
-int
-check_multiple_volume_entry (xlator_t *this, 
-			     xlator_t *subvol)
-{
-	int ret = -1;
-	int idx = 0;
-	map_private_t *priv = NULL;
-
-	priv = this->private;
-	
-	for (idx = 0; idx < priv->child_count; idx++) {
-		if (priv->xlarray[idx].xl == subvol) {
-			if (priv->xlarray[idx].mapped) {
-				gf_log (this->name, GF_LOG_ERROR,
-					"subvolume '%s' is already mapped",
-					subvol->name);
-				goto out;
-			}
-			priv->xlarray[idx].mapped = 1;
-			ret = 0;
-			goto out;
-		}
-	}
-
-	gf_log (this->name, GF_LOG_ERROR,
-		"subvolume '%s' is not found",
-		subvol->name);
-	
- out:
-	return ret;
-}
-
-int
-verify_dir_and_assign_subvol (xlator_t *this, 
-			      const char *directory, 
-			      const char *subvol)
-{
-	int            default_flag = 0;
-	int            ret  = -1;
-	int            idx  = 0;
-	map_private_t *priv = NULL;
-	xlator_list_t *trav = NULL;
-	struct map_pattern *tmp_map = NULL;
-
-	priv = this->private;
-
-	/* check if directory is valid, ie, its a top level dir, and 
-	 * not includes a '*' in it.
-	 */
-	if (!strcmp ("*", directory)) {
-		default_flag = 1;
-	} else {
-		if (directory[0] != '/') {
-			gf_log (this->name, GF_LOG_ERROR,
-				"map takes absolute path, starting with '/'. "
-				"not '%s'", directory);
-			goto out;
-		}
-		for (idx = 1; idx < (strlen (directory) - 1); idx++) {
-			if (directory[idx] == '/') {
-				gf_log (this->name, GF_LOG_ERROR,
-					"map takes only top level directory, "
-					"not '%s'", directory);
-				goto out;
-			}
-		}
-	}
-
-	/* Assign proper subvolume */
-	trav = this->children;
-	while (trav) {
-		if (!strcmp (trav->xlator->name, subvol)) {
-			
-			/* Check if there is another directory for 
-			 * same volume, if yes, return error.
-			 */
-			ret = check_multiple_volume_entry (this, 
-							   trav->xlator);
-			if (ret != 0) {
-				goto out;
-			}
-
-			ret = 0;
-			if (default_flag) {
-				if (priv->default_xl) {
-					ret = -1;
-					gf_log (this->name, GF_LOG_ERROR,
-						"'*' specified more than "
-						"once. don't confuse me!!!");
-				}
-
-				priv->default_xl = trav->xlator;
-				goto out;
-			}
-
-			tmp_map = GF_CALLOC (1, sizeof (struct map_pattern),
-                                             gf_map_mt_map_pattern);
-			tmp_map->xl = trav->xlator;
-			tmp_map->dir_len = strlen (directory);
-
-			/* make sure that the top level directory starts 
-			 * with '/' and ends without '/'
-			 */
-			tmp_map->directory = gf_strdup (directory);
-			if (directory[tmp_map->dir_len - 1] == '/') {
-				tmp_map->dir_len--;
-			}
-
-			if (!priv->map) 
-				priv->map = tmp_map;
-			else {
-				struct map_pattern *trav_map = NULL;
-				trav_map = priv->map;
-				while (trav_map->next)
-					trav_map = trav_map->next;
-				trav_map->next = tmp_map;
-			}
-			
-			goto out;
-		}
-
-		trav = trav->next;
-	}
-
-	gf_log (this->name, GF_LOG_ERROR, 
-		"map volume '%s' is not proper subvolume", subvol);
-
- out:
-	return ret;
-}
-
-int 
-assign_default_subvol (xlator_t *this, const char *default_xl)
-{
-	int ret = -1;
-	map_private_t *priv = NULL;
-	xlator_list_t *trav = NULL;
-
-	priv = this->private;
-	trav = this->children;
-
-	while (trav) {
-		if (!strcmp (trav->xlator->name, default_xl)) {
-			ret = check_multiple_volume_entry (this, 
-							   trav->xlator);
-			if (ret != 0) {
-				goto out;
-			}
-			if (priv->default_xl)
-				gf_log (this->name, GF_LOG_WARNING,
-					"default-volume option provided, "
-					"overriding earlier '*' option");
-			priv->default_xl = trav->xlator;
-			return 0;
-		}
-		trav = trav->next;
-	}
-
-	gf_log (this->name, GF_LOG_ERROR,
-		"default-volume value is not an valid subvolume. check again");
- out:
-	return -1;
-}
-
-void
-verify_if_all_subvolumes_got_used (xlator_t *this)
-{
-	int idx = 0;
-	map_private_t *priv = NULL;
-
-	priv = this->private;
-	
-	for (idx = 0; idx < priv->child_count; idx++) {
-		if (!priv->xlarray[idx].mapped) {
-			if (!priv->default_xl) {
-				priv->default_xl = priv->xlarray[idx].xl;
-				priv->xlarray[idx].mapped = 1;
-			} else {
-				gf_log (this->name, GF_LOG_WARNING,
-					"subvolume '%s' is not mapped to "
-					"any directory",
-					priv->xlarray[idx].xl->name);
-			}
-		}
-	}
-
-	if (!priv->default_xl) {
-		gf_log (this->name, GF_LOG_WARNING,
-			"default subvolume not specified, filesystem "
-			"may not work properly. Check 'map' translator "
-			"documentation for more info");
-	}
-
-	return ;
-}
diff --git a/xlators/cluster/map/src/map-mem-types.h b/xlators/cluster/map/src/map-mem-types.h
deleted file mode 100644
index 3e89f4736e4..00000000000
--- a/xlators/cluster/map/src/map-mem-types.h
+++ /dev/null
@@ -1,24 +0,0 @@
-/*
-   Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
-   This file is part of GlusterFS.
-
-   This file is licensed to you under your choice of the GNU Lesser
-   General Public License, version 3 or any later version (LGPLv3 or
-   later), or the GNU General Public License, version 2 (GPLv2), in all
-   cases as published by the Free Software Foundation.
-*/
-
-#ifndef __MAP_MEM_TYPES_H__
-#define __MAP_MEM_TYPES_H__
-
-#include "mem-types.h"
-
-enum gf_map_mem_types_ {
-        gf_map_mt_map_private_t = gf_common_mt_end + 1,
-        gf_map_mt_map_local_t,
-        gf_map_mt_map_xlator_array,
-        gf_map_mt_map_pattern,
-        gf_map_mt_end
-};
-#endif
-
diff --git a/xlators/cluster/map/src/map.c b/xlators/cluster/map/src/map.c
deleted file mode 100644
index aa2b23c29f1..00000000000
--- a/xlators/cluster/map/src/map.c
+++ /dev/null
@@ -1,2566 +0,0 @@
-/*
-   Copyright (c) 2010-2012 Red Hat, Inc. <http://www.redhat.com>
-   This file is part of GlusterFS.
-
-   This file is licensed to you under your choice of the GNU Lesser
-   General Public License, version 3 or any later version (LGPLv3 or
-   later), or the GNU General Public License, version 2 (GPLv2), in all
-   cases as published by the Free Software Foundation.
-*/
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
-
-#include "xlator.h"
-#include "map.h"
-
-/* TODO : 
- *     -> support for 'get' 'put' API in through xattrs.
- *     -> define the behavior of notify()
- */
-
-static int32_t
-map_stat_cbk (call_frame_t *frame,
-		  void *cookie,
-		  xlator_t *this,
-		  int32_t op_ret,
-		  int32_t op_errno,
-		  struct iatt *buf)
-
-{
-        call_frame_t *prev = NULL;
-        prev  = cookie;
-	
-	map_itransform (this, prev->this, buf->ia_ino, &buf->ia_ino);
-
-	STACK_UNWIND (frame, op_ret, op_errno, buf);
-	return 0;
-}
-
-static int32_t
-map_setattr_cbk (call_frame_t *frame,
-                 void *cookie,
-                 xlator_t *this,
-                 int32_t op_ret,
-                 int32_t op_errno,
-                 struct iatt *statpre,
-                 struct iatt *statpost)
-{
-        call_frame_t *prev = NULL;
-        prev  = cookie;
-
-	map_itransform (this, prev->this, statpre->ia_ino, &statpre->ia_ino);
-	map_itransform (this, prev->this, statpost->ia_ino, &statpost->ia_ino);
-
-	STACK_UNWIND (frame, op_ret, op_errno, statpre, statpost);
-	return 0;
-}
-
-static int32_t
-map_fsetattr_cbk (call_frame_t *frame,
-                  void *cookie,
-                  xlator_t *this,
-                  int32_t op_ret,
-                  int32_t op_errno,
-                  struct iatt *statpre,
-                  struct iatt *statpost)
-{
-        call_frame_t *prev = NULL;
-        prev  = cookie;
-
-	map_itransform (this, prev->this, statpre->ia_ino, &statpre->ia_ino);
-	map_itransform (this, prev->this, statpost->ia_ino, &statpost->ia_ino);
-
-	STACK_UNWIND (frame, op_ret, op_errno, statpre, statpost);
-	return 0;
-}
-
-static int32_t
-map_truncate_cbk (call_frame_t *frame,
-		      void *cookie,
-		      xlator_t *this,
-		      int32_t op_ret,
-		      int32_t op_errno,
-		      struct iatt *prebuf,
-                      struct iatt *postbuf)
-{
-        call_frame_t *prev = NULL;
-        prev  = cookie;
-	
-	map_itransform (this, prev->this, postbuf->ia_ino, &postbuf->ia_ino);
-
-	STACK_UNWIND (frame, op_ret, op_errno, prebuf, postbuf);
-	return 0;
-}
-
-static int32_t
-map_ftruncate_cbk (call_frame_t *frame,
-		       void *cookie,
-		       xlator_t *this,
-		       int32_t op_ret,
-		       int32_t op_errno,
-		       struct iatt *prebuf,
-                       struct iatt *postbuf)
-{
-        call_frame_t *prev = NULL;
-        prev  = cookie;
-	
-	map_itransform (this, prev->this, postbuf->ia_ino, &postbuf->ia_ino);
-
-	STACK_UNWIND (frame, op_ret, op_errno, prebuf, postbuf);
-	return 0;
-}
-
-
-static int32_t
-map_access_cbk (call_frame_t *frame,
-		    void *cookie,
-		    xlator_t *this,
-		    int32_t op_ret,
-		    int32_t op_errno)
-{
-	STACK_UNWIND (frame, op_ret, op_errno);
-	return 0;
-}
-
-static int32_t
-map_readlink_cbk (call_frame_t *frame,
-		      void *cookie,
-		      xlator_t *this,
-		      int32_t op_ret,
-		      int32_t op_errno,
-		      const char *path,
-                      struct iatt *sbuf)
-{
-	STACK_UNWIND (frame, op_ret, op_errno, path, sbuf);
-	return 0;
-}
-
-static int32_t
-map_unlink_cbk (call_frame_t *frame,
-		    void *cookie,
-		    xlator_t *this,
-		    int32_t op_ret,
-		    int32_t op_errno,
-                    struct iatt *preparent,
-                    struct iatt *postparent)
-{
-	STACK_UNWIND (frame, op_ret, op_errno, preparent, postparent);
-	return 0;
-}
-
-static int32_t
-map_rmdir_cbk (call_frame_t *frame,
-		   void *cookie,
-		   xlator_t *this,
-		   int32_t op_ret,
-		   int32_t op_errno,
-                   struct iatt *preparent,
-                   struct iatt *postparent)
-{
-	STACK_UNWIND (frame, op_ret, op_errno, preparent, postparent);
-	return 0;
-}
-
-
-static int32_t
-map_rename_cbk (call_frame_t *frame,
-		    void *cookie,
-		    xlator_t *this,
-		    int32_t op_ret,
-		    int32_t op_errno,
-		    struct iatt *buf,
-                    struct iatt *preoldparent,
-                    struct iatt *postoldparent,
-                    struct iatt *prenewparent,
-                    struct iatt *postnewparent)
-{
-        call_frame_t *prev = NULL;
-        prev  = cookie;
-	
-	map_itransform (this, prev->this, buf->ia_ino, &buf->ia_ino);
-
-	STACK_UNWIND (frame, op_ret, op_errno, buf);
-	return 0;
-}
-
-static int32_t
-map_link_cbk (call_frame_t *frame,
-		  void *cookie,
-		  xlator_t *this,
-		  int32_t op_ret,
-		  int32_t op_errno,
-		  inode_t *inode,
-                  struct iatt *buf,
-                  struct iatt *preparent,
-                  struct iatt *postparent)
-{
-        call_frame_t *prev = NULL;
-        prev  = cookie;
-	
-	map_itransform (this, prev->this, buf->ia_ino, &buf->ia_ino);
-
-	STACK_UNWIND (frame, op_ret, op_errno, inode,	buf);
-	return 0;
-}
-
-static int32_t
-map_open_cbk (call_frame_t *frame,
-		  void *cookie,
-		  xlator_t *this,
-		  int32_t op_ret,
-		  int32_t op_errno,
-		  fd_t *fd)
-{
-	STACK_UNWIND (frame, op_ret, op_errno, fd);
-	return 0;
-}
-
-static int32_t
-map_readv_cbk (call_frame_t *frame,
-               void *cookie,
-               xlator_t *this,
-               int32_t op_ret,
-               int32_t op_errno,
-               struct iovec *vector,
-               int32_t count,
-               struct iatt *stbuf,
-               struct iobref *iobref)
-{
-        call_frame_t *prev = NULL;
-        prev  = cookie;
-	
-	map_itransform (this, prev->this, stbuf->ia_ino, &stbuf->ia_ino);
-
-	STACK_UNWIND (frame, op_ret, op_errno, vector, count, stbuf, iobref);
-	return 0;
-}
-
-static int32_t
-map_writev_cbk (call_frame_t *frame,
-		    void *cookie,
-		    xlator_t *this,
-		    int32_t op_ret,
-		    int32_t op_errno,
-                    struct iatt *prebuf,
-		    struct iatt *postbuf)
-{
-        call_frame_t *prev = NULL;
-        prev  = cookie;
-	
-	map_itransform (this, prev->this, postbuf->ia_ino, &postbuf->ia_ino);
-
-	STACK_UNWIND (frame, op_ret, op_errno, prebuf, postbuf);
-	return 0;
-}
-
-static int32_t
-map_flush_cbk (call_frame_t *frame,
-		   void *cookie,
-		   xlator_t *this,
-		   int32_t op_ret,
-		   int32_t op_errno)
-{
-	STACK_UNWIND (frame, op_ret, op_errno);
-	return 0;
-}
-
-
-static int32_t
-map_fsync_cbk (call_frame_t *frame,
-		   void *cookie,
-		   xlator_t *this,
-		   int32_t op_ret,
-		   int32_t op_errno,
-                   struct iatt *prebuf,
-                   struct iatt *postbuf)
-{
-	STACK_UNWIND (frame, op_ret, op_errno, prebuf, postbuf);
-	return 0;
-}
-
-
-static int32_t
-map_fstat_cbk (call_frame_t *frame,
-		   void *cookie,
-		   xlator_t *this,
-		   int32_t op_ret,
-		   int32_t op_errno,
-		   struct iatt *buf)
-{
-        call_frame_t *prev = NULL;
-        prev  = cookie;
-	
-	map_itransform (this, prev->this, buf->ia_ino, &buf->ia_ino);
-
-	STACK_UNWIND (frame, op_ret, op_errno, buf);
-	return 0;
-}
-
-
-static int32_t
-map_getdents_cbk (call_frame_t *frame,
-		      void *cookie,
-		      xlator_t *this,
-		      int32_t op_ret,
-		      int32_t op_errno,
-		      dir_entry_t *entries,
-		      int32_t count)
-{
-	STACK_UNWIND (frame, op_ret, op_errno, entries, count);
-	return 0;
-}
-
-
-static int32_t
-map_setdents_cbk (call_frame_t *frame,
-		      void *cookie,
-		      xlator_t *this,
-		      int32_t op_ret,
-		      int32_t op_errno)
-{
-	STACK_UNWIND (frame, op_ret, op_errno);
-	return 0;
-}
-
-static int32_t
-map_fsyncdir_cbk (call_frame_t *frame,
-		      void *cookie,
-		      xlator_t *this,
-		      int32_t op_ret,
-		      int32_t op_errno)
-{
-	STACK_UNWIND (frame, op_ret, op_errno);
-	return 0;
-}
-
-
-static int32_t
-map_setxattr_cbk (call_frame_t *frame,
-		      void *cookie,
-		      xlator_t *this,
-		      int32_t op_ret,
-		      int32_t op_errno)
-{
-	STACK_UNWIND (frame, op_ret, op_errno);
-	return 0;
-}
-
-
-static int32_t
-map_fsetxattr_cbk (call_frame_t *frame,
-                       void *cookie,
-                       xlator_t *this,
-                       int32_t op_ret,
-                       int32_t op_errno)
-{
-	STACK_UNWIND (frame, op_ret, op_errno);
-	return 0;
-}
-
-
-static int32_t
-map_fgetxattr_cbk (call_frame_t *frame,
-                       void *cookie,
-                       xlator_t *this,
-                       int32_t op_ret,
-                       int32_t op_errno,
-                       dict_t *dict)
-{
-	STACK_UNWIND (frame, op_ret, op_errno, dict);
-	return 0;
-}
-
-
-
-static int32_t
-map_getxattr_cbk (call_frame_t *frame,
-                      void *cookie,
-                      xlator_t *this,
-                      int32_t op_ret,
-                      int32_t op_errno,
-                      dict_t *dict)
-{
-	STACK_UNWIND (frame, op_ret, op_errno, dict);
-	return 0;
-}
-
-int32_t
-map_xattrop_cbk (call_frame_t *frame,
-		     void *cookie,
-		     xlator_t *this,
-		     int32_t op_ret,
-		     int32_t op_errno,
-		     dict_t *dict)
-{
-	STACK_UNWIND (frame, op_ret, op_errno, dict);
-	return 0;
-}
-
-int32_t
-map_fxattrop_cbk (call_frame_t *frame,
-		      void *cookie,
-		      xlator_t *this,
-		      int32_t op_ret,
-		      int32_t op_errno,
-		      dict_t *dict)
-{
-	STACK_UNWIND (frame, op_ret, op_errno, dict);
-	return 0;
-}
-
-static int32_t
-map_removexattr_cbk (call_frame_t *frame,
-			 void *cookie,
-			 xlator_t *this,
-			 int32_t op_ret,
-			 int32_t op_errno)
-{
-	STACK_UNWIND (frame, op_ret, op_errno);
-	return 0;
-}
-
-
-static int32_t
-map_lk_cbk (call_frame_t *frame,
-		void *cookie,
-		xlator_t *this,
-		int32_t op_ret,
-		int32_t op_errno,
-		struct gf_flock *lock)
-{
-	STACK_UNWIND (frame, op_ret, op_errno, lock);
-	return 0;
-}
-
-
-static int32_t
-map_inodelk_cbk (call_frame_t *frame, void *cookie,
-		     xlator_t *this, int32_t op_ret, int32_t op_errno)
-
-{
-	STACK_UNWIND (frame, op_ret, op_errno);
-	return 0;
-}
-
-
-
-static int32_t
-map_finodelk_cbk (call_frame_t *frame, void *cookie,
-		      xlator_t *this, int32_t op_ret, int32_t op_errno)
-
-{
-	STACK_UNWIND (frame, op_ret, op_errno);
-	return 0;
-}
-
-
-static int32_t
-map_entrylk_cbk (call_frame_t *frame, void *cookie,
-		     xlator_t *this, int32_t op_ret, int32_t op_errno)
-
-{
-	STACK_UNWIND (frame, op_ret, op_errno);
-	return 0;
-}
-
-
-static int32_t
-map_fentrylk_cbk (call_frame_t *frame, void *cookie,
-		      xlator_t *this, int32_t op_ret, int32_t op_errno)
-
-{
-	STACK_UNWIND (frame, op_ret, op_errno);
-	return 0;
-}
-
-static int32_t
-map_newentry_cbk (call_frame_t *frame,
-		  void *cookie,
-		  xlator_t *this,
-		  int32_t op_ret,
-		  int32_t op_errno,
-		  inode_t *inode,
-                  struct iatt *buf,
-                  struct iatt *preparent,
-                  struct iatt *postparent)
-{
-        call_frame_t *prev = NULL;
-        prev  = cookie;
-	
-	map_itransform (this, prev->this, buf->ia_ino, &buf->ia_ino);
-
-	STACK_UNWIND (frame, op_ret, op_errno, inode, buf);
-	return 0;
-
-}
-
-
-static int32_t
-map_create_cbk (call_frame_t *frame,
-		void *cookie,
-		xlator_t *this,
-		int32_t op_ret,
-		int32_t op_errno,
-		fd_t *fd,
-		inode_t *inode,
-		struct iatt *buf,
-                struct iatt *preparent,
-                struct iatt *postparent)
-{
-        call_frame_t *prev = NULL;
-        prev  = cookie;
-	
-	map_itransform (this, prev->this, buf->ia_ino, &buf->ia_ino);
-
-	STACK_UNWIND (frame, op_ret, op_errno, fd, inode, buf);
-	return 0;
-}
-
-
-/*
- * map_normalize_stats -
- */
-void
-map_normalize_stats (struct statvfs *buf,
-		     unsigned long bsize,
-		     unsigned long frsize)
-{
-	double factor;
-
-	if (buf->f_bsize != bsize) {
-		factor = ((double) buf->f_bsize) / bsize;
-		buf->f_bsize  = bsize;
-		buf->f_bfree  = (fsblkcnt_t) (factor * buf->f_bfree);
-		buf->f_bavail = (fsblkcnt_t) (factor * buf->f_bavail);
-	}
-  
-	if (buf->f_frsize != frsize) {
-		factor = ((double) buf->f_frsize) / frsize;
-		buf->f_frsize = frsize;
-		buf->f_blocks = (fsblkcnt_t) (factor * buf->f_blocks);
-	}
-}
-
-
-int32_t
-map_statfs_cbk (call_frame_t *frame,
-		void *cookie,
-		xlator_t *this,
-		int32_t op_ret,
-		int32_t op_errno,
-		struct statvfs *stbuf)
-{
-	struct statvfs *dict_buf = NULL;
-	map_local_t  *local = NULL;
-	int           this_call_cnt = 0;
-	unsigned long bsize;
-	unsigned long frsize;
-
-	local = frame->local;
-
-	LOCK (&frame->lock);
-	{
-		this_call_cnt = --local->call_count;
-
-		if (op_ret == -1) {
-			local->op_errno = op_errno;
-			goto unlock;
-		}
-		local->op_ret = 0;
-		
-		/* when a call is successful, add it to local->dict */
-		dict_buf = &local->statvfs;
-		
-		if (dict_buf->f_bsize != 0) {
-			bsize  = max (dict_buf->f_bsize, 
-				      stbuf->f_bsize);
-			
-			frsize = max (dict_buf->f_frsize, 
-				      stbuf->f_frsize);
-			map_normalize_stats(dict_buf, bsize, frsize);
-			map_normalize_stats(stbuf, bsize, frsize);
-		} else {
-			dict_buf->f_bsize   = stbuf->f_bsize;
-			dict_buf->f_frsize  = stbuf->f_frsize;
-		}
-		
-		dict_buf->f_blocks += stbuf->f_blocks;
-		dict_buf->f_bfree  += stbuf->f_bfree;
-		dict_buf->f_bavail += stbuf->f_bavail;
-		dict_buf->f_files  += stbuf->f_files;
-		dict_buf->f_ffree  += stbuf->f_ffree;
-		dict_buf->f_favail += stbuf->f_favail;
-		dict_buf->f_fsid    = stbuf->f_fsid;
-		dict_buf->f_flag    = stbuf->f_flag;
-		dict_buf->f_namemax = stbuf->f_namemax;
-	}
-unlock:
-	UNLOCK (&frame->lock);
-
-	if (!this_call_cnt) {
-		STACK_UNWIND (frame, local->op_ret, local->op_errno,
-			      &local->statvfs);
-	}
-
-	return 0;
-}
-
-int32_t
-map_single_lookup_cbk (call_frame_t *frame,
-		       void *cookie,
-		       xlator_t *this,
-		       int32_t op_ret,
-		       int32_t op_errno,
-		       inode_t *inode,
-		       struct iatt *buf,
-                       dict_t *dict,
-                       struct iatt *postparent)
-{
-        call_frame_t *prev = NULL;
-        prev  = cookie;
-	
-	map_itransform (this, prev->this, buf->ia_ino, &buf->ia_ino);
-
-	STACK_UNWIND (frame, op_ret, op_errno, inode, buf, dict);
-
-	return 0;
-}
-
-int32_t
-map_root_lookup_cbk (call_frame_t *frame,
-                     void *cookie,
-                     xlator_t *this,
-                     int32_t op_ret,
-                     int32_t op_errno,
-                     inode_t *inode,
-                     struct iatt *buf,
-                     dict_t *dict,
-                     struct iatt *postparent)
-{
-	int          callcnt = 0;
-	map_local_t *local = NULL;
-	inode_t     *tmp_inode = NULL;
-	dict_t      *tmp_dict = NULL;
-
-	local = frame->local;
-	LOCK (&frame->lock);
-	{
-		callcnt = --local->call_count;
-		if ((op_ret == 0) && (local->op_ret == -1)) {
-			local->op_ret = 0;
-			local->stbuf = *buf;
-			if (dict)
-				local->dict = dict_ref (dict);
-			local->inode = inode_ref (inode);
-		}
-		if (op_ret == -1)
-			local->op_errno = op_errno;
-		
-	}
-	UNLOCK (&frame->lock);
-
-	if (!callcnt) {
-		tmp_dict = local->dict;
-		tmp_inode = local->inode;
-
-		STACK_UNWIND (frame, local->op_ret, 
-			      local->op_errno, local->inode, 
-			      &local->stbuf, local->dict);
-
-		inode_unref (local->inode);
-		if (tmp_dict)
-			dict_unref (tmp_dict);
-	}
-
-	return 0;
-}
-
-
-int32_t
-map_opendir_cbk (call_frame_t *frame,
-		 void *cookie,
-		 xlator_t *this,
-		 int32_t op_ret,
-		 int32_t op_errno,
-		 fd_t *fd)
-{
-	int callcnt = 0;
-	map_local_t *local = NULL;
-	fd_t *local_fd = NULL;
-
-	local = frame->local;
-	LOCK (&frame->lock);
-	{
-		callcnt = --local->call_count;
-
-		if (op_ret == -1) {
-			local->op_errno = op_errno;
-			goto unlock;
-		}
-
-		local->op_ret = 0;
-	}
- unlock:
-	UNLOCK (&frame->lock);
-
-	if (!callcnt) {
-		local_fd = local->fd;
-		local->fd = NULL;
-
-		STACK_UNWIND (frame, local->op_ret, 
-			      local->op_errno, local_fd);
-
-		fd_unref (local_fd);
-	}
-	return 0;
-}
-
-int32_t
-map_single_readdir_cbk (call_frame_t *frame,
-			void *cookie,
-			xlator_t *this,
-			int32_t op_ret,
-			int32_t op_errno,
-			gf_dirent_t *entries)
-{
-        call_frame_t *prev = NULL;
-	gf_dirent_t  *orig_entry = NULL;
-
-        prev  = cookie;
-
-	list_for_each_entry (orig_entry, &entries->list, list) {
-		map_itransform (this, prev->this, orig_entry->d_ino, 
-				&orig_entry->d_ino);
-	}
-	STACK_UNWIND (frame, op_ret, op_errno, entries);
-	
-	return 0;
-}
-
-
-int32_t
-map_single_readdirp_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                         int32_t op_ret, int32_t op_errno, gf_dirent_t *entries)
-{
-        call_frame_t *prev = NULL;
-	gf_dirent_t  *orig_entry = NULL;
-
-        prev  = cookie;
-
-	list_for_each_entry (orig_entry, &entries->list, list) {
-		map_itransform (this, prev->this, orig_entry->d_ino,
-				&orig_entry->d_ino);
-                orig_entry->d_stat.ia_ino = orig_entry->d_ino;
-	}
-	STACK_UNWIND (frame, op_ret, op_errno, entries);
-	return 0;
-}
-
-int
-map_readdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                 int op_ret, int op_errno, gf_dirent_t *orig_entries);
-
-int
-map_readdirp_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                 int op_ret, int op_errno, gf_dirent_t *orig_entries);
-
-int
-map_generic_readdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                         int op_ret, int op_errno, gf_dirent_t *orig_entries,
-                         int whichop)
-{
-	map_local_t  *local = NULL;
-	gf_dirent_t   entries;
-	gf_dirent_t  *orig_entry = NULL;
-	gf_dirent_t  *entry = NULL;
-	call_frame_t *prev = NULL;
-	xlator_t     *subvol = NULL;
-	xlator_t     *next_subvol = NULL;
-        off_t         next_offset = 0;
-	int           count = 0;
-	fd_t         *local_fd = NULL;
-
-	INIT_LIST_HEAD (&entries.list);
-	prev = cookie;
-	local = frame->local;
-
-	if (op_ret < 0)
-		goto done;
-
-	list_for_each_entry (orig_entry, &orig_entries->list, list) {
-		subvol = prev->this;
-
-		entry = gf_dirent_for_name (orig_entry->d_name);
-		if (!entry) {
-			gf_log (this->name, GF_LOG_ERROR,
-				"memory allocation failed :(");
-			goto unwind;
-		}
-		
-		map_itransform (this, subvol, orig_entry->d_ino,
-				&entry->d_ino);
-		map_itransform (this, subvol, orig_entry->d_off,
-				&entry->d_off);
-		
-                if (whichop == GF_FOP_READDIRP)
-                        entry->d_stat.ia_ino = entry->d_ino;
-		entry->d_type = orig_entry->d_type;
-		entry->d_len  = orig_entry->d_len;
-		
-		list_add_tail (&entry->list, &entries.list);
-		count++;
-                next_offset = orig_entry->d_off;
-	}
-
-	op_ret = count;
-
-done:
-	if (count == 0) {
-                /* non-zero next_offset means that
-                   EOF is not yet hit on the current subvol
-                */
-                if (next_offset == 0) {
-                        next_subvol = map_subvol_next (this, prev->this);
-                } else {
-                        next_subvol = prev->this;
-                }
-
-		if (!next_subvol) {
-			goto unwind;
-		}
-
-                if (whichop == GF_FOP_READDIR)
-                        STACK_WIND (frame, map_readdir_cbk, next_subvol,
-                                    next_subvol->fops->readdir, local->fd,
-                                    local->size, 0);
-                else
-                        STACK_WIND (frame, map_readdirp_cbk, next_subvol,
-                                    next_subvol->fops->readdirp, local->fd,
-                                    local->size, 0);
-		return 0;
-	}
-
-unwind:
-	if (op_ret < 0)
-		op_ret = 0;
-
-	local_fd = local->fd;
-	local->fd = NULL;
-
-	STACK_UNWIND (frame, op_ret, op_errno, &entries);
-
-	fd_unref (local_fd);
-
-	gf_dirent_free (&entries);
-
-        return 0;
-}
-
-
-int
-map_readdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                 int op_ret, int op_errno, gf_dirent_t *orig_entries)
-{
-        map_generic_readdir_cbk (frame, cookie, this, op_ret, op_errno,
-                                 orig_entries, GF_FOP_READDIR);
-        return 0;
-}
-
-
-int
-map_readdirp_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                  int op_ret, int op_errno, gf_dirent_t *orig_entries)
-{
-        map_generic_readdir_cbk (frame, cookie, this, op_ret, op_errno,
-                                 orig_entries, GF_FOP_READDIRP);
-        return 0;
-}
-
-
-/* Management operations */
-
-static int32_t
-map_checksum_cbk (call_frame_t *frame,
-		      void *cookie,
-		      xlator_t *this,
-		      int32_t op_ret,
-		      int32_t op_errno,
-		      uint8_t *file_checksum,
-		      uint8_t *dir_checksum)
-{
-	STACK_UNWIND (frame, op_ret, op_errno, file_checksum, dir_checksum);
-	return 0;
-}
-
-
-/* Fops starts here */
-
-int32_t
-map_stat (call_frame_t *frame,
-	  xlator_t *this,
-	  loc_t *loc)
-{
-	int32_t op_errno = 1;
-	xlator_t *subvol   = NULL;
-
-        VALIDATE_OR_GOTO (frame, err);
-        VALIDATE_OR_GOTO (this, err);
-        VALIDATE_OR_GOTO (loc, err);
-        VALIDATE_OR_GOTO (loc->inode, err);
-        VALIDATE_OR_GOTO (loc->path, err);
-
-	subvol = get_mapping_subvol_from_ctx (this, loc->inode);
-	if (!subvol) {
-		op_errno = EINVAL;
-		goto err;
-	}
-
-	STACK_WIND (frame, map_stat_cbk, subvol, subvol->fops->stat, loc);
-
-	return 0;
- err:
-	STACK_UNWIND (frame, -1, op_errno, NULL, NULL);
-
-	return 0;
-}
-
-int32_t
-map_setattr (call_frame_t *frame,
-             xlator_t *this,
-             loc_t *loc,
-             struct iatt *stbuf,
-             int32_t valid)
-{
-	int32_t op_errno = 1;
-	xlator_t *subvol   = NULL;
-
-        GF_VALIDATE_OR_GOTO ("map", this, err);
-        GF_VALIDATE_OR_GOTO (this->name, frame, err);
-        GF_VALIDATE_OR_GOTO (this->name, loc, err);
-        GF_VALIDATE_OR_GOTO (this->name, loc->inode, err);
-        GF_VALIDATE_OR_GOTO (this->name, loc->path, err);
-        GF_VALIDATE_OR_GOTO (this->name, stbuf, err);
-
-	subvol = get_mapping_subvol_from_ctx (this, loc->inode);
-	if (!subvol) {
-		op_errno = EINVAL;
-		goto err;
-	}
-
-	STACK_WIND (frame, map_setattr_cbk, subvol,
-		    subvol->fops->setattr, loc, stbuf, valid);
-	return 0;
- err:
-	STACK_UNWIND (frame, -1, op_errno, NULL, NULL);
-
-	return 0;
-}
-
-int32_t
-map_fsetattr (call_frame_t *frame,
-              xlator_t *this,
-              fd_t *fd,
-              struct iatt *stbuf,
-              int32_t valid)
-{
-	int32_t op_errno = 1;
-	xlator_t *subvol   = NULL;
-
-        GF_VALIDATE_OR_GOTO ("map", this, err);
-        GF_VALIDATE_OR_GOTO (this->name, frame, err);
-        GF_VALIDATE_OR_GOTO (this->name, fd, err);
-        GF_VALIDATE_OR_GOTO (this->name, stbuf, err);
-
-	subvol = get_mapping_subvol_from_ctx (this, fd->inode);
-	if (!subvol) {
-		op_errno = EINVAL;
-		goto err;
-	}
-
-	STACK_WIND (frame, map_fsetattr_cbk, subvol,
-		    subvol->fops->fsetattr, fd, stbuf, valid);
-	return 0;
- err:
-	STACK_UNWIND (frame, -1, op_errno, NULL, NULL);
-
-	return 0;
-}
-
-int32_t
-map_truncate (call_frame_t *frame,
-	      xlator_t *this,
-	      loc_t *loc,
-	      off_t offset)
-{
-	int32_t op_errno = 1;
-	xlator_t *subvol   = NULL;
-
-        VALIDATE_OR_GOTO (frame, err);
-        VALIDATE_OR_GOTO (this, err);
-        VALIDATE_OR_GOTO (loc, err);
-        VALIDATE_OR_GOTO (loc->inode, err);
-        VALIDATE_OR_GOTO (loc->path, err);
-
-	subvol = get_mapping_subvol_from_ctx (this, loc->inode);
-	if (!subvol) {
-		op_errno = EINVAL;
-		goto err;
-	}
-
-	STACK_WIND (frame, map_truncate_cbk, subvol, 
-                    subvol->fops->truncate, loc, offset);
-
-	return 0;
- err:
-	STACK_UNWIND (frame, -1, op_errno, NULL, NULL);
-
-	return 0;
-}
-
-int32_t
-map_ftruncate (call_frame_t *frame,
-	       xlator_t *this,
-	       fd_t *fd,
-	       off_t offset)
-{
-	int32_t op_errno = 1;
-	xlator_t *subvol   = NULL;
-
-        VALIDATE_OR_GOTO (frame, err);
-        VALIDATE_OR_GOTO (this, err);
-        VALIDATE_OR_GOTO (fd, err);
-        VALIDATE_OR_GOTO (fd->inode, err);
-
-	subvol = get_mapping_subvol_from_ctx (this, fd->inode);
-	if (!subvol) {
-		op_errno = EINVAL;
-		goto err;
-	}
-
-	STACK_WIND (frame, map_ftruncate_cbk, subvol,
-		    subvol->fops->ftruncate, fd, offset);
-
-	return 0;
- err:
-	STACK_UNWIND (frame, -1, op_errno, NULL, NULL);
-
-	return 0;
-}
-
-int32_t
-map_access (call_frame_t *frame,
-	    xlator_t *this,
-	    loc_t *loc,
-	    int32_t mask)
-{
-	int32_t op_errno = 1;
-	xlator_t *subvol   = NULL;
-
-        VALIDATE_OR_GOTO (frame, err);
-        VALIDATE_OR_GOTO (this, err);
-        VALIDATE_OR_GOTO (loc, err);
-        VALIDATE_OR_GOTO (loc->inode, err);
-        VALIDATE_OR_GOTO (loc->path, err);
-
-	subvol = get_mapping_subvol_from_ctx (this, loc->inode);
-	if (!subvol) {
-		op_errno = EINVAL;
-		goto err;
-	}
-
-	STACK_WIND (frame, map_access_cbk, subvol,
-		    subvol->fops->access, loc, mask);
-
-	return 0;
- err:
-	STACK_UNWIND (frame, -1, op_errno, NULL, NULL);
-
-	return 0;
-}
-
-
-int32_t
-map_readlink (call_frame_t *frame,
-	      xlator_t *this,
-	      loc_t *loc,
-	      size_t size)
-{
-	int32_t op_errno = 1;
-	xlator_t *subvol   = NULL;
-
-        VALIDATE_OR_GOTO (frame, err);
-        VALIDATE_OR_GOTO (this, err);
-        VALIDATE_OR_GOTO (loc, err);
-        VALIDATE_OR_GOTO (loc->inode, err);
-        VALIDATE_OR_GOTO (loc->path, err);
-
-	subvol = get_mapping_subvol_from_ctx (this, loc->inode);
-	if (!subvol) {
-		op_errno = EINVAL;
-		goto err;
-	}
-
-	STACK_WIND (frame, map_readlink_cbk, subvol,
-		    subvol->fops->readlink, loc, size);
-
-	return 0;
- err:
-	STACK_UNWIND (frame, -1, op_errno, NULL, NULL);
-
-	return 0;
-}
-
-int32_t
-map_unlink (call_frame_t *frame,
-	    xlator_t *this,
-	    loc_t *loc)
-{
-	int32_t op_errno = 1;
-	xlator_t *subvol   = NULL;
-
-        VALIDATE_OR_GOTO (frame, err);
-        VALIDATE_OR_GOTO (this, err);
-        VALIDATE_OR_GOTO (loc, err);
-        VALIDATE_OR_GOTO (loc->inode, err);
-        VALIDATE_OR_GOTO (loc->path, err);
-
-	subvol = get_mapping_subvol_from_ctx (this, loc->inode);
-	if (!subvol) {
-		op_errno = EINVAL;
-		goto err;
-	}
-
-	STACK_WIND (frame, map_unlink_cbk, subvol, subvol->fops->unlink, loc);
-
-	return 0;
- err:
-	STACK_UNWIND (frame, -1, op_errno, NULL, NULL);
-
-	return 0;
-}
-
-int32_t
-map_rmdir (call_frame_t *frame,
-	   xlator_t *this,
-	   loc_t *loc)
-{
-	int32_t op_errno = 1;
-	xlator_t *subvol   = NULL;
-
-        VALIDATE_OR_GOTO (frame, err);
-        VALIDATE_OR_GOTO (this, err);
-        VALIDATE_OR_GOTO (loc, err);
-        VALIDATE_OR_GOTO (loc->inode, err);
-        VALIDATE_OR_GOTO (loc->path, err);
-
-	subvol = get_mapping_subvol_from_ctx (this, loc->inode);
-	if (!subvol) {
-		op_errno = EINVAL;
-		goto err;
-	}
-
-	STACK_WIND (frame, map_rmdir_cbk, subvol, subvol->fops->rmdir, loc);
-
-	return 0;
- err:
-	STACK_UNWIND (frame, -1, op_errno, NULL, NULL);
-
-	return 0;
-}
-
-
-int32_t
-map_rename (call_frame_t *frame,
-	    xlator_t *this,
-	    loc_t *oldloc,
-	    loc_t *newloc)
-{
-	int32_t op_errno = 1;
-	xlator_t *old_subvol = NULL;
-	xlator_t *new_subvol = NULL;
-
-        VALIDATE_OR_GOTO (frame, err);
-        VALIDATE_OR_GOTO (this, err);
-        VALIDATE_OR_GOTO (oldloc, err);
-        VALIDATE_OR_GOTO (oldloc->inode, err);
-        VALIDATE_OR_GOTO (oldloc->path, err);
-        VALIDATE_OR_GOTO (newloc, err);
-
-	old_subvol = get_mapping_subvol_from_ctx (this, oldloc->inode);
-	if (!old_subvol) {
-		op_errno = EINVAL;
-		goto err;
-	}
-
-	if (newloc->path) {
-		new_subvol = get_mapping_subvol_from_path (this, newloc->path);
-		if (new_subvol && (new_subvol != old_subvol)) {
-			op_errno = EXDEV;
-			goto err;
-		}
-	}
-
-	STACK_WIND (frame, map_rename_cbk, old_subvol,
-		    old_subvol->fops->rename, oldloc, newloc);
-
-	return 0;
- err:
-	STACK_UNWIND (frame, -1, op_errno, NULL, NULL);
-
-	return 0;
-}
-
-
-int32_t
-map_link (call_frame_t *frame,
-	  xlator_t *this,
-	  loc_t *oldloc,
-	  loc_t *newloc)
-{
-	int32_t op_errno = 1;
-	xlator_t *old_subvol = NULL;
-	xlator_t *new_subvol = NULL;
-
-        VALIDATE_OR_GOTO (frame, err);
-        VALIDATE_OR_GOTO (this, err);
-        VALIDATE_OR_GOTO (oldloc, err);
-        VALIDATE_OR_GOTO (oldloc->inode, err);
-        VALIDATE_OR_GOTO (oldloc->path, err);
-        VALIDATE_OR_GOTO (newloc, err);
-
-	old_subvol = get_mapping_subvol_from_ctx (this, oldloc->inode);
-	if (!old_subvol) {
-		op_errno = EINVAL;
-		goto err;
-	}
-
-	if (newloc->path) {
-		new_subvol = get_mapping_subvol_from_path (this, newloc->path);
-		if (new_subvol && (new_subvol != old_subvol)) {
-			op_errno = EXDEV;
-			goto err;
-		}
-	}
-
-	STACK_WIND (frame, map_link_cbk, old_subvol,
-		    old_subvol->fops->link, oldloc, newloc);
-
-	return 0;
- err:
-	STACK_UNWIND (frame, -1, op_errno, NULL, NULL);
-
-	return 0;
-}
-
-
-int32_t
-map_open (call_frame_t *frame,
-	  xlator_t *this,
-	  loc_t *loc,
-	  int32_t flags, fd_t *fd, int wbflags)
-{
-	int32_t op_errno = 1;
-	xlator_t *subvol   = NULL;
-
-        VALIDATE_OR_GOTO (frame, err);
-        VALIDATE_OR_GOTO (this, err);
-        VALIDATE_OR_GOTO (loc, err);
-        VALIDATE_OR_GOTO (loc->inode, err);
-        VALIDATE_OR_GOTO (loc->path, err);
-
-	subvol = get_mapping_subvol_from_ctx (this, loc->inode);
-	if (!subvol) {
-		op_errno = EINVAL;
-		goto err;
-	}
-
-	STACK_WIND (frame, map_open_cbk, subvol, 
-                    subvol->fops->open, loc, flags, fd, wbflags);
-
-	return 0;
- err:
-	STACK_UNWIND (frame, -1, op_errno, NULL, NULL);
-
-	return 0;
-}
-
-int32_t
-map_readv (call_frame_t *frame,
-	   xlator_t *this,
-	   fd_t *fd,
-	   size_t size,
-	   off_t offset)
-{
-	int32_t op_errno = 1;
-	xlator_t *subvol   = NULL;
-
-        VALIDATE_OR_GOTO (frame, err);
-        VALIDATE_OR_GOTO (this, err);
-        VALIDATE_OR_GOTO (fd, err);
-        VALIDATE_OR_GOTO (fd->inode, err);
-
-	subvol = get_mapping_subvol_from_ctx (this, fd->inode);
-	if (!subvol) {
-		op_errno = EINVAL;
-		goto err;
-	}
-
-	STACK_WIND (frame, map_readv_cbk, subvol,
-		    subvol->fops->readv, fd, size, offset);
-
-	return 0;
- err:
-	STACK_UNWIND (frame, -1, op_errno, NULL, 0, NULL, NULL);
-
-	return 0;
-}
-
-int32_t
-map_writev (call_frame_t *frame,
-	    xlator_t *this,
-	    fd_t *fd,
-	    struct iovec *vector,
-	    int32_t count,
-	    off_t off,
-            struct iobref *iobref)
-{
-	int32_t op_errno = 1;
-	xlator_t *subvol   = NULL;
-
-        VALIDATE_OR_GOTO (frame, err);
-        VALIDATE_OR_GOTO (this, err);
-        VALIDATE_OR_GOTO (fd, err);
-        VALIDATE_OR_GOTO (fd->inode, err);
-
-	subvol = get_mapping_subvol_from_ctx (this, fd->inode);
-	if (!subvol) {
-		op_errno = EINVAL;
-		goto err;
-	}
-
-	STACK_WIND (frame, map_writev_cbk, subvol,
-		    subvol->fops->writev, fd, vector, count, off, iobref);
-
-	return 0;
- err:
-	STACK_UNWIND (frame, -1, op_errno, NULL, NULL);
-
-	return 0;
-}
-
-int32_t
-map_flush (call_frame_t *frame,
-	   xlator_t *this,
-	   fd_t *fd)
-{
-	int32_t op_errno = 1;
-	xlator_t *subvol   = NULL;
-
-        VALIDATE_OR_GOTO (frame, err);
-        VALIDATE_OR_GOTO (this, err);
-        VALIDATE_OR_GOTO (fd, err);
-        VALIDATE_OR_GOTO (fd->inode, err);
-
-	subvol = get_mapping_subvol_from_ctx (this, fd->inode);
-	if (!subvol) {
-		op_errno = EINVAL;
-		goto err;
-	}
-
-	STACK_WIND (frame, map_flush_cbk, subvol, subvol->fops->flush, fd);
-
-	return 0;
- err:
-	STACK_UNWIND (frame, -1, op_errno, NULL, NULL);
-
-	return 0;
-}
-
-
-int32_t
-map_fsync (call_frame_t *frame,
-	   xlator_t *this,
-	   fd_t *fd,
-	   int32_t flags)
-{
-	int32_t op_errno = 1;
-	xlator_t *subvol   = NULL;
-
-        VALIDATE_OR_GOTO (frame, err);
-        VALIDATE_OR_GOTO (this, err);
-        VALIDATE_OR_GOTO (fd, err);
-        VALIDATE_OR_GOTO (fd->inode, err);
-
-	subvol = get_mapping_subvol_from_ctx (this, fd->inode);
-	if (!subvol) {
-		op_errno = EINVAL;
-		goto err;
-	}
-
-	STACK_WIND (frame, map_fsync_cbk, subvol,
-		    subvol->fops->fsync, fd, flags);
-
-	return 0;
- err:
-	STACK_UNWIND (frame, -1, op_errno, NULL, NULL);
-
-	return 0;
-}
-
-int32_t
-map_fstat (call_frame_t *frame,
-	   xlator_t *this,
-	   fd_t *fd)
-{
-	int32_t op_errno = 1;
-	xlator_t *subvol   = NULL;
-
-        VALIDATE_OR_GOTO (frame, err);
-        VALIDATE_OR_GOTO (this, err);
-        VALIDATE_OR_GOTO (fd, err);
-        VALIDATE_OR_GOTO (fd->inode, err);
-
-	subvol = get_mapping_subvol_from_ctx (this, fd->inode);
-	if (!subvol) {
-		op_errno = EINVAL;
-		goto err;
-	}
-
-	STACK_WIND (frame, map_fstat_cbk, subvol, subvol->fops->fstat, fd);
-
-	return 0;
- err:
-	STACK_UNWIND (frame, -1, op_errno, NULL, NULL);
-
-	return 0;
-}
-
-int32_t
-map_getdents (call_frame_t *frame,
-	      xlator_t *this,
-	      fd_t *fd,
-	      size_t size,
-	      off_t offset,
-	      int32_t flag)
-{
-	int32_t op_errno = 1;
-	xlator_t *subvol   = NULL;
-
-        VALIDATE_OR_GOTO (frame, err);
-        VALIDATE_OR_GOTO (this, err);
-        VALIDATE_OR_GOTO (fd, err);
-        VALIDATE_OR_GOTO (fd->inode, err);
-
-	subvol = get_mapping_subvol_from_ctx (this, fd->inode);
-	if (!subvol) {
-		op_errno = EINVAL;
-		goto err;
-	}
-
-	STACK_WIND (frame, map_getdents_cbk, subvol,
-		    subvol->fops->getdents, fd, size, offset, flag);
-
-	return 0;
- err:
-	STACK_UNWIND (frame, -1, op_errno, NULL, NULL);
-
-	return 0;
-}
-
-int32_t
-map_setdents (call_frame_t *frame,
-	      xlator_t *this,
-	      fd_t *fd,
-	      int32_t flags,
-	      dir_entry_t *entries,
-	      int32_t count)
-{
-	int32_t op_errno = 1;
-	xlator_t *subvol   = NULL;
-
-        VALIDATE_OR_GOTO (frame, err);
-        VALIDATE_OR_GOTO (this, err);
-        VALIDATE_OR_GOTO (fd, err);
-        VALIDATE_OR_GOTO (fd->inode, err);
-
-	subvol = get_mapping_subvol_from_ctx (this, fd->inode);
-	if (!subvol) {
-		op_errno = EINVAL;
-		goto err;
-	}
-
-	STACK_WIND (frame, map_setdents_cbk, subvol,
-		    subvol->fops->setdents, fd, flags, entries, count);
-
-	return 0;
- err:
-	STACK_UNWIND (frame, -1, op_errno, NULL, NULL);
-
-	return 0;
-}
-
-
-int32_t
-map_fsyncdir (call_frame_t *frame,
-	      xlator_t *this,
-	      fd_t *fd,
-	      int32_t flags)
-{
-	int32_t op_errno = 1;
-	xlator_t *subvol   = NULL;
-
-        VALIDATE_OR_GOTO (frame, err);
-        VALIDATE_OR_GOTO (this, err);
-        VALIDATE_OR_GOTO (fd, err);
-        VALIDATE_OR_GOTO (fd->inode, err);
-
-	subvol = get_mapping_subvol_from_ctx (this, fd->inode);
-	if (!subvol) {
-		op_errno = EINVAL;
-		goto err;
-	}
-
-	STACK_WIND (frame, map_fsyncdir_cbk, subvol,
-		    subvol->fops->fsyncdir, fd, flags);
-
-	return 0;
- err:
-	STACK_UNWIND (frame, -1, op_errno, NULL, NULL);
-
-	return 0;
-}
-
-
-
-
-int32_t
-map_setxattr (call_frame_t *frame,
-	      xlator_t *this,
-	      loc_t *loc,
-	      dict_t *dict,
-	      int32_t flags)
-{
-	int32_t op_errno = 1;
-	xlator_t *subvol   = NULL;
-
-        VALIDATE_OR_GOTO (frame, err);
-        VALIDATE_OR_GOTO (this, err);
-        VALIDATE_OR_GOTO (loc, err);
-        VALIDATE_OR_GOTO (loc->inode, err);
-        VALIDATE_OR_GOTO (loc->path, err);
-
-	subvol = get_mapping_subvol_from_ctx (this, loc->inode);
-	if (!subvol) {
-		op_errno = EINVAL;
-		goto err;
-	}
-
-	STACK_WIND (frame, map_setxattr_cbk, subvol,
-		    subvol->fops->setxattr, loc, dict, flags);
-
-	return 0;
- err:
-	STACK_UNWIND (frame, -1, op_errno, NULL, NULL);
-
-	return 0;
-}
-
-int32_t
-map_getxattr (call_frame_t *frame,
-	      xlator_t *this,
-	      loc_t *loc,
-	      const char *name)
-{
-	int32_t op_errno = 1;
-	xlator_t *subvol   = NULL;
-
-        VALIDATE_OR_GOTO (frame, err);
-        VALIDATE_OR_GOTO (this, err);
-        VALIDATE_OR_GOTO (loc, err);
-        VALIDATE_OR_GOTO (loc->inode, err);
-        VALIDATE_OR_GOTO (loc->path, err);
-
-	subvol = get_mapping_subvol_from_ctx (this, loc->inode);
-	if (!subvol) {
-		op_errno = EINVAL;
-		goto err;
-	}
-
-	STACK_WIND (frame, map_getxattr_cbk, subvol,
-		    subvol->fops->getxattr, loc, name);
-
-	return 0;
- err:
-	STACK_UNWIND (frame, -1, op_errno, NULL, NULL);
-
-	return 0;
-}
-
-
-int32_t
-map_fsetxattr (call_frame_t *frame,
-               xlator_t *this,
-               fd_t *fd,
-               dict_t *dict,
-               int32_t flags)
-{
-	int32_t   op_errno = 1;
-	xlator_t *subvol   = NULL;
-
-        VALIDATE_OR_GOTO (frame, err);
-        VALIDATE_OR_GOTO (this, err);
-        VALIDATE_OR_GOTO (fd, err);
-        VALIDATE_OR_GOTO (fd->inode, err);
-
-	subvol = get_mapping_subvol_from_ctx (this, fd->inode);
-	if (!subvol) {
-		op_errno = EINVAL;
-		goto err;
-	}
-
-	STACK_WIND (frame, map_fsetxattr_cbk, subvol, 
-                    subvol->fops->fsetxattr, fd, dict, flags);
-
-	return 0;
- err:
-	STACK_UNWIND (frame, -1, op_errno, NULL, NULL);
-
-	return 0;
-}
-
-int32_t
-map_fgetxattr (call_frame_t *frame,
-               xlator_t *this,
-               fd_t *fd,
-               const char *name)
-{
-	int32_t   op_errno = 1;
-	xlator_t *subvol   = NULL;
-
-        VALIDATE_OR_GOTO (frame, err);
-        VALIDATE_OR_GOTO (this, err);
-        VALIDATE_OR_GOTO (fd, err);
-        VALIDATE_OR_GOTO (fd->inode, err);
-
-	subvol = get_mapping_subvol_from_ctx (this, fd->inode);
-	if (!subvol) {
-		op_errno = EINVAL;
-		goto err;
-	}
-
-	STACK_WIND (frame, map_fgetxattr_cbk, subvol,
-		    subvol->fops->fgetxattr, fd, name);
-
-	return 0;
- err:
-	STACK_UNWIND (frame, -1, op_errno, NULL, NULL);
-
-	return 0;
-}
-
-
-int32_t
-map_xattrop (call_frame_t *frame,
-	     xlator_t *this,
-	     loc_t *loc,
-	     gf_xattrop_flags_t flags,
-	     dict_t *dict)
-{
-	int32_t op_errno = 1;
-	xlator_t *subvol   = NULL;
-
-        VALIDATE_OR_GOTO (frame, err);
-        VALIDATE_OR_GOTO (this, err);
-        VALIDATE_OR_GOTO (loc, err);
-        VALIDATE_OR_GOTO (loc->inode, err);
-        VALIDATE_OR_GOTO (loc->path, err);
-
-	subvol = get_mapping_subvol_from_ctx (this, loc->inode);
-	if (!subvol) {
-		op_errno = EINVAL;
-		goto err;
-	}
-
-	STACK_WIND (frame, map_xattrop_cbk, subvol,
-		    subvol->fops->xattrop, loc, flags, dict);
-
-	return 0;
- err:
-	STACK_UNWIND (frame, -1, op_errno, NULL, NULL);
-
-	return 0;
-}
-
-int32_t
-map_fxattrop (call_frame_t *frame,
-	      xlator_t *this,
-	      fd_t *fd,
-	      gf_xattrop_flags_t flags,
-	      dict_t *dict)
-{
-	int32_t op_errno = 1;
-	xlator_t *subvol   = NULL;
-
-        VALIDATE_OR_GOTO (frame, err);
-        VALIDATE_OR_GOTO (this, err);
-        VALIDATE_OR_GOTO (fd, err);
-        VALIDATE_OR_GOTO (fd->inode, err);
-
-	subvol = get_mapping_subvol_from_ctx (this, fd->inode);
-	if (!subvol) {
-		op_errno = EINVAL;
-		goto err;
-	}
-
-	STACK_WIND (frame, map_fxattrop_cbk, subvol,
-		    subvol->fops->fxattrop, fd, flags, dict);
-
-        return 0;
- err:
-	STACK_UNWIND (frame, -1, op_errno, NULL, NULL);
-
-	return 0;
-}
-
-int32_t
-map_removexattr (call_frame_t *frame,
-		 xlator_t *this,
-		 loc_t *loc,
-		 const char *name)
-{
-	int32_t op_errno = 1;
-	xlator_t *subvol   = NULL;
-
-        VALIDATE_OR_GOTO (frame, err);
-        VALIDATE_OR_GOTO (this, err);
-        VALIDATE_OR_GOTO (loc, err);
-        VALIDATE_OR_GOTO (loc->inode, err);
-        VALIDATE_OR_GOTO (loc->path, err);
-
-	subvol = get_mapping_subvol_from_ctx (this, loc->inode);
-	if (!subvol) {
-		op_errno = EINVAL;
-		goto err;
-	}
-
-	STACK_WIND (frame, map_removexattr_cbk, subvol,
-		    subvol->fops->removexattr, loc, name);
-
-	return 0;
- err:
-	STACK_UNWIND (frame, -1, op_errno, NULL, NULL);
-
-	return 0;
-}
-
-int32_t
-map_lk (call_frame_t *frame,
-	xlator_t *this,
-	fd_t *fd,
-	int32_t cmd,
-	struct gf_flock *lock)
-{
-	int32_t op_errno = 1;
-	xlator_t *subvol   = NULL;
-
-        VALIDATE_OR_GOTO (frame, err);
-        VALIDATE_OR_GOTO (this, err);
-        VALIDATE_OR_GOTO (fd, err);
-        VALIDATE_OR_GOTO (fd->inode, err);
-
-	subvol = get_mapping_subvol_from_ctx (this, fd->inode);
-	if (!subvol) {
-		op_errno = EINVAL;
-		goto err;
-	}
-
-	STACK_WIND (frame, map_lk_cbk, subvol,
-		    subvol->fops->lk, fd, cmd, lock);
-
-	return 0;
- err:
-	STACK_UNWIND (frame, -1, op_errno, NULL, NULL);
-
-	return 0;
-}
-
-
-int32_t
-map_inodelk (call_frame_t *frame, xlator_t *this,
-	     const char *volume, loc_t *loc, int32_t cmd, struct gf_flock *lock)
-{
-	int32_t op_errno = 1;
-	xlator_t *subvol   = NULL;
-
-        VALIDATE_OR_GOTO (frame, err);
-        VALIDATE_OR_GOTO (this, err);
-        VALIDATE_OR_GOTO (loc, err);
-        VALIDATE_OR_GOTO (loc->inode, err);
-        VALIDATE_OR_GOTO (loc->path, err);
-
-	subvol = get_mapping_subvol_from_ctx (this, loc->inode);
-	if (!subvol) {
-		op_errno = EINVAL;
-		goto err;
-	}
-
-	STACK_WIND (frame, map_inodelk_cbk, subvol,
-		    subvol->fops->inodelk, volume, loc, cmd, lock);
-
-	return 0;
- err:
-	STACK_UNWIND (frame, -1, op_errno, NULL, NULL);
-
-	return 0;
-}
-
-
-int32_t
-map_finodelk (call_frame_t *frame, xlator_t *this,
-	      const char *volume, fd_t *fd, int32_t cmd, struct gf_flock *lock)
-{
-	int32_t op_errno = 1;
-	xlator_t *subvol   = NULL;
-
-        VALIDATE_OR_GOTO (frame, err);
-        VALIDATE_OR_GOTO (this, err);
-        VALIDATE_OR_GOTO (fd, err);
-        VALIDATE_OR_GOTO (fd->inode, err);
-
-	subvol = get_mapping_subvol_from_ctx (this, fd->inode);
-	if (!subvol) {
-		op_errno = EINVAL;
-		goto err;
-	}
-
-	STACK_WIND (frame, map_finodelk_cbk, subvol,
-		    subvol->fops->finodelk, volume, fd, cmd, lock);
-
-	return 0;
- err:
-	STACK_UNWIND (frame, -1, op_errno, NULL, NULL);
-
-	return 0;
-}
-
-int32_t
-map_entrylk (call_frame_t *frame, xlator_t *this,
-	     const char *volume, loc_t *loc, const char *basename,
-	     entrylk_cmd cmd, entrylk_type type)
-{
-	int32_t op_errno = 1;
-	xlator_t *subvol   = NULL;
-
-        VALIDATE_OR_GOTO (frame, err);
-        VALIDATE_OR_GOTO (this, err);
-        VALIDATE_OR_GOTO (loc, err);
-        VALIDATE_OR_GOTO (loc->inode, err);
-        VALIDATE_OR_GOTO (loc->path, err);
-
-	subvol = get_mapping_subvol_from_ctx (this, loc->inode);
-	if (!subvol) {
-		op_errno = EINVAL;
-		goto err;
-	}
-
-	STACK_WIND (frame, map_entrylk_cbk, subvol,
-		    subvol->fops->entrylk, volume, loc, basename, cmd, type);
-
-	return 0;
- err:
-	STACK_UNWIND (frame, -1, op_errno, NULL, NULL);
-
-	return 0;
-}
-
-int32_t
-map_fentrylk (call_frame_t *frame, xlator_t *this,
-	      const char *volume, fd_t *fd, const char *basename,
-	      entrylk_cmd cmd, entrylk_type type)
-{
-	int32_t op_errno = 1;
-	xlator_t *subvol   = NULL;
-
-        VALIDATE_OR_GOTO (frame, err);
-        VALIDATE_OR_GOTO (this, err);
-        VALIDATE_OR_GOTO (fd, err);
-        VALIDATE_OR_GOTO (fd->inode, err);
-
-	subvol = get_mapping_subvol_from_ctx (this, fd->inode);
-	if (!subvol) {
-		op_errno = EINVAL;
-		goto err;
-	}
-
-	STACK_WIND (frame, map_fentrylk_cbk, subvol,
-		    subvol->fops->fentrylk, volume, fd, basename, cmd, type);
-
-	return 0;
- err:
-	STACK_UNWIND (frame, -1, op_errno, NULL, NULL);
-
-	return 0;
-}
-
-int32_t
-map_checksum (call_frame_t *frame,
-	      xlator_t *this,
-	      loc_t *loc,
-	      int32_t flag)
-{
-	int32_t op_errno = 1;
-	xlator_t *subvol   = NULL;
-
-        VALIDATE_OR_GOTO (frame, err);
-        VALIDATE_OR_GOTO (this, err);
-        VALIDATE_OR_GOTO (loc, err);
-        VALIDATE_OR_GOTO (loc->inode, err);
-        VALIDATE_OR_GOTO (loc->path, err);
-
-	subvol = get_mapping_subvol_from_ctx (this, loc->inode);
-	if (!subvol) {
-		op_errno = EINVAL;
-		goto err;
-	}
-
-	STACK_WIND (frame, map_checksum_cbk, subvol,
-		    subvol->fops->checksum, loc, flag);
-
-	return 0;
- err:
-	STACK_UNWIND (frame, -1, op_errno, NULL, NULL);
-
-	return 0;
-}
-
-
-int32_t
-map_mknod (call_frame_t *frame,
-	   xlator_t *this,
-	   loc_t *loc,
-	   mode_t mode,
-	   dev_t rdev)
-{
-	int32_t op_errno = 1;
-	xlator_t *subvol   = NULL;
-
-        VALIDATE_OR_GOTO (frame, err);
-        VALIDATE_OR_GOTO (this, err);
-        VALIDATE_OR_GOTO (loc, err);
-        VALIDATE_OR_GOTO (loc->path, err);
-        VALIDATE_OR_GOTO (loc->inode, err);
-
-	subvol = get_mapping_subvol_from_path (this, loc->path);
-	if (!subvol) {
-		op_errno = EINVAL;
-		goto err;
-	}
-	
-	op_errno = inode_ctx_put (loc->inode, this, (uint64_t)(long)subvol);
-	if (op_errno != 0) {
-		gf_log (this->name, GF_LOG_ERROR,
-			"%s: failed to set subvolume ptr in inode ctx",
-			loc->path);
-	}
-
-	STACK_WIND (frame, map_newentry_cbk, subvol,
-		    subvol->fops->mknod, loc, mode, rdev);
-
-	return 0;
- err:
-	STACK_UNWIND (frame, -1, op_errno, NULL, NULL);
-
-	return 0;
-}
-
-int32_t
-map_mkdir (call_frame_t *frame,
-	   xlator_t *this,
-	   loc_t *loc,
-	   mode_t mode)
-{
-	int32_t op_errno = 1;
-	xlator_t *subvol   = NULL;
-
-        VALIDATE_OR_GOTO (frame, err);
-        VALIDATE_OR_GOTO (this, err);
-        VALIDATE_OR_GOTO (loc, err);
-        VALIDATE_OR_GOTO (loc->path, err);
-        VALIDATE_OR_GOTO (loc->inode, err);
-
-	subvol = get_mapping_subvol_from_path (this, loc->path);
-	if (!subvol) {
-		op_errno = EINVAL;
-		goto err;
-	}
-
-	op_errno = inode_ctx_put (loc->inode, this, (uint64_t)(long)subvol);
-	if (op_errno != 0) {
-		gf_log (this->name, GF_LOG_ERROR,
-			"%s: failed to set subvolume ptr in inode ctx",
-			loc->path);
-	}
-
-	STACK_WIND (frame, map_newentry_cbk, subvol,
-		    subvol->fops->mkdir, loc, mode);
-	return 0;
- err:
-	STACK_UNWIND (frame, -1, op_errno, NULL, NULL);
-
-	return 0;
-}
-
-
-int32_t
-map_symlink (call_frame_t *frame,
-	     xlator_t *this,
-	     const char *linkpath,
-	     loc_t *loc)
-{
-	int32_t op_errno = 1;
-	xlator_t *subvol   = NULL;
-
-        VALIDATE_OR_GOTO (frame, err);
-        VALIDATE_OR_GOTO (this, err);
-        VALIDATE_OR_GOTO (loc, err);
-        VALIDATE_OR_GOTO (loc->path, err);
-        VALIDATE_OR_GOTO (loc->inode, err);
-
-	subvol = get_mapping_subvol_from_path (this, loc->path);
-	if (!subvol) {
-		op_errno = EINVAL;
-		goto err;
-	}
-	
-	op_errno = inode_ctx_put (loc->inode, this, (uint64_t)(long)subvol);
-	if (op_errno != 0) {
-		gf_log (this->name, GF_LOG_ERROR,
-			"%s: failed to set subvolume ptr in inode ctx",
-			loc->path);
-	}
-
-	STACK_WIND (frame, map_newentry_cbk, subvol,
-		    subvol->fops->symlink, linkpath, loc);
-
-	return 0;
- err:
-	STACK_UNWIND (frame, -1, op_errno, NULL, NULL);
-
-	return 0;
-}
-
-int32_t
-map_create (call_frame_t *frame,
-	    xlator_t *this,
-	    loc_t *loc,
-	    int32_t flags,
-	    mode_t mode, fd_t *fd)
-{
-	int32_t op_errno = 1;
-	xlator_t *subvol   = NULL;
-
-        VALIDATE_OR_GOTO (frame, err);
-        VALIDATE_OR_GOTO (this, err);
-        VALIDATE_OR_GOTO (loc, err);
-        VALIDATE_OR_GOTO (loc->path, err);
-        VALIDATE_OR_GOTO (loc->inode, err);
-
-	subvol = get_mapping_subvol_from_path (this, loc->path);
-	if (!subvol) {
-		op_errno = EINVAL;
-		goto err;
-	}
-
-	op_errno = inode_ctx_put (loc->inode, this, (uint64_t)(long)subvol);
-	if (op_errno != 0) {
-		gf_log (this->name, GF_LOG_ERROR,
-			"%s: failed to set subvolume ptr in inode ctx",
-			loc->path);
-	}
-
-	STACK_WIND (frame, map_create_cbk, subvol,
-		    subvol->fops->create, loc, flags, mode, fd);
-
-	return 0;
- err:
-	STACK_UNWIND (frame, -1, op_errno, NULL, NULL);
-
-	return 0;
-}
-
-
-int32_t
-map_lookup (call_frame_t *frame,
-	    xlator_t *this,
-	    loc_t *loc,
-	    dict_t *xattr_req)
-{
-	int32_t op_errno = EINVAL;
-	xlator_t *subvol   = NULL;
-	map_local_t *local = NULL;
-	map_private_t *priv = NULL;
-	xlator_list_t *trav = NULL;
-
-        VALIDATE_OR_GOTO (frame, err);
-        VALIDATE_OR_GOTO (this, err);
-        VALIDATE_OR_GOTO (loc, err);
-        VALIDATE_OR_GOTO (loc->path, err);
-        VALIDATE_OR_GOTO (loc->inode, err);
-
-	priv = this->private;
-
-	if (loc->inode->ino == 1)
-		goto root_inode;
-
-	subvol = get_mapping_subvol_from_ctx (this, loc->inode);
-	if (!subvol) {
-		subvol = get_mapping_subvol_from_path (this, loc->path);
-		if (!subvol) {
-			goto err;
-		}
-
-		op_errno = inode_ctx_put (loc->inode, this, 
-					  (uint64_t)(long)subvol);
-		if (op_errno != 0) {
-			gf_log (this->name, GF_LOG_ERROR,
-				"%s: failed to set subvolume in inode ctx",
-				loc->path);
-		}
-	}
-
-	/* Just one callback */
-	STACK_WIND (frame, map_single_lookup_cbk, subvol,
-		    subvol->fops->lookup, loc, xattr_req);
-
-	return 0;
-
- root_inode:
-	local = GF_CALLOC (1, sizeof (map_local_t),
-                           gf_map_mt_map_local_t);
-
-	frame->local = local;
-	local->call_count = priv->child_count;
-	local->op_ret = -1;
-
-	trav = this->children;
-	while (trav) {
-		STACK_WIND (frame, map_root_lookup_cbk, trav->xlator,
-			    trav->xlator->fops->lookup, loc, xattr_req);
-		trav = trav->next;
-	}
-
-	return 0;
-
- err:
-	STACK_UNWIND (frame, -1, op_errno, NULL, NULL);
-
-	return 0;
-}
-
-
-int32_t
-map_statfs (call_frame_t *frame,
-	    xlator_t *this,
-	    loc_t *loc)
-{
-	int32_t op_errno = EINVAL;
-	xlator_t *subvol   = NULL;
-	map_local_t *local = NULL;
-	map_private_t *priv = NULL;
-	xlator_list_t *trav = NULL;
-
-        VALIDATE_OR_GOTO (frame, err);
-        VALIDATE_OR_GOTO (this, err);
-        VALIDATE_OR_GOTO (loc, err);
-        VALIDATE_OR_GOTO (loc->path, err);
-        VALIDATE_OR_GOTO (loc->inode, err);
-
-	if (loc->inode->ino == 1)
-		goto root_inode;
-	subvol = get_mapping_subvol_from_ctx (this, loc->inode);
-	if (!subvol) {
-		goto err;
-	}
-	
-	/* Just one callback */
-	STACK_WIND (frame, map_statfs_cbk, subvol, subvol->fops->statfs, loc);
-
-	return 0;
-
- root_inode:
-	local = GF_CALLOC (1, sizeof (map_local_t),
-                           gf_map_mt_map_local_t);
-
-	priv = this->private;
-	frame->local = local;
-	local->call_count = priv->child_count;
-	local->op_ret = -1;
-
-	trav = this->children;
-	while (trav) {
-		STACK_WIND (frame, map_statfs_cbk, trav->xlator,
-			    trav->xlator->fops->statfs, loc);
-		trav = trav->next;
-	}
-
-	return 0;
- err:
-	STACK_UNWIND (frame, -1, op_errno, NULL);
-
-	return 0;
-}
-
-int32_t
-map_opendir (call_frame_t *frame,
-	     xlator_t *this,
-	     loc_t *loc, fd_t *fd)
-{
-	int32_t op_errno = EINVAL;
-	xlator_t *subvol   = NULL;
-	map_local_t *local = NULL;
-	map_private_t *priv = NULL;
-	xlator_list_t *trav = NULL;
-
-        VALIDATE_OR_GOTO (frame, err);
-        VALIDATE_OR_GOTO (this, err);
-        VALIDATE_OR_GOTO (fd, err);
-        VALIDATE_OR_GOTO (fd->inode, err);
-
-	if (loc->inode->ino == 1) 
-		goto root_inode;
-	
-	subvol = get_mapping_subvol_from_ctx (this, fd->inode);
-	if (!subvol) {
-		goto err;
-	}
-	
-	/* Just one callback */
-	STACK_WIND (frame, map_opendir_cbk, subvol,
-		    subvol->fops->opendir, loc, fd);
-
-	return 0;
-
- root_inode:
-	local = GF_CALLOC (1, sizeof (map_local_t),
-                           gf_map_mt_map_local_t);
-
-	priv = this->private;
-	frame->local = local;
-	local->call_count = priv->child_count;
-	local->op_ret = -1;
-	local->fd = fd_ref (fd);
-
-	trav = this->children;
-	while (trav) {
-		STACK_WIND (frame, map_opendir_cbk, trav->xlator,
-			    trav->xlator->fops->opendir, loc, fd);
-		trav = trav->next;
-	}
-	
-	return 0;
- err:
-	STACK_UNWIND (frame, -1, op_errno, NULL);
-
-	return 0;
-}
-
-
-int32_t
-map_do_readdir (call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
-                off_t yoff, int whichop)
-{
-	int32_t        op_errno = EINVAL;
-	xlator_t      *subvol = NULL;
-	map_local_t   *local = NULL;
-	map_private_t *priv = NULL;
-	xlator_t      *xvol = NULL;
-	off_t          xoff = 0;
-
-        VALIDATE_OR_GOTO (frame, err);
-        VALIDATE_OR_GOTO (this, err);
-        VALIDATE_OR_GOTO (fd, err);
-        VALIDATE_OR_GOTO (fd->inode, err);
-
-	if (fd->inode->ino == 1) 
-		goto root_inode;
-
-	subvol = get_mapping_subvol_from_ctx (this, fd->inode);
-	if (!subvol) {
-		goto err;
-	}
-	
-	/* Just one callback */
-        if (whichop == GF_FOP_READDIR)
-                STACK_WIND (frame, map_single_readdir_cbk, subvol,
-                            subvol->fops->readdir, fd, size, yoff);
-        else
-                STACK_WIND (frame, map_single_readdirp_cbk, subvol,
-                            subvol->fops->readdirp, fd, size, yoff);
-
-	return 0;
-
- root_inode:
-	/* readdir on '/' */
-	local = GF_CALLOC (1, sizeof (map_local_t),
-                           gf_map_mt_map_local_t);
-	if (!local) {
-		gf_log (this->name, GF_LOG_ERROR,
-			"memory allocation failed :(");
-		op_errno = ENOMEM;
-		goto err;
-	}
-	
-	priv = this->private;
-	frame->local = local;
-	local->op_errno = ENOENT;
-	local->op_ret = -1;
-
-	local->fd = fd_ref (fd);
-	local->size = size;
-
-	map_deitransform (this, yoff, &xvol, (uint64_t *)&xoff);
-
-        if (whichop == GF_FOP_READDIR)
-                STACK_WIND (frame, map_readdir_cbk, xvol, xvol->fops->readdir,
-                            fd, size, xoff);
-        else
-                STACK_WIND (frame, map_readdirp_cbk, xvol, xvol->fops->readdirp,
-                            fd, size, xoff);
-
-	return 0;
- err:
-	STACK_UNWIND (frame, -1, op_errno, NULL);
-
-	return 0;
-}
-
-
-
-int32_t
-map_readdir (call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
-             off_t yoff)
-{
-        map_do_readdir (frame, this, fd, size, yoff, GF_FOP_READDIR);
-        return 0;
-}
-
-int32_t
-map_readdirp (call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
-              off_t yoff)
-{
-        map_do_readdir (frame, this, fd, size, yoff, GF_FOP_READDIRP);
-        return 0;
-}
-
-
-void
-fini (xlator_t *this)
-{
-	map_private_t *priv = NULL;
-	struct map_pattern *trav_map = NULL;
-	struct map_pattern *tmp_map  = NULL;
-
-	priv = this->private;
-
-	if (priv) {
-		GF_FREE (priv->xlarray);
-
-		trav_map = priv->map;
-		while (trav_map) {
-			tmp_map = trav_map;
-			trav_map = trav_map->next;
-			GF_FREE (tmp_map);
-		}
-
-		GF_FREE(priv);
-	}
-
-	return;
-}
-
-int32_t
-mem_acct_init (xlator_t *this)
-{
-        int     ret = -1;
-
-        if (!this)
-                return ret;
-
-        ret = xlator_mem_acct_init (this, gf_map_mt_end + 1);
-        
-        if (ret != 0) {
-                gf_log (this->name, GF_LOG_ERROR, "Memory accounting init"
-                                "failed");
-                return ret;
-        }
-
-        return ret;
-}
-
-int
-init (xlator_t *this)
-{
-	map_private_t *priv = NULL;
-	xlator_list_t *trav = NULL;
-	int   count = 0;
-	int   ret = -1;
-	char *pattern_string = NULL;
-	char *map_pair_str = NULL;
-	char *tmp_str = NULL;
-	char *tmp_str1 = NULL;
-	char *dup_map_pair = NULL;
-	char *dir_str = NULL;
-	char *subvol_str = NULL;
-	char *map_xl = NULL;
-
-
-	if (!this->children) {
-		gf_log (this->name,GF_LOG_ERROR,
-			"FATAL: map should have one or more child defined");
-		return -1;
-	}
-
-	if (!this->parents) {
-		gf_log (this->name, GF_LOG_WARNING,
-			"dangling volume. check volfile ");
-	}
-
-	priv = GF_CALLOC (1, sizeof (map_private_t),
-                          gf_map_mt_map_private_t);
-	this->private = priv;
-
-	/* allocate xlator array */
-	trav = this->children;
-	while (trav) {
-		count++;
-		trav = trav->next;
-	}
-	priv->xlarray = GF_CALLOC (1, sizeof (struct map_xlator_array) * count,
-                                   gf_map_mt_map_xlator_array);
-	priv->child_count = count;
-
-	/* build xlator array */
-	count = 0;
-	trav = this->children;
-	while (trav) {
-		priv->xlarray[count++].xl = trav->xlator;
-		trav = trav->next;
-	}
-
-	/* map dir1:brick1;dir2:brick2;dir3:brick3;*:brick4 */
-	ret = dict_get_str (this->options, "map-directory", &pattern_string);
-	if (ret != 0) {
-		gf_log (this->name, GF_LOG_ERROR, 
-			"map.pattern not given, can't continue");
-		goto err;
-	}
-	map_pair_str = strtok_r (pattern_string, ";", &tmp_str);
-	while (map_pair_str) {
-		dup_map_pair = gf_strdup (map_pair_str);
-		dir_str = strtok_r (dup_map_pair, ":", &tmp_str1);
-		if (!dir_str) {
-			gf_log (this->name, GF_LOG_ERROR, 
-				"directory string invalid");
-			goto err;
-		}
-		subvol_str = strtok_r (NULL, ":", &tmp_str1);
-		if (!subvol_str) {
-			gf_log (this->name, GF_LOG_ERROR, 
-				"mapping subvolume string invalid");
-			goto err;
-		}
-		ret = verify_dir_and_assign_subvol (this, 
-						    dir_str, 
-						    subvol_str);
-		if (ret != 0) {
-			gf_log (this->name, GF_LOG_ERROR, 
-				"verification failed");
-			goto err;
-		}
-		
-		GF_FREE (dup_map_pair);
-
-		map_pair_str = strtok_r (NULL, ";", &tmp_str);
-	}
-
-	/* default-volume brick4 */
-	ret = dict_get_str (this->options, "default-volume", &map_xl);
-	if (ret == 0) {
-		ret = assign_default_subvol (this, map_xl);
-		if (ret != 0) {
-			gf_log (this->name, GF_LOG_ERROR, 
-				"assigning default failed");
-			goto err;
-		}
-	}
-
-	verify_if_all_subvolumes_got_used (this);
-	
-	return 0;
- err:
-	fini (this);
-	return -1;
-}
-
-
-struct xlator_fops fops = {
-	.lookup      = map_lookup,
-	.mknod       = map_mknod,
-	.create      = map_create,
-
-	.stat        = map_stat,
-	.fstat       = map_fstat,
-	.truncate    = map_truncate,
-	.ftruncate   = map_ftruncate,
-	.access      = map_access,
-	.readlink    = map_readlink,
-	.setxattr    = map_setxattr,
-	.getxattr    = map_getxattr,
-	.fsetxattr   = map_fsetxattr,
-	.fgetxattr   = map_fgetxattr,
-	.removexattr = map_removexattr,
-	.open        = map_open,
-	.readv       = map_readv,
-	.writev      = map_writev,
-	.flush       = map_flush,
-	.fsync       = map_fsync,
-	.statfs      = map_statfs,
-	.lk          = map_lk,
-	.opendir     = map_opendir,
-	.readdir     = map_readdir,
-	.readdirp    = map_readdirp,
-	.fsyncdir    = map_fsyncdir,
-	.symlink     = map_symlink,
-	.unlink      = map_unlink,
-	.link        = map_link,
-	.mkdir       = map_mkdir,
-	.rmdir       = map_rmdir,
-	.rename      = map_rename,
-	.inodelk     = map_inodelk,
-	.finodelk    = map_finodelk,
-	.entrylk     = map_entrylk,
-	.fentrylk    = map_fentrylk,
-	.xattrop     = map_xattrop,
-	.fxattrop    = map_fxattrop,
-	.setdents    = map_setdents,
-	.getdents    = map_getdents,
-	.checksum    = map_checksum,
-        .setattr     = map_setattr,
-        .fsetattr    = map_fsetattr,
-};
-
-struct xlator_cbks cbks = {
-};
-
-struct volume_options options[] = {
-	{ .key   = {"map-directory"},  
-	  .type  = GF_OPTION_TYPE_ANY 
-	},
-	{ .key   = {"default-volume"},  
-	  .type  = GF_OPTION_TYPE_XLATOR 
-	},
-	
-	{ .key = {NULL} }
-};
diff --git a/xlators/cluster/map/src/map.h b/xlators/cluster/map/src/map.h
deleted file mode 100644
index 7703a543e28..00000000000
--- a/xlators/cluster/map/src/map.h
+++ /dev/null
@@ -1,67 +0,0 @@
-/*
-   Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
-   This file is part of GlusterFS.
-
-   This file is licensed to you under your choice of the GNU Lesser
-   General Public License, version 3 or any later version (LGPLv3 or
-   later), or the GNU General Public License, version 2 (GPLv2), in all
-   cases as published by the Free Software Foundation.
-*/
-#ifndef __MAP_H__
-#define __MAP_H__
-
-#include "xlator.h"
-#include "map-mem-types.h"
-
-struct map_pattern {
-	struct map_pattern *next;
-	xlator_t           *xl;
-	char               *directory;
-	int                 dir_len;
-};
-
-struct map_xlator_array {
-	xlator_t *xl;
-	int       mapped; /* yes/no */
-};
-
-typedef struct {
-	struct map_pattern      *map;
-	xlator_t                *default_xl;
-	struct map_xlator_array *xlarray;
-	int                      child_count;
-} map_private_t;
-
-typedef struct {
-	int32_t        op_ret;
-	int32_t        op_errno;
-	int            call_count;
-	struct statvfs statvfs;
-	struct iatt    stbuf;
-	inode_t       *inode;
-	dict_t        *dict;
-	fd_t          *fd;
-
-	size_t        size;
-} map_local_t;
-
-xlator_t *map_subvol_next (xlator_t *this, xlator_t *prev);
-int map_subvol_cnt (xlator_t *this, xlator_t *subvol);
-
-int map_itransform (xlator_t *this, xlator_t *subvol, 
-		    uint64_t x, uint64_t *y_p);
-int map_deitransform (xlator_t *this, uint64_t y, 
-		      xlator_t **subvol_p, uint64_t *x_p);
-
-
-xlator_t *get_mapping_subvol_from_path (xlator_t *this, const char *path);
-xlator_t *get_mapping_subvol_from_ctx (xlator_t *this, inode_t *inode);
-
-int check_multiple_volume_entry (xlator_t *this, xlator_t *subvol);
-int verify_dir_and_assign_subvol (xlator_t *this, 
-				  const char *directory, const char *subvol);
-int assign_default_subvol (xlator_t *this, const char *default_xl);
-void verify_if_all_subvolumes_got_used (xlator_t *this);
-
-
-#endif /* __MAP_H__ */
diff --git a/xlators/cluster/stripe/src/Makefile.am b/xlators/cluster/stripe/src/Makefile.am
deleted file mode 100644
index 4268d6f0382..00000000000
--- a/xlators/cluster/stripe/src/Makefile.am
+++ /dev/null
@@ -1,19 +0,0 @@
-xlator_LTLIBRARIES = stripe.la
-xlatordir = $(libdir)/glusterfs/$(PACKAGE_VERSION)/xlator/cluster
-
-stripe_la_LDFLAGS = -module -avoid-version
-
-stripe_la_SOURCES = stripe.c stripe-helpers.c \
-	$(top_builddir)/xlators/lib/src/libxlator.c
-
-stripe_la_LIBADD = $(top_builddir)/libglusterfs/src/libglusterfs.la
-
-noinst_HEADERS = stripe.h stripe-mem-types.h $(top_builddir)/xlators/lib/src/libxlator.h
-
-AM_CPPFLAGS = $(GF_CPPFLAGS) -I$(top_srcdir)/libglusterfs/src \
-	-I$(top_srcdir)/xlators/lib/src
-
-AM_CFLAGS = -Wall $(GF_CFLAGS)
-
-CLEANFILES = 
-
diff --git a/xlators/cluster/stripe/src/stripe-helpers.c b/xlators/cluster/stripe/src/stripe-helpers.c
deleted file mode 100644
index 3c12809d625..00000000000
--- a/xlators/cluster/stripe/src/stripe-helpers.c
+++ /dev/null
@@ -1,677 +0,0 @@
-/*
-  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
-  This file is part of GlusterFS.
-
-  This file is licensed to you under your choice of the GNU Lesser
-  General Public License, version 3 or any later version (LGPLv3 or
-  later), or the GNU General Public License, version 2 (GPLv2), in all
-  cases as published by the Free Software Foundation.
-*/
-
-#include <fnmatch.h>
-
-#include "stripe.h"
-#include "byte-order.h"
-#include "mem-types.h"
-#include "logging.h"
-
-void
-stripe_local_wipe (stripe_local_t *local)
-{
-        if (!local)
-                goto out;
-
-        loc_wipe (&local->loc);
-        loc_wipe (&local->loc2);
-
-        if (local->fd)
-                fd_unref (local->fd);
-
-        if (local->inode)
-                inode_unref (local->inode);
-
-        if (local->xattr)
-                dict_unref (local->xattr);
-
-        if (local->xdata)
-                dict_unref (local->xdata);
-
-out:
-        return;
-}
-
-
-
-int
-stripe_aggregate (dict_t *this, char *key, data_t *value, void *data)
-{
-        dict_t  *dst  = NULL;
-        int64_t *ptr  = 0, *size = NULL;
-        int32_t  ret  = -1;
-
-        dst = data;
-
-        if (strcmp (key, GF_XATTR_QUOTA_SIZE_KEY) == 0) {
-                ret = dict_get_bin (dst, key, (void **)&size);
-                if (ret < 0) {
-                        size = GF_CALLOC (1, sizeof (int64_t),
-                                          gf_common_mt_char);
-                        if (size == NULL) {
-                                gf_log ("stripe", GF_LOG_WARNING,
-                                        "memory allocation failed");
-                                goto out;
-                        }
-                        ret = dict_set_bin (dst, key, size, sizeof (int64_t));
-                        if (ret < 0) {
-                                gf_log ("stripe", GF_LOG_WARNING,
-                                        "stripe aggregate dict set failed");
-                                GF_FREE (size);
-                                goto out;
-                        }
-                }
-
-                ptr = data_to_bin (value);
-                if (ptr == NULL) {
-                        gf_log ("stripe", GF_LOG_WARNING, "data to bin failed");
-                        goto out;
-                }
-
-                *size = hton64 (ntoh64 (*size) + ntoh64 (*ptr));
-        } else if (strcmp (key, GF_CONTENT_KEY)) {
-                /* No need to aggregate 'CONTENT' data */
-                ret = dict_set (dst, key, value);
-                if (ret)
-                        gf_log ("stripe", GF_LOG_WARNING, "xattr dict set failed");
-        }
-
-out:
-        return 0;
-}
-
-
-void
-stripe_aggregate_xattr (dict_t *dst, dict_t *src)
-{
-        if ((dst == NULL) || (src == NULL)) {
-                goto out;
-        }
-
-        dict_foreach (src, stripe_aggregate, dst);
-out:
-        return;
-}
-
-
-int32_t
-stripe_xattr_aggregate (char *buffer, stripe_local_t *local, int32_t *total)
-{
-        int32_t              i     = 0;
-        int32_t              ret   = -1;
-        int32_t              len   = 0;
-        char                *sbuf  = NULL;
-        stripe_xattr_sort_t *xattr = NULL;
-
-        if (!buffer || !local || !local->xattr_list)
-                goto out;
-
-        sbuf = buffer;
-
-        for (i = 0; i < local->nallocs; i++) {
-                xattr = local->xattr_list + i;
-                len = xattr->xattr_len;
-
-                if (len && xattr && xattr->xattr_value) {
-                        memcpy (buffer, xattr->xattr_value, len);
-                        buffer += len;
-                        *buffer++ = ' ';
-                }
-        }
-
-        *--buffer = '\0';
-        if (total)
-                *total = buffer - sbuf;
-        ret = 0;
-
- out:
-        return ret;
-}
-
-int32_t
-stripe_free_xattr_str (stripe_local_t *local)
-{
-        int32_t              i     = 0;
-        int32_t              ret   = -1;
-        stripe_xattr_sort_t *xattr = NULL;
-
-        if (!local || !local->xattr_list)
-                goto out;
-
-        for (i = 0; i < local->nallocs; i++) {
-                xattr = local->xattr_list + i;
-
-                if (xattr && xattr->xattr_value)
-                        GF_FREE (xattr->xattr_value);
-        }
-
-        ret = 0;
- out:
-        return ret;
-}
-
-
-int32_t
-stripe_fill_lockinfo_xattr (xlator_t *this, stripe_local_t *local,
-                            void **xattr_serz)
-{
-        int32_t              ret   = -1, i = 0, len = 0;
-        dict_t              *tmp1  = NULL, *tmp2 = NULL;
-        char                *buf   = NULL;
-        stripe_xattr_sort_t *xattr = NULL;
-
-        if (xattr_serz == NULL) {
-                goto out;
-        }
-
-        tmp2 = dict_new ();
-
-        if (tmp2 == NULL) {
-                goto out;
-        }
-
-        for (i = 0; i < local->nallocs; i++) {
-                xattr = local->xattr_list + i;
-                len = xattr->xattr_len;
-
-                if (len && xattr && xattr->xattr_value) {
-                        ret = dict_reset (tmp2);
-                        if (ret < 0) {
-                                gf_log (this->name, GF_LOG_DEBUG,
-                                        "dict_reset failed (%s)",
-                                        strerror (-ret));
-                        }
-
-                        ret = dict_unserialize (xattr->xattr_value,
-                                                xattr->xattr_len,
-                                                &tmp2);
-                        if (ret < 0) {
-                                gf_log (this->name, GF_LOG_WARNING,
-                                        "dict_unserialize failed (%s)",
-                                        strerror (-ret));
-                                ret = -1;
-                                goto out;
-                        }
-
-                        tmp1 = dict_copy (tmp2, tmp1);
-                        if (tmp1 == NULL) {
-                                gf_log (this->name, GF_LOG_WARNING,
-                                        "dict_copy failed (%s)",
-                                        strerror (-ret));
-                                ret = -1;
-                                goto out;
-                        }
-                }
-        }
-
-        len = dict_serialized_length (tmp1);
-        if (len > 0) {
-                buf = GF_CALLOC (1, len, gf_common_mt_dict_t);
-                if (buf == NULL) {
-                        ret = -1;
-                        goto out;
-                }
-
-                ret = dict_serialize (tmp1, buf);
-                if (ret < 0) {
-                        gf_log (this->name, GF_LOG_WARNING,
-                                "dict_serialize failed (%s)", strerror (-ret));
-                        GF_FREE(buf);
-                        ret = -1;
-                        goto out;
-                }
-
-                *xattr_serz = buf;
-        }
-
-        ret = 0;
-out:
-        if (tmp1 != NULL) {
-                dict_unref (tmp1);
-        }
-
-        if (tmp2 != NULL) {
-                dict_unref (tmp2);
-        }
-
-        return ret;
-}
-
-
-int32_t
-stripe_fill_pathinfo_xattr (xlator_t *this, stripe_local_t *local,
-                            char **xattr_serz)
-{
-        int      ret             = -1;
-        int32_t  padding         = 0;
-        int32_t  tlen            = 0;
-        char stripe_size_str[20] = {0,};
-        char    *pathinfo_serz   = NULL;
-
-        if (!local) {
-                gf_log (this->name, GF_LOG_ERROR, "Possible NULL deref");
-                goto out;
-        }
-
-        (void) snprintf (stripe_size_str, 20, "%"PRId64,
-                         (long long) (local->fctx) ? local->fctx->stripe_size : 0);
-
-        /* extra bytes for decorations (brackets and <>'s) */
-        padding = strlen (this->name) + strlen (STRIPE_PATHINFO_HEADER)
-                + strlen (stripe_size_str) + 7;
-        local->xattr_total_len += (padding + 2);
-
-        pathinfo_serz = GF_CALLOC (local->xattr_total_len, sizeof (char),
-                                   gf_common_mt_char);
-        if (!pathinfo_serz)
-                goto out;
-
-        /* xlator info */
-        (void) sprintf (pathinfo_serz, "(<"STRIPE_PATHINFO_HEADER"%s:[%s]> ",
-                        this->name, stripe_size_str);
-
-        ret = stripe_xattr_aggregate (pathinfo_serz + padding, local, &tlen);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "Cannot aggregate pathinfo list");
-                GF_FREE(pathinfo_serz);
-                goto out;
-        }
-
-        *(pathinfo_serz + padding + tlen) = ')';
-        *(pathinfo_serz + padding + tlen + 1) = '\0';
-
-        *xattr_serz = pathinfo_serz;
-
-        ret = 0;
- out:
-        return ret;
-}
-
-/**
- * stripe_get_matching_bs - Get the matching block size for the given path.
- */
-int32_t
-stripe_get_matching_bs (const char *path, stripe_private_t *priv)
-{
-        struct stripe_options *trav       = NULL;
-        uint64_t               block_size = 0;
-
-        GF_VALIDATE_OR_GOTO ("stripe", priv, out);
-        GF_VALIDATE_OR_GOTO ("stripe", path, out);
-
-        LOCK (&priv->lock);
-        {
-                block_size = priv->block_size;
-                trav = priv->pattern;
-                while (trav) {
-                        if (!fnmatch (trav->path_pattern, path, FNM_NOESCAPE)) {
-                                block_size = trav->block_size;
-                                break;
-                        }
-                        trav = trav->next;
-                }
-        }
-        UNLOCK (&priv->lock);
-
-out:
-        return block_size;
-}
-
-int32_t
-stripe_ctx_handle (xlator_t *this, call_frame_t *prev, stripe_local_t *local,
-                   dict_t *dict)
-{
-        char            key[256]       = {0,};
-        data_t         *data            = NULL;
-        int32_t         index           = 0;
-        stripe_private_t *priv          = NULL;
-
-        priv = this->private;
-
-
-        if (!local->fctx) {
-                local->fctx =  GF_CALLOC (1, sizeof (stripe_fd_ctx_t),
-                                         gf_stripe_mt_stripe_fd_ctx_t);
-                if (!local->fctx) {
-                        local->op_errno = ENOMEM;
-                        local->op_ret = -1;
-                        goto out;
-                }
-
-                local->fctx->static_array = 0;
-        }
-        /* Stripe block size */
-        sprintf (key, "trusted.%s.stripe-size", this->name);
-        data = dict_get (dict, key);
-        if (!data) {
-                local->xattr_self_heal_needed = 1;
-                gf_log (this->name, GF_LOG_ERROR,
-                        "Failed to get stripe-size");
-                goto out;
-        } else {
-                if (!local->fctx->stripe_size) {
-                        local->fctx->stripe_size =
-                                     data_to_int64 (data);
-                }
-
-                if (local->fctx->stripe_size != data_to_int64 (data)) {
-                        gf_log (this->name, GF_LOG_WARNING,
-                                "stripe-size mismatch in blocks");
-                        local->xattr_self_heal_needed = 1;
-                }
-        }
-
-        /* Stripe count */
-        sprintf (key, "trusted.%s.stripe-count", this->name);
-        data = dict_get (dict, key);
-
-        if (!data) {
-                local->xattr_self_heal_needed = 1;
-                gf_log (this->name, GF_LOG_ERROR,
-                        "Failed to get stripe-count");
-                goto out;
-        }
-        if (!local->fctx->xl_array) {
-                local->fctx->stripe_count = data_to_int32 (data);
-                if (!local->fctx->stripe_count) {
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "error with stripe-count xattr");
-                        local->op_ret   = -1;
-                        local->op_errno = EIO;
-                        goto out;
-                }
-
-                local->fctx->xl_array = GF_CALLOC (local->fctx->stripe_count,
-                                                   sizeof (xlator_t *),
-                                                   gf_stripe_mt_xlator_t);
-
-                if (!local->fctx->xl_array) {
-                        local->op_errno = ENOMEM;
-                        local->op_ret   = -1;
-                        goto out;
-                }
-        }
-        if (local->fctx->stripe_count != data_to_int32 (data)) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "error with stripe-count xattr (%d != %d)",
-                        local->fctx->stripe_count, data_to_int32 (data));
-                local->op_ret   = -1;
-                local->op_errno = EIO;
-                goto out;
-        }
-
-        /* index */
-        sprintf (key, "trusted.%s.stripe-index", this->name);
-        data = dict_get (dict, key);
-        if (!data) {
-                local->xattr_self_heal_needed = 1;
-                gf_log (this->name, GF_LOG_ERROR,
-                        "Failed to get stripe-index");
-                goto out;
-        }
-        index = data_to_int32 (data);
-        if (index > priv->child_count) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "error with stripe-index xattr (%d)", index);
-                local->op_ret   = -1;
-                local->op_errno = EIO;
-                goto out;
-        }
-        if (local->fctx->xl_array) {
-                if (!local->fctx->xl_array[index])
-                        local->fctx->xl_array[index] = prev->this;
-        }
-
-	sprintf(key, "trusted.%s.stripe-coalesce", this->name);
-	data = dict_get(dict, key);
-	if (!data) {
-		/*
-		 * The file was probably created prior to coalesce support.
-		 * Assume non-coalesce mode for this file to maintain backwards
-		 * compatibility.
-		 */
-		gf_log(this->name, GF_LOG_DEBUG, "missing stripe-coalesce "
-			"attr, assume non-coalesce mode");
-		local->fctx->stripe_coalesce = 0;
-	} else {
-		local->fctx->stripe_coalesce = data_to_int32(data);
-	}
-
-
-out:
-        return 0;
-}
-
-int32_t
-stripe_xattr_request_build (xlator_t *this, dict_t *dict, uint64_t stripe_size,
-                            uint32_t stripe_count, uint32_t stripe_index,
-			    uint32_t stripe_coalesce)
-{
-        char            key[256]       = {0,};
-        int32_t         ret             = -1;
-
-        sprintf (key, "trusted.%s.stripe-size", this->name);
-        ret = dict_set_int64 (dict, key, stripe_size);
-        if (ret) {
-                gf_log (this->name, GF_LOG_WARNING,
-                        "failed to set %s in xattr_req dict", key);
-                goto out;
-        }
-
-        sprintf (key, "trusted.%s.stripe-count", this->name);
-        ret = dict_set_int32 (dict, key, stripe_count);
-        if (ret) {
-                gf_log (this->name, GF_LOG_WARNING,
-                        "failed to set %s in xattr_req dict", key);
-                goto out;
-        }
-
-        sprintf (key, "trusted.%s.stripe-index", this->name);
-        ret = dict_set_int32 (dict, key, stripe_index);
-        if (ret) {
-                gf_log (this->name, GF_LOG_WARNING,
-                        "failed to set %s in xattr_req dict", key);
-                goto out;
-        }
-
-	sprintf(key, "trusted.%s.stripe-coalesce", this->name);
-	ret = dict_set_int32(dict, key, stripe_coalesce);
-	if (ret) {
-		gf_log(this->name, GF_LOG_WARNING,
-			"failed to set %s in xattr_req_dict", key);
-		goto out;
-	}
-out:
-        return ret;
-}
-
-
-static int
-set_default_block_size (stripe_private_t *priv, char *num)
-{
-
-        int             ret = -1;
-        GF_VALIDATE_OR_GOTO ("stripe", THIS, out);
-        GF_VALIDATE_OR_GOTO (THIS->name, priv, out);
-        GF_VALIDATE_OR_GOTO (THIS->name, num, out);
-
-
-        if (gf_string2bytesize_uint64 (num, &priv->block_size) != 0) {
-                gf_log (THIS->name, GF_LOG_ERROR,
-                        "invalid number format \"%s\"", num);
-                goto out;
-        }
-
-        ret = 0;
-
- out:
-        return ret;
-
-}
-
-
-int
-set_stripe_block_size (xlator_t *this, stripe_private_t *priv, char *data)
-{
-        int                    ret = -1;
-        char                  *tmp_str = NULL;
-        char                  *tmp_str1 = NULL;
-        char                  *dup_str = NULL;
-        char                  *stripe_str = NULL;
-        char                  *pattern = NULL;
-        char                  *num = NULL;
-        struct stripe_options *temp_stripeopt = NULL;
-        struct stripe_options *stripe_opt = NULL;
-
-        if (!this || !priv || !data)
-                goto out;
-
-        /* Get the pattern for striping.
-           "option block-size *avi:10MB" etc */
-        stripe_str = strtok_r (data, ",", &tmp_str);
-        while (stripe_str) {
-                dup_str = gf_strdup (stripe_str);
-                stripe_opt = GF_CALLOC (1, sizeof (struct stripe_options),
-                                        gf_stripe_mt_stripe_options);
-                if (!stripe_opt) {
-                        goto out;
-                }
-
-                pattern = strtok_r (dup_str, ":", &tmp_str1);
-                num = strtok_r (NULL, ":", &tmp_str1);
-                if (!num) {
-                        num = pattern;
-                        pattern = "*";
-                        ret = set_default_block_size (priv, num);
-                        if (ret)
-                                goto out;
-                }
-                if (gf_string2bytesize_uint64 (num, &stripe_opt->block_size) != 0) {
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "invalid number format \"%s\"", num);
-                        goto out;
-                }
-
-                if (stripe_opt->block_size < STRIPE_MIN_BLOCK_SIZE) {
-                        gf_log (this->name, GF_LOG_ERROR, "Invalid Block-size: "
-                                "%s. Should be atleast %llu bytes", num,
-                                STRIPE_MIN_BLOCK_SIZE);
-                        goto out;
-                }
-                if (stripe_opt->block_size % 512) {
-                        gf_log (this->name, GF_LOG_ERROR, "Block-size: %s should"
-                                " be a multiple of 512 bytes", num);
-                        goto out;
-                }
-
-                memcpy (stripe_opt->path_pattern, pattern, strlen (pattern));
-
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "block-size : pattern %s : size %"PRId64,
-                        stripe_opt->path_pattern, stripe_opt->block_size);
-
-                if (priv->pattern)
-                        temp_stripeopt = NULL;
-                else
-                        temp_stripeopt = priv->pattern;
-
-                stripe_opt->next = temp_stripeopt;
-
-                priv->pattern = stripe_opt;
-                stripe_opt = NULL;
-
-                GF_FREE (dup_str);
-                dup_str = NULL;
-
-                stripe_str = strtok_r (NULL, ",", &tmp_str);
-        }
-
-        ret = 0;
-out:
-
-        GF_FREE (dup_str);
-
-        GF_FREE (stripe_opt);
-
-        return ret;
-}
-
-int32_t
-stripe_iatt_merge (struct iatt *from, struct iatt *to)
-{
-        if (to->ia_size < from->ia_size)
-                to->ia_size = from->ia_size;
-        if (to->ia_mtime < from->ia_mtime)
-                to->ia_mtime = from->ia_mtime;
-        if (to->ia_ctime < from->ia_ctime)
-                to->ia_ctime = from->ia_ctime;
-        if (to->ia_atime < from->ia_atime)
-                to->ia_atime = from->ia_atime;
-        return 0;
-}
-
-off_t
-coalesced_offset(off_t offset, uint64_t stripe_size, int stripe_count)
-{
-	size_t line_size = 0;
-	uint64_t stripe_num = 0;
-	off_t coalesced_offset = 0;
-
-	line_size = stripe_size * stripe_count;
-	stripe_num = offset / line_size;
-
-	coalesced_offset = (stripe_num * stripe_size) +
-		(offset % stripe_size);
-
-	return coalesced_offset;
-}
-
-off_t
-uncoalesced_size(off_t size, uint64_t stripe_size, int stripe_count,
-		int stripe_index)
-{
-        uint64_t nr_full_stripe_chunks = 0, mod = 0;
-
-        if (!size)
-                return size;
-
-	/*
-	 * Estimate the number of fully written stripes from the
-	 * local file size. Each stripe_size chunk corresponds to
-	 * a stripe.
-	 */
-        nr_full_stripe_chunks = (size / stripe_size) * stripe_count;
-        mod = size % stripe_size;
-
-        if (!mod) {
-                /*
-		 * There is no remainder, thus we could have overestimated
-		 * the size of the file in terms of chunks. Trim the number
-		 * of chunks by the following stripe members and leave it
-		 * up to those nodes to respond with a larger size (if
-		 * necessary).
-		 */
-                nr_full_stripe_chunks -= stripe_count -
-                        (stripe_index + 1);
-                size = nr_full_stripe_chunks * stripe_size;
-        } else {
-		/*
-		 * There is a remainder and thus we own the last chunk of the
-		 * file. Add the preceding stripe members of the final stripe
-		 * along with the remainder to calculate the exact size.
-		 */
-                nr_full_stripe_chunks += stripe_index;
-                size = nr_full_stripe_chunks * stripe_size + mod;
-        }
-
-        return size;
-}
diff --git a/xlators/cluster/stripe/src/stripe-mem-types.h b/xlators/cluster/stripe/src/stripe-mem-types.h
deleted file mode 100644
index e9ac9cf4648..00000000000
--- a/xlators/cluster/stripe/src/stripe-mem-types.h
+++ /dev/null
@@ -1,31 +0,0 @@
-/*
-  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
-  This file is part of GlusterFS.
-
-  This file is licensed to you under your choice of the GNU Lesser
-  General Public License, version 3 or any later version (LGPLv3 or
-  later), or the GNU General Public License, version 2 (GPLv2), in all
-  cases as published by the Free Software Foundation.
-*/
-
-
-#ifndef __STRIPE_MEM_TYPES_H__
-#define __STRIPE_MEM_TYPES_H__
-
-#include "mem-types.h"
-
-enum gf_stripe_mem_types_ {
-        gf_stripe_mt_iovec = gf_common_mt_end + 1,
-        gf_stripe_mt_stripe_replies,
-        gf_stripe_mt_stripe_fd_ctx_t,
-        gf_stripe_mt_char,
-        gf_stripe_mt_int8_t,
-        gf_stripe_mt_int32_t,
-        gf_stripe_mt_xlator_t,
-        gf_stripe_mt_stripe_private_t,
-        gf_stripe_mt_stripe_options,
-        gf_stripe_mt_xattr_sort_t,
-        gf_stripe_mt_end
-};
-#endif
-
diff --git a/xlators/cluster/stripe/src/stripe.c b/xlators/cluster/stripe/src/stripe.c
deleted file mode 100644
index 7ac131b2ca9..00000000000
--- a/xlators/cluster/stripe/src/stripe.c
+++ /dev/null
@@ -1,5752 +0,0 @@
-/*
-  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
-  This file is part of GlusterFS.
-
-  This file is licensed to you under your choice of the GNU Lesser
-  General Public License, version 3 or any later version (LGPLv3 or
-  later), or the GNU General Public License, version 2 (GPLv2), in all
-  cases as published by the Free Software Foundation.
-*/
-
-/**
- * xlators/cluster/stripe:
- *    Stripe translator, stripes the data across its child nodes,
- *    as per the options given in the volfile. The striping works
- *    fairly simple. It writes files at different offset as per
- *    calculation. So, 'ls -l' output at the real posix level will
- *    show file size bigger than the actual size. But when one does
- *    'df' or 'du <file>', real size of the file on the server is shown.
- *
- * WARNING:
- *  Stripe translator can't regenerate data if a child node gets disconnected.
- *  So, no 'self-heal' for stripe. Hence the advice, use stripe only when its
- *  very much necessary, or else, use it in combination with AFR, to have a
- *  backup copy.
- */
-#include <fnmatch.h>
-
-#include "stripe.h"
-#include "libxlator.h"
-#include "byte-order.h"
-#include "statedump.h"
-
-struct volume_options options[];
-
-int32_t
-stripe_sh_chown_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                     int32_t op_ret, int32_t op_errno,
-                     struct iatt *preop, struct iatt *postop, dict_t *xdata)
-{
-        int             callcnt = -1;
-        stripe_local_t *local   = NULL;
-
-        if (!this || !frame || !frame->local) {
-                gf_log ("stripe", GF_LOG_DEBUG, "possible NULL deref");
-                goto out;
-        }
-
-        local = frame->local;
-
-        LOCK (&frame->lock);
-        {
-                callcnt = --local->call_count;
-        }
-        UNLOCK (&frame->lock);
-
-        if (!callcnt) {
-                STRIPE_STACK_DESTROY (frame);
-        }
-out:
-        return 0;
-}
-
-int32_t
-stripe_sh_make_entry_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                          int32_t op_ret, int32_t op_errno, inode_t *inode,
-                          struct iatt *buf, struct iatt *preparent,
-                          struct iatt *postparent, dict_t *xdata)
-{
-        stripe_local_t *local = NULL;
-        call_frame_t    *prev = NULL;
-
-        if (!frame || !frame->local || !cookie || !this) {
-                gf_log ("stripe", GF_LOG_DEBUG, "possible NULL deref");
-                goto out;
-        }
-
-        prev  = cookie;
-        local = frame->local;
-
-        STACK_WIND (frame, stripe_sh_chown_cbk, prev->this,
-                    prev->this->fops->setattr, &local->loc,
-                    &local->stbuf, (GF_SET_ATTR_UID | GF_SET_ATTR_GID), NULL);
-
-out:
-        return 0;
-}
-
-int32_t
-stripe_entry_self_heal (call_frame_t *frame, xlator_t *this,
-                        stripe_local_t *local)
-{
-        xlator_list_t    *trav   = NULL;
-        call_frame_t     *rframe = NULL;
-        stripe_local_t   *rlocal = NULL;
-        stripe_private_t *priv   = NULL;
-        dict_t           *xdata  = NULL;
-        int               ret    = 0;
-
-        if (!local || !this || !frame) {
-                gf_log ("stripe", GF_LOG_DEBUG, "possible NULL deref");
-                goto out;
-        }
-
-        if (!(IA_ISREG (local->stbuf.ia_type) ||
-              IA_ISDIR (local->stbuf.ia_type)))
-                return 0;
-
-        priv = this->private;
-        trav = this->children;
-        rframe = copy_frame (frame);
-        if (!rframe) {
-                goto out;
-        }
-        rlocal = mem_get0 (this->local_pool);
-        if (!rlocal) {
-                goto out;
-        }
-        rframe->local = rlocal;
-        rlocal->call_count = priv->child_count;
-        loc_copy (&rlocal->loc, &local->loc);
-        memcpy (&rlocal->stbuf, &local->stbuf, sizeof (struct iatt));
-
-        xdata = dict_new ();
-        if (!xdata)
-                goto out;
-
-        ret = dict_set_static_bin (xdata, "gfid-req", local->stbuf.ia_gfid, 16);
-        if (ret)
-                gf_log (this->name, GF_LOG_WARNING,
-                        "%s: failed to set gfid-req", local->loc.path);
-
-        while (trav) {
-                if (IA_ISREG (local->stbuf.ia_type)) {
-                        STACK_WIND (rframe, stripe_sh_make_entry_cbk,
-                                    trav->xlator, trav->xlator->fops->mknod,
-                                    &local->loc,
-                                    st_mode_from_ia (local->stbuf.ia_prot,
-                                                     local->stbuf.ia_type),
-                                    0, 0, xdata);
-                }
-                if (IA_ISDIR (local->stbuf.ia_type)) {
-                        STACK_WIND (rframe, stripe_sh_make_entry_cbk,
-                                    trav->xlator, trav->xlator->fops->mkdir,
-                                    &local->loc,
-                                    st_mode_from_ia (local->stbuf.ia_prot,
-                                                     local->stbuf.ia_type),
-                                    0, xdata);
-                }
-                trav = trav->next;
-        }
-
-        if (xdata)
-                dict_unref (xdata);
-        return 0;
-
-out:
-        if (rframe)
-                STRIPE_STACK_DESTROY (rframe);
-        if (xdata)
-                dict_unref (xdata);
-
-        return 0;
-}
-
-
-int32_t
-stripe_lookup_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                   int32_t op_ret, int32_t op_errno, inode_t *inode,
-                   struct iatt *buf, dict_t *xdata, struct iatt *postparent)
-{
-        int32_t         callcnt     = 0;
-        stripe_local_t *local       = NULL;
-        call_frame_t   *prev        = NULL;
-        int             ret         = 0;
-
-        if (!this || !frame || !frame->local || !cookie) {
-                gf_log ("stripe", GF_LOG_DEBUG, "possible NULL deref");
-                goto out;
-        }
-
-        prev = cookie;
-        local = frame->local;
-
-        LOCK (&frame->lock);
-        {
-                callcnt = --local->call_count;
-
-                if (op_ret == -1) {
-                        if ((op_errno != ENOENT) || (op_errno != ESTALE))
-                                gf_log (this->name, GF_LOG_DEBUG,
-                                        "%s returned error %s",
-                                        prev->this->name,
-                                        strerror (op_errno));
-                        if (local->op_errno != ESTALE)
-                                local->op_errno = op_errno;
-                        if (((op_errno != ENOENT) && (op_errno != ENOTCONN)
-                              && (op_errno != ESTALE)) ||
-                            (prev->this == FIRST_CHILD (this)))
-                                local->failed = 1;
-                        if (op_errno == ENOENT)
-                                local->entry_self_heal_needed = 1;
-                }
-
-                if (op_ret >= 0) {
-                        local->op_ret = 0;
-                        if (IA_ISREG (buf->ia_type)) {
-                                ret = stripe_ctx_handle (this, prev, local,
-                                                         xdata);
-                                if (ret)
-                                        gf_log (this->name, GF_LOG_ERROR,
-                                                 "Error getting fctx info from"
-                                                 " dict");
-                        }
-
-                        if (FIRST_CHILD(this) == prev->this) {
-                                local->stbuf      = *buf;
-                                local->postparent = *postparent;
-                                local->inode = inode_ref (inode);
-                                if (xdata)
-                                        local->xdata = dict_ref (xdata);
-                                if (local->xattr) {
-                                        stripe_aggregate_xattr (local->xdata,
-                                                                local->xattr);
-                                        dict_unref (local->xattr);
-                                        local->xattr = NULL;
-                                }
-                        }
-
-                        if (!local->xdata && !local->xattr) {
-                                local->xattr = dict_ref (xdata);
-                        } else if (local->xdata) {
-                                stripe_aggregate_xattr (local->xdata, xdata);
-                        } else if (local->xattr) {
-                                stripe_aggregate_xattr (local->xattr, xdata);
-                        }
-
-                        local->stbuf_blocks      += buf->ia_blocks;
-                        local->postparent_blocks += postparent->ia_blocks;
-
-			correct_file_size(buf, local->fctx, prev);
-
-                        if (local->stbuf_size < buf->ia_size)
-                                local->stbuf_size = buf->ia_size;
-                        if (local->postparent_size < postparent->ia_size)
-                                local->postparent_size = postparent->ia_size;
-
-                        if (gf_uuid_is_null (local->ia_gfid))
-                                gf_uuid_copy (local->ia_gfid, buf->ia_gfid);
-
-                        /* Make sure the gfid on all the nodes are same */
-                        if (gf_uuid_compare (local->ia_gfid, buf->ia_gfid)) {
-                                gf_log (this->name, GF_LOG_WARNING,
-                                        "%s: gfid different on subvolume %s",
-                                        local->loc.path, prev->this->name);
-                        }
-                }
-        }
-        UNLOCK (&frame->lock);
-
-        if (!callcnt) {
-                if (local->op_ret == 0 && local->entry_self_heal_needed &&
-                    !gf_uuid_is_null (local->loc.inode->gfid))
-                        stripe_entry_self_heal (frame, this, local);
-
-                if (local->failed)
-                        local->op_ret = -1;
-
-                if (local->op_ret != -1) {
-                        local->stbuf.ia_blocks      = local->stbuf_blocks;
-                        local->stbuf.ia_size        = local->stbuf_size;
-                        local->postparent.ia_blocks = local->postparent_blocks;
-                        local->postparent.ia_size   = local->postparent_size;
-                        inode_ctx_put (local->inode, this,
-                                       (uint64_t) (long)local->fctx);
-                }
-
-                STRIPE_STACK_UNWIND (lookup, frame, local->op_ret,
-                                     local->op_errno, local->inode,
-                                     &local->stbuf, local->xdata,
-                                     &local->postparent);
-        }
-out:
-        return 0;
-}
-
-int32_t
-stripe_lookup (call_frame_t *frame, xlator_t *this, loc_t *loc,
-               dict_t *xdata)
-{
-        stripe_local_t   *local    = NULL;
-        xlator_list_t    *trav     = NULL;
-        stripe_private_t *priv     = NULL;
-        int32_t           op_errno = EINVAL;
-        int64_t           filesize = 0;
-        int               ret      = 0;
-        uint64_t          tmpctx   = 0;
-
-        VALIDATE_OR_GOTO (frame, err);
-        VALIDATE_OR_GOTO (this, err);
-        VALIDATE_OR_GOTO (loc, err);
-        VALIDATE_OR_GOTO (loc->inode, err);
-
-        priv = this->private;
-        trav = this->children;
-
-        /* Initialization */
-        local = mem_get0 (this->local_pool);
-        if (!local) {
-                op_errno = ENOMEM;
-                goto err;
-        }
-        local->op_ret = -1;
-        frame->local = local;
-        loc_copy (&local->loc, loc);
-
-        inode_ctx_get (local->inode, this, &tmpctx);
-        if (tmpctx)
-                local->fctx = (stripe_fd_ctx_t*) (long)tmpctx;
-
-        /* quick-read friendly changes */
-        if (xdata && dict_get (xdata, GF_CONTENT_KEY)) {
-                ret = dict_get_int64 (xdata, GF_CONTENT_KEY, &filesize);
-                if (!ret && (filesize > priv->block_size))
-                        dict_del (xdata, GF_CONTENT_KEY);
-        }
-
-        /* get stripe-size xattr on lookup. This would be required for
-         * open/read/write/pathinfo calls. Hence we send down the request
-         * even when type == IA_INVAL */
-
-	/*
-	 * We aren't guaranteed to have xdata here. We need the format info for
-	 * the file, so allocate xdata if necessary.
-	 */
-	if (!xdata)
-		xdata = dict_new();
-	else
-		xdata = dict_ref(xdata);
-
-        if (xdata && (IA_ISREG (loc->inode->ia_type) ||
-            (loc->inode->ia_type == IA_INVAL))) {
-                ret = stripe_xattr_request_build (this, xdata, 8, 4, 4, 0);
-                if (ret)
-                        gf_log (this->name , GF_LOG_ERROR, "Failed to build"
-                                " xattr request for %s", loc->path);
-
-        }
-
-        /* Every time in stripe lookup, all child nodes
-           should be looked up */
-        local->call_count = priv->child_count;
-        while (trav) {
-                STACK_WIND (frame, stripe_lookup_cbk, trav->xlator,
-                            trav->xlator->fops->lookup, loc, xdata);
-                trav = trav->next;
-        }
-
-	dict_unref(xdata);
-
-        return 0;
-err:
-        STRIPE_STACK_UNWIND (lookup, frame, -1, op_errno, NULL, NULL, NULL, NULL);
-        return 0;
-}
-
-
-int32_t
-stripe_stat_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                 int32_t op_ret, int32_t op_errno, struct iatt *buf, dict_t *xdata)
-{
-        int32_t         callcnt = 0;
-        stripe_local_t *local = NULL;
-        call_frame_t   *prev = NULL;
-
-        if (!this || !frame || !frame->local || !cookie) {
-                gf_log ("stripe", GF_LOG_DEBUG, "possible NULL deref");
-                goto out;
-        }
-        prev  = cookie;
-        local = frame->local;
-
-        LOCK (&frame->lock);
-        {
-                callcnt = --local->call_count;
-
-                if (op_ret == -1) {
-                        gf_log (this->name, GF_LOG_DEBUG,
-                                "%s returned error %s",
-                                prev->this->name, strerror (op_errno));
-                        local->op_errno = op_errno;
-                        if ((op_errno != ENOENT) ||
-                            (prev->this == FIRST_CHILD (this)))
-                                local->failed = 1;
-                }
-
-                if (op_ret == 0) {
-                        local->op_ret = 0;
-
-                        if (FIRST_CHILD(this) == prev->this) {
-                                local->stbuf = *buf;
-                        }
-
-                        local->stbuf_blocks += buf->ia_blocks;
-
-			correct_file_size(buf, local->fctx, prev);
-
-                        if (local->stbuf_size < buf->ia_size)
-                                local->stbuf_size = buf->ia_size;
-                }
-        }
-        UNLOCK (&frame->lock);
-
-        if (!callcnt) {
-                if (local->failed)
-                        local->op_ret = -1;
-
-                if (local->op_ret != -1) {
-                        local->stbuf.ia_size   = local->stbuf_size;
-                        local->stbuf.ia_blocks = local->stbuf_blocks;
-                }
-
-                STRIPE_STACK_UNWIND (stat, frame, local->op_ret,
-                                     local->op_errno, &local->stbuf, NULL);
-        }
-out:
-        return 0;
-}
-
-int32_t
-stripe_stat (call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xdata)
-{
-        xlator_list_t    *trav = NULL;
-        stripe_local_t   *local = NULL;
-        stripe_private_t *priv = NULL;
-	stripe_fd_ctx_t  *fctx = NULL;
-        int32_t           op_errno = EINVAL;
-
-        VALIDATE_OR_GOTO (frame, err);
-        VALIDATE_OR_GOTO (this, err);
-        VALIDATE_OR_GOTO (loc, err);
-        VALIDATE_OR_GOTO (loc->path, err);
-        VALIDATE_OR_GOTO (loc->inode, err);
-
-        priv = this->private;
-        trav = this->children;
-
-        if (priv->first_child_down) {
-                op_errno = ENOTCONN;
-                goto err;
-        }
-
-        /* Initialization */
-        local = mem_get0 (this->local_pool);
-        if (!local) {
-                op_errno = ENOMEM;
-                goto err;
-        }
-        local->op_ret = -1;
-        frame->local = local;
-        local->call_count = priv->child_count;
-
-	if (IA_ISREG(loc->inode->ia_type)) {
-		inode_ctx_get(loc->inode, this, (uint64_t *) &fctx);
-		if (!fctx)
-			goto err;
-		local->fctx = fctx;
-	}
-
-        while (trav) {
-                STACK_WIND (frame, stripe_stat_cbk, trav->xlator,
-                            trav->xlator->fops->stat, loc, NULL);
-                trav = trav->next;
-        }
-
-        return 0;
-
-err:
-        STRIPE_STACK_UNWIND (stat, frame, -1, op_errno, NULL, NULL);
-        return 0;
-}
-
-
-int32_t
-stripe_statfs_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                   int32_t op_ret, int32_t op_errno, struct statvfs *stbuf, dict_t *xdata)
-{
-        stripe_local_t *local = NULL;
-        int32_t         callcnt = 0;
-
-        if (!this || !frame || !frame->local) {
-                gf_log ("stripe", GF_LOG_DEBUG, "possible NULL deref");
-                goto out;
-        }
-        local = frame->local;
-
-        LOCK(&frame->lock);
-        {
-                callcnt = --local->call_count;
-
-                if (op_ret && (op_errno != ENOTCONN)) {
-                        local->op_errno = op_errno;
-                }
-                if (op_ret == 0) {
-                        struct statvfs *dict_buf = &local->statvfs_buf;
-                        dict_buf->f_bsize   = stbuf->f_bsize;
-                        dict_buf->f_frsize  = stbuf->f_frsize;
-                        dict_buf->f_blocks += stbuf->f_blocks;
-                        dict_buf->f_bfree  += stbuf->f_bfree;
-                        dict_buf->f_bavail += stbuf->f_bavail;
-                        dict_buf->f_files  += stbuf->f_files;
-                        dict_buf->f_ffree  += stbuf->f_ffree;
-                        dict_buf->f_favail += stbuf->f_favail;
-                        dict_buf->f_fsid    = stbuf->f_fsid;
-                        dict_buf->f_flag    = stbuf->f_flag;
-                        dict_buf->f_namemax = stbuf->f_namemax;
-                        local->op_ret = 0;
-                }
-        }
-        UNLOCK (&frame->lock);
-
-        if (!callcnt) {
-                STRIPE_STACK_UNWIND (statfs, frame, local->op_ret,
-                                     local->op_errno, &local->statvfs_buf, NULL);
-        }
-out:
-        return 0;
-}
-
-int32_t
-stripe_statfs (call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xdata)
-{
-        stripe_local_t   *local = NULL;
-        xlator_list_t    *trav = NULL;
-        stripe_private_t *priv = NULL;
-        int32_t           op_errno = EINVAL;
-
-        VALIDATE_OR_GOTO (frame, err);
-        VALIDATE_OR_GOTO (this, err);
-        VALIDATE_OR_GOTO (loc, err);
-
-        trav = this->children;
-        priv = this->private;
-
-        /* Initialization */
-        local = mem_get0 (this->local_pool);
-        if (!local) {
-                op_errno = ENOMEM;
-                goto err;
-        }
-        local->op_ret = -1;
-        local->op_errno = ENOTCONN;
-        frame->local = local;
-
-        local->call_count = priv->child_count;
-        while (trav) {
-                STACK_WIND (frame, stripe_statfs_cbk, trav->xlator,
-                            trav->xlator->fops->statfs, loc, NULL);
-                trav = trav->next;
-        }
-
-        return 0;
-err:
-        STRIPE_STACK_UNWIND (statfs, frame, -1, op_errno, NULL, NULL);
-        return 0;
-}
-
-
-
-int32_t
-stripe_truncate_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                     int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
-                     struct iatt *postbuf, dict_t *xdata)
-{
-        int32_t         callcnt = 0;
-        stripe_local_t *local = NULL;
-        call_frame_t   *prev = NULL;
-
-        if (!this || !frame || !frame->local || !cookie) {
-                gf_log ("stripe", GF_LOG_DEBUG, "possible NULL deref");
-                goto out;
-        }
-
-        prev  = cookie;
-        local = frame->local;
-
-        LOCK (&frame->lock);
-        {
-                callcnt = --local->call_count;
-
-                if (op_ret == -1) {
-                        gf_log (this->name, GF_LOG_DEBUG,
-                                "%s returned error %s",
-                                prev->this->name, strerror (op_errno));
-                        local->op_errno = op_errno;
-                        if ((op_errno != ENOENT) ||
-                            (prev->this == FIRST_CHILD (this)))
-                                local->failed = 1;
-                }
-
-                if (op_ret == 0) {
-                        local->op_ret = 0;
-                        if (FIRST_CHILD(this) == prev->this) {
-                                local->pre_buf  = *prebuf;
-                                local->post_buf = *postbuf;
-                        }
-
-                        local->prebuf_blocks  += prebuf->ia_blocks;
-                        local->postbuf_blocks += postbuf->ia_blocks;
-
-			correct_file_size(prebuf, local->fctx, prev);
-			correct_file_size(postbuf, local->fctx, prev);
-
-                        if (local->prebuf_size < prebuf->ia_size)
-                                local->prebuf_size = prebuf->ia_size;
-
-                        if (local->postbuf_size < postbuf->ia_size)
-                                local->postbuf_size = postbuf->ia_size;
-                }
-        }
-        UNLOCK (&frame->lock);
-
-        if (!callcnt) {
-                if (local->failed)
-                        local->op_ret = -1;
-
-                if (local->op_ret != -1) {
-                        local->pre_buf.ia_blocks  = local->prebuf_blocks;
-                        local->pre_buf.ia_size    = local->prebuf_size;
-                        local->post_buf.ia_blocks = local->postbuf_blocks;
-                        local->post_buf.ia_size   = local->postbuf_size;
-                }
-
-                STRIPE_STACK_UNWIND (truncate, frame, local->op_ret,
-                                     local->op_errno, &local->pre_buf,
-                                     &local->post_buf, NULL);
-        }
-out:
-        return 0;
-}
-
-int32_t
-stripe_truncate (call_frame_t *frame, xlator_t *this, loc_t *loc, off_t offset, dict_t *xdata)
-{
-        stripe_local_t   *local = NULL;
-        stripe_private_t *priv = NULL;
-	stripe_fd_ctx_t  *fctx = NULL;
-        int32_t           op_errno = EINVAL;
-	int		  i, eof_idx;
-	off_t		  dest_offset, tmp_offset;
-
-        VALIDATE_OR_GOTO (frame, err);
-        VALIDATE_OR_GOTO (this, err);
-        VALIDATE_OR_GOTO (loc, err);
-        VALIDATE_OR_GOTO (loc->path, err);
-        VALIDATE_OR_GOTO (loc->inode, err);
-
-        priv = this->private;
-
-        if (priv->first_child_down) {
-                op_errno = ENOTCONN;
-                goto err;
-        }
-
-        /* Initialization */
-        local = mem_get0 (this->local_pool);
-        if (!local) {
-                op_errno = ENOMEM;
-                goto err;
-        }
-        local->op_ret = -1;
-        frame->local = local;
-        local->call_count = priv->child_count;
-
-	inode_ctx_get(loc->inode, this, (uint64_t *) &fctx);
-	if (!fctx) {
-		gf_log(this->name, GF_LOG_ERROR, "no stripe context");
-		op_errno = EINVAL;
-		goto err;
-	}
-
-	local->fctx = fctx;
-	eof_idx = (offset / fctx->stripe_size) % fctx->stripe_count;
-
-	for (i = 0; i < fctx->stripe_count; i++) {
-		if (!fctx->xl_array[i]) {
-			gf_log(this->name, GF_LOG_ERROR,
-				"no xlator at index %d", i);
-			op_errno = EINVAL;
-			goto err;
-		}
-
-		if (fctx->stripe_coalesce) {
-			/*
-	 		 * The node that owns EOF is truncated to the exact
-	 		 * coalesced offset. Nodes prior to this index should
-	 		 * be rounded up to the size of the complete stripe,
-	 		 * while nodes after this index should be rounded down
-			 * to the size of the previous stripe.
-			 */
-			if (i < eof_idx)
-				tmp_offset = roof(offset, fctx->stripe_size *
-						fctx->stripe_count);
-			else if (i > eof_idx)
-				tmp_offset = floor(offset, fctx->stripe_size *
-						fctx->stripe_count);
-			else
-				tmp_offset = offset;
-
-			dest_offset = coalesced_offset(tmp_offset,
-					fctx->stripe_size, fctx->stripe_count);
-		} else {
-			dest_offset = offset;
-		}
-
-		STACK_WIND(frame, stripe_truncate_cbk, fctx->xl_array[i],
-			fctx->xl_array[i]->fops->truncate, loc, dest_offset,
-			NULL);
-	}
-
-        return 0;
-err:
-        STRIPE_STACK_UNWIND (truncate, frame, -1, op_errno, NULL, NULL, NULL);
-        return 0;
-}
-
-
-int32_t
-stripe_setattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                    int32_t op_ret, int32_t op_errno,
-                    struct iatt *preop, struct iatt *postop, dict_t *xdata)
-{
-        int32_t         callcnt = 0;
-        stripe_local_t *local = NULL;
-        call_frame_t   *prev = NULL;
-
-        if (!this || !frame || !frame->local || !cookie) {
-                gf_log ("stripe", GF_LOG_DEBUG, "possible NULL deref");
-                goto out;
-        }
-
-        prev  = cookie;
-        local = frame->local;
-
-        LOCK (&frame->lock);
-        {
-                callcnt = --local->call_count;
-
-                if (op_ret == -1) {
-                        gf_log (this->name, GF_LOG_DEBUG,
-                                "%s returned error %s",
-                                prev->this->name, strerror (op_errno));
-                        local->op_errno = op_errno;
-                        if ((op_errno != ENOENT) ||
-                            (prev->this == FIRST_CHILD (this)))
-                                local->failed = 1;
-                }
-
-                if (op_ret == 0) {
-                        local->op_ret = 0;
-
-                        if (FIRST_CHILD(this) == prev->this) {
-                                local->pre_buf  = *preop;
-                                local->post_buf = *postop;
-                        }
-
-                        local->prebuf_blocks  += preop->ia_blocks;
-                        local->postbuf_blocks += postop->ia_blocks;
-
-			correct_file_size(preop, local->fctx, prev);
-			correct_file_size(postop, local->fctx, prev);
-
-                        if (local->prebuf_size < preop->ia_size)
-                                local->prebuf_size = preop->ia_size;
-                        if (local->postbuf_size < postop->ia_size)
-                                local->postbuf_size = postop->ia_size;
-                }
-        }
-        UNLOCK (&frame->lock);
-
-        if (!callcnt) {
-                if (local->failed)
-                        local->op_ret = -1;
-
-                if (local->op_ret != -1) {
-                        local->pre_buf.ia_blocks  = local->prebuf_blocks;
-                        local->pre_buf.ia_size    = local->prebuf_size;
-                        local->post_buf.ia_blocks = local->postbuf_blocks;
-                        local->post_buf.ia_size   = local->postbuf_size;
-                }
-
-                STRIPE_STACK_UNWIND (setattr, frame, local->op_ret,
-                                     local->op_errno, &local->pre_buf,
-                                     &local->post_buf, NULL);
-        }
-out:
-        return 0;
-}
-
-
-int32_t
-stripe_setattr (call_frame_t *frame, xlator_t *this, loc_t *loc,
-                struct iatt *stbuf, int32_t valid, dict_t *xdata)
-{
-        xlator_list_t    *trav = NULL;
-        stripe_local_t   *local = NULL;
-        stripe_private_t *priv = NULL;
-	stripe_fd_ctx_t	 *fctx = NULL;
-        int32_t           op_errno = EINVAL;
-
-        VALIDATE_OR_GOTO (frame, err);
-        VALIDATE_OR_GOTO (this, err);
-        VALIDATE_OR_GOTO (loc, err);
-        VALIDATE_OR_GOTO (loc->path, err);
-        VALIDATE_OR_GOTO (loc->inode, err);
-
-        priv = this->private;
-        trav = this->children;
-
-        if (priv->first_child_down) {
-                op_errno = ENOTCONN;
-                goto err;
-        }
-
-        /* Initialization */
-        local = mem_get0 (this->local_pool);
-        if (!local) {
-                op_errno = ENOMEM;
-                goto err;
-        }
-        local->op_ret = -1;
-        frame->local = local;
-        if (!IA_ISDIR (loc->inode->ia_type) &&
-            !IA_ISREG (loc->inode->ia_type)) {
-                local->call_count = 1;
-                STACK_WIND (frame, stripe_setattr_cbk, FIRST_CHILD (this),
-                            FIRST_CHILD (this)->fops->setattr,
-                            loc, stbuf, valid, NULL);
-                return 0;
-        }
-
-	if (IA_ISREG(loc->inode->ia_type)) {
-		inode_ctx_get(loc->inode, this, (uint64_t *) &fctx);
-		if (!fctx)
-			goto err;
-		local->fctx = fctx;
-	}
-
-        local->call_count = priv->child_count;
-        while (trav) {
-                STACK_WIND (frame, stripe_setattr_cbk,
-                            trav->xlator, trav->xlator->fops->setattr,
-                            loc, stbuf, valid, NULL);
-                trav = trav->next;
-        }
-
-        return 0;
-err:
-        STRIPE_STACK_UNWIND (setattr, frame, -1, op_errno, NULL, NULL, NULL);
-        return 0;
-}
-
-
-int32_t
-stripe_fsetattr (call_frame_t *frame, xlator_t *this, fd_t *fd,
-                 struct iatt *stbuf, int32_t valid, dict_t *xdata)
-{
-        stripe_local_t   *local = NULL;
-        stripe_private_t *priv = NULL;
-        xlator_list_t    *trav = NULL;
-        int32_t           op_errno = EINVAL;
-
-        VALIDATE_OR_GOTO (frame, err);
-        VALIDATE_OR_GOTO (this, err);
-        VALIDATE_OR_GOTO (fd, err);
-        VALIDATE_OR_GOTO (fd->inode, err);
-
-        priv = this->private;
-        trav = this->children;
-
-        /* Initialization */
-        local = mem_get0 (this->local_pool);
-        if (!local) {
-                op_errno = ENOMEM;
-                goto err;
-        }
-        local->op_ret = -1;
-        frame->local = local;
-        local->call_count = priv->child_count;
-
-        while (trav) {
-                STACK_WIND (frame, stripe_setattr_cbk, trav->xlator,
-                            trav->xlator->fops->fsetattr, fd, stbuf, valid, NULL);
-                trav = trav->next;
-        }
-
-        return 0;
-err:
-        STRIPE_STACK_UNWIND (fsetattr, frame, -1, op_errno, NULL, NULL, NULL);
-        return 0;
-}
-
-int32_t
-stripe_stack_rename_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                         int32_t op_ret, int32_t op_errno, struct iatt *buf,
-                         struct iatt *preoldparent, struct iatt *postoldparent,
-                         struct iatt *prenewparent, struct iatt *postnewparent,
-                         dict_t *xdata)
-{
-        int32_t         callcnt = 0;
-        stripe_local_t *local = NULL;
-        call_frame_t   *prev = NULL;
-
-        if (!this || !frame || !frame->local || !cookie) {
-                gf_log ("stripe", GF_LOG_DEBUG, "possible NULL deref");
-                goto out;
-        }
-
-        prev  = cookie;
-        local = frame->local;
-
-        LOCK (&frame->lock);
-        {
-                callcnt = --local->call_count;
-
-                if (op_ret == -1) {
-                        gf_log (this->name, GF_LOG_DEBUG,
-                                "%s returned error %s",
-                                prev->this->name, strerror (op_errno));
-                        local->op_errno = op_errno;
-                        if ((op_errno != ENOENT) ||
-                            (prev->this == FIRST_CHILD (this)))
-                                local->failed = 1;
-                }
-
-                if (op_ret == 0) {
-                        local->op_ret = 0;
-
-                        local->stbuf.ia_blocks      += buf->ia_blocks;
-                        local->preparent.ia_blocks  += preoldparent->ia_blocks;
-                        local->postparent.ia_blocks += postoldparent->ia_blocks;
-                        local->pre_buf.ia_blocks    += prenewparent->ia_blocks;
-                        local->post_buf.ia_blocks   += postnewparent->ia_blocks;
-
-			correct_file_size(buf, local->fctx, prev);
-
-                        if (local->stbuf.ia_size < buf->ia_size)
-                                local->stbuf.ia_size =  buf->ia_size;
-
-                        if (local->preparent.ia_size < preoldparent->ia_size)
-                                local->preparent.ia_size = preoldparent->ia_size;
-
-                        if (local->postparent.ia_size < postoldparent->ia_size)
-                                local->postparent.ia_size = postoldparent->ia_size;
-
-                        if (local->pre_buf.ia_size < prenewparent->ia_size)
-                                local->pre_buf.ia_size = prenewparent->ia_size;
-
-                        if (local->post_buf.ia_size < postnewparent->ia_size)
-                                local->post_buf.ia_size = postnewparent->ia_size;
-                }
-        }
-        UNLOCK (&frame->lock);
-
-        if (!callcnt) {
-                if (local->failed)
-                        local->op_ret = -1;
-
-                STRIPE_STACK_UNWIND (rename, frame, local->op_ret, local->op_errno,
-                                     &local->stbuf, &local->preparent,
-                                     &local->postparent,  &local->pre_buf,
-                                     &local->post_buf, NULL);
-        }
-out:
-        return 0;
-}
-
-int32_t
-stripe_first_rename_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                         int32_t op_ret, int32_t op_errno, struct iatt *buf,
-                         struct iatt *preoldparent, struct iatt *postoldparent,
-                         struct iatt *prenewparent, struct iatt *postnewparent,
-                         dict_t *xdata)
-{
-        stripe_local_t *local = NULL;
-        xlator_list_t  *trav = NULL;
-
-        if (!this || !frame || !frame->local) {
-                gf_log ("stripe", GF_LOG_DEBUG, "possible NULL deref");
-                op_errno = EINVAL;
-                goto unwind;
-        }
-
-        if (op_ret == -1) {
-                goto unwind;
-        }
-
-        local = frame->local;
-        trav = this->children;
-
-        local->stbuf      = *buf;
-        local->preparent  = *preoldparent;
-        local->postparent = *postoldparent;
-        local->pre_buf    = *prenewparent;
-        local->post_buf   = *postnewparent;
-
-        local->op_ret = 0;
-        local->call_count--;
-
-        trav = trav->next; /* Skip first child */
-        while (trav) {
-                STACK_WIND (frame, stripe_stack_rename_cbk,
-                            trav->xlator, trav->xlator->fops->rename,
-                            &local->loc, &local->loc2, NULL);
-                trav = trav->next;
-        }
-        return 0;
-
-unwind:
-        STRIPE_STACK_UNWIND (rename, frame, -1, op_errno, buf, preoldparent,
-                             postoldparent, prenewparent, postnewparent, NULL);
-        return 0;
-}
-
-int32_t
-stripe_rename (call_frame_t *frame, xlator_t *this, loc_t *oldloc,
-               loc_t *newloc, dict_t *xdata)
-{
-        stripe_private_t *priv = NULL;
-        stripe_local_t   *local = NULL;
-        xlator_list_t    *trav = NULL;
-	stripe_fd_ctx_t	 *fctx = NULL;
-        int32_t           op_errno = EINVAL;
-
-        VALIDATE_OR_GOTO (frame, err);
-        VALIDATE_OR_GOTO (this, err);
-        VALIDATE_OR_GOTO (oldloc, err);
-        VALIDATE_OR_GOTO (oldloc->path, err);
-        VALIDATE_OR_GOTO (oldloc->inode, err);
-        VALIDATE_OR_GOTO (newloc, err);
-
-        priv = this->private;
-        trav = this->children;
-
-        /* If any one node is down, don't allow rename */
-        if (priv->nodes_down) {
-                op_errno = ENOTCONN;
-                goto err;
-        }
-
-        /* Initialization */
-        local = mem_get0 (this->local_pool);
-        if (!local) {
-                op_errno = ENOMEM;
-                goto err;
-        }
-
-        frame->local = local;
-
-        local->op_ret = -1;
-        loc_copy (&local->loc, oldloc);
-        loc_copy (&local->loc2, newloc);
-
-        local->call_count = priv->child_count;
-
-	if (IA_ISREG(oldloc->inode->ia_type)) {
-		inode_ctx_get(oldloc->inode, this, (uint64_t *) &fctx);
-		if (!fctx)
-			goto err;
-		local->fctx = fctx;
-	}
-
-        STACK_WIND (frame, stripe_first_rename_cbk, trav->xlator,
-                    trav->xlator->fops->rename, oldloc, newloc, NULL);
-
-        return 0;
-err:
-        STRIPE_STACK_UNWIND (rename, frame, -1, op_errno, NULL, NULL, NULL,
-                             NULL, NULL, NULL);
-        return 0;
-}
-int32_t
-stripe_first_unlink_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                         int32_t op_ret, int32_t op_errno, struct iatt *preparent,
-                         struct iatt *postparent, dict_t *xdata)
-{
-        stripe_local_t *local   = NULL;
-        call_frame_t   *prev = NULL;
-
-        if (!this || !frame || !frame->local || !cookie) {
-                gf_log ("stripe", GF_LOG_DEBUG, "possible NULL deref");
-                goto out;
-        }
-
-        prev  = cookie;
-        local = frame->local;
-
-        if (op_ret == -1) {
-                gf_log (this->name, GF_LOG_DEBUG, "%s returned %s",
-                        prev->this->name, strerror (op_errno));
-                goto out;
-        }
-        local->op_ret = 0;
-        local->preparent  = *preparent;
-        local->postparent = *postparent;
-        local->preparent_blocks  += preparent->ia_blocks;
-        local->postparent_blocks += postparent->ia_blocks;
-
-        STRIPE_STACK_UNWIND(unlink, frame, local->op_ret, local->op_errno,
-                            &local->preparent, &local->postparent, xdata);
-        return 0;
-out:
-        STRIPE_STACK_UNWIND (unlink, frame, -1, op_errno, NULL, NULL, NULL);
-
-        return 0;
-}
-
-
-
-
-int32_t
-stripe_unlink_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                   int32_t op_ret, int32_t op_errno, struct iatt *preparent,
-                   struct iatt *postparent, dict_t *xdata)
-{
-        int32_t         callcnt = 0;
-        stripe_local_t *local   = NULL;
-        call_frame_t   *prev = NULL;
-
-        if (!this || !frame || !frame->local || !cookie) {
-                gf_log ("stripe", GF_LOG_DEBUG, "possible NULL deref");
-                goto out;
-        }
-
-        prev  = cookie;
-        local = frame->local;
-
-        LOCK (&frame->lock);
-        {
-                callcnt = --local->call_count;
-
-                if (op_ret == -1) {
-                        gf_log (this->name, GF_LOG_DEBUG, "%s returned %s",
-                                prev->this->name, strerror (op_errno));
-                        local->op_errno = op_errno;
-                        if (op_errno != ENOENT) {
-                                local->failed = 1;
-                                local->op_ret = op_ret;
-                        }
-                }
-        }
-        UNLOCK (&frame->lock);
-
-        if (callcnt == 1) {
-                if (local->failed) {
-                        op_errno = local->op_errno;
-                        goto out;
-                }
-                STACK_WIND(frame, stripe_first_unlink_cbk, FIRST_CHILD (this),
-                           FIRST_CHILD (this)->fops->unlink, &local->loc,
-                           local->xflag, local->xdata);
-        }
-        return 0;
-out:
-        STRIPE_STACK_UNWIND (unlink, frame, -1, op_errno, NULL, NULL, NULL);
-
-        return 0;
-}
-
-int32_t
-stripe_unlink (call_frame_t *frame, xlator_t *this, loc_t *loc,
-               int xflag, dict_t *xdata)
-{
-        xlator_list_t    *trav = NULL;
-        stripe_local_t   *local = NULL;
-        stripe_private_t *priv = NULL;
-        int32_t           op_errno = EINVAL;
-
-        VALIDATE_OR_GOTO (frame, err);
-        VALIDATE_OR_GOTO (this, err);
-        VALIDATE_OR_GOTO (loc, err);
-        VALIDATE_OR_GOTO (loc->path, err);
-        VALIDATE_OR_GOTO (loc->inode, err);
-
-        priv = this->private;
-        trav = this->children;
-
-        if (priv->first_child_down) {
-                op_errno = ENOTCONN;
-                goto err;
-        }
-
-        /* Don't unlink a file if a node is down */
-        if (priv->nodes_down) {
-                op_errno = ENOTCONN;
-                goto err;
-        }
-
-        /* Initialization */
-        local = mem_get0 (this->local_pool);
-        if (!local) {
-                op_errno = ENOMEM;
-                goto err;
-        }
-        local->op_ret = -1;
-        loc_copy (&local->loc, loc);
-        local->xflag = xflag;
-
-        if (xdata)
-            local->xdata = dict_ref (xdata);
-
-        frame->local = local;
-        local->call_count = priv->child_count;
-        trav = trav->next; /* Skip the first child */
-
-        while (trav) {
-                STACK_WIND (frame, stripe_unlink_cbk,
-                            trav->xlator, trav->xlator->fops->unlink,
-                            loc, xflag, xdata);
-                trav = trav->next;
-        }
-
-        return 0;
-err:
-        STRIPE_STACK_UNWIND (unlink, frame, -1, op_errno, NULL, NULL, NULL);
-        return 0;
-}
-
-
-int32_t
-stripe_first_rmdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                        int32_t op_ret, int32_t op_errno,struct iatt *preparent,
-                        struct iatt *postparent, dict_t *xdata)
-{
-        stripe_local_t *local = NULL;
-
-        if (!this || !frame || !frame->local) {
-                gf_log ("stripe", GF_LOG_DEBUG, "possible NULL deref");
-                op_errno = EINVAL;
-                goto err;
-        }
-
-        if (op_ret == -1) {
-                goto err;
-        }
-
-        local = frame->local;
-        local->op_ret = 0;
-
-        local->call_count--; /* First child successful */
-
-        local->preparent  = *preparent;
-        local->postparent = *postparent;
-        local->preparent_size  = preparent->ia_size;
-        local->postparent_size = postparent->ia_size;
-        local->preparent_blocks  += preparent->ia_blocks;
-        local->postparent_blocks += postparent->ia_blocks;
-
-        STRIPE_STACK_UNWIND (rmdir, frame, local->op_ret, local->op_errno,
-                             &local->preparent, &local->postparent, xdata);
-        return 0;
-err:
-        STRIPE_STACK_UNWIND (rmdir, frame, op_ret, op_errno, NULL, NULL, NULL);
-        return 0;
-
-}
-
-int32_t
-stripe_rmdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                   int32_t op_ret, int32_t op_errno, struct iatt *preparent,
-                   struct iatt *postparent, dict_t *xdata)
-{
-        int32_t         callcnt = 0;
-        stripe_local_t *local   = NULL;
-        call_frame_t   *prev = NULL;
-
-        if (!this || !frame || !frame->local || !cookie) {
-                gf_log ("stripe", GF_LOG_DEBUG, "possible NULL deref");
-                goto out;
-        }
-
-        prev  = cookie;
-        local = frame->local;
-
-        LOCK (&frame->lock);
-        {
-                callcnt = --local->call_count;
-
-                if (op_ret == -1) {
-                        gf_log (this->name, GF_LOG_DEBUG, "%s returned %s",
-                                prev->this->name, strerror (op_errno));
-                        if (op_errno != ENOENT)
-                                local->failed = 1;
-                }
-        }
-        UNLOCK (&frame->lock);
-
-        if (callcnt == 1) {
-                if (local->failed)
-                        goto out;
-                STACK_WIND (frame, stripe_first_rmdir_cbk, FIRST_CHILD (this),
-                            FIRST_CHILD (this)->fops->rmdir, &local->loc,
-                            local->flags, NULL);
-        }
-        return 0;
-out:
-        STRIPE_STACK_UNWIND (rmdir, frame, -1, op_errno, NULL, NULL, NULL);
-        return 0;
-}
-
-int32_t
-stripe_rmdir (call_frame_t *frame, xlator_t *this, loc_t *loc, int flags, dict_t *xdata)
-{
-        xlator_list_t    *trav = NULL;
-        stripe_local_t   *local = NULL;
-        stripe_private_t *priv = NULL;
-        int32_t           op_errno = EINVAL;
-
-        VALIDATE_OR_GOTO (frame, err);
-        VALIDATE_OR_GOTO (this, err);
-        VALIDATE_OR_GOTO (loc, err);
-        VALIDATE_OR_GOTO (loc->path, err);
-        VALIDATE_OR_GOTO (loc->inode, err);
-
-        priv = this->private;
-        trav = this->children;
-
-        /* don't delete a directory if any of the subvolume is down */
-        if (priv->nodes_down) {
-                op_errno = ENOTCONN;
-                goto err;
-        }
-
-        /* Initialization */
-        local = mem_get0 (this->local_pool);
-        if (!local) {
-                op_errno = ENOMEM;
-                goto err;
-        }
-        local->op_ret = -1;
-        frame->local = local;
-        loc_copy (&local->loc, loc);
-        local->flags = flags;
-        local->call_count = priv->child_count;
-        trav = trav->next; /* skip the first child */
-
-        while (trav) {
-                STACK_WIND (frame, stripe_rmdir_cbk,  trav->xlator,
-                            trav->xlator->fops->rmdir, loc, flags, NULL);
-                trav = trav->next;
-        }
-
-        return 0;
-err:
-        STRIPE_STACK_UNWIND (rmdir, frame, -1, op_errno, NULL, NULL, NULL);
-        return 0;
-}
-
-
-int32_t
-stripe_mknod_ifreg_fail_unlink_cbk (call_frame_t *frame, void *cookie,
-                                    xlator_t *this, int32_t op_ret,
-                                    int32_t op_errno, struct iatt *preparent,
-                                    struct iatt *postparent, dict_t *xdata)
-{
-        int32_t         callcnt = 0;
-        stripe_local_t *local = NULL;
-
-        if (!this || !frame || !frame->local) {
-                gf_log ("stripe", GF_LOG_DEBUG, "possible NULL deref");
-                goto out;
-        }
-
-        local = frame->local;
-
-        LOCK (&frame->lock);
-        {
-                callcnt = --local->call_count;
-        }
-        UNLOCK (&frame->lock);
-
-        if (!callcnt) {
-                STRIPE_STACK_UNWIND (mknod, frame, local->op_ret, local->op_errno,
-                                     local->inode, &local->stbuf,
-                                     &local->preparent, &local->postparent, NULL);
-        }
-out:
-        return 0;
-}
-
-
-/**
- */
-int32_t
-stripe_mknod_ifreg_setxattr_cbk (call_frame_t *frame, void *cookie,
-                                 xlator_t *this, int32_t op_ret,
-                                 int32_t op_errno, dict_t *xdata)
-{
-        int32_t           callcnt = 0;
-        stripe_local_t   *local = NULL;
-        stripe_private_t *priv = NULL;
-        xlator_list_t    *trav = NULL;
-        call_frame_t     *prev = NULL;
-
-        if (!this || !frame || !frame->local || !cookie) {
-                gf_log ("stripe", GF_LOG_DEBUG, "possible NULL deref");
-                goto out;
-        }
-
-        prev  = cookie;
-        priv  = this->private;
-        local = frame->local;
-
-	LOCK (&frame->lock);
-        {
-                callcnt = --local->call_count;
-
-                if (op_ret == -1) {
-                        gf_log (this->name, GF_LOG_DEBUG,
-                                "%s returned error %s",
-                                prev->this->name, strerror (op_errno));
-                        local->op_ret = -1;
-                        local->op_errno = op_errno;
-                }
-        }
-        UNLOCK (&frame->lock);
-
-        if (!callcnt) {
-                if (local->op_ret == -1) {
-                        local->call_count = priv->child_count;
-                        while (trav) {
-                                STACK_WIND (frame,
-                                            stripe_mknod_ifreg_fail_unlink_cbk,
-                                            trav->xlator,
-                                            trav->xlator->fops->unlink,
-                                            &local->loc, 0, NULL);
-                                trav = trav->next;
-                        }
-                        return 0;
-                }
-
-                STRIPE_STACK_UNWIND (mknod, frame, local->op_ret, local->op_errno,
-                                     local->inode, &local->stbuf,
-                                     &local->preparent, &local->postparent, NULL);
-        }
-out:
-        return 0;
-}
-
-int32_t
-stripe_mknod_ifreg_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                        int32_t op_ret, int32_t op_errno, inode_t *inode,
-                        struct iatt *buf, struct iatt *preparent,
-                        struct iatt *postparent, dict_t *xdata)
-{
-        int32_t           callcnt = 0;
-        stripe_local_t   *local = NULL;
-        stripe_private_t *priv = NULL;
-        call_frame_t     *prev = NULL;
-        xlator_list_t    *trav = NULL;
-
-        if (!this || !frame || !frame->local || !cookie) {
-                gf_log ("stripe", GF_LOG_DEBUG, "possible NULL deref");
-                goto out;
-        }
-
-        prev  = cookie;
-        priv  = this->private;
-        local = frame->local;
-
-        LOCK (&frame->lock);
-        {
-                callcnt = --local->call_count;
-
-                if (op_ret == -1) {
-                        gf_log (this->name, GF_LOG_DEBUG,
-                                "%s returned error %s",
-                                prev->this->name, strerror (op_errno));
-                        if ((op_errno != ENOENT) ||
-                            (prev->this == FIRST_CHILD (this)))
-                                local->failed = 1;
-                        local->op_errno = op_errno;
-                }
-                if (op_ret >= 0) {
-                        local->op_ret = op_ret;
-
-                        /* Can be used as a mechanism to understand if mknod
-                           was successful in at least one place */
-                        if (gf_uuid_is_null (local->ia_gfid))
-                                gf_uuid_copy (local->ia_gfid, buf->ia_gfid);
-
-			if (stripe_ctx_handle(this, prev, local, xdata))
-				gf_log(this->name, GF_LOG_ERROR,
-					"Error getting fctx info from dict");
-
-                        local->stbuf_blocks += buf->ia_blocks;
-                        local->preparent_blocks  += preparent->ia_blocks;
-                        local->postparent_blocks += postparent->ia_blocks;
-
-			correct_file_size(buf, local->fctx, prev);
-
-                        if (local->stbuf_size < buf->ia_size)
-                                local->stbuf_size = buf->ia_size;
-                        if (local->preparent_size < preparent->ia_size)
-                                local->preparent_size = preparent->ia_size;
-                        if (local->postparent_size < postparent->ia_size)
-                                local->postparent_size = postparent->ia_size;
-                }
-        }
-        UNLOCK (&frame->lock);
-
-        if (!callcnt) {
-                if (local->failed)
-                        local->op_ret = -1;
-
-                if ((local->op_ret == -1) && !gf_uuid_is_null (local->ia_gfid)) {
-                        /* ia_gfid set means, at least on one node 'mknod'
-                           is successful */
-                        local->call_count = priv->child_count;
-                        trav = this->children;
-                        while (trav) {
-                                STACK_WIND (frame,
-                                            stripe_mknod_ifreg_fail_unlink_cbk,
-                                            trav->xlator,
-                                            trav->xlator->fops->unlink,
-                                            &local->loc, 0, NULL);
-                                trav = trav->next;
-                        }
-                        return 0;
-                }
-
-
-                if (local->op_ret != -1) {
-                        local->preparent.ia_blocks  = local->preparent_blocks;
-                        local->preparent.ia_size    = local->preparent_size;
-                        local->postparent.ia_blocks = local->postparent_blocks;
-                        local->postparent.ia_size   = local->postparent_size;
-                        local->stbuf.ia_size        = local->stbuf_size;
-                        local->stbuf.ia_blocks      = local->stbuf_blocks;
-                        inode_ctx_put (local->inode, this,
-                                       (uint64_t)(long) local->fctx);
-
-                }
-                STRIPE_STACK_UNWIND (mknod, frame, local->op_ret, local->op_errno,
-                                     local->inode, &local->stbuf,
-                                     &local->preparent, &local->postparent, NULL);
-        }
-out:
-        return 0;
-}
-
-
-int32_t
-stripe_mknod_first_ifreg_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                        int32_t op_ret, int32_t op_errno, inode_t *inode,
-                        struct iatt *buf, struct iatt *preparent,
-                        struct iatt *postparent, dict_t *xdata)
-{
-        stripe_local_t   *local = NULL;
-        stripe_private_t *priv = NULL;
-        call_frame_t     *prev = NULL;
-        xlator_list_t    *trav = NULL;
-        int               i    = 1;
-        dict_t           *dict           = NULL;
-        int               ret            = 0;
-        int               need_unref     = 0;
-
-        if (!this || !frame || !frame->local || !cookie) {
-                gf_log ("stripe", GF_LOG_DEBUG, "possible NULL deref");
-                goto out;
-        }
-
-        prev  = cookie;
-        priv  = this->private;
-        local = frame->local;
-        trav = this->children;
-
-        local->call_count--;
-
-        if (op_ret == -1) {
-                gf_log (this->name, GF_LOG_DEBUG, "%s returned error %s",
-                        prev->this->name, strerror (op_errno));
-                local->failed = 1;
-                local->op_errno = op_errno;
-                goto out;
-        }
-
-        local->op_ret = op_ret;
-
-        local->stbuf      = *buf;
-        local->preparent  = *preparent;
-        local->postparent = *postparent;
-
-        if (gf_uuid_is_null (local->ia_gfid))
-                gf_uuid_copy (local->ia_gfid, buf->ia_gfid);
-        local->preparent.ia_blocks  = local->preparent_blocks;
-        local->preparent.ia_size    = local->preparent_size;
-        local->postparent.ia_blocks = local->postparent_blocks;
-        local->postparent.ia_size   = local->postparent_size;
-        local->stbuf.ia_size        = local->stbuf_size;
-        local->stbuf.ia_blocks      = local->stbuf_blocks;
-
-        trav = trav->next;
-        while (trav) {
-                if (priv->xattr_supported) {
-                        dict = dict_new ();
-                        if (!dict) {
-                                gf_log (this->name, GF_LOG_ERROR,
-                                        "failed to allocate dict %s", local->loc.path);
-                        }
-                        need_unref = 1;
-
-                        dict_copy (local->xattr, dict);
-
-                        ret = stripe_xattr_request_build (this, dict,
-                                                          local->stripe_size,
-                                                          priv->child_count, i,
-							  priv->coalesce);
-                        if (ret)
-                                gf_log (this->name, GF_LOG_ERROR,
-                                        "Failed to build xattr request");
-
-                } else {
-                        dict = local->xattr;
-                }
-
-                STACK_WIND (frame, stripe_mknod_ifreg_cbk,
-                            trav->xlator, trav->xlator->fops->mknod,
-                            &local->loc, local->mode, local->rdev, 0, dict);
-                trav = trav->next;
-                i++;
-
-                if (dict && need_unref)
-                        dict_unref (dict);
-        }
-
-        return 0;
-
-out:
-
-       STRIPE_STACK_UNWIND (mknod, frame, op_ret, op_errno, NULL, NULL, NULL, NULL, NULL);
-       return 0;
-}
-
-
-int32_t
-stripe_single_mknod_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                         int32_t op_ret, int32_t op_errno, inode_t *inode,
-                         struct iatt *buf, struct iatt *preparent,
-                         struct iatt *postparent, dict_t *xdata)
-{
-        STRIPE_STACK_UNWIND (mknod, frame, op_ret, op_errno, inode, buf,
-                             preparent, postparent, xdata);
-        return 0;
-}
-
-
-int
-stripe_mknod (call_frame_t *frame, xlator_t *this, loc_t *loc, mode_t mode,
-              dev_t rdev, mode_t umask, dict_t *xdata)
-{
-        stripe_private_t *priv           = NULL;
-        stripe_local_t   *local          = NULL;
-        int32_t           op_errno       = EINVAL;
-        int32_t           i              = 0;
-        dict_t           *dict           = NULL;
-        int               ret            = 0;
-        int               need_unref     = 0;
-
-        VALIDATE_OR_GOTO (frame, err);
-        VALIDATE_OR_GOTO (this, err);
-        VALIDATE_OR_GOTO (loc, err);
-        VALIDATE_OR_GOTO (loc->path, err);
-        VALIDATE_OR_GOTO (loc->inode, err);
-
-        priv = this->private;
-
-        if (priv->first_child_down) {
-                op_errno = ENOTCONN;
-                goto err;
-        }
-
-        if (S_ISREG(mode)) {
-                /* NOTE: on older kernels (older than 2.6.9),
-                   creat() fops is sent as mknod() + open(). Hence handling
-                   S_IFREG files is necessary */
-                if (priv->nodes_down) {
-                        gf_log (this->name, GF_LOG_WARNING,
-                                "Some node down, returning EIO");
-                        op_errno = EIO;
-                        goto err;
-                }
-
-                /* Initialization */
-                local = mem_get0 (this->local_pool);
-                if (!local) {
-                        op_errno = ENOMEM;
-                        goto err;
-                }
-                local->op_ret = -1;
-                local->op_errno = ENOTCONN;
-                local->stripe_size = stripe_get_matching_bs (loc->path, priv);
-                frame->local = local;
-                local->inode = inode_ref (loc->inode);
-                loc_copy (&local->loc, loc);
-                local->xattr = dict_copy_with_ref (xdata, NULL);
-                local->mode = mode;
-                local->umask = umask;
-                local->rdev = rdev;
-
-                /* Every time in stripe lookup, all child nodes should
-                   be looked up */
-                local->call_count = priv->child_count;
-
-                if (priv->xattr_supported) {
-                        dict = dict_new ();
-                        if (!dict) {
-                                gf_log (this->name, GF_LOG_ERROR,
-                                        "failed to allocate dict %s", loc->path);
-                        }
-                        need_unref = 1;
-
-                        dict_copy (xdata, dict);
-
-                        ret = stripe_xattr_request_build (this, dict,
-                                                          local->stripe_size,
-                                                          priv->child_count,
-                                                          i, priv->coalesce);
-                        if (ret)
-                                gf_log (this->name, GF_LOG_ERROR,
-                                        "failed to build xattr request");
-                } else {
-                        dict = xdata;
-                }
-
-                STACK_WIND (frame, stripe_mknod_first_ifreg_cbk,
-                            FIRST_CHILD (this), FIRST_CHILD (this)->fops->mknod,
-                            loc, mode, rdev, umask, dict);
-
-                        if (dict && need_unref)
-                                dict_unref (dict);
-                return 0;
-        }
-
-        STACK_WIND (frame, stripe_single_mknod_cbk,
-                    FIRST_CHILD(this), FIRST_CHILD(this)->fops->mknod,
-                    loc, mode, rdev, umask, xdata);
-
-        return 0;
-err:
-        STRIPE_STACK_UNWIND (mknod, frame, -1, op_errno, NULL, NULL, NULL, NULL, NULL);
-        return 0;
-}
-
-
-int32_t
-stripe_mkdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                  int32_t op_ret, int32_t op_errno, inode_t *inode,
-                  struct iatt *buf, struct iatt *preparent,
-                  struct iatt *postparent, dict_t *xdata)
-{
-        int32_t         callcnt = 0;
-        stripe_local_t  *local   = NULL;
-        call_frame_t    *prev = NULL;
-
-        if (!this || !frame || !frame->local || !cookie) {
-                gf_log ("stripe", GF_LOG_DEBUG, "possible NULL deref");
-                goto out;
-        }
-
-        prev  = cookie;
-        local = frame->local;
-
-        LOCK (&frame->lock);
-        {
-                callcnt = --local->call_count;
-
-                if (op_ret == -1) {
-                        gf_log (this->name, GF_LOG_DEBUG,
-                                "%s returned error %s",
-                                prev->this->name, strerror (op_errno));
-                        local->op_errno = op_errno;
-                        if ((op_errno != ENOENT) ||
-                            (prev->this == FIRST_CHILD (this)))
-                                local->failed = 1;
-                }
-
-                if (op_ret >= 0) {
-                        local->op_ret = 0;
-
-                        local->stbuf_blocks      += buf->ia_blocks;
-                        local->preparent_blocks  += preparent->ia_blocks;
-                        local->postparent_blocks += postparent->ia_blocks;
-
-                        if (local->stbuf_size < buf->ia_size)
-                                local->stbuf_size = buf->ia_size;
-                        if (local->preparent_size < preparent->ia_size)
-                                local->preparent_size = preparent->ia_size;
-                        if (local->postparent_size < postparent->ia_size)
-                                local->postparent_size = postparent->ia_size;
-                }
-        }
-        UNLOCK (&frame->lock);
-
-        if (!callcnt) {
-                if (local->failed != -1) {
-                        local->preparent.ia_blocks  = local->preparent_blocks;
-                        local->preparent.ia_size    = local->preparent_size;
-                        local->postparent.ia_blocks = local->postparent_blocks;
-                        local->postparent.ia_size   = local->postparent_size;
-                        local->stbuf.ia_size        = local->stbuf_size;
-                        local->stbuf.ia_blocks      = local->stbuf_blocks;
-                }
-                STRIPE_STACK_UNWIND (mkdir, frame, local->op_ret,
-                                     local->op_errno, local->inode,
-                                     &local->stbuf, &local->preparent,
-                                     &local->postparent, NULL);
-        }
-out:
-        return 0;
-}
-
-
-int32_t
-stripe_first_mkdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                        int32_t op_ret, int32_t op_errno, inode_t *inode,
-                        struct iatt *buf, struct iatt *preparent,
-                        struct iatt *postparent, dict_t *xdata)
-{
-        stripe_local_t  *local   = NULL;
-        call_frame_t    *prev = NULL;
-        xlator_list_t        *trav = NULL;
-
-        if (!this || !frame || !frame->local || !cookie) {
-                gf_log ("stripe", GF_LOG_DEBUG, "possible NULL deref");
-                goto out;
-        }
-
-        prev  = cookie;
-        local = frame->local;
-        trav = this->children;
-
-        local->call_count--; /* first child is successful */
-        trav = trav->next;   /* skip first child */
-
-        if (op_ret == -1) {
-                gf_log (this->name, GF_LOG_DEBUG, "%s returned error %s",
-                        prev->this->name, strerror (op_errno));
-                local->op_errno = op_errno;
-                goto out;
-        }
-
-        local->op_ret = 0;
-
-        local->inode      = inode_ref (inode);
-        local->stbuf      = *buf;
-        local->postparent = *postparent;
-        local->preparent  = *preparent;
-
-        local->stbuf_blocks      += buf->ia_blocks;
-        local->preparent_blocks  += preparent->ia_blocks;
-        local->postparent_blocks += postparent->ia_blocks;
-
-        local->stbuf_size = buf->ia_size;
-        local->preparent_size = preparent->ia_size;
-        local->postparent_size = postparent->ia_size;
-
-        while (trav) {
-                STACK_WIND (frame, stripe_mkdir_cbk, trav->xlator,
-                            trav->xlator->fops->mkdir, &local->loc, local->mode,
-                            local->umask, local->xdata);
-                trav = trav->next;
-        }
-        return 0;
-out:
-        STRIPE_STACK_UNWIND (mkdir, frame, -1, op_errno, NULL, NULL, NULL,
-                             NULL, NULL);
-
-        return 0;
-
-}
-
-
-int
-stripe_mkdir (call_frame_t *frame, xlator_t *this, loc_t *loc, mode_t mode,
-              mode_t umask, dict_t *xdata)
-{
-        stripe_private_t *priv = NULL;
-        stripe_local_t   *local = NULL;
-        xlator_list_t    *trav = NULL;
-        int32_t           op_errno = 1;
-
-        VALIDATE_OR_GOTO (frame, err);
-        VALIDATE_OR_GOTO (this, err);
-        VALIDATE_OR_GOTO (loc, err);
-        VALIDATE_OR_GOTO (loc->path, err);
-        VALIDATE_OR_GOTO (loc->inode, err);
-
-        priv = this->private;
-        trav = this->children;
-
-        if (priv->first_child_down) {
-                op_errno = ENOTCONN;
-                goto err;
-        }
-
-        /* Initialization */
-        local = mem_get0 (this->local_pool);
-        if (!local) {
-                op_errno = ENOMEM;
-                goto err;
-        }
-        local->op_ret = -1;
-        local->call_count = priv->child_count;
-        if (xdata)
-                local->xdata = dict_ref (xdata);
-        local->mode  = mode;
-        local->umask = umask;
-        loc_copy (&local->loc, loc);
-        frame->local = local;
-
-        /* Every time in stripe lookup, all child nodes should be looked up */
-        STACK_WIND (frame, stripe_first_mkdir_cbk, trav->xlator,
-                    trav->xlator->fops->mkdir, loc, mode, umask, xdata);
-
-        return 0;
-err:
-        STRIPE_STACK_UNWIND (mkdir, frame, -1, op_errno, NULL, NULL, NULL, NULL, NULL);
-        return 0;
-}
-
-
-int32_t
-stripe_link_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                 int32_t op_ret, int32_t op_errno, inode_t *inode,
-                 struct iatt *buf, struct iatt *preparent,
-                 struct iatt *postparent, dict_t *xdata)
-{
-        int32_t         callcnt = 0;
-        stripe_local_t  *local   = NULL;
-        call_frame_t    *prev = NULL;
-	stripe_fd_ctx_t *fctx = NULL;
-
-        if (!this || !frame || !frame->local || !cookie) {
-                gf_log ("stripe", GF_LOG_DEBUG, "possible NULL deref");
-                goto out;
-        }
-
-        prev  = cookie;
-        local = frame->local;
-
-        LOCK (&frame->lock);
-        {
-                callcnt = --local->call_count;
-
-                if (op_ret == -1) {
-                        gf_log (this->name, GF_LOG_DEBUG,
-                                "%s returned error %s",
-                                prev->this->name, strerror (op_errno));
-                        local->op_errno = op_errno;
-                        if ((op_errno != ENOENT) ||
-                            (prev->this == FIRST_CHILD (this)))
-                                local->failed = 1;
-                }
-
-                if (op_ret >= 0) {
-                        local->op_ret = 0;
-
-			if (IA_ISREG(inode->ia_type)) {
-				inode_ctx_get(inode, this, (uint64_t *) &fctx);
-				if (!fctx) {
-					gf_log(this->name, GF_LOG_ERROR,
-						"failed to get stripe context");
-					op_ret = -1;
-					op_errno = EINVAL;
-				}
-			}
-
-                        if (FIRST_CHILD(this) == prev->this) {
-                                local->inode      = inode_ref (inode);
-                                local->stbuf      = *buf;
-                                local->postparent = *postparent;
-                                local->preparent  = *preparent;
-                        }
-                        local->stbuf_blocks      += buf->ia_blocks;
-                        local->preparent_blocks  += preparent->ia_blocks;
-                        local->postparent_blocks += postparent->ia_blocks;
-
-			correct_file_size(buf, fctx, prev);
-
-                        if (local->stbuf_size < buf->ia_size)
-                                local->stbuf_size = buf->ia_size;
-                        if (local->preparent_size < preparent->ia_size)
-                                local->preparent_size = preparent->ia_size;
-                        if (local->postparent_size < postparent->ia_size)
-                                local->postparent_size = postparent->ia_size;
-                }
-        }
-        UNLOCK (&frame->lock);
-
-        if (!callcnt) {
-                if (local->failed)
-                        local->op_ret = -1;
-
-                if (local->op_ret != -1) {
-                        local->preparent.ia_blocks  = local->preparent_blocks;
-                        local->preparent.ia_size    = local->preparent_size;
-                        local->postparent.ia_blocks = local->postparent_blocks;
-                        local->postparent.ia_size   = local->postparent_size;
-                        local->stbuf.ia_size        = local->stbuf_size;
-                        local->stbuf.ia_blocks      = local->stbuf_blocks;
-                }
-                STRIPE_STACK_UNWIND (link, frame, local->op_ret,
-                                     local->op_errno, local->inode,
-                                     &local->stbuf, &local->preparent,
-                                     &local->postparent, NULL);
-        }
-out:
-        return 0;
-}
-
-int32_t
-stripe_link (call_frame_t *frame, xlator_t *this, loc_t *oldloc, loc_t *newloc, dict_t *xdata)
-{
-        xlator_list_t    *trav = NULL;
-        stripe_local_t   *local = NULL;
-        stripe_private_t *priv = NULL;
-        int32_t           op_errno = 1;
-
-        VALIDATE_OR_GOTO (frame, err);
-        VALIDATE_OR_GOTO (this, err);
-        VALIDATE_OR_GOTO (oldloc, err);
-        VALIDATE_OR_GOTO (oldloc->path, err);
-        VALIDATE_OR_GOTO (oldloc->inode, err);
-
-        priv = this->private;
-        trav = this->children;
-
-        /* If any one node is down, don't allow link operation */
-        if (priv->nodes_down) {
-                op_errno = ENOTCONN;
-                goto err;
-        }
-
-        /* Initialization */
-        local = mem_get0 (this->local_pool);
-        if (!local) {
-                op_errno = ENOMEM;
-                goto err;
-        }
-        local->op_ret = -1;
-        frame->local = local;
-        local->call_count = priv->child_count;
-
-        /* Every time in stripe lookup, all child
-           nodes should be looked up */
-        while (trav) {
-                STACK_WIND (frame, stripe_link_cbk,
-                            trav->xlator, trav->xlator->fops->link,
-                            oldloc, newloc, NULL);
-                trav = trav->next;
-        }
-
-        return 0;
-err:
-        STRIPE_STACK_UNWIND (link, frame, -1, op_errno, NULL, NULL, NULL, NULL, NULL);
-        return 0;
-}
-
-int32_t
-stripe_create_fail_unlink_cbk (call_frame_t *frame, void *cookie,
-                               xlator_t *this, int32_t op_ret,
-                               int32_t op_errno, struct iatt *preparent,
-                               struct iatt *postparent, dict_t *xdata)
-{
-        int32_t         callcnt = 0;
-        stripe_local_t *local = NULL;
-
-        if (!this || !frame || !frame->local) {
-                gf_log ("stripe", GF_LOG_DEBUG, "possible NULL deref");
-                goto out;
-        }
-
-        local = frame->local;
-
-        LOCK (&frame->lock);
-        {
-                callcnt = --local->call_count;
-        }
-        UNLOCK (&frame->lock);
-
-        if (!callcnt) {
-                STRIPE_STACK_UNWIND (create, frame, local->op_ret, local->op_errno,
-                                     local->fd, local->inode, &local->stbuf,
-                                     &local->preparent, &local->postparent, NULL);
-        }
-out:
-        return 0;
-}
-
-
-int32_t
-stripe_create_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                   int32_t op_ret, int32_t op_errno, fd_t *fd,
-                   inode_t *inode, struct iatt *buf, struct iatt *preparent,
-                   struct iatt *postparent, dict_t *xdata)
-{
-        int32_t           callcnt = 0;
-        stripe_local_t   *local = NULL;
-        stripe_private_t *priv = NULL;
-        call_frame_t     *prev = NULL;
-        xlator_list_t    *trav = NULL;
-
-        if (!this || !frame || !frame->local || !cookie) {
-                gf_log ("stripe", GF_LOG_DEBUG, "possible NULL deref");
-                goto out;
-        }
-
-        prev  = cookie;
-        priv  = this->private;
-        local = frame->local;
-
-        LOCK (&frame->lock);
-        {
-                callcnt = --local->call_count;
-
-                if (op_ret == -1) {
-                        gf_log (this->name, GF_LOG_DEBUG,
-                                "%s returned error %s",
-                                prev->this->name, strerror (op_errno));
-                        local->failed = 1;
-                        local->op_errno = op_errno;
-                }
-
-                if (op_ret >= 0) {
-			if (IA_ISREG(buf->ia_type)) {
-				if (stripe_ctx_handle(this, prev, local, xdata))
-					gf_log(this->name, GF_LOG_ERROR,
-						"Error getting fctx info from "
-						"dict");
-			}
-
-                        local->op_ret = op_ret;
-
-                        local->stbuf_blocks += buf->ia_blocks;
-                        local->preparent_blocks  += preparent->ia_blocks;
-                        local->postparent_blocks += postparent->ia_blocks;
-
-			correct_file_size(buf, local->fctx, prev);
-
-                        if (local->stbuf_size < buf->ia_size)
-                                local->stbuf_size = buf->ia_size;
-                        if (local->preparent_size < preparent->ia_size)
-                                local->preparent_size = preparent->ia_size;
-                        if (local->postparent_size < postparent->ia_size)
-                                local->postparent_size = postparent->ia_size;
-                }
-        }
-        UNLOCK (&frame->lock);
-
-        if (!callcnt) {
-                if (local->failed)
-                        local->op_ret = -1;
-
-                if (local->op_ret == -1) {
-                        local->call_count = priv->child_count;
-                        trav = this->children;
-                        while (trav) {
-                                STACK_WIND (frame,
-                                            stripe_create_fail_unlink_cbk,
-                                            trav->xlator,
-                                            trav->xlator->fops->unlink,
-                                            &local->loc, 0, NULL);
-                                trav = trav->next;
-                        }
-
-                        return 0;
-                }
-
-                if (local->op_ret >= 0) {
-                        local->preparent.ia_blocks  = local->preparent_blocks;
-                        local->preparent.ia_size    = local->preparent_size;
-                        local->postparent.ia_blocks = local->postparent_blocks;
-                        local->postparent.ia_size   = local->postparent_size;
-                        local->stbuf.ia_size        = local->stbuf_size;
-                        local->stbuf.ia_blocks      = local->stbuf_blocks;
-
-			stripe_copy_xl_array(local->fctx->xl_array,
-				             priv->xl_array,
-					     local->fctx->stripe_count);
-			inode_ctx_put(local->inode, this,
-					(uint64_t) local->fctx);
-                }
-
-                /* Create itself has failed.. so return
-                   without setxattring */
-                STRIPE_STACK_UNWIND (create, frame, local->op_ret,
-                                     local->op_errno, local->fd,
-                                     local->inode, &local->stbuf,
-                                     &local->preparent, &local->postparent, NULL);
-        }
-
-out:
-        return 0;
-}
-
-
-
-int32_t
-stripe_first_create_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                   int32_t op_ret, int32_t op_errno, fd_t *fd,
-                   inode_t *inode, struct iatt *buf, struct iatt *preparent,
-                   struct iatt *postparent, dict_t *xdata)
-{
-        stripe_local_t   *local = NULL;
-        stripe_private_t *priv = NULL;
-        call_frame_t     *prev = NULL;
-        xlator_list_t    *trav = NULL;
-        int               i    = 1;
-        dict_t           *dict = NULL;
-        loc_t            *loc  = NULL;
-        int32_t           need_unref = 0;
-        int32_t           ret  = -1;
-
-        if (!this || !frame || !frame->local || !cookie) {
-                gf_log ("stripe", GF_LOG_DEBUG, "possible NULL deref");
-                goto out;
-        }
-
-        prev  = cookie;
-        priv  = this->private;
-        local = frame->local;
-        trav = this->children;
-        loc = &local->loc;
-
-        --local->call_count;
-
-        if (op_ret == -1) {
-                gf_log (this->name, GF_LOG_DEBUG, "%s returned error %s",
-                        prev->this->name, strerror (op_errno));
-                 local->failed = 1;
-                 local->op_errno = op_errno;
-        }
-
-        local->op_ret = 0;
-        /* Get the mapping in inode private */
-        /* Get the stat buf right */
-        local->stbuf      = *buf;
-        local->preparent  = *preparent;
-        local->postparent = *postparent;
-
-        local->stbuf_blocks += buf->ia_blocks;
-        local->preparent_blocks  += preparent->ia_blocks;
-        local->postparent_blocks += postparent->ia_blocks;
-
-        if (local->stbuf_size < buf->ia_size)
-              local->stbuf_size = buf->ia_size;
-        if (local->preparent_size < preparent->ia_size)
-              local->preparent_size = preparent->ia_size;
-        if (local->postparent_size < postparent->ia_size)
-              local->postparent_size = postparent->ia_size;
-
-        if (local->failed)
-                local->op_ret = -1;
-
-        if (local->op_ret == -1) {
-                local->call_count = 1;
-                STACK_WIND (frame, stripe_create_fail_unlink_cbk,
-                            FIRST_CHILD (this), FIRST_CHILD (this)->fops->unlink,
-                            &local->loc, 0, NULL);
-                return 0;
-        }
-
-        if (local->op_ret >= 0) {
-                local->preparent.ia_blocks  = local->preparent_blocks;
-                local->preparent.ia_size    = local->preparent_size;
-                local->postparent.ia_blocks = local->postparent_blocks;
-                local->postparent.ia_size   = local->postparent_size;
-                local->stbuf.ia_size        = local->stbuf_size;
-                local->stbuf.ia_blocks      = local->stbuf_blocks;
-        }
-
-        /* Send a setxattr request to nodes where the
-           files are created */
-        trav = trav->next;
-        while (trav) {
-                if (priv->xattr_supported) {
-                        dict = dict_new ();
-                        if (!dict) {
-                                gf_log (this->name, GF_LOG_ERROR,
-                                        "failed to allocate dict %s", loc->path);
-                        }
-                        need_unref = 1;
-
-                        dict_copy (local->xattr, dict);
-
-                        ret = stripe_xattr_request_build (this, dict,
-                                                          local->stripe_size,
-                                                          priv->child_count,
-                                                          i, priv->coalesce);
-                        if (ret)
-                                gf_log (this->name, GF_LOG_ERROR,
-                                        "failed to build xattr request");
-                } else {
-                        dict = local->xattr;
-                }
-
-                STACK_WIND (frame, stripe_create_cbk, trav->xlator,
-                            trav->xlator->fops->create, &local->loc,
-                            local->flags, local->mode, local->umask, local->fd,
-                            dict);
-                trav = trav->next;
-                if (need_unref && dict)
-                        dict_unref (dict);
-                i++;
-        }
-
-out:
-        return 0;
-}
-
-
-
-/**
- * stripe_create - If a block-size is specified for the 'name', create the
- *    file in all the child nodes. If not, create it in only first child.
- *
- * @name- complete path of the file to be created.
- */
-int32_t
-stripe_create (call_frame_t *frame, xlator_t *this, loc_t *loc,
-               int32_t flags, mode_t mode, mode_t umask, fd_t *fd, dict_t *xdata)
-{
-        stripe_private_t *priv = NULL;
-        stripe_local_t   *local = NULL;
-        int32_t           op_errno = EINVAL;
-        int               ret            = 0;
-        int               need_unref     = 0;
-        int               i              = 0;
-        dict_t           *dict           = NULL;
-
-        VALIDATE_OR_GOTO (frame, err);
-        VALIDATE_OR_GOTO (this, err);
-        VALIDATE_OR_GOTO (loc, err);
-        VALIDATE_OR_GOTO (loc->path, err);
-        VALIDATE_OR_GOTO (loc->inode, err);
-
-        priv = this->private;
-
-        /* files created in O_APPEND mode does not allow lseek() on fd */
-        flags &= ~O_APPEND;
-
-        if (priv->first_child_down || priv->nodes_down) {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "First node down, returning EIO");
-                op_errno = EIO;
-                goto err;
-        }
-
-        /* Initialization */
-        local = mem_get0 (this->local_pool);
-        if (!local) {
-                op_errno = ENOMEM;
-                goto err;
-        }
-        local->op_ret = -1;
-        local->op_errno = ENOTCONN;
-        local->stripe_size = stripe_get_matching_bs (loc->path, priv);
-        frame->local = local;
-        local->inode = inode_ref (loc->inode);
-        loc_copy (&local->loc, loc);
-        local->fd = fd_ref (fd);
-        local->flags = flags;
-        local->mode = mode;
-        local->umask = umask;
-        if (xdata)
-                local->xattr = dict_ref (xdata);
-
-        local->call_count = priv->child_count;
-        /* Send a setxattr request to nodes where the
-           files are created */
-
-        if (priv->xattr_supported) {
-                dict = dict_new ();
-                if (!dict) {
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "failed to allocate dict %s", loc->path);
-                }
-                need_unref = 1;
-
-                dict_copy (xdata, dict);
-
-                ret = stripe_xattr_request_build (this, dict,
-                                                  local->stripe_size,
-                                                  priv->child_count,
-                                                  i, priv->coalesce);
-                if (ret)
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "failed to build xattr request");
-        } else {
-                        dict = xdata;
-        }
-
-
-        STACK_WIND (frame, stripe_first_create_cbk, FIRST_CHILD (this),
-                    FIRST_CHILD (this)->fops->create, loc, flags, mode,
-                    umask, fd, dict);
-
-        if (need_unref && dict)
-                dict_unref (dict);
-
-
-        return 0;
-err:
-        STRIPE_STACK_UNWIND (create, frame, -1, op_errno, NULL, NULL, NULL,
-                             NULL, NULL, xdata);
-        return 0;
-}
-
-int32_t
-stripe_open_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                 int32_t op_ret, int32_t op_errno, fd_t *fd, dict_t *xdata)
-{
-        int32_t         callcnt = 0;
-        stripe_local_t *local = NULL;
-        call_frame_t   *prev = NULL;
-
-        if (!this || !frame || !frame->local || !cookie) {
-                gf_log ("stripe", GF_LOG_DEBUG, "possible NULL deref");
-                goto out;
-        }
-
-        prev  = cookie;
-        local = frame->local;
-
-        LOCK (&frame->lock);
-        {
-                callcnt = --local->call_count;
-
-                if (op_ret == -1) {
-
-                        gf_log (this->name, GF_LOG_DEBUG,
-                                "%s returned error %s",
-                                prev->this->name, strerror (op_errno));
-                        if ((op_errno != ENOENT) ||
-                            (prev->this == FIRST_CHILD (this)))
-                                local->failed = 1;
-                        local->op_errno = op_errno;
-                }
-
-                if (op_ret >= 0)
-                        local->op_ret = op_ret;
-        }
-        UNLOCK (&frame->lock);
-
-        if (!callcnt) {
-                if (local->failed)
-                        local->op_ret = -1;
-
-                STRIPE_STACK_UNWIND (open, frame, local->op_ret,
-                                     local->op_errno, local->fd, xdata);
-        }
-out:
-        return 0;
-}
-
-
-/**
- * stripe_open -
- */
-int32_t
-stripe_open (call_frame_t *frame, xlator_t *this, loc_t *loc,
-             int32_t flags, fd_t *fd, dict_t *xdata)
-{
-        stripe_local_t   *local = NULL;
-        stripe_private_t *priv = NULL;
-        xlator_list_t    *trav = NULL;
-        int32_t           op_errno = 1;
-
-        VALIDATE_OR_GOTO (frame, err);
-        VALIDATE_OR_GOTO (this, err);
-        VALIDATE_OR_GOTO (loc, err);
-        VALIDATE_OR_GOTO (loc->path, err);
-        VALIDATE_OR_GOTO (loc->inode, err);
-
-        priv = this->private;
-        trav = this->children;
-
-        if (priv->first_child_down) {
-                op_errno = ENOTCONN;
-                goto err;
-        }
-
-        /* Initialization */
-        local = mem_get0 (this->local_pool);
-        if (!local) {
-                op_errno = ENOMEM;
-                goto err;
-        }
-
-        /* files opened in O_APPEND mode does not allow lseek() on fd */
-        flags &= ~O_APPEND;
-
-        local->fd = fd_ref (fd);
-        frame->local = local;
-        loc_copy (&local->loc, loc);
-
-        /* Striped files */
-        local->flags = flags;
-        local->call_count = priv->child_count;
-        local->stripe_size = stripe_get_matching_bs (loc->path, priv);
-
-        while (trav) {
-                STACK_WIND (frame, stripe_open_cbk, trav->xlator,
-                            trav->xlator->fops->open,
-                            &local->loc, local->flags, local->fd,
-                            xdata);
-                trav = trav->next;
-        }
-        return 0;
-err:
-        STRIPE_STACK_UNWIND (open, frame, -1, op_errno, NULL, NULL);
-        return 0;
-}
-
-
-int32_t
-stripe_opendir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                    int32_t op_ret, int32_t op_errno, fd_t *fd, dict_t *xdata)
-{
-        int32_t         callcnt = 0;
-        stripe_local_t *local = NULL;
-        call_frame_t   *prev = NULL;
-
-        if (!this || !frame || !frame->local || !cookie) {
-                gf_log ("stripe", GF_LOG_DEBUG, "possible NULL deref");
-                goto out;
-        }
-
-        prev  = cookie;
-        local = frame->local;
-
-        LOCK (&frame->lock);
-        {
-                callcnt = --local->call_count;
-
-                if (op_ret == -1) {
-                        gf_log (this->name, GF_LOG_DEBUG,
-                                "%s returned error %s",
-                                prev->this->name, strerror (op_errno));
-                        local->op_ret = -1;
-                        local->op_errno = op_errno;
-                }
-
-                if (op_ret >= 0)
-                        local->op_ret = op_ret;
-        }
-        UNLOCK (&frame->lock);
-
-        if (!callcnt) {
-                STRIPE_STACK_UNWIND (opendir, frame, local->op_ret,
-                                     local->op_errno, local->fd, NULL);
-        }
-out:
-        return 0;
-}
-
-
-int32_t
-stripe_opendir (call_frame_t *frame, xlator_t *this, loc_t *loc, fd_t *fd, dict_t *xdata)
-{
-        xlator_list_t    *trav = NULL;
-        stripe_local_t   *local = NULL;
-        stripe_private_t *priv = NULL;
-        int32_t           op_errno = EINVAL;
-
-        VALIDATE_OR_GOTO (frame, err);
-        VALIDATE_OR_GOTO (this, err);
-        VALIDATE_OR_GOTO (loc, err);
-        VALIDATE_OR_GOTO (loc->path, err);
-        VALIDATE_OR_GOTO (loc->inode, err);
-
-        priv = this->private;
-        trav = this->children;
-
-        if (priv->first_child_down) {
-                op_errno = ENOTCONN;
-                goto err;
-        }
-
-        /* Initialization */
-        local = mem_get0 (this->local_pool);
-        if (!local) {
-                op_errno = ENOMEM;
-                goto err;
-        }
-        frame->local = local;
-        local->call_count = priv->child_count;
-        local->fd = fd_ref (fd);
-
-        while (trav) {
-                STACK_WIND (frame, stripe_opendir_cbk, trav->xlator,
-                            trav->xlator->fops->opendir, loc, fd, NULL);
-                trav = trav->next;
-        }
-
-        return 0;
-err:
-        STRIPE_STACK_UNWIND (opendir, frame, -1, op_errno, NULL, NULL);
-        return 0;
-}
-
-int32_t
-stripe_lk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-               int32_t op_ret, int32_t op_errno, struct gf_flock *lock, dict_t *xdata)
-{
-        int32_t         callcnt = 0;
-        stripe_local_t *local = NULL;
-        call_frame_t   *prev = NULL;
-
-        if (!this || !frame || !frame->local || !cookie) {
-                gf_log ("stripe", GF_LOG_DEBUG, "possible NULL deref");
-                goto out;
-        }
-
-        prev  = cookie;
-        local = frame->local;
-
-        LOCK (&frame->lock);
-        {
-                callcnt = --local->call_count;
-                if (op_ret == -1) {
-                        gf_log (this->name, GF_LOG_DEBUG,
-                                "%s returned error %s",
-                                prev->this->name, strerror (op_errno));
-                        local->op_errno = op_errno;
-                        if ((op_errno != ENOENT) ||
-                            (prev->this == FIRST_CHILD (this)))
-                                local->failed = 1;
-                }
-                if (op_ret >= 0) {
-                        if (FIRST_CHILD(this) == prev->this) {
-                                /* First successful call, copy the *lock */
-                                local->op_ret = op_ret;
-                                local->lock = *lock;
-                        }
-                }
-        }
-        UNLOCK (&frame->lock);
-
-        if (!callcnt) {
-                if (local->failed)
-                        local->op_ret = -1;
-                STRIPE_STACK_UNWIND (lk, frame, local->op_ret,
-                                     local->op_errno, &local->lock, NULL);
-        }
-out:
-        return 0;
-}
-
-int32_t
-stripe_lk (call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t cmd,
-           struct gf_flock *lock, dict_t *xdata)
-{
-        stripe_local_t   *local = NULL;
-        xlator_list_t    *trav = NULL;
-        stripe_private_t *priv = NULL;
-        int32_t           op_errno = EINVAL;
-
-        VALIDATE_OR_GOTO (frame, err);
-        VALIDATE_OR_GOTO (this, err);
-        VALIDATE_OR_GOTO (fd, err);
-        VALIDATE_OR_GOTO (fd->inode, err);
-
-        trav = this->children;
-        priv = this->private;
-
-        /* Initialization */
-        local = mem_get0 (this->local_pool);
-        if (!local) {
-                op_errno = ENOMEM;
-                goto err;
-        }
-        local->op_ret = -1;
-        frame->local = local;
-        local->call_count = priv->child_count;
-
-        while (trav) {
-                STACK_WIND (frame, stripe_lk_cbk, trav->xlator,
-                            trav->xlator->fops->lk, fd, cmd, lock, NULL);
-                trav = trav->next;
-        }
-
-        return 0;
-err:
-        STRIPE_STACK_UNWIND (lk, frame, -1, op_errno, NULL, NULL);
-        return 0;
-}
-
-
-int32_t
-stripe_flush_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                  int32_t op_ret, int32_t op_errno, dict_t *xdata)
-{
-        int32_t         callcnt = 0;
-        stripe_local_t *local   = NULL;
-        call_frame_t   *prev = NULL;
-
-        if (!this || !frame || !frame->local || !cookie) {
-                gf_log ("stripe", GF_LOG_DEBUG, "possible NULL deref");
-                goto out;
-        }
-
-        prev  = cookie;
-        local = frame->local;
-
-        LOCK (&frame->lock);
-        {
-                callcnt = --local->call_count;
-
-                if (op_ret == -1) {
-                        gf_log (this->name, GF_LOG_DEBUG,
-                                "%s returned %s",
-                                prev->this->name, strerror (op_errno));
-                        local->op_errno = op_errno;
-                        if ((op_errno != ENOENT) ||
-                            (prev->this == FIRST_CHILD (this)))
-                                local->failed = 1;
-                }
-                if (op_ret >= 0)
-                        local->op_ret = op_ret;
-        }
-        UNLOCK (&frame->lock);
-
-        if (!callcnt) {
-                if (local->failed)
-                        local->op_ret = -1;
-
-                STRIPE_STACK_UNWIND (flush, frame, local->op_ret,
-                                     local->op_errno, NULL);
-        }
-out:
-        return 0;
-}
-
-int32_t
-stripe_flush (call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *xdata)
-{
-        stripe_local_t   *local = NULL;
-        stripe_private_t *priv = NULL;
-        xlator_list_t    *trav = NULL;
-        int32_t           op_errno = 1;
-
-        VALIDATE_OR_GOTO (frame, err);
-        VALIDATE_OR_GOTO (this, err);
-        VALIDATE_OR_GOTO (fd, err);
-        VALIDATE_OR_GOTO (fd->inode, err);
-
-        priv = this->private;
-        trav = this->children;
-
-        if (priv->first_child_down) {
-                op_errno = ENOTCONN;
-                goto err;
-        }
-        /* Initialization */
-        local = mem_get0 (this->local_pool);
-        if (!local) {
-                op_errno = ENOMEM;
-                goto err;
-        }
-        local->op_ret = -1;
-        frame->local = local;
-        local->call_count = priv->child_count;
-
-        while (trav) {
-                STACK_WIND (frame, stripe_flush_cbk,  trav->xlator,
-                            trav->xlator->fops->flush, fd, NULL);
-                trav = trav->next;
-        }
-
-        return 0;
-err:
-        STRIPE_STACK_UNWIND (flush, frame, -1, op_errno, NULL);
-        return 0;
-}
-
-
-
-int32_t
-stripe_fsync_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                  int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
-                  struct iatt *postbuf, dict_t *xdata)
-{
-        int32_t         callcnt = 0;
-        stripe_local_t *local   = NULL;
-        call_frame_t   *prev = NULL;
-
-        if (!this || !frame || !frame->local || !cookie) {
-                gf_log ("stripe", GF_LOG_DEBUG, "possible NULL deref");
-                goto out;
-        }
-
-        prev = cookie;
-        local = frame->local;
-
-        LOCK (&frame->lock);
-        {
-                callcnt = --local->call_count;
-
-                if (op_ret == -1) {
-                        gf_log (this->name, GF_LOG_DEBUG,
-                                "%s returned %s",
-                                prev->this->name, strerror (op_errno));
-                        local->op_errno = op_errno;
-                        if ((op_errno != ENOENT) ||
-                            (prev->this == FIRST_CHILD (this)))
-                                local->failed = 1;
-                }
-                if (op_ret >= 0) {
-                        local->op_ret = op_ret;
-                        if (FIRST_CHILD(this) == prev->this) {
-                                local->pre_buf  = *prebuf;
-                                local->post_buf = *postbuf;
-                        }
-                        local->prebuf_blocks  += prebuf->ia_blocks;
-                        local->postbuf_blocks += postbuf->ia_blocks;
-
-			correct_file_size(prebuf, local->fctx, prev);
-			correct_file_size(postbuf, local->fctx, prev);
-
-                        if (local->prebuf_size < prebuf->ia_size)
-                                local->prebuf_size = prebuf->ia_size;
-
-                        if (local->postbuf_size < postbuf->ia_size)
-                                local->postbuf_size = postbuf->ia_size;
-                }
-        }
-        UNLOCK (&frame->lock);
-
-        if (!callcnt) {
-                if (local->failed)
-                        local->op_ret = -1;
-
-                if (local->op_ret != -1) {
-                        local->pre_buf.ia_blocks  = local->prebuf_blocks;
-                        local->pre_buf.ia_size    = local->prebuf_size;
-                        local->post_buf.ia_blocks = local->postbuf_blocks;
-                        local->post_buf.ia_size   = local->postbuf_size;
-                }
-
-                STRIPE_STACK_UNWIND (fsync, frame, local->op_ret,
-                                     local->op_errno, &local->pre_buf,
-                                     &local->post_buf, NULL);
-        }
-out:
-        return 0;
-}
-
-int32_t
-stripe_fsync (call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t flags, dict_t *xdata)
-{
-        stripe_local_t   *local = NULL;
-        stripe_private_t *priv = NULL;
-        xlator_list_t    *trav = NULL;
-	stripe_fd_ctx_t  *fctx = NULL;
-        int32_t           op_errno = 1;
-
-        VALIDATE_OR_GOTO (frame, err);
-        VALIDATE_OR_GOTO (this, err);
-        VALIDATE_OR_GOTO (fd, err);
-        VALIDATE_OR_GOTO (fd->inode, err);
-
-        priv = this->private;
-        trav = this->children;
-
-        /* Initialization */
-        local = mem_get0 (this->local_pool);
-        if (!local) {
-                op_errno = ENOMEM;
-                goto err;
-        }
-
-        frame->local = local;
-
-	inode_ctx_get(fd->inode, this, (uint64_t *) &fctx);
-	if (!fctx) {
-		op_errno = EINVAL;
-		goto err;
-	}
-	local->fctx = fctx;
-        local->op_ret = -1;
-        local->call_count = priv->child_count;
-
-        while (trav) {
-                STACK_WIND (frame, stripe_fsync_cbk, trav->xlator,
-                            trav->xlator->fops->fsync, fd, flags, NULL);
-                trav = trav->next;
-        }
-
-        return 0;
-err:
-        STRIPE_STACK_UNWIND (fsync, frame, -1, op_errno, NULL, NULL, NULL);
-        return 0;
-}
-
-int32_t
-stripe_fstat_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                  int32_t op_ret, int32_t op_errno, struct iatt *buf, dict_t *xdata)
-{
-        int32_t         callcnt = 0;
-        stripe_local_t *local = NULL;
-        call_frame_t   *prev = NULL;
-
-        if (!this || !frame || !frame->local || !cookie) {
-                gf_log ("stripe", GF_LOG_DEBUG, "possible NULL deref");
-                goto out;
-        }
-
-        prev  = cookie;
-        local = frame->local;
-
-        LOCK (&frame->lock);
-        {
-                callcnt = --local->call_count;
-
-                if (op_ret == -1) {
-                        gf_log (this->name, GF_LOG_DEBUG,
-                                "%s returned error %s",
-                                prev->this->name, strerror (op_errno));
-                        local->op_errno = op_errno;
-                        if ((op_errno != ENOENT) ||
-                            (prev->this == FIRST_CHILD (this)))
-                                local->failed = 1;
-                }
-
-                if (op_ret == 0) {
-                        local->op_ret = 0;
-
-                        if (FIRST_CHILD(this) == prev->this)
-                                local->stbuf = *buf;
-
-                        local->stbuf_blocks += buf->ia_blocks;
-
-			correct_file_size(buf, local->fctx, prev);
-
-                        if (local->stbuf_size < buf->ia_size)
-                                local->stbuf_size = buf->ia_size;
-                }
-        }
-        UNLOCK (&frame->lock);
-
-        if (!callcnt) {
-                if (local->failed)
-                        local->op_ret = -1;
-
-                if (local->op_ret != -1) {
-                        local->stbuf.ia_size   = local->stbuf_size;
-                        local->stbuf.ia_blocks = local->stbuf_blocks;
-                }
-
-                STRIPE_STACK_UNWIND (fstat, frame, local->op_ret,
-                                     local->op_errno, &local->stbuf, NULL);
-        }
-
-out:
-        return 0;
-}
-
-int32_t
-stripe_fstat (call_frame_t *frame,
-              xlator_t *this,
-              fd_t *fd, dict_t *xdata)
-{
-        stripe_local_t   *local = NULL;
-        stripe_private_t *priv = NULL;
-        xlator_list_t    *trav = NULL;
-	stripe_fd_ctx_t  *fctx = NULL;
-        int32_t           op_errno = 1;
-
-        VALIDATE_OR_GOTO (frame, err);
-        VALIDATE_OR_GOTO (this, err);
-        VALIDATE_OR_GOTO (fd, err);
-        VALIDATE_OR_GOTO (fd->inode, err);
-
-        priv = this->private;
-        trav = this->children;
-
-        /* Initialization */
-        local = mem_get0 (this->local_pool);
-        if (!local) {
-                op_errno = ENOMEM;
-                goto err;
-        }
-        local->op_ret = -1;
-        frame->local = local;
-        local->call_count = priv->child_count;
-
-	if (IA_ISREG(fd->inode->ia_type)) {
-		inode_ctx_get(fd->inode, this, (uint64_t *) &fctx);
-		if (!fctx)
-			goto err;
-		local->fctx = fctx;
-	}
-
-        while (trav) {
-                STACK_WIND (frame, stripe_fstat_cbk, trav->xlator,
-                            trav->xlator->fops->fstat, fd, NULL);
-                trav = trav->next;
-        }
-
-        return 0;
-err:
-        STRIPE_STACK_UNWIND (fstat, frame, -1, op_errno, NULL, NULL);
-        return 0;
-}
-
-
-int32_t
-stripe_ftruncate (call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset, dict_t *xdata)
-{
-        stripe_local_t   *local = NULL;
-        stripe_private_t *priv = NULL;
-	stripe_fd_ctx_t  *fctx = NULL;
-	int		  i, eof_idx;
-	off_t		  dest_offset, tmp_offset;
-        int32_t		  op_errno = 1;
-
-        VALIDATE_OR_GOTO (frame, err);
-        VALIDATE_OR_GOTO (this, err);
-        VALIDATE_OR_GOTO (fd, err);
-        VALIDATE_OR_GOTO (fd->inode, err);
-
-        priv = this->private;
-
-        /* Initialization */
-        local = mem_get0 (this->local_pool);
-        if (!local) {
-                op_errno = ENOMEM;
-                goto err;
-        }
-        local->op_ret = -1;
-        frame->local = local;
-        local->call_count = priv->child_count;
-
-	inode_ctx_get(fd->inode, this, (uint64_t *) &fctx);
-	if (!fctx) {
-		gf_log(this->name, GF_LOG_ERROR, "no stripe context");
-		op_errno = EINVAL;
-		goto err;
-	}
-	if (!fctx->stripe_count) {
-		gf_log(this->name, GF_LOG_ERROR, "no stripe count");
-		op_errno = EINVAL;
-		goto err;
-	}
-
-	local->fctx = fctx;
-	eof_idx = (offset / fctx->stripe_size) % fctx->stripe_count;
-
-	for (i = 0; i < fctx->stripe_count; i++) {
-		if (!fctx->xl_array[i]) {
-			gf_log(this->name, GF_LOG_ERROR, "no xlator at index "
-				"%d", i);
-			op_errno = EINVAL;
-			goto err;
-		}
-
-		if (fctx->stripe_coalesce) {
-			if (i < eof_idx)
-				tmp_offset = roof(offset, fctx->stripe_size *
-						fctx->stripe_count);
-			else if (i > eof_idx)
-				tmp_offset = floor(offset, fctx->stripe_size *
-						fctx->stripe_count);
-			else
-				tmp_offset = offset;
-
-			dest_offset = coalesced_offset(tmp_offset,
-				fctx->stripe_size, fctx->stripe_count);
-		} else {
-			dest_offset = offset;
-		}
-
-		STACK_WIND(frame, stripe_truncate_cbk, fctx->xl_array[i],
-			fctx->xl_array[i]->fops->ftruncate, fd, dest_offset,
-			NULL);
-	}
-
-        return 0;
-err:
-        STRIPE_STACK_UNWIND (ftruncate, frame, -1, op_errno, NULL, NULL, NULL);
-        return 0;
-}
-
-
-int32_t
-stripe_fsyncdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                     int32_t op_ret, int32_t op_errno, dict_t *xdata)
-{
-        int32_t         callcnt = 0;
-        stripe_local_t *local   = NULL;
-        call_frame_t   *prev = NULL;
-
-        if (!this || !frame || !frame->local || !cookie) {
-                gf_log ("stripe", GF_LOG_DEBUG, "possible NULL deref");
-                goto out;
-        }
-
-        prev  = cookie;
-        local = frame->local;
-
-        LOCK (&frame->lock);
-        {
-                callcnt = --local->call_count;
-
-                if (op_ret == -1) {
-                        gf_log (this->name, GF_LOG_DEBUG,
-                                "%s returned %s",
-                                prev->this->name, strerror (op_errno));
-                        local->op_errno = op_errno;
-                        if ((op_errno != ENOENT) ||
-                            (prev->this == FIRST_CHILD (this)))
-                                local->failed = 1;
-                }
-                if (op_ret >= 0)
-                        local->op_ret = op_ret;
-        }
-        UNLOCK (&frame->lock);
-
-        if (!callcnt) {
-                if (local->failed)
-                        local->op_ret = -1;
-
-                STRIPE_STACK_UNWIND (fsyncdir, frame, local->op_ret,
-                                     local->op_errno, NULL);
-        }
-out:
-        return 0;
-}
-
-int32_t
-stripe_fsyncdir (call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t flags, dict_t *xdata)
-{
-        stripe_local_t   *local = NULL;
-        stripe_private_t *priv = NULL;
-        xlator_list_t    *trav = NULL;
-        int32_t           op_errno = 1;
-
-        VALIDATE_OR_GOTO (frame, err);
-        VALIDATE_OR_GOTO (this, err);
-        VALIDATE_OR_GOTO (fd, err);
-        VALIDATE_OR_GOTO (fd->inode, err);
-
-        priv = this->private;
-        trav = this->children;
-
-        /* Initialization */
-        local = mem_get0 (this->local_pool);
-        if (!local) {
-                op_errno = ENOMEM;
-                goto err;
-        }
-        local->op_ret = -1;
-        frame->local = local;
-        local->call_count = priv->child_count;
-
-        while (trav) {
-                STACK_WIND (frame, stripe_fsyncdir_cbk, trav->xlator,
-                            trav->xlator->fops->fsyncdir, fd, flags, NULL);
-                trav = trav->next;
-        }
-
-        return 0;
-err:
-        STRIPE_STACK_UNWIND (fsyncdir, frame, -1, op_errno, NULL);
-        return 0;
-}
-
-
-int32_t
-stripe_readv_fstat_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                        int32_t op_ret, int32_t op_errno, struct iatt *buf, dict_t *xdata)
-{
-        int32_t         i = 0;
-        int32_t         callcnt = 0;
-        int32_t         count = 0;
-        stripe_local_t *local = NULL;
-        struct iovec   *vec = NULL;
-        struct iatt     tmp_stbuf = {0,};
-        struct iobref  *tmp_iobref = NULL;
-        struct iobuf   *iobuf = NULL;
-	call_frame_t   *prev = NULL;
-
-        if (!this || !frame || !frame->local) {
-                gf_log ("stripe", GF_LOG_DEBUG, "possible NULL deref");
-                goto out;
-        }
-
-        local = frame->local;
-	prev = cookie;
-
-        LOCK (&frame->lock);
-        {
-                callcnt = --local->call_count;
-                if (op_ret != -1) {
-			correct_file_size(buf, local->fctx, prev);
-                        if (local->stbuf_size < buf->ia_size)
-                                local->stbuf_size = buf->ia_size;
-		}
-        }
-        UNLOCK (&frame->lock);
-
-        if (!callcnt) {
-                op_ret = 0;
-
-                /* Keep extra space for filling in '\0's */
-                vec = GF_CALLOC ((local->count * 2), sizeof (struct iovec),
-                                 gf_stripe_mt_iovec);
-                if (!vec) {
-                        op_ret = -1;
-                        goto done;
-                }
-
-                for (i = 0; i < local->wind_count; i++) {
-                        if (local->replies[i].op_ret) {
-                                memcpy ((vec + count), local->replies[i].vector,
-                                        (local->replies[i].count * sizeof (struct iovec)));
-                                count +=  local->replies[i].count;
-                                op_ret += local->replies[i].op_ret;
-                        }
-                        if ((local->replies[i].op_ret <
-                             local->replies[i].requested_size) &&
-                            (local->stbuf_size > (local->offset + op_ret))) {
-                                /* Fill in 0s here */
-                                vec[count].iov_len  =
-                                        (local->replies[i].requested_size -
-                                         local->replies[i].op_ret);
-                                iobuf = iobuf_get2 (this->ctx->iobuf_pool,
-                                                    vec[count].iov_len);
-                                if (!iobuf) {
-                                        gf_log (this->name, GF_LOG_ERROR,
-                                                "Out of memory.");
-                                        op_ret = -1;
-                                        op_errno = ENOMEM;
-                                        goto done;
-                                }
-                                memset (iobuf->ptr, 0, vec[count].iov_len);
-                                vec[count].iov_base = iobuf->ptr;
-
-                                iobref_add (local->iobref, iobuf);
-                                iobuf_unref(iobuf);
-
-                                op_ret += vec[count].iov_len;
-                                count++;
-                        }
-                        GF_FREE (local->replies[i].vector);
-                }
-
-                /* FIXME: notice that st_ino, and st_dev (gen) will be
-                 * different than what inode will have. Make sure this doesn't
-                 * cause any bugs at higher levels */
-                memcpy (&tmp_stbuf, &local->replies[0].stbuf,
-                        sizeof (struct iatt));
-                tmp_stbuf.ia_size = local->stbuf_size;
-
-        done:
-                GF_FREE (local->replies);
-                tmp_iobref = local->iobref;
-                STRIPE_STACK_UNWIND (readv, frame, op_ret, op_errno, vec,
-                                     count, &tmp_stbuf, tmp_iobref, NULL);
-
-                iobref_unref (tmp_iobref);
-                GF_FREE (vec);
-        }
-out:
-        return 0;
-}
-
-/**
- * stripe_readv_cbk - get all the striped reads, and order it properly, send it
- *        to above layer after putting it in a single vector.
- */
-int32_t
-stripe_readv_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                  int32_t op_ret, int32_t op_errno, struct iovec *vector,
-                  int32_t count, struct iatt *stbuf, struct iobref *iobref, dict_t *xdata)
-{
-        int32_t         index = 0;
-        int32_t         callcnt = 0;
-        int32_t         final_count = 0;
-        int32_t         need_to_check_proper_size = 0;
-        call_frame_t   *mframe = NULL;
-        stripe_local_t *mlocal = NULL;
-        stripe_local_t *local = NULL;
-        struct iovec   *final_vec = NULL;
-        struct iatt     tmp_stbuf = {0,};
-        struct iatt    *tmp_stbuf_p = NULL; //need it for a warning
-        struct iobref  *tmp_iobref = NULL;
-        stripe_fd_ctx_t  *fctx = NULL;
-	call_frame_t	*prev = NULL;
-
-        if (!this || !frame || !frame->local || !cookie) {
-                gf_log ("stripe", GF_LOG_DEBUG, "possible NULL deref");
-                goto end;
-        }
-
-        local  = frame->local;
-        index  = local->node_index;
-	prev = cookie;
-        mframe = local->orig_frame;
-        if (!mframe)
-                goto out;
-
-        mlocal = mframe->local;
-        if (!mlocal)
-                goto out;
-
-        fctx = mlocal->fctx;
-
-        LOCK (&mframe->lock);
-        {
-                mlocal->replies[index].op_ret = op_ret;
-                mlocal->replies[index].op_errno = op_errno;
-                mlocal->replies[index].requested_size = local->readv_size;
-                if (op_ret >= 0) {
-                        mlocal->replies[index].stbuf  = *stbuf;
-                        mlocal->replies[index].count  = count;
-                        mlocal->replies[index].vector = iov_dup (vector, count);
-
-			correct_file_size(stbuf, fctx, prev);
-
-                        if (local->stbuf_size < stbuf->ia_size)
-                                local->stbuf_size = stbuf->ia_size;
-                        local->stbuf_blocks += stbuf->ia_blocks;
-
-                        if (!mlocal->iobref)
-                                mlocal->iobref = iobref_new ();
-                        iobref_merge (mlocal->iobref, iobref);
-                }
-                callcnt = ++mlocal->call_count;
-        }
-        UNLOCK(&mframe->lock);
-
-        if (callcnt == mlocal->wind_count) {
-                op_ret = 0;
-
-                for (index=0; index < mlocal->wind_count; index++) {
-                        /* check whether each stripe returned
-                         * 'expected' number of bytes */
-                        if (mlocal->replies[index].op_ret == -1) {
-                                op_ret = -1;
-                                op_errno = mlocal->replies[index].op_errno;
-                                break;
-                        }
-                        /* TODO: handle the 'holes' within the read range
-                           properly */
-                        if (mlocal->replies[index].op_ret <
-                            mlocal->replies[index].requested_size) {
-                                need_to_check_proper_size = 1;
-                        }
-
-                        op_ret       += mlocal->replies[index].op_ret;
-                        mlocal->count += mlocal->replies[index].count;
-                }
-                if (op_ret == -1)
-                        goto done;
-                if (need_to_check_proper_size)
-                        goto check_size;
-
-                final_vec = GF_CALLOC (mlocal->count, sizeof (struct iovec),
-                                       gf_stripe_mt_iovec);
-
-                if (!final_vec) {
-                        op_ret = -1;
-                        goto done;
-                }
-
-                for (index = 0; index < mlocal->wind_count; index++) {
-                        memcpy ((final_vec + final_count),
-                                mlocal->replies[index].vector,
-                                (mlocal->replies[index].count *
-                                 sizeof (struct iovec)));
-                        final_count +=  mlocal->replies[index].count;
-                        GF_FREE (mlocal->replies[index].vector);
-                }
-
-                /* FIXME: notice that st_ino, and st_dev (gen) will be
-                 * different than what inode will have. Make sure this doesn't
-                 * cause any bugs at higher levels */
-                memcpy (&tmp_stbuf, &mlocal->replies[0].stbuf,
-                        sizeof (struct iatt));
-                tmp_stbuf.ia_size = local->stbuf_size;
-                tmp_stbuf.ia_blocks = local->stbuf_blocks;
-
-        done:
-                /* */
-                GF_FREE (mlocal->replies);
-                tmp_iobref = mlocal->iobref;
-                /* work around for nfs truncated read. Bug 3774 */
-                tmp_stbuf_p = &tmp_stbuf;
-                WIPE (tmp_stbuf_p);
-                STRIPE_STACK_UNWIND (readv, mframe, op_ret, op_errno, final_vec,
-                                     final_count, &tmp_stbuf, tmp_iobref, NULL);
-
-                iobref_unref (tmp_iobref);
-                GF_FREE (final_vec);
-        }
-
-        goto out;
-
-check_size:
-        mlocal->call_count = fctx->stripe_count;
-
-        for (index = 0; index < fctx->stripe_count; index++) {
-                STACK_WIND (mframe, stripe_readv_fstat_cbk,
-                            (fctx->xl_array[index]),
-                            (fctx->xl_array[index])->fops->fstat,
-                            mlocal->fd, NULL);
-        }
-
-out:
-        STRIPE_STACK_DESTROY (frame);
-end:
-        return 0;
-}
-
-
-int32_t
-stripe_readv (call_frame_t *frame, xlator_t *this, fd_t *fd,
-              size_t size, off_t offset, uint32_t flags, dict_t *xdata)
-{
-        int32_t           op_errno = EINVAL;
-        int32_t           idx = 0;
-        int32_t           index = 0;
-        int32_t           num_stripe = 0;
-        int32_t           off_index = 0;
-        size_t            frame_size = 0;
-        off_t             rounded_end = 0;
-        uint64_t          tmp_fctx = 0;
-        uint64_t          stripe_size = 0;
-        off_t             rounded_start = 0;
-        off_t             frame_offset = offset;
-	off_t		  dest_offset = 0;
-        stripe_local_t   *local = NULL;
-        call_frame_t     *rframe = NULL;
-        stripe_local_t   *rlocal = NULL;
-        stripe_fd_ctx_t  *fctx = NULL;
-
-        VALIDATE_OR_GOTO (frame, err);
-        VALIDATE_OR_GOTO (this, err);
-        VALIDATE_OR_GOTO (fd, err);
-        VALIDATE_OR_GOTO (fd->inode, err);
-
-        inode_ctx_get (fd->inode, this, &tmp_fctx);
-        if (!tmp_fctx) {
-                op_errno = EBADFD;
-                goto err;
-        }
-        fctx = (stripe_fd_ctx_t *)(long)tmp_fctx;
-        stripe_size = fctx->stripe_size;
-
-        STRIPE_VALIDATE_FCTX (fctx, err);
-
-        if (!stripe_size) {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "Wrong stripe size for the file");
-                goto err;
-        }
-        /* The file is stripe across the child nodes. Send the read request
-         * to the child nodes appropriately after checking which region of
-         * the file is in which child node. Always '0-<stripe_size>' part of
-         * the file resides in the first child.
-         */
-        rounded_start = floor (offset, stripe_size);
-        rounded_end = roof (offset+size, stripe_size);
-        num_stripe = (rounded_end- rounded_start)/stripe_size;
-
-        local = mem_get0 (this->local_pool);
-        if (!local) {
-                op_errno = ENOMEM;
-                goto err;
-        }
-        frame->local = local;
-
-        /* This is where all the vectors should be copied. */
-        local->replies = GF_CALLOC (num_stripe, sizeof (struct stripe_replies),
-                                    gf_stripe_mt_stripe_replies);
-        if (!local->replies) {
-                op_errno = ENOMEM;
-                goto err;
-        }
-
-        off_index = (offset / stripe_size) % fctx->stripe_count;
-        local->wind_count = num_stripe;
-        local->readv_size = size;
-        local->offset     = offset;
-        local->fd         = fd_ref (fd);
-        local->fctx       = fctx;
-
-        for (index = off_index; index < (num_stripe + off_index); index++) {
-                rframe = copy_frame (frame);
-                rlocal = mem_get0 (this->local_pool);
-                if (!rlocal) {
-                        op_errno = ENOMEM;
-                        goto err;
-                }
-
-                frame_size = min (roof (frame_offset+1, stripe_size),
-                                  (offset + size)) - frame_offset;
-
-                rlocal->node_index = index - off_index;
-                rlocal->orig_frame = frame;
-                rlocal->readv_size = frame_size;
-                rframe->local = rlocal;
-                idx = (index % fctx->stripe_count);
-
-		if (fctx->stripe_coalesce)
-			dest_offset = coalesced_offset(frame_offset,
-				stripe_size, fctx->stripe_count);
-		else
-			dest_offset = frame_offset;
-
-                STACK_WIND (rframe, stripe_readv_cbk, fctx->xl_array[idx],
-                            fctx->xl_array[idx]->fops->readv,
-                            fd, frame_size, dest_offset, flags, xdata);
-
-                frame_offset += frame_size;
-        }
-
-        return 0;
-err:
-        if (rframe)
-                STRIPE_STACK_DESTROY (rframe);
-
-        STRIPE_STACK_UNWIND (readv, frame, -1, op_errno, NULL, 0, NULL, NULL, NULL);
-        return 0;
-}
-
-
-int32_t
-stripe_writev_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                   int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
-                   struct iatt *postbuf, dict_t *xdata)
-{
-        int32_t         callcnt = 0;
-        stripe_local_t *local = NULL;
-	stripe_local_t *mlocal = NULL;
-        call_frame_t   *prev = NULL;
-	call_frame_t   *mframe = NULL;
-	struct stripe_replies *reply = NULL;
-	int32_t		i = 0;
-
-        if (!this || !frame || !frame->local || !cookie) {
-                gf_log ("stripe", GF_LOG_DEBUG, "possible NULL deref");
-                goto out;
-        }
-
-        prev  = cookie;
-        local = frame->local;
-	mframe = local->orig_frame;
-	mlocal = mframe->local;
-
-        LOCK(&frame->lock);
-        {
-                callcnt = ++mlocal->call_count;
-
-		mlocal->replies[local->node_index].op_ret = op_ret;
-		mlocal->replies[local->node_index].op_errno = op_errno;
-
-                if (op_ret >= 0) {
-                        mlocal->post_buf = *postbuf;
-                        mlocal->pre_buf = *prebuf;
-
-			mlocal->prebuf_blocks  += prebuf->ia_blocks;
-			mlocal->postbuf_blocks += postbuf->ia_blocks;
-
-			correct_file_size(prebuf, mlocal->fctx, prev);
-			correct_file_size(postbuf, mlocal->fctx, prev);
-
-			if (mlocal->prebuf_size < prebuf->ia_size)
-				mlocal->prebuf_size = prebuf->ia_size;
-			if (mlocal->postbuf_size < postbuf->ia_size)
-				mlocal->postbuf_size = postbuf->ia_size;
-                }
-        }
-        UNLOCK (&frame->lock);
-
-        if ((callcnt == mlocal->wind_count) && mlocal->unwind) {
-		mlocal->pre_buf.ia_size = mlocal->prebuf_size;
-		mlocal->pre_buf.ia_blocks = mlocal->prebuf_blocks;
-		mlocal->post_buf.ia_size = mlocal->postbuf_size;
-		mlocal->post_buf.ia_blocks = mlocal->postbuf_blocks;
-
-		/*
-		 * Only return the number of consecutively written bytes up until
-		 * the first error. Only return an error if it occurs first.
-		 *
-		 * When a short write occurs, the application should retry at the
-		 * appropriate offset, at which point we'll potentially pass back
-		 * the error.
-		 */
-		for (i = 0, reply = mlocal->replies; i < mlocal->wind_count;
-			i++, reply++) {
-			if (reply->op_ret == -1) {
-				gf_log(this->name, GF_LOG_DEBUG, "reply %d "
-					"returned error %s", i,
-					strerror(reply->op_errno));
-				if (!mlocal->op_ret) {
-					mlocal->op_ret = -1;
-					mlocal->op_errno = reply->op_errno;
-				}
-				break;
-			}
-
-			mlocal->op_ret += reply->op_ret;
-
-			if (reply->op_ret < reply->requested_size)
-				break;
-		}
-
-		GF_FREE(mlocal->replies);
-
-                STRIPE_STACK_UNWIND (writev, mframe, mlocal->op_ret,
-                                     mlocal->op_errno, &mlocal->pre_buf,
-                                     &mlocal->post_buf, NULL);
-        }
-out:
-	STRIPE_STACK_DESTROY(frame);
-        return 0;
-}
-
-int32_t
-stripe_writev (call_frame_t *frame, xlator_t *this, fd_t *fd,
-               struct iovec *vector, int32_t count, off_t offset,
-               uint32_t flags, struct iobref *iobref, dict_t *xdata)
-{
-        struct iovec     *tmp_vec = NULL;
-        stripe_local_t   *local = NULL;
-        stripe_fd_ctx_t  *fctx = NULL;
-        int32_t           op_errno = 1;
-        int32_t           idx = 0;
-        int32_t           total_size = 0;
-        int32_t           offset_offset = 0;
-        int32_t           remaining_size = 0;
-        int32_t           tmp_count = count;
-        off_t             fill_size = 0;
-        uint64_t          stripe_size = 0;
-        uint64_t          tmp_fctx = 0;
-	off_t		  dest_offset = 0;
-	off_t		  rounded_start = 0;
-	off_t		  rounded_end = 0;
-	int32_t		  total_chunks = 0;
-	call_frame_t	  *wframe = NULL;
-	stripe_local_t	  *wlocal = NULL;
-
-        VALIDATE_OR_GOTO (frame, err);
-        VALIDATE_OR_GOTO (this, err);
-        VALIDATE_OR_GOTO (fd, err);
-        VALIDATE_OR_GOTO (fd->inode, err);
-
-        inode_ctx_get (fd->inode, this, &tmp_fctx);
-        if (!tmp_fctx) {
-                op_errno = EINVAL;
-                goto err;
-        }
-        fctx = (stripe_fd_ctx_t *)(long)tmp_fctx;
-        stripe_size = fctx->stripe_size;
-
-        STRIPE_VALIDATE_FCTX (fctx, err);
-
-        /* File has to be stripped across the child nodes */
-        for (idx = 0; idx< count; idx ++) {
-                total_size += vector[idx].iov_len;
-        }
-        remaining_size = total_size;
-
-        local = mem_get0 (this->local_pool);
-        if (!local) {
-                op_errno = ENOMEM;
-                goto err;
-        }
-        frame->local = local;
-        local->stripe_size = stripe_size;
-	local->fctx = fctx;
-
-        if (!stripe_size) {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "Wrong stripe size for the file");
-                op_errno = EINVAL;
-                goto err;
-        }
-
-	rounded_start = floor(offset, stripe_size);
-	rounded_end = roof(offset + total_size, stripe_size);
-	total_chunks = (rounded_end - rounded_start) / stripe_size;
-	local->replies = GF_CALLOC(total_chunks, sizeof(struct stripe_replies),
-				gf_stripe_mt_stripe_replies);
-	if (!local->replies) {
-		op_errno = ENOMEM;
-		goto err;
-	}
-
-	total_chunks = 0;
-        while (1) {
-		wframe = copy_frame(frame);
-		wlocal = mem_get0(this->local_pool);
-		if (!wlocal) {
-			op_errno = ENOMEM;
-			goto err;
-		}
-		wlocal->orig_frame = frame;
-		wframe->local = wlocal;
-
-                /* Send striped chunk of the vector to child
-                   nodes appropriately. */
-                idx = (((offset + offset_offset) /
-                        local->stripe_size) % fctx->stripe_count);
-
-                fill_size = (local->stripe_size -
-                             ((offset + offset_offset) % local->stripe_size));
-                if (fill_size > remaining_size)
-                        fill_size = remaining_size;
-
-                remaining_size -= fill_size;
-
-                tmp_count = iov_subset (vector, count, offset_offset,
-                                        offset_offset + fill_size, NULL);
-                tmp_vec = GF_CALLOC (tmp_count, sizeof (struct iovec),
-                                     gf_stripe_mt_iovec);
-                if (!tmp_vec) {
-                        op_errno = ENOMEM;
-                        goto err;
-                }
-                tmp_count = iov_subset (vector, count, offset_offset,
-                                        offset_offset + fill_size, tmp_vec);
-
-                local->wind_count++;
-                if (remaining_size == 0)
-                        local->unwind = 1;
-
-		/*
-		 * Store off the request index (with respect to the chunk of the
-		 * initial offset) and the size of the request. This is required
-		 * in the callback to calculate an appropriate return value in
-		 * the event of a write failure in one or more requests.
-		 */
-		wlocal->node_index = total_chunks;
-		local->replies[total_chunks].requested_size = fill_size;
-
-		dest_offset = offset + offset_offset;
-		if (fctx->stripe_coalesce)
-			dest_offset = coalesced_offset(dest_offset,
-					local->stripe_size, fctx->stripe_count);
-
-                STACK_WIND (wframe, stripe_writev_cbk, fctx->xl_array[idx],
-                            fctx->xl_array[idx]->fops->writev, fd, tmp_vec,
-                            tmp_count, dest_offset, flags, iobref,
-                            xdata);
-
-                GF_FREE (tmp_vec);
-                offset_offset += fill_size;
-		total_chunks++;
-                if (remaining_size == 0)
-                        break;
-        }
-
-        return 0;
-err:
-	if (wframe)
-		STRIPE_STACK_DESTROY(wframe);
-
-        STRIPE_STACK_UNWIND (writev, frame, -1, op_errno, NULL, NULL, NULL);
-        return 0;
-}
-
-
-int32_t
-stripe_fallocate_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
-                     int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
-                     struct iatt *postbuf, dict_t *xdata)
-{
-        int32_t         callcnt = 0;
-        stripe_local_t *local = NULL;
-	stripe_local_t *mlocal = NULL;
-        call_frame_t   *prev = NULL;
-	call_frame_t   *mframe = NULL;
-
-        if (!this || !frame || !frame->local || !cookie) {
-                gf_log ("stripe", GF_LOG_DEBUG, "possible NULL deref");
-                goto out;
-        }
-
-        prev  = cookie;
-        local = frame->local;
-	mframe = local->orig_frame;
-	mlocal = mframe->local;
-
-        LOCK(&frame->lock);
-        {
-                callcnt = ++mlocal->call_count;
-
-                if (op_ret == 0) {
-                        mlocal->post_buf = *postbuf;
-                        mlocal->pre_buf = *prebuf;
-
-			mlocal->prebuf_blocks  += prebuf->ia_blocks;
-			mlocal->postbuf_blocks += postbuf->ia_blocks;
-
-			correct_file_size(prebuf, mlocal->fctx, prev);
-			correct_file_size(postbuf, mlocal->fctx, prev);
-
-			if (mlocal->prebuf_size < prebuf->ia_size)
-				mlocal->prebuf_size = prebuf->ia_size;
-			if (mlocal->postbuf_size < postbuf->ia_size)
-				mlocal->postbuf_size = postbuf->ia_size;
-                }
-
-		/* return the first failure */
-		if (mlocal->op_ret == 0) {
-			mlocal->op_ret = op_ret;
-			mlocal->op_errno = op_errno;
-		}
-        }
-        UNLOCK (&frame->lock);
-
-        if ((callcnt == mlocal->wind_count) && mlocal->unwind) {
-		mlocal->pre_buf.ia_size = mlocal->prebuf_size;
-		mlocal->pre_buf.ia_blocks = mlocal->prebuf_blocks;
-		mlocal->post_buf.ia_size = mlocal->postbuf_size;
-		mlocal->post_buf.ia_blocks = mlocal->postbuf_blocks;
-
-                STRIPE_STACK_UNWIND (fallocate, mframe, mlocal->op_ret,
-                                     mlocal->op_errno, &mlocal->pre_buf,
-                                     &mlocal->post_buf, NULL);
-        }
-out:
-	STRIPE_STACK_DESTROY(frame);
-        return 0;
-}
-
-int32_t
-stripe_fallocate(call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t mode,
-		 off_t offset, size_t len, dict_t *xdata)
-{
-        stripe_local_t   *local = NULL;
-        stripe_fd_ctx_t  *fctx = NULL;
-        int32_t           op_errno = 1;
-        int32_t           idx = 0;
-        int32_t           offset_offset = 0;
-        int32_t           remaining_size = 0;
-        off_t             fill_size = 0;
-        uint64_t          stripe_size = 0;
-        uint64_t          tmp_fctx = 0;
-	off_t		  dest_offset = 0;
-	call_frame_t	  *fframe = NULL;
-	stripe_local_t	  *flocal = NULL;
-
-        VALIDATE_OR_GOTO (frame, err);
-        VALIDATE_OR_GOTO (this, err);
-        VALIDATE_OR_GOTO (fd, err);
-        VALIDATE_OR_GOTO (fd->inode, err);
-
-        inode_ctx_get (fd->inode, this, &tmp_fctx);
-        if (!tmp_fctx) {
-                op_errno = EINVAL;
-                goto err;
-        }
-        fctx = (stripe_fd_ctx_t *)(long)tmp_fctx;
-        stripe_size = fctx->stripe_size;
-
-        STRIPE_VALIDATE_FCTX (fctx, err);
-
-        remaining_size = len;
-
-        local = mem_get0 (this->local_pool);
-        if (!local) {
-                op_errno = ENOMEM;
-                goto err;
-        }
-        frame->local = local;
-        local->stripe_size = stripe_size;
-	local->fctx = fctx;
-
-        if (!stripe_size) {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "Wrong stripe size for the file");
-                op_errno = EINVAL;
-                goto err;
-        }
-
-        while (1) {
-		fframe = copy_frame(frame);
-		flocal = mem_get0(this->local_pool);
-		if (!flocal) {
-			op_errno = ENOMEM;
-			goto err;
-		}
-		flocal->orig_frame = frame;
-		fframe->local = flocal;
-
-		/* send fallocate request to the associated child node */
-                idx = (((offset + offset_offset) /
-                        local->stripe_size) % fctx->stripe_count);
-
-                fill_size = (local->stripe_size -
-                             ((offset + offset_offset) % local->stripe_size));
-                if (fill_size > remaining_size)
-                        fill_size = remaining_size;
-
-                remaining_size -= fill_size;
-
-                local->wind_count++;
-                if (remaining_size == 0)
-                        local->unwind = 1;
-
-		dest_offset = offset + offset_offset;
-		if (fctx->stripe_coalesce)
-			dest_offset = coalesced_offset(dest_offset,
-					local->stripe_size, fctx->stripe_count);
-
-		/*
-		 * TODO: Create a separate handler for coalesce mode that sends a
-		 * single fallocate per-child (since the ranges are linear).
-		 */
-		STACK_WIND(fframe, stripe_fallocate_cbk, fctx->xl_array[idx],
-			   fctx->xl_array[idx]->fops->fallocate, fd, mode,
-			   dest_offset, fill_size, xdata);
-
-                offset_offset += fill_size;
-                if (remaining_size == 0)
-                        break;
-        }
-
-        return 0;
-err:
-	if (fframe)
-		STRIPE_STACK_DESTROY(fframe);
-
-        STRIPE_STACK_UNWIND (fallocate, frame, -1, op_errno, NULL, NULL, NULL);
-        return 0;
-}
-
-
-int32_t
-stripe_discard_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
-                   int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
-                   struct iatt *postbuf, dict_t *xdata)
-{
-        int32_t         callcnt = 0;
-        stripe_local_t *local = NULL;
-	stripe_local_t *mlocal = NULL;
-        call_frame_t   *prev = NULL;
-	call_frame_t   *mframe = NULL;
-
-        if (!this || !frame || !frame->local || !cookie) {
-                gf_log ("stripe", GF_LOG_DEBUG, "possible NULL deref");
-                goto out;
-        }
-
-        prev  = cookie;
-        local = frame->local;
-	mframe = local->orig_frame;
-	mlocal = mframe->local;
-
-        LOCK(&frame->lock);
-        {
-                callcnt = ++mlocal->call_count;
-
-                if (op_ret == 0) {
-                        mlocal->post_buf = *postbuf;
-                        mlocal->pre_buf = *prebuf;
-
-			mlocal->prebuf_blocks  += prebuf->ia_blocks;
-			mlocal->postbuf_blocks += postbuf->ia_blocks;
-
-			correct_file_size(prebuf, mlocal->fctx, prev);
-			correct_file_size(postbuf, mlocal->fctx, prev);
-
-			if (mlocal->prebuf_size < prebuf->ia_size)
-				mlocal->prebuf_size = prebuf->ia_size;
-			if (mlocal->postbuf_size < postbuf->ia_size)
-				mlocal->postbuf_size = postbuf->ia_size;
-                }
-
-		/* return the first failure */
-		if (mlocal->op_ret == 0) {
-			mlocal->op_ret = op_ret;
-			mlocal->op_errno = op_errno;
-		}
-        }
-        UNLOCK (&frame->lock);
-
-        if ((callcnt == mlocal->wind_count) && mlocal->unwind) {
-		mlocal->pre_buf.ia_size = mlocal->prebuf_size;
-		mlocal->pre_buf.ia_blocks = mlocal->prebuf_blocks;
-		mlocal->post_buf.ia_size = mlocal->postbuf_size;
-		mlocal->post_buf.ia_blocks = mlocal->postbuf_blocks;
-
-                STRIPE_STACK_UNWIND (discard, mframe, mlocal->op_ret,
-                                     mlocal->op_errno, &mlocal->pre_buf,
-                                     &mlocal->post_buf, NULL);
-        }
-out:
-	STRIPE_STACK_DESTROY(frame);
-        return 0;
-}
-
-int32_t
-stripe_discard(call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
-	       size_t len, dict_t *xdata)
-{
-        stripe_local_t   *local = NULL;
-        stripe_fd_ctx_t  *fctx = NULL;
-        int32_t           op_errno = 1;
-        int32_t           idx = 0;
-        int32_t           offset_offset = 0;
-        int32_t           remaining_size = 0;
-        off_t             fill_size = 0;
-        uint64_t          stripe_size = 0;
-        uint64_t          tmp_fctx = 0;
-	off_t		  dest_offset = 0;
-	call_frame_t	  *fframe = NULL;
-	stripe_local_t	  *flocal = NULL;
-
-        VALIDATE_OR_GOTO (frame, err);
-        VALIDATE_OR_GOTO (this, err);
-        VALIDATE_OR_GOTO (fd, err);
-        VALIDATE_OR_GOTO (fd->inode, err);
-
-        inode_ctx_get (fd->inode, this, &tmp_fctx);
-        if (!tmp_fctx) {
-                op_errno = EINVAL;
-                goto err;
-        }
-        fctx = (stripe_fd_ctx_t *)(long)tmp_fctx;
-        stripe_size = fctx->stripe_size;
-
-        STRIPE_VALIDATE_FCTX (fctx, err);
-
-        remaining_size = len;
-
-        local = mem_get0 (this->local_pool);
-        if (!local) {
-                op_errno = ENOMEM;
-                goto err;
-        }
-        frame->local = local;
-        local->stripe_size = stripe_size;
-	local->fctx = fctx;
-
-        if (!stripe_size) {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "Wrong stripe size for the file");
-                op_errno = EINVAL;
-                goto err;
-        }
-
-        while (1) {
-		fframe = copy_frame(frame);
-		flocal = mem_get0(this->local_pool);
-		if (!flocal) {
-			op_errno = ENOMEM;
-			goto err;
-		}
-		flocal->orig_frame = frame;
-		fframe->local = flocal;
-
-		/* send discard request to the associated child node */
-                idx = (((offset + offset_offset) /
-                        local->stripe_size) % fctx->stripe_count);
-
-                fill_size = (local->stripe_size -
-                             ((offset + offset_offset) % local->stripe_size));
-                if (fill_size > remaining_size)
-                        fill_size = remaining_size;
-
-                remaining_size -= fill_size;
-
-                local->wind_count++;
-                if (remaining_size == 0)
-                        local->unwind = 1;
-
-		dest_offset = offset + offset_offset;
-		if (fctx->stripe_coalesce)
-			dest_offset = coalesced_offset(dest_offset,
-					local->stripe_size, fctx->stripe_count);
-
-		/*
-		 * TODO: Create a separate handler for coalesce mode that sends a
-		 * single discard per-child (since the ranges are linear).
-		 */
-		STACK_WIND(fframe, stripe_discard_cbk, fctx->xl_array[idx],
-			   fctx->xl_array[idx]->fops->discard, fd, dest_offset,
-			   fill_size, xdata);
-
-                offset_offset += fill_size;
-                if (remaining_size == 0)
-                        break;
-        }
-
-        return 0;
-err:
-	if (fframe)
-		STRIPE_STACK_DESTROY(fframe);
-
-        STRIPE_STACK_UNWIND (discard, frame, -1, op_errno, NULL, NULL, NULL);
-        return 0;
-}
-
-int32_t
-stripe_zerofill_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
-                   int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
-                   struct iatt *postbuf, dict_t *xdata)
-{
-        int32_t         callcnt    = 0;
-        stripe_local_t *local      = NULL;
-        stripe_local_t *mlocal     = NULL;
-        call_frame_t   *prev       = NULL;
-        call_frame_t   *mframe     = NULL;
-
-        GF_ASSERT (frame);
-
-        if (!this || !frame->local || !cookie) {
-                gf_log ("stripe", GF_LOG_DEBUG, "possible NULL deref");
-                goto out;
-        }
-
-        prev  = cookie;
-        local = frame->local;
-        mframe = local->orig_frame;
-        mlocal = mframe->local;
-
-        LOCK(&frame->lock);
-        {
-                callcnt = ++mlocal->call_count;
-
-                if (op_ret == 0) {
-                        mlocal->post_buf = *postbuf;
-                        mlocal->pre_buf = *prebuf;
-
-                        mlocal->prebuf_blocks  += prebuf->ia_blocks;
-                        mlocal->postbuf_blocks += postbuf->ia_blocks;
-
-                        correct_file_size(prebuf, mlocal->fctx, prev);
-                        correct_file_size(postbuf, mlocal->fctx, prev);
-
-                        if (mlocal->prebuf_size < prebuf->ia_size)
-                                mlocal->prebuf_size = prebuf->ia_size;
-                        if (mlocal->postbuf_size < postbuf->ia_size)
-                                mlocal->postbuf_size = postbuf->ia_size;
-                }
-
-                /* return the first failure */
-                if (mlocal->op_ret == 0) {
-                        mlocal->op_ret = op_ret;
-                        mlocal->op_errno = op_errno;
-                }
-        }
-        UNLOCK (&frame->lock);
-
-        if ((callcnt == mlocal->wind_count) && mlocal->unwind) {
-                mlocal->pre_buf.ia_size = mlocal->prebuf_size;
-                mlocal->pre_buf.ia_blocks = mlocal->prebuf_blocks;
-                mlocal->post_buf.ia_size = mlocal->postbuf_size;
-                mlocal->post_buf.ia_blocks = mlocal->postbuf_blocks;
-
-                STRIPE_STACK_UNWIND (zerofill, mframe, mlocal->op_ret,
-                                     mlocal->op_errno, &mlocal->pre_buf,
-                                     &mlocal->post_buf, NULL);
-        }
-out:
-        STRIPE_STACK_DESTROY(frame);
-        return 0;
-}
-
-int32_t
-stripe_zerofill(call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
-               off_t len, dict_t *xdata)
-{
-        stripe_local_t   *local            = NULL;
-        stripe_fd_ctx_t  *fctx             = NULL;
-        int32_t           op_errno         = 1;
-        int32_t           idx              = 0;
-        int32_t           offset_offset    = 0;
-        int32_t           remaining_size   = 0;
-        off_t             fill_size        = 0;
-        uint64_t          stripe_size      = 0;
-        uint64_t          tmp_fctx         = 0;
-        off_t             dest_offset      = 0;
-        call_frame_t     *fframe           = NULL;
-        stripe_local_t   *flocal           = NULL;
-
-        VALIDATE_OR_GOTO (frame, err);
-        VALIDATE_OR_GOTO (this, err);
-        VALIDATE_OR_GOTO (fd, err);
-        VALIDATE_OR_GOTO (fd->inode, err);
-
-        inode_ctx_get (fd->inode, this, &tmp_fctx);
-        if (!tmp_fctx) {
-                op_errno = EINVAL;
-                goto err;
-        }
-        fctx = (stripe_fd_ctx_t *)(long)tmp_fctx;
-        stripe_size = fctx->stripe_size;
-
-        STRIPE_VALIDATE_FCTX (fctx, err);
-
-        remaining_size = len;
-
-        local = mem_get0 (this->local_pool);
-        if (!local) {
-                op_errno = ENOMEM;
-                goto err;
-        }
-        frame->local = local;
-        local->stripe_size = stripe_size;
-        local->fctx = fctx;
-
-        if (!stripe_size) {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "Wrong stripe size for the file");
-                op_errno = EINVAL;
-                goto err;
-        }
-
-        while (1) {
-                fframe = copy_frame(frame);
-                flocal = mem_get0(this->local_pool);
-                if (!flocal) {
-                        op_errno = ENOMEM;
-                        goto err;
-                }
-                flocal->orig_frame = frame;
-                fframe->local = flocal;
-
-                idx = (((offset + offset_offset) /
-                        local->stripe_size) % fctx->stripe_count);
-
-                fill_size = (local->stripe_size -
-                             ((offset + offset_offset) % local->stripe_size));
-                if (fill_size > remaining_size)
-                        fill_size = remaining_size;
-
-                remaining_size -= fill_size;
-
-                local->wind_count++;
-                if (remaining_size == 0)
-                        local->unwind = 1;
-
-                dest_offset = offset + offset_offset;
-                if (fctx->stripe_coalesce)
-                        dest_offset = coalesced_offset(dest_offset,
-                                                       local->stripe_size,
-                                                       fctx->stripe_count);
-
-                STACK_WIND(fframe, stripe_zerofill_cbk, fctx->xl_array[idx],
-                           fctx->xl_array[idx]->fops->zerofill, fd,
-                           dest_offset, fill_size, xdata);
-                offset_offset += fill_size;
-                if (remaining_size == 0)
-                        break;
-        }
-
-        return 0;
-err:
-        if (fframe)
-                STRIPE_STACK_DESTROY(fframe);
-
-        STRIPE_STACK_UNWIND (zerofill, frame, -1, op_errno, NULL, NULL, NULL);
-        return 0;
-}
-
-int32_t
-stripe_release (xlator_t *this, fd_t *fd)
-{
-	return 0;
-}
-
-int
-stripe_forget (xlator_t *this, inode_t *inode)
-{
-        uint64_t          tmp_fctx = 0;
-        stripe_fd_ctx_t  *fctx = NULL;
-
-        VALIDATE_OR_GOTO (this, err);
-        VALIDATE_OR_GOTO (inode, err);
-
-        (void) inode_ctx_del (inode, this, &tmp_fctx);
-        if (!tmp_fctx) {
-                goto err;
-        }
-
-        fctx = (stripe_fd_ctx_t *)(long)tmp_fctx;
-
-        if (!fctx->static_array)
-                GF_FREE (fctx->xl_array);
-
-        GF_FREE (fctx);
-err:
-        return 0;
-}
-
-int32_t
-notify (xlator_t *this, int32_t event, void *data, ...)
-{
-        stripe_private_t *priv = NULL;
-        int               down_client = 0;
-        int               i = 0;
-        gf_boolean_t      heard_from_all_children = _gf_false;
-
-        if (!this)
-                return 0;
-
-        priv = this->private;
-        if (!priv)
-                return 0;
-
-        switch (event)
-        {
-        case GF_EVENT_CHILD_UP:
-        {
-                /* get an index number to set */
-                for (i = 0; i < priv->child_count; i++) {
-                        if (data == priv->xl_array[i])
-                                break;
-                }
-
-                if (priv->child_count == i) {
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "got GF_EVENT_CHILD_UP bad subvolume %s",
-                                data? ((xlator_t *)data)->name: NULL);
-                        break;
-                }
-
-                LOCK (&priv->lock);
-                {
-                        if (data == FIRST_CHILD (this))
-                                priv->first_child_down = 0;
-                        priv->last_event[i] = event;
-                }
-                UNLOCK (&priv->lock);
-        }
-        break;
-        case GF_EVENT_CHILD_CONNECTING:
-        {
-                // 'CONNECTING' doesn't ensure its CHILD_UP, so do nothing
-                goto out;
-        }
-        case GF_EVENT_CHILD_DOWN:
-        {
-                /* get an index number to set */
-                for (i = 0; i < priv->child_count; i++) {
-                        if (data == priv->xl_array[i])
-                                break;
-                }
-
-                if (priv->child_count == i) {
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "got GF_EVENT_CHILD_DOWN bad subvolume %s",
-                                data? ((xlator_t *)data)->name: NULL);
-                        break;
-                }
-
-                LOCK (&priv->lock);
-                {
-                        if (data == FIRST_CHILD (this))
-                                priv->first_child_down = 1;
-                        priv->last_event[i] = event;
-                }
-                UNLOCK (&priv->lock);
-        }
-        break;
-
-        default:
-        {
-                /* */
-                default_notify (this, event, data);
-                goto out;
-        }
-        break;
-        }
-
-        // Consider child as down if it's last_event is not CHILD_UP
-        for (i = 0, down_client = 0; i < priv->child_count; i++)
-                if (priv->last_event[i] != GF_EVENT_CHILD_UP)
-                        down_client++;
-
-        LOCK (&priv->lock);
-        {
-                priv->nodes_down = down_client;
-        }
-        UNLOCK (&priv->lock);
-
-        heard_from_all_children = _gf_true;
-        for (i = 0; i < priv->child_count; i++)
-                if (!priv->last_event[i])
-                        heard_from_all_children = _gf_false;
-
-        if (heard_from_all_children)
-                default_notify (this, event, data);
-out:
-        return 0;
-}
-
-int
-stripe_setxattr_cbk (call_frame_t *frame, void *cookie,
-                     xlator_t *this, int op_ret, int op_errno, dict_t *xdata)
-{
-        int ret = -1;
-        int call_cnt = 0;
-        stripe_local_t *local = NULL;
-
-        if (!frame || !frame->local || !this) {
-                gf_log ("", GF_LOG_ERROR, "Possible NULL deref");
-                return ret;
-        }
-
-        local = frame->local;
-
-        LOCK (&frame->lock);
-        {
-                call_cnt = --local->wind_count;
-
-                /**
-                 * We overwrite ->op_* values here for subsequent faliure
-                 * conditions, hence we propagate the last errno down the
-                 * stack.
-                 */
-                if (op_ret < 0) {
-                        local->op_ret = op_ret;
-                        local->op_errno = op_errno;
-                        goto unlock;
-                }
-        }
-
- unlock:
-        UNLOCK (&frame->lock);
-
-        if (!call_cnt) {
-                STRIPE_STACK_UNWIND (setxattr, frame, local->op_ret,
-                                     local->op_errno, xdata);
-        }
-
-        return 0;
-}
-
-#ifdef HAVE_BD_XLATOR
-int
-stripe_is_bd (dict_t *this, char *key, data_t *value, void *data)
-{
-        gf_boolean_t *is_bd = data;
-
-        if (data == NULL)
-                return 0;
-
-        if (XATTR_IS_BD (key))
-            *is_bd = _gf_true;
-
-        return 0;
-}
-
-static inline gf_boolean_t
-stripe_setxattr_is_bd (dict_t *dict)
-{
-        gf_boolean_t is_bd = _gf_false;
-
-        if (dict == NULL)
-                goto out;
-
-        dict_foreach (dict, stripe_is_bd, &is_bd);
-out:
-        return is_bd;
-}
-#else
-#define stripe_setxattr_is_bd(dict) _gf_false
-#endif
-
-int
-stripe_setxattr (call_frame_t *frame, xlator_t *this,
-                 loc_t *loc, dict_t *dict, int flags, dict_t *xdata)
-{
-        int32_t           op_errno = EINVAL;
-        xlator_list_t    *trav     = NULL;
-        stripe_private_t *priv     = NULL;
-        stripe_local_t   *local    = NULL;
-        int               i        = 0;
-        gf_boolean_t      is_bd    = _gf_false;
-
-        VALIDATE_OR_GOTO (frame, err);
-        VALIDATE_OR_GOTO (this, err);
-        VALIDATE_OR_GOTO (loc, err);
-        VALIDATE_OR_GOTO (loc->inode, err);
-
-        GF_IF_INTERNAL_XATTR_GOTO ("trusted.*stripe*", dict,
-                                   op_errno, err);
-
-        priv = this->private;
-        trav = this->children;
-
-        local = mem_get0 (this->local_pool);
-        if (!local) {
-                op_errno = ENOMEM;
-                goto err;
-        }
-
-        frame->local = local;
-        local->wind_count = priv->child_count;
-        local->op_ret = local->op_errno = 0;
-
-        is_bd = stripe_setxattr_is_bd (dict);
-
-        /**
-         * Set xattrs for directories on all subvolumes. Additionally
-         * this power is only given to a special client. Bd xlator
-         * also needs xattrs for regular files (ie LVs)
-         */
-        if (((frame->root->pid == GF_CLIENT_PID_GSYNCD) &&
-             IA_ISDIR (loc->inode->ia_type)) || is_bd) {
-                for (i = 0; i < priv->child_count; i++, trav = trav->next) {
-                        STACK_WIND (frame, stripe_setxattr_cbk,
-                                    trav->xlator, trav->xlator->fops->setxattr,
-                                    loc, dict, flags, xdata);
-                }
-        } else {
-                local->wind_count = 1;
-                STACK_WIND (frame, stripe_setxattr_cbk,
-                            FIRST_CHILD(this),
-                            FIRST_CHILD(this)->fops->setxattr,
-                            loc, dict, flags, xdata);
-        }
-
-        return 0;
-err:
-        STRIPE_STACK_UNWIND (setxattr, frame, -1,  op_errno, NULL);
-        return 0;
-}
-
-
-int
-stripe_fsetxattr_cbk (call_frame_t *frame, void *cookie,
-                      xlator_t *this, int op_ret, int op_errno, dict_t *xdata)
-{
-        STRIPE_STACK_UNWIND (fsetxattr, frame, op_ret, op_errno, xdata);
-        return 0;
-}
-
-
-int
-stripe_is_special_key (dict_t *this,
-                       char *key,
-                       data_t *value,
-                       void *data)
-{
-        gf_boolean_t *is_special = NULL;
-
-        if (data == NULL) {
-                goto out;
-        }
-
-        is_special = data;
-
-        if (XATTR_IS_LOCKINFO (key) || XATTR_IS_BD (key))
-                *is_special = _gf_true;
-
-out:
-        return 0;
-}
-
-int32_t
-stripe_fsetxattr_everyone_cbk (call_frame_t *frame, void *cookie,
-                               xlator_t *this, int32_t op_ret, int32_t op_errno,
-                               dict_t *xdata)
-{
-        int             call_count = 0;
-        stripe_local_t *local      = NULL;
-
-        local = frame->local;
-
-        LOCK (&frame->lock);
-        {
-                call_count = --local->wind_count;
-
-                if (op_ret < 0) {
-                        local->op_ret = op_ret;
-                        local->op_errno = op_errno;
-                }
-        }
-        UNLOCK (&frame->lock);
-
-        if (call_count == 0) {
-                STRIPE_STACK_UNWIND (fsetxattr, frame, local->op_ret,
-                                     local->op_errno, NULL);
-        }
-        return 0;
-}
-
-int
-stripe_fsetxattr_to_everyone (call_frame_t *frame, xlator_t *this, fd_t *fd,
-                              dict_t *dict, int flags, dict_t *xdata)
-{
-        xlator_list_t    *trav  = NULL;
-        stripe_private_t *priv  = NULL;
-        int               ret   = -1;
-        stripe_local_t   *local = NULL;
-
-        priv = this->private;
-
-        local = mem_get0 (this->local_pool);
-        if (local == NULL) {
-                goto out;
-        }
-
-        frame->local = local;
-
-        local->wind_count = priv->child_count;
-
-        trav = this->children;
-
-        while (trav) {
-                STACK_WIND (frame, stripe_fsetxattr_everyone_cbk,
-                            trav->xlator, trav->xlator->fops->fsetxattr,
-                            fd, dict, flags, xdata);
-                trav = trav->next;
-        }
-
-        ret = 0;
-out:
-        return ret;
-}
-
-static inline gf_boolean_t
-stripe_fsetxattr_is_special (dict_t *dict)
-{
-        gf_boolean_t is_spl = _gf_false;
-
-        if (dict == NULL) {
-                goto out;
-        }
-
-        dict_foreach (dict, stripe_is_special_key, &is_spl);
-
-out:
-        return is_spl;
-}
-
-int
-stripe_fsetxattr (call_frame_t *frame, xlator_t *this, fd_t *fd,
-                  dict_t *dict, int flags, dict_t *xdata)
-{
-        int32_t      op_ret = -1, ret = -1, op_errno = EINVAL;
-        gf_boolean_t is_spl = _gf_false;
-
-        VALIDATE_OR_GOTO (frame, err);
-        VALIDATE_OR_GOTO (this, err);
-        VALIDATE_OR_GOTO (fd, err);
-
-        GF_IF_INTERNAL_XATTR_GOTO ("trusted.*stripe*", dict,
-                                   op_errno, err);
-
-        is_spl = stripe_fsetxattr_is_special (dict);
-        if (is_spl) {
-                ret = stripe_fsetxattr_to_everyone (frame, this, fd, dict,
-                                                    flags, xdata);
-                if (ret < 0) {
-                        op_errno = ENOMEM;
-                        goto err;
-                }
-
-                goto out;
-        }
-
-        STACK_WIND (frame, stripe_fsetxattr_cbk,
-                    FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->fsetxattr,
-                    fd, dict, flags, xdata);
-out:
-        return 0;
-err:
-        STRIPE_STACK_UNWIND (fsetxattr, frame, op_ret, op_errno, NULL);
-        return 0;
-}
-
-int
-stripe_removexattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                        int32_t op_ret, int32_t op_errno, dict_t *xdata)
-{
-        STRIPE_STACK_UNWIND (removexattr, frame, op_ret, op_errno, xdata);
-        return 0;
-}
-
-int
-stripe_removexattr (call_frame_t *frame, xlator_t *this,
-                    loc_t *loc, const char *name, dict_t *xdata)
-{
-        int32_t         op_errno = EINVAL;
-
-        VALIDATE_OR_GOTO (this, err);
-
-        GF_IF_NATIVE_XATTR_GOTO ("trusted.*stripe*",
-                                 name, op_errno, err);
-
-        VALIDATE_OR_GOTO (frame, err);
-        VALIDATE_OR_GOTO (loc, err);
-
-        STACK_WIND (frame, stripe_removexattr_cbk,
-                    FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->removexattr,
-                    loc, name, xdata);
-        return 0;
-err:
-        STRIPE_STACK_UNWIND (removexattr, frame, -1,  op_errno, NULL);
-        return 0;
-}
-
-
-int
-stripe_fremovexattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                         int32_t op_ret, int32_t op_errno, dict_t *xdata)
-{
-        STRIPE_STACK_UNWIND (fremovexattr, frame, op_ret, op_errno, xdata);
-        return 0;
-}
-
-int
-stripe_fremovexattr (call_frame_t *frame, xlator_t *this,
-                     fd_t *fd, const char *name, dict_t *xdata)
-{
-        int32_t         op_ret   = -1;
-        int32_t         op_errno = EINVAL;
-
-        VALIDATE_OR_GOTO (frame, err);
-        VALIDATE_OR_GOTO (this, err);
-        VALIDATE_OR_GOTO (fd, err);
-
-        GF_IF_NATIVE_XATTR_GOTO ("trusted.*stripe*",
-                                 name, op_errno, err);
-
-        STACK_WIND (frame, stripe_fremovexattr_cbk,
-                    FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->fremovexattr,
-                    fd, name, xdata);
-        return 0;
- err:
-        STRIPE_STACK_UNWIND (fremovexattr, frame, op_ret, op_errno, xdata);
-        return 0;
-}
-
-int32_t
-stripe_readdirp_lookup_cbk (call_frame_t *frame, void *cookie,
-                            xlator_t *this, int op_ret, int op_errno,
-                            inode_t *inode, struct iatt *stbuf,
-                            dict_t *xattr, struct iatt *parent)
-{
-        stripe_local_t          *local          = NULL;
-        call_frame_t            *main_frame     = NULL;
-        stripe_local_t          *main_local     = NULL;
-        gf_dirent_t             *entry          = NULL;
-        call_frame_t            *prev           = NULL;
-        int                      done           = 0;
-
-        local = frame->local;
-        prev = cookie;
-
-        entry = local->dirent;
-
-        main_frame = local->orig_frame;
-        main_local = main_frame->local;
-        LOCK (&frame->lock);
-        {
-
-                local->call_count--;
-                if (!local->call_count)
-                        done = 1;
-                if (op_ret == -1) {
-                        local->op_errno = op_errno;
-                        local->op_ret = op_ret;
-                        goto unlock;
-                }
-
-		if (stripe_ctx_handle(this, prev, local, xattr))
-			gf_log(this->name, GF_LOG_ERROR,
-				"Error getting fctx info from dict.");
-
-		correct_file_size(stbuf, local->fctx, prev);
-
-                stripe_iatt_merge (stbuf, &entry->d_stat);
-                local->stbuf_blocks += stbuf->ia_blocks;
-        }
-unlock:
-        UNLOCK(&frame->lock);
-
-        if (done) {
-                inode_ctx_put (entry->inode, this,
-                               (uint64_t) (long)local->fctx);
-
-                done = 0;
-                LOCK (&main_frame->lock);
-                {
-                        main_local->wind_count--;
-                        if (!main_local->wind_count)
-                                done = 1;
-                        if (local->op_ret == -1) {
-                                main_local->op_errno = local->op_errno;
-                                main_local->op_ret = local->op_ret;
-                        }
-                        entry->d_stat.ia_blocks = local->stbuf_blocks;
-                }
-                UNLOCK (&main_frame->lock);
-                if (done) {
-                        main_frame->local = NULL;
-                        STRIPE_STACK_UNWIND (readdir, main_frame,
-                                             main_local->op_ret,
-                                             main_local->op_errno,
-                                             &main_local->entries, NULL);
-                        gf_dirent_free (&main_local->entries);
-                        stripe_local_wipe (main_local);
-                        mem_put (main_local);
-                }
-                frame->local = NULL;
-                stripe_local_wipe (local);
-                mem_put (local);
-                STRIPE_STACK_DESTROY (frame);
-        }
-
-        return 0;
-}
-
-int32_t
-stripe_readdirp_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                     int32_t op_ret, int32_t op_errno,
-                     gf_dirent_t *orig_entries, dict_t *xdata)
-{
-        stripe_local_t *local = NULL;
-        call_frame_t   *prev = NULL;
-        gf_dirent_t    *local_entry = NULL;
-        gf_dirent_t    *tmp_entry = NULL;
-        xlator_list_t  *trav = NULL;
-        loc_t          loc = {0, };
-        int32_t        count = 0;
-        stripe_private_t *priv = NULL;
-        int32_t        subvols = 0;
-        dict_t         *xattrs = NULL;
-        call_frame_t   *local_frame = NULL;
-        stripe_local_t *local_ent = NULL;
-
-        if (!this || !frame || !frame->local || !cookie) {
-                gf_log ("stripe", GF_LOG_DEBUG, "possible NULL deref");
-                goto out;
-        }
-        prev  = cookie;
-        local = frame->local;
-        trav = this->children;
-        priv = this->private;
-
-        subvols = priv->child_count;
-
-        LOCK (&frame->lock);
-        {
-                local->op_errno = op_errno;
-                local->op_ret = op_ret;
-
-                if (op_ret != -1) {
-                        list_splice_init (&orig_entries->list,
-                                          &local->entries.list);
-                        local->wind_count = op_ret;
-                }
-
-        }
-        UNLOCK (&frame->lock);
-
-        if (op_ret == -1) {
-                gf_log (this->name, GF_LOG_WARNING, "%s returned error %s",
-                        prev->this->name, strerror (op_errno));
-                goto out;
-        }
-
-        xattrs = dict_new ();
-        if (xattrs)
-                (void) stripe_xattr_request_build (this, xattrs, 0, 0, 0, 0);
-        count = op_ret;
-        list_for_each_entry_safe (local_entry, tmp_entry,
-                                  (&local->entries.list), list) {
-
-                if (!local_entry)
-                        break;
-                if (!IA_ISREG (local_entry->d_stat.ia_type) || !local_entry->inode) {
-                        LOCK (&frame->lock);
-                        {
-                                local->wind_count--;
-                                count = local->wind_count;
-                        }
-                        UNLOCK (&frame->lock);
-                        continue;
-                }
-
-                local_frame = copy_frame (frame);
-
-                if (!local_frame) {
-                        op_errno = ENOMEM;
-                        op_ret = -1;
-                        goto out;
-                }
-
-                local_ent = mem_get0 (this->local_pool);
-                if (!local_ent) {
-                        op_errno = ENOMEM;
-                        op_ret = -1;
-                        goto out;
-                }
-
-                loc.inode = inode_ref (local_entry->inode);
-
-                gf_uuid_copy (loc.gfid, local_entry->d_stat.ia_gfid);
-
-                local_ent->orig_frame = frame;
-
-                local_ent->call_count = subvols;
-
-                local_ent->dirent = local_entry;
-
-                local_frame->local = local_ent;
-
-                trav = this->children;
-                while (trav) {
-                        STACK_WIND (local_frame, stripe_readdirp_lookup_cbk,
-                                    trav->xlator, trav->xlator->fops->lookup,
-                                    &loc, xattrs);
-                        trav = trav->next;
-                }
-                loc_wipe (&loc);
-        }
-out:
-        if (!count) {
-                /* all entries are directories */
-                frame->local = NULL;
-                STRIPE_STACK_UNWIND (readdir, frame,
-                                     local ? local->op_ret : -1,
-                                     local ? local->op_errno : EINVAL,
-                                     local ? &local->entries : NULL,
-                                     NULL);
-                gf_dirent_free (&local->entries);
-                stripe_local_wipe (local);
-                mem_put (local);
-        }
-        if (xattrs)
-                dict_unref (xattrs);
-        return 0;
-
-}
-int32_t
-stripe_readdirp (call_frame_t *frame, xlator_t *this,
-                 fd_t *fd, size_t size, off_t off, dict_t *xdata)
-{
-        stripe_local_t  *local  = NULL;
-        stripe_private_t *priv = NULL;
-        xlator_list_t   *trav = NULL;
-        int             op_errno = -1;
-
-        VALIDATE_OR_GOTO (frame, err);
-        VALIDATE_OR_GOTO (this, err);
-        VALIDATE_OR_GOTO (fd, err);
-
-        priv = this->private;
-        trav = this->children;
-
-        if (priv->first_child_down) {
-                op_errno = ENOTCONN;
-                goto err;
-        }
-
-        /* Initialization */
-        local = mem_get0 (this->local_pool);
-        if (!local) {
-                op_errno = ENOMEM;
-                goto err;
-        }
-
-        frame->local = local;
-
-        local->fd = fd_ref (fd);
-
-        local->wind_count = 0;
-
-        local->count = 0;
-        local->op_ret = -1;
-        INIT_LIST_HEAD(&local->entries);
-
-        if (!trav)
-                goto err;
-
-        STACK_WIND (frame, stripe_readdirp_cbk, trav->xlator,
-                    trav->xlator->fops->readdirp, fd, size, off, xdata);
-        return 0;
-err:
-        op_errno = (op_errno == -1) ? errno : op_errno;
-        STRIPE_STACK_UNWIND (readdir, frame, -1, op_errno, NULL, NULL);
-
-        return 0;
-
-}
-
-int32_t
-mem_acct_init (xlator_t *this)
-{
-        int     ret = -1;
-
-        if (!this)
-                goto out;
-
-        ret = xlator_mem_acct_init (this, gf_stripe_mt_end + 1);
-
-        if (ret != 0) {
-                gf_log (this->name, GF_LOG_ERROR, "Memory accounting init"
-                        "failed");
-                goto out;
-        }
-
-out:
-        return ret;
-}
-
-static int
-clear_pattern_list (stripe_private_t *priv)
-{
-        struct stripe_options *prev = NULL;
-        struct stripe_options *trav = NULL;
-        int                    ret = -1;
-
-        GF_VALIDATE_OR_GOTO ("stripe", priv, out);
-
-        trav = priv->pattern;
-        priv->pattern = NULL;
-        while (trav) {
-                prev = trav;
-                trav = trav->next;
-                GF_FREE (prev);
-        }
-
-        ret = 0;
- out:
-        return ret;
-
-
-}
-
-
-int
-reconfigure (xlator_t *this, dict_t *options)
-{
-
-        stripe_private_t *priv = NULL;
-        data_t           *data = NULL;
-        int               ret = -1;
-        volume_option_t  *opt = NULL;
-
-        GF_ASSERT (this);
-        GF_ASSERT (this->private);
-
-        priv = this->private;
-
-
-        ret = 0;
-        LOCK (&priv->lock);
-        {
-                ret = clear_pattern_list (priv);
-                if (ret)
-                         goto unlock;
-
-                data = dict_get (options, "block-size");
-                if (data) {
-                        ret = set_stripe_block_size (this, priv, data->data);
-                        if (ret)
-                                goto unlock;
-                } else {
-                        opt = xlator_volume_option_get (this, "block-size");
-                        if (!opt) {
-                                gf_log (this->name, GF_LOG_WARNING,
-                                        "option 'block-size' not found");
-                                ret = -1;
-                                goto unlock;
-                        }
-
-                        if (gf_string2bytesize_uint64 (opt->default_value, &priv->block_size)){
-                                gf_log (this->name, GF_LOG_ERROR,
-                                        "Unable to set default block-size ");
-                                ret = -1;
-                                goto unlock;
-                        }
-                }
-
-		GF_OPTION_RECONF("coalesce", priv->coalesce, options, bool,
-				unlock);
-        }
- unlock:
-        UNLOCK (&priv->lock);
-        if (ret)
-                goto out;
-
-        ret = 0;
- out:
-        return ret;
-
-}
-
-/**
- * init - This function is called when xlator-graph gets initialized.
- *     The option given in volfiles are parsed here.
- * @this -
- */
-int32_t
-init (xlator_t *this)
-{
-        stripe_private_t *priv = NULL;
-        volume_option_t  *opt = NULL;
-        xlator_list_t    *trav = NULL;
-        data_t           *data = NULL;
-        int32_t           count = 0;
-        int               ret = -1;
-
-        if (!this)
-                goto out;
-
-        trav = this->children;
-        while (trav) {
-                count++;
-                trav = trav->next;
-        }
-
-        if (!count) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "stripe configured without \"subvolumes\" option. "
-                        "exiting");
-                goto out;
-        }
-
-        if (!this->parents) {
-                gf_log (this->name, GF_LOG_WARNING,
-                        "dangling volume. check volfile ");
-        }
-
-        if (count == 1) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "stripe configured with only one \"subvolumes\" option."
-                        " please check the volume. exiting");
-                goto out;
-        }
-
-        priv = GF_CALLOC (1, sizeof (stripe_private_t),
-                          gf_stripe_mt_stripe_private_t);
-
-        if (!priv)
-                goto out;
-        priv->xl_array = GF_CALLOC (count, sizeof (xlator_t *),
-                                    gf_stripe_mt_xlator_t);
-        if (!priv->xl_array)
-                goto out;
-
-        priv->last_event = GF_CALLOC (count, sizeof (int),
-                                      gf_stripe_mt_int32_t);
-        if (!priv->last_event)
-                goto out;
-
-        priv->child_count = count;
-        LOCK_INIT (&priv->lock);
-
-        trav = this->children;
-        count = 0;
-        while (trav) {
-                priv->xl_array[count++] = trav->xlator;
-                trav = trav->next;
-        }
-
-        if (count > 256) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "maximum number of stripe subvolumes supported "
-                        "is 256");
-                goto out;
-        }
-
-        ret = 0;
-        LOCK (&priv->lock);
-        {
-                opt = xlator_volume_option_get (this, "block-size");
-                if (!opt) {
-                        gf_log (this->name, GF_LOG_WARNING,
-                                "option 'block-size' not found");
-                        ret = -1;
-                        goto unlock;
-                }
-                if (gf_string2bytesize_uint64 (opt->default_value, &priv->block_size)){
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "Unable to set default block-size ");
-                        ret = -1;
-                        goto unlock;
-                }
-                /* option stripe-pattern *avi:1GB,*pdf:16K */
-                data = dict_get (this->options, "block-size");
-                if (data) {
-                        ret = set_stripe_block_size (this, priv, data->data);
-                        if (ret)
-                                goto unlock;
-                }
-        }
- unlock:
-        UNLOCK (&priv->lock);
-        if (ret)
-                goto out;
-
-        GF_OPTION_INIT ("use-xattr", priv->xattr_supported, bool, out);
-        /* notify related */
-        priv->nodes_down = priv->child_count;
-
-	GF_OPTION_INIT("coalesce", priv->coalesce, bool, out);
-
-        this->local_pool = mem_pool_new (stripe_local_t, 128);
-        if (!this->local_pool) {
-                ret = -1;
-                gf_log (this->name, GF_LOG_ERROR,
-                        "failed to create local_t's memory pool");
-                goto out;
-        }
-
-        this->private = priv;
-
-        ret = 0;
-out:
-        if (ret) {
-                if (priv) {
-                        GF_FREE (priv->xl_array);
-                        GF_FREE (priv);
-                }
-        }
-        return ret;
-}
-
-/**
- * fini -   Free all the private variables
- * @this -
- */
-void
-fini (xlator_t *this)
-{
-        stripe_private_t      *priv = NULL;
-        struct stripe_options *prev = NULL;
-        struct stripe_options *trav = NULL;
-
-        if (!this)
-                goto out;
-
-        priv = this->private;
-        if (priv) {
-                this->private = NULL;
-                GF_FREE (priv->xl_array);
-
-                trav = priv->pattern;
-                while (trav) {
-                        prev = trav;
-                        trav = trav->next;
-                        GF_FREE (prev);
-                }
-                GF_FREE (priv->last_event);
-                LOCK_DESTROY (&priv->lock);
-                GF_FREE (priv);
-        }
-
-out:
-        return;
-}
-
-int32_t
-stripe_getxattr_unwind (call_frame_t *frame,
-                        int op_ret, int op_errno, dict_t *dict, dict_t *xdata)
-
-{
-        STRIPE_STACK_UNWIND (getxattr, frame, op_ret, op_errno, dict, xdata);
-        return 0;
-}
-
-int
-stripe_internal_getxattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                              int op_ret, int op_errno, dict_t *xattr,
-                              dict_t *xdata)
-{
-
-        char        size_key[256]  = {0,};
-        char        index_key[256] = {0,};
-        char        count_key[256] = {0,};
-        char        coalesce_key[256] = {0,};
-
-        VALIDATE_OR_GOTO (frame, out);
-        VALIDATE_OR_GOTO (frame->local, out);
-
-        if (!xattr || (op_ret == -1))
-            goto out;
-
-        sprintf (size_key, "trusted.%s.stripe-size", this->name);
-        sprintf (count_key, "trusted.%s.stripe-count", this->name);
-        sprintf (index_key, "trusted.%s.stripe-index", this->name);
-	sprintf (coalesce_key, "trusted.%s.stripe-coalesce", this->name);
-
-        dict_del (xattr, size_key);
-        dict_del (xattr, count_key);
-        dict_del (xattr, index_key);
-        dict_del (xattr, coalesce_key);
-
-out:
-        STRIPE_STACK_UNWIND (getxattr, frame, op_ret, op_errno, xattr, xdata);
-
-        return 0;
-
-}
-
-int
-stripe_getxattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                     int op_ret, int op_errno, dict_t *xattr, dict_t *xdata)
-{
-        int                     call_cnt = 0;
-        stripe_local_t         *local = NULL;
-
-        VALIDATE_OR_GOTO (frame, out);
-        VALIDATE_OR_GOTO (frame->local, out);
-
-        local = frame->local;
-
-        LOCK (&frame->lock);
-        {
-                call_cnt = --local->wind_count;
-        }
-        UNLOCK (&frame->lock);
-
-        if (!xattr || (op_ret < 0))
-                goto out;
-
-        local->op_ret = 0;
-
-        if (!local->xattr) {
-                local->xattr = dict_ref (xattr);
-        } else {
-                stripe_aggregate_xattr (local->xattr, xattr);
-        }
-
-out:
-        if (!call_cnt) {
-                STRIPE_STACK_UNWIND (getxattr, frame, local->op_ret, op_errno,
-                                     local->xattr, xdata);
-        }
-
-        return 0;
-}
-
-int32_t
-stripe_vgetxattr_cbk (call_frame_t *frame, void *cookie,
-                      xlator_t *this, int32_t op_ret, int32_t op_errno,
-                      dict_t *dict, dict_t *xdata)
-{
-        stripe_local_t      *local         = NULL;
-        int32_t              callcnt       = 0;
-        int32_t              ret           = -1;
-        long                 cky           = 0;
-        void                *xattr_val     = NULL;
-        void                *xattr_serz    = NULL;
-        stripe_xattr_sort_t *xattr         = NULL;
-        dict_t              *stripe_xattr  = NULL;
-
-        if (!frame || !frame->local || !this) {
-                gf_log ("", GF_LOG_ERROR, "Possible NULL deref");
-                return ret;
-        }
-
-        local = frame->local;
-        cky = (long) cookie;
-
-        if (local->xsel[0] == '\0') {
-                gf_log (this->name, GF_LOG_ERROR, "Empty xattr in cbk");
-                return ret;
-        }
-
-        LOCK (&frame->lock);
-        {
-                callcnt = --local->wind_count;
-
-                if (!dict || (op_ret < 0))
-                        goto out;
-
-                if (!local->xattr_list)
-                        local->xattr_list = (stripe_xattr_sort_t *)
-                                GF_CALLOC (local->nallocs,
-                                           sizeof (stripe_xattr_sort_t),
-                                           gf_stripe_mt_xattr_sort_t);
-
-                if (local->xattr_list) {
-                        xattr = local->xattr_list + (int32_t) cky;
-
-                        ret = dict_get_ptr_and_len (dict, local->xsel,
-                                                    &xattr_val,
-                                                    &xattr->xattr_len);
-                        if (xattr->xattr_len == 0)
-                                goto out;
-
-                        xattr->pos = cky;
-                        xattr->xattr_value = gf_memdup (xattr_val,
-                                                        xattr->xattr_len);
-
-                        if (xattr->xattr_value != NULL)
-                                local->xattr_total_len += xattr->xattr_len + 1;
-                }
-        }
- out:
-        UNLOCK (&frame->lock);
-
-        if (!callcnt) {
-                if (!local->xattr_total_len)
-                        goto unwind;
-
-                stripe_xattr = dict_new ();
-                if (!stripe_xattr)
-                        goto unwind;
-
-                /* select filler based on ->xsel */
-                if (XATTR_IS_PATHINFO (local->xsel))
-                        ret = stripe_fill_pathinfo_xattr (this, local,
-                                                          (char **)&xattr_serz);
-                else if (XATTR_IS_LOCKINFO (local->xsel)) {
-                        ret = stripe_fill_lockinfo_xattr (this, local,
-                                                          &xattr_serz);
-                } else {
-                        gf_log (this->name, GF_LOG_WARNING,
-                                "Unknown xattr in xattr request");
-                        goto unwind;
-                }
-
-                if (!ret) {
-                        ret = dict_set_dynptr (stripe_xattr, local->xsel,
-                                               xattr_serz,
-                                               local->xattr_total_len);
-                        if (ret)
-                                gf_log (this->name, GF_LOG_ERROR,
-                                        "Can't set %s key in dict",
-                                        local->xsel);
-                }
-
-        unwind:
-                STRIPE_STACK_UNWIND (getxattr, frame, op_ret, op_errno,
-                                     stripe_xattr, NULL);
-
-                ret = stripe_free_xattr_str (local);
-
-                GF_FREE (local->xattr_list);
-
-                if (stripe_xattr)
-                        dict_unref (stripe_xattr);
-        }
-
-        return ret;
-}
-
-int
-stripe_marker_populate_args (call_frame_t *frame, int type, int *gauge,
-                            xlator_t **subvols)
-{
-        xlator_t         *this  = frame->this;
-        stripe_private_t *priv  = this->private;
-        stripe_local_t   *local = frame->local;
-        int              count  = 0;
-
-        count = priv->child_count;
-        if (MARKER_XTIME_TYPE == type) {
-                if (!IA_FILE_OR_DIR (local->loc.inode->ia_type))
-                        count = 1;
-        }
-        memcpy (subvols, priv->xl_array, sizeof (*subvols) * count);
-
-        return count;
-}
-
-int32_t
-stripe_getxattr (call_frame_t *frame, xlator_t *this,
-                 loc_t *loc, const char *name, dict_t *xdata)
-{
-        stripe_local_t    *local    = NULL;
-        xlator_list_t     *trav     = NULL;
-        stripe_private_t  *priv     = NULL;
-        int32_t            op_errno = EINVAL;
-        int                i        = 0;
-        int                ret      = 0;
-
-        VALIDATE_OR_GOTO (frame, err);
-        VALIDATE_OR_GOTO (this, err);
-        VALIDATE_OR_GOTO (loc, err);
-        VALIDATE_OR_GOTO (loc->path, err);
-        VALIDATE_OR_GOTO (loc->inode, err);
-
-        priv = this->private;
-        trav = this->children;
-
-        /* Initialization */
-        local = mem_get0 (this->local_pool);
-        if (!local) {
-                op_errno = ENOMEM;
-                goto err;
-        }
-        local->op_ret = -1;
-        frame->local = local;
-        loc_copy (&local->loc, loc);
-
-
-        if (name && strncmp (name, GF_XATTR_QUOTA_SIZE_KEY,
-                             strlen (GF_XATTR_QUOTA_SIZE_KEY)) == 0) {
-                local->wind_count = priv->child_count;
-
-                for (i = 0, trav=this->children; i < priv->child_count; i++,
-                             trav = trav->next) {
-                        STACK_WIND (frame, stripe_getxattr_cbk,
-                                    trav->xlator, trav->xlator->fops->getxattr,
-                                    loc, name, xdata);
-                }
-
-                return 0;
-        }
-
-        if (name && (XATTR_IS_PATHINFO (name))) {
-                if (IA_ISREG (loc->inode->ia_type)) {
-                        ret = inode_ctx_get (loc->inode, this,
-                                             (uint64_t *) &local->fctx);
-                        if (ret)
-                                gf_log (this->name, GF_LOG_ERROR,
-                                        "stripe size unavailable from fctx"
-                                        " relying on pathinfo could lead to"
-                                        " wrong results");
-                }
-
-                local->nallocs = local->wind_count = priv->child_count;
-                (void) strncpy (local->xsel, name, strlen (name));
-
-                /**
-                 * for xattrs that need info from all childs, fill ->xsel
-                 * as above and call the filler function in cbk based on
-                 * it
-                 */
-                for (i = 0, trav = this->children; i < priv->child_count; i++,
-                     trav = trav->next) {
-                        STACK_WIND_COOKIE (frame, stripe_vgetxattr_cbk,
-                                           (void *) (long) i, trav->xlator,
-                                           trav->xlator->fops->getxattr,
-                                           loc, name, xdata);
-                }
-
-                return 0;
-        }
-
-        if (cluster_handle_marker_getxattr (frame, loc, name, priv->vol_uuid,
-                                            stripe_getxattr_unwind,
-                                            stripe_marker_populate_args) == 0)
-                return 0;
-
-        STACK_WIND (frame, stripe_internal_getxattr_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->getxattr, loc, name, xdata);
-
-        return 0;
-
-err:
-        STRIPE_STACK_UNWIND (getxattr, frame, -1, op_errno, NULL, NULL);
-        return 0;
-}
-
-static inline gf_boolean_t
-stripe_is_special_xattr (const char *name)
-{
-        gf_boolean_t    is_spl = _gf_false;
-
-        if (!name) {
-                goto out;
-        }
-
-        if (!strncmp (name, GF_XATTR_LOCKINFO_KEY,
-                      strlen (GF_XATTR_LOCKINFO_KEY))
-            || XATTR_IS_PATHINFO (name))
-                is_spl = _gf_true;
-out:
-        return is_spl;
-}
-
-int32_t
-stripe_fgetxattr_from_everyone (call_frame_t *frame, xlator_t *this, fd_t *fd,
-                                const char *name, dict_t *xdata)
-{
-        stripe_local_t   *local = NULL;
-        stripe_private_t *priv  = NULL;
-        int32_t           ret   = -1, op_errno = 0;
-        int               i     = 0;
-        xlator_list_t    *trav  = NULL;
-
-        priv = this->private;
-
-        local = mem_get0 (this->local_pool);
-        if (!local) {
-                op_errno = ENOMEM;
-                goto err;
-        }
-
-        local->op_ret = -1;
-        frame->local = local;
-
-        strncpy (local->xsel, name, strlen (name));
-        local->nallocs = local->wind_count = priv->child_count;
-
-        for (i = 0, trav = this->children; i < priv->child_count; i++,
-                     trav = trav->next) {
-                STACK_WIND_COOKIE (frame, stripe_vgetxattr_cbk,
-                                   (void *) (long) i, trav->xlator,
-                                   trav->xlator->fops->fgetxattr,
-                                   fd, name, xdata);
-        }
-
-        return 0;
-
-err:
-        STACK_UNWIND_STRICT (fgetxattr, frame, -1, op_errno, NULL, NULL);
-        return ret;
-}
-
-int32_t
-stripe_fgetxattr (call_frame_t *frame, xlator_t *this, fd_t *fd,
-                  const char *name, dict_t *xdata)
-{
-        if (stripe_is_special_xattr (name)) {
-                stripe_fgetxattr_from_everyone (frame, this, fd, name, xdata);
-                goto out;
-        }
-
-        STACK_WIND (frame, stripe_internal_getxattr_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->fgetxattr, fd, name, xdata);
-
-out:
-        return 0;
-}
-
-
-
-int32_t
-stripe_priv_dump (xlator_t *this)
-{
-        char                    key[GF_DUMP_MAX_BUF_LEN];
-        int                     i = 0;
-        stripe_private_t       *priv = NULL;
-        int                     ret = -1;
-        struct stripe_options  *options = NULL;
-
-        GF_VALIDATE_OR_GOTO ("stripe", this, out);
-
-        priv = this->private;
-        if (!priv)
-                goto out;
-
-        ret = TRY_LOCK (&priv->lock);
-        if (ret != 0)
-                goto out;
-
-        gf_proc_dump_add_section("xlator.cluster.stripe.%s.priv", this->name);
-        gf_proc_dump_write("child_count","%d", priv->child_count);
-
-        for (i = 0; i < priv->child_count; i++) {
-                sprintf (key, "subvolumes[%d]", i);
-                gf_proc_dump_write (key, "%s.%s", priv->xl_array[i]->type,
-                                    priv->xl_array[i]->name);
-        }
-
-        options = priv->pattern;
-        while (options != NULL) {
-                gf_proc_dump_write ("path_pattern", "%s", priv->pattern->path_pattern);
-                gf_proc_dump_write ("options_block_size", "%ul", options->block_size);
-
-                options = options->next;
-        }
-
-        gf_proc_dump_write ("block_size", "%ul", priv->block_size);
-        gf_proc_dump_write ("nodes-down", "%d", priv->nodes_down);
-        gf_proc_dump_write ("first-child_down", "%d", priv->first_child_down);
-        gf_proc_dump_write ("xattr_supported", "%d", priv->xattr_supported);
-
-        UNLOCK (&priv->lock);
-
-out:
-        return ret;
-}
-
-struct xlator_fops fops = {
-        .stat           = stripe_stat,
-        .unlink         = stripe_unlink,
-        .rename         = stripe_rename,
-        .link           = stripe_link,
-        .truncate       = stripe_truncate,
-        .create         = stripe_create,
-        .open           = stripe_open,
-        .readv          = stripe_readv,
-        .writev         = stripe_writev,
-        .statfs         = stripe_statfs,
-        .flush          = stripe_flush,
-        .fsync          = stripe_fsync,
-        .ftruncate      = stripe_ftruncate,
-        .fstat          = stripe_fstat,
-        .mkdir          = stripe_mkdir,
-        .rmdir          = stripe_rmdir,
-        .lk             = stripe_lk,
-        .opendir        = stripe_opendir,
-        .fsyncdir       = stripe_fsyncdir,
-        .setattr        = stripe_setattr,
-        .fsetattr       = stripe_fsetattr,
-        .lookup         = stripe_lookup,
-        .mknod          = stripe_mknod,
-        .setxattr       = stripe_setxattr,
-        .fsetxattr      = stripe_fsetxattr,
-        .getxattr       = stripe_getxattr,
-        .fgetxattr      = stripe_fgetxattr,
-        .removexattr    = stripe_removexattr,
-        .fremovexattr   = stripe_fremovexattr,
-        .readdirp       = stripe_readdirp,
-	.fallocate	= stripe_fallocate,
-	.discard	= stripe_discard,
-        .zerofill       = stripe_zerofill,
-};
-
-struct xlator_cbks cbks = {
-        .release = stripe_release,
-        .forget  = stripe_forget,
-};
-
-struct xlator_dumpops dumpops = {
-        .priv = stripe_priv_dump,
-};
-
-struct volume_options options[] = {
-        { .key  = {"block-size"},
-          .type = GF_OPTION_TYPE_SIZE_LIST,
-          .default_value = "128KB",
-          .min = STRIPE_MIN_BLOCK_SIZE,
-          .description = "Size of the stripe unit that would be read "
-                         "from or written to the striped servers."
-        },
-        { .key  = {"use-xattr"},
-          .type = GF_OPTION_TYPE_BOOL,
-          .default_value = "true"
-        },
-	{ .key = {"coalesce"},
-	  .type = GF_OPTION_TYPE_BOOL,
-	  .default_value = "true",
-	  .description = "Enable/Disable coalesce mode to flatten striped "
-			 "files as stored on the server (i.e., eliminate holes "
-			 "caused by the traditional format)."
-	},
-        { .key  = {NULL} },
-};
diff --git a/xlators/cluster/stripe/src/stripe.h b/xlators/cluster/stripe/src/stripe.h
deleted file mode 100644
index 150d9e49f93..00000000000
--- a/xlators/cluster/stripe/src/stripe.h
+++ /dev/null
@@ -1,286 +0,0 @@
-/*
-  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
-  This file is part of GlusterFS.
-
-  This file is licensed to you under your choice of the GNU Lesser
-  General Public License, version 3 or any later version (LGPLv3 or
-  later), or the GNU General Public License, version 2 (GPLv2), in all
-  cases as published by the Free Software Foundation.
-*/
-
-
-#ifndef _STRIPE_H_
-#define _STRIPE_H_
-
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
-
-#include "xlator.h"
-#include "logging.h"
-#include "defaults.h"
-#include "common-utils.h"
-#include "compat.h"
-#include "compat-errno.h"
-#include "stripe-mem-types.h"
-#include "libxlator.h"
-#include <fnmatch.h>
-#include <signal.h>
-
-#define STRIPE_PATHINFO_HEADER "STRIPE:"
-#define STRIPE_MIN_BLOCK_SIZE  (16*GF_UNIT_KB)
-
-#define STRIPE_STACK_UNWIND(fop, frame, params ...) do {           \
-                stripe_local_t *__local = NULL;                    \
-                if (frame) {                                       \
-                        __local = frame->local;                    \
-                        frame->local = NULL;                       \
-                }                                                  \
-                STACK_UNWIND_STRICT (fop, frame, params);          \
-                if (__local) {                                     \
-                        stripe_local_wipe(__local);                \
-                        mem_put (__local);       \
-                }                                                  \
-        } while (0)
-
-#define STRIPE_STACK_DESTROY(frame) do {                        \
-                stripe_local_t *__local = NULL;                 \
-                __local = frame->local;                         \
-                frame->local = NULL;                            \
-                STACK_DESTROY (frame->root);                    \
-                if (__local) {                                  \
-                        stripe_local_wipe (__local);            \
-                        mem_put (__local);    \
-                }                                               \
-        } while (0)
-
-#define STRIPE_VALIDATE_FCTX(fctx, label) do {                  \
-        int     idx = 0;                                        \
-        if (!fctx) {                                            \
-                op_errno = EINVAL;                              \
-                goto label;                                     \
-        }                                                       \
-        for (idx = 0; idx < fctx->stripe_count; idx++) {        \
-                if (!fctx->xl_array[idx]) {                     \
-                        gf_log (this->name, GF_LOG_ERROR,       \
-                                "fctx->xl_array[%d] is NULL",   \
-                                idx);                           \
-                        op_errno = ESTALE;                      \
-                        goto label;                             \
-                }                                               \
-        }                                                       \
-       } while (0)
-
-typedef struct stripe_xattr_sort {
-        int   pos;
-        int   xattr_len;
-        char *xattr_value;
-} stripe_xattr_sort_t;
-
-/**
- * struct stripe_options : This keeps the pattern and the block-size
- *     information, which is used for striping on a file.
- */
-struct stripe_options {
-        struct stripe_options *next;
-        char                   path_pattern[256];
-        uint64_t               block_size;
-};
-
-/**
- * Private structure for stripe translator
- */
-struct stripe_private {
-        struct stripe_options  *pattern;
-        xlator_t              **xl_array;
-        uint64_t                block_size;
-        gf_lock_t               lock;
-        uint8_t                 nodes_down;
-        int8_t                  first_child_down;
-        int                    *last_event;
-        int8_t                  child_count;
-        gf_boolean_t            xattr_supported;  /* default yes */
-	gf_boolean_t		coalesce;
-        char                    vol_uuid[UUID_SIZE + 1];
-};
-
-/**
- * Used to keep info about the replies received from readv/writev calls
- */
-struct stripe_replies {
-        struct iovec *vector;
-        int32_t       count;    //count of vector
-        int32_t       op_ret;   //op_ret of readv
-        int32_t       op_errno;
-        int32_t       requested_size;
-        struct iatt   stbuf;    /* 'stbuf' is also a part of reply */
-};
-
-typedef struct _stripe_fd_ctx {
-        off_t      stripe_size;
-        int        stripe_count;
-	int	   stripe_coalesce;
-        int        static_array;
-        xlator_t **xl_array;
-} stripe_fd_ctx_t;
-
-
-/**
- * Local structure to be passed with all the frames in case of STACK_WIND
- */
-struct stripe_local; /* this itself is used inside the structure; */
-
-struct stripe_local {
-        struct stripe_local *next;
-        call_frame_t        *orig_frame;
-
-        stripe_fd_ctx_t     *fctx;
-
-        /* Used by _cbk functions */
-        struct iatt          stbuf;
-        struct iatt          pre_buf;
-        struct iatt          post_buf;
-        struct iatt          preparent;
-        struct iatt          postparent;
-
-        off_t                stbuf_size;
-        off_t                prebuf_size;
-        off_t                postbuf_size;
-        off_t                preparent_size;
-        off_t                postparent_size;
-
-        blkcnt_t             stbuf_blocks;
-        blkcnt_t             prebuf_blocks;
-        blkcnt_t             postbuf_blocks;
-        blkcnt_t             preparent_blocks;
-        blkcnt_t             postparent_blocks;
-
-        struct stripe_replies *replies;
-        struct statvfs        statvfs_buf;
-        dir_entry_t          *entry;
-
-        int8_t               revalidate;
-        int8_t               failed;
-        int8_t               unwind;
-
-        size_t               readv_size;
-        int32_t              entry_count;
-        int32_t              node_index;
-        int32_t              call_count;
-        int32_t              wind_count; /* used instead of child_cound
-                                            in case of read and write */
-        int32_t              op_ret;
-        int32_t              op_errno;
-        int32_t              count;
-        int32_t              flags;
-        char                *name;
-        inode_t             *inode;
-
-        loc_t                loc;
-        loc_t                loc2;
-
-        mode_t               mode;
-        dev_t                rdev;
-        /* For File I/O fops */
-        dict_t              *xdata;
-
-        stripe_xattr_sort_t *xattr_list;
-        int32_t              xattr_total_len;
-        int32_t              nallocs;
-        char xsel[256];
-
-        /* General usage */
-        off_t                offset;
-        off_t                stripe_size;
-
-        int xattr_self_heal_needed;
-        int entry_self_heal_needed;
-
-        int8_t              *list;
-        struct gf_flock         lock;
-        fd_t                *fd;
-        void                *value;
-        struct iobref       *iobref;
-        gf_dirent_t          entries;
-        gf_dirent_t         *dirent;
-        dict_t              *xattr;
-        uuid_t               ia_gfid;
-
-        int                  xflag;
-        mode_t               umask;
-};
-
-typedef struct stripe_local   stripe_local_t;
-typedef struct stripe_private stripe_private_t;
-
-/*
- * Determine the stripe index of a particular frame based on the translator.
- */
-static inline int32_t stripe_get_frame_index(stripe_fd_ctx_t *fctx,
-					     call_frame_t *prev)
-{
-	int32_t i, idx = -1;
-
-	for (i = 0; i < fctx->stripe_count; i++) {
-		if (fctx->xl_array[i] == prev->this) {
-			idx = i;
-			break;
-		}
-	}
-
-	return idx;
-}
-
-static inline void stripe_copy_xl_array(xlator_t **dst, xlator_t **src,
-					int count)
-{
-	int i;
-
-	for (i = 0; i < count; i++)
-		dst[i] = src[i];
-}
-
-void stripe_local_wipe (stripe_local_t *local);
-int32_t stripe_ctx_handle (xlator_t *this, call_frame_t *prev,
-                           stripe_local_t *local, dict_t *dict);
-void stripe_aggregate_xattr (dict_t *dst, dict_t *src);
-int32_t stripe_xattr_request_build (xlator_t *this, dict_t *dict,
-                                    uint64_t stripe_size, uint32_t stripe_count,
-                                    uint32_t stripe_index,
-				    uint32_t stripe_coalesce);
-int32_t stripe_get_matching_bs (const char *path, stripe_private_t *priv);
-int set_stripe_block_size (xlator_t *this, stripe_private_t *priv, char *data);
-int32_t stripe_iatt_merge (struct iatt *from, struct iatt *to);
-int32_t stripe_fill_pathinfo_xattr (xlator_t *this, stripe_local_t *local,
-                                    char **xattr_serz);
-int32_t stripe_free_xattr_str (stripe_local_t *local);
-int32_t stripe_xattr_aggregate (char *buffer, stripe_local_t *local,
-                                int32_t *total);
-off_t coalesced_offset(off_t offset, uint64_t stripe_size, int stripe_count);
-off_t uncoalesced_size(off_t size, uint64_t stripe_size, int stripe_count,
-			int stripe_index);
-int32_t
-stripe_fill_lockinfo_xattr (xlator_t *this, stripe_local_t *local,
-                            void **xattr_serz);
-
-/*
- * Adjust the size attribute for files if coalesce is enabled.
- */
-static inline void correct_file_size(struct iatt *buf, stripe_fd_ctx_t *fctx,
-	call_frame_t *prev)
-{
-	int index;
-
-	if (!IA_ISREG(buf->ia_type))
-		return;
-
-	if (!fctx || !fctx->stripe_coalesce)
-		return;
-
-	index = stripe_get_frame_index(fctx, prev);
-	buf->ia_size = uncoalesced_size(buf->ia_size, fctx->stripe_size,
-		fctx->stripe_count, index);
-}
-
-#endif /* _STRIPE_H_ */
diff --git a/xlators/debug/Makefile.am b/xlators/debug/Makefile.am
index b655554efec..88fac1c6d9e 100644
--- a/xlators/debug/Makefile.am
+++ b/xlators/debug/Makefile.am
@@ -1,3 +1,3 @@
-SUBDIRS = trace error-gen io-stats
+SUBDIRS = error-gen io-stats sink trace delay-gen
 
 CLEANFILES = 
diff --git a/xlators/features/changetimerecorder/Makefile.am b/xlators/debug/delay-gen/Makefile.am
index a985f42a877..a985f42a877 100644
--- a/xlators/features/changetimerecorder/Makefile.am
+++ b/xlators/debug/delay-gen/Makefile.am
diff --git a/xlators/debug/delay-gen/src/Makefile.am b/xlators/debug/delay-gen/src/Makefile.am
new file mode 100644
index 00000000000..8f758dec199
--- /dev/null
+++ b/xlators/debug/delay-gen/src/Makefile.am
@@ -0,0 +1,11 @@
+
+xlator_LTLIBRARIES = delay-gen.la
+xlatordir = $(libdir)/glusterfs/$(PACKAGE_VERSION)/xlator/debug
+delay_gen_la_LDFLAGS = -module $(GF_XLATOR_DEFAULT_LDFLAGS)
+delay_gen_la_SOURCES = delay-gen.c
+delay_gen_la_LIBADD = $(top_builddir)/libglusterfs/src/libglusterfs.la
+noinst_HEADERS = delay-gen.h delay-gen-mem-types.h delay-gen-messages.h
+AM_CPPFLAGS = $(GF_CPPFLAGS) -I$(top_srcdir)/libglusterfs/src \
+	   -I$(top_srcdir)/rpc/xdr/src -I$(top_builddir)/rpc/xdr/src
+AM_CFLAGS = -Wall -fno-strict-aliasing $(GF_CFLAGS)
+CLEANFILES =
diff --git a/xlators/debug/delay-gen/src/delay-gen-mem-types.h b/xlators/debug/delay-gen/src/delay-gen-mem-types.h
new file mode 100644
index 00000000000..c89a9217193
--- /dev/null
+++ b/xlators/debug/delay-gen/src/delay-gen-mem-types.h
@@ -0,0 +1,21 @@
+/*
+ *   Copyright (c) 2017 Red Hat, Inc. <http://www.redhat.com>
+ *   This file is part of GlusterFS.
+ *
+ *   This file is licensed to you under your choice of the GNU Lesser
+ *   General Public License, version 3 or any later version (LGPLv3 or
+ *   later), or the GNU General Public License, version 2 (GPLv2), in all
+ *   cases as published by the Free Software Foundation.
+ */
+
+#ifndef __DELAY_GEN_MEM_TYPES_H__
+#define __DELAY_GEN_MEM_TYPES_H__
+
+#include <glusterfs/mem-types.h>
+
+enum gf_delay_gen_mem_types_ {
+    gf_delay_gen_mt_dg_t = gf_common_mt_end + 1,
+    gf_delay_gen_mt_end
+};
+
+#endif /* __DELAY_GEN_MEM_TYPES_H__ */
diff --git a/xlators/debug/delay-gen/src/delay-gen-messages.h b/xlators/debug/delay-gen/src/delay-gen-messages.h
new file mode 100644
index 00000000000..bc98cec2885
--- /dev/null
+++ b/xlators/debug/delay-gen/src/delay-gen-messages.h
@@ -0,0 +1,26 @@
+/*
+ *   Copyright (c) 2017 Red Hat, Inc. <http://www.redhat.com>
+ *   This file is part of GlusterFS.
+ *
+ *   This file is licensed to you under your choice of the GNU Lesser
+ *   General Public License, version 3 or any later version (LGPLv3 or
+ *   later), or the GNU General Public License, version 2 (GPLv2), in all
+ *   cases as published by the Free Software Foundation.
+ */
+
+#ifndef __DELAY_GEN_MESSAGES_H__
+#define __DELAY_GEN_MESSAGES_H__
+
+#include <glusterfs/glfs-message-id.h>
+
+/* To add new message IDs, append new identifiers at the end of the list.
+ *
+ * Never remove a message ID. If it's not used anymore, you can rename it or
+ * leave it as it is, but not delete it. This is to prevent reutilization of
+ * IDs by other messages.
+ *
+ * The component name must match one of the entries defined in
+ * glfs-message-id.h.
+ */
+
+#endif /* __DELAY_GEN_MESSAGES_H__ */
diff --git a/xlators/debug/delay-gen/src/delay-gen.c b/xlators/debug/delay-gen/src/delay-gen.c
new file mode 100644
index 00000000000..4698f1fd785
--- /dev/null
+++ b/xlators/debug/delay-gen/src/delay-gen.c
@@ -0,0 +1,697 @@
+/*
+ *  Copyright (c) 2017 Red Hat, Inc. <http://www.redhat.com>
+ *  This file is part of GlusterFS.
+ *
+ *  This file is licensed to you under your choice of the GNU Lesser
+ *  General Public License, version 3 or any later version (LGPLv3 or
+ *  later), or the GNU General Public License, version 2 (GPLv2), in all
+ *  cases as published by the Free Software Foundation.
+ */
+
+#include "delay-gen.h"
+
+#define DELAY_GRANULARITY (1 << 20)
+
+#define DG_FOP(fop, name, frame, this, args...)                                \
+    do {                                                                       \
+        delay_gen(this, fop);                                                  \
+        default_##name(frame, this, args);                                     \
+    } while (0)
+
+int
+delay_gen(xlator_t *this, int fop)
+{
+    dg_t *dg = this->private;
+
+    if (!dg->enable[fop] || !dg->delay_ppm)
+        return 0;
+
+    if ((rand() % DELAY_GRANULARITY) < dg->delay_ppm)
+        gf_nanosleep(dg->delay_duration * GF_US_IN_NS);
+
+    return 0;
+}
+
+int32_t
+dg_rename(call_frame_t *frame, xlator_t *this, loc_t *oldloc, loc_t *newloc,
+          dict_t *xdata)
+{
+    DG_FOP(GF_FOP_RENAME, rename, frame, this, oldloc, newloc, xdata);
+    return 0;
+}
+
+int32_t
+dg_ipc(call_frame_t *frame, xlator_t *this, int32_t op, dict_t *xdata)
+{
+    DG_FOP(GF_FOP_IPC, ipc, frame, this, op, xdata);
+    return 0;
+}
+
+int32_t
+dg_setactivelk(call_frame_t *frame, xlator_t *this, loc_t *loc,
+               lock_migration_info_t *locklist, dict_t *xdata)
+{
+    DG_FOP(GF_FOP_SETACTIVELK, setactivelk, frame, this, loc, locklist, xdata);
+    return 0;
+}
+
+int32_t
+dg_flush(call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *xdata)
+{
+    DG_FOP(GF_FOP_FLUSH, flush, frame, this, fd, xdata);
+    return 0;
+}
+
+int32_t
+dg_readdir(call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
+           off_t off, dict_t *xdata)
+{
+    DG_FOP(GF_FOP_READDIR, readdir, frame, this, fd, size, off, xdata);
+    return 0;
+}
+
+int32_t
+dg_setxattr(call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *dict,
+            int32_t flags, dict_t *xdata)
+{
+    DG_FOP(GF_FOP_SETXATTR, setxattr, frame, this, loc, dict, flags, xdata);
+    return 0;
+}
+
+int32_t
+dg_mknod(call_frame_t *frame, xlator_t *this, loc_t *loc, mode_t mode,
+         dev_t rdev, mode_t umask, dict_t *xdata)
+{
+    DG_FOP(GF_FOP_MKNOD, mknod, frame, this, loc, mode, rdev, umask, xdata);
+    return 0;
+}
+
+int32_t
+dg_fsetxattr(call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *dict,
+             int32_t flags, dict_t *xdata)
+{
+    DG_FOP(GF_FOP_FSETXATTR, fsetxattr, frame, this, fd, dict, flags, xdata);
+    return 0;
+}
+
+int32_t
+dg_readv(call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
+         off_t offset, uint32_t flags, dict_t *xdata)
+{
+    DG_FOP(GF_FOP_READ, readv, frame, this, fd, size, offset, flags, xdata);
+    return 0;
+}
+
+int32_t
+dg_inodelk(call_frame_t *frame, xlator_t *this, const char *volume, loc_t *loc,
+           int32_t cmd, struct gf_flock *lock, dict_t *xdata)
+{
+    DG_FOP(GF_FOP_INODELK, inodelk, frame, this, volume, loc, cmd, lock, xdata);
+    return 0;
+}
+
+int32_t
+dg_fremovexattr(call_frame_t *frame, xlator_t *this, fd_t *fd, const char *name,
+                dict_t *xdata)
+{
+    DG_FOP(GF_FOP_FREMOVEXATTR, fremovexattr, frame, this, fd, name, xdata);
+    return 0;
+}
+
+int32_t
+dg_open(call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t flags,
+        fd_t *fd, dict_t *xdata)
+{
+    DG_FOP(GF_FOP_OPEN, open, frame, this, loc, flags, fd, xdata);
+    return 0;
+}
+
+int32_t
+dg_xattrop(call_frame_t *frame, xlator_t *this, loc_t *loc,
+           gf_xattrop_flags_t flags, dict_t *dict, dict_t *xdata)
+{
+    DG_FOP(GF_FOP_XATTROP, xattrop, frame, this, loc, flags, dict, xdata);
+    return 0;
+}
+
+int32_t
+dg_entrylk(call_frame_t *frame, xlator_t *this, const char *volume, loc_t *loc,
+           const char *basename, entrylk_cmd cmd, entrylk_type type,
+           dict_t *xdata)
+{
+    DG_FOP(GF_FOP_ENTRYLK, entrylk, frame, this, volume, loc, basename, cmd,
+           type, xdata);
+    return 0;
+}
+
+int32_t
+dg_getactivelk(call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xdata)
+{
+    DG_FOP(GF_FOP_GETACTIVELK, getactivelk, frame, this, loc, xdata);
+    return 0;
+}
+
+int32_t
+dg_finodelk(call_frame_t *frame, xlator_t *this, const char *volume, fd_t *fd,
+            int32_t cmd, struct gf_flock *lock, dict_t *xdata)
+{
+    DG_FOP(GF_FOP_FINODELK, finodelk, frame, this, volume, fd, cmd, lock,
+           xdata);
+    return 0;
+}
+
+int32_t
+dg_create(call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t flags,
+          mode_t mode, mode_t umask, fd_t *fd, dict_t *xdata)
+{
+    DG_FOP(GF_FOP_CREATE, create, frame, this, loc, flags, mode, umask, fd,
+           xdata);
+    return 0;
+}
+
+int32_t
+dg_discard(call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
+           size_t len, dict_t *xdata)
+{
+    DG_FOP(GF_FOP_DISCARD, discard, frame, this, fd, offset, len, xdata);
+    return 0;
+}
+
+int32_t
+dg_mkdir(call_frame_t *frame, xlator_t *this, loc_t *loc, mode_t mode,
+         mode_t umask, dict_t *xdata)
+{
+    DG_FOP(GF_FOP_MKDIR, mkdir, frame, this, loc, mode, umask, xdata);
+    return 0;
+}
+
+int32_t
+dg_lk(call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t cmd,
+      struct gf_flock *lock, dict_t *xdata)
+{
+    DG_FOP(GF_FOP_LK, lk, frame, this, fd, cmd, lock, xdata);
+    return 0;
+}
+
+int32_t
+dg_writev(call_frame_t *frame, xlator_t *this, fd_t *fd, struct iovec *vector,
+          int32_t count, off_t off, uint32_t flags, struct iobref *iobref,
+          dict_t *xdata)
+{
+    DG_FOP(GF_FOP_WRITE, writev, frame, this, fd, vector, count, off, flags,
+           iobref, xdata);
+    return 0;
+}
+
+int32_t
+dg_access(call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t mask,
+          dict_t *xdata)
+{
+    DG_FOP(GF_FOP_ACCESS, access, frame, this, loc, mask, xdata);
+    return 0;
+}
+
+int32_t
+dg_lookup(call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xdata)
+{
+    DG_FOP(GF_FOP_LOOKUP, lookup, frame, this, loc, xdata);
+    return 0;
+}
+
+int32_t
+dg_rmdir(call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t flags,
+         dict_t *xdata)
+{
+    DG_FOP(GF_FOP_RMDIR, rmdir, frame, this, loc, flags, xdata);
+    return 0;
+}
+
+int32_t
+dg_fallocate(call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t keep_size,
+             off_t offset, size_t len, dict_t *xdata)
+{
+    DG_FOP(GF_FOP_FALLOCATE, fallocate, frame, this, fd, keep_size, offset, len,
+           xdata);
+    return 0;
+}
+
+int32_t
+dg_fstat(call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *xdata)
+{
+    DG_FOP(GF_FOP_FSTAT, fstat, frame, this, fd, xdata);
+    return 0;
+}
+
+int32_t
+dg_lease(call_frame_t *frame, xlator_t *this, loc_t *loc,
+         struct gf_lease *lease, dict_t *xdata)
+{
+    DG_FOP(GF_FOP_LEASE, lease, frame, this, loc, lease, xdata);
+    return 0;
+}
+
+int32_t
+dg_stat(call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xdata)
+{
+    DG_FOP(GF_FOP_STAT, stat, frame, this, loc, xdata);
+    return 0;
+}
+
+int32_t
+dg_truncate(call_frame_t *frame, xlator_t *this, loc_t *loc, off_t offset,
+            dict_t *xdata)
+{
+    DG_FOP(GF_FOP_TRUNCATE, truncate, frame, this, loc, offset, xdata);
+    return 0;
+}
+
+int32_t
+dg_getxattr(call_frame_t *frame, xlator_t *this, loc_t *loc, const char *name,
+            dict_t *xdata)
+{
+    DG_FOP(GF_FOP_GETXATTR, getxattr, frame, this, loc, name, xdata);
+    return 0;
+}
+
+int32_t
+dg_symlink(call_frame_t *frame, xlator_t *this, const char *linkpath,
+           loc_t *loc, mode_t umask, dict_t *xdata)
+{
+    DG_FOP(GF_FOP_SYMLINK, symlink, frame, this, linkpath, loc, umask, xdata);
+    return 0;
+}
+
+int32_t
+dg_zerofill(call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
+            off_t len, dict_t *xdata)
+{
+    DG_FOP(GF_FOP_ZEROFILL, zerofill, frame, this, fd, offset, len, xdata);
+    return 0;
+}
+
+int32_t
+dg_fsyncdir(call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t flags,
+            dict_t *xdata)
+{
+    DG_FOP(GF_FOP_FSYNCDIR, fsyncdir, frame, this, fd, flags, xdata);
+    return 0;
+}
+
+int32_t
+dg_fgetxattr(call_frame_t *frame, xlator_t *this, fd_t *fd, const char *name,
+             dict_t *xdata)
+{
+    DG_FOP(GF_FOP_FGETXATTR, fgetxattr, frame, this, fd, name, xdata);
+    return 0;
+}
+
+int32_t
+dg_readdirp(call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
+            off_t off, dict_t *xdata)
+{
+    DG_FOP(GF_FOP_READDIRP, readdirp, frame, this, fd, size, off, xdata);
+    return 0;
+}
+
+int32_t
+dg_link(call_frame_t *frame, xlator_t *this, loc_t *oldloc, loc_t *newloc,
+        dict_t *xdata)
+{
+    DG_FOP(GF_FOP_LINK, link, frame, this, oldloc, newloc, xdata);
+    return 0;
+}
+
+int32_t
+dg_fxattrop(call_frame_t *frame, xlator_t *this, fd_t *fd,
+            gf_xattrop_flags_t flags, dict_t *dict, dict_t *xdata)
+{
+    DG_FOP(GF_FOP_FXATTROP, fxattrop, frame, this, fd, flags, dict, xdata);
+    return 0;
+}
+
+int32_t
+dg_ftruncate(call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
+             dict_t *xdata)
+{
+    DG_FOP(GF_FOP_FTRUNCATE, ftruncate, frame, this, fd, offset, xdata);
+    return 0;
+}
+
+int32_t
+dg_rchecksum(call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
+             int32_t len, dict_t *xdata)
+{
+    DG_FOP(GF_FOP_RCHECKSUM, rchecksum, frame, this, fd, offset, len, xdata);
+    return 0;
+}
+
+int32_t
+dg_unlink(call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t flags,
+          dict_t *xdata)
+{
+    DG_FOP(GF_FOP_UNLINK, unlink, frame, this, loc, flags, xdata);
+    return 0;
+}
+
+int32_t
+dg_fentrylk(call_frame_t *frame, xlator_t *this, const char *volume, fd_t *fd,
+            const char *basename, entrylk_cmd cmd, entrylk_type type,
+            dict_t *xdata)
+{
+    DG_FOP(GF_FOP_FENTRYLK, fentrylk, frame, this, volume, fd, basename, cmd,
+           type, xdata);
+    return 0;
+}
+
+int32_t
+dg_getspec(call_frame_t *frame, xlator_t *this, const char *key, int32_t flags)
+{
+    DG_FOP(GF_FOP_GETSPEC, getspec, frame, this, key, flags);
+    return 0;
+}
+
+int32_t
+dg_setattr(call_frame_t *frame, xlator_t *this, loc_t *loc, struct iatt *stbuf,
+           int32_t valid, dict_t *xdata)
+{
+    DG_FOP(GF_FOP_SETATTR, setattr, frame, this, loc, stbuf, valid, xdata);
+    return 0;
+}
+
+int32_t
+dg_fsync(call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t flags,
+         dict_t *xdata)
+{
+    DG_FOP(GF_FOP_FSYNC, fsync, frame, this, fd, flags, xdata);
+    return 0;
+}
+
+int32_t
+dg_statfs(call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xdata)
+{
+    DG_FOP(GF_FOP_STATFS, statfs, frame, this, loc, xdata);
+    return 0;
+}
+
+int32_t
+dg_seek(call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
+        gf_seek_what_t what, dict_t *xdata)
+{
+    DG_FOP(GF_FOP_SEEK, seek, frame, this, fd, offset, what, xdata);
+    return 0;
+}
+
+int32_t
+dg_fsetattr(call_frame_t *frame, xlator_t *this, fd_t *fd, struct iatt *stbuf,
+            int32_t valid, dict_t *xdata)
+{
+    DG_FOP(GF_FOP_FSETATTR, fsetattr, frame, this, fd, stbuf, valid, xdata);
+    return 0;
+}
+
+int32_t
+dg_opendir(call_frame_t *frame, xlator_t *this, loc_t *loc, fd_t *fd,
+           dict_t *xdata)
+{
+    DG_FOP(GF_FOP_OPENDIR, opendir, frame, this, loc, fd, xdata);
+    return 0;
+}
+
+int32_t
+dg_readlink(call_frame_t *frame, xlator_t *this, loc_t *loc, size_t size,
+            dict_t *xdata)
+{
+    DG_FOP(GF_FOP_READLINK, readlink, frame, this, loc, size, xdata);
+    return 0;
+}
+
+int32_t
+dg_removexattr(call_frame_t *frame, xlator_t *this, loc_t *loc,
+               const char *name, dict_t *xdata)
+{
+    DG_FOP(GF_FOP_REMOVEXATTR, removexattr, frame, this, loc, name, xdata);
+    return 0;
+}
+
+int32_t
+dg_forget(xlator_t *this, inode_t *inode)
+{
+    return 0;
+}
+
+int32_t
+dg_release(xlator_t *this, fd_t *fd)
+{
+    return 0;
+}
+
+int32_t
+dg_releasedir(xlator_t *this, fd_t *fd)
+{
+    return 0;
+}
+
+static int
+delay_gen_parse_fill_fops(dg_t *dg, char *enable_fops)
+{
+    char *op_no_str = NULL;
+    int op_no = -1;
+    int i = 0;
+    int ret = 0;
+    xlator_t *this = THIS;
+    char *saveptr = NULL;
+    char *dup_enable_fops = NULL;
+
+    if (strlen(enable_fops) == 0) {
+        for (i = GF_FOP_NULL + 1; i < GF_FOP_MAXVALUE; i++)
+            dg->enable[i] = 1;
+    } else {
+        dup_enable_fops = gf_strdup(enable_fops);
+        if (!dup_enable_fops) {
+            ret = -1;
+            goto out;
+        }
+        op_no_str = strtok_r(dup_enable_fops, ",", &saveptr);
+        while (op_no_str) {
+            op_no = gf_fop_int(op_no_str);
+            if (op_no == -1) {
+                gf_log(this->name, GF_LOG_WARNING, "Wrong option value %s",
+                       op_no_str);
+                ret = -1;
+                goto out;
+            } else {
+                dg->enable[op_no] = 1;
+            }
+
+            op_no_str = strtok_r(NULL, ",", &saveptr);
+        }
+    }
+out:
+    GF_FREE(dup_enable_fops);
+    return ret;
+}
+
+void
+delay_gen_set_delay_ppm(dg_t *dg, double percent)
+{
+    double ppm;
+
+    ppm = (percent / 100.0) * (double)DELAY_GRANULARITY;
+    dg->delay_ppm = ppm;
+}
+
+int32_t
+init(xlator_t *this)
+{
+    dg_t *dg = NULL;
+    int32_t ret = 0;
+    double delay_percent = 0;
+    char *delay_enable_fops = NULL;
+
+    if (!this->children || this->children->next) {
+        gf_log(this->name, GF_LOG_ERROR,
+               "delay-gen not configured with one subvolume");
+        ret = -1;
+        goto out;
+    }
+
+    if (!this->parents) {
+        gf_log(this->name, GF_LOG_WARNING, "dangling volume. check volfile ");
+    }
+
+    dg = GF_CALLOC(1, sizeof(*dg), gf_delay_gen_mt_dg_t);
+
+    if (!dg) {
+        ret = -1;
+        goto out;
+    }
+
+    ret = -1;
+
+    GF_OPTION_INIT("delay-percentage", delay_percent, percent, out);
+    GF_OPTION_INIT("enable", delay_enable_fops, str, out);
+    GF_OPTION_INIT("delay-duration", dg->delay_duration, int32, out);
+
+    delay_gen_set_delay_ppm(dg, delay_percent);
+
+    ret = delay_gen_parse_fill_fops(dg, delay_enable_fops);
+    if (ret)
+        goto out;
+
+    this->private = dg;
+
+    ret = 0;
+out:
+    if (ret)
+        GF_FREE(dg);
+    return ret;
+}
+
+void
+fini(xlator_t *this)
+{
+    GF_FREE(this->private);
+}
+
+int32_t
+mem_acct_init(xlator_t *this)
+{
+    int ret = -1;
+
+    if (!this)
+        return ret;
+
+    ret = xlator_mem_acct_init(this, gf_delay_gen_mt_end + 1);
+
+    if (ret != 0) {
+        gf_log(this->name, GF_LOG_ERROR,
+               "Memory accounting init"
+               " failed");
+        return ret;
+    }
+
+    return ret;
+}
+
+int32_t
+reconfigure(xlator_t *this, dict_t *dict)
+{
+    /*At the moment I don't see any need to implement this. In future
+     *if this is needed we can add code here.
+     */
+    return 0;
+}
+
+int
+notify(xlator_t *this, int event, void *data, ...)
+{
+    return default_notify(this, event, data);
+}
+
+struct xlator_fops fops = {
+    .rename = dg_rename,
+    .ipc = dg_ipc,
+    .setactivelk = dg_setactivelk,
+    .flush = dg_flush,
+    .readdir = dg_readdir,
+    .setxattr = dg_setxattr,
+    .mknod = dg_mknod,
+    .fsetxattr = dg_fsetxattr,
+    .readv = dg_readv,
+    .inodelk = dg_inodelk,
+    .fremovexattr = dg_fremovexattr,
+    .open = dg_open,
+    .xattrop = dg_xattrop,
+    .entrylk = dg_entrylk,
+    .getactivelk = dg_getactivelk,
+    .finodelk = dg_finodelk,
+    .create = dg_create,
+    .discard = dg_discard,
+    .mkdir = dg_mkdir,
+    .lk = dg_lk,
+    .writev = dg_writev,
+    .access = dg_access,
+    .lookup = dg_lookup,
+    .rmdir = dg_rmdir,
+    .fallocate = dg_fallocate,
+    .fstat = dg_fstat,
+    .lease = dg_lease,
+    .stat = dg_stat,
+    .truncate = dg_truncate,
+    .getxattr = dg_getxattr,
+    .symlink = dg_symlink,
+    .zerofill = dg_zerofill,
+    .fsyncdir = dg_fsyncdir,
+    .fgetxattr = dg_fgetxattr,
+    .readdirp = dg_readdirp,
+    .link = dg_link,
+    .fxattrop = dg_fxattrop,
+    .ftruncate = dg_ftruncate,
+    .rchecksum = dg_rchecksum,
+    .unlink = dg_unlink,
+    .fentrylk = dg_fentrylk,
+    .getspec = dg_getspec,
+    .setattr = dg_setattr,
+    .fsync = dg_fsync,
+    .statfs = dg_statfs,
+    .seek = dg_seek,
+    .fsetattr = dg_fsetattr,
+    .opendir = dg_opendir,
+    .readlink = dg_readlink,
+    .removexattr = dg_removexattr,
+};
+
+struct xlator_cbks cbks = {
+    .forget = dg_forget,
+    .release = dg_release,
+    .releasedir = dg_releasedir,
+};
+
+struct volume_options options[] = {
+    {
+        .key = {"delay-percentage"},
+        .type = GF_OPTION_TYPE_PERCENT,
+        .default_value = "10%",
+        .description = "Percentage delay of operations when enabled.",
+        .op_version = {GD_OP_VERSION_3_13_0},
+        .flags = OPT_FLAG_SETTABLE,
+        .tags = {"delay-gen"},
+    },
+
+    {
+        .key = {"delay-duration"},
+        .type = GF_OPTION_TYPE_INT,
+        .description = "Delay duration in micro seconds",
+        .default_value = "100000",
+        .op_version = {GD_OP_VERSION_3_13_0},
+        .flags = OPT_FLAG_SETTABLE,
+        .tags = {"delay-gen"},
+    },
+
+    {
+        .key = {"enable"},
+        .type = GF_OPTION_TYPE_STR,
+        .description = "Accepts a string which takes ',' separated fop "
+                       "strings to denote which fops are enabled for delay",
+        .op_version = {GD_OP_VERSION_3_13_0},
+        .flags = OPT_FLAG_SETTABLE,
+        .tags = {"delay-gen"},
+        .default_value = "",
+    },
+
+    {.key = {NULL}},
+};
+
+xlator_api_t xlator_api = {
+    .init = init,
+    .fini = fini,
+    .notify = notify,
+    .reconfigure = reconfigure,
+    .mem_acct_init = mem_acct_init,
+    .op_version = {GD_OP_VERSION_3_12_0},
+    .fops = &fops,
+    .cbks = &cbks,
+    .options = options,
+    .identifier = "delay-gen",
+    .category = GF_TECH_PREVIEW,
+};
diff --git a/xlators/debug/delay-gen/src/delay-gen.h b/xlators/debug/delay-gen/src/delay-gen.h
new file mode 100644
index 00000000000..afa95e5eb2d
--- /dev/null
+++ b/xlators/debug/delay-gen/src/delay-gen.h
@@ -0,0 +1,27 @@
+/*
+ *   Copyright (c) 2017 Red Hat, Inc. <http://www.redhat.com>
+ *   This file is part of GlusterFS.
+ *
+ *   This file is licensed to you under your choice of the GNU Lesser
+ *   General Public License, version 3 or any later version (LGPLv3 or
+ *   later), or the GNU General Public License, version 2 (GPLv2), in all
+ *   cases as published by the Free Software Foundation.
+ */
+
+#ifndef __DELAY_GEN_H__
+#define __DELAY_GEN_H__
+
+#include "delay-gen-mem-types.h"
+#include "delay-gen-messages.h"
+#include <glusterfs/glusterfs.h>
+#include <glusterfs/xlator.h>
+#include <glusterfs/defaults.h>
+
+typedef struct {
+    int enable[GF_FOP_MAXVALUE];
+    int op_count;
+    int delay_ppm;
+    int delay_duration;
+} dg_t;
+
+#endif /* __DELAY_GEN_H__ */
diff --git a/xlators/debug/error-gen/src/Makefile.am b/xlators/debug/error-gen/src/Makefile.am
index 5075c59a861..038d2e8e66d 100644
--- a/xlators/debug/error-gen/src/Makefile.am
+++ b/xlators/debug/error-gen/src/Makefile.am
@@ -2,14 +2,15 @@
 xlator_LTLIBRARIES = error-gen.la
 xlatordir = $(libdir)/glusterfs/$(PACKAGE_VERSION)/xlator/debug
 
-error_gen_la_LDFLAGS = -module -avoid-version
+error_gen_la_LDFLAGS = -module $(GF_XLATOR_DEFAULT_LDFLAGS)
 
 error_gen_la_SOURCES = error-gen.c
 error_gen_la_LIBADD = $(top_builddir)/libglusterfs/src/libglusterfs.la
 
 noinst_HEADERS = error-gen.h error-gen-mem-types.h
 
-AM_CPPFLAGS = $(GF_CPPFLAGS) -I$(top_srcdir)/libglusterfs/src
+AM_CPPFLAGS = $(GF_CPPFLAGS) -I$(top_srcdir)/libglusterfs/src \
+	-I$(top_srcdir)/rpc/xdr/src -I$(top_builddir)/rpc/xdr/src
 
 AM_CFLAGS = -Wall $(GF_CFLAGS)
 
diff --git a/xlators/debug/error-gen/src/error-gen-mem-types.h b/xlators/debug/error-gen/src/error-gen-mem-types.h
index f02280535df..b9b713af8fc 100644
--- a/xlators/debug/error-gen/src/error-gen-mem-types.h
+++ b/xlators/debug/error-gen/src/error-gen-mem-types.h
@@ -11,10 +11,10 @@
 #ifndef __ERROR_GEN_MEM_TYPES_H__
 #define __ERROR_GEN_MEM_TYPES_H__
 
-#include "mem-types.h"
+#include <glusterfs/mem-types.h>
 
 enum gf_error_gen_mem_types_ {
-        gf_error_gen_mt_eg_t = gf_common_mt_end + 1,
-        gf_error_gen_mt_end
+    gf_error_gen_mt_eg_t = gf_common_mt_end + 1,
+    gf_error_gen_mt_end
 };
 #endif
diff --git a/xlators/debug/error-gen/src/error-gen.c b/xlators/debug/error-gen/src/error-gen.c
index ec0874b354a..d45655ef4c3 100644
--- a/xlators/debug/error-gen/src/error-gen.c
+++ b/xlators/debug/error-gen/src/error-gen.c
@@ -7,2226 +7,1657 @@
    later), or the GNU General Public License, version 2 (GPLv2), in all
    cases as published by the Free Software Foundation.
 */
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
-
-#include "xlator.h"
+#include <glusterfs/xlator.h>
 #include "error-gen.h"
-#include "statedump.h"
-
-sys_error_t error_no_list[] = {
-        [GF_FOP_LOOKUP]            = { .error_no_count = 4,
-                                    .error_no = {ENOENT,ENOTDIR,
-                                                 ENAMETOOLONG,EAGAIN}},
-        [GF_FOP_STAT]              = { .error_no_count = 7,
-                                    .error_no = {EACCES,EBADF,EFAULT,
-                                                 ENAMETOOLONG,ENOENT,
-                                                 ENOMEM,ENOTDIR}},
-        [GF_FOP_READLINK]          = { .error_no_count = 8,
-                                    .error_no = {EACCES,EFAULT,EINVAL,EIO,
-                                                 ENAMETOOLONG,ENOENT,ENOMEM,
-                                                 ENOTDIR}},
-        [GF_FOP_MKNOD]             = { .error_no_count = 11,
-                                    .error_no = {EACCES,EEXIST,EFAULT,
-                                                 EINVAL,ENAMETOOLONG,
-                                                 ENOENT,ENOMEM,ENOSPC,
-                                                 ENOTDIR,EPERM,EROFS}},
-        [GF_FOP_MKDIR]             = { .error_no_count = 10,
-                                    .error_no = {EACCES,EEXIST,EFAULT,
-                                                 ENAMETOOLONG,ENOENT,
-                                                 ENOMEM,ENOSPC,ENOTDIR,
-						 EPERM,EROFS}},
-        [GF_FOP_UNLINK]            = { .error_no_count = 10,
-                                    .error_no = {EACCES,EBUSY,EFAULT,EIO,
-                                                 EISDIR,ENAMETOOLONG,
-                                                 ENOENT,ENOMEM,ENOTDIR,
-                                                 EPERM,EROFS}},
-        [GF_FOP_RMDIR]             = { .error_no_count = 8,
-                                    .error_no = {EACCES,EBUSY,EFAULT,
-                                                 ENOMEM,ENOTDIR,ENOTEMPTY,
-                                                 EPERM,EROFS}},
-        [GF_FOP_SYMLINK]           = { .error_no_count = 11,
-                                    .error_no = {EACCES,EEXIST,EFAULT,EIO,
-                                                 ENAMETOOLONG,ENOENT,ENOMEM,
-                                                 ENOSPC,ENOTDIR,EPERM,
-                                                 EROFS}},
-        [GF_FOP_RENAME]            = { .error_no_count = 13,
-                                    .error_no = {EACCES,EBUSY,EFAULT,
-                                                 EINVAL,EISDIR,EMLINK,
-                                                 ENAMETOOLONG,ENOENT,ENOMEM,
-                                                 ENOSPC,ENOTDIR,EEXIST,
-                                                 EXDEV}},
-        [GF_FOP_LINK]              = { .error_no_count = 13,
-                                    .error_no = {EACCES,EFAULT,EEXIST,EIO,
-                                                 EMLINK,ENAMETOOLONG,
-                                                 ENOENT,ENOMEM,ENOSPC,
-                                                 ENOTDIR,EPERM,EROFS,
-                                                 EXDEV}},
-        [GF_FOP_TRUNCATE]          = { .error_no_count = 10,
-                                    .error_no = {EACCES,EFAULT,EFBIG,
-                                                 EINTR,EINVAL,EIO,EISDIR,
-                                                 ENAMETOOLONG,ENOENT,
-                                                 EISDIR}},
-        [GF_FOP_CREATE]            = {.error_no_count = 10,
-                                   .error_no = {EACCES,EEXIST,EFAULT,
-                                                EISDIR,EMFILE,ENAMETOOLONG,
-                                                ENFILE,ENODEV,ENOENT,
-                                                ENODEV}},
-        [GF_FOP_OPEN]              = { .error_no_count = 10,
-                                    .error_no = {EACCES,EEXIST,EFAULT,
-                                                 EISDIR,EMFILE,
-                                                 ENAMETOOLONG,ENFILE,
-                                                 ENODEV,ENOENT,ENOMEM}},
-        [GF_FOP_READ]              = { .error_no_count = 5,
-                                    .error_no = {EINVAL,EBADF,EFAULT,EISDIR,
-                                                 ENAMETOOLONG}},
-        [GF_FOP_WRITE]             = { .error_no_count = 7,
-                                    .error_no = {EINVAL,EBADF,EFAULT,EISDIR,
-                                                 ENAMETOOLONG,ENOSPC,
-						 GF_ERROR_SHORT_WRITE}},
-        [GF_FOP_STATFS]            = {.error_no_count = 10,
-                                   .error_no = {EACCES,EBADF,EFAULT,EINTR,
-                                                EIO,ENAMETOOLONG,ENOENT,
-                                                ENOMEM,ENOSYS,ENOTDIR}},
-        [GF_FOP_FLUSH]             = { .error_no_count = 5,
-                                    .error_no = {EACCES,EFAULT,
-                                                 ENAMETOOLONG,ENOSYS,
-                                                 ENOENT}},
-        [GF_FOP_FSYNC]             = { .error_no_count = 4,
-                                    .error_no = {EBADF,EIO,EROFS,EINVAL}},
-        [GF_FOP_SETXATTR]          = { .error_no_count = 4,
-                                    .error_no = {EACCES,EBADF,EINTR,
-                                                 ENAMETOOLONG}},
-        [GF_FOP_GETXATTR]          = { .error_no_count = 4,
-                                    .error_no = {EACCES,EBADF,ENAMETOOLONG,
-                                                 EINTR}},
-        [GF_FOP_REMOVEXATTR]       = { .error_no_count = 4,
-                                    .error_no = {EACCES,EBADF,ENAMETOOLONG,
-                                                 EINTR}},
-        [GF_FOP_FSETXATTR]          = { .error_no_count = 4,
-                                        .error_no = {EACCES,EBADF,EINTR,
-                                                     ENAMETOOLONG}},
-        [GF_FOP_FGETXATTR]          = { .error_no_count = 4,
-                                        .error_no = {EACCES,EBADF,ENAMETOOLONG,
-                                                     EINTR}},
-        [GF_FOP_FREMOVEXATTR]       = { .error_no_count = 4,
-                                        .error_no = {EACCES,EBADF,ENAMETOOLONG,
-                                                     EINTR}},
-        [GF_FOP_OPENDIR]           = { .error_no_count = 8,
-                                    .error_no = {EACCES,EEXIST,EFAULT,
-                                                 EISDIR,EMFILE,
-                                                 ENAMETOOLONG,ENFILE,
-                                                 ENODEV}},
-        [GF_FOP_READDIR]           = { .error_no_count = 5,
-                                    .error_no = {EINVAL,EACCES,EBADF,
-                                                 EMFILE,ENOENT}},
-        [GF_FOP_READDIRP]          = { .error_no_count = 5,
-                                    .error_no = {EINVAL,EACCES,EBADF,
-                                                 EMFILE,ENOENT}},
-        [GF_FOP_FSYNCDIR]          = { .error_no_count = 4,
-                                    .error_no = {EBADF,EIO,EROFS,EINVAL}},
-        [GF_FOP_ACCESS]            = { .error_no_count = 8,
-                                    .error_no = {EACCES,ENAMETOOLONG,
-                                                 ENOENT,ENOTDIR,EROFS,
-                                                 EFAULT,EINVAL,EIO}},
-        [GF_FOP_FTRUNCATE]         = { .error_no_count = 9,
-                                    .error_no = {EACCES,EFAULT,EFBIG,
-                                                 EINTR,EINVAL,EIO,EISDIR,
-                                                 ENAMETOOLONG,ENOENT}},
-        [GF_FOP_FSTAT]             = { .error_no_count = 7,
-                                    .error_no = {EACCES,EBADF,EFAULT,
-                                                 ENAMETOOLONG,ENOENT,
-                                                 ENOMEM,ENOTDIR}},
-        [GF_FOP_LK]                = { .error_no_count = 4,
-                                    .error_no = {EACCES,EFAULT,ENOENT,
-                                                 EINTR}},
-        [GF_FOP_XATTROP]           = { .error_no_count = 5,
-                                    .error_no = {EACCES,EFAULT,
-                                                 ENAMETOOLONG,ENOSYS,
-                                                 ENOENT}},
-        [GF_FOP_FXATTROP]          = { .error_no_count = 4,
-                                    .error_no = {EBADF,EIO,EROFS,EINVAL}},
-        [GF_FOP_INODELK]           = { .error_no_count = 4,
-                                    .error_no = {EACCES,EBADF,EINTR,
-                                                 ENAMETOOLONG}},
-        [GF_FOP_FINODELK]          = { .error_no_count = 4,
-                                    .error_no = {EACCES,EBADF,EINTR,
-                                                 ENAMETOOLONG}},
-        [GF_FOP_ENTRYLK]           = { .error_no_count = 4,
-                                    .error_no = {EACCES,EBADF,
-                                                 ENAMETOOLONG,EINTR}},
-        [GF_FOP_FENTRYLK]          = { .error_no_count = 10,
-                                    .error_no = {EACCES,EEXIST,EFAULT,
-                                                 EISDIR,EMFILE,
-                                                 ENAMETOOLONG,ENFILE,
-                                                 ENODEV,ENOENT,ENOMEM}},
-        [GF_FOP_SETATTR]           = {.error_no_count = 11,
-                                    .error_no = {EACCES,EFAULT,EIO,
-                                                 ENAMETOOLONG,ENOENT,
-                                                 ENOMEM,ENOTDIR,EPERM,
-                                                 EROFS,EBADF,EIO}},
-        [GF_FOP_FSETATTR]          = { .error_no_count = 11,
-                                    .error_no = {EACCES,EFAULT,EIO,
-                                                 ENAMETOOLONG,ENOENT,
-                                                 ENOMEM,ENOTDIR,EPERM,
-                                                 EROFS,EBADF,EIO}},
-        [GF_FOP_GETSPEC]           = { .error_no_count = 4,
-                                    .error_no = {EACCES,EBADF,ENAMETOOLONG,
-                                                 EINTR}}
-};
-
-int
-generate_rand_no (int op_no)
-{
-        int             rand_no = 0;
-
-        if (op_no < GF_FOP_MAXVALUE)
-                rand_no = rand () % error_no_list[op_no].error_no_count;
-        return rand_no;
-}
+#include <glusterfs/statedump.h>
+#include <glusterfs/defaults.h>
 
-int
-conv_errno_to_int (char **error_no)
-{
-        if (!strcmp ((*error_no), "ENOENT"))
-                return ENOENT;
-        else if (!strcmp ((*error_no), "ENOTDIR"))
-                return ENOTDIR;
-        else if (!strcmp ((*error_no), "ENAMETOOLONG"))
-                return ENAMETOOLONG;
-        else if (!strcmp ((*error_no), "EACCES"))
-                return EACCES;
-        else if (!strcmp ((*error_no), "EBADF"))
-                return EBADF;
-        else if (!strcmp ((*error_no), "EFAULT"))
-                return EFAULT;
-        else if (!strcmp ((*error_no), "ENOMEM"))
-                return ENOMEM;
-        else if (!strcmp ((*error_no), "EINVAL"))
-                return EINVAL;
-        else if (!strcmp ((*error_no), "EIO"))
-                return EIO;
-        else if (!strcmp ((*error_no), "EEXIST"))
-                return EEXIST;
-        else if (!strcmp ((*error_no), "ENOSPC"))
-                return ENOSPC;
-        else if (!strcmp ((*error_no), "EPERM"))
-                return EPERM;
-        else if (!strcmp ((*error_no), "EROFS"))
-                return EROFS;
-        else if (!strcmp ((*error_no), "EBUSY"))
-                return EBUSY;
-        else if (!strcmp ((*error_no), "EISDIR"))
-                return EISDIR;
-        else if (!strcmp ((*error_no), "ENOTEMPTY"))
-                return ENOTEMPTY;
-        else if (!strcmp ((*error_no), "EMLINK"))
-                return EMLINK;
-        else if (!strcmp ((*error_no), "ENODEV"))
-                return ENODEV;
-        else if (!strcmp ((*error_no), "EXDEV"))
-                return EXDEV;
-        else if (!strcmp ((*error_no), "EMFILE"))
-                return EMFILE;
-        else if (!strcmp ((*error_no), "ENFILE"))
-                return ENFILE;
-        else if (!strcmp ((*error_no), "ENOSYS"))
-                return ENOSYS;
-        else if (!strcmp ((*error_no), "EINTR"))
-                return EINTR;
-        else if (!strcmp ((*error_no), "EFBIG"))
-                return EFBIG;
-	else if (!strcmp((*error_no), "GF_ERROR_SHORT_WRITE"))
-		return GF_ERROR_SHORT_WRITE;
-        else
-                return EAGAIN;
-}
-
-int
-get_fop_int (char **op_no_str)
-{
-        if (!strcmp ((*op_no_str), "lookup"))
-                return GF_FOP_LOOKUP;
-        else if (!strcmp ((*op_no_str), "stat"))
-                return GF_FOP_STAT;
-        else if (!strcmp ((*op_no_str), "readlink"))
-                return GF_FOP_READLINK;
-        else if (!strcmp ((*op_no_str), "mknod"))
-                return GF_FOP_MKNOD;
-        else if (!strcmp ((*op_no_str), "mkdir"))
-                return GF_FOP_MKDIR;
-        else if (!strcmp ((*op_no_str), "unlink"))
-                return GF_FOP_UNLINK;
-        else if (!strcmp ((*op_no_str), "rmdir"))
-                return GF_FOP_RMDIR;
-        else if (!strcmp ((*op_no_str), "symlink"))
-                return GF_FOP_SYMLINK;
-        else if (!strcmp ((*op_no_str), "rename"))
-                return GF_FOP_RENAME;
-        else if (!strcmp ((*op_no_str), "link"))
-                return GF_FOP_LINK;
-        else if (!strcmp ((*op_no_str), "truncate"))
-                return GF_FOP_TRUNCATE;
-        else if (!strcmp ((*op_no_str), "create"))
-                return GF_FOP_CREATE;
-        else if (!strcmp ((*op_no_str), "open"))
-                return GF_FOP_OPEN;
-        else if (!strcmp ((*op_no_str), "readv"))
-                return GF_FOP_READ;
-        else if (!strcmp ((*op_no_str), "writev"))
-                return GF_FOP_WRITE;
-        else if (!strcmp ((*op_no_str), "statfs"))
-                return GF_FOP_STATFS;
-        else if (!strcmp ((*op_no_str), "flush"))
-                return GF_FOP_FLUSH;
-        else if (!strcmp ((*op_no_str), "fsync"))
-                return GF_FOP_FSYNC;
-        else if (!strcmp ((*op_no_str), "setxattr"))
-                return GF_FOP_SETXATTR;
-        else if (!strcmp ((*op_no_str), "getxattr"))
-                return GF_FOP_GETXATTR;
-        else if (!strcmp ((*op_no_str), "removexattr"))
-                return GF_FOP_REMOVEXATTR;
-        else if (!strcmp ((*op_no_str), "fsetxattr"))
-                return GF_FOP_FSETXATTR;
-        else if (!strcmp ((*op_no_str), "fgetxattr"))
-                return GF_FOP_FGETXATTR;
-        else if (!strcmp ((*op_no_str), "fremovexattr"))
-                return GF_FOP_FREMOVEXATTR;
-        else if (!strcmp ((*op_no_str), "opendir"))
-                return GF_FOP_OPENDIR;
-        else if (!strcmp ((*op_no_str), "readdir"))
-                return GF_FOP_READDIR;
-        else if (!strcmp ((*op_no_str), "readdirp"))
-                return GF_FOP_READDIRP;
-        else if (!strcmp ((*op_no_str), "fsyncdir"))
-                return GF_FOP_FSYNCDIR;
-        else if (!strcmp ((*op_no_str), "access"))
-                return GF_FOP_ACCESS;
-        else if (!strcmp ((*op_no_str), "ftruncate"))
-                return GF_FOP_FTRUNCATE;
-        else if (!strcmp ((*op_no_str), "fstat"))
-                return GF_FOP_FSTAT;
-        else if (!strcmp ((*op_no_str), "lk"))
-                return GF_FOP_LK;
-        else if (!strcmp ((*op_no_str), "xattrop"))
-                return GF_FOP_XATTROP;
-        else if (!strcmp ((*op_no_str), "fxattrop"))
-                return GF_FOP_FXATTROP;
-        else if (!strcmp ((*op_no_str), "inodelk"))
-                return GF_FOP_INODELK;
-        else if (!strcmp ((*op_no_str), "finodelk"))
-                return GF_FOP_FINODELK;
-        else if (!strcmp ((*op_no_str), "etrylk"))
-                return GF_FOP_ENTRYLK;
-        else if (!strcmp ((*op_no_str), "fentrylk"))
-                return GF_FOP_FENTRYLK;
-        else if (!strcmp ((*op_no_str), "setattr"))
-                return GF_FOP_SETATTR;
-        else if (!strcmp ((*op_no_str), "fsetattr"))
-                return GF_FOP_FSETATTR;
-        else if (!strcmp ((*op_no_str), "getspec"))
-                return GF_FOP_GETSPEC;
-	else
-                return -1;
-}
-
-int
-error_gen (xlator_t *this, int op_no)
-{
-        eg_t             *egp = NULL;
-        int              count = 0;
-        int              failure_iter_no = GF_FAILURE_DEFAULT;
-        char             *error_no = NULL;
-        int              rand_no = 0;
-        int              ret = 0;
-
-        egp = this->private;
+/*
+ * The user can specify an error probability as a float percentage, but we
+ * store it internally as a numerator with this as the denominator.  When it's
+ * used, it's like this:
+ *
+ *    (rand() % FAILURE_GRANULARITY) < error_rate
+ *
+ * To minimize rounding errors from the modulo operation, it's good for this to
+ * be a power of two.
+ *
+ * (BTW this is just the normal case.  If "random-failure" is set, that does
+ * something completely different and this number is irrelevant.  See error_gen
+ * for the legacy code.)
+ */
+#define FAILURE_GRANULARITY (1 << 20)
 
-        LOCK (&egp->lock);
+sys_error_t error_no_list[] = {
+    [GF_FOP_LOOKUP] = {.error_no_count = 4,
+                       .error_no = {ENOENT, ENOTDIR, ENAMETOOLONG, EAGAIN}},
+    [GF_FOP_STAT] = {.error_no_count = 6,
+                     .error_no = {EACCES, EFAULT, ENAMETOOLONG, ENOENT, ENOMEM,
+                                  ENOTDIR}},
+    [GF_FOP_READLINK] = {.error_no_count = 8,
+                         .error_no = {EACCES, EFAULT, EINVAL, EIO, ENAMETOOLONG,
+                                      ENOENT, ENOMEM, ENOTDIR}},
+    [GF_FOP_MKNOD] = {.error_no_count = 11,
+                      .error_no = {EACCES, EEXIST, EFAULT, EINVAL, ENAMETOOLONG,
+                                   ENOENT, ENOMEM, ENOSPC, ENOTDIR, EPERM,
+                                   EROFS}},
+    [GF_FOP_MKDIR] = {.error_no_count = 10,
+                      .error_no = {EACCES, EEXIST, EFAULT, ENAMETOOLONG, ENOENT,
+                                   ENOMEM, ENOSPC, ENOTDIR, EPERM, EROFS}},
+    [GF_FOP_UNLINK] = {.error_no_count = 10,
+                       .error_no = {EACCES, EBUSY, EFAULT, EIO, EISDIR,
+                                    ENAMETOOLONG, ENOENT, ENOMEM, ENOTDIR,
+                                    EPERM, EROFS}},
+    [GF_FOP_RMDIR] = {.error_no_count = 8,
+                      .error_no = {EACCES, EBUSY, EFAULT, ENOMEM, ENOTDIR,
+                                   ENOTEMPTY, EPERM, EROFS}},
+    [GF_FOP_SYMLINK] = {.error_no_count = 11,
+                        .error_no = {EACCES, EEXIST, EFAULT, EIO, ENAMETOOLONG,
+                                     ENOENT, ENOMEM, ENOSPC, ENOTDIR, EPERM,
+                                     EROFS}},
+    [GF_FOP_RENAME] = {.error_no_count = 13,
+                       .error_no = {EACCES, EBUSY, EFAULT, EINVAL, EISDIR,
+                                    EMLINK, ENAMETOOLONG, ENOENT, ENOMEM,
+                                    ENOSPC, ENOTDIR, EEXIST, EXDEV}},
+    [GF_FOP_LINK] = {.error_no_count = 13,
+                     .error_no = {EACCES, EFAULT, EEXIST, EIO, EMLINK,
+                                  ENAMETOOLONG, ENOENT, ENOMEM, ENOSPC, ENOTDIR,
+                                  EPERM, EROFS, EXDEV}},
+    [GF_FOP_TRUNCATE] = {.error_no_count = 10,
+                         .error_no = {EACCES, EFAULT, EFBIG, EINTR, EINVAL, EIO,
+                                      EISDIR, ENAMETOOLONG, ENOENT, EISDIR}},
+    [GF_FOP_CREATE] = {.error_no_count = 10,
+                       .error_no = {EACCES, EEXIST, EFAULT, EISDIR, EMFILE,
+                                    ENAMETOOLONG, ENFILE, ENODEV, ENOENT,
+                                    ENODEV}},
+    [GF_FOP_OPEN] = {.error_no_count = 10,
+                     .error_no = {EACCES, EEXIST, EFAULT, EISDIR, EMFILE,
+                                  ENAMETOOLONG, ENFILE, ENODEV, ENOENT,
+                                  ENOMEM}},
+    [GF_FOP_READ] = {.error_no_count = 5,
+                     .error_no = {EINVAL, EBADF, EFAULT, EISDIR, ENAMETOOLONG}},
+    [GF_FOP_WRITE] = {.error_no_count = 7,
+                      .error_no = {EINVAL, EBADF, EFAULT, EISDIR, ENAMETOOLONG,
+                                   ENOSPC, GF_ERROR_SHORT_WRITE}},
+    [GF_FOP_STATFS] = {.error_no_count = 9,
+                       .error_no = {EACCES, EFAULT, EINTR, EIO, ENAMETOOLONG,
+                                    ENOENT, ENOMEM, ENOSYS, ENOTDIR}},
+    [GF_FOP_FLUSH] = {.error_no_count = 5,
+                      .error_no = {EACCES, EFAULT, ENAMETOOLONG, ENOSYS,
+                                   ENOENT}},
+    [GF_FOP_FSYNC] = {.error_no_count = 4,
+                      .error_no = {EBADF, EIO, EROFS, EINVAL}},
+    [GF_FOP_SETXATTR] = {.error_no_count = 3,
+                         .error_no = {EACCES, EINTR, ENAMETOOLONG}},
+    [GF_FOP_GETXATTR] = {.error_no_count = 3,
+                         .error_no = {EACCES, ENAMETOOLONG, EINTR}},
+    [GF_FOP_REMOVEXATTR] = {.error_no_count = 3,
+                            .error_no = {EACCES, ENAMETOOLONG, EINTR}},
+    [GF_FOP_FSETXATTR] = {.error_no_count = 4,
+                          .error_no = {EACCES, EBADF, EINTR, ENAMETOOLONG}},
+    [GF_FOP_FGETXATTR] = {.error_no_count = 4,
+                          .error_no = {EACCES, EBADF, ENAMETOOLONG, EINTR}},
+    [GF_FOP_FREMOVEXATTR] = {.error_no_count = 4,
+                             .error_no = {EACCES, EBADF, ENAMETOOLONG, EINTR}},
+    [GF_FOP_OPENDIR] = {.error_no_count = 8,
+                        .error_no = {EACCES, EEXIST, EFAULT, EISDIR, EMFILE,
+                                     ENAMETOOLONG, ENFILE, ENODEV}},
+    [GF_FOP_READDIR] = {.error_no_count = 5,
+                        .error_no = {EINVAL, EACCES, EBADF, EMFILE, ENOENT}},
+    [GF_FOP_READDIRP] = {.error_no_count = 5,
+                         .error_no = {EINVAL, EACCES, EBADF, EMFILE, ENOENT}},
+    [GF_FOP_FSYNCDIR] = {.error_no_count = 4,
+                         .error_no = {EBADF, EIO, EROFS, EINVAL}},
+    [GF_FOP_ACCESS] = {.error_no_count = 8,
+                       .error_no = {EACCES, ENAMETOOLONG, ENOENT, ENOTDIR,
+                                    EROFS, EFAULT, EINVAL, EIO}},
+    [GF_FOP_FTRUNCATE] = {.error_no_count = 9,
+                          .error_no = {EACCES, EFAULT, EFBIG, EINTR, EINVAL,
+                                       EIO, EISDIR, ENAMETOOLONG, ENOENT}},
+    [GF_FOP_FSTAT] = {.error_no_count = 7,
+                      .error_no = {EACCES, EBADF, EFAULT, ENAMETOOLONG, ENOENT,
+                                   ENOMEM, ENOTDIR}},
+    [GF_FOP_LK] = {.error_no_count = 4,
+                   .error_no = {EACCES, EFAULT, ENOENT, EINTR}},
+    [GF_FOP_XATTROP] = {.error_no_count = 5,
+                        .error_no = {EACCES, EFAULT, ENAMETOOLONG, ENOSYS,
+                                     ENOENT}},
+    [GF_FOP_FXATTROP] = {.error_no_count = 4,
+                         .error_no = {EBADF, EIO, EROFS, EINVAL}},
+    [GF_FOP_INODELK] = {.error_no_count = 3,
+                        .error_no = {EACCES, EINTR, ENAMETOOLONG}},
+    [GF_FOP_FINODELK] = {.error_no_count = 4,
+                         .error_no = {EACCES, EBADF, EINTR, ENAMETOOLONG}},
+    [GF_FOP_ENTRYLK] = {.error_no_count = 3,
+                        .error_no = {EACCES, ENAMETOOLONG, EINTR}},
+    [GF_FOP_FENTRYLK] = {.error_no_count = 10,
+                         .error_no = {EACCES, EEXIST, EFAULT, EISDIR, EMFILE,
+                                      ENAMETOOLONG, ENFILE, ENODEV, ENOENT,
+                                      ENOMEM}},
+    [GF_FOP_SETATTR] = {.error_no_count = 10,
+                        .error_no = {EACCES, EFAULT, EIO, ENAMETOOLONG, ENOENT,
+                                     ENOMEM, ENOTDIR, EPERM, EROFS, EIO}},
+    [GF_FOP_FSETATTR] = {.error_no_count = 11,
+                         .error_no = {EACCES, EFAULT, EIO, ENAMETOOLONG, ENOENT,
+                                      ENOMEM, ENOTDIR, EPERM, EROFS, EBADF,
+                                      EIO}},
+    [GF_FOP_GETSPEC] = {.error_no_count = 3,
+                        .error_no = {EACCES, ENAMETOOLONG, EINTR}}};
+
+int
+generate_rand_no(int op_no)
+{
+    int rand_no = 0;
+    int error_no_list_size = 0;
+
+    error_no_list_size = sizeof(error_no_list) / sizeof(error_no_list[0]);
+
+    if (op_no < error_no_list_size)
+        /* coverity[DC.WEAK_CRYPTO] */
+        rand_no = rand() % error_no_list[op_no].error_no_count;
+    return rand_no;
+}
+
+int
+conv_errno_to_int(char **error_no)
+{
+    if (!strcmp((*error_no), "ENOENT"))
+        return ENOENT;
+    else if (!strcmp((*error_no), "ENOTDIR"))
+        return ENOTDIR;
+    else if (!strcmp((*error_no), "ENAMETOOLONG"))
+        return ENAMETOOLONG;
+    else if (!strcmp((*error_no), "EACCES"))
+        return EACCES;
+    else if (!strcmp((*error_no), "EBADF"))
+        return EBADF;
+    else if (!strcmp((*error_no), "EFAULT"))
+        return EFAULT;
+    else if (!strcmp((*error_no), "ENOMEM"))
+        return ENOMEM;
+    else if (!strcmp((*error_no), "EINVAL"))
+        return EINVAL;
+    else if (!strcmp((*error_no), "EIO"))
+        return EIO;
+    else if (!strcmp((*error_no), "EEXIST"))
+        return EEXIST;
+    else if (!strcmp((*error_no), "ENOSPC"))
+        return ENOSPC;
+    else if (!strcmp((*error_no), "EPERM"))
+        return EPERM;
+    else if (!strcmp((*error_no), "EROFS"))
+        return EROFS;
+    else if (!strcmp((*error_no), "EBUSY"))
+        return EBUSY;
+    else if (!strcmp((*error_no), "EISDIR"))
+        return EISDIR;
+    else if (!strcmp((*error_no), "ENOTEMPTY"))
+        return ENOTEMPTY;
+    else if (!strcmp((*error_no), "EMLINK"))
+        return EMLINK;
+    else if (!strcmp((*error_no), "ENODEV"))
+        return ENODEV;
+    else if (!strcmp((*error_no), "EXDEV"))
+        return EXDEV;
+    else if (!strcmp((*error_no), "EMFILE"))
+        return EMFILE;
+    else if (!strcmp((*error_no), "ENFILE"))
+        return ENFILE;
+    else if (!strcmp((*error_no), "ENOSYS"))
+        return ENOSYS;
+    else if (!strcmp((*error_no), "EINTR"))
+        return EINTR;
+    else if (!strcmp((*error_no), "EFBIG"))
+        return EFBIG;
+    else if (!strcmp((*error_no), "GF_ERROR_SHORT_WRITE"))
+        return GF_ERROR_SHORT_WRITE;
+    else
+        return EAGAIN;
+}
+
+int
+error_gen(xlator_t *this, int op_no)
+{
+    eg_t *egp = NULL;
+    int count = 0;
+    int error_no_int = 0;
+    int rand_no = 0;
+    int ret = 0;
+    gf_boolean_t should_err = _gf_false;
+    int error_no_list_size = 0;
+
+    egp = this->private;
+
+    if (egp->random_failure) {
+        /*
+         * I honestly don't know why anyone would use this "feature"
+         * but I'll try to preserve its functionality anyway.  Without
+         * locking twice to update failure_iter_no and egp->op_count
+         * separately, then not locking at all to update
+         * egp->failure_iter_no.  That's not needed for compatibility,
+         * and it's abhorrently wrong.  I have *some* standards.
+         */
+        LOCK(&egp->lock);
         {
-                count = ++egp->op_count;
-                failure_iter_no = egp->failure_iter_no;
-                error_no = egp->error_no;
+            count = ++(egp->op_count);
+            error_no_int = egp->error_no_int;
+            if ((count % egp->failure_iter_no) == 0) {
+                egp->op_count = 0;
+                /* coverity[DC.WEAK_CRYPTO] */
+                egp->failure_iter_no = 3 + (rand() % GF_UNIVERSAL_ANSWER);
+                should_err = _gf_true;
+            }
         }
-        UNLOCK (&egp->lock);
-
-        if((count % failure_iter_no) == 0) {
-                LOCK (&egp->lock);
-                {
-                        egp->op_count = 0;
-                }
-                UNLOCK (&egp->lock);
-
-                if (error_no)
-                        ret = conv_errno_to_int (&error_no);
-                else {
-
-                        rand_no = generate_rand_no (op_no);
-                        if (op_no >= GF_FOP_MAXVALUE)
-                                op_no = 0;
-                        if (rand_no >= error_no_list[op_no].error_no_count)
-                                rand_no = 0;
-                        ret = error_no_list[op_no].error_no[rand_no];
-                }
-                if (egp->random_failure == _gf_true)
-                        egp->failure_iter_no = 3 + (rand () % GF_UNIVERSAL_ANSWER);
+        UNLOCK(&egp->lock);
+    } else {
+        /*
+         * It turns out that rand() is almost universally implemented
+         * as a linear congruential PRNG, which is about as cheap as
+         * it gets.  This gets us real random behavior, including
+         * phenomena like streaks and dry spells, with controllable
+         * long-term probability, cheaply.
+         */
+        if ((rand() % FAILURE_GRANULARITY) < egp->failure_iter_no) {
+            should_err = _gf_true;
         }
-        return ret;
-}
-
+    }
+
+    error_no_list_size = sizeof(error_no_list) / sizeof(error_no_list[0]);
+    if (should_err) {
+        if (error_no_int)
+            ret = error_no_int;
+        else {
+            rand_no = generate_rand_no(op_no);
+            if (op_no >= error_no_list_size)
+                op_no = 0;
+            if (rand_no >= error_no_list[op_no].error_no_count)
+                rand_no = 0;
+            ret = error_no_list[op_no].error_no[rand_no];
+        }
+    }
 
-int
-error_gen_lookup_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-		      int32_t op_ret, int32_t op_errno, inode_t *inode,
-		      struct iatt *buf, dict_t *xdata, struct iatt *postparent)
-{
-	STACK_UNWIND_STRICT (lookup, frame, op_ret, op_errno, inode,
-                             buf, xdata, postparent);
-        return 0;
+    return ret;
 }
 
-
 int
-error_gen_lookup (call_frame_t *frame, xlator_t *this, loc_t *loc,
-                  dict_t *xdata)
+error_gen_lookup(call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xdata)
 {
-	int              op_errno = 0;
-        eg_t            *egp = NULL;
-        int              enable = 1;
+    int op_errno = 0;
+    eg_t *egp = NULL;
+    int enable = 1;
 
-        egp = this->private;
-        enable = egp->enable[GF_FOP_LOOKUP];
+    egp = this->private;
+    enable = egp->enable[GF_FOP_LOOKUP];
 
-        if (enable)
-                op_errno = error_gen (this, GF_FOP_LOOKUP);
+    if (enable)
+        op_errno = error_gen(this, GF_FOP_LOOKUP);
 
-	if (op_errno) {
-		GF_ERROR(this, "unwind(-1, %s)", strerror (op_errno));
-		STACK_UNWIND_STRICT (lookup, frame, -1, op_errno, NULL, NULL, NULL,
-                                     NULL);
+    if (op_errno) {
+        GF_ERROR(this, "unwind(-1, %s)", strerror(op_errno));
+        STACK_UNWIND_STRICT(lookup, frame, -1, op_errno, NULL, NULL, NULL,
+                            NULL);
         return 0;
-	}
-
-	STACK_WIND (frame, error_gen_lookup_cbk,
-		    FIRST_CHILD(this),
-		    FIRST_CHILD(this)->fops->lookup,
-		    loc, xdata);
-        return 0;
-}
+    }
 
-
-int
-error_gen_stat_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-		    int32_t op_ret, int32_t op_errno, struct iatt *buf, dict_t *xdata)
-{
-	STACK_UNWIND_STRICT (stat, frame, op_ret, op_errno, buf, xdata);
-        return 0;
+    STACK_WIND_TAIL(frame, FIRST_CHILD(this), FIRST_CHILD(this)->fops->lookup,
+                    loc, xdata);
+    return 0;
 }
 
 int
-error_gen_stat (call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xdata)
+error_gen_stat(call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xdata)
 {
-	int              op_errno = 0;
-        eg_t            *egp = NULL;
-        int              enable = 1;
+    int op_errno = 0;
+    eg_t *egp = NULL;
+    int enable = 1;
 
-        egp = this->private;
-        enable = egp->enable[GF_FOP_STAT];
+    egp = this->private;
+    enable = egp->enable[GF_FOP_STAT];
 
-        if (enable)
-                op_errno = error_gen (this, GF_FOP_STAT);
+    if (enable)
+        op_errno = error_gen(this, GF_FOP_STAT);
 
-	if (op_errno) {
-		GF_ERROR(this, "unwind(-1, %s)", strerror (op_errno));
-		STACK_UNWIND_STRICT (stat, frame, -1, op_errno, NULL, xdata);
+    if (op_errno) {
+        GF_ERROR(this, "unwind(-1, %s)", strerror(op_errno));
+        STACK_UNWIND_STRICT(stat, frame, -1, op_errno, NULL, xdata);
         return 0;
-	}
+    }
 
-	STACK_WIND (frame, error_gen_stat_cbk,
-		    FIRST_CHILD(this),
-		    FIRST_CHILD(this)->fops->stat,
-		    loc, xdata);
-        return 0;
+    STACK_WIND_TAIL(frame, FIRST_CHILD(this), FIRST_CHILD(this)->fops->stat,
+                    loc, xdata);
+    return 0;
 }
 
-
 int
-error_gen_setattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                       int32_t op_ret, int32_t op_errno,
-                       struct iatt *preop, struct iatt *postop, dict_t *xdata)
+error_gen_setattr(call_frame_t *frame, xlator_t *this, loc_t *loc,
+                  struct iatt *stbuf, int32_t valid, dict_t *xdata)
 {
-	STACK_UNWIND_STRICT (setattr, frame, op_ret, op_errno, preop, postop, xdata);
+    int op_errno = 0;
+    eg_t *egp = NULL;
+    int enable = 1;
+
+    egp = this->private;
+    enable = egp->enable[GF_FOP_SETATTR];
+
+    if (enable)
+        op_errno = error_gen(this, GF_FOP_SETATTR);
+
+    if (op_errno) {
+        GF_ERROR(this, "unwind(-1, %s)", strerror(op_errno));
+        STACK_UNWIND_STRICT(setattr, frame, -1, op_errno, NULL, NULL, xdata);
         return 0;
-}
+    }
 
+    STACK_WIND_TAIL(frame, FIRST_CHILD(this), FIRST_CHILD(this)->fops->setattr,
+                    loc, stbuf, valid, xdata);
+    return 0;
+}
 
 int
-error_gen_setattr (call_frame_t *frame, xlator_t *this, loc_t *loc,
+error_gen_fsetattr(call_frame_t *frame, xlator_t *this, fd_t *fd,
                    struct iatt *stbuf, int32_t valid, dict_t *xdata)
 {
-	int              op_errno = 0;
-        eg_t            *egp = NULL;
-        int              enable = 1;
+    int op_errno = 0;
+    eg_t *egp = NULL;
+    int enable = 1;
 
-        egp = this->private;
-        enable = egp->enable[GF_FOP_SETATTR];
+    egp = this->private;
+    enable = egp->enable[GF_FOP_FSETATTR];
 
-        if (enable)
-                op_errno = error_gen (this, GF_FOP_SETATTR);
+    if (enable)
+        op_errno = error_gen(this, GF_FOP_FSETATTR);
 
-	if (op_errno) {
-		GF_ERROR(this, "unwind(-1, %s)", strerror (op_errno));
-		STACK_UNWIND_STRICT (setattr, frame, -1, op_errno, NULL, NULL, xdata);
+    if (op_errno) {
+        GF_ERROR(this, "unwind(-1, %s)", strerror(op_errno));
+        STACK_UNWIND_STRICT(fsetattr, frame, -1, op_errno, NULL, NULL, xdata);
         return 0;
-	}
+    }
 
-	STACK_WIND (frame, error_gen_setattr_cbk,
-		    FIRST_CHILD(this),
-		    FIRST_CHILD(this)->fops->setattr,
-		    loc, stbuf, valid, xdata);
-        return 0;
+    STACK_WIND_TAIL(frame, FIRST_CHILD(this), FIRST_CHILD(this)->fops->fsetattr,
+                    fd, stbuf, valid, xdata);
+    return 0;
 }
 
-
 int
-error_gen_fsetattr (call_frame_t *frame, xlator_t *this, fd_t *fd,
-                    struct iatt *stbuf, int32_t valid, dict_t *xdata)
+error_gen_truncate(call_frame_t *frame, xlator_t *this, loc_t *loc,
+                   off_t offset, dict_t *xdata)
 {
-	int             op_errno = 0;
-        eg_t            *egp = NULL;
-        int             enable = 1;
+    int op_errno = 0;
+    eg_t *egp = NULL;
+    int enable = 1;
 
-        egp = this->private;
-        enable = egp->enable[GF_FOP_FSETATTR];
+    egp = this->private;
+    enable = egp->enable[GF_FOP_TRUNCATE];
 
-        if (enable)
-                op_errno = error_gen (this, GF_FOP_FSETATTR);
+    if (enable)
+        op_errno = error_gen(this, GF_FOP_TRUNCATE);
 
-	if (op_errno) {
-		GF_ERROR(this, "unwind(-1, %s)", strerror (op_errno));
-		STACK_UNWIND_STRICT (fsetattr, frame, -1, op_errno, NULL, NULL, xdata);
+    if (op_errno) {
+        GF_ERROR(this, "unwind(-1, %s)", strerror(op_errno));
+        STACK_UNWIND_STRICT(truncate, frame, -1, op_errno, NULL, NULL, xdata);
         return 0;
-	}
+    }
 
-	STACK_WIND (frame, error_gen_setattr_cbk,
-		    FIRST_CHILD(this),
-		    FIRST_CHILD(this)->fops->fsetattr,
-		    fd, stbuf, valid, xdata);
-        return 0;
+    STACK_WIND_TAIL(frame, FIRST_CHILD(this), FIRST_CHILD(this)->fops->truncate,
+                    loc, offset, xdata);
+    return 0;
 }
 
-
 int
-error_gen_truncate_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-			int32_t op_ret, int32_t op_errno,
-			struct iatt *prebuf, struct iatt *postbuf, dict_t *xdata)
+error_gen_ftruncate(call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
+                    dict_t *xdata)
 {
-	STACK_UNWIND_STRICT (truncate, frame, op_ret, op_errno,
-                             prebuf, postbuf, xdata);
-        return 0;
-}
-
-
-int
-error_gen_truncate (call_frame_t *frame, xlator_t *this, loc_t *loc,
-                    off_t offset, dict_t *xdata)
-{
-	int              op_errno = 0;
-        eg_t            *egp = NULL;
-        int              enable = 1;
-
-        egp = this->private;
-        enable = egp->enable[GF_FOP_TRUNCATE];
+    int op_errno = 0;
+    eg_t *egp = NULL;
+    int enable = 1;
 
-        if (enable)
-                op_errno = error_gen (this, GF_FOP_TRUNCATE);
+    egp = this->private;
+    enable = egp->enable[GF_FOP_FTRUNCATE];
 
-	if (op_errno) {
-		GF_ERROR(this, "unwind(-1, %s)", strerror (op_errno));
-		STACK_UNWIND_STRICT (truncate, frame, -1, op_errno,
-                                     NULL, NULL, xdata);
-        return 0;
-	}
+    if (enable)
+        op_errno = error_gen(this, GF_FOP_FTRUNCATE);
 
-	STACK_WIND (frame, error_gen_truncate_cbk,
-		    FIRST_CHILD(this),
-		    FIRST_CHILD(this)->fops->truncate,
-		    loc, offset, xdata);
+    if (op_errno) {
+        GF_ERROR(this, "unwind(-1, %s)", strerror(op_errno));
+        STACK_UNWIND_STRICT(ftruncate, frame, -1, op_errno, NULL, NULL, xdata);
         return 0;
-}
-
+    }
 
-int
-error_gen_ftruncate_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-			 int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
-                         struct iatt *postbuf, dict_t *xdata)
-{
-	STACK_UNWIND_STRICT (ftruncate, frame, op_ret, op_errno,
-                             prebuf, postbuf, xdata);
-        return 0;
+    STACK_WIND_TAIL(frame, FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->ftruncate, fd, offset, xdata);
+    return 0;
 }
 
-
 int
-error_gen_ftruncate (call_frame_t *frame, xlator_t *this, fd_t *fd,
-		     off_t offset, dict_t *xdata)
+error_gen_access(call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t mask,
+                 dict_t *xdata)
 {
-	int              op_errno = 0;
-        eg_t            *egp =NULL;
-        int              enable = 1;
+    int op_errno = 0;
+    eg_t *egp = NULL;
+    int enable = 1;
 
-        egp = this->private;
-        enable = egp->enable[GF_FOP_FTRUNCATE];
+    egp = this->private;
+    enable = egp->enable[GF_FOP_ACCESS];
 
-        if (enable)
-                op_errno = error_gen (this, GF_FOP_FTRUNCATE);
+    if (enable)
+        op_errno = error_gen(this, GF_FOP_ACCESS);
 
-	if (op_errno) {
-		GF_ERROR(this, "unwind(-1, %s)", strerror (op_errno));
-		STACK_UNWIND_STRICT (ftruncate, frame, -1, op_errno,
-                                     NULL, NULL, xdata);
+    if (op_errno) {
+        GF_ERROR(this, "unwind(-1, %s)", strerror(op_errno));
+        STACK_UNWIND_STRICT(access, frame, -1, op_errno, xdata);
         return 0;
-	}
-
-	STACK_WIND (frame, error_gen_ftruncate_cbk,
-		    FIRST_CHILD(this),
-		    FIRST_CHILD(this)->fops->ftruncate,
-		    fd, offset, xdata);
-        return 0;
-}
+    }
 
-
-int
-error_gen_access_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-		      int32_t op_ret, int32_t op_errno, dict_t *xdata)
-{
-	STACK_UNWIND_STRICT (access, frame, op_ret, op_errno, xdata);
-        return 0;
+    STACK_WIND_TAIL(frame, FIRST_CHILD(this), FIRST_CHILD(this)->fops->access,
+                    loc, mask, xdata);
+    return 0;
 }
 
-
 int
-error_gen_access (call_frame_t *frame, xlator_t *this, loc_t *loc,
-		  int32_t mask, dict_t *xdata)
+error_gen_readlink(call_frame_t *frame, xlator_t *this, loc_t *loc, size_t size,
+                   dict_t *xdata)
 {
-	int              op_errno = 0;
-        eg_t            *egp = NULL;
-        int              enable = 1;
+    int op_errno = 0;
+    eg_t *egp = NULL;
+    int enable = 1;
 
-        egp = this->private;
-        enable = egp->enable[GF_FOP_ACCESS];
+    egp = this->private;
+    enable = egp->enable[GF_FOP_READLINK];
 
-        if (enable)
-                op_errno = error_gen (this, GF_FOP_ACCESS);
+    if (enable)
+        op_errno = error_gen(this, GF_FOP_READLINK);
 
-	if (op_errno) {
-		GF_ERROR(this, "unwind(-1, %s)", strerror (op_errno));
-		STACK_UNWIND_STRICT (access, frame, -1, op_errno, xdata);
+    if (op_errno) {
+        GF_ERROR(this, "unwind(-1, %s)", strerror(op_errno));
+        STACK_UNWIND_STRICT(readlink, frame, -1, op_errno, NULL, NULL, xdata);
         return 0;
-	}
+    }
 
-	STACK_WIND (frame, error_gen_access_cbk,
-		    FIRST_CHILD(this),
-		    FIRST_CHILD(this)->fops->access,
-		    loc, mask, xdata);
-        return 0;
-}
-
-
-int
-error_gen_readlink_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-			int32_t op_ret, int32_t op_errno,
-			const char *path, struct iatt *sbuf, dict_t *xdata)
-{
-	STACK_UNWIND_STRICT (readlink, frame, op_ret, op_errno, path, sbuf, xdata);
-        return 0;
+    STACK_WIND_TAIL(frame, FIRST_CHILD(this), FIRST_CHILD(this)->fops->readlink,
+                    loc, size, xdata);
+    return 0;
 }
 
-
 int
-error_gen_readlink (call_frame_t *frame, xlator_t *this, loc_t *loc,
-                    size_t size, dict_t *xdata)
+error_gen_mknod(call_frame_t *frame, xlator_t *this, loc_t *loc, mode_t mode,
+                dev_t rdev, mode_t umask, dict_t *xdata)
 {
-	int              op_errno = 0;
-        eg_t            *egp = NULL;
-        int              enable = 1;
+    int op_errno = 0;
+    eg_t *egp = NULL;
+    int enable = 1;
 
-        egp = this->private;
-        enable = egp->enable[GF_FOP_READLINK];
+    egp = this->private;
+    enable = egp->enable[GF_FOP_MKNOD];
 
-        if (enable)
-                op_errno = error_gen (this, GF_FOP_READLINK);
+    if (enable)
+        op_errno = error_gen(this, GF_FOP_MKNOD);
 
-	if (op_errno) {
-		GF_ERROR(this, "unwind(-1, %s)", strerror (op_errno));
-		STACK_UNWIND_STRICT (readlink, frame, -1, op_errno, NULL, NULL, xdata);
+    if (op_errno) {
+        GF_ERROR(this, "unwind(-1, %s)", strerror(op_errno));
+        STACK_UNWIND_STRICT(mknod, frame, -1, op_errno, NULL, NULL, NULL, NULL,
+                            xdata);
         return 0;
-	}
+    }
 
-	STACK_WIND (frame, error_gen_readlink_cbk,
-		    FIRST_CHILD(this),
-		    FIRST_CHILD(this)->fops->readlink,
-		    loc, size, xdata);
-        return 0;
+    STACK_WIND_TAIL(frame, FIRST_CHILD(this), FIRST_CHILD(this)->fops->mknod,
+                    loc, mode, rdev, umask, xdata);
+    return 0;
 }
 
-
 int
-error_gen_mknod_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-		     int32_t op_ret, int32_t op_errno, inode_t *inode,
-                     struct iatt *buf, struct iatt *preparent,
-                     struct iatt *postparent, dict_t *xdata)
+error_gen_mkdir(call_frame_t *frame, xlator_t *this, loc_t *loc, mode_t mode,
+                mode_t umask, dict_t *xdata)
 {
-	STACK_UNWIND_STRICT (mknod, frame, op_ret, op_errno,
-                             inode, buf,
-                             preparent, postparent, xdata);
-        return 0;
-}
+    int op_errno = 0;
+    eg_t *egp = NULL;
+    int enable = 1;
 
+    egp = this->private;
+    enable = egp->enable[GF_FOP_MKDIR];
 
-int
-error_gen_mknod (call_frame_t *frame, xlator_t *this, loc_t *loc,
-		 mode_t mode, dev_t rdev, mode_t umask, dict_t *xdata)
-{
-	int              op_errno = 0;
-        eg_t            *egp = NULL;
-        int              enable = 1;
-
-        egp = this->private;
-        enable = egp->enable[GF_FOP_MKNOD];
+    if (enable)
+        op_errno = error_gen(this, GF_FOP_MKDIR);
 
-        if (enable)
-                op_errno = error_gen (this, GF_FOP_MKNOD);
-
-	if (op_errno) {
-		GF_ERROR(this, "unwind(-1, %s)", strerror (op_errno));
-		STACK_UNWIND_STRICT (mknod, frame, -1, op_errno, NULL, NULL,
-                                     NULL, NULL, xdata);
+    if (op_errno) {
+        GF_ERROR(this, "unwind(-1, %s)", strerror(op_errno));
+        STACK_UNWIND_STRICT(mkdir, frame, -1, op_errno, NULL, NULL, NULL, NULL,
+                            xdata);
         return 0;
-	}
+    }
 
-	STACK_WIND (frame, error_gen_mknod_cbk,
-		    FIRST_CHILD(this),
-		    FIRST_CHILD(this)->fops->mknod,
-		    loc, mode, rdev, umask, xdata);
-        return 0;
+    STACK_WIND_TAIL(frame, FIRST_CHILD(this), FIRST_CHILD(this)->fops->mkdir,
+                    loc, mode, umask, xdata);
+    return 0;
 }
 
-
 int
-error_gen_mkdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-		     int32_t op_ret, int32_t op_errno, inode_t *inode,
-                     struct iatt *buf, struct iatt *preparent,
-                     struct iatt *postparent, dict_t *xdata)
+error_gen_unlink(call_frame_t *frame, xlator_t *this, loc_t *loc, int xflag,
+                 dict_t *xdata)
 {
-	STACK_UNWIND_STRICT (mkdir, frame, op_ret, op_errno,
-                             inode, buf,
-                             preparent, postparent, xdata);
-        return 0;
-}
+    int op_errno = 0;
+    eg_t *egp = NULL;
+    int enable = 1;
 
-int
-error_gen_mkdir (call_frame_t *frame, xlator_t *this,
-		 loc_t *loc, mode_t mode, mode_t umask, dict_t *xdata)
-{
-	int              op_errno = 0;
-        eg_t            *egp = NULL;
-        int              enable = 1;
-
-        egp = this->private;
-        enable = egp->enable[GF_FOP_MKDIR];
-
-        if (enable)
-                op_errno = error_gen (this, GF_FOP_MKDIR);
-
-	if (op_errno) {
-		GF_ERROR(this, "unwind(-1, %s)", strerror (op_errno));
-		STACK_UNWIND_STRICT (mkdir, frame, -1, op_errno, NULL, NULL,
-                                     NULL, NULL, xdata);
-                return 0;
-	}
-
-	STACK_WIND (frame, error_gen_mkdir_cbk,
-		    FIRST_CHILD(this),
-		    FIRST_CHILD(this)->fops->mkdir,
-		    loc, mode, umask, xdata);
-        return 0;
-}
+    egp = this->private;
+    enable = egp->enable[GF_FOP_UNLINK];
 
+    if (enable)
+        op_errno = error_gen(this, GF_FOP_UNLINK);
 
-int
-error_gen_unlink_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-		      int32_t op_ret, int32_t op_errno,
-                      struct iatt *preparent, struct iatt *postparent, dict_t *xdata)
-{
-	STACK_UNWIND_STRICT (unlink, frame, op_ret, op_errno,
-                             preparent, postparent, xdata);
+    if (op_errno) {
+        GF_ERROR(this, "unwind(-1, %s)", strerror(op_errno));
+        STACK_UNWIND_STRICT(unlink, frame, -1, op_errno, NULL, NULL, xdata);
         return 0;
-}
+    }
 
+    STACK_WIND_TAIL(frame, FIRST_CHILD(this), FIRST_CHILD(this)->fops->unlink,
+                    loc, xflag, xdata);
+    return 0;
+}
 
 int
-error_gen_unlink (call_frame_t *frame, xlator_t *this, loc_t *loc, int xflag,
-                  dict_t *xdata)
+error_gen_rmdir(call_frame_t *frame, xlator_t *this, loc_t *loc, int flags,
+                dict_t *xdata)
 {
-	int              op_errno = 0;
-        eg_t            *egp = NULL;
-        int              enable = 1;
-
-        egp = this->private;
-        enable = egp->enable[GF_FOP_UNLINK];
-
-        if (enable)
-                op_errno = error_gen (this, GF_FOP_UNLINK);
-
-	if (op_errno) {
-		GF_ERROR(this, "unwind(-1, %s)", strerror (op_errno));
-		STACK_UNWIND_STRICT (unlink, frame, -1, op_errno, NULL, NULL,
-                                     xdata);
-                return 0;
-	}
-
-	STACK_WIND (frame, error_gen_unlink_cbk,
-		    FIRST_CHILD(this),
-		    FIRST_CHILD(this)->fops->unlink,
-		    loc, xflag, xdata);
-        return 0;
-}
+    int op_errno = 0;
+    eg_t *egp = NULL;
+    int enable = 1;
 
+    egp = this->private;
+    enable = egp->enable[GF_FOP_RMDIR];
 
-int
-error_gen_rmdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-		     int32_t op_ret, int32_t op_errno,
-                     struct iatt *preparent, struct iatt *postparent,
-                     dict_t *xdata)
-{
-	STACK_UNWIND_STRICT (rmdir, frame, op_ret, op_errno,
-                             preparent, postparent, xdata);
+    if (enable)
+        op_errno = error_gen(this, GF_FOP_RMDIR);
+
+    if (op_errno) {
+        GF_ERROR(this, "unwind(-1, %s)", strerror(op_errno));
+        STACK_UNWIND_STRICT(rmdir, frame, -1, op_errno, NULL, NULL, xdata);
         return 0;
-}
+    }
 
+    STACK_WIND_TAIL(frame, FIRST_CHILD(this), FIRST_CHILD(this)->fops->rmdir,
+                    loc, flags, xdata);
+    return 0;
+}
 
 int
-error_gen_rmdir (call_frame_t *frame, xlator_t *this, loc_t *loc, int flags,
-                 dict_t *xdata)
+error_gen_symlink(call_frame_t *frame, xlator_t *this, const char *linkpath,
+                  loc_t *loc, mode_t umask, dict_t *xdata)
 {
-	int              op_errno = 0;
-        eg_t            *egp = NULL;
-        int              enable = 1;
+    int op_errno = 0;
+    eg_t *egp = NULL;
+    int enable = 1;
 
-        egp = this->private;
-        enable = egp->enable[GF_FOP_RMDIR];
+    egp = this->private;
+    enable = egp->enable[GF_FOP_SYMLINK];
 
-        if (enable)
-                op_errno = error_gen (this, GF_FOP_RMDIR);
+    if (enable)
+        op_errno = error_gen(this, GF_FOP_SYMLINK);
 
-	if (op_errno) {
-		GF_ERROR(this, "unwind(-1, %s)", strerror (op_errno));
-		STACK_UNWIND_STRICT (rmdir, frame, -1, op_errno, NULL, NULL, xdata);
+    if (op_errno) {
+        GF_ERROR(this, "unwind(-1, %s)", strerror(op_errno));
+        STACK_UNWIND_STRICT(symlink, frame, -1, op_errno, NULL, NULL, NULL,
+                            NULL, NULL); /* pre & post parent attr */
         return 0;
-	}
+    }
 
-	STACK_WIND (frame, error_gen_rmdir_cbk,
-		    FIRST_CHILD(this),
-		    FIRST_CHILD(this)->fops->rmdir,
-		    loc, flags, xdata);
-        return 0;
+    STACK_WIND_TAIL(frame, FIRST_CHILD(this), FIRST_CHILD(this)->fops->symlink,
+                    linkpath, loc, umask, xdata);
+    return 0;
 }
 
-
 int
-error_gen_symlink_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-		       int32_t op_ret, int32_t op_errno, inode_t *inode,
-                       struct iatt *buf, struct iatt *preparent,
-                       struct iatt *postparent, dict_t *xdata)
+error_gen_rename(call_frame_t *frame, xlator_t *this, loc_t *oldloc,
+                 loc_t *newloc, dict_t *xdata)
 {
-	STACK_UNWIND_STRICT (symlink, frame, op_ret, op_errno, inode, buf,
-                             preparent, postparent, xdata);
-        return 0;
-}
+    int op_errno = 0;
+    eg_t *egp = NULL;
+    int enable = 1;
 
+    egp = this->private;
+    enable = egp->enable[GF_FOP_RENAME];
 
-int
-error_gen_symlink (call_frame_t *frame, xlator_t *this, const char *linkpath,
-		   loc_t *loc, mode_t umask, dict_t *xdata)
-{
-	int              op_errno = 0;
-        eg_t            *egp = NULL;
-        int              enable = 1;
-
-        egp = this->private;
-        enable = egp->enable[GF_FOP_SYMLINK];
-
-        if (enable)
-                op_errno = error_gen (this, GF_FOP_SYMLINK);
-
-	if (op_errno) {
-		GF_ERROR(this, "unwind(-1, %s)", strerror (op_errno));
-		STACK_UNWIND_STRICT (symlink, frame, -1, op_errno, NULL, NULL,
-                                     NULL, NULL, NULL); /* pre & post parent attr */
-		return 0;
-	}
-
-	STACK_WIND (frame, error_gen_symlink_cbk,
-		    FIRST_CHILD(this),
-		    FIRST_CHILD(this)->fops->symlink,
-		    linkpath, loc, umask, xdata);
-        return 0;
-}
+    if (enable)
+        op_errno = error_gen(this, GF_FOP_RENAME);
 
-
-int
-error_gen_rename_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-		      int32_t op_ret, int32_t op_errno, struct iatt *buf,
-                      struct iatt *preoldparent, struct iatt *postoldparent,
-                      struct iatt *prenewparent, struct iatt *postnewparent,
-                      dict_t *xdata)
-{
-	STACK_UNWIND_STRICT (rename, frame, op_ret, op_errno, buf,
-                             preoldparent, postoldparent,
-                             prenewparent, postnewparent, xdata);
+    if (op_errno) {
+        GF_ERROR(this, "unwind(-1, %s)", strerror(op_errno));
+        STACK_UNWIND_STRICT(rename, frame, -1, op_errno, NULL, NULL, NULL, NULL,
+                            NULL, NULL);
         return 0;
-}
-
+    }
 
-int
-error_gen_rename (call_frame_t *frame, xlator_t *this,
-		  loc_t *oldloc, loc_t *newloc, dict_t *xdata)
-{
-	int              op_errno = 0;
-        eg_t            *egp = NULL;
-        int              enable = 1;
-
-        egp = this->private;
-        enable = egp->enable[GF_FOP_RENAME];
-
-        if (enable)
-                op_errno = error_gen (this, GF_FOP_RENAME);
-
-	if (op_errno) {
-		GF_ERROR(this, "unwind(-1, %s)", strerror (op_errno));
-		STACK_UNWIND_STRICT (rename, frame, -1, op_errno, NULL,
-                                     NULL, NULL, NULL, NULL, NULL);
-		return 0;
-	}
-
-	STACK_WIND (frame, error_gen_rename_cbk,
-		    FIRST_CHILD(this),
-		    FIRST_CHILD(this)->fops->rename,
-		    oldloc, newloc, xdata);
-        return 0;
+    STACK_WIND_TAIL(frame, FIRST_CHILD(this), FIRST_CHILD(this)->fops->rename,
+                    oldloc, newloc, xdata);
+    return 0;
 }
 
-
 int
-error_gen_link_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-		    int32_t op_ret, int32_t op_errno, inode_t *inode,
-                    struct iatt *buf, struct iatt *preparent,
-                    struct iatt *postparent, dict_t *xdata)
+error_gen_link(call_frame_t *frame, xlator_t *this, loc_t *oldloc,
+               loc_t *newloc, dict_t *xdata)
 {
-	STACK_UNWIND_STRICT (link, frame, op_ret, op_errno, inode, buf,
-                             preparent, postparent, xdata);
-        return 0;
-}
+    int op_errno = 0;
+    eg_t *egp = NULL;
+    int enable = 1;
 
+    egp = this->private;
+    enable = egp->enable[GF_FOP_LINK];
 
-int
-error_gen_link (call_frame_t *frame, xlator_t *this,
-		loc_t *oldloc, loc_t *newloc, dict_t *xdata)
-{
-	int             op_errno = 0;
-        eg_t            *egp = NULL;
-        int             enable = 1;
-
-        egp = this->private;
-        enable = egp->enable[GF_FOP_LINK];
-
-        if (enable)
-                op_errno = error_gen (this, GF_FOP_LINK);
-
-	if (op_errno) {
-		GF_ERROR(this, "unwind(-1, %s)", strerror (op_errno));
-		STACK_UNWIND_STRICT (link, frame, -1, op_errno, NULL, NULL,
-                                     NULL, NULL, NULL);
-		return 0;
-	}
-
-	STACK_WIND (frame, error_gen_link_cbk,
-		    FIRST_CHILD(this),
-		    FIRST_CHILD(this)->fops->link,
-		    oldloc, newloc, xdata);
-        return 0;
-}
-
+    if (enable)
+        op_errno = error_gen(this, GF_FOP_LINK);
 
-int
-error_gen_create_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-		      int32_t op_ret, int32_t op_errno,
-		      fd_t *fd, inode_t *inode, struct iatt *buf,
-                      struct iatt *preparent, struct iatt *postparent, dict_t *xdata)
-{
-	STACK_UNWIND_STRICT (create, frame, op_ret, op_errno, fd, inode, buf,
-                             preparent, postparent, xdata);
+    if (op_errno) {
+        GF_ERROR(this, "unwind(-1, %s)", strerror(op_errno));
+        STACK_UNWIND_STRICT(link, frame, -1, op_errno, NULL, NULL, NULL, NULL,
+                            NULL);
         return 0;
-}
+    }
 
+    STACK_WIND_TAIL(frame, FIRST_CHILD(this), FIRST_CHILD(this)->fops->link,
+                    oldloc, newloc, xdata);
+    return 0;
+}
 
 int
-error_gen_create (call_frame_t *frame, xlator_t *this, loc_t *loc,
-		  int32_t flags, mode_t mode, mode_t umask, fd_t *fd,
-                  dict_t *xdata)
+error_gen_create(call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t flags,
+                 mode_t mode, mode_t umask, fd_t *fd, dict_t *xdata)
 {
-	int             op_errno = 0;
-        eg_t            *egp = NULL;
-        int             enable = 1;
-
-        egp = this->private;
-        enable = egp->enable[GF_FOP_CREATE];
-
-        if (enable)
-                op_errno = error_gen (this, GF_FOP_CREATE);
-
-	if (op_errno) {
-		GF_ERROR(this, "unwind(-1, %s)", strerror (op_errno));
-		STACK_UNWIND_STRICT (create, frame, -1, op_errno, NULL, NULL,
-                                     NULL, NULL, NULL, NULL);
-		return 0;
-	}
-
-	STACK_WIND (frame, error_gen_create_cbk,
-		    FIRST_CHILD(this),
-		    FIRST_CHILD(this)->fops->create,
-		    loc, flags, mode, umask, fd, xdata);
-        return 0;
-}
+    int op_errno = 0;
+    eg_t *egp = NULL;
+    int enable = 1;
 
+    egp = this->private;
+    enable = egp->enable[GF_FOP_CREATE];
 
-int
-error_gen_open_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-		    int32_t op_ret, int32_t op_errno, fd_t *fd, dict_t *xdata)
-{
-	STACK_UNWIND_STRICT (open, frame, op_ret, op_errno, fd, xdata);
+    if (enable)
+        op_errno = error_gen(this, GF_FOP_CREATE);
+
+    if (op_errno) {
+        GF_ERROR(this, "unwind(-1, %s)", strerror(op_errno));
+        STACK_UNWIND_STRICT(create, frame, -1, op_errno, NULL, NULL, NULL, NULL,
+                            NULL, NULL);
         return 0;
-}
+    }
 
+    STACK_WIND_TAIL(frame, FIRST_CHILD(this), FIRST_CHILD(this)->fops->create,
+                    loc, flags, mode, umask, fd, xdata);
+    return 0;
+}
 
 int
-error_gen_open (call_frame_t *frame, xlator_t *this, loc_t *loc,
-		int32_t flags, fd_t *fd, dict_t *xdata)
+error_gen_open(call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t flags,
+               fd_t *fd, dict_t *xdata)
 {
-	int              op_errno = 0;
-        eg_t            *egp = NULL;
-        int              enable = 1;
+    int op_errno = 0;
+    eg_t *egp = NULL;
+    int enable = 1;
 
-        egp = this->private;
-        enable = egp->enable[GF_FOP_OPEN];
+    egp = this->private;
+    enable = egp->enable[GF_FOP_OPEN];
 
-        if (enable)
-                op_errno = error_gen (this, GF_FOP_OPEN);
+    if (enable)
+        op_errno = error_gen(this, GF_FOP_OPEN);
 
-	if (op_errno) {
-		GF_ERROR(this, "unwind(-1, %s)", strerror (op_errno));
-		STACK_UNWIND_STRICT (open, frame, -1, op_errno, NULL, xdata);
+    if (op_errno) {
+        GF_ERROR(this, "unwind(-1, %s)", strerror(op_errno));
+        STACK_UNWIND_STRICT(open, frame, -1, op_errno, NULL, xdata);
         return 0;
-	}
+    }
 
-	STACK_WIND (frame, error_gen_open_cbk,
-		    FIRST_CHILD(this),
-		    FIRST_CHILD(this)->fops->open,
-		    loc, flags, fd, xdata);
-        return 0;
+    STACK_WIND_TAIL(frame, FIRST_CHILD(this), FIRST_CHILD(this)->fops->open,
+                    loc, flags, fd, xdata);
+    return 0;
 }
 
-
 int
-error_gen_readv_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-		     int32_t op_ret, int32_t op_errno,
-		     struct iovec *vector, int32_t count,
-		     struct iatt *stbuf, struct iobref *iobref, dict_t *xdata)
+error_gen_readv(call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
+                off_t offset, uint32_t flags, dict_t *xdata)
 {
-	STACK_UNWIND_STRICT (readv, frame, op_ret, op_errno,
-                             vector, count, stbuf, iobref, xdata);
-        return 0;
-}
+    int op_errno = 0;
+    eg_t *egp = NULL;
+    int enable = 1;
 
+    egp = this->private;
+    enable = egp->enable[GF_FOP_READ];
 
-int
-error_gen_readv (call_frame_t *frame, xlator_t *this,
-		 fd_t *fd, size_t size, off_t offset, uint32_t flags, dict_t *xdata)
-{
-	int              op_errno = 0;
-        eg_t            *egp = NULL;
-        int              enable = 1;
-
-        egp = this->private;
-        enable = egp->enable[GF_FOP_READ];
-
-        if (enable)
-                op_errno = error_gen (this, GF_FOP_READ);
+    if (enable)
+        op_errno = error_gen(this, GF_FOP_READ);
 
-	if (op_errno) {
-		GF_ERROR(this, "unwind(-1, %s)", strerror (op_errno));
-		STACK_UNWIND_STRICT (readv, frame, -1, op_errno, NULL, 0,
-                                     NULL, NULL, xdata);
+    if (op_errno) {
+        GF_ERROR(this, "unwind(-1, %s)", strerror(op_errno));
+        STACK_UNWIND_STRICT(readv, frame, -1, op_errno, NULL, 0, NULL, NULL,
+                            xdata);
         return 0;
-	}
+    }
 
-
-	STACK_WIND (frame, error_gen_readv_cbk,
-		    FIRST_CHILD(this),
-		    FIRST_CHILD(this)->fops->readv,
-		    fd, size, offset, flags, xdata);
-        return 0;
+    STACK_WIND_TAIL(frame, FIRST_CHILD(this), FIRST_CHILD(this)->fops->readv,
+                    fd, size, offset, flags, xdata);
+    return 0;
 }
 
-
 int
-error_gen_writev_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-		      int32_t op_ret, int32_t op_errno,
-                      struct iatt *prebuf, struct iatt *postbuf, dict_t *xdata)
+error_gen_writev(call_frame_t *frame, xlator_t *this, fd_t *fd,
+                 struct iovec *vector, int32_t count, off_t off, uint32_t flags,
+                 struct iobref *iobref, dict_t *xdata)
 {
-	STACK_UNWIND_STRICT (writev, frame, op_ret, op_errno, prebuf, postbuf, xdata);
-        return 0;
-}
+    int op_errno = 0;
+    eg_t *egp = NULL;
+    int enable = 1;
+    struct iovec *shortvec = NULL;
 
+    egp = this->private;
+    enable = egp->enable[GF_FOP_WRITE];
 
-int
-error_gen_writev (call_frame_t *frame, xlator_t *this, fd_t *fd,
-		  struct iovec *vector, int32_t count,
-		  off_t off, uint32_t flags, struct iobref *iobref, dict_t *xdata)
-{
-	int              op_errno = 0;
-        eg_t            *egp = NULL;
-        int              enable = 1;
-
-        egp = this->private;
-        enable = egp->enable[GF_FOP_WRITE];
-
-        if (enable)
-                op_errno = error_gen (this, GF_FOP_WRITE);
-
-	if (op_errno == GF_ERROR_SHORT_WRITE) {
-		struct iovec *shortvec;
-
-		/*
-		 * A short write error returns some value less than what was
-		 * requested from a write. To simulate this, replace the vector
-		 * with one half the size;
-		 */
-		shortvec = iov_dup(vector, 1);
-		shortvec->iov_len /= 2;
-
-		STACK_WIND(frame, error_gen_writev_cbk, FIRST_CHILD(this),
-			   FIRST_CHILD(this)->fops->writev, fd, shortvec, count,
-			   off, flags, iobref, xdata);
-		GF_FREE(shortvec);
-		return 0;
-	} else if (op_errno) {
-		GF_ERROR(this, "unwind(-1, %s)", strerror (op_errno));
-		STACK_UNWIND_STRICT (writev, frame, -1, op_errno, NULL, NULL, xdata);
-        	return 0;
-	}
-
-	STACK_WIND (frame, error_gen_writev_cbk,
-		    FIRST_CHILD(this),
-		    FIRST_CHILD(this)->fops->writev,
-		    fd, vector, count, off, flags, iobref, xdata);
-        return 0;
-}
-
+    if (enable)
+        op_errno = error_gen(this, GF_FOP_WRITE);
 
-int
-error_gen_flush_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-		     int32_t op_ret, int32_t op_errno, dict_t *xdata)
-{
-	STACK_UNWIND_STRICT (flush, frame, op_ret, op_errno, xdata);
+    if (op_errno == GF_ERROR_SHORT_WRITE) {
+        /*
+         * A short write error returns some value less than what was
+         * requested from a write. To simulate this, replace the vector
+         * with one half the size;
+         */
+        shortvec = iov_dup(vector, 1);
+        shortvec->iov_len /= 2;
+        count = 1;
+        goto wind;
+    } else if (op_errno) {
+        GF_ERROR(this, "unwind(-1, %s)", strerror(op_errno));
+        STACK_UNWIND_STRICT(writev, frame, -1, op_errno, NULL, NULL, xdata);
         return 0;
-}
+    }
+wind:
+    STACK_WIND_TAIL(frame, FIRST_CHILD(this), FIRST_CHILD(this)->fops->writev,
+                    fd, shortvec ? shortvec : vector, count, off, flags, iobref,
+                    xdata);
 
+    if (shortvec)
+        GF_FREE(shortvec);
+    return 0;
+}
 
 int
-error_gen_flush (call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *xdata)
+error_gen_flush(call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *xdata)
 {
-	int              op_errno = 0;
-        eg_t            *egp = NULL;
-        int              enable = 1;
+    int op_errno = 0;
+    eg_t *egp = NULL;
+    int enable = 1;
 
-        egp = this->private;
-        enable = egp->enable[GF_FOP_FLUSH];
+    egp = this->private;
+    enable = egp->enable[GF_FOP_FLUSH];
 
-        if (enable)
-                op_errno = error_gen (this, GF_FOP_FLUSH);
+    if (enable)
+        op_errno = error_gen(this, GF_FOP_FLUSH);
 
-	if (op_errno) {
-		GF_ERROR(this, "unwind(-1, %s)", strerror (op_errno));
-		STACK_UNWIND_STRICT (flush, frame, -1, op_errno, xdata);
+    if (op_errno) {
+        GF_ERROR(this, "unwind(-1, %s)", strerror(op_errno));
+        STACK_UNWIND_STRICT(flush, frame, -1, op_errno, xdata);
         return 0;
-	}
+    }
 
-	STACK_WIND (frame, error_gen_flush_cbk,
-                    FIRST_CHILD(this),
-		    FIRST_CHILD(this)->fops->flush,
-		    fd, xdata);
-        return 0;
+    STACK_WIND_TAIL(frame, FIRST_CHILD(this), FIRST_CHILD(this)->fops->flush,
+                    fd, xdata);
+    return 0;
 }
 
-
 int
-error_gen_fsync_cbk (call_frame_t *frame, void *cookie,
-		     xlator_t *this, int32_t op_ret,
-		     int32_t op_errno, struct iatt *prebuf,
-                     struct iatt *postbuf, dict_t *xdata)
+error_gen_fsync(call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t flags,
+                dict_t *xdata)
 {
-	STACK_UNWIND_STRICT (fsync, frame, op_ret, op_errno, prebuf, postbuf, xdata);
-        return 0;
-}
+    int op_errno = 0;
+    eg_t *egp = NULL;
+    int enable = 1;
 
+    egp = this->private;
+    enable = egp->enable[GF_FOP_FSYNC];
 
-int
-error_gen_fsync (call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t flags, dict_t *xdata)
-{
-	int              op_errno = 0;
-        eg_t            *egp = NULL;
-        int              enable = 1;
-
-        egp = this->private;
-        enable = egp->enable[GF_FOP_FSYNC];
-
-        if (enable)
-                op_errno = error_gen (this, GF_FOP_FSYNC);
+    if (enable)
+        op_errno = error_gen(this, GF_FOP_FSYNC);
 
-	if (op_errno) {
-		GF_ERROR(this, "unwind(-1, %s)", strerror (op_errno));
-		STACK_UNWIND_STRICT (fsync, frame, -1, op_errno, NULL, NULL, xdata);
+    if (op_errno) {
+        GF_ERROR(this, "unwind(-1, %s)", strerror(op_errno));
+        STACK_UNWIND_STRICT(fsync, frame, -1, op_errno, NULL, NULL, xdata);
         return 0;
-	}
+    }
 
-	STACK_WIND (frame, error_gen_fsync_cbk,
-		    FIRST_CHILD(this),
-		    FIRST_CHILD(this)->fops->fsync,
-		    fd, flags, xdata);
-        return 0;
+    STACK_WIND_TAIL(frame, FIRST_CHILD(this), FIRST_CHILD(this)->fops->fsync,
+                    fd, flags, xdata);
+    return 0;
 }
 
-
 int
-error_gen_fstat_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                     int32_t op_ret, int32_t op_errno, struct iatt *buf, dict_t *xdata)
+error_gen_fstat(call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *xdata)
 {
-	STACK_UNWIND_STRICT (fstat, frame, op_ret, op_errno, buf, xdata);
-        return 0;
-}
-
-
-int
-error_gen_fstat (call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *xdata)
-{
-	int              op_errno = 0;
-        eg_t            *egp = NULL;
-        int              enable = 1;
+    int op_errno = 0;
+    eg_t *egp = NULL;
+    int enable = 1;
 
-        egp = this->private;
-        enable = egp->enable[GF_FOP_FSTAT];
+    egp = this->private;
+    enable = egp->enable[GF_FOP_FSTAT];
 
-        if (enable)
-                op_errno = error_gen (this, GF_FOP_FSTAT);
+    if (enable)
+        op_errno = error_gen(this, GF_FOP_FSTAT);
 
-	if (op_errno) {
-		GF_ERROR(this, "unwind(-1, %s)", strerror (op_errno));
-		STACK_UNWIND_STRICT (fstat, frame, -1, op_errno, NULL, xdata);
+    if (op_errno) {
+        GF_ERROR(this, "unwind(-1, %s)", strerror(op_errno));
+        STACK_UNWIND_STRICT(fstat, frame, -1, op_errno, NULL, xdata);
         return 0;
-	}
+    }
 
-	STACK_WIND (frame, error_gen_fstat_cbk,
-		    FIRST_CHILD(this),
-		    FIRST_CHILD(this)->fops->fstat,
-		    fd, xdata);
-        return 0;
+    STACK_WIND_TAIL(frame, FIRST_CHILD(this), FIRST_CHILD(this)->fops->fstat,
+                    fd, xdata);
+    return 0;
 }
 
-
 int
-error_gen_opendir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-		       int32_t op_ret, int32_t op_errno, fd_t *fd, dict_t *xdata)
-{
-	STACK_UNWIND_STRICT (opendir, frame, op_ret, op_errno, fd, xdata);
-        return 0;
-}
-
-
-int
-error_gen_opendir (call_frame_t *frame, xlator_t *this, loc_t *loc, fd_t *fd, dict_t *xdata)
+error_gen_opendir(call_frame_t *frame, xlator_t *this, loc_t *loc, fd_t *fd,
+                  dict_t *xdata)
 {
-	int              op_errno = 0;
-        eg_t            *egp = NULL;
-        int              enable = 1;
+    int op_errno = 0;
+    eg_t *egp = NULL;
+    int enable = 1;
 
-        egp = this->private;
-        enable = egp->enable[GF_FOP_OPENDIR];
+    egp = this->private;
+    enable = egp->enable[GF_FOP_OPENDIR];
 
-        if (enable)
-                op_errno = error_gen (this, GF_FOP_OPENDIR);
+    if (enable)
+        op_errno = error_gen(this, GF_FOP_OPENDIR);
 
-	if (op_errno) {
-		GF_ERROR(this, "unwind(-1, %s)", strerror (op_errno));
-		STACK_UNWIND_STRICT (opendir, frame, -1, op_errno, NULL, xdata);
+    if (op_errno) {
+        GF_ERROR(this, "unwind(-1, %s)", strerror(op_errno));
+        STACK_UNWIND_STRICT(opendir, frame, -1, op_errno, NULL, xdata);
         return 0;
-	}
+    }
 
-	STACK_WIND (frame, error_gen_opendir_cbk,
-		    FIRST_CHILD(this),
-		    FIRST_CHILD(this)->fops->opendir,
-		    loc, fd, xdata);
-        return 0;
+    STACK_WIND_TAIL(frame, FIRST_CHILD(this), FIRST_CHILD(this)->fops->opendir,
+                    loc, fd, xdata);
+    return 0;
 }
 
 int
-error_gen_fsyncdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-			int32_t op_ret, int32_t op_errno, dict_t *xdata)
-{
-	STACK_UNWIND_STRICT (fsyncdir, frame, op_ret, op_errno, xdata);
-        return 0;
-}
-
-
-int
-error_gen_fsyncdir (call_frame_t *frame, xlator_t *this, fd_t *fd,
-                    int32_t flags, dict_t *xdata)
+error_gen_fsyncdir(call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t flags,
+                   dict_t *xdata)
 {
-	int              op_errno = 0;
-        eg_t            *egp = NULL;
-        int              enable = 1;
+    int op_errno = 0;
+    eg_t *egp = NULL;
+    int enable = 1;
 
-        egp = this->private;
-        enable = egp->enable[GF_FOP_FSYNCDIR];
+    egp = this->private;
+    enable = egp->enable[GF_FOP_FSYNCDIR];
 
-        if (enable)
-                op_errno = error_gen (this, GF_FOP_FSYNCDIR);
+    if (enable)
+        op_errno = error_gen(this, GF_FOP_FSYNCDIR);
 
-	if (op_errno) {
-		GF_ERROR(this, "unwind(-1, %s)", strerror (op_errno));
-		STACK_UNWIND_STRICT (fsyncdir, frame, -1, op_errno, xdata);
+    if (op_errno) {
+        GF_ERROR(this, "unwind(-1, %s)", strerror(op_errno));
+        STACK_UNWIND_STRICT(fsyncdir, frame, -1, op_errno, xdata);
         return 0;
-	}
+    }
 
-	STACK_WIND (frame, error_gen_fsyncdir_cbk,
-		    FIRST_CHILD(this),
-		    FIRST_CHILD(this)->fops->fsyncdir,
-		    fd, flags, xdata);
-        return 0;
+    STACK_WIND_TAIL(frame, FIRST_CHILD(this), FIRST_CHILD(this)->fops->fsyncdir,
+                    fd, flags, xdata);
+    return 0;
 }
 
-
 int
-error_gen_statfs_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-		      int32_t op_ret, int32_t op_errno, struct statvfs *buf, dict_t *xdata)
+error_gen_statfs(call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xdata)
 {
-	STACK_UNWIND_STRICT (statfs, frame, op_ret, op_errno, buf, xdata);
-        return 0;
-}
+    int op_errno = 0;
+    eg_t *egp = NULL;
+    int enable = 1;
 
+    egp = this->private;
+    enable = egp->enable[GF_FOP_STATFS];
 
-int
-error_gen_statfs (call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xdata)
-{
-	int              op_errno = 0;
-        eg_t            *egp = NULL;
-        int              enable = 1;
+    if (enable)
+        op_errno = error_gen(this, GF_FOP_STATFS);
 
-        egp = this->private;
-        enable = egp->enable[GF_FOP_STATFS];
-
-        if (enable)
-                op_errno = error_gen (this, GF_FOP_STATFS);
-
-	if (op_errno) {
-		GF_ERROR(this, "unwind(-1, %s)", strerror (op_errno));
-		STACK_UNWIND_STRICT (statfs, frame, -1, op_errno, NULL, xdata);
+    if (op_errno) {
+        GF_ERROR(this, "unwind(-1, %s)", strerror(op_errno));
+        STACK_UNWIND_STRICT(statfs, frame, -1, op_errno, NULL, xdata);
         return 0;
-	}
-
-	STACK_WIND (frame, error_gen_statfs_cbk,
-		    FIRST_CHILD(this),
-		    FIRST_CHILD(this)->fops->statfs,
-		    loc, xdata);
-        return 0;
-}
+    }
 
-
-int
-error_gen_setxattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-			int32_t op_ret, int32_t op_errno, dict_t *xdata)
-{
-	STACK_UNWIND_STRICT (setxattr, frame, op_ret, op_errno, xdata);
-        return 0;
+    STACK_WIND_TAIL(frame, FIRST_CHILD(this), FIRST_CHILD(this)->fops->statfs,
+                    loc, xdata);
+    return 0;
 }
 
-
 int
-error_gen_setxattr (call_frame_t *frame, xlator_t *this, loc_t *loc,
-		    dict_t *dict, int32_t flags, dict_t *xdata)
+error_gen_setxattr(call_frame_t *frame, xlator_t *this, loc_t *loc,
+                   dict_t *dict, int32_t flags, dict_t *xdata)
 {
-	int              op_errno = 0;
-        eg_t            *egp = NULL;
-        int              enable = 1;
+    int op_errno = 0;
+    eg_t *egp = NULL;
+    int enable = 1;
 
-        egp = this->private;
-        enable = egp->enable[GF_FOP_SETXATTR];
+    egp = this->private;
+    enable = egp->enable[GF_FOP_SETXATTR];
 
-        if (enable)
-                op_errno = error_gen (this, GF_FOP_SETXATTR);
+    if (enable)
+        op_errno = error_gen(this, GF_FOP_SETXATTR);
 
-	if (op_errno) {
-		GF_ERROR(this, "unwind(-1, %s)", strerror (op_errno));
-		STACK_UNWIND_STRICT (setxattr, frame, -1, op_errno, xdata);
+    if (op_errno) {
+        GF_ERROR(this, "unwind(-1, %s)", strerror(op_errno));
+        STACK_UNWIND_STRICT(setxattr, frame, -1, op_errno, xdata);
         return 0;
-	}
+    }
 
-	STACK_WIND (frame, error_gen_setxattr_cbk,
-		    FIRST_CHILD(this),
-		    FIRST_CHILD(this)->fops->setxattr,
-		    loc, dict, flags, xdata);
-        return 0;
+    STACK_WIND_TAIL(frame, FIRST_CHILD(this), FIRST_CHILD(this)->fops->setxattr,
+                    loc, dict, flags, xdata);
+    return 0;
 }
 
-
 int
-error_gen_getxattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-			int32_t op_ret, int32_t op_errno, dict_t *dict, dict_t *xdata)
+error_gen_getxattr(call_frame_t *frame, xlator_t *this, loc_t *loc,
+                   const char *name, dict_t *xdata)
 {
-	STACK_UNWIND_STRICT (getxattr, frame, op_ret, op_errno, dict, xdata);
-        return 0;
-}
+    int op_errno = 0;
+    eg_t *egp = NULL;
+    int enable = 1;
 
+    egp = this->private;
+    enable = egp->enable[GF_FOP_GETXATTR];
 
-int
-error_gen_getxattr (call_frame_t *frame, xlator_t *this, loc_t *loc,
-		    const char *name, dict_t *xdata)
-{
-	int              op_errno = 0;
-        eg_t            *egp = NULL;
-        int              enable = 1;
+    if (enable)
+        op_errno = error_gen(this, GF_FOP_GETXATTR);
 
-        egp = this->private;
-        enable = egp->enable[GF_FOP_GETXATTR];
-
-        if (enable)
-                op_errno = error_gen (this, GF_FOP_GETXATTR);
-
-	if (op_errno) {
-		GF_ERROR(this, "unwind(-1, %s)", strerror (op_errno));
-		STACK_UNWIND_STRICT (getxattr, frame, -1, op_errno, NULL, xdata);
+    if (op_errno) {
+        GF_ERROR(this, "unwind(-1, %s)", strerror(op_errno));
+        STACK_UNWIND_STRICT(getxattr, frame, -1, op_errno, NULL, xdata);
         return 0;
-	}
+    }
 
-	STACK_WIND (frame, error_gen_getxattr_cbk,
-		    FIRST_CHILD(this),
-		    FIRST_CHILD(this)->fops->getxattr,
-		    loc, name, xdata);
-        return 0;
+    STACK_WIND_TAIL(frame, FIRST_CHILD(this), FIRST_CHILD(this)->fops->getxattr,
+                    loc, name, xdata);
+    return 0;
 }
 
 int
-error_gen_fsetxattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                         int32_t op_ret, int32_t op_errno, dict_t *xdata)
-{
-	STACK_UNWIND_STRICT (fsetxattr, frame, op_ret, op_errno, xdata);
-        return 0;
-}
-
-
-int
-error_gen_fsetxattr (call_frame_t *frame, xlator_t *this, fd_t *fd,
-                     dict_t *dict, int32_t flags, dict_t *xdata)
+error_gen_fsetxattr(call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *dict,
+                    int32_t flags, dict_t *xdata)
 {
-	int              op_errno = 0;
-        eg_t            *egp = NULL;
-        int              enable = 1;
-
-        egp = this->private;
-        enable = egp->enable[GF_FOP_FSETXATTR];
+    int op_errno = 0;
+    eg_t *egp = NULL;
+    int enable = 1;
 
-        if (enable)
-                op_errno = error_gen (this, GF_FOP_FSETXATTR);
+    egp = this->private;
+    enable = egp->enable[GF_FOP_FSETXATTR];
 
-	if (op_errno) {
-		GF_ERROR(this, "unwind(-1, %s)", strerror (op_errno));
-		STACK_UNWIND_STRICT (fsetxattr, frame, -1, op_errno, xdata);
-        return 0;
-	}
+    if (enable)
+        op_errno = error_gen(this, GF_FOP_FSETXATTR);
 
-	STACK_WIND (frame, error_gen_fsetxattr_cbk,
-		    FIRST_CHILD(this),
-		    FIRST_CHILD(this)->fops->fsetxattr,
-		    fd, dict, flags, xdata);
+    if (op_errno) {
+        GF_ERROR(this, "unwind(-1, %s)", strerror(op_errno));
+        STACK_UNWIND_STRICT(fsetxattr, frame, -1, op_errno, xdata);
         return 0;
-}
-
+    }
 
-int
-error_gen_fgetxattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                         int32_t op_ret, int32_t op_errno, dict_t *dict, dict_t *xdata)
-{
-	STACK_UNWIND_STRICT (fgetxattr, frame, op_ret, op_errno, dict, xdata);
-        return 0;
+    STACK_WIND_TAIL(frame, FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->fsetxattr, fd, dict, flags, xdata);
+    return 0;
 }
 
-
 int
-error_gen_fgetxattr (call_frame_t *frame, xlator_t *this, fd_t *fd,
-                     const char *name, dict_t *xdata)
+error_gen_fgetxattr(call_frame_t *frame, xlator_t *this, fd_t *fd,
+                    const char *name, dict_t *xdata)
 {
-	int              op_errno = 0;
-        eg_t            *egp = NULL;
-        int              enable = 1;
-
-        egp = this->private;
-        enable = egp->enable[GF_FOP_FGETXATTR];
+    int op_errno = 0;
+    eg_t *egp = NULL;
+    int enable = 1;
 
-        if (enable)
-                op_errno = error_gen (this, GF_FOP_FGETXATTR);
+    egp = this->private;
+    enable = egp->enable[GF_FOP_FGETXATTR];
 
-	if (op_errno) {
-		GF_ERROR(this, "unwind(-1, %s)", strerror (op_errno));
-		STACK_UNWIND_STRICT (fgetxattr, frame, -1, op_errno, NULL, xdata);
-        return 0;
-	}
+    if (enable)
+        op_errno = error_gen(this, GF_FOP_FGETXATTR);
 
-	STACK_WIND (frame, error_gen_fgetxattr_cbk,
-		    FIRST_CHILD(this),
-		    FIRST_CHILD(this)->fops->fgetxattr,
-		    fd, name, xdata);
+    if (op_errno) {
+        GF_ERROR(this, "unwind(-1, %s)", strerror(op_errno));
+        STACK_UNWIND_STRICT(fgetxattr, frame, -1, op_errno, NULL, xdata);
         return 0;
-}
-
+    }
 
-int
-error_gen_xattrop_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                       int32_t op_ret, int32_t op_errno, dict_t *dict, dict_t *xdata)
-{
-	STACK_UNWIND_STRICT (xattrop, frame, op_ret, op_errno, dict, xdata);
-        return 0;
+    STACK_WIND_TAIL(frame, FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->fgetxattr, fd, name, xdata);
+    return 0;
 }
 
-
 int
-error_gen_xattrop (call_frame_t *frame, xlator_t *this, loc_t *loc,
-		   gf_xattrop_flags_t flags, dict_t *dict, dict_t *xdata)
+error_gen_xattrop(call_frame_t *frame, xlator_t *this, loc_t *loc,
+                  gf_xattrop_flags_t flags, dict_t *dict, dict_t *xdata)
 {
-	int              op_errno = 0;
-        eg_t            *egp = NULL;
-        int              enable = 1;
+    int op_errno = 0;
+    eg_t *egp = NULL;
+    int enable = 1;
 
-        egp = this->private;
-        enable = egp->enable[GF_FOP_XATTROP];
+    egp = this->private;
+    enable = egp->enable[GF_FOP_XATTROP];
 
-        if (enable)
-                op_errno = error_gen (this, GF_FOP_XATTROP);
+    if (enable)
+        op_errno = error_gen(this, GF_FOP_XATTROP);
 
-	if (op_errno) {
-		GF_ERROR(this, "unwind(-1, %s)", strerror (op_errno));
-		STACK_UNWIND_STRICT (xattrop, frame, -1, op_errno, NULL, xdata);
+    if (op_errno) {
+        GF_ERROR(this, "unwind(-1, %s)", strerror(op_errno));
+        STACK_UNWIND_STRICT(xattrop, frame, -1, op_errno, NULL, xdata);
         return 0;
-	}
+    }
 
-	STACK_WIND (frame, error_gen_xattrop_cbk,
-		    FIRST_CHILD(this),
-		    FIRST_CHILD(this)->fops->xattrop,
-		    loc, flags, dict, xdata);
-        return 0;
-}
-
-
-int
-error_gen_fxattrop_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-			int32_t op_ret, int32_t op_errno, dict_t *dict, dict_t *xdata)
-{
-	STACK_UNWIND_STRICT (fxattrop, frame, op_ret, op_errno, dict, xdata);
-        return 0;
+    STACK_WIND_TAIL(frame, FIRST_CHILD(this), FIRST_CHILD(this)->fops->xattrop,
+                    loc, flags, dict, xdata);
+    return 0;
 }
 
-
 int
-error_gen_fxattrop (call_frame_t *frame, xlator_t *this, fd_t *fd,
-		    gf_xattrop_flags_t flags, dict_t *dict, dict_t *xdata)
+error_gen_fxattrop(call_frame_t *frame, xlator_t *this, fd_t *fd,
+                   gf_xattrop_flags_t flags, dict_t *dict, dict_t *xdata)
 {
-	int              op_errno = 0;
-        eg_t            *egp = NULL;
-        int              enable = 1;
+    int op_errno = 0;
+    eg_t *egp = NULL;
+    int enable = 1;
 
-        egp = this->private;
-        enable = egp->enable[GF_FOP_FXATTROP];
+    egp = this->private;
+    enable = egp->enable[GF_FOP_FXATTROP];
 
-        if (enable)
-                op_errno = error_gen (this, GF_FOP_FXATTROP);
+    if (enable)
+        op_errno = error_gen(this, GF_FOP_FXATTROP);
 
-	if (op_errno) {
-		GF_ERROR(this, "unwind(-1, %s)", strerror (op_errno));
-		STACK_UNWIND_STRICT (fxattrop, frame, -1, op_errno, NULL, xdata);
+    if (op_errno) {
+        GF_ERROR(this, "unwind(-1, %s)", strerror(op_errno));
+        STACK_UNWIND_STRICT(fxattrop, frame, -1, op_errno, NULL, xdata);
         return 0;
-	}
+    }
 
-	STACK_WIND (frame, error_gen_fxattrop_cbk,
-		    FIRST_CHILD(this),
-		    FIRST_CHILD(this)->fops->fxattrop,
-		    fd, flags, dict, xdata);
-        return 0;
+    STACK_WIND_TAIL(frame, FIRST_CHILD(this), FIRST_CHILD(this)->fops->fxattrop,
+                    fd, flags, dict, xdata);
+    return 0;
 }
 
-
 int
-error_gen_removexattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-			   int32_t op_ret, int32_t op_errno, dict_t *xdata)
+error_gen_removexattr(call_frame_t *frame, xlator_t *this, loc_t *loc,
+                      const char *name, dict_t *xdata)
 {
-	STACK_UNWIND_STRICT (removexattr, frame, op_ret, op_errno, xdata);
-        return 0;
-}
+    int op_errno = 0;
+    eg_t *egp = NULL;
+    int enable = 1;
 
+    egp = this->private;
+    enable = egp->enable[GF_FOP_REMOVEXATTR];
 
-int
-error_gen_removexattr (call_frame_t *frame, xlator_t *this, loc_t *loc,
-		       const char *name, dict_t *xdata)
-{
-	int              op_errno = 0;
-        eg_t            *egp = NULL;
-        int              enable = 1;
-
-        egp = this->private;
-        enable = egp->enable[GF_FOP_REMOVEXATTR];
-
-        if (enable)
-                op_errno = error_gen (this, GF_FOP_REMOVEXATTR);
+    if (enable)
+        op_errno = error_gen(this, GF_FOP_REMOVEXATTR);
 
-	if (op_errno) {
-		GF_ERROR(this, "unwind(-1, %s)", strerror (op_errno));
-		STACK_UNWIND_STRICT (removexattr, frame, -1, op_errno, xdata);
+    if (op_errno) {
+        GF_ERROR(this, "unwind(-1, %s)", strerror(op_errno));
+        STACK_UNWIND_STRICT(removexattr, frame, -1, op_errno, xdata);
         return 0;
-	}
+    }
 
-	STACK_WIND (frame, error_gen_removexattr_cbk,
-		    FIRST_CHILD(this),
-		    FIRST_CHILD(this)->fops->removexattr,
-		    loc, name, xdata);
-        return 0;
+    STACK_WIND_TAIL(frame, FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->removexattr, loc, name, xdata);
+    return 0;
 }
 
 int
-error_gen_fremovexattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-			   int32_t op_ret, int32_t op_errno, dict_t *xdata)
+error_gen_fremovexattr(call_frame_t *frame, xlator_t *this, fd_t *fd,
+                       const char *name, dict_t *xdata)
 {
-	STACK_UNWIND_STRICT (fremovexattr, frame, op_ret, op_errno, xdata);
-        return 0;
-}
-
-
-int
-error_gen_fremovexattr (call_frame_t *frame, xlator_t *this, fd_t *fd,
-                        const char *name, dict_t *xdata)
-{
-	int              op_errno = 0;
-        eg_t            *egp = NULL;
-        int              enable = 1;
+    int op_errno = 0;
+    eg_t *egp = NULL;
+    int enable = 1;
 
-        egp = this->private;
-        enable = egp->enable[GF_FOP_FREMOVEXATTR];
+    egp = this->private;
+    enable = egp->enable[GF_FOP_FREMOVEXATTR];
 
-        if (enable)
-                op_errno = error_gen (this, GF_FOP_FREMOVEXATTR);
+    if (enable)
+        op_errno = error_gen(this, GF_FOP_FREMOVEXATTR);
 
-	if (op_errno) {
-		GF_ERROR(this, "unwind(-1, %s)", strerror (op_errno));
-		STACK_UNWIND_STRICT (fremovexattr, frame, -1, op_errno, xdata);
+    if (op_errno) {
+        GF_ERROR(this, "unwind(-1, %s)", strerror(op_errno));
+        STACK_UNWIND_STRICT(fremovexattr, frame, -1, op_errno, xdata);
         return 0;
-	}
+    }
 
-	STACK_WIND (frame, error_gen_fremovexattr_cbk,
-		    FIRST_CHILD(this),
-		    FIRST_CHILD(this)->fops->fremovexattr,
-		    fd, name, xdata);
-        return 0;
-}
-
-
-int
-error_gen_lk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-		  int32_t op_ret, int32_t op_errno, struct gf_flock *lock, dict_t *xdata)
-{
-	STACK_UNWIND_STRICT (lk, frame, op_ret, op_errno, lock, xdata);
-        return 0;
+    STACK_WIND_TAIL(frame, FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->fremovexattr, fd, name, xdata);
+    return 0;
 }
 
-
 int
-error_gen_lk (call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t cmd,
-	      struct gf_flock *lock, dict_t *xdata)
+error_gen_lk(call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t cmd,
+             struct gf_flock *lock, dict_t *xdata)
 {
-	int              op_errno = 0;
-        eg_t            *egp = NULL;
-        int              enable = 1;
+    int op_errno = 0;
+    eg_t *egp = NULL;
+    int enable = 1;
 
-        egp = this->private;
-        enable = egp->enable[GF_FOP_LK];
+    egp = this->private;
+    enable = egp->enable[GF_FOP_LK];
 
-        if (enable)
-                op_errno = error_gen (this, GF_FOP_LK);
+    if (enable)
+        op_errno = error_gen(this, GF_FOP_LK);
 
-	if (op_errno) {
-		GF_ERROR(this, "unwind(-1, %s)", strerror (op_errno));
-		STACK_UNWIND_STRICT (lk, frame, -1, op_errno, NULL, xdata);
+    if (op_errno) {
+        GF_ERROR(this, "unwind(-1, %s)", strerror(op_errno));
+        STACK_UNWIND_STRICT(lk, frame, -1, op_errno, NULL, xdata);
         return 0;
-	}
+    }
 
-	STACK_WIND (frame, error_gen_lk_cbk,
-		    FIRST_CHILD(this),
-		    FIRST_CHILD(this)->fops->lk,
-		    fd, cmd, lock, xdata);
-        return 0;
+    STACK_WIND_TAIL(frame, FIRST_CHILD(this), FIRST_CHILD(this)->fops->lk, fd,
+                    cmd, lock, xdata);
+    return 0;
 }
 
-
 int
-error_gen_inodelk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                       int32_t op_ret, int32_t op_errno, dict_t *xdata)
+error_gen_inodelk(call_frame_t *frame, xlator_t *this, const char *volume,
+                  loc_t *loc, int32_t cmd, struct gf_flock *lock, dict_t *xdata)
 {
-	STACK_UNWIND_STRICT (inodelk, frame, op_ret, op_errno, xdata);
-        return 0;
-}
-
-
-int
-error_gen_inodelk (call_frame_t *frame, xlator_t *this,
-		   const char *volume, loc_t *loc, int32_t cmd,
-                   struct gf_flock *lock, dict_t *xdata)
-{
-	int              op_errno = 0;
-        eg_t            *egp = NULL;
-        int              enable = 1;
-
-        egp = this->private;
-        enable = egp->enable[GF_FOP_INODELK];
+    int op_errno = 0;
+    eg_t *egp = NULL;
+    int enable = 1;
 
-        if (enable)
-                op_errno = error_gen (this, GF_FOP_INODELK);
+    egp = this->private;
+    enable = egp->enable[GF_FOP_INODELK];
 
-	if (op_errno) {
-		GF_ERROR(this, "unwind(-1, %s)", strerror (op_errno));
-		STACK_UNWIND_STRICT (inodelk, frame, -1, op_errno, xdata);
-        return 0;
-	}
+    if (enable)
+        op_errno = error_gen(this, GF_FOP_INODELK);
 
-	STACK_WIND (frame, error_gen_inodelk_cbk,
-		    FIRST_CHILD(this),
-		    FIRST_CHILD(this)->fops->inodelk,
-		    volume, loc, cmd, lock, xdata);
+    if (op_errno) {
+        GF_ERROR(this, "unwind(-1, %s)", strerror(op_errno));
+        STACK_UNWIND_STRICT(inodelk, frame, -1, op_errno, xdata);
         return 0;
-}
-
+    }
 
-int
-error_gen_finodelk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                        int32_t op_ret, int32_t op_errno, dict_t *xdata)
-{
-	STACK_UNWIND_STRICT (finodelk, frame, op_ret, op_errno, xdata);
-        return 0;
+    STACK_WIND_TAIL(frame, FIRST_CHILD(this), FIRST_CHILD(this)->fops->inodelk,
+                    volume, loc, cmd, lock, xdata);
+    return 0;
 }
 
-
 int
-error_gen_finodelk (call_frame_t *frame, xlator_t *this,
-		    const char *volume, fd_t *fd, int32_t cmd,
-                    struct gf_flock *lock, dict_t *xdata)
+error_gen_finodelk(call_frame_t *frame, xlator_t *this, const char *volume,
+                   fd_t *fd, int32_t cmd, struct gf_flock *lock, dict_t *xdata)
 {
-	int              op_errno = 0;
-        eg_t            *egp = NULL;
-        int              enable = 1;
-
-        egp = this->private;
-        enable = egp->enable[GF_FOP_FINODELK];
+    int op_errno = 0;
+    eg_t *egp = NULL;
+    int enable = 1;
 
-        if (enable)
-                op_errno = error_gen (this, GF_FOP_FINODELK);
+    egp = this->private;
+    enable = egp->enable[GF_FOP_FINODELK];
 
-	if (op_errno) {
-		GF_ERROR(this, "unwind(-1, %s)", strerror (op_errno));
-		STACK_UNWIND_STRICT (finodelk, frame, -1, op_errno, xdata);
-        return 0;
-	}
+    if (enable)
+        op_errno = error_gen(this, GF_FOP_FINODELK);
 
-	STACK_WIND (frame, error_gen_finodelk_cbk,
-		    FIRST_CHILD(this),
-		    FIRST_CHILD(this)->fops->finodelk,
-		    volume, fd, cmd, lock, xdata);
+    if (op_errno) {
+        GF_ERROR(this, "unwind(-1, %s)", strerror(op_errno));
+        STACK_UNWIND_STRICT(finodelk, frame, -1, op_errno, xdata);
         return 0;
-}
-
+    }
 
-int
-error_gen_entrylk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                       int32_t op_ret, int32_t op_errno, dict_t *xdata)
-{
-	STACK_UNWIND_STRICT (entrylk, frame, op_ret, op_errno, xdata);
-        return 0;
+    STACK_WIND_TAIL(frame, FIRST_CHILD(this), FIRST_CHILD(this)->fops->finodelk,
+                    volume, fd, cmd, lock, xdata);
+    return 0;
 }
 
-
 int
-error_gen_entrylk (call_frame_t *frame, xlator_t *this,
-		   const char *volume, loc_t *loc, const char *basename,
-		   entrylk_cmd cmd, entrylk_type type, dict_t *xdata)
+error_gen_entrylk(call_frame_t *frame, xlator_t *this, const char *volume,
+                  loc_t *loc, const char *basename, entrylk_cmd cmd,
+                  entrylk_type type, dict_t *xdata)
 {
-	int              op_errno = 0;
-        eg_t            *egp = NULL;
-        int              enable = 1;
+    int op_errno = 0;
+    eg_t *egp = NULL;
+    int enable = 1;
 
-        egp = this->private;
-        enable = egp->enable[GF_FOP_ENTRYLK];
+    egp = this->private;
+    enable = egp->enable[GF_FOP_ENTRYLK];
 
-        if (enable)
-                op_errno = error_gen (this, GF_FOP_ENTRYLK);
+    if (enable)
+        op_errno = error_gen(this, GF_FOP_ENTRYLK);
 
-	if (op_errno) {
-		GF_ERROR(this, "unwind(-1, %s)", strerror (op_errno));
-		STACK_UNWIND_STRICT (entrylk, frame, -1, op_errno, xdata);
+    if (op_errno) {
+        GF_ERROR(this, "unwind(-1, %s)", strerror(op_errno));
+        STACK_UNWIND_STRICT(entrylk, frame, -1, op_errno, xdata);
         return 0;
-	}
+    }
 
-	STACK_WIND (frame, error_gen_entrylk_cbk,
-		    FIRST_CHILD(this),
-		    FIRST_CHILD(this)->fops->entrylk,
-		    volume, loc, basename, cmd, type, xdata);
-        return 0;
+    STACK_WIND_TAIL(frame, FIRST_CHILD(this), FIRST_CHILD(this)->fops->entrylk,
+                    volume, loc, basename, cmd, type, xdata);
+    return 0;
 }
 
-
 int
-error_gen_fentrylk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                        int32_t op_ret, int32_t op_errno, dict_t *xdata)
+error_gen_fentrylk(call_frame_t *frame, xlator_t *this, const char *volume,
+                   fd_t *fd, const char *basename, entrylk_cmd cmd,
+                   entrylk_type type, dict_t *xdata)
 {
-	STACK_UNWIND_STRICT (fentrylk, frame, op_ret, op_errno, xdata);
-        return 0;
-}
+    int op_errno = 0;
+    eg_t *egp = NULL;
+    int enable = 1;
 
+    egp = this->private;
+    enable = egp->enable[GF_FOP_FENTRYLK];
 
-int
-error_gen_fentrylk (call_frame_t *frame, xlator_t *this,
-		    const char *volume, fd_t *fd, const char *basename,
-		    entrylk_cmd cmd, entrylk_type type, dict_t *xdata)
-{
-	int             op_errno = 0;
-        eg_t            *egp = NULL;
-        int             enable = 1;
-
-        egp = this->private;
-        enable = egp->enable[GF_FOP_FENTRYLK];
-
-        if (enable)
-                op_errno = error_gen (this, GF_FOP_FENTRYLK);
-
-	if (op_errno) {
-		GF_ERROR(this, "unwind(-1, %s)", strerror (op_errno));
-		STACK_UNWIND_STRICT (fentrylk, frame, -1, op_errno, xdata);
-                return 0;
-	}
-
-	STACK_WIND (frame, error_gen_fentrylk_cbk,
-		    FIRST_CHILD(this),
-		    FIRST_CHILD(this)->fops->fentrylk,
-		    volume, fd, basename, cmd, type, xdata);
+    if (enable)
+        op_errno = error_gen(this, GF_FOP_FENTRYLK);
+
+    if (op_errno) {
+        GF_ERROR(this, "unwind(-1, %s)", strerror(op_errno));
+        STACK_UNWIND_STRICT(fentrylk, frame, -1, op_errno, xdata);
         return 0;
+    }
+
+    STACK_WIND_TAIL(frame, FIRST_CHILD(this), FIRST_CHILD(this)->fops->fentrylk,
+                    volume, fd, basename, cmd, type, xdata);
+    return 0;
 }
 
+int
+error_gen_getspec(call_frame_t *frame, xlator_t *this, const char *key,
+                  int32_t flags)
+{
+    int op_errno = 0;
+    eg_t *egp = NULL;
+    int enable = 1;
 
-/* Management operations */
+    egp = this->private;
+    enable = egp->enable[GF_FOP_GETSPEC];
 
+    if (enable)
+        op_errno = error_gen(this, GF_FOP_GETSPEC);
 
-int
-error_gen_getspec_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-		       int32_t op_ret, int32_t op_errno, char *spec_data)
-{
-	STACK_UNWIND_STRICT (getspec, frame, op_ret, op_errno, spec_data);
+    if (op_errno) {
+        GF_ERROR(this, "unwind(-1, %s)", strerror(op_errno));
+        STACK_UNWIND_STRICT(getspec, frame, -1, op_errno, NULL);
         return 0;
-}
+    }
 
-
-int
-error_gen_getspec (call_frame_t *frame, xlator_t *this, const char *key,
-		   int32_t flags)
-{
-	int              op_errno = 0;
-        eg_t            *egp = NULL;
-        int              enable = 1;
-
-        egp = this->private;
-        enable = egp->enable[GF_FOP_GETSPEC];
-
-        if (enable)
-                op_errno = error_gen (this, GF_FOP_GETSPEC);
-
-	if (op_errno) {
-		GF_ERROR(this, "unwind(-1, %s)", strerror (op_errno));
-		STACK_UNWIND_STRICT (getspec, frame, -1, op_errno, NULL);
-		return 0;
-	}
-
-	STACK_WIND (frame, error_gen_getspec_cbk,
-		    FIRST_CHILD(this),
-		    FIRST_CHILD(this)->fops->getspec,
-		    key, flags);
-	return 0;
+    STACK_WIND_TAIL(frame, FIRST_CHILD(this), FIRST_CHILD(this)->fops->getspec,
+                    key, flags);
+    return 0;
 }
 
-
 int
-error_gen_readdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-		       int32_t op_ret, int32_t op_errno, gf_dirent_t *entries,
-                       dict_t *xdata)
+error_gen_readdir(call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
+                  off_t off, dict_t *xdata)
 {
-	STACK_UNWIND_STRICT (readdir, frame, op_ret, op_errno, entries, xdata);
-	return 0;
-}
+    int op_errno = 0;
+    eg_t *egp = NULL;
+    int enable = 1;
 
+    egp = this->private;
+    enable = egp->enable[GF_FOP_READDIR];
 
-int
-error_gen_readdir (call_frame_t *frame, xlator_t *this, fd_t *fd,
-		   size_t size, off_t off, dict_t *xdata)
-{
-	int              op_errno = 0;
-        eg_t            *egp = NULL;
-        int              enable = 1;
-
-        egp = this->private;
-        enable = egp->enable[GF_FOP_READDIR];
-
-        if (enable)
-                op_errno = error_gen (this, GF_FOP_READDIR);
-
-	if (op_errno) {
-		GF_ERROR(this, "unwind(-1, %s)", strerror (op_errno));
-		STACK_UNWIND_STRICT (readdir, frame, -1, op_errno, NULL, xdata);
-		return 0;
-	}
-
-	STACK_WIND (frame, error_gen_readdir_cbk,
-		    FIRST_CHILD(this),
-		    FIRST_CHILD(this)->fops->readdir,
-		    fd, size, off, xdata);
-	return 0;
-}
+    if (enable)
+        op_errno = error_gen(this, GF_FOP_READDIR);
 
+    if (op_errno) {
+        GF_ERROR(this, "unwind(-1, %s)", strerror(op_errno));
+        STACK_UNWIND_STRICT(readdir, frame, -1, op_errno, NULL, xdata);
+        return 0;
+    }
 
-int
-error_gen_readdirp_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                        int32_t op_ret, int32_t op_errno, gf_dirent_t *entries,
-                        dict_t *xdata)
-{
-	STACK_UNWIND_STRICT (readdirp, frame, op_ret, op_errno, entries, xdata);
-	return 0;
+    STACK_WIND_TAIL(frame, FIRST_CHILD(this), FIRST_CHILD(this)->fops->readdir,
+                    fd, size, off, xdata);
+    return 0;
 }
 
-
 int
-error_gen_readdirp (call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
-                    off_t off, dict_t *dict)
+error_gen_readdirp(call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
+                   off_t off, dict_t *dict)
 {
-	int              op_errno = 0;
-        eg_t            *egp = NULL;
-        int              enable = 1;
+    int op_errno = 0;
+    eg_t *egp = NULL;
+    int enable = 1;
 
-        egp = this->private;
-        enable = egp->enable[GF_FOP_READDIRP];
+    egp = this->private;
+    enable = egp->enable[GF_FOP_READDIRP];
 
-        if (enable)
-                op_errno = error_gen (this, GF_FOP_READDIRP);
+    if (enable)
+        op_errno = error_gen(this, GF_FOP_READDIRP);
 
-	if (op_errno) {
-		GF_ERROR(this, "unwind(-1, %s)", strerror (op_errno));
-		STACK_UNWIND_STRICT (readdirp, frame, -1, op_errno, NULL, NULL);
-		return 0;
-	}
+    if (op_errno) {
+        GF_ERROR(this, "unwind(-1, %s)", strerror(op_errno));
+        STACK_UNWIND_STRICT(readdirp, frame, -1, op_errno, NULL, NULL);
+        return 0;
+    }
 
-	STACK_WIND (frame, error_gen_readdirp_cbk,
-                    FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->readdirp,
+    STACK_WIND_TAIL(frame, FIRST_CHILD(this), FIRST_CHILD(this)->fops->readdirp,
                     fd, size, off, dict);
-	return 0;
+    return 0;
 }
 
 static void
-error_gen_set_failure (eg_t *pvt, int percent)
+error_gen_set_failure(eg_t *pvt, double percent)
 {
-        GF_ASSERT (pvt);
+    double ppm;
+
+    GF_ASSERT(pvt);
 
-        if (percent)
-                pvt->failure_iter_no = 100/percent;
-        else
-                pvt->failure_iter_no = 100/GF_FAILURE_DEFAULT;
+    ppm = (percent / 100.0) * (double)FAILURE_GRANULARITY;
+    pvt->failure_iter_no = (int)ppm;
 }
 
 static void
-error_gen_parse_fill_fops (eg_t *pvt, char *enable_fops)
+error_gen_parse_fill_fops(eg_t *pvt, char *enable_fops)
 {
-        char            *op_no_str = NULL;
-        int              op_no = -1;
-        int              i = 0;
-        xlator_t        *this = THIS;
-        char            *saveptr = NULL;
+    char *op_no_str = NULL;
+    int op_no = -1;
+    int i = 0;
+    xlator_t *this = THIS;
+    char *saveptr = NULL;
 
-        GF_ASSERT (pvt);
-        GF_ASSERT (this);
+    GF_ASSERT(pvt);
+    GF_ASSERT(this);
 
+    for (i = 0; i < GF_FOP_MAXVALUE; i++)
+        pvt->enable[i] = 0;
+
+    if (!enable_fops) {
+        gf_log(this->name, GF_LOG_WARNING, "All fops are enabled.");
         for (i = 0; i < GF_FOP_MAXVALUE; i++)
-                pvt->enable[i] = 0;
-
-        if (!enable_fops) {
-                gf_log (this->name, GF_LOG_WARNING,
-                        "All fops are enabled.");
-                for (i = 0; i < GF_FOP_MAXVALUE; i++)
-                        pvt->enable[i] = 1;
-        } else {
-                op_no_str = strtok_r (enable_fops, ",", &saveptr);
-                while (op_no_str) {
-                        op_no = get_fop_int (&op_no_str);
-                        if (op_no == -1) {
-                                gf_log (this->name, GF_LOG_WARNING,
-                                        "Wrong option value %s", op_no_str);
-                        } else
-                                pvt->enable[op_no] = 1;
-
-                        op_no_str = strtok_r (NULL, ",", &saveptr);
-                }
+            pvt->enable[i] = 1;
+    } else {
+        op_no_str = strtok_r(enable_fops, ",", &saveptr);
+        while (op_no_str) {
+            op_no = gf_fop_int(op_no_str);
+            if (op_no == -1) {
+                gf_log(this->name, GF_LOG_WARNING, "Wrong option value %s",
+                       op_no_str);
+            } else
+                pvt->enable[op_no] = 1;
+
+            op_no_str = strtok_r(NULL, ",", &saveptr);
         }
+    }
 }
 
 int32_t
-error_gen_priv_dump (xlator_t *this)
+error_gen_priv_dump(xlator_t *this)
 {
-        char            key_prefix[GF_DUMP_MAX_BUF_LEN];
-        int             ret = -1;
-        eg_t            *conf = NULL;
+    char key_prefix[GF_DUMP_MAX_BUF_LEN];
+    int ret = -1;
+    eg_t *conf = NULL;
 
-        if (!this)
-            goto out;
+    if (!this)
+        goto out;
 
-        conf = this->private;
-        if (!conf)
-            goto out;
+    conf = this->private;
+    if (!conf)
+        goto out;
 
-        ret = TRY_LOCK(&conf->lock);
-        if (ret != 0) {
-                return ret;
-        }
+    ret = TRY_LOCK(&conf->lock);
+    if (ret != 0) {
+        return ret;
+    }
 
-        gf_proc_dump_add_section("xlator.debug.error-gen.%s.priv", this->name);
-        gf_proc_dump_build_key(key_prefix,"xlator.debug.error-gen","%s.priv",
-                               this->name);
+    gf_proc_dump_add_section("xlator.debug.error-gen.%s.priv", this->name);
+    gf_proc_dump_build_key(key_prefix, "xlator.debug.error-gen", "%s.priv",
+                           this->name);
 
-        gf_proc_dump_write("op_count", "%d", conf->op_count);
-        gf_proc_dump_write("failure_iter_no", "%d", conf->failure_iter_no);
-        gf_proc_dump_write("error_no", "%s", conf->error_no);
-        gf_proc_dump_write("random_failure", "%d", conf->random_failure);
+    gf_proc_dump_write("op_count", "%d", conf->op_count);
+    gf_proc_dump_write("failure_iter_no", "%d", conf->failure_iter_no);
+    gf_proc_dump_write("error_no", "%d", conf->error_no_int);
+    gf_proc_dump_write("random_failure", "%d", conf->random_failure);
 
-        UNLOCK(&conf->lock);
+    UNLOCK(&conf->lock);
 out:
-        return ret;
+    return ret;
 }
 
 int32_t
-mem_acct_init (xlator_t *this)
+mem_acct_init(xlator_t *this)
 {
-        int     ret = -1;
-
-        if (!this)
-                return ret;
+    int ret = -1;
 
-        ret = xlator_mem_acct_init (this, gf_error_gen_mt_end + 1);
+    if (!this)
+        return ret;
 
-        if (ret != 0) {
-                gf_log (this->name, GF_LOG_ERROR, "Memory accounting init"
-                        " failed");
-                return ret;
-        }
+    ret = xlator_mem_acct_init(this, gf_error_gen_mt_end + 1);
 
+    if (ret != 0) {
+        gf_log(this->name, GF_LOG_ERROR,
+               "Memory accounting init"
+               " failed");
         return ret;
+    }
+
+    return ret;
 }
 
 int
-reconfigure (xlator_t *this, dict_t *options)
+reconfigure(xlator_t *this, dict_t *options)
 {
-        eg_t            *pvt = NULL;
-        int32_t          ret = 0;
-        char            *error_enable_fops = NULL;
-        int32_t          failure_percent_int = 0;
+    eg_t *pvt = NULL;
+    int32_t ret = 0;
+    char *error_enable_fops = NULL;
+    char *error_no = NULL;
+    double failure_percent_dbl = 0.0;
 
-        if (!this || !this->private)
-                goto out;
+    if (!this || !this->private)
+        goto out;
 
-        pvt = this->private;
+    pvt = this->private;
 
-        GF_OPTION_RECONF ("error-no", pvt->error_no, options, str, out);
+    ret = -1;
 
-        GF_OPTION_RECONF ("failure", failure_percent_int, options, int32,
-                          out);
+    GF_OPTION_RECONF("error-no", error_no, options, str, out);
 
-        GF_OPTION_RECONF ("enable", error_enable_fops, options, str, out);
+    if (error_no)
+        pvt->error_no_int = conv_errno_to_int(&error_no);
 
-        GF_OPTION_RECONF ("random-failure", pvt->random_failure, options,
-                          bool, out);
+    GF_OPTION_RECONF("failure", failure_percent_dbl, options, percent, out);
 
-        error_gen_parse_fill_fops (pvt, error_enable_fops);
-        error_gen_set_failure (pvt, failure_percent_int);
+    GF_OPTION_RECONF("enable", error_enable_fops, options, str, out);
 
-        ret = 0;
+    GF_OPTION_RECONF("random-failure", pvt->random_failure, options, bool, out);
+
+    error_gen_parse_fill_fops(pvt, error_enable_fops);
+    error_gen_set_failure(pvt, failure_percent_dbl);
+
+    ret = 0;
 out:
-        gf_log (this->name, GF_LOG_DEBUG, "reconfigure returning %d", ret);
-        return ret;
+    gf_log(this ? this->name : "error-gen", GF_LOG_DEBUG,
+           "reconfigure returning %d", ret);
+    return ret;
 }
 
 int
-init (xlator_t *this)
+init(xlator_t *this)
 {
-        eg_t            *pvt = NULL;
-        int32_t          ret = 0;
-        char            *error_enable_fops = NULL;
-        int32_t          failure_percent_int = 0;
-
-        if (!this->children || this->children->next) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "error-gen not configured with one subvolume");
-                ret = -1;
-                goto out;
-        }
+    eg_t *pvt = NULL;
+    int32_t ret = 0;
+    char *error_enable_fops = NULL;
+    char *error_no = NULL;
+    double failure_percent_dbl = 0.0;
 
-        if (!this->parents) {
-                gf_log (this->name, GF_LOG_WARNING,
-                        "dangling volume. check volfile ");
-        }
+    if (!this->children || this->children->next) {
+        gf_log(this->name, GF_LOG_ERROR,
+               "error-gen not configured with one subvolume");
+        ret = -1;
+        goto out;
+    }
 
-        pvt = GF_CALLOC (1, sizeof (eg_t), gf_error_gen_mt_eg_t);
+    if (!this->parents) {
+        gf_log(this->name, GF_LOG_WARNING, "dangling volume. check volfile ");
+    }
 
-        if (!pvt) {
-                ret = -1;
-                goto out;
-        }
+    pvt = GF_CALLOC(1, sizeof(eg_t), gf_error_gen_mt_eg_t);
+
+    if (!pvt) {
+        ret = -1;
+        goto out;
+    }
+
+    LOCK_INIT(&pvt->lock);
 
-        LOCK_INIT (&pvt->lock);
+    ret = -1;
 
-        GF_OPTION_INIT ("error-no", pvt->error_no, str, out);
+    GF_OPTION_INIT("error-no", error_no, str, out);
 
-        GF_OPTION_INIT ("failure", failure_percent_int, int32, out);
+    if (error_no)
+        pvt->error_no_int = conv_errno_to_int(&error_no);
 
-        GF_OPTION_INIT ("enable", error_enable_fops, str, out);
+    GF_OPTION_INIT("failure", failure_percent_dbl, percent, out);
 
-        GF_OPTION_INIT ("random-failure", pvt->random_failure, bool, out);
+    GF_OPTION_INIT("enable", error_enable_fops, str, out);
 
+    GF_OPTION_INIT("random-failure", pvt->random_failure, bool, out);
 
-        error_gen_parse_fill_fops (pvt, error_enable_fops);
-        error_gen_set_failure (pvt, failure_percent_int);
+    error_gen_parse_fill_fops(pvt, error_enable_fops);
+    error_gen_set_failure(pvt, failure_percent_dbl);
 
-        this->private = pvt;
+    this->private = pvt;
 
-        /* Give some seed value here */
-        srand (time(NULL));
+    /* Give some seed value here. */
+    srand(gf_time());
+
+    ret = 0;
 out:
-        if (ret)
-                GF_FREE (pvt);
-        return ret;
+    if (ret)
+        GF_FREE(pvt);
+    return ret;
 }
 
-
 void
-fini (xlator_t *this)
+fini(xlator_t *this)
 {
-        eg_t            *pvt = NULL;
+    eg_t *pvt = NULL;
 
-        if (!this)
-                 return;
-        pvt = this->private;
-
-        if (pvt) {
-                LOCK_DESTROY (&pvt->lock);
-                GF_FREE (pvt);
-                gf_log (this->name, GF_LOG_DEBUG, "fini called");
-        }
+    if (!this)
         return;
+    pvt = this->private;
+
+    if (pvt) {
+        LOCK_DESTROY(&pvt->lock);
+        GF_FREE(pvt);
+        gf_log(this->name, GF_LOG_DEBUG, "fini called");
+    }
+    return;
 }
 
 struct xlator_dumpops dumpops = {
-        .priv = error_gen_priv_dump,
+    .priv = error_gen_priv_dump,
 };
 
-struct xlator_fops cbks;
+struct xlator_cbks cbks;
 
 struct xlator_fops fops = {
-	.lookup      = error_gen_lookup,
-	.stat        = error_gen_stat,
-	.readlink    = error_gen_readlink,
-	.mknod       = error_gen_mknod,
-	.mkdir       = error_gen_mkdir,
-	.unlink      = error_gen_unlink,
-	.rmdir       = error_gen_rmdir,
-	.symlink     = error_gen_symlink,
-	.rename      = error_gen_rename,
-	.link        = error_gen_link,
-	.truncate    = error_gen_truncate,
-	.create      = error_gen_create,
-	.open        = error_gen_open,
-	.readv       = error_gen_readv,
-	.writev      = error_gen_writev,
-	.statfs      = error_gen_statfs,
-	.flush       = error_gen_flush,
-	.fsync       = error_gen_fsync,
-	.setxattr    = error_gen_setxattr,
-	.getxattr    = error_gen_getxattr,
-	.removexattr = error_gen_removexattr,
-	.fsetxattr    = error_gen_fsetxattr,
-	.fgetxattr    = error_gen_fgetxattr,
-	.fremovexattr = error_gen_fremovexattr,
-	.opendir     = error_gen_opendir,
-	.readdir     = error_gen_readdir,
-	.readdirp    = error_gen_readdirp,
-	.fsyncdir    = error_gen_fsyncdir,
-	.access      = error_gen_access,
-	.ftruncate   = error_gen_ftruncate,
-	.fstat       = error_gen_fstat,
-	.lk          = error_gen_lk,
-	.lookup_cbk  = error_gen_lookup_cbk,
-	.xattrop     = error_gen_xattrop,
-	.fxattrop    = error_gen_fxattrop,
-	.inodelk     = error_gen_inodelk,
-	.finodelk    = error_gen_finodelk,
-	.entrylk     = error_gen_entrylk,
-	.fentrylk    = error_gen_fentrylk,
-        .setattr     = error_gen_setattr,
-        .fsetattr    = error_gen_fsetattr,
-	.getspec     = error_gen_getspec,
+    .lookup = error_gen_lookup,
+    .stat = error_gen_stat,
+    .readlink = error_gen_readlink,
+    .mknod = error_gen_mknod,
+    .mkdir = error_gen_mkdir,
+    .unlink = error_gen_unlink,
+    .rmdir = error_gen_rmdir,
+    .symlink = error_gen_symlink,
+    .rename = error_gen_rename,
+    .link = error_gen_link,
+    .truncate = error_gen_truncate,
+    .create = error_gen_create,
+    .open = error_gen_open,
+    .readv = error_gen_readv,
+    .writev = error_gen_writev,
+    .statfs = error_gen_statfs,
+    .flush = error_gen_flush,
+    .fsync = error_gen_fsync,
+    .setxattr = error_gen_setxattr,
+    .getxattr = error_gen_getxattr,
+    .removexattr = error_gen_removexattr,
+    .fsetxattr = error_gen_fsetxattr,
+    .fgetxattr = error_gen_fgetxattr,
+    .fremovexattr = error_gen_fremovexattr,
+    .opendir = error_gen_opendir,
+    .readdir = error_gen_readdir,
+    .readdirp = error_gen_readdirp,
+    .fsyncdir = error_gen_fsyncdir,
+    .access = error_gen_access,
+    .ftruncate = error_gen_ftruncate,
+    .fstat = error_gen_fstat,
+    .lk = error_gen_lk,
+    .xattrop = error_gen_xattrop,
+    .fxattrop = error_gen_fxattrop,
+    .inodelk = error_gen_inodelk,
+    .finodelk = error_gen_finodelk,
+    .entrylk = error_gen_entrylk,
+    .fentrylk = error_gen_fentrylk,
+    .setattr = error_gen_setattr,
+    .fsetattr = error_gen_fsetattr,
+    .getspec = error_gen_getspec,
 };
 
 struct volume_options options[] = {
-        { .key  = {"failure"},
-          .type = GF_OPTION_TYPE_INT,
-          .description = "Percentage failure of operations when enabled.",
-        },
-
-        { .key  = {"error-no"},
-          .value = {"ENOENT","ENOTDIR","ENAMETOOLONG","EACCES","EBADF",
-                    "EFAULT","ENOMEM","EINVAL","EIO","EEXIST","ENOSPC",
-                    "EPERM","EROFS","EBUSY","EISDIR","ENOTEMPTY","EMLINK"
-                    "ENODEV","EXDEV","EMFILE","ENFILE","ENOSYS","EINTR",
-                    "EFBIG","EAGAIN","GF_ERROR_SHORT_WRITE"},
-          .type = GF_OPTION_TYPE_STR,
-        },
-
-        { .key = {"random-failure"},
-          .type = GF_OPTION_TYPE_BOOL,
-          .default_value = "off",
-        },
-
-        { .key  = {"enable"},
-          .type = GF_OPTION_TYPE_STR,
-        },
-
-        { .key  = {NULL} }
+    {
+        .key = {"failure"},
+        .type = GF_OPTION_TYPE_PERCENT,
+        .description = "Percentage failure of operations when enabled.",
+    },
+
+    {
+        .key = {"error-no"},
+        .value = {"ENOENT",
+                  "ENOTDIR",
+                  "ENAMETOOLONG",
+                  "EACCES",
+                  "EBADF",
+                  "EFAULT",
+                  "ENOMEM",
+                  "EINVAL",
+                  "EIO",
+                  "EEXIST",
+                  "ENOSPC",
+                  "EPERM",
+                  "EROFS",
+                  "EBUSY",
+                  "EISDIR",
+                  "ENOTEMPTY",
+                  "EMLINK"
+                  "ENODEV",
+                  "EXDEV",
+                  "EMFILE",
+                  "ENFILE",
+                  "ENOSYS",
+                  "EINTR",
+                  "EFBIG",
+                  "EAGAIN",
+                  "GF_ERROR_SHORT_WRITE"},
+        .type = GF_OPTION_TYPE_STR,
+        .op_version = {3},
+        .tags = {"error-gen"},
+        .flags = OPT_FLAG_SETTABLE,
+
+    },
+
+    {
+        .key = {"random-failure"},
+        .type = GF_OPTION_TYPE_BOOL,
+        .default_value = "off",
+        .op_version = {3},
+        .tags = {"error-gen"},
+        .flags = OPT_FLAG_SETTABLE,
+    },
+
+    {
+        .key = {"enable", "error-fops"},
+        .type = GF_OPTION_TYPE_STR,
+        .description = "Accepts a string which takes ',' separated fop "
+                       "strings to denote which fops are enabled for error",
+        .op_version = {3},
+        .tags = {"error-gen"},
+        .flags = OPT_FLAG_SETTABLE,
+    },
+
+    {.key = {NULL}},
+};
+
+xlator_api_t xlator_api = {
+    .init = init,
+    .fini = fini,
+    .reconfigure = reconfigure,
+    .mem_acct_init = mem_acct_init,
+    .op_version = {1},
+    .dumpops = &dumpops,
+    .fops = &fops,
+    .cbks = &cbks,
+    .options = options,
+    .identifier = "error-gen",
+    .category = GF_TECH_PREVIEW,
 };
diff --git a/xlators/debug/error-gen/src/error-gen.h b/xlators/debug/error-gen/src/error-gen.h
index d92c2306240..2478cd5b21c 100644
--- a/xlators/debug/error-gen/src/error-gen.h
+++ b/xlators/debug/error-gen/src/error-gen.h
@@ -10,11 +10,6 @@
 #ifndef _ERROR_GEN_H
 #define _ERROR_GEN_H
 
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
-
 #include "error-gen-mem-types.h"
 
 #define GF_FAILURE_DEFAULT 10
@@ -27,22 +22,28 @@
  * range.
  */
 enum GF_PSEUDO_ERRORS {
-	GF_ERROR_SHORT_WRITE = 1000,	/* short writev return value */
-	GF_ERROR_MAX
+    GF_ERROR_SHORT_WRITE = 1000, /* short writev return value */
+    GF_ERROR_MAX
 };
 
 typedef struct {
-        int enable[GF_FOP_MAXVALUE];
-        int op_count;
-        int failure_iter_no;
-        char *error_no;
-        gf_boolean_t random_failure;
-        gf_lock_t lock;
+    int enable[GF_FOP_MAXVALUE];
+    int op_count;
+    /*
+     * This is only an iteration number in the random-failure case.  For
+     * the normal controlled-probability case, it's actually a numerator
+     * for the failure probability (see FAILURE_GRANULARITY declaration).
+     * It's just not worth blowing up the diff by changing it.
+     */
+    int failure_iter_no;
+    int error_no_int;
+    gf_boolean_t random_failure;
+    gf_lock_t lock;
 } eg_t;
 
 typedef struct {
-        int error_no_count;
-	int error_no[20];
+    int error_no_count;
+    int error_no[20];
 } sys_error_t;
 
 #endif
diff --git a/xlators/debug/io-stats/src/Makefile.am b/xlators/debug/io-stats/src/Makefile.am
index dff294cd84e..c69f3caf0fe 100644
--- a/xlators/debug/io-stats/src/Makefile.am
+++ b/xlators/debug/io-stats/src/Makefile.am
@@ -2,7 +2,7 @@
 xlator_LTLIBRARIES = io-stats.la
 xlatordir = $(libdir)/glusterfs/$(PACKAGE_VERSION)/xlator/debug
 
-io_stats_la_LDFLAGS = -module -avoid-version
+io_stats_la_LDFLAGS = -module $(GF_XLATOR_DEFAULT_LDFLAGS)
 
 io_stats_la_SOURCES = io-stats.c
 io_stats_la_LIBADD = $(top_builddir)/libglusterfs/src/libglusterfs.la
@@ -10,8 +10,9 @@ io_stats_la_LIBADD = $(top_builddir)/libglusterfs/src/libglusterfs.la
 noinst_HEADERS = io-stats-mem-types.h
 
 AM_CPPFLAGS = $(GF_CPPFLAGS) -I$(top_srcdir)/libglusterfs/src \
-	-I$(top_srcdir)/rpc/xdr/src \
-	-I$(top_srcdir)/rpc/rpc-lib/src
+	-I$(top_srcdir)/rpc/xdr/src -I$(top_builddir)/rpc/xdr/src \
+	-I$(top_srcdir)/rpc/rpc-lib/src \
+	-DDATADIR=\"$(localstatedir)\"
 
 AM_CFLAGS = -Wall $(GF_CFLAGS)
 
diff --git a/xlators/debug/io-stats/src/io-stats-mem-types.h b/xlators/debug/io-stats/src/io-stats-mem-types.h
index c30dfb17e7c..51d38d8b97c 100644
--- a/xlators/debug/io-stats/src/io-stats-mem-types.h
+++ b/xlators/debug/io-stats/src/io-stats-mem-types.h
@@ -11,14 +11,17 @@
 #ifndef __IO_STATS_MEM_TYPES_H__
 #define __IO_STATS_MEM_TYPES_H__
 
-#include "mem-types.h"
+#include <glusterfs/mem-types.h>
+
+extern const char *__progname;
 
 enum gf_io_stats_mem_types_ {
-        gf_io_stats_mt_ios_conf = gf_common_mt_end + 1,
-        gf_io_stats_mt_ios_fd,
-        gf_io_stats_mt_ios_stat,
-        gf_io_stats_mt_ios_stat_list,
-        gf_io_stats_mt_end
+    gf_io_stats_mt_ios_conf = gf_common_mt_end + 1,
+    gf_io_stats_mt_ios_fd,
+    gf_io_stats_mt_ios_stat,
+    gf_io_stats_mt_ios_stat_list,
+    gf_io_stats_mt_ios_sample_buf,
+    gf_io_stats_mt_ios_sample,
+    gf_io_stats_mt_end
 };
 #endif
-
diff --git a/xlators/debug/io-stats/src/io-stats.c b/xlators/debug/io-stats/src/io-stats.c
index f79db1ad19a..aa00c446e5a 100644
--- a/xlators/debug/io-stats/src/io-stats.c
+++ b/xlators/debug/io-stats/src/io-stats.c
@@ -7,11 +7,8 @@
    later), or the GNU General Public License, version 2 (GPLv2), in all
    cases as published by the Free Software Foundation.
 */
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#include "xlator.h"
-#endif
+#include <glusterfs/xlator.h>
+#include <glusterfs/syscall.h>
 
 /**
  * xlators/debug/io_stats :
@@ -20,3236 +17,4464 @@
  *
  *  a) total read data - since process start, last interval and per fd
  *  b) total write data - since process start, last interval and per fd
- *  c) counts of read IO block size - since process start, last interval and per fd
- *  d) counts of write IO block size - since process start, last interval and per fd
- *  e) counts of all FOP types passing through it
+ *  c) counts of read IO block size - since process start, last interval and per
+ * fd d) counts of write IO block size - since process start, last interval and
+ * per fd e) counts of all FOP types passing through it
  *
- *  Usage: setfattr -n io-stats-dump /tmp/filename /mnt/gluster
+ *  Usage: setfattr -n trusted.io-stats-dump /tmp/filename /mnt/gluster
+ *      output is written to /tmp/filename.<iostats xlator instance name>
  *
  */
 
 #include <fnmatch.h>
 #include <errno.h>
-#include "glusterfs.h"
-#include "xlator.h"
+#include <glusterfs/glusterfs.h>
+#include <glusterfs/xlator.h>
 #include "io-stats-mem-types.h"
 #include <stdarg.h>
-#include "defaults.h"
-#include "logging.h"
-#include "cli1-xdr.h"
-#include "statedump.h"
+#include <glusterfs/defaults.h>
+#include <glusterfs/logging.h>
+#include <glusterfs/statedump.h>
+#include <glusterfs/syncop.h>
+#include <pwd.h>
+#include <grp.h>
+#include <glusterfs/upcall-utils.h>
+#include <glusterfs/async.h>
 
 #define MAX_LIST_MEMBERS 100
+#define DEFAULT_PWD_BUF_SZ 16384
+#define DEFAULT_GRP_BUF_SZ 16384
+#define IOS_BLOCK_COUNT_SIZE 32
+
+#define IOS_STATS_DUMP_DIR DEFAULT_VAR_RUN_DIRECTORY
 
 typedef enum {
-        IOS_STATS_TYPE_NONE,
-        IOS_STATS_TYPE_OPEN,
-        IOS_STATS_TYPE_READ,
-        IOS_STATS_TYPE_WRITE,
-        IOS_STATS_TYPE_OPENDIR,
-        IOS_STATS_TYPE_READDIRP,
-        IOS_STATS_TYPE_READ_THROUGHPUT,
-        IOS_STATS_TYPE_WRITE_THROUGHPUT,
-        IOS_STATS_TYPE_MAX
-}ios_stats_type_t;
+    IOS_STATS_TYPE_NONE,
+    IOS_STATS_TYPE_OPEN,
+    IOS_STATS_TYPE_READ,
+    IOS_STATS_TYPE_WRITE,
+    IOS_STATS_TYPE_OPENDIR,
+    IOS_STATS_TYPE_READDIRP,
+    IOS_STATS_TYPE_READ_THROUGHPUT,
+    IOS_STATS_TYPE_WRITE_THROUGHPUT,
+    IOS_STATS_TYPE_MAX
+} ios_stats_type_t;
 
 typedef enum {
-        IOS_STATS_THRU_READ,
-        IOS_STATS_THRU_WRITE,
-        IOS_STATS_THRU_MAX,
-}ios_stats_thru_t;
+    IOS_STATS_THRU_READ,
+    IOS_STATS_THRU_WRITE,
+    IOS_STATS_THRU_MAX,
+} ios_stats_thru_t;
+
+/* This is same as gf1_cli_info_op */
+/* had to be defined here again, so we have modularity between
+ xdr, xlator, and library functions */
+typedef enum ios_info_op {
+    GF_IOS_INFO_NONE = 0,
+    GF_IOS_INFO_ALL = 1,
+    GF_IOS_INFO_INCREMENTAL = 2,
+    GF_IOS_INFO_CUMULATIVE = 3,
+    GF_IOS_INFO_CLEAR = 4,
+} ios_info_op_t;
 
 struct ios_stat_lat {
-        struct timeval  time;
-        double          throughput;
+    struct timeval time;
+    double throughput;
 };
 
 struct ios_stat {
-        gf_lock_t       lock;
-        uuid_t          gfid;
-        char           *filename;
-        uint64_t        counters [IOS_STATS_TYPE_MAX];
-        struct ios_stat_lat thru_counters [IOS_STATS_THRU_MAX];
-        int             refcnt;
+    gf_lock_t lock;
+    uuid_t gfid;
+    char *filename;
+    gf_atomic_t counters[IOS_STATS_TYPE_MAX];
+    struct ios_stat_lat thru_counters[IOS_STATS_THRU_MAX];
+    gf_atomic_t refcnt;
 };
 
 struct ios_stat_list {
-        struct list_head  list;
-        struct ios_stat  *iosstat;
-        double            value;
+    struct list_head list;
+    struct ios_stat *iosstat;
+    double value;
 };
 
 struct ios_stat_head {
-       gf_lock_t                lock;
-       double                   min_cnt;
-       uint64_t                 members;
-       struct ios_stat_list    *iosstats;
+    gf_lock_t lock;
+    double min_cnt;
+    uint64_t members;
+    struct ios_stat_list *iosstats;
 };
 
+typedef struct _ios_sample_t {
+    uid_t uid;
+    gid_t gid;
+    char identifier[UNIX_PATH_MAX];
+    glusterfs_fop_t fop_type;
+    struct timeval timestamp;
+    double elapsed;
+} ios_sample_t;
+
+typedef struct _ios_sample_buf_t {
+    uint64_t pos;              /* Position in write buffer */
+    uint64_t size;             /* Size of ring buffer */
+    uint64_t collected;        /* Number of samples we've collected */
+    uint64_t observed;         /* Number of FOPs we've observed */
+    ios_sample_t *ios_samples; /* Our list of samples */
+} ios_sample_buf_t;
+
 struct ios_lat {
-        double      min;
-        double      max;
-        double      avg;
-        uint64_t    total;
+    double min;
+    double max;
+    double avg;
+    uint64_t total;
 };
 
 struct ios_global_stats {
-        uint64_t        data_written;
-        uint64_t        data_read;
-        uint64_t        block_count_write[32];
-        uint64_t        block_count_read[32];
-        uint64_t        fop_hits[GF_FOP_MAXVALUE];
-        struct timeval  started_at;
-        struct ios_lat  latency[GF_FOP_MAXVALUE];
-        uint64_t        nr_opens;
-        uint64_t        max_nr_opens;
-        struct timeval  max_openfd_time;
+    gf_atomic_t data_written;
+    gf_atomic_t data_read;
+    gf_atomic_t block_count_write[IOS_BLOCK_COUNT_SIZE];
+    gf_atomic_t block_count_read[IOS_BLOCK_COUNT_SIZE];
+    gf_atomic_t fop_hits[GF_FOP_MAXVALUE];
+    gf_atomic_t upcall_hits[GF_UPCALL_FLAGS_MAXVALUE];
+    time_t started_at;
+    struct ios_lat latency[GF_FOP_MAXVALUE];
+    uint64_t nr_opens;
+    uint64_t max_nr_opens;
+    struct timeval max_openfd_time;
 };
 
+typedef enum {
+    IOS_DUMP_TYPE_NONE = 0,
+    IOS_DUMP_TYPE_FILE = 1,
+    IOS_DUMP_TYPE_DICT = 2,
+    IOS_DUMP_TYPE_JSON_FILE = 3,
+    IOS_DUMP_TYPE_SAMPLES = 4,
+    IOS_DUMP_TYPE_MAX = 5
+} ios_dump_type_t;
 
 struct ios_conf {
-        gf_lock_t                 lock;
-        struct ios_global_stats   cumulative;
-        uint64_t                  increment;
-        struct ios_global_stats   incremental;
-        gf_boolean_t              dump_fd_stats;
-        gf_boolean_t              count_fop_hits;
-        gf_boolean_t              measure_latency;
-        struct ios_stat_head      list[IOS_STATS_TYPE_MAX];
-        struct ios_stat_head      thru_list[IOS_STATS_THRU_MAX];
+    gf_lock_t lock;
+    struct ios_global_stats cumulative;
+    uint64_t increment;
+    struct ios_global_stats incremental;
+    gf_boolean_t dump_fd_stats;
+    gf_boolean_t count_fop_hits;
+    gf_boolean_t measure_latency;
+    struct ios_stat_head list[IOS_STATS_TYPE_MAX];
+    struct ios_stat_head thru_list[IOS_STATS_THRU_MAX];
+    int32_t ios_dump_interval;
+    pthread_t dump_thread;
+    gf_boolean_t dump_thread_should_die;
+    gf_boolean_t dump_thread_running;
+    gf_lock_t ios_sampling_lock;
+    int32_t ios_sample_interval;
+    int32_t ios_sample_buf_size;
+    ios_sample_buf_t *ios_sample_buf;
+    struct dnscache *dnscache;
+    int32_t ios_dnscache_ttl_sec;
+    /*
+     * What we really need here is just a unique value to keep files
+     * created by this instance distinct from those created by any other.
+     * On the client side this isn't a problem, so we just use the
+     * translator name.  On the server side conflicts can occur, so the
+     * volfile-generation code automatically sets this (via an option)
+     * to be the brick path.
+     *
+     * NB While the *field* name has changed, it didn't seem worth changing
+     * all of the cases where "xlator_name" is used as a *variable* name.
+     */
+    char *unique_id;
+    ios_dump_type_t dump_format;
 };
 
-
 struct ios_fd {
-        char           *filename;
-        uint64_t        data_written;
-        uint64_t        data_read;
-        uint64_t        block_count_write[32];
-        uint64_t        block_count_read[32];
-        struct timeval  opened_at;
+    char *filename;
+    gf_atomic_t data_written;
+    gf_atomic_t data_read;
+    gf_atomic_t block_count_write[IOS_BLOCK_COUNT_SIZE];
+    gf_atomic_t block_count_read[IOS_BLOCK_COUNT_SIZE];
+    struct timeval opened_at;
 };
 
-typedef enum {
-        IOS_DUMP_TYPE_NONE = 0,
-        IOS_DUMP_TYPE_FILE = 1,
-        IOS_DUMP_TYPE_DICT = 2,
-        IOS_DUMP_TYPE_MAX  = 3
-} ios_dump_type_t;
-
 struct ios_dump_args {
-        ios_dump_type_t type;
-        union {
-                FILE *logfp;
-                dict_t *dict;
-        } u;
+    ios_dump_type_t type;
+    union {
+        FILE *logfp;
+        dict_t *dict;
+    } u;
 };
 
-typedef int (*block_dump_func) (xlator_t *, struct ios_dump_args*,
-                                    int , int , uint64_t ) ;
+typedef int (*block_dump_func)(xlator_t *, struct ios_dump_args *, int, int,
+                               uint64_t);
 
 struct ios_local {
-        struct timeval  wind_at;
-        struct timeval  unwind_at;
+    struct timeval wind_at;
+    struct timeval unwind_at;
 };
 
 struct volume_options options[];
 
-inline static int
-is_fop_latency_started (call_frame_t *frame)
-{
-        GF_ASSERT (frame);
-        struct timeval epoch = {0,};
-        return memcmp (&frame->begin, &epoch, sizeof (epoch));
-}
-
-#define END_FOP_LATENCY(frame, op)                                      \
-        do {                                                            \
-                struct ios_conf  *conf = NULL;                          \
-                                                                        \
-                conf = this->private;                                   \
-                if (conf && conf->measure_latency) {                    \
-                        gettimeofday (&frame->end, NULL);               \
-                        update_ios_latency (conf, frame, GF_FOP_##op);  \
-                }                                                       \
-        } while (0)
-
-#define START_FOP_LATENCY(frame)                                         \
-        do {                                                             \
-                struct ios_conf  *conf = NULL;                           \
-                                                                         \
-                conf = this->private;                                    \
-                if (conf && conf->measure_latency) {                     \
-                        gettimeofday (&frame->begin, NULL);              \
-                } else {                                                 \
-                        memset (&frame->begin, 0, sizeof (frame->begin));\
-                }                                                        \
-        } while (0)
-
-
-#define BUMP_FOP(op)                                                    \
-        do {                                                            \
-                struct ios_conf  *conf = NULL;                          \
-                                                                        \
-                conf = this->private;                                   \
-                if (!conf)                                              \
-                        break;                                          \
-                conf->cumulative.fop_hits[GF_FOP_##op]++;               \
-                conf->incremental.fop_hits[GF_FOP_##op]++;              \
-        } while (0)
-
-#define UPDATE_PROFILE_STATS(frame, op)                                       \
-        do {                                                                  \
-                struct ios_conf  *conf = NULL;                                \
-                                                                              \
-                if (!is_fop_latency_started (frame))                          \
-                        break;                                                \
-                conf = this->private;                                         \
-                LOCK (&conf->lock);                                           \
-                {                                                             \
-                        if (conf && conf->measure_latency &&                  \
-                            conf->count_fop_hits) {                           \
-                                BUMP_FOP(op);                                 \
-                                gettimeofday (&frame->end, NULL);             \
-                                update_ios_latency (conf, frame, GF_FOP_##op);\
-                        }                                                     \
-                }                                                             \
-                UNLOCK (&conf->lock);                                         \
-        } while (0)
-
-#define BUMP_READ(fd, len)                                              \
-        do {                                                            \
-                struct ios_conf  *conf = NULL;                          \
-                struct ios_fd    *iosfd = NULL;                         \
-                int               lb2 = 0;                              \
-                                                                        \
-                conf = this->private;                                   \
-                lb2 = log_base2 (len);                                  \
-                ios_fd_ctx_get (fd, this, &iosfd);                      \
-                if (!conf)                                              \
-                        break;                                          \
-                                                                        \
-                LOCK (&conf->lock);                                     \
-                {                                                       \
-                        conf->cumulative.data_read += len;              \
-                        conf->incremental.data_read += len;             \
-                        conf->cumulative.block_count_read[lb2]++;       \
-                        conf->incremental.block_count_read[lb2]++;      \
-                                                                        \
-                        if (iosfd) {                                    \
-                                iosfd->data_read += len;                \
-                                iosfd->block_count_read[lb2]++;         \
-                        }                                               \
-                }                                                       \
-                UNLOCK (&conf->lock);                                   \
-        } while (0)
-
-
-#define BUMP_WRITE(fd, len)                                             \
-        do {                                                            \
-                struct ios_conf  *conf = NULL;                          \
-                struct ios_fd    *iosfd = NULL;                         \
-                int               lb2 = 0;                              \
-                                                                        \
-                conf = this->private;                                   \
-                lb2 = log_base2 (len);                                  \
-                ios_fd_ctx_get (fd, this, &iosfd);                      \
-                if (!conf)                                              \
-                        break;                                          \
-                                                                        \
-                LOCK (&conf->lock);                                     \
-                {                                                       \
-                        conf->cumulative.data_written += len;           \
-                        conf->incremental.data_written += len;          \
-                        conf->cumulative.block_count_write[lb2]++;      \
-                        conf->incremental.block_count_write[lb2]++;     \
-                                                                        \
-                        if (iosfd) {                                    \
-                                iosfd->data_written += len;             \
-                                iosfd->block_count_write[lb2]++;        \
-                        }                                               \
-                }                                                       \
-                UNLOCK (&conf->lock);                                   \
-        } while (0)
-
-
-#define BUMP_STATS(iosstat, type)                                               \
-        do {                                                                    \
-                struct ios_conf         *conf = NULL;                           \
-                uint64_t                 value = 0;                             \
-                                                                                \
-                conf = this->private;                                           \
-                                                                                \
-                LOCK(&iosstat->lock);                                           \
-                {                                                               \
-                        iosstat->counters[type]++;                              \
-                        value = iosstat->counters[type];                        \
-                }                                                               \
-                UNLOCK (&iosstat->lock);                                        \
-                ios_stat_add_to_list (&conf->list[type],                        \
-                                     value, iosstat);                           \
-                                                                                \
-        } while (0)
-
-
-#define BUMP_THROUGHPUT(iosstat, type)						\
-        do {									\
-                struct ios_conf         *conf = NULL;				\
-                double                   elapsed;				\
-                struct timeval          *begin, *end;				\
-                double                   throughput;				\
-               int                      flag = 0;                              \
-                                                                                \
-                begin = &frame->begin;						\
-                end   = &frame->end;						\
-                                                                                \
-                elapsed = (end->tv_sec - begin->tv_sec) * 1e6			\
-                        + (end->tv_usec - begin->tv_usec);			\
-                throughput = op_ret / elapsed;					\
-                                                                                \
-                conf = this->private;						\
-                LOCK(&iosstat->lock);						\
-                {								\
-                        if (iosstat->thru_counters[type].throughput             \
-                                <= throughput) {                                \
-                                iosstat->thru_counters[type].throughput =       \
-                                                                throughput;     \
-                                gettimeofday (&iosstat->                        \
-                                             thru_counters[type].time, NULL);   \
-                               flag = 1;                                       \
-                        }							\
-                }								\
-                UNLOCK (&iosstat->lock);					\
-               if (flag)                                                       \
-                       ios_stat_add_to_list (&conf->thru_list[type],           \
-                                               throughput, iosstat);           \
-        } while (0)
-
-int
-ios_fd_ctx_get (fd_t *fd, xlator_t *this, struct ios_fd **iosfd)
-{
-        uint64_t      iosfd64 = 0;
-        unsigned long iosfdlong = 0;
-        int           ret = 0;
-
-        ret = fd_ctx_get (fd, this, &iosfd64);
-        iosfdlong = iosfd64;
-        if (ret != -1)
-                *iosfd = (void *) iosfdlong;
-
-        return ret;
+static int
+is_fop_latency_started(call_frame_t *frame)
+{
+    GF_ASSERT(frame);
+    struct timeval epoch = {
+        0,
+    };
+    return memcmp(&frame->begin, &epoch, sizeof(epoch));
 }
 
+#define _IOS_SAMP_DIR DEFAULT_LOG_FILE_DIRECTORY "/samples"
+#ifdef GF_LINUX_HOST_OS
+#define _IOS_DUMP_DIR DATADIR "/lib/glusterd/stats"
+#else
+#define _IOS_DUMP_DIR DATADIR "/db/glusterd/stats"
+#endif
 
+#define END_FOP_LATENCY(frame, op)                                             \
+    do {                                                                       \
+        struct ios_conf *conf = NULL;                                          \
+                                                                               \
+        conf = this->private;                                                  \
+        if (conf && conf->measure_latency) {                                   \
+            timespec_now(&frame->end);                                         \
+            update_ios_latency(conf, frame, GF_FOP_##op);                      \
+        }                                                                      \
+    } while (0)
+
+#define START_FOP_LATENCY(frame)                                               \
+    do {                                                                       \
+        struct ios_conf *conf = NULL;                                          \
+                                                                               \
+        conf = this->private;                                                  \
+        if (conf && conf->measure_latency) {                                   \
+            timespec_now(&frame->begin);                                       \
+        } else {                                                               \
+            memset(&frame->begin, 0, sizeof(frame->begin));                    \
+        }                                                                      \
+    } while (0)
+
+#define BUMP_FOP(op)                                                           \
+    do {                                                                       \
+        struct ios_conf *conf = NULL;                                          \
+                                                                               \
+        conf = this->private;                                                  \
+        if (!conf)                                                             \
+            break;                                                             \
+        GF_ATOMIC_INC(conf->cumulative.fop_hits[GF_FOP_##op]);                 \
+        GF_ATOMIC_INC(conf->incremental.fop_hits[GF_FOP_##op]);                \
+    } while (0)
+
+#define UPDATE_PROFILE_STATS(frame, op)                                        \
+    do {                                                                       \
+        struct ios_conf *conf = NULL;                                          \
+                                                                               \
+        if (!is_fop_latency_started(frame))                                    \
+            break;                                                             \
+        conf = this->private;                                                  \
+        if (conf && conf->measure_latency && conf->count_fop_hits) {           \
+            BUMP_FOP(op);                                                      \
+            timespec_now(&frame->end);                                         \
+            update_ios_latency(conf, frame, GF_FOP_##op);                      \
+        }                                                                      \
+    } while (0)
+
+#define BUMP_THROUGHPUT(iosstat, type)                                         \
+    do {                                                                       \
+        struct ios_conf *conf = NULL;                                          \
+        double elapsed;                                                        \
+        struct timespec *begin, *end;                                          \
+        double throughput;                                                     \
+        int flag = 0;                                                          \
+        struct timeval tv = {                                                  \
+            0,                                                                 \
+        };                                                                     \
+                                                                               \
+        begin = &frame->begin;                                                 \
+        end = &frame->end;                                                     \
+                                                                               \
+        elapsed = gf_tsdiff(begin, end) / 1000.0;                              \
+        throughput = op_ret / elapsed;                                         \
+                                                                               \
+        conf = this->private;                                                  \
+        gettimeofday(&tv, NULL);                                               \
+        LOCK(&iosstat->lock);                                                  \
+        {                                                                      \
+            if (iosstat->thru_counters[type].throughput <= throughput) {       \
+                iosstat->thru_counters[type].throughput = throughput;          \
+                memcpy(&iosstat->thru_counters[type].time, &tv,                \
+                       sizeof(struct timeval));                                \
+                flag = 1;                                                      \
+            }                                                                  \
+        }                                                                      \
+        UNLOCK(&iosstat->lock);                                                \
+        if (flag)                                                              \
+            ios_stat_add_to_list(&conf->thru_list[type], throughput, iosstat); \
+    } while (0)
 
-int
-ios_fd_ctx_set (fd_t *fd, xlator_t *this, struct ios_fd *iosfd)
+static int
+ios_fd_ctx_get(fd_t *fd, xlator_t *this, struct ios_fd **iosfd)
 {
-        uint64_t   iosfd64 = 0;
-        int        ret = 0;
+    uint64_t iosfd64 = 0;
+    unsigned long iosfdlong = 0;
+    int ret = 0;
 
-        iosfd64 = (unsigned long) iosfd;
-        ret = fd_ctx_set (fd, this, iosfd64);
+    ret = fd_ctx_get(fd, this, &iosfd64);
+    iosfdlong = iosfd64;
+    if (ret != -1)
+        *iosfd = (void *)iosfdlong;
 
-        return ret;
+    return ret;
 }
 
-int
-ios_stat_ref (struct ios_stat *iosstat)
+static int
+ios_fd_ctx_set(fd_t *fd, xlator_t *this, struct ios_fd *iosfd)
 {
-        LOCK (&iosstat->lock);
-        {
-                iosstat->refcnt++;
+    uint64_t iosfd64 = 0;
+    int ret = 0;
+
+    iosfd64 = (unsigned long)iosfd;
+    ret = fd_ctx_set(fd, this, iosfd64);
+
+    return ret;
+}
+
+static int
+ios_stat_ref(struct ios_stat *iosstat)
+{
+    uint64_t refcnt = 0;
+    refcnt = GF_ATOMIC_INC(iosstat->refcnt);
+
+    return refcnt;
+}
+
+static int
+ios_stat_unref(struct ios_stat *iosstat)
+{
+    int cleanup = 0;
+    uint64_t refcnt = 0;
+
+    refcnt = GF_ATOMIC_DEC(iosstat->refcnt);
+    if (refcnt == 0) {
+        if (iosstat->filename) {
+            GF_FREE(iosstat->filename);
+            iosstat->filename = NULL;
         }
-        UNLOCK (&iosstat->lock);
+        cleanup = 1;
+    }
+
+    if (cleanup) {
+        LOCK_DESTROY(&iosstat->lock);
+        GF_FREE(iosstat);
+        iosstat = NULL;
+    }
 
-        return iosstat->refcnt;
+    return 0;
 }
 
-int
-ios_stat_unref (struct ios_stat *iosstat)
+static int
+ios_stat_add_to_list(struct ios_stat_head *list_head, uint64_t value,
+                     struct ios_stat *iosstat)
 {
-        int cleanup = 0;
-        LOCK (&iosstat->lock);
+    struct ios_stat_list *new = NULL;
+    struct ios_stat_list *entry = NULL;
+    struct ios_stat_list *t = NULL;
+    struct ios_stat_list *list_entry = NULL;
+    struct ios_stat_list *tmp = NULL;
+    struct ios_stat_list *last = NULL;
+    struct ios_stat *stat = NULL;
+    int cnt = 0;
+    int found = 0;
+    int reposition = 0;
+    double min_count = 0;
+
+    LOCK(&list_head->lock);
+    {
+        if (list_head->min_cnt == 0)
+            list_head->min_cnt = value;
+        if ((list_head->members == MAX_LIST_MEMBERS) &&
+            (list_head->min_cnt > value))
+            goto out;
+
+        list_for_each_entry_safe(entry, t, &list_head->iosstats->list, list)
         {
-                iosstat->refcnt--;
-                if (iosstat->refcnt == 0) {
-                        if (iosstat->filename) {
-                                GF_FREE (iosstat->filename);
-                                iosstat->filename = NULL;
-                        }
-                        cleanup = 1;
+            cnt++;
+            if (cnt == list_head->members)
+                last = entry;
+
+            if (!gf_uuid_compare(iosstat->gfid, entry->iosstat->gfid)) {
+                list_entry = entry;
+                found = cnt;
+                entry->value = value;
+                if (!reposition) {
+                    if (cnt == list_head->members)
+                        list_head->min_cnt = value;
+                    goto out;
                 }
+                break;
+            } else if (entry->value <= value && !reposition) {
+                reposition = cnt;
+                tmp = entry;
+                if (cnt == list_head->members - 1)
+                    min_count = entry->value;
+            }
         }
-        UNLOCK (&iosstat->lock);
-
-        if (cleanup) {
-                LOCK_DESTROY (&iosstat->lock);
-                GF_FREE (iosstat);
-                iosstat = NULL;
+        if (found) {
+            list_del(&list_entry->list);
+            list_add_tail(&list_entry->list, &tmp->list);
+            if (min_count)
+                list_head->min_cnt = min_count;
+            goto out;
+        } else if (list_head->members == MAX_LIST_MEMBERS && reposition) {
+            new = GF_CALLOC(1, sizeof(*new), gf_io_stats_mt_ios_stat_list);
+            new->iosstat = iosstat;
+            new->value = value;
+            ios_stat_ref(iosstat);
+            list_add_tail(&new->list, &tmp->list);
+            if (last) {
+                stat = last->iosstat;
+                last->iosstat = NULL;
+                ios_stat_unref(stat);
+                list_del(&last->list);
+                GF_FREE(last);
+            }
+            if (reposition == MAX_LIST_MEMBERS)
+                list_head->min_cnt = value;
+            else if (min_count) {
+                list_head->min_cnt = min_count;
+            }
+        } else if (list_head->members < MAX_LIST_MEMBERS) {
+            new = GF_CALLOC(1, sizeof(*new), gf_io_stats_mt_ios_stat_list);
+            new->iosstat = iosstat;
+            new->value = value;
+            ios_stat_ref(iosstat);
+            if (reposition) {
+                list_add_tail(&new->list, &tmp->list);
+            } else {
+                list_add_tail(&new->list, &entry->list);
+            }
+            list_head->members++;
+            if (list_head->min_cnt > value)
+                list_head->min_cnt = value;
         }
+    }
+out:
+    UNLOCK(&list_head->lock);
+    return 0;
+}
 
-        return 0;
+static void
+ios_bump_read(xlator_t *this, fd_t *fd, size_t len)
+{
+    struct ios_conf *conf = NULL;
+    struct ios_fd *iosfd = NULL;
+    int lb2 = 0;
+
+    conf = this->private;
+    lb2 = log_base2(len);
+    ios_fd_ctx_get(fd, this, &iosfd);
+    if (!conf)
+        return;
+
+    GF_ATOMIC_ADD(conf->cumulative.data_read, len);
+    GF_ATOMIC_ADD(conf->incremental.data_read, len);
+    GF_ATOMIC_INC(conf->cumulative.block_count_read[lb2]);
+    GF_ATOMIC_INC(conf->incremental.block_count_read[lb2]);
+
+    if (iosfd) {
+        GF_ATOMIC_ADD(iosfd->data_read, len);
+        GF_ATOMIC_INC(iosfd->block_count_read[lb2]);
+    }
 }
 
-int
-ios_inode_ctx_set (inode_t *inode, xlator_t *this, struct ios_stat *iosstat)
+static void
+ios_bump_write(xlator_t *this, fd_t *fd, size_t len)
 {
-        uint64_t   iosstat64 = 0;
-        int        ret     = 0;
+    struct ios_conf *conf = NULL;
+    struct ios_fd *iosfd = NULL;
+    int lb2 = 0;
+
+    conf = this->private;
+    lb2 = log_base2(len);
+    ios_fd_ctx_get(fd, this, &iosfd);
+    if (!conf)
+        return;
 
-        ios_stat_ref (iosstat);
-        iosstat64 = (unsigned long )iosstat;
-        ret = inode_ctx_put (inode, this, iosstat64);
-        return ret;
+    GF_ATOMIC_ADD(conf->cumulative.data_written, len);
+    GF_ATOMIC_ADD(conf->incremental.data_written, len);
+    GF_ATOMIC_INC(conf->cumulative.block_count_write[lb2]);
+    GF_ATOMIC_INC(conf->incremental.block_count_write[lb2]);
+
+    if (iosfd) {
+        GF_ATOMIC_ADD(iosfd->data_written, len);
+        GF_ATOMIC_INC(iosfd->block_count_write[lb2]);
+    }
 }
 
-int
-ios_inode_ctx_get (inode_t *inode, xlator_t *this, struct ios_stat **iosstat)
+static void
+ios_bump_upcall(xlator_t *this, gf_upcall_flags_t event)
 {
-        uint64_t      iosstat64 = 0;
-        unsigned long iosstatlong = 0;
-        int           ret = 0;
+    struct ios_conf *conf = NULL;
+
+    conf = this->private;
+    if (!conf)
+        return;
+    if (conf->count_fop_hits) {
+        GF_ATOMIC_INC(conf->cumulative.upcall_hits[event]);
+        GF_ATOMIC_INC(conf->incremental.upcall_hits[event]);
+    }
+}
 
-        ret = inode_ctx_get (inode, this, &iosstat64);
-        iosstatlong = iosstat64;
-        if (ret != -1)
-                *iosstat = (void *) iosstatlong;
+static void
+ios_bump_stats(xlator_t *this, struct ios_stat *iosstat, ios_stats_type_t type)
+{
+    struct ios_conf *conf = NULL;
+    uint64_t value = 0;
 
-        return ret;
+    conf = this->private;
 
+    value = GF_ATOMIC_INC(iosstat->counters[type]);
+    ios_stat_add_to_list(&conf->list[type], value, iosstat);
 }
 
 int
-ios_stat_add_to_list (struct ios_stat_head *list_head, uint64_t value,
-                            struct ios_stat *iosstat)
+ios_inode_ctx_set(inode_t *inode, xlator_t *this, struct ios_stat *iosstat)
 {
-        struct ios_stat_list *new = NULL;
-        struct ios_stat_list *entry = NULL;
-        struct ios_stat_list *t = NULL;
-        struct ios_stat_list *list_entry = NULL;
-        struct ios_stat_list *tmp = NULL;
-        struct ios_stat_list *last = NULL;
-        struct ios_stat      *stat = NULL;
-        int                   cnt = 0;
-        int                   found = 0;
-        int                   reposition = 0;
-        double                min_count = 0;
+    uint64_t iosstat64 = 0;
+    int ret = 0;
 
-        LOCK (&list_head->lock);
-        {
+    ios_stat_ref(iosstat);
+    iosstat64 = (unsigned long)iosstat;
+    ret = inode_ctx_put(inode, this, iosstat64);
+    return ret;
+}
 
-                if (list_head->min_cnt == 0)
-                        list_head->min_cnt = value;
-                if ((list_head->members == MAX_LIST_MEMBERS) &&
-                     (list_head->min_cnt > value))
-                        goto out;
-
-                list_for_each_entry_safe (entry, t,
-                              &list_head->iosstats->list, list) {
-                        cnt++;
-                        if (cnt == list_head->members)
-                                last = entry;
-
-                        if (!gf_uuid_compare (iosstat->gfid,
-                            entry->iosstat->gfid)) {
-                                list_entry = entry;
-                                found = cnt;
-                                entry->value = value;
-                                if (!reposition) {
-                                        if (cnt == list_head->members)
-                                                list_head->min_cnt = value;
-                                        goto out;
-                                }
-                                break;
-                        } else if (entry->value <= value && !reposition) {
-                                reposition = cnt;
-                                tmp = entry;
-                                if (cnt == list_head->members - 1)
-                                        min_count = entry->value;
-                        }
-                }
-                if (found) {
-                        list_del (&list_entry->list);
-                        list_add_tail (&list_entry->list, &tmp->list);
-                        if (min_count)
-                                list_head->min_cnt = min_count;
-                        goto out;
-                } else if (list_head->members == MAX_LIST_MEMBERS && reposition) {
-                        new = GF_CALLOC (1, sizeof (*new),
-                                        gf_io_stats_mt_ios_stat_list);
-                        new->iosstat = iosstat;
-                        new->value = value;
-                        ios_stat_ref (iosstat);
-                        list_add_tail (&new->list, &tmp->list);
-                        if (last) {
-                                stat = last->iosstat;
-                                last->iosstat = NULL;
-                                ios_stat_unref (stat);
-                                list_del (&last->list);
-                                GF_FREE (last);
-                        }
-                        if (reposition == MAX_LIST_MEMBERS)
-                                list_head->min_cnt = value;
-                        else if (min_count) {
-                                list_head->min_cnt = min_count;
-                        }
-                } else if (list_head->members < MAX_LIST_MEMBERS) {
-                        new  = GF_CALLOC (1, sizeof (*new),
-                                          gf_io_stats_mt_ios_stat_list);
-                        new->iosstat = iosstat;
-                        new->value = value;
-                        ios_stat_ref (iosstat);
-                        if (reposition) {
-                                list_add_tail (&new->list, &tmp->list);
-                        } else {
-                                list_add_tail (&new->list, &entry->list);
-                        }
-                        list_head->members++;
-                        if (list_head->min_cnt > value)
-                                list_head->min_cnt = value;
-                }
-        }
-out:
-        UNLOCK (&list_head->lock);
-        return 0;
+int
+ios_inode_ctx_get(inode_t *inode, xlator_t *this, struct ios_stat **iosstat)
+{
+    uint64_t iosstat64 = 0;
+    unsigned long iosstatlong = 0;
+    int ret = 0;
+
+    ret = inode_ctx_get(inode, this, &iosstat64);
+    iosstatlong = iosstat64;
+    if (ret != -1)
+        *iosstat = (void *)iosstatlong;
+
+    return ret;
 }
 
-static inline int
-ios_stats_cleanup (xlator_t *this, inode_t *inode)
+/*
+ * So why goto all this trouble?  Why not just queue up some samples in
+ * a big list and malloc away?  Well malloc is expensive relative
+ * to what we are measuring, so cannot have any malloc's (or worse
+ * callocs) in our measurement code paths.  Instead, we are going to
+ * pre-allocate a circular buffer and collect a maximum number of samples.
+ * Prior to dumping them all we'll create a new buffer and swap the
+ * old buffer with the new, and then proceed to dump the statistics
+ * in our dump thread.
+ *
+ */
+ios_sample_buf_t *
+ios_create_sample_buf(size_t buf_size)
 {
+    ios_sample_buf_t *ios_sample_buf = NULL;
+    ios_sample_t *ios_samples = NULL;
+
+    ios_sample_buf = GF_CALLOC(1, sizeof(*ios_sample_buf),
+                               gf_io_stats_mt_ios_sample_buf);
+    if (!ios_sample_buf)
+        goto err;
+
+    ios_samples = GF_CALLOC(buf_size, sizeof(*ios_samples),
+                            gf_io_stats_mt_ios_sample);
+
+    if (!ios_samples)
+        goto err;
+
+    ios_sample_buf->ios_samples = ios_samples;
+    ios_sample_buf->size = buf_size;
+    ios_sample_buf->pos = 0;
+    ios_sample_buf->observed = 0;
+    ios_sample_buf->collected = 0;
+
+    return ios_sample_buf;
+err:
+    GF_FREE(ios_sample_buf);
+    return NULL;
+}
 
-        struct ios_stat *iosstat = NULL;
-        uint64_t         iosstat64 = 0;
+void
+ios_destroy_sample_buf(ios_sample_buf_t *ios_sample_buf)
+{
+    GF_FREE(ios_sample_buf->ios_samples);
+    GF_FREE(ios_sample_buf);
+}
 
-        inode_ctx_del (inode, this, &iosstat64);
-        if (!iosstat64) {
-                gf_log (this->name, GF_LOG_WARNING,
-                        "could not get inode ctx");
-                return 0;
-        }
-        iosstat = (void *) (long)iosstat64;
-        if (iosstat) {
-                ios_stat_unref (iosstat);
-        }
+static int
+ios_init_sample_buf(struct ios_conf *conf)
+{
+    int32_t ret = -1;
+
+    GF_ASSERT(conf);
+    LOCK(&conf->lock);
+    conf->ios_sample_buf = ios_create_sample_buf(conf->ios_sample_buf_size);
+    if (!conf->ios_sample_buf)
+        goto out;
+    ret = 0;
+out:
+    UNLOCK(&conf->lock);
+    return ret;
+}
+
+static int
+ios_stats_cleanup(xlator_t *this, inode_t *inode)
+{
+    struct ios_stat *iosstat = NULL;
+    uint64_t iosstat64 = 0;
+
+    inode_ctx_del(inode, this, &iosstat64);
+    if (!iosstat64) {
+        gf_log(this->name, GF_LOG_WARNING, "could not get inode ctx");
         return 0;
+    }
+    iosstat = (void *)(long)iosstat64;
+    if (iosstat) {
+        ios_stat_unref(iosstat);
+    }
+    return 0;
 }
 
-#define ios_log(this, logfp, fmt ...)                           \
-        do {                                                    \
-                if (logfp) {                                    \
-                        fprintf (logfp, fmt);                   \
-                        fprintf (logfp, "\n");                  \
-                }                                               \
-                gf_log (this->name, GF_LOG_INFO, fmt);        \
-        } while (0)
+#define ios_log(this, logfp, fmt...)                                           \
+    do {                                                                       \
+        if (logfp) {                                                           \
+            fprintf(logfp, fmt);                                               \
+            fprintf(logfp, "\n");                                              \
+        }                                                                      \
+        gf_log(this->name, GF_LOG_DEBUG, fmt);                                 \
+    } while (0)
 
 int
-ios_dump_file_stats (struct ios_stat_head *list_head, xlator_t *this, FILE* logfp)
+ios_dump_file_stats(struct ios_stat_head *list_head, xlator_t *this,
+                    FILE *logfp)
 {
-        struct ios_stat_list *entry = NULL;
+    struct ios_stat_list *entry = NULL;
 
-        LOCK (&list_head->lock);
+    LOCK(&list_head->lock);
+    {
+        list_for_each_entry(entry, &list_head->iosstats->list, list)
         {
-                list_for_each_entry (entry, &list_head->iosstats->list, list) {
-                        ios_log (this, logfp, "%-12.0f %s",
-                                entry->value, entry->iosstat->filename);
-                }
+            ios_log(this, logfp, "%-12.0f %s", entry->value,
+                    entry->iosstat->filename);
         }
-        UNLOCK (&list_head->lock);
-        return 0;
+    }
+    UNLOCK(&list_head->lock);
+    return 0;
 }
 
 int
-ios_dump_throughput_stats (struct ios_stat_head *list_head, xlator_t *this,
-                            FILE* logfp, ios_stats_type_t type)
+ios_dump_throughput_stats(struct ios_stat_head *list_head, xlator_t *this,
+                          FILE *logfp, ios_stats_thru_t type)
 {
-        struct ios_stat_list *entry = NULL;
-        struct timeval        time  = {0, };
-        char                  timestr[256] = {0, };
-
-        LOCK (&list_head->lock);
+    struct ios_stat_list *entry = NULL;
+    char timestr[GF_TIMESTR_SIZE] = {
+        0,
+    };
+
+    LOCK(&list_head->lock);
+    {
+        list_for_each_entry(entry, &list_head->iosstats->list, list)
         {
-                list_for_each_entry (entry, &list_head->iosstats->list, list) {
-                        gf_time_fmt (timestr, sizeof timestr,
-                                     entry->iosstat->thru_counters[type].time.tv_sec,
-                                     gf_timefmt_FT);
-                        snprintf (timestr + strlen (timestr), sizeof timestr - strlen (timestr),
-                          ".%"GF_PRI_SUSECONDS, time.tv_usec);
-
-                        ios_log (this, logfp, "%s \t %-10.2f  \t  %s",
-                                 timestr, entry->value, entry->iosstat->filename);
-                }
+            gf_time_fmt_tv(timestr, sizeof timestr,
+                           &entry->iosstat->thru_counters[type].time,
+                           gf_timefmt_FT);
+
+            ios_log(this, logfp, "%s \t %-10.2f  \t  %s", timestr, entry->value,
+                    entry->iosstat->filename);
         }
-        UNLOCK (&list_head->lock);
-        return 0;
+    }
+    UNLOCK(&list_head->lock);
+    return 0;
 }
 
 int
-io_stats_dump_global_to_logfp (xlator_t *this, struct ios_global_stats *stats,
-                               struct timeval *now, int interval, FILE* logfp)
+_io_stats_get_key_prefix(xlator_t *this, char **key_prefix)
 {
-        int                   i = 0;
-        int                   per_line = 0;
-        int                   index = 0;
-        struct ios_stat_head *list_head = NULL;
-        struct ios_conf      *conf = NULL;
-        char                  timestr[256] = {0, };
-        char                  str_header[128] = {0};
-        char                  str_read[128] = {0};
-        char                  str_write[128] = {0};
-
-        conf = this->private;
+    char *key_root = "gluster";
+    char *xlator_name = NULL;
+    char *instance_name = NULL;
+    size_t key_len = 0;
+    int bytes_written = 0;
+    int i = 0;
+    int ret = 0;
+    struct ios_conf *conf = this->private;
+
+    xlator_name = strdupa(conf->unique_id);
+    for (i = 0; i < strlen(xlator_name); i++) {
+        if (xlator_name[i] == '/')
+            xlator_name[i] = '_';
+    }
+
+    instance_name = this->instance_name;
+    if (this->name && strcmp(this->name, "glustershd") == 0) {
+        xlator_name = "shd";
+    } else if (this->prev && strcmp(this->prev->name, "nfs-server") == 0) {
+        xlator_name = "nfsd";
+        if (this->prev->instance_name)
+            instance_name = strdupa(this->prev->instance_name);
+    }
+
+    if (strcmp(__progname, "glusterfsd") == 0)
+        key_root = "gluster.brick";
+
+    if (instance_name) {
+        /* +3 for 2 x "." + NULL */
+        key_len = strlen(key_root) + strlen(xlator_name) +
+                  strlen(instance_name) + 3;
+        *key_prefix = GF_CALLOC(key_len, sizeof(char), gf_common_mt_char);
+        if (!*key_prefix) {
+            ret = -ENOMEM;
+            goto err;
+        }
+        bytes_written = snprintf(*key_prefix, key_len, "%s.%s.%s", key_root,
+                                 xlator_name, instance_name);
+        if (bytes_written != key_len - 1) {
+            ret = -EINVAL;
+            goto err;
+        }
+    } else {
+        /* +2 for 1 x "." + NULL */
+        key_len = strlen(key_root) + strlen(xlator_name) + 2;
+        *key_prefix = GF_CALLOC(key_len, sizeof(char), gf_common_mt_char);
+        if (!*key_prefix) {
+            ret = -ENOMEM;
+            goto err;
+        }
+        bytes_written = snprintf(*key_prefix, key_len, "%s.%s", key_root,
+                                 xlator_name);
+        if (bytes_written != key_len - 1) {
+            ret = -EINVAL;
+            goto err;
+        }
+    }
+    return 0;
+err:
+    GF_FREE(*key_prefix);
+    *key_prefix = NULL;
+    return ret;
+}
 
-        if (interval == -1)
-                ios_log (this, logfp, "\n=== Cumulative stats ===");
-        else
-                ios_log (this, logfp, "\n=== Interval %d stats ===",
-                         interval);
-        ios_log (this, logfp, "      Duration : %"PRId64" secs",
-                 (uint64_t) (now->tv_sec - stats->started_at.tv_sec));
-        ios_log (this, logfp, "     BytesRead : %"PRId64,
-                 stats->data_read);
-        ios_log (this, logfp, "  BytesWritten : %"PRId64"\n",
-                 stats->data_written);
-
-        snprintf (str_header, sizeof (str_header), "%-12s %c", "Block Size", ':');
-        snprintf (str_read, sizeof (str_read), "%-12s %c", "Read Count", ':');
-        snprintf (str_write, sizeof (str_write), "%-12s %c", "Write Count", ':');
-        index = 14;
-        for (i = 0; i < 32; i++) {
-                if ((stats->block_count_read[i] == 0) &&
-                    (stats->block_count_write[i] == 0))
-                        continue;
-                per_line++;
-
-                snprintf (str_header+index, sizeof (str_header)-index,
-                          "%16dB+", (1<<i));
-                if (stats->block_count_read[i])
-                        snprintf (str_read+index, sizeof (str_read)-index,
-                                  "%18"PRId64, stats->block_count_read[i]);
-                else    snprintf (str_read+index, sizeof (str_read)-index,
-                                  "%18s", "0");
-                if (stats->block_count_write[i])
-                        snprintf (str_write+index, sizeof (str_write)-index,
-                                  "%18"PRId64, stats->block_count_write[i]);
-                else    snprintf (str_write+index, sizeof (str_write)-index,
-                                  "%18s", "0");
-
-                index += 18;
-                if (per_line == 3) {
-                        ios_log (this, logfp, "%s", str_header);
-                        ios_log (this, logfp, "%s", str_read);
-                        ios_log (this, logfp, "%s\n", str_write);
-
-                        memset (str_header, 0, sizeof (str_header));
-                        memset (str_read, 0, sizeof (str_read));
-                        memset (str_write, 0, sizeof (str_write));
-
-                        snprintf (str_header, sizeof (str_header), "%-12s %c",
-                                  "Block Size", ':');
-                        snprintf (str_read, sizeof (str_read), "%-12s %c",
-                                  "Read Count", ':');
-                        snprintf (str_write, sizeof (str_write), "%-12s %c",
-                                  "Write Count", ':');
-
-                        index = 14;
-                        per_line = 0;
-                }
+int
+io_stats_dump_global_to_json_logfp(xlator_t *this,
+                                   struct ios_global_stats *stats, time_t now,
+                                   int interval, FILE *logfp)
+{
+    int i = 0;
+    int j = 0;
+    struct ios_conf *conf = NULL;
+    char *key_prefix = NULL;
+    char *str_prefix = NULL;
+    char *lc_fop_name = NULL;
+    int ret = 1; /* Default to error */
+    int rw_size;
+    char *rw_unit = NULL;
+    uint64_t fop_hits;
+    float fop_lat_ave;
+    float fop_lat_min;
+    float fop_lat_max;
+    double interval_sec;
+    double fop_ave_usec = 0.0;
+    double fop_ave_usec_sum = 0.0;
+    double weighted_fop_ave_usec = 0.0;
+    double weighted_fop_ave_usec_sum = 0.0;
+    long total_fop_hits = 0;
+    loc_t unused_loc = {
+        0,
+    };
+    dict_t *xattr = NULL;
+
+    interval_sec = (double)(now - stats->started_at);
+
+    conf = this->private;
+
+    ret = _io_stats_get_key_prefix(this, &key_prefix);
+    if (ret) {
+        goto out;
+    }
+
+    if (interval == -1) {
+        str_prefix = "aggr";
+
+    } else {
+        str_prefix = "inter";
+    }
+    ios_log(this, logfp, "{");
+
+    for (i = 0; i < 31; i++) {
+        rw_size = (1 << i);
+        if (rw_size >= 1024 * 1024) {
+            rw_size = rw_size / (1024 * 1024);
+            rw_unit = "mb";
+        } else if (rw_size >= 1024) {
+            rw_size = rw_size / 1024;
+            rw_unit = "kb";
+        } else {
+            rw_unit = "b";
         }
 
-        if (per_line != 0) {
-                ios_log (this, logfp, "%s", str_header);
-                ios_log (this, logfp, "%s", str_read);
-                ios_log (this, logfp, "%s\n", str_write);
+        if (interval == -1) {
+            ios_log(this, logfp, "\"%s.%s.read_%d%s\": %" GF_PRI_ATOMIC ",",
+                    key_prefix, str_prefix, rw_size, rw_unit,
+                    GF_ATOMIC_GET(stats->block_count_read[i]));
+            ios_log(this, logfp, "\"%s.%s.write_%d%s\": %" GF_PRI_ATOMIC ",",
+                    key_prefix, str_prefix, rw_size, rw_unit,
+                    GF_ATOMIC_GET(stats->block_count_write[i]));
+        } else {
+            ios_log(this, logfp, "\"%s.%s.read_%d%s_per_sec\": %0.2lf,",
+                    key_prefix, str_prefix, rw_size, rw_unit,
+                    (double)(GF_ATOMIC_GET(stats->block_count_read[i]) /
+                             interval_sec));
+            ios_log(this, logfp, "\"%s.%s.write_%d%s_per_sec\": %0.2lf,",
+                    key_prefix, str_prefix, rw_size, rw_unit,
+                    (double)(GF_ATOMIC_GET(stats->block_count_write[i]) /
+                             interval_sec));
+        }
+    }
+
+    if (interval == -1) {
+        ios_log(this, logfp, "\"%s.%s.fds.open_count\": %" PRId64 ",",
+                key_prefix, str_prefix, conf->cumulative.nr_opens);
+        ios_log(this, logfp, "\"%s.%s.fds.max_open_count\": %" PRId64 ",",
+                key_prefix, str_prefix, conf->cumulative.max_nr_opens);
+    }
+
+    for (i = 0; i < GF_FOP_MAXVALUE; i++) {
+        lc_fop_name = strdupa(gf_fop_list[i]);
+        for (j = 0; lc_fop_name[j]; j++) {
+            lc_fop_name[j] = tolower(lc_fop_name[j]);
         }
 
-        ios_log (this, logfp, "%-13s %10s %14s %14s %14s", "Fop",
-                 "Call Count", "Avg-Latency", "Min-Latency",
-                 "Max-Latency");
-        ios_log (this, logfp, "%-13s %10s %14s %14s %14s", "---", "----------",
-                 "-----------", "-----------", "-----------");
-
-        for (i = 0; i < GF_FOP_MAXVALUE; i++) {
-                if (stats->fop_hits[i] && !stats->latency[i].avg)
-                        ios_log (this, logfp, "%-13s %10"PRId64" %11s "
-                                 "us %11s us %11s us", gf_fop_list[i],
-                                 stats->fop_hits[i], "0", "0", "0");
-                else if (stats->fop_hits[i] && stats->latency[i].avg)
-                        ios_log (this, logfp, "%-13s %10"PRId64" %11.2lf us "
-                                 "%11.2lf us %11.2lf us", gf_fop_list[i],
-                                 stats->fop_hits[i], stats->latency[i].avg,
-                                 stats->latency[i].min, stats->latency[i].max);
+        fop_hits = GF_ATOMIC_GET(stats->fop_hits[i]);
+        fop_lat_ave = 0.0;
+        fop_lat_min = 0.0;
+        fop_lat_max = 0.0;
+        if (fop_hits) {
+            if (stats->latency[i].avg) {
+                fop_lat_ave = stats->latency[i].avg;
+                fop_lat_min = stats->latency[i].min;
+                fop_lat_max = stats->latency[i].max;
+            }
+        }
+        if (interval == -1) {
+            ios_log(this, logfp, "\"%s.%s.fop.%s.count\": %" GF_PRI_ATOMIC ",",
+                    key_prefix, str_prefix, lc_fop_name, fop_hits);
+        } else {
+            ios_log(this, logfp, "\"%s.%s.fop.%s.per_sec\": %0.2lf,",
+                    key_prefix, str_prefix, lc_fop_name,
+                    (double)(fop_hits / interval_sec));
         }
-        ios_log (this, logfp, "------ ----- ----- ----- ----- ----- ----- ----- "
-                 " ----- ----- ----- -----\n");
 
+        ios_log(this, logfp, "\"%s.%s.fop.%s.latency_ave_usec\": %0.2lf,",
+                key_prefix, str_prefix, lc_fop_name, fop_lat_ave);
+        ios_log(this, logfp, "\"%s.%s.fop.%s.latency_min_usec\": %0.2lf,",
+                key_prefix, str_prefix, lc_fop_name, fop_lat_min);
+        ios_log(this, logfp, "\"%s.%s.fop.%s.latency_max_usec\": %0.2lf,",
+                key_prefix, str_prefix, lc_fop_name, fop_lat_max);
+
+        fop_ave_usec_sum += fop_lat_ave;
+        weighted_fop_ave_usec_sum += fop_hits * fop_lat_ave;
+        total_fop_hits += fop_hits;
+    }
+
+    if (total_fop_hits) {
+        weighted_fop_ave_usec = weighted_fop_ave_usec_sum / total_fop_hits;
+        /* Extra key that does not print out an entry w/ 0.00 for
+         * intervals with no data
+         */
+        ios_log(this, logfp,
+                "\"%s.%s.fop.weighted_latency_ave_usec_nozerofill\": "
+                "%0.4lf,",
+                key_prefix, str_prefix, weighted_fop_ave_usec);
+    }
+    ios_log(this, logfp, "\"%s.%s.fop.weighted_latency_ave_usec\": %0.4lf,",
+            key_prefix, str_prefix, weighted_fop_ave_usec);
+    ios_log(this, logfp, "\"%s.%s.fop.weighted_fop_count\": %ld,", key_prefix,
+            str_prefix, total_fop_hits);
+
+    fop_ave_usec = fop_ave_usec_sum / GF_FOP_MAXVALUE;
+    ios_log(this, logfp, "\"%s.%s.fop.unweighted_latency_ave_usec\":%0.4lf,",
+            key_prefix, str_prefix, fop_ave_usec);
+
+    for (i = 0; i < GF_UPCALL_FLAGS_MAXVALUE; i++) {
+        lc_fop_name = strdupa(gf_upcall_list[i]);
+        for (j = 0; lc_fop_name[j]; j++) {
+            lc_fop_name[j] = tolower(lc_fop_name[j]);
+        }
+        fop_hits = GF_ATOMIC_GET(stats->upcall_hits[i]);
         if (interval == -1) {
-                LOCK (&conf->lock);
-                {
-                        gf_time_fmt (timestr, sizeof timestr,
-                                     conf->cumulative.max_openfd_time.tv_sec,
-                                     gf_timefmt_FT);
-                        snprintf (timestr + strlen (timestr), sizeof timestr - strlen (timestr),
-                                  ".%"GF_PRI_SUSECONDS,
-                                  conf->cumulative.max_openfd_time.tv_usec);
-                        ios_log (this, logfp, "Current open fd's: %"PRId64
-                                 " Max open fd's: %"PRId64" time %s",
-                                 conf->cumulative.nr_opens,
-                                 conf->cumulative.max_nr_opens, timestr);
-                }
-                UNLOCK (&conf->lock);
-                ios_log (this, logfp, "\n==========Open File Stats========");
-                ios_log (this, logfp, "\nCOUNT:  \t  FILE NAME");
-                list_head = &conf->list[IOS_STATS_TYPE_OPEN];
-                ios_dump_file_stats (list_head, this, logfp);
-
-
-                ios_log (this, logfp, "\n==========Read File Stats========");
-                ios_log (this, logfp, "\nCOUNT:  \t  FILE NAME");
-                list_head = &conf->list[IOS_STATS_TYPE_READ];
-                ios_dump_file_stats (list_head, this, logfp);
-
-                ios_log (this, logfp, "\n==========Write File Stats========");
-                ios_log (this, logfp, "\nCOUNT:  \t  FILE NAME");
-                list_head = &conf->list[IOS_STATS_TYPE_WRITE];
-                ios_dump_file_stats (list_head, this, logfp);
-
-                ios_log (this, logfp, "\n==========Directory open stats========");
-                ios_log (this, logfp, "\nCOUNT:  \t  DIRECTORY NAME");
-                list_head = &conf->list[IOS_STATS_TYPE_OPENDIR];
-                ios_dump_file_stats (list_head, this, logfp);
-
-                ios_log (this, logfp, "\n========Directory readdirp Stats=======");
-                ios_log (this, logfp, "\nCOUNT:  \t  DIRECTORY NAME");
-                list_head = &conf->list[IOS_STATS_TYPE_READDIRP];
-                ios_dump_file_stats (list_head, this, logfp);
-
-                ios_log (this, logfp, "\n========Read Throughput File Stats=====");
-                ios_log (this, logfp, "\nTIMESTAMP \t\t\t THROUGHPUT(KBPS)"
-                         "\tFILE NAME");
-                list_head = &conf->thru_list[IOS_STATS_THRU_READ];
-                ios_dump_throughput_stats(list_head, this, logfp, IOS_STATS_TYPE_READ);
-
-                ios_log (this, logfp, "\n======Write Throughput File Stats======");
-                ios_log (this, logfp, "\nTIMESTAMP \t\t\t THROUGHPUT(KBPS)"
-                         "\tFILE NAME");
-                list_head = &conf->thru_list[IOS_STATS_THRU_WRITE];
-                ios_dump_throughput_stats (list_head, this, logfp, IOS_STATS_TYPE_WRITE);
+            ios_log(this, logfp, "\"%s.%s.fop.%s.count\": %" GF_PRI_ATOMIC ",",
+                    key_prefix, str_prefix, lc_fop_name, fop_hits);
+        } else {
+            ios_log(this, logfp, "\"%s.%s.fop.%s.per_sec\": %0.2lf,",
+                    key_prefix, str_prefix, lc_fop_name,
+                    (double)(fop_hits / interval_sec));
         }
-        return 0;
+    }
+
+    ret = syncop_getxattr(this, &unused_loc, &xattr, IO_THREADS_QUEUE_SIZE_KEY,
+                          NULL, NULL);
+    if (xattr) {
+        /*
+         * Iterate over the dictionary returned to us by io-threads and
+         * dump the results to the stats file.
+         */
+        data_pair_t *curr = NULL;
+
+        dict_foreach_inline(xattr, curr)
+        {
+            ios_log(this, logfp, "\"%s.%s.%s.queue_size\": %d,", key_prefix,
+                    str_prefix, curr->key, data_to_int32(curr->value));
+        }
+
+        /* Free the dictionary */
+        dict_unref(xattr);
+    } else {
+        gf_log(this->name, GF_LOG_WARNING,
+               "Unable to get queue size counts from "
+               "the io-threads translator!");
+    }
+
+    if (interval == -1) {
+        ios_log(this, logfp, "\"%s.%s.uptime\": %" PRIu64 ",", key_prefix,
+                str_prefix, (uint64_t)(now - stats->started_at));
+        ios_log(this, logfp,
+                "\"%s.%s.bytes_read\": "
+                "%" GF_PRI_ATOMIC ",",
+                key_prefix, str_prefix, GF_ATOMIC_GET(stats->data_read));
+        ios_log(this, logfp,
+                "\"%s.%s.bytes_written\": "
+                "%" GF_PRI_ATOMIC "",
+                key_prefix, str_prefix, GF_ATOMIC_GET(stats->data_written));
+    } else {
+        ios_log(this, logfp, "\"%s.%s.sample_interval_sec\": %0.2lf,",
+                key_prefix, str_prefix, interval_sec);
+        ios_log(this, logfp, "\"%s.%s.bytes_read_per_sec\": %0.2lf,",
+                key_prefix, str_prefix,
+                (double)(GF_ATOMIC_GET(stats->data_read) / interval_sec));
+        ios_log(this, logfp, "\"%s.%s.bytes_written_per_sec\": %0.2lf",
+                key_prefix, str_prefix,
+                (double)(GF_ATOMIC_GET(stats->data_written) / interval_sec));
+    }
+
+    ios_log(this, logfp, "}");
+    ret = 0;
+out:
+    GF_FREE(key_prefix);
+    return ret;
 }
 
+char *
+_resolve_username(xlator_t *this, uid_t uid)
+{
+    struct passwd pwd;
+    struct passwd *pwd_result = NULL;
+    size_t pwd_buf_len;
+    char *pwd_buf = NULL;
+    char *ret = NULL;
+
+    /* Prepare our buffer for the uid->username translation */
+#ifdef _SC_GETGR_R_SIZE_MAX
+    pwd_buf_len = sysconf(_SC_GETGR_R_SIZE_MAX);
+#else
+    pwd_buf_len = -1;
+#endif
+    if (pwd_buf_len == -1) {
+        pwd_buf_len = DEFAULT_PWD_BUF_SZ; /* per the man page */
+    }
+
+    pwd_buf = alloca(pwd_buf_len);
+    if (!pwd_buf)
+        goto err;
+
+    getpwuid_r(uid, &pwd, pwd_buf, pwd_buf_len, &pwd_result);
+    if (!pwd_result)
+        goto err;
+
+    ret = gf_strdup(pwd.pw_name);
+    if (ret)
+        return ret;
+    else
+        gf_log(this->name, GF_LOG_ERROR,
+               "gf_strdup failed, failing username "
+               "resolution.");
+err:
+    return ret;
+}
+
+char *
+_resolve_group_name(xlator_t *this, gid_t gid)
+{
+    struct group grp;
+    struct group *grp_result = NULL;
+    size_t grp_buf_len;
+    char *grp_buf = NULL;
+    char *ret = NULL;
+
+    /* Prepare our buffer for the gid->group name translation */
+#ifdef _SC_GETGR_R_SIZE_MAX
+    grp_buf_len = sysconf(_SC_GETGR_R_SIZE_MAX);
+#else
+    grp_buf_len = -1;
+#endif
+    if (grp_buf_len == -1) {
+        grp_buf_len = DEFAULT_GRP_BUF_SZ; /* per the man page */
+    }
+
+    grp_buf = alloca(grp_buf_len);
+    if (!grp_buf) {
+        goto err;
+    }
+
+    if (getgrgid_r(gid, &grp, grp_buf, grp_buf_len, &grp_result) != 0)
+        goto err;
+
+    if (!grp_result)
+        goto err;
+
+    ret = gf_strdup(grp.gr_name);
+    if (ret)
+        return ret;
+    else
+        gf_log(this->name, GF_LOG_ERROR,
+               "gf_strdup failed, failing username "
+               "resolution.");
+err:
+    return ret;
+}
+
+/*
+ * This function writes out a latency sample to a given file descriptor
+ * and beautifies the output in the process.
+ */
+void
+_io_stats_write_latency_sample(xlator_t *this, ios_sample_t *sample,
+                               FILE *logfp)
+{
+    double epoch_time = 0.00;
+    char *xlator_name = NULL;
+    char *instance_name = NULL;
+    char *hostname = NULL;
+    char *identifier = NULL;
+    char *port = NULL;
+    char *port_pos = NULL;
+    char *group_name = NULL;
+    char *username = NULL;
+    struct ios_conf *conf = NULL;
+
+    conf = this->private;
+
+    epoch_time = (sample->timestamp).tv_sec +
+                 ((sample->timestamp).tv_usec / 1000000.0);
+
+    if (strlen(sample->identifier) == 0) {
+        hostname = "Unknown";
+        port = "Unknown";
+    } else {
+        identifier = strdupa(sample->identifier);
+        port_pos = strrchr(identifier, ':');
+        if (!port_pos || strlen(port_pos) < 2)
+            goto err;
+        port = strdupa(port_pos + 1);
+        if (!port)
+            goto err;
+        *port_pos = '\0';
+        hostname = gf_rev_dns_lookup_cached(identifier, conf->dnscache);
+        if (!hostname)
+            hostname = "Unknown";
+    }
+
+    xlator_name = conf->unique_id;
+    if (!xlator_name || strlen(xlator_name) == 0)
+        xlator_name = "Unknown";
+
+    instance_name = this->instance_name;
+    if (!instance_name || strlen(instance_name) == 0)
+        instance_name = "N/A";
+
+    /* Resolve the UID to a string username */
+    username = _resolve_username(this, sample->uid);
+    if (!username) {
+        username = GF_MALLOC(30, gf_common_mt_char);
+        if (!username) {
+            goto out;
+        }
+        sprintf(username, "%d", (int32_t)sample->uid);
+    }
+
+    /* Resolve the GID to a string group name */
+    group_name = _resolve_group_name(this, sample->gid);
+    if (!group_name) {
+        group_name = GF_MALLOC(30, gf_common_mt_char);
+        if (!group_name) {
+            goto out;
+        }
+        sprintf(group_name, "%d", (int32_t)sample->gid);
+    }
+
+    ios_log(this, logfp, "%0.6lf,%s,%s,%0.4lf,%s,%s,%s,%s,%s,%s", epoch_time,
+            fop_enum_to_pri_string(sample->fop_type),
+            gf_fop_string(sample->fop_type), sample->elapsed, xlator_name,
+            instance_name, username, group_name, hostname, port);
+    goto out;
+err:
+    gf_log(this->name, GF_LOG_ERROR, "Error parsing socket identifier");
+out:
+    GF_FREE(group_name);
+    GF_FREE(username);
+}
+
+/*
+ * Takes our current sample buffer in conf->io_sample_buf, and saves
+ * a reference to this, init's a new buffer, and then dumps out the
+ * contents of the saved reference.
+ */
 int
-io_stats_dump_global_to_dict (xlator_t *this, struct ios_global_stats *stats,
-                              struct timeval *now, int interval, dict_t *dict)
+io_stats_dump_latency_samples_logfp(xlator_t *this, FILE *logfp)
 {
-        int             ret = 0;
-        char            key[256] = {0};
-        uint64_t        sec = 0;
-        int             i = 0;
-        uint64_t        count = 0;
+    uint64_t i = 0;
+    struct ios_conf *conf = NULL;
+    ios_sample_buf_t *sample_buf = NULL;
+    int ret = 1; /* Default to error */
+
+    conf = this->private;
+
+    /* Save pointer to old buffer; the CS equivalent of
+     * Indiana Jones: https://www.youtube.com/watch?v=Pr-8AP0To4k,
+     * though ours will end better I hope!
+     */
+    sample_buf = conf->ios_sample_buf;
+    if (!sample_buf) {
+        gf_log(this->name, GF_LOG_WARNING, "Sampling buffer is null, bailing!");
+        goto out;
+    }
+
+    /* Empty case, nothing to do, exit. */
+    if (sample_buf->collected == 0) {
+        gf_log(this->name, GF_LOG_DEBUG, "No samples, dump not required.");
+        ret = 0;
+        goto out;
+    }
+
+    /* Init a new buffer, so we are free to work on the one we saved a
+     * reference to above.
+     */
+    if (ios_init_sample_buf(conf) != 0) {
+        gf_log(this->name, GF_LOG_WARNING,
+               "Failed to init new sampling buffer, out of memory?");
+        goto out;
+    }
+
+    /* Wrap-around case, dump from pos to sample_buf->size -1
+     * and then from 0 to sample_buf->pos (covered off by
+     * "simple case")
+     */
+    if (sample_buf->collected > sample_buf->pos + 1) {
+        for (i = sample_buf->pos; i < sample_buf->size; i++) {
+            _io_stats_write_latency_sample(this, &(sample_buf->ios_samples[i]),
+                                           logfp);
+        }
+    }
+
+    /* Simple case: Dump from 0 to sample_buf->pos */
+    for (i = 0; i < sample_buf->pos; i++) {
+        _io_stats_write_latency_sample(this, &(sample_buf->ios_samples[i]),
+                                       logfp);
+    }
+    ios_destroy_sample_buf(sample_buf);
 
-        GF_ASSERT (stats);
-        GF_ASSERT (now);
-        GF_ASSERT (dict);
-        GF_ASSERT (this);
+out:
+    return ret;
+}
 
-        if (interval == -1)
-                snprintf (key, sizeof (key), "cumulative");
+int
+io_stats_dump_global_to_logfp(xlator_t *this, struct ios_global_stats *stats,
+                              time_t now, int interval, FILE *logfp)
+{
+    int i = 0;
+    int per_line = 0;
+    int index = 0;
+    struct ios_stat_head *list_head = NULL;
+    struct ios_conf *conf = NULL;
+    char timestr[GF_TIMESTR_SIZE] = {
+        0,
+    };
+    char str_header[128] = {0};
+    char str_read[128] = {0};
+    char str_write[128] = {0};
+    uint64_t fop_hits = 0;
+    uint64_t block_count_read = 0;
+    uint64_t block_count_write = 0;
+
+    conf = this->private;
+
+    if (interval == -1)
+        ios_log(this, logfp, "\n=== Cumulative stats ===");
+    else
+        ios_log(this, logfp, "\n=== Interval %d stats ===", interval);
+    ios_log(this, logfp, "      Duration : %" PRIu64 " secs",
+            (uint64_t)(now - stats->started_at));
+    ios_log(this, logfp, "     BytesRead : %" GF_PRI_ATOMIC,
+            GF_ATOMIC_GET(stats->data_read));
+    ios_log(this, logfp, "  BytesWritten : %" GF_PRI_ATOMIC "\n",
+            GF_ATOMIC_GET(stats->data_written));
+
+    snprintf(str_header, sizeof(str_header), "%-12s %c", "Block Size", ':');
+    snprintf(str_read, sizeof(str_read), "%-12s %c", "Read Count", ':');
+    snprintf(str_write, sizeof(str_write), "%-12s %c", "Write Count", ':');
+    index = 14;
+    for (i = 0; i < IOS_BLOCK_COUNT_SIZE; i++) {
+        block_count_read = GF_ATOMIC_GET(stats->block_count_read[i]);
+        block_count_write = GF_ATOMIC_GET(stats->block_count_write[i]);
+        if ((block_count_read == 0) && (block_count_write == 0))
+            continue;
+        per_line++;
+
+        snprintf(str_header + index, sizeof(str_header) - index, "%16dB+",
+                 (1 << i));
+        if (block_count_read)
+            snprintf(str_read + index, sizeof(str_read) - index, "%18" PRId64,
+                     block_count_read);
         else
-                snprintf (key, sizeof (key), "interval");
-        ret = dict_set_int32 (dict, key, interval);
-        if (ret)
-                gf_log (this->name, GF_LOG_ERROR, "failed to set "
-                        "interval %d", interval);
-
-        memset (key, 0, sizeof (key));
-        snprintf (key, sizeof (key), "%d-duration", interval);
-        sec = (uint64_t) (now->tv_sec - stats->started_at.tv_sec);
-        ret = dict_set_uint64 (dict, key, sec);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "failed to set "
-                        "duration(%d) - %"PRId64, interval, sec);
-                goto out;
+            snprintf(str_read + index, sizeof(str_read) - index, "%18s", "0");
+        if (block_count_write)
+            snprintf(str_write + index, sizeof(str_write) - index,
+                     "%18" GF_PRI_ATOMIC, block_count_write);
+        else
+            snprintf(str_write + index, sizeof(str_write) - index, "%18s", "0");
+
+        index += 18;
+        if (per_line == 3) {
+            ios_log(this, logfp, "%s", str_header);
+            ios_log(this, logfp, "%s", str_read);
+            ios_log(this, logfp, "%s\n", str_write);
+
+            snprintf(str_header, sizeof(str_header), "%-12s %c", "Block Size",
+                     ':');
+            snprintf(str_read, sizeof(str_read), "%-12s %c", "Read Count", ':');
+            snprintf(str_write, sizeof(str_write), "%-12s %c", "Write Count",
+                     ':');
+
+            index = 14;
+            per_line = 0;
         }
+    }
+
+    if (per_line != 0) {
+        ios_log(this, logfp, "%s", str_header);
+        ios_log(this, logfp, "%s", str_read);
+        ios_log(this, logfp, "%s\n", str_write);
+    }
+
+    ios_log(this, logfp, "%-13s %10s %14s %14s %14s", "Fop", "Call Count",
+            "Avg-Latency", "Min-Latency", "Max-Latency");
+    ios_log(this, logfp, "%-13s %10s %14s %14s %14s", "---", "----------",
+            "-----------", "-----------", "-----------");
+
+    for (i = 0; i < GF_FOP_MAXVALUE; i++) {
+        fop_hits = GF_ATOMIC_GET(stats->fop_hits[i]);
+        if (fop_hits && !stats->latency[i].avg)
+            ios_log(this, logfp,
+                    "%-13s %10" GF_PRI_ATOMIC
+                    " %11s "
+                    "us %11s us %11s us",
+                    gf_fop_list[i], fop_hits, "0", "0", "0");
+        else if (fop_hits && stats->latency[i].avg)
+            ios_log(this, logfp,
+                    "%-13s %10" GF_PRI_ATOMIC
+                    " "
+                    "%11.2lf us %11.2lf us %11.2lf us",
+                    gf_fop_list[i], fop_hits, stats->latency[i].avg,
+                    stats->latency[i].min, stats->latency[i].max);
+    }
+
+    for (i = 0; i < GF_UPCALL_FLAGS_MAXVALUE; i++) {
+        fop_hits = GF_ATOMIC_GET(stats->upcall_hits[i]);
+        if (fop_hits)
+            ios_log(this, logfp,
+                    "%-13s %10" PRId64
+                    " %11s "
+                    "us %11s us %11s us",
+                    gf_upcall_list[i], fop_hits, "0", "0", "0");
+    }
+
+    ios_log(this, logfp,
+            "------ ----- ----- ----- ----- ----- ----- ----- "
+            " ----- ----- ----- -----\n");
+
+    if (interval == -1) {
+        LOCK(&conf->lock);
+        {
+            gf_time_fmt_tv(timestr, sizeof timestr,
+                           &conf->cumulative.max_openfd_time, gf_timefmt_FT);
+            ios_log(this, logfp,
+                    "Current open fd's: %" PRId64 " Max open fd's: %" PRId64
+                    " time %s",
+                    conf->cumulative.nr_opens, conf->cumulative.max_nr_opens,
+                    timestr);
+        }
+        UNLOCK(&conf->lock);
+        ios_log(this, logfp, "\n==========Open File Stats========");
+        ios_log(this, logfp, "\nCOUNT:  \t  FILE NAME");
+        list_head = &conf->list[IOS_STATS_TYPE_OPEN];
+        ios_dump_file_stats(list_head, this, logfp);
+
+        ios_log(this, logfp, "\n==========Read File Stats========");
+        ios_log(this, logfp, "\nCOUNT:  \t  FILE NAME");
+        list_head = &conf->list[IOS_STATS_TYPE_READ];
+        ios_dump_file_stats(list_head, this, logfp);
+
+        ios_log(this, logfp, "\n==========Write File Stats========");
+        ios_log(this, logfp, "\nCOUNT:  \t  FILE NAME");
+        list_head = &conf->list[IOS_STATS_TYPE_WRITE];
+        ios_dump_file_stats(list_head, this, logfp);
+
+        ios_log(this, logfp, "\n==========Directory open stats========");
+        ios_log(this, logfp, "\nCOUNT:  \t  DIRECTORY NAME");
+        list_head = &conf->list[IOS_STATS_TYPE_OPENDIR];
+        ios_dump_file_stats(list_head, this, logfp);
+
+        ios_log(this, logfp, "\n========Directory readdirp Stats=======");
+        ios_log(this, logfp, "\nCOUNT:  \t  DIRECTORY NAME");
+        list_head = &conf->list[IOS_STATS_TYPE_READDIRP];
+        ios_dump_file_stats(list_head, this, logfp);
+
+        ios_log(this, logfp, "\n========Read Throughput File Stats=====");
+        ios_log(this, logfp,
+                "\nTIMESTAMP \t\t\t THROUGHPUT(KBPS)"
+                "\tFILE NAME");
+        list_head = &conf->thru_list[IOS_STATS_THRU_READ];
+        ios_dump_throughput_stats(list_head, this, logfp, IOS_STATS_THRU_READ);
+
+        ios_log(this, logfp, "\n======Write Throughput File Stats======");
+        ios_log(this, logfp,
+                "\nTIMESTAMP \t\t\t THROUGHPUT(KBPS)"
+                "\tFILE NAME");
+        list_head = &conf->thru_list[IOS_STATS_THRU_WRITE];
+        ios_dump_throughput_stats(list_head, this, logfp, IOS_STATS_THRU_WRITE);
+    }
+    return 0;
+}
 
-        memset (key, 0, sizeof (key));
-        snprintf (key, sizeof (key), "%d-total-read", interval);
-        ret = dict_set_uint64 (dict, key, stats->data_read);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "failed to set total "
-                       "read(%d) - %"PRId64, interval, stats->data_read);
+int
+io_stats_dump_global_to_dict(xlator_t *this, struct ios_global_stats *stats,
+                             time_t now, int interval, dict_t *dict)
+{
+    int ret = 0;
+    char key[64] = {0};
+    uint64_t sec = 0;
+    int i = 0;
+    uint64_t count = 0;
+    uint64_t fop_hits = 0;
+
+    GF_ASSERT(stats);
+    GF_ASSERT(now);
+    GF_ASSERT(dict);
+    GF_ASSERT(this);
+
+    if (interval == -1)
+        snprintf(key, sizeof(key), "cumulative");
+    else
+        snprintf(key, sizeof(key), "interval");
+    ret = dict_set_int32(dict, key, interval);
+    if (ret)
+        gf_log(this->name, GF_LOG_ERROR,
+               "failed to set "
+               "interval %d",
+               interval);
+
+    snprintf(key, sizeof(key), "%d-duration", interval);
+    sec = now - stats->started_at;
+    ret = dict_set_uint64(dict, key, sec);
+    if (ret) {
+        gf_log(this->name, GF_LOG_ERROR,
+               "failed to set "
+               "duration(%d) - %" PRId64,
+               interval, sec);
+        goto out;
+    }
+
+    snprintf(key, sizeof(key), "%d-total-read", interval);
+    ret = dict_set_uint64(dict, key, GF_ATOMIC_GET(stats->data_read));
+    if (ret) {
+        gf_log(this->name, GF_LOG_ERROR,
+               "failed to set total "
+               "read(%d) - %" GF_PRI_ATOMIC,
+               interval, GF_ATOMIC_GET(stats->data_read));
+        goto out;
+    }
+
+    snprintf(key, sizeof(key), "%d-total-write", interval);
+    ret = dict_set_uint64(dict, key, GF_ATOMIC_GET(stats->data_written));
+    if (ret) {
+        gf_log(this->name, GF_LOG_ERROR,
+               "failed to set total "
+               "write(%d) - %" GF_PRI_ATOMIC,
+               interval, GF_ATOMIC_GET(stats->data_written));
+        goto out;
+    }
+    for (i = 0; i < 32; i++) {
+        count = GF_ATOMIC_GET(stats->block_count_read[i]);
+        if (count) {
+            snprintf(key, sizeof(key), "%d-read-%d", interval, (1 << i));
+            ret = dict_set_uint64(dict, key, count);
+            if (ret) {
+                gf_log(this->name, GF_LOG_ERROR,
+                       "failed to "
+                       "set read-%db+, with: %" PRId64,
+                       (1 << i), count);
                 goto out;
+            }
         }
-
-        memset (key, 0, sizeof (key));
-        snprintf (key, sizeof (key), "%d-total-write", interval);
-        ret = dict_set_uint64 (dict, key, stats->data_written);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "failed to set total "
-                        "write(%d) - %"PRId64, interval, stats->data_written);
+    }
+
+    for (i = 0; i < IOS_BLOCK_COUNT_SIZE; i++) {
+        count = GF_ATOMIC_GET(stats->block_count_write[i]);
+        if (count) {
+            snprintf(key, sizeof(key), "%d-write-%d", interval, (1 << i));
+            ret = dict_set_uint64(dict, key, count);
+            if (ret) {
+                gf_log(this->name, GF_LOG_ERROR,
+                       "failed to "
+                       "set write-%db+, with: %" PRId64,
+                       (1 << i), count);
                 goto out;
+            }
         }
-        for (i = 0; i < 32; i++) {
-                if (stats->block_count_read[i]) {
-                        memset (key, 0, sizeof (key));
-                        snprintf (key, sizeof (key), "%d-read-%d", interval,
-                                  (1 << i));
-                        count = stats->block_count_read[i];
-                        ret = dict_set_uint64 (dict, key, count);
-                        if (ret) {
-                                gf_log (this->name, GF_LOG_ERROR, "failed to "
-                                        "set read-%db+, with: %"PRId64,
-                                        (1<<i), count);
-                                goto out;
-                        }
-                }
+    }
+
+    for (i = 0; i < GF_FOP_MAXVALUE; i++) {
+        fop_hits = GF_ATOMIC_GET(stats->fop_hits[i]);
+        if (fop_hits == 0)
+            continue;
+        snprintf(key, sizeof(key), "%d-%d-hits", interval, i);
+        ret = dict_set_uint64(dict, key, fop_hits);
+        if (ret) {
+            gf_log(this->name, GF_LOG_ERROR,
+                   "failed to set "
+                   "%s-fop-hits: %" GF_PRI_ATOMIC,
+                   gf_fop_list[i], fop_hits);
+            goto out;
         }
 
-        for (i = 0; i < 32; i++) {
-                if (stats->block_count_write[i]) {
-                        snprintf (key, sizeof (key), "%d-write-%d", interval,
-                                  (1<<i));
-                        count = stats->block_count_write[i];
-                        ret = dict_set_uint64 (dict, key, count);
-                        if (ret) {
-                                gf_log (this->name, GF_LOG_ERROR, "failed to "
-                                        "set write-%db+, with: %"PRId64,
-                                        (1<<i), count);
-                                goto out;
-                        }
-                }
+        if (stats->latency[i].avg == 0)
+            continue;
+        snprintf(key, sizeof(key), "%d-%d-avglatency", interval, i);
+        ret = dict_set_double(dict, key, stats->latency[i].avg);
+        if (ret) {
+            gf_log(this->name, GF_LOG_ERROR,
+                   "failed to set %s "
+                   "avglatency(%d) with %f",
+                   gf_fop_list[i], interval, stats->latency[i].avg);
+            goto out;
         }
-
-        for (i = 0; i < GF_FOP_MAXVALUE; i++) {
-                if (stats->fop_hits[i] == 0)
-                        continue;
-                snprintf (key, sizeof (key), "%d-%d-hits", interval, i);
-                ret = dict_set_uint64 (dict, key, stats->fop_hits[i]);
-                if (ret) {
-                        gf_log (this->name, GF_LOG_ERROR, "failed to "
-                                "set %s-fop-hits: %"PRIu64, gf_fop_list[i],
-                                stats->fop_hits[i]);
-                        goto out;
-                }
-
-                if (stats->latency[i].avg == 0)
-                        continue;
-                snprintf (key, sizeof (key), "%d-%d-avglatency", interval, i);
-                ret = dict_set_double (dict, key, stats->latency[i].avg);
-                if (ret) {
-                        gf_log (this->name, GF_LOG_ERROR, "failed to set %s "
-                                "avglatency(%d) with %f", gf_fop_list[i],
-                                interval, stats->latency[i].avg);
-                        goto out;
-                }
-                snprintf (key, sizeof (key), "%d-%d-minlatency", interval, i);
-                ret = dict_set_double (dict, key, stats->latency[i].min);
-                if (ret) {
-                        gf_log (this->name, GF_LOG_ERROR, "failed to set %s "
-                                "minlatency(%d) with %f", gf_fop_list[i],
-                                interval, stats->latency[i].min);
-                        goto out;
-                }
-                snprintf (key, sizeof (key), "%d-%d-maxlatency", interval, i);
-                ret = dict_set_double (dict, key, stats->latency[i].max);
-                if (ret) {
-                        gf_log (this->name, GF_LOG_ERROR, "failed to set %s "
-                                "maxlatency(%d) with %f", gf_fop_list[i],
-                                interval, stats->latency[i].max);
-                        goto out;
-                }
+        snprintf(key, sizeof(key), "%d-%d-minlatency", interval, i);
+        ret = dict_set_double(dict, key, stats->latency[i].min);
+        if (ret) {
+            gf_log(this->name, GF_LOG_ERROR,
+                   "failed to set %s "
+                   "minlatency(%d) with %f",
+                   gf_fop_list[i], interval, stats->latency[i].min);
+            goto out;
+        }
+        snprintf(key, sizeof(key), "%d-%d-maxlatency", interval, i);
+        ret = dict_set_double(dict, key, stats->latency[i].max);
+        if (ret) {
+            gf_log(this->name, GF_LOG_ERROR,
+                   "failed to set %s "
+                   "maxlatency(%d) with %f",
+                   gf_fop_list[i], interval, stats->latency[i].max);
+            goto out;
         }
+    }
+    for (i = 0; i < GF_UPCALL_FLAGS_MAXVALUE; i++) {
+        fop_hits = GF_ATOMIC_GET(stats->upcall_hits[i]);
+        if (fop_hits == 0)
+            continue;
+        snprintf(key, sizeof(key), "%d-%d-upcall-hits", interval, i);
+        ret = dict_set_uint64(dict, key, fop_hits);
+        if (ret) {
+            gf_log(this->name, GF_LOG_ERROR,
+                   "failed to "
+                   "set %s-upcall-hits: %" PRIu64,
+                   gf_upcall_list[i], fop_hits);
+            goto out;
+        }
+    }
 out:
-        gf_log (this->name, GF_LOG_DEBUG, "returning %d", ret);
-        return ret;
+    gf_log(this->name, GF_LOG_DEBUG, "returning %d", ret);
+    return ret;
 }
 
 int
-io_stats_dump_global (xlator_t *this, struct ios_global_stats *stats,
-                      struct timeval *now, int interval,
-                      struct ios_dump_args *args)
+io_stats_dump_global(xlator_t *this, struct ios_global_stats *stats, time_t now,
+                     int interval, struct ios_dump_args *args)
 {
-        int     ret = -1;
-
-        GF_ASSERT (args);
-        GF_ASSERT (now);
-        GF_ASSERT (stats);
-        GF_ASSERT (this);
-
-
-
-        switch (args->type) {
+    int ret = -1;
+
+    GF_ASSERT(args);
+    GF_ASSERT(now);
+    GF_ASSERT(stats);
+    GF_ASSERT(this);
+
+    switch (args->type) {
+        case IOS_DUMP_TYPE_JSON_FILE:
+            ret = io_stats_dump_global_to_json_logfp(this, stats, now, interval,
+                                                     args->u.logfp);
+            break;
         case IOS_DUMP_TYPE_FILE:
-                ret = io_stats_dump_global_to_logfp (this, stats, now,
-                                                     interval, args->u.logfp);
-        break;
+            ret = io_stats_dump_global_to_logfp(this, stats, now, interval,
+                                                args->u.logfp);
+            break;
         case IOS_DUMP_TYPE_DICT:
-                ret = io_stats_dump_global_to_dict (this, stats, now,
-                                                    interval, args->u.dict);
-        break;
+            ret = io_stats_dump_global_to_dict(this, stats, now, interval,
+                                               args->u.dict);
+            break;
         default:
-                GF_ASSERT (0);
-                ret = -1;
-        break;
-        }
-        return ret;
+            GF_ASSERT(0);
+            ret = -1;
+            break;
+    }
+    return ret;
 }
 
 int
-ios_dump_args_init (struct ios_dump_args *args, ios_dump_type_t type,
-                    void *output)
+ios_dump_args_init(struct ios_dump_args *args, ios_dump_type_t type,
+                   void *output)
 {
-        int             ret = 0;
+    int ret = 0;
 
-        GF_ASSERT (args);
-        GF_ASSERT (type > IOS_DUMP_TYPE_NONE && type < IOS_DUMP_TYPE_MAX);
-        GF_ASSERT (output);
+    GF_ASSERT(args);
+    GF_ASSERT(type > IOS_DUMP_TYPE_NONE && type < IOS_DUMP_TYPE_MAX);
+    GF_ASSERT(output);
 
-        args->type = type;
-        switch (args->type) {
+    args->type = type;
+    switch (args->type) {
+        case IOS_DUMP_TYPE_JSON_FILE:
         case IOS_DUMP_TYPE_FILE:
-                args->u.logfp = output;
-                break;
+            args->u.logfp = output;
+            break;
         case IOS_DUMP_TYPE_DICT:
-                args->u.dict = output;
-                break;
+            args->u.dict = output;
+            break;
         default:
-                GF_ASSERT (0);
-                ret = -1;
-        }
+            GF_ASSERT(0);
+            ret = -1;
+    }
 
-        return ret;
+    return ret;
 }
 
 static void
-ios_global_stats_clear (struct ios_global_stats *stats, struct timeval *now)
+ios_global_stats_clear(struct ios_global_stats *stats, time_t now)
 {
-        GF_ASSERT (stats);
-        GF_ASSERT (now);
+    GF_ASSERT(stats);
+    GF_ASSERT(now);
 
-        memset (stats, 0, sizeof (*stats));
-        stats->started_at = *now;
+    memset(stats, 0, sizeof(*stats));
+    stats->started_at = now;
 }
 
 int
-io_stats_dump (xlator_t *this, struct ios_dump_args *args,
-               gf1_cli_info_op op, gf_boolean_t is_peek)
+io_stats_dump(xlator_t *this, struct ios_dump_args *args, ios_info_op_t op,
+              gf_boolean_t is_peek)
 {
-        struct ios_conf         *conf = NULL;
-        struct ios_global_stats  cumulative = {0, };
-        struct ios_global_stats  incremental = {0, };
-        int                      increment = 0;
-        struct timeval           now;
+    struct ios_conf *conf = NULL;
+    struct ios_global_stats cumulative = {};
+    struct ios_global_stats incremental = {};
+    int increment = 0;
+    time_t now = 0;
 
-        GF_ASSERT (this);
-        GF_ASSERT (args);
-        GF_ASSERT (args->type > IOS_DUMP_TYPE_NONE);
-        GF_ASSERT (args->type < IOS_DUMP_TYPE_MAX);
+    GF_ASSERT(this);
+    GF_ASSERT(args);
+    GF_ASSERT(args->type > IOS_DUMP_TYPE_NONE);
+    GF_ASSERT(args->type < IOS_DUMP_TYPE_MAX);
 
-        conf = this->private;
+    conf = this->private;
+    now = gf_time();
 
-        gettimeofday (&now, NULL);
-        LOCK (&conf->lock);
-        {
-                if (op == GF_CLI_INFO_ALL ||
-                    op == GF_CLI_INFO_CUMULATIVE)
-                        cumulative  = conf->cumulative;
+    LOCK(&conf->lock);
+    {
+        if (op == GF_IOS_INFO_ALL || op == GF_IOS_INFO_CUMULATIVE)
+            cumulative = conf->cumulative;
 
-                if (op == GF_CLI_INFO_ALL ||
-                    op == GF_CLI_INFO_INCREMENTAL) {
-                        incremental = conf->incremental;
-                        increment = conf->increment;
+        if (op == GF_IOS_INFO_ALL || op == GF_IOS_INFO_INCREMENTAL) {
+            incremental = conf->incremental;
+            increment = conf->increment;
 
-                        if (!is_peek) {
-                                increment = conf->increment++;
+            if (!is_peek) {
+                increment = conf->increment++;
 
-                                ios_global_stats_clear (&conf->incremental,
-                                                        &now);
-                        }
-                }
+                ios_global_stats_clear(&conf->incremental, now);
+            }
         }
-        UNLOCK (&conf->lock);
+    }
+    UNLOCK(&conf->lock);
 
-        if (op == GF_CLI_INFO_ALL ||
-            op == GF_CLI_INFO_CUMULATIVE)
-                io_stats_dump_global (this, &cumulative, &now, -1, args);
+    if (op == GF_IOS_INFO_ALL || op == GF_IOS_INFO_CUMULATIVE)
+        io_stats_dump_global(this, &cumulative, now, -1, args);
 
-        if (op == GF_CLI_INFO_ALL ||
-            op == GF_CLI_INFO_INCREMENTAL)
-                io_stats_dump_global (this, &incremental, &now, increment, args);
+    if (op == GF_IOS_INFO_ALL || op == GF_IOS_INFO_INCREMENTAL)
+        io_stats_dump_global(this, &incremental, now, increment, args);
 
-        return 0;
+    return 0;
 }
 
-
 int
-io_stats_dump_fd (xlator_t *this, struct ios_fd *iosfd)
+io_stats_dump_fd(xlator_t *this, struct ios_fd *iosfd)
 {
-        struct ios_conf         *conf = NULL;
-        struct timeval           now;
-        uint64_t                 sec = 0;
-        uint64_t                 usec = 0;
-        int                      i = 0;
-
-        conf = this->private;
-
-        if (!conf->dump_fd_stats)
-                return 0;
-
-        if (!iosfd)
-                return 0;
+    struct ios_conf *conf = NULL;
+    struct timeval now;
+    int i = 0;
+    double usecs = 0;
+    uint64_t data_read = 0;
+    uint64_t data_written = 0;
+    uint64_t block_count_read = 0;
+    uint64_t block_count_write = 0;
+
+    conf = this->private;
+
+    if (!conf->dump_fd_stats)
+        return 0;
 
-        gettimeofday (&now, NULL);
+    if (!iosfd)
+        return 0;
 
-        if (iosfd->opened_at.tv_usec > now.tv_usec) {
-                now.tv_usec += 1000000;
-                now.tv_usec--;
-        }
+    gettimeofday(&now, NULL);
+    usecs = gf_tvdiff(&iosfd->opened_at, &now);
+
+    gf_log(this->name, GF_LOG_INFO, "--- fd stats ---");
+
+    if (iosfd->filename)
+        gf_log(this->name, GF_LOG_INFO, "      Filename : %s", iosfd->filename);
+
+    if (usecs)
+        gf_log(this->name, GF_LOG_INFO, "      Lifetime : %lf secs", usecs);
+
+    data_read = GF_ATOMIC_GET(iosfd->data_read);
+    if (data_read)
+        gf_log(this->name, GF_LOG_INFO, "     BytesRead : %" PRId64 " bytes",
+               data_read);
+
+    data_written = GF_ATOMIC_GET(iosfd->data_written);
+    if (data_written)
+        gf_log(this->name, GF_LOG_INFO, "  BytesWritten : %" PRId64 " bytes",
+               data_written);
+
+    for (i = 0; i < 32; i++) {
+        block_count_read = GF_ATOMIC_GET(iosfd->block_count_read[i]);
+        if (block_count_read)
+            gf_log(this->name, GF_LOG_INFO,
+                   " Read %06db+ :"
+                   "%" PRId64,
+                   (1 << i), block_count_read);
+    }
+    for (i = 0; i < IOS_BLOCK_COUNT_SIZE; i++) {
+        block_count_write = GF_ATOMIC_GET(iosfd->block_count_write[i]);
+        if (block_count_write)
+            gf_log(this->name, GF_LOG_INFO, "Write %06db+ : %" PRId64, (1 << i),
+                   block_count_write);
+    }
+    return 0;
+}
 
-        sec = now.tv_sec - iosfd->opened_at.tv_sec;
-        usec = now.tv_usec - iosfd->opened_at.tv_usec;
-
-        gf_log (this->name, GF_LOG_INFO,
-                "--- fd stats ---");
-
-        if (iosfd->filename)
-                gf_log (this->name, GF_LOG_INFO,
-                        "      Filename : %s",
-                        iosfd->filename);
-
-        if (sec)
-                gf_log (this->name, GF_LOG_INFO,
-                        "      Lifetime : %"PRId64"secs, %"PRId64"usecs",
-                        sec, usec);
-
-        if (iosfd->data_read)
-                gf_log (this->name, GF_LOG_INFO,
-                        "     BytesRead : %"PRId64" bytes",
-                        iosfd->data_read);
-
-        if (iosfd->data_written)
-                gf_log (this->name, GF_LOG_INFO,
-                        "  BytesWritten : %"PRId64" bytes",
-                        iosfd->data_written);
-
-        for (i = 0; i < 32; i++) {
-                if (iosfd->block_count_read[i])
-                        gf_log (this->name, GF_LOG_INFO,
-                                " Read %06db+ : %"PRId64,
-                                (1 << i), iosfd->block_count_read[i]);
-        }
-        for (i = 0; i < 32; i++) {
-                if (iosfd->block_count_write[i])
-                        gf_log (this->name, GF_LOG_INFO,
-                                "Write %06db+ : %"PRId64,
-                                (1 << i), iosfd->block_count_write[i]);
-        }
-        return 0;
+void
+collect_ios_latency_sample(struct ios_conf *conf, glusterfs_fop_t fop_type,
+                           double elapsed, call_frame_t *frame)
+{
+    ios_sample_buf_t *ios_sample_buf = NULL;
+    ios_sample_t *ios_sample = NULL;
+    struct timespec *timestamp = NULL;
+    call_stack_t *root = NULL;
+
+    ios_sample_buf = conf->ios_sample_buf;
+    LOCK(&conf->ios_sampling_lock);
+    if (conf->ios_sample_interval == 0 ||
+        ios_sample_buf->observed % conf->ios_sample_interval != 0)
+        goto out;
+
+    timestamp = &frame->begin;
+    root = frame->root;
+
+    ios_sample = &(ios_sample_buf->ios_samples[ios_sample_buf->pos]);
+    ios_sample->elapsed = elapsed;
+    ios_sample->fop_type = fop_type;
+    ios_sample->uid = root->uid;
+    ios_sample->gid = root->gid;
+    (ios_sample->timestamp).tv_sec = timestamp->tv_sec;
+    (ios_sample->timestamp).tv_usec = timestamp->tv_nsec / 1000;
+    memcpy(&ios_sample->identifier, &root->identifier,
+           sizeof(root->identifier));
+
+    /* We've reached the end of the circular buffer, start from the
+     * beginning. */
+    if (ios_sample_buf->pos == (ios_sample_buf->size - 1))
+        ios_sample_buf->pos = 0;
+    else
+        ios_sample_buf->pos++;
+    ios_sample_buf->collected++;
+out:
+    ios_sample_buf->observed++;
+    UNLOCK(&conf->ios_sampling_lock);
+    return;
 }
 
 static void
-update_ios_latency_stats (struct ios_global_stats   *stats, double elapsed,
-                          glusterfs_fop_t op)
+update_ios_latency_stats(struct ios_global_stats *stats, double elapsed,
+                         glusterfs_fop_t op)
 {
-        double avg;
+    double avg;
 
-        GF_ASSERT (stats);
+    GF_ASSERT(stats);
 
-        stats->latency[op].total += elapsed;
+    stats->latency[op].total += elapsed;
 
-        if (!stats->latency[op].min)
-                stats->latency[op].min = elapsed;
-        if (stats->latency[op].min > elapsed)
-                stats->latency[op].min = elapsed;
-        if (stats->latency[op].max < elapsed)
-                stats->latency[op].max = elapsed;
+    if (!stats->latency[op].min)
+        stats->latency[op].min = elapsed;
+    if (stats->latency[op].min > elapsed)
+        stats->latency[op].min = elapsed;
+    if (stats->latency[op].max < elapsed)
+        stats->latency[op].max = elapsed;
 
-        avg = stats->latency[op].avg;
+    avg = stats->latency[op].avg;
 
-        stats->latency[op].avg = avg + (elapsed - avg) / stats->fop_hits[op];
+    stats->latency[op].avg = avg + (elapsed - avg) /
+                                       GF_ATOMIC_GET(stats->fop_hits[op]);
 }
 
 int
-update_ios_latency (struct ios_conf *conf, call_frame_t *frame,
-                    glusterfs_fop_t op)
+update_ios_latency(struct ios_conf *conf, call_frame_t *frame,
+                   glusterfs_fop_t op)
 {
-        double elapsed;
-        struct timeval *begin, *end;
+    double elapsed;
+    struct timespec *begin, *end;
 
-        begin = &frame->begin;
-        end   = &frame->end;
+    begin = &frame->begin;
+    end = &frame->end;
 
-        elapsed = (end->tv_sec - begin->tv_sec) * 1e6
-                + (end->tv_usec - begin->tv_usec);
+    elapsed = gf_tsdiff(begin, end) / 1000.0;
 
-        update_ios_latency_stats (&conf->cumulative, elapsed, op);
-        update_ios_latency_stats (&conf->incremental, elapsed, op);
+    update_ios_latency_stats(&conf->cumulative, elapsed, op);
+    update_ios_latency_stats(&conf->incremental, elapsed, op);
+    collect_ios_latency_sample(conf, op, elapsed, frame);
 
-        return 0;
+    return 0;
 }
 
 int32_t
-io_stats_dump_stats_to_dict (xlator_t *this, dict_t *resp,
-                             ios_stats_type_t flags, int32_t list_cnt)
-{
-        struct ios_conf         *conf = NULL;
-        int                      cnt  = 0;
-        char                     key[256];
-        struct ios_stat_head    *list_head = NULL;
-        struct ios_stat_list    *entry = NULL;
-        int                      ret = -1;
-        ios_stats_thru_t         index = IOS_STATS_THRU_MAX;
-        char                     timestr[256] = {0, };
-        char                    *dict_timestr = NULL;
-
-        conf = this->private;
-
-        switch (flags) {
-                case IOS_STATS_TYPE_OPEN:
-                        list_head = &conf->list[IOS_STATS_TYPE_OPEN];
-                        LOCK (&conf->lock);
-                        {
-                                ret = dict_set_uint64 (resp, "current-open",
-                                                     conf->cumulative.nr_opens);
-                                if (ret)
-                                        goto unlock;
-                                ret = dict_set_uint64 (resp, "max-open",
-                                                       conf->cumulative.max_nr_opens);
-
-                                gf_time_fmt (timestr, sizeof timestr,
-                                             conf->cumulative.max_openfd_time.tv_sec,
-                                             gf_timefmt_FT);
-                                if (conf->cumulative.max_openfd_time.tv_sec)
-                                        snprintf (timestr + strlen (timestr), sizeof timestr - strlen (timestr),
-                                                  ".%"GF_PRI_SUSECONDS,
-                                                  conf->cumulative.max_openfd_time.tv_usec);
-
-                                dict_timestr = gf_strdup (timestr);
-                                if (!dict_timestr)
-                                        goto unlock;
-                                ret = dict_set_dynstr (resp, "max-openfd-time",
-                                                       dict_timestr);
-                                if (ret)
-                                        goto unlock;
-                        }
+io_stats_dump_stats_to_dict(xlator_t *this, dict_t *resp,
+                            ios_stats_type_t flags, int32_t list_cnt)
+{
+    struct ios_conf *conf = NULL;
+    int cnt = 0;
+    char key[32];
+    int keylen;
+    struct ios_stat_head *list_head = NULL;
+    struct ios_stat_list *entry = NULL;
+    int ret = -1;
+    ios_stats_thru_t index = IOS_STATS_THRU_MAX;
+    char timestr[GF_TIMESTR_SIZE] = {
+        0,
+    };
+    char *dict_timestr = NULL;
+
+    conf = this->private;
+
+    switch (flags) {
+        case IOS_STATS_TYPE_OPEN:
+            list_head = &conf->list[IOS_STATS_TYPE_OPEN];
+            LOCK(&conf->lock);
+            {
+                ret = dict_set_uint64(resp, "current-open",
+                                      conf->cumulative.nr_opens);
+                if (ret)
+                    goto unlock;
+                ret = dict_set_uint64(resp, "max-open",
+                                      conf->cumulative.max_nr_opens);
+
+                gf_time_fmt_tv(timestr, sizeof timestr,
+                               &conf->cumulative.max_openfd_time,
+                               gf_timefmt_FT);
+
+                dict_timestr = gf_strdup(timestr);
+                if (!dict_timestr)
+                    goto unlock;
+                ret = dict_set_dynstr(resp, "max-openfd-time", dict_timestr);
+                if (ret)
+                    goto unlock;
+            }
         unlock:
-                        UNLOCK (&conf->lock);
-                        /* Do not proceed if we came here because of some error
-                         * during the dict operation */
-                        if (ret)
-                                goto out;
-                        break;
-                case IOS_STATS_TYPE_READ:
-                        list_head = &conf->list[IOS_STATS_TYPE_READ];
-                        break;
-                case IOS_STATS_TYPE_WRITE:
-                        list_head = &conf->list[IOS_STATS_TYPE_WRITE];
-                        break;
-                case IOS_STATS_TYPE_OPENDIR:
-                        list_head = &conf->list[IOS_STATS_TYPE_OPENDIR];
-                        break;
-                case IOS_STATS_TYPE_READDIRP:
-                        list_head = &conf->list[IOS_STATS_TYPE_READDIRP];
-                        break;
-                case IOS_STATS_TYPE_READ_THROUGHPUT:
-                        list_head = &conf->thru_list[IOS_STATS_THRU_READ];
-                        index = IOS_STATS_THRU_READ;
-                        break;
-                case IOS_STATS_TYPE_WRITE_THROUGHPUT:
-                        list_head = &conf->thru_list[IOS_STATS_THRU_WRITE];
-                        index = IOS_STATS_THRU_WRITE;
-                        break;
-
-                default:
-                       goto out;
-        }
-        ret = dict_set_int32 (resp, "top-op", flags);
-        if (!list_cnt)
+            UNLOCK(&conf->lock);
+            /* Do not proceed if we came here because of some error
+             * during the dict operation */
+            if (ret)
                 goto out;
-        LOCK (&list_head->lock);
-        {
-                list_for_each_entry (entry, &list_head->iosstats->list, list) {
-
-                        cnt++;
-                        snprintf (key, 256, "%s-%d", "filename", cnt);
-                        ret = dict_set_str (resp, key, entry->iosstat->filename);
-                        if (ret)
-                                goto unlock_list_head;
-                         snprintf (key, 256, "%s-%d", "value",cnt);
-                         ret = dict_set_uint64 (resp, key, entry->value);
-                         if (ret)
-                                 goto unlock_list_head;
-                         if (index != IOS_STATS_THRU_MAX) {
-                                 snprintf (key, 256, "%s-%d", "time-sec", cnt);
-                                 ret = dict_set_int32 (resp, key,
-                                         entry->iosstat->thru_counters[index].time.tv_sec);
-                                 if (ret)
-                                         goto unlock_list_head;
-                                 snprintf (key, 256, "%s-%d", "time-usec", cnt);
-                                 ret = dict_set_int32 (resp, key,
-                                         entry->iosstat->thru_counters[index].time.tv_usec);
-                                 if (ret)
-                                         goto unlock_list_head;
-                         }
-                         if (cnt == list_cnt)
-                                 break;
+            break;
+        case IOS_STATS_TYPE_READ:
+            list_head = &conf->list[IOS_STATS_TYPE_READ];
+            break;
+        case IOS_STATS_TYPE_WRITE:
+            list_head = &conf->list[IOS_STATS_TYPE_WRITE];
+            break;
+        case IOS_STATS_TYPE_OPENDIR:
+            list_head = &conf->list[IOS_STATS_TYPE_OPENDIR];
+            break;
+        case IOS_STATS_TYPE_READDIRP:
+            list_head = &conf->list[IOS_STATS_TYPE_READDIRP];
+            break;
+        case IOS_STATS_TYPE_READ_THROUGHPUT:
+            list_head = &conf->thru_list[IOS_STATS_THRU_READ];
+            index = IOS_STATS_THRU_READ;
+            break;
+        case IOS_STATS_TYPE_WRITE_THROUGHPUT:
+            list_head = &conf->thru_list[IOS_STATS_THRU_WRITE];
+            index = IOS_STATS_THRU_WRITE;
+            break;
 
-                }
+        default:
+            goto out;
+    }
+    ret = dict_set_int32_sizen(resp, "top-op", flags);
+    if (!list_cnt)
+        goto out;
+    LOCK(&list_head->lock);
+    {
+        list_for_each_entry(entry, &list_head->iosstats->list, list)
+        {
+            cnt++;
+            keylen = snprintf(key, sizeof(key), "filename-%d", cnt);
+            ret = dict_set_strn(resp, key, keylen, entry->iosstat->filename);
+            if (ret)
+                goto unlock_list_head;
+            snprintf(key, sizeof(key), "value-%d", cnt);
+            ret = dict_set_uint64(resp, key, entry->value);
+            if (ret)
+                goto unlock_list_head;
+            if (index != IOS_STATS_THRU_MAX) {
+                keylen = snprintf(key, sizeof(key), "time-sec-%d", cnt);
+                ret = dict_set_int32n(
+                    resp, key, keylen,
+                    entry->iosstat->thru_counters[index].time.tv_sec);
+                if (ret)
+                    goto unlock_list_head;
+                keylen = snprintf(key, sizeof(key), "time-usec-%d", cnt);
+                ret = dict_set_int32n(
+                    resp, key, keylen,
+                    entry->iosstat->thru_counters[index].time.tv_usec);
+                if (ret)
+                    goto unlock_list_head;
+            }
+            if (cnt == list_cnt)
+                break;
         }
+    }
 unlock_list_head:
-        UNLOCK (&list_head->lock);
-        /* ret is !=0 if some dict operation in the above critical region
-         * failed. */
-        if (ret)
-                goto out;
-        ret = dict_set_int32 (resp, "members", cnt);
- out:
-        return ret;
+    UNLOCK(&list_head->lock);
+    /* ret is !=0 if some dict operation in the above critical region
+     * failed. */
+    if (ret)
+        goto out;
+    ret = dict_set_int32_sizen(resp, "members", cnt);
+out:
+    return ret;
 }
 
-int
-io_stats_create_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                     int32_t op_ret, int32_t op_errno, fd_t *fd,
-                     inode_t *inode, struct iatt *buf,
-                     struct iatt *preparent, struct iatt *postparent, dict_t *xdata)
+static struct ios_stat *
+ios_init_iosstat(xlator_t *this, char *path, uuid_t gfid, inode_t *inode)
 {
-        struct ios_fd *iosfd = NULL;
-        char          *path = NULL;
-        struct ios_stat *iosstat = NULL;
-        struct ios_conf   *conf = NULL;
+    struct ios_stat *iosstat = NULL;
+    int i = 0;
 
-        conf = this->private;
+    iosstat = GF_CALLOC(1, sizeof(*iosstat), gf_io_stats_mt_ios_stat);
+    if (!iosstat)
+        goto out;
 
-        path = frame->local;
-        frame->local = NULL;
+    iosstat->filename = gf_strdup(path);
+    gf_uuid_copy(iosstat->gfid, gfid);
+    LOCK_INIT(&iosstat->lock);
 
-        if (!path)
-                goto unwind;
-
-        if (op_ret < 0) {
-                GF_FREE (path);
-                goto unwind;
-        }
+    for (i = 0; i < IOS_STATS_TYPE_MAX; i++)
+        GF_ATOMIC_INIT(iosstat->counters[i], 0);
 
-        iosfd = GF_CALLOC (1, sizeof (*iosfd), gf_io_stats_mt_ios_fd);
-        if (!iosfd) {
-                GF_FREE (path);
-                goto unwind;
-        }
+    ios_inode_ctx_set(inode, this, iosstat);
 
-        iosfd->filename = path;
-        gettimeofday (&iosfd->opened_at, NULL);
+out:
+    return iosstat;
+}
 
-        ios_fd_ctx_set (fd, this, iosfd);
-        LOCK (&conf->lock);
-        {
-                conf->cumulative.nr_opens++;
-                if (conf->cumulative.nr_opens > conf->cumulative.max_nr_opens) {
-                        conf->cumulative.max_nr_opens = conf->cumulative.nr_opens;
-                        conf->cumulative.max_openfd_time = iosfd->opened_at;
-                }
+int
+io_stats_create_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                    int32_t op_ret, int32_t op_errno, fd_t *fd, inode_t *inode,
+                    struct iatt *buf, struct iatt *preparent,
+                    struct iatt *postparent, dict_t *xdata)
+{
+    struct ios_fd *iosfd = NULL;
+    char *path = NULL;
+    struct ios_stat *iosstat = NULL;
+    struct ios_conf *conf = NULL;
+
+    conf = this->private;
+
+    path = frame->local;
+    frame->local = NULL;
+
+    if (!path)
+        goto unwind;
+
+    if (op_ret < 0) {
+        GF_FREE(path);
+        goto unwind;
+    }
+
+    iosfd = GF_CALLOC(1, sizeof(*iosfd), gf_io_stats_mt_ios_fd);
+    if (!iosfd) {
+        GF_FREE(path);
+        goto unwind;
+    }
+
+    iosfd->filename = path;
+    gettimeofday(&iosfd->opened_at, NULL);
+
+    ios_fd_ctx_set(fd, this, iosfd);
+    LOCK(&conf->lock);
+    {
+        conf->cumulative.nr_opens++;
+        if (conf->cumulative.nr_opens > conf->cumulative.max_nr_opens) {
+            conf->cumulative.max_nr_opens = conf->cumulative.nr_opens;
+            conf->cumulative.max_openfd_time = iosfd->opened_at;
         }
-        UNLOCK (&conf->lock);
+    }
+    UNLOCK(&conf->lock);
 
-        iosstat = GF_CALLOC (1, sizeof (*iosstat), gf_io_stats_mt_ios_stat);
-        if (!iosstat) {
-                GF_FREE (path);
-                goto unwind;
-        }
-        iosstat->filename = gf_strdup (path);
-        gf_uuid_copy (iosstat->gfid, buf->ia_gfid);
-        LOCK_INIT (&iosstat->lock);
-        ios_inode_ctx_set (fd->inode, this, iosstat);
+    iosstat = ios_init_iosstat(this, path, buf->ia_gfid, inode);
+    if (!iosstat)
+        GF_FREE(path);
 
 unwind:
-        UPDATE_PROFILE_STATS (frame, CREATE);
-        STACK_UNWIND_STRICT (create, frame, op_ret, op_errno, fd, inode, buf,
-                             preparent, postparent, xdata);
-        return 0;
+    UPDATE_PROFILE_STATS(frame, CREATE);
+    STACK_UNWIND_STRICT(create, frame, op_ret, op_errno, fd, inode, buf,
+                        preparent, postparent, xdata);
+    return 0;
 }
 
-
 int
-io_stats_open_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                   int32_t op_ret, int32_t op_errno, fd_t *fd, dict_t *xdata)
+io_stats_open_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                  int32_t op_ret, int32_t op_errno, fd_t *fd, dict_t *xdata)
 {
-        struct ios_fd *iosfd = NULL;
-        char          *path = NULL;
-        struct   ios_stat *iosstat = NULL;
-        struct ios_conf   *conf = NULL;
-
-        conf = this->private;
-        path = frame->local;
-        frame->local = NULL;
-
-        if (!path)
-                goto unwind;
-
-        if (op_ret < 0) {
-                GF_FREE (path);
-                goto unwind;
-        }
-
-        iosfd = GF_CALLOC (1, sizeof (*iosfd), gf_io_stats_mt_ios_fd);
-        if (!iosfd) {
-                GF_FREE (path);
-                goto unwind;
-        }
-
-        iosfd->filename = path;
-        gettimeofday (&iosfd->opened_at, NULL);
-
-        ios_fd_ctx_set (fd, this, iosfd);
-
-        ios_inode_ctx_get (fd->inode, this, &iosstat);
-        if (!iosstat) {
-                iosstat = GF_CALLOC (1, sizeof (*iosstat),
-                                     gf_io_stats_mt_ios_stat);
-                if (iosstat) {
-                        iosstat->filename = gf_strdup (path);
-                        gf_uuid_copy (iosstat->gfid, fd->inode->gfid);
-                        LOCK_INIT (&iosstat->lock);
-                        ios_inode_ctx_set (fd->inode, this, iosstat);
-                }
-        }
-
-        LOCK (&conf->lock);
-        {
-                conf->cumulative.nr_opens++;
-                if (conf->cumulative.nr_opens > conf->cumulative.max_nr_opens) {
-                        conf->cumulative.max_nr_opens = conf->cumulative.nr_opens;
-                        conf->cumulative.max_openfd_time = iosfd->opened_at;
-                }
-        }
-        UNLOCK (&conf->lock);
-        if (iosstat) {
-              BUMP_STATS (iosstat, IOS_STATS_TYPE_OPEN);
-              iosstat = NULL;
+    struct ios_fd *iosfd = NULL;
+    char *path = NULL;
+    struct ios_stat *iosstat = NULL;
+    struct ios_conf *conf = NULL;
+    int i = 0;
+
+    conf = this->private;
+    path = frame->local;
+    frame->local = NULL;
+
+    if (!path)
+        goto unwind;
+
+    if (op_ret < 0) {
+        GF_FREE(path);
+        goto unwind;
+    }
+
+    iosfd = GF_CALLOC(1, sizeof(*iosfd), gf_io_stats_mt_ios_fd);
+    if (!iosfd) {
+        GF_FREE(path);
+        goto unwind;
+    }
+
+    iosfd->filename = path;
+    GF_ATOMIC_INIT(iosfd->data_read, 0);
+    GF_ATOMIC_INIT(iosfd->data_written, 0);
+    for (i = 0; i < IOS_BLOCK_COUNT_SIZE; i++) {
+        GF_ATOMIC_INIT(iosfd->block_count_write[i], 0);
+        GF_ATOMIC_INIT(iosfd->block_count_read[i], 0);
+    }
+    gettimeofday(&iosfd->opened_at, NULL);
+
+    ios_fd_ctx_set(fd, this, iosfd);
+
+    ios_inode_ctx_get(fd->inode, this, &iosstat);
+    if (!iosstat) {
+        iosstat = ios_init_iosstat(this, path, fd->inode->gfid, fd->inode);
+    }
+
+    LOCK(&conf->lock);
+    {
+        conf->cumulative.nr_opens++;
+        if (conf->cumulative.nr_opens > conf->cumulative.max_nr_opens) {
+            conf->cumulative.max_nr_opens = conf->cumulative.nr_opens;
+            conf->cumulative.max_openfd_time = iosfd->opened_at;
         }
+    }
+    UNLOCK(&conf->lock);
+    if (iosstat) {
+        ios_bump_stats(this, iosstat, IOS_STATS_TYPE_OPEN);
+        iosstat = NULL;
+    }
 unwind:
-        UPDATE_PROFILE_STATS (frame, OPEN);
-
-        STACK_UNWIND_STRICT (open, frame, op_ret, op_errno, fd, xdata);
-        return 0;
+    UPDATE_PROFILE_STATS(frame, OPEN);
 
+    STACK_UNWIND_STRICT(open, frame, op_ret, op_errno, fd, xdata);
+    return 0;
 }
 
-
 int
-io_stats_stat_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                   int32_t op_ret, int32_t op_errno, struct iatt *buf, dict_t *xdata)
+io_stats_stat_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                  int32_t op_ret, int32_t op_errno, struct iatt *buf,
+                  dict_t *xdata)
 {
-        UPDATE_PROFILE_STATS (frame, STAT);
-        STACK_UNWIND_STRICT (stat, frame, op_ret, op_errno, buf, xdata);
-        return 0;
+    UPDATE_PROFILE_STATS(frame, STAT);
+    STACK_UNWIND_STRICT(stat, frame, op_ret, op_errno, buf, xdata);
+    return 0;
 }
 
+int
+io_stats_readv_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                   int32_t op_ret, int32_t op_errno, struct iovec *vector,
+                   int32_t count, struct iatt *buf, struct iobref *iobref,
+                   dict_t *xdata)
+{
+    int len = 0;
+    fd_t *fd = NULL;
+    struct ios_stat *iosstat = NULL;
+
+    fd = frame->local;
+    frame->local = NULL;
+
+    if (op_ret > 0) {
+        len = iov_length(vector, count);
+        ios_bump_read(this, fd, len);
+    }
+
+    UPDATE_PROFILE_STATS(frame, READ);
+    ios_inode_ctx_get(fd->inode, this, &iosstat);
+
+    if (iosstat) {
+        ios_bump_stats(this, iosstat, IOS_STATS_TYPE_READ);
+        BUMP_THROUGHPUT(iosstat, IOS_STATS_THRU_READ);
+        iosstat = NULL;
+    }
+
+    STACK_UNWIND_STRICT(readv, frame, op_ret, op_errno, vector, count, buf,
+                        iobref, xdata);
+    return 0;
+}
 
 int
-io_stats_readv_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                    int32_t op_ret, int32_t op_errno,
-                    struct iovec *vector, int32_t count,
-                    struct iatt *buf, struct iobref *iobref, dict_t *xdata)
+io_stats_writev_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                    int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
+                    struct iatt *postbuf, dict_t *xdata)
 {
-        int              len = 0;
-        fd_t            *fd = NULL;
-        struct ios_stat *iosstat = NULL;
+    struct ios_stat *iosstat = NULL;
+    inode_t *inode = NULL;
 
-        fd = frame->local;
+    UPDATE_PROFILE_STATS(frame, WRITE);
+    if (frame->local) {
+        inode = frame->local;
         frame->local = NULL;
-
-        if (op_ret > 0) {
-                len = iov_length (vector, count);
-                BUMP_READ (fd, len);
-        }
-
-        UPDATE_PROFILE_STATS (frame, READ);
-        ios_inode_ctx_get (fd->inode, this, &iosstat);
-
+        ios_inode_ctx_get(inode, this, &iosstat);
         if (iosstat) {
-              BUMP_STATS (iosstat, IOS_STATS_TYPE_READ);
-              BUMP_THROUGHPUT (iosstat, IOS_STATS_THRU_READ);
-              iosstat = NULL;
+            ios_bump_stats(this, iosstat, IOS_STATS_TYPE_WRITE);
+            BUMP_THROUGHPUT(iosstat, IOS_STATS_THRU_WRITE);
+            inode = NULL;
+            iosstat = NULL;
         }
+    }
 
-        STACK_UNWIND_STRICT (readv, frame, op_ret, op_errno,
-                             vector, count, buf, iobref, xdata);
-        return 0;
-
+    STACK_UNWIND_STRICT(writev, frame, op_ret, op_errno, prebuf, postbuf,
+                        xdata);
+    return 0;
 }
 
-
 int
-io_stats_writev_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                     int32_t op_ret, int32_t op_errno,
-                     struct iatt *prebuf, struct iatt *postbuf, dict_t *xdata)
+io_stats_copy_file_range_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                             int32_t op_ret, int32_t op_errno,
+                             struct iatt *stbuf, struct iatt *prebuf_dst,
+                             struct iatt *postbuf_dst, dict_t *xdata)
 {
-        struct ios_stat *iosstat = NULL;
-        inode_t         *inode   = NULL;
-
-        UPDATE_PROFILE_STATS (frame, WRITE);
-        if (frame->local){
-                inode = frame->local;
-                frame->local = NULL;
-                ios_inode_ctx_get (inode, this, &iosstat);
-                if (iosstat) {
-                        BUMP_STATS (iosstat, IOS_STATS_TYPE_WRITE);
-                        BUMP_THROUGHPUT (iosstat, IOS_STATS_THRU_WRITE);
-                        inode = NULL;
-                        iosstat = NULL;
-                }
-        }
-
-        STACK_UNWIND_STRICT (writev, frame, op_ret, op_errno, prebuf, postbuf, xdata);
-        return 0;
+    UPDATE_PROFILE_STATS(frame, COPY_FILE_RANGE);
 
+    STACK_UNWIND_STRICT(copy_file_range, frame, op_ret, op_errno, stbuf,
+                        prebuf_dst, postbuf_dst, xdata);
+    return 0;
 }
 
-
-
-
 int
-io_stats_readdirp_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                       int32_t op_ret, int32_t op_errno, gf_dirent_t *buf, dict_t *xdata)
+io_stats_readdirp_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                      int32_t op_ret, int32_t op_errno, gf_dirent_t *buf,
+                      dict_t *xdata)
 {
-        struct ios_stat *iosstat = NULL;
-        inode_t         *inode   = frame->local;
+    struct ios_stat *iosstat = NULL;
+    inode_t *inode = frame->local;
 
-        frame->local = NULL;
+    frame->local = NULL;
 
-        UPDATE_PROFILE_STATS (frame, READDIRP);
+    UPDATE_PROFILE_STATS(frame, READDIRP);
 
-        ios_inode_ctx_get (inode, this, &iosstat);
+    ios_inode_ctx_get(inode, this, &iosstat);
 
-        if (iosstat) {
-              BUMP_STATS (iosstat, IOS_STATS_TYPE_READDIRP);
-              iosstat = NULL;
-        }
+    if (iosstat) {
+        ios_bump_stats(this, iosstat, IOS_STATS_TYPE_READDIRP);
+        iosstat = NULL;
+    }
 
-        STACK_UNWIND_STRICT (readdirp, frame, op_ret, op_errno, buf, xdata);
-        return 0;
+    STACK_UNWIND_STRICT(readdirp, frame, op_ret, op_errno, buf, xdata);
+    return 0;
 }
 
-
 int
-io_stats_readdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                      int32_t op_ret, int32_t op_errno, gf_dirent_t *buf, dict_t *xdata)
+io_stats_readdir_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                     int32_t op_ret, int32_t op_errno, gf_dirent_t *buf,
+                     dict_t *xdata)
 {
-        UPDATE_PROFILE_STATS (frame, READDIR);
-        STACK_UNWIND_STRICT (readdir, frame, op_ret, op_errno, buf, xdata);
-        return 0;
+    UPDATE_PROFILE_STATS(frame, READDIR);
+    STACK_UNWIND_STRICT(readdir, frame, op_ret, op_errno, buf, xdata);
+    return 0;
 }
 
-
 int
-io_stats_fsync_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                    int32_t op_ret, int32_t op_errno,
-                    struct iatt *prebuf, struct iatt *postbuf, dict_t *xdata)
+io_stats_fsync_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                   int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
+                   struct iatt *postbuf, dict_t *xdata)
 {
-        UPDATE_PROFILE_STATS (frame, FSYNC);
-        STACK_UNWIND_STRICT (fsync, frame, op_ret, op_errno, prebuf, postbuf, xdata);
-        return 0;
+    UPDATE_PROFILE_STATS(frame, FSYNC);
+    STACK_UNWIND_STRICT(fsync, frame, op_ret, op_errno, prebuf, postbuf, xdata);
+    return 0;
 }
 
-
 int
-io_stats_setattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                      int32_t op_ret, int32_t op_errno,
-                      struct iatt *preop, struct iatt *postop, dict_t *xdata)
+io_stats_setattr_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                     int32_t op_ret, int32_t op_errno, struct iatt *preop,
+                     struct iatt *postop, dict_t *xdata)
 {
-        UPDATE_PROFILE_STATS (frame, SETATTR);
-        STACK_UNWIND_STRICT (setattr, frame, op_ret, op_errno, preop, postop, xdata);
-        return 0;
+    UPDATE_PROFILE_STATS(frame, SETATTR);
+    STACK_UNWIND_STRICT(setattr, frame, op_ret, op_errno, preop, postop, xdata);
+    return 0;
 }
 
-
 int
-io_stats_unlink_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                     int32_t op_ret, int32_t op_errno,
-                     struct iatt *preparent, struct iatt *postparent, dict_t *xdata)
+io_stats_unlink_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                    int32_t op_ret, int32_t op_errno, struct iatt *preparent,
+                    struct iatt *postparent, dict_t *xdata)
 {
-        UPDATE_PROFILE_STATS (frame, UNLINK);
-        STACK_UNWIND_STRICT (unlink, frame, op_ret, op_errno,
-                             preparent, postparent, xdata);
-        return 0;
-
+    UPDATE_PROFILE_STATS(frame, UNLINK);
+    STACK_UNWIND_STRICT(unlink, frame, op_ret, op_errno, preparent, postparent,
+                        xdata);
+    return 0;
 }
 
-
 int
-io_stats_rename_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                     int32_t op_ret, int32_t op_errno, struct iatt *buf,
-                     struct iatt *preoldparent, struct iatt *postoldparent,
-                     struct iatt *prenewparent, struct iatt *postnewparent, dict_t *xdata)
+io_stats_rename_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                    int32_t op_ret, int32_t op_errno, struct iatt *buf,
+                    struct iatt *preoldparent, struct iatt *postoldparent,
+                    struct iatt *prenewparent, struct iatt *postnewparent,
+                    dict_t *xdata)
 {
-        UPDATE_PROFILE_STATS (frame, RENAME);
-        STACK_UNWIND_STRICT (rename, frame, op_ret, op_errno, buf,
-                             preoldparent, postoldparent,
-                             prenewparent, postnewparent, xdata);
-        return 0;
+    UPDATE_PROFILE_STATS(frame, RENAME);
+    STACK_UNWIND_STRICT(rename, frame, op_ret, op_errno, buf, preoldparent,
+                        postoldparent, prenewparent, postnewparent, xdata);
+    return 0;
 }
 
-
 int
-io_stats_readlink_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                       int32_t op_ret, int32_t op_errno, const char *buf,
-                       struct iatt *sbuf, dict_t *xdata)
+io_stats_readlink_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                      int32_t op_ret, int32_t op_errno, const char *buf,
+                      struct iatt *sbuf, dict_t *xdata)
 {
-        UPDATE_PROFILE_STATS (frame, READLINK);
-        STACK_UNWIND_STRICT (readlink, frame, op_ret, op_errno, buf, sbuf, xdata);
-        return 0;
+    UPDATE_PROFILE_STATS(frame, READLINK);
+    STACK_UNWIND_STRICT(readlink, frame, op_ret, op_errno, buf, sbuf, xdata);
+    return 0;
 }
 
-
 int
-io_stats_lookup_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                     int32_t op_ret, int32_t op_errno,
-                     inode_t *inode, struct iatt *buf,
-                     dict_t *xdata, struct iatt *postparent)
+io_stats_lookup_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                    int32_t op_ret, int32_t op_errno, inode_t *inode,
+                    struct iatt *buf, dict_t *xdata, struct iatt *postparent)
 {
-        UPDATE_PROFILE_STATS (frame, LOOKUP);
-        STACK_UNWIND_STRICT (lookup, frame, op_ret, op_errno, inode, buf, xdata,
-                             postparent);
-        return 0;
+    UPDATE_PROFILE_STATS(frame, LOOKUP);
+    STACK_UNWIND_STRICT(lookup, frame, op_ret, op_errno, inode, buf, xdata,
+                        postparent);
+    return 0;
 }
 
-
 int
-io_stats_symlink_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                      int32_t op_ret, int32_t op_errno,
-                      inode_t *inode, struct iatt *buf,
-                      struct iatt *preparent, struct iatt *postparent, dict_t *xdata)
+io_stats_symlink_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                     int32_t op_ret, int32_t op_errno, inode_t *inode,
+                     struct iatt *buf, struct iatt *preparent,
+                     struct iatt *postparent, dict_t *xdata)
 {
-        UPDATE_PROFILE_STATS (frame, SYMLINK);
-        STACK_UNWIND_STRICT (symlink, frame, op_ret, op_errno, inode, buf,
-                             preparent, postparent, xdata);
-        return 0;
+    UPDATE_PROFILE_STATS(frame, SYMLINK);
+    STACK_UNWIND_STRICT(symlink, frame, op_ret, op_errno, inode, buf, preparent,
+                        postparent, xdata);
+    return 0;
 }
 
-
 int
-io_stats_mknod_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                    int32_t op_ret, int32_t op_errno,
-                    inode_t *inode, struct iatt *buf,
-                    struct iatt *preparent, struct iatt *postparent, dict_t *xdata)
+io_stats_mknod_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                   int32_t op_ret, int32_t op_errno, inode_t *inode,
+                   struct iatt *buf, struct iatt *preparent,
+                   struct iatt *postparent, dict_t *xdata)
 {
-        UPDATE_PROFILE_STATS (frame, MKNOD);
-        STACK_UNWIND_STRICT (mknod, frame, op_ret, op_errno, inode, buf,
-                             preparent, postparent, xdata);
-        return 0;
+    UPDATE_PROFILE_STATS(frame, MKNOD);
+    STACK_UNWIND_STRICT(mknod, frame, op_ret, op_errno, inode, buf, preparent,
+                        postparent, xdata);
+    return 0;
 }
 
-
 int
-io_stats_mkdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                    int32_t op_ret, int32_t op_errno,
-                    inode_t *inode, struct iatt *buf,
-                    struct iatt *preparent, struct iatt *postparent,
-                    dict_t *xdata)
+io_stats_mkdir_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                   int32_t op_ret, int32_t op_errno, inode_t *inode,
+                   struct iatt *buf, struct iatt *preparent,
+                   struct iatt *postparent, dict_t *xdata)
 {
-        struct ios_stat *iosstat = NULL;
-        char   *path = frame->local;
+    char *path = frame->local;
 
-        if (!path)
-                goto unwind;
+    if (!path)
+        goto unwind;
 
-        UPDATE_PROFILE_STATS (frame, MKDIR);
-        if (op_ret < 0)
-                goto unwind;
+    UPDATE_PROFILE_STATS(frame, MKDIR);
+    if (op_ret < 0)
+        goto unwind;
 
-        iosstat = GF_CALLOC (1, sizeof (*iosstat), gf_io_stats_mt_ios_stat);
-        if (iosstat) {
-                LOCK_INIT (&iosstat->lock);
-                iosstat->filename = gf_strdup(path);
-                gf_uuid_copy (iosstat->gfid, buf->ia_gfid);
-                ios_inode_ctx_set (inode, this, iosstat);
-        }
+    /* allocate a struct ios_stat and set the inode ctx */
+    ios_init_iosstat(this, path, buf->ia_gfid, inode);
 
 unwind:
-        /* local is assigned with path */
-        GF_FREE (frame->local);
-        frame->local = NULL;
-        STACK_UNWIND_STRICT (mkdir, frame, op_ret, op_errno, inode, buf,
-                             preparent, postparent, xdata);
-        return 0;
+    /* local is assigned with path */
+    GF_FREE(frame->local);
+    frame->local = NULL;
+    STACK_UNWIND_STRICT(mkdir, frame, op_ret, op_errno, inode, buf, preparent,
+                        postparent, xdata);
+    return 0;
 }
 
-
 int
-io_stats_link_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                   int32_t op_ret, int32_t op_errno,
-                   inode_t *inode, struct iatt *buf,
-                   struct iatt *preparent, struct iatt *postparent, dict_t *xdata)
+io_stats_link_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                  int32_t op_ret, int32_t op_errno, inode_t *inode,
+                  struct iatt *buf, struct iatt *preparent,
+                  struct iatt *postparent, dict_t *xdata)
 {
-        UPDATE_PROFILE_STATS (frame, LINK);
-        STACK_UNWIND_STRICT (link, frame, op_ret, op_errno, inode, buf,
-                             preparent, postparent, xdata);
-        return 0;
+    UPDATE_PROFILE_STATS(frame, LINK);
+    STACK_UNWIND_STRICT(link, frame, op_ret, op_errno, inode, buf, preparent,
+                        postparent, xdata);
+    return 0;
 }
 
-
 int
-io_stats_flush_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                    int32_t op_ret, int32_t op_errno, dict_t *xdata)
+io_stats_flush_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                   int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
-        UPDATE_PROFILE_STATS (frame, FLUSH);
-        STACK_UNWIND_STRICT (flush, frame, op_ret, op_errno, xdata);
-        return 0;
+    UPDATE_PROFILE_STATS(frame, FLUSH);
+    STACK_UNWIND_STRICT(flush, frame, op_ret, op_errno, xdata);
+    return 0;
 }
 
-
 int
-io_stats_opendir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                      int32_t op_ret, int32_t op_errno, fd_t *fd, dict_t *xdata)
+io_stats_opendir_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                     int32_t op_ret, int32_t op_errno, fd_t *fd, dict_t *xdata)
 {
-        struct ios_stat *iosstat = NULL;
-        int              ret     = -1;
+    struct ios_stat *iosstat = NULL;
+    int ret = -1;
 
-        UPDATE_PROFILE_STATS (frame, OPENDIR);
-        if (op_ret < 0)
-                goto unwind;
+    UPDATE_PROFILE_STATS(frame, OPENDIR);
+    if (op_ret < 0)
+        goto unwind;
 
-        ios_fd_ctx_set (fd, this, 0);
+    ios_fd_ctx_set(fd, this, 0);
 
-        ret = ios_inode_ctx_get (fd->inode, this, &iosstat);
-        if (!ret)
-                BUMP_STATS (iosstat, IOS_STATS_TYPE_OPENDIR);
+    ret = ios_inode_ctx_get(fd->inode, this, &iosstat);
+    if (!ret)
+        ios_bump_stats(this, iosstat, IOS_STATS_TYPE_OPENDIR);
 
 unwind:
-        STACK_UNWIND_STRICT (opendir, frame, op_ret, op_errno, fd, xdata);
-        return 0;
+    STACK_UNWIND_STRICT(opendir, frame, op_ret, op_errno, fd, xdata);
+    return 0;
 }
 
-
 int
-io_stats_rmdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                    int32_t op_ret, int32_t op_errno,
-                    struct iatt *preparent, struct iatt *postparent, dict_t *xdata)
+io_stats_rmdir_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                   int32_t op_ret, int32_t op_errno, struct iatt *preparent,
+                   struct iatt *postparent, dict_t *xdata)
 {
+    UPDATE_PROFILE_STATS(frame, RMDIR);
 
-        UPDATE_PROFILE_STATS (frame, RMDIR);
-
-        STACK_UNWIND_STRICT (rmdir, frame, op_ret, op_errno,
-                             preparent, postparent, xdata);
-        return 0;
+    STACK_UNWIND_STRICT(rmdir, frame, op_ret, op_errno, preparent, postparent,
+                        xdata);
+    return 0;
 }
 
-
 int
-io_stats_truncate_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                       int32_t op_ret, int32_t op_errno,
-                       struct iatt *prebuf, struct iatt *postbuf, dict_t *xdata)
+io_stats_truncate_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                      int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
+                      struct iatt *postbuf, dict_t *xdata)
 {
-        UPDATE_PROFILE_STATS (frame, TRUNCATE);
-        STACK_UNWIND_STRICT (truncate, frame, op_ret, op_errno,
-                             prebuf, postbuf, xdata);
-        return 0;
+    UPDATE_PROFILE_STATS(frame, TRUNCATE);
+    STACK_UNWIND_STRICT(truncate, frame, op_ret, op_errno, prebuf, postbuf,
+                        xdata);
+    return 0;
 }
 
-
 int
-io_stats_statfs_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                     int32_t op_ret, int32_t op_errno, struct statvfs *buf, dict_t *xdata)
+io_stats_statfs_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                    int32_t op_ret, int32_t op_errno, struct statvfs *buf,
+                    dict_t *xdata)
 {
-        UPDATE_PROFILE_STATS (frame, STATFS);
-        STACK_UNWIND_STRICT (statfs, frame, op_ret, op_errno, buf, xdata);
-        return 0;
+    UPDATE_PROFILE_STATS(frame, STATFS);
+    STACK_UNWIND_STRICT(statfs, frame, op_ret, op_errno, buf, xdata);
+    return 0;
 }
 
-
 int
-io_stats_setxattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                       int32_t op_ret, int32_t op_errno, dict_t *xdata)
+io_stats_setxattr_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                      int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
-        UPDATE_PROFILE_STATS (frame, SETXATTR);
-        STACK_UNWIND_STRICT (setxattr, frame, op_ret, op_errno, xdata);
-        return 0;
+    UPDATE_PROFILE_STATS(frame, SETXATTR);
+    STACK_UNWIND_STRICT(setxattr, frame, op_ret, op_errno, xdata);
+    return 0;
 }
 
-
 int
-io_stats_getxattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                       int32_t op_ret, int32_t op_errno, dict_t *dict, dict_t *xdata)
+io_stats_getxattr_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                      int32_t op_ret, int32_t op_errno, dict_t *dict,
+                      dict_t *xdata)
 {
-        UPDATE_PROFILE_STATS (frame, GETXATTR);
-        STACK_UNWIND_STRICT (getxattr, frame, op_ret, op_errno, dict, xdata);
-        return 0;
+    UPDATE_PROFILE_STATS(frame, GETXATTR);
+    STACK_UNWIND_STRICT(getxattr, frame, op_ret, op_errno, dict, xdata);
+    return 0;
 }
 
-
 int
-io_stats_removexattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                          int32_t op_ret, int32_t op_errno, dict_t *xdata)
+io_stats_removexattr_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                         int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
-        UPDATE_PROFILE_STATS (frame, REMOVEXATTR);
-        STACK_UNWIND_STRICT (removexattr, frame, op_ret, op_errno, xdata);
-        return 0;
+    UPDATE_PROFILE_STATS(frame, REMOVEXATTR);
+    STACK_UNWIND_STRICT(removexattr, frame, op_ret, op_errno, xdata);
+    return 0;
 }
 
 int
-io_stats_fsetxattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                        int32_t op_ret, int32_t op_errno, dict_t *xdata)
+io_stats_fsetxattr_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                       int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
-        UPDATE_PROFILE_STATS (frame, FSETXATTR);
-        STACK_UNWIND_STRICT (fsetxattr, frame, op_ret, op_errno, xdata);
-        return 0;
+    UPDATE_PROFILE_STATS(frame, FSETXATTR);
+    STACK_UNWIND_STRICT(fsetxattr, frame, op_ret, op_errno, xdata);
+    return 0;
 }
 
-
 int
-io_stats_fgetxattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                        int32_t op_ret, int32_t op_errno, dict_t *dict, dict_t *xdata)
+io_stats_fgetxattr_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                       int32_t op_ret, int32_t op_errno, dict_t *dict,
+                       dict_t *xdata)
 {
-        UPDATE_PROFILE_STATS (frame, FGETXATTR);
-        STACK_UNWIND_STRICT (fgetxattr, frame, op_ret, op_errno, dict, xdata);
-        return 0;
+    UPDATE_PROFILE_STATS(frame, FGETXATTR);
+    STACK_UNWIND_STRICT(fgetxattr, frame, op_ret, op_errno, dict, xdata);
+    return 0;
 }
 
-
 int
-io_stats_fremovexattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                           int32_t op_ret, int32_t op_errno, dict_t *xdata)
+io_stats_fremovexattr_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                          int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
-        UPDATE_PROFILE_STATS (frame, FREMOVEXATTR);
-        STACK_UNWIND_STRICT (fremovexattr, frame, op_ret, op_errno, xdata);
-        return 0;
+    UPDATE_PROFILE_STATS(frame, FREMOVEXATTR);
+    STACK_UNWIND_STRICT(fremovexattr, frame, op_ret, op_errno, xdata);
+    return 0;
 }
 
-
 int
-io_stats_fsyncdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                       int32_t op_ret, int32_t op_errno, dict_t *xdata)
+io_stats_fsyncdir_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                      int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
-        UPDATE_PROFILE_STATS (frame, FSYNCDIR);
-        STACK_UNWIND_STRICT (fsyncdir, frame, op_ret, op_errno, xdata);
-        return 0;
+    UPDATE_PROFILE_STATS(frame, FSYNCDIR);
+    STACK_UNWIND_STRICT(fsyncdir, frame, op_ret, op_errno, xdata);
+    return 0;
 }
 
-
 int
-io_stats_access_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                     int32_t op_ret, int32_t op_errno, dict_t *xdata)
+io_stats_access_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                    int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
-        UPDATE_PROFILE_STATS (frame, ACCESS);
-        STACK_UNWIND_STRICT (access, frame, op_ret, op_errno, xdata);
-        return 0;
+    UPDATE_PROFILE_STATS(frame, ACCESS);
+    STACK_UNWIND_STRICT(access, frame, op_ret, op_errno, xdata);
+    return 0;
 }
 
-
 int
-io_stats_ftruncate_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                        int32_t op_ret, int32_t op_errno,
-                        struct iatt *prebuf, struct iatt *postbuf, dict_t *xdata)
+io_stats_ftruncate_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                       int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
+                       struct iatt *postbuf, dict_t *xdata)
 {
-        UPDATE_PROFILE_STATS (frame, FTRUNCATE);
-        STACK_UNWIND_STRICT (ftruncate, frame, op_ret, op_errno,
-                             prebuf, postbuf, xdata);
-        return 0;
+    UPDATE_PROFILE_STATS(frame, FTRUNCATE);
+    STACK_UNWIND_STRICT(ftruncate, frame, op_ret, op_errno, prebuf, postbuf,
+                        xdata);
+    return 0;
 }
 
-
 int
-io_stats_fstat_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                    int32_t op_ret, int32_t op_errno, struct iatt *buf, dict_t *xdata)
+io_stats_fstat_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                   int32_t op_ret, int32_t op_errno, struct iatt *buf,
+                   dict_t *xdata)
 {
-        UPDATE_PROFILE_STATS (frame, FSTAT);
-        STACK_UNWIND_STRICT (fstat, frame, op_ret, op_errno, buf, xdata);
-        return 0;
+    UPDATE_PROFILE_STATS(frame, FSTAT);
+    STACK_UNWIND_STRICT(fstat, frame, op_ret, op_errno, buf, xdata);
+    return 0;
 }
 
-
 int
 io_stats_fallocate_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
-		       int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
-		       struct iatt *postbuf, dict_t *xdata)
+                       int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
+                       struct iatt *postbuf, dict_t *xdata)
 {
-	UPDATE_PROFILE_STATS(frame, FALLOCATE);
-	STACK_UNWIND_STRICT(fallocate, frame, op_ret, op_errno, prebuf, postbuf,
-			    xdata);
-	return 0;
+    UPDATE_PROFILE_STATS(frame, FALLOCATE);
+    STACK_UNWIND_STRICT(fallocate, frame, op_ret, op_errno, prebuf, postbuf,
+                        xdata);
+    return 0;
 }
 
-
 int
 io_stats_discard_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
-		     int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
-		     struct iatt *postbuf, dict_t *xdata)
+                     int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
+                     struct iatt *postbuf, dict_t *xdata)
 {
-	UPDATE_PROFILE_STATS(frame, DISCARD);
-	STACK_UNWIND_STRICT(discard, frame, op_ret, op_errno, prebuf, postbuf,
-			    xdata);
-	return 0;
+    UPDATE_PROFILE_STATS(frame, DISCARD);
+    STACK_UNWIND_STRICT(discard, frame, op_ret, op_errno, prebuf, postbuf,
+                        xdata);
+    return 0;
 }
 
 int
 io_stats_zerofill_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
-                     int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
-                     struct iatt *postbuf, dict_t *xdata)
+                      int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
+                      struct iatt *postbuf, dict_t *xdata)
 {
-        UPDATE_PROFILE_STATS(frame, ZEROFILL);
-        STACK_UNWIND_STRICT(zerofill, frame, op_ret, op_errno, prebuf, postbuf,
-                            xdata);
-        return 0;
+    UPDATE_PROFILE_STATS(frame, ZEROFILL);
+    STACK_UNWIND_STRICT(zerofill, frame, op_ret, op_errno, prebuf, postbuf,
+                        xdata);
+    return 0;
+}
+
+int32_t
+io_stats_ipc_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                 int32_t op_ret, int32_t op_errno, dict_t *xdata)
+{
+    UPDATE_PROFILE_STATS(frame, IPC);
+    STACK_UNWIND_STRICT(ipc, frame, op_ret, op_errno, xdata);
+    return 0;
 }
 
 int
-io_stats_lk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                 int32_t op_ret, int32_t op_errno, struct gf_flock *lock, dict_t *xdata)
+io_stats_lk_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                int32_t op_ret, int32_t op_errno, struct gf_flock *lock,
+                dict_t *xdata)
 {
-        UPDATE_PROFILE_STATS (frame, LK);
-        STACK_UNWIND_STRICT (lk, frame, op_ret, op_errno, lock, xdata);
-        return 0;
+    UPDATE_PROFILE_STATS(frame, LK);
+    STACK_UNWIND_STRICT(lk, frame, op_ret, op_errno, lock, xdata);
+    return 0;
 }
 
+int
+io_stats_entrylk_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                     int32_t op_ret, int32_t op_errno, dict_t *xdata)
+{
+    UPDATE_PROFILE_STATS(frame, ENTRYLK);
+    STACK_UNWIND_STRICT(entrylk, frame, op_ret, op_errno, xdata);
+    return 0;
+}
 
 int
-io_stats_entrylk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+io_stats_fentrylk_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
                       int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
-        UPDATE_PROFILE_STATS (frame, ENTRYLK);
-        STACK_UNWIND_STRICT (entrylk, frame, op_ret, op_errno, xdata);
-        return 0;
+    UPDATE_PROFILE_STATS(frame, FENTRYLK);
+    STACK_UNWIND_STRICT(fentrylk, frame, op_ret, op_errno, xdata);
+    return 0;
 }
 
-
 int
-io_stats_xattrop_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                      int32_t op_ret, int32_t op_errno, dict_t *dict, dict_t *xdata)
+io_stats_rchecksum_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                       int32_t op_ret, int32_t op_errno, uint32_t weak_checksum,
+                       uint8_t *strong_checksum, dict_t *xdata)
 {
-        UPDATE_PROFILE_STATS (frame, XATTROP);
-        STACK_UNWIND_STRICT (xattrop, frame, op_ret, op_errno, dict, xdata);
-        return 0;
+    UPDATE_PROFILE_STATS(frame, RCHECKSUM);
+    STACK_UNWIND_STRICT(rchecksum, frame, op_ret, op_errno, weak_checksum,
+                        strong_checksum, xdata);
+    return 0;
 }
 
+int
+io_stats_seek_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                  int32_t op_ret, int32_t op_errno, off_t offset, dict_t *xdata)
+{
+    UPDATE_PROFILE_STATS(frame, SEEK);
+    STACK_UNWIND_STRICT(seek, frame, op_ret, op_errno, offset, xdata);
+    return 0;
+}
 
 int
-io_stats_fxattrop_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                       int32_t op_ret, int32_t op_errno, dict_t *dict, dict_t *xdata)
+io_stats_lease_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                   int32_t op_ret, int32_t op_errno, struct gf_lease *lease,
+                   dict_t *xdata)
 {
-        UPDATE_PROFILE_STATS (frame, FXATTROP);
-        STACK_UNWIND_STRICT (fxattrop, frame, op_ret, op_errno, dict, xdata);
-        return 0;
+    UPDATE_PROFILE_STATS(frame, LEASE);
+    STACK_UNWIND_STRICT(lease, frame, op_ret, op_errno, lease, xdata);
+    return 0;
 }
 
+int
+io_stats_getactivelk_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                         int32_t op_ret, int32_t op_errno,
+                         lock_migration_info_t *locklist, dict_t *xdata)
+{
+    UPDATE_PROFILE_STATS(frame, GETACTIVELK);
+    STACK_UNWIND_STRICT(getactivelk, frame, op_ret, op_errno, locklist, xdata);
+    return 0;
+}
 
 int
-io_stats_inodelk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                      int32_t op_ret, int32_t op_errno, dict_t *xdata)
+io_stats_setactivelk_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                         int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
-        UPDATE_PROFILE_STATS (frame, INODELK);
-        STACK_UNWIND_STRICT (inodelk, frame, op_ret, op_errno, xdata);
-        return 0;
+    UPDATE_PROFILE_STATS(frame, SETACTIVELK);
+    STACK_UNWIND_STRICT(setactivelk, frame, op_ret, op_errno, xdata);
+    return 0;
 }
 
 int
-io_stats_entrylk (call_frame_t *frame, xlator_t *this,
-                  const char *volume, loc_t *loc, const char *basename,
-                  entrylk_cmd cmd, entrylk_type type, dict_t *xdata)
+io_stats_compound_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                      int32_t op_ret, int32_t op_errno, void *data,
+                      dict_t *xdata)
 {
-        START_FOP_LATENCY (frame);
+    UPDATE_PROFILE_STATS(frame, COMPOUND);
+    STACK_UNWIND_STRICT(compound, frame, op_ret, op_errno, data, xdata);
+    return 0;
+}
 
-        STACK_WIND (frame, io_stats_entrylk_cbk,
-                    FIRST_CHILD (this),
-                    FIRST_CHILD (this)->fops->entrylk,
-                    volume, loc, basename, cmd, type, xdata);
-        return 0;
+int
+io_stats_xattrop_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                     int32_t op_ret, int32_t op_errno, dict_t *dict,
+                     dict_t *xdata)
+{
+    UPDATE_PROFILE_STATS(frame, XATTROP);
+    STACK_UNWIND_STRICT(xattrop, frame, op_ret, op_errno, dict, xdata);
+    return 0;
 }
 
+int
+io_stats_fxattrop_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                      int32_t op_ret, int32_t op_errno, dict_t *dict,
+                      dict_t *xdata)
+{
+    UPDATE_PROFILE_STATS(frame, FXATTROP);
+    STACK_UNWIND_STRICT(fxattrop, frame, op_ret, op_errno, dict, xdata);
+    return 0;
+}
 
 int
-io_stats_inodelk (call_frame_t *frame, xlator_t *this,
-                  const char *volume, loc_t *loc, int32_t cmd, struct gf_flock *flock, dict_t *xdata)
+io_stats_inodelk_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                     int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
+    UPDATE_PROFILE_STATS(frame, INODELK);
+    STACK_UNWIND_STRICT(inodelk, frame, op_ret, op_errno, xdata);
+    return 0;
+}
 
-        START_FOP_LATENCY (frame);
+int
+io_stats_entrylk(call_frame_t *frame, xlator_t *this, const char *volume,
+                 loc_t *loc, const char *basename, entrylk_cmd cmd,
+                 entrylk_type type, dict_t *xdata)
+{
+    START_FOP_LATENCY(frame);
 
-        STACK_WIND (frame, io_stats_inodelk_cbk,
-                    FIRST_CHILD (this),
-                    FIRST_CHILD (this)->fops->inodelk,
-                    volume, loc, cmd, flock, xdata);
-        return 0;
+    STACK_WIND(frame, io_stats_entrylk_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->entrylk, volume, loc, basename, cmd,
+               type, xdata);
+    return 0;
 }
 
-
 int
-io_stats_finodelk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                       int32_t op_ret, int32_t op_errno, dict_t *xdata)
+io_stats_fentrylk(call_frame_t *frame, xlator_t *this, const char *volume,
+                  fd_t *fd, const char *basename, entrylk_cmd cmd,
+                  entrylk_type type, dict_t *xdata)
 {
+    START_FOP_LATENCY(frame);
 
-        UPDATE_PROFILE_STATS (frame, FINODELK);
-        STACK_UNWIND_STRICT (finodelk, frame, op_ret, op_errno, xdata);
-        return 0;
+    STACK_WIND(frame, io_stats_fentrylk_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->fentrylk, volume, fd, basename, cmd,
+               type, xdata);
+    return 0;
 }
 
-
 int
-io_stats_finodelk (call_frame_t *frame, xlator_t *this, const char *volume,
-                   fd_t *fd, int32_t cmd, struct gf_flock *flock, dict_t *xdata)
+io_stats_inodelk(call_frame_t *frame, xlator_t *this, const char *volume,
+                 loc_t *loc, int32_t cmd, struct gf_flock *flock, dict_t *xdata)
 {
-        START_FOP_LATENCY (frame);
+    START_FOP_LATENCY(frame);
 
-        STACK_WIND (frame, io_stats_finodelk_cbk,
-                    FIRST_CHILD (this),
-                    FIRST_CHILD (this)->fops->finodelk,
-                    volume, fd, cmd, flock, xdata);
-        return 0;
+    STACK_WIND(frame, io_stats_inodelk_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->inodelk, volume, loc, cmd, flock,
+               xdata);
+    return 0;
 }
 
+int
+io_stats_finodelk_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                      int32_t op_ret, int32_t op_errno, dict_t *xdata)
+{
+    UPDATE_PROFILE_STATS(frame, FINODELK);
+    STACK_UNWIND_STRICT(finodelk, frame, op_ret, op_errno, xdata);
+    return 0;
+}
 
 int
-io_stats_xattrop (call_frame_t *frame, xlator_t *this, loc_t *loc,
-                  gf_xattrop_flags_t flags, dict_t *dict, dict_t *xdata)
+io_stats_finodelk(call_frame_t *frame, xlator_t *this, const char *volume,
+                  fd_t *fd, int32_t cmd, struct gf_flock *flock, dict_t *xdata)
 {
-        START_FOP_LATENCY (frame);
+    START_FOP_LATENCY(frame);
 
-        STACK_WIND (frame, io_stats_xattrop_cbk,
-                    FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->xattrop,
-                    loc, flags, dict, xdata);
-        return 0;
+    STACK_WIND(frame, io_stats_finodelk_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->finodelk, volume, fd, cmd, flock,
+               xdata);
+    return 0;
 }
 
-
 int
-io_stats_fxattrop (call_frame_t *frame, xlator_t *this, fd_t *fd,
-                   gf_xattrop_flags_t flags, dict_t *dict, dict_t *xdata)
+io_stats_xattrop(call_frame_t *frame, xlator_t *this, loc_t *loc,
+                 gf_xattrop_flags_t flags, dict_t *dict, dict_t *xdata)
 {
-        START_FOP_LATENCY (frame);
+    START_FOP_LATENCY(frame);
 
-        STACK_WIND (frame, io_stats_fxattrop_cbk,
-                    FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->fxattrop,
-                    fd, flags, dict, xdata);
-        return 0;
+    STACK_WIND(frame, io_stats_xattrop_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->xattrop, loc, flags, dict, xdata);
+    return 0;
 }
 
-
 int
-io_stats_lookup (call_frame_t *frame, xlator_t *this,
-                 loc_t *loc, dict_t *xdata)
+io_stats_fxattrop(call_frame_t *frame, xlator_t *this, fd_t *fd,
+                  gf_xattrop_flags_t flags, dict_t *dict, dict_t *xdata)
 {
-        START_FOP_LATENCY (frame);
+    START_FOP_LATENCY(frame);
 
-        STACK_WIND (frame, io_stats_lookup_cbk,
-                    FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->lookup,
-                    loc, xdata);
-        return 0;
+    STACK_WIND(frame, io_stats_fxattrop_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->fxattrop, fd, flags, dict, xdata);
+    return 0;
 }
 
-
 int
-io_stats_stat (call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xdata)
+io_stats_lookup(call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xdata)
 {
-        START_FOP_LATENCY (frame);
+    START_FOP_LATENCY(frame);
 
-        STACK_WIND (frame, io_stats_stat_cbk,
-                    FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->stat,
-                    loc, xdata);
-        return 0;
+    STACK_WIND(frame, io_stats_lookup_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->lookup, loc, xdata);
+    return 0;
 }
 
-
 int
-io_stats_readlink (call_frame_t *frame, xlator_t *this,
-                   loc_t *loc, size_t size, dict_t *xdata)
+io_stats_stat(call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xdata)
 {
-        START_FOP_LATENCY (frame);
+    START_FOP_LATENCY(frame);
 
-        STACK_WIND (frame, io_stats_readlink_cbk,
-                    FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->readlink,
-                    loc, size, xdata);
-        return 0;
+    STACK_WIND(frame, io_stats_stat_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->stat, loc, xdata);
+    return 0;
 }
 
-
 int
-io_stats_mknod (call_frame_t *frame, xlator_t *this, loc_t *loc,
-                mode_t mode, dev_t dev, mode_t umask, dict_t *xdata)
+io_stats_readlink(call_frame_t *frame, xlator_t *this, loc_t *loc, size_t size,
+                  dict_t *xdata)
 {
-        START_FOP_LATENCY (frame);
+    START_FOP_LATENCY(frame);
 
-        STACK_WIND (frame, io_stats_mknod_cbk,
-                    FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->mknod,
-                    loc, mode, dev, umask, xdata);
-        return 0;
+    STACK_WIND(frame, io_stats_readlink_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->readlink, loc, size, xdata);
+    return 0;
 }
 
+int
+io_stats_mknod(call_frame_t *frame, xlator_t *this, loc_t *loc, mode_t mode,
+               dev_t dev, mode_t umask, dict_t *xdata)
+{
+    START_FOP_LATENCY(frame);
+
+    STACK_WIND(frame, io_stats_mknod_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->mknod, loc, mode, dev, umask, xdata);
+    return 0;
+}
 
 int
-io_stats_mkdir (call_frame_t *frame, xlator_t *this,
-                loc_t *loc, mode_t mode, mode_t umask, dict_t *xdata)
+io_stats_mkdir(call_frame_t *frame, xlator_t *this, loc_t *loc, mode_t mode,
+               mode_t umask, dict_t *xdata)
 {
-        if (loc->path)
-                frame->local = gf_strdup (loc->path);
+    if (loc->path)
+        frame->local = gf_strdup(loc->path);
 
-        START_FOP_LATENCY (frame);
+    START_FOP_LATENCY(frame);
 
-        STACK_WIND (frame, io_stats_mkdir_cbk,
-                    FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->mkdir,
-                    loc, mode, umask, xdata);
-        return 0;
+    STACK_WIND(frame, io_stats_mkdir_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->mkdir, loc, mode, umask, xdata);
+    return 0;
 }
 
-
 int
-io_stats_unlink (call_frame_t *frame, xlator_t *this,
-                 loc_t *loc, int xflag, dict_t *xdata)
+io_stats_unlink(call_frame_t *frame, xlator_t *this, loc_t *loc, int xflag,
+                dict_t *xdata)
 {
-        START_FOP_LATENCY (frame);
+    START_FOP_LATENCY(frame);
 
-        STACK_WIND (frame, io_stats_unlink_cbk,
-                    FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->unlink,
-                    loc, xflag, xdata);
-        return 0;
+    STACK_WIND(frame, io_stats_unlink_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->unlink, loc, xflag, xdata);
+    return 0;
 }
 
-
 int
-io_stats_rmdir (call_frame_t *frame, xlator_t *this,
-                loc_t *loc, int flags, dict_t *xdata)
+io_stats_rmdir(call_frame_t *frame, xlator_t *this, loc_t *loc, int flags,
+               dict_t *xdata)
 {
-        START_FOP_LATENCY (frame);
+    START_FOP_LATENCY(frame);
 
-        STACK_WIND (frame, io_stats_rmdir_cbk,
-                    FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->rmdir,
-                    loc, flags, xdata);
-        return 0;
+    STACK_WIND(frame, io_stats_rmdir_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->rmdir, loc, flags, xdata);
+    return 0;
 }
 
-
 int
-io_stats_symlink (call_frame_t *frame, xlator_t *this, const char *linkpath,
-                  loc_t *loc, mode_t umask, dict_t *xdata)
+io_stats_symlink(call_frame_t *frame, xlator_t *this, const char *linkpath,
+                 loc_t *loc, mode_t umask, dict_t *xdata)
 {
-        START_FOP_LATENCY (frame);
+    START_FOP_LATENCY(frame);
 
-        STACK_WIND (frame, io_stats_symlink_cbk,
-                    FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->symlink,
-                    linkpath, loc, umask, xdata);
-        return 0;
+    STACK_WIND(frame, io_stats_symlink_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->symlink, linkpath, loc, umask, xdata);
+    return 0;
 }
 
-
 int
-io_stats_rename (call_frame_t *frame, xlator_t *this,
-                 loc_t *oldloc, loc_t *newloc, dict_t *xdata)
+io_stats_rename(call_frame_t *frame, xlator_t *this, loc_t *oldloc,
+                loc_t *newloc, dict_t *xdata)
 {
-        START_FOP_LATENCY (frame);
+    START_FOP_LATENCY(frame);
 
-        STACK_WIND (frame, io_stats_rename_cbk,
-                    FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->rename,
-                    oldloc, newloc, xdata);
-        return 0;
+    STACK_WIND(frame, io_stats_rename_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->rename, oldloc, newloc, xdata);
+    return 0;
 }
 
-
 int
-io_stats_link (call_frame_t *frame, xlator_t *this,
-               loc_t *oldloc, loc_t *newloc, dict_t *xdata)
+io_stats_link(call_frame_t *frame, xlator_t *this, loc_t *oldloc, loc_t *newloc,
+              dict_t *xdata)
 {
-        START_FOP_LATENCY (frame);
+    START_FOP_LATENCY(frame);
 
-        STACK_WIND (frame, io_stats_link_cbk,
-                    FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->link,
-                    oldloc, newloc, xdata);
-        return 0;
+    STACK_WIND(frame, io_stats_link_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->link, oldloc, newloc, xdata);
+    return 0;
 }
 
-
 int
-io_stats_setattr (call_frame_t *frame, xlator_t *this,
-                  loc_t *loc, struct iatt *stbuf, int32_t valid, dict_t *xdata)
+io_stats_setattr(call_frame_t *frame, xlator_t *this, loc_t *loc,
+                 struct iatt *stbuf, int32_t valid, dict_t *xdata)
 {
-        START_FOP_LATENCY (frame);
+    START_FOP_LATENCY(frame);
 
-        STACK_WIND (frame, io_stats_setattr_cbk,
-                    FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->setattr,
-                    loc, stbuf, valid, xdata);
-        return 0;
+    STACK_WIND(frame, io_stats_setattr_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->setattr, loc, stbuf, valid, xdata);
+    return 0;
 }
 
-
 int
-io_stats_truncate (call_frame_t *frame, xlator_t *this,
-                   loc_t *loc, off_t offset, dict_t *xdata)
+io_stats_truncate(call_frame_t *frame, xlator_t *this, loc_t *loc, off_t offset,
+                  dict_t *xdata)
 {
-        START_FOP_LATENCY (frame);
+    START_FOP_LATENCY(frame);
 
-        STACK_WIND (frame, io_stats_truncate_cbk,
-                    FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->truncate,
-                    loc, offset, xdata);
-        return 0;
+    STACK_WIND(frame, io_stats_truncate_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->truncate, loc, offset, xdata);
+    return 0;
 }
 
-
 int
-io_stats_open (call_frame_t *frame, xlator_t *this, loc_t *loc,
-               int32_t flags, fd_t *fd, dict_t *xdata)
+io_stats_open(call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t flags,
+              fd_t *fd, dict_t *xdata)
 {
-        if (loc->path)
-                frame->local = gf_strdup (loc->path);
+    if (loc->path)
+        frame->local = gf_strdup(loc->path);
 
-        START_FOP_LATENCY (frame);
+    START_FOP_LATENCY(frame);
 
-        STACK_WIND (frame, io_stats_open_cbk,
-                    FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->open,
-                    loc, flags, fd, xdata);
-        return 0;
+    STACK_WIND(frame, io_stats_open_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->open, loc, flags, fd, xdata);
+    return 0;
 }
 
-
 int
-io_stats_create (call_frame_t *frame, xlator_t *this,
-                 loc_t *loc, int32_t flags, mode_t mode,
-                 mode_t umask, fd_t *fd, dict_t *xdata)
+io_stats_create(call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t flags,
+                mode_t mode, mode_t umask, fd_t *fd, dict_t *xdata)
 {
-        if (loc->path)
-                frame->local = gf_strdup (loc->path);
+    if (loc->path)
+        frame->local = gf_strdup(loc->path);
 
-        START_FOP_LATENCY (frame);
+    START_FOP_LATENCY(frame);
 
-        STACK_WIND (frame, io_stats_create_cbk,
-                    FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->create,
-                    loc, flags, mode, umask, fd, xdata);
-        return 0;
+    STACK_WIND(frame, io_stats_create_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->create, loc, flags, mode, umask, fd,
+               xdata);
+    return 0;
 }
 
-
 int
-io_stats_readv (call_frame_t *frame, xlator_t *this,
-                fd_t *fd, size_t size, off_t offset, uint32_t flags, dict_t *xdata)
+io_stats_readv(call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
+               off_t offset, uint32_t flags, dict_t *xdata)
 {
-        frame->local = fd;
+    frame->local = fd;
 
-        START_FOP_LATENCY (frame);
+    START_FOP_LATENCY(frame);
 
-        STACK_WIND (frame, io_stats_readv_cbk,
-                    FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->readv,
-                    fd, size, offset, flags, xdata);
-        return 0;
+    STACK_WIND(frame, io_stats_readv_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->readv, fd, size, offset, flags, xdata);
+    return 0;
 }
 
-
 int
-io_stats_writev (call_frame_t *frame, xlator_t *this,
-                 fd_t *fd, struct iovec *vector,
-                 int32_t count, off_t offset,
-                 uint32_t flags, struct iobref *iobref, dict_t *xdata)
+io_stats_writev(call_frame_t *frame, xlator_t *this, fd_t *fd,
+                struct iovec *vector, int32_t count, off_t offset,
+                uint32_t flags, struct iobref *iobref, dict_t *xdata)
 {
-        int                 len = 0;
+    int len = 0;
 
-        if (fd->inode)
-                frame->local = fd->inode;
-        len = iov_length (vector, count);
+    if (fd->inode)
+        frame->local = fd->inode;
+    len = iov_length(vector, count);
 
-        BUMP_WRITE (fd, len);
-        START_FOP_LATENCY (frame);
-
-        STACK_WIND (frame, io_stats_writev_cbk,
-                    FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->writev,
-                    fd, vector, count, offset, flags, iobref, xdata);
-        return 0;
+    ios_bump_write(this, fd, len);
+    START_FOP_LATENCY(frame);
 
+    STACK_WIND(frame, io_stats_writev_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->writev, fd, vector, count, offset,
+               flags, iobref, xdata);
+    return 0;
 }
 
-
 int
-io_stats_statfs (call_frame_t *frame, xlator_t *this,
-                 loc_t *loc, dict_t *xdata)
+io_stats_copy_file_range(call_frame_t *frame, xlator_t *this, fd_t *fd_in,
+                         off_t off_in, fd_t *fd_out, off_t off_out, size_t len,
+                         uint32_t flags, dict_t *xdata)
 {
-        START_FOP_LATENCY (frame);
+    START_FOP_LATENCY(frame);
 
-        STACK_WIND (frame, io_stats_statfs_cbk,
-                    FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->statfs,
-                    loc, xdata);
-        return 0;
+    STACK_WIND(frame, io_stats_copy_file_range_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->copy_file_range, fd_in, off_in, fd_out,
+               off_out, len, flags, xdata);
+    return 0;
 }
 
-
 int
-io_stats_flush (call_frame_t *frame, xlator_t *this,
-                fd_t *fd, dict_t *xdata)
+io_stats_statfs(call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xdata)
 {
-        START_FOP_LATENCY (frame);
+    START_FOP_LATENCY(frame);
 
-        STACK_WIND (frame, io_stats_flush_cbk,
-                    FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->flush,
-                    fd, xdata);
-        return 0;
+    STACK_WIND(frame, io_stats_statfs_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->statfs, loc, xdata);
+    return 0;
 }
 
-
 int
-io_stats_fsync (call_frame_t *frame, xlator_t *this,
-                fd_t *fd, int32_t flags, dict_t *xdata)
+io_stats_flush(call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *xdata)
 {
-        START_FOP_LATENCY (frame);
+    START_FOP_LATENCY(frame);
 
-        STACK_WIND (frame, io_stats_fsync_cbk,
-                    FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->fsync,
-                    fd, flags, xdata);
-        return 0;
+    STACK_WIND(frame, io_stats_flush_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->flush, fd, xdata);
+    return 0;
 }
 
-
 int
-conditional_dump (dict_t *dict, char *key, data_t *value, void *data)
+io_stats_fsync(call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t flags,
+               dict_t *xdata)
 {
-        struct {
-                xlator_t       *this;
-                inode_t        *inode;
-                const char     *path;
-        } *stub;
-        xlator_t             *this = NULL;
-        char                 *filename = NULL;
-        FILE                 *logfp = NULL;
-        struct ios_dump_args args = {0};
+    START_FOP_LATENCY(frame);
 
-        stub  = data;
-        this  = stub->this;
+    STACK_WIND(frame, io_stats_fsync_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->fsync, fd, flags, xdata);
+    return 0;
+}
 
-        filename = alloca (value->len + 1);
-        memset (filename, 0, value->len + 1);
-        memcpy (filename, data_to_str (value), value->len);
+int
+conditional_dump(dict_t *dict, char *key, data_t *value, void *data)
+{
+    struct {
+        xlator_t *this;
+        inode_t *inode;
+        const char *path;
+    } * stub;
+    xlator_t *this = NULL;
+    char *filename = NULL;
+    FILE *logfp = NULL;
+    struct ios_dump_args args = {0};
+    int pid, namelen, dirlen;
+    char dump_key[100];
+    char *slash_ptr = NULL;
+    char *path_in_value = NULL;
+    char *identifier = NULL;
+    struct ios_conf *conf = NULL;
+
+    stub = data;
+    this = stub->this;
+    conf = this->private;
+
+    /* Don't do this on 'brick-side', only do this on client side */
+    /* Addresses CVE-2018-14659 */
+    if (this->ctx->process_mode != GF_CLIENT_PROCESS) {
+        gf_log(this->name, GF_LOG_DEBUG,
+               "taking io-stats dump using setxattr not permitted on brick."
+               " Use 'gluster profile' instead");
+        return -1;
+    }
+
+    /* Create a file name that is appended with the io-stats instance
+    name as well. This helps when there is more than a single io-stats
+    instance in the graph, or the client and server processes are running
+    on the same node */
+    /* For the sanity of where the file should be located, we should make
+       sure file is written only inside RUNDIR (ie, /var/run/gluster) */
+    /* TODO: provide an option to dump it to different directory of
+       choice, based on options */
+    /* name format: /var/run/gluster/<passed in path/filename>.<xlator name
+     * slashes to -> */
+
+    path_in_value = alloca0(value->len + 1);
+
+    /* We need a memcpy here because of the way dict_unserialize works */
+
+    memcpy(path_in_value, data_to_str(value), value->len);
+    path_in_value[value->len] = '\0';
+
+    if (strstr(path_in_value, "../")) {
+        gf_log(this->name, GF_LOG_ERROR, "%s: no \"../\" allowed in path",
+               path_in_value);
+        return -1;
+    }
+
+    if (path_in_value[0] == '/') {
+        path_in_value = path_in_value + 1;
+    }
+
+    dirlen = strlen(IOS_STATS_DUMP_DIR);
+    if (conf->unique_id) {
+        /* this->name will be the same for all bricks of the volume */
+        identifier = conf->unique_id;
+    } else {
+        identifier = this->name;
+    }
+
+    namelen = (dirlen + value->len + strlen(identifier) + 3);
+    /* +3 for '/', '.' and '\0' added in snprintf below*/
+
+    filename = alloca0(namelen);
+    snprintf(filename, namelen, "%s/%s.%s", IOS_STATS_DUMP_DIR, path_in_value,
+             identifier);
+
+    /* convert any slashes to '-' so that fopen works correctly */
+    slash_ptr = strchr(filename + dirlen + 1, '/');
+    while (slash_ptr) {
+        *slash_ptr = '-';
+        slash_ptr = strchr(slash_ptr, '/');
+    }
+
+    pid = getpid();
+
+    if (!strncmp(filename, "", 1)) {
+        gf_log(this->name, GF_LOG_ERROR, "No filename given");
+        return -1;
+    }
+    logfp = fopen(filename, "w+");
+    if (!logfp) {
+        gf_log(this->name, GF_LOG_ERROR,
+               "failed to open %s "
+               "for writing",
+               filename);
+        return -1;
+    }
+    sprintf(dump_key, "*io*stat*%d_json_dump", pid);
+    if (fnmatch(dump_key, key, 0) == 0) {
+        (void)ios_dump_args_init(&args, IOS_DUMP_TYPE_JSON_FILE, logfp);
+    } else {
+        (void)ios_dump_args_init(&args, IOS_DUMP_TYPE_FILE, logfp);
+    }
+    io_stats_dump(this, &args, GF_IOS_INFO_ALL, _gf_false);
+    fclose(logfp);
+    return 0;
+}
 
-        if (fnmatch ("*io*stat*dump", key, 0) == 0) {
+int
+_ios_destroy_dump_thread(struct ios_conf *conf)
+{
+    conf->dump_thread_should_die = _gf_true;
+    if (conf->dump_thread_running) {
+        (void)pthread_cancel(conf->dump_thread);
+        (void)pthread_join(conf->dump_thread, NULL);
+    }
+    return 0;
+}
 
-                if (!strncmp (filename, "", 1)) {
-                        gf_log (this->name, GF_LOG_ERROR, "No filename given");
-                        return -1;
-                }
-                logfp = fopen (filename, "w+");
-                if (!logfp) {
-                        gf_log (this->name, GF_LOG_ERROR, "failed to open %s "
-                                "for writing", filename);
-                        return -1;
-                    }
-                (void) ios_dump_args_init (&args, IOS_DUMP_TYPE_FILE,
-                                           logfp);
-                io_stats_dump (this, &args, GF_CLI_INFO_ALL, _gf_false);
-                fclose (logfp);
+void *
+_ios_dump_thread(xlator_t *this)
+{
+    struct ios_conf *conf = NULL;
+    FILE *stats_logfp = NULL;
+    FILE *samples_logfp = NULL;
+    struct ios_dump_args args = {0};
+    int i;
+    int stats_bytes_written = 0;
+    int samples_bytes_written = 0;
+    char stats_filename[PATH_MAX];
+    char samples_filename[PATH_MAX];
+    char *xlator_name;
+    char *instance_name;
+    gf_boolean_t log_stats_fopen_failure = _gf_true;
+    gf_boolean_t log_samples_fopen_failure = _gf_true;
+    int old_cancel_type;
+
+    conf = this->private;
+    gf_log(this->name, GF_LOG_INFO,
+           "IO stats dump thread started, "
+           "polling IO stats every %d seconds",
+           conf->ios_dump_interval);
+    xlator_name = strdupa(conf->unique_id);
+    for (i = 0; i < strlen(xlator_name); i++) {
+        if (xlator_name[i] == '/')
+            xlator_name[i] = '_';
+    }
+    instance_name = this->instance_name;
+    if (this->name && strcmp(this->name, "glustershd") == 0) {
+        xlator_name = "shd";
+    } else if (this->prev && strcmp(this->prev->name, "nfs-server") == 0) {
+        xlator_name = "nfsd";
+        instance_name = this->prev->instance_name;
+    }
+    if (sys_mkdir(_IOS_DUMP_DIR, S_IRWXU | S_IRWXO | S_IRWXG) == (-1)) {
+        if (errno != EEXIST) {
+            gf_log(this->name, GF_LOG_ERROR,
+                   "could not create stats-dump directory %s", _IOS_DUMP_DIR);
+            goto out;
         }
-        return 0;
+    }
+    if (sys_mkdir(_IOS_SAMP_DIR, S_IRWXU | S_IRWXO | S_IRWXG) == (-1)) {
+        if (errno != EEXIST) {
+            gf_log(this->name, GF_LOG_ERROR,
+                   "could not create stats-sample directory %s", _IOS_SAMP_DIR);
+            goto out;
+        }
+    }
+    if (instance_name) {
+        stats_bytes_written = snprintf(stats_filename, PATH_MAX,
+                                       "%s/%s_%s_%s.dump", _IOS_DUMP_DIR,
+                                       __progname, xlator_name, instance_name);
+        samples_bytes_written = snprintf(
+            samples_filename, PATH_MAX, "%s/%s_%s_%s.samp", _IOS_SAMP_DIR,
+            __progname, xlator_name, instance_name);
+    } else {
+        stats_bytes_written = snprintf(stats_filename, PATH_MAX,
+                                       "%s/%s_%s.dump", _IOS_DUMP_DIR,
+                                       __progname, xlator_name);
+        samples_bytes_written = snprintf(samples_filename, PATH_MAX,
+                                         "%s/%s_%s.samp", _IOS_SAMP_DIR,
+                                         __progname, xlator_name);
+    }
+    if ((stats_bytes_written >= PATH_MAX) ||
+        (samples_bytes_written >= PATH_MAX)) {
+        gf_log(this->name, GF_LOG_ERROR,
+               "Invalid path for stats dump (%s) and/or latency "
+               "samples (%s)",
+               stats_filename, samples_filename);
+        goto out;
+    }
+    while (1) {
+        if (conf->dump_thread_should_die)
+            break;
+        (void)pthread_setcanceltype(PTHREAD_CANCEL_ASYNCHRONOUS,
+                                    &old_cancel_type);
+        sleep(conf->ios_dump_interval);
+        (void)pthread_setcanceltype(PTHREAD_CANCEL_DEFERRED, &old_cancel_type);
+        /*
+         * It's not clear whether we should reopen this each time, or
+         * just hold it open and rewind/truncate on each iteration.
+         * Leaving it alone for now.
+         */
+        stats_logfp = fopen(stats_filename, "w+");
+        if (stats_logfp) {
+            (void)ios_dump_args_init(&args, conf->dump_format, stats_logfp);
+            io_stats_dump(this, &args, GF_IOS_INFO_ALL, _gf_false);
+            fclose(stats_logfp);
+            log_stats_fopen_failure = _gf_true;
+        } else if (log_stats_fopen_failure) {
+            gf_log(this->name, GF_LOG_ERROR,
+                   "could not open stats-dump file %s (%s)", stats_filename,
+                   strerror(errno));
+            log_stats_fopen_failure = _gf_false;
+        }
+        samples_logfp = fopen(samples_filename, "w+");
+        if (samples_logfp) {
+            io_stats_dump_latency_samples_logfp(this, samples_logfp);
+            fclose(samples_logfp);
+            log_samples_fopen_failure = _gf_true;
+        } else if (log_samples_fopen_failure) {
+            gf_log(this->name, GF_LOG_ERROR,
+                   "could not open samples-dump file %s (%s)", samples_filename,
+                   strerror(errno));
+            log_samples_fopen_failure = _gf_false;
+        }
+    }
+out:
+    conf->dump_thread_running = _gf_false;
+    gf_log(this->name, GF_LOG_INFO, "IO stats dump thread terminated");
+    return NULL;
 }
 
+static gf_boolean_t
+match_special_xattr(dict_t *d, char *k, data_t *val, void *mdata)
+{
+    gf_boolean_t ret = _gf_false;
+    if (fnmatch("*io*stat*dump", k, 0) == 0) {
+        ret = _gf_true;
+    }
+
+    return ret;
+}
 
 int
-io_stats_setxattr (call_frame_t *frame, xlator_t *this,
-                   loc_t *loc, dict_t *dict,
-                   int32_t flags, dict_t *xdata)
+io_stats_setxattr(call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *dict,
+                  int32_t flags, dict_t *xdata)
 {
-        struct {
-                xlator_t     *this;
-                inode_t      *inode;
-                const char   *path;
-        } stub;
+    struct {
+        xlator_t *this;
+        inode_t *inode;
+        const char *path;
+    } stub;
 
-        stub.this  = this;
-        stub.inode = loc->inode;
-        stub.path  = loc->path;
+    stub.this = this;
+    stub.inode = loc->inode;
+    stub.path = loc->path;
 
-        dict_foreach (dict, conditional_dump, &stub);
+    (void)dict_foreach_match(dict, match_special_xattr, NULL, conditional_dump,
+                             &stub);
 
-        START_FOP_LATENCY (frame);
+    START_FOP_LATENCY(frame);
 
-        STACK_WIND (frame, io_stats_setxattr_cbk,
-                    FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->setxattr,
-                    loc, dict, flags, xdata);
-        return 0;
+    STACK_WIND(frame, io_stats_setxattr_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->setxattr, loc, dict, flags, xdata);
+    return 0;
 }
 
-
 int
-io_stats_getxattr (call_frame_t *frame, xlator_t *this,
-                   loc_t *loc, const char *name, dict_t *xdata)
+io_stats_getxattr(call_frame_t *frame, xlator_t *this, loc_t *loc,
+                  const char *name, dict_t *xdata)
 {
-        START_FOP_LATENCY (frame);
+    START_FOP_LATENCY(frame);
 
-        STACK_WIND (frame, io_stats_getxattr_cbk,
-                    FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->getxattr,
-                    loc, name, xdata);
-        return 0;
+    STACK_WIND(frame, io_stats_getxattr_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->getxattr, loc, name, xdata);
+    return 0;
 }
 
-
 int
-io_stats_removexattr (call_frame_t *frame, xlator_t *this,
-                      loc_t *loc, const char *name, dict_t *xdata)
+io_stats_removexattr(call_frame_t *frame, xlator_t *this, loc_t *loc,
+                     const char *name, dict_t *xdata)
 {
-        START_FOP_LATENCY (frame);
+    START_FOP_LATENCY(frame);
 
-        STACK_WIND (frame, io_stats_removexattr_cbk,
-                    FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->removexattr,
-                    loc, name, xdata);
-        return 0;
+    STACK_WIND(frame, io_stats_removexattr_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->removexattr, loc, name, xdata);
+    return 0;
 }
 
-
 int
-io_stats_fsetxattr (call_frame_t *frame, xlator_t *this,
-                    fd_t *fd, dict_t *dict,
-                    int32_t flags, dict_t *xdata)
+io_stats_fsetxattr(call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *dict,
+                   int32_t flags, dict_t *xdata)
 {
-        START_FOP_LATENCY (frame);
+    START_FOP_LATENCY(frame);
 
-        STACK_WIND (frame, io_stats_fsetxattr_cbk,
-                    FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->fsetxattr,
-                    fd, dict, flags, xdata);
-        return 0;
+    STACK_WIND(frame, io_stats_fsetxattr_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->fsetxattr, fd, dict, flags, xdata);
+    return 0;
 }
 
-
 int
-io_stats_fgetxattr (call_frame_t *frame, xlator_t *this,
-                    fd_t *fd, const char *name, dict_t *xdata)
+io_stats_fgetxattr(call_frame_t *frame, xlator_t *this, fd_t *fd,
+                   const char *name, dict_t *xdata)
 {
-        START_FOP_LATENCY (frame);
+    START_FOP_LATENCY(frame);
 
-        STACK_WIND (frame, io_stats_fgetxattr_cbk,
-                    FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->fgetxattr,
-                    fd, name, xdata);
-        return 0;
+    STACK_WIND(frame, io_stats_fgetxattr_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->fgetxattr, fd, name, xdata);
+    return 0;
 }
 
-
 int
-io_stats_fremovexattr (call_frame_t *frame, xlator_t *this,
-                       fd_t *fd, const char *name, dict_t *xdata)
+io_stats_fremovexattr(call_frame_t *frame, xlator_t *this, fd_t *fd,
+                      const char *name, dict_t *xdata)
 {
-        START_FOP_LATENCY (frame);
+    START_FOP_LATENCY(frame);
 
-        STACK_WIND (frame, io_stats_fremovexattr_cbk,
-                    FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->fremovexattr,
-                    fd, name, xdata);
-        return 0;
+    STACK_WIND(frame, io_stats_fremovexattr_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->fremovexattr, fd, name, xdata);
+    return 0;
 }
 
-
 int
-io_stats_opendir (call_frame_t *frame, xlator_t *this,
-                  loc_t *loc, fd_t *fd, dict_t *xdata)
+io_stats_opendir(call_frame_t *frame, xlator_t *this, loc_t *loc, fd_t *fd,
+                 dict_t *xdata)
 {
+    START_FOP_LATENCY(frame);
 
-        START_FOP_LATENCY (frame);
-
-        STACK_WIND (frame, io_stats_opendir_cbk,
-                    FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->opendir,
-                    loc, fd, xdata);
-        return 0;
+    STACK_WIND(frame, io_stats_opendir_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->opendir, loc, fd, xdata);
+    return 0;
 }
 
 int
-io_stats_readdirp (call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
-                   off_t offset, dict_t *dict)
+io_stats_readdirp(call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
+                  off_t offset, dict_t *dict)
 {
-        frame->local = fd->inode;
-        START_FOP_LATENCY (frame);
+    frame->local = fd->inode;
+    START_FOP_LATENCY(frame);
 
-        STACK_WIND (frame, io_stats_readdirp_cbk,
-                    FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->readdirp,
-                    fd, size, offset, dict);
-        return 0;
+    STACK_WIND(frame, io_stats_readdirp_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->readdirp, fd, size, offset, dict);
+    return 0;
 }
 
-
 int
-io_stats_readdir (call_frame_t *frame, xlator_t *this,
-                  fd_t *fd, size_t size, off_t offset, dict_t *xdata)
+io_stats_readdir(call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
+                 off_t offset, dict_t *xdata)
 {
-        START_FOP_LATENCY (frame);
+    START_FOP_LATENCY(frame);
 
-        STACK_WIND (frame, io_stats_readdir_cbk,
-                    FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->readdir,
-                    fd, size, offset, xdata);
-        return 0;
+    STACK_WIND(frame, io_stats_readdir_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->readdir, fd, size, offset, xdata);
+    return 0;
 }
 
-
 int
-io_stats_fsyncdir (call_frame_t *frame, xlator_t *this,
-                   fd_t *fd, int32_t datasync, dict_t *xdata)
+io_stats_fsyncdir(call_frame_t *frame, xlator_t *this, fd_t *fd,
+                  int32_t datasync, dict_t *xdata)
 {
-        START_FOP_LATENCY (frame);
+    START_FOP_LATENCY(frame);
 
-        STACK_WIND (frame, io_stats_fsyncdir_cbk,
-                    FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->fsyncdir,
-                    fd, datasync, xdata);
-        return 0;
+    STACK_WIND(frame, io_stats_fsyncdir_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->fsyncdir, fd, datasync, xdata);
+    return 0;
 }
 
-
 int
-io_stats_access (call_frame_t *frame, xlator_t *this,
-                 loc_t *loc, int32_t mask, dict_t *xdata)
+io_stats_access(call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t mask,
+                dict_t *xdata)
 {
-        START_FOP_LATENCY (frame);
+    START_FOP_LATENCY(frame);
 
-        STACK_WIND (frame, io_stats_access_cbk,
-                    FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->access,
-                    loc, mask, xdata);
-        return 0;
+    STACK_WIND(frame, io_stats_access_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->access, loc, mask, xdata);
+    return 0;
 }
 
-
 int
-io_stats_ftruncate (call_frame_t *frame, xlator_t *this,
-                    fd_t *fd, off_t offset, dict_t *xdata)
+io_stats_ftruncate(call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
+                   dict_t *xdata)
 {
-        START_FOP_LATENCY (frame);
+    START_FOP_LATENCY(frame);
 
-        STACK_WIND (frame, io_stats_ftruncate_cbk,
-                    FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->ftruncate,
-                    fd, offset, xdata);
-        return 0;
+    STACK_WIND(frame, io_stats_ftruncate_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->ftruncate, fd, offset, xdata);
+    return 0;
 }
 
-
 int
-io_stats_fsetattr (call_frame_t *frame, xlator_t *this,
-                   fd_t *fd, struct iatt *stbuf, int32_t valid, dict_t *xdata)
+io_stats_fsetattr(call_frame_t *frame, xlator_t *this, fd_t *fd,
+                  struct iatt *stbuf, int32_t valid, dict_t *xdata)
 {
-        START_FOP_LATENCY (frame);
+    START_FOP_LATENCY(frame);
 
-        STACK_WIND (frame, io_stats_setattr_cbk,
-                    FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->fsetattr,
-                    fd, stbuf, valid, xdata);
-        return 0;
+    STACK_WIND(frame, io_stats_setattr_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->fsetattr, fd, stbuf, valid, xdata);
+    return 0;
 }
 
-
 int
-io_stats_fstat (call_frame_t *frame, xlator_t *this,
-                fd_t *fd, dict_t *xdata)
+io_stats_fstat(call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *xdata)
 {
-        START_FOP_LATENCY (frame);
+    START_FOP_LATENCY(frame);
 
-        STACK_WIND (frame, io_stats_fstat_cbk,
-                    FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->fstat,
-                    fd, xdata);
-        return 0;
+    STACK_WIND(frame, io_stats_fstat_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->fstat, fd, xdata);
+    return 0;
 }
 
-
 int
 io_stats_fallocate(call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t mode,
-		   off_t offset, size_t len, dict_t *xdata)
+                   off_t offset, size_t len, dict_t *xdata)
 {
-	START_FOP_LATENCY(frame);
+    START_FOP_LATENCY(frame);
 
-	STACK_WIND(frame, io_stats_fallocate_cbk, FIRST_CHILD(this),
-		   FIRST_CHILD(this)->fops->fallocate, fd, mode, offset, len,
-		   xdata);
+    STACK_WIND(frame, io_stats_fallocate_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->fallocate, fd, mode, offset, len,
+               xdata);
 
-	return 0;
+    return 0;
 }
 
-
 int
 io_stats_discard(call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
-		 size_t len, dict_t *xdata)
+                 size_t len, dict_t *xdata)
 {
-	START_FOP_LATENCY(frame);
+    START_FOP_LATENCY(frame);
 
-	STACK_WIND(frame, io_stats_discard_cbk, FIRST_CHILD(this),
-		   FIRST_CHILD(this)->fops->discard, fd, offset, len, xdata);
+    STACK_WIND(frame, io_stats_discard_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->discard, fd, offset, len, xdata);
 
-	return 0;
+    return 0;
 }
 
 int
 io_stats_zerofill(call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
-                 off_t len, dict_t *xdata)
+                  off_t len, dict_t *xdata)
 {
-        START_FOP_LATENCY(frame);
+    START_FOP_LATENCY(frame);
 
-        STACK_WIND(frame, io_stats_zerofill_cbk, FIRST_CHILD(this),
-                   FIRST_CHILD(this)->fops->zerofill, fd, offset, len, xdata);
+    STACK_WIND(frame, io_stats_zerofill_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->zerofill, fd, offset, len, xdata);
 
-        return 0;
+    return 0;
 }
 
+int32_t
+io_stats_ipc(call_frame_t *frame, xlator_t *this, int32_t op, dict_t *xdata)
+{
+    START_FOP_LATENCY(frame);
+
+    STACK_WIND(frame, io_stats_ipc_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->ipc, op, xdata);
+    return 0;
+}
 
 int
-io_stats_lk (call_frame_t *frame, xlator_t *this,
-             fd_t *fd, int32_t cmd, struct gf_flock *lock, dict_t *xdata)
+io_stats_lk(call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t cmd,
+            struct gf_flock *lock, dict_t *xdata)
 {
-        START_FOP_LATENCY (frame);
+    START_FOP_LATENCY(frame);
 
-        STACK_WIND (frame, io_stats_lk_cbk,
-                    FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->lk,
-                    fd, cmd, lock, xdata);
-        return 0;
+    STACK_WIND(frame, io_stats_lk_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->lk, fd, cmd, lock, xdata);
+    return 0;
 }
 
+int
+io_stats_rchecksum(call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
+                   int32_t len, dict_t *xdata)
+{
+    START_FOP_LATENCY(frame);
+
+    STACK_WIND(frame, io_stats_rchecksum_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->rchecksum, fd, offset, len, xdata);
+    return 0;
+}
 
 int
-io_stats_release (xlator_t *this, fd_t *fd)
+io_stats_seek(call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
+              gf_seek_what_t what, dict_t *xdata)
 {
-        struct ios_fd  *iosfd = NULL;
-        struct ios_conf *conf = NULL;
+    START_FOP_LATENCY(frame);
 
-        BUMP_FOP (RELEASE);
+    STACK_WIND(frame, io_stats_seek_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->seek, fd, offset, what, xdata);
+    return 0;
+}
 
-        conf = this->private;
+int
+io_stats_lease(call_frame_t *frame, xlator_t *this, loc_t *loc,
+               struct gf_lease *lease, dict_t *xdata)
+{
+    START_FOP_LATENCY(frame);
 
-        LOCK (&conf->lock);
-        {
-                conf->cumulative.nr_opens--;
-        }
-        UNLOCK (&conf->lock);
+    STACK_WIND(frame, io_stats_lease_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->lease, loc, lease, xdata);
+    return 0;
+}
 
-        ios_fd_ctx_get (fd, this, &iosfd);
-        if (iosfd) {
-                io_stats_dump_fd (this, iosfd);
+int
+io_stats_getactivelk(call_frame_t *frame, xlator_t *this, loc_t *loc,
+                     dict_t *xdata)
+{
+    START_FOP_LATENCY(frame);
 
-                GF_FREE (iosfd->filename);
-                GF_FREE (iosfd);
-        }
+    STACK_WIND(frame, io_stats_getactivelk_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->getactivelk, loc, xdata);
+    return 0;
+}
 
-        return 0;
+int
+io_stats_setactivelk(call_frame_t *frame, xlator_t *this, loc_t *loc,
+                     lock_migration_info_t *locklist, dict_t *xdata)
+{
+    START_FOP_LATENCY(frame);
+
+    STACK_WIND(frame, io_stats_setactivelk_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->setactivelk, loc, locklist, xdata);
+    return 0;
 }
 
+int
+io_stats_compound(call_frame_t *frame, xlator_t *this, void *args,
+                  dict_t *xdata)
+{
+    START_FOP_LATENCY(frame);
+
+    STACK_WIND(frame, io_stats_compound_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->compound, args, xdata);
+    return 0;
+}
 
 int
-io_stats_releasedir (xlator_t *this, fd_t *fd)
+io_stats_release(xlator_t *this, fd_t *fd)
 {
-        BUMP_FOP (RELEASEDIR);
+    struct ios_fd *iosfd = NULL;
+    struct ios_conf *conf = NULL;
 
-        return 0;
+    BUMP_FOP(RELEASE);
+
+    conf = this->private;
+    if (conf) {
+        LOCK(&conf->lock);
+        {
+            conf->cumulative.nr_opens--;
+        }
+        UNLOCK(&conf->lock);
+    }
+
+    ios_fd_ctx_get(fd, this, &iosfd);
+    if (iosfd) {
+        io_stats_dump_fd(this, iosfd);
+
+        GF_FREE(iosfd->filename);
+        GF_FREE(iosfd);
+    }
+
+    return 0;
 }
 
+int
+io_stats_releasedir(xlator_t *this, fd_t *fd)
+{
+    BUMP_FOP(RELEASEDIR);
+
+    return 0;
+}
 
 int
-io_stats_forget (xlator_t *this, inode_t *inode)
+io_stats_forget(xlator_t *this, inode_t *inode)
 {
-        BUMP_FOP (FORGET);
-        ios_stats_cleanup (this, inode);
-        return 0;
+    BUMP_FOP(FORGET);
+    ios_stats_cleanup(this, inode);
+    return 0;
 }
 
 static int
-ios_init_top_stats (struct ios_conf *conf)
+ios_init_top_stats(struct ios_conf *conf)
 {
-        int     i = 0;
+    int i = 0;
 
-        GF_ASSERT (conf);
+    GF_ASSERT(conf);
 
-        for (i = 0; i <IOS_STATS_TYPE_MAX; i++) {
-                conf->list[i].iosstats = GF_CALLOC (1,
-                                         sizeof(*conf->list[i].iosstats),
-                                         gf_io_stats_mt_ios_stat);
+    for (i = 0; i < IOS_STATS_TYPE_MAX; i++) {
+        conf->list[i].iosstats = GF_CALLOC(1, sizeof(*conf->list[i].iosstats),
+                                           gf_io_stats_mt_ios_stat);
 
-                if (!conf->list[i].iosstats)
-                        return -1;
+        if (!conf->list[i].iosstats)
+            return -1;
 
-                INIT_LIST_HEAD(&conf->list[i].iosstats->list);
-                LOCK_INIT (&conf->list[i].lock);
-        }
+        INIT_LIST_HEAD(&conf->list[i].iosstats->list);
+        LOCK_INIT(&conf->list[i].lock);
+    }
 
-        for (i = 0; i < IOS_STATS_THRU_MAX; i ++) {
-                conf->thru_list[i].iosstats = GF_CALLOC (1,
-                                 sizeof (*conf->thru_list[i].iosstats),
-                                 gf_io_stats_mt_ios_stat);
+    for (i = 0; i < IOS_STATS_THRU_MAX; i++) {
+        conf->thru_list[i].iosstats = GF_CALLOC(
+            1, sizeof(*conf->thru_list[i].iosstats), gf_io_stats_mt_ios_stat);
 
-                if (!conf->thru_list[i].iosstats)
-                        return -1;
+        if (!conf->thru_list[i].iosstats)
+            return -1;
 
-                INIT_LIST_HEAD(&conf->thru_list[i].iosstats->list);
-                LOCK_INIT (&conf->thru_list[i].lock);
-        }
+        INIT_LIST_HEAD(&conf->thru_list[i].iosstats->list);
+        LOCK_INIT(&conf->thru_list[i].lock);
+    }
 
-        return 0;
+    return 0;
 }
 
 static void
-ios_destroy_top_stats (struct ios_conf *conf)
-{
-        int                     i = 0;
-        struct ios_stat_head    *list_head = NULL;
-        struct ios_stat_list    *entry     = NULL;
-        struct ios_stat_list    *tmp       = NULL;
-        struct ios_stat_list    *list      = NULL;
-        struct ios_stat         *stat      = NULL;
-
-        GF_ASSERT (conf);
-
-        LOCK (&conf->lock);
-
-        conf->cumulative.nr_opens = 0;
-        conf->cumulative.max_nr_opens = 0;
-        conf->cumulative.max_openfd_time.tv_sec = 0;
-        conf->cumulative.max_openfd_time.tv_usec = 0;
-
-        for (i = 0; i < IOS_STATS_TYPE_MAX; i++) {
-                list_head = &conf->list[i];
-                if (!list_head)
-                        continue;
-                list_for_each_entry_safe (entry, tmp,
-                                          &list_head->iosstats->list, list) {
-                        list = entry;
-                        stat = list->iosstat;
-                        ios_stat_unref (stat);
-                        list_del (&list->list);
-                        GF_FREE (list);
-                        list_head->members--;
-                }
+ios_destroy_top_stats(struct ios_conf *conf)
+{
+    int i = 0;
+    struct ios_stat_head *list_head = NULL;
+    struct ios_stat_list *entry = NULL;
+    struct ios_stat_list *tmp = NULL;
+    struct ios_stat_list *list = NULL;
+    struct ios_stat *stat = NULL;
+
+    GF_ASSERT(conf);
+
+    LOCK(&conf->lock);
+
+    conf->cumulative.nr_opens = 0;
+    conf->cumulative.max_nr_opens = 0;
+    conf->cumulative.max_openfd_time.tv_sec = 0;
+    conf->cumulative.max_openfd_time.tv_usec = 0;
+
+    for (i = 0; i < IOS_STATS_TYPE_MAX; i++) {
+        list_head = &conf->list[i];
+        if (!list_head)
+            continue;
+        list_for_each_entry_safe(entry, tmp, &list_head->iosstats->list, list)
+        {
+            list = entry;
+            stat = list->iosstat;
+            ios_stat_unref(stat);
+            list_del(&list->list);
+            GF_FREE(list);
+            list_head->members--;
         }
-
-        for (i = 0; i < IOS_STATS_THRU_MAX; i++) {
-                list_head = &conf->thru_list[i];
-                if (!list_head)
-                        continue;
-                list_for_each_entry_safe (entry, tmp,
-                                          &list_head->iosstats->list, list) {
-                        list = entry;
-                        stat = list->iosstat;
-                        ios_stat_unref (stat);
-                        list_del (&list->list);
-                        GF_FREE (list);
-                        list_head->members--;
-                }
+        GF_FREE(list_head->iosstats);
+    }
+
+    for (i = 0; i < IOS_STATS_THRU_MAX; i++) {
+        list_head = &conf->thru_list[i];
+        if (!list_head)
+            continue;
+        list_for_each_entry_safe(entry, tmp, &list_head->iosstats->list, list)
+        {
+            list = entry;
+            stat = list->iosstat;
+            ios_stat_unref(stat);
+            list_del(&list->list);
+            GF_FREE(list);
+            list_head->members--;
         }
+        GF_FREE(list_head->iosstats);
+    }
 
-        UNLOCK (&conf->lock);
+    UNLOCK(&conf->lock);
 
-        return;
+    return;
 }
 
-static int
-io_stats_clear (struct ios_conf *conf)
+static void
+io_stats_clear(struct ios_conf *conf)
 {
-        struct timeval      now;
-        int                 ret = -1;
-
-        GF_ASSERT (conf);
-
-        if (!gettimeofday (&now, NULL))
-        {
-            LOCK (&conf->lock);
-            {
-                    ios_global_stats_clear (&conf->cumulative, &now);
-                    ios_global_stats_clear (&conf->incremental, &now);
-                    conf->increment = 0;
-            }
-            UNLOCK (&conf->lock);
-            ret = 0;
-        }
-
-        return ret;
+    time_t now = 0;
+
+    GF_ASSERT(conf);
+    now = gf_time();
+
+    LOCK(&conf->lock);
+    {
+        ios_global_stats_clear(&conf->cumulative, now);
+        ios_global_stats_clear(&conf->incremental, now);
+        conf->increment = 0;
+    }
+    UNLOCK(&conf->lock);
 }
 
 int32_t
-io_priv (xlator_t *this)
-{
-        int                 i;
-        char                key[GF_DUMP_MAX_BUF_LEN];
-        char                key_prefix_cumulative[GF_DUMP_MAX_BUF_LEN];
-        char                key_prefix_incremental[GF_DUMP_MAX_BUF_LEN];
-        double              min, max, avg;
-        uint64_t            count, total;
-        struct ios_conf    *conf = NULL;
-
-        conf = this->private;
-        if (!conf)
-                return -1;
-
-        if(!conf->count_fop_hits || !conf->measure_latency)
-                return -1;
-
-        gf_proc_dump_write("cumulative.data_read", "%"PRIu64,
-                                                conf->cumulative.data_read);
-        gf_proc_dump_write("cumulative.data_written", "%"PRIu64,
-                                                conf->cumulative.data_written);
-
-        gf_proc_dump_write("incremental.data_read", "%"PRIu64,
-                                                conf->incremental.data_read);
-        gf_proc_dump_write("incremental.data_written", "%"PRIu64,
-                                                conf->incremental.data_written);
-
-        snprintf (key_prefix_cumulative, GF_DUMP_MAX_BUF_LEN, "%s.cumulative",
-                                                                    this->name);
-        snprintf (key_prefix_incremental, GF_DUMP_MAX_BUF_LEN, "%s.incremental",
-                                                                    this->name);
-
-        for (i = 0; i < GF_FOP_MAXVALUE; i++) {
-                count = conf->cumulative.fop_hits[i];
-                total = conf->cumulative.latency[i].total;
-                min = conf->cumulative.latency[i].min;
-                max = conf->cumulative.latency[i].max;
-                avg = conf->cumulative.latency[i].avg;
-
-                gf_proc_dump_build_key (key, key_prefix_cumulative,
-                                        (char *)gf_fop_list[i]);
-
-                gf_proc_dump_write (key,"%"PRId64",%"PRId64",%.03f,%.03f,%.03f",
-                                    count, total, min, max, avg);
-
-                count = conf->incremental.fop_hits[i];
-                total = conf->incremental.latency[i].total;
-                min = conf->incremental.latency[i].min;
-                max = conf->incremental.latency[i].max;
-                avg = conf->incremental.latency[i].avg;
-
-                gf_proc_dump_build_key (key, key_prefix_incremental,
-                                        (char *)gf_fop_list[i]);
-
-                gf_proc_dump_write (key,"%"PRId64",%"PRId64",%.03f,%.03f,%.03f",
-                                    count, total, min, max, avg);
-
-        }
-
-        return 0;
+io_priv(xlator_t *this)
+{
+    int i;
+    char key[GF_DUMP_MAX_BUF_LEN];
+    char key_prefix_cumulative[GF_DUMP_MAX_BUF_LEN];
+    char key_prefix_incremental[GF_DUMP_MAX_BUF_LEN];
+    double min, max, avg;
+    uint64_t count, total;
+    struct ios_conf *conf = NULL;
+
+    conf = this->private;
+    if (!conf)
+        return -1;
+
+    if (!conf->count_fop_hits || !conf->measure_latency)
+        return -1;
+
+    gf_proc_dump_write("cumulative.data_read", "%" GF_PRI_ATOMIC,
+                       GF_ATOMIC_GET(conf->cumulative.data_read));
+    gf_proc_dump_write("cumulative.data_written", "%" GF_PRI_ATOMIC,
+                       GF_ATOMIC_GET(conf->cumulative.data_written));
+
+    gf_proc_dump_write("incremental.data_read", "%" GF_PRI_ATOMIC,
+                       GF_ATOMIC_GET(conf->incremental.data_read));
+    gf_proc_dump_write("incremental.data_written", "%" GF_PRI_ATOMIC,
+                       GF_ATOMIC_GET(conf->incremental.data_written));
+
+    snprintf(key_prefix_cumulative, GF_DUMP_MAX_BUF_LEN, "%s.cumulative",
+             this->name);
+    snprintf(key_prefix_incremental, GF_DUMP_MAX_BUF_LEN, "%s.incremental",
+             this->name);
+
+    for (i = 0; i < GF_FOP_MAXVALUE; i++) {
+        count = GF_ATOMIC_GET(conf->cumulative.fop_hits[i]);
+        total = conf->cumulative.latency[i].total;
+        min = conf->cumulative.latency[i].min;
+        max = conf->cumulative.latency[i].max;
+        avg = conf->cumulative.latency[i].avg;
+
+        gf_proc_dump_build_key(key, key_prefix_cumulative, "%s",
+                               (char *)gf_fop_list[i]);
+
+        gf_proc_dump_write(key, "%" PRId64 ",%" PRId64 ",%.03f,%.03f,%.03f",
+                           count, total, min, max, avg);
+
+        count = GF_ATOMIC_GET(conf->incremental.fop_hits[i]);
+        total = conf->incremental.latency[i].total;
+        min = conf->incremental.latency[i].min;
+        max = conf->incremental.latency[i].max;
+        avg = conf->incremental.latency[i].avg;
+
+        gf_proc_dump_build_key(key, key_prefix_incremental, "%s",
+                               (char *)gf_fop_list[i]);
+
+        gf_proc_dump_write(key, "%" PRId64 ",%" PRId64 ",%.03f,%.03f,%.03f",
+                           count, total, min, max, avg);
+    }
+
+    return 0;
 }
 
-int
-reconfigure (xlator_t *this, dict_t *options)
+static void
+ios_set_log_format_code(struct ios_conf *conf, char *dump_format_str)
 {
-        struct ios_conf    *conf = NULL;
-        int                 ret = -1;
-        char               *sys_log_str = NULL;
-        char               *log_format_str = NULL;
-        char               *logger_str    = NULL;
-        int                 sys_log_level = -1;
-        char               *log_str = NULL;
-        int                 log_level = -1;
-        int                 log_format = -1;
-        int                 logger = -1;
-        uint32_t            log_buf_size = 0;
-        uint32_t            log_flush_timeout = 0;
-
-        if (!this || !this->private)
-                goto out;
-
-        conf = this->private;
+    if (strcmp(dump_format_str, "json") == 0)
+        conf->dump_format = IOS_DUMP_TYPE_JSON_FILE;
+    else if (strcmp(dump_format_str, "text") == 0)
+        conf->dump_format = IOS_DUMP_TYPE_FILE;
+    else if (strcmp(dump_format_str, "dict") == 0)
+        conf->dump_format = IOS_DUMP_TYPE_DICT;
+    else if (strcmp(dump_format_str, "samples") == 0)
+        conf->dump_format = IOS_DUMP_TYPE_SAMPLES;
+}
 
-        GF_OPTION_RECONF ("dump-fd-stats", conf->dump_fd_stats, options, bool,
-                          out);
+void
+xlator_set_loglevel(xlator_t *this, int log_level)
+{
+    glusterfs_ctx_t *ctx = NULL;
+    glusterfs_graph_t *active = NULL;
+    xlator_t *top = NULL;
+    xlator_t *trav = this;
 
-        GF_OPTION_RECONF ("count-fop-hits", conf->count_fop_hits, options, bool,
-                          out);
+    ctx = this->ctx;
+    GF_ASSERT(ctx);
+    active = ctx->active;
+    top = active->first;
 
-        GF_OPTION_RECONF ("latency-measurement", conf->measure_latency,
-                          options, bool, out);
+    if (log_level == -1)
+        return;
 
-        GF_OPTION_RECONF ("sys-log-level", sys_log_str, options, str, out);
-        if (sys_log_str) {
-                sys_log_level = glusterd_check_log_level (sys_log_str);
-                set_sys_log_level (sys_log_level);
-        }
+    if (ctx->cmd_args.brick_mux) {
+        /* Set log-level for all brick xlators */
+        top->loglevel = log_level;
 
-        GF_OPTION_RECONF ("log-level", log_str, options, str, out);
-        if (log_str) {
-                log_level = glusterd_check_log_level (log_str);
-                gf_log_set_loglevel (log_level);
-        }
+        /* Set log-level for parent xlator */
+        if (this->parents)
+            this->parents->xlator->loglevel = log_level;
 
-        GF_OPTION_RECONF ("logger", logger_str, options, str, out);
-        if (logger_str) {
-                logger = gf_check_logger (logger_str);
-                gf_log_set_logger (logger);
+        while (trav) {
+            trav->loglevel = log_level;
+            trav = trav->next;
         }
+    } else {
+        gf_log_set_loglevel(this->ctx, log_level);
+    }
+}
 
-        GF_OPTION_RECONF ("log-format", log_format_str, options, str, out);
-        if (log_format_str) {
-                log_format = gf_check_log_format (log_format_str);
-                gf_log_set_logformat (log_format);
+int
+reconfigure(xlator_t *this, dict_t *options)
+{
+    struct ios_conf *conf = NULL;
+    int ret = -1;
+    char *sys_log_str = NULL;
+    char *log_format_str = NULL;
+    char *logger_str = NULL;
+    char *dump_format_str = NULL;
+    int sys_log_level = -1;
+    char *log_str = NULL;
+    int log_level = -1;
+    int log_format = -1;
+    int logger = -1;
+    uint32_t log_buf_size = 0;
+    uint32_t log_flush_timeout = 0;
+    int32_t old_dump_interval;
+    int32_t threads;
+
+    if (!this || !this->private)
+        goto out;
+
+    conf = this->private;
+
+    GF_OPTION_RECONF("dump-fd-stats", conf->dump_fd_stats, options, bool, out);
+
+    GF_OPTION_RECONF("count-fop-hits", conf->count_fop_hits, options, bool,
+                     out);
+
+    GF_OPTION_RECONF("latency-measurement", conf->measure_latency, options,
+                     bool, out);
+
+    old_dump_interval = conf->ios_dump_interval;
+    GF_OPTION_RECONF("ios-dump-interval", conf->ios_dump_interval, options,
+                     int32, out);
+    if ((old_dump_interval <= 0) && (conf->ios_dump_interval > 0)) {
+        conf->dump_thread_running = _gf_true;
+        conf->dump_thread_should_die = _gf_false;
+        ret = gf_thread_create(&conf->dump_thread, NULL,
+                               (void *)&_ios_dump_thread, this, "iosdump");
+        if (ret) {
+            conf->dump_thread_running = _gf_false;
+            gf_log(this ? this->name : "io-stats", GF_LOG_ERROR,
+                   "Failed to start thread"
+                   "while reconfigure. Returning %d",
+                   ret);
+            goto out;
         }
-
-        GF_OPTION_RECONF ("log-buf-size", log_buf_size, options, uint32, out);
-        gf_log_set_log_buf_size (log_buf_size);
-
-        GF_OPTION_RECONF ("log-flush-timeout", log_flush_timeout, options,
-                          time, out);
-        gf_log_set_log_flush_timeout (log_flush_timeout);
-
-        ret = 0;
+    } else if ((old_dump_interval > 0) && (conf->ios_dump_interval == 0)) {
+        _ios_destroy_dump_thread(conf);
+    }
+
+    GF_OPTION_RECONF("ios-sample-interval", conf->ios_sample_interval, options,
+                     int32, out);
+    GF_OPTION_RECONF("ios-dump-format", dump_format_str, options, str, out);
+    ios_set_log_format_code(conf, dump_format_str);
+    GF_OPTION_RECONF("ios-sample-buf-size", conf->ios_sample_buf_size, options,
+                     int32, out);
+    GF_OPTION_RECONF("sys-log-level", sys_log_str, options, str, out);
+    if (sys_log_str) {
+        sys_log_level = glusterd_check_log_level(sys_log_str);
+        set_sys_log_level(sys_log_level);
+    }
+
+    GF_OPTION_RECONF("log-level", log_str, options, str, out);
+    if (log_str) {
+        log_level = glusterd_check_log_level(log_str);
+        /* Set loglevel for all children and server xlators */
+        xlator_set_loglevel(this, log_level);
+    }
+
+    GF_OPTION_RECONF("logger", logger_str, options, str, out);
+    if (logger_str) {
+        logger = gf_check_logger(logger_str);
+        gf_log_set_logger(logger);
+    }
+
+    GF_OPTION_RECONF("log-format", log_format_str, options, str, out);
+    if (log_format_str) {
+        log_format = gf_check_log_format(log_format_str);
+        gf_log_set_logformat(log_format);
+    }
+
+    GF_OPTION_RECONF("log-buf-size", log_buf_size, options, uint32, out);
+    gf_log_set_log_buf_size(log_buf_size);
+
+    GF_OPTION_RECONF("log-flush-timeout", log_flush_timeout, options, time,
+                     out);
+    gf_log_set_log_flush_timeout(log_flush_timeout);
+
+    GF_OPTION_RECONF("threads", threads, options, int32, out);
+    gf_async_adjust_threads(threads);
+
+    ret = 0;
 out:
-        gf_log (this ? this->name : "io-stats",
-                        GF_LOG_DEBUG, "reconfigure returning %d", ret);
-        return ret;
+    gf_log(this ? this->name : "io-stats", GF_LOG_DEBUG,
+           "reconfigure returning %d", ret);
+    return ret;
 }
 
-
 int32_t
-mem_acct_init (xlator_t *this)
+mem_acct_init(xlator_t *this)
 {
-        int     ret = -1;
+    int ret = -1;
 
-        if (!this)
-                return ret;
-
-        ret = xlator_mem_acct_init (this, gf_io_stats_mt_end + 1);
+    if (!this)
+        return ret;
 
-        if (ret != 0) {
-                gf_log (this->name, GF_LOG_ERROR, "Memory accounting init"
-                        " failed");
-                return ret;
-        }
+    ret = xlator_mem_acct_init(this, gf_io_stats_mt_end + 1);
 
+    if (ret != 0) {
+        gf_log(this->name, GF_LOG_ERROR,
+               "Memory accounting init"
+               " failed");
         return ret;
+    }
+
+    return ret;
 }
 
 void
-ios_conf_destroy (struct ios_conf *conf)
+ios_conf_destroy(struct ios_conf *conf)
 {
-        if (!conf)
-                return;
+    if (!conf)
+        return;
 
-        ios_destroy_top_stats (conf);
-        LOCK_DESTROY (&conf->lock);
-        GF_FREE(conf);
+    ios_destroy_top_stats(conf);
+    _ios_destroy_dump_thread(conf);
+    ios_destroy_sample_buf(conf->ios_sample_buf);
+    LOCK_DESTROY(&conf->lock);
+    gf_dnscache_deinit(conf->dnscache);
+    GF_FREE(conf);
 }
 
-int
-init (xlator_t *this)
+static void
+ios_init_stats(struct ios_global_stats *stats)
 {
-        struct ios_conf    *conf = NULL;
-        char               *sys_log_str = NULL;
-        char               *logger_str  = NULL;
-        char               *log_format_str  = NULL;
-        int                 logger = -1;
-        int                 log_format = -1;
-        int                 sys_log_level = -1;
-        char               *log_str = NULL;
-        int                 log_level = -1;
-        int                 ret = -1;
-        uint32_t            log_buf_size = 0;
-        uint32_t            log_flush_timeout = 0;
+    int i = 0;
 
-        if (!this)
-                return -1;
+    GF_ATOMIC_INIT(stats->data_read, 0);
+    GF_ATOMIC_INIT(stats->data_written, 0);
 
-        if (!this->children) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "io_stats translator requires atleast one subvolume");
-                return -1;
-        }
+    for (i = 0; i < IOS_BLOCK_COUNT_SIZE; i++) {
+        GF_ATOMIC_INIT(stats->block_count_write[i], 0);
+        GF_ATOMIC_INIT(stats->block_count_read[i], 0);
+    }
 
-        if (!this->parents) {
-                /* This is very much valid as io-stats currently is loaded
-                 * on top of volumes on both client and server, hence this is
-                 * not an warning message */
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "dangling volume. check volfile ");
-        }
+    for (i = 0; i < GF_FOP_MAXVALUE; i++)
+        GF_ATOMIC_INIT(stats->fop_hits[i], 0);
 
-        conf = GF_CALLOC (1, sizeof(*conf), gf_io_stats_mt_ios_conf);
-
-        if (!conf)
-                goto out;
+    for (i = 0; i < GF_UPCALL_FLAGS_MAXVALUE; i++)
+        GF_ATOMIC_INIT(stats->upcall_hits[i], 0);
 
-        /*
-         * Init it just after calloc, so that we are sure the lock is inited
-         * in case of error paths.
-         */
-        LOCK_INIT (&conf->lock);
-
-        gettimeofday (&conf->cumulative.started_at, NULL);
-        gettimeofday (&conf->incremental.started_at, NULL);
-
-        ret = ios_init_top_stats (conf);
-        if (ret)
-                goto out;
-
-        GF_OPTION_INIT ("dump-fd-stats", conf->dump_fd_stats, bool, out);
-
-        GF_OPTION_INIT ("count-fop-hits", conf->count_fop_hits, bool, out);
-
-        GF_OPTION_INIT ("latency-measurement", conf->measure_latency,
-                          bool, out);
-
-        GF_OPTION_INIT ("sys-log-level", sys_log_str, str, out);
-        if (sys_log_str) {
-                sys_log_level = glusterd_check_log_level (sys_log_str);
-                set_sys_log_level (sys_log_level);
-        }
-
-        GF_OPTION_INIT ("log-level", log_str, str, out);
-        if (log_str) {
-                log_level = glusterd_check_log_level (log_str);
-                gf_log_set_loglevel (log_level);
-        }
-
-        GF_OPTION_INIT ("logger", logger_str, str, out);
-        if (logger_str) {
-                logger = gf_check_logger (logger_str);
-                gf_log_set_logger (logger);
-        }
+    stats->started_at = gf_time();
+}
 
-        GF_OPTION_INIT ("log-format", log_format_str, str, out);
-        if (log_format_str) {
-                log_format = gf_check_log_format (log_format_str);
-                gf_log_set_logformat (log_format);
+int
+init(xlator_t *this)
+{
+    struct ios_conf *conf = NULL;
+    char *volume_id = NULL;
+    char *sys_log_str = NULL;
+    char *logger_str = NULL;
+    char *log_format_str = NULL;
+    char *dump_format_str = NULL;
+    int logger = -1;
+    int log_format = -1;
+    int sys_log_level = -1;
+    char *log_str = NULL;
+    int log_level = -1;
+    int ret = -1;
+    uint32_t log_buf_size = 0;
+    uint32_t log_flush_timeout = 0;
+    int32_t threads;
+
+    if (!this)
+        return -1;
+
+    if (!this->children) {
+        gf_log(this->name, GF_LOG_ERROR,
+               "io_stats translator requires at least one subvolume");
+        return -1;
+    }
+
+    if (!this->parents) {
+        /* This is very much valid as io-stats currently is loaded
+         * on top of volumes on both client and server, hence this is
+         * not an warning message */
+        gf_log(this->name, GF_LOG_DEBUG, "dangling volume. check volfile ");
+    }
+
+    conf = GF_CALLOC(1, sizeof(*conf), gf_io_stats_mt_ios_conf);
+
+    if (!conf)
+        goto out;
+
+    if (dict_get_str(this->options, "unique-id", &conf->unique_id) != 0) {
+        /* This is always set on servers, so we must be a client. */
+        conf->unique_id = this->name;
+    }
+
+    ret = dict_get_strn(this->options, "volume-id", SLEN("volume-id"),
+                        &volume_id);
+    if (!ret) {
+        strncpy(this->graph->volume_id, volume_id, GF_UUID_BUF_SIZE);
+    }
+    /*
+     * Init it just after calloc, so that we are sure the lock is inited
+     * in case of error paths.
+     */
+    LOCK_INIT(&conf->lock);
+    LOCK_INIT(&conf->ios_sampling_lock);
+
+    ios_init_stats(&conf->cumulative);
+    ios_init_stats(&conf->incremental);
+
+    ret = ios_init_top_stats(conf);
+    if (ret)
+        goto out;
+
+    GF_OPTION_INIT("dump-fd-stats", conf->dump_fd_stats, bool, out);
+
+    GF_OPTION_INIT("count-fop-hits", conf->count_fop_hits, bool, out);
+
+    GF_OPTION_INIT("latency-measurement", conf->measure_latency, bool, out);
+
+    GF_OPTION_INIT("ios-dump-interval", conf->ios_dump_interval, int32, out);
+
+    GF_OPTION_INIT("ios-sample-interval", conf->ios_sample_interval, int32,
+                   out);
+
+    GF_OPTION_INIT("ios-dump-format", dump_format_str, str, out);
+    ios_set_log_format_code(conf, dump_format_str);
+
+    GF_OPTION_INIT("ios-sample-buf-size", conf->ios_sample_buf_size, int32,
+                   out);
+
+    ret = ios_init_sample_buf(conf);
+    if (ret) {
+        gf_log(this->name, GF_LOG_ERROR, "Out of memory.");
+        goto out;
+    }
+
+    GF_OPTION_INIT("ios-dnscache-ttl-sec", conf->ios_dnscache_ttl_sec, int32,
+                   out);
+    conf->dnscache = gf_dnscache_init(conf->ios_dnscache_ttl_sec);
+    if (!conf->dnscache) {
+        ret = -1;
+        goto out;
+    }
+
+    GF_OPTION_INIT("sys-log-level", sys_log_str, str, out);
+    if (sys_log_str) {
+        sys_log_level = glusterd_check_log_level(sys_log_str);
+        set_sys_log_level(sys_log_level);
+    }
+
+    GF_OPTION_INIT("log-level", log_str, str, out);
+    if (log_str) {
+        log_level = glusterd_check_log_level(log_str);
+        if (DEFAULT_LOG_LEVEL != log_level)
+            gf_log_set_loglevel(this->ctx, log_level);
+    }
+
+    GF_OPTION_INIT("logger", logger_str, str, out);
+    if (logger_str) {
+        logger = gf_check_logger(logger_str);
+        gf_log_set_logger(logger);
+    }
+
+    GF_OPTION_INIT("log-format", log_format_str, str, out);
+    if (log_format_str) {
+        log_format = gf_check_log_format(log_format_str);
+        gf_log_set_logformat(log_format);
+    }
+
+    GF_OPTION_INIT("log-buf-size", log_buf_size, uint32, out);
+    gf_log_set_log_buf_size(log_buf_size);
+
+    GF_OPTION_INIT("log-flush-timeout", log_flush_timeout, time, out);
+    gf_log_set_log_flush_timeout(log_flush_timeout);
+
+    GF_OPTION_INIT("threads", threads, int32, out);
+    gf_async_adjust_threads(threads);
+
+    this->private = conf;
+    if (conf->ios_dump_interval > 0) {
+        conf->dump_thread_running = _gf_true;
+        conf->dump_thread_should_die = _gf_false;
+        ret = gf_thread_create(&conf->dump_thread, NULL,
+                               (void *)&_ios_dump_thread, this, "iosdump");
+        if (ret) {
+            conf->dump_thread_running = _gf_false;
+            gf_log(this ? this->name : "io-stats", GF_LOG_ERROR,
+                   "Failed to start thread"
+                   "in init. Returning %d",
+                   ret);
+            goto out;
         }
-
-        GF_OPTION_INIT ("log-buf-size", log_buf_size, uint32, out);
-        gf_log_set_log_buf_size (log_buf_size);
-
-        GF_OPTION_INIT ("log-flush-timeout", log_flush_timeout, time, out);
-        gf_log_set_log_flush_timeout (log_flush_timeout);
-
-
-        this->private = conf;
-        ret = 0;
+    }
+    return 0;
 out:
-        if (!this->private) {
-                ios_conf_destroy (conf);
-                ret = -1;
-        }
-
-        return ret;
+    ios_conf_destroy(conf);
+    return ret;
 }
 
 void
-fini (xlator_t *this)
+fini(xlator_t *this)
 {
-        struct ios_conf *conf = NULL;
+    struct ios_conf *conf = NULL;
 
-        if (!this)
-                return;
+    if (!this)
+        return;
 
-        conf = this->private;
-        this->private = NULL;
+    conf = this->private;
 
-        ios_conf_destroy (conf);
-        gf_log (this->name, GF_LOG_INFO,
-                "io-stats translator unloaded");
-        return;
+    ios_conf_destroy(conf);
+    this->private = NULL;
+    gf_log(this->name, GF_LOG_INFO, "io-stats translator unloaded");
+    return;
 }
 
 int
-notify (xlator_t *this, int32_t event, void *data, ...)
+notify(xlator_t *this, int32_t event, void *data, ...)
 {
-        int          ret = 0;
-        struct ios_dump_args args = {0};
-        dict_t       *output = NULL;
-        dict_t       *dict = NULL;
-        int32_t       op = 0;
-        int32_t       list_cnt = 0;
-        double        throughput = 0;
-        double        time = 0;
-        gf_boolean_t  is_peek = _gf_false;
-        va_list ap;
-
-        dict = data;
-        va_start (ap, data);
-        output = va_arg (ap, dict_t*);
-        va_end (ap);
-        switch (event) {
+    int ret = 0;
+    struct ios_dump_args args = {0};
+    dict_t *output = NULL;
+    dict_t *dict = NULL;
+    int32_t op = 0;
+    int32_t list_cnt = 0;
+    double throughput = 0;
+    double time = 0;
+    gf_boolean_t is_peek = _gf_false;
+    va_list ap;
+    struct gf_upcall *up_data = NULL;
+    struct gf_upcall_cache_invalidation *up_ci = NULL;
+
+    dict = data;
+    va_start(ap, data);
+    output = va_arg(ap, dict_t *);
+    va_end(ap);
+    switch (event) {
         case GF_EVENT_TRANSLATOR_INFO:
-                ret = dict_get_str_boolean (dict, "clear-stats", _gf_false);
+            ret = dict_get_str_boolean(dict, "clear-stats", _gf_false);
+            if (ret) {
+                ret = dict_set_int32(output, "top-op", op);
                 if (ret) {
-                        ret = dict_set_int32 (output, "top-op", op);
-                        if (ret) {
-                                gf_log (this->name, GF_LOG_ERROR,
-                                        "Failed to set top-op in dict");
-                                goto out;
-                        }
-                        ios_destroy_top_stats (this->private);
-                        ret = ios_init_top_stats (this->private);
+                    gf_log(this->name, GF_LOG_ERROR,
+                           "Failed to set top-op in dict");
+                    goto out;
+                }
+                ios_destroy_top_stats(this->private);
+                ret = ios_init_top_stats(this->private);
+                if (ret)
+                    gf_log(this->name, GF_LOG_ERROR,
+                           "Failed to reset top stats");
+                ret = dict_set_int32(output, "stats-cleared", ret ? 0 : 1);
+                if (ret)
+                    gf_log(this->name, GF_LOG_ERROR,
+                           "Failed to set stats-cleared"
+                           " in dict");
+                goto out;
+            }
+
+            ret = dict_get_int32(dict, "top-op", &op);
+            if (!ret) {
+                ret = dict_get_int32(dict, "list-cnt", &list_cnt);
+                if (op > IOS_STATS_TYPE_NONE && op < IOS_STATS_TYPE_MAX)
+                    ret = io_stats_dump_stats_to_dict(this, output, op,
+                                                      list_cnt);
+                if (op == IOS_STATS_TYPE_READ_THROUGHPUT ||
+                    op == IOS_STATS_TYPE_WRITE_THROUGHPUT) {
+                    ret = dict_get_double(dict, "throughput", &throughput);
+                    if (!ret) {
+                        ret = dict_get_double(dict, "time", &time);
+                        if (ret)
+                            goto out;
+                        ret = dict_set_double(output, "throughput", throughput);
                         if (ret)
-                                gf_log (this->name, GF_LOG_ERROR,
-                                        "Failed to reset top stats");
-                        ret = dict_set_int32 (output, "stats-cleared",
-                                              ret ? 0 : 1);
+                            goto out;
+                        ret = dict_set_double(output, "time", time);
                         if (ret)
-                                gf_log (this->name, GF_LOG_ERROR,
-                                        "Failed to set stats-cleared"
-                                        " in dict");
-                        goto out;
+                            goto out;
+                    }
+                    ret = 0;
+                }
+            } else {
+                ret = dict_get_int32(dict, "info-op", &op);
+                if (ret || op < GF_IOS_INFO_ALL || GF_IOS_INFO_CLEAR < op)
+                    op = GF_IOS_INFO_ALL;
+
+                ret = dict_set_int32(output, "info-op", op);
+                if (ret) {
+                    gf_log(this->name, GF_LOG_ERROR,
+                           "Failed to set info-op in dict");
+                    goto out;
                 }
 
-                ret = dict_get_int32 (dict, "top-op", &op);
-                if (!ret) {
-                        ret = dict_get_int32 (dict, "list-cnt", &list_cnt);
-                        if (op > IOS_STATS_TYPE_NONE &&
-                            op < IOS_STATS_TYPE_MAX)
-                                ret = io_stats_dump_stats_to_dict (this, output,
-                                                             op, list_cnt);
-                        if (op == IOS_STATS_TYPE_READ_THROUGHPUT ||
-                                op == IOS_STATS_TYPE_WRITE_THROUGHPUT) {
-                                ret = dict_get_double (dict, "throughput",
-                                                        &throughput);
-                                if (!ret) {
-                                        ret = dict_get_double (dict, "time",
-                                                                &time);
-                                        if (ret)
-                                                goto out;
-                                        ret = dict_set_double (output,
-                                                "throughput", throughput);
-                                        if (ret)
-                                                goto out;
-                                        ret = dict_set_double (output, "time",
-                                                        time);
-                                        if (ret)
-                                                goto out;
-                                }
-                                ret = 0;
-
-                        }
+                if (GF_IOS_INFO_CLEAR == op) {
+                    io_stats_clear(this->private);
+
+                    ret = dict_set_int32(output, "stats-cleared", 1);
+                    if (ret)
+                        gf_log(this->name, GF_LOG_ERROR,
+                               "Failed to set stats-cleared"
+                               " in dict");
                 } else {
-                        ret = dict_get_int32 (dict, "info-op", &op);
-                        if (ret || op < GF_CLI_INFO_ALL ||
-                            GF_CLI_INFO_CLEAR < op)
-                            op = GF_CLI_INFO_ALL;
-
-                        ret = dict_set_int32 (output, "info-op", op);
-                        if (ret) {
-                                gf_log (this->name, GF_LOG_ERROR,
-                                        "Failed to set info-op in dict");
-                                goto out;
-                        }
-
-                        if (GF_CLI_INFO_CLEAR == op) {
-                                ret = io_stats_clear (this->private);
-                                if (ret)
-                                        gf_log (this->name, GF_LOG_ERROR,
-                                                "Failed to clear info stats");
-
-                                ret = dict_set_int32 (output, "stats-cleared",
-                                              ret ? 0 : 1);
-                                if (ret)
-                                        gf_log (this->name, GF_LOG_ERROR,
-                                                "Failed to set stats-cleared"
-                                                " in dict");
-                        }
-                        else {
-                                ret = dict_get_str_boolean (dict, "peek",
-                                                            _gf_false);
-                                if (-1 != ret)
-                                        is_peek = ret;
-
-                                (void) ios_dump_args_init (&args,
-                                                IOS_DUMP_TYPE_DICT, output);
-                                ret = io_stats_dump (this, &args, op, is_peek);
-                        }
+                    ret = dict_get_str_boolean(dict, "peek", _gf_false);
+                    if (-1 != ret)
+                        is_peek = ret;
+
+                    (void)ios_dump_args_init(&args, IOS_DUMP_TYPE_DICT, output);
+                    ret = io_stats_dump(this, &args, op, is_peek);
                 }
-                break;
-        default:
-                default_notify (this, event, data);
-                break;
+            }
+            break;
+        case GF_EVENT_UPCALL:
+            up_data = (struct gf_upcall *)data;
+            ios_bump_upcall(this, GF_UPCALL);
+
+            switch (up_data->event_type) {
+                case GF_UPCALL_RECALL_LEASE:
+                    ios_bump_upcall(this, GF_UPCALL_LEASE_RECALL);
+                    break;
+                case GF_UPCALL_CACHE_INVALIDATION:
+                    up_ci = (struct gf_upcall_cache_invalidation *)
+                                up_data->data;
+                    if (up_ci->flags & (UP_XATTR | UP_XATTR_RM))
+                        ios_bump_upcall(this, GF_UPCALL_CI_XATTR);
+                    if (up_ci->flags & IATT_UPDATE_FLAGS)
+                        ios_bump_upcall(this, GF_UPCALL_CI_STAT);
+                    if (up_ci->flags & UP_RENAME_FLAGS)
+                        ios_bump_upcall(this, GF_UPCALL_CI_RENAME);
+                    if (up_ci->flags & UP_FORGET)
+                        ios_bump_upcall(this, GF_UPCALL_CI_FORGET);
+                    if (up_ci->flags & UP_NLINK)
+                        ios_bump_upcall(this, GF_UPCALL_CI_NLINK);
+                    break;
+                default:
+                    gf_msg_debug(this->name, 0,
+                                 "Unknown upcall event "
+                                 "type :%d",
+                                 up_data->event_type);
+                    break;
+            }
 
-        }
+            default_notify(this, event, data);
+            break;
+        default:
+            default_notify(this, event, data);
+            break;
+    }
 out:
-        return ret;
+    return ret;
 }
 
-struct xlator_dumpops dumpops = {
-        .priv    = io_priv
-};
+struct xlator_dumpops dumpops = {.priv = io_priv};
 
 struct xlator_fops fops = {
-        .stat        = io_stats_stat,
-        .readlink    = io_stats_readlink,
-        .mknod       = io_stats_mknod,
-        .mkdir       = io_stats_mkdir,
-        .unlink      = io_stats_unlink,
-        .rmdir       = io_stats_rmdir,
-        .symlink     = io_stats_symlink,
-        .rename      = io_stats_rename,
-        .link        = io_stats_link,
-        .truncate    = io_stats_truncate,
-        .open        = io_stats_open,
-        .readv       = io_stats_readv,
-        .writev      = io_stats_writev,
-        .statfs      = io_stats_statfs,
-        .flush       = io_stats_flush,
-        .fsync       = io_stats_fsync,
-        .setxattr    = io_stats_setxattr,
-        .getxattr    = io_stats_getxattr,
-        .removexattr = io_stats_removexattr,
-        .fsetxattr    = io_stats_fsetxattr,
-        .fgetxattr    = io_stats_fgetxattr,
-        .fremovexattr = io_stats_fremovexattr,
-        .opendir     = io_stats_opendir,
-        .readdir     = io_stats_readdir,
-        .readdirp    = io_stats_readdirp,
-        .fsyncdir    = io_stats_fsyncdir,
-        .access      = io_stats_access,
-        .ftruncate   = io_stats_ftruncate,
-        .fstat       = io_stats_fstat,
-        .create      = io_stats_create,
-        .lk          = io_stats_lk,
-        .inodelk     = io_stats_inodelk,
-        .finodelk    = io_stats_finodelk,
-        .entrylk     = io_stats_entrylk,
-        .lookup      = io_stats_lookup,
-        .xattrop     = io_stats_xattrop,
-        .fxattrop    = io_stats_fxattrop,
-        .setattr     = io_stats_setattr,
-        .fsetattr    = io_stats_fsetattr,
-	.fallocate   = io_stats_fallocate,
-	.discard     = io_stats_discard,
-        .zerofill    = io_stats_zerofill,
+    .stat = io_stats_stat,
+    .readlink = io_stats_readlink,
+    .mknod = io_stats_mknod,
+    .mkdir = io_stats_mkdir,
+    .unlink = io_stats_unlink,
+    .rmdir = io_stats_rmdir,
+    .symlink = io_stats_symlink,
+    .rename = io_stats_rename,
+    .link = io_stats_link,
+    .truncate = io_stats_truncate,
+    .open = io_stats_open,
+    .readv = io_stats_readv,
+    .writev = io_stats_writev,
+    .statfs = io_stats_statfs,
+    .flush = io_stats_flush,
+    .fsync = io_stats_fsync,
+    .setxattr = io_stats_setxattr,
+    .getxattr = io_stats_getxattr,
+    .removexattr = io_stats_removexattr,
+    .fsetxattr = io_stats_fsetxattr,
+    .fgetxattr = io_stats_fgetxattr,
+    .fremovexattr = io_stats_fremovexattr,
+    .opendir = io_stats_opendir,
+    .readdir = io_stats_readdir,
+    .readdirp = io_stats_readdirp,
+    .fsyncdir = io_stats_fsyncdir,
+    .access = io_stats_access,
+    .ftruncate = io_stats_ftruncate,
+    .fstat = io_stats_fstat,
+    .create = io_stats_create,
+    .lk = io_stats_lk,
+    .inodelk = io_stats_inodelk,
+    .finodelk = io_stats_finodelk,
+    .entrylk = io_stats_entrylk,
+    .fentrylk = io_stats_fentrylk,
+    .lookup = io_stats_lookup,
+    .xattrop = io_stats_xattrop,
+    .fxattrop = io_stats_fxattrop,
+    .setattr = io_stats_setattr,
+    .fsetattr = io_stats_fsetattr,
+    .fallocate = io_stats_fallocate,
+    .discard = io_stats_discard,
+    .zerofill = io_stats_zerofill,
+    .ipc = io_stats_ipc,
+    .rchecksum = io_stats_rchecksum,
+    .seek = io_stats_seek,
+    .lease = io_stats_lease,
+    .getactivelk = io_stats_getactivelk,
+    .setactivelk = io_stats_setactivelk,
+    .compound = io_stats_compound,
+    .copy_file_range = io_stats_copy_file_range,
 };
 
 struct xlator_cbks cbks = {
-        .release     = io_stats_release,
-        .releasedir  = io_stats_releasedir,
-        .forget      = io_stats_forget,
+    .release = io_stats_release,
+    .releasedir = io_stats_releasedir,
+    .forget = io_stats_forget,
 };
 
 struct volume_options options[] = {
-        { .key  = {"dump-fd-stats"},
-          .type = GF_OPTION_TYPE_BOOL,
-          .default_value = "off",
-          .description = "If on stats related to file-operations would be "
-                         "tracked inside GlusterFS data-structures."
-        },
-        { .key  = { "latency-measurement" },
-          .type = GF_OPTION_TYPE_BOOL,
-          .default_value = "off",
-          .description = "If on stats related to the latency of each operation "
-                         "would be tracked inside GlusterFS data-structures. "
-        },
-        { .key  = {"count-fop-hits"},
-          .type = GF_OPTION_TYPE_BOOL,
-        },
-        { .key = {"log-level"},
-          .type = GF_OPTION_TYPE_STR,
-          .value = { "DEBUG", "WARNING", "ERROR", "INFO",
-                     "CRITICAL", "NONE", "TRACE"}
-        },
-
-        /* These are synthetic entries to assist validation of CLI's  *
-         *  volume set  command                                       */
-        { .key = {"client-log-level"},
-          .type = GF_OPTION_TYPE_STR,
-          .default_value = "INFO",
-          .description = "Changes the log-level of the clients",
-          .value = { "DEBUG", "WARNING", "ERROR", "INFO",
-                     "CRITICAL", "NONE", "TRACE"}
-        },
-        { .key = {"sys-log-level"},
-          .type = GF_OPTION_TYPE_STR,
-          .default_value = "CRITICAL",
-          .description = "Gluster's syslog log-level",
-          .value = { "WARNING", "ERROR", "INFO", "CRITICAL"}
-        },
-        { .key = {"brick-log-level"},
-          .type = GF_OPTION_TYPE_STR,
-          .default_value = "INFO",
-          .description = "Changes the log-level of the bricks",
-          .value = { "DEBUG", "WARNING", "ERROR", "INFO",
-                     "CRITICAL", "NONE", "TRACE"}
-        },
-        { .key = {"logger"},
-          .type = GF_OPTION_TYPE_STR,
-          .value = { GF_LOGGER_GLUSTER_LOG, GF_LOGGER_SYSLOG}
-        },
-        { .key = {"client-logger"},
-          .type = GF_OPTION_TYPE_STR,
-          .default_value = GF_LOGGER_GLUSTER_LOG,
-          .description = "Changes the logging sub-system to log to, for the "
-                         "clients",
-          .value = { GF_LOGGER_GLUSTER_LOG, GF_LOGGER_SYSLOG}
-        },
-        { .key = {"brick-logger"},
-          .type = GF_OPTION_TYPE_STR,
-          .default_value = GF_LOGGER_GLUSTER_LOG,
-          .description = "Changes the logging sub-system to log to, for the "
-                         "bricks",
-          .value = { GF_LOGGER_GLUSTER_LOG, GF_LOGGER_SYSLOG}
-        },
-        { .key = {"log-format"},
-          .type = GF_OPTION_TYPE_STR,
-          .value = { GF_LOG_FORMAT_NO_MSG_ID, GF_LOG_FORMAT_WITH_MSG_ID}
-        },
-        { .key = {"client-log-format"},
-          .type = GF_OPTION_TYPE_STR,
-          .default_value = GF_LOG_FORMAT_WITH_MSG_ID,
-          .description = "Changes log format for the clients",
-          .value = { GF_LOG_FORMAT_NO_MSG_ID, GF_LOG_FORMAT_WITH_MSG_ID}
-        },
-        { .key = {"brick-log-format"},
-          .type = GF_OPTION_TYPE_STR,
-          .default_value = GF_LOG_FORMAT_WITH_MSG_ID,
-          .description = "Changes the log format for the bricks",
-          .value = { GF_LOG_FORMAT_NO_MSG_ID, GF_LOG_FORMAT_WITH_MSG_ID}
-        },
-        { .key  = {"log-buf-size"},
-          .type = GF_OPTION_TYPE_INT,
-          .min  = GF_LOG_LRU_BUFSIZE_MIN,
-          .max  = GF_LOG_LRU_BUFSIZE_MAX,
-          .default_value = "5",
-        },
-        { .key  = {"client-log-buf-size"},
-          .type = GF_OPTION_TYPE_INT,
-          .min  = GF_LOG_LRU_BUFSIZE_MIN,
-          .max  = GF_LOG_LRU_BUFSIZE_MAX,
-          .default_value = "5",
-          .description = "This option determines the maximum number of unique "
-                         "log messages that can be buffered for a time equal to"
-                         " the value of the option client-log-flush-timeout."
-        },
-        { .key  = {"brick-log-buf-size"},
-          .type = GF_OPTION_TYPE_INT,
-          .min  = GF_LOG_LRU_BUFSIZE_MIN,
-          .max  = GF_LOG_LRU_BUFSIZE_MAX,
-          .default_value = "5",
-          .description = "This option determines the maximum number of unique "
-                         "log messages that can be buffered for a time equal to"
-                         " the value of the option brick-log-flush-timeout."
-        },
-        { .key  = {"log-flush-timeout"},
-          .type = GF_OPTION_TYPE_TIME,
-          .min  = GF_LOG_FLUSH_TIMEOUT_MIN,
-          .max  = GF_LOG_FLUSH_TIMEOUT_MAX,
-          .default_value = "120",
-        },
-        { .key  = {"client-log-flush-timeout"},
-          .type = GF_OPTION_TYPE_TIME,
-          .min  = GF_LOG_FLUSH_TIMEOUT_MIN,
-          .max  = GF_LOG_FLUSH_TIMEOUT_MAX,
-          .default_value = "120",
-          .description = "This option determines the maximum number of unique "
-                         "log messages that can be buffered for a time equal to"
-                         " the value of the option client-log-flush-timeout."
-        },
-        { .key  = {"brick-log-flush-timeout"},
-          .type = GF_OPTION_TYPE_TIME,
-          .min  = GF_LOG_FLUSH_TIMEOUT_MIN,
-          .max  = GF_LOG_FLUSH_TIMEOUT_MAX,
-          .default_value = "120",
-          .description = "This option determines the maximum number of unique "
-                         "log messages that can be buffered for a time equal to"
-                         " the value of the option brick-log-flush-timeout."
-        },
-        { .key  = {NULL} },
+    {.key = {"dump-fd-stats"},
+     .op_version = {1},
+     .flags = OPT_FLAG_SETTABLE | OPT_FLAG_DOC,
+     .tags = {"io-stats"},
+     .type = GF_OPTION_TYPE_BOOL,
+     .default_value = "off",
+     .description = "If on stats related to file-operations would be "
+                    "tracked inside GlusterFS data-structures."},
+    {.key = {"ios-dump-interval"},
+     .type = GF_OPTION_TYPE_INT,
+     .op_version = {1},
+     .flags = OPT_FLAG_SETTABLE | OPT_FLAG_DOC,
+     .tags = {"io-stats"},
+     .min = 0,
+     .max = 3600,
+     .default_value = "0",
+     .description = "Interval (in seconds) at which to auto-dump "
+                    "statistics. Zero disables automatic dumping."},
+    {.key = {"ios-sample-interval"},
+     .type = GF_OPTION_TYPE_INT,
+     .op_version = {1},
+     .flags = OPT_FLAG_SETTABLE | OPT_FLAG_DOC,
+     .tags = {"io-stats"},
+     .min = 0,
+     .max = 65535,
+     .default_value = "0",
+     .description = "Interval in which we want to collect FOP latency "
+                    "samples.  2 means collect a sample every 2nd FOP."},
+    {.key = {"ios-dump-format"},
+     .type = GF_OPTION_TYPE_STR,
+     .op_version = {GD_OP_VERSION_3_12_0},
+     .flags = OPT_FLAG_SETTABLE | OPT_FLAG_DOC,
+     .tags = {"io-stats"},
+     .default_value = "json",
+     .description = " The dump-format option specifies the format in which"
+                    " to dump the statistics. Select between \"text\", "
+                    "\"json\", \"dict\" and \"samples\". Default is "
+                    "\"json\".",
+     .value = {"text", "json", "dict", "samples"}},
+    {.key = {"ios-sample-buf-size"},
+     .type = GF_OPTION_TYPE_INT,
+     .op_version = {1},
+     .flags = OPT_FLAG_SETTABLE | OPT_FLAG_DOC,
+     .tags = {"io-stats"},
+     .min = 1024,
+     .max = 1024 * 1024,
+     .default_value = "65535",
+     .description = "The maximum size of our FOP sampling ring buffer."},
+    {.key = {"ios-dnscache-ttl-sec"},
+     .type = GF_OPTION_TYPE_INT,
+     .op_version = {1},
+     .flags = OPT_FLAG_SETTABLE | OPT_FLAG_DOC,
+     .tags = {"io-stats"},
+     .min = 1,
+     .max = 3600 * 72,
+     .default_value = "86400",
+     .description = "The interval after wish a cached DNS entry will be "
+                    "re-validated.  Default: 24 hrs"},
+    {.key = {"latency-measurement"},
+     .type = GF_OPTION_TYPE_BOOL,
+     .op_version = {1},
+     .flags = OPT_FLAG_SETTABLE | OPT_FLAG_DOC,
+     .tags = {"io-stats"},
+     .default_value = "off",
+     .description = "If on stats related to the latency of each operation "
+                    "would be tracked inside GlusterFS data-structures. "},
+    {
+        .key = {"count-fop-hits"},
+        .type = GF_OPTION_TYPE_BOOL,
+        .op_version = {1},
+        .flags = OPT_FLAG_SETTABLE,
+        .tags = {"io-stats"},
+    },
+    {.key = {"log-level"},
+     .type = GF_OPTION_TYPE_STR,
+     .value = {"DEBUG", "WARNING", "ERROR", "INFO", "CRITICAL", "NONE",
+               "TRACE"}},
+
+    /* These are synthetic entries to assist validation of CLI's  *
+     *  volume set  command                                       */
+    {.key = {"client-log-level"},
+     .type = GF_OPTION_TYPE_STR,
+     .op_version = {1},
+     .flags = OPT_FLAG_SETTABLE | OPT_FLAG_CLIENT_OPT | OPT_FLAG_DOC,
+     .tags = {"io-stats"},
+     .default_value = "INFO",
+     .description = "Changes the log-level of the clients",
+     .value = {"DEBUG", "WARNING", "ERROR", "INFO", "CRITICAL", "NONE",
+               "TRACE"}},
+    {.key = {"sys-log-level"},
+     .type = GF_OPTION_TYPE_STR,
+     .default_value = "CRITICAL",
+     .description = "Gluster's syslog log-level",
+     .value = {"WARNING", "ERROR", "INFO", "CRITICAL"}},
+    {.key = {"brick-log-level"},
+     .type = GF_OPTION_TYPE_STR,
+     .op_version = {1},
+     .flags = OPT_FLAG_SETTABLE | OPT_FLAG_DOC,
+     .tags = {"io-stats"},
+     .default_value = "INFO",
+     .description = "Changes the log-level of the bricks",
+     .value = {"DEBUG", "WARNING", "ERROR", "INFO", "CRITICAL", "NONE",
+               "TRACE"}},
+    {.key = {"logger"},
+     .type = GF_OPTION_TYPE_STR,
+     .value = {GF_LOGGER_GLUSTER_LOG, GF_LOGGER_SYSLOG}},
+    {.key = {"client-logger"},
+     .type = GF_OPTION_TYPE_STR,
+     .op_version = {GD_OP_VERSION_3_6_0},
+     .flags = OPT_FLAG_SETTABLE | OPT_FLAG_CLIENT_OPT | OPT_FLAG_DOC,
+     .tags = {"io-stats"},
+     .default_value = GF_LOGGER_GLUSTER_LOG,
+     .description = "Changes the logging sub-system to log to, for the "
+                    "clients",
+     .value = {GF_LOGGER_GLUSTER_LOG, GF_LOGGER_SYSLOG}},
+    {.key = {"brick-logger"},
+     .type = GF_OPTION_TYPE_STR,
+     .op_version = {GD_OP_VERSION_3_6_0},
+     .flags = OPT_FLAG_SETTABLE | OPT_FLAG_DOC,
+     .tags = {"io-stats"},
+     .default_value = GF_LOGGER_GLUSTER_LOG,
+     .description = "Changes the logging sub-system to log to, for the "
+                    "bricks",
+     .value = {GF_LOGGER_GLUSTER_LOG, GF_LOGGER_SYSLOG}},
+    {.key = {"log-format"},
+     .type = GF_OPTION_TYPE_STR,
+     .value = {GF_LOG_FORMAT_NO_MSG_ID, GF_LOG_FORMAT_WITH_MSG_ID}},
+    {.key = {"client-log-format"},
+     .type = GF_OPTION_TYPE_STR,
+     .op_version = {GD_OP_VERSION_3_6_0},
+     .flags = OPT_FLAG_SETTABLE | OPT_FLAG_CLIENT_OPT | OPT_FLAG_DOC,
+     .tags = {"io-stats"},
+     .default_value = GF_LOG_FORMAT_WITH_MSG_ID,
+     .description = "Changes log format for the clients",
+     .value = {GF_LOG_FORMAT_NO_MSG_ID, GF_LOG_FORMAT_WITH_MSG_ID}},
+    {.key = {"brick-log-format"},
+     .type = GF_OPTION_TYPE_STR,
+     .op_version = {GD_OP_VERSION_3_6_0},
+     .flags = OPT_FLAG_SETTABLE | OPT_FLAG_DOC,
+     .tags = {"io-stats"},
+     .default_value = GF_LOG_FORMAT_WITH_MSG_ID,
+     .description = "Changes the log format for the bricks",
+     .value = {GF_LOG_FORMAT_NO_MSG_ID, GF_LOG_FORMAT_WITH_MSG_ID}},
+    {
+        .key = {"log-buf-size"},
+        .type = GF_OPTION_TYPE_INT,
+        .min = GF_LOG_LRU_BUFSIZE_MIN,
+        .max = GF_LOG_LRU_BUFSIZE_MAX,
+        .default_value = "5",
+    },
+    {.key = {"client-log-buf-size"},
+     .type = GF_OPTION_TYPE_INT,
+     .op_version = {GD_OP_VERSION_3_6_0},
+     .flags = OPT_FLAG_SETTABLE | OPT_FLAG_CLIENT_OPT | OPT_FLAG_DOC,
+     .tags = {"io-stats"},
+     .min = GF_LOG_LRU_BUFSIZE_MIN,
+     .max = GF_LOG_LRU_BUFSIZE_MAX,
+     .default_value = "5",
+     .description = "This option determines the maximum number of unique "
+                    "log messages that can be buffered for a time equal to"
+                    " the value of the option client-log-flush-timeout."},
+    {.key = {"brick-log-buf-size"},
+     .type = GF_OPTION_TYPE_INT,
+     .op_version = {GD_OP_VERSION_3_6_0},
+     .flags = OPT_FLAG_SETTABLE | OPT_FLAG_DOC,
+     .tags = {"io-stats"},
+     .min = GF_LOG_LRU_BUFSIZE_MIN,
+     .max = GF_LOG_LRU_BUFSIZE_MAX,
+     .default_value = "5",
+     .description = "This option determines the maximum number of unique "
+                    "log messages that can be buffered for a time equal to"
+                    " the value of the option brick-log-flush-timeout."},
+    {
+        .key = {"log-flush-timeout"},
+        .type = GF_OPTION_TYPE_TIME,
+        .min = GF_LOG_FLUSH_TIMEOUT_MIN,
+        .max = GF_LOG_FLUSH_TIMEOUT_MAX,
+        .default_value = "120",
+    },
+    {.key = {"client-log-flush-timeout"},
+     .type = GF_OPTION_TYPE_TIME,
+     .op_version = {GD_OP_VERSION_3_6_0},
+     .flags = OPT_FLAG_SETTABLE | OPT_FLAG_CLIENT_OPT | OPT_FLAG_DOC,
+     .tags = {"io-stats"},
+     .min = GF_LOG_FLUSH_TIMEOUT_MIN,
+     .max = GF_LOG_FLUSH_TIMEOUT_MAX,
+     .default_value = "120",
+     .description = "This option determines the maximum number of unique "
+                    "log messages that can be buffered for a time equal to"
+                    " the value of the option client-log-flush-timeout."},
+    {.key = {"brick-log-flush-timeout"},
+     .type = GF_OPTION_TYPE_TIME,
+     .op_version = {GD_OP_VERSION_3_6_0},
+     .flags = OPT_FLAG_SETTABLE | OPT_FLAG_DOC,
+     .tags = {"io-stats"},
+     .min = GF_LOG_FLUSH_TIMEOUT_MIN,
+     .max = GF_LOG_FLUSH_TIMEOUT_MAX,
+     .default_value = "120",
+     .description = "This option determines the maximum number of unique "
+                    "log messages that can be buffered for a time equal to"
+                    " the value of the option brick-log-flush-timeout."},
+    {.key = {"unique-id"},
+     .type = GF_OPTION_TYPE_STR,
+     .default_value = "/no/such/path",
+     .description = "Unique ID for our files."},
+    {.key = {"global-threading"},
+     .type = GF_OPTION_TYPE_BOOL,
+     .default_value = "off",
+     .op_version = {GD_OP_VERSION_6_0},
+     .flags = OPT_FLAG_SETTABLE,
+     .tags = {"io-stats", "threading"},
+     .description = "This option enables the global threading support for "
+                    "bricks. If enabled, it's recommended to also enable "
+                    "'performance.iot-pass-through'"},
+    {.key = {"threads"}, .type = GF_OPTION_TYPE_INT},
+    {.key = {"brick-threads"},
+     .type = GF_OPTION_TYPE_INT,
+     .default_value = "16",
+     .min = 0,
+     .max = GF_ASYNC_MAX_THREADS,
+     .op_version = {GD_OP_VERSION_6_0},
+     .flags = OPT_FLAG_SETTABLE | OPT_FLAG_DOC,
+     .tags = {"io-stats", "threading"},
+     .description = "When global threading is used, this value determines the "
+                    "maximum amount of threads that can be created on bricks"},
+    {.key = {"client-threads"},
+     .type = GF_OPTION_TYPE_INT,
+     .default_value = "16",
+     .min = 0,
+     .max = GF_ASYNC_MAX_THREADS,
+     .op_version = {GD_OP_VERSION_6_0},
+     .flags = OPT_FLAG_SETTABLE | OPT_FLAG_DOC | OPT_FLAG_CLIENT_OPT,
+     .tags = {"io-stats", "threading"},
+     .description = "When global threading is used, this value determines the "
+                    "maximum amount of threads that can be created on clients"},
+    {.key = {"volume-id"},
+     .type = GF_OPTION_TYPE_STR,
+     .op_version = {GD_OP_VERSION_7_1},
+     .tags = {"global", "volume-id"},
+     .description =
+         "This option points to the 'unique' UUID particular to this "
+         "volume, which would be set in 'graph->volume_id'"},
+    {.key = {NULL}},
+};
 
+xlator_api_t xlator_api = {
+    .init = init,
+    .fini = fini,
+    .notify = notify,
+    .reconfigure = reconfigure,
+    .mem_acct_init = mem_acct_init,
+    .op_version = {1}, /* Present from the initial version */
+    .dumpops = &dumpops,
+    .fops = &fops,
+    .cbks = &cbks,
+    .options = options,
+    .identifier = "io-stats",
+    .category = GF_MAINTAINED,
 };
diff --git a/xlators/features/qemu-block/Makefile.am b/xlators/debug/sink/Makefile.am
index af437a64d6d..f2689244371 100644
--- a/xlators/features/qemu-block/Makefile.am
+++ b/xlators/debug/sink/Makefile.am
@@ -1 +1,2 @@
 SUBDIRS = src
+
diff --git a/xlators/debug/sink/src/Makefile.am b/xlators/debug/sink/src/Makefile.am
new file mode 100644
index 00000000000..f952c2ce6bc
--- /dev/null
+++ b/xlators/debug/sink/src/Makefile.am
@@ -0,0 +1,14 @@
+xlator_LTLIBRARIES = sink.la
+xlatordir = $(libdir)/glusterfs/$(PACKAGE_VERSION)/xlator/debug
+
+AM_CPPFLAGS = $(GF_CPPFLAGS) -I$(top_srcdir)/libglusterfs/src \
+        -I$(top_builddir)/rpc/xdr/src
+AM_CFLAGS = -Wall $(GF_CFLAGS)
+
+sink_la_LDFLAGS = -module $(GF_XLATOR_DEFAULT_LDFLAGS)
+
+sink_la_SOURCES = sink.c
+sink_la_LIBADD = $(top_builddir)/libglusterfs/src/libglusterfs.la
+
+CLEANFILES =
+
diff --git a/xlators/debug/sink/src/sink.c b/xlators/debug/sink/src/sink.c
new file mode 100644
index 00000000000..9822bbb732e
--- /dev/null
+++ b/xlators/debug/sink/src/sink.c
@@ -0,0 +1,94 @@
+/*
+   Copyright (c) 2017 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
+
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
+
+#include <glusterfs/xlator.h>
+#include <glusterfs/defaults.h>
+
+int32_t
+init(xlator_t *this)
+{
+    return 0;
+}
+
+void
+fini(xlator_t *this)
+{
+    return;
+}
+
+/*
+ * notify - when parent sends PARENT_UP, send CHILD_UP event from here
+ */
+int32_t
+notify(xlator_t *this, int32_t event, void *data, ...)
+{
+    switch (event) {
+        case GF_EVENT_PARENT_UP:
+            /* Tell the parent that this xlator is up */
+            default_notify(this, GF_EVENT_CHILD_UP, data);
+            break;
+        case GF_EVENT_PARENT_DOWN:
+            /* Tell the parent that this xlator is down */
+            default_notify(this, GF_EVENT_CHILD_DOWN, data);
+            break;
+        default:
+            break;
+    }
+
+    return 0;
+}
+
+/*
+ * A lookup on "/" is done while mounting or glfs_init() is performed. This
+ * needs to return a valid directory for the root of the mountpoint.
+ *
+ * In case this xlator is used for more advanced debugging, it will need to be
+ * extended to support different LOOKUPs too.
+ */
+static int32_t
+sink_lookup(call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xdata)
+{
+    struct iatt stbuf = {
+        0,
+    };
+    struct iatt postparent = {
+        0,
+    };
+
+    /* the root of the volume always need to be a directory */
+    stbuf.ia_type = IA_IFDIR;
+
+    STACK_UNWIND_STRICT(lookup, frame, 0, 0, loc ? loc->inode : NULL, &stbuf,
+                        xdata, &postparent);
+
+    return 0;
+}
+
+struct xlator_fops fops = {
+    .lookup = sink_lookup,
+};
+
+struct xlator_cbks cbks = {};
+
+struct volume_options options[] = {
+    {.key = {NULL}},
+};
+
+xlator_api_t xlator_api = {
+    .init = init,
+    .fini = fini,
+    .notify = notify,
+    .op_version = {GD_OP_VERSION_3_12_0},
+    .fops = &fops,
+    .cbks = &cbks,
+    .options = options,
+    .identifier = "sink",
+    .category = GF_TECH_PREVIEW,
+};
diff --git a/xlators/debug/trace/src/Makefile.am b/xlators/debug/trace/src/Makefile.am
index 7b2597b4db6..a37ea63af04 100644
--- a/xlators/debug/trace/src/Makefile.am
+++ b/xlators/debug/trace/src/Makefile.am
@@ -2,13 +2,14 @@
 xlator_LTLIBRARIES = trace.la
 xlatordir = $(libdir)/glusterfs/$(PACKAGE_VERSION)/xlator/debug
 
-trace_la_LDFLAGS = -module -avoid-version
+trace_la_LDFLAGS = -module $(GF_XLATOR_DEFAULT_LDFLAGS)
 
 trace_la_SOURCES = trace.c
 trace_la_LIBADD = $(top_builddir)/libglusterfs/src/libglusterfs.la
 
 noinst_HEADERS = trace.h trace-mem-types.h
-AM_CPPFLAGS = $(GF_CPPFLAGS) -I$(top_srcdir)/libglusterfs/src
+AM_CPPFLAGS = $(GF_CPPFLAGS) -I$(top_srcdir)/libglusterfs/src \
+	-I$(top_srcdir)/rpc/xdr/src -I$(top_builddir)/rpc/xdr/src
 
 AM_CFLAGS = -Wall $(GF_CFLAGS)
 
diff --git a/xlators/debug/trace/src/trace-mem-types.h b/xlators/debug/trace/src/trace-mem-types.h
index 9fa7d97c2ca..18a7e0414a6 100644
--- a/xlators/debug/trace/src/trace-mem-types.h
+++ b/xlators/debug/trace/src/trace-mem-types.h
@@ -8,14 +8,13 @@
    cases as published by the Free Software Foundation.
 */
 
-
 #ifndef __TRACE_MEM_TYPES_H__
 #define __TRACE_MEM_TYPES_H__
 
-#include "mem-types.h"
+#include <glusterfs/mem-types.h>
 
 enum gf_trace_mem_types_ {
-        gf_trace_mt_trace_conf_t = gf_common_mt_end + 1,
-        gf_trace_mt_end
+    gf_trace_mt_trace_conf_t = gf_common_mt_end + 1,
+    gf_trace_mt_end
 };
 #endif
diff --git a/xlators/debug/trace/src/trace.c b/xlators/debug/trace/src/trace.c
index 88a41ae480f..6ed0ca00342 100644
--- a/xlators/debug/trace/src/trace.c
+++ b/xlators/debug/trace/src/trace.c
@@ -17,3282 +17,3518 @@
  *    their _cbk functions, which later passes the call to next layer.
  *    Very helpful translator for debugging.
  */
-#define TRACE_STAT_TO_STR(buf, str) trace_stat_to_str (buf, str, sizeof (str))
+#define TRACE_STAT_TO_STR(buf, str) trace_stat_to_str(buf, str, sizeof(str))
 
-static inline void
+static void
 trace_stat_to_str(struct iatt *buf, char *str, size_t len)
 {
-        char     atime_buf[256]    = {0,};
-        char     mtime_buf[256]    = {0,};
-        char     ctime_buf[256]    = {0,};
-
-        if (!buf)
-                return;
+    char atime_buf[GF_TIMESTR_SIZE] = {
+        0,
+    };
+    char mtime_buf[GF_TIMESTR_SIZE] = {
+        0,
+    };
+    char ctime_buf[GF_TIMESTR_SIZE] = {
+        0,
+    };
+
+    if (!buf)
+        return;
 
-        gf_time_fmt (atime_buf, sizeof atime_buf, buf->ia_atime,
-                     gf_timefmt_dirent);
+    gf_time_fmt(atime_buf, sizeof atime_buf, buf->ia_atime, gf_timefmt_dirent);
 
-        gf_time_fmt (mtime_buf, sizeof mtime_buf, buf->ia_mtime,
-                     gf_timefmt_dirent);
+    gf_time_fmt(mtime_buf, sizeof mtime_buf, buf->ia_mtime, gf_timefmt_dirent);
 
-        gf_time_fmt (ctime_buf, sizeof ctime_buf, buf->ia_ctime,
-                     gf_timefmt_dirent);
+    gf_time_fmt(ctime_buf, sizeof ctime_buf, buf->ia_ctime, gf_timefmt_dirent);
 
-        snprintf (str, len,
-                  "gfid=%s ino=%"PRIu64", mode=%o, "
-                  "nlink=%"GF_PRI_NLINK", uid=%u, "
-                  "gid=%u, size=%"PRIu64", "
-                  "blocks=%"PRIu64", atime=%s, "
-                  "mtime=%s, ctime=%s",
-                  uuid_utoa (buf->ia_gfid),
-                  buf->ia_ino,
-                  st_mode_from_ia (buf->ia_prot, buf->ia_type),
-                  buf->ia_nlink, buf->ia_uid,
-                  buf->ia_gid, buf->ia_size,
-                  buf->ia_blocks, atime_buf,
-                  mtime_buf, ctime_buf);
+    snprintf(str, len,
+             "gfid=%s ino=%" PRIu64
+             ", mode=%o, "
+             "nlink=%" GF_PRI_NLINK ", uid=%u, gid=%u, size=%" PRIu64
+             ", "
+             "blocks=%" PRIu64
+             ", atime=%s mtime=%s ctime=%s "
+             "atime_sec=%" PRId64 ", atime_nsec=%" PRIu32
+             ","
+             " mtime_sec=%" PRId64 ", mtime_nsec=%" PRIu32
+             ", "
+             "ctime_sec=%" PRId64 ", ctime_nsec=%" PRIu32 "",
+             uuid_utoa(buf->ia_gfid), buf->ia_ino,
+             st_mode_from_ia(buf->ia_prot, buf->ia_type), buf->ia_nlink,
+             buf->ia_uid, buf->ia_gid, buf->ia_size, buf->ia_blocks, atime_buf,
+             mtime_buf, ctime_buf, buf->ia_atime, buf->ia_atime_nsec,
+             buf->ia_mtime, buf->ia_mtime_nsec, buf->ia_ctime,
+             buf->ia_ctime_nsec);
 }
 
-
 int
-dump_history_trace (circular_buffer_t *cb, void *data)
+dump_history_trace(circular_buffer_t *cb, void *data)
 {
-        char     timestr[256] = {0,};
+    char timestr[GF_TIMESTR_SIZE] = {
+        0,
+    };
 
-        /* Since we are continuing with adding entries to the buffer even when
-           gettimeofday () fails, it's safe to check tm and then dump the time
-           at which the entry was added to the buffer */
+    /* Since we are continuing with adding entries to the buffer even when
+       gettimeofday () fails, it's safe to check tm and then dump the time
+       at which the entry was added to the buffer */
 
-        gf_time_fmt (timestr, sizeof timestr, cb->tv.tv_sec, gf_timefmt_Ymd_T);
-        snprintf (timestr + strlen (timestr), 256 - strlen (timestr),
-                  ".%"GF_PRI_SUSECONDS, cb->tv.tv_usec);
-        gf_proc_dump_write ("TIME", "%s", timestr);
+    gf_time_fmt_tv(timestr, sizeof timestr, &cb->tv, gf_timefmt_Ymd_T);
+    gf_proc_dump_write("TIME", "%s", timestr);
 
-        gf_proc_dump_write ("FOP", "%s\n", cb->data);
-
-        return 0;
-}
+    gf_proc_dump_write("FOP", "%s\n", (char *)cb->data);
 
-int
-trace_create_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                  int32_t op_ret, int32_t op_errno, fd_t *fd,
-                  inode_t *inode, struct iatt *buf,
-                  struct iatt *preparent, struct iatt *postparent,
-                  dict_t *xdata)
-{
-        char          statstr[4096]       = {0, };
-        char          preparentstr[4096]  = {0, };
-        char          postparentstr[4096] = {0, };
-        trace_conf_t  *conf               = NULL;
-
-        conf = this->private;
-
-        if (!conf->log_file && !conf->log_history)
-		goto out;
-        if (trace_fop_names[GF_FOP_CREATE].enabled) {
-                char  string[4096] = {0,};
-                if (op_ret >= 0) {
-                        TRACE_STAT_TO_STR (buf, statstr);
-                        TRACE_STAT_TO_STR (preparent, preparentstr);
-                        TRACE_STAT_TO_STR (postparent, postparentstr);
-
-                        snprintf (string, sizeof (string),
-                                  "%"PRId64": gfid=%s (op_ret=%d, fd=%p"
-                                  "*stbuf {%s}, *preparent {%s}, "
-                                  "*postparent = {%s})",
-                                  frame->root->unique,
-                                  uuid_utoa (inode->gfid), op_ret, fd,
-                                  statstr, preparentstr, postparentstr);
-
-                        /* for 'release' log */
-                        fd_ctx_set (fd, this, 0);
-                } else {
-                        snprintf (string, sizeof (string),
-                                  "%"PRId64": (op_ret=%d, op_errno=%d)",
-                                  frame->root->unique, op_ret,
-                                  op_errno);
-                }
-                LOG_ELEMENT (conf, string);
-        }
-out:
-        TRACE_STACK_UNWIND (create, frame, op_ret, op_errno, fd, inode, buf,
-                            preparent, postparent, xdata);
-        return 0;
+    return 0;
 }
 
 int
-trace_open_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                int32_t op_ret, int32_t op_errno, fd_t *fd, dict_t *xdata)
+trace_create_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                 int32_t op_ret, int32_t op_errno, fd_t *fd, inode_t *inode,
+                 struct iatt *buf, struct iatt *preparent,
+                 struct iatt *postparent, dict_t *xdata)
 {
-        trace_conf_t      *conf = NULL;
+    char statstr[1024] = {
+        0,
+    };
+    char preparentstr[1024] = {
+        0,
+    };
+    char postparentstr[1024] = {
+        0,
+    };
+    trace_conf_t *conf = NULL;
 
-        conf = this->private;
+    conf = this->private;
 
-        if (!conf->log_file && !conf->log_history)
-		goto out;
-        if (trace_fop_names[GF_FOP_OPEN].enabled) {
-                char     string[4096] = {0,};
-                snprintf (string, sizeof (string),
-                          "%"PRId64": gfid=%s op_ret=%d, op_errno=%d, "
-                          "*fd=%p", frame->root->unique,
-                          uuid_utoa (frame->local), op_ret, op_errno,
-                          fd);
+    if (!conf->log_file && !conf->log_history)
+        goto out;
+    if (trace_fop_names[GF_FOP_CREATE].enabled) {
+        char string[4096] = {
+            0,
+        };
+        if (op_ret >= 0) {
+            TRACE_STAT_TO_STR(buf, statstr);
+            TRACE_STAT_TO_STR(preparent, preparentstr);
+            TRACE_STAT_TO_STR(postparent, postparentstr);
 
-                LOG_ELEMENT (conf, string);
-        }
-
-out:
-        /* for 'release' log */
-        if (op_ret >= 0)
-                fd_ctx_set (fd, this, 0);
-
-        TRACE_STACK_UNWIND (open, frame, op_ret, op_errno, fd, xdata);
-        return 0;
-}
+            snprintf(string, sizeof(string),
+                     "%" PRId64
+                     ": gfid=%s (op_ret=%d, fd=%p"
+                     "*stbuf {%s}, *preparent {%s}, "
+                     "*postparent = {%s})",
+                     frame->root->unique, uuid_utoa(inode->gfid), op_ret, fd,
+                     statstr, preparentstr, postparentstr);
 
-int
-trace_stat_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                int32_t op_ret, int32_t op_errno, struct iatt *buf,
-                dict_t *xdata)
-{
-        char          statstr[4096] = {0, };
-        trace_conf_t  *conf         = NULL;
-
-        conf = this->private;
-
-        if (!conf->log_file && !conf->log_history)
-		goto out;
-        if (trace_fop_names[GF_FOP_STAT].enabled) {
-                char string[4096] = {0,};
-                if (op_ret == 0) {
-                        TRACE_STAT_TO_STR (buf, statstr);
-                        snprintf (string, sizeof (string),
-                                  "%"PRId64": gfid=%s op_ret=%d buf=%s",
-                                  frame->root->unique,
-                                  uuid_utoa (frame->local), op_ret,
-                                  statstr);
-                } else {
-                        snprintf (string, sizeof (string),
-                                  "%"PRId64": gfid=%s op_ret=%d, "
-                                  "op_errno=%d)",
-                                  frame->root->unique,
-                                  uuid_utoa (frame->local), op_ret,
-                                  op_errno);
-                }
-                LOG_ELEMENT (conf, string);
+            /* for 'release' log */
+            fd_ctx_set(fd, this, 0);
+        } else {
+            snprintf(string, sizeof(string),
+                     "%" PRId64 ": (op_ret=%d, op_errno=%d)",
+                     frame->root->unique, op_ret, op_errno);
         }
+        LOG_ELEMENT(conf, string);
+    }
 out:
-        TRACE_STACK_UNWIND (stat, frame, op_ret, op_errno, buf, xdata);
-        return 0;
+    TRACE_STACK_UNWIND(create, frame, op_ret, op_errno, fd, inode, buf,
+                       preparent, postparent, xdata);
+    return 0;
 }
 
 int
-trace_readv_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                 int32_t op_ret, int32_t op_errno, struct iovec *vector,
-                 int32_t count, struct iatt *buf, struct iobref *iobref,
-                 dict_t *xdata)
+trace_open_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+               int32_t op_ret, int32_t op_errno, fd_t *fd, dict_t *xdata)
 {
-        char          statstr[4096] = {0, };
-        trace_conf_t  *conf         = NULL;
-
-        conf = this->private;
-
-        if (!conf->log_file && !conf->log_history)
-		goto out;
-        if (trace_fop_names[GF_FOP_READ].enabled) {
-                char  string[4096] = {0,};
-                if (op_ret >= 0) {
-                        TRACE_STAT_TO_STR (buf, statstr);
-                        snprintf (string, sizeof (string),
-                                  "%"PRId64": gfid=%s op_ret=%d buf=%s",
-                                  frame->root->unique,
-                                  uuid_utoa (frame->local), op_ret,
-                                  statstr);
-                } else {
-                        snprintf (string, sizeof (string),
-                                  "%"PRId64": gfid=%s op_ret=%d, "
-                                  "op_errno=%d)",
-                                  frame->root->unique,
-                                  uuid_utoa (frame->local), op_ret,
-                                  op_errno);
-                }
-                LOG_ELEMENT (conf, string);
-        }
-out:
-        TRACE_STACK_UNWIND (readv, frame, op_ret, op_errno, vector, count,
-                            buf, iobref, xdata);
-        return 0;
-}
-
-int
-trace_writev_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                  int32_t op_ret, int32_t op_errno,
-                  struct iatt *prebuf, struct iatt *postbuf, dict_t *xdata)
-{
-        char         preopstr[4096]  = {0, };
-        char         postopstr[4096] = {0, };
-        trace_conf_t *conf           = NULL;
-
-        conf = this->private;
-
-        if (!conf->log_file && !conf->log_history)
-		goto out;
-        if (trace_fop_names[GF_FOP_WRITE].enabled) {
-                char  string[4096] = {0,};
-                if (op_ret >= 0) {
-                        TRACE_STAT_TO_STR (prebuf, preopstr);
-                        TRACE_STAT_TO_STR (postbuf, postopstr);
-
-                        snprintf (string, sizeof (string),
-                                  "%"PRId64": (op_ret=%d, "
-                                  "*prebuf = {%s}, *postbuf = {%s})",
-                                  frame->root->unique, op_ret,
-                                  preopstr, postopstr);
-                } else {
-                        snprintf (string, sizeof (string),
-                                  "%"PRId64": gfid=%s op_ret=%d, "
-                                  "op_errno=%d", frame->root->unique,
-                                  uuid_utoa (frame->local), op_ret,
-                                  op_errno);
-                }
-                LOG_ELEMENT (conf, string);
-        }
-out:
-        TRACE_STACK_UNWIND (writev, frame, op_ret, op_errno, prebuf, postbuf,
-                            xdata);
-        return 0;
-}
+    trace_conf_t *conf = NULL;
 
-int
-trace_readdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                   int32_t op_ret, int32_t op_errno, gf_dirent_t *buf,
-                   dict_t *xdata)
-{
-        trace_conf_t  *conf = NULL;
+    conf = this->private;
 
-        conf = this->private;
+    if (!conf->log_file && !conf->log_history)
+        goto out;
+    if (trace_fop_names[GF_FOP_OPEN].enabled) {
+        char string[4096] = {
+            0,
+        };
+        snprintf(string, sizeof(string),
+                 "%" PRId64
+                 ": gfid=%s op_ret=%d, op_errno=%d, "
+                 "*fd=%p",
+                 frame->root->unique, uuid_utoa(frame->local), op_ret, op_errno,
+                 fd);
 
-        if (!conf->log_file && !conf->log_history)
-		goto out;
-        if (trace_fop_names[GF_FOP_READDIR].enabled) {
-                char    string[4096]  = {0,};
-                snprintf (string, sizeof (string),
-                          "%"PRId64" : gfid=%s op_ret=%d, op_errno=%d",
-                          frame->root->unique, uuid_utoa (frame->local),
-                          op_ret, op_errno);
+        LOG_ELEMENT(conf, string);
+    }
 
-                LOG_ELEMENT (conf, string);
-        }
 out:
-        TRACE_STACK_UNWIND (readdir, frame, op_ret, op_errno, buf, xdata);
+    /* for 'release' log */
+    if (op_ret >= 0)
+        fd_ctx_set(fd, this, 0);
 
-        return 0;
+    TRACE_STACK_UNWIND(open, frame, op_ret, op_errno, fd, xdata);
+    return 0;
 }
 
 int
-trace_readdirp_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                    int32_t op_ret, int32_t op_errno, gf_dirent_t *buf,
-                    dict_t *xdata)
+trace_stat_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+               int32_t op_ret, int32_t op_errno, struct iatt *buf,
+               dict_t *xdata)
 {
-        trace_conf_t   *conf = NULL;
-
-        conf = this->private;
-
-        if (!conf->log_file && !conf->log_history)
-		goto out;
-        if (trace_fop_names[GF_FOP_READDIRP].enabled) {
-                char        string[4096] = {0,};
-                snprintf (string, sizeof (string),
-                          "%"PRId64" : gfid=%s op_ret=%d, op_errno=%d",
-                          frame->root->unique, uuid_utoa (frame->local),
-                          op_ret, op_errno);
-
-                LOG_ELEMENT (conf, string);
-        }
-
-out:
-        TRACE_STACK_UNWIND (readdirp, frame, op_ret, op_errno, buf, xdata);
-        return 0;
-}
-
-int
-trace_fsync_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                 int32_t op_ret, int32_t op_errno,
-                 struct iatt *prebuf, struct iatt *postbuf, dict_t *xdata)
+    char statstr[1024] = {
+        0,
+    };
+    trace_conf_t *conf = NULL;
+
+    conf = this->private;
+
+    if (!conf->log_file && !conf->log_history)
+        goto out;
+    if (trace_fop_names[GF_FOP_STAT].enabled) {
+        char string[4096] = {
+            0,
+        };
+        if (op_ret == 0) {
+            TRACE_STAT_TO_STR(buf, statstr);
+            (void)snprintf(
+                string, sizeof(string), "%" PRId64 ": gfid=%s op_ret=%d buf=%s",
+                frame->root->unique, uuid_utoa(frame->local), op_ret, statstr);
+        } else {
+            (void)snprintf(string, sizeof(string),
+                           "%" PRId64
+                           ": gfid=%s op_ret=%d, "
+                           "op_errno=%d)",
+                           frame->root->unique, uuid_utoa(frame->local), op_ret,
+                           op_errno);
+        }
+        LOG_ELEMENT(conf, string);
+    }
+out:
+    TRACE_STACK_UNWIND(stat, frame, op_ret, op_errno, buf, xdata);
+    return 0;
+}
+
+int
+trace_readv_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                int32_t op_ret, int32_t op_errno, struct iovec *vector,
+                int32_t count, struct iatt *buf, struct iobref *iobref,
+                dict_t *xdata)
 {
-        char          preopstr[4096]  = {0, };
-        char          postopstr[4096] = {0, };
-        trace_conf_t  *conf           = NULL;
-
-        conf = this->private;
-
-        if (!conf->log_file && !conf->log_history)
-		goto out;
-        if (trace_fop_names[GF_FOP_FSYNC].enabled) {
-                char  string[4096] = {0,};
-                if (op_ret == 0) {
-                        TRACE_STAT_TO_STR (prebuf, preopstr);
-                        TRACE_STAT_TO_STR (postbuf, postopstr);
-
-                        snprintf (string, sizeof (string),
-                                  "%"PRId64": (op_ret=%d, "
-                                  "*prebuf = {%s}, *postbuf = {%s}",
-                                  frame->root->unique, op_ret,
-                                  preopstr, postopstr);
-                } else {
-                        snprintf (string, sizeof (string),
-                                  "%"PRId64": gfid=%s op_ret=%d, "
-                                  "op_errno=%d", frame->root->unique,
-                                  uuid_utoa (frame->local), op_ret,
-                                  op_errno);
-
-                }
-                LOG_ELEMENT (conf, string);
-        }
-out:
-        TRACE_STACK_UNWIND (fsync, frame, op_ret, op_errno, prebuf, postbuf,
-                            xdata);
-
-        return 0;
-}
-
-int
-trace_setattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                   int32_t op_ret, int32_t op_errno,
-                   struct iatt *statpre, struct iatt *statpost, dict_t *xdata)
+    char statstr[1024] = {
+        0,
+    };
+    trace_conf_t *conf = NULL;
+
+    conf = this->private;
+
+    if (!conf->log_file && !conf->log_history)
+        goto out;
+    if (trace_fop_names[GF_FOP_READ].enabled) {
+        char string[4096] = {
+            0,
+        };
+        if (op_ret >= 0) {
+            TRACE_STAT_TO_STR(buf, statstr);
+            snprintf(
+                string, sizeof(string), "%" PRId64 ": gfid=%s op_ret=%d buf=%s",
+                frame->root->unique, uuid_utoa(frame->local), op_ret, statstr);
+        } else {
+            snprintf(string, sizeof(string),
+                     "%" PRId64
+                     ": gfid=%s op_ret=%d, "
+                     "op_errno=%d)",
+                     frame->root->unique, uuid_utoa(frame->local), op_ret,
+                     op_errno);
+        }
+        LOG_ELEMENT(conf, string);
+    }
+out:
+    TRACE_STACK_UNWIND(readv, frame, op_ret, op_errno, vector, count, buf,
+                       iobref, xdata);
+    return 0;
+}
+
+int
+trace_writev_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                 int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
+                 struct iatt *postbuf, dict_t *xdata)
+{
+    char preopstr[1024] = {
+        0,
+    };
+    char postopstr[1024] = {
+        0,
+    };
+    trace_conf_t *conf = NULL;
+
+    conf = this->private;
+
+    if (!conf->log_file && !conf->log_history)
+        goto out;
+    if (trace_fop_names[GF_FOP_WRITE].enabled) {
+        char string[4096] = {
+            0,
+        };
+        if (op_ret >= 0) {
+            TRACE_STAT_TO_STR(prebuf, preopstr);
+            TRACE_STAT_TO_STR(postbuf, postopstr);
+
+            snprintf(string, sizeof(string),
+                     "%" PRId64
+                     ": (op_ret=%d, "
+                     "*prebuf = {%s}, *postbuf = {%s})",
+                     frame->root->unique, op_ret, preopstr, postopstr);
+        } else {
+            snprintf(string, sizeof(string),
+                     "%" PRId64
+                     ": gfid=%s op_ret=%d, "
+                     "op_errno=%d",
+                     frame->root->unique, uuid_utoa(frame->local), op_ret,
+                     op_errno);
+        }
+        LOG_ELEMENT(conf, string);
+    }
+out:
+    TRACE_STACK_UNWIND(writev, frame, op_ret, op_errno, prebuf, postbuf, xdata);
+    return 0;
+}
+
+int
+trace_readdir_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                  int32_t op_ret, int32_t op_errno, gf_dirent_t *buf,
+                  dict_t *xdata)
 {
-        char          preopstr[4096]  = {0, };
-        char          postopstr[4096] = {0, };
-        trace_conf_t  *conf           = NULL;
+    trace_conf_t *conf = NULL;
 
-        conf = this->private;
+    conf = this->private;
 
-        if (!conf->log_file && !conf->log_history)
-		goto out;
-        if (trace_fop_names[GF_FOP_SETATTR].enabled) {
-                char  string[4096]  = {0,};
-                if (op_ret == 0) {
-                        TRACE_STAT_TO_STR (statpre, preopstr);
-                        TRACE_STAT_TO_STR (statpost, postopstr);
-
-                        snprintf (string, sizeof (string),
-                                  "%"PRId64": (op_ret=%d, "
-                                  "*prebuf = {%s}, *postbuf = {%s})",
-                                  frame->root->unique, op_ret,
-                                  preopstr, postopstr);
-                } else {
-                        snprintf (string, sizeof (string),
-                                  "%"PRId64": gfid=%s op_ret=%d, "
-                                  "op_errno=%d)", frame->root->unique,
-                                  uuid_utoa (frame->local), op_ret,
-                                  op_errno);
-                }
-                LOG_ELEMENT (conf, string);
-        }
-out:
-        TRACE_STACK_UNWIND (setattr, frame, op_ret, op_errno, statpre,
-                            statpost, xdata);
-        return 0;
-}
-
-int
-trace_fsetattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                    int32_t op_ret, int32_t op_errno,
-                    struct iatt *statpre, struct iatt *statpost, dict_t *xdata)
-{
-        char          preopstr[4096]  = {0, };
-        char          postopstr[4096] = {0, };
-        trace_conf_t  *conf           = NULL;
-
-        conf = this->private;
-
-        if (!conf->log_file && !conf->log_history)
-		goto out;
-        if (trace_fop_names[GF_FOP_FSETATTR].enabled) {
-                char  string[4096] = {0,};
-                if (op_ret == 0) {
-                        TRACE_STAT_TO_STR (statpre, preopstr);
-                        TRACE_STAT_TO_STR (statpost, postopstr);
-
-                        snprintf (string, sizeof (string),
-                                  "%"PRId64": (op_ret=%d, "
-                                  "*prebuf = {%s}, *postbuf = {%s})",
-                                  frame->root->unique, op_ret,
-                                  preopstr, postopstr);
-                } else {
-                        snprintf (string, sizeof (string),
-                                  "%"PRId64": gfid=%s op_ret=%d, op_errno=%d)",
-                                  frame->root->unique, uuid_utoa (frame->local),
-                                  op_ret, op_errno);
-                }
-                LOG_ELEMENT (conf, string);
-        }
-out:
-        TRACE_STACK_UNWIND (fsetattr, frame, op_ret, op_errno,
-                            statpre, statpost, xdata);
-        return 0;
-}
+    if (!conf->log_file && !conf->log_history)
+        goto out;
+    if (trace_fop_names[GF_FOP_READDIR].enabled) {
+        char string[4096] = {
+            0,
+        };
+        snprintf(string, sizeof(string),
+                 "%" PRId64 " : gfid=%s op_ret=%d, op_errno=%d",
+                 frame->root->unique, uuid_utoa(frame->local), op_ret,
+                 op_errno);
 
-int
-trace_unlink_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                  int32_t op_ret, int32_t op_errno,
-                  struct iatt *preparent, struct iatt *postparent,
-                  dict_t *xdata)
-{
-        char          preparentstr[4096]  = {0, };
-        char          postparentstr[4096] = {0, };
-        trace_conf_t  *conf               = NULL;
-
-        conf = this->private;
-
-        if (!conf->log_file && !conf->log_history)
-		goto out;
-        if (trace_fop_names[GF_FOP_UNLINK].enabled) {
-                char string[4096] = {0,};
-                if (op_ret == 0) {
-                        TRACE_STAT_TO_STR (preparent, preparentstr);
-                        TRACE_STAT_TO_STR (postparent, postparentstr);
-
-                        snprintf (string, sizeof (string),
-                                  "%"PRId64": gfid=%s op_ret=%d, "
-                                  " *preparent = {%s}, "
-                                  "*postparent = {%s})",
-                                  frame->root->unique,
-                                  uuid_utoa (frame->local),
-                                  op_ret, preparentstr,
-                                  postparentstr);
-                } else {
-                        snprintf (string, sizeof (string),
-                                  "%"PRId64": gfid=%s op_ret=%d, "
-                                  "op_errno=%d)",
-                                  frame->root->unique,
-                                  uuid_utoa (frame->local), op_ret,
-                                  op_errno);
-                }
-                LOG_ELEMENT (conf, string);
-        }
+        LOG_ELEMENT(conf, string);
+    }
 out:
-        TRACE_STACK_UNWIND (unlink, frame, op_ret, op_errno,
-                            preparent, postparent, xdata);
-        return 0;
-}
+    TRACE_STACK_UNWIND(readdir, frame, op_ret, op_errno, buf, xdata);
 
-int
-trace_rename_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                  int32_t op_ret, int32_t op_errno, struct iatt *buf,
-                  struct iatt *preoldparent, struct iatt *postoldparent,
-                  struct iatt *prenewparent, struct iatt *postnewparent,
-                  dict_t *xdata)
-{
-        char           statstr[4096]          = {0, };
-        char           preoldparentstr[4096]  = {0, };
-        char           postoldparentstr[4096] = {0, };
-        char           prenewparentstr[4096]  = {0, };
-        char           postnewparentstr[4096] = {0, };
-        trace_conf_t   *conf                  = NULL;
-
-        conf = this->private;
-
-        if (!conf->log_file && !conf->log_history)
-		goto out;
-        if (trace_fop_names[GF_FOP_RENAME].enabled) {
-                char  string[4096] = {0,};
-                if (op_ret == 0) {
-                        TRACE_STAT_TO_STR (buf, statstr);
-                        TRACE_STAT_TO_STR (preoldparent, preoldparentstr);
-                        TRACE_STAT_TO_STR (postoldparent, postoldparentstr);
-                        TRACE_STAT_TO_STR (prenewparent, prenewparentstr);
-                        TRACE_STAT_TO_STR (postnewparent, postnewparentstr);
-
-                        snprintf (string, sizeof (string),
-                                  "%"PRId64": (op_ret=%d, "
-                                  "*stbuf = {%s}, *preoldparent = {%s},"
-                                  " *postoldparent = {%s}"
-                                  " *prenewparent = {%s}, "
-                                  "*postnewparent = {%s})",
-                                  frame->root->unique, op_ret, statstr,
-                                  preoldparentstr, postoldparentstr,
-                                  prenewparentstr, postnewparentstr);
-                } else {
-                        snprintf (string, sizeof (string),
-                                  "%"PRId64": gfid=%s op_ret=%d, "
-                                  "op_errno=%d", frame->root->unique,
-                                  uuid_utoa (frame->local),
-                                  op_ret, op_errno);
-
-                }
-                LOG_ELEMENT (conf, string);
-        }
-out:
-        TRACE_STACK_UNWIND (rename, frame, op_ret, op_errno, buf,
-                            preoldparent, postoldparent,
-                            prenewparent, postnewparent, xdata);
-        return 0;
-}
-
-int
-trace_readlink_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                    int32_t op_ret, int32_t op_errno,
-                    const char *buf, struct iatt *stbuf, dict_t *xdata)
-{
-        char          statstr[4096] = {0, };
-        trace_conf_t  *conf         = NULL;
-
-        conf = this->private;
-
-        if (!conf->log_file && !conf->log_history)
-		goto out;
-        if (trace_fop_names[GF_FOP_READLINK].enabled) {
-                char string[4096] = {0,};
-                if (op_ret == 0) {
-                        TRACE_STAT_TO_STR (stbuf, statstr);
-                        snprintf (string, sizeof (string),
-                                  "%"PRId64": (op_ret=%d, op_errno=%d,"
-                                  "buf=%s, stbuf = { %s })",
-                                  frame->root->unique, op_ret, op_errno,
-                                  buf, statstr);
-                } else {
-                        snprintf (string, sizeof (string),
-                                  "%"PRId64": gfid=%s op_ret=%d, "
-                                  "op_errno=%d",
-                                  frame->root->unique,
-                                  uuid_utoa (frame->local), op_ret,
-                                  op_errno);
-                }
-
-                LOG_ELEMENT (conf, string);
-        }
-out:
-        TRACE_STACK_UNWIND (readlink, frame, op_ret, op_errno, buf, stbuf,
-                            xdata);
-        return 0;
-}
-
-int
-trace_lookup_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                  int32_t op_ret, int32_t op_errno,
-                  inode_t *inode, struct iatt *buf,
-                  dict_t *xdata, struct iatt *postparent)
-{
-        char          statstr[4096]       = {0, };
-        char          postparentstr[4096] = {0, };
-        trace_conf_t  *conf               = NULL;
-
-        conf = this->private;
-
-        if (!conf->log_file && !conf->log_history)
-		goto out;
-        if (trace_fop_names[GF_FOP_LOOKUP].enabled) {
-                char  string[4096] = {0,};
-                if (op_ret == 0) {
-                        TRACE_STAT_TO_STR (buf, statstr);
-                        TRACE_STAT_TO_STR (postparent, postparentstr);
-                        /* print buf->ia_gfid instead of inode->gfid,
-                         * since if the inode is not yet linked to the
-                         * inode table (fresh lookup) then null gfid
-                         * will be printed.
-                         */
-                        snprintf (string, sizeof (string),
-                                  "%"PRId64": gfid=%s (op_ret=%d "
-                                  "*buf {%s}, *postparent {%s}",
-                                  frame->root->unique,
-                                  uuid_utoa (buf->ia_gfid),
-                                  op_ret, statstr, postparentstr);
-
-                        /* For 'forget' */
-                        inode_ctx_put (inode, this, 0);
-                } else {
-                        snprintf (string, sizeof (string),
-                                  "%"PRId64": gfid=%s op_ret=%d, "
-                                  "op_errno=%d)",
-                                  frame->root->unique,
-                                  uuid_utoa (frame->local), op_ret,
-                                  op_errno);
-                }
-                LOG_ELEMENT (conf, string);
-        }
-out:
-        TRACE_STACK_UNWIND (lookup, frame, op_ret, op_errno, inode, buf,
-                            xdata, postparent);
-        return 0;
+    return 0;
 }
 
 int
-trace_symlink_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                   int32_t op_ret, int32_t op_errno,
-                   inode_t *inode, struct iatt *buf,
-                   struct iatt *preparent, struct iatt *postparent,
+trace_readdirp_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                   int32_t op_ret, int32_t op_errno, gf_dirent_t *buf,
                    dict_t *xdata)
 {
-        char          statstr[4096]       = {0, };
-        char          preparentstr[4096]  = {0, };
-        char          postparentstr[4096] = {0, };
-        trace_conf_t  *conf               = NULL;
-
-        conf = this->private;
-
-        if (!conf->log_file && !conf->log_history)
-		goto out;
-        if (trace_fop_names[GF_FOP_SYMLINK].enabled) {
-                char  string[4096] = {0,};
-                if (op_ret == 0) {
-                        TRACE_STAT_TO_STR (buf, statstr);
-                        TRACE_STAT_TO_STR (preparent, preparentstr);
-                        TRACE_STAT_TO_STR (postparent, postparentstr);
-
-                        snprintf (string, sizeof (string),
-                                  "%"PRId64": gfid=%s (op_ret=%d "
-                                  "*stbuf = {%s}, *preparent = {%s}, "
-                                  "*postparent = {%s})",
-                                  frame->root->unique,
-                                  uuid_utoa (inode->gfid),
-                                  op_ret, statstr, preparentstr,
-                                  postparentstr);
-                } else {
-                        snprintf (string, sizeof (string),
-                                  "%"PRId64": op_ret=%d, op_errno=%d",
-                                  frame->root->unique, op_ret,
-                                  op_errno);
-                }
-                LOG_ELEMENT (conf, string);
-        }
-out:
-        TRACE_STACK_UNWIND (symlink, frame, op_ret, op_errno, inode, buf,
-                            preparent, postparent, xdata);
-        return 0;
-}
-
-int
-trace_mknod_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                 int32_t op_ret, int32_t op_errno,
-                 inode_t *inode, struct iatt *buf,
-                 struct iatt *preparent, struct iatt *postparent, dict_t *xdata)
-{
-        char          statstr[4096]       = {0, };
-        char          preparentstr[4096]  = {0, };
-        char          postparentstr[4096] = {0, };
-        trace_conf_t  *conf               = NULL;
-
-        conf = this->private;
-
-        if (!conf->log_file && !conf->log_history)
-		goto out;
-        char string[4096]  = {0,};
-        if (trace_fop_names[GF_FOP_MKNOD].enabled) {
-                if (op_ret == 0) {
-                        TRACE_STAT_TO_STR (buf, statstr);
-                        TRACE_STAT_TO_STR (preparent, preparentstr);
-                        TRACE_STAT_TO_STR (postparent, postparentstr);
-
-                        snprintf (string, sizeof (string),
-                                  "%"PRId64": gfid=%s (op_ret=%d "
-                                  "*stbuf = {%s}, *preparent = {%s}, "
-                                  "*postparent = {%s})",
-                                  frame->root->unique,
-                                  uuid_utoa (inode->gfid),
-                                  op_ret, statstr, preparentstr,
-                                  postparentstr);
-                } else {
-                        snprintf (string, sizeof (string),
-                                  "%"PRId64": (op_ret=%d, op_errno=%d)",
-                                  frame->root->unique, op_ret,
-                                  op_errno);
-                }
-                LOG_ELEMENT (conf, string);
-        }
-out:
-        TRACE_STACK_UNWIND (mknod, frame, op_ret, op_errno, inode, buf,
-                            preparent, postparent, xdata);
-        return 0;
-}
-
-int
-trace_mkdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                 int32_t op_ret, int32_t op_errno,
-                 inode_t *inode, struct iatt *buf,
-                 struct iatt *preparent, struct iatt *postparent, dict_t *xdata)
-{
-        char          statstr[4096]       = {0, };
-        char          preparentstr[4096]  = {0, };
-        char          postparentstr[4096] = {0, };
-        trace_conf_t  *conf               = NULL;
-
-        conf = this->private;
-
-        if (!conf->log_file && !conf->log_history)
-		goto out;
-        if (trace_fop_names[GF_FOP_MKDIR].enabled) {
-                char  string[4096]  = {0,};
-                if (op_ret == 0) {
-                        TRACE_STAT_TO_STR (buf, statstr);
-                        TRACE_STAT_TO_STR (preparent, preparentstr);
-                        TRACE_STAT_TO_STR (postparent, postparentstr);
-
-                        snprintf (string, sizeof (string),
-                                  "%"PRId64": gfid=%s (op_ret=%d "
-                                  ", *stbuf = {%s}, *prebuf = {%s}, "
-                                  "*postbuf = {%s} )",
-                                  frame->root->unique,
-                                  uuid_utoa (inode->gfid),
-                                  op_ret, statstr, preparentstr,
-                                  postparentstr);
-                } else {
-                        snprintf (string, sizeof (string),
-                                  "%"PRId64": (op_ret=%d, op_errno=%d)",
-                                  frame->root->unique, op_ret,
-                                  op_errno);
-                }
-                LOG_ELEMENT (conf, string);
-        }
-out:
-        TRACE_STACK_UNWIND (mkdir, frame, op_ret, op_errno, inode, buf,
-                            preparent, postparent, xdata);
-        return 0;
-}
-
-int
-trace_link_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                int32_t op_ret, int32_t op_errno,
-                inode_t *inode, struct iatt *buf,
-                struct iatt *preparent, struct iatt *postparent, dict_t *xdata)
-{
-        char          statstr[4096]       = {0, };
-        char          preparentstr[4096]  = {0, };
-        char          postparentstr[4096] = {0, };
-        trace_conf_t  *conf               = NULL;
-
-        conf = this->private;
-
-        if (!conf->log_file && !conf->log_history)
-		goto out;
-        char  string[4096]  = {0,};
-        if (trace_fop_names[GF_FOP_LINK].enabled) {
-                if (op_ret == 0) {
-                        TRACE_STAT_TO_STR (buf, statstr);
-                        TRACE_STAT_TO_STR (preparent, preparentstr);
-                        TRACE_STAT_TO_STR (postparent, postparentstr);
-
-                        snprintf (string, sizeof (string),
-                                  "%"PRId64": (op_ret=%d, "
-                                  "*stbuf = {%s},  *prebuf = {%s},"
-                                  " *postbuf = {%s})",
-                                  frame->root->unique, op_ret,
-                                  statstr, preparentstr, postparentstr);
-                } else {
-                        snprintf (string, sizeof (string),
-                                  "%"PRId64": gfid=%s op_ret=%d, "
-                                  "op_errno=%d",
-                                  frame->root->unique,
-                                  uuid_utoa (frame->local),
-                                  op_ret, op_errno);
-                }
-                LOG_ELEMENT (conf, string);
-        }
-out:
-        TRACE_STACK_UNWIND (link, frame, op_ret, op_errno, inode, buf,
-                            preparent, postparent, xdata);
-        return 0;
-}
-
-int
-trace_flush_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                 int32_t op_ret, int32_t op_errno, dict_t *xdata)
+    int count = 0;
+    char statstr[1024] = {
+        0,
+    };
+    char string[4096] = {
+        0,
+    };
+    trace_conf_t *conf = NULL;
+    gf_dirent_t *entry = NULL;
+
+    conf = this->private;
+
+    if (!conf->log_file && !conf->log_history)
+        goto out;
+    if (trace_fop_names[GF_FOP_READDIRP].enabled) {
+        snprintf(string, sizeof(string),
+                 "%" PRId64 " : gfid=%s op_ret=%d, op_errno=%d",
+                 frame->root->unique, uuid_utoa(frame->local), op_ret,
+                 op_errno);
+
+        LOG_ELEMENT(conf, string);
+    }
+    if (op_ret < 0)
+        goto out;
+
+    list_for_each_entry(entry, &buf->list, list)
+    {
+        count++;
+        TRACE_STAT_TO_STR(&entry->d_stat, statstr);
+        snprintf(string, sizeof(string),
+                 "entry no. %d, pargfid=%s, "
+                 "bname=%s *buf {%s}",
+                 count, uuid_utoa(frame->local), entry->d_name, statstr);
+        LOG_ELEMENT(conf, string);
+    }
+
+out:
+    TRACE_STACK_UNWIND(readdirp, frame, op_ret, op_errno, buf, xdata);
+    return 0;
+}
+
+int
+trace_fsync_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
+                struct iatt *postbuf, dict_t *xdata)
+{
+    char preopstr[1024] = {
+        0,
+    };
+    char postopstr[1024] = {
+        0,
+    };
+    trace_conf_t *conf = NULL;
+
+    conf = this->private;
+
+    if (!conf->log_file && !conf->log_history)
+        goto out;
+    if (trace_fop_names[GF_FOP_FSYNC].enabled) {
+        char string[4096] = {
+            0,
+        };
+        if (op_ret == 0) {
+            TRACE_STAT_TO_STR(prebuf, preopstr);
+            TRACE_STAT_TO_STR(postbuf, postopstr);
+
+            snprintf(string, sizeof(string),
+                     "%" PRId64
+                     ": (op_ret=%d, "
+                     "*prebuf = {%s}, *postbuf = {%s}",
+                     frame->root->unique, op_ret, preopstr, postopstr);
+        } else {
+            snprintf(string, sizeof(string),
+                     "%" PRId64
+                     ": gfid=%s op_ret=%d, "
+                     "op_errno=%d",
+                     frame->root->unique, uuid_utoa(frame->local), op_ret,
+                     op_errno);
+        }
+        LOG_ELEMENT(conf, string);
+    }
+out:
+    TRACE_STACK_UNWIND(fsync, frame, op_ret, op_errno, prebuf, postbuf, xdata);
+
+    return 0;
+}
+
+int
+trace_setattr_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                  int32_t op_ret, int32_t op_errno, struct iatt *statpre,
+                  struct iatt *statpost, dict_t *xdata)
+{
+    char preopstr[1024] = {
+        0,
+    };
+    char postopstr[1024] = {
+        0,
+    };
+    trace_conf_t *conf = NULL;
+
+    conf = this->private;
+
+    if (!conf->log_file && !conf->log_history)
+        goto out;
+    if (trace_fop_names[GF_FOP_SETATTR].enabled) {
+        char string[4096] = {
+            0,
+        };
+        if (op_ret == 0) {
+            TRACE_STAT_TO_STR(statpre, preopstr);
+            TRACE_STAT_TO_STR(statpost, postopstr);
+
+            snprintf(string, sizeof(string),
+                     "%" PRId64
+                     ": (op_ret=%d, "
+                     "*prebuf = {%s}, *postbuf = {%s})",
+                     frame->root->unique, op_ret, preopstr, postopstr);
+        } else {
+            snprintf(string, sizeof(string),
+                     "%" PRId64
+                     ": gfid=%s op_ret=%d, "
+                     "op_errno=%d)",
+                     frame->root->unique, uuid_utoa(frame->local), op_ret,
+                     op_errno);
+        }
+        LOG_ELEMENT(conf, string);
+    }
+out:
+    TRACE_STACK_UNWIND(setattr, frame, op_ret, op_errno, statpre, statpost,
+                       xdata);
+    return 0;
+}
+
+int
+trace_fsetattr_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                   int32_t op_ret, int32_t op_errno, struct iatt *statpre,
+                   struct iatt *statpost, dict_t *xdata)
+{
+    char preopstr[1024] = {
+        0,
+    };
+    char postopstr[1024] = {
+        0,
+    };
+    trace_conf_t *conf = NULL;
+
+    conf = this->private;
+
+    if (!conf->log_file && !conf->log_history)
+        goto out;
+    if (trace_fop_names[GF_FOP_FSETATTR].enabled) {
+        char string[4096] = {
+            0,
+        };
+        if (op_ret == 0) {
+            TRACE_STAT_TO_STR(statpre, preopstr);
+            TRACE_STAT_TO_STR(statpost, postopstr);
+
+            snprintf(string, sizeof(string),
+                     "%" PRId64
+                     ": (op_ret=%d, "
+                     "*prebuf = {%s}, *postbuf = {%s})",
+                     frame->root->unique, op_ret, preopstr, postopstr);
+        } else {
+            snprintf(string, sizeof(string),
+                     "%" PRId64 ": gfid=%s op_ret=%d, op_errno=%d)",
+                     frame->root->unique, uuid_utoa(frame->local), op_ret,
+                     op_errno);
+        }
+        LOG_ELEMENT(conf, string);
+    }
+out:
+    TRACE_STACK_UNWIND(fsetattr, frame, op_ret, op_errno, statpre, statpost,
+                       xdata);
+    return 0;
+}
+
+int
+trace_unlink_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                 int32_t op_ret, int32_t op_errno, struct iatt *preparent,
+                 struct iatt *postparent, dict_t *xdata)
+{
+    char preparentstr[1024] = {
+        0,
+    };
+    char postparentstr[1024] = {
+        0,
+    };
+    trace_conf_t *conf = NULL;
+
+    conf = this->private;
+
+    if (!conf->log_file && !conf->log_history)
+        goto out;
+    if (trace_fop_names[GF_FOP_UNLINK].enabled) {
+        char string[4096] = {
+            0,
+        };
+        if (op_ret == 0) {
+            TRACE_STAT_TO_STR(preparent, preparentstr);
+            TRACE_STAT_TO_STR(postparent, postparentstr);
+
+            snprintf(string, sizeof(string),
+                     "%" PRId64
+                     ": gfid=%s op_ret=%d, "
+                     " *preparent = {%s}, "
+                     "*postparent = {%s})",
+                     frame->root->unique, uuid_utoa(frame->local), op_ret,
+                     preparentstr, postparentstr);
+        } else {
+            snprintf(string, sizeof(string),
+                     "%" PRId64
+                     ": gfid=%s op_ret=%d, "
+                     "op_errno=%d)",
+                     frame->root->unique, uuid_utoa(frame->local), op_ret,
+                     op_errno);
+        }
+        LOG_ELEMENT(conf, string);
+    }
+out:
+    TRACE_STACK_UNWIND(unlink, frame, op_ret, op_errno, preparent, postparent,
+                       xdata);
+    return 0;
+}
+
+int
+trace_rename_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                 int32_t op_ret, int32_t op_errno, struct iatt *buf,
+                 struct iatt *preoldparent, struct iatt *postoldparent,
+                 struct iatt *prenewparent, struct iatt *postnewparent,
+                 dict_t *xdata)
+{
+    char statstr[1024] = {
+        0,
+    };
+    char preoldparentstr[1024] = {
+        0,
+    };
+    char postoldparentstr[1024] = {
+        0,
+    };
+    char prenewparentstr[1024] = {
+        0,
+    };
+    char postnewparentstr[1024] = {
+        0,
+    };
+    trace_conf_t *conf = NULL;
+
+    conf = this->private;
+
+    if (!conf->log_file && !conf->log_history)
+        goto out;
+    if (trace_fop_names[GF_FOP_RENAME].enabled) {
+        char string[6044] = {
+            0,
+        };
+        if (op_ret == 0) {
+            TRACE_STAT_TO_STR(buf, statstr);
+            TRACE_STAT_TO_STR(preoldparent, preoldparentstr);
+            TRACE_STAT_TO_STR(postoldparent, postoldparentstr);
+            TRACE_STAT_TO_STR(prenewparent, prenewparentstr);
+            TRACE_STAT_TO_STR(postnewparent, postnewparentstr);
+
+            snprintf(string, sizeof(string),
+                     "%" PRId64
+                     ": (op_ret=%d, "
+                     "*stbuf = {%s}, *preoldparent = {%s},"
+                     " *postoldparent = {%s}"
+                     " *prenewparent = {%s}, "
+                     "*postnewparent = {%s})",
+                     frame->root->unique, op_ret, statstr, preoldparentstr,
+                     postoldparentstr, prenewparentstr, postnewparentstr);
+        } else {
+            snprintf(string, sizeof(string),
+                     "%" PRId64
+                     ": gfid=%s op_ret=%d, "
+                     "op_errno=%d",
+                     frame->root->unique, uuid_utoa(frame->local), op_ret,
+                     op_errno);
+        }
+        LOG_ELEMENT(conf, string);
+    }
+out:
+    TRACE_STACK_UNWIND(rename, frame, op_ret, op_errno, buf, preoldparent,
+                       postoldparent, prenewparent, postnewparent, xdata);
+    return 0;
+}
+
+int
+trace_readlink_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                   int32_t op_ret, int32_t op_errno, const char *buf,
+                   struct iatt *stbuf, dict_t *xdata)
+{
+    char statstr[1024] = {
+        0,
+    };
+    trace_conf_t *conf = NULL;
+
+    conf = this->private;
+
+    if (!conf->log_file && !conf->log_history)
+        goto out;
+    if (trace_fop_names[GF_FOP_READLINK].enabled) {
+        char string[4096] = {
+            0,
+        };
+        if (op_ret == 0) {
+            TRACE_STAT_TO_STR(stbuf, statstr);
+            snprintf(string, sizeof(string),
+                     "%" PRId64
+                     ": (op_ret=%d, op_errno=%d,"
+                     "buf=%s, stbuf = { %s })",
+                     frame->root->unique, op_ret, op_errno, buf, statstr);
+        } else {
+            snprintf(string, sizeof(string),
+                     "%" PRId64
+                     ": gfid=%s op_ret=%d, "
+                     "op_errno=%d",
+                     frame->root->unique, uuid_utoa(frame->local), op_ret,
+                     op_errno);
+        }
+
+        LOG_ELEMENT(conf, string);
+    }
+out:
+    TRACE_STACK_UNWIND(readlink, frame, op_ret, op_errno, buf, stbuf, xdata);
+    return 0;
+}
+
+int
+trace_lookup_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                 int32_t op_ret, int32_t op_errno, inode_t *inode,
+                 struct iatt *buf, dict_t *xdata, struct iatt *postparent)
+{
+    char statstr[1024] = {
+        0,
+    };
+    char postparentstr[1024] = {
+        0,
+    };
+    trace_conf_t *conf = NULL;
+
+    conf = this->private;
+
+    if (!conf->log_file && !conf->log_history)
+        goto out;
+    if (trace_fop_names[GF_FOP_LOOKUP].enabled) {
+        char string[4096] = {
+            0,
+        };
+        if (op_ret == 0) {
+            TRACE_STAT_TO_STR(buf, statstr);
+            TRACE_STAT_TO_STR(postparent, postparentstr);
+            /* print buf->ia_gfid instead of inode->gfid,
+             * since if the inode is not yet linked to the
+             * inode table (fresh lookup) then null gfid
+             * will be printed.
+             */
+            snprintf(string, sizeof(string),
+                     "%" PRId64
+                     ": gfid=%s (op_ret=%d "
+                     "*buf {%s}, *postparent {%s}",
+                     frame->root->unique, uuid_utoa(buf->ia_gfid), op_ret,
+                     statstr, postparentstr);
+
+            /* For 'forget' */
+            inode_ctx_put(inode, this, 0);
+        } else {
+            snprintf(string, sizeof(string),
+                     "%" PRId64
+                     ": gfid=%s op_ret=%d, "
+                     "op_errno=%d)",
+                     frame->root->unique, uuid_utoa(frame->local), op_ret,
+                     op_errno);
+        }
+        LOG_ELEMENT(conf, string);
+    }
+out:
+    TRACE_STACK_UNWIND(lookup, frame, op_ret, op_errno, inode, buf, xdata,
+                       postparent);
+    return 0;
+}
+
+int
+trace_symlink_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                  int32_t op_ret, int32_t op_errno, inode_t *inode,
+                  struct iatt *buf, struct iatt *preparent,
+                  struct iatt *postparent, dict_t *xdata)
+{
+    char statstr[1024] = {
+        0,
+    };
+    char preparentstr[1024] = {
+        0,
+    };
+    char postparentstr[1024] = {
+        0,
+    };
+    trace_conf_t *conf = NULL;
+
+    conf = this->private;
+
+    if (!conf->log_file && !conf->log_history)
+        goto out;
+    if (trace_fop_names[GF_FOP_SYMLINK].enabled) {
+        char string[4096] = {
+            0,
+        };
+        if (op_ret == 0) {
+            TRACE_STAT_TO_STR(buf, statstr);
+            TRACE_STAT_TO_STR(preparent, preparentstr);
+            TRACE_STAT_TO_STR(postparent, postparentstr);
+
+            snprintf(string, sizeof(string),
+                     "%" PRId64
+                     ": gfid=%s (op_ret=%d "
+                     "*stbuf = {%s}, *preparent = {%s}, "
+                     "*postparent = {%s})",
+                     frame->root->unique, uuid_utoa(inode->gfid), op_ret,
+                     statstr, preparentstr, postparentstr);
+        } else {
+            snprintf(string, sizeof(string),
+                     "%" PRId64 ": op_ret=%d, op_errno=%d", frame->root->unique,
+                     op_ret, op_errno);
+        }
+        LOG_ELEMENT(conf, string);
+    }
+out:
+    TRACE_STACK_UNWIND(symlink, frame, op_ret, op_errno, inode, buf, preparent,
+                       postparent, xdata);
+    return 0;
+}
+
+int
+trace_mknod_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                int32_t op_ret, int32_t op_errno, inode_t *inode,
+                struct iatt *buf, struct iatt *preparent,
+                struct iatt *postparent, dict_t *xdata)
+{
+    char statstr[1024] = {
+        0,
+    };
+    char preparentstr[1024] = {
+        0,
+    };
+    char postparentstr[1024] = {
+        0,
+    };
+    trace_conf_t *conf = NULL;
+
+    conf = this->private;
+
+    if (!conf->log_file && !conf->log_history)
+        goto out;
+    char string[4096] = {
+        0,
+    };
+    if (trace_fop_names[GF_FOP_MKNOD].enabled) {
+        if (op_ret == 0) {
+            TRACE_STAT_TO_STR(buf, statstr);
+            TRACE_STAT_TO_STR(preparent, preparentstr);
+            TRACE_STAT_TO_STR(postparent, postparentstr);
+
+            snprintf(string, sizeof(string),
+                     "%" PRId64
+                     ": gfid=%s (op_ret=%d "
+                     "*stbuf = {%s}, *preparent = {%s}, "
+                     "*postparent = {%s})",
+                     frame->root->unique, uuid_utoa(inode->gfid), op_ret,
+                     statstr, preparentstr, postparentstr);
+        } else {
+            snprintf(string, sizeof(string),
+                     "%" PRId64 ": (op_ret=%d, op_errno=%d)",
+                     frame->root->unique, op_ret, op_errno);
+        }
+        LOG_ELEMENT(conf, string);
+    }
+out:
+    TRACE_STACK_UNWIND(mknod, frame, op_ret, op_errno, inode, buf, preparent,
+                       postparent, xdata);
+    return 0;
+}
+
+int
+trace_mkdir_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                int32_t op_ret, int32_t op_errno, inode_t *inode,
+                struct iatt *buf, struct iatt *preparent,
+                struct iatt *postparent, dict_t *xdata)
+{
+    char statstr[1024] = {
+        0,
+    };
+    char preparentstr[1024] = {
+        0,
+    };
+    char postparentstr[1024] = {
+        0,
+    };
+    trace_conf_t *conf = NULL;
+
+    conf = this->private;
+
+    if (!conf->log_file && !conf->log_history)
+        goto out;
+    if (trace_fop_names[GF_FOP_MKDIR].enabled) {
+        char string[4096] = {
+            0,
+        };
+        if (op_ret == 0) {
+            TRACE_STAT_TO_STR(buf, statstr);
+            TRACE_STAT_TO_STR(preparent, preparentstr);
+            TRACE_STAT_TO_STR(postparent, postparentstr);
+
+            snprintf(string, sizeof(string),
+                     "%" PRId64
+                     ": gfid=%s (op_ret=%d "
+                     ", *stbuf = {%s}, *prebuf = {%s}, "
+                     "*postbuf = {%s} )",
+                     frame->root->unique, uuid_utoa(inode->gfid), op_ret,
+                     statstr, preparentstr, postparentstr);
+        } else {
+            snprintf(string, sizeof(string),
+                     "%" PRId64 ": (op_ret=%d, op_errno=%d)",
+                     frame->root->unique, op_ret, op_errno);
+        }
+        LOG_ELEMENT(conf, string);
+    }
+out:
+    TRACE_STACK_UNWIND(mkdir, frame, op_ret, op_errno, inode, buf, preparent,
+                       postparent, xdata);
+    return 0;
+}
+
+int
+trace_link_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+               int32_t op_ret, int32_t op_errno, inode_t *inode,
+               struct iatt *buf, struct iatt *preparent,
+               struct iatt *postparent, dict_t *xdata)
+{
+    char statstr[1024] = {
+        0,
+    };
+    char preparentstr[1024] = {
+        0,
+    };
+    char postparentstr[1024] = {
+        0,
+    };
+    trace_conf_t *conf = NULL;
+
+    conf = this->private;
+
+    if (!conf->log_file && !conf->log_history)
+        goto out;
+    char string[4096] = {
+        0,
+    };
+    if (trace_fop_names[GF_FOP_LINK].enabled) {
+        if (op_ret == 0) {
+            TRACE_STAT_TO_STR(buf, statstr);
+            TRACE_STAT_TO_STR(preparent, preparentstr);
+            TRACE_STAT_TO_STR(postparent, postparentstr);
+
+            snprintf(string, sizeof(string),
+                     "%" PRId64
+                     ": (op_ret=%d, "
+                     "*stbuf = {%s},  *prebuf = {%s},"
+                     " *postbuf = {%s})",
+                     frame->root->unique, op_ret, statstr, preparentstr,
+                     postparentstr);
+        } else {
+            snprintf(string, sizeof(string),
+                     "%" PRId64
+                     ": gfid=%s op_ret=%d, "
+                     "op_errno=%d",
+                     frame->root->unique, uuid_utoa(frame->local), op_ret,
+                     op_errno);
+        }
+        LOG_ELEMENT(conf, string);
+    }
+out:
+    TRACE_STACK_UNWIND(link, frame, op_ret, op_errno, inode, buf, preparent,
+                       postparent, xdata);
+    return 0;
+}
+
+int
+trace_flush_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                int32_t op_ret, int32_t op_errno, dict_t *xdata)
+{
+    trace_conf_t *conf = NULL;
+
+    conf = this->private;
+
+    if (!conf->log_file && !conf->log_history)
+        goto out;
+    char string[4096] = {
+        0,
+    };
+    if (trace_fop_names[GF_FOP_FLUSH].enabled) {
+        snprintf(string, sizeof(string),
+                 "%" PRId64 ": gfid=%s op_ret=%d, op_errno=%d",
+                 frame->root->unique, uuid_utoa(frame->local), op_ret,
+                 op_errno);
+
+        LOG_ELEMENT(conf, string);
+    }
+out:
+    TRACE_STACK_UNWIND(flush, frame, op_ret, op_errno, xdata);
+    return 0;
+}
+
+int
+trace_opendir_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                  int32_t op_ret, int32_t op_errno, fd_t *fd, dict_t *xdata)
+{
+    trace_conf_t *conf = NULL;
+
+    conf = this->private;
+
+    if (!conf->log_file && !conf->log_history)
+        goto out;
+    char string[4096] = {
+        0,
+    };
+    if (trace_fop_names[GF_FOP_OPENDIR].enabled) {
+        snprintf(string, sizeof(string),
+                 "%" PRId64
+                 ": gfid=%s op_ret=%d, op_errno=%d,"
+                 " fd=%p",
+                 frame->root->unique, uuid_utoa(frame->local), op_ret, op_errno,
+                 fd);
+
+        LOG_ELEMENT(conf, string);
+    }
+out:
+    /* for 'releasedir' log */
+    if (op_ret >= 0)
+        fd_ctx_set(fd, this, 0);
+
+    TRACE_STACK_UNWIND(opendir, frame, op_ret, op_errno, fd, xdata);
+    return 0;
+}
+
+int
+trace_rmdir_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                int32_t op_ret, int32_t op_errno, struct iatt *preparent,
+                struct iatt *postparent, dict_t *xdata)
+{
+    char preparentstr[1024] = {
+        0,
+    };
+    char postparentstr[1024] = {
+        0,
+    };
+    trace_conf_t *conf = NULL;
+
+    conf = this->private;
+
+    if (!conf->log_file && !conf->log_history)
+        goto out;
+    if (trace_fop_names[GF_FOP_RMDIR].enabled) {
+        char string[4096] = {
+            0,
+        };
+        if (op_ret == 0) {
+            TRACE_STAT_TO_STR(preparent, preparentstr);
+            TRACE_STAT_TO_STR(postparent, postparentstr);
+
+            snprintf(string, sizeof(string),
+                     "%" PRId64
+                     ": gfid=%s op_ret=%d, "
+                     "*prebuf={%s},  *postbuf={%s}",
+                     frame->root->unique, uuid_utoa(frame->local), op_ret,
+                     preparentstr, postparentstr);
+        } else {
+            snprintf(string, sizeof(string),
+                     "%" PRId64
+                     ": gfid=%s op_ret=%d, "
+                     "op_errno=%d",
+                     frame->root->unique, uuid_utoa(frame->local), op_ret,
+                     op_errno);
+        }
+        LOG_ELEMENT(conf, string);
+    }
+out:
+    TRACE_STACK_UNWIND(rmdir, frame, op_ret, op_errno, preparent, postparent,
+                       xdata);
+    return 0;
+}
+
+int
+trace_truncate_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                   int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
+                   struct iatt *postbuf, dict_t *xdata)
+{
+    char preopstr[1024] = {
+        0,
+    };
+    char postopstr[1024] = {
+        0,
+    };
+    trace_conf_t *conf = NULL;
+
+    conf = this->private;
+
+    if (!conf->log_file && !conf->log_history)
+        goto out;
+    if (trace_fop_names[GF_FOP_TRUNCATE].enabled) {
+        char string[4096] = {
+            0,
+        };
+        if (op_ret == 0) {
+            TRACE_STAT_TO_STR(prebuf, preopstr);
+            TRACE_STAT_TO_STR(postbuf, postopstr);
+
+            snprintf(string, sizeof(string),
+                     "%" PRId64
+                     ": (op_ret=%d, "
+                     "*prebuf = {%s}, *postbuf = {%s} )",
+                     frame->root->unique, op_ret, preopstr, postopstr);
+        } else {
+            snprintf(string, sizeof(string),
+                     "%" PRId64
+                     ": gfid=%s op_ret=%d, "
+                     "op_errno=%d",
+                     frame->root->unique, uuid_utoa(frame->local), op_ret,
+                     op_errno);
+        }
+        LOG_ELEMENT(conf, string);
+    }
+out:
+    TRACE_STACK_UNWIND(truncate, frame, op_ret, op_errno, prebuf, postbuf,
+                       xdata);
+    return 0;
+}
+
+int
+trace_statfs_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                 int32_t op_ret, int32_t op_errno, struct statvfs *buf,
+                 dict_t *xdata)
+{
+    trace_conf_t *conf = NULL;
+
+    conf = this->private;
+
+    if (!conf->log_file && !conf->log_history)
+        goto out;
+    if (trace_fop_names[GF_FOP_STATFS].enabled) {
+        char string[4096] = {
+            0,
+        };
+        if (op_ret == 0) {
+            snprintf(string, sizeof(string),
+                     "%" PRId64
+                     ": ({f_bsize=%lu, "
+                     "f_frsize=%lu, "
+                     "f_blocks=%" GF_PRI_FSBLK ", f_bfree=%" GF_PRI_FSBLK
+                     ", "
+                     "f_bavail=%" GF_PRI_FSBLK
+                     ", "
+                     "f_files=%" GF_PRI_FSBLK
+                     ", "
+                     "f_ffree=%" GF_PRI_FSBLK
+                     ", "
+                     "f_favail=%" GF_PRI_FSBLK
+                     ", "
+                     "f_fsid=%lu, f_flag=%lu, "
+                     "f_namemax=%lu}) => ret=%d",
+                     frame->root->unique, buf->f_bsize, buf->f_frsize,
+                     buf->f_blocks, buf->f_bfree, buf->f_bavail, buf->f_files,
+                     buf->f_ffree, buf->f_favail, buf->f_fsid, buf->f_flag,
+                     buf->f_namemax, op_ret);
+        } else {
+            snprintf(string, sizeof(string),
+                     "%" PRId64
+                     ": (op_ret=%d, "
+                     "op_errno=%d)",
+                     frame->root->unique, op_ret, op_errno);
+        }
+        LOG_ELEMENT(conf, string);
+    }
+out:
+    TRACE_STACK_UNWIND(statfs, frame, op_ret, op_errno, buf, xdata);
+    return 0;
+}
+
+int
+trace_setxattr_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                   int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
-        trace_conf_t   *conf = NULL;
+    trace_conf_t *conf = NULL;
 
-        conf = this->private;
+    conf = this->private;
 
-        if (!conf->log_file && !conf->log_history)
-		goto out;
-        char      string[4096] = {0,};
-        if (trace_fop_names[GF_FOP_FLUSH].enabled) {
-                snprintf (string, sizeof (string),
-                          "%"PRId64": gfid=%s op_ret=%d, op_errno=%d",
-                          frame->root->unique, uuid_utoa (frame->local),
-                          op_ret, op_errno);
+    if (!conf->log_file && !conf->log_history)
+        goto out;
+    if (trace_fop_names[GF_FOP_SETXATTR].enabled) {
+        char string[4096] = {
+            0,
+        };
+        snprintf(string, sizeof(string),
+                 "%" PRId64 ": gfid=%s op_ret=%d, op_errno=%d",
+                 frame->root->unique, uuid_utoa(frame->local), op_ret,
+                 op_errno);
 
-                LOG_ELEMENT (conf, string);
-        }
+        LOG_ELEMENT(conf, string);
+    }
 out:
-        TRACE_STACK_UNWIND (flush, frame, op_ret, op_errno, xdata);
-        return 0;
+    TRACE_STACK_UNWIND(setxattr, frame, op_ret, op_errno, xdata);
+    return 0;
 }
 
 int
-trace_opendir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                   int32_t op_ret, int32_t op_errno, fd_t *fd, dict_t *xdata)
+trace_getxattr_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                   int32_t op_ret, int32_t op_errno, dict_t *dict,
+                   dict_t *xdata)
 {
-        trace_conf_t   *conf = NULL;
+    trace_conf_t *conf = NULL;
 
-        conf = this->private;
+    conf = this->private;
 
-        if (!conf->log_file && !conf->log_history)
-		goto out;
-        char    string[4096] = {0,};
-        if (trace_fop_names[GF_FOP_OPENDIR].enabled) {
-                snprintf (string, sizeof (string),
-                          "%"PRId64": gfid=%s op_ret=%d, op_errno=%d,"
-                          " fd=%p",
-                          frame->root->unique, uuid_utoa (frame->local),
-                          op_ret, op_errno, fd);
+    if (!conf->log_file && !conf->log_history)
+        goto out;
+    if (trace_fop_names[GF_FOP_GETXATTR].enabled) {
+        char string[4096] = {
+            0,
+        };
+        snprintf(string, sizeof(string),
+                 "%" PRId64
+                 ": gfid=%s op_ret=%d, op_errno=%d,"
+                 " dict=%p",
+                 frame->root->unique, uuid_utoa(frame->local), op_ret, op_errno,
+                 dict);
 
-                LOG_ELEMENT (conf, string);
-        }
+        LOG_ELEMENT(conf, string);
+    }
 out:
-        /* for 'releasedir' log */
-        if (op_ret >= 0)
-                fd_ctx_set (fd, this, 0);
-
-        TRACE_STACK_UNWIND (opendir, frame, op_ret, op_errno, fd, xdata);
-        return 0;
-}
-
-int
-trace_rmdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                 int32_t op_ret, int32_t op_errno,
-                 struct iatt *preparent, struct iatt *postparent, dict_t *xdata)
-{
-        char           preparentstr[4096]  = {0, };
-        char           postparentstr[4096] = {0, };
-        trace_conf_t   *conf               = NULL;
-
-        conf = this->private;
-
-        if (!conf->log_file && !conf->log_history)
-		goto out;
-        if (trace_fop_names[GF_FOP_RMDIR].enabled) {
-                char  string[4096] = {0,};
-                if (op_ret == 0) {
-                        TRACE_STAT_TO_STR (preparent, preparentstr);
-                        TRACE_STAT_TO_STR (postparent, postparentstr);
-
-                        snprintf (string, sizeof (string),
-                                  "%"PRId64": gfid=%s op_ret=%d, "
-                                  "*prebuf={%s},  *postbuf={%s}",
-                                  frame->root->unique,
-                                  uuid_utoa (frame->local),
-                                  op_ret, preparentstr, postparentstr);
-                } else {
-                        snprintf (string, sizeof (string),
-                                  "%"PRId64": gfid=%s op_ret=%d, "
-                                  "op_errno=%d", frame->root->unique,
-                                  uuid_utoa (frame->local),
-                                  op_ret, op_errno);
-                }
-                LOG_ELEMENT (conf, string);
-        }
-out:
-        TRACE_STACK_UNWIND (rmdir, frame, op_ret, op_errno,
-                            preparent, postparent, xdata);
-        return 0;
-}
-
-int
-trace_truncate_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                    int32_t op_ret, int32_t op_errno,
-                    struct iatt *prebuf, struct iatt *postbuf, dict_t *xdata)
-{
-        char           preopstr[4096]  = {0, };
-        char           postopstr[4096] = {0, };
-        trace_conf_t   *conf           = NULL;
-
-        conf = this->private;
-
-        if (!conf->log_file && !conf->log_history)
-		goto out;
-        if (trace_fop_names[GF_FOP_TRUNCATE].enabled) {
-                char   string[4096] = {0,};
-                if (op_ret == 0) {
-                        TRACE_STAT_TO_STR (prebuf, preopstr);
-                        TRACE_STAT_TO_STR (postbuf, postopstr);
-
-                        snprintf (string, sizeof (string),
-                                  "%"PRId64": (op_ret=%d, "
-                                  "*prebuf = {%s}, *postbuf = {%s} )",
-                                  frame->root->unique, op_ret,
-                                  preopstr, postopstr);
-                } else {
-                        snprintf (string, sizeof (string),
-                                  "%"PRId64": gfid=%s op_ret=%d, "
-                                  "op_errno=%d", frame->root->unique,
-                                  uuid_utoa (frame->local), op_ret,
-                                  op_errno);
-                }
-                LOG_ELEMENT (conf, string);
-        }
-out:
-        TRACE_STACK_UNWIND (truncate, frame, op_ret, op_errno, prebuf,
-                            postbuf, xdata);
-        return 0;
-}
+    TRACE_STACK_UNWIND(getxattr, frame, op_ret, op_errno, dict, xdata);
 
-int
-trace_statfs_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                  int32_t op_ret, int32_t op_errno, struct statvfs *buf,
-                  dict_t *xdata)
-{
-        trace_conf_t  *conf = NULL;
-
-        conf = this->private;
-
-        if (!conf->log_file && !conf->log_history)
-		goto out;
-        if (trace_fop_names[GF_FOP_STATFS].enabled) {
-                char   string[4096] = {0,};
-                if (op_ret == 0) {
-                        snprintf (string, sizeof (string),
-                                  "%"PRId64": ({f_bsize=%lu, "
-                                  "f_frsize=%lu, "
-                                  "f_blocks=%"GF_PRI_FSBLK
-                                  ", f_bfree=%"GF_PRI_FSBLK", "
-                                  "f_bavail=%"GF_PRI_FSBLK", "
-                                  "f_files=%"GF_PRI_FSBLK", "
-                                  "f_ffree=%"GF_PRI_FSBLK", "
-                                  "f_favail=%"GF_PRI_FSBLK", "
-                                  "f_fsid=%lu, f_flag=%lu, "
-                                  "f_namemax=%lu}) => ret=%d",
-                                  frame->root->unique, buf->f_bsize,
-                                  buf->f_frsize, buf->f_blocks,
-                                  buf->f_bfree, buf->f_bavail,
-                                  buf->f_files, buf->f_ffree,
-                                  buf->f_favail, buf->f_fsid,
-                                  buf->f_flag, buf->f_namemax, op_ret);
-                } else {
-                        snprintf (string, sizeof (string),
-                                  "%"PRId64": (op_ret=%d, "
-                                  "op_errno=%d)",
-                                  frame->root->unique, op_ret,
-                                  op_errno);
-                }
-                LOG_ELEMENT (conf, string);
-        }
-out:
-        TRACE_STACK_UNWIND (statfs, frame, op_ret, op_errno, buf, xdata);
-        return 0;
+    return 0;
 }
 
 int
-trace_setxattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+trace_fsetxattr_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
                     int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
-        trace_conf_t     *conf = NULL;
+    trace_conf_t *conf = NULL;
 
-        conf = this->private;
+    conf = this->private;
 
-        if (!conf->log_file && !conf->log_history)
-		goto out;
-        if (trace_fop_names[GF_FOP_SETXATTR].enabled) {
-                char      string[4096] = {0,};
-                snprintf (string, sizeof (string),
-                          "%"PRId64": gfid=%s op_ret=%d, op_errno=%d",
-                          frame->root->unique,
-                          uuid_utoa (frame->local), op_ret,
-                          op_errno);
+    if (!conf->log_file && !conf->log_history)
+        goto out;
+    if (trace_fop_names[GF_FOP_FSETXATTR].enabled) {
+        char string[4096] = {
+            0,
+        };
+        snprintf(string, sizeof(string),
+                 "%" PRId64 ": gfid=%s op_ret=%d, op_errno=%d",
+                 frame->root->unique, uuid_utoa(frame->local), op_ret,
+                 op_errno);
 
-                LOG_ELEMENT (conf, string);
-        }
+        LOG_ELEMENT(conf, string);
+    }
 out:
-        TRACE_STACK_UNWIND (setxattr, frame, op_ret, op_errno, xdata);
-        return 0;
+    TRACE_STACK_UNWIND(fsetxattr, frame, op_ret, op_errno, xdata);
+    return 0;
 }
 
 int
-trace_getxattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+trace_fgetxattr_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
                     int32_t op_ret, int32_t op_errno, dict_t *dict,
                     dict_t *xdata)
 {
-        trace_conf_t    *conf = NULL;
+    trace_conf_t *conf = NULL;
 
-        conf = this->private;
+    conf = this->private;
 
-        if (!conf->log_file && !conf->log_history)
-		goto out;
-        if (trace_fop_names[GF_FOP_GETXATTR].enabled) {
-                char       string[4096] = {0,};
-                snprintf (string, sizeof (string),
-                          "%"PRId64": gfid=%s op_ret=%d, op_errno=%d,"
-                          " dict=%p", frame->root->unique,
-                          uuid_utoa (frame->local), op_ret, op_errno,
-                          dict);
+    if (!conf->log_file && !conf->log_history)
+        goto out;
+    if (trace_fop_names[GF_FOP_FGETXATTR].enabled) {
+        char string[4096] = {
+            0,
+        };
+        snprintf(string, sizeof(string),
+                 "%" PRId64
+                 ": gfid=%s op_ret=%d, op_errno=%d,"
+                 " dict=%p",
+                 frame->root->unique, uuid_utoa(frame->local), op_ret, op_errno,
+                 dict);
 
-                LOG_ELEMENT (conf, string);
-        }
+        LOG_ELEMENT(conf, string);
+    }
 out:
-        TRACE_STACK_UNWIND (getxattr, frame, op_ret, op_errno, dict, xdata);
-
-        return 0;
-}
-
-int
-trace_fsetxattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                     int32_t op_ret, int32_t op_errno, dict_t *xdata)
-{
-        trace_conf_t   *conf = NULL;
-
-        conf = this->private;
+    TRACE_STACK_UNWIND(fgetxattr, frame, op_ret, op_errno, dict, xdata);
 
-        if (!conf->log_file && !conf->log_history)
-                goto out;
-        if (trace_fop_names[GF_FOP_FSETXATTR].enabled) {
-                char    string[4096] = {0,};
-                snprintf (string, sizeof (string),
-                          "%"PRId64": gfid=%s op_ret=%d, op_errno=%d",
-                          frame->root->unique,
-                          uuid_utoa (frame->local), op_ret, op_errno);
-
-                LOG_ELEMENT (conf, string);
-        }
-out:
-        TRACE_STACK_UNWIND (fsetxattr, frame, op_ret, op_errno, xdata);
-        return 0;
+    return 0;
 }
 
 int
-trace_fgetxattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                     int32_t op_ret, int32_t op_errno, dict_t *dict,
-                     dict_t *xdata)
+trace_removexattr_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                      int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
-        trace_conf_t    *conf = NULL;
+    trace_conf_t *conf = NULL;
 
-        conf = this->private;
+    conf = this->private;
 
-        if (!conf->log_file && !conf->log_history)
-		goto out;
-        if (trace_fop_names[GF_FOP_FGETXATTR].enabled) {
-                char      string[4096]  = {0,};
-                snprintf (string, sizeof (string),
-                          "%"PRId64": gfid=%s op_ret=%d, op_errno=%d,"
-                          " dict=%p", frame->root->unique,
-                          uuid_utoa (frame->local), op_ret, op_errno,
-                          dict);
+    if (!conf->log_file && !conf->log_history)
+        goto out;
+    if (trace_fop_names[GF_FOP_REMOVEXATTR].enabled) {
+        char string[4096] = {
+            0,
+        };
+        snprintf(string, sizeof(string),
+                 "%" PRId64 ": gfid=%s op_ret=%d, op_errno=%d",
+                 frame->root->unique, uuid_utoa(frame->local), op_ret,
+                 op_errno);
 
-                LOG_ELEMENT (conf, string);
-        }
+        LOG_ELEMENT(conf, string);
+    }
 out:
-        TRACE_STACK_UNWIND (fgetxattr, frame, op_ret, op_errno, dict, xdata);
+    TRACE_STACK_UNWIND(removexattr, frame, op_ret, op_errno, xdata);
 
-        return 0;
+    return 0;
 }
 
 int
-trace_removexattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                       int32_t op_ret, int32_t op_errno, dict_t *xdata)
+trace_fsyncdir_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                   int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
-        trace_conf_t  *conf = NULL;
+    trace_conf_t *conf = NULL;
 
-        conf = this->private;
+    conf = this->private;
 
-        if (!conf->log_file && !conf->log_history)
-		goto out;
-        if (trace_fop_names[GF_FOP_REMOVEXATTR].enabled) {
-                char       string[4096] = {0,};
-                snprintf (string, sizeof (string),
-                          "%"PRId64": gfid=%s op_ret=%d, op_errno=%d",
-                          frame->root->unique,
-                          uuid_utoa (frame->local), op_ret, op_errno);
+    if (!conf->log_file && !conf->log_history)
+        goto out;
+    if (trace_fop_names[GF_FOP_FSYNCDIR].enabled) {
+        char string[4096] = {
+            0,
+        };
+        snprintf(string, sizeof(string),
+                 "%" PRId64 ": gfid=%s op_ret=%d, op_errno=%d",
+                 frame->root->unique, uuid_utoa(frame->local), op_ret,
+                 op_errno);
 
-                LOG_ELEMENT (conf, string);
-        }
+        LOG_ELEMENT(conf, string);
+    }
 out:
-        TRACE_STACK_UNWIND (removexattr, frame, op_ret, op_errno, xdata);
-
-        return 0;
+    TRACE_STACK_UNWIND(fsyncdir, frame, op_ret, op_errno, xdata);
+    return 0;
 }
 
 int
-trace_fsyncdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                    int32_t op_ret, int32_t op_errno, dict_t *xdata)
+trace_access_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                 int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
-        trace_conf_t     *conf = NULL;
-
-        conf = this->private;
-
-        if (!conf->log_file && !conf->log_history)
-		goto out;
-        if (trace_fop_names[GF_FOP_FSYNCDIR].enabled) {
-                char       string[4096]   =  {0,};
-                snprintf (string, sizeof (string),
-                          "%"PRId64": gfid=%s op_ret=%d, op_errno=%d",
-                          frame->root->unique,
-                          uuid_utoa (frame->local), op_ret, op_errno);
-
-                LOG_ELEMENT (conf, string);
-        }
-out:
-        TRACE_STACK_UNWIND (fsyncdir, frame, op_ret, op_errno, xdata);
-        return 0;
-}
-
-int
-trace_access_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+    trace_conf_t *conf = NULL;
+
+    conf = this->private;
+
+    if (!conf->log_file && !conf->log_history)
+        goto out;
+    if (trace_fop_names[GF_FOP_ACCESS].enabled) {
+        char string[4096] = {
+            0,
+        };
+        snprintf(string, sizeof(string),
+                 "%" PRId64
+                 ": gfid=%s op_ret=%d, "
+                 "op_errno=%d)",
+                 frame->root->unique, uuid_utoa(frame->local), op_ret,
+                 op_errno);
+
+        LOG_ELEMENT(conf, string);
+    }
+out:
+    TRACE_STACK_UNWIND(access, frame, op_ret, op_errno, xdata);
+    return 0;
+}
+
+int
+trace_ftruncate_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                    int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
+                    struct iatt *postbuf, dict_t *xdata)
+{
+    char prebufstr[1024] = {
+        0,
+    };
+    char postbufstr[1024] = {
+        0,
+    };
+    trace_conf_t *conf = NULL;
+
+    conf = this->private;
+
+    if (!conf->log_file && !conf->log_history)
+        goto out;
+    if (trace_fop_names[GF_FOP_FTRUNCATE].enabled) {
+        char string[4096] = {
+            0,
+        };
+        if (op_ret == 0) {
+            TRACE_STAT_TO_STR(prebuf, prebufstr);
+            TRACE_STAT_TO_STR(postbuf, postbufstr);
+
+            snprintf(string, sizeof(string),
+                     "%" PRId64
+                     ": op_ret=%d, "
+                     "*prebuf = {%s}, *postbuf = {%s} )",
+                     frame->root->unique, op_ret, prebufstr, postbufstr);
+        } else {
+            snprintf(string, sizeof(string),
+                     "%" PRId64
+                     ": gfid=%s op_ret=%d, "
+                     "op_errno=%d",
+                     frame->root->unique, uuid_utoa(frame->local), op_ret,
+                     op_errno);
+        }
+        LOG_ELEMENT(conf, string);
+    }
+out:
+    TRACE_STACK_UNWIND(ftruncate, frame, op_ret, op_errno, prebuf, postbuf,
+                       xdata);
+    return 0;
+}
+
+int
+trace_fstat_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                int32_t op_ret, int32_t op_errno, struct iatt *buf,
+                dict_t *xdata)
+{
+    char statstr[1024] = {
+        0,
+    };
+    trace_conf_t *conf = NULL;
+
+    conf = this->private;
+
+    if (!conf->log_file && !conf->log_history)
+        goto out;
+    if (trace_fop_names[GF_FOP_FSTAT].enabled) {
+        char string[4096] = {0.};
+        if (op_ret == 0) {
+            TRACE_STAT_TO_STR(buf, statstr);
+            snprintf(string, sizeof(string),
+                     "%" PRId64
+                     ": gfid=%s op_ret=%d "
+                     "buf=%s",
+                     frame->root->unique, uuid_utoa(frame->local), op_ret,
+                     statstr);
+        } else {
+            snprintf(string, sizeof(string),
+                     "%" PRId64
+                     ": gfid=%s op_ret=%d, "
+                     "op_errno=%d",
+                     frame->root->unique, uuid_utoa(frame->local), op_ret,
+                     op_errno);
+        }
+        LOG_ELEMENT(conf, string);
+    }
+out:
+    TRACE_STACK_UNWIND(fstat, frame, op_ret, op_errno, buf, xdata);
+    return 0;
+}
+
+int
+trace_lk_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int32_t op_ret,
+             int32_t op_errno, struct gf_flock *lock, dict_t *xdata)
+{
+    trace_conf_t *conf = NULL;
+
+    conf = this->private;
+
+    if (!conf->log_file && !conf->log_history)
+        goto out;
+    if (trace_fop_names[GF_FOP_LK].enabled) {
+        char string[4096] = {
+            0,
+        };
+        if (op_ret == 0) {
+            snprintf(string, sizeof(string),
+                     "%" PRId64
+                     ": gfid=%s op_ret=%d, "
+                     "{l_type=%d, l_whence=%d, "
+                     "l_start=%" PRId64
+                     ", "
+                     "l_len=%" PRId64 ", l_pid=%u})",
+                     frame->root->unique, uuid_utoa(frame->local), op_ret,
+                     lock->l_type, lock->l_whence, lock->l_start, lock->l_len,
+                     lock->l_pid);
+        } else {
+            snprintf(string, sizeof(string),
+                     "%" PRId64
+                     ": gfid=%s op_ret=%d, "
+                     "op_errno=%d)",
+                     frame->root->unique, uuid_utoa(frame->local), op_ret,
+                     op_errno);
+        }
+
+        LOG_ELEMENT(conf, string);
+    }
+out:
+    TRACE_STACK_UNWIND(lk, frame, op_ret, op_errno, lock, xdata);
+    return 0;
+}
+
+int
+trace_entrylk_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
                   int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
-        trace_conf_t   *conf = NULL;
+    trace_conf_t *conf = NULL;
 
-        conf = this->private;
+    conf = this->private;
 
-        if (!conf->log_file && !conf->log_history)
-		goto out;
-        if (trace_fop_names[GF_FOP_ACCESS].enabled) {
-                char      string[4096] = {0,};
-                snprintf (string, sizeof (string),
-                          "%"PRId64": gfid=%s op_ret=%d, "
-                          "op_errno=%d)", frame->root->unique,
-                          uuid_utoa (frame->local), op_ret, op_errno);
+    if (!conf->log_file && !conf->log_history)
+        goto out;
+    if (trace_fop_names[GF_FOP_ENTRYLK].enabled) {
+        char string[4096] = {
+            0,
+        };
+        snprintf(string, sizeof(string),
+                 "%" PRId64 ": gfid=%s op_ret=%d, op_errno=%d",
+                 frame->root->unique, uuid_utoa(frame->local), op_ret,
+                 op_errno);
 
-                LOG_ELEMENT (conf, string);
-        }
-out:
-        TRACE_STACK_UNWIND (access, frame, op_ret, op_errno, xdata);
-        return 0;
-}
-
-int
-trace_ftruncate_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                     int32_t op_ret, int32_t op_errno,
-                     struct iatt *prebuf, struct iatt *postbuf, dict_t *xdata)
-{
-        char          prebufstr[4096]  = {0, };
-        char          postbufstr[4096] = {0, };
-        trace_conf_t  *conf            = NULL;
-
-        conf = this->private;
-
-        if (!conf->log_file && !conf->log_history)
-		goto out;
-        if (trace_fop_names[GF_FOP_FTRUNCATE].enabled) {
-                char  string[4096]  = {0,};
-                if (op_ret == 0) {
-                        TRACE_STAT_TO_STR (prebuf, prebufstr);
-                        TRACE_STAT_TO_STR (postbuf, postbufstr);
-
-                        snprintf (string, sizeof (string),
-                                  "%"PRId64": op_ret=%d, "
-                                  "*prebuf = {%s}, *postbuf = {%s} )",
-                                  frame->root->unique, op_ret,
-                                  prebufstr, postbufstr);
-                } else {
-                        snprintf (string, sizeof (string),
-                                  "%"PRId64": gfid=%s op_ret=%d, "
-                                  "op_errno=%d", frame->root->unique,
-                                  uuid_utoa (frame->local), op_ret,
-                                  op_errno);
-                }
-                LOG_ELEMENT (conf, string);
-        }
-out:
-        TRACE_STACK_UNWIND (ftruncate, frame, op_ret, op_errno, prebuf, postbuf,
-                            xdata);
-        return 0;
-}
-
-int
-trace_fstat_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                 int32_t op_ret, int32_t op_errno, struct iatt *buf, dict_t *xdata)
-{
-        char          statstr[4096] = {0, };
-        trace_conf_t  *conf         = NULL;
-
-        conf = this->private;
-
-        if (!conf->log_file && !conf->log_history)
-		goto out;
-        if (trace_fop_names[GF_FOP_FSTAT].enabled) {
-                char string[4096]  = {0.};
-                if (op_ret == 0) {
-                        TRACE_STAT_TO_STR (buf, statstr);
-                        snprintf (string, sizeof (string),
-                                  "%"PRId64": gfid=%s op_ret=%d "
-                                  "buf=%s", frame->root->unique,
-                                  uuid_utoa (frame->local), op_ret,
-                                  statstr);
-                } else {
-                        snprintf (string, sizeof (string),
-                                  "%"PRId64": gfid=%s op_ret=%d, "
-                                  "op_errno=%d", frame->root->unique,
-                                  uuid_utoa (frame->local), op_ret,
-                                  op_errno);
-                }
-                LOG_ELEMENT (conf, string);
-        }
+        LOG_ELEMENT(conf, string);
+    }
 out:
-        TRACE_STACK_UNWIND (fstat, frame, op_ret, op_errno, buf, xdata);
-        return 0;
+    TRACE_STACK_UNWIND(entrylk, frame, op_ret, op_errno, xdata);
+    return 0;
 }
 
 int
-trace_lk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-              int32_t op_ret, int32_t op_errno, struct gf_flock *lock,
-              dict_t *xdata)
-{
-        trace_conf_t   *conf = NULL;
-
-        conf = this->private;
-
-        if (!conf->log_file && !conf->log_history)
-		goto out;
-        if (trace_fop_names[GF_FOP_LK].enabled) {
-                char      string[4096] = {0,};
-                if (op_ret == 0) {
-                        snprintf (string, sizeof (string),
-                                  "%"PRId64": gfid=%s op_ret=%d, "
-                                  "{l_type=%d, l_whence=%d, "
-                                  "l_start=%"PRId64", "
-                                  "l_len=%"PRId64", l_pid=%u})",
-                                  frame->root->unique,
-                                  uuid_utoa (frame->local),
-                                  op_ret, lock->l_type, lock->l_whence,
-                                  lock->l_start, lock->l_len,
-                                  lock->l_pid);
-                } else {
-                        snprintf (string, sizeof (string),
-                                  "%"PRId64": gfid=%s op_ret=%d, "
-                                  "op_errno=%d)", frame->root->unique,
-                                  uuid_utoa (frame->local), op_ret,
-                                  op_errno);
-                }
-
-                LOG_ELEMENT (conf, string);
-        }
-out:
-        TRACE_STACK_UNWIND (lk, frame, op_ret, op_errno, lock, xdata);
-        return 0;
-}
-
-int
-trace_entrylk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+trace_fentrylk_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
                    int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
-        trace_conf_t  *conf = NULL;
+    trace_conf_t *conf = NULL;
 
-        conf = this->private;
+    conf = this->private;
 
-        if (!conf->log_file && !conf->log_history)
-		goto out;
-        if (trace_fop_names[GF_FOP_ENTRYLK].enabled) {
-                char   string[4096] = {0,};
-                snprintf (string, sizeof (string),
-                          "%"PRId64": gfid=%s op_ret=%d, op_errno=%d",
-                          frame->root->unique,
-                          uuid_utoa (frame->local), op_ret, op_errno);
+    if (!conf->log_file && !conf->log_history)
+        goto out;
+    if (trace_fop_names[GF_FOP_FENTRYLK].enabled) {
+        char string[4096] = {
+            0,
+        };
+        snprintf(string, sizeof(string),
+                 "%" PRId64 ": gfid=%s op_ret=%d, op_errno=%d",
+                 frame->root->unique, uuid_utoa(frame->local), op_ret,
+                 op_errno);
 
-                LOG_ELEMENT (conf, string);
-        }
+        LOG_ELEMENT(conf, string);
+    }
 out:
-        TRACE_STACK_UNWIND (entrylk, frame, op_ret, op_errno, xdata);
-        return 0;
+    TRACE_STACK_UNWIND(fentrylk, frame, op_ret, op_errno, xdata);
+    return 0;
 }
 
 int
-trace_fentrylk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                    int32_t op_ret, int32_t op_errno, dict_t *xdata)
+trace_xattrop_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                  int32_t op_ret, int32_t op_errno, dict_t *dict, dict_t *xdata)
 {
-        trace_conf_t    *conf = NULL;
+    trace_conf_t *conf = NULL;
 
-        conf = this->private;
+    conf = this->private;
 
-        if (!conf->log_file && !conf->log_history)
-		goto out;
-        if (trace_fop_names[GF_FOP_FENTRYLK].enabled) {
-                char      string[4096] = {0,};
-                snprintf (string, sizeof (string),
-                          "%"PRId64": gfid=%s op_ret=%d, op_errno=%d",
-                          frame->root->unique,
-                          uuid_utoa (frame->local), op_ret, op_errno);
+    if (!conf->log_file && !conf->log_history)
+        goto out;
+    if (trace_fop_names[GF_FOP_XATTROP].enabled) {
+        char string[4096] = {
+            0,
+        };
+        snprintf(string, sizeof(string),
+                 "%" PRId64 ": gfid=%s op_ret=%d, op_errno=%d",
+                 frame->root->unique, uuid_utoa(frame->local), op_ret,
+                 op_errno);
 
-                LOG_ELEMENT (conf, string);
-        }
+        LOG_ELEMENT(conf, string);
+    }
 out:
-        TRACE_STACK_UNWIND (fentrylk, frame, op_ret, op_errno, xdata);
-        return 0;
+    TRACE_STACK_UNWIND(xattrop, frame, op_ret, op_errno, dict, xdata);
+    return 0;
 }
 
 int
-trace_xattrop_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+trace_fxattrop_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
                    int32_t op_ret, int32_t op_errno, dict_t *dict,
                    dict_t *xdata)
 {
-        trace_conf_t  *conf = NULL;
+    trace_conf_t *conf = NULL;
 
-        conf = this->private;
+    conf = this->private;
 
-        if (!conf->log_file && !conf->log_history)
-		goto out;
-        if (trace_fop_names[GF_FOP_XATTROP].enabled) {
-                char    string[4096] = {0,};
-                snprintf (string, sizeof (string),
-                          "%"PRId64": gfid=%s op_ret=%d, op_errno=%d",
-                          frame->root->unique,
-                          uuid_utoa (frame->local), op_ret, op_errno);
+    if (!conf->log_file && !conf->log_history)
+        goto out;
+    if (trace_fop_names[GF_FOP_FXATTROP].enabled) {
+        char string[4096] = {
+            0,
+        };
+        snprintf(string, sizeof(string),
+                 "%" PRId64 ": gfid=%s op_ret=%d, op_errno=%d",
+                 frame->root->unique, uuid_utoa(frame->local), op_ret,
+                 op_errno);
 
-                LOG_ELEMENT (conf, string);
-        }
+        LOG_ELEMENT(conf, string);
+    }
 out:
-        TRACE_STACK_UNWIND (xattrop, frame, op_ret, op_errno, dict, xdata);
-        return 0;
+    TRACE_STACK_UNWIND(fxattrop, frame, op_ret, op_errno, dict, xdata);
+    return 0;
 }
 
 int
-trace_fxattrop_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                    int32_t op_ret, int32_t op_errno, dict_t *dict,
-                    dict_t *xdata)
+trace_inodelk_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                  int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
-        trace_conf_t    *conf = NULL;
+    trace_conf_t *conf = NULL;
 
-        conf = this->private;
+    conf = this->private;
 
-        if (!conf->log_file && !conf->log_history)
-		goto out;
-        if (trace_fop_names[GF_FOP_FXATTROP].enabled) {
-                char      string[4096] = {0,};
-                snprintf (string, sizeof (string),
-                          "%"PRId64": gfid=%s op_ret=%d, op_errno=%d",
-                          frame->root->unique,
-                          uuid_utoa (frame->local), op_ret, op_errno);
+    if (!conf->log_file && !conf->log_history)
+        goto out;
+    if (trace_fop_names[GF_FOP_INODELK].enabled) {
+        char string[4096] = {
+            0,
+        };
+        snprintf(string, sizeof(string),
+                 "%" PRId64 ": gfid=%s op_ret=%d, op_errno=%d",
+                 frame->root->unique, uuid_utoa(frame->local), op_ret,
+                 op_errno);
 
-                LOG_ELEMENT (conf, string);
-        }
+        LOG_ELEMENT(conf, string);
+    }
 out:
-        TRACE_STACK_UNWIND (fxattrop, frame, op_ret, op_errno, dict, xdata);
-        return 0;
+    TRACE_STACK_UNWIND(inodelk, frame, op_ret, op_errno, xdata);
+    return 0;
 }
 
 int
-trace_inodelk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+trace_finodelk_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
                    int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
-        trace_conf_t   *conf = NULL;
+    trace_conf_t *conf = NULL;
 
-        conf = this->private;
+    conf = this->private;
 
-        if (!conf->log_file && !conf->log_history)
-		goto out;
-        if (trace_fop_names[GF_FOP_INODELK].enabled) {
-                char       string[4096] = {0,};
-                snprintf (string, sizeof (string),
-                          "%"PRId64": gfid=%s op_ret=%d, op_errno=%d",
-                          frame->root->unique,
-                          uuid_utoa (frame->local),op_ret, op_errno);
+    if (!conf->log_file && !conf->log_history)
+        goto out;
+    if (trace_fop_names[GF_FOP_FINODELK].enabled) {
+        char string[4096] = {
+            0,
+        };
+        snprintf(string, sizeof(string),
+                 "%" PRId64 ": gfid=%s op_ret=%d, op_errno=%d",
+                 frame->root->unique, uuid_utoa(frame->local), op_ret,
+                 op_errno);
 
-                LOG_ELEMENT (conf, string);
-        }
+        LOG_ELEMENT(conf, string);
+    }
 out:
-        TRACE_STACK_UNWIND (inodelk, frame, op_ret, op_errno, xdata);
-        return 0;
+    TRACE_STACK_UNWIND(finodelk, frame, op_ret, op_errno, xdata);
+    return 0;
 }
 
 int
-trace_finodelk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                    int32_t op_ret, int32_t op_errno, dict_t *xdata)
+trace_rchecksum_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                    int32_t op_ret, int32_t op_errno, uint32_t weak_checksum,
+                    uint8_t *strong_checksum, dict_t *xdata)
 {
-        trace_conf_t   *conf = NULL;
+    trace_conf_t *conf = NULL;
 
-        conf = this->private;
+    conf = this->private;
 
-        if (!conf->log_file && !conf->log_history)
-		goto out;
-        if (trace_fop_names[GF_FOP_FINODELK].enabled) {
-                char      string[4096] = {0,};
-                snprintf (string, sizeof (string),
-                          "%"PRId64": gfid=%s op_ret=%d, op_errno=%d",
-                          frame->root->unique,
-                          uuid_utoa (frame->local), op_ret, op_errno);
+    if (!conf->log_file && !conf->log_history)
+        goto out;
+    if (trace_fop_names[GF_FOP_RCHECKSUM].enabled) {
+        char string[4096] = {
+            0,
+        };
+        snprintf(string, sizeof(string),
+                 "%" PRId64 ": gfid=%s op_ret=%d op_errno=%d",
+                 frame->root->unique, uuid_utoa(frame->local), op_ret,
+                 op_errno);
 
-                LOG_ELEMENT (conf, string);
-        }
-out:
-        TRACE_STACK_UNWIND (finodelk, frame, op_ret, op_errno, xdata);
-        return 0;
-}
-
-int
-trace_rchecksum_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                     int32_t op_ret, int32_t op_errno,
-                     uint32_t weak_checksum, uint8_t *strong_checksum,
-                     dict_t *xdata)
-{
-        trace_conf_t   *conf = NULL;
-
-        conf = this->private;
-
-        if (!conf->log_file && !conf->log_history)
-                goto out;
-        if (trace_fop_names[GF_FOP_RCHECKSUM].enabled) {
-                char      string[4096] = {0,};
-                snprintf (string, sizeof (string),
-                          "%"PRId64": gfid=%s op_ret=%d op_errno=%d",
-                          frame->root->unique,
-                          uuid_utoa (frame->local), op_ret, op_errno);
-
-                LOG_ELEMENT (conf, string);
-        }
+        LOG_ELEMENT(conf, string);
+    }
 
 out:
-        TRACE_STACK_UNWIND (rchecksum, frame, op_ret, op_errno, weak_checksum,
-                            strong_checksum, xdata);
+    TRACE_STACK_UNWIND(rchecksum, frame, op_ret, op_errno, weak_checksum,
+                       strong_checksum, xdata);
 
-        return 0;
+    return 0;
 }
 
 /* *_cbk section over <----------> fop section start */
 
 int
-trace_entrylk (call_frame_t *frame, xlator_t *this,
-               const char *volume, loc_t *loc, const char *basename,
-               entrylk_cmd cmd, entrylk_type type, dict_t *xdata)
+trace_entrylk(call_frame_t *frame, xlator_t *this, const char *volume,
+              loc_t *loc, const char *basename, entrylk_cmd cmd,
+              entrylk_type type, dict_t *xdata)
 {
-        trace_conf_t   *conf = NULL;
+    trace_conf_t *conf = NULL;
 
-        conf = this->private;
+    conf = this->private;
 
-        if (!conf->log_file && !conf->log_history)
-		goto out;
-        if (trace_fop_names[GF_FOP_ENTRYLK].enabled) {
-                char     string[4096] = {0,};
-                snprintf (string, sizeof (string),
-                          "%"PRId64": gfid=%s volume=%s, (path=%s "
-                          "basename=%s, cmd=%s, type=%s)",
-                          frame->root->unique,
-                          uuid_utoa (loc->inode->gfid),
-                          volume, loc->path, basename,
-                          ((cmd == ENTRYLK_LOCK) ? "ENTRYLK_LOCK" :
-                           "ENTRYLK_UNLOCK"),
-                          ((type == ENTRYLK_RDLCK) ? "ENTRYLK_RDLCK" :
-                           "ENTRYLK_WRLCK"));
+    if (!conf->log_file && !conf->log_history)
+        goto out;
+    if (trace_fop_names[GF_FOP_ENTRYLK].enabled) {
+        char string[4096] = {
+            0,
+        };
+        snprintf(string, sizeof(string),
+                 "%" PRId64
+                 ": gfid=%s volume=%s, (path=%s "
+                 "basename=%s, cmd=%s, type=%s)",
+                 frame->root->unique, uuid_utoa(loc->inode->gfid), volume,
+                 loc->path, basename,
+                 ((cmd == ENTRYLK_LOCK) ? "ENTRYLK_LOCK" : "ENTRYLK_UNLOCK"),
+                 ((type == ENTRYLK_RDLCK) ? "ENTRYLK_RDLCK" : "ENTRYLK_WRLCK"));
 
-                frame->local = loc->inode->gfid;
+        frame->local = loc->inode->gfid;
 
-                LOG_ELEMENT (conf, string);
-        }
+        LOG_ELEMENT(conf, string);
+    }
 
 out:
-        STACK_WIND (frame, trace_entrylk_cbk,
-                    FIRST_CHILD (this),
-                    FIRST_CHILD (this)->fops->entrylk,
-                    volume, loc, basename, cmd, type, xdata);
-        return 0;
+    STACK_WIND(frame, trace_entrylk_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->entrylk, volume, loc, basename, cmd,
+               type, xdata);
+    return 0;
 }
 
 int
-trace_inodelk (call_frame_t *frame, xlator_t *this, const char *volume,
-               loc_t *loc, int32_t cmd, struct gf_flock *flock, dict_t *xdata)
+trace_inodelk(call_frame_t *frame, xlator_t *this, const char *volume,
+              loc_t *loc, int32_t cmd, struct gf_flock *flock, dict_t *xdata)
 {
-        char         *cmd_str  = NULL;
-        char         *type_str = NULL;
-        trace_conf_t *conf     = NULL;
+    char *cmd_str = NULL;
+    char *type_str = NULL;
+    trace_conf_t *conf = NULL;
 
-        conf = this->private;
+    conf = this->private;
 
-        if (!conf->log_file && !conf->log_history)
-		goto out;
-        if (trace_fop_names[GF_FOP_INODELK].enabled) {
-                char string[4096]  = {0,};
-                switch (cmd) {
+    if (!conf->log_file && !conf->log_history)
+        goto out;
+    if (trace_fop_names[GF_FOP_INODELK].enabled) {
+        char string[4096] = {
+            0,
+        };
+        switch (cmd) {
 #if F_GETLK != F_GETLK64
-                case F_GETLK64:
+            case F_GETLK64:
 #endif
-                case F_GETLK:
-                        cmd_str = "GETLK";
-                        break;
+            case F_GETLK:
+                cmd_str = "GETLK";
+                break;
 
 #if F_SETLK != F_SETLK64
-                case F_SETLK64:
+            case F_SETLK64:
 #endif
-                case F_SETLK:
-                        cmd_str = "SETLK";
-                        break;
+            case F_SETLK:
+                cmd_str = "SETLK";
+                break;
 
 #if F_SETLKW != F_SETLKW64
-                case F_SETLKW64:
+            case F_SETLKW64:
 #endif
-                case F_SETLKW:
-                        cmd_str = "SETLKW";
-                        break;
-
-                default:
-                        cmd_str = "UNKNOWN";
-                        break;
-                }
-
-                switch (flock->l_type) {
-                case F_RDLCK:
-                        type_str = "READ";
-                        break;
-                case F_WRLCK:
-                        type_str = "WRITE";
-                        break;
-                case F_UNLCK:
-                        type_str = "UNLOCK";
-                        break;
-                default:
-                        type_str = "UNKNOWN";
-                        break;
-                }
-
-                snprintf (string, sizeof (string),
-                          "%"PRId64": gfid=%s volume=%s, (path=%s "
-                          "cmd=%s, type=%s, start=%llu, len=%llu, "
-                          "pid=%llu)", frame->root->unique,
-                          uuid_utoa (loc->inode->gfid), volume,
-                          loc->path, cmd_str, type_str,
-                          (unsigned long long)flock->l_start,
-                          (unsigned long long) flock->l_len,
-                          (unsigned long long) flock->l_pid);
-
-                frame->local = loc->inode->gfid;
-
-                LOG_ELEMENT (conf, string);
+            case F_SETLKW:
+                cmd_str = "SETLKW";
+                break;
+
+            default:
+                cmd_str = "UNKNOWN";
+                break;
+        }
+
+        switch (flock->l_type) {
+            case F_RDLCK:
+                type_str = "READ";
+                break;
+            case F_WRLCK:
+                type_str = "WRITE";
+                break;
+            case F_UNLCK:
+                type_str = "UNLOCK";
+                break;
+            default:
+                type_str = "UNKNOWN";
+                break;
         }
 
+        snprintf(
+            string, sizeof(string),
+            "%" PRId64
+            ": gfid=%s volume=%s, (path=%s "
+            "cmd=%s, type=%s, start=%llu, len=%llu, "
+            "pid=%llu)",
+            frame->root->unique, uuid_utoa(loc->inode->gfid), volume, loc->path,
+            cmd_str, type_str, (unsigned long long)flock->l_start,
+            (unsigned long long)flock->l_len, (unsigned long long)flock->l_pid);
+
+        frame->local = loc->inode->gfid;
+
+        LOG_ELEMENT(conf, string);
+    }
+
 out:
-        STACK_WIND (frame, trace_inodelk_cbk,
-                    FIRST_CHILD (this),
-                    FIRST_CHILD (this)->fops->inodelk,
-                    volume, loc, cmd, flock, xdata);
-        return 0;
+    STACK_WIND(frame, trace_inodelk_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->inodelk, volume, loc, cmd, flock,
+               xdata);
+    return 0;
 }
 
 int
-trace_finodelk (call_frame_t *frame, xlator_t *this, const char *volume,
-                fd_t *fd, int32_t cmd, struct gf_flock *flock, dict_t *xdata)
+trace_finodelk(call_frame_t *frame, xlator_t *this, const char *volume,
+               fd_t *fd, int32_t cmd, struct gf_flock *flock, dict_t *xdata)
 {
-        char          *cmd_str  = NULL;
-        char          *type_str = NULL;
-        trace_conf_t  *conf     = NULL;
+    char *cmd_str = NULL;
+    char *type_str = NULL;
+    trace_conf_t *conf = NULL;
 
-        conf = this->private;
+    conf = this->private;
 
-        if (!conf->log_file && !conf->log_history)
-		goto out;
-        if (trace_fop_names[GF_FOP_FINODELK].enabled) {
-                char  string[4096] = {0,};
-                switch (cmd) {
+    if (!conf->log_file && !conf->log_history)
+        goto out;
+    if (trace_fop_names[GF_FOP_FINODELK].enabled) {
+        char string[4096] = {
+            0,
+        };
+        switch (cmd) {
 #if F_GETLK != F_GETLK64
-                case F_GETLK64:
+            case F_GETLK64:
 #endif
-                case F_GETLK:
-                        cmd_str = "GETLK";
-                        break;
+            case F_GETLK:
+                cmd_str = "GETLK";
+                break;
 
 #if F_SETLK != F_SETLK64
-                case F_SETLK64:
+            case F_SETLK64:
 #endif
-                case F_SETLK:
-                        cmd_str = "SETLK";
-                        break;
+            case F_SETLK:
+                cmd_str = "SETLK";
+                break;
 
 #if F_SETLKW != F_SETLKW64
-                case F_SETLKW64:
+            case F_SETLKW64:
 #endif
-                case F_SETLKW:
-                        cmd_str = "SETLKW";
-                        break;
-
-                default:
-                        cmd_str = "UNKNOWN";
-                        break;
-                }
-
-                switch (flock->l_type) {
-                case F_RDLCK:
-                        type_str = "READ";
-                        break;
-                case F_WRLCK:
-                        type_str = "WRITE";
-                        break;
-                case F_UNLCK:
-                        type_str = "UNLOCK";
-                        break;
-                default:
-                        type_str = "UNKNOWN";
-                        break;
-                }
-
-                snprintf (string, sizeof (string),
-                          "%"PRId64": gfid=%s volume=%s, (fd =%p "
-                          "cmd=%s, type=%s, start=%llu, len=%llu, "
-                          "pid=%llu)", frame->root->unique,
-                          uuid_utoa (fd->inode->gfid), volume, fd,
-                          cmd_str, type_str,
-                          (unsigned long long) flock->l_start,
-                          (unsigned long long) flock->l_len,
-                          (unsigned long long) flock->l_pid);
-
-                frame->local = fd->inode->gfid;
-
-                LOG_ELEMENT (conf, string);
+            case F_SETLKW:
+                cmd_str = "SETLKW";
+                break;
+
+            default:
+                cmd_str = "UNKNOWN";
+                break;
+        }
+
+        switch (flock->l_type) {
+            case F_RDLCK:
+                type_str = "READ";
+                break;
+            case F_WRLCK:
+                type_str = "WRITE";
+                break;
+            case F_UNLCK:
+                type_str = "UNLOCK";
+                break;
+            default:
+                type_str = "UNKNOWN";
+                break;
         }
+
+        snprintf(string, sizeof(string),
+                 "%" PRId64
+                 ": gfid=%s volume=%s, (fd =%p "
+                 "cmd=%s, type=%s, start=%llu, len=%llu, "
+                 "pid=%llu)",
+                 frame->root->unique, uuid_utoa(fd->inode->gfid), volume, fd,
+                 cmd_str, type_str, (unsigned long long)flock->l_start,
+                 (unsigned long long)flock->l_len,
+                 (unsigned long long)flock->l_pid);
+
+        frame->local = fd->inode->gfid;
+
+        LOG_ELEMENT(conf, string);
+    }
 out:
-        STACK_WIND (frame, trace_finodelk_cbk,
-                    FIRST_CHILD (this),
-                    FIRST_CHILD (this)->fops->finodelk,
-                    volume, fd, cmd, flock, xdata);
-        return 0;
+    STACK_WIND(frame, trace_finodelk_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->finodelk, volume, fd, cmd, flock,
+               xdata);
+    return 0;
 }
 
 int
-trace_xattrop (call_frame_t *frame, xlator_t *this, loc_t *loc,
-               gf_xattrop_flags_t flags, dict_t *dict, dict_t *xdata)
+trace_xattrop(call_frame_t *frame, xlator_t *this, loc_t *loc,
+              gf_xattrop_flags_t flags, dict_t *dict, dict_t *xdata)
 {
-        trace_conf_t   *conf = NULL;
+    trace_conf_t *conf = NULL;
 
-        conf = this->private;
+    conf = this->private;
 
-        if (!conf->log_file && !conf->log_history)
-		goto out;
-        if (trace_fop_names[GF_FOP_XATTROP].enabled) {
-                char    string[4096]  =  {0,};
-                snprintf (string, sizeof (string),
-                          "%"PRId64": gfid=%s (path=%s flags=%d)",
-                          frame->root->unique,
-                          uuid_utoa (loc->inode->gfid), loc->path,
-                          flags);
+    if (!conf->log_file && !conf->log_history)
+        goto out;
+    if (trace_fop_names[GF_FOP_XATTROP].enabled) {
+        char string[4096] = {
+            0,
+        };
+        snprintf(string, sizeof(string),
+                 "%" PRId64 ": gfid=%s (path=%s flags=%d)", frame->root->unique,
+                 uuid_utoa(loc->inode->gfid), loc->path, flags);
 
-                frame->local = loc->inode->gfid;
+        frame->local = loc->inode->gfid;
 
-                LOG_ELEMENT (conf, string);
-        }
+        LOG_ELEMENT(conf, string);
+    }
 
 out:
-        STACK_WIND (frame, trace_xattrop_cbk,
-                    FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->xattrop,
-                    loc, flags, dict, xdata);
+    STACK_WIND(frame, trace_xattrop_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->xattrop, loc, flags, dict, xdata);
 
-        return 0;
+    return 0;
 }
 
 int
-trace_fxattrop (call_frame_t *frame, xlator_t *this, fd_t *fd,
-                gf_xattrop_flags_t flags, dict_t *dict, dict_t *xdata)
+trace_fxattrop(call_frame_t *frame, xlator_t *this, fd_t *fd,
+               gf_xattrop_flags_t flags, dict_t *dict, dict_t *xdata)
 {
-        trace_conf_t    *conf = NULL;
+    trace_conf_t *conf = NULL;
 
-        conf = this->private;
+    conf = this->private;
 
-        if (!conf->log_file && !conf->log_history)
-		goto out;
-        if (trace_fop_names[GF_FOP_FXATTROP].enabled) {
-                char    string[4096]  = {0,};
-                snprintf (string, sizeof (string),
-                          "%"PRId64": gfid=%s fd=%p, flags=%d",
-                          frame->root->unique,
-                          uuid_utoa (fd->inode->gfid), fd, flags);
+    if (!conf->log_file && !conf->log_history)
+        goto out;
+    if (trace_fop_names[GF_FOP_FXATTROP].enabled) {
+        char string[4096] = {
+            0,
+        };
+        snprintf(string, sizeof(string), "%" PRId64 ": gfid=%s fd=%p, flags=%d",
+                 frame->root->unique, uuid_utoa(fd->inode->gfid), fd, flags);
 
-                frame->local = fd->inode->gfid;
+        frame->local = fd->inode->gfid;
 
-                LOG_ELEMENT (conf, string);
-        }
+        LOG_ELEMENT(conf, string);
+    }
 
 out:
-        STACK_WIND (frame, trace_fxattrop_cbk,
-                    FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->fxattrop,
-                    fd, flags, dict, xdata);
+    STACK_WIND(frame, trace_fxattrop_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->fxattrop, fd, flags, dict, xdata);
 
-        return 0;
+    return 0;
 }
 
 int
-trace_lookup (call_frame_t *frame, xlator_t *this,
-              loc_t *loc, dict_t *xdata)
+trace_lookup(call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xdata)
 {
-        trace_conf_t *conf = NULL;
+    trace_conf_t *conf = NULL;
 
-        conf = this->private;
+    conf = this->private;
 
-        if (!conf->log_file && !conf->log_history)
-		goto out;
-        if (trace_fop_names[GF_FOP_LOOKUP].enabled) {
-                char string[4096] = {0,};
-                /* TODO: print all the keys mentioned in xattr_req */
-                snprintf (string, sizeof (string),
-                          "%"PRId64": gfid=%s path=%s",
-                          frame->root->unique,
-                          uuid_utoa (loc->inode->gfid), loc->path);
+    if (!conf->log_file && !conf->log_history)
+        goto out;
+    if (trace_fop_names[GF_FOP_LOOKUP].enabled) {
+        char string[4096] = {
+            0,
+        };
+        /* TODO: print all the keys mentioned in xattr_req */
+        snprintf(string, sizeof(string), "%" PRId64 ": gfid=%s path=%s",
+                 frame->root->unique, uuid_utoa(loc->inode->gfid), loc->path);
 
-                frame->local = loc->inode->gfid;
+        frame->local = loc->inode->gfid;
 
-                LOG_ELEMENT (conf, string);
-        }
+        LOG_ELEMENT(conf, string);
+    }
 
 out:
-        STACK_WIND (frame, trace_lookup_cbk,
-                    FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->lookup,
-                    loc, xdata);
+    STACK_WIND(frame, trace_lookup_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->lookup, loc, xdata);
 
-        return 0;
+    return 0;
 }
 
 int
-trace_stat (call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xdata)
+trace_stat(call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xdata)
 {
-        trace_conf_t  *conf = NULL;
+    trace_conf_t *conf = NULL;
 
-        conf = this->private;
+    conf = this->private;
 
-        if (!conf->log_file && !conf->log_history)
-                goto out;
-        if (trace_fop_names[GF_FOP_STAT].enabled) {
-                char  string[4096]  = {0,};
-                snprintf (string, sizeof (string),
-                          "%"PRId64": gfid=%s path=%s",
-                          frame->root->unique,
-                          uuid_utoa (loc->inode->gfid), loc->path);
+    if (!conf->log_file && !conf->log_history)
+        goto out;
+    if (trace_fop_names[GF_FOP_STAT].enabled) {
+        char string[4096] = {
+            0,
+        };
+        snprintf(string, sizeof(string), "%" PRId64 ": gfid=%s path=%s",
+                 frame->root->unique, uuid_utoa(loc->inode->gfid), loc->path);
 
-                frame->local = loc->inode->gfid;
+        frame->local = loc->inode->gfid;
 
-                LOG_ELEMENT (conf, string);
-        }
+        LOG_ELEMENT(conf, string);
+    }
 
 out:
-        STACK_WIND (frame, trace_stat_cbk,
-                    FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->stat,
-                    loc, xdata);
+    STACK_WIND(frame, trace_stat_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->stat, loc, xdata);
 
-        return 0;
+    return 0;
 }
 
 int
-trace_readlink (call_frame_t *frame, xlator_t *this, loc_t *loc, size_t size,
-                dict_t *xdata)
+trace_readlink(call_frame_t *frame, xlator_t *this, loc_t *loc, size_t size,
+               dict_t *xdata)
 {
-        trace_conf_t   *conf = NULL;
+    trace_conf_t *conf = NULL;
 
-        conf = this->private;
+    conf = this->private;
 
-        if (!conf->log_file && !conf->log_history)
-		goto out;
-        if (trace_fop_names[GF_FOP_READLINK].enabled) {
-                char      string[4096] = {0,};
-                snprintf (string, sizeof (string),
-                          "%"PRId64": gfid=%s path=%s, "
-                          "size=%"GF_PRI_SIZET")", frame->root->unique,
-                          uuid_utoa (loc->inode->gfid), loc->path,
-                          size);
+    if (!conf->log_file && !conf->log_history)
+        goto out;
+    if (trace_fop_names[GF_FOP_READLINK].enabled) {
+        char string[4096] = {
+            0,
+        };
+        snprintf(string, sizeof(string),
+                 "%" PRId64
+                 ": gfid=%s path=%s, "
+                 "size=%" GF_PRI_SIZET ")",
+                 frame->root->unique, uuid_utoa(loc->inode->gfid), loc->path,
+                 size);
 
-                frame->local = loc->inode->gfid;
+        frame->local = loc->inode->gfid;
 
-                LOG_ELEMENT (conf, string);
-        }
+        LOG_ELEMENT(conf, string);
+    }
 
 out:
-        STACK_WIND (frame, trace_readlink_cbk,
-                    FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->readlink,
-                    loc, size, xdata);
+    STACK_WIND(frame, trace_readlink_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->readlink, loc, size, xdata);
 
-        return 0;
+    return 0;
 }
 
 int
-trace_mknod (call_frame_t *frame, xlator_t *this, loc_t *loc,
-             mode_t mode, dev_t dev, mode_t umask, dict_t *xdata)
+trace_mknod(call_frame_t *frame, xlator_t *this, loc_t *loc, mode_t mode,
+            dev_t dev, mode_t umask, dict_t *xdata)
 {
-        trace_conf_t  *conf = NULL;
+    trace_conf_t *conf = NULL;
 
-        conf = this->private;
+    conf = this->private;
 
-        if (!conf->log_file && !conf->log_history)
-                goto out;
-        if (trace_fop_names[GF_FOP_MKNOD].enabled) {
-                char     string[4096]  =  {0,};
-                snprintf (string, sizeof (string),
-                          "%"PRId64": gfid=%s path=%s mode=%d "
-                          "umask=0%o, dev=%"GF_PRI_DEV")",
-                          frame->root->unique,
-                          uuid_utoa (loc->inode->gfid), loc->path,
-                          mode, umask, dev);
+    if (!conf->log_file && !conf->log_history)
+        goto out;
+    if (trace_fop_names[GF_FOP_MKNOD].enabled) {
+        char string[4096] = {
+            0,
+        };
+        snprintf(string, sizeof(string),
+                 "%" PRId64
+                 ": gfid=%s path=%s mode=%d "
+                 "umask=0%o, dev=%" GF_PRI_DEV ")",
+                 frame->root->unique, uuid_utoa(loc->inode->gfid), loc->path,
+                 mode, umask, dev);
 
-                LOG_ELEMENT (conf, string);
-        }
+        LOG_ELEMENT(conf, string);
+    }
 
 out:
-        STACK_WIND (frame, trace_mknod_cbk,
-                    FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->mknod,
-                    loc, mode, dev, umask, xdata);
+    STACK_WIND(frame, trace_mknod_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->mknod, loc, mode, dev, umask, xdata);
 
-        return 0;
+    return 0;
 }
 
 int
-trace_mkdir (call_frame_t *frame, xlator_t *this, loc_t *loc, mode_t mode,
-             mode_t umask, dict_t *xdata)
+trace_mkdir(call_frame_t *frame, xlator_t *this, loc_t *loc, mode_t mode,
+            mode_t umask, dict_t *xdata)
 {
-        trace_conf_t   *conf = NULL;
+    trace_conf_t *conf = NULL;
 
-        conf = this->private;
+    conf = this->private;
 
-        if (!conf->log_file && !conf->log_history)
-		goto out;
-        if (trace_fop_names[GF_FOP_MKDIR].enabled) {
-                char     string[4096]  =  {0,};
-                snprintf (string, sizeof (string),
-                          "%"PRId64": gfid=%s path=%s mode=%d"
-                          " umask=0%o", frame->root->unique,
-                          uuid_utoa (loc->inode->gfid), loc->path,
-                          mode, umask);
+    if (!conf->log_file && !conf->log_history)
+        goto out;
+    if (trace_fop_names[GF_FOP_MKDIR].enabled) {
+        char string[4096] = {
+            0,
+        };
+        snprintf(string, sizeof(string),
+                 "%" PRId64
+                 ": gfid=%s path=%s mode=%d"
+                 " umask=0%o",
+                 frame->root->unique, uuid_utoa(loc->inode->gfid), loc->path,
+                 mode, umask);
 
-                LOG_ELEMENT (conf, string);
-        }
+        LOG_ELEMENT(conf, string);
+    }
 
 out:
-        STACK_WIND (frame, trace_mkdir_cbk,
-                    FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->mkdir,
-                    loc, mode, umask, xdata);
-        return 0;
+    STACK_WIND(frame, trace_mkdir_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->mkdir, loc, mode, umask, xdata);
+    return 0;
 }
 
 int
-trace_unlink (call_frame_t *frame, xlator_t *this, loc_t *loc, int xflag,
-              dict_t *xdata)
+trace_unlink(call_frame_t *frame, xlator_t *this, loc_t *loc, int xflag,
+             dict_t *xdata)
 {
-        trace_conf_t   *conf = NULL;
+    trace_conf_t *conf = NULL;
 
-        conf = this->private;
+    conf = this->private;
 
-        if (!conf->log_file && !conf->log_history)
-		goto out;
-        if (trace_fop_names[GF_FOP_UNLINK].enabled) {
-                char     string[4096]  =  {0,};
-                snprintf (string, sizeof (string),
-                          "%"PRId64": gfid=%s path=%s flag=%d",
-                          frame->root->unique,
-                          uuid_utoa (loc->inode->gfid), loc->path,
-                          xflag);
+    if (!conf->log_file && !conf->log_history)
+        goto out;
+    if (trace_fop_names[GF_FOP_UNLINK].enabled) {
+        char string[4096] = {
+            0,
+        };
+        snprintf(string, sizeof(string), "%" PRId64 ": gfid=%s path=%s flag=%d",
+                 frame->root->unique, uuid_utoa(loc->inode->gfid), loc->path,
+                 xflag);
 
-                frame->local = loc->inode->gfid;
+        frame->local = loc->inode->gfid;
 
-                LOG_ELEMENT (conf, string);
-        }
+        LOG_ELEMENT(conf, string);
+    }
 out:
-        STACK_WIND (frame, trace_unlink_cbk,
-                    FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->unlink,
-                    loc, xflag, xdata);
-        return 0;
+    STACK_WIND(frame, trace_unlink_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->unlink, loc, xflag, xdata);
+    return 0;
 }
 
 int
-trace_rmdir (call_frame_t *frame, xlator_t *this, loc_t *loc, int flags,
-             dict_t *xdata)
+trace_rmdir(call_frame_t *frame, xlator_t *this, loc_t *loc, int flags,
+            dict_t *xdata)
 {
-        trace_conf_t    *conf = NULL;
+    trace_conf_t *conf = NULL;
 
-        conf = this->private;
+    conf = this->private;
 
-        if (!conf->log_file && !conf->log_history)
-		goto out;
-        if (trace_fop_names[GF_FOP_RMDIR].enabled) {
-                char     string[4096]  =  {0,};
-                snprintf (string, sizeof (string),
-                          "%"PRId64": gfid=%s path=%s flags=%d",
-                          frame->root->unique,
-                          uuid_utoa (loc->inode->gfid), loc->path,
-                          flags);
+    if (!conf->log_file && !conf->log_history)
+        goto out;
+    if (trace_fop_names[GF_FOP_RMDIR].enabled) {
+        char string[4096] = {
+            0,
+        };
+        snprintf(string, sizeof(string),
+                 "%" PRId64 ": gfid=%s path=%s flags=%d", frame->root->unique,
+                 uuid_utoa(loc->inode->gfid), loc->path, flags);
 
-                frame->local = loc->inode->gfid;
+        frame->local = loc->inode->gfid;
 
-                LOG_ELEMENT (conf, string);
-        }
+        LOG_ELEMENT(conf, string);
+    }
 
 out:
-        STACK_WIND (frame, trace_rmdir_cbk,
-                    FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->rmdir,
-                    loc, flags, xdata);
+    STACK_WIND(frame, trace_rmdir_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->rmdir, loc, flags, xdata);
 
-        return 0;
+    return 0;
 }
 
 int
-trace_symlink (call_frame_t *frame, xlator_t *this, const char *linkpath,
-               loc_t *loc, mode_t umask, dict_t *xdata)
+trace_symlink(call_frame_t *frame, xlator_t *this, const char *linkpath,
+              loc_t *loc, mode_t umask, dict_t *xdata)
 {
-        trace_conf_t     *conf = NULL;
+    trace_conf_t *conf = NULL;
 
-        conf = this->private;
+    conf = this->private;
 
-        if (!conf->log_file && !conf->log_history)
-		goto out;
-        if (trace_fop_names[GF_FOP_SYMLINK].enabled) {
-                char     string[4096]   =  {0,};
-                snprintf (string, sizeof (string),
-                          "%"PRId64": gfid=%s linkpath=%s, path=%s"
-                          " umask=0%o", frame->root->unique,
-                          uuid_utoa (loc->inode->gfid), linkpath,
-                          loc->path, umask);
+    if (!conf->log_file && !conf->log_history)
+        goto out;
+    if (trace_fop_names[GF_FOP_SYMLINK].enabled) {
+        char string[4096] = {
+            0,
+        };
+        snprintf(string, sizeof(string),
+                 "%" PRId64
+                 ": gfid=%s linkpath=%s, path=%s"
+                 " umask=0%o",
+                 frame->root->unique, uuid_utoa(loc->inode->gfid), linkpath,
+                 loc->path, umask);
 
-                LOG_ELEMENT (conf, string);
-        }
+        LOG_ELEMENT(conf, string);
+    }
 
 out:
-        STACK_WIND (frame, trace_symlink_cbk,
-                    FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->symlink,
-                    linkpath, loc, umask, xdata);
+    STACK_WIND(frame, trace_symlink_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->symlink, linkpath, loc, umask, xdata);
 
-        return 0;
+    return 0;
 }
 
 int
-trace_rename (call_frame_t *frame, xlator_t *this, loc_t *oldloc, loc_t *newloc,
-              dict_t *xdata)
+trace_rename(call_frame_t *frame, xlator_t *this, loc_t *oldloc, loc_t *newloc,
+             dict_t *xdata)
 {
-        char         oldgfid[50] = {0,};
-        char         newgfid[50] = {0,};
-        trace_conf_t *conf       = NULL;
+    char oldgfid[50] = {
+        0,
+    };
+    char newgfid[50] = {
+        0,
+    };
+    trace_conf_t *conf = NULL;
 
-        conf = this->private;
+    conf = this->private;
 
-        if (!conf->log_file && !conf->log_history)
-		goto out;
-        if (trace_fop_names[GF_FOP_RENAME].enabled) {
-                char string[4096] = {0,};
-                if (newloc->inode)
-                        uuid_utoa_r (newloc->inode->gfid, newgfid);
-                else
-                        strcpy (newgfid, "0");
+    if (!conf->log_file && !conf->log_history)
+        goto out;
+    if (trace_fop_names[GF_FOP_RENAME].enabled) {
+        char string[4096] = {
+            0,
+        };
+        if (newloc->inode)
+            uuid_utoa_r(newloc->inode->gfid, newgfid);
+        else
+            strcpy(newgfid, "0");
 
-                uuid_utoa_r (oldloc->inode->gfid, oldgfid);
+        uuid_utoa_r(oldloc->inode->gfid, oldgfid);
 
-                snprintf (string, sizeof (string),
-                          "%"PRId64": oldgfid=%s oldpath=%s --> "
-                          "newgfid=%s newpath=%s",
-                          frame->root->unique, oldgfid,
-                          oldloc->path, newgfid, newloc->path);
+        snprintf(string, sizeof(string),
+                 "%" PRId64
+                 ": oldgfid=%s oldpath=%s --> "
+                 "newgfid=%s newpath=%s",
+                 frame->root->unique, oldgfid, oldloc->path, newgfid,
+                 newloc->path);
 
-                frame->local = oldloc->inode->gfid;
+        frame->local = oldloc->inode->gfid;
 
-                LOG_ELEMENT (conf, string);
-        }
+        LOG_ELEMENT(conf, string);
+    }
 
 out:
-        STACK_WIND (frame, trace_rename_cbk,
-                    FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->rename,
-                    oldloc, newloc, xdata);
+    STACK_WIND(frame, trace_rename_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->rename, oldloc, newloc, xdata);
 
-        return 0;
+    return 0;
 }
 
 int
-trace_link (call_frame_t *frame, xlator_t *this, loc_t *oldloc, loc_t *newloc,
-            dict_t *xdata)
+trace_link(call_frame_t *frame, xlator_t *this, loc_t *oldloc, loc_t *newloc,
+           dict_t *xdata)
 {
-        char         oldgfid[50] = {0,};
-        char         newgfid[50] = {0,};
-        trace_conf_t *conf       = NULL;
+    char oldgfid[50] = {
+        0,
+    };
+    char newgfid[50] = {
+        0,
+    };
+    trace_conf_t *conf = NULL;
 
-        conf = this->private;
+    conf = this->private;
 
-        if (!conf->log_file && !conf->log_history)
-		goto out;
-        if (trace_fop_names[GF_FOP_LINK].enabled) {
-                char string[4096]  = {0,};
-                if (newloc->inode)
-                        uuid_utoa_r (newloc->inode->gfid, newgfid);
-                else
-                        strcpy (newgfid, "0");
+    if (!conf->log_file && !conf->log_history)
+        goto out;
+    if (trace_fop_names[GF_FOP_LINK].enabled) {
+        char string[4096] = {
+            0,
+        };
+        if (newloc->inode)
+            uuid_utoa_r(newloc->inode->gfid, newgfid);
+        else
+            strcpy(newgfid, "0");
 
-                uuid_utoa_r (oldloc->inode->gfid, oldgfid);
+        uuid_utoa_r(oldloc->inode->gfid, oldgfid);
 
-                snprintf (string, sizeof (string),
-                          "%"PRId64": oldgfid=%s oldpath=%s --> "
-                          "newgfid=%s newpath=%s", frame->root->unique,
-                          oldgfid, oldloc->path, newgfid,
-                          newloc->path);
+        snprintf(string, sizeof(string),
+                 "%" PRId64
+                 ": oldgfid=%s oldpath=%s --> "
+                 "newgfid=%s newpath=%s",
+                 frame->root->unique, oldgfid, oldloc->path, newgfid,
+                 newloc->path);
 
-                frame->local = oldloc->inode->gfid;
+        frame->local = oldloc->inode->gfid;
 
-                LOG_ELEMENT (conf, string);
-        }
+        LOG_ELEMENT(conf, string);
+    }
 
 out:
-        STACK_WIND (frame, trace_link_cbk,
-                    FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->link,
-                    oldloc, newloc, xdata);
-        return 0;
+    STACK_WIND(frame, trace_link_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->link, oldloc, newloc, xdata);
+    return 0;
 }
 
 int
-trace_setattr (call_frame_t *frame, xlator_t *this, loc_t *loc,
-               struct iatt *stbuf, int32_t valid, dict_t *xdata)
+trace_setattr(call_frame_t *frame, xlator_t *this, loc_t *loc,
+              struct iatt *stbuf, int32_t valid, dict_t *xdata)
 {
-        char         actime_str[256]  = {0,};
-        char         modtime_str[256] = {0,};
-        trace_conf_t *conf            = NULL;
-
-        conf = this->private;
-
-        if (!conf->log_file && !conf->log_history)
-		goto out;
-        if (trace_fop_names[GF_FOP_SETATTR].enabled) {
-                char     string[4096]  =  {0,};
-                if (valid & GF_SET_ATTR_MODE) {
-                        snprintf (string, sizeof (string),
-                                  "%"PRId64": gfid=%s path=%s mode=%o)",
-                                  frame->root->unique,
-                                  uuid_utoa (loc->inode->gfid),
-                                  loc->path,
-                                  st_mode_from_ia (stbuf->ia_prot,
-                                                   stbuf->ia_type));
-
-                        LOG_ELEMENT (conf, string);
-                        memset (string, 0 , sizeof (string));
-                }
-
-                if (valid & (GF_SET_ATTR_UID | GF_SET_ATTR_GID)) {
-                        snprintf (string, sizeof (string),
-                                  "%"PRId64": gfid=%s path=%s uid=%o,"
-                                  " gid=%o", frame->root->unique,
-                                  uuid_utoa (loc->inode->gfid),
-                                  loc->path, stbuf->ia_uid,
-                                  stbuf->ia_gid);
-
-                        LOG_ELEMENT (conf, string);
-                        memset (string, 0 , sizeof (string));
-                }
-
-                if (valid & (GF_SET_ATTR_ATIME | GF_SET_ATTR_MTIME)) {
-                        gf_time_fmt (actime_str, sizeof actime_str,
-                                     stbuf->ia_atime, gf_timefmt_bdT);
-
-                        gf_time_fmt (modtime_str, sizeof modtime_str,
-                                     stbuf->ia_mtime, gf_timefmt_bdT);
-
-                        snprintf (string, sizeof (string),
-                                  "%"PRId64": gfid=%s path=%s "
-                                  "ia_atime=%s, ia_mtime=%s",
-                                  frame->root->unique,
-                                  uuid_utoa (loc->inode->gfid),
-                                  loc->path, actime_str, modtime_str);
-
-                        LOG_ELEMENT (conf, string);
-                        memset (string, 0 , sizeof (string));
-                }
-                frame->local = loc->inode->gfid;
+    char actime_str[GF_TIMESTR_SIZE] = {
+        0,
+    };
+    char modtime_str[GF_TIMESTR_SIZE] = {
+        0,
+    };
+    trace_conf_t *conf = NULL;
+
+    conf = this->private;
+
+    if (!conf->log_file && !conf->log_history)
+        goto out;
+    if (trace_fop_names[GF_FOP_SETATTR].enabled) {
+        char string[4096] = {
+            0,
+        };
+        if (valid & GF_SET_ATTR_MODE) {
+            snprintf(
+                string, sizeof(string), "%" PRId64 ": gfid=%s path=%s mode=%o)",
+                frame->root->unique, uuid_utoa(loc->inode->gfid), loc->path,
+                st_mode_from_ia(stbuf->ia_prot, stbuf->ia_type));
+
+            LOG_ELEMENT(conf, string);
+            memset(string, 0, sizeof(string));
+        }
+
+        if (valid & (GF_SET_ATTR_UID | GF_SET_ATTR_GID)) {
+            snprintf(string, sizeof(string),
+                     "%" PRId64
+                     ": gfid=%s path=%s uid=%o,"
+                     " gid=%o",
+                     frame->root->unique, uuid_utoa(loc->inode->gfid),
+                     loc->path, stbuf->ia_uid, stbuf->ia_gid);
+
+            LOG_ELEMENT(conf, string);
+            memset(string, 0, sizeof(string));
+        }
+
+        if (valid & (GF_SET_ATTR_ATIME | GF_SET_ATTR_MTIME)) {
+            gf_time_fmt(actime_str, sizeof actime_str, stbuf->ia_atime,
+                        gf_timefmt_bdT);
+
+            gf_time_fmt(modtime_str, sizeof modtime_str, stbuf->ia_mtime,
+                        gf_timefmt_bdT);
+
+            snprintf(string, sizeof(string),
+                     "%" PRId64
+                     ": gfid=%s path=%s "
+                     "ia_atime=%s, ia_mtime=%s",
+                     frame->root->unique, uuid_utoa(loc->inode->gfid),
+                     loc->path, actime_str, modtime_str);
+
+            LOG_ELEMENT(conf, string);
+            memset(string, 0, sizeof(string));
         }
+        frame->local = loc->inode->gfid;
+    }
 
 out:
-        STACK_WIND (frame, trace_setattr_cbk,
-                    FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->setattr,
-                    loc, stbuf, valid, xdata);
+    STACK_WIND(frame, trace_setattr_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->setattr, loc, stbuf, valid, xdata);
 
-        return 0;
+    return 0;
 }
 
 int
-trace_fsetattr (call_frame_t *frame, xlator_t *this, fd_t *fd,
-                struct iatt *stbuf, int32_t valid, dict_t *xdata)
+trace_fsetattr(call_frame_t *frame, xlator_t *this, fd_t *fd,
+               struct iatt *stbuf, int32_t valid, dict_t *xdata)
 {
-        char           actime_str[256]  = {0,};
-        char           modtime_str[256] = {0,};
-        trace_conf_t   *conf            = NULL;
+    char actime_str[GF_TIMESTR_SIZE] = {
+        0,
+    };
+    char modtime_str[GF_TIMESTR_SIZE] = {
+        0,
+    };
+    trace_conf_t *conf = NULL;
+
+    conf = this->private;
+
+    if (!conf->log_file && !conf->log_history)
+        goto out;
+    if (trace_fop_names[GF_FOP_FSETATTR].enabled) {
+        char string[4096] = {
+            0,
+        };
+        if (valid & GF_SET_ATTR_MODE) {
+            snprintf(string, sizeof(string),
+                     "%" PRId64 ": gfid=%s fd=%p, mode=%o", frame->root->unique,
+                     uuid_utoa(fd->inode->gfid), fd,
+                     st_mode_from_ia(stbuf->ia_prot, stbuf->ia_type));
 
-        conf = this->private;
+            LOG_ELEMENT(conf, string);
+            memset(string, 0, sizeof(string));
+        }
 
-        if (!conf->log_file && !conf->log_history)
-		goto out;
-        if (trace_fop_names[GF_FOP_FSETATTR].enabled) {
-                char     string[4096]  =  {0,};
-                if (valid & GF_SET_ATTR_MODE) {
-                        snprintf (string, sizeof (string),
-                                  "%"PRId64": gfid=%s fd=%p, mode=%o",
-                                  frame->root->unique,
-                                  uuid_utoa (fd->inode->gfid), fd,
-                                  st_mode_from_ia (stbuf->ia_prot,
-                                                   stbuf->ia_type));
+        if (valid & (GF_SET_ATTR_UID | GF_SET_ATTR_GID)) {
+            snprintf(string, sizeof(string),
+                     "%" PRId64
+                     ": gfid=%s fd=%p, uid=%o, "
+                     "gid=%o",
+                     frame->root->unique, uuid_utoa(fd->inode->gfid), fd,
+                     stbuf->ia_uid, stbuf->ia_gid);
 
-                        LOG_ELEMENT (conf, string);
-                        memset (string, 0, sizeof (string));
-                }
+            LOG_ELEMENT(conf, string);
+            memset(string, 0, sizeof(string));
+        }
 
-                if (valid & (GF_SET_ATTR_UID | GF_SET_ATTR_GID)) {
-                        snprintf (string, sizeof (string),
-                                  "%"PRId64": gfid=%s fd=%p, uid=%o, "
-                                  "gid=%o", frame->root->unique,
-                                  uuid_utoa (fd->inode->gfid),
-                                  fd, stbuf->ia_uid, stbuf->ia_gid);
+        if (valid & (GF_SET_ATTR_ATIME | GF_SET_ATTR_MTIME)) {
+            gf_time_fmt(actime_str, sizeof actime_str, stbuf->ia_atime,
+                        gf_timefmt_bdT);
 
-                        LOG_ELEMENT (conf, string);
-                        memset (string, 0, sizeof (string));
-                }
+            gf_time_fmt(modtime_str, sizeof modtime_str, stbuf->ia_mtime,
+                        gf_timefmt_bdT);
 
-                if (valid & (GF_SET_ATTR_ATIME | GF_SET_ATTR_MTIME)) {
-                        gf_time_fmt (actime_str, sizeof actime_str,
-                                     stbuf->ia_atime, gf_timefmt_bdT);
+            snprintf(string, sizeof(string),
+                     "%" PRId64
+                     ": gfid=%s fd=%p "
+                     "ia_atime=%s, ia_mtime=%s",
+                     frame->root->unique, uuid_utoa(fd->inode->gfid), fd,
+                     actime_str, modtime_str);
 
-                        gf_time_fmt (modtime_str, sizeof modtime_str,
-                                     stbuf->ia_mtime, gf_timefmt_bdT);
+            LOG_ELEMENT(conf, string);
+            memset(string, 0, sizeof(string));
+        }
+        frame->local = fd->inode->gfid;
+    }
 
-                        snprintf (string, sizeof (string),
-                                  "%"PRId64": gfid=%s fd=%p "
-                                  "ia_atime=%s, ia_mtime=%s",
-                                  frame->root->unique,
-                                  uuid_utoa (fd->inode->gfid),
-                                  fd, actime_str, modtime_str);
+out:
+    STACK_WIND(frame, trace_fsetattr_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->fsetattr, fd, stbuf, valid, xdata);
 
-                        LOG_ELEMENT (conf, string);
-                        memset (string, 0, sizeof (string));
-                }
-                frame->local = fd->inode->gfid;
-        }
+    return 0;
+}
+
+static int
+trace_seek_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+               int32_t op_ret, int32_t op_errno, off_t offset, dict_t *xdata)
+{
+    trace_conf_t *conf = this->private;
+
+    if (!conf->log_file && !conf->log_history)
+        goto out;
+    if (trace_fop_names[GF_FOP_SEEK].enabled) {
+        char string[4096] = {
+            0,
+        };
+        snprintf(string, sizeof(string),
+                 "%" PRId64
+                 ": gfid=%s op_ret=%d op_errno=%d, "
+                 "offset=%" PRId64 "",
+                 frame->root->unique, uuid_utoa(frame->local), op_ret, op_errno,
+                 offset);
+        LOG_ELEMENT(conf, string);
+    }
+out:
+    TRACE_STACK_UNWIND(seek, frame, op_ret, op_errno, offset, xdata);
+    return 0;
+}
+
+static int
+trace_seek(call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
+           gf_seek_what_t what, dict_t *xdata)
+{
+    trace_conf_t *conf = this->private;
 
+    if (!conf->log_file && !conf->log_history)
+        goto out;
+    if (trace_fop_names[GF_FOP_SEEK].enabled) {
+        char string[4096] = {
+            0,
+        };
+        snprintf(string, sizeof(string),
+                 "%" PRId64
+                 ": gfid=%s fd=%p "
+                 "offset=%" PRId64 " what=%d",
+                 frame->root->unique, uuid_utoa(fd->inode->gfid), fd, offset,
+                 what);
+
+        frame->local = fd->inode->gfid;
+        LOG_ELEMENT(conf, string);
+    }
 out:
-        STACK_WIND (frame, trace_fsetattr_cbk,
-                    FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->fsetattr,
-                    fd, stbuf, valid, xdata);
+    STACK_WIND(frame, trace_seek_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->seek, fd, offset, what, xdata);
 
-        return 0;
+    return 0;
 }
 
 int
-trace_truncate (call_frame_t *frame, xlator_t *this, loc_t *loc,
-                off_t offset, dict_t *xdata)
+trace_truncate(call_frame_t *frame, xlator_t *this, loc_t *loc, off_t offset,
+               dict_t *xdata)
 {
-        trace_conf_t   *conf = NULL;
+    trace_conf_t *conf = NULL;
 
-        conf = this->private;
+    conf = this->private;
 
-        if (!conf->log_file && !conf->log_history)
-		goto out;
-        if (trace_fop_names[GF_FOP_TRUNCATE].enabled) {
-                char     string[4096]  =  {0,};
-                snprintf (string, sizeof (string),
-                          "%"PRId64": gfid=%s path=%s, "
-                          "offset=%"PRId64"", frame->root->unique,
-                          uuid_utoa (loc->inode->gfid), loc->path,
-                          offset);
+    if (!conf->log_file && !conf->log_history)
+        goto out;
+    if (trace_fop_names[GF_FOP_TRUNCATE].enabled) {
+        char string[4096] = {
+            0,
+        };
+        snprintf(string, sizeof(string),
+                 "%" PRId64
+                 ": gfid=%s path=%s, "
+                 "offset=%" PRId64 "",
+                 frame->root->unique, uuid_utoa(loc->inode->gfid), loc->path,
+                 offset);
 
-                frame->local = loc->inode->gfid;
+        frame->local = loc->inode->gfid;
 
-                LOG_ELEMENT (conf, string);
-        }
+        LOG_ELEMENT(conf, string);
+    }
 
 out:
-        STACK_WIND (frame, trace_truncate_cbk,
-                    FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->truncate,
-                    loc, offset, xdata);
+    STACK_WIND(frame, trace_truncate_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->truncate, loc, offset, xdata);
 
-        return 0;
+    return 0;
 }
 
 int
-trace_open (call_frame_t *frame, xlator_t *this, loc_t *loc,
-            int32_t flags, fd_t *fd, dict_t *xdata)
+trace_open(call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t flags,
+           fd_t *fd, dict_t *xdata)
 {
-        trace_conf_t   *conf = NULL;
+    trace_conf_t *conf = NULL;
 
-        conf = this->private;
+    conf = this->private;
 
-        if (!conf->log_file && !conf->log_history)
-		goto out;
-        if (trace_fop_names[GF_FOP_OPEN].enabled) {
-                char      string[4096]  =  {0,};
-                snprintf (string, sizeof (string),
-                          "%"PRId64": gfid=%s path=%s flags=%d fd=%p",
-                          frame->root->unique,
-                          uuid_utoa (loc->inode->gfid), loc->path,
-                          flags, fd);
+    if (!conf->log_file && !conf->log_history)
+        goto out;
+    if (trace_fop_names[GF_FOP_OPEN].enabled) {
+        char string[4096] = {
+            0,
+        };
+        snprintf(string, sizeof(string),
+                 "%" PRId64 ": gfid=%s path=%s flags=%d fd=%p",
+                 frame->root->unique, uuid_utoa(loc->inode->gfid), loc->path,
+                 flags, fd);
 
-                frame->local = loc->inode->gfid;
+        frame->local = loc->inode->gfid;
 
-                LOG_ELEMENT (conf, string);
-        }
+        LOG_ELEMENT(conf, string);
+    }
 
 out:
-        STACK_WIND (frame, trace_open_cbk,
-                    FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->open,
-                    loc, flags, fd, xdata);
-        return 0;
+    STACK_WIND(frame, trace_open_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->open, loc, flags, fd, xdata);
+    return 0;
 }
 
 int
-trace_create (call_frame_t *frame, xlator_t *this, loc_t *loc,
-              int32_t flags, mode_t mode, mode_t umask, fd_t *fd,
-              dict_t *xdata)
+trace_create(call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t flags,
+             mode_t mode, mode_t umask, fd_t *fd, dict_t *xdata)
 {
-        trace_conf_t   *conf = NULL;
+    trace_conf_t *conf = NULL;
 
-        conf = this->private;
+    conf = this->private;
 
-        if (!conf->log_file && !conf->log_history)
-		goto out;
-        if (trace_fop_names[GF_FOP_CREATE].enabled) {
-                char    string[4096]  =  {0,};
-                snprintf (string, sizeof (string),
-                          "%"PRId64": gfid=%s path=%s, fd=%p, "
-                          "flags=0%o mode=0%o umask=0%o",
-                          frame->root->unique,
-                          uuid_utoa (loc->inode->gfid), loc->path,
-                          fd, flags, mode, umask);
+    if (!conf->log_file && !conf->log_history)
+        goto out;
+    if (trace_fop_names[GF_FOP_CREATE].enabled) {
+        char string[4096] = {
+            0,
+        };
+        snprintf(string, sizeof(string),
+                 "%" PRId64
+                 ": gfid=%s path=%s, fd=%p, "
+                 "flags=0%o mode=0%o umask=0%o",
+                 frame->root->unique, uuid_utoa(loc->inode->gfid), loc->path,
+                 fd, flags, mode, umask);
 
-                LOG_ELEMENT (conf, string);
-        }
+        LOG_ELEMENT(conf, string);
+    }
 
 out:
-        STACK_WIND (frame, trace_create_cbk,
-                    FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->create,
-                    loc, flags, mode, umask, fd, xdata);
-        return 0;
+    STACK_WIND(frame, trace_create_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->create, loc, flags, mode, umask, fd,
+               xdata);
+    return 0;
 }
 
 int
-trace_readv (call_frame_t *frame, xlator_t *this, fd_t *fd,
-             size_t size, off_t offset, uint32_t flags, dict_t *xdata)
+trace_readv(call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
+            off_t offset, uint32_t flags, dict_t *xdata)
 {
-        trace_conf_t  *conf = NULL;
+    trace_conf_t *conf = NULL;
 
-        conf = this->private;
+    conf = this->private;
 
-        if (!conf->log_file && !conf->log_history)
-		goto out;
-        if (trace_fop_names[GF_FOP_READ].enabled) {
-                char     string[4096]  =  {0,};
-                snprintf (string, sizeof (string),
-                          "%"PRId64": gfid=%s fd=%p, size=%"
-                          GF_PRI_SIZET"offset=%"PRId64" flags=0%x)",
-                          frame->root->unique,
-                          uuid_utoa (fd->inode->gfid), fd, size,
-                          offset, flags);
+    if (!conf->log_file && !conf->log_history)
+        goto out;
+    if (trace_fop_names[GF_FOP_READ].enabled) {
+        char string[4096] = {
+            0,
+        };
+        snprintf(string, sizeof(string),
+                 "%" PRId64 ": gfid=%s fd=%p, size=%" GF_PRI_SIZET
+                 "offset=%" PRId64 " flags=0%x)",
+                 frame->root->unique, uuid_utoa(fd->inode->gfid), fd, size,
+                 offset, flags);
 
-                frame->local = fd->inode->gfid;
+        frame->local = fd->inode->gfid;
 
-                LOG_ELEMENT (conf, string);
-        }
+        LOG_ELEMENT(conf, string);
+    }
 
 out:
-        STACK_WIND (frame, trace_readv_cbk,
-                    FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->readv,
-                    fd, size, offset, flags, xdata);
-        return 0;
+    STACK_WIND(frame, trace_readv_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->readv, fd, size, offset, flags, xdata);
+    return 0;
 }
 
 int
-trace_writev (call_frame_t *frame, xlator_t *this, fd_t *fd,
-              struct iovec *vector, int32_t count,
-              off_t offset, uint32_t flags, struct iobref *iobref, dict_t *xdata)
+trace_writev(call_frame_t *frame, xlator_t *this, fd_t *fd,
+             struct iovec *vector, int32_t count, off_t offset, uint32_t flags,
+             struct iobref *iobref, dict_t *xdata)
 {
-        trace_conf_t    *conf = NULL;
+    trace_conf_t *conf = NULL;
+    int i = 0;
+    size_t total_size = 0;
 
-        conf = this->private;
+    conf = this->private;
 
-        if (!conf->log_file && !conf->log_history)
-		goto out;
-        if (trace_fop_names[GF_FOP_WRITE].enabled) {
-                char     string[4096]  =  {0,};
-                snprintf (string, sizeof (string),
-                          "%"PRId64": gfid=%s fd=%p, count=%d, "
-                          " offset=%"PRId64" flags=0%x)",
-                          frame->root->unique,
-                          uuid_utoa (fd->inode->gfid), fd, count,
-                          offset, flags);
+    if (!conf->log_file && !conf->log_history)
+        goto out;
+    if (trace_fop_names[GF_FOP_WRITE].enabled) {
+        char string[4096] = {
+            0,
+        };
+        for (i = 0; i < count; i++)
+            total_size += vector[i].iov_len;
 
-                frame->local = fd->inode->gfid;
+        snprintf(string, sizeof(string),
+                 "%" PRId64
+                 ": gfid=%s fd=%p, count=%d, "
+                 " offset=%" PRId64 " flags=0%x write_size=%zu",
+                 frame->root->unique, uuid_utoa(fd->inode->gfid), fd, count,
+                 offset, flags, total_size);
 
-                LOG_ELEMENT (conf, string);
-        }
+        frame->local = fd->inode->gfid;
+
+        LOG_ELEMENT(conf, string);
+    }
 
 out:
-        STACK_WIND (frame, trace_writev_cbk,
-                    FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->writev,
-                    fd, vector, count, offset, flags, iobref, xdata);
-        return 0;
+    STACK_WIND(frame, trace_writev_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->writev, fd, vector, count, offset,
+               flags, iobref, xdata);
+    return 0;
 }
 
 int
-trace_statfs (call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xdata)
+trace_statfs(call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xdata)
 {
-        trace_conf_t  *conf = NULL;
+    trace_conf_t *conf = NULL;
 
-        conf = this->private;
+    conf = this->private;
 
-        if (!conf->log_file && !conf->log_history)
-		goto out;
-        if (trace_fop_names[GF_FOP_STATFS].enabled) {
-                char  string[4096]  =  {0,};
-                snprintf (string, sizeof (string),
-                          "%"PRId64": gfid=%s path=%s",
-                          frame->root->unique, (loc->inode)?
-                          uuid_utoa (loc->inode->gfid):"0", loc->path);
+    if (!conf->log_file && !conf->log_history)
+        goto out;
+    if (trace_fop_names[GF_FOP_STATFS].enabled) {
+        char string[4096] = {
+            0,
+        };
+        snprintf(string, sizeof(string), "%" PRId64 ": gfid=%s path=%s",
+                 frame->root->unique,
+                 (loc->inode) ? uuid_utoa(loc->inode->gfid) : "0", loc->path);
 
-                LOG_ELEMENT (conf, string);
-        }
+        LOG_ELEMENT(conf, string);
+    }
 
 out:
-        STACK_WIND (frame, trace_statfs_cbk,
-                    FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->statfs,
-                    loc, xdata);
-        return 0;
+    STACK_WIND(frame, trace_statfs_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->statfs, loc, xdata);
+    return 0;
 }
 
 int
-trace_flush (call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *xdata)
+trace_flush(call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *xdata)
 {
-        trace_conf_t  *conf = NULL;
+    trace_conf_t *conf = NULL;
 
-        conf = this->private;
+    conf = this->private;
 
-        if (!conf->log_file && !conf->log_history)
-		goto out;
-        if (trace_fop_names[GF_FOP_FLUSH].enabled) {
-                char    string[4096]  =  {0,};
-                snprintf (string, sizeof (string),
-                          "%"PRId64": gfid=%s fd=%p",
-                          frame->root->unique,
-                          uuid_utoa (fd->inode->gfid), fd);
+    if (!conf->log_file && !conf->log_history)
+        goto out;
+    if (trace_fop_names[GF_FOP_FLUSH].enabled) {
+        char string[4096] = {
+            0,
+        };
+        snprintf(string, sizeof(string), "%" PRId64 ": gfid=%s fd=%p",
+                 frame->root->unique, uuid_utoa(fd->inode->gfid), fd);
 
-                frame->local = fd->inode->gfid;
+        frame->local = fd->inode->gfid;
 
-                LOG_ELEMENT (conf, string);
-        }
+        LOG_ELEMENT(conf, string);
+    }
 
 out:
-        STACK_WIND (frame, trace_flush_cbk,
-                    FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->flush,
-                    fd, xdata);
-        return 0;
+    STACK_WIND(frame, trace_flush_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->flush, fd, xdata);
+    return 0;
 }
 
 int
-trace_fsync (call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t flags,
-             dict_t *xdata)
+trace_fsync(call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t flags,
+            dict_t *xdata)
 {
-        trace_conf_t   *conf = NULL;
+    trace_conf_t *conf = NULL;
 
-        conf = this->private;
+    conf = this->private;
 
-        if (!conf->log_file && !conf->log_history)
-		goto out;
-        if (trace_fop_names[GF_FOP_FSYNC].enabled) {
-                char     string[4096]  =  {0,};
-                snprintf (string, sizeof (string),
-                          "%"PRId64": gfid=%s flags=%d fd=%p",
-                          frame->root->unique,
-                          uuid_utoa (fd->inode->gfid), flags, fd);
+    if (!conf->log_file && !conf->log_history)
+        goto out;
+    if (trace_fop_names[GF_FOP_FSYNC].enabled) {
+        char string[4096] = {
+            0,
+        };
+        snprintf(string, sizeof(string), "%" PRId64 ": gfid=%s flags=%d fd=%p",
+                 frame->root->unique, uuid_utoa(fd->inode->gfid), flags, fd);
 
-                frame->local = fd->inode->gfid;
+        frame->local = fd->inode->gfid;
 
-                LOG_ELEMENT (conf, string);
-        }
+        LOG_ELEMENT(conf, string);
+    }
 
 out:
-        STACK_WIND (frame, trace_fsync_cbk,
-                    FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->fsync,
-                    fd, flags, xdata);
-        return 0;
+    STACK_WIND(frame, trace_fsync_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->fsync, fd, flags, xdata);
+    return 0;
 }
 
 int
-trace_setxattr (call_frame_t *frame, xlator_t *this,
-                loc_t *loc, dict_t *dict, int32_t flags, dict_t *xdata)
+trace_setxattr(call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *dict,
+               int32_t flags, dict_t *xdata)
 {
-        trace_conf_t    *conf = NULL;
+    trace_conf_t *conf = NULL;
 
-        conf = this->private;
+    conf = this->private;
 
-        if (!conf->log_file && !conf->log_history)
-		goto out;
-        if (trace_fop_names[GF_FOP_SETXATTR].enabled) {
-                char     string[4096]  =  {0,};
-                snprintf (string, sizeof (string),
-                          "%"PRId64": gfid=%s path=%s flags=%d",
-                          frame->root->unique,
-                          uuid_utoa (loc->inode->gfid), loc->path,
-                          flags);
+    if (!conf->log_file && !conf->log_history)
+        goto out;
+    if (trace_fop_names[GF_FOP_SETXATTR].enabled) {
+        char string[4096] = {
+            0,
+        };
+        snprintf(string, sizeof(string),
+                 "%" PRId64 ": gfid=%s path=%s flags=%d", frame->root->unique,
+                 uuid_utoa(loc->inode->gfid), loc->path, flags);
 
-                frame->local = loc->inode->gfid;
+        frame->local = loc->inode->gfid;
 
-                LOG_ELEMENT (conf, string);
-        }
+        LOG_ELEMENT(conf, string);
+    }
 
 out:
-        STACK_WIND (frame, trace_setxattr_cbk,
-                    FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->setxattr,
-                    loc, dict, flags, xdata);
-        return 0;
+    STACK_WIND(frame, trace_setxattr_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->setxattr, loc, dict, flags, xdata);
+    return 0;
 }
 
 int
-trace_getxattr (call_frame_t *frame, xlator_t *this,
-                loc_t *loc, const char *name, dict_t *xdata)
+trace_getxattr(call_frame_t *frame, xlator_t *this, loc_t *loc,
+               const char *name, dict_t *xdata)
 {
-        trace_conf_t   *conf = NULL;
+    trace_conf_t *conf = NULL;
 
-        conf = this->private;
+    conf = this->private;
 
-        if (!conf->log_file && !conf->log_history)
-		goto out;
-        if (trace_fop_names[GF_FOP_GETXATTR].enabled) {
-                char    string[4096]  =  {0,};
-                snprintf (string, sizeof (string),
-                          "%"PRId64": gfid=%s path=%s name=%s",
-                          frame->root->unique,
-                          uuid_utoa (loc->inode->gfid), loc->path,
-                          name);
+    if (!conf->log_file && !conf->log_history)
+        goto out;
+    if (trace_fop_names[GF_FOP_GETXATTR].enabled) {
+        char string[4096] = {
+            0,
+        };
+        snprintf(string, sizeof(string), "%" PRId64 ": gfid=%s path=%s name=%s",
+                 frame->root->unique, uuid_utoa(loc->inode->gfid), loc->path,
+                 name);
 
-                frame->local = loc->inode->gfid;
+        frame->local = loc->inode->gfid;
 
-                LOG_ELEMENT (conf, string);
-        }
+        LOG_ELEMENT(conf, string);
+    }
 
 out:
-        STACK_WIND (frame, trace_getxattr_cbk,
-                    FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->getxattr,
-                    loc, name, xdata);
-        return 0;
+    STACK_WIND(frame, trace_getxattr_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->getxattr, loc, name, xdata);
+    return 0;
 }
 
 int
-trace_removexattr (call_frame_t *frame, xlator_t *this,
-                   loc_t *loc, const char *name, dict_t *xdata)
+trace_removexattr(call_frame_t *frame, xlator_t *this, loc_t *loc,
+                  const char *name, dict_t *xdata)
 {
-        trace_conf_t   *conf = NULL;
+    trace_conf_t *conf = NULL;
 
-        conf = this->private;
+    conf = this->private;
 
-        if (!conf->log_file && !conf->log_history)
-		goto out;
-        if (trace_fop_names[GF_FOP_REMOVEXATTR].enabled) {
-                char   string[4096]  =  {0,};
-                snprintf (string, sizeof (string),
-                          "%"PRId64": gfid=%s path=%s name=%s",
-                          frame->root->unique,
-                          uuid_utoa (loc->inode->gfid), loc->path,
-                          name);
+    if (!conf->log_file && !conf->log_history)
+        goto out;
+    if (trace_fop_names[GF_FOP_REMOVEXATTR].enabled) {
+        char string[4096] = {
+            0,
+        };
+        snprintf(string, sizeof(string), "%" PRId64 ": gfid=%s path=%s name=%s",
+                 frame->root->unique, uuid_utoa(loc->inode->gfid), loc->path,
+                 name);
 
-                frame->local = loc->inode->gfid;
+        frame->local = loc->inode->gfid;
 
-                LOG_ELEMENT (conf, string);
-        }
+        LOG_ELEMENT(conf, string);
+    }
 
 out:
-        STACK_WIND (frame, trace_removexattr_cbk,
-                    FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->removexattr,
-                    loc, name, xdata);
+    STACK_WIND(frame, trace_removexattr_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->removexattr, loc, name, xdata);
 
-        return 0;
+    return 0;
 }
 
 int
-trace_opendir (call_frame_t *frame, xlator_t *this, loc_t *loc, fd_t *fd,
-               dict_t *xdata)
+trace_opendir(call_frame_t *frame, xlator_t *this, loc_t *loc, fd_t *fd,
+              dict_t *xdata)
 {
-        trace_conf_t   *conf = NULL;
+    trace_conf_t *conf = NULL;
 
-        conf = this->private;
+    conf = this->private;
 
-        if (!conf->log_file && !conf->log_history)
-		goto out;
-        if (trace_fop_names[GF_FOP_OPENDIR].enabled) {
-                char   string[4096]  =  {0,};
-                snprintf (string, sizeof (string),
-                          "%"PRId64": gfid=%s path=%s fd=%p",
-                          frame->root->unique,
-                          uuid_utoa (loc->inode->gfid), loc->path, fd);
+    if (!conf->log_file && !conf->log_history)
+        goto out;
+    if (trace_fop_names[GF_FOP_OPENDIR].enabled) {
+        char string[4096] = {
+            0,
+        };
+        snprintf(string, sizeof(string), "%" PRId64 ": gfid=%s path=%s fd=%p",
+                 frame->root->unique, uuid_utoa(loc->inode->gfid), loc->path,
+                 fd);
 
-                frame->local = loc->inode->gfid;
+        frame->local = loc->inode->gfid;
 
-                LOG_ELEMENT (conf, string);
-        }
+        LOG_ELEMENT(conf, string);
+    }
 
 out:
-        STACK_WIND (frame, trace_opendir_cbk,
-                    FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->opendir,
-                    loc, fd, xdata);
-        return 0;
+    STACK_WIND(frame, trace_opendir_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->opendir, loc, fd, xdata);
+    return 0;
 }
 
 int
-trace_readdirp (call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
-                off_t offset, dict_t *dict)
+trace_readdirp(call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
+               off_t offset, dict_t *dict)
 {
-        trace_conf_t *conf = NULL;
+    trace_conf_t *conf = NULL;
 
-        conf = this->private;
+    conf = this->private;
 
-        if (!conf->log_file && !conf->log_history)
-		goto out;
-        if (trace_fop_names[GF_FOP_READDIRP].enabled) {
-                char   string[4096]  =  {0,};
-                snprintf (string, sizeof (string),
-                          "%"PRId64": gfid=%s fd=%p, size=%"GF_PRI_SIZET
-                          ", offset=%"PRId64" dict=%p",
-                          frame->root->unique,
-                          uuid_utoa (fd->inode->gfid), fd, size,
-                          offset, dict);
+    if (!conf->log_file && !conf->log_history)
+        goto out;
+    if (trace_fop_names[GF_FOP_READDIRP].enabled) {
+        char string[4096] = {
+            0,
+        };
+        snprintf(string, sizeof(string),
+                 "%" PRId64 ": gfid=%s fd=%p, size=%" GF_PRI_SIZET
+                 ", offset=%" PRId64 " dict=%p",
+                 frame->root->unique, uuid_utoa(fd->inode->gfid), fd, size,
+                 offset, dict);
 
-                frame->local = fd->inode->gfid;
+        frame->local = fd->inode->gfid;
 
-                LOG_ELEMENT (conf, string);
-        }
+        LOG_ELEMENT(conf, string);
+    }
 
 out:
-        STACK_WIND (frame, trace_readdirp_cbk,
-                    FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->readdirp,
-                    fd, size, offset, dict);
+    STACK_WIND(frame, trace_readdirp_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->readdirp, fd, size, offset, dict);
 
-        return 0;
+    return 0;
 }
 
 int
-trace_readdir (call_frame_t *frame, xlator_t *this, fd_t *fd,
-               size_t size, off_t offset, dict_t *xdata)
+trace_readdir(call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
+              off_t offset, dict_t *xdata)
 {
-        trace_conf_t  *conf = NULL;
+    trace_conf_t *conf = NULL;
 
-        conf = this->private;
+    conf = this->private;
 
-        if (!conf->log_file && !conf->log_history)
-		goto out;
-        if (trace_fop_names[GF_FOP_READDIR].enabled) {
-                char   string[4096]  =  {0,};
-                snprintf (string, sizeof (string),
-                          "%"PRId64": gfid=%s fd=%p, size=%"GF_PRI_SIZET
-                          ", offset=%"PRId64,
-                          frame->root->unique,
-                          uuid_utoa (fd->inode->gfid), fd, size,
-                          offset);
+    if (!conf->log_file && !conf->log_history)
+        goto out;
+    if (trace_fop_names[GF_FOP_READDIR].enabled) {
+        char string[4096] = {
+            0,
+        };
+        snprintf(string, sizeof(string),
+                 "%" PRId64 ": gfid=%s fd=%p, size=%" GF_PRI_SIZET
+                 ", offset=%" PRId64,
+                 frame->root->unique, uuid_utoa(fd->inode->gfid), fd, size,
+                 offset);
 
-                frame->local = fd->inode->gfid;
+        frame->local = fd->inode->gfid;
 
-                LOG_ELEMENT (conf, string);
-        }
+        LOG_ELEMENT(conf, string);
+    }
 
 out:
-        STACK_WIND (frame, trace_readdir_cbk,
-                    FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->readdir,
-                    fd, size, offset, xdata);
+    STACK_WIND(frame, trace_readdir_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->readdir, fd, size, offset, xdata);
 
-        return 0;
+    return 0;
 }
 
 int
-trace_fsyncdir (call_frame_t *frame, xlator_t *this,
-                fd_t *fd, int32_t datasync, dict_t *xdata)
+trace_fsyncdir(call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t datasync,
+               dict_t *xdata)
 {
-        trace_conf_t   *conf = NULL;
+    trace_conf_t *conf = NULL;
 
-        conf = this->private;
+    conf = this->private;
 
-        if (!conf->log_file && !conf->log_history)
-		goto out;
-        if (trace_fop_names[GF_FOP_FSYNCDIR].enabled) {
-                char     string[4096]  =  {0,};
-                snprintf (string, sizeof (string),
-                          "%"PRId64": gfid=%s datasync=%d fd=%p",
-                          frame->root->unique,
-                          uuid_utoa (fd->inode->gfid), datasync, fd);
+    if (!conf->log_file && !conf->log_history)
+        goto out;
+    if (trace_fop_names[GF_FOP_FSYNCDIR].enabled) {
+        char string[4096] = {
+            0,
+        };
+        snprintf(string, sizeof(string),
+                 "%" PRId64 ": gfid=%s datasync=%d fd=%p", frame->root->unique,
+                 uuid_utoa(fd->inode->gfid), datasync, fd);
 
-                frame->local = fd->inode->gfid;
+        frame->local = fd->inode->gfid;
 
-                LOG_ELEMENT (conf, string);
-        }
+        LOG_ELEMENT(conf, string);
+    }
 
 out:
-        STACK_WIND (frame, trace_fsyncdir_cbk,
-                    FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->fsyncdir,
-                    fd, datasync, xdata);
-        return 0;
+    STACK_WIND(frame, trace_fsyncdir_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->fsyncdir, fd, datasync, xdata);
+    return 0;
 }
 
 int
-trace_access (call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t mask,
-              dict_t *xdata)
+trace_access(call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t mask,
+             dict_t *xdata)
 {
-        trace_conf_t  *conf = NULL;
+    trace_conf_t *conf = NULL;
 
-        conf = this->private;
+    conf = this->private;
 
-        if (!conf->log_file && !conf->log_history)
-		goto out;
-        if (trace_fop_names[GF_FOP_ACCESS].enabled) {
-                char     string[4096]  =  {0,};
-                snprintf (string, sizeof (string),
-                          "%"PRId64": gfid=%s path=%s mask=0%o",
-                          frame->root->unique,
-                          uuid_utoa (loc->inode->gfid),
-                          loc->path, mask);
+    if (!conf->log_file && !conf->log_history)
+        goto out;
+    if (trace_fop_names[GF_FOP_ACCESS].enabled) {
+        char string[4096] = {
+            0,
+        };
+        snprintf(string, sizeof(string),
+                 "%" PRId64 ": gfid=%s path=%s mask=0%o", frame->root->unique,
+                 uuid_utoa(loc->inode->gfid), loc->path, mask);
 
-                frame->local = loc->inode->gfid;
+        frame->local = loc->inode->gfid;
 
-                LOG_ELEMENT (conf, string);
-        }
+        LOG_ELEMENT(conf, string);
+    }
 
 out:
-        STACK_WIND (frame, trace_access_cbk,
-                    FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->access,
-                    loc, mask, xdata);
-        return 0;
+    STACK_WIND(frame, trace_access_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->access, loc, mask, xdata);
+    return 0;
 }
 
 int32_t
-trace_rchecksum (call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
-                 int32_t len, dict_t *xdata)
+trace_rchecksum(call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
+                int32_t len, dict_t *xdata)
 {
+    trace_conf_t *conf = NULL;
 
-        trace_conf_t *conf = NULL;
-
-        conf = this->private;
+    conf = this->private;
 
-        if (!conf->log_file && !conf->log_history)
-		goto out;
-        if (trace_fop_names[GF_FOP_RCHECKSUM].enabled) {
-                char    string[4096]  =  {0,};
-                snprintf (string, sizeof (string),
-                          "%"PRId64": gfid=%s offset=%"PRId64
-                          "len=%u fd=%p", frame->root->unique,
-                          uuid_utoa (fd->inode->gfid), offset, len, fd);
+    if (!conf->log_file && !conf->log_history)
+        goto out;
+    if (trace_fop_names[GF_FOP_RCHECKSUM].enabled) {
+        char string[4096] = {
+            0,
+        };
+        snprintf(string, sizeof(string),
+                 "%" PRId64 ": gfid=%s offset=%" PRId64 "len=%u fd=%p",
+                 frame->root->unique, uuid_utoa(fd->inode->gfid), offset, len,
+                 fd);
 
-                frame->local = fd->inode->gfid;
+        frame->local = fd->inode->gfid;
 
-                LOG_ELEMENT (conf, string);
-        }
+        LOG_ELEMENT(conf, string);
+    }
 
 out:
-        STACK_WIND (frame, trace_rchecksum_cbk,
-                    FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->rchecksum,
-                    fd, offset, len, xdata);
-
-        return 0;
+    STACK_WIND(frame, trace_rchecksum_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->rchecksum, fd, offset, len, xdata);
 
+    return 0;
 }
 
 int32_t
-trace_fentrylk (call_frame_t *frame, xlator_t *this, const char *volume,
-                fd_t *fd, const char *basename, entrylk_cmd cmd,
-                entrylk_type type, dict_t *xdata)
-{
-        trace_conf_t   *conf = NULL;
-
-        conf = this->private;
-
-        if (!conf->log_file && !conf->log_history)
-		goto out;
-        if (trace_fop_names[GF_FOP_FENTRYLK].enabled) {
-                char      string[4096]   =  {0,};
-                snprintf (string, sizeof (string),
-                          "%"PRId64": gfid=%s volume=%s, (fd=%p "
-                          "basename=%s, cmd=%s, type=%s)",
-                          frame->root->unique,
-                          uuid_utoa (fd->inode->gfid), volume, fd,
-                          basename,
-                          ((cmd == ENTRYLK_LOCK) ? "ENTRYLK_LOCK" :
-                           "ENTRYLK_UNLOCK"),
-                          ((type == ENTRYLK_RDLCK) ? "ENTRYLK_RDLCK" :
-                           "ENTRYLK_WRLCK"));
-
-                frame->local = fd->inode->gfid;
-
-                LOG_ELEMENT (conf, string);
-        }
+trace_fentrylk(call_frame_t *frame, xlator_t *this, const char *volume,
+               fd_t *fd, const char *basename, entrylk_cmd cmd,
+               entrylk_type type, dict_t *xdata)
+{
+    trace_conf_t *conf = NULL;
 
-out:
-        STACK_WIND (frame, trace_fentrylk_cbk,
-                    FIRST_CHILD (this),
-                    FIRST_CHILD (this)->fops->fentrylk,
-                    volume, fd, basename, cmd, type, xdata);
-        return 0;
+    conf = this->private;
+
+    if (!conf->log_file && !conf->log_history)
+        goto out;
+    if (trace_fop_names[GF_FOP_FENTRYLK].enabled) {
+        char string[4096] = {
+            0,
+        };
+        snprintf(string, sizeof(string),
+                 "%" PRId64
+                 ": gfid=%s volume=%s, (fd=%p "
+                 "basename=%s, cmd=%s, type=%s)",
+                 frame->root->unique, uuid_utoa(fd->inode->gfid), volume, fd,
+                 basename,
+                 ((cmd == ENTRYLK_LOCK) ? "ENTRYLK_LOCK" : "ENTRYLK_UNLOCK"),
+                 ((type == ENTRYLK_RDLCK) ? "ENTRYLK_RDLCK" : "ENTRYLK_WRLCK"));
+
+        frame->local = fd->inode->gfid;
+
+        LOG_ELEMENT(conf, string);
+    }
 
+out:
+    STACK_WIND(frame, trace_fentrylk_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->fentrylk, volume, fd, basename, cmd,
+               type, xdata);
+    return 0;
 }
 
 int32_t
-trace_fgetxattr (call_frame_t *frame, xlator_t *this, fd_t *fd,
-                 const char *name, dict_t *xdata)
+trace_fgetxattr(call_frame_t *frame, xlator_t *this, fd_t *fd, const char *name,
+                dict_t *xdata)
 {
-        trace_conf_t   *conf = NULL;
+    trace_conf_t *conf = NULL;
 
-        conf = this->private;
+    conf = this->private;
 
-        if (!conf->log_file && !conf->log_history)
-                goto out;
-        if (trace_fop_names[GF_FOP_FGETXATTR].enabled) {
-                char     string[4096]  =  {0,};
-                snprintf (string, sizeof (string),
-                          "%"PRId64": gfid=%s fd=%p name=%s",
-                          frame->root->unique,
-                          uuid_utoa (fd->inode->gfid), fd, name);
+    if (!conf->log_file && !conf->log_history)
+        goto out;
+    if (trace_fop_names[GF_FOP_FGETXATTR].enabled) {
+        char string[4096] = {
+            0,
+        };
+        snprintf(string, sizeof(string), "%" PRId64 ": gfid=%s fd=%p name=%s",
+                 frame->root->unique, uuid_utoa(fd->inode->gfid), fd, name);
 
-                frame->local = fd->inode->gfid;
+        frame->local = fd->inode->gfid;
 
-                LOG_ELEMENT (conf, string);
-        }
+        LOG_ELEMENT(conf, string);
+    }
 
 out:
-        STACK_WIND (frame, trace_fgetxattr_cbk,
-                    FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->fgetxattr,
-                    fd, name, xdata);
-        return 0;
+    STACK_WIND(frame, trace_fgetxattr_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->fgetxattr, fd, name, xdata);
+    return 0;
 }
 
 int32_t
-trace_fsetxattr (call_frame_t *frame, xlator_t *this, fd_t *fd,
-                 dict_t *dict, int32_t flags, dict_t *xdata)
+trace_fsetxattr(call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *dict,
+                int32_t flags, dict_t *xdata)
 {
-        trace_conf_t   *conf = NULL;
+    trace_conf_t *conf = NULL;
 
-        conf = this->private;
+    conf = this->private;
 
-        if (!conf->log_file && !conf->log_history)
-		goto out;
-        if (trace_fop_names[GF_FOP_FSETXATTR].enabled) {
-                char   string[4096]  =  {0,};
-                snprintf (string, sizeof (string),
-                          "%"PRId64": gfid=%s fd=%p flags=%d",
-                          frame->root->unique,
-                          uuid_utoa (fd->inode->gfid), fd, flags);
+    if (!conf->log_file && !conf->log_history)
+        goto out;
+    if (trace_fop_names[GF_FOP_FSETXATTR].enabled) {
+        char string[4096] = {
+            0,
+        };
+        snprintf(string, sizeof(string), "%" PRId64 ": gfid=%s fd=%p flags=%d",
+                 frame->root->unique, uuid_utoa(fd->inode->gfid), fd, flags);
 
-                frame->local = fd->inode->gfid;
+        frame->local = fd->inode->gfid;
 
-                LOG_ELEMENT (conf, string);
-        }
+        LOG_ELEMENT(conf, string);
+    }
 
 out:
-        STACK_WIND (frame, trace_fsetxattr_cbk,
-                    FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->fsetxattr,
-                    fd, dict, flags, xdata);
-        return 0;
+    STACK_WIND(frame, trace_fsetxattr_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->fsetxattr, fd, dict, flags, xdata);
+    return 0;
 }
 
 int
-trace_ftruncate (call_frame_t *frame, xlator_t *this,
-                 fd_t *fd, off_t offset, dict_t *xdata)
+trace_ftruncate(call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
+                dict_t *xdata)
 {
-        trace_conf_t  *conf = NULL;
+    trace_conf_t *conf = NULL;
 
-        conf = this->private;
+    conf = this->private;
 
-        if (!conf->log_file && !conf->log_history)
-		goto out;
-        if (trace_fop_names[GF_FOP_FTRUNCATE].enabled) {
-                char    string[4096]  =  {0,};
-                snprintf (string, sizeof (string),
-                          "%"PRId64": gfid=%s offset=%"PRId64" fd=%p",
-                          frame->root->unique,
-                          uuid_utoa (fd->inode->gfid), offset, fd);
+    if (!conf->log_file && !conf->log_history)
+        goto out;
+    if (trace_fop_names[GF_FOP_FTRUNCATE].enabled) {
+        char string[4096] = {
+            0,
+        };
+        snprintf(string, sizeof(string),
+                 "%" PRId64 ": gfid=%s offset=%" PRId64 " fd=%p",
+                 frame->root->unique, uuid_utoa(fd->inode->gfid), offset, fd);
 
-                frame->local = fd->inode->gfid;
+        frame->local = fd->inode->gfid;
 
-                LOG_ELEMENT (conf, string);
-        }
+        LOG_ELEMENT(conf, string);
+    }
 
 out:
-        STACK_WIND (frame, trace_ftruncate_cbk,
-                    FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->ftruncate,
-                    fd, offset, xdata);
+    STACK_WIND(frame, trace_ftruncate_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->ftruncate, fd, offset, xdata);
 
-        return 0;
+    return 0;
 }
 
 int
-trace_fstat (call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *xdata)
+trace_fstat(call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *xdata)
 {
-        trace_conf_t   *conf = NULL;
+    trace_conf_t *conf = NULL;
 
-        conf = this->private;
+    conf = this->private;
 
-        if (!conf->log_file && !conf->log_history)
-		goto out;
-        if (trace_fop_names[GF_FOP_FSTAT].enabled) {
-                char   string[4096]  =  {0,};
-                snprintf (string, sizeof (string),
-                          "%"PRId64": gfid=%s fd=%p",
-                          frame->root->unique,
-                          uuid_utoa (fd->inode->gfid), fd);
+    if (!conf->log_file && !conf->log_history)
+        goto out;
+    if (trace_fop_names[GF_FOP_FSTAT].enabled) {
+        char string[4096] = {
+            0,
+        };
+        snprintf(string, sizeof(string), "%" PRId64 ": gfid=%s fd=%p",
+                 frame->root->unique, uuid_utoa(fd->inode->gfid), fd);
 
-                frame->local = fd->inode->gfid;
+        frame->local = fd->inode->gfid;
 
-                LOG_ELEMENT (conf, string);
-        }
+        LOG_ELEMENT(conf, string);
+    }
 
 out:
-        STACK_WIND (frame, trace_fstat_cbk,
-                    FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->fstat,
-                    fd, xdata);
-        return 0;
+    STACK_WIND(frame, trace_fstat_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->fstat, fd, xdata);
+    return 0;
 }
 
 int
-trace_lk (call_frame_t *frame, xlator_t *this, fd_t *fd,
-          int32_t cmd, struct gf_flock *lock, dict_t *xdata)
+trace_lk(call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t cmd,
+         struct gf_flock *lock, dict_t *xdata)
 {
-        trace_conf_t   *conf = NULL;
+    trace_conf_t *conf = NULL;
 
-        conf = this->private;
+    conf = this->private;
 
-        if (!conf->log_file && !conf->log_history)
-		goto out;
-        if (trace_fop_names[GF_FOP_LK].enabled) {
-                char     string[4096]  =  {0,};
-                snprintf (string, sizeof (string),
-                          "%"PRId64": gfid=%s fd=%p, cmd=%d, "
-                          "lock {l_type=%d, "
-                          "l_whence=%d, l_start=%"PRId64", "
-                          "l_len=%"PRId64", l_pid=%u})",
-                          frame->root->unique,
-                          uuid_utoa (fd->inode->gfid), fd, cmd,
-                          lock->l_type, lock->l_whence,
-                          lock->l_start, lock->l_len, lock->l_pid);
+    if (!conf->log_file && !conf->log_history)
+        goto out;
+    if (trace_fop_names[GF_FOP_LK].enabled) {
+        char string[4096] = {
+            0,
+        };
+        snprintf(string, sizeof(string),
+                 "%" PRId64
+                 ": gfid=%s fd=%p, cmd=%d, "
+                 "lock {l_type=%d, "
+                 "l_whence=%d, l_start=%" PRId64
+                 ", "
+                 "l_len=%" PRId64 ", l_pid=%u})",
+                 frame->root->unique, uuid_utoa(fd->inode->gfid), fd, cmd,
+                 lock->l_type, lock->l_whence, lock->l_start, lock->l_len,
+                 lock->l_pid);
 
-                frame->local = fd->inode->gfid;
+        frame->local = fd->inode->gfid;
 
-                LOG_ELEMENT (conf, string);
-        }
+        LOG_ELEMENT(conf, string);
+    }
 
 out:
-        STACK_WIND (frame, trace_lk_cbk,
-                    FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->lk,
-                    fd, cmd, lock, xdata);
-        return 0;
+    STACK_WIND(frame, trace_lk_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->lk, fd, cmd, lock, xdata);
+    return 0;
 }
 
 int32_t
-trace_forget (xlator_t *this, inode_t *inode)
+trace_forget(xlator_t *this, inode_t *inode)
 {
-        trace_conf_t   *conf = NULL;
+    trace_conf_t *conf = NULL;
 
-        conf = this->private;
-        /* If user want to understand when a lookup happens,
-           he should know about 'forget' too */
-        if (!conf->log_file && !conf->log_history)
-		goto out;
-        if (trace_fop_names[GF_FOP_LOOKUP].enabled) {
-                char   string[4096]  =  {0,};
-                snprintf (string, sizeof (string),
-                          "gfid=%s", uuid_utoa (inode->gfid));
+    conf = this->private;
+    /* If user want to understand when a lookup happens,
+       he should know about 'forget' too */
+    if (!conf->log_file && !conf->log_history)
+        goto out;
+    if (trace_fop_names[GF_FOP_LOOKUP].enabled) {
+        char string[4096] = {
+            0,
+        };
+        snprintf(string, sizeof(string), "gfid=%s", uuid_utoa(inode->gfid));
 
-                LOG_ELEMENT (conf, string);
-        }
+        LOG_ELEMENT(conf, string);
+    }
 
 out:
-        return 0;
+    return 0;
 }
 
 int32_t
-trace_releasedir (xlator_t *this, fd_t *fd)
+trace_releasedir(xlator_t *this, fd_t *fd)
 {
-        trace_conf_t  *conf = NULL;
+    trace_conf_t *conf = NULL;
 
-        conf = this->private;
+    conf = this->private;
 
-        if (!conf->log_file && !conf->log_history)
-		goto out;
-        if (trace_fop_names[GF_FOP_OPENDIR].enabled) {
-                char   string[4096]  =  {0,};
-                snprintf (string, sizeof (string),
-                          "gfid=%s fd=%p",
-                          uuid_utoa (fd->inode->gfid), fd);
+    if (!conf->log_file && !conf->log_history)
+        goto out;
+    if (trace_fop_names[GF_FOP_OPENDIR].enabled) {
+        char string[4096] = {
+            0,
+        };
+        snprintf(string, sizeof(string), "gfid=%s fd=%p",
+                 uuid_utoa(fd->inode->gfid), fd);
 
-                LOG_ELEMENT (conf, string);
-        }
+        LOG_ELEMENT(conf, string);
+    }
 
 out:
-        return 0;
+    return 0;
 }
 
 int32_t
-trace_release (xlator_t *this, fd_t *fd)
+trace_release(xlator_t *this, fd_t *fd)
 {
-        trace_conf_t   *conf = NULL;
+    trace_conf_t *conf = NULL;
 
-        conf = this->private;
+    conf = this->private;
 
-        if (!conf->log_file && !conf->log_history)
-		goto out;
-        if (trace_fop_names[GF_FOP_OPEN].enabled ||
-            trace_fop_names[GF_FOP_CREATE].enabled) {
-                char   string[4096]  =  {0,};
-                snprintf (string, sizeof (string),
-                          "gfid=%s fd=%p",
-                          uuid_utoa (fd->inode->gfid), fd);
+    if (!conf->log_file && !conf->log_history)
+        goto out;
+    if (trace_fop_names[GF_FOP_OPEN].enabled ||
+        trace_fop_names[GF_FOP_CREATE].enabled) {
+        char string[4096] = {
+            0,
+        };
+        snprintf(string, sizeof(string), "gfid=%s fd=%p",
+                 uuid_utoa(fd->inode->gfid), fd);
 
-                LOG_ELEMENT (conf, string);
-        }
+        LOG_ELEMENT(conf, string);
+    }
 
 out:
-        return 0;
+    return 0;
 }
 
-
 void
-enable_all_calls (int enabled)
+enable_all_calls(int enabled)
 {
-        int i;
+    int i;
 
-        for (i = 0; i < GF_FOP_MAXVALUE; i++)
-                trace_fop_names[i].enabled = enabled;
+    for (i = 0; i < GF_FOP_MAXVALUE; i++)
+        trace_fop_names[i].enabled = enabled;
 }
 
 void
-enable_call (const char *name, int enabled)
+enable_call(const char *name, int enabled)
 {
-        int i;
-        for (i = 0; i < GF_FOP_MAXVALUE; i++)
-                if (!strcasecmp(trace_fop_names[i].name, name))
-                        trace_fop_names[i].enabled = enabled;
+    int i;
+    for (i = 0; i < GF_FOP_MAXVALUE; i++)
+        if (!strcasecmp(trace_fop_names[i].name, name))
+            trace_fop_names[i].enabled = enabled;
 }
 
-
 /*
   include = 1 for "include-ops"
   = 0 for "exclude-ops"
 */
 void
-process_call_list (const char *list, int include)
+process_call_list(const char *list, int include)
 {
-        enable_all_calls (include ? 0 : 1);
+    enable_all_calls(include ? 0 : 1);
 
-        char *call = strsep ((char **)&list, ",");
+    char *call = strsep((char **)&list, ",");
 
-        while (call) {
-                enable_call (call, include);
-                call = strsep ((char **)&list, ",");
-        }
+    while (call) {
+        enable_call(call, include);
+        call = strsep((char **)&list, ",");
+    }
 }
 
 int32_t
-trace_dump_history (xlator_t *this)
-{
-        int ret = -1;
-        char key_prefix[GF_DUMP_MAX_BUF_LEN] = {0,};
-        trace_conf_t *conf = NULL;
-
-        GF_VALIDATE_OR_GOTO ("trace", this, out);
-        GF_VALIDATE_OR_GOTO (this->name, this->history, out);
-
-        conf = this->private;
-        // Is it ok to return silently if log-history option his off?
-        if (conf && conf->log_history == _gf_true) {
-                gf_proc_dump_build_key (key_prefix, "xlator.debug.trace",
-                                        "history");
-                gf_proc_dump_add_section (key_prefix);
-                eh_dump (this->history, NULL, dump_history_trace);
-        }
-        ret = 0;
+trace_dump_history(xlator_t *this)
+{
+    int ret = -1;
+    char key_prefix[GF_DUMP_MAX_BUF_LEN] = {
+        0,
+    };
+    trace_conf_t *conf = NULL;
+
+    GF_VALIDATE_OR_GOTO("trace", this, out);
+    GF_VALIDATE_OR_GOTO(this->name, this->history, out);
+
+    conf = this->private;
+    // Is it ok to return silently if log-history option his off?
+    if (conf && conf->log_history == _gf_true) {
+        gf_proc_dump_build_key(key_prefix, "xlator.debug.trace", "history");
+        gf_proc_dump_add_section("%s", key_prefix);
+        eh_dump(this->history, NULL, dump_history_trace);
+    }
+    ret = 0;
 
 out:
-        return ret;
+    return ret;
 }
 
 int32_t
-mem_acct_init (xlator_t *this)
+mem_acct_init(xlator_t *this)
 {
-        int     ret = -1;
+    int ret = -1;
 
-        if (!this)
-                return ret;
+    if (!this)
+        return ret;
 
-        ret = xlator_mem_acct_init (this, gf_trace_mt_end + 1);
-
-        if (ret != 0) {
-                gf_log (this->name, GF_LOG_ERROR, "Memory accounting init"
-                        " failed");
-                return ret;
-        }
+    ret = xlator_mem_acct_init(this, gf_trace_mt_end + 1);
 
+    if (ret != 0) {
+        gf_log(this->name, GF_LOG_ERROR,
+               "Memory accounting init"
+               " failed");
         return ret;
+    }
+
+    return ret;
 }
 
 int
-reconfigure (xlator_t *this, dict_t *options)
+reconfigure(xlator_t *this, dict_t *options)
 {
-        int32_t       ret            = -1;
-        trace_conf_t    *conf           = NULL;
-        char            *includes = NULL, *excludes = NULL;
+    int32_t ret = -1;
+    trace_conf_t *conf = NULL;
+    char *includes = NULL, *excludes = NULL;
 
-        GF_VALIDATE_OR_GOTO ("quick-read", this, out);
-        GF_VALIDATE_OR_GOTO (this->name, this->private, out);
-        GF_VALIDATE_OR_GOTO (this->name, options, out);
+    GF_VALIDATE_OR_GOTO("quick-read", this, out);
+    GF_VALIDATE_OR_GOTO(this->name, this->private, out);
+    GF_VALIDATE_OR_GOTO(this->name, options, out);
 
-        conf = this->private;
+    conf = this->private;
 
-        includes = data_to_str (dict_get (options, "include-ops"));
-        excludes = data_to_str (dict_get (options, "exclude-ops"));
+    includes = data_to_str(dict_get(options, "include-ops"));
+    excludes = data_to_str(dict_get(options, "exclude-ops"));
 
-        {
-                int i;
-                for (i = 0; i < GF_FOP_MAXVALUE; i++) {
-                        if (gf_fop_list[i])
-                                strncpy (trace_fop_names[i].name,
-                                         gf_fop_list[i],
-                                         strlen (gf_fop_list[i]));
-                        else
-                                strncpy (trace_fop_names[i].name, ":O",
-                                         strlen (":O"));
-                        trace_fop_names[i].enabled = 1;
-                }
+    {
+        int i;
+        for (i = 0; i < GF_FOP_MAXVALUE; i++) {
+            if (gf_fop_list[i])
+                strncpy(trace_fop_names[i].name, gf_fop_list[i],
+                        sizeof(trace_fop_names[i].name));
+            else
+                strncpy(trace_fop_names[i].name, ":0",
+                        sizeof(trace_fop_names[i].name));
+            trace_fop_names[i].enabled = 1;
+            trace_fop_names[i].name[sizeof(trace_fop_names[i].name) - 1] = 0;
         }
+    }
 
-        if (includes && excludes) {
-                gf_log (this->name,
-                        GF_LOG_ERROR,
-                        "must specify only one of 'include-ops' and "
-                        "'exclude-ops'");
-                goto out;
-        }
+    if (includes && excludes) {
+        gf_log(this->name, GF_LOG_ERROR,
+               "must specify only one of 'include-ops' and "
+               "'exclude-ops'");
+        goto out;
+    }
 
-        if (includes)
-                process_call_list (includes, 1);
-        if (excludes)
-                process_call_list (excludes, 0);
+    if (includes)
+        process_call_list(includes, 1);
+    if (excludes)
+        process_call_list(excludes, 0);
 
-        /* Should resizing of the event-history be allowed in reconfigure?
-         * for which a new event_history might have to be allocated and the
-         * older history has to be freed.
-         */
-        GF_OPTION_RECONF ("log-file", conf->log_file, options, bool, out);
+    /* Should resizing of the event-history be allowed in reconfigure?
+     * for which a new event_history might have to be allocated and the
+     * older history has to be freed.
+     */
+    GF_OPTION_RECONF("log-file", conf->log_file, options, bool, out);
 
-        GF_OPTION_RECONF ("log-history", conf->log_history, options, bool, out);
+    GF_OPTION_RECONF("log-history", conf->log_history, options, bool, out);
 
-        ret = 0;
+    ret = 0;
 
 out:
-        return ret;
+    return ret;
 }
 
 int32_t
-init (xlator_t *this)
-{
-        dict_t *options = NULL;
-        char *includes = NULL, *excludes = NULL;
-        char *forced_loglevel = NULL;
-        eh_t *history = NULL;
-        int  ret = -1;
-        size_t  history_size = TRACE_DEFAULT_HISTORY_SIZE;
-        trace_conf_t    *conf = NULL;
-
-        if (!this)
-                return -1;
-
-        if (!this->children || this->children->next) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "trace translator requires one subvolume");
-                return -1;
-        }
-        if (!this->parents) {
-                gf_log (this->name, GF_LOG_WARNING,
-                        "dangling volume. check volfile ");
-        }
-
-        conf = GF_CALLOC (1, sizeof (trace_conf_t), gf_trace_mt_trace_conf_t);
-        if (!conf) {
-                gf_log (this->name, GF_LOG_ERROR, "cannot allocate "
-                        "xl->private");
-                return -1;
-        }
-
-        options = this->options;
-        includes = data_to_str (dict_get (options, "include-ops"));
-        excludes = data_to_str (dict_get (options, "exclude-ops"));
-
-        {
-                int i;
-                for (i = 0; i < GF_FOP_MAXVALUE; i++) {
-                        if (gf_fop_list[i])
-                                strncpy (trace_fop_names[i].name,
-                                         gf_fop_list[i],
-                                         strlen (gf_fop_list[i]));
-                        else
-                                strncpy (trace_fop_names[i].name, ":O",
-                                         strlen (":O"));
-                        trace_fop_names[i].enabled = 1;
-                }
-        }
-
-        if (includes && excludes) {
-                gf_log (this->name,
-                        GF_LOG_ERROR,
-                        "must specify only one of 'include-ops' and "
-                        "'exclude-ops'");
-                return -1;
-        }
-
-        if (includes)
-                process_call_list (includes, 1);
-        if (excludes)
-                process_call_list (excludes, 0);
-
-
-        GF_OPTION_INIT ("history-size", conf->history_size, size, out);
-
-        gf_log (this->name, GF_LOG_INFO, "history size %"GF_PRI_SIZET,
-                history_size);
-
-        GF_OPTION_INIT ("log-file", conf->log_file, bool, out);
-
-        gf_log (this->name, GF_LOG_INFO, "logging to file %s",
-                (conf->log_file == _gf_true)?"enabled":"disabled");
-
-        GF_OPTION_INIT ("log-history", conf->log_history, bool, out);
-
-        gf_log (this->name, GF_LOG_DEBUG, "logging to history %s",
-                (conf->log_history == _gf_true)?"enabled":"disabled");
-
-        history = eh_new (history_size, _gf_false, NULL);
-        if (!history) {
-                gf_log (this->name, GF_LOG_ERROR, "event history cannot be "
-                        "initialized");
-                return -1;
-        }
-
-        this->history = history;
-
-        conf->trace_log_level = GF_LOG_INFO;
-
-        if (dict_get (options, "force-log-level")) {
-                forced_loglevel = data_to_str (dict_get (options,
-                                                         "force-log-level"));
-                if (!forced_loglevel)
-                        goto setloglevel;
-
-                if (strcmp (forced_loglevel, "INFO") == 0)
-                        conf->trace_log_level = GF_LOG_INFO;
-                else if (strcmp (forced_loglevel, "TRACE") == 0)
-                        conf->trace_log_level = GF_LOG_TRACE;
-                else if (strcmp (forced_loglevel, "ERROR") == 0)
-                        conf->trace_log_level = GF_LOG_ERROR;
-                else if (strcmp (forced_loglevel, "DEBUG") == 0)
-                        conf->trace_log_level = GF_LOG_DEBUG;
-                else if (strcmp (forced_loglevel, "WARNING") == 0)
-                        conf->trace_log_level = GF_LOG_WARNING;
-                else if (strcmp (forced_loglevel, "CRITICAL") == 0)
-                        conf->trace_log_level = GF_LOG_CRITICAL;
-                else if (strcmp (forced_loglevel, "NONE") == 0)
-                        conf->trace_log_level = GF_LOG_NONE;
-        }
+init(xlator_t *this)
+{
+    dict_t *options = NULL;
+    char *includes = NULL, *excludes = NULL;
+    char *forced_loglevel = NULL;
+    eh_t *history = NULL;
+    int ret = -1;
+    uint64_t history_size = TRACE_DEFAULT_HISTORY_SIZE;
+    trace_conf_t *conf = NULL;
+
+    if (!this)
+        return -1;
+
+    if (!this->children || this->children->next) {
+        gf_log(this->name, GF_LOG_ERROR,
+               "trace translator requires one subvolume");
+        return -1;
+    }
+    if (!this->parents) {
+        gf_log(this->name, GF_LOG_WARNING, "dangling volume. check volfile ");
+    }
+
+    conf = GF_CALLOC(1, sizeof(trace_conf_t), gf_trace_mt_trace_conf_t);
+    if (!conf) {
+        gf_log(this->name, GF_LOG_ERROR,
+               "cannot allocate "
+               "xl->private");
+        return -1;
+    }
+
+    options = this->options;
+    includes = data_to_str(dict_get(options, "include-ops"));
+    excludes = data_to_str(dict_get(options, "exclude-ops"));
+
+    {
+        int i;
+        for (i = 0; i < GF_FOP_MAXVALUE; i++) {
+            if (gf_fop_list[i])
+                strncpy(trace_fop_names[i].name, gf_fop_list[i],
+                        sizeof(trace_fop_names[i].name));
+            else
+                strncpy(trace_fop_names[i].name, ":O",
+                        sizeof(trace_fop_names[i].name));
+            trace_fop_names[i].enabled = 1;
+            trace_fop_names[i].name[sizeof(trace_fop_names[i].name) - 1] = 0;
+        }
+    }
+
+    if (includes && excludes) {
+        gf_log(this->name, GF_LOG_ERROR,
+               "must specify only one of 'include-ops' and "
+               "'exclude-ops'");
+        return -1;
+    }
+
+    if (includes)
+        process_call_list(includes, 1);
+    if (excludes)
+        process_call_list(excludes, 0);
+
+    GF_OPTION_INIT("history-size", history_size, size, out);
+    conf->history_size = history_size;
+
+    gf_log(this->name, GF_LOG_INFO, "history size %" PRIu64, history_size);
+
+    GF_OPTION_INIT("log-file", conf->log_file, bool, out);
+
+    gf_log(this->name, GF_LOG_INFO, "logging to file %s",
+           (conf->log_file == _gf_true) ? "enabled" : "disabled");
+
+    GF_OPTION_INIT("log-history", conf->log_history, bool, out);
+
+    gf_log(this->name, GF_LOG_DEBUG, "logging to history %s",
+           (conf->log_history == _gf_true) ? "enabled" : "disabled");
+
+    history = eh_new(history_size, _gf_false, NULL);
+    if (!history) {
+        gf_log(this->name, GF_LOG_ERROR,
+               "event history cannot be "
+               "initialized");
+        return -1;
+    }
+
+    this->history = history;
+
+    conf->trace_log_level = GF_LOG_INFO;
+
+    if (dict_get(options, "force-log-level")) {
+        forced_loglevel = data_to_str(dict_get(options, "force-log-level"));
+        if (!forced_loglevel)
+            goto setloglevel;
+
+        if (strcmp(forced_loglevel, "INFO") == 0)
+            conf->trace_log_level = GF_LOG_INFO;
+        else if (strcmp(forced_loglevel, "TRACE") == 0)
+            conf->trace_log_level = GF_LOG_TRACE;
+        else if (strcmp(forced_loglevel, "ERROR") == 0)
+            conf->trace_log_level = GF_LOG_ERROR;
+        else if (strcmp(forced_loglevel, "DEBUG") == 0)
+            conf->trace_log_level = GF_LOG_DEBUG;
+        else if (strcmp(forced_loglevel, "WARNING") == 0)
+            conf->trace_log_level = GF_LOG_WARNING;
+        else if (strcmp(forced_loglevel, "CRITICAL") == 0)
+            conf->trace_log_level = GF_LOG_CRITICAL;
+        else if (strcmp(forced_loglevel, "NONE") == 0)
+            conf->trace_log_level = GF_LOG_NONE;
+    }
 
 setloglevel:
-        gf_log_set_loglevel (conf->trace_log_level);
-        this->private = conf;
-        ret = 0;
-out:
-        if (ret == -1) {
-                if (history)
-                        GF_FREE (history);
-                if (conf)
-                        GF_FREE (conf);
-        }
+    gf_log_set_loglevel(this->ctx, conf->trace_log_level);
+    this->private = conf;
+    ret = 0;
+out:
+    if (ret == -1) {
+        if (history)
+            GF_FREE(history);
+        if (conf)
+            GF_FREE(conf);
+    }
 
-        return ret;
+    return ret;
 }
 
 void
-fini (xlator_t *this)
+fini(xlator_t *this)
 {
-        if (!this)
-                return;
+    if (!this)
+        return;
 
-        if (this->history)
-                eh_destroy (this->history);
+    if (this->history)
+        eh_destroy(this->history);
 
-        gf_log (this->name, GF_LOG_INFO,
-                "trace translator unloaded");
-        return;
+    gf_log(this->name, GF_LOG_INFO, "trace translator unloaded");
+    return;
 }
 
 struct xlator_fops fops = {
-        .stat        = trace_stat,
-        .readlink    = trace_readlink,
-        .mknod       = trace_mknod,
-        .mkdir       = trace_mkdir,
-        .unlink      = trace_unlink,
-        .rmdir       = trace_rmdir,
-        .symlink     = trace_symlink,
-        .rename      = trace_rename,
-        .link        = trace_link,
-        .truncate    = trace_truncate,
-        .open        = trace_open,
-        .readv       = trace_readv,
-        .writev      = trace_writev,
-        .statfs      = trace_statfs,
-        .flush       = trace_flush,
-        .fsync       = trace_fsync,
-        .setxattr    = trace_setxattr,
-        .getxattr    = trace_getxattr,
-        .fsetxattr   = trace_fsetxattr,
-        .fgetxattr   = trace_fgetxattr,
-        .removexattr = trace_removexattr,
-        .opendir     = trace_opendir,
-        .readdir     = trace_readdir,
-        .readdirp    = trace_readdirp,
-        .fsyncdir    = trace_fsyncdir,
-        .access      = trace_access,
-        .ftruncate   = trace_ftruncate,
-        .fstat       = trace_fstat,
-        .create      = trace_create,
-        .lk          = trace_lk,
-        .inodelk     = trace_inodelk,
-        .finodelk    = trace_finodelk,
-        .entrylk     = trace_entrylk,
-        .fentrylk    = trace_fentrylk,
-        .lookup      = trace_lookup,
-        .rchecksum   = trace_rchecksum,
-        .xattrop     = trace_xattrop,
-        .fxattrop    = trace_fxattrop,
-        .setattr     = trace_setattr,
-        .fsetattr    = trace_fsetattr,
+    .stat = trace_stat,
+    .readlink = trace_readlink,
+    .mknod = trace_mknod,
+    .mkdir = trace_mkdir,
+    .unlink = trace_unlink,
+    .rmdir = trace_rmdir,
+    .symlink = trace_symlink,
+    .rename = trace_rename,
+    .link = trace_link,
+    .truncate = trace_truncate,
+    .open = trace_open,
+    .readv = trace_readv,
+    .writev = trace_writev,
+    .statfs = trace_statfs,
+    .flush = trace_flush,
+    .fsync = trace_fsync,
+    .setxattr = trace_setxattr,
+    .getxattr = trace_getxattr,
+    .fsetxattr = trace_fsetxattr,
+    .fgetxattr = trace_fgetxattr,
+    .removexattr = trace_removexattr,
+    .opendir = trace_opendir,
+    .readdir = trace_readdir,
+    .readdirp = trace_readdirp,
+    .fsyncdir = trace_fsyncdir,
+    .access = trace_access,
+    .ftruncate = trace_ftruncate,
+    .fstat = trace_fstat,
+    .create = trace_create,
+    .lk = trace_lk,
+    .inodelk = trace_inodelk,
+    .finodelk = trace_finodelk,
+    .entrylk = trace_entrylk,
+    .fentrylk = trace_fentrylk,
+    .lookup = trace_lookup,
+    .rchecksum = trace_rchecksum,
+    .xattrop = trace_xattrop,
+    .fxattrop = trace_fxattrop,
+    .setattr = trace_setattr,
+    .fsetattr = trace_fsetattr,
+    .seek = trace_seek,
 };
 
 struct xlator_cbks cbks = {
-        .release     = trace_release,
-        .releasedir  = trace_releasedir,
-        .forget      = trace_forget,
+    .release = trace_release,
+    .releasedir = trace_releasedir,
+    .forget = trace_forget,
 };
 
 struct volume_options options[] = {
-        { .key  = {"include-ops", "include"},
-          .type = GF_OPTION_TYPE_STR,
-          /*.value = { ""} */
-        },
-        { .key  = {"exclude-ops", "exclude"},
-          .type = GF_OPTION_TYPE_STR
-          /*.value = { ""} */
-        },
-        { .key  = {"history-size"},
-          .type = GF_OPTION_TYPE_SIZET,
-          .default_value = "1024",
-        },
-        { .key  = {"log-file"},
-          .type = GF_OPTION_TYPE_BOOL,
-          .default_value = "no",
-        },
-        { .key  = {"log-history"},
-          .type = GF_OPTION_TYPE_BOOL,
-          .default_value = "no",
-        },
-        { .key  = {NULL} },
+    {
+        .key = {"include-ops", "include"},
+        .type = GF_OPTION_TYPE_STR,
+        /*.value = { ""} */
+    },
+    {
+        .key = {"exclude-ops", "exclude"},
+        .type = GF_OPTION_TYPE_STR
+        /*.value = { ""} */
+    },
+    {
+        .key = {"history-size"},
+        .type = GF_OPTION_TYPE_SIZET,
+        .default_value = "1024",
+    },
+    {
+        .key = {"log-file"},
+        .type = GF_OPTION_TYPE_BOOL,
+        .default_value = "no",
+    },
+    {
+        .key = {"log-history"},
+        .type = GF_OPTION_TYPE_BOOL,
+        .default_value = "no",
+    },
+    {.key = {NULL}},
 };
 
-struct xlator_dumpops dumpops = {
-        .history = trace_dump_history
+struct xlator_dumpops dumpops = {.history = trace_dump_history};
+
+xlator_api_t xlator_api = {
+    .init = init,
+    .fini = fini,
+    .reconfigure = reconfigure,
+    .mem_acct_init = mem_acct_init,
+    .op_version = {1},
+    .dumpops = &dumpops,
+    .fops = &fops,
+    .cbks = &cbks,
+    .options = options,
+    .identifier = "trace",
+    .category = GF_TECH_PREVIEW,
 };
diff --git a/xlators/debug/trace/src/trace.h b/xlators/debug/trace/src/trace.h
index 62d1bc9c921..b16304799da 100644
--- a/xlators/debug/trace/src/trace.h
+++ b/xlators/debug/trace/src/trace.h
@@ -8,54 +8,48 @@
    cases as published by the Free Software Foundation.
 */
 
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
-
 #include <time.h>
 #include <errno.h>
-#include "glusterfs.h"
-#include "xlator.h"
-#include "common-utils.h"
-#include "event-history.h"
-#include "logging.h"
-#include "circ-buff.h"
-#include "statedump.h"
-#include "options.h"
+#include <glusterfs/glusterfs.h>
+#include <glusterfs/xlator.h>
+#include <glusterfs/common-utils.h>
+#include <glusterfs/event-history.h>
+#include <glusterfs/logging.h>
+#include <glusterfs/circ-buff.h>
+#include <glusterfs/statedump.h>
+#include <glusterfs/options.h>
 
 #define TRACE_DEFAULT_HISTORY_SIZE 1024
 
 typedef struct {
-        /* Since the longest fop name is fremovexattr i.e 12 characters, array size
-         * is kept 24, i.e double of the maximum.
-         */
-        char name[24];
-        int enabled;
+    /* Since the longest fop name is fremovexattr i.e 12 characters, array size
+     * is kept 24, i.e double of the maximum.
+     */
+    char name[24];
+    int enabled;
 } trace_fop_name_t;
 
 trace_fop_name_t trace_fop_names[GF_FOP_MAXVALUE];
 
 typedef struct {
-        gf_boolean_t log_file;
-        gf_boolean_t log_history;
-        size_t       history_size;
-        int trace_log_level;
+    gf_boolean_t log_file;
+    gf_boolean_t log_history;
+    uint64_t history_size;
+    int trace_log_level;
 } trace_conf_t;
 
-#define TRACE_STACK_UNWIND(op, frame, params ...)                       \
-        do {                                                            \
-                frame->local = NULL;                                    \
-                STACK_UNWIND_STRICT (op, frame, params);                \
-        } while (0);
-
-#define LOG_ELEMENT(_conf, _string)                                     \
-        do {                                                            \
-                if (_conf) {                                            \
-                        if ((_conf->log_history) == _gf_true)           \
-                                gf_log_eh ("%s", _string);              \
-                        if ((_conf->log_file) == _gf_true)              \
-                                gf_log (THIS->name, _conf->trace_log_level, \
-                                        "%s", _string);                 \
-                }                                                       \
-        } while (0);
+#define TRACE_STACK_UNWIND(op, frame, params...)                               \
+    do {                                                                       \
+        frame->local = NULL;                                                   \
+        STACK_UNWIND_STRICT(op, frame, params);                                \
+    } while (0);
+
+#define LOG_ELEMENT(_conf, _string)                                            \
+    do {                                                                       \
+        if (_conf) {                                                           \
+            if ((_conf->log_history) == _gf_true)                              \
+                gf_log_eh("%s", _string);                                      \
+            if ((_conf->log_file) == _gf_true)                                 \
+                gf_log(THIS->name, _conf->trace_log_level, "%s", _string);     \
+        }                                                                      \
+    } while (0);
diff --git a/xlators/encryption/Makefile.am b/xlators/encryption/Makefile.am
deleted file mode 100644
index 36efc6698bd..00000000000
--- a/xlators/encryption/Makefile.am
+++ /dev/null
@@ -1,3 +0,0 @@
-SUBDIRS = rot-13 crypt
-
-CLEANFILES = 
diff --git a/xlators/encryption/crypt/src/Makefile.am b/xlators/encryption/crypt/src/Makefile.am
deleted file mode 100644
index d7ebac61884..00000000000
--- a/xlators/encryption/crypt/src/Makefile.am
+++ /dev/null
@@ -1,24 +0,0 @@
-if ENABLE_CRYPT_XLATOR
-
-xlator_LTLIBRARIES = crypt.la
-xlatordir = $(libdir)/glusterfs/$(PACKAGE_VERSION)/xlator/encryption
-
-crypt_la_LDFLAGS = -module -avoid-version
-
-crypt_la_SOURCES = keys.c data.c metadata.c atom.c crypt.c
-crypt_la_LIBADD = $(top_builddir)/libglusterfs/src/libglusterfs.la -lssl -lcrypto
-
-noinst_HEADERS = crypt-common.h crypt-mem-types.h crypt.h metadata.h
-
-AM_CPPFLAGS = $(GF_CPPFLAGS) -I$(top_srcdir)/libglusterfs/src
-
-AM_CFLAGS = -Wall $(GF_CFLAGS)
-
-CLEANFILES =
-
-else
-
-noinst_DIST = keys.c data.c metadata.c atom.c crypt.c
-noinst_HEADERS = crypt-common.h crypt-mem-types.h crypt.h metadata.h
-
-endif
diff --git a/xlators/encryption/crypt/src/atom.c b/xlators/encryption/crypt/src/atom.c
deleted file mode 100644
index 1ec41495ca1..00000000000
--- a/xlators/encryption/crypt/src/atom.c
+++ /dev/null
@@ -1,962 +0,0 @@
-/*
-  Copyright (c) 2008-2013 Red Hat, Inc. <http://www.redhat.com>
-  This file is part of GlusterFS.
-
-  This file is licensed to you under your choice of the GNU Lesser
-  General Public License, version 3 or any later version (LGPLv3 or
-  later), or the GNU General Public License, version 2 (GPLv2), in all
-  cases as published by the Free Software Foundation.
-*/
-
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
-
-#include "defaults.h"
-#include "crypt-common.h"
-#include "crypt.h"
-
-/*
- *                              Glossary
- *
- *
- * cblock            (or cipher block). A logical unit in a file.
- *                   cblock size is defined as the number of bits
- *                   in an input (or output) block of the block
- *                   cipher (*). Cipher block size is a property of
- *                   cipher algorithm. E.g. cblock size is 64 bits
- *                   for DES, 128 bits for AES, etc.
- *
- * atomic cipher     A cipher algorithm, which requires some chunks of
- * algorithm         text to be padded at left and(or) right sides before
- *                   cipher transaform.
- *
- *
- * block (atom)      Minimal chunk of file's data, which doesn't require
- *                   padding. We'll consider logical units in a file of
- *                   block size (atom size).
- *
- * cipher algorithm  Atomic cipher algorithm, which requires the last
- * with EOF issue    incomplete cblock in a file to be padded with some
- *                   data (usually zeros).
- *
- *
- * operation, which  reading/writing from offset, which is not aligned to
- * forms a gap at    to atom size
- * the beginning
- *
- *
- * operation, which  reading/writing count bytes starting from offset off,
- * forms a gap at    so that off+count is not aligned to atom_size
- * the end
- *
- * head block        the first atom affected by an operation, which forms
- *                   a gap at the beginning, or(and) at the end.
- *                   Сomment. Head block has at least one gap (either at
- *                   the beginning, or at the end)
- *
- *
- * tail block        the last atom different from head, affected by an
- *                   operation, which forms a gap at the end.
- *                   Сomment: Tail block has exactly one gap (at the end).
- *
- *
- * partial block     head or tail block
- *
- *
- * full block        block without gaps.
- *
- *
- * (*) Recommendation for Block Cipher Modes of Operation
- *     Methods and Techniques
- *     NIST Special Publication 800-38A Edition 2001
- */
-
-/*
- * atom->offset_at()
- */
-static off_t offset_at_head(struct avec_config *conf)
-{
-	return conf->aligned_offset;
-}
-
-static off_t offset_at_hole_head(call_frame_t *frame,
-					 struct object_cipher_info *object)
-{
-	return offset_at_head(get_hole_conf(frame));
-}
-
-static off_t offset_at_data_head(call_frame_t *frame,
-					 struct object_cipher_info *object)
-{
-	return offset_at_head(get_data_conf(frame));
-}
-
-
-static off_t offset_at_tail(struct avec_config *conf,
-				    struct object_cipher_info *object)
-{
-	return conf->aligned_offset +
-		(conf->off_in_head ? get_atom_size(object) : 0) +
-		(conf->nr_full_blocks << get_atom_bits(object));
-}
-
-static off_t offset_at_hole_tail(call_frame_t *frame,
-					 struct object_cipher_info *object)
-{
-	return offset_at_tail(get_hole_conf(frame), object);
-}
-
-
-static off_t offset_at_data_tail(call_frame_t *frame,
-					 struct object_cipher_info *object)
-{
-	return offset_at_tail(get_data_conf(frame), object);
-}
-
-static off_t offset_at_full(struct avec_config *conf,
-				    struct object_cipher_info *object)
-{
-	return conf->aligned_offset +
-		(conf->off_in_head ? get_atom_size(object) : 0);
-}
-
-static off_t offset_at_data_full(call_frame_t *frame,
-					 struct object_cipher_info *object)
-{
-	return offset_at_full(get_data_conf(frame), object);
-}
-
-static off_t offset_at_hole_full(call_frame_t *frame,
-					 struct object_cipher_info *object)
-{
-	return offset_at_full(get_hole_conf(frame), object);
-}
-
-/*
- * atom->io_size_nopad()
- */
-
-static uint32_t io_size_nopad_head(struct avec_config *conf,
-				   struct object_cipher_info *object)
-{
-	uint32_t gap_at_beg;
-	uint32_t gap_at_end;
-
-	check_head_block(conf);
-
-	gap_at_beg = conf->off_in_head;
-
-	if (has_tail_block(conf) || has_full_blocks(conf) || conf->off_in_tail == 0 )
-		gap_at_end = 0;
-	else
-		gap_at_end = get_atom_size(object) - conf->off_in_tail;
-
-	return get_atom_size(object) - (gap_at_beg + gap_at_end);
-}
-
-static uint32_t io_size_nopad_tail(struct avec_config *conf,
-				   struct object_cipher_info *object)
-{
-	check_tail_block(conf);
-	return conf->off_in_tail;
-}
-
-static uint32_t io_size_nopad_full(struct avec_config *conf,
-				   struct object_cipher_info *object)
-{	
-	check_full_block(conf);
-	return get_atom_size(object);
-}
-
-static uint32_t io_size_nopad_data_head(call_frame_t *frame,
-					struct object_cipher_info *object)
-{
-	return io_size_nopad_head(get_data_conf(frame), object);
-}
-
-static uint32_t io_size_nopad_hole_head(call_frame_t *frame,
-					struct object_cipher_info *object)
-{
-	return io_size_nopad_head(get_hole_conf(frame), object);
-}
-
-static uint32_t io_size_nopad_data_tail(call_frame_t *frame,
-					struct object_cipher_info *object)
-{
-	return io_size_nopad_tail(get_data_conf(frame), object);
-}
-
-static uint32_t io_size_nopad_hole_tail(call_frame_t *frame,
-					struct object_cipher_info *object)
-{
-	return io_size_nopad_tail(get_hole_conf(frame), object);
-}
-
-static uint32_t io_size_nopad_data_full(call_frame_t *frame,
-					struct object_cipher_info *object)
-{
-	return io_size_nopad_full(get_data_conf(frame), object);
-}
-
-static uint32_t io_size_nopad_hole_full(call_frame_t *frame,
-					struct object_cipher_info *object)
-{
-	return io_size_nopad_full(get_hole_conf(frame), object);
-}
-
-static uint32_t offset_in_head(struct avec_config *conf)
-{
-	check_cursor_head(conf);
-
-	return conf->off_in_head;
-}
-
-static uint32_t offset_in_tail(call_frame_t *frame,
-			       struct object_cipher_info *object)
-{
-	return 0;
-}
-
-static uint32_t offset_in_full(struct avec_config *conf,
-			       struct object_cipher_info *object)
-{
-	check_cursor_full(conf);
-
-	if (has_head_block(conf))
-		return (conf->cursor - 1) << get_atom_bits(object);
-	else
-		return conf->cursor << get_atom_bits(object);
-}
-
-static uint32_t offset_in_data_head(call_frame_t *frame,
-				    struct object_cipher_info *object)
-{
-	return offset_in_head(get_data_conf(frame));
-}
-
-static uint32_t offset_in_hole_head(call_frame_t *frame,
-				    struct object_cipher_info *object)
-{
-	return offset_in_head(get_hole_conf(frame));
-}
-
-static uint32_t offset_in_data_full(call_frame_t *frame,
-				    struct object_cipher_info *object)
-{
-	return offset_in_full(get_data_conf(frame), object);
-}
-
-static uint32_t offset_in_hole_full(call_frame_t *frame,
-				    struct object_cipher_info *object)
-{
-	return offset_in_full(get_hole_conf(frame), object);
-}
-
-/*
- * atom->rmw()
- */
-/*
- * Pre-conditions:
- * @vec contains plain text of the latest
- * version.
- *
- * Uptodate gaps of the @partial block with
- * this plain text, encrypt the whole block
- * and write the result to disk.
- */
-static int32_t rmw_partial_block(call_frame_t *frame,
-				 void *cookie,
-				 xlator_t *this,
-				 int32_t op_ret,
-				 int32_t op_errno,
-				 struct iovec *vec,
-				 int32_t count,
-				 struct iatt *stbuf,
-				 struct iobref *iobref,
-				 struct rmw_atom *atom)
-{
-	size_t was_read = 0;
-	uint64_t file_size;
-	crypt_local_t *local = frame->local;
-	struct object_cipher_info *object = &local->info->cinfo;
-
-	struct iovec *partial = atom->get_iovec(frame, 0);
-	struct avec_config *conf = atom->get_config(frame);
-	end_writeback_handler_t end_writeback_partial_block;
-#if DEBUG_CRYPT
-	gf_boolean_t check_last_cblock = _gf_false;
-#endif
-	local->op_ret = op_ret;
-	local->op_errno = op_errno;
-
-	if (op_ret < 0)
-		goto exit;
-
-	file_size = local->cur_file_size;
-	was_read = op_ret;
-
-	if (atom->locality == HEAD_ATOM && conf->off_in_head) {
-		/*
-		 * head atom with a non-uptodate gap
-		 * at the beginning
-		 *
-		 * fill the gap with plain text of the
-		 * latest version. Convert a part of hole
-		 * (if any) to zeros.
-		 */
-		int32_t i;
-		int32_t copied = 0;
-		int32_t to_gap; /* amount of data needed to uptodate
-				   the gap at the beginning */
-#if 0
-		int32_t hole = 0; /* The part of the hole which
-				   * got in the head block */
-#endif /* 0 */
-		to_gap = conf->off_in_head;
-
-		if (was_read < to_gap) {
-			if (file_size >
-			    offset_at_head(conf) + was_read) {
-				/*
-				 * It is impossible to uptodate
-				 * head block: too few bytes have
-				 * been read from disk, so that
-				 * partial write is impossible.
-				 *
-				 * It could happen because of many
-				 * reasons: IO errors, (meta)data
-				 * corruption in the local file system,
-				 * etc.
-				 */
-				gf_log(this->name, GF_LOG_WARNING,
-				     "Can not uptodate a gap at the beginning");
-				local->op_ret = -1;
-				local->op_errno = EIO;
-				goto exit;
-			}
-#if 0
-			hole = to_gap - was_read;
-#endif /* 0 */
-			to_gap = was_read;
-		}
-		/*
-		 * uptodate the gap at the beginning
-		 */
-		for (i = 0; i < count && copied < to_gap; i++) {
-			int32_t to_copy;
-
-			to_copy = vec[i].iov_len;
-			if (to_copy > to_gap - copied)
-				to_copy = to_gap - copied;
-
-			memcpy(partial->iov_base, vec[i].iov_base, to_copy);
-			copied += to_copy;
-		}
-#if 0
-		/*
-		 * If possible, convert part of the
-		 * hole, which got in the head block
-		 */
-		ret = TRY_LOCK(&local->hole_lock);
-		if (!ret) {
-			if (local->hole_handled)
-				/*
-				 * already converted by
-				 * crypt_writev_cbk()
-				 */
-				UNLOCK(&local->hole_lock);
-			else {
-				/*
-				 * convert the part of the hole
-				 * which got in the head block
-				 * to zeros.
-				 *
-				 * Update the orig_offset to make
-				 * sure writev_cbk() won't care
-				 * about this part of the hole.
-				 *
-				 */
-				memset(partial->iov_base + to_gap, 0, hole);
-
-				conf->orig_offset -= hole;
-				conf->orig_size += hole;
-				UNLOCK(&local->hole_lock);
-			}
-		}
-		else /*
-		      * conversion is being performed
-		      * by crypt_writev_cbk()
-		      */
-			;
-#endif /* 0 */
-	}
-	if (atom->locality == TAIL_ATOM ||
-	    (!has_tail_block(conf) && conf->off_in_tail)) {
-		/*
-		 * tail atom, or head atom with a non-uptodate
-		 * gap at the end.
-		 *
-		 * fill the gap at the end of the block
-		 * with plain text of the latest version.
-		 * Pad the result, (if needed)
-		 */
-		int32_t i;
-		int32_t to_gap;
-		int copied;
-		off_t off_in_tail;
-		int32_t to_copy;
-
-		off_in_tail = conf->off_in_tail;
-		to_gap = conf->gap_in_tail;
-
-		if (to_gap && was_read < off_in_tail + to_gap) {
-			/*
-			 * It is impossible to uptodate
-			 * the gap at the end: too few bytes
-			 * have been read from disk, so that
-			 * partial write is impossible.
-			 *
-			 * It could happen because of many
-			 * reasons: IO errors, (meta)data
-			 * corruption in the local file system,
-			 * etc.
-			 */
-			gf_log(this->name, GF_LOG_WARNING,
-			       "Can not uptodate a gap at the end");
-			local->op_ret = -1;
-			local->op_errno = EIO;
-			goto exit;
-		}
-		/*
-		 * uptodate the gap at the end
-		 */
-		copied = 0;
-		to_copy = to_gap;
-		for(i = count - 1; i >= 0 && to_copy > 0; i--) {
-			uint32_t from_vec, off_in_vec;
-
-			off_in_vec = 0;
-			from_vec = vec[i].iov_len;
-			if (from_vec > to_copy) {
-				off_in_vec = from_vec - to_copy;
-				from_vec = to_copy;
-			}
-			memcpy(partial->iov_base +
-			       off_in_tail + to_gap - copied - from_vec,
-			       vec[i].iov_base + off_in_vec,
-			       from_vec);
-
-			gf_log(this->name, GF_LOG_DEBUG,
-			       "uptodate %d bytes at tail. Offset at target(source): %d(%d)",
-			       (int)from_vec,
-			       (int)off_in_tail + to_gap - copied - from_vec,
-			       (int)off_in_vec);
-
-			copied += from_vec;
-			to_copy -= from_vec;
-		}
-		partial->iov_len = off_in_tail + to_gap;
-
-		if (object_alg_should_pad(object)) {
-			int32_t resid = 0;
-			resid = partial->iov_len & (object_alg_blksize(object) - 1);
-			if (resid) {
-				/*
-				 * append a new EOF padding
-				 */
-				local->eof_padding_size =
-					object_alg_blksize(object) - resid;
-
-				gf_log(this->name, GF_LOG_DEBUG,
-				       "set padding size %d",
-				       local->eof_padding_size);
-
-				memset(partial->iov_base + partial->iov_len,
-				       1,
-				       local->eof_padding_size);
-				partial->iov_len += local->eof_padding_size;
-#if DEBUG_CRYPT
-				gf_log(this->name, GF_LOG_DEBUG,
-				       "pad cblock with %d zeros:",
-				       local->eof_padding_size);
-				dump_cblock(this,
-					    (unsigned char *)partial->iov_base +
-					    partial->iov_len - object_alg_blksize(object));
-				check_last_cblock = _gf_true;
-#endif
-			}
-		}
-	}
-	/*
-	 * encrypt the whole block
-	 */
-	encrypt_aligned_iov(object,
-			    partial,
-			    1,
-			    atom->offset_at(frame, object));
-#if DEBUG_CRYPT
-	if (check_last_cblock == _gf_true) {
-		gf_log(this->name, GF_LOG_DEBUG,
-		       "encrypt last cblock with offset %llu",
-		       (unsigned long long)atom->offset_at(frame, object));
-		dump_cblock(this, (unsigned char *)partial->iov_base +
-			    partial->iov_len - object_alg_blksize(object));
-	}
-#endif
-	set_local_io_params_writev(frame, object, atom,
-				   atom->offset_at(frame, object),
-				   iovec_get_size(partial, 1));
-	/*
-	 * write the whole block to disk
-	 */
-	end_writeback_partial_block = dispatch_end_writeback(local->fop);
-	conf->cursor ++;
-	STACK_WIND(frame,
-		   end_writeback_partial_block,
-		   FIRST_CHILD(this),
-		   FIRST_CHILD(this)->fops->writev,
-		   local->fd,
-		   partial,
-		   1,
-		   atom->offset_at(frame, object),
-		   local->flags,
-		   local->iobref_data,
-		   local->xdata);
-
-	gf_log("crypt", GF_LOG_DEBUG,
-	       "submit partial block: %d bytes from %d offset",
-	       (int)iovec_get_size(partial, 1),
-	       (int)atom->offset_at(frame, object));
- exit:
-	return 0;
-}
-
-/*
- * Perform a (read-)modify-write sequence.
- * This should be performed only after approval
- * of upper server-side manager, i.e. the caller
- * needs to make sure this is his turn to rmw.
- */
-void submit_partial(call_frame_t *frame,
-		    xlator_t *this,
-		    fd_t *fd,
-		    atom_locality_type ltype)
-{
-	int32_t ret;
-	dict_t *dict;
-	struct rmw_atom *atom;
-	crypt_local_t *local = frame->local;
-	struct object_cipher_info *object = &local->info->cinfo;
-
-	atom = atom_by_types(local->active_setup, ltype);
-	/*
-	 * To perform the "read" component of the read-modify-write
-	 * sequence the crypt translator does stack_wind to itself.
-	 * 
-	 * Pass current file size to crypt_readv()
-	 */
-	dict = dict_new();
-	if (!dict) {
-		/*
-		 * FIXME: Handle the error
-		 */
-		gf_log("crypt", GF_LOG_WARNING, "Can not alloc dict");
-		return;
-	}
-	ret = dict_set(dict,
-		       FSIZE_XATTR_PREFIX,
-		       data_from_uint64(local->cur_file_size));
-	if (ret) {
-		/*
-		 * FIXME: Handle the error
-		 */
-		dict_unref(dict);
-		gf_log("crypt", GF_LOG_WARNING, "Can not set dict");
-		goto exit;
-	}
-	STACK_WIND(frame,
-		   atom->rmw,
-		   this,
-		   this->fops->readv, /* crypt_readv */
-		   fd,
-		   atom->count_to_uptodate(frame, object), /* count */
-		   atom->offset_at(frame, object), /* offset to read from */
-		   0,
-		   dict);
- exit:
-	dict_unref(dict);
-}
-
-/*
- * submit blocks of FULL_ATOM type
- */
-void submit_full(call_frame_t *frame, xlator_t *this)
-{
-	crypt_local_t *local = frame->local;
-	struct object_cipher_info *object = &local->info->cinfo;
-	struct rmw_atom *atom = atom_by_types(local->active_setup, FULL_ATOM);
-	uint32_t count; /* total number of full blocks to submit */
-	uint32_t granularity; /* number of blocks to submit in one iteration */
-
-	uint64_t off_in_file; /* start offset in the file, bytes */
-	uint32_t off_in_atom; /* start offset in the atom, blocks */
-	uint32_t blocks_written = 0; /* blocks written for this submit */
-
-	struct avec_config *conf = atom->get_config(frame);
-	end_writeback_handler_t end_writeback_full_block;
-	/*
-	 * Write full blocks by groups of granularity size.
-	 */
-	end_writeback_full_block = dispatch_end_writeback(local->fop);
-
-	if (is_ordered_mode(frame)) {
-		uint32_t skip = has_head_block(conf) ? 1 : 0;
-		count = 1;
-		granularity = 1;
-		/*
-		 * calculate start offset using cursor value;
-		 * here we should take into accout head block,
-		 * which corresponds to cursor value 0.
-		 */
-		off_in_file = atom->offset_at(frame, object) +
-			((conf->cursor - skip) << get_atom_bits(object));
-		off_in_atom = conf->cursor - skip;
-	}
-	else {
-		/*
-		 * in parallel mode
-		 */
-		count = conf->nr_full_blocks;
-		granularity = MAX_IOVEC;
-		off_in_file = atom->offset_at(frame, object);
-		off_in_atom = 0;
-	}
-	while (count) {
-		uint32_t blocks_to_write = count;
-
-		if (blocks_to_write > granularity)
-			blocks_to_write = granularity;
-		if (conf->type == HOLE_ATOM)
-			/*
-			 * reset iovec before encryption
-			 */
-			memset(atom->get_iovec(frame, 0)->iov_base,
-			       0,
-			       get_atom_size(object));
-		/*
-		 * encrypt the group
-		 */
-		encrypt_aligned_iov(object,
-				    atom->get_iovec(frame,
-						    off_in_atom +
-						    blocks_written),
-				    blocks_to_write,
-				    off_in_file + (blocks_written <<
-						   get_atom_bits(object)));
-		
-		set_local_io_params_writev(frame, object, atom,
-		        off_in_file + (blocks_written << get_atom_bits(object)),
-			blocks_to_write <<  get_atom_bits(object));
-
-		conf->cursor += blocks_to_write;
-
-		STACK_WIND(frame,
-			   end_writeback_full_block,
-			   FIRST_CHILD(this),
-			   FIRST_CHILD(this)->fops->writev,
-			   local->fd,
-			   atom->get_iovec(frame, off_in_atom + blocks_written),
-			   blocks_to_write,
-			   off_in_file + (blocks_written << get_atom_bits(object)),
-			   local->flags,
-			   local->iobref_data ? local->iobref_data : local->iobref,
-			   local->xdata);
-
-		gf_log("crypt", GF_LOG_DEBUG, "submit %d full blocks from %d offset",
-		       blocks_to_write,
-		       (int)(off_in_file + (blocks_written << get_atom_bits(object))));
-
-		count -= blocks_to_write;
-		blocks_written += blocks_to_write;
-	}
-	return;
-}
-
-static int32_t rmw_data_head(call_frame_t *frame,
-			     void *cookie,
-			     xlator_t *this,
-			     int32_t op_ret,
-			     int32_t op_errno,
-			     struct iovec *vec,
-			     int32_t count,
-			     struct iatt *stbuf,
-			     struct iobref *iobref,
-			     dict_t *xdata)
-{
-	return rmw_partial_block(frame,
-				 cookie,
-				 this,
-				 op_ret,
-				 op_errno,
-				 vec,
-				 count,
-				 stbuf,
-				 iobref,
-				 atom_by_types(DATA_ATOM, HEAD_ATOM));
-}
-
-static int32_t rmw_data_tail(call_frame_t *frame,
-			     void *cookie,
-			     xlator_t *this,
-			     int32_t op_ret,
-			     int32_t op_errno,
-			     struct iovec *vec,
-			     int32_t count,
-			     struct iatt *stbuf,
-			     struct iobref *iobref,
-			     dict_t *xdata)
-{
-	return rmw_partial_block(frame,
-				 cookie,
-				 this,
-				 op_ret,
-				 op_errno,
-				 vec,
-				 count,
-				 stbuf,
-				 iobref,
-				 atom_by_types(DATA_ATOM, TAIL_ATOM));
-}
-
-static int32_t rmw_hole_head(call_frame_t *frame,
-			     void *cookie,
-			     xlator_t *this,
-			     int32_t op_ret,
-			     int32_t op_errno,
-			     struct iovec *vec,
-			     int32_t count,
-			     struct iatt *stbuf,
-			     struct iobref *iobref,
-			     dict_t *xdata)
-{
-	return rmw_partial_block(frame,
-				 cookie,
-				 this,
-				 op_ret,
-				 op_errno,
-				 vec,
-				 count,
-				 stbuf,
-				 iobref,
-				 atom_by_types(HOLE_ATOM, HEAD_ATOM));
-}
-
-static int32_t rmw_hole_tail(call_frame_t *frame,
-			     void *cookie,
-			     xlator_t *this,
-			     int32_t op_ret,
-			     int32_t op_errno,
-			     struct iovec *vec,
-			     int32_t count,
-			     struct iatt *stbuf,
-			     struct iobref *iobref,
-			     dict_t *xdata)
-{
-	return rmw_partial_block(frame,
-				 cookie,
-				 this,
-				 op_ret,
-				 op_errno,
-				 vec,
-				 count,
-				 stbuf,
-				 iobref,
-				 atom_by_types(HOLE_ATOM, TAIL_ATOM));
-}
-
-/*
- * atom->count_to_uptodate()
- */
-static uint32_t count_to_uptodate_head(struct avec_config *conf,
-				       struct object_cipher_info *object)
-{
-	if (conf->acount == 1 && conf->off_in_tail)
-		return get_atom_size(object);
-	else
-		/* there is no need to read the whole head block */
-		return conf->off_in_head;
-}
-
-static uint32_t count_to_uptodate_tail(struct avec_config *conf,
-				       struct object_cipher_info *object)
-{
-	/* we need to read the whole tail block */
-	return get_atom_size(object);
-}
-
-static uint32_t count_to_uptodate_data_head(call_frame_t *frame,
-					    struct object_cipher_info *object)
-{
-	return count_to_uptodate_head(get_data_conf(frame), object);
-}
-
-static uint32_t count_to_uptodate_data_tail(call_frame_t *frame,
-					    struct object_cipher_info *object)
-{
-	return count_to_uptodate_tail(get_data_conf(frame), object);
-}
-
-static uint32_t count_to_uptodate_hole_head(call_frame_t *frame,
-					    struct object_cipher_info *object)
-{
-	return count_to_uptodate_head(get_hole_conf(frame), object);
-}
-
-static uint32_t count_to_uptodate_hole_tail(call_frame_t *frame,
-					    struct object_cipher_info *object)
-{
-	return count_to_uptodate_tail(get_hole_conf(frame), object);
-}
-
-/* atom->get_config() */
-
-static struct avec_config *get_config_data(call_frame_t *frame)
-{
-	return &((crypt_local_t *)frame->local)->data_conf;
-}
-
-static struct avec_config *get_config_hole(call_frame_t *frame)
-{
-	return &((crypt_local_t *)frame->local)->hole_conf;
-}
-
-/*
- * atom->get_iovec()
- */
-static struct iovec *get_iovec_hole_head(call_frame_t *frame,
-					 uint32_t count)
-{
-	struct avec_config *conf = get_hole_conf(frame);
-
-	return conf->avec;
-}
-
-static struct iovec *get_iovec_hole_full(call_frame_t *frame,
-					 uint32_t count)
-{
-	struct avec_config *conf = get_hole_conf(frame);
-
-	return conf->avec + (conf->off_in_head ? 1 : 0);
-}
-
-static inline struct iovec *get_iovec_hole_tail(call_frame_t *frame,
-						uint32_t count)
-{
-	struct avec_config *conf = get_hole_conf(frame);
-
-	return conf->avec + (conf->blocks_in_pool - 1);
-}
-
-static inline struct iovec *get_iovec_data_head(call_frame_t *frame,
-						uint32_t count)
-{
-	struct avec_config *conf = get_data_conf(frame);
-
-	return conf->avec;
-}
-
-static inline struct iovec *get_iovec_data_full(call_frame_t *frame,
-						uint32_t count)
-{
-	struct avec_config *conf = get_data_conf(frame);
-
-	return conf->avec + (conf->off_in_head ? 1 : 0) + count;
-}
-
-static inline struct iovec *get_iovec_data_tail(call_frame_t *frame,
-						uint32_t count)
-{
-	struct avec_config *conf = get_data_conf(frame);
-
-	return conf->avec +
-		(conf->off_in_head ? 1 : 0) +
-		conf->nr_full_blocks;
-}
-
-static struct rmw_atom atoms[LAST_DATA_TYPE][LAST_LOCALITY_TYPE] = {
-	[DATA_ATOM][HEAD_ATOM] =
-	{ .locality = HEAD_ATOM,
-	  .rmw = rmw_data_head,
-	  .offset_at = offset_at_data_head,
-	  .offset_in = offset_in_data_head,
-	  .get_iovec = get_iovec_data_head,
-	  .io_size_nopad = io_size_nopad_data_head,
-	  .count_to_uptodate = count_to_uptodate_data_head,
-	  .get_config = get_config_data
-	},
-	[DATA_ATOM][TAIL_ATOM] =
-	{ .locality = TAIL_ATOM,
-	  .rmw = rmw_data_tail,
-	  .offset_at = offset_at_data_tail,
-	  .offset_in = offset_in_tail,
-	  .get_iovec = get_iovec_data_tail,
-	  .io_size_nopad = io_size_nopad_data_tail,
-	  .count_to_uptodate = count_to_uptodate_data_tail,
-	  .get_config = get_config_data
-	},
-	[DATA_ATOM][FULL_ATOM] =
-	{ .locality = FULL_ATOM,
-	  .offset_at = offset_at_data_full,
-	  .offset_in = offset_in_data_full,
-	  .get_iovec = get_iovec_data_full,
-	  .io_size_nopad = io_size_nopad_data_full,
-	  .get_config = get_config_data
-	},
-	[HOLE_ATOM][HEAD_ATOM] =
-	{ .locality = HEAD_ATOM,
-	  .rmw = rmw_hole_head,
-	  .offset_at = offset_at_hole_head,
-	  .offset_in = offset_in_hole_head,
-	  .get_iovec = get_iovec_hole_head,
-	  .io_size_nopad = io_size_nopad_hole_head,
-	  .count_to_uptodate = count_to_uptodate_hole_head,
-	  .get_config = get_config_hole
-	},
-	[HOLE_ATOM][TAIL_ATOM] =
-	{ .locality = TAIL_ATOM,
-	  .rmw = rmw_hole_tail,
-	  .offset_at = offset_at_hole_tail,
-	  .offset_in = offset_in_tail,
-	  .get_iovec = get_iovec_hole_tail,
-	  .io_size_nopad = io_size_nopad_hole_tail,
-	  .count_to_uptodate = count_to_uptodate_hole_tail,
-	  .get_config = get_config_hole
-	},
-	[HOLE_ATOM][FULL_ATOM] =
-	{ .locality = FULL_ATOM,
-	  .offset_at = offset_at_hole_full,
-	  .offset_in = offset_in_hole_full,
-	  .get_iovec = get_iovec_hole_full,
-	  .io_size_nopad = io_size_nopad_hole_full,
-	  .get_config = get_config_hole
-	}
-};
-
-struct rmw_atom *atom_by_types(atom_data_type data,
-			       atom_locality_type locality)
-{
-	return &atoms[data][locality];
-}
-
-/*
-  Local variables:
-  c-indentation-style: "K&R"
-  mode-name: "LC"
-  c-basic-offset: 8
-  tab-width: 8
-  fill-column: 80
-  scroll-step: 1
-  End:
-*/
diff --git a/xlators/encryption/crypt/src/crypt-common.h b/xlators/encryption/crypt/src/crypt-common.h
deleted file mode 100644
index 7c212ad5d25..00000000000
--- a/xlators/encryption/crypt/src/crypt-common.h
+++ /dev/null
@@ -1,141 +0,0 @@
-/*
-  Copyright (c) 2008-2013 Red Hat, Inc. <http://www.redhat.com>
-  This file is part of GlusterFS.
-
-  This file is licensed to you under your choice of the GNU Lesser
-  General Public License, version 3 or any later version (LGPLv3 or
-  later), or the GNU General Public License, version 2 (GPLv2), in all
-  cases as published by the Free Software Foundation.
-*/
-
-#ifndef __CRYPT_COMMON_H__
-#define __CRYPT_COMMON_H__
-
-#define INVAL_SUBVERSION_NUMBER (0xff)
-#define CRYPT_INVAL_OP   (GF_FOP_NULL)
-
-#define CRYPTO_FORMAT_PREFIX  "trusted.glusterfs.crypt.att.cfmt"
-#define FSIZE_XATTR_PREFIX    "trusted.glusterfs.crypt.att.size"
-#define SUBREQ_PREFIX         "trusted.glusterfs.crypt.msg.sreq"
-#define FSIZE_MSG_PREFIX      "trusted.glusterfs.crypt.msg.size"
-#define DE_MSG_PREFIX         "trusted.glusterfs.crypt.msg.dent"
-#define REQUEST_ID_PREFIX     "trusted.glusterfs.crypt.msg.rqid"
-#define MSGFLAGS_PREFIX       "trusted.glusterfs.crypt.msg.xfgs"
-
-
-/* messages for crypt_open() */
-#define MSGFLAGS_REQUEST_MTD_RLOCK   1 /* take read lock and don't unlock */
-#define MSGFLAGS_REQUEST_MTD_WLOCK   2 /* take write lock and don't unlock */
-
-#define AES_BLOCK_BITS (4) /* AES_BLOCK_SIZE == 1 << AES_BLOCK_BITS */
-
-#define noop   do {; } while (0)
-#define cassert(cond) ({ switch (-1) { case (cond): case 0: break; } })
-#define __round_mask(x, y) ((__typeof__(x))((y)-1))
-#define round_up(x, y) ((((x)-1) | __round_mask(x, y))+1)
-
-/*
- * Format of file's metadata
- */
-struct crypt_format {
-	uint8_t loader_id;  /* version of metadata loader */
-	uint8_t versioned[0]; /* file's metadata of specific version */
-} __attribute__((packed));
-
-typedef enum {
-	AES_CIPHER_ALG,
-	LAST_CIPHER_ALG
-} cipher_alg_t;
-
-typedef enum {
-	XTS_CIPHER_MODE,
-	LAST_CIPHER_MODE
-} cipher_mode_t;
-
-typedef enum {
-	MTD_LOADER_V1,
-	LAST_MTD_LOADER
-} mtd_loader_id;
-
-static inline void msgflags_set_mtd_rlock(uint32_t *flags)
-{
-	*flags |= MSGFLAGS_REQUEST_MTD_RLOCK;
-}
-
-static inline void msgflags_set_mtd_wlock(uint32_t *flags)
-{
-	*flags |= MSGFLAGS_REQUEST_MTD_WLOCK;
-}
-
-static inline gf_boolean_t msgflags_check_mtd_rlock(uint32_t *flags)
-{
-	return *flags & MSGFLAGS_REQUEST_MTD_RLOCK;
-}
-
-static inline gf_boolean_t msgflags_check_mtd_wlock(uint32_t *flags)
-{
-	return *flags & MSGFLAGS_REQUEST_MTD_WLOCK;
-}
-
-static inline gf_boolean_t msgflags_check_mtd_lock(uint32_t *flags)
-{
-	return  msgflags_check_mtd_rlock(flags) ||
-		msgflags_check_mtd_wlock(flags);
-}
-
-/*
- * returns number of logical blocks occupied
- * (maybe partially) by @count bytes
- * at offset @start.
- */
-static inline off_t logical_blocks_occupied(uint64_t start, off_t count,
-					    int blkbits)
-{
-	return ((start + count - 1) >> blkbits) - (start >> blkbits) + 1;
-}
-
-/*
- * are two bytes (represented by offsets @off1
- * and @off2 respectively) in the same logical
- * block.
- */
-static inline int in_same_lblock(uint64_t off1, uint64_t off2,
-				 int blkbits)
-{
-	return off1 >> blkbits == off2 >> blkbits;
-}
-
-static inline void dump_cblock(xlator_t *this, unsigned char *buf)
-{
-	gf_log(this->name, GF_LOG_DEBUG,
-	       "dump cblock: %x %x %x %x %x %x %x %x %x %x %x %x %x %x %x %x",
-	       (buf)[0],
-	       (buf)[1],
-	       (buf)[2],
-	       (buf)[3],
-	       (buf)[4],
-	       (buf)[5],
-	       (buf)[6],
-	       (buf)[7],
-	       (buf)[8],
-	       (buf)[9],
-	       (buf)[10],
-	       (buf)[11],
-	       (buf)[12],
-	       (buf)[13],
-	       (buf)[14],
-	       (buf)[15]);
-}
-
-#endif /* __CRYPT_COMMON_H__ */
-
-/*
-  Local variables:
-  c-indentation-style: "K&R"
-  mode-name: "LC"
-  c-basic-offset: 8
-  tab-width: 8
-  fill-column: 80
-  scroll-step: 1
-  End:
-*/
diff --git a/xlators/encryption/crypt/src/crypt-mem-types.h b/xlators/encryption/crypt/src/crypt-mem-types.h
deleted file mode 100644
index 2eab921fcba..00000000000
--- a/xlators/encryption/crypt/src/crypt-mem-types.h
+++ /dev/null
@@ -1,44 +0,0 @@
-/*
-  Copyright (c) 2008-2013 Red Hat, Inc. <http://www.redhat.com>
-  This file is part of GlusterFS.
-
-  This file is licensed to you under your choice of the GNU Lesser
-  General Public License, version 3 or any later version (LGPLv3 or
-  later), or the GNU General Public License, version 2 (GPLv2), in all
-  cases as published by the Free Software Foundation.
-*/
-
-
-#ifndef __CRYPT_MEM_TYPES_H__
-#define __CRYPT_MEM_TYPES_H__
-
-#include "mem-types.h"
-
-enum gf_crypt_mem_types_ {
-	gf_crypt_mt_priv = gf_common_mt_end + 1,
-	gf_crypt_mt_inode,
-	gf_crypt_mt_data,
-	gf_crypt_mt_mtd,
-	gf_crypt_mt_loc,
-	gf_crypt_mt_iatt,
-	gf_crypt_mt_key,
-	gf_crypt_mt_iovec,
-	gf_crypt_mt_char,
-	gf_crypt_mt_end,
-};
-
-#endif /* __CRYPT_MEM_TYPES_H__ */
-
-/*
-  Local variables:
-  c-indentation-style: "K&R"
-  mode-name: "LC"
-  c-basic-offset: 8
-  tab-width: 8
-  fill-column: 80
-  scroll-step: 1
-  End:
-*/
-
-
-
diff --git a/xlators/encryption/crypt/src/crypt.c b/xlators/encryption/crypt/src/crypt.c
deleted file mode 100644
index 2c296061ff7..00000000000
--- a/xlators/encryption/crypt/src/crypt.c
+++ /dev/null
@@ -1,4526 +0,0 @@
-/*
-  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
-  This file is part of GlusterFS.
-
-  This file is licensed to you under your choice of the GNU Lesser
-  General Public License, version 3 or any later version (LGPLv3 or
-  later), or the GNU General Public License, version 2 (GPLv2), in all
-  cases as published by the Free Software Foundation.
-*/
-#include <ctype.h>
-#include <sys/uio.h>
-
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
-
-#include "glusterfs.h"
-#include "xlator.h"
-#include "logging.h"
-#include "defaults.h"
-
-#include "crypt-common.h"
-#include "crypt.h"
-
-static void init_inode_info_head(struct crypt_inode_info *info, fd_t *fd);
-static int32_t init_inode_info_tail(struct crypt_inode_info *info,
-				    struct master_cipher_info *master);
-static int32_t prepare_for_submit_hole(call_frame_t *frame, xlator_t *this,
-				       uint64_t from, off_t size);
-static int32_t load_file_size(call_frame_t *frame, void *cookie,
-			      xlator_t *this, int32_t op_ret, int32_t op_errno,
-			      dict_t *dict, dict_t *xdata);
-static void do_ordered_submit(call_frame_t *frame, xlator_t *this,
-			      atom_data_type dtype);
-static void do_parallel_submit(call_frame_t *frame, xlator_t *this,
-			       atom_data_type dtype);
-static void put_one_call_open(call_frame_t *frame);
-static void put_one_call_readv(call_frame_t *frame, xlator_t *this);
-static void put_one_call_writev(call_frame_t *frame, xlator_t *this);
-static void put_one_call_ftruncate(call_frame_t *frame, xlator_t *this);
-static void free_avec(struct iovec *avec, char **pool, int blocks_in_pool);
-static void free_avec_data(crypt_local_t *local);
-static void free_avec_hole(crypt_local_t *local);
-
-static crypt_local_t *crypt_alloc_local(call_frame_t *frame, xlator_t *this,
-					glusterfs_fop_t fop)
-{
-	crypt_local_t *local = NULL;
-
-	local = mem_get0(this->local_pool);
-	if (!local) {
-		gf_log(this->name, GF_LOG_ERROR, "out of memory");
-		return NULL;
-	}
-	local->fop = fop;
-	LOCK_INIT(&local->hole_lock);
-	LOCK_INIT(&local->call_lock);
-	LOCK_INIT(&local->rw_count_lock);
-
-	frame->local = local;
-	return local;
-}
-
-struct crypt_inode_info *get_crypt_inode_info(inode_t *inode, xlator_t *this)
-{
-	int ret;
-	uint64_t value = 0;
-	struct crypt_inode_info *info;
-
-	ret = inode_ctx_get(inode, this, &value);
-	if (ret == -1) {
-		gf_log (this->name, GF_LOG_WARNING,
-			"Can not get inode info");
-		return NULL;
-	}
-	info = (struct crypt_inode_info *)(long)value;
-	if (info == NULL) {
-		gf_log (this->name, GF_LOG_WARNING,
-			"Can not obtain inode info");
-		return NULL;
-	}
-	return info;
-}
-
-static struct crypt_inode_info *local_get_inode_info(crypt_local_t *local,
-						      xlator_t *this)
-{
-	if (local->info)
-		return local->info;
-	local->info = get_crypt_inode_info(local->fd->inode, this);
-	return local->info;
-}
-
-static struct crypt_inode_info *alloc_inode_info(crypt_local_t *local,
-						 loc_t *loc)
-{
-	struct crypt_inode_info *info;
-
-	info = GF_CALLOC(1, sizeof(*info), gf_crypt_mt_inode);
-	if (!info) {
-		local->op_ret = -1;
-		local->op_errno = ENOMEM;
-		gf_log ("crypt", GF_LOG_WARNING,
-			"Can not allocate inode info");
-		return NULL;
-	}
-	memset(info, 0, sizeof(*info));
-#if DEBUG_CRYPT
-	info->loc = GF_CALLOC(1, sizeof(*loc), gf_crypt_mt_loc);
-	if (!info->loc) {
-		gf_log("crypt", GF_LOG_WARNING, "Can not allocate loc");
-		GF_FREE(info);
-		return NULL;
-	}
-	if (loc_copy(info->loc, loc)){
-		GF_FREE(info->loc);
-		GF_FREE(info);
-		return NULL;
-	}
-#endif /* DEBUG_CRYPT */
-
-	local->info = info;
-	return info;
-}
-
-static void free_inode_info(struct crypt_inode_info *info)
-{
-#if DEBUG_CRYPT
-	loc_wipe(info->loc);
-	GF_FREE(info->loc);
-#endif
-	memset(info, 0, sizeof(*info));
-	GF_FREE(info);
-}
-
-int crypt_forget (xlator_t *this, inode_t *inode)
-{
-        uint64_t ctx_addr = 0;
-        if (!inode_ctx_del (inode, this, &ctx_addr))
-                free_inode_info((struct crypt_inode_info *)(long)ctx_addr);
-        return 0;
-}
-
-#if DEBUG_CRYPT
-static void check_read(call_frame_t *frame, xlator_t *this, int32_t read,
-		       struct iovec *vec, int32_t count, struct iatt *stbuf)
-{
-	crypt_local_t *local = frame->local;
-	struct object_cipher_info *object = get_object_cinfo(local->info);
-	struct avec_config *conf = &local->data_conf;
-	uint32_t resid = stbuf->ia_size & (object_alg_blksize(object) - 1);
-
-	if (read <= 0)
-		return;
-	if (read != iovec_get_size(vec, count))
-		gf_log ("crypt", GF_LOG_DEBUG,
-			"op_ret differs from amount of read bytes");
-
-	if (object_alg_should_pad(object) && (read & (object_alg_blksize(object) - 1)))
-		gf_log ("crypt", GF_LOG_DEBUG,
-			"bad amount of read bytes (!= 0 mod(cblock size))");
-
-	if (conf->aligned_offset + read >
-	    stbuf->ia_size + (resid ? object_alg_blksize(object) - resid : 0))
-		gf_log ("crypt", GF_LOG_DEBUG,
-			"bad amount of read bytes (too large))");
-
-}
-
-#define PT_BYTES_TO_DUMP (32)
-static void dump_plain_text(crypt_local_t *local, struct iovec *avec)
-{
-	int32_t to_dump;
-	char str[PT_BYTES_TO_DUMP + 1];
-
-	if (!avec)
-		return;
-	to_dump = avec->iov_len;
-	if (to_dump > PT_BYTES_TO_DUMP)
-		to_dump = PT_BYTES_TO_DUMP;
-	memcpy(str, avec->iov_base, to_dump);
-	memset(str + to_dump, '0', 1);
-	gf_log("crypt", GF_LOG_DEBUG, "Read file: %s", str);
-}
-
-static int32_t data_conf_invariant(struct avec_config *conf)
-{
-	return conf->acount ==
-		!!has_head_block(conf) +
-		!!has_tail_block(conf)+
-		conf->nr_full_blocks;
-}
-
-static int32_t hole_conf_invariant(struct avec_config *conf)
-{
-	return conf->blocks_in_pool ==
-		!!has_head_block(conf) +
-		!!has_tail_block(conf)+
-		!!has_full_blocks(conf);
-}
-
-static void crypt_check_conf(struct avec_config *conf)
-{
-	int32_t ret = 0;
-	const char *msg;
-
-	switch (conf->type) {
-	case DATA_ATOM:
-		msg = "data";
-		ret = data_conf_invariant(conf);
-		break;
-	case HOLE_ATOM:
-		msg = "hole";
-		ret = hole_conf_invariant(conf);
-		break;
-	default:
-		msg = "unknown";
-	}
-	if (!ret)
-		gf_log("crypt", GF_LOG_DEBUG, "bad %s conf", msg);
-}
-
-static void check_buf(call_frame_t *frame, xlator_t *this, struct iatt *buf)
-{
-	crypt_local_t *local = frame->local;
-	struct object_cipher_info *object = &local->info->cinfo;
-	uint64_t local_file_size;
-
-	switch(local->fop) {
-	case GF_FOP_FTRUNCATE:
-		return;
-	case GF_FOP_WRITE:
-		local_file_size = local->new_file_size;
-		break;
-	case GF_FOP_READ:
-		if (parent_is_crypt_xlator(frame, this))
-			return;
-		local_file_size = local->cur_file_size;
-		break;
-	default:
-		gf_log("crypt", GF_LOG_DEBUG, "bad file operation");
-		return;
-	}
-	if (buf->ia_size != round_up(local_file_size,
-					 object_alg_blksize(object)))
-		gf_log("crypt", GF_LOG_DEBUG,
-		       "bad ia_size in buf (%llu), should be %llu",
-		       (unsigned long long)buf->ia_size,
-		       (unsigned long long)round_up(local_file_size,
-						    object_alg_blksize(object)));
-}
-
-#else
-#define check_read(frame, this, op_ret, vec, count, stbuf) noop
-#define dump_plain_text(local, avec) noop
-#define crypt_check_conf(conf) noop
-#define check_buf(frame, this, buf) noop
-#endif /* DEBUG_CRYPT */
-
-/*
- * Pre-conditions:
- * @vec represents a ciphertext of expanded size and
- * aligned offset.
- *
- * Compound a temporal vector @avec with block-aligned
- * components, decrypt and fix it up to represent a chunk
- * of data corresponding to the original size and offset.
- * Pass the result to the next translator.
- */
-int32_t crypt_readv_cbk(call_frame_t *frame,
-			void *cookie,
-			xlator_t *this,
-			int32_t op_ret,
-			int32_t op_errno,
-			struct iovec *vec,
-			int32_t count,
-			struct iatt *stbuf,
-			struct iobref *iobref,
-			dict_t *xdata)
-{
-	crypt_local_t *local = frame->local;
-	struct avec_config *conf = &local->data_conf;
-	struct object_cipher_info *object = &local->info->cinfo;
-
-	struct iovec *avec;
-	uint32_t i;
-	uint32_t to_vec;
-	uint32_t to_user;
-
-	check_buf(frame, this, stbuf);
-	check_read(frame, this, op_ret, vec, count, stbuf);
-
-	local->op_ret = op_ret;
-	local->op_errno = op_errno;
-	local->iobref = iobref_ref(iobref);
-
-	local->buf = *stbuf;
-	local->buf.ia_size = local->cur_file_size;
-
-	if (op_ret <= 0 || count == 0 || vec[0].iov_len == 0)
-		goto put_one_call;
-
-	if (conf->orig_offset >= local->cur_file_size) {
-		local->op_ret = 0;
-		goto put_one_call;
-	}
-	/* 
-	 * correct config params with real file size
-	 * and actual amount of bytes read
-	 */
-	set_config_offsets(frame, this,
-			   conf->orig_offset, op_ret, DATA_ATOM, 0);
-
-	if (conf->orig_offset + conf->orig_size > local->cur_file_size)
-		conf->orig_size = local->cur_file_size - conf->orig_offset;
-	/*
-	 * calculate amount of data to be returned
-	 * to user.
-	 */
-	to_user = op_ret;
-	if (conf->aligned_offset + to_user <= conf->orig_offset) {
-		gf_log(this->name, GF_LOG_WARNING, "Incomplete read");
-		local->op_ret = -1;
-		local->op_errno = EIO;
-		goto put_one_call;
-	}
-	to_user -= (conf->aligned_offset - conf->orig_offset);
-
-	if (to_user > conf->orig_size)
-		to_user = conf->orig_size;
-	local->rw_count = to_user;
-
-	op_errno = set_config_avec_data(this, local,
-					conf, object, vec, count);
-	if (op_errno) {
-		local->op_ret = -1;
-		local->op_errno = op_errno;
-		goto put_one_call;
-	}
-	avec = conf->avec;
-#if DEBUG_CRYPT
-	if (conf->off_in_tail != 0 &&
-	    conf->off_in_tail < object_alg_blksize(object) &&
-	    object_alg_should_pad(object))
-		gf_log(this->name, GF_LOG_DEBUG, "Bad offset in tail %d",
-		       conf->off_in_tail);
-	if (iovec_get_size(vec, count) != 0 &&
-	    in_same_lblock(conf->orig_offset + iovec_get_size(vec, count) - 1,
-			   local->cur_file_size - 1,
-			   object_alg_blkbits(object))) {
-		gf_log(this->name, GF_LOG_DEBUG, "Compound last cblock");
-		dump_cblock(this,
-			    (unsigned char *)(avec[conf->acount - 1].iov_base) +
-			    avec[conf->acount - 1].iov_len - object_alg_blksize(object));
-		dump_cblock(this,
-			    (unsigned char *)(vec[count - 1].iov_base) +
-			    vec[count - 1].iov_len - object_alg_blksize(object));
-	}
-#endif
-	decrypt_aligned_iov(object, avec,
-			    conf->acount, conf->aligned_offset);
-	/*
-	 * pass proper plain data to user
-	 */
-	avec[0].iov_base += (conf->aligned_offset - conf->orig_offset);
-	avec[0].iov_len  -= (conf->aligned_offset - conf->orig_offset);
-
-	to_vec = to_user;
-	for (i = 0; i < conf->acount; i++) {
-		if (avec[i].iov_len > to_vec)
-			avec[i].iov_len = to_vec;
-		to_vec -= avec[i].iov_len;
-	}
- put_one_call:
-	put_one_call_readv(frame, this);
-	return 0;
-}
-
-static int32_t do_readv(call_frame_t *frame,
-			void *cookie,
-			xlator_t *this,
-			int32_t op_ret,
-			int32_t op_errno,
-			dict_t *dict,
-			dict_t *xdata)
-{
-	data_t *data;
-	crypt_local_t *local = frame->local;
-
-	if (op_ret < 0)
-		goto error;	
-	/*
-	 * extract regular file size
-	 */
-	data = dict_get(dict, FSIZE_XATTR_PREFIX);
-	if (!data) {
-		gf_log("crypt", GF_LOG_WARNING, "Regular file size not found");
-		op_errno = EIO;
-		goto error;
-	}
-	local->cur_file_size = data_to_uint64(data);
-	
-	get_one_call(frame);
-	STACK_WIND(frame,
-		   crypt_readv_cbk,
-		   FIRST_CHILD (this),
-		   FIRST_CHILD (this)->fops->readv,
-		   local->fd,
-		   /*
-		    * FIXME: read amount can be reduced
-		    */
-		   local->data_conf.expanded_size,
-		   local->data_conf.aligned_offset,
-		   local->flags,
-		   local->xdata);
-	return 0;
- error:
-	local->op_ret = -1;
-	local->op_errno = op_errno;
-
-	get_one_call(frame);
-	put_one_call_readv(frame, this);
-	return 0;
-}
-
-static int32_t crypt_readv_finodelk_cbk(call_frame_t *frame,
-					void *cookie,
-					xlator_t *this,
-					int32_t op_ret,
-					int32_t op_errno,
-					dict_t *xdata)
-{
-	crypt_local_t *local = frame->local;
-
-	if (op_ret < 0)
-		goto error;
-	/*
-	 * An access has been granted,
-	 * retrieve file size
-	 */
-	STACK_WIND(frame,
-		   do_readv,
-		   FIRST_CHILD(this),
-		   FIRST_CHILD(this)->fops->fgetxattr,
-		   local->fd,
-		   FSIZE_XATTR_PREFIX,
-		   NULL);
-	return  0;
- error:
-	fd_unref(local->fd);
-	if (local->xdata)
-		dict_unref(local->xdata);
-	STACK_UNWIND_STRICT(readv,
-			    frame,
-			    -1,
-			    op_errno,
-			    NULL,
-			    0,
-			    NULL,
-			    NULL,
-			    NULL);
-	return 0;
-}
-
-static int32_t readv_trivial_completion(call_frame_t *frame,
-					void *cookie,
-					xlator_t *this,
-					int32_t op_ret,
-					int32_t op_errno,
-					struct iatt *buf,
-					dict_t *xdata)
-{
-	crypt_local_t *local = frame->local;
-
-	local->op_ret = op_ret;
-	local->op_errno = op_errno;
-
-	if (op_ret < 0) {
-		gf_log(this->name, GF_LOG_WARNING,
-		       "stat failed (%d)", op_errno);
-		goto error;
-	}
-	local->buf = *buf;
-	STACK_WIND(frame,
-		   load_file_size,
-		   FIRST_CHILD(this),
-		   FIRST_CHILD(this)->fops->getxattr,
-		   local->loc,
-		   FSIZE_XATTR_PREFIX,
-		   NULL);
-	return 0;	
- error:
-	STACK_UNWIND_STRICT(readv, frame, op_ret, op_errno,
-			    NULL, 0, NULL, NULL, NULL);
-	return 0;
-}
-
-int32_t crypt_readv(call_frame_t *frame,
-		    xlator_t *this,
-		    fd_t *fd,
-		    size_t size,
-		    off_t offset,
-		    uint32_t flags, dict_t *xdata)
-{
-	int32_t ret;
-	crypt_local_t *local;
-	struct crypt_inode_info *info;
-	struct gf_flock lock = {0, };
-
-#if DEBUG_CRYPT
-	gf_log("crypt", GF_LOG_DEBUG, "reading %d bytes from offset %llu",
-	       (int)size, (long long)offset);
-	if (parent_is_crypt_xlator(frame, this))
-		gf_log("crypt", GF_LOG_DEBUG, "parent is crypt");
-#endif	
-	local = crypt_alloc_local(frame, this, GF_FOP_READ);
-	if (!local) {
-		ret = ENOMEM;
-		goto error;
-	}
-	if (size == 0)
-		goto trivial;
-
-	local->fd = fd_ref(fd);
-	local->flags = flags;
-
-	info = local_get_inode_info(local, this);
-	if (info == NULL) {
-		ret = EINVAL;
-		fd_unref(fd);
-		goto error;
-	}
-	if (!object_alg_atomic(&info->cinfo)) {
-		ret = EINVAL;
-		fd_unref(fd);
-		goto error;
-	}
-	set_config_offsets(frame, this, offset, size,
-			   DATA_ATOM, 0);
-	if (parent_is_crypt_xlator(frame, this)) {
-		data_t *data;
-		/*
-		 * We are called by crypt_writev (or cypt_ftruncate)
-		 * to perform the "read" component of the read-modify-write
-		 * (or read-prune-write) sequence for some atom;
-		 *
-		 * don't ask for access:
-		 * it has already been acquired
-		 *
-		 * Retrieve current file size
-		 */
-		if (!xdata) {
-			gf_log("crypt", GF_LOG_WARNING,
-			       "Regular file size hasn't been passed");
-			ret = EIO;
-			goto error;
-		}
-		data = dict_get(xdata, FSIZE_XATTR_PREFIX);
-		if (!data) {
-			gf_log("crypt", GF_LOG_WARNING,
-			       "Regular file size not found");
-			ret = EIO;
-			goto error;
-		}
-		local->old_file_size =
-			local->cur_file_size = data_to_uint64(data);
-
-		get_one_call(frame);
-		STACK_WIND(frame,
-			   crypt_readv_cbk,
-			   FIRST_CHILD(this),
-			   FIRST_CHILD(this)->fops->readv,
-			   local->fd,
-			   /*
-			    * FIXME: read amount can be reduced
-			    */
-			   local->data_conf.expanded_size,
-			   local->data_conf.aligned_offset,
-			   flags,
-			   NULL);
-		return 0;
-	}
-	if (xdata)
-		local->xdata = dict_ref(xdata);
-
-	lock.l_len    = 0;
-        lock.l_start  = 0;
-        lock.l_type   = F_RDLCK;
-        lock.l_whence = SEEK_SET;
-
-	STACK_WIND(frame,
-		   crypt_readv_finodelk_cbk,
-		   FIRST_CHILD(this),
-		   FIRST_CHILD(this)->fops->finodelk,
-		   this->name,
-		   fd,
-		   F_SETLKW,
-		   &lock,
-		   NULL);
-	return 0;
- trivial:
-	STACK_WIND(frame,
-		   readv_trivial_completion,
-		   FIRST_CHILD(this),
-		   FIRST_CHILD(this)->fops->fstat,
-		   fd,
-		   NULL);
-	return 0;
- error:
-	STACK_UNWIND_STRICT(readv,
-			    frame,
-			    -1,
-			    ret,
-			    NULL,
-			    0,
-			    NULL,
-			    NULL,
-			    NULL);
-	return 0;
-}
-
-void set_local_io_params_writev(call_frame_t *frame,
-				struct object_cipher_info *object,
-				struct rmw_atom *atom,
-				off_t io_offset,
-				uint32_t io_size)
-{
-	crypt_local_t *local = frame->local;
-
-	local->io_offset = io_offset;
-	local->io_size = io_size;
-
-	local->io_offset_nopad =
-		atom->offset_at(frame, object) + atom->offset_in(frame, object);
-
-	gf_log("crypt", GF_LOG_DEBUG,
-	       "set nopad offset to %llu",
-	       (unsigned long long)local->io_offset_nopad);
-
-	local->io_size_nopad = atom->io_size_nopad(frame, object);
-
-	gf_log("crypt", GF_LOG_DEBUG,
-	       "set nopad size to %llu",
-	       (unsigned long long)local->io_size_nopad);
-
-	local->update_disk_file_size = 0;
-	/*
-	 * NOTE: eof_padding_size is 0 for all full atoms;
-	 * For head and tail atoms it will be set up at rmw_partial block()
-	 */
-	local->new_file_size = local->cur_file_size;
-
-	if (local->io_offset_nopad + local->io_size_nopad > local->cur_file_size) {
-
-		local->new_file_size = local->io_offset_nopad + local->io_size_nopad;
-
-		gf_log("crypt", GF_LOG_DEBUG,
-		       "set new file size to %llu",
-		       (unsigned long long)local->new_file_size);
-
-		local->update_disk_file_size = 1;
-	}
-}
-
-void set_local_io_params_ftruncate(call_frame_t *frame,
-				   struct object_cipher_info *object)
-{
-	uint32_t resid;
-	crypt_local_t *local = frame->local;
-	struct avec_config *conf = &local->data_conf;
-
-	resid = conf->orig_offset & (object_alg_blksize(object) - 1);
-	if (resid) {
-		local->eof_padding_size =
-			object_alg_blksize(object) - resid;
-		local->new_file_size = conf->aligned_offset;
-		local->update_disk_file_size = 0;
-		/*
-		 * file size will be updated
-		 * in the ->writev() stack,
-		 * when submitting file tail
-		 */
-	}
-	else {	
-		local->eof_padding_size = 0;
-		local->new_file_size = conf->orig_offset;
-		local->update_disk_file_size = 1;
-		/*
-		 * file size will be updated
-		 * in this ->ftruncate stack
-		 */
-	}
-}
-
-static inline void submit_head(call_frame_t *frame, xlator_t *this)
-{
-	crypt_local_t *local = frame->local;
-	submit_partial(frame, this, local->fd, HEAD_ATOM);
-}
-
-static inline void submit_tail(call_frame_t *frame, xlator_t *this)
-{
-	crypt_local_t *local = frame->local;
-	submit_partial(frame, this, local->fd, TAIL_ATOM);
-}
-
-static void submit_hole(call_frame_t *frame, xlator_t *this)
-{
-	/*
-	 * hole conversion always means
-	 * appended write and goes in ordered fashion
-	 */
-	do_ordered_submit(frame, this, HOLE_ATOM);
-}
-
-static void submit_data(call_frame_t *frame, xlator_t *this)
-{
-	if (is_ordered_mode(frame)) {
-		do_ordered_submit(frame, this, DATA_ATOM);
-		return;
-	}
-	gf_log("crypt", GF_LOG_WARNING, "Bad submit mode");
-	get_nr_calls(frame, nr_calls_data(frame));
-	do_parallel_submit(frame, this, DATA_ATOM);
-	return;
-}
-
-/*
- * heplers called by writev_cbk, fruncate_cbk in ordered mode
- */
-
-static inline int32_t should_submit_hole(crypt_local_t *local)
-{
-	struct avec_config *conf = &local->hole_conf;
-
-	return conf->avec != NULL;
-}
-
-static inline int32_t should_resume_submit_hole(crypt_local_t *local)
-{
-	struct avec_config *conf = &local->hole_conf;
-
-	if (local->fop == GF_FOP_WRITE && has_tail_block(conf))
-		/*
-		 * Don't submit a part of hole, which
-		 * fits into a data block:
- 		 * this part of hole will be converted
-		 * as a gap filled by zeros in data head
-		 * block.
-		 */
-		return conf->cursor < conf->acount - 1;
-	else
-		return conf->cursor < conf->acount;
-}
-
-static inline int32_t should_resume_submit_data(call_frame_t *frame)
-{
-	crypt_local_t *local = frame->local;
-	struct avec_config *conf = &local->data_conf;
-
-	if (is_ordered_mode(frame))
-		return conf->cursor < conf->acount;
-	/*
-	 * parallel writes
-	 */
-	return 0;
-}
-
-static inline int32_t should_submit_data_after_hole(crypt_local_t *local)
-{
-	return local->data_conf.avec != NULL;
-}
-
-static void update_local_file_params(call_frame_t *frame,
-				     xlator_t *this,
-				     struct iatt *prebuf,
-				     struct iatt *postbuf)
-{
-	crypt_local_t *local = frame->local;
-
-	check_buf(frame, this, postbuf);
-
-	local->prebuf = *prebuf;
-	local->postbuf = *postbuf;
-
-	local->prebuf.ia_size  = local->cur_file_size;
-	local->postbuf.ia_size = local->new_file_size;
-
-	local->cur_file_size = local->new_file_size;	
-}
-
-static int32_t end_writeback_writev(call_frame_t *frame,
-				    void *cookie,
-				    xlator_t *this,
-				    int32_t op_ret,
-				    int32_t op_errno,
-				    struct iatt *prebuf,
-				    struct iatt *postbuf,
-				    dict_t *xdata)
-{
-	crypt_local_t *local = frame->local;
-
-	local->op_ret = op_ret;
-	local->op_errno = op_errno;
-
-	if (op_ret <= 0) {
-		gf_log(this->name, GF_LOG_WARNING,
-		       "writev iteration failed");
-		goto put_one_call;
-	}
-	/*
-	 * op_ret includes paddings (atom's head, atom's tail and EOF)
-	 */
-	if (op_ret < local->io_size) {
-		gf_log(this->name, GF_LOG_WARNING,
-		       "Incomplete writev iteration");
-		goto put_one_call;
-	}
-	op_ret -= local->eof_padding_size;
-	local->op_ret = op_ret;
-
-	update_local_file_params(frame, this, prebuf, postbuf);
-
-	if (data_write_in_progress(local)) {
-
-		LOCK(&local->rw_count_lock);
-		local->rw_count += op_ret;
-		UNLOCK(&local->rw_count_lock);
-
-		if (should_resume_submit_data(frame))
-			submit_data(frame, this);
-	}
-	else {
-		/*
-		 * hole conversion is going on;
-		 * don't take into account written zeros
-		 */
-		if (should_resume_submit_hole(local))
-			submit_hole(frame, this);
-
-		else if (should_submit_data_after_hole(local))
-			submit_data(frame, this);
-	}
- put_one_call:
-	put_one_call_writev(frame, this);
-	return 0;
-}
-
-#define crypt_writev_cbk end_writeback_writev
-
-#define HOLE_WRITE_CHUNK_BITS 12
-#define HOLE_WRITE_CHUNK_SIZE (1 << HOLE_WRITE_CHUNK_BITS)
-
-/*
- * Convert hole of size @size at offset @off to
- * zeros and prepare respective iovecs for submit.
- * The hole lock should be held.
- *
- * Pre-conditions:
- * @local->file_size is set and valid.
- */
-int32_t prepare_for_submit_hole(call_frame_t *frame, xlator_t *this,
-				uint64_t off, off_t size)
-{
-	int32_t ret;
-	crypt_local_t *local = frame->local;
-	struct object_cipher_info *object = &local->info->cinfo;
-
-	set_config_offsets(frame, this, off, size, HOLE_ATOM, 1);
-
-	ret = set_config_avec_hole(this, local,
-				   &local->hole_conf, object, local->fop);
-	crypt_check_conf(&local->hole_conf);
-
-	return ret;
-}
-
-/*
- * prepare for submit @count bytes at offset @from
- */
-int32_t prepare_for_submit_data(call_frame_t *frame, xlator_t *this,
-				off_t from, int32_t size, struct iovec *vec,
-				int32_t vec_count, int32_t setup_gap)
-{
-	uint32_t ret;
-	crypt_local_t *local = frame->local;
-	struct object_cipher_info *object = &local->info->cinfo;
-
-	set_config_offsets(frame, this, from, size,
-			   DATA_ATOM, setup_gap);
-
-	ret = set_config_avec_data(this, local,
-				   &local->data_conf, object, vec, vec_count);
-	crypt_check_conf(&local->data_conf);
-
-	return ret;
-}
-
-static void free_avec(struct iovec *avec,
-		      char **pool, int blocks_in_pool)
-{
-	if (!avec)
-		return;
-	GF_FREE(pool);
-	GF_FREE(avec);
-}
-
-static void free_avec_data(crypt_local_t *local)
-{
-	return free_avec(local->data_conf.avec,
-			 local->data_conf.pool,
-			 local->data_conf.blocks_in_pool);
-}
-
-static void free_avec_hole(crypt_local_t *local)
-{
-	return free_avec(local->hole_conf.avec,
-			 local->hole_conf.pool,
-			 local->hole_conf.blocks_in_pool);
-}
-
-
-static void do_parallel_submit(call_frame_t *frame, xlator_t *this,
-			       atom_data_type dtype)
-{
-	crypt_local_t *local = frame->local;
-	struct avec_config *conf;
-
-	local->active_setup = dtype;
-	conf = conf_by_type(frame, dtype);
-
-	if (has_head_block(conf))
-		submit_head(frame, this);
-
-	if (has_full_blocks(conf))
-		submit_full(frame, this);
-
-	if (has_tail_block(conf))
-		submit_tail(frame, this);
-	return;
-}
-
-static void do_ordered_submit(call_frame_t *frame, xlator_t *this,
-			      atom_data_type dtype)
-{
-	crypt_local_t *local = frame->local;
-	struct avec_config *conf;
-
-	local->active_setup = dtype;
-	conf = conf_by_type(frame, dtype);
-
-	if (should_submit_head_block(conf)) {
-		get_one_call_nolock(frame);
-		submit_head(frame, this);
-	}
-	else if (should_submit_full_block(conf)) {
-		get_one_call_nolock(frame);
-		submit_full(frame, this);
-	}
-	else if (should_submit_tail_block(conf)) {
-		get_one_call_nolock(frame);
-		submit_tail(frame, this);
-	}
-	else
-		gf_log("crypt", GF_LOG_DEBUG,
-		       "nothing has been submitted in ordered mode");
-	return;
-}
-
-static int32_t do_writev(call_frame_t *frame,
-			 void *cookie,
-			 xlator_t *this,
-			 int32_t op_ret,
-			 int32_t op_errno,
-			 dict_t *dict,
-			 dict_t *xdata)
-{
-	data_t *data;
-	crypt_local_t *local = frame->local;
-	struct object_cipher_info *object = &local->info->cinfo;
-	/*
-	 * extract regular file size
-	 */
-	data = dict_get(dict, FSIZE_XATTR_PREFIX);
-	if (!data) {
-		gf_log("crypt", GF_LOG_WARNING, "Regular file size not found");
-		op_ret = -1;
-		op_errno = EIO;
-		goto error;
-	}
-	local->old_file_size = local->cur_file_size = data_to_uint64(data);
-
-	set_gap_at_end(frame, object, &local->data_conf, DATA_ATOM);
-
-	if (local->cur_file_size < local->data_conf.orig_offset) {
-		/*
-		 * Set up hole config
-		 */
-		op_errno = prepare_for_submit_hole(frame,
-			       this,
-			       local->cur_file_size,
-			       local->data_conf.orig_offset - local->cur_file_size);
-		if (op_errno) {
-			local->op_ret = -1;
-			local->op_errno = op_errno;
-			goto error;
-		}
-	}
-	if (should_submit_hole(local))
-		submit_hole(frame, this);
-	else 
-		submit_data(frame, this);
-	return 0;
- error:
-	get_one_call_nolock(frame);
-	put_one_call_writev(frame, this);
-	return 0;
-}
-
-static int32_t crypt_writev_finodelk_cbk(call_frame_t *frame,
-					 void *cookie,
-					 xlator_t *this,
-					 int32_t op_ret,
-					 int32_t op_errno,
-					 dict_t *xdata)
-{
-	crypt_local_t *local = frame->local;
-
-	local->op_ret = op_ret;
-	local->op_errno = op_errno;
-
-	if (op_ret < 0)
-		goto error;
-	/*
-	 * An access has been granted,
-	 * retrieve file size first
-	 */
-	STACK_WIND(frame,
-		   do_writev,
-		   FIRST_CHILD(this),
-		   FIRST_CHILD(this)->fops->fgetxattr,
-		   local->fd,
-		   FSIZE_XATTR_PREFIX,
-		   NULL);
-	return 0;
- error:
-	get_one_call_nolock(frame);
-	put_one_call_writev(frame, this);
-	return 0;	
-}
-
-static int32_t writev_trivial_completion(call_frame_t *frame,
-					 void *cookie,
-					 xlator_t *this,
-					 int32_t op_ret,
-					 int32_t op_errno,
-					 struct iatt *buf,
-					 dict_t *dict)
-{
-	crypt_local_t *local = frame->local;
-	
-	local->op_ret = op_ret;
-	local->op_errno = op_errno;
-	local->prebuf = *buf;
-	local->postbuf = *buf;
-
-	local->prebuf.ia_size = local->cur_file_size;
-	local->postbuf.ia_size = local->cur_file_size;
-
-	get_one_call(frame);
-	put_one_call_writev(frame, this);
-	return 0;
-}
-
-int crypt_writev(call_frame_t *frame,
-		 xlator_t *this,
-		 fd_t *fd,
-		 struct iovec *vec,
-		 int32_t count,
-		 off_t offset,
-		 uint32_t flags,
-		 struct iobref *iobref,
-		 dict_t *xdata)
-{
-	int32_t ret;
-	crypt_local_t *local;
-	struct crypt_inode_info *info;
-	struct gf_flock lock = {0, };
-#if DEBUG_CRYPT
-	gf_log ("crypt", GF_LOG_DEBUG, "writing %d bytes from offset %llu",
-		(int)iovec_get_size(vec, count), (long long)offset);
-#endif
-	local = crypt_alloc_local(frame, this, GF_FOP_WRITE);
-	if (!local) {
-		ret = ENOMEM;
-		goto error;
-	}
-	local->fd = fd_ref(fd);
-
-	if (iobref)
-		local->iobref = iobref_ref(iobref);
-	/*
-	 * to update real file size on the server
-	 */
-	local->xattr = dict_new();
-	if (!local->xattr) {
-		ret = ENOMEM;
-		goto error;
-	}
-	local->flags = flags;
-
-	info = local_get_inode_info(local, this);
-	if (info == NULL) {
-		ret = EINVAL;
-		goto error;
-	}
-	if (!object_alg_atomic(&info->cinfo)) {
-		ret = EINVAL;
-		goto error;
-	}
-	if (iovec_get_size(vec, count) == 0)
-		goto trivial;
-
-	ret = prepare_for_submit_data(frame, this, offset,
-				      iovec_get_size(vec, count),
-				      vec, count, 0 /* don't setup gup
-						       in tail: we don't
-						       know file size yet */);
-	if (ret)
-		goto error;
-
-	if (parent_is_crypt_xlator(frame, this)) {
-		data_t *data;
-		/*
-		 * we are called by shinking crypt_ftruncate(),
-		 * which doesn't perform hole conversion;
-		 *
-		 * don't ask for access:
-		 * it has already been acquired
-		 */
-
-		/*
-		 * extract file size
-		 */
-		if (!xdata) {
-			gf_log("crypt", GF_LOG_WARNING,
-			       "Regular file size hasn't been passed");
-			ret = EIO;
-			goto error;
-		}
-		data = dict_get(xdata, FSIZE_XATTR_PREFIX);
-		if (!data) {
-			gf_log("crypt", GF_LOG_WARNING,
-			       "Regular file size not found");
-			ret = EIO;
-			goto error;
-		}
-		local->old_file_size =
-			local->cur_file_size = data_to_uint64(data);
-
-		submit_data(frame, this);
-		return 0;
-	}
-	if (xdata)
-		local->xdata = dict_ref(xdata);
-	/*
-	 * lock the file and retrieve its size
-	 */
-	lock.l_len    = 0;
-        lock.l_start  = 0;
-        lock.l_type   = F_WRLCK;
-        lock.l_whence = SEEK_SET;
-
-	STACK_WIND(frame,
-		   crypt_writev_finodelk_cbk,
-		   FIRST_CHILD(this),
-		   FIRST_CHILD(this)->fops->finodelk,
-		   this->name,
-		   fd,
-		   F_SETLKW,
-		   &lock,
-		   NULL);
-	return 0;
- trivial:
-	STACK_WIND(frame,
-		   writev_trivial_completion,
-		   FIRST_CHILD(this),
-		   FIRST_CHILD(this)->fops->fstat,
-		   fd,
-		   NULL);
-	return 0;
- error:
-	if (local && local->fd)
-		fd_unref(fd);
-	if (local && local->iobref)
-		iobref_unref(iobref);
-	if (local && local->xdata)
-		dict_unref(xdata);
-	if (local && local->xattr)
-		dict_unref(local->xattr);
-	if (local && local->info)
-		free_inode_info(local->info);
-
-	STACK_UNWIND_STRICT(writev, frame, -1, ret, NULL, NULL, NULL);
-	return 0;
-}
-
-int32_t prepare_for_prune(call_frame_t *frame, xlator_t *this, uint64_t offset)
-{
-	set_config_offsets(frame, this,
-			   offset,
-			   0, /* count */
-			   DATA_ATOM,
-			   0 /* since we prune, there is no
-				gap in tail to uptodate */);
-	return 0;
-}
-
-/*
- * Finish the read-prune-modify sequence
- *
- * Can be invoked as
- * 1) ->ftruncate_cbk() for cblock-aligned, or trivial prune
- * 2) ->writev_cbk() for non-cblock-aligned prune
- */
-
-static int32_t prune_complete(call_frame_t *frame,
-			      void *cookie,
-			      xlator_t *this,
-			      int32_t op_ret,
-			      int32_t op_errno,
-			      struct iatt *prebuf,
-			      struct iatt *postbuf,
-			      dict_t *xdata)
-{
-	crypt_local_t *local = frame->local;
-
-	local->op_ret = op_ret;
-	local->op_errno = op_errno;
-
-	update_local_file_params(frame, this, prebuf, postbuf);
-
-	put_one_call_ftruncate(frame, this);
-	return 0;
-}
-
-/*
- * This is called as ->ftruncate_cbk()
- *
- * Perform the "write" component of the
- * read-prune-write sequence.
- *
- * submuit the rest of the file
- */
-static int32_t prune_submit_file_tail(call_frame_t *frame,
-				      void *cookie,
-				      xlator_t *this,
-				      int32_t op_ret,
-				      int32_t op_errno,
-				      struct iatt *prebuf,
-				      struct iatt *postbuf,
-				      dict_t *xdata)
-{
-	crypt_local_t *local = frame->local;
-	struct avec_config *conf = &local->data_conf;
-	dict_t *dict;
-
-	if (op_ret < 0)
-		goto put_one_call;
-
-	if (local->xdata) {
-		dict_unref(local->xdata);
-		local->xdata = NULL;
-	}
-	if (xdata)
-		local->xdata = dict_ref(xdata);
-
-	dict = dict_new();
-	if (!dict) {
-		op_errno = ENOMEM;
-		goto error;
-	}
-
-	update_local_file_params(frame, this, prebuf, postbuf);
-	local->new_file_size = conf->orig_offset;
-
-	/*
-	 * The rest of the file is a partial block and, hence,
-	 * should be written via RMW sequence, so the crypt xlator
-	 * does STACK_WIND to itself.
-	 *
-	 * Pass current file size to crypt_writev()
-	 */
-	op_errno = dict_set(dict,
-			    FSIZE_XATTR_PREFIX,
-			    data_from_uint64(local->cur_file_size));
-	if (op_errno) {
-		gf_log("crypt", GF_LOG_WARNING,
-		       "can not set key to update file size");
-		dict_unref(dict);
-		goto error;
-	}
-	gf_log("crypt", GF_LOG_DEBUG,
-	       "passing current file size (%llu) to crypt_writev",
-	       (unsigned long long)local->cur_file_size);
-	/*
-	 * Padding will be filled with
-	 * zeros by rmw_partial_block()
-	 */
-	STACK_WIND(frame,
-		   prune_complete,
-		   this,
-		   this->fops->writev, /* crypt_writev */
-		   local->fd,
-		   &local->vec,
-		   1,
-		   conf->aligned_offset, /* offset to write from */
-		   0,
-		   local->iobref,
-		   dict);
-
-	dict_unref(dict);
-	return 0;
- error:
-	local->op_ret = -1;
-	local->op_errno = op_errno;
- put_one_call:
-	put_one_call_ftruncate(frame, this);
-	return 0;
-}
-
-/*
- * This is called as a callback of ->writev() invoked in behalf
- * of ftruncate(): it can be
- * 1) ordered writes issued by hole conversion in the case of
- *    expanded truncate, or
- * 2) an rmw partial data block issued by non-cblock-aligned
- * prune.
- */
-int32_t end_writeback_ftruncate(call_frame_t *frame,
-				void *cookie,
-				xlator_t *this,
-				int32_t op_ret,
-				int32_t op_errno,
-				struct iatt *prebuf,
-				struct iatt *postbuf,
-				dict_t *xdata)
-{
-	crypt_local_t  *local = frame->local;
-	/*
-	 * if nothing has been written,
-	 * then it must be an error
-	 */
-	local->op_ret = op_ret;
-	local->op_errno = op_errno;
-
-	if (op_ret < 0)
-		goto put_one_call;
-
-	update_local_file_params(frame, this, prebuf, postbuf);
-
-	if (data_write_in_progress(local))
-		/* case (2) */
-		goto put_one_call;
-	/* case (1) */
-	if (should_resume_submit_hole(local))
-		submit_hole(frame, this);
-	/*
-	 * case of hole, when we should't resume
-	 */
- put_one_call:
-	put_one_call_ftruncate(frame, this);
-	return 0;
-}
-
-/*
- * Perform prune and write components of the
- * read-prune-write sequence.
- *
- * Called as ->readv_cbk()
- *
- * Pre-conditions:
- * @vec contains the latest atom of the file
- * (plain text)
- */
-static int32_t prune_write(call_frame_t *frame,
-			   void *cookie,
-			   xlator_t *this,
-			   int32_t op_ret,
-			   int32_t op_errno,
-			   struct iovec *vec,
-			   int32_t count,
-			   struct iatt *stbuf,
-			   struct iobref *iobref,
-			   dict_t *xdata)
-{
-	int32_t i;
-	size_t to_copy;
-	size_t copied = 0;
-	crypt_local_t *local = frame->local;
-	struct avec_config *conf = &local->data_conf;
-
-	local->op_ret = op_ret;
-	local->op_errno = op_errno;
-	if (op_ret == -1)
-		goto put_one_call;
-
-	/*
-	 * At first, uptodate head block
-	 */
-	if (iovec_get_size(vec, count) < conf->off_in_head) {
-		gf_log(this->name, GF_LOG_WARNING,
-		       "Failed to uptodate head block for prune");
-		local->op_ret = -1;
-		local->op_errno = EIO;
-		goto put_one_call;
-	}
-	local->vec.iov_len = conf->off_in_head;
-	local->vec.iov_base = GF_CALLOC(1, local->vec.iov_len,
-					gf_crypt_mt_data);
-
-	if (local->vec.iov_base == NULL) {
-	        gf_log(this->name, GF_LOG_WARNING,
-                       "Failed to calloc head block for prune");
-		local->op_ret = -1;
-		local->op_errno = ENOMEM;
-	        goto put_one_call;
-	}
-	for (i = 0; i < count; i++) {
-		to_copy = vec[i].iov_len;
-		if (to_copy > local->vec.iov_len - copied)
-			to_copy = local->vec.iov_len - copied;
-
-		memcpy((char *)local->vec.iov_base + copied,
-		       vec[i].iov_base,
-		       to_copy);
-		copied += to_copy;
-		if (copied == local->vec.iov_len)
-			break;
-	}
-	/*
-	 * perform prune with aligned offset
-	 * (i.e. at this step we prune a bit
-	 * more then it is needed
-	 */
-	STACK_WIND(frame,
-		   prune_submit_file_tail,
-		   FIRST_CHILD(this),
-		   FIRST_CHILD(this)->fops->ftruncate,
-		   local->fd,
-		   conf->aligned_offset,
-		   local->xdata);
-	return 0;
- put_one_call:
-	put_one_call_ftruncate(frame, this);
-	return 0;
-}
-
-/*
- * Perform a read-prune-write sequence
- */
-int32_t read_prune_write(call_frame_t *frame, xlator_t *this)
-{
-	int32_t ret = 0;
-	dict_t *dict = NULL;
-	crypt_local_t  *local = frame->local;
-	struct avec_config *conf = &local->data_conf;
-	struct object_cipher_info *object = &local->info->cinfo;
-
-	set_local_io_params_ftruncate(frame, object);
-	get_one_call_nolock(frame);
-
-	if ((conf->orig_offset & (object_alg_blksize(object) - 1)) == 0) {
-		/*
-		 * cblock-aligned prune:
-		 * we don't need read and write components,
-		 * just cut file body
-		 */
-		gf_log("crypt", GF_LOG_DEBUG,
-		       "prune without RMW (at offset %llu",
-		       (unsigned long long)conf->orig_offset);
-
-		STACK_WIND(frame,
-			   prune_complete,
-			   FIRST_CHILD(this),
-			   FIRST_CHILD(this)->fops->ftruncate,
-			   local->fd,
-			   conf->orig_offset,
-			   local->xdata);
-		return 0;
-	}
-	gf_log("crypt", GF_LOG_DEBUG,
-	       "prune with RMW (at offset %llu",
-	       (unsigned long long)conf->orig_offset);	
-	/*
-	 * We are about to perform the "read" component of the
-	 * read-prune-write sequence. It means that we need to
-	 * read encrypted data from disk and decrypt it.
-	 * So, the crypt translator does STACK_WIND to itself.
-	 * 
-	 * Pass current file size to crypt_readv()
-	 
-	 */
-	dict = dict_new();
-	if (!dict) {
-		gf_log("crypt", GF_LOG_WARNING, "Can not alloc dict");
-		ret = ENOMEM;
-		goto exit;
-	}
-	ret = dict_set(dict,
-		       FSIZE_XATTR_PREFIX,
-		       data_from_uint64(local->cur_file_size));
-	if (ret) {
-		gf_log("crypt", GF_LOG_WARNING, "Can not set dict");
-		goto exit;
-	}
-	STACK_WIND(frame,
-		   prune_write,
-		   this,
-		   this->fops->readv, /* crypt_readv */
-		   local->fd,
-		   get_atom_size(object), /* bytes to read */
-		   conf->aligned_offset, /* offset to read from */
-		   0,
-		   dict);
- exit:
-	if (dict)
-		dict_unref(dict);
-	return ret;
-}
-
-/*
- * File prune is more complicated than expand.
- * First we need to read the latest atom to not lose info
- * needed for proper update. Also we need to make sure that
- * every component of read-prune-write sequence leaves data
- * consistent
- *
- * Non-cblock aligned prune is performed as read-prune-write
- * sequence:
- *
- * 1) read the latest atom;
- * 2) perform cblock-aligned prune
- * 3) issue a write request for the end-of-file
- */
-int32_t prune_file(call_frame_t *frame, xlator_t *this, uint64_t offset)
-{
-	int32_t ret;
-
-	ret = prepare_for_prune(frame, this, offset);
-	if (ret)
-		return ret;
-	return read_prune_write(frame, this);
-}
-
-int32_t expand_file(call_frame_t *frame, xlator_t *this,
-		    uint64_t offset)
-{
-	int32_t ret;
-	crypt_local_t *local = frame->local;
-
-	ret = prepare_for_submit_hole(frame, this,
-				      local->old_file_size,
-				      offset - local->old_file_size);
-	if (ret)
-		return ret;
-	submit_hole(frame, this);
-	return 0;
-}
-
-static int32_t ftruncate_trivial_completion(call_frame_t *frame,
-					    void *cookie,
-					    xlator_t *this,
-					    int32_t op_ret,
-					    int32_t op_errno,
-					    struct iatt *buf,
-					    dict_t *dict)
-{
-	crypt_local_t *local = frame->local;
-
-	local->op_ret = op_ret;
-	local->op_errno = op_errno;
-	local->prebuf = *buf;
-	local->postbuf = *buf;
-
-	local->prebuf.ia_size = local->cur_file_size;
-	local->postbuf.ia_size = local->cur_file_size;
-
-	get_one_call(frame);
-	put_one_call_ftruncate(frame, this);
-	return 0;
-}
-
-static int32_t do_ftruncate(call_frame_t *frame,
-			    void *cookie,
-			    xlator_t *this,
-			    int32_t op_ret,
-			    int32_t op_errno,
-			    dict_t *dict,
-			    dict_t *xdata)
-{
-	data_t *data;
-	crypt_local_t *local = frame->local;
-
-	if (op_ret)
-		goto error;	
-	/*
-	 * extract regular file size
-	 */
-	data = dict_get(dict, FSIZE_XATTR_PREFIX);
-	if (!data) {
-		gf_log("crypt", GF_LOG_WARNING, "Regular file size not found");
-		op_errno = EIO;
-		goto error;
-	}
-	local->old_file_size = local->cur_file_size = data_to_uint64(data);
-
-	if (local->data_conf.orig_offset == local->cur_file_size) {
-#if DEBUG_CRYPT
-		gf_log("crypt", GF_LOG_DEBUG,
-		       "trivial ftruncate (current file size %llu)",
-		       (unsigned long long)local->cur_file_size);
-#endif
-		goto trivial;
-	}
-	else if (local->data_conf.orig_offset < local->cur_file_size) {
-#if DEBUG_CRYPT
-		gf_log("crypt", GF_LOG_DEBUG, "prune from %llu to %llu",
-		       (unsigned long long)local->cur_file_size,
-		       (unsigned long long)local->data_conf.orig_offset);
-#endif
-		op_errno = prune_file(frame,
-				      this,
-				      local->data_conf.orig_offset);
-	}
-	else {
-#if DEBUG_CRYPT
-		gf_log("crypt", GF_LOG_DEBUG, "expand from %llu to %llu",
-		       (unsigned long long)local->cur_file_size,
-		       (unsigned long long)local->data_conf.orig_offset);
-#endif
-		op_errno = expand_file(frame,
-				       this,
-				       local->data_conf.orig_offset);
-	}
-	if (op_errno)
-		goto error;
-	return 0;
- trivial:
-	STACK_WIND(frame,
-		   ftruncate_trivial_completion,
-		   FIRST_CHILD(this),
-		   FIRST_CHILD(this)->fops->fstat,
-		   local->fd,
-		   NULL);
-	return 0;
- error:
-	/*
-	 * finish with ftruncate
-	 */
-	local->op_ret = -1;
-	local->op_errno = op_errno;
-
-	get_one_call_nolock(frame);
-	put_one_call_ftruncate(frame, this);
-	return 0;
-}
-
-static int32_t crypt_ftruncate_finodelk_cbk(call_frame_t *frame,
-					    void *cookie,
-					    xlator_t *this,
-					    int32_t op_ret,
-					    int32_t op_errno,
-					    dict_t *xdata)
-{
-	crypt_local_t *local = frame->local;
-
-	local->op_ret = op_ret;
-	local->op_errno = op_errno;
-
-	if (op_ret < 0)
-		goto error;
-	/*
-	 * An access has been granted,
-	 * retrieve file size first
-	 */
-	STACK_WIND(frame,
-		   do_ftruncate,
-		   FIRST_CHILD(this),
-		   FIRST_CHILD(this)->fops->fgetxattr,
-		   local->fd,
-		   FSIZE_XATTR_PREFIX,
-		   NULL);
-	return 0;
- error:
-	get_one_call_nolock(frame);
-	put_one_call_ftruncate(frame, this);
-	return 0;
-}
-
-/*
- * ftruncate is performed in 2 steps:
- * . receive file size;
- * . expand or prune file.
- */
-static int32_t crypt_ftruncate(call_frame_t *frame,
-			       xlator_t *this,
-			       fd_t *fd,
-			       off_t offset,
-			       dict_t *xdata)
-{
-	int32_t ret;
-	crypt_local_t *local;
-	struct crypt_inode_info *info;
-	struct gf_flock lock = {0, };
-
-	local = crypt_alloc_local(frame, this, GF_FOP_FTRUNCATE);
-	if (!local) {
-		ret = ENOMEM;
-		goto error;
-	}
-	local->xattr = dict_new();
-	if (!local->xattr) {
-		ret = ENOMEM;
-		goto error;
-	}
-	local->fd = fd_ref(fd);
-	info = local_get_inode_info(local, this);
-	if (info == NULL) {
-		ret = EINVAL;
-		goto error;
-	}
-	if (!object_alg_atomic(&info->cinfo)) {
-		ret = EINVAL;
-		goto error;
-	}
-	local->data_conf.orig_offset = offset;
-	if (xdata)
-		local->xdata = dict_ref(xdata);
-
-	lock.l_len    = 0;
-        lock.l_start  = 0;
-        lock.l_type   = F_WRLCK;
-        lock.l_whence = SEEK_SET;
-
-	STACK_WIND(frame,
-		   crypt_ftruncate_finodelk_cbk,
-		   FIRST_CHILD(this),
-		   FIRST_CHILD(this)->fops->finodelk,
-		   this->name,
-		   fd,
-		   F_SETLKW,
-		   &lock,
-		   NULL);
-	return 0;
- error:
-	if (local && local->fd)
-		fd_unref(fd);
-	if (local && local->xdata)
-		dict_unref(xdata);
-	if (local && local->xattr)
-		dict_unref(local->xattr);
-	if (local && local->info)
-		free_inode_info(local->info);
-
-	STACK_UNWIND_STRICT(ftruncate, frame, -1, ret, NULL, NULL, NULL);
-	return 0;
-}
-
-/* ->flush_cbk() */
-int32_t truncate_end(call_frame_t *frame,
-		     void *cookie,
-		     xlator_t *this,
-		     int32_t op_ret,
-		     int32_t op_errno,
-		     dict_t *xdata)
-{
-	crypt_local_t *local = frame->local;
-
-	STACK_UNWIND_STRICT(truncate,
-			    frame,
-			    op_ret,
-			    op_errno,
-			    &local->prebuf,
-			    &local->postbuf,
-			    local->xdata);
-	return 0;
-}
-
-/* ftruncate_cbk() */
-int32_t truncate_flush(call_frame_t *frame,
-		       void *cookie,
-		       xlator_t *this,
-		       int32_t op_ret,
-		       int32_t op_errno,
-		       struct iatt *prebuf,
-		       struct iatt *postbuf,
-		       dict_t *xdata)
-{
-	crypt_local_t *local = frame->local;
-	fd_t *fd = local->fd;
-	local->prebuf = *prebuf;
-	local->postbuf = *postbuf;
-
-	STACK_WIND(frame,
-		   truncate_end,
-		   FIRST_CHILD(this),
-		   FIRST_CHILD(this)->fops->flush,
-		   fd,
-		   NULL);
-	fd_unref(fd);
-	return 0;
-}
-
-/*
- * is called as ->open_cbk()
- */
-static int32_t truncate_begin(call_frame_t *frame,
-			      void *cookie,
-			      xlator_t *this,
-			      int32_t op_ret,
-			      int32_t op_errno,
-			      fd_t *fd,
-			      dict_t *xdata)
-{
-	crypt_local_t *local = frame->local;
-
-	if (op_ret < 0) {
-		fd_unref(fd);
-		STACK_UNWIND_STRICT(truncate,
-				    frame,
-				    op_ret,
-				    op_errno, NULL, NULL, NULL);
-		return 0;
-	}
-	/*
-	 * crypt_truncate() is implemented via crypt_ftruncate(),
-	 * so the crypt xlator does STACK_WIND to itself here
-	 */
-	STACK_WIND(frame,
-		   truncate_flush,
-		   this,
-		   this->fops->ftruncate, /* crypt_ftruncate */
-		   fd,
-		   local->offset,
-		   NULL);
-	return 0;
-}
-
-/*
- * crypt_truncate() is implemented via crypt_ftruncate() as a
- * sequence crypt_open() - crypt_ftruncate() - truncate_flush()
- */
-int32_t crypt_truncate(call_frame_t *frame,
-		       xlator_t *this,
-		       loc_t *loc,
-		       off_t offset,
-		       dict_t *xdata)
-{
-	fd_t *fd;
-	crypt_local_t *local;
-
-#if DEBUG_CRYPT
-	gf_log(this->name, GF_LOG_DEBUG,
-	       "truncate file %s at offset %llu",
-	       loc->path, (unsigned long long)offset);
-#endif
-	local = crypt_alloc_local(frame, this, GF_FOP_TRUNCATE);
-	if (!local)
-		goto error;
-
-	fd = fd_create(loc->inode, frame->root->pid);
-	if (!fd) {
-		gf_log(this->name, GF_LOG_ERROR, "Can not create fd");
-		goto error;
-	}
-	local->fd = fd;
-	local->offset = offset;
-	local->xdata = xdata;
-	STACK_WIND(frame,
-		   truncate_begin,
-		   this,
-		   this->fops->open, /* crypt_open() */
-		   loc,
-		   O_RDWR,
-		   fd,
-		   NULL);
-	return 0;
- error:
-	STACK_UNWIND_STRICT(truncate, frame, -1, EINVAL, NULL, NULL, NULL);
-	return 0;
-}
-
-end_writeback_handler_t dispatch_end_writeback(glusterfs_fop_t fop)
-{
-	switch (fop) {
-	case GF_FOP_WRITE:
-		return end_writeback_writev;
-	case GF_FOP_FTRUNCATE:
-		return end_writeback_ftruncate;
-	default:
-		gf_log("crypt", GF_LOG_WARNING, "Bad wb operation %d", fop);
-		return NULL;
-	}
-}
-
-/*
- * true, if the caller needs metadata string
- */
-static int32_t is_custom_mtd(dict_t *xdata)
-{
-	data_t *data;
-	uint32_t flags;
-
-	if (!xdata)
-		return 0;
-
-	data = dict_get(xdata, MSGFLAGS_PREFIX);
-	if (!data)
-		return 0;
-	if (data->len != sizeof(uint32_t)) {
-		gf_log("crypt", GF_LOG_WARNING,
-		       "Bad msgflags size (%d)", data->len);
-		return -1;
-	}
-	flags = *((uint32_t *)data->data);
-	return msgflags_check_mtd_lock(&flags);
-}
-
-static int32_t crypt_open_done(call_frame_t *frame,
-			       void *cookie,
-			       xlator_t *this,
-			       int32_t op_ret,
-			       int32_t op_errno, dict_t *xdata)
-{
-	crypt_local_t *local = frame->local;
-
-	local->op_ret = op_ret;
-	local->op_errno = op_errno;
-	if (op_ret < 0)
-		gf_log(this->name, GF_LOG_WARNING, "mtd unlock failed (%d)",
-		       op_errno);
-	put_one_call_open(frame);
-	return 0;
-}
-
-static void crypt_open_tail(call_frame_t *frame, xlator_t *this)
-{
-	struct gf_flock  lock  = {0, };
-	crypt_local_t *local = frame->local;
-
-        lock.l_type   = F_UNLCK;
-        lock.l_whence = SEEK_SET;
-        lock.l_start  = 0;
-        lock.l_len    = 0;
-        lock.l_pid    = 0;
-
-	STACK_WIND(frame,
-		   crypt_open_done,
-		   FIRST_CHILD(this),
-		   FIRST_CHILD(this)->fops->finodelk,
-		   this->name,
-		   local->fd,
-		   F_SETLKW,
-		   &lock,
-		   NULL);
-}
-
-/*
- * load private inode info at open time
- * called as ->fgetxattr_cbk()
- */
-static int load_mtd_open(call_frame_t *frame,
-			 void *cookie,
-			 xlator_t *this,
-			 int32_t op_ret,
-			 int32_t op_errno,
-			 dict_t *dict,
-			 dict_t *xdata)
-{
-	int32_t ret;
-	gf_boolean_t upload_info;
-	data_t *mtd;
-	uint64_t value = 0;
-	struct crypt_inode_info *info;
-	crypt_local_t *local = frame->local;
-	crypt_private_t *priv = this->private;
-
-	local->op_ret = op_ret;
-	local->op_errno = op_errno;
-
-	if (local->fd->inode->ia_type == IA_IFLNK)
-		goto exit;
-	if (op_ret < 0)
-		goto exit;
-	/*
-	 * first, check for cached info
-	 */
-	ret = inode_ctx_get(local->fd->inode, this, &value);
-	if (ret != -1) {
-		info = (struct crypt_inode_info *)(long)value;
-		if (info == NULL) {
-			gf_log(this->name, GF_LOG_WARNING,
-			       "Inode info expected, but not found");
-			local->op_ret = -1;
-			local->op_errno = EIO;
-			goto exit;
-		}
-		/*
-		 * info has been found in the cache
-		 */
-		upload_info = _gf_false;
-	}
-	else {
-		/*
-		 * info hasn't been found in the cache.
-		 */
-		info = alloc_inode_info(local, local->loc);
-		if (!info) {
-			local->op_ret = -1;
-			local->op_errno = ENOMEM;
-			goto exit;
-		}
-		init_inode_info_head(info, local->fd);
-		upload_info = _gf_true;
-	}
-	/*
-	 * extract metadata
-	 */
-	mtd = dict_get(dict, CRYPTO_FORMAT_PREFIX);
-	if (!mtd) {
-		local->op_ret = -1;
-		local->op_errno = ENOENT;
-		gf_log (this->name, GF_LOG_WARNING,
-			"Format string wasn't found");
-		goto exit;
-	}
-	/*
-	 * authenticate metadata against the path
-	 */	
-	ret = open_format((unsigned char *)mtd->data,
-			  mtd->len,
-			  local->loc,
-			  info,
-			  get_master_cinfo(priv),
-			  local,
-			  upload_info);
-	if (ret) {
-		local->op_ret = -1;
-		local->op_errno = ret;
-		goto exit;
-	}
-	if (upload_info) {
-		ret = init_inode_info_tail(info, get_master_cinfo(priv));
-		if (ret) {
-			local->op_ret = -1;
-			local->op_errno = ret;
-			goto exit;	
-		}
-		ret = inode_ctx_put(local->fd->inode,
-				    this, (uint64_t)(long)info);
-		if (ret == -1) {
-			local->op_ret = -1;
-			local->op_errno = EIO;
-			goto exit;
-		}
-	}
-	if (local->custom_mtd) {
-		/*
-		 * pass the metadata string to the customer
-		 */
-		ret = dict_set_static_bin(local->xdata,
-					  CRYPTO_FORMAT_PREFIX,
-					  mtd->data,
-					  mtd->len);
-		if (ret) {
-			local->op_ret = -1;
-			local->op_errno = ret;
-			goto exit;
-		}
-	}
- exit:
-	if (!local->custom_mtd)
-		crypt_open_tail(frame, this);
-	else
-		put_one_call_open(frame);
-	return 0;
-}
-
-static int32_t crypt_open_finodelk_cbk(call_frame_t *frame,
-				       void *cookie,
-				       xlator_t *this,
-				       int32_t op_ret,
-				       int32_t op_errno,
-				       dict_t *xdata)
-{
-	crypt_local_t *local = frame->local;
-
-	local->op_ret = op_ret;
-	local->op_errno = op_errno;
-
-	if (op_ret < 0) {
-		gf_log(this->name, GF_LOG_WARNING, "finodelk (LOCK) failed");
-		goto exit;
-	}
-	STACK_WIND(frame,
-		   load_mtd_open,
-		   FIRST_CHILD(this),
-		   FIRST_CHILD(this)->fops->fgetxattr,
-		   local->fd,
-		   CRYPTO_FORMAT_PREFIX,
-		   NULL);
-	return 0;
- exit:
-	put_one_call_open(frame);
-	return 0;
-}
-
-/*
- * verify metadata against the specified pathname
- */
-static int32_t crypt_open_cbk(call_frame_t *frame,
-			      void *cookie,
-			      xlator_t *this,
-			      int32_t op_ret,
-			      int32_t op_errno,
-			      fd_t *fd,
-			      dict_t *xdata)
-{
-	struct gf_flock lock = {0, };
-	crypt_local_t *local = frame->local;
-
-	local->op_ret = op_ret;
-	local->op_errno = op_errno;
-
-	if (local->fd->inode->ia_type == IA_IFLNK)
-		goto exit;
-	if (op_ret < 0)
-		goto exit;
-	if (xdata)
-		local->xdata = dict_ref(xdata);
-	else if (local->custom_mtd){
-		local->xdata = dict_new();
-		if (!local->xdata) {
-			local->op_ret = -1;
-			local->op_errno = ENOMEM;
-			gf_log ("crypt", GF_LOG_ERROR,
-				"Can not get new dict for mtd string");
-			goto exit;
-		}
-	}
-	lock.l_len    = 0;
-        lock.l_start  = 0;
-        lock.l_type   = local->custom_mtd ? F_WRLCK : F_RDLCK;
-        lock.l_whence = SEEK_SET;
-
-	STACK_WIND(frame,
-		   crypt_open_finodelk_cbk,
-		   FIRST_CHILD(this),
-		   FIRST_CHILD(this)->fops->finodelk,
-		   this->name,
-		   fd,
-		   F_SETLKW,
-		   &lock,
-		   NULL);
-	return 0;
- exit:
-	put_one_call_open(frame);
-	return 0;
-}
-
-static int32_t crypt_open(call_frame_t *frame,
-			  xlator_t *this,
-			  loc_t *loc,
-			  int32_t flags,
-			  fd_t *fd,
-			  dict_t *xdata)
-{
-	int32_t ret = ENOMEM;
-	crypt_local_t *local;
-
-	local = crypt_alloc_local(frame, this, GF_FOP_OPEN);
-	if (!local)
-		goto error;
-	local->loc = GF_CALLOC(1, sizeof(*loc), gf_crypt_mt_loc);
-	if (!local->loc) {
-		ret = ENOMEM;
-		goto error;
-	}
-	memset(local->loc, 0, sizeof(*local->loc));
-	ret = loc_copy(local->loc, loc);
-	if (ret) {
-		GF_FREE(local->loc);
-		goto error;
-	}
-	local->fd = fd_ref(fd);
-
-	ret = is_custom_mtd(xdata);
-	if (ret < 0) {
-		loc_wipe(local->loc);
-		GF_FREE(local->loc);
-		ret = EINVAL;
-		goto error;
-	}
-	local->custom_mtd = ret;
-
-	if ((flags & O_ACCMODE) == O_WRONLY)
-		/*
-		 * we can't open O_WRONLY, because
-		 * we need to do read-modify-write
-		 */
-		flags = (flags & ~O_ACCMODE) | O_RDWR;
-	/*
-	 * Make sure that out translated offsets
-	 * and counts won't be ignored
-	 */
-	flags &= ~O_APPEND;
-	get_one_call_nolock(frame);
-	STACK_WIND(frame,
-		   crypt_open_cbk,
-		   FIRST_CHILD(this),
-		   FIRST_CHILD(this)->fops->open,
-		   loc,
-		   flags,
-		   fd,
-		   xdata);
-	return 0;
- error:
-	STACK_UNWIND_STRICT(open,
-			    frame,
-			    -1,
-			    ret,
-			    NULL,
-			    NULL);
-	return 0;
-}
-
-static int32_t init_inode_info_tail(struct crypt_inode_info *info,
-				    struct master_cipher_info *master)
-{
-	int32_t ret;
-	struct object_cipher_info *object = &info->cinfo;
-
-#if DEBUG_CRYPT
-	gf_log("crypt", GF_LOG_DEBUG, "Init inode info for object %s",
-	       uuid_utoa(info->oid));
-#endif
-	ret = data_cipher_algs[object->o_alg][object->o_mode].set_private(info,
-									master);
-	if (ret) {
-		gf_log("crypt", GF_LOG_ERROR, "Set private info failed");
-		return ret;
-	}
-	return 0;
-}
-
-/*
- * Init inode info at ->create() time
- */
-static void init_inode_info_create(struct crypt_inode_info *info,
-				   struct master_cipher_info *master,
-				   data_t *data)
-{
-	struct object_cipher_info *object;
-
-	info->nr_minor = CRYPT_XLATOR_ID;
-	memcpy(info->oid, data->data, data->len);
-
-	object = &info->cinfo;
-
-	object->o_alg        = master->m_alg;
-	object->o_mode       = master->m_mode;
-	object->o_block_bits = master->m_block_bits;
-	object->o_dkey_size  = master->m_dkey_size;
-}
-
-static void init_inode_info_head(struct crypt_inode_info *info, fd_t *fd)
-{
-	memcpy(info->oid, fd->inode->gfid, sizeof(uuid_t));
-}
-
-static int32_t crypt_create_done(call_frame_t *frame,
-				 void *cookie,
-				 xlator_t *this,
-				 int32_t op_ret,
-				 int32_t op_errno, dict_t *xdata)
-{
-	crypt_private_t *priv = this->private;
-	crypt_local_t *local = frame->local;
-	struct crypt_inode_info *info = local->info;
-	fd_t *local_fd = local->fd;
-	dict_t *local_xdata = local->xdata;
-	inode_t *local_inode = local->inode;
-
-	if (op_ret < 0) {
-		free_inode_info(info);
-		goto unwind;
-	}
-	op_errno = init_inode_info_tail(info, get_master_cinfo(priv));
-	if (op_errno) {
-		op_ret = -1;
-		free_inode_info(info);
-		goto unwind;
-	}
-	/*
-	 * FIXME: drop major subversion number
-	 */
-	op_ret = inode_ctx_put(local->fd->inode, this, (uint64_t)(long)info);
-	if (op_ret == -1) {
-		op_errno = EIO;
-		free_inode_info(info);
-		goto unwind;
-	}
- unwind:
-	free_format(local);
-	STACK_UNWIND_STRICT(create,
-			    frame,
-			    op_ret,
-			    op_errno,
-			    local_fd,
-			    local_inode,
-			    &local->buf,
-			    &local->prebuf,
-			    &local->postbuf,
-			    local_xdata);
-	fd_unref(local_fd);
-	inode_unref(local_inode);
-	if (local_xdata)
-		dict_unref(local_xdata);
-	return 0;
-}
-
-static int crypt_create_tail(call_frame_t *frame,
-			     void *cookie,
-			     xlator_t *this,
-			     int32_t op_ret,
-			     int32_t op_errno,
-			     dict_t *xdata)
-{
-	struct gf_flock  lock  = {0, };
-	crypt_local_t *local = frame->local;
-	fd_t *local_fd = local->fd;
-	dict_t *local_xdata = local->xdata;
-	inode_t *local_inode = local->inode;
-
-	dict_unref(local->xattr);
-
-	if (op_ret < 0)
-		goto error;
-
-        lock.l_type   = F_UNLCK;
-        lock.l_whence = SEEK_SET;
-        lock.l_start  = 0;
-        lock.l_len    = 0;
-        lock.l_pid    = 0;
-
-	STACK_WIND(frame,
-		   crypt_create_done,
-		   FIRST_CHILD(this),
-		   FIRST_CHILD(this)->fops->finodelk,
-		   this->name,
-		   local->fd,
-		   F_SETLKW,
-		   &lock,
-		   NULL);
-	return 0;
- error:
-	free_inode_info(local->info);
-	free_format(local);
-
-	STACK_UNWIND_STRICT(create,
-			    frame,
-			    op_ret,
-			    op_errno,
-			    local_fd,
-			    local_inode,
-			    &local->buf,
-			    &local->prebuf,
-			    &local->postbuf,
-			    local_xdata);
-
-	fd_unref(local_fd);
-	inode_unref(local_inode);
-	if (local_xdata)
-		dict_unref(local_xdata);
-	return 0;
-}
-
-static int32_t crypt_create_finodelk_cbk(call_frame_t *frame,
-					 void *cookie,
-					 xlator_t *this,
-					 int32_t op_ret,
-					 int32_t op_errno,
-					 dict_t *xdata)
-{
-	crypt_local_t *local = frame->local;
-	struct crypt_inode_info *info = local->info;
-
-	if (op_ret < 0)
-		goto error;
-
-	STACK_WIND(frame,
-		   crypt_create_tail,
-		   FIRST_CHILD(this),
-		   FIRST_CHILD(this)->fops->fsetxattr,
-		   local->fd,
-		   local->xattr, /* CRYPTO_FORMAT_PREFIX */
-		   0,
-		   NULL);
-	return 0;
- error: 
-	free_inode_info(info);
-	free_format(local);
-	fd_unref(local->fd);
-	dict_unref(local->xattr);
-	if (local->xdata)
-		dict_unref(local->xdata);
-
-	STACK_UNWIND_STRICT(create,
-			    frame,
-			    op_ret,
-			    op_errno,
-			    NULL,
-			    NULL,
-			    NULL,
-			    NULL,
-			    NULL,
-			    NULL);
-	return 0;
-}
-
-/*
- * Create and store crypt-specific format on disk;
- * Populate cache with private inode info
- */
-static int32_t crypt_create_cbk(call_frame_t *frame,
-				void *cookie,
-				xlator_t *this,
-				int32_t op_ret,
-				int32_t op_errno,
-				fd_t *fd,
-				inode_t *inode,
-				struct iatt *buf,
-				struct iatt *preparent,
-				struct iatt *postparent,
-				dict_t *xdata)
-{
-	struct gf_flock lock = {0, };
-	crypt_local_t *local = frame->local;
-	struct crypt_inode_info *info = local->info;
-
-	if (op_ret < 0)
-		goto error;
-	if (xdata)
-		local->xdata = dict_ref(xdata);	
-	local->inode = inode_ref(inode);
-	local->buf = *buf;
-	local->prebuf = *preparent;
-	local->postbuf = *postparent;
-
-        lock.l_len    = 0;
-        lock.l_start  = 0;
-        lock.l_type   = F_WRLCK;
-        lock.l_whence = SEEK_SET;
-
-        STACK_WIND(frame,
-		   crypt_create_finodelk_cbk,
-		   FIRST_CHILD(this),
-		   FIRST_CHILD(this)->fops->finodelk,
-		   this->name,
-		   local->fd,
-		   F_SETLKW,
-		   &lock,
-		   NULL);	
-	return 0;
- error:
-	free_inode_info(info);
-	free_format(local);
-	fd_unref(local->fd);
-	dict_unref(local->xattr);
-
-	STACK_UNWIND_STRICT(create,
-			    frame,
-			    op_ret,
-			    op_errno,
-			    NULL, NULL, NULL,
-			    NULL, NULL, NULL);
-	return 0;
-}
-
-static int32_t crypt_create(call_frame_t *frame,
-			    xlator_t *this,
-			    loc_t *loc,
-			    int32_t flags,
-			    mode_t mode,
-			    mode_t umask,
-			    fd_t *fd,
-			    dict_t *xdata)
-{
-	int ret;
-	data_t *data;
-	crypt_local_t *local;
-	crypt_private_t *priv;
-	struct master_cipher_info *master;
-	struct crypt_inode_info *info;
-
-	priv = this->private;
-	master = get_master_cinfo(priv);
-
-	if (master_alg_atomic(master)) {
-		/*
-		 * We can't open O_WRONLY, because we
-		 * need to do read-modify-write.
-		 */
-		if ((flags & O_ACCMODE) == O_WRONLY)
-			flags = (flags & ~O_ACCMODE) | O_RDWR;
-		/*
-		 * Make sure that out translated offsets
-		 * and counts won't be ignored
-		 */
-		flags &= ~O_APPEND;
-	}
-	local = crypt_alloc_local(frame, this, GF_FOP_CREATE);
-	if (!local) {
-		ret = ENOMEM;
-		goto error;
-	}
-	data = dict_get(xdata, "gfid-req");
-	if (!data) {
-		ret = EINVAL;
-		gf_log("crypt", GF_LOG_WARNING, "gfid not found");
-		goto error;
-	}
-	if (data->len != sizeof(uuid_t)) {
-		ret = EINVAL;
-		gf_log("crypt", GF_LOG_WARNING,
-		       "bad gfid size (%d), should be %d",
-		       (int)data->len, (int)sizeof(uuid_t));
-		goto error;
-	}
-	info = alloc_inode_info(local, loc);
-	if (!info){
-		ret = ENOMEM;
-		goto error;
-	}
-	/*
-	 * NOTE:
-	 * format has to be created BEFORE
-	 * proceeding to the untrusted server
-	 */
-	ret = alloc_format_create(local);
-	if (ret) {
-		free_inode_info(info);
-		goto error;
-	}
-	init_inode_info_create(info, master, data);
-
-	ret = create_format(local->format,
-			    loc,
-			    info,
-			    master);
-	if (ret) {
-		free_inode_info(info);
-		goto error;		
-	}
-	local->xattr = dict_new();
-	if (!local->xattr) {
-		free_inode_info(info);
-		free_format(local);	
-		goto error;
-	}
-	ret = dict_set_static_bin(local->xattr,
-				  CRYPTO_FORMAT_PREFIX,
-				  local->format,
-				  new_format_size());
-	if (ret) {
-		dict_unref(local->xattr);
-		free_inode_info(info);
-		free_format(local);
-		goto error;
-	}
-	ret = dict_set(local->xattr, FSIZE_XATTR_PREFIX, data_from_uint64(0));
-	if (ret) {
-		dict_unref(local->xattr);
-		free_inode_info(info);
-		free_format(local);
-		goto error;
-	}
-	local->fd = fd_ref(fd);
-
-	STACK_WIND(frame,
-		   crypt_create_cbk,
-		   FIRST_CHILD(this),
-		   FIRST_CHILD(this)->fops->create,
-		   loc,
-		   flags,
-		   mode,
-		   umask,
-		   fd,
-		   xdata);
-	return 0;
- error:
-	gf_log("crypt", GF_LOG_WARNING, "can not create file");
-	STACK_UNWIND_STRICT(create,
-			    frame,
-			    -1,
-			    ret,
-			    NULL, NULL, NULL,
-			    NULL, NULL, NULL);
-	return 0;
-}
-
-/*
- * FIXME: this should depends on the version of format string
- */
-static int32_t filter_crypt_xattr(dict_t *dict,
-				  char *key, data_t *value, void *data)
-{
-        dict_del(dict, key);
-        return 0;
-}
-
-static int32_t crypt_fsetxattr(call_frame_t *frame,
-			       xlator_t *this,
-			       fd_t *fd,
-			       dict_t *dict,
-			       int32_t flags, dict_t *xdata)
-{
-	dict_foreach_fnmatch(dict, "trusted.glusterfs.crypt*",
-			     filter_crypt_xattr, NULL);
-	STACK_WIND(frame,
-		   default_fsetxattr_cbk,
-		   FIRST_CHILD(this),
-		   FIRST_CHILD(this)->fops->fsetxattr,
-		   fd,
-		   dict,
-		   flags,
-		   xdata);
-	return 0;
-}
-
-/*
- * TBD: verify file metadata before wind
- */
-static int32_t crypt_setxattr(call_frame_t *frame,
-			      xlator_t *this,
-			      loc_t *loc,
-			      dict_t *dict,
-			      int32_t flags, dict_t *xdata)
-{
-	dict_foreach_fnmatch(dict, "trusted.glusterfs.crypt*",
-			     filter_crypt_xattr, NULL);
-	STACK_WIND(frame,
-		   default_setxattr_cbk,
-		   FIRST_CHILD(this),
-		   FIRST_CHILD(this)->fops->setxattr,
-		   loc,
-		   dict,
-		   flags,
-		   xdata);
-	return 0;
-}
-
-/*
- * called as flush_cbk()
- */
-static int32_t linkop_end(call_frame_t *frame,
-			  void *cookie,
-			  xlator_t *this,
-			  int32_t op_ret,
-			  int32_t op_errno,
-			  dict_t *xdata)
-{
-	crypt_local_t *local = frame->local;
-	linkop_unwind_handler_t unwind_fn;
-	unwind_fn = linkop_unwind_dispatch(local->fop);
-
-	local->op_ret = op_ret;
-	local->op_errno = op_errno;
-
-	if (op_ret < 0 &&
-	    op_errno == ENOENT &&
-	    local->loc->inode->ia_type == IA_IFLNK) {
-		local->op_ret = 0;
-		local->op_errno = 0;
-	}
-	unwind_fn(frame);
-	return 0;
-}
-
-/*
- * unpin inode on the server
- */
-static int32_t link_flush(call_frame_t *frame,
-			  void *cookie,
-			  xlator_t *this,
-			  int32_t op_ret,
-			  int32_t op_errno,
-			  inode_t *inode,
-			  struct iatt *buf,
-			  struct iatt *preparent,
-			  struct iatt *postparent, dict_t *xdata)
-{
-	crypt_local_t *local = frame->local;
-
-	if (op_ret < 0)
-		goto error;
-	if (local->xdata) {
-		dict_unref(local->xdata);
-		local->xdata = NULL;
-	}
-	if (xdata)
-		local->xdata = dict_ref(xdata);
-	local->inode = inode_ref(inode);
-	local->buf = *buf;
-	local->prebuf = *preparent;
-	local->postbuf = *postparent;
-
-	STACK_WIND(frame,
-		   linkop_end,
-		   FIRST_CHILD(this),
-		   FIRST_CHILD(this)->fops->flush,
-		   local->fd,
-		   NULL);
-	return 0;
- error:
-	local->op_ret = -1;
-	local->op_errno = op_errno;
-	link_unwind(frame);
-	return 0;
-}
-
-void link_unwind(call_frame_t *frame)
-{
-	crypt_local_t *local = frame->local;
-	dict_t *xdata;
-	dict_t *xattr;
-	inode_t *inode;
-
-	if (!local) {
-		STACK_UNWIND_STRICT(link,
-				    frame,
-				    -1,
-				    ENOMEM,
-				    NULL,
-				    NULL,
-				    NULL,
-				    NULL,
-				    NULL);
-		return;
-	}
-	xdata = local->xdata;
-	xattr = local->xattr;
-	inode = local->inode;
-
-	if (local->loc){
-		loc_wipe(local->loc);
-		GF_FREE(local->loc);
-	}
-	if (local->newloc) {
-		loc_wipe(local->newloc);
-		GF_FREE(local->newloc);
-	}
-	if (local->fd)
-		fd_unref(local->fd);
-	if (local->format)
-		GF_FREE(local->format);
-
-	STACK_UNWIND_STRICT(link,
-			    frame,
-			    local->op_ret,
-			    local->op_errno,
-			    inode,
-			    &local->buf,
-			    &local->prebuf,
-			    &local->postbuf,
-			    xdata);
-	if (xdata)
-		dict_unref(xdata);
-	if (xattr)
-		dict_unref(xattr);
-	if (inode)
-		inode_unref(inode);
-}
-
-void link_wind(call_frame_t *frame, xlator_t *this)
-{
-	crypt_local_t *local = frame->local;
-
-	STACK_WIND(frame,
-		   link_flush,
-		   FIRST_CHILD(this),
-		   FIRST_CHILD(this)->fops->link,
-		   local->loc,
-		   local->newloc,
-		   local->xdata);
-}
-
-/*
- * unlink()
- */
-static int32_t unlink_flush(call_frame_t *frame,
-			    void *cookie,
-			    xlator_t *this,
-			    int32_t op_ret,
-			    int32_t op_errno,
-			    struct iatt *preparent,
-			    struct iatt *postparent, dict_t *xdata)
-{
-	crypt_local_t *local = frame->local;
-
-	if (op_ret < 0)
-		goto error;
-	local->prebuf = *preparent;
-	local->postbuf = *postparent;
-	if (local->xdata) {
-		dict_unref(local->xdata);
-		local->xdata = NULL;
-	}
-	if (xdata)
-		local->xdata = dict_ref(xdata);
-
-	STACK_WIND(frame,
-		   linkop_end,
-		   FIRST_CHILD(this),
-		   FIRST_CHILD(this)->fops->flush,
-		   local->fd,
-		   NULL);
-	return 0;
- error:
-	local->op_ret = -1;
-	local->op_errno = op_errno;
-	unlink_unwind(frame);
-	return 0;
-}
-
-void unlink_unwind(call_frame_t *frame)
-{
-	crypt_local_t *local = frame->local;
-	dict_t *xdata;
-	dict_t *xattr;
-
-	if (!local) {
-		STACK_UNWIND_STRICT(unlink,
-				    frame,
-				    -1,
-				    ENOMEM,
-				    NULL,
-				    NULL,
-				    NULL);
-		return;
-	}
-	xdata = local->xdata;
-	xattr = local->xattr;
-	if (local->loc){
-		loc_wipe(local->loc);
-		GF_FREE(local->loc);
-	}
-	if (local->fd)
-		fd_unref(local->fd);
-	if (local->format)
-		GF_FREE(local->format);
-
-	STACK_UNWIND_STRICT(unlink,
-			    frame,
-			    local->op_ret,
-			    local->op_errno,
-			    &local->prebuf,
-			    &local->postbuf,
-			    xdata);
-	if (xdata)
-		dict_unref(xdata);
-	if (xattr)
-		dict_unref(xattr);
-}
-
-void unlink_wind(call_frame_t *frame, xlator_t *this)
-{
-	crypt_local_t *local = frame->local;
-
-	STACK_WIND(frame,
-		   unlink_flush,
-		   FIRST_CHILD(this),
-		   FIRST_CHILD(this)->fops->unlink,
-		   local->loc,
-		   local->flags,
-		   local->xdata);
-}
-
-void rename_unwind(call_frame_t *frame)
-{
-	crypt_local_t *local = frame->local;
-	dict_t *xdata;
-	dict_t *xattr;
-	struct iatt *prenewparent;
-	struct iatt *postnewparent;
-
-	if (!local) {
-		STACK_UNWIND_STRICT(rename,
-				    frame,
-				    -1,
-				    ENOMEM,
-				    NULL,
-				    NULL,
-				    NULL,
-				    NULL,
-				    NULL,
-				    NULL);	
-		return;
-	}
-	xdata = local->xdata;
-	xattr = local->xattr;
-	prenewparent = local->prenewparent;
-	postnewparent = local->postnewparent;	
-
-	if (local->loc){
-		loc_wipe(local->loc);
-		GF_FREE(local->loc);
-	}
-	if (local->newloc){
-		loc_wipe(local->newloc);
-		GF_FREE(local->newloc);
-	}
-	if (local->fd)
-		fd_unref(local->fd);
-	if (local->format)
-		GF_FREE(local->format);
-
-	STACK_UNWIND_STRICT(rename,
-			    frame,
-			    local->op_ret,
-			    local->op_errno,
-			    &local->buf,
-			    &local->prebuf,
-			    &local->postbuf,
-			    prenewparent,
-			    postnewparent,
-			    xdata);
-	if (xdata)
-		dict_unref(xdata);
-	if (xattr)
-		dict_unref(xattr);
-	if (prenewparent)
-		GF_FREE(prenewparent);
-	if (postnewparent)
-		GF_FREE(postnewparent);
-}
-
-/*
- * called as flush_cbk()
- */
-static int32_t rename_end(call_frame_t *frame,
-			  void *cookie,
-			  xlator_t *this,
-			  int32_t op_ret,
-			  int32_t op_errno,
-			  dict_t *xdata)
-{
-	crypt_local_t *local = frame->local;
-
-	local->op_ret = op_ret;
-	local->op_errno = op_errno;
-
-	rename_unwind(frame);
-	return 0;
-}
-
-static int32_t rename_flush(call_frame_t *frame,
-			    void *cookie,
-			    xlator_t *this,
-			    int32_t op_ret,
-			    int32_t op_errno,
-			    struct iatt *buf,
-			    struct iatt *preoldparent,
-			    struct iatt *postoldparent,
-			    struct iatt *prenewparent,
-			    struct iatt *postnewparent,
-			    dict_t *xdata)
-{
-	crypt_local_t *local = frame->local;
-
-	if (op_ret < 0)
-		goto error;
-	dict_unref(local->xdata);
-	local->xdata = NULL;
-	if (xdata)
-		local->xdata = dict_ref(xdata);
-
-	local->buf = *buf;
-	local->prebuf = *preoldparent;
-	local->postbuf = *postoldparent;
-	if (prenewparent) {
-		local->prenewparent = GF_CALLOC(1, sizeof(*prenewparent),
-						gf_crypt_mt_iatt);
-		if (!local->prenewparent) {
-			op_errno = ENOMEM;
-			goto error;
-		}
-		*local->prenewparent = *prenewparent;
-	}
-	if (postnewparent) {
-		local->postnewparent = GF_CALLOC(1, sizeof(*postnewparent),
-						 gf_crypt_mt_iatt);
-		if (!local->postnewparent) {
-			op_errno = ENOMEM;
-			goto error;
-		}
-		*local->postnewparent = *postnewparent;
-	}
-	STACK_WIND(frame,
-		   rename_end,
-		   FIRST_CHILD(this),
-		   FIRST_CHILD(this)->fops->flush,
-		   local->fd,
-		   NULL);
-	return 0;
- error:
-	local->op_ret = -1;
-	local->op_errno = op_errno;
-	rename_unwind(frame);
-	return 0;
-}
-
-void rename_wind(call_frame_t *frame, xlator_t *this)
-{
-	crypt_local_t *local = frame->local;
-
-	STACK_WIND(frame,
-		   rename_flush,
-		   FIRST_CHILD(this),
-		   FIRST_CHILD(this)->fops->rename,
-		   local->loc,
-		   local->newloc,
-		   local->xdata);
-}
-
-static int32_t __do_linkop(call_frame_t *frame,
-			   void *cookie,
-			   xlator_t *this,
-			   int32_t op_ret,
-			   int32_t op_errno, dict_t *xdata)
-{
-	crypt_local_t *local = frame->local;
-	linkop_wind_handler_t wind_fn;
-	linkop_unwind_handler_t unwind_fn;
-
-	wind_fn = linkop_wind_dispatch(local->fop);
-	unwind_fn = linkop_unwind_dispatch(local->fop);
-
-	local->op_ret = op_ret;
-	local->op_errno = op_errno;
-
-	if (op_ret >= 0)
-		wind_fn(frame, this);
-	else {
-		gf_log(this->name, GF_LOG_WARNING, "mtd unlock failed (%d)",
-		       op_errno);
-		unwind_fn(frame);
-	}
-	return 0;
-}
-
-static int32_t do_linkop(call_frame_t *frame,
-			 void *cookie,
-			 xlator_t *this,
-			 int32_t op_ret,
-			 int32_t op_errno,
-			 dict_t *xdata)
-{
-	struct gf_flock  lock  = {0, };
-	crypt_local_t *local = frame->local;
-	linkop_unwind_handler_t unwind_fn;
-
-	unwind_fn = linkop_unwind_dispatch(local->fop);
-	local->op_ret = op_ret;
-	local->op_errno = op_errno;
-
-	if(op_ret < 0)
-		goto error;
-
-	lock.l_type   = F_UNLCK;
-        lock.l_whence = SEEK_SET;
-        lock.l_start  = 0;
-        lock.l_len    = 0;
-        lock.l_pid    = 0;
-
-	STACK_WIND(frame,
-		   __do_linkop,
-		   FIRST_CHILD(this),
-		   FIRST_CHILD(this)->fops->finodelk,
-		   this->name,
-		   local->fd,
-		   F_SETLKW,
-		   &lock,
-		   NULL);
-	return 0;
- error:		
-	unwind_fn(frame);
-	return 0;
-}
-
-/*
- * Update the metadata string (against the new pathname);
- * submit the result
- */
-static int32_t linkop_begin(call_frame_t *frame,
-			    void *cookie,
-			    xlator_t *this,
-			    int32_t op_ret,
-			    int32_t op_errno,
-			    fd_t *fd,
-			    dict_t *xdata)
-{
-	gf_boolean_t upload_info;
-	crypt_local_t *local = frame->local;
-	crypt_private_t *priv = this->private;
-	struct crypt_inode_info *info;
-	data_t *old_mtd;
-	uint32_t new_mtd_size;
-	uint64_t value = 0;
-	void (*unwind_fn)(call_frame_t *frame);
-	mtd_op_t mop;
-
-	unwind_fn = linkop_unwind_dispatch(local->fop);
-	mop = linkop_mtdop_dispatch(local->fop);
-
-	if (op_ret < 0)
-		/*
-		 * verification failed
-		 */
-		goto error;
-
-	old_mtd = dict_get(xdata, CRYPTO_FORMAT_PREFIX);
-	if (!old_mtd) {
-		op_errno = EIO;
-		gf_log (this->name, GF_LOG_DEBUG,
-			"Metadata string wasn't found");
-		goto error;
-	}
-	new_mtd_size = format_size(mop, old_mtd->len);
-	op_errno = alloc_format(local, new_mtd_size);
-	if (op_errno)
-		goto error;
-	/*
-	 * check for cached info 
-	 */
-	op_ret = inode_ctx_get(fd->inode, this, &value);
-	if (op_ret != -1) {
-		info = (struct crypt_inode_info *)(long)value;
-		if (info == NULL) {
-			gf_log (this->name, GF_LOG_WARNING,
-				"Inode info was not found");
-			op_errno = EINVAL;
-			goto error;
-		}
-		/*
-		 * info was found in the cache
-		 */
-		local->info = info;
-		upload_info = _gf_false;
-	}
-	else {
-		/*
-		 * info wasn't found in the cache;
-		 */
-		info = alloc_inode_info(local, local->loc);
-		if (!info)
-			goto error;
-		init_inode_info_head(info, fd);
-		local->info = info;
-		upload_info = _gf_true;
-	}
-	op_errno = open_format((unsigned char *)old_mtd->data,
-			       old_mtd->len,
-			       local->loc,
-			       info,
-			       get_master_cinfo(priv),
-			       local,
-			       upload_info);
-	if (op_errno)
-		goto error;
-	if (upload_info == _gf_true) {
-		op_errno = init_inode_info_tail(info,
-						get_master_cinfo(priv));
-		if (op_errno)
-			goto error;
-		op_errno = inode_ctx_put(fd->inode, this,
-					 (uint64_t)(long)(info));
-		if (op_errno == -1) {
-			op_errno = EIO;
-			goto error;
-		}
-	}
-	/*
-	 * update the format string (append/update/cup a MAC)
-	 */
-	op_errno = update_format(local->format,
-				 (unsigned char *)old_mtd->data,
-				 old_mtd->len,
-				 local->mac_idx,
-				 mop,
-				 local->newloc,
-				 info,
-				 get_master_cinfo(priv),
-				 local);
-	if (op_errno)
-		goto error;
-	/*
-	 * store the new format string on the server
-	 */
-	if (new_mtd_size) {
-		op_errno = dict_set_static_bin(local->xattr,
-					       CRYPTO_FORMAT_PREFIX,
-					       local->format,
-					       new_mtd_size);
-		if (op_errno)
-			goto error;
-	}
-	STACK_WIND(frame,
-		   do_linkop,
-		   FIRST_CHILD(this),
-		   FIRST_CHILD(this)->fops->setxattr,
-		   local->loc,
-		   local->xattr,
-		   0,
-		   NULL);
-	return 0;
- error:
-	local->op_ret = -1;
-	local->op_errno = op_errno;
-	unwind_fn(frame);
-	return 0;
-}
-
-static int32_t linkop_grab_local(call_frame_t *frame,
-				 xlator_t *this,
-				 loc_t *oldloc,
-				 loc_t *newloc,
-				 int flags, dict_t *xdata,
-				 glusterfs_fop_t op)
-{
-	int32_t ret = ENOMEM;
-	fd_t *fd;
-	crypt_local_t *local;
-
-	local = crypt_alloc_local(frame, this, op);
-	if (!local)
-		goto error;
-	if (xdata)
-		local->xdata = dict_ref(xdata);
-
-	fd = fd_create(oldloc->inode, frame->root->pid);
-	if (!fd) {
-		gf_log(this->name, GF_LOG_ERROR, "Can not create fd");
-		goto error;
-	}
-	local->fd = fd;
-	local->flags = flags;
-	local->loc = GF_CALLOC(1, sizeof(*oldloc), gf_crypt_mt_loc);
-	if (!local->loc)
-		goto error;
-	memset(local->loc, 0, sizeof(*local->loc));
-	ret = loc_copy(local->loc, oldloc);
-	if (ret) {
-		GF_FREE(local->loc);
-		local->loc = NULL;
-		goto error;
-	}
-	if (newloc) {
-		local->newloc = GF_CALLOC(1, sizeof(*newloc), gf_crypt_mt_loc);
-		if (!local->newloc) {
-			loc_wipe(local->loc);
-			GF_FREE(local->loc);
-			goto error;
-		}
-		memset(local->newloc, 0, sizeof(*local->newloc));
-		ret = loc_copy(local->newloc, newloc);
-		if (ret) {
-			loc_wipe(local->loc);
-			GF_FREE(local->loc);
-			GF_FREE(local->newloc);
-			goto error;
-		}
-	}
-	local->xattr = dict_new();
-	if (!local->xattr) {
-		gf_log(this->name, GF_LOG_ERROR, "Can not create dict");
-		ret = ENOMEM;
-		goto error;
-	}
-	return 0;
-
-error:
-        if (local) {
-                if (local->xdata)
-                        dict_unref(local->xdata);
-                if (local->fd)
-                        fd_unref(local->fd);
-                local->fd = 0;
-                local->loc = NULL;
-                local->newloc = NULL;
-                local->op_ret = -1;
-                local->op_errno = ret;
-        }
-
-        return ret;
-}
-
-/*
- * read and verify locked metadata against the old pathname (via open);
- * update the metadata string in accordance with the new pathname;
- * submit modified metadata;
- * wind;
- */
-static int32_t linkop(call_frame_t *frame,
-		      xlator_t *this,
-		      loc_t *oldloc,
-		      loc_t *newloc,
-		      int flags,
-		      dict_t *xdata,
-		      glusterfs_fop_t op)
-{
-	int32_t ret;
-	dict_t *dict;
-	crypt_local_t *local;
-	void (*unwind_fn)(call_frame_t *frame);
-        void (*wind_fn)(call_frame_t *frame, xlator_t *this);
-
-        wind_fn = linkop_wind_dispatch(op);
-	unwind_fn = linkop_unwind_dispatch(op);
-
-	ret = linkop_grab_local(frame, this, oldloc, newloc, flags, xdata, op);
-	local = frame->local;
-	if (ret)
-		goto error;
-
-        if (local->fd->inode->ia_type == IA_IFLNK)
-                goto wind;
-
-	dict = dict_new();
-	if (!dict) {
-		gf_log(this->name, GF_LOG_ERROR, "Can not create dict");
-		ret = ENOMEM;
-		goto error;
-	}
-	/*
-	 * Set a message to crypt_open() that we need
-	 * locked metadata string.
-	 * All link operations (link, unlink, rename)
-	 * need write lock
-	 */
-	msgflags_set_mtd_wlock(&local->msgflags);
-	ret = dict_set_static_bin(dict,
-				  MSGFLAGS_PREFIX,
-				  &local->msgflags,
-				  sizeof(local->msgflags));
-	if (ret) {
-		gf_log(this->name, GF_LOG_ERROR, "Can not set dict");
-		dict_unref(dict);
-		goto error;
-	}
-	/*
-	 * verify metadata against the old pathname
-	 * and retrieve locked metadata string
-	 */
-	STACK_WIND(frame,
-		   linkop_begin,
-		   this,
-		   this->fops->open, /* crypt_open() */
-		   oldloc,
-		   O_RDWR,
-		   local->fd,
-		   dict);
-	dict_unref(dict);
-	return 0;
-
-wind:
-        wind_fn(frame, this);
-        return 0;
-
-error:
-	local->op_ret = -1;
-	local->op_errno = ret;
-	unwind_fn(frame);
-	return 0;
-}
-
-static int32_t crypt_link(call_frame_t *frame, xlator_t *this,
-			  loc_t *oldloc, loc_t *newloc, dict_t *xdata)
-{
-	return linkop(frame, this, oldloc, newloc, 0, xdata, GF_FOP_LINK);
-}
-
-static int32_t crypt_unlink(call_frame_t *frame, xlator_t *this,
-			    loc_t *loc, int flags, dict_t *xdata)
-{
-	return linkop(frame, this, loc, NULL, flags, xdata, GF_FOP_UNLINK);
-}
-
-static int32_t crypt_rename(call_frame_t *frame, xlator_t *this,
-			    loc_t *oldloc, loc_t *newloc, dict_t *xdata)
-{
-	return linkop(frame, this, oldloc, newloc, 0, xdata, GF_FOP_RENAME);
-}
-
-static void put_one_call_open(call_frame_t *frame)
-{
-	crypt_local_t *local = frame->local;
-	if (put_one_call(local)) {
-		fd_t *fd = local->fd;
-		loc_t *loc = local->loc;
-		dict_t *xdata = local->xdata;
-
-		STACK_UNWIND_STRICT(open,
-				    frame,
-				    local->op_ret,
-				    local->op_errno,
-				    fd,
-				    xdata);
-		fd_unref(fd);
-		if (xdata)
-			dict_unref(xdata);
-		loc_wipe(loc);
-		GF_FREE(loc);
-	}
-}
-
-static int32_t __crypt_readv_done(call_frame_t *frame,
-				  void *cookie,
-				  xlator_t *this,
-				  int32_t op_ret,
-				  int32_t op_errno, dict_t *xdata)
-{
-	crypt_local_t *local = frame->local;
-	fd_t *local_fd = local->fd;
-	dict_t *local_xdata = local->xdata;
-	/* read deals with data configs only */
-	struct iovec *avec = local->data_conf.avec;
-	char **pool = local->data_conf.pool;
-	int blocks_in_pool = local->data_conf.blocks_in_pool;
-	struct iobref *iobref = local->iobref;
-	struct iobref *iobref_data = local->iobref_data;
-
-	if (op_ret < 0) {
-		gf_log(this->name, GF_LOG_WARNING,
-		       "readv unlock failed (%d)", op_errno);
-		if (local->op_ret >= 0) {
-			local->op_ret = op_ret;
-			local->op_errno = op_errno;
-		}
-	}
-	dump_plain_text(local, avec);
-
-	gf_log("crypt", GF_LOG_DEBUG,
-	       "readv: ret_to_user: %d, iovec len: %d, ia_size: %llu",
-	       (int)(local->rw_count > 0 ? local->rw_count : local->op_ret),
-	       (int)(local->rw_count > 0 ? iovec_get_size(avec, local->data_conf.acount) : 0),
-	       (unsigned long long)local->buf.ia_size);
-
-	STACK_UNWIND_STRICT(readv,
-			    frame,
-			    local->rw_count > 0 ? local->rw_count : local->op_ret,
-			    local->op_errno,
-			    avec,
-			    avec ? local->data_conf.acount : 0,
-			    &local->buf,
-			    local->iobref,
-			    local_xdata);
-
-	free_avec(avec, pool, blocks_in_pool);
-	fd_unref(local_fd);
-	if (local_xdata)
-		dict_unref(local_xdata);
-	if (iobref)
-		iobref_unref(iobref);
-	if (iobref_data)
-		iobref_unref(iobref_data);
-	return 0;
-}
-
-static void crypt_readv_done(call_frame_t *frame, xlator_t *this)
-{
-	if (parent_is_crypt_xlator(frame, this))
-		/*
-		 *  don't unlock (it will be done by the parent)
-		 */
-		__crypt_readv_done(frame, NULL, this, 0, 0, NULL);
-	else {
-		crypt_local_t *local = frame->local;
-		struct gf_flock  lock  = {0, };
-
-		lock.l_type   = F_UNLCK;
-		lock.l_whence = SEEK_SET;
-		lock.l_start  = 0;
-		lock.l_len    = 0;
-		lock.l_pid    = 0;
-
-		STACK_WIND(frame,
-			   __crypt_readv_done,
-			   FIRST_CHILD(this),
-			   FIRST_CHILD(this)->fops->finodelk,
-			   this->name,
-			   local->fd,
-			   F_SETLKW,
-			   &lock,
-			   NULL);
-	}
-}
-
-static void put_one_call_readv(call_frame_t *frame, xlator_t *this)
-{
-	crypt_local_t *local = frame->local;
-	if (put_one_call(local))
-		crypt_readv_done(frame, this);
-}
-
-static int32_t __crypt_writev_done(call_frame_t *frame,
-				   void *cookie,
-				   xlator_t *this,
-				   int32_t op_ret,
-				   int32_t op_errno, dict_t *xdata)
-{
-	crypt_local_t *local = frame->local;
-	fd_t *local_fd = local->fd;
-	dict_t *local_xdata = local->xdata;
-	int32_t ret_to_user;
-
-	if (local->xattr)
-		dict_unref(local->xattr);
-	/*
-	 * Calculate amout of butes to be returned
-	 * to user. We need to subtract paddings that
-	 * have been written as a part of atom.
-	 */
-	/*
-	 * subtract head padding
-	 */
-	if (local->rw_count == 0)
-		/*
-		 * Nothing has been written, it must be an error
-		 */
-		ret_to_user = local->op_ret;
-	else if (local->rw_count <= local->data_conf.off_in_head) {
-		gf_log("crypt", GF_LOG_WARNING, "Incomplete write");
-		ret_to_user = 0;
-	}
-	else
-		ret_to_user = local->rw_count -
-			local->data_conf.off_in_head;
-	/*
-	 * subtract tail padding
-	 */
-	if (ret_to_user > local->data_conf.orig_size)
-		ret_to_user = local->data_conf.orig_size;
-
-	if (local->iobref)
-		iobref_unref(local->iobref);
-	if (local->iobref_data)
-		iobref_unref(local->iobref_data);
-	free_avec_data(local);
-	free_avec_hole(local);
-
-	gf_log("crypt", GF_LOG_DEBUG,
-	       "writev: ret_to_user: %d", ret_to_user);
-
-	STACK_UNWIND_STRICT(writev,
-			    frame,
-			    ret_to_user,
-			    local->op_errno,
-			    &local->prebuf,
-			    &local->postbuf,
-			    local_xdata);
-	fd_unref(local_fd);
-	if (local_xdata)
-		dict_unref(local_xdata);
-	return 0;
-}
-
-static int32_t crypt_writev_done(call_frame_t *frame,
-				 void *cookie,
-				 xlator_t *this,
-				 int32_t op_ret,
-				 int32_t op_errno,
-				 dict_t *xdata)
-{
-	crypt_local_t *local = frame->local;
-
-	if (op_ret < 0)
-		gf_log("crypt", GF_LOG_WARNING, "can not update file size");
-
-	if (parent_is_crypt_xlator(frame, this))
-		/*
-		 * don't unlock (it will be done by the parent)
-		 */
-		__crypt_writev_done(frame, NULL, this, 0, 0, NULL);
-	else {
-		struct gf_flock  lock  = {0, };
-
-		lock.l_type   = F_UNLCK;
-		lock.l_whence = SEEK_SET;
-		lock.l_start  = 0;
-		lock.l_len    = 0;
-		lock.l_pid    = 0;
-
-		STACK_WIND(frame,
-			   __crypt_writev_done,
-			   FIRST_CHILD(this),
-			   FIRST_CHILD(this)->fops->finodelk,
-			   this->name,
-			   local->fd,
-			   F_SETLKW,
-			   &lock,
-			   NULL);
-	}
-	return 0;
-}
-
-static void put_one_call_writev(call_frame_t *frame, xlator_t *this)
-{
-	crypt_local_t *local = frame->local;
-	if (put_one_call(local)) {
-		if (local->update_disk_file_size) {
-			int32_t ret;
-			/*
-			 * update file size, unlock the file and unwind
-			 */
-			ret = dict_set(local->xattr,
-				       FSIZE_XATTR_PREFIX,
-				       data_from_uint64(local->cur_file_size));
-			if (ret) {
-				gf_log("crypt", GF_LOG_WARNING,
-				       "can not set key to update file size");
-				crypt_writev_done(frame, NULL,
-						  this, 0, 0, NULL);
-				return;
-			}
-			gf_log("crypt", GF_LOG_DEBUG,
-			       "Updating disk file size to %llu",
-			       (unsigned long long)local->cur_file_size);
-			STACK_WIND(frame,
-				   crypt_writev_done,
-				   FIRST_CHILD(this),
-				   FIRST_CHILD(this)->fops->fsetxattr,
-				   local->fd,
-				   local->xattr, /* CRYPTO_FORMAT_PREFIX */
-				   0,
-				   NULL);
-		}
-		else
-			crypt_writev_done(frame, NULL, this, 0, 0, NULL);
-	}
-}
-
-static int32_t __crypt_ftruncate_done(call_frame_t *frame,
-				      void *cookie,
-				      xlator_t *this,
-				      int32_t op_ret,
-				      int32_t op_errno, dict_t *xdata)
-{
-	crypt_local_t *local = frame->local;
-	fd_t *local_fd = local->fd;
-	dict_t *local_xdata = local->xdata;
-	char *iobase = local->vec.iov_base;
-
-	if (op_ret < 0) {
-		gf_log(this->name, GF_LOG_WARNING,
-		       "ftruncate unlock failed (%d)", op_errno);
-		if (local->op_ret >= 0) {
-			local->op_ret = op_ret;
-			local->op_errno = op_errno;
-		}
-	}
-	if (local->iobref_data)
-		iobref_unref(local->iobref_data);
-	free_avec_data(local);
-	free_avec_hole(local);
-
-	gf_log("crypt", GF_LOG_DEBUG,
-	       "ftruncate, return to user: presize=%llu, postsize=%llu",
-	       (unsigned long long)local->prebuf.ia_size,
-	       (unsigned long long)local->postbuf.ia_size);
-
-	STACK_UNWIND_STRICT(ftruncate,
-			    frame,
-			    local->op_ret < 0 ? -1 : 0,
-			    local->op_errno,
-			    &local->prebuf,
-			    &local->postbuf,
-			    local_xdata);
-	fd_unref(local_fd);
-	if (local_xdata)
-		dict_unref(local_xdata);
-	if (iobase)
-		GF_FREE(iobase);
-	return 0;
-}
-
-static int32_t crypt_ftruncate_done(call_frame_t *frame,
-				    void *cookie,
-				    xlator_t *this,
-				    int32_t op_ret,
-				    int32_t op_errno,
-				    dict_t *xdata)
-{
-	crypt_local_t *local = frame->local;
-	struct gf_flock  lock  = {0, };
-
-	dict_unref(local->xattr);
-	if (op_ret < 0)
-		gf_log("crypt", GF_LOG_WARNING, "can not update file size");
-
-	lock.l_type   = F_UNLCK;
-	lock.l_whence = SEEK_SET;
-	lock.l_start  = 0;
-	lock.l_len    = 0;
-	lock.l_pid    = 0;
-
-	STACK_WIND(frame,
-		   __crypt_ftruncate_done,
-		   FIRST_CHILD(this),
-		   FIRST_CHILD(this)->fops->finodelk,
-		   this->name,
-		   local->fd,
-		   F_SETLKW,
-		   &lock,
-		   NULL);
-	return 0;
-}
-
-static void put_one_call_ftruncate(call_frame_t *frame, xlator_t *this)
-{
-	crypt_local_t *local = frame->local;
-	if (put_one_call(local)) {
-		if (local->update_disk_file_size) {
-			int32_t ret;
-			/*
-			 * update file size, unlock the file and unwind
-			 */
-			ret = dict_set(local->xattr,
-				       FSIZE_XATTR_PREFIX,
-				       data_from_uint64(local->cur_file_size));
-			if (ret) {
-				gf_log("crypt", GF_LOG_WARNING,
-				       "can not set key to update file size");
-				crypt_ftruncate_done(frame, NULL,
-						     this, 0, 0, NULL);
-				return;
-			}
-			gf_log("crypt", GF_LOG_DEBUG,
-			       "Updating disk file size to %llu",
-			       (unsigned long long)local->cur_file_size);
-			STACK_WIND(frame,
-				   crypt_ftruncate_done,
-				   FIRST_CHILD(this),
-				   FIRST_CHILD(this)->fops->fsetxattr,
-				   local->fd,
-				   local->xattr, /* CRYPTO_FORMAT_PREFIX */
-				   0,
-				   NULL);
-		}
-		else
-			crypt_ftruncate_done(frame, NULL, this, 0, 0, NULL);
-	}
-}
-
-/*
- * load regular file size for some FOPs
- */
-static int32_t load_file_size(call_frame_t *frame,
-			      void *cookie,
-			      xlator_t *this,
-			      int32_t op_ret,
-			      int32_t op_errno,
-			      dict_t *dict,
-			      dict_t *xdata)
-{
-	data_t *data;
-	crypt_local_t *local = frame->local;
-
-	dict_t *local_xdata = local->xdata;
-	inode_t *local_inode = local->inode;
-
-	if (op_ret < 0)
-		goto unwind;
-	/*
-	 * load regular file size
-	 */
-	data = dict_get(dict, FSIZE_XATTR_PREFIX);
-	if (!data) {
-		if (local->xdata)
-			dict_unref(local->xdata);
-		gf_log("crypt", GF_LOG_WARNING, "Regular file size not found");
-		op_ret = -1;
-		op_errno = EIO;
-		goto unwind;
-	}
-	local->buf.ia_size = data_to_uint64(data);
-
-	gf_log(this->name, GF_LOG_DEBUG,
-	       "FOP %d: Translate regular file to %llu",
-	       local->fop,
-	       (unsigned long long)local->buf.ia_size);
- unwind:	
-	if (local->fd)
-		fd_unref(local->fd);
-	if (local->loc) {
-		loc_wipe(local->loc);
-		GF_FREE(local->loc);
-	}
-	switch (local->fop) {
-	case GF_FOP_FSTAT:
-		STACK_UNWIND_STRICT(fstat,
-				    frame,
-				    op_ret,
-				    op_errno,
-				    op_ret >= 0 ? &local->buf : NULL,
-				    local->xdata);
-		break;
-	case GF_FOP_STAT:
-		STACK_UNWIND_STRICT(stat,
-				    frame,
-				    op_ret,
-				    op_errno,
-				    op_ret >= 0 ? &local->buf : NULL,
-				    local->xdata);
-		break;
-	case GF_FOP_LOOKUP:
-		STACK_UNWIND_STRICT(lookup,
-				    frame,
-				    op_ret,
-				    op_errno,
-				    op_ret >= 0 ? local->inode : NULL,
-				    op_ret >= 0 ? &local->buf : NULL,				    
-				    local->xdata,
-				    op_ret >= 0 ? &local->postbuf : NULL);
-		break;
-	case GF_FOP_READ:
-		STACK_UNWIND_STRICT(readv,
-				    frame,
-				    op_ret,
-				    op_errno,
-				    NULL,
-				    0,
-				    op_ret >= 0 ? &local->buf : NULL,
-				    NULL,
-				    NULL);
-		break;
-	default:
-		gf_log(this->name, GF_LOG_WARNING,
-		       "Improper file operation %d", local->fop);
-	}
-	if (local_xdata)
-		dict_unref(local_xdata);
-	if (local_inode)
-		inode_unref(local_inode);
-	return 0;
-}
-
-static int32_t crypt_stat_common_cbk(call_frame_t *frame,
-				     void *cookie,
-				     xlator_t *this,
-				     int32_t op_ret,
-				     int32_t op_errno,
-				     struct iatt *buf, dict_t *xdata)
-{
-	crypt_local_t *local = frame->local;
-
-	if (op_ret < 0)
-		goto unwind;
-	if (!IA_ISREG(buf->ia_type))
-		goto unwind;
-
-	local->buf = *buf;
-	if (xdata)
-		local->xdata = dict_ref(xdata);
-
-	switch (local->fop) {
-	case GF_FOP_FSTAT:
-		STACK_WIND(frame,
-			   load_file_size,
-			   FIRST_CHILD(this),
-			   FIRST_CHILD(this)->fops->fgetxattr,
-			   local->fd,
-			   FSIZE_XATTR_PREFIX,
-			   NULL);
-		break;
-	case GF_FOP_STAT:
-		STACK_WIND(frame,
-			   load_file_size,
-			   FIRST_CHILD(this),
-			   FIRST_CHILD(this)->fops->getxattr,
-			   local->loc,
-			   FSIZE_XATTR_PREFIX,
-			   NULL);
-		break;
-	default:
-		gf_log (this->name, GF_LOG_WARNING,
-			"Improper file operation %d", local->fop);
-	}
-	return 0;
- unwind:
-	if (local->fd)
-		fd_unref(local->fd);
-	if (local->loc) {
-		loc_wipe(local->loc);
-		GF_FREE(local->loc);
-	}
-	switch (local->fop) {
-	case GF_FOP_FSTAT:
-		STACK_UNWIND_STRICT(fstat,
-				    frame,
-				    op_ret,
-				    op_errno,
-				    op_ret >= 0 ? buf : NULL,
-				    op_ret >= 0 ? xdata : NULL);
-		break;
-	case GF_FOP_STAT:
-		STACK_UNWIND_STRICT(stat,
-				    frame,
-				    op_ret,
-				    op_errno,
-				    op_ret >= 0 ? buf : NULL,
-				    op_ret >= 0 ? xdata : NULL);
-		break;
-	default:
-		gf_log (this->name, GF_LOG_WARNING,
-			"Improper file operation %d", local->fop);
-	}
-	return 0;
-}
-
-static int32_t crypt_fstat(call_frame_t *frame,
-			   xlator_t *this,
-			   fd_t *fd, dict_t *xdata)
-{
-	crypt_local_t *local;
-
-	local = crypt_alloc_local(frame, this, GF_FOP_FSTAT);
-	if (!local)
-		goto error;
-	local->fd = fd_ref(fd);
-	STACK_WIND(frame,
-		   crypt_stat_common_cbk,
-		   FIRST_CHILD(this),
-		   FIRST_CHILD(this)->fops->fstat,
-		   fd,
-		   xdata);	
-	return 0;
- error:
-	STACK_UNWIND_STRICT(fstat,
-			    frame,
-			    -1,
-			    ENOMEM,
-			    NULL,
-			    NULL);
-	return 0;
-}
-
-static int32_t crypt_stat(call_frame_t *frame,
-			  xlator_t *this,
-			  loc_t *loc, dict_t *xdata)
-{
-	int32_t ret;
-	crypt_local_t *local;
-
-	local = crypt_alloc_local(frame, this, GF_FOP_STAT);
-	if (!local)
-		goto error;
-	local->loc = GF_CALLOC(1, sizeof(*loc), gf_crypt_mt_loc);
-	if (!local->loc)
-		goto error;
-	memset(local->loc, 0, sizeof(*local->loc));
-	ret = loc_copy(local->loc, loc);
-	if (ret) {
-		GF_FREE(local->loc);
-		goto error;
-	}
-	STACK_WIND(frame,
-		   crypt_stat_common_cbk,
-		   FIRST_CHILD(this),
-		   FIRST_CHILD(this)->fops->stat,
-		   loc,
-		   xdata);	
-	return 0;
- error:
-	STACK_UNWIND_STRICT(stat,
-			    frame,
-			    -1,
-			    ENOMEM,
-			    NULL,
-			    NULL);
-	return 0;
-}
-
-static int32_t crypt_lookup_cbk(call_frame_t *frame,
-				void *cookie,
-				xlator_t *this,
-				int32_t op_ret,
-				int32_t op_errno,
-				inode_t *inode,
-				struct iatt *buf, dict_t *xdata,
-				struct iatt *postparent)
-{
-	crypt_local_t *local = frame->local;
-
-	if (op_ret < 0)
-		goto unwind;
-	if (!IA_ISREG(buf->ia_type))
-		goto unwind;
-
-	local->inode = inode_ref(inode);
-	local->buf = *buf;
-	local->postbuf = *postparent;
-	if (xdata)
-		local->xdata = dict_ref(xdata);
-	gf_uuid_copy(local->loc->gfid, buf->ia_gfid);
-
-	STACK_WIND(frame,
-		   load_file_size,
-		   FIRST_CHILD(this),
-		   FIRST_CHILD(this)->fops->getxattr,
-		   local->loc,
-		   FSIZE_XATTR_PREFIX,
-		   NULL);
-	return 0;
- unwind:
-	loc_wipe(local->loc);
-	GF_FREE(local->loc);
-	STACK_UNWIND_STRICT(lookup,
-			    frame,
-			    op_ret,
-			    op_errno,
-			    inode,
-			    buf,
-			    xdata,
-			    postparent);
-	return 0;
-}
-
-static int32_t crypt_lookup(call_frame_t *frame,
-			    xlator_t *this,
-			    loc_t *loc, dict_t *xdata)
-{
-	int32_t ret;
-	crypt_local_t *local;
-
-	local = crypt_alloc_local(frame, this, GF_FOP_LOOKUP);
-	if (!local)
-		goto error;
-	local->loc = GF_CALLOC(1, sizeof(*loc), gf_crypt_mt_loc);
-	if (!local->loc)
-		goto error;
-	memset(local->loc, 0, sizeof(*local->loc));
-	ret = loc_copy(local->loc, loc);
-	if (ret) {
-		GF_FREE(local->loc);
-		goto error;
-	}
-	gf_log(this->name, GF_LOG_DEBUG, "Lookup %s", loc->path);
-	STACK_WIND(frame,
-		   crypt_lookup_cbk,
-		   FIRST_CHILD(this),
-		   FIRST_CHILD(this)->fops->lookup,
-		   loc,
-		   xdata);
-	return 0;
- error:
-	STACK_UNWIND_STRICT(lookup,
-			    frame,
-			    -1,
-			    ENOMEM,
-			    NULL,
-			    NULL,
-			    NULL,
-			    NULL);
-	return 0;
-}
-
-/*
- * for every regular directory entry find its real file size
- * and update stat's buf properly
- */
-static int32_t crypt_readdirp_cbk(call_frame_t *frame,
-				  void *cookie,
-				  xlator_t *this,
-				  int32_t op_ret,
-				  int32_t op_errno,
-				  gf_dirent_t *entries, dict_t *xdata)
-{
-	gf_dirent_t *entry = NULL;
-
-	if (op_ret < 0)
-		goto unwind;
-
-	list_for_each_entry (entry, (&entries->list), list) {
-		data_t *data;
-
-		if (!IA_ISREG(entry->d_stat.ia_type))
-			continue;
-		data = dict_get(entry->dict, FSIZE_XATTR_PREFIX);
-		if (!data){
-			gf_log("crypt", GF_LOG_WARNING,
-			       "Regular file size of direntry not found");
-			op_errno = EIO;
-			op_ret = -1;
-			break;
-		}
-		entry->d_stat.ia_size = data_to_uint64(data);
-	}
- unwind:
-	STACK_UNWIND_STRICT(readdirp, frame, op_ret, op_errno, entries, xdata);
-	return 0;
-}
-
-/*
- * ->readdirp() fills in-core inodes, so we need to set proper
- * file sizes for all directory entries of the parent @fd.
- * Actual updates take place in ->crypt_readdirp_cbk()
- */
-static int32_t crypt_readdirp(call_frame_t *frame, xlator_t *this,
-			      fd_t *fd, size_t size, off_t offset,
-			      dict_t *xdata)
-{
-	int32_t ret = ENOMEM;
-
-	if (!xdata) {
-		xdata = dict_new();
-		if (!xdata)
-			goto error;
-	}
-	else
-		dict_ref(xdata);
-	/*
-	 * make sure that we'll have real file sizes at ->readdirp_cbk()
-	 */
-	ret = dict_set(xdata, FSIZE_XATTR_PREFIX, data_from_uint64(0));
-	if (ret) {
-		dict_unref(xdata);
-		goto error;
-	}
-	STACK_WIND(frame,
-		   crypt_readdirp_cbk,
-		   FIRST_CHILD(this),
-		   FIRST_CHILD(this)->fops->readdirp,
-		   fd,
-		   size,
-		   offset,
-		   xdata);
-	dict_unref(xdata);
-	return 0;
- error:
-	STACK_UNWIND_STRICT(readdirp, frame, -1, ret, NULL, NULL);
-	return 0;
-}
-
-static int32_t crypt_access(call_frame_t *frame,
-			    xlator_t *this,
-			    loc_t *loc,
-			    int32_t mask, dict_t *xdata)
-{
-	gf_log(this->name, GF_LOG_WARNING,
-	       "NFS mounts of encrypted volumes are unsupported");
-	STACK_UNWIND_STRICT(access, frame, -1, EPERM, NULL);
-	return 0;
-}
-
-int32_t master_set_block_size (xlator_t *this, crypt_private_t *priv,
-			       dict_t *options)
-{
-	uint64_t block_size = 0;
-	struct master_cipher_info *master = get_master_cinfo(priv);
-
-	if (options != NULL)
-		GF_OPTION_RECONF("block-size", block_size, options,
-				 size_uint64, error);
-	else
-		GF_OPTION_INIT("block-size", block_size, size_uint64, error);
-
-	switch (block_size) {
-	case 512:
-		master->m_block_bits = 9;
-		break;
-	case 1024:
-		master->m_block_bits = 10;
-		break;
-	case 2048:
-		master->m_block_bits = 11;
-		break;
-	case 4096:
-		master->m_block_bits = 12;
-		break;
-	default:
-		gf_log("crypt", GF_LOG_ERROR,
-		       "FATAL: unsupported block size %llu",
-		       (unsigned long long)block_size);
-		goto error;
-	}
-	return 0;
- error:
-	return -1;
-}
-
-int32_t master_set_alg(xlator_t *this, crypt_private_t *priv)
-{
-	struct master_cipher_info *master = get_master_cinfo(priv);
-	master->m_alg = AES_CIPHER_ALG;
-	return 0;
-}
-
-int32_t master_set_mode(xlator_t *this, crypt_private_t *priv)
-{
-	struct master_cipher_info *master = get_master_cinfo(priv);
-	master->m_mode = XTS_CIPHER_MODE;
-	return 0;
-}
-
-/*
- * set key size in bits to the master info
- * Pre-conditions: cipher mode in the master info is uptodate.
- */
-static int master_set_data_key_size (xlator_t *this, crypt_private_t *priv,
-				     dict_t *options)
-{
-	int32_t ret;
-	uint64_t key_size = 0;
-	struct master_cipher_info *master = get_master_cinfo(priv);
-
-	if (options != NULL)
-		GF_OPTION_RECONF("data-key-size", key_size, options,
-				 uint64, error);
-	else
-		GF_OPTION_INIT("data-key-size", key_size, uint64, error);
-
-	ret = data_cipher_algs[master->m_alg][master->m_mode].check_key(key_size);
-	if (ret) {
-		gf_log("crypt", GF_LOG_ERROR,
-		       "FATAL: wrong bin key size %llu for alg %d mode %d",
-		       (unsigned long long)key_size,
-		       (int)master->m_alg,
-		       (int)master->m_mode);
-		goto error;
-	}
-	master->m_dkey_size = key_size;
-	return 0;
- error:
-	return -1;
-}
-
-static int is_hex(char *s) {
-	return  ('0' <= *s && *s <= '9') || ('a' <= *s && *s <= 'f');
-}
-
-static int parse_hex_buf(xlator_t *this, char *src, unsigned char *dst,
-			 int hex_size)
-{
-	int i;
-	int hex_byte = 0;
-
-	for (i = 0; i < (hex_size / 2); i++) {
-		if (!is_hex(src + i*2) || !is_hex(src + i*2 + 1)) {
-			gf_log("crypt", GF_LOG_ERROR,
-			       "FATAL: not hex symbol in key");
-			return -1;
-		}
-		if (sscanf(src + i*2, "%2x", &hex_byte) != 1) {
-			gf_log("crypt", GF_LOG_ERROR,
-			       "FATAL: can not parse hex key");
-			return -1;
-		}
-		dst[i] = hex_byte & 0xff;
-	}
-	return 0;
-}
-
-/*
- * Parse options;
- * install master volume key
- */
-int32_t master_set_master_vol_key(xlator_t *this, crypt_private_t *priv)
-{
-	int32_t ret;
-	FILE *file = NULL;
-
-	int32_t key_size;
-	char *opt_key_file_pathname = NULL;
-
-	unsigned char bin_buf[MASTER_VOL_KEY_SIZE];
-	char hex_buf[2 * MASTER_VOL_KEY_SIZE];
-
-	struct master_cipher_info *master = get_master_cinfo(priv);
-	/*
-	 * extract master key passed via option
-	 */
-	GF_OPTION_INIT("master-key", opt_key_file_pathname, path, bad_key);
-
-	if (!opt_key_file_pathname) {
-		gf_log(this->name, GF_LOG_ERROR, "FATAL: missing master key");
-		return -1;
-	}
-	gf_log(this->name, GF_LOG_DEBUG, "handling file key %s",
-	       opt_key_file_pathname);
-
-	file = fopen(opt_key_file_pathname, "r");
-	if (file == NULL) {
-		gf_log(this->name, GF_LOG_ERROR,
-		       "FATAL: can not open file with master key");
-		return -1;
-	}
-	/*
-	 * extract hex key
-	 */
-	key_size = fread(hex_buf, 1, sizeof(hex_buf), file);
-	if (key_size < sizeof(hex_buf)) {
-		gf_log(this->name, GF_LOG_ERROR,
-		       "FATAL: master key is too short");
-		goto bad_key;
-	}
-	ret = parse_hex_buf(this, hex_buf, bin_buf, key_size);
-	if (ret)
-		goto bad_key;
-	memcpy(master->m_key, bin_buf, MASTER_VOL_KEY_SIZE);
-	memset(hex_buf, 0, sizeof(hex_buf));
-	fclose(file);
-
- 	memset(bin_buf, 0, sizeof(bin_buf));
-	return 0;
- bad_key:
-	gf_log(this->name, GF_LOG_ERROR, "FATAL: bad master key");
-	if (file)
-		fclose(file);
-	memset(bin_buf, 0, sizeof(bin_buf));
-	return -1;
-}
-
-/*
- * Derive volume key for object-id authentication
- */
-int32_t master_set_nmtd_vol_key(xlator_t *this, crypt_private_t *priv)
-{
-	return get_nmtd_vol_key(get_master_cinfo(priv));
-}
-
-int32_t crypt_init_xlator(xlator_t *this)
-{
-	int32_t ret;
-	crypt_private_t *priv = this->private; 
-
-	ret = master_set_alg(this, priv);
-	if (ret)
-		return ret;
-	ret = master_set_mode(this, priv);
-	if (ret)
-		return ret;
-	ret = master_set_block_size(this, priv, NULL);
-	if (ret)
-		return ret;
-	ret = master_set_data_key_size(this, priv, NULL);
-	if (ret)
-		return ret;
-	ret = master_set_master_vol_key(this, priv);
-	if (ret)
-		return ret;
-	return master_set_nmtd_vol_key(this, priv);
-}
-
-static int32_t crypt_alloc_private(xlator_t *this)
-{
-	this->private = GF_CALLOC(1, sizeof(crypt_private_t), gf_crypt_mt_priv);
-	if (!this->private) {
-		gf_log("crypt", GF_LOG_ERROR,
-		       "Can not allocate memory for private data");
-		return ENOMEM;
-	}
-	return 0;
-}
-
-static void crypt_free_private(xlator_t *this)
-{
-	crypt_private_t *priv = this->private;
-	if (priv) {
-		memset(priv, 0, sizeof(*priv));
-		GF_FREE(priv);
-	}
-}
-
-int32_t
-mem_acct_init (xlator_t *this)
-{
-        int     ret = -1;
-
-        if (!this)
-                return ret;
-
-        ret = xlator_mem_acct_init (this, gf_crypt_mt_end);
-
-        if (ret != 0) {
-                gf_log(this->name, GF_LOG_ERROR, "Memory accounting init"
-                       "failed");
-                return ret;
-        }
-
-        return ret;
-}
-
-int32_t reconfigure (xlator_t *this, dict_t *options)
-{
-	int32_t ret = -1;
-	crypt_private_t *priv = NULL;
-
-	GF_VALIDATE_OR_GOTO ("crypt", this, error);
-	GF_VALIDATE_OR_GOTO (this->name, this->private, error);
-	GF_VALIDATE_OR_GOTO (this->name, options, error);
-
-	priv = this->private;
-
-	ret = master_set_block_size(this, priv, options);
-	if (ret) {
-		gf_log("this->name", GF_LOG_ERROR,
-		       "Failed to reconfure block size");
-		goto error;
-	}
-	ret = master_set_data_key_size(this, priv, options);
-	if (ret) {
-		gf_log("this->name", GF_LOG_ERROR,
-		       "Failed to reconfure data key size");
-		goto error;
-	}
-	return 0;
- error:
-	return ret;
-}
-
-int32_t init(xlator_t *this)
-{
-	int32_t ret;
-
-	if (!this->children || this->children->next) {
-		gf_log ("crypt", GF_LOG_ERROR,
-			"FATAL: crypt should have exactly one child");
-		return EINVAL;
-	}
-	if (!this->parents) {
-		gf_log (this->name, GF_LOG_WARNING,
-			"dangling volume. check volfile ");
-	}
-	ret = crypt_alloc_private(this);
-	if (ret)
-		return ret;
-	ret = crypt_init_xlator(this);
-	if (ret)
-		goto error;
-	this->local_pool = mem_pool_new(crypt_local_t, 64);
-        if (!this->local_pool) {
-		gf_log(this->name, GF_LOG_ERROR,
-		       "failed to create local_t's memory pool");
-		ret = ENOMEM;
-                goto error;
-        }
-	gf_log ("crypt", GF_LOG_INFO, "crypt xlator loaded");
-	return 0;
- error:
-	crypt_free_private(this);
-	return ret;
-}
-
-void fini (xlator_t *this)
-{
-	crypt_free_private(this);
-}
-
-struct xlator_fops fops = {
-	.readv        = crypt_readv,
-	.writev       = crypt_writev,
-	.truncate     = crypt_truncate,
-	.ftruncate    = crypt_ftruncate,
-	.setxattr     = crypt_setxattr,
-	.fsetxattr    = crypt_fsetxattr,
-	.link         = crypt_link,
-	.unlink       = crypt_unlink,
-	.rename       = crypt_rename,
-	.open         = crypt_open,
-	.create       = crypt_create,
-	.stat         = crypt_stat,
-	.fstat        = crypt_fstat,
-	.lookup       = crypt_lookup,
-	.readdirp     = crypt_readdirp,
-	.access       = crypt_access   
-};
-
-struct xlator_cbks cbks = {
-	.forget       = crypt_forget
-};
-
-struct volume_options options[] = {
-	{ .key = {"master-key"},
-	  .type = GF_OPTION_TYPE_PATH,
-	  .description = "Pathname of regular file which contains master volume key"
-	},
-	{ .key = {"data-key-size"},
-	  .type = GF_OPTION_TYPE_SIZET,
-	  .description = "Data key size (bits)",
-	  .min = 256,
-	  .max = 512,
-	  .default_value = "256",
-	},
-	{ .key = {"block-size"},
-	  .type = GF_OPTION_TYPE_SIZET,
-	  .description = "Atom size (bits)",
-	  .min = 512,
-	  .max = 4096,
-	  .default_value = "4096"
-	},
-	{ .key  = {NULL} },
-};
-
-/*
-  Local variables:
-  c-indentation-style: "K&R"
-  mode-name: "LC"
-  c-basic-offset: 8
-  tab-width: 8
-  fill-column: 80
-  scroll-step: 1
-  End:
-*/
diff --git a/xlators/encryption/crypt/src/crypt.h b/xlators/encryption/crypt/src/crypt.h
deleted file mode 100644
index 31bb59d8765..00000000000
--- a/xlators/encryption/crypt/src/crypt.h
+++ /dev/null
@@ -1,904 +0,0 @@
-/*
-  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
-  This file is part of GlusterFS.
-
-  This file is licensed to you under your choice of the GNU Lesser
-  General Public License, version 3 or any later version (LGPLv3 or
-  later), or the GNU General Public License, version 2 (GPLv2), in all
-  cases as published by the Free Software Foundation.
-*/
-
-#ifndef __CRYPT_H__
-#define __CRYPT_H__
-
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
-#include <openssl/aes.h>
-#include <openssl/evp.h>
-#include <openssl/sha.h>
-#include <openssl/hmac.h>
-#include <openssl/cmac.h>
-#include <openssl/modes.h>
-#include "crypt-mem-types.h"
-#include "compat.h"
-
-#define CRYPT_XLATOR_ID  (0)
-
-#define MAX_IOVEC_BITS (3)
-#define MAX_IOVEC (1 << MAX_IOVEC_BITS)
-#define KEY_FACTOR_BITS  (6)
-
-#define DEBUG_CRYPT (0)
-#define TRIVIAL_TFM (0)
-
-#define CRYPT_MIN_BLOCK_BITS (9)
-#define CRYPT_MAX_BLOCK_BITS (12)
-
-#define MASTER_VOL_KEY_SIZE (32)
-#define NMTD_VOL_KEY_SIZE (16)
-
-#if !defined(GF_LINUX_HOST_OS)
-typedef off_t loff_t;
-#endif
-
-struct crypt_key {
-	uint32_t len;
-	const char *label;
-};
-
-/*
- * Add new key types to the end of this
- * enumeration but before LAST_KEY_TYPE
- */
-typedef enum {
-	MASTER_VOL_KEY,
-	NMTD_VOL_KEY,
-	NMTD_LINK_KEY,
-	EMTD_FILE_KEY,
-	DATA_FILE_KEY_256,
-	DATA_FILE_KEY_512,
-	LAST_KEY_TYPE
-}crypt_key_type;
-
-struct kderive_context {
-	const unsigned char *pkey;/* parent key */
-	uint32_t pkey_len;        /* parent key size, bits */
-	uint32_t ckey_len;        /* child key size, bits */
-	unsigned char *fid;       /* fixed input data, NIST 800-108, 5.1 */
-	uint32_t fid_len;         /* fid len, bytes */
-	unsigned char *out;       /* contains child keying material */
-	uint32_t out_len;         /* out len, bytes */
-};
-
-typedef enum {
-	DATA_ATOM,
-	HOLE_ATOM,
-	LAST_DATA_TYPE
-}atom_data_type;
-
-typedef enum {
-	HEAD_ATOM,
-	TAIL_ATOM,
-	FULL_ATOM,
-	LAST_LOCALITY_TYPE
-}atom_locality_type;
-
-typedef enum {
-	MTD_CREATE,
-	MTD_APPEND,
-	MTD_OVERWRITE,
-	MTD_CUT,
-	MTD_LAST_OP
-} mtd_op_t;
-
-struct xts128_context {
-	void      *key1, *key2;
-	block128_f block1,block2;
-};
-
-struct object_cipher_info {
-	cipher_alg_t  o_alg;
-        cipher_mode_t o_mode;
-	uint32_t      o_block_bits;
-        uint32_t      o_dkey_size; /* raw data key size in bits */
-	union {
-		struct {
-			unsigned char ivec[16];
-			AES_KEY dkey[2];
-			AES_KEY tkey;    /* key used for tweaking */
-			XTS128_CONTEXT xts;
-		} aes_xts;
-	} u;
-};
-
-struct master_cipher_info {
-	/*
-	 * attributes inherited by newly created regular files
-	 */
-	cipher_alg_t  m_alg;
-        cipher_mode_t m_mode;
-	uint32_t      m_block_bits;
-        uint32_t      m_dkey_size; /* raw key size in bits */
-	/*
-	 * master key
-	 */
-	unsigned char m_key[MASTER_VOL_KEY_SIZE];
-	/*
-	 * volume key for oid authentication
-	 */
-	unsigned char m_nmtd_key[NMTD_VOL_KEY_SIZE];
-};
-
-/*
-* This info is not changed during file's life
- */
-struct crypt_inode_info {
-#if DEBUG_CRYPT
-	loc_t *loc; /* pathname that the file has been
-		       opened, or created with */
-#endif
-	uint16_t nr_minor;
-	uuid_t oid;
-	struct object_cipher_info cinfo;
-};
-
-/*
- * this should locate in secure memory
- */
-typedef struct {
-	struct master_cipher_info master;
-} crypt_private_t;
-
-static inline struct master_cipher_info *get_master_cinfo(crypt_private_t *priv)
-{
-	return &priv->master;
-}
-
-static inline struct object_cipher_info *get_object_cinfo(struct crypt_inode_info
-							 *info)
-{
-	return &info->cinfo;
-}
-
-/*
- * this describes layouts and properties
- * of atoms in an aligned vector
- */
-struct avec_config {
-	uint32_t atom_size;
-	atom_data_type type;
-	size_t orig_size;
-	off_t orig_offset;
-	size_t expanded_size;
-	off_t aligned_offset;
-
-	uint32_t off_in_head;
-	uint32_t off_in_tail;
-	uint32_t gap_in_tail;
-	uint32_t nr_full_blocks;
-
-	struct iovec *avec; /* aligned vector */
-	uint32_t acount; /* number of avec components. The same
-			  * as number of occupied logical blocks */
-	char **pool;
-	uint32_t blocks_in_pool;
-	uint32_t cursor; /* makes sense only for ordered writes,
-			  * so there is no races on this counter.
-			  *
-			  * Cursor is per-config object, we don't
-			  * reset cursor for atoms of different
-			  * localities (head, tail, full)
-			  */
-};
-
-
-typedef struct {
-	glusterfs_fop_t fop; /* code of FOP this local info built for */
-	fd_t *fd;
-	inode_t *inode;
-	loc_t *loc;
-	int32_t mac_idx;
-	loc_t *newloc;
-	int32_t flags;
-	int32_t wbflags;
-	struct crypt_inode_info *info;
-	struct iobref *iobref;
-	struct iobref *iobref_data;
-	off_t offset;
-
-	uint64_t old_file_size; /* per FOP, retrieved under lock held */
-	uint64_t cur_file_size; /* per iteration, before issuing IOs */
-	uint64_t new_file_size; /* per iteration, after issuing IOs */
-
-	uint64_t io_offset; /* offset of IOs issued per iteration */
-	uint64_t io_offset_nopad; /* offset of user's data in the atom */
-	uint32_t io_size; /* size of IOs issued per iteration */
-	uint32_t io_size_nopad; /* size of user's data in the IOs */
-	uint32_t eof_padding_size; /* size od EOF padding in the IOs */
-
-	gf_lock_t call_lock; /* protect nr_calls from many cbks */
-	int32_t nr_calls;
-
-	atom_data_type active_setup; /* which setup (hole or date)
-					is currently active */
-	/* data setup */
-	struct avec_config data_conf;
-
-	/* hole setup */
-	int hole_conv_in_proggress;
-	gf_lock_t hole_lock; /* protect hole config from many cbks */
-	int hole_handled;
-	struct avec_config hole_conf;
-	struct iatt buf;
-	struct iatt prebuf;
-	struct iatt postbuf;
-	struct iatt *prenewparent;
-	struct iatt *postnewparent;
-	int32_t op_ret;
-	int32_t op_errno;
-	int32_t rw_count; /* total read or written */
-	gf_lock_t rw_count_lock; /* protect the counter above */
-	unsigned char *format; /* for create, update format string */
-	uint32_t format_size;
-	uint32_t msgflags; /* messages for crypt_open() */
-	dict_t *xdata;
-	dict_t *xattr;
-	struct iovec vec; /* contains last file's atom for
-			     read-prune-write sequence */
-	gf_boolean_t custom_mtd;
-	/*
-	 * the next 3 fields are used by readdir and friends
-	 */
-	gf_dirent_t *de; /* directory entry */
-	char *de_path; /* pathname of directory entry */
-	uint32_t de_prefix_len; /* length of the parent's pathname */
-	gf_dirent_t *entries;
-
-	uint32_t update_disk_file_size:1;
-} crypt_local_t;
-
-/* This represents a (read)modify-write atom */
-struct rmw_atom {
-	atom_locality_type locality;
-	/*
-	 * read-modify-write sequence of the atom
-	 */
-	int32_t (*rmw)(call_frame_t *frame,
-		       void *cookie,
-		       xlator_t *this,
-		       int32_t op_ret,
-		       int32_t op_errno,
-		       struct iovec *vec,
-		       int32_t count,
-		       struct iatt *stbuf,
-		       struct iobref *iobref,
-		       dict_t *xdata);
-	/*
-	 * offset of the logical block in a file
-	 */
-	loff_t (*offset_at)(call_frame_t *frame,
-			    struct object_cipher_info *object);
-	/*
-	 * IO offset in an atom
-	 */
-	uint32_t (*offset_in)(call_frame_t *frame,
-			      struct object_cipher_info *object);
-	/*
-	 * number of bytes of plain text of this atom that user
-	 * wants to read/write.
-	 * It can be smaller than atom_size in the case of head
-	 * or tail atoms.
-	 */
-	uint32_t (*io_size_nopad)(call_frame_t *frame,
-				  struct object_cipher_info *object);
-	/*
-	 * which iovec represents the atom
-	 */
-	struct iovec *(*get_iovec)(call_frame_t *frame, uint32_t count);
-	/*
-	 * how many bytes of partial block should be uptodated by
-	 * reading from disk.
-	 * This is used to perform a read component of RMW (read-modify-write).
-	 */
-	uint32_t (*count_to_uptodate)(call_frame_t *frame, struct object_cipher_info *object);
-	struct avec_config *(*get_config)(call_frame_t *frame);
-};
-
-struct data_cipher_alg {
-	gf_boolean_t atomic; /* true means that algorithm requires
-				to pad data before cipher transform */
-	gf_boolean_t should_pad; /* true means that algorithm requires
-				    to pad the end of file with extra-data */
-	uint32_t blkbits; /* blksize = 1 << blkbits */
-	/*
-	 * any preliminary sanity checks goes here
-	 */
-	int32_t (*init)(void);
-	/*
-	 * set alg-mode specific inode info
-	 */
-	int32_t (*set_private)(struct crypt_inode_info *info,
-			       struct master_cipher_info *master);
-	/*
-	 * check alg-mode specific data key
-	 */
-	int32_t (*check_key)(uint32_t key_size);
-	void (*set_iv)(off_t offset, struct object_cipher_info *object);
-	int32_t (*encrypt)(const unsigned char *from, unsigned char *to,
-			   size_t length, off_t offset, const int enc,
-			   struct object_cipher_info *object);
-};
-
-/*
- * version-dependent metadata loader
- */
-struct crypt_mtd_loader {
-	/*
-	 * return core format size
-	 */
-	size_t (*format_size)(mtd_op_t op, size_t old_size);
-	/*
-	 * pack version-specific metadata of an object
-	 * at ->create()
-	 */
-	int32_t (*create_format)(unsigned char *wire,
-				 loc_t *loc,
-				 struct crypt_inode_info *info,
-				 struct master_cipher_info *master);
-	/*
-	 * extract version-specific metadata of an object
-	 * at ->open() time
-	 */
-	int32_t (*open_format)(unsigned char *wire,
-			       int32_t len,
-			       loc_t *loc,
-			       struct crypt_inode_info *info,
-			       struct master_cipher_info *master,
-			       crypt_local_t *local,
-			       gf_boolean_t load_info);
-	int32_t (*update_format)(unsigned char *new,
-				 unsigned char *old,
-				 size_t old_len,
-				 int32_t mac_idx,
-				 mtd_op_t op,
-				 loc_t *loc,
-				 struct crypt_inode_info *info,
-				 struct master_cipher_info *master,
-				 crypt_local_t *local);
-};
-
-typedef int32_t (*end_writeback_handler_t)(call_frame_t *frame,
-					   void *cookie,
-					   xlator_t *this,
-					   int32_t op_ret,
-					   int32_t op_errno,
-					   struct iatt *prebuf,
-					   struct iatt *postbuf,
-					   dict_t *xdata);
-typedef void (*linkop_wind_handler_t)(call_frame_t *frame, xlator_t *this);
-typedef void (*linkop_unwind_handler_t)(call_frame_t *frame);
-
-
-/* Declarations */
-
-/* keys.c */
-extern struct crypt_key crypt_keys[LAST_KEY_TYPE];
-int32_t get_nmtd_vol_key(struct master_cipher_info *master);
-int32_t get_nmtd_link_key(loc_t *loc,
-			  struct master_cipher_info *master,
-			  unsigned char *result);
-int32_t get_emtd_file_key(struct crypt_inode_info *info,
-			  struct master_cipher_info *master,
-			  unsigned char *result);
-int32_t get_data_file_key(struct crypt_inode_info *info,
-			  struct master_cipher_info *master,
-			  uint32_t keysize,
-			  unsigned char *key);
-/* data.c */
-extern struct data_cipher_alg data_cipher_algs[LAST_CIPHER_ALG][LAST_CIPHER_MODE];
-void encrypt_aligned_iov(struct object_cipher_info *object,
-			 struct iovec *vec,
-			 int count,
-			 off_t off);
-void decrypt_aligned_iov(struct object_cipher_info *object,
-			 struct iovec *vec,
-			 int count,
-			 off_t off);
-int32_t align_iov_by_atoms(xlator_t *this,
-			   crypt_local_t *local,
-			   struct object_cipher_info *object,
-			   struct iovec *vec /* input vector */,
-			   int32_t count /* number of vec components */,
-			   struct iovec *avec /* aligned vector */,
-			   char **blocks /* pool of blocks */,
-			   uint32_t *blocks_allocated,
-			   struct avec_config *conf);
-int32_t set_config_avec_data(xlator_t *this,
-			     crypt_local_t *local,
-			     struct avec_config *conf,
-			     struct object_cipher_info *object,
-			     struct iovec *vec,
-			     int32_t vec_count);
-int32_t set_config_avec_hole(xlator_t *this,
-			     crypt_local_t *local,
-			     struct avec_config *conf,
-			     struct object_cipher_info *object,
-			     glusterfs_fop_t fop);
-void set_gap_at_end(call_frame_t *frame, struct object_cipher_info *object,
-		    struct avec_config *conf, atom_data_type dtype);
-void set_config_offsets(call_frame_t *frame,
-			xlator_t *this,
-			uint64_t offset,
-			uint64_t count,
-			atom_data_type dtype,
-			int32_t setup_gap_in_tail);
-
-/* metadata.c */
-extern struct crypt_mtd_loader mtd_loaders [LAST_MTD_LOADER];
-
-int32_t alloc_format(crypt_local_t *local, size_t size);
-int32_t alloc_format_create(crypt_local_t *local);
-void free_format(crypt_local_t *local);
-size_t format_size(mtd_op_t op, size_t old_size);
-size_t new_format_size(void);
-int32_t open_format(unsigned char *str, int32_t len, loc_t *loc,
-		    struct crypt_inode_info *info,
-		    struct master_cipher_info *master, crypt_local_t *local,
-		    gf_boolean_t load_info);
-int32_t update_format(unsigned char *new, unsigned char *old,
-		      size_t old_len, int32_t mac_idx, mtd_op_t op, loc_t *loc,
-		      struct crypt_inode_info *info,
-		      struct master_cipher_info *master,
-		      crypt_local_t *local);
-int32_t create_format(unsigned char *wire,
-		      loc_t *loc,
-		      struct crypt_inode_info *info,
-		      struct master_cipher_info *master);
-
-/* atom.c */
-struct rmw_atom *atom_by_types(atom_data_type data,
-			       atom_locality_type locality);
-void submit_partial(call_frame_t *frame,
-		    xlator_t *this,
-		    fd_t *fd,
-		    atom_locality_type ltype);
-void submit_full(call_frame_t *frame, xlator_t *this);
-
-/* crypt.c */
-
-end_writeback_handler_t dispatch_end_writeback(glusterfs_fop_t fop);
-static size_t iovec_get_size(struct iovec *vec, uint32_t count);
-void set_local_io_params_writev(call_frame_t *frame,
-				struct object_cipher_info *object,
-				struct rmw_atom *atom, off_t io_offset,
-				uint32_t io_size);
-void link_wind(call_frame_t *frame, xlator_t *this);
-void unlink_wind(call_frame_t *frame, xlator_t *this);
-void link_unwind(call_frame_t *frame);
-void unlink_unwind(call_frame_t *frame);
-void rename_wind(call_frame_t *frame, xlator_t *this);
-void rename_unwind(call_frame_t *frame);
-
-/* Inline functions */
-
-static inline size_t iovec_get_size(struct iovec *vec, uint32_t count)
-{
-	int i;
-	size_t size = 0;
-	for (i = 0; i < count; i++)
-		size += vec[i].iov_len;
-	return size;
-}
-
-static inline int32_t crypt_xlator_id(void)
-{
-	return CRYPT_XLATOR_ID;
-}
-
-static inline mtd_loader_id current_mtd_loader(void)
-{
-	return MTD_LOADER_V1;
-}
-
-static inline uint32_t master_key_size (void)
-{
-	return crypt_keys[MASTER_VOL_KEY].len >> 3;
-}
-
-static inline uint32_t nmtd_vol_key_size (void)
-{
-	return crypt_keys[NMTD_VOL_KEY].len >> 3;
-}
-
-static inline uint32_t alg_mode_blkbits(cipher_alg_t alg,
-					cipher_mode_t mode)
-{
-	return data_cipher_algs[alg][mode].blkbits;
-}
-
-static inline uint32_t alg_mode_blksize(cipher_alg_t alg,
-					cipher_mode_t mode)
-{
-	return 1 << alg_mode_blkbits(alg, mode);
-}
-
-static inline gf_boolean_t alg_mode_atomic(cipher_alg_t alg,
-					   cipher_mode_t mode)
-{
-	return data_cipher_algs[alg][mode].atomic;
-}
-
-static inline gf_boolean_t alg_mode_should_pad(cipher_alg_t alg,
-					       cipher_mode_t mode)
-{
-	return data_cipher_algs[alg][mode].should_pad;
-}
-
-static inline uint32_t master_alg_blksize(struct master_cipher_info *mr)
-{
-	return alg_mode_blksize(mr->m_alg, mr->m_mode);
-}
-
-static inline uint32_t master_alg_blkbits(struct master_cipher_info *mr)
-{
-	return alg_mode_blkbits(mr->m_alg, mr->m_mode);
-}
-
-static inline gf_boolean_t master_alg_atomic(struct master_cipher_info *mr)
-{
-	return alg_mode_atomic(mr->m_alg, mr->m_mode);
-}
-
-static inline gf_boolean_t master_alg_should_pad(struct master_cipher_info *mr)
-{
-	return alg_mode_should_pad(mr->m_alg, mr->m_mode);
-}
-
-static inline uint32_t object_alg_blksize(struct object_cipher_info *ob)
-{
-	return alg_mode_blksize(ob->o_alg, ob->o_mode);
-}
-
-static inline uint32_t object_alg_blkbits(struct object_cipher_info *ob)
-{
-	return alg_mode_blkbits(ob->o_alg, ob->o_mode);
-}
-
-static inline gf_boolean_t object_alg_atomic(struct object_cipher_info *ob)
-{
-	return alg_mode_atomic(ob->o_alg, ob->o_mode);
-}
-
-static inline gf_boolean_t object_alg_should_pad(struct object_cipher_info *ob)
-{
-	return alg_mode_should_pad(ob->o_alg, ob->o_mode);
-}
-
-static inline uint32_t aes_raw_key_size(struct master_cipher_info *master)
-{
-	return master->m_dkey_size >> 3;
-}
-
-static inline struct avec_config *get_hole_conf(call_frame_t *frame)
-{
-	return &(((crypt_local_t *)frame->local)->hole_conf);
-}
-
-static inline struct avec_config *get_data_conf(call_frame_t *frame)
-{
-	return &(((crypt_local_t *)frame->local)->data_conf);
-}
-
-static inline int32_t get_atom_bits (struct object_cipher_info *object)
-{
-	return object->o_block_bits;
-}
-
-static inline int32_t get_atom_size (struct object_cipher_info *object)
-{
-	return 1 << get_atom_bits(object);
-}
-
-static inline int32_t has_head_block(struct avec_config *conf)
-{
-	return conf->off_in_head ||
-		(conf->acount == 1 && conf->off_in_tail);
-}
-
-static inline int32_t has_tail_block(struct avec_config *conf)
-{
-	return conf->off_in_tail && conf->acount > 1;
-}
-
-static inline int32_t has_full_blocks(struct avec_config *conf)
-{
-	return conf->nr_full_blocks;
-}
-
-static inline int32_t should_submit_head_block(struct avec_config *conf)
-{
-	return has_head_block(conf) && (conf->cursor == 0);
-}
-
-static inline int32_t should_submit_tail_block(struct avec_config *conf)
-{
-	return has_tail_block(conf) && (conf->cursor == conf->acount - 1);
-}
-
-static inline int32_t should_submit_full_block(struct avec_config *conf)
-{
-	uint32_t start = has_head_block(conf) ? 1 : 0;
-
-	return has_full_blocks(conf) &&
-		conf->cursor >= start &&
-		conf->cursor < start + conf->nr_full_blocks;
-}
-
-#if DEBUG_CRYPT
-static inline void crypt_check_input_len(size_t len,
-					 struct object_cipher_info *object)
-{
-	if (object_alg_should_pad(object) && (len & (object_alg_blksize(object) - 1)))
-		gf_log ("crypt", GF_LOG_DEBUG, "bad input len: %d", (int)len);
-}
-
-static inline void check_head_block(struct avec_config *conf)
-{
-	if (!has_head_block(conf))
-		gf_log("crypt", GF_LOG_DEBUG, "not a head atom");
-}
-
-static inline void check_tail_block(struct avec_config *conf)
-{
-	if (!has_tail_block(conf))
-		gf_log("crypt", GF_LOG_DEBUG, "not a tail atom");
-}
-
-static inline void check_full_block(struct avec_config *conf)
-{
-	if (!has_full_blocks(conf))
-		gf_log("crypt", GF_LOG_DEBUG, "not a full atom");
-}
-
-static inline void check_cursor_head(struct avec_config *conf)
-{
-	if (!has_head_block(conf))
-		gf_log("crypt",
-		       GF_LOG_DEBUG, "Illegal call of head atom method");
-	else if (conf->cursor != 0)
-		gf_log("crypt",
-		       GF_LOG_DEBUG, "Cursor (%d) is not at head atom",
-		       conf->cursor);
-}
-
-static inline void check_cursor_full(struct avec_config *conf)
-{
-	if (!has_full_blocks(conf))
-		gf_log("crypt",
-		       GF_LOG_DEBUG, "Illegal call of full atom method");
-	if (has_head_block(conf) && (conf->cursor == 0))
-		gf_log("crypt",
-		       GF_LOG_DEBUG, "Cursor is not at full atom");
-}
-
-/*
- * FIXME: use avec->iov_len to check setup
- */
-static inline int data_local_invariant(crypt_local_t *local)
-{
-	return 0;
-}
-
-#else
-#define crypt_check_input_len(len, object) noop
-#define check_head_block(conf) noop
-#define check_tail_block(conf) noop
-#define check_full_block(conf) noop
-#define check_cursor_head(conf) noop
-#define check_cursor_full(conf) noop
-
-#endif /* DEBUG_CRYPT */
-
-static inline struct avec_config *conf_by_type(call_frame_t *frame,
-					       atom_data_type dtype)
-{
-	struct avec_config *conf = NULL;
-
-	switch (dtype) {
-	case HOLE_ATOM:
-		conf = get_hole_conf(frame);
-		break;
-	case DATA_ATOM:
-		conf = get_data_conf(frame);
-		break;
-	default:
-		gf_log("crypt", GF_LOG_DEBUG, "bad atom type");
-	}
-	return conf;
-}
-
-static inline uint32_t nr_calls_head(struct avec_config *conf)
-{
-	return has_head_block(conf) ? 1 : 0;
-}
-
-static inline uint32_t nr_calls_tail(struct avec_config *conf)
-{
-	return has_tail_block(conf) ? 1 : 0;
-}
-
-static inline uint32_t nr_calls_full(struct avec_config *conf)
-{
-	switch(conf->type) {
-	case HOLE_ATOM:
-		return has_full_blocks(conf);
-	case DATA_ATOM:
-		return has_full_blocks(conf) ?
-			logical_blocks_occupied(0,
-						conf->nr_full_blocks,
-						MAX_IOVEC_BITS)	: 0;
-	default:
-		gf_log("crypt", GF_LOG_DEBUG, "bad atom data type");
-		return 0;
-	}
-}
-
-static inline uint32_t nr_calls(struct avec_config *conf)
-{
-	return nr_calls_head(conf) + nr_calls_tail(conf) + nr_calls_full(conf);
-}
-
-static inline uint32_t nr_calls_data(call_frame_t *frame)
-{
-	return nr_calls(get_data_conf(frame));
-}
-
-static inline uint32_t nr_calls_hole(call_frame_t *frame)
-{
-	return nr_calls(get_hole_conf(frame));
-}
-
-static inline void get_one_call_nolock(call_frame_t *frame)
-{
-	crypt_local_t *local = frame->local;
-
-	++local->nr_calls;
-
-	//gf_log("crypt", GF_LOG_DEBUG, "get %d calls", 1);
-}
-
-static inline void get_one_call(call_frame_t *frame)
-{
-	crypt_local_t *local = frame->local;
-
-	LOCK(&local->call_lock);
-	get_one_call_nolock(frame);
-	UNLOCK(&local->call_lock);
-}
-
-static inline void get_nr_calls_nolock(call_frame_t *frame, int32_t nr)
-{
-	crypt_local_t *local = frame->local;
-
-	local->nr_calls += nr;
-
-	//gf_log("crypt", GF_LOG_DEBUG, "get %d calls", nr);
-}
-
-static inline void get_nr_calls(call_frame_t *frame, int32_t nr)
-{
-	crypt_local_t *local = frame->local;
-
-	LOCK(&local->call_lock);
-	get_nr_calls_nolock(frame, nr);
-	UNLOCK(&local->call_lock);
-}
-
-static inline int put_one_call(crypt_local_t *local)
-{
-	uint32_t last = 0;
-
-	LOCK(&local->call_lock);
-	if (--local->nr_calls == 0)
-		last = 1;
-
-	//gf_log("crypt", GF_LOG_DEBUG, "put %d calls", 1);
-
-	UNLOCK(&local->call_lock);
-	return last;
-}
-
-static inline int is_appended_write(call_frame_t *frame)
-{
-	crypt_local_t *local = frame->local;
-	struct avec_config *conf = get_data_conf(frame);
-
-	return conf->orig_offset + conf->orig_size > local->old_file_size;
-}
-
-static inline int is_ordered_mode(call_frame_t *frame)
-{
-#if 0
-	crypt_local_t *local = frame->local;
-	return local->fop == GF_FOP_FTRUNCATE ||
-		(local->fop == GF_FOP_WRITE && is_appended_write(frame));
-#endif
-	return 1;
-}
-
-static inline int32_t hole_conv_completed(crypt_local_t *local)
-{
-	struct avec_config *conf = &local->hole_conf;
-	return conf->cursor == conf->acount;
-}
-
-static inline int32_t data_write_in_progress(crypt_local_t *local)
-{
-	return local->active_setup == DATA_ATOM;
-}
-
-static inline int32_t parent_is_crypt_xlator(call_frame_t *frame,
-					     xlator_t *this)
-{
-	return frame->parent->this == this;
-}
-
-static inline linkop_wind_handler_t linkop_wind_dispatch(glusterfs_fop_t fop)
-{
-	switch(fop){
-	case GF_FOP_LINK:
-		return link_wind;
-	case GF_FOP_UNLINK:
-		return unlink_wind;
-	case GF_FOP_RENAME:
-		return rename_wind;
-	default:
-		gf_log("crypt", GF_LOG_ERROR, "Bad link operation %d", fop);
-		return NULL;
-	}
-}
-
-static inline linkop_unwind_handler_t linkop_unwind_dispatch(glusterfs_fop_t fop)
-{
-	switch(fop){
-	case GF_FOP_LINK:
-		return link_unwind;
-	case GF_FOP_UNLINK:
-		return unlink_unwind;
-	case GF_FOP_RENAME:
-		return rename_unwind;
-	default:
-		gf_log("crypt", GF_LOG_ERROR, "Bad link operation %d", fop);
-		return NULL;
-	}
-}
-
-static inline mtd_op_t linkop_mtdop_dispatch(glusterfs_fop_t fop)
-{
-	switch (fop) {
-	case GF_FOP_LINK:
-		return MTD_APPEND;
-	case GF_FOP_UNLINK:
-		return MTD_CUT;
-	case GF_FOP_RENAME:
-		return MTD_OVERWRITE;
-	default:
-		gf_log("crypt", GF_LOG_WARNING, "Bad link operation %d", fop);
-		return MTD_LAST_OP;
-	}
-}
-
-#endif /* __CRYPT_H__ */
-
-/*
-  Local variables:
-  c-indentation-style: "K&R"
-  mode-name: "LC"
-  c-basic-offset: 8
-  tab-width: 8
-  fill-column: 80
-  scroll-step: 1
-  End:
-*/
diff --git a/xlators/encryption/crypt/src/data.c b/xlators/encryption/crypt/src/data.c
deleted file mode 100644
index 762fa554ac2..00000000000
--- a/xlators/encryption/crypt/src/data.c
+++ /dev/null
@@ -1,769 +0,0 @@
-/*
-  Copyright (c) 2008-2013 Red Hat, Inc. <http://www.redhat.com>
-  This file is part of GlusterFS.
-
-  This file is licensed to you under your choice of the GNU Lesser
-  General Public License, version 3 or any later version (LGPLv3 or
-  later), or the GNU General Public License, version 2 (GPLv2), in all
-  cases as published by the Free Software Foundation.
-*/
-
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
-
-#include "defaults.h"
-#include "crypt-common.h"
-#include "crypt.h"
-
-static void set_iv_aes_xts(off_t offset, struct object_cipher_info *object)
-{
-	unsigned char *ivec;
-
-	ivec = object->u.aes_xts.ivec;
-
-	/* convert the tweak into a little-endian byte
-	 * array (IEEE P1619/D16, May 2007, section 5.1)
-	 */
-
-	*((uint64_t *)ivec) = htole64(offset);
-
-	/* ivec is padded with zeroes */
-}
-
-static int32_t aes_set_keys_common(unsigned char *raw_key, uint32_t key_size,
-				   AES_KEY *keys)
-{
-	int32_t ret;
-
-	ret = AES_set_encrypt_key(raw_key,
-				  key_size,
-				  &keys[AES_ENCRYPT]);
-	if (ret) {
-		gf_log("crypt", GF_LOG_ERROR, "Set encrypt key failed");
-		return ret;
-	}
-	ret = AES_set_decrypt_key(raw_key,
-				  key_size,
-				  &keys[AES_DECRYPT]);
-	if (ret) {
-		gf_log("crypt", GF_LOG_ERROR, "Set decrypt key failed");
-		return ret;
-	}
-	return 0;
-}
-
-/*
- * set private cipher info for xts mode
- */
-static int32_t set_private_aes_xts(struct crypt_inode_info *info,
-				   struct master_cipher_info *master)
-{
-	int ret;
-	struct object_cipher_info *object = get_object_cinfo(info);
-	unsigned char *data_key;
-	uint32_t subkey_size;
-
-	/* init tweak value */
-	memset(object->u.aes_xts.ivec, 0, 16);
-
-	data_key = GF_CALLOC(1, object->o_dkey_size, gf_crypt_mt_key);
-	if (!data_key)
-		return ENOMEM;
-
-	/*
-	 * retrieve data keying meterial
-	 */
-	ret = get_data_file_key(info, master, object->o_dkey_size, data_key);
-	if (ret) {
-		gf_log("crypt", GF_LOG_ERROR, "Failed to retrieve data key");
-		GF_FREE(data_key);
-		return ret;
-	}
-	/*
-	 * parse compound xts key
-	 */
-	subkey_size = object->o_dkey_size >> 4; /* (xts-key-size-in-bytes / 2) */
-	/*
-	 * install key for data encryption
-	 */
-	ret = aes_set_keys_common(data_key,
-				  subkey_size << 3, object->u.aes_xts.dkey);
-	if (ret) {
-		GF_FREE(data_key);
-		return ret;
-	}
-	/*
-	 * set up key used to encrypt tweaks
-	 */
-	ret = AES_set_encrypt_key(data_key + subkey_size,
-				  object->o_dkey_size / 2,
-				  &object->u.aes_xts.tkey);
-	if (ret < 0)
-		gf_log("crypt", GF_LOG_ERROR, "Set tweak key failed");
-
-	GF_FREE(data_key);
-	return ret;
-}
-
-static int32_t aes_xts_init(void)
-{
-	cassert(AES_BLOCK_SIZE == (1 << AES_BLOCK_BITS));
-	return 0;
-}
-
-static int32_t check_key_aes_xts(uint32_t keysize)
-{
-	switch(keysize) {
-	case 256:
-	case 512:
-		return 0;
-	default:
-		break;
-	}
-	return -1;
-}
-
-static int32_t encrypt_aes_xts(const unsigned char *from,
-			       unsigned char *to, size_t length,
-			       off_t offset, const int enc,
-			       struct object_cipher_info *object)
-{
-	XTS128_CONTEXT ctx;
-	if (enc) {
-		ctx.key1 = &object->u.aes_xts.dkey[AES_ENCRYPT];
-		ctx.block1 = (block128_f)AES_encrypt;
-	}
-	else {
-		ctx.key1 = &object->u.aes_xts.dkey[AES_DECRYPT];
-		ctx.block1 = (block128_f)AES_decrypt;
-	}
-	ctx.key2 = &object->u.aes_xts.tkey;
-	ctx.block2 = (block128_f)AES_encrypt;
-
-	return CRYPTO_xts128_encrypt(&ctx,
-				     object->u.aes_xts.ivec,
-				     from,
-				     to,
-				     length, enc);
-}
-
-/*
- * Cipher input chunk @from of length @len;
- * @to: result of cipher transform;
- * @off: offset in a file (must be cblock-aligned);
- */
-static void cipher_data(struct object_cipher_info *object,
-			char *from,
-			char *to,
-			off_t off,
-			size_t len,
-			const int enc)
-{
-	crypt_check_input_len(len, object);
-
-#if TRIVIAL_TFM && DEBUG_CRYPT
-	return;
-#endif
-	data_cipher_algs[object->o_alg][object->o_mode].set_iv(off, object);
-	data_cipher_algs[object->o_alg][object->o_mode].encrypt
-		((const unsigned char *)from,
-		 (unsigned char *)to,
-		 len,
-		 off,
-		 enc,
-		 object);
-}
-
-#define MAX_CIPHER_CHUNK (1 << 30)
-
-/*
- * Do cipher (encryption/decryption) transform of a
- * continuous region of memory.
- *
- * @len: a number of bytes to transform;
- * @buf: data to transform;
- * @off: offset in a file, should be block-aligned
- *       for atomic cipher modes and ksize-aligned
- *       for other modes).
- * @dir: direction of transform (encrypt/decrypt).
- */
-static void cipher_region(struct object_cipher_info *object,
-			  char *from,
-			  char *to,
-			  off_t off,
-			  size_t len,
-			  int dir)
-{
-	while (len > 0) {
-		size_t to_cipher;
-
-		to_cipher = len;
-		if (to_cipher > MAX_CIPHER_CHUNK)
-			to_cipher = MAX_CIPHER_CHUNK;
-
-		/* this will reset IV */
-		cipher_data(object,
-			    from,
-			    to,
-			    off,
-			    to_cipher,
-			    dir);
-		from += to_cipher;
-		to   += to_cipher;
-		off  += to_cipher;
-		len  -= to_cipher;
-	}
-}
-
-/*
- * Do cipher transform (encryption/decryption) of
- * plaintext/ciphertext represented by @vec.
- *
- * Pre-conditions: @vec represents a continuous piece
- * of data in a file at offset @off to be ciphered
- * (encrypted/decrypted).
- * @count is the number of vec's components. All the
- * components must be block-aligned, the caller is
- * responsible for this. @dir is "direction" of
- * transform (encrypt/decrypt).
- */
-static void cipher_aligned_iov(struct object_cipher_info *object,
-			       struct iovec *vec,
-			       int count,
-			       off_t off,
-			       int32_t dir)
-{
-	int i;
-	int len = 0;
-
-	for (i = 0; i < count; i++) {
-		cipher_region(object,
-			      vec[i].iov_base,
-			      vec[i].iov_base,
-			      off + len,
-			      vec[i].iov_len,
-			      dir);
-		len += vec[i].iov_len;
-	}
-}
-
-void encrypt_aligned_iov(struct object_cipher_info *object,
-			 struct iovec *vec,
-			 int count,
-			 off_t off)
-{
-	cipher_aligned_iov(object, vec, count, off, 1);
-}
-
-void decrypt_aligned_iov(struct object_cipher_info *object,
-			 struct iovec *vec,
-			 int count,
-			 off_t off)
-{
-	cipher_aligned_iov(object, vec, count, off, 0);
-}
-
-#if DEBUG_CRYPT
-static void compound_stream(struct iovec *vec, int count, char *buf, off_t skip)
-{
-	int i;
-	int off = 0;
-	for (i = 0; i < count; i++) {
-		memcpy(buf + off,
-		       vec[i].iov_base + skip,
-		       vec[i].iov_len - skip);
-
-		off += (vec[i].iov_len - skip);
-		skip = 0;
-	}
-}
-
-static void check_iovecs(struct iovec *vec, int cnt,
-			 struct iovec *avec, int acnt, uint32_t off_in_head)
-{
-	char *s1, *s2;
-	uint32_t size, asize;
-
-	size = iovec_get_size(vec, cnt);
-	asize = iovec_get_size(avec, acnt) - off_in_head;
-	if (size != asize) {
-		gf_log("crypt", GF_LOG_DEBUG, "size %d is not eq asize %d",
-		       size, asize);
-		return;
-	}
-	s1 = GF_CALLOC(1, size, gf_crypt_mt_data);
-	if (!s1) {
-		gf_log("crypt", GF_LOG_DEBUG, "Can not allocate stream ");
-		return;
-	}
-	s2 = GF_CALLOC(1, asize, gf_crypt_mt_data);
-	if (!s2) {
-		GF_FREE(s1);
-		gf_log("crypt", GF_LOG_DEBUG, "Can not allocate stream ");
-		return;
-	}
-	compound_stream(vec, cnt, s1, 0);
-	compound_stream(avec, acnt, s2, off_in_head);
-	if (memcmp(s1, s2, size))
-		gf_log("crypt", GF_LOG_DEBUG, "chunks of different data");
-	GF_FREE(s1);
-	GF_FREE(s2);
-}
-
-#else
-#define check_iovecs(vec, count, avec, avecn, off) noop
-#endif /* DEBUG_CRYPT */
-
-static char *data_alloc_block(xlator_t *this, crypt_local_t *local,
-			      int32_t block_size)
-{
-	struct iobuf *iobuf = NULL;
-
-	iobuf = iobuf_get2(this->ctx->iobuf_pool, block_size);
-	if (!iobuf) {
-		gf_log("crypt", GF_LOG_ERROR,
-		       "Failed to get iobuf");
-		return NULL;
-	}
-	if (!local->iobref_data) {
-		local->iobref_data = iobref_new();
-		if (!local->iobref_data) {
-			gf_log("crypt", GF_LOG_ERROR,
-			       "Failed to get iobref");
-			iobuf_unref(iobuf);
-			return NULL;
-		}
-	}
-	iobref_add(local->iobref_data, iobuf);
-	return iobuf->ptr;
-}
-
-/*
- * Compound @avec, which represent the same data
- * chunk as @vec, but has aligned components of
- * specified block size. Alloc blocks, if needed.
- * In particular, incomplete head and tail blocks
- * must be allocated.
- * Put number of allocated blocks to @num_blocks.
- *
- * Example:
- *
- * input: data chunk represented by 4 components
- * [AB],[BC],[CD],[DE];
- * output: 5 logical blocks (0, 1, 2, 3, 4).
- *
- *   A     B       C     D               E
- *   *-----*+------*-+---*----+--------+-*
- *   |     ||      | |   |    |        | |
- * *-+-----+*------+-*---+----*--------*-+------*
- * 0        1        2        3        4
- *
- * 0 - incomplete compound (head);
- * 1, 2 - full compound;
- * 3 - full non-compound (the case of reuse);
- * 4 - incomplete non-compound (tail).
- */
-int32_t align_iov_by_atoms(xlator_t *this,
-			   crypt_local_t *local,
-			   struct object_cipher_info *object,
-			   struct iovec *vec /* input vector */,
-			   int32_t count /* number of vec components */,
-			   struct iovec *avec /* aligned vector */,
-			   char **blocks /* pool of blocks */,
-			   uint32_t *blocks_allocated,
-			   struct avec_config *conf)
-{
-	int vecn = 0;      /* number of the current component in vec */
-	int avecn = 0;     /* number of the current component in avec */
-	off_t vec_off = 0; /* offset in the current vec component,
-			    * i.e. the number of bytes have already
-			    * been copied */
-	int32_t block_size = get_atom_size(object);
-	size_t to_process; /* number of vec's bytes to copy and(or) re-use */
-	int32_t off_in_head = conf->off_in_head;
-
-	to_process = iovec_get_size(vec, count);
-
-	while (to_process > 0) {
-		if (off_in_head ||
-		    vec[vecn].iov_len - vec_off < block_size) {
-			/*
-			 * less than block_size:
-			 * the case of incomplete (head or tail),
-			 * or compound block
-			 */
-			size_t copied = 0;
-			/*
-			 * populate the pool with a new block
-			 */
-			blocks[*blocks_allocated] = data_alloc_block(this,
-								    local,
-								    block_size);
-			if (!blocks[*blocks_allocated])
-				return -ENOMEM;
-			memset(blocks[*blocks_allocated], 0, off_in_head);
-			/*
-			 * fill the block with vec components
-			 */
-			do {
-				size_t to_copy;
-
-				to_copy = vec[vecn].iov_len - vec_off;
-				if (to_copy > block_size - off_in_head)
-					to_copy = block_size - off_in_head;
-
-				memcpy(blocks[*blocks_allocated] + off_in_head + copied,
-				       vec[vecn].iov_base + vec_off,
-				       to_copy);
-
-				copied += to_copy;
-				to_process -= to_copy;
-
-				vec_off += to_copy;
-				if (vec_off == vec[vecn].iov_len) {
-					/* finished with this vecn */
-					vec_off = 0;
-					vecn++;
-				}
-			} while (copied < (block_size - off_in_head) && to_process > 0);
-			/*
-			 * update avec
-			 */
-			avec[avecn].iov_len = off_in_head + copied;
-			avec[avecn].iov_base = blocks[*blocks_allocated];
-
-			(*blocks_allocated)++;
-			off_in_head = 0;
-		} else {
-			/*
-			 * the rest of the current vec component
-			 * is not less than block_size, so reuse
-			 * the memory buffer of the component.
-			 */
-			size_t to_reuse;
-			to_reuse = (to_process > block_size ?
-				    block_size :
-				    to_process);
-			avec[avecn].iov_len = to_reuse;
-			avec[avecn].iov_base = vec[vecn].iov_base + vec_off;
-
-			vec_off += to_reuse;
-			if (vec_off == vec[vecn].iov_len) {
-				/* finished with this vecn */
-				vec_off = 0;
-				vecn++;
-			}
-			to_process -= to_reuse;
-		}
-		avecn++;
-	}
-	check_iovecs(vec, count, avec, avecn, conf->off_in_head);
-	return 0;
-}
-
-/*
- * allocate and setup aligned vector for data submission
- * Pre-condition: @conf is set.
- */
-int32_t set_config_avec_data(xlator_t *this,
-			     crypt_local_t *local,
-			     struct avec_config *conf,
-			     struct object_cipher_info *object,
-			     struct iovec *vec,
-			     int32_t vec_count)
-{
-	int32_t ret = ENOMEM;
-	struct iovec *avec;
-	char **pool;
-	uint32_t blocks_in_pool = 0;
-
-	conf->type = DATA_ATOM;
-
-	avec = GF_CALLOC(conf->acount, sizeof(*avec), gf_crypt_mt_iovec);
-	if (!avec)
-		return ret;
-	pool = GF_CALLOC(conf->acount, sizeof(pool), gf_crypt_mt_char);
-	if (!pool) {
-		GF_FREE(avec);
-		return ret;
-	}
-	if (!vec) {
-		/*
-		 * degenerated case: no data
-		 */
-		pool[0] = data_alloc_block(this, local, get_atom_size(object));
-		if (!pool[0])
-			goto free;
-		blocks_in_pool = 1;
-		avec->iov_base = pool[0];
-		avec->iov_len = conf->off_in_tail;
-	}
-	else {
-		ret = align_iov_by_atoms(this, local, object, vec, vec_count,
-					 avec, pool, &blocks_in_pool, conf);
-		if (ret)
-			goto free;
-	}
-	conf->avec = avec;
-	conf->pool = pool;
-	conf->blocks_in_pool = blocks_in_pool;
-	return 0;
- free:
-	GF_FREE(avec);
-	GF_FREE(pool);
-	return ret;
-}
-
-/*
- * allocate and setup aligned vector for hole submission
- */
-int32_t set_config_avec_hole(xlator_t *this,
-			     crypt_local_t *local,
-			     struct avec_config *conf,
-			     struct object_cipher_info *object,
-			     glusterfs_fop_t fop)
-{
-	uint32_t i, idx;
-	struct iovec *avec;
-	char **pool;
-	uint32_t num_blocks;
-	uint32_t blocks_in_pool = 0;
-
-	conf->type = HOLE_ATOM;
-
-	num_blocks = conf->acount -
-		(conf->nr_full_blocks ? conf->nr_full_blocks - 1 : 0);
-
-	switch (fop) {
-	case GF_FOP_WRITE:
-		/*
-		 * hole goes before data
-		 */
-		if (num_blocks == 1 && conf->off_in_tail != 0)
-		/*
-		 * we won't submit a hole which fits into
-		 * a data atom: this part of hole will be
-		 * submitted with data write
-		 */
-			return 0;
-		break;
-	case GF_FOP_FTRUNCATE:
-		/*
-		 * expanding truncate, hole goes after data,
-		 * and will be submited in any case.
- 		 */
-		break;
-	default:
-		gf_log("crypt", GF_LOG_WARNING,
-		       "bad file operation %d", fop);
-		return 0;
-	}
-	avec = GF_CALLOC(num_blocks, sizeof(*avec), gf_crypt_mt_iovec);
-	if (!avec)
-		return ENOMEM;
-	pool = GF_CALLOC(num_blocks, sizeof(pool), gf_crypt_mt_char);
-	if (!pool) {
-		GF_FREE(avec);
-		return ENOMEM;
-	}
-	for (i = 0; i < num_blocks; i++) {
-		pool[i] = data_alloc_block(this, local, get_atom_size(object));
-		if (pool[i] == NULL)
-			goto free;
-		blocks_in_pool++;
-	}
-	if (has_head_block(conf)) {
-		/* set head block */
-		idx = 0;
-		avec[idx].iov_base = pool[idx];
-		avec[idx].iov_len = get_atom_size(object);
-		memset(avec[idx].iov_base + conf->off_in_head,
-		       0,
-		       get_atom_size(object) - conf->off_in_head);
-	}
-	if (has_tail_block(conf)) {
-		/* set tail block */
-		idx = num_blocks - 1;
-		avec[idx].iov_base = pool[idx];
-		avec[idx].iov_len = get_atom_size(object);
-		memset(avec[idx].iov_base, 0, conf->off_in_tail);
-	}
-	if (has_full_blocks(conf)) {
-		/* set full block */
-		idx = conf->off_in_head ? 1 : 0;
-		avec[idx].iov_base = pool[idx];
-		avec[idx].iov_len = get_atom_size(object);
-		/*
-		 * since we re-use the buffer,
-		 * zeroes will be set every time
-		 * before encryption, see submit_full()
-		 */
-	}
-	conf->avec = avec;
-	conf->pool = pool;
-	conf->blocks_in_pool = blocks_in_pool;
-	return 0;
- free:
-	GF_FREE(avec);
-	GF_FREE(pool);
-	return ENOMEM;
-}
-
-/* A helper for setting up config of partial atoms (which
- * participate in read-modify-write sequence).
- *
- * Calculate and setup precise amount of "extra-bytes"
- * that should be uptodated at the end of partial (not
- * necessarily tail!) block.
- *
- * Pre-condition: local->old_file_size is valid!
- * @conf contains setup, which is enough for correct calculation
- * of has_tail_block(), ->get_offset().
- */
-void set_gap_at_end(call_frame_t *frame, struct object_cipher_info *object,
-		    struct avec_config *conf, atom_data_type dtype)
-{
-	uint32_t to_block;
-	crypt_local_t *local = frame->local;
-	uint64_t old_file_size = local->old_file_size;
-	struct rmw_atom *partial = atom_by_types(dtype,
-						 has_tail_block(conf) ?
-						 TAIL_ATOM : HEAD_ATOM);
-
-	if (old_file_size <= partial->offset_at(frame, object))
-		to_block = 0;
-	else {
-		to_block = old_file_size - partial->offset_at(frame, object);
-		if (to_block > get_atom_size(object))
-			to_block = get_atom_size(object);
-	}
-	if (to_block > conf->off_in_tail)
-		conf->gap_in_tail = to_block - conf->off_in_tail;
-	else
-		/*
-		 * nothing to uptodate
-		 */
-		conf->gap_in_tail = 0;
-}
-
-/*
- * fill struct avec_config with offsets layouts
- */
-void set_config_offsets(call_frame_t *frame,
-			xlator_t *this,
-			uint64_t offset,
-			uint64_t count,
-			atom_data_type dtype,
-			int32_t set_gap)
-{
-	crypt_local_t *local;
-	struct object_cipher_info *object;
-	struct avec_config *conf;
-	uint32_t resid;
-
-	uint32_t atom_size;
-	uint32_t atom_bits;
-
-	size_t orig_size;
-	off_t orig_offset;
-	size_t expanded_size;
-	off_t aligned_offset;
-
-	uint32_t off_in_head = 0;
-	uint32_t off_in_tail = 0;
-	uint32_t nr_full_blocks;
-	int32_t size_full_blocks;
-
-	uint32_t acount; /* number of alifned components to write.
-			  * The same as number of occupied logical
-			  * blocks (atoms)
-			  */
-	local = frame->local;
-	object = &local->info->cinfo;
-	conf = (dtype == DATA_ATOM ?
-		get_data_conf(frame) : get_hole_conf(frame));
-
-	orig_offset = offset;
-	orig_size = count;
-
-	atom_size = get_atom_size(object);
-	atom_bits = get_atom_bits(object);
-
-	/*
-	 * Round-down the start,
-	 * round-up the end.
-	 */
-	resid = offset & (uint64_t)(atom_size - 1);
-
-	if (resid)
-		off_in_head = resid;
-	aligned_offset = offset - off_in_head;
-	expanded_size = orig_size + off_in_head;
-
-	/* calculate tail,
-	   expand size forward  */
-	resid = (offset + orig_size) & (uint64_t)(atom_size - 1);
-
-	if (resid) {
-		off_in_tail = resid;
-		expanded_size += (atom_size - off_in_tail);
-	}
-	/*
-	 * calculate number of occupied blocks
-	 */
-	acount = expanded_size >> atom_bits;
-	/*
-	 * calculate number of full blocks
-	 */
-	size_full_blocks = expanded_size;
-	if (off_in_head)
-		size_full_blocks -= atom_size;
-	if (off_in_tail && size_full_blocks > 0)
-		size_full_blocks -= atom_size;
-	nr_full_blocks = size_full_blocks >> atom_bits;
-
-	conf->atom_size = atom_size;
-	conf->orig_size = orig_size;
-	conf->orig_offset = orig_offset;
-	conf->expanded_size = expanded_size;
-	conf->aligned_offset = aligned_offset;
-
-	conf->off_in_head = off_in_head;
-	conf->off_in_tail = off_in_tail;
-	conf->nr_full_blocks = nr_full_blocks;
-	conf->acount = acount;
-	/*
-	 * Finally, calculate precise amount of
-	 * "extra-bytes" that should be uptodated
-	 * at the end.
-	 * Only if RMW is expected.
-	 */
-	if (off_in_tail && set_gap)
-		set_gap_at_end(frame, object, conf, dtype);
-}
-
-struct data_cipher_alg data_cipher_algs[LAST_CIPHER_ALG][LAST_CIPHER_MODE] = {
-	[AES_CIPHER_ALG][XTS_CIPHER_MODE] =
-	{ .atomic = _gf_true,
-	  .should_pad = _gf_true,
-	  .blkbits = AES_BLOCK_BITS,
-	  .init = aes_xts_init,
-	  .set_private = set_private_aes_xts,
-	  .check_key = check_key_aes_xts,
-	  .set_iv = set_iv_aes_xts,
-	  .encrypt = encrypt_aes_xts
-	}
-};
-
-/*
-  Local variables:
-  c-indentation-style: "K&R"
-  mode-name: "LC"
-  c-basic-offset: 8
-  tab-width: 8
-  fill-column: 80
-  scroll-step: 1
-  End:
-*/
diff --git a/xlators/encryption/crypt/src/keys.c b/xlators/encryption/crypt/src/keys.c
deleted file mode 100644
index 4a1d3bb5a09..00000000000
--- a/xlators/encryption/crypt/src/keys.c
+++ /dev/null
@@ -1,302 +0,0 @@
-/*
-  Copyright (c) 2008-2013 Red Hat, Inc. <http://www.redhat.com>
-  This file is part of GlusterFS.
-
-  This file is licensed to you under your choice of the GNU Lesser
-  General Public License, version 3 or any later version (LGPLv3 or
-  later), or the GNU General Public License, version 2 (GPLv2), in all
-  cases as published by the Free Software Foundation.
-*/
-
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
-
-#include "defaults.h"
-#include "crypt-common.h"
-#include "crypt.h"
-
-/* Key hierarchy
-                  
-                   +----------------+
-                   | MASTER_VOL_KEY |
-                   +-------+--------+
-                           |
-                           |
-          +----------------+----------------+
-          |                |                |
-          |                |                |
-  +-------+------+ +-------+-------+ +------+--------+
-  | NMTD_VOL_KEY | | EMTD_FILE_KEY | | DATA_FILE_KEY |
-  +-------+------+ +---------------+ +---------------+
-          |
-          |
-  +-------+-------+
-  | NMTD_LINK_KEY |
-  +---------------+
-
- */
-
-#if DEBUG_CRYPT
-static void check_prf_iters(uint32_t num_iters)
-{
-	if (num_iters == 0)
-		gf_log ("crypt", GF_LOG_DEBUG,
-			"bad number of prf iterations : %d", num_iters);
-}
-#else
-#define check_prf_iters(num_iters) noop
-#endif /* DEBUG_CRYPT */
-
-unsigned char crypt_fake_oid[16] =
-	{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0};
-
-/*
- * derive key in the counter mode using
- * sha256-based HMAC as PRF, see
- * NIST Special Publication 800-108, 5.1)
- */
-
-#define PRF_OUTPUT_SIZE  SHA256_DIGEST_LENGTH
-
-static int32_t kderive_init(struct kderive_context *ctx,
-			    const unsigned char *pkey, /* parent key */
-			    uint32_t pkey_size,  /* parent key size */
-			    const unsigned char *idctx, /* id-context */
-			    uint32_t idctx_size,
-			    crypt_key_type type  /* type of child key */)
-{
-	unsigned char *pos;
-	uint32_t llen = strlen(crypt_keys[type].label);	
-	/*
-	 * Compoud the fixed input data for KDF:
-	 * [i]_2 || Label || 0x00 || Id-Context || [L]_2),
-	 * NIST SP 800-108, 5.1
-	 */
-	ctx->fid_len =
-		sizeof(uint32_t) +
-		llen +
-		1 +
-		idctx_size +
-		sizeof(uint32_t);
-
-	ctx->fid = GF_CALLOC(ctx->fid_len, 1, gf_crypt_mt_key);
-	if (!ctx->fid)
-		return ENOMEM;
-	ctx->out_len = round_up(crypt_keys[type].len >> 3,
-				PRF_OUTPUT_SIZE);
-	ctx->out = GF_CALLOC(ctx->out_len, 1, gf_crypt_mt_key);
-	if (!ctx->out) {
-		GF_FREE(ctx->fid);
-		return ENOMEM;
-	}
-	ctx->pkey = pkey;
-	ctx->pkey_len = pkey_size;
-	ctx->ckey_len = crypt_keys[type].len;
-
-	pos = ctx->fid;
-
-	/* counter will be set up in kderive_rfn() */
-	pos += sizeof(uint32_t);
-
-	memcpy(pos, crypt_keys[type].label, llen);
-	pos += llen;
-
-	/* set up zero octet */
-	*pos = 0;
-	pos += 1;
-
-	memcpy(pos, idctx, idctx_size);
-	pos += idctx_size;
-
-	*((uint32_t *)pos) = htobe32(ctx->ckey_len);
-
-	return 0;
-}
-
-static void kderive_update(struct kderive_context *ctx)
-{	
-	uint32_t i;
-	HMAC_CTX hctx;
-	unsigned char *pos = ctx->out;
-	uint32_t *p_iter = (uint32_t *)ctx->fid;
-	uint32_t num_iters = ctx->out_len / PRF_OUTPUT_SIZE;
-
-	check_prf_iters(num_iters);
-
-	HMAC_CTX_init(&hctx);
-	for (i = 0; i < num_iters; i++) {
-		/*
-		 * update the iteration number in the fid
-		 */
-		*p_iter = htobe32(i);
-		HMAC_Init_ex(&hctx,
-			     ctx->pkey, ctx->pkey_len >> 3,
-			     EVP_sha256(),
-			     NULL);
-		HMAC_Update(&hctx, ctx->fid, ctx->fid_len);
-		HMAC_Final(&hctx, pos, NULL);
-
-		pos += PRF_OUTPUT_SIZE;
-	}
-	HMAC_CTX_cleanup(&hctx);
-}
-
-static void kderive_final(struct kderive_context *ctx, unsigned char *child)
-{
-	memcpy(child, ctx->out, ctx->ckey_len >> 3);
-	GF_FREE(ctx->fid);
-	GF_FREE(ctx->out);
-	memset(ctx, 0, sizeof(*ctx));
-}
-
-/*
- * derive per-volume key for object ids aithentication
- */
-int32_t get_nmtd_vol_key(struct master_cipher_info *master)
-{
-	int32_t ret;
-	struct kderive_context ctx;
-
-	ret = kderive_init(&ctx,
-			   master->m_key,
-			   master_key_size(),
-			   crypt_fake_oid, sizeof(uuid_t), NMTD_VOL_KEY);
-	if (ret)
-		return ret;
-	kderive_update(&ctx);
-	kderive_final(&ctx, master->m_nmtd_key);
-	return 0;
-}
-
-/*
- * derive per-link key for aithentication of non-encrypted
- * meta-data (nmtd)
- */
-int32_t get_nmtd_link_key(loc_t *loc,
-			  struct master_cipher_info *master,
-			  unsigned char *result)
-{
-	int32_t ret;
-	struct kderive_context ctx;
-
-	ret = kderive_init(&ctx,
-			   master->m_nmtd_key,
-			   nmtd_vol_key_size(),
-			   (const unsigned char *)loc->path,
-			   strlen(loc->path), NMTD_LINK_KEY);
-	if (ret)
-		return ret;
-	kderive_update(&ctx);
-	kderive_final(&ctx, result);
-	return 0;
-}
-
-/*
- * derive per-file key for encryption and authentication
- * of encrypted part of metadata (emtd)
- */
-int32_t get_emtd_file_key(struct crypt_inode_info *info,
-			  struct master_cipher_info *master,
-			  unsigned char *result)
-{
-	int32_t ret;
-	struct kderive_context ctx;
-
-	ret = kderive_init(&ctx,
-			   master->m_key,
-			   master_key_size(),
-			   info->oid, sizeof(uuid_t), EMTD_FILE_KEY);
-	if (ret)
-		return ret;
-	kderive_update(&ctx);
-	kderive_final(&ctx, result);
-	return 0;
-}
-
-static int32_t data_key_type_by_size(uint32_t keysize, crypt_key_type *type)
-{
-	int32_t ret = 0;
-	switch (keysize) {
-	case 256:
-		*type = DATA_FILE_KEY_256;
-		break;
-	case 512:
-		*type = DATA_FILE_KEY_512;
-		break;
-	default:
-		gf_log("crypt", GF_LOG_ERROR, "Unsupported data key size %d",
-		       keysize);
-		ret = ENOTSUP;
-		break;
-	}
-	return ret;
-}
-
-/*
- * derive per-file key for data encryption
- */
-int32_t get_data_file_key(struct crypt_inode_info *info,
-			  struct master_cipher_info *master,
-			  uint32_t keysize,
-			  unsigned char *key)
-{
-	int32_t ret;
-	struct kderive_context ctx;
-	crypt_key_type type;
-
-	ret = data_key_type_by_size(keysize, &type);
-	if (ret)
-		return ret;
-	ret = kderive_init(&ctx,
-			   master->m_key,
-			   master_key_size(),
-			   info->oid, sizeof(uuid_t), type);
-	if (ret)
-		return ret;
-	kderive_update(&ctx);
-	kderive_final(&ctx, key);
-	return 0;
-}
-
-/*
- * NOTE: Don't change existing keys: it will break compatibility;
- */
-struct crypt_key crypt_keys[LAST_KEY_TYPE] = {
-	[MASTER_VOL_KEY] =
-	{ .len = MASTER_VOL_KEY_SIZE << 3,
-	  .label = "volume-master",
-	},
-	[NMTD_VOL_KEY] =
-	{ .len = NMTD_VOL_KEY_SIZE << 3,
-	  .label = "volume-nmtd-key-generation"
-	},
-	[NMTD_LINK_KEY] =
-	{ .len = 128,
-	  .label = "link-nmtd-authentication"
-	},
-	[EMTD_FILE_KEY] =
-	{ .len = 128,
-	  .label = "file-emtd-encryption-and-auth"
-	},
-	[DATA_FILE_KEY_256] =
-	{ .len = 256,
-	  .label = "file-data-encryption-256"
-	},
-	[DATA_FILE_KEY_512] =
-	{ .len = 512,
-	  .label = "file-data-encryption-512"
-	}
-};
-
-/*
-  Local variables:
-  c-indentation-style: "K&R"
-  mode-name: "LC"
-  c-basic-offset: 8
-  tab-width: 8
-  fill-column: 80
-  scroll-step: 1
-  End:
-*/
diff --git a/xlators/encryption/crypt/src/metadata.c b/xlators/encryption/crypt/src/metadata.c
deleted file mode 100644
index a4124ab8d6b..00000000000
--- a/xlators/encryption/crypt/src/metadata.c
+++ /dev/null
@@ -1,619 +0,0 @@
-/*
-  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
-  This file is part of GlusterFS.
-
-  This file is licensed to you under your choice of the GNU Lesser
-  General Public License, version 3 or any later version (LGPLv3 or
-  later), or the GNU General Public License, version 2 (GPLv2), in all
-  cases as published by the Free Software Foundation.
-*/
-
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
-
-#include "defaults.h"
-#include "crypt-common.h"
-#include "crypt.h"
-#include "metadata.h"
-
-int32_t alloc_format(crypt_local_t *local, size_t size)
-{
-	if (size > 0) {
-		local->format = GF_CALLOC(1, size, gf_crypt_mt_mtd);
-		if (!local->format)
-			return ENOMEM;
-	}
-	local->format_size = size;
-	return 0;
-}
-
-int32_t alloc_format_create(crypt_local_t *local)
-{
-	return alloc_format(local, new_format_size());
-}
-
-void free_format(crypt_local_t *local)
-{
-	GF_FREE(local->format);
-}
-
-/*
- * Check compatibility with extracted metadata
- */
-static int32_t check_file_metadata(struct crypt_inode_info *info)
-{
-	struct object_cipher_info *object = &info->cinfo;
-
-	if (info->nr_minor != CRYPT_XLATOR_ID) {
-		gf_log("crypt", GF_LOG_WARNING,
-		       "unsupported minor subversion %d", info->nr_minor);
-		return EINVAL;
-	}
-	if (object->o_alg > LAST_CIPHER_ALG) {
-		gf_log("crypt", GF_LOG_WARNING,
-		       "unsupported cipher algorithm %d",
-		       object->o_alg);
-		return EINVAL;
-	}
-	if (object->o_mode > LAST_CIPHER_MODE) {
-		gf_log("crypt", GF_LOG_WARNING,
-		       "unsupported cipher mode %d",
-		       object->o_mode);
-		return EINVAL;
-	}
-	if (object->o_block_bits < CRYPT_MIN_BLOCK_BITS ||
-	    object->o_block_bits > CRYPT_MAX_BLOCK_BITS) {
-		gf_log("crypt", GF_LOG_WARNING, "unsupported block bits %d",
-		       object->o_block_bits);
-		return EINVAL;
-	}
-	/* TBD: check data key size */
-	return 0;
-}
-
-static size_t format_size_v1(mtd_op_t op, size_t old_size)
-{
-	
-	switch (op) {
-	case MTD_CREATE:
-		return sizeof(struct mtd_format_v1);
-	case MTD_OVERWRITE:
-		return old_size;
-	case MTD_APPEND:
-		return old_size + NMTD_8_MAC_SIZE;
-	case MTD_CUT:
-		if (old_size > sizeof(struct mtd_format_v1))
-			return old_size - NMTD_8_MAC_SIZE;
-		else
-			return 0;
-	default:
-		gf_log("crypt", GF_LOG_WARNING, "Bad mtd operation");
-		return 0;
-	}
-}
-
-/*
- * Calculate size of the updated format string.
- * Returned zero means that we don't need to update the format string.
- */
-size_t format_size(mtd_op_t op, size_t old_size)
-{
-	size_t versioned;
-
-	versioned = mtd_loaders[current_mtd_loader()].format_size(op,
-				 old_size - sizeof(struct crypt_format));
-	if (versioned != 0)
-		return versioned + sizeof(struct crypt_format);
-	return 0;
-}
-
-/*
- * size of the format string of newly created file (nr_links = 1)
- */
-size_t new_format_size(void)
-{
-	return format_size(MTD_CREATE, 0);
-}
-
-/*
- * Calculate per-link MAC by pathname
- */
-static int32_t calc_link_mac_v1(struct mtd_format_v1 *fmt,
-				loc_t *loc,
-				unsigned char *result,
-				struct crypt_inode_info *info,
-				struct master_cipher_info *master)
-{
-	int32_t ret;	
-	unsigned char nmtd_link_key[16];
-	CMAC_CTX *cctx;
-	size_t len;
-
-	ret = get_nmtd_link_key(loc, master, nmtd_link_key);
-	if (ret) {
-		gf_log("crypt", GF_LOG_ERROR, "Can not get nmtd link key");
-		return -1;
-	}
-	cctx = CMAC_CTX_new();
-	if (!cctx) {
-		gf_log("crypt", GF_LOG_ERROR, "CMAC_CTX_new failed");
-		return -1; 
-	}
-	ret = CMAC_Init(cctx, nmtd_link_key, sizeof(nmtd_link_key),
-			EVP_aes_128_cbc(), 0);
-	if (!ret) {
-		gf_log("crypt", GF_LOG_ERROR, "CMAC_Init failed");
-		CMAC_CTX_free(cctx);
-		return -1;
-	}
-	ret = CMAC_Update(cctx, get_NMTD_V1(info), SIZE_OF_NMTD_V1);
-	if (!ret) {
-		gf_log("crypt", GF_LOG_ERROR, "CMAC_Update failed");
-		CMAC_CTX_free(cctx);
-		return -1;
-	}
-	ret = CMAC_Final(cctx, result, &len);
-	CMAC_CTX_free(cctx);
-	if (!ret) {
-		gf_log("crypt", GF_LOG_ERROR, "CMAC_Final failed");
-		return -1;
-	}
-	return 0;
-}
-
-/*
- * Create per-link MAC of index @idx by pathname
- */
-static int32_t create_link_mac_v1(struct mtd_format_v1 *fmt,
-				  uint32_t idx,
-				  loc_t *loc,
-				  struct crypt_inode_info *info,
-				  struct master_cipher_info *master)
-{
-	int32_t ret;
-	unsigned char *mac;
-	unsigned char cmac[16];
-
-	mac = get_NMTD_V1_MAC(fmt) + idx * SIZE_OF_NMTD_V1_MAC;
-
-	ret = calc_link_mac_v1(fmt, loc, cmac, info, master);
-	if (ret)
-		return -1;
-	memcpy(mac, cmac, SIZE_OF_NMTD_V1_MAC);
-	return 0;
-}
-
-static int32_t create_format_v1(unsigned char *wire,
-				loc_t *loc,
-				struct crypt_inode_info *info,
-				struct master_cipher_info *master)
-{
-	int32_t ret;
-	struct mtd_format_v1 *fmt;
-	unsigned char mtd_key[16];
-	AES_KEY EMTD_KEY;
-	unsigned char nmtd_link_key[16];
-	uint32_t ad;
-	GCM128_CONTEXT *gctx;
-
-	fmt = (struct mtd_format_v1 *)wire;
-
-	fmt->minor_id = info->nr_minor;
-	fmt->alg_id = AES_CIPHER_ALG;
-	fmt->dkey_factor = master->m_dkey_size >> KEY_FACTOR_BITS;
-	fmt->block_bits = master->m_block_bits;
-	fmt->mode_id = master->m_mode;
-	/*
-	 * retrieve keys for the parts of metadata
-	 */
-	ret = get_emtd_file_key(info, master, mtd_key);
-	if (ret)
-		return ret;
-	ret = get_nmtd_link_key(loc, master, nmtd_link_key);
-	if (ret)
-		return ret;
-	
-	AES_set_encrypt_key(mtd_key, sizeof(mtd_key)*8, &EMTD_KEY);
-
-	gctx = CRYPTO_gcm128_new(&EMTD_KEY, (block128_f)AES_encrypt);
-
-	/* TBD: Check return values */
-
-	CRYPTO_gcm128_setiv(gctx, info->oid, sizeof(uuid_t));
-
-	ad = htole32(MTD_LOADER_V1);
-	ret = CRYPTO_gcm128_aad(gctx, (const unsigned char *)&ad, sizeof(ad));
-	if (ret) {
-		gf_log("crypt", GF_LOG_ERROR, " CRYPTO_gcm128_aad failed");
-		CRYPTO_gcm128_release(gctx);
-		return ret;
-	}
-	ret = CRYPTO_gcm128_encrypt(gctx,
-				    get_EMTD_V1(fmt),
-				    get_EMTD_V1(fmt),
-				    SIZE_OF_EMTD_V1);
-	if (ret) {
-		gf_log("crypt", GF_LOG_ERROR, " CRYPTO_gcm128_encrypt failed");
-		CRYPTO_gcm128_release(gctx);
-		return ret;
-	}
-	/*
-	 * set MAC of encrypted part of metadata
-	 */
-	CRYPTO_gcm128_tag(gctx, get_EMTD_V1_MAC(fmt), SIZE_OF_EMTD_V1_MAC);
-	CRYPTO_gcm128_release(gctx);
-	/*
-	 * set the first MAC of non-encrypted part of metadata 
-	 */
-	return create_link_mac_v1(fmt, 0, loc, info, master);
-}
-
-/*
- * Called by fops:
- * ->create();
- * ->link();
- *
- * Pack common and version-specific parts of file's metadata
- * Pre-conditions: @info contains valid object-id.
- */
-int32_t create_format(unsigned char *wire,
-		      loc_t *loc,
-		      struct crypt_inode_info *info,
-		      struct master_cipher_info *master)
-{
-	struct crypt_format *fmt = (struct crypt_format *)wire;
-
-	fmt->loader_id = current_mtd_loader();
-
-	wire += sizeof(struct crypt_format);
-	return mtd_loaders[current_mtd_loader()].create_format(wire, loc,
-							       info, master);
-}
-
-/*
- * Append or overwrite per-link mac of @mac_idx index
- * in accordance with the new pathname
- */
-int32_t appov_link_mac_v1(unsigned char *new,
-			  unsigned char *old,
-			  uint32_t old_size,
-			  int32_t mac_idx,
-			  loc_t *loc,
-			  struct crypt_inode_info *info,
-			  struct master_cipher_info *master,
-			  crypt_local_t *local)
-{
-	memcpy(new, old, old_size);
-	return create_link_mac_v1((struct mtd_format_v1 *)new, mac_idx,
-				  loc, info, master);
-}
-
-/*
- * Cut per-link mac of @mac_idx index
- */
-static int32_t cut_link_mac_v1(unsigned char *new,
-			       unsigned char *old,			       
-			       uint32_t old_size,
-			       int32_t mac_idx,
-			       loc_t *loc,
-			       struct crypt_inode_info *info,
-			       struct master_cipher_info *master,
-			       crypt_local_t *local)
-{
-	memcpy(new,
-	       old,
-	       sizeof(struct mtd_format_v1) + NMTD_8_MAC_SIZE * (mac_idx - 1));
-
-	memcpy(new + sizeof(struct mtd_format_v1) + NMTD_8_MAC_SIZE * (mac_idx - 1),
-	       old + sizeof(struct mtd_format_v1) + NMTD_8_MAC_SIZE * mac_idx,
-	       old_size - (sizeof(struct mtd_format_v1) + NMTD_8_MAC_SIZE * mac_idx));
-	return 0;
-}
-
-int32_t update_format_v1(unsigned char *new,
-			 unsigned char *old,
-				size_t old_len,
-				int32_t mac_idx, /* of old name */
-				mtd_op_t op,
-				loc_t *loc,
-				struct crypt_inode_info *info,
-				struct master_cipher_info *master,
-				crypt_local_t *local)
-{
-	switch (op) {
-	case MTD_APPEND:
-		mac_idx = 1 + (old_len - sizeof(struct mtd_format_v1))/8;
-	case MTD_OVERWRITE:
-		return appov_link_mac_v1(new, old, old_len, mac_idx,
-					 loc, info, master, local);
-	case MTD_CUT:
-		return cut_link_mac_v1(new, old, old_len, mac_idx,
-				       loc, info, master, local);
-	default:
-		gf_log("crypt", GF_LOG_ERROR, "Bad  mtd operation %d", op);
-		return -1;
-	}
-}
-
-/*
- * Called by fops:
- *
- * ->link()
- * ->unlink()
- * ->rename()
- *
- */
-int32_t update_format(unsigned char *new,
-		      unsigned char *old,
-		      size_t old_len,
-		      int32_t mac_idx,
-		      mtd_op_t op,
-		      loc_t *loc,
-		      struct crypt_inode_info *info,
-		      struct master_cipher_info *master,
-		      crypt_local_t *local)
-{
-	if (!new)
-		return 0;
-	memcpy(new, old, sizeof(struct crypt_format));
-
-	old += sizeof(struct crypt_format);
-	new += sizeof(struct crypt_format);
-	old_len -= sizeof(struct crypt_format);
-
-	return mtd_loaders[current_mtd_loader()].update_format(new, old,
-							       old_len,
-							       mac_idx, op,
-							       loc, info,
-							       master, local);
-}
-
-/*
- * Perform preliminary checks of found metadata
- * Return < 0 on errors;
- * Return number of object-id MACs (>= 1) on success
- */
-int32_t check_format_v1(uint32_t len, unsigned char *wire)
-{
-	uint32_t nr_links;
-
-	if (len < sizeof(struct mtd_format_v1)) {
-		gf_log("crypt", GF_LOG_ERROR,
-		       "v1-loader: bad metadata size %d", len);
-		goto error;
-	}
-	len -= sizeof(struct mtd_format_v1);
-	if (len % sizeof(nmtd_8_mac_t)) {
-		gf_log("crypt", GF_LOG_ERROR,
-		       "v1-loader: bad metadata format");
-		goto error;
-	}
-	nr_links = 1 + len / sizeof(nmtd_8_mac_t);
-	if (nr_links > _POSIX_LINK_MAX)
-		goto error;
-	return nr_links;
- error:
-	return EIO;
-}
-
-/*
- * Verify per-link MAC specified by index @idx
- *
- * return:
- * -1 on errors;
- *  0 on failed verification;
- *  1 on successful verification
- */
-static int32_t verify_link_mac_v1(struct mtd_format_v1 *fmt,
-				  uint32_t idx /* index of the mac to verify */,
-				  loc_t *loc,
-				  struct crypt_inode_info *info,
-				  struct master_cipher_info *master)
-{
-	int32_t ret;
-	unsigned char *mac;
-	unsigned char cmac[16];
-
-	mac = get_NMTD_V1_MAC(fmt) + idx * SIZE_OF_NMTD_V1_MAC;
-
-	ret = calc_link_mac_v1(fmt, loc, cmac, info, master);
-	if (ret)
-		return -1;
-	if (memcmp(cmac, mac, SIZE_OF_NMTD_V1_MAC))
-		return 0;
-	return 1;
-}
-
-/*
- * Lookup per-link MAC by pathname.
- *
- * return index of the MAC, if it was found;
- * return < 0 on errors, or if the MAC wasn't found
- */
-static int32_t lookup_link_mac_v1(struct mtd_format_v1 *fmt,
-				  uint32_t nr_macs,
-				  loc_t *loc,
-				  struct crypt_inode_info *info,
-				  struct master_cipher_info *master)
-{
-	int32_t ret;
-	uint32_t idx;
-
-	for (idx = 0; idx < nr_macs; idx++) {
-		ret = verify_link_mac_v1(fmt, idx, loc, info, master);
-		if (ret < 0)
-			return ret;
-		if (ret > 0)
-			return idx;
-	}
-	return -ENOENT;
-}
-
-/*
- * Extract version-specific part of metadata
- */
-static int32_t open_format_v1(unsigned char *wire,
-			      int32_t len,
-			      loc_t *loc,
-			      struct crypt_inode_info *info,
-			      struct master_cipher_info *master,
-			      crypt_local_t *local,
-			      gf_boolean_t load_info)
-{
-	int32_t ret;
-	int32_t num_nmtd_macs;
-	struct mtd_format_v1 *fmt;
-	unsigned char mtd_key[16];
-	AES_KEY EMTD_KEY;
-	GCM128_CONTEXT *gctx;
-	uint32_t ad;
-	emtd_8_mac_t gmac;
-	struct object_cipher_info *object;
-
-	num_nmtd_macs = check_format_v1(len, wire);
-	if (num_nmtd_macs <= 0)
-		return EIO;
-
-	ret = lookup_link_mac_v1((struct mtd_format_v1 *)wire,
-				 num_nmtd_macs, loc, info, master);
-	if (ret < 0) {
-		gf_log("crypt", GF_LOG_ERROR, "NMTD verification failed");
-		return EINVAL;
-	}
-
-	local->mac_idx = ret;
-	if (load_info == _gf_false)
-		/* the case of partial open */
-		return 0;
-
-	fmt = GF_CALLOC(1, len, gf_crypt_mt_mtd);
-	if (!fmt)
-		return ENOMEM;
-	memcpy(fmt, wire, len);
-
-	object = &info->cinfo;
-
-	ret = get_emtd_file_key(info, master, mtd_key);
-	if (ret) {
-		gf_log("crypt", GF_LOG_ERROR, "Can not retrieve metadata key");
-		goto out;
-	}
-	/*
-	 * decrypt encrypted meta-data
-	 */
-	ret = AES_set_encrypt_key(mtd_key, sizeof(mtd_key)*8, &EMTD_KEY);
-	if (ret < 0) {
-		gf_log("crypt", GF_LOG_ERROR, "Can not set encrypt key");
-		ret = EIO;
-		goto out;
-	}
-	gctx = CRYPTO_gcm128_new(&EMTD_KEY, (block128_f)AES_encrypt);
-	if (!gctx) {
-		gf_log("crypt", GF_LOG_ERROR, "Can not alloc gcm context");
-		ret = ENOMEM;
-		goto out;
-	}
-	CRYPTO_gcm128_setiv(gctx, info->oid, sizeof(uuid_t));
-
-	ad = htole32(MTD_LOADER_V1);
-	ret = CRYPTO_gcm128_aad(gctx, (const unsigned char *)&ad, sizeof(ad));
-	if (ret) {
-		gf_log("crypt", GF_LOG_ERROR, " CRYPTO_gcm128_aad failed");
-		CRYPTO_gcm128_release(gctx);
-		ret = EIO;
-		goto out;
-	}
-	ret = CRYPTO_gcm128_decrypt(gctx,
-				    get_EMTD_V1(fmt),
-				    get_EMTD_V1(fmt),
-				    SIZE_OF_EMTD_V1);
-	if (ret) {
-		gf_log("crypt", GF_LOG_ERROR, " CRYPTO_gcm128_decrypt failed");
-		CRYPTO_gcm128_release(gctx);
-		ret = EIO;
-		goto out;
-	}
-	/*
-	 * verify metadata
-	 */
-	CRYPTO_gcm128_tag(gctx, gmac, sizeof(gmac));
-	CRYPTO_gcm128_release(gctx);
-	if (memcmp(gmac, get_EMTD_V1_MAC(fmt), SIZE_OF_EMTD_V1_MAC)) {
-		gf_log("crypt", GF_LOG_ERROR, "EMTD verification failed");
-		ret = EINVAL;
-		goto out;
-	}
-	/*
-	 * load verified metadata to the private part of inode
-	 */
-	info->nr_minor = fmt->minor_id;
-
-	object->o_alg = fmt->alg_id;
-	object->o_dkey_size = fmt->dkey_factor << KEY_FACTOR_BITS;
-	object->o_block_bits = fmt->block_bits;
-	object->o_mode = fmt->mode_id;
-
-	ret = check_file_metadata(info);
- out:
-	GF_FREE(fmt);
-	return ret;
-}
-
-/*
- * perform metadata authentication against @loc->path;
- * extract crypt-specific attribtes and populate @info
- * with them (optional)
- */
-int32_t open_format(unsigned char *str,
-		    int32_t len,
-		    loc_t *loc,
-		    struct crypt_inode_info *info,
-		    struct master_cipher_info *master,
-		    crypt_local_t *local,
-		    gf_boolean_t load_info)
-{
-	struct crypt_format *fmt;
-	if (len < sizeof(*fmt)) {
-		gf_log("crypt", GF_LOG_ERROR, "Bad core format");
-		return EIO;
-	}
-	fmt = (struct crypt_format *)str;
-
-	if (fmt->loader_id >= LAST_MTD_LOADER) {
-		gf_log("crypt", GF_LOG_ERROR,
-		       "Unsupported loader id %d", fmt->loader_id);
-		return EINVAL;
-	}
-	str += sizeof(*fmt);
-	len -= sizeof(*fmt);
-
-	return mtd_loaders[fmt->loader_id].open_format(str,
-						       len,
-						       loc,
-						       info,
-						       master,
-						       local,
-						       load_info);
-}
-
-struct crypt_mtd_loader mtd_loaders [LAST_MTD_LOADER] = {
-	[MTD_LOADER_V1] = 
-	{.format_size = format_size_v1,
-	 .create_format = create_format_v1,
-	 .open_format = open_format_v1,
-	 .update_format = update_format_v1
-	}
-};
-
-/*
-  Local variables:
-  c-indentation-style: "K&R"
-  mode-name: "LC"
-  c-basic-offset: 8
-  tab-width: 8
-  fill-column: 80
-  scroll-step: 1
-  End:
-*/
diff --git a/xlators/encryption/crypt/src/metadata.h b/xlators/encryption/crypt/src/metadata.h
deleted file mode 100644
index a92f149ef50..00000000000
--- a/xlators/encryption/crypt/src/metadata.h
+++ /dev/null
@@ -1,74 +0,0 @@
-/*
-  Copyright (c) 2008-2013 Red Hat, Inc. <http://www.redhat.com>
-  This file is part of GlusterFS.
-
-  This file is licensed to you under your choice of the GNU Lesser
-  General Public License, version 3 or any later version (LGPLv3 or
-  later), or the GNU General Public License, version 2 (GPLv2), in all
-  cases as published by the Free Software Foundation.
-*/
-
-#ifndef __METADATA_H__
-#define __METADATA_H__
-
-#define NMTD_8_MAC_SIZE   (8)
-#define EMTD_8_MAC_SIZE   (8)
-
-typedef uint8_t nmtd_8_mac_t[NMTD_8_MAC_SIZE];
-typedef uint8_t emtd_8_mac_t[EMTD_8_MAC_SIZE] ;
-
-/*
- * Version "v1" of file's metadata.
- * Metadata of this version has 4 components:
- * 
- * 1) EMTD (Encrypted part of MeTaData);
- * 2) NMTD (Non-encrypted part of MeTaData);
- * 3) EMTD_MAC; (EMTD Message Authentication Code);
- * 4) Array of per-link NMTD MACs (for every (hard)link it includes
- *    exactly one MAC)
- */
-struct mtd_format_v1 {
-	/* EMTD, encrypted part of meta-data */
-	uint8_t            alg_id;  /* cipher algorithm id (only AES for now) */
-        uint8_t            mode_id; /* cipher mode id; (only XTS for now) */
-	uint8_t            block_bits; /* encoded block size */
-	uint8_t            minor_id; /* client translator id */
-	uint8_t            dkey_factor; /* encoded size of the data key */
-	/* MACs */
-	emtd_8_mac_t       gmac; /* MAC of the encrypted meta-data, 8 bytes */
-	nmtd_8_mac_t       omac; /* per-link MACs of the non-encrypted
-				  * meta-data: at least one such MAC is always
-				  * present */
-} __attribute__((packed));
-
-/*
- * NMTD, the non-encrypted part of metadata of version "v1"
- * is file's gfid, which is generated on trusted machines.
- */
-#define SIZE_OF_NMTD_V1 (sizeof(uuid_t))
-#define SIZE_OF_EMTD_V1 (offsetof(struct mtd_format_v1, gmac) -		\
-			 offsetof(struct mtd_format_v1, alg_id))
-#define SIZE_OF_NMTD_V1_MAC (NMTD_8_MAC_SIZE)
-#define SIZE_OF_EMTD_V1_MAC (EMTD_8_MAC_SIZE)
-
-static inline unsigned char *get_EMTD_V1(struct mtd_format_v1 *format)
-{
-        return &format->alg_id;
-}
-
-static inline unsigned char *get_NMTD_V1(struct crypt_inode_info *info)
-{
-        return info->oid;
-}
-
-static inline unsigned char *get_EMTD_V1_MAC(struct mtd_format_v1 *format)
-{
-        return format->gmac;
-}
-
-static inline unsigned char *get_NMTD_V1_MAC(struct mtd_format_v1 *format)
-{
-        return format->omac;
-}
-
-#endif /* __METADATA_H__ */
diff --git a/xlators/encryption/rot-13/src/rot-13.c b/xlators/encryption/rot-13/src/rot-13.c
deleted file mode 100644
index 1bcfe0192f6..00000000000
--- a/xlators/encryption/rot-13/src/rot-13.c
+++ /dev/null
@@ -1,201 +0,0 @@
-/*
-   Copyright (c) 2006-2012 Red Hat, Inc. <http://www.redhat.com>
-   This file is part of GlusterFS.
-
-   This file is licensed to you under your choice of the GNU Lesser
-   General Public License, version 3 or any later version (LGPLv3 or
-   later), or the GNU General Public License, version 2 (GPLv2), in all
-   cases as published by the Free Software Foundation.
-*/
-#include <ctype.h>
-#include <sys/uio.h>
-
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
-
-#include "glusterfs.h"
-#include "xlator.h"
-#include "logging.h"
-
-#include "rot-13.h"
-
-/*
- * This is a rot13 ``encryption'' xlator. It rot13's data when
- * writing to disk and rot13's it back when reading it.
- * This xlator is meant as an example, NOT FOR PRODUCTION
- * USE ;) (hence no error-checking)
- */
-
-void
-rot13 (char *buf, int len)
-{
-	int i;
-	for (i = 0; i < len; i++) {
-		if (buf[i] >= 'a' && buf[i] <= 'z')
-			buf[i] = 'a' + ((buf[i] - 'a' + 13) % 26);
-		else if (buf[i] >= 'A' && buf[i] <= 'Z')
-			buf[i] = 'A' + ((buf[i] - 'A' + 13) % 26);
-	}
-}
-
-void
-rot13_iovec (struct iovec *vector, int count)
-{
-	int i;
-	for (i = 0; i < count; i++) {
-		rot13 (vector[i].iov_base, vector[i].iov_len);
-	}
-}
-
-int32_t
-rot13_readv_cbk (call_frame_t *frame,
-                 void *cookie,
-                 xlator_t *this,
-                 int32_t op_ret,
-                 int32_t op_errno,
-                 struct iovec *vector,
-                 int32_t count,
-		 struct iatt *stbuf,
-                 struct iobref *iobref, dict_t *xdata)
-{
-	rot_13_private_t *priv = (rot_13_private_t *)this->private;
-
-	if (priv->decrypt_read)
-		rot13_iovec (vector, count);
-
-	STACK_UNWIND_STRICT (readv, frame, op_ret, op_errno, vector, count,
-                             stbuf, iobref, xdata);
-	return 0;
-}
-
-int32_t
-rot13_readv (call_frame_t *frame,
-             xlator_t *this,
-             fd_t *fd,
-             size_t size,
-             off_t offset, uint32_t flags, dict_t *xdata)
-{
-	STACK_WIND (frame,
-		    rot13_readv_cbk,
-		    FIRST_CHILD (this),
-		    FIRST_CHILD (this)->fops->readv,
-		    fd, size, offset, flags, xdata);
-	return 0;
-}
-
-int32_t
-rot13_writev_cbk (call_frame_t *frame,
-                  void *cookie,
-                  xlator_t *this,
-                  int32_t op_ret,
-                  int32_t op_errno,
-                  struct iatt *prebuf,
-		  struct iatt *postbuf, dict_t *xdata)
-{
-	STACK_UNWIND_STRICT (writev, frame, op_ret, op_errno, prebuf, postbuf,
-                             xdata);
-	return 0;
-}
-
-int32_t
-rot13_writev (call_frame_t *frame,
-              xlator_t *this,
-              fd_t *fd,
-              struct iovec *vector,
-              int32_t count,
-              off_t offset, uint32_t flags,
-              struct iobref *iobref, dict_t *xdata)
-{
-	rot_13_private_t *priv = (rot_13_private_t *)this->private;
-	if (priv->encrypt_write)
-		rot13_iovec (vector, count);
-
-	STACK_WIND (frame,
-		    rot13_writev_cbk,
-		    FIRST_CHILD (this),
-		    FIRST_CHILD (this)->fops->writev,
-		    fd, vector, count, offset, flags,
-                    iobref, xdata);
-	return 0;
-}
-
-int32_t
-init (xlator_t *this)
-{
-	data_t *data = NULL;
-	rot_13_private_t *priv = NULL;
-
-	if (!this->children || this->children->next) {
-		gf_log ("rot13", GF_LOG_ERROR,
-			"FATAL: rot13 should have exactly one child");
-		return -1;
-	}
-
-	if (!this->parents) {
-		gf_log (this->name, GF_LOG_WARNING,
-			"dangling volume. check volfile ");
-	}
-
-	priv = GF_CALLOC (sizeof (rot_13_private_t), 1, 0);
-        if (!priv)
-                return -1;
-
-	priv->decrypt_read = 1;
-	priv->encrypt_write = 1;
-
-	data = dict_get (this->options, "encrypt-write");
-	if (data) {
-		if (gf_string2boolean (data->data, &priv->encrypt_write) == -1) {
-			gf_log (this->name, GF_LOG_ERROR,
-				"encrypt-write takes only boolean options");
-			GF_FREE (priv);
-			return -1;
-		}
-	}
-
-	data = dict_get (this->options, "decrypt-read");
-	if (data) {
-		if (gf_string2boolean (data->data, &priv->decrypt_read) == -1) {
-			gf_log (this->name, GF_LOG_ERROR,
-				"decrypt-read takes only boolean options");
-			GF_FREE (priv);
-			return -1;
-		}
-	}
-
-	this->private = priv;
-	gf_log ("rot13", GF_LOG_DEBUG, "rot13 xlator loaded");
-	return 0;
-}
-
-void
-fini (xlator_t *this)
-{
-	rot_13_private_t *priv = this->private;
-
-        if (!priv)
-                return;
-        this->private = NULL;
-	GF_FREE (priv);
-
-	return;
-}
-
-struct xlator_fops fops = {
-	.readv        = rot13_readv,
-	.writev       = rot13_writev
-};
-
-struct xlator_cbks cbks;
-
-struct volume_options options[] = {
-	{ .key  = {"encrypt-write"},
-	  .type = GF_OPTION_TYPE_BOOL
-	},
-	{ .key  = {"decrypt-read"},
-	  .type = GF_OPTION_TYPE_BOOL
-	},
-	{ .key  = {NULL} },
-};
diff --git a/xlators/features/Makefile.am b/xlators/features/Makefile.am
index 7e5783f4f30..c57897f11ea 100644
--- a/xlators/features/Makefile.am
+++ b/xlators/features/Makefile.am
@@ -1,5 +1,14 @@
-SUBDIRS = locks quota read-only mac-compat quiesce marker index barrier arbiter\
-          protect compress changelog changetimerecorder ganesha gfid-access $(GLUPY_SUBDIR) qemu-block \
-          upcall snapview-client snapview-server trash shard bit-rot #path-converter # filter
+if BUILD_CLOUDSYNC
+  CLOUDSYNC_DIR = cloudsync
+endif
+
+if BUILD_METADISP
+  METADISP_DIR = metadisp
+endif
+
+SUBDIRS = locks quota read-only quiesce marker index barrier arbiter upcall \
+	compress changelog gfid-access snapview-client snapview-server trash \
+	shard bit-rot leases selinux sdfs namespace $(CLOUDSYNC_DIR) thin-arbiter \
+	utime $(METADISP_DIR)
 
 CLEANFILES =
diff --git a/xlators/features/arbiter/src/Makefile.am b/xlators/features/arbiter/src/Makefile.am
index 090095d8860..badc42f37be 100644
--- a/xlators/features/arbiter/src/Makefile.am
+++ b/xlators/features/arbiter/src/Makefile.am
@@ -1,14 +1,18 @@
+if WITH_SERVER
 xlator_LTLIBRARIES = arbiter.la
+endif
+
 xlatordir = $(libdir)/glusterfs/$(PACKAGE_VERSION)/xlator/features
 
-arbiter_la_LDFLAGS = -module -avoid-version
+arbiter_la_LDFLAGS = -module $(GF_XLATOR_DEFAULT_LDFLAGS)
 
 arbiter_la_SOURCES = arbiter.c
 arbiter_la_LIBADD = $(top_builddir)/libglusterfs/src/libglusterfs.la
 
 noinst_HEADERS = arbiter.h arbiter-mem-types.h
 
-AM_CPPFLAGS = $(GF_CPPFLAGS) -I$(top_srcdir)/libglusterfs/src
+AM_CPPFLAGS = $(GF_CPPFLAGS) -I$(top_srcdir)/libglusterfs/src \
+	-I$(top_srcdir)/rpc/xdr/src -I$(top_builddir)/rpc/xdr/src
 
 AM_CFLAGS = -Wall $(GF_CFLAGS)
 
diff --git a/xlators/features/arbiter/src/arbiter-mem-types.h b/xlators/features/arbiter/src/arbiter-mem-types.h
index 200b59de695..05d18374c46 100644
--- a/xlators/features/arbiter/src/arbiter-mem-types.h
+++ b/xlators/features/arbiter/src/arbiter-mem-types.h
@@ -9,11 +9,10 @@
 
 #ifndef __ARBITER_MEM_TYPES_H__
 #define __ARBITER_MEM_TYPES_H__
-#include "mem-types.h"
+#include <glusterfs/mem-types.h>
 
 typedef enum gf_arbiter_mem_types_ {
-        gf_arbiter_mt_inode_ctx_t =  gf_common_mt_end + 1,
-        gf_arbiter_mt_iatt,
-        gf_arbiter_mt_end
+    gf_arbiter_mt_inode_ctx_t = gf_common_mt_end + 1,
+    gf_arbiter_mt_end
 } gf_arbiter_mem_types_t;
 #endif
diff --git a/xlators/features/arbiter/src/arbiter.c b/xlators/features/arbiter/src/arbiter.c
index 87145da5680..83a97e3354b 100644
--- a/xlators/features/arbiter/src/arbiter.c
+++ b/xlators/features/arbiter/src/arbiter.c
@@ -8,317 +8,373 @@
   cases as published by the Free Software Foundation.
 */
 
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
-
 #include "arbiter.h"
 #include "arbiter-mem-types.h"
-#include "glusterfs.h"
-#include "xlator.h"
-#include "logging.h"
-
-void
-arbiter_inode_ctx_destroy (arbiter_inode_ctx_t *ctx)
-{
-        if (!ctx)
-                return;
-        GF_FREE (ctx->iattbuf);
-        GF_FREE (ctx);
-}
+#include <glusterfs/glusterfs.h>
+#include <glusterfs/xlator.h>
+#include <glusterfs/logging.h>
 
 static arbiter_inode_ctx_t *
-__arbiter_inode_ctx_get (inode_t *inode, xlator_t *this)
+__arbiter_inode_ctx_get(inode_t *inode, xlator_t *this)
 {
-
-        arbiter_inode_ctx_t *ctx = NULL;
-        int ret = 0;
-        uint64_t ctx_addr = 0;
-
-        ret = __inode_ctx_get (inode, this, &ctx_addr);
-        if (ret == 0) {
-                ctx = (arbiter_inode_ctx_t *) (long) ctx_addr;
-                goto out;
-        }
-
-        ctx = GF_CALLOC (1, sizeof (*ctx), gf_arbiter_mt_inode_ctx_t);
-        if (!ctx)
-                goto fail;
-        ctx->iattbuf = GF_CALLOC (1, sizeof (*ctx->iattbuf),
-                                  gf_arbiter_mt_iatt);
-        if (!ctx->iattbuf)
-                goto fail;
-        ret = __inode_ctx_put (inode, this, (uint64_t)ctx);
-        if (ret) {
-                gf_log_callingfn (this->name, GF_LOG_ERROR, "failed to "
-                                  "set the inode ctx (%s)",
-                                  uuid_utoa (inode->gfid));
-                goto fail;
-        }
+    arbiter_inode_ctx_t *ctx = NULL;
+    int ret = 0;
+    uint64_t ctx_addr = 0;
+
+    ret = __inode_ctx_get(inode, this, &ctx_addr);
+    if (ret == 0) {
+        ctx = (arbiter_inode_ctx_t *)(long)ctx_addr;
+        goto out;
+    }
+
+    ctx = GF_CALLOC(1, sizeof(*ctx), gf_arbiter_mt_inode_ctx_t);
+    if (!ctx)
+        goto out;
+
+    ret = __inode_ctx_put(inode, this, (uint64_t)(uintptr_t)ctx);
+    if (ret) {
+        GF_FREE(ctx);
+        ctx = NULL;
+        gf_log_callingfn(this->name, GF_LOG_ERROR,
+                         "failed to "
+                         "set the inode ctx (%s)",
+                         uuid_utoa(inode->gfid));
+    }
 out:
-        return ctx;
-fail:
-        arbiter_inode_ctx_destroy (ctx);
-        return NULL;
+    return ctx;
 }
 
 static arbiter_inode_ctx_t *
-arbiter_inode_ctx_get (inode_t *inode, xlator_t *this)
+arbiter_inode_ctx_get(inode_t *inode, xlator_t *this)
 {
-        arbiter_inode_ctx_t *ctx = NULL;
-
-        LOCK(&inode->lock);
-        {
-                ctx = __arbiter_inode_ctx_get (inode, this);
-        }
-        UNLOCK(&inode->lock);
-        return ctx;
+    arbiter_inode_ctx_t *ctx = NULL;
+
+    LOCK(&inode->lock);
+    {
+        ctx = __arbiter_inode_ctx_get(inode, this);
+    }
+    UNLOCK(&inode->lock);
+    return ctx;
 }
 
 int32_t
-arbiter_lookup_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                    int32_t op_ret, int32_t op_errno, inode_t *inode,
-                    struct iatt *buf, dict_t *xdata, struct iatt *postparent)
+arbiter_lookup_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                   int32_t op_ret, int32_t op_errno, inode_t *inode,
+                   struct iatt *buf, dict_t *xdata, struct iatt *postparent)
 {
-        arbiter_inode_ctx_t *ctx = NULL;
-
-        if (op_ret != 0)
-                goto unwind;
-        ctx = arbiter_inode_ctx_get (inode, this);
-        if (!ctx) {
-                op_ret = -1;
-                op_errno = ENOMEM;
-                goto unwind;
-        }
-        memcpy (ctx->iattbuf, buf, sizeof (*ctx->iattbuf));
+    arbiter_inode_ctx_t *ctx = NULL;
+
+    if (op_ret != 0)
+        goto unwind;
+    ctx = arbiter_inode_ctx_get(inode, this);
+    if (!ctx) {
+        op_ret = -1;
+        op_errno = ENOMEM;
+        goto unwind;
+    }
+    memcpy(&ctx->iattbuf, buf, sizeof(ctx->iattbuf));
 
 unwind:
-        STACK_UNWIND_STRICT (lookup, frame, op_ret, op_errno, inode, buf,
-                             xdata, postparent);
-        return 0;
+    STACK_UNWIND_STRICT(lookup, frame, op_ret, op_errno, inode, buf, xdata,
+                        postparent);
+    return 0;
 }
 
 int32_t
-arbiter_lookup (call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xdata)
+arbiter_lookup(call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xdata)
 {
-        STACK_WIND (frame, arbiter_lookup_cbk, FIRST_CHILD(this),
-                         FIRST_CHILD(this)->fops->lookup, loc, xdata);
-        return 0;
+    STACK_WIND(frame, arbiter_lookup_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->lookup, loc, xdata);
+    return 0;
 }
 
 int32_t
-arbiter_readv (call_frame_t *frame,  xlator_t *this, fd_t *fd, size_t size,
-               off_t offset, uint32_t flags, dict_t *xdata)
+arbiter_truncate(call_frame_t *frame, xlator_t *this, loc_t *loc, off_t offset,
+                 dict_t *xdata)
 {
-        STACK_UNWIND_STRICT (readv, frame, -1, ENOTCONN, NULL, 0, NULL, NULL,
-                             NULL);
-        return 0;
+    arbiter_inode_ctx_t *ctx = NULL;
+    struct iatt *buf = NULL;
+    int32_t op_ret = 0;
+    int32_t op_errno = 0;
+
+    ctx = arbiter_inode_ctx_get(loc->inode, this);
+    if (!ctx) {
+        op_ret = -1;
+        op_errno = ENOMEM;
+        goto unwind;
+    }
+    buf = &ctx->iattbuf;
+unwind:
+    STACK_UNWIND_STRICT(truncate, frame, op_ret, op_errno, buf, buf, NULL);
+    return 0;
 }
 
 int32_t
-arbiter_truncate (call_frame_t *frame, xlator_t *this, loc_t *loc, off_t offset,
+arbiter_ftruncate(call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
                   dict_t *xdata)
+
 {
-        arbiter_inode_ctx_t *ctx      = NULL;
-        struct iatt         *buf      = NULL;
-        int32_t              op_ret   = 0;
-        int32_t              op_errno = 0;
-
-        ctx = arbiter_inode_ctx_get (loc->inode, this);
-        if (!ctx) {
-                op_ret = -1;
-                op_errno = ENOMEM;
-                goto unwind;
-        }
-        buf = ctx->iattbuf;
+    arbiter_inode_ctx_t *ctx = NULL;
+    struct iatt *buf = NULL;
+    int32_t op_ret = 0;
+    int32_t op_errno = 0;
+
+    ctx = arbiter_inode_ctx_get(fd->inode, this);
+    if (!ctx) {
+        op_ret = -1;
+        op_errno = ENOMEM;
+        goto unwind;
+    }
+    buf = &ctx->iattbuf;
 unwind:
-        STACK_UNWIND_STRICT (truncate, frame, op_ret, op_errno, buf, buf,
-                             xdata);
-        return 0;
+    STACK_UNWIND_STRICT(ftruncate, frame, op_ret, op_errno, buf, buf, NULL);
+    return 0;
 }
 
-int32_t
-arbiter_ftruncate (call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
-                   dict_t *xdata)
-
+dict_t *
+arbiter_fill_writev_xdata(fd_t *fd, dict_t *xdata, xlator_t *this)
 {
-        arbiter_inode_ctx_t *ctx      = NULL;
-        struct iatt         *buf      = NULL;
-        int32_t              op_ret   = 0;
-        int32_t              op_errno = 0;
-
-        ctx = arbiter_inode_ctx_get (fd->inode, this);
-        if (!ctx) {
-                op_ret = -1;
-                op_errno = ENOMEM;
-                goto unwind;
+    dict_t *rsp_xdata = NULL;
+    int32_t ret = 0;
+    int is_append = 1;
+
+    if (!fd || !fd->inode || gf_uuid_is_null(fd->inode->gfid)) {
+        goto out;
+    }
+
+    if (!xdata)
+        goto out;
+
+    rsp_xdata = dict_new();
+    if (!rsp_xdata)
+        goto out;
+
+    if (dict_get(xdata, GLUSTERFS_OPEN_FD_COUNT)) {
+        ret = dict_set_uint32(rsp_xdata, GLUSTERFS_OPEN_FD_COUNT,
+                              fd->inode->fd_count);
+        if (ret < 0) {
+            gf_msg_debug(this->name, 0,
+                         "Failed to set dict value"
+                         " for GLUSTERFS_OPEN_FD_COUNT");
         }
-        buf = ctx->iattbuf;
-unwind:
-        STACK_UNWIND_STRICT (ftruncate, frame, op_ret, op_errno, buf, buf,
-                             xdata);
-        return 0;
+    }
+    if (dict_get(xdata, GLUSTERFS_WRITE_IS_APPEND)) {
+        ret = dict_set_uint32(rsp_xdata, GLUSTERFS_WRITE_IS_APPEND, is_append);
+        if (ret < 0) {
+            gf_msg_debug(this->name, 0,
+                         "Failed to set dict value"
+                         " for GLUSTERFS_WRITE_IS_APPEND");
+        }
+    }
+out:
+    return rsp_xdata;
 }
 
 int32_t
-arbiter_writev (call_frame_t *frame, xlator_t *this, fd_t *fd,
-                struct iovec *vector, int32_t count, off_t off, uint32_t flags,
-                struct iobref *iobref, dict_t *xdata)
+arbiter_writev(call_frame_t *frame, xlator_t *this, fd_t *fd,
+               struct iovec *vector, int32_t count, off_t off, uint32_t flags,
+               struct iobref *iobref, dict_t *xdata)
 {
-        arbiter_inode_ctx_t *ctx      = NULL;
-        struct iatt         *buf      = NULL;
-        int                  op_ret   = 0;
-        int                  op_errno = 0;
-
-        ctx = arbiter_inode_ctx_get (fd->inode, this);
-        if (!ctx) {
-                op_ret = -1;
-                op_errno = ENOMEM;
-                goto unwind;
-        }
-        buf = ctx->iattbuf;
-        op_ret = iov_length (vector, count);
+    arbiter_inode_ctx_t *ctx = NULL;
+    struct iatt *buf = NULL;
+    dict_t *rsp_xdata = NULL;
+    int op_ret = 0;
+    int op_errno = 0;
+
+    ctx = arbiter_inode_ctx_get(fd->inode, this);
+    if (!ctx) {
+        op_ret = -1;
+        op_errno = ENOMEM;
+        goto unwind;
+    }
+    buf = &ctx->iattbuf;
+    op_ret = iov_length(vector, count);
+    rsp_xdata = arbiter_fill_writev_xdata(fd, xdata, this);
 unwind:
-        STACK_UNWIND_STRICT (writev, frame, op_ret, op_errno, buf, buf, xdata);
-        return 0;
+    STACK_UNWIND_STRICT(writev, frame, op_ret, op_errno, buf, buf, rsp_xdata);
+    if (rsp_xdata)
+        dict_unref(rsp_xdata);
+    return 0;
 }
 
 int32_t
-arbiter_fallocate (call_frame_t *frame, xlator_t *this, fd_t *fd,
+arbiter_fallocate(call_frame_t *frame, xlator_t *this, fd_t *fd,
                   int32_t keep_size, off_t offset, size_t len, dict_t *xdata)
 {
-        arbiter_inode_ctx_t *ctx      = NULL;
-        struct iatt         *buf      = NULL;
-        int                  op_ret   = 0;
-        int                  op_errno = 0;
-
-        ctx = arbiter_inode_ctx_get (fd->inode, this);
-        if (!ctx) {
-                op_ret = -1;
-                op_errno = ENOMEM;
-                goto unwind;
-        }
-        buf = ctx->iattbuf;
+    arbiter_inode_ctx_t *ctx = NULL;
+    struct iatt *buf = NULL;
+    int op_ret = 0;
+    int op_errno = 0;
+
+    ctx = arbiter_inode_ctx_get(fd->inode, this);
+    if (!ctx) {
+        op_ret = -1;
+        op_errno = ENOMEM;
+        goto unwind;
+    }
+    buf = &ctx->iattbuf;
 unwind:
-        STACK_UNWIND_STRICT(fallocate, frame, op_ret, op_errno, buf, buf,
-                            xdata);
-        return 0;
+    STACK_UNWIND_STRICT(fallocate, frame, op_ret, op_errno, buf, buf, NULL);
+    return 0;
 }
 
 int32_t
-arbiter_discard (call_frame_t *frame, xlator_t *this, fd_t *fd,
-                off_t offset, size_t len, dict_t *xdata)
+arbiter_discard(call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
+                size_t len, dict_t *xdata)
 {
-        arbiter_inode_ctx_t *ctx      = NULL;
-        struct iatt         *buf      = NULL;
-        int                  op_ret   = 0;
-        int                  op_errno = 0;
-
-        ctx = arbiter_inode_ctx_get (fd->inode, this);
-        if (!ctx) {
-                op_ret = -1;
-                op_errno = ENOMEM;
-                goto unwind;
-        }
-        buf = ctx->iattbuf;
+    arbiter_inode_ctx_t *ctx = NULL;
+    struct iatt *buf = NULL;
+    int op_ret = 0;
+    int op_errno = 0;
+
+    ctx = arbiter_inode_ctx_get(fd->inode, this);
+    if (!ctx) {
+        op_ret = -1;
+        op_errno = ENOMEM;
+        goto unwind;
+    }
+    buf = &ctx->iattbuf;
 unwind:
-        STACK_UNWIND_STRICT(discard, frame, op_ret, op_errno, buf, buf, xdata);
-        return 0;
+    STACK_UNWIND_STRICT(discard, frame, op_ret, op_errno, buf, buf, NULL);
+    return 0;
 }
 
 int32_t
-arbiter_zerofill (call_frame_t *frame, xlator_t *this, fd_t *fd,
-                  off_t offset, off_t len, dict_t *xdata)
+arbiter_zerofill(call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
+                 off_t len, dict_t *xdata)
 {
-        arbiter_inode_ctx_t *ctx      = NULL;
-        struct iatt         *buf      = NULL;
-        int                  op_ret   = 0;
-        int                  op_errno = 0;
-
-        ctx = arbiter_inode_ctx_get (fd->inode, this);
-        if (!ctx) {
-                op_ret = -1;
-                op_errno = ENOMEM;
-                goto unwind;
-        }
-        buf = ctx->iattbuf;
+    arbiter_inode_ctx_t *ctx = NULL;
+    struct iatt *buf = NULL;
+    int op_ret = 0;
+    int op_errno = 0;
+
+    ctx = arbiter_inode_ctx_get(fd->inode, this);
+    if (!ctx) {
+        op_ret = -1;
+        op_errno = ENOMEM;
+        goto unwind;
+    }
+    buf = &ctx->iattbuf;
 unwind:
-        STACK_UNWIND_STRICT(zerofill, frame, op_ret, op_errno, buf, buf, xdata);
-        return 0;
+    STACK_UNWIND_STRICT(zerofill, frame, op_ret, op_errno, buf, buf, NULL);
+    return 0;
 }
 
-int32_t
-mem_acct_init (xlator_t *this)
+static int32_t
+arbiter_readv(call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
+              off_t offset, uint32_t flags, dict_t *xdata)
+{
+    STACK_UNWIND_STRICT(readv, frame, -1, ENOSYS, NULL, 0, NULL, NULL, NULL);
+    return 0;
+}
+
+static int32_t
+arbiter_seek(call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
+             gf_seek_what_t what, dict_t *xdata)
 {
-        int ret = -1;
+    STACK_UNWIND_STRICT(seek, frame, -1, ENOSYS, 0, xdata);
+    return 0;
+}
 
-        ret = xlator_mem_acct_init (this, gf_arbiter_mt_end + 1);
-        if (ret)
-                gf_log (this->name, GF_LOG_ERROR, "Memory accounting "
-                        "initialization failed.");
-        return ret;
+int32_t
+mem_acct_init(xlator_t *this)
+{
+    int ret = -1;
+
+    ret = xlator_mem_acct_init(this, gf_arbiter_mt_end + 1);
+    if (ret)
+        gf_log(this->name, GF_LOG_ERROR,
+               "Memory accounting "
+               "initialization failed.");
+    return ret;
 }
 
 int
-reconfigure (xlator_t *this, dict_t *options)
+reconfigure(xlator_t *this, dict_t *options)
 {
-
-        return 0;
+    return 0;
 }
 
 int
-arbiter_forget (xlator_t *this, inode_t *inode)
+arbiter_forget(xlator_t *this, inode_t *inode)
 {
-        arbiter_inode_ctx_t *ctx = NULL;
-        uint64_t ctx_addr = 0;
-
-        inode_ctx_del (inode, this, &ctx_addr);
-        if (!ctx_addr)
-                return 0;
-        ctx = (arbiter_inode_ctx_t *) (long) ctx_addr;
-        GF_FREE (ctx);
+    arbiter_inode_ctx_t *ctx = NULL;
+    uint64_t ctx_addr = 0;
+
+    inode_ctx_del(inode, this, &ctx_addr);
+    if (!ctx_addr)
         return 0;
+    ctx = (arbiter_inode_ctx_t *)(long)ctx_addr;
+    GF_FREE(ctx);
+    return 0;
 }
 
 int32_t
-init (xlator_t *this)
+init(xlator_t *this)
 {
+    if (!this->children || this->children->next) {
+        gf_log(this->name, GF_LOG_ERROR,
+               "'arbiter' not configured with exactly one child");
+        return -1;
+    }
 
-        if (!this->children || this->children->next) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "'arbiter' not configured with exactly one child");
-                return -1;
-        }
-
-        if (!this->parents)
-                gf_log (this->name, GF_LOG_ERROR,
-                        "dangling volume. check volfile ");
+    if (!this->parents)
+        gf_log(this->name, GF_LOG_ERROR, "dangling volume. check volfile ");
 
-        return 0;
+    return 0;
 }
 
 void
-fini (xlator_t *this)
+fini(xlator_t *this)
 {
-        return;
+    return;
 }
 
 struct xlator_fops fops = {
-        .lookup = arbiter_lookup,
-        .readv  = arbiter_readv,
-        .truncate = arbiter_truncate,
-        .writev = arbiter_writev,
-        .ftruncate = arbiter_ftruncate,
-        .fallocate = arbiter_fallocate,
-        .discard = arbiter_discard,
-        .zerofill = arbiter_zerofill,
+    .lookup = arbiter_lookup,
+
+    /* Return success for these inode write FOPS without winding it down to
+     * posix; this is needed for AFR write transaction logic to work.*/
+    .truncate = arbiter_truncate,
+    .writev = arbiter_writev,
+    .ftruncate = arbiter_ftruncate,
+    .fallocate = arbiter_fallocate,
+    .discard = arbiter_discard,
+    .zerofill = arbiter_zerofill,
+
+    /* AFR is not expected to wind these inode read FOPS initiated by the
+     * application to the arbiter brick. But in case a bug causes them
+     * to be called, we return ENOSYS. */
+    .readv = arbiter_readv,
+    .seek = arbiter_seek,
+
+    /* The following inode read FOPS initiated by the application are not
+     * wound by AFR either but internal logic like  shd, glfsheal and
+     * client side healing in AFR will send them for selfheal/ inode refresh
+     * operations etc.,so we need to wind them down to posix:
+     *
+     * (f)stat, readdir(p), readlink, (f)getxattr.*/
+
+    /* All other FOPs not listed here are safe to be wound down to posix.*/
 };
 
 struct xlator_cbks cbks = {
-        .forget = arbiter_forget,
+    .forget = arbiter_forget,
 };
 
 struct volume_options options[] = {
-        { .key  = {NULL} },
+    {.key = {NULL}},
+};
+
+xlator_api_t xlator_api = {
+    .init = init,
+    .fini = fini,
+    .reconfigure = reconfigure,
+    .mem_acct_init = mem_acct_init,
+    .op_version = {1}, /* Present from the initial version */
+    .fops = &fops,
+    .cbks = &cbks,
+    .options = options,
+    .identifier = "arbiter",
+    .category = GF_MAINTAINED,
 };
diff --git a/xlators/features/arbiter/src/arbiter.h b/xlators/features/arbiter/src/arbiter.h
index 69ce9cb4fa3..546db7b751a 100644
--- a/xlators/features/arbiter/src/arbiter.h
+++ b/xlators/features/arbiter/src/arbiter.h
@@ -11,16 +11,11 @@
 #ifndef _ARBITER_H
 #define _ARBITER_H
 
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
-
-#include "locking.h"
-#include "common-utils.h"
+#include <glusterfs/locking.h>
+#include <glusterfs/common-utils.h>
 
 typedef struct arbiter_inode_ctx_ {
-        struct iatt *iattbuf;
+    struct iatt iattbuf;
 } arbiter_inode_ctx_t;
 
 #endif /* _ARBITER_H */
diff --git a/xlators/features/barrier/src/Makefile.am b/xlators/features/barrier/src/Makefile.am
index 8859be328d3..25099bc56e5 100644
--- a/xlators/features/barrier/src/Makefile.am
+++ b/xlators/features/barrier/src/Makefile.am
@@ -1,7 +1,7 @@
 xlator_LTLIBRARIES = barrier.la
 xlatordir = $(libdir)/glusterfs/$(PACKAGE_VERSION)/xlator/features
 
-barrier_la_LDFLAGS = -module -avoid-version
+barrier_la_LDFLAGS = -module $(GF_XLATOR_DEFAULT_LDFLAGS)
 
 barrier_la_SOURCES = barrier.c
 
@@ -9,7 +9,8 @@ barrier_la_LIBADD = $(top_builddir)/libglusterfs/src/libglusterfs.la
 
 noinst_HEADERS = barrier.h barrier-mem-types.h
 
-AM_CPPFLAGS = $(GF_CPPFLAGS) -I$(top_srcdir)/libglusterfs/src
+AM_CPPFLAGS = $(GF_CPPFLAGS) -I$(top_srcdir)/libglusterfs/src \
+	-I$(top_srcdir)/rpc/xdr/src -I$(top_builddir)/rpc/xdr/src
 
 AM_CFLAGS = -Wall $(GF_CFLAGS)
 
diff --git a/xlators/features/barrier/src/barrier-mem-types.h b/xlators/features/barrier/src/barrier-mem-types.h
index 36647a66966..71ed7898d9c 100644
--- a/xlators/features/barrier/src/barrier-mem-types.h
+++ b/xlators/features/barrier/src/barrier-mem-types.h
@@ -11,10 +11,10 @@
 #ifndef __BARRIER_MEM_TYPES_H__
 #define __BARRIER_MEM_TYPES_H__
 
-#include "mem-types.h"
+#include <glusterfs/mem-types.h>
 
 enum gf_barrier_mem_types_ {
-        gf_barrier_mt_priv_t = gf_common_mt_end + 1,
-        gf_barrier_mt_end
+    gf_barrier_mt_priv_t = gf_common_mt_end + 1,
+    gf_barrier_mt_end
 };
 #endif
diff --git a/xlators/features/barrier/src/barrier.c b/xlators/features/barrier/src/barrier.c
index 9620a73e71b..852bbacb99d 100644
--- a/xlators/features/barrier/src/barrier.c
+++ b/xlators/features/barrier/src/barrier.c
@@ -8,797 +8,802 @@
    cases as published by the Free Software Foundation.
 */
 
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
-
 #include "barrier.h"
-#include "defaults.h"
-#include "call-stub.h"
+#include <glusterfs/defaults.h>
+#include <glusterfs/call-stub.h>
 
-#include "statedump.h"
+#include <glusterfs/statedump.h>
 
 void
-barrier_local_set_gfid (call_frame_t *frame, uuid_t gfid, xlator_t *this)
-{
-        if (gfid) {
-                uuid_t *id = GF_MALLOC (sizeof (uuid_t), gf_common_mt_uuid_t);
-                if (!id) {
-                        gf_log (this->name, GF_LOG_WARNING, "Could not set gfid"
-                                ". gfid will not be dumped in statedump file.");
-                        return;
-                }
-                gf_uuid_copy (*id, gfid);
-                frame->local = id;
+barrier_local_set_gfid(call_frame_t *frame, uuid_t gfid, xlator_t *this)
+{
+    if (gfid) {
+        uuid_t *id = GF_MALLOC(sizeof(uuid_t), gf_common_mt_uuid_t);
+        if (!id) {
+            gf_log(this->name, GF_LOG_WARNING,
+                   "Could not set gfid"
+                   ". gfid will not be dumped in statedump file.");
+            return;
         }
+        gf_uuid_copy(*id, gfid);
+        frame->local = id;
+    }
 }
 
 void
-barrier_local_free_gfid (call_frame_t *frame)
+barrier_local_free_gfid(call_frame_t *frame)
 {
-        if (frame->local) {
-                GF_FREE (frame->local);
-                frame->local = NULL;
-        }
+    if (frame->local) {
+        GF_FREE(frame->local);
+        frame->local = NULL;
+    }
+}
+
+int32_t
+barrier_truncate_cbk_resume(call_frame_t *frame, void *cookie, xlator_t *this,
+                            int32_t op_ret, int32_t op_errno,
+                            struct iatt *prebuf, struct iatt *postbuf,
+                            dict_t *xdata)
+{
+    barrier_local_free_gfid(frame);
+    STACK_UNWIND_STRICT(truncate, frame, op_ret, op_errno, prebuf, postbuf,
+                        xdata);
+    return 0;
 }
 
 int32_t
-barrier_truncate_cbk_resume (call_frame_t *frame, void *cookie, xlator_t *this,
+barrier_ftruncate_cbk_resume(call_frame_t *frame, void *cookie, xlator_t *this,
                              int32_t op_ret, int32_t op_errno,
                              struct iatt *prebuf, struct iatt *postbuf,
                              dict_t *xdata)
 {
-        barrier_local_free_gfid (frame);
-        STACK_UNWIND_STRICT (truncate, frame, op_ret, op_errno, prebuf, postbuf,
-                             xdata);
-        return 0;
+    barrier_local_free_gfid(frame);
+    STACK_UNWIND_STRICT(ftruncate, frame, op_ret, op_errno, prebuf, postbuf,
+                        xdata);
+    return 0;
 }
 
 int32_t
-barrier_ftruncate_cbk_resume (call_frame_t *frame, void *cookie, xlator_t *this,
-                              int32_t op_ret, int32_t op_errno,
-                              struct iatt *prebuf, struct iatt *postbuf,
-                              dict_t *xdata)
-{
-        barrier_local_free_gfid (frame);
-        STACK_UNWIND_STRICT (ftruncate, frame, op_ret, op_errno, prebuf,
-                             postbuf, xdata);
-        return 0;
+barrier_unlink_cbk_resume(call_frame_t *frame, void *cookie, xlator_t *this,
+                          int32_t op_ret, int32_t op_errno,
+                          struct iatt *preparent, struct iatt *postparent,
+                          dict_t *xdata)
+{
+    barrier_local_free_gfid(frame);
+    STACK_UNWIND_STRICT(unlink, frame, op_ret, op_errno, preparent, postparent,
+                        xdata);
+    return 0;
 }
 
 int32_t
-barrier_unlink_cbk_resume (call_frame_t *frame, void *cookie, xlator_t *this,
-                           int32_t op_ret, int32_t op_errno,
-                           struct iatt *preparent, struct iatt *postparent,
-                           dict_t *xdata)
-{
-        barrier_local_free_gfid (frame);
-        STACK_UNWIND_STRICT (unlink, frame, op_ret, op_errno, preparent,
-                             postparent, xdata);
-        return 0;
+barrier_rmdir_cbk_resume(call_frame_t *frame, void *cookie, xlator_t *this,
+                         int32_t op_ret, int32_t op_errno,
+                         struct iatt *preparent, struct iatt *postparent,
+                         dict_t *xdata)
+{
+    barrier_local_free_gfid(frame);
+    STACK_UNWIND_STRICT(rmdir, frame, op_ret, op_errno, preparent, postparent,
+                        xdata);
+    return 0;
 }
 
 int32_t
-barrier_rmdir_cbk_resume (call_frame_t *frame, void *cookie, xlator_t *this,
-                          int32_t op_ret, int32_t op_errno,
-                          struct iatt *preparent, struct iatt *postparent,
+barrier_rename_cbk_resume(call_frame_t *frame, void *cookie, xlator_t *this,
+                          int32_t op_ret, int32_t op_errno, struct iatt *buf,
+                          struct iatt *preoldparent, struct iatt *postoldparent,
+                          struct iatt *prenewparent, struct iatt *postnewparent,
                           dict_t *xdata)
 {
-        barrier_local_free_gfid (frame);
-        STACK_UNWIND_STRICT (rmdir, frame, op_ret, op_errno, preparent,
-                             postparent, xdata);
-        return 0;
+    barrier_local_free_gfid(frame);
+    STACK_UNWIND_STRICT(rename, frame, op_ret, op_errno, buf, preoldparent,
+                        postoldparent, prenewparent, postnewparent, xdata);
+    return 0;
 }
 
 int32_t
-barrier_rename_cbk_resume (call_frame_t *frame, void *cookie, xlator_t *this,
-                           int32_t op_ret, int32_t op_errno, struct iatt *buf,
-                           struct iatt *preoldparent, struct iatt *postoldparent,
-                           struct iatt *prenewparent, struct iatt *postnewparent,
-                           dict_t *xdata)
-{
-        barrier_local_free_gfid (frame);
-        STACK_UNWIND_STRICT (rename, frame, op_ret, op_errno, buf, preoldparent,
-                             postoldparent, prenewparent, postnewparent, xdata);
-        return 0;
+barrier_writev_cbk_resume(call_frame_t *frame, void *cookie, xlator_t *this,
+                          int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
+                          struct iatt *postbuf, dict_t *xdata)
+{
+    barrier_local_free_gfid(frame);
+    STACK_UNWIND_STRICT(writev, frame, op_ret, op_errno, prebuf, postbuf,
+                        xdata);
+    return 0;
 }
 
 int32_t
-barrier_writev_cbk_resume (call_frame_t *frame, void *cookie, xlator_t *this,
-                           int32_t op_ret, int32_t op_errno,
-                           struct iatt *prebuf, struct iatt *postbuf,
-                           dict_t *xdata)
-{
-        barrier_local_free_gfid (frame);
-        STACK_UNWIND_STRICT (writev, frame, op_ret, op_errno, prebuf, postbuf,
-                             xdata);
-        return 0;
+barrier_fsync_cbk_resume(call_frame_t *frame, void *cookie, xlator_t *this,
+                         int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
+                         struct iatt *postbuf, dict_t *xdata)
+{
+    barrier_local_free_gfid(frame);
+    STACK_UNWIND_STRICT(fsync, frame, op_ret, op_errno, prebuf, postbuf, xdata);
+    return 0;
 }
 
 int32_t
-barrier_fsync_cbk_resume (call_frame_t *frame, void *cookie, xlator_t *this,
-                          int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
-                          struct iatt *postbuf, dict_t *xdata)
+barrier_removexattr_cbk_resume(call_frame_t *frame, void *cookie,
+                               xlator_t *this, int32_t op_ret, int32_t op_errno,
+                               dict_t *xdata)
 {
-        barrier_local_free_gfid (frame);
-        STACK_UNWIND_STRICT (fsync, frame, op_ret, op_errno, prebuf, postbuf,
-                             xdata);
-        return 0;
+    barrier_local_free_gfid(frame);
+    STACK_UNWIND_STRICT(removexattr, frame, op_ret, op_errno, xdata);
+    return 0;
 }
 
 int32_t
-barrier_removexattr_cbk_resume (call_frame_t *frame, void *cookie,
+barrier_fremovexattr_cbk_resume(call_frame_t *frame, void *cookie,
                                 xlator_t *this, int32_t op_ret,
                                 int32_t op_errno, dict_t *xdata)
 {
-        barrier_local_free_gfid (frame);
-        STACK_UNWIND_STRICT (removexattr, frame, op_ret, op_errno, xdata);
-        return 0;
+    barrier_local_free_gfid(frame);
+    STACK_UNWIND_STRICT(fremovexattr, frame, op_ret, op_errno, xdata);
+    return 0;
 }
 
 int32_t
-barrier_fremovexattr_cbk_resume (call_frame_t *frame, void *cookie,
-                                 xlator_t *this, int32_t op_ret,
-                                 int32_t op_errno, dict_t *xdata)
+barrier_writev_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                   int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
+                   struct iatt *postbuf, dict_t *xdata)
 {
-        barrier_local_free_gfid (frame);
-        STACK_UNWIND_STRICT (fremovexattr, frame, op_ret, op_errno, xdata);
-        return 0;
+    BARRIER_FOP_CBK(writev, out, frame, this, op_ret, op_errno, prebuf, postbuf,
+                    xdata);
+out:
+    return 0;
 }
 
 int32_t
-barrier_writev_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                    int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
-                    struct iatt *postbuf,
-                    dict_t *xdata)
+barrier_fremovexattr_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                         int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
-        BARRIER_FOP_CBK (writev, out, frame, this, op_ret, op_errno,
-                         prebuf, postbuf, xdata);
+    BARRIER_FOP_CBK(fremovexattr, out, frame, this, op_ret, op_errno, xdata);
 out:
-        return 0;
+    return 0;
 }
 
 int32_t
-barrier_fremovexattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                          int32_t op_ret, int32_t op_errno, dict_t *xdata)
+barrier_removexattr_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                        int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
-        BARRIER_FOP_CBK (fremovexattr, out, frame, this, op_ret, op_errno,
-                         xdata);
+    BARRIER_FOP_CBK(removexattr, out, frame, this, op_ret, op_errno, xdata);
 out:
-        return 0;
+    return 0;
 }
 
 int32_t
-barrier_removexattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                         int32_t op_ret, int32_t op_errno, dict_t *xdata)
+barrier_truncate_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                     int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
+                     struct iatt *postbuf, dict_t *xdata)
 {
-        BARRIER_FOP_CBK (removexattr, out, frame, this, op_ret, op_errno,
-                         xdata);
+    BARRIER_FOP_CBK(truncate, out, frame, this, op_ret, op_errno, prebuf,
+                    postbuf, xdata);
 out:
-        return 0;
+    return 0;
 }
 
 int32_t
-barrier_truncate_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+barrier_ftruncate_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
                       int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
                       struct iatt *postbuf, dict_t *xdata)
 {
-        BARRIER_FOP_CBK (truncate, out, frame, this, op_ret, op_errno, prebuf,
-                         postbuf, xdata);
+    BARRIER_FOP_CBK(ftruncate, out, frame, this, op_ret, op_errno, prebuf,
+                    postbuf, xdata);
 out:
-        return 0;
+    return 0;
 }
 
 int32_t
-barrier_ftruncate_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                       int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
-                       struct iatt *postbuf, dict_t *xdata)
-{
-        BARRIER_FOP_CBK (ftruncate, out, frame, this, op_ret, op_errno, prebuf,
-                         postbuf, xdata);
+barrier_rename_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                   int32_t op_ret, int32_t op_errno, struct iatt *buf,
+                   struct iatt *preoldparent, struct iatt *postoldparent,
+                   struct iatt *prenewparent, struct iatt *postnewparent,
+                   dict_t *xdata)
+{
+    BARRIER_FOP_CBK(rename, out, frame, this, op_ret, op_errno, buf,
+                    preoldparent, postoldparent, prenewparent, postnewparent,
+                    xdata);
 out:
-        return 0;
+    return 0;
 }
 
 int32_t
-barrier_rename_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                    int32_t op_ret, int32_t op_errno, struct iatt *buf,
-                    struct iatt *preoldparent, struct iatt *postoldparent,
-                    struct iatt *prenewparent, struct iatt *postnewparent,
-                    dict_t *xdata)
-{
-        BARRIER_FOP_CBK (rename, out, frame, this, op_ret, op_errno, buf,
-                         preoldparent, postoldparent, prenewparent,
-                         postnewparent, xdata);
+barrier_rmdir_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                  int32_t op_ret, int32_t op_errno, struct iatt *preparent,
+                  struct iatt *postparent, dict_t *xdata)
+{
+    BARRIER_FOP_CBK(rmdir, out, frame, this, op_ret, op_errno, preparent,
+                    postparent, xdata);
 out:
-        return 0;
+    return 0;
 }
 
 int32_t
-barrier_rmdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+barrier_unlink_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
                    int32_t op_ret, int32_t op_errno, struct iatt *preparent,
                    struct iatt *postparent, dict_t *xdata)
 {
-        BARRIER_FOP_CBK (rmdir, out, frame, this, op_ret, op_errno, preparent,
-                         postparent, xdata);
+    BARRIER_FOP_CBK(unlink, out, frame, this, op_ret, op_errno, preparent,
+                    postparent, xdata);
 out:
-        return 0;
+    return 0;
 }
 
 int32_t
-barrier_unlink_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                    int32_t op_ret, int32_t op_errno, struct iatt *preparent,
-                    struct iatt *postparent, dict_t *xdata)
+barrier_fsync_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                  int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
+                  struct iatt *postbuf, dict_t *xdata)
 {
-        BARRIER_FOP_CBK (unlink, out, frame, this, op_ret, op_errno, preparent,
-                         postparent, xdata);
+    BARRIER_FOP_CBK(fsync, out, frame, this, op_ret, op_errno, prebuf, postbuf,
+                    xdata);
 out:
-        return 0;
+    return 0;
 }
 
 int32_t
-barrier_fsync_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                   int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
-                   struct iatt *postbuf, dict_t *xdata)
+barrier_writev(call_frame_t *frame, xlator_t *this, fd_t *fd,
+               struct iovec *vector, int32_t count, off_t off, uint32_t flags,
+               struct iobref *iobref, dict_t *xdata)
 {
-        BARRIER_FOP_CBK (fsync, out, frame, this, op_ret, op_errno,
-                         prebuf, postbuf, xdata);
-out:
-        return 0;
-}
-
-int32_t
-barrier_writev (call_frame_t *frame, xlator_t *this, fd_t *fd,
-                struct iovec *vector, int32_t count, off_t off, uint32_t flags,
-                struct iobref *iobref, dict_t *xdata)
-{
-        if (!((flags | fd->flags) & (O_SYNC | O_DSYNC))) {
-                STACK_WIND_TAIL (frame, FIRST_CHILD(this),
-                                 FIRST_CHILD(this)->fops->writev,
-                                 fd, vector, count, off, flags, iobref, xdata);
-
-                return 0;
-        }
+    if (!((flags | fd->flags) & (O_SYNC | O_DSYNC))) {
+        STACK_WIND_TAIL(frame, FIRST_CHILD(this),
+                        FIRST_CHILD(this)->fops->writev, fd, vector, count, off,
+                        flags, iobref, xdata);
 
-        barrier_local_set_gfid (frame, fd->inode->gfid, this);
-        STACK_WIND (frame, barrier_writev_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->writev, fd, vector, count,
-                    off, flags, iobref, xdata);
         return 0;
+    }
+
+    barrier_local_set_gfid(frame, fd->inode->gfid, this);
+    STACK_WIND(frame, barrier_writev_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->writev, fd, vector, count, off, flags,
+               iobref, xdata);
+    return 0;
 }
 
 int32_t
-barrier_fremovexattr (call_frame_t *frame, xlator_t *this, fd_t *fd,
-                      const char *name, dict_t *xdata)
+barrier_fremovexattr(call_frame_t *frame, xlator_t *this, fd_t *fd,
+                     const char *name, dict_t *xdata)
 {
-        barrier_local_set_gfid (frame, fd->inode->gfid, this);
-        STACK_WIND (frame, barrier_fremovexattr_cbk, FIRST_CHILD (this),
-                    FIRST_CHILD (this)->fops->fremovexattr,
-                    fd, name, xdata);
-        return 0;
+    barrier_local_set_gfid(frame, fd->inode->gfid, this);
+    STACK_WIND(frame, barrier_fremovexattr_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->fremovexattr, fd, name, xdata);
+    return 0;
 }
 
 int32_t
-barrier_removexattr (call_frame_t *frame, xlator_t *this, loc_t *loc,
-                     const char *name, dict_t *xdata)
+barrier_removexattr(call_frame_t *frame, xlator_t *this, loc_t *loc,
+                    const char *name, dict_t *xdata)
 {
-        barrier_local_set_gfid (frame, loc->inode->gfid, this);
-        STACK_WIND (frame, barrier_removexattr_cbk, FIRST_CHILD (this),
-                    FIRST_CHILD (this)->fops->removexattr,
-                    loc, name, xdata);
-        return 0;
+    barrier_local_set_gfid(frame, loc->inode->gfid, this);
+    STACK_WIND(frame, barrier_removexattr_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->removexattr, loc, name, xdata);
+    return 0;
 }
 
 int32_t
-barrier_truncate (call_frame_t *frame, xlator_t *this, loc_t *loc,
-                  off_t offset, dict_t *xdata)
+barrier_truncate(call_frame_t *frame, xlator_t *this, loc_t *loc, off_t offset,
+                 dict_t *xdata)
 {
-        barrier_local_set_gfid (frame, loc->inode->gfid, this);
-        STACK_WIND (frame, barrier_truncate_cbk, FIRST_CHILD (this),
-                    FIRST_CHILD (this)->fops->truncate,
-                    loc, offset, xdata);
-        return 0;
+    barrier_local_set_gfid(frame, loc->inode->gfid, this);
+    STACK_WIND(frame, barrier_truncate_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->truncate, loc, offset, xdata);
+    return 0;
 }
 
-
 int32_t
-barrier_rename (call_frame_t *frame, xlator_t *this, loc_t *oldloc,
-                loc_t *newloc, dict_t *xdata)
+barrier_rename(call_frame_t *frame, xlator_t *this, loc_t *oldloc,
+               loc_t *newloc, dict_t *xdata)
 {
-        barrier_local_set_gfid (frame, oldloc->inode->gfid, this);
-        STACK_WIND (frame, barrier_rename_cbk, FIRST_CHILD (this),
-                    FIRST_CHILD (this)->fops->rename,
-                    oldloc, newloc, xdata);
-        return 0;
+    barrier_local_set_gfid(frame, oldloc->inode->gfid, this);
+    STACK_WIND(frame, barrier_rename_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->rename, oldloc, newloc, xdata);
+    return 0;
 }
 
 int
-barrier_rmdir (call_frame_t *frame, xlator_t *this, loc_t *loc, int flags,
-               dict_t *xdata)
+barrier_rmdir(call_frame_t *frame, xlator_t *this, loc_t *loc, int flags,
+              dict_t *xdata)
 {
-        barrier_local_set_gfid (frame, loc->inode->gfid, this);
-        STACK_WIND (frame, barrier_rmdir_cbk, FIRST_CHILD (this),
-                    FIRST_CHILD (this)->fops->rmdir,
-                    loc, flags, xdata);
-        return 0;
+    barrier_local_set_gfid(frame, loc->inode->gfid, this);
+    STACK_WIND(frame, barrier_rmdir_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->rmdir, loc, flags, xdata);
+    return 0;
 }
 
 int32_t
-barrier_unlink (call_frame_t *frame, xlator_t *this,
-                loc_t *loc, int xflag, dict_t *xdata)
+barrier_unlink(call_frame_t *frame, xlator_t *this, loc_t *loc, int xflag,
+               dict_t *xdata)
 {
-        barrier_local_set_gfid (frame, loc->inode->gfid, this);
-        STACK_WIND (frame, barrier_unlink_cbk, FIRST_CHILD (this),
-                    FIRST_CHILD (this)->fops->unlink,
-                    loc, xflag, xdata);
-        return 0;
+    barrier_local_set_gfid(frame, loc->inode->gfid, this);
+    STACK_WIND(frame, barrier_unlink_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->unlink, loc, xflag, xdata);
+    return 0;
 }
 
 int32_t
-barrier_ftruncate (call_frame_t *frame, xlator_t *this, fd_t *fd,
-                   off_t offset, dict_t *xdata)
+barrier_ftruncate(call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
+                  dict_t *xdata)
 {
-        barrier_local_set_gfid (frame, fd->inode->gfid, this);
-        STACK_WIND (frame, barrier_ftruncate_cbk, FIRST_CHILD (this),
-                    FIRST_CHILD (this)->fops->ftruncate,
-                    fd, offset, xdata);
-        return 0;
+    barrier_local_set_gfid(frame, fd->inode->gfid, this);
+    STACK_WIND(frame, barrier_ftruncate_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->ftruncate, fd, offset, xdata);
+    return 0;
 }
 
 int32_t
-barrier_fsync (call_frame_t *frame, xlator_t *this, fd_t *fd,
-               int32_t flags, dict_t *xdata)
+barrier_fsync(call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t flags,
+              dict_t *xdata)
 {
-        barrier_local_set_gfid (frame, fd->inode->gfid, this);
-        STACK_WIND (frame, barrier_fsync_cbk, FIRST_CHILD (this),
-                    FIRST_CHILD (this)->fops->fsync,
-                    fd, flags, xdata);
-        return 0;
+    barrier_local_set_gfid(frame, fd->inode->gfid, this);
+    STACK_WIND(frame, barrier_fsync_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->fsync, fd, flags, xdata);
+    return 0;
 }
 
 call_stub_t *
-__barrier_dequeue (xlator_t *this, struct list_head *queue)
+__barrier_dequeue(xlator_t *this, struct list_head *queue)
 {
-        call_stub_t    *stub            = NULL;
-        barrier_priv_t *priv            = NULL;
+    call_stub_t *stub = NULL;
+    barrier_priv_t *priv = NULL;
 
-        priv = this->private;
-        GF_ASSERT (priv);
+    priv = this->private;
+    GF_ASSERT(priv);
 
-        if (list_empty (queue))
-                goto out;
+    if (list_empty(queue))
+        goto out;
 
-        stub = list_entry (queue->next, call_stub_t, list);
-        list_del_init (&stub->list);
+    stub = list_entry(queue->next, call_stub_t, list);
+    list_del_init(&stub->list);
 
 out:
-        return stub;
+    return stub;
 }
 
 void
-barrier_dequeue_all (xlator_t *this, struct list_head *queue)
+barrier_dequeue_all(xlator_t *this, struct list_head *queue)
 {
-        call_stub_t            *stub    = NULL;
+    call_stub_t *stub = NULL;
 
-        gf_log (this->name, GF_LOG_INFO, "Dequeuing all the barriered fops");
+    gf_log(this->name, GF_LOG_INFO, "Dequeuing all the barriered fops");
 
-        /* TODO: Start the below task in a new thread */
-        while ((stub = __barrier_dequeue (this, queue)))
-                call_resume (stub);
+    /* TODO: Start the below task in a new thread */
+    while ((stub = __barrier_dequeue(this, queue)))
+        call_resume(stub);
 
-        gf_log (this->name, GF_LOG_INFO, "Dequeuing the barriered fops is "
-                                         "finished");
-        return;
+    gf_log(this->name, GF_LOG_INFO,
+           "Dequeuing the barriered fops is "
+           "finished");
+    return;
 }
 
 void
-barrier_timeout (void *data)
+barrier_timeout(void *data)
 {
-        xlator_t               *this    = NULL;
-        barrier_priv_t         *priv    = NULL;
-        struct list_head        queue   = {0,};
+    xlator_t *this = NULL;
+    barrier_priv_t *priv = NULL;
+    struct list_head queue = {
+        0,
+    };
 
-        this = data;
-        THIS = this;
-        priv = this->private;
+    this = data;
+    THIS = this;
+    priv = this->private;
 
-        INIT_LIST_HEAD (&queue);
+    INIT_LIST_HEAD(&queue);
 
-        gf_log (this->name, GF_LOG_CRITICAL, "Disabling barrier because of "
-                                             "the barrier timeout.");
+    gf_log(this->name, GF_LOG_CRITICAL,
+           "Disabling barrier because of "
+           "the barrier timeout.");
 
-        LOCK (&priv->lock);
-        {
-                __barrier_disable (this, &queue);
-        }
-        UNLOCK (&priv->lock);
+    LOCK(&priv->lock);
+    {
+        __barrier_disable(this, &queue);
+    }
+    UNLOCK(&priv->lock);
 
-        barrier_dequeue_all (this, &queue);
+    barrier_dequeue_all(this, &queue);
 
-        return;
+    return;
 }
 
 void
-__barrier_enqueue (xlator_t *this, call_stub_t *stub)
+__barrier_enqueue(xlator_t *this, call_stub_t *stub)
 {
-        barrier_priv_t *priv    = NULL;
+    barrier_priv_t *priv = NULL;
 
-        priv = this->private;
-        GF_ASSERT (priv);
+    priv = this->private;
+    GF_ASSERT(priv);
 
-        list_add_tail (&stub->list, &priv->queue);
-        priv->queue_size++;
+    list_add_tail(&stub->list, &priv->queue);
+    priv->queue_size++;
 
-        return;
+    return;
 }
 
 void
-__barrier_disable (xlator_t *this, struct list_head *queue)
+__barrier_disable(xlator_t *this, struct list_head *queue)
 {
-        GF_UNUSED int   ret     = 0;
-        barrier_priv_t *priv    = NULL;
+    GF_UNUSED int ret = 0;
+    barrier_priv_t *priv = NULL;
 
-        priv = this->private;
-        GF_ASSERT (priv);
+    priv = this->private;
+    GF_ASSERT(priv);
 
-        if (priv->timer) {
-                ret = gf_timer_call_cancel (this->ctx, priv->timer);
-                priv->timer = NULL;
-        }
+    if (priv->timer) {
+        ret = gf_timer_call_cancel(this->ctx, priv->timer);
+        priv->timer = NULL;
+    }
 
-        list_splice_init (&priv->queue, queue);
-        priv->queue_size = 0;
-        priv->barrier_enabled = _gf_false;
+    list_splice_init(&priv->queue, queue);
+    priv->queue_size = 0;
+    priv->barrier_enabled = _gf_false;
 }
 
 int
-__barrier_enable (xlator_t *this, barrier_priv_t *priv)
+__barrier_enable(xlator_t *this, barrier_priv_t *priv)
 {
-        int             ret     = -1;
+    int ret = -1;
 
-        priv->timer = gf_timer_call_after (this->ctx, priv->timeout,
-                                           barrier_timeout, (void *) this);
-        if (!priv->timer) {
-                gf_log (this->name, GF_LOG_CRITICAL, "Couldn't add barrier "
-                                                     "timeout event.");
-                goto out;
-        }
+    priv->timer = gf_timer_call_after(this->ctx, priv->timeout, barrier_timeout,
+                                      (void *)this);
+    if (!priv->timer) {
+        gf_log(this->name, GF_LOG_CRITICAL,
+               "Couldn't add barrier "
+               "timeout event.");
+        goto out;
+    }
 
-        priv->barrier_enabled = _gf_true;
-        ret = 0;
+    priv->barrier_enabled = _gf_true;
+    ret = 0;
 out:
-        return ret;
+    return ret;
 }
 
 int
-notify (xlator_t *this, int event, void *data, ...)
-{
-        barrier_priv_t  *priv                   = NULL;
-        dict_t          *dict                   = NULL;
-        gf_boolean_t     past                   = _gf_false;
-        int              ret                    = -1;
-        gf_boolean_t     barrier_enabled        = _gf_false;
-        struct list_head queue                  = {0,};
-
-        priv = this->private;
-        GF_ASSERT (priv);
-        INIT_LIST_HEAD (&queue);
-
-        switch (event) {
-        case GF_EVENT_TRANSLATOR_OP:
-        {
-                dict = data;
-                barrier_enabled = dict_get_str_boolean (dict, "barrier", -1);
-
-                if (barrier_enabled == -1) {
-                        gf_log (this->name, GF_LOG_ERROR, "Could not fetch "
-                                " barrier key from the dictionary.");
-                        goto out;
-                }
-
-                LOCK (&priv->lock);
-                {
-                        past = priv->barrier_enabled;
-
-                        switch (past) {
-                        case _gf_false:
-                                if (barrier_enabled) {
-                                        ret = __barrier_enable (this,priv);
-                                        if (ret)
-                                                goto unlock;
-                                } else {
-                                        gf_log (this->name, GF_LOG_ERROR,
-                                                "Already disabled.");
-                                        goto unlock;
-                                }
-                                break;
-
-                        case _gf_true:
-                                if (!barrier_enabled) {
-                                        __barrier_disable(this, &queue);
-                                } else {
-                                        gf_log (this->name, GF_LOG_ERROR,
-                                                "Already enabled");
-                                        goto unlock;
-                                }
-                                break;
-                        }
+notify(xlator_t *this, int event, void *data, ...)
+{
+    barrier_priv_t *priv = this->private;
+    dict_t *dict = NULL;
+    int ret = -1;
+    int barrier_enabled = _gf_false;
+    struct list_head queue = {
+        0,
+    };
+
+    GF_ASSERT(priv);
+    INIT_LIST_HEAD(&queue);
+
+    switch (event) {
+        case GF_EVENT_TRANSLATOR_OP: {
+            dict = data;
+            barrier_enabled = dict_get_str_boolean(dict, "barrier", -1);
+
+            if (barrier_enabled == -1) {
+                gf_log(this->name, GF_LOG_ERROR,
+                       "Could not fetch "
+                       " barrier key from the dictionary.");
+                goto out;
+            }
+
+            LOCK(&priv->lock);
+            {
+                if (!priv->barrier_enabled) {
+                    if (barrier_enabled) {
+                        ret = __barrier_enable(this, priv);
+                    } else {
+                        UNLOCK(&priv->lock);
+                        gf_log(this->name, GF_LOG_ERROR, "Already disabled.");
+                        goto post_unlock;
+                    }
+                } else {
+                    if (!barrier_enabled) {
+                        __barrier_disable(this, &queue);
                         ret = 0;
+                    } else {
+                        UNLOCK(&priv->lock);
+                        gf_log(this->name, GF_LOG_ERROR, "Already enabled");
+                        goto post_unlock;
+                    }
                 }
-unlock:
-                UNLOCK (&priv->lock);
+            }
+            UNLOCK(&priv->lock);
+        post_unlock:
+            if (!list_empty(&queue))
+                barrier_dequeue_all(this, &queue);
 
-                if (!list_empty (&queue))
-                        barrier_dequeue_all (this, &queue);
-
-                break;
-        }
-        default:
-        {
-                default_notify (this, event, data);
-                ret = 0;
-                goto out;
+            break;
         }
+        default: {
+            default_notify(this, event, data);
+            ret = 0;
+            goto out;
         }
+    }
 out:
-        return ret;
+    return ret;
 }
 
 int
-reconfigure (xlator_t *this, dict_t *options)
-{
-        barrier_priv_t  *priv                   = NULL;
-        gf_boolean_t     past                   = _gf_false;
-        int              ret                    = -1;
-        gf_boolean_t     barrier_enabled        = _gf_false;
-        uint32_t         timeout                = {0,};
-        struct list_head queue                  = {0,};
-
-        priv = this->private;
-        GF_ASSERT (priv);
-
-        GF_OPTION_RECONF ("barrier", barrier_enabled, options, bool, out);
-        GF_OPTION_RECONF ("barrier-timeout", timeout, options, time, out);
-
-        INIT_LIST_HEAD (&queue);
-
-        LOCK (&priv->lock);
-        {
-                past = priv->barrier_enabled;
-
-                switch (past) {
-                case _gf_false:
-                        if (barrier_enabled) {
-                                ret = __barrier_enable (this, priv);
-                                if (ret) {
-                                        goto unlock;
-                                }
-                        }
-                        break;
-
-                case _gf_true:
-                        if (!barrier_enabled) {
-                                __barrier_disable (this, &queue);
-
-                        }
-                        break;
+reconfigure(xlator_t *this, dict_t *options)
+{
+    barrier_priv_t *priv = NULL;
+    int ret = -1;
+    gf_boolean_t barrier_enabled = _gf_false;
+    uint32_t timeout = {
+        0,
+    };
+    struct list_head queue = {
+        0,
+    };
+
+    priv = this->private;
+    GF_ASSERT(priv);
+
+    GF_OPTION_RECONF("barrier", barrier_enabled, options, bool, out);
+    GF_OPTION_RECONF("barrier-timeout", timeout, options, time, out);
+
+    INIT_LIST_HEAD(&queue);
+
+    LOCK(&priv->lock);
+    {
+        if (!priv->barrier_enabled) {
+            if (barrier_enabled) {
+                ret = __barrier_enable(this, priv);
+                if (ret) {
+                    goto unlock;
                 }
-                priv->timeout.tv_sec = timeout;
-                ret = 0;
+            }
+        } else {
+            if (!barrier_enabled) {
+                __barrier_disable(this, &queue);
+            }
         }
+        priv->timeout.tv_sec = timeout;
+        ret = 0;
+    }
 unlock:
-        UNLOCK (&priv->lock);
+    UNLOCK(&priv->lock);
 
-        if (!list_empty (&queue))
-                barrier_dequeue_all (this, &queue);
+    if (!list_empty(&queue))
+        barrier_dequeue_all(this, &queue);
 
 out:
-        return ret;
+    return ret;
 }
 
 int32_t
-mem_acct_init (xlator_t *this)
+mem_acct_init(xlator_t *this)
 {
-        int     ret = -1;
+    int ret = -1;
 
-        ret = xlator_mem_acct_init (this, gf_barrier_mt_end + 1);
-        if (ret)
-                gf_log (this->name, GF_LOG_ERROR, "Memory accounting "
-                        "initialization failed.");
+    ret = xlator_mem_acct_init(this, gf_barrier_mt_end + 1);
+    if (ret)
+        gf_log(this->name, GF_LOG_ERROR,
+               "Memory accounting "
+               "initialization failed.");
 
-        return ret;
+    return ret;
 }
 
 int
-init (xlator_t *this)
+init(xlator_t *this)
 {
-        int                     ret     = -1;
-        barrier_priv_t         *priv    = NULL;
-        uint32_t                timeout = {0,};
+    int ret = -1;
+    barrier_priv_t *priv = NULL;
+    uint32_t timeout = {
+        0,
+    };
 
-        if (!this->children || this->children->next) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "'barrier' not configured with exactly one child");
-                goto out;
-        }
+    if (!this->children || this->children->next) {
+        gf_log(this->name, GF_LOG_ERROR,
+               "'barrier' not configured with exactly one child");
+        goto out;
+    }
 
-        if (!this->parents)
-                gf_log (this->name, GF_LOG_WARNING,
-                        "dangling volume. check volfile ");
+    if (!this->parents)
+        gf_log(this->name, GF_LOG_WARNING, "dangling volume. check volfile ");
 
-        priv = GF_CALLOC (1, sizeof (*priv), gf_barrier_mt_priv_t);
-        if (!priv)
-                goto out;
+    priv = GF_CALLOC(1, sizeof(*priv), gf_barrier_mt_priv_t);
+    if (!priv)
+        goto out;
 
-        LOCK_INIT (&priv->lock);
+    LOCK_INIT(&priv->lock);
 
-        GF_OPTION_INIT ("barrier", priv->barrier_enabled, bool, out);
-        GF_OPTION_INIT ("barrier-timeout", timeout, time, out);
-        priv->timeout.tv_sec = timeout;
+    GF_OPTION_INIT("barrier", priv->barrier_enabled, bool, out);
+    GF_OPTION_INIT("barrier-timeout", timeout, time, out);
+    priv->timeout.tv_sec = timeout;
 
-        INIT_LIST_HEAD (&priv->queue);
+    INIT_LIST_HEAD(&priv->queue);
 
-        if (priv->barrier_enabled) {
-                ret = __barrier_enable (this, priv);
-                if (ret == -1)
-                        goto out;
-        }
+    if (priv->barrier_enabled) {
+        ret = __barrier_enable(this, priv);
+        if (ret == -1)
+            goto out;
+    }
 
-        this->private = priv;
-        ret = 0;
+    this->private = priv;
+    ret = 0;
 out:
-        return ret;
+    if (ret && priv)
+        GF_FREE(priv);
+
+    return ret;
 }
 
 void
-fini (xlator_t *this)
+fini(xlator_t *this)
 {
-        barrier_priv_t         *priv    = NULL;
-        struct list_head        queue   = {0,};
+    barrier_priv_t *priv = NULL;
+    struct list_head queue = {
+        0,
+    };
 
-        priv = this->private;
-        if (!priv)
-                goto out;
+    priv = this->private;
+    if (!priv)
+        goto out;
 
-        INIT_LIST_HEAD (&queue);
+    INIT_LIST_HEAD(&queue);
 
-        gf_log (this->name, GF_LOG_INFO, "Disabling barriering and dequeuing "
-                                         "all the queued fops");
-        LOCK (&priv->lock);
-        {
-                __barrier_disable (this, &queue);
-        }
-        UNLOCK (&priv->lock);
+    gf_log(this->name, GF_LOG_INFO,
+           "Disabling barriering and dequeuing "
+           "all the queued fops");
+    LOCK(&priv->lock);
+    {
+        __barrier_disable(this, &queue);
+    }
+    UNLOCK(&priv->lock);
 
-        if (!list_empty (&queue))
-                barrier_dequeue_all (this, &queue);
+    if (!list_empty(&queue))
+        barrier_dequeue_all(this, &queue);
 
-        this->private = NULL;
+    this->private = NULL;
 
-        LOCK_DESTROY (&priv->lock);
-        GF_FREE (priv);
+    LOCK_DESTROY(&priv->lock);
+    GF_FREE(priv);
 out:
-        return;
+    return;
 }
 
 static void
-barrier_dump_stub (call_stub_t *stub, char *prefix)
+barrier_dump_stub(call_stub_t *stub, char *prefix)
 {
-        char key[GF_DUMP_MAX_BUF_LEN] = {0,};
+    char key[GF_DUMP_MAX_BUF_LEN] = {
+        0,
+    };
 
-        gf_proc_dump_build_key (key, prefix, "fop");
-        gf_proc_dump_write (key, "%s", gf_fop_list[stub->fop]);
+    gf_proc_dump_build_key(key, prefix, "fop");
+    gf_proc_dump_write(key, "%s", gf_fop_list[stub->fop]);
 
-        if (stub->frame->local) {
-                gf_proc_dump_build_key (key, prefix, "gfid");
-                gf_proc_dump_write (key, "%s",
-                                    uuid_utoa (*(uuid_t*)(stub->frame->local)));
-        }
-        if (stub->args.loc.path) {
-                gf_proc_dump_build_key (key, prefix, "path");
-                gf_proc_dump_write (key, "%s", stub->args.loc.path);
-        }
-        if (stub->args.loc.name) {
-                gf_proc_dump_build_key (key, prefix, "name");
-                gf_proc_dump_write (key, "%s", stub->args.loc.name);
-        }
+    if (stub->frame->local) {
+        gf_proc_dump_build_key(key, prefix, "gfid");
+        gf_proc_dump_write(key, "%s",
+                           uuid_utoa(*(uuid_t *)(stub->frame->local)));
+    }
+    if (stub->args.loc.path) {
+        gf_proc_dump_build_key(key, prefix, "path");
+        gf_proc_dump_write(key, "%s", stub->args.loc.path);
+    }
+    if (stub->args.loc.name) {
+        gf_proc_dump_build_key(key, prefix, "name");
+        gf_proc_dump_write(key, "%s", stub->args.loc.name);
+    }
 
-        return;
+    return;
 }
 
 static void
-__barrier_dump_queue (barrier_priv_t *priv)
+__barrier_dump_queue(barrier_priv_t *priv)
 {
-        call_stub_t *stub = NULL;
-        char key[GF_DUMP_MAX_BUF_LEN] = {0,};
-        int i = 0;
+    call_stub_t *stub = NULL;
+    char key[GF_DUMP_MAX_BUF_LEN] = {
+        0,
+    };
+    int i = 0;
 
-        GF_VALIDATE_OR_GOTO ("barrier", priv, out);
+    GF_VALIDATE_OR_GOTO("barrier", priv, out);
 
-        list_for_each_entry (stub, &priv->queue, list) {
-                snprintf (key, sizeof (key), "stub.%d", i++);
-                gf_proc_dump_add_section (key);
-                barrier_dump_stub(stub, key);
-        }
+    list_for_each_entry(stub, &priv->queue, list)
+    {
+        snprintf(key, sizeof(key), "stub.%d", i++);
+        gf_proc_dump_add_section("%s", key);
+        barrier_dump_stub(stub, key);
+    }
 
 out:
-        return;
+    return;
 }
 
 int
-barrier_dump_priv (xlator_t *this)
-{
-        int ret = -1;
-        char key[GF_DUMP_MAX_BUF_LEN] = {0,};
-        barrier_priv_t *priv = NULL;
-
-        GF_VALIDATE_OR_GOTO ("barrier", this, out);
-
-        priv = this->private;
-        if (!priv)
-                return 0;
-
-        gf_proc_dump_build_key (key, "xlator.features.barrier", "priv");
-        gf_proc_dump_add_section (key);
-
-        LOCK (&priv->lock);
-        {
-                gf_proc_dump_build_key (key, "barrier", "enabled");
-                gf_proc_dump_write (key, "%d", priv->barrier_enabled);
-                gf_proc_dump_build_key (key, "barrier", "timeout");
-                gf_proc_dump_write (key, "%"PRId64, priv->timeout.tv_sec);
-                if (priv->barrier_enabled) {
-                        gf_proc_dump_build_key (key, "barrier", "queue_size");
-                        gf_proc_dump_write (key, "%d", priv->queue_size);
-                        __barrier_dump_queue (priv);
-                }
+barrier_dump_priv(xlator_t *this)
+{
+    int ret = -1;
+    char key[GF_DUMP_MAX_BUF_LEN] = {
+        0,
+    };
+    barrier_priv_t *priv = NULL;
+
+    GF_VALIDATE_OR_GOTO("barrier", this, out);
+
+    priv = this->private;
+    if (!priv)
+        return 0;
+
+    gf_proc_dump_build_key(key, "xlator.features.barrier", "priv");
+    gf_proc_dump_add_section("%s", key);
+    gf_proc_dump_build_key(key, "barrier", "enabled");
+
+    LOCK(&priv->lock);
+    {
+        gf_proc_dump_write(key, "%d", priv->barrier_enabled);
+        gf_proc_dump_build_key(key, "barrier", "timeout");
+        gf_proc_dump_write(key, "%ld", priv->timeout.tv_sec);
+        if (priv->barrier_enabled) {
+            gf_proc_dump_build_key(key, "barrier", "queue_size");
+            gf_proc_dump_write(key, "%d", priv->queue_size);
+            __barrier_dump_queue(priv);
         }
-        UNLOCK (&priv->lock);
+    }
+    UNLOCK(&priv->lock);
 
 out:
-        return ret;
+    return ret;
 }
 
 struct xlator_fops fops = {
 
-        /* Barrier Class fops */
-        .rmdir          = barrier_rmdir,
-        .unlink         = barrier_unlink,
-        .rename         = barrier_rename,
-        .removexattr    = barrier_removexattr,
-        .fremovexattr   = barrier_fremovexattr,
-        .truncate       = barrier_truncate,
-        .ftruncate      = barrier_ftruncate,
-        .fsync          = barrier_fsync,
-
-        /* Writes with only O_SYNC flag */
-        .writev         = barrier_writev,
+    /* Barrier Class fops */
+    .rmdir = barrier_rmdir,
+    .unlink = barrier_unlink,
+    .rename = barrier_rename,
+    .removexattr = barrier_removexattr,
+    .fremovexattr = barrier_fremovexattr,
+    .truncate = barrier_truncate,
+    .ftruncate = barrier_ftruncate,
+    .fsync = barrier_fsync,
+
+    /* Writes with only O_SYNC flag */
+    .writev = barrier_writev,
 };
 
 struct xlator_dumpops dumpops = {
-        .priv = barrier_dump_priv,
+    .priv = barrier_dump_priv,
 };
 
 struct xlator_cbks cbks;
 
 struct volume_options options[] = {
-        { .key  = {"barrier"},
-          .type = GF_OPTION_TYPE_BOOL,
-          .default_value = "off",
-          .description = "When \"on\", blocks acknowledgements to application "
-                         "for file operations such as rmdir, rename, unlink, "
-                         "removexattr, fremovexattr, truncate, ftruncate, "
-                         "write (with O_SYNC), fsync. It is turned \"off\" by "
-                         "default."
-        },
-        { .key = {"barrier-timeout"},
-          .type = GF_OPTION_TYPE_TIME,
-          .default_value = BARRIER_TIMEOUT,
-          .description = "After 'timeout' seconds since the time 'barrier' "
-                         "option was set to \"on\", acknowledgements to file "
-                         "operations are no longer blocked and previously "
-                         "blocked acknowledgements are sent to the application"
-        },
-        { .key  = {NULL} },
+    {.key = {"barrier"},
+     .type = GF_OPTION_TYPE_BOOL,
+     .default_value = "disable",
+     .op_version = {GD_OP_VERSION_3_6_0},
+     .flags = OPT_FLAG_SETTABLE,
+     .description = "When \"enabled\", blocks acknowledgements to application "
+                    "for file operations such as rmdir, rename, unlink, "
+                    "removexattr, fremovexattr, truncate, ftruncate, "
+                    "write (with O_SYNC), fsync. It is turned \"off\" by "
+                    "default."},
+    {.key = {"barrier-timeout"},
+     .type = GF_OPTION_TYPE_TIME,
+     .default_value = BARRIER_TIMEOUT,
+     .op_version = {GD_OP_VERSION_3_6_0},
+     .flags = OPT_FLAG_SETTABLE,
+     .description = "After 'timeout' seconds since the time 'barrier' "
+                    "option was set to \"on\", acknowledgements to file "
+                    "operations are no longer blocked and previously "
+                    "blocked acknowledgements are sent to the application"},
+    {.key = {NULL}},
+};
+
+xlator_api_t xlator_api = {
+    .init = init,
+    .fini = fini,
+    .notify = notify,
+    .reconfigure = reconfigure,
+    .mem_acct_init = mem_acct_init,
+    .op_version = {1}, /* Present from the initial version */
+    .dumpops = &dumpops,
+    .fops = &fops,
+    .cbks = &cbks,
+    .options = options,
+    .identifier = "barrier",
+    .category = GF_MAINTAINED,
 };
diff --git a/xlators/features/barrier/src/barrier.h b/xlators/features/barrier/src/barrier.h
index 0d646f90474..1337f311f7d 100644
--- a/xlators/features/barrier/src/barrier.h
+++ b/xlators/features/barrier/src/barrier.h
@@ -12,71 +12,78 @@
 #define __BARRIER_H__
 
 #include "barrier-mem-types.h"
-#include "xlator.h"
-#include "timer.h"
-#include "call-stub.h"
+#include <glusterfs/xlator.h>
+#include <glusterfs/timer.h>
+#include <glusterfs/call-stub.h>
 
-#define BARRIER_FOP_CBK(fop_name, label, frame, this, params ...)       \
-        do {                                                            \
-                barrier_priv_t         *_priv           = NULL;         \
-                call_stub_t            *_stub           = NULL;         \
-                gf_boolean_t            _barrier_enabled= _gf_false;    \
-                struct list_head        queue           = {0, };        \
-                                                                        \
-                INIT_LIST_HEAD (&queue);                                \
-                                                                        \
-                _priv = this->private;                                  \
-                GF_ASSERT (_priv);                                      \
-                                                                        \
-                LOCK (&_priv->lock);                                    \
-                {                                                       \
-                        if (_priv->barrier_enabled) {                   \
-                                _barrier_enabled = _priv->barrier_enabled;\
-                                                                        \
-                                _stub = fop_##fop_name##_cbk_stub       \
-                                        (frame,                         \
-                                         barrier_##fop_name##_cbk_resume,\
-                                         params);                       \
-                                if (!_stub) {                           \
-                                        __barrier_disable (this, &queue);\
-                                        goto unlock;                    \
-                                }                                       \
-                                                                        \
-                                __barrier_enqueue (this, _stub);        \
-                        }                                               \
-                }                                                       \
-unlock:                                                                 \
-                UNLOCK (&_priv->lock);                                  \
-                                                                        \
-                if (_stub)                                              \
-                        goto label;                                     \
-                                                                        \
-                if (_barrier_enabled && !_stub) {                       \
-                        gf_log (this->name, GF_LOG_CRITICAL,            \
-                                "Failed to barrier FOPs, disabling "    \
-                                "barrier. FOP: %s, ERROR: %s",          \
-                                #fop_name, strerror (ENOMEM));          \
-                        barrier_dequeue_all (this, &queue);             \
-                }                                                       \
-                barrier_local_free_gfid (frame);                        \
-                STACK_UNWIND_STRICT (fop_name, frame, params);          \
-                goto label;                                             \
-        } while (0)
+#define BARRIER_FOP_CBK(fop_name, label, frame, this, params...)               \
+    do {                                                                       \
+        barrier_priv_t *_priv = NULL;                                          \
+        call_stub_t *_stub = NULL;                                             \
+        gf_boolean_t _barrier_enabled = _gf_false;                             \
+        struct list_head queue = {                                             \
+            0,                                                                 \
+        };                                                                     \
+                                                                               \
+        INIT_LIST_HEAD(&queue);                                                \
+                                                                               \
+        _priv = this->private;                                                 \
+        GF_ASSERT(_priv);                                                      \
+                                                                               \
+        LOCK(&_priv->lock);                                                    \
+        {                                                                      \
+            if (_priv->barrier_enabled) {                                      \
+                _barrier_enabled = _priv->barrier_enabled;                     \
+                                                                               \
+                _stub = fop_##fop_name##_cbk_stub(                             \
+                    frame, barrier_##fop_name##_cbk_resume, params);           \
+                if (!_stub) {                                                  \
+                    __barrier_disable(this, &queue);                           \
+                    goto unlock;                                               \
+                }                                                              \
+                                                                               \
+                __barrier_enqueue(this, _stub);                                \
+            }                                                                  \
+        }                                                                      \
+    unlock:                                                                    \
+        UNLOCK(&_priv->lock);                                                  \
+                                                                               \
+        if (_stub)                                                             \
+            goto label;                                                        \
+                                                                               \
+        if (_barrier_enabled && !_stub) {                                      \
+            gf_log(this->name, GF_LOG_CRITICAL,                                \
+                   "Failed to barrier FOPs, disabling "                        \
+                   "barrier. FOP: %s, ERROR: %s",                              \
+                   #fop_name, strerror(ENOMEM));                               \
+            barrier_dequeue_all(this, &queue);                                 \
+        }                                                                      \
+        barrier_local_free_gfid(frame);                                        \
+        STACK_UNWIND_STRICT(fop_name, frame, params);                          \
+        goto label;                                                            \
+    } while (0)
 
 typedef struct {
-        gf_timer_t       *timer;
-        gf_boolean_t      barrier_enabled;
-        gf_lock_t         lock;
-        struct list_head  queue;
-        struct timespec   timeout;
-        uint32_t          queue_size;
+    gf_timer_t *timer;
+    gf_lock_t lock;
+    struct list_head queue;
+    struct timespec timeout;
+    uint32_t queue_size;
+    gf_boolean_t barrier_enabled;
+    char _pad[3]; /* manual padding */
 } barrier_priv_t;
 
-int __barrier_enable (xlator_t *this, barrier_priv_t *priv);
-void __barrier_enqueue (xlator_t *this, call_stub_t *stub);
-void __barrier_disable (xlator_t *this, struct list_head *queue);
-void barrier_timeout (void *data);
-void barrier_dequeue_all (xlator_t *this, struct list_head *queue);
-call_stub_t *__barrier_dequeue (xlator_t *this, struct list_head *queue);
+int
+__barrier_enable(xlator_t *this, barrier_priv_t *priv);
+void
+__barrier_enqueue(xlator_t *this, call_stub_t *stub);
+void
+__barrier_disable(xlator_t *this, struct list_head *queue);
+void
+barrier_timeout(void *data);
+void
+barrier_dequeue_all(xlator_t *this, struct list_head *queue);
+call_stub_t *
+__barrier_dequeue(xlator_t *this, struct list_head *queue);
 
 #endif
diff --git a/xlators/features/bit-rot/src/bitd/Makefile.am b/xlators/features/bit-rot/src/bitd/Makefile.am
index 160e3b653df..6db800e6565 100644
--- a/xlators/features/bit-rot/src/bitd/Makefile.am
+++ b/xlators/features/bit-rot/src/bitd/Makefile.am
@@ -1,20 +1,23 @@
+if WITH_SERVER
 xlator_LTLIBRARIES = bit-rot.la
+endif
 xlatordir = $(libdir)/glusterfs/$(PACKAGE_VERSION)/xlator/features
 
-bit_rot_la_LDFLAGS = -module -avoid-version
+bit_rot_la_LDFLAGS = -module $(GF_XLATOR_DEFAULT_LDFLAGS)
 
 AM_CPPFLAGS = $(GF_CPPFLAGS) -I$(top_srcdir)/libglusterfs/src \
-		 -I$(top_srcdir)/rpc/xdr/src/ \
-		 -I$(top_srcdir)/rpc/rpc-lib/src \
-                 -I$(CONTRIBDIR)/timer-wheel \
-                 -I$(top_srcdir)/xlators/features/bit-rot/src/stub
+	-I$(top_srcdir)/rpc/xdr/src/ -I$(top_builddir)/rpc/xdr/src/ \
+	-I$(top_srcdir)/rpc/rpc-lib/src -I$(CONTRIBDIR)/timer-wheel \
+	-I$(top_srcdir)/xlators/features/bit-rot/src/stub
 
-bit_rot_la_SOURCES = bit-rot.c bit-rot-scrub.c
+bit_rot_la_SOURCES = bit-rot.c bit-rot-scrub.c bit-rot-ssm.c \
+		     bit-rot-scrub-status.c
 bit_rot_la_LIBADD = $(top_builddir)/libglusterfs/src/libglusterfs.la \
-                    $(top_builddir)/xlators/features/changelog/lib/src/libgfchangelog.la
+	$(top_builddir)/xlators/features/changelog/lib/src/libgfchangelog.la
 
-noinst_HEADERS = bit-rot.h bit-rot-scrub.h
+noinst_HEADERS = bit-rot.h bit-rot-scrub.h bit-rot-bitd-messages.h bit-rot-ssm.h \
+		 bit-rot-scrub-status.h
 
-AM_CFLAGS = -Wall $(GF_CFLAGS)
+AM_CFLAGS = -Wall -DBR_RATE_LIMIT_SIGNER $(GF_CFLAGS)
 
 CLEANFILES =
diff --git a/xlators/features/bit-rot/src/bitd/bit-rot-bitd-messages.h b/xlators/features/bit-rot/src/bitd/bit-rot-bitd-messages.h
new file mode 100644
index 00000000000..5bc5103a27c
--- /dev/null
+++ b/xlators/features/bit-rot/src/bitd/bit-rot-bitd-messages.h
@@ -0,0 +1,101 @@
+/*
+ Copyright (c) 2015 Red Hat, Inc. <http://www.redhat.com>
+ This file is part of GlusterFS.
+
+ This file is licensed to you under your choice of the GNU Lesser
+ General Public License, version 3 or any later version (LGPLv3 or
+ later), or the GNU General Public License, version 2 (GPLv2), in all
+ cases as published by the Free Software Foundation.
+ */
+
+#ifndef _BITROT_BITD_MESSAGES_H_
+#define _BITROT_BITD_MESSAGES_H_
+
+#include <glusterfs/glfs-message-id.h>
+
+/* To add new message IDs, append new identifiers at the end of the list.
+ *
+ * Never remove a message ID. If it's not used anymore, you can rename it or
+ * leave it as it is, but not delete it. This is to prevent reutilization of
+ * IDs by other messages.
+ *
+ * The component name must match one of the entries defined in
+ * glfs-message-id.h.
+ */
+
+GLFS_MSGID(BITROT_BITD, BRB_MSG_FD_CREATE_FAILED, BRB_MSG_READV_FAILED,
+           BRB_MSG_BLOCK_READ_FAILED, BRB_MSG_CALC_CHECKSUM_FAILED,
+           BRB_MSG_NO_MEMORY, BRB_MSG_GET_SIGN_FAILED, BRB_MSG_SET_SIGN_FAILED,
+           BRB_MSG_OP_FAILED, BRB_MSG_READ_AND_SIGN_FAILED, BRB_MSG_SIGN_FAILED,
+           BRB_MSG_GET_SUBVOL_FAILED, BRB_MSG_SET_TIMER_FAILED,
+           BRB_MSG_GET_INFO_FAILED, BRB_MSG_PATH_FAILED, BRB_MSG_MARK_BAD_FILE,
+           BRB_MSG_TRIGGER_SIGN, BRB_MSG_REGISTER_FAILED,
+           BRB_MSG_CRAWLING_START, BRB_MSG_SPAWN_FAILED,
+           BRB_MSG_INVALID_SUBVOL_CHILD, BRB_MSG_SKIP_OBJECT, BRB_MSG_NO_CHILD,
+           BRB_MSG_CHECKSUM_MISMATCH, BRB_MSG_MARK_CORRUPTED,
+           BRB_MSG_CRAWLING_FINISH, BRB_MSG_CALC_ERROR, BRB_MSG_LOOKUP_FAILED,
+           BRB_MSG_PARTIAL_VERSION_PRESENCE, BRB_MSG_MEM_ACNT_FAILED,
+           BRB_MSG_TIMER_WHEEL_UNAVAILABLE, BRB_MSG_BITROT_LOADED,
+           BRB_MSG_SCALE_DOWN_FAILED, BRB_MSG_SCALE_UP_FAILED,
+           BRB_MSG_SCALE_DOWN_SCRUBBER, BRB_MSG_SCALING_UP_SCRUBBER,
+           BRB_MSG_UNKNOWN_THROTTLE, BRB_MSG_RATE_LIMIT_INFO,
+           BRB_MSG_SCRUB_INFO, BRB_MSG_CONNECTED_TO_BRICK, BRB_MSG_BRICK_INFO,
+           BRB_MSG_SUBVOL_CONNECT_FAILED, BRB_MSG_INVALID_SUBVOL,
+           BRB_MSG_RESCHEDULE_SCRUBBER_FAILED, BRB_MSG_SCRUB_START,
+           BRB_MSG_SCRUB_FINISH, BRB_MSG_SCRUB_RUNNING,
+           BRB_MSG_SCRUB_RESCHEDULED, BRB_MSG_SCRUB_TUNABLE,
+           BRB_MSG_SCRUB_THREAD_CLEANUP, BRB_MSG_SCRUBBER_CLEANED,
+           BRB_MSG_GENERIC_SSM_INFO, BRB_MSG_ZERO_TIMEOUT_BUG,
+           BRB_MSG_BAD_OBJ_READDIR_FAIL, BRB_MSG_SSM_FAILED,
+           BRB_MSG_SCRUB_WAIT_FAILED, BRB_MSG_TRIGGER_SIGN_FAILED,
+           BRB_MSG_EVENT_UNHANDLED, BRB_MSG_COULD_NOT_SCHEDULE_SCRUB,
+           BRB_MSG_THREAD_CREATION_FAILED, BRB_MSG_MEM_POOL_ALLOC,
+           BRB_MSG_SAVING_HASH_FAILED);
+
+#define BRB_MSG_FD_CREATE_FAILED_STR "failed to create fd for the inode"
+#define BRB_MSG_READV_FAILED_STR "readv failed"
+#define BRB_MSG_BLOCK_READ_FAILED_STR "reading block failed"
+#define BRB_MSG_NO_MEMORY_STR "failed to allocate memory"
+#define BRB_MSG_CALC_CHECKSUM_FAILED_STR "calculating checksum failed"
+#define BRB_MSG_GET_SIGN_FAILED_STR "failed to get the signature"
+#define BRB_MSG_SET_SIGN_FAILED_STR "signing failed"
+#define BRB_MSG_OP_FAILED_STR "failed on object"
+#define BRB_MSG_TRIGGER_SIGN_FAILED_STR "Could not trigger signing"
+#define BRB_MSG_READ_AND_SIGN_FAILED_STR "reading and signing of object failed"
+#define BRB_MSG_SET_TIMER_FAILED_STR "Failed to allocate object expiry timer"
+#define BRB_MSG_GET_SUBVOL_FAILED_STR                                          \
+    "failed to get the subvolume for the brick"
+#define BRB_MSG_PATH_FAILED_STR "path failed"
+#define BRB_MSG_SKIP_OBJECT_STR "Entry is marked corrupted. skipping"
+#define BRB_MSG_PARTIAL_VERSION_PRESENCE_STR                                   \
+    "PArtial version xattr presence detected, ignoring"
+#define BRB_MSG_TRIGGER_SIGN_STR "Triggering signing"
+#define BRB_MSG_CRAWLING_START_STR                                             \
+    "Crawling brick, scanning for unsigned objects"
+#define BRB_MSG_CRAWLING_FINISH_STR "Completed crawling brick"
+#define BRB_MSG_REGISTER_FAILED_STR "Register to changelog failed"
+#define BRB_MSG_SPAWN_FAILED_STR "failed to spawn"
+#define BRB_MSG_CONNECTED_TO_BRICK_STR "Connected to brick"
+#define BRB_MSG_LOOKUP_FAILED_STR "lookup on root failed"
+#define BRB_MSG_GET_INFO_FAILED_STR "failed to get stub info"
+#define BRB_MSG_SCRUB_THREAD_CLEANUP_STR "Error cleaning up scanner thread"
+#define BRB_MSG_SCRUBBER_CLEANED_STR "clened up scrubber for brick"
+#define BRB_MSG_SUBVOL_CONNECT_FAILED_STR                                      \
+    "callback handler for subvolume failed"
+#define BRB_MSG_MEM_ACNT_FAILED_STR "Memory accounting init failed"
+#define BRB_MSG_EVENT_UNHANDLED_STR "Event unhandled for child"
+#define BRB_MSG_INVALID_SUBVOL_STR "Got event from invalid subvolume"
+#define BRB_MSG_RESCHEDULE_SCRUBBER_FAILED_STR                                 \
+    "on demand scrub schedule failed. Scrubber is not in pending state."
+#define BRB_MSG_COULD_NOT_SCHEDULE_SCRUB_STR                                   \
+    "Could not schedule ondemand scrubbing. Scrubbing will continue "          \
+    "according to old frequency."
+#define BRB_MSG_THREAD_CREATION_FAILED_STR "thread creation failed"
+#define BRB_MSG_RATE_LIMIT_INFO_STR "Rate Limit Info"
+#define BRB_MSG_MEM_POOL_ALLOC_STR "failed to allocate mem-pool for timer"
+#define BRB_MSG_NO_CHILD_STR "FATAL: no children"
+#define BRB_MSG_TIMER_WHEEL_UNAVAILABLE_STR "global timer wheel unavailable"
+#define BRB_MSG_BITROT_LOADED_STR "bit-rot xlator loaded"
+#define BRB_MSG_SAVING_HASH_FAILED_STR                                         \
+    "failed to allocate memory for saving hash of the object"
+#endif /* !_BITROT_BITD_MESSAGES_H_ */
diff --git a/xlators/features/bit-rot/src/bitd/bit-rot-scrub-status.c b/xlators/features/bit-rot/src/bitd/bit-rot-scrub-status.c
new file mode 100644
index 00000000000..5cef2ffa5e5
--- /dev/null
+++ b/xlators/features/bit-rot/src/bitd/bit-rot-scrub-status.c
@@ -0,0 +1,78 @@
+/*
+  Copyright (c) 2016 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
+*/
+
+#include <string.h>
+#include <stdio.h>
+
+#include "bit-rot-scrub-status.h"
+
+void
+br_inc_unsigned_file_count(br_scrub_stats_t *scrub_stat)
+{
+    if (!scrub_stat)
+        return;
+
+    pthread_mutex_lock(&scrub_stat->lock);
+    {
+        scrub_stat->unsigned_files++;
+    }
+    pthread_mutex_unlock(&scrub_stat->lock);
+}
+
+void
+br_inc_scrubbed_file(br_scrub_stats_t *scrub_stat)
+{
+    if (!scrub_stat)
+        return;
+
+    pthread_mutex_lock(&scrub_stat->lock);
+    {
+        scrub_stat->scrubbed_files++;
+    }
+    pthread_mutex_unlock(&scrub_stat->lock);
+}
+
+void
+br_update_scrub_start_time(br_scrub_stats_t *scrub_stat, time_t time)
+{
+    if (!scrub_stat)
+        return;
+
+    pthread_mutex_lock(&scrub_stat->lock);
+    {
+        scrub_stat->scrub_start_time = time;
+    }
+    pthread_mutex_unlock(&scrub_stat->lock);
+}
+
+void
+br_update_scrub_finish_time(br_scrub_stats_t *scrub_stat, char *timestr,
+                            time_t time)
+{
+    int lst_size = 0;
+
+    if (!scrub_stat)
+        return;
+
+    lst_size = sizeof(scrub_stat->last_scrub_time);
+    if (strlen(timestr) >= lst_size)
+        return;
+
+    pthread_mutex_lock(&scrub_stat->lock);
+    {
+        scrub_stat->scrub_end_time = time;
+
+        scrub_stat->scrub_duration = scrub_stat->scrub_end_time -
+                                     scrub_stat->scrub_start_time;
+
+        snprintf(scrub_stat->last_scrub_time, lst_size, "%s", timestr);
+    }
+    pthread_mutex_unlock(&scrub_stat->lock);
+}
diff --git a/xlators/features/bit-rot/src/bitd/bit-rot-scrub-status.h b/xlators/features/bit-rot/src/bitd/bit-rot-scrub-status.h
new file mode 100644
index 00000000000..f022aa831eb
--- /dev/null
+++ b/xlators/features/bit-rot/src/bitd/bit-rot-scrub-status.h
@@ -0,0 +1,50 @@
+/*
+   Copyright (c) 2016 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
+
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
+
+#ifndef __BIT_ROT_SCRUB_STATUS_H__
+#define __BIT_ROT_SCRUB_STATUS_H__
+
+#include <stdint.h>
+#include <sys/time.h>
+#include <pthread.h>
+
+#include <glusterfs/common-utils.h>
+
+struct br_scrub_stats {
+    uint64_t scrubbed_files; /* Total number of scrubbed files. */
+
+    uint64_t unsigned_files; /* Total number of unsigned files. */
+
+    uint64_t scrub_duration; /* Duration of last scrub. */
+
+    char last_scrub_time[GF_TIMESTR_SIZE]; /* Last scrub completion time. */
+
+    time_t scrub_start_time; /* Scrubbing starting time. */
+
+    time_t scrub_end_time; /* Scrubbing finishing time. */
+
+    int8_t scrub_running; /* Whether scrub running or not. */
+
+    pthread_mutex_t lock;
+};
+
+typedef struct br_scrub_stats br_scrub_stats_t;
+
+void
+br_inc_unsigned_file_count(br_scrub_stats_t *scrub_stat);
+void
+br_inc_scrubbed_file(br_scrub_stats_t *scrub_stat);
+void
+br_update_scrub_start_time(br_scrub_stats_t *scrub_stat, time_t time);
+void
+br_update_scrub_finish_time(br_scrub_stats_t *scrub_stat, char *timestr,
+                            time_t time);
+
+#endif /* __BIT_ROT_SCRUB_STATUS_H__ */
diff --git a/xlators/features/bit-rot/src/bitd/bit-rot-scrub.c b/xlators/features/bit-rot/src/bitd/bit-rot-scrub.c
index 1e9c2035c81..289dd53f610 100644
--- a/xlators/features/bit-rot/src/bitd/bit-rot-scrub.c
+++ b/xlators/features/bit-rot/src/bitd/bit-rot-scrub.c
@@ -8,56 +8,70 @@
    cases as published by the Free Software Foundation.
 */
 
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
-
+#include <math.h>
 #include <ctype.h>
 #include <sys/uio.h>
 
-#include "glusterfs.h"
-#include "xlator.h"
-#include "logging.h"
+#include <glusterfs/glusterfs.h>
+#include <glusterfs/logging.h>
+#include <glusterfs/common-utils.h>
 
-#include "bit-rot.h"
 #include "bit-rot-scrub.h"
 #include <pthread.h>
+#include "bit-rot-bitd-messages.h"
+#include "bit-rot-scrub-status.h"
+#include <glusterfs/events.h>
+
+struct br_scrubbers {
+    pthread_t scrubthread;
+
+    struct list_head list;
+};
+
+struct br_fsscan_entry {
+    void *data;
+
+    loc_t parent;
+
+    gf_dirent_t *entry;
+
+    struct br_scanfs *fsscan; /* backpointer to subvolume scanner */
+
+    struct list_head list;
+};
 
 /**
  * fetch signature extended attribute from an object's fd.
  * NOTE: On success @xattr is not unref'd as @sign points
  * to the dictionary value.
  */
-static inline int32_t
-bitd_fetch_signature (xlator_t *this, br_child_t *child,
-                      fd_t *fd, dict_t **xattr, br_isignature_out_t **sign)
+static int32_t
+bitd_fetch_signature(xlator_t *this, br_child_t *child, fd_t *fd,
+                     dict_t **xattr, br_isignature_out_t **sign)
 {
-       int32_t ret = -1;
-
-        ret = syncop_fgetxattr (child->xl, fd, xattr,
-                               GLUSTERFS_GET_OBJECT_SIGNATURE, NULL, NULL);
-        if (ret < 0) {
-                br_log_object (this, "fgetxattr", fd->inode->gfid, -ret);
-                goto out;
-        }
+    int32_t ret = -1;
 
-        ret = dict_get_ptr
-                (*xattr, GLUSTERFS_GET_OBJECT_SIGNATURE, (void **) sign);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "failed to extract signature info [GFID: %s]",
-                        uuid_utoa (fd->inode->gfid));
-                goto unref_dict;
-        }
+    ret = syncop_fgetxattr(child->xl, fd, xattr, GLUSTERFS_GET_OBJECT_SIGNATURE,
+                           NULL, NULL);
+    if (ret < 0) {
+        br_log_object(this, "fgetxattr", fd->inode->gfid, -ret);
+        goto out;
+    }
 
-        return 0;
+    ret = dict_get_ptr(*xattr, GLUSTERFS_GET_OBJECT_SIGNATURE, (void **)sign);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, BRB_MSG_GET_SIGN_FAILED,
+               "failed to extract signature info [GFID: %s]",
+               uuid_utoa(fd->inode->gfid));
+        goto unref_dict;
+    }
 
- unref_dict:
-        dict_unref (*xattr);
- out:
-        return -1;
+    return 0;
 
+unref_dict:
+    dict_unref(*xattr);
+out:
+    return -1;
 }
 
 /**
@@ -70,77 +84,87 @@ bitd_fetch_signature (xlator_t *this, br_child_t *child,
  */
 
 int32_t
-bitd_scrub_post_compute_check (xlator_t *this,
-                               br_child_t *child,
-                               fd_t *fd, unsigned long version,
-                               br_isignature_out_t **signature)
+bitd_scrub_post_compute_check(xlator_t *this, br_child_t *child, fd_t *fd,
+                              unsigned long version,
+                              br_isignature_out_t **signature,
+                              br_scrub_stats_t *scrub_stat,
+                              gf_boolean_t skip_stat)
 {
-        int32_t              ret     = 0;
-        size_t               signlen = 0;
-        dict_t              *xattr   = NULL;
-        br_isignature_out_t *signptr = NULL;
+    int32_t ret = 0;
+    size_t signlen = 0;
+    dict_t *xattr = NULL;
+    br_isignature_out_t *signptr = NULL;
 
-        ret = bitd_fetch_signature (this, child, fd, &xattr, &signptr);
-        if (ret < 0)
-                goto out;
+    ret = bitd_fetch_signature(this, child, fd, &xattr, &signptr);
+    if (ret < 0) {
+        if (!skip_stat)
+            br_inc_unsigned_file_count(scrub_stat);
+        goto out;
+    }
 
-        /**
-         * Either the object got dirtied during the time the signature was
-         * calculated OR the version we saved during pre-compute check does
-         * not match now, implying that the object got dirtied and signed in
-         * between scrubs pre & post compute checks (checksum window).
-         *
-         * The log entry looks pretty ugly, but helps in debugging..
-         */
-        if (signptr->stale || (signptr->version != version)) {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "<STAGE: POST> Object [GFID: %s] either has a stale "
-                        "signature OR underwent signing during checksumming "
-                        "{Stale: %d | Version: %lu,%lu}",
-                        uuid_utoa (fd->inode->gfid), (signptr->stale) ? 1 : 0,
-                        version, signptr->version);
-                ret = -1;
-                goto unref_dict;
-        }
+    /**
+     * Either the object got dirtied during the time the signature was
+     * calculated OR the version we saved during pre-compute check does
+     * not match now, implying that the object got dirtied and signed in
+     * between scrubs pre & post compute checks (checksum window).
+     *
+     * The log entry looks pretty ugly, but helps in debugging..
+     */
+    if (signptr->stale || (signptr->version != version)) {
+        if (!skip_stat)
+            br_inc_unsigned_file_count(scrub_stat);
+        gf_msg_debug(this->name, 0,
+                     "<STAGE: POST> Object [GFID: %s] "
+                     "either has a stale signature OR underwent "
+                     "signing during checksumming {Stale: %d | "
+                     "Version: %lu,%lu}",
+                     uuid_utoa(fd->inode->gfid), (signptr->stale) ? 1 : 0,
+                     version, signptr->version);
+        ret = -1;
+        goto unref_dict;
+    }
 
-        signlen = signptr->signaturelen;
-        *signature = GF_CALLOC (1, sizeof (br_isignature_out_t) + signlen,
-                                gf_common_mt_char);
+    signlen = signptr->signaturelen;
+    *signature = GF_MALLOC(sizeof(br_isignature_out_t) + signlen,
+                           gf_common_mt_char);
 
-        (void) memcpy (*signature, signptr,
-                       sizeof (br_isignature_out_t) + signlen);
+    (void)memcpy(*signature, signptr, sizeof(br_isignature_out_t) + signlen);
 
- unref_dict:
-        dict_unref (xattr);
- out:
-        return ret;
+    (*signature)->signaturelen = signlen;
 
+unref_dict:
+    dict_unref(xattr);
+out:
+    return ret;
 }
 
-static inline int32_t
-bitd_signature_staleness (xlator_t *this,
-                          br_child_t *child, fd_t *fd,
-                          int *stale, unsigned long *version)
+static int32_t
+bitd_signature_staleness(xlator_t *this, br_child_t *child, fd_t *fd,
+                         int *stale, unsigned long *version,
+                         br_scrub_stats_t *scrub_stat, gf_boolean_t skip_stat)
 {
-        int32_t ret = -1;
-        dict_t *xattr = NULL;
-        br_isignature_out_t *signptr = NULL;
+    int32_t ret = -1;
+    dict_t *xattr = NULL;
+    br_isignature_out_t *signptr = NULL;
 
-        ret = bitd_fetch_signature (this, child, fd, &xattr, &signptr);
-        if (ret < 0)
-                goto out;
+    ret = bitd_fetch_signature(this, child, fd, &xattr, &signptr);
+    if (ret < 0) {
+        if (!skip_stat)
+            br_inc_unsigned_file_count(scrub_stat);
+        goto out;
+    }
 
-        /**
-         * save verison for validation in post compute stage
-         * c.f. bitd_scrub_post_compute_check()
-         */
-        *stale = signptr->stale ? 1 : 0;
-        *version = signptr->version;
+    /**
+     * save version for validation in post compute stage
+     * c.f. bitd_scrub_post_compute_check()
+     */
+    *stale = signptr->stale ? 1 : 0;
+    *version = signptr->version;
 
-        dict_unref (xattr);
+    dict_unref(xattr);
 
- out:
-        return ret;
+out:
+    return ret;
 }
 
 /**
@@ -152,88 +176,102 @@ bitd_signature_staleness (xlator_t *this,
  *  - has stale signature
  */
 int32_t
-bitd_scrub_pre_compute_check (xlator_t *this, br_child_t *child,
-                              fd_t *fd, unsigned long *version)
+bitd_scrub_pre_compute_check(xlator_t *this, br_child_t *child, fd_t *fd,
+                             unsigned long *version,
+                             br_scrub_stats_t *scrub_stat,
+                             gf_boolean_t skip_stat)
 {
-        int     stale = 0;
-        int32_t ret   = -1;
+    int stale = 0;
+    int32_t ret = -1;
 
-        if (bitd_is_bad_file (this, child, NULL, fd)) {
-                gf_log (this->name, GF_LOG_WARNING,
-                        "Object [GFID: %s] is marked corrupted, skipping..",
-                        uuid_utoa (fd->inode->gfid));
-                goto out;
-        }
+    if (bitd_is_bad_file(this, child, NULL, fd)) {
+        gf_msg(this->name, GF_LOG_WARNING, 0, BRB_MSG_SKIP_OBJECT,
+               "Object [GFID: %s] is marked corrupted, skipping..",
+               uuid_utoa(fd->inode->gfid));
+        goto out;
+    }
 
-        ret = bitd_signature_staleness (this, child, fd, &stale, version);
-        if (!ret && stale) {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "<STAGE: PRE> Object [GFID: %s] has stale signature",
-                        uuid_utoa (fd->inode->gfid));
-                ret = -1;
-        }
+    ret = bitd_signature_staleness(this, child, fd, &stale, version, scrub_stat,
+                                   skip_stat);
+    if (!ret && stale) {
+        if (!skip_stat)
+            br_inc_unsigned_file_count(scrub_stat);
+        gf_msg_debug(this->name, 0,
+                     "<STAGE: PRE> Object [GFID: %s] "
+                     "has stale signature",
+                     uuid_utoa(fd->inode->gfid));
+        ret = -1;
+    }
 
- out:
-        return ret;
+out:
+    return ret;
 }
 
-/* static inline int */
+/* static int */
 int
-bitd_compare_ckum (xlator_t *this,
-                   br_isignature_out_t *sign,
-                   unsigned char *md, inode_t *linked_inode,
-                   gf_dirent_t *entry, fd_t *fd, br_child_t *child)
-{
-        int   ret = -1;
-        dict_t *xattr = NULL;
-
-        GF_VALIDATE_OR_GOTO ("bit-rot", this, out);
-        GF_VALIDATE_OR_GOTO (this->name, sign, out);
-        GF_VALIDATE_OR_GOTO (this->name, fd, out);
-        GF_VALIDATE_OR_GOTO (this->name, child, out);
-        GF_VALIDATE_OR_GOTO (this->name, linked_inode, out);
-        GF_VALIDATE_OR_GOTO (this->name, md, out);
-        GF_VALIDATE_OR_GOTO (this->name, entry, out);
-
-        if (strncmp
-            (sign->signature, (char *) md, strlen (sign->signature)) == 0) {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "Entry %s [GFID: %s] matches calculated checksum",
-                        entry->d_name, uuid_utoa (linked_inode->gfid));
-                return 0;
-        }
+bitd_compare_ckum(xlator_t *this, br_isignature_out_t *sign, unsigned char *md,
+                  inode_t *linked_inode, gf_dirent_t *entry, fd_t *fd,
+                  br_child_t *child, loc_t *loc)
+{
+    int ret = -1;
+    dict_t *xattr = NULL;
 
-        gf_log (this->name, GF_LOG_WARNING,
-                "Object checksumsum mismatch: %s [GFID: %s]",
-                entry->d_name, uuid_utoa (linked_inode->gfid));
+    GF_VALIDATE_OR_GOTO("bit-rot", this, out);
+    GF_VALIDATE_OR_GOTO(this->name, sign, out);
+    GF_VALIDATE_OR_GOTO(this->name, fd, out);
+    GF_VALIDATE_OR_GOTO(this->name, child, out);
+    GF_VALIDATE_OR_GOTO(this->name, linked_inode, out);
+    GF_VALIDATE_OR_GOTO(this->name, md, out);
+    GF_VALIDATE_OR_GOTO(this->name, entry, out);
 
-        /* Perform bad-file marking */
-        xattr = dict_new ();
-        if (!xattr) {
-                ret = -1;
-                goto out;
-        }
+    if (strncmp(sign->signature, (char *)md, sign->signaturelen) == 0) {
+        gf_msg_debug(this->name, 0,
+                     "%s [GFID: %s | Brick: %s] "
+                     "matches calculated checksum",
+                     loc->path, uuid_utoa(linked_inode->gfid),
+                     child->brick_path);
+        return 0;
+    }
 
-        ret = dict_set_int32 (xattr, "trusted.glusterfs.bad-file", _gf_true);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "Error setting bad-file marker for %s [GFID: %s]",
-                        entry->d_name, uuid_utoa (linked_inode->gfid));
-                goto dictfree;
-        }
+    gf_msg(this->name, GF_LOG_DEBUG, 0, BRB_MSG_CHECKSUM_MISMATCH,
+           "Object checksum mismatch: %s [GFID: %s | Brick: %s]", loc->path,
+           uuid_utoa(linked_inode->gfid), child->brick_path);
+    gf_msg(this->name, GF_LOG_ALERT, 0, BRB_MSG_CHECKSUM_MISMATCH,
+           "CORRUPTION DETECTED: Object %s {Brick: %s | GFID: %s}", loc->path,
+           child->brick_path, uuid_utoa(linked_inode->gfid));
 
-        gf_log (this->name, GF_LOG_INFO, "Marking %s [GFID: %s] as corrupted..",
-                entry->d_name, uuid_utoa (linked_inode->gfid));
-        ret = syncop_fsetxattr (child->xl, fd, xattr, 0, NULL, NULL);
-        if (ret)
-                gf_log (this->name, GF_LOG_ERROR,
-                        "Error marking object %s [GFID: %s] as corrupted",
-                        entry->d_name, uuid_utoa (linked_inode->gfid));
+    /* Perform bad-file marking */
+    xattr = dict_new();
+    if (!xattr) {
+        ret = -1;
+        goto out;
+    }
+
+    ret = dict_set_int32(xattr, BITROT_OBJECT_BAD_KEY, _gf_true);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, BRB_MSG_MARK_BAD_FILE,
+               "Error setting bad-file marker for %s [GFID: %s | "
+               "Brick: %s]",
+               loc->path, uuid_utoa(linked_inode->gfid), child->brick_path);
+        goto dictfree;
+    }
 
- dictfree:
-        dict_unref (xattr);
- out:
-        return ret;
+    gf_msg(this->name, GF_LOG_ALERT, 0, BRB_MSG_MARK_CORRUPTED,
+           "Marking"
+           " %s [GFID: %s | Brick: %s] as corrupted..",
+           loc->path, uuid_utoa(linked_inode->gfid), child->brick_path);
+    gf_event(EVENT_BITROT_BAD_FILE, "gfid=%s;path=%s;brick=%s",
+             uuid_utoa(linked_inode->gfid), loc->path, child->brick_path);
+    ret = syncop_fsetxattr(child->xl, fd, xattr, 0, NULL, NULL);
+    if (ret)
+        gf_msg(this->name, GF_LOG_ERROR, 0, BRB_MSG_MARK_BAD_FILE,
+               "Error marking object %s [GFID: %s] as corrupted", loc->path,
+               uuid_utoa(linked_inode->gfid));
+
+dictfree:
+    dict_unref(xattr);
+out:
+    return ret;
 }
 
 /**
@@ -244,149 +282,1789 @@ bitd_compare_ckum (xlator_t *this,
  * signs with SHA256).
  */
 int
-bitd_start_scrub (xlator_t *subvol,
-                  gf_dirent_t *entry, loc_t *parent, void *data)
-{
-        int32_t              ret           = -1;
-        fd_t                *fd            = NULL;
-        loc_t                loc           = {0, };
-        struct iatt          iatt          = {0, };
-        struct iatt          parent_buf    = {0, };
-        pid_t                pid           = 0;
-        br_child_t          *child         = NULL;
-        xlator_t            *this          = NULL;
-        unsigned char       *md            = NULL;
-        inode_t             *linked_inode  = NULL;
-        br_isignature_out_t *sign          = NULL;
-        unsigned long        signedversion = 0;
-
-        GF_VALIDATE_OR_GOTO ("bit-rot", subvol, out);
-        GF_VALIDATE_OR_GOTO ("bit-rot", data, out);
-
-        child = data;
-        this = child->this;
-
-        pid = GF_CLIENT_PID_SCRUB;
-
-        ret = br_prepare_loc (this, child, parent, entry, &loc);
-        if (!ret)
-                goto out;
+br_scrubber_scrub_begin(xlator_t *this, struct br_fsscan_entry *fsentry)
+{
+    int32_t ret = -1;
+    fd_t *fd = NULL;
+    loc_t loc = {
+        0,
+    };
+    struct iatt iatt = {
+        0,
+    };
+    struct iatt parent_buf = {
+        0,
+    };
+    pid_t pid = 0;
+    br_child_t *child = NULL;
+    unsigned char *md = NULL;
+    inode_t *linked_inode = NULL;
+    br_isignature_out_t *sign = NULL;
+    unsigned long signedversion = 0;
+    gf_dirent_t *entry = NULL;
+    br_private_t *priv = NULL;
+    loc_t *parent = NULL;
+    gf_boolean_t skip_stat = _gf_false;
+    uuid_t shard_root_gfid = {
+        0,
+    };
 
-        syncopctx_setfspid (&pid);
+    GF_VALIDATE_OR_GOTO("bit-rot", fsentry, out);
 
-        ret = syncop_lookup (child->xl, &loc, &iatt, &parent_buf, NULL, NULL);
-        if (ret) {
-                br_log_object_path (this, "lookup", loc.path, -ret);
-                goto out;
+    entry = fsentry->entry;
+    parent = &fsentry->parent;
+    child = fsentry->data;
+
+    priv = this->private;
+
+    GF_VALIDATE_OR_GOTO("bit-rot", entry, out);
+    GF_VALIDATE_OR_GOTO("bit-rot", parent, out);
+    GF_VALIDATE_OR_GOTO("bit-rot", child, out);
+    GF_VALIDATE_OR_GOTO("bit-rot", priv, out);
+
+    pid = GF_CLIENT_PID_SCRUB;
+
+    ret = br_prepare_loc(this, child, parent, entry, &loc);
+    if (!ret)
+        goto out;
+
+    syncopctx_setfspid(&pid);
+
+    ret = syncop_lookup(child->xl, &loc, &iatt, &parent_buf, NULL, NULL);
+    if (ret) {
+        br_log_object_path(this, "lookup", loc.path, -ret);
+        goto out;
+    }
+
+    linked_inode = inode_link(loc.inode, parent->inode, loc.name, &iatt);
+    if (linked_inode)
+        inode_lookup(linked_inode);
+
+    gf_msg_debug(this->name, 0, "Scrubbing object %s [GFID: %s]", entry->d_name,
+                 uuid_utoa(linked_inode->gfid));
+
+    if (iatt.ia_type != IA_IFREG) {
+        gf_msg_debug(this->name, 0, "%s is not a regular file", entry->d_name);
+        ret = 0;
+        goto unref_inode;
+    }
+
+    if (IS_DHT_LINKFILE_MODE((&iatt))) {
+        gf_msg_debug(this->name, 0, "%s is a dht sticky bit file",
+                     entry->d_name);
+        ret = 0;
+        goto unref_inode;
+    }
+
+    /* skip updating scrub statistics for shard entries */
+    gf_uuid_parse(SHARD_ROOT_GFID, shard_root_gfid);
+    if (gf_uuid_compare(loc.pargfid, shard_root_gfid) == 0)
+        skip_stat = _gf_true;
+
+    /**
+     * open() an fd for subsequent operations
+     */
+    fd = fd_create(linked_inode, 0);
+    if (!fd) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, BRB_MSG_FD_CREATE_FAILED,
+               "failed to create fd for inode %s",
+               uuid_utoa(linked_inode->gfid));
+        goto unref_inode;
+    }
+
+    ret = syncop_open(child->xl, &loc, O_RDWR, fd, NULL, NULL);
+    if (ret) {
+        br_log_object(this, "open", linked_inode->gfid, -ret);
+        ret = -1;
+        goto unrefd;
+    }
+
+    fd_bind(fd);
+
+    /**
+     * perform pre compute checks before initiating checksum
+     * computation
+     *  - presence of bad object
+     *  - signature staleness
+     */
+    ret = bitd_scrub_pre_compute_check(this, child, fd, &signedversion,
+                                       &priv->scrub_stat, skip_stat);
+    if (ret)
+        goto unrefd; /* skip this object */
+
+    /* if all's good, proceed to calculate the hash */
+    md = GF_MALLOC(SHA256_DIGEST_LENGTH, gf_common_mt_char);
+    if (!md)
+        goto unrefd;
+
+    ret = br_calculate_obj_checksum(md, child, fd, &iatt);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, BRB_MSG_CALC_ERROR,
+               "error calculating hash for object [GFID: %s]",
+               uuid_utoa(fd->inode->gfid));
+        ret = -1;
+        goto free_md;
+    }
+
+    /**
+     * perform post compute checks as an object's signature may have
+     * become stale while scrubber calculated checksum.
+     */
+    ret = bitd_scrub_post_compute_check(this, child, fd, signedversion, &sign,
+                                        &priv->scrub_stat, skip_stat);
+    if (ret)
+        goto free_md;
+
+    ret = bitd_compare_ckum(this, sign, md, linked_inode, entry, fd, child,
+                            &loc);
+
+    if (!skip_stat)
+        br_inc_scrubbed_file(&priv->scrub_stat);
+
+    GF_FREE(sign); /* allocated on post-compute */
+
+    /** fd_unref() takes care of closing fd.. like syncop_close() */
+
+free_md:
+    GF_FREE(md);
+unrefd:
+    fd_unref(fd);
+unref_inode:
+    inode_unref(linked_inode);
+out:
+    loc_wipe(&loc);
+    return ret;
+}
+
+static void
+_br_lock_cleaner(void *arg)
+{
+    pthread_mutex_t *mutex = arg;
+
+    pthread_mutex_unlock(mutex);
+}
+
+static void
+wait_for_scrubbing(xlator_t *this, struct br_scanfs *fsscan)
+{
+    br_private_t *priv = NULL;
+    struct br_scrubber *fsscrub = NULL;
+
+    priv = this->private;
+    fsscrub = &priv->fsscrub;
+
+    pthread_cleanup_push(_br_lock_cleaner, &fsscan->waitlock);
+    pthread_mutex_lock(&fsscan->waitlock);
+    {
+        pthread_cleanup_push(_br_lock_cleaner, &fsscrub->mutex);
+        pthread_mutex_lock(&fsscrub->mutex);
+        {
+            list_replace_init(&fsscan->queued, &fsscan->ready);
+
+            /* wake up scrubbers */
+            pthread_cond_broadcast(&fsscrub->cond);
         }
+        pthread_mutex_unlock(&fsscrub->mutex);
+        pthread_cleanup_pop(0);
 
-        linked_inode = inode_link (loc.inode, parent->inode, loc.name, &iatt);
-        if (linked_inode)
-                inode_lookup (linked_inode);
+        while (fsscan->entries != 0)
+            pthread_cond_wait(&fsscan->waitcond, &fsscan->waitlock);
+    }
+    pthread_mutex_unlock(&fsscan->waitlock);
+    pthread_cleanup_pop(0);
+}
 
-        gf_log (this->name, GF_LOG_DEBUG, "Scrubbing object %s [GFID: %s]",
-                entry->d_name, uuid_utoa (linked_inode->gfid));
+static void
+_br_fsscan_inc_entry_count(struct br_scanfs *fsscan)
+{
+    fsscan->entries++;
+}
 
-        if (iatt.ia_type != IA_IFREG) {
-                gf_log (this->name, GF_LOG_DEBUG, "%s is not a regular "
-                        "file", entry->d_name);
-                ret = 0;
-                goto unref_inode;
+static void
+_br_fsscan_dec_entry_count(struct br_scanfs *fsscan)
+{
+    if (--fsscan->entries == 0) {
+        pthread_mutex_lock(&fsscan->waitlock);
+        {
+            pthread_cond_signal(&fsscan->waitcond);
         }
+        pthread_mutex_unlock(&fsscan->waitlock);
+    }
+}
+
+static void
+_br_fsscan_collect_entry(struct br_scanfs *fsscan,
+                         struct br_fsscan_entry *fsentry)
+{
+    list_add_tail(&fsentry->list, &fsscan->queued);
+    _br_fsscan_inc_entry_count(fsscan);
+}
+
+#define NR_ENTRIES (1 << 7) /* ..bulk scrubbing */
+
+int
+br_fsscanner_handle_entry(xlator_t *subvol, gf_dirent_t *entry, loc_t *parent,
+                          void *data)
+{
+    int32_t ret = -1;
+    int scrub = 0;
+    br_child_t *child = NULL;
+    xlator_t *this = NULL;
+    struct br_scanfs *fsscan = NULL;
+    struct br_fsscan_entry *fsentry = NULL;
+
+    GF_VALIDATE_OR_GOTO("bit-rot", subvol, error_return);
+    GF_VALIDATE_OR_GOTO("bit-rot", data, error_return);
+
+    child = data;
+    this = child->this;
+    fsscan = &child->fsscan;
+
+    _mask_cancellation();
+
+    fsentry = GF_CALLOC(1, sizeof(*fsentry), gf_br_mt_br_fsscan_entry_t);
+    if (!fsentry)
+        goto error_return;
+
+    {
+        fsentry->data = data;
+        fsentry->fsscan = &child->fsscan;
+
+        /* copy parent loc */
+        ret = loc_copy(&fsentry->parent, parent);
+        if (ret)
+            goto dealloc;
+
+        /* copy child entry */
+        fsentry->entry = entry_copy(entry);
+        if (!fsentry->entry)
+            goto locwipe;
+
+        INIT_LIST_HEAD(&fsentry->list);
+    }
+
+    LOCK(&fsscan->entrylock);
+    {
+        _br_fsscan_collect_entry(fsscan, fsentry);
 
         /**
-         * open() an fd for subsequent opertaions
+         * need not be a equality check as entries may be pushed
+         * back onto the scanned queue when thread(s) are cleaned.
          */
-        fd = fd_create (linked_inode, 0);
-        if (!fd) {
-                gf_log (this->name, GF_LOG_ERROR, "failed to create fd for "
-                        "inode %s", uuid_utoa (linked_inode->gfid));
-                goto unref_inode;
+        if (fsscan->entries >= NR_ENTRIES)
+            scrub = 1;
+    }
+    UNLOCK(&fsscan->entrylock);
+
+    _unmask_cancellation();
+
+    if (scrub)
+        wait_for_scrubbing(this, fsscan);
+
+    return 0;
+
+locwipe:
+    loc_wipe(&fsentry->parent);
+dealloc:
+    GF_FREE(fsentry);
+error_return:
+    return -1;
+}
+
+int32_t
+br_fsscan_deactivate(xlator_t *this)
+{
+    int ret = 0;
+    br_private_t *priv = NULL;
+    br_scrub_state_t nstate = 0;
+    struct br_monitor *scrub_monitor = NULL;
+
+    priv = this->private;
+    scrub_monitor = &priv->scrub_monitor;
+
+    ret = gf_tw_del_timer(priv->timer_wheel, scrub_monitor->timer);
+    if (ret == 0) {
+        nstate = BR_SCRUB_STATE_STALLED;
+        gf_msg(this->name, GF_LOG_INFO, 0, BRB_MSG_SCRUB_INFO,
+               "Volume is under active scrubbing. Pausing scrub..");
+    } else {
+        nstate = BR_SCRUB_STATE_PAUSED;
+        gf_msg(this->name, GF_LOG_INFO, 0, BRB_MSG_SCRUB_INFO,
+               "Scrubber paused");
+    }
+
+    _br_monitor_set_scrub_state(scrub_monitor, nstate);
+
+    return 0;
+}
+
+static void
+br_scrubber_log_time(xlator_t *this, const char *sfx)
+{
+    char timestr[GF_TIMESTR_SIZE] = {
+        0,
+    };
+    br_private_t *priv = NULL;
+    time_t now = 0;
+
+    now = gf_time();
+    priv = this->private;
+
+    gf_time_fmt(timestr, sizeof(timestr), now, gf_timefmt_FT);
+
+    if (strcasecmp(sfx, "started") == 0) {
+        br_update_scrub_start_time(&priv->scrub_stat, now);
+        gf_msg(this->name, GF_LOG_INFO, 0, BRB_MSG_SCRUB_START,
+               "Scrubbing %s at %s", sfx, timestr);
+    } else {
+        br_update_scrub_finish_time(&priv->scrub_stat, timestr, now);
+        gf_msg(this->name, GF_LOG_INFO, 0, BRB_MSG_SCRUB_FINISH,
+               "Scrubbing %s at %s", sfx, timestr);
+    }
+}
+
+static void
+br_fsscanner_log_time(xlator_t *this, br_child_t *child, const char *sfx)
+{
+    char timestr[GF_TIMESTR_SIZE] = {
+        0,
+    };
+    time_t now = 0;
+
+    now = gf_time();
+    gf_time_fmt(timestr, sizeof(timestr), now, gf_timefmt_FT);
+
+    if (strcasecmp(sfx, "started") == 0) {
+        gf_msg_debug(this->name, 0, "Scrubbing \"%s\" %s at %s",
+                     child->brick_path, sfx, timestr);
+    } else {
+        gf_msg_debug(this->name, 0, "Scrubbing \"%s\" %s at %s",
+                     child->brick_path, sfx, timestr);
+    }
+}
+
+void
+br_child_set_scrub_state(br_child_t *child, gf_boolean_t state)
+{
+    child->active_scrubbing = state;
+}
+
+static void
+br_fsscanner_wait_until_kicked(xlator_t *this, br_child_t *child)
+{
+    br_private_t *priv = NULL;
+    struct br_monitor *scrub_monitor = NULL;
+
+    priv = this->private;
+    scrub_monitor = &priv->scrub_monitor;
+
+    pthread_cleanup_push(_br_lock_cleaner, &scrub_monitor->wakelock);
+    pthread_mutex_lock(&scrub_monitor->wakelock);
+    {
+        while (!scrub_monitor->kick)
+            pthread_cond_wait(&scrub_monitor->wakecond,
+                              &scrub_monitor->wakelock);
+
+        /* Child lock is to synchronize with disconnect events */
+        pthread_cleanup_push(_br_lock_cleaner, &child->lock);
+        pthread_mutex_lock(&child->lock);
+        {
+            scrub_monitor->active_child_count++;
+            br_child_set_scrub_state(child, _gf_true);
         }
+        pthread_mutex_unlock(&child->lock);
+        pthread_cleanup_pop(0);
+    }
+    pthread_mutex_unlock(&scrub_monitor->wakelock);
+    pthread_cleanup_pop(0);
+}
 
-        ret = syncop_open (child->xl, &loc, O_RDWR, fd, NULL, NULL);
-        if (ret) {
-                br_log_object (this, "open", linked_inode->gfid, -ret);
-                ret = -1;
-                goto unrefd;
+static void
+br_scrubber_entry_control(xlator_t *this)
+{
+    br_private_t *priv = NULL;
+    struct br_monitor *scrub_monitor = NULL;
+
+    priv = this->private;
+    scrub_monitor = &priv->scrub_monitor;
+
+    LOCK(&scrub_monitor->lock);
+    {
+        /* Move the state to BR_SCRUB_STATE_ACTIVE */
+        if (scrub_monitor->state == BR_SCRUB_STATE_PENDING)
+            scrub_monitor->state = BR_SCRUB_STATE_ACTIVE;
+        br_scrubber_log_time(this, "started");
+        priv->scrub_stat.scrub_running = 1;
+    }
+    UNLOCK(&scrub_monitor->lock);
+}
+
+static void
+br_scrubber_exit_control(xlator_t *this)
+{
+    br_private_t *priv = NULL;
+    struct br_monitor *scrub_monitor = NULL;
+
+    priv = this->private;
+    scrub_monitor = &priv->scrub_monitor;
+
+    LOCK(&scrub_monitor->lock);
+    {
+        br_scrubber_log_time(this, "finished");
+        priv->scrub_stat.scrub_running = 0;
+
+        if (scrub_monitor->state == BR_SCRUB_STATE_ACTIVE) {
+            (void)br_fsscan_activate(this);
+        } else {
+            UNLOCK(&scrub_monitor->lock);
+            gf_msg(this->name, GF_LOG_INFO, 0, BRB_MSG_SCRUB_INFO,
+                   "Volume waiting to get rescheduled..");
+            return;
+        }
+    }
+    UNLOCK(&scrub_monitor->lock);
+}
+
+static void
+br_fsscanner_entry_control(xlator_t *this, br_child_t *child)
+{
+    br_fsscanner_log_time(this, child, "started");
+}
+
+static void
+br_fsscanner_exit_control(xlator_t *this, br_child_t *child)
+{
+    br_private_t *priv = NULL;
+    struct br_monitor *scrub_monitor = NULL;
+
+    priv = this->private;
+    scrub_monitor = &priv->scrub_monitor;
+
+    if (!_br_is_child_connected(child)) {
+        gf_msg(this->name, GF_LOG_WARNING, 0, BRB_MSG_SCRUB_INFO,
+               "Brick [%s] disconnected while scrubbing. Scrubbing "
+               "might be incomplete",
+               child->brick_path);
+    }
+
+    br_fsscanner_log_time(this, child, "finished");
+
+    pthread_cleanup_push(_br_lock_cleaner, &scrub_monitor->wakelock);
+    pthread_mutex_lock(&scrub_monitor->wakelock);
+    {
+        scrub_monitor->active_child_count--;
+        pthread_cleanup_push(_br_lock_cleaner, &child->lock);
+        pthread_mutex_lock(&child->lock);
+        {
+            br_child_set_scrub_state(child, _gf_false);
         }
+        pthread_mutex_unlock(&child->lock);
+        pthread_cleanup_pop(0);
 
-        fd_bind (fd);
+        if (scrub_monitor->active_child_count == 0) {
+            /* The last child has finished scrubbing.
+             * Set the kick to false and  wake up other
+             * children who are waiting for the last
+             * child to complete scrubbing.
+             */
+            scrub_monitor->kick = _gf_false;
+            pthread_cond_broadcast(&scrub_monitor->wakecond);
 
-        /**
-         * perform pre compute checks before initiating checksum
-         * computation
-         *  - presence of bad object
-         *  - signature staleness
-         */
-        ret = bitd_scrub_pre_compute_check (this, child, fd, &signedversion);
+            /* Signal monitor thread waiting for the all
+             * the children to finish scrubbing.
+             */
+            pthread_cleanup_push(_br_lock_cleaner, &scrub_monitor->donelock);
+            pthread_mutex_lock(&scrub_monitor->donelock);
+            {
+                scrub_monitor->done = _gf_true;
+                pthread_cond_signal(&scrub_monitor->donecond);
+            }
+            pthread_mutex_unlock(&scrub_monitor->donelock);
+            pthread_cleanup_pop(0);
+        } else {
+            while (scrub_monitor->active_child_count)
+                pthread_cond_wait(&scrub_monitor->wakecond,
+                                  &scrub_monitor->wakelock);
+        }
+    }
+    pthread_mutex_unlock(&scrub_monitor->wakelock);
+    pthread_cleanup_pop(0);
+}
+
+void *
+br_fsscanner(void *arg)
+{
+    loc_t loc = {
+        0,
+    };
+    br_child_t *child = NULL;
+    xlator_t *this = NULL;
+    struct br_scanfs *fsscan = NULL;
+
+    child = arg;
+    this = child->this;
+    fsscan = &child->fsscan;
+
+    THIS = this;
+    loc.inode = child->table->root;
+
+    while (1) {
+        br_fsscanner_wait_until_kicked(this, child);
+        {
+            /* precursor for scrub */
+            br_fsscanner_entry_control(this, child);
+
+            /* scrub */
+            (void)syncop_ftw(child->xl, &loc, GF_CLIENT_PID_SCRUB, child,
+                             br_fsscanner_handle_entry);
+            if (!list_empty(&fsscan->queued))
+                wait_for_scrubbing(this, fsscan);
+
+            /* scrub exit criteria */
+            br_fsscanner_exit_control(this, child);
+        }
+    }
+
+    return NULL;
+}
+
+/**
+ * Keep this routine extremely simple and do not ever try to acquire
+ * child->lock here: it may lead to deadlock. Scrubber state is
+ * modified in br_fsscanner(). An intermediate state change to pause
+ * changes the scrub state to the _correct_ state by identifying a
+ * non-pending timer.
+ */
+void
+br_kickstart_scanner(struct gf_tw_timer_list *timer, void *data,
+                     unsigned long calltime)
+{
+    xlator_t *this = NULL;
+    struct br_monitor *scrub_monitor = data;
+    br_private_t *priv = NULL;
+
+    THIS = this = scrub_monitor->this;
+    priv = this->private;
+
+    /* Reset scrub statistics */
+    priv->scrub_stat.scrubbed_files = 0;
+    priv->scrub_stat.unsigned_files = 0;
+
+    /* Moves state from PENDING to ACTIVE */
+    (void)br_scrubber_entry_control(this);
+
+    /* kickstart scanning.. */
+    pthread_mutex_lock(&scrub_monitor->wakelock);
+    {
+        scrub_monitor->kick = _gf_true;
+        GF_ASSERT(scrub_monitor->active_child_count == 0);
+        pthread_cond_broadcast(&scrub_monitor->wakecond);
+    }
+    pthread_mutex_unlock(&scrub_monitor->wakelock);
+
+    return;
+}
+
+static uint32_t
+br_fsscan_calculate_delta(uint32_t times)
+{
+    return times;
+}
+
+#define BR_SCRUB_ONDEMAND (1)
+#define BR_SCRUB_MINUTE (60)
+#define BR_SCRUB_HOURLY (60 * 60)
+#define BR_SCRUB_DAILY (1 * 24 * 60 * 60)
+#define BR_SCRUB_WEEKLY (7 * 24 * 60 * 60)
+#define BR_SCRUB_BIWEEKLY (14 * 24 * 60 * 60)
+#define BR_SCRUB_MONTHLY (30 * 24 * 60 * 60)
+
+static unsigned int
+br_fsscan_calculate_timeout(scrub_freq_t freq)
+{
+    uint32_t timo = 0;
+
+    switch (freq) {
+        case BR_FSSCRUB_FREQ_MINUTE:
+            timo = br_fsscan_calculate_delta(BR_SCRUB_MINUTE);
+            break;
+        case BR_FSSCRUB_FREQ_HOURLY:
+            timo = br_fsscan_calculate_delta(BR_SCRUB_HOURLY);
+            break;
+        case BR_FSSCRUB_FREQ_DAILY:
+            timo = br_fsscan_calculate_delta(BR_SCRUB_DAILY);
+            break;
+        case BR_FSSCRUB_FREQ_WEEKLY:
+            timo = br_fsscan_calculate_delta(BR_SCRUB_WEEKLY);
+            break;
+        case BR_FSSCRUB_FREQ_BIWEEKLY:
+            timo = br_fsscan_calculate_delta(BR_SCRUB_BIWEEKLY);
+            break;
+        case BR_FSSCRUB_FREQ_MONTHLY:
+            timo = br_fsscan_calculate_delta(BR_SCRUB_MONTHLY);
+            break;
+        default:
+            timo = 0;
+    }
+
+    return timo;
+}
+
+int32_t
+br_fsscan_schedule(xlator_t *this)
+{
+    uint32_t timo = 0;
+    br_private_t *priv = NULL;
+    char timestr[GF_TIMESTR_SIZE] = {
+        0,
+    };
+    struct br_scrubber *fsscrub = NULL;
+    struct gf_tw_timer_list *timer = NULL;
+    struct br_monitor *scrub_monitor = NULL;
+
+    priv = this->private;
+    fsscrub = &priv->fsscrub;
+    scrub_monitor = &priv->scrub_monitor;
+
+    scrub_monitor->boot = gf_time();
+
+    timo = br_fsscan_calculate_timeout(fsscrub->frequency);
+    if (timo == 0) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, BRB_MSG_ZERO_TIMEOUT_BUG,
+               "BUG: Zero schedule timeout");
+        goto error_return;
+    }
+
+    scrub_monitor->timer = GF_CALLOC(1, sizeof(*scrub_monitor->timer),
+                                     gf_br_stub_mt_br_scanner_freq_t);
+    if (!scrub_monitor->timer)
+        goto error_return;
+
+    timer = scrub_monitor->timer;
+    INIT_LIST_HEAD(&timer->entry);
+
+    timer->data = scrub_monitor;
+    timer->expires = timo;
+    timer->function = br_kickstart_scanner;
+
+    gf_tw_add_timer(priv->timer_wheel, timer);
+    _br_monitor_set_scrub_state(scrub_monitor, BR_SCRUB_STATE_PENDING);
+
+    gf_time_fmt(timestr, sizeof(timestr), (scrub_monitor->boot + timo),
+                gf_timefmt_FT);
+    gf_msg(this->name, GF_LOG_INFO, 0, BRB_MSG_SCRUB_INFO,
+           "Scrubbing is "
+           "scheduled to run at %s",
+           timestr);
+
+    return 0;
+
+error_return:
+    return -1;
+}
+
+int32_t
+br_fsscan_activate(xlator_t *this)
+{
+    uint32_t timo = 0;
+    char timestr[GF_TIMESTR_SIZE] = {
+        0,
+    };
+    time_t now = 0;
+    br_private_t *priv = NULL;
+    struct br_scrubber *fsscrub = NULL;
+    struct br_monitor *scrub_monitor = NULL;
+
+    priv = this->private;
+    fsscrub = &priv->fsscrub;
+    scrub_monitor = &priv->scrub_monitor;
+
+    now = gf_time();
+    timo = br_fsscan_calculate_timeout(fsscrub->frequency);
+    if (timo == 0) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, BRB_MSG_ZERO_TIMEOUT_BUG,
+               "BUG: Zero schedule timeout");
+        return -1;
+    }
+
+    pthread_mutex_lock(&scrub_monitor->donelock);
+    {
+        scrub_monitor->done = _gf_false;
+    }
+    pthread_mutex_unlock(&scrub_monitor->donelock);
+
+    gf_time_fmt(timestr, sizeof(timestr), now + timo, gf_timefmt_FT);
+    (void)gf_tw_mod_timer(priv->timer_wheel, scrub_monitor->timer, timo);
+
+    _br_monitor_set_scrub_state(scrub_monitor, BR_SCRUB_STATE_PENDING);
+    gf_msg(this->name, GF_LOG_INFO, 0, BRB_MSG_SCRUB_INFO,
+           "Scrubbing is "
+           "rescheduled to run at %s",
+           timestr);
+
+    return 0;
+}
+
+int32_t
+br_fsscan_reschedule(xlator_t *this)
+{
+    int32_t ret = 0;
+    uint32_t timo = 0;
+    char timestr[GF_TIMESTR_SIZE] = {
+        0,
+    };
+    time_t now = 0;
+    br_private_t *priv = NULL;
+    struct br_scrubber *fsscrub = NULL;
+    struct br_monitor *scrub_monitor = NULL;
+
+    priv = this->private;
+    fsscrub = &priv->fsscrub;
+    scrub_monitor = &priv->scrub_monitor;
+
+    if (!fsscrub->frequency_reconf)
+        return 0;
+
+    now = gf_time();
+    timo = br_fsscan_calculate_timeout(fsscrub->frequency);
+    if (timo == 0) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, BRB_MSG_ZERO_TIMEOUT_BUG,
+               "BUG: Zero schedule timeout");
+        return -1;
+    }
+
+    gf_time_fmt(timestr, sizeof(timestr), now + timo, gf_timefmt_FT);
+
+    pthread_mutex_lock(&scrub_monitor->donelock);
+    {
+        scrub_monitor->done = _gf_false;
+    }
+    pthread_mutex_unlock(&scrub_monitor->donelock);
+
+    ret = gf_tw_mod_timer_pending(priv->timer_wheel, scrub_monitor->timer,
+                                  timo);
+    if (ret == 0)
+        gf_msg(this->name, GF_LOG_INFO, 0, BRB_MSG_SCRUB_INFO,
+               "Scrubber is currently running and would be "
+               "rescheduled after completion");
+    else {
+        _br_monitor_set_scrub_state(scrub_monitor, BR_SCRUB_STATE_PENDING);
+        gf_msg(this->name, GF_LOG_INFO, 0, BRB_MSG_SCRUB_INFO,
+               "Scrubbing rescheduled to run at %s", timestr);
+    }
+
+    return 0;
+}
+
+int32_t
+br_fsscan_ondemand(xlator_t *this)
+{
+    int32_t ret = 0;
+    uint32_t timo = 0;
+    char timestr[GF_TIMESTR_SIZE] = {
+        0,
+    };
+    time_t now = 0;
+    br_private_t *priv = NULL;
+    struct br_monitor *scrub_monitor = NULL;
+
+    priv = this->private;
+    scrub_monitor = &priv->scrub_monitor;
+
+    now = gf_time();
+    timo = BR_SCRUB_ONDEMAND;
+    gf_time_fmt(timestr, sizeof(timestr), now + timo, gf_timefmt_FT);
+
+    pthread_mutex_lock(&scrub_monitor->donelock);
+    {
+        scrub_monitor->done = _gf_false;
+    }
+    pthread_mutex_unlock(&scrub_monitor->donelock);
+
+    ret = gf_tw_mod_timer_pending(priv->timer_wheel, scrub_monitor->timer,
+                                  timo);
+    if (ret == 0)
+        gf_msg(this->name, GF_LOG_INFO, 0, BRB_MSG_SCRUB_INFO,
+               "Scrubber is currently running and would be "
+               "rescheduled after completion");
+    else {
+        _br_monitor_set_scrub_state(scrub_monitor, BR_SCRUB_STATE_PENDING);
+        gf_msg(this->name, GF_LOG_INFO, 0, BRB_MSG_SCRUB_INFO,
+               "Ondemand Scrubbing scheduled to run at %s", timestr);
+    }
+
+    return 0;
+}
+
+#define BR_SCRUB_THREAD_SCALE_LAZY 0
+#define BR_SCRUB_THREAD_SCALE_NORMAL 0.4
+#define BR_SCRUB_THREAD_SCALE_AGGRESSIVE 1.0
+
+#ifndef M_E
+#define M_E 2.718
+#endif
+
+/**
+ * This is just a simple exponential scale to a fixed value selected
+ * per throttle config. We probably need to be more smart and select
+ * the scale based on the number of processor cores too.
+ */
+static unsigned int
+br_scrubber_calc_scale(xlator_t *this, br_private_t *priv,
+                       scrub_throttle_t throttle)
+{
+    unsigned int scale = 0;
+
+    switch (throttle) {
+        case BR_SCRUB_THROTTLE_VOID:
+        case BR_SCRUB_THROTTLE_STALLED:
+            scale = 0;
+            break;
+        case BR_SCRUB_THROTTLE_LAZY:
+            scale = priv->child_count * pow(M_E, BR_SCRUB_THREAD_SCALE_LAZY);
+            break;
+        case BR_SCRUB_THROTTLE_NORMAL:
+            scale = priv->child_count * pow(M_E, BR_SCRUB_THREAD_SCALE_NORMAL);
+            break;
+        case BR_SCRUB_THROTTLE_AGGRESSIVE:
+            scale = priv->child_count *
+                    pow(M_E, BR_SCRUB_THREAD_SCALE_AGGRESSIVE);
+            break;
+        default:
+            gf_msg(this->name, GF_LOG_ERROR, 0, BRB_MSG_UNKNOWN_THROTTLE,
+                   "Unknown throttle %d", throttle);
+    }
+
+    return scale;
+}
+
+static br_child_t *
+_br_scrubber_get_next_child(struct br_scrubber *fsscrub)
+{
+    br_child_t *child = NULL;
+
+    child = list_first_entry(&fsscrub->scrublist, br_child_t, list);
+    list_rotate_left(&fsscrub->scrublist);
+
+    return child;
+}
+
+static void
+_br_scrubber_get_entry(br_child_t *child, struct br_fsscan_entry **fsentry)
+{
+    struct br_scanfs *fsscan = &child->fsscan;
+
+    if (list_empty(&fsscan->ready))
+        return;
+    *fsentry = list_first_entry(&fsscan->ready, struct br_fsscan_entry, list);
+    list_del_init(&(*fsentry)->list);
+}
+
+static void
+_br_scrubber_find_scrubbable_entry(struct br_scrubber *fsscrub,
+                                   struct br_fsscan_entry **fsentry)
+{
+    br_child_t *child = NULL;
+    br_child_t *firstchild = NULL;
+
+    while (1) {
+        while (list_empty(&fsscrub->scrublist))
+            pthread_cond_wait(&fsscrub->cond, &fsscrub->mutex);
+
+        firstchild = NULL;
+        for (child = _br_scrubber_get_next_child(fsscrub); child != firstchild;
+             child = _br_scrubber_get_next_child(fsscrub)) {
+            if (!firstchild)
+                firstchild = child;
+
+            _br_scrubber_get_entry(child, fsentry);
+            if (*fsentry)
+                break;
+        }
+
+        if (*fsentry)
+            break;
+
+        /* nothing to work on.. wait till available */
+        pthread_cond_wait(&fsscrub->cond, &fsscrub->mutex);
+    }
+}
+
+static void
+br_scrubber_pick_entry(struct br_scrubber *fsscrub,
+                       struct br_fsscan_entry **fsentry)
+{
+    pthread_cleanup_push(_br_lock_cleaner, &fsscrub->mutex);
+
+    pthread_mutex_lock(&fsscrub->mutex);
+    {
+        *fsentry = NULL;
+        _br_scrubber_find_scrubbable_entry(fsscrub, fsentry);
+    }
+    pthread_mutex_unlock(&fsscrub->mutex);
+
+    pthread_cleanup_pop(0);
+}
+
+struct br_scrub_entry {
+    gf_boolean_t scrubbed;
+    struct br_fsscan_entry *fsentry;
+};
+
+/**
+ * We need to be a bit careful here. These thread(s) are prone to cancellations
+ * when threads are scaled down (depending on the thottling value configured)
+ * and pausing scrub. A thread can get cancelled while it's waiting for entries
+ * in the ->pending queue or when an object is undergoing scrubbing.
+ */
+static void
+br_scrubber_entry_handle(void *arg)
+{
+    struct br_scanfs *fsscan = NULL;
+    struct br_scrub_entry *sentry = NULL;
+    struct br_fsscan_entry *fsentry = NULL;
+
+    sentry = arg;
+
+    fsentry = sentry->fsentry;
+    fsscan = fsentry->fsscan;
+
+    LOCK(&fsscan->entrylock);
+    {
+        if (sentry->scrubbed) {
+            _br_fsscan_dec_entry_count(fsscan);
+
+            /* cleanup ->entry */
+            fsentry->data = NULL;
+            fsentry->fsscan = NULL;
+            loc_wipe(&fsentry->parent);
+            gf_dirent_entry_free(fsentry->entry);
+
+            GF_FREE(sentry->fsentry);
+        } else {
+            /* (re)queue the entry again for scrub */
+            _br_fsscan_collect_entry(fsscan, sentry->fsentry);
+        }
+    }
+    UNLOCK(&fsscan->entrylock);
+}
+
+static void
+br_scrubber_scrub_entry(xlator_t *this, struct br_fsscan_entry *fsentry)
+{
+    struct br_scrub_entry sentry = {
+        0,
+    };
+
+    sentry.scrubbed = 0;
+    sentry.fsentry = fsentry;
+
+    pthread_cleanup_push(br_scrubber_entry_handle, &sentry);
+    {
+        (void)br_scrubber_scrub_begin(this, fsentry);
+        sentry.scrubbed = 1;
+    }
+    pthread_cleanup_pop(1);
+}
+
+void *
+br_scrubber_proc(void *arg)
+{
+    xlator_t *this = NULL;
+    struct br_scrubber *fsscrub = NULL;
+    struct br_fsscan_entry *fsentry = NULL;
+
+    fsscrub = arg;
+    THIS = this = fsscrub->this;
+
+    while (1) {
+        br_scrubber_pick_entry(fsscrub, &fsentry);
+        br_scrubber_scrub_entry(this, fsentry);
+        sleep(1);
+    }
+
+    return NULL;
+}
+
+static int32_t
+br_scrubber_scale_up(xlator_t *this, struct br_scrubber *fsscrub,
+                     unsigned int v1, unsigned int v2)
+{
+    int i = 0;
+    int32_t ret = -1;
+    int diff = 0;
+    struct br_scrubbers *scrub = NULL;
+
+    diff = (int)(v2 - v1);
+
+    gf_msg(this->name, GF_LOG_INFO, 0, BRB_MSG_SCALING_UP_SCRUBBER,
+           "Scaling up scrubbers [%d => %d]", v1, v2);
+
+    for (i = 0; i < diff; i++) {
+        scrub = GF_CALLOC(diff, sizeof(*scrub), gf_br_mt_br_scrubber_t);
+        if (!scrub)
+            break;
+
+        INIT_LIST_HEAD(&scrub->list);
+        ret = gf_thread_create(&scrub->scrubthread, NULL, br_scrubber_proc,
+                               fsscrub, "brsproc");
         if (ret)
-                goto unrefd; /* skip this object */
+            break;
 
-        /* if all's good, proceed to calculate the hash */
-        md = GF_CALLOC (SHA256_DIGEST_LENGTH, sizeof (*md),
-                        gf_common_mt_char);
-        if (!md)
-                goto unrefd;
+        fsscrub->nr_scrubbers++;
+        list_add_tail(&scrub->list, &fsscrub->scrubbers);
+    }
 
-        ret = br_calculate_obj_checksum (md, child, fd, &iatt);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "error calculating hash "
-                        "for object [GFID: %s]", uuid_utoa (fd->inode->gfid));
-                ret = -1;
-                goto free_md;
+    if ((i != diff) && !scrub)
+        goto error_return;
+
+    if (i != diff) /* degraded scaling.. */
+        gf_msg(this->name, GF_LOG_WARNING, 0, BRB_MSG_SCALE_UP_FAILED,
+               "Could not fully scale up to %d scrubber(s). Spawned "
+               "%d/%d [total scrubber(s): %d]",
+               v2, i, diff, (v1 + i));
+
+    return 0;
+
+error_return:
+    return -1;
+}
+
+static int32_t
+br_scrubber_scale_down(xlator_t *this, struct br_scrubber *fsscrub,
+                       unsigned int v1, unsigned int v2)
+{
+    int i = 0;
+    int diff = 0;
+    int32_t ret = -1;
+    struct br_scrubbers *scrub = NULL;
+
+    diff = (int)(v1 - v2);
+
+    gf_msg(this->name, GF_LOG_INFO, 0, BRB_MSG_SCALE_DOWN_SCRUBBER,
+           "Scaling down scrubbers [%d => %d]", v1, v2);
+
+    for (i = 0; i < diff; i++) {
+        scrub = list_first_entry(&fsscrub->scrubbers, struct br_scrubbers,
+                                 list);
+
+        list_del_init(&scrub->list);
+        ret = gf_thread_cleanup_xint(scrub->scrubthread);
+        if (ret)
+            break;
+        GF_FREE(scrub);
+
+        fsscrub->nr_scrubbers--;
+    }
+
+    if (ret) {
+        gf_msg(this->name, GF_LOG_WARNING, 0, BRB_MSG_SCALE_DOWN_FAILED,
+               "Could not fully scale down "
+               "to %d scrubber(s). Terminated %d/%d [total "
+               "scrubber(s): %d]",
+               v1, i, diff, (v2 - i));
+        ret = 0;
+    }
+
+    return ret;
+}
+
+static int32_t
+br_scrubber_configure(xlator_t *this, br_private_t *priv,
+                      struct br_scrubber *fsscrub, scrub_throttle_t nthrottle)
+{
+    int32_t ret = 0;
+    unsigned int v1 = 0;
+    unsigned int v2 = 0;
+
+    v1 = fsscrub->nr_scrubbers;
+    v2 = br_scrubber_calc_scale(this, priv, nthrottle);
+
+    if (v1 == v2)
+        return 0;
+
+    if (v1 > v2)
+        ret = br_scrubber_scale_down(this, fsscrub, v1, v2);
+    else
+        ret = br_scrubber_scale_up(this, fsscrub, v1, v2);
+
+    return ret;
+}
+
+static int32_t
+br_scrubber_fetch_option(xlator_t *this, char *opt, dict_t *options,
+                         char **value)
+{
+    if (options)
+        GF_OPTION_RECONF(opt, *value, options, str, error_return);
+    else
+        GF_OPTION_INIT(opt, *value, str, error_return);
+
+    return 0;
+
+error_return:
+    return -1;
+}
+
+/* internal "throttle" override */
+#define BR_SCRUB_STALLED "STALLED"
+
+/* TODO: token buket spec */
+static int32_t
+br_scrubber_handle_throttle(xlator_t *this, br_private_t *priv, dict_t *options,
+                            gf_boolean_t scrubstall)
+{
+    int32_t ret = 0;
+    char *tmp = NULL;
+    struct br_scrubber *fsscrub = NULL;
+    scrub_throttle_t nthrottle = BR_SCRUB_THROTTLE_VOID;
+
+    fsscrub = &priv->fsscrub;
+    fsscrub->throttle_reconf = _gf_false;
+
+    ret = br_scrubber_fetch_option(this, "scrub-throttle", options, &tmp);
+    if (ret)
+        goto error_return;
+
+    if (scrubstall)
+        tmp = BR_SCRUB_STALLED;
+
+    if (strcasecmp(tmp, "lazy") == 0)
+        nthrottle = BR_SCRUB_THROTTLE_LAZY;
+    else if (strcasecmp(tmp, "normal") == 0)
+        nthrottle = BR_SCRUB_THROTTLE_NORMAL;
+    else if (strcasecmp(tmp, "aggressive") == 0)
+        nthrottle = BR_SCRUB_THROTTLE_AGGRESSIVE;
+    else if (strcasecmp(tmp, BR_SCRUB_STALLED) == 0)
+        nthrottle = BR_SCRUB_THROTTLE_STALLED;
+    else
+        goto error_return;
+
+    /* on failure old throttling value is preserved */
+    ret = br_scrubber_configure(this, priv, fsscrub, nthrottle);
+    if (ret)
+        goto error_return;
+
+    if (fsscrub->throttle != nthrottle)
+        fsscrub->throttle_reconf = _gf_true;
+
+    fsscrub->throttle = nthrottle;
+    return 0;
+
+error_return:
+    return -1;
+}
+
+static int32_t
+br_scrubber_handle_stall(xlator_t *this, br_private_t *priv, dict_t *options,
+                         gf_boolean_t *scrubstall)
+{
+    int32_t ret = 0;
+    char *tmp = NULL;
+
+    ret = br_scrubber_fetch_option(this, "scrub-state", options, &tmp);
+    if (ret)
+        goto error_return;
+
+    if (strcasecmp(tmp, "pause") == 0) /* anything else is active */
+        *scrubstall = _gf_true;
+
+    return 0;
+
+error_return:
+    return -1;
+}
+
+static int32_t
+br_scrubber_handle_freq(xlator_t *this, br_private_t *priv, dict_t *options,
+                        gf_boolean_t scrubstall)
+{
+    int32_t ret = -1;
+    char *tmp = NULL;
+    scrub_freq_t frequency = BR_FSSCRUB_FREQ_HOURLY;
+    struct br_scrubber *fsscrub = NULL;
+
+    fsscrub = &priv->fsscrub;
+    fsscrub->frequency_reconf = _gf_true;
+
+    ret = br_scrubber_fetch_option(this, "scrub-freq", options, &tmp);
+    if (ret)
+        goto error_return;
+
+    if (scrubstall)
+        tmp = BR_SCRUB_STALLED;
+
+    if (strcasecmp(tmp, "hourly") == 0) {
+        frequency = BR_FSSCRUB_FREQ_HOURLY;
+    } else if (strcasecmp(tmp, "daily") == 0) {
+        frequency = BR_FSSCRUB_FREQ_DAILY;
+    } else if (strcasecmp(tmp, "weekly") == 0) {
+        frequency = BR_FSSCRUB_FREQ_WEEKLY;
+    } else if (strcasecmp(tmp, "biweekly") == 0) {
+        frequency = BR_FSSCRUB_FREQ_BIWEEKLY;
+    } else if (strcasecmp(tmp, "monthly") == 0) {
+        frequency = BR_FSSCRUB_FREQ_MONTHLY;
+    } else if (strcasecmp(tmp, "minute") == 0) {
+        frequency = BR_FSSCRUB_FREQ_MINUTE;
+    } else if (strcasecmp(tmp, BR_SCRUB_STALLED) == 0) {
+        frequency = BR_FSSCRUB_FREQ_STALLED;
+    } else
+        goto error_return;
+
+    if (fsscrub->frequency == frequency)
+        fsscrub->frequency_reconf = _gf_false;
+    else
+        fsscrub->frequency = frequency;
+
+    return 0;
+
+error_return:
+    return -1;
+}
+
+static void
+br_scrubber_log_option(xlator_t *this, br_private_t *priv,
+                       gf_boolean_t scrubstall)
+{
+    struct br_scrubber *fsscrub = &priv->fsscrub;
+    char *scrub_throttle_str[] = {
+        [BR_SCRUB_THROTTLE_LAZY] = "lazy",
+        [BR_SCRUB_THROTTLE_NORMAL] = "normal",
+        [BR_SCRUB_THROTTLE_AGGRESSIVE] = "aggressive",
+        [BR_SCRUB_THROTTLE_STALLED] = "stalled",
+    };
+
+    char *scrub_freq_str[] = {
+        [0] = "",
+        [BR_FSSCRUB_FREQ_HOURLY] = "hourly",
+        [BR_FSSCRUB_FREQ_DAILY] = "daily",
+        [BR_FSSCRUB_FREQ_WEEKLY] = "weekly",
+        [BR_FSSCRUB_FREQ_BIWEEKLY] = "biweekly",
+        [BR_FSSCRUB_FREQ_MONTHLY] = "monthly (30 days)",
+        [BR_FSSCRUB_FREQ_MINUTE] = "every minute",
+    };
+
+    if (scrubstall)
+        return; /* logged as pause */
+
+    if (fsscrub->frequency_reconf || fsscrub->throttle_reconf) {
+        if (fsscrub->throttle == BR_SCRUB_THROTTLE_VOID)
+            return;
+        gf_msg(this->name, GF_LOG_INFO, 0, BRB_MSG_SCRUB_TUNABLE,
+               "SCRUB TUNABLES:: [Frequency: %s, Throttle: %s]",
+               scrub_freq_str[fsscrub->frequency],
+               scrub_throttle_str[fsscrub->throttle]);
+    }
+}
+
+int32_t
+br_scrubber_handle_options(xlator_t *this, br_private_t *priv, dict_t *options)
+{
+    int32_t ret = 0;
+    gf_boolean_t scrubstall = _gf_false; /* not as dangerous as it sounds */
+
+    ret = br_scrubber_handle_stall(this, priv, options, &scrubstall);
+    if (ret)
+        goto error_return;
+
+    ret = br_scrubber_handle_throttle(this, priv, options, scrubstall);
+    if (ret)
+        goto error_return;
+
+    ret = br_scrubber_handle_freq(this, priv, options, scrubstall);
+    if (ret)
+        goto error_return;
+
+    br_scrubber_log_option(this, priv, scrubstall);
+
+    return 0;
+
+error_return:
+    return -1;
+}
+
+inode_t *
+br_lookup_bad_obj_dir(xlator_t *this, br_child_t *child, uuid_t gfid)
+{
+    struct iatt statbuf = {
+        0,
+    };
+    inode_table_t *table = NULL;
+    int32_t ret = -1;
+    loc_t loc = {
+        0,
+    };
+    inode_t *linked_inode = NULL;
+    int32_t op_errno = 0;
+
+    GF_VALIDATE_OR_GOTO("bit-rot-scrubber", this, out);
+    GF_VALIDATE_OR_GOTO(this->name, this->private, out);
+    GF_VALIDATE_OR_GOTO(this->name, child, out);
+
+    table = child->table;
+
+    loc.inode = inode_new(table);
+    if (!loc.inode) {
+        gf_msg(this->name, GF_LOG_ERROR, ENOMEM, BRB_MSG_NO_MEMORY,
+               "failed to allocate a new inode for"
+               "bad object directory");
+        goto out;
+    }
+
+    gf_uuid_copy(loc.gfid, gfid);
+
+    ret = syncop_lookup(child->xl, &loc, &statbuf, NULL, NULL, NULL);
+    if (ret < 0) {
+        op_errno = -ret;
+        gf_msg(this->name, GF_LOG_ERROR, 0, BRB_MSG_LOOKUP_FAILED,
+               "failed to lookup the bad "
+               "objects directory (gfid: %s (%s))",
+               uuid_utoa(gfid), strerror(op_errno));
+        goto out;
+    }
+
+    linked_inode = inode_link(loc.inode, NULL, NULL, &statbuf);
+    if (linked_inode)
+        inode_lookup(linked_inode);
+
+out:
+    loc_wipe(&loc);
+    return linked_inode;
+}
+
+int32_t
+br_read_bad_object_dir(xlator_t *this, br_child_t *child, fd_t *fd,
+                       dict_t *dict)
+{
+    gf_dirent_t entries;
+    gf_dirent_t *entry = NULL;
+    int32_t ret = -1;
+    off_t offset = 0;
+    int32_t count = 0;
+    char key[32] = {
+        0,
+    };
+    dict_t *out_dict = NULL;
+
+    INIT_LIST_HEAD(&entries.list);
+
+    while ((ret = syncop_readdir(child->xl, fd, 131072, offset, &entries, NULL,
+                                 &out_dict))) {
+        if (ret < 0)
+            goto out;
+
+        list_for_each_entry(entry, &entries.list, list)
+        {
+            offset = entry->d_off;
+
+            snprintf(key, sizeof(key), "quarantine-%d", count);
+
+            /*
+             * ignore the dict_set errors for now. The intention is
+             * to get as many bad objects as possible instead of
+             * erroring out at the first failure.
+             */
+            ret = dict_set_dynstr_with_alloc(dict, key, entry->d_name);
+            if (!ret)
+                count++;
+
+            if (out_dict) {
+                dict_copy(out_dict, dict);
+                dict_unref(out_dict);
+                out_dict = NULL;
+            }
         }
 
-        /**
-         * perform post compute checks as an object's signature may have
-         * become stale while scrubber calculated checksum.
-         */
-        ret = bitd_scrub_post_compute_check (this, child,
-                                             fd, signedversion, &sign);
+        gf_dirent_free(&entries);
+    }
+
+    ret = count;
+    ret = dict_set_int32_sizen(dict, "count", count);
+
+out:
+    return ret;
+}
+
+int32_t
+br_get_bad_objects_from_child(xlator_t *this, dict_t *dict, br_child_t *child)
+{
+    inode_t *inode = NULL;
+    inode_table_t *table = NULL;
+    fd_t *fd = NULL;
+    int32_t ret = -1;
+    loc_t loc = {
+        0,
+    };
+    int32_t op_errno = 0;
+
+    GF_VALIDATE_OR_GOTO("bit-rot-scrubber", this, out);
+    GF_VALIDATE_OR_GOTO(this->name, this->private, out);
+    GF_VALIDATE_OR_GOTO(this->name, child, out);
+    GF_VALIDATE_OR_GOTO(this->name, dict, out);
+
+    table = child->table;
+
+    inode = inode_find(table, BR_BAD_OBJ_CONTAINER);
+    if (!inode) {
+        inode = br_lookup_bad_obj_dir(this, child, BR_BAD_OBJ_CONTAINER);
+        if (!inode)
+            goto out;
+    }
+
+    fd = fd_create(inode, 0);
+    if (!fd) {
+        gf_msg(this->name, GF_LOG_ERROR, ENOMEM, BRB_MSG_FD_CREATE_FAILED,
+               "fd creation for the bad "
+               "objects directory failed (gfid: %s)",
+               uuid_utoa(BR_BAD_OBJ_CONTAINER));
+        goto out;
+    }
+
+    loc.inode = inode;
+    gf_uuid_copy(loc.gfid, inode->gfid);
+
+    ret = syncop_opendir(child->xl, &loc, fd, NULL, NULL);
+    if (ret < 0) {
+        op_errno = -ret;
+        fd_unref(fd);
+        fd = NULL;
+        gf_msg(this->name, GF_LOG_ERROR, op_errno, BRB_MSG_FD_CREATE_FAILED,
+               "failed to open the bad "
+               "objects directory %s",
+               uuid_utoa(BR_BAD_OBJ_CONTAINER));
+        goto out;
+    }
+
+    fd_bind(fd);
+
+    ret = br_read_bad_object_dir(this, child, fd, dict);
+    if (ret < 0) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, BRB_MSG_BAD_OBJ_READDIR_FAIL,
+               "readdir of the bad "
+               "objects directory (%s) failed ",
+               uuid_utoa(BR_BAD_OBJ_CONTAINER));
+        goto out;
+    }
+
+    ret = 0;
+
+out:
+    loc_wipe(&loc);
+    if (fd)
+        fd_unref(fd);
+    return ret;
+}
+
+int32_t
+br_collect_bad_objects_of_child(xlator_t *this, br_child_t *child, dict_t *dict,
+                                dict_t *child_dict, int32_t total_count)
+{
+    int32_t ret = -1;
+    int32_t count = 0;
+    char key[32] = {
+        0,
+    };
+    char main_key[32] = {
+        0,
+    };
+    int32_t j = 0;
+    int32_t tmp_count = 0;
+    char *entry = NULL;
+    char tmp[PATH_MAX] = {
+        0,
+    };
+    char *path = NULL;
+    int32_t len = 0;
+
+    ret = dict_get_int32_sizen(child_dict, "count", &count);
+    if (ret)
+        goto out;
+
+    tmp_count = total_count;
+
+    for (j = 0; j < count; j++) {
+        len = snprintf(key, sizeof(key), "quarantine-%d", j);
+        ret = dict_get_strn(child_dict, key, len, &entry);
         if (ret)
-                goto free_md;
+            continue;
+
+        ret = dict_get_str(child_dict, entry, &path);
+        len = snprintf(tmp, PATH_MAX, "%s ==> BRICK: %s\n path: %s", entry,
+                       child->brick_path, path);
+        if ((len < 0) || (len >= PATH_MAX)) {
+            continue;
+        }
+        snprintf(main_key, sizeof(main_key), "quarantine-%d", tmp_count);
+
+        ret = dict_set_dynstr_with_alloc(dict, main_key, tmp);
+        if (!ret)
+            tmp_count++;
+        path = NULL;
+    }
+
+    ret = tmp_count;
+
+out:
+    return ret;
+}
+
+int32_t
+br_collect_bad_objects_from_children(xlator_t *this, dict_t *dict)
+{
+    int32_t ret = -1;
+    dict_t *child_dict = NULL;
+    int32_t i = 0;
+    int32_t total_count = 0;
+    br_child_t *child = NULL;
+    br_private_t *priv = NULL;
+    dict_t *tmp_dict = NULL;
+
+    priv = this->private;
+    tmp_dict = dict;
+
+    for (i = 0; i < priv->child_count; i++) {
+        child = &priv->children[i];
+        GF_ASSERT(child);
+        if (!_br_is_child_connected(child))
+            continue;
+
+        child_dict = dict_new();
+        if (!child_dict) {
+            gf_msg(this->name, GF_LOG_ERROR, ENOMEM, BRB_MSG_NO_MEMORY,
+                   "failed to allocate dict");
+            continue;
+        }
+        ret = br_get_bad_objects_from_child(this, child_dict, child);
+        /*
+         * Continue asking the remaining children for the list of
+         * bad objects even though getting the list from one of them
+         * fails.
+         */
+        if (ret) {
+            dict_unref(child_dict);
+            continue;
+        }
+
+        ret = br_collect_bad_objects_of_child(this, child, tmp_dict, child_dict,
+                                              total_count);
+        if (ret < 0) {
+            dict_unref(child_dict);
+            continue;
+        }
+
+        total_count = ret;
+        dict_unref(child_dict);
+        child_dict = NULL;
+    }
+
+    ret = dict_set_int32(tmp_dict, "total-count", total_count);
+
+    return ret;
+}
+
+int32_t
+br_get_bad_objects_list(xlator_t *this, dict_t **dict)
+{
+    int32_t ret = -1;
+    dict_t *tmp_dict = NULL;
+
+    GF_VALIDATE_OR_GOTO("bir-rot-scrubber", this, out);
+    GF_VALIDATE_OR_GOTO(this->name, dict, out);
 
-        ret = bitd_compare_ckum (this, sign, md,
-                                 linked_inode, entry, fd, child);
+    tmp_dict = *dict;
+    if (!tmp_dict) {
+        tmp_dict = dict_new();
+        if (!tmp_dict) {
+            gf_msg(this->name, GF_LOG_ERROR, ENOMEM, BRB_MSG_NO_MEMORY,
+                   "failed to allocate dict");
+            goto out;
+        }
+        *dict = tmp_dict;
+    }
+
+    ret = br_collect_bad_objects_from_children(this, tmp_dict);
+
+out:
+    return ret;
+}
 
-        GF_FREE (sign); /* alloced on post-compute */
+static int
+wait_for_scrub_to_finish(xlator_t *this)
+{
+    int ret = -1;
+    br_private_t *priv = NULL;
+    struct br_monitor *scrub_monitor = NULL;
+
+    priv = this->private;
+    scrub_monitor = &priv->scrub_monitor;
 
-        /** fd_unref() takes care of closing fd.. like syncop_close() */
+    GF_VALIDATE_OR_GOTO("bit-rot", scrub_monitor, out);
+    GF_VALIDATE_OR_GOTO("bit-rot", this, out);
 
- free_md:
-        GF_FREE (md);
- unrefd:
-        fd_unref (fd);
- unref_inode:
-        inode_unref (linked_inode);
- out:
-        loc_wipe (&loc);
-        return ret;
+    gf_msg(this->name, GF_LOG_INFO, 0, BRB_MSG_SCRUB_INFO,
+           "Waiting for all children to start and finish scrub");
+
+    pthread_mutex_lock(&scrub_monitor->donelock);
+    {
+        while (!scrub_monitor->done)
+            pthread_cond_wait(&scrub_monitor->donecond,
+                              &scrub_monitor->donelock);
+    }
+    pthread_mutex_unlock(&scrub_monitor->donelock);
+    ret = 0;
+out:
+    return ret;
 }
 
-#define BR_SCRUB_THROTTLE_COUNT 30
-#define BR_SCRUB_THROTTLE_ZZZ   60
+/**
+ * This function is executed in a separate thread. This is scrubber monitor
+ * thread that takes care of state machine.
+ */
 void *
-br_scrubber (void *arg)
+br_monitor_thread(void *arg)
 {
-        loc_t       loc   = {0,};
-        xlator_t   *this  = NULL;
-        br_child_t *child = NULL;
+    int32_t ret = 0;
+    xlator_t *this = NULL;
+    br_private_t *priv = NULL;
+    struct br_monitor *scrub_monitor = NULL;
+
+    this = arg;
+    priv = this->private;
 
-        child = arg;
-        this = child->this;
+    /*
+     * Since, this is the topmost xlator, THIS has to be set by bit-rot
+     * xlator itself (STACK_WIND won't help in this case). Also it has
+     * to be done for each thread that gets spawned. Otherwise, a new
+     * thread will get global_xlator's pointer when it does "THIS".
+     */
+    THIS = this;
 
-        THIS = this;
+    scrub_monitor = &priv->scrub_monitor;
 
-        loc.inode = child->table->root;
-        while (1) {
-                (void) syncop_ftw_throttle
-                           (child->xl, &loc,
-                            GF_CLIENT_PID_SCRUB, child, bitd_start_scrub,
-                            BR_SCRUB_THROTTLE_COUNT, BR_SCRUB_THROTTLE_ZZZ);
+    pthread_mutex_lock(&scrub_monitor->mutex);
+    {
+        while (!scrub_monitor->inited)
+            pthread_cond_wait(&scrub_monitor->cond, &scrub_monitor->mutex);
+    }
+    pthread_mutex_unlock(&scrub_monitor->mutex);
 
-                sleep (BR_SCRUB_THROTTLE_ZZZ);
+    /* this needs to be serialized with reconfigure() */
+    pthread_mutex_lock(&priv->lock);
+    {
+        ret = br_scrub_state_machine(this, _gf_false);
+    }
+    pthread_mutex_unlock(&priv->lock);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, -ret, BRB_MSG_SSM_FAILED,
+               "Scrub state machine failed");
+        goto out;
+    }
+
+    while (1) {
+        /* Wait for all children to finish scrubbing */
+        ret = wait_for_scrub_to_finish(this);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, -ret, BRB_MSG_SCRUB_WAIT_FAILED,
+                   "Scrub wait failed");
+            goto out;
         }
 
-        return NULL;
+        /* scrub exit criteria: Move the state to PENDING */
+        br_scrubber_exit_control(this);
+    }
+
+out:
+    return NULL;
+}
+
+static void
+br_set_scrub_state(struct br_monitor *scrub_monitor, br_scrub_state_t state)
+{
+    LOCK(&scrub_monitor->lock);
+    {
+        _br_monitor_set_scrub_state(scrub_monitor, state);
+    }
+    UNLOCK(&scrub_monitor->lock);
+}
+
+int32_t
+br_scrubber_monitor_init(xlator_t *this, br_private_t *priv)
+{
+    struct br_monitor *scrub_monitor = NULL;
+    int ret = 0;
+
+    scrub_monitor = &priv->scrub_monitor;
+
+    LOCK_INIT(&scrub_monitor->lock);
+    scrub_monitor->this = this;
+
+    scrub_monitor->inited = _gf_false;
+    pthread_mutex_init(&scrub_monitor->mutex, NULL);
+    pthread_cond_init(&scrub_monitor->cond, NULL);
+
+    scrub_monitor->kick = _gf_false;
+    scrub_monitor->active_child_count = 0;
+    pthread_mutex_init(&scrub_monitor->wakelock, NULL);
+    pthread_cond_init(&scrub_monitor->wakecond, NULL);
+
+    scrub_monitor->done = _gf_false;
+    pthread_mutex_init(&scrub_monitor->donelock, NULL);
+    pthread_cond_init(&scrub_monitor->donecond, NULL);
+
+    /* Set the state to INACTIVE */
+    br_set_scrub_state(&priv->scrub_monitor, BR_SCRUB_STATE_INACTIVE);
+
+    /* Start the monitor thread */
+    ret = gf_thread_create(&scrub_monitor->thread, NULL, br_monitor_thread,
+                           this, "brmon");
+    if (ret != 0) {
+        gf_msg(this->name, GF_LOG_ERROR, -ret, BRB_MSG_SPAWN_FAILED,
+               "monitor thread creation failed");
+        ret = -1;
+        goto err;
+    }
+
+    return 0;
+err:
+    pthread_mutex_destroy(&scrub_monitor->mutex);
+    pthread_cond_destroy(&scrub_monitor->cond);
+
+    pthread_mutex_destroy(&scrub_monitor->wakelock);
+    pthread_cond_destroy(&scrub_monitor->wakecond);
+
+    pthread_mutex_destroy(&scrub_monitor->donelock);
+    pthread_cond_destroy(&scrub_monitor->donecond);
+
+    LOCK_DESTROY(&scrub_monitor->lock);
+
+    return ret;
+}
+
+int32_t
+br_scrubber_init(xlator_t *this, br_private_t *priv)
+{
+    struct br_scrubber *fsscrub = NULL;
+    int ret = 0;
+
+    priv->tbf = tbf_init(NULL, 0);
+    if (!priv->tbf)
+        return -1;
+
+    ret = br_scrubber_monitor_init(this, priv);
+    if (ret)
+        return -1;
+
+    fsscrub = &priv->fsscrub;
+
+    fsscrub->this = this;
+    fsscrub->throttle = BR_SCRUB_THROTTLE_VOID;
+
+    pthread_mutex_init(&fsscrub->mutex, NULL);
+    pthread_cond_init(&fsscrub->cond, NULL);
+
+    fsscrub->nr_scrubbers = 0;
+    INIT_LIST_HEAD(&fsscrub->scrubbers);
+    INIT_LIST_HEAD(&fsscrub->scrublist);
+
+    return 0;
 }
diff --git a/xlators/features/bit-rot/src/bitd/bit-rot-scrub.h b/xlators/features/bit-rot/src/bitd/bit-rot-scrub.h
index daec9ad8196..4e5f67bc021 100644
--- a/xlators/features/bit-rot/src/bitd/bit-rot-scrub.h
+++ b/xlators/features/bit-rot/src/bitd/bit-rot-scrub.h
@@ -8,9 +8,39 @@
    cases as published by the Free Software Foundation.
 */
 
-#ifndef __BIT_ROT__SCRUB_H__
+#ifndef __BIT_ROT_SCRUB_H__
 #define __BIT_ROT_SCRUB_H__
 
-void *br_scrubber (void *);
+#include <glusterfs/xlator.h>
+#include "bit-rot.h"
+
+void *
+br_fsscanner(void *);
+
+int32_t
+br_fsscan_schedule(xlator_t *);
+int32_t
+br_fsscan_reschedule(xlator_t *);
+int32_t
+br_fsscan_activate(xlator_t *);
+int32_t
+br_fsscan_deactivate(xlator_t *);
+int32_t
+br_fsscan_ondemand(xlator_t *);
+
+int32_t
+br_scrubber_handle_options(xlator_t *, br_private_t *, dict_t *);
+
+int32_t
+br_scrubber_monitor_init(xlator_t *, br_private_t *);
+
+int32_t
+br_scrubber_init(xlator_t *, br_private_t *);
+
+int32_t
+br_collect_bad_objects_from_children(xlator_t *this, dict_t *dict);
+
+void
+br_child_set_scrub_state(br_child_t *, gf_boolean_t);
 
 #endif /* __BIT_ROT_SCRUB_H__ */
diff --git a/xlators/features/bit-rot/src/bitd/bit-rot-ssm.c b/xlators/features/bit-rot/src/bitd/bit-rot-ssm.c
new file mode 100644
index 00000000000..753e31a3b23
--- /dev/null
+++ b/xlators/features/bit-rot/src/bitd/bit-rot-ssm.c
@@ -0,0 +1,124 @@
+/*
+   Copyright (c) 2015 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
+
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
+
+#include "bit-rot-ssm.h"
+#include "bit-rot-scrub.h"
+#include "bit-rot-bitd-messages.h"
+
+int
+br_scrub_ssm_noop(xlator_t *this)
+{
+    return 0;
+}
+
+int
+br_scrub_ssm_state_pause(xlator_t *this)
+{
+    br_private_t *priv = NULL;
+    struct br_monitor *scrub_monitor = NULL;
+
+    priv = this->private;
+    scrub_monitor = &priv->scrub_monitor;
+
+    gf_msg(this->name, GF_LOG_INFO, 0, BRB_MSG_GENERIC_SSM_INFO,
+           "Scrubber paused");
+    _br_monitor_set_scrub_state(scrub_monitor, BR_SCRUB_STATE_PAUSED);
+    return 0;
+}
+
+int
+br_scrub_ssm_state_ipause(xlator_t *this)
+{
+    br_private_t *priv = NULL;
+    struct br_monitor *scrub_monitor = NULL;
+
+    priv = this->private;
+    scrub_monitor = &priv->scrub_monitor;
+
+    gf_msg(this->name, GF_LOG_INFO, 0, BRB_MSG_GENERIC_SSM_INFO,
+           "Scrubber paused");
+    _br_monitor_set_scrub_state(scrub_monitor, BR_SCRUB_STATE_IPAUSED);
+    return 0;
+}
+
+int
+br_scrub_ssm_state_active(xlator_t *this)
+{
+    br_private_t *priv = NULL;
+    struct br_monitor *scrub_monitor = NULL;
+
+    priv = this->private;
+    scrub_monitor = &priv->scrub_monitor;
+
+    if (scrub_monitor->done) {
+        (void)br_fsscan_activate(this);
+    } else {
+        gf_msg(this->name, GF_LOG_INFO, 0, BRB_MSG_GENERIC_SSM_INFO,
+               "Scrubbing resumed");
+        _br_monitor_set_scrub_state(scrub_monitor, BR_SCRUB_STATE_ACTIVE);
+    }
+
+    return 0;
+}
+
+int
+br_scrub_ssm_state_stall(xlator_t *this)
+{
+    br_private_t *priv = NULL;
+    struct br_monitor *scrub_monitor = NULL;
+
+    priv = this->private;
+    scrub_monitor = &priv->scrub_monitor;
+
+    gf_msg(this->name, GF_LOG_INFO, 0, BRB_MSG_GENERIC_SSM_INFO,
+           "Volume is under active scrubbing. Pausing scrub..");
+    _br_monitor_set_scrub_state(scrub_monitor, BR_SCRUB_STATE_STALLED);
+    return 0;
+}
+
+static br_scrub_ssm_call *br_scrub_ssm[BR_SCRUB_MAXSTATES][BR_SCRUB_MAXEVENTS] =
+    {
+        /* INACTIVE */
+        {br_fsscan_schedule, br_scrub_ssm_state_ipause, br_scrub_ssm_noop},
+        /* PENDING  */
+        {br_fsscan_reschedule, br_fsscan_deactivate, br_fsscan_ondemand},
+        /* ACTIVE   */
+        {br_scrub_ssm_noop, br_scrub_ssm_state_stall, br_scrub_ssm_noop},
+        /* PAUSED   */
+        {br_fsscan_activate, br_scrub_ssm_noop, br_scrub_ssm_noop},
+        /* IPAUSED  */
+        {br_fsscan_schedule, br_scrub_ssm_noop, br_scrub_ssm_noop},
+        /* STALLED  */
+        {br_scrub_ssm_state_active, br_scrub_ssm_noop, br_scrub_ssm_noop},
+};
+
+int32_t
+br_scrub_state_machine(xlator_t *this, gf_boolean_t scrub_ondemand)
+{
+    br_private_t *priv = NULL;
+    br_scrub_ssm_call *call = NULL;
+    struct br_scrubber *fsscrub = NULL;
+    br_scrub_state_t currstate = 0;
+    br_scrub_event_t event = 0;
+    struct br_monitor *scrub_monitor = NULL;
+
+    priv = this->private;
+    fsscrub = &priv->fsscrub;
+    scrub_monitor = &priv->scrub_monitor;
+
+    currstate = scrub_monitor->state;
+    if (scrub_ondemand)
+        event = BR_SCRUB_EVENT_ONDEMAND;
+    else
+        event = _br_child_get_scrub_event(fsscrub);
+
+    call = br_scrub_ssm[currstate][event];
+    return call(this);
+}
diff --git a/xlators/features/bit-rot/src/bitd/bit-rot-ssm.h b/xlators/features/bit-rot/src/bitd/bit-rot-ssm.h
new file mode 100644
index 00000000000..37b45a42eac
--- /dev/null
+++ b/xlators/features/bit-rot/src/bitd/bit-rot-ssm.h
@@ -0,0 +1,38 @@
+/*
+   Copyright (c) 2015 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
+
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
+
+#ifndef __BIT_ROT_SSM_H__
+#define __BIT_ROT_SSM_H__
+
+#include <glusterfs/xlator.h>
+
+typedef enum br_scrub_state {
+    BR_SCRUB_STATE_INACTIVE = 0,
+    BR_SCRUB_STATE_PENDING,
+    BR_SCRUB_STATE_ACTIVE,
+    BR_SCRUB_STATE_PAUSED,
+    BR_SCRUB_STATE_IPAUSED,
+    BR_SCRUB_STATE_STALLED,
+    BR_SCRUB_MAXSTATES,
+} br_scrub_state_t;
+
+typedef enum br_scrub_event {
+    BR_SCRUB_EVENT_SCHEDULE = 0,
+    BR_SCRUB_EVENT_PAUSE,
+    BR_SCRUB_EVENT_ONDEMAND,
+    BR_SCRUB_MAXEVENTS,
+} br_scrub_event_t;
+
+struct br_monitor;
+
+int32_t
+br_scrub_state_machine(xlator_t *, gf_boolean_t);
+
+#endif /* __BIT_ROT_SSM_H__ */
diff --git a/xlators/features/bit-rot/src/bitd/bit-rot.c b/xlators/features/bit-rot/src/bitd/bit-rot.c
index 1202ca260eb..a2f1c343a1d 100644
--- a/xlators/features/bit-rot/src/bitd/bit-rot.c
+++ b/xlators/features/bit-rot/src/bitd/bit-rot.c
@@ -8,95 +8,82 @@
    cases as published by the Free Software Foundation.
 */
 
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
-
 #include <ctype.h>
-#include <sys/uio.h>
 
-#include "glusterfs.h"
-#include "xlator.h"
-#include "logging.h"
-#include "compat-errno.h"
+#include <glusterfs/logging.h>
+#include <glusterfs/compat-errno.h>
 
 #include "bit-rot.h"
 #include "bit-rot-scrub.h"
 #include <pthread.h>
+#include "bit-rot-bitd-messages.h"
 
-static int
-br_find_child_index (xlator_t *this, xlator_t *child)
-{
-        br_private_t *priv   = NULL;
-        int           i      = -1;
-        int           index  = -1;
+#define BR_HASH_CALC_READ_SIZE (128 * 1024)
 
-        GF_VALIDATE_OR_GOTO ("bit-rot", this, out);
-        GF_VALIDATE_OR_GOTO (this->name, this->private, out);
-        GF_VALIDATE_OR_GOTO (this->name, child, out);
+typedef int32_t(br_child_handler)(xlator_t *, br_child_t *);
 
-        priv = this->private;
+struct br_child_event {
+    xlator_t *this;
 
-        for (i = 0; i < priv->child_count; i++) {
-                if (child == priv->children[i].xl) {
-                        index = i;
-                        break;
-                }
-        }
+    br_child_t *child;
 
-out:
-        return index;
-}
+    br_child_handler *call;
 
-static void
-br_free_children (xlator_t *this)
+    struct list_head list;
+};
+
+static int
+br_find_child_index(xlator_t *this, xlator_t *child)
 {
-        br_private_t *priv = NULL;
-        int32_t       i    = 0;
-        br_child_t   *child = NULL;
+    br_private_t *priv = NULL;
+    int i = -1;
+    int index = -1;
 
-        priv = this->private;
+    GF_VALIDATE_OR_GOTO("bit-rot", this, out);
+    GF_VALIDATE_OR_GOTO(this->name, this->private, out);
+    GF_VALIDATE_OR_GOTO(this->name, child, out);
 
-        for (i = 0; i < priv->child_count; i++) {
-                child = &priv->children[i];
-                mem_pool_destroy (child->timer_pool);
-                list_del_init (&priv->children[i].list);
-        }
+    priv = this->private;
 
-        GF_FREE (priv->children);
+    for (i = 0; i < priv->child_count; i++) {
+        if (child == priv->children[i].xl) {
+            index = i;
+            break;
+        }
+    }
 
-        priv->children = NULL;
+out:
+    return index;
 }
 
 br_child_t *
-br_get_child_from_brick_path (xlator_t *this, char *brick_path)
+br_get_child_from_brick_path(xlator_t *this, char *brick_path)
 {
-        br_private_t *priv  = NULL;
-        br_child_t   *child = NULL;
-        br_child_t   *tmp   = NULL;
-        int           i     = 0;
+    br_private_t *priv = NULL;
+    br_child_t *child = NULL;
+    br_child_t *tmp = NULL;
+    int i = 0;
 
-        GF_VALIDATE_OR_GOTO ("bit-rot", this, out);
-        GF_VALIDATE_OR_GOTO (this->name, this->private, out);
-        GF_VALIDATE_OR_GOTO (this->name, brick_path, out);
+    GF_VALIDATE_OR_GOTO("bit-rot", this, out);
+    GF_VALIDATE_OR_GOTO(this->name, this->private, out);
+    GF_VALIDATE_OR_GOTO(this->name, brick_path, out);
 
-        priv = this->private;
+    priv = this->private;
 
-        pthread_mutex_lock (&priv->lock);
-        {
-                for (i = 0; i < priv->child_count; i++) {
-                        tmp = &priv->children[i];
-                        if (!strcmp (tmp->brick_path, brick_path)) {
-                                child = tmp;
-                                break;
-                        }
-                }
+    pthread_mutex_lock(&priv->lock);
+    {
+        for (i = 0; i < priv->child_count; i++) {
+            tmp = &priv->children[i];
+            if (!strcmp(tmp->brick_path, brick_path)) {
+                child = tmp;
+                break;
+            }
         }
-        pthread_mutex_unlock (&priv->lock);
+    }
+    pthread_mutex_unlock(&priv->lock);
 
 out:
-        return child;
+    return child;
 }
 
 /**
@@ -104,18 +91,18 @@ out:
  * needed -- later.
  */
 void *
-br_brick_init (void *xl, struct gf_brick_spec *brick)
+br_brick_init(void *xl, struct gf_brick_spec *brick)
 {
-        return brick;
+    return brick;
 }
 
 /**
  * and cleanup things here when allocated br_brick_init().
  */
 void
-br_brick_fini (void *xl, char *brick, void *data)
+br_brick_fini(void *xl, char *brick, void *data)
 {
-        return;
+    return;
 }
 
 /**
@@ -126,111 +113,110 @@ br_brick_fini (void *xl, char *brick, void *data)
  * FIX: Send the string length as part of the signature struct and
  *      change stub to handle this change.
  */
-static inline br_isignature_t *
-br_prepare_signature (const unsigned char *sign,
-                      unsigned long hashlen,
-                      int8_t hashtype, br_object_t *object)
+static br_isignature_t *
+br_prepare_signature(const unsigned char *sign, unsigned long hashlen,
+                     int8_t hashtype, br_object_t *object)
 {
-        br_isignature_t *signature = NULL;
+    br_isignature_t *signature = NULL;
 
-        /* TODO: use mem-pool */
-        signature = GF_CALLOC (1, signature_size (hashlen + 1),
-                               gf_br_stub_mt_signature_t);
-        if (!signature)
-                return NULL;
+    /* TODO: use mem-pool */
+    signature = GF_CALLOC(1, signature_size(hashlen + 1),
+                          gf_br_stub_mt_signature_t);
+    if (!signature)
+        return NULL;
 
-        /* object version */
-        signature->signedversion = object->signedversion;
+    /* object version */
+    signature->signedversion = object->signedversion;
 
-        /* signature length & type */
-        signature->signaturelen = hashlen;
-        signature->signaturetype = hashtype;
+    /* signature length & type */
+    signature->signaturelen = hashlen;
+    signature->signaturetype = hashtype;
 
-        /* signature itself */
-        memcpy (signature->signature, (char *)sign, hashlen);
-        signature->signature[hashlen+1] = '\0';
+    /* signature itself */
+    memcpy(signature->signature, (char *)sign, hashlen);
+    signature->signature[hashlen + 1] = '\0';
 
-        return signature;
+    return signature;
 }
 
 gf_boolean_t
-bitd_is_bad_file (xlator_t *this, br_child_t *child, loc_t *loc, fd_t *fd)
+bitd_is_bad_file(xlator_t *this, br_child_t *child, loc_t *loc, fd_t *fd)
 {
-        int32_t       ret      = -1;
-        dict_t       *xattr    = NULL;
-        inode_t      *inode    = NULL;
-        gf_boolean_t  bad_file = _gf_false;
+    int32_t ret = -1;
+    dict_t *xattr = NULL;
+    inode_t *inode = NULL;
+    gf_boolean_t bad_file = _gf_false;
 
-        GF_VALIDATE_OR_GOTO ("bit-rot", this, out);
+    GF_VALIDATE_OR_GOTO("bit-rot", this, out);
 
-        inode = (loc) ? loc->inode : fd->inode;
+    inode = (loc) ? loc->inode : fd->inode;
 
-        if (fd)
-                ret = syncop_fgetxattr (child->xl, fd, &xattr,
-                                        "trusted.glusterfs.bad-file", NULL,
-                                        NULL);
-        else if (loc)
-                ret = syncop_getxattr (child->xl, loc, &xattr,
-                                       "trusted.glusterfs.bad-file", NULL,
-                                       NULL);
+    if (fd)
+        ret = syncop_fgetxattr(child->xl, fd, &xattr, BITROT_OBJECT_BAD_KEY,
+                               NULL, NULL);
+    else if (loc)
+        ret = syncop_getxattr(child->xl, loc, &xattr, BITROT_OBJECT_BAD_KEY,
+                              NULL, NULL);
 
-        if (!ret) {
-                gf_log (this->name, GF_LOG_DEBUG, "[GFID: %s] is marked "
-                        "corrupted", uuid_utoa (inode->gfid));
-                bad_file = _gf_true;
-        }
+    if (!ret) {
+        gf_msg_debug(this->name, 0, "[GFID: %s] is marked corrupted",
+                     uuid_utoa(inode->gfid));
+        bad_file = _gf_true;
+    }
 
-        if (xattr)
-                dict_unref (xattr);
+    if (xattr)
+        dict_unref(xattr);
 
 out:
-        return bad_file;
+    return bad_file;
 }
 
 /**
  * Do a lookup on the gfid present within the object.
  */
-static inline int32_t
-br_object_lookup (xlator_t *this, br_object_t *object,
-                  struct iatt *iatt, inode_t **linked_inode)
+static int32_t
+br_object_lookup(xlator_t *this, br_object_t *object, struct iatt *iatt,
+                 inode_t **linked_inode)
 {
-	int      ret          = -EINVAL;
-	loc_t    loc          = {0, };
-	inode_t *inode        = NULL;
-
-        GF_VALIDATE_OR_GOTO ("bit-rot", this, out);
-        GF_VALIDATE_OR_GOTO (this->name, object, out);
-
-	inode = inode_find (object->child->table, object->gfid);
-
-        if (inode)
-                loc.inode = inode;
-        else
-                loc.inode = inode_new (object->child->table);
-
-	if (!loc.inode) {
-                ret = -ENOMEM;
-		goto out;
-        }
-
-	gf_uuid_copy (loc.gfid, object->gfid);
-
-	ret = syncop_lookup (object->child->xl, &loc, iatt, NULL, NULL, NULL);
-	if (ret < 0)
-		goto out;
-
-        /*
-         * The file might have been deleted by the application
-         * after getting the event, but before doing a lookup.
-         * So use linked_inode after inode_link is done.
-         */
-	*linked_inode = inode_link (loc.inode, NULL, NULL, iatt);
-	if (*linked_inode)
-		inode_lookup (*linked_inode);
+    int ret = -EINVAL;
+    loc_t loc = {
+        0,
+    };
+    inode_t *inode = NULL;
+
+    GF_VALIDATE_OR_GOTO("bit-rot", this, out);
+    GF_VALIDATE_OR_GOTO(this->name, object, out);
+
+    inode = inode_find(object->child->table, object->gfid);
+
+    if (inode)
+        loc.inode = inode;
+    else
+        loc.inode = inode_new(object->child->table);
+
+    if (!loc.inode) {
+        ret = -ENOMEM;
+        goto out;
+    }
+
+    gf_uuid_copy(loc.gfid, object->gfid);
+
+    ret = syncop_lookup(object->child->xl, &loc, iatt, NULL, NULL, NULL);
+    if (ret < 0)
+        goto out;
+
+    /*
+     * The file might have been deleted by the application
+     * after getting the event, but before doing a lookup.
+     * So use linked_inode after inode_link is done.
+     */
+    *linked_inode = inode_link(loc.inode, NULL, NULL, iatt);
+    if (*linked_inode)
+        inode_lookup(*linked_inode);
 
 out:
-	loc_wipe (&loc);
-	return ret;
+    loc_wipe(&loc);
+    return ret;
 }
 
 /**
@@ -238,43 +224,45 @@ out:
  * know that open is being done by bitd because syncop framework does not allow
  * passing xdata -- may be use frame->root->pid itself.
  */
-static inline int32_t
-br_object_open (xlator_t *this,
-                br_object_t *object, inode_t *inode, fd_t **openfd)
+static int32_t
+br_object_open(xlator_t *this, br_object_t *object, inode_t *inode,
+               fd_t **openfd)
 {
-        int32_t      ret   = -1;
-        fd_t        *fd   = NULL;
-        loc_t        loc   = {0, };
-
-        GF_VALIDATE_OR_GOTO ("bit-rot", this, out);
-        GF_VALIDATE_OR_GOTO (this->name, object, out);
-        GF_VALIDATE_OR_GOTO (this->name, inode, out);
-
-        ret = -EINVAL;
-        fd = fd_create (inode, 0);
-        if (!fd) {
-                gf_log (this->name, GF_LOG_ERROR, "failed to create fd for the "
-                        "inode %s", uuid_utoa (inode->gfid));
-                goto out;
-        }
-
-        loc.inode = inode_ref (inode);
-	gf_uuid_copy (loc.gfid, inode->gfid);
-
-        ret = syncop_open (object->child->xl, &loc, O_RDONLY, fd, NULL, NULL);
-	if (ret) {
-                br_log_object (this, "open", inode->gfid, -ret);
-		fd_unref (fd);
-		fd = NULL;
-	} else {
-		fd_bind (fd);
-                *openfd = fd;
-	}
-
-        loc_wipe (&loc);
+    int32_t ret = -1;
+    fd_t *fd = NULL;
+    loc_t loc = {
+        0,
+    };
+
+    GF_VALIDATE_OR_GOTO("bit-rot", this, out);
+    GF_VALIDATE_OR_GOTO(this->name, object, out);
+    GF_VALIDATE_OR_GOTO(this->name, inode, out);
+
+    ret = -EINVAL;
+    fd = fd_create(inode, 0);
+    if (!fd) {
+        gf_smsg(this->name, GF_LOG_ERROR, 0, BRB_MSG_FD_CREATE_FAILED,
+                "gfid=%s", uuid_utoa(inode->gfid), NULL);
+        goto out;
+    }
+
+    loc.inode = inode_ref(inode);
+    gf_uuid_copy(loc.gfid, inode->gfid);
+
+    ret = syncop_open(object->child->xl, &loc, O_RDONLY, fd, NULL, NULL);
+    if (ret) {
+        br_log_object(this, "open", inode->gfid, -ret);
+        fd_unref(fd);
+        fd = NULL;
+    } else {
+        fd_bind(fd);
+        *openfd = fd;
+    }
+
+    loc_wipe(&loc);
 
 out:
-        return ret;
+    return ret;
 }
 
 /**
@@ -282,301 +270,406 @@ out:
  * and return the buffer.
  */
 static int32_t
-br_object_read_block_and_sign (xlator_t *this, fd_t *fd, br_child_t *child,
-                               off_t offset, size_t size, SHA256_CTX *sha256)
+br_object_read_block_and_sign(xlator_t *this, fd_t *fd, br_child_t *child,
+                              off_t offset, size_t size, SHA256_CTX *sha256)
 {
-        int32_t        ret    = -1;
-        struct iovec  *iovec  = NULL;
-        struct iobref *iobref = NULL;
-        int            count  = 0;
-        int            i      = 0;
-
-        GF_VALIDATE_OR_GOTO ("bit-rot", this, out);
-        GF_VALIDATE_OR_GOTO (this->name, fd, out);
-        GF_VALIDATE_OR_GOTO (this->name, fd->inode, out);
-        GF_VALIDATE_OR_GOTO (this->name, child, out);
-
-        ret = syncop_readv (child->xl, fd,
-                            size, offset, 0, &iovec, &count, &iobref, NULL,
-                            NULL);
-
-        if (ret < 0) {
-                gf_log (this->name, GF_LOG_ERROR, "readv on %s failed (%s)",
-                        uuid_utoa (fd->inode->gfid), strerror (errno));
-                ret = -1;
-                goto out;
-        }
-
-        if (ret == 0)
-                goto out;
-
-        for (i = 0; i < count; i++) {
-                SHA256_Update (sha256,
-                               (const unsigned char *) (iovec[i].iov_base),
-                               iovec[i].iov_len);
+    int32_t ret = -1;
+    tbf_t *tbf = NULL;
+    struct iovec *iovec = NULL;
+    struct iobref *iobref = NULL;
+    br_private_t *priv = NULL;
+    int count = 0;
+    int i = 0;
+
+    GF_VALIDATE_OR_GOTO("bit-rot", this, out);
+    GF_VALIDATE_OR_GOTO(this->name, fd, out);
+    GF_VALIDATE_OR_GOTO(this->name, fd->inode, out);
+    GF_VALIDATE_OR_GOTO(this->name, child, out);
+    GF_VALIDATE_OR_GOTO(this->name, this->private, out);
+
+    priv = this->private;
+
+    GF_VALIDATE_OR_GOTO(this->name, priv->tbf, out);
+    tbf = priv->tbf;
+
+    ret = syncop_readv(child->xl, fd, size, offset, 0, &iovec, &count, &iobref,
+                       NULL, NULL, NULL);
+
+    if (ret < 0) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, BRB_MSG_READV_FAILED,
+                "gfid=%s", uuid_utoa(fd->inode->gfid), NULL);
+        ret = -1;
+        goto out;
+    }
+
+    if (ret == 0)
+        goto out;
+
+    for (i = 0; i < count; i++) {
+        TBF_THROTTLE_BEGIN(tbf, TBF_OP_HASH, iovec[i].iov_len);
+        {
+            SHA256_Update(sha256, (const unsigned char *)(iovec[i].iov_base),
+                          iovec[i].iov_len);
         }
+        TBF_THROTTLE_BEGIN(tbf, TBF_OP_HASH, iovec[i].iov_len);
+    }
 
- out:
-        if (iovec)
-                GF_FREE (iovec);
+out:
+    if (iovec)
+        GF_FREE(iovec);
 
-        if (iobref)
-                iobref_unref (iobref);
+    if (iobref)
+        iobref_unref(iobref);
 
-        return ret;
+    return ret;
 }
 
 int32_t
-br_calculate_obj_checksum (unsigned char *md,
-                           br_child_t *child, fd_t *fd, struct iatt *iatt)
+br_calculate_obj_checksum(unsigned char *md, br_child_t *child, fd_t *fd,
+                          struct iatt *iatt)
 {
-        int32_t   ret    = -1;
-        off_t     offset = 0;
-        size_t    block  = 128 * 1024;  /* 128K block size */
-        xlator_t *this   = NULL;
-
-        SHA256_CTX       sha256;
+    int32_t ret = -1;
+    off_t offset = 0;
+    size_t block = BR_HASH_CALC_READ_SIZE;
+    xlator_t *this = NULL;
 
-        GF_VALIDATE_OR_GOTO ("bit-rot", child, out);
-        GF_VALIDATE_OR_GOTO ("bit-rot", iatt, out);
-        GF_VALIDATE_OR_GOTO ("bit-rot", fd, out);
+    SHA256_CTX sha256;
 
-        this = child->this;
+    GF_VALIDATE_OR_GOTO("bit-rot", child, out);
+    GF_VALIDATE_OR_GOTO("bit-rot", iatt, out);
+    GF_VALIDATE_OR_GOTO("bit-rot", fd, out);
 
-        SHA256_Init (&sha256);
+    this = child->this;
 
-        while (1) {
-                ret = br_object_read_block_and_sign (this, fd, child,
-                                                     offset, block, &sha256);
-                if (ret < 0) {
-                        gf_log (this->name, GF_LOG_ERROR, "reading block with "
-                                "offset %lu of object %s failed", offset,
-                                uuid_utoa (fd->inode->gfid));
-                        break;
-                }
+    SHA256_Init(&sha256);
 
-                if (ret == 0)
-                        break;
-
-                offset += ret;
+    while (1) {
+        ret = br_object_read_block_and_sign(this, fd, child, offset, block,
+                                            &sha256);
+        if (ret < 0) {
+            gf_smsg(this->name, GF_LOG_ERROR, 0, BRB_MSG_BLOCK_READ_FAILED,
+                    "offset=%" PRIu64, offset, "object-gfid=%s",
+                    uuid_utoa(fd->inode->gfid), NULL);
+            break;
         }
 
         if (ret == 0)
-                SHA256_Final (md, &sha256);
+            break;
 
- out:
-        return ret;
+        offset += ret;
+    }
+
+    if (ret == 0)
+        SHA256_Final(md, &sha256);
+
+out:
+    return ret;
 }
 
-static inline int32_t
-br_object_checksum (unsigned char *md,
-                    br_object_t *object, fd_t *fd, struct iatt *iatt)
+static int32_t
+br_object_checksum(unsigned char *md, br_object_t *object, fd_t *fd,
+                   struct iatt *iatt)
 {
-        return br_calculate_obj_checksum (md, object->child, fd,  iatt);
+    return br_calculate_obj_checksum(md, object->child, fd, iatt);
 }
 
-static inline int32_t
-br_object_read_sign (inode_t *linked_inode, fd_t *fd, br_object_t *object,
-                     struct iatt *iatt)
+static int32_t
+br_object_read_sign(inode_t *linked_inode, fd_t *fd, br_object_t *object,
+                    struct iatt *iatt)
 {
-        int32_t          ret           = -1;
-        xlator_t        *this          = NULL;
-        dict_t          *xattr         = NULL;
-        unsigned char   *md            = NULL;
-        br_isignature_t *sign          = NULL;
-
-        GF_VALIDATE_OR_GOTO ("bit-rot", object, out);
-        GF_VALIDATE_OR_GOTO ("bit-rot", linked_inode, out);
-        GF_VALIDATE_OR_GOTO ("bit-rot", fd, out);
-
-        this = object->this;
-
-        md = GF_CALLOC (SHA256_DIGEST_LENGTH, sizeof (*md), gf_common_mt_char);
-        if (!md) {
-                gf_log (this->name, GF_LOG_ERROR, "failed to allocate memory "
-                        "for saving hash of the object %s",
-                        uuid_utoa (fd->inode->gfid));
-                goto out;
-        }
-
-        ret = br_object_checksum (md, object, fd, iatt);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "calculating checksum for "
-                        "the object %s failed", uuid_utoa (linked_inode->gfid));
-                goto free_signature;
-        }
-
-        sign = br_prepare_signature (md, SHA256_DIGEST_LENGTH,
-                                     BR_SIGNATURE_TYPE_SHA256, object);
-        if (!sign) {
-                gf_log (this->name, GF_LOG_ERROR, "failed to get the signature "
-                        "for the object %s", uuid_utoa (fd->inode->gfid));
-                goto free_signature;
-        }
-
-        xattr = dict_for_key_value
-                (GLUSTERFS_SET_OBJECT_SIGNATURE,
-                 (void *)sign, signature_size (SHA256_DIGEST_LENGTH));
-
-        if (!xattr) {
-                gf_log (this->name, GF_LOG_ERROR, "dict allocation for signing"
-                        " failed for the object %s",
-                        uuid_utoa (fd->inode->gfid));
-                goto free_isign;
-        }
-
-        ret = syncop_fsetxattr (object->child->xl, fd, xattr, 0, NULL, NULL);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "fsetxattr of signature to "
-                        "the object %s failed", uuid_utoa (fd->inode->gfid));
-                goto unref_dict;
-        }
-
-        ret = 0;
-
- unref_dict:
-        dict_unref (xattr);
- free_isign:
-        GF_FREE (sign);
- free_signature:
-        GF_FREE (md);
- out:
-        return ret;
+    int32_t ret = -1;
+    xlator_t *this = NULL;
+    dict_t *xattr = NULL;
+    unsigned char *md = NULL;
+    br_isignature_t *sign = NULL;
+
+    GF_VALIDATE_OR_GOTO("bit-rot", object, out);
+    GF_VALIDATE_OR_GOTO("bit-rot", linked_inode, out);
+    GF_VALIDATE_OR_GOTO("bit-rot", fd, out);
+
+    this = object->this;
+
+    md = GF_MALLOC(SHA256_DIGEST_LENGTH, gf_common_mt_char);
+    if (!md) {
+        gf_smsg(this->name, GF_LOG_ERROR, ENOMEM, BRB_MSG_SAVING_HASH_FAILED,
+                "object-gfid=%s", uuid_utoa(fd->inode->gfid), NULL);
+        goto out;
+    }
+
+    ret = br_object_checksum(md, object, fd, iatt);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_ERROR, 0, BRB_MSG_CALC_CHECKSUM_FAILED,
+                "object-gfid=%s", uuid_utoa(linked_inode->gfid), NULL);
+        goto free_signature;
+    }
+
+    sign = br_prepare_signature(md, SHA256_DIGEST_LENGTH,
+                                BR_SIGNATURE_TYPE_SHA256, object);
+    if (!sign) {
+        gf_smsg(this->name, GF_LOG_ERROR, 0, BRB_MSG_GET_SIGN_FAILED,
+                "object-gfid=%s", uuid_utoa(fd->inode->gfid), NULL);
+        goto free_signature;
+    }
+
+    xattr = dict_for_key_value(GLUSTERFS_SET_OBJECT_SIGNATURE, (void *)sign,
+                               signature_size(SHA256_DIGEST_LENGTH), _gf_true);
+
+    if (!xattr) {
+        gf_smsg(this->name, GF_LOG_ERROR, 0, BRB_MSG_SET_SIGN_FAILED,
+                "dict-allocation object-gfid=%s", uuid_utoa(fd->inode->gfid),
+                NULL);
+        goto free_isign;
+    }
+
+    ret = syncop_fsetxattr(object->child->xl, fd, xattr, 0, NULL, NULL);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_ERROR, 0, BRB_MSG_SET_SIGN_FAILED,
+                "fsetxattr object-gfid=%s", uuid_utoa(fd->inode->gfid), NULL);
+        goto unref_dict;
+    }
+
+    ret = 0;
+
+unref_dict:
+    dict_unref(xattr);
+free_isign:
+    GF_FREE(sign);
+free_signature:
+    GF_FREE(md);
+out:
+    return ret;
 }
 
-static inline int br_object_sign_softerror (int32_t op_errno)
+static int
+br_object_sign_softerror(int32_t op_errno)
 {
-        return ((op_errno == ENOENT) || (op_errno == ESTALE)
-                || (op_errno == ENODATA));
+    return ((op_errno == ENOENT) || (op_errno == ESTALE) ||
+            (op_errno == ENODATA));
 }
 
 void
-br_log_object (xlator_t *this, char *op, uuid_t gfid, int32_t op_errno)
+br_log_object(xlator_t *this, char *op, uuid_t gfid, int32_t op_errno)
 {
-        int softerror = br_object_sign_softerror (op_errno);
-        gf_log (this->name, (softerror) ? GF_LOG_DEBUG : GF_LOG_ERROR,
-                "%s() failed on object %s [reason: %s]",
-                op, uuid_utoa (gfid), strerror (op_errno));
+    int softerror = br_object_sign_softerror(op_errno);
+    if (softerror) {
+        gf_msg_debug(this->name, 0,
+                     "%s() failed on object %s "
+                     "[reason: %s]",
+                     op, uuid_utoa(gfid), strerror(op_errno));
+    } else {
+        gf_smsg(this->name, GF_LOG_ERROR, op_errno, BRB_MSG_OP_FAILED, "op=%s",
+                op, "gfid=%s", uuid_utoa(gfid), NULL);
+    }
 }
 
 void
-br_log_object_path (xlator_t *this, char *op,
-                    const char *path, int32_t op_errno)
+br_log_object_path(xlator_t *this, char *op, const char *path, int32_t op_errno)
 {
-        int softerror = br_object_sign_softerror (op_errno);
-        gf_log (this->name, (softerror) ? GF_LOG_DEBUG : GF_LOG_ERROR,
-                "%s() failed on object %s [reason: %s]",
-                op, path, strerror (op_errno));
+    int softerror = br_object_sign_softerror(op_errno);
+    if (softerror) {
+        gf_msg_debug(this->name, 0,
+                     "%s() failed on object %s "
+                     "[reason: %s]",
+                     op, path, strerror(op_errno));
+    } else {
+        gf_smsg(this->name, GF_LOG_ERROR, op_errno, BRB_MSG_OP_FAILED, "op=%s",
+                op, "path=%s", path, NULL);
+    }
 }
 
-/**
- * Sign a given object. This routine runs full throttle. There needs to be
- * some form of priority scheduling and/or read burstness to avoid starving
- * (or kicking) client I/O's.
- */
-static inline int32_t br_sign_object (br_object_t *object)
+static void
+br_trigger_sign(xlator_t *this, br_child_t *child, inode_t *linked_inode,
+                loc_t *loc, gf_boolean_t need_reopen)
 {
-        int32_t         ret           = -1;
-        inode_t        *linked_inode  = NULL;
-        xlator_t       *this          = NULL;
-        fd_t           *fd            = NULL;
-        struct iatt     iatt          = {0, };
-        pid_t           pid           = GF_CLIENT_PID_BITD;
-
-        GF_VALIDATE_OR_GOTO ("bit-rot", object, out);
-
-        this = object->this;
-
-        /**
-         * FIXME: This is required as signing an object is restricted to
-         * clients with special frame->root->pid. Change the way client
-         * pid is set.
-         */
-        syncopctx_setfspid (&pid);
+    fd_t *fd = NULL;
+    int32_t ret = -1;
+    uint32_t val = 0;
+    dict_t *dict = NULL;
+    pid_t pid = GF_CLIENT_PID_BITD;
+
+    syncopctx_setfspid(&pid);
+
+    val = (need_reopen == _gf_true) ? BR_OBJECT_REOPEN : BR_OBJECT_RESIGN;
+
+    dict = dict_new();
+    if (!dict)
+        goto out;
+
+    ret = dict_set_uint32(dict, BR_REOPEN_SIGN_HINT_KEY, val);
+    if (ret)
+        goto cleanup_dict;
+
+    ret = -1;
+    fd = fd_create(linked_inode, 0);
+    if (!fd) {
+        gf_smsg(this->name, GF_LOG_ERROR, 0, BRB_MSG_FD_CREATE_FAILED,
+                "gfid=%s", uuid_utoa(linked_inode->gfid), NULL);
+        goto cleanup_dict;
+    }
+
+    ret = syncop_open(child->xl, loc, O_RDWR, fd, NULL, NULL);
+    if (ret) {
+        br_log_object(this, "open", linked_inode->gfid, -ret);
+        goto unref_fd;
+    }
+
+    fd_bind(fd);
+
+    ret = syncop_fsetxattr(child->xl, fd, dict, 0, NULL, NULL);
+    if (ret)
+        br_log_object(this, "fsetxattr", linked_inode->gfid, -ret);
+
+    /* passthough: fd_unref() */
+
+unref_fd:
+    fd_unref(fd);
+cleanup_dict:
+    dict_unref(dict);
+out:
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_WARNING, 0, BRB_MSG_TRIGGER_SIGN_FAILED,
+                "gfid=%s", uuid_utoa(linked_inode->gfid), "reopen-hint-val=%d",
+                val, NULL);
+    }
+}
 
-        ret = br_object_lookup (this, object, &iatt, &linked_inode);
-        if (ret) {
-                br_log_object (this, "lookup", object->gfid, -ret);
-                goto out;
-        }
+static void
+br_object_resign(xlator_t *this, br_object_t *object, inode_t *linked_inode)
+{
+    loc_t loc = {
+        0,
+    };
 
-        ret = br_object_open (this, object, linked_inode, &fd);
-        if (!fd) {
-                br_log_object (this, "open", object->gfid, -ret);
-                goto unref_inode;
-        }
+    loc.inode = inode_ref(linked_inode);
+    gf_uuid_copy(loc.gfid, linked_inode->gfid);
 
-        /**
-         * we have an open file descriptor on the object. from here on,
-         * do not be generous to file operation errors.
-         */
-        gf_log (this->name, GF_LOG_DEBUG,
-                "Signing object [%s]", uuid_utoa (linked_inode->gfid));
-
-        ret = br_object_read_sign (linked_inode, fd, object, &iatt);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "reading and signing of the "
-                        "object %s failed", uuid_utoa (linked_inode->gfid));
-                goto unref_fd;
-        }
+    br_trigger_sign(this, object->child, linked_inode, &loc, _gf_false);
 
-        ret = 0;
+    loc_wipe(&loc);
+}
 
- unref_fd:
-        fd_unref (fd);
- unref_inode:
-        inode_unref (linked_inode);
- out:
-        return ret;
+/**
+ * Sign a given object. This routine runs full throttle. There needs to be
+ * some form of priority scheduling and/or read burstness to avoid starving
+ * (or kicking) client I/O's.
+ */
+static int32_t
+br_sign_object(br_object_t *object)
+{
+    int32_t ret = -1;
+    inode_t *linked_inode = NULL;
+    xlator_t *this = NULL;
+    fd_t *fd = NULL;
+    struct iatt iatt = {
+        0,
+    };
+    pid_t pid = GF_CLIENT_PID_BITD;
+    br_sign_state_t sign_info = BR_SIGN_NORMAL;
+
+    GF_VALIDATE_OR_GOTO("bit-rot", object, out);
+
+    this = object->this;
+
+    /**
+     * FIXME: This is required as signing an object is restricted to
+     * clients with special frame->root->pid. Change the way client
+     * pid is set.
+     */
+    syncopctx_setfspid(&pid);
+
+    ret = br_object_lookup(this, object, &iatt, &linked_inode);
+    if (ret) {
+        br_log_object(this, "lookup", object->gfid, -ret);
+        goto out;
+    }
+
+    /**
+     * For fd's that have notified for reopening, we send an explicit
+     * open() followed by a dummy write() call. This triggers the
+     * actual signing of the object.
+     */
+    sign_info = ntohl(object->sign_info);
+    if (sign_info == BR_SIGN_REOPEN_WAIT) {
+        br_object_resign(this, object, linked_inode);
+        goto unref_inode;
+    }
+
+    ret = br_object_open(this, object, linked_inode, &fd);
+    if (!fd) {
+        br_log_object(this, "open", object->gfid, -ret);
+        goto unref_inode;
+    }
+
+    /**
+     * we have an open file descriptor on the object. from here on,
+     * do not be generous to file operation errors.
+     */
+    gf_msg_debug(this->name, 0, "Signing object [%s]",
+                 uuid_utoa(linked_inode->gfid));
+
+    ret = br_object_read_sign(linked_inode, fd, object, &iatt);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_ERROR, 0, BRB_MSG_READ_AND_SIGN_FAILED,
+                "gfid=%s", uuid_utoa(linked_inode->gfid), NULL);
+        goto unref_fd;
+    }
+
+    ret = 0;
+
+unref_fd:
+    fd_unref(fd);
+unref_inode:
+    inode_unref(linked_inode);
+out:
+    return ret;
 }
 
-static inline br_object_t *__br_pick_object (br_private_t *priv)
+static br_object_t *
+__br_pick_object(br_private_t *priv)
 {
-        br_object_t *object = NULL;
+    br_object_t *object = NULL;
 
-        while (list_empty (&priv->obj_queue->objects)) {
-                pthread_cond_wait (&priv->object_cond, &priv->lock);
-        }
+    while (list_empty(&priv->obj_queue->objects)) {
+        pthread_cond_wait(&priv->object_cond, &priv->lock);
+    }
 
-        object = list_first_entry
-                (&priv->obj_queue->objects, br_object_t, list);
-        list_del_init (&object->list);
+    object = list_first_entry(&priv->obj_queue->objects, br_object_t, list);
+    list_del_init(&object->list);
 
-        return object;
+    return object;
 }
 
 /**
  * This is the place where the signing of the objects is triggered.
  */
 void *
-br_process_object (void *arg)
+br_process_object(void *arg)
 {
-        xlator_t     *this   = NULL;
-        br_object_t  *object = NULL;
-        br_private_t *priv   = NULL;
-        int32_t       ret    = -1;
-
-        this = arg;
-        priv = this->private;
-
-        THIS = this;
-
-        for (;;) {
-                pthread_mutex_lock (&priv->lock);
-                {
-                        object = __br_pick_object (priv);
-                }
-                pthread_mutex_unlock (&priv->lock);
-
-                ret = br_sign_object (object);
-                if (ret && !br_object_sign_softerror (-ret))
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "SIGNING FAILURE [%s]",
-                                uuid_utoa (object->gfid));
-                GF_FREE (object);
+    xlator_t *this = NULL;
+    br_object_t *object = NULL;
+    br_private_t *priv = NULL;
+    int32_t ret = -1;
+
+    this = arg;
+    priv = this->private;
+
+    THIS = this;
+
+    for (;;) {
+        pthread_mutex_lock(&priv->lock);
+        {
+            object = __br_pick_object(priv);
         }
+        pthread_mutex_unlock(&priv->lock);
 
-        return NULL;
+        ret = br_sign_object(object);
+        if (ret && !br_object_sign_softerror(-ret))
+            gf_smsg(this->name, GF_LOG_ERROR, 0, BRB_MSG_SET_SIGN_FAILED,
+                    "gfid=%s", uuid_utoa(object->gfid), NULL);
+        GF_FREE(object);
+    }
+
+    return NULL;
 }
 
 /**
@@ -593,70 +686,97 @@ br_process_object (void *arg)
  * NOTE: use call_time to instrument signing time in br_sign_object().
  */
 void
-br_add_object_to_queue (struct gf_tw_timer_list *timer,
-                        void *data, unsigned long call_time)
+br_add_object_to_queue(struct gf_tw_timer_list *timer, void *data,
+                       unsigned long call_time)
 {
-        br_object_t   *object = NULL;
-        xlator_t      *this   = NULL;
-        br_private_t  *priv   = NULL;
-
-        object = data;
-        this   = object->this;
-        priv   = this->private;
-
-        pthread_mutex_lock (&priv->lock);
-        {
-                list_add_tail (&object->list, &priv->obj_queue->objects);
-                pthread_cond_broadcast (&priv->object_cond);
-        }
-        pthread_mutex_unlock (&priv->lock);
-
-        mem_put (timer);
-        return;
+    br_object_t *object = NULL;
+    xlator_t *this = NULL;
+    br_private_t *priv = NULL;
+
+    object = data;
+    this = object->this;
+    priv = this->private;
+
+    THIS = this;
+
+    pthread_mutex_lock(&priv->lock);
+    {
+        list_add_tail(&object->list, &priv->obj_queue->objects);
+        pthread_cond_broadcast(&priv->object_cond);
+    }
+    pthread_mutex_unlock(&priv->lock);
+
+    if (timer)
+        mem_put(timer);
+    return;
 }
 
-static inline br_object_t *
-br_initialize_object (xlator_t *this, br_child_t *child, changelog_event_t *ev)
+static br_object_t *
+br_initialize_object(xlator_t *this, br_child_t *child, changelog_event_t *ev)
 {
-        br_object_t *object = NULL;
+    br_object_t *object = NULL;
 
-        object = GF_CALLOC (1, sizeof (*object), gf_br_mt_br_object_t);
-        if (!object)
-                goto out;
-        INIT_LIST_HEAD (&object->list);
+    object = GF_CALLOC(1, sizeof(*object), gf_br_mt_br_object_t);
+    if (!object)
+        goto out;
+    INIT_LIST_HEAD(&object->list);
 
-        object->this  = this;
-        object->child = child;
-        gf_uuid_copy (object->gfid, ev->u.releasebr.gfid);
+    object->this = this;
+    object->child = child;
+    gf_uuid_copy(object->gfid, ev->u.releasebr.gfid);
 
-        /* NOTE: it's BE, but no worry */
-        object->signedversion = ev->u.releasebr.version;
+    /* NOTE: it's BE, but no worry */
+    object->signedversion = ev->u.releasebr.version;
+    object->sign_info = ev->u.releasebr.sign_info;
 
 out:
-        return object;
+    return object;
 }
 
-static inline struct gf_tw_timer_list *
-br_initialize_timer (xlator_t *this, br_object_t *object, br_child_t *child,
-                     changelog_event_t *ev)
+static struct gf_tw_timer_list *
+br_initialize_timer(xlator_t *this, br_object_t *object, br_child_t *child,
+                    changelog_event_t *ev)
 {
-        br_private_t  *priv = NULL;
-        struct gf_tw_timer_list *timer = NULL;
+    br_private_t *priv = NULL;
+    struct gf_tw_timer_list *timer = NULL;
 
-        priv = this->private;
+    priv = this->private;
 
-        timer = mem_get0 (child->timer_pool);
-        if (!timer)
-                goto out;
-        INIT_LIST_HEAD (&timer->entry);
+    timer = mem_get0(child->timer_pool);
+    if (!timer)
+        goto out;
+    INIT_LIST_HEAD(&timer->entry);
+
+    timer->expires = priv->expiry_time;
+    if (!timer->expires)
+        timer->expires = 1;
 
-        timer->data     = object;
-        timer->expires  = priv->expiry_time;
-        timer->function = br_add_object_to_queue;
-        gf_tw_add_timer (priv->timer_wheel, timer);
+    timer->data = object;
+    timer->function = br_add_object_to_queue;
+    gf_tw_add_timer(priv->timer_wheel, timer);
 
 out:
-        return timer;
+    return timer;
+}
+
+static int32_t
+br_schedule_object_reopen(xlator_t *this, br_object_t *object,
+                          br_child_t *child, changelog_event_t *ev)
+{
+    struct gf_tw_timer_list *timer = NULL;
+
+    timer = br_initialize_timer(this, object, child, ev);
+    if (!timer)
+        gf_smsg(this->name, GF_LOG_ERROR, 0, BRB_MSG_SET_TIMER_FAILED,
+                "gfid=%s", uuid_utoa(object->gfid), NULL);
+    return timer ? 0 : -1;
+}
+
+static int32_t
+br_object_quicksign(xlator_t *this, br_object_t *object)
+{
+    br_add_object_to_queue(NULL, object, 0ULL);
+    return 0;
 }
 
 /**
@@ -669,194 +789,146 @@ out:
  * object as a single alloc and bifurcate their respective pointers.
  */
 void
-br_brick_callback (void *xl, char *brick,
-                   void *data, changelog_event_t *ev)
+br_brick_callback(void *xl, char *brick, void *data, changelog_event_t *ev)
 {
-        uuid_t gfid = {0,};
-        xlator_t                *this   = NULL;
-        br_object_t             *object = NULL;
-        br_child_t              *child  = NULL;
-        int32_t                  flags  = 0;
-        struct gf_tw_timer_list *timer  = NULL;
-
-        this = xl;
-
-        GF_VALIDATE_OR_GOTO (this->name, ev, out);
-        GF_VALIDATE_OR_GOTO ("bit-rot", this, out);
-        GF_VALIDATE_OR_GOTO (this->name, this->private, out);
-
-        GF_ASSERT (ev->ev_type == CHANGELOG_OP_TYPE_BR_RELEASE);
-        GF_ASSERT (!gf_uuid_is_null (ev->u.releasebr.gfid));
-
-        gf_uuid_copy (gfid, ev->u.releasebr.gfid);
-
-        gf_log (this->name, GF_LOG_DEBUG,
-                "RELEASE EVENT [GFID %s]", uuid_utoa (gfid));
-
-        flags = (int32_t)ntohl (ev->u.releasebr.flags);
-        if (flags == O_RDONLY) {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "Read only fd [GFID: %s], ignoring signing..",
-                        uuid_utoa (gfid));
-                goto out;
-        }
-
-        child = br_get_child_from_brick_path (this, brick);
-        if (!child) {
-                gf_log (this->name, GF_LOG_ERROR, "failed to get the subvolume "
-                        "for the brick %s", brick);
-                goto out;
-        }
-
-        object = br_initialize_object (this, child, ev);
-        if (!object) {
-                gf_log (this->name, GF_LOG_ERROR, "failed to allocate "
-                        "object memory [GFID: %s]", uuid_utoa (gfid));
-                goto out;
-        }
-
-        timer = br_initialize_timer (this, object, child, ev);
-        if (!timer) {
-                gf_log (this->name, GF_LOG_ERROR, "failed to allocate "
-                        "object expiry timer [GFID: %s]", uuid_utoa (gfid));
-                goto free_object;
-        }
-
-        gf_log (this->name, GF_LOG_DEBUG, "->callback: brick [%s], type [%d]\n",
-                brick, ev->ev_type);
-
-        return;
-
- free_object:
-        GF_FREE (object);
+    int32_t ret = 0;
+    uuid_t gfid = {
+        0,
+    };
+    xlator_t *this = NULL;
+    br_object_t *object = NULL;
+    br_child_t *child = NULL;
+    br_sign_state_t sign_info = BR_SIGN_INVALID;
+
+    this = xl;
+
+    GF_VALIDATE_OR_GOTO(this->name, ev, out);
+    GF_VALIDATE_OR_GOTO("bit-rot", this, out);
+    GF_VALIDATE_OR_GOTO(this->name, this->private, out);
+
+    GF_ASSERT(ev->ev_type == CHANGELOG_OP_TYPE_BR_RELEASE);
+    GF_ASSERT(!gf_uuid_is_null(ev->u.releasebr.gfid));
+
+    gf_uuid_copy(gfid, ev->u.releasebr.gfid);
+
+    gf_msg_debug(this->name, 0, "RELEASE EVENT [GFID %s]", uuid_utoa(gfid));
+
+    child = br_get_child_from_brick_path(this, brick);
+    if (!child) {
+        gf_smsg(this->name, GF_LOG_ERROR, 0, BRB_MSG_GET_SUBVOL_FAILED,
+                "brick=%s", brick, NULL);
+        goto out;
+    }
+
+    object = br_initialize_object(this, child, ev);
+    if (!object) {
+        gf_smsg(this->name, GF_LOG_ERROR, ENOMEM, BRB_MSG_NO_MEMORY,
+                "object-gfid=%s", uuid_utoa(gfid), NULL);
+        goto out;
+    }
+
+    /* sanity check */
+    sign_info = ntohl(object->sign_info);
+    GF_ASSERT(sign_info != BR_SIGN_NORMAL);
+
+    if (sign_info == BR_SIGN_REOPEN_WAIT)
+        ret = br_schedule_object_reopen(this, object, child, ev);
+    else
+        ret = br_object_quicksign(this, object);
+
+    if (ret)
+        goto free_object;
+
+    gf_msg_debug(this->name, 0, "->callback: brick [%s], type [%d]\n", brick,
+                 ev->ev_type);
+    return;
+
+free_object:
+    GF_FREE(object);
 out:
-        return;
+    return;
 }
 
 void
-br_fill_brick_spec (struct gf_brick_spec *brick, char *path)
+br_fill_brick_spec(struct gf_brick_spec *brick, char *path)
 {
-        brick->brick_path = gf_strdup (path);
-        brick->filter = CHANGELOG_OP_TYPE_BR_RELEASE;
-
-        brick->init         = br_brick_init;
-        brick->fini         = br_brick_fini;
-        brick->callback     = br_brick_callback;
-        brick->connected    = NULL;
-        brick->disconnected = NULL;
+    brick->brick_path = gf_strdup(path);
+    brick->filter = CHANGELOG_OP_TYPE_BR_RELEASE;
+
+    brick->init = br_brick_init;
+    brick->fini = br_brick_fini;
+    brick->callback = br_brick_callback;
+    brick->connected = NULL;
+    brick->disconnected = NULL;
 }
 
-static inline gf_boolean_t
-br_time_equal (br_child_t *child, struct timeval *tv)
+static gf_boolean_t
+br_check_object_need_sign(xlator_t *this, dict_t *xattr, br_child_t *child)
 {
-        if ((child->tv.tv_sec == tv->tv_sec) &&
-            (child->tv.tv_usec == tv->tv_usec))
-                return _gf_true;
+    int32_t ret = -1;
+    gf_boolean_t need_sign = _gf_false;
+    br_isignature_out_t *sign = NULL;
 
-        return _gf_false;
-}
+    GF_VALIDATE_OR_GOTO("bit-rot", this, out);
+    GF_VALIDATE_OR_GOTO(this->name, xattr, out);
+    GF_VALIDATE_OR_GOTO(this->name, child, out);
 
-static inline gf_boolean_t
-br_check_object_need_sign (xlator_t *this, dict_t *xattr, br_child_t *child)
-{
-        int32_t              ret       = -1;
-        gf_boolean_t         need_sign = _gf_false;
-        struct timeval       tv        = {0,};
-        br_isignature_out_t *sign      = NULL;
-
-        GF_VALIDATE_OR_GOTO ("bit-rot", this, out);
-        GF_VALIDATE_OR_GOTO (this->name, xattr, out);
-        GF_VALIDATE_OR_GOTO (this->name, child, out);
-
-        ret = dict_get_ptr (xattr, GLUSTERFS_GET_OBJECT_SIGNATURE,
-                            (void **)&sign);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "failed to get object signature info");
-                goto out;
-        }
+    ret = dict_get_ptr(xattr, GLUSTERFS_GET_OBJECT_SIGNATURE, (void **)&sign);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_ERROR, 0, BRB_MSG_GET_SIGN_FAILED,
+                "object-info", NULL);
+        goto out;
+    }
 
-        tv.tv_sec  = ntohl (sign->time[0]);
-        tv.tv_usec = ntohl (sign->time[1]);
-
-        /* Object has been opened and hence dirty. Do not sign it */
-        if (sign->stale && !br_time_equal (child, &tv))
-                need_sign = _gf_true;
+    /* Object has been opened and hence dirty. Do not sign it */
+    if (sign->stale)
+        need_sign = _gf_true;
 
 out:
-        return need_sign;
-}
-
-static inline void
-br_trigger_sign (xlator_t *this, br_child_t *child, inode_t *linked_inode,
-                 loc_t *loc)
-{
-        fd_t      *fd = NULL;
-        int32_t    ret = -1;
-
-        fd = fd_create (linked_inode, 0);
-        if (!fd) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "Failed to create fd [GFID %s]",
-                        uuid_utoa (linked_inode->gfid));
-                goto out;
-        }
-
-        ret = syncop_open (child->xl, loc, O_RDWR, fd, NULL, NULL);
-	if (ret) {
-                br_log_object (this, "open", linked_inode->gfid, -ret);
-		fd_unref (fd);
-		fd = NULL;
-	} else {
-		fd_bind (fd);
-	}
-
-        if (fd)
-                syncop_close (fd);
-
-out:
-        return;
+    return need_sign;
 }
 
 int32_t
-br_prepare_loc (xlator_t *this, br_child_t *child, loc_t *parent,
-                gf_dirent_t *entry, loc_t *loc)
+br_prepare_loc(xlator_t *this, br_child_t *child, loc_t *parent,
+               gf_dirent_t *entry, loc_t *loc)
 {
-        int32_t  ret   = -1;
-        inode_t *inode = NULL;
-
-        inode = inode_grep (child->table, parent->inode, entry->d_name);
-        if (!inode)
-                loc->inode = inode_new (child->table);
-        else {
-                loc->inode = inode;
-                if (loc->inode->ia_type != IA_IFREG) {
-                        gf_log (this->name, GF_LOG_DEBUG, "%s is not a regular "
-                                "file", entry->d_name);
-                        ret = 0;
-                        goto out;
-                }
+    int32_t ret = -1;
+    inode_t *inode = NULL;
+
+    inode = inode_grep(child->table, parent->inode, entry->d_name);
+    if (!inode)
+        loc->inode = inode_new(child->table);
+    else {
+        loc->inode = inode;
+        if (loc->inode->ia_type != IA_IFREG) {
+            gf_msg_debug(this->name, 0,
+                         "%s is not a regular "
+                         "file",
+                         entry->d_name);
+            ret = 0;
+            goto out;
         }
+    }
 
-        loc->parent = inode_ref (parent->inode);
-        gf_uuid_copy (loc->pargfid, parent->inode->gfid);
+    loc->parent = inode_ref(parent->inode);
+    gf_uuid_copy(loc->pargfid, parent->inode->gfid);
 
-        ret = inode_path (parent->inode, entry->d_name, (char **)&loc->path);
-        if (ret < 0 || !loc->path) {
-                gf_log (this->name, GF_LOG_ERROR, "inode_path on %s "
-                        "(parent: %s) failed", entry->d_name,
-                        uuid_utoa (parent->inode->gfid));
-                goto out;
-        }
+    ret = inode_path(parent->inode, entry->d_name, (char **)&loc->path);
+    if (ret < 0 || !loc->path) {
+        gf_smsg(this->name, GF_LOG_ERROR, 0, BRB_MSG_PATH_FAILED,
+                "inode_path=%s", entry->d_name, "parent-gfid=%s",
+                uuid_utoa(parent->inode->gfid), NULL);
+        goto out;
+    }
 
-        loc->name = strrchr (loc->path, '/');
-        if (loc->name)
-                loc->name++;
+    loc->name = strrchr(loc->path, '/');
+    if (loc->name)
+        loc->name++;
 
-        ret = 1;
+    ret = 1;
 
 out:
-        return ret;
+    return ret;
 }
 
 /**
@@ -866,138 +938,174 @@ out:
  * last run for whatever reason (node crashes, reboots, etc..) become
  * candidates for signing. This allows the signature to "catch up" with
  * the current state of the object. Triggering signing is easy: perform
- * an open() followed by a close() therby resulting in call boomerang.
+ * an open() followed by a close() thereby resulting in call boomerang.
  * (though not back to itself :))
  */
 int
-bitd_oneshot_crawl (xlator_t *subvol,
-                    gf_dirent_t *entry, loc_t *parent, void *data)
+bitd_oneshot_crawl(xlator_t *subvol, gf_dirent_t *entry, loc_t *parent,
+                   void *data)
 {
-        int           op_errno     = 0;
-        br_child_t   *child        = NULL;
-        xlator_t     *this         = NULL;
-        loc_t         loc          = {0, };
-        struct iatt   iatt         = {0, };
-        struct iatt   parent_buf   = {0, };
-        dict_t       *xattr        = NULL;
-        int32_t       ret          = -1;
-        inode_t      *linked_inode = NULL;
-        gf_boolean_t  need_signing = _gf_false;
-
-        GF_VALIDATE_OR_GOTO ("bit-rot", subvol, out);
-        GF_VALIDATE_OR_GOTO ("bit-rot", data, out);
-
-        child = data;
-        this = child->this;
-
-        ret = br_prepare_loc (this, child, parent, entry, &loc);
-        if (!ret)
-                goto out;
-
-        ret = syncop_lookup (child->xl, &loc, &iatt, &parent_buf, NULL, NULL);
-        if (ret) {
-                br_log_object_path (this, "lookup", loc.path, -ret);
-                goto out;
-        }
-
-        linked_inode = inode_link (loc.inode, parent->inode, loc.name, &iatt);
-        if (linked_inode)
-                inode_lookup (linked_inode);
-
-        if (iatt.ia_type != IA_IFREG) {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "%s is not a regular file, skipping..", entry->d_name);
-                ret = 0;
-                goto unref_inode;
-        }
+    int op_errno = 0;
+    br_child_t *child = NULL;
+    xlator_t *this = NULL;
+    loc_t loc = {
+        0,
+    };
+    struct iatt iatt = {
+        0,
+    };
+    struct iatt parent_buf = {
+        0,
+    };
+    dict_t *xattr = NULL;
+    int32_t ret = -1;
+    inode_t *linked_inode = NULL;
+    gf_boolean_t need_signing = _gf_false;
+    gf_boolean_t need_reopen = _gf_true;
+
+    GF_VALIDATE_OR_GOTO("bit-rot", subvol, out);
+    GF_VALIDATE_OR_GOTO("bit-rot", data, out);
+
+    child = data;
+    this = child->this;
+
+    ret = br_prepare_loc(this, child, parent, entry, &loc);
+    if (!ret)
+        goto out;
+
+    ret = syncop_lookup(child->xl, &loc, &iatt, &parent_buf, NULL, NULL);
+    if (ret) {
+        br_log_object_path(this, "lookup", loc.path, -ret);
+        goto out;
+    }
+
+    linked_inode = inode_link(loc.inode, parent->inode, loc.name, &iatt);
+    if (linked_inode)
+        inode_lookup(linked_inode);
+
+    if (iatt.ia_type != IA_IFREG) {
+        gf_msg_debug(this->name, 0,
+                     "%s is not a regular file, "
+                     "skipping..",
+                     entry->d_name);
+        ret = 0;
+        goto unref_inode;
+    }
+
+    /**
+     * As of now, 2 cases  are possible and handled.
+     * 1) GlusterFS is upgraded from a previous version which does not
+     *    have any idea about bit-rot and have data in the filesystem.
+     *    In this case syncop_getxattr fails with ENODATA and the object
+     *    is signed. (In real, when crawler sends lookup, bit-rot-stub
+     *    creates the xattrs before returning lookup reply)
+     * 2) Bit-rot was not enabled or BitD was does for some reasons, during
+     *    which some files were created, but since BitD was down, were not
+     *    signed.
+     * If the file was just created and was being written some data when
+     * the down BitD came up, then bit-rot stub should be intelligent to
+     * identify this case (by comparing the ongoing version or by checking
+     * if there are any fds present for that inode) and handle properly.
+     */
+
+    if (bitd_is_bad_file(this, child, &loc, NULL)) {
+        gf_smsg(this->name, GF_LOG_WARNING, 0, BRB_MSG_SKIP_OBJECT, "path=%s",
+                loc.path, NULL);
+        goto unref_inode;
+    }
+
+    ret = syncop_getxattr(child->xl, &loc, &xattr,
+                          GLUSTERFS_GET_OBJECT_SIGNATURE, NULL, NULL);
+    if (ret < 0) {
+        op_errno = -ret;
+        br_log_object(this, "getxattr", linked_inode->gfid, op_errno);
 
         /**
-         * As of now, 2 cases  are possible and handled.
-         * 1) GlusterFS is upgraded from a previous version which does not
-         *    have any idea about bit-rot and have data in the filesystem.
-         *    In this case syncop_getxattr fails with ENODATA and the object
-         *    is signed. (In real, when crawler sends lookup, bit-rot-stub
-         *    creates the xattrs before returning lookup reply)
-         * 2) Bit-rot was not enabled or BitD was dows for some reasons, during
-         *    which some files were created, but since BitD was down, were not
-         *    signed.
-         * If the file was just created and was being written some data when
-         * the down BitD came up, then bit-rot stub should be intelligent to
-         * identify this case (by comparing the ongoing version or by checking
-         * if there are any fds present for that inode) and handle properly.
+         * No need to sign the zero byte objects as the signing
+         * happens upon first modification of the object.
          */
+        if (op_errno == ENODATA && (iatt.ia_size != 0))
+            need_signing = _gf_true;
+        if (op_errno == EINVAL)
+            gf_smsg(this->name, GF_LOG_WARNING, 0,
+                    BRB_MSG_PARTIAL_VERSION_PRESENCE, "gfid=%s",
+                    uuid_utoa(linked_inode->gfid), NULL);
+    } else {
+        need_signing = br_check_object_need_sign(this, xattr, child);
 
-        if (bitd_is_bad_file (this, child, &loc, NULL)) {
-                gf_log (this->name, GF_LOG_WARNING,
-                        "Entry [%s] is marked corrupted.. skipping.", loc.path);
-                goto unref_inode;
-        }
-
-        ret = syncop_getxattr (child->xl, &loc, &xattr,
-                               GLUSTERFS_GET_OBJECT_SIGNATURE, NULL, NULL);
-        if (ret < 0) {
-                op_errno = -ret;
-                br_log_object (this, "getxattr", linked_inode->gfid, op_errno);
-
-                if (op_errno == ENODATA)
-                        need_signing = _gf_true;
-                if (op_errno == EINVAL)
-                        gf_log (this->name, GF_LOG_WARNING, "Partial version "
-                                "xattr presence detected, ignoring [GFID: %s]",
-                                uuid_utoa (linked_inode->gfid));
-        } else {
-                need_signing = br_check_object_need_sign (this, xattr, child);
+        /*
+         * If we are here means, bitrot daemon has started. Is it just
+         * a simple restart of the daemon or is it started because the
+         * feature is enabled is something hard to determine. Hence,
+         * if need_signing is false (because bit-rot version and signature
+         * are present), then still go ahead and sign it.
+         */
+        if (!need_signing) {
+            need_signing = _gf_true;
+            need_reopen = _gf_true;
         }
+    }
 
-        if (!need_signing)
-                goto unref_dict;
+    if (!need_signing)
+        goto unref_dict;
 
-        gf_log (this->name, GF_LOG_INFO,
-                "Triggering signing for %s [GFID: %s | Brick: %s]",
-                loc.path, uuid_utoa (linked_inode->gfid), child->brick_path);
-        br_trigger_sign (this, child, linked_inode, &loc);
+    gf_smsg(this->name, GF_LOG_INFO, 0, BRB_MSG_TRIGGER_SIGN, "path=%s",
+            loc.path, "gfid=%s", uuid_utoa(linked_inode->gfid), "Brick-path=%s",
+            child->brick_path, NULL);
+    br_trigger_sign(this, child, linked_inode, &loc, need_reopen);
 
-        ret = 0;
+    ret = 0;
 
- unref_dict:
-        if (xattr)
-                dict_unref (xattr);
- unref_inode:
-        inode_unref (linked_inode);
- out:
-        loc_wipe (&loc);
+unref_dict:
+    if (xattr)
+        dict_unref(xattr);
+unref_inode:
+    inode_unref(linked_inode);
+out:
+    loc_wipe(&loc);
 
-        return ret;
+    return ret;
 }
 
 #define BR_CRAWL_THROTTLE_COUNT 50
-#define BR_CRAWL_THROTTLE_ZZZ   5
+#define BR_CRAWL_THROTTLE_ZZZ 5
 
 void *
-br_oneshot_signer (void *arg)
+br_oneshot_signer(void *arg)
 {
-        loc_t       loc   = {0,};
-        xlator_t   *this  = NULL;
-        br_child_t *child = NULL;
+    loc_t loc = {
+        0,
+    };
+    xlator_t *this = NULL;
+    br_child_t *child = NULL;
 
-        child = arg;
-        this = child->this;
+    child = arg;
+    this = child->this;
 
-        THIS = this;
+    THIS = this;
 
-        gf_log (this->name, GF_LOG_INFO, "Crawling brick [%s], scanning "
-                "for unsigned objects", child->brick_path);
+    gf_smsg(this->name, GF_LOG_INFO, 0, BRB_MSG_CRAWLING_START, "brick-path=%s",
+            child->brick_path, NULL);
 
-        loc.inode = child->table->root;
-        (void) syncop_ftw_throttle
-                         (child->xl, &loc,
-                         GF_CLIENT_PID_BITD, child, bitd_oneshot_crawl,
-                         BR_CRAWL_THROTTLE_COUNT, BR_CRAWL_THROTTLE_ZZZ);
+    loc.inode = child->table->root;
+    (void)syncop_ftw_throttle(child->xl, &loc, GF_CLIENT_PID_BITD, child,
+                              bitd_oneshot_crawl, BR_CRAWL_THROTTLE_COUNT,
+                              BR_CRAWL_THROTTLE_ZZZ);
 
-        gf_log (this->name, GF_LOG_INFO,
-                "Completed crawling brick [%s]", child->brick_path);
+    gf_smsg(this->name, GF_LOG_INFO, 0, BRB_MSG_CRAWLING_FINISH,
+            "brick-path=%s", child->brick_path, NULL);
 
-        return NULL;
+    return NULL;
+}
+
+static void
+br_set_child_state(br_child_t *child, br_child_state_t state)
+{
+    pthread_mutex_lock(&child->lock);
+    {
+        _br_set_child_state(child, state);
+    }
+    pthread_mutex_unlock(&child->lock);
 }
 
 /**
@@ -1011,55 +1119,158 @@ br_oneshot_signer (void *arg)
  * Also, we register to the changelog library to subscribe for event
  * notifications.
  */
-static inline int32_t
-br_enact_signer (xlator_t *this, br_child_t *child, br_stub_init_t *stub)
+static int32_t
+br_enact_signer(xlator_t *this, br_child_t *child, br_stub_init_t *stub)
 {
-        int32_t ret = 0;
-        struct gf_brick_spec *brick = NULL;
-
-        brick = GF_CALLOC (1, sizeof (struct gf_brick_spec),
-                           gf_common_mt_gf_brick_spec_t);
-        if (!brick)
-                goto error_return;
-
-        br_fill_brick_spec (brick, stub->export);
-        ret = gf_changelog_register_generic
-                         (brick, 1, 1, this->ctx->cmd_args.log_file, -1, this);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "Register to changelog failed"
-                        " [Reason: %s]", strerror (errno));
-                goto dealloc;
-        }
+    int32_t ret = 0;
+    br_private_t *priv = NULL;
+    struct gf_brick_spec *brick = NULL;
+
+    priv = this->private;
+
+    brick = GF_CALLOC(1, sizeof(struct gf_brick_spec),
+                      gf_common_mt_gf_brick_spec_t);
+    if (!brick)
+        goto error_return;
+
+    br_fill_brick_spec(brick, stub->export);
+    ret = gf_changelog_register_generic(brick, 1, 1,
+                                        this->ctx->cmd_args.log_file, -1, this);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, BRB_MSG_REGISTER_FAILED, NULL);
+        goto dealloc;
+    }
+
+    child->threadrunning = 0;
+    ret = gf_thread_create(&child->thread, NULL, br_oneshot_signer, child,
+                           "brosign");
+    if (ret)
+        gf_smsg(this->name, GF_LOG_WARNING, 0, BRB_MSG_SPAWN_FAILED,
+                "FS-crawler-thread", NULL);
+    else
+        child->threadrunning = 1;
+
+    /* it's OK to continue, "old" objects would be signed when modified */
+    list_add_tail(&child->list, &priv->signing);
+    return 0;
+
+dealloc:
+    GF_FREE(brick);
+error_return:
+    return -1;
+}
 
-        child->threadrunning = 0;
-        ret = gf_thread_create (&child->thread, NULL, br_oneshot_signer, child);
-        if (ret)
-                gf_log (this->name, GF_LOG_WARNING,
-                        "failed to spawn FS crawler thread");
-        else
-                child->threadrunning = 1;
+static int32_t
+br_launch_scrubber(xlator_t *this, br_child_t *child, struct br_scanfs *fsscan,
+                   struct br_scrubber *fsscrub)
+{
+    int32_t ret = -1;
+    br_private_t *priv = NULL;
+    struct br_monitor *scrub_monitor = NULL;
+
+    priv = this->private;
+
+    scrub_monitor = &priv->scrub_monitor;
+    ret = gf_thread_create(&child->thread, NULL, br_fsscanner, child,
+                           "brfsscan");
+    if (ret != 0) {
+        gf_smsg(this->name, GF_LOG_ALERT, 0, BRB_MSG_SPAWN_FAILED,
+                "bitrot-scrubber-daemon Brick-path=%s", child->brick_path,
+                NULL);
+        goto error_return;
+    }
+
+    /* Signal monitor to kick off state machine*/
+    pthread_mutex_lock(&scrub_monitor->mutex);
+    {
+        if (!scrub_monitor->inited)
+            pthread_cond_signal(&scrub_monitor->cond);
+        scrub_monitor->inited = _gf_true;
+    }
+    pthread_mutex_unlock(&scrub_monitor->mutex);
+
+    /**
+     * Everything has been setup.. add this subvolume to scrubbers
+     * list.
+     */
+    pthread_mutex_lock(&fsscrub->mutex);
+    {
+        list_add_tail(&child->list, &fsscrub->scrublist);
+        pthread_cond_broadcast(&fsscrub->cond);
+    }
+    pthread_mutex_unlock(&fsscrub->mutex);
+
+    return 0;
+
+error_return:
+    return -1;
+}
 
-        /* it's OK to continue, "old" objects would be signed when modified */
-        return 0;
+static int32_t
+br_enact_scrubber(xlator_t *this, br_child_t *child)
+{
+    int32_t ret = 0;
+    br_private_t *priv = NULL;
+    struct br_scanfs *fsscan = NULL;
+    struct br_scrubber *fsscrub = NULL;
+
+    priv = this->private;
+
+    fsscan = &child->fsscan;
+    fsscrub = &priv->fsscrub;
+
+    /**
+     * if this child already witnesses a successful connection earlier
+     * there's no need to initialize mutexes, condvars, etc..
+     */
+    if (_br_child_witnessed_connection(child))
+        return br_launch_scrubber(this, child, fsscan, fsscrub);
+
+    LOCK_INIT(&fsscan->entrylock);
+    pthread_mutex_init(&fsscan->waitlock, NULL);
+    pthread_cond_init(&fsscan->waitcond, NULL);
+
+    fsscan->entries = 0;
+    INIT_LIST_HEAD(&fsscan->queued);
+    INIT_LIST_HEAD(&fsscan->ready);
 
- dealloc:
-        GF_FREE (brick);
- error_return:
-        return -1;
+    ret = br_launch_scrubber(this, child, fsscan, fsscrub);
+    if (ret)
+        goto error_return;
+
+    return 0;
+
+error_return:
+    LOCK_DESTROY(&fsscan->entrylock);
+    pthread_mutex_destroy(&fsscan->waitlock);
+    pthread_cond_destroy(&fsscan->waitcond);
+
+    return -1;
 }
 
-static inline int32_t
-br_enact_scrubber (xlator_t *this, br_child_t *child)
+static int32_t
+br_child_enaction(xlator_t *this, br_child_t *child, br_stub_init_t *stub)
 {
-        int32_t ret = 0;
+    int32_t ret = -1;
+    br_private_t *priv = this->private;
 
-        ret = gf_thread_create (&child->thread, NULL, br_scrubber, child);
-        if (ret != 0) {
-                ret = -1;
-                gf_log (this->name, GF_LOG_ERROR, "failed to spawn scrubber");
+    pthread_mutex_lock(&child->lock);
+    {
+        if (priv->iamscrubber)
+            ret = br_enact_scrubber(this, child);
+        else
+            ret = br_enact_signer(this, child, stub);
+
+        if (!ret) {
+            child->witnessed = 1;
+            _br_set_child_state(child, BR_CHILD_STATE_CONNECTED);
+            gf_smsg(this->name, GF_LOG_INFO, 0, BRB_MSG_CONNECTED_TO_BRICK,
+                    "brick-path=%s", child->brick_path, NULL);
         }
+    }
+    pthread_mutex_unlock(&child->lock);
 
-        return ret;
+    return ret;
 }
 
 /**
@@ -1069,70 +1280,171 @@ br_enact_scrubber (xlator_t *this, br_child_t *child)
  * by getxattr() on a virtual key. Depending on the configuration, the
  * process either acts as a signer or a scrubber.
  */
-static inline int32_t
-br_brick_connect (xlator_t *this, br_child_t *child)
+int32_t
+br_brick_connect(xlator_t *this, br_child_t *child)
 {
-        int32_t         ret      = -1;
-        loc_t           loc      = {0, };
-        struct iatt     buf      = {0, };
-        struct iatt     parent   = {0, };
-        br_stub_init_t *stub     = NULL;
-        dict_t         *xattr    = NULL;
-        br_private_t   *priv     = NULL;
-        int             op_errno = 0;
-
-        GF_VALIDATE_OR_GOTO ("bit-rot", this, out);
-        GF_VALIDATE_OR_GOTO (this->name, child, out);
-        GF_VALIDATE_OR_GOTO (this->name, this->private, out);
-
-        priv = this->private;
-
-        loc.inode = inode_ref (child->table->root);
-        gf_uuid_copy (loc.gfid, loc.inode->gfid);
-        loc.path = gf_strdup ("/");
-
-        ret = syncop_lookup (child->xl, &loc, &buf, &parent, NULL, NULL);
-        if (ret) {
-                op_errno = -ret;
-                ret = -1;
-                gf_log (this->name, GF_LOG_ERROR, "lookup on root failed "
-                        "[Reason: %s]", strerror (op_errno));
-                goto wipeloc;
-        }
+    int32_t ret = -1;
+    loc_t loc = {
+        0,
+    };
+    struct iatt buf = {
+        0,
+    };
+    struct iatt parent = {
+        0,
+    };
+    br_stub_init_t *stub = NULL;
+    dict_t *xattr = NULL;
+    int op_errno = 0;
+
+    GF_VALIDATE_OR_GOTO("bit-rot", this, out);
+    GF_VALIDATE_OR_GOTO(this->name, child, out);
+    GF_VALIDATE_OR_GOTO(this->name, this->private, out);
+
+    br_child_set_scrub_state(child, _gf_false);
+    br_set_child_state(child, BR_CHILD_STATE_INITIALIZING);
+
+    loc.inode = inode_ref(child->table->root);
+    gf_uuid_copy(loc.gfid, loc.inode->gfid);
+    loc.path = gf_strdup("/");
+
+    ret = syncop_lookup(child->xl, &loc, &buf, &parent, NULL, NULL);
+    if (ret) {
+        op_errno = -ret;
+        ret = -1;
+        gf_smsg(this->name, GF_LOG_ERROR, op_errno, BRB_MSG_LOOKUP_FAILED,
+                NULL);
+        goto wipeloc;
+    }
+
+    ret = syncop_getxattr(child->xl, &loc, &xattr,
+                          GLUSTERFS_GET_BR_STUB_INIT_TIME, NULL, NULL);
+    if (ret) {
+        op_errno = -ret;
+        ret = -1;
+        gf_smsg(this->name, GF_LOG_ERROR, op_errno, BRB_MSG_GET_INFO_FAILED,
+                NULL);
+        goto wipeloc;
+    }
+
+    ret = dict_get_ptr(xattr, GLUSTERFS_GET_BR_STUB_INIT_TIME, (void **)&stub);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_ERROR, 0, BRB_MSG_GET_INFO_FAILED, NULL);
+        goto free_dict;
+    }
+
+    memcpy(child->brick_path, stub->export, strlen(stub->export) + 1);
+    child->tv.tv_sec = ntohl(stub->timebuf[0]);
+    child->tv.tv_usec = ntohl(stub->timebuf[1]);
+
+    ret = br_child_enaction(this, child, stub);
+
+free_dict:
+    dict_unref(xattr);
+wipeloc:
+    loc_wipe(&loc);
+out:
+    if (ret)
+        br_set_child_state(child, BR_CHILD_STATE_CONNFAILED);
+    return ret;
+}
 
-        ret = syncop_getxattr (child->xl, &loc, &xattr,
-                               GLUSTERFS_GET_BR_STUB_INIT_TIME, NULL, NULL);
-        if (ret) {
-                op_errno = -ret;
-                ret = -1;
-                gf_log (this->name, GF_LOG_ERROR, "failed to get stub info "
-                        "[Reason: %s]", strerror (op_errno));
-                goto wipeloc;
-        }
+/* TODO: cleanup signer */
+static int32_t
+br_cleanup_signer(xlator_t *this, br_child_t *child)
+{
+    return 0;
+}
 
-        ret = dict_get_ptr (xattr, GLUSTERFS_GET_BR_STUB_INIT_TIME,
-                            (void **)&stub);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "failed to extract stub information");
-                goto free_dict;
-        }
+static int32_t
+br_cleanup_scrubber(xlator_t *this, br_child_t *child)
+{
+    int32_t ret = 0;
+    br_private_t *priv = NULL;
+    struct br_scrubber *fsscrub = NULL;
+    struct br_monitor *scrub_monitor = NULL;
+
+    priv = this->private;
+    fsscrub = &priv->fsscrub;
+    scrub_monitor = &priv->scrub_monitor;
+
+    if (_br_is_child_scrub_active(child)) {
+        scrub_monitor->active_child_count--;
+        br_child_set_scrub_state(child, _gf_false);
+    }
+
+    /**
+     * 0x0: child (brick) goes out of rotation
+     *
+     * This is fully safe w.r.t. entries for this child being actively
+     * scrubbed. Each of the scrubber thread(s) would finish scrubbing
+     * the entry (probably failing due to disconnection) and either
+     * putting the entry back into the queue or continuing further.
+     * Either way, pending entries for this child's queue need not be
+     * drained; entries just sit there in the queued/ready list to be
+     * consumed later upon re-connection.
+     */
+    pthread_mutex_lock(&fsscrub->mutex);
+    {
+        list_del_init(&child->list);
+    }
+    pthread_mutex_unlock(&fsscrub->mutex);
+
+    /**
+     * 0x1: cleanup scanner thread
+     *
+     * The pending timer needs to be removed _after_ cleaning up the
+     * filesystem scanner (scheduling the next scrub time is not a
+     * cancellation point).
+     */
+    ret = gf_thread_cleanup_xint(child->thread);
+    if (ret)
+        gf_smsg(this->name, GF_LOG_INFO, 0, BRB_MSG_SCRUB_THREAD_CLEANUP, NULL);
+
+    gf_smsg(this->name, GF_LOG_INFO, 0, BRB_MSG_SCRUBBER_CLEANED,
+            "brick-path=%s", child->brick_path, NULL);
+
+    return 0;
+}
 
-        memcpy (child->brick_path, stub->export, strlen (stub->export) + 1);
-        child->tv.tv_sec = ntohl (stub->timebuf[0]);
-        child->tv.tv_usec = ntohl (stub->timebuf[0]);
+/**
+ * OK.. this child has made it's mind to go down the drain. So,
+ * let's clean up what it touched. (NOTE: there's no need to clean
+ * the inode table, it's just reused taking care of stale inodes)
+ */
+int32_t
+br_brick_disconnect(xlator_t *this, br_child_t *child)
+{
+    int32_t ret = 0;
+    struct br_monitor *scrub_monitor = NULL;
+    br_private_t *priv = this->private;
+
+    scrub_monitor = &priv->scrub_monitor;
+
+    /* Lock order should be wakelock and then child lock to
+     * dead locks.
+     */
+    pthread_mutex_lock(&scrub_monitor->wakelock);
+    {
+        pthread_mutex_lock(&child->lock);
+        {
+            if (!_br_is_child_connected(child))
+                goto unblock;
 
-        if (priv->iamscrubber)
-                ret = br_enact_scrubber (this, child);
-        else
-                ret = br_enact_signer (this, child, stub);
+            /* child is on death row.. */
+            _br_set_child_state(child, BR_CHILD_STATE_DISCONNECTED);
 
- free_dict:
-        dict_unref (xattr);
- wipeloc:
-        loc_wipe (&loc);
- out:
-        return ret;
+            if (priv->iamscrubber)
+                ret = br_cleanup_scrubber(this, child);
+            else
+                ret = br_cleanup_signer(this, child);
+        }
+    unblock:
+        pthread_mutex_unlock(&child->lock);
+    }
+    pthread_mutex_unlock(&scrub_monitor->wakelock);
+
+    return ret;
 }
 
 /**
@@ -1141,346 +1453,701 @@ br_brick_connect (xlator_t *this, br_child_t *child)
  * information regarding that brick (such as brick path).
  */
 void *
-br_handle_events (void *arg)
+br_handle_events(void *arg)
 {
-        xlator_t     *this  = NULL;
-        br_private_t *priv  = NULL;
-        br_child_t   *child = NULL;
-        int32_t       ret   = -1;
-
-        this = arg;
-        priv = this->private;
+    int32_t ret = 0;
+    xlator_t *this = NULL;
+    br_private_t *priv = NULL;
+    br_child_t *child = NULL;
+    struct br_child_event *childev = NULL;
+
+    this = arg;
+    priv = this->private;
+
+    /*
+     * Since, this is the topmost xlator, THIS has to be set by bit-rot
+     * xlator itself (STACK_WIND won't help in this case). Also it has
+     * to be done for each thread that gets spawned. Otherwise, a new
+     * thread will get global_xlator's pointer when it does "THIS".
+     */
+    THIS = this;
+
+    while (1) {
+        pthread_mutex_lock(&priv->lock);
+        {
+            while (list_empty(&priv->bricks))
+                pthread_cond_wait(&priv->cond, &priv->lock);
 
-        /*
-         * Since, this is the topmost xlator, THIS has to be set by bit-rot
-         * xlator itself (STACK_WIND wont help in this case). Also it has
-         * to be done for each thread that gets spawned. Otherwise, a new
-         * thread will get global_xlator's pointer when it does "THIS".
-         */
-        THIS = this;
-
-        while (1) {
-                pthread_mutex_lock (&priv->lock);
-                {
-                        while (list_empty (&priv->bricks)) {
-                                pthread_cond_wait (&priv->cond,
-                                                   &priv->lock);
-                        }
-
-                        child = list_entry (priv->bricks.next, br_child_t,
-                                            list);
-                        if (child && child->child_up) {
-                                ret = br_brick_connect (this, child);
-                                if (ret == -1)
-                                        gf_log (this->name, GF_LOG_ERROR,
-                                                "failed to connect to the "
-                                                "child (subvolume: %s)",
-                                                child->xl->name);
-                                else
-                                        list_del_init (&child->list);
-                        }
-
-                }
-                pthread_mutex_unlock (&priv->lock);
+            childev = list_first_entry(&priv->bricks, struct br_child_event,
+                                       list);
+            list_del_init(&childev->list);
         }
+        pthread_mutex_unlock(&priv->lock);
 
-        return NULL;
+        child = childev->child;
+        ret = childev->call(this, child);
+        if (ret)
+            gf_smsg(this->name, GF_LOG_ERROR, 0, BRB_MSG_SUBVOL_CONNECT_FAILED,
+                    "name=%s", child->xl->name, NULL);
+        GF_FREE(childev);
+    }
+
+    return NULL;
 }
 
 int32_t
-mem_acct_init (xlator_t *this)
+mem_acct_init(xlator_t *this)
 {
-        int32_t     ret = -1;
-
-        if (!this)
-                return ret;
+    int32_t ret = -1;
 
-        ret = xlator_mem_acct_init (this, gf_br_stub_mt_end + 1);
+    if (!this)
+        return ret;
 
-        if (ret != 0) {
-                gf_log (this->name, GF_LOG_WARNING, "Memory accounting"
-                        " init failed");
-                return ret;
-        }
+    ret = xlator_mem_acct_init(this, gf_br_stub_mt_end + 1);
 
+    if (ret != 0) {
+        gf_smsg(this->name, GF_LOG_WARNING, 0, BRB_MSG_MEM_ACNT_FAILED, NULL);
         return ret;
+    }
+
+    return ret;
+}
+
+static void
+_br_qchild_event(xlator_t *this, br_child_t *child, br_child_handler *call)
+{
+    br_private_t *priv = NULL;
+    struct br_child_event *childev = NULL;
+
+    priv = this->private;
+
+    childev = GF_CALLOC(1, sizeof(*childev), gf_br_mt_br_child_event_t);
+    if (!childev) {
+        gf_smsg(this->name, GF_LOG_ERROR, ENOMEM, BRB_MSG_EVENT_UNHANDLED,
+                "Brick-name=%s", child->xl->name, NULL);
+        return;
+    }
+
+    INIT_LIST_HEAD(&childev->list);
+    childev->this = this;
+    childev->child = child;
+    childev->call = call;
+
+    list_add_tail(&childev->list, &priv->bricks);
 }
 
 int
-notify (xlator_t *this, int32_t event, void *data, ...)
+br_scrubber_status_get(xlator_t *this, dict_t **dict)
 {
-        xlator_t                *subvol = NULL;
-        br_private_t            *priv   = NULL;
-        int                      idx    = -1;
-        br_child_t              *child  = NULL;
+    int ret = -1;
+    br_private_t *priv = NULL;
+    struct br_scrub_stats *scrub_stats = NULL;
+
+    priv = this->private;
+
+    GF_VALIDATE_OR_GOTO("bit-rot", priv, out);
+
+    scrub_stats = &priv->scrub_stat;
+
+    ret = br_get_bad_objects_list(this, dict);
+    if (ret) {
+        gf_msg_debug(this->name, 0,
+                     "Failed to collect corrupt "
+                     "files");
+    }
+
+    ret = dict_set_int8(*dict, "scrub-running", scrub_stats->scrub_running);
+    if (ret) {
+        gf_msg_debug(this->name, 0,
+                     "Failed setting scrub_running "
+                     "entry to the dictionary");
+    }
+
+    ret = dict_set_uint64(*dict, "scrubbed-files", scrub_stats->scrubbed_files);
+    if (ret) {
+        gf_msg_debug(this->name, 0,
+                     "Failed to setting scrubbed file "
+                     "entry to the dictionary");
+    }
+
+    ret = dict_set_uint64(*dict, "unsigned-files", scrub_stats->unsigned_files);
+    if (ret) {
+        gf_msg_debug(this->name, 0,
+                     "Failed to set unsigned file count"
+                     " entry to the dictionary");
+    }
+
+    ret = dict_set_uint64(*dict, "scrub-duration", scrub_stats->scrub_duration);
+    if (ret) {
+        gf_msg_debug(this->name, 0,
+                     "Failed to set scrub duration"
+                     " entry to the dictionary");
+    }
+
+    ret = dict_set_dynstr_with_alloc(*dict, "last-scrub-time",
+                                     scrub_stats->last_scrub_time);
+    if (ret) {
+        gf_msg_debug(this->name, 0,
+                     "Failed to set "
+                     "last scrub time value");
+    }
 
-        subvol = (xlator_t *)data;
-        priv = this->private;
+out:
+    return ret;
+}
 
-        gf_log (this->name, GF_LOG_TRACE, "Notification received: %d",
-                event);
+int
+notify(xlator_t *this, int32_t event, void *data, ...)
+{
+    int idx = -1;
+    int ret = -1;
+    xlator_t *subvol = NULL;
+    br_child_t *child = NULL;
+    br_private_t *priv = NULL;
+    dict_t *output = NULL;
+    va_list ap;
+    struct br_monitor *scrub_monitor = NULL;
+
+    subvol = (xlator_t *)data;
+    priv = this->private;
+    scrub_monitor = &priv->scrub_monitor;
 
-        switch (event) {
+    gf_msg_trace(this->name, 0, "Notification received: %d", event);
+
+    idx = br_find_child_index(this, subvol);
+
+    switch (event) {
         case GF_EVENT_CHILD_UP:
-                /* should this be done under lock? or is it ok to do it
-                   without lock? */
-                idx = br_find_child_index (this, subvol);
-
-                pthread_mutex_lock (&priv->lock);
-                {
-                        if (idx < 0) {
-                                gf_log (this->name, GF_LOG_ERROR, "got child "
-                                        "up from invalid subvolume");
-                        } else {
-                                child = &priv->children[idx];
-                                if (child->child_up != 1)
-                                        child->child_up = 1;
-                                if (!child->xl)
-                                        child->xl = subvol;
-                                if (!child->table)
-                                        child->table = inode_table_new (4096,
-                                                                       subvol);
-                                priv->up_children++;
-                                list_add_tail (&child->list, &priv->bricks);
-                                pthread_cond_signal (&priv->cond);
-                        }
-                }
-                pthread_mutex_unlock (&priv->lock);
-                break;
+            if (idx < 0) {
+                gf_smsg(this->name, GF_LOG_ERROR, 0, BRB_MSG_INVALID_SUBVOL,
+                        "event=%d", event, NULL);
+                goto out;
+            }
+
+            pthread_mutex_lock(&priv->lock);
+            {
+                child = &priv->children[idx];
+                if (child->child_up == 1)
+                    goto unblock_0;
+                priv->up_children++;
+
+                child->child_up = 1;
+                child->xl = subvol;
+                if (!child->table)
+                    child->table = inode_table_new(4096, subvol);
+
+                _br_qchild_event(this, child, br_brick_connect);
+                pthread_cond_signal(&priv->cond);
+            }
+        unblock_0:
+            pthread_mutex_unlock(&priv->lock);
+
+            if (priv->up_children == priv->child_count)
+                default_notify(this, event, data);
+            break;
 
-        case GF_EVENT_CHILD_MODIFIED:
-                idx = br_find_child_index (this, subvol);
-                if (idx < 0) {
-                        gf_log (this->name, GF_LOG_ERROR, "received child up "
-                                "from invalid subvolume");
-                        goto out;
-                }
-                priv = this->private;
-                /* ++(priv->generation); */
-                break;
         case GF_EVENT_CHILD_DOWN:
-                idx = br_find_child_index (this, subvol);
-                if (idx < 0) {
-                        gf_log (this->name, GF_LOG_ERROR, "received child down "
-                                "from invalid subvolume");
-                        goto out;
-                }
-
-                pthread_mutex_lock (&priv->lock);
-                {
-                        if (priv->children[idx].child_up == 1) {
-                                priv->children[idx].child_up = 0;
-                                priv->up_children--;
-                        }
-                }
-                pthread_mutex_unlock (&priv->lock);
-                break;
-        case GF_EVENT_PARENT_UP:
-                default_notify (this, GF_EVENT_PARENT_UP, data);
-                break;
-        }
+            if (idx < 0) {
+                gf_smsg(this->name, GF_LOG_ERROR, 0, BRB_MSG_INVALID_SUBVOL,
+                        "event=%d", event, NULL);
+                goto out;
+            }
+
+            pthread_mutex_lock(&priv->lock);
+            {
+                child = &priv->children[idx];
+                if (child->child_up == 0)
+                    goto unblock_1;
+
+                child->child_up = 0;
+                priv->up_children--;
+
+                _br_qchild_event(this, child, br_brick_disconnect);
+                pthread_cond_signal(&priv->cond);
+            }
+        unblock_1:
+            pthread_mutex_unlock(&priv->lock);
+
+            if (priv->up_children == 0)
+                default_notify(this, event, data);
+            break;
+
+        case GF_EVENT_SCRUB_STATUS:
+            gf_msg_debug(this->name, GF_LOG_INFO,
+                         "BitRot scrub status "
+                         "called");
+            va_start(ap, data);
+            output = va_arg(ap, dict_t *);
+            va_end(ap);
+
+            ret = br_scrubber_status_get(this, &output);
+            gf_msg_debug(this->name, 0, "returning %d", ret);
+            break;
+
+        case GF_EVENT_SCRUB_ONDEMAND:
+            gf_log(this->name, GF_LOG_INFO,
+                   "BitRot scrub ondemand "
+                   "called");
+
+            if (scrub_monitor->state != BR_SCRUB_STATE_PENDING) {
+                gf_smsg(this->name, GF_LOG_ERROR, 0,
+                        BRB_MSG_RESCHEDULE_SCRUBBER_FAILED, "Current-state=%d",
+                        scrub_monitor->state, NULL);
+                return -2;
+            }
+
+            /* Needs synchronization with reconfigure thread */
+            pthread_mutex_lock(&priv->lock);
+            {
+                ret = br_scrub_state_machine(this, _gf_true);
+            }
+            pthread_mutex_unlock(&priv->lock);
+
+            if (ret) {
+                gf_smsg(this->name, GF_LOG_ERROR, 0,
+                        BRB_MSG_COULD_NOT_SCHEDULE_SCRUB, NULL);
+            }
+            gf_msg_debug(this->name, 0, "returning %d", ret);
+            break;
+        default:
+            default_notify(this, event, data);
+    }
 
 out:
-        return 0;
+    return 0;
+}
+
+static void
+br_fini_signer(xlator_t *this, br_private_t *priv)
+{
+    int i = 0;
+
+    if (priv == NULL)
+        return;
+
+    for (; i < priv->signer_th_count; i++) {
+        (void)gf_thread_cleanup_xint(priv->obj_queue->workers[i]);
+    }
+    GF_FREE(priv->obj_queue->workers);
+
+    pthread_cond_destroy(&priv->object_cond);
 }
 
 /**
  * Initialize signer specific structures, spawn worker threads.
  */
 
-static inline void
-br_fini_signer (xlator_t *this, br_private_t *priv)
+static int32_t
+br_init_signer(xlator_t *this, br_private_t *priv)
 {
-        int i = 0;
-        int ret = 0;
-
-        for (; i < BR_WORKERS; i++) {
-                (void) gf_thread_cleanup_xint (priv->obj_queue->workers[i]);
+    int i = 0;
+    int32_t ret = -1;
+
+    /* initialize gfchangelog xlator context */
+    ret = gf_changelog_init(this);
+    if (ret)
+        goto out;
+
+    pthread_cond_init(&priv->object_cond, NULL);
+
+    priv->obj_queue = GF_CALLOC(1, sizeof(*priv->obj_queue),
+                                gf_br_mt_br_ob_n_wk_t);
+    if (!priv->obj_queue)
+        goto cleanup_cond;
+    INIT_LIST_HEAD(&priv->obj_queue->objects);
+
+    priv->obj_queue->workers = GF_CALLOC(
+        priv->signer_th_count, sizeof(pthread_t), gf_br_mt_br_worker_t);
+    if (!priv->obj_queue->workers)
+        goto cleanup_obj_queue;
+
+    for (i = 0; i < priv->signer_th_count; i++) {
+        ret = gf_thread_create(&priv->obj_queue->workers[i], NULL,
+                               br_process_object, this, "brpobj");
+        if (ret != 0) {
+            gf_smsg(this->name, GF_LOG_ERROR, -ret,
+                    BRB_MSG_THREAD_CREATION_FAILED, NULL);
+            ret = -1;
+            goto cleanup_threads;
         }
+    }
 
-        pthread_cond_destroy (&priv->object_cond);
-        ret = gf_tw_cleanup_timers (priv->timer_wheel);
-        if (ret == 0) {
-                priv->timer_wheel = NULL;
-        }
+    return 0;
+
+cleanup_threads:
+    for (i--; i >= 0; i--) {
+        (void)gf_thread_cleanup_xint(priv->obj_queue->workers[i]);
+    }
+    GF_FREE(priv->obj_queue->workers);
+
+cleanup_obj_queue:
+    GF_FREE(priv->obj_queue);
+
+cleanup_cond:
+    /* that's explicit */
+    pthread_cond_destroy(&priv->object_cond);
+out:
+    return -1;
 }
 
-static inline int32_t
-br_init_signer (xlator_t *this, br_private_t *priv)
+/**
+ * For signer, only rate limit CPU usage (during hash calculation) when
+ * compiled with -DBR_RATE_LIMIT_SIGNER cflags, else let it run full
+ * throttle.
+ */
+static int32_t
+br_rate_limit_signer(xlator_t *this, int child_count, int numbricks)
 {
-        int rc = 0;
-        int i = 0;
-        int32_t ret = -1;
+    br_private_t *priv = NULL;
+    tbf_opspec_t spec = {
+        0,
+    };
+
+    priv = this->private;
+
+    spec.op = TBF_OP_HASH;
+    spec.rate = 0;
+    spec.maxlimit = 0;
+
+    /**
+     * OK. Most implementations of TBF I've come across generate tokens
+     * every second (UML, etc..) and some chose sub-second granularity
+     * (blk-iothrottle cgroups). TBF algorithm itself does not enforce
+     * any logic for choosing generation interval and it seems pretty
+     * logical as one could jack up token count per interval w.r.t.
+     * generation rate.
+     *
+     * Value used here is chosen based on a series of test(s) performed
+     * to balance object signing time and not maxing out on all available
+     * CPU cores. It's obvious to have seconds granularity and jack up
+     * token count per interval, thereby achieving close to similar
+     * results. Let's stick to this as it seems to be working fine for
+     * the set of ops that are throttled.
+     **/
+    spec.token_gen_interval = 600000; /* In usec */
+
+#ifdef BR_RATE_LIMIT_SIGNER
+
+    double contribution = 0;
+    contribution = ((double)1 - ((double)child_count / (double)numbricks));
+    if (contribution == 0)
+        contribution = 1;
+    spec.rate = BR_HASH_CALC_READ_SIZE * contribution;
+    spec.maxlimit = priv->signer_th_count * BR_HASH_CALC_READ_SIZE;
 
-        /* initialize gfchangelog xlator context */
-        ret = gf_changelog_init (this);
-        if (ret)
-                goto out;
+#endif
 
-        priv->timer_wheel = gf_tw_init_timers ();
-        if (!priv->timer_wheel) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "failed to initialize the timer wheel");
-                goto out;
-        }
+    if (!spec.rate)
+        gf_smsg(this->name, GF_LOG_INFO, 0, BRB_MSG_RATE_LIMIT_INFO,
+                "FULL THROTTLE", NULL);
+    else
+        gf_smsg(this->name, GF_LOG_INFO, 0, BRB_MSG_RATE_LIMIT_INFO,
+                "tokens/sec-rate=%lu", spec.rate, "maxlimit=%lu", spec.maxlimit,
+                NULL);
 
-        pthread_cond_init (&priv->object_cond, NULL);
-
-        priv->obj_queue = GF_CALLOC (1, sizeof (*priv->obj_queue),
-                                     gf_br_mt_br_ob_n_wk_t);
-        if (!priv->obj_queue)
-                goto cleanup_timer;
-        INIT_LIST_HEAD (&priv->obj_queue->objects);
-
-        for (i = 0; i < BR_WORKERS; i++) {
-                ret = gf_thread_create (&priv->obj_queue->workers[i], NULL,
-                                        br_process_object, this);
-                if (ret != 0) {
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "thread creation failed (%s)", strerror (-ret));
-                        ret = -1;
-                        goto cleanup_threads;
-                }
-        }
+    priv->tbf = tbf_init(&spec, 1);
+    return priv->tbf ? 0 : -1;
+}
 
-        return 0;
+static int32_t
+br_signer_handle_options(xlator_t *this, br_private_t *priv, dict_t *options)
+{
+    if (options) {
+        GF_OPTION_RECONF("expiry-time", priv->expiry_time, options, uint32,
+                         error_return);
+        GF_OPTION_RECONF("signer-threads", priv->signer_th_count, options,
+                         uint32, error_return);
+    } else {
+        GF_OPTION_INIT("expiry-time", priv->expiry_time, uint32, error_return);
+        GF_OPTION_INIT("signer-threads", priv->signer_th_count, uint32,
+                       error_return);
+    }
+
+    return 0;
+
+error_return:
+    return -1;
+}
 
- cleanup_threads:
-        for (i--; i >= 0; i--) {
-                (void) gf_thread_cleanup_xint (priv->obj_queue->workers[i]);
-        }
+static int32_t
+br_signer_init(xlator_t *this, br_private_t *priv)
+{
+    int32_t ret = 0;
+    int numbricks = 0;
+
+    GF_OPTION_INIT("expiry-time", priv->expiry_time, uint32, error_return);
+    GF_OPTION_INIT("brick-count", numbricks, int32, error_return);
+    GF_OPTION_INIT("signer-threads", priv->signer_th_count, uint32,
+                   error_return);
+
+    ret = br_rate_limit_signer(this, priv->child_count, numbricks);
+    if (ret)
+        goto error_return;
+
+    ret = br_init_signer(this, priv);
+    if (ret)
+        goto cleanup_tbf;
 
-        GF_FREE (priv->obj_queue);
+    return 0;
 
- cleanup_timer:
-        /* that's explicit */
-        pthread_cond_destroy (&priv->object_cond);
-        rc = gf_tw_cleanup_timers (priv->timer_wheel);
-        if (rc == 0) {
-                priv->timer_wheel = NULL;
+cleanup_tbf:
+    /* cleanup TBF */
+error_return:
+    return -1;
+}
+
+static void
+br_free_scrubber_monitor(xlator_t *this, br_private_t *priv)
+{
+    struct br_monitor *scrub_monitor = &priv->scrub_monitor;
+
+    if (scrub_monitor->timer) {
+        (void)gf_tw_del_timer(priv->timer_wheel, scrub_monitor->timer);
+
+        GF_FREE(scrub_monitor->timer);
+        scrub_monitor->timer = NULL;
+    }
+
+    (void)gf_thread_cleanup_xint(scrub_monitor->thread);
+
+    /* Clean up cond and mutex variables */
+    pthread_mutex_destroy(&scrub_monitor->mutex);
+    pthread_cond_destroy(&scrub_monitor->cond);
+
+    pthread_mutex_destroy(&scrub_monitor->wakelock);
+    pthread_cond_destroy(&scrub_monitor->wakecond);
+
+    pthread_mutex_destroy(&scrub_monitor->donelock);
+    pthread_cond_destroy(&scrub_monitor->donecond);
+
+    LOCK_DESTROY(&scrub_monitor->lock);
+}
+
+static void
+br_free_children(xlator_t *this, br_private_t *priv, int count)
+{
+    br_child_t *child = NULL;
+
+    for (--count; count >= 0; count--) {
+        child = &priv->children[count];
+        mem_pool_destroy(child->timer_pool);
+        pthread_mutex_destroy(&child->lock);
+    }
+
+    GF_FREE(priv->children);
+    priv->children = NULL;
+}
+
+static int
+br_init_children(xlator_t *this, br_private_t *priv)
+{
+    int i = 0;
+    br_child_t *child = NULL;
+    xlator_list_t *trav = NULL;
+
+    priv->child_count = xlator_subvolume_count(this);
+    priv->children = GF_CALLOC(priv->child_count, sizeof(*priv->children),
+                               gf_br_mt_br_child_t);
+    if (!priv->children)
+        goto err;
+
+    trav = this->children;
+    while (trav) {
+        child = &priv->children[i];
+
+        pthread_mutex_init(&child->lock, NULL);
+        child->witnessed = 0;
+
+        br_set_child_state(child, BR_CHILD_STATE_DISCONNECTED);
+
+        child->this = this;
+        child->xl = trav->xlator;
+
+        child->timer_pool = mem_pool_new(struct gf_tw_timer_list, 4096);
+        if (!child->timer_pool) {
+            gf_smsg(this->name, GF_LOG_ERROR, ENOMEM, BRB_MSG_MEM_POOL_ALLOC,
+                    NULL);
+            errno = ENOMEM;
+            goto freechild;
         }
 
- out:
-        return -1;
+        INIT_LIST_HEAD(&child->list);
+
+        i++;
+        trav = trav->next;
+    }
+
+    return 0;
+
+freechild:
+    br_free_children(this, priv, i);
+err:
+    return -1;
 }
 
 int32_t
-init (xlator_t *this)
+init(xlator_t *this)
 {
-        int            i    = 0;
-        int32_t        ret  = -1;
-	br_private_t  *priv = NULL;
-        xlator_list_t *trav = NULL;
-
-	if (!this->children) {
-		gf_log (this->name, GF_LOG_ERROR, "FATAL: no children");
-		goto out;
-	}
-
-        priv = GF_CALLOC (1, sizeof (*priv), gf_br_mt_br_private_t);
-        if (!priv) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "failed to allocate memory (->priv)");
-                goto out;
-        }
+    int32_t ret = -1;
+    br_private_t *priv = NULL;
 
-        GF_OPTION_INIT ("scrubber", priv->iamscrubber, bool, out);
-        GF_OPTION_INIT ("expiry-time", priv->expiry_time, int32, out);
-
-        priv->child_count = xlator_subvolume_count (this);
-        priv->children = GF_CALLOC (priv->child_count, sizeof (*priv->children),
-                                    gf_br_mt_br_child_t);
-        if (!priv->children)
-                goto free_priv;
-
-        trav = this->children;
-        while (trav) {
-                priv->children[i].this = this;
-                priv->children[i].xl = trav->xlator;
-
-                priv->children[i].timer_pool =
-                                  mem_pool_new (struct gf_tw_timer_list,  4096);
-                if (!priv->children[i].timer_pool) {
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "failed to allocate mem-pool for timer");
-                        errno = ENOMEM;
-                        goto free_children;
-                }
-
-                i++;
-                trav = trav->next;
-        }
+    if (!this->children) {
+        gf_smsg(this->name, GF_LOG_ERROR, 0, BRB_MSG_NO_CHILD, NULL);
+        goto out;
+    }
 
-        pthread_mutex_init (&priv->lock, NULL);
-        pthread_cond_init (&priv->cond, NULL);
+    priv = GF_CALLOC(1, sizeof(*priv), gf_br_mt_br_private_t);
+    if (!priv) {
+        gf_smsg(this->name, GF_LOG_ERROR, ENOMEM, BRB_MSG_NO_MEMORY, NULL);
+        goto out;
+    }
 
-        for (i = 0; i < priv->child_count; i++)
-                INIT_LIST_HEAD (&priv->children[i].list);
-        INIT_LIST_HEAD (&priv->bricks);
+    GF_OPTION_INIT("scrubber", priv->iamscrubber, bool, free_priv);
 
-	this->private = priv;
+    ret = br_init_children(this, priv);
+    if (ret)
+        goto free_priv;
 
-        if (!priv->iamscrubber) {
-                ret = br_init_signer (this, priv);
-                if (ret)
-                        goto cleanup_mutex;
-        }
+    pthread_mutex_init(&priv->lock, NULL);
+    pthread_cond_init(&priv->cond, NULL);
 
-        ret = gf_thread_create (&priv->thread, NULL, br_handle_events, this);
-        if (ret != 0) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "thread creation failed (%s)", strerror (-ret));
-                ret = -1;
-        }
+    INIT_LIST_HEAD(&priv->bricks);
+    INIT_LIST_HEAD(&priv->signing);
 
-        if (!ret) {
-                gf_log (this->name, GF_LOG_INFO,
-                        "bit-rot xlator loaded in \"%s\" mode",
-                        (priv->iamscrubber) ? "SCRUBBER" : "SIGNER");
-                return 0;
-        }
+    priv->timer_wheel = glusterfs_ctx_tw_get(this->ctx);
+    if (!priv->timer_wheel) {
+        gf_smsg(this->name, GF_LOG_ERROR, 0, BRB_MSG_TIMER_WHEEL_UNAVAILABLE,
+                NULL);
+        goto cleanup;
+    }
 
- cleanup_mutex:
-        (void) pthread_cond_destroy (&priv->cond);
-        (void) pthread_mutex_destroy (&priv->lock);
- free_children:
-        for (i = 0; i < priv->child_count; i++) {
-                if (priv->children[i].timer_pool)
-                        mem_pool_destroy (priv->children[i].timer_pool);
-        }
+    this->private = priv;
+
+    if (!priv->iamscrubber) {
+        ret = br_signer_init(this, priv);
+        if (!ret)
+            ret = br_signer_handle_options(this, priv, NULL);
+    } else {
+        ret = br_scrubber_init(this, priv);
+        if (!ret)
+            ret = br_scrubber_handle_options(this, priv, NULL);
+    }
+
+    if (ret)
+        goto cleanup;
+
+    ret = gf_thread_create(&priv->thread, NULL, br_handle_events, this,
+                           "brhevent");
+    if (ret != 0) {
+        gf_smsg(this->name, GF_LOG_ERROR, -ret, BRB_MSG_THREAD_CREATION_FAILED,
+                NULL);
+        ret = -1;
+    }
+
+    if (!ret) {
+        gf_smsg(this->name, GF_LOG_INFO, 0, BRB_MSG_BITROT_LOADED, "mode=%s",
+                (priv->iamscrubber) ? "SCRUBBER" : "SIGNER", NULL);
+        return 0;
+    }
+
+cleanup:
+    (void)pthread_cond_destroy(&priv->cond);
+    (void)pthread_mutex_destroy(&priv->lock);
 
-        GF_FREE (priv->children);
- free_priv:
-        GF_FREE (priv);
- out:
-        this->private = NULL;
-        return -1;
+    br_free_children(this, priv, priv->child_count);
+
+free_priv:
+    GF_FREE(priv);
+out:
+    this->private = NULL;
+    return -1;
 }
 
 void
-fini (xlator_t *this)
+fini(xlator_t *this)
 {
-        int ret = 0;
-	br_private_t *priv = this->private;
+    br_private_t *priv = this->private;
+
+    if (!priv)
+        return;
 
-        if (!priv)
-                return;
+    if (!priv->iamscrubber)
+        br_fini_signer(this, priv);
+    else
+        (void)br_free_scrubber_monitor(this, priv);
 
-        if (!priv->iamscrubber)
-                br_fini_signer (this, priv);
-        br_free_children (this);
-        if (priv->timer_wheel) {
-                ret = gf_tw_cleanup_timers (priv->timer_wheel);
+    br_free_children(this, priv, priv->child_count);
 
-                if (ret == 0) {
-                        priv->timer_wheel = NULL;
-                }
-        }
+    this->private = NULL;
+    GF_FREE(priv);
+
+    glusterfs_ctx_tw_put(this->ctx);
+
+    return;
+}
+
+static void
+br_reconfigure_monitor(xlator_t *this)
+{
+    int32_t ret = 0;
+
+    ret = br_scrub_state_machine(this, _gf_false);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_ERROR, 0, BRB_MSG_COULD_NOT_SCHEDULE_SCRUB,
+                NULL);
+    }
+}
+
+static int
+br_reconfigure_scrubber(xlator_t *this, dict_t *options)
+{
+    int32_t ret = -1;
+    br_private_t *priv = NULL;
+
+    priv = this->private;
+
+    pthread_mutex_lock(&priv->lock);
+    {
+        ret = br_scrubber_handle_options(this, priv, options);
+    }
+    pthread_mutex_unlock(&priv->lock);
+
+    if (ret)
+        goto err;
+
+    /* change state for all _up_ subvolume(s) */
+    pthread_mutex_lock(&priv->lock);
+    {
+        br_reconfigure_monitor(this);
+    }
+    pthread_mutex_unlock(&priv->lock);
+
+err:
+    return ret;
+}
+
+static int
+br_reconfigure_signer(xlator_t *this, dict_t *options)
+{
+    br_private_t *priv = this->private;
+
+    return br_signer_handle_options(this, priv, options);
+}
 
-        this->private = NULL;
-	GF_FREE (priv);
+int
+reconfigure(xlator_t *this, dict_t *options)
+{
+    int ret = 0;
+    br_private_t *priv = NULL;
+
+    priv = this->private;
 
-	return;
+    if (priv->iamscrubber)
+        ret = br_reconfigure_scrubber(this, options);
+    else
+        ret = br_reconfigure_signer(this, options);
+
+    return ret;
 }
 
 struct xlator_fops fops;
@@ -1488,16 +2155,78 @@ struct xlator_fops fops;
 struct xlator_cbks cbks;
 
 struct volume_options options[] = {
-        { .key = {"expiry-time"},
-          .type = GF_OPTION_TYPE_INT,
-          .default_value = "120",
-          .description = "default time duration for which an object waits "
-                         "before it is signed",
-        },
-        { .key = {"scrubber"},
-          .type = GF_OPTION_TYPE_BOOL,
-          .default_value = "false",
-          .description = "option to run as a scrubber",
-        },
-	{ .key  = {NULL} },
+    {
+        .key = {"expiry-time"},
+        .type = GF_OPTION_TYPE_INT,
+        .default_value = SIGNING_TIMEOUT,
+        .op_version = {GD_OP_VERSION_3_7_0},
+        .flags = OPT_FLAG_SETTABLE,
+        .description = "Waiting time for an object on which it waits "
+                       "before it is signed",
+    },
+    {
+        .key = {"brick-count"},
+        .type = GF_OPTION_TYPE_STR,
+        .description = "Total number of bricks for the current node for "
+                       "all volumes in the trusted storage pool.",
+    },
+    {
+        .key = {"scrubber", "scrub"},
+        .type = GF_OPTION_TYPE_BOOL,
+        .default_value = "false",
+        .op_version = {GD_OP_VERSION_3_7_0},
+        .flags = OPT_FLAG_SETTABLE | OPT_FLAG_FORCE,
+        .description = "option to run as a scrubber",
+    },
+    {
+        .key = {"scrub-throttle"},
+        .type = GF_OPTION_TYPE_STR,
+        .default_value = "lazy",
+        .op_version = {GD_OP_VERSION_3_7_0},
+        .flags = OPT_FLAG_SETTABLE,
+        .description = "Scrub-throttle value is a measure of how fast "
+                       "or slow the scrubber scrubs the filesystem for "
+                       "volume <VOLNAME>",
+    },
+    {
+        .key = {"scrub-freq"},
+        .type = GF_OPTION_TYPE_STR,
+        .default_value = "biweekly",
+        .op_version = {GD_OP_VERSION_3_7_0},
+        .flags = OPT_FLAG_SETTABLE,
+        .description = "Scrub frequency for volume <VOLNAME>",
+    },
+    {
+        .key = {"scrub-state"},
+        .type = GF_OPTION_TYPE_STR,
+        .default_value = "active",
+        .op_version = {GD_OP_VERSION_4_0_0},
+        .flags = OPT_FLAG_SETTABLE,
+        .description = "Pause/Resume scrub. Upon resume, scrubber "
+                       "continues from where it left off.",
+    },
+    {
+        .key = {"signer-threads"},
+        .type = GF_OPTION_TYPE_INT,
+        .default_value = BR_WORKERS,
+        .op_version = {GD_OP_VERSION_8_0},
+        .flags = OPT_FLAG_SETTABLE,
+        .description = "Number of signing process threads. As a best "
+                       "practice, set this to the number of processor cores",
+    },
+    {.key = {NULL}},
+};
+
+xlator_api_t xlator_api = {
+    .init = init,
+    .fini = fini,
+    .notify = notify,
+    .reconfigure = reconfigure,
+    .mem_acct_init = mem_acct_init,
+    .op_version = {1}, /* Present from the initial version */
+    .fops = &fops,
+    .cbks = &cbks,
+    .options = options,
+    .identifier = "bit-rot",
+    .category = GF_MAINTAINED,
 };
diff --git a/xlators/features/bit-rot/src/bitd/bit-rot.h b/xlators/features/bit-rot/src/bitd/bit-rot.h
index a634a1fa76f..8ac7dcdac3d 100644
--- a/xlators/features/bit-rot/src/bitd/bit-rot.h
+++ b/xlators/features/bit-rot/src/bitd/bit-rot.h
@@ -7,126 +7,296 @@
    later), or the GNU General Public License, version 2 (GPLv2), in all
    cases as published by the Free Software Foundation.
 */
+
 #ifndef __BIT_ROT_H__
 #define __BIT_ROT_H__
 
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
-
-#include "glusterfs.h"
-#include "logging.h"
-#include "dict.h"
-#include "xlator.h"
-#include "defaults.h"
-#include "syncop.h"
-#include "syncop-utils.h"
+#include <glusterfs/glusterfs.h>
+#include <glusterfs/logging.h>
+#include <glusterfs/dict.h>
+#include <glusterfs/xlator.h>
+#include <glusterfs/defaults.h>
+#include <glusterfs/syncop.h>
+#include <glusterfs/syncop-utils.h>
 #include "changelog.h"
 #include "timer-wheel.h"
 
+#include <glusterfs/throttle-tbf.h>
+#include "bit-rot-ssm.h"
+
 #include "bit-rot-common.h"
 #include "bit-rot-stub-mem-types.h"
+#include "bit-rot-scrub-status.h"
 
 #include <openssl/sha.h>
 
-/* TODO: make this configurable */
-#define BR_WORKERS 8
+typedef enum scrub_throttle {
+    BR_SCRUB_THROTTLE_VOID = -1,
+    BR_SCRUB_THROTTLE_LAZY = 0,
+    BR_SCRUB_THROTTLE_NORMAL = 1,
+    BR_SCRUB_THROTTLE_AGGRESSIVE = 2,
+    BR_SCRUB_THROTTLE_STALLED = 3,
+} scrub_throttle_t;
+
+typedef enum scrub_freq {
+    BR_FSSCRUB_FREQ_HOURLY = 1,
+    BR_FSSCRUB_FREQ_DAILY,
+    BR_FSSCRUB_FREQ_WEEKLY,
+    BR_FSSCRUB_FREQ_BIWEEKLY,
+    BR_FSSCRUB_FREQ_MONTHLY,
+    BR_FSSCRUB_FREQ_MINUTE,
+    BR_FSSCRUB_FREQ_STALLED,
+} scrub_freq_t;
+
+#define signature_size(hl) (sizeof(br_isignature_t) + hl + 1)
+
+struct br_scanfs {
+    gf_lock_t entrylock;
+
+    pthread_mutex_t waitlock;
+    pthread_cond_t waitcond;
+
+    unsigned int entries;
+    struct list_head queued;
+    struct list_head ready;
+};
 
-#define signature_size(hl) (sizeof (br_isignature_t) + hl + 1)
+/* just need three states to track child status */
+typedef enum br_child_state {
+    BR_CHILD_STATE_CONNECTED = 1,
+    BR_CHILD_STATE_INITIALIZING,
+    BR_CHILD_STATE_CONNFAILED,
+    BR_CHILD_STATE_DISCONNECTED,
+} br_child_state_t;
 
 struct br_child {
-        char child_up;                /* Indicates whether this child is
-                                         up or not */
-        xlator_t *xl;                 /* client xlator corresponding to
-                                         this child */
-        inode_table_t *table;         /* inode table for this child */
-        char brick_path[PATH_MAX];    /* brick export directory of this
-                                         child */
-        struct list_head list;        /* hook to attach to the list of
-                                         UP children */
-        xlator_t *this;               /* Bit rot xlator */
-
-        pthread_t thread;             /* initial crawler for unsigned
-                                         object(s) */
-        int threadrunning;            /* active thread */
-
-        struct mem_pool *timer_pool;  /* timer-wheel's timer mem-pool */
-
-        struct timeval tv;
+    pthread_mutex_t lock;     /* protects child state */
+    char witnessed;           /* witnessed at least one successful
+                                 connection */
+    br_child_state_t c_state; /* current state of this child */
+
+    char child_up;             /* Indicates whether this child is
+                                  up or not */
+    xlator_t *xl;              /* client xlator corresponding to
+                                  this child */
+    inode_table_t *table;      /* inode table for this child */
+    char brick_path[PATH_MAX]; /* brick export directory of this
+                                  child */
+    struct list_head list;     /* hook to attach to the list of
+                                  UP children */
+    xlator_t *this;            /* Bit rot xlator */
+
+    pthread_t thread;  /* initial crawler for unsigned
+                          object(s) or scrub crawler */
+    int threadrunning; /* active thread */
+
+    struct mem_pool *timer_pool; /* timer-wheel's timer mem-pool */
+
+    struct timeval tv;
+
+    struct br_scanfs fsscan; /* per subvolume FS scanner */
+
+    gf_boolean_t active_scrubbing; /* Actively scrubbing or not */
 };
 
 typedef struct br_child br_child_t;
 
 struct br_obj_n_workers {
-        struct list_head objects;         /* queue of objects expired from the
-                                             timer wheel and ready to be picked
-                                             up for signing */
-        pthread_t workers[BR_WORKERS];    /* Threads which pick up the objects
-                                             from the above queue and start
-                                             signing each object */
+    struct list_head objects; /* queue of objects expired from the
+                                 timer wheel and ready to be picked
+                                 up for signing */
+    pthread_t *workers;       /* Threads which pick up the objects
+                                 from the above queue and start
+                                 signing each object */
 };
 
-typedef struct br_obj_n_workers br_obj_n_workers_t;
+struct br_scrubber {
+    xlator_t *this;
 
-struct br_private {
-        pthread_mutex_t lock;
-
-        struct list_head bricks;          /* list of bricks from which CHILD_UP
-                                             has been received */
-
-        pthread_cond_t cond;              /* handling CHILD_UP notifications */
-        pthread_cond_t object_cond;       /* handling signing of objects */
-        int child_count;
-        br_child_t *children;             /* list of subvolumes */
-        int up_children;
-        pthread_t thread;                 /* thread for connecting each UP
-                                             child with changelog */
-        struct tvec_base *timer_wheel;    /* timer wheel where the objects which
-                                             changelog has sent sits and waits
-                                             for expiry */
-        br_obj_n_workers_t *obj_queue;    /* place holder for all the objects
-                                             that are expired from timer wheel
-                                             and ready to be picked up for
-                                             signing and the workers which sign
-                                             the objects */
-        int32_t expiry_time;              /* objects "wait" time */
-
-        gf_boolean_t iamscrubber;         /* function as a fs scrubber */
+    scrub_throttle_t throttle;
+
+    /**
+     * frequency of scanning for this subvolume. this should
+     * normally be per-child, but since all children follow the
+     * same frequency for a volume, this option ends up here
+     * instead of br_child_t.
+     */
+    scrub_freq_t frequency;
+
+    gf_boolean_t frequency_reconf;
+    gf_boolean_t throttle_reconf;
+
+    pthread_mutex_t mutex;
+    pthread_cond_t cond;
+
+    unsigned int nr_scrubbers;
+    struct list_head scrubbers;
+
+    /**
+     * list of "rotatable" subvolume(s) undergoing scrubbing
+     */
+    struct list_head scrublist;
+};
+
+struct br_monitor {
+    gf_lock_t lock;
+    pthread_t thread; /* Monitor thread */
+
+    gf_boolean_t inited;
+    pthread_mutex_t mutex;
+    pthread_cond_t cond; /* Thread starts and will be waiting on cond.
+                            First child which is up wakes this up */
+
+    xlator_t *this;
+    /* scheduler */
+    uint32_t boot;
+
+    int32_t active_child_count; /* Number of children currently scrubbing */
+    gf_boolean_t kick;          /* This variable tracks the scrubber is
+                                 * kicked or not. Both 'kick' and
+                                 * 'active_child_count' uses the same pair
+                                 * of mutex-cond variable, i.e, wakelock and
+                                 * wakecond. */
+
+    pthread_mutex_t wakelock;
+    pthread_cond_t wakecond;
+
+    gf_boolean_t done;
+    pthread_mutex_t donelock;
+    pthread_cond_t donecond;
+
+    struct gf_tw_timer_list *timer;
+    br_scrub_state_t state; /* current scrub state */
 };
 
+typedef struct br_obj_n_workers br_obj_n_workers_t;
+
 typedef struct br_private br_private_t;
 
+typedef void (*br_scrubbed_file_update)(br_private_t *priv);
+
+struct br_private {
+    pthread_mutex_t lock;
+
+    struct list_head bricks; /* list of bricks from which enents
+                                have been received */
+
+    struct list_head signing;
+
+    pthread_cond_t object_cond; /* handling signing of objects */
+    int child_count;
+    br_child_t *children; /* list of subvolumes */
+    int up_children;
+
+    pthread_cond_t cond; /* handling CHILD_UP notifications */
+    pthread_t thread;    /* thread for connecting each UP
+                            child with changelog */
+
+    struct tvec_base *timer_wheel; /* timer wheel where the objects which
+                                      changelog has sent sits and waits
+                                      for expiry */
+    br_obj_n_workers_t *obj_queue; /* place holder for all the objects
+                                      that are expired from timer wheel
+                                      and ready to be picked up for
+                                      signing and the workers which sign
+                                      the objects */
+
+    uint32_t expiry_time; /* objects "wait" time */
+
+    uint32_t signer_th_count; /* Number of signing process threads */
+
+    tbf_t *tbf; /* token bucket filter */
+
+    gf_boolean_t iamscrubber; /* function as a fs scrubber */
+
+    struct br_scrub_stats scrub_stat; /* statistics of scrub*/
+
+    struct br_scrubber fsscrub; /* scrubbers for this subvolume */
+
+    struct br_monitor scrub_monitor; /* scrubber monitor */
+};
+
 struct br_object {
-        xlator_t *this;
+    xlator_t *this;
 
-        uuid_t gfid;
+    uuid_t gfid;
 
-        unsigned long signedversion;    /* version aginst which this object will
-                                           be signed */
-        br_child_t *child;              /* object's subvolume */
+    unsigned long signedversion; /* version against which this object will
+                                    be signed */
+    br_child_t *child;           /* object's subvolume */
 
-        struct list_head list;          /* hook to add to the queue once the
-                                           object is expired from timer wheel */
-        void *data;
+    int sign_info;
+
+    struct list_head list; /* hook to add to the queue once the
+                              object is expired from timer wheel */
+    void *data;
 };
 
 typedef struct br_object br_object_t;
+typedef int32_t(br_scrub_ssm_call)(xlator_t *);
 
 void
-br_log_object (xlator_t *, char *, uuid_t, int32_t);
+br_log_object(xlator_t *, char *, uuid_t, int32_t);
 
 void
-br_log_object_path (xlator_t *, char *, const char *, int32_t);
+br_log_object_path(xlator_t *, char *, const char *, int32_t);
 
 int32_t
-br_calculate_obj_checksum (unsigned char *,
-                           br_child_t *, fd_t *, struct iatt *);
+br_calculate_obj_checksum(unsigned char *, br_child_t *, fd_t *, struct iatt *);
 
 int32_t
-br_prepare_loc (xlator_t *, br_child_t *, loc_t *, gf_dirent_t *, loc_t *);
+br_prepare_loc(xlator_t *, br_child_t *, loc_t *, gf_dirent_t *, loc_t *);
 
 gf_boolean_t
-bitd_is_bad_file (xlator_t *, br_child_t *, loc_t *, fd_t *);
+bitd_is_bad_file(xlator_t *, br_child_t *, loc_t *, fd_t *);
+
+static inline void
+_br_set_child_state(br_child_t *child, br_child_state_t state)
+{
+    child->c_state = state;
+}
+
+static inline int
+_br_is_child_connected(br_child_t *child)
+{
+    return (child->c_state == BR_CHILD_STATE_CONNECTED);
+}
+
+static inline int
+_br_is_child_scrub_active(br_child_t *child)
+{
+    return child->active_scrubbing;
+}
+
+static inline int
+_br_child_failed_conn(br_child_t *child)
+{
+    return (child->c_state == BR_CHILD_STATE_CONNFAILED);
+}
+
+static inline int
+_br_child_witnessed_connection(br_child_t *child)
+{
+    return (child->witnessed == 1);
+}
+
+/* scrub state */
+static inline void
+_br_monitor_set_scrub_state(struct br_monitor *scrub_monitor,
+                            br_scrub_state_t state)
+{
+    scrub_monitor->state = state;
+}
+
+static inline br_scrub_event_t
+_br_child_get_scrub_event(struct br_scrubber *fsscrub)
+{
+    return (fsscrub->frequency == BR_FSSCRUB_FREQ_STALLED)
+               ? BR_SCRUB_EVENT_PAUSE
+               : BR_SCRUB_EVENT_SCHEDULE;
+}
+
+int32_t
+br_get_bad_objects_list(xlator_t *this, dict_t **dict);
 
 #endif /* __BIT_ROT_H__ */
diff --git a/xlators/features/bit-rot/src/stub/Makefile.am b/xlators/features/bit-rot/src/stub/Makefile.am
index 579dd1c02b6..f13de7145fc 100644
--- a/xlators/features/bit-rot/src/stub/Makefile.am
+++ b/xlators/features/bit-rot/src/stub/Makefile.am
@@ -1,15 +1,19 @@
+if WITH_SERVER
 xlator_LTLIBRARIES = bitrot-stub.la
+endif
 xlatordir = $(libdir)/glusterfs/$(PACKAGE_VERSION)/xlator/features
 
-bitrot_stub_la_LDFLAGS = -module -avoid-version
+bitrot_stub_la_LDFLAGS = -module $(GF_XLATOR_DEFAULT_LDFLAGS)
 
-bitrot_stub_la_SOURCES = bit-rot-stub.c
+bitrot_stub_la_SOURCES = bit-rot-stub-helpers.c bit-rot-stub.c
 bitrot_stub_la_LIBADD = $(top_builddir)/libglusterfs/src/libglusterfs.la
 
 noinst_HEADERS = bit-rot-stub.h bit-rot-common.h bit-rot-stub-mem-types.h \
-                 bit-rot-object-version.h
+	bit-rot-object-version.h bit-rot-stub-messages.h
 
-AM_CPPFLAGS = $(GF_CPPFLAGS) -I$(top_srcdir)/libglusterfs/src
+AM_CPPFLAGS = $(GF_CPPFLAGS) -I$(top_srcdir)/libglusterfs/src \
+	-I$(top_srcdir)/rpc/xdr/src -I$(top_builddir)/rpc/xdr/src \
+	-I$(top_srcdir)/rpc/rpc-lib/src
 
 AM_CFLAGS = -Wall $(GF_CFLAGS)
 
diff --git a/xlators/features/bit-rot/src/stub/bit-rot-common.h b/xlators/features/bit-rot/src/stub/bit-rot-common.h
index 699323170d3..20561aa7764 100644
--- a/xlators/features/bit-rot/src/stub/bit-rot-common.h
+++ b/xlators/features/bit-rot/src/stub/bit-rot-common.h
@@ -11,59 +11,74 @@
 #ifndef __BIT_ROT_COMMON_H__
 #define __BIT_ROT_COMMON_H__
 
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
-
-#include "glusterfs.h"
+#include <glusterfs/glusterfs.h>
 #include "bit-rot-object-version.h"
 
-#define BR_VXATTR_VERSION   (1 << 0)
+#define BR_VXATTR_VERSION (1 << 0)
 #define BR_VXATTR_SIGNATURE (1 << 1)
 
 #define BR_VXATTR_SIGN_MISSING (BR_VXATTR_SIGNATURE)
-#define BR_VXATTR_ALL_MISSING                           \
-        (BR_VXATTR_VERSION | BR_VXATTR_SIGNATURE)
+#define BR_VXATTR_ALL_MISSING (BR_VXATTR_VERSION | BR_VXATTR_SIGNATURE)
+
+#define BR_BAD_OBJ_CONTAINER                                                   \
+    (uuid_t) { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8 }
 
 typedef enum br_vxattr_state {
-        BR_VXATTR_STATUS_FULL     = 0,
-        BR_VXATTR_STATUS_MISSING  = 1,
-        BR_VXATTR_STATUS_UNSIGNED = 2,
-        BR_VXATTR_STATUS_INVALID  = 3,
+    BR_VXATTR_STATUS_FULL = 0,
+    BR_VXATTR_STATUS_MISSING = 1,
+    BR_VXATTR_STATUS_UNSIGNED = 2,
+    BR_VXATTR_STATUS_INVALID = 3,
 } br_vxattr_status_t;
 
+typedef enum br_sign_state {
+    BR_SIGN_INVALID = -1,
+    BR_SIGN_NORMAL = 0,
+    BR_SIGN_REOPEN_WAIT = 1,
+    BR_SIGN_QUICK = 2,
+} br_sign_state_t;
+
 static inline br_vxattr_status_t
-br_version_xattr_state (dict_t *xattr,
-                        br_version_t **obuf, br_signature_t **sbuf)
+br_version_xattr_state(dict_t *xattr, br_version_t **obuf,
+                       br_signature_t **sbuf, gf_boolean_t *objbad)
 {
-        int32_t             ret    = 0;
-        int32_t             vxattr = 0;
-        br_vxattr_status_t  status;
-
-        ret = dict_get_bin (xattr, BITROT_CURRENT_VERSION_KEY, (void **)obuf);
-        if (ret)
-                vxattr |= BR_VXATTR_VERSION;
-
-        ret = dict_get_bin (xattr, BITROT_SIGNING_VERSION_KEY, (void **)sbuf);
-        if (ret)
-                vxattr |= BR_VXATTR_SIGNATURE;
-
-        switch (vxattr) {
+    int32_t ret = 0;
+    int32_t vxattr = 0;
+    br_vxattr_status_t status;
+    void *data = NULL;
+
+    /**
+     * The key being present in the dict indicates the xattr was set on
+     * disk. The presence of xattr itself as of now is suffecient to say
+     * the the object is bad.
+     */
+    *objbad = _gf_false;
+    ret = dict_get_bin(xattr, BITROT_OBJECT_BAD_KEY, (void **)&data);
+    if (!ret)
+        *objbad = _gf_true;
+
+    ret = dict_get_bin(xattr, BITROT_CURRENT_VERSION_KEY, (void **)obuf);
+    if (ret)
+        vxattr |= BR_VXATTR_VERSION;
+
+    ret = dict_get_bin(xattr, BITROT_SIGNING_VERSION_KEY, (void **)sbuf);
+    if (ret)
+        vxattr |= BR_VXATTR_SIGNATURE;
+
+    switch (vxattr) {
         case 0:
-                status = BR_VXATTR_STATUS_FULL;
-                break;
+            status = BR_VXATTR_STATUS_FULL;
+            break;
         case BR_VXATTR_SIGN_MISSING:
-                status = BR_VXATTR_STATUS_UNSIGNED;
-                break;
+            status = BR_VXATTR_STATUS_UNSIGNED;
+            break;
         case BR_VXATTR_ALL_MISSING:
-                status = BR_VXATTR_STATUS_MISSING;
-                break;
+            status = BR_VXATTR_STATUS_MISSING;
+            break;
         default:
-                status = BR_VXATTR_STATUS_INVALID;
-        }
+            status = BR_VXATTR_STATUS_INVALID;
+    }
 
-        return status;
+    return status;
 }
 
 /**
@@ -71,13 +86,13 @@ br_version_xattr_state (dict_t *xattr,
  * signing.
  */
 typedef struct br_isignature_in {
-        int8_t signaturetype;            /* signature type            */
+    int8_t signaturetype; /* signature type            */
 
-        unsigned long signedversion;     /* version against which the
-                                            object was signed         */
+    unsigned long signedversion; /* version against which the
+                                    object was signed         */
 
-        size_t signaturelen;             /* signature length          */
-        char signature[0];               /* object signature          */
+    size_t signaturelen; /* signature length          */
+    char signature[0];   /* object signature          */
 } br_isignature_t;
 
 /**
@@ -85,75 +100,79 @@ typedef struct br_isignature_in {
  * verification.
  */
 typedef struct br_isignature_out {
-        char stale;                      /* stale signature?          */
+    char stale; /* stale signature?          */
 
-        unsigned long version;           /* current signed version    */
+    unsigned long version; /* current signed version    */
 
-        uint32_t time[2];                /* time when the object
-                                            got dirtied               */
+    uint32_t time[2]; /* time when the object
+                         got dirtied               */
 
-        int8_t signaturetype;            /* hash type                 */
-        size_t signaturelen;             /* signature length          */
-        char   signature[0];             /* signature (hash)          */
+    int8_t signaturetype; /* hash type                 */
+    size_t signaturelen;  /* signature length          */
+    char signature[0];    /* signature (hash)          */
 } br_isignature_out_t;
 
 typedef struct br_stub_init {
-        uint32_t timebuf[2];
-        char export[PATH_MAX];
+    uint32_t timebuf[2];
+    char export[PATH_MAX];
 } br_stub_init_t;
 
 typedef enum {
-        BR_SIGNATURE_TYPE_VOID   = -1,   /* object is not signed       */
-        BR_SIGNATURE_TYPE_ZERO   = 0,    /* min boundary               */
-        BR_SIGNATURE_TYPE_SHA256 = 1,    /* signed with SHA256         */
-        BR_SIGNATURE_TYPE_MAX    = 2,    /* max boundary               */
+    BR_SIGNATURE_TYPE_VOID = -1,  /* object is not signed       */
+    BR_SIGNATURE_TYPE_ZERO = 0,   /* min boundary               */
+    BR_SIGNATURE_TYPE_SHA256 = 1, /* signed with SHA256         */
+    BR_SIGNATURE_TYPE_MAX = 2,    /* max boundary               */
 } br_signature_type;
 
 /* BitRot stub start time (virtual xattr) */
-#define GLUSTERFS_GET_BR_STUB_INIT_TIME  "trusted.glusterfs.bit-rot.stub-init"
+#define GLUSTERFS_GET_BR_STUB_INIT_TIME "trusted.glusterfs.bit-rot.stub-init"
+
+/* signing/reopen hint */
+#define BR_OBJECT_RESIGN 0
+#define BR_OBJECT_REOPEN 1
+#define BR_REOPEN_SIGN_HINT_KEY "trusted.glusterfs.bit-rot.reopen-hint"
 
 static inline int
-br_is_signature_type_valid (int8_t signaturetype)
+br_is_signature_type_valid(int8_t signaturetype)
 {
-        return ((signaturetype > BR_SIGNATURE_TYPE_ZERO)
-                 && (signaturetype < BR_SIGNATURE_TYPE_MAX));
+    return ((signaturetype > BR_SIGNATURE_TYPE_ZERO) &&
+            (signaturetype < BR_SIGNATURE_TYPE_MAX));
 }
 
 static inline void
-br_set_default_ongoingversion (br_version_t *buf, uint32_t *tv)
+br_set_default_ongoingversion(br_version_t *buf, uint32_t *tv)
 {
-        buf->ongoingversion = BITROT_DEFAULT_CURRENT_VERSION;
-        buf->timebuf[0] = tv[0];
-        buf->timebuf[1] = tv[1];
+    buf->ongoingversion = BITROT_DEFAULT_CURRENT_VERSION;
+    buf->timebuf[0] = tv[0];
+    buf->timebuf[1] = tv[1];
 }
 
 static inline void
-br_set_default_signature (br_signature_t *buf, size_t *size)
+br_set_default_signature(br_signature_t *buf, size_t *size)
 {
-        buf->signaturetype = (int8_t) BR_SIGNATURE_TYPE_VOID;
-        buf->signedversion = BITROT_DEFAULT_SIGNING_VERSION;
+    buf->signaturetype = (int8_t)BR_SIGNATURE_TYPE_VOID;
+    buf->signedversion = BITROT_DEFAULT_SIGNING_VERSION;
 
-        *size = sizeof (br_signature_t); /* no signature */
+    *size = sizeof(br_signature_t); /* no signature */
 }
 
 static inline void
-br_set_ongoingversion (br_version_t *buf,
-                       unsigned long version, uint32_t *tv)
+br_set_ongoingversion(br_version_t *buf, unsigned long version, uint32_t *tv)
 {
-        buf->ongoingversion = version;
-        buf->timebuf[0] = tv[0];
-        buf->timebuf[1] = tv[1];
+    buf->ongoingversion = version;
+    buf->timebuf[0] = tv[0];
+    buf->timebuf[1] = tv[1];
 }
 
 static inline void
-br_set_signature (br_signature_t *buf,
-                  br_isignature_t *sign, size_t signaturelen, size_t *size)
+br_set_signature(br_signature_t *buf, br_isignature_t *sign,
+                 size_t signaturelen, size_t *size)
 {
-        buf->signaturetype  = sign->signaturetype;
-        buf->signedversion  = ntohl (sign->signedversion);
+    buf->signaturetype = sign->signaturetype;
+    buf->signedversion = ntohl(sign->signedversion);
 
-        memcpy (buf->signature, sign->signature, signaturelen);
-        *size = sizeof (br_signature_t) + signaturelen;
+    memcpy(buf->signature, sign->signature, signaturelen);
+    *size = sizeof(br_signature_t) + signaturelen;
 }
 
 #endif /* __BIT_ROT_COMMON_H__ */
diff --git a/xlators/features/bit-rot/src/stub/bit-rot-object-version.h b/xlators/features/bit-rot/src/stub/bit-rot-object-version.h
index 1f2497aebe9..7ae6a5200df 100644
--- a/xlators/features/bit-rot/src/stub/bit-rot-object-version.h
+++ b/xlators/features/bit-rot/src/stub/bit-rot-object-version.h
@@ -15,16 +15,16 @@
  * on-disk formats for ongoing version and object signature.
  */
 typedef struct br_version {
-        unsigned long ongoingversion;
-        uint32_t timebuf[2];
+    unsigned long ongoingversion;
+    uint32_t timebuf[2];
 } br_version_t;
 
-typedef struct __attribute__ ((__packed__)) br_signature {
-        int8_t signaturetype;
+typedef struct __attribute__((__packed__)) br_signature {
+    int8_t signaturetype;
 
-        unsigned long signedversion;
+    unsigned long signedversion;
 
-        char signature[0];
+    char signature[0];
 } br_signature_t;
 
 #endif
diff --git a/xlators/features/bit-rot/src/stub/bit-rot-stub-helpers.c b/xlators/features/bit-rot/src/stub/bit-rot-stub-helpers.c
new file mode 100644
index 00000000000..8ac13a09941
--- /dev/null
+++ b/xlators/features/bit-rot/src/stub/bit-rot-stub-helpers.c
@@ -0,0 +1,796 @@
+/*
+   Copyright (c) 2015 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
+
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
+
+#include "bit-rot-stub.h"
+
+br_stub_fd_t *
+br_stub_fd_new(void)
+{
+    br_stub_fd_t *br_stub_fd = NULL;
+
+    br_stub_fd = GF_CALLOC(1, sizeof(*br_stub_fd), gf_br_stub_mt_br_stub_fd_t);
+
+    return br_stub_fd;
+}
+
+int
+__br_stub_fd_ctx_set(xlator_t *this, fd_t *fd, br_stub_fd_t *br_stub_fd)
+{
+    uint64_t value = 0;
+    int ret = -1;
+
+    GF_VALIDATE_OR_GOTO("bit-rot-stub", this, out);
+    GF_VALIDATE_OR_GOTO(this->name, fd, out);
+    GF_VALIDATE_OR_GOTO(this->name, br_stub_fd, out);
+
+    value = (uint64_t)(long)br_stub_fd;
+
+    ret = __fd_ctx_set(fd, this, value);
+
+out:
+    return ret;
+}
+
+br_stub_fd_t *
+__br_stub_fd_ctx_get(xlator_t *this, fd_t *fd)
+{
+    br_stub_fd_t *br_stub_fd = NULL;
+    uint64_t value = 0;
+    int ret = -1;
+
+    GF_VALIDATE_OR_GOTO("bit-rot-stub", this, out);
+    GF_VALIDATE_OR_GOTO(this->name, fd, out);
+
+    ret = __fd_ctx_get(fd, this, &value);
+    if (ret)
+        return NULL;
+
+    br_stub_fd = (br_stub_fd_t *)((long)value);
+
+out:
+    return br_stub_fd;
+}
+
+br_stub_fd_t *
+br_stub_fd_ctx_get(xlator_t *this, fd_t *fd)
+{
+    br_stub_fd_t *br_stub_fd = NULL;
+
+    GF_VALIDATE_OR_GOTO("bit-rot-stub", this, out);
+    GF_VALIDATE_OR_GOTO(this->name, fd, out);
+
+    LOCK(&fd->lock);
+    {
+        br_stub_fd = __br_stub_fd_ctx_get(this, fd);
+    }
+    UNLOCK(&fd->lock);
+
+out:
+    return br_stub_fd;
+}
+
+int32_t
+br_stub_fd_ctx_set(xlator_t *this, fd_t *fd, br_stub_fd_t *br_stub_fd)
+{
+    int32_t ret = -1;
+
+    GF_VALIDATE_OR_GOTO("bit-rot-stub", this, out);
+    GF_VALIDATE_OR_GOTO(this->name, fd, out);
+    GF_VALIDATE_OR_GOTO(this->name, br_stub_fd, out);
+
+    LOCK(&fd->lock);
+    {
+        ret = __br_stub_fd_ctx_set(this, fd, br_stub_fd);
+    }
+    UNLOCK(&fd->lock);
+
+out:
+    return ret;
+}
+
+/**
+ * Adds an entry to the bad objects directory.
+ * @gfid: gfid of the bad object being added to the bad objects directory
+ */
+int
+br_stub_add(xlator_t *this, uuid_t gfid)
+{
+    char gfid_path[BR_PATH_MAX_PLUS] = {0};
+    char bad_gfid_path[BR_PATH_MAX_PLUS] = {0};
+    int ret = 0;
+    br_stub_private_t *priv = NULL;
+    struct stat st = {0};
+
+    priv = this->private;
+    GF_ASSERT_AND_GOTO_WITH_ERROR(this->name, !gf_uuid_is_null(gfid), out,
+                                  errno, EINVAL);
+
+    snprintf(gfid_path, sizeof(gfid_path), "%s/%s", priv->stub_basepath,
+             uuid_utoa(gfid));
+
+    ret = sys_stat(gfid_path, &st);
+    if (!ret)
+        goto out;
+    snprintf(bad_gfid_path, sizeof(bad_gfid_path), "%s/stub-%s",
+             priv->stub_basepath, uuid_utoa(priv->bad_object_dir_gfid));
+
+    ret = sys_link(bad_gfid_path, gfid_path);
+    if (ret) {
+        if ((errno != ENOENT) && (errno != EMLINK) && (errno != EEXIST))
+            goto out;
+
+        /*
+         * Continue with success. At least we'll have half of the
+         * functionality, in the sense, object is marked bad and
+         * would be inaccessible. It's only scrub status that would
+         * show up less number of objects. That's fine as we'll have
+         * the log files that will have the missing information.
+         */
+        gf_smsg(this->name, GF_LOG_WARNING, errno, BRS_MSG_LINK_FAIL, "gfid=%s",
+                uuid_utoa(gfid), NULL);
+    }
+
+    return 0;
+out:
+    return -1;
+}
+
+int
+br_stub_del(xlator_t *this, uuid_t gfid)
+{
+    int32_t op_errno __attribute__((unused)) = 0;
+    br_stub_private_t *priv = NULL;
+    int ret = 0;
+    char gfid_path[BR_PATH_MAX_PLUS] = {0};
+
+    priv = this->private;
+    GF_ASSERT_AND_GOTO_WITH_ERROR(this->name, !gf_uuid_is_null(gfid), out,
+                                  op_errno, EINVAL);
+    snprintf(gfid_path, sizeof(gfid_path), "%s/%s", priv->stub_basepath,
+             uuid_utoa(gfid));
+    ret = sys_unlink(gfid_path);
+    if (ret && (errno != ENOENT)) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, BRS_MSG_BAD_OBJ_UNLINK_FAIL,
+                "path=%s", gfid_path, NULL);
+        ret = -errno;
+        goto out;
+    }
+
+    ret = 0;
+
+out:
+    return ret;
+}
+
+static int
+br_stub_check_stub_directory(xlator_t *this, char *fullpath)
+{
+    int ret = 0;
+    struct stat st = {
+        0,
+    };
+    char oldpath[BR_PATH_MAX_PLUS] = {0};
+    br_stub_private_t *priv = NULL;
+
+    priv = this->private;
+
+    snprintf(oldpath, sizeof(oldpath), "%s/%s", priv->export,
+             OLD_BR_STUB_QUARANTINE_DIR);
+
+    ret = sys_stat(fullpath, &st);
+    if (!ret && !S_ISDIR(st.st_mode))
+        goto error_return;
+    if (ret) {
+        if (errno != ENOENT)
+            goto error_return;
+        ret = sys_stat(oldpath, &st);
+        if (ret)
+            ret = mkdir_p(fullpath, 0600, _gf_true);
+        else
+            ret = sys_rename(oldpath, fullpath);
+    }
+
+    if (ret)
+        gf_smsg(this->name, GF_LOG_ERROR, errno, BRS_MSG_BAD_OBJECT_DIR_FAIL,
+                "create-path=%s", fullpath, NULL);
+    return ret;
+
+error_return:
+    gf_smsg(this->name, GF_LOG_ERROR, errno, BRS_MSG_BAD_OBJECT_DIR_FAIL,
+            "verify-path=%s", fullpath, NULL);
+    return -1;
+}
+
+/**
+ * Function to create the container for the bad objects within the bad objects
+ * directory.
+ */
+static int
+br_stub_check_stub_file(xlator_t *this, char *path)
+{
+    int ret = 0;
+    int fd = -1;
+    struct stat st = {
+        0,
+    };
+
+    ret = sys_stat(path, &st);
+    if (!ret && !S_ISREG(st.st_mode))
+        goto error_return;
+    if (ret) {
+        if (errno != ENOENT)
+            goto error_return;
+        fd = sys_creat(path, 0);
+        if (fd < 0)
+            gf_smsg(this->name, GF_LOG_ERROR, errno,
+                    BRS_MSG_BAD_OBJECT_DIR_FAIL, "create-path=%s", path, NULL);
+    }
+
+    if (fd >= 0) {
+        sys_close(fd);
+        ret = 0;
+    }
+
+    return ret;
+
+error_return:
+    gf_smsg(this->name, GF_LOG_ERROR, errno, BRS_MSG_BAD_OBJECT_DIR_FAIL,
+            "verify-path=%s", path, NULL);
+    return -1;
+}
+
+int
+br_stub_dir_create(xlator_t *this, br_stub_private_t *priv)
+{
+    int ret = -1;
+    char fullpath[BR_PATH_MAX_PLUS] = {
+        0,
+    };
+    char stub_gfid_path[BR_PATH_MAX_PLUS] = {
+        0,
+    };
+
+    gf_uuid_copy(priv->bad_object_dir_gfid, BR_BAD_OBJ_CONTAINER);
+
+    if (snprintf(fullpath, sizeof(fullpath), "%s", priv->stub_basepath) >=
+        sizeof(fullpath))
+        goto out;
+
+    if (snprintf(stub_gfid_path, sizeof(stub_gfid_path), "%s/stub-%s",
+                 priv->stub_basepath, uuid_utoa(priv->bad_object_dir_gfid)) >=
+        sizeof(stub_gfid_path))
+        goto out;
+
+    ret = br_stub_check_stub_directory(this, fullpath);
+    if (ret)
+        goto out;
+    ret = br_stub_check_stub_file(this, stub_gfid_path);
+    if (ret)
+        goto out;
+
+    return 0;
+
+out:
+    return -1;
+}
+
+call_stub_t *
+__br_stub_dequeue(struct list_head *callstubs)
+{
+    call_stub_t *stub = NULL;
+
+    if (!list_empty(callstubs)) {
+        stub = list_entry(callstubs->next, call_stub_t, list);
+        list_del_init(&stub->list);
+    }
+
+    return stub;
+}
+
+void
+__br_stub_enqueue(struct list_head *callstubs, call_stub_t *stub)
+{
+    list_add_tail(&stub->list, callstubs);
+}
+
+void
+br_stub_worker_enqueue(xlator_t *this, call_stub_t *stub)
+{
+    br_stub_private_t *priv = NULL;
+
+    priv = this->private;
+    pthread_mutex_lock(&priv->container.bad_lock);
+    {
+        __br_stub_enqueue(&priv->container.bad_queue, stub);
+        pthread_cond_signal(&priv->container.bad_cond);
+    }
+    pthread_mutex_unlock(&priv->container.bad_lock);
+}
+
+void *
+br_stub_worker(void *data)
+{
+    br_stub_private_t *priv = NULL;
+    xlator_t *this = NULL;
+    call_stub_t *stub = NULL;
+
+    THIS = data;
+    this = data;
+    priv = this->private;
+
+    for (;;) {
+        pthread_mutex_lock(&priv->container.bad_lock);
+        {
+            while (list_empty(&priv->container.bad_queue)) {
+                (void)pthread_cond_wait(&priv->container.bad_cond,
+                                        &priv->container.bad_lock);
+            }
+
+            stub = __br_stub_dequeue(&priv->container.bad_queue);
+        }
+        pthread_mutex_unlock(&priv->container.bad_lock);
+
+        if (stub) /* guard against spurious wakeups */
+            call_resume(stub);
+    }
+
+    return NULL;
+}
+
+int32_t
+br_stub_lookup_wrapper(call_frame_t *frame, xlator_t *this, loc_t *loc,
+                       dict_t *xattr_req)
+{
+    br_stub_private_t *priv = NULL;
+    struct stat lstatbuf = {0};
+    int ret = 0;
+    int32_t op_errno = EINVAL;
+    int32_t op_ret = -1;
+    struct iatt stbuf = {
+        0,
+    };
+    struct iatt postparent = {
+        0,
+    };
+    dict_t *xattr = NULL;
+    gf_boolean_t ver_enabled = _gf_false;
+
+    BR_STUB_VER_ENABLED_IN_CALLPATH(frame, ver_enabled);
+    priv = this->private;
+    BR_STUB_VER_COND_GOTO(priv, (!ver_enabled), done);
+
+    VALIDATE_OR_GOTO(loc, done);
+    if (gf_uuid_compare(loc->gfid, priv->bad_object_dir_gfid))
+        goto done;
+
+    ret = sys_lstat(priv->stub_basepath, &lstatbuf);
+    if (ret) {
+        gf_msg_debug(this->name, errno,
+                     "Stat failed on stub bad "
+                     "object dir");
+        op_errno = errno;
+        goto done;
+    } else if (!S_ISDIR(lstatbuf.st_mode)) {
+        gf_msg_debug(this->name, errno,
+                     "bad object container is not "
+                     "a directory");
+        op_errno = ENOTDIR;
+        goto done;
+    }
+
+    iatt_from_stat(&stbuf, &lstatbuf);
+    gf_uuid_copy(stbuf.ia_gfid, priv->bad_object_dir_gfid);
+
+    op_ret = op_errno = 0;
+    xattr = dict_new();
+    if (!xattr) {
+        op_ret = -1;
+        op_errno = ENOMEM;
+    }
+
+done:
+    STACK_UNWIND_STRICT(lookup, frame, op_ret, op_errno, loc->inode, &stbuf,
+                        xattr, &postparent);
+    if (xattr)
+        dict_unref(xattr);
+    return 0;
+}
+
+static int
+is_bad_gfid_file_current(char *filename, uuid_t gfid)
+{
+    char current_stub_gfid[GF_UUID_BUF_SIZE + 16] = {
+        0,
+    };
+
+    snprintf(current_stub_gfid, sizeof current_stub_gfid, "stub-%s",
+             uuid_utoa(gfid));
+    return (!strcmp(filename, current_stub_gfid));
+}
+
+static void
+check_delete_stale_bad_file(xlator_t *this, char *filename)
+{
+    int ret = 0;
+    struct stat st = {0};
+    char filepath[BR_PATH_MAX_PLUS] = {0};
+    br_stub_private_t *priv = NULL;
+
+    priv = this->private;
+
+    if (is_bad_gfid_file_current(filename, priv->bad_object_dir_gfid))
+        return;
+
+    snprintf(filepath, sizeof(filepath), "%s/%s", priv->stub_basepath,
+             filename);
+
+    ret = sys_stat(filepath, &st);
+    if (!ret && st.st_nlink == 1)
+        sys_unlink(filepath);
+}
+
+static int
+br_stub_fill_readdir(fd_t *fd, br_stub_fd_t *fctx, DIR *dir, off_t off,
+                     size_t size, gf_dirent_t *entries)
+{
+    off_t in_case = -1;
+    off_t last_off = 0;
+    size_t filled = 0;
+    int count = 0;
+    int32_t this_size = -1;
+    gf_dirent_t *this_entry = NULL;
+    xlator_t *this = NULL;
+    struct dirent *entry = NULL;
+    struct dirent scratch[2] = {
+        {
+            0,
+        },
+    };
+
+    this = THIS;
+    if (!off) {
+        rewinddir(dir);
+    } else {
+        seekdir(dir, off);
+#ifndef GF_LINUX_HOST_OS
+        if ((u_long)telldir(dir) != off && off != fctx->bad_object.dir_eof) {
+            gf_smsg(THIS->name, GF_LOG_ERROR, 0,
+                    BRS_MSG_BAD_OBJECT_DIR_SEEK_FAIL, "off=(0x%llx)", off,
+                    "dir=%p", dir, NULL);
+            errno = EINVAL;
+            count = -1;
+            goto out;
+        }
+#endif /* GF_LINUX_HOST_OS */
+    }
+
+    while (filled <= size) {
+        in_case = (u_long)telldir(dir);
+
+        if (in_case == -1) {
+            gf_smsg(THIS->name, GF_LOG_ERROR, 0,
+                    BRS_MSG_BAD_OBJECT_DIR_TELL_FAIL, "dir=%p", dir, "err=%s",
+                    strerror(errno), NULL);
+            goto out;
+        }
+
+        errno = 0;
+        entry = sys_readdir(dir, scratch);
+        if (!entry || errno != 0) {
+            if (errno == EBADF) {
+                gf_smsg(THIS->name, GF_LOG_WARNING, 0,
+                        BRS_MSG_BAD_OBJECT_DIR_READ_FAIL, "dir=%p", dir,
+                        "err=%s", strerror(errno), NULL);
+                goto out;
+            }
+            break;
+        }
+
+        if (!strcmp(entry->d_name, ".") || !strcmp(entry->d_name, ".."))
+            continue;
+
+        if (!strncmp(entry->d_name, "stub-", strlen("stub-"))) {
+            check_delete_stale_bad_file(this, entry->d_name);
+            continue;
+        }
+
+        this_size = max(sizeof(gf_dirent_t), sizeof(gfs3_dirplist)) +
+                    strlen(entry->d_name) + 1;
+
+        if (this_size + filled > size) {
+            seekdir(dir, in_case);
+#ifndef GF_LINUX_HOST_OS
+            if ((u_long)telldir(dir) != in_case &&
+                in_case != fctx->bad_object.dir_eof) {
+                gf_smsg(THIS->name, GF_LOG_ERROR, 0,
+                        BRS_MSG_BAD_OBJECT_DIR_SEEK_FAIL, "in_case=(0x%llx)",
+                        in_case, "dir=%p", dir, NULL);
+                errno = EINVAL;
+                count = -1;
+                goto out;
+            }
+#endif /* GF_LINUX_HOST_OS */
+            break;
+        }
+
+        this_entry = gf_dirent_for_name(entry->d_name);
+
+        if (!this_entry) {
+            gf_smsg(THIS->name, GF_LOG_ERROR, 0,
+                    BRS_MSG_CREATE_GF_DIRENT_FAILED, "entry-name=%s",
+                    entry->d_name, "err=%s", strerror(errno), NULL);
+            goto out;
+        }
+        /*
+         * we store the offset of next entry here, which is
+         * probably not intended, but code using syncop_readdir()
+         * (glfs-heal.c, afr-self-heald.c, pump.c) rely on it
+         * for directory read resumption.
+         */
+        last_off = (u_long)telldir(dir);
+        this_entry->d_off = last_off;
+        this_entry->d_ino = entry->d_ino;
+
+        list_add_tail(&this_entry->list, &entries->list);
+
+        filled += this_size;
+        count++;
+    }
+
+    if ((!sys_readdir(dir, scratch) && (errno == 0))) {
+        /* Indicate EOF */
+        errno = ENOENT;
+        /* Remember EOF offset for later detection */
+        fctx->bad_object.dir_eof = last_off;
+    }
+out:
+    return count;
+}
+
+int32_t
+br_stub_readdir_wrapper(call_frame_t *frame, xlator_t *this, fd_t *fd,
+                        size_t size, off_t off, dict_t *xdata)
+{
+    br_stub_fd_t *fctx = NULL;
+    DIR *dir = NULL;
+    int ret = -1;
+    int32_t op_ret = -1;
+    int32_t op_errno = 0;
+    int count = 0;
+    gf_dirent_t entries;
+    gf_boolean_t xdata_unref = _gf_false;
+    dict_t *dict = NULL;
+
+    INIT_LIST_HEAD(&entries.list);
+
+    fctx = br_stub_fd_ctx_get(this, fd);
+    if (!fctx) {
+        gf_smsg(this->name, GF_LOG_WARNING, 0, BRS_MSG_GET_FD_CONTEXT_FAILED,
+                "fd=%p", fd, NULL);
+        op_errno = -ret;
+        goto done;
+    }
+
+    dir = fctx->bad_object.dir;
+
+    if (!dir) {
+        gf_smsg(this->name, GF_LOG_WARNING, 0, BRS_MSG_BAD_HANDLE_DIR_NULL,
+                "fd=%p", fd, NULL);
+        op_errno = EINVAL;
+        goto done;
+    }
+
+    count = br_stub_fill_readdir(fd, fctx, dir, off, size, &entries);
+
+    /* pick ENOENT to indicate EOF */
+    op_errno = errno;
+    op_ret = count;
+
+    dict = xdata;
+    (void)br_stub_bad_objects_path(this, fd, &entries, &dict);
+    if (!xdata && dict) {
+        xdata = dict;
+        xdata_unref = _gf_true;
+    }
+
+done:
+    STACK_UNWIND_STRICT(readdir, frame, op_ret, op_errno, &entries, xdata);
+    gf_dirent_free(&entries);
+    if (xdata_unref)
+        dict_unref(xdata);
+    return 0;
+}
+
+/**
+ * This function is called to mainly obtain the paths of the corrupt
+ * objects (files as of now). Currently scrub status prints only the
+ * gfid of the corrupted files. Reason is, bitrot-stub maintains the
+ * list of the corrupted objects as entries inside the quarantine
+ * directory (<brick export>/.glusterfs/quarantine)
+ *
+ * And the name of each entry in the qurantine directory is the gfid
+ * of the corrupted object. So scrub status will just show that info.
+ * But it helps the users a lot if the actual path to the object is
+ * also reported. Hence the below function to get that information.
+ * The function allocates a new dict to be returned (if it does not
+ * get one from the caller of readdir i.e. scrubber as of now), and
+ * stores the paths of each corrupted gfid there. The gfid is used as
+ * the key and path is used as the value.
+ *
+ * NOTE: The path will be there in following situations
+ * 1) gfid2path option has been enabled (posix xlator option)
+ *    and the corrupted file contains the path as an extended
+ *    attribute.
+ * 2) If the gfid2path option is not enabled, OR if the xattr
+ *    is absent, then the inode table should have it.
+ *    The path will be there if a name based lookup has happened
+ *    on the file which has been corrupted. With lookup a inode and
+ *    dentry would be created in the inode table. And the path is
+ *    constructed using the in memory inode and dentry. If a lookup
+ *    has not happened OR the inode corresponding to the corrupted
+ *    file does not exist in the inode table (because it got purged
+ *    as lru limit of the inodes exceeded) OR a nameless lookup had
+ *    happened to populate the inode in the inode table, then the
+ *    path will not be printed in scrub and only the gfid will be there.
+ **/
+int
+br_stub_bad_objects_path(xlator_t *this, fd_t *fd, gf_dirent_t *entries,
+                         dict_t **dict)
+{
+    gf_dirent_t *entry = NULL;
+    inode_t *inode = NULL;
+    char *hpath = NULL;
+    uuid_t gfid = {0};
+    int ret = -1;
+    dict_t *tmp_dict = NULL;
+    char str_gfid[64] = {0};
+
+    if (list_empty(&entries->list))
+        return 0;
+
+    tmp_dict = *dict;
+
+    if (!tmp_dict) {
+        tmp_dict = dict_new();
+        /*
+         * If the allocation of dict fails then no need treat it
+         * it as a error. This path (or function) is executed when
+         * "gluster volume bitrot <volume name> scrub status" is
+         * executed, to get the list of the corrupted objects.
+         * And the motive of this function is to get the paths of
+         * the corrupted objects. If the dict allocation fails, then
+         * the scrub status will only show the gfids of those corrupted
+         * objects (which is the behavior as of the time of this patch
+         * being worked upon). So just return and only the gfids will
+         * be shown.
+         */
+        if (!tmp_dict) {
+            gf_smsg(this->name, GF_LOG_ERROR, 0, BRS_MSG_ALLOC_FAILED, NULL);
+            goto out;
+        }
+    }
+
+    list_for_each_entry(entry, &entries->list, list)
+    {
+        gf_uuid_clear(gfid);
+        gf_uuid_parse(entry->d_name, gfid);
+
+        inode = inode_find(fd->inode->table, gfid);
+
+        /* No need to check the return value here.
+         * Because @hpath is examined.
+         */
+        (void)br_stub_get_path_of_gfid(this, fd->inode, inode, gfid, &hpath);
+
+        if (hpath) {
+            gf_msg_debug(this->name, 0,
+                         "path of the corrupted "
+                         "object (gfid: %s) is %s",
+                         uuid_utoa(gfid), hpath);
+            br_stub_entry_xattr_fill(this, hpath, entry, tmp_dict);
+        } else
+            gf_smsg(this->name, GF_LOG_WARNING, 0, BRS_MSG_PATH_GET_FAILED,
+                    "gfid=%s", uuid_utoa_r(gfid, str_gfid), NULL);
+
+        inode = NULL;
+        hpath = NULL;
+    }
+
+    ret = 0;
+    *dict = tmp_dict;
+
+out:
+    return ret;
+}
+
+int
+br_stub_get_path_of_gfid(xlator_t *this, inode_t *parent, inode_t *inode,
+                         uuid_t gfid, char **path)
+{
+    int32_t ret = -1;
+    char gfid_str[64] = {0};
+
+    GF_VALIDATE_OR_GOTO("bitrot-stub", this, out);
+    GF_VALIDATE_OR_GOTO(this->name, parent, out);
+    GF_VALIDATE_OR_GOTO(this->name, path, out);
+
+    /* Above, No need to validate the @inode for hard resolution. Because
+     * inode can be NULL and if it is NULL, then syncop_gfid_to_path_hard
+     * will allocate a new inode and proceed. So no need to bother about
+     * @inode. Because we need it only to send a syncop_getxattr call
+     * from inside syncop_gfid_to_path_hard. And getxattr fetches the
+     * path from the backend.
+     */
+
+    ret = syncop_gfid_to_path_hard(parent->table, FIRST_CHILD(this), gfid,
+                                   inode, path, _gf_true);
+    if (ret < 0)
+        gf_smsg(this->name, GF_LOG_WARNING, 0, BRS_MSG_PATH_GET_FAILED,
+                "gfid=%s", uuid_utoa_r(gfid, gfid_str), NULL);
+
+    /*
+     * Try with soft resolution of path if hard resolve fails. Because
+     * checking the xattr on disk to get the path of a inode (or gfid)
+     * is dependent on whether that option is enabled in the posix
+     * xlator or not. If it is not enabled, then hard resolution by
+     * checking the on disk xattr fails.
+     *
+     * Thus in such situations fall back to the soft resolution which
+     * mainly depends on the inode_path() function. And for using
+     * inode_path, @inode has to be linked i.e. a successful lookup should
+     * have happened on the gfid (or the path) to link the inode to the
+     * inode table. And if @inode is NULL, means, the inode has not been
+     * found in the inode table and better not to do inode_path() on the
+     * inode which has not been linked.
+     */
+    if (ret < 0 && inode) {
+        ret = syncop_gfid_to_path_hard(parent->table, FIRST_CHILD(this), gfid,
+                                       inode, path, _gf_false);
+        if (ret < 0)
+            gf_smsg(this->name, GF_LOG_WARNING, 0, BRS_MSG_PATH_GET_FAILED,
+                    "from-memory  gfid=%s", uuid_utoa_r(gfid, gfid_str), NULL);
+    }
+
+out:
+    return ret;
+}
+
+/**
+ * NOTE: If the file has multiple hardlinks (in gluster volume
+ * namespace), the path would be one of the hardlinks. Its up to
+ * the user to find the remaining hardlinks (using find -samefile)
+ * and remove them.
+ **/
+void
+br_stub_entry_xattr_fill(xlator_t *this, char *hpath, gf_dirent_t *entry,
+                         dict_t *dict)
+{
+    int32_t ret = -1;
+
+    GF_VALIDATE_OR_GOTO("bit-rot-stub", this, out);
+    GF_VALIDATE_OR_GOTO(this->name, hpath, out);
+
+    /*
+     * Use the entry->d_name (which is nothing but the gfid of the
+     * corrupted object) as the key. And the value will be the actual
+     * path of that object (or file).
+     *
+     * ALso ignore the dict_set errors. scrubber will get the gfid of
+     * the corrupted object for sure. So, for now lets just log the
+     * dict_set_dynstr failure and move on.
+     */
+
+    ret = dict_set_dynstr(dict, entry->d_name, hpath);
+    if (ret)
+        gf_smsg(this->name, GF_LOG_WARNING, 0, BRS_MSG_DICT_SET_FAILED,
+                "path=%s", hpath, "object-name=%s", entry->d_name, NULL);
+out:
+    return;
+}
diff --git a/xlators/features/bit-rot/src/stub/bit-rot-stub-mem-types.h b/xlators/features/bit-rot/src/stub/bit-rot-stub-mem-types.h
index 492278639b4..9d93caf069f 100644
--- a/xlators/features/bit-rot/src/stub/bit-rot-stub-mem-types.h
+++ b/xlators/features/bit-rot/src/stub/bit-rot-stub-mem-types.h
@@ -11,18 +11,26 @@
 #ifndef _BR_MEM_TYPES_H
 #define _BR_MEM_TYPES_H
 
-#include "mem-types.h"
+#include <glusterfs/mem-types.h>
 
 enum br_mem_types {
-        gf_br_stub_mt_private_t   = gf_common_mt_end + 1,
-        gf_br_stub_mt_version_t,
-        gf_br_stub_mt_inode_ctx_t,
-        gf_br_stub_mt_signature_t,
-        gf_br_mt_br_private_t,
-        gf_br_mt_br_child_t,
-        gf_br_mt_br_object_t,
-        gf_br_mt_br_ob_n_wk_t,
-        gf_br_stub_mt_end
+    gf_br_stub_mt_private_t = gf_common_mt_end + 1,
+    gf_br_stub_mt_version_t,
+    gf_br_stub_mt_inode_ctx_t,
+    gf_br_stub_mt_signature_t,
+    gf_br_mt_br_private_t,
+    gf_br_mt_br_child_t,
+    gf_br_mt_br_object_t,
+    gf_br_mt_br_ob_n_wk_t,
+    gf_br_mt_br_scrubber_t,
+    gf_br_mt_br_fsscan_entry_t,
+    gf_br_stub_mt_br_stub_fd_t,
+    gf_br_stub_mt_br_scanner_freq_t,
+    gf_br_stub_mt_sigstub_t,
+    gf_br_mt_br_child_event_t,
+    gf_br_stub_mt_misc,
+    gf_br_mt_br_worker_t,
+    gf_br_stub_mt_end,
 };
 
 #endif
diff --git a/xlators/features/bit-rot/src/stub/bit-rot-stub-messages.h b/xlators/features/bit-rot/src/stub/bit-rot-stub-messages.h
new file mode 100644
index 00000000000..6c15a166f18
--- /dev/null
+++ b/xlators/features/bit-rot/src/stub/bit-rot-stub-messages.h
@@ -0,0 +1,117 @@
+/*
+ Copyright (c) 2015 Red Hat, Inc. <http://www.redhat.com>
+ This file is part of GlusterFS.
+
+ This file is licensed to you under your choice of the GNU Lesser
+ General Public License, version 3 or any later version (LGPLv3 or
+ later), or the GNU General Public License, version 2 (GPLv2), in all
+ cases as published by the Free Software Foundation.
+ */
+
+#ifndef _BITROT_STUB_MESSAGES_H_
+#define _BITROT_STUB_MESSAGES_H_
+
+#include <glusterfs/glfs-message-id.h>
+
+/* To add new message IDs, append new identifiers at the end of the list.
+ *
+ * Never remove a message ID. If it's not used anymore, you can rename it or
+ * leave it as it is, but not delete it. This is to prevent reutilization of
+ * IDs by other messages.
+ *
+ * The component name must match one of the entries defined in
+ * glfs-message-id.h.
+ */
+
+GLFS_MSGID(BITROT_STUB, BRS_MSG_NO_MEMORY, BRS_MSG_SET_EVENT_FAILED,
+           BRS_MSG_MEM_ACNT_FAILED, BRS_MSG_CREATE_FRAME_FAILED,
+           BRS_MSG_SET_CONTEXT_FAILED, BRS_MSG_CHANGE_VERSION_FAILED,
+           BRS_MSG_ADD_FD_TO_LIST_FAILED, BRS_MSG_SET_FD_CONTEXT_FAILED,
+           BRS_MSG_CREATE_ANONYMOUS_FD_FAILED, BRS_MSG_NO_CHILD,
+           BRS_MSG_STUB_ALLOC_FAILED, BRS_MSG_GET_INODE_CONTEXT_FAILED,
+           BRS_MSG_CANCEL_SIGN_THREAD_FAILED, BRS_MSG_ADD_FD_TO_INODE,
+           BRS_MSG_SIGN_VERSION_ERROR, BRS_MSG_BAD_OBJ_MARK_FAIL,
+           BRS_MSG_NON_SCRUB_BAD_OBJ_MARK, BRS_MSG_REMOVE_INTERNAL_XATTR,
+           BRS_MSG_SET_INTERNAL_XATTR, BRS_MSG_BAD_OBJECT_ACCESS,
+           BRS_MSG_BAD_CONTAINER_FAIL, BRS_MSG_BAD_OBJECT_DIR_FAIL,
+           BRS_MSG_BAD_OBJECT_DIR_SEEK_FAIL, BRS_MSG_BAD_OBJECT_DIR_TELL_FAIL,
+           BRS_MSG_BAD_OBJECT_DIR_READ_FAIL, BRS_MSG_GET_FD_CONTEXT_FAILED,
+           BRS_MSG_BAD_HANDLE_DIR_NULL, BRS_MSG_BAD_OBJ_THREAD_FAIL,
+           BRS_MSG_BAD_OBJ_DIR_CLOSE_FAIL, BRS_MSG_LINK_FAIL,
+           BRS_MSG_BAD_OBJ_UNLINK_FAIL, BRS_MSG_DICT_SET_FAILED,
+           BRS_MSG_PATH_GET_FAILED, BRS_MSG_NULL_LOCAL,
+           BRS_MSG_SPAWN_SIGN_THRD_FAILED, BRS_MSG_KILL_SIGN_THREAD,
+           BRS_MSG_NON_BITD_PID, BRS_MSG_SIGN_PREPARE_FAIL,
+           BRS_MSG_USING_DEFAULT_THREAD_SIZE, BRS_MSG_ALLOC_MEM_FAILED,
+           BRS_MSG_DICT_ALLOC_FAILED, BRS_MSG_CREATE_GF_DIRENT_FAILED,
+           BRS_MSG_ALLOC_FAILED, BRS_MSG_PATH_XATTR_GET_FAILED,
+           BRS_MSG_VERSION_PREPARE_FAIL);
+
+#define BRS_MSG_MEM_ACNT_FAILED_STR "Memory accounting init failed"
+#define BRS_MSG_BAD_OBJ_THREAD_FAIL_STR "pthread_init failed"
+#define BRS_MSG_USING_DEFAULT_THREAD_SIZE_STR "Using default thread stack size"
+#define BRS_MSG_NO_CHILD_STR "FATAL: no children"
+#define BRS_MSG_SPAWN_SIGN_THRD_FAILED_STR                                     \
+    "failed to create the new thread for signer"
+#define BRS_MSG_BAD_CONTAINER_FAIL_STR                                         \
+    "failed to launch the thread for storing bad gfids"
+#define BRS_MSG_CANCEL_SIGN_THREAD_FAILED_STR                                  \
+    "Could not cancel sign serializer thread"
+#define BRS_MSG_KILL_SIGN_THREAD_STR "killed the signer thread"
+#define BRS_MSG_GET_INODE_CONTEXT_FAILED_STR                                   \
+    "failed to init the inode context for the inode"
+#define BRS_MSG_ADD_FD_TO_INODE_STR "failed to add fd to the inode"
+#define BRS_MSG_NO_MEMORY_STR "local allocation failed"
+#define BRS_MSG_BAD_OBJECT_ACCESS_STR "bad object accessed. Returning"
+#define BRS_MSG_SIGN_VERSION_ERROR_STR "Signing version exceeds current version"
+#define BRS_MSG_NON_BITD_PID_STR                                               \
+    "PID from where signature request came, does not belong to bit-rot "       \
+    "daemon. Unwinding the fop"
+#define BRS_MSG_SIGN_PREPARE_FAIL_STR                                          \
+    "failed to prepare the signature. Unwinding the fop"
+#define BRS_MSG_VERSION_PREPARE_FAIL_STR                                       \
+    "failed to prepare the version. Unwinding the fop"
+#define BRS_MSG_STUB_ALLOC_FAILED_STR "failed to allocate stub fop, Unwinding"
+#define BRS_MSG_BAD_OBJ_MARK_FAIL_STR "failed to mark object as bad"
+#define BRS_MSG_NON_SCRUB_BAD_OBJ_MARK_STR                                     \
+    "bad object marking is not from the scrubber"
+#define BRS_MSG_ALLOC_MEM_FAILED_STR "failed to allocate memory"
+#define BRS_MSG_SET_INTERNAL_XATTR_STR "called on the internal xattr"
+#define BRS_MSG_REMOVE_INTERNAL_XATTR_STR "removexattr called on internal xattr"
+#define BRS_MSG_CREATE_ANONYMOUS_FD_FAILED_STR                                 \
+    "failed to create anonymous fd for the inode"
+#define BRS_MSG_ADD_FD_TO_LIST_FAILED_STR "failed add fd to the list"
+#define BRS_MSG_SET_FD_CONTEXT_FAILED_STR                                      \
+    "failed to set the fd context for the file"
+#define BRS_MSG_NULL_LOCAL_STR "local is NULL"
+#define BRS_MSG_DICT_ALLOC_FAILED_STR                                          \
+    "dict allocation failed: cannot send IPC FOP to changelog"
+#define BRS_MSG_SET_EVENT_FAILED_STR "cannot set release event in dict"
+#define BRS_MSG_CREATE_FRAME_FAILED_STR "create_frame() failure"
+#define BRS_MSG_BAD_OBJ_DIR_CLOSE_FAIL_STR "closedir error"
+#define BRS_MSG_LINK_FAIL_STR "failed to record gfid"
+#define BRS_MSG_BAD_OBJ_UNLINK_FAIL_STR                                        \
+    "failed to delete bad object link from quaratine directory"
+#define BRS_MSG_BAD_OBJECT_DIR_FAIL_STR "failed stub directory"
+#define BRS_MSG_BAD_OBJECT_DIR_SEEK_FAIL_STR                                   \
+    "seekdir failed. Invalid argument (offset reused from another DIR * "      \
+    "structure)"
+#define BRS_MSG_BAD_OBJECT_DIR_TELL_FAIL_STR "telldir failed on dir"
+#define BRS_MSG_BAD_OBJECT_DIR_READ_FAIL_STR "readdir failed on dir"
+#define BRS_MSG_CREATE_GF_DIRENT_FAILED_STR "could not create gf_dirent"
+#define BRS_MSG_GET_FD_CONTEXT_FAILED_STR "pfd is NULL"
+#define BRS_MSG_BAD_HANDLE_DIR_NULL_STR "dir if NULL"
+#define BRS_MSG_ALLOC_FAILED_STR                                               \
+    "failed to allocate new dict for saving the paths of the corrupted "       \
+    "objects. Scrub status will only display the gfid"
+#define BRS_MSG_PATH_GET_FAILED_STR "failed to get the path"
+#define BRS_MSG_PATH_XATTR_GET_FAILED_STR                                      \
+    "failed to get the path xattr from disk for the gfid. Trying to get path " \
+    "from the memory"
+#define BRS_MSG_DICT_SET_FAILED_STR                                            \
+    "failed to set the actual path as the value in the dict for the "          \
+    "corrupted object"
+#define BRS_MSG_SET_CONTEXT_FAILED_STR                                         \
+    "could not set fd context for release callback"
+#define BRS_MSG_CHANGE_VERSION_FAILED_STR "change version failed"
+#endif /* !_BITROT_STUB_MESSAGES_H_ */
diff --git a/xlators/features/bit-rot/src/stub/bit-rot-stub.c b/xlators/features/bit-rot/src/stub/bit-rot-stub.c
index 0db500659b5..447dd47ff41 100644
--- a/xlators/features/bit-rot/src/stub/bit-rot-stub.c
+++ b/xlators/features/bit-rot/src/stub/bit-rot-stub.c
@@ -10,174 +10,433 @@
 
 #include <ctype.h>
 #include <sys/uio.h>
+#include <signal.h>
 
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
-
-#include "glusterfs.h"
-#include "xlator.h"
-#include "logging.h"
+#include <glusterfs/glusterfs.h>
+#include <glusterfs/logging.h>
 #include "changelog.h"
-#include "compat-errno.h"
+#include <glusterfs/compat-errno.h>
+#include <glusterfs/call-stub.h>
 
 #include "bit-rot-stub.h"
 #include "bit-rot-stub-mem-types.h"
-
+#include "bit-rot-stub-messages.h"
 #include "bit-rot-common.h"
 
-#define BR_STUB_REQUEST_COOKIE  0x1
+#define BR_STUB_REQUEST_COOKIE 0x1
 
-int32_t
-mem_acct_init (xlator_t *this)
+void
+br_stub_lock_cleaner(void *arg)
 {
-        int32_t ret = -1;
+    pthread_mutex_t *clean_mutex = arg;
 
-        if (!this)
-                return ret;
+    pthread_mutex_unlock(clean_mutex);
+    return;
+}
 
-        ret = xlator_mem_acct_init (this, gf_br_stub_mt_end + 1);
+void *
+br_stub_signth(void *);
 
-        if (ret != 0) {
-                gf_log (this->name, GF_LOG_WARNING, "Memory accounting"
-                        " init failed");
-                return ret;
-        }
+struct br_stub_signentry {
+    unsigned long v;
 
+    call_stub_t *stub;
+
+    struct list_head list;
+};
+
+int32_t
+mem_acct_init(xlator_t *this)
+{
+    int32_t ret = -1;
+
+    if (!this)
         return ret;
+
+    ret = xlator_mem_acct_init(this, gf_br_stub_mt_end + 1);
+
+    if (ret != 0) {
+        gf_smsg(this->name, GF_LOG_WARNING, 0, BRS_MSG_MEM_ACNT_FAILED, NULL);
+        return ret;
+    }
+
+    return ret;
+}
+
+int
+br_stub_bad_object_container_init(xlator_t *this, br_stub_private_t *priv)
+{
+    pthread_attr_t w_attr;
+    int ret = -1;
+
+    ret = pthread_cond_init(&priv->container.bad_cond, NULL);
+    if (ret != 0) {
+        gf_smsg(this->name, GF_LOG_ERROR, 0, BRS_MSG_BAD_OBJ_THREAD_FAIL,
+                "cond_init ret=%d", ret, NULL);
+        goto out;
+    }
+
+    ret = pthread_mutex_init(&priv->container.bad_lock, NULL);
+    if (ret != 0) {
+        gf_smsg(this->name, GF_LOG_ERROR, 0, BRS_MSG_BAD_OBJ_THREAD_FAIL,
+                "mutex_init ret=%d", ret, NULL);
+        goto cleanup_cond;
+    }
+
+    ret = pthread_attr_init(&w_attr);
+    if (ret != 0) {
+        gf_smsg(this->name, GF_LOG_ERROR, 0, BRS_MSG_BAD_OBJ_THREAD_FAIL,
+                "attr_init ret=%d", ret, NULL);
+        goto cleanup_lock;
+    }
+
+    ret = pthread_attr_setstacksize(&w_attr, BAD_OBJECT_THREAD_STACK_SIZE);
+    if (ret == EINVAL) {
+        gf_smsg(this->name, GF_LOG_WARNING, 0,
+                BRS_MSG_USING_DEFAULT_THREAD_SIZE, NULL);
+    }
+
+    INIT_LIST_HEAD(&priv->container.bad_queue);
+    ret = br_stub_dir_create(this, priv);
+    if (ret < 0)
+        goto cleanup_lock;
+
+    ret = gf_thread_create(&priv->container.thread, &w_attr, br_stub_worker,
+                           this, "brswrker");
+    if (ret)
+        goto cleanup_attr;
+
+    return 0;
+
+cleanup_attr:
+    pthread_attr_destroy(&w_attr);
+cleanup_lock:
+    pthread_mutex_destroy(&priv->container.bad_lock);
+cleanup_cond:
+    pthread_cond_destroy(&priv->container.bad_cond);
+out:
+    return -1;
 }
 
 int32_t
-init (xlator_t *this)
+init(xlator_t *this)
 {
-        char *tmp = NULL;
-        struct timeval tv = {0,};
-	br_stub_private_t *priv = NULL;
+    int ret = 0;
+    char *tmp = NULL;
+    struct timeval tv = {
+        0,
+    };
+    br_stub_private_t *priv = NULL;
 
-	if (!this->children) {
-		gf_log (this->name, GF_LOG_ERROR, "FATAL: no children");
-		goto error_return;
-	}
+    if (!this->children) {
+        gf_smsg(this->name, GF_LOG_ERROR, 0, BRS_MSG_NO_CHILD, NULL);
+        goto error_return;
+    }
 
-        priv = GF_CALLOC (1, sizeof (*priv), gf_br_stub_mt_private_t);
-        if (!priv)
-                goto error_return;
+    priv = GF_CALLOC(1, sizeof(*priv), gf_br_stub_mt_private_t);
+    if (!priv)
+        goto error_return;
 
-        priv->local_pool = mem_pool_new (br_stub_local_t, 512);
-        if (!priv->local_pool)
-                goto free_priv;
+    priv->local_pool = mem_pool_new(br_stub_local_t, 512);
+    if (!priv->local_pool)
+        goto free_priv;
 
-        GF_OPTION_INIT ("bitrot", priv->go, bool, free_mempool);
+    GF_OPTION_INIT("bitrot", priv->do_versioning, bool, free_mempool);
 
-        GF_OPTION_INIT ("export", tmp, str, free_mempool);
-        memcpy (priv->export, tmp, strlen (tmp) + 1);
+    GF_OPTION_INIT("export", tmp, str, free_mempool);
 
-        (void) gettimeofday (&tv, NULL);
+    if (snprintf(priv->export, PATH_MAX, "%s", tmp) >= PATH_MAX)
+        goto free_mempool;
 
-        /* boot time is in network endian format */
-        priv->boot[0] = htonl (tv.tv_sec);
-        priv->boot[1] = htonl (tv.tv_usec);
+    if (snprintf(priv->stub_basepath, sizeof(priv->stub_basepath), "%s/%s",
+                 priv->export,
+                 BR_STUB_QUARANTINE_DIR) >= sizeof(priv->stub_basepath))
+        goto free_mempool;
 
-        gf_log (this->name, GF_LOG_DEBUG, "bit-rot stub loaded");
-	this->private = priv;
-        return 0;
+    (void)gettimeofday(&tv, NULL);
 
- free_mempool:
-        mem_pool_destroy (priv->local_pool);
- free_priv:
-        GF_FREE (priv);
- error_return:
-        return -1;
-}
+    /* boot time is in network endian format */
+    priv->boot[0] = htonl(tv.tv_sec);
+    priv->boot[1] = htonl(tv.tv_usec);
 
-void
-fini (xlator_t *this)
-{
-	br_stub_private_t *priv = this->private;
+    pthread_mutex_init(&priv->lock, NULL);
+    pthread_cond_init(&priv->cond, NULL);
+    INIT_LIST_HEAD(&priv->squeue);
 
-        if (!priv)
-                return;
-        this->private = NULL;
-	GF_FREE (priv);
+    /* Thread creations need 'this' to be passed so that THIS can be
+     * assigned inside the thread. So setting this->private here.
+     */
+    this->private = priv;
+    if (!priv->do_versioning)
+        return 0;
 
-	return;
+    ret = gf_thread_create(&priv->signth, NULL, br_stub_signth, this,
+                           "brssign");
+    if (ret != 0) {
+        gf_smsg(this->name, GF_LOG_WARNING, 0, BRS_MSG_SPAWN_SIGN_THRD_FAILED,
+                NULL);
+        goto cleanup_lock;
+    }
+
+    ret = br_stub_bad_object_container_init(this, priv);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_ERROR, 0, BRS_MSG_BAD_CONTAINER_FAIL, NULL);
+        goto cleanup_lock;
+    }
+
+    gf_msg_debug(this->name, 0, "bit-rot stub loaded");
+
+    return 0;
+
+cleanup_lock:
+    pthread_cond_destroy(&priv->cond);
+    pthread_mutex_destroy(&priv->lock);
+free_mempool:
+    mem_pool_destroy(priv->local_pool);
+    priv->local_pool = NULL;
+free_priv:
+    GF_FREE(priv);
+    this->private = NULL;
+error_return:
+    return -1;
 }
 
-static inline int
-br_stub_alloc_versions (br_version_t **obuf,
-                        br_signature_t **sbuf, size_t signaturelen)
+/* TODO:
+ * As of now enabling bitrot option does 2 things.
+ * 1) Start the Bitrot Daemon which signs the objects (currently files only)
+ *    upon getting notified by the stub.
+ * 2) Enable versioning of the objects. Object versions (again files only) are
+ *    incremented upon modification.
+ * So object versioning is tied to bitrot daemon's signing. In future, object
+ * versioning might be necessary for other things as well apart from bit-rot
+ * detection (well that's the objective of bringing in object-versioning :)).
+ * In that case, better to make versioning a new option and letting it to be
+ * enabled despite bit-rot detection is not needed.
+ * Ex: ICAP.
+ */
+int32_t
+reconfigure(xlator_t *this, dict_t *options)
 {
-        void *mem = NULL;
-        size_t size = 0;
+    int32_t ret = -1;
+    br_stub_private_t *priv = NULL;
 
-        if (obuf)
-                size += sizeof (br_version_t);
-        if (sbuf)
-                size += sizeof (br_signature_t) + signaturelen;
+    priv = this->private;
 
-        mem = GF_CALLOC (1, size, gf_br_stub_mt_version_t);
-        if (!mem)
-                goto error_return;
+    GF_OPTION_RECONF("bitrot", priv->do_versioning, options, bool, err);
+    if (priv->do_versioning && !priv->signth) {
+        ret = gf_thread_create(&priv->signth, NULL, br_stub_signth, this,
+                               "brssign");
+        if (ret != 0) {
+            gf_smsg(this->name, GF_LOG_WARNING, 0,
+                    BRS_MSG_SPAWN_SIGN_THRD_FAILED, NULL);
+            goto err;
+        }
 
-        if (obuf) {
-                *obuf = (br_version_t *)mem;
-                mem = ((char *)mem + sizeof (br_version_t));
+        ret = br_stub_bad_object_container_init(this, priv);
+        if (ret) {
+            gf_smsg(this->name, GF_LOG_ERROR, 0, BRS_MSG_BAD_CONTAINER_FAIL,
+                    NULL);
+            goto err;
         }
-        if (sbuf) {
-                *sbuf = (br_signature_t *)mem;
+    } else {
+        if (priv->signth) {
+            if (gf_thread_cleanup_xint(priv->signth)) {
+                gf_smsg(this->name, GF_LOG_ERROR, 0,
+                        BRS_MSG_CANCEL_SIGN_THREAD_FAILED, NULL);
+            } else {
+                gf_smsg(this->name, GF_LOG_INFO, 0, BRS_MSG_KILL_SIGN_THREAD,
+                        NULL);
+                priv->signth = 0;
+            }
         }
 
+        if (priv->container.thread) {
+            if (gf_thread_cleanup_xint(priv->container.thread)) {
+                gf_smsg(this->name, GF_LOG_ERROR, 0,
+                        BRS_MSG_CANCEL_SIGN_THREAD_FAILED, NULL);
+            }
+            priv->container.thread = 0;
+        }
+    }
+
+    ret = 0;
+    return ret;
+err:
+    if (priv->signth) {
+        if (gf_thread_cleanup_xint(priv->signth)) {
+            gf_smsg(this->name, GF_LOG_ERROR, 0,
+                    BRS_MSG_CANCEL_SIGN_THREAD_FAILED, NULL);
+        }
+        priv->signth = 0;
+    }
+
+    if (priv->container.thread) {
+        if (gf_thread_cleanup_xint(priv->container.thread)) {
+            gf_smsg(this->name, GF_LOG_ERROR, 0,
+                    BRS_MSG_CANCEL_SIGN_THREAD_FAILED, NULL);
+        }
+        priv->container.thread = 0;
+    }
+    ret = -1;
+    return ret;
+}
+
+int
+notify(xlator_t *this, int event, void *data, ...)
+{
+    br_stub_private_t *priv = NULL;
+
+    if (!this)
         return 0;
 
- error_return:
-        return -1;
+    priv = this->private;
+    if (!priv)
+        return 0;
+
+    default_notify(this, event, data);
+    return 0;
 }
 
-static inline void
-br_stub_dealloc_versions (void *mem)
+void
+fini(xlator_t *this)
 {
-        GF_FREE (mem);
+    int32_t ret = 0;
+    br_stub_private_t *priv = this->private;
+    struct br_stub_signentry *sigstub = NULL;
+    call_stub_t *stub = NULL;
+
+    if (!priv)
+        return;
+
+    if (!priv->do_versioning)
+        goto cleanup;
+
+    ret = gf_thread_cleanup_xint(priv->signth);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_ERROR, 0, BRS_MSG_CANCEL_SIGN_THREAD_FAILED,
+                NULL);
+        goto out;
+    }
+    priv->signth = 0;
+
+    while (!list_empty(&priv->squeue)) {
+        sigstub = list_first_entry(&priv->squeue, struct br_stub_signentry,
+                                   list);
+        list_del_init(&sigstub->list);
+
+        call_stub_destroy(sigstub->stub);
+        GF_FREE(sigstub);
+    }
+
+    ret = gf_thread_cleanup_xint(priv->container.thread);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_ERROR, 0, BRS_MSG_CANCEL_SIGN_THREAD_FAILED,
+                NULL);
+        goto out;
+    }
+
+    priv->container.thread = 0;
+
+    while (!list_empty(&priv->container.bad_queue)) {
+        stub = list_first_entry(&priv->container.bad_queue, call_stub_t, list);
+        list_del_init(&stub->list);
+        call_stub_destroy(stub);
+    }
+
+    pthread_mutex_destroy(&priv->container.bad_lock);
+    pthread_cond_destroy(&priv->container.bad_cond);
+
+cleanup:
+    pthread_mutex_destroy(&priv->lock);
+    pthread_cond_destroy(&priv->cond);
+
+    if (priv->local_pool) {
+        mem_pool_destroy(priv->local_pool);
+        priv->local_pool = NULL;
+    }
+
+    this->private = NULL;
+    GF_FREE(priv);
+
+out:
+    return;
 }
 
-static inline br_stub_local_t *
-br_stub_alloc_local (xlator_t *this)
+static int
+br_stub_alloc_versions(br_version_t **obuf, br_signature_t **sbuf,
+                       size_t signaturelen)
 {
-        br_stub_private_t *priv = this->private;
+    void *mem = NULL;
+    size_t size = 0;
+
+    if (obuf)
+        size += sizeof(br_version_t);
+    if (sbuf)
+        size += sizeof(br_signature_t) + signaturelen;
+
+    mem = GF_CALLOC(1, size, gf_br_stub_mt_version_t);
+    if (!mem)
+        goto error_return;
+
+    if (obuf) {
+        *obuf = (br_version_t *)mem;
+        mem = ((char *)mem + sizeof(br_version_t));
+    }
+    if (sbuf) {
+        *sbuf = (br_signature_t *)mem;
+    }
+
+    return 0;
+
+error_return:
+    return -1;
+}
 
-        return mem_get0 (priv->local_pool);
+static void
+br_stub_dealloc_versions(void *mem)
+{
+    GF_FREE(mem);
 }
 
-static inline void
-br_stub_dealloc_local (br_stub_local_t *ptr)
+static br_stub_local_t *
+br_stub_alloc_local(xlator_t *this)
 {
-        mem_put (ptr);
+    br_stub_private_t *priv = this->private;
+
+    return mem_get0(priv->local_pool);
 }
 
-static inline int
-br_stub_prepare_version_request (xlator_t *this, dict_t *dict,
+static void
+br_stub_dealloc_local(br_stub_local_t *ptr)
+{
+    if (!ptr)
+        return;
+
+    mem_put(ptr);
+}
+
+static int
+br_stub_prepare_version_request(xlator_t *this, dict_t *dict,
                                 br_version_t *obuf, unsigned long oversion)
 {
-        br_stub_private_t *priv = NULL;
+    br_stub_private_t *priv = NULL;
 
-        priv = this->private;
-        br_set_ongoingversion (obuf, oversion, priv->boot);
+    priv = this->private;
+    br_set_ongoingversion(obuf, oversion, priv->boot);
 
-        return dict_set_static_bin (dict, BITROT_CURRENT_VERSION_KEY,
-                                    (void *)obuf, sizeof (br_version_t));
+    return dict_set_bin(dict, BITROT_CURRENT_VERSION_KEY, (void *)obuf,
+                        sizeof(br_version_t));
 }
 
-static inline int
-br_stub_prepare_signing_request (dict_t *dict,
-                                 br_signature_t *sbuf,
-                                 br_isignature_t *sign, size_t signaturelen)
+static int
+br_stub_prepare_signing_request(dict_t *dict, br_signature_t *sbuf,
+                                br_isignature_t *sign, size_t signaturelen)
 {
-        size_t size = 0;
+    size_t size = 0;
 
-        br_set_signature (sbuf, sign, signaturelen, &size);
+    br_set_signature(sbuf, sign, signaturelen, &size);
 
-        return dict_set_static_bin (dict, BITROT_SIGNING_VERSION_KEY,
-                                    (void *)sbuf, size);
+    return dict_set_bin(dict, BITROT_SIGNING_VERSION_KEY, (void *)sbuf, size);
 }
 
 /**
@@ -186,1038 +445,2831 @@ br_stub_prepare_signing_request (dict_t *dict,
  * context, hence the inode is marked dirty. this routine also
  * initializes the transient inode version.
  */
-static inline int
-br_stub_init_inode_versions (xlator_t *this, fd_t *fd, inode_t *inode,
-                             unsigned long version, gf_boolean_t markdirty)
-{
-        int32_t ret = 0;
-        br_stub_inode_ctx_t *ctx = NULL;
-
-        ctx = GF_CALLOC (1, sizeof (br_stub_inode_ctx_t),
-                         gf_br_stub_mt_inode_ctx_t);
-        if (!ctx)
-                goto error_return;
-
-        (markdirty) ? __br_stub_mark_inode_dirty (ctx)
-                : __br_stub_mark_inode_synced (ctx);
-        __br_stub_set_ongoing_version (ctx, version);
-        __br_stub_reset_release_counters (ctx);
-
-        if (fd) {
-                br_stub_require_release_call (this, fd);
-                __br_stub_track_openfd (fd, ctx);
-        }
-        ret = br_stub_set_inode_ctx (this, inode, ctx);
+static int
+br_stub_init_inode_versions(xlator_t *this, fd_t *fd, inode_t *inode,
+                            unsigned long version, gf_boolean_t markdirty,
+                            gf_boolean_t bad_object, uint64_t *ctx_addr)
+{
+    int32_t ret = 0;
+    br_stub_inode_ctx_t *ctx = NULL;
+
+    ctx = GF_CALLOC(1, sizeof(br_stub_inode_ctx_t), gf_br_stub_mt_inode_ctx_t);
+    if (!ctx)
+        goto error_return;
+
+    INIT_LIST_HEAD(&ctx->fd_list);
+    (markdirty) ? __br_stub_mark_inode_dirty(ctx)
+                : __br_stub_mark_inode_synced(ctx);
+    __br_stub_set_ongoing_version(ctx, version);
+
+    if (bad_object)
+        __br_stub_mark_object_bad(ctx);
+
+    if (fd) {
+        ret = br_stub_add_fd_to_inode(this, fd, ctx);
         if (ret)
-                goto free_ctx;
-        return 0;
+            goto free_ctx;
+    }
 
- free_ctx:
-        GF_FREE (ctx);
- error_return:
-        return -1;
+    ret = br_stub_set_inode_ctx(this, inode, ctx);
+    if (ret)
+        goto free_ctx;
+
+    if (ctx_addr)
+        *ctx_addr = (uint64_t)(uintptr_t)ctx;
+    return 0;
+
+free_ctx:
+    GF_FREE(ctx);
+error_return:
+    return -1;
 }
 
 /**
  * modify the ongoing version of an inode.
  */
-static inline int
-br_stub_mod_inode_versions (xlator_t *this,
-                            fd_t *fd, inode_t *inode, unsigned long version)
+static int
+br_stub_mod_inode_versions(xlator_t *this, fd_t *fd, inode_t *inode,
+                           unsigned long version)
 {
-        int32_t ret = -1;
-        br_stub_inode_ctx_t *ctx = 0;
-
-        LOCK (&inode->lock);
-        {
-                ctx = __br_stub_get_ongoing_version_ctx (this, inode, NULL);
-                if (ctx == NULL)
-                        goto unblock;
-                if (__br_stub_is_inode_dirty (ctx)) {
-                        __br_stub_set_ongoing_version (ctx, version);
-                        __br_stub_mark_inode_synced (ctx);
-                }
-
-                __br_stub_track_openfd (fd, ctx);
-                ret = 0;
+    int32_t ret = -1;
+    br_stub_inode_ctx_t *ctx = 0;
+
+    LOCK(&inode->lock);
+    {
+        ctx = __br_stub_get_ongoing_version_ctx(this, inode, NULL);
+        if (ctx == NULL)
+            goto unblock;
+        if (__br_stub_is_inode_dirty(ctx)) {
+            __br_stub_set_ongoing_version(ctx, version);
+            __br_stub_mark_inode_synced(ctx);
         }
- unblock:
-        UNLOCK (&inode->lock);
 
-        return ret;
+        ret = 0;
+    }
+unblock:
+    UNLOCK(&inode->lock);
+
+    return ret;
 }
 
-static inline void
-br_stub_fill_local (br_stub_local_t *local,
-                    call_stub_t *stub, fd_t *fd, inode_t *inode, uuid_t gfid,
-                    int versioningtype, unsigned long memversion, int dirty)
+static void
+br_stub_fill_local(br_stub_local_t *local, call_stub_t *stub, fd_t *fd,
+                   inode_t *inode, uuid_t gfid, int versioningtype,
+                   unsigned long memversion)
 {
-        local->fopstub = stub;
-        local->versioningtype = versioningtype;
-        local->u.context.version = memversion;
-        if (fd)
-                local->u.context.fd = fd_ref (fd);
-        if (inode)
-                local->u.context.inode = inode_ref (inode);
-        gf_uuid_copy (local->u.context.gfid, gfid);
+    local->fopstub = stub;
+    local->versioningtype = versioningtype;
+    local->u.context.version = memversion;
+    if (fd)
+        local->u.context.fd = fd_ref(fd);
+    if (inode)
+        local->u.context.inode = inode_ref(inode);
+    gf_uuid_copy(local->u.context.gfid, gfid);
+}
 
-        /* mark inode dirty/fresh according to durability */
-        local->u.context.markdirty = (dirty) ? _gf_true : _gf_false;
+static void
+br_stub_cleanup_local(br_stub_local_t *local)
+{
+    if (!local)
+        return;
+
+    local->fopstub = NULL;
+    local->versioningtype = 0;
+    local->u.context.version = 0;
+    if (local->u.context.fd) {
+        fd_unref(local->u.context.fd);
+        local->u.context.fd = NULL;
+    }
+    if (local->u.context.inode) {
+        inode_unref(local->u.context.inode);
+        local->u.context.inode = NULL;
+    }
+    memset(local->u.context.gfid, '\0', sizeof(uuid_t));
 }
 
-static inline void
-br_stub_cleanup_local (br_stub_local_t *local)
+static int
+br_stub_need_versioning(xlator_t *this, fd_t *fd, gf_boolean_t *versioning,
+                        gf_boolean_t *modified, br_stub_inode_ctx_t **ctx)
 {
-        local->fopstub = NULL;
-        local->versioningtype = 0;
-        local->u.context.version = 0;
-        if (local->u.context.fd) {
-                fd_unref (local->u.context.fd);
-                local->u.context.fd = NULL;
-        }
-        if (local->u.context.inode) {
-                inode_unref (local->u.context.inode);
-                local->u.context.inode = NULL;
+    int32_t ret = -1;
+    uint64_t ctx_addr = 0;
+    br_stub_inode_ctx_t *c = NULL;
+    unsigned long version = BITROT_DEFAULT_CURRENT_VERSION;
+
+    *versioning = _gf_false;
+    *modified = _gf_false;
+
+    /* Bitrot stub inode context was initialized only in lookup, create
+     * and mknod cbk path. Object versioning was enabled by default
+     * irrespective of bitrot enabled or not. But it's made optional now.
+     * As a consequence there could be cases where getting inode ctx would
+     * fail because it's not set yet.
+     * e.g., If versioning (with bitrot enable) is enabled while I/O is
+     * happening, it could directly get other fops like writev without
+     * lookup, where getting inode ctx would fail. Hence initialize the
+     * inode ctx on failure to get ctx. This is done in all places where
+     * applicable.
+     */
+    ret = br_stub_get_inode_ctx(this, fd->inode, &ctx_addr);
+    if (ret < 0) {
+        ret = br_stub_init_inode_versions(this, fd, fd->inode, version,
+                                          _gf_true, _gf_false, &ctx_addr);
+        if (ret) {
+            gf_smsg(this->name, GF_LOG_ERROR, 0,
+                    BRS_MSG_GET_INODE_CONTEXT_FAILED, "gfid=%s",
+                    uuid_utoa(fd->inode->gfid), NULL);
+            goto error_return;
         }
-        local->u.context.markdirty = _gf_true;
-        memset (local->u.context.gfid, '\0', sizeof (uuid_t));
+    }
+
+    c = (br_stub_inode_ctx_t *)(long)ctx_addr;
+
+    LOCK(&fd->inode->lock);
+    {
+        if (__br_stub_is_inode_dirty(c))
+            *versioning = _gf_true;
+        if (__br_stub_is_inode_modified(c))
+            *modified = _gf_true;
+    }
+    UNLOCK(&fd->inode->lock);
+
+    if (ctx)
+        *ctx = c;
+    return 0;
+
+error_return:
+    return -1;
 }
 
-/**
- * callback for inode/fd full versioning
- */
-int
-br_stub_inode_fullversioning_cbk (call_frame_t *frame,
-                                  void *cookie, xlator_t *this,
-                                  int op_ret, int op_errno, dict_t *xdata)
-{
-        fd_t            *fd      = NULL;
-        inode_t         *inode   = NULL;
-        unsigned long    version = 0;
-        gf_boolean_t     dirty   = _gf_true;
-        br_stub_local_t *local   = NULL;
-
-        local = (br_stub_local_t *)frame->local;
-
-        /* be graceful to EEXIST */
-        if ((op_ret < 0) && (op_errno == EEXIST)) {
-                op_ret = 0;
-                goto done;
+static int32_t
+br_stub_anon_fd_ctx(xlator_t *this, fd_t *fd, br_stub_inode_ctx_t *ctx)
+{
+    int32_t ret = -1;
+    br_stub_fd_t *br_stub_fd = NULL;
+
+    br_stub_fd = br_stub_fd_ctx_get(this, fd);
+    if (!br_stub_fd) {
+        ret = br_stub_add_fd_to_inode(this, fd, ctx);
+        if (ret) {
+            gf_smsg(this->name, GF_LOG_ERROR, 0, BRS_MSG_ADD_FD_TO_INODE,
+                    "gfid=%s", uuid_utoa(fd->inode->gfid), NULL);
+            goto out;
         }
+    }
 
-        if (op_ret < 0)
-                goto done;
+    ret = 0;
 
-        fd      = local->u.context.fd;
-        inode   = local->u.context.inode;
-        version = local->u.context.version;
-        dirty   = local->u.context.markdirty;
+out:
+    return ret;
+}
 
-        op_ret = br_stub_init_inode_versions (this, fd, inode, version, dirty);
-        if (op_ret < 0)
-                op_errno = EINVAL;
+static int
+br_stub_versioning_prep(call_frame_t *frame, xlator_t *this, fd_t *fd,
+                        br_stub_inode_ctx_t *ctx)
+{
+    int32_t ret = -1;
+    br_stub_local_t *local = NULL;
+
+    local = br_stub_alloc_local(this);
+    if (!local) {
+        gf_smsg(this->name, GF_LOG_ERROR, ENOMEM, BRS_MSG_NO_MEMORY, "gfid=%s",
+                uuid_utoa(fd->inode->gfid), NULL);
+        goto error_return;
+    }
+
+    if (fd_is_anonymous(fd)) {
+        ret = br_stub_anon_fd_ctx(this, fd, ctx);
+        if (ret)
+            goto free_local;
+    }
 
- done:
-        frame->local = NULL;
-        if (op_ret < 0)
-                call_unwind_error (local->fopstub, op_ret, op_errno);
-        else
-                call_resume (local->fopstub);
-        br_stub_cleanup_local (local);
-        br_stub_dealloc_local (local);
+    frame->local = local;
 
-        return 0;
+    return 0;
+
+free_local:
+    br_stub_dealloc_local(local);
+error_return:
+    return -1;
+}
+
+static int
+br_stub_mark_inode_modified(xlator_t *this, br_stub_local_t *local)
+{
+    fd_t *fd = NULL;
+    int32_t ret = 0;
+    uint64_t ctx_addr = 0;
+    br_stub_inode_ctx_t *ctx = NULL;
+    unsigned long version = BITROT_DEFAULT_CURRENT_VERSION;
+
+    fd = local->u.context.fd;
+
+    ret = br_stub_get_inode_ctx(this, fd->inode, &ctx_addr);
+    if (ret < 0) {
+        ret = br_stub_init_inode_versions(this, fd, fd->inode, version,
+                                          _gf_true, _gf_false, &ctx_addr);
+        if (ret)
+            goto error_return;
+    }
+
+    ctx = (br_stub_inode_ctx_t *)(long)ctx_addr;
+
+    LOCK(&fd->inode->lock);
+    {
+        __br_stub_set_inode_modified(ctx);
+    }
+    UNLOCK(&fd->inode->lock);
+
+    return 0;
+
+error_return:
+    return -1;
 }
 
+/**
+ * The possible return values from br_stub_is_bad_object () are:
+ * 1) 0  => as per the inode context object is not bad
+ * 2) -1 => Failed to get the inode context itself
+ * 3) -2 => As per the inode context object is bad
+ * Both -ve values means the fop which called this function is failed
+ * and error is returned upwards.
+ */
+static int
+br_stub_check_bad_object(xlator_t *this, inode_t *inode, int32_t *op_ret,
+                         int32_t *op_errno)
+{
+    int ret = -1;
+    unsigned long version = BITROT_DEFAULT_CURRENT_VERSION;
+
+    ret = br_stub_is_bad_object(this, inode);
+    if (ret == -2) {
+        gf_smsg(this->name, GF_LOG_ERROR, 0, BRS_MSG_BAD_OBJECT_ACCESS,
+                "gfid=%s", uuid_utoa(inode->gfid), NULL);
+        *op_ret = -1;
+        *op_errno = EIO;
+    }
+
+    if (ret == -1) {
+        ret = br_stub_init_inode_versions(this, NULL, inode, version, _gf_true,
+                                          _gf_false, NULL);
+        if (ret) {
+            gf_smsg(this->name, GF_LOG_ERROR, 0,
+                    BRS_MSG_GET_INODE_CONTEXT_FAILED, "gfid=%s",
+                    uuid_utoa(inode->gfid), NULL);
+            *op_ret = -1;
+            *op_errno = EINVAL;
+        }
+    }
+
+    return ret;
+}
+
+/**
+ * callback for inode/fd versioning
+ */
 int
-br_stub_fd_incversioning_cbk (call_frame_t *frame,
-                              void *cookie, xlator_t *this,
-                              int op_ret, int op_errno, dict_t *xdata)
-{
-        fd_t            *fd      = NULL;
-        inode_t         *inode   = NULL;
-        unsigned long    version = 0;
-        br_stub_local_t *local   = NULL;
-
-        local = (br_stub_local_t *)frame->local;
-        if (op_ret < 0)
-                goto done;
-        fd      = local->u.context.fd;
-        inode   = local->u.context.inode;
-        version = local->u.context.version;
-
-        op_ret = br_stub_mod_inode_versions (this, fd, inode, version);
-        if (op_ret < 0)
-                op_errno = EINVAL;
-
- done:
-        frame->local = NULL;
-        if (op_ret < 0)
-                call_unwind_error (local->fopstub, -1, op_errno);
-        else
-                call_resume (local->fopstub);
-        br_stub_cleanup_local (local);
-        br_stub_dealloc_local (local);
+br_stub_fd_incversioning_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                             int op_ret, int op_errno, dict_t *xdata)
+{
+    fd_t *fd = NULL;
+    inode_t *inode = NULL;
+    unsigned long version = 0;
+    br_stub_local_t *local = NULL;
 
-        return 0;
+    local = (br_stub_local_t *)frame->local;
+    if (op_ret < 0)
+        goto done;
+    fd = local->u.context.fd;
+    inode = local->u.context.inode;
+    version = local->u.context.version;
+
+    op_ret = br_stub_mod_inode_versions(this, fd, inode, version);
+    if (op_ret < 0)
+        op_errno = EINVAL;
+
+done:
+    if (op_ret < 0) {
+        frame->local = NULL;
+        call_unwind_error(local->fopstub, -1, op_errno);
+        br_stub_cleanup_local(local);
+        br_stub_dealloc_local(local);
+    } else {
+        call_resume(local->fopstub);
+    }
+    return 0;
 }
 
 /**
  * Initial object versioning
  *
  * Version persists two (2) extended attributes as explained below:
- *   1. Current (ongoing) version: This is incremented on an open()
- *      or creat() and is the running version for an object.
+ *   1. Current (ongoing) version: This is incremented on an writev ()
+ *      or truncate () and is the running version for an object.
  *   2. Signing version: This is the version against which an object
  *      was signed (checksummed).
  *
  * During initial versioning, both ongoing and signing versions are
- * set of one and zero respectively. An open() call increments the
+ * set of one and zero respectively. A write() call increments the
  * ongoing version as an indication of modification to the object.
  * Additionally this needs to be persisted on disk and needs to be
  * durable: fsync().. :-/
- * As an optimization only the first open() synchronizes the ongoing
- * version to disk, subsequent open()s before the *last* release()
+ * As an optimization only the first write() synchronizes the ongoing
+ * version to disk, subsequent write()s before the *last* release()
  * are no-op's.
  *
  * create(), just like lookup() initializes the object versions to
- * the default, but persists the version to disk. As an optimization
- * this is not a durable operation: in case of a crash, hard reboot
- * etc.. absence of versioning xattrs is ignored in scrubber along
- * with the one time crawler explicitly triggering signing for such
- * objects.
+ * the default. As an optimization this is not a durable operation:
+ * in case of a crash, hard reboot etc.. absence of versioning xattrs
+ * is ignored in scrubber along with the one time crawler explicitly
+ * triggering signing for such objects.
  *
- * c.f. br_stub_open_cbk() / br_stub_create_cbk()
+ * c.f. br_stub_writev() / br_stub_truncate()
  */
 
 /**
  * perform full or incremental versioning on an inode pointd by an
  * fd. incremental versioning is done when an inode is dirty and a
- * writeback is trigerred.
+ * writeback is triggered.
  */
 
 int
-br_stub_fd_versioning (xlator_t *this, call_frame_t *frame,
-                       call_stub_t *stub, dict_t *dict, fd_t *fd,
-                       br_stub_version_cbk *callback, unsigned long memversion,
-                       int versioningtype, int durable, int dirty)
-{
-        int32_t          ret   = -1;
-        int              flags = 0;
-        dict_t          *xdata = NULL;
-        br_stub_local_t *local = NULL;
-
-        if (durable) {
-                xdata = dict_new ();
-                if (!xdata)
-                        goto done;
-                ret = dict_set_int32 (xdata, GLUSTERFS_DURABLE_OP, 0);
-                if (ret)
-                        goto dealloc_xdata;
+br_stub_fd_versioning(xlator_t *this, call_frame_t *frame, call_stub_t *stub,
+                      dict_t *dict, fd_t *fd, br_stub_version_cbk *callback,
+                      unsigned long memversion, int versioningtype, int durable)
+{
+    int32_t ret = -1;
+    int flags = 0;
+    dict_t *xdata = NULL;
+    br_stub_local_t *local = NULL;
+
+    xdata = dict_new();
+    if (!xdata)
+        goto done;
+
+    ret = dict_set_int32(xdata, GLUSTERFS_INTERNAL_FOP_KEY, 1);
+    if (ret)
+        goto dealloc_xdata;
+
+    if (durable) {
+        ret = dict_set_int32(xdata, GLUSTERFS_DURABLE_OP, 0);
+        if (ret)
+            goto dealloc_xdata;
+    }
+
+    local = frame->local;
+
+    br_stub_fill_local(local, stub, fd, fd->inode, fd->inode->gfid,
+                       versioningtype, memversion);
+
+    STACK_WIND(frame, callback, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->fsetxattr, fd, dict, flags, xdata);
+
+    ret = 0;
+
+dealloc_xdata:
+    dict_unref(xdata);
+done:
+    return ret;
+}
+
+static int
+br_stub_perform_incversioning(xlator_t *this, call_frame_t *frame,
+                              call_stub_t *stub, fd_t *fd,
+                              br_stub_inode_ctx_t *ctx)
+{
+    int32_t ret = -1;
+    dict_t *dict = NULL;
+    br_version_t *obuf = NULL;
+    unsigned long writeback_version = 0;
+    int op_errno = 0;
+    br_stub_local_t *local = NULL;
+
+    op_errno = EINVAL;
+    local = frame->local;
+
+    writeback_version = __br_stub_writeback_version(ctx);
+
+    op_errno = ENOMEM;
+    dict = dict_new();
+    if (!dict)
+        goto out;
+    ret = br_stub_alloc_versions(&obuf, NULL, 0);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_ERROR, 0, BRS_MSG_ALLOC_MEM_FAILED,
+                "gfid=%s", uuid_utoa(fd->inode->gfid), NULL);
+        goto out;
+    }
+    ret = br_stub_prepare_version_request(this, dict, obuf, writeback_version);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_ERROR, 0, BRS_MSG_VERSION_PREPARE_FAIL,
+                "gfid=%s", uuid_utoa(fd->inode->gfid), NULL);
+        br_stub_dealloc_versions(obuf);
+        goto out;
+    }
+
+    ret = br_stub_fd_versioning(
+        this, frame, stub, dict, fd, br_stub_fd_incversioning_cbk,
+        writeback_version, BR_STUB_INCREMENTAL_VERSIONING, !WRITEBACK_DURABLE);
+out:
+    if (dict)
+        dict_unref(dict);
+    if (ret) {
+        if (local)
+            frame->local = NULL;
+        call_unwind_error(stub, -1, op_errno);
+        if (local) {
+            br_stub_cleanup_local(local);
+            br_stub_dealloc_local(local);
         }
+    }
 
-        local = br_stub_alloc_local (this);
-        if (!local) {
-                ret = -1;
-                goto dealloc_xdata;
+    return ret;
+}
+
+/** {{{ */
+
+/* fsetxattr() */
+
+int32_t
+br_stub_perform_objsign(call_frame_t *frame, xlator_t *this, fd_t *fd,
+                        dict_t *dict, int flags, dict_t *xdata)
+{
+    STACK_WIND(frame, default_fsetxattr_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->fsetxattr, fd, dict, flags, xdata);
+
+    dict_unref(xdata);
+    return 0;
+}
+
+void *
+br_stub_signth(void *arg)
+{
+    xlator_t *this = arg;
+    br_stub_private_t *priv = this->private;
+    struct br_stub_signentry *sigstub = NULL;
+
+    THIS = this;
+    while (1) {
+        /*
+         * Disabling bit-rot feature leads to this particular thread
+         * getting cleaned up by reconfigure via a call to the function
+         * gf_thread_cleanup_xint (which in turn calls pthread_cancel
+         * and pthread_join). But, if this thread had held the mutex
+         * &priv->lock at the time of cancellation, then it leads to
+         * deadlock in future when bit-rot feature is enabled (which
+         * again spawns this thread which cant hold the lock as the
+         * mutex is still held by the previous instance of the thread
+         * which got killed). Also, the br_stub_handle_object_signature
+         * function which is called whenever file has to be signed
+         * also gets blocked as it too attempts to acquire &priv->lock.
+         *
+         * So, arrange for the lock to be unlocked as part of the
+         * cleanup of this thread using pthread_cleanup_push and
+         * pthread_cleanup_pop.
+         */
+        pthread_cleanup_push(br_stub_lock_cleaner, &priv->lock);
+        pthread_mutex_lock(&priv->lock);
+        {
+            while (list_empty(&priv->squeue))
+                pthread_cond_wait(&priv->cond, &priv->lock);
+
+            sigstub = list_first_entry(&priv->squeue, struct br_stub_signentry,
+                                       list);
+            list_del_init(&sigstub->list);
         }
+        pthread_mutex_unlock(&priv->lock);
+        pthread_cleanup_pop(0);
 
-        if (versioningtype == BR_STUB_FULL_VERSIONING)
-                flags |= XATTR_CREATE;
+        call_resume(sigstub->stub);
 
-        br_stub_fill_local (local, stub, fd,
-                            fd->inode, fd->inode->gfid,
-                            versioningtype, memversion, dirty);
+        GF_FREE(sigstub);
+    }
 
-        frame->local = local;
-        STACK_WIND (frame, callback,
-                    FIRST_CHILD (this), FIRST_CHILD (this)->fops->fsetxattr,
-                    fd, dict, flags, xdata);
+    return NULL;
+}
 
-        ret = 0;
+static gf_boolean_t
+br_stub_internal_xattr(dict_t *dict)
+{
+    if (dict_get(dict, GLUSTERFS_SET_OBJECT_SIGNATURE) ||
+        dict_get(dict, GLUSTERFS_GET_OBJECT_SIGNATURE) ||
+        dict_get(dict, BR_REOPEN_SIGN_HINT_KEY) ||
+        dict_get(dict, BITROT_OBJECT_BAD_KEY) ||
+        dict_get(dict, BITROT_SIGNING_VERSION_KEY) ||
+        dict_get(dict, BITROT_CURRENT_VERSION_KEY))
+        return _gf_true;
+
+    return _gf_false;
+}
 
- dealloc_xdata:
-        if (durable)
-                dict_unref (xdata);
- done:
-        return ret;
+int
+orderq(struct list_head *elem1, struct list_head *elem2)
+{
+    struct br_stub_signentry *s1 = NULL;
+    struct br_stub_signentry *s2 = NULL;
+
+    s1 = list_entry(elem1, struct br_stub_signentry, list);
+    s2 = list_entry(elem2, struct br_stub_signentry, list);
+
+    return (s1->v > s2->v);
 }
 
-static inline int
-br_stub_perform_fullversioning (xlator_t *this, call_frame_t *frame,
-                                call_stub_t *stub, fd_t *fd)
+static int
+br_stub_compare_sign_version(xlator_t *this, inode_t *inode,
+                             br_signature_t *sbuf, dict_t *dict,
+                             int *fakesuccess)
 {
-        int32_t         ret      = -1;
-        dict_t         *dict     = NULL;
-        br_version_t   *obuf     = NULL;
-        int             op_errno = 0;
+    int32_t ret = -1;
+    uint64_t tmp_ctx = 0;
+    gf_boolean_t invalid = _gf_false;
+    br_stub_inode_ctx_t *ctx = NULL;
+
+    GF_VALIDATE_OR_GOTO("bit-rot-stub", this, out);
+    GF_VALIDATE_OR_GOTO(this->name, inode, out);
+    GF_VALIDATE_OR_GOTO(this->name, sbuf, out);
+    GF_VALIDATE_OR_GOTO(this->name, dict, out);
+
+    ret = br_stub_get_inode_ctx(this, inode, &tmp_ctx);
+    if (ret) {
+        dict_del(dict, BITROT_SIGNING_VERSION_KEY);
+        goto out;
+    }
+
+    ctx = (br_stub_inode_ctx_t *)(long)tmp_ctx;
+
+    LOCK(&inode->lock);
+    {
+        if (ctx->currentversion < sbuf->signedversion) {
+            invalid = _gf_true;
+        } else if (ctx->currentversion > sbuf->signedversion) {
+            gf_msg_debug(this->name, 0,
+                         "\"Signing version\" "
+                         "(%lu) lower than \"Current version \" "
+                         "(%lu)",
+                         ctx->currentversion, sbuf->signedversion);
+            *fakesuccess = 1;
+        }
+    }
+    UNLOCK(&inode->lock);
+
+    if (invalid) {
+        ret = -1;
+        gf_smsg(this->name, GF_LOG_WARNING, 0, BRS_MSG_SIGN_VERSION_ERROR,
+                "Signing-ver=%lu", sbuf->signedversion, "current-ver=%lu",
+                ctx->currentversion, NULL);
+    }
+
+out:
+    return ret;
+}
 
-        op_errno = ENOMEM;
-        dict = dict_new ();
-        if (!dict)
-                goto done;
-        ret = br_stub_alloc_versions (&obuf, NULL, 0);
-        if (ret)
-                goto dealloc_dict;
+static int
+br_stub_prepare_signature(xlator_t *this, dict_t *dict, inode_t *inode,
+                          br_isignature_t *sign, int *fakesuccess)
+{
+    int32_t ret = -1;
+    size_t signaturelen = 0;
+    br_signature_t *sbuf = NULL;
+
+    if (!br_is_signature_type_valid(sign->signaturetype))
+        goto out;
+
+    signaturelen = sign->signaturelen;
+    ret = br_stub_alloc_versions(NULL, &sbuf, signaturelen);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_ERROR, 0, BRS_MSG_ALLOC_MEM_FAILED,
+                "gfid=%s", uuid_utoa(inode->gfid), NULL);
+        ret = -1;
+        goto out;
+    }
+    ret = br_stub_prepare_signing_request(dict, sbuf, sign, signaturelen);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_ERROR, 0, BRS_MSG_SIGN_PREPARE_FAIL,
+                "gfid=%s", uuid_utoa(inode->gfid), NULL);
+        ret = -1;
+        br_stub_dealloc_versions(sbuf);
+        goto out;
+    }
+
+    /* At this point sbuf has been added to dict, so the memory will be freed
+     * when the data from the dict is destroyed
+     */
+    ret = br_stub_compare_sign_version(this, inode, sbuf, dict, fakesuccess);
+out:
+    return ret;
+}
 
-        op_errno = EINVAL;
-        ret = br_stub_prepare_version_request (this, dict, obuf,
-                                               BITROT_DEFAULT_CURRENT_VERSION);
-        if (ret)
-                goto dealloc_versions;
+static void
+br_stub_handle_object_signature(call_frame_t *frame, xlator_t *this, fd_t *fd,
+                                dict_t *dict, br_isignature_t *sign,
+                                dict_t *xdata)
+{
+    int32_t ret = -1;
+    int32_t op_ret = -1;
+    int32_t op_errno = EINVAL;
+    int fakesuccess = 0;
+    br_stub_private_t *priv = NULL;
+    struct br_stub_signentry *sigstub = NULL;
+
+    priv = this->private;
+
+    if (frame->root->pid != GF_CLIENT_PID_BITD) {
+        gf_smsg(this->name, GF_LOG_WARNING, op_errno, BRS_MSG_NON_BITD_PID,
+                "PID=%d", frame->root->pid, NULL);
+        goto dofop;
+    }
+
+    ret = br_stub_prepare_signature(this, dict, fd->inode, sign, &fakesuccess);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_WARNING, 0, BRS_MSG_SIGN_PREPARE_FAIL,
+                "gfid=%s", uuid_utoa(fd->inode->gfid), NULL);
+        goto dofop;
+    }
+    if (fakesuccess) {
+        op_ret = op_errno = 0;
+        goto dofop;
+    }
+
+    dict_del(dict, GLUSTERFS_SET_OBJECT_SIGNATURE);
+
+    ret = -1;
+    if (!xdata) {
+        xdata = dict_new();
+        if (!xdata)
+            goto dofop;
+    } else {
+        dict_ref(xdata);
+    }
+
+    ret = dict_set_int32(xdata, GLUSTERFS_DURABLE_OP, 0);
+    if (ret)
+        goto unref_dict;
 
-        /**
-         * Version extended attributes need not be durable at this point of
-         * time. If the objects (inode) data gets persisted on disk but the
-         * version extended attributes are lost due to a crash/power failure,
-         * a subsequent lookup marks the objects signature as stale. This way,
-         * dentry operation times do not shoot up.
-         */
-        ret = br_stub_fd_versioning (this, frame, stub, dict, fd,
-                                     br_stub_inode_fullversioning_cbk,
-                                     BITROT_DEFAULT_CURRENT_VERSION,
-                                     BR_STUB_FULL_VERSIONING, !WRITEBACK_DURABLE, 0);
-
- dealloc_versions:
-        br_stub_dealloc_versions (obuf);
- dealloc_dict:
-        dict_unref (dict);
- done:
-        if (ret)
-                call_unwind_error (stub, -1, op_errno);
-        return ret;
+    /* prepare dispatch stub to order object signing */
+    sigstub = GF_CALLOC(1, sizeof(*sigstub), gf_br_stub_mt_sigstub_t);
+    if (!sigstub)
+        goto unref_dict;
+
+    INIT_LIST_HEAD(&sigstub->list);
+    sigstub->v = ntohl(sign->signedversion);
+    sigstub->stub = fop_fsetxattr_stub(frame, br_stub_perform_objsign, fd, dict,
+                                       0, xdata);
+    if (!sigstub->stub)
+        goto cleanup_stub;
+
+    pthread_mutex_lock(&priv->lock);
+    {
+        list_add_order(&sigstub->list, &priv->squeue, orderq);
+        pthread_cond_signal(&priv->cond);
+    }
+    pthread_mutex_unlock(&priv->lock);
+
+    return;
+
+cleanup_stub:
+    GF_FREE(sigstub);
+unref_dict:
+    dict_unref(xdata);
+dofop:
+    STACK_UNWIND_STRICT(fsetxattr, frame, op_ret, op_errno, NULL);
 }
 
-static inline int
-br_stub_perform_incversioning (xlator_t *this,
-                               call_frame_t *frame, call_stub_t *stub,
-                               fd_t *fd, br_stub_inode_ctx_t *ctx)
+int32_t
+br_stub_fsetxattr_resume(call_frame_t *frame, void *cookie, xlator_t *this,
+                         int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
-        int32_t        ret               = -1;
-        dict_t        *dict              = NULL;
-        inode_t       *inode             = NULL;
-        br_version_t  *obuf              = NULL;
-        unsigned long  writeback_version = 0;
-        int            op_errno          = 0;
+    int32_t ret = -1;
+    br_stub_local_t *local = NULL;
 
-        inode = fd->inode;
+    local = frame->local;
+    frame->local = NULL;
 
+    ret = br_stub_mark_inode_modified(this, local);
+    if (ret) {
+        op_ret = -1;
         op_errno = EINVAL;
-        ret = br_stub_require_release_call (this, fd);
-        if (ret)
-                goto done;
+    }
 
-        LOCK (&inode->lock);
-        {
-                if (__br_stub_is_inode_dirty (ctx))
-                        writeback_version = __br_stub_writeback_version (ctx);
-                else
-                        __br_stub_track_openfd (fd, ctx);
-        }
-        UNLOCK (&inode->lock);
+    STACK_UNWIND_STRICT(fsetxattr, frame, op_ret, op_errno, xdata);
 
-        if (!writeback_version) {
-                ret = 0;
-                goto done;
-        }
+    br_stub_cleanup_local(local);
+    br_stub_dealloc_local(local);
 
-        /* inode requires writeback to disk */
-        op_errno = ENOMEM;
-        dict = dict_new ();
-        if (!dict)
-                goto done;
-        ret = br_stub_alloc_versions (&obuf, NULL, 0);
-        if (ret)
-                goto dealloc_dict;
-        ret = br_stub_prepare_version_request (this, dict,
-                                               obuf, writeback_version);
-        if (ret)
-                goto dealloc_versions;
-
-        ret = br_stub_fd_versioning
-                (this, frame, stub, dict,
-                 fd, br_stub_fd_incversioning_cbk, writeback_version,
-                 BR_STUB_INCREMENTAL_VERSIONING, WRITEBACK_DURABLE, 0);
-
- dealloc_versions:
-        br_stub_dealloc_versions (obuf);
- dealloc_dict:
-        dict_unref (dict);
- done:
-        if (!ret && !writeback_version)
-                call_resume (stub);
-        if (ret)
-                call_unwind_error (stub, -1, op_errno);
-        return ret;
+    return 0;
 }
 
-/** {{{ */
+/**
+ * Handles object reopens. Object reopens can be of 3 types. 2 are from
+ * oneshot crawler and 1 from the regular signer.
+ * ONESHOT CRAWLER:
+ * For those objects which were created before bitrot was enabled. oneshow
+ * crawler crawls the namespace and signs all the objects. It has to do
+ * the versioning before making bit-rot-stub send a sign notification.
+ * So it sends fsetxattr with BR_OBJECT_REOPEN as the value. And bit-rot-stub
+ * upon getting BR_OBJECT_REOPEN value checks if the version has to be
+ * increased or not. By default the version will be increased. But if the
+ * object is modified before BR_OBJECT_REOPEN from oneshot crawler, then
+ * versioning need not be done. In that case simply a success is returned.
+ * SIGNER:
+ * Signer wait for 2 minutes upon getting the notification from bit-rot-stub
+ * and then it sends a dummy write (in reality a fsetxattr) call, to change
+ * the state of the inode from REOPEN_WAIT to SIGN_QUICK. The funny part here
+ * is though the inode's state is REOPEN_WAIT, the call sent by signer is
+ * BR_OBJECT_RESIGN. Once the state is changed to SIGN_QUICK, then yet another
+ * notification is sent upon release (RESIGN would have happened via fsetxattr,
+ * so a fd is needed) and the object is signed truly this time.
+ * There is a challenge in the above RESIGN method by signer. After sending
+ * the 1st notification, the inode could be forgotten before RESIGN request
+ * is received. In that case, the inode's context (the newly looked up inode)
+ * would not indicate the inode as being modified (it would be in the default
+ * state) and because of this, a SIGN_QUICK notification to truly sign the
+ * object would not be sent. So, this is how its handled.
+ * if (request == RESIGN) {
+ *    if (inode->sign_info == NORMAL) {
+ *        mark_inode_non_dirty;
+ *        mark_inode_modified;
+ *    }
+ *    GOBACK (means unwind without doing versioning)
+ * }
+ */
+static void
+br_stub_handle_object_reopen(call_frame_t *frame, xlator_t *this, fd_t *fd,
+                             uint32_t val)
+{
+    int32_t ret = -1;
+    int32_t op_ret = -1;
+    int32_t op_errno = EINVAL;
+    call_stub_t *stub = NULL;
+    gf_boolean_t inc_version = _gf_false;
+    gf_boolean_t modified = _gf_false;
+    br_stub_inode_ctx_t *ctx = NULL;
+    br_stub_local_t *local = NULL;
+    gf_boolean_t goback = _gf_true;
+
+    ret = br_stub_need_versioning(this, fd, &inc_version, &modified, &ctx);
+    if (ret)
+        goto unwind;
+
+    LOCK(&fd->inode->lock);
+    {
+        if ((val == BR_OBJECT_REOPEN) && inc_version)
+            goback = _gf_false;
+        if (val == BR_OBJECT_RESIGN && ctx->info_sign == BR_SIGN_NORMAL) {
+            __br_stub_mark_inode_synced(ctx);
+            __br_stub_set_inode_modified(ctx);
+        }
+        (void)__br_stub_inode_sign_state(ctx, GF_FOP_FSETXATTR, fd);
+    }
+    UNLOCK(&fd->inode->lock);
+
+    if (goback) {
+        op_ret = op_errno = 0;
+        goto unwind;
+    }
+
+    ret = br_stub_versioning_prep(frame, this, fd, ctx);
+    if (ret)
+        goto unwind;
+    local = frame->local;
+
+    stub = fop_fsetxattr_cbk_stub(frame, br_stub_fsetxattr_resume, 0, 0, NULL);
+    if (!stub) {
+        gf_smsg(this->name, GF_LOG_ERROR, 0, BRS_MSG_STUB_ALLOC_FAILED,
+                "fsetxattr gfid=%s", uuid_utoa(fd->inode->gfid), NULL);
+        goto cleanup_local;
+    }
+
+    (void)br_stub_perform_incversioning(this, frame, stub, fd, ctx);
+    return;
+
+cleanup_local:
+    br_stub_cleanup_local(local);
+    br_stub_dealloc_local(local);
+
+unwind:
+    frame->local = NULL;
+    STACK_UNWIND_STRICT(fsetxattr, frame, op_ret, op_errno, NULL);
+}
 
-/* fsetxattr() */
+/**
+ * This function only handles bad file identification. Instead of checking in
+ * fops like open, readv, writev whether the object is bad or not by doing
+ * getxattr calls, better to catch them when scrubber marks it as bad.
+ * So this callback is called only when the fsetxattr is sent by the scrubber
+ * to mark the object as bad.
+ */
+int
+br_stub_fsetxattr_bad_object_cbk(call_frame_t *frame, void *cookie,
+                                 xlator_t *this, int32_t op_ret,
+                                 int32_t op_errno, dict_t *xdata)
+{
+    br_stub_local_t *local = NULL;
+    int32_t ret = -1;
+
+    local = frame->local;
+    frame->local = NULL;
+
+    if (op_ret < 0)
+        goto unwind;
+
+    /*
+     * What to do if marking the object as bad fails? (i.e. in memory
+     * marking within the inode context. If we are here means fsetxattr
+     * fop has succeeded on disk and the bad object xattr has been set).
+     * We can return failure to scruber, but there is nothing the scrubber
+     * can do with it (it might assume that the on disk setxattr itself has
+     * failed). The main purpose of this operation is to help identify the
+     * bad object by checking the inode context itself (thus avoiding the
+     * necessity of doing a getxattr fop on the disk).
+     *
+     * So as of now, success itself is being returned even though inode
+     * context set operation fails.
+     * In future if there is any change in the policy which can handle this,
+     * then appropriate response should be sent (i.e. success or error).
+     */
+    ret = br_stub_mark_object_bad(this, local->u.context.inode);
+    if (ret)
+        gf_smsg(this->name, GF_LOG_ERROR, 0, BRS_MSG_BAD_OBJ_MARK_FAIL,
+                "gfid=%s", uuid_utoa(local->u.context.inode->gfid), NULL);
+
+    ret = br_stub_add(this, local->u.context.inode->gfid);
+
+unwind:
+    STACK_UNWIND_STRICT(fsetxattr, frame, op_ret, op_errno, xdata);
+    br_stub_cleanup_local(local);
+    br_stub_dealloc_local(local);
+    return 0;
+}
+
+static int32_t
+br_stub_handle_bad_object_key(call_frame_t *frame, xlator_t *this, fd_t *fd,
+                              dict_t *dict, int flags, dict_t *xdata)
+{
+    br_stub_local_t *local = NULL;
+    int32_t op_ret = -1;
+    int32_t op_errno = EINVAL;
+
+    if (frame->root->pid != GF_CLIENT_PID_SCRUB) {
+        gf_smsg(this->name, GF_LOG_ERROR, 0, BRS_MSG_NON_SCRUB_BAD_OBJ_MARK,
+                "gfid=%s", uuid_utoa(fd->inode->gfid), NULL);
+        goto unwind;
+    }
+
+    local = br_stub_alloc_local(this);
+    if (!local) {
+        gf_smsg(this->name, GF_LOG_ERROR, 0, BRS_MSG_ALLOC_MEM_FAILED,
+                "fsetxattr gfid=%s", uuid_utoa(fd->inode->gfid), NULL);
+        op_ret = -1;
+        op_errno = ENOMEM;
+        goto unwind;
+    }
+
+    br_stub_fill_local(local, NULL, fd, fd->inode, fd->inode->gfid,
+                       BR_STUB_NO_VERSIONING, 0);
+    frame->local = local;
+
+    STACK_WIND(frame, br_stub_fsetxattr_bad_object_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->fsetxattr, fd, dict, flags, xdata);
+    return 0;
+unwind:
+    STACK_UNWIND_STRICT(fsetxattr, frame, op_ret, op_errno, NULL);
+    return 0;
+}
 
-static inline int
-br_stub_prepare_signature (xlator_t *this, dict_t *dict,
-                           inode_t *inode, br_isignature_t *sign)
+/**
+ * As of now, versioning is done by the stub (though as a setxattr
+ * operation) as part of inode modification operations such as writev,
+ * truncate, ftruncate. And signing is done by BitD by a fsetxattr call.
+ * So any kind of setxattr coming on the versioning and the signing xattr is
+ * not allowed (i.e. BITROT_CURRENT_VERSION_KEY and BITROT_SIGNING_VERSION_KEY).
+ * In future if BitD/scrubber are allowed to change the versioning
+ * xattrs (though I cannot see a reason for it as of now), then the below
+ * function can be modified to block setxattr on version for only applications.
+ *
+ * NOTE: BitD sends sign request on GLUSTERFS_SET_OBJECT_SIGNATURE key.
+ *       BITROT_SIGNING_VERSION_KEY is the xattr used to save the signature.
+ *
+ */
+static int32_t
+br_stub_handle_internal_xattr(call_frame_t *frame, xlator_t *this, fd_t *fd,
+                              char *key)
 {
-        int32_t ret = 0;
-        size_t signaturelen = 0;
-        br_signature_t *sbuf = NULL;
+    int32_t op_ret = -1;
+    int32_t op_errno = EINVAL;
 
-        if (!br_is_signature_type_valid (sign->signaturetype))
-                goto error_return;
+    gf_smsg(this->name, GF_LOG_ERROR, 0, BRS_MSG_SET_INTERNAL_XATTR,
+            "setxattr key=%s", key, "inode-gfid=%s", uuid_utoa(fd->inode->gfid),
+            NULL);
 
-        signaturelen = sign->signaturelen;
-        ret = br_stub_alloc_versions (NULL, &sbuf, signaturelen);
-        if (ret)
-                goto error_return;
-        ret = br_stub_prepare_signing_request (dict, sbuf, sign, signaturelen);
-        if (ret)
-                goto dealloc_versions;
-        return 0;
+    STACK_UNWIND_STRICT(fsetxattr, frame, op_ret, op_errno, NULL);
+    return 0;
+}
 
- dealloc_versions:
-        br_stub_dealloc_versions (sbuf);
- error_return:
-        return -1;
+static void
+br_stub_dump_xattr(xlator_t *this, dict_t *dict, int *op_errno)
+{
+    char *format = "(%s:%s)";
+    char *dump = NULL;
+
+    dump = GF_CALLOC(1, BR_STUB_DUMP_STR_SIZE, gf_br_stub_mt_misc);
+    if (!dump) {
+        *op_errno = ENOMEM;
+        goto out;
+    }
+    dict_dump_to_str(dict, dump, BR_STUB_DUMP_STR_SIZE, format);
+    gf_smsg(this->name, GF_LOG_ERROR, 0, BRS_MSG_SET_INTERNAL_XATTR,
+            "fsetxattr dump=%s", dump, NULL);
+out:
+    if (dump) {
+        GF_FREE(dump);
+    }
+    return;
 }
 
 int
-br_stub_fsetxattr (call_frame_t *frame, xlator_t *this,
-                   fd_t *fd, dict_t *dict, int flags, dict_t *xdata)
+br_stub_fsetxattr(call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *dict,
+                  int flags, dict_t *xdata)
 {
-        int32_t          ret  = 0;
-        br_isignature_t *sign = NULL;
-        gf_boolean_t     xref = _gf_false;
+    int32_t ret = 0;
+    uint32_t val = 0;
+    br_isignature_t *sign = NULL;
+    br_stub_private_t *priv = NULL;
+    int32_t op_ret = -1;
+    int32_t op_errno = EINVAL;
+
+    priv = this->private;
+
+    if ((frame->root->pid != GF_CLIENT_PID_BITD &&
+         frame->root->pid != GF_CLIENT_PID_SCRUB) &&
+        br_stub_internal_xattr(dict)) {
+        br_stub_dump_xattr(this, dict, &op_errno);
+        goto unwind;
+    }
+
+    if (!priv->do_versioning)
+        goto wind;
+
+    if (!IA_ISREG(fd->inode->ia_type))
+        goto wind;
+
+    /* object signature request */
+    ret = dict_get_bin(dict, GLUSTERFS_SET_OBJECT_SIGNATURE, (void **)&sign);
+    if (!ret) {
+        gf_msg_debug(this->name, 0, "got SIGNATURE request on %s",
+                     uuid_utoa(fd->inode->gfid));
+        br_stub_handle_object_signature(frame, this, fd, dict, sign, xdata);
+        goto done;
+    }
 
-        if (!IA_ISREG (fd->inode->ia_type))
-                goto wind;
-        ret = dict_get_bin (dict, GLUSTERFS_SET_OBJECT_SIGNATURE,
-                            (void **) &sign);
-        if (ret < 0)
-                goto wind;
-        if (frame->root->pid != GF_CLIENT_PID_BITD)
-                goto unwind;
+    /* signing xattr */
+    if (dict_get(dict, BITROT_SIGNING_VERSION_KEY)) {
+        br_stub_handle_internal_xattr(frame, this, fd,
+                                      BITROT_SIGNING_VERSION_KEY);
+        goto done;
+    }
 
-        ret = br_stub_prepare_signature (this, dict, fd->inode, sign);
-        if (ret)
-                goto unwind;
-        dict_del (dict, GLUSTERFS_SET_OBJECT_SIGNATURE);
-
-        if (!xdata) {
-                xdata = dict_new ();
-                if (!xdata)
-                        goto unwind;
-        } else {
-                dict_ref (xdata);
-        }
+    /* version xattr */
+    if (dict_get(dict, BITROT_CURRENT_VERSION_KEY)) {
+        br_stub_handle_internal_xattr(frame, this, fd,
+                                      BITROT_CURRENT_VERSION_KEY);
+        goto done;
+    }
 
-        xref = _gf_true;
-        ret = dict_set_int32 (xdata, GLUSTERFS_DURABLE_OP, 0);
-        if (ret)
-                goto unwind;
+    if (dict_get(dict, GLUSTERFS_GET_OBJECT_SIGNATURE)) {
+        br_stub_handle_internal_xattr(frame, this, fd,
+                                      GLUSTERFS_GET_OBJECT_SIGNATURE);
+        goto done;
+    }
 
- wind:
-        STACK_WIND (frame, default_setxattr_cbk,
-                    FIRST_CHILD (this), FIRST_CHILD (this)->fops->fsetxattr, fd,
-                    dict, flags, xdata);
+    /* object reopen request */
+    ret = dict_get_uint32(dict, BR_REOPEN_SIGN_HINT_KEY, &val);
+    if (!ret) {
+        br_stub_handle_object_reopen(frame, this, fd, val);
         goto done;
+    }
 
- unwind:
-        STACK_UNWIND_STRICT (setxattr, frame, -1, EINVAL, NULL);
- done:
-        if (xref)
-                dict_unref (xdata);
-        return 0;
+    /* handle bad object */
+    if (dict_get(dict, BITROT_OBJECT_BAD_KEY)) {
+        br_stub_handle_bad_object_key(frame, this, fd, dict, flags, xdata);
+        goto done;
+    }
+
+wind:
+    STACK_WIND(frame, default_fsetxattr_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->fsetxattr, fd, dict, flags, xdata);
+    return 0;
+
+unwind:
+    STACK_UNWIND_STRICT(fsetxattr, frame, op_ret, op_errno, NULL);
+
+done:
+    return 0;
+}
+
+/**
+ * Currently BitD and scrubber are doing fsetxattr to either sign the object
+ * or to mark it as bad. Hence setxattr on any of those keys is denied directly
+ * without checking from where the fop is coming.
+ * Later, if BitD or Scrubber does setxattr of those keys, then appropriate
+ * check has to be added below.
+ */
+int
+br_stub_setxattr(call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *dict,
+                 int flags, dict_t *xdata)
+{
+    int32_t op_ret = -1;
+    int32_t op_errno = EINVAL;
+
+    if (br_stub_internal_xattr(dict)) {
+        br_stub_dump_xattr(this, dict, &op_errno);
+        goto unwind;
+    }
+
+    STACK_WIND_TAIL(frame, FIRST_CHILD(this), FIRST_CHILD(this)->fops->setxattr,
+                    loc, dict, flags, xdata);
+    return 0;
+unwind:
+    STACK_UNWIND_STRICT(setxattr, frame, op_ret, op_errno, NULL);
+    return 0;
 }
 
 /** }}} */
 
+/** {{{ */
+
+/* {f}removexattr() */
+
+int32_t
+br_stub_removexattr(call_frame_t *frame, xlator_t *this, loc_t *loc,
+                    const char *name, dict_t *xdata)
+{
+    int32_t op_ret = -1;
+    int32_t op_errno = EINVAL;
+
+    if (!strcmp(BITROT_OBJECT_BAD_KEY, name) ||
+        !strcmp(BITROT_SIGNING_VERSION_KEY, name) ||
+        !strcmp(BITROT_CURRENT_VERSION_KEY, name)) {
+        gf_smsg(this->name, GF_LOG_WARNING, 0, BRS_MSG_REMOVE_INTERNAL_XATTR,
+                "name=%s", name, "file-path=%s", loc->path, NULL);
+        goto unwind;
+    }
+
+    STACK_WIND_TAIL(frame, FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->removexattr, loc, name, xdata);
+    return 0;
+unwind:
+    STACK_UNWIND_STRICT(removexattr, frame, op_ret, op_errno, NULL);
+    return 0;
+}
+
+int32_t
+br_stub_fremovexattr(call_frame_t *frame, xlator_t *this, fd_t *fd,
+                     const char *name, dict_t *xdata)
+{
+    int32_t op_ret = -1;
+    int32_t op_errno = EINVAL;
+
+    if (!strcmp(BITROT_OBJECT_BAD_KEY, name) ||
+        !strcmp(BITROT_SIGNING_VERSION_KEY, name) ||
+        !strcmp(BITROT_CURRENT_VERSION_KEY, name)) {
+        gf_smsg(this->name, GF_LOG_WARNING, 0, BRS_MSG_REMOVE_INTERNAL_XATTR,
+                "name=%s", name, "inode-gfid=%s", uuid_utoa(fd->inode->gfid),
+                NULL);
+        goto unwind;
+    }
+
+    STACK_WIND_TAIL(frame, FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->fremovexattr, fd, name, xdata);
+    return 0;
+unwind:
+    STACK_UNWIND_STRICT(fremovexattr, frame, op_ret, op_errno, NULL);
+    return 0;
+}
+
+/** }}} */
 
 /** {{{ */
 
 /* {f}getxattr() */
 
 int
-br_stub_listxattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                       int op_ret, int op_errno, dict_t *xattr, dict_t *xdata)
+br_stub_listxattr_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                      int op_ret, int op_errno, dict_t *xattr, dict_t *xdata)
 {
-        if (op_ret < 0)
-                goto unwind;
+    if (op_ret < 0)
+        goto unwind;
 
-        br_stub_remove_vxattrs (xattr);
+    br_stub_remove_vxattrs(xattr, _gf_true);
 
- unwind:
-        STACK_UNWIND (frame, op_ret, op_errno, xattr, xdata);
-        return 0;
+unwind:
+    STACK_UNWIND_STRICT(getxattr, frame, op_ret, op_errno, xattr, xdata);
+    return 0;
 }
 
+/**
+ * ONE SHOT CRAWLER from BitD signs the objects that it encounters while
+ * crawling, if the object is identified as stale by the stub. Stub follows
+ * the below logic to mark an object as stale or not.
+ * If the ongoing version and the signed_version match, then the object is not
+ * stale. Just return. Otherwise if they does not match, then it means one
+ * of the below things.
+ * 1) If the inode does not need write back of the version and the sign state is
+ *    is NORMAL, then some active i/o is going on the object. So skip it.
+ *    A notification will be sent to trigger the sign once the release is
+ *    received on the object.
+ * 2) If inode does not need writeback of the version and the sign state is
+ *    either reopen wait or quick sign, then it means:
+ *    A) BitD restarted and it is not sure whether the object it encountered
+ *       while crawling is in its timer wheel or not. Since there is no way to
+ *       scan the timer wheel as of now, ONE SHOT CRAWLER just goes ahead and
+ *       signs the object. Since the inode does not need writeback, version will
+ *       not be incremented and directly the object will be signed.
+ * 3) If the inode needs writeback, then it means the inode was forgotten after
+ *    the versioning and it has to be signed now.
+ *
+ * This is the algorithm followed:
+ * if (ongoing_version == signed_version); then
+ *     object_is_not_stale;
+ *     return;
+ * else; then
+ *      if (!inode_needs_writeback && inode_sign_state != NORMAL); then
+ *            object_is_stale;
+ *      if (inode_needs_writeback); then
+ *            object_is_stale;
+ *
+ * For SCRUBBER, no need to check for the sign state and inode writeback.
+ * If the ondisk ongoingversion and the ondisk signed version does not match,
+ * then treat the object as stale.
+ */
+char
+br_stub_is_object_stale(xlator_t *this, call_frame_t *frame, inode_t *inode,
+                        br_version_t *obuf, br_signature_t *sbuf)
+{
+    uint64_t ctx_addr = 0;
+    br_stub_inode_ctx_t *ctx = NULL;
+    int32_t ret = -1;
+    char stale = 0;
+
+    if (obuf->ongoingversion == sbuf->signedversion)
+        goto out;
+
+    if (frame->root->pid == GF_CLIENT_PID_SCRUB) {
+        stale = 1;
+        goto out;
+    }
+
+    ret = br_stub_get_inode_ctx(this, inode, &ctx_addr);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_ERROR, 0, BRS_MSG_GET_INODE_CONTEXT_FAILED,
+                "gfid=%s", uuid_utoa(inode->gfid), NULL);
+        goto out;
+    }
+
+    ctx = (br_stub_inode_ctx_t *)(long)ctx_addr;
+
+    LOCK(&inode->lock);
+    {
+        if ((!__br_stub_is_inode_dirty(ctx) &&
+             ctx->info_sign != BR_SIGN_NORMAL) ||
+            __br_stub_is_inode_dirty(ctx))
+            stale = 1;
+    }
+    UNLOCK(&inode->lock);
+
+out:
+    return stale;
+}
 
 int
-br_stub_getxattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                      int op_ret, int op_errno, dict_t *xattr, dict_t *xdata)
+br_stub_getxattr_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                     int op_ret, int op_errno, dict_t *xattr, dict_t *xdata)
 {
-        int32_t              ret          = 0;
-        size_t               totallen     = 0;
-        size_t               signaturelen = 0;
-        br_version_t        *obuf         = NULL;
-        br_signature_t      *sbuf         = NULL;
-        br_isignature_out_t *sign         = NULL;
-        br_vxattr_status_t   status;
+    int32_t ret = 0;
+    size_t totallen = 0;
+    size_t signaturelen = 0;
+    br_stub_private_t *priv = NULL;
+    br_version_t *obuf = NULL;
+    br_signature_t *sbuf = NULL;
+    br_isignature_out_t *sign = NULL;
+    br_vxattr_status_t status;
+    br_stub_local_t *local = NULL;
+    inode_t *inode = NULL;
+    gf_boolean_t bad_object = _gf_false;
+    gf_boolean_t ver_enabled = _gf_false;
+
+    BR_STUB_VER_ENABLED_IN_CALLPATH(frame, ver_enabled);
+    priv = this->private;
+
+    if (op_ret < 0)
+        goto unwind;
+    BR_STUB_VER_COND_GOTO(priv, (!ver_enabled), delkeys);
+
+    if (cookie != (void *)BR_STUB_REQUEST_COOKIE)
+        goto unwind;
+
+    local = frame->local;
+    frame->local = NULL;
+    if (!local) {
+        op_ret = -1;
+        op_errno = EINVAL;
+        goto unwind;
+    }
+    inode = local->u.context.inode;
+
+    op_ret = -1;
+    status = br_version_xattr_state(xattr, &obuf, &sbuf, &bad_object);
+
+    op_errno = EIO;
+    if (bad_object)
+        goto delkeys;
+
+    op_errno = EINVAL;
+    if (status == BR_VXATTR_STATUS_INVALID)
+        goto delkeys;
+
+    op_errno = ENODATA;
+    if ((status == BR_VXATTR_STATUS_MISSING) ||
+        (status == BR_VXATTR_STATUS_UNSIGNED))
+        goto delkeys;
+
+    /**
+     * okay.. we have enough information to satisfy the request,
+     * namely: version and signing extended attribute. what's
+     * pending is the signature length -- that's figured out
+     * indirectly via the size of the _whole_ xattr and the
+     * on-disk signing xattr header size.
+     */
+    op_errno = EINVAL;
+    ret = dict_get_uint32(xattr, BITROT_SIGNING_XATTR_SIZE_KEY,
+                          (uint32_t *)&signaturelen);
+    if (ret)
+        goto delkeys;
+
+    signaturelen -= sizeof(br_signature_t);
+    totallen = sizeof(br_isignature_out_t) + signaturelen;
+
+    op_errno = ENOMEM;
+    sign = GF_CALLOC(1, totallen, gf_br_stub_mt_signature_t);
+    if (!sign)
+        goto delkeys;
+
+    sign->time[0] = obuf->timebuf[0];
+    sign->time[1] = obuf->timebuf[1];
+
+    /* Object's dirty state & current signed version */
+    sign->version = sbuf->signedversion;
+    sign->stale = br_stub_is_object_stale(this, frame, inode, obuf, sbuf);
+
+    /* Object's signature */
+    sign->signaturelen = signaturelen;
+    sign->signaturetype = sbuf->signaturetype;
+    (void)memcpy(sign->signature, sbuf->signature, signaturelen);
+
+    op_errno = EINVAL;
+    ret = dict_set_bin(xattr, GLUSTERFS_GET_OBJECT_SIGNATURE, (void *)sign,
+                       totallen);
+    if (ret < 0) {
+        GF_FREE(sign);
+        goto delkeys;
+    }
+    op_errno = 0;
+    op_ret = totallen;
+
+delkeys:
+    br_stub_remove_vxattrs(xattr, _gf_true);
+
+unwind:
+    STACK_UNWIND_STRICT(getxattr, frame, op_ret, op_errno, xattr, xdata);
+    br_stub_cleanup_local(local);
+    br_stub_dealloc_local(local);
+    return 0;
+}
 
-        if (op_ret < 0)
-                goto unwind;
-        if (cookie != (void *) BR_STUB_REQUEST_COOKIE)
-                goto unwind;
+static void
+br_stub_send_stub_init_time(call_frame_t *frame, xlator_t *this)
+{
+    int op_ret = 0;
+    int op_errno = 0;
+    dict_t *xattr = NULL;
+    br_stub_init_t stub = {
+        {
+            0,
+        },
+    };
+    br_stub_private_t *priv = NULL;
 
-        op_ret   = -1;
-        status = br_version_xattr_state (xattr, &obuf, &sbuf);
+    priv = this->private;
 
-        op_errno = EINVAL;
-        if (status == BR_VXATTR_STATUS_INVALID)
-                goto delkeys;
+    xattr = dict_new();
+    if (!xattr) {
+        op_ret = -1;
+        op_errno = ENOMEM;
+        goto unwind;
+    }
 
-        op_errno = ENODATA;
-        if ((status == BR_VXATTR_STATUS_MISSING)
-            || (status == BR_VXATTR_STATUS_UNSIGNED))
-                goto delkeys;
+    stub.timebuf[0] = priv->boot[0];
+    stub.timebuf[1] = priv->boot[1];
+    memcpy(stub.export, priv->export, strlen(priv->export) + 1);
 
-        /**
-         * okay.. we have enough information to satisfy the request,
-         * namely: version and signing extended attribute. what's
-         * pending is the signature length -- that's figured out
-         * indirectly via the size of the _whole_ xattr and the
-         * on-disk signing xattr header size.
-         */
+    op_ret = dict_set_static_bin(xattr, GLUSTERFS_GET_BR_STUB_INIT_TIME,
+                                 (void *)&stub, sizeof(br_stub_init_t));
+    if (op_ret < 0) {
         op_errno = EINVAL;
-        ret = dict_get_uint32 (xattr, BITROT_SIGNING_XATTR_SIZE_KEY,
-                               (uint32_t *)&signaturelen);
-        if (ret)
-                goto delkeys;
+        goto unwind;
+    }
 
-        signaturelen -= sizeof (br_signature_t);
-        totallen = sizeof (br_isignature_out_t) + signaturelen;
+    op_ret = sizeof(br_stub_init_t);
 
-        op_errno = ENOMEM;
-        sign = GF_CALLOC (1, totallen, gf_br_stub_mt_signature_t);
-        if (!sign)
-                goto delkeys;
+unwind:
+    STACK_UNWIND_STRICT(getxattr, frame, op_ret, op_errno, xattr, NULL);
 
-        sign->time[0] = obuf->timebuf[0];
-        sign->time[1] = obuf->timebuf[1];
+    if (xattr)
+        dict_unref(xattr);
+}
 
-        /* Object's dirty state & current signed version */
-        sign->version = sbuf->signedversion;
-        sign->stale = (obuf->ongoingversion != sbuf->signedversion) ? 1 : 0;
+int
+br_stub_getxattr(call_frame_t *frame, xlator_t *this, loc_t *loc,
+                 const char *name, dict_t *xdata)
+{
+    void *cookie = NULL;
+    static uuid_t rootgfid = {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1};
+    fop_getxattr_cbk_t cbk = br_stub_getxattr_cbk;
+    int32_t op_ret = -1;
+    int32_t op_errno = EINVAL;
+    br_stub_local_t *local = NULL;
+    br_stub_private_t *priv = NULL;
+
+    GF_VALIDATE_OR_GOTO("bit-rot-stub", this, unwind);
+    GF_VALIDATE_OR_GOTO(this->name, loc, unwind);
+    GF_VALIDATE_OR_GOTO(this->name, this->private, unwind);
+    GF_VALIDATE_OR_GOTO(this->name, loc->inode, unwind);
+
+    if (!name) {
+        cbk = br_stub_listxattr_cbk;
+        goto wind;
+    }
+
+    if (br_stub_is_internal_xattr(name))
+        goto unwind;
+
+    priv = this->private;
+    BR_STUB_VER_NOT_ACTIVE_THEN_GOTO(frame, priv, wind);
+
+    /**
+     * If xattr is node-uuid and the inode is marked bad, return EIO.
+     * Returning EIO would result in AFR to choose correct node-uuid
+     * corresponding to the subvolume * where the good copy of the
+     * file resides.
+     */
+    if (IA_ISREG(loc->inode->ia_type) && XATTR_IS_NODE_UUID(name) &&
+        br_stub_check_bad_object(this, loc->inode, &op_ret, &op_errno)) {
+        goto unwind;
+    }
+
+    /**
+     * this special extended attribute is allowed only on root
+     */
+    if (name &&
+        (strncmp(name, GLUSTERFS_GET_BR_STUB_INIT_TIME,
+                 sizeof(GLUSTERFS_GET_BR_STUB_INIT_TIME) - 1) == 0) &&
+        ((gf_uuid_compare(loc->gfid, rootgfid) == 0) ||
+         (gf_uuid_compare(loc->inode->gfid, rootgfid) == 0))) {
+        BR_STUB_RESET_LOCAL_NULL(frame);
+        br_stub_send_stub_init_time(frame, this);
+        return 0;
+    }
 
-        /* Object's signature */
-        sign->signaturelen  = signaturelen;
-        sign->signaturetype = sbuf->signaturetype;
-        (void) memcpy (sign->signature, sbuf->signature, signaturelen);
+    if (!IA_ISREG(loc->inode->ia_type))
+        goto wind;
 
-        op_errno = EINVAL;
-        ret = dict_set_bin (xattr, GLUSTERFS_GET_OBJECT_SIGNATURE,
-                            (void *)sign, totallen);
-        if (ret < 0)
-                goto delkeys;
-        op_errno = 0;
-        op_ret = totallen;
+    if (name && (strncmp(name, GLUSTERFS_GET_OBJECT_SIGNATURE,
+                         sizeof(GLUSTERFS_GET_OBJECT_SIGNATURE) - 1) == 0)) {
+        cookie = (void *)BR_STUB_REQUEST_COOKIE;
 
- delkeys:
-        br_stub_remove_vxattrs (xattr);
+        local = br_stub_alloc_local(this);
+        if (!local) {
+            op_ret = -1;
+            op_errno = ENOMEM;
+            goto unwind;
+        }
 
- unwind:
-        STACK_UNWIND (frame, op_ret, op_errno, xattr, xdata);
-        return 0;
+        br_stub_fill_local(local, NULL, NULL, loc->inode, loc->inode->gfid,
+                           BR_STUB_NO_VERSIONING, 0);
+        frame->local = local;
+    }
+
+wind:
+    STACK_WIND_COOKIE(frame, cbk, cookie, FIRST_CHILD(this),
+                      FIRST_CHILD(this)->fops->getxattr, loc, name, xdata);
+    return 0;
+unwind:
+    BR_STUB_RESET_LOCAL_NULL(frame);
+    STACK_UNWIND_STRICT(getxattr, frame, op_ret, op_errno, NULL, NULL);
+    return 0;
 }
 
-static inline void
-br_stub_send_stub_init_time (call_frame_t *frame, xlator_t *this)
+int
+br_stub_fgetxattr(call_frame_t *frame, xlator_t *this, fd_t *fd,
+                  const char *name, dict_t *xdata)
 {
-        int op_ret = 0;
-        int op_errno = 0;
-        dict_t *xattr = NULL;
-        br_stub_init_t stub = {{0,},};
-        br_stub_private_t *priv = NULL;
+    void *cookie = NULL;
+    static uuid_t rootgfid = {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1};
+    fop_fgetxattr_cbk_t cbk = br_stub_getxattr_cbk;
+    int32_t op_ret = -1;
+    int32_t op_errno = EINVAL;
+    br_stub_local_t *local = NULL;
+    br_stub_private_t *priv = NULL;
+
+    priv = this->private;
+
+    if (!name) {
+        cbk = br_stub_listxattr_cbk;
+        goto wind;
+    }
+
+    if (br_stub_is_internal_xattr(name))
+        goto unwind;
+
+    BR_STUB_VER_NOT_ACTIVE_THEN_GOTO(frame, priv, wind);
+
+    /**
+     * If xattr is node-uuid and the inode is marked bad, return EIO.
+     * Returning EIO would result in AFR to choose correct node-uuid
+     * corresponding to the subvolume * where the good copy of the
+     * file resides.
+     */
+    if (IA_ISREG(fd->inode->ia_type) && XATTR_IS_NODE_UUID(name) &&
+        br_stub_check_bad_object(this, fd->inode, &op_ret, &op_errno)) {
+        goto unwind;
+    }
+
+    /**
+     * this special extended attribute is allowed only on root
+     */
+    if (name &&
+        (strncmp(name, GLUSTERFS_GET_BR_STUB_INIT_TIME,
+                 sizeof(GLUSTERFS_GET_BR_STUB_INIT_TIME) - 1) == 0) &&
+        (gf_uuid_compare(fd->inode->gfid, rootgfid) == 0)) {
+        BR_STUB_RESET_LOCAL_NULL(frame);
+        br_stub_send_stub_init_time(frame, this);
+        return 0;
+    }
+
+    if (!IA_ISREG(fd->inode->ia_type))
+        goto wind;
 
-        priv = this->private;
+    if (name && (strncmp(name, GLUSTERFS_GET_OBJECT_SIGNATURE,
+                         sizeof(GLUSTERFS_GET_OBJECT_SIGNATURE) - 1) == 0)) {
+        cookie = (void *)BR_STUB_REQUEST_COOKIE;
 
-        xattr = dict_new ();
-        if (!xattr) {
-                op_ret = -1;
-                op_errno = ENOMEM;
-                goto unwind;
+        local = br_stub_alloc_local(this);
+        if (!local) {
+            op_ret = -1;
+            op_errno = ENOMEM;
+            goto unwind;
         }
 
-        stub.timebuf[0] = priv->boot[0];
-        stub.timebuf[1] = priv->boot[1];
-        memcpy (stub.export, priv->export, strlen (priv->export) + 1);
+        br_stub_fill_local(local, NULL, fd, fd->inode, fd->inode->gfid,
+                           BR_STUB_NO_VERSIONING, 0);
+        frame->local = local;
+    }
+
+wind:
+    STACK_WIND_COOKIE(frame, cbk, cookie, FIRST_CHILD(this),
+                      FIRST_CHILD(this)->fops->fgetxattr, fd, name, xdata);
+    return 0;
+unwind:
+    BR_STUB_RESET_LOCAL_NULL(frame);
+    STACK_UNWIND_STRICT(fgetxattr, frame, op_ret, op_errno, NULL, NULL);
+    return 0;
+}
 
-        op_ret = dict_set_static_bin (xattr, GLUSTERFS_GET_BR_STUB_INIT_TIME,
-                                      (void *) &stub, sizeof (br_stub_init_t));
-        if (op_ret < 0) {
-                op_errno = EINVAL;
-                goto unwind;
-        }
+int32_t
+br_stub_readv(call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
+              off_t offset, uint32_t flags, dict_t *xdata)
+{
+    int32_t op_ret = -1;
+    int32_t op_errno = EINVAL;
+    int32_t ret = -1;
+    br_stub_private_t *priv = NULL;
+
+    GF_VALIDATE_OR_GOTO("bit-rot-stub", this, unwind);
+    GF_VALIDATE_OR_GOTO(this->name, frame, unwind);
+    GF_VALIDATE_OR_GOTO(this->name, this->private, unwind);
+    GF_VALIDATE_OR_GOTO(this->name, fd, unwind);
+    GF_VALIDATE_OR_GOTO(this->name, fd->inode, unwind);
+
+    priv = this->private;
+    if (!priv->do_versioning)
+        goto wind;
+
+    ret = br_stub_check_bad_object(this, fd->inode, &op_ret, &op_errno);
+    if (ret)
+        goto unwind;
+
+wind:
+    STACK_WIND_TAIL(frame, FIRST_CHILD(this), FIRST_CHILD(this)->fops->readv,
+                    fd, size, offset, flags, xdata);
+    return 0;
+
+unwind:
+    STACK_UNWIND_STRICT(readv, frame, op_ret, op_errno, NULL, 0, NULL, NULL,
+                        NULL);
+    return 0;
+}
+
+/**
+ * The first write response on the first fd in the list of fds will set
+ * the flag to indicate that the inode is modified. The subsequent write
+ * respnses coming on either the first fd or some other fd will not change
+ * the fd. The inode-modified flag is unset only upon release of all the
+ * fds.
+ */
+int32_t
+br_stub_writev_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                   int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
+                   struct iatt *postbuf, dict_t *xdata)
+{
+    int32_t ret = 0;
+    br_stub_local_t *local = NULL;
 
-        op_ret = sizeof (br_stub_init_t);
+    local = frame->local;
+    frame->local = NULL;
 
- unwind:
-        STACK_UNWIND (frame, op_ret, op_errno, xattr, NULL);
+    if (op_ret < 0)
+        goto unwind;
 
-        if (xattr)
-                dict_unref (xattr);
+    ret = br_stub_mark_inode_modified(this, local);
+    if (ret) {
+        op_ret = -1;
+        op_errno = EINVAL;
+    }
+
+unwind:
+    STACK_UNWIND_STRICT(writev, frame, op_ret, op_errno, prebuf, postbuf,
+                        xdata);
+
+    br_stub_cleanup_local(local);
+    br_stub_dealloc_local(local);
+
+    return 0;
 }
 
-int
-br_stub_getxattr (call_frame_t *frame, xlator_t *this,
-                  loc_t *loc, const char *name, dict_t *xdata)
+int32_t
+br_stub_writev_resume(call_frame_t *frame, xlator_t *this, fd_t *fd,
+                      struct iovec *vector, int32_t count, off_t offset,
+                      uint32_t flags, struct iobref *iobref, dict_t *xdata)
 {
-        void *cookie = NULL;
-        uuid_t rootgfid = {0, };
-        fop_getxattr_cbk_t cbk = br_stub_getxattr_cbk;
+    STACK_WIND(frame, br_stub_writev_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->writev, fd, vector, count, offset,
+               flags, iobref, xdata);
+    return 0;
+}
 
-        rootgfid[15] = 1;
+/**
+ * This is probably the most crucial part about the whole versioning thing.
+ * There's absolutely no differentiation as such between an anonymous fd
+ * and a regular fd except the fd context initialization. Object versioning
+ * is performed when the inode is dirty. Parallel write operations are no
+ * special with each write performing object versioning followed by marking
+ * the inode as non-dirty (synced). This is followed by the actual operation
+ * (writev() in this case) which on a success marks the inode as modified.
+ * This prevents signing of objects that have not been modified.
+ */
+int32_t
+br_stub_writev(call_frame_t *frame, xlator_t *this, fd_t *fd,
+               struct iovec *vector, int32_t count, off_t offset,
+               uint32_t flags, struct iobref *iobref, dict_t *xdata)
+{
+    call_stub_t *stub = NULL;
+    int32_t op_ret = -1;
+    int32_t op_errno = EINVAL;
+    gf_boolean_t inc_version = _gf_false;
+    gf_boolean_t modified = _gf_false;
+    br_stub_inode_ctx_t *ctx = NULL;
+    int32_t ret = -1;
+    fop_writev_cbk_t cbk = default_writev_cbk;
+    br_stub_local_t *local = NULL;
+    br_stub_private_t *priv = NULL;
+
+    GF_VALIDATE_OR_GOTO("bit-rot-stub", this, unwind);
+    GF_VALIDATE_OR_GOTO(this->name, this->private, unwind);
+    GF_VALIDATE_OR_GOTO(this->name, frame, unwind);
+    GF_VALIDATE_OR_GOTO(this->name, fd, unwind);
+
+    priv = this->private;
+    if (!priv->do_versioning)
+        goto wind;
+
+    ret = br_stub_need_versioning(this, fd, &inc_version, &modified, &ctx);
+    if (ret)
+        goto unwind;
+
+    ret = br_stub_check_bad_object(this, fd->inode, &op_ret, &op_errno);
+    if (ret)
+        goto unwind;
+
+    /**
+     * The inode is not dirty and also witnessed at least one successful
+     * modification operation. Therefore, subsequent operations need not
+     * perform any special tracking.
+     */
+    if (!inc_version && modified)
+        goto wind;
+
+    /**
+     * okay.. so, either the inode needs versioning or the modification
+     * needs to be tracked. ->cbk is set to the appropriate callback
+     * routine for this.
+     * NOTE: ->local needs to be deallocated on failures from here on.
+     */
+    ret = br_stub_versioning_prep(frame, this, fd, ctx);
+    if (ret)
+        goto unwind;
+
+    local = frame->local;
+    if (!inc_version) {
+        br_stub_fill_local(local, NULL, fd, fd->inode, fd->inode->gfid,
+                           BR_STUB_NO_VERSIONING, 0);
+        cbk = br_stub_writev_cbk;
+        goto wind;
+    }
+
+    stub = fop_writev_stub(frame, br_stub_writev_resume, fd, vector, count,
+                           offset, flags, iobref, xdata);
+
+    if (!stub) {
+        gf_smsg(this->name, GF_LOG_ERROR, 0, BRS_MSG_STUB_ALLOC_FAILED,
+                "write  gfid=%s", uuid_utoa(fd->inode->gfid), NULL);
+        goto cleanup_local;
+    }
+
+    /* Perform Versioning */
+    return br_stub_perform_incversioning(this, frame, stub, fd, ctx);
+
+wind:
+    STACK_WIND(frame, cbk, FIRST_CHILD(this), FIRST_CHILD(this)->fops->writev,
+               fd, vector, count, offset, flags, iobref, xdata);
+    return 0;
+
+cleanup_local:
+    br_stub_cleanup_local(local);
+    br_stub_dealloc_local(local);
+
+unwind:
+    frame->local = NULL;
+    STACK_UNWIND_STRICT(writev, frame, op_ret, op_errno, NULL, NULL, NULL);
+
+    return 0;
+}
 
-        if (!name) {
-                cbk = br_stub_listxattr_cbk;
-                goto wind;
-        }
+int32_t
+br_stub_ftruncate_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                      int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
+                      struct iatt *postbuf, dict_t *xdata)
+{
+    int32_t ret = -1;
+    br_stub_local_t *local = NULL;
 
-        if (br_stub_is_internal_xattr (name)) {
-                STACK_UNWIND (frame, -1, EINVAL, NULL, NULL);
-                return 0;
-        }
+    local = frame->local;
+    frame->local = NULL;
 
-        /**
-         * this special extended attribute is allowed only on root
-         */
-        if (name
-            && (strncmp (name, GLUSTERFS_GET_BR_STUB_INIT_TIME,
-                          strlen (GLUSTERFS_GET_BR_STUB_INIT_TIME)) == 0)
-            && ((gf_uuid_compare (loc->gfid, rootgfid) == 0)
-                || (gf_uuid_compare (loc->inode->gfid, rootgfid) == 0))) {
-                br_stub_send_stub_init_time (frame, this);
-                return 0;
-        }
+    if (op_ret < 0)
+        goto unwind;
 
-        if (!IA_ISREG (loc->inode->ia_type))
-                goto wind;
+    ret = br_stub_mark_inode_modified(this, local);
+    if (ret) {
+        op_ret = -1;
+        op_errno = EINVAL;
+    }
 
-        if (name && (strncmp (name, GLUSTERFS_GET_OBJECT_SIGNATURE,
-                              strlen (GLUSTERFS_GET_OBJECT_SIGNATURE)) == 0)) {
-                cookie = (void *) BR_STUB_REQUEST_COOKIE;
-        }
+unwind:
+    STACK_UNWIND_STRICT(ftruncate, frame, op_ret, op_errno, prebuf, postbuf,
+                        xdata);
 
- wind:
-        STACK_WIND_COOKIE
-                      (frame, cbk, cookie, FIRST_CHILD (this),
-                       FIRST_CHILD (this)->fops->getxattr, loc, name, xdata);
-        return 0;
+    br_stub_cleanup_local(local);
+    br_stub_dealloc_local(local);
+
+    return 0;
 }
 
-int
-br_stub_fgetxattr (call_frame_t *frame, xlator_t *this,
-                   fd_t *fd, const char *name, dict_t *xdata)
+int32_t
+br_stub_ftruncate_resume(call_frame_t *frame, xlator_t *this, fd_t *fd,
+                         off_t offset, dict_t *xdata)
 {
-        void *cookie = NULL;
-        uuid_t rootgfid = {0, };
-        fop_fgetxattr_cbk_t cbk = br_stub_getxattr_cbk;
+    STACK_WIND(frame, br_stub_ftruncate_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->ftruncate, fd, offset, xdata);
+    return 0;
+}
 
-        rootgfid[15] = 1;
+/* c.f. br_stub_writev() for explanation */
+int32_t
+br_stub_ftruncate(call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
+                  dict_t *xdata)
+{
+    br_stub_local_t *local = NULL;
+    call_stub_t *stub = NULL;
+    int32_t op_ret = -1;
+    int32_t op_errno = EINVAL;
+    gf_boolean_t inc_version = _gf_false;
+    gf_boolean_t modified = _gf_false;
+    br_stub_inode_ctx_t *ctx = NULL;
+    int32_t ret = -1;
+    fop_ftruncate_cbk_t cbk = default_ftruncate_cbk;
+    br_stub_private_t *priv = NULL;
+
+    GF_VALIDATE_OR_GOTO("bit-rot-stub", this, unwind);
+    GF_VALIDATE_OR_GOTO(this->name, this->private, unwind);
+    GF_VALIDATE_OR_GOTO(this->name, frame, unwind);
+    GF_VALIDATE_OR_GOTO(this->name, fd, unwind);
+
+    priv = this->private;
+    if (!priv->do_versioning)
+        goto wind;
+
+    ret = br_stub_need_versioning(this, fd, &inc_version, &modified, &ctx);
+    if (ret)
+        goto unwind;
+
+    ret = br_stub_check_bad_object(this, fd->inode, &op_ret, &op_errno);
+    if (ret)
+        goto unwind;
+
+    if (!inc_version && modified)
+        goto wind;
+
+    ret = br_stub_versioning_prep(frame, this, fd, ctx);
+    if (ret)
+        goto unwind;
+
+    local = frame->local;
+    if (!inc_version) {
+        br_stub_fill_local(local, NULL, fd, fd->inode, fd->inode->gfid,
+                           BR_STUB_NO_VERSIONING, 0);
+        cbk = br_stub_ftruncate_cbk;
+        goto wind;
+    }
+
+    stub = fop_ftruncate_stub(frame, br_stub_ftruncate_resume, fd, offset,
+                              xdata);
+    if (!stub) {
+        gf_smsg(this->name, GF_LOG_ERROR, 0, BRS_MSG_STUB_ALLOC_FAILED,
+                "ftruncate gfid=%s", uuid_utoa(fd->inode->gfid), NULL);
+        goto cleanup_local;
+    }
+
+    return br_stub_perform_incversioning(this, frame, stub, fd, ctx);
+
+wind:
+    STACK_WIND(frame, cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->ftruncate, fd, offset, xdata);
+    return 0;
+
+cleanup_local:
+    br_stub_cleanup_local(local);
+    br_stub_dealloc_local(local);
+
+unwind:
+    frame->local = NULL;
+    STACK_UNWIND_STRICT(ftruncate, frame, op_ret, op_errno, NULL, NULL, NULL);
+
+    return 0;
+}
 
-        if (!name) {
-                cbk = br_stub_listxattr_cbk;
-                goto wind;
-        }
+int32_t
+br_stub_truncate_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                     int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
+                     struct iatt *postbuf, dict_t *xdata)
+{
+    int32_t ret = 0;
+    br_stub_local_t *local = NULL;
 
-        if (br_stub_is_internal_xattr (name)) {
-                STACK_UNWIND (frame, -1, EINVAL, NULL, NULL);
-                return 0;
-        }
+    local = frame->local;
+    frame->local = NULL;
 
-        /**
-         * this special extended attribute is allowed only on root
-         */
-        if (name
-            && (strncmp (name, GLUSTERFS_GET_BR_STUB_INIT_TIME,
-                         strlen (GLUSTERFS_GET_BR_STUB_INIT_TIME)) == 0)
-            && (gf_uuid_compare (fd->inode->gfid, rootgfid) == 0)) {
-                br_stub_send_stub_init_time (frame, this);
-                return 0;
-        }
+    if (op_ret < 0)
+        goto unwind;
 
-        if (!IA_ISREG (fd->inode->ia_type))
-                goto wind;
+    ret = br_stub_mark_inode_modified(this, local);
+    if (ret) {
+        op_ret = -1;
+        op_errno = EINVAL;
+    }
+
+unwind:
+    STACK_UNWIND_STRICT(truncate, frame, op_ret, op_errno, prebuf, postbuf,
+                        xdata);
+    br_stub_cleanup_local(local);
+    br_stub_dealloc_local(local);
+    return 0;
+}
 
-        if (name && (strncmp (name, GLUSTERFS_GET_OBJECT_SIGNATURE,
-                              strlen (GLUSTERFS_GET_OBJECT_SIGNATURE)) == 0)) {
-                cookie = (void *) BR_STUB_REQUEST_COOKIE;
-        }
+int32_t
+br_stub_truncate_resume(call_frame_t *frame, xlator_t *this, loc_t *loc,
+                        off_t offset, dict_t *xdata)
+{
+    br_stub_local_t *local = frame->local;
 
- wind:
-        STACK_WIND_COOKIE
-                      (frame, cbk, cookie, FIRST_CHILD (this),
-                       FIRST_CHILD (this)->fops->fgetxattr, fd, name, xdata);
-        return 0;
+    fd_unref(local->u.context.fd);
+    STACK_WIND(frame, br_stub_ftruncate_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->truncate, loc, offset, xdata);
+    return 0;
 }
 
-/** }}} */
+/**
+ * Bit-rot-stub depends heavily on the fd based operations to for doing
+ * versioning and sending notification. It starts tracking the operation
+ * upon getting first fd based modify operation by doing versioning and
+ * sends notification when last fd using which the inode was modified is
+ * released.
+ * But for truncate there is no fd and hence it becomes difficult to do
+ * the versioning and send notification. It is handled by doing versioning
+ * on an anonymous fd. The fd will be valid till the completion of the
+ * truncate call. It guarantees that release on this anonymous fd will happen
+ * after the truncate call and notification is sent after the truncate call.
+ *
+ * c.f. br_writev_cbk() for explanation
+ */
+int32_t
+br_stub_truncate(call_frame_t *frame, xlator_t *this, loc_t *loc, off_t offset,
+                 dict_t *xdata)
+{
+    br_stub_local_t *local = NULL;
+    call_stub_t *stub = NULL;
+    int32_t op_ret = -1;
+    int32_t op_errno = EINVAL;
+    gf_boolean_t inc_version = _gf_false;
+    gf_boolean_t modified = _gf_false;
+    br_stub_inode_ctx_t *ctx = NULL;
+    int32_t ret = -1;
+    fd_t *fd = NULL;
+    fop_truncate_cbk_t cbk = default_truncate_cbk;
+    br_stub_private_t *priv = NULL;
+
+    GF_VALIDATE_OR_GOTO("bit-rot-stub", this, unwind);
+    GF_VALIDATE_OR_GOTO(this->name, this->private, unwind);
+    GF_VALIDATE_OR_GOTO(this->name, frame, unwind);
+    GF_VALIDATE_OR_GOTO(this->name, loc, unwind);
+    GF_VALIDATE_OR_GOTO(this->name, loc->inode, unwind);
+
+    priv = this->private;
+    if (!priv->do_versioning)
+        goto wind;
+
+    fd = fd_anonymous(loc->inode);
+    if (!fd) {
+        gf_smsg(this->name, GF_LOG_ERROR, 0, BRS_MSG_CREATE_ANONYMOUS_FD_FAILED,
+                "inode-gfid=%s", uuid_utoa(loc->inode->gfid), NULL);
+        goto unwind;
+    }
+
+    ret = br_stub_need_versioning(this, fd, &inc_version, &modified, &ctx);
+    if (ret)
+        goto cleanup_fd;
+
+    ret = br_stub_check_bad_object(this, fd->inode, &op_ret, &op_errno);
+    if (ret)
+        goto unwind;
+
+    if (!inc_version && modified)
+        goto wind;
+
+    ret = br_stub_versioning_prep(frame, this, fd, ctx);
+    if (ret)
+        goto cleanup_fd;
+
+    local = frame->local;
+    if (!inc_version) {
+        br_stub_fill_local(local, NULL, fd, fd->inode, fd->inode->gfid,
+                           BR_STUB_NO_VERSIONING, 0);
+        cbk = br_stub_truncate_cbk;
+        goto wind;
+    }
+
+    stub = fop_truncate_stub(frame, br_stub_truncate_resume, loc, offset,
+                             xdata);
+    if (!stub) {
+        gf_smsg(this->name, GF_LOG_ERROR, 0, BRS_MSG_STUB_ALLOC_FAILED,
+                "truncate gfid=%s", uuid_utoa(fd->inode->gfid), NULL);
+        goto cleanup_local;
+    }
+
+    return br_stub_perform_incversioning(this, frame, stub, fd, ctx);
+
+wind:
+    STACK_WIND(frame, cbk, FIRST_CHILD(this), FIRST_CHILD(this)->fops->truncate,
+               loc, offset, xdata);
+    if (fd)
+        fd_unref(fd);
+    return 0;
+
+cleanup_local:
+    br_stub_cleanup_local(local);
+    br_stub_dealloc_local(local);
+cleanup_fd:
+    fd_unref(fd);
+unwind:
+    frame->local = NULL;
+    STACK_UNWIND_STRICT(truncate, frame, op_ret, op_errno, NULL, NULL, NULL);
+
+    return 0;
+}
 
+/** }}} */
 
 /** {{{ */
 
 /* open() */
 
+/**
+ * It's probably worth mentioning a bit about why some of the housekeeping
+ * work is done in open() call path, rather than the callback path.
+ * Two (or more) open()'s in parallel can race and lead to a situation
+ * where a release() gets triggered (possibly after a series of write()
+ * calls) when *other* open()'s have still not reached callback path
+ * thereby having an active fd on an inode that is in process of getting
+ * signed with the current version.
+ *
+ * Maintaining fd list in the call path ensures that a release() would
+ * not be triggered if an open() call races ahead (followed by a close())
+ * threby finding non-empty fd list.
+ */
+
 int
-br_stub_open_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                  int op_ret, int op_errno, fd_t *fd, dict_t *xdata)
+br_stub_open(call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t flags,
+             fd_t *fd, dict_t *xdata)
 {
-        int32_t              ret      = 0;
-        uint64_t             ctx_addr = 0;
-        br_stub_inode_ctx_t *ctx      = NULL;
-        call_stub_t         *stub     = NULL;
+    int32_t ret = -1;
+    br_stub_inode_ctx_t *ctx = NULL;
+    uint64_t ctx_addr = 0;
+    int32_t op_ret = -1;
+    int32_t op_errno = EINVAL;
+    br_stub_private_t *priv = NULL;
+    unsigned long version = BITROT_DEFAULT_CURRENT_VERSION;
+
+    GF_VALIDATE_OR_GOTO("bit-rot-stub", this, unwind);
+    GF_VALIDATE_OR_GOTO(this->name, this->private, unwind);
+    GF_VALIDATE_OR_GOTO(this->name, loc, unwind);
+    GF_VALIDATE_OR_GOTO(this->name, fd, unwind);
+    GF_VALIDATE_OR_GOTO(this->name, fd->inode, unwind);
+
+    priv = this->private;
+
+    if (!priv->do_versioning)
+        goto wind;
+
+    ret = br_stub_get_inode_ctx(this, fd->inode, &ctx_addr);
+    if (ret) {
+        ret = br_stub_init_inode_versions(this, fd, fd->inode, version,
+                                          _gf_true, _gf_false, &ctx_addr);
+        if (ret) {
+            gf_smsg(this->name, GF_LOG_ERROR, 0,
+                    BRS_MSG_GET_INODE_CONTEXT_FAILED, "path=%s", loc->path,
+                    "gfid=%s", uuid_utoa(fd->inode->gfid), NULL);
+            goto unwind;
+        }
+    }
+
+    ctx = (br_stub_inode_ctx_t *)(long)ctx_addr;
+
+    ret = br_stub_check_bad_object(this, fd->inode, &op_ret, &op_errno);
+    if (ret)
+        goto unwind;
+
+    if (frame->root->pid == GF_CLIENT_PID_SCRUB)
+        goto wind;
+
+    if (flags == O_RDONLY)
+        goto wind;
+
+    ret = br_stub_add_fd_to_inode(this, fd, ctx);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_ERROR, 0, BRS_MSG_ADD_FD_TO_LIST_FAILED,
+                "gfid=%s", uuid_utoa(fd->inode->gfid), NULL);
+        goto unwind;
+    }
+
+wind:
+    STACK_WIND(frame, default_open_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->open, loc, flags, fd, xdata);
+    return 0;
+unwind:
+    STACK_UNWIND_STRICT(open, frame, op_ret, op_errno, NULL, NULL);
+    return 0;
+}
 
-        if (op_ret < 0)
-                goto unwind;
-        if (cookie != (void *) BR_STUB_REQUEST_COOKIE)
-                goto unwind;
+/** }}} */
 
-        ret = br_stub_get_inode_ctx (this, fd->inode, &ctx_addr);
-        if (ret < 0)
-                goto unwind;
+/** {{{ */
 
-        stub = fop_open_cbk_stub (frame, NULL, op_ret, op_errno, fd, xdata);
-        if (!stub) {
-                op_ret = -1;
-                op_errno = EINVAL;
-                goto unwind;
-        }
+/* creat() */
 
-        /**
-         * Ongoing version needs to be incremented. If the inode is not dirty,
-         * things are simple: increment the ongoing version safely and be done.
-         * If inode is dirty, a writeback to disk is required. This is tricky in
-         * case of multiple open()'s as ongoing version needs to be incremented
-         * on a successful writeback. It's probably safe to remember the ongoing
-         * version before writeback and *assigning* it in the callback, but that
-         * may lead to a trustable checksum to be treated as stale by scrubber
-         * (the case where the in-memory ongoing version is lesser than the
-         * on-disk version). Therefore, *all* open() calls (which might have
-         * come in parallel) try to synchronize the next ongoing version to
-         * disk. In the callback path, the winner marks the inode as synced
-         * therby loosing open() calls become no-op's.
-         */
-        ctx = (br_stub_inode_ctx_t *) (long) ctx_addr;
-        return br_stub_perform_incversioning (this, frame, stub, fd, ctx);
+/**
+ * This routine registers a release callback for the given fd and adds the
+ * fd to the inode context fd tracking list.
+ */
+int32_t
+br_stub_add_fd_to_inode(xlator_t *this, fd_t *fd, br_stub_inode_ctx_t *ctx)
+{
+    int32_t ret = -1;
+    br_stub_fd_t *br_stub_fd = NULL;
+
+    ret = br_stub_require_release_call(this, fd, &br_stub_fd);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_ERROR, 0, BRS_MSG_SET_FD_CONTEXT_FAILED,
+                "gfid=%s", uuid_utoa(fd->inode->gfid), NULL);
+        goto out;
+    }
+
+    LOCK(&fd->inode->lock);
+    {
+        list_add_tail(&ctx->fd_list, &br_stub_fd->list);
+    }
+    UNLOCK(&fd->inode->lock);
+
+    ret = 0;
+
+out:
+    return ret;
+}
 
- unwind:
-        STACK_UNWIND_STRICT (open, frame,
-                             op_ret, op_errno, fd, xdata);
-        return 0;
+int
+br_stub_create_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                   int op_ret, int op_errno, fd_t *fd, inode_t *inode,
+                   struct iatt *stbuf, struct iatt *preparent,
+                   struct iatt *postparent, dict_t *xdata)
+{
+    int32_t ret = 0;
+    uint64_t ctx_addr = 0;
+    br_stub_inode_ctx_t *ctx = NULL;
+    unsigned long version = BITROT_DEFAULT_CURRENT_VERSION;
+    br_stub_private_t *priv = NULL;
+
+    priv = this->private;
+
+    if (op_ret < 0)
+        goto unwind;
+
+    if (!priv->do_versioning)
+        goto unwind;
+
+    ret = br_stub_get_inode_ctx(this, fd->inode, &ctx_addr);
+    if (ret < 0) {
+        ret = br_stub_init_inode_versions(this, fd, inode, version, _gf_true,
+                                          _gf_false, &ctx_addr);
+        if (ret) {
+            op_ret = -1;
+            op_errno = EINVAL;
+        }
+    } else {
+        ctx = (br_stub_inode_ctx_t *)(long)ctx_addr;
+        ret = br_stub_add_fd_to_inode(this, fd, ctx);
+    }
+
+unwind:
+    STACK_UNWIND_STRICT(create, frame, op_ret, op_errno, fd, inode, stbuf,
+                        preparent, postparent, xdata);
+    return 0;
 }
 
 int
-br_stub_open (call_frame_t *frame, xlator_t *this,
-              loc_t *loc, int32_t flags, fd_t *fd, dict_t *xdata)
+br_stub_create(call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t flags,
+               mode_t mode, mode_t umask, fd_t *fd, dict_t *xdata)
 {
-        void *cookie = NULL;
+    GF_VALIDATE_OR_GOTO("bit-rot-stub", this, unwind);
+    GF_VALIDATE_OR_GOTO(this->name, loc, unwind);
+    GF_VALIDATE_OR_GOTO(this->name, loc->inode, unwind);
+    GF_VALIDATE_OR_GOTO(this->name, fd, unwind);
+    GF_VALIDATE_OR_GOTO(this->name, fd->inode, unwind);
+
+    STACK_WIND(frame, br_stub_create_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->create, loc, flags, mode, umask, fd,
+               xdata);
+    return 0;
+unwind:
+    STACK_UNWIND_STRICT(create, frame, -1, EINVAL, NULL, NULL, NULL, NULL, NULL,
+                        NULL);
+    return 0;
+}
 
-        if (!flags)
-                goto wind;
-        if (frame->root->pid == GF_CLIENT_PID_SCRUB)
-                goto wind;
-        cookie = (void *) BR_STUB_REQUEST_COOKIE;
+int
+br_stub_mknod_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int op_ret,
+                  int op_errno, inode_t *inode, struct iatt *stbuf,
+                  struct iatt *preparent, struct iatt *postparent,
+                  dict_t *xdata)
+{
+    int32_t ret = -1;
+    unsigned long version = BITROT_DEFAULT_CURRENT_VERSION;
+    br_stub_private_t *priv = NULL;
 
- wind:
-        STACK_WIND_COOKIE (frame, br_stub_open_cbk, cookie,
-                           FIRST_CHILD (this), FIRST_CHILD (this)->fops->open,
-                           loc, flags, fd, xdata);
-        return 0;
+    priv = this->private;
+
+    if (op_ret < 0)
+        goto unwind;
+
+    if (!priv->do_versioning)
+        goto unwind;
+
+    ret = br_stub_init_inode_versions(this, NULL, inode, version, _gf_true,
+                                      _gf_false, NULL);
+    /**
+     * Like lookup, if init_inode_versions fail, return EINVAL
+     */
+    if (ret) {
+        op_ret = -1;
+        op_errno = EINVAL;
+    }
+
+unwind:
+    STACK_UNWIND_STRICT(mknod, frame, op_ret, op_errno, inode, stbuf, preparent,
+                        postparent, xdata);
+    return 0;
+}
+
+int
+br_stub_mknod(call_frame_t *frame, xlator_t *this, loc_t *loc, mode_t mode,
+              dev_t dev, mode_t umask, dict_t *xdata)
+{
+    GF_VALIDATE_OR_GOTO("bit-rot-stub", this, unwind);
+    GF_VALIDATE_OR_GOTO(this->name, loc, unwind);
+    GF_VALIDATE_OR_GOTO(this->name, loc->inode, unwind);
+
+    STACK_WIND(frame, br_stub_mknod_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->mknod, loc, mode, dev, umask, xdata);
+    return 0;
+unwind:
+    STACK_UNWIND_STRICT(mknod, frame, -1, EINVAL, NULL, NULL, NULL, NULL, NULL);
+    return 0;
 }
 
 /** }}} */
 
+/**
+ * As of now, only lookup searches for bad object xattr and marks the
+ * object as bad in its inode context if the xattr is present. But there
+ * is a possibility that, at the time of the lookup the object was not
+ * marked bad (i.e. bad object xattr was not set), and later its marked
+ * as bad. In this case, object is not bad, so when a fop such as open or
+ * readv or writev comes on the object, the fop will be sent downward instead
+ * of sending as error upwards.
+ * The solution for this is to do a getxattr for the below list of fops.
+ * lookup, readdirp, open, readv, writev.
+ * But doing getxattr for each of the above fops might be costly.
+ * So another method followed is to catch the bad file marking by the scrubber
+ * and set that info within the object's inode context. In this way getxattr
+ * calls can be avoided and bad objects can be caught instantly. Fetching the
+ * xattr is needed only in lookups when there is a brick restart or inode
+ * forget.
+ *
+ * If the dict (@xattr) is NULL, then how should that be handled? Fail the
+ * lookup operation? Or let it continue with version being initialized to
+ * BITROT_DEFAULT_CURRENT_VERSION. But what if the version was different
+ * on disk (and also a right signature was there), but posix failed to
+ * successfully allocate the dict? Posix does not treat call back xdata
+ * creattion failure as the lookup failure.
+ */
+static int32_t
+br_stub_lookup_version(xlator_t *this, uuid_t gfid, inode_t *inode,
+                       dict_t *xattr)
+{
+    unsigned long version = 0;
+    br_version_t *obuf = NULL;
+    br_signature_t *sbuf = NULL;
+    br_vxattr_status_t status;
+    gf_boolean_t bad_object = _gf_false;
+
+    /**
+     * versioning xattrs were requested from POSIX. if available, figure
+     * out the correct version to use in the inode context (start with
+     * the default version if unavailable). As of now versions are not
+     * persisted on-disk. The inode is marked dirty, so that the first
+     * operation (such as write(), etc..) triggers synchronization to
+     * disk.
+     */
+    status = br_version_xattr_state(xattr, &obuf, &sbuf, &bad_object);
+    version = ((status == BR_VXATTR_STATUS_FULL) ||
+               (status == BR_VXATTR_STATUS_UNSIGNED))
+                  ? obuf->ongoingversion
+                  : BITROT_DEFAULT_CURRENT_VERSION;
+
+    /**
+     * If signature is there, but version is not there then that status is
+     * is treated as INVALID. So in that case, we should not initialize the
+     * inode context with wrong version names etc.
+     */
+    if (status == BR_VXATTR_STATUS_INVALID)
+        return -1;
+
+    return br_stub_init_inode_versions(this, NULL, inode, version, _gf_true,
+                                       bad_object, NULL);
+}
 
 /** {{{ */
 
-/* creat() */
+int32_t
+br_stub_opendir(call_frame_t *frame, xlator_t *this, loc_t *loc, fd_t *fd,
+                dict_t *xdata)
+{
+    br_stub_private_t *priv = NULL;
+    br_stub_fd_t *fd_ctx = NULL;
+    int32_t op_ret = -1;
+    int32_t op_errno = EINVAL;
 
-int
-br_stub_create_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                    int op_ret, int op_errno, fd_t *fd, inode_t *inode,
-                    struct iatt *stbuf, struct iatt *preparent,
-                    struct iatt *postparent, dict_t *xdata)
+    priv = this->private;
+    if (gf_uuid_compare(fd->inode->gfid, priv->bad_object_dir_gfid))
+        goto normal;
+
+    fd_ctx = br_stub_fd_new();
+    if (!fd_ctx) {
+        op_errno = ENOMEM;
+        goto unwind;
+    }
+
+    fd_ctx->bad_object.dir_eof = -1;
+    fd_ctx->bad_object.dir = sys_opendir(priv->stub_basepath);
+    if (!fd_ctx->bad_object.dir) {
+        op_errno = errno;
+        goto err_freectx;
+    }
+
+    op_ret = br_stub_fd_ctx_set(this, fd, fd_ctx);
+    if (!op_ret)
+        goto unwind;
+
+    sys_closedir(fd_ctx->bad_object.dir);
+
+err_freectx:
+    GF_FREE(fd_ctx);
+unwind:
+    STACK_UNWIND_STRICT(opendir, frame, op_ret, op_errno, fd, NULL);
+    return 0;
+
+normal:
+    STACK_WIND(frame, default_opendir_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->opendir, loc, fd, xdata);
+    return 0;
+}
+
+int32_t
+br_stub_readdir(call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
+                off_t off, dict_t *xdata)
 {
-        int32_t ret = 0;
-        uint64_t ctx_addr = 0;
-        call_stub_t *stub = NULL;
-        br_stub_inode_ctx_t *ctx = NULL;
+    call_stub_t *stub = NULL;
+    br_stub_private_t *priv = NULL;
+
+    priv = this->private;
+    if (!priv->do_versioning)
+        goto out;
+
+    if (gf_uuid_compare(fd->inode->gfid, priv->bad_object_dir_gfid))
+        goto out;
+    stub = fop_readdir_stub(frame, br_stub_readdir_wrapper, fd, size, off,
+                            xdata);
+    if (!stub) {
+        STACK_UNWIND_STRICT(readdir, frame, -1, ENOMEM, NULL, NULL);
+        return 0;
+    }
+    br_stub_worker_enqueue(this, stub);
+    return 0;
+out:
+    STACK_WIND(frame, default_readdir_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->readdir, fd, size, off, xdata);
+    return 0;
+}
 
-        if (op_ret < 0)
-                goto unwind;
+int
+br_stub_readdirp_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                     int op_ret, int op_errno, gf_dirent_t *entries,
+                     dict_t *dict)
+{
+    int32_t ret = 0;
+    uint64_t ctxaddr = 0;
+    gf_dirent_t *entry = NULL;
+    br_stub_private_t *priv = NULL;
+    gf_boolean_t ver_enabled = _gf_false;
+
+    BR_STUB_VER_ENABLED_IN_CALLPATH(frame, ver_enabled);
+    priv = this->private;
+    BR_STUB_VER_COND_GOTO(priv, (!ver_enabled), unwind);
+
+    if (op_ret < 0)
+        goto unwind;
+
+    list_for_each_entry(entry, &entries->list, list)
+    {
+        if ((strcmp(entry->d_name, ".") == 0) ||
+            (strcmp(entry->d_name, "..") == 0))
+            continue;
+
+        if (!IA_ISREG(entry->d_stat.ia_type))
+            continue;
+
+        /*
+         * Readdirp for most part is a bulk lookup for all the entries
+         * present in the directory being read. Ideally, for each
+         * entry, the handling should be similar to that of a lookup
+         * callback. But for now, just keeping this as it has been
+         * until now (which means, this comment has been added much
+         * later as part of a change that wanted to send the flag
+         * of true/false to br_stub_remove_vxattrs to indicate whether
+         * the bad-object xattr should be removed from the entry->dict
+         * or not). Until this change, the function br_stub_remove_vxattrs
+         * was just removing all the xattrs associated with bit-rot-stub
+         * (like version, bad-object, signature etc). But, there are
+         * scenarios where we only want to send bad-object xattr and not
+         * others. So this comment is part of that change which also
+         * mentions about another possible change that might be needed
+         * in future.
+         * But for now, adding _gf_true means functionally its same as
+         * what this function was doing before. Just remove all the stub
+         * related xattrs.
+         */
+        ret = br_stub_get_inode_ctx(this, entry->inode, &ctxaddr);
+        if (ret < 0)
+            ctxaddr = 0;
+        if (ctxaddr) { /* already has the context */
+            br_stub_remove_vxattrs(entry->dict, _gf_true);
+            continue;
+        }
 
-        stub = fop_create_cbk_stub (frame, NULL, op_ret, op_errno, fd, inode,
-                                    stbuf, preparent, postparent, xdata);
-        if (!stub) {
-                op_ret = -1;
-                op_errno = EINVAL;
-                goto unwind;
+        ret = br_stub_lookup_version(this, entry->inode->gfid, entry->inode,
+                                     entry->dict);
+        br_stub_remove_vxattrs(entry->dict, _gf_true);
+        if (ret) {
+            /**
+             * there's no per-file granularity support in case of
+             * failure. let's fail the entire request for now..
+             */
+            break;
         }
+    }
 
-        ret = br_stub_get_inode_ctx (this, fd->inode, &ctx_addr);
-        if (ret < 0)
-                ctx_addr = 0;
-        ctx = (br_stub_inode_ctx_t *) (long) ctx_addr;
+    if (ret) {
+        op_ret = -1;
+        op_errno = EINVAL;
+    }
 
-        /* see comment in br_stub_open_cbk().. */
-        return (ctx)
-                ? br_stub_perform_incversioning (this, frame, stub, fd, ctx)
-                : br_stub_perform_fullversioning (this, frame, stub, fd);
+unwind:
+    STACK_UNWIND_STRICT(readdirp, frame, op_ret, op_errno, entries, dict);
 
- unwind:
-        STACK_UNWIND_STRICT (create, frame, op_ret, op_errno,
-                             fd, inode, stbuf, preparent, postparent, xdata);
-        return 0;
+    return 0;
 }
 
 int
-br_stub_create (call_frame_t *frame,
-                xlator_t *this, loc_t *loc, int32_t flags,
-                mode_t mode, mode_t umask, fd_t *fd, dict_t *xdata)
+br_stub_readdirp(call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
+                 off_t offset, dict_t *dict)
 {
-        STACK_WIND (frame, br_stub_create_cbk, FIRST_CHILD (this),
-                    FIRST_CHILD (this)->fops->create,
-                    loc, flags, mode, umask, fd, xdata);
-        return 0;
+    int32_t ret = -1;
+    int op_errno = 0;
+    gf_boolean_t xref = _gf_false;
+    br_stub_private_t *priv = NULL;
+
+    priv = this->private;
+    BR_STUB_VER_NOT_ACTIVE_THEN_GOTO(frame, priv, wind);
+
+    op_errno = ENOMEM;
+    if (!dict) {
+        dict = dict_new();
+        if (!dict)
+            goto unwind;
+    } else {
+        dict = dict_ref(dict);
+    }
+
+    xref = _gf_true;
+
+    op_errno = EINVAL;
+    ret = dict_set_uint32(dict, BITROT_CURRENT_VERSION_KEY, 0);
+    if (ret)
+        goto unwind;
+    ret = dict_set_uint32(dict, BITROT_SIGNING_VERSION_KEY, 0);
+    if (ret)
+        goto unwind;
+    ret = dict_set_uint32(dict, BITROT_OBJECT_BAD_KEY, 0);
+    if (ret)
+        goto unwind;
+
+wind:
+    STACK_WIND(frame, br_stub_readdirp_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->readdirp, fd, size, offset, dict);
+    goto unref_dict;
+
+unwind:
+    if (frame->local == (void *)0x1)
+        frame->local = NULL;
+    STACK_UNWIND_STRICT(readdirp, frame, -1, op_errno, NULL, NULL);
+    return 0;
+
+unref_dict:
+    if (xref)
+        dict_unref(dict);
+    return 0;
 }
 
 /** }}} */
 
-static inline int32_t
-br_stub_lookup_version (xlator_t *this,
-                        uuid_t gfid, inode_t *inode, dict_t *xattr)
+/** {{{ */
+
+/* lookup() */
+
+/**
+ * This function mainly handles the ENOENT error for the bad objects. Though
+ * br_stub_forget () handles removal of the link for the bad object from the
+ * quarantine directory, its better to handle it in lookup as well, where
+ * a failed lookup on a bad object with ENOENT, will trigger deletion of the
+ * link for the bad object from quarantine directory. So whoever comes first
+ * either forget () or lookup () will take care of removing the link.
+ */
+void
+br_stub_handle_lookup_error(xlator_t *this, inode_t *inode, int32_t op_errno)
 {
-        unsigned long       version = 0;
-        br_version_t       *obuf    = NULL;
-        br_signature_t     *sbuf    = NULL;
-        br_vxattr_status_t  status;
+    int32_t ret = -1;
+    uint64_t ctx_addr = 0;
+    br_stub_inode_ctx_t *ctx = NULL;
 
-        /**
-         * versioning xattrs were requested from POSIX. if available, figure
-         * out the correct version to use in the inode context (start with
-         * the default version if unavailable). As of now versions are not
-         * persisted on-disk. The inode is marked dirty, so that the first
-         * operation (such as open(), etc..) would trigger synchronization
-         * to disk.
-         */
-        status = br_version_xattr_state (xattr, &obuf, &sbuf);
+    if (op_errno != ENOENT)
+        goto out;
 
-        /**
-         * stub does not know how to handle presence of signature but not
-         * the object version, therefore, in such cases, bail out..
+    if (!inode_is_linked(inode))
+        goto out;
+
+    ret = br_stub_get_inode_ctx(this, inode, &ctx_addr);
+    if (ret)
+        goto out;
+
+    ctx = (br_stub_inode_ctx_t *)(long)ctx_addr;
+
+    LOCK(&inode->lock);
+    {
+        if (__br_stub_is_bad_object(ctx))
+            (void)br_stub_del(this, inode->gfid);
+    }
+    UNLOCK(&inode->lock);
+
+    if (__br_stub_is_bad_object(ctx)) {
+        /* File is not present, might be deleted for recovery,
+         * del the bitrot inode context
          */
-        if (status == BR_VXATTR_STATUS_INVALID) {
-                gf_log (this->name, GF_LOG_ERROR, "Invalid versioning xattrs. "
-                        "Bailing out [GFID: %s]", uuid_utoa (gfid));
-                return -1;
+        ctx_addr = 0;
+        inode_ctx_del(inode, this, &ctx_addr);
+        if (ctx_addr) {
+            ctx = (br_stub_inode_ctx_t *)(long)ctx_addr;
+            GF_FREE(ctx);
         }
+    }
 
-        version = ((status == BR_VXATTR_STATUS_FULL)
-                   || (status == BR_VXATTR_STATUS_UNSIGNED))
-                        ? obuf->ongoingversion : BITROT_DEFAULT_CURRENT_VERSION;
-        return br_stub_init_inode_versions (this, NULL,
-                                            inode, version, _gf_true);
+out:
+    return;
 }
 
+int
+br_stub_lookup_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                   int op_ret, int op_errno, inode_t *inode, struct iatt *stbuf,
+                   dict_t *xattr, struct iatt *postparent)
+{
+    int32_t ret = 0;
+    br_stub_private_t *priv = NULL;
+    gf_boolean_t ver_enabled = _gf_false;
+    gf_boolean_t remove_bad_file_marker = _gf_true;
 
-/** {{{ */
+    BR_STUB_VER_ENABLED_IN_CALLPATH(frame, ver_enabled);
+    priv = this->private;
 
-int
-br_stub_readdirp_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                      int op_ret, int op_errno, gf_dirent_t *entries,
-                      dict_t *dict)
-{
-        int32_t      ret     = 0;
-        uint64_t     ctxaddr = 0;
-        gf_dirent_t *entry   = NULL;
-
-        if (op_ret < 0)
-                goto unwind;
-
-        list_for_each_entry (entry, &entries->list, list) {
-                if ((strcmp (entry->d_name, ".") == 0)
-                    || (strcmp (entry->d_name, "..") == 0))
-                        continue;
-
-                if (!IA_ISREG (entry->d_stat.ia_type))
-                        continue;
-
-                ret = br_stub_get_inode_ctx (this, entry->inode, &ctxaddr);
-                if (ret < 0)
-                        ctxaddr = 0;
-                if (ctxaddr) { /* already has the context */
-                        br_stub_remove_vxattrs (entry->dict);
-                        continue;
-                }
-
-                ret = br_stub_lookup_version
-                        (this, entry->inode->gfid, entry->inode, entry->dict);
-                br_stub_remove_vxattrs (entry->dict);
-                if (ret) {
-                        /**
-                         * there's no per-file granularity support in case of
-                         * failure. let's fail the entire request for now..
-                         */
-                        break;
-                }
-        }
+    if (op_ret < 0) {
+        (void)br_stub_handle_lookup_error(this, inode, op_errno);
 
+        /*
+         * If the lookup error is not ENOENT, then it is better
+         * to send the bad file marker to the higher layer (if
+         * it has been set)
+         */
+        if (op_errno != ENOENT)
+            remove_bad_file_marker = _gf_false;
+        goto delkey;
+    }
+
+    BR_STUB_VER_COND_GOTO(priv, (!ver_enabled), delkey);
+
+    if (!IA_ISREG(stbuf->ia_type))
+        goto unwind;
+
+    /**
+     * If the object is bad, then "bad inode" marker has to be sent back
+     * in resoinse, for revalidated lookups as well. Some xlators such as
+     * quick-read might cache the data in revalidated lookup as fresh
+     * lookup would anyway have sent "bad inode" marker.
+     * In general send bad inode marker for every lookup operation on the
+     * bad object.
+     */
+    if (cookie != (void *)BR_STUB_REQUEST_COOKIE) {
+        ret = br_stub_mark_xdata_bad_object(this, inode, xattr);
         if (ret) {
-                op_ret   = -1;
-                op_errno = EINVAL;
+            op_ret = -1;
+            op_errno = EIO;
+            /*
+             * This flag ensures that in the label @delkey below,
+             * bad file marker is not removed from the dictinary,
+             * but other virtual xattrs (such as version, signature)
+             * are removed.
+             */
+            remove_bad_file_marker = _gf_false;
         }
+        goto delkey;
+    }
 
- unwind:
-        STACK_UNWIND_STRICT (readdirp, frame, op_ret, op_errno, entries, dict);
+    ret = br_stub_lookup_version(this, stbuf->ia_gfid, inode, xattr);
+    if (ret < 0) {
+        op_ret = -1;
+        op_errno = EINVAL;
+        goto delkey;
+    }
+
+    /**
+     * If the object is bad, send "bad inode" marker back in response
+     * for xlator(s) to act accordingly (such as quick-read, etc..)
+     */
+    ret = br_stub_mark_xdata_bad_object(this, inode, xattr);
+    if (ret) {
+        /**
+         * aaha! bad object, but sorry we would not
+         * satisfy the request on allocation failures.
+         */
+        op_ret = -1;
+        op_errno = EIO;
+        goto delkey;
+    }
+
+delkey:
+    br_stub_remove_vxattrs(xattr, remove_bad_file_marker);
+unwind:
+    STACK_UNWIND_STRICT(lookup, frame, op_ret, op_errno, inode, stbuf, xattr,
+                        postparent);
+
+    return 0;
+}
+
+int
+br_stub_lookup(call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xdata)
+{
+    int32_t ret = 0;
+    int op_errno = 0;
+    void *cookie = NULL;
+    uint64_t ctx_addr = 0;
+    gf_boolean_t xref = _gf_false;
+    br_stub_private_t *priv = NULL;
+    call_stub_t *stub = NULL;
+
+    GF_VALIDATE_OR_GOTO("bit-rot-stub", this, unwind);
+    GF_VALIDATE_OR_GOTO(this->name, loc, unwind);
+    GF_VALIDATE_OR_GOTO(this->name, loc->inode, unwind);
+
+    priv = this->private;
+
+    BR_STUB_VER_NOT_ACTIVE_THEN_GOTO(frame, priv, wind);
 
+    if (!gf_uuid_compare(loc->gfid, priv->bad_object_dir_gfid) ||
+        !gf_uuid_compare(loc->pargfid, priv->bad_object_dir_gfid)) {
+        stub = fop_lookup_stub(frame, br_stub_lookup_wrapper, loc, xdata);
+        if (!stub) {
+            op_errno = ENOMEM;
+            goto unwind;
+        }
+        br_stub_worker_enqueue(this, stub);
         return 0;
+    }
+
+    ret = br_stub_get_inode_ctx(this, loc->inode, &ctx_addr);
+    if (ret < 0)
+        ctx_addr = 0;
+    if (ctx_addr != 0)
+        goto wind;
+
+    /**
+     * fresh lookup: request version keys from POSIX
+     */
+    op_errno = ENOMEM;
+    if (!xdata) {
+        xdata = dict_new();
+        if (!xdata)
+            goto unwind;
+    } else {
+        xdata = dict_ref(xdata);
+    }
+
+    xref = _gf_true;
+
+    /**
+     * Requesting both xattrs provides a way of sanity checking the
+     * object. Anomaly checking is done in cbk by examining absence
+     * of either or both xattrs.
+     */
+    op_errno = EINVAL;
+    ret = dict_set_uint32(xdata, BITROT_CURRENT_VERSION_KEY, 0);
+    if (ret)
+        goto unwind;
+    ret = dict_set_uint32(xdata, BITROT_SIGNING_VERSION_KEY, 0);
+    if (ret)
+        goto unwind;
+    ret = dict_set_uint32(xdata, BITROT_OBJECT_BAD_KEY, 0);
+    if (ret)
+        goto unwind;
+    cookie = (void *)BR_STUB_REQUEST_COOKIE;
+
+wind:
+    STACK_WIND_COOKIE(frame, br_stub_lookup_cbk, cookie, FIRST_CHILD(this),
+                      FIRST_CHILD(this)->fops->lookup, loc, xdata);
+    goto dealloc_dict;
+
+unwind:
+    if (frame->local == (void *)0x1)
+        frame->local = NULL;
+    STACK_UNWIND_STRICT(lookup, frame, -1, op_errno, NULL, NULL, NULL, NULL);
+dealloc_dict:
+    if (xref)
+        dict_unref(xdata);
+    return 0;
 }
 
+/** }}} */
+
+/** {{{ */
+
+/* stat */
 int
-br_stub_readdirp (call_frame_t *frame, xlator_t *this,
-                  fd_t *fd, size_t size, off_t offset, dict_t *dict)
+br_stub_stat(call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xdata)
 {
-        int32_t ret = -1;
-        int op_errno = 0;
-        gf_boolean_t xref = _gf_false;
+    int32_t ret = 0;
+    int32_t op_ret = -1;
+    int32_t op_errno = EINVAL;
+    br_stub_private_t *priv = NULL;
 
-        op_errno = ENOMEM;
-        if (!dict) {
-                dict = dict_new ();
-                if (!dict)
-                        goto unwind;
-        } else {
-                dict = dict_ref (dict);
-        }
+    priv = this->private;
 
-        xref = _gf_true;
+    if (!priv->do_versioning)
+        goto wind;
 
-        op_errno = EINVAL;
-        ret = dict_set_uint32 (dict, BITROT_CURRENT_VERSION_KEY, 0);
-        if (ret)
-                goto unwind;
-        ret = dict_set_uint32 (dict, BITROT_SIGNING_VERSION_KEY, 0);
-        if (ret)
-                goto unwind;
+    if (!IA_ISREG(loc->inode->ia_type))
+        goto wind;
 
-        STACK_WIND (frame, br_stub_readdirp_cbk, FIRST_CHILD (this),
-                    FIRST_CHILD(this)->fops->readdirp, fd, size,
-                    offset, dict);
-        goto unref_dict;
+    ret = br_stub_check_bad_object(this, loc->inode, &op_ret, &op_errno);
+    if (ret)
+        goto unwind;
 
- unwind:
-        STACK_UNWIND_STRICT (readdirp, frame, -1, op_errno, NULL, NULL);
-        return 0;
+wind:
+    STACK_WIND_TAIL(frame, FIRST_CHILD(this), FIRST_CHILD(this)->fops->stat,
+                    loc, xdata);
+    return 0;
 
- unref_dict:
-        if (xref)
-                dict_unref (dict);
-        return 0;
+unwind:
+    STACK_UNWIND_STRICT(stat, frame, op_ret, op_errno, NULL, NULL);
+    return 0;
 }
 
-/** }}} */
+/* fstat */
+int
+br_stub_fstat(call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *xdata)
+{
+    int32_t ret = 0;
+    int32_t op_ret = -1;
+    int32_t op_errno = EINVAL;
+    br_stub_private_t *priv = NULL;
 
+    priv = this->private;
 
-/** {{{ */
+    if (!priv->do_versioning)
+        goto wind;
 
-/* lookup() */
+    if (!IA_ISREG(fd->inode->ia_type))
+        goto wind;
 
-int
-br_stub_lookup_cbk (call_frame_t *frame, void *cookie,
-                    xlator_t *this, int op_ret, int op_errno, inode_t *inode,
-                    struct iatt *stbuf, dict_t *xattr, struct iatt *postparent)
-{
-        int32_t ret = 0;
-
-        if (op_ret < 0)
-                goto unwind;
-        if (!IA_ISREG (stbuf->ia_type))
-                goto unwind;
-        if (cookie != (void *) BR_STUB_REQUEST_COOKIE)
-                goto delkey;
-
-        ret = br_stub_lookup_version (this, stbuf->ia_gfid, inode, xattr);
-        if (ret < 0) {
-                op_ret   = -1;
-                op_errno = EINVAL;
-        }
+    ret = br_stub_check_bad_object(this, fd->inode, &op_ret, &op_errno);
+    if (ret)
+        goto unwind;
 
- delkey:
-        br_stub_remove_vxattrs (xattr);
- unwind:
-        STACK_UNWIND_STRICT (lookup, frame,
-                             op_ret, op_errno, inode, stbuf, xattr, postparent);
+wind:
+    STACK_WIND_TAIL(frame, FIRST_CHILD(this), FIRST_CHILD(this)->fops->fstat,
+                    fd, xdata);
+    return 0;
 
-        return 0;
+unwind:
+    STACK_UNWIND_STRICT(fstat, frame, op_ret, op_errno, NULL, NULL);
+    return 0;
 }
 
+/** }}} */
+
+/** {{{ */
+
+/* unlink() */
+
 int
-br_stub_lookup (call_frame_t *frame,
-                xlator_t *this, loc_t *loc, dict_t *xdata)
+br_stub_unlink_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                   int32_t op_ret, int32_t op_errno, struct iatt *preparent,
+                   struct iatt *postparent, dict_t *xdata)
 {
-        int32_t ret = 0;
-        int op_errno = 0;
-        void *cookie = NULL;
-        uint64_t ctx_addr = 0;
-        gf_boolean_t xref = _gf_false;
+    br_stub_local_t *local = NULL;
+    inode_t *inode = NULL;
+    uint64_t ctx_addr = 0;
+    br_stub_inode_ctx_t *ctx = NULL;
+    int32_t ret = -1;
+    br_stub_private_t *priv = NULL;
+    gf_boolean_t ver_enabled = _gf_false;
+
+    BR_STUB_VER_ENABLED_IN_CALLPATH(frame, ver_enabled);
+    priv = this->private;
+    BR_STUB_VER_COND_GOTO(priv, (!ver_enabled), unwind);
+
+    local = frame->local;
+    frame->local = NULL;
+
+    if (op_ret < 0)
+        goto unwind;
+
+    if (!local) {
+        gf_smsg(this->name, GF_LOG_WARNING, 0, BRS_MSG_NULL_LOCAL, NULL);
+        goto unwind;
+    }
+    inode = local->u.context.inode;
+    if (!IA_ISREG(inode->ia_type))
+        goto unwind;
+
+    ret = br_stub_get_inode_ctx(this, inode, &ctx_addr);
+    if (ret) {
+        /**
+         * If the inode is bad AND context is not there, then there
+         * is a possibility of the gfid of the object being listed
+         * in the quarantine directory and will be shown in the
+         * bad objects list. So continuing with the fop with a
+         * warning log. The entry from the quarantine directory
+         * has to be removed manually. Its not a good idea to fail
+         * the fop, as the object has already been deleted.
+         */
+        gf_smsg(this->name, GF_LOG_WARNING, 0, BRS_MSG_GET_INODE_CONTEXT_FAILED,
+                "inode-gfid=%s", uuid_utoa(inode->gfid), NULL);
+        goto unwind;
+    }
 
-        ret = br_stub_get_inode_ctx (this, loc->inode, &ctx_addr);
-        if (ret < 0)
-                ctx_addr = 0;
-        if (ctx_addr != 0)
-                goto wind;
+    ctx = (br_stub_inode_ctx_t *)(long)ctx_addr;
 
+    LOCK(&inode->lock);
+    {
         /**
-         * fresh lookup: request version keys from POSIX
+         * Ignoring the return value of br_stub_del ().
+         * There is not much that can be done if unlinking
+         * of the entry in the quarantine directory fails.
+         * The failure is logged.
          */
+        if (__br_stub_is_bad_object(ctx))
+            (void)br_stub_del(this, inode->gfid);
+    }
+    UNLOCK(&inode->lock);
+
+unwind:
+    STACK_UNWIND_STRICT(unlink, frame, op_ret, op_errno, preparent, postparent,
+                        xdata);
+    br_stub_cleanup_local(local);
+    br_stub_dealloc_local(local);
+    return 0;
+}
+
+int
+br_stub_unlink(call_frame_t *frame, xlator_t *this, loc_t *loc, int flag,
+               dict_t *xdata)
+{
+    br_stub_local_t *local = NULL;
+    int32_t op_ret = -1;
+    int32_t op_errno = 0;
+    br_stub_private_t *priv = NULL;
+
+    priv = this->private;
+    BR_STUB_VER_NOT_ACTIVE_THEN_GOTO(frame, priv, wind);
+
+    local = br_stub_alloc_local(this);
+    if (!local) {
+        op_ret = -1;
         op_errno = ENOMEM;
-        if (!xdata) {
-                xdata = dict_new ();
-                if (!xdata)
-                        goto unwind;
-        } else {
-                xdata = dict_ref (xdata);
-        }
+        gf_smsg(this->name, GF_LOG_ERROR, ENOMEM, BRS_MSG_ALLOC_MEM_FAILED,
+                "local path=%s", loc->path, "gfid=%s",
+                uuid_utoa(loc->inode->gfid), NULL);
+        goto unwind;
+    }
 
-        xref = _gf_true;
+    br_stub_fill_local(local, NULL, NULL, loc->inode, loc->inode->gfid,
+                       BR_STUB_NO_VERSIONING, 0);
 
-        /**
-         * Requesting both xattrs provides a way of sanity checking the
-         * object. Anomaly checking is done in cbk by examining absence
-         * of either or both xattrs.
-         */
-        op_errno = EINVAL;
-        ret = dict_set_uint32 (xdata, BITROT_CURRENT_VERSION_KEY, 0);
-        if (ret)
-                goto unwind;
-        ret = dict_set_uint32 (xdata, BITROT_SIGNING_VERSION_KEY, 0);
-        if (ret)
-                goto unwind;
-        cookie = (void *) BR_STUB_REQUEST_COOKIE;
+    frame->local = local;
 
- wind:
-        STACK_WIND_COOKIE (frame, br_stub_lookup_cbk, cookie,
-                           FIRST_CHILD(this), FIRST_CHILD(this)->fops->lookup,
-                           loc, xdata);
-        goto dealloc_dict;
+wind:
+    STACK_WIND(frame, br_stub_unlink_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->unlink, loc, flag, xdata);
+    return 0;
 
- unwind:
-        STACK_UNWIND_STRICT (lookup, frame,
-                             -1, op_errno, NULL, NULL, NULL, NULL);
- dealloc_dict:
-        if (xref)
-                dict_unref (xdata);
-        return 0;
+unwind:
+    if (frame->local == (void *)0x1)
+        frame->local = NULL;
+    STACK_UNWIND_STRICT(unlink, frame, op_ret, op_errno, NULL, NULL, NULL);
+    return 0;
 }
 
 /** }}} */
@@ -1227,19 +3279,20 @@ br_stub_lookup (call_frame_t *frame,
 /* forget() */
 
 int
-br_stub_forget (xlator_t *this, inode_t *inode)
+br_stub_forget(xlator_t *this, inode_t *inode)
 {
-        uint64_t ctx_addr = 0;
-        br_stub_inode_ctx_t *ctx = NULL;
+    uint64_t ctx_addr = 0;
+    br_stub_inode_ctx_t *ctx = NULL;
+
+    inode_ctx_del(inode, this, &ctx_addr);
+    if (!ctx_addr)
+        return 0;
 
-        inode_ctx_del (inode, this, &ctx_addr);
-        if (!ctx_addr)
-                return 0;
+    ctx = (br_stub_inode_ctx_t *)(long)ctx_addr;
 
-        ctx = (br_stub_inode_ctx_t *) (long) ctx_addr;
-        GF_FREE (ctx);
+    GF_FREE(ctx);
 
-        return 0;
+    return 0;
 }
 
 /** }}} */
@@ -1247,94 +3300,188 @@ br_stub_forget (xlator_t *this, inode_t *inode)
 /** {{{ */
 
 int32_t
-br_stub_noop (call_frame_t *frame, void *cookie, xlator_t *this,
-              int32_t op_ret, int32_t op_errno, dict_t *xdata)
+br_stub_noop(call_frame_t *frame, void *cookie, xlator_t *this, int32_t op_ret,
+             int32_t op_errno, dict_t *xdata)
 {
-        STACK_DESTROY (frame->root);
-        return 0;
+    STACK_DESTROY(frame->root);
+    return 0;
 }
 
-static inline void
-br_stub_send_ipc_fop (xlator_t *this,
-                      fd_t *fd, unsigned long releaseversion, int32_t flags)
+static void
+br_stub_send_ipc_fop(xlator_t *this, fd_t *fd, unsigned long releaseversion,
+                     int sign_info)
 {
-        int32_t op = 0;
-        int32_t ret = 0;
-        dict_t *xdata = NULL;
-        call_frame_t *frame = NULL;
-        changelog_event_t ev = {0,};
+    int32_t op = 0;
+    int32_t ret = 0;
+    dict_t *xdata = NULL;
+    call_frame_t *frame = NULL;
+    changelog_event_t ev = {
+        0,
+    };
+
+    ev.ev_type = CHANGELOG_OP_TYPE_BR_RELEASE;
+    ev.u.releasebr.version = releaseversion;
+    ev.u.releasebr.sign_info = sign_info;
+    gf_uuid_copy(ev.u.releasebr.gfid, fd->inode->gfid);
+
+    xdata = dict_new();
+    if (!xdata) {
+        gf_smsg(this->name, GF_LOG_WARNING, ENOMEM, BRS_MSG_DICT_ALLOC_FAILED,
+                NULL);
+        goto out;
+    }
+
+    ret = dict_set_static_bin(xdata, "RELEASE-EVENT", &ev, CHANGELOG_EV_SIZE);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_WARNING, 0, BRS_MSG_SET_EVENT_FAILED, NULL);
+        goto dealloc_dict;
+    }
 
-        ev.ev_type = CHANGELOG_OP_TYPE_BR_RELEASE;
-        ev.u.releasebr.flags = flags;
-        ev.u.releasebr.version = releaseversion;
-        gf_uuid_copy (ev.u.releasebr.gfid, fd->inode->gfid);
+    frame = create_frame(this, this->ctx->pool);
+    if (!frame) {
+        gf_smsg(this->name, GF_LOG_WARNING, 0, BRS_MSG_CREATE_FRAME_FAILED,
+                NULL);
+        goto dealloc_dict;
+    }
 
-        xdata = dict_new ();
-        if (!xdata) {
-                gf_log (this->name, GF_LOG_WARNING,
-                        "dict allocation failed: cannot send IPC FOP "
-                        "to changelog");
-                goto out;
-        }
+    op = GF_IPC_TARGET_CHANGELOG;
+    STACK_WIND(frame, br_stub_noop, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->ipc, op, xdata);
 
-        ret = dict_set_static_bin (xdata,
-                                   "RELEASE-EVENT", &ev, CHANGELOG_EV_SIZE);
-        if (ret) {
-                gf_log (this->name, GF_LOG_WARNING,
-                        "cannot set release event in dict");
-                goto dealloc_dict;
-        }
+dealloc_dict:
+    dict_unref(xdata);
+out:
+    return;
+}
 
-        frame = create_frame (this, this->ctx->pool);
-        if (!frame) {
-                gf_log (this->name, GF_LOG_WARNING, "create_frame() failure");
-                goto dealloc_dict;
-        }
+/**
+ * This is how the state machine of sign info works:
+ * 3 states:
+ * 1) BR_SIGN_NORMAL => The default State of the inode
+ * 2) BR_SIGN_REOPEN_WAIT => A release has been sent and is waiting for reopen
+ * 3) BR_SIGN_QUICK => reopen has happened and this release should trigger sign
+ * 2 events:
+ * 1) GF_FOP_RELEASE
+ * 2) GF_FOP_WRITE (actually a dummy write for BitD)
+ *
+ * This is how states are changed based on events:
+ * EVENT: GF_FOP_RELEASE:
+ * if (state == BR_SIGN_NORMAL) ; then
+ *     set state = BR_SIGN_REOPEN_WAIT;
+ * if (state == BR_SIGN_QUICK); then
+ *     set state = BR_SIGN_NORMAL;
+ * EVENT: GF_FOP_WRITE:
+ *  if (state == BR_SIGN_REOPEN_WAIT); then
+ *     set state = BR_SIGN_QUICK;
+ */
+br_sign_state_t
+__br_stub_inode_sign_state(br_stub_inode_ctx_t *ctx, glusterfs_fop_t fop,
+                           fd_t *fd)
+{
+    br_sign_state_t sign_info = BR_SIGN_INVALID;
 
-        op = GF_IPC_TARGET_CHANGELOG;
-        STACK_WIND (frame, br_stub_noop, FIRST_CHILD (this),
-                    FIRST_CHILD (this)->fops->ipc, op, xdata);
+    switch (fop) {
+        case GF_FOP_FSETXATTR:
+            sign_info = ctx->info_sign = BR_SIGN_QUICK;
+            break;
 
- dealloc_dict:
-        dict_unref (xdata);
- out:
-        return;
+        case GF_FOP_RELEASE:
+            GF_ASSERT(ctx->info_sign != BR_SIGN_REOPEN_WAIT);
+
+            if (ctx->info_sign == BR_SIGN_NORMAL) {
+                sign_info = ctx->info_sign = BR_SIGN_REOPEN_WAIT;
+            } else {
+                sign_info = ctx->info_sign;
+                ctx->info_sign = BR_SIGN_NORMAL;
+            }
+
+            break;
+        default:
+            break;
+    }
+
+    return sign_info;
 }
 
 int32_t
-br_stub_release (xlator_t *this, fd_t *fd)
+br_stub_release(xlator_t *this, fd_t *fd)
 {
-        int32_t ret = 0;
-        int32_t flags = 0;
-        inode_t *inode = NULL;
-        unsigned long releaseversion = 0;
-        br_stub_inode_ctx_t *ctx = NULL;
+    int32_t ret = 0;
+    int32_t flags = 0;
+    inode_t *inode = NULL;
+    unsigned long releaseversion = 0;
+    br_stub_inode_ctx_t *ctx = NULL;
+    uint64_t tmp = 0;
+    br_stub_fd_t *br_stub_fd = NULL;
+    int32_t signinfo = 0;
+
+    inode = fd->inode;
+
+    LOCK(&inode->lock);
+    {
+        ctx = __br_stub_get_ongoing_version_ctx(this, inode, NULL);
+        if (ctx == NULL)
+            goto unblock;
+        br_stub_fd = br_stub_fd_ctx_get(this, fd);
+        if (br_stub_fd) {
+            list_del_init(&br_stub_fd->list);
+        }
 
-        inode = fd->inode;
+        ret = __br_stub_can_trigger_release(inode, ctx, &releaseversion);
+        if (!ret)
+            goto unblock;
 
-        LOCK (&inode->lock);
-        {
-                ctx = __br_stub_get_ongoing_version_ctx (this, inode, NULL);
-                if (ctx == NULL)
-                        goto unblock;
-                __br_stub_track_release (ctx);
-                ret = __br_stub_can_trigger_release
-                                 (inode, ctx, &releaseversion, &flags);
-                if (ret) {
-                        GF_ASSERT (__br_stub_is_inode_dirty (ctx) == 0);
-                        __br_stub_mark_inode_dirty (ctx);
-                }
-        }
- unblock:
-        UNLOCK (&inode->lock);
+        signinfo = __br_stub_inode_sign_state(ctx, GF_FOP_RELEASE, fd);
+        signinfo = htonl(signinfo);
 
-        if (ret) {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "releaseversion: %lu|flags: %d", releaseversion, flags);
-                br_stub_send_ipc_fop (this, fd, releaseversion, flags);
+        /* inode back to initital state: mark dirty */
+        if (ctx->info_sign == BR_SIGN_NORMAL) {
+            __br_stub_mark_inode_dirty(ctx);
+            __br_stub_unset_inode_modified(ctx);
         }
+    }
+unblock:
+    UNLOCK(&inode->lock);
 
-        return 0;
+    if (ret) {
+        gf_msg_debug(this->name, 0,
+                     "releaseversion: %lu | flags: %d "
+                     "| signinfo: %d",
+                     (unsigned long)ntohl(releaseversion), flags,
+                     ntohl(signinfo));
+        br_stub_send_ipc_fop(this, fd, releaseversion, signinfo);
+    }
+
+    ret = fd_ctx_del(fd, this, &tmp);
+    br_stub_fd = (br_stub_fd_t *)(long)tmp;
+
+    GF_FREE(br_stub_fd);
+
+    return 0;
+}
+
+int32_t
+br_stub_releasedir(xlator_t *this, fd_t *fd)
+{
+    br_stub_fd_t *fctx = NULL;
+    uint64_t ctx = 0;
+    int ret = 0;
+
+    ret = fd_ctx_del(fd, this, &ctx);
+    if (ret < 0)
+        goto out;
+
+    fctx = (br_stub_fd_t *)(long)ctx;
+    if (fctx->bad_object.dir) {
+        ret = sys_closedir(fctx->bad_object.dir);
+        if (ret)
+            gf_smsg(this->name, GF_LOG_ERROR, 0, BRS_MSG_BAD_OBJ_DIR_CLOSE_FAIL,
+                    "error=%s", strerror(errno), NULL);
+    }
+
+    GF_FREE(fctx);
+out:
+    return 0;
 }
 
 /** }}} */
@@ -1344,84 +3491,100 @@ br_stub_release (xlator_t *this, fd_t *fd)
 /* ictxmerge */
 
 void
-br_stub_ictxmerge (xlator_t *this, fd_t *fd,
-                   inode_t *inode, inode_t *linked_inode)
+br_stub_ictxmerge(xlator_t *this, fd_t *fd, inode_t *inode,
+                  inode_t *linked_inode)
 {
-        int32_t ret = 0;
-        uint64_t ctxaddr = 0;
-        uint64_t lctxaddr = 0;
-        br_stub_inode_ctx_t *ctx = NULL;
-        br_stub_inode_ctx_t *lctx = NULL;
+    int32_t ret = 0;
+    uint64_t ctxaddr = 0;
+    uint64_t lctxaddr = 0;
+    br_stub_inode_ctx_t *ctx = NULL;
+    br_stub_inode_ctx_t *lctx = NULL;
+    br_stub_fd_t *br_stub_fd = NULL;
+
+    ret = br_stub_get_inode_ctx(this, inode, &ctxaddr);
+    if (ret < 0)
+        goto done;
+    ctx = (br_stub_inode_ctx_t *)(uintptr_t)ctxaddr;
 
-        ret = br_stub_get_inode_ctx (this, inode, &ctxaddr);
+    LOCK(&linked_inode->lock);
+    {
+        ret = __br_stub_get_inode_ctx(this, linked_inode, &lctxaddr);
         if (ret < 0)
-                goto done;
-        ctx = (br_stub_inode_ctx_t *) ctxaddr;
-
-        LOCK (&linked_inode->lock);
-        {
-                ret = __br_stub_get_inode_ctx (this, linked_inode, &lctxaddr);
-                if (ret < 0)
-                        goto unblock;
-                lctx = (br_stub_inode_ctx_t *) lctxaddr;
-
-                if (__br_stub_is_inode_dirty (lctx)) {
-                        /**
-                         * RACY code: An inode can end up in this situation
-                         * after a lookup() or after a create() followed by
-                         * a release(). Even if we distinguish b/w the two,
-                         * there needs to be more infrastructure built up
-                         * in stub to handle these races. Note, that it's
-                         * probably OK to ignore the race iff the version
-                         * was initialized on the very first lookup(), i.e.,
-                         * [ongoingversion: default].
-                         *
-                         * FIXME: fixup races [create(1..n)/lookup(1..n)].
-                         */
-                        GF_ASSERT (lctx->currentversion
-                                      == BITROT_DEFAULT_CURRENT_VERSION);
-                        __br_stub_track_openfd (fd, lctx);
-                        __br_stub_mark_inode_synced (lctx);
-                } else {
-                        GF_ASSERT (ctx->currentversion <= lctx->currentversion);
-                        __br_stub_track_openfd (fd, lctx);
-                }
+            goto unblock;
+        lctx = (br_stub_inode_ctx_t *)(uintptr_t)lctxaddr;
+
+        GF_ASSERT(list_is_singular(&ctx->fd_list));
+        br_stub_fd = list_first_entry(&ctx->fd_list, br_stub_fd_t, list);
+        if (br_stub_fd) {
+            GF_ASSERT(br_stub_fd->fd == fd);
+            list_move_tail(&br_stub_fd->list, &lctx->fd_list);
         }
- unblock:
-        UNLOCK (&linked_inode->lock);
+    }
+unblock:
+    UNLOCK(&linked_inode->lock);
 
- done:
-        return;
+done:
+    return;
 }
 
 /** }}} */
 
-
 struct xlator_fops fops = {
-        .lookup    = br_stub_lookup,
-        .open      = br_stub_open,
-        .create    = br_stub_create,
-        .readdirp  = br_stub_readdirp,
-        .getxattr  = br_stub_getxattr,
-        .fgetxattr = br_stub_fgetxattr,
-        .fsetxattr = br_stub_fsetxattr,
+    .lookup = br_stub_lookup,
+    .stat = br_stub_stat,
+    .fstat = br_stub_fstat,
+    .open = br_stub_open,
+    .create = br_stub_create,
+    .readdirp = br_stub_readdirp,
+    .getxattr = br_stub_getxattr,
+    .fgetxattr = br_stub_fgetxattr,
+    .fsetxattr = br_stub_fsetxattr,
+    .writev = br_stub_writev,
+    .truncate = br_stub_truncate,
+    .ftruncate = br_stub_ftruncate,
+    .mknod = br_stub_mknod,
+    .readv = br_stub_readv,
+    .removexattr = br_stub_removexattr,
+    .fremovexattr = br_stub_fremovexattr,
+    .setxattr = br_stub_setxattr,
+    .opendir = br_stub_opendir,
+    .readdir = br_stub_readdir,
+    .unlink = br_stub_unlink,
 };
 
 struct xlator_cbks cbks = {
-        .forget    = br_stub_forget,
-        .release   = br_stub_release,
-        .ictxmerge = br_stub_ictxmerge,
+    .forget = br_stub_forget,
+    .release = br_stub_release,
+    .ictxmerge = br_stub_ictxmerge,
 };
 
 struct volume_options options[] = {
-        { .key = {"bitrot"},
-          .type = GF_OPTION_TYPE_BOOL,
-          .default_value = "on",
-          .description = "enable/disable bitrot stub"
-        },
-        { .key = {"export"},
-          .type = GF_OPTION_TYPE_PATH,
-          .description = "brick path for versioning"
-        },
-	{ .key  = {NULL} },
+    {.key = {"bitrot"},
+     .type = GF_OPTION_TYPE_BOOL,
+     .default_value = "off",
+     .op_version = {GD_OP_VERSION_3_7_0},
+     .flags = OPT_FLAG_SETTABLE | OPT_FLAG_FORCE,
+     .tags = {"bitrot"},
+     .description = "enable/disable bitrot stub"},
+    {.key = {"export"},
+     .type = GF_OPTION_TYPE_PATH,
+     .op_version = {GD_OP_VERSION_3_7_0},
+     .tags = {"bitrot"},
+     .description = "brick path for versioning",
+     .default_value = "{{ brick.path }}"},
+    {.key = {NULL}},
+};
+
+xlator_api_t xlator_api = {
+    .init = init,
+    .fini = fini,
+    .notify = notify,
+    .reconfigure = reconfigure,
+    .mem_acct_init = mem_acct_init,
+    .op_version = {1}, /* Present from the initial version */
+    .fops = &fops,
+    .cbks = &cbks,
+    .options = options,
+    .identifier = "bitrot-stub",
+    .category = GF_MAINTAINED,
 };
diff --git a/xlators/features/bit-rot/src/stub/bit-rot-stub.h b/xlators/features/bit-rot/src/stub/bit-rot-stub.h
index 5db0e321c20..edd79a77e4f 100644
--- a/xlators/features/bit-rot/src/stub/bit-rot-stub.h
+++ b/xlators/features/bit-rot/src/stub/bit-rot-stub.h
@@ -1,270 +1,515 @@
- /*
-   Copyright (c) 2015 Red Hat, Inc. <http://www.redhat.com>
-   This file is part of GlusterFS.
-
-   This file is licensed to you under your choice of the GNU Lesser
-   General Public License, version 3 or any later version (LGPLv3 or
-   later), or the GNU General Public License, version 2 (GPLv2), in all
-   cases as published by the Free Software Foundation.
+/*
+  Copyright (c) 2015 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 #ifndef __BIT_ROT_STUB_H__
 #define __BIT_ROT_STUB_H__
 
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
+#include <glusterfs/glusterfs.h>
+#include <glusterfs/logging.h>
+#include <glusterfs/dict.h>
+#include <glusterfs/xlator.h>
+#include <glusterfs/defaults.h>
+#include <glusterfs/call-stub.h>
+#include "bit-rot-stub-mem-types.h"
+#include <glusterfs/syscall.h>
+#include <glusterfs/common-utils.h>
+#include "bit-rot-common.h"
+#include "bit-rot-stub-messages.h"
+#include "glusterfs3-xdr.h"
+#include <glusterfs/syncop.h>
+#include <glusterfs/syncop-utils.h>
 
-#include "glusterfs.h"
-#include "logging.h"
-#include "dict.h"
-#include "xlator.h"
-#include "defaults.h"
-#include "call-stub.h"
+#define BAD_OBJECT_THREAD_STACK_SIZE ((size_t)(1024 * 1024))
+#define BR_STUB_DUMP_STR_SIZE 65536
 
-#include "bit-rot-common.h"
+#define BR_PATH_MAX_EXTRA (PATH_MAX + 1024)
+#define BR_PATH_MAX_PLUS (PATH_MAX + 2048)
+
+/*
+ * Oops. Spelling mistake. Correcting it
+ */
+#define OLD_BR_STUB_QUARANTINE_DIR GF_HIDDEN_PATH "/quanrantine"
+#define BR_STUB_QUARANTINE_DIR GF_HIDDEN_PATH "/quarantine"
 
-typedef int (br_stub_version_cbk) (call_frame_t *, void *,
-                                   xlator_t *, int32_t, int32_t, dict_t *);
+/* do not reference frame->local in cbk unless initialized.
+ * Assigned 0x1 marks verisoning flag between call path and
+ * cbk path.
+ */
+#define BR_STUB_VER_NOT_ACTIVE_THEN_GOTO(frame, priv, label)                   \
+    do {                                                                       \
+        if (priv->do_versioning)                                               \
+            frame->local = (void *)0x1;                                        \
+        else                                                                   \
+            goto label;                                                        \
+    } while (0)
+
+#define BR_STUB_VER_COND_GOTO(priv, cond, label)                               \
+    do {                                                                       \
+        if (!priv->do_versioning || cond)                                      \
+            goto label;                                                        \
+    } while (0)
+
+#define BR_STUB_VER_ENABLED_IN_CALLPATH(frame, flag)                           \
+    do {                                                                       \
+        if (frame->local)                                                      \
+            flag = _gf_true;                                                   \
+        if (frame->local == (void *)0x1)                                       \
+            frame->local = NULL;                                               \
+    } while (0)
+
+#define BR_STUB_RESET_LOCAL_NULL(frame)                                        \
+    do {                                                                       \
+        if (frame->local == (void *)0x1)                                       \
+            frame->local = NULL;                                               \
+    } while (0)
+
+typedef int(br_stub_version_cbk)(call_frame_t *, void *, xlator_t *, int32_t,
+                                 int32_t, dict_t *);
 
 typedef struct br_stub_inode_ctx {
-        int need_writeback;                     /* does the inode need
-                                                      a writeback to disk? */
-        unsigned long currentversion;           /* ongoing version */
-
-        struct release {
-                int32_t ordflags;
-                unsigned long opencount;        /* number of open()s before
-                                                   final release() */
-                unsigned long releasecount;     /* number of release()s */
-        } releasectx;
-#define BR_STUB_REQUIRE_RELEASE_CBK 0x0E0EA0E
+    int need_writeback;           /* does the inode need
+                                        a writeback to disk? */
+    unsigned long currentversion; /* ongoing version */
+
+    int info_sign;
+    struct list_head fd_list; /* list of open fds or fds participating in
+                                 write operations */
+    gf_boolean_t bad_object;
 } br_stub_inode_ctx_t;
 
+typedef struct br_stub_fd {
+    fd_t *fd;
+    struct list_head list;
+    struct bad_object_dir {
+        DIR *dir;
+        off_t dir_eof;
+    } bad_object;
+} br_stub_fd_t;
 
-#define I_DIRTY  (1<<0)        /* inode needs writeback */
-#define WRITEBACK_DURABLE 1    /* writeback is durable */
+#define I_DIRTY (1 << 0) /* inode needs writeback */
+#define I_MODIFIED (1 << 1)
+#define WRITEBACK_DURABLE 1 /* writeback is durable */
 
 /**
  * This could just have been a plain struct without unions and all,
  * but we may need additional things in the future.
  */
 typedef struct br_stub_local {
-        call_stub_t *fopstub;   /* stub for original fop */
-
-        int versioningtype;     /* not much used atm */
-
-        union {
-                struct br_stub_ctx {
-                        fd_t          *fd;
-                        uuid_t         gfid;
-                        inode_t       *inode;
-                        unsigned long  version;
-                        gf_boolean_t   markdirty;
-                } context;
-        } u;
+    call_stub_t *fopstub; /* stub for original fop */
+
+    int versioningtype; /* not much used atm */
+
+    union {
+        struct br_stub_ctx {
+            fd_t *fd;
+            uuid_t gfid;
+            inode_t *inode;
+            unsigned long version;
+        } context;
+    } u;
 } br_stub_local_t;
 
-#define BR_STUB_FULL_VERSIONING (1<<0)
-#define BR_STUB_INCREMENTAL_VERSIONING (1<<1)
+#define BR_STUB_NO_VERSIONING (1 << 0)
+#define BR_STUB_INCREMENTAL_VERSIONING (1 << 1)
 
 typedef struct br_stub_private {
-        gf_boolean_t go;
+    gf_boolean_t do_versioning;
+
+    uint32_t boot[2];
+    char export[PATH_MAX];
 
-        uint32_t boot[2];
-        char export[PATH_MAX];
+    pthread_mutex_t lock;
+    pthread_cond_t cond;
 
-        struct mem_pool *local_pool;
+    struct list_head squeue; /* ordered signing queue */
+    pthread_t signth;
+    struct bad_objects_container {
+        pthread_t thread;
+        pthread_mutex_t bad_lock;
+        pthread_cond_t bad_cond;
+        struct list_head bad_queue;
+    } container;
+    struct mem_pool *local_pool;
+
+    char stub_basepath[BR_PATH_MAX_EXTRA];
+
+    uuid_t bad_object_dir_gfid;
 } br_stub_private_t;
 
+br_stub_fd_t *
+br_stub_fd_new(void);
+
+int
+__br_stub_fd_ctx_set(xlator_t *this, fd_t *fd, br_stub_fd_t *br_stub_fd);
+
+br_stub_fd_t *
+__br_stub_fd_ctx_get(xlator_t *this, fd_t *fd);
+
+br_stub_fd_t *
+br_stub_fd_ctx_get(xlator_t *this, fd_t *fd);
+
+int32_t
+br_stub_fd_ctx_set(xlator_t *this, fd_t *fd, br_stub_fd_t *br_stub_fd);
+
+static inline gf_boolean_t
+__br_stub_is_bad_object(br_stub_inode_ctx_t *ctx)
+{
+    return ctx->bad_object;
+}
+
+static inline void
+__br_stub_mark_object_bad(br_stub_inode_ctx_t *ctx)
+{
+    ctx->bad_object = _gf_true;
+}
+
 /* inode writeback helpers */
 static inline void
-__br_stub_mark_inode_dirty (br_stub_inode_ctx_t *ctx)
+__br_stub_mark_inode_dirty(br_stub_inode_ctx_t *ctx)
+{
+    ctx->need_writeback |= I_DIRTY;
+}
+
+static inline void
+__br_stub_mark_inode_synced(br_stub_inode_ctx_t *ctx)
+{
+    ctx->need_writeback &= ~I_DIRTY;
+}
+
+static inline int
+__br_stub_is_inode_dirty(br_stub_inode_ctx_t *ctx)
 {
-        ctx->need_writeback |= I_DIRTY;
+    return (ctx->need_writeback & I_DIRTY);
 }
 
+/* inode mofification markers */
 static inline void
-__br_stub_mark_inode_synced (br_stub_inode_ctx_t *ctx)
+__br_stub_set_inode_modified(br_stub_inode_ctx_t *ctx)
 {
-        ctx->need_writeback &= ~I_DIRTY;
+    ctx->need_writeback |= I_MODIFIED;
+}
+
+static inline void
+__br_stub_unset_inode_modified(br_stub_inode_ctx_t *ctx)
+{
+    ctx->need_writeback &= ~I_MODIFIED;
 }
 
 static inline int
-__br_stub_is_inode_dirty (br_stub_inode_ctx_t *ctx)
+__br_stub_is_inode_modified(br_stub_inode_ctx_t *ctx)
 {
-        return (ctx->need_writeback & I_DIRTY);
+    return (ctx->need_writeback & I_MODIFIED);
 }
 
 static inline int
-br_stub_require_release_call (xlator_t *this, fd_t *fd)
+br_stub_require_release_call(xlator_t *this, fd_t *fd, br_stub_fd_t **fd_ctx)
 {
-        int32_t ret = 0;
-
-        ret = fd_ctx_set (fd, this,
-                          (uint64_t)(long)BR_STUB_REQUIRE_RELEASE_CBK);
-        if (ret)
-                gf_log (this->name, GF_LOG_WARNING,
-                        "could not set fd context (for release callback");
-        return ret;
+    int32_t ret = 0;
+    br_stub_fd_t *br_stub_fd = NULL;
+
+    br_stub_fd = br_stub_fd_new();
+    if (!br_stub_fd)
+        return -1;
+
+    br_stub_fd->fd = fd;
+    INIT_LIST_HEAD(&br_stub_fd->list);
+
+    ret = br_stub_fd_ctx_set(this, fd, br_stub_fd);
+    if (ret)
+        gf_smsg(this->name, GF_LOG_WARNING, 0, BRS_MSG_SET_CONTEXT_FAILED,
+                NULL);
+    else
+        *fd_ctx = br_stub_fd;
+
+    return ret;
 }
 
 /* get/set inode context helpers */
 
 static inline int
-__br_stub_get_inode_ctx (xlator_t *this,
-                         inode_t *inode, uint64_t *ctx)
+__br_stub_get_inode_ctx(xlator_t *this, inode_t *inode, uint64_t *ctx)
 {
-        return __inode_ctx_get (inode, this, ctx);
+    return __inode_ctx_get(inode, this, ctx);
 }
 
 static inline int
-br_stub_get_inode_ctx (xlator_t *this,
-                       inode_t *inode, uint64_t *ctx)
+br_stub_get_inode_ctx(xlator_t *this, inode_t *inode, uint64_t *ctx)
 {
-        return inode_ctx_get (inode, this, ctx);
+    int ret = -1;
+
+    LOCK(&inode->lock);
+    {
+        ret = __br_stub_get_inode_ctx(this, inode, ctx);
+    }
+    UNLOCK(&inode->lock);
+
+    return ret;
 }
 
 static inline int
-br_stub_set_inode_ctx (xlator_t *this,
-                       inode_t *inode, br_stub_inode_ctx_t *ctx)
+br_stub_set_inode_ctx(xlator_t *this, inode_t *inode, br_stub_inode_ctx_t *ctx)
 {
-        uint64_t ctx_addr = (uint64_t) ctx;
-        return inode_ctx_set (inode, this, &ctx_addr);
+    uint64_t ctx_addr = (uint64_t)(uintptr_t)ctx;
+    return inode_ctx_set(inode, this, &ctx_addr);
 }
 
 /* version get/set helpers */
 
 static inline unsigned long
-__br_stub_writeback_version (br_stub_inode_ctx_t *ctx)
+__br_stub_writeback_version(br_stub_inode_ctx_t *ctx)
 {
-        return (ctx->currentversion + 1);
+    return (ctx->currentversion + 1);
 }
 
 static inline void
-__br_stub_set_ongoing_version (br_stub_inode_ctx_t *ctx, unsigned long version)
+__br_stub_set_ongoing_version(br_stub_inode_ctx_t *ctx, unsigned long version)
 {
+    if (ctx->currentversion < version)
         ctx->currentversion = version;
+    else
+        gf_smsg("bit-rot-stub", GF_LOG_WARNING, 0,
+                BRS_MSG_CHANGE_VERSION_FAILED, "current version=%lu",
+                ctx->currentversion, "new version=%lu", version, NULL);
 }
 
-static inline void
-__br_stub_reset_release_counters (br_stub_inode_ctx_t *ctx)
+static inline int
+__br_stub_can_trigger_release(inode_t *inode, br_stub_inode_ctx_t *ctx,
+                              unsigned long *version)
 {
-        ctx->releasectx.ordflags = 0;
-        ctx->releasectx.opencount = 0;
-        ctx->releasectx.releasecount = 0;
+    /**
+     * If the inode is modified, then it has to be dirty. An inode is
+     * marked dirty once version is increased. Its marked as modified
+     * when the modification call (write/truncate) which triggered
+     * the versioning is successful.
+     */
+    if (__br_stub_is_inode_modified(ctx) && list_empty(&ctx->fd_list) &&
+        (ctx->info_sign != BR_SIGN_REOPEN_WAIT)) {
+        GF_ASSERT(__br_stub_is_inode_dirty(ctx) == 0);
+
+        if (version)
+            *version = htonl(ctx->currentversion);
+        return 1;
+    }
+
+    return 0;
 }
 
-static inline void
-__br_stub_track_release (br_stub_inode_ctx_t *ctx)
+static inline int32_t
+br_stub_get_ongoing_version(xlator_t *this, inode_t *inode,
+                            unsigned long *version)
 {
-        ++ctx->releasectx.releasecount;
+    int32_t ret = 0;
+    uint64_t ctx_addr = 0;
+    br_stub_inode_ctx_t *ctx = NULL;
+
+    LOCK(&inode->lock);
+    {
+        ret = __inode_ctx_get(inode, this, &ctx_addr);
+        if (ret < 0)
+            goto unblock;
+        ctx = (br_stub_inode_ctx_t *)(long)ctx_addr;
+        *version = ctx->currentversion;
+    }
+unblock:
+    UNLOCK(&inode->lock);
+
+    return ret;
 }
 
-static inline void
-___br_stub_track_open (br_stub_inode_ctx_t *ctx)
+/**
+ * fetch the current version from inode and return the context.
+ * inode->lock should be held before invoking this as context
+ * *needs* to be valid in the caller.
+ */
+static inline br_stub_inode_ctx_t *
+__br_stub_get_ongoing_version_ctx(xlator_t *this, inode_t *inode,
+                                  unsigned long *version)
 {
-        ++ctx->releasectx.opencount;
+    int32_t ret = 0;
+    uint64_t ctx_addr = 0;
+    br_stub_inode_ctx_t *ctx = NULL;
+
+    ret = __inode_ctx_get(inode, this, &ctx_addr);
+    if (ret < 0)
+        return NULL;
+    ctx = (br_stub_inode_ctx_t *)(long)ctx_addr;
+    if (version)
+        *version = ctx->currentversion;
+
+    return ctx;
 }
 
-static inline void
-___br_stub_track_open_flags (fd_t *fd, br_stub_inode_ctx_t *ctx)
+/* filter for xattr fetch */
+static inline int
+br_stub_is_internal_xattr(const char *name)
 {
-        ctx->releasectx.ordflags |= fd->flags;
+    if (name && ((strncmp(name, BITROT_CURRENT_VERSION_KEY,
+                          SLEN(BITROT_CURRENT_VERSION_KEY)) == 0) ||
+                 (strncmp(name, BITROT_SIGNING_VERSION_KEY,
+                          SLEN(BITROT_SIGNING_VERSION_KEY)) == 0)))
+        return 1;
+    return 0;
 }
 
 static inline void
-__br_stub_track_openfd (fd_t *fd, br_stub_inode_ctx_t *ctx)
+br_stub_remove_vxattrs(dict_t *xattr, gf_boolean_t remove_bad_marker)
 {
-        ___br_stub_track_open (ctx);
-        ___br_stub_track_open_flags (fd, ctx);
+    if (xattr) {
+        /*
+         * When a file is corrupted, bad-object should be
+         * set in the dict. But, other info such as version,
+         * signature etc should not be set. Hence the flag
+         * remove_bad_marker. The consumer should know whether
+         * to send the bad-object info in the dict or not.
+         */
+        if (remove_bad_marker)
+            dict_del(xattr, BITROT_OBJECT_BAD_KEY);
+        dict_del(xattr, BITROT_CURRENT_VERSION_KEY);
+        dict_del(xattr, BITROT_SIGNING_VERSION_KEY);
+        dict_del(xattr, BITROT_SIGNING_XATTR_SIZE_KEY);
+    }
 }
 
+/**
+ * This function returns the below values for different situations
+ * 0  => as per the inode context object is not bad
+ * -1 => Failed to get the inode context itself
+ * -2 => As per the inode context object is bad
+ * Both -ve values means the fop which called this function is failed
+ * and error is returned upwards.
+ * In future if needed or more errors have to be handled, then those
+ * errors can be made into enums.
+ */
 static inline int
-__br_stub_can_trigger_release (inode_t *inode,
-                               br_stub_inode_ctx_t *ctx,
-                               unsigned long *version, int32_t *flags)
+br_stub_is_bad_object(xlator_t *this, inode_t *inode)
 {
-        if (list_empty (&inode->fd_list)
-            && (ctx->releasectx.releasecount == ctx->releasectx.opencount)) {
-                if (flags)
-                        *flags = htonl (ctx->releasectx.ordflags);
-                if (version)
-                        *version = htonl (ctx->currentversion);
-
-                __br_stub_reset_release_counters (ctx);
-                return 1;
-        }
-
-        return 0;
+    int bad_object = 0;
+    gf_boolean_t tmp = _gf_false;
+    uint64_t ctx_addr = 0;
+    br_stub_inode_ctx_t *ctx = NULL;
+    int32_t ret = -1;
+
+    ret = br_stub_get_inode_ctx(this, inode, &ctx_addr);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_ERROR, 0, BRS_MSG_GET_INODE_CONTEXT_FAILED,
+                "inode-gfid=%s", uuid_utoa(inode->gfid), NULL);
+        bad_object = -1;
+        goto out;
+    }
+
+    ctx = (br_stub_inode_ctx_t *)(long)ctx_addr;
+
+    LOCK(&inode->lock);
+    {
+        tmp = __br_stub_is_bad_object(ctx);
+        if (tmp)
+            bad_object = -2;
+    }
+    UNLOCK(&inode->lock);
+
+out:
+    return bad_object;
 }
 
 static inline int32_t
-br_stub_get_ongoing_version (xlator_t *this,
-                             inode_t *inode, unsigned long *version)
+br_stub_mark_object_bad(xlator_t *this, inode_t *inode)
 {
-        int32_t ret = 0;
-        uint64_t ctx_addr = 0;
-        br_stub_inode_ctx_t *ctx = NULL;
-
-        LOCK (&inode->lock);
-        {
-                ret = __inode_ctx_get (inode, this, &ctx_addr);
-                if (ret < 0)
-                        goto unblock;
-                ctx = (br_stub_inode_ctx_t *) (long) ctx_addr;
-                *version = ctx->currentversion;
-        }
- unblock:
-        UNLOCK (&inode->lock);
-
-        return ret;
+    int32_t ret = -1;
+    uint64_t ctx_addr = 0;
+    br_stub_inode_ctx_t *ctx = NULL;
+
+    ret = br_stub_get_inode_ctx(this, inode, &ctx_addr);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_ERROR, 0, BRS_MSG_GET_INODE_CONTEXT_FAILED,
+                "inode-gfid=%s", uuid_utoa(inode->gfid), NULL);
+        goto out;
+    }
+
+    ctx = (br_stub_inode_ctx_t *)(long)ctx_addr;
+
+    LOCK(&inode->lock);
+    {
+        __br_stub_mark_object_bad(ctx);
+    }
+    UNLOCK(&inode->lock);
+
+out:
+    return ret;
 }
 
 /**
- * fetch the current version from inode and return the context.
- * inode->lock should be held before invoking this as context
- * *needs* to be valid in the caller.
+ * There is a possibility that dict_set might fail. The o/p of dict_set is
+ * given to the caller and the caller has to decide what to do.
  */
-static inline br_stub_inode_ctx_t *
-__br_stub_get_ongoing_version_ctx (xlator_t *this,
-                                   inode_t *inode, unsigned long *version)
+static inline int32_t
+br_stub_mark_xdata_bad_object(xlator_t *this, inode_t *inode, dict_t *xdata)
 {
-        int32_t ret = 0;
-        uint64_t ctx_addr = 0;
-        br_stub_inode_ctx_t *ctx = NULL;
+    int32_t ret = 0;
 
-        ret = __inode_ctx_get (inode, this, &ctx_addr);
-        if (ret < 0)
-                return NULL;
-        ctx = (br_stub_inode_ctx_t *) (long) ctx_addr;
-        if (version)
-                *version = ctx->currentversion;
+    if (br_stub_is_bad_object(this, inode) == -2)
+        ret = dict_set_int32(xdata, GLUSTERFS_BAD_INODE, 1);
 
-        return ctx;
+    return ret;
 }
 
-/* filter for xattr fetch */
-static inline int
-br_stub_is_internal_xattr (const char *name)
-{
-        if (name
-            && ((strncmp (name, BITROT_CURRENT_VERSION_KEY,
-                          strlen (BITROT_CURRENT_VERSION_KEY)) == 0)
-                || (strncmp (name, BITROT_SIGNING_VERSION_KEY,
-                             strlen (BITROT_SIGNING_VERSION_KEY)) == 0)))
-                return 1;
-        return 0;
-}
+int32_t
+br_stub_add_fd_to_inode(xlator_t *this, fd_t *fd, br_stub_inode_ctx_t *ctx);
 
-static inline void
-br_stub_remove_vxattrs (dict_t *xattr)
-{
-        if (xattr) {
-                dict_del (xattr, BITROT_CURRENT_VERSION_KEY);
-                dict_del (xattr, BITROT_SIGNING_VERSION_KEY);
-                dict_del (xattr, BITROT_SIGNING_XATTR_SIZE_KEY);
-        }
-}
+br_sign_state_t
+__br_stub_inode_sign_state(br_stub_inode_ctx_t *ctx, glusterfs_fop_t fop,
+                           fd_t *fd);
+
+int
+br_stub_dir_create(xlator_t *this, br_stub_private_t *priv);
+
+int
+br_stub_add(xlator_t *this, uuid_t gfid);
+
+int32_t
+br_stub_create_stub_gfid(xlator_t *this, char *stub_gfid_path, uuid_t gfid);
+
+int
+br_stub_dir_create(xlator_t *this, br_stub_private_t *priv);
+
+call_stub_t *
+__br_stub_dequeue(struct list_head *callstubs);
+
+void
+__br_stub_enqueue(struct list_head *callstubs, call_stub_t *stub);
+
+void
+br_stub_worker_enqueue(xlator_t *this, call_stub_t *stub);
+
+void *
+br_stub_worker(void *data);
+
+int32_t
+br_stub_lookup_wrapper(call_frame_t *frame, xlator_t *this, loc_t *loc,
+                       dict_t *xattr_req);
+
+int32_t
+br_stub_readdir_wrapper(call_frame_t *frame, xlator_t *this, fd_t *fd,
+                        size_t size, off_t off, dict_t *xdata);
+
+int
+br_stub_del(xlator_t *this, uuid_t gfid);
+
+int
+br_stub_bad_objects_path(xlator_t *this, fd_t *fd, gf_dirent_t *entries,
+                         dict_t **dict);
+
+void
+br_stub_entry_xattr_fill(xlator_t *this, char *hpath, gf_dirent_t *entry,
+                         dict_t *dict);
+
+int
+br_stub_get_path_of_gfid(xlator_t *this, inode_t *parent, inode_t *inode,
+                         uuid_t gfid, char **path);
 
 #endif /* __BIT_ROT_STUB_H__ */
diff --git a/xlators/features/changelog/lib/examples/c/get-changes-multi.c b/xlators/features/changelog/lib/examples/c/get-changes-multi.c
index 3741bdf6edc..5ea5bbb6630 100644
--- a/xlators/features/changelog/lib/examples/c/get-changes-multi.c
+++ b/xlators/features/changelog/lib/examples/c/get-changes-multi.c
@@ -25,64 +25,66 @@
 
 #include "changelog.h"
 
-void *brick_init (void *xl, struct gf_brick_spec *brick)
+void *
+brick_init(void *xl, struct gf_brick_spec *brick)
 {
-        return brick;
+    return brick;
 }
 
-void brick_fini (void *xl, char *brick, void *data)
+void
+brick_fini(void *xl, char *brick, void *data)
 {
-        return;
+    return;
 }
 
-void brick_callback (void *xl, char *brick,
-                    void *data, changelog_event_t *ev)
+void
+brick_callback(void *xl, char *brick, void *data, changelog_event_t *ev)
 {
-        printf ("->callback: (brick,type) [%s:%d]\n", brick, ev->ev_type);
+    printf("->callback: (brick,type) [%s:%d]\n", brick, ev->ev_type);
 }
 
-void fill_brick_spec (struct gf_brick_spec *brick, char *path)
+void
+fill_brick_spec(struct gf_brick_spec *brick, char *path)
 {
-        brick->brick_path = strdup (path);
-        brick->filter = CHANGELOG_OP_TYPE_BR_RELEASE;
-
-        brick->init         = brick_init;
-        brick->fini         = brick_fini;
-        brick->callback     = brick_callback;
-        brick->connected    = NULL;
-        brick->disconnected = NULL;
+    brick->brick_path = strdup(path);
+    brick->filter = CHANGELOG_OP_TYPE_BR_RELEASE;
+
+    brick->init = brick_init;
+    brick->fini = brick_fini;
+    brick->callback = brick_callback;
+    brick->connected = NULL;
+    brick->disconnected = NULL;
 }
 
 int
-main (int argc, char **argv)
+main(int argc, char **argv)
 {
-        int ret = 0;
-        void *bricks = NULL;
-        struct gf_brick_spec *brick = NULL;
+    int ret = 0;
+    void *bricks = NULL;
+    struct gf_brick_spec *brick = NULL;
 
-        bricks = calloc (2, sizeof (struct gf_brick_spec));
-        if (!bricks)
-                goto error_return;
+    bricks = calloc(2, sizeof(struct gf_brick_spec));
+    if (!bricks)
+        goto error_return;
 
-        brick = (struct gf_brick_spec *)bricks;
-        fill_brick_spec (brick, "/export/z1/zwoop");
+    brick = (struct gf_brick_spec *)bricks;
+    fill_brick_spec(brick, "/export/z1/zwoop");
 
-        brick++;
-        fill_brick_spec (brick, "/export/z2/zwoop");
+    brick++;
+    fill_brick_spec(brick, "/export/z2/zwoop");
 
-        ret = gf_changelog_init (NULL);
-        if (ret)
-                goto error_return;
+    ret = gf_changelog_init(NULL);
+    if (ret)
+        goto error_return;
 
-        ret = gf_changelog_register_generic ((struct gf_brick_spec *)bricks, 2,
-                                             0, "/tmp/multi-changes.log", 9,
-                                             NULL);
-        if (ret)
-                goto error_return;
+    ret = gf_changelog_register_generic((struct gf_brick_spec *)bricks, 2, 0,
+                                        "/tmp/multi-changes.log", 9, NULL);
+    if (ret)
+        goto error_return;
 
-        /* let callbacks do the job */
-        select (0, NULL, NULL, NULL, NULL);
+    /* let callbacks do the job */
+    select(0, NULL, NULL, NULL, NULL);
 
- error_return:
-        return -1;
+error_return:
+    return -1;
 }
diff --git a/xlators/features/changelog/lib/examples/c/get-changes.c b/xlators/features/changelog/lib/examples/c/get-changes.c
index 0b2808c7e35..8bc651c24a4 100644
--- a/xlators/features/changelog/lib/examples/c/get-changes.c
+++ b/xlators/features/changelog/lib/examples/c/get-changes.c
@@ -27,61 +27,67 @@
 
 #include "changelog.h"
 
-#define handle_error(fn)                                \
-        printf ("%s (reason: %s)\n", fn, strerror (errno))
+#define handle_error(fn) printf("%s (reason: %s)\n", fn, strerror(errno))
 
 int
-main (int argc, char ** argv)
+main(int argc, char **argv)
 {
-        int     i           = 0;
-        int     ret         = 0;
-        ssize_t nr_changes  = 0;
-        ssize_t changes     = 0;
-        char fbuf[PATH_MAX] = {0,};
-
-        /* get changes for brick "/home/vshankar/export/yow/yow-1" */
-        ret = gf_changelog_register ("/export/z1/zwoop",
-                                     "/tmp/scratch", "/tmp/change.log", 9, 5);
-        if (ret) {
-                handle_error ("register failed");
-                goto out;
+    int i = 0;
+    int ret = 0;
+    ssize_t nr_changes = 0;
+    ssize_t changes = 0;
+    char fbuf[PATH_MAX] = {
+        0,
+    };
+
+    ret = gf_changelog_init(NULL);
+    if (ret) {
+        handle_error("Init failed");
+        goto out;
+    }
+
+    /* get changes for brick "/home/vshankar/export/yow/yow-1" */
+    ret = gf_changelog_register("/export/z1/zwoop", "/tmp/scratch",
+                                "/tmp/change.log", 9, 5);
+    if (ret) {
+        handle_error("register failed");
+        goto out;
+    }
+
+    while (1) {
+        i = 0;
+        nr_changes = gf_changelog_scan();
+        if (nr_changes < 0) {
+            handle_error("scan(): ");
+            break;
         }
 
-        while (1) {
-                i = 0;
-                nr_changes = gf_changelog_scan ();
-                if (nr_changes < 0) {
-                        handle_error ("scan(): ");
-                        break;
-                }
+        if (nr_changes == 0)
+            goto next;
 
-                if (nr_changes == 0)
-                        goto next;
+        printf("Got %ld changelog files\n", nr_changes);
 
-                printf ("Got %ld changelog files\n", nr_changes);
+        while ((changes = gf_changelog_next_change(fbuf, PATH_MAX)) > 0) {
+            printf("changelog file [%d]: %s\n", ++i, fbuf);
 
-                while ( (changes =
-                         gf_changelog_next_change (fbuf, PATH_MAX)) > 0) {
-                        printf ("changelog file [%d]: %s\n", ++i, fbuf);
+            /* process changelog */
+            /* ... */
+            /* ... */
+            /* ... */
+            /* done processing */
 
-                        /* process changelog */
-                        /* ... */
-                        /* ... */
-                        /* ... */
-                        /* done processing */
-
-                        ret = gf_changelog_done (fbuf);
-                        if (ret)
-                                handle_error ("gf_changelog_done");
-                }
+            ret = gf_changelog_done(fbuf);
+            if (ret)
+                handle_error("gf_changelog_done");
+        }
 
-                if (changes == -1)
-                        handle_error ("gf_changelog_next_change");
+        if (changes == -1)
+            handle_error("gf_changelog_next_change");
 
-        next:
-                sleep (10);
-        }
+    next:
+        sleep(10);
+    }
 
- out:
-        return ret;
+out:
+    return ret;
 }
diff --git a/xlators/features/changelog/lib/examples/c/get-history.c b/xlators/features/changelog/lib/examples/c/get-history.c
index 2e1ff3c767f..3e888d75ca6 100644
--- a/xlators/features/changelog/lib/examples/c/get-history.c
+++ b/xlators/features/changelog/lib/examples/c/get-history.c
@@ -27,84 +27,90 @@
 
 #include "changelog.h"
 
-#define handle_error(fn)                                \
-        printf ("%s (reason: %s)\n", fn, strerror (errno))
+#define handle_error(fn) printf("%s (reason: %s)\n", fn, strerror(errno))
 
 int
-main (int argc, char ** argv)
+main(int argc, char **argv)
 {
-        int     i            = 0;
-        int     ret          = 0;
-        ssize_t nr_changes   = 0;
-        ssize_t changes      = 0;
-        char fbuf[PATH_MAX]  = {0,};
-        unsigned long end_ts = 0;
-
-        ret = gf_changelog_register ("/export/z1/zwoop",
-                                     "/tmp/scratch_v1", "/tmp/changes.log",
-                                     9, 5);
-        if (ret) {
-                handle_error ("register failed");
-                goto out;
+    int i = 0;
+    int ret = 0;
+    ssize_t nr_changes = 0;
+    ssize_t changes = 0;
+    char fbuf[PATH_MAX] = {
+        0,
+    };
+    unsigned long end_ts = 0;
+
+    ret = gf_changelog_init(NULL);
+    if (ret) {
+        handle_error("init failed");
+        goto out;
+    }
+
+    ret = gf_changelog_register("/export/z1/zwoop", "/tmp/scratch_v1",
+                                "/tmp/changes.log", 9, 5);
+    if (ret) {
+        handle_error("register failed");
+        goto out;
+    }
+
+    int a, b;
+    printf("give the two numbers start and end\t");
+    scanf("%d%d", &a, &b);
+    ret = gf_history_changelog("/export/z1/zwoop/.glusterfs/changelogs", a, b,
+                               3, &end_ts);
+    if (ret == -1) {
+        printf("history failed");
+        goto out;
+    }
+
+    printf("end time till when changelog available : %d , ret(%d) \t", end_ts,
+           ret);
+    fflush(stdout);
+
+    while (1) {
+        nr_changes = gf_history_changelog_scan();
+        printf("scanned, nr_changes : %d\n", nr_changes);
+        if (nr_changes < 0) {
+            handle_error("scan(): ");
+            break;
         }
 
-        int a, b;
-        printf ("give the two numbers start and end\t");
-        scanf ("%d%d", &a, &b);
-        ret = gf_history_changelog ("/export/z1/zwoop/.glusterfs/changelogs",
-                                    a, b, 3, &end_ts);
-        if (ret == -1) {
-                printf ("history failed");
-                goto out;
+        if (nr_changes == 0) {
+            printf("done scanning \n");
+            goto out;
         }
 
-        printf ("end time till when changelog available : %d , ret(%d) \t", end_ts, ret);
-        fflush(stdout);
-
-        while (1) {
-                nr_changes = gf_history_changelog_scan ();
-                printf ("scanned, nr_changes : %d\n",nr_changes);
-                if (nr_changes < 0) {
-                        handle_error ("scan(): ");
-                        break;
-                }
-
-                if (nr_changes == 0) {
-                        printf ("done scanning \n");
-                        goto out;
-                }
-
-                printf ("Got %ld changelog files\n", nr_changes);
-
-                while ( (changes =
-                         gf_history_changelog_next_change (fbuf, PATH_MAX)) > 0) {
-                        printf ("changelog file [%d]: %s\n", ++i, fbuf);
-
-                        /* process changelog */
-                        /* ... */
-                        /* ... */
-                        /* ... */
-                        /* done processing */
-
-                        ret = gf_history_changelog_done (fbuf);
-                        if (ret)
-                                handle_error ("gf_changelog_done");
-                }
-                /*
-                if (changes == -1)
-                        handle_error ("gf_changelog_next_change");
-                if (nr_changes ==1){
-                        printf("continue scanning\n");
-                }
-
-                if(nr_changes == 0){
-                        printf("done scanning \n");
-                        goto out;
-                }
-                */
+        printf("Got %ld changelog files\n", nr_changes);
+
+        while ((changes = gf_history_changelog_next_change(fbuf, PATH_MAX)) >
+               0) {
+            printf("changelog file [%d]: %s\n", ++i, fbuf);
+
+            /* process changelog */
+            /* ... */
+            /* ... */
+            /* ... */
+            /* done processing */
+
+            ret = gf_history_changelog_done(fbuf);
+            if (ret)
+                handle_error("gf_changelog_done");
+        }
+        /*
+        if (changes == -1)
+                handle_error ("gf_changelog_next_change");
+        if (nr_changes ==1){
+                printf("continue scanning\n");
         }
 
+        if(nr_changes == 0){
+                printf("done scanning \n");
+                goto out;
+        }
+        */
+    }
 
 out:
-        return ret;
+    return ret;
 }
diff --git a/xlators/features/changelog/lib/examples/python/changes.py b/xlators/features/changelog/lib/examples/python/changes.py
index d21db8eab2e..c410d3b000d 100644..100755
--- a/xlators/features/changelog/lib/examples/python/changes.py
+++ b/xlators/features/changelog/lib/examples/python/changes.py
@@ -1,5 +1,6 @@
-#!/usr/bin/python
+#!/usr/bin/python3
 
+from __future__ import print_function
 import os
 import sys
 import time
@@ -10,23 +11,24 @@ cl = libgfchangelog.Changes()
 def get_changes(brick, scratch_dir, log_file, log_level, interval):
     change_list = []
     try:
+        cl.cl_init()
         cl.cl_register(brick, scratch_dir, log_file, log_level)
         while True:
             cl.cl_scan()
             change_list = cl.cl_getchanges()
             if change_list:
-                print change_list
+                print(change_list)
             for change in change_list:
-                print('done with %s' % (change))
+                print(('done with %s' % (change)))
                 cl.cl_done(change)
             time.sleep(interval)
     except OSError:
         ex = sys.exc_info()[1]
-        print ex
+        print(ex)
 
 if __name__ == '__main__':
-    if len(sys.argv) != 5:
-        print("usage: %s <brick> <scratch-dir> <log-file> <fetch-interval>"
-              % (sys.argv[0]))
+    if len(sys.argv) != 6:
+        print(("usage: %s <brick> <scratch-dir> <log-file> <fetch-interval>"
+              % (sys.argv[0])))
         sys.exit(1)
     get_changes(sys.argv[1], sys.argv[2], sys.argv[3], 9, int(sys.argv[4]))
diff --git a/xlators/features/changelog/lib/examples/python/libgfchangelog.py b/xlators/features/changelog/lib/examples/python/libgfchangelog.py
index 68ec3baf144..2da9f2d2a8c 100644
--- a/xlators/features/changelog/lib/examples/python/libgfchangelog.py
+++ b/xlators/features/changelog/lib/examples/python/libgfchangelog.py
@@ -3,7 +3,8 @@ from ctypes import *
 from ctypes.util import find_library
 
 class Changes(object):
-    libgfc = CDLL(find_library("gfchangelog"), use_errno=True)
+    libgfc = CDLL(find_library("gfchangelog"), mode=RTLD_GLOBAL,
+                  use_errno=True)
 
     @classmethod
     def geterrno(cls):
@@ -19,6 +20,12 @@ class Changes(object):
         return getattr(cls.libgfc, call)
 
     @classmethod
+    def cl_init(cls):
+        ret = cls._get_api('gf_changelog_init')(None)
+        if ret == -1:
+            cls.raise_changelog_err()
+
+    @classmethod
     def cl_register(cls, brick, path, log_file, log_level, retries = 0):
         ret = cls._get_api('gf_changelog_register')(brick, path,
                                                     log_file, log_level, retries)
diff --git a/xlators/features/changelog/lib/src/Makefile.am b/xlators/features/changelog/lib/src/Makefile.am
index 7da4f7c78ae..c933ec53ed2 100644
--- a/xlators/features/changelog/lib/src/Makefile.am
+++ b/xlators/features/changelog/lib/src/Makefile.am
@@ -1,29 +1,33 @@
 libgfchangelog_la_CFLAGS = -Wall $(GF_CFLAGS) $(GF_DARWIN_LIBGLUSTERFS_CFLAGS) \
-			-DDATADIR=\"$(localstatedir)\"
+	-DDATADIR=\"$(localstatedir)\"
 
-libgfchangelog_la_CPPFLAGS = $(GF_CPPFLAGS) -D__USE_FILE_OFFSET64 -fpic \
-				-I../../../src/ -I$(top_srcdir)/libglusterfs/src \
-				-I$(top_srcdir)/xlators/features/changelog/src \
-				-I$(top_srcdir)/rpc/xdr/src -I$(top_srcdir)/rpc/rpc-lib/src \
-				-I$(top_srcdir)/rpc/rpc-transport/socket/src \
-				-DDATADIR=\"$(localstatedir)\"
+libgfchangelog_la_CPPFLAGS = $(GF_CPPFLAGS) -D__USE_FILE_OFFSET64 -D__USE_LARGEFILE64 -fpic \
+	-I../../../src/ -I$(top_srcdir)/libglusterfs/src \
+	-I$(top_srcdir)/xlators/features/changelog/src \
+	-I$(top_srcdir)/rpc/xdr/src -I$(top_builddir)/rpc/xdr/src \
+	-I$(top_srcdir)/rpc/rpc-lib/src \
+	-I$(top_srcdir)/rpc/rpc-transport/socket/src \
+	-DDATADIR=\"$(localstatedir)\"
 
 libgfchangelog_la_LIBADD = $(top_builddir)/libglusterfs/src/libglusterfs.la \
-			   $(top_builddir)/rpc/xdr/src/libgfxdr.la \
-			   $(top_builddir)/rpc/rpc-lib/src/libgfrpc.la
+	$(top_builddir)/rpc/xdr/src/libgfxdr.la \
+	$(top_builddir)/rpc/rpc-lib/src/libgfrpc.la
 
-libgfchangelog_la_LDFLAGS = $(GF_LDFLAGS) -version-info $(LIBGFCHANGELOG_LT_VERSION)
+libgfchangelog_la_LDFLAGS = $(GF_LDFLAGS) \
+        -version-info $(LIBGFCHANGELOG_LT_VERSION) \
+        $(GF_NO_UNDEFINED)
 
-libgfchangelogdir = $(includedir)/glusterfs/gfchangelog
 lib_LTLIBRARIES = libgfchangelog.la
 
 CONTRIB_BUILDDIR = $(top_builddir)/contrib
 
-libgfchangelog_la_SOURCES = gf-changelog.c gf-changelog-journal-handler.c gf-changelog-helpers.c \
-		gf-changelog-api.c gf-history-changelog.c gf-changelog-rpc.c gf-changelog-reborp.c \
-		$(top_srcdir)/xlators/features/changelog/src/changelog-rpc-common.c
+libgfchangelog_la_SOURCES = gf-changelog.c gf-changelog-journal-handler.c \
+	gf-changelog-helpers.c gf-changelog-api.c gf-history-changelog.c \
+	gf-changelog-rpc.c gf-changelog-reborp.c \
+	$(top_srcdir)/xlators/features/changelog/src/changelog-rpc-common.c
 
-noinst_HEADERS = gf-changelog-helpers.h gf-changelog-rpc.h gf-changelog-journal.h
+noinst_HEADERS = gf-changelog-helpers.h gf-changelog-rpc.h \
+	gf-changelog-journal.h changelog-lib-messages.h
 
 CLEANFILES =
 
diff --git a/xlators/features/changelog/lib/src/changelog-lib-messages.h b/xlators/features/changelog/lib/src/changelog-lib-messages.h
new file mode 100644
index 00000000000..d7fe7274353
--- /dev/null
+++ b/xlators/features/changelog/lib/src/changelog-lib-messages.h
@@ -0,0 +1,74 @@
+/*
+ Copyright (c) 2015 Red Hat, Inc. <http://www.redhat.com>
+ This file is part of GlusterFS.
+
+ This file is licensed to you under your choice of the GNU Lesser
+ General Public License, version 3 or any later version (LGPLv3 or
+ later), or the GNU General Public License, version 2 (GPLv2), in all
+ cases as published by the Free Software Foundation.
+ */
+
+#ifndef _CHANGELOG_LIB_MESSAGES_H_
+#define _CHANGELOG_LIB_MESSAGES_H_
+
+#include <glusterfs/glfs-message-id.h>
+
+/* To add new message IDs, append new identifiers at the end of the list.
+ *
+ * Never remove a message ID. If it's not used anymore, you can rename it or
+ * leave it as it is, but not delete it. This is to prevent reutilization of
+ * IDs by other messages.
+ *
+ * The component name must match one of the entries defined in
+ * glfs-message-id.h.
+ */
+
+GLFS_MSGID(
+    CHANGELOG_LIB, CHANGELOG_LIB_MSG_OPEN_FAILED,
+    CHANGELOG_LIB_MSG_FAILED_TO_RMDIR,
+    CHANGELOG_LIB_MSG_SCRATCH_DIR_ENTRIES_CREATION_ERROR,
+    CHANGELOG_LIB_MSG_THREAD_CREATION_FAILED, CHANGELOG_LIB_MSG_OPENDIR_ERROR,
+    CHANGELOG_LIB_MSG_RENAME_FAILED, CHANGELOG_LIB_MSG_READ_ERROR,
+    CHANGELOG_LIB_MSG_HTIME_ERROR, CHANGELOG_LIB_MSG_GET_TIME_ERROR,
+    CHANGELOG_LIB_MSG_WRITE_FAILED, CHANGELOG_LIB_MSG_PTHREAD_ERROR,
+    CHANGELOG_LIB_MSG_MMAP_FAILED, CHANGELOG_LIB_MSG_MUNMAP_FAILED,
+    CHANGELOG_LIB_MSG_ASCII_ERROR, CHANGELOG_LIB_MSG_STAT_FAILED,
+    CHANGELOG_LIB_MSG_GET_XATTR_FAILED, CHANGELOG_LIB_MSG_PUBLISH_ERROR,
+    CHANGELOG_LIB_MSG_PARSE_ERROR, CHANGELOG_LIB_MSG_MIN_MAX_INFO,
+    CHANGELOG_LIB_MSG_CLEANUP_ERROR, CHANGELOG_LIB_MSG_UNLINK_FAILED,
+    CHANGELOG_LIB_MSG_NOTIFY_REGISTER_FAILED,
+    CHANGELOG_LIB_MSG_INVOKE_RPC_FAILED, CHANGELOG_LIB_MSG_DRAINING_EVENT_INFO,
+    CHANGELOG_LIB_MSG_CLEANING_BRICK_ENTRY_INFO,
+    CHANGELOG_LIB_MSG_FREEING_ENTRY_INFO, CHANGELOG_LIB_MSG_XDR_DECODING_FAILED,
+    CHANGELOG_LIB_MSG_NOTIFY_REGISTER_INFO,
+    CHANGELOG_LIB_MSG_THREAD_CLEANUP_WARNING,
+    CHANGELOG_LIB_MSG_COPY_FROM_BUFFER_FAILED,
+    CHANGELOG_LIB_MSG_PTHREAD_JOIN_FAILED, CHANGELOG_LIB_MSG_HIST_FAILED,
+    CHANGELOG_LIB_MSG_DRAINED_EVENT_INFO, CHANGELOG_LIB_MSG_PARSE_ERROR_CEASED,
+    CHANGELOG_LIB_MSG_REQUESTING_INFO, CHANGELOG_LIB_MSG_FINAL_INFO);
+
+#define CHANGELOG_LIB_MSG_NOTIFY_REGISTER_INFO_STR "Registering brick"
+#define CHANGELOG_LIB_MSG_RENAME_FAILED_STR "error moving changelog file"
+#define CHANGELOG_LIB_MSG_OPEN_FAILED_STR "cannot open changelog file"
+#define CHANGELOG_LIB_MSG_UNLINK_FAILED_STR "failed to unlink"
+#define CHANGELOG_LIB_MSG_FAILED_TO_RMDIR_STR "failed to rmdir"
+#define CHANGELOG_LIB_MSG_STAT_FAILED_STR "stat failed on changelog file"
+#define CHANGELOG_LIB_MSG_PARSE_ERROR_STR "could not parse changelog"
+#define CHANGELOG_LIB_MSG_PARSE_ERROR_CEASED_STR                               \
+    "parsing error, ceased publishing..."
+#define CHANGELOG_LIB_MSG_HTIME_ERROR_STR "fop failed on htime file"
+#define CHANGELOG_LIB_MSG_GET_XATTR_FAILED_STR                                 \
+    "error extracting max timstamp from htime file"
+#define CHANGELOG_LIB_MSG_MIN_MAX_INFO_STR "changelogs min max"
+#define CHANGELOG_LIB_MSG_REQUESTING_INFO_STR "Requesting historical changelogs"
+#define CHANGELOG_LIB_MSG_FINAL_INFO_STR "FINAL"
+#define CHANGELOG_LIB_MSG_HIST_FAILED_STR                                      \
+    "Requested changelog range is not available"
+#define CHANGELOG_LIB_MSG_GET_TIME_ERROR_STR "wrong result"
+#define CHANGELOG_LIB_MSG_CLEANING_BRICK_ENTRY_INFO_STR                        \
+    "Cleaning brick entry for brick"
+#define CHANGELOG_LIB_MSG_DRAINING_EVENT_INFO_STR "Draining event"
+#define CHANGELOG_LIB_MSG_DRAINED_EVENT_INFO_STR "Drained event"
+#define CHANGELOG_LIB_MSG_FREEING_ENTRY_INFO_STR "freeing entry"
+
+#endif /* !_CHANGELOG_MESSAGES_H_ */
diff --git a/xlators/features/changelog/lib/src/gf-changelog-api.c b/xlators/features/changelog/lib/src/gf-changelog-api.c
index eee3ad1eefd..81a5cbfec10 100644
--- a/xlators/features/changelog/lib/src/gf-changelog-api.c
+++ b/xlators/features/changelog/lib/src/gf-changelog-api.c
@@ -8,63 +8,65 @@
    cases as published by the Free Software Foundation.
 */
 
-#include "compat-uuid.h"
-#include "globals.h"
-#include "glusterfs.h"
+#include <glusterfs/compat-uuid.h>
+#include <glusterfs/globals.h>
+#include <glusterfs/glusterfs.h>
+#include <glusterfs/syscall.h>
 
 #include "gf-changelog-helpers.h"
 #include "gf-changelog-journal.h"
 #include "changelog-mem-types.h"
+#include "changelog-lib-messages.h"
 
 int
-gf_changelog_done (char *file)
+gf_changelog_done(char *file)
 {
-        int                     ret    = -1;
-        char                   *buffer = NULL;
-        xlator_t               *this   = NULL;
-        gf_changelog_journal_t *jnl    = NULL;
-        char to_path[PATH_MAX]         = {0,};
-
-        errno = EINVAL;
-
-        this = THIS;
-        if (!this)
-                goto out;
-
-        jnl = (gf_changelog_journal_t *) GF_CHANGELOG_GET_API_PTR (this);
-        if (!jnl)
-                goto out;
-
-        if (!file || !strlen (file))
-                goto out;
-
-        /* make sure 'file' is inside ->jnl_working_dir */
-        buffer = realpath (file, NULL);
-        if (!buffer)
-                goto out;
-
-        if (strncmp (jnl->jnl_working_dir,
-                     buffer, strlen (jnl->jnl_working_dir)))
-                goto out;
-
-        (void) snprintf (to_path, PATH_MAX, "%s%s",
-                         jnl->jnl_processed_dir, basename (buffer));
-        gf_log (this->name, GF_LOG_DEBUG,
-                "moving %s to processed directory", file);
-        ret = rename (buffer, to_path);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "cannot move %s to %s (reason: %s)",
-                        file, to_path, strerror (errno));
-                goto out;
-        }
-
-        ret = 0;
+    int ret = -1;
+    char *buffer = NULL;
+    xlator_t *this = NULL;
+    gf_changelog_journal_t *jnl = NULL;
+    char to_path[PATH_MAX] = {
+        0,
+    };
+
+    errno = EINVAL;
+
+    this = THIS;
+    if (!this)
+        goto out;
+
+    jnl = (gf_changelog_journal_t *)GF_CHANGELOG_GET_API_PTR(this);
+    if (!jnl)
+        goto out;
+
+    if (!file || !strlen(file))
+        goto out;
+
+    /* make sure 'file' is inside ->jnl_working_dir */
+    buffer = realpath(file, NULL);
+    if (!buffer)
+        goto out;
+
+    if (strncmp(jnl->jnl_working_dir, buffer, strlen(jnl->jnl_working_dir)))
+        goto out;
+
+    (void)snprintf(to_path, PATH_MAX, "%s%s", jnl->jnl_processed_dir,
+                   basename(buffer));
+    gf_msg_debug(this->name, 0, "moving %s to processed directory", file);
+    ret = sys_rename(buffer, to_path);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno,
+                CHANGELOG_LIB_MSG_RENAME_FAILED, "from=%s", file, "to=%s",
+                to_path, NULL);
+        goto out;
+    }
+
+    ret = 0;
 
- out:
-        if (buffer)
-                free (buffer); /* allocated by realpath() */
-        return ret;
+out:
+    if (buffer)
+        free(buffer); /* allocated by realpath() */
+    return ret;
 }
 
 /**
@@ -72,28 +74,28 @@ gf_changelog_done (char *file)
  *  for a set of changelogs, start from the beginning
  */
 int
-gf_changelog_start_fresh ()
+gf_changelog_start_fresh()
 {
-        xlator_t *this = NULL;
-        gf_changelog_journal_t *jnl = NULL;
+    xlator_t *this = NULL;
+    gf_changelog_journal_t *jnl = NULL;
 
-        this = THIS;
-        if (!this)
-                goto out;
+    this = THIS;
+    if (!this)
+        goto out;
 
-        errno = EINVAL;
+    errno = EINVAL;
 
-        jnl = (gf_changelog_journal_t *) GF_CHANGELOG_GET_API_PTR (this);
-        if (!jnl)
-                goto out;
+    jnl = (gf_changelog_journal_t *)GF_CHANGELOG_GET_API_PTR(this);
+    if (!jnl)
+        goto out;
 
-        if (gf_ftruncate (jnl->jnl_fd, 0))
-                goto out;
+    if (gf_ftruncate(jnl->jnl_fd, 0))
+        goto out;
 
-        return 0;
+    return 0;
 
- out:
-        return -1;
+out:
+    return -1;
 }
 
 /**
@@ -102,40 +104,42 @@ gf_changelog_start_fresh ()
  * consumed.
  */
 ssize_t
-gf_changelog_next_change (char *bufptr, size_t maxlen)
+gf_changelog_next_change(char *bufptr, size_t maxlen)
 {
-        ssize_t         size       = -1;
-        int             tracker_fd = 0;
-        xlator_t       *this       = NULL;
-        gf_changelog_journal_t *jnl = NULL;
-        char buffer[PATH_MAX]      = {0,};
+    ssize_t size = -1;
+    int tracker_fd = 0;
+    xlator_t *this = NULL;
+    gf_changelog_journal_t *jnl = NULL;
+    char buffer[PATH_MAX] = {
+        0,
+    };
 
-        errno = EINVAL;
+    errno = EINVAL;
 
-        this = THIS;
-        if (!this)
-                goto out;
+    this = THIS;
+    if (!this)
+        goto out;
 
-        jnl = (gf_changelog_journal_t *) GF_CHANGELOG_GET_API_PTR (this);
-        if (!jnl)
-                goto out;
+    jnl = (gf_changelog_journal_t *)GF_CHANGELOG_GET_API_PTR(this);
+    if (!jnl)
+        goto out;
 
-        tracker_fd = jnl->jnl_fd;
+    tracker_fd = jnl->jnl_fd;
 
-        size = gf_readline (tracker_fd, buffer, maxlen);
-        if (size < 0) {
-                size = -1;
-                goto out;
-        }
+    size = gf_readline(tracker_fd, buffer, maxlen);
+    if (size < 0) {
+        size = -1;
+        goto out;
+    }
 
-        if (size == 0)
-                goto out;
+    if (size == 0)
+        goto out;
 
-        memcpy (bufptr, buffer, size - 1);
-        bufptr[size - 1] = '\0';
+    memcpy(bufptr, buffer, size - 1);
+    bufptr[size - 1] = '\0';
 
 out:
-        return size;
+    return size;
 }
 
 /**
@@ -147,78 +151,74 @@ out:
  * This call also acts as a cancellation point for the consumer.
  */
 ssize_t
-gf_changelog_scan ()
+gf_changelog_scan()
 {
-        int             ret        = 0;
-        int             tracker_fd = 0;
-        size_t          len        = 0;
-        size_t          off        = 0;
-        xlator_t       *this       = NULL;
-        size_t          nr_entries = 0;
-        gf_changelog_journal_t *jnl = NULL;
-        struct dirent  *entryp     = NULL;
-        struct dirent  *result     = NULL;
-        char buffer[PATH_MAX]      = {0,};
-
-        this = THIS;
-        if (!this)
-                goto out;
-
-        jnl = (gf_changelog_journal_t *) GF_CHANGELOG_GET_API_PTR (this);
-        if (!jnl)
-                goto out;
-        if (JNL_IS_API_DISCONNECTED (jnl)) {
-                errno = ENOTCONN;
-                goto out;
+    int tracker_fd = 0;
+    size_t off = 0;
+    xlator_t *this = NULL;
+    size_t nr_entries = 0;
+    gf_changelog_journal_t *jnl = NULL;
+    struct dirent *entry = NULL;
+    struct dirent scratch[2] = {
+        {
+            0,
+        },
+    };
+    char buffer[PATH_MAX] = {
+        0,
+    };
+
+    this = THIS;
+    if (!this)
+        goto out;
+
+    jnl = (gf_changelog_journal_t *)GF_CHANGELOG_GET_API_PTR(this);
+    if (!jnl)
+        goto out;
+    if (JNL_IS_API_DISCONNECTED(jnl)) {
+        errno = ENOTCONN;
+        goto out;
+    }
+
+    errno = EINVAL;
+
+    tracker_fd = jnl->jnl_fd;
+    if (gf_ftruncate(tracker_fd, 0))
+        goto out;
+
+    rewinddir(jnl->jnl_dir);
+
+    for (;;) {
+        errno = 0;
+        entry = sys_readdir(jnl->jnl_dir, scratch);
+        if (!entry || errno != 0)
+            break;
+
+        if (!strcmp(basename(entry->d_name), ".") ||
+            !strcmp(basename(entry->d_name), ".."))
+            continue;
+
+        nr_entries++;
+
+        GF_CHANGELOG_FILL_BUFFER(jnl->jnl_processing_dir, buffer, off,
+                                 strlen(jnl->jnl_processing_dir));
+        GF_CHANGELOG_FILL_BUFFER(entry->d_name, buffer, off,
+                                 strlen(entry->d_name));
+        GF_CHANGELOG_FILL_BUFFER("\n", buffer, off, 1);
+
+        if (gf_changelog_write(tracker_fd, buffer, off) != off) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, CHANGELOG_LIB_MSG_WRITE_FAILED,
+                   "error writing changelog filename"
+                   " to tracker file");
+            break;
         }
+        off = 0;
+    }
 
-        errno = EINVAL;
-
-        tracker_fd = jnl->jnl_fd;
-        if (gf_ftruncate (tracker_fd, 0))
-                goto out;
-
-        len = offsetof(struct dirent, d_name)
-                + pathconf(jnl->jnl_processing_dir, _PC_NAME_MAX) + 1;
-        entryp = GF_CALLOC (1, len,
-                            gf_changelog_mt_libgfchangelog_dirent_t);
-        if (!entryp)
-                goto out;
-
-        rewinddir (jnl->jnl_dir);
-        while (1) {
-                ret = readdir_r (jnl->jnl_dir, entryp, &result);
-                if (ret || !result)
-                        break;
-
-                if (!strcmp (basename (entryp->d_name), ".")
-                     || !strcmp (basename (entryp->d_name), ".."))
-                        continue;
-
-                nr_entries++;
-
-                GF_CHANGELOG_FILL_BUFFER (jnl->jnl_processing_dir,
-                                          buffer, off,
-                                          strlen (jnl->jnl_processing_dir));
-                GF_CHANGELOG_FILL_BUFFER (entryp->d_name, buffer,
-                                          off, strlen (entryp->d_name));
-                GF_CHANGELOG_FILL_BUFFER ("\n", buffer, off, 1);
-
-                if (gf_changelog_write (tracker_fd, buffer, off) != off) {
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "error writing changelog filename"
-                                " to tracker file");
-                        break;
-                }
-                off = 0;
-        }
-
-        GF_FREE (entryp);
-
-        if (!result) {
-                if (gf_lseek (tracker_fd, 0, SEEK_SET) != -1)
-                        return nr_entries;
-        }
- out:
-        return -1;
+    if (!entry) {
+        if (gf_lseek(tracker_fd, 0, SEEK_SET) != -1)
+            return nr_entries;
+    }
+out:
+    return -1;
 }
diff --git a/xlators/features/changelog/lib/src/gf-changelog-helpers.c b/xlators/features/changelog/lib/src/gf-changelog-helpers.c
index 6bf709dc664..75f8a6dfc08 100644
--- a/xlators/features/changelog/lib/src/gf-changelog-helpers.c
+++ b/xlators/features/changelog/lib/src/gf-changelog-helpers.c
@@ -10,40 +10,37 @@
 
 #include "changelog-mem-types.h"
 #include "gf-changelog-helpers.h"
-
-ssize_t gf_changelog_read_path (int fd, char *buffer, size_t bufsize)
-{
-        return read (fd, buffer, bufsize);
-}
+#include "changelog-lib-messages.h"
+#include <glusterfs/syscall.h>
 
 size_t
-gf_changelog_write (int fd, char *buffer, size_t len)
+gf_changelog_write(int fd, char *buffer, size_t len)
 {
-        ssize_t size = 0;
-        size_t written = 0;
+    ssize_t size = 0;
+    size_t written = 0;
 
-        while (written < len) {
-                size = write (fd,
-                              buffer + written, len - written);
-                if (size <= 0)
-                        break;
+    while (written < len) {
+        size = sys_write(fd, buffer + written, len - written);
+        if (size <= 0)
+            break;
 
-                written += size;
-        }
+        written += size;
+    }
 
-        return written;
+    return written;
 }
 
 void
-gf_rfc3986_encode (unsigned char *s, char *enc, char *estr)
+gf_rfc3986_encode_space_newline(unsigned char *s, char *enc, char *estr)
 {
-        for (; *s; s++) {
-                if (estr[*s])
-                        sprintf(enc, "%c", estr[*s]);
-                else
-                        sprintf(enc, "%%%02X", *s);
-                while (*++enc);
-        }
+    for (; *s; s++) {
+        if (estr[*s])
+            sprintf(enc, "%c", estr[*s]);
+        else
+            sprintf(enc, "%%%02X", *s);
+        while (*++enc)
+            ;
+    }
 }
 
 /**
@@ -56,157 +53,118 @@ gf_rfc3986_encode (unsigned char *s, char *enc, char *estr)
  * that can be done via @fflush(fp), @ftruncate(fd) and @fseek(fp),
  * but this involves mixing POSIX file descriptors and stream FILE *).
  *
- * NOTE: This implmentation still does work with more than one fd's
+ * NOTE: This implementation still does work with more than one fd's
  *       used to perform gf_readline(). For this very reason it's not
  *       made a part of libglusterfs.
  */
 
-static pthread_key_t rl_key;
-static pthread_once_t rl_once = PTHREAD_ONCE_INIT;
-
-static void
-readline_destructor (void *ptr)
-{
-        GF_FREE (ptr);
-}
-
-static void
-readline_once (void)
-{
-        pthread_key_create (&rl_key, readline_destructor);
-}
+static __thread read_line_t thread_tsd = {};
 
 static ssize_t
-my_read (read_line_t *tsd, int fd, char *ptr)
-{
-        if (tsd->rl_cnt <= 0) {
-                if ( (tsd->rl_cnt = read (fd, tsd->rl_buf, MAXLINE)) < 0 )
-                        return -1;
-                else if (tsd->rl_cnt == 0)
-                        return 0;
-                tsd->rl_bufptr = tsd->rl_buf;
-        }
-
-        tsd->rl_cnt--;
-        *ptr = *tsd->rl_bufptr++;
-        return 1;
-}
-
-static int
-gf_readline_init_once (read_line_t **tsd)
+my_read(read_line_t *tsd, int fd, char *ptr)
 {
-        if (pthread_once (&rl_once, readline_once) != 0)
-                return -1;
+    if (tsd->rl_cnt <= 0) {
+        tsd->rl_cnt = sys_read(fd, tsd->rl_buf, MAXLINE);
 
-        *tsd = pthread_getspecific (rl_key);
-        if (*tsd)
-                goto out;
-
-        *tsd = GF_CALLOC (1, sizeof (**tsd),
-                          gf_changelog_mt_libgfchangelog_rl_t);
-        if (!*tsd)
-                return -1;
-
-        if (pthread_setspecific (rl_key, *tsd) != 0)
-                return -1;
+        if (tsd->rl_cnt < 0)
+            return -1;
+        else if (tsd->rl_cnt == 0)
+            return 0;
+        tsd->rl_bufptr = tsd->rl_buf;
+    }
 
- out:
-        return 0;
+    tsd->rl_cnt--;
+    *ptr = *tsd->rl_bufptr++;
+    return 1;
 }
 
 ssize_t
-gf_readline (int fd, void *vptr, size_t maxlen)
+gf_readline(int fd, void *vptr, size_t maxlen)
 {
-        size_t       n   = 0;
-        size_t       rc  = 0;
-        char         c   = ' ';
-        char        *ptr = NULL;
-        read_line_t *tsd = NULL;
-
-        if (gf_readline_init_once (&tsd))
-                return -1;
-
-        ptr = vptr;
-        for (n = 1; n < maxlen; n++) {
-                if ( (rc = my_read (tsd, fd, &c)) == 1 ) {
-                        *ptr++ = c;
-                        if (c == '\n')
-                                break;
-                } else if (rc == 0) {
-                        *ptr = '\0';
-                        return (n - 1);
-                } else
-                        return -1;
-        }
-
-        *ptr = '\0';
-        return n;
-
+    size_t n = 0;
+    size_t rc = 0;
+    char c = ' ';
+    char *ptr = NULL;
+    read_line_t *tsd = &thread_tsd;
+
+    ptr = vptr;
+    for (n = 1; n < maxlen; n++) {
+        if ((rc = my_read(tsd, fd, &c)) == 1) {
+            *ptr++ = c;
+            if (c == '\n')
+                break;
+        } else if (rc == 0) {
+            *ptr = '\0';
+            return (n - 1);
+        } else
+            return -1;
+    }
+
+    *ptr = '\0';
+    return n;
 }
 
 off_t
-gf_lseek (int fd, off_t offset, int whence)
+gf_lseek(int fd, off_t offset, int whence)
 {
-        off_t        off = 0;
-        read_line_t *tsd = NULL;
+    off_t off = 0;
+    read_line_t *tsd = &thread_tsd;
 
-        if (gf_readline_init_once (&tsd))
-                return -1;
+    off = sys_lseek(fd, offset, whence);
+    if (off == -1)
+        return -1;
 
-        if ( (off = lseek (fd, offset, whence)) == -1)
-                return -1;
+    tsd->rl_cnt = 0;
+    tsd->rl_bufptr = tsd->rl_buf;
 
-        tsd->rl_cnt = 0;
-        tsd->rl_bufptr = tsd->rl_buf;
-
-        return off;
+    return off;
 }
 
 int
-gf_ftruncate (int fd, off_t length)
+gf_ftruncate(int fd, off_t length)
 {
-        read_line_t *tsd = NULL;
+    read_line_t *tsd = &thread_tsd;
 
-        if (gf_readline_init_once (&tsd))
-                return -1;
+    if (sys_ftruncate(fd, 0))
+        return -1;
 
-        if (ftruncate (fd, 0))
-                return -1;
+    tsd->rl_cnt = 0;
+    tsd->rl_bufptr = tsd->rl_buf;
 
-        tsd->rl_cnt = 0;
-        tsd->rl_bufptr = tsd->rl_buf;
-
-        return 0;
+    return 0;
 }
 
 int
-gf_thread_cleanup (xlator_t *this, pthread_t thread)
+gf_thread_cleanup(xlator_t *this, pthread_t thread)
 {
-        int ret = 0;
-        void *res = NULL;
-
-        ret = pthread_cancel (thread);
-        if (ret != 0) {
-                gf_log (this->name, GF_LOG_WARNING,
-                        "Failed to send cancellation to thread");
-                goto error_return;
-        }
-
-        ret = pthread_join (thread, &res);
-        if (ret != 0) {
-                gf_log (this->name, GF_LOG_WARNING,
-                        "failed to join thread");
-                goto error_return;
-        }
-
-        if (res != PTHREAD_CANCELED) {
-                gf_log (this->name, GF_LOG_WARNING,
-                        "Thread could not be cleaned up");
-                goto error_return;
-        }
-
-        return 0;
-
- error_return:
-        return -1;
+    int ret = 0;
+    void *res = NULL;
+
+    ret = pthread_cancel(thread);
+    if (ret != 0) {
+        gf_msg(this->name, GF_LOG_WARNING, 0,
+               CHANGELOG_LIB_MSG_THREAD_CLEANUP_WARNING,
+               "Failed to send cancellation to thread");
+        goto error_return;
+    }
+
+    ret = pthread_join(thread, &res);
+    if (ret != 0) {
+        gf_msg(this->name, GF_LOG_WARNING, 0,
+               CHANGELOG_LIB_MSG_THREAD_CLEANUP_WARNING,
+               "failed to join thread");
+        goto error_return;
+    }
+
+    if (res != PTHREAD_CANCELED) {
+        gf_msg(this->name, GF_LOG_WARNING, 0,
+               CHANGELOG_LIB_MSG_THREAD_CLEANUP_WARNING,
+               "Thread could not be cleaned up");
+        goto error_return;
+    }
+
+    return 0;
+
+error_return:
+    return -1;
 }
diff --git a/xlators/features/changelog/lib/src/gf-changelog-helpers.h b/xlators/features/changelog/lib/src/gf-changelog-helpers.h
index adde1e57bbf..9c609d33172 100644
--- a/xlators/features/changelog/lib/src/gf-changelog-helpers.h
+++ b/xlators/features/changelog/lib/src/gf-changelog-helpers.h
@@ -14,36 +14,37 @@
 #include <unistd.h>
 #include <dirent.h>
 #include <limits.h>
-#include <pthread.h>
+#include <glusterfs/locking.h>
 
-#include <xlator.h>
+#include <glusterfs/xlator.h>
 
 #include "changelog.h"
 
 #include "changelog-rpc-common.h"
 #include "gf-changelog-journal.h"
 
-#define GF_CHANGELOG_TRACKER  "tracker"
+#define GF_CHANGELOG_TRACKER "tracker"
 
-#define GF_CHANGELOG_CURRENT_DIR    ".current"
-#define GF_CHANGELOG_PROCESSED_DIR  ".processed"
+#define GF_CHANGELOG_CURRENT_DIR ".current"
+#define GF_CHANGELOG_PROCESSED_DIR ".processed"
 #define GF_CHANGELOG_PROCESSING_DIR ".processing"
-#define GF_CHANGELOG_HISTORY_DIR    ".history"
+#define GF_CHANGELOG_HISTORY_DIR ".history"
 #define TIMESTAMP_LENGTH 10
 
 #ifndef MAXLINE
 #define MAXLINE 4096
 #endif
 
-#define GF_CHANGELOG_FILL_BUFFER(ptr, ascii, off, len) do {     \
-                memcpy (ascii + off, ptr, len);                 \
-                off += len;                                     \
-        } while (0)
+#define GF_CHANGELOG_FILL_BUFFER(ptr, ascii, off, len)                         \
+    do {                                                                       \
+        memcpy(ascii + off, ptr, len);                                         \
+        off += len;                                                            \
+    } while (0)
 
 typedef struct read_line {
-        int rl_cnt;
-        char *rl_bufptr;
-        char rl_buf[MAXLINE];
+    int rl_cnt;
+    char *rl_bufptr;
+    char rl_buf[MAXLINE];
 } read_line_t;
 
 struct gf_changelog;
@@ -55,51 +56,50 @@ struct gf_event;
  * ->next_seq holds the next _expected_ sequence number.
  */
 struct gf_event_list {
-        pthread_mutex_t lock;               /* protects this structure */
-        pthread_cond_t  cond;
+    pthread_mutex_t lock; /* protects this structure */
+    pthread_cond_t cond;
 
-        pthread_t invoker;
+    pthread_t invoker;
 
-        unsigned long next_seq;             /* next sequence number expected:
-                                               zero during bootstrap */
+    unsigned long next_seq; /* next sequence number expected:
+                               zero during bootstrap */
 
-        struct gf_changelog *entry;         /* backpointer to it's brick
-                                               encapsulator (entry) */
-        struct list_head events;            /* list of events */
+    struct gf_changelog *entry; /* backpointer to it's brick
+                                   encapsulator (entry) */
+    struct list_head events;    /* list of events */
 };
 
 /**
  * include a refcount if it's of use by additional layers
  */
 struct gf_event {
-        int count;
+    int count;
 
-        unsigned long seq;
+    unsigned long seq;
 
-        struct list_head list;
+    struct list_head list;
 
-        struct iovec iov[0];
+    struct iovec iov[0];
 };
-#define GF_EVENT_CALLOC_SIZE(cnt, len)                                  \
-        (sizeof (struct gf_event) + (cnt * sizeof (struct iovec)) + len)
+#define GF_EVENT_CALLOC_SIZE(cnt, len)                                         \
+    (sizeof(struct gf_event) + (cnt * sizeof(struct iovec)) + len)
 
 /**
  * assign the base address of the IO vector to the correct memory
 o * area and set it's addressable length.
  */
-#define GF_EVENT_ASSIGN_IOVEC(vec, event, len, pos)                     \
-        do {                                                            \
-                vec->iov_base = ((char *)event) +                       \
-                        sizeof (struct gf_event) +                      \
-                        (event->count * sizeof (struct iovec)) + pos;   \
-                vec->iov_len = len;                                     \
-                pos += len;                                             \
-        } while (0)
+#define GF_EVENT_ASSIGN_IOVEC(vec, event, len, pos)                            \
+    do {                                                                       \
+        vec->iov_base = ((char *)event) + sizeof(struct gf_event) +            \
+                        (event->count * sizeof(struct iovec)) + pos;           \
+        vec->iov_len = len;                                                    \
+        pos += len;                                                            \
+    } while (0)
 
 typedef enum gf_changelog_conn_state {
-        GF_CHANGELOG_CONN_STATE_PENDING = 0,
-        GF_CHANGELOG_CONN_STATE_ACCEPTED,
-        GF_CHANGELOG_CONN_STATE_DISCONNECTED,
+    GF_CHANGELOG_CONN_STATE_PENDING = 0,
+    GF_CHANGELOG_CONN_STATE_ACCEPTED,
+    GF_CHANGELOG_CONN_STATE_DISCONNECTED,
 } gf_changelog_conn_state_t;
 
 /**
@@ -107,153 +107,149 @@ typedef enum gf_changelog_conn_state {
  * notifications are streamed.
  */
 typedef struct gf_changelog {
-        gf_lock_t statelock;
-        gf_changelog_conn_state_t connstate;
+    gf_lock_t statelock;
+    gf_changelog_conn_state_t connstate;
 
-        xlator_t *this;
+    xlator_t *this;
 
-        struct list_head list;              /* list of instances */
+    struct list_head list; /* list of instances */
 
-        char brick[PATH_MAX];               /* brick path for this end-point */
+    char brick[PATH_MAX]; /* brick path for this end-point */
 
-        changelog_rpc_t grpc;               /* rpc{-clnt,svc} for this brick */
-#define RPC_PROBER(ent)  ent->grpc.rpc
-#define RPC_REBORP(ent)  ent->grpc.svc
-#define RPC_SOCK(ent)    ent->grpc.sock
+    changelog_rpc_t grpc; /* rpc{-clnt,svc} for this brick */
+#define RPC_PROBER(ent) ent->grpc.rpc
+#define RPC_REBORP(ent) ent->grpc.svc
+#define RPC_SOCK(ent) ent->grpc.sock
 
-        unsigned int notify;                /* notification flag(s) */
+    unsigned int notify; /* notification flag(s) */
 
-        FINI       *fini;                   /* destructor callback */
-        CALLBACK   *callback;               /* event callback dispatcher */
-        CONNECT    *connected;              /* connect callback */
-        DISCONNECT *disconnected;           /* disconnection callback */
+    FINI *fini;               /* destructor callback */
+    CALLBACK *callback;       /* event callback dispatcher */
+    CONNECT *connected;       /* connect callback */
+    DISCONNECT *disconnected; /* disconnection callback */
 
-        void *ptr;                          /* owner specific private data */
-        xlator_t *invokerxl;                /* consumers _this_, if valid,
-                                               assigned to THIS before cbk is
-                                               invoked */
+    void *ptr;           /* owner specific private data */
+    xlator_t *invokerxl; /* consumers _this_, if valid,
+                            assigned to THIS before cbk is
+                            invoked */
 
-        gf_boolean_t ordered;
+    gf_boolean_t ordered;
 
-        void (*queueevent) (struct gf_event_list *, struct gf_event *);
-        void (*pickevent) (struct gf_event_list *, struct gf_event **);
+    void (*queueevent)(struct gf_event_list *, struct gf_event *);
+    void (*pickevent)(struct gf_event_list *, struct gf_event **);
 
-        struct gf_event_list event;
+    struct gf_event_list event;
 } gf_changelog_t;
 
 static inline int
-gf_changelog_filter_check (gf_changelog_t *entry, changelog_event_t *event)
+gf_changelog_filter_check(gf_changelog_t *entry, changelog_event_t *event)
 {
-        if (event->ev_type & entry->notify)
-                return 1;
-        return 0;
+    if (event->ev_type & entry->notify)
+        return 1;
+    return 0;
 }
 
-#define GF_NEED_ORDERED_EVENTS(ent)  (ent->ordered == _gf_true)
+#define GF_NEED_ORDERED_EVENTS(ent) (ent->ordered == _gf_true)
 
 /** private structure */
 typedef struct gf_private {
-        pthread_mutex_t lock;            /* protects ->connections, cleanups */
-        pthread_cond_t  cond;
+    pthread_mutex_t lock; /* protects ->connections, cleanups */
+    pthread_cond_t cond;
 
-        void *api;                       /* pointer for API access */
+    void *api; /* pointer for API access */
 
-        pthread_t poller;                /* event poller thread */
-        pthread_t connectionjanitor;     /* connection cleaner */
+    pthread_t poller;            /* event poller thread */
+    pthread_t connectionjanitor; /* connection cleaner */
 
-        struct list_head connections;    /* list of connections */
-        struct list_head cleanups;       /* list of connection to be
-                                            cleaned up */
+    struct list_head connections; /* list of connections */
+    struct list_head cleanups;    /* list of connection to be
+                                     cleaned up */
 } gf_private_t;
 
-#define GF_CHANGELOG_GET_API_PTR(this) (((gf_private_t *) this->private)->api)
+#define GF_CHANGELOG_GET_API_PTR(this) (((gf_private_t *)this->private)->api)
 
 /**
  * upcall: invoke callback with _correct_ THIS
  */
-#define GF_CHANGELOG_INVOKE_CBK(this, cbk, brick, args ...)             \
-        do {                                                            \
-                xlator_t *old_this = NULL;                              \
-                xlator_t *invokerxl = NULL;                             \
-                                                                        \
-                invokerxl = entry->invokerxl;                           \
-                old_this = this;                                        \
-                                                                        \
-                if (invokerxl) {                                        \
-                        THIS = invokerxl;                               \
-                }                                                       \
-                                                                        \
-                cbk (invokerxl, brick, args);                           \
-                THIS = old_this;                                        \
-                                                                        \
-        } while (0)
-
-#define SAVE_THIS(xl)                           \
-        do {                                    \
-                old_this = xl;                  \
-                THIS = master;                  \
-        } while (0)
-
-#define RESTORE_THIS()                          \
-        do {                                    \
-                if (old_this)                   \
-                        THIS = old_this;        \
-        } while (0)
+#define GF_CHANGELOG_INVOKE_CBK(this, cbk, brick, args...)                     \
+    do {                                                                       \
+        xlator_t *old_this = NULL;                                             \
+        xlator_t *invokerxl = NULL;                                            \
+                                                                               \
+        invokerxl = entry->invokerxl;                                          \
+        old_this = this;                                                       \
+                                                                               \
+        if (invokerxl) {                                                       \
+            THIS = invokerxl;                                                  \
+        }                                                                      \
+                                                                               \
+        cbk(invokerxl, brick, args);                                           \
+        THIS = old_this;                                                       \
+                                                                               \
+    } while (0)
+
+#define SAVE_THIS(xl)                                                          \
+    do {                                                                       \
+        old_this = xl;                                                         \
+        THIS = master;                                                         \
+    } while (0)
+
+#define RESTORE_THIS()                                                         \
+    do {                                                                       \
+        if (old_this)                                                          \
+            THIS = old_this;                                                   \
+    } while (0)
 
 /** APIs and the rest */
 
 void *
-gf_changelog_process (void *data);
-
-ssize_t
-gf_changelog_read_path (int fd, char *buffer, size_t bufsize);
+gf_changelog_process(void *data);
 
 void
-gf_rfc3986_encode (unsigned char *s, char *enc, char *estr);
+gf_rfc3986_encode_space_newline(unsigned char *s, char *enc, char *estr);
 
 size_t
-gf_changelog_write (int fd, char *buffer, size_t len);
+gf_changelog_write(int fd, char *buffer, size_t len);
 
 ssize_t
-gf_readline (int fd, void *vptr, size_t maxlen);
+gf_readline(int fd, void *vptr, size_t maxlen);
 
 int
-gf_ftruncate (int fd, off_t length);
+gf_ftruncate(int fd, off_t length);
 
 off_t
-gf_lseek (int fd, off_t offset, int whence);
+gf_lseek(int fd, off_t offset, int whence);
 
 int
-gf_changelog_consume (xlator_t *this,
-                      gf_changelog_journal_t *jnl,
-                      char *from_path, gf_boolean_t no_publish);
+gf_changelog_consume(xlator_t *this, gf_changelog_journal_t *jnl,
+                     char *from_path, gf_boolean_t no_publish);
 int
-gf_changelog_publish (xlator_t *this,
-                      gf_changelog_journal_t *jnl, char *from_path);
+gf_changelog_publish(xlator_t *this, gf_changelog_journal_t *jnl,
+                     char *from_path);
 int
-gf_thread_cleanup (xlator_t *this, pthread_t thread);
+gf_thread_cleanup(xlator_t *this, pthread_t thread);
 void *
-gf_changelog_callback_invoker (void *arg);
+gf_changelog_callback_invoker(void *arg);
 
 int
-gf_cleanup_event (xlator_t *, struct gf_event_list *);
+gf_cleanup_event(xlator_t *, struct gf_event_list *);
 
 /* (un)ordered event queueing */
 void
-queue_ordered_event (struct gf_event_list *, struct gf_event *);
+queue_ordered_event(struct gf_event_list *, struct gf_event *);
 
 void
-queue_unordered_event (struct gf_event_list *, struct gf_event *);
+queue_unordered_event(struct gf_event_list *, struct gf_event *);
 
 /* (un)ordered event picking */
 void
-pick_event_ordered (struct gf_event_list *, struct gf_event **);
+pick_event_ordered(struct gf_event_list *, struct gf_event **);
 
 void
-pick_event_unordered (struct gf_event_list *, struct gf_event **);
+pick_event_unordered(struct gf_event_list *, struct gf_event **);
 
 /* connection janitor thread */
 void *
-gf_changelog_connection_janitor (void *);
+gf_changelog_connection_janitor(void *);
 
 #endif
diff --git a/xlators/features/changelog/lib/src/gf-changelog-journal-handler.c b/xlators/features/changelog/lib/src/gf-changelog-journal-handler.c
index 4029a562dfa..7f6e2329e71 100644
--- a/xlators/features/changelog/lib/src/gf-changelog-journal-handler.c
+++ b/xlators/features/changelog/lib/src/gf-changelog-journal-handler.c
@@ -8,9 +8,11 @@
    cases as published by the Free Software Foundation.
 */
 
-#include "compat-uuid.h"
-#include "globals.h"
-#include "glusterfs.h"
+#include <glusterfs/compat-uuid.h>
+#include <glusterfs/globals.h>
+#include <glusterfs/glusterfs.h>
+#include <glusterfs/syscall.h>
+#include <glusterfs/compat-errno.h>
 
 #include "gf-changelog-helpers.h"
 
@@ -19,86 +21,111 @@
 #include "changelog-mem-types.h"
 
 #include "gf-changelog-journal.h"
+#include "changelog-lib-messages.h"
 
 extern int byebye;
 
+enum changelog_versions { VERSION_1_1 = 0, VERSION_1_2 = 1 };
+
 /**
  * number of gfid records after fop number
  */
-int nr_gfids[] = {
-        [GF_FOP_MKNOD]   = 1,
-        [GF_FOP_MKDIR]   = 1,
-        [GF_FOP_UNLINK]  = 1,
-        [GF_FOP_RMDIR]   = 1,
-        [GF_FOP_SYMLINK] = 1,
-        [GF_FOP_RENAME]  = 2,
-        [GF_FOP_LINK]    = 1,
-        [GF_FOP_CREATE]  = 1,
-};
-
-int nr_extra_recs[] = {
-        [GF_FOP_MKNOD]   = 3,
-        [GF_FOP_MKDIR]   = 3,
-        [GF_FOP_UNLINK]  = 0,
-        [GF_FOP_RMDIR]   = 0,
-        [GF_FOP_SYMLINK] = 0,
-        [GF_FOP_RENAME]  = 0,
-        [GF_FOP_LINK]    = 0,
-        [GF_FOP_CREATE]  = 3,
-};
+int nr_gfids[2][GF_FOP_MAXVALUE] = {{
+                                        [GF_FOP_MKNOD] = 1,
+                                        [GF_FOP_MKDIR] = 1,
+                                        [GF_FOP_UNLINK] = 1,
+                                        [GF_FOP_RMDIR] = 1,
+                                        [GF_FOP_SYMLINK] = 1,
+                                        [GF_FOP_RENAME] = 2,
+                                        [GF_FOP_LINK] = 1,
+                                        [GF_FOP_CREATE] = 1,
+                                    },
+                                    {
+                                        [GF_FOP_MKNOD] = 1,
+                                        [GF_FOP_MKDIR] = 1,
+                                        [GF_FOP_UNLINK] = 2,
+                                        [GF_FOP_RMDIR] = 2,
+                                        [GF_FOP_SYMLINK] = 1,
+                                        [GF_FOP_RENAME] = 2,
+                                        [GF_FOP_LINK] = 1,
+                                        [GF_FOP_CREATE] = 1,
+                                    }};
+
+int nr_extra_recs[2][GF_FOP_MAXVALUE] = {{
+                                             [GF_FOP_MKNOD] = 3,
+                                             [GF_FOP_MKDIR] = 3,
+                                             [GF_FOP_UNLINK] = 0,
+                                             [GF_FOP_RMDIR] = 0,
+                                             [GF_FOP_SYMLINK] = 0,
+                                             [GF_FOP_RENAME] = 0,
+                                             [GF_FOP_LINK] = 0,
+                                             [GF_FOP_CREATE] = 3,
+                                         },
+                                         {
+                                             [GF_FOP_MKNOD] = 3,
+                                             [GF_FOP_MKDIR] = 3,
+                                             [GF_FOP_UNLINK] = 0,
+                                             [GF_FOP_RMDIR] = 0,
+                                             [GF_FOP_SYMLINK] = 0,
+                                             [GF_FOP_RENAME] = 0,
+                                             [GF_FOP_LINK] = 0,
+                                             [GF_FOP_CREATE] = 3,
+                                         }};
 
 static char *
-binary_to_ascii (uuid_t uuid)
+binary_to_ascii(uuid_t uuid)
 {
-        return uuid_utoa (uuid);
+    return uuid_utoa(uuid);
 }
 
 static char *
-conv_noop (char *ptr) { return ptr; }
-
-#define VERIFY_SEPARATOR(ptr, plen, perr)       \
-        {                                       \
-                if (*(ptr + plen) != '\0') {    \
-                        perr = 1;               \
-                        break;                  \
-                }                               \
-        }
-
-#define MOVER_MOVE(mover, nleft, bytes)         \
-        {                                       \
-                mover += bytes;                 \
-                nleft -= bytes;                 \
-        }                                       \
-
-#define PARSE_GFID(mov, ptr, le, fn, perr)                      \
-        {                                                       \
-                VERIFY_SEPARATOR (mov, le, perr);               \
-                ptr = fn (mov);                                 \
-                if (!ptr) {                                     \
-                        perr = 1;                               \
-                        break;                                  \
-                }                                               \
-        }
-
-#define FILL_AND_MOVE(pt, buf, of, mo, nl, le)                          \
-        {                                                               \
-                GF_CHANGELOG_FILL_BUFFER (pt, buf, of, strlen (pt));    \
-                MOVER_MOVE (mo, nl, le);                                \
-        }
-
-
-#define PARSE_GFID_MOVE(ptr, uuid, mover, nleft, perr)          \
-        {                                                       \
-                memcpy (uuid, mover, sizeof (uuid_t));          \
-                ptr = binary_to_ascii (uuid);                   \
-                if (!ptr) {                                     \
-                        perr = 1;                               \
-                        break;                                  \
-                }                                               \
-                MOVER_MOVE (mover, nleft, sizeof (uuid_t));     \
-        }                                                       \
+conv_noop(char *ptr)
+{
+    return ptr;
+}
 
-#define LINE_BUFSIZE  (3*PATH_MAX) /* enough buffer for extra chars too */
+#define VERIFY_SEPARATOR(ptr, plen, perr)                                      \
+    {                                                                          \
+        if (*(ptr + plen) != '\0') {                                           \
+            perr = 1;                                                          \
+            break;                                                             \
+        }                                                                      \
+    }
+
+#define MOVER_MOVE(mover, nleft, bytes)                                        \
+    {                                                                          \
+        mover += bytes;                                                        \
+        nleft -= bytes;                                                        \
+    }
+
+#define PARSE_GFID(mov, ptr, le, fn, perr)                                     \
+    {                                                                          \
+        VERIFY_SEPARATOR(mov, le, perr);                                       \
+        ptr = fn(mov);                                                         \
+        if (!ptr) {                                                            \
+            perr = 1;                                                          \
+            break;                                                             \
+        }                                                                      \
+    }
+
+#define FILL_AND_MOVE(pt, buf, of, mo, nl, le)                                 \
+    {                                                                          \
+        GF_CHANGELOG_FILL_BUFFER(pt, buf, of, strlen(pt));                     \
+        MOVER_MOVE(mo, nl, le);                                                \
+    }
+
+#define PARSE_GFID_MOVE(ptr, uuid, mover, nleft, perr)                         \
+    {                                                                          \
+        memcpy(uuid, mover, sizeof(uuid_t));                                   \
+        ptr = binary_to_ascii(uuid);                                           \
+        if (!ptr) {                                                            \
+            perr = 1;                                                          \
+            break;                                                             \
+        }                                                                      \
+        MOVER_MOVE(mover, nleft, sizeof(uuid_t));                              \
+    }
+
+#define LINE_BUFSIZE (3 * PATH_MAX) /* enough buffer for extra chars too */
 
 /**
  * using mmap() makes parsing easy. fgets() cannot be used here as
@@ -113,104 +140,107 @@ conv_noop (char *ptr) { return ptr; }
  */
 
 static int
-gf_changelog_parse_binary (xlator_t *this,
-                           gf_changelog_journal_t *jnl,
-                           int from_fd, int to_fd,
-                           size_t start_offset, struct stat *stbuf)
+gf_changelog_parse_binary(xlator_t *this, gf_changelog_journal_t *jnl,
+                          int from_fd, int to_fd, size_t start_offset,
+                          struct stat *stbuf, int version_idx)
 
 {
-        int     ret              = -1;
-        off_t   off              = 0;
-        off_t   nleft            = 0;
-        uuid_t  uuid             = {0,};
-        char   *ptr              = NULL;
-        char   *bname_start      = NULL;
-        char   *bname_end        = NULL;
-        char   *mover            = NULL;
-        void   *start            = NULL;
-        char    current_mover    = ' ';
-        size_t  blen             = 0;
-        int     parse_err        = 0;
-        char ascii[LINE_BUFSIZE] = {0,};
-
-        nleft = stbuf->st_size;
-
-        start = mmap (NULL, nleft, PROT_READ, MAP_PRIVATE, from_fd, 0);
-        if (start == MAP_FAILED) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "mmap() error (reason: %s)", strerror (errno));
-                goto out;
-        }
-
-        mover = start;
-
-        MOVER_MOVE (mover, nleft, start_offset);
-
-        while (nleft > 0) {
-
-                off = blen = 0;
-                ptr = bname_start = bname_end = NULL;
-
-                current_mover = *mover;
-
-                switch (current_mover) {
-                case 'D':
-                case 'M':
-                        MOVER_MOVE (mover, nleft, 1);
-                        PARSE_GFID_MOVE (ptr, uuid, mover, nleft, parse_err);
-
-                        break;
-
-                case 'E':
-                        MOVER_MOVE (mover, nleft, 1);
-                        PARSE_GFID_MOVE (ptr, uuid, mover, nleft, parse_err);
+    int ret = -1;
+    off_t off = 0;
+    off_t nleft = 0;
+    uuid_t uuid = {
+        0,
+    };
+    char *ptr = NULL;
+    char *bname_start = NULL;
+    char *bname_end = NULL;
+    char *mover = NULL;
+    void *start = NULL;
+    char current_mover = ' ';
+    size_t blen = 0;
+    int parse_err = 0;
+    char *ascii = NULL;
+
+    ascii = GF_CALLOC(LINE_BUFSIZE, sizeof(char), gf_common_mt_char);
+
+    nleft = stbuf->st_size;
+
+    start = mmap(NULL, nleft, PROT_READ, MAP_PRIVATE, from_fd, 0);
+    if (start == MAP_FAILED) {
+        gf_msg(this->name, GF_LOG_ERROR, errno, CHANGELOG_LIB_MSG_MMAP_FAILED,
+               "mmap() error");
+        goto out;
+    }
+
+    mover = start;
+
+    MOVER_MOVE(mover, nleft, start_offset);
+
+    while (nleft > 0) {
+        off = blen = 0;
+        ptr = bname_start = bname_end = NULL;
+
+        current_mover = *mover;
+
+        switch (current_mover) {
+            case 'D':
+            case 'M':
+                MOVER_MOVE(mover, nleft, 1);
+                PARSE_GFID_MOVE(ptr, uuid, mover, nleft, parse_err);
 
-                        bname_start = mover;
-                        bname_end = strchr (mover, '\n');
-                        if (bname_end == NULL) {
-                                parse_err = 1;
-                                break;
-                        }
-
-                        blen = bname_end - bname_start;
-                        MOVER_MOVE (mover, nleft, blen);
+                break;
 
-                        break;
+            case 'E':
+                MOVER_MOVE(mover, nleft, 1);
+                PARSE_GFID_MOVE(ptr, uuid, mover, nleft, parse_err);
 
-                default:
-                        parse_err = 1;
+                bname_start = mover;
+                bname_end = strchr(mover, '\n');
+                if (bname_end == NULL) {
+                    parse_err = 1;
+                    break;
                 }
 
-                if (parse_err)
-                        break;
+                blen = bname_end - bname_start;
+                MOVER_MOVE(mover, nleft, blen);
 
-                GF_CHANGELOG_FILL_BUFFER (&current_mover, ascii, off, 1);
-                GF_CHANGELOG_FILL_BUFFER (" ", ascii, off, 1);
-                GF_CHANGELOG_FILL_BUFFER (ptr, ascii, off, strlen (ptr));
-                if (blen)
-                        GF_CHANGELOG_FILL_BUFFER (bname_start,
-                                                  ascii, off, blen);
-                GF_CHANGELOG_FILL_BUFFER ("\n", ascii, off, 1);
-
-                if (gf_changelog_write (to_fd, ascii, off) != off) {
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "processing binary changelog failed due to "
-                                " error in writing ascii change (reason: %s)",
-                                strerror (errno));
-                        break;
-                }
+                break;
+
+            default:
+                parse_err = 1;
+        }
 
-                MOVER_MOVE (mover, nleft, 1);
+        if (parse_err)
+            break;
+
+        GF_CHANGELOG_FILL_BUFFER(&current_mover, ascii, off, 1);
+        GF_CHANGELOG_FILL_BUFFER(" ", ascii, off, 1);
+        GF_CHANGELOG_FILL_BUFFER(ptr, ascii, off, strlen(ptr));
+        if (blen)
+            GF_CHANGELOG_FILL_BUFFER(bname_start, ascii, off, blen);
+        GF_CHANGELOG_FILL_BUFFER("\n", ascii, off, 1);
+
+        if (gf_changelog_write(to_fd, ascii, off) != off) {
+            gf_msg(this->name, GF_LOG_ERROR, errno,
+                   CHANGELOG_LIB_MSG_ASCII_ERROR,
+                   "processing binary changelog failed due to "
+                   " error in writing ascii change");
+            break;
         }
 
-        if ((nleft == 0) && (!parse_err))
-                ret = 0;
+        MOVER_MOVE(mover, nleft, 1);
+    }
 
-        if (munmap (start, stbuf->st_size))
-                gf_log (this->name, GF_LOG_ERROR,
-                        "munmap() error (reason: %s)", strerror (errno));
- out:
-        return ret;
+    if ((nleft == 0) && (!parse_err))
+        ret = 0;
+
+    if (munmap(start, stbuf->st_size))
+        gf_msg(this->name, GF_LOG_ERROR, errno, CHANGELOG_LIB_MSG_MUNMAP_FAILED,
+               "munmap() error");
+out:
+    if (ascii)
+        GF_FREE(ascii);
+    return ret;
 }
 
 /**
@@ -219,768 +249,781 @@ gf_changelog_parse_binary (xlator_t *this,
  *  - use fop name rather than fop number
  */
 static int
-gf_changelog_parse_ascii (xlator_t *this,
-                          gf_changelog_journal_t *jnl,
-                          int from_fd, int to_fd,
-                          size_t start_offset, struct stat *stbuf)
+gf_changelog_parse_ascii(xlator_t *this, gf_changelog_journal_t *jnl,
+                         int from_fd, int to_fd, size_t start_offset,
+                         struct stat *stbuf, int version_idx)
 {
-        int           ng            = 0;
-        int           ret           = -1;
-        int           fop           = 0;
-        int           len           = 0;
-        off_t         off           = 0;
-        off_t         nleft         = 0;
-        char         *ptr           = NULL;
-        char         *eptr          = NULL;
-        void         *start         = NULL;
-        char         *mover         = NULL;
-        int           parse_err     = 0;
-        char          current_mover = ' ';
-        char ascii[LINE_BUFSIZE]    = {0,};
-        const char   *fopname       = NULL;
-
-        nleft = stbuf->st_size;
-
-        start = mmap (NULL, nleft, PROT_READ, MAP_PRIVATE, from_fd, 0);
-        if (start == MAP_FAILED) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "mmap() error (reason: %s)", strerror (errno));
-                goto out;
-        }
-
-        mover = start;
-
-        MOVER_MOVE (mover, nleft, start_offset);
-
-        while (nleft > 0) {
-                off = 0;
-                current_mover = *mover;
-
-                GF_CHANGELOG_FILL_BUFFER (&current_mover, ascii, off, 1);
-                GF_CHANGELOG_FILL_BUFFER (" ", ascii, off, 1);
-
-                switch (current_mover) {
-                case 'D':
-                        MOVER_MOVE (mover, nleft, 1);
-
-                        /* target gfid */
-                        PARSE_GFID (mover, ptr, UUID_CANONICAL_FORM_LEN,
-                                    conv_noop, parse_err);
-                        FILL_AND_MOVE(ptr, ascii, off,
-                                      mover, nleft, UUID_CANONICAL_FORM_LEN);
-                        break;
-                case 'M':
-                        MOVER_MOVE (mover, nleft, 1);
+    int ng = 0;
+    int ret = -1;
+    int fop = 0;
+    int len = 0;
+    off_t off = 0;
+    off_t nleft = 0;
+    char *ptr = NULL;
+    char *eptr = NULL;
+    void *start = NULL;
+    char *mover = NULL;
+    int parse_err = 0;
+    char current_mover = ' ';
+    char *ascii = NULL;
+    const char *fopname = NULL;
+
+    ascii = GF_CALLOC(LINE_BUFSIZE, sizeof(char), gf_common_mt_char);
+
+    nleft = stbuf->st_size;
+
+    start = mmap(NULL, nleft, PROT_READ, MAP_PRIVATE, from_fd, 0);
+    if (start == MAP_FAILED) {
+        gf_msg(this->name, GF_LOG_ERROR, errno, CHANGELOG_LIB_MSG_MMAP_FAILED,
+               "mmap() error");
+        goto out;
+    }
+
+    mover = start;
+
+    MOVER_MOVE(mover, nleft, start_offset);
+
+    while (nleft > 0) {
+        off = 0;
+        current_mover = *mover;
+
+        GF_CHANGELOG_FILL_BUFFER(&current_mover, ascii, off, 1);
+        GF_CHANGELOG_FILL_BUFFER(" ", ascii, off, 1);
+
+        switch (current_mover) {
+            case 'D':
+                MOVER_MOVE(mover, nleft, 1);
+
+                /* target gfid */
+                PARSE_GFID(mover, ptr, UUID_CANONICAL_FORM_LEN, conv_noop,
+                           parse_err);
+                FILL_AND_MOVE(ptr, ascii, off, mover, nleft,
+                              UUID_CANONICAL_FORM_LEN);
+                break;
+            case 'M':
+                MOVER_MOVE(mover, nleft, 1);
+
+                /* target gfid */
+                PARSE_GFID(mover, ptr, UUID_CANONICAL_FORM_LEN, conv_noop,
+                           parse_err);
+                FILL_AND_MOVE(ptr, ascii, off, mover, nleft,
+                              UUID_CANONICAL_FORM_LEN);
+                FILL_AND_MOVE(" ", ascii, off, mover, nleft, 1);
+
+                /* fop */
+                len = strlen(mover);
+                VERIFY_SEPARATOR(mover, len, parse_err);
+
+                fop = atoi(mover);
+                fopname = gf_fop_list[fop];
+                if (fopname == NULL) {
+                    parse_err = 1;
+                    break;
+                }
 
-                        /* target gfid */
-                        PARSE_GFID (mover, ptr, UUID_CANONICAL_FORM_LEN,
-                                    conv_noop, parse_err);
-                        FILL_AND_MOVE (ptr, ascii, off,
-                                       mover, nleft, UUID_CANONICAL_FORM_LEN);
-                        FILL_AND_MOVE (" ", ascii, off, mover, nleft, 1);
+                MOVER_MOVE(mover, nleft, len);
 
-                        /* fop */
-                        len = strlen (mover);
-                        VERIFY_SEPARATOR (mover, len, parse_err);
+                len = strlen(fopname);
+                GF_CHANGELOG_FILL_BUFFER(fopname, ascii, off, len);
 
-                        fop = atoi (mover);
-                        fopname = gf_fop_list[fop];
-                        if (fopname == NULL) {
-                                parse_err = 1;
-                                break;
-                        }
+                break;
 
-                        MOVER_MOVE (mover, nleft, len);
+            case 'E':
+                MOVER_MOVE(mover, nleft, 1);
+
+                /* target gfid */
+                PARSE_GFID(mover, ptr, UUID_CANONICAL_FORM_LEN, conv_noop,
+                           parse_err);
+                FILL_AND_MOVE(ptr, ascii, off, mover, nleft,
+                              UUID_CANONICAL_FORM_LEN);
+                FILL_AND_MOVE(" ", ascii, off, mover, nleft, 1);
+
+                /* fop */
+                len = strlen(mover);
+                VERIFY_SEPARATOR(mover, len, parse_err);
+
+                fop = atoi(mover);
+                fopname = gf_fop_list[fop];
+                if (fopname == NULL) {
+                    parse_err = 1;
+                    break;
+                }
 
-                        len = strlen (fopname);
-                        GF_CHANGELOG_FILL_BUFFER (fopname, ascii, off, len);
+                MOVER_MOVE(mover, nleft, len);
 
-                        break;
+                len = strlen(fopname);
+                GF_CHANGELOG_FILL_BUFFER(fopname, ascii, off, len);
 
-                case 'E':
-                        MOVER_MOVE (mover, nleft, 1);
-
-                        /* target gfid */
-                        PARSE_GFID (mover, ptr, UUID_CANONICAL_FORM_LEN,
-                                    conv_noop, parse_err);
-                        FILL_AND_MOVE (ptr, ascii, off,
-                                       mover, nleft, UUID_CANONICAL_FORM_LEN);
-                        FILL_AND_MOVE (" ", ascii, off,
-                                       mover, nleft, 1);
-
-                        /* fop */
-                        len = strlen (mover);
-                        VERIFY_SEPARATOR (mover, len, parse_err);
-
-                        fop = atoi (mover);
-                        fopname = gf_fop_list[fop];
-                        if (fopname == NULL) {
-                                parse_err = 1;
-                                break;
-                        }
-
-                        MOVER_MOVE (mover, nleft, len);
-
-                        len = strlen (fopname);
-                        GF_CHANGELOG_FILL_BUFFER (fopname, ascii, off, len);
-
-                        ng = nr_extra_recs[fop];
-                        for (; ng > 0; ng--) {
-                                MOVER_MOVE (mover, nleft, 1);
-                                len = strlen (mover);
-                                VERIFY_SEPARATOR (mover, len, parse_err);
-
-                                GF_CHANGELOG_FILL_BUFFER (" ", ascii, off, 1);
-                                FILL_AND_MOVE (mover, ascii,
-                                               off, mover, nleft, len);
-                        }
-
-                        /* pargfid + bname */
-                        ng = nr_gfids[fop];
-                        while (ng-- > 0) {
-                                MOVER_MOVE (mover, nleft, 1);
-                                len = strlen (mover);
-                                GF_CHANGELOG_FILL_BUFFER (" ", ascii, off, 1);
-
-                                PARSE_GFID (mover, ptr, len,
-                                            conv_noop, parse_err);
-                                eptr = calloc (3, strlen (ptr));
-                                if (!eptr) {
-                                        parse_err = 1;
-                                        break;
-                                }
-
-                                gf_rfc3986_encode ((unsigned char *) ptr,
-                                                   eptr, jnl->rfc3986);
-                                FILL_AND_MOVE (eptr, ascii, off,
-                                               mover, nleft, len);
-                                free (eptr);
-                        }
+                ng = nr_extra_recs[version_idx][fop];
+                for (; ng > 0; ng--) {
+                    MOVER_MOVE(mover, nleft, 1);
+                    len = strlen(mover);
+                    VERIFY_SEPARATOR(mover, len, parse_err);
 
-                        break;
-                default:
-                        parse_err = 1;
+                    GF_CHANGELOG_FILL_BUFFER(" ", ascii, off, 1);
+                    FILL_AND_MOVE(mover, ascii, off, mover, nleft, len);
                 }
 
-                if (parse_err)
+                /* pargfid + bname */
+                ng = nr_gfids[version_idx][fop];
+                while (ng-- > 0) {
+                    MOVER_MOVE(mover, nleft, 1);
+                    len = strlen(mover);
+                    if (!len) {
+                        MOVER_MOVE(mover, nleft, 1);
+                        continue;
+                    }
+
+                    GF_CHANGELOG_FILL_BUFFER(" ", ascii, off, 1);
+
+                    PARSE_GFID(mover, ptr, len, conv_noop, parse_err);
+                    eptr = calloc(3, strlen(ptr));
+                    if (!eptr) {
+                        parse_err = 1;
                         break;
+                    }
 
-                GF_CHANGELOG_FILL_BUFFER ("\n", ascii, off, 1);
-
-                if (gf_changelog_write (to_fd, ascii, off) != off) {
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "processing ascii changelog failed due to "
-                                " error in writing change (reason: %s)",
-                                strerror (errno));
-                        break;
+                    gf_rfc3986_encode_space_newline((unsigned char *)ptr, eptr,
+                                                    jnl->rfc3986_space_newline);
+                    FILL_AND_MOVE(eptr, ascii, off, mover, nleft, len);
+                    free(eptr);
                 }
 
-                MOVER_MOVE (mover, nleft, 1);
-
+                break;
+            default:
+                parse_err = 1;
         }
 
-        if ((nleft == 0) && (!parse_err))
-                ret = 0;
+        if (parse_err)
+            break;
 
-        if (munmap (start, stbuf->st_size))
-                gf_log (this->name, GF_LOG_ERROR,
-                        "munmap() error (reason: %s)", strerror (errno));
+        GF_CHANGELOG_FILL_BUFFER("\n", ascii, off, 1);
 
- out:
-        return ret;
-}
+        if (gf_changelog_write(to_fd, ascii, off) != off) {
+            gf_msg(this->name, GF_LOG_ERROR, errno,
+                   CHANGELOG_LIB_MSG_ASCII_ERROR,
+                   "processing ascii changelog failed due to "
+                   " error in writing change");
+            break;
+        }
 
-#define COPY_BUFSIZE  8192
-static int
-gf_changelog_copy (xlator_t *this, int from_fd, int to_fd)
-{
-        ssize_t size                  = 0;
-        char   buffer[COPY_BUFSIZE+1] = {0,};
+        MOVER_MOVE(mover, nleft, 1);
+    }
 
-        while (1) {
-                size = read (from_fd, buffer, COPY_BUFSIZE);
-                if (size <= 0)
-                        break;
+    if ((nleft == 0) && (!parse_err))
+        ret = 0;
 
-                if (gf_changelog_write (to_fd,
-                                        buffer, size) != size) {
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "error processing ascii changlog");
-                        size = -1;
-                        break;
-                }
-        }
+    if (munmap(start, stbuf->st_size))
+        gf_msg(this->name, GF_LOG_ERROR, errno, CHANGELOG_LIB_MSG_MUNMAP_FAILED,
+               "munmap() error");
+
+out:
+    if (ascii)
+        GF_FREE(ascii);
 
-        return (size < 0 ? -1 : 0);
+    return ret;
 }
 
 static int
-gf_changelog_decode (xlator_t *this, gf_changelog_journal_t *jnl,
-                     int from_fd, int to_fd, struct stat *stbuf, int *zerob)
+gf_changelog_decode(xlator_t *this, gf_changelog_journal_t *jnl, int from_fd,
+                    int to_fd, struct stat *stbuf, int *zerob)
 {
-        int    ret        = -1;
-        int    encoding   = -1;
-        size_t elen       = 0;
-        char buffer[1024] = {0,};
-
-        CHANGELOG_GET_ENCODING (from_fd, buffer, 1024, encoding, elen);
-        if (encoding == -1) /* unknown encoding */
-                goto out;
-
-        if (!CHANGELOG_VALID_ENCODING (encoding))
-                goto out;
-
-        if (elen == stbuf->st_size) {
-                *zerob = 1;
-                goto out;
-        }
-
-        /**
-         * start processing after the header
-         */
-        lseek (from_fd, elen, SEEK_SET);
-
-        switch (encoding) {
+    int ret = -1;
+    int encoding = -1;
+    int major_version = -1;
+    int minor_version = -1;
+    int version_idx = -1;
+    size_t elen = 0;
+    char buffer[1024] = {
+        0,
+    };
+
+    CHANGELOG_GET_HEADER_INFO(from_fd, buffer, sizeof(buffer), encoding,
+                              major_version, minor_version, elen);
+    if (encoding == -1) /* unknown encoding */
+        goto out;
+
+    if (major_version == -1) /* unknown major version */
+        goto out;
+
+    if (minor_version == -1) /* unknown minor version */
+        goto out;
+
+    if (!CHANGELOG_VALID_ENCODING(encoding))
+        goto out;
+
+    if (elen == stbuf->st_size) {
+        *zerob = 1;
+        goto out;
+    }
+
+    if (major_version == 1 && minor_version == 1) {
+        version_idx = VERSION_1_1;
+    } else if (major_version == 1 && minor_version == 2) {
+        version_idx = VERSION_1_2;
+    }
+
+    if (version_idx == -1) /* unknown version number */
+        goto out;
+
+    /**
+     * start processing after the header
+     */
+    if (sys_lseek(from_fd, elen, SEEK_SET) < 0) {
+        goto out;
+    }
+    switch (encoding) {
         case CHANGELOG_ENCODE_BINARY:
-                /**
-                 * this ideally should have been a part of changelog-encoders.c
-                 * (ie. part of the changelog translator).
-                 */
-                ret = gf_changelog_parse_binary (this, jnl, from_fd,
-                                                 to_fd, elen, stbuf);
-                break;
+            /**
+             * this ideally should have been a part of changelog-encoders.c
+             * (ie. part of the changelog translator).
+             */
+            ret = gf_changelog_parse_binary(this, jnl, from_fd, to_fd, elen,
+                                            stbuf, version_idx);
+            break;
 
         case CHANGELOG_ENCODE_ASCII:
-                ret = gf_changelog_parse_ascii (this, jnl, from_fd,
-                                                to_fd, elen, stbuf);
-                break;
-        default:
-                ret = gf_changelog_copy (this, from_fd, to_fd);
-        }
+            ret = gf_changelog_parse_ascii(this, jnl, from_fd, to_fd, elen,
+                                           stbuf, version_idx);
+            break;
+    }
 
- out:
-        return ret;
+out:
+    return ret;
 }
 
 int
-gf_changelog_publish (xlator_t *this,
-                      gf_changelog_journal_t *jnl, char *from_path)
+gf_changelog_publish(xlator_t *this, gf_changelog_journal_t *jnl,
+                     char *from_path)
 {
-        int         ret        = 0;
-        char dest[PATH_MAX]    = {0,};
-        char to_path[PATH_MAX] = {0,};
-        struct stat stbuf      = {0,};
-
-        (void) snprintf (to_path, PATH_MAX, "%s%s",
-                         jnl->jnl_current_dir, basename (from_path));
-
-        /* handle zerob file that wont exist in current */
-        ret = stat (to_path, &stbuf);
-        if (ret) {
-                if (errno == ENOENT)
-                        ret = 0;
-                goto out;
-        }
+    int ret = 0;
+    char dest[PATH_MAX] = {
+        0,
+    };
+    char to_path[PATH_MAX] = {
+        0,
+    };
+    struct stat stbuf = {
+        0,
+    };
+
+    if (snprintf(to_path, PATH_MAX, "%s%s", jnl->jnl_current_dir,
+                 basename(from_path)) >= PATH_MAX)
+        return -1;
 
-        (void) snprintf (dest, PATH_MAX, "%s%s",
-                         jnl->jnl_processing_dir, basename (from_path));
+    /* handle zerob file that won't exist in current */
+    ret = sys_stat(to_path, &stbuf);
+    if (ret) {
+        if (errno == ENOENT)
+            ret = 0;
+        goto out;
+    }
 
-        ret = rename (to_path, dest);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "error moving %s to processing dir"
-                        " (reason: %s)", to_path, strerror (errno));
-        }
+    if (snprintf(dest, PATH_MAX, "%s%s", jnl->jnl_processing_dir,
+                 basename(from_path)) >= PATH_MAX)
+        return -1;
+
+    ret = sys_rename(to_path, dest);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno,
+                CHANGELOG_LIB_MSG_RENAME_FAILED, "from=%s", to_path, "to=%s",
+                dest, NULL);
+    }
 
 out:
-        return ret;
+    return ret;
 }
 
 int
-gf_changelog_consume (xlator_t *this,
-                      gf_changelog_journal_t *jnl,
-                      char *from_path, gf_boolean_t no_publish)
+gf_changelog_consume(xlator_t *this, gf_changelog_journal_t *jnl,
+                     char *from_path, gf_boolean_t no_publish)
 {
-        int         ret        = -1;
-        int         fd1        = 0;
-        int         fd2        = 0;
-        int         zerob      = 0;
-        struct stat stbuf      = {0,};
-        char dest[PATH_MAX]    = {0,};
-        char to_path[PATH_MAX] = {0,};
-
-        ret = stat (from_path, &stbuf);
-        if (ret || !S_ISREG(stbuf.st_mode)) {
-                ret = -1;
-                gf_log (this->name, GF_LOG_ERROR,
-                        "stat failed on changelog file: %s", from_path);
-                goto out;
-        }
-
-        fd1 = open (from_path, O_RDONLY);
-        if (fd1 < 0) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "cannot open changelog file: %s (reason: %s)",
-                        from_path, strerror (errno));
-                goto out;
-        }
-
-        (void) snprintf (to_path, PATH_MAX, "%s%s",
-                         jnl->jnl_current_dir, basename (from_path));
-        (void) snprintf (dest, PATH_MAX, "%s%s",
-                         jnl->jnl_processing_dir, basename (from_path));
-
-        fd2 = open (to_path, O_CREAT | O_TRUNC | O_RDWR,
-                    S_IRUSR | S_IWUSR | S_IRGRP | S_IROTH);
-        if (fd2 < 0) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "cannot create ascii changelog file %s (reason %s)",
-                        to_path, strerror (errno));
+    int ret = -1;
+    int fd1 = 0;
+    int fd2 = 0;
+    int zerob = 0;
+    struct stat stbuf = {
+        0,
+    };
+    char dest[PATH_MAX] = {
+        0,
+    };
+    char to_path[PATH_MAX] = {
+        0,
+    };
+
+    if (snprintf(to_path, PATH_MAX, "%s%s", jnl->jnl_current_dir,
+                 basename(from_path)) >= PATH_MAX)
+        goto out;
+    if (snprintf(dest, PATH_MAX, "%s%s", jnl->jnl_processing_dir,
+                 basename(from_path)) >= PATH_MAX)
+        goto out;
+
+    ret = sys_stat(from_path, &stbuf);
+    if (ret || !S_ISREG(stbuf.st_mode)) {
+        ret = -1;
+        gf_smsg(this->name, GF_LOG_ERROR, errno, CHANGELOG_LIB_MSG_STAT_FAILED,
+                "path=%s", from_path, NULL);
+        goto out;
+    }
+
+    fd1 = open(from_path, O_RDONLY);
+    if (fd1 < 0) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, CHANGELOG_LIB_MSG_OPEN_FAILED,
+                "path=%s", from_path, NULL);
+        goto out;
+    }
+
+    fd2 = open(to_path, O_CREAT | O_TRUNC | O_RDWR,
+               S_IRUSR | S_IWUSR | S_IRGRP | S_IROTH);
+    if (fd2 < 0) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, CHANGELOG_LIB_MSG_OPEN_FAILED,
+                "path=%s", to_path, NULL);
+        goto close_fd;
+    } else {
+        ret = gf_changelog_decode(this, jnl, fd1, fd2, &stbuf, &zerob);
+
+        sys_close(fd2);
+
+        if (!ret) {
+            /* move it to processing on a successful
+               decode */
+            if (no_publish == _gf_true)
                 goto close_fd;
-        } else {
-                ret = gf_changelog_decode (this, jnl, fd1,
-                                           fd2, &stbuf, &zerob);
-
-                close (fd2);
-
-                if (!ret) {
-                        /* move it to processing on a successful
-                           decode */
-                        if (no_publish == _gf_true)
-                                goto close_fd;
-                        ret = rename (to_path, dest);
-                        if (ret)
-                                gf_log (this->name, GF_LOG_ERROR,
-                                        "error moving %s to processing dir"
-                                        " (reason: %s)", to_path,
-                                        strerror (errno));
-                }
+            ret = sys_rename(to_path, dest);
+            if (ret)
+                gf_smsg(this->name, GF_LOG_ERROR, errno,
+                        CHANGELOG_LIB_MSG_RENAME_FAILED, "from=%s", to_path,
+                        "to=%s", dest, NULL);
+        }
 
-                /* remove it from .current if it's an empty file */
-                if (zerob) {
-                        /* zerob changelogs must be unlinked */
-                        ret = unlink (to_path);
-                        if (ret)
-                                gf_log (this->name, GF_LOG_ERROR,
-                                        "could not unlink %s (reason: %s)",
-                                        to_path, strerror (errno));
-                }
+        /* remove it from .current if it's an empty file */
+        if (zerob) {
+            /* zerob changelogs must be unlinked */
+            ret = sys_unlink(to_path);
+            if (ret)
+                gf_smsg(this->name, GF_LOG_ERROR, errno,
+                        CHANGELOG_LIB_MSG_UNLINK_FAILED, "name=empty changelog",
+                        "path=%s", to_path, NULL);
         }
+    }
 
- close_fd:
-        close (fd1);
+close_fd:
+    sys_close(fd1);
 
- out:
-        return ret;
+out:
+    return ret;
 }
 
 void *
-gf_changelog_process (void *data)
+gf_changelog_process(void *data)
 {
-        int ret = 0;
-        xlator_t *this = NULL;
-        gf_changelog_journal_t *jnl = NULL;
-        gf_changelog_entry_t *entry = NULL;
-        gf_changelog_processor_t *jnl_proc = NULL;
-
-        jnl = data;
-        jnl_proc = jnl->jnl_proc;
-        THIS = jnl->this;
-        this = jnl->this;
-
-        while (1) {
-                pthread_mutex_lock (&jnl_proc->lock);
-                {
-                        while (list_empty (&jnl_proc->entries)) {
-                                jnl_proc->waiting = _gf_true;
-                                pthread_cond_wait
-                                        (&jnl_proc->cond, &jnl_proc->lock);
-                        }
-
-                        entry = list_first_entry (&jnl_proc->entries,
-                                                  gf_changelog_entry_t, list);
-                        list_del (&entry->list);
-                        jnl_proc->waiting = _gf_false;
-                }
-                pthread_mutex_unlock (&jnl_proc->lock);
+    xlator_t *this = NULL;
+    gf_changelog_journal_t *jnl = NULL;
+    gf_changelog_entry_t *entry = NULL;
+    gf_changelog_processor_t *jnl_proc = NULL;
+
+    jnl = data;
+    jnl_proc = jnl->jnl_proc;
+    THIS = jnl->this;
+    this = jnl->this;
+
+    while (1) {
+        pthread_mutex_lock(&jnl_proc->lock);
+        {
+            while (list_empty(&jnl_proc->entries)) {
+                jnl_proc->waiting = _gf_true;
+                pthread_cond_wait(&jnl_proc->cond, &jnl_proc->lock);
+            }
 
-                if (entry) {
-                        ret = gf_changelog_consume (this, jnl,
-                                                    entry->path, _gf_false);
-                        GF_FREE (entry);
-                }
+            entry = list_first_entry(&jnl_proc->entries, gf_changelog_entry_t,
+                                     list);
+            if (entry)
+                list_del(&entry->list);
+
+            jnl_proc->waiting = _gf_false;
         }
+        pthread_mutex_unlock(&jnl_proc->lock);
 
-        return NULL;
+        if (entry) {
+            (void)gf_changelog_consume(this, jnl, entry->path, _gf_false);
+            GF_FREE(entry);
+        }
+    }
+
+    return NULL;
 }
 
-inline void
-gf_changelog_queue_journal (gf_changelog_processor_t *jnl_proc,
-                            changelog_event_t *event)
+void
+gf_changelog_queue_journal(gf_changelog_processor_t *jnl_proc,
+                           changelog_event_t *event)
 {
-        size_t len = 0;
-        gf_changelog_entry_t *entry = NULL;
+    size_t len = 0;
+    gf_changelog_entry_t *entry = NULL;
 
-        entry = GF_CALLOC (1, sizeof (gf_changelog_entry_t),
-                           gf_changelog_mt_libgfchangelog_entry_t);
-        if (!entry)
-                return;
-        INIT_LIST_HEAD (&entry->list);
+    entry = GF_CALLOC(1, sizeof(gf_changelog_entry_t),
+                      gf_changelog_mt_libgfchangelog_entry_t);
+    if (!entry)
+        return;
+    INIT_LIST_HEAD(&entry->list);
 
-        len = strlen (event->u.journal.path);
-        (void)memcpy (entry->path, event->u.journal.path, len+1);
+    len = strlen(event->u.journal.path);
+    (void)memcpy(entry->path, event->u.journal.path, len + 1);
+    entry->path[len] = '\0';
 
-        pthread_mutex_lock (&jnl_proc->lock);
-        {
-                list_add_tail (&entry->list, &jnl_proc->entries);
-                if (jnl_proc->waiting)
-                        pthread_cond_signal (&jnl_proc->cond);
-        }
-        pthread_mutex_unlock (&jnl_proc->lock);
+    pthread_mutex_lock(&jnl_proc->lock);
+    {
+        list_add_tail(&entry->list, &jnl_proc->entries);
+        if (jnl_proc->waiting)
+            pthread_cond_signal(&jnl_proc->cond);
+    }
+    pthread_mutex_unlock(&jnl_proc->lock);
 
-        return;
+    return;
 }
 
 void
-gf_changelog_handle_journal (void *xl, char *brick,
-                             void *cbkdata, changelog_event_t *event)
+gf_changelog_handle_journal(void *xl, char *brick, void *cbkdata,
+                            changelog_event_t *event)
 {
-        int                       ret      = 0;
-        gf_changelog_journal_t   *jnl      = NULL;
-        gf_changelog_processor_t *jnl_proc = NULL;
+    gf_changelog_journal_t *jnl = NULL;
+    gf_changelog_processor_t *jnl_proc = NULL;
 
-        jnl      = cbkdata;
-        jnl_proc = jnl->jnl_proc;
+    jnl = cbkdata;
+    jnl_proc = jnl->jnl_proc;
 
-        gf_changelog_queue_journal (jnl_proc, event);
+    gf_changelog_queue_journal(jnl_proc, event);
 }
 
 void
-gf_changelog_journal_disconnect (void *xl, char *brick, void *data)
+gf_changelog_journal_disconnect(void *xl, char *brick, void *data)
 {
-        gf_changelog_journal_t *jnl = NULL;
+    gf_changelog_journal_t *jnl = NULL;
 
-        jnl = data;
+    jnl = data;
 
-        pthread_spin_lock (&jnl->lock);
-        {
-                JNL_SET_API_STATE (jnl, JNL_API_DISCONNECTED);
-        };
-        pthread_spin_unlock (&jnl->lock);
+    pthread_spin_lock(&jnl->lock);
+    {
+        JNL_SET_API_STATE(jnl, JNL_API_DISCONNECTED);
+    };
+    pthread_spin_unlock(&jnl->lock);
 }
 
 void
-gf_changelog_journal_connect (void *xl, char *brick, void *data)
+gf_changelog_journal_connect(void *xl, char *brick, void *data)
 {
-        gf_changelog_journal_t *jnl = NULL;
+    gf_changelog_journal_t *jnl = NULL;
 
-        jnl = data;
+    jnl = data;
 
-        pthread_spin_lock (&jnl->lock);
-        {
-                JNL_SET_API_STATE (jnl, JNL_API_CONNECTED);
-        };
-        pthread_spin_unlock (&jnl->lock);
+    pthread_spin_lock(&jnl->lock);
+    {
+        JNL_SET_API_STATE(jnl, JNL_API_CONNECTED);
+    };
+    pthread_spin_unlock(&jnl->lock);
 
-        return;
+    return;
 }
 
 void
-gf_changelog_cleanup_processor (gf_changelog_journal_t *jnl)
+gf_changelog_cleanup_processor(gf_changelog_journal_t *jnl)
 {
-        int ret = 0;
-        xlator_t *this = NULL;
-        gf_changelog_processor_t *jnl_proc = NULL;
+    int ret = 0;
+    xlator_t *this = NULL;
+    gf_changelog_processor_t *jnl_proc = NULL;
 
-        this = THIS;
-        if (!this || !jnl || !jnl->jnl_proc)
-                goto error_return;
+    this = THIS;
+    if (!this || !jnl || !jnl->jnl_proc)
+        goto error_return;
 
-        jnl_proc = jnl->jnl_proc;
+    jnl_proc = jnl->jnl_proc;
 
-        ret = gf_thread_cleanup (this, jnl_proc->processor);
-        if (ret != 0) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "failed to cleanup processor thread");
-                goto error_return;
-        }
+    ret = gf_thread_cleanup(this, jnl_proc->processor);
+    if (ret != 0) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, CHANGELOG_LIB_MSG_CLEANUP_ERROR,
+               "failed to cleanup processor thread");
+        goto error_return;
+    }
 
-        (void)pthread_mutex_destroy (&jnl_proc->lock);
-        (void)pthread_cond_destroy (&jnl_proc->cond);
+    (void)pthread_mutex_destroy(&jnl_proc->lock);
+    (void)pthread_cond_destroy(&jnl_proc->cond);
 
-        GF_FREE (jnl_proc);
+    GF_FREE(jnl_proc);
 
- error_return:
-        return;
+error_return:
+    return;
 }
 
 int
-gf_changelog_init_processor (gf_changelog_journal_t *jnl)
+gf_changelog_init_processor(gf_changelog_journal_t *jnl)
 {
-        int ret = -1;
-        gf_changelog_processor_t *jnl_proc = NULL;
+    int ret = -1;
+    gf_changelog_processor_t *jnl_proc = NULL;
 
-        jnl_proc = GF_CALLOC (1, sizeof (gf_changelog_processor_t),
-                              gf_changelog_mt_libgfchangelog_t);
-        if (!jnl_proc)
-                goto error_return;
-
-        ret = pthread_mutex_init (&jnl_proc->lock, NULL);
-        if (ret != 0)
-                goto free_jnl_proc;
-        ret = pthread_cond_init (&jnl_proc->cond, NULL);
-        if (ret != 0)
-                goto cleanup_mutex;
-
-        INIT_LIST_HEAD (&jnl_proc->entries);
-        ret = pthread_create (&jnl_proc->processor,
-                              NULL, gf_changelog_process, jnl);
-        if (ret != 0)
-                goto cleanup_cond;
-        jnl_proc->waiting = _gf_false;
-
-        jnl->jnl_proc = jnl_proc;
-        return 0;
-
- cleanup_cond:
-        (void) pthread_cond_destroy (&jnl_proc->cond);
- cleanup_mutex:
-        (void) pthread_mutex_destroy (&jnl_proc->lock);
- free_jnl_proc:
-        GF_FREE (jnl_proc);
- error_return:
-        return -1;
+    jnl_proc = GF_CALLOC(1, sizeof(gf_changelog_processor_t),
+                         gf_changelog_mt_libgfchangelog_t);
+    if (!jnl_proc)
+        goto error_return;
+
+    ret = pthread_mutex_init(&jnl_proc->lock, NULL);
+    if (ret != 0)
+        goto free_jnl_proc;
+    ret = pthread_cond_init(&jnl_proc->cond, NULL);
+    if (ret != 0)
+        goto cleanup_mutex;
+
+    INIT_LIST_HEAD(&jnl_proc->entries);
+    jnl_proc->waiting = _gf_false;
+    jnl->jnl_proc = jnl_proc;
+
+    ret = gf_thread_create(&jnl_proc->processor, NULL, gf_changelog_process,
+                           jnl, "clogproc");
+    if (ret != 0) {
+        jnl->jnl_proc = NULL;
+        goto cleanup_cond;
+    }
+
+    return 0;
+
+cleanup_cond:
+    (void)pthread_cond_destroy(&jnl_proc->cond);
+cleanup_mutex:
+    (void)pthread_mutex_destroy(&jnl_proc->lock);
+free_jnl_proc:
+    GF_FREE(jnl_proc);
+error_return:
+    return -1;
 }
 
 static void
-gf_changelog_cleanup_fds (gf_changelog_journal_t *jnl)
+gf_changelog_cleanup_fds(gf_changelog_journal_t *jnl)
 {
-        /* tracker fd */
-        if (jnl->jnl_fd != -1)
-                close (jnl->jnl_fd);
-        /* processing dir */
-        if (jnl->jnl_dir)
-                closedir (jnl->jnl_dir);
-
-        if (jnl->jnl_working_dir)
-                free (jnl->jnl_working_dir); /* allocated by realpath */
+    /* tracker fd */
+    if (jnl->jnl_fd != -1)
+        sys_close(jnl->jnl_fd);
+    /* processing dir */
+    if (jnl->jnl_dir)
+        sys_closedir(jnl->jnl_dir);
+
+    if (jnl->jnl_working_dir)
+        free(jnl->jnl_working_dir); /* allocated by realpath */
 }
 
 static int
-gf_changelog_open_dirs (xlator_t *this, gf_changelog_journal_t *jnl)
+gf_changelog_open_dirs(xlator_t *this, gf_changelog_journal_t *jnl)
 {
-        int  ret                    = -1;
-        DIR *dir                    = NULL;
-        int  tracker_fd             = 0;
-        char tracker_path[PATH_MAX] = {0,};
-
-        /* .current */
-        (void) snprintf (jnl->jnl_current_dir, PATH_MAX,
-                         "%s/"GF_CHANGELOG_CURRENT_DIR"/",
-                         jnl->jnl_working_dir);
-        ret = recursive_rmdir (jnl->jnl_current_dir);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "Failed to rmdir: %s, err: %s",
-                        jnl->jnl_current_dir, strerror (errno));
-                goto out;
-        }
-        ret = mkdir_p (jnl->jnl_current_dir, 0600, _gf_false);
-        if (ret)
-                goto out;
-
-        /* .processed */
-        (void) snprintf (jnl->jnl_processed_dir, PATH_MAX,
-                         "%s/"GF_CHANGELOG_PROCESSED_DIR"/",
-                         jnl->jnl_working_dir);
-        ret = mkdir_p (jnl->jnl_processed_dir, 0600, _gf_false);
-        if (ret)
-                goto out;
-
-        /* .processing */
-        (void) snprintf (jnl->jnl_processing_dir, PATH_MAX,
-                         "%s/"GF_CHANGELOG_PROCESSING_DIR"/",
-                         jnl->jnl_working_dir);
-        ret = recursive_rmdir (jnl->jnl_processing_dir);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "Failed to rmdir: %s, err: %s",
-                        jnl->jnl_processing_dir, strerror (errno));
-                goto out;
-        }
-
-        ret = mkdir_p (jnl->jnl_processing_dir, 0600, _gf_false);
-        if (ret)
-                goto out;
-
-        dir = opendir (jnl->jnl_processing_dir);
-        if (!dir) {
-                gf_log ("", GF_LOG_ERROR,
-                        "opendir() error [reason: %s]", strerror (errno));
-                goto out;
-        }
-
-        jnl->jnl_dir = dir;
-
-        (void) snprintf (tracker_path, PATH_MAX,
-                         "%s/"GF_CHANGELOG_TRACKER, jnl->jnl_working_dir);
-
-        tracker_fd = open (tracker_path, O_CREAT | O_APPEND | O_RDWR,
-                           S_IRUSR | S_IWUSR | S_IRGRP | S_IROTH);
-        if (tracker_fd < 0) {
-                closedir (jnl->jnl_dir);
-                ret = -1;
-                goto out;
-        }
-
-        jnl->jnl_fd = tracker_fd;
-        ret = 0;
- out:
-        return ret;
+    int ret = -1;
+    DIR *dir = NULL;
+    int tracker_fd = 0;
+    char tracker_path[PATH_MAX] = {
+        0,
+    };
+
+    /* .current */
+    (void)snprintf(jnl->jnl_current_dir, PATH_MAX,
+                   "%s/" GF_CHANGELOG_CURRENT_DIR "/", jnl->jnl_working_dir);
+    ret = recursive_rmdir(jnl->jnl_current_dir);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno,
+                CHANGELOG_LIB_MSG_FAILED_TO_RMDIR, "path=%s",
+                jnl->jnl_current_dir, NULL);
+        goto out;
+    }
+    ret = mkdir_p(jnl->jnl_current_dir, 0600, _gf_false);
+    if (ret)
+        goto out;
+
+    /* .processed */
+    (void)snprintf(jnl->jnl_processed_dir, PATH_MAX,
+                   "%s/" GF_CHANGELOG_PROCESSED_DIR "/", jnl->jnl_working_dir);
+    ret = mkdir_p(jnl->jnl_processed_dir, 0600, _gf_false);
+    if (ret)
+        goto out;
+
+    /* .processing */
+    (void)snprintf(jnl->jnl_processing_dir, PATH_MAX,
+                   "%s/" GF_CHANGELOG_PROCESSING_DIR "/", jnl->jnl_working_dir);
+    ret = recursive_rmdir(jnl->jnl_processing_dir);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno,
+                CHANGELOG_LIB_MSG_FAILED_TO_RMDIR, "path=%s",
+                jnl->jnl_processing_dir, NULL);
+        goto out;
+    }
+
+    ret = mkdir_p(jnl->jnl_processing_dir, 0600, _gf_false);
+    if (ret)
+        goto out;
+
+    dir = sys_opendir(jnl->jnl_processing_dir);
+    if (!dir) {
+        gf_msg("", GF_LOG_ERROR, errno, CHANGELOG_LIB_MSG_OPENDIR_ERROR,
+               "opendir() error");
+        goto out;
+    }
+
+    jnl->jnl_dir = dir;
+
+    (void)snprintf(tracker_path, PATH_MAX, "%s/" GF_CHANGELOG_TRACKER,
+                   jnl->jnl_working_dir);
+
+    tracker_fd = open(tracker_path, O_CREAT | O_APPEND | O_RDWR,
+                      S_IRUSR | S_IWUSR | S_IRGRP | S_IROTH);
+    if (tracker_fd < 0) {
+        sys_closedir(jnl->jnl_dir);
+        ret = -1;
+        goto out;
+    }
+
+    jnl->jnl_fd = tracker_fd;
+    ret = 0;
+out:
+    return ret;
 }
 
 int
-gf_changelog_init_history (xlator_t *this,
-                           gf_changelog_journal_t *jnl,
-                           char *brick_path)
+gf_changelog_init_history(xlator_t *this, gf_changelog_journal_t *jnl,
+                          char *brick_path)
 {
-        int i   = 0;
-        int ret = 0;
-        char hist_scratch_dir[PATH_MAX] = {0,};
-
-        jnl->hist_jnl = GF_CALLOC (1, sizeof (*jnl),
-                         gf_changelog_mt_libgfchangelog_t);
-        if (!jnl->hist_jnl)
-                goto error_return;
-
-        jnl->hist_jnl->jnl_dir = NULL;
-        jnl->hist_jnl->jnl_fd =  -1;
-
-        (void) snprintf (hist_scratch_dir, PATH_MAX,
-                         "%s/"GF_CHANGELOG_HISTORY_DIR"/",
-                         jnl->jnl_working_dir);
-
-        ret = mkdir_p (hist_scratch_dir, 0600, _gf_false);
-        if (ret)
-                goto dealloc_hist;
-
-        jnl->hist_jnl->jnl_working_dir = realpath (hist_scratch_dir, NULL);
-        if (!jnl->hist_jnl->jnl_working_dir)
-                goto dealloc_hist;
-
-        ret = gf_changelog_open_dirs (this, jnl->hist_jnl);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "could not create entries in history scratch dir");
-                goto dealloc_hist;
-        }
-
-        (void) strncpy (jnl->hist_jnl->jnl_brickpath, brick_path, PATH_MAX-1);
-        jnl->hist_jnl->jnl_brickpath[PATH_MAX-1] = 0;
+    int i = 0;
+    int ret = 0;
+    char hist_scratch_dir[PATH_MAX] = {
+        0,
+    };
 
-        for (i = 0; i < 256; i++) {
-                jnl->hist_jnl->rfc3986[i] =
-                        (isalnum(i) || i == '~' ||
-                        i == '-' || i == '.' || i == '_') ? i : 0;
-        }
-
-        return 0;
-
- dealloc_hist:
-        GF_FREE (jnl->hist_jnl);
-        jnl->hist_jnl = NULL;
- error_return:
-        return -1;
+    jnl->hist_jnl = GF_CALLOC(1, sizeof(*jnl),
+                              gf_changelog_mt_libgfchangelog_t);
+    if (!jnl->hist_jnl)
+        goto error_return;
+
+    jnl->hist_jnl->jnl_dir = NULL;
+    jnl->hist_jnl->jnl_fd = -1;
+
+    (void)snprintf(hist_scratch_dir, PATH_MAX,
+                   "%s/" GF_CHANGELOG_HISTORY_DIR "/", jnl->jnl_working_dir);
+
+    ret = mkdir_p(hist_scratch_dir, 0600, _gf_false);
+    if (ret)
+        goto dealloc_hist;
+
+    jnl->hist_jnl->jnl_working_dir = realpath(hist_scratch_dir, NULL);
+    if (!jnl->hist_jnl->jnl_working_dir)
+        goto dealloc_hist;
+
+    ret = gf_changelog_open_dirs(this, jnl->hist_jnl);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, CHANGELOG_LIB_MSG_OPENDIR_ERROR,
+               "could not create entries in history scratch dir");
+        goto dealloc_hist;
+    }
+
+    if (snprintf(jnl->hist_jnl->jnl_brickpath, PATH_MAX, "%s", brick_path) >=
+        PATH_MAX)
+        goto dealloc_hist;
+
+    for (i = 0; i < 256; i++) {
+        jnl->hist_jnl->rfc3986_space_newline[i] = (i == ' ' || i == '\n' ||
+                                                   i == '%')
+                                                      ? 0
+                                                      : i;
+    }
+
+    return 0;
+
+dealloc_hist:
+    GF_FREE(jnl->hist_jnl);
+    jnl->hist_jnl = NULL;
+error_return:
+    return -1;
 }
 
 void
-gf_changelog_journal_fini (void *xl, char *brick, void *data)
+gf_changelog_journal_fini(void *xl, char *brick, void *data)
 {
-        int ret = 0;
-        xlator_t *this = NULL;
-        gf_changelog_journal_t *jnl = NULL;
+    gf_changelog_journal_t *jnl = NULL;
 
-        this = xl;
-        jnl = data;
+    jnl = data;
 
-        gf_changelog_cleanup_processor (jnl);
+    gf_changelog_cleanup_processor(jnl);
 
-        gf_changelog_cleanup_fds (jnl);
-        if (jnl->hist_jnl)
-                gf_changelog_cleanup_fds (jnl->hist_jnl);
+    gf_changelog_cleanup_fds(jnl);
+    if (jnl->hist_jnl)
+        gf_changelog_cleanup_fds(jnl->hist_jnl);
 
-        GF_FREE (jnl);
+    GF_FREE(jnl);
 }
 
 void *
-gf_changelog_journal_init (void *xl, struct gf_brick_spec *brick)
+gf_changelog_journal_init(void *xl, struct gf_brick_spec *brick)
 {
-        int                     i           = 0;
-        int                     ret         = 0;
-        xlator_t               *this        = NULL;
-        struct stat             buf         = {0,};
-        char                   *scratch_dir = NULL;
-        gf_changelog_journal_t *jnl         = NULL;
-
-        this = xl;
-        scratch_dir = (char *) brick->ptr;
-
-        jnl = GF_CALLOC (1, sizeof (gf_changelog_journal_t),
-                         gf_changelog_mt_libgfchangelog_t);
-        if (!jnl)
-                goto error_return;
-
-        if (stat (scratch_dir, &buf) && errno == ENOENT) {
-                ret = mkdir_p (scratch_dir, 0600, _gf_true);
-                if (ret)
-                        goto dealloc_private;
-        }
-
-        jnl->jnl_working_dir = realpath (scratch_dir, NULL);
-        if (!jnl->jnl_working_dir)
-                goto dealloc_private;
-
-        ret = gf_changelog_open_dirs (this, jnl);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "could not create entries in scratch dir");
-                goto dealloc_private;
-        }
-
-        (void) strncpy (jnl->jnl_brickpath, brick->brick_path, PATH_MAX-1);
-        jnl->jnl_brickpath[PATH_MAX-1] = 0;
-
-        /* RFC 3986 {de,en}coding */
-        for (i = 0; i < 256; i++) {
-                jnl->rfc3986[i] =
-                        (isalnum(i) || i == '~' ||
-                        i == '-' || i == '.' || i == '_') ? i : 0;
-        }
-
-        ret = gf_changelog_init_history (this, jnl, brick->brick_path);
-        if (ret)
-                goto cleanup_fds;
-
-        /* initialize journal processor */
-        jnl->this = this;
-        ret = gf_changelog_init_processor (jnl);
+    int i = 0;
+    int ret = 0;
+    xlator_t *this = NULL;
+    struct stat buf = {
+        0,
+    };
+    char *scratch_dir = NULL;
+    gf_changelog_journal_t *jnl = NULL;
+
+    this = xl;
+    scratch_dir = (char *)brick->ptr;
+
+    jnl = GF_CALLOC(1, sizeof(gf_changelog_journal_t),
+                    gf_changelog_mt_libgfchangelog_t);
+    if (!jnl)
+        goto error_return;
+
+    if (snprintf(jnl->jnl_brickpath, PATH_MAX, "%s", brick->brick_path) >=
+        PATH_MAX)
+        goto dealloc_private;
+
+    if (sys_stat(scratch_dir, &buf) && errno == ENOENT) {
+        ret = mkdir_p(scratch_dir, 0600, _gf_true);
         if (ret)
-                goto cleanup_fds;
-
-        JNL_SET_API_STATE (jnl, JNL_API_CONN_INPROGESS);
-        ret = pthread_spin_init (&jnl->lock, 0);
-        if (ret != 0)
-                goto cleanup_processor;
-        return jnl;
-
- cleanup_processor:
-        gf_changelog_cleanup_processor (jnl);
- cleanup_fds:
-        gf_changelog_cleanup_fds (jnl);
-        if (jnl->hist_jnl)
-                gf_changelog_cleanup_fds (jnl->hist_jnl);
- dealloc_private:
-        GF_FREE (jnl);
- error_return:
-        return NULL;
+            goto dealloc_private;
+    }
+
+    jnl->jnl_working_dir = realpath(scratch_dir, NULL);
+    if (!jnl->jnl_working_dir)
+        goto dealloc_private;
+
+    ret = gf_changelog_open_dirs(this, jnl);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, CHANGELOG_LIB_MSG_OPENDIR_ERROR,
+               "could not create entries in scratch dir");
+        goto dealloc_private;
+    }
+
+    /* RFC 3986 {de,en}coding */
+    for (i = 0; i < 256; i++) {
+        jnl->rfc3986_space_newline[i] = (i == ' ' || i == '\n' || i == '%') ? 0
+                                                                            : i;
+    }
+
+    ret = gf_changelog_init_history(this, jnl, brick->brick_path);
+    if (ret)
+        goto cleanup_fds;
+
+    /* initialize journal processor */
+    jnl->this = this;
+    ret = gf_changelog_init_processor(jnl);
+    if (ret)
+        goto cleanup_fds;
+
+    JNL_SET_API_STATE(jnl, JNL_API_CONN_INPROGESS);
+    ret = pthread_spin_init(&jnl->lock, 0);
+    if (ret != 0)
+        goto cleanup_processor;
+    return jnl;
+
+cleanup_processor:
+    gf_changelog_cleanup_processor(jnl);
+cleanup_fds:
+    gf_changelog_cleanup_fds(jnl);
+    if (jnl->hist_jnl)
+        gf_changelog_cleanup_fds(jnl->hist_jnl);
+dealloc_private:
+    GF_FREE(jnl);
+error_return:
+    return NULL;
 }
diff --git a/xlators/features/changelog/lib/src/gf-changelog-journal.h b/xlators/features/changelog/lib/src/gf-changelog-journal.h
index e91807c80b6..ba5b9bf827e 100644
--- a/xlators/features/changelog/lib/src/gf-changelog-journal.h
+++ b/xlators/features/changelog/lib/src/gf-changelog-journal.h
@@ -17,91 +17,91 @@
 #include "changelog.h"
 
 enum api_conn {
-        JNL_API_CONNECTED,
-        JNL_API_CONN_INPROGESS,
-        JNL_API_DISCONNECTED,
+    JNL_API_CONNECTED,
+    JNL_API_CONN_INPROGESS,
+    JNL_API_DISCONNECTED,
 };
 
 typedef struct gf_changelog_entry {
-        char path[PATH_MAX];
+    char path[PATH_MAX];
 
-        struct list_head list;
+    struct list_head list;
 } gf_changelog_entry_t;
 
 typedef struct gf_changelog_processor {
-        pthread_mutex_t  lock;     /* protects ->entries */
-        pthread_cond_t   cond;     /* waiter during empty list */
-        gf_boolean_t     waiting;
+    pthread_mutex_t lock; /* protects ->entries */
+    pthread_cond_t cond;  /* waiter during empty list */
+    gf_boolean_t waiting;
 
-        pthread_t processor;       /* thread-id of journal processing thread */
+    pthread_t processor; /* thread-id of journal processing thread */
 
-        struct list_head entries;
+    struct list_head entries;
 } gf_changelog_processor_t;
 
 typedef struct gf_changelog_journal {
-        DIR *jnl_dir;                       /* 'processing' directory stream */
+    DIR *jnl_dir; /* 'processing' directory stream */
 
-        int jnl_fd;                         /* fd to the tracker file */
+    int jnl_fd; /* fd to the tracker file */
 
-        char jnl_brickpath[PATH_MAX];       /* brick path for this end-point */
+    char jnl_brickpath[PATH_MAX]; /* brick path for this end-point */
 
-        gf_changelog_processor_t *jnl_proc;
+    gf_changelog_processor_t *jnl_proc;
 
-        char *jnl_working_dir;              /* scratch directory */
+    char *jnl_working_dir; /* scratch directory */
 
-        char jnl_current_dir[PATH_MAX];
-        char jnl_processed_dir[PATH_MAX];
-        char jnl_processing_dir[PATH_MAX];
+    char jnl_current_dir[PATH_MAX];
+    char jnl_processed_dir[PATH_MAX];
+    char jnl_processing_dir[PATH_MAX];
 
-        char rfc3986[256];                  /* RFC 3986 string encoding */
+    char rfc3986_space_newline[256]; /* RFC 3986 string encoding */
 
-        struct gf_changelog_journal *hist_jnl;
-        int hist_done;                      /* holds 0 done scanning,
-                                               1 keep scanning and -1 error */
+    struct gf_changelog_journal *hist_jnl;
+    int hist_done; /* holds 0 done scanning,
+                      1 keep scanning and -1 error */
 
-        pthread_spinlock_t lock;
-        int connected;
-        xlator_t *this;
+    pthread_spinlock_t lock;
+    int connected;
+    xlator_t *this;
 } gf_changelog_journal_t;
 
-#define JNL_SET_API_STATE(jnl, state)  (jnl->connected = state)
-#define JNL_IS_API_DISCONNECTED(jnl)  (jnl->connected == JNL_API_DISCONNECTED)
+#define JNL_SET_API_STATE(jnl, state) (jnl->connected = state)
+#define JNL_IS_API_DISCONNECTED(jnl) (jnl->connected == JNL_API_DISCONNECTED)
 
 /* History API */
 typedef struct gf_changelog_history_data {
-        int           len;
+    int len;
 
-        int           htime_fd;
+    int htime_fd;
 
-        /* parallelism count */
-        int           n_parallel;
+    /* parallelism count */
+    int n_parallel;
 
-        /* history from, to indexes */
-        unsigned long from;
-        unsigned long to;
-        xlator_t      *this;
+    /* history from, to indexes */
+    unsigned long from;
+    unsigned long to;
+    xlator_t *this;
 } gf_changelog_history_data_t;
 
 typedef struct gf_changelog_consume_data {
-        /** set of inputs */
+    /** set of inputs */
 
-        /* fd to read from */
-        int             fd;
+    /* fd to read from */
+    int fd;
 
-        /* from @offset */
-        off_t           offset;
+    /* from @offset */
+    off_t offset;
 
-        xlator_t       *this;
+    xlator_t *this;
 
-        gf_changelog_journal_t *jnl;
+    gf_changelog_journal_t *jnl;
 
-        /** set of outputs */
+    /** set of outputs */
 
-        /* return value */
-        int retval;
+    /* return value */
+    int retval;
 
-        /* journal processed */
-        char changelog[PATH_MAX];
+    /* journal processed */
+    char changelog[PATH_MAX];
 } gf_changelog_consume_data_t;
 
 /* event handler */
diff --git a/xlators/features/changelog/lib/src/gf-changelog-reborp.c b/xlators/features/changelog/lib/src/gf-changelog-reborp.c
index 457d829e69b..56b11cbb705 100644
--- a/xlators/features/changelog/lib/src/gf-changelog-reborp.c
+++ b/xlators/features/changelog/lib/src/gf-changelog-reborp.c
@@ -13,183 +13,132 @@
 
 #include "gf-changelog-helpers.h"
 #include "changelog-rpc-common.h"
+#include "changelog-lib-messages.h"
+
+#include <glusterfs/syscall.h>
 
 /**
  * Reverse socket: actual data transfer handler. Connection
  * initiator is PROBER, data transfer is REBORP.
  */
 
-struct rpcsvc_program *gf_changelog_reborp_programs[];
+static struct rpcsvc_program *gf_changelog_reborp_programs[];
 
 void *
-gf_changelog_connection_janitor (void *arg)
+gf_changelog_connection_janitor(void *arg)
 {
-        int32_t ret = 0;
-        xlator_t *this = NULL;
-        gf_private_t *priv = NULL;
-        gf_changelog_t *entry = NULL;
-        struct gf_event *event = NULL;
-        struct gf_event_list *ev = NULL;
-        unsigned long drained = 0;
-
-        this = arg;
-        THIS = this;
-
-        priv = this->private;
-
-        while (1) {
-                pthread_mutex_lock (&priv->lock);
-                {
-                        while (list_empty (&priv->cleanups))
-                                pthread_cond_wait (&priv->cond, &priv->lock);
-
-                        entry = list_first_entry (&priv->cleanups,
-                                                  gf_changelog_t, list);
-                        list_del_init (&entry->list);
-                }
-                pthread_mutex_unlock (&priv->lock);
-
-                drained = 0;
-                ev = &entry->event;
-
-                gf_log (this->name, GF_LOG_INFO,
-                        "Cleaning brick entry for brick %s", entry->brick);
-
-                /* 0x0: disbale rpc-clnt */
-                rpc_clnt_disable (RPC_PROBER (entry));
-
-                /* 0x1: cleanup callback invoker thread */
-                ret = gf_cleanup_event (this, ev);
-                if (ret)
-                        continue;
-
-                /* 0x2: drain pending events */
-                while (!list_empty (&ev->events)) {
-                        event = list_first_entry (&ev->events,
-                                                  struct gf_event, list);
-                        gf_log (this->name, GF_LOG_INFO,
-                                "Draining event [Seq: %lu, Payload: %d]",
-                                event->seq, event->count);
-
-                        GF_FREE (event);
-                        drained++;
-                }
-
-                gf_log (this->name, GF_LOG_INFO,
-                        "Drained %lu events", drained);
-
-                /* 0x3: freeup brick entry */
-                gf_log (this->name, GF_LOG_INFO, "freeing entry %p", entry);
-                LOCK_DESTROY (&entry->statelock);
-                GF_FREE (entry);
-        }
+    int32_t ret = 0;
+    xlator_t *this = NULL;
+    gf_private_t *priv = NULL;
+    gf_changelog_t *entry = NULL;
+    struct gf_event *event = NULL;
+    struct gf_event_list *ev = NULL;
+    unsigned long drained = 0;
 
-        return NULL;
-}
+    this = arg;
+    THIS = this;
 
-static inline void
-__gf_changelog_set_conn_state (gf_changelog_t *entry,
-                               gf_changelog_conn_state_t state)
-{
-        entry->connstate = state;
-}
+    priv = this->private;
 
-/**
- * state check login to gaurd access object after free
- */
-static inline void
-gf_changelog_check_event (gf_private_t *priv,
-                          gf_changelog_t *entry, rpcsvc_event_t event)
-{
-        gf_boolean_t needfree = _gf_false;
-        gf_changelog_conn_state_t laststate;
-        /**
-         * need to handle couple of connection states to gaurd correct
-         * freeing of object.
-         */
-        LOCK (&entry->statelock);
+    while (1) {
+        pthread_mutex_lock(&priv->lock);
         {
-                laststate = entry->connstate;
-                if (event == RPCSVC_EVENT_ACCEPT) {
-                        __gf_changelog_set_conn_state
-                                (entry, GF_CHANGELOG_CONN_STATE_ACCEPTED);
-
-                        if (laststate == GF_CHANGELOG_CONN_STATE_DISCONNECTED)
-                                needfree = _gf_true;
-                }
-
-                if (event == RPCSVC_EVENT_DISCONNECT) {
-                        __gf_changelog_set_conn_state
-                                (entry, GF_CHANGELOG_CONN_STATE_DISCONNECTED);
-
-                        if (laststate == GF_CHANGELOG_CONN_STATE_ACCEPTED)
-                                needfree = _gf_true;
-                }
+            while (list_empty(&priv->cleanups))
+                pthread_cond_wait(&priv->cond, &priv->lock);
+
+            entry = list_first_entry(&priv->cleanups, gf_changelog_t, list);
+            list_del_init(&entry->list);
+        }
+        pthread_mutex_unlock(&priv->lock);
+
+        drained = 0;
+        ev = &entry->event;
+
+        gf_smsg(this->name, GF_LOG_INFO, 0,
+                CHANGELOG_LIB_MSG_CLEANING_BRICK_ENTRY_INFO, "brick=%s",
+                entry->brick, NULL);
+
+        /* 0x0: disable rpc-clnt */
+        rpc_clnt_disable(RPC_PROBER(entry));
+
+        /* 0x1: cleanup callback invoker thread */
+        ret = gf_cleanup_event(this, ev);
+        if (ret)
+            continue;
+
+        /* 0x2: drain pending events */
+        while (!list_empty(&ev->events)) {
+            event = list_first_entry(&ev->events, struct gf_event, list);
+            gf_smsg(this->name, GF_LOG_INFO, 0,
+                    CHANGELOG_LIB_MSG_DRAINING_EVENT_INFO, "seq=%lu",
+                    event->seq, "payload=%d", event->count, NULL);
+
+            GF_FREE(event);
+            drained++;
         }
-        UNLOCK (&entry->statelock);
-
-        /**
-         * TODO:
-         * Handle the  race between ACCEPT and DISCONNECT in the
-         * reconnect code. So purging of entry is deliberately
-         * avoided here. It will be handled in the reconnect code.
-         */
+
+        gf_smsg(this->name, GF_LOG_INFO, 0,
+                CHANGELOG_LIB_MSG_DRAINED_EVENT_INFO, "num=%lu", drained, NULL);
+
+        /* 0x3: freeup brick entry */
+        gf_smsg(this->name, GF_LOG_INFO, 0,
+                CHANGELOG_LIB_MSG_FREEING_ENTRY_INFO, "entry=%p", entry, NULL);
+        LOCK_DESTROY(&entry->statelock);
+        GF_FREE(entry);
+    }
+
+    return NULL;
 }
 
 int
-gf_changelog_reborp_rpcsvc_notify (rpcsvc_t *rpc, void *mydata,
-                                   rpcsvc_event_t event, void *data)
+gf_changelog_reborp_rpcsvc_notify(rpcsvc_t *rpc, void *mydata,
+                                  rpcsvc_event_t event, void *data)
 {
-        int             ret      = 0;
-        xlator_t       *this     = NULL;
-        gf_private_t   *priv     = NULL;
-        gf_changelog_t *entry    = NULL;
-        char sock[UNIX_PATH_MAX] = {0,};
+    int ret = 0;
+    xlator_t *this = NULL;
+    gf_changelog_t *entry = NULL;
 
-        if (!(event == RPCSVC_EVENT_ACCEPT ||
-              event == RPCSVC_EVENT_DISCONNECT))
-                return 0;
+    if (!(event == RPCSVC_EVENT_ACCEPT || event == RPCSVC_EVENT_DISCONNECT))
+        return 0;
 
-        entry = mydata;
-        this = entry->this;
-        priv = this->private;
+    entry = mydata;
+    this = entry->this;
 
-        switch (event) {
+    switch (event) {
         case RPCSVC_EVENT_ACCEPT:
-                ret = unlink (RPC_SOCK(entry));
-                if (ret != 0)
-                        gf_log (this->name, GF_LOG_WARNING, "failed to unlink "
-                                "reverse socket %s", RPC_SOCK (entry));
-                if (entry->connected)
-                        GF_CHANGELOG_INVOKE_CBK (this, entry->connected,
-                                                 entry->brick, entry->ptr);
-                break;
+            ret = sys_unlink(RPC_SOCK(entry));
+            if (ret != 0)
+                gf_smsg(this->name, GF_LOG_WARNING, errno,
+                        CHANGELOG_LIB_MSG_UNLINK_FAILED, "name=reverse socket",
+                        "path=%s", RPC_SOCK(entry), NULL);
+            if (entry->connected)
+                GF_CHANGELOG_INVOKE_CBK(this, entry->connected, entry->brick,
+                                        entry->ptr);
+            break;
         case RPCSVC_EVENT_DISCONNECT:
-                if (entry->disconnected)
-                        GF_CHANGELOG_INVOKE_CBK (this, entry->disconnected,
-                                                 entry->brick, entry->ptr);
-                /* passthrough */
+            if (entry->disconnected)
+                GF_CHANGELOG_INVOKE_CBK(this, entry->disconnected, entry->brick,
+                                        entry->ptr);
+            /* passthrough */
         default:
-                break;
-        }
-
-        /* gf_changelog_check_event (priv, entry, event); */
+            break;
+    }
 
-        return 0;
+    return 0;
 }
 
 rpcsvc_t *
-gf_changelog_reborp_init_rpc_listner (xlator_t *this,
-                                      char *path, char *sock, void *cbkdata)
+gf_changelog_reborp_init_rpc_listner(xlator_t *this, char *path, char *sock,
+                                     void *cbkdata)
 {
-        CHANGELOG_MAKE_TMP_SOCKET_PATH (path, sock, UNIX_PATH_MAX);
-        return changelog_rpc_server_init (this, sock, cbkdata,
-                                          gf_changelog_reborp_rpcsvc_notify,
-                                          gf_changelog_reborp_programs);
+    CHANGELOG_MAKE_TMP_SOCKET_PATH(path, sock, UNIX_PATH_MAX);
+    return changelog_rpc_server_init(this, sock, cbkdata,
+                                     gf_changelog_reborp_rpcsvc_notify,
+                                     gf_changelog_reborp_programs);
 }
 
 /**
- * This is dirty and painful as of now untill there is event filtering in the
+ * This is dirty and painful as of now until there is event filtering in the
  * server. The entire event buffer is scanned and interested events are picked,
  * whereas we _should_ be notified with the events we were interested in
  * (selected at the time of probe). As of now this is complete BS and needs
@@ -197,30 +146,28 @@ gf_changelog_reborp_init_rpc_listner (xlator_t *this,
  *
  * @FIXME: cleanup this bugger once server filters events.
  */
-inline void
-gf_changelog_invoke_callback (gf_changelog_t *entry,
-                              struct iovec **vec, int payloadcnt)
+void
+gf_changelog_invoke_callback(gf_changelog_t *entry, struct iovec **vec,
+                             int payloadcnt)
 {
-        int i = 0;
-        int evsize = 0;
-        xlator_t *this = NULL;
-        changelog_event_t *event = NULL;
-
-        this = entry->this;
-
-        for (; i < payloadcnt; i++) {
-                event = (changelog_event_t *)vec[i]->iov_base;
-                evsize = vec[i]->iov_len / CHANGELOG_EV_SIZE;
-
-                for (; evsize > 0; evsize--, event++) {
-                        if (gf_changelog_filter_check (entry, event)) {
-                                GF_CHANGELOG_INVOKE_CBK (this,
-                                                         entry->callback,
-                                                         entry->brick,
-                                                         entry->ptr, event);
-                        }
-                }
+    int i = 0;
+    int evsize = 0;
+    xlator_t *this = NULL;
+    changelog_event_t *event = NULL;
+
+    this = entry->this;
+
+    for (; i < payloadcnt; i++) {
+        event = (changelog_event_t *)vec[i]->iov_base;
+        evsize = vec[i]->iov_len / CHANGELOG_EV_SIZE;
+
+        for (; evsize > 0; evsize--, event++) {
+            if (gf_changelog_filter_check(entry, event)) {
+                GF_CHANGELOG_INVOKE_CBK(this, entry->callback, entry->brick,
+                                        entry->ptr, event);
+            }
         }
+    }
 }
 
 /**
@@ -230,218 +177,219 @@ gf_changelog_invoke_callback (gf_changelog_t *entry,
  * dynamically allocated and ordered.
  */
 
-inline int
-__is_expected_sequence (struct gf_event_list *ev, struct gf_event *event)
+int
+__is_expected_sequence(struct gf_event_list *ev, struct gf_event *event)
 {
-        return (ev->next_seq == event->seq);
+    return (ev->next_seq == event->seq);
 }
 
-inline int
-__can_process_event (struct gf_event_list *ev, struct gf_event **event)
+int
+__can_process_event(struct gf_event_list *ev, struct gf_event **event)
 {
-        *event = list_first_entry (&ev->events, struct gf_event, list);
+    *event = list_first_entry(&ev->events, struct gf_event, list);
 
-        if (__is_expected_sequence (ev, *event)) {
-                list_del (&(*event)->list);
-                ev->next_seq++;
-                return 1;
-        }
+    if (__is_expected_sequence(ev, *event)) {
+        list_del(&(*event)->list);
+        ev->next_seq++;
+        return 1;
+    }
 
-        return 0;
+    return 0;
 }
 
 void
-pick_event_ordered (struct gf_event_list *ev, struct gf_event **event)
+pick_event_ordered(struct gf_event_list *ev, struct gf_event **event)
 {
-        pthread_mutex_lock (&ev->lock);
-        {
-                while (list_empty (&ev->events)
-                       || !__can_process_event (ev, event))
-                        pthread_cond_wait (&ev->cond, &ev->lock);
-        }
-        pthread_mutex_unlock (&ev->lock);
+    pthread_mutex_lock(&ev->lock);
+    {
+        while (list_empty(&ev->events) || !__can_process_event(ev, event))
+            pthread_cond_wait(&ev->cond, &ev->lock);
+    }
+    pthread_mutex_unlock(&ev->lock);
 }
 
 void
-pick_event_unordered (struct gf_event_list *ev, struct gf_event **event)
+pick_event_unordered(struct gf_event_list *ev, struct gf_event **event)
 {
-        pthread_mutex_lock (&ev->lock);
-        {
-                while (list_empty (&ev->events))
-                        pthread_cond_wait (&ev->cond, &ev->lock);
-                *event = list_first_entry (&ev->events, struct gf_event, list);
-                list_del (&(*event)->list);
-        }
-        pthread_mutex_unlock (&ev->lock);
+    pthread_mutex_lock(&ev->lock);
+    {
+        while (list_empty(&ev->events))
+            pthread_cond_wait(&ev->cond, &ev->lock);
+        *event = list_first_entry(&ev->events, struct gf_event, list);
+        list_del(&(*event)->list);
+    }
+    pthread_mutex_unlock(&ev->lock);
 }
 
 void *
-gf_changelog_callback_invoker (void *arg)
+gf_changelog_callback_invoker(void *arg)
 {
-        int                   ret    = 0;
-        xlator_t             *this   = NULL;
-        gf_changelog_t       *entry  = NULL;
-        struct iovec         *vec    = NULL;
-        struct gf_event      *event  = NULL;
-        struct gf_event_list *ev     = NULL;
+    xlator_t *this = NULL;
+    gf_changelog_t *entry = NULL;
+    struct iovec *vec = NULL;
+    struct gf_event *event = NULL;
+    struct gf_event_list *ev = NULL;
 
-        ev    = arg;
-        entry = ev->entry;
-        THIS = this = entry->this;
+    ev = arg;
+    entry = ev->entry;
+    THIS = this = entry->this;
 
-        while (1) {
-                entry->pickevent (ev, &event);
+    while (1) {
+        entry->pickevent(ev, &event);
 
-                vec = (struct iovec *) &event->iov;
-                gf_changelog_invoke_callback (entry, &vec, event->count);
+        vec = (struct iovec *)&event->iov;
+        gf_changelog_invoke_callback(entry, &vec, event->count);
 
-                GF_FREE (event);
-        }
+        GF_FREE(event);
+    }
 
-        return NULL;
+    return NULL;
 }
 
 static int
-orderfn (struct list_head *pos1, struct list_head *pos2)
+orderfn(struct list_head *pos1, struct list_head *pos2)
 {
-        struct gf_event *event1 = NULL;
-        struct gf_event *event2 = NULL;
+    struct gf_event *event1 = NULL;
+    struct gf_event *event2 = NULL;
 
-        event1 = list_entry (pos1, struct gf_event, list);
-        event2 = list_entry (pos2, struct gf_event, list);
+    event1 = list_entry(pos1, struct gf_event, list);
+    event2 = list_entry(pos2, struct gf_event, list);
 
-        if  (event1->seq > event2->seq)
-                return 1;
-        return -1;
+    if (event1->seq > event2->seq)
+        return 1;
+    return -1;
 }
 
 void
-queue_ordered_event (struct gf_event_list *ev, struct gf_event *event)
+queue_ordered_event(struct gf_event_list *ev, struct gf_event *event)
 {
-        /* add event to the ordered event list and wake up listner(s) */
-        pthread_mutex_lock (&ev->lock);
-        {
-                list_add_order (&event->list, &ev->events, orderfn);
-                if (!ev->next_seq)
-                        ev->next_seq = event->seq;
-                if (ev->next_seq == event->seq)
-                        pthread_cond_signal (&ev->cond);
-        }
-        pthread_mutex_unlock (&ev->lock);
+    /* add event to the ordered event list and wake up listener(s) */
+    pthread_mutex_lock(&ev->lock);
+    {
+        list_add_order(&event->list, &ev->events, orderfn);
+        if (!ev->next_seq)
+            ev->next_seq = event->seq;
+        if (ev->next_seq == event->seq)
+            pthread_cond_signal(&ev->cond);
+    }
+    pthread_mutex_unlock(&ev->lock);
 }
 
 void
-queue_unordered_event (struct gf_event_list *ev, struct gf_event *event)
+queue_unordered_event(struct gf_event_list *ev, struct gf_event *event)
 {
-        /* add event to the tail of the queue and wake up listener(s) */
-        pthread_mutex_lock (&ev->lock);
-        {
-                list_add_tail (&event->list, &ev->events);
-                pthread_cond_signal (&ev->cond);
-        }
-        pthread_mutex_unlock (&ev->lock);
+    /* add event to the tail of the queue and wake up listener(s) */
+    pthread_mutex_lock(&ev->lock);
+    {
+        list_add_tail(&event->list, &ev->events);
+        pthread_cond_signal(&ev->cond);
+    }
+    pthread_mutex_unlock(&ev->lock);
 }
 
 int
-gf_changelog_event_handler (rpcsvc_request_t *req,
-                            xlator_t *this, gf_changelog_t *entry)
+gf_changelog_event_handler(rpcsvc_request_t *req, xlator_t *this,
+                           gf_changelog_t *entry)
 {
-        int                   i          = 0;
-        size_t                payloadlen = 0;
-        ssize_t               len        = 0;
-        int                   payloadcnt = 0;
-        changelog_event_req   rpc_req    = {0,};
-        changelog_event_rsp   rpc_rsp    = {0,};
-        struct iovec         *vec        = NULL;
-        struct gf_event      *event      = NULL;
-        struct gf_event_list *ev         = NULL;
-
-        ev = &entry->event;
-
-        len = xdr_to_generic (req->msg[0],
-                              &rpc_req, (xdrproc_t)xdr_changelog_event_req);
-        if (len < 0) {
-                gf_log (this->name, GF_LOG_ERROR, "xdr decoding failed");
-                req->rpc_err = GARBAGE_ARGS;
-                goto handle_xdr_error;
-        }
-
-        if (len < req->msg[0].iov_len) {
-                payloadcnt = 1;
-                payloadlen = (req->msg[0].iov_len - len);
-        }
-        for (i = 1; i < req->count; i++) {
-                payloadcnt++;
-                payloadlen += req->msg[i].iov_len;
-        }
-
-        event = GF_CALLOC (1, GF_EVENT_CALLOC_SIZE (payloadcnt, payloadlen),
-                           gf_changelog_mt_libgfchangelog_event_t);
-        if (!event)
-                goto handle_xdr_error;
-        INIT_LIST_HEAD (&event->list);
-
-        payloadlen   = 0;
-        event->seq   = rpc_req.seq;
-        event->count = payloadcnt;
-
-        /* deep copy IO vectors */
-        vec = &event->iov[0];
-        GF_EVENT_ASSIGN_IOVEC (vec, event,
-                               (req->msg[0].iov_len - len), payloadlen);
-        (void) memcpy (vec->iov_base,
-                       req->msg[0].iov_base + len, vec->iov_len);
-
-        for (i = 1; i < req->count; i++) {
-                vec = &event->iov[i];
-                GF_EVENT_ASSIGN_IOVEC (vec, event,
-                                       req->msg[i].iov_len, payloadlen);
-                (void) memcpy (event->iov[i].iov_base,
-                               req->msg[i].iov_base, req->msg[i].iov_len);
-        }
-
-        gf_log (this->name, GF_LOG_DEBUG,
-                "seq: %lu [%s] (time: %lu.%lu), (vec: %d, len: %ld)",
-                rpc_req.seq, entry->brick, rpc_req.tv_sec,
-                rpc_req.tv_usec, payloadcnt, payloadlen);
-
-        /* dispatch event */
-        entry->queueevent (ev, event);
-
-        /* ack sequence number */
-        rpc_rsp.op_ret = 0;
-        rpc_rsp.seq    = rpc_req.seq;
-
-        goto submit_rpc;
-
- handle_xdr_error:
-        rpc_rsp.op_ret = -1;
-        rpc_rsp.seq    = 0;     /* invalid */
- submit_rpc:
-        return changelog_rpc_sumbit_reply (req, &rpc_rsp, NULL, 0, NULL,
-                                           (xdrproc_t)xdr_changelog_event_rsp);
+    int i = 0;
+    size_t payloadlen = 0;
+    ssize_t len = 0;
+    int payloadcnt = 0;
+    changelog_event_req rpc_req = {
+        0,
+    };
+    changelog_event_rsp rpc_rsp = {
+        0,
+    };
+    struct iovec *vec = NULL;
+    struct gf_event *event = NULL;
+    struct gf_event_list *ev = NULL;
+
+    ev = &entry->event;
+
+    len = xdr_to_generic(req->msg[0], &rpc_req,
+                         (xdrproc_t)xdr_changelog_event_req);
+    if (len < 0) {
+        gf_msg(this->name, GF_LOG_ERROR, 0,
+               CHANGELOG_LIB_MSG_XDR_DECODING_FAILED, "xdr decoding failed");
+        req->rpc_err = GARBAGE_ARGS;
+        goto handle_xdr_error;
+    }
+
+    if (len < req->msg[0].iov_len) {
+        payloadcnt = 1;
+        payloadlen = (req->msg[0].iov_len - len);
+    }
+    for (i = 1; i < req->count; i++) {
+        payloadcnt++;
+        payloadlen += req->msg[i].iov_len;
+    }
+
+    event = GF_CALLOC(1, GF_EVENT_CALLOC_SIZE(payloadcnt, payloadlen),
+                      gf_changelog_mt_libgfchangelog_event_t);
+    if (!event)
+        goto handle_xdr_error;
+    INIT_LIST_HEAD(&event->list);
+
+    payloadlen = 0;
+    event->seq = rpc_req.seq;
+    event->count = payloadcnt;
+
+    /* deep copy IO vectors */
+    vec = &event->iov[0];
+    GF_EVENT_ASSIGN_IOVEC(vec, event, (req->msg[0].iov_len - len), payloadlen);
+    (void)memcpy(vec->iov_base, req->msg[0].iov_base + len, vec->iov_len);
+
+    for (i = 1; i < req->count; i++) {
+        vec = &event->iov[i];
+        GF_EVENT_ASSIGN_IOVEC(vec, event, req->msg[i].iov_len, payloadlen);
+        (void)memcpy(event->iov[i].iov_base, req->msg[i].iov_base,
+                     req->msg[i].iov_len);
+    }
+
+    gf_msg_debug(this->name, 0,
+                 "seq: %" PRIu64 " [%s] (time: %" PRIu64 ".%" PRIu64
+                 "), "
+                 "(vec: %d, len: %zd)",
+                 rpc_req.seq, entry->brick, rpc_req.tv_sec, rpc_req.tv_usec,
+                 payloadcnt, payloadlen);
+
+    /* dispatch event */
+    entry->queueevent(ev, event);
+
+    /* ack sequence number */
+    rpc_rsp.op_ret = 0;
+    rpc_rsp.seq = rpc_req.seq;
+
+    goto submit_rpc;
+
+handle_xdr_error:
+    rpc_rsp.op_ret = -1;
+    rpc_rsp.seq = 0; /* invalid */
+submit_rpc:
+    return changelog_rpc_sumbit_reply(req, &rpc_rsp, NULL, 0, NULL,
+                                      (xdrproc_t)xdr_changelog_event_rsp);
 }
 
 int
-gf_changelog_reborp_handle_event (rpcsvc_request_t *req)
+gf_changelog_reborp_handle_event(rpcsvc_request_t *req)
 {
-        xlator_t       *this  = NULL;
-        rpcsvc_t       *svc   = NULL;
-        gf_changelog_t *entry = NULL;
+    xlator_t *this = NULL;
+    rpcsvc_t *svc = NULL;
+    gf_changelog_t *entry = NULL;
 
-        svc = rpcsvc_request_service (req);
-        entry = svc->mydata;
+    svc = rpcsvc_request_service(req);
+    entry = svc->mydata;
 
-        this = THIS = entry->this;
+    this = THIS = entry->this;
 
-        return gf_changelog_event_handler (req, this, entry);
+    return gf_changelog_event_handler(req, this, entry);
 }
 
-rpcsvc_actor_t gf_changelog_reborp_actors[CHANGELOG_REV_PROC_MAX] = {
-        [CHANGELOG_REV_PROC_EVENT] = {
-                "CHANGELOG EVENT HANDLER", CHANGELOG_REV_PROC_EVENT,
-                gf_changelog_reborp_handle_event, NULL, 0, DRC_NA
-        },
+static rpcsvc_actor_t gf_changelog_reborp_actors[CHANGELOG_REV_PROC_MAX] = {
+    [CHANGELOG_REV_PROC_EVENT] = {"CHANGELOG EVENT HANDLER",
+                                  gf_changelog_reborp_handle_event, NULL,
+                                  CHANGELOG_REV_PROC_EVENT, DRC_NA, 0},
 };
 
 /**
@@ -450,16 +398,16 @@ rpcsvc_actor_t gf_changelog_reborp_actors[CHANGELOG_REV_PROC_MAX] = {
  * and that's required to invoke the callback with the appropriate
  * brick path and it's private data.
  */
-struct rpcsvc_program gf_changelog_reborp_prog = {
-        .progname  = "LIBGFCHANGELOG REBORP",
-        .prognum   = CHANGELOG_REV_RPC_PROCNUM,
-        .progver   = CHANGELOG_REV_RPC_PROCVER,
-        .numactors = CHANGELOG_REV_PROC_MAX,
-        .actors    = gf_changelog_reborp_actors,
-        .synctask  = _gf_false,
+static struct rpcsvc_program gf_changelog_reborp_prog = {
+    .progname = "LIBGFCHANGELOG REBORP",
+    .prognum = CHANGELOG_REV_RPC_PROCNUM,
+    .progver = CHANGELOG_REV_RPC_PROCVER,
+    .numactors = CHANGELOG_REV_PROC_MAX,
+    .actors = gf_changelog_reborp_actors,
+    .synctask = _gf_false,
 };
 
-struct rpcsvc_program *gf_changelog_reborp_programs[] = {
-        &gf_changelog_reborp_prog,
-        NULL,
+static struct rpcsvc_program *gf_changelog_reborp_programs[] = {
+    &gf_changelog_reborp_prog,
+    NULL,
 };
diff --git a/xlators/features/changelog/lib/src/gf-changelog-rpc.c b/xlators/features/changelog/lib/src/gf-changelog-rpc.c
index 270632bc71b..8ec6ffbcebc 100644
--- a/xlators/features/changelog/lib/src/gf-changelog-rpc.c
+++ b/xlators/features/changelog/lib/src/gf-changelog-rpc.c
@@ -16,31 +16,32 @@ struct rpc_clnt_program gf_changelog_clnt;
 
 /* TODO: piggyback reconnect to called (upcall) */
 int
-gf_changelog_rpc_notify (struct rpc_clnt *rpc,
-                         void *mydata, rpc_clnt_event_t event, void *data)
+gf_changelog_rpc_notify(struct rpc_clnt *rpc, void *mydata,
+                        rpc_clnt_event_t event, void *data)
 {
-        switch (event) {
+    switch (event) {
         case RPC_CLNT_CONNECT:
-                rpc_clnt_set_connected (&rpc->conn);
-                break;
+            break;
         case RPC_CLNT_DISCONNECT:
         case RPC_CLNT_MSG:
         case RPC_CLNT_DESTROY:
-                break;
-        }
+        case RPC_CLNT_PING:
+            break;
+    }
 
-        return 0;
+    return 0;
 }
 
 struct rpc_clnt *
-gf_changelog_rpc_init (xlator_t *this, gf_changelog_t *entry)
+gf_changelog_rpc_init(xlator_t *this, gf_changelog_t *entry)
 {
-        char sockfile[UNIX_PATH_MAX] = {0,};
+    char sockfile[UNIX_PATH_MAX] = {
+        0,
+    };
 
-        CHANGELOG_MAKE_SOCKET_PATH (entry->brick,
-                                    sockfile, UNIX_PATH_MAX);
-        return changelog_rpc_client_init (this, entry,
-                                          sockfile, gf_changelog_rpc_notify);
+    CHANGELOG_MAKE_SOCKET_PATH(entry->brick, sockfile, UNIX_PATH_MAX);
+    return changelog_rpc_client_init(this, entry, sockfile,
+                                     gf_changelog_rpc_notify);
 }
 
 /**
@@ -48,52 +49,50 @@ gf_changelog_rpc_init (xlator_t *this, gf_changelog_t *entry)
  */
 
 int
-gf_probe_changelog_cbk (struct rpc_req *req,
-                        struct iovec *iovec, int count, void *myframe)
+gf_probe_changelog_cbk(struct rpc_req *req, struct iovec *iovec, int count,
+                       void *myframe)
 {
-        return 0;
+    return 0;
 }
 
 int
-gf_probe_changelog_filter (call_frame_t *frame, xlator_t *this, void *data)
+gf_probe_changelog_filter(call_frame_t *frame, xlator_t *this, void *data)
 {
-        int ret = 0;
-        char *sock = NULL;
-        gf_changelog_t *entry = NULL;
-        changelog_probe_req req = {0,};
-
-        entry = data;
-        sock = RPC_SOCK (entry);
-
-        (void) memcpy (&req.sock, sock, strlen (sock));
-        req.filter = entry->notify;
-
-        /* invoke RPC */
-        return changelog_rpc_sumbit_req (RPC_PROBER (entry), (void *) &req,
-                                         frame, &gf_changelog_clnt,
-                                         CHANGELOG_RPC_PROBE_FILTER, NULL, 0,
-                                         NULL, this, gf_probe_changelog_cbk,
-                                         (xdrproc_t) xdr_changelog_probe_req);
+    char *sock = NULL;
+    gf_changelog_t *entry = NULL;
+    changelog_probe_req req = {
+        0,
+    };
+
+    entry = data;
+    sock = RPC_SOCK(entry);
+
+    (void)memcpy(&req.sock, sock, strlen(sock));
+    req.filter = entry->notify;
+
+    /* invoke RPC */
+    return changelog_rpc_sumbit_req(
+        RPC_PROBER(entry), (void *)&req, frame, &gf_changelog_clnt,
+        CHANGELOG_RPC_PROBE_FILTER, NULL, 0, NULL, this, gf_probe_changelog_cbk,
+        (xdrproc_t)xdr_changelog_probe_req);
 }
 
 int
-gf_changelog_invoke_rpc (xlator_t *this, gf_changelog_t *entry, int procidx)
+gf_changelog_invoke_rpc(xlator_t *this, gf_changelog_t *entry, int procidx)
 {
-        return changelog_invoke_rpc (this, RPC_PROBER (entry),
-                                     &gf_changelog_clnt, procidx, entry);
+    return changelog_invoke_rpc(this, RPC_PROBER(entry), &gf_changelog_clnt,
+                                procidx, entry);
 }
 
 struct rpc_clnt_procedure gf_changelog_procs[CHANGELOG_RPC_PROC_MAX] = {
-        [CHANGELOG_RPC_PROC_NULL] = {"NULL", NULL},
-        [CHANGELOG_RPC_PROBE_FILTER] = {
-                "PROBE FILTER", gf_probe_changelog_filter
-        },
+    [CHANGELOG_RPC_PROC_NULL] = {"NULL", NULL},
+    [CHANGELOG_RPC_PROBE_FILTER] = {"PROBE FILTER", gf_probe_changelog_filter},
 };
 
 struct rpc_clnt_program gf_changelog_clnt = {
-        .progname  = "LIBGFCHANGELOG",
-        .prognum   = CHANGELOG_RPC_PROGNUM,
-        .progver   = CHANGELOG_RPC_PROGVER,
-        .numproc   = CHANGELOG_RPC_PROC_MAX,
-        .proctable = gf_changelog_procs,
+    .progname = "LIBGFCHANGELOG",
+    .prognum = CHANGELOG_RPC_PROGNUM,
+    .progver = CHANGELOG_RPC_PROGVER,
+    .numproc = CHANGELOG_RPC_PROC_MAX,
+    .proctable = gf_changelog_procs,
 };
diff --git a/xlators/features/changelog/lib/src/gf-changelog-rpc.h b/xlators/features/changelog/lib/src/gf-changelog-rpc.h
index 1c982eef809..5c82d6f1c08 100644
--- a/xlators/features/changelog/lib/src/gf-changelog-rpc.h
+++ b/xlators/features/changelog/lib/src/gf-changelog-rpc.h
@@ -11,16 +11,18 @@
 #ifndef __GF_CHANGELOG_RPC_H
 #define __GF_CHANGELOG_RPC_H
 
-#include "xlator.h"
+#include <glusterfs/xlator.h>
 
 #include "gf-changelog-helpers.h"
 #include "changelog-rpc-common.h"
 
-struct rpc_clnt *gf_changelog_rpc_init (xlator_t *, gf_changelog_t *);
+struct rpc_clnt *
+gf_changelog_rpc_init(xlator_t *, gf_changelog_t *);
 
-int gf_changelog_invoke_rpc (xlator_t *, gf_changelog_t *, int);
+int
+gf_changelog_invoke_rpc(xlator_t *, gf_changelog_t *, int);
 
 rpcsvc_t *
-gf_changelog_reborp_init_rpc_listner (xlator_t *, char *, char *, void *);
+gf_changelog_reborp_init_rpc_listner(xlator_t *, char *, char *, void *);
 
 #endif
diff --git a/xlators/features/changelog/lib/src/gf-changelog.c b/xlators/features/changelog/lib/src/gf-changelog.c
index 82abc662632..57c3d39ef76 100644
--- a/xlators/features/changelog/lib/src/gf-changelog.c
+++ b/xlators/features/changelog/lib/src/gf-changelog.c
@@ -22,11 +22,11 @@
 #endif
 #include <string.h>
 
-#include "globals.h"
-#include "glusterfs.h"
-#include "logging.h"
-#include "defaults.h"
-#include "syncop.h"
+#include <glusterfs/globals.h>
+#include <glusterfs/glusterfs.h>
+#include <glusterfs/logging.h>
+#include <glusterfs/defaults.h>
+#include <glusterfs/syncop.h>
 
 #include "gf-changelog-rpc.h"
 #include "gf-changelog-helpers.h"
@@ -34,6 +34,7 @@
 /* from the changelog translator */
 #include "changelog-misc.h"
 #include "changelog-mem-types.h"
+#include "changelog-lib-messages.h"
 
 /**
  * Global singleton xlator pointer for the library, initialized
@@ -44,275 +45,315 @@
  */
 xlator_t *master = NULL;
 
-static inline
-gf_private_t *gf_changelog_alloc_priv ()
+static inline gf_private_t *
+gf_changelog_alloc_priv()
 {
-        int ret = 0;
-        gf_private_t *priv = NULL;
-
-        priv = GF_CALLOC (1, sizeof (*priv), gf_changelog_mt_priv_t);
-        if (!priv)
-                goto error_return;
-        INIT_LIST_HEAD (&priv->connections);
-        INIT_LIST_HEAD (&priv->cleanups);
-
-        ret = pthread_mutex_init (&priv->lock, NULL);
-        if (ret != 0)
-                goto free_priv;
-        ret = pthread_cond_init (&priv->cond, NULL);
-        if (ret != 0)
-                goto cleanup_mutex;
-
-        priv->api = NULL;
-        return priv;
-
- cleanup_mutex:
-        (void) pthread_mutex_destroy (&priv->lock);
- free_priv:
-        GF_FREE (priv);
- error_return:
-        return NULL;
+    int ret = 0;
+    gf_private_t *priv = NULL;
+
+    priv = GF_CALLOC(1, sizeof(*priv), gf_changelog_mt_priv_t);
+    if (!priv)
+        goto error_return;
+    INIT_LIST_HEAD(&priv->connections);
+    INIT_LIST_HEAD(&priv->cleanups);
+
+    ret = pthread_mutex_init(&priv->lock, NULL);
+    if (ret != 0)
+        goto free_priv;
+    ret = pthread_cond_init(&priv->cond, NULL);
+    if (ret != 0)
+        goto cleanup_mutex;
+
+    priv->api = NULL;
+    return priv;
+
+cleanup_mutex:
+    (void)pthread_mutex_destroy(&priv->lock);
+free_priv:
+    GF_FREE(priv);
+error_return:
+    return NULL;
 }
 
-#define GF_CHANGELOG_EVENT_POOL_SIZE   16384
+#define GF_CHANGELOG_EVENT_POOL_SIZE 16384
 #define GF_CHANGELOG_EVENT_THREAD_COUNT 4
 
 static int
-gf_changelog_ctx_defaults_init (glusterfs_ctx_t *ctx)
+gf_changelog_ctx_defaults_init(glusterfs_ctx_t *ctx)
 {
-        cmd_args_t    *cmd_args = NULL;
-        struct rlimit  lim = {0, };
-        call_pool_t   *pool = NULL;
-        int            ret         = -1;
+    cmd_args_t *cmd_args = NULL;
+    struct rlimit lim = {
+        0,
+    };
+    call_pool_t *pool = NULL;
+    int ret = -1;
+
+    ret = xlator_mem_acct_init(THIS, gf_changelog_mt_end);
+    if (ret != 0)
+        return -1;
 
-        ret = xlator_mem_acct_init (THIS, gf_changelog_mt_end);
-        if (ret != 0)
-                return -1;
+    ctx->process_uuid = generate_glusterfs_ctx_id();
+    if (!ctx->process_uuid)
+        return -1;
 
-        ctx->process_uuid = generate_glusterfs_ctx_id ();
-        if (!ctx->process_uuid)
-                return -1;
+    ctx->page_size = 128 * GF_UNIT_KB;
 
-        ctx->page_size  = 128 * GF_UNIT_KB;
+    ctx->iobuf_pool = iobuf_pool_new();
+    if (!ctx->iobuf_pool)
+        goto free_pool;
 
-        ctx->iobuf_pool = iobuf_pool_new ();
-        if (!ctx->iobuf_pool)
-                return -1;
+    ctx->event_pool = gf_event_pool_new(GF_CHANGELOG_EVENT_POOL_SIZE,
+                                        GF_CHANGELOG_EVENT_THREAD_COUNT);
+    if (!ctx->event_pool)
+        goto free_pool;
 
-        ctx->event_pool = event_pool_new (GF_CHANGELOG_EVENT_POOL_SIZE,
-                                          GF_CHANGELOG_EVENT_THREAD_COUNT);
-        if (!ctx->event_pool)
-                return -1;
+    pool = GF_CALLOC(1, sizeof(call_pool_t),
+                     gf_changelog_mt_libgfchangelog_call_pool_t);
+    if (!pool)
+        goto free_pool;
 
-        pool = GF_CALLOC (1, sizeof (call_pool_t),
-                          gf_changelog_mt_libgfchangelog_call_pool_t);
-        if (!pool)
-                return -1;
+    /* frame_mem_pool size 112 * 64 */
+    pool->frame_mem_pool = mem_pool_new(call_frame_t, 32);
+    if (!pool->frame_mem_pool)
+        goto free_pool;
 
-        /* frame_mem_pool size 112 * 64 */
-        pool->frame_mem_pool = mem_pool_new (call_frame_t, 32);
-        if (!pool->frame_mem_pool)
-                return -1;
+    /* stack_mem_pool size 256 * 128 */
+    pool->stack_mem_pool = mem_pool_new(call_stack_t, 16);
 
-        /* stack_mem_pool size 256 * 128 */
-        pool->stack_mem_pool = mem_pool_new (call_stack_t, 16);
+    if (!pool->stack_mem_pool)
+        goto free_pool;
 
-        if (!pool->stack_mem_pool)
-                return -1;
+    ctx->stub_mem_pool = mem_pool_new(call_stub_t, 16);
+    if (!ctx->stub_mem_pool)
+        goto free_pool;
 
-        ctx->stub_mem_pool = mem_pool_new (call_stub_t, 16);
-        if (!ctx->stub_mem_pool)
-                return -1;
+    ctx->dict_pool = mem_pool_new(dict_t, 32);
+    if (!ctx->dict_pool)
+        goto free_pool;
 
-        ctx->dict_pool = mem_pool_new (dict_t, 32);
-        if (!ctx->dict_pool)
-                return -1;
+    ctx->dict_pair_pool = mem_pool_new(data_pair_t, 512);
+    if (!ctx->dict_pair_pool)
+        goto free_pool;
 
-        ctx->dict_pair_pool = mem_pool_new (data_pair_t, 512);
-        if (!ctx->dict_pair_pool)
-                return -1;
+    ctx->dict_data_pool = mem_pool_new(data_t, 512);
+    if (!ctx->dict_data_pool)
+        goto free_pool;
 
-        ctx->dict_data_pool = mem_pool_new (data_t, 512);
-        if (!ctx->dict_data_pool)
-                return -1;
+    ctx->logbuf_pool = mem_pool_new(log_buf_t, 256);
+    if (!ctx->logbuf_pool)
+        goto free_pool;
 
-        INIT_LIST_HEAD (&pool->all_frames);
-        LOCK_INIT (&pool->lock);
-        ctx->pool = pool;
+    INIT_LIST_HEAD(&pool->all_frames);
+    LOCK_INIT(&pool->lock);
+    ctx->pool = pool;
 
-        pthread_mutex_init (&(ctx->lock), NULL);
+    LOCK_INIT(&ctx->lock);
 
-        cmd_args = &ctx->cmd_args;
+    cmd_args = &ctx->cmd_args;
 
-        INIT_LIST_HEAD (&cmd_args->xlator_options);
+    INIT_LIST_HEAD(&cmd_args->xlator_options);
 
-        lim.rlim_cur = RLIM_INFINITY;
-        lim.rlim_max = RLIM_INFINITY;
-        setrlimit (RLIMIT_CORE, &lim);
+    lim.rlim_cur = RLIM_INFINITY;
+    lim.rlim_max = RLIM_INFINITY;
+    setrlimit(RLIMIT_CORE, &lim);
 
-        return 0;
+    return 0;
+
+free_pool:
+    if (pool) {
+        GF_FREE(pool->frame_mem_pool);
+
+        GF_FREE(pool->stack_mem_pool);
+
+        GF_FREE(pool);
+    }
+
+    GF_FREE(ctx->stub_mem_pool);
+
+    GF_FREE(ctx->dict_pool);
+
+    GF_FREE(ctx->dict_pair_pool);
+
+    GF_FREE(ctx->dict_data_pool);
+
+    GF_FREE(ctx->logbuf_pool);
+
+    GF_FREE(ctx->iobuf_pool);
+
+    GF_FREE(ctx->event_pool);
+
+    return -1;
 }
 
 /* TODO: cleanup ctx defaults */
 void
-gf_changelog_cleanup_this (xlator_t *this)
+gf_changelog_cleanup_this(xlator_t *this)
 {
-        glusterfs_ctx_t *ctx = NULL;
+    glusterfs_ctx_t *ctx = NULL;
+
+    if (!this)
+        return;
 
-        if (!this)
-                return;
+    ctx = this->ctx;
+    syncenv_destroy(ctx->env);
+    free(ctx);
 
-        ctx = this->ctx;
-        syncenv_destroy (ctx->env);
-        free (ctx);
+    this->private = NULL;
+    this->ctx = NULL;
 
-        this->private = NULL;
-        this->ctx = NULL;
+    mem_pools_fini();
 }
 
 static int
-gf_changelog_init_context ()
+gf_changelog_init_context()
 {
-        glusterfs_ctx_t *ctx = NULL;
+    glusterfs_ctx_t *ctx = NULL;
 
-        ctx = glusterfs_ctx_new ();
-        if (!ctx)
-                goto error_return;
+    ctx = glusterfs_ctx_new();
+    if (!ctx)
+        goto error_return;
 
-        if (glusterfs_globals_init (ctx))
-                goto free_ctx;
+    if (glusterfs_globals_init(ctx))
+        goto free_ctx;
 
-        THIS->ctx = ctx;
-        if (gf_changelog_ctx_defaults_init (ctx))
-                goto free_ctx;
+    THIS->ctx = ctx;
+    if (gf_changelog_ctx_defaults_init(ctx))
+        goto free_ctx;
 
-        ctx->env = syncenv_new (0, 0, 0);
-        if (!ctx->env)
-                goto free_ctx;
-        return 0;
+    ctx->env = syncenv_new(0, 0, 0);
+    if (!ctx->env)
+        goto free_ctx;
+    return 0;
 
- free_ctx:
-        free (ctx);
-        THIS->ctx = NULL;
- error_return:
-        return -1;
+free_ctx:
+    free(ctx);
+    THIS->ctx = NULL;
+error_return:
+    return -1;
 }
 
 static int
-gf_changelog_init_master ()
+gf_changelog_init_master()
 {
-        return gf_changelog_init_context ();
+    int ret = 0;
+
+    ret = gf_changelog_init_context();
+    mem_pools_init();
+
+    return ret;
 }
 
 /* TODO: cleanup clnt/svc on failure */
 int
-gf_changelog_setup_rpc (xlator_t *this,
-                        gf_changelog_t *entry, int proc)
+gf_changelog_setup_rpc(xlator_t *this, gf_changelog_t *entry, int proc)
 {
-        int              ret = 0;
-        rpcsvc_t        *svc = NULL;
-        struct rpc_clnt *rpc = NULL;
-
-        /**
-         * Initialize a connect back socket. A probe() RPC call to the server
-         * triggers a reverse connect.
-         */
-        svc = gf_changelog_reborp_init_rpc_listner (this, entry->brick,
-                                                    RPC_SOCK (entry), entry);
-        if (!svc)
-                goto error_return;
-        RPC_REBORP (entry) = svc;
-
-        /* Initialize an RPC client */
-        rpc = gf_changelog_rpc_init (this, entry);
-        if (!rpc)
-                goto error_return;
-        RPC_PROBER (entry) = rpc;
-
-        /**
-         * @FIXME
-         * till we have connection state machine, let's delay the RPC call
-         * for now..
-         */
-        sleep (2);
-
-        /**
-         * Probe changelog translator for reverse connection. After a successful
-         * call, there's less use of the client and can be disconnected, but
-         * let's leave the connection active for any future RPC calls.
-         */
-        ret = gf_changelog_invoke_rpc (this, entry, proc);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "Could not initiate probe RPC, bailing out!!!");
-                goto error_return;
-        }
-
-        return 0;
-
- error_return:
-        return -1;
+    int ret = 0;
+    rpcsvc_t *svc = NULL;
+    struct rpc_clnt *rpc = NULL;
+
+    /**
+     * Initialize a connect back socket. A probe() RPC call to the server
+     * triggers a reverse connect.
+     */
+    svc = gf_changelog_reborp_init_rpc_listner(this, entry->brick,
+                                               RPC_SOCK(entry), entry);
+    if (!svc)
+        goto error_return;
+    RPC_REBORP(entry) = svc;
+
+    /* Initialize an RPC client */
+    rpc = gf_changelog_rpc_init(this, entry);
+    if (!rpc)
+        goto error_return;
+    RPC_PROBER(entry) = rpc;
+
+    /**
+     * @FIXME
+     * till we have connection state machine, let's delay the RPC call
+     * for now..
+     */
+    sleep(2);
+
+    /**
+     * Probe changelog translator for reverse connection. After a successful
+     * call, there's less use of the client and can be disconnected, but
+     * let's leave the connection active for any future RPC calls.
+     */
+    ret = gf_changelog_invoke_rpc(this, entry, proc);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, CHANGELOG_LIB_MSG_INVOKE_RPC_FAILED,
+               "Could not initiate probe RPC, bailing out!!!");
+        goto error_return;
+    }
+
+    return 0;
+
+error_return:
+    return -1;
 }
 
 int
-gf_cleanup_event (xlator_t *this, struct gf_event_list *ev)
+gf_cleanup_event(xlator_t *this, struct gf_event_list *ev)
 {
-        int ret = 0;
-
-        ret = gf_thread_cleanup (this, ev->invoker);
-        if (ret) {
-                gf_log (this->name, GF_LOG_WARNING,
-                        "cannot cleanup callback invoker thread "
-                        " [reason: %s]. Not freeing resources",
-                        strerror (-ret));
-                return -1;
-        }
+    int ret = 0;
+
+    ret = gf_thread_cleanup(this, ev->invoker);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_WARNING, -ret,
+               CHANGELOG_LIB_MSG_CLEANUP_ERROR,
+               "cannot cleanup callback invoker thread."
+               " Not freeing resources");
+        return -1;
+    }
 
-        ev->entry = NULL;
+    ev->entry = NULL;
 
-        return 0;
+    return 0;
 }
 
 static int
-gf_init_event (gf_changelog_t *entry)
+gf_init_event(gf_changelog_t *entry)
 {
-        int ret = 0;
-        struct gf_event_list *ev = NULL;
-
-        ev = &entry->event;
-        ev->entry = entry;
-
-        ret = pthread_mutex_init (&ev->lock, NULL);
-        if (ret != 0)
-                goto error_return;
-        ret = pthread_cond_init (&ev->cond, NULL);
-        if (ret != 0)
-                goto cleanup_mutex;
-        INIT_LIST_HEAD (&ev->events);
-
-        ev->next_seq = 0;  /* bootstrap sequencing */
-
-        ret = gf_thread_create (&ev->invoker, NULL,
-                                gf_changelog_callback_invoker, ev);
-        if (ret != 0)
-                goto cleanup_cond;
-
-        if (GF_NEED_ORDERED_EVENTS (entry)) {
-                entry->pickevent  = pick_event_ordered;
-                entry->queueevent = queue_ordered_event;
-        } else {
-                entry->pickevent  = pick_event_unordered;
-                entry->queueevent = queue_unordered_event;
-        }
-
-        return 0;
-
- cleanup_cond:
-        (void) pthread_cond_destroy (&ev->cond);
- cleanup_mutex:
-        (void) pthread_mutex_destroy (&ev->lock);
- error_return:
-        return -1;
+    int ret = 0;
+    struct gf_event_list *ev = NULL;
+
+    ev = &entry->event;
+    ev->entry = entry;
+
+    ret = pthread_mutex_init(&ev->lock, NULL);
+    if (ret != 0)
+        goto error_return;
+    ret = pthread_cond_init(&ev->cond, NULL);
+    if (ret != 0)
+        goto cleanup_mutex;
+    INIT_LIST_HEAD(&ev->events);
+
+    ev->next_seq = 0; /* bootstrap sequencing */
+
+    if (GF_NEED_ORDERED_EVENTS(entry)) {
+        entry->pickevent = pick_event_ordered;
+        entry->queueevent = queue_ordered_event;
+    } else {
+        entry->pickevent = pick_event_unordered;
+        entry->queueevent = queue_unordered_event;
+    }
+
+    ret = gf_thread_create(&ev->invoker, NULL, gf_changelog_callback_invoker,
+                           ev, "clogcbki");
+    if (ret != 0) {
+        entry->pickevent = NULL;
+        entry->queueevent = NULL;
+        goto cleanup_cond;
+    }
+
+    return 0;
+
+cleanup_cond:
+    (void)pthread_cond_destroy(&ev->cond);
+cleanup_mutex:
+    (void)pthread_mutex_destroy(&ev->lock);
+error_return:
+    return -1;
 }
 
 /**
@@ -322,243 +363,241 @@ gf_init_event (gf_changelog_t *entry)
  *  - destroy rpc{-clnt, svc}
  */
 int
-gf_cleanup_brick_connection (xlator_t *this, gf_changelog_t *entry)
+gf_cleanup_brick_connection(xlator_t *this, gf_changelog_t *entry)
 {
-        return 0;
+    return 0;
 }
 
 int
-gf_cleanup_connections (xlator_t *this)
+gf_cleanup_connections(xlator_t *this)
 {
-        return 0;
+    return 0;
 }
 
 static int
-gf_setup_brick_connection (xlator_t *this,
-                           struct gf_brick_spec *brick,
-                           gf_boolean_t ordered, void *xl)
+gf_setup_brick_connection(xlator_t *this, struct gf_brick_spec *brick,
+                          gf_boolean_t ordered, void *xl)
 {
-        int ret = 0;
-        gf_private_t *priv = NULL;
-        gf_changelog_t *entry = NULL;
-
-        priv = this->private;
-
-        if (!brick->callback || !brick->init || !brick->fini)
-                goto error_return;
-
-        entry = GF_CALLOC (1, sizeof (*entry),
-                           gf_changelog_mt_libgfchangelog_t);
-        if (!entry)
-                goto error_return;
-        INIT_LIST_HEAD (&entry->list);
-
-        LOCK_INIT (&entry->statelock);
-        entry->connstate = GF_CHANGELOG_CONN_STATE_PENDING;
-
-        entry->notify = brick->filter;
-        (void) strncpy (entry->brick, brick->brick_path, PATH_MAX-1);
-        entry->brick[PATH_MAX-1] = 0;
-
-        entry->this = this;
-        entry->invokerxl = xl;
-
-        entry->ordered = ordered;
-        ret = gf_init_event (entry);
-        if (ret)
-                goto free_entry;
-
-        entry->fini         = brick->fini;
-        entry->callback     = brick->callback;
-        entry->connected    = brick->connected;
-        entry->disconnected = brick->disconnected;
-
-        entry->ptr = brick->init (this, brick);
-        if (!entry->ptr)
-                goto cleanup_event;
-        priv->api = entry->ptr;  /* pointer to API, if required */
-
-        pthread_mutex_lock (&priv->lock);
-        {
-                list_add_tail (&entry->list, &priv->connections);
-        }
-        pthread_mutex_unlock (&priv->lock);
-
-        ret = gf_changelog_setup_rpc (this, entry, CHANGELOG_RPC_PROBE_FILTER);
-        if (ret)
-                goto cleanup_event;
-        return 0;
-
- cleanup_event:
-        (void) gf_cleanup_event (this, &entry->event);
- free_entry:
-        gf_log (this->name, GF_LOG_DEBUG, "freeing entry %p", entry);
-        list_del (&entry->list); /* FIXME: kludge for now */
-        GF_FREE (entry);
- error_return:
-        return -1;
+    int ret = 0;
+    gf_private_t *priv = NULL;
+    gf_changelog_t *entry = NULL;
+
+    priv = this->private;
+
+    if (!brick->callback || !brick->init || !brick->fini)
+        goto error_return;
+
+    entry = GF_CALLOC(1, sizeof(*entry), gf_changelog_mt_libgfchangelog_t);
+    if (!entry)
+        goto error_return;
+    INIT_LIST_HEAD(&entry->list);
+
+    LOCK_INIT(&entry->statelock);
+    entry->connstate = GF_CHANGELOG_CONN_STATE_PENDING;
+
+    entry->notify = brick->filter;
+    if (snprintf(entry->brick, PATH_MAX, "%s", brick->brick_path) >= PATH_MAX)
+        goto free_entry;
+
+    entry->this = this;
+    entry->invokerxl = xl;
+
+    entry->ordered = ordered;
+    ret = gf_init_event(entry);
+    if (ret)
+        goto free_entry;
+
+    entry->fini = brick->fini;
+    entry->callback = brick->callback;
+    entry->connected = brick->connected;
+    entry->disconnected = brick->disconnected;
+
+    entry->ptr = brick->init(this, brick);
+    if (!entry->ptr)
+        goto cleanup_event;
+    priv->api = entry->ptr; /* pointer to API, if required */
+
+    pthread_mutex_lock(&priv->lock);
+    {
+        list_add_tail(&entry->list, &priv->connections);
+    }
+    pthread_mutex_unlock(&priv->lock);
+
+    ret = gf_changelog_setup_rpc(this, entry, CHANGELOG_RPC_PROBE_FILTER);
+    if (ret)
+        goto cleanup_event;
+    return 0;
+
+cleanup_event:
+    (void)gf_cleanup_event(this, &entry->event);
+free_entry:
+    gf_msg_debug(this->name, 0, "freeing entry %p", entry);
+    list_del(&entry->list); /* FIXME: kludge for now */
+    GF_FREE(entry);
+error_return:
+    return -1;
 }
 
 int
-gf_changelog_register_brick (xlator_t *this,
-                             struct gf_brick_spec *brick,
-                             gf_boolean_t ordered, void *xl)
+gf_changelog_register_brick(xlator_t *this, struct gf_brick_spec *brick,
+                            gf_boolean_t ordered, void *xl)
 {
-        return gf_setup_brick_connection (this, brick, ordered, xl);
+    return gf_setup_brick_connection(this, brick, ordered, xl);
 }
 
 static int
-gf_changelog_setup_logging (xlator_t *this, char *logfile, int loglevel)
+gf_changelog_setup_logging(xlator_t *this, char *logfile, int loglevel)
 {
-        /* passing ident as NULL means to use default ident for syslog */
-        if (gf_log_init (this->ctx, logfile, NULL))
-                return -1;
+    /* passing ident as NULL means to use default ident for syslog */
+    if (gf_log_init(this->ctx, logfile, NULL))
+        return -1;
 
-        gf_log_set_loglevel ((loglevel == -1) ? GF_LOG_INFO :
-                             loglevel);
-        return 0;
+    gf_log_set_loglevel(this->ctx, (loglevel == -1) ? GF_LOG_INFO : loglevel);
+    return 0;
 }
 
 static int
-gf_changelog_set_master (xlator_t *master, void *xl)
+gf_changelog_set_master(xlator_t *master, void *xl)
 {
-        int32_t ret = 0;
-        xlator_t *this = NULL;
-        xlator_t *old_this = NULL;
-        gf_private_t *priv = NULL;
-
-        this = xl;
-        if (!this || !this->ctx) {
-                ret = gf_changelog_init_master ();
-                if (ret)
-                        return -1;
-                this = THIS;
-        }
+    int32_t ret = 0;
+    xlator_t *this = NULL;
+    xlator_t *old_this = NULL;
+    gf_private_t *priv = NULL;
+
+    this = xl;
+    if (!this || !this->ctx) {
+        ret = gf_changelog_init_master();
+        if (ret)
+            return -1;
+        this = THIS;
+    }
 
-        master->ctx = this->ctx;
+    master->ctx = this->ctx;
 
-        INIT_LIST_HEAD (&master->volume_options);
-        SAVE_THIS (THIS);
+    INIT_LIST_HEAD(&master->volume_options);
+    SAVE_THIS(THIS);
 
-        ret = xlator_mem_acct_init (THIS, gf_changelog_mt_end);
-        if (ret != 0)
-                goto restore_this;
+    ret = xlator_mem_acct_init(THIS, gf_changelog_mt_end);
+    if (ret != 0)
+        goto restore_this;
 
-        priv = gf_changelog_alloc_priv ();
-        if (!priv) {
-                ret = -1;
-                goto restore_this;
-        }
+    priv = gf_changelog_alloc_priv();
+    if (!priv) {
+        ret = -1;
+        goto restore_this;
+    }
 
-        if (!xl) {
-                /* poller thread */
-                ret = gf_thread_create (&priv->poller,
-                                        NULL, changelog_rpc_poller, THIS);
-                if (ret != 0) {
-                        GF_FREE (priv);
-                        gf_log (master->name, GF_LOG_ERROR,
-                                "failed to spawn poller thread");
-                        goto restore_this;
-                }
+    if (!xl) {
+        /* poller thread */
+        ret = gf_thread_create(&priv->poller, NULL, changelog_rpc_poller, THIS,
+                               "clogpoll");
+        if (ret != 0) {
+            GF_FREE(priv);
+            gf_msg(master->name, GF_LOG_ERROR, 0,
+                   CHANGELOG_LIB_MSG_THREAD_CREATION_FAILED,
+                   "failed to spawn poller thread");
+            goto restore_this;
         }
+    }
 
-        master->private = priv;
+    master->private = priv;
 
- restore_this:
-        RESTORE_THIS ();
+restore_this:
+    RESTORE_THIS();
 
-        return ret;
+    return ret;
 }
 
 int
-gf_changelog_init (void *xl)
+gf_changelog_init(void *xl)
 {
-        int ret = 0;
-        gf_private_t *priv = NULL;
-
-        if (master)
-                return 0;
-
-        master = calloc (1, sizeof (*master));
-        if (!master)
-                goto error_return;
-
-        master->name = strdup ("gfchangelog");
-        if (!master->name)
-                goto dealloc_master;
-
-        ret = gf_changelog_set_master (master, xl);
-        if (ret)
-                goto dealloc_name;
-
-        priv = master->private;
-        ret = gf_thread_create (&priv->connectionjanitor, NULL,
-                                gf_changelog_connection_janitor, master);
-        if (ret != 0) {
-                /* TODO: cleanup priv, mutex (poller thread for !xl) */
-                goto dealloc_name;
-        }
+    int ret = 0;
+    gf_private_t *priv = NULL;
 
+    if (master)
         return 0;
 
- dealloc_name:
-        free (master->name);
- dealloc_master:
-        free (master);
-        master = NULL;
- error_return:
-        return -1;
+    master = calloc(1, sizeof(*master));
+    if (!master)
+        goto error_return;
+
+    master->name = strdup("gfchangelog");
+    if (!master->name)
+        goto dealloc_master;
+
+    ret = gf_changelog_set_master(master, xl);
+    if (ret)
+        goto dealloc_name;
+
+    priv = master->private;
+    ret = gf_thread_create(&priv->connectionjanitor, NULL,
+                           gf_changelog_connection_janitor, master, "clogjan");
+    if (ret != 0) {
+        /* TODO: cleanup priv, mutex (poller thread for !xl) */
+        goto dealloc_name;
+    }
+
+    return 0;
+
+dealloc_name:
+    free(master->name);
+dealloc_master:
+    free(master);
+    master = NULL;
+error_return:
+    return -1;
 }
 
 int
-gf_changelog_register_generic (struct gf_brick_spec *bricks, int count,
-                               int ordered, char *logfile, int lvl, void *xl)
+gf_changelog_register_generic(struct gf_brick_spec *bricks, int count,
+                              int ordered, char *logfile, int lvl, void *xl)
 {
-        int                   ret        = 0;
-        xlator_t             *this       = NULL;
-        xlator_t             *old_this   = NULL;
-        struct gf_brick_spec *brick      = NULL;
-        gf_boolean_t          need_order = _gf_false;
+    int ret = 0;
+    xlator_t *this = NULL;
+    xlator_t *old_this = NULL;
+    struct gf_brick_spec *brick = NULL;
+    gf_boolean_t need_order = _gf_false;
 
-        SAVE_THIS (xl);
+    SAVE_THIS(xl);
 
-        this = THIS;
-        if (!this)
-                goto error_return;
+    this = THIS;
+    if (!this)
+        goto error_return;
 
-        ret = gf_changelog_setup_logging (this, logfile, lvl);
-        if (ret)
-                goto error_return;
+    ret = gf_changelog_setup_logging(this, logfile, lvl);
+    if (ret)
+        goto error_return;
 
-        need_order = (ordered) ? _gf_true : _gf_false;
+    need_order = (ordered) ? _gf_true : _gf_false;
 
-        brick = bricks;
-        while (count--) {
-                gf_log (this->name, GF_LOG_INFO,
-                        "Registering brick: %s [notify filter: %d]",
-                        brick->brick_path, brick->filter);
+    brick = bricks;
+    while (count--) {
+        gf_smsg(this->name, GF_LOG_INFO, 0,
+                CHANGELOG_LIB_MSG_NOTIFY_REGISTER_INFO, "brick=%s",
+                brick->brick_path, "notify_filter=%d", brick->filter, NULL);
 
-                ret = gf_changelog_register_brick (this, brick, need_order, xl);
-                if (ret != 0) {
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "Error registering with changelog xlator");
-                        break;
-                }
-
-                brick++;
+        ret = gf_changelog_register_brick(this, brick, need_order, xl);
+        if (ret != 0) {
+            gf_msg(this->name, GF_LOG_ERROR, 0,
+                   CHANGELOG_LIB_MSG_NOTIFY_REGISTER_FAILED,
+                   "Error registering with changelog xlator");
+            break;
         }
 
-        if (ret != 0)
-                goto cleanup_inited_bricks;
+        brick++;
+    }
 
-        RESTORE_THIS();
-        return 0;
+    if (ret != 0)
+        goto cleanup_inited_bricks;
 
- cleanup_inited_bricks:
-        gf_cleanup_connections (this);
- error_return:
-        RESTORE_THIS();
-        return -1;
+    RESTORE_THIS();
+    return 0;
+
+cleanup_inited_bricks:
+    gf_cleanup_connections(this);
+error_return:
+    RESTORE_THIS();
+    return -1;
 }
 
 /**
@@ -585,27 +624,29 @@ gf_changelog_register_generic (struct gf_brick_spec *bricks, int count,
  * For generic API, refer gf_changelog_register_generic().
  */
 int
-gf_changelog_register (char *brick_path, char *scratch_dir,
-                       char *log_file, int log_level, int max_reconnects)
+gf_changelog_register(char *brick_path, char *scratch_dir, char *log_file,
+                      int log_level, int max_reconnects)
 {
-        struct gf_brick_spec brick = {0,};
+    struct gf_brick_spec brick = {
+        0,
+    };
 
-        if (master)
-                THIS = master;
-        else
-                return -1;
+    if (master)
+        THIS = master;
+    else
+        return -1;
 
-        brick.brick_path = brick_path;
-        brick.filter     = CHANGELOG_OP_TYPE_JOURNAL;
+    brick.brick_path = brick_path;
+    brick.filter = CHANGELOG_OP_TYPE_JOURNAL;
 
-        brick.init         = gf_changelog_journal_init;
-        brick.fini         = gf_changelog_journal_fini;
-        brick.callback     = gf_changelog_handle_journal;
-        brick.connected    = gf_changelog_journal_connect;
-        brick.disconnected = gf_changelog_journal_disconnect;
+    brick.init = gf_changelog_journal_init;
+    brick.fini = gf_changelog_journal_fini;
+    brick.callback = gf_changelog_handle_journal;
+    brick.connected = gf_changelog_journal_connect;
+    brick.disconnected = gf_changelog_journal_disconnect;
 
-        brick.ptr = scratch_dir;
+    brick.ptr = scratch_dir;
 
-        return gf_changelog_register_generic (&brick, 1, 1,
-                                              log_file, log_level, NULL);
+    return gf_changelog_register_generic(&brick, 1, 1, log_file, log_level,
+                                         NULL);
 }
diff --git a/xlators/features/changelog/lib/src/gf-history-changelog.c b/xlators/features/changelog/lib/src/gf-history-changelog.c
index 50afb3fe0c7..a16219f3664 100644
--- a/xlators/features/changelog/lib/src/gf-history-changelog.c
+++ b/xlators/features/changelog/lib/src/gf-history-changelog.c
@@ -8,16 +8,17 @@
 #endif
 #include <string.h>
 
-#include "globals.h"
-#include "glusterfs.h"
-#include "logging.h"
-#include "syscall.h"
+#include <glusterfs/globals.h>
+#include <glusterfs/glusterfs.h>
+#include <glusterfs/logging.h>
+#include <glusterfs/syscall.h>
 
 #include "gf-changelog-helpers.h"
 #include "gf-changelog-journal.h"
 
 /* from the changelog translator */
 #include "changelog-misc.h"
+#include "changelog-lib-messages.h"
 #include "changelog-mem-types.h"
 
 /**
@@ -35,59 +36,60 @@
  *   -1: On error.
  */
 int
-gf_history_changelog_done (char *file)
+gf_history_changelog_done(char *file)
 {
-        int                     ret               = -1;
-        char                   *buffer            = NULL;
-        xlator_t               *this              = NULL;
-        gf_changelog_journal_t *jnl               = NULL;
-        gf_changelog_journal_t *hist_jnl          = NULL;
-        char                    to_path[PATH_MAX] = {0,};
+    int ret = -1;
+    char *buffer = NULL;
+    xlator_t *this = NULL;
+    gf_changelog_journal_t *jnl = NULL;
+    gf_changelog_journal_t *hist_jnl = NULL;
+    char to_path[PATH_MAX] = {
+        0,
+    };
+
+    errno = EINVAL;
+
+    this = THIS;
+    if (!this)
+        goto out;
+
+    jnl = (gf_changelog_journal_t *)GF_CHANGELOG_GET_API_PTR(this);
+    if (!jnl)
+        goto out;
+
+    hist_jnl = jnl->hist_jnl;
+    if (!hist_jnl)
+        goto out;
+
+    if (!file || !strlen(file))
+        goto out;
+
+    /* make sure 'file' is inside ->jnl_working_dir */
+    buffer = realpath(file, NULL);
+    if (!buffer)
+        goto out;
+
+    if (strncmp(hist_jnl->jnl_working_dir, buffer,
+                strlen(hist_jnl->jnl_working_dir)))
+        goto out;
+
+    (void)snprintf(to_path, PATH_MAX, "%s%s", hist_jnl->jnl_processed_dir,
+                   basename(buffer));
+    gf_msg_debug(this->name, 0, "moving %s to processed directory", file);
+    ret = sys_rename(buffer, to_path);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno,
+                CHANGELOG_LIB_MSG_RENAME_FAILED, "from=%s", file, "to=%s",
+                to_path, NULL);
+        goto out;
+    }
+
+    ret = 0;
 
-        errno = EINVAL;
-
-        this = THIS;
-        if (!this)
-                goto out;
-
-        jnl = (gf_changelog_journal_t *) GF_CHANGELOG_GET_API_PTR (this);
-        if (!jnl)
-                goto out;
-
-        hist_jnl = jnl->hist_jnl;
-        if (!hist_jnl)
-                goto out;
-
-        if (!file || !strlen (file))
-                goto out;
-
-        /* make sure 'file' is inside ->jnl_working_dir */
-        buffer = realpath (file, NULL);
-        if (!buffer)
-                goto out;
-
-        if (strncmp (hist_jnl->jnl_working_dir,
-                     buffer, strlen (hist_jnl->jnl_working_dir)))
-                goto out;
-
-        (void) snprintf (to_path, PATH_MAX, "%s%s",
-                         hist_jnl->jnl_processed_dir, basename (buffer));
-        gf_log (this->name, GF_LOG_DEBUG,
-                "moving %s to processed directory", file);
-        ret = rename (buffer, to_path);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "cannot move %s to %s (reason: %s)",
-                        file, to_path, strerror (errno));
-                goto out;
-        }
-
-        ret = 0;
-
- out:
-        if (buffer)
-                free (buffer); /* allocated by realpath() */
-        return ret;
+out:
+    if (buffer)
+        free(buffer); /* allocated by realpath() */
+    return ret;
 }
 
 /**
@@ -101,33 +103,33 @@ gf_history_changelog_done (char *file)
  *    -1: On error.
  */
 int
-gf_history_changelog_start_fresh ()
+gf_history_changelog_start_fresh()
 {
-        xlator_t               *this     = NULL;
-        gf_changelog_journal_t *jnl      = NULL;
-        gf_changelog_journal_t *hist_jnl = NULL;
+    xlator_t *this = NULL;
+    gf_changelog_journal_t *jnl = NULL;
+    gf_changelog_journal_t *hist_jnl = NULL;
 
-        this = THIS;
-        if (!this)
-                goto out;
+    this = THIS;
+    if (!this)
+        goto out;
 
-        errno = EINVAL;
+    errno = EINVAL;
 
-        jnl = (gf_changelog_journal_t *) GF_CHANGELOG_GET_API_PTR (this);
-        if (!jnl)
-                goto out;
+    jnl = (gf_changelog_journal_t *)GF_CHANGELOG_GET_API_PTR(this);
+    if (!jnl)
+        goto out;
 
-        hist_jnl = jnl->hist_jnl;
-        if (!hist_jnl)
-                goto out;
+    hist_jnl = jnl->hist_jnl;
+    if (!hist_jnl)
+        goto out;
 
-        if (gf_ftruncate (hist_jnl->jnl_fd, 0))
-                goto out;
+    if (gf_ftruncate(hist_jnl->jnl_fd, 0))
+        goto out;
 
-        return 0;
+    return 0;
 
- out:
-        return -1;
+out:
+    return -1;
 }
 
 /**
@@ -146,50 +148,52 @@ gf_history_changelog_start_fresh ()
  *     -1  : On error.
  */
 ssize_t
-gf_history_changelog_next_change (char *bufptr, size_t maxlen)
+gf_history_changelog_next_change(char *bufptr, size_t maxlen)
 {
-        ssize_t                 size             = -1;
-        int                     tracker_fd       = 0;
-        xlator_t               *this             = NULL;
-        gf_changelog_journal_t *jnl              = NULL;
-        gf_changelog_journal_t *hist_jnl         = NULL;
-        char                    buffer[PATH_MAX] = {0,};
-
-        if (maxlen > PATH_MAX) {
-                errno = ENAMETOOLONG;
-                goto out;
-        }
+    ssize_t size = -1;
+    int tracker_fd = 0;
+    xlator_t *this = NULL;
+    gf_changelog_journal_t *jnl = NULL;
+    gf_changelog_journal_t *hist_jnl = NULL;
+    char buffer[PATH_MAX] = {
+        0,
+    };
 
-        errno = EINVAL;
+    if (maxlen > PATH_MAX) {
+        errno = ENAMETOOLONG;
+        goto out;
+    }
 
-        this = THIS;
-        if (!this)
-                goto out;
+    errno = EINVAL;
 
-        jnl = (gf_changelog_journal_t *) GF_CHANGELOG_GET_API_PTR (this);
-        if (!jnl)
-                goto out;
+    this = THIS;
+    if (!this)
+        goto out;
 
-        hist_jnl = jnl->hist_jnl;
-        if (!hist_jnl)
-                goto out;
+    jnl = (gf_changelog_journal_t *)GF_CHANGELOG_GET_API_PTR(this);
+    if (!jnl)
+        goto out;
 
-        tracker_fd = hist_jnl->jnl_fd;
+    hist_jnl = jnl->hist_jnl;
+    if (!hist_jnl)
+        goto out;
 
-        size = gf_readline (tracker_fd, buffer, maxlen);
-        if (size < 0) {
-                size = -1;
-                goto out;
-        }
+    tracker_fd = hist_jnl->jnl_fd;
 
-        if (size == 0)
-                goto out;
+    size = gf_readline(tracker_fd, buffer, maxlen);
+    if (size < 0) {
+        size = -1;
+        goto out;
+    }
+
+    if (size == 0)
+        goto out;
 
-        memcpy (bufptr, buffer, size - 1);
-        bufptr[size - 1] = '\0';
+    memcpy(bufptr, buffer, size - 1);
+    bufptr[size - 1] = '\0';
 
 out:
-        return size;
+    return size;
 }
 
 /**
@@ -210,105 +214,100 @@ out:
  *
  */
 ssize_t
-gf_history_changelog_scan ()
+gf_history_changelog_scan()
 {
-        int                     ret          = 0;
-        int                     tracker_fd   = 0;
-        size_t                  len          = 0;
-        size_t                  off          = 0;
-        xlator_t               *this         = NULL;
-        size_t                  nr_entries   = 0;
-        gf_changelog_journal_t *jnl          = NULL;
-        gf_changelog_journal_t *hist_jnl     = NULL;
-        struct dirent          *entryp       = NULL;
-        struct dirent          *result       = NULL;
-        char buffer[PATH_MAX]                = {0,};
-        static int              is_last_scan;
-
-        this = THIS;
-        if (!this)
-                goto out;
+    int tracker_fd = 0;
+    size_t off = 0;
+    xlator_t *this = NULL;
+    size_t nr_entries = 0;
+    gf_changelog_journal_t *jnl = NULL;
+    gf_changelog_journal_t *hist_jnl = NULL;
+    struct dirent *entry = NULL;
+    struct dirent scratch[2] = {
+        {
+            0,
+        },
+    };
+    char buffer[PATH_MAX] = {
+        0,
+    };
+    static int is_last_scan;
+
+    this = THIS;
+    if (!this)
+        goto out;
+
+    jnl = (gf_changelog_journal_t *)GF_CHANGELOG_GET_API_PTR(this);
+    if (!jnl)
+        goto out;
+    if (JNL_IS_API_DISCONNECTED(jnl)) {
+        errno = ENOTCONN;
+        goto out;
+    }
+
+    hist_jnl = jnl->hist_jnl;
+    if (!hist_jnl)
+        goto out;
+
+retry:
+    if (is_last_scan == 1)
+        return 0;
+    if (hist_jnl->hist_done == 0)
+        is_last_scan = 1;
 
-        jnl = (gf_changelog_journal_t *) GF_CHANGELOG_GET_API_PTR (this);
-        if (!jnl)
-                goto out;
-        if (JNL_IS_API_DISCONNECTED (jnl)) {
-                errno = ENOTCONN;
-                goto out;
-        }
+    errno = EINVAL;
+    if (hist_jnl->hist_done == -1)
+        goto out;
 
-        hist_jnl = jnl->hist_jnl;
-        if (!hist_jnl)
-                goto out;
+    tracker_fd = hist_jnl->jnl_fd;
 
- retry:
-        if (is_last_scan == 1)
-                return 0;
-        if (hist_jnl->hist_done == 0)
-                is_last_scan = 1;
+    if (gf_ftruncate(tracker_fd, 0))
+        goto out;
 
-        errno = EINVAL;
-        if (hist_jnl->hist_done == -1)
-                goto out;
+    rewinddir(hist_jnl->jnl_dir);
 
-        tracker_fd = hist_jnl->jnl_fd;
+    for (;;) {
+        errno = 0;
+        entry = sys_readdir(hist_jnl->jnl_dir, scratch);
+        if (!entry || errno != 0)
+            break;
 
-        if (gf_ftruncate (tracker_fd, 0))
-                goto out;
+        if (strcmp(basename(entry->d_name), ".") == 0 ||
+            strcmp(basename(entry->d_name), "..") == 0)
+            continue;
 
-        len = offsetof (struct dirent, d_name)
-                + pathconf (hist_jnl->jnl_processing_dir, _PC_NAME_MAX) + 1;
-        entryp = GF_CALLOC (1, len,
-                            gf_changelog_mt_libgfchangelog_dirent_t);
-        if (!entryp)
-                goto out;
+        nr_entries++;
+
+        GF_CHANGELOG_FILL_BUFFER(hist_jnl->jnl_processing_dir, buffer, off,
+                                 strlen(hist_jnl->jnl_processing_dir));
+        GF_CHANGELOG_FILL_BUFFER(entry->d_name, buffer, off,
+                                 strlen(entry->d_name));
+        GF_CHANGELOG_FILL_BUFFER("\n", buffer, off, 1);
 
-        rewinddir (hist_jnl->jnl_dir);
-        while (1) {
-                ret = readdir_r (hist_jnl->jnl_dir, entryp, &result);
-                if (ret || !result)
-                        break;
-
-                if ( !strcmp (basename (entryp->d_name), ".")
-                     || !strcmp (basename (entryp->d_name), "..") )
-                        continue;
-
-                nr_entries++;
-
-                GF_CHANGELOG_FILL_BUFFER (hist_jnl->jnl_processing_dir,
-                                          buffer, off,
-                                          strlen (hist_jnl->jnl_processing_dir));
-                GF_CHANGELOG_FILL_BUFFER (entryp->d_name, buffer,
-                                          off, strlen (entryp->d_name));
-                GF_CHANGELOG_FILL_BUFFER ("\n", buffer, off, 1);
-
-                if (gf_changelog_write (tracker_fd, buffer, off) != off) {
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "error writing changelog filename"
-                                " to tracker file");
-                        break;
-                }
-                off = 0;
+        if (gf_changelog_write(tracker_fd, buffer, off) != off) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, CHANGELOG_LIB_MSG_WRITE_FAILED,
+                   "error writing changelog filename"
+                   " to tracker file");
+            break;
         }
+        off = 0;
+    }
 
-        GF_FREE (entryp);
-
-        gf_log (this->name, GF_LOG_DEBUG,
-                "hist_done %d, is_last_scan: %d", hist_jnl->hist_done,
-                is_last_scan);
-
-        if (!result) {
-                if (gf_lseek (tracker_fd, 0, SEEK_SET) != -1) {
-                        if (nr_entries > 0)
-                                return nr_entries;
-                        else {
-                                sleep(1);
-                                goto retry;
-                        }
-                }
+    gf_msg_debug(this->name, 0, "hist_done %d, is_last_scan: %d",
+                 hist_jnl->hist_done, is_last_scan);
+
+    if (!entry) {
+        if (gf_lseek(tracker_fd, 0, SEEK_SET) != -1) {
+            if (nr_entries > 0)
+                return nr_entries;
+            else {
+                sleep(1);
+                goto retry;
+            }
         }
- out:
-        return -1;
+    }
+out:
+    return -1;
 }
 
 /*
@@ -316,35 +315,36 @@ gf_history_changelog_scan ()
  * Returns 0 on success(updates given time-stamp), -1 on failure.
  */
 int
-gf_history_get_timestamp (int fd, int index, int len,
-                          unsigned long *ts)
+gf_history_get_timestamp(int fd, int index, int len, unsigned long *ts)
 {
-        xlator_t        *this             = NULL;
-        int             n_read            = -1;
-        char            path_buf[PATH_MAX]= {0,};
-        char            *iter             = path_buf;
-        size_t          offset            = index * (len+1);
-        unsigned long   value             = 0;
-        int             ret               = 0;
-
-        this = THIS;
-        if (!this) {
-                return -1;
-        }
-
-        n_read = pread (fd, path_buf, len, offset);
-        if (n_read < 0 ) {
-                ret = -1;
-                gf_log ( this->name, GF_LOG_ERROR,
-                         "could not read from htime file");
-                goto out;
-        }
-        iter+= len - TIMESTAMP_LENGTH;
-        sscanf (iter, "%lu",&value);
+    xlator_t *this = NULL;
+    int n_read = -1;
+    char path_buf[PATH_MAX] = {
+        0,
+    };
+    char *iter = path_buf;
+    size_t offset = index * (len + 1);
+    unsigned long value = 0;
+    int ret = 0;
+
+    this = THIS;
+    if (!this) {
+        return -1;
+    }
+
+    n_read = sys_pread(fd, path_buf, len, offset);
+    if (n_read < 0) {
+        ret = -1;
+        gf_msg(this->name, GF_LOG_ERROR, errno, CHANGELOG_LIB_MSG_READ_ERROR,
+               "could not read from htime file");
+        goto out;
+    }
+    iter += len - TIMESTAMP_LENGTH;
+    sscanf(iter, "%lu", &value);
 out:
-        if(ret == 0)
-                *ts = value;
-        return ret;
+    if (ret == 0)
+        *ts = value;
+    return ret;
 }
 
 /*
@@ -352,38 +352,37 @@ out:
  * Checks whether @value is there next to @target_index or not
  */
 int
-gf_history_check ( int fd, int target_index, unsigned long value, int len)
+gf_history_check(int fd, int target_index, unsigned long value, int len)
 {
-        int             ret = 0;
-        unsigned long   ts1 = 0;
-        unsigned long   ts2 = 0;
-
-        if (target_index == 0) {
-                ret = gf_history_get_timestamp (fd, target_index, len, &ts1);
-                if (ret == -1)
-                        goto out;
-                if (value <= ts1)
-                        goto out;
-                else {
-                        ret = -1;
-                        goto out;
-                }
-        }
+    int ret = 0;
+    unsigned long ts1 = 0;
+    unsigned long ts2 = 0;
 
-        ret = gf_history_get_timestamp (fd, target_index, len, &ts1);
-        if (ret ==-1)
-                goto out;
-        ret = gf_history_get_timestamp (fd, target_index -1, len, &ts2);
-        if (ret ==-1)
-                goto out;
-
-        if ( (value <= ts1) && (value > ts2) ) {
-                goto out;
-        }
-        else
-                ret = -1;
+    if (target_index == 0) {
+        ret = gf_history_get_timestamp(fd, target_index, len, &ts1);
+        if (ret == -1)
+            goto out;
+        if (value <= ts1)
+            goto out;
+        else {
+            ret = -1;
+            goto out;
+        }
+    }
+
+    ret = gf_history_get_timestamp(fd, target_index, len, &ts1);
+    if (ret == -1)
+        goto out;
+    ret = gf_history_get_timestamp(fd, target_index - 1, len, &ts2);
+    if (ret == -1)
+        goto out;
+
+    if ((value <= ts1) && (value > ts2)) {
+        goto out;
+    } else
+        ret = -1;
 out:
-        return ret;
+    return ret;
 }
 
 /*
@@ -403,115 +402,134 @@ out:
  */
 
 int
-gf_history_b_search (int fd, unsigned long value,
-                     unsigned long from, unsigned long to, int len)
+gf_history_b_search(int fd, unsigned long value, unsigned long from,
+                    unsigned long to, int len)
 {
-        int             m_index   = -1;
-        unsigned long   cur_value = 0;
-        unsigned long   ts1       = 0;
-        int             ret       = 0;
-
-        m_index = (from + to)/2;
-
-        if ( (to - from) <=1 ) {
-                /* either one or 2 changelogs left */
-                if ( to != from ) {
-                        /* check if value is less or greater than to
-                         * return accordingly
-                         */
-                        ret = gf_history_get_timestamp (fd, from, len, &ts1);
-                        if (ret ==-1)
-                                goto out;
-                        if ( ts1 >= value) {
-                                /* actually compatision should be
-                                 * exactly == but considering
-                                 *
-                                 * case of only 2 changelogs in htime file
-                                 */
-                                return from;
-                        }
-                        else
-                                return to;
-                }
-                else
-                        return to;
-        }
-
-        ret = gf_history_get_timestamp (fd, m_index, len, &cur_value);
+    int m_index = -1;
+    unsigned long cur_value = 0;
+    unsigned long ts1 = 0;
+    int ret = 0;
+
+    m_index = (from + to) / 2;
+
+    if ((to - from) <= 1) {
+        /* either one or 2 changelogs left */
+        if (to != from) {
+            /* check if value is less or greater than to
+             * return accordingly
+             */
+            ret = gf_history_get_timestamp(fd, from, len, &ts1);
+            if (ret == -1)
+                goto out;
+            if (ts1 >= value) {
+                /* actually compatision should be
+                 * exactly == but considering
+                 *
+                 * case of only 2 changelogs in htime file
+                 */
+                return from;
+            } else
+                return to;
+        } else
+            return to;
+    }
+
+    ret = gf_history_get_timestamp(fd, m_index, len, &cur_value);
+    if (ret == -1)
+        goto out;
+    if (cur_value == value) {
+        return m_index;
+    } else if (value > cur_value) {
+        ret = gf_history_get_timestamp(fd, m_index + 1, len, &cur_value);
         if (ret == -1)
-                goto out;
-        if (cur_value == value) {
+            goto out;
+        if (value < cur_value)
+            return m_index + 1;
+        else
+            return gf_history_b_search(fd, value, m_index + 1, to, len);
+    } else {
+        if (m_index == 0) {
+            /*  we are sure that values exists
+             *  in this htime file
+             */
+            return 0;
+        } else {
+            ret = gf_history_get_timestamp(fd, m_index - 1, len, &cur_value);
+            if (ret == -1)
+                goto out;
+            if (value > cur_value) {
                 return m_index;
+            } else
+                return gf_history_b_search(fd, value, from, m_index - 1, len);
         }
-        else if (value > cur_value) {
-                ret = gf_history_get_timestamp (fd, m_index+1, len, &cur_value);
-                if (ret == -1)
-                        goto out;
-                if (value < cur_value)
-                        return m_index + 1;
-                else
-                        return gf_history_b_search (fd, value,
-                                                    m_index+1, to, len);
-        }
-        else {
-                if (m_index ==0) {
-                       /*  we are sure that values exists
-                        *  in this htime file
-                        */
-                        return 0;
-                }
-                else {
-                        ret = gf_history_get_timestamp (fd, m_index-1, len,
-                                                        &cur_value);
-                        if (ret == -1)
-                                goto out;
-                        if (value > cur_value) {
-                                return m_index;
-                        }
-                        else
-                                return gf_history_b_search (fd, value, from,
-                                                            m_index-1, len);
-                }
-        }
+    }
 out:
-        return -1;
+    return -1;
 }
 
-void *
-gf_changelog_consume_wrap (void* data)
+/*
+ * Description: Checks if the changelog path is usable or not,
+ *              which is differentiated by checking for "changelog"
+ *              in the path and not "CHANGELOG".
+ *
+ * Returns:
+ * 1 : Yes, usable ( contains "CHANGELOG" )
+ * 0 : No, Not usable ( contains, "changelog")
+ */
+int
+gf_is_changelog_usable(char *cl_path)
 {
-        int                          ret   = -1;
-        ssize_t                      nread = 0;
-        xlator_t                    *this  = NULL;
-        gf_changelog_consume_data_t *ccd   = NULL;
+    int ret = -1;
+    const char low_c[] = "changelog";
+    char *str_ret = NULL;
+    char *bname = NULL;
 
-        ccd = (gf_changelog_consume_data_t *) data;
-        this = ccd->this;
+    bname = basename(cl_path);
 
-        ccd->retval = -1;
+    str_ret = strstr(bname, low_c);
 
-        nread = pread (ccd->fd, ccd->changelog, PATH_MAX, ccd->offset);
-        if (nread < 0) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "cannot read from history metadata file (reason %s)",
-                        strerror (errno));
-                goto out;
-        }
+    if (str_ret != NULL)
+        ret = 0;
+    else
+        ret = 1;
 
-        /* TODO: handle short reads and EOF. */
+    return ret;
+}
 
-        ret = gf_changelog_consume (ccd->this,
-                                    ccd->jnl, ccd->changelog, _gf_true);
+void *
+gf_changelog_consume_wrap(void *data)
+{
+    int ret = -1;
+    ssize_t nread = 0;
+    xlator_t *this = NULL;
+    gf_changelog_consume_data_t *ccd = NULL;
+
+    ccd = (gf_changelog_consume_data_t *)data;
+    this = ccd->this;
+
+    ccd->retval = -1;
+
+    nread = sys_pread(ccd->fd, ccd->changelog, PATH_MAX - 1, ccd->offset);
+    if (nread < 0) {
+        gf_msg(this->name, GF_LOG_ERROR, errno, CHANGELOG_LIB_MSG_READ_ERROR,
+               "cannot read from history metadata file");
+        goto out;
+    }
+
+    /* TODO: handle short reads and EOF. */
+    if (gf_is_changelog_usable(ccd->changelog) == 1) {
+        ret = gf_changelog_consume(ccd->this, ccd->jnl, ccd->changelog,
+                                   _gf_true);
         if (ret) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "could not parse changelog: %s", ccd->changelog);
-                goto out;
+            gf_smsg(this->name, GF_LOG_ERROR, 0, CHANGELOG_LIB_MSG_PARSE_ERROR,
+                    "name=%s", ccd->changelog, NULL);
+            goto out;
         }
+    }
+    ccd->retval = 0;
 
-        ccd->retval = 0;
-
- out:
-        return NULL;
+out:
+    return NULL;
 }
 
 /**
@@ -520,128 +538,132 @@ gf_changelog_consume_wrap (void* data)
  * to index "to" in open htime file whose fd is "fd".
  */
 
-#define MAX_PARALLELS  10
+#define MAX_PARALLELS 10
 
 void *
-gf_history_consume (void * data)
+gf_history_consume(void *data)
 {
-        xlator_t                    *this              = NULL;
-        gf_changelog_journal_t              *jnl               = NULL;
-        gf_changelog_journal_t              *hist_jnl          = NULL;
-        int                          ret               = 0;
-        int                          iter              = 0;
-        int                          fd                = -1;
-        int                          from              = -1;
-        int                          to                = -1;
-        int                          len               = -1;
-        int                          n_parallel        = 0;
-        int                          n_envoked         = 0;
-        gf_boolean_t                 publish           = _gf_true;
-        pthread_t th_id[MAX_PARALLELS]                 = {0,};
-        gf_changelog_history_data_t *hist_data         = NULL;
-        gf_changelog_consume_data_t ccd[MAX_PARALLELS] = {{0},};
-        gf_changelog_consume_data_t *curr              = NULL;
-
-        hist_data = (gf_changelog_history_data_t *) data;
-        if (hist_data == NULL) {
-                ret = -1;
-                goto out;
-        }
-
-        fd         = hist_data->htime_fd;
-        from       = hist_data->from;
-        to         = hist_data->to;
-        len        = hist_data->len;
-        n_parallel = hist_data->n_parallel;
-
-        THIS = hist_data->this;
-        this = hist_data->this;
-        if (!this) {
-                ret = -1;
-                goto out;
-        }
-
-        jnl = (gf_changelog_journal_t *) GF_CHANGELOG_GET_API_PTR (this);
-        if (!jnl) {
-                ret = -1;
-                goto out;
-        }
-
-        hist_jnl = jnl->hist_jnl;
-        if (!hist_jnl) {
-                ret = -1;
-                goto out;
-        }
-
-        while (from <= to) {
-                n_envoked = 0;
-
-                for (iter = 0 ; (iter < n_parallel) && (from <= to); iter++) {
-                        curr = &ccd[iter];
-
-                        curr->this   = this;
-                        curr->jnl    = hist_jnl;
-                        curr->fd     = fd;
-                        curr->offset = from * (len + 1);
-
-                        curr->retval = 0;
-                        memset (curr->changelog, '\0', PATH_MAX);
-
-                        ret = pthread_create (&th_id[iter], NULL,
-                                              gf_changelog_consume_wrap, curr);
-                        if (ret) {
-                                gf_log ( this->name, GF_LOG_ERROR,
-                                        "could not create consume-thread"
-                                        " reason (%s)", strerror (ret));
-                                ret = -1;
-                                goto sync;
-                        } else
-                                n_envoked++;
-
-                        from++;
-                }
-
-        sync:
-                for (iter = 0; iter < n_envoked; iter++) {
-                        ret = pthread_join (th_id[iter], NULL);
-                        if (ret) {
-                                publish = _gf_false;
-                                gf_log (this->name, GF_LOG_ERROR,
-                                        "pthread_join() error %s",
-                                        strerror (ret));
-                                /* try to join the rest */
-                                continue;
-                        }
-
-                        if (publish == _gf_false)
-                                continue;
-
-                        curr = &ccd[iter];
-                        if (ccd->retval) {
-                                publish = _gf_false;
-                                gf_log (this->name, GF_LOG_ERROR,
-                                        "parsing error, ceased publishing...");
-                                continue;
-                        }
-
-                        ret = gf_changelog_publish (curr->this,
-                                                    curr->jnl, curr->changelog);
-                        if (ret) {
-                                publish = _gf_false;
-                                gf_log (this->name, GF_LOG_ERROR,
-                                        "publish error, ceased publishing...");
-                        }
-                }
-        }
-
-       /* informing "parsing done". */
-        hist_jnl->hist_done = (publish == _gf_true) ? 0 : -1;
+    xlator_t *this = NULL;
+    gf_changelog_journal_t *jnl = NULL;
+    gf_changelog_journal_t *hist_jnl = NULL;
+    int ret = 0;
+    int iter = 0;
+    int fd = -1;
+    int from = -1;
+    int to = -1;
+    int len = -1;
+    int n_parallel = 0;
+    int n_envoked = 0;
+    gf_boolean_t publish = _gf_true;
+    pthread_t th_id[MAX_PARALLELS] = {
+        0,
+    };
+    gf_changelog_history_data_t *hist_data = NULL;
+    gf_changelog_consume_data_t ccd[MAX_PARALLELS] = {
+        {0},
+    };
+    gf_changelog_consume_data_t *curr = NULL;
+
+    hist_data = (gf_changelog_history_data_t *)data;
+    if (hist_data == NULL) {
+        ret = -1;
+        goto out;
+    }
+
+    fd = hist_data->htime_fd;
+    from = hist_data->from;
+    to = hist_data->to;
+    len = hist_data->len;
+    n_parallel = hist_data->n_parallel;
+
+    THIS = hist_data->this;
+    this = hist_data->this;
+    if (!this) {
+        ret = -1;
+        goto out;
+    }
+
+    jnl = (gf_changelog_journal_t *)GF_CHANGELOG_GET_API_PTR(this);
+    if (!jnl) {
+        ret = -1;
+        goto out;
+    }
+
+    hist_jnl = jnl->hist_jnl;
+    if (!hist_jnl) {
+        ret = -1;
+        goto out;
+    }
+
+    while (from <= to) {
+        n_envoked = 0;
+
+        for (iter = 0; (iter < n_parallel) && (from <= to); iter++) {
+            curr = &ccd[iter];
+
+            curr->this = this;
+            curr->jnl = hist_jnl;
+            curr->fd = fd;
+            curr->offset = from * (len + 1);
+
+            curr->retval = 0;
+            memset(curr->changelog, '\0', PATH_MAX);
+
+            ret = gf_thread_create(&th_id[iter], NULL,
+                                   gf_changelog_consume_wrap, curr,
+                                   "clogc%03hx", (iter + 1) & 0x3ff);
+            if (ret) {
+                gf_msg(this->name, GF_LOG_ERROR, ret,
+                       CHANGELOG_LIB_MSG_THREAD_CREATION_FAILED,
+                       "could not create consume-thread");
+                goto sync;
+            } else
+                n_envoked++;
+
+            from++;
+        }
+
+    sync:
+        for (iter = 0; iter < n_envoked; iter++) {
+            ret = pthread_join(th_id[iter], NULL);
+            if (ret) {
+                publish = _gf_false;
+                gf_msg(this->name, GF_LOG_ERROR, ret,
+                       CHANGELOG_LIB_MSG_PTHREAD_JOIN_FAILED,
+                       "pthread_join() error");
+                /* try to join the rest */
+                continue;
+            }
+
+            if (publish == _gf_false)
+                continue;
+
+            curr = &ccd[iter];
+            if (ccd->retval) {
+                publish = _gf_false;
+                gf_smsg(this->name, GF_LOG_ERROR, 0,
+                        CHANGELOG_LIB_MSG_PARSE_ERROR_CEASED, NULL);
+                continue;
+            }
+
+            ret = gf_changelog_publish(curr->this, curr->jnl, curr->changelog);
+            if (ret) {
+                publish = _gf_false;
+                gf_msg(this->name, GF_LOG_ERROR, 0,
+                       CHANGELOG_LIB_MSG_PUBLISH_ERROR,
+                       "publish error, ceased publishing...");
+            }
+        }
+    }
+
+    /* informing "parsing done". */
+    hist_jnl->hist_done = (publish == _gf_true) ? 0 : -1;
 
 out:
-        if (fd != -1)
-                close (fd);
-        GF_FREE (hist_data);
-        return NULL;
+    if (fd != -1)
+        (void)sys_close(fd);
+    GF_FREE(hist_data);
+    return NULL;
 }
 
 /**
@@ -671,267 +693,328 @@ out:
  *   -2 : Ignore this metadata file and process next
  */
 int
-gf_changelog_extract_min_max (const char *dname, const char *htime_dir,
-                              int *fd, unsigned long *total,
-                              unsigned long *min_ts, unsigned long *max_ts)
+gf_changelog_extract_min_max(const char *dname, const char *htime_dir, int *fd,
+                             unsigned long *total, unsigned long *min_ts,
+                             unsigned long *max_ts)
 {
-        int          ret          = -1;
-        xlator_t    *this         = NULL;
-        char htime_file[PATH_MAX] = {0,};
-        struct stat  stbuf        = {0,};
-        char        *iter         = NULL;
-        char x_value[30]          = {0,};
-
-        this = THIS;
-
-        snprintf (htime_file, PATH_MAX, "%s/%s", htime_dir, dname);
+    int ret = -1;
+    xlator_t *this = NULL;
+    char htime_file[PATH_MAX] = {
+        0,
+    };
+    struct stat stbuf = {
+        0,
+    };
+    char *iter = NULL;
+    char x_value[30] = {
+        0,
+    };
+
+    this = THIS;
+
+    snprintf(htime_file, PATH_MAX, "%s/%s", htime_dir, dname);
+
+    iter = (htime_file + strlen(htime_file) - TIMESTAMP_LENGTH);
+    sscanf(iter, "%lu", min_ts);
+
+    ret = sys_stat(htime_file, &stbuf);
+    if (ret) {
+        ret = -1;
+        gf_smsg(this->name, GF_LOG_ERROR, errno, CHANGELOG_LIB_MSG_HTIME_ERROR,
+                "op=stat", "path=%s", htime_file, NULL);
+        goto out;
+    }
+
+    /* ignore everything except regular files */
+    if (!S_ISREG(stbuf.st_mode)) {
+        ret = -2;
+        goto out;
+    }
+
+    *fd = open(htime_file, O_RDONLY);
+    if (*fd < 0) {
+        ret = -1;
+        gf_smsg(this->name, GF_LOG_ERROR, errno, CHANGELOG_LIB_MSG_HTIME_ERROR,
+                "op=open", "path=%s", htime_file, NULL);
+        goto out;
+    }
+
+    /* Looks good, extract max timestamp */
+    ret = sys_fgetxattr(*fd, HTIME_KEY, x_value, sizeof(x_value));
+    if (ret < 0) {
+        ret = -1;
+        gf_smsg(this->name, GF_LOG_ERROR, errno,
+                CHANGELOG_LIB_MSG_GET_XATTR_FAILED, "path=%s", htime_file,
+                NULL);
+        goto out;
+    }
+
+    sscanf(x_value, "%lu:%lu", max_ts, total);
+    gf_smsg(this->name, GF_LOG_INFO, 0, CHANGELOG_LIB_MSG_MIN_MAX_INFO,
+            "min=%lu", *min_ts, "max=%lu", *max_ts, "total_changelogs=%lu",
+            *total, NULL);
+
+    ret = 0;
 
-        iter = (htime_file + strlen (htime_file) - TIMESTAMP_LENGTH);
-        sscanf (iter ,"%lu",min_ts);
+out:
+    return ret;
+}
 
-        ret = stat (htime_file, &stbuf);
+/* gf_history_changelog returns actual_end and spawns threads to
+ * parse historical changelogs. The return values are as follows.
+ *     0 : On success
+ *     1 : Successful, but partial historical changelogs available,
+ *         end time falls into different htime file or future time
+ *    -2 : Error, requested historical changelog not available, not
+ *         even partial
+ *    -1 : On any error
+ */
+int
+gf_history_changelog(char *changelog_dir, unsigned long start,
+                     unsigned long end, int n_parallel,
+                     unsigned long *actual_end)
+{
+    int ret = 0;
+    int len = -1;
+    int fd = -1;
+    int n_read = -1;
+    unsigned long min_ts = 0;
+    unsigned long max_ts = 0;
+    unsigned long end2 = 0;
+    unsigned long ts1 = 0;
+    unsigned long ts2 = 0;
+    unsigned long to = 0;
+    unsigned long from = 0;
+    unsigned long total_changelog = 0;
+    xlator_t *this = NULL;
+    gf_changelog_journal_t *jnl = NULL;
+    gf_changelog_journal_t *hist_jnl = NULL;
+    gf_changelog_history_data_t *hist_data = NULL;
+    DIR *dirp = NULL;
+    struct dirent *entry = NULL;
+    struct dirent scratch[2] = {
+        {
+            0,
+        },
+    };
+    pthread_t consume_th = 0;
+    char htime_dir[PATH_MAX] = {
+        0,
+    };
+    char buffer[PATH_MAX] = {
+        0,
+    };
+    gf_boolean_t partial_history = _gf_false;
+
+    pthread_attr_t attr;
+
+    this = THIS;
+    if (!this) {
+        ret = -1;
+        goto out;
+    }
+
+    ret = pthread_attr_init(&attr);
+    if (ret != 0) {
+        gf_msg(this->name, GF_LOG_ERROR, errno, CHANGELOG_LIB_MSG_PTHREAD_ERROR,
+               "Pthread init failed");
+        return -1;
+    }
+
+    jnl = (gf_changelog_journal_t *)GF_CHANGELOG_GET_API_PTR(this);
+    if (!jnl) {
+        ret = -1;
+        goto out;
+    }
+
+    hist_jnl = (gf_changelog_journal_t *)jnl->hist_jnl;
+    if (!hist_jnl) {
+        ret = -1;
+        goto out;
+    }
+
+    gf_smsg(this->name, GF_LOG_INFO, 0, CHANGELOG_LIB_MSG_REQUESTING_INFO,
+            "start=%lu", start, "end=%lu", end, NULL);
+
+    /* basic sanity check */
+    if (start > end || n_parallel <= 0) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, CHANGELOG_LIB_MSG_HIST_FAILED,
+                "start=%lu", start, "end=%lu", end, "thread_count=%d",
+                n_parallel, NULL);
+        ret = -1;
+        goto out;
+    }
+
+    /* cap parallelism count */
+    if (n_parallel > MAX_PARALLELS)
+        n_parallel = MAX_PARALLELS;
+
+    CHANGELOG_FILL_HTIME_DIR(changelog_dir, htime_dir);
+
+    dirp = sys_opendir(htime_dir);
+    if (dirp == NULL) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, CHANGELOG_LIB_MSG_HTIME_ERROR,
+                "op=opendir", "path=%s", htime_dir, NULL);
+        ret = -1;
+        goto out;
+    }
+
+    for (;;) {
+        errno = 0;
+
+        entry = sys_readdir(dirp, scratch);
+
+        if (!entry || errno != 0) {
+            gf_smsg(this->name, GF_LOG_ERROR, errno,
+                    CHANGELOG_LIB_MSG_HIST_FAILED, "start=%lu", start,
+                    "end=%lu", end, NULL);
+            ret = -2;
+            break;
+        }
+
+        ret = gf_changelog_extract_min_max(entry->d_name, htime_dir, &fd,
+                                           &total_changelog, &min_ts, &max_ts);
         if (ret) {
-                ret = -1;
-                gf_log (this->name, GF_LOG_ERROR,
-                        "stat() failed on htime file %s (reason %s)",
-                        htime_file, strerror (errno));
-                goto out;
-        }
-
-        /* ignore everything except regular files */
-        if (!S_ISREG (stbuf.st_mode)) {
-                ret = -2;
-                goto out;
-        }
-
-        *fd = open (htime_file, O_RDONLY);
-        if (*fd < 0) {
-                ret = -1;
-                gf_log (this->name, GF_LOG_ERROR,
-                        "open() failed for htime %s (reasong %s)",
-                        htime_file, strerror (errno));
-                goto out;
+            if (-2 == ret)
+                continue;
+            goto out;
         }
 
-        /* Looks good, extract max timestamp */
-        ret = sys_fgetxattr (*fd, HTIME_KEY, x_value, sizeof (x_value));
-        if (ret < 0) {
+        if (start >= min_ts && start < max_ts) {
+            /**
+             * TODO: handle short reads later...
+             */
+            n_read = sys_read(fd, buffer, PATH_MAX);
+            if (n_read < 0) {
                 ret = -1;
-                gf_log (this->name, GF_LOG_ERROR,
-                        "error extracting max timstamp from htime file"
-                        " %s (reason %s)", htime_file, strerror (errno));
+                gf_msg(this->name, GF_LOG_ERROR, errno,
+                       CHANGELOG_LIB_MSG_READ_ERROR,
+                       "unable to read htime file");
                 goto out;
-        }
-
-        sscanf (x_value, "%lu:%lu", max_ts, total);
-        gf_log (this->name, GF_LOG_INFO,
-                "MIN: %lu, MAX: %lu, TOTAL CHANGELOGS: %lu",
-                *min_ts, *max_ts, *total);
-
-        ret = 0;
+            }
 
- out:
-        return ret;
-}
+            len = strlen(buffer);
 
-int
-gf_history_changelog (char* changelog_dir, unsigned long start,
-                      unsigned long end, int n_parallel,
-                      unsigned long *actual_end)
-{
-        int                             ret                     = 0;
-        int                             len                     = -1;
-        int                             fd                      = -1;
-        int                             n_read                  = -1;
-        unsigned long                   min_ts                  = 0;
-        unsigned long                   max_ts                  = 0;
-        unsigned long                   end2                    = 0;
-        unsigned long                   ts1                     = 0;
-        unsigned long                   ts2                     = 0;
-        unsigned long                   to                      = 0;
-        unsigned long                   from                    = 0;
-        unsigned long                   total_changelog         = 0;
-        xlator_t                        *this                   = NULL;
-        gf_changelog_journal_t                  *jnl                    = NULL;
-        gf_changelog_journal_t                  *hist_jnl               = NULL;
-        gf_changelog_history_data_t     *hist_data              = NULL;
-        DIR                             *dirp                   = NULL;
-        struct dirent                   *dp                     = NULL;
-        pthread_t                       consume_th              = 0;
-        char                            htime_dir[PATH_MAX]     = {0,};
-        char                            buffer[PATH_MAX]        = {0,};
-
-        pthread_attr_t attr;
-
-        ret = pthread_attr_init (&attr);
-        if (ret != 0) {
-                return -1;
-        }
+            /**
+             * search @start in the htime file returning it's index
+             * (@from)
+             */
+            from = gf_history_b_search(fd, start, 0, total_changelog - 1, len);
 
-        this = THIS;
-        if (!this) {
+            /* ensuring correctness of gf_b_search */
+            if (gf_history_check(fd, from, start, len) != 0) {
                 ret = -1;
-                goto out;
-        }
-
-        jnl = (gf_changelog_journal_t *) GF_CHANGELOG_GET_API_PTR (this);
-        if (!jnl) {
+                gf_smsg(this->name, GF_LOG_ERROR, 0,
+                        CHANGELOG_LIB_MSG_GET_TIME_ERROR, "for=start",
+                        "start=%lu", start, "idx=%lu", from, NULL);
+                goto out;
+            }
+
+            end2 = (end <= max_ts) ? end : max_ts;
+
+            /* Check if end falls out of same HTIME file. The end
+             * falling to a different htime file or changelog
+             * disable-enable is detected only after 20 seconds.
+             * This is required because, applications generally
+             * asks historical changelogs till current time and
+             * it is possible changelog is not rolled over yet.
+             * So, buffer time of default rollover time plus 5
+             * seconds is subtracted.  If the application requests
+             * the end time with in half a minute of changelog
+             * disable, it's not detected as changelog disable and
+             * it's application's responsibility to retry after
+             * 20 seconds before confirming it as partial history.
+             */
+            if ((end - 20) > max_ts) {
+                partial_history = _gf_true;
+            }
+
+            /**
+             * search @end2 in htime file returning it's index (@to)
+             */
+            to = gf_history_b_search(fd, end2, 0, total_changelog - 1, len);
+
+            if (gf_history_check(fd, to, end2, len) != 0) {
                 ret = -1;
+                gf_smsg(this->name, GF_LOG_ERROR, 0,
+                        CHANGELOG_LIB_MSG_GET_TIME_ERROR, "for=end",
+                        "start=%lu", end2, "idx=%lu", to, NULL);
                 goto out;
-        }
+            }
 
-        hist_jnl = (gf_changelog_journal_t *) jnl->hist_jnl;
-        if (!hist_jnl) {
-                ret = -1;
+            ret = gf_history_get_timestamp(fd, from, len, &ts1);
+            if (ret == -1)
                 goto out;
-        }
 
-        /* basic sanity check */
-        if (start > end || n_parallel <= 0) {
-                ret = -1;
+            ret = gf_history_get_timestamp(fd, to, len, &ts2);
+            if (ret == -1)
                 goto out;
-        }
 
-        /* cap parallelism count */
-        if (n_parallel > MAX_PARALLELS)
-                n_parallel = MAX_PARALLELS;
+            gf_smsg(this->name, GF_LOG_INFO, 0, CHANGELOG_LIB_MSG_FINAL_INFO,
+                    "from=%lu", ts1, "to=%lu", ts2, "changes=%lu",
+                    (to - from + 1), NULL);
 
-        CHANGELOG_FILL_HTIME_DIR (changelog_dir, htime_dir);
+            hist_data = GF_CALLOC(1, sizeof(gf_changelog_history_data_t),
+                                  gf_changelog_mt_history_data_t);
 
-        dirp = opendir (htime_dir);
-        if (dirp == NULL) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "open dir on htime failed : %s (reason: %s)",
-                        htime_dir, strerror (errno));
+            hist_data->htime_fd = fd;
+            hist_data->from = from;
+            hist_data->to = to;
+            hist_data->len = len;
+            hist_data->n_parallel = n_parallel;
+            hist_data->this = this;
+
+            ret = pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_DETACHED);
+            if (ret != 0) {
+                gf_msg(this->name, GF_LOG_ERROR, ret,
+                       CHANGELOG_LIB_MSG_PTHREAD_ERROR,
+                       "unable to sets the detach"
+                       " state attribute");
                 ret = -1;
                 goto out;
-        }
+            }
 
-        while ((dp = readdir (dirp)) != NULL) {
-                ret = gf_changelog_extract_min_max (dp->d_name, htime_dir,
-                                                    &fd, &total_changelog,
-                                                    &min_ts, &max_ts);
-                if (ret) {
-                        if (-2 == ret)
-                                continue;
-                        goto out;
-                }
-
-                if (start >= min_ts && start < max_ts) {
-                        /**
-                         * TODO: handle short reads later...
-                         */
-                        n_read = read (fd, buffer, PATH_MAX);
-                        if (n_read < 0) {
-                                ret = -1;
-                                gf_log ( this->name, GF_LOG_ERROR,
-                                        "unable to read htime file");
-                                goto out;
-                        }
-
-                        len = strlen (buffer);
-
-                        /**
-                         * search @start in the htime file returning it's index
-                         * (@from)
-                         */
-                        from = gf_history_b_search (fd, start, 0,
-                                                   total_changelog - 1, len);
-
-                        /* ensuring correctness of gf_b_search */
-                        if (gf_history_check (fd, from, start, len) != 0) {
-                                ret = -1;
-                                gf_log (this->name, GF_LOG_ERROR,
-                                        "wrong result for start: %lu idx: %lu",
-                                        start, from);
-                                goto out;
-                        }
-
-                        end2 = (end <= max_ts) ? end : max_ts;
-
-                        /**
-                         * search @end2 in htime file returning it's index (@to)
-                         */
-                        to = gf_history_b_search (fd, end2,
-                                                  0, total_changelog - 1, len);
-
-                        if (gf_history_check (fd, to, end2, len) != 0) {
-                                ret = -1;
-                                gf_log (this->name, GF_LOG_ERROR,
-                                        "wrong result for start: %lu idx: %lu",
-                                        end2, to);
-                                goto out;
-                        }
-
-                        ret = gf_history_get_timestamp (fd, from, len, &ts1);
-                        if (ret == -1)
-                                goto out;
-
-                        ret = gf_history_get_timestamp (fd, to, len, &ts2);
-                        if (ret == -1)
-                                goto out;
-
-                        gf_log (this->name, GF_LOG_INFO,
-                                "FINAL: from: %lu, to: %lu, changes: %lu",
-                                ts1, ts2, (to - from + 1));
-
-                        hist_data =  GF_CALLOC (1,
-                                     sizeof (gf_changelog_history_data_t),
-                                     gf_changelog_mt_history_data_t);
-
-                        hist_data->htime_fd   = fd;
-                        hist_data->from       = from;
-                        hist_data->to         = to;
-                        hist_data->len        = len;
-                        hist_data->n_parallel = n_parallel;
-                        hist_data->this       = this;
-
-                        ret = pthread_attr_setdetachstate
-                                (&attr, PTHREAD_CREATE_DETACHED);
-                        if (ret != 0) {
-                                gf_log (this->name, GF_LOG_ERROR,
-                                        "unable to sets the detach"
-                                        " state attribute, reason(%s)",
-                                        strerror (ret));
-                                ret = -1;
-                                goto out;
-                        }
-
-                        /* spawn a thread for background parsing & publishing */
-                        ret = pthread_create (&consume_th, &attr,
-                                              gf_history_consume, hist_data);
-                        if (ret) {
-                                gf_log (this->name, GF_LOG_ERROR,
-                                        "creation of consume parent-thread"
-                                        " failed. reason(%s)", strerror (ret));
-                                ret = -1;
-                                goto out;
-                        }
-
-                        goto out;
-
-                } /* end of range check */
-
-        } /* end of readdir() */
-
-        if (!from || !to)
+            /* spawn a thread for background parsing & publishing */
+            ret = gf_thread_create(&consume_th, &attr, gf_history_consume,
+                                   hist_data, "cloghcon");
+            if (ret) {
+                gf_msg(this->name, GF_LOG_ERROR, ret,
+                       CHANGELOG_LIB_MSG_THREAD_CREATION_FAILED,
+                       "creation of consume parent-thread"
+                       " failed.");
                 ret = -1;
+                goto out;
+            }
 
-out:
-        if (dirp != NULL)
-                closedir (dirp);
-
-        if (ret < 0) {
-                if (fd != -1)
-                        close (fd);
-                GF_FREE (hist_data);
-                (void) pthread_attr_destroy (&attr);
+            goto out;
 
-                return ret;
+        } else { /* end of range check */
+            gf_smsg(this->name, GF_LOG_ERROR, errno,
+                    CHANGELOG_LIB_MSG_HIST_FAILED, "start=%lu", start,
+                    "end=%lu", end, "chlog_min=%lu", min_ts, "chlog_max=%lu",
+                    max_ts, NULL);
         }
+    } /* end of readdir() */
 
-        hist_jnl->hist_done = 1;
-        *actual_end = ts2;
+out:
+    if (dirp != NULL)
+        (void)sys_closedir(dirp);
+
+    if (ret < 0) {
+        if (fd != -1)
+            (void)sys_close(fd);
+        GF_FREE(hist_data);
+        (void)pthread_attr_destroy(&attr);
 
         return ret;
+    }
+
+    hist_jnl->hist_done = 1;
+    *actual_end = ts2;
+
+    if (partial_history) {
+        ret = 1;
+    }
+
+    return ret;
 }
diff --git a/xlators/features/changelog/src/Makefile.am b/xlators/features/changelog/src/Makefile.am
index 8712b9d059f..eee7dfa238d 100644
--- a/xlators/features/changelog/src/Makefile.am
+++ b/xlators/features/changelog/src/Makefile.am
@@ -3,24 +3,26 @@ xlator_LTLIBRARIES = changelog.la
 xlatordir = $(libdir)/glusterfs/$(PACKAGE_VERSION)/xlator/features
 
 noinst_HEADERS = changelog-helpers.h changelog-mem-types.h changelog-rt.h \
-		 changelog-rpc-common.h changelog-misc.h changelog-encoders.h \
-		 changelog-rpc-common.h changelog-rpc.h changelog-ev-handle.h
+	changelog-rpc-common.h changelog-misc.h changelog-encoders.h \
+	changelog-rpc-common.h changelog-rpc.h changelog-ev-handle.h \
+	changelog-messages.h
 
-changelog_la_LDFLAGS = -module -avoid-version
+changelog_la_LDFLAGS = -module $(GF_XLATOR_DEFAULT_LDFLAGS)
 
 changelog_la_SOURCES = changelog.c changelog-rt.c changelog-helpers.c \
-		changelog-encoders.c changelog-rpc.c changelog-barrier.c \
-		changelog-rpc-common.c changelog-ev-handle.c
+	changelog-encoders.c changelog-rpc.c changelog-barrier.c \
+	changelog-rpc-common.c changelog-ev-handle.c
 changelog_la_LIBADD = $(top_builddir)/libglusterfs/src/libglusterfs.la \
-		      $(top_builddir)/rpc/xdr/src/libgfxdr.la \
-		      $(top_builddir)/rpc/rpc-lib/src/libgfrpc.la
+	$(top_builddir)/rpc/xdr/src/libgfxdr.la \
+	$(top_builddir)/rpc/rpc-lib/src/libgfrpc.la
 
 AM_CPPFLAGS = $(GF_CPPFLAGS) -I$(top_srcdir)/libglusterfs/src \
-	      -I$(top_srcdir)/rpc/xdr/src -I$(top_srcdir)/rpc/rpc-lib/src \
-	      -I$(top_srcdir)/rpc/rpc-transport/socket/src \
-	      -I$(top_srcdir)/xlators/features/changelog/lib/src/ \
-	      -fPIC -D_FILE_OFFSET_BITS=64 -D_GNU_SOURCE -D$(GF_HOST_OS) \
-	      -DDATADIR=\"$(localstatedir)\"
+	-I$(top_srcdir)/rpc/xdr/src -I$(top_builddir)/rpc/xdr/src \
+	-I$(top_srcdir)/rpc/rpc-lib/src \
+	-I$(top_srcdir)/rpc/rpc-transport/socket/src \
+	-I$(top_srcdir)/xlators/features/changelog/lib/src/ \
+	-fPIC -D_FILE_OFFSET_BITS=64 -D_GNU_SOURCE -D$(GF_HOST_OS) \
+	-DDATADIR=\"$(localstatedir)\"
 
 AM_CFLAGS = -Wall $(GF_CFLAGS)
 
diff --git a/xlators/features/changelog/src/changelog-barrier.c b/xlators/features/changelog/src/changelog-barrier.c
index 22cfa617e1b..0fb89ddb127 100644
--- a/xlators/features/changelog/src/changelog-barrier.c
+++ b/xlators/features/changelog/src/changelog-barrier.c
@@ -9,121 +9,123 @@
 */
 
 #include "changelog-helpers.h"
-#include "call-stub.h"
+#include "changelog-messages.h"
+#include <glusterfs/call-stub.h>
 
 /* Enqueue a stub*/
 void
-__chlog_barrier_enqueue (xlator_t *this, call_stub_t *stub)
+__chlog_barrier_enqueue(xlator_t *this, call_stub_t *stub)
 {
-        changelog_priv_t *priv    = NULL;
+    changelog_priv_t *priv = NULL;
 
-        priv = this->private;
-        GF_ASSERT (priv);
+    priv = this->private;
+    GF_ASSERT(priv);
 
-        list_add_tail (&stub->list, &priv->queue);
-        priv->queue_size++;
+    list_add_tail(&stub->list, &priv->queue);
+    priv->queue_size++;
 
-        return;
+    return;
 }
 
 /* Dequeue a stub */
 call_stub_t *
-__chlog_barrier_dequeue (xlator_t *this, struct list_head *queue)
+__chlog_barrier_dequeue(xlator_t *this, struct list_head *queue)
 {
-        call_stub_t      *stub            = NULL;
-        changelog_priv_t *priv            = NULL;
+    call_stub_t *stub = NULL;
+    changelog_priv_t *priv = NULL;
 
-        priv = this->private;
-        GF_ASSERT (priv);
+    priv = this->private;
+    GF_ASSERT(priv);
 
-        if (list_empty (queue))
-                goto out;
+    if (list_empty(queue))
+        goto out;
 
-        stub = list_entry (queue->next, call_stub_t, list);
-        list_del_init (&stub->list);
+    stub = list_entry(queue->next, call_stub_t, list);
+    list_del_init(&stub->list);
 
 out:
-        return stub;
+    return stub;
 }
 
 /* Dequeue all the stubs and call corresponding resume functions */
 void
-chlog_barrier_dequeue_all (xlator_t *this, struct list_head *queue)
+chlog_barrier_dequeue_all(xlator_t *this, struct list_head *queue)
 {
-        call_stub_t            *stub    = NULL;
+    call_stub_t *stub = NULL;
 
-        gf_log (this->name, GF_LOG_INFO,
-                "Dequeuing all the changelog barriered fops");
+    gf_smsg(this->name, GF_LOG_INFO, 0, CHANGELOG_MSG_DEQUEUING_BARRIER_FOPS,
+            NULL);
 
-        while ((stub = __chlog_barrier_dequeue (this, queue)))
-                call_resume (stub);
+    while ((stub = __chlog_barrier_dequeue(this, queue)))
+        call_resume(stub);
 
-        gf_log (this->name, GF_LOG_INFO,
-                "Dequeuing changelog barriered fops is finished");
-        return;
+    gf_smsg(this->name, GF_LOG_INFO, 0,
+            CHANGELOG_MSG_DEQUEUING_BARRIER_FOPS_FINISHED, NULL);
+    return;
 }
 
 /* Function called on changelog barrier timeout */
 void
-chlog_barrier_timeout (void *data)
+chlog_barrier_timeout(void *data)
 {
-        xlator_t               *this    = NULL;
-        changelog_priv_t       *priv    = NULL;
-        struct list_head        queue   = {0,};
+    xlator_t *this = NULL;
+    changelog_priv_t *priv = NULL;
+    struct list_head queue = {
+        0,
+    };
 
-        this = data;
-        THIS = this;
-        priv = this->private;
+    this = data;
+    THIS = this;
+    priv = this->private;
 
-        INIT_LIST_HEAD (&queue);
+    INIT_LIST_HEAD(&queue);
 
-        gf_log (this->name, GF_LOG_ERROR,
-                "Disabling changelog barrier because of the timeout.");
+    gf_smsg(this->name, GF_LOG_ERROR, 0, CHANGELOG_MSG_BARRIER_TIMEOUT, NULL);
 
-        LOCK (&priv->lock);
-        {
-                __chlog_barrier_disable (this, &queue);
-        }
-        UNLOCK (&priv->lock);
+    LOCK(&priv->lock);
+    {
+        __chlog_barrier_disable(this, &queue);
+    }
+    UNLOCK(&priv->lock);
 
-        chlog_barrier_dequeue_all (this, &queue);
+    chlog_barrier_dequeue_all(this, &queue);
 
-        return;
+    return;
 }
 
 /* Disable changelog barrier enable flag */
 void
-__chlog_barrier_disable (xlator_t *this, struct list_head *queue)
+__chlog_barrier_disable(xlator_t *this, struct list_head *queue)
 {
-        changelog_priv_t  *priv   = this->private;
-        GF_ASSERT (priv);
+    changelog_priv_t *priv = this->private;
+    GF_ASSERT(priv);
 
-        if (priv->timer) {
-                gf_timer_call_cancel (this->ctx, priv->timer);
-                priv->timer = NULL;
-        }
+    if (priv->timer) {
+        gf_timer_call_cancel(this->ctx, priv->timer);
+        priv->timer = NULL;
+    }
 
-        list_splice_init (&priv->queue, queue);
-        priv->queue_size = 0;
-        priv->barrier_enabled = _gf_false;
+    list_splice_init(&priv->queue, queue);
+    priv->queue_size = 0;
+    priv->barrier_enabled = _gf_false;
 }
 
 /* Enable chagelog barrier enable with timer */
 int
-__chlog_barrier_enable (xlator_t *this, changelog_priv_t *priv)
+__chlog_barrier_enable(xlator_t *this, changelog_priv_t *priv)
 {
-        int             ret     = -1;
-
-        priv->timer = gf_timer_call_after (this->ctx, priv->timeout,
-                                           chlog_barrier_timeout, (void *)this);
-        if (!priv->timer) {
-                gf_log (this->name, GF_LOG_CRITICAL,
-                        "Couldn't add changelog barrier timeout event.");
-                goto out;
-        }
-
-        priv->barrier_enabled = _gf_true;
-        ret = 0;
+    int ret = -1;
+
+    priv->timer = gf_timer_call_after(this->ctx, priv->timeout,
+                                      chlog_barrier_timeout, (void *)this);
+    if (!priv->timer) {
+        gf_smsg(this->name, GF_LOG_CRITICAL, 0,
+                CHANGELOG_MSG_TIMEOUT_ADD_FAILED, NULL);
+        goto out;
+    }
+
+    priv->barrier_enabled = _gf_true;
+    ret = 0;
 out:
-        return ret;
+    return ret;
 }
diff --git a/xlators/features/changelog/src/changelog-encoders.c b/xlators/features/changelog/src/changelog-encoders.c
index ea9db4061ca..63754516c2e 100644
--- a/xlators/features/changelog/src/changelog-encoders.c
+++ b/xlators/features/changelog/src/changelog-encoders.c
@@ -8,190 +8,225 @@
    cases as published by the Free Software Foundation.
 */
 
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
-
 #include "changelog-encoders.h"
 
 size_t
-entry_fn (void *data, char *buffer, gf_boolean_t encode)
+entry_fn(void *data, char *buffer, gf_boolean_t encode)
 {
-        char    *tmpbuf = NULL;
-        size_t  bufsz  = 0;
-        struct changelog_entry_fields *ce = NULL;
-
-        ce = (struct changelog_entry_fields *) data;
-
-        if (encode) {
-                tmpbuf = uuid_utoa (ce->cef_uuid);
-                CHANGELOG_FILL_BUFFER (buffer, bufsz, tmpbuf, strlen (tmpbuf));
-        } else {
-                CHANGELOG_FILL_BUFFER (buffer, bufsz,
-                                       ce->cef_uuid, sizeof (uuid_t));
-        }
+    char *tmpbuf = NULL;
+    size_t bufsz = 0;
+    struct changelog_entry_fields *ce = NULL;
+
+    ce = (struct changelog_entry_fields *)data;
+
+    if (encode) {
+        tmpbuf = uuid_utoa(ce->cef_uuid);
+        CHANGELOG_FILL_BUFFER(buffer, bufsz, tmpbuf, strlen(tmpbuf));
+    } else {
+        CHANGELOG_FILL_BUFFER(buffer, bufsz, ce->cef_uuid, sizeof(uuid_t));
+    }
+
+    CHANGELOG_FILL_BUFFER(buffer, bufsz, "/", 1);
+    CHANGELOG_FILL_BUFFER(buffer, bufsz, ce->cef_bname, strlen(ce->cef_bname));
+    return bufsz;
+}
 
-        CHANGELOG_FILL_BUFFER (buffer, bufsz, "/", 1);
-        CHANGELOG_FILL_BUFFER (buffer, bufsz,
-                               ce->cef_bname, strlen (ce->cef_bname));
-        return bufsz;
+size_t
+del_entry_fn(void *data, char *buffer, gf_boolean_t encode)
+{
+    char *tmpbuf = NULL;
+    size_t bufsz = 0;
+    struct changelog_entry_fields *ce = NULL;
+
+    ce = (struct changelog_entry_fields *)data;
+
+    if (encode) {
+        tmpbuf = uuid_utoa(ce->cef_uuid);
+        CHANGELOG_FILL_BUFFER(buffer, bufsz, tmpbuf, strlen(tmpbuf));
+    } else {
+        CHANGELOG_FILL_BUFFER(buffer, bufsz, ce->cef_uuid, sizeof(uuid_t));
+    }
+
+    CHANGELOG_FILL_BUFFER(buffer, bufsz, "/", 1);
+    CHANGELOG_FILL_BUFFER(buffer, bufsz, ce->cef_bname, strlen(ce->cef_bname));
+    CHANGELOG_FILL_BUFFER(buffer, bufsz, "\0", 1);
+
+    if (ce->cef_path[0] == '\0') {
+        CHANGELOG_FILL_BUFFER(buffer, bufsz, "\0", 1);
+    } else {
+        CHANGELOG_FILL_BUFFER(buffer, bufsz, ce->cef_path,
+                              strlen(ce->cef_path));
+    }
+
+    return bufsz;
 }
 
 size_t
-fop_fn (void *data, char *buffer, gf_boolean_t encode)
+fop_fn(void *data, char *buffer, gf_boolean_t encode)
 {
-        char buf[10]          = {0,};
-        size_t         bufsz = 0;
-        glusterfs_fop_t fop   = 0;
+    char buf[10] = {
+        0,
+    };
+    size_t bufsz = 0;
+    glusterfs_fop_t fop = 0;
 
-        fop = *(glusterfs_fop_t *) data;
+    fop = *(glusterfs_fop_t *)data;
 
-        if (encode) {
-                (void) snprintf (buf, sizeof (buf), "%d", fop);
-                CHANGELOG_FILL_BUFFER (buffer, bufsz, buf, strlen (buf));
-        } else
-                CHANGELOG_FILL_BUFFER (buffer, bufsz, &fop, sizeof (fop));
+    if (encode) {
+        (void)snprintf(buf, sizeof(buf), "%d", fop);
+        CHANGELOG_FILL_BUFFER(buffer, bufsz, buf, strlen(buf));
+    } else
+        CHANGELOG_FILL_BUFFER(buffer, bufsz, &fop, sizeof(fop));
 
-        return bufsz;
+    return bufsz;
 }
 
 size_t
-number_fn (void *data, char *buffer, gf_boolean_t encode)
+number_fn(void *data, char *buffer, gf_boolean_t encode)
 {
-        size_t       bufsz = 0;
-        unsigned int nr    = 0;
-        char buf[20]       = {0,};
+    size_t bufsz = 0;
+    unsigned int nr = 0;
+    char buf[20] = {
+        0,
+    };
+
+    nr = *(unsigned int *)data;
 
-        nr = *(unsigned int *) data;
+    if (encode) {
+        (void)snprintf(buf, sizeof(buf), "%u", nr);
+        CHANGELOG_FILL_BUFFER(buffer, bufsz, buf, strlen(buf));
+    } else
+        CHANGELOG_FILL_BUFFER(buffer, bufsz, &nr, sizeof(unsigned int));
 
-        if (encode) {
-                (void) snprintf (buf, sizeof (buf), "%u", nr);
-                CHANGELOG_FILL_BUFFER (buffer, bufsz, buf, strlen (buf));
-        } else
-                CHANGELOG_FILL_BUFFER (buffer, bufsz, &nr, sizeof (unsigned int));
+    return bufsz;
+}
+
+void
+entry_free_fn(void *data)
+{
+    changelog_opt_t *co = data;
+
+    if (!co)
+        return;
 
-        return bufsz;
+    GF_FREE(co->co_entry.cef_bname);
 }
 
 void
-entry_free_fn (void *data)
+del_entry_free_fn(void *data)
 {
-        changelog_opt_t *co = data;
+    changelog_opt_t *co = data;
 
-        if (!co)
-                return;
+    if (!co)
+        return;
 
-        GF_FREE (co->co_entry.cef_bname);
+    GF_FREE(co->co_entry.cef_bname);
+    GF_FREE(co->co_entry.cef_path);
 }
 
 /**
  * try to write all data in one shot
  */
 
-static inline void
-changelog_encode_write_xtra (changelog_log_data_t *cld,
-                             char *buffer, size_t *off, gf_boolean_t encode)
+static void
+changelog_encode_write_xtra(changelog_log_data_t *cld, char *buffer,
+                            size_t *off, gf_boolean_t encode)
 {
-        int              i      = 0;
-        size_t           offset = 0;
-        void            *data   = NULL;
-        changelog_opt_t *co     = NULL;
-
-        offset = *off;
-
-        co = (changelog_opt_t *) cld->cld_ptr;
-
-        for (; i < cld->cld_xtra_records; i++, co++) {
-                CHANGELOG_FILL_BUFFER (buffer, offset, "\0", 1);
-
-                switch (co->co_type) {
-                case CHANGELOG_OPT_REC_FOP:
-                        data = &co->co_fop;
-                        break;
-                case CHANGELOG_OPT_REC_ENTRY:
-                        data = &co->co_entry;
-                        break;
-                case CHANGELOG_OPT_REC_UINT32:
-                        data = &co->co_uint32;
-                        break;
-                }
-
-                if (co->co_convert)
-                        offset += co->co_convert (data,
-                                                  buffer + offset, encode);
-                else /* no coversion: write it out as it is */
-                        CHANGELOG_FILL_BUFFER (buffer, offset,
-                                               data, co->co_len);
+    int i = 0;
+    size_t offset = 0;
+    void *data = NULL;
+    changelog_opt_t *co = NULL;
+
+    offset = *off;
+
+    co = (changelog_opt_t *)cld->cld_ptr;
+
+    for (; i < cld->cld_xtra_records; i++, co++) {
+        CHANGELOG_FILL_BUFFER(buffer, offset, "\0", 1);
+
+        switch (co->co_type) {
+            case CHANGELOG_OPT_REC_FOP:
+                data = &co->co_fop;
+                break;
+            case CHANGELOG_OPT_REC_ENTRY:
+                data = &co->co_entry;
+                break;
+            case CHANGELOG_OPT_REC_UINT32:
+                data = &co->co_uint32;
+                break;
         }
 
-        *off = offset;
+        if (co->co_convert)
+            offset += co->co_convert(data, buffer + offset, encode);
+        else /* no coversion: write it out as it is */
+            CHANGELOG_FILL_BUFFER(buffer, offset, data, co->co_len);
+    }
+
+    *off = offset;
 }
 
 int
-changelog_encode_ascii (xlator_t *this, changelog_log_data_t *cld)
+changelog_encode_ascii(xlator_t *this, changelog_log_data_t *cld)
 {
-        size_t            off      = 0;
-        size_t            gfid_len = 0;
-        char             *gfid_str = NULL;
-        char             *buffer   = NULL;
-        changelog_priv_t *priv     = NULL;
+    size_t off = 0;
+    size_t gfid_len = 0;
+    char *gfid_str = NULL;
+    char *buffer = NULL;
+    changelog_priv_t *priv = NULL;
 
-        priv = this->private;
+    priv = this->private;
 
-        gfid_str = uuid_utoa (cld->cld_gfid);
-        gfid_len = strlen (gfid_str);
+    gfid_str = uuid_utoa(cld->cld_gfid);
+    gfid_len = strlen(gfid_str);
 
-        /* extra bytes for decorations */
-        buffer = alloca (gfid_len + cld->cld_ptr_len + 10);
-        CHANGELOG_STORE_ASCII (priv, buffer,
-                               off, gfid_str, gfid_len, cld);
+    /* extra bytes for decorations */
+    buffer = alloca(gfid_len + cld->cld_ptr_len + 10);
+    CHANGELOG_STORE_ASCII(priv, buffer, off, gfid_str, gfid_len, cld);
 
-        if (cld->cld_xtra_records)
-                changelog_encode_write_xtra (cld, buffer, &off, _gf_true);
+    if (cld->cld_xtra_records)
+        changelog_encode_write_xtra(cld, buffer, &off, _gf_true);
 
-        CHANGELOG_FILL_BUFFER (buffer, off, "\0", 1);
+    CHANGELOG_FILL_BUFFER(buffer, off, "\0", 1);
 
-        return changelog_write_change (priv, buffer, off);
+    return changelog_write_change(priv, buffer, off);
 }
 
 int
-changelog_encode_binary (xlator_t *this, changelog_log_data_t *cld)
+changelog_encode_binary(xlator_t *this, changelog_log_data_t *cld)
 {
-        size_t            off    = 0;
-        char             *buffer = NULL;
-        changelog_priv_t *priv   = NULL;
+    size_t off = 0;
+    char *buffer = NULL;
+    changelog_priv_t *priv = NULL;
 
-        priv = this->private;
+    priv = this->private;
 
-        /* extra bytes for decorations */
-        buffer = alloca (sizeof (uuid_t) + cld->cld_ptr_len + 10);
-        CHANGELOG_STORE_BINARY (priv, buffer, off, cld->cld_gfid, cld);
+    /* extra bytes for decorations */
+    buffer = alloca(sizeof(uuid_t) + cld->cld_ptr_len + 10);
+    CHANGELOG_STORE_BINARY(priv, buffer, off, cld->cld_gfid, cld);
 
-        if (cld->cld_xtra_records)
-                changelog_encode_write_xtra (cld, buffer, &off, _gf_false);
+    if (cld->cld_xtra_records)
+        changelog_encode_write_xtra(cld, buffer, &off, _gf_false);
 
-        CHANGELOG_FILL_BUFFER (buffer, off, "\0", 1);
+    CHANGELOG_FILL_BUFFER(buffer, off, "\0", 1);
 
-        return changelog_write_change (priv, buffer, off);
+    return changelog_write_change(priv, buffer, off);
 }
 
-static struct changelog_encoder
-cb_encoder[] = {
-        [CHANGELOG_ENCODE_BINARY] =
+static struct changelog_encoder cb_encoder[] = {
+    [CHANGELOG_ENCODE_BINARY] =
         {
-                .encoder = CHANGELOG_ENCODE_BINARY,
-                .encode = changelog_encode_binary,
+            .encoder = CHANGELOG_ENCODE_BINARY,
+            .encode = changelog_encode_binary,
         },
-        [CHANGELOG_ENCODE_ASCII] =
+    [CHANGELOG_ENCODE_ASCII] =
         {
-                .encoder = CHANGELOG_ENCODE_ASCII,
-                .encode = changelog_encode_ascii,
+            .encoder = CHANGELOG_ENCODE_ASCII,
+            .encode = changelog_encode_ascii,
         },
 };
 
 void
 changelog_encode_change(changelog_priv_t *priv)
 {
-        priv->ce = &cb_encoder[priv->encode_mode];
+    priv->ce = &cb_encoder[priv->encode_mode];
 }
diff --git a/xlators/features/changelog/src/changelog-encoders.h b/xlators/features/changelog/src/changelog-encoders.h
index c5dcc8a77d9..26252696d56 100644
--- a/xlators/features/changelog/src/changelog-encoders.h
+++ b/xlators/features/changelog/src/changelog-encoders.h
@@ -11,37 +11,39 @@
 #ifndef _CHANGELOG_ENCODERS_H
 #define _CHANGELOG_ENCODERS_H
 
-#include "xlator.h"
-#include "defaults.h"
+#include <glusterfs/xlator.h>
+#include <glusterfs/defaults.h>
 
 #include "changelog-helpers.h"
 
-#define CHANGELOG_STORE_ASCII(priv, buf, off, gfid, gfid_len, cld) do { \
-                CHANGELOG_FILL_BUFFER (buffer, off,                     \
-                                       priv->maps[cld->cld_type], 1);   \
-                CHANGELOG_FILL_BUFFER (buffer,                          \
-                                       off, gfid, gfid_len);            \
-        } while (0)
+#define CHANGELOG_STORE_ASCII(priv, buf, off, gfid, gfid_len, cld)             \
+    do {                                                                       \
+        CHANGELOG_FILL_BUFFER(buffer, off, priv->maps[cld->cld_type], 1);      \
+        CHANGELOG_FILL_BUFFER(buffer, off, gfid, gfid_len);                    \
+    } while (0)
 
-#define CHANGELOG_STORE_BINARY(priv, buf, off, gfid, cld) do {          \
-                CHANGELOG_FILL_BUFFER (buffer, off,                     \
-                                       priv->maps[cld->cld_type], 1);   \
-                CHANGELOG_FILL_BUFFER (buffer,                          \
-                                       off, gfid, sizeof (uuid_t));     \
-        } while (0)
+#define CHANGELOG_STORE_BINARY(priv, buf, off, gfid, cld)                      \
+    do {                                                                       \
+        CHANGELOG_FILL_BUFFER(buffer, off, priv->maps[cld->cld_type], 1);      \
+        CHANGELOG_FILL_BUFFER(buffer, off, gfid, sizeof(uuid_t));              \
+    } while (0)
 
 size_t
-entry_fn (void *data, char *buffer, gf_boolean_t encode);
+entry_fn(void *data, char *buffer, gf_boolean_t encode);
 size_t
-fop_fn (void *data, char *buffer, gf_boolean_t encode);
+del_entry_fn(void *data, char *buffer, gf_boolean_t encode);
 size_t
-number_fn (void *data, char *buffer, gf_boolean_t encode);
+fop_fn(void *data, char *buffer, gf_boolean_t encode);
+size_t
+number_fn(void *data, char *buffer, gf_boolean_t encode);
+void
+entry_free_fn(void *data);
 void
-entry_free_fn (void *data);
+del_entry_free_fn(void *data);
 int
-changelog_encode_binary (xlator_t *, changelog_log_data_t *);
+changelog_encode_binary(xlator_t *, changelog_log_data_t *);
 int
-changelog_encode_ascii (xlator_t *, changelog_log_data_t *);
+changelog_encode_ascii(xlator_t *, changelog_log_data_t *);
 void
 changelog_encode_change(changelog_priv_t *);
 
diff --git a/xlators/features/changelog/src/changelog-ev-handle.c b/xlators/features/changelog/src/changelog-ev-handle.c
index ca7443cfd22..aa94459de5a 100644
--- a/xlators/features/changelog/src/changelog-ev-handle.c
+++ b/xlators/features/changelog/src/changelog-ev-handle.c
@@ -14,19 +14,19 @@
 
 struct rpc_clnt_program changelog_ev_program;
 
-#define NR_IOVEC  (MAX_IOVEC - 3)
+#define NR_IOVEC (MAX_IOVEC - 3)
 struct ev_rpc_vec {
-        int count;
-        struct iovec vector[NR_IOVEC];
+    int count;
+    struct iovec vector[NR_IOVEC];
 
-        /* sequence number */
-        unsigned long seq;
+    /* sequence number */
+    unsigned long seq;
 };
 
 struct ev_rpc {
-        rbuf_list_t     *rlist;
-        struct rpc_clnt *rpc;
-        struct ev_rpc_vec vec;
+    rbuf_list_t *rlist;
+    struct rpc_clnt *rpc;
+    struct ev_rpc_vec vec;
 };
 
 /**
@@ -35,203 +35,229 @@ struct ev_rpc {
  * intelligence can be built into the server.
  */
 int
-changelog_event_dispatch_cbk (struct rpc_req *req,
-                              struct iovec *iov, int count, void *myframe)
+changelog_event_dispatch_cbk(struct rpc_req *req, struct iovec *iov, int count,
+                             void *myframe)
 {
-        return 0;
+    return 0;
 }
 
 /* dispatcher RPC */
-inline int
-changelog_dispatch_vec (call_frame_t *frame, xlator_t *this,
-                        struct rpc_clnt *rpc, struct ev_rpc_vec *vec)
+int
+changelog_dispatch_vec(call_frame_t *frame, xlator_t *this,
+                       struct rpc_clnt *rpc, struct ev_rpc_vec *vec)
+{
+    struct timeval tv = {
+        0,
+    };
+    changelog_event_req req = {
+        0,
+    };
+
+    (void)gettimeofday(&tv, NULL);
+
+    /**
+     * Event dispatch RPC header contains a sequence number for each
+     * dispatch. This allows the receiver to order the request before
+     * processing.
+     */
+    req.seq = vec->seq;
+    req.tv_sec = tv.tv_sec;
+    req.tv_usec = tv.tv_usec;
+
+    return changelog_rpc_sumbit_req(
+        rpc, (void *)&req, frame, &changelog_ev_program,
+        CHANGELOG_REV_PROC_EVENT, vec->vector, vec->count, NULL, this,
+        changelog_event_dispatch_cbk, (xdrproc_t)xdr_changelog_event_req);
+}
+
+int
+changelog_event_dispatch_rpc(call_frame_t *frame, xlator_t *this, void *data)
 {
-         struct timeval      tv  = {0,};
-         changelog_event_req req = {0,};
-
-         (void) gettimeofday (&tv, NULL);
-
-         /**
-          * Event dispatch RPC header contains a sequence number for each
-          * dispatch. This allows the reciever to order the request before
-          * processing.
-          */
-         req.seq     = vec->seq;
-         req.tv_sec  = tv.tv_sec;
-         req.tv_usec = tv.tv_usec;
-
-         return changelog_rpc_sumbit_req (rpc, (void *)&req,
-                                          frame, &changelog_ev_program,
-                                          CHANGELOG_REV_PROC_EVENT,
-                                          vec->vector, vec->count, NULL,
-                                          this, changelog_event_dispatch_cbk,
-                                          (xdrproc_t) xdr_changelog_event_req);
- }
-
- int
- changelog_event_dispatch_rpc (call_frame_t *frame, xlator_t *this, void *data)
- {
-         int                idx      = 0;
-         int                count    = 0;
-         int                ret      = 0;
-         unsigned long      range    = 0;
-         unsigned long      sequence = 0;
-         rbuf_iovec_t      *rvec     = NULL;
-         struct ev_rpc     *erpc     = NULL;
-         struct rlist_iter  riter    = {{0,},};
-
-         /* dispatch NR_IOVEC IO vectors at a time. */
-
-         erpc = data;
-         RLIST_GET_SEQ (erpc->rlist, sequence, range);
-
-         rlist_iter_init (&riter, erpc->rlist);
-
-         rvec_for_each_entry (rvec, &riter) {
-                 idx = count % NR_IOVEC;
-                 if (++count == NR_IOVEC) {
-                         erpc->vec.vector[idx] = rvec->iov;
-                         erpc->vec.seq = sequence++;
-                         erpc->vec.count = NR_IOVEC;
-
-                         ret = changelog_dispatch_vec (frame, this,
-                                                       erpc->rpc, &erpc->vec);
-                         if (ret)
-                                 break;
-                         count = 0;
-                         continue;
-                 }
-
-                 erpc->vec.vector[idx] = rvec->iov;
-         }
-
-         if (ret)
-                 goto error_return;
-
-         idx = count % NR_IOVEC;
-         if (idx) {
-                 erpc->vec.seq = sequence;
-                 erpc->vec.count = idx;
-
-                 ret = changelog_dispatch_vec (frame, this,
-                                               erpc->rpc, &erpc->vec);
-         }
-
- error_return:
-         return ret;
+    int idx = 0;
+    int count = 0;
+    int ret = 0;
+    unsigned long sequence = 0;
+    rbuf_iovec_t *rvec = NULL;
+    struct ev_rpc *erpc = NULL;
+    struct rlist_iter riter = {
+        {
+            0,
+        },
+    };
+
+    /* dispatch NR_IOVEC IO vectors at a time. */
+
+    erpc = data;
+    sequence = erpc->rlist->seq[0];
+
+    rlist_iter_init(&riter, erpc->rlist);
+
+    rvec_for_each_entry(rvec, &riter)
+    {
+        idx = count % NR_IOVEC;
+        if (++count == NR_IOVEC) {
+            erpc->vec.vector[idx] = rvec->iov;
+            erpc->vec.seq = sequence++;
+            erpc->vec.count = NR_IOVEC;
+
+            ret = changelog_dispatch_vec(frame, this, erpc->rpc, &erpc->vec);
+            if (ret)
+                break;
+            count = 0;
+            continue;
+        }
+
+        erpc->vec.vector[idx] = rvec->iov;
+    }
+
+    if (ret)
+        goto error_return;
+
+    idx = count % NR_IOVEC;
+    if (idx) {
+        erpc->vec.seq = sequence;
+        erpc->vec.count = idx;
+
+        ret = changelog_dispatch_vec(frame, this, erpc->rpc, &erpc->vec);
+    }
+
+error_return:
+    return ret;
 }
 
 int
-changelog_rpc_notify (struct rpc_clnt *rpc,
-                      void *mydata, rpc_clnt_event_t event, void *data)
+changelog_rpc_notify(struct rpc_clnt *rpc, void *mydata, rpc_clnt_event_t event,
+                     void *data)
 {
-        xlator_t                *this      = NULL;
-        changelog_rpc_clnt_t    *crpc      = NULL;
-        changelog_clnt_t        *c_clnt    = NULL;
-        changelog_priv_t        *priv      = NULL;
-        changelog_ev_selector_t *selection = NULL;
+    xlator_t *this = NULL;
+    changelog_rpc_clnt_t *crpc = NULL;
+    changelog_clnt_t *c_clnt = NULL;
+    changelog_priv_t *priv = NULL;
+    changelog_ev_selector_t *selection = NULL;
+    uint64_t clntcnt = 0;
+    uint64_t xprtcnt = 0;
 
-        crpc = mydata;
-        this = crpc->this;
-        c_clnt = crpc->c_clnt;
+    crpc = mydata;
+    this = crpc->this;
+    c_clnt = crpc->c_clnt;
 
-        priv = this->private;
+    priv = this->private;
 
-        switch (event) {
+    switch (event) {
         case RPC_CLNT_CONNECT:
-                rpc_clnt_set_connected (&rpc->conn);
-                selection = &priv->ev_selection;
+            selection = &priv->ev_selection;
+            GF_ATOMIC_INC(priv->clntcnt);
 
-                LOCK (&c_clnt->wait_lock);
+            LOCK(&c_clnt->wait_lock);
+            {
+                LOCK(&c_clnt->active_lock);
                 {
-                        LOCK (&c_clnt->active_lock);
-                        {
-                                changelog_select_event (this, selection,
-                                                        crpc->filter);
-                                list_move_tail (&crpc->list, &c_clnt->active);
-                        }
-                        UNLOCK (&c_clnt->active_lock);
+                    changelog_select_event(this, selection, crpc->filter);
+                    list_move_tail(&crpc->list, &c_clnt->active);
                 }
-                UNLOCK (&c_clnt->wait_lock);
+                UNLOCK(&c_clnt->active_lock);
+            }
+            UNLOCK(&c_clnt->wait_lock);
 
-                break;
+            break;
         case RPC_CLNT_DISCONNECT:
-                rpc_clnt_disable (crpc->rpc);
-                selection = &priv->ev_selection;
+            rpc_clnt_disable(crpc->rpc);
 
-                LOCK (&crpc->lock);
-                {
-                        changelog_deselect_event (this, selection,
-                                                  crpc->filter);
-                        changelog_set_disconnect_flag (crpc, _gf_true);
-                }
-                UNLOCK (&crpc->lock);
+            /* rpc_clnt_disable doesn't unref the rpc. It just marks
+             * the rpc as disabled and cancels reconnection timer.
+             * Hence unref the rpc object to free it.
+             */
+            rpc_clnt_unref(crpc->rpc);
 
-                break;
+            if (priv)
+                selection = &priv->ev_selection;
+
+            LOCK(&crpc->lock);
+            {
+                if (selection)
+                    changelog_deselect_event(this, selection, crpc->filter);
+                changelog_set_disconnect_flag(crpc, _gf_true);
+            }
+            UNLOCK(&crpc->lock);
+            LOCK(&c_clnt->active_lock);
+            {
+                list_del_init(&crpc->list);
+            }
+            UNLOCK(&c_clnt->active_lock);
+
+            break;
         case RPC_CLNT_MSG:
         case RPC_CLNT_DESTROY:
-                break;
-        }
-
-        return 0;
+            /* Free up mydata */
+            changelog_rpc_clnt_unref(crpc);
+            clntcnt = GF_ATOMIC_DEC(priv->clntcnt);
+            xprtcnt = GF_ATOMIC_GET(priv->xprtcnt);
+            if (this->cleanup_starting) {
+                if (!clntcnt && !xprtcnt)
+                    changelog_process_cleanup_event(this);
+            }
+            break;
+        case RPC_CLNT_PING:
+            break;
+    }
+
+    return 0;
 }
 
 void *
-changelog_ev_connector (void *data)
+changelog_ev_connector(void *data)
 {
-        xlator_t             *this   = NULL;
-        changelog_clnt_t     *c_clnt = NULL;
-        changelog_rpc_clnt_t *crpc   = NULL;
+    xlator_t *this = NULL;
+    changelog_clnt_t *c_clnt = NULL;
+    changelog_rpc_clnt_t *crpc = NULL;
 
-        c_clnt = data;
-        this = c_clnt->this;
+    c_clnt = data;
+    this = c_clnt->this;
 
-        while (1) {
-                pthread_mutex_lock (&c_clnt->pending_lock);
-                {
-                        while (list_empty (&c_clnt->pending))
-                                pthread_cond_wait (&c_clnt->pending_cond,
-                                                   &c_clnt->pending_lock);
-                        crpc = list_first_entry (&c_clnt->pending,
-                                                 changelog_rpc_clnt_t, list);
-                        crpc->rpc =
-                                changelog_rpc_client_init (this, crpc,
-                                                           crpc->sock,
-                                                           changelog_rpc_notify);
-                        if (!crpc->rpc) {
-                                gf_log (this->name, GF_LOG_ERROR, "failed to "
-                                        "connect back.. <%s>", crpc->sock);
-                                crpc->cleanup (crpc);
-                                goto mutex_unlock;
-                        }
-
-                        LOCK (&c_clnt->wait_lock);
-                        {
-                                list_move_tail (&crpc->list, &c_clnt->waitq);
-                        }
-                        UNLOCK (&c_clnt->wait_lock);
-                }
-        mutex_unlock:
-                pthread_mutex_unlock (&c_clnt->pending_lock);
+    while (1) {
+        pthread_mutex_lock(&c_clnt->pending_lock);
+        {
+            while (list_empty(&c_clnt->pending))
+                pthread_cond_wait(&c_clnt->pending_cond, &c_clnt->pending_lock);
+            crpc = list_first_entry(&c_clnt->pending, changelog_rpc_clnt_t,
+                                    list);
+            crpc->rpc = changelog_rpc_client_init(this, crpc, crpc->sock,
+                                                  changelog_rpc_notify);
+            if (!crpc->rpc) {
+                gf_smsg(this->name, GF_LOG_ERROR, 0,
+                        CHANGELOG_MSG_RPC_CONNECT_ERROR, "path=%s", crpc->sock,
+                        NULL);
+                crpc->cleanup(crpc);
+                goto mutex_unlock;
+            }
+
+            LOCK(&c_clnt->wait_lock);
+            {
+                list_move_tail(&crpc->list, &c_clnt->waitq);
+            }
+            UNLOCK(&c_clnt->wait_lock);
         }
+    mutex_unlock:
+        pthread_mutex_unlock(&c_clnt->pending_lock);
+    }
 
-        return NULL;
+    return NULL;
 }
 
 void
-changelog_ev_cleanup_connections (xlator_t *this, changelog_clnt_t *c_clnt)
+changelog_ev_cleanup_connections(xlator_t *this, changelog_clnt_t *c_clnt)
 {
-        int ret = 0;
-        changelog_rpc_clnt_t *crpc = NULL;
+    changelog_rpc_clnt_t *crpc = NULL;
 
-        /* cleanup active connections */
-        LOCK (&c_clnt->active_lock);
+    /* cleanup active connections */
+    LOCK(&c_clnt->active_lock);
+    {
+        list_for_each_entry(crpc, &c_clnt->active, list)
         {
-                list_for_each_entry (crpc, &c_clnt->active, list) {
-                        rpc_clnt_disable (crpc->rpc);
-                }
+            rpc_clnt_disable(crpc->rpc);
         }
-        UNLOCK (&c_clnt->active_lock);
+    }
+    UNLOCK(&c_clnt->active_lock);
 }
 
 /**
@@ -241,142 +267,146 @@ changelog_ev_cleanup_connections (xlator_t *this, changelog_clnt_t *c_clnt)
  * a performance bottleneck.
  */
 
-static inline changelog_rpc_clnt_t *
-get_client (changelog_clnt_t *c_clnt, struct list_head **next)
+static changelog_rpc_clnt_t *
+get_client(changelog_clnt_t *c_clnt, struct list_head **next)
 {
-        changelog_rpc_clnt_t *crpc = NULL;
-
-        LOCK (&c_clnt->active_lock);
-        {
-                if (*next == &c_clnt->active)
-                        goto unblock;
-                crpc = list_entry (*next, changelog_rpc_clnt_t, list);
-                changelog_rpc_clnt_ref (crpc);
-                *next = (*next)->next;
-        }
- unblock:
-        UNLOCK (&c_clnt->active_lock);
-
-        return crpc;
+    changelog_rpc_clnt_t *crpc = NULL;
+
+    LOCK(&c_clnt->active_lock);
+    {
+        if (*next == &c_clnt->active)
+            goto unblock;
+        crpc = list_entry(*next, changelog_rpc_clnt_t, list);
+        /* ref rpc as DISCONNECT might unref the rpc asynchronously */
+        changelog_rpc_clnt_ref(crpc);
+        rpc_clnt_ref(crpc->rpc);
+        *next = (*next)->next;
+    }
+unblock:
+    UNLOCK(&c_clnt->active_lock);
+
+    return crpc;
 }
 
-static inline void
-put_client (changelog_clnt_t *c_clnt, changelog_rpc_clnt_t *crpc)
+static void
+put_client(changelog_clnt_t *c_clnt, changelog_rpc_clnt_t *crpc)
 {
-        LOCK (&c_clnt->active_lock);
-        {
-                changelog_rpc_clnt_unref (crpc);
-        }
-        UNLOCK (&c_clnt->active_lock);
+    LOCK(&c_clnt->active_lock);
+    {
+        rpc_clnt_unref(crpc->rpc);
+        changelog_rpc_clnt_unref(crpc);
+    }
+    UNLOCK(&c_clnt->active_lock);
 }
 
 void
-_dispatcher (rbuf_list_t *rlist, void *arg)
+_dispatcher(rbuf_list_t *rlist, void *arg)
 {
-        int                   ret    = 0;
-        xlator_t             *this   = NULL;
-        changelog_clnt_t     *c_clnt = NULL;
-        changelog_rpc_clnt_t *crpc   = NULL;
-        changelog_rpc_clnt_t *tmp    = NULL;
-        struct ev_rpc         erpc   = {0,};
-        struct list_head     *next   = NULL;
-
-        c_clnt = arg;
-        this = c_clnt->this;
-
-        erpc.rlist = rlist;
-        next = c_clnt->active.next;
-
-        while (1) {
-                crpc = get_client (c_clnt, &next);
-                if (!crpc)
-                        break;
-                erpc.rpc = crpc->rpc;
-                ret = changelog_invoke_rpc (this, crpc->rpc,
-                                            &changelog_ev_program,
-                                            CHANGELOG_REV_PROC_EVENT, &erpc);
-                put_client (c_clnt, crpc);
-        }
+    xlator_t *this = NULL;
+    changelog_clnt_t *c_clnt = NULL;
+    changelog_rpc_clnt_t *crpc = NULL;
+    struct ev_rpc erpc = {
+        0,
+    };
+    struct list_head *next = NULL;
+
+    c_clnt = arg;
+    this = c_clnt->this;
+
+    erpc.rlist = rlist;
+    next = c_clnt->active.next;
+
+    while (1) {
+        crpc = get_client(c_clnt, &next);
+        if (!crpc)
+            break;
+        erpc.rpc = crpc->rpc;
+        (void)changelog_invoke_rpc(this, crpc->rpc, &changelog_ev_program,
+                                   CHANGELOG_REV_PROC_EVENT, &erpc);
+        put_client(c_clnt, crpc);
+    }
 }
 
 /** this is called under rotbuff's lock */
 void
-sequencer (rbuf_list_t *rlist, void *mydata)
+sequencer(rbuf_list_t *rlist, void *mydata)
 {
-        unsigned long     range  = 0;
-        changelog_clnt_t *c_clnt = 0;
+    unsigned long range = 0;
+    changelog_clnt_t *c_clnt = 0;
 
-        c_clnt = mydata;
+    c_clnt = mydata;
 
-        range = (RLIST_ENTRY_COUNT (rlist)) / NR_IOVEC;
-        if ((RLIST_ENTRY_COUNT (rlist)) % NR_IOVEC)
-                range++;
-        RLIST_STORE_SEQ (rlist, c_clnt->sequence, range);
+    range = (RLIST_ENTRY_COUNT(rlist)) / NR_IOVEC;
+    if ((RLIST_ENTRY_COUNT(rlist)) % NR_IOVEC)
+        range++;
+    RLIST_STORE_SEQ(rlist, c_clnt->sequence, range);
 
-        c_clnt->sequence += range;
+    c_clnt->sequence += range;
 }
 
 void *
-changelog_ev_dispatch (void *data)
+changelog_ev_dispatch(void *data)
 {
-        int                   ret    = 0;
-        void                 *opaque = NULL;
-        xlator_t             *this   = NULL;
-        changelog_clnt_t     *c_clnt = NULL;
-        struct timeval        tv     = {0,};
-
-        c_clnt = data;
-        this = c_clnt->this;
-
-        while (1) {
-                /* TODO: change this to be pthread cond based.. later */
-                tv.tv_sec = 1;
-                tv.tv_usec = 0;
-                select (0, NULL, NULL, NULL, &tv);
-
-                ret = rbuf_get_buffer (c_clnt->rbuf,
-                                       &opaque, sequencer, c_clnt);
-                if (ret != RBUF_CONSUMABLE) {
-                        if (ret != RBUF_EMPTY)
-                                gf_log (this->name, GF_LOG_WARNING,
-                                        "Failed to get buffer for RPC dispatch "
-                                        "[rbuf retval: %d]", ret);
-                        continue;
-                }
-
-                ret = rbuf_wait_for_completion (c_clnt->rbuf,
-                                                opaque, _dispatcher, c_clnt);
-                if (ret)
-                        gf_log (this->name, GF_LOG_WARNING,
-                                "failed to put buffer after consumption");
+    int ret = 0;
+    void *opaque = NULL;
+    xlator_t *this = NULL;
+    changelog_clnt_t *c_clnt = NULL;
+    struct timeval tv = {
+        0,
+    };
+
+    c_clnt = data;
+    this = c_clnt->this;
+
+    while (1) {
+        /* TODO: change this to be pthread cond based.. later */
+
+        tv.tv_sec = 1;
+        tv.tv_usec = 0;
+        select(0, NULL, NULL, NULL, &tv);
+
+        ret = rbuf_get_buffer(c_clnt->rbuf, &opaque, sequencer, c_clnt);
+        if (ret != RBUF_CONSUMABLE) {
+            if (ret != RBUF_EMPTY)
+                gf_smsg(this->name, GF_LOG_WARNING, 0,
+                        CHANGELOG_MSG_BUFFER_STARVATION_ERROR,
+                        "Failed to get buffer for RPC dispatch",
+                        "rbuf_retval=%d", ret, NULL);
+            continue;
         }
 
-        return NULL;
+        ret = rbuf_wait_for_completion(c_clnt->rbuf, opaque, _dispatcher,
+                                       c_clnt);
+        if (ret)
+            gf_smsg(this->name, GF_LOG_WARNING, 0,
+                    CHANGELOG_MSG_PUT_BUFFER_FAILED, NULL);
+    }
+
+    return NULL;
 }
 
 void
-changelog_ev_queue_connection (changelog_clnt_t *c_clnt,
-                               changelog_rpc_clnt_t *crpc)
+changelog_ev_queue_connection(changelog_clnt_t *c_clnt,
+                              changelog_rpc_clnt_t *crpc)
 {
-        pthread_mutex_lock (&c_clnt->pending_lock);
-        {
-                list_add_tail (&crpc->list, &c_clnt->pending);
-                pthread_cond_signal (&c_clnt->pending_cond);
-        }
-        pthread_mutex_unlock (&c_clnt->pending_lock);
+    pthread_mutex_lock(&c_clnt->pending_lock);
+    {
+        list_add_tail(&crpc->list, &c_clnt->pending);
+        pthread_cond_signal(&c_clnt->pending_cond);
+    }
+    pthread_mutex_unlock(&c_clnt->pending_lock);
 }
 
 struct rpc_clnt_procedure changelog_ev_procs[CHANGELOG_REV_PROC_MAX] = {
-        [CHANGELOG_REV_PROC_NULL]  = {"NULL", NULL},
-        [CHANGELOG_REV_PROC_EVENT] = {
-                "EVENT DISPATCH", changelog_event_dispatch_rpc
-        },
+    [CHANGELOG_REV_PROC_NULL] = {"NULL", NULL},
+    [CHANGELOG_REV_PROC_EVENT] = {"EVENT DISPATCH",
+                                  changelog_event_dispatch_rpc},
 };
 
 struct rpc_clnt_program changelog_ev_program = {
-        .progname  = "CHANGELOG EVENT DISPATCHER",
-        .prognum   = CHANGELOG_REV_RPC_PROCNUM,
-        .progver   = CHANGELOG_REV_RPC_PROCVER,
-        .numproc   = CHANGELOG_REV_PROC_MAX,
-        .proctable = changelog_ev_procs,
+    .progname = "CHANGELOG EVENT DISPATCHER",
+    .prognum = CHANGELOG_REV_RPC_PROCNUM,
+    .progver = CHANGELOG_REV_RPC_PROCVER,
+    .numproc = CHANGELOG_REV_PROC_MAX,
+    .proctable = changelog_ev_procs,
 };
diff --git a/xlators/features/changelog/src/changelog-ev-handle.h b/xlators/features/changelog/src/changelog-ev-handle.h
index eef0492a9ee..cc1af58a276 100644
--- a/xlators/features/changelog/src/changelog-ev-handle.h
+++ b/xlators/features/changelog/src/changelog-ev-handle.h
@@ -11,74 +11,67 @@
 #ifndef __CHANGELOG_EV_HANDLE_H
 #define __CHANGELOG_EV_HANDLE_H
 
-#include "list.h"
-#include "xlator.h"
+#include <glusterfs/list.h>
+#include <glusterfs/xlator.h>
 #include "rpc-clnt.h"
 
-#include "rot-buffs.h"
+#include <glusterfs/rot-buffs.h>
 
 struct changelog_clnt;
 
 typedef struct changelog_rpc_clnt {
-        xlator_t *this;
+    xlator_t *this;
 
-        gf_lock_t lock;
+    gf_lock_t lock;
 
-        unsigned long ref;
-        gf_boolean_t  disconnected;
+    gf_atomic_t ref;
+    gf_boolean_t disconnected;
 
-        unsigned int filter;
-        char sock[UNIX_PATH_MAX];
+    unsigned int filter;
+    char sock[UNIX_PATH_MAX];
 
-        struct changelog_clnt *c_clnt;   /* back pointer to list holder */
+    struct changelog_clnt *c_clnt; /* back pointer to list holder */
 
-        struct rpc_clnt *rpc;            /* RPC client endpoint */
+    struct rpc_clnt *rpc; /* RPC client endpoint */
 
-        struct list_head list;           /* ->pending, ->waitq, ->active */
+    struct list_head list; /* ->pending, ->waitq, ->active */
 
-        void (*cleanup)
-        (struct changelog_rpc_clnt *);   /* cleanup handler */
+    void (*cleanup)(struct changelog_rpc_clnt *); /* cleanup handler */
 } changelog_rpc_clnt_t;
 
 static inline void
-changelog_rpc_clnt_ref (changelog_rpc_clnt_t *crpc)
+changelog_rpc_clnt_ref(changelog_rpc_clnt_t *crpc)
 {
-        LOCK (&crpc->lock);
-        {
-                ++crpc->ref;
-        }
-        UNLOCK (&crpc->lock);
+    GF_ATOMIC_INC(crpc->ref);
 }
 
 static inline void
-changelog_set_disconnect_flag (changelog_rpc_clnt_t *crpc, gf_boolean_t flag)
+changelog_set_disconnect_flag(changelog_rpc_clnt_t *crpc, gf_boolean_t flag)
 {
-        crpc->disconnected = flag;
+    crpc->disconnected = flag;
 }
 
 static inline int
-changelog_rpc_clnt_is_disconnected (changelog_rpc_clnt_t *crpc)
+changelog_rpc_clnt_is_disconnected(changelog_rpc_clnt_t *crpc)
 {
-        return (crpc->disconnected == _gf_true);
+    return (crpc->disconnected == _gf_true);
 }
 
 static inline void
-changelog_rpc_clnt_unref (changelog_rpc_clnt_t *crpc)
+changelog_rpc_clnt_unref(changelog_rpc_clnt_t *crpc)
 {
-        gf_boolean_t gone = _gf_false;
-
-        LOCK (&crpc->lock);
-        {
-                if (!(--crpc->ref)
-                    && changelog_rpc_clnt_is_disconnected (crpc)) {
-                        list_del (&crpc->list);
-                        gone = _gf_true;
-                }
-        }
-        UNLOCK (&crpc->lock);
-
-        if (gone)
-                crpc->cleanup (crpc);
+    gf_boolean_t gone = _gf_false;
+    uint64_t ref = 0;
+
+    ref = GF_ATOMIC_DEC(crpc->ref);
+
+    if (!ref && changelog_rpc_clnt_is_disconnected(crpc)) {
+        list_del(&crpc->list);
+        gone = _gf_true;
+    }
+
+    if (gone)
+        crpc->cleanup(crpc);
 }
 
 /**
@@ -106,35 +99,38 @@ changelog_rpc_clnt_unref (changelog_rpc_clnt_t *crpc)
  */
 
 typedef struct changelog_clnt {
-        xlator_t *this;
+    xlator_t *this;
 
-        /* pending connections */
-        pthread_mutex_t pending_lock;
-        pthread_cond_t pending_cond;
-        struct list_head pending;
+    /* pending connections */
+    pthread_mutex_t pending_lock;
+    pthread_cond_t pending_cond;
+    struct list_head pending;
 
-        /* current active connections */
-        gf_lock_t active_lock;
-        struct list_head active;
+    /* current active connections */
+    gf_lock_t active_lock;
+    struct list_head active;
 
-        gf_lock_t wait_lock;
-        struct list_head waitq;
+    gf_lock_t wait_lock;
+    struct list_head waitq;
 
-        /* consumer part of rot-buffs */
-        rbuf_t *rbuf;
-        unsigned long sequence;
+    /* consumer part of rot-buffs */
+    rbuf_t *rbuf;
+    unsigned long sequence;
 } changelog_clnt_t;
 
-void *changelog_ev_connector (void *);
+void *
+changelog_ev_connector(void *);
 
-void *changelog_ev_dispatch (void *);
+void *
+changelog_ev_dispatch(void *);
 
 /* APIs */
 void
-changelog_ev_queue_connection (changelog_clnt_t *, changelog_rpc_clnt_t *);
+changelog_ev_queue_connection(changelog_clnt_t *, changelog_rpc_clnt_t *);
 
 void
-changelog_ev_cleanup_connections (xlator_t *, changelog_clnt_t *);
+changelog_ev_cleanup_connections(xlator_t *, changelog_clnt_t *);
 
+void
+changelog_process_cleanup_event(xlator_t *);
 #endif
-
diff --git a/xlators/features/changelog/src/changelog-helpers.c b/xlators/features/changelog/src/changelog-helpers.c
index ca326043acc..e561997d858 100644
--- a/xlators/features/changelog/src/changelog-helpers.c
+++ b/xlators/features/changelog/src/changelog-helpers.c
@@ -8,466 +8,812 @@
    cases as published by the Free Software Foundation.
 */
 
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
-
-#include "xlator.h"
-#include "defaults.h"
-#include "logging.h"
-#include "iobuf.h"
-#include "syscall.h"
+#include <glusterfs/xlator.h>
+#include <glusterfs/defaults.h>
+#include <glusterfs/logging.h>
+#include <glusterfs/iobuf.h>
+#include <glusterfs/syscall.h>
 
 #include "changelog-helpers.h"
 #include "changelog-encoders.h"
 #include "changelog-mem-types.h"
+#include "changelog-messages.h"
 
 #include "changelog-encoders.h"
 #include "changelog-rpc-common.h"
 #include <pthread.h>
-
-static inline void
-__mask_cancellation (xlator_t *this)
-{
-        int ret = 0;
-
-        ret = pthread_setcancelstate (PTHREAD_CANCEL_DISABLE, NULL);
-        if (ret)
-                gf_log (this->name, GF_LOG_WARNING,
-                        "failed to disable thread cancellation");
-}
-
-static inline void
-__unmask_cancellation (xlator_t *this)
-{
-        int ret = 0;
-
-        ret = pthread_setcancelstate (PTHREAD_CANCEL_ENABLE, NULL);
-        if (ret)
-                gf_log (this->name, GF_LOG_WARNING,
-                        "failed to enable thread cancellation");
-}
+#include <time.h>
 
 static void
-changelog_cleanup_free_mutex (void *arg_mutex)
+changelog_cleanup_free_mutex(void *arg_mutex)
 {
-    pthread_mutex_t *p_mutex = (pthread_mutex_t*) arg_mutex;
+    pthread_mutex_t *p_mutex = (pthread_mutex_t *)arg_mutex;
 
     if (p_mutex)
-            pthread_mutex_unlock(p_mutex);
+        pthread_mutex_unlock(p_mutex);
 }
 
 int
-changelog_thread_cleanup (xlator_t *this, pthread_t thr_id)
+changelog_thread_cleanup(xlator_t *this, pthread_t thr_id)
 {
-        int   ret    = 0;
-        void *retval = NULL;
-
-        /* send a cancel request to the thread */
-        ret = pthread_cancel (thr_id);
-        if (ret != 0) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "could not cancel thread (reason: %s)",
-                        strerror (errno));
-                goto out;
-        }
-
-        ret = pthread_join (thr_id, &retval);
-        if ((ret != 0) || (retval != PTHREAD_CANCELED)) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "cancel request not adhered as expected"
-                        " (reason: %s)", strerror (errno));
-        }
+    int ret = 0;
+    void *retval = NULL;
+
+    /* send a cancel request to the thread */
+    ret = pthread_cancel(thr_id);
+    if (ret != 0) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno,
+                CHANGELOG_MSG_PTHREAD_CANCEL_FAILED, NULL);
+        goto out;
+    }
+
+    ret = pthread_join(thr_id, &retval);
+    if ((ret != 0) || (retval != PTHREAD_CANCELED)) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno,
+                CHANGELOG_MSG_PTHREAD_CANCEL_FAILED, NULL);
+    }
 
- out:
-        return ret;
+out:
+    return ret;
 }
 
-inline void *
-changelog_get_usable_buffer (changelog_local_t *local)
+void *
+changelog_get_usable_buffer(changelog_local_t *local)
 {
-        changelog_log_data_t *cld = NULL;
+    changelog_log_data_t *cld = NULL;
 
-        if (!local)
-                return NULL;
-
-        cld = &local->cld;
-        if (!cld->cld_iobuf)
-                return NULL;
+    if (!local)
+        return NULL;
 
-        return cld->cld_iobuf->ptr;
-}
+    cld = &local->cld;
+    if (!cld->cld_iobuf)
+        return NULL;
 
-static inline int
-changelog_selector_index (unsigned int selector)
-{
-        return (ffs (selector) - 1);
+    return cld->cld_iobuf->ptr;
 }
 
-inline int
-changelog_ev_selected (xlator_t *this,
-                       changelog_ev_selector_t *selection,
-                       unsigned int selector)
+static int
+changelog_selector_index(unsigned int selector)
 {
-        int idx = 0;
-
-        idx = changelog_selector_index (selector);
-        gf_log (this->name, GF_LOG_DEBUG,
-                "selector ref count for %d (idx: %d): %d",
-                selector, idx, selection->ref[idx]);
-        /* this can be lockless */
-        return (idx < CHANGELOG_EV_SELECTION_RANGE
-                 && (selection->ref[idx] > 0));
+    return (ffs(selector) - 1);
 }
 
-inline void
-changelog_select_event (xlator_t *this,
-                        changelog_ev_selector_t *selection,
-                        unsigned int selector)
+int
+changelog_ev_selected(xlator_t *this, changelog_ev_selector_t *selection,
+                      unsigned int selector)
 {
-        int idx = 0;
+    int idx = 0;
 
-        LOCK (&selection->reflock);
-        {
-                while (selector) {
-                        idx = changelog_selector_index (selector);
-                        if (idx < CHANGELOG_EV_SELECTION_RANGE) {
-                                selection->ref[idx]++;
-                                gf_log (this->name, GF_LOG_DEBUG,
-                                        "selecting event %d", idx);
-                        }
-                        selector &= ~(1 << idx);
-                }
-        }
-        UNLOCK (&selection->reflock);
+    idx = changelog_selector_index(selector);
+    gf_msg_debug(this->name, 0, "selector ref count for %d (idx: %d): %d",
+                 selector, idx, selection->ref[idx]);
+    /* this can be lockless */
+    return (idx < CHANGELOG_EV_SELECTION_RANGE && (selection->ref[idx] > 0));
 }
 
-inline void
-changelog_deselect_event (xlator_t *this,
-                          changelog_ev_selector_t *selection,
-                          unsigned int selector)
+void
+changelog_select_event(xlator_t *this, changelog_ev_selector_t *selection,
+                       unsigned int selector)
 {
-        int idx = 0;
-
-        LOCK (&selection->reflock);
-        {
-                while (selector) {
-                        idx = changelog_selector_index (selector);
-                        if (idx < CHANGELOG_EV_SELECTION_RANGE) {
-                                selection->ref[idx]--;
-                                gf_log (this->name, GF_LOG_DEBUG,
-                                        "de-selecting event %d", idx);
-                        }
-                        selector &= ~(1 << idx);
-                }
+    int idx = 0;
+
+    LOCK(&selection->reflock);
+    {
+        while (selector) {
+            idx = changelog_selector_index(selector);
+            if (idx < CHANGELOG_EV_SELECTION_RANGE) {
+                selection->ref[idx]++;
+                gf_msg_debug(this->name, 0, "selecting event %d", idx);
+            }
+            selector &= ~(1 << idx);
         }
-        UNLOCK (&selection->reflock);
+    }
+    UNLOCK(&selection->reflock);
 }
 
-inline int
-changelog_init_event_selection (xlator_t *this,
-                                changelog_ev_selector_t *selection)
+void
+changelog_deselect_event(xlator_t *this, changelog_ev_selector_t *selection,
+                         unsigned int selector)
 {
-        int ret = 0;
-        int j = CHANGELOG_EV_SELECTION_RANGE;
-
-        ret = LOCK_INIT (&selection->reflock);
-        if (ret != 0)
-                return -1;
-
-        LOCK (&selection->reflock);
-        {
-                while (j--) {
-                        selection->ref[j] = 0;
-                }
+    int idx = 0;
+
+    LOCK(&selection->reflock);
+    {
+        while (selector) {
+            idx = changelog_selector_index(selector);
+            if (idx < CHANGELOG_EV_SELECTION_RANGE) {
+                selection->ref[idx]--;
+                gf_msg_debug(this->name, 0, "de-selecting event %d", idx);
+            }
+            selector &= ~(1 << idx);
         }
-        UNLOCK (&selection->reflock);
-
-        return 0;
+    }
+    UNLOCK(&selection->reflock);
 }
 
-inline int
-changelog_cleanup_event_selection (xlator_t *this,
-                                   changelog_ev_selector_t *selection)
+int
+changelog_init_event_selection(xlator_t *this,
+                               changelog_ev_selector_t *selection)
 {
-        int ret = 0;
-        int j = CHANGELOG_EV_SELECTION_RANGE;
+    int ret = 0;
+    int j = CHANGELOG_EV_SELECTION_RANGE;
 
-        LOCK (&selection->reflock);
-        {
-                while (j--) {
-                        if (selection->ref[j] > 0)
-                                gf_log (this->name, GF_LOG_WARNING,
-                                        "changelog event selection cleaning up "
-                                        " on active references");
-                }
+    ret = LOCK_INIT(&selection->reflock);
+    if (ret != 0)
+        return -1;
+
+    LOCK(&selection->reflock);
+    {
+        while (j--) {
+            selection->ref[j] = 0;
         }
-        UNLOCK (&selection->reflock);
+    }
+    UNLOCK(&selection->reflock);
 
-        return LOCK_DESTROY (&selection->reflock);
+    return 0;
 }
 
-static inline void
-changelog_perform_dispatch (xlator_t *this,
-                            changelog_priv_t *priv, void *mem, size_t size)
+static void
+changelog_perform_dispatch(xlator_t *this, changelog_priv_t *priv, void *mem,
+                           size_t size)
 {
-        char *buf    = NULL;
-        void *opaque = NULL;
-
-        buf = rbuf_reserve_write_area (priv->rbuf, size, &opaque);
-        if (!buf) {
-                gf_log_callingfn (this->name,
-                                  GF_LOG_WARNING, "failed to dispatch event");
-                return;
-        }
+    char *buf = NULL;
+    void *opaque = NULL;
+
+    buf = rbuf_reserve_write_area(priv->rbuf, size, &opaque);
+    if (!buf) {
+        gf_msg_callingfn(this->name, GF_LOG_WARNING, 0,
+                         CHANGELOG_MSG_DISPATCH_EVENT_FAILED,
+                         "failed to dispatch event");
+        return;
+    }
 
-        memcpy (buf, mem, size);
-        rbuf_write_complete (opaque);
+    memcpy(buf, mem, size);
+    rbuf_write_complete(opaque);
 }
 
-inline void
-changelog_dispatch_event (xlator_t *this,
-                          changelog_priv_t *priv, changelog_event_t *ev)
+void
+changelog_dispatch_event(xlator_t *this, changelog_priv_t *priv,
+                         changelog_event_t *ev)
 {
-        changelog_ev_selector_t *selection = NULL;
+    changelog_ev_selector_t *selection = NULL;
 
-        selection = &priv->ev_selection;
-        if (changelog_ev_selected (this, selection, ev->ev_type)) {
-                changelog_perform_dispatch (this, priv, ev, CHANGELOG_EV_SIZE);
-        }
+    selection = &priv->ev_selection;
+    if (changelog_ev_selected(this, selection, ev->ev_type)) {
+        changelog_perform_dispatch(this, priv, ev, CHANGELOG_EV_SIZE);
+    }
 }
 
-inline void
-changelog_set_usable_record_and_length (changelog_local_t *local,
-                                        size_t len, int xr)
+void
+changelog_set_usable_record_and_length(changelog_local_t *local, size_t len,
+                                       int xr)
 {
-        changelog_log_data_t *cld = NULL;
+    changelog_log_data_t *cld = NULL;
 
-        cld = &local->cld;
+    cld = &local->cld;
 
-        cld->cld_ptr_len = len;
-        cld->cld_xtra_records = xr;
+    cld->cld_ptr_len = len;
+    cld->cld_xtra_records = xr;
 }
 
 void
-changelog_local_cleanup (xlator_t *xl, changelog_local_t *local)
+changelog_local_cleanup(xlator_t *xl, changelog_local_t *local)
 {
-        int                   i   = 0;
-        changelog_opt_t      *co  = NULL;
-        changelog_log_data_t *cld = NULL;
+    int i = 0;
+    changelog_opt_t *co = NULL;
+    changelog_log_data_t *cld = NULL;
 
-        if (!local)
-                return;
+    if (!local)
+        return;
 
-        cld = &local->cld;
+    cld = &local->cld;
 
-        /* cleanup dynamic allocation for extra records */
-        if (cld->cld_xtra_records) {
-                co = (changelog_opt_t *) cld->cld_ptr;
-                for (; i < cld->cld_xtra_records; i++, co++)
-                        if (co->co_free)
-                                co->co_free (co);
-        }
+    /* cleanup dynamic allocation for extra records */
+    if (cld->cld_xtra_records) {
+        co = (changelog_opt_t *)cld->cld_ptr;
+        for (; i < cld->cld_xtra_records; i++, co++)
+            if (co->co_free)
+                co->co_free(co);
+    }
 
-        CHANGELOG_IOBUF_UNREF (cld->cld_iobuf);
+    CHANGELOG_IOBUF_UNREF(cld->cld_iobuf);
 
-        if (local->inode)
-                inode_unref (local->inode);
+    if (local->inode)
+        inode_unref(local->inode);
 
-        mem_put (local);
+    mem_put(local);
 }
 
-inline int
-changelog_write (int fd, char *buffer, size_t len)
+int
+changelog_write(int fd, char *buffer, size_t len)
 {
-        ssize_t size = 0;
-        size_t written = 0;
+    ssize_t size = 0;
+    size_t written = 0;
 
-        while (written < len) {
-                size = write (fd,
-                              buffer + written, len - written);
-                if (size <= 0)
-                        break;
+    while (written < len) {
+        size = sys_write(fd, buffer + written, len - written);
+        if (size <= 0)
+            break;
 
-                written += size;
-        }
+        written += size;
+    }
 
-        return (written != len);
+    return (written != len);
 }
 
 int
-htime_update (xlator_t *this,
-              changelog_priv_t *priv, unsigned long ts,
-              char * buffer)
+htime_update(xlator_t *this, changelog_priv_t *priv, time_t ts, char *buffer)
 {
-        char changelog_path[PATH_MAX+1]   = {0,};
-        int len                           = -1;
-        char x_value[25]                  = {0,};
-        /* time stamp(10) + : (1) + rolltime (12 ) + buffer (2) */
-        int ret                           = 0;
-
-        if (priv->htime_fd ==-1) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "Htime fd not available for updation");
-                ret = -1;
-                goto out;
-        }
-        strcpy (changelog_path, buffer);
-        len = strlen (changelog_path);
-        changelog_path[len] = '\0'; /* redundant */
-
-        if (changelog_write (priv->htime_fd, (void*) changelog_path, len+1 ) < 0) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "Htime file content write failed");
-                ret =-1;
-                goto out;
+    char changelog_path[PATH_MAX + 1] = {
+        0,
+    };
+    int len = -1;
+    char x_value[25] = {
+        0,
+    };
+    /* time stamp(10) + : (1) + rolltime (12 ) + buffer (2) */
+    int ret = 0;
+
+    if (priv->htime_fd == -1) {
+        gf_smsg(this->name, GF_LOG_ERROR, 0, CHANGELOG_MSG_HTIME_ERROR,
+                "reason=fd not available", NULL);
+        ret = -1;
+        goto out;
+    }
+    len = snprintf(changelog_path, PATH_MAX, "%s", buffer);
+    if (len >= PATH_MAX) {
+        ret = -1;
+        goto out;
+    }
+    if (changelog_write(priv->htime_fd, (void *)changelog_path, len + 1) < 0) {
+        gf_smsg(this->name, GF_LOG_ERROR, 0, CHANGELOG_MSG_HTIME_ERROR,
+                "reason=write failed", NULL);
+        ret = -1;
+        goto out;
+    }
+
+    len = snprintf(x_value, sizeof(x_value), "%ld:%d", ts,
+                   priv->rollover_count);
+    if (len >= sizeof(x_value)) {
+        ret = -1;
+        goto out;
+    }
+
+    if (sys_fsetxattr(priv->htime_fd, HTIME_KEY, x_value, len, XATTR_REPLACE)) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, CHANGELOG_MSG_HTIME_ERROR,
+                "reason=xattr updation failed", "XATTR_REPLACE=true",
+                "changelog=%s", changelog_path, NULL);
+
+        if (sys_fsetxattr(priv->htime_fd, HTIME_KEY, x_value, len, 0)) {
+            gf_smsg(this->name, GF_LOG_ERROR, errno, CHANGELOG_MSG_HTIME_ERROR,
+                    "reason=xattr updation failed", "changelog=%s",
+                    changelog_path, NULL);
+            ret = -1;
+            goto out;
         }
+    }
 
-        sprintf (x_value,"%lu:%d",ts, priv->rollover_count);
+    priv->rollover_count += 1;
 
-        if (sys_fsetxattr (priv->htime_fd, HTIME_KEY, x_value,
-                       strlen (x_value),  XATTR_REPLACE)) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "Htime xattr updation failed, "
-                        "reason (%s)",strerror (errno));
-                goto out;
-        }
+out:
+    return ret;
+}
+
+/*
+ * Description: Check if the changelog to rollover is empty or not.
+ * It is assumed that fd passed is already verified.
+ *
+ * Returns:
+ * 1 : If found empty, changed path from "CHANGELOG.<TS>" to "changelog.<TS>"
+ * 0 : If NOT empty, proceed usual.
+ */
+int
+cl_is_empty(xlator_t *this, int fd)
+{
+    int ret = -1;
+    size_t elen = 0;
+    int encoding = -1;
+    char buffer[1024] = {
+        0,
+    };
+    struct stat stbuf = {
+        0,
+    };
+    int major_version = -1;
+    int minor_version = -1;
+
+    ret = sys_fstat(fd, &stbuf);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, CHANGELOG_MSG_FSTAT_OP_FAILED,
+                NULL);
+        goto out;
+    }
+
+    ret = sys_lseek(fd, 0, SEEK_SET);
+    if (ret == -1) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, CHANGELOG_MSG_LSEEK_OP_FAILED,
+                NULL);
+        goto out;
+    }
+
+    CHANGELOG_GET_HEADER_INFO(fd, buffer, sizeof(buffer), encoding,
+                              major_version, minor_version, elen);
+
+    if (elen == stbuf.st_size) {
+        ret = 1;
+    } else {
+        ret = 0;
+    }
 
-        priv->rollover_count +=1;
+out:
+    return ret;
+}
 
+/*
+ * Description: Updates "CHANGELOG" to "changelog" for writing changelog path
+ * to htime file.
+ *
+ * Returns:
+ * 0  : Success
+ * -1 : Error
+ */
+int
+update_path(xlator_t *this, char *cl_path)
+{
+    const char low_cl[] = "changelog";
+    const char up_cl[] = "CHANGELOG";
+    char *found = NULL;
+    int ret = -1;
+
+    found = strstr(cl_path, up_cl);
+
+    if (found == NULL) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, CHANGELOG_MSG_PATH_NOT_FOUND,
+                NULL);
+        goto out;
+    } else {
+        memcpy(found, low_cl, sizeof(low_cl) - 1);
+    }
+
+    ret = 0;
 out:
-        return ret;
+    return ret;
 }
 
 static int
-changelog_rollover_changelog (xlator_t *this,
-                              changelog_priv_t *priv, unsigned long ts)
+changelog_rollover_changelog(xlator_t *this, changelog_priv_t *priv, time_t ts)
 {
-        int   ret            = -1;
-        int   notify         = 0;
-        char ofile[PATH_MAX] = {0,};
-        char nfile[PATH_MAX] = {0,};
-        changelog_event_t ev = {0,};
-
-        if (priv->changelog_fd != -1) {
-                ret = fsync (priv->changelog_fd);
-                if (ret < 0) {
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "fsync failed (reason: %s)",
-                                strerror (errno));
-                }
-                close (priv->changelog_fd);
-                priv->changelog_fd = -1;
+    int ret = -1;
+    int notify = 0;
+    int cl_empty_flag = 0;
+    struct tm *gmt;
+    char yyyymmdd[40];
+    char ofile[PATH_MAX] = {
+        0,
+    };
+    char nfile[PATH_MAX] = {
+        0,
+    };
+    char nfile_dir[PATH_MAX] = {
+        0,
+    };
+    changelog_event_t ev = {
+        0,
+    };
+
+    if (priv->changelog_fd != -1) {
+        ret = sys_fsync(priv->changelog_fd);
+        if (ret < 0) {
+            gf_smsg(this->name, GF_LOG_ERROR, errno,
+                    CHANGELOG_MSG_FSYNC_OP_FAILED, NULL);
+        }
+        ret = cl_is_empty(this, priv->changelog_fd);
+        if (ret == 1) {
+            cl_empty_flag = 1;
+        } else if (ret == -1) {
+            /* Log error but proceed as usual */
+            gf_smsg(this->name, GF_LOG_WARNING, 0,
+                    CHANGELOG_MSG_DETECT_EMPTY_CHANGELOG_FAILED, NULL);
         }
+        sys_close(priv->changelog_fd);
+        priv->changelog_fd = -1;
+    }
 
-        (void) snprintf (ofile, PATH_MAX,
-                         "%s/"CHANGELOG_FILE_NAME, priv->changelog_dir);
-        (void) snprintf (nfile, PATH_MAX,
-                         "%s/"CHANGELOG_FILE_NAME".%lu",
-                         priv->changelog_dir, ts);
+    /* Get GMT time. */
+    gmt = gmtime(&ts);
 
-        ret = rename (ofile, nfile);
-        if (!ret)
-                notify = 1;
+    strftime(yyyymmdd, sizeof(yyyymmdd), "%Y/%m/%d", gmt);
 
-        if (ret && (errno == ENOENT)) {
-                ret = 0;
+    (void)snprintf(ofile, PATH_MAX, "%s/" CHANGELOG_FILE_NAME,
+                   priv->changelog_dir);
+    (void)snprintf(nfile, PATH_MAX, "%s/%s/" CHANGELOG_FILE_NAME ".%ld",
+                   priv->changelog_dir, yyyymmdd, ts);
+    (void)snprintf(nfile_dir, PATH_MAX, "%s/%s", priv->changelog_dir, yyyymmdd);
+
+    if (cl_empty_flag == 1) {
+        ret = sys_unlink(ofile);
+        if (ret) {
+            gf_smsg(this->name, GF_LOG_ERROR, errno,
+                    CHANGELOG_MSG_UNLINK_OP_FAILED, "path=%s", ofile, NULL);
+            ret = 0; /* Error in unlinking empty changelog should
+                        not break further changelog operation, so
+                        reset return value to 0*/
+        }
+    } else {
+        ret = sys_rename(ofile, nfile);
+
+        /* Changelog file rename gets ENOENT when parent dir doesn't exist */
+        if (errno == ENOENT) {
+            ret = mkdir_p(nfile_dir, 0600, _gf_true);
+
+            if ((ret == -1) && (EEXIST != errno)) {
+                gf_smsg(this->name, GF_LOG_ERROR, errno,
+                        CHANGELOG_MSG_MKDIR_ERROR, "%s", nfile_dir, NULL);
                 goto out;
+            }
+
+            ret = sys_rename(ofile, nfile);
         }
 
+        if (ret && (errno == ENOENT)) {
+            ret = 0;
+            goto out;
+        }
         if (ret) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "error renaming %s -> %s (reason %s)",
-                        ofile, nfile, strerror (errno));
+            gf_smsg(this->name, GF_LOG_ERROR, errno, CHANGELOG_MSG_RENAME_ERROR,
+                    "from=%s", ofile, "to=%s", nfile, NULL);
         }
+    }
 
-        if (!ret) {
-                ret = htime_update (this, priv, ts, nfile);
-                if (ret == -1) {
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "could not update htime file");
-                        goto out;
-                }
+    if (!ret && (cl_empty_flag == 0)) {
+        notify = 1;
+    }
+
+    if (!ret) {
+        if (cl_empty_flag) {
+            update_path(this, nfile);
+        }
+        ret = htime_update(this, priv, ts, nfile);
+        if (ret == -1) {
+            gf_smsg(this->name, GF_LOG_ERROR, 0, CHANGELOG_MSG_HTIME_ERROR,
+                    NULL);
+            goto out;
         }
+    }
 
-        if (notify) {
-                ev.ev_type = CHANGELOG_OP_TYPE_JOURNAL;
-                memcpy (ev.u.journal.path, nfile, strlen (nfile) + 1);
-                changelog_dispatch_event (this, priv, &ev);
-
-                /* If this is explicit rollover initiated by snapshot,
-                 * wakeup reconfigure thread waiting for changelog to
-                 * rollover
-                 */
-                if (priv->explicit_rollover) {
-                        priv->explicit_rollover = _gf_false;
-
-                        ret = pthread_mutex_lock (&priv->bn.bnotify_mutex);
-                        CHANGELOG_PTHREAD_ERROR_HANDLE_0 (ret, out);
-                        {
-                                priv->bn.bnotify = _gf_false;
-                                ret = pthread_cond_signal
-                                        (&priv->bn.bnotify_cond);
-                                CHANGELOG_PTHREAD_ERROR_HANDLE_0 (ret, out);
-                                gf_log (this->name, GF_LOG_INFO,
-                                        "Changelog published: %s signalled"
-                                        " bnotify", nfile);
-                        }
-                        ret = pthread_mutex_unlock (&priv->bn.bnotify_mutex);
-                        CHANGELOG_PTHREAD_ERROR_HANDLE_0 (ret, out);
-                }
+    if (notify) {
+        ev.ev_type = CHANGELOG_OP_TYPE_JOURNAL;
+        memcpy(ev.u.journal.path, nfile, strlen(nfile) + 1);
+        changelog_dispatch_event(this, priv, &ev);
+    }
+out:
+    /* If this is explicit rollover initiated by snapshot,
+     * wakeup reconfigure thread waiting for changelog to
+     * rollover. This should happen even in failure cases as
+     * well otherwise snapshot will timeout and fail. Hence
+     * moved under out.
+     */
+    if (priv->explicit_rollover) {
+        priv->explicit_rollover = _gf_false;
+
+        pthread_mutex_lock(&priv->bn.bnotify_mutex);
+        {
+            if (ret) {
+                priv->bn.bnotify_error = _gf_true;
+                gf_smsg(this->name, GF_LOG_ERROR, 0,
+                        CHANGELOG_MSG_EXPLICIT_ROLLOVER_FAILED, NULL);
+            } else {
+                gf_smsg(this->name, GF_LOG_INFO, 0, CHANGELOG_MSG_BNOTIFY_INFO,
+                        "changelog=%s", nfile, NULL);
+            }
+            priv->bn.bnotify = _gf_false;
+            pthread_cond_signal(&priv->bn.bnotify_cond);
         }
- out:
-        return ret;
+        pthread_mutex_unlock(&priv->bn.bnotify_mutex);
+    }
+    return ret;
 }
 
-/* Returns 0 on successful creation of htime file
- * returns -1 on failure or error
+int
+filter_cur_par_dirs(const struct dirent *entry)
+{
+    if (entry == NULL)
+        return 0;
+
+    if ((strcmp(entry->d_name, ".") == 0) || (strcmp(entry->d_name, "..") == 0))
+        return 0;
+    else
+        return 1;
+}
+
+/*
+ * find_current_htime:
+ *       It finds the latest htime file and sets the HTIME_CURRENT
+ *       xattr.
+ *       RETURN VALUE:
+ *           -1 : Error
+ *           ret: Number of directory entries;
  */
+
 int
-htime_open (xlator_t *this,
-            changelog_priv_t * priv, unsigned long ts)
+find_current_htime(int ht_dir_fd, const char *ht_dir_path, char *ht_file_bname)
 {
-        int fd                          = -1;
-        int ret                         = 0;
-        char ht_dir_path[PATH_MAX]      = {0,};
-        char ht_file_path[PATH_MAX]     = {0,};
-        int flags                       = 0;
-
-        CHANGELOG_FILL_HTIME_DIR(priv->changelog_dir, ht_dir_path);
-
-        /* get the htime file name in ht_file_path */
-        (void) snprintf (ht_file_path,PATH_MAX,"%s/%s.%lu",ht_dir_path,
-                        HTIME_FILE_NAME, ts);
-
-        flags |= (O_CREAT | O_RDWR | O_SYNC);
-        fd = open (ht_file_path, flags,
-                        S_IRUSR | S_IWUSR | S_IRGRP | S_IROTH);
-        if (fd < 0) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "unable to open/create htime file: %s"
-                        "(reason: %s)", ht_file_path, strerror (errno));
-                ret = -1;
-                goto out;
+    struct dirent **namelist = NULL;
+    int ret = 0;
+    int cnt = 0;
+    int i = 0;
+    xlator_t *this = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+    GF_ASSERT(ht_dir_path);
+
+    cnt = scandir(ht_dir_path, &namelist, filter_cur_par_dirs, alphasort);
+    if (cnt < 0) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, CHANGELOG_MSG_SCAN_DIR_FAILED,
+                NULL);
+    } else if (cnt > 0) {
+        if (snprintf(ht_file_bname, NAME_MAX, "%s",
+                     namelist[cnt - 1]->d_name) >= NAME_MAX) {
+            ret = -1;
+            goto out;
+        }
+        if (sys_fsetxattr(ht_dir_fd, HTIME_CURRENT, ht_file_bname,
+                          strlen(ht_file_bname), 0)) {
+            gf_smsg(this->name, GF_LOG_ERROR, errno,
+                    CHANGELOG_MSG_FSETXATTR_FAILED, "HTIME_CURRENT", NULL);
+            ret = -1;
+            goto out;
+        }
 
+        if (sys_fsync(ht_dir_fd) < 0) {
+            gf_smsg(this->name, GF_LOG_ERROR, errno,
+                    CHANGELOG_MSG_FSYNC_OP_FAILED, NULL);
+            ret = -1;
+            goto out;
         }
+    }
 
-        if (sys_fsetxattr (fd, HTIME_KEY, HTIME_INITIAL_VALUE,
-                       sizeof (HTIME_INITIAL_VALUE)-1,  0)) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "Htime xattr initialization failed");
-                ret = -1;
-                goto out;
+out:
+    for (i = 0; i < cnt; i++)
+        free(namelist[i]);
+    free(namelist);
+
+    if (ret)
+        cnt = ret;
+
+    return cnt;
+}
+
+/* Returns 0 on successful open of htime file
+ * returns -1 on failure or error
+ */
+int
+htime_open(xlator_t *this, changelog_priv_t *priv, time_t ts)
+{
+    int ht_file_fd = -1;
+    int ht_dir_fd = -1;
+    int ret = 0;
+    int cnt = 0;
+    char ht_dir_path[PATH_MAX] = {
+        0,
+    };
+    char ht_file_path[PATH_MAX] = {
+        0,
+    };
+    char ht_file_bname[NAME_MAX] = {
+        0,
+    };
+    char x_value[NAME_MAX] = {
+        0,
+    };
+    int flags = 0;
+    unsigned long min_ts = 0;
+    unsigned long max_ts = 0;
+    unsigned long total = 0;
+    unsigned long total1 = 0;
+    ssize_t size = 0;
+    struct stat stat_buf = {
+        0,
+    };
+    unsigned long record_len = 0;
+    int32_t len = 0;
+
+    CHANGELOG_FILL_HTIME_DIR(priv->changelog_dir, ht_dir_path);
+
+    /* Open htime directory to get HTIME_CURRENT */
+    ht_dir_fd = open(ht_dir_path, O_RDONLY);
+    if (ht_dir_fd == -1) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, CHANGELOG_MSG_OPEN_FAILED,
+                "path=%s", ht_dir_path, NULL);
+        ret = -1;
+        goto out;
+    }
+
+    size = sys_fgetxattr(ht_dir_fd, HTIME_CURRENT, ht_file_bname,
+                         sizeof(ht_file_bname));
+    if (size < 0) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, CHANGELOG_MSG_FGETXATTR_FAILED,
+                "name=HTIME_CURRENT", NULL);
+
+        /* If upgrade scenario, find the latest HTIME.TSTAMP file
+         * and use the same. If error, create a new HTIME.TSTAMP
+         * file.
+         */
+        cnt = find_current_htime(ht_dir_fd, ht_dir_path, ht_file_bname);
+        if (cnt <= 0) {
+            gf_smsg(this->name, GF_LOG_INFO, errno,
+                    CHANGELOG_MSG_NO_HTIME_CURRENT, NULL);
+            sys_close(ht_dir_fd);
+            return htime_create(this, priv, ts);
         }
 
-        /* save this htime_fd in priv->htime_fd */
-        priv->htime_fd = fd;
-        /* initialize rollover-number in priv to 1 */
-        priv->rollover_count = 1;
+        gf_smsg(this->name, GF_LOG_ERROR, errno,
+                CHANGELOG_MSG_HTIME_CURRENT_ERROR, NULL);
+    }
+
+    gf_smsg(this->name, GF_LOG_INFO, 0, CHANGELOG_MSG_HTIME_CURRENT, "path=%s",
+            ht_file_bname, NULL);
+    len = snprintf(ht_file_path, PATH_MAX, "%s/%s", ht_dir_path, ht_file_bname);
+    if ((len < 0) || (len >= PATH_MAX)) {
+        ret = -1;
+        goto out;
+    }
+
+    /* Open in append mode as existing htime file is used */
+    flags |= (O_RDWR | O_SYNC | O_APPEND);
+    ht_file_fd = open(ht_file_path, flags,
+                      S_IRUSR | S_IWUSR | S_IRGRP | S_IROTH);
+    if (ht_file_fd < 0) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, CHANGELOG_MSG_OPEN_FAILED,
+                "path=%s", ht_file_path, NULL);
+        ret = -1;
+        goto out;
+    }
+
+    /* save this htime_fd in priv->htime_fd */
+    priv->htime_fd = ht_file_fd;
+
+    ret = sys_fstat(ht_file_fd, &stat_buf);
+    if (ret < 0) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, CHANGELOG_MSG_HTIME_STAT_ERROR,
+                "path=%s", ht_file_path, NULL);
+        ret = -1;
+        goto out;
+    }
+
+    /* Initialize rollover-number in priv to current number */
+    size = sys_fgetxattr(ht_file_fd, HTIME_KEY, x_value, sizeof(x_value));
+    if (size < 0) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, CHANGELOG_MSG_FGETXATTR_FAILED,
+                "name=%s", HTIME_KEY, "path=%s", ht_file_path, NULL);
+        ret = -1;
+        goto out;
+    }
+
+    sscanf(x_value, "%lu:%lu", &max_ts, &total);
+
+    /* 22 = 1(/) + 20(CHANGELOG.TIMESTAMP) + 1(\x00) */
+    record_len = strlen(priv->changelog_dir) + 22;
+    total1 = stat_buf.st_size / record_len;
+    if (total != total1) {
+        gf_smsg(this->name, GF_LOG_INFO, 0, CHANGELOG_MSG_TOTAL_LOG_INFO,
+                "xattr_total=%lu", total, "size_total=%lu", total1, NULL);
+    }
+
+    gf_smsg(this->name, GF_LOG_INFO, 0, CHANGELOG_MSG_TOTAL_LOG_INFO, "min=%lu",
+            min_ts, "max=%lu", max_ts, "total_changelogs=%lu", total, NULL);
+
+    if (total < total1)
+        priv->rollover_count = total1 + 1;
+    else
+        priv->rollover_count = total + 1;
 
 out:
-        return ret;
+    if (ht_dir_fd != -1)
+        sys_close(ht_dir_fd);
+    return ret;
+}
+
+/* Returns 0 on successful creation of htime file
+ * returns -1 on failure or error
+ */
+int
+htime_create(xlator_t *this, changelog_priv_t *priv, time_t ts)
+{
+    int ht_file_fd = -1;
+    int ht_dir_fd = -1;
+    int ret = 0;
+    char ht_dir_path[PATH_MAX] = {
+        0,
+    };
+    char ht_file_path[PATH_MAX] = {
+        0,
+    };
+    char ht_file_bname[NAME_MAX + 1] = {
+        0,
+    };
+    int flags = 0;
+    int32_t len = 0;
+
+    gf_smsg(this->name, GF_LOG_INFO, 0, CHANGELOG_MSG_NEW_HTIME_FILE,
+            "name=%ld", ts, NULL);
+
+    CHANGELOG_FILL_HTIME_DIR(priv->changelog_dir, ht_dir_path);
+
+    /* get the htime file name in ht_file_path */
+    len = snprintf(ht_file_path, PATH_MAX, "%s/%s.%ld", ht_dir_path,
+                   HTIME_FILE_NAME, ts);
+    if ((len < 0) || (len >= PATH_MAX)) {
+        ret = -1;
+        goto out;
+    }
+
+    flags |= (O_CREAT | O_RDWR | O_SYNC);
+    ht_file_fd = open(ht_file_path, flags,
+                      S_IRUSR | S_IWUSR | S_IRGRP | S_IROTH);
+    if (ht_file_fd < 0) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, CHANGELOG_MSG_OPEN_FAILED,
+                "path=%s", ht_file_path, NULL);
+        ret = -1;
+        goto out;
+    }
+
+    if (sys_fsetxattr(ht_file_fd, HTIME_KEY, HTIME_INITIAL_VALUE,
+                      sizeof(HTIME_INITIAL_VALUE) - 1, 0)) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno,
+                CHANGELOG_MSG_XATTR_INIT_FAILED, NULL);
+        ret = -1;
+        goto out;
+    }
+
+    ret = sys_fsync(ht_file_fd);
+    if (ret < 0) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, CHANGELOG_MSG_FSYNC_OP_FAILED,
+                NULL);
+        goto out;
+    }
+
+    /* save this htime_fd in priv->htime_fd */
+    priv->htime_fd = ht_file_fd;
+
+    ht_file_fd = -1;
+
+    /* Set xattr HTIME_CURRENT on htime directory to htime filename */
+    ht_dir_fd = open(ht_dir_path, O_RDONLY);
+    if (ht_dir_fd == -1) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, CHANGELOG_MSG_OPEN_FAILED,
+                "path=%s", ht_dir_path, NULL);
+        ret = -1;
+        goto out;
+    }
+
+    (void)snprintf(ht_file_bname, sizeof(ht_file_bname), "%s.%ld",
+                   HTIME_FILE_NAME, ts);
+    if (sys_fsetxattr(ht_dir_fd, HTIME_CURRENT, ht_file_bname,
+                      strlen(ht_file_bname), 0)) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, CHANGELOG_MSG_FSETXATTR_FAILED,
+                " HTIME_CURRENT", NULL);
+        ret = -1;
+        goto out;
+    }
+
+    ret = sys_fsync(ht_dir_fd);
+    if (ret < 0) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, CHANGELOG_MSG_FSYNC_OP_FAILED,
+                NULL);
+        goto out;
+    }
+
+    /* initialize rollover-number in priv to 1 */
+    priv->rollover_count = 1;
+
+out:
+    if (ht_dir_fd != -1)
+        sys_close(ht_dir_fd);
+    if (ht_file_fd != -1)
+        sys_close(ht_file_fd);
+    return ret;
 }
 
 /* Description:
@@ -479,48 +825,53 @@ out:
  *      -1  : On failure.
  */
 int
-changelog_snap_open (xlator_t *this,
-                         changelog_priv_t *priv)
+changelog_snap_open(xlator_t *this, changelog_priv_t *priv)
 {
-        int fd                        = -1;
-        int ret                       = 0;
-        int flags                     = 0;
-        char buffer[1024]             = {0,};
-        char c_snap_path[PATH_MAX]    = {0,};
-        char csnap_dir_path[PATH_MAX] = {0,};
-
-        CHANGELOG_FILL_CSNAP_DIR(priv->changelog_dir, csnap_dir_path);
-
-        (void) snprintf (c_snap_path, PATH_MAX,
-                        "%s/"CSNAP_FILE_NAME,
-                        csnap_dir_path);
-
-        flags |= (O_CREAT | O_RDWR | O_TRUNC);
-
-        fd = open (c_snap_path, flags,
-                        S_IRUSR | S_IWUSR | S_IRGRP | S_IROTH);
-        if (fd < 0) {
-                gf_log (this->name, GF_LOG_ERROR,
-                                "unable to open %s file "
-                                "reason:(%s)", c_snap_path, strerror (errno));
-                ret = -1;
-                goto out;
-        }
-        priv->c_snap_fd = fd;
-
-        (void) snprintf (buffer, 1024, CHANGELOG_HEADER,
-                        CHANGELOG_VERSION_MAJOR,
-                        CHANGELOG_VERSION_MINOR,
-                        priv->ce->encoder);
-        ret = changelog_snap_write_change (priv, buffer, strlen (buffer));
-        if (ret < 0) {
-                close (priv->c_snap_fd);
-                priv->c_snap_fd = -1;
-                goto out;
-        }
+    int fd = -1;
+    int ret = 0;
+    int flags = 0;
+    char buffer[1024] = {
+        0,
+    };
+    char c_snap_path[PATH_MAX] = {
+        0,
+    };
+    char csnap_dir_path[PATH_MAX] = {
+        0,
+    };
+    int32_t len = 0;
+
+    CHANGELOG_FILL_CSNAP_DIR(priv->changelog_dir, csnap_dir_path);
+
+    len = snprintf(c_snap_path, PATH_MAX, "%s/" CSNAP_FILE_NAME,
+                   csnap_dir_path);
+    if ((len < 0) || (len >= PATH_MAX)) {
+        ret = -1;
+        goto out;
+    }
+
+    flags |= (O_CREAT | O_RDWR | O_TRUNC);
+
+    fd = open(c_snap_path, flags, S_IRUSR | S_IWUSR | S_IRGRP | S_IROTH);
+    if (fd < 0) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, CHANGELOG_MSG_OPEN_FAILED,
+                "path=%s", c_snap_path, NULL);
+        ret = -1;
+        goto out;
+    }
+    priv->c_snap_fd = fd;
+
+    (void)snprintf(buffer, 1024, CHANGELOG_HEADER, CHANGELOG_VERSION_MAJOR,
+                   CHANGELOG_VERSION_MINOR, priv->ce->encoder);
+    ret = changelog_snap_write_change(priv, buffer, strlen(buffer));
+    if (ret < 0) {
+        sys_close(priv->c_snap_fd);
+        priv->c_snap_fd = -1;
+        goto out;
+    }
 
 out:
-        return ret;
+    return ret;
 }
 
 /*
@@ -531,16 +882,15 @@ out:
  *      -1 : On Failure.
  */
 int
-changelog_snap_logging_start (xlator_t *this,
-                                  changelog_priv_t *priv)
+changelog_snap_logging_start(xlator_t *this, changelog_priv_t *priv)
 {
-        int ret = 0;
+    int ret = 0;
 
-        ret = changelog_snap_open (this, priv);
-        gf_log (this->name, GF_LOG_INFO,
-                        "Now starting to log in call path");
+    ret = changelog_snap_open(this, priv);
+    gf_smsg(this->name, GF_LOG_INFO, 0, CHANGELOG_MSG_SNAP_INFO, "starting",
+            NULL);
 
-        return ret;
+    return ret;
 }
 
 /*
@@ -551,116 +901,104 @@ changelog_snap_logging_start (xlator_t *this,
  *      -1 : On Failure.
  */
 int
-changelog_snap_logging_stop (xlator_t *this,
-                changelog_priv_t *priv)
+changelog_snap_logging_stop(xlator_t *this, changelog_priv_t *priv)
 {
-        int ret         = 0;
+    int ret = 0;
 
-        close (priv->c_snap_fd);
-        priv->c_snap_fd = -1;
+    sys_close(priv->c_snap_fd);
+    priv->c_snap_fd = -1;
 
-        gf_log (this->name, GF_LOG_INFO,
-                        "Stopped to log in call path");
+    gf_smsg(this->name, GF_LOG_INFO, 0, CHANGELOG_MSG_SNAP_INFO, "Stopped",
+            NULL);
 
-        return ret;
+    return ret;
 }
 
 int
-changelog_open_journal (xlator_t *this,
-                        changelog_priv_t *priv)
+changelog_open_journal(xlator_t *this, changelog_priv_t *priv)
 {
-        int fd                        = 0;
-        int ret                       = -1;
-        int flags                     = 0;
-        char buffer[1024]             = {0,};
-        char changelog_path[PATH_MAX] = {0,};
-
-        (void) snprintf (changelog_path, PATH_MAX,
-                         "%s/"CHANGELOG_FILE_NAME,
-                         priv->changelog_dir);
-
-        flags |= (O_CREAT | O_RDWR);
-        if (priv->fsync_interval == 0)
-                flags |= O_SYNC;
-
-        fd = open (changelog_path, flags,
-                   S_IRUSR | S_IWUSR | S_IRGRP | S_IROTH);
-        if (fd < 0) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "unable to open/create changelog file %s"
-                        " (reason: %s). change-logging will be"
-                        " inactive", changelog_path, strerror (errno));
-                goto out;
-        }
-
-        priv->changelog_fd = fd;
-
-        (void) snprintf (buffer, 1024, CHANGELOG_HEADER,
-                         CHANGELOG_VERSION_MAJOR,
-                         CHANGELOG_VERSION_MINOR,
-                         priv->ce->encoder);
-        ret = changelog_write_change (priv, buffer, strlen (buffer));
-        if (ret) {
-                close (priv->changelog_fd);
-                priv->changelog_fd = -1;
-                goto out;
-        }
+    int fd = 0;
+    int ret = -1;
+    int flags = 0;
+    char buffer[1024] = {
+        0,
+    };
+    char changelog_path[PATH_MAX] = {
+        0,
+    };
+
+    (void)snprintf(changelog_path, PATH_MAX, "%s/" CHANGELOG_FILE_NAME,
+                   priv->changelog_dir);
+
+    flags |= (O_CREAT | O_RDWR);
+    if (priv->fsync_interval == 0)
+        flags |= O_SYNC;
+
+    fd = open(changelog_path, flags, S_IRUSR | S_IWUSR | S_IRGRP | S_IROTH);
+    if (fd < 0) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, CHANGELOG_MSG_OPEN_FAILED,
+                "path=%s", changelog_path, NULL);
+        goto out;
+    }
+
+    priv->changelog_fd = fd;
+
+    (void)snprintf(buffer, 1024, CHANGELOG_HEADER, CHANGELOG_VERSION_MAJOR,
+                   CHANGELOG_VERSION_MINOR, priv->ce->encoder);
+    ret = changelog_write_change(priv, buffer, strlen(buffer));
+    if (ret) {
+        sys_close(priv->changelog_fd);
+        priv->changelog_fd = -1;
+        goto out;
+    }
+
+    ret = 0;
 
-        ret = 0;
-
- out:
-        return ret;
+out:
+    return ret;
 }
 
 int
-changelog_start_next_change (xlator_t *this,
-                             changelog_priv_t *priv,
-                             unsigned long ts, gf_boolean_t finale)
+changelog_start_next_change(xlator_t *this, changelog_priv_t *priv, time_t ts,
+                            gf_boolean_t finale)
 {
-        int ret = -1;
+    int ret = -1;
 
-        ret = changelog_rollover_changelog (this, priv, ts);
+    ret = changelog_rollover_changelog(this, priv, ts);
 
-        if (!ret && !finale)
-                ret = changelog_open_journal (this, priv);
+    if (!ret && !finale)
+        ret = changelog_open_journal(this, priv);
 
-        return ret;
+    return ret;
 }
 
 /**
  * return the length of entry
  */
-inline size_t
-changelog_entry_length ()
+size_t
+changelog_entry_length()
 {
-        return sizeof (changelog_log_data_t);
+    return sizeof(changelog_log_data_t);
 }
 
-int
-changelog_fill_rollover_data (changelog_log_data_t *cld, gf_boolean_t is_last)
+void
+changelog_fill_rollover_data(changelog_log_data_t *cld, gf_boolean_t is_last)
 {
-        struct timeval tv = {0,};
-
-        cld->cld_type = CHANGELOG_TYPE_ROLLOVER;
-
-        if (gettimeofday (&tv, NULL))
-                return -1;
-
-        cld->cld_roll_time = (unsigned long) tv.tv_sec;
-        cld->cld_finale = is_last;
-        return 0;
+    cld->cld_type = CHANGELOG_TYPE_ROLLOVER;
+    cld->cld_roll_time = gf_time();
+    cld->cld_finale = is_last;
 }
 
 int
-changelog_snap_write_change (changelog_priv_t *priv, char *buffer, size_t len)
+changelog_snap_write_change(changelog_priv_t *priv, char *buffer, size_t len)
 {
-        return changelog_write (priv->c_snap_fd, buffer, len);
+    return changelog_write(priv->c_snap_fd, buffer, len);
 }
 
 int
-changelog_write_change (changelog_priv_t *priv, char *buffer, size_t len)
+changelog_write_change(changelog_priv_t *priv, char *buffer, size_t len)
 {
-        return changelog_write (priv->changelog_fd, buffer, len);
+    return changelog_write(priv->changelog_fd, buffer, len);
 }
 
 /*
@@ -673,237 +1011,230 @@ changelog_write_change (changelog_priv_t *priv, char *buffer, size_t len)
  *     -1 : On Failure.
  */
 int
-changelog_snap_handle_ascii_change (xlator_t *this,
-                                    changelog_log_data_t *cld)
+changelog_snap_handle_ascii_change(xlator_t *this, changelog_log_data_t *cld)
 {
-        size_t            off      = 0;
-        size_t            gfid_len = 0;
-        char             *gfid_str = NULL;
-        char             *buffer   = NULL;
-        changelog_priv_t *priv     = NULL;
-        int               ret      = 0;
-
-        if (this == NULL) {
-                ret = -1;
-                goto out;
-        }
-
-        priv = this->private;
-
-        if (priv == NULL) {
-                ret = -1;
-                goto out;
-        }
-
-        gfid_str = uuid_utoa (cld->cld_gfid);
-        gfid_len = strlen (gfid_str);
-
-        /*  extra bytes for decorations */
-        buffer = alloca (gfid_len + cld->cld_ptr_len + 10);
-        CHANGELOG_STORE_ASCII (priv, buffer,
-                        off, gfid_str, gfid_len, cld);
-
-        CHANGELOG_FILL_BUFFER (buffer, off, "\0", 1);
-
-        ret = changelog_snap_write_change (priv, buffer, off);
-
-        if (ret < 0) {
-                gf_log (this->name, GF_LOG_ERROR,
-                                "error writing csnap to disk");
-        }
-        gf_log (this->name, GF_LOG_INFO,
-                        "Successfully wrote to csnap");
-        ret = 0;
+    size_t off = 0;
+    size_t gfid_len = 0;
+    char *gfid_str = NULL;
+    char *buffer = NULL;
+    changelog_priv_t *priv = NULL;
+    int ret = 0;
+
+    if (this == NULL) {
+        ret = -1;
+        goto out;
+    }
+
+    priv = this->private;
+
+    if (priv == NULL) {
+        ret = -1;
+        goto out;
+    }
+
+    gfid_str = uuid_utoa(cld->cld_gfid);
+    gfid_len = strlen(gfid_str);
+
+    /*  extra bytes for decorations */
+    buffer = alloca(gfid_len + cld->cld_ptr_len + 10);
+    CHANGELOG_STORE_ASCII(priv, buffer, off, gfid_str, gfid_len, cld);
+
+    CHANGELOG_FILL_BUFFER(buffer, off, "\0", 1);
+
+    ret = changelog_snap_write_change(priv, buffer, off);
+
+    if (ret < 0) {
+        gf_smsg(this->name, GF_LOG_ERROR, 0, CHANGELOG_MSG_WRITE_FAILED,
+                "csnap", NULL);
+    }
+    gf_smsg(this->name, GF_LOG_INFO, 0, CHANGELOG_MSG_WROTE_TO_CSNAP, NULL);
+    ret = 0;
 out:
-        return ret;
+    return ret;
 }
 
-inline int
-changelog_handle_change (xlator_t *this,
-                         changelog_priv_t *priv, changelog_log_data_t *cld)
+int
+changelog_handle_change(xlator_t *this, changelog_priv_t *priv,
+                        changelog_log_data_t *cld)
 {
-        int ret = 0;
-
-        if (CHANGELOG_TYPE_IS_ROLLOVER (cld->cld_type)) {
-                changelog_encode_change (priv);
-                ret = changelog_start_next_change (this, priv,
-                                                   cld->cld_roll_time,
-                                                   cld->cld_finale);
-                if (ret)
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "Problem rolling over changelog(s)");
-                goto out;
-        }
+    int ret = 0;
 
-        /**
-         * case when there is reconfigure done (disabling changelog) and there
-         * are still fops that have updates in prgress.
-         */
-        if (priv->changelog_fd == -1)
-                return 0;
-
-        if (CHANGELOG_TYPE_IS_FSYNC (cld->cld_type)) {
-                ret = fsync (priv->changelog_fd);
-                if (ret < 0) {
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "fsync failed (reason: %s)",
-                                strerror (errno));
-                }
-                goto out;
-        }
+    if (CHANGELOG_TYPE_IS_ROLLOVER(cld->cld_type)) {
+        changelog_encode_change(priv);
+        ret = changelog_start_next_change(this, priv, cld->cld_roll_time,
+                                          cld->cld_finale);
+        if (ret)
+            gf_smsg(this->name, GF_LOG_ERROR, 0,
+                    CHANGELOG_MSG_GET_TIME_OP_FAILED, NULL);
+        goto out;
+    }
+
+    /**
+     * case when there is reconfigure done (disabling changelog) and there
+     * are still fops that have updates in prgress.
+     */
+    if (priv->changelog_fd == -1)
+        return 0;
 
-        ret = priv->ce->encode (this, cld);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "error writing changelog to disk");
+    if (CHANGELOG_TYPE_IS_FSYNC(cld->cld_type)) {
+        ret = sys_fsync(priv->changelog_fd);
+        if (ret < 0) {
+            gf_smsg(this->name, GF_LOG_ERROR, errno,
+                    CHANGELOG_MSG_FSYNC_OP_FAILED, NULL);
         }
+        goto out;
+    }
 
- out:
-        return ret;
+    ret = priv->ce->encode(this, cld);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_ERROR, 0, CHANGELOG_MSG_WRITE_FAILED,
+                "changelog", NULL);
+    }
+
+out:
+    return ret;
 }
 
 changelog_local_t *
-changelog_local_init (xlator_t *this, inode_t *inode,
-                      uuid_t gfid, int xtra_records,
-                      gf_boolean_t update_flag)
+changelog_local_init(xlator_t *this, inode_t *inode, uuid_t gfid,
+                     int xtra_records, gf_boolean_t update_flag)
 {
-        changelog_local_t *local = NULL;
-        struct iobuf      *iobuf = NULL;
+    changelog_local_t *local = NULL;
+    struct iobuf *iobuf = NULL;
 
-        /**
-         * We relax the presence of inode if @update_flag is true.
-         * The caller (implmentation of the fop) needs to be careful to
-         * not blindly use local->inode.
-         */
-        if (!update_flag && !inode) {
-                gf_log_callingfn (this->name, GF_LOG_WARNING,
-                                  "inode needed for version checking !!!");
-                goto out;
-        }
+    /**
+     * We relax the presence of inode if @update_flag is true.
+     * The caller (implementation of the fop) needs to be careful to
+     * not blindly use local->inode.
+     */
+    if (!update_flag && !inode) {
+        gf_msg_callingfn(this->name, GF_LOG_WARNING, 0,
+                         CHANGELOG_MSG_INODE_NOT_FOUND,
+                         "inode needed for version checking !!!");
 
-        if (xtra_records) {
-                iobuf = iobuf_get2 (this->ctx->iobuf_pool,
-                                    xtra_records * CHANGELOG_OPT_RECORD_LEN);
-                if (!iobuf)
-                        goto out;
-        }
+        goto out;
+    }
 
-        local = mem_get0 (this->local_pool);
-        if (!local) {
-                CHANGELOG_IOBUF_UNREF (iobuf);
-                goto out;
-        }
+    if (xtra_records) {
+        iobuf = iobuf_get2(this->ctx->iobuf_pool,
+                           xtra_records * CHANGELOG_OPT_RECORD_LEN);
+        if (!iobuf)
+            goto out;
+    }
 
-        local->update_no_check = update_flag;
+    local = mem_get0(this->local_pool);
+    if (!local) {
+        CHANGELOG_IOBUF_UNREF(iobuf);
+        goto out;
+    }
 
-        gf_uuid_copy (local->cld.cld_gfid, gfid);
+    local->update_no_check = update_flag;
 
-        local->cld.cld_iobuf = iobuf;
-        local->cld.cld_xtra_records = 0; /* set by the caller */
+    gf_uuid_copy(local->cld.cld_gfid, gfid);
 
-        if (inode)
-                local->inode = inode_ref (inode);
+    local->cld.cld_iobuf = iobuf;
+    local->cld.cld_xtra_records = 0; /* set by the caller */
 
- out:
-        return local;
+    if (inode)
+        local->inode = inode_ref(inode);
+
+out:
+    return local;
 }
 
 int
-changelog_forget (xlator_t *this, inode_t *inode)
+changelog_forget(xlator_t *this, inode_t *inode)
 {
-        uint64_t ctx_addr = 0;
-        changelog_inode_ctx_t *ctx = NULL;
+    uint64_t ctx_addr = 0;
+    changelog_inode_ctx_t *ctx = NULL;
 
-        inode_ctx_del (inode, this, &ctx_addr);
-        if (!ctx_addr)
-                return 0;
+    inode_ctx_del(inode, this, &ctx_addr);
+    if (!ctx_addr)
+        return 0;
 
-        ctx = (changelog_inode_ctx_t *) (long) ctx_addr;
-        GF_FREE (ctx);
+    ctx = (changelog_inode_ctx_t *)(long)ctx_addr;
+    GF_FREE(ctx);
 
-        return 0;
+    return 0;
 }
 
 int
-changelog_inject_single_event (xlator_t *this,
-                               changelog_priv_t *priv,
-                               changelog_log_data_t *cld)
+changelog_inject_single_event(xlator_t *this, changelog_priv_t *priv,
+                              changelog_log_data_t *cld)
 {
-        return priv->cd.dispatchfn (this, priv, priv->cd.cd_data, cld, NULL);
+    return priv->cd.dispatchfn(this, priv, priv->cd.cd_data, cld, NULL);
 }
 
 /* Wait till all the black fops are drained */
 void
-changelog_drain_black_fops (xlator_t *this, changelog_priv_t *priv)
+changelog_drain_black_fops(xlator_t *this, changelog_priv_t *priv)
 {
-        int ret = 0;
-
-        /* clean up framework of pthread_mutex is required here as
-         * 'reconfigure' terminates the changelog_rollover thread
-         * on graph change.
-         */
-        pthread_cleanup_push (changelog_cleanup_free_mutex,
-                                        &priv->dm.drain_black_mutex);
-        ret = pthread_mutex_lock (&priv->dm.drain_black_mutex);
-        if (ret)
-                gf_log (this->name, GF_LOG_ERROR, "pthread error:"
-                        " Error:%d", ret);
-        while (priv->dm.black_fop_cnt > 0) {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "Condtional wait on black fops: %ld",
-                        priv->dm.black_fop_cnt);
-                priv->dm.drain_wait_black = _gf_true;
-                ret = pthread_cond_wait (&priv->dm.drain_black_cond,
-                                         &priv->dm.drain_black_mutex);
-                if (ret)
-                        gf_log (this->name, GF_LOG_ERROR, "pthread"
-                                " cond wait failed: Error:%d", ret);
-        }
-        priv->dm.drain_wait_black = _gf_false;
-        ret = pthread_mutex_unlock (&priv->dm.drain_black_mutex);
-        pthread_cleanup_pop (0);
+    int ret = 0;
+
+    /* clean up framework of pthread_mutex is required here as
+     * 'reconfigure' terminates the changelog_rollover thread
+     * on graph change.
+     */
+    pthread_cleanup_push(changelog_cleanup_free_mutex,
+                         &priv->dm.drain_black_mutex);
+    ret = pthread_mutex_lock(&priv->dm.drain_black_mutex);
+    if (ret)
+        gf_smsg(this->name, GF_LOG_ERROR, errno, CHANGELOG_MSG_PTHREAD_ERROR,
+                "error=%d", ret, NULL);
+    while (priv->dm.black_fop_cnt > 0) {
+        gf_msg_debug(this->name, 0, "Conditional wait on black fops: %ld",
+                     priv->dm.black_fop_cnt);
+        priv->dm.drain_wait_black = _gf_true;
+        ret = pthread_cond_wait(&priv->dm.drain_black_cond,
+                                &priv->dm.drain_black_mutex);
         if (ret)
-                gf_log (this->name, GF_LOG_ERROR, "pthread error:"
-                        " Error:%d", ret);
-        gf_log (this->name, GF_LOG_DEBUG,
-                "Woke up: Conditional wait on black fops");
+            gf_smsg(this->name, GF_LOG_ERROR, errno,
+                    CHANGELOG_MSG_PTHREAD_COND_WAIT_FAILED, "error=%d", ret,
+                    NULL);
+    }
+    priv->dm.drain_wait_black = _gf_false;
+    ret = pthread_mutex_unlock(&priv->dm.drain_black_mutex);
+    if (ret)
+        gf_smsg(this->name, GF_LOG_ERROR, errno, CHANGELOG_MSG_PTHREAD_ERROR,
+                "error=%d", ret, NULL);
+    pthread_cleanup_pop(0);
+    gf_msg_debug(this->name, 0, "Woke up: Conditional wait on black fops");
 }
 
 /* Wait till all the white  fops are drained */
 void
-changelog_drain_white_fops (xlator_t *this, changelog_priv_t *priv)
+changelog_drain_white_fops(xlator_t *this, changelog_priv_t *priv)
 {
-        int ret = 0;
-
-        /* clean up framework of pthread_mutex is required here as
-         * 'reconfigure' terminates the changelog_rollover thread
-         * on graph change.
-         */
-        pthread_cleanup_push (changelog_cleanup_free_mutex,
-                                        &priv->dm.drain_white_mutex);
-        ret = pthread_mutex_lock (&priv->dm.drain_white_mutex);
+    int ret = 0;
+
+    /* clean up framework of pthread_mutex is required here as
+     * 'reconfigure' terminates the changelog_rollover thread
+     * on graph change.
+     */
+    pthread_cleanup_push(changelog_cleanup_free_mutex,
+                         &priv->dm.drain_white_mutex);
+    ret = pthread_mutex_lock(&priv->dm.drain_white_mutex);
+    if (ret)
+        gf_smsg(this->name, GF_LOG_ERROR, errno, CHANGELOG_MSG_PTHREAD_ERROR,
+                "error=%d", ret, NULL);
+    while (priv->dm.white_fop_cnt > 0) {
+        gf_msg_debug(this->name, 0, "Conditional wait on white fops : %ld",
+                     priv->dm.white_fop_cnt);
+        priv->dm.drain_wait_white = _gf_true;
+        ret = pthread_cond_wait(&priv->dm.drain_white_cond,
+                                &priv->dm.drain_white_mutex);
         if (ret)
-                gf_log (this->name, GF_LOG_ERROR, "pthread error:"
-                        " Error:%d", ret);
-        while (priv->dm.white_fop_cnt > 0) {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "Condtional wait on white fops : %ld",
-                        priv->dm.white_fop_cnt);
-                priv->dm.drain_wait_white = _gf_true;
-                ret = pthread_cond_wait (&priv->dm.drain_white_cond,
-                                         &priv->dm.drain_white_mutex);
-                if (ret)
-                        gf_log (this->name, GF_LOG_ERROR, "pthread"
-                                " cond wait failed: Error:%d", ret);
-        }
-        priv->dm.drain_wait_white = _gf_false;
-        ret = pthread_mutex_unlock (&priv->dm.drain_white_mutex);
-        if (ret)
-                gf_log (this->name, GF_LOG_ERROR, "pthread error:"
-                        " Error:%d", ret);
-        pthread_cleanup_pop (0);
-        gf_log (this->name, GF_LOG_DEBUG,
-                "Woke up: Conditional wait on white fops");
+            gf_smsg(this->name, GF_LOG_ERROR, errno,
+                    CHANGELOG_MSG_PTHREAD_COND_WAIT_FAILED, "error=%d", ret,
+                    NULL);
+    }
+    priv->dm.drain_wait_white = _gf_false;
+    ret = pthread_mutex_unlock(&priv->dm.drain_white_mutex);
+    if (ret)
+        gf_smsg(this->name, GF_LOG_ERROR, errno, CHANGELOG_MSG_PTHREAD_ERROR,
+                "error=%d", ret, NULL);
+    pthread_cleanup_pop(0);
+    gf_msg_debug(this->name, 0, "Woke up: Conditional wait on white fops");
 }
 
 /**
@@ -911,196 +1242,194 @@ changelog_drain_white_fops (xlator_t *this, changelog_priv_t *priv)
  * a certain time etc..). move them into separate routine.
  */
 void *
-changelog_rollover (void *data)
+changelog_rollover(void *data)
 {
-        int                     ret             = 0;
-        xlator_t               *this            = NULL;
-        struct timeval          tv              = {0,};
-        changelog_log_data_t    cld             = {0,};
-        changelog_time_slice_t *slice           = NULL;
-        changelog_priv_t       *priv            = data;
-        int                     max_fd          = 0;
-        char                    buf[1]          = {0};
-        int                     len             = 0;
-
-        fd_set                  rset;
-
-        this = priv->cr.this;
-        slice = &priv->slice;
-
-        while (1) {
-                (void) pthread_testcancel();
-
-                tv.tv_sec  = priv->rollover_time;
-                tv.tv_usec = 0;
-                FD_ZERO(&rset);
-                FD_SET(priv->cr.rfd, &rset);
-                max_fd = priv->cr.rfd;
-                max_fd = max_fd + 1;
-
-               /* It seems there is a race between actual rollover and explicit
-                * rollover. But it is handled. If actual rollover is being
-                * done and the explicit rollover event comes, the event is
-                * not missed. The next select will immediately wakeup to
-                * handle explicit wakeup.
-                */
-
-                ret = select (max_fd, &rset, NULL, NULL, &tv);
-                if (ret == -1) {
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "select failed: %s", strerror(errno));
-                        continue;
-                } else if (ret && FD_ISSET(priv->cr.rfd, &rset)) {
-                        gf_log (this->name, GF_LOG_INFO,
-                                "Explicit wakeup of select on barrier notify");
-                        len = read(priv->cr.rfd, buf, 1);
-                        if (len == 0) {
-                                gf_log (this->name, GF_LOG_ERROR, "BUG: Got EOF"
-                                        " from reconfigure notification pipe");
-                                continue;
-                        }
-                        if (len < 0) {
-                                gf_log (this->name, GF_LOG_ERROR,
-                                        "Failed to read wakeup data");
-                                continue;
-                        }
-                        /* Lock is not required as same thread is modifying.*/
-                        priv->explicit_rollover = _gf_true;
-                } else {
-                        gf_log (this->name, GF_LOG_DEBUG,
-                                "select wokeup on timeout");
-                }
+    int ret = 0;
+    xlator_t *this = NULL;
+    struct timespec tv = {
+        0,
+    };
+    changelog_log_data_t cld = {
+        0,
+    };
+    changelog_time_slice_t *slice = NULL;
+    changelog_priv_t *priv = data;
+
+    this = priv->cr.this;
+    slice = &priv->slice;
+
+    while (1) {
+        (void)pthread_testcancel();
+
+        tv.tv_sec = gf_time() + priv->rollover_time;
+        tv.tv_nsec = 0;
+        ret = 0; /* Reset ret to zero */
+
+        /* The race between actual rollover and explicit rollover is
+         * handled. If actual rollover is being done and the
+         * explicit rollover event comes, the event is not missed.
+         * Since explicit rollover sets 'cr.notify' to true, this
+         * thread doesn't wait on 'pthread_cond_timedwait'.
+         */
+        pthread_cleanup_push(changelog_cleanup_free_mutex, &priv->cr.lock);
+        pthread_mutex_lock(&priv->cr.lock);
+        {
+            while (ret == 0 && !priv->cr.notify)
+                ret = pthread_cond_timedwait(&priv->cr.cond, &priv->cr.lock,
+                                             &tv);
+            if (ret == 0)
+                priv->cr.notify = _gf_false;
+        }
+        pthread_mutex_unlock(&priv->cr.lock);
+        pthread_cleanup_pop(0);
+
+        if (ret == 0) {
+            gf_smsg(this->name, GF_LOG_INFO, 0, CHANGELOG_MSG_BARRIER_INFO,
+                    NULL);
+            priv->explicit_rollover = _gf_true;
+        } else if (ret && ret != ETIMEDOUT) {
+            gf_smsg(this->name, GF_LOG_ERROR, errno,
+                    CHANGELOG_MSG_SELECT_FAILED, NULL);
+            continue;
+        } else if (ret && ret == ETIMEDOUT) {
+            gf_msg_debug(this->name, 0, "Wokeup on timeout");
+        }
 
-               /* Reading curent_color without lock is fine here
-                * as it is only modified here and is next to reading.
-                */
-                if (priv->current_color == FOP_COLOR_BLACK) {
-                        LOCK(&priv->lock);
-                                priv->current_color = FOP_COLOR_WHITE;
-                        UNLOCK(&priv->lock);
-                        gf_log (this->name, GF_LOG_DEBUG, "Black fops"
-                                " to be drained:%ld",priv->dm.black_fop_cnt);
-                        changelog_drain_black_fops (this, priv);
-                } else {
-                        LOCK(&priv->lock);
-                                priv->current_color = FOP_COLOR_BLACK;
-                        UNLOCK(&priv->lock);
-                        gf_log (this->name, GF_LOG_DEBUG, "White fops"
-                                " to be drained:%ld",priv->dm.white_fop_cnt);
-                        changelog_drain_white_fops (this, priv);
-                }
+        /* Reading curent_color without lock is fine here
+         * as it is only modified here and is next to reading.
+         */
+        if (priv->current_color == FOP_COLOR_BLACK) {
+            LOCK(&priv->lock);
+            priv->current_color = FOP_COLOR_WHITE;
+            UNLOCK(&priv->lock);
+            gf_msg_debug(this->name, 0,
+                         "Black fops"
+                         " to be drained:%ld",
+                         priv->dm.black_fop_cnt);
+            changelog_drain_black_fops(this, priv);
+        } else {
+            LOCK(&priv->lock);
+            priv->current_color = FOP_COLOR_BLACK;
+            UNLOCK(&priv->lock);
+            gf_msg_debug(this->name, 0,
+                         "White fops"
+                         " to be drained:%ld",
+                         priv->dm.white_fop_cnt);
+            changelog_drain_white_fops(this, priv);
+        }
 
-                /* Adding delay of 1 second only during explicit rollover:
-                 *
-                 * Changelog rollover can happen either due to actual
-                 * or the explict rollover during snapshot. Actual
-                 * rollover is controlled by tuneable called 'rollover-time'.
-                 * The minimum granularity for rollover-time is 1 second.
-                 * Explicit rollover is asynchronous in nature and happens
-                 * during snapshot.
-                 *
-                 * Basically, rollover renames the current CHANGELOG file
-                 * to CHANGELOG.TIMESTAMP. Let's assume, at time 't1',
-                 * actual and explicit rollover raced against  each
-                 * other and actual rollover won the race renaming the
-                 * CHANGELOG file to CHANGELOG.t1 and opens a new
-                 * CHANGELOG file. There is high chance that, an immediate
-                 * explicit rollover at time 't1' can happen with in the same
-                 * second to rename CHANGELOG file to CHANGELOG.t1 resulting in
-                 * purging the earlier CHANGELOG.t1 file created by actual
-                 * rollover. So adding a delay of 1 second guarantees unique
-                 * CHANGELOG.TIMESTAMP during  explicit rollover.
-                 */
-                if (priv->explicit_rollover == _gf_true)
-                        sleep (1);
-
-                ret = changelog_fill_rollover_data (&cld, _gf_false);
-                if (ret) {
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "failed to fill rollover data");
-                        continue;
-                }
+        /* Adding delay of 1 second only during explicit rollover:
+         *
+         * Changelog rollover can happen either due to actual
+         * or the explicit rollover during snapshot. Actual
+         * rollover is controlled by tuneable called 'rollover-time'.
+         * The minimum granularity for rollover-time is 1 second.
+         * Explicit rollover is asynchronous in nature and happens
+         * during snapshot.
+         *
+         * Basically, rollover renames the current CHANGELOG file
+         * to CHANGELOG.TIMESTAMP. Let's assume, at time 't1',
+         * actual and explicit rollover raced against  each
+         * other and actual rollover won the race renaming the
+         * CHANGELOG file to CHANGELOG.t1 and opens a new
+         * CHANGELOG file. There is high chance that, an immediate
+         * explicit rollover at time 't1' can happen with in the same
+         * second to rename CHANGELOG file to CHANGELOG.t1 resulting in
+         * purging the earlier CHANGELOG.t1 file created by actual
+         * rollover. So adding a delay of 1 second guarantees unique
+         * CHANGELOG.TIMESTAMP during  explicit rollover.
+         */
+        if (priv->explicit_rollover == _gf_true)
+            sleep(1);
 
-                __mask_cancellation (this);
+        changelog_fill_rollover_data(&cld, _gf_false);
 
-                LOCK (&priv->lock);
-                {
-                        ret = changelog_inject_single_event (this, priv, &cld);
-                        if (!ret)
-                                SLICE_VERSION_UPDATE (slice);
-                }
-                UNLOCK (&priv->lock);
+        _mask_cancellation();
 
-                __unmask_cancellation (this);
+        LOCK(&priv->lock);
+        {
+            ret = changelog_inject_single_event(this, priv, &cld);
+            if (!ret)
+                SLICE_VERSION_UPDATE(slice);
         }
+        UNLOCK(&priv->lock);
 
-        return NULL;
+        _unmask_cancellation();
+    }
+
+    return NULL;
 }
 
 void *
-changelog_fsync_thread (void *data)
+changelog_fsync_thread(void *data)
 {
-        int                   ret  = 0;
-        xlator_t             *this = NULL;
-        struct timeval        tv   = {0,};
-        changelog_log_data_t  cld  = {0,};
-        changelog_priv_t     *priv = data;
-
-        this = priv->cf.this;
-        cld.cld_type = CHANGELOG_TYPE_FSYNC;
-
-        while (1) {
-                (void) pthread_testcancel();
-
-                tv.tv_sec  = priv->fsync_interval;
-                tv.tv_usec = 0;
-
-                ret = select (0, NULL, NULL, NULL, &tv);
-                if (ret)
-                        continue;
+    int ret = 0;
+    xlator_t *this = NULL;
+    struct timeval tv = {
+        0,
+    };
+    changelog_log_data_t cld = {
+        0,
+    };
+    changelog_priv_t *priv = data;
+
+    this = priv->cf.this;
+    cld.cld_type = CHANGELOG_TYPE_FSYNC;
+
+    while (1) {
+        (void)pthread_testcancel();
+
+        tv.tv_sec = priv->fsync_interval;
+        tv.tv_usec = 0;
+
+        ret = select(0, NULL, NULL, NULL, &tv);
+        if (ret)
+            continue;
 
-                __mask_cancellation (this);
+        _mask_cancellation();
 
-                ret = changelog_inject_single_event (this, priv, &cld);
-                if (ret)
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "failed to inject fsync event");
+        ret = changelog_inject_single_event(this, priv, &cld);
+        if (ret)
+            gf_smsg(this->name, GF_LOG_ERROR, 0,
+                    CHANGELOG_MSG_INJECT_FSYNC_FAILED, NULL);
 
-                __unmask_cancellation (this);
-        }
+        _unmask_cancellation();
+    }
 
-        return NULL;
+    return NULL;
 }
 
 /* macros for inode/changelog version checks */
 
-#define INODE_VERSION_UPDATE(priv, inode, iver, slice, type) do {       \
-                LOCK (&inode->lock);                                    \
-                {                                                       \
-                        LOCK (&priv->lock);                             \
-                        {                                               \
-                                *iver = slice->changelog_version[type]; \
-                        }                                               \
-                        UNLOCK (&priv->lock);                           \
-                }                                                       \
-                UNLOCK (&inode->lock);                                  \
-        } while (0)
-
-#define INODE_VERSION_EQUALS_SLICE(priv, ver, slice, type, upd) do {    \
-                LOCK (&priv->lock);                                     \
-                {                                                       \
-                        upd = (ver == slice->changelog_version[type])   \
-                                ? _gf_false : _gf_true;                 \
-                }                                                       \
-                UNLOCK (&priv->lock);                                   \
-        } while (0)
+#define INODE_VERSION_UPDATE(priv, inode, iver, slice, type)                   \
+    do {                                                                       \
+        LOCK(&inode->lock);                                                    \
+        {                                                                      \
+            LOCK(&priv->lock);                                                 \
+            {                                                                  \
+                *iver = slice->changelog_version[type];                        \
+            }                                                                  \
+            UNLOCK(&priv->lock);                                               \
+        }                                                                      \
+        UNLOCK(&inode->lock);                                                  \
+    } while (0)
+
+#define INODE_VERSION_EQUALS_SLICE(priv, ver, slice, type, upd)                \
+    do {                                                                       \
+        LOCK(&priv->lock);                                                     \
+        {                                                                      \
+            upd = (ver == slice->changelog_version[type]) ? _gf_false          \
+                                                          : _gf_true;          \
+        }                                                                      \
+        UNLOCK(&priv->lock);                                                   \
+    } while (0)
 
 static int
-__changelog_inode_ctx_set (xlator_t *this,
-                           inode_t *inode, changelog_inode_ctx_t *ctx)
+__changelog_inode_ctx_set(xlator_t *this, inode_t *inode,
+                          changelog_inode_ctx_t *ctx)
 {
-        uint64_t ctx_addr = (uint64_t) ctx;
-        return __inode_ctx_set (inode, this, &ctx_addr);
+    uint64_t ctx_addr = (uint64_t)(uintptr_t)ctx;
+    return __inode_ctx_set(inode, this, &ctx_addr);
 }
 
 /**
@@ -1108,56 +1437,53 @@ __changelog_inode_ctx_set (xlator_t *this,
  * for a particular type.
  */
 changelog_inode_ctx_t *
-__changelog_inode_ctx_get (xlator_t *this,
-                           inode_t *inode, unsigned long **iver,
-                           unsigned long *version, changelog_log_type type)
+__changelog_inode_ctx_get(xlator_t *this, inode_t *inode, unsigned long **iver,
+                          unsigned long *version, changelog_log_type type)
 {
-        int                    ret      = 0;
-        uint64_t               ctx_addr = 0;
-        changelog_inode_ctx_t *ctx      = NULL;
-
-        ret = __inode_ctx_get (inode, this, &ctx_addr);
-        if (ret < 0)
-                ctx_addr = 0;
-        if (ctx_addr != 0) {
-                ctx = (changelog_inode_ctx_t *) (long)ctx_addr;
-                goto out;
-        }
+    int ret = 0;
+    uint64_t ctx_addr = 0;
+    changelog_inode_ctx_t *ctx = NULL;
+
+    ret = __inode_ctx_get(inode, this, &ctx_addr);
+    if (ret < 0)
+        ctx_addr = 0;
+    if (ctx_addr != 0) {
+        ctx = (changelog_inode_ctx_t *)(long)ctx_addr;
+        goto out;
+    }
+
+    ctx = GF_CALLOC(1, sizeof(*ctx), gf_changelog_mt_inode_ctx_t);
+    if (!ctx)
+        goto out;
+
+    ret = __changelog_inode_ctx_set(this, inode, ctx);
+    if (ret) {
+        GF_FREE(ctx);
+        ctx = NULL;
+    }
 
-        ctx = GF_CALLOC (1, sizeof (*ctx), gf_changelog_mt_inode_ctx_t);
-        if (!ctx)
-                goto out;
-
-        ret = __changelog_inode_ctx_set (this, inode, ctx);
-        if (ret) {
-                GF_FREE (ctx);
-                ctx = NULL;
-        }
-
- out:
-        if (ctx && iver && version) {
-                *iver = CHANGELOG_INODE_VERSION_TYPE (ctx, type);
-                *version = **iver;
-        }
+out:
+    if (ctx && iver && version) {
+        *iver = CHANGELOG_INODE_VERSION_TYPE(ctx, type);
+        *version = **iver;
+    }
 
-        return ctx;
+    return ctx;
 }
 
 static changelog_inode_ctx_t *
-changelog_inode_ctx_get (xlator_t *this,
-                         inode_t *inode, unsigned long **iver,
-                         unsigned long *version, changelog_log_type type)
+changelog_inode_ctx_get(xlator_t *this, inode_t *inode, unsigned long **iver,
+                        unsigned long *version, changelog_log_type type)
 {
-        changelog_inode_ctx_t *ctx = NULL;
+    changelog_inode_ctx_t *ctx = NULL;
 
-        LOCK (&inode->lock);
-        {
-                ctx = __changelog_inode_ctx_get (this,
-                                                 inode, iver, version, type);
-        }
-        UNLOCK (&inode->lock);
+    LOCK(&inode->lock);
+    {
+        ctx = __changelog_inode_ctx_get(this, inode, iver, version, type);
+    }
+    UNLOCK(&inode->lock);
 
-        return ctx;
+    return ctx;
 }
 
 /**
@@ -1260,60 +1586,58 @@ changelog_inode_ctx_get (xlator_t *this,
  * then there is no need to record an update (as the equality of the two version
  * signifies an update was recorded in the current time slice).
  */
-inline void
-changelog_update (xlator_t *this, changelog_priv_t *priv,
-                  changelog_local_t *local, changelog_log_type type)
+void
+changelog_update(xlator_t *this, changelog_priv_t *priv,
+                 changelog_local_t *local, changelog_log_type type)
 {
-        int                     ret        = 0;
-        unsigned long          *iver       = NULL;
-        unsigned long           version    = 0;
-        inode_t                *inode      = NULL;
-        changelog_time_slice_t *slice      = NULL;
-        changelog_inode_ctx_t  *ctx        = NULL;
-        changelog_log_data_t   *cld_0      = NULL;
-        changelog_log_data_t   *cld_1      = NULL;
-        changelog_local_t      *next_local = NULL;
-        gf_boolean_t            need_upd   = _gf_true;
-
-        slice = &priv->slice;
+    int ret = 0;
+    unsigned long *iver = NULL;
+    unsigned long version = 0;
+    inode_t *inode = NULL;
+    changelog_time_slice_t *slice = NULL;
+    changelog_inode_ctx_t *ctx = NULL;
+    changelog_log_data_t *cld_0 = NULL;
+    changelog_log_data_t *cld_1 = NULL;
+    changelog_local_t *next_local = NULL;
+    gf_boolean_t need_upd = _gf_true;
+
+    slice = &priv->slice;
+
+    /**
+     * for fops that do not require inode version checking
+     */
+    if (local->update_no_check)
+        goto update;
+
+    inode = local->inode;
+
+    ctx = changelog_inode_ctx_get(this, inode, &iver, &version, type);
+    if (!ctx)
+        goto update;
+
+    INODE_VERSION_EQUALS_SLICE(priv, version, slice, type, need_upd);
+
+update:
+    if (need_upd) {
+        cld_0 = &local->cld;
+        cld_0->cld_type = type;
+
+        if ((next_local = local->prev_entry) != NULL) {
+            cld_1 = &next_local->cld;
+            cld_1->cld_type = type;
+        }
+
+        ret = priv->cd.dispatchfn(this, priv, priv->cd.cd_data, cld_0, cld_1);
 
         /**
-         * for fops that do not require inode version checking
+         * update after the dispatcher has successfully done
+         * it's job.
          */
-        if (local->update_no_check)
-                goto update;
-
-        inode = local->inode;
-
-        ctx = changelog_inode_ctx_get (this,
-                                       inode, &iver, &version, type);
-        if (!ctx)
-                goto update;
-
-        INODE_VERSION_EQUALS_SLICE (priv, version, slice, type, need_upd);
-
- update:
-        if (need_upd) {
-                cld_0 = &local->cld;
-                cld_0->cld_type = type;
-
-                if ( (next_local = local->prev_entry) != NULL ) {
-                        cld_1 = &next_local->cld;
-                        cld_1->cld_type = type;
-                }
-
-                ret = priv->cd.dispatchfn (this, priv,
-                                           priv->cd.cd_data, cld_0, cld_1);
-
-                /**
-                 * update after the dispatcher has successfully done
-                 * it's job.
-                 */
-                if (!local->update_no_check && iver && !ret)
-                        INODE_VERSION_UPDATE (priv, inode, iver, slice, type);
-        }
+        if (!local->update_no_check && iver && !ret)
+            INODE_VERSION_UPDATE(priv, inode, iver, slice, type);
+    }
 
-        return;
+    return;
 }
 
 /* Begin: Geo-rep snapshot dependency changes */
@@ -1328,216 +1652,326 @@ changelog_update (xlator_t *this, changelog_priv_t *priv,
  * not incremented fop count but color is assigned black.
  */
 
-inline void
-changelog_color_fop_and_inc_cnt (xlator_t *this, changelog_priv_t *priv,
-                                                 changelog_local_t *local)
+void
+changelog_color_fop_and_inc_cnt(xlator_t *this, changelog_priv_t *priv,
+                                changelog_local_t *local)
 {
-        if (!priv || !local)
-                return;
+    if (!priv || !local)
+        return;
 
-        LOCK (&priv->lock);
-        {
-                local->color = priv->current_color;
-                changelog_inc_fop_cnt (this, priv, local);
-        }
-        UNLOCK (&priv->lock);
+    LOCK(&priv->lock);
+    {
+        local->color = priv->current_color;
+        changelog_inc_fop_cnt(this, priv, local);
+    }
+    UNLOCK(&priv->lock);
 }
 
 /* Increments the respective fop counter based on the fop color */
-inline void
-changelog_inc_fop_cnt (xlator_t *this, changelog_priv_t *priv,
-                                       changelog_local_t *local)
+void
+changelog_inc_fop_cnt(xlator_t *this, changelog_priv_t *priv,
+                      changelog_local_t *local)
 {
-        int ret = 0;
-
-        if (local) {
-                if (local->color == FOP_COLOR_BLACK) {
-                        ret = pthread_mutex_lock (&priv->dm.drain_black_mutex);
-                        CHANGELOG_PTHREAD_ERROR_HANDLE_0 (ret, out);
-                        {
-                                priv->dm.black_fop_cnt++;
-                        }
-                        ret = pthread_mutex_unlock(&priv->dm.drain_black_mutex);
-                        CHANGELOG_PTHREAD_ERROR_HANDLE_0 (ret, out);
-                } else {
-                        ret = pthread_mutex_lock (&priv->dm.drain_white_mutex);
-                        CHANGELOG_PTHREAD_ERROR_HANDLE_0 (ret, out);
-                        {
-                                priv->dm.white_fop_cnt++;
-                        }
-                        ret = pthread_mutex_unlock(&priv->dm.drain_white_mutex);
-                        CHANGELOG_PTHREAD_ERROR_HANDLE_0 (ret, out);
-                }
+    int ret = 0;
+
+    if (local) {
+        if (local->color == FOP_COLOR_BLACK) {
+            ret = pthread_mutex_lock(&priv->dm.drain_black_mutex);
+            CHANGELOG_PTHREAD_ERROR_HANDLE_0(ret, out);
+            {
+                priv->dm.black_fop_cnt++;
+            }
+            ret = pthread_mutex_unlock(&priv->dm.drain_black_mutex);
+            CHANGELOG_PTHREAD_ERROR_HANDLE_0(ret, out);
+        } else {
+            ret = pthread_mutex_lock(&priv->dm.drain_white_mutex);
+            CHANGELOG_PTHREAD_ERROR_HANDLE_0(ret, out);
+            {
+                priv->dm.white_fop_cnt++;
+            }
+            ret = pthread_mutex_unlock(&priv->dm.drain_white_mutex);
+            CHANGELOG_PTHREAD_ERROR_HANDLE_0(ret, out);
         }
- out:
-        return;
+    }
+out:
+    return;
 }
 
 /* Decrements the respective fop counter based on the fop color */
-inline void
-changelog_dec_fop_cnt (xlator_t *this, changelog_priv_t *priv,
-                                       changelog_local_t *local)
+void
+changelog_dec_fop_cnt(xlator_t *this, changelog_priv_t *priv,
+                      changelog_local_t *local)
 {
-        int ret = 0;
-
-        if (local) {
-                if (local->color == FOP_COLOR_BLACK) {
-                        ret = pthread_mutex_lock (&priv->dm.drain_black_mutex);
-                        CHANGELOG_PTHREAD_ERROR_HANDLE_0 (ret, out);
-                        {
-                                priv->dm.black_fop_cnt--;
-                                if (priv->dm.black_fop_cnt == 0 &&
-                                    priv->dm.drain_wait_black == _gf_true) {
-                                        ret = pthread_cond_signal (
-                                                    &priv->dm.drain_black_cond);
-                                        CHANGELOG_PTHREAD_ERROR_HANDLE_0 (ret,
-                                                                           out);
-                                        gf_log (this->name, GF_LOG_DEBUG,
-                                                "Signalled draining of black");
-                                }
-                        }
-                        ret = pthread_mutex_unlock(&priv->dm.drain_black_mutex);
-                        CHANGELOG_PTHREAD_ERROR_HANDLE_0 (ret, out);
-                } else {
-                        ret = pthread_mutex_lock (&priv->dm.drain_white_mutex);
-                        CHANGELOG_PTHREAD_ERROR_HANDLE_0 (ret, out);
-                        {
-                                priv->dm.white_fop_cnt--;
-                                if (priv->dm.white_fop_cnt == 0 &&
-                                    priv->dm.drain_wait_white == _gf_true) {
-                                        ret = pthread_cond_signal (
-                                                    &priv->dm.drain_white_cond);
-                                        CHANGELOG_PTHREAD_ERROR_HANDLE_0 (ret,
-                                                                           out);
-                                        gf_log (this->name, GF_LOG_DEBUG,
-                                                "Signalled draining of white");
-                                }
-                        }
-                        ret = pthread_mutex_unlock(&priv->dm.drain_white_mutex);
-                        CHANGELOG_PTHREAD_ERROR_HANDLE_0 (ret, out);
+    int ret = 0;
+
+    if (local) {
+        if (local->color == FOP_COLOR_BLACK) {
+            ret = pthread_mutex_lock(&priv->dm.drain_black_mutex);
+            CHANGELOG_PTHREAD_ERROR_HANDLE_0(ret, out);
+            {
+                priv->dm.black_fop_cnt--;
+                if (priv->dm.black_fop_cnt == 0 &&
+                    priv->dm.drain_wait_black == _gf_true) {
+                    ret = pthread_cond_signal(&priv->dm.drain_black_cond);
+                    CHANGELOG_PTHREAD_ERROR_HANDLE_2(
+                        ret, out, priv->dm.drain_black_mutex);
+                    gf_msg_debug(this->name, 0,
+                                 "Signalled "
+                                 "draining of black");
+                }
+            }
+            ret = pthread_mutex_unlock(&priv->dm.drain_black_mutex);
+            CHANGELOG_PTHREAD_ERROR_HANDLE_0(ret, out);
+        } else {
+            ret = pthread_mutex_lock(&priv->dm.drain_white_mutex);
+            CHANGELOG_PTHREAD_ERROR_HANDLE_0(ret, out);
+            {
+                priv->dm.white_fop_cnt--;
+                if (priv->dm.white_fop_cnt == 0 &&
+                    priv->dm.drain_wait_white == _gf_true) {
+                    ret = pthread_cond_signal(&priv->dm.drain_white_cond);
+                    CHANGELOG_PTHREAD_ERROR_HANDLE_2(
+                        ret, out, priv->dm.drain_white_mutex);
+                    gf_msg_debug(this->name, 0,
+                                 "Signalled "
+                                 "draining of white");
                 }
+            }
+            ret = pthread_mutex_unlock(&priv->dm.drain_white_mutex);
+            CHANGELOG_PTHREAD_ERROR_HANDLE_0(ret, out);
         }
- out:
-        return;
+    }
+out:
+    return;
 }
 
 /* Write to a pipe setup between changelog main thread and changelog
  * rollover thread to initiate explicit rollover of changelog journal.
  */
-inline int
-changelog_barrier_notify (changelog_priv_t *priv, char *buf)
+int
+changelog_barrier_notify(changelog_priv_t *priv, char *buf)
 {
-        int ret = 0;
-
-        LOCK(&priv->lock);
-                ret = changelog_write (priv->cr_wfd, buf, 1);
-        UNLOCK(&priv->lock);
-        return ret;
+    int ret = 0;
+
+    pthread_mutex_lock(&priv->cr.lock);
+    {
+        ret = pthread_cond_signal(&priv->cr.cond);
+        priv->cr.notify = _gf_true;
+    }
+    pthread_mutex_unlock(&priv->cr.lock);
+    return ret;
 }
 
 /* Clean up flags set on barrier notification */
-inline void
-changelog_barrier_cleanup (xlator_t *this, changelog_priv_t *priv,
-                                                struct list_head *queue)
+void
+changelog_barrier_cleanup(xlator_t *this, changelog_priv_t *priv,
+                          struct list_head *queue)
 {
-        int ret = 0;
-
-        LOCK (&priv->bflags.lock);
-                priv->bflags.barrier_ext = _gf_false;
-        UNLOCK (&priv->bflags.lock);
-
-        ret = pthread_mutex_lock (&priv->bn.bnotify_mutex);
-        CHANGELOG_PTHREAD_ERROR_HANDLE_0 (ret, out);
-        {
-                priv->bn.bnotify = _gf_false;
-        }
-        ret = pthread_mutex_unlock (&priv->bn.bnotify_mutex);
-        CHANGELOG_PTHREAD_ERROR_HANDLE_0 (ret, out);
+    int ret = 0;
+
+    LOCK(&priv->bflags.lock);
+    priv->bflags.barrier_ext = _gf_false;
+    UNLOCK(&priv->bflags.lock);
+
+    ret = pthread_mutex_lock(&priv->bn.bnotify_mutex);
+    CHANGELOG_PTHREAD_ERROR_HANDLE_0(ret, out);
+    {
+        priv->bn.bnotify = _gf_false;
+    }
+    ret = pthread_mutex_unlock(&priv->bn.bnotify_mutex);
+    CHANGELOG_PTHREAD_ERROR_HANDLE_0(ret, out);
+
+    /* Disable changelog barrier and dequeue fops */
+    LOCK(&priv->lock);
+    {
+        if (priv->barrier_enabled == _gf_true)
+            __chlog_barrier_disable(this, queue);
+        else
+            ret = -1;
+    }
+    UNLOCK(&priv->lock);
+    if (ret == 0)
+        chlog_barrier_dequeue_all(this, queue);
 
-        /* Disable changelog barrier and dequeue fops */
-        LOCK (&priv->lock);
-        {
-                if (priv->barrier_enabled == _gf_true)
-                        __chlog_barrier_disable (this, queue);
-                else
-                        ret = -1;
-        }
-        UNLOCK (&priv->lock);
-        if (ret == 0)
-                chlog_barrier_dequeue_all(this, queue);
-
- out:
-        return;
+out:
+    return;
 }
 /* End: Geo-Rep snapshot dependency changes */
 
 int32_t
-changelog_fill_entry_buf (call_frame_t *frame, xlator_t *this,
-                          loc_t *loc, changelog_local_t **local)
+changelog_fill_entry_buf(call_frame_t *frame, xlator_t *this, loc_t *loc,
+                         changelog_local_t **local)
 {
-        changelog_opt_t  *co       = NULL;
-        size_t            xtra_len = 0;
-        char             *dup_path = NULL;
-        char             *bname    = NULL;
-        inode_t          *parent   = NULL;
-
-        GF_ASSERT (this);
-
-        parent = inode_parent (loc->inode, 0, 0);
-        if (!parent) {
-                gf_log (this->name, GF_LOG_ERROR, "Parent inode not found"
-                        " for gfid: %s", uuid_utoa (loc->inode->gfid));
-                goto err;
-        }
+    changelog_opt_t *co = NULL;
+    size_t xtra_len = 0;
+    char *dup_path = NULL;
+    char *bname = NULL;
+    inode_t *parent = NULL;
+
+    GF_ASSERT(this);
+
+    parent = inode_parent(loc->inode, 0, 0);
+    if (!parent) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, CHANGELOG_MSG_INODE_NOT_FOUND,
+                "type=parent", "gfid=%s", uuid_utoa(loc->inode->gfid), NULL);
+        goto err;
+    }
+
+    CHANGELOG_INIT_NOCHECK(this, *local, loc->inode, loc->inode->gfid, 5);
+    if (!(*local)) {
+        gf_smsg(this->name, GF_LOG_ERROR, 0, CHANGELOG_MSG_LOCAL_INIT_FAILED,
+                NULL);
+        goto err;
+    }
+
+    co = changelog_get_usable_buffer(*local);
+    if (!co) {
+        gf_smsg(this->name, GF_LOG_ERROR, 0, CHANGELOG_MSG_GET_BUFFER_FAILED,
+                NULL);
+        goto err;
+    }
+
+    if (loc->inode->ia_type == IA_IFDIR) {
+        CHANGLOG_FILL_FOP_NUMBER(co, GF_FOP_MKDIR, fop_fn, xtra_len);
+        co++;
+        CHANGELOG_FILL_UINT32(co, S_IFDIR | 0755, number_fn, xtra_len);
+        co++;
+    } else {
+        CHANGLOG_FILL_FOP_NUMBER(co, GF_FOP_CREATE, fop_fn, xtra_len);
+        co++;
+        CHANGELOG_FILL_UINT32(co, S_IFREG | 0644, number_fn, xtra_len);
+        co++;
+    }
 
-        CHANGELOG_INIT_NOCHECK (this, *local, loc->inode, loc->inode->gfid, 5);
-        if (!(*local)) {
-                gf_log (this->name, GF_LOG_ERROR, "changelog local"
-                        " initiatilization failed");
-                goto err;
-        }
+    CHANGELOG_FILL_UINT32(co, frame->root->uid, number_fn, xtra_len);
+    co++;
+
+    CHANGELOG_FILL_UINT32(co, frame->root->gid, number_fn, xtra_len);
+    co++;
+
+    dup_path = gf_strdup(loc->path);
+    bname = basename(dup_path);
+
+    CHANGELOG_FILL_ENTRY(co, parent->gfid, bname, entry_fn, entry_free_fn,
+                         xtra_len, err);
+    changelog_set_usable_record_and_length(*local, xtra_len, 5);
+
+    if (dup_path)
+        GF_FREE(dup_path);
+    if (parent)
+        inode_unref(parent);
+    return 0;
 
-        co = changelog_get_usable_buffer (*local);
-        if (!co) {
-                gf_log (this->name, GF_LOG_ERROR, "Failed to get buffer");
-                goto err;
+err:
+    if (dup_path)
+        GF_FREE(dup_path);
+    if (parent)
+        inode_unref(parent);
+    return -1;
+}
+
+/*
+ * resolve_pargfid_to_path:
+ *      It converts given pargfid to path by doing recursive readlinks at the
+ * backend. If bname is given, it suffixes bname to pargfid to form the
+ * complete path else it doesn't. It allocates memory for the path and is
+ * caller's responsibility to free the same. If bname is NULL and pargfid
+ * is ROOT, then it returns "."
+ */
+
+int
+resolve_pargfid_to_path(xlator_t *this, const uuid_t pgfid, char **path,
+                        char *bname)
+{
+    char *linkname = NULL;
+    char *dir_handle = NULL;
+    char *pgfidstr = NULL;
+    char *saveptr = NULL;
+    ssize_t len = 0;
+    int ret = 0;
+    uuid_t tmp_gfid = {
+        0,
+    };
+    uuid_t pargfid = {
+        0,
+    };
+    changelog_priv_t *priv = NULL;
+    char gpath[PATH_MAX] = {
+        0,
+    };
+    char result[PATH_MAX] = {
+        0,
+    };
+    char *dir_name = NULL;
+    char pre_dir_name[PATH_MAX] = {
+        0,
+    };
+
+    GF_ASSERT(this);
+    priv = this->private;
+    GF_ASSERT(priv);
+
+    gf_uuid_copy(pargfid, pgfid);
+    if (!path || gf_uuid_is_null(pargfid)) {
+        ret = -1;
+        goto out;
+    }
+
+    if (__is_root_gfid(pargfid)) {
+        if (bname)
+            *path = gf_strdup(bname);
+        else
+            *path = gf_strdup(".");
+        return ret;
+    }
+
+    dir_handle = alloca(PATH_MAX);
+    linkname = alloca(PATH_MAX);
+    (void)snprintf(gpath, PATH_MAX, "%s/.glusterfs/", priv->changelog_brick);
+
+    while (!(__is_root_gfid(pargfid))) {
+        len = snprintf(dir_handle, PATH_MAX, "%s/%02x/%02x/%s", gpath,
+                       pargfid[0], pargfid[1], uuid_utoa(pargfid));
+        if ((len < 0) || (len >= PATH_MAX)) {
+            ret = -1;
+            goto out;
         }
 
-        if (loc->inode->ia_type == IA_IFDIR) {
-                CHANGLOG_FILL_FOP_NUMBER (co, GF_FOP_MKDIR, fop_fn, xtra_len);
-                co++;
-                CHANGELOG_FILL_UINT32 (co, S_IFDIR|0755, number_fn, xtra_len);
-                co++;
-        } else {
-                CHANGLOG_FILL_FOP_NUMBER (co, GF_FOP_CREATE, fop_fn, xtra_len);
-                co++;
-                CHANGELOG_FILL_UINT32 (co, S_IFREG|0644, number_fn, xtra_len);
-                co++;
+        len = sys_readlink(dir_handle, linkname, PATH_MAX);
+        if (len < 0) {
+            gf_smsg(this->name, GF_LOG_ERROR, errno,
+                    CHANGELOG_MSG_READLINK_OP_FAILED,
+                    "could not read the "
+                    "link from the gfid handle",
+                    "handle=%s", dir_handle, NULL);
+            ret = -1;
+            goto out;
         }
 
-        CHANGELOG_FILL_UINT32 (co, frame->root->uid, number_fn, xtra_len);
-        co++;
+        linkname[len] = '\0';
 
-        CHANGELOG_FILL_UINT32 (co, frame->root->gid, number_fn, xtra_len);
-        co++;
+        pgfidstr = strtok_r(linkname + strlen("../../00/00/"), "/", &saveptr);
+        dir_name = strtok_r(NULL, "/", &saveptr);
 
-        dup_path = gf_strdup (loc->path);
-        bname = basename (dup_path);
+        len = snprintf(result, PATH_MAX, "%s/%s", dir_name, pre_dir_name);
+        if ((len < 0) || (len >= PATH_MAX)) {
+            ret = -1;
+            goto out;
+        }
+        if (snprintf(pre_dir_name, len + 1, "%s", result) >= len + 1) {
+            ret = -1;
+            goto out;
+        }
 
-        CHANGELOG_FILL_ENTRY (co, parent->gfid, bname, entry_fn, entry_free_fn,
-                              xtra_len, err);
-        changelog_set_usable_record_and_length (*local, xtra_len, 5);
+        gf_uuid_parse(pgfidstr, tmp_gfid);
+        gf_uuid_copy(pargfid, tmp_gfid);
+    }
 
-        if (dup_path)
-                GF_FREE (dup_path);
-        if (parent)
-                inode_unref (parent);
-        return 0;
+    if (bname)
+        strncat(result, bname, strlen(bname) + 1);
 
-err:
-        if (dup_path)
-                GF_FREE (dup_path);
-        if (parent)
-                inode_unref (parent);
-        return -1;
+    *path = gf_strdup(result);
+
+out:
+    return ret;
 }
diff --git a/xlators/features/changelog/src/changelog-helpers.h b/xlators/features/changelog/src/changelog-helpers.h
index 765d5fd878b..38fa7590c32 100644
--- a/xlators/features/changelog/src/changelog-helpers.h
+++ b/xlators/features/changelog/src/changelog-helpers.h
@@ -11,62 +11,63 @@
 #ifndef _CHANGELOG_HELPERS_H
 #define _CHANGELOG_HELPERS_H
 
-#include "locking.h"
-#include "timer.h"
+#include <glusterfs/locking.h>
+#include <glusterfs/timer.h>
 #include "pthread.h"
-#include "iobuf.h"
-#include "rot-buffs.h"
+#include <glusterfs/iobuf.h>
+#include <glusterfs/rot-buffs.h>
 
 #include "changelog-misc.h"
-#include "call-stub.h"
+#include <glusterfs/call-stub.h>
 
 #include "rpcsvc.h"
 #include "changelog-ev-handle.h"
 
 #include "changelog.h"
+#include "changelog-messages.h"
 
 /**
  * the changelog entry
  */
 typedef struct changelog_log_data {
-        /* rollover related */
-        unsigned long cld_roll_time;
+    /* rollover related */
+    time_t cld_roll_time;
 
-        /* reopen changelog? */
-        gf_boolean_t cld_finale;
+    /* reopen changelog? */
+    gf_boolean_t cld_finale;
 
-        changelog_log_type cld_type;
+    changelog_log_type cld_type;
 
-        /**
-         * sincd gfid is _always_ a necessity, it's not a part
-         * of the iobuf. by doing this we do not add any overhead
-         * for data and metadata related fops.
-         */
-        uuid_t        cld_gfid;
+    /**
+     * sincd gfid is _always_ a necessity, it's not a part
+     * of the iobuf. by doing this we do not add any overhead
+     * for data and metadata related fops.
+     */
+    uuid_t cld_gfid;
 
-        /**
-         * iobufs are used for optionals records: pargfid, path,
-         * write offsets etc.. It's the fop implementers job
-         * to allocate (iobuf_get() in the fop) and get unref'ed
-         * in the callback (CHANGELOG_STACK_UNWIND).
-         */
-        struct iobuf *cld_iobuf;
+    /**
+     * iobufs are used for optionals records: pargfid, path,
+     * write offsets etc.. It's the fop implementers job
+     * to allocate (iobuf_get() in the fop) and get unref'ed
+     * in the callback (CHANGELOG_STACK_UNWIND).
+     */
+    struct iobuf *cld_iobuf;
 
 #define cld_ptr cld_iobuf->ptr
 
-        /**
-         * after allocation you can point this to the length of
-         * usable data, but make sure it does not exceed the
-         * the size of the requested iobuf.
-         */
-        size_t        cld_iobuf_len;
+    /**
+     * after allocation you can point this to the length of
+     * usable data, but make sure it does not exceed the
+     * the size of the requested iobuf.
+     */
+    size_t cld_iobuf_len;
 
 #define cld_ptr_len cld_iobuf_len
 
-        /**
-         * number of optional records
-         */
-        int cld_xtra_records;
+    /**
+     * number of optional records
+     */
+    int cld_xtra_records;
 } changelog_log_data_t;
 
 /**
@@ -76,54 +77,48 @@ typedef struct changelog_log_data {
 typedef struct changelog_priv changelog_priv_t;
 
 typedef struct changelog_dispatcher {
-        void *cd_data;
-        int (*dispatchfn) (xlator_t *, changelog_priv_t *, void *,
-                           changelog_log_data_t *, changelog_log_data_t *);
+    void *cd_data;
+    int (*dispatchfn)(xlator_t *, changelog_priv_t *, void *,
+                      changelog_log_data_t *, changelog_log_data_t *);
 } changelog_dispatcher_t;
 
 struct changelog_bootstrap {
-        changelog_mode_t mode;
-        int (*ctor) (xlator_t *, changelog_dispatcher_t *);
-        int (*dtor) (xlator_t *, changelog_dispatcher_t *);
+    changelog_mode_t mode;
+    int (*ctor)(xlator_t *, changelog_dispatcher_t *);
+    int (*dtor)(xlator_t *, changelog_dispatcher_t *);
 };
 
 struct changelog_encoder {
-        changelog_encoder_t encoder;
-        int (*encode) (xlator_t *, changelog_log_data_t *);
+    changelog_encoder_t encoder;
+    int (*encode)(xlator_t *, changelog_log_data_t *);
 };
 
-
 /* xlator private */
 
 typedef struct changelog_time_slice {
-        /**
-         * just in case we need nanosecond granularity some day.
-         * field is unused as of now (maybe we'd need it later).
-         */
-        struct timeval tv_start;
-
-        /**
-         * version of changelog file, incremented each time changes
-         * rollover.
-         */
-        unsigned long changelog_version[CHANGELOG_MAX_TYPE];
+    /**
+     * version of changelog file, incremented each time changes
+     * rollover.
+     */
+    unsigned long changelog_version[CHANGELOG_MAX_TYPE];
 } changelog_time_slice_t;
 
 typedef struct changelog_rollover {
-        /* rollover thread */
-        pthread_t rollover_th;
+    /* rollover thread */
+    pthread_t rollover_th;
 
-        xlator_t *this;
+    xlator_t *this;
 
-        /* read end of pipe used as event from barrier on snapshot */
-        int rfd;
+    pthread_mutex_t lock;
+    pthread_cond_t cond;
+    gf_boolean_t notify;
 } changelog_rollover_t;
 
 typedef struct changelog_fsync {
-        /* fsync() thread */
-        pthread_t fsync_th;
+    /* fsync() thread */
+    pthread_t fsync_th;
 
-        xlator_t *this;
+    xlator_t *this;
 } changelog_fsync_t;
 
 /* Draining during changelog rollover (for geo-rep snapshot dependency):
@@ -143,197 +138,219 @@ typedef struct changelog_fsync {
  */
 
 typedef enum chlog_fop_color {
-         FOP_COLOR_BLACK,
-         FOP_COLOR_WHITE
+    FOP_COLOR_BLACK,
+    FOP_COLOR_WHITE
 } chlog_fop_color_t;
 
 /* Barrier notify variable */
 typedef struct barrier_notify {
-         pthread_mutex_t        bnotify_mutex;
-         pthread_cond_t         bnotify_cond;
-         gf_boolean_t           bnotify;
+    pthread_mutex_t bnotify_mutex;
+    pthread_cond_t bnotify_cond;
+    gf_boolean_t bnotify;
+    gf_boolean_t bnotify_error;
 } barrier_notify_t;
 
 /* Two separate mutex and conditional variable set is used
  * to drain white and black fops. */
 
 typedef struct drain_mgmt {
-         pthread_mutex_t        drain_black_mutex;
-         pthread_cond_t         drain_black_cond;
-         pthread_mutex_t        drain_white_mutex;
-         pthread_cond_t         drain_white_cond;
-         /* Represents black fops count in-transit */
-         unsigned long          black_fop_cnt;
-         /* Represents white fops count in-transit */
-         unsigned long          white_fop_cnt;
-         gf_boolean_t           drain_wait_black;
-         gf_boolean_t           drain_wait_white;
+    pthread_mutex_t drain_black_mutex;
+    pthread_cond_t drain_black_cond;
+    pthread_mutex_t drain_white_mutex;
+    pthread_cond_t drain_white_cond;
+    /* Represents black fops count in-transit */
+    unsigned long black_fop_cnt;
+    /* Represents white fops count in-transit */
+    unsigned long white_fop_cnt;
+    gf_boolean_t drain_wait_black;
+    gf_boolean_t drain_wait_white;
 } drain_mgmt_t;
 
 /* External barrier as a result of snap on/off indicating flag*/
 typedef struct barrier_flags {
-        gf_lock_t lock;
-        gf_boolean_t barrier_ext;
+    gf_lock_t lock;
+    gf_boolean_t barrier_ext;
 } barrier_flags_t;
 
 /* Event selection */
 typedef struct changelog_ev_selector {
-        gf_lock_t reflock;
+    gf_lock_t reflock;
 
-        /**
-         * Array of references for each selection bit.
-         */
-        unsigned int ref[CHANGELOG_EV_SELECTION_RANGE];
+    /**
+     * Array of references for each selection bit.
+     */
+    unsigned int ref[CHANGELOG_EV_SELECTION_RANGE];
 } changelog_ev_selector_t;
 
-
 /* changelog's private structure */
 struct changelog_priv {
-        gf_boolean_t active;
+    /* changelog journalling */
+    gf_boolean_t active;
+
+    /* changelog live notifications */
+    gf_boolean_t rpc_active;
+
+    /* to generate unique socket file per brick */
+    char *changelog_brick;
+
+    /* logging directory */
+    char *changelog_dir;
+
+    /* htime directory */
+    char *htime_dir;
 
-        /* to generate unique socket file per brick */
-        char *changelog_brick;
+    /* one file for all changelog types */
+    int changelog_fd;
 
-        /* logging directory */
-        char *changelog_dir;
+    /* htime fd for current changelog session */
+    int htime_fd;
 
-        /* htime directory */
-        char *htime_dir;
+    /*  c_snap_fd is fd for call-path changelog */
+    int c_snap_fd;
 
-        /* one file for all changelog types */
-        int changelog_fd;
+    /* rollover_count used by htime */
+    int rollover_count;
 
-        /* htime fd for current changelog session */
-        int htime_fd;
+    gf_lock_t lock;
 
-        /*  c_snap_fd is fd for call-path changelog */
-        int c_snap_fd;
+    /*  lock to synchronize CSNAP updation */
+    gf_lock_t c_snap_lock;
 
-        /* rollover_count used by htime */
-        int  rollover_count;
+    /* written end of the pipe */
+    int wfd;
 
-        gf_lock_t lock;
+    /* rollover time */
+    int32_t rollover_time;
 
-        /*  lock to synchronize CSNAP updation */
-        gf_lock_t c_snap_lock;
+    /* fsync() interval */
+    int32_t fsync_interval;
 
-        /* written end of the pipe */
-        int wfd;
+    /* changelog type maps */
+    const char *maps[CHANGELOG_MAX_TYPE];
 
-        /* rollover time */
-        int32_t rollover_time;
+    /* time slicer */
+    changelog_time_slice_t slice;
 
-        /* fsync() interval */
-        int32_t fsync_interval;
+    /* context of the updater */
+    changelog_dispatcher_t cd;
 
-        /* changelog type maps */
-        const char *maps[CHANGELOG_MAX_TYPE];
+    /* context of the rollover thread */
+    changelog_rollover_t cr;
 
-        /* time slicer */
-        changelog_time_slice_t slice;
+    /* context of fsync thread */
+    changelog_fsync_t cf;
 
-        /* context of the updater */
-        changelog_dispatcher_t cd;
+    /* operation mode */
+    changelog_mode_t op_mode;
 
-        /* context of the rollover thread */
-        changelog_rollover_t cr;
+    /* bootstrap routine for 'current' logger */
+    struct changelog_bootstrap *cb;
 
-        /* context of fsync thread */
-        changelog_fsync_t cf;
+    /* encoder mode */
+    changelog_encoder_t encode_mode;
 
-        /* operation mode */
-        changelog_mode_t op_mode;
+    /* encoder */
+    struct changelog_encoder *ce;
 
-        /* bootstrap routine for 'current' logger */
-        struct changelog_bootstrap *cb;
+    /**
+     * snapshot dependency changes
+     */
 
-        /* encoder mode */
-        changelog_encoder_t encode_mode;
+    /* Draining of fops*/
+    drain_mgmt_t dm;
 
-        /* encoder */
-        struct changelog_encoder *ce;
+    /* Represents the active color. Initially by default black */
+    chlog_fop_color_t current_color;
 
-        /**
-         * snapshot dependency changes
-         */
+    /* flag to determine explicit rollover is triggered */
+    gf_boolean_t explicit_rollover;
 
-        /* Draining of fops*/
-        drain_mgmt_t dm;
+    /* barrier notification variable protected by mutex */
+    barrier_notify_t bn;
 
-        /* Represents the active color. Initially by default black */
-        chlog_fop_color_t current_color;
+    /* barrier on/off indicating flags */
+    barrier_flags_t bflags;
 
-        /* write end of pipe to do explicit rollover on barrier during snap */
-        int cr_wfd;
+    /* changelog barrier on/off indicating flag */
+    gf_boolean_t barrier_enabled;
+    struct list_head queue;
+    uint32_t queue_size;
+    gf_timer_t *timer;
+    struct timespec timeout;
 
-        /* flag to determine explicit rollover is triggered */
-        gf_boolean_t explicit_rollover;
+    /**
+     * buffers, RPC, event selection, notifications and other
+     * beasts.
+     */
 
-        /* barrier notification variable protected by mutex */
-        barrier_notify_t bn;
+    /* epoll pthread */
+    pthread_t poller;
 
-        /* barrier on/off indicating flags */
-        barrier_flags_t bflags;
+    /* rotational buffer */
+    rbuf_t *rbuf;
 
-        /* changelog barrier on/off indicating flag */
-        gf_boolean_t      barrier_enabled;
-        struct list_head  queue;
-        uint32_t          queue_size;
-        gf_timer_t       *timer;
-        struct timespec   timeout;
+    /* changelog RPC server */
+    rpcsvc_t *rpc;
 
-        /**
-         * buffers, RPC, event selection, notifications and other
-         * beasts.
-         */
+    /* event selection */
+    changelog_ev_selector_t ev_selection;
 
-        /* epoll pthread */
-        pthread_t poller;
+    /* client handling (reverse connection) */
+    pthread_t connector;
 
-        /* rotational buffer */
-        rbuf_t *rbuf;
+    int nr_dispatchers;
+    pthread_t *ev_dispatcher;
 
-        /* changelog RPC server */
-        rpcsvc_t *rpc;
+    changelog_clnt_t connections;
 
-        /* event selection */
-        changelog_ev_selector_t ev_selection;
+    /* glusterfind dependency to capture paths on deleted entries*/
+    gf_boolean_t capture_del_path;
 
-        /* client handling (reverse connection) */
-        pthread_t connector;
+    /* Save total no. of listners */
+    gf_atomic_t listnercnt;
 
-        int nr_dispatchers;
-        pthread_t *ev_dispatcher;
+    /* Save total no. of xprt are associated with listner */
+    gf_atomic_t xprtcnt;
 
-        changelog_clnt_t connections;
+    /* Save xprt list */
+    struct list_head xprt_list;
+
+    /* Save total no. of client connection */
+    gf_atomic_t clntcnt;
+
+    /* Save cleanup brick in victim */
+    xlator_t *victim;
+
+    /* Status to save cleanup notify status */
+    gf_boolean_t notify_down;
 };
 
 struct changelog_local {
-        inode_t              *inode;
-        gf_boolean_t          update_no_check;
+    inode_t *inode;
+    gf_boolean_t update_no_check;
 
-        changelog_log_data_t  cld;
+    changelog_log_data_t cld;
 
-        /**
-         * ->prev_entry is used in cases when there needs to be
-         * additional changelog entry for the parent (eg. rename)
-         * It's analogous to ->next in single linked list world,
-         * but we call it as ->prev_entry... ha ha ha
-         */
-        struct changelog_local *prev_entry;
+    /**
+     * ->prev_entry is used in cases when there needs to be
+     * additional changelog entry for the parent (eg. rename)
+     * It's analogous to ->next in single linked list world,
+     * but we call it as ->prev_entry... ha ha ha
+     */
+    struct changelog_local *prev_entry;
 
-        /* snap dependency changes */
-        chlog_fop_color_t color;
+    /* snap dependency changes */
+    chlog_fop_color_t color;
 };
 
 typedef struct changelog_local changelog_local_t;
 
 /* inode version is stored in inode ctx */
 typedef struct changelog_inode_ctx {
-        unsigned long iversion[CHANGELOG_MAX_TYPE];
+    unsigned long iversion[CHANGELOG_MAX_TYPE];
 } changelog_inode_ctx_t;
 
-#define CHANGELOG_INODE_VERSION_TYPE(ctx, type)  &(ctx->iversion[type])
+#define CHANGELOG_INODE_VERSION_TYPE(ctx, type) &(ctx->iversion[type])
 
 /**
  * Optional Records:
@@ -341,242 +358,276 @@ typedef struct changelog_inode_ctx {
  *  @changelog_opt_t struct. The array is allocated via @iobufs.
  */
 typedef enum {
-        CHANGELOG_OPT_REC_FOP,
-        CHANGELOG_OPT_REC_ENTRY,
-        CHANGELOG_OPT_REC_UINT32,
+    CHANGELOG_OPT_REC_FOP,
+    CHANGELOG_OPT_REC_ENTRY,
+    CHANGELOG_OPT_REC_UINT32,
 } changelog_optional_rec_type_t;
 
 struct changelog_entry_fields {
-        uuid_t  cef_uuid;
-        char   *cef_bname;
+    uuid_t cef_uuid;
+    char *cef_bname;
+    char *cef_path;
 };
 
 typedef struct {
-        /**
-         * @co_covert can be used to do post-processing of the record before
-         * it's persisted to the CHANGELOG. If this is NULL, then the record
-         * is persisted as per it's in memory format.
-         */
-        size_t (*co_convert) (void *data, char *buffer, gf_boolean_t encode);
-
-        /* release routines */
-        void (*co_free) (void *data);
-
-        /* type of the field */
-        changelog_optional_rec_type_t co_type;
-
-        /**
-         * sizeof of the 'valid' field in the union. This field is not used if
-         * @co_convert is specified.
-         */
-        size_t co_len;
-
-        union {
-                unsigned int                  co_uint32;
-                glusterfs_fop_t               co_fop;
-                struct changelog_entry_fields co_entry;
-        };
+    /**
+     * @co_covert can be used to do post-processing of the record before
+     * it's persisted to the CHANGELOG. If this is NULL, then the record
+     * is persisted as per it's in memory format.
+     */
+    size_t (*co_convert)(void *data, char *buffer, gf_boolean_t encode);
+
+    /* release routines */
+    void (*co_free)(void *data);
+
+    /* type of the field */
+    changelog_optional_rec_type_t co_type;
+
+    /**
+     * sizeof of the 'valid' field in the union. This field is not used if
+     * @co_convert is specified.
+     */
+    size_t co_len;
+
+    union {
+        unsigned int co_uint32;
+        glusterfs_fop_t co_fop;
+        struct changelog_entry_fields co_entry;
+    };
 } changelog_opt_t;
 
-#define CHANGELOG_OPT_RECORD_LEN  sizeof (changelog_opt_t)
+#define CHANGELOG_OPT_RECORD_LEN sizeof(changelog_opt_t)
 
 /**
  * helpers routines
  */
 
 int
-changelog_thread_cleanup (xlator_t *this, pthread_t thr_id);
+changelog_thread_cleanup(xlator_t *this, pthread_t thr_id);
 
 void *
-changelog_get_usable_buffer (changelog_local_t *local);
+changelog_get_usable_buffer(changelog_local_t *local);
 
 void
-changelog_set_usable_record_and_length (changelog_local_t *local,
-                                        size_t len, int xr);
+changelog_set_usable_record_and_length(changelog_local_t *local, size_t len,
+                                       int xr);
 void
-changelog_local_cleanup (xlator_t *xl, changelog_local_t *local);
+changelog_local_cleanup(xlator_t *xl, changelog_local_t *local);
 changelog_local_t *
-changelog_local_init (xlator_t *this, inode_t *inode, uuid_t gfid,
-                      int xtra_records, gf_boolean_t update_flag);
+changelog_local_init(xlator_t *this, inode_t *inode, uuid_t gfid,
+                     int xtra_records, gf_boolean_t update_flag);
 int
-changelog_start_next_change (xlator_t *this,
-                             changelog_priv_t *priv,
-                             unsigned long ts, gf_boolean_t finale);
+changelog_start_next_change(xlator_t *this, changelog_priv_t *priv, time_t ts,
+                            gf_boolean_t finale);
 int
-changelog_open_journal (xlator_t *this, changelog_priv_t *priv);
-int
-changelog_fill_rollover_data (changelog_log_data_t *cld, gf_boolean_t is_last);
+changelog_open_journal(xlator_t *this, changelog_priv_t *priv);
+void
+changelog_fill_rollover_data(changelog_log_data_t *cld, gf_boolean_t is_last);
 int
-changelog_inject_single_event (xlator_t *this,
-                               changelog_priv_t *priv,
-                               changelog_log_data_t *cld);
+changelog_inject_single_event(xlator_t *this, changelog_priv_t *priv,
+                              changelog_log_data_t *cld);
 size_t
-changelog_entry_length ();
+changelog_entry_length();
 int
-changelog_write (int fd, char *buffer, size_t len);
+changelog_write(int fd, char *buffer, size_t len);
 int
-changelog_write_change (changelog_priv_t *priv, char *buffer, size_t len);
+changelog_write_change(changelog_priv_t *priv, char *buffer, size_t len);
 int
-changelog_handle_change (xlator_t *this,
-                         changelog_priv_t *priv, changelog_log_data_t *cld);
+changelog_handle_change(xlator_t *this, changelog_priv_t *priv,
+                        changelog_log_data_t *cld);
 void
-changelog_update (xlator_t *this, changelog_priv_t *priv,
-                  changelog_local_t *local, changelog_log_type type);
+changelog_update(xlator_t *this, changelog_priv_t *priv,
+                 changelog_local_t *local, changelog_log_type type);
 void *
-changelog_rollover (void *data);
+changelog_rollover(void *data);
 void *
-changelog_fsync_thread (void *data);
+changelog_fsync_thread(void *data);
+int
+changelog_forget(xlator_t *this, inode_t *inode);
 int
-changelog_forget (xlator_t *this, inode_t *inode);
+htime_update(xlator_t *this, changelog_priv_t *priv, time_t ts, char *buffer);
 int
-htime_update (xlator_t *this, changelog_priv_t *priv,
-              unsigned long ts, char * buffer);
+htime_open(xlator_t *this, changelog_priv_t *priv, time_t ts);
 int
-htime_open (xlator_t *this, changelog_priv_t * priv, unsigned long ts);
+htime_create(xlator_t *this, changelog_priv_t *priv, time_t ts);
 
 /* Geo-Rep snapshot dependency changes */
 void
-changelog_color_fop_and_inc_cnt (xlator_t *this, changelog_priv_t *priv,
-                                                 changelog_local_t *local);
+changelog_color_fop_and_inc_cnt(xlator_t *this, changelog_priv_t *priv,
+                                changelog_local_t *local);
 void
-changelog_inc_fop_cnt (xlator_t *this, changelog_priv_t *priv,
-                                       changelog_local_t *local);
+changelog_inc_fop_cnt(xlator_t *this, changelog_priv_t *priv,
+                      changelog_local_t *local);
 void
-changelog_dec_fop_cnt (xlator_t *this, changelog_priv_t *priv,
-                                       changelog_local_t *local);
+changelog_dec_fop_cnt(xlator_t *this, changelog_priv_t *priv,
+                      changelog_local_t *local);
 int
-changelog_barrier_notify (changelog_priv_t *priv, char* buf);
+changelog_barrier_notify(changelog_priv_t *priv, char *buf);
 void
-changelog_barrier_cleanup (xlator_t *this, changelog_priv_t *priv,
-                                                struct list_head *queue);
+changelog_barrier_cleanup(xlator_t *this, changelog_priv_t *priv,
+                          struct list_head *queue);
 void
-changelog_drain_white_fops (xlator_t *this, changelog_priv_t *priv);
+changelog_drain_white_fops(xlator_t *this, changelog_priv_t *priv);
 void
-changelog_drain_black_fops (xlator_t *this, changelog_priv_t *priv);
+changelog_drain_black_fops(xlator_t *this, changelog_priv_t *priv);
 
 /* Crash consistency of changelog wrt snapshot */
 int
-changelog_snap_logging_stop ( xlator_t *this, changelog_priv_t *priv);
+changelog_snap_logging_stop(xlator_t *this, changelog_priv_t *priv);
 int
-changelog_snap_logging_start ( xlator_t *this, changelog_priv_t *priv);
+changelog_snap_logging_start(xlator_t *this, changelog_priv_t *priv);
 int
-changelog_snap_open ( xlator_t *this, changelog_priv_t *priv);
+changelog_snap_open(xlator_t *this, changelog_priv_t *priv);
 int
-changelog_snap_handle_ascii_change (xlator_t *this,
-                changelog_log_data_t *cld);
+changelog_snap_handle_ascii_change(xlator_t *this, changelog_log_data_t *cld);
 int
-changelog_snap_write_change (changelog_priv_t *priv, char *buffer, size_t len);
+changelog_snap_write_change(changelog_priv_t *priv, char *buffer, size_t len);
 
 /* Changelog barrier routines */
-void __chlog_barrier_enqueue (xlator_t *this, call_stub_t *stub);
-void __chlog_barrier_disable (xlator_t *this, struct list_head *queue);
-void chlog_barrier_dequeue_all (xlator_t *this, struct list_head *queue);
-call_stub_t *__chlog_barrier_dequeue (xlator_t *this, struct list_head *queue);
-int __chlog_barrier_enable (xlator_t *this, changelog_priv_t *priv);
+void
+__chlog_barrier_enqueue(xlator_t *this, call_stub_t *stub);
+void
+__chlog_barrier_disable(xlator_t *this, struct list_head *queue);
+void
+chlog_barrier_dequeue_all(xlator_t *this, struct list_head *queue);
+call_stub_t *
+__chlog_barrier_dequeue(xlator_t *this, struct list_head *queue);
+int
+__chlog_barrier_enable(xlator_t *this, changelog_priv_t *priv);
 
 int32_t
-changelog_fill_entry_buf (call_frame_t *frame, xlator_t *this,
-                          loc_t *loc, changelog_local_t **local);
+changelog_fill_entry_buf(call_frame_t *frame, xlator_t *this, loc_t *loc,
+                         changelog_local_t **local);
 
 /* event selection routines */
-inline void changelog_select_event (xlator_t *,
-                                    changelog_ev_selector_t *, unsigned int);
-inline void changelog_deselect_event (xlator_t *,
-                                      changelog_ev_selector_t *, unsigned int);
-inline int changelog_init_event_selection (xlator_t *,
-                                           changelog_ev_selector_t *);
-inline int changelog_cleanup_event_selection (xlator_t *,
-                                              changelog_ev_selector_t *);
-inline int changelog_ev_selected (xlator_t *,
-                                  changelog_ev_selector_t *, unsigned int);
-inline void
-changelog_dispatch_event (xlator_t *, changelog_priv_t *, changelog_event_t *);
+void
+changelog_select_event(xlator_t *, changelog_ev_selector_t *, unsigned int);
+void
+changelog_deselect_event(xlator_t *, changelog_ev_selector_t *, unsigned int);
+int
+changelog_init_event_selection(xlator_t *, changelog_ev_selector_t *);
+int
+changelog_ev_selected(xlator_t *, changelog_ev_selector_t *, unsigned int);
+void
+changelog_dispatch_event(xlator_t *, changelog_priv_t *, changelog_event_t *);
 
 changelog_inode_ctx_t *
-__changelog_inode_ctx_get (xlator_t *, inode_t *, unsigned long **,
-                           unsigned long *, changelog_log_type);
+__changelog_inode_ctx_get(xlator_t *, inode_t *, unsigned long **,
+                          unsigned long *, changelog_log_type);
+int
+resolve_pargfid_to_path(xlator_t *this, const uuid_t gfid, char **path,
+                        char *bname);
 
 /* macros */
 
-#define CHANGELOG_STACK_UNWIND(fop, frame, params ...) do {             \
-                changelog_local_t *__local = NULL;                      \
-                xlator_t          *__xl    = NULL;                      \
-                if (frame) {                                            \
-                        __local      = frame->local;                    \
-                        __xl         = frame->this;                     \
-                        frame->local = NULL;                            \
-                }                                                       \
-                STACK_UNWIND_STRICT (fop, frame, params);               \
-                if (__local && __local->prev_entry)                     \
-                        changelog_local_cleanup (__xl,                  \
-                                                 __local->prev_entry);  \
-                changelog_local_cleanup (__xl, __local);                \
-        } while (0)
-
-#define CHANGELOG_IOBUF_REF(iobuf) do {         \
-                if (iobuf)                      \
-                        iobuf_ref (iobuf);      \
-        } while (0)
-
-#define CHANGELOG_IOBUF_UNREF(iobuf) do {       \
-                if (iobuf)                      \
-                        iobuf_unref (iobuf);    \
-        } while (0)
-
-#define CHANGELOG_FILL_BUFFER(buffer, off, val, len) do {       \
-                memcpy (buffer + off, val, len);                \
-                off += len;                                     \
-        } while (0)
-
-#define SLICE_VERSION_UPDATE(slice) do {                \
-                int i = 0;                              \
-                for (; i < CHANGELOG_MAX_TYPE; i++) {   \
-                        slice->changelog_version[i]++;  \
-                }                                       \
-        } while (0)
-
-#define CHANGELOG_FILL_UINT32(co, number, converter, xlen) do { \
-                co->co_convert = converter;                     \
-                co->co_free = NULL;                             \
-                co->co_type = CHANGELOG_OPT_REC_UINT32;         \
-                co->co_uint32 = number;                         \
-                xlen += sizeof (unsigned int);                 \
-        } while (0)
-
-#define CHANGLOG_FILL_FOP_NUMBER(co, fop, converter, xlen) do { \
-                co->co_convert = converter;                     \
-                co->co_free = NULL;                             \
-                co->co_type = CHANGELOG_OPT_REC_FOP;            \
-                co->co_fop = fop;                               \
-                xlen += sizeof (fop);                           \
-        } while (0)
-
-#define CHANGELOG_FILL_ENTRY(co, pargfid, bname,                        \
-                             converter, freefn, xlen, label)            \
-        do {                                                            \
-                co->co_convert = converter;                             \
-                co->co_free = freefn;                                   \
-                co->co_type = CHANGELOG_OPT_REC_ENTRY;                  \
-                gf_uuid_copy (co->co_entry.cef_uuid, pargfid);          \
-                co->co_entry.cef_bname = gf_strdup(bname);              \
-                if (!co->co_entry.cef_bname)                            \
-                        goto label;                                     \
-                xlen += (UUID_CANONICAL_FORM_LEN + strlen (bname));     \
-        } while (0)
-
-#define CHANGELOG_INIT(this, local, inode, gfid, xrec)                  \
-        local = changelog_local_init (this, inode, gfid, xrec, _gf_false)
-
-#define CHANGELOG_INIT_NOCHECK(this, local, inode, gfid, xrec)          \
-        local = changelog_local_init (this, inode, gfid, xrec, _gf_true)
-
-#define CHANGELOG_NOT_ACTIVE_THEN_GOTO(frame, priv, label) do { \
-                if (!priv->active)                              \
-                        goto label;                             \
-                /* ignore rebalance process's activity. */      \
-                if (frame->root->pid == GF_CLIENT_PID_DEFRAG)   \
-                        goto label;                             \
-        } while (0)
+#define CHANGELOG_STACK_UNWIND(fop, frame, params...)                          \
+    do {                                                                       \
+        changelog_local_t *__local = NULL;                                     \
+        xlator_t *__xl = NULL;                                                 \
+        if (frame) {                                                           \
+            __local = frame->local;                                            \
+            __xl = frame->this;                                                \
+            frame->local = NULL;                                               \
+        }                                                                      \
+        STACK_UNWIND_STRICT(fop, frame, params);                               \
+        if (__local && __local->prev_entry)                                    \
+            changelog_local_cleanup(__xl, __local->prev_entry);                \
+        changelog_local_cleanup(__xl, __local);                                \
+    } while (0)
+
+#define CHANGELOG_IOBUF_REF(iobuf)                                             \
+    do {                                                                       \
+        if (iobuf)                                                             \
+            iobuf_ref(iobuf);                                                  \
+    } while (0)
+
+#define CHANGELOG_IOBUF_UNREF(iobuf)                                           \
+    do {                                                                       \
+        if (iobuf)                                                             \
+            iobuf_unref(iobuf);                                                \
+    } while (0)
+
+#define CHANGELOG_FILL_BUFFER(buffer, off, val, len)                           \
+    do {                                                                       \
+        memcpy(buffer + off, val, len);                                        \
+        off += len;                                                            \
+    } while (0)
+
+#define SLICE_VERSION_UPDATE(slice)                                            \
+    do {                                                                       \
+        int i = 0;                                                             \
+        for (; i < CHANGELOG_MAX_TYPE; i++) {                                  \
+            slice->changelog_version[i]++;                                     \
+        }                                                                      \
+    } while (0)
+
+#define CHANGELOG_FILL_UINT32(co, number, converter, xlen)                     \
+    do {                                                                       \
+        co->co_convert = converter;                                            \
+        co->co_free = NULL;                                                    \
+        co->co_type = CHANGELOG_OPT_REC_UINT32;                                \
+        co->co_uint32 = number;                                                \
+        xlen += sizeof(unsigned int);                                          \
+    } while (0)
+
+#define CHANGLOG_FILL_FOP_NUMBER(co, fop, converter, xlen)                     \
+    do {                                                                       \
+        co->co_convert = converter;                                            \
+        co->co_free = NULL;                                                    \
+        co->co_type = CHANGELOG_OPT_REC_FOP;                                   \
+        co->co_fop = fop;                                                      \
+        xlen += sizeof(fop);                                                   \
+    } while (0)
+
+#define CHANGELOG_FILL_ENTRY(co, pargfid, bname, converter, freefn, xlen,      \
+                             label)                                            \
+    do {                                                                       \
+        co->co_convert = converter;                                            \
+        co->co_free = freefn;                                                  \
+        co->co_type = CHANGELOG_OPT_REC_ENTRY;                                 \
+        gf_uuid_copy(co->co_entry.cef_uuid, pargfid);                          \
+        co->co_entry.cef_bname = gf_strdup(bname);                             \
+        if (!co->co_entry.cef_bname)                                           \
+            goto label;                                                        \
+        xlen += (UUID_CANONICAL_FORM_LEN + strlen(bname));                     \
+    } while (0)
+
+#define CHANGELOG_FILL_ENTRY_DIR_PATH(co, pargfid, bname, converter,           \
+                                      del_freefn, xlen, label, capture_del)    \
+    do {                                                                       \
+        co->co_convert = converter;                                            \
+        co->co_free = del_freefn;                                              \
+        co->co_type = CHANGELOG_OPT_REC_ENTRY;                                 \
+        gf_uuid_copy(co->co_entry.cef_uuid, pargfid);                          \
+        co->co_entry.cef_bname = gf_strdup(bname);                             \
+        if (!co->co_entry.cef_bname)                                           \
+            goto label;                                                        \
+        xlen += (UUID_CANONICAL_FORM_LEN + strlen(bname));                     \
+        if (!capture_del ||                                                    \
+            resolve_pargfid_to_path(this, pargfid, &(co->co_entry.cef_path),   \
+                                    co->co_entry.cef_bname)) {                 \
+            co->co_entry.cef_path = gf_strdup("\0");                           \
+            xlen += 1;                                                         \
+        } else {                                                               \
+            xlen += (strlen(co->co_entry.cef_path));                           \
+        }                                                                      \
+    } while (0)
+
+#define CHANGELOG_INIT(this, local, inode, gfid, xrec)                         \
+    local = changelog_local_init(this, inode, gfid, xrec, _gf_false)
+
+#define CHANGELOG_INIT_NOCHECK(this, local, inode, gfid, xrec)                 \
+    local = changelog_local_init(this, inode, gfid, xrec, _gf_true)
+
+#define CHANGELOG_NOT_ACTIVE_THEN_GOTO(frame, priv, label)                     \
+    do {                                                                       \
+        if (!priv->active)                                                     \
+            goto label;                                                        \
+        /* ignore rebalance process's activity. */                             \
+        if ((frame->root->pid == GF_CLIENT_PID_DEFRAG) ||                      \
+            (frame->root->pid == GF_CLIENT_PID_TIER_DEFRAG))                   \
+            goto label;                                                        \
+    } while (0)
 
 /* If it is a METADATA entry and fop num being GF_FOP_NULL, don't
  * log in the changelog as it is of no use. And also if it is
@@ -585,63 +636,81 @@ __changelog_inode_ctx_get (xlator_t *, inode_t *, unsigned long **,
  * to same changelog will be missed. Hence check for boundary
  * condition.
  */
-#define CHANGELOG_OP_BOUNDARY_CHECK(frame, label) do {          \
-                if (frame->root->op <= GF_FOP_NULL ||           \
-                    frame->root->op >= GF_FOP_MAXVALUE)         \
-                        goto label;                             \
-        } while (0)
+#define CHANGELOG_OP_BOUNDARY_CHECK(frame, label)                              \
+    do {                                                                       \
+        if (frame->root->op <= GF_FOP_NULL ||                                  \
+            frame->root->op >= GF_FOP_MAXVALUE)                                \
+            goto label;                                                        \
+    } while (0)
 
 /**
  * ignore internal fops for all clients except AFR self-heal daemon
  */
-#define CHANGELOG_IF_INTERNAL_FOP_THEN_GOTO(frame, dict, label) do {    \
-                if ((frame->root->pid != GF_CLIENT_PID_AFR_SELF_HEALD)  \
-                    && dict                                             \
-                    && dict_get (dict, GLUSTERFS_INTERNAL_FOP_KEY))     \
-                        goto label;                                     \
-        } while (0)
-
-#define CHANGELOG_COND_GOTO(priv, cond, label) do {                    \
-                if (!priv->active || cond)                             \
-                        goto label;                                    \
-        } while (0)
+#define CHANGELOG_IF_INTERNAL_FOP_THEN_GOTO(frame, dict, label)                \
+    do {                                                                       \
+        if ((frame->root->pid != GF_CLIENT_PID_SELF_HEALD) && dict &&          \
+            dict_get(dict, GLUSTERFS_INTERNAL_FOP_KEY))                        \
+            goto label;                                                        \
+    } while (0)
+
+#define CHANGELOG_COND_GOTO(priv, cond, label)                                 \
+    do {                                                                       \
+        if (!priv->active || cond)                                             \
+            goto label;                                                        \
+    } while (0)
 
 /* Begin: Geo-Rep snapshot dependency changes */
 
-#define DICT_ERROR         -1
-#define BARRIER_OFF         0
-#define BARRIER_ON          1
-#define DICT_DEFAULT        2
-
-#define CHANGELOG_NOT_ON_THEN_GOTO(priv, ret, label) do {                      \
-                if (!priv->active) {                                           \
-                        gf_log (this->name, GF_LOG_WARNING,                    \
-                                "Changelog is not active, return success");    \
-                        ret = 0;                                               \
-                        goto label;                                            \
-                }                                                              \
-        } while (0)
+#define DICT_ERROR -1
+#define BARRIER_OFF 0
+#define BARRIER_ON 1
+#define DICT_DEFAULT 2
+
+#define CHANGELOG_NOT_ON_THEN_GOTO(priv, ret, label)                           \
+    do {                                                                       \
+        if (!priv->active) {                                                   \
+            gf_smsg(this->name, GF_LOG_WARNING, 0,                             \
+                    CHANGELOG_MSG_CHANGELOG_NOT_ACTIVE, NULL);                 \
+            ret = 0;                                                           \
+            goto label;                                                        \
+        }                                                                      \
+    } while (0)
 
 /* Log pthread error and goto label */
-#define CHANGELOG_PTHREAD_ERROR_HANDLE_0(ret, label) do {                      \
-                if (ret) {                                                     \
-                        gf_log (this->name, GF_LOG_ERROR,                      \
-                                "pthread error: Error: %d", ret);              \
-                        ret = -1;                                              \
-                        goto label;                                            \
-                }                                                              \
-        } while (0)
+#define CHANGELOG_PTHREAD_ERROR_HANDLE_0(ret, label)                           \
+    do {                                                                       \
+        if (ret) {                                                             \
+            gf_smsg(this->name, GF_LOG_ERROR, 0, CHANGELOG_MSG_PTHREAD_ERROR,  \
+                    "error=%d", ret, NULL);                                    \
+            ret = -1;                                                          \
+            goto label;                                                        \
+        }                                                                      \
+    } while (0);
 
 /* Log pthread error, set flag and goto label */
-#define CHANGELOG_PTHREAD_ERROR_HANDLE_1(ret, label, flag) do {                \
-                if (ret) {                                                     \
-                        gf_log (this->name, GF_LOG_ERROR,                      \
-                                "pthread error: Error: %d", ret);              \
-                        ret = -1;                                              \
-                        flag = _gf_true;                                       \
-                        goto label;                                            \
-                }                                                              \
-        } while (0)
+#define CHANGELOG_PTHREAD_ERROR_HANDLE_1(ret, label, flag)                     \
+    do {                                                                       \
+        if (ret) {                                                             \
+            gf_smsg(this->name, GF_LOG_ERROR, 0, CHANGELOG_MSG_PTHREAD_ERROR,  \
+                    "error=%d", ret, NULL);                                    \
+            ret = -1;                                                          \
+            flag = _gf_true;                                                   \
+            goto label;                                                        \
+        }                                                                      \
+    } while (0)
+
+/* Log pthread error, unlock mutex and goto label */
+#define CHANGELOG_PTHREAD_ERROR_HANDLE_2(ret, label, mutex)                    \
+    do {                                                                       \
+        if (ret) {                                                             \
+            gf_smsg(this->name, GF_LOG_ERROR, 0, CHANGELOG_MSG_PTHREAD_ERROR,  \
+                    "error=%d", ret, NULL);                                    \
+            ret = -1;                                                          \
+            pthread_mutex_unlock(&mutex);                                      \
+            goto label;                                                        \
+        }                                                                      \
+    } while (0)
+
 /* End: Geo-Rep snapshot dependency changes */
 
 #endif /* _CHANGELOG_HELPERS_H */
diff --git a/xlators/features/changelog/src/changelog-mem-types.h b/xlators/features/changelog/src/changelog-mem-types.h
index 1618f722f6c..a2d8a9cbe93 100644
--- a/xlators/features/changelog/src/changelog-mem-types.h
+++ b/xlators/features/changelog/src/changelog-mem-types.h
@@ -11,25 +11,24 @@
 #ifndef _CHANGELOG_MEM_TYPES_H
 #define _CHANGELOG_MEM_TYPES_H
 
-#include "mem-types.h"
+#include <glusterfs/mem-types.h>
 
 enum gf_changelog_mem_types {
-        gf_changelog_mt_priv_t                     = gf_common_mt_end + 1,
-        gf_changelog_mt_str_t                      = gf_common_mt_end + 2,
-        gf_changelog_mt_batch_t                    = gf_common_mt_end + 3,
-        gf_changelog_mt_rt_t                       = gf_common_mt_end + 4,
-        gf_changelog_mt_inode_ctx_t                = gf_common_mt_end + 5,
-        gf_changelog_mt_rpc_clnt_t                 = gf_common_mt_end + 6,
-        gf_changelog_mt_libgfchangelog_t           = gf_common_mt_end + 7,
-        gf_changelog_mt_libgfchangelog_entry_t     = gf_common_mt_end + 8,
-        gf_changelog_mt_libgfchangelog_rl_t        = gf_common_mt_end + 9,
-        gf_changelog_mt_libgfchangelog_dirent_t    = gf_common_mt_end + 10,
-        gf_changelog_mt_changelog_buffer_t         = gf_common_mt_end + 11,
-        gf_changelog_mt_history_data_t             = gf_common_mt_end + 12,
-        gf_changelog_mt_libgfchangelog_call_pool_t = gf_common_mt_end + 13,
-        gf_changelog_mt_libgfchangelog_event_t     = gf_common_mt_end + 14,
-        gf_changelog_mt_ev_dispatcher_t            = gf_common_mt_end + 15,
-        gf_changelog_mt_end
+    gf_changelog_mt_priv_t = gf_common_mt_end + 1,
+    gf_changelog_mt_str_t = gf_common_mt_end + 2,
+    gf_changelog_mt_batch_t = gf_common_mt_end + 3,
+    gf_changelog_mt_rt_t = gf_common_mt_end + 4,
+    gf_changelog_mt_inode_ctx_t = gf_common_mt_end + 5,
+    gf_changelog_mt_rpc_clnt_t = gf_common_mt_end + 6,
+    gf_changelog_mt_libgfchangelog_t = gf_common_mt_end + 7,
+    gf_changelog_mt_libgfchangelog_entry_t = gf_common_mt_end + 8,
+    gf_changelog_mt_libgfchangelog_rl_t = gf_common_mt_end + 9,
+    gf_changelog_mt_changelog_buffer_t = gf_common_mt_end + 10,
+    gf_changelog_mt_history_data_t = gf_common_mt_end + 11,
+    gf_changelog_mt_libgfchangelog_call_pool_t = gf_common_mt_end + 12,
+    gf_changelog_mt_libgfchangelog_event_t = gf_common_mt_end + 13,
+    gf_changelog_mt_ev_dispatcher_t = gf_common_mt_end + 14,
+    gf_changelog_mt_end
 };
 
 #endif
diff --git a/xlators/features/changelog/src/changelog-messages.h b/xlators/features/changelog/src/changelog-messages.h
new file mode 100644
index 00000000000..cb0e16c85d8
--- /dev/null
+++ b/xlators/features/changelog/src/changelog-messages.h
@@ -0,0 +1,172 @@
+/*
+ Copyright (c) 2015 Red Hat, Inc. <http://www.redhat.com>
+ This file is part of GlusterFS.
+
+ This file is licensed to you under your choice of the GNU Lesser
+ General Public License, version 3 or any later version (LGPLv3 or
+ later), or the GNU General Public License, version 2 (GPLv2), in all
+ cases as published by the Free Software Foundation.
+ */
+
+#ifndef _CHANGELOG_MESSAGES_H_
+#define _CHANGELOG_MESSAGES_H_
+
+#include <glusterfs/glfs-message-id.h>
+
+/* To add new message IDs, append new identifiers at the end of the list.
+ *
+ * Never remove a message ID. If it's not used anymore, you can rename it or
+ * leave it as it is, but not delete it. This is to prevent reutilization of
+ * IDs by other messages.
+ *
+ * The component name must match one of the entries defined in
+ * glfs-message-id.h.
+ */
+
+GLFS_MSGID(
+    CHANGELOG, CHANGELOG_MSG_OPEN_FAILED, CHANGELOG_MSG_BARRIER_FOP_FAILED,
+    CHANGELOG_MSG_VOL_MISCONFIGURED, CHANGELOG_MSG_RENAME_ERROR,
+    CHANGELOG_MSG_READ_ERROR, CHANGELOG_MSG_HTIME_ERROR,
+    CHANGELOG_MSG_PTHREAD_MUTEX_INIT_FAILED,
+    CHANGELOG_MSG_PTHREAD_COND_INIT_FAILED, CHANGELOG_MSG_CHILD_MISCONFIGURED,
+    CHANGELOG_MSG_DIR_OPTIONS_NOT_SET, CHANGELOG_MSG_CLOSE_ERROR,
+    CHANGELOG_MSG_PIPE_CREATION_ERROR, CHANGELOG_MSG_DICT_GET_FAILED,
+    CHANGELOG_MSG_BARRIER_INFO, CHANGELOG_MSG_BARRIER_ERROR,
+    CHANGELOG_MSG_GET_TIME_OP_FAILED, CHANGELOG_MSG_WRITE_FAILED,
+    CHANGELOG_MSG_PTHREAD_ERROR, CHANGELOG_MSG_INODE_NOT_FOUND,
+    CHANGELOG_MSG_FSYNC_OP_FAILED, CHANGELOG_MSG_TOTAL_LOG_INFO,
+    CHANGELOG_MSG_SNAP_INFO, CHANGELOG_MSG_SELECT_FAILED,
+    CHANGELOG_MSG_FCNTL_FAILED, CHANGELOG_MSG_BNOTIFY_INFO,
+    CHANGELOG_MSG_ENTRY_BUF_INFO, CHANGELOG_MSG_CHANGELOG_NOT_ACTIVE,
+    CHANGELOG_MSG_LOCAL_INIT_FAILED, CHANGELOG_MSG_NOTIFY_REGISTER_FAILED,
+    CHANGELOG_MSG_PROGRAM_NAME_REG_FAILED, CHANGELOG_MSG_HANDLE_PROBE_ERROR,
+    CHANGELOG_MSG_SET_FD_CONTEXT, CHANGELOG_MSG_FREEUP_FAILED,
+    CHANGELOG_MSG_RECONFIGURE, CHANGELOG_MSG_RPC_SUBMIT_REPLY_FAILED,
+    CHANGELOG_MSG_RPC_BUILD_ERROR, CHANGELOG_MSG_RPC_CONNECT_ERROR,
+    CHANGELOG_MSG_RPC_START_ERROR, CHANGELOG_MSG_BUFFER_STARVATION_ERROR,
+    CHANGELOG_MSG_SCAN_DIR_FAILED, CHANGELOG_MSG_FSETXATTR_FAILED,
+    CHANGELOG_MSG_FGETXATTR_FAILED, CHANGELOG_MSG_CLEANUP_ON_ACTIVE_REF,
+    CHANGELOG_MSG_DISPATCH_EVENT_FAILED, CHANGELOG_MSG_PUT_BUFFER_FAILED,
+    CHANGELOG_MSG_PTHREAD_COND_WAIT_FAILED, CHANGELOG_MSG_PTHREAD_CANCEL_FAILED,
+    CHANGELOG_MSG_INJECT_FSYNC_FAILED, CHANGELOG_MSG_CREATE_FRAME_FAILED,
+    CHANGELOG_MSG_FSTAT_OP_FAILED, CHANGELOG_MSG_LSEEK_OP_FAILED,
+    CHANGELOG_MSG_STRSTR_OP_FAILED, CHANGELOG_MSG_UNLINK_OP_FAILED,
+    CHANGELOG_MSG_DETECT_EMPTY_CHANGELOG_FAILED,
+    CHANGELOG_MSG_READLINK_OP_FAILED, CHANGELOG_MSG_EXPLICIT_ROLLOVER_FAILED,
+    CHANGELOG_MSG_RPCSVC_NOTIFY_FAILED, CHANGELOG_MSG_MEMORY_INIT_FAILED,
+    CHANGELOG_MSG_NO_MEMORY, CHANGELOG_MSG_HTIME_STAT_ERROR,
+    CHANGELOG_MSG_HTIME_CURRENT_ERROR, CHANGELOG_MSG_BNOTIFY_COND_INFO,
+    CHANGELOG_MSG_NO_HTIME_CURRENT, CHANGELOG_MSG_HTIME_CURRENT,
+    CHANGELOG_MSG_NEW_HTIME_FILE, CHANGELOG_MSG_MKDIR_ERROR,
+    CHANGELOG_MSG_PATH_NOT_FOUND, CHANGELOG_MSG_XATTR_INIT_FAILED,
+    CHANGELOG_MSG_WROTE_TO_CSNAP, CHANGELOG_MSG_UNUSED_0,
+    CHANGELOG_MSG_GET_BUFFER_FAILED, CHANGELOG_MSG_BARRIER_STATE_NOTIFY,
+    CHANGELOG_MSG_BARRIER_DISABLED, CHANGELOG_MSG_BARRIER_ALREADY_DISABLED,
+    CHANGELOG_MSG_BARRIER_ON_ERROR, CHANGELOG_MSG_BARRIER_ENABLE,
+    CHANGELOG_MSG_BARRIER_KEY_NOT_FOUND, CHANGELOG_MSG_ERROR_IN_DICT_GET,
+    CHANGELOG_MSG_UNUSED_1, CHANGELOG_MSG_UNUSED_2,
+    CHANGELOG_MSG_DEQUEUING_BARRIER_FOPS,
+    CHANGELOG_MSG_DEQUEUING_BARRIER_FOPS_FINISHED,
+    CHANGELOG_MSG_BARRIER_TIMEOUT, CHANGELOG_MSG_TIMEOUT_ADD_FAILED,
+    CHANGELOG_MSG_CLEANUP_ALREADY_SET);
+
+#define CHANGELOG_MSG_BARRIER_FOP_FAILED_STR                                   \
+    "failed to barrier FOPs, disabling changelog barrier"
+#define CHANGELOG_MSG_MEMORY_INIT_FAILED_STR "memory accounting init failed"
+#define CHANGELOG_MSG_NO_MEMORY_STR "failed to create local memory pool"
+#define CHANGELOG_MSG_ENTRY_BUF_INFO_STR                                       \
+    "Entry cannot be captured for gfid, Capturing DATA entry."
+#define CHANGELOG_MSG_PTHREAD_ERROR_STR "pthread error"
+#define CHANGELOG_MSG_PTHREAD_MUTEX_INIT_FAILED_STR "pthread_mutex_init failed"
+#define CHANGELOG_MSG_PTHREAD_COND_INIT_FAILED_STR "pthread_cond_init failed"
+#define CHANGELOG_MSG_HTIME_ERROR_STR "failed to update HTIME file"
+#define CHANGELOG_MSG_HTIME_STAT_ERROR_STR "unable to stat htime file"
+#define CHANGELOG_MSG_HTIME_CURRENT_ERROR_STR "Error extracting HTIME_CURRENT."
+#define CHANGELOG_MSG_UNLINK_OP_FAILED_STR "error unlinking empty changelog"
+#define CHANGELOG_MSG_RENAME_ERROR_STR "error renaming"
+#define CHANGELOG_MSG_MKDIR_ERROR_STR "unable to create directory"
+#define CHANGELOG_MSG_BNOTIFY_INFO_STR                                         \
+    "Explicit rollover changelog signaling bnotify"
+#define CHANGELOG_MSG_BNOTIFY_COND_INFO_STR "Woke up: bnotify conditional wait"
+#define CHANGELOG_MSG_RECONFIGURE_STR "Reconfigure: Changelog Enable"
+#define CHANGELOG_MSG_NO_HTIME_CURRENT_STR                                     \
+    "HTIME_CURRENT not found. Changelog enabled before init"
+#define CHANGELOG_MSG_HTIME_CURRENT_STR "HTIME_CURRENT"
+#define CHANGELOG_MSG_NEW_HTIME_FILE_STR                                       \
+    "Changelog enable: Creating new HTIME file"
+#define CHANGELOG_MSG_FGETXATTR_FAILED_STR "fgetxattr failed"
+#define CHANGELOG_MSG_TOTAL_LOG_INFO_STR "changelog info"
+#define CHANGELOG_MSG_PTHREAD_COND_WAIT_FAILED_STR "pthread cond wait failed"
+#define CHANGELOG_MSG_INODE_NOT_FOUND_STR "inode not found"
+#define CHANGELOG_MSG_READLINK_OP_FAILED_STR                                   \
+    "could not read the link from the gfid handle"
+#define CHANGELOG_MSG_OPEN_FAILED_STR "unable to open file"
+#define CHANGELOG_MSG_RPC_CONNECT_ERROR_STR "failed to connect back"
+#define CHANGELOG_MSG_BUFFER_STARVATION_ERROR_STR                              \
+    "Failed to get buffer for RPC dispatch"
+#define CHANGELOG_MSG_PTHREAD_CANCEL_FAILED_STR "could not cancel thread"
+#define CHANGELOG_MSG_FSTAT_OP_FAILED_STR "Could not stat (CHANGELOG)"
+#define CHANGELOG_MSG_LSEEK_OP_FAILED_STR "Could not lseek (changelog)"
+#define CHANGELOG_MSG_PATH_NOT_FOUND_STR                                       \
+    "Could not find CHANGELOG in changelog path"
+#define CHANGELOG_MSG_FSYNC_OP_FAILED_STR "fsync failed"
+#define CHANGELOG_MSG_DETECT_EMPTY_CHANGELOG_FAILED_STR                        \
+    "Error detecting empty changelog"
+#define CHANGELOG_MSG_EXPLICIT_ROLLOVER_FAILED_STR                             \
+    "Fail snapshot because of previous errors"
+#define CHANGELOG_MSG_SCAN_DIR_FAILED_STR "scandir failed"
+#define CHANGELOG_MSG_FSETXATTR_FAILED_STR "fsetxattr failed"
+#define CHANGELOG_MSG_XATTR_INIT_FAILED_STR "Htime xattr initialization failed"
+#define CHANGELOG_MSG_SNAP_INFO_STR "log in call path"
+#define CHANGELOG_MSG_WRITE_FAILED_STR "error writing to disk"
+#define CHANGELOG_MSG_WROTE_TO_CSNAP_STR "Successfully wrote to csnap"
+#define CHANGELOG_MSG_GET_TIME_OP_FAILED_STR "Problem rolling over changelog(s)"
+#define CHANGELOG_MSG_BARRIER_INFO_STR "Explicit wakeup on barrier notify"
+#define CHANGELOG_MSG_SELECT_FAILED_STR "pthread_cond_timedwait failed"
+#define CHANGELOG_MSG_INJECT_FSYNC_FAILED_STR "failed to inject fsync event"
+#define CHANGELOG_MSG_LOCAL_INIT_FAILED_STR                                    \
+    "changelog local initialization failed"
+#define CHANGELOG_MSG_GET_BUFFER_FAILED_STR "Failed to get buffer"
+#define CHANGELOG_MSG_SET_FD_CONTEXT_STR                                       \
+    "could not set fd context(for release cbk)"
+#define CHANGELOG_MSG_DICT_GET_FAILED_STR "Barrier failed"
+#define CHANGELOG_MSG_BARRIER_STATE_NOTIFY_STR "Barrier notification"
+#define CHANGELOG_MSG_BARRIER_ERROR_STR                                        \
+    "Received another barrier off notification while already off"
+#define CHANGELOG_MSG_BARRIER_DISABLED_STR "disabled changelog barrier"
+#define CHANGELOG_MSG_BARRIER_ALREADY_DISABLED_STR                             \
+    "Changelog barrier already disabled"
+#define CHANGELOG_MSG_BARRIER_ON_ERROR_STR                                     \
+    "Received another barrier on notification when last one is not served yet"
+#define CHANGELOG_MSG_BARRIER_ENABLE_STR "Enabled changelog barrier"
+#define CHANGELOG_MSG_BARRIER_KEY_NOT_FOUND_STR "barrier key not found"
+#define CHANGELOG_MSG_ERROR_IN_DICT_GET_STR                                    \
+    "Something went wrong in dict_get_str_boolean"
+#define CHANGELOG_MSG_DIR_OPTIONS_NOT_SET_STR "changelog-dir option is not set"
+#define CHANGELOG_MSG_FREEUP_FAILED_STR "could not cleanup bootstrapper"
+#define CHANGELOG_MSG_CHILD_MISCONFIGURED_STR                                  \
+    "translator needs a single subvolume"
+#define CHANGELOG_MSG_VOL_MISCONFIGURED_STR                                    \
+    "dangling volume. please check volfile"
+#define CHANGELOG_MSG_DEQUEUING_BARRIER_FOPS_STR                               \
+    "Dequeuing all the changelog barriered fops"
+#define CHANGELOG_MSG_DEQUEUING_BARRIER_FOPS_FINISHED_STR                      \
+    "Dequeuing changelog barriered fops is finished"
+#define CHANGELOG_MSG_BARRIER_TIMEOUT_STR                                      \
+    "Disabling changelog barrier because of the timeout"
+#define CHANGELOG_MSG_TIMEOUT_ADD_FAILED_STR                                   \
+    "Couldn't add changelog barrier timeout event"
+#define CHANGELOG_MSG_RPC_BUILD_ERROR_STR "failed to build rpc options"
+#define CHANGELOG_MSG_NOTIFY_REGISTER_FAILED_STR "failed to register notify"
+#define CHANGELOG_MSG_RPC_START_ERROR_STR "failed to start rpc"
+#define CHANGELOG_MSG_CREATE_FRAME_FAILED_STR "failed to create frame"
+#define CHANGELOG_MSG_RPC_SUBMIT_REPLY_FAILED_STR "failed to serialize reply"
+#define CHANGELOG_MSG_PROGRAM_NAME_REG_FAILED_STR "cannot register program"
+#define CHANGELOG_MSG_CHANGELOG_NOT_ACTIVE_STR                                 \
+    "Changelog is not active, return success"
+#define CHANGELOG_MSG_PUT_BUFFER_FAILED_STR                                    \
+    "failed to put buffer after consumption"
+#define CHANGELOG_MSG_CLEANUP_ALREADY_SET_STR                                  \
+    "cleanup_starting flag is already set for xl"
+#define CHANGELOG_MSG_HANDLE_PROBE_ERROR_STR "xdr decoding error"
+#endif /* !_CHANGELOG_MESSAGES_H_ */
diff --git a/xlators/features/changelog/src/changelog-misc.h b/xlators/features/changelog/src/changelog-misc.h
index b45302ad099..e2addc09414 100644
--- a/xlators/features/changelog/src/changelog-misc.h
+++ b/xlators/features/changelog/src/changelog-misc.h
@@ -11,118 +11,121 @@
 #ifndef _CHANGELOG_MISC_H
 #define _CHANGELOG_MISC_H
 
-#include "glusterfs.h"
-#include "common-utils.h"
+#include <glusterfs/glusterfs.h>
+#include <glusterfs/common-utils.h>
 
-#define CHANGELOG_MAX_TYPE  3
+#define CHANGELOG_MAX_TYPE 4
 #define CHANGELOG_FILE_NAME "CHANGELOG"
 #define HTIME_FILE_NAME "HTIME"
 #define CSNAP_FILE_NAME "CHANGELOG.SNAP"
 #define HTIME_KEY "trusted.glusterfs.htime"
+#define HTIME_CURRENT "trusted.glusterfs.current_htime"
 #define HTIME_INITIAL_VALUE "0:0"
 
-#define CHANGELOG_VERSION_MAJOR  1
-#define CHANGELOG_VERSION_MINOR  1
+#define CHANGELOG_VERSION_MAJOR 1
+#define CHANGELOG_VERSION_MINOR 2
 
-#define CHANGELOG_UNIX_SOCK  DEFAULT_VAR_RUN_DIRECTORY"/changelog-%s.sock"
-#define CHANGELOG_TMP_UNIX_SOCK  DEFAULT_VAR_RUN_DIRECTORY"/.%s%lu.sock"
+#define CHANGELOG_UNIX_SOCK DEFAULT_VAR_RUN_DIRECTORY "/changelog-%s.sock"
+#define CHANGELOG_TMP_UNIX_SOCK DEFAULT_VAR_RUN_DIRECTORY "/.%s%lu.sock"
 
 /**
  * header starts with the version and the format of the changelog.
  * 'version' not much of a use now.
  */
-#define CHANGELOG_HEADER                                                \
-        "GlusterFS Changelog | version: v%d.%d | encoding : %d\n"
-
-#define CHANGELOG_MAKE_SOCKET_PATH(brick_path, sockpath, len) do {      \
-                char md5_sum[MD5_DIGEST_LENGTH*2+1] = {0,};             \
-                md5_wrapper((unsigned char *) brick_path,               \
-                            strlen(brick_path),                         \
-                            md5_sum);                                   \
-                (void) snprintf (sockpath, len,                         \
-                                 CHANGELOG_UNIX_SOCK, md5_sum);         \
-        } while (0)
-
-#define CHANGELOG_MAKE_TMP_SOCKET_PATH(brick_path, sockpath, len) do {  \
-                unsigned long pid = 0;                                  \
-                char md5_sum[MD5_DIGEST_LENGTH*2+1] = {0,};             \
-                pid = (unsigned long) getpid ();                        \
-                md5_wrapper((unsigned char *) brick_path,               \
-                            strlen(brick_path),                         \
-                            md5_sum);                                   \
-                (void) snprintf (sockpath,                              \
-                                 len, CHANGELOG_TMP_UNIX_SOCK,          \
-                                 md5_sum, pid);                         \
-        } while (0)
-
+#define CHANGELOG_HEADER                                                       \
+    "GlusterFS Changelog | version: v%d.%d | encoding : %d\n"
+
+#define CHANGELOG_MAKE_SOCKET_PATH(brick_path, sockpath, len)                  \
+    do {                                                                       \
+        char xxh64[GF_XXH64_DIGEST_LENGTH * 2 + 1] = {                         \
+            0,                                                                 \
+        };                                                                     \
+        gf_xxh64_wrapper((unsigned char *)brick_path, strlen(brick_path),      \
+                         GF_XXHSUM64_DEFAULT_SEED, xxh64);                     \
+        (void)snprintf(sockpath, len, CHANGELOG_UNIX_SOCK, xxh64);             \
+    } while (0)
+
+#define CHANGELOG_MAKE_TMP_SOCKET_PATH(brick_path, sockpath, len)              \
+    do {                                                                       \
+        unsigned long pid = 0;                                                 \
+        char xxh64[GF_XXH64_DIGEST_LENGTH * 2 + 1] = {                         \
+            0,                                                                 \
+        };                                                                     \
+        pid = (unsigned long)getpid();                                         \
+        gf_xxh64_wrapper((unsigned char *)brick_path, strlen(brick_path),      \
+                         GF_XXHSUM64_DEFAULT_SEED, xxh64);                     \
+        (void)snprintf(sockpath, len, CHANGELOG_TMP_UNIX_SOCK, xxh64, pid);    \
+    } while (0)
 
 /**
  * ... used by libgfchangelog.
  */
-#define CHANGELOG_GET_ENCODING(fd, buffer, len, enc, enc_len) do {      \
-                FILE *fp;                                               \
-                int fd_dup, maj, min;                                   \
-                                                                        \
-                enc = -1;                                               \
-                fd_dup = dup (fd);                                      \
-                                                                        \
-                if (fd_dup != -1) {                                     \
-                        fp = fdopen (fd_dup, "r");                      \
-                        if (fp) {                                       \
-                                if (fgets (buffer, len, fp)) {          \
-                                        elen = strlen (buffer);         \
-                                        sscanf (buffer,                 \
-                                                CHANGELOG_HEADER,       \
-                                                &maj, &min, &enc);      \
-                                }                                       \
-                                fclose (fp);                            \
-                        } else {                                        \
-                                close (fd_dup);                         \
-                        }                                               \
-                }                                                       \
-        } while (0)
-
-#define CHANGELOG_FILL_HTIME_DIR(changelog_dir, path) do {      \
-                strcpy (path, changelog_dir);                   \
-                strcat (path, "/htime");                        \
-        } while(0)
-
-#define CHANGELOG_FILL_CSNAP_DIR(changelog_dir, path) do {      \
-                strcpy (path, changelog_dir);                   \
-                strcat (path, "/csnap");                        \
-        } while(0)
+#define CHANGELOG_GET_HEADER_INFO(fd, buffer, len, enc, maj, min, elen)        \
+    do {                                                                       \
+        FILE *fp;                                                              \
+        int fd_dup;                                                            \
+                                                                               \
+        enc = -1;                                                              \
+        maj = -1;                                                              \
+        min = -1;                                                              \
+        fd_dup = dup(fd);                                                      \
+                                                                               \
+        if (fd_dup != -1) {                                                    \
+            fp = fdopen(fd_dup, "r");                                          \
+            if (fp) {                                                          \
+                if (fgets(buffer, len, fp)) {                                  \
+                    elen = strlen(buffer);                                     \
+                    sscanf(buffer, CHANGELOG_HEADER, &maj, &min, &enc);        \
+                }                                                              \
+                fclose(fp);                                                    \
+            } else {                                                           \
+                sys_close(fd_dup);                                             \
+            }                                                                  \
+        }                                                                      \
+    } while (0)
+
+#define CHANGELOG_FILL_HTIME_DIR(changelog_dir, path)                          \
+    do {                                                                       \
+        snprintf(path, sizeof(path), "%s/htime", changelog_dir);               \
+    } while (0)
+
+#define CHANGELOG_FILL_CSNAP_DIR(changelog_dir, path)                          \
+    do {                                                                       \
+        snprintf(path, sizeof(path), "%s/csnap", changelog_dir);               \
+    } while (0)
 /**
- * everything after 'CHANGELOG_TYPE_ENTRY' are internal types
+ * everything after 'CHANGELOG_TYPE_METADATA_XATTR' are internal types
  * (ie. none of the fops trigger this type of event), hence
- * CHANGELOG_MAX_TYPE = 3
+ * CHANGELOG_MAX_TYPE = 4
  */
 typedef enum {
-        CHANGELOG_TYPE_DATA = 0,
-        CHANGELOG_TYPE_METADATA,
-        CHANGELOG_TYPE_ENTRY,
-        CHANGELOG_TYPE_ROLLOVER,
-        CHANGELOG_TYPE_FSYNC,
+    CHANGELOG_TYPE_DATA = 0,
+    CHANGELOG_TYPE_METADATA,
+    CHANGELOG_TYPE_ENTRY,
+    CHANGELOG_TYPE_METADATA_XATTR,
+    CHANGELOG_TYPE_ROLLOVER,
+    CHANGELOG_TYPE_FSYNC,
 } changelog_log_type;
 
 /* operation modes - RT for now */
 typedef enum {
-        CHANGELOG_MODE_RT = 0,
+    CHANGELOG_MODE_RT = 0,
 } changelog_mode_t;
 
 /* encoder types */
 
 typedef enum {
-        CHANGELOG_ENCODE_MIN = 0,
-        CHANGELOG_ENCODE_BINARY,
-        CHANGELOG_ENCODE_ASCII,
-        CHANGELOG_ENCODE_MAX,
+    CHANGELOG_ENCODE_MIN = 0,
+    CHANGELOG_ENCODE_BINARY,
+    CHANGELOG_ENCODE_ASCII,
+    CHANGELOG_ENCODE_MAX,
 } changelog_encoder_t;
 
-#define CHANGELOG_VALID_ENCODING(enc)                                   \
-        (enc > CHANGELOG_ENCODE_MIN && enc < CHANGELOG_ENCODE_MAX)
+#define CHANGELOG_VALID_ENCODING(enc)                                          \
+    (enc > CHANGELOG_ENCODE_MIN && enc < CHANGELOG_ENCODE_MAX)
 
-#define CHANGELOG_TYPE_IS_ENTRY(type)  (type == CHANGELOG_TYPE_ENTRY)
-#define CHANGELOG_TYPE_IS_ROLLOVER(type)  (type == CHANGELOG_TYPE_ROLLOVER)
-#define CHANGELOG_TYPE_IS_FSYNC(type)  (type == CHANGELOG_TYPE_FSYNC)
+#define CHANGELOG_TYPE_IS_ENTRY(type) (type == CHANGELOG_TYPE_ENTRY)
+#define CHANGELOG_TYPE_IS_ROLLOVER(type) (type == CHANGELOG_TYPE_ROLLOVER)
+#define CHANGELOG_TYPE_IS_FSYNC(type) (type == CHANGELOG_TYPE_FSYNC)
 
 #endif /* _CHANGELOG_MISC_H */
diff --git a/xlators/features/changelog/src/changelog-rpc-common.c b/xlators/features/changelog/src/changelog-rpc-common.c
index 76db6696ae8..125246a17e1 100644
--- a/xlators/features/changelog/src/changelog-rpc-common.c
+++ b/xlators/features/changelog/src/changelog-rpc-common.c
@@ -9,7 +9,9 @@
 */
 
 #include "changelog-rpc-common.h"
+#include "changelog-messages.h"
 
+#include <glusterfs/syscall.h>
 /**
 *****************************************************
                   Client Interface
@@ -22,60 +24,63 @@
  */
 
 void *
-changelog_rpc_poller (void *arg)
+changelog_rpc_poller(void *arg)
 {
-        xlator_t *this = arg;
+    xlator_t *this = arg;
 
-        (void) event_dispatch (this->ctx->event_pool);
-        return NULL;
+    (void)gf_event_dispatch(this->ctx->event_pool);
+    return NULL;
 }
 
 struct rpc_clnt *
-changelog_rpc_client_init (xlator_t *this, void *cbkdata,
-                           char *sockfile, rpc_clnt_notify_t fn)
+changelog_rpc_client_init(xlator_t *this, void *cbkdata, char *sockfile,
+                          rpc_clnt_notify_t fn)
 {
-        int              ret         = 0;
-        struct rpc_clnt *rpc         = NULL;
-        dict_t          *options     = NULL;
-
-        if (!cbkdata)
-                cbkdata = this;
-
-        options = dict_new ();
-        if (!options)
-                goto error_return;
-
-        ret = rpc_transport_unix_options_build (&options, sockfile, 0);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "failed to build rpc options");
-                goto dealloc_dict;
-        }
-
-        rpc = rpc_clnt_new (options, this->ctx, this->name, 16);
-        if (!rpc)
-                goto dealloc_dict;
-
-        ret = rpc_clnt_register_notify (rpc, fn, cbkdata);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "failed to register notify");
-                goto dealloc_rpc_clnt;
-        }
-
-        ret = rpc_clnt_start (rpc);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "failed to start rpc");
-                goto dealloc_rpc_clnt;
-        }
-
-        return rpc;
-
- dealloc_rpc_clnt:
-        rpc_clnt_unref (rpc);
- dealloc_dict:
-        dict_unref (options);
- error_return:
-        return NULL;
+    int ret = 0;
+    struct rpc_clnt *rpc = NULL;
+    dict_t *options = NULL;
+
+    if (!cbkdata)
+        cbkdata = this;
+
+    options = dict_new();
+    if (!options)
+        goto error_return;
+
+    ret = rpc_transport_unix_options_build(options, sockfile, 0);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_ERROR, 0, CHANGELOG_MSG_RPC_BUILD_ERROR,
+                NULL);
+        goto dealloc_dict;
+    }
+
+    rpc = rpc_clnt_new(options, this, this->name, 16);
+    if (!rpc)
+        goto dealloc_dict;
+
+    ret = rpc_clnt_register_notify(rpc, fn, cbkdata);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_ERROR, 0,
+                CHANGELOG_MSG_NOTIFY_REGISTER_FAILED, NULL);
+        goto dealloc_rpc_clnt;
+    }
+
+    ret = rpc_clnt_start(rpc);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_ERROR, 0, CHANGELOG_MSG_RPC_START_ERROR,
+                NULL);
+        goto dealloc_rpc_clnt;
+    }
+
+    dict_unref(options);
+    return rpc;
+
+dealloc_rpc_clnt:
+    rpc_clnt_unref(rpc);
+dealloc_dict:
+    dict_unref(options);
+error_return:
+    return NULL;
 }
 
 /**
@@ -83,94 +88,96 @@ changelog_rpc_client_init (xlator_t *this, void *cbkdata,
  * RPC server.
  */
 int
-changelog_rpc_sumbit_req (struct rpc_clnt *rpc, void *req,
-                          call_frame_t *frame, rpc_clnt_prog_t *prog,
-                          int procnum, struct iovec *payload, int payloadcnt,
-                          struct iobref *iobref, xlator_t *this,
-                          fop_cbk_fn_t cbkfn, xdrproc_t xdrproc)
+changelog_rpc_sumbit_req(struct rpc_clnt *rpc, void *req, call_frame_t *frame,
+                         rpc_clnt_prog_t *prog, int procnum,
+                         struct iovec *payload, int payloadcnt,
+                         struct iobref *iobref, xlator_t *this,
+                         fop_cbk_fn_t cbkfn, xdrproc_t xdrproc)
 {
-        int           ret        = 0;
-        int           count      = 0;
-        struct iovec  iov        = {0, };
-        struct iobuf *iobuf      = NULL;
-        char          new_iobref = 0;
-        ssize_t       xdr_size   = 0;
-
-        GF_ASSERT (this);
+    int ret = 0;
+    int count = 0;
+    struct iovec iov = {
+        0,
+    };
+    struct iobuf *iobuf = NULL;
+    char new_iobref = 0;
+    ssize_t xdr_size = 0;
 
-        if (req) {
-                xdr_size = xdr_sizeof (xdrproc, req);
+    GF_ASSERT(this);
 
-                iobuf = iobuf_get2 (this->ctx->iobuf_pool, xdr_size);
-                if (!iobuf) {
-                        goto out;
-                };
+    if (req) {
+        xdr_size = xdr_sizeof(xdrproc, req);
 
-                if (!iobref) {
-                        iobref = iobref_new ();
-                        if (!iobref) {
-                                goto out;
-                        }
+        iobuf = iobuf_get2(this->ctx->iobuf_pool, xdr_size);
+        if (!iobuf) {
+            goto out;
+        };
 
-                        new_iobref = 1;
-                }
+        if (!iobref) {
+            iobref = iobref_new();
+            if (!iobref) {
+                goto out;
+            }
 
-                iobref_add (iobref, iobuf);
+            new_iobref = 1;
+        }
 
-                iov.iov_base = iobuf->ptr;
-                iov.iov_len  = iobuf_size (iobuf);
+        iobref_add(iobref, iobuf);
 
-                /* Create the xdr payload */
-                ret = xdr_serialize_generic (iov, req, xdrproc);
-                if (ret == -1) {
-                        goto out;
-                }
+        iov.iov_base = iobuf->ptr;
+        iov.iov_len = iobuf_size(iobuf);
 
-                iov.iov_len = ret;
-                count = 1;
+        /* Create the xdr payload */
+        ret = xdr_serialize_generic(iov, req, xdrproc);
+        if (ret == -1) {
+            goto out;
         }
 
-        ret = rpc_clnt_submit (rpc, prog, procnum, cbkfn, &iov, count,
-                               payload, payloadcnt, iobref, frame, NULL,
-                               0, NULL, 0, NULL);
+        iov.iov_len = ret;
+        count = 1;
+    }
+
+    ret = rpc_clnt_submit(rpc, prog, procnum, cbkfn, &iov, count, payload,
+                          payloadcnt, iobref, frame, NULL, 0, NULL, 0, NULL);
 
- out:
-        if (new_iobref)
-                iobref_unref (iobref);
-        if (iobuf)
-                iobuf_unref (iobuf);
-        return ret;
+out:
+    if (new_iobref)
+        iobref_unref(iobref);
+    if (iobuf)
+        iobuf_unref(iobuf);
+    return ret;
 }
 
 /**
  * Entry point to perform a remote procedure call
  */
 int
-changelog_invoke_rpc (xlator_t *this, struct rpc_clnt *rpc,
-                      rpc_clnt_prog_t *prog, int procidx, void *arg)
+changelog_invoke_rpc(xlator_t *this, struct rpc_clnt *rpc,
+                     rpc_clnt_prog_t *prog, int procidx, void *arg)
 {
-        int                   ret   = 0;
-        call_frame_t         *frame = NULL;
-        rpc_clnt_procedure_t *proc  = NULL;
+    int ret = 0;
+    call_frame_t *frame = NULL;
+    rpc_clnt_procedure_t *proc = NULL;
 
-        if (!this || !prog)
-                goto error_return;
+    if (!this || !prog)
+        goto error_return;
 
-        frame = create_frame (this, this->ctx->pool);
-        if (!frame) {
-                gf_log (this->name, GF_LOG_ERROR, "failed to create frame");
-                goto error_return;
-        }
+    frame = create_frame(this, this->ctx->pool);
+    if (!frame) {
+        gf_smsg(this->name, GF_LOG_ERROR, 0, CHANGELOG_MSG_CREATE_FRAME_FAILED,
+                NULL);
+        goto error_return;
+    }
 
-        proc = &prog->proctable[procidx];
-        if (proc->fn)
-                ret = proc->fn (frame, this, arg);
+    proc = &prog->proctable[procidx];
+    if (proc->fn)
+        ret = proc->fn(frame, this, arg);
 
-        STACK_DESTROY (frame->root);
-        return ret;
+    STACK_DESTROY(frame->root);
+    return ret;
 
- error_return:
-        return -1;
+error_return:
+    return -1;
 }
 
 /**
@@ -180,155 +187,173 @@ changelog_invoke_rpc (xlator_t *this, struct rpc_clnt *rpc,
 */
 
 struct iobuf *
-__changelog_rpc_serialize_reply (rpcsvc_request_t *req, void *arg,
-                                 struct iovec *outmsg, xdrproc_t xdrproc)
+__changelog_rpc_serialize_reply(rpcsvc_request_t *req, void *arg,
+                                struct iovec *outmsg, xdrproc_t xdrproc)
 {
-        struct iobuf *iob      = NULL;
-        ssize_t       retlen   = 0;
-        ssize_t       rsp_size = 0;
+    struct iobuf *iob = NULL;
+    ssize_t retlen = 0;
+    ssize_t rsp_size = 0;
 
-        rsp_size = xdr_sizeof (xdrproc, arg);
-        iob = iobuf_get2 (req->svc->ctx->iobuf_pool, rsp_size);
-        if (!iob)
-                goto error_return;
+    rsp_size = xdr_sizeof(xdrproc, arg);
+    iob = iobuf_get2(req->svc->ctx->iobuf_pool, rsp_size);
+    if (!iob)
+        goto error_return;
 
-        iobuf_to_iovec (iob, outmsg);
+    iobuf_to_iovec(iob, outmsg);
 
-        retlen = xdr_serialize_generic (*outmsg, arg, xdrproc);
-        if (retlen == -1)
-                goto unref_iob;
+    retlen = xdr_serialize_generic(*outmsg, arg, xdrproc);
+    if (retlen == -1)
+        goto unref_iob;
 
-        outmsg->iov_len = retlen;
-        return iob;
+    outmsg->iov_len = retlen;
+    return iob;
 
- unref_iob:
-        iobuf_unref (iob);
- error_return:
-        return NULL;
+unref_iob:
+    iobuf_unref(iob);
+error_return:
+    return NULL;
 }
 
 int
-changelog_rpc_sumbit_reply (rpcsvc_request_t *req,
-                            void *arg, struct iovec *payload, int payloadcount,
-                            struct iobref *iobref, xdrproc_t xdrproc)
+changelog_rpc_sumbit_reply(rpcsvc_request_t *req, void *arg,
+                           struct iovec *payload, int payloadcount,
+                           struct iobref *iobref, xdrproc_t xdrproc)
 {
-        int           ret        = -1;
-        struct iobuf *iob        = NULL;
-        struct iovec  iov        = {0,};
-        char          new_iobref = 0;
-
-        if (!req)
-                goto return_ret;
-
-        if (!iobref) {
-                iobref = iobref_new ();
-                if (!iobref)
-                        goto return_ret;
-                new_iobref = 1;
-        }
-
-        iob = __changelog_rpc_serialize_reply (req, arg, &iov, xdrproc);
-        if (!iob)
-                gf_log ("", GF_LOG_ERROR, "failed to serialize reply");
-        else
-                iobref_add (iobref, iob);
-
-        ret = rpcsvc_submit_generic (req, &iov,
-                                     1, payload, payloadcount, iobref);
-
-        if (new_iobref)
-                iobref_unref (iobref);
-        if (iob)
-                iobuf_unref (iob);
- return_ret:
-        return ret;
+    int ret = -1;
+    struct iobuf *iob = NULL;
+    struct iovec iov = {
+        0,
+    };
+    char new_iobref = 0;
+
+    if (!req)
+        goto return_ret;
+
+    if (!iobref) {
+        iobref = iobref_new();
+        if (!iobref)
+            goto return_ret;
+        new_iobref = 1;
+    }
+
+    iob = __changelog_rpc_serialize_reply(req, arg, &iov, xdrproc);
+    if (!iob)
+        gf_smsg("", GF_LOG_ERROR, 0, CHANGELOG_MSG_RPC_SUBMIT_REPLY_FAILED,
+                NULL);
+    else
+        iobref_add(iobref, iob);
+
+    ret = rpcsvc_submit_generic(req, &iov, 1, payload, payloadcount, iobref);
+
+    if (new_iobref)
+        iobref_unref(iobref);
+    if (iob)
+        iobuf_unref(iob);
+return_ret:
+    return ret;
 }
 
 void
-changelog_rpc_server_destroy (xlator_t *this, rpcsvc_t *rpc, char *sockfile,
-                              rpcsvc_notify_t fn, struct rpcsvc_program **progs)
+changelog_rpc_server_destroy(xlator_t *this, rpcsvc_t *rpc, char *sockfile,
+                             rpcsvc_notify_t fn, struct rpcsvc_program **progs)
 {
-        rpcsvc_listener_t      *listener = NULL;
-        rpcsvc_listener_t      *next    = NULL;
-        struct rpcsvc_program *prog    = NULL;
-
-        while (*progs) {
-                prog = *progs;
-                (void) rpcsvc_program_unregister (rpc, prog);
+    rpcsvc_listener_t *listener = NULL;
+    rpcsvc_listener_t *next = NULL;
+    struct rpcsvc_program *prog = NULL;
+    rpc_transport_t *trans = NULL;
+
+    if (!rpc)
+        return;
+
+    while (*progs) {
+        prog = *progs;
+        (void)rpcsvc_program_unregister(rpc, prog);
+        progs++;
+    }
+
+    list_for_each_entry_safe(listener, next, &rpc->listeners, list)
+    {
+        if (listener->trans) {
+            trans = listener->trans;
+            rpc_transport_disconnect(trans, _gf_false);
         }
-
-        list_for_each_entry_safe (listener, next, &rpc->listeners, list) {
-                rpcsvc_listener_destroy (listener);
+    }
+
+    (void)rpcsvc_unregister_notify(rpc, fn, this);
+
+    /* TODO Avoid freeing rpc object in case of brick multiplex
+       after freeing rpc object svc->rpclock corrupted and it takes
+       more time to detach a brick
+    */
+    if (!this->cleanup_starting) {
+        if (rpc->rxpool) {
+            mem_pool_destroy(rpc->rxpool);
+            rpc->rxpool = NULL;
         }
-
-        (void) rpcsvc_unregister_notify (rpc, fn, this);
-        unlink (sockfile);
-
-        GF_FREE (rpc);
+        GF_FREE(rpc);
+    }
 }
 
 rpcsvc_t *
-changelog_rpc_server_init (xlator_t *this, char *sockfile, void *cbkdata,
-                           rpcsvc_notify_t fn, struct rpcsvc_program **progs)
+changelog_rpc_server_init(xlator_t *this, char *sockfile, void *cbkdata,
+                          rpcsvc_notify_t fn, struct rpcsvc_program **progs)
 {
-        int                    j       = 0;
-        int                    ret     = 0;
-        rpcsvc_t              *rpc     = NULL;
-        dict_t                *options = NULL;
-        struct rpcsvc_program *prog    = NULL;
-
-        if (!cbkdata)
-                cbkdata = this;
-
-        options = dict_new ();
-        if (!options)
-                goto error_return;
-
-        ret = rpcsvc_transport_unix_options_build (&options, sockfile);
-        if (ret)
-                goto dealloc_dict;
-
-        rpc = rpcsvc_init (this, this->ctx, options, 8);
-        if (rpc == NULL) {
-                gf_log (this->name, GF_LOG_ERROR, "failed to init rpc");
-                goto dealloc_dict;
-        }
+    int ret = 0;
+    rpcsvc_t *rpc = NULL;
+    dict_t *options = NULL;
+    struct rpcsvc_program *prog = NULL;
 
-        ret = rpcsvc_register_notify (rpc, fn, cbkdata);
-        if (ret) {
-                gf_log (this->name,
-                        GF_LOG_ERROR, "failed to register notify function");
-                goto dealloc_rpc;
-        }
+    if (!cbkdata)
+        cbkdata = this;
 
-        ret = rpcsvc_create_listeners (rpc, options, this->name);
-        if (ret != 1) {
-                gf_log (this->name,
-                        GF_LOG_DEBUG, "failed to create listeners");
-                goto dealloc_rpc;
-        }
+    options = dict_new();
+    if (!options)
+        return NULL;
 
-        while (*progs) {
-                prog = *progs;
-                ret = rpcsvc_program_register (rpc, prog);
-                if (ret) {
-                        gf_log (this->name,
-                                GF_LOG_ERROR, "cannot register program "
-                                "(name: %s, prognum: %d, pogver: %d)",
-                                prog->progname, prog->prognum, prog->progver);
-                        goto dealloc_rpc;
-                }
-
-                progs++;
+    ret = rpcsvc_transport_unix_options_build(options, sockfile);
+    if (ret)
+        goto dealloc_dict;
+
+    rpc = rpcsvc_init(this, this->ctx, options, 8);
+    if (rpc == NULL) {
+        gf_smsg(this->name, GF_LOG_ERROR, 0, CHANGELOG_MSG_RPC_START_ERROR,
+                NULL);
+        goto dealloc_dict;
+    }
+
+    ret = rpcsvc_register_notify(rpc, fn, cbkdata);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_ERROR, 0,
+                CHANGELOG_MSG_NOTIFY_REGISTER_FAILED, NULL);
+        goto dealloc_rpc;
+    }
+
+    ret = rpcsvc_create_listeners(rpc, options, this->name);
+    if (ret != 1) {
+        gf_msg_debug(this->name, 0, "failed to create listeners");
+        goto dealloc_rpc;
+    }
+
+    while (*progs) {
+        prog = *progs;
+        ret = rpcsvc_program_register(rpc, prog, _gf_false);
+        if (ret) {
+            gf_smsg(this->name, GF_LOG_ERROR, 0,
+                    CHANGELOG_MSG_PROGRAM_NAME_REG_FAILED, "name%s",
+                    prog->progname, "prognum=%d", prog->prognum, "pogver=%d",
+                    prog->progver, NULL);
+            goto dealloc_rpc;
         }
 
-        dict_unref (options);
-        return rpc;
+        progs++;
+    }
 
- dealloc_rpc:
-        GF_FREE (rpc);
- dealloc_dict:
-        dict_unref (options);
- error_return:
-        return NULL;
+    dict_unref(options);
+    return rpc;
+
+dealloc_rpc:
+    GF_FREE(rpc);
+dealloc_dict:
+    dict_unref(options);
+    return NULL;
 }
diff --git a/xlators/features/changelog/src/changelog-rpc-common.h b/xlators/features/changelog/src/changelog-rpc-common.h
index 95c850c9400..4d9aa2c694b 100644
--- a/xlators/features/changelog/src/changelog-rpc-common.h
+++ b/xlators/features/changelog/src/changelog-rpc-common.h
@@ -13,8 +13,8 @@
 
 #include "rpcsvc.h"
 #include "rpc-clnt.h"
-#include "event.h"
-#include "call-stub.h"
+#include <glusterfs/gf-event.h>
+#include <glusterfs/call-stub.h>
 
 #include "changelog-xdr.h"
 #include "xdr-generic.h"
@@ -24,61 +24,62 @@
 /**
  * Let's keep this non-configurable for now.
  */
-#define NR_ROTT_BUFFS  4
+#define NR_ROTT_BUFFS 4
 #define NR_DISPATCHERS (NR_ROTT_BUFFS - 1)
 
 enum changelog_rpc_procnum {
-        CHANGELOG_RPC_PROC_NULL    = 0,
-        CHANGELOG_RPC_PROBE_FILTER = 1,
-        CHANGELOG_RPC_PROC_MAX     = 2,
+    CHANGELOG_RPC_PROC_NULL = 0,
+    CHANGELOG_RPC_PROBE_FILTER = 1,
+    CHANGELOG_RPC_PROC_MAX = 2,
 };
 
-#define CHANGELOG_RPC_PROGNUM   1885957735
-#define CHANGELOG_RPC_PROGVER   1
+#define CHANGELOG_RPC_PROGNUM 1885957735
+#define CHANGELOG_RPC_PROGVER 1
 
 /**
  * reverse connection: data xfer path
  */
 enum changelog_reverse_rpc_procnum {
-        CHANGELOG_REV_PROC_NULL  = 0,
-        CHANGELOG_REV_PROC_EVENT = 1,
-        CHANGELOG_REV_PROC_MAX   = 2,
+    CHANGELOG_REV_PROC_NULL = 0,
+    CHANGELOG_REV_PROC_EVENT = 1,
+    CHANGELOG_REV_PROC_MAX = 2,
 };
 
-#define CHANGELOG_REV_RPC_PROCNUM   1886350951
-#define CHANGELOG_REV_RPC_PROCVER   1
+#define CHANGELOG_REV_RPC_PROCNUM 1886350951
+#define CHANGELOG_REV_RPC_PROCVER 1
 
 typedef struct changelog_rpc {
-        rpcsvc_t        *svc;
-        struct rpc_clnt *rpc;
-        char             sock[UNIX_PATH_MAX];  /* tied to server */
+    rpcsvc_t *svc;
+    struct rpc_clnt *rpc;
+    char sock[UNIX_PATH_MAX]; /* tied to server */
 } changelog_rpc_t;
 
 /* event poller */
-void *changelog_rpc_poller (void *);
+void *
+changelog_rpc_poller(void *);
 
 /* CLIENT API */
 struct rpc_clnt *
-changelog_rpc_client_init (xlator_t *, void *, char *, rpc_clnt_notify_t);
+changelog_rpc_client_init(xlator_t *, void *, char *, rpc_clnt_notify_t);
 
 int
-changelog_rpc_sumbit_req (struct rpc_clnt *, void *, call_frame_t *,
-                          rpc_clnt_prog_t *, int , struct iovec *, int,
-                          struct iobref *, xlator_t *, fop_cbk_fn_t, xdrproc_t);
+changelog_rpc_sumbit_req(struct rpc_clnt *, void *, call_frame_t *,
+                         rpc_clnt_prog_t *, int, struct iovec *, int,
+                         struct iobref *, xlator_t *, fop_cbk_fn_t, xdrproc_t);
 
 int
-changelog_invoke_rpc (xlator_t *, struct rpc_clnt *,
-                      rpc_clnt_prog_t *, int , void *);
+changelog_invoke_rpc(xlator_t *, struct rpc_clnt *, rpc_clnt_prog_t *, int,
+                     void *);
 
 /* SERVER API */
 int
-changelog_rpc_sumbit_reply (rpcsvc_request_t *, void *,
-                            struct iovec *, int, struct iobref *, xdrproc_t);
+changelog_rpc_sumbit_reply(rpcsvc_request_t *, void *, struct iovec *, int,
+                           struct iobref *, xdrproc_t);
 rpcsvc_t *
-changelog_rpc_server_init (xlator_t *, char *, void*,
-                           rpcsvc_notify_t, struct rpcsvc_program **);
+changelog_rpc_server_init(xlator_t *, char *, void *, rpcsvc_notify_t,
+                          struct rpcsvc_program **);
 void
-changelog_rpc_server_destroy (xlator_t *, rpcsvc_t *, char *,
-                              rpcsvc_notify_t, struct rpcsvc_program **);
+changelog_rpc_server_destroy(xlator_t *, rpcsvc_t *, char *, rpcsvc_notify_t,
+                             struct rpcsvc_program **);
 
 #endif
diff --git a/xlators/features/changelog/src/changelog-rpc.c b/xlators/features/changelog/src/changelog-rpc.c
index 04326456d31..440b88091a6 100644
--- a/xlators/features/changelog/src/changelog-rpc.c
+++ b/xlators/features/changelog/src/changelog-rpc.c
@@ -8,216 +8,346 @@
    cases as published by the Free Software Foundation.
 */
 
+#include <glusterfs/syscall.h>
 #include "changelog-rpc.h"
 #include "changelog-mem-types.h"
 #include "changelog-ev-handle.h"
 
-struct rpcsvc_program *changelog_programs[];
+static struct rpcsvc_program *changelog_programs[];
 
 static void
-changelog_cleanup_dispatchers (xlator_t *this,
-                               changelog_priv_t *priv, int count)
+changelog_cleanup_dispatchers(xlator_t *this, changelog_priv_t *priv, int count)
 {
-        for (; count >= 0; count--) {
-                (void) changelog_thread_cleanup
-                        (this, priv->ev_dispatcher[count]);
-        }
+    for (count--; count >= 0; count--) {
+        (void)changelog_thread_cleanup(this, priv->ev_dispatcher[count]);
+        priv->ev_dispatcher[count] = 0;
+    }
 }
 
-static int
-changelog_cleanup_rpc_threads (xlator_t *this, changelog_priv_t *priv)
+int
+changelog_cleanup_rpc_threads(xlator_t *this, changelog_priv_t *priv)
 {
-        int ret = 0;
-        changelog_clnt_t *conn = NULL;
-
-        conn = &priv->connections;
-        if (!conn)
-                return 0;
-
-        /** terminate RPC thread(s) */
-        ret = changelog_thread_cleanup (this, priv->connector);
-        if (ret != 0)
-                goto error_return;
-        /** terminate dispatcher thread(s) */
-        changelog_cleanup_dispatchers (this, priv, priv->nr_dispatchers);
-
-        /* TODO: what about pending and waiting connections? */
-        changelog_ev_cleanup_connections (this, conn);
-
-        /* destroy locks */
-        ret = pthread_mutex_destroy (&conn->pending_lock);
-        if (ret != 0)
-                goto error_return;
-        ret = pthread_cond_destroy (&conn->pending_cond);
-        if (ret != 0)
-                goto error_return;
-        ret = LOCK_DESTROY (&conn->active_lock);
-        if (ret != 0)
-                goto error_return;
-        ret = LOCK_DESTROY (&conn->wait_lock);
-        if (ret != 0)
-                goto error_return;
+    int ret = 0;
+    changelog_clnt_t *conn = NULL;
+
+    conn = &priv->connections;
+    if (!conn)
         return 0;
 
- error_return:
-        return -1;
+    /** terminate RPC thread(s) */
+    ret = changelog_thread_cleanup(this, priv->connector);
+    if (ret != 0)
+        goto error_return;
+    priv->connector = 0;
+
+    /** terminate dispatcher thread(s) */
+    changelog_cleanup_dispatchers(this, priv, priv->nr_dispatchers);
+
+    /* destroy locks */
+    ret = pthread_mutex_destroy(&conn->pending_lock);
+    if (ret != 0)
+        goto error_return;
+    ret = pthread_cond_destroy(&conn->pending_cond);
+    if (ret != 0)
+        goto error_return;
+    ret = LOCK_DESTROY(&conn->active_lock);
+    if (ret != 0)
+        goto error_return;
+    ret = LOCK_DESTROY(&conn->wait_lock);
+    if (ret != 0)
+        goto error_return;
+    return 0;
+
+error_return:
+    return -1;
 }
 
 static int
-changelog_init_rpc_threads (xlator_t *this, changelog_priv_t *priv,
-                            rbuf_t *rbuf, int nr_dispatchers)
+changelog_init_rpc_threads(xlator_t *this, changelog_priv_t *priv, rbuf_t *rbuf,
+                           int nr_dispatchers)
+{
+    int j = 0;
+    int ret = 0;
+    changelog_clnt_t *conn = NULL;
+
+    conn = &priv->connections;
+
+    conn->this = this;
+    conn->rbuf = rbuf;
+    conn->sequence = 1; /* start with sequence number one */
+
+    INIT_LIST_HEAD(&conn->pending);
+    INIT_LIST_HEAD(&conn->active);
+    INIT_LIST_HEAD(&conn->waitq);
+
+    ret = pthread_mutex_init(&conn->pending_lock, NULL);
+    if (ret)
+        goto error_return;
+    ret = pthread_cond_init(&conn->pending_cond, NULL);
+    if (ret)
+        goto cleanup_pending_lock;
+
+    ret = LOCK_INIT(&conn->active_lock);
+    if (ret)
+        goto cleanup_pending_cond;
+    ret = LOCK_INIT(&conn->wait_lock);
+    if (ret)
+        goto cleanup_active_lock;
+
+    /* spawn reverse connection thread */
+    ret = gf_thread_create(&priv->connector, NULL, changelog_ev_connector, conn,
+                           "clogecon");
+    if (ret != 0)
+        goto cleanup_wait_lock;
+
+    /* spawn dispatcher thread(s) */
+    priv->ev_dispatcher = GF_CALLOC(nr_dispatchers, sizeof(pthread_t),
+                                    gf_changelog_mt_ev_dispatcher_t);
+    if (!priv->ev_dispatcher)
+        goto cleanup_connector;
+
+    /* spawn dispatcher threads */
+    for (; j < nr_dispatchers; j++) {
+        ret = gf_thread_create(&priv->ev_dispatcher[j], NULL,
+                               changelog_ev_dispatch, conn, "clogd%03hx",
+                               j & 0x3ff);
+        if (ret != 0) {
+            changelog_cleanup_dispatchers(this, priv, j);
+            break;
+        }
+    }
+
+    if (ret != 0)
+        goto cleanup_connector;
+
+    priv->nr_dispatchers = nr_dispatchers;
+    return 0;
+
+cleanup_connector:
+    (void)pthread_cancel(priv->connector);
+cleanup_wait_lock:
+    LOCK_DESTROY(&conn->wait_lock);
+cleanup_active_lock:
+    LOCK_DESTROY(&conn->active_lock);
+cleanup_pending_cond:
+    (void)pthread_cond_destroy(&conn->pending_cond);
+cleanup_pending_lock:
+    (void)pthread_mutex_destroy(&conn->pending_lock);
+error_return:
+    return -1;
+}
+
+int
+changelog_rpcsvc_notify(rpcsvc_t *rpc, void *xl, rpcsvc_event_t event,
+                        void *data)
 {
-        int               j    = 0;
-        int               ret  = 0;
-        changelog_clnt_t *conn = NULL;
-
-        conn = &priv->connections;
-
-        conn->this = this;
-        conn->rbuf = rbuf;
-        conn->sequence = 1; /* start with sequence number one */
-
-        INIT_LIST_HEAD (&conn->pending);
-        INIT_LIST_HEAD (&conn->active);
-        INIT_LIST_HEAD (&conn->waitq);
-
-        ret = pthread_mutex_init (&conn->pending_lock, NULL);
-        if (ret)
-                goto error_return;
-        ret = pthread_cond_init (&conn->pending_cond, NULL);
-        if (ret)
-                goto cleanup_pending_lock;
-
-        ret = LOCK_INIT (&conn->active_lock);
-        if (ret)
-                goto cleanup_pending_cond;
-        ret = LOCK_INIT (&conn->wait_lock);
-        if (ret)
-                goto cleanup_active_lock;
-
-        /* spawn reverse connection thread */
-        ret = pthread_create (&priv->connector,
-                              NULL, changelog_ev_connector, conn);
-        if (ret != 0)
-                goto cleanup_wait_lock;
-
-        /* spawn dispatcher thread(s) */
-        priv->ev_dispatcher = GF_CALLOC (nr_dispatchers, sizeof(pthread_t),
-                                         gf_changelog_mt_ev_dispatcher_t);
-        if (!priv->ev_dispatcher)
-                goto cleanup_connector;
-
-        /* spawn dispatcher threads */
-        for (; j < nr_dispatchers; j++) {
-                ret = pthread_create (&priv->ev_dispatcher[j],
-                                      NULL, changelog_ev_dispatch, conn);
-                if (ret != 0) {
-                        changelog_cleanup_dispatchers (this, priv, --j);
-                        break;
+    xlator_t *this = NULL;
+    rpc_transport_t *trans = NULL;
+    rpc_transport_t *xprt = NULL;
+    rpc_transport_t *xp_next = NULL;
+    changelog_priv_t *priv = NULL;
+    uint64_t listnercnt = 0;
+    uint64_t xprtcnt = 0;
+    uint64_t clntcnt = 0;
+    rpcsvc_listener_t *listener = NULL;
+    rpcsvc_listener_t *next = NULL;
+    gf_boolean_t listner_found = _gf_false;
+    socket_private_t *sockpriv = NULL;
+
+    if (!xl || !data || !rpc) {
+        gf_msg_callingfn("changelog", GF_LOG_WARNING, 0,
+                         CHANGELOG_MSG_RPCSVC_NOTIFY_FAILED,
+                         "Calling rpc_notify without initializing");
+        goto out;
+    }
+
+    this = xl;
+    trans = data;
+    priv = this->private;
+
+    if (!priv) {
+        gf_msg_callingfn("changelog", GF_LOG_WARNING, 0,
+                         CHANGELOG_MSG_RPCSVC_NOTIFY_FAILED,
+                         "Calling rpc_notify without priv initializing");
+        goto out;
+    }
+
+    if (event == RPCSVC_EVENT_ACCEPT) {
+        GF_ATOMIC_INC(priv->xprtcnt);
+        LOCK(&priv->lock);
+        {
+            list_add_tail(&trans->list, &priv->xprt_list);
+        }
+        UNLOCK(&priv->lock);
+        goto out;
+    }
+
+    if (event == RPCSVC_EVENT_DISCONNECT) {
+        list_for_each_entry_safe(listener, next, &rpc->listeners, list)
+        {
+            if (listener && listener->trans) {
+                if (listener->trans == trans) {
+                    listnercnt = GF_ATOMIC_DEC(priv->listnercnt);
+                    listner_found = _gf_true;
+                    rpcsvc_listener_destroy(listener);
                 }
+            }
         }
 
-        if (ret != 0)
-                goto cleanup_connector;
+        if (listnercnt > 0) {
+            goto out;
+        }
+        if (listner_found) {
+            LOCK(&priv->lock);
+            list_for_each_entry_safe(xprt, xp_next, &priv->xprt_list, list)
+            {
+                sockpriv = (socket_private_t *)(xprt->private);
+                gf_log("changelog", GF_LOG_INFO,
+                       "Send disconnect"
+                       " on socket %d",
+                       sockpriv->sock);
+                rpc_transport_disconnect(xprt, _gf_false);
+            }
+            UNLOCK(&priv->lock);
+            goto out;
+        }
+        LOCK(&priv->lock);
+        {
+            list_del_init(&trans->list);
+        }
+        UNLOCK(&priv->lock);
 
-        priv->nr_dispatchers = nr_dispatchers;
-        return 0;
+        xprtcnt = GF_ATOMIC_DEC(priv->xprtcnt);
+        clntcnt = GF_ATOMIC_GET(priv->clntcnt);
+        if (!xprtcnt && !clntcnt) {
+            changelog_process_cleanup_event(this);
+        }
+    }
 
- cleanup_connector:
-        (void) pthread_cancel (priv->connector);
- cleanup_wait_lock:
-        (void) LOCK_DESTROY (&conn->wait_lock);
- cleanup_active_lock:
-        (void) LOCK_DESTROY (&conn->active_lock);
- cleanup_pending_cond:
-        (void) pthread_cond_destroy (&conn->pending_cond);
- cleanup_pending_lock:
-        (void) pthread_mutex_destroy (&conn->pending_lock);
- error_return:
-        return -1;
+out:
+    return 0;
 }
 
-int
-changelog_rpcsvc_notify (rpcsvc_t *rpc,
-                         void *xl, rpcsvc_event_t event, void *data)
+void
+changelog_process_cleanup_event(xlator_t *this)
 {
-        return 0;
+    gf_boolean_t cleanup_notify = _gf_false;
+    changelog_priv_t *priv = NULL;
+    char sockfile[UNIX_PATH_MAX] = {
+        0,
+    };
+
+    if (!this)
+        return;
+    priv = this->private;
+    if (!priv)
+        return;
+
+    LOCK(&priv->lock);
+    {
+        cleanup_notify = priv->notify_down;
+        priv->notify_down = _gf_true;
+    }
+    UNLOCK(&priv->lock);
+
+    if (priv->victim && !cleanup_notify) {
+        default_notify(this, GF_EVENT_PARENT_DOWN, priv->victim);
+
+        if (priv->rpc) {
+            /* sockfile path could have been saved to avoid this */
+            CHANGELOG_MAKE_SOCKET_PATH(priv->changelog_brick, sockfile,
+                                       UNIX_PATH_MAX);
+            sys_unlink(sockfile);
+            (void)rpcsvc_unregister_notify(priv->rpc, changelog_rpcsvc_notify,
+                                           this);
+            if (priv->rpc->rxpool) {
+                mem_pool_destroy(priv->rpc->rxpool);
+                priv->rpc->rxpool = NULL;
+            }
+            GF_FREE(priv->rpc);
+            priv->rpc = NULL;
+        }
+    }
 }
 
 void
-changelog_destroy_rpc_listner (xlator_t *this, changelog_priv_t *priv)
+changelog_destroy_rpc_listner(xlator_t *this, changelog_priv_t *priv)
 {
-        char sockfile[UNIX_PATH_MAX] = {0,};
-
-        /* sockfile path could have been saved to avoid this */
-        CHANGELOG_MAKE_SOCKET_PATH (priv->changelog_brick,
-                                    sockfile, UNIX_PATH_MAX);
-        changelog_rpc_server_destroy (this,
-                                      priv->rpc, sockfile,
-                                      changelog_rpcsvc_notify,
-                                      changelog_programs);
-        (void) changelog_cleanup_rpc_threads (this, priv);
+    char sockfile[UNIX_PATH_MAX] = {
+        0,
+    };
+
+    /* sockfile path could have been saved to avoid this */
+    CHANGELOG_MAKE_SOCKET_PATH(priv->changelog_brick, sockfile, UNIX_PATH_MAX);
+    changelog_rpc_server_destroy(this, priv->rpc, sockfile,
+                                 changelog_rpcsvc_notify, changelog_programs);
 }
 
 rpcsvc_t *
-changelog_init_rpc_listner (xlator_t *this, changelog_priv_t *priv,
+changelog_init_rpc_listener(xlator_t *this, changelog_priv_t *priv,
                             rbuf_t *rbuf, int nr_dispatchers)
 {
-        int ret = 0;
-        char sockfile[UNIX_PATH_MAX] = {0,};
-
-        ret = changelog_init_rpc_threads (this, priv, rbuf, nr_dispatchers);
-        if (ret)
-                return NULL;
-
-        CHANGELOG_MAKE_SOCKET_PATH (priv->changelog_brick,
-                                    sockfile, UNIX_PATH_MAX);
-        return changelog_rpc_server_init (this, sockfile, NULL,
-                                          changelog_rpcsvc_notify,
-                                          changelog_programs);
+    int ret = 0;
+    char sockfile[UNIX_PATH_MAX] = {
+        0,
+    };
+    rpcsvc_t *svcp;
+
+    ret = changelog_init_rpc_threads(this, priv, rbuf, nr_dispatchers);
+    if (ret)
+        return NULL;
+
+    CHANGELOG_MAKE_SOCKET_PATH(priv->changelog_brick, sockfile, UNIX_PATH_MAX);
+    (void)sys_unlink(sockfile);
+    svcp = changelog_rpc_server_init(
+        this, sockfile, NULL, changelog_rpcsvc_notify, changelog_programs);
+    return svcp;
 }
 
 void
-changelog_rpc_clnt_cleanup (changelog_rpc_clnt_t *crpc)
+changelog_rpc_clnt_cleanup(changelog_rpc_clnt_t *crpc)
 {
-        if (!crpc)
-                return;
-        crpc->c_clnt = NULL;
-        (void) LOCK_DESTROY (&crpc->lock);
-        GF_FREE (crpc);
+    if (!crpc)
+        return;
+    crpc->c_clnt = NULL;
+    LOCK_DESTROY(&crpc->lock);
+    GF_FREE(crpc);
 }
 
-inline changelog_rpc_clnt_t *
-changelog_rpc_clnt_init (xlator_t *this,
-                         changelog_probe_req *rpc_req, changelog_clnt_t *c_clnt)
+static changelog_rpc_clnt_t *
+changelog_rpc_clnt_init(xlator_t *this, changelog_probe_req *rpc_req,
+                        changelog_clnt_t *c_clnt)
 {
-        int ret = 0;
-        changelog_rpc_clnt_t *crpc = NULL;
-
-        crpc = GF_CALLOC (1, sizeof (*crpc), gf_changelog_mt_rpc_clnt_t);
-        if (!crpc)
-                goto error_return;
-        INIT_LIST_HEAD (&crpc->list);
-
-        crpc->ref = 0;
-        changelog_set_disconnect_flag (crpc, _gf_false);
-
-        crpc->filter = rpc_req->filter;
-        (void) memcpy (crpc->sock, rpc_req->sock, strlen (rpc_req->sock));
-
-        crpc->this = this;
-        crpc->c_clnt = c_clnt;
-        crpc->cleanup = changelog_rpc_clnt_cleanup;
-
-        ret = LOCK_INIT (&crpc->lock);
-        if (ret != 0)
-                goto dealloc_crpc;
-        return crpc;
-
- dealloc_crpc:
-        GF_FREE (crpc);
- error_return:
-        return NULL;
+    int ret = 0;
+    changelog_rpc_clnt_t *crpc = NULL;
+
+    crpc = GF_CALLOC(1, sizeof(*crpc), gf_changelog_mt_rpc_clnt_t);
+    if (!crpc)
+        goto error_return;
+    INIT_LIST_HEAD(&crpc->list);
+
+    /* Take a ref, the last unref will be on RPC_CLNT_DESTROY
+     * which comes as a result of last rpc_clnt_unref.
+     */
+    GF_ATOMIC_INIT(crpc->ref, 1);
+    changelog_set_disconnect_flag(crpc, _gf_false);
+
+    crpc->filter = rpc_req->filter;
+    (void)memcpy(crpc->sock, rpc_req->sock, strlen(rpc_req->sock));
+
+    crpc->this = this;
+    crpc->c_clnt = c_clnt;
+    crpc->cleanup = changelog_rpc_clnt_cleanup;
+
+    ret = LOCK_INIT(&crpc->lock);
+    if (ret != 0)
+        goto dealloc_crpc;
+    return crpc;
+
+dealloc_crpc:
+    GF_FREE(crpc);
+error_return:
+    return NULL;
 }
 
 /**
@@ -231,70 +361,80 @@ changelog_rpc_clnt_init (xlator_t *this,
  */
 
 int
-changelog_handle_probe (rpcsvc_request_t *req)
+changelog_handle_probe(rpcsvc_request_t *req)
 {
-        int                   ret     = 0;
-        xlator_t             *this    = NULL;
-        rpcsvc_t             *svc     = NULL;
-        changelog_priv_t     *priv    = NULL;
-        changelog_clnt_t     *c_clnt  = NULL;
-        changelog_rpc_clnt_t *crpc    = NULL;
-
-        changelog_probe_req   rpc_req = {0,};
-        changelog_probe_rsp   rpc_rsp = {0,};
-
-        ret = xdr_to_generic (req->msg[0],
-                              &rpc_req, (xdrproc_t)xdr_changelog_probe_req);
-        if (ret < 0) {
-                gf_log ("", GF_LOG_ERROR, "xdr decoding error");
-                req->rpc_err = GARBAGE_ARGS;
-                goto handle_xdr_error;
-        }
-
-        /* ->xl hidden in rpcsvc */
-        svc    = rpcsvc_request_service (req);
-        this   = svc->mydata;
-        priv   = this->private;
-        c_clnt = &priv->connections;
-
-        crpc = changelog_rpc_clnt_init (this, &rpc_req, c_clnt);
-        if (!crpc)
-                goto handle_xdr_error;
-
-        changelog_ev_queue_connection (c_clnt, crpc);
-        rpc_rsp.op_ret = 0;
-
-        goto submit_rpc;
-
- handle_xdr_error:
-        rpc_rsp.op_ret = -1;
- submit_rpc:
-        (void) changelog_rpc_sumbit_reply (req, &rpc_rsp, NULL, 0, NULL,
-                                           (xdrproc_t)xdr_changelog_probe_rsp);
+    int ret = 0;
+    xlator_t *this = NULL;
+    rpcsvc_t *svc = NULL;
+    changelog_priv_t *priv = NULL;
+    changelog_clnt_t *c_clnt = NULL;
+    changelog_rpc_clnt_t *crpc = NULL;
+
+    changelog_probe_req rpc_req = {
+        0,
+    };
+    changelog_probe_rsp rpc_rsp = {
+        0,
+    };
+
+    this = req->trans->xl;
+    if (this->cleanup_starting) {
+        gf_smsg(this->name, GF_LOG_DEBUG, 0, CHANGELOG_MSG_CLEANUP_ALREADY_SET,
+                NULL);
         return 0;
+    }
+
+    ret = xdr_to_generic(req->msg[0], &rpc_req,
+                         (xdrproc_t)xdr_changelog_probe_req);
+    if (ret < 0) {
+        gf_smsg("", GF_LOG_ERROR, 0, CHANGELOG_MSG_HANDLE_PROBE_ERROR, NULL);
+        req->rpc_err = GARBAGE_ARGS;
+        goto handle_xdr_error;
+    }
+
+    /* ->xl hidden in rpcsvc */
+    svc = rpcsvc_request_service(req);
+    this = svc->xl;
+    priv = this->private;
+    c_clnt = &priv->connections;
+
+    crpc = changelog_rpc_clnt_init(this, &rpc_req, c_clnt);
+    if (!crpc)
+        goto handle_xdr_error;
+
+    changelog_ev_queue_connection(c_clnt, crpc);
+    rpc_rsp.op_ret = 0;
+
+    goto submit_rpc;
+
+handle_xdr_error:
+    rpc_rsp.op_ret = -1;
+submit_rpc:
+    (void)changelog_rpc_sumbit_reply(req, &rpc_rsp, NULL, 0, NULL,
+                                     (xdrproc_t)xdr_changelog_probe_rsp);
+    return 0;
 }
 
 /**
  * RPC declarations
  */
 
-rpcsvc_actor_t changelog_svc_actors[CHANGELOG_RPC_PROC_MAX] = {
-        [CHANGELOG_RPC_PROBE_FILTER] = {
-                "CHANGELOG PROBE FILTER", CHANGELOG_RPC_PROBE_FILTER,
-                changelog_handle_probe, NULL, 0, DRC_NA
-        },
+static rpcsvc_actor_t changelog_svc_actors[CHANGELOG_RPC_PROC_MAX] = {
+    [CHANGELOG_RPC_PROBE_FILTER] = {"CHANGELOG PROBE FILTER",
+                                    changelog_handle_probe, NULL,
+                                    CHANGELOG_RPC_PROBE_FILTER, DRC_NA, 0},
 };
 
-struct rpcsvc_program changelog_svc_prog = {
-        .progname  = CHANGELOG_RPC_PROGNAME,
-        .prognum   = CHANGELOG_RPC_PROGNUM,
-        .progver   = CHANGELOG_RPC_PROGVER,
-        .numactors = CHANGELOG_RPC_PROC_MAX,
-        .actors    = changelog_svc_actors,
-        .synctask  = _gf_true,
+static struct rpcsvc_program changelog_svc_prog = {
+    .progname = CHANGELOG_RPC_PROGNAME,
+    .prognum = CHANGELOG_RPC_PROGNUM,
+    .progver = CHANGELOG_RPC_PROGVER,
+    .numactors = CHANGELOG_RPC_PROC_MAX,
+    .actors = changelog_svc_actors,
+    .synctask = _gf_true,
 };
 
-struct rpcsvc_program *changelog_programs[] = {
-        &changelog_svc_prog,
-        NULL,
+static struct rpcsvc_program *changelog_programs[] = {
+    &changelog_svc_prog,
+    NULL,
 };
diff --git a/xlators/features/changelog/src/changelog-rpc.h b/xlators/features/changelog/src/changelog-rpc.h
index 0df96684b6c..b1707565249 100644
--- a/xlators/features/changelog/src/changelog-rpc.h
+++ b/xlators/features/changelog/src/changelog-rpc.h
@@ -11,19 +11,21 @@
 #ifndef __CHANGELOG_RPC_H
 #define __CHANGELOG_RPC_H
 
-#include "xlator.h"
+#include <glusterfs/xlator.h>
 #include "changelog-helpers.h"
 
 /* one time */
 #include "socket.h"
 #include "changelog-rpc-common.h"
 
-#define CHANGELOG_RPC_PROGNAME  "GlusterFS Changelog"
+#define CHANGELOG_RPC_PROGNAME "GlusterFS Changelog"
 
 rpcsvc_t *
-changelog_init_rpc_listner (xlator_t *, changelog_priv_t *, rbuf_t *, int);
+changelog_init_rpc_listener(xlator_t *, changelog_priv_t *, rbuf_t *, int);
 
 void
-changelog_destroy_rpc_listner (xlator_t *, changelog_priv_t *);
+changelog_destroy_rpc_listner(xlator_t *, changelog_priv_t *);
 
+int
+changelog_cleanup_rpc_threads(xlator_t *this, changelog_priv_t *priv);
 #endif
diff --git a/xlators/features/changelog/src/changelog-rt.c b/xlators/features/changelog/src/changelog-rt.c
index c147f68ca85..841545ae359 100644
--- a/xlators/features/changelog/src/changelog-rt.c
+++ b/xlators/features/changelog/src/changelog-rt.c
@@ -8,65 +8,59 @@
    cases as published by the Free Software Foundation.
 */
 
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
-
-#include "xlator.h"
-#include "defaults.h"
-#include "logging.h"
+#include <glusterfs/xlator.h>
+#include <glusterfs/defaults.h>
+#include <glusterfs/logging.h>
 
 #include "changelog-rt.h"
 #include "changelog-mem-types.h"
 
 int
-changelog_rt_init (xlator_t *this, changelog_dispatcher_t *cd)
+changelog_rt_init(xlator_t *this, changelog_dispatcher_t *cd)
 {
-        changelog_rt_t *crt = NULL;
+    changelog_rt_t *crt = NULL;
 
-        crt = GF_CALLOC (1, sizeof (*crt),
-                         gf_changelog_mt_rt_t);
-        if (!crt)
-                return -1;
+    crt = GF_CALLOC(1, sizeof(*crt), gf_changelog_mt_rt_t);
+    if (!crt)
+        return -1;
 
-        LOCK_INIT (&crt->lock);
+    LOCK_INIT(&crt->lock);
 
-        cd->cd_data = crt;
-        cd->dispatchfn = &changelog_rt_enqueue;
+    cd->cd_data = crt;
+    cd->dispatchfn = &changelog_rt_enqueue;
 
-        return 0;
+    return 0;
 }
 
 int
-changelog_rt_fini (xlator_t *this, changelog_dispatcher_t *cd)
+changelog_rt_fini(xlator_t *this, changelog_dispatcher_t *cd)
 {
-        changelog_rt_t *crt = NULL;
+    changelog_rt_t *crt = NULL;
 
-        crt = cd->cd_data;
+    crt = cd->cd_data;
 
-        LOCK_DESTROY (&crt->lock);
-        GF_FREE (crt);
+    LOCK_DESTROY(&crt->lock);
+    GF_FREE(crt);
 
-        return 0;
+    return 0;
 }
 
 int
-changelog_rt_enqueue (xlator_t *this, changelog_priv_t *priv, void *cbatch,
-                      changelog_log_data_t *cld_0, changelog_log_data_t *cld_1)
+changelog_rt_enqueue(xlator_t *this, changelog_priv_t *priv, void *cbatch,
+                     changelog_log_data_t *cld_0, changelog_log_data_t *cld_1)
 {
-        int             ret = 0;
-        changelog_rt_t *crt = NULL;
+    int ret = 0;
+    changelog_rt_t *crt = NULL;
 
-        crt = (changelog_rt_t *) cbatch;
+    crt = (changelog_rt_t *)cbatch;
 
-        LOCK (&crt->lock);
-        {
-                ret = changelog_handle_change (this, priv, cld_0);
-                if (!ret && cld_1)
-                        ret = changelog_handle_change (this, priv, cld_1);
-        }
-        UNLOCK (&crt->lock);
+    LOCK(&crt->lock);
+    {
+        ret = changelog_handle_change(this, priv, cld_0);
+        if (!ret && cld_1)
+            ret = changelog_handle_change(this, priv, cld_1);
+    }
+    UNLOCK(&crt->lock);
 
-        return ret;
+    return ret;
 }
diff --git a/xlators/features/changelog/src/changelog-rt.h b/xlators/features/changelog/src/changelog-rt.h
index 1fc2bbc5bb9..28b9827d85b 100644
--- a/xlators/features/changelog/src/changelog-rt.h
+++ b/xlators/features/changelog/src/changelog-rt.h
@@ -11,23 +11,23 @@
 #ifndef _CHANGELOG_RT_H
 #define _CHANGELOG_RT_H
 
-#include "locking.h"
-#include "timer.h"
+#include <glusterfs/locking.h>
+#include <glusterfs/timer.h>
 #include "pthread.h"
 
 #include "changelog-helpers.h"
 
 /* unused as of now - may be you would need it later */
 typedef struct changelog_rt {
-        gf_lock_t lock;
+    gf_lock_t lock;
 } changelog_rt_t;
 
 int
-changelog_rt_init (xlator_t *this, changelog_dispatcher_t *cd);
+changelog_rt_init(xlator_t *this, changelog_dispatcher_t *cd);
 int
-changelog_rt_fini (xlator_t *this, changelog_dispatcher_t *cd);
+changelog_rt_fini(xlator_t *this, changelog_dispatcher_t *cd);
 int
-changelog_rt_enqueue (xlator_t *this, changelog_priv_t *priv, void *cbatch,
-                      changelog_log_data_t *cld_0, changelog_log_data_t *cld_1);
+changelog_rt_enqueue(xlator_t *this, changelog_priv_t *priv, void *cbatch,
+                     changelog_log_data_t *cld_0, changelog_log_data_t *cld_1);
 
 #endif /* _CHANGELOG_RT_H */
diff --git a/xlators/features/changelog/src/changelog.c b/xlators/features/changelog/src/changelog.c
index e6553c7f6bb..6a6e5af859e 100644
--- a/xlators/features/changelog/src/changelog.c
+++ b/xlators/features/changelog/src/changelog.c
@@ -8,34 +8,38 @@
    cases as published by the Free Software Foundation.
 */
 
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
-
-#include "xlator.h"
-#include "defaults.h"
-#include "logging.h"
-#include "iobuf.h"
+#include <glusterfs/xlator.h>
+#include <glusterfs/defaults.h>
+#include <glusterfs/syscall.h>
+#include <glusterfs/logging.h>
+#include <glusterfs/iobuf.h>
 
 #include "changelog-rt.h"
 
 #include "changelog-encoders.h"
 #include "changelog-mem-types.h"
+#include "changelog-messages.h"
 
 #include <pthread.h>
+#include <signal.h>
 
 #include "changelog-rpc.h"
-
-static struct changelog_bootstrap
-cb_bootstrap[] = {
-        {
-                .mode = CHANGELOG_MODE_RT,
-                .ctor = changelog_rt_init,
-                .dtor = changelog_rt_fini,
-        },
+#include "errno.h"
+
+static struct changelog_bootstrap cb_bootstrap[] = {
+    {
+        .mode = CHANGELOG_MODE_RT,
+        .ctor = changelog_rt_init,
+        .dtor = changelog_rt_fini,
+    },
 };
 
+static int
+changelog_init_rpc(xlator_t *this, changelog_priv_t *priv);
+
+static int
+changelog_init(xlator_t *this, changelog_priv_t *priv);
+
 /* Entry operations - TYPE III */
 
 /**
@@ -47,1021 +51,1052 @@ cb_bootstrap[] = {
 /* rmdir */
 
 int32_t
-changelog_rmdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                     int32_t op_ret, int32_t op_errno, struct iatt *preparent,
-                     struct iatt *postparent, dict_t *xdata)
+changelog_rmdir_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                    int32_t op_ret, int32_t op_errno, struct iatt *preparent,
+                    struct iatt *postparent, dict_t *xdata)
 {
-        changelog_priv_t  *priv  = NULL;
-        changelog_local_t *local = NULL;
+    changelog_priv_t *priv = NULL;
+    changelog_local_t *local = NULL;
 
-        priv  = this->private;
-        local = frame->local;
+    priv = this->private;
+    local = frame->local;
 
-        CHANGELOG_COND_GOTO (priv, ((op_ret < 0) || !local), unwind);
+    CHANGELOG_COND_GOTO(priv, ((op_ret < 0) || !local), unwind);
 
-        changelog_update (this, priv, local, CHANGELOG_TYPE_ENTRY);
+    changelog_update(this, priv, local, CHANGELOG_TYPE_ENTRY);
 
- unwind:
-        changelog_dec_fop_cnt (this, priv, local);
-        CHANGELOG_STACK_UNWIND (rmdir, frame, op_ret, op_errno,
-                                preparent, postparent, xdata);
-        return 0;
+unwind:
+    changelog_dec_fop_cnt(this, priv, local);
+    CHANGELOG_STACK_UNWIND(rmdir, frame, op_ret, op_errno, preparent,
+                           postparent, xdata);
+    return 0;
 }
 
 int32_t
-changelog_rmdir_resume (call_frame_t *frame, xlator_t *this,
-                        loc_t *loc, int xflags, dict_t *xdata)
+changelog_rmdir_resume(call_frame_t *frame, xlator_t *this, loc_t *loc,
+                       int xflags, dict_t *xdata)
 {
-        changelog_priv_t *priv     = NULL;
+    changelog_priv_t *priv = NULL;
 
-        priv = this->private;
+    priv = this->private;
 
-        gf_log (this->name, GF_LOG_DEBUG, "Dequeue rmdir");
-        changelog_color_fop_and_inc_cnt (this, priv, frame->local);
-        STACK_WIND (frame, changelog_rmdir_cbk,
-                    FIRST_CHILD (this), FIRST_CHILD (this)->fops->rmdir,
-                    loc, xflags, xdata);
-        return 0;
+    gf_msg_debug(this->name, 0, "Dequeue rmdir");
+    changelog_color_fop_and_inc_cnt(this, priv, frame->local);
+    STACK_WIND(frame, changelog_rmdir_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->rmdir, loc, xflags, xdata);
+    return 0;
 }
 
 int32_t
-changelog_rmdir (call_frame_t *frame, xlator_t *this,
-                 loc_t *loc, int xflags, dict_t *xdata)
-{
-        size_t                  xtra_len                = 0;
-        changelog_priv_t       *priv                    = NULL;
-        changelog_opt_t        *co                      = NULL;
-        call_stub_t            *stub                    = NULL;
-        struct list_head        queue                   = {0, };
-        gf_boolean_t            barrier_enabled         = _gf_false;
-
-        INIT_LIST_HEAD (&queue);
-
-        priv = this->private;
-        CHANGELOG_NOT_ACTIVE_THEN_GOTO (frame, priv, wind);
-
-        CHANGELOG_INIT_NOCHECK (this, frame->local,
-                                NULL, loc->inode->gfid, 2);
-
-        co = changelog_get_usable_buffer (frame->local);
-        if (!co)
-                goto wind;
-
-        CHANGLOG_FILL_FOP_NUMBER (co, frame->root->op, fop_fn, xtra_len);
-
-        co++;
-        CHANGELOG_FILL_ENTRY (co, loc->pargfid, loc->name,
-                              entry_fn, entry_free_fn, xtra_len, wind);
-
-        changelog_set_usable_record_and_length (frame->local, xtra_len, 2);
-
-/* changelog barrier */
-        /* Color assignment and increment of fop_cnt for rmdir/unlink/rename
-         * should be made with in priv lock if changelog barrier is not enabled.
-         * Because if counter is not incremented yet, draining wakes up and
-         * publishes the changelog but later these fops might hit the disk and
-         * present in snapped volume but where as the intention is these fops
-         * should not be present in snapped volume.
-         */
-        LOCK (&priv->lock);
-        {
-                if ((barrier_enabled = priv->barrier_enabled)) {
-                        stub = fop_rmdir_stub (frame, changelog_rmdir_resume,
-                                                loc, xflags, xdata);
-                        if (!stub)
-                               __chlog_barrier_disable (this, &queue);
-                        else
-                               __chlog_barrier_enqueue (this, stub);
-                } else {
-                        ((changelog_local_t *)frame->local)->color
-                                                          = priv->current_color;
-                        changelog_inc_fop_cnt (this, priv, frame->local);
-                }
-        }
-        UNLOCK (&priv->lock);
-
-        if (barrier_enabled && stub) {
-                gf_log (this->name, GF_LOG_DEBUG, "Enqueue rmdir");
-                goto out;
-        }
-        if (barrier_enabled && !stub) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "Failed to barrier FOPs, disabling changelog barrier "
-                        "FOP: rmdir, ERROR: %s", strerror (ENOMEM));
-                chlog_barrier_dequeue_all (this, &queue);
+changelog_rmdir(call_frame_t *frame, xlator_t *this, loc_t *loc, int xflags,
+                dict_t *xdata)
+{
+    size_t xtra_len = 0;
+    changelog_priv_t *priv = NULL;
+    changelog_opt_t *co = NULL;
+    call_stub_t *stub = NULL;
+    struct list_head queue = {
+        0,
+    };
+    gf_boolean_t barrier_enabled = _gf_false;
+
+    INIT_LIST_HEAD(&queue);
+
+    priv = this->private;
+    CHANGELOG_NOT_ACTIVE_THEN_GOTO(frame, priv, wind);
+
+    CHANGELOG_INIT_NOCHECK(this, frame->local, NULL, loc->inode->gfid, 2);
+
+    co = changelog_get_usable_buffer(frame->local);
+    if (!co)
+        goto wind;
+
+    CHANGLOG_FILL_FOP_NUMBER(co, frame->root->op, fop_fn, xtra_len);
+
+    co++;
+    if (priv->capture_del_path) {
+        CHANGELOG_FILL_ENTRY_DIR_PATH(co, loc->pargfid, loc->name, del_entry_fn,
+                                      del_entry_free_fn, xtra_len, wind,
+                                      _gf_true);
+    } else {
+        CHANGELOG_FILL_ENTRY_DIR_PATH(co, loc->pargfid, loc->name, del_entry_fn,
+                                      del_entry_free_fn, xtra_len, wind,
+                                      _gf_false);
+    }
+
+    changelog_set_usable_record_and_length(frame->local, xtra_len, 2);
+
+    /* changelog barrier */
+    /* Color assignment and increment of fop_cnt for rmdir/unlink/rename
+     * should be made with in priv lock if changelog barrier is not enabled.
+     * Because if counter is not incremented yet, draining wakes up and
+     * publishes the changelog but later these fops might hit the disk and
+     * present in snapped volume but where as the intention is these fops
+     * should not be present in snapped volume.
+     */
+    LOCK(&priv->lock);
+    {
+        if ((barrier_enabled = priv->barrier_enabled)) {
+            stub = fop_rmdir_stub(frame, changelog_rmdir_resume, loc, xflags,
+                                  xdata);
+            if (!stub)
+                __chlog_barrier_disable(this, &queue);
+            else
+                __chlog_barrier_enqueue(this, stub);
+        } else {
+            ((changelog_local_t *)frame->local)->color = priv->current_color;
+            changelog_inc_fop_cnt(this, priv, frame->local);
         }
-
-/* changelog barrier */
-
- wind:
-        STACK_WIND (frame, changelog_rmdir_cbk,
-                    FIRST_CHILD (this), FIRST_CHILD (this)->fops->rmdir,
-                    loc, xflags, xdata);
- out:
-        return 0;
+    }
+    UNLOCK(&priv->lock);
+
+    if (barrier_enabled && stub) {
+        gf_msg_debug(this->name, 0, "Enqueue rmdir");
+        goto out;
+    }
+    if (barrier_enabled && !stub) {
+        gf_smsg(this->name, GF_LOG_ERROR, ENOMEM,
+                CHANGELOG_MSG_BARRIER_FOP_FAILED, "fop=rmdir", NULL);
+        chlog_barrier_dequeue_all(this, &queue);
+    }
+
+    /* changelog barrier */
+
+wind:
+    STACK_WIND(frame, changelog_rmdir_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->rmdir, loc, xflags, xdata);
+out:
+    return 0;
 }
 
 /* unlink */
 
 int32_t
-changelog_unlink_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                      int32_t op_ret, int32_t op_errno, struct iatt *preparent,
-                      struct iatt *postparent, dict_t *xdata)
+changelog_unlink_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                     int32_t op_ret, int32_t op_errno, struct iatt *preparent,
+                     struct iatt *postparent, dict_t *xdata)
 {
-        changelog_priv_t  *priv  = NULL;
-        changelog_local_t *local = NULL;
+    changelog_priv_t *priv = NULL;
+    changelog_local_t *local = NULL;
 
-        priv  = this->private;
-        local = frame->local;
+    priv = this->private;
+    local = frame->local;
 
-        CHANGELOG_COND_GOTO (priv, ((op_ret < 0) || !local), unwind);
+    CHANGELOG_COND_GOTO(priv, ((op_ret < 0) || !local), unwind);
 
-        changelog_update (this, priv, local, CHANGELOG_TYPE_ENTRY);
+    changelog_update(this, priv, local, CHANGELOG_TYPE_ENTRY);
 
- unwind:
-        changelog_dec_fop_cnt (this, priv, local);
-        CHANGELOG_STACK_UNWIND (unlink, frame, op_ret, op_errno,
-                                preparent, postparent, xdata);
-        return 0;
+unwind:
+    changelog_dec_fop_cnt(this, priv, local);
+    CHANGELOG_STACK_UNWIND(unlink, frame, op_ret, op_errno, preparent,
+                           postparent, xdata);
+    return 0;
 }
 
 int32_t
-changelog_unlink_resume (call_frame_t *frame, xlator_t *this,
-                         loc_t *loc, int xflags, dict_t *xdata)
+changelog_unlink_resume(call_frame_t *frame, xlator_t *this, loc_t *loc,
+                        int xflags, dict_t *xdata)
 {
-        changelog_priv_t *priv     = NULL;
+    changelog_priv_t *priv = NULL;
 
-        priv = this->private;
+    priv = this->private;
 
-        gf_log (this->name, GF_LOG_DEBUG, "Dequeue unlink");
-        changelog_color_fop_and_inc_cnt (this, priv, frame->local);
-        STACK_WIND (frame, changelog_unlink_cbk,
-                    FIRST_CHILD (this), FIRST_CHILD (this)->fops->unlink,
-                    loc, xflags, xdata);
-        return 0;
+    gf_msg_debug(this->name, 0, "Dequeue unlink");
+    changelog_color_fop_and_inc_cnt(this, priv, frame->local);
+    STACK_WIND(frame, changelog_unlink_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->unlink, loc, xflags, xdata);
+    return 0;
 }
 
 int32_t
-changelog_unlink (call_frame_t *frame, xlator_t *this,
-                  loc_t *loc, int xflags, dict_t *xdata)
-{
-        size_t                  xtra_len                = 0;
-        changelog_priv_t       *priv                    = NULL;
-        changelog_opt_t        *co                      = NULL;
-        call_stub_t            *stub                    = NULL;
-        struct list_head        queue                   = {0, };
-        gf_boolean_t            barrier_enabled         = _gf_false;
+changelog_unlink(call_frame_t *frame, xlator_t *this, loc_t *loc, int xflags,
+                 dict_t *xdata)
+{
+    size_t xtra_len = 0;
+    changelog_priv_t *priv = NULL;
+    changelog_opt_t *co = NULL;
+    call_stub_t *stub = NULL;
+    struct list_head queue = {
+        0,
+    };
+    gf_boolean_t barrier_enabled = _gf_false;
+    dht_changelog_rename_info_t *info = NULL;
+    int ret = 0;
+    char *old_name = NULL;
+    char *new_name = NULL;
+    char *nname = NULL;
+
+    INIT_LIST_HEAD(&queue);
+    priv = this->private;
+
+    CHANGELOG_NOT_ACTIVE_THEN_GOTO(frame, priv, wind);
+
+    ret = dict_get_bin(xdata, DHT_CHANGELOG_RENAME_OP_KEY, (void **)&info);
+    if (!ret) { /* special case: unlink considered as rename */
+        /* 3 == fop + oldloc + newloc */
+        old_name = alloca(info->oldname_len);
+        new_name = alloca(info->newname_len);
+        CHANGELOG_INIT_NOCHECK(this, frame->local, NULL, loc->inode->gfid, 3);
 
-        INIT_LIST_HEAD (&queue);
+        co = changelog_get_usable_buffer(frame->local);
+        if (!co)
+            goto wind;
 
-        priv = this->private;
-        CHANGELOG_NOT_ACTIVE_THEN_GOTO (frame, priv, wind);
-        CHANGELOG_IF_INTERNAL_FOP_THEN_GOTO (frame, xdata, wind);
+        CHANGLOG_FILL_FOP_NUMBER(co, GF_FOP_RENAME, fop_fn, xtra_len);
 
-        CHANGELOG_INIT_NOCHECK (this, frame->local, NULL, loc->inode->gfid, 2);
+        co++;
+        strncpy(old_name, info->buffer, info->oldname_len);
+        CHANGELOG_FILL_ENTRY(co, info->old_pargfid, old_name, entry_fn,
+                             entry_free_fn, xtra_len, wind);
 
-        co = changelog_get_usable_buffer (frame->local);
+        co++;
+        /* new name resides just after old name */
+        nname = info->buffer + info->oldname_len;
+        strncpy(new_name, nname, info->newname_len);
+        CHANGELOG_FILL_ENTRY(co, info->new_pargfid, new_name, entry_fn,
+                             entry_free_fn, xtra_len, wind);
+
+        changelog_set_usable_record_and_length(frame->local, xtra_len, 3);
+    } else { /* default unlink */
+        CHANGELOG_IF_INTERNAL_FOP_THEN_GOTO(frame, xdata, wind);
+        CHANGELOG_INIT_NOCHECK(this, frame->local, NULL, loc->inode->gfid, 2);
+
+        co = changelog_get_usable_buffer(frame->local);
         if (!co)
-                goto wind;
+            goto wind;
 
-        CHANGLOG_FILL_FOP_NUMBER (co, frame->root->op, fop_fn, xtra_len);
+        CHANGLOG_FILL_FOP_NUMBER(co, frame->root->op, fop_fn, xtra_len);
 
         co++;
-        CHANGELOG_FILL_ENTRY (co, loc->pargfid, loc->name,
-                              entry_fn, entry_free_fn, xtra_len, wind);
-
-        changelog_set_usable_record_and_length (frame->local, xtra_len, 2);
-
-/* changelog barrier */
-        LOCK (&priv->lock);
-        {
-                if ((barrier_enabled = priv->barrier_enabled)) {
-                        stub = fop_unlink_stub (frame, changelog_unlink_resume,
-                                                loc, xflags, xdata);
-                        if (!stub)
-                               __chlog_barrier_disable (this, &queue);
-                        else
-                               __chlog_barrier_enqueue (this, stub);
-                } else {
-                        ((changelog_local_t *)frame->local)->color
-                                                          = priv->current_color;
-                        changelog_inc_fop_cnt (this, priv, frame->local);
-                }
+        if (priv->capture_del_path) {
+            CHANGELOG_FILL_ENTRY_DIR_PATH(co, loc->pargfid, loc->name,
+                                          del_entry_fn, del_entry_free_fn,
+                                          xtra_len, wind, _gf_true);
+        } else {
+            CHANGELOG_FILL_ENTRY_DIR_PATH(co, loc->pargfid, loc->name,
+                                          del_entry_fn, del_entry_free_fn,
+                                          xtra_len, wind, _gf_false);
         }
-        UNLOCK (&priv->lock);
 
-        if (barrier_enabled && stub) {
-                gf_log (this->name, GF_LOG_DEBUG, "Enqueue unlink");
-                goto out;
-        }
-        if (barrier_enabled && !stub) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "Failed to barrier FOPs, disabling changelog barrier "
-                        "FOP: unlink, ERROR: %s", strerror (ENOMEM));
-                chlog_barrier_dequeue_all (this, &queue);
+        changelog_set_usable_record_and_length(frame->local, xtra_len, 2);
+    }
+
+    /* changelog barrier */
+    LOCK(&priv->lock);
+    {
+        if ((barrier_enabled = priv->barrier_enabled)) {
+            stub = fop_unlink_stub(frame, changelog_unlink_resume, loc, xflags,
+                                   xdata);
+            if (!stub)
+                __chlog_barrier_disable(this, &queue);
+            else
+                __chlog_barrier_enqueue(this, stub);
+        } else {
+            ((changelog_local_t *)frame->local)->color = priv->current_color;
+            changelog_inc_fop_cnt(this, priv, frame->local);
         }
-
-/* changelog barrier */
-
- wind:
-        STACK_WIND (frame, changelog_unlink_cbk,
-                    FIRST_CHILD (this), FIRST_CHILD (this)->fops->unlink,
-                    loc, xflags, xdata);
- out:
-        return 0;
+    }
+    UNLOCK(&priv->lock);
+
+    if (barrier_enabled && stub) {
+        gf_msg_debug(this->name, 0, "Enqueue unlink");
+        goto out;
+    }
+    if (barrier_enabled && !stub) {
+        gf_smsg(this->name, GF_LOG_ERROR, ENOMEM,
+                CHANGELOG_MSG_BARRIER_FOP_FAILED, "fop=unlink", NULL);
+        chlog_barrier_dequeue_all(this, &queue);
+    }
+
+    /* changelog barrier */
+
+wind:
+    STACK_WIND(frame, changelog_unlink_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->unlink, loc, xflags, xdata);
+out:
+    return 0;
 }
 
 /* rename */
 
 int32_t
-changelog_rename_cbk (call_frame_t *frame,
-                      void *cookie, xlator_t *this,
-                      int32_t op_ret, int32_t op_errno,
-                      struct iatt *buf, struct iatt *preoldparent,
-                      struct iatt *postoldparent, struct iatt *prenewparent,
-                      struct iatt *postnewparent, dict_t *xdata)
-{
-        changelog_priv_t  *priv  = NULL;
-        changelog_local_t *local = NULL;
-
-        priv  = this->private;
-        local = frame->local;
-
-        CHANGELOG_COND_GOTO (priv, ((op_ret < 0) || !local), unwind);
-
-        changelog_update (this, priv, local, CHANGELOG_TYPE_ENTRY);
-
- unwind:
-        changelog_dec_fop_cnt (this, priv, local);
-        CHANGELOG_STACK_UNWIND (rename, frame, op_ret, op_errno,
-                                buf, preoldparent, postoldparent,
-                                prenewparent, postnewparent, xdata);
-        return 0;
+changelog_rename_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                     int32_t op_ret, int32_t op_errno, struct iatt *buf,
+                     struct iatt *preoldparent, struct iatt *postoldparent,
+                     struct iatt *prenewparent, struct iatt *postnewparent,
+                     dict_t *xdata)
+{
+    changelog_priv_t *priv = NULL;
+    changelog_local_t *local = NULL;
+
+    priv = this->private;
+    local = frame->local;
+    CHANGELOG_COND_GOTO(priv, ((op_ret < 0) || !local), unwind);
+    changelog_update(this, priv, local, CHANGELOG_TYPE_ENTRY);
+unwind:
+    changelog_dec_fop_cnt(this, priv, local);
+    CHANGELOG_STACK_UNWIND(rename, frame, op_ret, op_errno, buf, preoldparent,
+                           postoldparent, prenewparent, postnewparent, xdata);
+    return 0;
 }
 
-
 int32_t
-changelog_rename_resume (call_frame_t *frame, xlator_t *this,
-                         loc_t *oldloc, loc_t *newloc, dict_t *xdata)
+changelog_rename_resume(call_frame_t *frame, xlator_t *this, loc_t *oldloc,
+                        loc_t *newloc, dict_t *xdata)
 {
-        changelog_priv_t *priv     = NULL;
+    changelog_priv_t *priv = NULL;
 
-        priv = this->private;
+    priv = this->private;
 
-        gf_log (this->name, GF_LOG_DEBUG, "Dequeue rename");
-        changelog_color_fop_and_inc_cnt (this, priv, frame->local);
-        STACK_WIND (frame, changelog_rename_cbk,
-                    FIRST_CHILD (this), FIRST_CHILD (this)->fops->rename,
-                    oldloc, newloc, xdata);
-        return 0;
+    gf_msg_debug(this->name, 0, "Dequeue rename");
+    changelog_color_fop_and_inc_cnt(this, priv, frame->local);
+    STACK_WIND(frame, changelog_rename_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->rename, oldloc, newloc, xdata);
+    return 0;
 }
 
 int32_t
-changelog_rename (call_frame_t *frame, xlator_t *this,
-                  loc_t *oldloc, loc_t *newloc, dict_t *xdata)
-{
-        size_t                  xtra_len                = 0;
-        changelog_priv_t       *priv                    = NULL;
-        changelog_opt_t        *co                      = NULL;
-        call_stub_t            *stub                    = NULL;
-        struct list_head        queue                   = {0, };
-        gf_boolean_t            barrier_enabled         = _gf_false;
-
-        INIT_LIST_HEAD (&queue);
-
-        priv = this->private;
-        CHANGELOG_NOT_ACTIVE_THEN_GOTO (frame, priv, wind);
-
-        /* 3 == fop + oldloc + newloc */
-        CHANGELOG_INIT_NOCHECK (this, frame->local,
-                                NULL, oldloc->inode->gfid, 3);
-
-        co = changelog_get_usable_buffer (frame->local);
-        if (!co)
-                goto wind;
-
-        CHANGLOG_FILL_FOP_NUMBER (co, frame->root->op, fop_fn, xtra_len);
-
-        co++;
-        CHANGELOG_FILL_ENTRY (co, oldloc->pargfid, oldloc->name,
-                              entry_fn, entry_free_fn, xtra_len, wind);
-
-        co++;
-        CHANGELOG_FILL_ENTRY (co, newloc->pargfid, newloc->name,
-                              entry_fn, entry_free_fn, xtra_len, wind);
-
-        changelog_set_usable_record_and_length (frame->local, xtra_len, 3);
-/* changelog barrier */
-        LOCK (&priv->lock);
-        {
-                if ((barrier_enabled = priv->barrier_enabled)) {
-                        stub = fop_rename_stub (frame, changelog_rename_resume,
-                                                oldloc, newloc, xdata);
-                        if (!stub)
-                               __chlog_barrier_disable (this, &queue);
-                        else
-                               __chlog_barrier_enqueue (this, stub);
-                } else {
-                        ((changelog_local_t *)frame->local)->color
-                                                          = priv->current_color;
-                        changelog_inc_fop_cnt (this, priv, frame->local);
-                }
-        }
-        UNLOCK (&priv->lock);
-
-        if (barrier_enabled && stub) {
-                gf_log (this->name, GF_LOG_DEBUG, "Enqueue rename");
-                goto out;
-        }
-        if (barrier_enabled && !stub) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "Failed to barrier FOPs, disabling changelog barrier "
-                        "FOP: rename, ERROR: %s", strerror (ENOMEM));
-                chlog_barrier_dequeue_all (this, &queue);
+changelog_rename(call_frame_t *frame, xlator_t *this, loc_t *oldloc,
+                 loc_t *newloc, dict_t *xdata)
+{
+    size_t xtra_len = 0;
+    changelog_priv_t *priv = NULL;
+    changelog_opt_t *co = NULL;
+    call_stub_t *stub = NULL;
+    struct list_head queue = {
+        0,
+    };
+    gf_boolean_t barrier_enabled = _gf_false;
+    dht_changelog_rename_info_t *info = NULL;
+    int ret = 0;
+
+    INIT_LIST_HEAD(&queue);
+
+    priv = this->private;
+    CHANGELOG_NOT_ACTIVE_THEN_GOTO(frame, priv, wind);
+
+    ret = dict_get_bin(xdata, DHT_CHANGELOG_RENAME_OP_KEY, (void **)&info);
+    if (ret && oldloc->inode->ia_type != IA_IFDIR) {
+        /* xdata "NOT" set for a non-directory,
+         * Special rename => avoid logging */
+        goto wind;
+    }
+
+    /* 3 == fop + oldloc + newloc */
+    CHANGELOG_INIT_NOCHECK(this, frame->local, NULL, oldloc->inode->gfid, 3);
+
+    co = changelog_get_usable_buffer(frame->local);
+    if (!co)
+        goto wind;
+
+    CHANGLOG_FILL_FOP_NUMBER(co, frame->root->op, fop_fn, xtra_len);
+
+    co++;
+    CHANGELOG_FILL_ENTRY(co, oldloc->pargfid, oldloc->name, entry_fn,
+                         entry_free_fn, xtra_len, wind);
+
+    co++;
+    CHANGELOG_FILL_ENTRY(co, newloc->pargfid, newloc->name, entry_fn,
+                         entry_free_fn, xtra_len, wind);
+
+    changelog_set_usable_record_and_length(frame->local, xtra_len, 3);
+    /* changelog barrier */
+    LOCK(&priv->lock);
+    {
+        if ((barrier_enabled = priv->barrier_enabled)) {
+            stub = fop_rename_stub(frame, changelog_rename_resume, oldloc,
+                                   newloc, xdata);
+            if (!stub)
+                __chlog_barrier_disable(this, &queue);
+            else
+                __chlog_barrier_enqueue(this, stub);
+        } else {
+            ((changelog_local_t *)frame->local)->color = priv->current_color;
+            changelog_inc_fop_cnt(this, priv, frame->local);
         }
-/* changelog barrier */
-
- wind:
-        STACK_WIND (frame, changelog_rename_cbk,
-                    FIRST_CHILD (this), FIRST_CHILD (this)->fops->rename,
-                    oldloc, newloc, xdata);
- out:
-        return 0;
+    }
+    UNLOCK(&priv->lock);
+
+    if (barrier_enabled && stub) {
+        gf_msg_debug(this->name, 0, "Enqueue rename");
+        goto out;
+    }
+    if (barrier_enabled && !stub) {
+        gf_smsg(this->name, GF_LOG_ERROR, ENOMEM,
+                CHANGELOG_MSG_BARRIER_FOP_FAILED, "fop=rename", NULL);
+        chlog_barrier_dequeue_all(this, &queue);
+    }
+    /* changelog barrier */
+
+wind:
+    STACK_WIND(frame, changelog_rename_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->rename, oldloc, newloc, xdata);
+out:
+    return 0;
 }
 
 /* link */
 
 int32_t
-changelog_link_cbk (call_frame_t *frame,
-                    void *cookie, xlator_t *this, int32_t op_ret,
-                    int32_t op_errno, inode_t *inode,
-                    struct iatt *buf, struct iatt *preparent,
-                    struct iatt *postparent, dict_t *xdata)
+changelog_link_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                   int32_t op_ret, int32_t op_errno, inode_t *inode,
+                   struct iatt *buf, struct iatt *preparent,
+                   struct iatt *postparent, dict_t *xdata)
 {
-        changelog_priv_t  *priv  = NULL;
-        changelog_local_t *local = NULL;
+    changelog_priv_t *priv = NULL;
+    changelog_local_t *local = NULL;
 
-        priv  = this->private;
-        local = frame->local;
+    priv = this->private;
+    local = frame->local;
 
-        CHANGELOG_COND_GOTO (priv, ((op_ret < 0) || !local), unwind);
+    CHANGELOG_COND_GOTO(priv, ((op_ret < 0) || !local), unwind);
 
-        changelog_update (this, priv, local, CHANGELOG_TYPE_ENTRY);
+    changelog_update(this, priv, local, CHANGELOG_TYPE_ENTRY);
 
- unwind:
-        changelog_dec_fop_cnt (this, priv, local);
-        CHANGELOG_STACK_UNWIND (link, frame, op_ret, op_errno,
-                                inode, buf, preparent, postparent, xdata);
-        return 0;
+unwind:
+    changelog_dec_fop_cnt(this, priv, local);
+    CHANGELOG_STACK_UNWIND(link, frame, op_ret, op_errno, inode, buf, preparent,
+                           postparent, xdata);
+    return 0;
 }
 
 int32_t
-changelog_link_resume (call_frame_t *frame, xlator_t *this,
-                        loc_t *oldloc, loc_t *newloc, dict_t *xdata)
+changelog_link_resume(call_frame_t *frame, xlator_t *this, loc_t *oldloc,
+                      loc_t *newloc, dict_t *xdata)
 {
-        changelog_priv_t *priv     = NULL;
+    changelog_priv_t *priv = NULL;
 
-        GF_VALIDATE_OR_GOTO ("changelog", this, out);
-        GF_VALIDATE_OR_GOTO ("changelog", this->fops, out);
-        GF_VALIDATE_OR_GOTO ("changelog", frame, out);
+    GF_VALIDATE_OR_GOTO("changelog", this, out);
+    GF_VALIDATE_OR_GOTO("changelog", this->fops, out);
+    GF_VALIDATE_OR_GOTO("changelog", frame, out);
 
-        priv = this->private;
+    priv = this->private;
 
-        gf_log (this->name, GF_LOG_DEBUG, "Dequeuing link");
-        changelog_color_fop_and_inc_cnt (this, priv, frame->local);
-        STACK_WIND (frame, changelog_link_cbk,
-                    FIRST_CHILD (this), FIRST_CHILD (this)->fops->link,
-                    oldloc, newloc, xdata);
-        return 0;
+    gf_msg_debug(this->name, 0, "Dequeuing link");
+    changelog_color_fop_and_inc_cnt(this, priv, frame->local);
+    STACK_WIND(frame, changelog_link_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->link, oldloc, newloc, xdata);
+    return 0;
 out:
-        return -1;
+    return -1;
 }
 int32_t
-changelog_link (call_frame_t *frame,
-                xlator_t *this, loc_t *oldloc,
-                loc_t *newloc, dict_t *xdata)
-{
-        size_t            xtra_len         = 0;
-        changelog_priv_t *priv             = NULL;
-        changelog_opt_t  *co               = NULL;
-        call_stub_t      *stub             = NULL;
-        struct list_head queue             = {0, };
-        gf_boolean_t     barrier_enabled   = _gf_false;
-
-        priv = this->private;
-
-        CHANGELOG_NOT_ACTIVE_THEN_GOTO (frame, priv, wind);
-        CHANGELOG_IF_INTERNAL_FOP_THEN_GOTO (frame, xdata, wind);
-
-        CHANGELOG_INIT_NOCHECK (this, frame->local, NULL, oldloc->gfid, 2);
-
-        co = changelog_get_usable_buffer (frame->local);
-        if (!co)
-                goto wind;
-
-        CHANGLOG_FILL_FOP_NUMBER (co, frame->root->op, fop_fn, xtra_len);
-
-        co++;
-        CHANGELOG_FILL_ENTRY (co, newloc->pargfid, newloc->name,
-                              entry_fn, entry_free_fn, xtra_len, wind);
-
-        changelog_set_usable_record_and_length (frame->local, xtra_len, 2);
-
-        LOCK (&priv->lock);
-        {
-                if ((barrier_enabled = priv->barrier_enabled)) {
-                        stub = fop_link_stub (frame, changelog_link_resume,
-                                               oldloc, newloc, xdata);
-                        if (!stub)
-                               __chlog_barrier_disable (this, &queue);
-                        else
-                               __chlog_barrier_enqueue (this, stub);
-                } else {
-                        ((changelog_local_t *)frame->local)->color
-                                                          = priv->current_color;
-                        changelog_inc_fop_cnt (this, priv, frame->local);
-                }
-        }
-        UNLOCK (&priv->lock);
-
-        if (barrier_enabled && stub) {
-                gf_log (this->name, GF_LOG_DEBUG, "Enqueued link");
-                goto out;
-        }
-
-        if (barrier_enabled && !stub) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "Failed to barrier FOPs, disabling changelog barrier "
-                        "FOP: link, ERROR: %s", strerror (ENOMEM));
-                chlog_barrier_dequeue_all (this, &queue);
+changelog_link(call_frame_t *frame, xlator_t *this, loc_t *oldloc,
+               loc_t *newloc, dict_t *xdata)
+{
+    size_t xtra_len = 0;
+    changelog_priv_t *priv = NULL;
+    changelog_opt_t *co = NULL;
+    call_stub_t *stub = NULL;
+    struct list_head queue = {
+        0,
+    };
+    gf_boolean_t barrier_enabled = _gf_false;
+
+    priv = this->private;
+
+    CHANGELOG_NOT_ACTIVE_THEN_GOTO(frame, priv, wind);
+    CHANGELOG_IF_INTERNAL_FOP_THEN_GOTO(frame, xdata, wind);
+
+    CHANGELOG_INIT_NOCHECK(this, frame->local, NULL, oldloc->gfid, 2);
+
+    co = changelog_get_usable_buffer(frame->local);
+    if (!co)
+        goto wind;
+
+    CHANGLOG_FILL_FOP_NUMBER(co, frame->root->op, fop_fn, xtra_len);
+
+    co++;
+    CHANGELOG_FILL_ENTRY(co, newloc->pargfid, newloc->name, entry_fn,
+                         entry_free_fn, xtra_len, wind);
+
+    changelog_set_usable_record_and_length(frame->local, xtra_len, 2);
+
+    LOCK(&priv->lock);
+    {
+        if ((barrier_enabled = priv->barrier_enabled)) {
+            stub = fop_link_stub(frame, changelog_link_resume, oldloc, newloc,
+                                 xdata);
+            if (!stub)
+                __chlog_barrier_disable(this, &queue);
+            else
+                __chlog_barrier_enqueue(this, stub);
+        } else {
+            ((changelog_local_t *)frame->local)->color = priv->current_color;
+            changelog_inc_fop_cnt(this, priv, frame->local);
         }
- wind:
-        STACK_WIND (frame, changelog_link_cbk,
-                    FIRST_CHILD (this), FIRST_CHILD (this)->fops->link,
-                    oldloc, newloc, xdata);
+    }
+    UNLOCK(&priv->lock);
+
+    if (barrier_enabled && stub) {
+        gf_msg_debug(this->name, 0, "Enqueued link");
+        goto out;
+    }
+
+    if (barrier_enabled && !stub) {
+        gf_smsg(this->name, GF_LOG_ERROR, 0, CHANGELOG_MSG_BARRIER_FOP_FAILED,
+                "fop=link", NULL);
+        chlog_barrier_dequeue_all(this, &queue);
+    }
+wind:
+    STACK_WIND(frame, changelog_link_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->link, oldloc, newloc, xdata);
 out:
-        return 0;
+    return 0;
 }
 
 /* mkdir */
 
 int32_t
-changelog_mkdir_cbk (call_frame_t *frame,
-                     void *cookie, xlator_t *this, int32_t op_ret,
-                     int32_t op_errno, inode_t *inode,
-                     struct iatt *buf, struct iatt *preparent,
-                     struct iatt *postparent, dict_t *xdata)
+changelog_mkdir_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                    int32_t op_ret, int32_t op_errno, inode_t *inode,
+                    struct iatt *buf, struct iatt *preparent,
+                    struct iatt *postparent, dict_t *xdata)
 {
-        changelog_priv_t  *priv  = NULL;
-        changelog_local_t *local = NULL;
+    changelog_priv_t *priv = NULL;
+    changelog_local_t *local = NULL;
 
-        priv  = this->private;
-        local = frame->local;
+    priv = this->private;
+    local = frame->local;
 
-        CHANGELOG_COND_GOTO (priv, ((op_ret < 0) || !local), unwind);
+    CHANGELOG_COND_GOTO(priv, ((op_ret < 0) || !local), unwind);
 
-        changelog_update (this, priv, local, CHANGELOG_TYPE_ENTRY);
+    changelog_update(this, priv, local, CHANGELOG_TYPE_ENTRY);
 
- unwind:
-        changelog_dec_fop_cnt (this, priv, local);
-        CHANGELOG_STACK_UNWIND (mkdir, frame, op_ret, op_errno,
-                                inode, buf, preparent, postparent, xdata);
-        return 0;
+unwind:
+    changelog_dec_fop_cnt(this, priv, local);
+    CHANGELOG_STACK_UNWIND(mkdir, frame, op_ret, op_errno, inode, buf,
+                           preparent, postparent, xdata);
+    return 0;
 }
 
 int32_t
-changelog_mkdir_resume (call_frame_t *frame, xlator_t *this,
-                        loc_t *loc, mode_t mode,
-                        mode_t umask, dict_t *xdata)
+changelog_mkdir_resume(call_frame_t *frame, xlator_t *this, loc_t *loc,
+                       mode_t mode, mode_t umask, dict_t *xdata)
 {
-        changelog_priv_t *priv     = NULL;
+    changelog_priv_t *priv = NULL;
 
-        GF_VALIDATE_OR_GOTO ("changelog", this, out);
-        GF_VALIDATE_OR_GOTO ("changelog", this->fops, out);
-        GF_VALIDATE_OR_GOTO ("changelog", frame, out);
+    GF_VALIDATE_OR_GOTO("changelog", this, out);
+    GF_VALIDATE_OR_GOTO("changelog", this->fops, out);
+    GF_VALIDATE_OR_GOTO("changelog", frame, out);
 
-        priv = this->private;
+    priv = this->private;
 
-        gf_log (this->name, GF_LOG_DEBUG, "Dequeuing mkdir");
-        changelog_color_fop_and_inc_cnt (this, priv, frame->local);
-        STACK_WIND (frame, changelog_mkdir_cbk,
-                    FIRST_CHILD (this), FIRST_CHILD (this)->fops->mkdir,
-                    loc, mode, umask, xdata);
-        return 0;
+    gf_msg_debug(this->name, 0, "Dequeuing mkdir");
+    changelog_color_fop_and_inc_cnt(this, priv, frame->local);
+    STACK_WIND(frame, changelog_mkdir_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->mkdir, loc, mode, umask, xdata);
+    return 0;
 out:
-        return -1;
+    return -1;
 }
 
 int32_t
-changelog_mkdir (call_frame_t *frame, xlator_t *this,
-                 loc_t *loc, mode_t mode, mode_t umask, dict_t *xdata)
-{
-        int               ret              = -1;
-        uuid_t            gfid             = {0,};
-        void             *uuid_req         = NULL;
-        size_t            xtra_len         = 0;
-        changelog_priv_t *priv             = NULL;
-        changelog_opt_t  *co               = NULL;
-        call_stub_t      *stub             = NULL;
-        struct list_head queue             = {0, };
-        gf_boolean_t     barrier_enabled   = _gf_false;
-
-        priv = this->private;
-        CHANGELOG_NOT_ACTIVE_THEN_GOTO (frame, priv, wind);
-
-        ret = dict_get_ptr (xdata, "gfid-req", &uuid_req);
-        if (ret) {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "failed to get gfid from dict");
-                goto wind;
-        }
-        gf_uuid_copy (gfid, uuid_req);
-
-        CHANGELOG_INIT_NOCHECK (this, frame->local, NULL, gfid, 5);
-
-        co = changelog_get_usable_buffer (frame->local);
-        if (!co)
-                goto wind;
-
-        CHANGLOG_FILL_FOP_NUMBER (co, frame->root->op, fop_fn, xtra_len);
-        co++;
-
-        CHANGELOG_FILL_UINT32 (co, S_IFDIR | mode, number_fn, xtra_len);
-        co++;
-
-        CHANGELOG_FILL_UINT32 (co, frame->root->uid, number_fn, xtra_len);
-        co++;
-
-        CHANGELOG_FILL_UINT32 (co, frame->root->gid, number_fn, xtra_len);
-        co++;
-
-        CHANGELOG_FILL_ENTRY (co, loc->pargfid, loc->name,
-                              entry_fn, entry_free_fn, xtra_len, wind);
-
-        changelog_set_usable_record_and_length (frame->local, xtra_len, 5);
-
-        LOCK (&priv->lock);
-        {
-                if ((barrier_enabled = priv->barrier_enabled)) {
-                        stub = fop_mkdir_stub (frame, changelog_mkdir_resume,
-                                               loc, mode, umask, xdata);
-                        if (!stub)
-                               __chlog_barrier_disable (this, &queue);
-                        else
-                               __chlog_barrier_enqueue (this, stub);
-                } else {
-                        ((changelog_local_t *)frame->local)->color
-                                                          = priv->current_color;
-                        changelog_inc_fop_cnt (this, priv, frame->local);
-                }
-        }
-        UNLOCK (&priv->lock);
-
-        if (barrier_enabled && stub) {
-                gf_log (this->name, GF_LOG_DEBUG, "Enqueued mkdir");
-                goto out;
-        }
-
-        if (barrier_enabled && !stub) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "Failed to barrier FOPs, disabling changelog barrier "
-                        "FOP: mkdir, ERROR: %s", strerror (ENOMEM));
-                chlog_barrier_dequeue_all (this, &queue);
+changelog_mkdir(call_frame_t *frame, xlator_t *this, loc_t *loc, mode_t mode,
+                mode_t umask, dict_t *xdata)
+{
+    int ret = -1;
+    uuid_t gfid = {
+        0,
+    };
+    size_t xtra_len = 0;
+    changelog_priv_t *priv = NULL;
+    changelog_opt_t *co = NULL;
+    call_stub_t *stub = NULL;
+    struct list_head queue = {
+        0,
+    };
+    gf_boolean_t barrier_enabled = _gf_false;
+
+    priv = this->private;
+    CHANGELOG_NOT_ACTIVE_THEN_GOTO(frame, priv, wind);
+
+    ret = dict_get_gfuuid(xdata, "gfid-req", &gfid);
+    if (ret) {
+        gf_msg_debug(this->name, 0, "failed to get gfid from dict");
+        goto wind;
+    }
+
+    CHANGELOG_INIT_NOCHECK(this, frame->local, NULL, gfid, 5);
+
+    co = changelog_get_usable_buffer(frame->local);
+    if (!co)
+        goto wind;
+
+    CHANGLOG_FILL_FOP_NUMBER(co, frame->root->op, fop_fn, xtra_len);
+    co++;
+
+    CHANGELOG_FILL_UINT32(co, S_IFDIR | mode, number_fn, xtra_len);
+    co++;
+
+    CHANGELOG_FILL_UINT32(co, frame->root->uid, number_fn, xtra_len);
+    co++;
+
+    CHANGELOG_FILL_UINT32(co, frame->root->gid, number_fn, xtra_len);
+    co++;
+
+    CHANGELOG_FILL_ENTRY(co, loc->pargfid, loc->name, entry_fn, entry_free_fn,
+                         xtra_len, wind);
+
+    changelog_set_usable_record_and_length(frame->local, xtra_len, 5);
+
+    LOCK(&priv->lock);
+    {
+        if ((barrier_enabled = priv->barrier_enabled)) {
+            stub = fop_mkdir_stub(frame, changelog_mkdir_resume, loc, mode,
+                                  umask, xdata);
+            if (!stub)
+                __chlog_barrier_disable(this, &queue);
+            else
+                __chlog_barrier_enqueue(this, stub);
+        } else {
+            ((changelog_local_t *)frame->local)->color = priv->current_color;
+            changelog_inc_fop_cnt(this, priv, frame->local);
         }
-
- wind:
-        STACK_WIND (frame, changelog_mkdir_cbk,
-                    FIRST_CHILD (this), FIRST_CHILD (this)->fops->mkdir,
-                    loc, mode, umask, xdata);
+    }
+    UNLOCK(&priv->lock);
+
+    if (barrier_enabled && stub) {
+        gf_msg_debug(this->name, 0, "Enqueued mkdir");
+        goto out;
+    }
+
+    if (barrier_enabled && !stub) {
+        gf_smsg(this->name, GF_LOG_ERROR, ENOMEM,
+                CHANGELOG_MSG_BARRIER_FOP_FAILED, "fop=mkdir", NULL);
+        chlog_barrier_dequeue_all(this, &queue);
+    }
+
+wind:
+    STACK_WIND(frame, changelog_mkdir_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->mkdir, loc, mode, umask, xdata);
 out:
-        return 0;
+    return 0;
 }
 
 /* symlink */
 
 int32_t
-changelog_symlink_cbk (call_frame_t *frame,
-                       void *cookie, xlator_t *this,
-                       int32_t op_ret, int32_t op_errno,
-                       inode_t *inode, struct iatt *buf, struct iatt *preparent,
-                       struct iatt *postparent, dict_t *xdata)
+changelog_symlink_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                      int32_t op_ret, int32_t op_errno, inode_t *inode,
+                      struct iatt *buf, struct iatt *preparent,
+                      struct iatt *postparent, dict_t *xdata)
 {
-        changelog_priv_t  *priv  = NULL;
-        changelog_local_t *local = NULL;
+    changelog_priv_t *priv = NULL;
+    changelog_local_t *local = NULL;
 
-        priv  = this->private;
-        local = frame->local;
+    priv = this->private;
+    local = frame->local;
 
-        CHANGELOG_COND_GOTO (priv, ((op_ret < 0) || !local), unwind);
+    CHANGELOG_COND_GOTO(priv, ((op_ret < 0) || !local), unwind);
 
-        changelog_update (this, priv, local, CHANGELOG_TYPE_ENTRY);
+    changelog_update(this, priv, local, CHANGELOG_TYPE_ENTRY);
 
- unwind:
-        changelog_dec_fop_cnt (this, priv, local);
-        CHANGELOG_STACK_UNWIND (symlink, frame, op_ret, op_errno,
-                                inode, buf, preparent, postparent, xdata);
-        return 0;
+unwind:
+    changelog_dec_fop_cnt(this, priv, local);
+    CHANGELOG_STACK_UNWIND(symlink, frame, op_ret, op_errno, inode, buf,
+                           preparent, postparent, xdata);
+    return 0;
 }
 
-
 int32_t
-changelog_symlink_resume (call_frame_t *frame, xlator_t *this,
-                          const char *linkname, loc_t *loc,
-                          mode_t umask, dict_t *xdata)
+changelog_symlink_resume(call_frame_t *frame, xlator_t *this,
+                         const char *linkname, loc_t *loc, mode_t umask,
+                         dict_t *xdata)
 {
-        changelog_priv_t *priv     = NULL;
+    changelog_priv_t *priv = NULL;
 
-        GF_VALIDATE_OR_GOTO ("changelog", this, out);
-        GF_VALIDATE_OR_GOTO ("changelog", this->fops, out);
-        GF_VALIDATE_OR_GOTO ("changelog", frame, out);
+    GF_VALIDATE_OR_GOTO("changelog", this, out);
+    GF_VALIDATE_OR_GOTO("changelog", this->fops, out);
+    GF_VALIDATE_OR_GOTO("changelog", frame, out);
 
-        priv = this->private;
+    priv = this->private;
 
-        gf_log (this->name, GF_LOG_DEBUG, "Dequeuing symlink");
-        changelog_color_fop_and_inc_cnt (this, priv, frame->local);
-        STACK_WIND (frame, changelog_symlink_cbk,
-                    FIRST_CHILD (this), FIRST_CHILD (this)->fops->symlink,
-                    linkname, loc, umask, xdata);
-        return 0;
+    gf_msg_debug(this->name, 0, "Dequeuing symlink");
+    changelog_color_fop_and_inc_cnt(this, priv, frame->local);
+    STACK_WIND(frame, changelog_symlink_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->symlink, linkname, loc, umask, xdata);
+    return 0;
 out:
-        return -1;
+    return -1;
 }
 
 int32_t
-changelog_symlink (call_frame_t *frame, xlator_t *this,
-                   const char *linkname, loc_t *loc,
-                   mode_t umask, dict_t *xdata)
-{
-        int               ret              = -1;
-        size_t            xtra_len         = 0;
-        uuid_t            gfid             = {0,};
-        void             *uuid_req         = NULL;
-        changelog_priv_t *priv             = NULL;
-        changelog_opt_t  *co               = NULL;
-        call_stub_t      *stub             = NULL;
-        struct list_head queue             = {0, };
-        gf_boolean_t     barrier_enabled   = _gf_false;
-
-        priv = this->private;
-        CHANGELOG_NOT_ACTIVE_THEN_GOTO (frame, priv, wind);
-
-        ret = dict_get_ptr (xdata, "gfid-req", &uuid_req);
-        if (ret) {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "failed to get gfid from dict");
-                goto wind;
-        }
-        gf_uuid_copy (gfid, uuid_req);
-
-        CHANGELOG_INIT_NOCHECK (this, frame->local, NULL, gfid, 2);
-
-        co = changelog_get_usable_buffer (frame->local);
-        if (!co)
-                goto wind;
-
-        CHANGLOG_FILL_FOP_NUMBER (co, frame->root->op, fop_fn, xtra_len);
-        co++;
-
-        CHANGELOG_FILL_ENTRY (co, loc->pargfid, loc->name,
-                              entry_fn, entry_free_fn, xtra_len, wind);
-
-        changelog_set_usable_record_and_length (frame->local, xtra_len, 2);
-
-        LOCK (&priv->lock);
-        {
-                if ((barrier_enabled = priv->barrier_enabled)) {
-                        stub = fop_symlink_stub (frame,
-                                                 changelog_symlink_resume,
-                                                 linkname, loc, umask, xdata);
-                        if (!stub)
-                               __chlog_barrier_disable (this, &queue);
-                        else
-                               __chlog_barrier_enqueue (this, stub);
-                } else {
-                        ((changelog_local_t *)frame->local)->color
-                                                          = priv->current_color;
-                        changelog_inc_fop_cnt (this, priv, frame->local);
-                }
-        }
-        UNLOCK (&priv->lock);
-
-        if (barrier_enabled && stub) {
-                gf_log (this->name, GF_LOG_DEBUG, "Enqueued symlink");
-                goto out;
-        }
-
-        if (barrier_enabled && !stub) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "Failed to barrier FOPs, disabling changelog barrier "
-                        "FOP: symlink, ERROR: %s", strerror (ENOMEM));
-                chlog_barrier_dequeue_all (this, &queue);
+changelog_symlink(call_frame_t *frame, xlator_t *this, const char *linkname,
+                  loc_t *loc, mode_t umask, dict_t *xdata)
+{
+    int ret = -1;
+    size_t xtra_len = 0;
+    uuid_t gfid = {
+        0,
+    };
+    changelog_priv_t *priv = NULL;
+    changelog_opt_t *co = NULL;
+    call_stub_t *stub = NULL;
+    struct list_head queue = {
+        0,
+    };
+    gf_boolean_t barrier_enabled = _gf_false;
+
+    priv = this->private;
+    CHANGELOG_NOT_ACTIVE_THEN_GOTO(frame, priv, wind);
+
+    ret = dict_get_gfuuid(xdata, "gfid-req", &gfid);
+    if (ret) {
+        gf_msg_debug(this->name, 0, "failed to get gfid from dict");
+        goto wind;
+    }
+
+    CHANGELOG_INIT_NOCHECK(this, frame->local, NULL, gfid, 2);
+
+    co = changelog_get_usable_buffer(frame->local);
+    if (!co)
+        goto wind;
+
+    CHANGLOG_FILL_FOP_NUMBER(co, frame->root->op, fop_fn, xtra_len);
+    co++;
+
+    CHANGELOG_FILL_ENTRY(co, loc->pargfid, loc->name, entry_fn, entry_free_fn,
+                         xtra_len, wind);
+
+    changelog_set_usable_record_and_length(frame->local, xtra_len, 2);
+
+    LOCK(&priv->lock);
+    {
+        if ((barrier_enabled = priv->barrier_enabled)) {
+            stub = fop_symlink_stub(frame, changelog_symlink_resume, linkname,
+                                    loc, umask, xdata);
+            if (!stub)
+                __chlog_barrier_disable(this, &queue);
+            else
+                __chlog_barrier_enqueue(this, stub);
+        } else {
+            ((changelog_local_t *)frame->local)->color = priv->current_color;
+            changelog_inc_fop_cnt(this, priv, frame->local);
         }
-
- wind:
-        STACK_WIND (frame, changelog_symlink_cbk,
-                    FIRST_CHILD (this), FIRST_CHILD (this)->fops->symlink,
-                    linkname, loc, umask, xdata);
+    }
+    UNLOCK(&priv->lock);
+
+    if (barrier_enabled && stub) {
+        gf_msg_debug(this->name, 0, "Enqueued symlink");
+        goto out;
+    }
+
+    if (barrier_enabled && !stub) {
+        gf_smsg(this->name, GF_LOG_ERROR, ENOMEM,
+                CHANGELOG_MSG_BARRIER_FOP_FAILED, "fop=symlink", NULL);
+        chlog_barrier_dequeue_all(this, &queue);
+    }
+
+wind:
+    STACK_WIND(frame, changelog_symlink_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->symlink, linkname, loc, umask, xdata);
 out:
-        return 0;
+    return 0;
 }
 
 /* mknod */
 
 int32_t
-changelog_mknod_cbk (call_frame_t *frame,
-                     void *cookie, xlator_t *this,
-                     int32_t op_ret, int32_t op_errno, inode_t *inode,
-                     struct iatt *buf, struct iatt *preparent,
-                     struct iatt *postparent, dict_t *xdata)
+changelog_mknod_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                    int32_t op_ret, int32_t op_errno, inode_t *inode,
+                    struct iatt *buf, struct iatt *preparent,
+                    struct iatt *postparent, dict_t *xdata)
 {
-        changelog_priv_t  *priv  = NULL;
-        changelog_local_t *local = NULL;
+    changelog_priv_t *priv = NULL;
+    changelog_local_t *local = NULL;
 
-        priv  = this->private;
-        local = frame->local;
+    priv = this->private;
+    local = frame->local;
 
-        CHANGELOG_COND_GOTO (priv, ((op_ret < 0) || !local), unwind);
+    CHANGELOG_COND_GOTO(priv, ((op_ret < 0) || !local), unwind);
 
-        changelog_update (this, priv, local, CHANGELOG_TYPE_ENTRY);
+    changelog_update(this, priv, local, CHANGELOG_TYPE_ENTRY);
 
- unwind:
-        changelog_dec_fop_cnt (this, priv, local);
-        CHANGELOG_STACK_UNWIND (mknod, frame, op_ret, op_errno,
-                                inode, buf, preparent, postparent, xdata);
-        return 0;
+unwind:
+    changelog_dec_fop_cnt(this, priv, local);
+    CHANGELOG_STACK_UNWIND(mknod, frame, op_ret, op_errno, inode, buf,
+                           preparent, postparent, xdata);
+    return 0;
 }
 
 int32_t
-changelog_mknod_resume (call_frame_t *frame, xlator_t *this,
-                        loc_t *loc, mode_t mode, dev_t rdev,
-                        mode_t umask, dict_t *xdata)
+changelog_mknod_resume(call_frame_t *frame, xlator_t *this, loc_t *loc,
+                       mode_t mode, dev_t rdev, mode_t umask, dict_t *xdata)
 {
-        changelog_priv_t *priv     = NULL;
+    changelog_priv_t *priv = NULL;
 
-        GF_VALIDATE_OR_GOTO ("changelog", this, out);
-        GF_VALIDATE_OR_GOTO ("changelog", this->fops, out);
-        GF_VALIDATE_OR_GOTO ("changelog", frame, out);
+    GF_VALIDATE_OR_GOTO("changelog", this, out);
+    GF_VALIDATE_OR_GOTO("changelog", this->fops, out);
+    GF_VALIDATE_OR_GOTO("changelog", frame, out);
 
-        priv = this->private;
+    priv = this->private;
 
-        gf_log (this->name, GF_LOG_DEBUG, "Dequeuing mknod");
-        changelog_color_fop_and_inc_cnt (this, priv, frame->local);
-        STACK_WIND (frame, changelog_mknod_cbk,
-                    FIRST_CHILD (this), FIRST_CHILD (this)->fops->mknod,
-                    loc, mode, rdev, umask, xdata);
-        return 0;
+    gf_msg_debug(this->name, 0, "Dequeuing mknod");
+    changelog_color_fop_and_inc_cnt(this, priv, frame->local);
+    STACK_WIND(frame, changelog_mknod_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->mknod, loc, mode, rdev, umask, xdata);
+    return 0;
 out:
-        return -1;
+    return -1;
 }
 
 int32_t
-changelog_mknod (call_frame_t *frame,
-                 xlator_t *this, loc_t *loc,
-                 mode_t mode, dev_t dev, mode_t umask, dict_t *xdata)
-{
-        int               ret              = -1;
-        uuid_t            gfid             = {0,};
-        void             *uuid_req         = NULL;
-        size_t            xtra_len         = 0;
-        changelog_priv_t *priv             = NULL;
-        changelog_opt_t  *co               = NULL;
-        call_stub_t      *stub             = NULL;
-        struct list_head queue             = {0, };
-        gf_boolean_t     barrier_enabled   = _gf_false;
-
-        priv = this->private;
-        CHANGELOG_NOT_ACTIVE_THEN_GOTO (frame, priv, wind);
-        CHANGELOG_IF_INTERNAL_FOP_THEN_GOTO (frame, xdata, wind);
-
-        ret = dict_get_ptr (xdata, "gfid-req", &uuid_req);
-        if (ret) {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "failed to get gfid from dict");
-                goto wind;
-        }
-        gf_uuid_copy (gfid, uuid_req);
-
-        CHANGELOG_INIT_NOCHECK (this, frame->local, NULL, gfid, 5);
-
-        co = changelog_get_usable_buffer (frame->local);
-        if (!co)
-                goto wind;
-
-        CHANGLOG_FILL_FOP_NUMBER (co, frame->root->op, fop_fn, xtra_len);
-        co++;
-
-        CHANGELOG_FILL_UINT32 (co, mode, number_fn, xtra_len);
-        co++;
-
-        CHANGELOG_FILL_UINT32 (co, frame->root->uid, number_fn, xtra_len);
-        co++;
-
-        CHANGELOG_FILL_UINT32 (co, frame->root->gid, number_fn, xtra_len);
-        co++;
-
-        CHANGELOG_FILL_ENTRY (co, loc->pargfid, loc->name,
-                              entry_fn, entry_free_fn, xtra_len, wind);
-
-        changelog_set_usable_record_and_length (frame->local, xtra_len, 5);
-
-        LOCK (&priv->lock);
-        {
-                if ((barrier_enabled = priv->barrier_enabled)) {
-                        stub = fop_mknod_stub (frame, changelog_mknod_resume,
-                                               loc, mode, dev, umask, xdata);
-                        if (!stub)
-                               __chlog_barrier_disable (this, &queue);
-                        else
-                               __chlog_barrier_enqueue (this, stub);
-                } else {
-                        ((changelog_local_t *)frame->local)->color
-                                                          = priv->current_color;
-                        changelog_inc_fop_cnt (this, priv, frame->local);
-                }
-        }
-        UNLOCK (&priv->lock);
-
-        if (barrier_enabled && stub) {
-                gf_log (this->name, GF_LOG_DEBUG, "Enqueued mknod");
-                goto out;
-        }
-
-        if (barrier_enabled && !stub) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "Failed to barrier FOPs, disabling changelog barrier "
-                        "FOP: mknod, ERROR: %s", strerror (ENOMEM));
-                chlog_barrier_dequeue_all (this, &queue);
+changelog_mknod(call_frame_t *frame, xlator_t *this, loc_t *loc, mode_t mode,
+                dev_t dev, mode_t umask, dict_t *xdata)
+{
+    int ret = -1;
+    uuid_t gfid = {
+        0,
+    };
+    size_t xtra_len = 0;
+    changelog_priv_t *priv = NULL;
+    changelog_opt_t *co = NULL;
+    call_stub_t *stub = NULL;
+    struct list_head queue = {
+        0,
+    };
+    gf_boolean_t barrier_enabled = _gf_false;
+
+    priv = this->private;
+
+    /* Check whether changelog active */
+    if (!(priv->active))
+        goto wind;
+
+    /* Check whether rebalance activity */
+    if (frame->root->pid == GF_CLIENT_PID_DEFRAG)
+        goto wind;
+
+    /* If tier-dht linkto is SET, ignore about verifiying :
+     * 1. Whether internal fop AND
+     * 2. Whether tier rebalance process activity (this will help in
+     * recording mknod if tier rebalance process calls this mknod) */
+    if (!(dict_get(xdata, "trusted.tier.tier-dht.linkto"))) {
+        CHANGELOG_IF_INTERNAL_FOP_THEN_GOTO(frame, xdata, wind);
+        if (frame->root->pid == GF_CLIENT_PID_TIER_DEFRAG)
+            goto wind;
+    }
+
+    ret = dict_get_gfuuid(xdata, "gfid-req", &gfid);
+    if (ret) {
+        gf_msg_debug(this->name, 0, "failed to get gfid from dict");
+        goto wind;
+    }
+
+    CHANGELOG_INIT_NOCHECK(this, frame->local, NULL, gfid, 5);
+
+    co = changelog_get_usable_buffer(frame->local);
+    if (!co)
+        goto wind;
+
+    CHANGLOG_FILL_FOP_NUMBER(co, frame->root->op, fop_fn, xtra_len);
+    co++;
+
+    CHANGELOG_FILL_UINT32(co, mode, number_fn, xtra_len);
+    co++;
+
+    CHANGELOG_FILL_UINT32(co, frame->root->uid, number_fn, xtra_len);
+    co++;
+
+    CHANGELOG_FILL_UINT32(co, frame->root->gid, number_fn, xtra_len);
+    co++;
+
+    CHANGELOG_FILL_ENTRY(co, loc->pargfid, loc->name, entry_fn, entry_free_fn,
+                         xtra_len, wind);
+
+    changelog_set_usable_record_and_length(frame->local, xtra_len, 5);
+
+    LOCK(&priv->lock);
+    {
+        if ((barrier_enabled = priv->barrier_enabled)) {
+            stub = fop_mknod_stub(frame, changelog_mknod_resume, loc, mode, dev,
+                                  umask, xdata);
+            if (!stub)
+                __chlog_barrier_disable(this, &queue);
+            else
+                __chlog_barrier_enqueue(this, stub);
+        } else {
+            ((changelog_local_t *)frame->local)->color = priv->current_color;
+            changelog_inc_fop_cnt(this, priv, frame->local);
         }
-
- wind:
-        STACK_WIND (frame, changelog_mknod_cbk,
-                    FIRST_CHILD (this), FIRST_CHILD (this)->fops->mknod,
-                    loc, mode, dev, umask, xdata);
+    }
+    UNLOCK(&priv->lock);
+
+    if (barrier_enabled && stub) {
+        gf_msg_debug(this->name, 0, "Enqueued mknod");
+        goto out;
+    }
+
+    if (barrier_enabled && !stub) {
+        gf_smsg(this->name, GF_LOG_ERROR, ENOMEM,
+                CHANGELOG_MSG_BARRIER_FOP_FAILED, "fop=mknod", NULL);
+        chlog_barrier_dequeue_all(this, &queue);
+    }
+
+wind:
+    STACK_WIND(frame, changelog_mknod_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->mknod, loc, mode, dev, umask, xdata);
 out:
-        return 0;
+    return 0;
 }
 
-/* creat */
+/* create */
 
 int32_t
-changelog_create_cbk (call_frame_t *frame,
-                      void *cookie, xlator_t *this,
-                      int32_t op_ret, int32_t op_errno,
-                      fd_t *fd, inode_t *inode, struct iatt *buf,
-                      struct iatt *preparent,
-                      struct iatt *postparent, dict_t *xdata)
+changelog_create_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                     int32_t op_ret, int32_t op_errno, fd_t *fd, inode_t *inode,
+                     struct iatt *buf, struct iatt *preparent,
+                     struct iatt *postparent, dict_t *xdata)
 {
-        int32_t ret = 0;
-        changelog_priv_t  *priv  = NULL;
-        changelog_local_t *local = NULL;
-        changelog_event_t  ev    = {0,};
+    int32_t ret = 0;
+    changelog_priv_t *priv = NULL;
+    changelog_local_t *local = NULL;
+    changelog_event_t ev = {
+        0,
+    };
 
-        priv  = this->private;
-        local = frame->local;
+    priv = this->private;
+    local = frame->local;
 
-        CHANGELOG_COND_GOTO (priv, ((op_ret < 0) || !local), unwind);
+    CHANGELOG_COND_GOTO(priv, ((op_ret < 0) || !local), unwind);
 
-        /* fill the event structure.. similar to open() */
-        ev.ev_type = CHANGELOG_OP_TYPE_CREATE;
-        gf_uuid_copy (ev.u.create.gfid, buf->ia_gfid);
-        ev.u.create.flags = fd->flags;
-        changelog_dispatch_event (this, priv, &ev);
+    /* fill the event structure.. similar to open() */
+    ev.ev_type = CHANGELOG_OP_TYPE_CREATE;
+    gf_uuid_copy(ev.u.create.gfid, buf->ia_gfid);
+    ev.u.create.flags = fd->flags;
+    changelog_dispatch_event(this, priv, &ev);
 
-        if (changelog_ev_selected
-                   (this, &priv->ev_selection, CHANGELOG_OP_TYPE_RELEASE)) {
-                ret = fd_ctx_set (fd, this, (uint64_t)(long) 0x1);
-                if (ret)
-                        gf_log (this->name, GF_LOG_WARNING,
-                                "could not set fd context (for release cbk)");
-        }
+    if (changelog_ev_selected(this, &priv->ev_selection,
+                              CHANGELOG_OP_TYPE_RELEASE)) {
+        ret = fd_ctx_set(fd, this, (uint64_t)(long)0x1);
+        if (ret)
+            gf_smsg(this->name, GF_LOG_WARNING, 0, CHANGELOG_MSG_SET_FD_CONTEXT,
+                    NULL);
+    }
 
-        changelog_update (this, priv, local, CHANGELOG_TYPE_ENTRY);
+    changelog_update(this, priv, local, CHANGELOG_TYPE_ENTRY);
 
- unwind:
-        changelog_dec_fop_cnt (this, priv, local);
-        CHANGELOG_STACK_UNWIND (create, frame,
-                                op_ret, op_errno, fd, inode,
-                                buf, preparent, postparent, xdata);
-        return 0;
+unwind:
+    changelog_dec_fop_cnt(this, priv, local);
+    CHANGELOG_STACK_UNWIND(create, frame, op_ret, op_errno, fd, inode, buf,
+                           preparent, postparent, xdata);
+    return 0;
 }
 
 int32_t
-changelog_create_resume (call_frame_t *frame, xlator_t *this,
-                         loc_t *loc, int32_t flags, mode_t mode,
-                         mode_t umask, fd_t *fd, dict_t *xdata)
+changelog_create_resume(call_frame_t *frame, xlator_t *this, loc_t *loc,
+                        int32_t flags, mode_t mode, mode_t umask, fd_t *fd,
+                        dict_t *xdata)
 {
-        changelog_priv_t *priv     = NULL;
+    changelog_priv_t *priv = NULL;
 
-        GF_VALIDATE_OR_GOTO ("changelog", this, out);
-        GF_VALIDATE_OR_GOTO ("changelog", this->fops, out);
-        GF_VALIDATE_OR_GOTO ("changelog", frame, out);
+    GF_VALIDATE_OR_GOTO("changelog", this, out);
+    GF_VALIDATE_OR_GOTO("changelog", this->fops, out);
+    GF_VALIDATE_OR_GOTO("changelog", frame, out);
 
-        priv = this->private;
+    priv = this->private;
 
-        gf_log (this->name, GF_LOG_DEBUG, "Dequeuing create");
-        changelog_color_fop_and_inc_cnt (this, priv, frame->local);
-        STACK_WIND (frame, changelog_create_cbk,
-                    FIRST_CHILD (this), FIRST_CHILD (this)->fops->create,
-                    loc, flags, mode, umask, fd, xdata);
-        return 0;
+    gf_msg_debug(this->name, 0, "Dequeuing create");
+    changelog_color_fop_and_inc_cnt(this, priv, frame->local);
+    STACK_WIND(frame, changelog_create_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->create, loc, flags, mode, umask, fd,
+               xdata);
+    return 0;
 
 out:
-        return -1;
+    return -1;
 }
 
 int32_t
-changelog_create (call_frame_t *frame, xlator_t *this,
-                  loc_t *loc, int32_t flags, mode_t mode,
-                  mode_t umask, fd_t *fd, dict_t *xdata)
-{
-        int               ret              = -1;
-        uuid_t            gfid             = {0,};
-        void             *uuid_req         = NULL;
-        changelog_opt_t  *co               = NULL;
-        changelog_priv_t *priv             = NULL;
-        size_t            xtra_len         = 0;
-        call_stub_t      *stub             = NULL;
-        struct list_head queue             = {0, };
-        gf_boolean_t     barrier_enabled   = _gf_false;
-
-        priv = this->private;
-        CHANGELOG_NOT_ACTIVE_THEN_GOTO (frame, priv, wind);
-
-        ret = dict_get_ptr (xdata, "gfid-req", &uuid_req);
-        if (ret) {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "failed to get gfid from dict");
-                goto wind;
-        }
-        gf_uuid_copy (gfid, uuid_req);
-
-        /* init with two extra records */
-        CHANGELOG_INIT_NOCHECK (this, frame->local, NULL, gfid, 5);
-        if (!frame->local)
-                goto wind;
-
-        co = changelog_get_usable_buffer (frame->local);
-        if (!co)
-                goto wind;
-
-        CHANGLOG_FILL_FOP_NUMBER (co, frame->root->op, fop_fn, xtra_len);
-        co++;
-
-        CHANGELOG_FILL_UINT32 (co, mode, number_fn, xtra_len);
-        co++;
-
-        CHANGELOG_FILL_UINT32 (co, frame->root->uid, number_fn, xtra_len);
-        co++;
-
-        CHANGELOG_FILL_UINT32 (co, frame->root->gid, number_fn, xtra_len);
-        co++;
-
-        CHANGELOG_FILL_ENTRY (co, loc->pargfid, loc->name,
-                              entry_fn, entry_free_fn, xtra_len, wind);
-
-        changelog_set_usable_record_and_length (frame->local, xtra_len, 5);
-
-        LOCK (&priv->lock);
-        {
-                if ((barrier_enabled = priv->barrier_enabled)) {
-                        stub = fop_create_stub (frame, changelog_create_resume,
-                                               loc, flags, mode, umask, fd,
-                                               xdata);
-                        if (!stub)
-                               __chlog_barrier_disable (this, &queue);
-                        else
-                               __chlog_barrier_enqueue (this, stub);
-                } else {
-                        ((changelog_local_t *)frame->local)->color
-                                                          = priv->current_color;
-                        changelog_inc_fop_cnt (this, priv, frame->local);
-                }
-        }
-        UNLOCK (&priv->lock);
-
-        if (barrier_enabled && stub) {
-                gf_log (this->name, GF_LOG_DEBUG, "Enqueued create");
-                goto out;
-        }
-
-        if (barrier_enabled && !stub) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "Failed to barrier FOPs, disabling changelog barrier "
-                        "FOP: create, ERROR: %s", strerror (ENOMEM));
-                chlog_barrier_dequeue_all (this, &queue);
+changelog_create(call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t flags,
+                 mode_t mode, mode_t umask, fd_t *fd, dict_t *xdata)
+{
+    int ret = -1;
+    uuid_t gfid = {
+        0,
+    };
+    changelog_opt_t *co = NULL;
+    changelog_priv_t *priv = NULL;
+    size_t xtra_len = 0;
+    call_stub_t *stub = NULL;
+    struct list_head queue = {
+        0,
+    };
+    gf_boolean_t barrier_enabled = _gf_false;
+
+    priv = this->private;
+    CHANGELOG_NOT_ACTIVE_THEN_GOTO(frame, priv, wind);
+
+    ret = dict_get_gfuuid(xdata, "gfid-req", &gfid);
+    if (ret) {
+        gf_msg_debug(this->name, 0, "failed to get gfid from dict");
+        goto wind;
+    }
+
+    /* init with two extra records */
+    CHANGELOG_INIT_NOCHECK(this, frame->local, NULL, gfid, 5);
+    if (!frame->local)
+        goto wind;
+
+    co = changelog_get_usable_buffer(frame->local);
+    if (!co)
+        goto wind;
+
+    CHANGLOG_FILL_FOP_NUMBER(co, frame->root->op, fop_fn, xtra_len);
+    co++;
+
+    CHANGELOG_FILL_UINT32(co, mode, number_fn, xtra_len);
+    co++;
+
+    CHANGELOG_FILL_UINT32(co, frame->root->uid, number_fn, xtra_len);
+    co++;
+
+    CHANGELOG_FILL_UINT32(co, frame->root->gid, number_fn, xtra_len);
+    co++;
+
+    CHANGELOG_FILL_ENTRY(co, loc->pargfid, loc->name, entry_fn, entry_free_fn,
+                         xtra_len, wind);
+
+    changelog_set_usable_record_and_length(frame->local, xtra_len, 5);
+
+    LOCK(&priv->lock);
+    {
+        if ((barrier_enabled = priv->barrier_enabled)) {
+            stub = fop_create_stub(frame, changelog_create_resume, loc, flags,
+                                   mode, umask, fd, xdata);
+            if (!stub)
+                __chlog_barrier_disable(this, &queue);
+            else
+                __chlog_barrier_enqueue(this, stub);
+        } else {
+            ((changelog_local_t *)frame->local)->color = priv->current_color;
+            changelog_inc_fop_cnt(this, priv, frame->local);
         }
-
- wind:
-        STACK_WIND (frame, changelog_create_cbk,
-                    FIRST_CHILD (this), FIRST_CHILD (this)->fops->create,
-                    loc, flags, mode, umask, fd, xdata);
+    }
+    UNLOCK(&priv->lock);
+
+    if (barrier_enabled && stub) {
+        gf_msg_debug(this->name, 0, "Enqueued create");
+        goto out;
+    }
+
+    if (barrier_enabled && !stub) {
+        gf_smsg(this->name, GF_LOG_ERROR, ENOMEM,
+                CHANGELOG_MSG_BARRIER_FOP_FAILED, "fop=create", NULL);
+        chlog_barrier_dequeue_all(this, &queue);
+    }
+
+wind:
+    STACK_WIND(frame, changelog_create_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->create, loc, flags, mode, umask, fd,
+               xdata);
 out:
-        return 0;
+    return 0;
 }
 
 /* }}} */
 
-
 /* Metadata modification fops - TYPE II */
 
 /* {{{ */
@@ -1069,259 +1104,253 @@ out:
 /* {f}setattr */
 
 int32_t
-changelog_fsetattr_cbk (call_frame_t *frame,
-                        void *cookie, xlator_t *this, int32_t op_ret,
-                        int32_t op_errno, struct iatt *preop_stbuf,
-                        struct iatt *postop_stbuf, dict_t *xdata)
+changelog_fsetattr_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                       int32_t op_ret, int32_t op_errno,
+                       struct iatt *preop_stbuf, struct iatt *postop_stbuf,
+                       dict_t *xdata)
 {
-        changelog_priv_t  *priv  = NULL;
-        changelog_local_t *local = NULL;
-
-        priv  = this->private;
-        local = frame->local;
+    changelog_priv_t *priv = NULL;
+    changelog_local_t *local = NULL;
 
-        CHANGELOG_COND_GOTO (priv, ((op_ret < 0) || !local), unwind);
+    priv = this->private;
+    local = frame->local;
 
-        changelog_update (this, priv, local, CHANGELOG_TYPE_METADATA);
+    CHANGELOG_COND_GOTO(priv, ((op_ret < 0) || !local), unwind);
 
- unwind:
-        changelog_dec_fop_cnt (this, priv, local);
-        CHANGELOG_STACK_UNWIND (fsetattr, frame, op_ret, op_errno,
-                                preop_stbuf, postop_stbuf, xdata);
-
-        return 0;
+    changelog_update(this, priv, local, CHANGELOG_TYPE_METADATA);
 
+unwind:
+    changelog_dec_fop_cnt(this, priv, local);
+    CHANGELOG_STACK_UNWIND(fsetattr, frame, op_ret, op_errno, preop_stbuf,
+                           postop_stbuf, xdata);
 
+    return 0;
 }
 
 int32_t
-changelog_fsetattr (call_frame_t *frame,
-                    xlator_t *this, fd_t *fd,
-                    struct iatt *stbuf, int32_t valid, dict_t *xdata)
+changelog_fsetattr(call_frame_t *frame, xlator_t *this, fd_t *fd,
+                   struct iatt *stbuf, int32_t valid, dict_t *xdata)
 {
-        changelog_priv_t *priv     = NULL;
-        changelog_opt_t  *co       = NULL;
-        size_t            xtra_len = 0;
-
-        priv = this->private;
-        CHANGELOG_NOT_ACTIVE_THEN_GOTO (frame, priv, wind);
+    changelog_priv_t *priv = NULL;
+    changelog_opt_t *co = NULL;
+    size_t xtra_len = 0;
 
-        CHANGELOG_OP_BOUNDARY_CHECK (frame, wind);
+    priv = this->private;
+    CHANGELOG_NOT_ACTIVE_THEN_GOTO(frame, priv, wind);
 
-        CHANGELOG_INIT (this, frame->local,
-                        fd->inode, fd->inode->gfid, 1);
-        if (!frame->local)
-                goto wind;
+    CHANGELOG_OP_BOUNDARY_CHECK(frame, wind);
 
-        co = changelog_get_usable_buffer (frame->local);
-        if (!co)
-                goto wind;
+    CHANGELOG_INIT(this, frame->local, fd->inode, fd->inode->gfid, 1);
+    if (!frame->local)
+        goto wind;
 
-        CHANGLOG_FILL_FOP_NUMBER (co, frame->root->op, fop_fn, xtra_len);
+    co = changelog_get_usable_buffer(frame->local);
+    if (!co)
+        goto wind;
 
-        changelog_set_usable_record_and_length (frame->local, xtra_len, 1);
-
- wind:
-        changelog_color_fop_and_inc_cnt (this, priv, frame->local);
-        STACK_WIND (frame, changelog_fsetattr_cbk,
-                    FIRST_CHILD (this), FIRST_CHILD (this)->fops->fsetattr,
-                    fd, stbuf, valid, xdata);
-        return 0;
+    CHANGLOG_FILL_FOP_NUMBER(co, frame->root->op, fop_fn, xtra_len);
 
+    changelog_set_usable_record_and_length(frame->local, xtra_len, 1);
 
+wind:
+    changelog_color_fop_and_inc_cnt(this, priv, frame->local);
+    STACK_WIND(frame, changelog_fsetattr_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->fsetattr, fd, stbuf, valid, xdata);
+    return 0;
 }
 
 int32_t
-changelog_setattr_cbk (call_frame_t *frame,
-                       void *cookie, xlator_t *this, int32_t op_ret,
-                       int32_t op_errno, struct iatt *preop_stbuf,
-                       struct iatt *postop_stbuf, dict_t *xdata)
+changelog_setattr_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                      int32_t op_ret, int32_t op_errno,
+                      struct iatt *preop_stbuf, struct iatt *postop_stbuf,
+                      dict_t *xdata)
 {
-        changelog_priv_t  *priv  = NULL;
-        changelog_local_t *local = NULL;
+    changelog_priv_t *priv = NULL;
+    changelog_local_t *local = NULL;
 
-        priv  = this->private;
-        local = frame->local;
+    priv = this->private;
+    local = frame->local;
 
-        CHANGELOG_COND_GOTO (priv, ((op_ret < 0) || !local), unwind);
+    CHANGELOG_COND_GOTO(priv, ((op_ret < 0) || !local), unwind);
 
-        changelog_update (this, priv, local, CHANGELOG_TYPE_METADATA);
+    changelog_update(this, priv, local, CHANGELOG_TYPE_METADATA);
 
- unwind:
-        changelog_dec_fop_cnt (this, priv, local);
-        CHANGELOG_STACK_UNWIND (setattr, frame, op_ret, op_errno,
-                                preop_stbuf, postop_stbuf, xdata);
+unwind:
+    changelog_dec_fop_cnt(this, priv, local);
+    CHANGELOG_STACK_UNWIND(setattr, frame, op_ret, op_errno, preop_stbuf,
+                           postop_stbuf, xdata);
 
-        return 0;
+    return 0;
 }
 
 int32_t
-changelog_setattr (call_frame_t *frame,
-                   xlator_t *this, loc_t *loc,
-                   struct iatt *stbuf, int32_t valid, dict_t *xdata)
+changelog_setattr(call_frame_t *frame, xlator_t *this, loc_t *loc,
+                  struct iatt *stbuf, int32_t valid, dict_t *xdata)
 {
-        changelog_priv_t *priv     = NULL;
-        changelog_opt_t  *co       = NULL;
-        size_t            xtra_len = 0;
+    changelog_priv_t *priv = NULL;
+    changelog_opt_t *co = NULL;
+    size_t xtra_len = 0;
+    uuid_t shard_root_gfid = {
+        0,
+    };
 
-        priv = this->private;
-        CHANGELOG_NOT_ACTIVE_THEN_GOTO (frame, priv, wind);
+    priv = this->private;
+    CHANGELOG_NOT_ACTIVE_THEN_GOTO(frame, priv, wind);
 
-        CHANGELOG_OP_BOUNDARY_CHECK (frame, wind);
+    CHANGELOG_IF_INTERNAL_FOP_THEN_GOTO(frame, xdata, wind);
 
-        CHANGELOG_INIT (this, frame->local,
-                        loc->inode, loc->inode->gfid, 1);
-        if (!frame->local)
-                goto wind;
+    /* Do not record META on .shard */
+    gf_uuid_parse(SHARD_ROOT_GFID, shard_root_gfid);
+    if (gf_uuid_compare(loc->gfid, shard_root_gfid) == 0) {
+        goto wind;
+    }
 
-        co = changelog_get_usable_buffer (frame->local);
-        if (!co)
-                goto wind;
+    CHANGELOG_OP_BOUNDARY_CHECK(frame, wind);
 
-        CHANGLOG_FILL_FOP_NUMBER (co, frame->root->op, fop_fn, xtra_len);
+    CHANGELOG_INIT(this, frame->local, loc->inode, loc->inode->gfid, 1);
+    if (!frame->local)
+        goto wind;
 
-        changelog_set_usable_record_and_length (frame->local, xtra_len, 1);
+    co = changelog_get_usable_buffer(frame->local);
+    if (!co)
+        goto wind;
 
- wind:
-        changelog_color_fop_and_inc_cnt (this, priv, frame->local);
-        STACK_WIND (frame, changelog_setattr_cbk,
-                    FIRST_CHILD (this), FIRST_CHILD (this)->fops->setattr,
-                    loc, stbuf, valid, xdata);
-        return 0;
+    CHANGLOG_FILL_FOP_NUMBER(co, frame->root->op, fop_fn, xtra_len);
+
+    changelog_set_usable_record_and_length(frame->local, xtra_len, 1);
+
+wind:
+    changelog_color_fop_and_inc_cnt(this, priv, frame->local);
+    STACK_WIND(frame, changelog_setattr_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->setattr, loc, stbuf, valid, xdata);
+    return 0;
 }
 
 /* {f}removexattr */
 
 int32_t
-changelog_fremovexattr_cbk (call_frame_t *frame,
-                            void *cookie, xlator_t *this,
-                            int32_t op_ret, int32_t op_errno, dict_t *xdata)
+changelog_fremovexattr_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                           int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
-        changelog_priv_t  *priv  = NULL;
-        changelog_local_t *local = NULL;
+    changelog_priv_t *priv = NULL;
+    changelog_local_t *local = NULL;
 
-        priv  = this->private;
-        local = frame->local;
+    priv = this->private;
+    local = frame->local;
 
-        CHANGELOG_COND_GOTO (priv, ((op_ret < 0) || !local), unwind);
+    CHANGELOG_COND_GOTO(priv, ((op_ret < 0) || !local), unwind);
 
-        changelog_update (this, priv, local, CHANGELOG_TYPE_METADATA);
+    changelog_update(this, priv, local, CHANGELOG_TYPE_METADATA_XATTR);
 
- unwind:
-        changelog_dec_fop_cnt (this, priv, local);
-        CHANGELOG_STACK_UNWIND (fremovexattr, frame, op_ret, op_errno, xdata);
+unwind:
+    changelog_dec_fop_cnt(this, priv, local);
+    CHANGELOG_STACK_UNWIND(fremovexattr, frame, op_ret, op_errno, xdata);
 
-        return 0;
+    return 0;
 }
 
 int32_t
-changelog_fremovexattr (call_frame_t *frame, xlator_t *this,
-                        fd_t *fd, const char *name, dict_t *xdata)
+changelog_fremovexattr(call_frame_t *frame, xlator_t *this, fd_t *fd,
+                       const char *name, dict_t *xdata)
 {
-        changelog_priv_t *priv     = NULL;
-        changelog_opt_t  *co       = NULL;
-        size_t            xtra_len = 0;
+    changelog_priv_t *priv = NULL;
+    changelog_opt_t *co = NULL;
+    size_t xtra_len = 0;
 
-        priv = this->private;
-        CHANGELOG_NOT_ACTIVE_THEN_GOTO (frame, priv, wind);
+    priv = this->private;
+    CHANGELOG_NOT_ACTIVE_THEN_GOTO(frame, priv, wind);
 
-        CHANGELOG_OP_BOUNDARY_CHECK (frame, wind);
+    CHANGELOG_OP_BOUNDARY_CHECK(frame, wind);
 
-        CHANGELOG_INIT (this, frame->local,
-                        fd->inode, fd->inode->gfid, 1);
+    CHANGELOG_INIT(this, frame->local, fd->inode, fd->inode->gfid, 1);
 
-        co = changelog_get_usable_buffer (frame->local);
-        if (!co)
-                goto wind;
+    co = changelog_get_usable_buffer(frame->local);
+    if (!co)
+        goto wind;
 
-        CHANGLOG_FILL_FOP_NUMBER (co, frame->root->op, fop_fn, xtra_len);
+    CHANGLOG_FILL_FOP_NUMBER(co, frame->root->op, fop_fn, xtra_len);
 
-        changelog_set_usable_record_and_length (frame->local, xtra_len, 1);
+    changelog_set_usable_record_and_length(frame->local, xtra_len, 1);
 
- wind:
-        changelog_color_fop_and_inc_cnt (this, priv, frame->local);
-        STACK_WIND (frame, changelog_fremovexattr_cbk,
-                    FIRST_CHILD (this), FIRST_CHILD (this)->fops->fremovexattr,
-                    fd, name, xdata);
-        return 0;
+wind:
+    changelog_color_fop_and_inc_cnt(this, priv, frame->local);
+    STACK_WIND(frame, changelog_fremovexattr_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->fremovexattr, fd, name, xdata);
+    return 0;
 }
 
 int32_t
-changelog_removexattr_cbk (call_frame_t *frame,
-                           void *cookie, xlator_t *this,
-                           int32_t op_ret, int32_t op_errno, dict_t *xdata)
+changelog_removexattr_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                          int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
-        changelog_priv_t  *priv  = NULL;
-        changelog_local_t *local = NULL;
+    changelog_priv_t *priv = NULL;
+    changelog_local_t *local = NULL;
 
-        priv  = this->private;
-        local = frame->local;
+    priv = this->private;
+    local = frame->local;
 
-        CHANGELOG_COND_GOTO (priv, ((op_ret < 0) || !local), unwind);
+    CHANGELOG_COND_GOTO(priv, ((op_ret < 0) || !local), unwind);
 
-        changelog_update (this, priv, local, CHANGELOG_TYPE_METADATA);
+    changelog_update(this, priv, local, CHANGELOG_TYPE_METADATA_XATTR);
 
- unwind:
-        changelog_dec_fop_cnt (this, priv, local);
-        CHANGELOG_STACK_UNWIND (removexattr, frame, op_ret, op_errno, xdata);
+unwind:
+    changelog_dec_fop_cnt(this, priv, local);
+    CHANGELOG_STACK_UNWIND(removexattr, frame, op_ret, op_errno, xdata);
 
-        return 0;
+    return 0;
 }
 
 int32_t
-changelog_removexattr (call_frame_t *frame, xlator_t *this,
-                       loc_t *loc, const char *name, dict_t *xdata)
+changelog_removexattr(call_frame_t *frame, xlator_t *this, loc_t *loc,
+                      const char *name, dict_t *xdata)
 {
-        changelog_priv_t *priv     = NULL;
-        changelog_opt_t  *co       = NULL;
-        size_t            xtra_len = 0;
+    changelog_priv_t *priv = NULL;
+    changelog_opt_t *co = NULL;
+    size_t xtra_len = 0;
 
-        priv = this->private;
-        CHANGELOG_NOT_ACTIVE_THEN_GOTO (frame, priv, wind);
+    priv = this->private;
+    CHANGELOG_NOT_ACTIVE_THEN_GOTO(frame, priv, wind);
 
-        CHANGELOG_OP_BOUNDARY_CHECK (frame, wind);
+    CHANGELOG_OP_BOUNDARY_CHECK(frame, wind);
 
-        CHANGELOG_INIT (this, frame->local,
-                        loc->inode, loc->inode->gfid, 1);
+    CHANGELOG_INIT(this, frame->local, loc->inode, loc->inode->gfid, 1);
 
-        co = changelog_get_usable_buffer (frame->local);
-        if (!co)
-                goto wind;
+    co = changelog_get_usable_buffer(frame->local);
+    if (!co)
+        goto wind;
 
-        CHANGLOG_FILL_FOP_NUMBER (co, frame->root->op, fop_fn, xtra_len);
+    CHANGLOG_FILL_FOP_NUMBER(co, frame->root->op, fop_fn, xtra_len);
 
-        changelog_set_usable_record_and_length (frame->local, xtra_len, 1);
+    changelog_set_usable_record_and_length(frame->local, xtra_len, 1);
 
- wind:
-        changelog_color_fop_and_inc_cnt (this, priv, frame->local);
-        STACK_WIND (frame, changelog_removexattr_cbk,
-                    FIRST_CHILD (this), FIRST_CHILD (this)->fops->removexattr,
-                    loc, name, xdata);
-        return 0;
+wind:
+    changelog_color_fop_and_inc_cnt(this, priv, frame->local);
+    STACK_WIND(frame, changelog_removexattr_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->removexattr, loc, name, xdata);
+    return 0;
 }
 
 /* {f}setxattr */
 
 int32_t
-changelog_setxattr_cbk (call_frame_t *frame,
-                        void *cookie, xlator_t *this,
-                        int32_t op_ret, int32_t op_errno, dict_t *xdata)
+changelog_setxattr_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                       int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
-        changelog_priv_t  *priv  = NULL;
-        changelog_local_t *local = NULL;
+    changelog_priv_t *priv = NULL;
+    changelog_local_t *local = NULL;
 
-        priv  = this->private;
-        local = frame->local;
+    priv = this->private;
+    local = frame->local;
 
-        CHANGELOG_COND_GOTO (priv, ((op_ret < 0) || !local), unwind);
+    CHANGELOG_COND_GOTO(priv, ((op_ret < 0) || !local), unwind);
 
-        changelog_update (this, priv, local, CHANGELOG_TYPE_METADATA);
+    changelog_update(this, priv, local, CHANGELOG_TYPE_METADATA_XATTR);
 
- unwind:
-        changelog_dec_fop_cnt (this, priv, local);
-        CHANGELOG_STACK_UNWIND (setxattr, frame, op_ret, op_errno, xdata);
+unwind:
+    changelog_dec_fop_cnt(this, priv, local);
+    CHANGELOG_STACK_UNWIND(setxattr, frame, op_ret, op_errno, xdata);
 
-        return 0;
+    return 0;
 }
 
 /* changelog_handle_virtual_xattr:
@@ -1334,150 +1363,254 @@ changelog_setxattr_cbk (call_frame_t *frame,
  *           any other value: ENOTSUP is returned.
  */
 static void
-changelog_handle_virtual_xattr (call_frame_t *frame, xlator_t *this,
-                                loc_t *loc, dict_t *dict)
+changelog_handle_virtual_xattr(call_frame_t *frame, xlator_t *this, loc_t *loc,
+                               dict_t *dict)
 {
-        changelog_priv_t     *priv         = NULL;
-        changelog_local_t    *local        = NULL;
-        int32_t               value        = 0;
-        int                   ret          = 0;
-        int                   dict_ret     = 0;
-        gf_boolean_t          valid        = _gf_false;
+    changelog_priv_t *priv = NULL;
+    changelog_local_t *local = NULL;
+    int32_t value = 0;
+    int ret = 0;
+    int dict_ret = 0;
+    gf_boolean_t valid = _gf_false;
 
-        priv = this->private;
-        GF_ASSERT (priv);
+    priv = this->private;
+    GF_ASSERT(priv);
 
-        dict_ret = dict_get_int32 (dict, GF_XATTR_TRIGGER_SYNC, &value);
+    dict_ret = dict_get_int32(dict, GF_XATTR_TRIGGER_SYNC, &value);
 
-        if ((dict_ret == 0 && value == 1) && ((loc->inode->ia_type == IA_IFDIR)
-            || (loc->inode->ia_type == IA_IFREG)))
-                valid = _gf_true;
+    if ((dict_ret == 0 && value == 1) && ((loc->inode->ia_type == IA_IFDIR) ||
+                                          (loc->inode->ia_type == IA_IFREG)))
+        valid = _gf_true;
 
-        if (valid) {
-                ret = changelog_fill_entry_buf (frame, this, loc, &local);
-                if (ret) {
-                        gf_log (this->name, GF_LOG_INFO, "Entry cannot be"
-                                " captured for gfid: %s. Capturing DATA"
-                                " entry.", uuid_utoa (loc->inode->gfid));
-                        goto unwind;
-                }
-                changelog_update (this, priv, local, CHANGELOG_TYPE_ENTRY);
-
- unwind:
-                /* Capture DATA only if it's a file. */
-                if (loc->inode->ia_type != IA_IFDIR)
-                        changelog_update (this, priv, frame->local,
-                                          CHANGELOG_TYPE_DATA);
-                /* Assign local to prev_entry, so unwind will take
-                 * care of cleanup. */
-                ((changelog_local_t *)(frame->local))->prev_entry = local;
-                CHANGELOG_STACK_UNWIND (setxattr, frame, 0, 0, NULL);
-                return;
-        } else {
-                CHANGELOG_STACK_UNWIND (setxattr, frame, -1, ENOTSUP, NULL);
-                return;
+    if (valid) {
+        ret = changelog_fill_entry_buf(frame, this, loc, &local);
+        if (ret) {
+            gf_smsg(this->name, GF_LOG_INFO, 0, CHANGELOG_MSG_ENTRY_BUF_INFO,
+                    "gfid=%s", uuid_utoa(loc->inode->gfid), NULL);
+            goto unwind;
         }
+        changelog_update(this, priv, local, CHANGELOG_TYPE_ENTRY);
+
+    unwind:
+        /* Capture DATA only if it's a file. */
+        if (loc->inode->ia_type != IA_IFDIR)
+            changelog_update(this, priv, frame->local, CHANGELOG_TYPE_DATA);
+        /* Assign local to prev_entry, so unwind will take
+         * care of cleanup. */
+        ((changelog_local_t *)(frame->local))->prev_entry = local;
+        CHANGELOG_STACK_UNWIND(setxattr, frame, 0, 0, NULL);
+        return;
+    } else {
+        CHANGELOG_STACK_UNWIND(setxattr, frame, -1, ENOTSUP, NULL);
+        return;
+    }
 }
 
 int32_t
-changelog_setxattr (call_frame_t *frame,
-                    xlator_t *this, loc_t *loc,
-                    dict_t *dict, int32_t flags, dict_t *xdata)
+changelog_setxattr(call_frame_t *frame, xlator_t *this, loc_t *loc,
+                   dict_t *dict, int32_t flags, dict_t *xdata)
 {
-        changelog_priv_t *priv     = NULL;
-        changelog_opt_t  *co       = NULL;
-        size_t            xtra_len = 0;
+    changelog_priv_t *priv = NULL;
+    changelog_opt_t *co = NULL;
+    size_t xtra_len = 0;
 
-        priv = this->private;
-        CHANGELOG_NOT_ACTIVE_THEN_GOTO (frame, priv, wind);
+    priv = this->private;
+    CHANGELOG_NOT_ACTIVE_THEN_GOTO(frame, priv, wind);
 
-        CHANGELOG_OP_BOUNDARY_CHECK (frame, wind);
+    CHANGELOG_OP_BOUNDARY_CHECK(frame, wind);
 
-        CHANGELOG_INIT (this, frame->local,
-                        loc->inode, loc->inode->gfid, 1);
+    CHANGELOG_INIT(this, frame->local, loc->inode, loc->inode->gfid, 1);
 
-        /* On setting this virtual xattr on a file, an explicit data
-         * sync is triggered from geo-rep as CREATE|DATA entry is
-         * recorded in changelog based on xattr value.
-         */
-        if (dict_get (dict, GF_XATTR_TRIGGER_SYNC)) {
-                changelog_handle_virtual_xattr (frame, this, loc, dict);
-                return 0;
-        }
+    /* On setting this virtual xattr on a file, an explicit data
+     * sync is triggered from geo-rep as CREATE|DATA entry is
+     * recorded in changelog based on xattr value.
+     */
+    if (dict_get(dict, GF_XATTR_TRIGGER_SYNC)) {
+        changelog_handle_virtual_xattr(frame, this, loc, dict);
+        return 0;
+    }
 
-        co = changelog_get_usable_buffer (frame->local);
-        if (!co)
-                goto wind;
+    co = changelog_get_usable_buffer(frame->local);
+    if (!co)
+        goto wind;
 
-        CHANGLOG_FILL_FOP_NUMBER (co, frame->root->op, fop_fn, xtra_len);
+    CHANGLOG_FILL_FOP_NUMBER(co, frame->root->op, fop_fn, xtra_len);
 
-        changelog_set_usable_record_and_length (frame->local, xtra_len, 1);
+    changelog_set_usable_record_and_length(frame->local, xtra_len, 1);
 
- wind:
-        changelog_color_fop_and_inc_cnt (this, priv, frame->local);
-        STACK_WIND (frame, changelog_setxattr_cbk,
-                    FIRST_CHILD (this), FIRST_CHILD (this)->fops->setxattr,
-                    loc, dict, flags, xdata);
-        return 0;
+wind:
+    changelog_color_fop_and_inc_cnt(this, priv, frame->local);
+    STACK_WIND(frame, changelog_setxattr_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->setxattr, loc, dict, flags, xdata);
+    return 0;
 }
 
 int32_t
-changelog_fsetxattr_cbk (call_frame_t *frame,
-                         void *cookie, xlator_t *this, int32_t op_ret,
-                         int32_t op_errno, dict_t *xdata)
+changelog_fsetxattr_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                        int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
-        changelog_priv_t  *priv  = NULL;
-        changelog_local_t *local = NULL;
+    changelog_priv_t *priv = NULL;
+    changelog_local_t *local = NULL;
 
-        priv  = this->private;
-        local = frame->local;
+    priv = this->private;
+    local = frame->local;
 
-        CHANGELOG_COND_GOTO (priv, ((op_ret < 0) || !local), unwind);
+    CHANGELOG_COND_GOTO(priv, ((op_ret < 0) || !local), unwind);
 
-        changelog_update (this, priv, local, CHANGELOG_TYPE_METADATA);
+    changelog_update(this, priv, local, CHANGELOG_TYPE_METADATA_XATTR);
 
- unwind:
-        changelog_dec_fop_cnt (this, priv, local);
-        CHANGELOG_STACK_UNWIND (fsetxattr, frame, op_ret, op_errno, xdata);
+unwind:
+    changelog_dec_fop_cnt(this, priv, local);
+    CHANGELOG_STACK_UNWIND(fsetxattr, frame, op_ret, op_errno, xdata);
 
-        return 0;
+    return 0;
 }
 
 int32_t
-changelog_fsetxattr (call_frame_t *frame,
-                     xlator_t *this, fd_t *fd, dict_t *dict,
-                     int32_t flags, dict_t *xdata)
+changelog_fsetxattr(call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *dict,
+                    int32_t flags, dict_t *xdata)
 {
-        changelog_priv_t *priv     = NULL;
-        changelog_opt_t  *co       = NULL;
-        size_t            xtra_len = 0;
+    changelog_priv_t *priv = NULL;
+    changelog_opt_t *co = NULL;
+    size_t xtra_len = 0;
 
-        priv = this->private;
-        CHANGELOG_NOT_ACTIVE_THEN_GOTO (frame, priv, wind);
+    priv = this->private;
+    CHANGELOG_NOT_ACTIVE_THEN_GOTO(frame, priv, wind);
+    CHANGELOG_IF_INTERNAL_FOP_THEN_GOTO(frame, xdata, wind);
 
-        CHANGELOG_OP_BOUNDARY_CHECK (frame, wind);
+    CHANGELOG_OP_BOUNDARY_CHECK(frame, wind);
 
-        CHANGELOG_INIT (this, frame->local,
-                        fd->inode, fd->inode->gfid, 1);
+    CHANGELOG_INIT(this, frame->local, fd->inode, fd->inode->gfid, 1);
 
-        co = changelog_get_usable_buffer (frame->local);
-        if (!co)
-                goto wind;
+    co = changelog_get_usable_buffer(frame->local);
+    if (!co)
+        goto wind;
 
-        CHANGLOG_FILL_FOP_NUMBER (co, frame->root->op, fop_fn, xtra_len);
+    CHANGLOG_FILL_FOP_NUMBER(co, frame->root->op, fop_fn, xtra_len);
 
-        changelog_set_usable_record_and_length (frame->local, xtra_len, 1);
+    changelog_set_usable_record_and_length(frame->local, xtra_len, 1);
 
- wind:
-        changelog_color_fop_and_inc_cnt (this, priv, frame->local);
-        STACK_WIND (frame, changelog_fsetxattr_cbk,
-                    FIRST_CHILD (this), FIRST_CHILD (this)->fops->fsetxattr,
-                    fd, dict, flags, xdata);
-        return 0;
+wind:
+    changelog_color_fop_and_inc_cnt(this, priv, frame->local);
+    STACK_WIND(frame, changelog_fsetxattr_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->fsetxattr, fd, dict, flags, xdata);
+    return 0;
 }
 
-/* }}} */
+int32_t
+changelog_xattrop_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                      int32_t op_ret, int32_t op_errno, dict_t *xattr,
+                      dict_t *xdata)
+{
+    changelog_priv_t *priv = NULL;
+    changelog_local_t *local = NULL;
 
+    priv = this->private;
+    local = frame->local;
+
+    CHANGELOG_COND_GOTO(priv, ((op_ret < 0) || !local), unwind);
+
+    changelog_update(this, priv, local, CHANGELOG_TYPE_METADATA);
+
+unwind:
+    changelog_dec_fop_cnt(this, priv, local);
+    CHANGELOG_STACK_UNWIND(xattrop, frame, op_ret, op_errno, xattr, xdata);
+
+    return 0;
+}
+
+int32_t
+changelog_xattrop(call_frame_t *frame, xlator_t *this, loc_t *loc,
+                  gf_xattrop_flags_t optype, dict_t *xattr, dict_t *xdata)
+{
+    changelog_priv_t *priv = NULL;
+    changelog_opt_t *co = NULL;
+    size_t xtra_len = 0;
+    int ret = 0;
+    void *size_attr = NULL;
+
+    priv = this->private;
+    CHANGELOG_NOT_ACTIVE_THEN_GOTO(frame, priv, wind);
+    ret = dict_get_ptr(xattr, GF_XATTR_SHARD_FILE_SIZE, &size_attr);
+    if (ret)
+        goto wind;
+
+    CHANGELOG_OP_BOUNDARY_CHECK(frame, wind);
+
+    CHANGELOG_INIT(this, frame->local, loc->inode, loc->inode->gfid, 1);
+
+    co = changelog_get_usable_buffer(frame->local);
+    if (!co)
+        goto wind;
+
+    CHANGLOG_FILL_FOP_NUMBER(co, frame->root->op, fop_fn, xtra_len);
+
+    changelog_set_usable_record_and_length(frame->local, xtra_len, 1);
+
+wind:
+    changelog_color_fop_and_inc_cnt(this, priv, frame->local);
+    STACK_WIND(frame, changelog_xattrop_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->xattrop, loc, optype, xattr, xdata);
+    return 0;
+}
+
+int32_t
+changelog_fxattrop_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                       int32_t op_ret, int32_t op_errno, dict_t *xattr,
+                       dict_t *xdata)
+{
+    changelog_priv_t *priv = NULL;
+    changelog_local_t *local = NULL;
+
+    priv = this->private;
+    local = frame->local;
+
+    CHANGELOG_COND_GOTO(priv, ((op_ret < 0) || !local), unwind);
+
+    changelog_update(this, priv, local, CHANGELOG_TYPE_METADATA_XATTR);
+
+unwind:
+    changelog_dec_fop_cnt(this, priv, local);
+    CHANGELOG_STACK_UNWIND(fxattrop, frame, op_ret, op_errno, xattr, xdata);
+
+    return 0;
+}
+
+int32_t
+changelog_fxattrop(call_frame_t *frame, xlator_t *this, fd_t *fd,
+                   gf_xattrop_flags_t optype, dict_t *xattr, dict_t *xdata)
+{
+    changelog_priv_t *priv = NULL;
+    changelog_opt_t *co = NULL;
+    size_t xtra_len = 0;
+    void *size_attr = NULL;
+    int ret = 0;
+
+    priv = this->private;
+    CHANGELOG_NOT_ACTIVE_THEN_GOTO(frame, priv, wind);
+    ret = dict_get_ptr(xattr, GF_XATTR_SHARD_FILE_SIZE, &size_attr);
+    if (ret)
+        goto wind;
+
+    CHANGELOG_OP_BOUNDARY_CHECK(frame, wind);
+
+    CHANGELOG_INIT(this, frame->local, fd->inode, fd->inode->gfid, 1);
+
+    co = changelog_get_usable_buffer(frame->local);
+    if (!co)
+        goto wind;
+
+    CHANGLOG_FILL_FOP_NUMBER(co, frame->root->op, fop_fn, xtra_len);
+
+    changelog_set_usable_record_and_length(frame->local, xtra_len, 1);
+
+wind:
+    changelog_color_fop_and_inc_cnt(this, priv, frame->local);
+    STACK_WIND(frame, changelog_fxattrop_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->fxattrop, fd, optype, xattr, xdata);
+    return 0;
+}
+/* }}} */
 
 /* Data modification fops - TYPE I */
 
@@ -1486,164 +1619,151 @@ changelog_fsetxattr (call_frame_t *frame,
 /* {f}truncate() */
 
 int32_t
-changelog_truncate_cbk (call_frame_t *frame,
-                        void *cookie, xlator_t *this, int32_t op_ret,
-                        int32_t op_errno, struct iatt *prebuf,
-                        struct iatt *postbuf, dict_t *xdata)
+changelog_truncate_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                       int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
+                       struct iatt *postbuf, dict_t *xdata)
 {
-        changelog_priv_t  *priv  = NULL;
-        changelog_local_t *local = NULL;
+    changelog_priv_t *priv = NULL;
+    changelog_local_t *local = NULL;
 
-        priv  = this->private;
-        local = frame->local;
+    priv = this->private;
+    local = frame->local;
 
-        CHANGELOG_COND_GOTO (priv, ((op_ret < 0) || !local), unwind);
+    CHANGELOG_COND_GOTO(priv, ((op_ret < 0) || !local), unwind);
 
-        changelog_update (this, priv, local, CHANGELOG_TYPE_DATA);
+    changelog_update(this, priv, local, CHANGELOG_TYPE_DATA);
 
- unwind:
-        changelog_dec_fop_cnt (this, priv, local);
-        CHANGELOG_STACK_UNWIND (truncate, frame,
-                                op_ret, op_errno, prebuf, postbuf, xdata);
-        return 0;
+unwind:
+    changelog_dec_fop_cnt(this, priv, local);
+    CHANGELOG_STACK_UNWIND(truncate, frame, op_ret, op_errno, prebuf, postbuf,
+                           xdata);
+    return 0;
 }
 
 int32_t
-changelog_truncate (call_frame_t *frame,
-                    xlator_t *this, loc_t *loc, off_t offset, dict_t *xdata)
+changelog_truncate(call_frame_t *frame, xlator_t *this, loc_t *loc,
+                   off_t offset, dict_t *xdata)
 {
-        changelog_priv_t *priv = NULL;
+    changelog_priv_t *priv = NULL;
 
-        priv = this->private;
-        CHANGELOG_NOT_ACTIVE_THEN_GOTO (frame, priv, wind);
+    priv = this->private;
+    CHANGELOG_NOT_ACTIVE_THEN_GOTO(frame, priv, wind);
 
-        CHANGELOG_INIT (this, frame->local,
-                        loc->inode, loc->inode->gfid, 0);
-        LOCK(&priv->c_snap_lock);
-        {
-                if (priv->c_snap_fd != -1 &&
-                    priv->barrier_enabled == _gf_true) {
-                        changelog_snap_handle_ascii_change (this,
-                              &( ((changelog_local_t *)(frame->local))->cld));
-                }
+    CHANGELOG_INIT(this, frame->local, loc->inode, loc->inode->gfid, 0);
+    LOCK(&priv->c_snap_lock);
+    {
+        if (priv->c_snap_fd != -1 && priv->barrier_enabled == _gf_true) {
+            changelog_snap_handle_ascii_change(
+                this, &(((changelog_local_t *)(frame->local))->cld));
         }
-        UNLOCK(&priv->c_snap_lock);
+    }
+    UNLOCK(&priv->c_snap_lock);
 
-
- wind:
-        changelog_color_fop_and_inc_cnt (this, priv, frame->local);
-        STACK_WIND (frame, changelog_truncate_cbk,
-                    FIRST_CHILD (this), FIRST_CHILD (this)->fops->truncate,
-                    loc, offset, xdata);
-        return 0;
+wind:
+    changelog_color_fop_and_inc_cnt(this, priv, frame->local);
+    STACK_WIND(frame, changelog_truncate_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->truncate, loc, offset, xdata);
+    return 0;
 }
 
 int32_t
-changelog_ftruncate_cbk (call_frame_t *frame,
-                         void *cookie, xlator_t *this, int32_t op_ret,
-                         int32_t op_errno, struct iatt *prebuf,
-                         struct iatt *postbuf, dict_t *xdata)
+changelog_ftruncate_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                        int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
+                        struct iatt *postbuf, dict_t *xdata)
 {
-        changelog_priv_t  *priv  = NULL;
-        changelog_local_t *local = NULL;
+    changelog_priv_t *priv = NULL;
+    changelog_local_t *local = NULL;
 
-        priv  = this->private;
-        local = frame->local;
+    priv = this->private;
+    local = frame->local;
 
-        CHANGELOG_COND_GOTO (priv, ((op_ret < 0) || !local), unwind);
+    CHANGELOG_COND_GOTO(priv, ((op_ret < 0) || !local), unwind);
 
-        changelog_update (this, priv, local, CHANGELOG_TYPE_DATA);
+    changelog_update(this, priv, local, CHANGELOG_TYPE_DATA);
 
- unwind:
-        changelog_dec_fop_cnt (this, priv, local);
-        CHANGELOG_STACK_UNWIND (ftruncate, frame,
-                                op_ret, op_errno, prebuf, postbuf, xdata);
-        return 0;
+unwind:
+    changelog_dec_fop_cnt(this, priv, local);
+    CHANGELOG_STACK_UNWIND(ftruncate, frame, op_ret, op_errno, prebuf, postbuf,
+                           xdata);
+    return 0;
 }
 
 int32_t
-changelog_ftruncate (call_frame_t *frame,
-                     xlator_t *this, fd_t *fd, off_t offset, dict_t *xdata)
+changelog_ftruncate(call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
+                    dict_t *xdata)
 {
-        changelog_priv_t *priv = NULL;
+    changelog_priv_t *priv = NULL;
 
-        priv = this->private;
-        CHANGELOG_NOT_ACTIVE_THEN_GOTO (frame, priv, wind);
+    priv = this->private;
+    CHANGELOG_NOT_ACTIVE_THEN_GOTO(frame, priv, wind);
 
-        CHANGELOG_INIT (this, frame->local,
-                        fd->inode, fd->inode->gfid, 0);
-        LOCK(&priv->c_snap_lock);
-        {
-                if (priv->c_snap_fd != -1 &&
-                    priv->barrier_enabled == _gf_true) {
-                        changelog_snap_handle_ascii_change (this,
-                              &( ((changelog_local_t *)(frame->local))->cld));
-                }
+    CHANGELOG_INIT(this, frame->local, fd->inode, fd->inode->gfid, 0);
+    LOCK(&priv->c_snap_lock);
+    {
+        if (priv->c_snap_fd != -1 && priv->barrier_enabled == _gf_true) {
+            changelog_snap_handle_ascii_change(
+                this, &(((changelog_local_t *)(frame->local))->cld));
         }
-        UNLOCK(&priv->c_snap_lock);
+    }
+    UNLOCK(&priv->c_snap_lock);
 
- wind:
-        changelog_color_fop_and_inc_cnt (this, priv, frame->local);
-        STACK_WIND (frame, changelog_ftruncate_cbk,
-                    FIRST_CHILD (this), FIRST_CHILD (this)->fops->ftruncate,
-                    fd, offset, xdata);
-        return 0;
+wind:
+    changelog_color_fop_and_inc_cnt(this, priv, frame->local);
+    STACK_WIND(frame, changelog_ftruncate_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->ftruncate, fd, offset, xdata);
+    return 0;
 }
 
 /* writev() */
 
 int32_t
-changelog_writev_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                    int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
-                    struct iatt *postbuf,
-                    dict_t *xdata)
+changelog_writev_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                     int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
+                     struct iatt *postbuf, dict_t *xdata)
 {
-        changelog_priv_t  *priv  = NULL;
-        changelog_local_t *local = NULL;
+    changelog_priv_t *priv = NULL;
+    changelog_local_t *local = NULL;
 
-        priv  = this->private;
-        local = frame->local;
+    priv = this->private;
+    local = frame->local;
 
-        CHANGELOG_COND_GOTO (priv, ((op_ret <= 0) || !local), unwind);
+    CHANGELOG_COND_GOTO(priv, ((op_ret <= 0) || !local), unwind);
 
-        changelog_update (this, priv, local, CHANGELOG_TYPE_DATA);
+    changelog_update(this, priv, local, CHANGELOG_TYPE_DATA);
 
- unwind:
-        changelog_dec_fop_cnt (this, priv, local);
-        CHANGELOG_STACK_UNWIND (writev, frame,
-                                op_ret, op_errno, prebuf, postbuf, xdata);
-        return 0;
+unwind:
+    changelog_dec_fop_cnt(this, priv, local);
+    CHANGELOG_STACK_UNWIND(writev, frame, op_ret, op_errno, prebuf, postbuf,
+                           xdata);
+    return 0;
 }
 
 int32_t
-changelog_writev (call_frame_t *frame,
-                  xlator_t *this, fd_t *fd, struct iovec *vector,
-                  int32_t count, off_t offset, uint32_t flags,
-                  struct iobref *iobref, dict_t *xdata)
+changelog_writev(call_frame_t *frame, xlator_t *this, fd_t *fd,
+                 struct iovec *vector, int32_t count, off_t offset,
+                 uint32_t flags, struct iobref *iobref, dict_t *xdata)
 {
-        changelog_priv_t *priv = NULL;
+    changelog_priv_t *priv = NULL;
 
-        priv = this->private;
-        CHANGELOG_NOT_ACTIVE_THEN_GOTO (frame, priv, wind);
+    priv = this->private;
+    CHANGELOG_NOT_ACTIVE_THEN_GOTO(frame, priv, wind);
 
-        CHANGELOG_INIT (this, frame->local,
-                        fd->inode, fd->inode->gfid, 0);
-        LOCK(&priv->c_snap_lock);
-        {
-                if (priv->c_snap_fd != -1 &&
-                    priv->barrier_enabled == _gf_true) {
-                        changelog_snap_handle_ascii_change (this,
-                              &( ((changelog_local_t *)(frame->local))->cld));
-                }
+    CHANGELOG_INIT(this, frame->local, fd->inode, fd->inode->gfid, 0);
+    LOCK(&priv->c_snap_lock);
+    {
+        if (priv->c_snap_fd != -1 && priv->barrier_enabled == _gf_true) {
+            changelog_snap_handle_ascii_change(
+                this, &(((changelog_local_t *)(frame->local))->cld));
         }
-        UNLOCK(&priv->c_snap_lock);
+    }
+    UNLOCK(&priv->c_snap_lock);
 
- wind:
-        changelog_color_fop_and_inc_cnt (this, priv, frame->local);
-        STACK_WIND (frame, changelog_writev_cbk, FIRST_CHILD (this),
-                    FIRST_CHILD (this)->fops->writev, fd, vector,
-                    count, offset, flags, iobref, xdata);
-        return 0;
+wind:
+    changelog_color_fop_and_inc_cnt(this, priv, frame->local);
+    STACK_WIND(frame, changelog_writev_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->writev, fd, vector, count, offset,
+               flags, iobref, xdata);
+    return 0;
 }
 
 /* }}} */
@@ -1652,83 +1772,79 @@ changelog_writev (call_frame_t *frame,
 
 /* {{{ */
 
-
-
 int
-changelog_open_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                    int op_ret, int op_errno, fd_t *fd, dict_t *xdata)
-{
-        int                ret    = 0;
-        void              *opaque = NULL;
-        char              *buf    = NULL;
-        ssize_t            buflen = 0;
-        changelog_priv_t  *priv   = NULL;
-        changelog_event_t  ev     = {0,};
-        gf_boolean_t logopen = _gf_false;
-
-        priv = this->private;
-        if (frame->local) {
-                frame->local = NULL;
-                logopen = _gf_true;
-        }
-
-        CHANGELOG_COND_GOTO (priv, ((op_ret < 0) || !logopen), unwind);
-
-        /* fill the event structure */
-        ev.ev_type = CHANGELOG_OP_TYPE_OPEN;
-        gf_uuid_copy (ev.u.open.gfid, fd->inode->gfid);
-        ev.u.open.flags = fd->flags;
-        changelog_dispatch_event (this, priv, &ev);
-
-        if (changelog_ev_selected
-                   (this, &priv->ev_selection, CHANGELOG_OP_TYPE_RELEASE)) {
-                ret = fd_ctx_set (fd, this, (uint64_t)(long) 0x1);
-                if (ret)
-                        gf_log (this->name, GF_LOG_WARNING,
-                                "could not set fd context (for release cbk)");
-        }
+changelog_open_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                   int op_ret, int op_errno, fd_t *fd, dict_t *xdata)
+{
+    int ret = 0;
+    changelog_priv_t *priv = NULL;
+    changelog_event_t ev = {
+        0,
+    };
+    gf_boolean_t logopen = _gf_false;
+
+    priv = this->private;
+    if (frame->local) {
+        frame->local = NULL;
+        logopen = _gf_true;
+    }
+
+    CHANGELOG_COND_GOTO(priv, ((op_ret < 0) || !logopen), unwind);
+
+    /* fill the event structure */
+    ev.ev_type = CHANGELOG_OP_TYPE_OPEN;
+    gf_uuid_copy(ev.u.open.gfid, fd->inode->gfid);
+    ev.u.open.flags = fd->flags;
+    changelog_dispatch_event(this, priv, &ev);
+
+    if (changelog_ev_selected(this, &priv->ev_selection,
+                              CHANGELOG_OP_TYPE_RELEASE)) {
+        ret = fd_ctx_set(fd, this, (uint64_t)(long)0x1);
+        if (ret)
+            gf_smsg(this->name, GF_LOG_WARNING, 0, CHANGELOG_MSG_SET_FD_CONTEXT,
+                    NULL);
+    }
 
- unwind:
-        CHANGELOG_STACK_UNWIND (open, frame, op_ret, op_errno, fd, xdata);
-        return 0;
+unwind:
+    CHANGELOG_STACK_UNWIND(open, frame, op_ret, op_errno, fd, xdata);
+    return 0;
 }
 
 int
-changelog_open (call_frame_t *frame, xlator_t *this,
-                loc_t *loc, int flags, fd_t *fd, dict_t *xdata)
+changelog_open(call_frame_t *frame, xlator_t *this, loc_t *loc, int flags,
+               fd_t *fd, dict_t *xdata)
 {
-        changelog_priv_t *priv = NULL;
+    changelog_priv_t *priv = NULL;
 
-        priv = this->private;
-        CHANGELOG_NOT_ACTIVE_THEN_GOTO (frame, priv, wind);
+    priv = this->private;
+    CHANGELOG_NOT_ACTIVE_THEN_GOTO(frame, priv, wind);
 
-        frame->local = (void *)0x1; /* do not dereference in ->cbk */
+    frame->local = (void *)0x1; /* do not dereference in ->cbk */
 
- wind:
-        STACK_WIND (frame, changelog_open_cbk, FIRST_CHILD (this),
-                    FIRST_CHILD (this)->fops->open, loc, flags, fd, xdata);
-        return 0;
+wind:
+    STACK_WIND(frame, changelog_open_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->open, loc, flags, fd, xdata);
+    return 0;
 }
 
 /* }}} */
 
 /* {{{ */
 
-
 /* }}} */
 
 int32_t
-_changelog_generic_dispatcher (dict_t *dict,
-                               char *key, data_t *value, void *data)
+_changelog_generic_dispatcher(dict_t *dict, char *key, data_t *value,
+                              void *data)
 {
-        xlator_t *this = NULL;
-        changelog_priv_t *priv = NULL;
+    xlator_t *this = NULL;
+    changelog_priv_t *priv = NULL;
 
-        this = data;
-        priv = this->private;
+    this = data;
+    priv = this->private;
 
-        changelog_dispatch_event (this, priv, (changelog_event_t *)value->data);
-        return 0;
+    changelog_dispatch_event(this, priv, (changelog_event_t *)value->data);
+    return 0;
 }
 
 /**
@@ -1737,46 +1853,45 @@ _changelog_generic_dispatcher (dict_t *dict,
  * traverses the dictionary).
  */
 int32_t
-changelog_ipc (call_frame_t *frame, xlator_t *this, int32_t op, dict_t *xdata)
+changelog_ipc(call_frame_t *frame, xlator_t *this, int32_t op, dict_t *xdata)
 {
-        if (op != GF_IPC_TARGET_CHANGELOG)
-                goto wind;
+    if (op != GF_IPC_TARGET_CHANGELOG)
+        goto wind;
 
-        /* it's for us, do the job */
-        if (xdata)
-                (void) dict_foreach (xdata,
-                                     _changelog_generic_dispatcher, this);
+    /* it's for us, do the job */
+    if (xdata)
+        (void)dict_foreach(xdata, _changelog_generic_dispatcher, this);
 
-        STACK_UNWIND_STRICT (ipc, frame, 0, 0, NULL);
-        return 0;
+    STACK_UNWIND_STRICT(ipc, frame, 0, 0, NULL);
+    return 0;
 
- wind:
-        STACK_WIND (frame, default_ipc_cbk, FIRST_CHILD (this),
-                    FIRST_CHILD (this)->fops->ipc, op, xdata);
-        return 0;
+wind:
+    STACK_WIND(frame, default_ipc_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->ipc, op, xdata);
+    return 0;
 }
 
-
 /* {{{ */
 
 int32_t
-changelog_release (xlator_t *this, fd_t *fd)
+changelog_release(xlator_t *this, fd_t *fd)
 {
-        changelog_event_t ev = {0,};
-        changelog_priv_t *priv = NULL;
+    changelog_event_t ev = {
+        0,
+    };
+    changelog_priv_t *priv = NULL;
 
-        priv = this->private;
+    priv = this->private;
 
-        ev.ev_type = CHANGELOG_OP_TYPE_RELEASE;
-        gf_uuid_copy (ev.u.release.gfid, fd->inode->gfid);
-        changelog_dispatch_event (this, priv, &ev);
+    ev.ev_type = CHANGELOG_OP_TYPE_RELEASE;
+    gf_uuid_copy(ev.u.release.gfid, fd->inode->gfid);
+    changelog_dispatch_event(this, priv, &ev);
 
-        (void) fd_ctx_del (fd, this, NULL);
+    (void)fd_ctx_del(fd, this, NULL);
 
-        return 0;
+    return 0;
 }
 
-
 /* }}} */
 
 /**
@@ -1791,930 +1906,1084 @@ changelog_release (xlator_t *this, fd_t *fd)
  * needed if there are more operation modes in the future.
  */
 static void
-changelog_assign_opmode (changelog_priv_t *priv, char *mode)
+changelog_assign_opmode(changelog_priv_t *priv, char *mode)
 {
-        if ( strncmp (mode, "realtime", 8) == 0 ) {
-                priv->op_mode = CHANGELOG_MODE_RT;
-        }
+    if (strncmp(mode, "realtime", 8) == 0) {
+        priv->op_mode = CHANGELOG_MODE_RT;
+    }
 }
 
 static void
-changelog_assign_encoding (changelog_priv_t *priv, char *enc)
+changelog_assign_encoding(changelog_priv_t *priv, char *enc)
 {
-        if ( strncmp (enc, "binary", 6) == 0 ) {
-                priv->encode_mode = CHANGELOG_ENCODE_BINARY;
-        } else if ( strncmp (enc, "ascii", 5) == 0 ) {
-                priv->encode_mode = CHANGELOG_ENCODE_ASCII;
-        }
+    if (strncmp(enc, "binary", 6) == 0) {
+        priv->encode_mode = CHANGELOG_ENCODE_BINARY;
+    } else if (strncmp(enc, "ascii", 5) == 0) {
+        priv->encode_mode = CHANGELOG_ENCODE_ASCII;
+    }
 }
 
 static void
 changelog_assign_barrier_timeout(changelog_priv_t *priv, uint32_t timeout)
 {
-       LOCK (&priv->lock);
-       {
-               priv->timeout.tv_sec = timeout;
-       }
-       UNLOCK (&priv->lock);
+    LOCK(&priv->lock);
+    {
+        priv->timeout.tv_sec = timeout;
+    }
+    UNLOCK(&priv->lock);
 }
 
 /* cleanup any helper threads that are running */
 static void
-changelog_cleanup_helper_threads (xlator_t *this, changelog_priv_t *priv)
-{
-        int ret = 0;
-
-        if (priv->cr.rollover_th) {
-                (void) changelog_thread_cleanup (this, priv->cr.rollover_th);
-                priv->cr.rollover_th = 0;
-                ret = close (priv->cr_wfd);
-                if (ret)
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "error closing write end of rollover pipe"
-                                " (reason: %s)", strerror (errno));
-        }
+changelog_cleanup_helper_threads(xlator_t *this, changelog_priv_t *priv)
+{
+    if (priv->cr.rollover_th) {
+        (void)changelog_thread_cleanup(this, priv->cr.rollover_th);
+        priv->cr.rollover_th = 0;
+    }
 
-        if (priv->cf.fsync_th) {
-                (void) changelog_thread_cleanup (this, priv->cf.fsync_th);
-                priv->cf.fsync_th = 0;
-        }
+    if (priv->cf.fsync_th) {
+        (void)changelog_thread_cleanup(this, priv->cf.fsync_th);
+        priv->cf.fsync_th = 0;
+    }
 }
 
 /* spawn helper thread; cleaning up in case of errors */
 static int
-changelog_spawn_helper_threads (xlator_t *this, changelog_priv_t *priv)
-{
-        int ret = 0;
-        int flags = 0;
-        int pipe_fd[2] = {0, 0};
-
-        /* Geo-Rep snapshot dependency:
-         *
-         * To implement explicit rollover of changlog journal on barrier
-         * notification, a pipe is created to communicate between
-         * 'changelog_rollover' thread and changelog main thread. The select
-         * call used to wait till roll-over time in changelog_rollover thread
-         * is modified to wait on read end of the pipe. When barrier
-         * notification comes (i.e, in 'reconfigure'), select in
-         * changelog_rollover thread is woken up explicitly by writing into
-         * the write end of the pipe in 'reconfigure'.
-         */
-
-        ret = pipe (pipe_fd);
-        if (ret == -1) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "Cannot create pipe (reason: %s)", strerror (errno));
-                goto out;
-        }
-
-        /* writer is non-blocking */
-        flags = fcntl (pipe_fd[1], F_GETFL);
-        flags |= O_NONBLOCK;
-
-        ret = fcntl (pipe_fd[1], F_SETFL, flags);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "failed to set O_NONBLOCK flag");
-                goto out;
-        }
-
-        priv->cr_wfd = pipe_fd[1];
-        priv->cr.rfd = pipe_fd[0];
-
-        priv->cr.this = this;
-        ret = gf_thread_create (&priv->cr.rollover_th,
-				NULL, changelog_rollover, priv);
-        if (ret)
-                goto out;
-
-        if (priv->fsync_interval) {
-                priv->cf.this = this;
-                ret = gf_thread_create (&priv->cf.fsync_th,
-					NULL, changelog_fsync_thread, priv);
-        }
-
-        if (ret)
-                changelog_cleanup_helper_threads (this, priv);
+changelog_spawn_helper_threads(xlator_t *this, changelog_priv_t *priv)
+{
+    int ret = 0;
+
+    /* Geo-Rep snapshot dependency:
+     *
+     * To implement explicit rollover of changlog journal on barrier
+     * notification, a pipe is created to communicate between
+     * 'changelog_rollover' thread and changelog main thread. The select
+     * call used to wait till roll-over time in changelog_rollover thread
+     * is modified to wait on read end of the pipe. When barrier
+     * notification comes (i.e, in 'reconfigure'), select in
+     * changelog_rollover thread is woken up explicitly by writing into
+     * the write end of the pipe in 'reconfigure'.
+     */
+
+    priv->cr.notify = _gf_false;
+    priv->cr.this = this;
+    ret = gf_thread_create(&priv->cr.rollover_th, NULL, changelog_rollover,
+                           priv, "clogro");
+    if (ret)
+        goto out;
+
+    if (priv->fsync_interval) {
+        priv->cf.this = this;
+        ret = gf_thread_create(&priv->cf.fsync_th, NULL, changelog_fsync_thread,
+                               priv, "clogfsyn");
+    }
+
+    if (ret)
+        changelog_cleanup_helper_threads(this, priv);
 
- out:
-        return ret;
+out:
+    return ret;
 }
 
 int
-notify (xlator_t *this, int event, void *data, ...)
-{
-        changelog_priv_t       *priv            = NULL;
-        dict_t                 *dict            = NULL;
-        char                    buf[1]          = {1};
-        int                     barrier         = DICT_DEFAULT;
-        gf_boolean_t            bclean_req      = _gf_false;
-        int                     ret             = 0;
-        struct list_head        queue           = {0, };
+notify(xlator_t *this, int event, void *data, ...)
+{
+    changelog_priv_t *priv = NULL;
+    dict_t *dict = NULL;
+    char buf[1] = {1};
+    int barrier = DICT_DEFAULT;
+    gf_boolean_t bclean_req = _gf_false;
+    int ret = 0;
+    int ret1 = 0;
+    struct list_head queue = {
+        0,
+    };
+    uint64_t xprtcnt = 0;
+    uint64_t clntcnt = 0;
+    changelog_clnt_t *conn = NULL;
+    gf_boolean_t cleanup_notify = _gf_false;
+    char sockfile[UNIX_PATH_MAX] = {
+        0,
+    };
+    rpcsvc_listener_t *listener = NULL;
+    rpcsvc_listener_t *next = NULL;
+
+    INIT_LIST_HEAD(&queue);
+
+    priv = this->private;
+    if (!priv)
+        goto out;
+
+    if (event == GF_EVENT_PARENT_DOWN) {
+        priv->victim = data;
+        gf_log(this->name, GF_LOG_INFO,
+               "cleanup changelog rpc connection of brick %s",
+               priv->victim->name);
+
+        if (priv->rpc_active) {
+            this->cleanup_starting = 1;
+            changelog_destroy_rpc_listner(this, priv);
+            conn = &priv->connections;
+            if (conn)
+                changelog_ev_cleanup_connections(this, conn);
+            xprtcnt = GF_ATOMIC_GET(priv->xprtcnt);
+            clntcnt = GF_ATOMIC_GET(priv->clntcnt);
+            if (!xprtcnt && !clntcnt) {
+                LOCK(&priv->lock);
+                {
+                    cleanup_notify = priv->notify_down;
+                    priv->notify_down = _gf_true;
+                }
+                UNLOCK(&priv->lock);
+                if (priv->rpc) {
+                    list_for_each_entry_safe(listener, next,
+                                             &priv->rpc->listeners, list)
+                    {
+                        if (listener->trans) {
+                            rpc_transport_unref(listener->trans);
+                        }
+                    }
+                    rpcsvc_destroy(priv->rpc);
+                    priv->rpc = NULL;
+                }
+                CHANGELOG_MAKE_SOCKET_PATH(priv->changelog_brick, sockfile,
+                                           UNIX_PATH_MAX);
+                sys_unlink(sockfile);
+                if (!cleanup_notify)
+                    default_notify(this, GF_EVENT_PARENT_DOWN, data);
+            }
+        } else {
+            default_notify(this, GF_EVENT_PARENT_DOWN, data);
+        }
+        goto out;
+    }
 
-        INIT_LIST_HEAD (&queue);
+    if (event == GF_EVENT_TRANSLATOR_OP) {
+        dict = data;
 
-        priv = this->private;
-        if (!priv)
-                goto out;
+        barrier = dict_get_str_boolean(dict, "barrier", DICT_DEFAULT);
 
-        if (event == GF_EVENT_TRANSLATOR_OP) {
+        switch (barrier) {
+            case DICT_ERROR:
+                gf_smsg(this->name, GF_LOG_ERROR, 0,
+                        CHANGELOG_MSG_DICT_GET_FAILED, "dict_get_str_boolean",
+                        NULL);
+                ret = -1;
+                goto out;
 
-                dict = data;
+            case BARRIER_OFF:
+                gf_smsg(this->name, GF_LOG_INFO, 0,
+                        CHANGELOG_MSG_BARRIER_STATE_NOTIFY, "off", NULL);
 
-                barrier = dict_get_str_boolean (dict, "barrier", DICT_DEFAULT);
+                CHANGELOG_NOT_ON_THEN_GOTO(priv, ret, out);
+                LOCK(&priv->c_snap_lock);
+                {
+                    changelog_snap_logging_stop(this, priv);
+                }
+                UNLOCK(&priv->c_snap_lock);
 
-                switch (barrier) {
-                case DICT_ERROR:
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "Barrier dict_get_str_boolean failed");
+                LOCK(&priv->bflags.lock);
+                {
+                    if (priv->bflags.barrier_ext == _gf_false)
                         ret = -1;
-                        goto out;
-
-                case BARRIER_OFF:
-                        gf_log (this->name, GF_LOG_INFO,
-                                "Barrier off notification");
+                }
+                UNLOCK(&priv->bflags.lock);
 
-                        CHANGELOG_NOT_ON_THEN_GOTO(priv, ret, out);
-                        LOCK(&priv->c_snap_lock);
-                        {
-                                changelog_snap_logging_stop (this, priv);
-                        }
-                        UNLOCK(&priv->c_snap_lock);
+                if (ret == -1) {
+                    gf_smsg(this->name, GF_LOG_ERROR, 0,
+                            CHANGELOG_MSG_BARRIER_ERROR, NULL);
+                    goto out;
+                }
 
-                        LOCK (&priv->bflags.lock);
-                        {
-                                if (priv->bflags.barrier_ext == _gf_false)
-                                        ret = -1;
-                        }
-                        UNLOCK (&priv->bflags.lock);
+                /* Stop changelog barrier and dequeue all fops */
+                LOCK(&priv->lock);
+                {
+                    if (priv->barrier_enabled == _gf_true)
+                        __chlog_barrier_disable(this, &queue);
+                    else
+                        ret = -1;
+                }
+                UNLOCK(&priv->lock);
+                /* If ret = -1, then changelog barrier is already
+                 * disabled because of error or timeout.
+                 */
+                if (ret == 0) {
+                    chlog_barrier_dequeue_all(this, &queue);
+                    gf_smsg(this->name, GF_LOG_INFO, 0,
+                            CHANGELOG_MSG_BARRIER_DISABLED, NULL);
+                } else {
+                    gf_smsg(this->name, GF_LOG_ERROR, 0,
+                            CHANGELOG_MSG_BARRIER_ALREADY_DISABLED, NULL);
+                }
 
-                        if (ret == -1 ) {
-                                gf_log (this->name, GF_LOG_ERROR, "Received"
-                                        " another barrier off notification"
-                                        " while already off");
-                                goto out;
-                        }
+                LOCK(&priv->bflags.lock);
+                {
+                    priv->bflags.barrier_ext = _gf_false;
+                }
+                UNLOCK(&priv->bflags.lock);
 
-                        /* Stop changelog barrier and dequeue all fops */
-                        LOCK (&priv->lock);
-                        {
-                                if (priv->barrier_enabled == _gf_true)
-                                        __chlog_barrier_disable (this, &queue);
-                                else
-                                        ret = -1;
-                        }
-                        UNLOCK (&priv->lock);
-                        /* If ret = -1, then changelog barrier is already
-                         * disabled because of error or timeout.
-                         */
-                        if (ret == 0) {
-                                chlog_barrier_dequeue_all(this, &queue);
-                                gf_log(this->name, GF_LOG_INFO,
-                                       "Disabled changelog barrier");
-                        } else {
-                                gf_log (this->name, GF_LOG_ERROR,
-                                        "Changelog barrier already disabled");
-                        }
+                goto out;
 
-                        LOCK (&priv->bflags.lock);
-                        {
-                                priv->bflags.barrier_ext = _gf_false;
-                        }
-                        UNLOCK (&priv->bflags.lock);
+            case BARRIER_ON:
+                gf_smsg(this->name, GF_LOG_INFO, 0,
+                        CHANGELOG_MSG_BARRIER_STATE_NOTIFY, "on", NULL);
 
-                        goto out;
+                CHANGELOG_NOT_ON_THEN_GOTO(priv, ret, out);
+                LOCK(&priv->c_snap_lock);
+                {
+                    changelog_snap_logging_start(this, priv);
+                }
+                UNLOCK(&priv->c_snap_lock);
 
-                case BARRIER_ON:
-                        gf_log (this->name, GF_LOG_INFO,
-                                "Barrier on notification");
+                LOCK(&priv->bflags.lock);
+                {
+                    if (priv->bflags.barrier_ext == _gf_true)
+                        ret = -1;
+                    else
+                        priv->bflags.barrier_ext = _gf_true;
+                }
+                UNLOCK(&priv->bflags.lock);
 
-                        CHANGELOG_NOT_ON_THEN_GOTO(priv, ret, out);
-                        LOCK(&priv->c_snap_lock);
-                        {
-                                changelog_snap_logging_start (this, priv);
-                        }
-                        UNLOCK(&priv->c_snap_lock);
-
-                        LOCK (&priv->bflags.lock);
-                        {
-                                if (priv->bflags.barrier_ext == _gf_true)
-                                        ret = -1;
-                                else
-                                        priv->bflags.barrier_ext = _gf_true;
-                        }
-                        UNLOCK (&priv->bflags.lock);
+                if (ret == -1) {
+                    gf_smsg(this->name, GF_LOG_ERROR, 0,
+                            CHANGELOG_MSG_BARRIER_ON_ERROR, NULL);
+                    goto out;
+                }
 
-                        if (ret == -1 ) {
-                                gf_log (this->name, GF_LOG_ERROR, "Received"
-                                        " another barrier on notification when"
-                                        " last one is not served yet");
-                                goto out;
-                        }
+                ret = pthread_mutex_lock(&priv->bn.bnotify_mutex);
+                CHANGELOG_PTHREAD_ERROR_HANDLE_1(ret, out, bclean_req);
+                {
+                    priv->bn.bnotify = _gf_true;
+                }
+                ret = pthread_mutex_unlock(&priv->bn.bnotify_mutex);
+                CHANGELOG_PTHREAD_ERROR_HANDLE_1(ret, out, bclean_req);
 
-                        ret = pthread_mutex_lock (&priv->bn.bnotify_mutex);
-                        CHANGELOG_PTHREAD_ERROR_HANDLE_1 (ret, out,
-                                                                    bclean_req);
-                        {
-                                priv->bn.bnotify = _gf_true;
-                        }
-                        ret = pthread_mutex_unlock (&priv->bn.bnotify_mutex);
-                        CHANGELOG_PTHREAD_ERROR_HANDLE_1 (ret, out,
-                                                                    bclean_req);
-
-                        /* Start changelog barrier */
-                        LOCK (&priv->lock);
-                        {
-                                ret = __chlog_barrier_enable (this, priv);
-                        }
-                        UNLOCK (&priv->lock);
-                        if (ret == -1) {
-                                changelog_barrier_cleanup (this, priv, &queue);
-                                goto out;
-                        }
+                /* Start changelog barrier */
+                LOCK(&priv->lock);
+                {
+                    ret = __chlog_barrier_enable(this, priv);
+                }
+                UNLOCK(&priv->lock);
+                if (ret == -1) {
+                    changelog_barrier_cleanup(this, priv, &queue);
+                    goto out;
+                }
 
-                        gf_log(this->name, GF_LOG_INFO,
-                                           "Enabled changelog barrier");
+                gf_smsg(this->name, GF_LOG_INFO, 0,
+                        CHANGELOG_MSG_BARRIER_ENABLE, NULL);
 
-                        ret = changelog_barrier_notify(priv, buf);
-                        if (ret) {
-                                gf_log (this->name, GF_LOG_ERROR,
-                                        "Explicit roll over: write failed");
-                                changelog_barrier_cleanup (this, priv, &queue);
-                                ret = -1;
-                                goto out;
-                        }
+                ret = changelog_barrier_notify(priv, buf);
+                if (ret) {
+                    gf_smsg(this->name, GF_LOG_ERROR, 0,
+                            CHANGELOG_MSG_WRITE_FAILED, "Explicit roll over",
+                            NULL);
+                    changelog_barrier_cleanup(this, priv, &queue);
+                    ret = -1;
+                    goto out;
+                }
 
-                        ret = pthread_mutex_lock (&priv->bn.bnotify_mutex);
-                        CHANGELOG_PTHREAD_ERROR_HANDLE_1 (ret, out,
-                                                                    bclean_req);
-                        {
-                                /* The while condition check is required here to
-                                 * handle spurious wakeup of cond wait that can
-                                 * happen with pthreads. See man page */
-                                while (priv->bn.bnotify == _gf_true) {
-                                        ret = pthread_cond_wait (
-                                                       &priv->bn.bnotify_cond,
-                                                       &priv->bn.bnotify_mutex);
-                                        CHANGELOG_PTHREAD_ERROR_HANDLE_1 (ret,
-                                                                          out,
-                                                                    bclean_req);
-                                }
-                        }
-                        ret = pthread_mutex_unlock (&priv->bn.bnotify_mutex);
-                        CHANGELOG_PTHREAD_ERROR_HANDLE_1 (ret, out, bclean_req);
-                        gf_log (this->name, GF_LOG_INFO,
-                                "Woke up: bnotify conditional wait");
+                ret = pthread_mutex_lock(&priv->bn.bnotify_mutex);
+                CHANGELOG_PTHREAD_ERROR_HANDLE_1(ret, out, bclean_req);
+                {
+                    /* The while condition check is required here to
+                     * handle spurious wakeup of cond wait that can
+                     * happen with pthreads. See man page */
+                    while (priv->bn.bnotify == _gf_true) {
+                        ret = pthread_cond_wait(&priv->bn.bnotify_cond,
+                                                &priv->bn.bnotify_mutex);
+                        CHANGELOG_PTHREAD_ERROR_HANDLE_1(ret, out, bclean_req);
+                    }
+                    if (priv->bn.bnotify_error == _gf_true) {
+                        ret = -1;
+                        priv->bn.bnotify_error = _gf_false;
+                    }
+                }
+                ret1 = pthread_mutex_unlock(&priv->bn.bnotify_mutex);
+                CHANGELOG_PTHREAD_ERROR_HANDLE_1(ret1, out, bclean_req);
+                gf_smsg(this->name, GF_LOG_INFO, 0,
+                        CHANGELOG_MSG_BNOTIFY_COND_INFO, NULL);
 
-                        ret = 0;
-                        goto out;
+                goto out;
 
-                case DICT_DEFAULT:
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "barrier key not found");
-                        ret = -1;
-                        goto out;
+            case DICT_DEFAULT:
+                gf_smsg(this->name, GF_LOG_ERROR, 0,
+                        CHANGELOG_MSG_BARRIER_KEY_NOT_FOUND, NULL);
+                ret = -1;
+                goto out;
 
-                default:
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "Something went bad in dict_get_str_boolean");
-                        ret = -1;
-                        goto out;
-                }
-        } else {
-                ret = default_notify (this, event, data);
+            default:
+                gf_smsg(this->name, GF_LOG_ERROR, EINVAL,
+                        CHANGELOG_MSG_ERROR_IN_DICT_GET, NULL);
+                ret = -1;
+                goto out;
         }
+    } else {
+        ret = default_notify(this, event, data);
+    }
 
- out:
-        if (bclean_req)
-                changelog_barrier_cleanup (this, priv, &queue);
+out:
+    if (bclean_req)
+        changelog_barrier_cleanup(this, priv, &queue);
 
-        return ret;
+    return ret;
 }
 
 int32_t
-mem_acct_init (xlator_t *this)
+mem_acct_init(xlator_t *this)
 {
-        int     ret = -1;
-
-        if (!this)
-                return ret;
+    int ret = -1;
 
-        ret = xlator_mem_acct_init (this, gf_changelog_mt_end + 1);
+    if (!this)
+        return ret;
 
-        if (ret != 0) {
-                gf_log (this->name, GF_LOG_WARNING, "Memory accounting"
-                        " init failed");
-                return ret;
-        }
+    ret = xlator_mem_acct_init(this, gf_changelog_mt_end + 1);
 
+    if (ret != 0) {
+        gf_smsg(this->name, GF_LOG_WARNING, ENOMEM,
+                CHANGELOG_MSG_MEMORY_INIT_FAILED, NULL);
         return ret;
+    }
+
+    return ret;
 }
 
 static int
-changelog_init (xlator_t *this, changelog_priv_t *priv)
+changelog_init(xlator_t *this, changelog_priv_t *priv)
 {
-        int                  i   = 0;
-        int                  ret = -1;
-        struct timeval       tv  = {0,};
-        changelog_log_data_t cld = {0,};
-
-        ret = gettimeofday (&tv, NULL);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "gettimeofday() failure");
-                goto out;
-        }
+    int i = 0;
+    int ret = 0;
+    changelog_log_data_t cld = {
+        0,
+    };
 
-        priv->slice.tv_start = tv;
+    priv->maps[CHANGELOG_TYPE_DATA] = "D ";
+    priv->maps[CHANGELOG_TYPE_METADATA] = "M ";
+    priv->maps[CHANGELOG_TYPE_METADATA_XATTR] = "M ";
+    priv->maps[CHANGELOG_TYPE_ENTRY] = "E ";
 
-        priv->maps[CHANGELOG_TYPE_DATA]     = "D ";
-        priv->maps[CHANGELOG_TYPE_METADATA] = "M ";
-        priv->maps[CHANGELOG_TYPE_ENTRY]    = "E ";
+    for (; i < CHANGELOG_MAX_TYPE; i++) {
+        /* start with version 1 */
+        priv->slice.changelog_version[i] = 1;
+    }
 
-        for (; i < CHANGELOG_MAX_TYPE; i++) {
-                /* start with version 1 */
-                priv->slice.changelog_version[i] = 1;
-        }
-
-        if (!priv->active)
-                return ret;
+    if (!priv->active)
+        return ret;
 
-        /**
-         * start with a fresh changelog file every time. this is done
-         * in case there was an encoding change. so... things are kept
-         * simple here.
-         */
-        ret = changelog_fill_rollover_data (&cld, _gf_false);
-        if(ret)
-                goto out;
+    /**
+     * start with a fresh changelog file every time. this is done
+     * in case there was an encoding change. so... things are kept
+     * simple here.
+     */
+    changelog_fill_rollover_data(&cld, _gf_false);
 
-        ret = htime_open (this, priv, cld.cld_roll_time);
-        /* call htime open with cld's rollover_time */
-        if (ret)
-                goto out;
+    ret = htime_open(this, priv, cld.cld_roll_time);
+    /* call htime open with cld's rollover_time */
+    if (ret)
+        goto out;
 
-        LOCK (&priv->lock);
-        {
-                ret = changelog_inject_single_event (this, priv, &cld);
-        }
-        UNLOCK (&priv->lock);
+    LOCK(&priv->lock);
+    {
+        ret = changelog_inject_single_event(this, priv, &cld);
+    }
+    UNLOCK(&priv->lock);
 
-        /* ... and finally spawn the helpers threads */
-        ret = changelog_spawn_helper_threads (this, priv);
+    /* ... and finally spawn the helpers threads */
+    ret = changelog_spawn_helper_threads(this, priv);
 
- out:
-        return ret;
+out:
+    return ret;
 }
 
 /**
  * Init barrier related condition variables and locks
  */
 static int
-changelog_barrier_pthread_init (xlator_t *this, changelog_priv_t *priv)
-{
-        gf_boolean_t    bn_mutex_init         = _gf_false;
-        gf_boolean_t    bn_cond_init          = _gf_false;
-        gf_boolean_t    dm_mutex_black_init   = _gf_false;
-        gf_boolean_t    dm_cond_black_init    = _gf_false;
-        gf_boolean_t    dm_mutex_white_init   = _gf_false;
-        gf_boolean_t    dm_cond_white_init    = _gf_false;
-        int             ret                   = 0;
-
-        if ((ret = pthread_mutex_init(&priv->bn.bnotify_mutex, NULL)) != 0) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "bnotify pthread_mutex_init failed (%d)", ret);
-                ret = -1;
-                goto out;
-        }
-        bn_mutex_init = _gf_true;
-
-        if ((ret = pthread_cond_init(&priv->bn.bnotify_cond, NULL)) != 0) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "bnotify pthread_cond_init failed (%d)", ret);
-                ret = -1;
-                goto out;
-        }
-        bn_cond_init = _gf_true;
-
-        if ((ret = pthread_mutex_init(&priv->dm.drain_black_mutex, NULL)) != 0)
-        {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "drain_black pthread_mutex_init failed (%d)", ret);
-                ret = -1;
-                goto out;
-        }
-        dm_mutex_black_init = _gf_true;
-
-        if ((ret = pthread_cond_init(&priv->dm.drain_black_cond, NULL)) != 0) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "drain_black pthread_cond_init failed (%d)", ret);
-                ret = -1;
-                goto out;
-        }
-        dm_cond_black_init = _gf_true;
-
-        if ((ret = pthread_mutex_init(&priv->dm.drain_white_mutex, NULL)) != 0)
-        {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "drain_white pthread_mutex_init failed (%d)", ret);
-                ret = -1;
-                goto out;
-        }
-        dm_mutex_white_init = _gf_true;
-
-        if ((ret = pthread_cond_init(&priv->dm.drain_white_cond, NULL)) != 0) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "drain_white pthread_cond_init failed (%d)", ret);
-                ret = -1;
-                goto out;
-        }
-        dm_cond_white_init = _gf_true;
- out:
-        if (ret) {
-                if (bn_mutex_init)
-                        pthread_mutex_destroy(&priv->bn.bnotify_mutex);
-                if (bn_cond_init)
-                        pthread_cond_destroy (&priv->bn.bnotify_cond);
-                if (dm_mutex_black_init)
-                        pthread_mutex_destroy(&priv->dm.drain_black_mutex);
-                if (dm_cond_black_init)
-                        pthread_cond_destroy (&priv->dm.drain_black_cond);
-                if (dm_mutex_white_init)
-                        pthread_mutex_destroy(&priv->dm.drain_white_mutex);
-                if (dm_cond_white_init)
-                        pthread_cond_destroy (&priv->dm.drain_white_cond);
-        }
-        return ret;
+changelog_barrier_pthread_init(xlator_t *this, changelog_priv_t *priv)
+{
+    gf_boolean_t bn_mutex_init = _gf_false;
+    gf_boolean_t bn_cond_init = _gf_false;
+    gf_boolean_t dm_mutex_black_init = _gf_false;
+    gf_boolean_t dm_cond_black_init = _gf_false;
+    gf_boolean_t dm_mutex_white_init = _gf_false;
+    gf_boolean_t dm_cond_white_init = _gf_false;
+    gf_boolean_t cr_mutex_init = _gf_false;
+    gf_boolean_t cr_cond_init = _gf_false;
+    int ret = 0;
+
+    if ((ret = pthread_mutex_init(&priv->bn.bnotify_mutex, NULL)) != 0) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno,
+                CHANGELOG_MSG_PTHREAD_MUTEX_INIT_FAILED, "name=bnotify",
+                "ret=%d", ret, NULL);
+        ret = -1;
+        goto out;
+    }
+    bn_mutex_init = _gf_true;
+
+    if ((ret = pthread_cond_init(&priv->bn.bnotify_cond, NULL)) != 0) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno,
+                CHANGELOG_MSG_PTHREAD_COND_INIT_FAILED, "name=bnotify",
+                "ret=%d", ret, NULL);
+        ret = -1;
+        goto out;
+    }
+    bn_cond_init = _gf_true;
+
+    if ((ret = pthread_mutex_init(&priv->dm.drain_black_mutex, NULL)) != 0) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno,
+                CHANGELOG_MSG_PTHREAD_MUTEX_INIT_FAILED, "name=drain_black",
+                "ret=%d", ret, NULL);
+        ret = -1;
+        goto out;
+    }
+    dm_mutex_black_init = _gf_true;
+
+    if ((ret = pthread_cond_init(&priv->dm.drain_black_cond, NULL)) != 0) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno,
+                CHANGELOG_MSG_PTHREAD_COND_INIT_FAILED, "name=drain_black",
+                "ret=%d", ret, NULL);
+        ret = -1;
+        goto out;
+    }
+    dm_cond_black_init = _gf_true;
+
+    if ((ret = pthread_mutex_init(&priv->dm.drain_white_mutex, NULL)) != 0) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno,
+                CHANGELOG_MSG_PTHREAD_MUTEX_INIT_FAILED, "name=drain_white",
+                "ret=%d", ret, NULL);
+        ret = -1;
+        goto out;
+    }
+    dm_mutex_white_init = _gf_true;
+
+    if ((ret = pthread_cond_init(&priv->dm.drain_white_cond, NULL)) != 0) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno,
+                CHANGELOG_MSG_PTHREAD_COND_INIT_FAILED, "name=drain_white",
+                "ret=%d", ret, NULL);
+        ret = -1;
+        goto out;
+    }
+    dm_cond_white_init = _gf_true;
+
+    if ((pthread_mutex_init(&priv->cr.lock, NULL)) != 0) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno,
+                CHANGELOG_MSG_PTHREAD_MUTEX_INIT_FAILED,
+                "name=changelog_rollover", "ret=%d", ret, NULL);
+        ret = -1;
+        goto out;
+    }
+    cr_mutex_init = _gf_true;
+
+    if ((pthread_cond_init(&priv->cr.cond, NULL)) != 0) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno,
+                CHANGELOG_MSG_PTHREAD_COND_INIT_FAILED,
+                "changelog_rollover cond init failed", "ret=%d", ret, NULL);
+        ret = -1;
+        goto out;
+    }
+    cr_cond_init = _gf_true;
+out:
+    if (ret) {
+        if (bn_mutex_init)
+            pthread_mutex_destroy(&priv->bn.bnotify_mutex);
+        if (bn_cond_init)
+            pthread_cond_destroy(&priv->bn.bnotify_cond);
+        if (dm_mutex_black_init)
+            pthread_mutex_destroy(&priv->dm.drain_black_mutex);
+        if (dm_cond_black_init)
+            pthread_cond_destroy(&priv->dm.drain_black_cond);
+        if (dm_mutex_white_init)
+            pthread_mutex_destroy(&priv->dm.drain_white_mutex);
+        if (dm_cond_white_init)
+            pthread_cond_destroy(&priv->dm.drain_white_cond);
+        if (cr_mutex_init)
+            pthread_mutex_destroy(&priv->cr.lock);
+        if (cr_cond_init)
+            pthread_cond_destroy(&priv->cr.cond);
+    }
+    return ret;
 }
 
 /* Destroy barrier related condition variables and locks */
-static inline void
-changelog_barrier_pthread_destroy (changelog_priv_t *priv)
+static void
+changelog_barrier_pthread_destroy(changelog_priv_t *priv)
 {
-        pthread_mutex_destroy (&priv->bn.bnotify_mutex);
-        pthread_cond_destroy (&priv->bn.bnotify_cond);
-        pthread_mutex_destroy (&priv->dm.drain_black_mutex);
-        pthread_cond_destroy (&priv->dm.drain_black_cond);
-        pthread_mutex_destroy (&priv->dm.drain_white_mutex);
-        pthread_cond_destroy (&priv->dm.drain_white_cond);
-        LOCK_DESTROY (&priv->bflags.lock);
+    pthread_mutex_destroy(&priv->bn.bnotify_mutex);
+    pthread_cond_destroy(&priv->bn.bnotify_cond);
+    pthread_mutex_destroy(&priv->dm.drain_black_mutex);
+    pthread_cond_destroy(&priv->dm.drain_black_cond);
+    pthread_mutex_destroy(&priv->dm.drain_white_mutex);
+    pthread_cond_destroy(&priv->dm.drain_white_cond);
+    pthread_mutex_destroy(&priv->cr.lock);
+    pthread_cond_destroy(&priv->cr.cond);
+    LOCK_DESTROY(&priv->bflags.lock);
 }
 
-int
-reconfigure (xlator_t *this, dict_t *options)
-{
-        int                     ret            = 0;
-        char                   *tmp            = NULL;
-        changelog_priv_t       *priv           = NULL;
-        gf_boolean_t            active_earlier = _gf_true;
-        gf_boolean_t            active_now     = _gf_true;
-        changelog_time_slice_t *slice          = NULL;
-        changelog_log_data_t    cld            = {0,};
-        char    htime_dir[PATH_MAX]            = {0,};
-        char    csnap_dir[PATH_MAX]            = {0,};
-        struct timeval          tv             = {0,};
-        uint32_t                timeout        = 0;
-
-        priv = this->private;
-        if (!priv)
-                goto out;
-
-        ret = -1;
-        active_earlier = priv->active;
-
-        /* first stop the rollover and the fsync thread */
-        changelog_cleanup_helper_threads (this, priv);
-
-        GF_OPTION_RECONF ("changelog-dir", tmp, options, str, out);
-        if (!tmp) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "\"changelog-dir\" option is not set");
-                goto out;
-        }
-
-        GF_FREE (priv->changelog_dir);
-        priv->changelog_dir = gf_strdup (tmp);
-        if (!priv->changelog_dir)
-                goto out;
-
-        ret = mkdir_p (priv->changelog_dir, 0600, _gf_true);
-
-        if (ret)
-                goto out;
-        CHANGELOG_FILL_HTIME_DIR(priv->changelog_dir, htime_dir);
-        ret = mkdir_p (htime_dir, 0600, _gf_true);
+static void
+changelog_cleanup_rpc(xlator_t *this, changelog_priv_t *priv)
+{
+    /* terminate rpc server */
+    if (!this->cleanup_starting)
+        changelog_destroy_rpc_listner(this, priv);
 
-        if (ret)
-                goto out;
+    (void)changelog_cleanup_rpc_threads(this, priv);
+    /* cleanup rot buffs */
+    rbuf_dtor(priv->rbuf);
 
-        CHANGELOG_FILL_CSNAP_DIR(priv->changelog_dir, csnap_dir);
-        ret = mkdir_p (csnap_dir, 0600, _gf_true);
+    /* cleanup poller thread */
+    if (priv->poller)
+        (void)changelog_thread_cleanup(this, priv->poller);
+}
 
+int
+reconfigure(xlator_t *this, dict_t *options)
+{
+    int ret = 0;
+    char *tmp = NULL;
+    changelog_priv_t *priv = NULL;
+    gf_boolean_t active_earlier = _gf_true;
+    gf_boolean_t active_now = _gf_true;
+    gf_boolean_t rpc_active_earlier = _gf_true;
+    gf_boolean_t rpc_active_now = _gf_true;
+    gf_boolean_t iniate_rpc = _gf_false;
+    changelog_time_slice_t *slice = NULL;
+    changelog_log_data_t cld = {
+        0,
+    };
+    char htime_dir[PATH_MAX] = {
+        0,
+    };
+    char csnap_dir[PATH_MAX] = {
+        0,
+    };
+    uint32_t timeout = 0;
+
+    priv = this->private;
+    if (!priv)
+        goto out;
+
+    ret = -1;
+    active_earlier = priv->active;
+    rpc_active_earlier = priv->rpc_active;
+
+    /* first stop the rollover and the fsync thread */
+    changelog_cleanup_helper_threads(this, priv);
+
+    GF_OPTION_RECONF("changelog-dir", tmp, options, str, out);
+    if (!tmp) {
+        gf_smsg(this->name, GF_LOG_ERROR, 0, CHANGELOG_MSG_DIR_OPTIONS_NOT_SET,
+                NULL);
+        goto out;
+    }
+
+    GF_FREE(priv->changelog_dir);
+    priv->changelog_dir = gf_strdup(tmp);
+    if (!priv->changelog_dir)
+        goto out;
+
+    ret = mkdir_p(priv->changelog_dir, 0600, _gf_true);
+
+    if (ret)
+        goto out;
+    CHANGELOG_FILL_HTIME_DIR(priv->changelog_dir, htime_dir);
+    ret = mkdir_p(htime_dir, 0600, _gf_true);
+
+    if (ret)
+        goto out;
+
+    CHANGELOG_FILL_CSNAP_DIR(priv->changelog_dir, csnap_dir);
+    ret = mkdir_p(csnap_dir, 0600, _gf_true);
+
+    if (ret)
+        goto out;
+
+    GF_OPTION_RECONF("changelog", active_now, options, bool, out);
+    GF_OPTION_RECONF("changelog-notification", rpc_active_now, options, bool,
+                     out);
+
+    /* If journalling is enabled, enable rpc notifications */
+    if (active_now && !active_earlier) {
+        if (!rpc_active_earlier)
+            iniate_rpc = _gf_true;
+    }
+
+    if (rpc_active_now && !rpc_active_earlier) {
+        iniate_rpc = _gf_true;
+    }
+
+    /* TODO: Disable of changelog-notifications is not supported for now
+     * as there is no clean way of cleaning up of rpc resources
+     */
+
+    if (iniate_rpc) {
+        ret = changelog_init_rpc(this, priv);
         if (ret)
-                goto out;
+            goto out;
+        priv->rpc_active = _gf_true;
+    }
 
-        GF_OPTION_RECONF ("changelog", active_now, options, bool, out);
+    /**
+     * changelog_handle_change() handles changes that could possibly
+     * have been submit changes before changelog deactivation.
+     */
+    if (!active_now)
+        priv->active = _gf_false;
 
-        /**
-         * changelog_handle_change() handles changes that could possibly
-         * have been submit changes before changelog deactivation.
-         */
-        if (!active_now)
-                priv->active = _gf_false;
+    GF_OPTION_RECONF("op-mode", tmp, options, str, out);
+    changelog_assign_opmode(priv, tmp);
 
-        GF_OPTION_RECONF ("op-mode", tmp, options, str, out);
-        changelog_assign_opmode (priv, tmp);
+    tmp = NULL;
 
-        tmp = NULL;
+    GF_OPTION_RECONF("encoding", tmp, options, str, out);
+    changelog_assign_encoding(priv, tmp);
 
-        GF_OPTION_RECONF ("encoding", tmp, options, str, out);
-        changelog_assign_encoding (priv, tmp);
+    GF_OPTION_RECONF("rollover-time", priv->rollover_time, options, int32, out);
+    GF_OPTION_RECONF("fsync-interval", priv->fsync_interval, options, int32,
+                     out);
+    GF_OPTION_RECONF("changelog-barrier-timeout", timeout, options, time, out);
+    changelog_assign_barrier_timeout(priv, timeout);
 
-        GF_OPTION_RECONF ("rollover-time",
-                          priv->rollover_time, options, int32, out);
-        GF_OPTION_RECONF ("fsync-interval",
-                          priv->fsync_interval, options, int32, out);
-        GF_OPTION_RECONF ("changelog-barrier-timeout",
-                          timeout, options, time, out);
-        changelog_assign_barrier_timeout (priv, timeout);
+    GF_OPTION_RECONF("capture-del-path", priv->capture_del_path, options, bool,
+                     out);
 
-        if (active_now || active_earlier) {
-                ret = changelog_fill_rollover_data (&cld, !active_now);
-                if (ret)
-                        goto out;
+    if (active_now || active_earlier) {
+        changelog_fill_rollover_data(&cld, !active_now);
 
-                slice = &priv->slice;
+        slice = &priv->slice;
 
-                LOCK (&priv->lock);
-                {
-                        ret = changelog_inject_single_event (this, priv, &cld);
-                        if (!ret && active_now)
-                                SLICE_VERSION_UPDATE (slice);
-                }
-                UNLOCK (&priv->lock);
-
-                if (ret)
-                        goto out;
-
-                if (active_now) {
-                        if (!active_earlier) {
-                                if (gettimeofday(&tv, NULL) ) {
-                                        gf_log (this->name, GF_LOG_ERROR,
-                                                 "unable to fetch htime");
-                                        ret = -1;
-                                        goto out;
-                                }
-                                htime_open(this, priv, tv.tv_sec);
-                        }
-                        ret = changelog_spawn_helper_threads (this, priv);
-                }
+        LOCK(&priv->lock);
+        {
+            ret = changelog_inject_single_event(this, priv, &cld);
+            if (!ret && active_now)
+                SLICE_VERSION_UPDATE(slice);
         }
+        UNLOCK(&priv->lock);
 
- out:
-        if (ret) {
-                /* TODO */
-        } else {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "changelog reconfigured");
-                if (active_now && priv)
-                        priv->active = _gf_true;
+        if (ret)
+            goto out;
+
+        if (active_now) {
+            if (!active_earlier) {
+                gf_smsg(this->name, GF_LOG_INFO, 0, CHANGELOG_MSG_RECONFIGURE,
+                        NULL);
+                htime_create(this, priv, gf_time());
+            }
+            ret = changelog_spawn_helper_threads(this, priv);
         }
+    }
 
-        return ret;
+out:
+    if (ret) {
+        /* TODO */
+    } else {
+        gf_msg_debug(this->name, 0, "changelog reconfigured");
+        if (active_now && priv)
+            priv->active = _gf_true;
+    }
+
+    return ret;
 }
 
 static void
-changelog_freeup_options (xlator_t *this, changelog_priv_t *priv)
+changelog_freeup_options(xlator_t *this, changelog_priv_t *priv)
 {
-        int ret = 0;
+    int ret = 0;
 
-        ret = priv->cb->dtor (this, &priv->cd);
-        if (ret)
-                gf_log (this->name, GF_LOG_ERROR,
-                        "could not cleanup bootstrapper");
-        GF_FREE (priv->changelog_brick);
-        GF_FREE (priv->changelog_dir);
+    ret = priv->cb->dtor(this, &priv->cd);
+    if (ret)
+        gf_smsg(this->name, GF_LOG_ERROR, 0, CHANGELOG_MSG_FREEUP_FAILED, NULL);
+    GF_FREE(priv->changelog_brick);
+    GF_FREE(priv->changelog_dir);
 }
 
 static int
-changelog_init_options (xlator_t *this, changelog_priv_t *priv)
+changelog_init_options(xlator_t *this, changelog_priv_t *priv)
 {
-        int       ret            = 0;
-        char     *tmp            = NULL;
-        uint32_t  timeout        = 0;
-        char htime_dir[PATH_MAX] = {0,};
-        char csnap_dir[PATH_MAX] = {0,};
+    int ret = 0;
+    char *tmp = NULL;
+    uint32_t timeout = 0;
+    char htime_dir[PATH_MAX] = {
+        0,
+    };
+    char csnap_dir[PATH_MAX] = {
+        0,
+    };
 
-        GF_OPTION_INIT ("changelog-brick", tmp, str, error_return);
-        priv->changelog_brick = gf_strdup (tmp);
-        if (!priv->changelog_brick)
-                goto error_return;
+    GF_OPTION_INIT("changelog-brick", tmp, str, error_return);
+    priv->changelog_brick = gf_strdup(tmp);
+    if (!priv->changelog_brick)
+        goto error_return;
 
-        tmp = NULL;
+    tmp = NULL;
 
-        GF_OPTION_INIT ("changelog-dir", tmp, str, dealloc_1);
-        priv->changelog_dir = gf_strdup (tmp);
-        if (!priv->changelog_dir)
-                goto dealloc_1;
+    GF_OPTION_INIT("changelog-dir", tmp, str, dealloc_1);
+    priv->changelog_dir = gf_strdup(tmp);
+    if (!priv->changelog_dir)
+        goto dealloc_1;
 
-        tmp = NULL;
+    tmp = NULL;
 
-        /**
-         * create the directory even if change-logging would be inactive
-         * so that consumers can _look_ into it (finding nothing...)
-         */
-        ret = mkdir_p (priv->changelog_dir, 0600, _gf_true);
+    /**
+     * create the directory even if change-logging would be inactive
+     * so that consumers can _look_ into it (finding nothing...)
+     */
+    ret = mkdir_p(priv->changelog_dir, 0600, _gf_true);
 
-        if (ret)
-                goto dealloc_2;
+    if (ret)
+        goto dealloc_2;
 
-        CHANGELOG_FILL_HTIME_DIR (priv->changelog_dir, htime_dir);
-        ret = mkdir_p (htime_dir, 0600, _gf_true);
-        if (ret)
-                goto dealloc_2;
+    CHANGELOG_FILL_HTIME_DIR(priv->changelog_dir, htime_dir);
+    ret = mkdir_p(htime_dir, 0600, _gf_true);
+    if (ret)
+        goto dealloc_2;
 
-        CHANGELOG_FILL_CSNAP_DIR (priv->changelog_dir, csnap_dir);
-        ret = mkdir_p (csnap_dir, 0600, _gf_true);
-        if (ret)
-                goto dealloc_2;
+    CHANGELOG_FILL_CSNAP_DIR(priv->changelog_dir, csnap_dir);
+    ret = mkdir_p(csnap_dir, 0600, _gf_true);
+    if (ret)
+        goto dealloc_2;
 
-        GF_OPTION_INIT ("changelog", priv->active, bool, dealloc_2);
+    GF_OPTION_INIT("changelog", priv->active, bool, dealloc_2);
+    GF_OPTION_INIT("changelog-notification", priv->rpc_active, bool, dealloc_2);
+    GF_OPTION_INIT("capture-del-path", priv->capture_del_path, bool, dealloc_2);
 
-        GF_OPTION_INIT ("op-mode", tmp, str, dealloc_2);
-        changelog_assign_opmode (priv, tmp);
+    GF_OPTION_INIT("op-mode", tmp, str, dealloc_2);
+    changelog_assign_opmode(priv, tmp);
 
-        tmp = NULL;
+    tmp = NULL;
 
-        GF_OPTION_INIT ("encoding", tmp, str, dealloc_2);
-        changelog_assign_encoding (priv, tmp);
-        changelog_encode_change (priv);
+    GF_OPTION_INIT("encoding", tmp, str, dealloc_2);
+    changelog_assign_encoding(priv, tmp);
+    changelog_encode_change(priv);
 
-        GF_OPTION_INIT ("rollover-time",
-                        priv->rollover_time, int32, dealloc_2);
+    GF_OPTION_INIT("rollover-time", priv->rollover_time, int32, dealloc_2);
 
-        GF_OPTION_INIT ("fsync-interval",
-                        priv->fsync_interval, int32, dealloc_2);
+    GF_OPTION_INIT("fsync-interval", priv->fsync_interval, int32, dealloc_2);
 
-        GF_OPTION_INIT ("changelog-barrier-timeout",
-                        timeout, time, dealloc_2);
-        changelog_assign_barrier_timeout (priv, timeout);
+    GF_OPTION_INIT("changelog-barrier-timeout", timeout, time, dealloc_2);
+    changelog_assign_barrier_timeout(priv, timeout);
 
-        GF_ASSERT (cb_bootstrap[priv->op_mode].mode == priv->op_mode);
-        priv->cb = &cb_bootstrap[priv->op_mode];
+    GF_ASSERT(cb_bootstrap[priv->op_mode].mode == priv->op_mode);
+    priv->cb = &cb_bootstrap[priv->op_mode];
 
-        /* ... now bootstrap the logger */
-        ret = priv->cb->ctor (this, &priv->cd);
-        if (ret)
-                goto dealloc_2;
+    /* ... now bootstrap the logger */
+    ret = priv->cb->ctor(this, &priv->cd);
+    if (ret)
+        goto dealloc_2;
 
-        priv->changelog_fd = -1;
+    priv->changelog_fd = -1;
 
-        return 0;
+    return 0;
 
- dealloc_2:
-        GF_FREE (priv->changelog_dir);
- dealloc_1:
-        GF_FREE (priv->changelog_brick);
- error_return:
-        return -1;
-}
-
-static void
-changelog_cleanup_rpc (xlator_t *this, changelog_priv_t *priv)
-{
-        /* terminate rpc server */
-        changelog_destroy_rpc_listner (this, priv);
-
-        /* cleanup rot buffs */
-        rbuf_dtor (priv->rbuf);
-
-        /* cleanup poller thread */
-        if (priv->poller)
-                (void) changelog_thread_cleanup (this, priv->poller);
+dealloc_2:
+    GF_FREE(priv->changelog_dir);
+dealloc_1:
+    GF_FREE(priv->changelog_brick);
+error_return:
+    return -1;
 }
 
 static int
-changelog_init_rpc (xlator_t *this, changelog_priv_t *priv)
+changelog_init_rpc(xlator_t *this, changelog_priv_t *priv)
 {
-        int        ret      = 0;
-        rpcsvc_t  *rpc      = NULL;
-        changelog_ev_selector_t *selection = NULL;
+    rpcsvc_t *rpc = NULL;
+    changelog_ev_selector_t *selection = NULL;
 
-        selection = &priv->ev_selection;
+    selection = &priv->ev_selection;
 
-        /* initialize event selection */
-        changelog_init_event_selection (this, selection);
+    /* initialize event selection */
+    changelog_init_event_selection(this, selection);
 
-        priv->rbuf = rbuf_init (NR_ROTT_BUFFS);
-        if (!priv->rbuf)
-                goto cleanup_thread;
+    priv->rbuf = rbuf_init(NR_ROTT_BUFFS);
+    if (!priv->rbuf)
+        goto cleanup_thread;
 
-        rpc = changelog_init_rpc_listner (this, priv,
-                                          priv->rbuf, NR_DISPATCHERS);
-        if (!rpc)
-                goto cleanup_rbuf;
-        priv->rpc = rpc;
+    rpc = changelog_init_rpc_listener(this, priv, priv->rbuf, NR_DISPATCHERS);
+    if (!rpc)
+        goto cleanup_rbuf;
+    priv->rpc = rpc;
 
-        return 0;
+    return 0;
 
- cleanup_rbuf:
-        rbuf_dtor (priv->rbuf);
- cleanup_thread:
-        if (priv->poller)
-                (void) changelog_thread_cleanup (this, priv->poller);
+cleanup_rbuf:
+    rbuf_dtor(priv->rbuf);
+cleanup_thread:
+    if (priv->poller)
+        (void)changelog_thread_cleanup(this, priv->poller);
 
-        return -1;
+    return -1;
 }
 
 int32_t
-init (xlator_t *this)
-{
-        int               ret  = -1;
-        char             *tmp  = NULL;
-        changelog_priv_t *priv = NULL;
-
-        GF_VALIDATE_OR_GOTO ("changelog", this, error_return);
-
-        if (!this->children || this->children->next) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "translator needs a single subvolume");
-                goto error_return;
-        }
-
-        if (!this->parents) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "dangling volume. please check volfile");
-                goto error_return;
-        }
-
-        priv = GF_CALLOC (1, sizeof (*priv), gf_changelog_mt_priv_t);
-        if (!priv)
-                goto error_return;
-
-        this->local_pool = mem_pool_new (changelog_local_t, 64);
-        if (!this->local_pool) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "failed to create local memory pool");
-                goto cleanup_priv;
-        }
-
-        LOCK_INIT (&priv->lock);
-        LOCK_INIT (&priv->c_snap_lock);
-
-        ret = changelog_init_options (this, priv);
-        if (ret)
-                goto cleanup_mempool;
-
-        /* snap dependency changes */
-        priv->dm.black_fop_cnt = 0;
-        priv->dm.white_fop_cnt = 0;
-        priv->dm.drain_wait_black = _gf_false;
-        priv->dm.drain_wait_white = _gf_false;
-        priv->current_color = FOP_COLOR_BLACK;
-        priv->explicit_rollover = _gf_false;
-
-        /* Mutex is not needed as threads are not spawned yet */
-        priv->bn.bnotify = _gf_false;
-        ret = changelog_barrier_pthread_init (this, priv);
-        if (ret)
-                goto cleanup_options;
-        LOCK_INIT (&priv->bflags.lock);
-        priv->bflags.barrier_ext = _gf_false;
-
-        /* Changelog barrier init */
-        INIT_LIST_HEAD (&priv->queue);
-        priv->barrier_enabled = _gf_false;
-
+init(xlator_t *this)
+{
+    int ret = -1;
+    changelog_priv_t *priv = NULL;
+
+    GF_VALIDATE_OR_GOTO("changelog", this, error_return);
+
+    if (!this->children || this->children->next) {
+        gf_smsg(this->name, GF_LOG_ERROR, 0, CHANGELOG_MSG_CHILD_MISCONFIGURED,
+                NULL);
+        goto error_return;
+    }
+
+    if (!this->parents) {
+        gf_smsg(this->name, GF_LOG_ERROR, 0, CHANGELOG_MSG_VOL_MISCONFIGURED,
+                NULL);
+        goto error_return;
+    }
+
+    priv = GF_CALLOC(1, sizeof(*priv), gf_changelog_mt_priv_t);
+    if (!priv)
+        goto error_return;
+
+    this->local_pool = mem_pool_new(changelog_local_t, 64);
+    if (!this->local_pool) {
+        gf_smsg(this->name, GF_LOG_ERROR, ENOMEM, CHANGELOG_MSG_NO_MEMORY,
+                NULL);
+        goto cleanup_priv;
+    }
+
+    LOCK_INIT(&priv->lock);
+    LOCK_INIT(&priv->c_snap_lock);
+    GF_ATOMIC_INIT(priv->listnercnt, 0);
+    GF_ATOMIC_INIT(priv->clntcnt, 0);
+    GF_ATOMIC_INIT(priv->xprtcnt, 0);
+    INIT_LIST_HEAD(&priv->xprt_list);
+    priv->htime_fd = -1;
+
+    ret = changelog_init_options(this, priv);
+    if (ret)
+        goto cleanup_mempool;
+
+    /* snap dependency changes */
+    priv->dm.black_fop_cnt = 0;
+    priv->dm.white_fop_cnt = 0;
+    priv->dm.drain_wait_black = _gf_false;
+    priv->dm.drain_wait_white = _gf_false;
+    priv->current_color = FOP_COLOR_BLACK;
+    priv->explicit_rollover = _gf_false;
+
+    priv->cr.notify = _gf_false;
+    /* Mutex is not needed as threads are not spawned yet */
+    priv->bn.bnotify = _gf_false;
+    priv->bn.bnotify_error = _gf_false;
+    ret = changelog_barrier_pthread_init(this, priv);
+    if (ret)
+        goto cleanup_options;
+    LOCK_INIT(&priv->bflags.lock);
+    priv->bflags.barrier_ext = _gf_false;
+
+    /* Changelog barrier init */
+    INIT_LIST_HEAD(&priv->queue);
+    priv->barrier_enabled = _gf_false;
+
+    if (priv->rpc_active || priv->active) {
         /* RPC ball rolling.. */
-        ret = changelog_init_rpc (this, priv);
-        if (ret)
-                goto cleanup_barrier;
-
-        ret = changelog_init (this, priv);
+        ret = changelog_init_rpc(this, priv);
         if (ret)
-                goto cleanup_rpc;
-        gf_log (this->name, GF_LOG_DEBUG, "changelog translator loaded");
-
-        this->private = priv;
-        return 0;
-
- cleanup_rpc:
-        changelog_cleanup_rpc (this, priv);
- cleanup_barrier:
-        changelog_barrier_pthread_destroy (priv);
- cleanup_options:
-        changelog_freeup_options (this, priv);
- cleanup_mempool:
-        mem_pool_destroy (this->local_pool);
- cleanup_priv:
-        GF_FREE (priv);
- error_return:
-        this->private = NULL;
-        return -1;
+            goto cleanup_barrier;
+        priv->rpc_active = _gf_true;
+    }
+
+    ret = changelog_init(this, priv);
+    if (ret)
+        goto cleanup_rpc;
+
+    gf_msg_debug(this->name, 0, "changelog translator loaded");
+
+    this->private = priv;
+    return 0;
+
+cleanup_rpc:
+    if (priv->rpc_active) {
+        changelog_cleanup_rpc(this, priv);
+    }
+cleanup_barrier:
+    changelog_barrier_pthread_destroy(priv);
+cleanup_options:
+    changelog_freeup_options(this, priv);
+cleanup_mempool:
+    mem_pool_destroy(this->local_pool);
+    this->local_pool = NULL;
+cleanup_priv:
+    GF_FREE(priv);
+error_return:
+    this->private = NULL;
+    return -1;
 }
 
 void
-fini (xlator_t *this)
+fini(xlator_t *this)
 {
-        changelog_priv_t *priv = NULL;
+    changelog_priv_t *priv = NULL;
+    struct list_head queue = {
+        0,
+    };
 
-        priv = this->private;
+    priv = this->private;
+
+    if (priv) {
+        if (priv->active || priv->rpc_active) {
+            /* terminate RPC server/threads */
+            changelog_cleanup_rpc(this, priv);
+            GF_FREE(priv->ev_dispatcher);
+        }
+        /* call barrier_disable to cancel timer */
+        if (priv->barrier_enabled)
+            __chlog_barrier_disable(this, &queue);
 
-        if (priv) {
-                /* terminate RPC server/threads */
-                changelog_cleanup_rpc (this, priv);
+        /* cleanup barrier related objects */
+        changelog_barrier_pthread_destroy(priv);
 
-                /* cleanup barrier related objects */
-                changelog_barrier_pthread_destroy (priv);
+        /* cleanup helper threads */
+        changelog_cleanup_helper_threads(this, priv);
 
-                /* cleanup allocated options */
-                changelog_freeup_options (this, priv);
+        /* cleanup allocated options */
+        changelog_freeup_options(this, priv);
 
-                /* deallocate mempool */
-                mem_pool_destroy (this->local_pool);
+        /* deallocate mempool */
+        mem_pool_destroy(this->local_pool);
 
-                /* finally, dealloac private variable */
-                GF_FREE (priv);
+        if (priv->htime_fd != -1) {
+            sys_close(priv->htime_fd);
         }
 
-        this->private = NULL;
+        /* finally, dealloac private variable */
+        GF_FREE(priv);
+    }
 
-        return;
+    this->private = NULL;
+    this->local_pool = NULL;
+
+    return;
 }
 
 struct xlator_fops fops = {
-        .open         = changelog_open,
-        .mknod        = changelog_mknod,
-        .mkdir        = changelog_mkdir,
-        .create       = changelog_create,
-        .symlink      = changelog_symlink,
-        .writev       = changelog_writev,
-        .truncate     = changelog_truncate,
-        .ftruncate    = changelog_ftruncate,
-        .link         = changelog_link,
-        .rename       = changelog_rename,
-        .unlink       = changelog_unlink,
-        .rmdir        = changelog_rmdir,
-        .setattr      = changelog_setattr,
-        .fsetattr     = changelog_fsetattr,
-        .setxattr     = changelog_setxattr,
-        .fsetxattr    = changelog_fsetxattr,
-        .removexattr  = changelog_removexattr,
-        .fremovexattr = changelog_fremovexattr,
-        .ipc          = changelog_ipc,
+    .open = changelog_open,
+    .mknod = changelog_mknod,
+    .mkdir = changelog_mkdir,
+    .create = changelog_create,
+    .symlink = changelog_symlink,
+    .writev = changelog_writev,
+    .truncate = changelog_truncate,
+    .ftruncate = changelog_ftruncate,
+    .link = changelog_link,
+    .rename = changelog_rename,
+    .unlink = changelog_unlink,
+    .rmdir = changelog_rmdir,
+    .setattr = changelog_setattr,
+    .fsetattr = changelog_fsetattr,
+    .setxattr = changelog_setxattr,
+    .fsetxattr = changelog_fsetxattr,
+    .removexattr = changelog_removexattr,
+    .fremovexattr = changelog_fremovexattr,
+    .ipc = changelog_ipc,
+    .xattrop = changelog_xattrop,
+    .fxattrop = changelog_fxattrop,
 };
 
 struct xlator_cbks cbks = {
-        .forget = changelog_forget,
-        .release = changelog_release,
+    .forget = changelog_forget,
+    .release = changelog_release,
 };
 
 struct volume_options options[] = {
-        {.key = {"changelog"},
-         .type = GF_OPTION_TYPE_BOOL,
-         .default_value = "off",
-         .description = "enable/disable change-logging"
-        },
-        {.key = {"changelog-brick"},
-         .type = GF_OPTION_TYPE_PATH,
-         .description = "brick path to generate unique socket file name."
-                       " should be the export directory of the volume strictly."
-        },
-        {.key = {"changelog-dir"},
-         .type = GF_OPTION_TYPE_PATH,
-         .description = "directory for the changelog files"
-        },
-        {.key = {"op-mode"},
-         .type = GF_OPTION_TYPE_STR,
-         .default_value = "realtime",
-         .value = {"realtime"},
-         .description = "operation mode - futuristic operation modes"
-        },
-        {.key = {"encoding"},
-         .type = GF_OPTION_TYPE_STR,
-         .default_value = "ascii",
-         .value = {"binary", "ascii"},
-         .description = "encoding type for changelogs"
-        },
-        {.key = {"rollover-time"},
-         .default_value = "15",
-         .type = GF_OPTION_TYPE_TIME,
-         .description = "time to switch to a new changelog file (in seconds)"
-        },
-        {.key = {"fsync-interval"},
-         .type = GF_OPTION_TYPE_TIME,
-         .default_value = "5",
-         .description = "do not open CHANGELOG file with O_SYNC mode."
-                        " instead perform fsync() at specified intervals"
-        },
-        { .key = {"changelog-barrier-timeout"},
-          .type = GF_OPTION_TYPE_TIME,
-          .default_value = BARRIER_TIMEOUT,
-          .description = "After 'timeout' seconds since the time 'barrier' "
-                         "option was set to \"on\", unlink/rmdir/rename  "
-                         "operations are no longer blocked and previously "
-                         "blocked fops are allowed to go through"
-        },
-        {.key = {NULL}
-        },
+    {.key = {"changelog"},
+     .type = GF_OPTION_TYPE_BOOL,
+     .default_value = "off",
+     .description = "enable/disable change-logging",
+     .op_version = {3},
+     .flags = OPT_FLAG_SETTABLE,
+     .level = OPT_STATUS_BASIC,
+     .tags = {"journal", "georep", "glusterfind"}},
+    {.key = {"changelog-notification"},
+     .type = GF_OPTION_TYPE_BOOL,
+     .default_value = "off",
+     .description = "enable/disable changelog live notification",
+     .op_version = {3},
+     .level = OPT_STATUS_BASIC,
+     .tags = {"bitrot", "georep"}},
+    {.key = {"changelog-brick"},
+     .type = GF_OPTION_TYPE_PATH,
+     .description = "brick path to generate unique socket file name."
+                    " should be the export directory of the volume strictly.",
+     .default_value = "{{ brick.path }}",
+     .op_version = {3},
+     .tags = {"journal"}},
+    {.key = {"changelog-dir"},
+     .type = GF_OPTION_TYPE_PATH,
+     .description = "directory for the changelog files",
+     .default_value = "{{ brick.path }}/.glusterfs/changelogs",
+     .op_version = {3},
+     .flags = OPT_FLAG_SETTABLE,
+     .level = OPT_STATUS_ADVANCED,
+     .tags = {"journal", "georep", "glusterfind"}},
+    {.key = {"op-mode"},
+     .type = GF_OPTION_TYPE_STR,
+     .default_value = "realtime",
+     .value = {"realtime"},
+     .description = "operation mode - futuristic operation modes",
+     .op_version = {3},
+     .tags = {"journal"}},
+    {.key = {"encoding"},
+     .type = GF_OPTION_TYPE_STR,
+     .default_value = "ascii",
+     .value = {"binary", "ascii"},
+     .description = "encoding type for changelogs",
+     .op_version = {3},
+     .flags = OPT_FLAG_SETTABLE,
+     .level = OPT_STATUS_ADVANCED,
+     .tags = {"journal"}},
+    {.key = {"rollover-time"},
+     .default_value = "15",
+     .type = GF_OPTION_TYPE_TIME,
+     .description = "time to switch to a new changelog file (in seconds)",
+     .op_version = {3},
+     .flags = OPT_FLAG_SETTABLE,
+     .level = OPT_STATUS_ADVANCED,
+     .tags = {"journal", "georep", "glusterfind"}},
+    {.key = {"fsync-interval"},
+     .type = GF_OPTION_TYPE_TIME,
+     .default_value = "5",
+     .description = "do not open CHANGELOG file with O_SYNC mode."
+                    " instead perform fsync() at specified intervals",
+     .op_version = {3},
+     .flags = OPT_FLAG_SETTABLE,
+     .level = OPT_STATUS_ADVANCED,
+     .tags = {"journal"}},
+    {.key = {"changelog-barrier-timeout"},
+     .type = GF_OPTION_TYPE_TIME,
+     .default_value = BARRIER_TIMEOUT,
+     .description = "After 'timeout' seconds since the time 'barrier' "
+                    "option was set to \"on\", unlink/rmdir/rename  "
+                    "operations are no longer blocked and previously "
+                    "blocked fops are allowed to go through",
+     .op_version = {3},
+     .flags = OPT_FLAG_SETTABLE,
+     .level = OPT_STATUS_ADVANCED,
+     .tags = {"journal"}},
+    {.key = {"capture-del-path"},
+     .type = GF_OPTION_TYPE_BOOL,
+     .default_value = "off",
+     .description = "enable/disable capturing paths of deleted entries",
+     .op_version = {3},
+     .flags = OPT_FLAG_SETTABLE,
+     .level = OPT_STATUS_BASIC,
+     .tags = {"journal", "glusterfind"}},
+    {.key = {NULL}},
+};
+
+xlator_api_t xlator_api = {
+    .init = init,
+    .fini = fini,
+    .notify = notify,
+    .reconfigure = reconfigure,
+    .mem_acct_init = mem_acct_init,
+    .op_version = {1}, /* Present from the initial version */
+    .fops = &fops,
+    .cbks = &cbks,
+    .options = options,
+    .identifier = "changelog",
+    .category = GF_MAINTAINED,
 };
diff --git a/xlators/features/changetimerecorder/src/Makefile.am b/xlators/features/changetimerecorder/src/Makefile.am
deleted file mode 100644
index e79be53c063..00000000000
--- a/xlators/features/changetimerecorder/src/Makefile.am
+++ /dev/null
@@ -1,23 +0,0 @@
-xlatordir = $(libdir)/glusterfs/$(PACKAGE_VERSION)/xlator/features
-
-# changetimerecorder can only get build when libgfdb is enabled
-if BUILD_GFDB
-   xlator_LTLIBRARIES = changetimerecorder.la
-endif
-
-changetimerecorder_la_LDFLAGS = -module -avoid-version
-
-changetimerecorder_la_SOURCES = changetimerecorder.c ctr-helper.c
-
-changetimerecorder_la_LIBADD = $(top_builddir)/libglusterfs/src/libglusterfs.la\
-			$(top_builddir)/libglusterfs/src/gfdb/libgfdb.la
-
-noinst_HEADERS = changetimerecorder.h ctr_mem_types.h ctr-helper.h
-
-AM_CPPFLAGS = $(GF_CPPFLAGS) -I$(top_srcdir)/libglusterfs/src \
-		-I$(top_srcdir)/libglusterfs/src/gfdb \
-		-DDATADIR=\"$(localstatedir)\"
-
-AM_CFLAGS = -Wall $(GF_CFLAGS) $(SQLITE_CFLAGS)
-
-CLEANFILES =
diff --git a/xlators/features/changetimerecorder/src/changetimerecorder.c b/xlators/features/changetimerecorder/src/changetimerecorder.c
deleted file mode 100644
index 80deefbd4b2..00000000000
--- a/xlators/features/changetimerecorder/src/changetimerecorder.c
+++ /dev/null
@@ -1,1211 +0,0 @@
-/*
-   Copyright (c) 2015 Red Hat, Inc. <http://www.redhat.com>
-   This file is part of GlusterFS.
-
-   This file is licensed to you under your choice of the GNU Lesser
-   General Public License, version 3 or any later version (LGPLv3 or
-   later), or the GNU General Public License, version 2 (GPLv2), in all
-   cases as published by the Free Software Foundation.
-*/
-#include <ctype.h>
-#include <sys/uio.h>
-
-#include "gfdb_sqlite3.h"
-#include "ctr-helper.h"
-
-/****************************WRITEV********************************************/
-int32_t
-ctr_writev_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                    int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
-                    struct iatt *postbuf,
-                    dict_t *xdata)
-{
-        int ret = -1;
-
-        CTR_IS_DISABLED_THEN_GOTO(this, out);
-
-        ret = ctr_insert_unwind(frame, this,
-                        GFDB_FOP_INODE_WRITE, GFDB_FOP_UNWIND);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "Failed inserting writev unwind");
-        }
-
-
-out:
-        STACK_UNWIND_STRICT (writev, frame, op_ret, op_errno, prebuf,
-                        postbuf, xdata);
-
-        return 0;
-}
-
-int32_t
-ctr_writev (call_frame_t *frame, xlator_t *this, fd_t *fd,
-            struct iovec *vector, int32_t count, off_t off,
-            uint32_t flags,
-            struct iobref *iobref, dict_t *xdata)
-{
-        int ret = -1;
-        gf_ctr_inode_context_t ctr_inode_cx;
-        gf_ctr_inode_context_t *_inode_cx = &ctr_inode_cx;
-
-        CTR_IS_DISABLED_THEN_GOTO(this, out);
-        CTR_IF_INTERNAL_FOP_THEN_GOTO (frame, xdata, out);
-
-        /*Fill ctr inode context*/
-        FILL_CTR_INODE_CONTEXT(_inode_cx, fd->inode->ia_type,
-                                fd->inode->gfid, NULL, NULL,
-                                GFDB_FOP_INODE_WRITE, GFDB_FOP_WIND);
-
-        /*record into the database*/
-        ret = ctr_insert_wind(frame, this, _inode_cx);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR,
-                                "Failed inserting writev wind");
-        }
-
-out:
-        STACK_WIND (frame, ctr_writev_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->writev, fd, vector, count,
-                    off, flags, iobref, xdata);
-
-        return 0;
-}
-
-/******************************setattr*****************************************/
-
-int32_t
-ctr_setattr_cbk (call_frame_t *frame,
-                       void *cookie, xlator_t *this, int32_t op_ret,
-                       int32_t op_errno, struct iatt *preop_stbuf,
-                       struct iatt *postop_stbuf, dict_t *xdata)
-{
-
-        int ret = -1;
-
-        CTR_IS_DISABLED_THEN_GOTO(this, out);
-
-        ret = ctr_insert_unwind(frame, this,
-                        GFDB_FOP_INODE_WRITE, GFDB_FOP_UNWIND);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "Failed inserting setattr unwind");
-        }
-
-out:
-        STACK_UNWIND_STRICT (setattr, frame, op_ret, op_errno, preop_stbuf,
-                       postop_stbuf, xdata);
-
-        return 0;
-}
-
-int32_t
-ctr_setattr (call_frame_t *frame,
-                   xlator_t *this, loc_t *loc,
-                   struct iatt *stbuf, int32_t valid, dict_t *xdata)
-{
-
-        int ret = -1;
-        gf_ctr_inode_context_t ctr_inode_cx;
-        gf_ctr_inode_context_t *_inode_cx = &ctr_inode_cx;
-
-        CTR_IS_DISABLED_THEN_GOTO(this, out);
-        CTR_IF_INTERNAL_FOP_THEN_GOTO (frame, xdata, out);
-
-         /*Fill ctr inode context*/
-        FILL_CTR_INODE_CONTEXT(_inode_cx, loc->inode->ia_type,
-                loc->inode->gfid, NULL, NULL, GFDB_FOP_INODE_WRITE,
-                GFDB_FOP_WIND);
-
-        /*record into the database*/
-        ret = ctr_insert_wind(frame, this, _inode_cx);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR,
-                                "Failed inserting setattr wind");
-        }
-out:
-
-        STACK_WIND (frame, ctr_setattr_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->setattr, loc, stbuf,
-                    valid, xdata);
-
-        return 0;
-}
-
-/****************************fremovexattr************************************/
-
-int32_t
-ctr_fremovexattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                          int32_t op_ret, int32_t op_errno, dict_t *xdata)
-{
-        int ret = -1;
-
-        CTR_IS_DISABLED_THEN_GOTO(this, out);
-
-
-        ret = ctr_insert_unwind(frame, this,
-                        GFDB_FOP_INODE_WRITE, GFDB_FOP_UNWIND);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "Failed inserting fremovexattr unwind");
-        }
-
-out:
-        STACK_UNWIND_STRICT (fremovexattr, frame, op_ret, op_errno, xdata);
-
-        return 0;
-}
-
-int32_t
-ctr_fremovexattr (call_frame_t *frame, xlator_t *this, fd_t *fd,
-                      const char *name, dict_t *xdata)
-{
-        int ret = -1;
-        gf_ctr_inode_context_t ctr_inode_cx;
-        gf_ctr_inode_context_t *_inode_cx = &ctr_inode_cx;
-
-        CTR_IS_DISABLED_THEN_GOTO(this, out);
-        CTR_IF_INTERNAL_FOP_THEN_GOTO (frame, xdata, out);
-
-
-         /*Fill ctr inode context*/
-        FILL_CTR_INODE_CONTEXT(_inode_cx, fd->inode->ia_type,
-                fd->inode->gfid, NULL, NULL, GFDB_FOP_INODE_WRITE,
-                GFDB_FOP_WIND);
-
-        /*record into the database*/
-        ret = ctr_insert_wind(frame, this, _inode_cx);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR,
-                                "Failed inserting fremovexattr wind");
-        }
-
-out:
-        STACK_WIND (frame, ctr_fremovexattr_cbk, FIRST_CHILD (this),
-                    FIRST_CHILD (this)->fops->fremovexattr,
-                    fd, name, xdata);
-        return 0;
-}
-
-/****************************removexattr*************************************/
-
-int32_t
-ctr_removexattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                         int32_t op_ret, int32_t op_errno, dict_t *xdata)
-{
-        int ret = -1;
-
-        CTR_IS_DISABLED_THEN_GOTO(this, out);
-        CTR_IF_INTERNAL_FOP_THEN_GOTO (frame, xdata, out);
-
-
-        ret = ctr_insert_unwind(frame, this,
-                        GFDB_FOP_INODE_WRITE, GFDB_FOP_UNWIND);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "Failed inserting removexattr unwind");
-        }
-
-out:
-        STACK_UNWIND_STRICT (removexattr, frame, op_ret, op_errno, xdata);
-
-        return 0;
-}
-
-int32_t
-ctr_removexattr (call_frame_t *frame, xlator_t *this, loc_t *loc,
-                     const char *name, dict_t *xdata)
-{
-        int ret = -1;
-        gf_ctr_inode_context_t ctr_inode_cx;
-        gf_ctr_inode_context_t *_inode_cx = &ctr_inode_cx;
-
-        CTR_IS_DISABLED_THEN_GOTO(this, out);
-        CTR_IF_INTERNAL_FOP_THEN_GOTO (frame, xdata, out);
-
-
-         /*Fill ctr inode context*/
-        FILL_CTR_INODE_CONTEXT(_inode_cx, loc->inode->ia_type,
-                loc->inode->gfid, NULL, NULL, GFDB_FOP_INODE_WRITE,
-                GFDB_FOP_WIND);
-
-        /*record into the database*/
-        ret = ctr_insert_wind(frame, this, _inode_cx);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR,
-                                "Failed inserting removexattr wind");
-        }
-
-out:
-        STACK_WIND (frame, ctr_removexattr_cbk, FIRST_CHILD (this),
-                    FIRST_CHILD (this)->fops->removexattr,
-                    loc, name, xdata);
-        return 0;
-}
-
-/****************************truncate****************************************/
-
-int32_t
-ctr_truncate_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                      int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
-                      struct iatt *postbuf, dict_t *xdata)
-{
-        int ret = -1;
-
-        CTR_IS_DISABLED_THEN_GOTO(this, out);
-
-
-        ret = ctr_insert_unwind(frame, this,
-                        GFDB_FOP_INODE_WRITE, GFDB_FOP_UNWIND);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "Failed inserting truncate unwind");
-        }
-
-
-out:
-        STACK_UNWIND_STRICT (truncate, frame, op_ret, op_errno, prebuf,
-                      postbuf, xdata);
-
-        return 0;
-}
-
-int32_t
-ctr_truncate (call_frame_t *frame, xlator_t *this, loc_t *loc,
-                  off_t offset, dict_t *xdata)
-{
-        int ret = -1;
-        gf_ctr_inode_context_t ctr_inode_cx;
-        gf_ctr_inode_context_t *_inode_cx = &ctr_inode_cx;
-
-        CTR_IS_DISABLED_THEN_GOTO(this, out);
-        CTR_IF_INTERNAL_FOP_THEN_GOTO (frame, xdata, out);
-
-         /*Fill ctr inode context*/
-        FILL_CTR_INODE_CONTEXT(_inode_cx, loc->inode->ia_type,
-                loc->inode->gfid, NULL, NULL, GFDB_FOP_INODE_WRITE,
-                GFDB_FOP_WIND);
-
-        /*record into the database*/
-        ret = ctr_insert_wind(frame, this, _inode_cx);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR,
-                                "Failed inserting truncate wind");
-        }
-out:
-        STACK_WIND (frame, ctr_truncate_cbk, FIRST_CHILD (this),
-                    FIRST_CHILD (this)->fops->truncate,
-                    loc, offset, xdata);
-        return 0;
-}
-
-/****************************ftruncate***************************************/
-
-int32_t
-ctr_ftruncate_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                       int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
-                       struct iatt *postbuf, dict_t *xdata)
-{
-        int ret = -1;
-
-        CTR_IS_DISABLED_THEN_GOTO(this, out);
-
-        ret = ctr_insert_unwind(frame, this,
-                        GFDB_FOP_INODE_WRITE, GFDB_FOP_UNWIND);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "Failed inserting ftruncate unwind");
-        }
-
-out:
-        STACK_UNWIND_STRICT (ftruncate, frame, op_ret, op_errno, prebuf,
-                      postbuf, xdata);
-
-        return 0;
-}
-
-int32_t
-ctr_ftruncate (call_frame_t *frame, xlator_t *this, fd_t *fd,
-                   off_t offset, dict_t *xdata)
-{
-        int ret = -1;
-        gf_ctr_inode_context_t ctr_inode_cx;
-        gf_ctr_inode_context_t *_inode_cx = &ctr_inode_cx;
-
-        CTR_IS_DISABLED_THEN_GOTO(this, out);
-        CTR_IF_INTERNAL_FOP_THEN_GOTO (frame, xdata, out);
-
-         /*Fill ctr inode context*/
-        FILL_CTR_INODE_CONTEXT(_inode_cx, fd->inode->ia_type,
-                fd->inode->gfid, NULL, NULL, GFDB_FOP_INODE_WRITE,
-                GFDB_FOP_WIND);
-
-        /*record into the database*/
-        ret = ctr_insert_wind(frame, this, _inode_cx);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR,
-                                "Failed inserting ftruncate wind");
-        }
-
-out:
-        STACK_WIND (frame, ctr_ftruncate_cbk, FIRST_CHILD (this),
-                    FIRST_CHILD (this)->fops->ftruncate,
-                    fd, offset, xdata);
-        return 0;
-}
-
-/****************************rename******************************************/
-
-int32_t
-ctr_rename_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                    int32_t op_ret, int32_t op_errno, struct iatt *buf,
-                    struct iatt *preoldparent, struct iatt *postoldparent,
-                    struct iatt *prenewparent, struct iatt *postnewparent,
-                    dict_t *xdata)
-{
-        int ret = -1;
-
-        CTR_IS_DISABLED_THEN_GOTO(this, out);
-
-        ret = ctr_insert_unwind(frame, this,
-                        GFDB_FOP_DENTRY_WRITE, GFDB_FOP_UNWIND);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "Failed inserting rename unwind");
-        }
-
-out:
-        STACK_UNWIND_STRICT (rename, frame, op_ret, op_errno, buf,
-                             preoldparent, postoldparent, prenewparent,
-                             postnewparent,
-                             xdata);
-
-        return 0;
-}
-
-int32_t
-ctr_rename (call_frame_t *frame, xlator_t *this, loc_t *oldloc,
-                loc_t *newloc, dict_t *xdata)
-{
-        int ret = -1;
-        gf_ctr_inode_context_t ctr_inode_cx;
-        gf_ctr_inode_context_t *_inode_cx = &ctr_inode_cx;
-        gf_ctr_link_context_t new_link_cx, old_link_cx;
-        gf_ctr_link_context_t *_nlink_cx = &new_link_cx;
-        gf_ctr_link_context_t *_olink_cx = &old_link_cx;
-
-        CTR_IS_DISABLED_THEN_GOTO(this, out);
-        CTR_IF_INTERNAL_FOP_THEN_GOTO (frame, xdata, out);
-
-        /*Fill old link context*/
-        FILL_CTR_LINK_CX(_olink_cx, oldloc->pargfid, oldloc->name,
-                        oldloc->path);
-
-        /*Fill new link context*/
-        FILL_CTR_LINK_CX(_nlink_cx, newloc->pargfid, newloc->name,
-                        newloc->path);
-
-         /*Fill ctr inode context*/
-        FILL_CTR_INODE_CONTEXT(_inode_cx, oldloc->inode->ia_type,
-                oldloc->inode->gfid, _nlink_cx, _olink_cx,
-                GFDB_FOP_DENTRY_WRITE, GFDB_FOP_WIND);
-
-        /*record into the database*/
-        ret = ctr_insert_wind(frame, this, _inode_cx);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR,
-                                "Failed inserting rename wind");
-        }
-
-out:
-        STACK_WIND (frame, ctr_rename_cbk, FIRST_CHILD (this),
-                    FIRST_CHILD (this)->fops->rename,
-                    oldloc, newloc, xdata);
-        return 0;
-}
-
-/****************************unlink******************************************/
-int32_t
-ctr_unlink_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                    int32_t op_ret, int32_t op_errno, struct iatt *preparent,
-                    struct iatt *postparent, dict_t *xdata)
-{
-        int ret                                 = -1;
-        uint32_t remaining_links                = -1;
-
-        CTR_IS_DISABLED_THEN_GOTO(this, out);
-
-        if (!xdata)
-                goto out;
-
-        /*
-         *
-         * Extracting CTR_RESPONSE_LINK_COUNT_XDATA from POSIX Xlator
-         *
-         * */
-        ret = dict_get_uint32 (xdata , CTR_RESPONSE_LINK_COUNT_XDATA,
-                                &remaining_links);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "Failed to getting CTR_RESPONSE_LINK_COUNT_XDATA");
-                remaining_links = -1;
-        }
-
-        /*As the xdata is no more required by CTR Xlator.*/
-        if (xdata) {
-                dict_unref (xdata);
-        }
-
-        /*This is not the only link*/
-        if (remaining_links != 1) {
-
-                ret = ctr_insert_unwind(frame, this, GFDB_FOP_DENTRY_WRITE,
-                                        GFDB_FOP_UNDEL);
-                if (ret) {
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "Failed inserting unlink unwind");
-                }
-        }
-        /*Last link that was deleted*/
-        else if (remaining_links == 1) {
-                ret = ctr_insert_unwind(frame, this, GFDB_FOP_DENTRY_WRITE,
-                                        GFDB_FOP_UNDEL_ALL);
-                if (ret) {
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "Failed inserting unlink unwind");
-                }
-        }
-
-
-
-out:
-        STACK_UNWIND_STRICT (unlink, frame, op_ret, op_errno, preparent,
-                        postparent, NULL);
-
-        return 0;
-}
-
-int32_t
-ctr_unlink (call_frame_t *frame, xlator_t *this,
-                loc_t *loc, int xflag, dict_t *xdata)
-{
-        int ret = -1;
-        gf_ctr_inode_context_t ctr_inode_cx;
-        gf_ctr_inode_context_t *_inode_cx = &ctr_inode_cx;
-        gf_ctr_link_context_t ctr_link_cx;
-        gf_ctr_link_context_t *_link_cx = &ctr_link_cx;
-
-        GF_ASSERT (frame);
-
-        CTR_IS_DISABLED_THEN_GOTO(this, out);
-
-        /*Fill link context*/
-        FILL_CTR_LINK_CX(_link_cx, loc->pargfid, loc->name, loc->path);
-
-         /*Fill ctr inode context*/
-        FILL_CTR_INODE_CONTEXT(_inode_cx, loc->inode->ia_type,
-                loc->inode->gfid, _link_cx, NULL,
-                GFDB_FOP_DENTRY_WRITE, GFDB_FOP_WDEL);
-
-        /*Internal FOP*/
-        _inode_cx->is_internal_fop = CTR_IS_INTERNAL_FOP(frame, xdata);
-
-        /*record into the database*/
-        ret = ctr_insert_wind(frame, this, _inode_cx);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR,
-                                "Failed inserting unlink wind");
-        }
-
-
-
-        /*
-         *
-         * Sending CTR_REQUEST_LINK_COUNT_XDATA
-         * to POSIX Xlator to send link count in unwind path
-         *
-         * */
-        /*create xdata if NULL*/
-        if (!xdata)
-                xdata = dict_new();
-        if (!xdata) {
-                gf_log (this->name, GF_LOG_ERROR,
-                                "xdata is NULL :"
-                                "Cannot send CTR_REQUEST_LINK_COUNT_XDATA"
-                                "to posix");
-                goto out;
-        }
-
-        ret = dict_set_int32 (xdata, CTR_REQUEST_LINK_COUNT_XDATA, 1);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR,
-                                "Failed setting CTR_REQUEST_LINK_COUNT_XDATA");
-                if (xdata) {
-                        dict_unref (xdata);
-                }
-                goto out;
-        }
-
-
-
-out:
-        STACK_WIND (frame, ctr_unlink_cbk, FIRST_CHILD (this),
-                    FIRST_CHILD (this)->fops->unlink,
-                    loc, xflag, xdata);
-        return 0;
-}
-
-/****************************fsync******************************************/
-int32_t
-ctr_fsync_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                   int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
-                   struct iatt *postbuf, dict_t *xdata)
-{
-        int ret = -1;
-
-        CTR_IS_DISABLED_THEN_GOTO(this, out);
-
-        ret = ctr_insert_unwind(frame, this, GFDB_FOP_INODE_WRITE,
-                                GFDB_FOP_UNWIND);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "Failed inserting fsync unwind");
-        }
-
-out:
-        STACK_UNWIND_STRICT (fsync, frame, op_ret, op_errno, prebuf, postbuf,
-                      xdata);
-
-        return 0;
-}
-
-int32_t
-ctr_fsync (call_frame_t *frame, xlator_t *this, fd_t *fd,
-               int32_t flags, dict_t *xdata)
-{
-        int ret = -1;
-        gf_ctr_inode_context_t ctr_inode_cx;
-        gf_ctr_inode_context_t *_inode_cx = &ctr_inode_cx;
-
-        CTR_IS_DISABLED_THEN_GOTO(this, out);
-        CTR_IF_INTERNAL_FOP_THEN_GOTO (frame, xdata, out);
-
-         /*Fill ctr inode context*/
-        FILL_CTR_INODE_CONTEXT(_inode_cx, fd->inode->ia_type,
-                fd->inode->gfid, NULL, NULL,
-                GFDB_FOP_INODE_WRITE, GFDB_FOP_WIND);
-
-        /*record into the database*/
-        ret = ctr_insert_wind(frame, this, _inode_cx);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR,
-                                "Failed inserting fsync wind");
-        }
-
-out:
-        STACK_WIND (frame, ctr_fsync_cbk, FIRST_CHILD (this),
-                    FIRST_CHILD (this)->fops->fsync,
-                    fd, flags, xdata);
-        return 0;
-}
-
-/****************************setxattr****************************************/
-
-int
-ctr_setxattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                  int32_t op_ret, int32_t op_errno, dict_t *xdata)
-{
-        int ret = -1;
-
-        CTR_IS_DISABLED_THEN_GOTO(this, out);
-
-        ret = ctr_insert_unwind(frame, this, GFDB_FOP_INODE_WRITE,
-                                GFDB_FOP_UNWIND);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "Failed inserting fsync unwind");
-        }
-
-
-out:
-        STACK_UNWIND_STRICT (setxattr, frame, op_ret, op_errno, xdata);
-
-        return 0;
-}
-
-int
-ctr_setxattr (call_frame_t *frame, xlator_t *this,
-              loc_t *loc, dict_t *xattr, int flags, dict_t *xdata)
-{
-        int ret = -1;
-        gf_ctr_inode_context_t ctr_inode_cx;
-        gf_ctr_inode_context_t *_inode_cx = &ctr_inode_cx;
-
-        CTR_IS_DISABLED_THEN_GOTO(this, out);
-        CTR_IF_INTERNAL_FOP_THEN_GOTO (frame, xdata, out);
-
-         /*Fill ctr inode context*/
-        FILL_CTR_INODE_CONTEXT(_inode_cx, loc->inode->ia_type,
-                loc->inode->gfid, NULL, NULL,
-                GFDB_FOP_INODE_WRITE, GFDB_FOP_WIND);
-
-        /*record into the database*/
-        ret = ctr_insert_wind(frame, this, _inode_cx);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR,
-                                "Failed inserting setxattr wind");
-        }
-
-out:
-        STACK_WIND (frame, ctr_setxattr_cbk, FIRST_CHILD (this),
-                    FIRST_CHILD (this)->fops->setxattr,
-                    loc, xattr, flags, xdata);
-        return 0;
-}
-
-/****************************mknod*******************************************/
-
-
-int32_t
-ctr_mknod_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                 int32_t op_ret, int32_t op_errno, inode_t *inode,
-                 struct iatt *buf, struct iatt *preparent,
-                 struct iatt *postparent, dict_t *xdata)
-{
-        int ret = -1;
-
-        CTR_IS_DISABLED_THEN_GOTO(this, out);
-
-        ret = ctr_insert_unwind(frame, this, GFDB_FOP_CREATE_WRITE,
-                                GFDB_FOP_UNWIND);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "Failed inserting mknod unwind");
-        }
-
-out:
-        STACK_UNWIND_STRICT (mknod, frame, op_ret, op_errno, inode, buf,
-                preparent, postparent, xdata);
-
-        return 0;
-}
-
-
-int
-ctr_mknod (call_frame_t *frame, xlator_t *this,
-           loc_t *loc, mode_t mode, dev_t rdev, mode_t umask, dict_t *xdata)
-{
-        int ret = -1;
-        gf_ctr_inode_context_t ctr_inode_cx;
-        gf_ctr_inode_context_t *_inode_cx = &ctr_inode_cx;
-        gf_ctr_link_context_t  ctr_link_cx;
-        gf_ctr_link_context_t  *_link_cx = &ctr_link_cx;
-        void *uuid_req                          = NULL;
-        uuid_t gfid                             = {0,};
-        uuid_t *ptr_gfid                        = &gfid;
-
-        CTR_IS_DISABLED_THEN_GOTO(this, out);
-
-        GF_ASSERT(frame);
-        GF_ASSERT(frame->root);
-
-        /*get gfid from xdata dict*/
-        ret = dict_get_ptr (xdata, "gfid-req", &uuid_req);
-        if (ret) {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "failed to get gfid from dict");
-                goto out;
-        }
-        gf_uuid_copy (gfid, uuid_req);
-
-        /*fill ctr link context*/
-        FILL_CTR_LINK_CX (_link_cx, loc->pargfid, loc->name, loc->path);
-
-         /*Fill ctr inode context*/
-        FILL_CTR_INODE_CONTEXT (_inode_cx, loc->inode->ia_type,
-                *ptr_gfid, _link_cx, NULL,
-                GFDB_FOP_CREATE_WRITE, GFDB_FOP_WIND);
-
-        /*Internal FOP*/
-        _inode_cx->is_internal_fop = CTR_IS_INTERNAL_FOP(frame, xdata);
-
-        /*record into the database*/
-        ret = ctr_insert_wind(frame, this, _inode_cx);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR,
-                                "Failed inserting mknod wind");
-        }
-
-out:
-        STACK_WIND (frame, ctr_mknod_cbk, FIRST_CHILD (this),
-                    FIRST_CHILD (this)->fops->mknod,
-                    loc, mode, rdev, umask, xdata);
-        return 0;
-}
-
-/****************************create******************************************/
-int
-ctr_create_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                int op_ret, int op_errno,
-                fd_t *fd, inode_t *inode, struct iatt *stbuf,
-                struct iatt *preparent, struct iatt *postparent,
-                dict_t *xdata)
-{
-        int ret = -1;
-
-        CTR_IS_DISABLED_THEN_GOTO(this, out);
-
-        ret = ctr_insert_unwind(frame, this, GFDB_FOP_CREATE_WRITE,
-                                GFDB_FOP_UNWIND);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "Failed inserting create unwind");
-        }
-
-out:
-        STACK_UNWIND_STRICT (create, frame, op_ret, op_errno, fd, inode,
-                             stbuf,
-                        preparent, postparent, xdata);
-
-        return 0;
-}
-
-int
-ctr_create (call_frame_t *frame, xlator_t *this,
-            loc_t *loc, int32_t flags, mode_t mode,
-            mode_t umask, fd_t *fd, dict_t *xdata)
-{
-        int ret = -1;
-        gf_ctr_inode_context_t ctr_inode_cx;
-        gf_ctr_inode_context_t *_inode_cx = &ctr_inode_cx;
-        gf_ctr_link_context_t  ctr_link_cx;
-        gf_ctr_link_context_t  *_link_cx = &ctr_link_cx;
-        void             *uuid_req              = NULL;
-        uuid_t            gfid                  = {0,};
-        uuid_t            *ptr_gfid             = &gfid;
-
-        CTR_IS_DISABLED_THEN_GOTO(this, out);
-
-        GF_ASSERT(frame);
-        GF_ASSERT(frame->root);
-
-        /*Get GFID from Xdata dict*/
-        ret = dict_get_ptr (xdata, "gfid-req", &uuid_req);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "failed to get gfid from dict");
-                goto out;
-        }
-        gf_uuid_copy (gfid, uuid_req);
-
-        /*fill ctr link context*/
-        FILL_CTR_LINK_CX(_link_cx, loc->pargfid, loc->name, loc->path);
-
-         /*Fill ctr inode context*/
-        FILL_CTR_INODE_CONTEXT(_inode_cx, loc->inode->ia_type,
-                *ptr_gfid, _link_cx, NULL,
-                GFDB_FOP_CREATE_WRITE, GFDB_FOP_WIND);
-
-        /*Internal FOP*/
-        _inode_cx->is_internal_fop = CTR_IS_INTERNAL_FOP(frame, xdata);
-
-        /*record into the database*/
-        ret = ctr_insert_wind(frame, this, &ctr_inode_cx);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR,
-                                "Failed inserting create wind");
-        }
-out:
-        STACK_WIND (frame, ctr_create_cbk, FIRST_CHILD (this),
-                    FIRST_CHILD (this)->fops->create,
-                    loc, flags, mode, umask, fd, xdata);
-        return 0;
-}
-
-/****************************link********************************************/
-
-int
-ctr_link_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-              int op_ret, int op_errno,
-              inode_t *inode, struct iatt *stbuf, struct iatt *preparent,
-              struct iatt *postparent, dict_t *xdata)
-{
-        int ret = -1;
-
-        CTR_IS_DISABLED_THEN_GOTO(this, out);
-
-        ret = ctr_insert_unwind(frame, this, GFDB_FOP_DENTRY_WRITE,
-                                GFDB_FOP_UNWIND);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "Failed inserting create unwind");
-        }
-
-out:
-        STACK_UNWIND_STRICT (link, frame, op_ret, op_errno, inode, stbuf,
-                       preparent, postparent, xdata);
-        return 0;
-}
-
-int
-ctr_link (call_frame_t *frame, xlator_t *this,
-          loc_t *oldloc, loc_t *newloc, dict_t *xdata)
-{
-        int ret = -1;
-        gf_ctr_inode_context_t ctr_inode_cx;
-        gf_ctr_inode_context_t *_inode_cx = &ctr_inode_cx;
-        gf_ctr_link_context_t  ctr_link_cx;
-        gf_ctr_link_context_t  *_link_cx = &ctr_link_cx;
-
-        CTR_IS_DISABLED_THEN_GOTO(this, out);
-
-        GF_ASSERT(frame);
-        GF_ASSERT(frame->root);
-
-        /*fill ctr link context*/
-        FILL_CTR_LINK_CX(_link_cx, newloc->pargfid, newloc->name,
-                        newloc->path);
-
-         /*Fill ctr inode context*/
-        FILL_CTR_INODE_CONTEXT(_inode_cx, oldloc->inode->ia_type,
-                oldloc->inode->gfid, _link_cx, NULL,
-                GFDB_FOP_DENTRY_WRITE, GFDB_FOP_WIND);
-
-        /*Internal FOP*/
-        _inode_cx->is_internal_fop = CTR_IS_INTERNAL_FOP(frame, xdata);
-
-        /*record into the database*/
-        ret = ctr_insert_wind(frame, this, _inode_cx);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR,
-                                "Failed inserting link wind");
-        }
-
-out:
-        STACK_WIND (frame, ctr_link_cbk, FIRST_CHILD (this),
-                    FIRST_CHILD (this)->fops->link,
-                    oldloc, newloc, xdata);
-        return 0;
-}
-
-/******************************readv*****************************************/
-int ctr_readv_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-               int op_ret, int op_errno,
-               struct iovec *vector, int count, struct iatt *stbuf,
-               struct iobref *iobref, dict_t *xdata) {
-
-        int ret = -1;
-
-        CTR_IS_DISABLED_THEN_GOTO(this, out);
-
-        ret = ctr_insert_unwind(frame, this, GFDB_FOP_INODE_READ,
-                                GFDB_FOP_UNWIND);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "Failed inserting create unwind");
-        }
-
-out:
-        STACK_UNWIND_STRICT (readv, frame, op_ret, op_errno, vector, count,
-                                stbuf, iobref, xdata);
-        return 0;
-}
-
-
-int
-ctr_readv (call_frame_t *frame, xlator_t *this,
-           fd_t *fd, size_t size, off_t off, uint32_t flags, dict_t *xdata)
-{
-        int ret = -1;
-        gf_ctr_inode_context_t ctr_inode_cx;
-        gf_ctr_inode_context_t *_inode_cx = &ctr_inode_cx;
-
-        CTR_IS_DISABLED_THEN_GOTO(this, out);
-        CTR_IF_INTERNAL_FOP_THEN_GOTO (frame, xdata, out);
-
-         /*Fill ctr inode context*/
-        FILL_CTR_INODE_CONTEXT(_inode_cx, fd->inode->ia_type,
-                fd->inode->gfid, NULL, NULL,
-                GFDB_FOP_INODE_READ, GFDB_FOP_WIND);
-
-        /*record into the database*/
-        ret = ctr_insert_wind(frame, this, _inode_cx);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR,
-                                "Failed inserting readv wind");
-        }
-
-out:
-        STACK_WIND (frame, ctr_readv_cbk, FIRST_CHILD (this),
-                    FIRST_CHILD (this)->fops->readv,
-                    fd, size, off, flags, xdata);
-        return 0;
-}
-/******************************************************************************/
-
-int
-reconfigure (xlator_t *this, dict_t *options)
-{
-        char *temp_str = NULL;
-        int ret = 0;
-        gf_ctr_private_t *_priv = NULL;
-
-        _priv = this->private;
-        if (dict_get_str(options, "changetimerecorder.frequency",
-                         &temp_str)) {
-                gf_log(this->name, GF_LOG_INFO, "set!");
-        }
-
-        GF_OPTION_RECONF ("ctr-enabled", _priv->enabled, options,
-                          bool, out);
-
-        GF_OPTION_RECONF ("record-counters", _priv->ctr_record_counter, options,
-                          bool, out);
-
-        GF_OPTION_RECONF ("record-exit", _priv->ctr_record_unwind, options,
-                          bool, out);
-
-        GF_OPTION_RECONF ("record-entry", _priv->ctr_record_wind, options,
-                          bool, out);
-
-out:
-
-        return ret;
-}
-
-/****************************init********************************************/
-
-int32_t
-init (xlator_t *this)
-{
-        gf_ctr_private_t *_priv = NULL;
-        int ret_db              = -1;
-        dict_t *params_dict      = NULL;
-
-        GF_VALIDATE_OR_GOTO ("ctr", this, error);
-
-        if (!this->children || this->children->next) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "FATAL: ctr should have exactly one child");
-                goto error;
-        }
-
-        if (!this->parents) {
-                gf_log (this->name, GF_LOG_WARNING,
-                        "dangling volume. check volfile ");
-        }
-
-        _priv = GF_CALLOC (1, sizeof (*_priv), gf_ctr_mt_private_t);
-        if (!_priv) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "Calloc didnt work!!!");
-                goto error;
-        }
-
-        /*Default values for the translator*/
-        _priv->ctr_record_wind          = _gf_true;
-        _priv->ctr_record_unwind        = _gf_false;
-        _priv->ctr_hot_brick            = _gf_false;
-        _priv->gfdb_db_type             = GFDB_SQLITE3;
-        _priv->gfdb_sync_type           = GFDB_DB_SYNC;
-        _priv->enabled                  = _gf_true;
-        _priv->_db_conn                 = NULL;
-
-        /*Extract ctr xlator options*/
-        ret_db = extract_ctr_options (this, _priv);
-        if (ret_db) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "Failed extracting ctr xlator options");
-                goto error;
-        }
-
-        params_dict = dict_new ();
-        if (!params_dict) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "DB Params cannot initialized!");
-                goto error;
-        }
-
-        /*Extract db params options*/
-        ret_db = extract_db_params(this, params_dict, _priv->gfdb_db_type);
-        if (ret_db) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "Failed extracting db params options");
-                goto error;
-        }
-
-        /*Create a memory pool for ctr xlator*/
-        this->local_pool = mem_pool_new (gf_ctr_local_t, 64);
-        if (!this->local_pool) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "failed to create local memory pool");
-                goto error;
-        }
-
-        /*Initialize Database Connection*/
-        _priv->_db_conn = init_db(params_dict, _priv->gfdb_db_type);
-        if (!_priv->_db_conn) {
-                gf_log (this->name, GF_LOG_ERROR,
-                       "FATAL: Failed initializing data base");
-                        goto error;
-        }
-
-        ret_db = 0;
-        goto out;
-
-/*Error handling */
-error:
-
-        if (this)
-                mem_pool_destroy (this->local_pool);
-
-        if (_priv) {
-                GF_FREE (_priv->ctr_db_path);
-        }
-        GF_FREE (_priv);
-
-        if (params_dict)
-                dict_unref (params_dict);
-
-        return -1;
-
-out:
-
-        if (params_dict)
-                dict_unref (params_dict);
-
-        this->private = (void *)_priv;
-        return 0;
-}
-
-int32_t
-mem_acct_init (xlator_t *this)
-{
-        int     ret = -1;
-
-        GF_VALIDATE_OR_GOTO ("ctr", this, out);
-
-        ret = xlator_mem_acct_init (this, gf_ctr_mt_end + 1);
-
-        if (ret != 0) {
-                gf_log (this->name, GF_LOG_ERROR, "Memory accounting init"
-                        "failed");
-                return ret;
-        }
-out:
-        return ret;
-}
-
-
-void
-fini (xlator_t *this)
-{
-        gf_ctr_private_t *priv = NULL;
-
-        priv = this->private;
-
-        if (priv) {
-                if (fini_db (priv->_db_conn)) {
-                        gf_log (this->name, GF_LOG_WARNING, "Failed closing "
-                        "db connection");
-                }
-                GF_FREE (priv->ctr_db_path);
-        }
-        GF_FREE (priv);
-        mem_pool_destroy (this->local_pool);
-
-        return;
-}
-
-struct xlator_fops fops = {
-        /*write fops */
-        .mknod       = ctr_mknod,
-        .create      = ctr_create,
-        .truncate    = ctr_truncate,
-        .ftruncate   = ctr_ftruncate,
-        .setxattr    = ctr_setxattr,
-        .removexattr = ctr_removexattr,
-        .unlink      = ctr_unlink,
-        .link        = ctr_link,
-        .rename      = ctr_rename,
-        .writev      = ctr_writev,
-        .setattr      = ctr_setattr,
-        /*read fops*/
-        .readv       = ctr_readv
-};
-
-struct xlator_cbks cbks;
-
-struct volume_options options[] = {
-        { .key  = {"ctr-enabled",},
-          .type = GF_OPTION_TYPE_BOOL,
-          .value = {"on", "off"},
-          .default_value = "off",
-          .description = "Enables the CTR"
-        },
-        { .key  = {"record-entry"},
-          .type = GF_OPTION_TYPE_BOOL,
-          .value = {"on", "off"},
-          .default_value = "on"
-        },
-        { .key  = {"record-exit"},
-          .type = GF_OPTION_TYPE_BOOL,
-          .value = {"on", "off"},
-          .default_value = "off"
-        },
-        { .key  = {"record-counters"},
-          .type = GF_OPTION_TYPE_BOOL,
-          .value = {"on", "off"},
-          .default_value = "off"
-        },
-        { .key  = {"hot-brick"},
-          .type = GF_OPTION_TYPE_BOOL,
-          .value = {"on", "off"},
-          .default_value = "off"
-        },
-        { .key  = {"db-type"},
-          .type = GF_OPTION_TYPE_STR,
-          .value = {"hashfile", "rocksdb", "changelog", "sqlite3",
-                    "hyperdex"},
-          .default_value = "sqlite3"
-        },
-        { .key  = {"db-sync"},
-          .type = GF_OPTION_TYPE_STR,
-          .value = {"sync", "async"},
-          .default_value = "sync"
-        },
-        { .key  = {"db-path"},
-          .type = GF_OPTION_TYPE_PATH
-        },
-        { .key  = {"db-name"},
-          .type = GF_OPTION_TYPE_STR
-        },
-        { .key  = {GFDB_SQL_PARAM_SYNC},
-          .type = GF_OPTION_TYPE_STR,
-          .value = {"off", "normal", "full"},
-          .default_value = "normal"
-        },
-        { .key  = {GFDB_SQL_PARAM_JOURNAL_MODE},
-          .type = GF_OPTION_TYPE_STR,
-          .value = {"delete", "truncate", "persist", "memory", "wal", "off"},
-          .default_value = "wal"
-        },
-        { .key  = {GFDB_SQL_PARAM_AUTO_VACUUM},
-          .type = GF_OPTION_TYPE_STR,
-          .value = {"off", "full", "incr"},
-          .default_value = "off"
-        },
-        { .key  = {GFDB_SQL_PARAM_WAL_AUTOCHECK},
-          .type = GF_OPTION_TYPE_INT,
-          .default_value = "1000"
-        },
-        { .key  = {GFDB_SQL_PARAM_CACHE_SIZE},
-          .type = GF_OPTION_TYPE_INT,
-          .default_value = "1000"
-        },
-        { .key  = {GFDB_SQL_PARAM_PAGE_SIZE},
-          .type = GF_OPTION_TYPE_INT,
-          .default_value = "4096"
-        },
-        { .key  = {NULL} },
-};
diff --git a/xlators/features/changetimerecorder/src/changetimerecorder.h b/xlators/features/changetimerecorder/src/changetimerecorder.h
deleted file mode 100644
index 6a963a56a2f..00000000000
--- a/xlators/features/changetimerecorder/src/changetimerecorder.h
+++ /dev/null
@@ -1,26 +0,0 @@
-/*
-   Copyright (c) 2006-2015 Red Hat, Inc. <http://www.redhat.com>
-   This file is part of GlusterFS.
-
-   This file is licensed to you under your choice of the GNU Lesser
-   General Public License, version 3 or any later version (LGPLv3 or
-   later), or the GNU General Public License, version 2 (GPLv2), in all
-   cases as published by the Free Software Foundation.
-*/
-
-#ifndef __CTR_H
-#define __CTR_H
-
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
-
-#include "glusterfs.h"
-#include "xlator.h"
-#include "logging.h"
-#include "common-utils.h"
-#include "ctr_mem_types.h"
-#include "ctr-helper.h"
-
-#endif /* __CTR_H */
diff --git a/xlators/features/changetimerecorder/src/ctr-helper.c b/xlators/features/changetimerecorder/src/ctr-helper.c
deleted file mode 100644
index da5a3deed96..00000000000
--- a/xlators/features/changetimerecorder/src/ctr-helper.c
+++ /dev/null
@@ -1,279 +0,0 @@
-/*
-   Copyright (c) 2015 Red Hat, Inc. <http://www.redhat.com>
-   This file is part of GlusterFS.
-
-   This file is licensed to you under your choice of the GNU Lesser
-   General Public License, version 3 or any later version (LGPLv3 or
-   later), or the GNU General Public License, version 2 (GPLv2), in all
-   cases as published by the Free Software Foundation.
-*/
-
-#include "gfdb_sqlite3.h"
-#include "ctr-helper.h"
-
-
-/*******************************************************************************
- *
- *                      Fill unwind into db record
- *
- ******************************************************************************/
-int
-fill_db_record_for_unwind(xlator_t              *this,
-                          gf_ctr_local_t        *ctr_local,
-                          gfdb_fop_type_t       fop_type,
-                          gfdb_fop_path_t       fop_path)
-{
-        int ret                         = -1;
-        gfdb_time_t *ctr_uwtime         = NULL;
-        gf_ctr_private_t *_priv                 = NULL;
-
-        GF_ASSERT (this);
-        _priv = this->private;
-        GF_ASSERT (_priv);
-
-        GF_ASSERT(ctr_local);
-
-        /*If not unwind path error*/
-        if (!isunwindpath(fop_path)) {
-                gf_log (this->name, GF_LOG_ERROR, "Wrong fop_path."
-                        "Should be unwind");
-                goto out;
-        }
-
-        ctr_uwtime = &CTR_DB_REC(ctr_local).gfdb_unwind_change_time;
-        CTR_DB_REC(ctr_local).gfdb_fop_path = fop_path;
-        CTR_DB_REC(ctr_local).gfdb_fop_type = fop_type;
-
-        ret = gettimeofday (ctr_uwtime, NULL);
-        if (ret == -1) {
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "Error filling unwind time record %s",
-                                strerror(errno));
-                        goto out;
-                }
-
-        /* Special case i.e if its a tier rebalance
-         * + cold tier brick
-         * + its a create/mknod FOP
-         * we record unwind time as zero */
-        if (ctr_local->client_pid == GF_CLIENT_PID_TIER_DEFRAG
-                && (!_priv->ctr_hot_brick)
-                && isdentrycreatefop(fop_type)) {
-                memset(ctr_uwtime, 0, sizeof(*ctr_uwtime));
-        }
-        ret = 0;
-out:
-        return ret;
-}
-
-
-/*******************************************************************************
- *
- *                      Fill wind into db record
- *
- ******************************************************************************/
-int
-fill_db_record_for_wind (xlator_t               *this,
-                        gf_ctr_local_t          *ctr_local,
-                        gf_ctr_inode_context_t  *ctr_inode_cx)
-{
-        int ret                                 = -1;
-        gfdb_time_t *ctr_wtime                  = NULL;
-        gf_ctr_private_t *_priv                 = NULL;
-
-        GF_ASSERT (this);
-        _priv = this->private;
-        GF_ASSERT (_priv);
-        GF_ASSERT (ctr_local);
-        IS_CTR_INODE_CX_SANE (ctr_inode_cx);
-
-        /*if not wind path error!*/
-        if (!iswindpath(ctr_inode_cx->fop_path)) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "Wrong fop_path. Should be wind");
-                goto out;
-        }
-
-        ctr_wtime = &CTR_DB_REC(ctr_local).gfdb_wind_change_time;
-        CTR_DB_REC(ctr_local).gfdb_fop_path = ctr_inode_cx->fop_path;
-        CTR_DB_REC(ctr_local).gfdb_fop_type = ctr_inode_cx->fop_type;
-
-        ret = gettimeofday (ctr_wtime, NULL);
-        if (ret) {
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "Error filling wind time record %s",
-                                strerror(errno));
-                        goto out;
-        }
-
-        /* Special case i.e if its a tier rebalance
-         * + cold tier brick
-         * + its a create/mknod FOP
-         * we record wind time as zero */
-        if (ctr_local->client_pid == GF_CLIENT_PID_TIER_DEFRAG
-                && (!_priv->ctr_hot_brick)
-                && isdentrycreatefop(ctr_inode_cx->fop_type)) {
-                memset(ctr_wtime, 0, sizeof(*ctr_wtime));
-        }
-
-        /*Copy gfid into db record*/
-        gf_uuid_copy (CTR_DB_REC(ctr_local).gfid, *(ctr_inode_cx->gfid));
-
-        /*Hard Links*/
-        if (isdentryfop(ctr_inode_cx->fop_type)) {
-                /*new link fop*/
-                if (NEW_LINK_CX(ctr_inode_cx)) {
-                        gf_uuid_copy (CTR_DB_REC(ctr_local).pargfid,
-                                *((NEW_LINK_CX(ctr_inode_cx))->pargfid));
-                        strcpy (CTR_DB_REC(ctr_local).file_name,
-                                NEW_LINK_CX(ctr_inode_cx)->basename);
-                        strcpy (CTR_DB_REC(ctr_local).file_path,
-                                NEW_LINK_CX(ctr_inode_cx)->basepath);
-                }
-                /*rename fop*/
-                if (OLD_LINK_CX(ctr_inode_cx)) {
-                        gf_uuid_copy (CTR_DB_REC(ctr_local).old_pargfid,
-                                *((OLD_LINK_CX(ctr_inode_cx))->pargfid));
-                        strcpy (CTR_DB_REC(ctr_local).old_file_name,
-                                OLD_LINK_CX(ctr_inode_cx)->basename);
-                        strcpy (CTR_DB_REC(ctr_local).old_path,
-                                OLD_LINK_CX(ctr_inode_cx)->basepath);
-                }
-        }
-
-        ret = 0;
-out:
-        /*On error roll back and clean the record*/
-        if (ret == -1) {
-                CLEAR_CTR_DB_RECORD (ctr_local);
-        }
-        return ret;
-}
-
-
-/******************************************************************************
- *
- *                      CTR xlator init related functions
- *
- *
- * ****************************************************************************/
-static int
-extract_sql_params(xlator_t *this, dict_t *params_dict)
-{
-        int ret                         = -1;
-        char *db_path                   = NULL;
-        char *db_name                   = NULL;
-        char *db_full_path              = NULL;
-
-        GF_ASSERT (this);
-        GF_ASSERT (params_dict);
-
-        /*Extract the path of the db*/
-        db_path = NULL;
-        GET_DB_PARAM_FROM_DICT_DEFAULT(this->name, this->options, "db-path",
-                                        db_path, "/var/run/gluster/");
-
-        /*Extract the name of the db*/
-        db_name = NULL;
-        GET_DB_PARAM_FROM_DICT_DEFAULT(this->name, this->options, "db-name",
-                                        db_name, "gf_ctr_db.db");
-
-        /*Construct full path of the db*/
-        ret = gf_asprintf(&db_full_path, "%s/%s", db_path, db_name);
-        if (ret < 0) {
-                gf_log (GFDB_DATA_STORE, GF_LOG_ERROR,
-                                "Construction of full db path failed!");
-                goto out;
-        }
-
-        /*Setting the SQL DB Path*/
-        SET_DB_PARAM_TO_DICT(this->name, params_dict, GFDB_SQL_PARAM_DBPATH,
-                                db_full_path, ret, out);
-
-        /*Extact rest of the sql params*/
-        ret = gfdb_set_sql_params(this->name, this->options, params_dict);
-        if (ret) {
-                gf_log (GFDB_DATA_STORE, GF_LOG_ERROR,
-                                "Failed setting values to sql param dict!");
-        }
-
-        ret = 0;
-
-out:
-        if (ret)
-                GF_FREE (db_full_path);
-        return ret;
-}
-
-
-
-int extract_db_params(xlator_t *this, dict_t *params_dict,
-                                                gfdb_db_type_t db_type) {
-
-        int ret = -1;
-
-        GF_ASSERT (this);
-        GF_ASSERT (params_dict);
-
-        switch (db_type) {
-        case GFDB_SQLITE3:
-                ret = extract_sql_params(this, params_dict);
-                if (ret)
-                        goto out;
-                break;
-        case GFDB_ROCKS_DB:
-        case GFDB_HYPERDEX:
-        case GFDB_HASH_FILE_STORE:
-        case GFDB_INVALID_DB:
-        case GFDB_DB_END:
-                ret = -1;
-                break;
-        }
-        ret = 0;
-out:
-        return ret;
-}
-
-int extract_ctr_options (xlator_t *this, gf_ctr_private_t *_priv) {
-        int ret         = -1;
-        char *_val_str  = NULL;
-
-        GF_ASSERT (this);
-        GF_ASSERT (_priv);
-
-        /*Checking if the CTR Translator is enabled. By Default its enabled*/
-        _priv->enabled = _gf_false;
-        GF_OPTION_INIT ("ctr-enabled", _priv->enabled, bool, out);
-        if (!_priv->enabled) {
-                gf_log (GFDB_DATA_STORE, GF_LOG_ERROR,
-                                "CTR Xlator is Disable!");
-                ret = 0;
-                goto out;
-        }
-
-        /*Extract db type*/
-        GF_OPTION_INIT ("db-type", _val_str, str, out);
-        _priv->gfdb_db_type = gf_string2gfdbdbtype(_val_str);
-
-        /*Extract flag for record on wind*/
-        GF_OPTION_INIT ("record-entry", _priv->ctr_record_wind, bool, out);
-
-        /*Extract flag for record on unwind*/
-        GF_OPTION_INIT ("record-exit", _priv->ctr_record_unwind, bool, out);
-
-        /*Extract flag for record on counters*/
-        GF_OPTION_INIT ("record-counters", _priv->ctr_record_counter, bool,
-                        out);
-
-        /*Extract flag for hot tier brick*/
-        GF_OPTION_INIT ("hot-brick", _priv->ctr_hot_brick, bool, out);
-
-        /*Extract flag for sync mode*/
-        GF_OPTION_INIT ("db-sync", _val_str, str, out);
-        _priv->gfdb_sync_type = gf_string2gfdbdbsync(_val_str);
-
-        ret = 0;
-
-out:
-        return ret;
-}
diff --git a/xlators/features/changetimerecorder/src/ctr-helper.h b/xlators/features/changetimerecorder/src/ctr-helper.h
deleted file mode 100644
index 6f4ad5cbe7a..00000000000
--- a/xlators/features/changetimerecorder/src/ctr-helper.h
+++ /dev/null
@@ -1,478 +0,0 @@
-/*
-   Copyright (c) 2015 Red Hat, Inc. <http://www.redhat.com>
-   This file is part of GlusterFS.
-
-   This file is licensed to you under your choice of the GNU Lesser
-   General Public License, version 3 or any later version (LGPLv3 or
-   later), or the GNU General Public License, version 2 (GPLv2), in all
-   cases as published by the Free Software Foundation.
-*/
-
-#ifndef __CTR_HELPER_H
-#define __CTR_HELPER_H
-
-
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
-
-#include "xlator.h"
-#include "ctr_mem_types.h"
-#include "iatt.h"
-#include "glusterfs.h"
-#include "xlator.h"
-#include "logging.h"
-#include "common-utils.h"
-#include <time.h>
-#include <sys/time.h>
-
-#include "gfdb_data_store.h"
-
-/*CTR Xlator Private structure*/
-typedef struct gf_ctr_private {
-        gf_boolean_t                    enabled;
-        char                            *ctr_db_path;
-        gf_boolean_t                    ctr_hot_brick;
-        gf_boolean_t                    ctr_record_wind;
-        gf_boolean_t                    ctr_record_unwind;
-        gf_boolean_t                    ctr_record_counter;
-        gfdb_db_type_t                  gfdb_db_type;
-        gfdb_sync_type_t                gfdb_sync_type;
-        gfdb_conn_node_t                *_db_conn;
-} gf_ctr_private_t;
-
-
-/*
- * gf_ctr_local_t is the ctr xlator local data structure that is stored in
- * the call_frame of each FOP.
- *
- * gfdb_db_record: The gf_ctr_local contains a gfdb_db_record object, which is
- * used by the insert_record() api from the libgfdb. The gfdb_db_record object
- * will contain all the inode and hardlink(only for dentry fops: create,
- * mknod,link, unlink, rename).The ctr_local is keep alive till the unwind
- * call and will be release during the unwind. The same gfdb_db_record will
- * used for the unwind insert_record() api, to record unwind in the database.
- *
- * ia_inode_type in gf_ctr_local will tell the type of the inode. This is
- * important for during the unwind path. As we will not have the inode during
- * the unwind path. We would have include this in the gfdb_db_record itself
- * but currently we record only file inode information.
- *
- * is_internal_fop in gf_ctr_local will tell us if this is a internal fop and
- * take special/no action. We dont record change/acces times or increement heat
- * counter for internal fops from rebalancer.
- * */
-typedef struct gf_ctr_local {
-        gfdb_db_record_t        gfdb_db_record;
-        ia_type_t               ia_inode_type;
-        gf_boolean_t            is_internal_fop;
-        gf_client_pid_t          client_pid;
-} gf_ctr_local_t;
-/*
- * Easy access of gfdb_db_record of ctr_local
- * */
-#define CTR_DB_REC(ctr_local)\
-        (ctr_local->gfdb_db_record)
-
-/*Clear db record*/
-#define CLEAR_CTR_DB_RECORD(ctr_local)\
-do {\
-        ctr_local->gfdb_db_record.gfdb_fop_path = GFDB_FOP_INVALID;\
-        memset(&(ctr_local->gfdb_db_record.gfdb_wind_change_time),\
-                0, sizeof(gfdb_time_t));\
-        memset(&(ctr_local->gfdb_db_record.gfdb_unwind_change_time),\
-                0, sizeof(gfdb_time_t));\
-        gf_uuid_clear (ctr_local->gfdb_db_record.gfid);\
-        gf_uuid_clear (ctr_local->gfdb_db_record.pargfid);\
-        memset(ctr_local->gfdb_db_record.file_name, 0, PATH_MAX);\
-        memset(ctr_local->gfdb_db_record.old_file_name, 0, PATH_MAX);\
-        ctr_local->gfdb_db_record.gfdb_fop_type = GFDB_FOP_INVALID_OP;\
-        ctr_local->ia_inode_type = IA_INVAL;\
-} while (0)
-
-
-static gf_ctr_local_t *
-init_ctr_local_t (xlator_t *this) {
-
-        gf_ctr_local_t  *ctr_local     = NULL;
-
-        GF_ASSERT(this);
-
-        ctr_local = mem_get0 (this->local_pool);
-        if (!ctr_local) {
-                gf_log (GFDB_DATA_STORE, GF_LOG_ERROR,
-                        "Error while creating ctr local");
-                goto out;
-        }
-
-        CLEAR_CTR_DB_RECORD (ctr_local);
-out:
-        return ctr_local;
-}
-
-static void
-free_ctr_local (gf_ctr_local_t *ctr_local)
-{
-        if (ctr_local)
-                mem_put (ctr_local);
-}
-
-
-
-/******************************************************************************
- *
- *
- *                      Context Carrier Structures
- *
- *
- * ****************************************************************************/
-
-/*
- * Context Carrier structures are used to carry relavent information about
- * inodes and links from the fops calls to the ctr_insert_wind.
- * These structure just have pointers to the original data and donot
- * do a deep copy of any data. This info is deep copied to
- * ctr_local->gfdb_db_record and passed to insert_record() api of libgfdb. This
- * info remains persistent for the unwind in  ctr_local->gfdb_db_record
- * and once used will be destroyed.
- *
- * gf_ctr_link_context_t : Context structure for hard links
- * gf_ctr_inode_context_t : Context structure for inodes
- *
- * */
-
- /*Context Carrier Structure for hard links*/
-typedef struct gf_ctr_link_context {
-        uuid_t                  *pargfid;
-        const char              *basename;
-        /*basepath is redundent. Will go off*/
-        const char              *basepath;
-} gf_ctr_link_context_t;
-
- /*Context Carrier Structure for inodes*/
-typedef struct gf_ctr_inode_context {
-        ia_type_t               ia_type;
-        uuid_t                  *gfid;
-        gf_ctr_link_context_t   *new_link_cx;
-        gf_ctr_link_context_t   *old_link_cx;
-        gfdb_fop_type_t         fop_type;
-        gfdb_fop_path_t         fop_path;
-        gf_boolean_t            is_internal_fop;
-} gf_ctr_inode_context_t;
-
-
-/*******************Util Macros for Context Carrier Structures*****************/
-
-/*Checks if ctr_link_cx is sane!*/
-#define IS_CTR_LINK_CX_SANE(ctr_link_cx)\
-do {\
-        if (ctr_link_cx) {\
-                if (ctr_link_cx->pargfid)\
-                        GF_ASSERT (*(ctr_link_cx->pargfid));\
-                GF_ASSERT (ctr_link_cx->basename);\
-                GF_ASSERT (ctr_link_cx->basepath);\
-        };\
-} while (0)
-
-/*Clear and fill the ctr_link_context with values*/
-#define FILL_CTR_LINK_CX(ctr_link_cx, _pargfid, _basename, _basepath)\
-do {\
-        GF_ASSERT (ctr_link_cx);\
-        GF_ASSERT (_pargfid);\
-        GF_ASSERT (_basename);\
-        GF_ASSERT (_basepath);\
-        memset (ctr_link_cx, 0, sizeof (*ctr_link_cx));\
-        ctr_link_cx->pargfid = &_pargfid;\
-        ctr_link_cx->basename = _basename;\
-        ctr_link_cx->basepath = _basepath;\
-} while (0)
-
-#define NEW_LINK_CX(ctr_inode_cx)\
-        ctr_inode_cx->new_link_cx\
-
-#define OLD_LINK_CX(ctr_inode_cx)\
-        ctr_inode_cx->old_link_cx\
-
-/*Checks if ctr_inode_cx is sane!*/
-#define IS_CTR_INODE_CX_SANE(ctr_inode_cx)\
-do {\
-        GF_ASSERT (ctr_inode_cx);\
-        GF_ASSERT (ctr_inode_cx->gfid);\
-        GF_ASSERT (*(ctr_inode_cx->gfid));\
-        GF_ASSERT (ctr_inode_cx->fop_type != GFDB_FOP_INVALID_OP);\
-        GF_ASSERT (ctr_inode_cx->fop_path != GFDB_FOP_INVALID);\
-        IS_CTR_LINK_CX_SANE (NEW_LINK_CX(ctr_inode_cx));\
-        IS_CTR_LINK_CX_SANE (OLD_LINK_CX(ctr_inode_cx));\
-} while (0)
-
-/*Clear and fill the ctr_inode_context with values*/
-#define FILL_CTR_INODE_CONTEXT(ctr_inode_cx,\
-                                _ia_type,\
-                                _gfid,\
-                                _new_link_cx,\
-                                _old_link_cx,\
-                                _fop_type,\
-                                _fop_path)\
-do {\
-        GF_ASSERT(ctr_inode_cx);\
-        GF_ASSERT(_gfid);\
-        GF_ASSERT(_fop_type != GFDB_FOP_INVALID_OP);\
-        GF_ASSERT(_fop_path != GFDB_FOP_INVALID);\
-        memset(ctr_inode_cx, 0, sizeof(*ctr_inode_cx));\
-        ctr_inode_cx->ia_type = _ia_type;\
-        ctr_inode_cx->gfid = &_gfid;\
-        IS_CTR_LINK_CX_SANE(NEW_LINK_CX(ctr_inode_cx));\
-        if (_new_link_cx)\
-                NEW_LINK_CX(ctr_inode_cx) = _new_link_cx;\
-        IS_CTR_LINK_CX_SANE(OLD_LINK_CX(ctr_inode_cx));\
-        if (_old_link_cx)\
-                OLD_LINK_CX(ctr_inode_cx) = _old_link_cx;\
-        ctr_inode_cx->fop_type = _fop_type;\
-        ctr_inode_cx->fop_path = _fop_path;\
-} while (0)
-
-/******************************************************************************
- *
- *                      Util functions or macros used by
- *                      insert wind and insert unwind
- *
- * ****************************************************************************/
-
-/*
- * If a rebalancer fop
- * */
-#define REBALANCE_FOP(frame)\
-        (frame->root->pid == GF_CLIENT_PID_DEFRAG)
-
-/*
- * If its a tiering rebalancer fop
- * */
-#define TIER_REBALANCE_FOP(frame)\
-        (frame->root->pid == GF_CLIENT_PID_TIER_DEFRAG)
-
-/*
- * If its a AFR SELF HEAL
- * */
- #define AFR_SELF_HEAL_FOP(frame)\
-        (frame->root->pid == GF_CLIENT_PID_AFR_SELF_HEALD)
-
-/*
- * if a rebalancer fop goto
- * */
-#define CTR_IF_REBALANCE_FOP_THEN_GOTO(frame, label)\
-do {\
-        if (REBALANCE_FOP (frame))\
-                goto label;\
-} while (0)
-
-/*
- * Internal fop
- *
- * */
-#define CTR_IS_INTERNAL_FOP(frame, dict)\
-        (AFR_SELF_HEAL_FOP (frame) \
-        || REBALANCE_FOP (frame) \
-        || TIER_REBALANCE_FOP (frame) \
-        || (dict && \
-        dict_get (dict, GLUSTERFS_INTERNAL_FOP_KEY)))
-
-/**
- * ignore internal fops for all clients except AFR self-heal daemon
- */
-#define CTR_IF_INTERNAL_FOP_THEN_GOTO(frame, dict, label)\
-do {\
-        GF_ASSERT(frame);\
-        GF_ASSERT(frame->root);\
-        if (CTR_IS_INTERNAL_FOP(frame, dict)) \
-                        goto label; \
-} while (0)
-
-
-/*
- * IS CTR Xlator is disabled then goto to label
- * */
- #define CTR_IS_DISABLED_THEN_GOTO(this, label)\
- do {\
-        gf_ctr_private_t *_priv = NULL;\
-        GF_ASSERT (this);\
-        GF_ASSERT (this->private);\
-        _priv = this->private;\
-        if (!_priv->enabled)\
-                goto label;\
- } while (0)
-
-int
-fill_db_record_for_unwind (xlator_t              *this,
-                          gf_ctr_local_t        *ctr_local,
-                          gfdb_fop_type_t       fop_type,
-                          gfdb_fop_path_t       fop_path);
-
-int
-fill_db_record_for_wind (xlator_t                *this,
-                        gf_ctr_local_t          *ctr_local,
-                        gf_ctr_inode_context_t  *ctr_inode_cx);
-
-/*******************************************************************************
- *                              CTR INSERT WIND
- * *****************************************************************************
- * Function used to insert/update record into the database during a wind fop
- * This function creates ctr_local structure into the frame of the fop
- * call.
- * ****************************************************************************/
-static inline int
-ctr_insert_wind (call_frame_t                    *frame,
-                xlator_t                        *this,
-                gf_ctr_inode_context_t          *ctr_inode_cx)
-{
-        int ret                         = -1;
-        gf_ctr_private_t *_priv         = NULL;
-        gf_ctr_local_t *ctr_local       = NULL;
-
-        GF_ASSERT(frame);
-        GF_ASSERT(frame->root);
-        GF_ASSERT(this);
-        IS_CTR_INODE_CX_SANE(ctr_inode_cx);
-
-        _priv = this->private;
-        GF_ASSERT (_priv);
-
-        GF_ASSERT(_priv->_db_conn);
-
-        /*If record_wind option of CTR is on record wind for
-         * regular files only*/
-        if (_priv->ctr_record_wind && ctr_inode_cx->ia_type != IA_IFDIR) {
-                frame->local = init_ctr_local_t (this);
-                if (!frame->local) {
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "WIND: Error while creating ctr local");
-                        goto out;
-                };
-                ctr_local = frame->local;
-                ctr_local->client_pid = frame->root->pid;
-                ctr_local->is_internal_fop = ctr_inode_cx->is_internal_fop;
-
-                /* Decide whether to record counters or not */
-                CTR_DB_REC(ctr_local).do_record_counters =
-                                                _priv->ctr_record_counter &&
-                                                !(ctr_local->is_internal_fop);
-
-                /* Decide whether to record times or not
-                 * For non internal FOPS record times as usual*/
-                if (!ctr_local->is_internal_fop) {
-                        CTR_DB_REC(ctr_local).do_record_times =
-                                                (_priv->ctr_record_wind
-                                                || _priv->ctr_record_unwind);
-                }
-                /* when its a internal FOPS*/
-                else {
-                        /* Record times only for create
-                         * i.e when the inode is created */
-                        CTR_DB_REC(ctr_local).do_record_times =
-                                (isdentrycreatefop(ctr_inode_cx->fop_type)) ?
-                                        _gf_true : _gf_false;
-                }
-
-                /*Fill the db record for insertion*/
-                ret = fill_db_record_for_wind (this, ctr_local, ctr_inode_cx);
-                if (ret) {
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "WIND: Error filling  ctr local");
-                        goto out;
-                }
-                /*Insert the db record*/
-                ret = insert_record (_priv->_db_conn,
-                                        &ctr_local->gfdb_db_record);
-                if (ret) {
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "WIND: Inserting of record failed!");
-                        goto out;
-                }
-        }
-        ret = 0;
-out:
-
-        if (ret) {
-                free_ctr_local (ctr_local);
-                frame->local = NULL;
-        }
-
-        return ret;
-}
-
-
-
-
-/*******************************************************************************
- *                             CTR INSERT UNWIND
- * *****************************************************************************
- * Function used to insert/update record into the database during a unwind fop
- * This function destroys ctr_local structure into the frame of the fop
- * call at the end.
- * ****************************************************************************/
-static inline int
-ctr_insert_unwind (call_frame_t          *frame,
-                  xlator_t              *this,
-                  gfdb_fop_type_t       fop_type,
-                  gfdb_fop_path_t       fop_path)
-{
-        int ret = -1;
-        gf_ctr_private_t *_priv         = NULL;
-        gf_ctr_local_t *ctr_local       = NULL;
-
-        GF_ASSERT(frame);
-        GF_ASSERT(this);
-
-        _priv = this->private;
-        GF_ASSERT (_priv);
-
-        GF_ASSERT(_priv->_db_conn);
-
-        ctr_local = frame->local;
-
-        if (ctr_local
-            && (_priv->ctr_record_unwind || isdentryfop(fop_type))
-            && (ctr_local->ia_inode_type != IA_IFDIR)) {
-
-                CTR_DB_REC(ctr_local).do_record_uwind_time =
-                                                _priv->ctr_record_unwind;
-
-                ret = fill_db_record_for_unwind(this, ctr_local, fop_type,
-                                               fop_path);
-                if (ret == -1) {
-                        gf_log(this->name, GF_LOG_ERROR, "UNWIND: Error"
-                                "filling ctr local");
-                        goto out;
-                }
-
-                ret = insert_record(_priv->_db_conn,
-                                        &ctr_local->gfdb_db_record);
-                if (ret == -1) {
-                        gf_log(this->name, GF_LOG_ERROR, "UNWIND: Error"
-                                "filling ctr local");
-                        goto out;
-                }
-        }
-        ret = 0;
-out:
-        free_ctr_local (ctr_local);
-        frame->local = NULL;
-        return ret;
-}
-
-/******************************************************************************
- *
- *                      CTR xlator init related functions
- *
- *
- * ****************************************************************************/
-int
-extract_db_params (xlator_t              *this,
-                  dict_t                *params_dict,
-                  gfdb_db_type_t        db_type);
-
-int
-extract_ctr_options (xlator_t            *this,
-                    gf_ctr_private_t    *_priv);
-
-#endif
diff --git a/xlators/features/ganesha/Makefile.am b/xlators/features/cloudsync/Makefile.am
index a985f42a877..a985f42a877 100644
--- a/xlators/features/ganesha/Makefile.am
+++ b/xlators/features/cloudsync/Makefile.am
diff --git a/xlators/features/cloudsync/src/Makefile.am b/xlators/features/cloudsync/src/Makefile.am
new file mode 100644
index 00000000000..e2a277e372b
--- /dev/null
+++ b/xlators/features/cloudsync/src/Makefile.am
@@ -0,0 +1,46 @@
+SUBDIRS = cloudsync-plugins
+
+xlator_LTLIBRARIES = cloudsync.la
+
+xlatordir = $(libdir)/glusterfs/$(PACKAGE_VERSION)/xlator/features
+
+cloudsync_sources = cloudsync.c
+
+CLOUDSYNC_SRC = $(top_srcdir)/xlators/features/cloudsync/src
+CLOUDSYNC_BLD = $(top_builddir)/xlators/features/cloudsync/src
+
+cloudsynccommon_sources = $(CLOUDSYNC_SRC)/cloudsync-common.c
+
+noinst_HEADERS = $(CLOUDSYNC_BLD)/cloudsync.h \
+		 $(CLOUDSYNC_BLD)/cloudsync-mem-types.h \
+		 $(CLOUDSYNC_BLD)/cloudsync-messages.h \
+		 $(CLOUDSYNC_BLD)/cloudsync-common.h
+
+cloudsync_la_SOURCES = $(cloudsync_sources) $(cloudsynccommon_sources)
+
+nodist_cloudsync_la_SOURCES = cloudsync-autogen-fops.c cloudsync-autogen-fops.h
+BUILT_SOURCES = cloudsync-autogen-fops.h
+
+cloudsync_la_LDFLAGS = -module $(GF_XLATOR_DEFAULT_LDFLAGS)
+
+cloudsync_la_LIBADD = $(top_builddir)/libglusterfs/src/libglusterfs.la $(LIB_DL)
+
+AM_CPPFLAGS = $(GF_CPPFLAGS) -I$(top_srcdir)/libglusterfs/src -I$(top_srcdir)/rpc/xdr/src -I$(top_builddir)/rpc/xdr/src \
+	      -DCS_PLUGINDIR=\"$(libdir)/glusterfs/$(PACKAGE_VERSION)/cloudsync-plugins\"
+AM_CFLAGS = -Wall -fno-strict-aliasing $(GF_CFLAGS)
+
+noinst_PYTHON = cloudsync-fops-c.py cloudsync-fops-h.py
+EXTRA_DIST = cloudsync-autogen-fops-tmpl.c cloudsync-autogen-fops-tmpl.h
+
+cloudsync-autogen-fops.c: cloudsync-fops-c.py cloudsync-autogen-fops-tmpl.c
+	$(PYTHON) $(CLOUDSYNC_SRC)/cloudsync-fops-c.py \
+	$(CLOUDSYNC_SRC)/cloudsync-autogen-fops-tmpl.c > $@
+
+cloudsync-autogen-fops.h: cloudsync-fops-h.py cloudsync-autogen-fops-tmpl.h
+	$(PYTHON) $(CLOUDSYNC_SRC)/cloudsync-fops-h.py \
+	$(CLOUDSYNC_SRC)/cloudsync-autogen-fops-tmpl.h > $@
+
+CLEANFILES = $(nodist_cloudsync_la_SOURCES)
+
+uninstall-local:
+	rm -f $(DESTDIR)$(xlatordir)/cloudsync.so
diff --git a/xlators/features/cloudsync/src/cloudsync-autogen-fops-tmpl.c b/xlators/features/cloudsync/src/cloudsync-autogen-fops-tmpl.c
new file mode 100644
index 00000000000..ee63f983980
--- /dev/null
+++ b/xlators/features/cloudsync/src/cloudsync-autogen-fops-tmpl.c
@@ -0,0 +1,30 @@
+/*
+  Copyright (c) 2008-2015 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
+*/
+
+/* File: cloudsync-autogen-fops-tmpl.c
+ * This file contains the CLOUDSYNC autogenerated FOPs. This is run through
+ * the code generator, generator.py to generate the required FOPs.
+ */
+
+#ifndef _CONFIG_H
+#define _CONFIG_H
+#include "config.h"
+#endif
+
+#include <dlfcn.h>
+
+#include <glusterfs/glusterfs.h>
+#include <glusterfs/xlator.h>
+#include <glusterfs/defaults.h>
+#include "cloudsync.h"
+#include "cloudsync-common.h"
+#include <glusterfs/call-stub.h>
+
+#pragma generate
diff --git a/xlators/features/cloudsync/src/cloudsync-autogen-fops-tmpl.h b/xlators/features/cloudsync/src/cloudsync-autogen-fops-tmpl.h
new file mode 100644
index 00000000000..d922c77d8aa
--- /dev/null
+++ b/xlators/features/cloudsync/src/cloudsync-autogen-fops-tmpl.h
@@ -0,0 +1,24 @@
+/*
+  Copyright (c) 2008-2015 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
+*/
+
+/* File: clousync-autogen-fops-tmpl.h
+ * This file contains the cloudsync autogenerated FOPs declarations.
+ */
+
+#ifndef _CLOUDSYNC_AUTOGEN_FOPS_H
+#define _CLOUDSYNC_AUTOGEN_FOPS_H
+
+#include <glusterfs/xlator.h>
+#include "cloudsync.h"
+#include "cloudsync-common.h"
+
+#pragma generate
+
+#endif /* _CLOUDSYNC_AUTOGEN_FOPS_H */
diff --git a/xlators/features/cloudsync/src/cloudsync-common.c b/xlators/features/cloudsync/src/cloudsync-common.c
new file mode 100644
index 00000000000..445a31b90e7
--- /dev/null
+++ b/xlators/features/cloudsync/src/cloudsync-common.c
@@ -0,0 +1,60 @@
+/*
+  Copyright (c) 2018 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
+*/
+
+#include "cloudsync-common.h"
+
+void
+cs_xattrinfo_wipe(cs_local_t *local)
+{
+    if (local->xattrinfo.lxattr) {
+        if (local->xattrinfo.lxattr->file_path)
+            GF_FREE(local->xattrinfo.lxattr->file_path);
+
+        if (local->xattrinfo.lxattr->volname)
+            GF_FREE(local->xattrinfo.lxattr->volname);
+
+        GF_FREE(local->xattrinfo.lxattr);
+    }
+}
+
+void
+cs_local_wipe(xlator_t *this, cs_local_t *local)
+{
+    if (!local)
+        return;
+
+    loc_wipe(&local->loc);
+
+    if (local->fd) {
+        fd_unref(local->fd);
+        local->fd = NULL;
+    }
+
+    if (local->stub) {
+        call_stub_destroy(local->stub);
+        local->stub = NULL;
+    }
+
+    if (local->xattr_req)
+        dict_unref(local->xattr_req);
+
+    if (local->xattr_rsp)
+        dict_unref(local->xattr_rsp);
+
+    if (local->dlfd)
+        fd_unref(local->dlfd);
+
+    if (local->remotepath)
+        GF_FREE(local->remotepath);
+
+    cs_xattrinfo_wipe(local);
+
+    mem_put(local);
+}
diff --git a/xlators/features/cloudsync/src/cloudsync-common.h b/xlators/features/cloudsync/src/cloudsync-common.h
new file mode 100644
index 00000000000..11d233460a4
--- /dev/null
+++ b/xlators/features/cloudsync/src/cloudsync-common.h
@@ -0,0 +1,134 @@
+/*
+  Copyright (c) 2018 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
+*/
+#ifndef _CLOUDSYNC_COMMON_H
+#define _CLOUDSYNC_COMMON_H
+
+#include <glusterfs/glusterfs.h>
+#include <glusterfs/call-stub.h>
+#include <glusterfs/xlator.h>
+#include <glusterfs/syncop.h>
+#include <glusterfs/compat-errno.h>
+#include "cloudsync-mem-types.h"
+#include "cloudsync-messages.h"
+
+typedef struct cs_loc_xattr {
+    char *file_path;
+    uuid_t uuid;
+    uuid_t gfid;
+    char *volname;
+} cs_loc_xattr_t;
+
+typedef struct cs_size_xattr {
+    uint64_t size;
+    uint64_t blksize;
+    uint64_t blocks;
+} cs_size_xattr_t;
+
+typedef struct cs_local {
+    loc_t loc;
+    fd_t *fd;
+    call_stub_t *stub;
+    call_frame_t *main_frame;
+    int op_errno;
+    int op_ret;
+    fd_t *dlfd;
+    off_t dloffset;
+    struct iatt stbuf;
+    dict_t *xattr_rsp;
+    dict_t *xattr_req;
+    glusterfs_fop_t fop;
+    gf_boolean_t locked;
+    int call_cnt;
+    inode_t *inode;
+    char *remotepath;
+
+    struct {
+        /* offset, flags and size are the information needed
+         * by read fop for remote read operation. These will be
+         * populated in cloudsync read fop, before being passed
+         * on to the plugin performing remote read.
+         */
+        off_t offset;
+        uint32_t flags;
+        size_t size;
+        cs_loc_xattr_t *lxattr;
+    } xattrinfo;
+
+} cs_local_t;
+
+typedef int (*fop_download_t)(call_frame_t *frame, void *config);
+
+typedef int (*fop_remote_read_t)(call_frame_t *, void *);
+
+typedef void *(*store_init)(xlator_t *this);
+
+typedef int (*store_reconfigure)(xlator_t *this, dict_t *options);
+
+typedef void (*store_fini)(void *config);
+
+struct cs_remote_stores {
+    char *name;                    /* store name */
+    void *config;                  /* store related information */
+    fop_download_t dlfop;          /* store specific download function */
+    fop_remote_read_t rdfop;       /* store specific read function */
+    store_init init;               /* store init to initialize store config */
+    store_reconfigure reconfigure; /* reconfigure store config */
+    store_fini fini;
+    void *handle; /* shared library handle*/
+};
+
+typedef struct cs_private {
+    xlator_t *this;
+    struct cs_remote_stores *stores;
+    gf_boolean_t abortdl;
+    pthread_spinlock_t lock;
+    gf_boolean_t remote_read;
+} cs_private_t;
+
+void
+cs_local_wipe(xlator_t *this, cs_local_t *local);
+
+void
+cs_xattrinfo_wipe(cs_local_t *local);
+
+#define CS_STACK_UNWIND(fop, frame, params...)                                 \
+    do {                                                                       \
+        cs_local_t *__local = NULL;                                            \
+        xlator_t *__xl = NULL;                                                 \
+        if (frame) {                                                           \
+            __xl = frame->this;                                                \
+            __local = frame->local;                                            \
+            frame->local = NULL;                                               \
+        }                                                                      \
+        STACK_UNWIND_STRICT(fop, frame, params);                               \
+        cs_local_wipe(__xl, __local);                                          \
+    } while (0)
+
+#define CS_STACK_DESTROY(frame)                                                \
+    do {                                                                       \
+        cs_local_t *__local = NULL;                                            \
+        xlator_t *__xl = NULL;                                                 \
+        __xl = frame->this;                                                    \
+        __local = frame->local;                                                \
+        frame->local = NULL;                                                   \
+        STACK_DESTROY(frame->root);                                            \
+        cs_local_wipe(__xl, __local);                                          \
+    } while (0)
+
+typedef struct store_methods {
+    int (*fop_download)(call_frame_t *frame, void *config);
+    int (*fop_remote_read)(call_frame_t *, void *);
+    /* return type should be the store config */
+    void *(*fop_init)(xlator_t *this);
+    int (*fop_reconfigure)(xlator_t *this, dict_t *options);
+    void (*fop_fini)(void *config);
+} store_methods_t;
+
+#endif /* _CLOUDSYNC_COMMON_H */
diff --git a/xlators/features/cloudsync/src/cloudsync-fops-c.py b/xlators/features/cloudsync/src/cloudsync-fops-c.py
new file mode 100755
index 00000000000..c27df97ae58
--- /dev/null
+++ b/xlators/features/cloudsync/src/cloudsync-fops-c.py
@@ -0,0 +1,324 @@
+#!/usr/bin/python3
+
+from __future__ import print_function
+import os
+import sys
+
+curdir = os.path.dirname(sys.argv[0])
+gendir = os.path.join(curdir, '../../../../libglusterfs/src')
+sys.path.append(gendir)
+from generator import ops, fop_subs, cbk_subs, generate
+
+FD_DATA_MODIFYING_OP_FOP_TEMPLATE = """
+int32_t
+cs_@NAME@ (call_frame_t *frame, xlator_t *this,
+           @LONG_ARGS@)
+{
+        int                         op_errno        = EINVAL ;
+        cs_local_t                 *local           = NULL;
+        int                         ret             = 0;
+        cs_inode_ctx_t             *ctx             = NULL;
+        gf_cs_obj_state             state           = -1;
+
+        VALIDATE_OR_GOTO (frame, err);
+        VALIDATE_OR_GOTO (this, err);
+        VALIDATE_OR_GOTO (fd, err);
+
+        local = cs_local_init (this, frame, NULL, fd, GF_FOP_@UPNAME@);
+        if (!local) {
+
+                gf_msg (this->name, GF_LOG_ERROR, 0, 0, "local init failed");
+                op_errno = ENOMEM;
+                goto err;
+        }
+
+        __cs_inode_ctx_get (this, fd->inode, &ctx);
+
+        if (ctx)
+                state = __cs_get_file_state (fd->inode, ctx);
+        else
+                state = GF_CS_LOCAL;
+
+        xdata = xdata ? dict_ref (xdata) : dict_new ();
+
+        if (!xdata) {
+                gf_msg (this->name, GF_LOG_ERROR, 0, 0, "insufficient memory");
+                op_errno = ENOMEM;
+                goto err;
+        }
+
+        local->xattr_req = xdata;
+
+        ret = dict_set_uint32 (local->xattr_req, GF_CS_OBJECT_STATUS, 1);
+        if (ret) {
+                gf_msg (this->name, GF_LOG_ERROR, 0, 0, "dict_set failed key:"
+                        " %s", GF_CS_OBJECT_STATUS);
+                goto err;
+        }
+
+        local->stub = fop_@NAME@_stub (frame, cs_resume_@NAME@,
+                                       @SHORT_ARGS@);
+        if (!local->stub) {
+                gf_msg (this->name, GF_LOG_ERROR, 0, 0, "insufficient memory");
+                op_errno = ENOMEM;
+                goto err;
+        }
+
+
+        if (state == GF_CS_LOCAL) {
+                STACK_WIND (frame, cs_@NAME@_cbk,
+                            FIRST_CHILD(this), FIRST_CHILD(this)->fops->@NAME@,
+                            @SHORT_ARGS@);
+        } else {
+                local->call_cnt++;
+                ret = locate_and_execute (frame);
+                if (ret) {
+                        op_errno = ENOMEM;
+                        goto err;
+                }
+        }
+
+        return 0;
+
+err:
+        CS_STACK_UNWIND (@NAME@, frame, -1, op_errno, @CBK_ERROR_ARGS@);
+
+        return 0;
+}
+"""
+
+FD_DATA_MODIFYING_RESUME_OP_FOP_TEMPLATE = """
+int32_t
+cs_resume_@NAME@ (call_frame_t *frame, xlator_t *this,
+                  @LONG_ARGS@)
+{
+        int              ret    = 0;
+
+        ret = cs_resume_postprocess (this, frame, fd->inode);
+        if (ret) {
+                goto unwind;
+        }
+
+        cs_inodelk_unlock (frame);
+
+        STACK_WIND (frame, cs_@NAME@_cbk,
+                    FIRST_CHILD(this), FIRST_CHILD(this)->fops->@NAME@,
+                    @SHORT_ARGS@);
+
+        return 0;
+
+unwind:
+
+        cs_inodelk_unlock (frame);
+
+        cs_common_cbk (frame);
+
+        return 0;
+}
+"""
+FD_DATA_MODIFYING_OP_FOP_CBK_TEMPLATE = """
+int32_t
+cs_@NAME@_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+               int32_t op_ret, int32_t op_errno,
+               @LONG_ARGS@)
+{
+        cs_local_t      *local = NULL;
+        int              ret = 0;
+        uint64_t         val = 0;
+        fd_t            *fd = NULL;
+
+        local = frame->local;
+        fd = local->fd;
+
+        /* Do we need lock here? */
+        local->call_cnt++;
+
+        if (op_ret == -1) {
+                ret = dict_get_uint64 (xdata, GF_CS_OBJECT_STATUS, &val);
+                if (ret == 0) {
+                        if (val == GF_CS_ERROR) {
+                                gf_msg (this->name, GF_LOG_ERROR, 0, 0,
+                                        "could not get file state, unwinding");
+                                op_ret = -1;
+                                op_errno = EIO;
+                                goto unwind;
+                        } else {
+                                __cs_inode_ctx_update (this, fd->inode, val);
+                                gf_msg (this->name, GF_LOG_INFO, 0, 0,
+                                        " state = %" PRIu64, val);
+
+                                if (local->call_cnt == 1 &&
+                                    (val == GF_CS_REMOTE ||
+                                     val == GF_CS_DOWNLOADING))  {
+                                        gf_msg (this->name, GF_LOG_INFO, 0,
+                                                0, " will repair and download "
+                                                "the file, current state : %"
+                                                PRIu64, val);
+                                        goto repair;
+                                } else {
+                                        gf_msg (this->name, GF_LOG_ERROR, 0, 0,
+                                                "second @NAME@, Unwinding");
+                                        goto unwind;
+                                }
+                        }
+                } else {
+                        gf_msg (this->name, GF_LOG_ERROR, 0, 0, "file state "
+                                "could not be figured, unwinding");
+                        goto unwind;
+                }
+        } else {
+                /* successful @NAME@ => file is local */
+                __cs_inode_ctx_update (this, fd->inode, GF_CS_LOCAL);
+                gf_msg (this->name, GF_LOG_INFO, 0, 0, "state : GF_CS_LOCAL"
+                        ", @NAME@ successful");
+
+                goto unwind;
+        }
+
+repair:
+        ret = locate_and_execute (frame);
+        if (ret) {
+                goto unwind;
+        }
+
+        return 0;
+
+unwind:
+        CS_STACK_UNWIND (@NAME@, frame, op_ret, op_errno, @SHORT_ARGS@);
+
+        return 0;
+}
+"""
+
+LOC_STAT_OP_FOP_TEMPLATE = """
+int32_t
+cs_@NAME@ (call_frame_t *frame, xlator_t *this,
+           @LONG_ARGS@)
+{
+        int              op_errno = EINVAL;
+        cs_local_t      *local = NULL;
+        int              ret   = 0;
+
+        local = cs_local_init (this, frame, loc, NULL, GF_FOP_@UPNAME@);
+        if (!local) {
+                gf_msg (this->name, GF_LOG_ERROR, 0, 0, "local is NULL");
+                op_errno = ENOMEM;
+                goto err;
+        }
+
+        if (loc->inode->ia_type == IA_IFDIR)
+                goto wind;
+
+        xdata = xdata ? dict_ref (xdata) : dict_new ();
+
+        if (!xdata) {
+                gf_msg (this->name, GF_LOG_ERROR, 0, 0, "insufficient memory");
+                op_errno = ENOMEM;
+                goto err;
+        }
+
+        local->xattr_req = xdata;
+
+        ret = dict_set_uint32 (local->xattr_req, GF_CS_OBJECT_STATUS, 1);
+        if (ret) {
+                gf_msg (this->name, GF_LOG_ERROR, 0, 0, "dict_set failed key:"
+                        " %s", GF_CS_OBJECT_STATUS);
+                goto err;
+        }
+
+wind:
+        STACK_WIND (frame, cs_@NAME@_cbk, FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->@NAME@,
+                    @SHORT_ARGS@);
+
+        return 0;
+err:
+        CS_STACK_UNWIND (@NAME@, frame, -1, op_errno, @CBK_ERROR_ARGS@);
+
+        return 0;
+}
+"""
+
+LOC_STAT_OP_FOP_CBK_TEMPLATE = """
+int32_t
+cs_@NAME@_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+               int32_t op_ret, int32_t op_errno,
+               @LONG_ARGS@)
+{
+        int              ret = 0;
+        uint64_t         val = 0;
+        loc_t           *loc = NULL;
+        cs_local_t      *local = NULL;
+
+        local = frame->local;
+
+        loc = &local->loc;
+
+        if (op_ret == 0) {
+                ret = dict_get_uint64 (xdata, GF_CS_OBJECT_STATUS, &val);
+                if (!ret) {
+                        ret = __cs_inode_ctx_update (this, loc->inode, val);
+                        if (ret) {
+                                gf_msg (this->name, GF_LOG_ERROR, 0, 0,
+                                        "ctx update failed");
+                        }
+                }
+        } else {
+                cs_inode_ctx_reset (this, loc->inode);
+        }
+
+        CS_STACK_UNWIND (@NAME@, frame, op_ret, op_errno, @SHORT_ARGS@);
+
+        return 0;
+}
+"""
+
+# All xlator FOPs are covered in the following section just to create a clarity
+# The lists themselves are not used.
+entry_ops = ['mknod', 'mkdir', 'unlink', 'rmdir', 'symlink', 'rename', 'link',
+             'create']
+special_ops = ['statfs', 'lookup', 'ipc', 'compound', 'icreate', 'namelink']
+ignored_ops = ['getspec']
+inode_ops = ['stat', 'readlink', 'truncate', 'open', 'setxattr', 'getxattr',
+             'removexattr', 'opendir', 'access', 'inodelk', 'entrylk',
+             'xattrop', 'setattr', 'lease', 'getactivelk', 'setactivelk',
+             'discover']
+fd_ops = ['readv', 'writev', 'flush', 'fsync', 'fsyncdir', 'ftruncate',
+          'fstat', 'lk', 'readdir', 'finodelk', 'fentrylk', 'fxattrop',
+          'fsetxattr', 'fgetxattr', 'rchecksum', 'fsetattr', 'readdirp',
+          'fremovexattr', 'fallocate', 'discard', 'zerofill', 'seek']
+
+
+# These are the current actual lists used to generate the code
+
+# The following list contains fops which are fd based that modifies data
+fd_data_modify_op_fop_template = ['writev', 'flush', 'fsync',
+                                  'ftruncate', 'rchecksum', 'fallocate',
+                                  'discard', 'zerofill', 'seek']
+
+# The following list contains fops which are entry based that does not change
+# data
+loc_stat_op_fop_template = ['lookup', 'stat', 'discover', 'access', 'setattr',
+                            'getattr']
+
+# These fops need a separate implementation
+special_fops = ['statfs', 'setxattr', 'unlink', 'getxattr',
+                'truncate', 'fstat', 'readv', 'readdirp']
+
+def gen_defaults():
+    for name in ops:
+        if name in fd_data_modify_op_fop_template:
+            print(generate(FD_DATA_MODIFYING_OP_FOP_CBK_TEMPLATE, name, cbk_subs))
+            print(generate(FD_DATA_MODIFYING_RESUME_OP_FOP_TEMPLATE, name, fop_subs))
+            print(generate(FD_DATA_MODIFYING_OP_FOP_TEMPLATE, name, fop_subs))
+        elif name in loc_stat_op_fop_template:
+            print(generate(LOC_STAT_OP_FOP_CBK_TEMPLATE, name, cbk_subs))
+            print(generate(LOC_STAT_OP_FOP_TEMPLATE, name, fop_subs))
+
+for l in open(sys.argv[1], 'r').readlines():
+    if l.find('#pragma generate') != -1:
+        print("/* BEGIN GENERATED CODE - DO NOT MODIFY */")
+        gen_defaults()
+        print("/* END GENERATED CODE */")
+    else:
+        print(l[:-1])
diff --git a/xlators/features/cloudsync/src/cloudsync-fops-h.py b/xlators/features/cloudsync/src/cloudsync-fops-h.py
new file mode 100755
index 00000000000..faa2de651a7
--- /dev/null
+++ b/xlators/features/cloudsync/src/cloudsync-fops-h.py
@@ -0,0 +1,31 @@
+#!/usr/bin/python3
+
+from __future__ import print_function
+import os
+import sys
+
+curdir = os.path.dirname(sys.argv[0])
+gendir = os.path.join(curdir, '../../../../libglusterfs/src')
+sys.path.append(gendir)
+from generator import ops, fop_subs, cbk_subs, generate
+
+OP_FOP_TEMPLATE = """
+int32_t
+cs_@NAME@ (call_frame_t *frame, xlator_t *this,
+                   @LONG_ARGS@);
+"""
+
+def gen_defaults():
+    for name, value in ops.items():
+        if name == 'getspec':
+            continue
+        print(generate(OP_FOP_TEMPLATE, name, fop_subs))
+
+
+for l in open(sys.argv[1], 'r').readlines():
+    if l.find('#pragma generate') != -1:
+        print("/* BEGIN GENERATED CODE - DO NOT MODIFY */")
+        gen_defaults()
+        print("/* END GENERATED CODE */")
+    else:
+        print(l[:-1])
diff --git a/xlators/features/cloudsync/src/cloudsync-mem-types.h b/xlators/features/cloudsync/src/cloudsync-mem-types.h
new file mode 100644
index 00000000000..220346405d0
--- /dev/null
+++ b/xlators/features/cloudsync/src/cloudsync-mem-types.h
@@ -0,0 +1,22 @@
+/*
+ *   Copyright (c) 2018 Red Hat, Inc. <http://www.redhat.com>
+ *   This file is part of GlusterFS.
+ *
+ *   This file is licensed to you under your choice of the GNU Lesser
+ *   General Public License, version 3 or any later version (LGPLv3 or
+ *   later), or the GNU General Public License, version 2 (GPLv2), in all
+ *   cases as published by the Free Software Foundation.
+ */
+
+#ifndef __CLOUDSYNC_MEM_TYPES_H__
+#define __CLOUDSYNC_MEM_TYPES_H__
+
+#include <glusterfs/mem-types.h>
+enum cs_mem_types_ {
+    gf_cs_mt_cs_private_t = gf_common_mt_end + 1,
+    gf_cs_mt_cs_remote_stores_t,
+    gf_cs_mt_cs_inode_ctx_t,
+    gf_cs_mt_cs_lxattr_t,
+    gf_cs_mt_end
+};
+#endif /* __CLOUDSYNC_MEM_TYPES_H__ */
diff --git a/xlators/features/cloudsync/src/cloudsync-messages.h b/xlators/features/cloudsync/src/cloudsync-messages.h
new file mode 100644
index 00000000000..fb08f72de7f
--- /dev/null
+++ b/xlators/features/cloudsync/src/cloudsync-messages.h
@@ -0,0 +1,16 @@
+/*
+ *   Copyright (c) 2018 Red Hat, Inc. <http://www.redhat.com>
+ *   This file is part of GlusterFS.
+ *
+ *   This file is licensed to you under your choice of the GNU Lesser
+ *   General Public License, version 3 or any later version (LGPLv3 or
+ *   later), or the GNU General Public License, version 2 (GPLv2), in all
+ *   cases as published by the Free Software Foundation.
+ */
+
+#ifndef __CLOUDSYNC_MESSAGES_H__
+#define __CLOUDSYNC_MESSAGES_H__
+
+/*TODO: define relevant message ids */
+
+#endif /* __CLOUDSYNC_MESSAGES_H__ */
diff --git a/xlators/storage/bd/Makefile.am b/xlators/features/cloudsync/src/cloudsync-plugins/Makefile.am
index a985f42a877..a985f42a877 100644
--- a/xlators/storage/bd/Makefile.am
+++ b/xlators/features/cloudsync/src/cloudsync-plugins/Makefile.am
diff --git a/xlators/features/cloudsync/src/cloudsync-plugins/src/Makefile.am b/xlators/features/cloudsync/src/cloudsync-plugins/src/Makefile.am
new file mode 100644
index 00000000000..fb6b0580c6d
--- /dev/null
+++ b/xlators/features/cloudsync/src/cloudsync-plugins/src/Makefile.am
@@ -0,0 +1,11 @@
+if BUILD_AMAZONS3_PLUGIN
+  AMAZONS3_DIR = cloudsyncs3
+endif
+
+if BUILD_CVLT_PLUGIN
+  CVLT_DIR = cvlt
+endif
+
+SUBDIRS = ${AMAZONS3_DIR} ${CVLT_DIR}
+
+CLEANFILES =
diff --git a/xlators/encryption/rot-13/Makefile.am b/xlators/features/cloudsync/src/cloudsync-plugins/src/cloudsyncs3/Makefile.am
index d471a3f9243..a985f42a877 100644
--- a/xlators/encryption/rot-13/Makefile.am
+++ b/xlators/features/cloudsync/src/cloudsync-plugins/src/cloudsyncs3/Makefile.am
@@ -1,3 +1,3 @@
 SUBDIRS = src
 
-CLEANFILES = 
+CLEANFILES =
diff --git a/xlators/features/cloudsync/src/cloudsync-plugins/src/cloudsyncs3/src/Makefile.am b/xlators/features/cloudsync/src/cloudsync-plugins/src/cloudsyncs3/src/Makefile.am
new file mode 100644
index 00000000000..6509426ef87
--- /dev/null
+++ b/xlators/features/cloudsync/src/cloudsync-plugins/src/cloudsyncs3/src/Makefile.am
@@ -0,0 +1,12 @@
+csp_LTLIBRARIES = cloudsyncs3.la
+cspdir = $(libdir)/glusterfs/$(PACKAGE_VERSION)/cloudsync-plugins
+
+cloudsyncs3_la_SOURCES = libcloudsyncs3.c  $(top_srcdir)/xlators/features/cloudsync/src/cloudsync-common.c
+cloudsyncs3_la_LIBADD = $(top_builddir)/libglusterfs/src/libglusterfs.la
+cloudsyncs3_la_LDFLAGS = -module -export-symbols $(top_srcdir)/xlators/features/cloudsync/src/cloudsync-plugins/src/cloudsyncs3/src/libcloudsyncs3.sym $(GF_XLATOR_LDFLAGS)
+AM_CPPFLAGS = $(GF_CPPFLAGS) -I$(top_srcdir)/libglusterfs/src   -I$(top_srcdir)/rpc/xdr/src -I$(top_builddir)/rpc/xdr/src -lcurlpp -lcryptopp
+noinst_HEADERS = libcloudsyncs3.h libcloudsyncs3-mem-types.h
+AM_CFLAGS = -Wall -fno-strict-aliasing $(GF_CFLAGS) -lcurl -lcrypto -I$(top_srcdir)/xlators/features/cloudsync/src
+CLEANFILES =
+
+EXTRA_DIST = libcloudsyncs3.sym
diff --git a/xlators/features/cloudsync/src/cloudsync-plugins/src/cloudsyncs3/src/libcloudsyncs3-mem-types.h b/xlators/features/cloudsync/src/cloudsync-plugins/src/cloudsyncs3/src/libcloudsyncs3-mem-types.h
new file mode 100644
index 00000000000..7ccfcc9f4b6
--- /dev/null
+++ b/xlators/features/cloudsync/src/cloudsync-plugins/src/cloudsyncs3/src/libcloudsyncs3-mem-types.h
@@ -0,0 +1,19 @@
+/*
+ *   Copyright (c) 2018 Red Hat, Inc. <http://www.redhat.com>
+ *   This file is part of GlusterFS.
+ *
+ *   This file is licensed to you under your choice of the GNU Lesser
+ *   General Public License, version 3 or any later version (LGPLv3 or
+ *   later), or the GNU General Public License, version 2 (GPLv2), in all
+ *   cases as published by the Free Software Foundation.
+ */
+
+#ifndef __LIBAWS_MEM_TYPES_H__
+#define __LIBAWS_MEM_TYPES_H__
+
+#include <glusterfs/mem-types.h>
+enum libaws_mem_types_ {
+    gf_libaws_mt_aws_private_t = gf_common_mt_end + 1,
+    gf_libaws_mt_end
+};
+#endif /* __CLOUDSYNC_MEM_TYPES_H__ */
diff --git a/xlators/features/cloudsync/src/cloudsync-plugins/src/cloudsyncs3/src/libcloudsyncs3.c b/xlators/features/cloudsync/src/cloudsync-plugins/src/cloudsyncs3/src/libcloudsyncs3.c
new file mode 100644
index 00000000000..23c3599825a
--- /dev/null
+++ b/xlators/features/cloudsync/src/cloudsync-plugins/src/cloudsyncs3/src/libcloudsyncs3.c
@@ -0,0 +1,584 @@
+/*
+  Copyright (c) 2018 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
+*/
+
+#include <stdlib.h>
+#include <openssl/hmac.h>
+#include <openssl/evp.h>
+#include <openssl/bio.h>
+#include <openssl/buffer.h>
+#include <openssl/crypto.h>
+#include <curl/curl.h>
+#include <glusterfs/xlator.h>
+#include <glusterfs/glusterfs.h>
+#include "libcloudsyncs3.h"
+#include "cloudsync-common.h"
+
+#define RESOURCE_SIZE 4096
+
+store_methods_t store_ops = {
+    .fop_download = aws_download_s3,
+    .fop_init = aws_init,
+    .fop_reconfigure = aws_reconfigure,
+    .fop_fini = aws_fini,
+};
+
+typedef struct aws_private {
+    char *hostname;
+    char *bucketid;
+    char *awssekey;
+    char *awskeyid;
+    gf_boolean_t abortdl;
+    pthread_spinlock_t lock;
+} aws_private_t;
+
+void *
+aws_init(xlator_t *this)
+{
+    aws_private_t *priv = NULL;
+    char *temp_str = NULL;
+    int ret = 0;
+
+    priv = GF_CALLOC(1, sizeof(aws_private_t), gf_libaws_mt_aws_private_t);
+    if (!priv) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, 0, "insufficient memory");
+        return NULL;
+    }
+
+    priv->abortdl = _gf_false;
+
+    pthread_spin_init(&priv->lock, PTHREAD_PROCESS_PRIVATE);
+
+    pthread_spin_lock(&(priv->lock));
+    {
+        if (dict_get_str(this->options, "s3plugin-seckey", &temp_str) == 0) {
+            priv->awssekey = gf_strdup(temp_str);
+            if (!priv->awssekey) {
+                gf_msg(this->name, GF_LOG_ERROR, ENOMEM, 0,
+                       "initializing aws secret key failed");
+                ret = -1;
+                goto unlock;
+            }
+        }
+
+        if (dict_get_str(this->options, "s3plugin-keyid", &temp_str) == 0) {
+            priv->awskeyid = gf_strdup(temp_str);
+            if (!priv->awskeyid) {
+                gf_msg(this->name, GF_LOG_ERROR, ENOMEM, 0,
+                       "initializing aws key ID failed");
+                ret = -1;
+                goto unlock;
+            }
+        }
+
+        if (dict_get_str(this->options, "s3plugin-bucketid", &temp_str) == 0) {
+            priv->bucketid = gf_strdup(temp_str);
+            if (!priv->bucketid) {
+                gf_msg(this->name, GF_LOG_ERROR, ENOMEM, 0,
+                       "initializing aws bucketid failed");
+
+                ret = -1;
+                goto unlock;
+            }
+        }
+
+        if (dict_get_str(this->options, "s3plugin-hostname", &temp_str) == 0) {
+            priv->hostname = gf_strdup(temp_str);
+            if (!priv->hostname) {
+                gf_msg(this->name, GF_LOG_ERROR, ENOMEM, 0,
+                       "initializing aws hostname failed");
+
+                ret = -1;
+                goto unlock;
+            }
+        }
+
+        gf_msg_debug(this->name, 0,
+                     "stored key: %s id: %s "
+                     "bucketid %s hostname: %s",
+                     priv->awssekey, priv->awskeyid, priv->bucketid,
+                     priv->hostname);
+    }
+unlock:
+    pthread_spin_unlock(&(priv->lock));
+
+    if (ret == -1) {
+        GF_FREE(priv->awskeyid);
+        GF_FREE(priv->awssekey);
+        GF_FREE(priv->bucketid);
+        GF_FREE(priv->hostname);
+        GF_FREE(priv);
+        priv = NULL;
+    }
+
+    return (void *)priv;
+}
+
+int
+aws_reconfigure(xlator_t *this, dict_t *options)
+{
+    aws_private_t *priv = NULL;
+    char *temp_str = NULL;
+    int ret = 0;
+    cs_private_t *cspriv = NULL;
+
+    cspriv = this->private;
+
+    priv = cspriv->stores->config;
+
+    if (!priv) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, 0, "null priv");
+        return -1;
+    }
+
+    pthread_spin_lock(&(priv->lock));
+    {
+        if (dict_get_str(options, "s3plugin-seckey", &temp_str) == 0) {
+            priv->awssekey = gf_strdup(temp_str);
+            if (!priv->awssekey) {
+                gf_msg(this->name, GF_LOG_ERROR, ENOMEM, 0,
+                       "initializing aws secret key failed");
+                ret = -1;
+                goto out;
+            }
+        }
+
+        if (dict_get_str(options, "s3plugin-keyid", &temp_str) == 0) {
+            priv->awskeyid = gf_strdup(temp_str);
+            if (!priv->awskeyid) {
+                gf_msg(this->name, GF_LOG_ERROR, ENOMEM, 0,
+                       "initializing aws key ID failed");
+                ret = -1;
+                goto out;
+            }
+        }
+
+        if (dict_get_str(options, "s3plugin-bucketid", &temp_str) == 0) {
+            priv->bucketid = gf_strdup(temp_str);
+            if (!priv->bucketid) {
+                gf_msg(this->name, GF_LOG_ERROR, ENOMEM, 0,
+                       "initializing aws bucketid failed");
+                ret = -1;
+                goto out;
+            }
+        }
+
+        if (dict_get_str(options, "s3plugin-hostname", &temp_str) == 0) {
+            priv->hostname = gf_strdup(temp_str);
+            if (!priv->hostname) {
+                gf_msg(this->name, GF_LOG_ERROR, ENOMEM, 0,
+                       "initializing aws hostname failed");
+                ret = -1;
+                goto out;
+            }
+        }
+    }
+out:
+    pthread_spin_unlock(&(priv->lock));
+
+    gf_msg_debug(this->name, 0,
+                 "stored key: %s id: %s "
+                 "bucketid %s hostname: %s",
+                 priv->awssekey, priv->awskeyid, priv->bucketid,
+                 priv->hostname);
+
+    return ret;
+}
+
+void
+aws_fini(void *config)
+{
+    aws_private_t *priv = NULL;
+
+    priv = (aws_private_t *)priv;
+
+    if (priv) {
+        GF_FREE(priv->hostname);
+        GF_FREE(priv->bucketid);
+        GF_FREE(priv->awssekey);
+        GF_FREE(priv->awskeyid);
+
+        pthread_spin_destroy(&priv->lock);
+        GF_FREE(priv);
+    }
+}
+
+int32_t
+mem_acct_init(xlator_t *this)
+{
+    int ret = -1;
+
+    GF_VALIDATE_OR_GOTO("dht", this, out);
+
+    ret = xlator_mem_acct_init(this, gf_libaws_mt_end + 1);
+
+    if (ret != 0) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, 0, "Memory accounting init failed");
+        return ret;
+    }
+out:
+    return ret;
+}
+char *
+aws_form_request(char *resource, char **date, char *reqtype, char *bucketid,
+                 char *filepath)
+{
+    char httpdate[256];
+    time_t ctime;
+    struct tm *gtime = NULL;
+    char *sign_req = NULL;
+    int signreq_len = -1;
+    int date_len = -1;
+    int res_len = -1;
+
+    ctime = gf_time();
+    gtime = gmtime(&ctime);
+
+    date_len = strftime(httpdate, sizeof(httpdate),
+                        "%a, %d %b %Y %H:%M:%S +0000", gtime);
+
+    *date = gf_strndup(httpdate, date_len);
+    if (*date == NULL) {
+        gf_msg("CS", GF_LOG_ERROR, ENOMEM, 0,
+               "memory allocation "
+               "failure for date");
+        goto out;
+    }
+
+    res_len = snprintf(resource, RESOURCE_SIZE, "%s/%s", bucketid, filepath);
+
+    gf_msg_debug("CS", 0, "resource %s", resource);
+
+    /* 6 accounts for the 4 new line chars, one forward slash and
+     * one null char */
+    signreq_len = res_len + date_len + strlen(reqtype) + 6;
+
+    sign_req = GF_MALLOC(signreq_len, gf_common_mt_char);
+    if (sign_req == NULL) {
+        gf_msg("CS", GF_LOG_ERROR, ENOMEM, 0,
+               "memory allocation "
+               "failure for sign_req");
+        goto out;
+    }
+
+    snprintf(sign_req, signreq_len, "%s\n\n%s\n%s\n/%s", reqtype, "", *date,
+             resource);
+
+out:
+    return sign_req;
+}
+
+char *
+aws_b64_encode(const unsigned char *input, int length)
+{
+    BIO *bio, *b64;
+    BUF_MEM *bptr;
+    char *buff = NULL;
+
+    b64 = BIO_new(BIO_f_base64());
+    bio = BIO_new(BIO_s_mem());
+    b64 = BIO_push(b64, bio);
+    BIO_write(b64, input, length);
+    BIO_flush(b64);
+    BIO_get_mem_ptr(b64, &bptr);
+
+    buff = GF_MALLOC(bptr->length, gf_common_mt_char);
+    memcpy(buff, bptr->data, bptr->length - 1);
+    buff[bptr->length - 1] = 0;
+
+    BIO_free_all(b64);
+
+    return buff;
+}
+
+char *
+aws_sign_request(char *const str, char *awssekey)
+{
+#if (OPENSSL_VERSION_NUMBER < 0x1010002f)
+    HMAC_CTX ctx;
+#endif
+    HMAC_CTX *pctx = NULL;
+    ;
+
+    unsigned char md[256];
+    unsigned len;
+    char *base64 = NULL;
+
+#if (OPENSSL_VERSION_NUMBER < 0x1010002f)
+    HMAC_CTX_init(&ctx);
+    pctx = &ctx;
+#else
+    pctx = HMAC_CTX_new();
+#endif
+    HMAC_Init_ex(pctx, awssekey, strlen(awssekey), EVP_sha1(), NULL);
+    HMAC_Update(pctx, (unsigned char *)str, strlen(str));
+    HMAC_Final(pctx, (unsigned char *)md, &len);
+
+#if (OPENSSL_VERSION_NUMBER < 0x1010002f)
+    HMAC_CTX_cleanup(pctx);
+#else
+    HMAC_CTX_free(pctx);
+#endif
+    base64 = aws_b64_encode(md, len);
+
+    return base64;
+}
+
+int
+aws_dlwritev_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int op_ret,
+                 int op_errno, struct iatt *prebuf, struct iatt *postbuf,
+                 dict_t *xdata)
+{
+    aws_private_t *priv = NULL;
+
+    if (op_ret == -1) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, op_errno,
+               "write failed "
+               ". Aborting Download");
+
+        priv = this->private;
+        pthread_spin_lock(&(priv->lock));
+        {
+            priv->abortdl = _gf_true;
+        }
+        pthread_spin_unlock(&(priv->lock));
+    }
+
+    CS_STACK_DESTROY(frame);
+
+    return op_ret;
+}
+
+size_t
+aws_write_callback(void *dlbuf, size_t size, size_t nitems, void *mainframe)
+{
+    call_frame_t *frame = NULL;
+    fd_t *dlfd = NULL;
+    int ret = 0;
+    cs_local_t *local = NULL;
+    struct iovec iov = {
+        0,
+    };
+    struct iobref *iobref = NULL;
+    struct iobuf *iobuf = NULL;
+    struct iovec dliov = {
+        0,
+    };
+    size_t tsize = 0;
+    xlator_t *this = NULL;
+    cs_private_t *xl_priv = NULL;
+    aws_private_t *priv = NULL;
+    call_frame_t *dlframe = NULL;
+
+    frame = (call_frame_t *)mainframe;
+    this = frame->this;
+    xl_priv = this->private;
+    priv = xl_priv->stores->config;
+
+    pthread_spin_lock(&(priv->lock));
+    {
+        /* returning size other than the size passed from curl will
+         * abort further download*/
+        if (priv->abortdl) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, 0, "aborting download");
+            pthread_spin_unlock(&(priv->lock));
+            return 0;
+        }
+    }
+    pthread_spin_unlock(&(priv->lock));
+
+    local = frame->local;
+    dlfd = local->dlfd;
+    tsize = size * nitems;
+
+    dliov.iov_base = (void *)dlbuf;
+    dliov.iov_len = tsize;
+
+    ret = iobuf_copy(this->ctx->iobuf_pool, &dliov, 1, &iobref, &iobuf, &iov);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, 0, "iobuf_copy failed");
+        goto out;
+    }
+
+    /* copy frame */
+    dlframe = copy_frame(frame);
+    if (!dlframe) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, 0, "copy_frame failed");
+        tsize = 0;
+        goto out;
+    }
+
+    STACK_WIND(dlframe, aws_dlwritev_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->writev, dlfd, &iov, 1, local->dloffset,
+               0, iobref, NULL);
+
+    local->dloffset += tsize;
+
+out:
+    if (iobuf)
+        iobuf_unref(iobuf);
+    if (iobref)
+        iobref_unref(iobref);
+
+    return tsize;
+}
+
+int
+aws_download_s3(call_frame_t *frame, void *config)
+{
+    char *buf;
+    int bufsize = -1;
+    CURL *handle = NULL;
+    struct curl_slist *slist = NULL;
+    struct curl_slist *tmp = NULL;
+    xlator_t *this = NULL;
+    int ret = 0;
+    int debug = 1;
+    CURLcode res;
+    char errbuf[CURL_ERROR_SIZE];
+    size_t len = 0;
+    long responsecode;
+    char *sign_req = NULL;
+    char *date = NULL;
+    char *const reqtype = "GET";
+    char *signature = NULL;
+    cs_local_t *local = NULL;
+    char resource[RESOURCE_SIZE] = {
+        0,
+    };
+    aws_private_t *priv = NULL;
+
+    this = frame->this;
+
+    local = frame->local;
+
+    priv = (aws_private_t *)config;
+
+    if (!priv->bucketid || !priv->hostname || !priv->awssekey ||
+        !priv->awskeyid) {
+        ret = -1;
+        goto out;
+    }
+
+    sign_req = aws_form_request(resource, &date, reqtype, priv->bucketid,
+                                local->remotepath);
+    if (!sign_req) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, 0,
+               "null sign_req, "
+               "aborting download");
+        ret = -1;
+        goto out;
+    }
+
+    gf_msg_debug("CS", 0, "sign_req %s date %s", sign_req, date);
+
+    signature = aws_sign_request(sign_req, priv->awssekey);
+    if (!signature) {
+        gf_msg("CS", GF_LOG_ERROR, 0, 0,
+               "null signature, "
+               "aborting download");
+        ret = -1;
+        goto out;
+    }
+
+    handle = curl_easy_init();
+    this = frame->this;
+
+    /* special numbers 6, 20, 10 accounts for static characters in the
+     * below snprintf string format arguments*/
+    bufsize = strlen(date) + 6 + strlen(priv->awskeyid) + strlen(signature) +
+              20 + strlen(priv->hostname) + 10;
+
+    buf = (char *)alloca(bufsize);
+    if (!buf) {
+        gf_msg("CS", GF_LOG_ERROR, ENOMEM, 0,
+               "mem allocation "
+               "failed for buf");
+        ret = -1;
+        goto out;
+    }
+
+    snprintf(buf, bufsize, "Date: %s", date);
+    slist = curl_slist_append(slist, buf);
+    snprintf(buf, bufsize, "Authorization: AWS %s:%s", priv->awskeyid,
+             signature);
+    slist = curl_slist_append(slist, buf);
+    snprintf(buf, bufsize, "https://%s/%s", priv->hostname, resource);
+
+    if (gf_log_get_loglevel() >= GF_LOG_DEBUG) {
+        tmp = slist;
+        while (tmp) {
+            gf_msg_debug(this->name, 0, "slist for curl - %s", tmp->data);
+            tmp = tmp->next;
+        }
+    }
+
+    curl_easy_setopt(handle, CURLOPT_HTTPHEADER, slist);
+    curl_easy_setopt(handle, CURLOPT_URL, buf);
+    curl_easy_setopt(handle, CURLOPT_WRITEFUNCTION, aws_write_callback);
+    curl_easy_setopt(handle, CURLOPT_WRITEDATA, frame);
+    curl_easy_setopt(handle, CURLOPT_VERBOSE, debug);
+    curl_easy_setopt(handle, CURLOPT_ERRORBUFFER, errbuf);
+
+    res = curl_easy_perform(handle);
+    if (res != CURLE_OK) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, 0, "download failed. err: %s\n",
+               curl_easy_strerror(res));
+        ret = -1;
+        len = strlen(errbuf);
+        if (len) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, 0, "curl failure %s", errbuf);
+        } else {
+            gf_msg(this->name, GF_LOG_ERROR, 0, 0,
+                   "curl error "
+                   "%s\n",
+                   curl_easy_strerror(res));
+        }
+    }
+
+    if (res == CURLE_OK) {
+        curl_easy_getinfo(handle, CURLINFO_RESPONSE_CODE, &responsecode);
+        gf_msg_debug(this->name, 0, "response code %ld", responsecode);
+        if (responsecode != 200) {
+            ret = -1;
+            gf_msg(this->name, GF_LOG_ERROR, 0, 0, "curl download failed");
+        }
+    }
+
+    curl_slist_free_all(slist);
+    curl_easy_cleanup(handle);
+
+out:
+    if (sign_req)
+        GF_FREE(sign_req);
+    if (date)
+        GF_FREE(date);
+    if (signature)
+        GF_FREE(signature);
+
+    return ret;
+}
+
+struct volume_options cs_options[] = {
+    {.key = {"s3plugin-seckey"},
+     .type = GF_OPTION_TYPE_STR,
+     .description = "aws secret key"},
+    {.key = {"s3plugin-keyid"},
+     .type = GF_OPTION_TYPE_STR,
+     .description = "aws key ID"
+
+    },
+    {.key = {"s3plugin-bucketid"},
+     .type = GF_OPTION_TYPE_STR,
+     .description = "aws bucketid"},
+    {.key = {"s3plugin-hostname"},
+     .type = GF_OPTION_TYPE_STR,
+     .description = "aws hostname e.g. s3.amazonaws.com"},
+    {.key = {NULL}},
+};
diff --git a/xlators/features/cloudsync/src/cloudsync-plugins/src/cloudsyncs3/src/libcloudsyncs3.h b/xlators/features/cloudsync/src/cloudsync-plugins/src/cloudsyncs3/src/libcloudsyncs3.h
new file mode 100644
index 00000000000..85ae669486b
--- /dev/null
+++ b/xlators/features/cloudsync/src/cloudsync-plugins/src/cloudsyncs3/src/libcloudsyncs3.h
@@ -0,0 +1,50 @@
+/*
+  Copyright (c) 2018 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
+*/
+#ifndef _LIBAWS_H
+#define _LIBAWS_H
+
+#include <glusterfs/glusterfs.h>
+#include <glusterfs/call-stub.h>
+#include <glusterfs/xlator.h>
+#include <glusterfs/syncop.h>
+#include <curl/curl.h>
+#include "cloudsync-common.h"
+#include "libcloudsyncs3-mem-types.h"
+
+char *
+aws_b64_encode(const unsigned char *input, int length);
+
+size_t
+aws_write_callback(void *dlbuf, size_t size, size_t nitems, void *mainframe);
+
+int
+aws_download_s3(call_frame_t *frame, void *config);
+
+int
+aws_dlwritev_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int op_ret,
+                 int op_errno, struct iatt *prebuf, struct iatt *postbuf,
+                 dict_t *xdata);
+
+void *
+aws_init(xlator_t *this);
+
+int
+aws_reconfigure(xlator_t *this, dict_t *options);
+
+char *
+aws_form_request(char *resource, char **date, char *reqtype, char *bucketid,
+                 char *filepath);
+char *
+aws_sign_request(char *const str, char *awssekey);
+
+void
+aws_fini(void *config);
+
+#endif
diff --git a/xlators/features/cloudsync/src/cloudsync-plugins/src/cloudsyncs3/src/libcloudsyncs3.sym b/xlators/features/cloudsync/src/cloudsync-plugins/src/cloudsyncs3/src/libcloudsyncs3.sym
new file mode 100644
index 00000000000..0bc273670d5
--- /dev/null
+++ b/xlators/features/cloudsync/src/cloudsync-plugins/src/cloudsyncs3/src/libcloudsyncs3.sym
@@ -0,0 +1 @@
+store_ops
diff --git a/xlators/cluster/map/Makefile.am b/xlators/features/cloudsync/src/cloudsync-plugins/src/cvlt/Makefile.am
index d471a3f9243..a985f42a877 100644
--- a/xlators/cluster/map/Makefile.am
+++ b/xlators/features/cloudsync/src/cloudsync-plugins/src/cvlt/Makefile.am
@@ -1,3 +1,3 @@
 SUBDIRS = src
 
-CLEANFILES = 
+CLEANFILES =
diff --git a/xlators/features/cloudsync/src/cloudsync-plugins/src/cvlt/src/Makefile.am b/xlators/features/cloudsync/src/cloudsync-plugins/src/cvlt/src/Makefile.am
new file mode 100644
index 00000000000..b512464f157
--- /dev/null
+++ b/xlators/features/cloudsync/src/cloudsync-plugins/src/cvlt/src/Makefile.am
@@ -0,0 +1,12 @@
+csp_LTLIBRARIES = cloudsynccvlt.la
+cspdir = $(libdir)/glusterfs/$(PACKAGE_VERSION)/cloudsync-plugins
+
+cloudsynccvlt_la_SOURCES = libcvlt.c  $(top_srcdir)/xlators/features/cloudsync/src/cloudsync-common.c
+cloudsynccvlt_la_LIBADD = $(top_builddir)/libglusterfs/src/libglusterfs.la
+cloudsynccvlt_la_LDFLAGS = -module -avoid-version -export-symbols $(top_srcdir)/xlators/features/cloudsync/src/cloudsync-plugins/src/cvlt/src/libcloudsynccvlt.sym
+AM_CPPFLAGS = $(GF_CPPFLAGS) -I$(top_srcdir)/libglusterfs/src   -I$(top_srcdir)/rpc/xdr/src -I$(top_builddir)/rpc/xdr/src
+noinst_HEADERS = archivestore.h libcvlt.h libcvlt-mem-types.h cvlt-messages.h
+AM_CFLAGS = -Wall -fno-strict-aliasing $(GF_CFLAGS) -I$(top_srcdir)/xlators/features/cloudsync/src
+CLEANFILES =
+
+EXTRA_DIST = libcloudsynccvlt.sym
diff --git a/xlators/features/cloudsync/src/cloudsync-plugins/src/cvlt/src/archivestore.h b/xlators/features/cloudsync/src/cloudsync-plugins/src/cvlt/src/archivestore.h
new file mode 100644
index 00000000000..7230ef77337
--- /dev/null
+++ b/xlators/features/cloudsync/src/cloudsync-plugins/src/cvlt/src/archivestore.h
@@ -0,0 +1,203 @@
+/*
+  Copyright (c) 2018 Commvault Systems, Inc. <http://www.commvault.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
+*/
+
+#ifndef __ARCHIVESTORE_H__
+#define __ARCHIVESTORE_H__
+
+#include <stdlib.h>
+#include <stddef.h>
+#include <stdint.h>
+#include <dlfcn.h>
+#include <uuid/uuid.h>
+
+#define CS_XATTR_ARCHIVE_UUID "trusted.cloudsync.uuid"
+#define CS_XATTR_PRODUCT_ID "trusted.cloudsync.product-id"
+#define CS_XATTR_STORE_ID "trusted.cloudsync.store-id"
+
+struct _archstore_methods;
+typedef struct _archstore_methods archstore_methods_t;
+
+struct _archstore_desc {
+    void *priv; /* Private field for store mgmt.   */
+                /* To be used only by archive store*/
+};
+typedef struct _archstore_desc archstore_desc_t;
+
+struct _archstore_info {
+    char *id;         /* Identifier for the archivestore */
+    uint32_t idlen;   /* Length of identifier string     */
+    char *prod;       /* Name of the data mgmt. product  */
+    uint32_t prodlen; /* Length of the product string    */
+};
+typedef struct _archstore_info archstore_info_t;
+
+struct _archstore_fileinfo {
+    uuid_t uuid;         /* uuid of the file                */
+    char *path;          /* file path                       */
+    uint32_t pathlength; /* length of file path             */
+};
+typedef struct _archstore_fileinfo archstore_fileinfo_t;
+
+struct _app_callback_info {
+    archstore_info_t *src_archstore;
+    archstore_fileinfo_t *src_archfile;
+    archstore_info_t *dest_archstore;
+    archstore_fileinfo_t *dest_archfile;
+};
+typedef struct _app_callback_info app_callback_info_t;
+
+typedef void (*app_callback_t)(archstore_desc_t *, app_callback_info_t *,
+                               void *, int64_t, int32_t);
+
+enum _archstore_scan_type { FULL = 1, INCREMENTAL = 2 };
+typedef enum _archstore_scan_type archstore_scan_type_t;
+
+typedef int32_t archstore_errno_t;
+
+/*
+ * Initialize archive store.
+ * arg1  pointer to structure containing archive store information
+ * arg2  error number if any generated during the initialization
+ * arg3  name of the log file
+ */
+typedef int32_t (*init_archstore_t)(archstore_desc_t *, archstore_errno_t *,
+                                    const char *);
+
+/*
+ * Clean up archive store.
+ * arg1  pointer to structure containing archive store information
+ * arg2  error number if any generated during the cleanup
+ */
+typedef int32_t (*term_archstore_t)(archstore_desc_t *, archstore_errno_t *);
+
+/*
+ * Read the contents of the file from archive store
+ * arg1  pointer to structure containing archive store description
+ * arg2  pointer to structure containing archive store information
+ * arg3  pointer to structure containing information about file to be read
+ * arg4  offset in the file from which data should be read
+ * arg5  buffer where the data should be read
+ * arg6  number of bytes of data to be read
+ * arg7  error number if any generated during the read from file
+ * arg8  callback handler to be invoked after the data is read
+ * arg9  cookie to be passed when callback is invoked
+ */
+typedef int32_t (*read_archstore_t)(archstore_desc_t *, archstore_info_t *,
+                                    archstore_fileinfo_t *, off_t, char *,
+                                    size_t, archstore_errno_t *, app_callback_t,
+                                    void *);
+
+/*
+ * Restore the contents of the file from archive store
+ * This is basically in-place restore
+ * arg1  pointer to structure containing archive store description
+ * arg2  pointer to structure containing archive store information
+ * arg3  pointer to structure containing information about file to be restored
+ * arg4  error number if any generated during the file restore
+ * arg5  callback to be invoked after the file is restored
+ * arg6  cookie to be passed when callback is invoked
+ */
+typedef int32_t (*recall_archstore_t)(archstore_desc_t *, archstore_info_t *,
+                                      archstore_fileinfo_t *,
+                                      archstore_errno_t *, app_callback_t,
+                                      void *);
+
+/*
+ * Restore the contents of the file from archive store to a different store
+ * This is basically out-of-place restore
+ * arg1  pointer to structure containing archive store description
+ * arg2  pointer to structure containing source archive store information
+ * arg3  pointer to structure containing information about file to be restored
+ * arg4  pointer to structure containing destination archive store information
+ * arg5  pointer to structure containing information about the location to
+         which the file will be restored
+ * arg6  error number if any generated during the file restore
+ * arg7  callback to be invoked after the file is restored
+ * arg8  cookie to be passed when callback is invoked
+ */
+typedef int32_t (*restore_archstore_t)(archstore_desc_t *, archstore_info_t *,
+                                       archstore_fileinfo_t *,
+                                       archstore_info_t *,
+                                       archstore_fileinfo_t *,
+                                       archstore_errno_t *, app_callback_t,
+                                       void *);
+
+/*
+ * Archive the contents of the file to archive store
+ * arg1  pointer to structure containing archive store description
+ * arg2  pointer to structure containing source archive store information
+ * arg3  pointer to structure containing information about files to be archived
+ * arg4  pointer to structure containing destination archive store information
+ * arg5  pointer to structure containing information about files that failed
+ *       to be archived
+ * arg6  error number if any generated during the file archival
+ * arg7  callback to be invoked after the file is archived
+ * arg8  cookie to be passed when callback is invoked
+ */
+typedef int32_t (*archive_archstore_t)(archstore_desc_t *, archstore_info_t *,
+                                       archstore_fileinfo_t *,
+                                       archstore_info_t *,
+                                       archstore_fileinfo_t *,
+                                       archstore_errno_t *, app_callback_t,
+                                       void *);
+
+/*
+ * Backup list of files provided in the input file
+ * arg1  pointer to structure containing archive store description
+ * arg2  pointer to structure containing source archive store information
+ * arg3  pointer to structure containing information about files to be backed up
+ * arg4  pointer to structure containing destination archive store information
+ * arg5  pointer to structure containing information about files that failed
+ *       to be backed up
+ * arg6  error number if any generated during the file archival
+ * arg7  callback to be invoked after the file is archived
+ * arg8  cookie to be passed when callback is invoked
+ */
+typedef int32_t (*backup_archstore_t)(archstore_desc_t *, archstore_info_t *,
+                                      archstore_fileinfo_t *,
+                                      archstore_info_t *,
+                                      archstore_fileinfo_t *,
+                                      archstore_errno_t *, app_callback_t,
+                                      void *);
+
+/*
+ * Scan the contents of a store and determine the files which need to be
+ * backed up.
+ * arg1  pointer to structure containing archive store description
+ * arg2  pointer to structure containing archive store information
+ * arg3  type of scan whether full or incremental
+ * arg4  path to file that contains list of files to be backed up
+ * arg5  error number if any generated during scan operation
+ */
+typedef int32_t (*scan_archstore_t)(archstore_desc_t *, archstore_info_t *,
+                                    archstore_scan_type_t, char *,
+                                    archstore_errno_t *);
+
+struct _archstore_methods {
+    init_archstore_t init;
+    term_archstore_t fini;
+    backup_archstore_t backup;
+    archive_archstore_t archive;
+    scan_archstore_t scan;
+    restore_archstore_t restore;
+    recall_archstore_t recall;
+    read_archstore_t read;
+};
+
+typedef int (*get_archstore_methods_t)(archstore_methods_t *);
+
+/*
+ * Single function that will be invoked by applications for extracting
+ * the function pointers to all data management functions.
+ */
+int32_t
+get_archstore_methods(archstore_methods_t *);
+
+#endif /* End of __ARCHIVESTORE_H__ */
diff --git a/xlators/features/cloudsync/src/cloudsync-plugins/src/cvlt/src/cvlt-messages.h b/xlators/features/cloudsync/src/cloudsync-plugins/src/cvlt/src/cvlt-messages.h
new file mode 100644
index 00000000000..57c9aa77da0
--- /dev/null
+++ b/xlators/features/cloudsync/src/cloudsync-plugins/src/cvlt/src/cvlt-messages.h
@@ -0,0 +1,30 @@
+/*
+ Copyright (c) 2015 Red Hat, Inc. <http://www.redhat.com>
+ This file is part of GlusterFS.
+
+ This file is licensed to you under your choice of the GNU Lesser
+ General Public License, version 3 or any later version (LGPLv3 or
+ later), or the GNU General Public License, version 2 (GPLv2), in all
+ cases as published by the Free Software Foundation.
+ */
+
+#ifndef _CVLT_MESSAGES_H_
+#define _CVLT_MESSAGES_H_
+
+#include <glusterfs/glfs-message-id.h>
+
+/* To add new message IDs, append new identifiers at the end of the list.
+ *
+ * Never remove a message ID. If it's not used anymore, you can rename it or
+ * leave it as it is, but not delete it. This is to prevent reutilization of
+ * IDs by other messages.
+ *
+ * The component name must match one of the entries defined in
+ * glfs-message-id.h.
+ */
+
+GLFS_MSGID(CVLT, CVLT_EXTRACTION_FAILED, CVLT_FREE,
+           CVLT_RESOURCE_ALLOCATION_FAILED, CVLT_RESTORE_FAILED,
+           CVLT_READ_FAILED, CVLT_NO_MEMORY, CVLT_DLOPEN_FAILED);
+
+#endif /* !_CVLT_MESSAGES_H_ */
diff --git a/xlators/features/cloudsync/src/cloudsync-plugins/src/cvlt/src/libcloudsynccvlt.sym b/xlators/features/cloudsync/src/cloudsync-plugins/src/cvlt/src/libcloudsynccvlt.sym
new file mode 100644
index 00000000000..0bc273670d5
--- /dev/null
+++ b/xlators/features/cloudsync/src/cloudsync-plugins/src/cvlt/src/libcloudsynccvlt.sym
@@ -0,0 +1 @@
+store_ops
diff --git a/xlators/features/cloudsync/src/cloudsync-plugins/src/cvlt/src/libcvlt-mem-types.h b/xlators/features/cloudsync/src/cloudsync-plugins/src/cvlt/src/libcvlt-mem-types.h
new file mode 100644
index 00000000000..c24fab8bfe7
--- /dev/null
+++ b/xlators/features/cloudsync/src/cloudsync-plugins/src/cvlt/src/libcvlt-mem-types.h
@@ -0,0 +1,19 @@
+/*
+ *   Copyright (c) 2018 Commvault Systems, Inc. <http://www.commvault.com>
+ *   This file is part of GlusterFS.
+ *
+ *   This file is licensed to you under your choice of the GNU Lesser
+ *   General Public License, version 3 or any later version (LGPLv3 or
+ *   later), or the GNU General Public License, version 2 (GPLv2), in all
+ *   cases as published by the Free Software Foundation.
+ */
+
+#ifndef __LIBCVLT_MEM_TYPES_H__
+#define __LIBCVLT_MEM_TYPES_H__
+
+#include <glusterfs/mem-types.h>
+enum libcvlt_mem_types_ {
+    gf_libcvlt_mt_cvlt_private_t = gf_common_mt_end + 1,
+    gf_libcvlt_mt_end
+};
+#endif /* __LIBCVLT_MEM_TYPES_H__ */
diff --git a/xlators/features/cloudsync/src/cloudsync-plugins/src/cvlt/src/libcvlt.c b/xlators/features/cloudsync/src/cloudsync-plugins/src/cvlt/src/libcvlt.c
new file mode 100644
index 00000000000..5b7272bb448
--- /dev/null
+++ b/xlators/features/cloudsync/src/cloudsync-plugins/src/cvlt/src/libcvlt.c
@@ -0,0 +1,842 @@
+#include <stdlib.h>
+#include <glusterfs/xlator.h>
+#include <glusterfs/glusterfs.h>
+#include "libcvlt.h"
+#include "cloudsync-common.h"
+#include "cvlt-messages.h"
+
+#define LIBARCHIVE_SO "libopenarchive.so"
+#define ALIGN_SIZE 4096
+#define CVLT_TRAILER "cvltv1"
+
+store_methods_t store_ops = {
+    .fop_download = cvlt_download,
+    .fop_init = cvlt_init,
+    .fop_reconfigure = cvlt_reconfigure,
+    .fop_fini = cvlt_fini,
+    .fop_remote_read = cvlt_read,
+};
+
+static const int32_t num_req = 32;
+static const int32_t num_iatt = 32;
+static char *plugin = "cvlt_cloudSync";
+
+int32_t
+mem_acct_init(xlator_t *this)
+{
+    int ret = -1;
+
+    if (!this)
+        return ret;
+
+    ret = xlator_mem_acct_init(this, gf_libcvlt_mt_end + 1);
+
+    if (ret != 0) {
+        return ret;
+    }
+
+    return ret;
+}
+
+static void
+cvlt_free_resources(archive_t *arch)
+{
+    /*
+     * We will release all the resources that were allocated by the xlator.
+     * Check whether there are any buffers which have not been released
+     * back to a mempool.
+     */
+
+    if (arch->handle) {
+        dlclose(arch->handle);
+    }
+
+    if (arch->iobuf_pool) {
+        iobuf_pool_destroy(arch->iobuf_pool);
+    }
+
+    if (arch->req_pool) {
+        mem_pool_destroy(arch->req_pool);
+        arch->req_pool = NULL;
+    }
+
+    return;
+}
+
+static int32_t
+cvlt_extract_store_fops(xlator_t *this, archive_t *arch)
+{
+    int32_t op_ret = -1;
+    get_archstore_methods_t get_archstore_methods;
+
+    /*
+     * libopenarchive.so defines methods for performing data management
+     * operations. We will extract the methods from library and these
+     * methods will be invoked for moving data between glusterfs volume
+     * and the data management product.
+     */
+
+    VALIDATE_OR_GOTO(arch, err);
+
+    arch->handle = dlopen(LIBARCHIVE_SO, RTLD_NOW);
+    if (!arch->handle) {
+        gf_msg(plugin, GF_LOG_ERROR, 0, CVLT_DLOPEN_FAILED,
+               " failed to open %s ", LIBARCHIVE_SO);
+        return op_ret;
+    }
+
+    dlerror(); /* Clear any existing error */
+
+    get_archstore_methods = dlsym(arch->handle, "get_archstore_methods");
+    if (!get_archstore_methods) {
+        gf_msg(plugin, GF_LOG_ERROR, 0, CVLT_EXTRACTION_FAILED,
+               " Error extracting get_archstore_methods()");
+        dlclose(arch->handle);
+        arch->handle = NULL;
+        return op_ret;
+    }
+
+    op_ret = get_archstore_methods(&(arch->fops));
+    if (op_ret) {
+        gf_msg(plugin, GF_LOG_ERROR, 0, CVLT_EXTRACTION_FAILED,
+               " Failed to extract methods in get_archstore_methods");
+        dlclose(arch->handle);
+        arch->handle = NULL;
+        return op_ret;
+    }
+
+err:
+    return op_ret;
+}
+
+static int32_t
+cvlt_alloc_resources(xlator_t *this, archive_t *arch, int num_req, int num_iatt)
+{
+    /*
+     * Initialize information about all the memory pools that will be
+     * used by this xlator.
+     */
+    arch->nreqs = 0;
+
+    arch->req_pool = NULL;
+
+    arch->handle = NULL;
+    arch->xl = this;
+
+    arch->req_pool = mem_pool_new(cvlt_request_t, num_req);
+    if (!arch->req_pool) {
+        goto err;
+    }
+
+    arch->iobuf_pool = iobuf_pool_new();
+    if (!arch->iobuf_pool) {
+        goto err;
+    }
+
+    if (cvlt_extract_store_fops(this, arch)) {
+        goto err;
+    }
+
+    return 0;
+
+err:
+
+    return -1;
+}
+
+static void
+cvlt_req_init(cvlt_request_t *req)
+{
+    sem_init(&(req->sem), 0, 0);
+
+    return;
+}
+
+static void
+cvlt_req_destroy(cvlt_request_t *req)
+{
+    if (req->iobuf) {
+        iobuf_unref(req->iobuf);
+    }
+
+    if (req->iobref) {
+        iobref_unref(req->iobref);
+    }
+
+    sem_destroy(&(req->sem));
+
+    return;
+}
+
+static cvlt_request_t *
+cvlt_alloc_req(archive_t *arch)
+{
+    cvlt_request_t *reqptr = NULL;
+
+    if (!arch) {
+        goto err;
+    }
+
+    if (arch->req_pool) {
+        reqptr = mem_get0(arch->req_pool);
+        if (reqptr) {
+            cvlt_req_init(reqptr);
+        }
+    }
+
+    if (reqptr) {
+        LOCK(&(arch->lock));
+        arch->nreqs++;
+        UNLOCK(&(arch->lock));
+    }
+
+err:
+    return reqptr;
+}
+
+static int32_t
+cvlt_free_req(archive_t *arch, cvlt_request_t *reqptr)
+{
+    if (!reqptr) {
+        goto err;
+    }
+
+    if (!arch) {
+        goto err;
+    }
+
+    if (arch->req_pool) {
+        /*
+         * Free the request resources if they exist.
+         */
+
+        cvlt_req_destroy(reqptr);
+        mem_put(reqptr);
+
+        LOCK(&(arch->lock));
+        arch->nreqs--;
+        UNLOCK(&(arch->lock));
+    }
+
+    return 0;
+
+err:
+    return -1;
+}
+
+static int32_t
+cvlt_init_xlator(xlator_t *this, archive_t *arch, int num_req, int num_iatt)
+{
+    int32_t ret = -1;
+    int32_t errnum = -1;
+    int32_t locked = 0;
+
+    /*
+     * Perform all the initializations needed for brining up the xlator.
+     */
+    if (!arch) {
+        goto err;
+    }
+
+    LOCK_INIT(&(arch->lock));
+    LOCK(&(arch->lock));
+
+    locked = 1;
+
+    ret = cvlt_alloc_resources(this, arch, num_req, num_iatt);
+
+    if (ret) {
+        goto err;
+    }
+
+    /*
+     * Now that the fops have been extracted initialize the store
+     */
+    ret = arch->fops.init(&(arch->descinfo), &errnum, plugin);
+    if (ret) {
+        goto err;
+    }
+
+    UNLOCK(&(arch->lock));
+    locked = 0;
+    ret = 0;
+
+    return ret;
+
+err:
+    if (arch) {
+        cvlt_free_resources(arch);
+
+        if (locked) {
+            UNLOCK(&(arch->lock));
+        }
+    }
+
+    return ret;
+}
+
+static int32_t
+cvlt_term_xlator(archive_t *arch)
+{
+    int32_t errnum = -1;
+
+    if (!arch) {
+        goto err;
+    }
+
+    LOCK(&(arch->lock));
+
+    /*
+     * Release the resources that have been allocated inside store
+     */
+    arch->fops.fini(&(arch->descinfo), &errnum);
+
+    cvlt_free_resources(arch);
+
+    UNLOCK(&(arch->lock));
+
+    GF_FREE(arch);
+
+    return 0;
+
+err:
+    return -1;
+}
+
+static int32_t
+cvlt_init_store_info(archive_t *priv, archstore_info_t *store_info)
+{
+    if (!store_info) {
+        return -1;
+    }
+
+    store_info->prod = priv->product_id;
+    store_info->prodlen = strlen(priv->product_id);
+
+    store_info->id = priv->store_id;
+    store_info->idlen = strlen(priv->store_id);
+
+    return 0;
+}
+
+static int32_t
+cvlt_init_file_info(cs_loc_xattr_t *xattr, archstore_fileinfo_t *file_info)
+{
+    if (!xattr || !file_info) {
+        return -1;
+    }
+
+    gf_uuid_copy(file_info->uuid, xattr->uuid);
+    file_info->path = xattr->file_path;
+    file_info->pathlength = strlen(xattr->file_path);
+
+    return 0;
+}
+
+static int32_t
+cvlt_init_gluster_store_info(cs_loc_xattr_t *xattr,
+                             archstore_info_t *store_info)
+{
+    static char *product = "glusterfs";
+
+    if (!xattr || !store_info) {
+        return -1;
+    }
+
+    store_info->prod = product;
+    store_info->prodlen = strlen(product);
+
+    store_info->id = xattr->volname;
+    store_info->idlen = strlen(xattr->volname);
+
+    return 0;
+}
+
+static int32_t
+cvlt_init_gluster_file_info(cs_loc_xattr_t *xattr,
+                            archstore_fileinfo_t *file_info)
+{
+    if (!xattr || !file_info) {
+        return -1;
+    }
+
+    gf_uuid_copy(file_info->uuid, xattr->gfid);
+    file_info->path = xattr->file_path;
+    file_info->pathlength = strlen(xattr->file_path);
+
+    return 0;
+}
+
+static void
+cvlt_copy_stat_info(struct iatt *buf, cs_size_xattr_t *xattrs)
+{
+    /*
+     * If the file was archived then the reported size will not be a
+     * correct one. We need to fix this.
+     */
+    if (buf && xattrs) {
+        buf->ia_size = xattrs->size;
+        buf->ia_blksize = xattrs->blksize;
+        buf->ia_blocks = xattrs->blocks;
+    }
+
+    return;
+}
+
+static void
+cvlt_readv_complete(archstore_desc_t *desc, app_callback_info_t *cbkinfo,
+                    void *cookie, int64_t op_ret, int32_t op_errno)
+{
+    struct iovec iov;
+    xlator_t *this = NULL;
+    struct iatt postbuf = {
+        0,
+    };
+    call_frame_t *frame = NULL;
+    cvlt_request_t *req = (cvlt_request_t *)cookie;
+    cs_local_t *local = NULL;
+    cs_private_t *cspriv = NULL;
+    archive_t *priv = NULL;
+
+    frame = req->frame;
+    this = frame->this;
+    local = frame->local;
+
+    cspriv = this->private;
+    priv = (archive_t *)cspriv->stores->config;
+
+    if (strcmp(priv->trailer, CVLT_TRAILER)) {
+        op_ret = -1;
+        op_errno = EINVAL;
+        goto out;
+    }
+
+    gf_msg_debug(plugin, 0,
+                 " Read callback invoked offset:%" PRIu64 "bytes: %" PRIu64
+                 " op : %d ret : %" PRId64 " errno : %d",
+                 req->offset, req->bytes, req->op_type, op_ret, op_errno);
+
+    if (op_ret < 0) {
+        goto out;
+    }
+
+    req->iobref = iobref_new();
+    if (!req->iobref) {
+        op_ret = -1;
+        op_errno = ENOMEM;
+        goto out;
+    }
+
+    iobref_add(req->iobref, req->iobuf);
+    iov.iov_base = iobuf_ptr(req->iobuf);
+    iov.iov_len = op_ret;
+
+    cvlt_copy_stat_info(&postbuf, &(req->szxattr));
+
+    /*
+     * Hack to notify higher layers of EOF.
+     */
+    if (!postbuf.ia_size || (req->offset + iov.iov_len >= postbuf.ia_size)) {
+        gf_msg_debug(plugin, 0, " signalling end-of-file for uuid=%s",
+                     uuid_utoa(req->file_info.uuid));
+        op_errno = ENOENT;
+    }
+
+out:
+
+    STACK_UNWIND_STRICT(readv, frame, op_ret, op_errno, &iov, 1, &postbuf,
+                        req->iobref, local->xattr_rsp);
+
+    cvlt_free_req(priv, req);
+
+    return;
+}
+
+static void
+cvlt_download_complete(archstore_desc_t *store, app_callback_info_t *cbk_info,
+                       void *cookie, int64_t ret, int errcode)
+{
+    cvlt_request_t *req = (cvlt_request_t *)cookie;
+
+    gf_msg_debug(plugin, 0,
+                 " Download callback invoked  ret : %" PRId64 " errno : %d",
+                 ret, errcode);
+
+    req->op_ret = ret;
+    req->op_errno = errcode;
+    sem_post(&(req->sem));
+
+    return;
+}
+
+void *
+cvlt_init(xlator_t *this)
+{
+    int ret = 0;
+    archive_t *priv = NULL;
+
+    if (!this->children || this->children->next) {
+        gf_msg(plugin, GF_LOG_ERROR, ENOMEM, 0,
+               "should have exactly one child");
+        ret = -1;
+        goto out;
+    }
+
+    if (!this->parents) {
+        gf_msg(plugin, GF_LOG_ERROR, ENOMEM, 0,
+               "dangling volume. check volfile");
+        ret = -1;
+        goto out;
+    }
+
+    priv = GF_CALLOC(1, sizeof(archive_t), gf_libcvlt_mt_cvlt_private_t);
+    if (!priv) {
+        ret = -1;
+        goto out;
+    }
+
+    priv->trailer = CVLT_TRAILER;
+    if (cvlt_init_xlator(this, priv, num_req, num_iatt)) {
+        gf_msg(plugin, GF_LOG_ERROR, ENOMEM, 0, "xlator init failed");
+        ret = -1;
+        goto out;
+    }
+
+    GF_OPTION_INIT("cloudsync-store-id", priv->store_id, str, out);
+    GF_OPTION_INIT("cloudsync-product-id", priv->product_id, str, out);
+
+    gf_msg(plugin, GF_LOG_INFO, 0, 0,
+           "store id is : %s "
+           "product id is : %s.",
+           priv->store_id, priv->product_id);
+out:
+    if (ret == -1) {
+        cvlt_term_xlator(priv);
+        return (NULL);
+    }
+    return priv;
+}
+
+int
+cvlt_reconfigure(xlator_t *this, dict_t *options)
+{
+    cs_private_t *cspriv = NULL;
+    archive_t *priv = NULL;
+
+    cspriv = this->private;
+    priv = (archive_t *)cspriv->stores->config;
+
+    if (strcmp(priv->trailer, CVLT_TRAILER))
+        goto out;
+
+    GF_OPTION_RECONF("cloudsync-store-id", priv->store_id, options, str, out);
+
+    GF_OPTION_RECONF("cloudsync-product-id", priv->product_id, options, str,
+                     out);
+    gf_msg_debug(plugin, 0,
+                 "store id is : %s "
+                 "product id is : %s.",
+                 priv->store_id, priv->product_id);
+    return 0;
+out:
+    return -1;
+}
+
+void
+cvlt_fini(void *config)
+{
+    archive_t *priv = NULL;
+
+    priv = (archive_t *)config;
+
+    if (strcmp(priv->trailer, CVLT_TRAILER))
+        return;
+
+    cvlt_term_xlator(priv);
+    gf_msg(plugin, GF_LOG_INFO, 0, CVLT_FREE, " released xlator resources");
+    return;
+}
+
+int
+cvlt_download(call_frame_t *frame, void *config)
+{
+    archive_t *parch = NULL;
+    cs_local_t *local = frame->local;
+    cs_loc_xattr_t *locxattr = local->xattrinfo.lxattr;
+    cvlt_request_t *req = NULL;
+    archstore_info_t dest_storeinfo;
+    archstore_fileinfo_t dest_fileinfo;
+    int32_t op_ret, op_errno;
+
+    parch = (archive_t *)config;
+
+    if (strcmp(parch->trailer, CVLT_TRAILER)) {
+        op_ret = -1;
+        op_errno = EINVAL;
+        goto err;
+    }
+
+    gf_msg_debug(plugin, 0, " download invoked for uuid = %s  gfid=%s ",
+                 locxattr->uuid, uuid_utoa(locxattr->gfid));
+
+    if (!(parch->fops.restore)) {
+        op_errno = ELIBBAD;
+        goto err;
+    }
+
+    /*
+     * Download needs to be processed. Allocate a request.
+     */
+    req = cvlt_alloc_req(parch);
+
+    if (!req) {
+        gf_msg(plugin, GF_LOG_ERROR, ENOMEM, CVLT_RESOURCE_ALLOCATION_FAILED,
+               " failed to allocated request for gfid=%s",
+               uuid_utoa(locxattr->gfid));
+        op_errno = ENOMEM;
+        goto err;
+    }
+
+    /*
+     * Initialize the request object.
+     */
+    req->op_type = CVLT_RESTORE_OP;
+    req->frame = frame;
+
+    /*
+     * The file is currently residing inside a data management store.
+     * To restore the file contents we need to provide the information
+     * about data management store.
+     */
+    op_ret = cvlt_init_store_info(parch, &(req->store_info));
+    if (op_ret < 0) {
+        gf_msg(plugin, GF_LOG_ERROR, 0, CVLT_EXTRACTION_FAILED,
+               " failed to extract store info for gfid=%s",
+               uuid_utoa(locxattr->gfid));
+        goto err;
+    }
+
+    op_ret = cvlt_init_file_info(locxattr, &(req->file_info));
+    if (op_ret < 0) {
+        gf_msg(plugin, GF_LOG_ERROR, 0, CVLT_EXTRACTION_FAILED,
+               " failed to extract file info for gfid=%s",
+               uuid_utoa(locxattr->gfid));
+        goto err;
+    }
+
+    /*
+     * We need to perform in-place restore of the file from data management
+     * store to gusterfs volume.
+     */
+    op_ret = cvlt_init_gluster_store_info(locxattr, &dest_storeinfo);
+    if (op_ret < 0) {
+        gf_msg(plugin, GF_LOG_ERROR, 0, CVLT_EXTRACTION_FAILED,
+               " failed to extract destination store info for gfid=%s",
+               uuid_utoa(locxattr->gfid));
+        goto err;
+    }
+
+    op_ret = cvlt_init_gluster_file_info(locxattr, &dest_fileinfo);
+    if (op_ret < 0) {
+        gf_msg(plugin, GF_LOG_ERROR, 0, CVLT_EXTRACTION_FAILED,
+               " failed to extract file info for gfid=%s",
+               uuid_utoa(locxattr->gfid));
+        goto err;
+    }
+
+    /*
+     * Submit the restore request.
+     */
+    op_ret = parch->fops.restore(&(parch->descinfo), &(req->store_info),
+                                 &(req->file_info), &dest_storeinfo,
+                                 &dest_fileinfo, &op_errno,
+                                 cvlt_download_complete, req);
+    if (op_ret < 0) {
+        gf_msg(plugin, GF_LOG_ERROR, 0, CVLT_RESTORE_FAILED,
+               " failed to restore file gfid=%s from data management store",
+               uuid_utoa(locxattr->gfid));
+        goto err;
+    }
+
+    /*
+     * Wait for the restore to complete.
+     */
+    sem_wait(&(req->sem));
+
+    if (req->op_ret < 0) {
+        gf_msg(plugin, GF_LOG_ERROR, 0, CVLT_RESTORE_FAILED,
+               " restored failed for gfid=%s", uuid_utoa(locxattr->gfid));
+        goto err;
+    }
+
+    if (req) {
+        cvlt_free_req(parch, req);
+    }
+
+    return 0;
+
+err:
+
+    if (req) {
+        cvlt_free_req(parch, req);
+    }
+
+    return -1;
+}
+
+int
+cvlt_read(call_frame_t *frame, void *config)
+{
+    int32_t op_ret = -1;
+    int32_t op_errno = 0;
+    archive_t *parch = NULL;
+    cvlt_request_t *req = NULL;
+    struct iovec iov = {
+        0,
+    };
+    struct iobref *iobref;
+    size_t size = 0;
+    off_t off = 0;
+
+    cs_local_t *local = frame->local;
+    cs_loc_xattr_t *locxattr = local->xattrinfo.lxattr;
+
+    size = local->xattrinfo.size;
+    off = local->xattrinfo.offset;
+
+    parch = (archive_t *)config;
+
+    if (strcmp(parch->trailer, CVLT_TRAILER)) {
+        op_ret = -1;
+        op_errno = EINVAL;
+        goto err;
+    }
+
+    gf_msg_debug(plugin, 0,
+                 " read invoked for gfid = %s offset = %" PRIu64
+                 " file_size = %" PRIu64,
+                 uuid_utoa(locxattr->gfid), off, local->stbuf.ia_size);
+
+    if (off >= local->stbuf.ia_size) {
+        /*
+         * Hack to notify higher layers of EOF.
+         */
+
+        op_errno = ENOENT;
+        op_ret = 0;
+
+        gf_msg(plugin, GF_LOG_ERROR, 0, CVLT_READ_FAILED,
+               " reporting end-of-file for gfid=%s", uuid_utoa(locxattr->gfid));
+
+        goto err;
+    }
+
+    if (!size) {
+        op_errno = EINVAL;
+
+        gf_msg(plugin, GF_LOG_ERROR, 0, CVLT_READ_FAILED,
+               " zero size read attempted on gfid=%s",
+               uuid_utoa(locxattr->gfid));
+        goto err;
+    }
+
+    if (!(parch->fops.read)) {
+        op_errno = ELIBBAD;
+        goto err;
+    }
+
+    /*
+     * The read request need to be processed. Allocate a request.
+     */
+    req = cvlt_alloc_req(parch);
+
+    if (!req) {
+        gf_msg(plugin, GF_LOG_ERROR, ENOMEM, CVLT_NO_MEMORY,
+               " failed to allocated request for gfid=%s",
+               uuid_utoa(locxattr->gfid));
+        op_errno = ENOMEM;
+        goto err;
+    }
+
+    req->iobuf = iobuf_get_page_aligned(parch->iobuf_pool, size, ALIGN_SIZE);
+    if (!req->iobuf) {
+        op_errno = ENOMEM;
+        goto err;
+    }
+
+    /*
+     * Initialize the request object.
+     */
+    req->op_type = CVLT_READ_OP;
+    req->offset = off;
+    req->bytes = size;
+    req->frame = frame;
+    req->szxattr.size = local->stbuf.ia_size;
+    req->szxattr.blocks = local->stbuf.ia_blocks;
+    req->szxattr.blksize = local->stbuf.ia_blksize;
+
+    /*
+     * The file is currently residing inside a data management store.
+     * To read the file contents we need to provide the information
+     * about data management store.
+     */
+    op_ret = cvlt_init_store_info(parch, &(req->store_info));
+    if (op_ret < 0) {
+        gf_msg(plugin, GF_LOG_ERROR, 0, CVLT_EXTRACTION_FAILED,
+               " failed to extract store info for gfid=%s"
+               " offset=%" PRIu64 " size=%" GF_PRI_SIZET
+               ", "
+               " buf=%p",
+               uuid_utoa(locxattr->gfid), off, size, req->iobuf->ptr);
+        goto err;
+    }
+
+    op_ret = cvlt_init_file_info(locxattr, &(req->file_info));
+    if (op_ret < 0) {
+        gf_msg(plugin, GF_LOG_ERROR, 0, CVLT_EXTRACTION_FAILED,
+               " failed to extract file info for gfid=%s"
+               " offset=%" PRIu64 " size=%" GF_PRI_SIZET
+               ", "
+               " buf=%p",
+               uuid_utoa(locxattr->gfid), off, size, req->iobuf->ptr);
+        goto err;
+    }
+
+    /*
+     * Submit the read request.
+     */
+    op_ret = parch->fops.read(&(parch->descinfo), &(req->store_info),
+                              &(req->file_info), off, req->iobuf->ptr, size,
+                              &op_errno, cvlt_readv_complete, req);
+
+    if (op_ret < 0) {
+        gf_msg(plugin, GF_LOG_ERROR, 0, CVLT_EXTRACTION_FAILED,
+               " read failed on gfid=%s"
+               " offset=%" PRIu64 " size=%" GF_PRI_SIZET
+               ", "
+               " buf=%p",
+               uuid_utoa(locxattr->gfid), off, size, req->iobuf->ptr);
+        goto err;
+    }
+
+    return 0;
+
+err:
+
+    iobref = iobref_new();
+    gf_msg_debug(plugin, 0, " read unwinding stack op_ret = %d, op_errno = %d",
+                 op_ret, op_errno);
+
+    STACK_UNWIND_STRICT(readv, frame, op_ret, op_errno, &iov, 1,
+                        &(local->stbuf), iobref, local->xattr_rsp);
+
+    if (iobref) {
+        iobref_unref(iobref);
+    }
+
+    if (req) {
+        cvlt_free_req(parch, req);
+    }
+
+    return 0;
+}
diff --git a/xlators/features/cloudsync/src/cloudsync-plugins/src/cvlt/src/libcvlt.h b/xlators/features/cloudsync/src/cloudsync-plugins/src/cvlt/src/libcvlt.h
new file mode 100644
index 00000000000..c45ac948f6c
--- /dev/null
+++ b/xlators/features/cloudsync/src/cloudsync-plugins/src/cvlt/src/libcvlt.h
@@ -0,0 +1,84 @@
+/*
+  Copyright (c) 2018 Commvault Systems, Inc. <http://www.commvault.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
+*/
+#ifndef _LIBCVLT_H
+#define _LIBCVLT_H
+
+#include <semaphore.h>
+#include <glusterfs/xlator.h>
+#include <glusterfs/glusterfs.h>
+#include <glusterfs/call-stub.h>
+#include <glusterfs/syncop.h>
+#include <glusterfs/compat-errno.h>
+#include "cloudsync-common.h"
+#include "libcvlt-mem-types.h"
+#include "archivestore.h"
+
+enum _cvlt_op {
+    CVLT_READ_OP = 1,
+    CVLT_WRITE_OP = 2,
+    CVLT_RESTORE_OP = 3,
+    CVLT_ARCHIVE_OP = 4,
+    CVLT_LOOKUP_OP = 5,
+    CVLT_XATTR_OP = 6,
+    CVLT_STAT_OP = 7,
+    CVLT_FSTAT_op = 8,
+    CVLT_UNDEF_OP = 127
+};
+typedef enum _cvlt_op cvlt_op_t;
+
+struct _archive;
+struct _cvlt_request {
+    uint64_t offset;
+    uint64_t bytes;
+    struct iobuf *iobuf;
+    struct iobref *iobref;
+    call_frame_t *frame;
+    cvlt_op_t op_type;
+    int32_t op_ret;
+    int32_t op_errno;
+    xlator_t *this;
+    sem_t sem;
+    archstore_info_t store_info;
+    archstore_fileinfo_t file_info;
+    cs_size_xattr_t szxattr;
+};
+typedef struct _cvlt_request cvlt_request_t;
+
+struct _archive {
+    gf_lock_t lock;                /* lock for controlling access   */
+    xlator_t *xl;                  /* xlator                        */
+    void *handle;                  /* handle returned from dlopen   */
+    int32_t nreqs;                 /* num requests active           */
+    struct mem_pool *req_pool;     /* pool for requests             */
+    struct iobuf_pool *iobuf_pool; /* iobuff pool                   */
+    archstore_desc_t descinfo;     /* Archive store descriptor info */
+    archstore_methods_t fops;      /* function pointers             */
+    char *product_id;
+    char *store_id;
+    char *trailer;
+};
+typedef struct _archive archive_t;
+
+void *
+cvlt_init(xlator_t *);
+
+int
+cvlt_reconfigure(xlator_t *, dict_t *);
+
+void
+cvlt_fini(void *);
+
+int
+cvlt_download(call_frame_t *, void *);
+
+int
+cvlt_read(call_frame_t *, void *);
+
+#endif
diff --git a/xlators/features/cloudsync/src/cloudsync.c b/xlators/features/cloudsync/src/cloudsync.c
new file mode 100644
index 00000000000..7f0b9e563b8
--- /dev/null
+++ b/xlators/features/cloudsync/src/cloudsync.c
@@ -0,0 +1,2076 @@
+/*
+ *   Copyright (c) 2018 Red Hat, Inc. <http://www.redhat.com>
+ *   This file is part of GlusterFS.
+ *
+ *   This file is licensed to you under your choice of the GNU Lesser
+ *   General Public License, version 3 or any later version (LGPLv3 or
+ *   later), or the GNU General Public License, version 2 (GPLv2), in all
+ *   cases as published by the Free Software Foundation.
+ */
+
+#include <glusterfs/glusterfs.h>
+#include <glusterfs/xlator.h>
+#include <glusterfs/defaults.h>
+#include "cloudsync.h"
+#include "cloudsync-common.h"
+#include <glusterfs/call-stub.h>
+#include "cloudsync-autogen-fops.h"
+
+#include <string.h>
+#include <dlfcn.h>
+
+static void
+cs_cleanup_private(cs_private_t *priv)
+{
+    if (priv) {
+        if (priv->stores) {
+            priv->stores->fini(priv->stores->config);
+            GF_FREE(priv->stores);
+        }
+
+        pthread_spin_destroy(&priv->lock);
+        GF_FREE(priv);
+    }
+
+    return;
+}
+
+static struct cs_plugin plugins[] = {
+    {.name = "cloudsyncs3",
+     .library = "cloudsyncs3.so",
+     .description = "cloudsync s3 store."},
+#if defined(__linux__)
+    {.name = "cvlt",
+     .library = "cloudsynccvlt.so",
+     .description = "Commvault content store."},
+#endif
+    {.name = NULL},
+};
+
+int
+cs_init(xlator_t *this)
+{
+    cs_private_t *priv = NULL;
+    gf_boolean_t per_vol = _gf_false;
+    int ret = 0;
+    char *libpath = NULL;
+    store_methods_t *store_methods = NULL;
+    void *handle = NULL;
+    char *temp_str = NULL;
+    int index = 0;
+    char *libname = NULL;
+
+    priv = GF_CALLOC(1, sizeof(*priv), gf_cs_mt_cs_private_t);
+    if (!priv) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, 0, "insufficient memory");
+        goto out;
+    }
+
+    priv->this = this;
+
+    this->local_pool = mem_pool_new(cs_local_t, 512);
+    if (!this->local_pool) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, ENOMEM, "initialisation failed.");
+        ret = -1;
+        goto out;
+    }
+
+    this->private = priv;
+
+    GF_OPTION_INIT("cloudsync-remote-read", priv->remote_read, bool, out);
+
+    /* temp workaround. Should be configurable through glusterd*/
+    per_vol = _gf_true;
+
+    if (per_vol) {
+        if (dict_get_str_sizen(this->options, "cloudsync-storetype",
+                               &temp_str) == 0) {
+            for (index = 0; plugins[index].name; index++) {
+                if (!strcmp(temp_str, plugins[index].name)) {
+                    libname = plugins[index].library;
+                    break;
+                }
+            }
+        } else {
+            ret = 0;
+        }
+
+        if (!libname) {
+            gf_msg(this->name, GF_LOG_WARNING, 0, 0, "no plugin enabled");
+            ret = 0;
+            goto out;
+        }
+
+        ret = gf_asprintf(&libpath, "%s/%s", CS_PLUGINDIR, libname);
+        if (ret == -1) {
+            goto out;
+        }
+
+        handle = dlopen(libpath, RTLD_NOW);
+        if (!handle) {
+            gf_msg(this->name, GF_LOG_WARNING, 0, 0,
+                   "could not "
+                   "load the required library. %s",
+                   dlerror());
+            ret = 0;
+            goto out;
+        } else {
+            gf_msg(this->name, GF_LOG_INFO, 0, 0,
+                   "loading library:%s successful", libname);
+        }
+
+        priv->stores = GF_CALLOC(1, sizeof(struct cs_remote_stores),
+                                 gf_cs_mt_cs_remote_stores_t);
+        if (!priv->stores) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, 0,
+                   "Could not "
+                   "allocate memory for priv->stores");
+            ret = -1;
+            goto out;
+        }
+
+        (void)dlerror(); /* clear out previous error string */
+
+        /* load library methods */
+        store_methods = (store_methods_t *)dlsym(handle, "store_ops");
+        if (!store_methods) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, 0, "null store_methods %s",
+                   dlerror());
+            ret = -1;
+            goto out;
+        }
+
+        (void)dlerror();
+
+        if (priv->remote_read) {
+            priv->stores->rdfop = store_methods->fop_remote_read;
+            if (!priv->stores->rdfop) {
+                gf_msg(this->name, GF_LOG_ERROR, 0, 0,
+                       "failed to get"
+                       " read fop %s",
+                       dlerror());
+                ret = -1;
+                goto out;
+            }
+        }
+
+        priv->stores->dlfop = store_methods->fop_download;
+        if (!priv->stores->dlfop) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, 0,
+                   "failed to get"
+                   " download fop %s",
+                   dlerror());
+            ret = -1;
+            goto out;
+        }
+
+        (void)dlerror();
+        priv->stores->init = store_methods->fop_init;
+        if (!priv->stores->init) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, 0,
+                   "failed to get"
+                   " init fop %s",
+                   dlerror());
+            ret = -1;
+            goto out;
+        }
+
+        (void)dlerror();
+        priv->stores->reconfigure = store_methods->fop_reconfigure;
+        if (!priv->stores->reconfigure) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, 0,
+                   "failed to get"
+                   " reconfigure fop %s",
+                   dlerror());
+            ret = -1;
+            goto out;
+        }
+
+        priv->stores->handle = handle;
+
+        priv->stores->config = (void *)((priv->stores->init)(this));
+        if (!priv->stores->config) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, 0, "null config");
+            ret = -1;
+            goto out;
+        }
+    }
+
+    ret = 0;
+
+out:
+    if (ret == -1) {
+        if (this->local_pool) {
+            mem_pool_destroy(this->local_pool);
+            this->local_pool = NULL;
+        }
+
+        cs_cleanup_private(priv);
+
+        if (handle) {
+            dlclose(handle);
+        }
+    }
+
+    GF_FREE(libpath);
+
+    return ret;
+}
+
+int
+cs_forget(xlator_t *this, inode_t *inode)
+{
+    uint64_t ctx_int = 0;
+    cs_inode_ctx_t *ctx = NULL;
+
+    inode_ctx_del(inode, this, &ctx_int);
+    if (!ctx_int)
+        return 0;
+
+    ctx = (cs_inode_ctx_t *)(uintptr_t)ctx_int;
+
+    GF_FREE(ctx);
+    return 0;
+}
+
+void
+cs_fini(xlator_t *this)
+{
+    cs_private_t *priv = NULL;
+    priv = this->private;
+
+    cs_cleanup_private(priv);
+}
+
+int
+cs_reconfigure(xlator_t *this, dict_t *options)
+{
+    cs_private_t *priv = NULL;
+    int ret = 0;
+
+    priv = this->private;
+    if (!priv) {
+        ret = -1;
+        goto out;
+    }
+
+    GF_OPTION_RECONF("cloudsync-remote-read", priv->remote_read, options, bool,
+                     out);
+
+    /* needed only for per volume configuration*/
+    ret = priv->stores->reconfigure(this, options);
+
+out:
+    return ret;
+}
+
+int32_t
+cs_mem_acct_init(xlator_t *this)
+{
+    int ret = -1;
+
+    GF_VALIDATE_OR_GOTO("cloudsync", this, out);
+
+    ret = xlator_mem_acct_init(this, gf_cs_mt_end + 1);
+
+    if (ret != 0) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, 0, "Memory accounting init failed");
+        return ret;
+    }
+out:
+    return ret;
+}
+
+int32_t
+cs_readdirp(call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
+            off_t off, dict_t *xdata)
+{
+    int ret = 0;
+    int op_errno = ENOMEM;
+
+    if (!xdata) {
+        xdata = dict_new();
+        if (!xdata) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, ENOMEM,
+                   "failed to create "
+                   "dict");
+            goto err;
+        }
+    }
+
+    ret = dict_set_uint32(xdata, GF_CS_OBJECT_STATUS, 1);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, 0,
+               "dict_set failed key:"
+               " %s",
+               GF_CS_OBJECT_STATUS);
+        goto err;
+    }
+
+    STACK_WIND(frame, default_readdirp_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->readdirp, fd, size, off, xdata);
+    return 0;
+err:
+    STACK_UNWIND_STRICT(readdirp, frame, -1, op_errno, NULL, NULL);
+    return 0;
+}
+
+int32_t
+cs_truncate_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
+                struct iatt *postbuf, dict_t *xdata)
+{
+    cs_local_t *local = NULL;
+    int ret = 0;
+    uint64_t val = 0;
+
+    local = frame->local;
+
+    local->call_cnt++;
+
+    if (op_ret == -1) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, 0, "truncate failed");
+        ret = dict_get_uint64(xdata, GF_CS_OBJECT_STATUS, &val);
+        if (ret == 0) {
+            if (val == GF_CS_ERROR) {
+                gf_msg(this->name, GF_LOG_ERROR, 0, 0,
+                       "could not get file state, unwinding");
+                op_ret = -1;
+                op_errno = EIO;
+                goto unwind;
+            } else {
+                __cs_inode_ctx_update(this, local->loc.inode, val);
+                gf_msg(this->name, GF_LOG_INFO, 0, 0, " state = %" PRIu64, val);
+
+                if (local->call_cnt == 1 &&
+                    (val == GF_CS_REMOTE || val == GF_CS_DOWNLOADING)) {
+                    gf_msg(this->name, GF_LOG_WARNING, 0, 0,
+                           "will repair and download "
+                           "the file, current state : %" PRIu64,
+                           val);
+                    goto repair;
+                } else {
+                    gf_msg(this->name, GF_LOG_ERROR, 0, 0,
+                           "second truncate, Unwinding");
+                    goto unwind;
+                }
+            }
+        } else {
+            gf_msg(this->name, GF_LOG_ERROR, 0, 0,
+                   "file state "
+                   "could not be figured, unwinding");
+            goto unwind;
+        }
+    } else {
+        /* successful write => file is local */
+        __cs_inode_ctx_update(this, local->loc.inode, GF_CS_LOCAL);
+        gf_msg(this->name, GF_LOG_INFO, 0, 0,
+               "state : GF_CS_LOCAL"
+               ", truncate successful");
+
+        goto unwind;
+    }
+
+repair:
+    ret = locate_and_execute(frame);
+    if (ret) {
+        goto unwind;
+    }
+
+    return 0;
+
+unwind:
+    CS_STACK_UNWIND(truncate, frame, op_ret, op_errno, prebuf, postbuf, xdata);
+    return 0;
+}
+
+int32_t
+cs_truncate(call_frame_t *frame, xlator_t *this, loc_t *loc, off_t offset,
+            dict_t *xdata)
+{
+    cs_local_t *local = NULL;
+    int ret = 0;
+    cs_inode_ctx_t *ctx = NULL;
+    gf_cs_obj_state state = -1;
+
+    VALIDATE_OR_GOTO(frame, err);
+    VALIDATE_OR_GOTO(this, err);
+    VALIDATE_OR_GOTO(loc, err);
+
+    local = cs_local_init(this, frame, loc, NULL, GF_FOP_TRUNCATE);
+    if (!local) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, 0, "local init failed");
+        goto err;
+    }
+
+    __cs_inode_ctx_get(this, loc->inode, &ctx);
+
+    if (ctx)
+        state = __cs_get_file_state(loc->inode, ctx);
+    else
+        state = GF_CS_LOCAL;
+
+    local->xattr_req = xdata ? dict_ref(xdata) : (xdata = dict_new());
+
+    ret = dict_set_uint32(local->xattr_req, GF_CS_OBJECT_STATUS, 1);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, 0,
+               "dict_set failed key:"
+               " %s",
+               GF_CS_OBJECT_STATUS);
+        goto err;
+    }
+
+    local->stub = fop_truncate_stub(frame, cs_resume_truncate, loc, offset,
+                                    xdata);
+    if (!local->stub) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, 0, "insufficient memory");
+        goto err;
+    }
+
+    if (state == GF_CS_LOCAL) {
+        STACK_WIND(frame, cs_truncate_cbk, FIRST_CHILD(this),
+                   FIRST_CHILD(this)->fops->truncate, loc, offset, xdata);
+
+    } else {
+        local->call_cnt++;
+        ret = locate_and_execute(frame);
+        if (ret) {
+            goto err;
+        }
+    }
+
+    return 0;
+err:
+    CS_STACK_UNWIND(truncate, frame, -1, ENOMEM, NULL, NULL, NULL);
+    return 0;
+}
+
+int32_t
+cs_statfs_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int32_t op_ret,
+              int32_t op_errno, struct statvfs *buf, dict_t *xdata)
+{
+    STACK_UNWIND_STRICT(statfs, frame, op_ret, op_errno, buf, xdata);
+    return 0;
+}
+
+int32_t
+cs_statfs(call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xdata)
+{
+    STACK_WIND(frame, cs_statfs_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->statfs, loc, xdata);
+    return 0;
+}
+
+int32_t
+cs_getxattr_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                int32_t op_ret, int32_t op_errno, dict_t *dict, dict_t *xdata)
+{
+    STACK_UNWIND_STRICT(getxattr, frame, op_ret, op_errno, dict, xdata);
+    return 0;
+}
+
+int32_t
+cs_getxattr(call_frame_t *frame, xlator_t *this, loc_t *loc, const char *name,
+            dict_t *xattr_req)
+{
+    STACK_WIND(frame, cs_getxattr_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->getxattr, loc, name, xattr_req);
+    return 0;
+}
+
+int32_t
+cs_setxattr_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                int32_t op_ret, int32_t op_errno, dict_t *xdata)
+{
+    cs_local_t *local = NULL;
+
+    local = frame->local;
+
+    if (local->locked)
+        cs_inodelk_unlock(frame);
+
+    CS_STACK_UNWIND(setxattr, frame, op_ret, op_errno, xdata);
+
+    return 0;
+}
+
+int32_t
+cs_setxattr(call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *dict,
+            int32_t flags, dict_t *xdata)
+{
+    data_t *tmp = NULL;
+    cs_local_t *local = NULL;
+    int ret = 0;
+
+    VALIDATE_OR_GOTO(frame, err);
+    VALIDATE_OR_GOTO(this, err);
+
+    local = cs_local_init(this, frame, loc, NULL, GF_FOP_SETXATTR);
+    if (!local) {
+        ret = -1;
+        goto err;
+    }
+
+    local->xattr_req = xdata ? dict_ref(xdata) : (xdata = dict_new());
+
+    tmp = dict_get_sizen(dict, GF_CS_OBJECT_UPLOAD_COMPLETE);
+    if (tmp) {
+        /* Value of key should be the atime */
+        local->stub = fop_setxattr_stub(frame, cs_resume_setxattr, loc, dict,
+                                        flags, xdata);
+
+        if (!local->stub)
+            goto err;
+
+        ret = locate_and_execute(frame);
+        if (ret) {
+            goto err;
+        }
+
+        return 0;
+    }
+
+    STACK_WIND(frame, cs_setxattr_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->setxattr, loc, dict, flags, xdata);
+    return 0;
+err:
+    CS_STACK_UNWIND(setxattr, frame, -1, errno, NULL);
+    return 0;
+}
+
+int32_t
+cs_fgetxattr_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                 int32_t op_ret, int32_t op_errno, dict_t *dict, dict_t *xdata)
+{
+    STACK_UNWIND_STRICT(fgetxattr, frame, op_ret, op_errno, dict, xdata);
+    return 0;
+}
+
+int32_t
+cs_fgetxattr(call_frame_t *frame, xlator_t *this, fd_t *fd, const char *name,
+             dict_t *xdata)
+{
+    STACK_WIND(frame, cs_fgetxattr_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->fgetxattr, fd, name, xdata);
+    return 0;
+}
+
+int32_t
+cs_fsetxattr_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                 int32_t op_ret, int32_t op_errno, dict_t *xdata)
+{
+    STACK_UNWIND_STRICT(fsetxattr, frame, op_ret, op_errno, xdata);
+    return 0;
+}
+
+int32_t
+cs_fsetxattr(call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *dict,
+             int32_t flags, dict_t *xdata)
+{
+    STACK_WIND(frame, cs_fsetxattr_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->fsetxattr, fd, dict, flags, xdata);
+    return 0;
+}
+
+int32_t
+cs_unlink_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int32_t op_ret,
+              int32_t op_errno, struct iatt *preparent, struct iatt *postparent,
+              dict_t *xdata)
+{
+    STACK_UNWIND_STRICT(unlink, frame, op_ret, op_errno, preparent, postparent,
+                        xdata);
+    return 0;
+}
+
+int32_t
+cs_unlink(call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t flags,
+          dict_t *xattr_req)
+{
+    cs_local_t *local = NULL;
+    int ret = 0;
+
+    local = cs_local_init(this, frame, loc, NULL, GF_FOP_UNLINK);
+    if (!local)
+        goto err;
+
+    local->xattr_req = xattr_req ? dict_ref(xattr_req) : dict_new();
+
+    ret = dict_set_uint32(local->xattr_req, GF_CS_OBJECT_STATUS, 1);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, 0,
+               "dict_set failed key:"
+               " %s",
+               GF_CS_OBJECT_STATUS);
+        goto err;
+    }
+    STACK_WIND(frame, cs_unlink_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->unlink, loc, flags, local->xattr_req);
+    return 0;
+err:
+    CS_STACK_UNWIND(unlink, frame, -1, errno, NULL, NULL, NULL);
+    return 0;
+}
+
+int32_t
+cs_open_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int32_t op_ret,
+            int32_t op_errno, fd_t *fd, dict_t *xdata)
+{
+    int ret = 0;
+    uint64_t val = 0;
+
+    if (op_ret == 0) {
+        ret = dict_get_uint64(xdata, GF_CS_OBJECT_STATUS, &val);
+        if (!ret) {
+            ret = __cs_inode_ctx_update(this, fd->inode, val);
+            if (ret) {
+                gf_msg(this->name, GF_LOG_ERROR, 0, 0, "ctx update failed");
+            }
+        }
+    } else {
+        cs_inode_ctx_reset(this, fd->inode);
+    }
+
+    CS_STACK_UNWIND(open, frame, op_ret, op_errno, fd, xdata);
+    return 0;
+}
+
+int32_t
+cs_open(call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t flags,
+        fd_t *fd, dict_t *xattr_req)
+{
+    cs_local_t *local = NULL;
+    int ret = 0;
+
+    local = cs_local_init(this, frame, NULL, fd, GF_FOP_OPEN);
+    if (!local)
+        goto err;
+
+    local->xattr_req = xattr_req ? dict_ref(xattr_req) : dict_new();
+
+    ret = dict_set_uint32(local->xattr_req, GF_CS_OBJECT_STATUS, 1);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, 0,
+               "dict_set failed key:"
+               " %s",
+               GF_CS_OBJECT_STATUS);
+        goto err;
+    }
+
+    STACK_WIND(frame, cs_open_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->open, loc, flags, fd, local->xattr_req);
+    return 0;
+err:
+    CS_STACK_UNWIND(open, frame, -1, errno, NULL, NULL);
+    return 0;
+}
+
+int32_t
+cs_fstat_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int32_t op_ret,
+             int32_t op_errno, struct iatt *buf, dict_t *xdata)
+{
+    int ret = 0;
+    uint64_t val = 0;
+    fd_t *fd = NULL;
+    cs_local_t *local = NULL;
+
+    local = frame->local;
+
+    fd = local->fd;
+
+    if (op_ret == 0) {
+        ret = dict_get_uint64(xdata, GF_CS_OBJECT_STATUS, &val);
+        if (!ret) {
+            gf_msg_debug(this->name, 0, "state %" PRIu64, val);
+            ret = __cs_inode_ctx_update(this, fd->inode, val);
+            if (ret) {
+                gf_msg(this->name, GF_LOG_ERROR, 0, 0, "ctx update failed");
+            }
+        }
+    } else {
+        cs_inode_ctx_reset(this, fd->inode);
+    }
+
+    CS_STACK_UNWIND(fstat, frame, op_ret, op_errno, buf, xdata);
+
+    return 0;
+}
+
+int32_t
+cs_fstat(call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *xattr_req)
+{
+    cs_local_t *local = NULL;
+    int ret = 0;
+
+    local = cs_local_init(this, frame, NULL, fd, GF_FOP_FSTAT);
+    if (!local)
+        goto err;
+
+    if (fd->inode->ia_type == IA_IFDIR)
+        goto wind;
+
+    local->xattr_req = xattr_req ? dict_ref(xattr_req) : dict_new();
+
+    ret = dict_set_uint32(local->xattr_req, GF_CS_OBJECT_STATUS, 1);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, 0,
+               "dict_set failed key:"
+               " %s",
+               GF_CS_OBJECT_STATUS);
+        goto err;
+    }
+
+wind:
+    STACK_WIND(frame, cs_fstat_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->fstat, fd, local->xattr_req);
+    return 0;
+err:
+    CS_STACK_UNWIND(fstat, frame, -1, errno, NULL, NULL);
+    return 0;
+}
+
+cs_local_t *
+cs_local_init(xlator_t *this, call_frame_t *frame, loc_t *loc, fd_t *fd,
+              glusterfs_fop_t fop)
+{
+    cs_local_t *local = NULL;
+    int ret = 0;
+
+    local = mem_get0(this->local_pool);
+    if (!local)
+        goto out;
+
+    if (loc) {
+        ret = loc_copy(&local->loc, loc);
+        if (ret)
+            goto out;
+    }
+
+    if (fd) {
+        local->fd = fd_ref(fd);
+    }
+
+    local->op_ret = -1;
+    local->op_errno = EUCLEAN;
+    local->fop = fop;
+    local->dloffset = 0;
+    frame->local = local;
+    local->locked = _gf_false;
+    local->call_cnt = 0;
+out:
+    if (ret) {
+        if (local)
+            mem_put(local);
+        local = NULL;
+    }
+
+    return local;
+}
+
+call_frame_t *
+cs_lock_frame(call_frame_t *parent_frame)
+{
+    call_frame_t *lock_frame = NULL;
+
+    lock_frame = copy_frame(parent_frame);
+
+    if (lock_frame == NULL)
+        goto out;
+
+    set_lk_owner_from_ptr(&lock_frame->root->lk_owner, parent_frame->root);
+
+out:
+    return lock_frame;
+}
+
+void
+cs_lock_wipe(call_frame_t *lock_frame)
+{
+    CS_STACK_DESTROY(lock_frame);
+}
+
+int32_t
+cs_inodelk_unlock_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                      int32_t op_ret, int32_t op_errno, dict_t *xdata)
+{
+    cs_lock_wipe(frame);
+
+    return 0;
+}
+
+int
+cs_inodelk_unlock(call_frame_t *main_frame)
+{
+    xlator_t *this = NULL;
+    struct gf_flock flock = {
+        0,
+    };
+    call_frame_t *lock_frame = NULL;
+    cs_local_t *lock_local = NULL;
+    cs_local_t *main_local = NULL;
+    int ret = 0;
+
+    this = main_frame->this;
+    main_local = main_frame->local;
+
+    lock_frame = cs_lock_frame(main_frame);
+    if (!lock_frame)
+        goto out;
+
+    lock_local = cs_local_init(this, lock_frame, NULL, NULL, 0);
+    if (!lock_local)
+        goto out;
+
+    ret = cs_build_loc(&lock_local->loc, main_frame);
+    if (ret) {
+        goto out;
+    }
+
+    flock.l_type = F_UNLCK;
+
+    main_local->locked = _gf_false;
+
+    STACK_WIND(lock_frame, cs_inodelk_unlock_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->inodelk, CS_LOCK_DOMAIN,
+               &lock_local->loc, F_SETLKW, &flock, NULL);
+
+    return 0;
+
+out:
+    gf_msg(this->name, GF_LOG_ERROR, 0, 0,
+           "Stale lock would be found on"
+           " server");
+
+    if (lock_frame)
+        cs_lock_wipe(lock_frame);
+
+    return 0;
+}
+
+int
+cs_download_task(void *arg)
+{
+    call_frame_t *frame = NULL;
+    xlator_t *this = NULL;
+    cs_private_t *priv = NULL;
+    int ret = -1;
+    char *sign_req = NULL;
+    fd_t *fd = NULL;
+    cs_local_t *local = NULL;
+    dict_t *dict = NULL;
+
+    frame = (call_frame_t *)arg;
+
+    this = frame->this;
+
+    priv = this->private;
+
+    if (!priv->stores) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, 0,
+               "No remote store "
+               "plugins found");
+        ret = -1;
+        goto out;
+    }
+
+    local = frame->local;
+
+    if (local->fd)
+        fd = fd_anonymous(local->fd->inode);
+    else
+        fd = fd_anonymous(local->loc.inode);
+
+    if (!fd) {
+        gf_msg("CS", GF_LOG_ERROR, 0, 0, "fd creation failed");
+        ret = -1;
+        goto out;
+    }
+
+    local->dlfd = fd;
+    local->dloffset = 0;
+
+    dict = dict_new();
+    if (!dict) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, ENOMEM,
+               "failed to create "
+               "dict");
+        ret = -1;
+        goto out;
+    }
+
+    ret = dict_set_uint32(dict, GF_CS_OBJECT_DOWNLOADING, 1);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, 0, "dict_set failed");
+        ret = -1;
+        goto out;
+    }
+
+    ret = syncop_fsetxattr(this, local->fd, dict, 0, NULL, NULL);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, 0,
+               "fsetxattr failed "
+               "key %s",
+               GF_CS_OBJECT_DOWNLOADING);
+        ret = -1;
+        goto out;
+    }
+    /*this calling method is for per volume setting */
+    ret = priv->stores->dlfop(frame, priv->stores->config);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, 0,
+               "download failed"
+               ", remotepath: %s",
+               local->remotepath);
+
+        /*using dlfd as it is anonymous and have RDWR flag*/
+        ret = syncop_ftruncate(FIRST_CHILD(this), local->dlfd, 0, NULL, NULL,
+                               NULL, NULL);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, -ret, "ftruncate failed");
+        } else {
+            gf_msg_debug(this->name, 0, "ftruncate succeed");
+        }
+
+        ret = -1;
+        goto out;
+    } else {
+        gf_msg(this->name, GF_LOG_INFO, 0, 0,
+               "download success, path"
+               " : %s",
+               local->remotepath);
+
+        ret = syncop_fremovexattr(this, local->fd, GF_CS_OBJECT_REMOTE, NULL,
+                                  NULL);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, -ret,
+                   "removexattr failed, remotexattr");
+            ret = -1;
+            goto out;
+        } else {
+            gf_msg_debug(this->name, 0,
+                         "fremovexattr success, "
+                         "path : %s",
+                         local->remotepath);
+        }
+
+        ret = syncop_fremovexattr(this, local->fd, GF_CS_OBJECT_DOWNLOADING,
+                                  NULL, NULL);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, -ret,
+                   "removexattr failed, downloading xattr, path %s",
+                   local->remotepath);
+            ret = -1;
+            goto out;
+        } else {
+            gf_msg_debug(this->name, 0,
+                         "fremovexattr success"
+                         " path  %s",
+                         local->remotepath);
+        }
+    }
+
+out:
+    GF_FREE(sign_req);
+
+    if (dict)
+        dict_unref(dict);
+
+    if (fd) {
+        fd_unref(fd);
+        local->dlfd = NULL;
+    }
+
+    return ret;
+}
+
+int
+cs_download(call_frame_t *frame)
+{
+    int ret = 0;
+    cs_local_t *local = NULL;
+    xlator_t *this = NULL;
+
+    local = frame->local;
+    this = frame->this;
+
+    if (!local->remotepath) {
+        ret = -1;
+        gf_msg(this->name, GF_LOG_ERROR, 0, 0,
+               "remote path not"
+               " available. Check posix logs to resolve");
+        goto out;
+    }
+
+    ret = cs_download_task((void *)frame);
+out:
+    return ret;
+}
+
+int
+cs_set_xattr_req(call_frame_t *frame)
+{
+    cs_local_t *local = NULL;
+    GF_UNUSED int ret = 0;
+
+    local = frame->local;
+
+    /* When remote reads are performed (i.e. reads on remote store),
+     * there needs to be a way to associate a file on gluster volume
+     * with its correspnding file on the remote store. In order to do
+     * that, a unique key can be maintained as an xattr
+     * (GF_CS_XATTR_ARCHIVE_UUID)on the stub file on gluster bricks.
+     * This xattr should be provided to the plugin to
+     * perform the read fop on the correct file. This assumes that the file
+     * hierarchy and name need not be the same on remote store as that of
+     * the gluster volume.
+     */
+    ret = dict_set_sizen_str_sizen(local->xattr_req, GF_CS_XATTR_ARCHIVE_UUID,
+                                   "1");
+
+    return 0;
+}
+
+int
+cs_update_xattrs(call_frame_t *frame, dict_t *xdata)
+{
+    cs_local_t *local = NULL;
+    xlator_t *this = NULL;
+    int size = -1;
+    GF_UNUSED int ret = 0;
+
+    local = frame->local;
+    this = frame->this;
+
+    local->xattrinfo.lxattr = GF_CALLOC(1, sizeof(cs_loc_xattr_t),
+                                        gf_cs_mt_cs_lxattr_t);
+    if (!local->xattrinfo.lxattr) {
+        local->op_ret = -1;
+        local->op_errno = ENOMEM;
+        goto err;
+    }
+
+    gf_uuid_copy(local->xattrinfo.lxattr->gfid, local->loc.gfid);
+
+    if (local->remotepath) {
+        local->xattrinfo.lxattr->file_path = gf_strdup(local->remotepath);
+        if (!local->xattrinfo.lxattr->file_path) {
+            local->op_ret = -1;
+            local->op_errno = ENOMEM;
+            goto err;
+        }
+    }
+
+    ret = dict_get_gfuuid(xdata, GF_CS_XATTR_ARCHIVE_UUID,
+                          &(local->xattrinfo.lxattr->uuid));
+
+    if (ret) {
+        gf_uuid_clear(local->xattrinfo.lxattr->uuid);
+    }
+    size = strlen(this->name) - strlen("-cloudsync") + 1;
+    local->xattrinfo.lxattr->volname = GF_CALLOC(1, size, gf_common_mt_char);
+    if (!local->xattrinfo.lxattr->volname) {
+        local->op_ret = -1;
+        local->op_errno = ENOMEM;
+        goto err;
+    }
+    strncpy(local->xattrinfo.lxattr->volname, this->name, size - 1);
+    local->xattrinfo.lxattr->volname[size - 1] = '\0';
+
+    return 0;
+err:
+    cs_xattrinfo_wipe(local);
+    return -1;
+}
+
+int
+cs_serve_readv(call_frame_t *frame, off_t offset, size_t size, uint32_t flags)
+{
+    xlator_t *this = NULL;
+    cs_private_t *priv = NULL;
+    int ret = -1;
+    fd_t *fd = NULL;
+    cs_local_t *local = NULL;
+
+    local = frame->local;
+    this = frame->this;
+    priv = this->private;
+
+    if (!local->remotepath) {
+        ret = -1;
+        gf_msg(this->name, GF_LOG_ERROR, 0, 0,
+               "remote path not"
+               " available. Check posix logs to resolve");
+        goto out;
+    }
+
+    if (!priv->stores) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, 0,
+               "No remote store "
+               "plugins found");
+        ret = -1;
+        goto out;
+    }
+
+    if (local->fd) {
+        fd = fd_anonymous(local->fd->inode);
+    } else {
+        fd = fd_anonymous(local->loc.inode);
+    }
+
+    local->xattrinfo.size = size;
+    local->xattrinfo.offset = offset;
+    local->xattrinfo.flags = flags;
+
+    if (!fd) {
+        gf_msg("CS", GF_LOG_ERROR, 0, 0, "fd creation failed");
+        ret = -1;
+        goto out;
+    }
+
+    local->dlfd = fd;
+    local->dloffset = offset;
+
+    /*this calling method is for per volume setting */
+    ret = priv->stores->rdfop(frame, priv->stores->config);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, 0,
+               "read failed"
+               ", remotepath: %s",
+               local->remotepath);
+        ret = -1;
+        goto out;
+    } else {
+        gf_msg(this->name, GF_LOG_INFO, 0, 0,
+               "read success, path"
+               " : %s",
+               local->remotepath);
+    }
+
+out:
+    if (fd) {
+        fd_unref(fd);
+        local->dlfd = NULL;
+    }
+    return ret;
+}
+
+int32_t
+cs_readv_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int32_t op_ret,
+             int32_t op_errno, struct iovec *vector, int32_t count,
+             struct iatt *stbuf, struct iobref *iobref, dict_t *xdata)
+{
+    cs_local_t *local = NULL;
+    int ret = 0;
+    uint64_t val = 0;
+    fd_t *fd = NULL;
+
+    local = frame->local;
+    fd = local->fd;
+
+    local->call_cnt++;
+
+    if (op_ret == -1) {
+        ret = dict_get_uint64(xdata, GF_CS_OBJECT_STATUS, &val);
+        if (ret == 0) {
+            if (val == GF_CS_ERROR) {
+                gf_msg(this->name, GF_LOG_ERROR, 0, 0,
+                       "could not get file state, unwinding");
+                op_ret = -1;
+                op_errno = EIO;
+                goto unwind;
+            } else {
+                __cs_inode_ctx_update(this, fd->inode, val);
+                gf_msg(this->name, GF_LOG_INFO, 0, 0, " state = %" PRIu64, val);
+
+                if (local->call_cnt == 1 &&
+                    (val == GF_CS_REMOTE || val == GF_CS_DOWNLOADING)) {
+                    gf_msg(this->name, GF_LOG_INFO, 0, 0,
+                           " will read from remote : %" PRIu64, val);
+                    goto repair;
+                } else {
+                    gf_msg(this->name, GF_LOG_ERROR, 0, 0,
+                           "second readv, Unwinding");
+                    goto unwind;
+                }
+            }
+        } else {
+            gf_msg(this->name, GF_LOG_ERROR, 0, 0,
+                   "file state "
+                   "could not be figured, unwinding");
+            goto unwind;
+        }
+    } else {
+        /* successful readv => file is local */
+        __cs_inode_ctx_update(this, fd->inode, GF_CS_LOCAL);
+        gf_msg(this->name, GF_LOG_INFO, 0, 0,
+               "state : GF_CS_LOCAL"
+               ", readv successful");
+
+        goto unwind;
+    }
+
+repair:
+    ret = locate_and_execute(frame);
+    if (ret) {
+        goto unwind;
+    }
+
+    return 0;
+
+unwind:
+    CS_STACK_UNWIND(readv, frame, op_ret, op_errno, vector, count, stbuf,
+                    iobref, xdata);
+
+    return 0;
+}
+
+int32_t
+cs_resume_readv(call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
+                off_t offset, uint32_t flags, dict_t *xdata)
+{
+    int ret = 0;
+
+    ret = cs_resume_postprocess(this, frame, fd->inode);
+    if (ret) {
+        goto unwind;
+    }
+
+    cs_inodelk_unlock(frame);
+
+    STACK_WIND(frame, cs_readv_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->readv, fd, size, offset, flags, xdata);
+
+    return 0;
+
+unwind:
+    cs_inodelk_unlock(frame);
+
+    cs_common_cbk(frame);
+
+    return 0;
+}
+
+int32_t
+cs_resume_remote_readv(call_frame_t *frame, xlator_t *this, fd_t *fd,
+                       size_t size, off_t offset, uint32_t flags, dict_t *xdata)
+{
+    int ret = 0;
+    cs_local_t *local = NULL;
+    gf_cs_obj_state state = -1;
+    cs_inode_ctx_t *ctx = NULL;
+
+    cs_inodelk_unlock(frame);
+
+    local = frame->local;
+    if (!local) {
+        ret = -1;
+        goto unwind;
+    }
+
+    __cs_inode_ctx_get(this, fd->inode, &ctx);
+
+    state = __cs_get_file_state(fd->inode, ctx);
+    if (state == GF_CS_ERROR) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, 0,
+               "status is GF_CS_ERROR."
+               " Aborting readv");
+        local->op_ret = -1;
+        local->op_errno = EREMOTE;
+        ret = -1;
+        goto unwind;
+    }
+
+    /* Serve readv from remote store only if it is remote. */
+    gf_msg_debug(this->name, 0, "status of file %s is %d",
+                 local->remotepath ? local->remotepath : "", state);
+
+    /* We will reach this condition if local inode ctx had REMOTE
+     * state when the control was in cs_readv but after stat
+     * we got an updated state saying that the file is LOCAL.
+     */
+    if (state == GF_CS_LOCAL) {
+        STACK_WIND(frame, cs_readv_cbk, FIRST_CHILD(this),
+                   FIRST_CHILD(this)->fops->readv, fd, size, offset, flags,
+                   xdata);
+    } else if (state == GF_CS_REMOTE) {
+        ret = cs_resume_remote_readv_postprocess(this, frame, fd->inode, offset,
+                                                 size, flags);
+        /* Failed to submit the remote readv fop to plugin */
+        if (ret) {
+            local->op_ret = -1;
+            local->op_errno = EREMOTE;
+            goto unwind;
+        }
+        /* When the file is in any other intermediate state,
+         * we should not perform remote reads.
+         */
+    } else {
+        local->op_ret = -1;
+        local->op_errno = EINVAL;
+        goto unwind;
+    }
+
+    return 0;
+
+unwind:
+    cs_common_cbk(frame);
+
+    return 0;
+}
+
+int32_t
+cs_readv(call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
+         off_t offset, uint32_t flags, dict_t *xdata)
+{
+    int op_errno = ENOMEM;
+    cs_local_t *local = NULL;
+    int ret = 0;
+    cs_inode_ctx_t *ctx = NULL;
+    gf_cs_obj_state state = -1;
+    cs_private_t *priv = NULL;
+
+    VALIDATE_OR_GOTO(frame, err);
+    VALIDATE_OR_GOTO(this, err);
+    VALIDATE_OR_GOTO(fd, err);
+
+    priv = this->private;
+
+    local = cs_local_init(this, frame, NULL, fd, GF_FOP_READ);
+    if (!local) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, 0, "local init failed");
+        goto err;
+    }
+
+    __cs_inode_ctx_get(this, fd->inode, &ctx);
+
+    if (ctx)
+        state = __cs_get_file_state(fd->inode, ctx);
+    else
+        state = GF_CS_LOCAL;
+
+    local->xattr_req = xdata ? dict_ref(xdata) : (xdata = dict_new());
+
+    ret = dict_set_uint32(local->xattr_req, GF_CS_OBJECT_STATUS, 1);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, 0,
+               "dict_set failed key:"
+               " %s",
+               GF_CS_OBJECT_STATUS);
+        goto err;
+    }
+
+    if (priv->remote_read) {
+        local->stub = fop_readv_stub(frame, cs_resume_remote_readv, fd, size,
+                                     offset, flags, xdata);
+    } else {
+        local->stub = fop_readv_stub(frame, cs_resume_readv, fd, size, offset,
+                                     flags, xdata);
+    }
+    if (!local->stub) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, 0, "insufficient memory");
+        goto err;
+    }
+
+    if (state == GF_CS_LOCAL) {
+        STACK_WIND(frame, cs_readv_cbk, FIRST_CHILD(this),
+                   FIRST_CHILD(this)->fops->readv, fd, size, offset, flags,
+                   xdata);
+    } else {
+        local->call_cnt++;
+        ret = locate_and_execute(frame);
+        if (ret) {
+            goto err;
+        }
+    }
+
+    return 0;
+
+err:
+    CS_STACK_UNWIND(readv, frame, -1, op_errno, NULL, -1, NULL, NULL, NULL);
+
+    return 0;
+}
+
+int
+cs_resume_remote_readv_postprocess(xlator_t *this, call_frame_t *frame,
+                                   inode_t *inode, off_t offset, size_t size,
+                                   uint32_t flags)
+{
+    int ret = 0;
+
+    ret = cs_serve_readv(frame, offset, size, flags);
+
+    return ret;
+}
+
+int
+cs_stat_check_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int op_ret,
+                  int op_errno, struct iatt *stbuf, dict_t *xdata)
+{
+    cs_local_t *local = NULL;
+    call_stub_t *stub = NULL;
+    char *filepath = NULL;
+    int ret = 0;
+    inode_t *inode = NULL;
+    uint64_t val = 0;
+
+    local = frame->local;
+
+    if (op_ret == -1) {
+        local->op_ret = op_ret;
+        local->op_errno = op_errno;
+        gf_msg(this->name, GF_LOG_ERROR, 0, op_errno, "stat check failed");
+        goto err;
+    } else {
+        if (local->fd)
+            inode = local->fd->inode;
+        else
+            inode = local->loc.inode;
+
+        if (!inode) {
+            local->op_ret = -1;
+            local->op_errno = EINVAL;
+            gf_msg(this->name, GF_LOG_ERROR, 0, 0,
+                   "null inode "
+                   "returned");
+            goto err;
+        }
+
+        ret = dict_get_uint64(xdata, GF_CS_OBJECT_STATUS, &val);
+        if (ret == 0) {
+            if (val == GF_CS_ERROR) {
+                cs_inode_ctx_reset(this, inode);
+                local->op_ret = -1;
+                local->op_errno = EIO;
+                gf_msg(this->name, GF_LOG_ERROR, 0, 0,
+                       "status = GF_CS_ERROR. failed to get "
+                       " file state");
+                goto err;
+            } else {
+                ret = __cs_inode_ctx_update(this, inode, val);
+                gf_msg_debug(this->name, 0, "status : %" PRIu64, val);
+                if (ret) {
+                    gf_msg(this->name, GF_LOG_ERROR, 0, 0, "ctx update failed");
+                    local->op_ret = -1;
+                    local->op_errno = ENOMEM;
+                    goto err;
+                }
+            }
+        } else {
+            gf_msg_debug(this->name, 0, "status not found in dict");
+            local->op_ret = -1;
+            local->op_errno = ENOMEM;
+            goto err;
+        }
+
+        ret = dict_get_str_sizen(xdata, GF_CS_OBJECT_REMOTE, &filepath);
+        if (filepath) {
+            gf_msg_debug(this->name, 0, "filepath returned %s", filepath);
+            local->remotepath = gf_strdup(filepath);
+            if (!local->remotepath) {
+                local->op_ret = -1;
+                local->op_errno = ENOMEM;
+                goto err;
+            }
+        } else {
+            gf_msg_debug(this->name, 0, "NULL filepath");
+        }
+
+        ret = cs_update_xattrs(frame, xdata);
+        if (ret)
+            goto err;
+
+        local->op_ret = 0;
+        local->xattr_rsp = dict_ref(xdata);
+        memcpy(&local->stbuf, stbuf, sizeof(struct iatt));
+    }
+
+    stub = local->stub;
+    local->stub = NULL;
+    call_resume(stub);
+
+    return 0;
+err:
+    cs_inodelk_unlock(frame);
+
+    cs_common_cbk(frame);
+
+    return 0;
+}
+
+int
+cs_do_stat_check(call_frame_t *main_frame)
+{
+    cs_local_t *local = NULL;
+    xlator_t *this = NULL;
+    int ret = 0;
+
+    local = main_frame->local;
+    this = main_frame->this;
+
+    ret = dict_set_uint32(local->xattr_req, GF_CS_OBJECT_REPAIR, 256);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, 0, "dict_set failed");
+        goto err;
+    }
+
+    cs_set_xattr_req(main_frame);
+
+    if (local->fd) {
+        STACK_WIND(main_frame, cs_stat_check_cbk, FIRST_CHILD(this),
+                   FIRST_CHILD(this)->fops->fstat, local->fd, local->xattr_req);
+    } else {
+        STACK_WIND(main_frame, cs_stat_check_cbk, FIRST_CHILD(this),
+                   FIRST_CHILD(this)->fops->stat, &local->loc,
+                   local->xattr_req);
+    }
+
+    return 0;
+
+err:
+    cs_inodelk_unlock(main_frame);
+
+    cs_common_cbk(main_frame);
+
+    return 0;
+}
+
+void
+cs_common_cbk(call_frame_t *frame)
+{
+    glusterfs_fop_t fop = -1;
+    cs_local_t *local = NULL;
+
+    local = frame->local;
+
+    fop = local->fop;
+
+    /*Note: Only the failure case needs to be handled here. Since for
+     * successful stat check the fop will resume anyway. The unwind can
+     * happen from the fop_cbk and each cbk can unlock the inodelk in case
+     * a lock was taken before. The lock status can be stored in frame */
+
+    /* for failure case  */
+
+    /*TODO: add other fops*/
+    switch (fop) {
+        case GF_FOP_WRITE:
+            CS_STACK_UNWIND(writev, frame, local->op_ret, local->op_errno, NULL,
+                            NULL, NULL);
+            break;
+
+        case GF_FOP_SETXATTR:
+            CS_STACK_UNWIND(setxattr, frame, local->op_ret, local->op_errno,
+                            NULL);
+            break;
+        case GF_FOP_READ:
+            CS_STACK_UNWIND(readv, frame, local->op_ret, local->op_errno, NULL,
+                            0, NULL, NULL, NULL);
+            break;
+        case GF_FOP_FTRUNCATE:
+            CS_STACK_UNWIND(ftruncate, frame, local->op_ret, local->op_errno,
+                            NULL, NULL, NULL);
+            break;
+
+        case GF_FOP_TRUNCATE:
+            CS_STACK_UNWIND(truncate, frame, local->op_ret, local->op_errno,
+                            NULL, NULL, NULL);
+            break;
+        default:
+            break;
+    }
+
+    return;
+}
+
+int
+cs_blocking_inodelk_cbk(call_frame_t *lock_frame, void *cookie, xlator_t *this,
+                        int32_t op_ret, int32_t op_errno, dict_t *xdata)
+{
+    cs_local_t *main_local = NULL;
+    call_frame_t *main_frame = NULL;
+    cs_local_t *lock_local = NULL;
+
+    lock_local = lock_frame->local;
+
+    main_frame = lock_local->main_frame;
+    main_local = main_frame->local;
+
+    if (op_ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, 0, "inodelk failed");
+        main_local->op_errno = op_errno;
+        main_local->op_ret = op_ret;
+        goto err;
+    }
+
+    main_local->locked = _gf_true;
+
+    cs_lock_wipe(lock_frame);
+
+    cs_do_stat_check(main_frame);
+
+    return 0;
+err:
+    cs_common_cbk(main_frame);
+
+    cs_lock_wipe(lock_frame);
+
+    return 0;
+}
+
+int
+cs_build_loc(loc_t *loc, call_frame_t *frame)
+{
+    cs_local_t *local = NULL;
+    int ret = -1;
+
+    local = frame->local;
+
+    if (local->fd) {
+        loc->inode = inode_ref(local->fd->inode);
+        if (loc->inode) {
+            gf_uuid_copy(loc->gfid, loc->inode->gfid);
+            ret = 0;
+            goto out;
+        } else {
+            ret = -1;
+            goto out;
+        }
+    } else {
+        loc->inode = inode_ref(local->loc.inode);
+        if (loc->inode) {
+            gf_uuid_copy(loc->gfid, loc->inode->gfid);
+            ret = 0;
+            goto out;
+        } else {
+            ret = -1;
+            goto out;
+        }
+    }
+out:
+    return ret;
+}
+
+int
+cs_blocking_inodelk(call_frame_t *parent_frame)
+{
+    call_frame_t *lock_frame = NULL;
+    cs_local_t *lock_local = NULL;
+    xlator_t *this = NULL;
+    struct gf_flock flock = {
+        0,
+    };
+    int ret = 0;
+
+    this = parent_frame->this;
+
+    lock_frame = cs_lock_frame(parent_frame);
+    if (!lock_frame) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, 0, "insuffcient memory");
+        goto err;
+    }
+
+    lock_local = cs_local_init(this, lock_frame, NULL, NULL, 0);
+    if (!lock_local) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, 0, "local init failed");
+        goto err;
+    }
+
+    lock_local->main_frame = parent_frame;
+
+    flock.l_type = F_WRLCK;
+
+    ret = cs_build_loc(&lock_local->loc, parent_frame);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, 0, "build_loc failed");
+        goto err;
+    }
+
+    STACK_WIND(lock_frame, cs_blocking_inodelk_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->inodelk, CS_LOCK_DOMAIN,
+               &lock_local->loc, F_SETLKW, &flock, NULL);
+
+    return 0;
+err:
+    if (lock_frame)
+        cs_lock_wipe(lock_frame);
+
+    return -1;
+}
+
+int
+locate_and_execute(call_frame_t *frame)
+{
+    int ret = 0;
+
+    ret = cs_blocking_inodelk(frame);
+
+    if (ret)
+        return -1;
+    else
+        return 0;
+}
+
+int32_t
+cs_resume_truncate(call_frame_t *frame, xlator_t *this, loc_t *loc,
+                   off_t offset, dict_t *xattr_req)
+{
+    cs_local_t *local = NULL;
+    int ret = 0;
+
+    local = frame->local;
+
+    ret = cs_resume_postprocess(this, frame, loc->inode);
+    if (ret) {
+        goto unwind;
+    }
+
+    cs_inodelk_unlock(frame);
+
+    STACK_WIND(frame, cs_truncate_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->truncate, loc, offset,
+               local->xattr_req);
+
+    return 0;
+
+unwind:
+    cs_inodelk_unlock(frame);
+
+    cs_common_cbk(frame);
+
+    return 0;
+}
+
+int32_t
+cs_resume_setxattr(call_frame_t *frame, xlator_t *this, loc_t *loc,
+                   dict_t *dict, int32_t flags, dict_t *xdata)
+{
+    cs_local_t *local = NULL;
+    cs_inode_ctx_t *ctx = NULL;
+    gf_cs_obj_state state = GF_CS_ERROR;
+
+    local = frame->local;
+
+    __cs_inode_ctx_get(this, loc->inode, &ctx);
+
+    state = __cs_get_file_state(loc->inode, ctx);
+
+    if (state == GF_CS_ERROR) {
+        /* file is already remote */
+        local->op_ret = -1;
+        local->op_errno = EINVAL;
+        gf_msg(this->name, GF_LOG_WARNING, 0, 0,
+               "file %s , could not figure file state", loc->path);
+        goto unwind;
+    }
+
+    if (state == GF_CS_REMOTE) {
+        /* file is already remote */
+        local->op_ret = -1;
+        local->op_errno = EINVAL;
+        gf_msg(this->name, GF_LOG_WARNING, 0, EINVAL,
+               "file %s is already remote", loc->path);
+        goto unwind;
+    }
+
+    if (state == GF_CS_DOWNLOADING) {
+        gf_msg(this->name, GF_LOG_WARNING, 0, 0,
+               " file is in downloading state.");
+        local->op_ret = -1;
+        local->op_errno = EINVAL;
+        goto unwind;
+    }
+
+    STACK_WIND(frame, cs_setxattr_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->setxattr, loc, dict, flags,
+               local->xattr_req);
+
+    return 0;
+unwind:
+    cs_inodelk_unlock(frame);
+
+    cs_common_cbk(frame);
+
+    return 0;
+}
+
+gf_cs_obj_state
+__cs_get_file_state(inode_t *inode, cs_inode_ctx_t *ctx)
+{
+    gf_cs_obj_state state = -1;
+
+    if (!ctx)
+        return GF_CS_ERROR;
+
+    LOCK(&inode->lock);
+    {
+        state = ctx->state;
+    }
+    UNLOCK(&inode->lock);
+
+    return state;
+}
+
+void
+__cs_inode_ctx_get(xlator_t *this, inode_t *inode, cs_inode_ctx_t **ctx)
+{
+    uint64_t ctxint = 0;
+    int ret = 0;
+
+    LOCK(&inode->lock);
+    {
+        ret = __inode_ctx_get(inode, this, &ctxint);
+    }
+    UNLOCK(&inode->lock);
+
+    if (ret)
+        *ctx = NULL;
+    else
+        *ctx = (cs_inode_ctx_t *)(uintptr_t)ctxint;
+
+    return;
+}
+
+int
+__cs_inode_ctx_update(xlator_t *this, inode_t *inode, uint64_t val)
+{
+    cs_inode_ctx_t *ctx = NULL;
+    uint64_t ctxint = 0;
+    int ret = 0;
+
+    LOCK(&inode->lock);
+    {
+        ret = __inode_ctx_get(inode, this, &ctxint);
+        if (ret) {
+            ctx = GF_CALLOC(1, sizeof(*ctx), gf_cs_mt_cs_inode_ctx_t);
+            if (!ctx) {
+                gf_msg(this->name, GF_LOG_ERROR, 0, 0, "ctx allocation failed");
+                ret = -1;
+                goto out;
+            }
+
+            ctx->state = val;
+
+            ctxint = (uint64_t)(uintptr_t)ctx;
+
+            ret = __inode_ctx_set(inode, this, &ctxint);
+            if (ret) {
+                GF_FREE(ctx);
+                goto out;
+            }
+        } else {
+            ctx = (cs_inode_ctx_t *)(uintptr_t)ctxint;
+
+            ctx->state = val;
+        }
+    }
+
+out:
+    UNLOCK(&inode->lock);
+
+    return ret;
+}
+
+int
+cs_inode_ctx_reset(xlator_t *this, inode_t *inode)
+{
+    cs_inode_ctx_t *ctx = NULL;
+    uint64_t ctxint = 0;
+
+    inode_ctx_del(inode, this, &ctxint);
+    if (!ctxint) {
+        return 0;
+    }
+
+    ctx = (cs_inode_ctx_t *)(uintptr_t)ctxint;
+
+    GF_FREE(ctx);
+    return 0;
+}
+
+int
+cs_resume_postprocess(xlator_t *this, call_frame_t *frame, inode_t *inode)
+{
+    cs_local_t *local = NULL;
+    gf_cs_obj_state state = -1;
+    cs_inode_ctx_t *ctx = NULL;
+    int ret = 0;
+
+    local = frame->local;
+    if (!local) {
+        ret = -1;
+        goto out;
+    }
+
+    __cs_inode_ctx_get(this, inode, &ctx);
+
+    state = __cs_get_file_state(inode, ctx);
+    if (state == GF_CS_ERROR) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, 0,
+               "status is GF_CS_ERROR."
+               " Aborting write");
+        local->op_ret = -1;
+        local->op_errno = EREMOTE;
+        ret = -1;
+        goto out;
+    }
+
+    if (state == GF_CS_REMOTE || state == GF_CS_DOWNLOADING) {
+        gf_msg_debug(this->name, 0, "status is %d", state);
+        ret = cs_download(frame);
+        if (ret == 0) {
+            gf_msg_debug(this->name, 0, "Winding for Final Write");
+        } else {
+            gf_msg(this->name, GF_LOG_ERROR, 0, 0,
+                   " download failed, unwinding writev");
+            local->op_ret = -1;
+            local->op_errno = EREMOTE;
+            ret = -1;
+        }
+    }
+out:
+    return ret;
+}
+
+int32_t
+cs_fdctx_to_dict(xlator_t *this, fd_t *fd, dict_t *dict)
+{
+    return 0;
+}
+
+int32_t
+cs_inode(xlator_t *this)
+{
+    return 0;
+}
+
+int32_t
+cs_inode_to_dict(xlator_t *this, dict_t *dict)
+{
+    return 0;
+}
+
+int32_t
+cs_history(xlator_t *this)
+{
+    return 0;
+}
+
+int32_t
+cs_fd(xlator_t *this)
+{
+    return 0;
+}
+
+int32_t
+cs_fd_to_dict(xlator_t *this, dict_t *dict)
+{
+    return 0;
+}
+
+int32_t
+cs_fdctx(xlator_t *this, fd_t *fd)
+{
+    return 0;
+}
+
+int32_t
+cs_inodectx(xlator_t *this, inode_t *ino)
+{
+    return 0;
+}
+
+int32_t
+cs_inodectx_to_dict(xlator_t *this, inode_t *ino, dict_t *dict)
+{
+    return 0;
+}
+
+int32_t
+cs_priv_to_dict(xlator_t *this, dict_t *dict, char *brickname)
+{
+    return 0;
+}
+
+int32_t
+cs_priv(xlator_t *this)
+{
+    return 0;
+}
+
+int
+cs_notify(xlator_t *this, int event, void *data, ...)
+{
+    return default_notify(this, event, data);
+}
+
+struct xlator_fops cs_fops = {
+    .stat = cs_stat,
+    .readdirp = cs_readdirp,
+    .truncate = cs_truncate,
+    .seek = cs_seek,
+    .statfs = cs_statfs,
+    .fallocate = cs_fallocate,
+    .discard = cs_discard,
+    .getxattr = cs_getxattr,
+    .writev = cs_writev,
+    .setxattr = cs_setxattr,
+    .fgetxattr = cs_fgetxattr,
+    .lookup = cs_lookup,
+    .fsetxattr = cs_fsetxattr,
+    .readv = cs_readv,
+    .ftruncate = cs_ftruncate,
+    .rchecksum = cs_rchecksum,
+    .unlink = cs_unlink,
+    .open = cs_open,
+    .fstat = cs_fstat,
+    .zerofill = cs_zerofill,
+};
+
+struct xlator_cbks cs_cbks = {
+    .forget = cs_forget,
+};
+
+struct xlator_dumpops cs_dumpops = {
+    .fdctx_to_dict = cs_fdctx_to_dict,
+    .inode = cs_inode,
+    .inode_to_dict = cs_inode_to_dict,
+    .history = cs_history,
+    .fd = cs_fd,
+    .fd_to_dict = cs_fd_to_dict,
+    .fdctx = cs_fdctx,
+    .inodectx = cs_inodectx,
+    .inodectx_to_dict = cs_inodectx_to_dict,
+    .priv_to_dict = cs_priv_to_dict,
+    .priv = cs_priv,
+};
+
+struct volume_options cs_options[] = {
+    {.key = {"cloudsync-storetype"},
+     .type = GF_OPTION_TYPE_STR,
+     .description = "Defines which remote store is enabled"},
+    {.key = {"cloudsync-remote-read"},
+     .type = GF_OPTION_TYPE_BOOL,
+     .description = "Defines a remote read fop when on"},
+    {.key = {"cloudsync-store-id"},
+     .type = GF_OPTION_TYPE_STR,
+     .description = "Defines a volume wide store id"},
+    {.key = {"cloudsync-product-id"},
+     .type = GF_OPTION_TYPE_STR,
+     .description = "Defines a volume wide product id"},
+    {.key = {NULL}},
+};
+
+xlator_api_t xlator_api = {
+    .init = cs_init,
+    .fini = cs_fini,
+    .notify = cs_notify,
+    .reconfigure = cs_reconfigure,
+    .mem_acct_init = cs_mem_acct_init,
+    .dumpops = &cs_dumpops,
+    .fops = &cs_fops,
+    .cbks = &cs_cbks,
+    .options = cs_options,
+    .identifier = "cloudsync",
+    .category = GF_TECH_PREVIEW,
+};
diff --git a/xlators/features/cloudsync/src/cloudsync.h b/xlators/features/cloudsync/src/cloudsync.h
new file mode 100644
index 00000000000..d24141978d6
--- /dev/null
+++ b/xlators/features/cloudsync/src/cloudsync.h
@@ -0,0 +1,123 @@
+/*
+ *   Copyright (c) 2018 Red Hat, Inc. <http://www.redhat.com>
+ *   This file is part of GlusterFS.
+ *
+ *   This file is licensed to you under your choice of the GNU Lesser
+ *   General Public License, version 3 or any later version (LGPLv3 or
+ *   later), or the GNU General Public License, version 2 (GPLv2), in all
+ *   cases as published by the Free Software Foundation.
+ */
+
+#ifndef __CLOUDSYNC_H__
+#define __CLOUDSYNC_H__
+
+#include <glusterfs/glusterfs.h>
+#include <glusterfs/xlator.h>
+#include <glusterfs/defaults.h>
+#include <glusterfs/syncop.h>
+#include <glusterfs/call-stub.h>
+#include "cloudsync-common.h"
+#include "cloudsync-autogen-fops.h"
+
+#define ALIGN_SIZE 4096
+#define CS_LOCK_DOMAIN "cs.protect.file.stat"
+typedef struct cs_dlstore {
+    off_t off;
+    struct iovec *vector;
+    int32_t count;
+    struct iobref *iobref;
+    uint32_t flags;
+} cs_dlstore;
+
+typedef struct cs_inode_ctx {
+    cs_loc_xattr_t locxattr;
+    gf_cs_obj_state state;
+} cs_inode_ctx_t;
+
+struct cs_plugin {
+    char *name;        /* store name */
+    char *library;     /* library to load for the given store */
+    char *description; /* description about the store */
+};
+
+cs_local_t *
+cs_local_init(xlator_t *this, call_frame_t *frame, loc_t *loc, fd_t *fd,
+              glusterfs_fop_t fop);
+
+int
+locate_and_execute(call_frame_t *frame);
+
+int32_t
+cs_resume_setxattr(call_frame_t *frame, xlator_t *this, loc_t *loc,
+                   dict_t *dict, int32_t flags, dict_t *xdata);
+
+int32_t
+cs_inodelk_unlock_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                      int32_t op_ret, int32_t op_errno, dict_t *xdata);
+
+size_t
+cs_write_callback(void *lcurlbuf, size_t size, size_t nitems, void *frame);
+
+void
+cs_common_cbk(call_frame_t *frame);
+
+gf_boolean_t
+cs_is_file_remote(struct iatt *stbuf, dict_t *xattr);
+
+int32_t
+cs_setxattr_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                int32_t op_ret, int32_t op_errno, dict_t *xdata);
+int
+cs_build_loc(loc_t *loc, call_frame_t *frame);
+
+int
+cs_blocking_inodelk_cbk(call_frame_t *lock_frame, void *cookie, xlator_t *this,
+                        int32_t op_ret, int32_t op_errno, dict_t *xdata);
+
+int
+cs_read_authinfo(xlator_t *this);
+
+int
+__cs_inode_ctx_update(xlator_t *this, inode_t *inode, uint64_t val);
+
+int
+cs_inode_ctx_reset(xlator_t *this, inode_t *inode);
+
+void
+__cs_inode_ctx_get(xlator_t *this, inode_t *inode, cs_inode_ctx_t **ctx);
+
+gf_cs_obj_state
+__cs_get_file_state(inode_t *inode, cs_inode_ctx_t *ctx);
+
+int
+cs_inodelk_unlock(call_frame_t *main_frame);
+
+int
+cs_resume_postprocess(xlator_t *this, call_frame_t *frame, inode_t *inode);
+
+int32_t
+cs_truncate_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
+                struct iatt *postbuf, dict_t *xdata);
+int32_t
+cs_resume_truncate(call_frame_t *frame, xlator_t *this, loc_t *loc,
+                   off_t offset, dict_t *xattr_req);
+
+int32_t
+cs_readv_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int32_t op_ret,
+             int32_t op_errno, struct iovec *vector, int32_t count,
+             struct iatt *stbuf, struct iobref *iobref, dict_t *xdata);
+int32_t
+cs_resume_readv(call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
+                off_t offset, uint32_t flags, dict_t *xdata);
+int32_t
+cs_readv(call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
+         off_t offset, uint32_t flags, dict_t *xdata);
+
+int
+cs_resume_remote_readv_postprocess(xlator_t *this, call_frame_t *frame,
+                                   inode_t *inode, off_t offset, size_t size,
+                                   uint32_t flags);
+int
+cs_serve_readv(call_frame_t *frame, off_t offset, size_t size, uint32_t flags);
+#endif /* __CLOUDSYNC_H__ */
diff --git a/xlators/features/compress/src/Makefile.am b/xlators/features/compress/src/Makefile.am
index 2f79e66a82d..98271a9f3fc 100644
--- a/xlators/features/compress/src/Makefile.am
+++ b/xlators/features/compress/src/Makefile.am
@@ -4,13 +4,15 @@ xlatordir = $(libdir)/glusterfs/$(PACKAGE_VERSION)/xlator/features
 
 noinst_HEADERS = cdc.h cdc-mem-types.h
 
-cdc_la_LDFLAGS = -module -avoid-version
+cdc_la_LDFLAGS = -module $(GF_XLATOR_DEFAULT_LDFLAGS)
 
 cdc_la_SOURCES = cdc.c cdc-helper.c
 cdc_la_LIBADD = $(top_builddir)/libglusterfs/src/libglusterfs.la $(ZLIB_LIBS)
 
-AM_CPPFLAGS = $(GF_CPPFLAGS) -I$(top_srcdir)/libglusterfs/src -fPIC \
-	-D_FILE_OFFSET_BITS=64 -D_GNU_SOURCE -D$(GF_HOST_OS) $(LIBZ_CFLAGS)
+AM_CPPFLAGS = $(GF_CPPFLAGS) -I$(top_srcdir)/libglusterfs/src \
+	-I$(top_srcdir)/rpc/xdr/src -I$(top_builddir)/rpc/xdr/src \
+	-fPIC -D_FILE_OFFSET_BITS=64 -D_GNU_SOURCE -D$(GF_HOST_OS) \
+	$(LIBZ_CFLAGS)
 
 AM_CFLAGS = -Wall $(GF_CFLAGS)
 
diff --git a/xlators/features/compress/src/cdc-helper.c b/xlators/features/compress/src/cdc-helper.c
index 79d60a4924b..f973ff56cf5 100644
--- a/xlators/features/compress/src/cdc-helper.c
+++ b/xlators/features/compress/src/cdc-helper.c
@@ -8,13 +8,9 @@
    cases as published by the Free Software Foundation.
 */
 
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
-
-#include "glusterfs.h"
-#include "logging.h"
+#include <glusterfs/glusterfs.h>
+#include <glusterfs/logging.h>
+#include <glusterfs/syscall.h>
 
 #include "cdc.h"
 #include "cdc-mem-types.h"
@@ -38,118 +34,110 @@
  * gzip_header is added only during debugging.
  * Refer to the function cdc_dump_iovec_to_disk
  */
-static const char gzip_header[10] =
-        {
-                '\037', '\213', Z_DEFLATED, 0,
-                0,      0,      0,          0,
-                0,      GF_CDC_OS_ID
-        };
+static const char gzip_header[10] = {'\037', '\213', Z_DEFLATED,  0, 0, 0, 0,
+                                     0,      0,      GF_CDC_OS_ID};
 
 static int32_t
-cdc_next_iovec (xlator_t *this, cdc_info_t *ci)
+cdc_next_iovec(xlator_t *this, cdc_info_t *ci)
 {
-        int ret = -1;
-
-        ci->ncount++;
-        /* check for iovec overflow -- should not happen */
-        if (ci->ncount == MAX_IOVEC) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "Zlib output buffer overflow"
-                        " ->ncount (%d) | ->MAX_IOVEC (%d)",
-                        ci->ncount, MAX_IOVEC);
-                goto out;
-        }
-
-        ret = 0;
-
- out:
-        return ret;
+    int ret = -1;
+
+    ci->ncount++;
+    /* check for iovec overflow -- should not happen */
+    if (ci->ncount == MAX_IOVEC) {
+        gf_log(this->name, GF_LOG_ERROR,
+               "Zlib output buffer overflow"
+               " ->ncount (%d) | ->MAX_IOVEC (%d)",
+               ci->ncount, MAX_IOVEC);
+        goto out;
+    }
+
+    ret = 0;
+
+out:
+    return ret;
 }
 
 static void
-cdc_put_long (unsigned char *string, unsigned long x)
+cdc_put_long(unsigned char *string, unsigned long x)
 {
-        string[0] = (unsigned char) (x & 0xff);
-        string[1] = (unsigned char) ((x & 0xff00) >> 8);
-        string[2] = (unsigned char) ((x & 0xff0000) >> 16);
-        string[3] = (unsigned char) ((x & 0xff000000) >> 24);
+    string[0] = (unsigned char)(x & 0xff);
+    string[1] = (unsigned char)((x & 0xff00) >> 8);
+    string[2] = (unsigned char)((x & 0xff0000) >> 16);
+    string[3] = (unsigned char)((x & 0xff000000) >> 24);
 }
 
 static unsigned long
-cdc_get_long (unsigned char *buf)
+cdc_get_long(unsigned char *buf)
 {
-        return ((unsigned long) buf[0])
-                | (((unsigned long) buf[1]) << 8)
-                | (((unsigned long) buf[2]) << 16)
-                | (((unsigned long) buf[3]) << 24);
+    return ((unsigned long)buf[0]) | (((unsigned long)buf[1]) << 8) |
+           (((unsigned long)buf[2]) << 16) | (((unsigned long)buf[3]) << 24);
 }
 
 static int32_t
-cdc_init_gzip_trailer (xlator_t *this, cdc_priv_t *priv, cdc_info_t *ci)
+cdc_init_gzip_trailer(xlator_t *this, cdc_priv_t *priv, cdc_info_t *ci)
 {
-        int   ret = -1;
-        char *buf = NULL;
+    int ret = -1;
+    char *buf = NULL;
 
-        ret = cdc_next_iovec (this, ci);
-        if (ret)
-                goto out;
+    ret = cdc_next_iovec(this, ci);
+    if (ret)
+        goto out;
 
-        buf = CURR_VEC(ci).iov_base =
-                (char *) GF_CALLOC (1, GF_CDC_VALIDATION_SIZE,
-                                    gf_cdc_mt_gzip_trailer_t);
+    buf = CURR_VEC(ci).iov_base = (char *)GF_CALLOC(1, GF_CDC_VALIDATION_SIZE,
+                                                    gf_cdc_mt_gzip_trailer_t);
 
-        if (!CURR_VEC(ci).iov_base)
-                goto out;
+    if (!CURR_VEC(ci).iov_base)
+        goto out;
 
-        CURR_VEC(ci).iov_len = GF_CDC_VALIDATION_SIZE;
+    CURR_VEC(ci).iov_len = GF_CDC_VALIDATION_SIZE;
 
-        cdc_put_long ((unsigned char *)&buf[0], ci->crc);
-        cdc_put_long ((unsigned char *)&buf[4], ci->stream.total_in);
+    cdc_put_long((unsigned char *)&buf[0], ci->crc);
+    cdc_put_long((unsigned char *)&buf[4], ci->stream.total_in);
 
-        ret = 0;
+    ret = 0;
 
- out:
-        return ret;
+out:
+    return ret;
 }
 
 static int32_t
-cdc_alloc_iobuf_and_init_vec (xlator_t *this,
-                              cdc_priv_t *priv, cdc_info_t *ci,
-                              int size)
+cdc_alloc_iobuf_and_init_vec(xlator_t *this, cdc_priv_t *priv, cdc_info_t *ci,
+                             int size)
 {
-        int           ret       = -1;
-        int           alloc_len = 0;
-        struct iobuf *iobuf     = NULL;
+    int ret = -1;
+    int alloc_len = 0;
+    struct iobuf *iobuf = NULL;
 
-        ret = cdc_next_iovec (this, ci);
-        if (ret)
-                goto out;
+    ret = cdc_next_iovec(this, ci);
+    if (ret)
+        goto out;
 
-        alloc_len = size ? size : ci->buffer_size;
+    alloc_len = size ? size : ci->buffer_size;
 
-        iobuf = iobuf_get2 (this->ctx->iobuf_pool, alloc_len);
-        if (!iobuf)
-                goto out;
+    iobuf = iobuf_get2(this->ctx->iobuf_pool, alloc_len);
+    if (!iobuf)
+        goto out;
 
-        ret = iobref_add (ci->iobref, iobuf);
-        if (ret)
-                goto out;
+    ret = iobref_add(ci->iobref, iobuf);
+    if (ret)
+        goto out;
 
-        /* Initialize this iovec */
-        CURR_VEC(ci).iov_base = iobuf->ptr;
-        CURR_VEC(ci).iov_len  = alloc_len;
+    /* Initialize this iovec */
+    CURR_VEC(ci).iov_base = iobuf->ptr;
+    CURR_VEC(ci).iov_len = alloc_len;
 
-        ret = 0;
+    ret = 0;
 
- out:
-        return ret;
+out:
+    return ret;
 }
 
 static void
-cdc_init_zlib_output_stream (cdc_priv_t *priv, cdc_info_t *ci, int size)
+cdc_init_zlib_output_stream(cdc_priv_t *priv, cdc_info_t *ci, int size)
 {
-        ci->stream.next_out  = (unsigned char *) CURR_VEC(ci).iov_base;
-        ci->stream.avail_out = size ? size : ci->buffer_size;
+    ci->stream.next_out = (unsigned char *)CURR_VEC(ci).iov_base;
+    ci->stream.avail_out = size ? size : ci->buffer_size;
 }
 
 /* This routine is for testing and debugging only.
@@ -157,391 +145,383 @@ cdc_init_zlib_output_stream (cdc_priv_t *priv, cdc_info_t *ci, int size)
  * So each gzip dump file is at least 18 bytes in size.
  */
 void
-cdc_dump_iovec_to_disk (xlator_t *this, cdc_info_t *ci, const char *file)
+cdc_dump_iovec_to_disk(xlator_t *this, cdc_info_t *ci, const char *file)
 {
-        int    i      = 0;
-        int    fd     = 0;
-        size_t written = 0;
-        size_t total_written = 0;
-
-        fd = open (file, O_WRONLY|O_CREAT|O_TRUNC, 0777 );
-        if (fd < 0) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "Cannot open file: %s", file);
-                return;
-        }
-
-        written = write (fd, (char *) gzip_header, 10);
+    int i = 0;
+    int fd = 0;
+    size_t written = 0;
+    size_t total_written = 0;
+
+    fd = open(file, O_WRONLY | O_CREAT | O_TRUNC, 0777);
+    if (fd < 0) {
+        gf_log(this->name, GF_LOG_ERROR, "Cannot open file: %s", file);
+        return;
+    }
+
+    written = sys_write(fd, (char *)gzip_header, 10);
+    total_written += written;
+    for (i = 0; i < ci->ncount; i++) {
+        written = sys_write(fd, (char *)ci->vec[i].iov_base,
+                            ci->vec[i].iov_len);
         total_written += written;
-        for (i = 0; i < ci->ncount; i++) {
-                written = write (fd, (char *) ci->vec[i].iov_base, ci->vec[i].iov_len);
-                total_written += written;
-        }
+    }
 
-        gf_log (this->name, GF_LOG_DEBUG,
-                        "dump'd %zu bytes to %s", total_written, GF_CDC_DEBUG_DUMP_FILE );
+    gf_log(this->name, GF_LOG_DEBUG, "dump'd %zu bytes to %s", total_written,
+           GF_CDC_DEBUG_DUMP_FILE);
 
-        close (fd);
+    sys_close(fd);
 }
 
 static int32_t
-cdc_flush_libz_buffer (cdc_priv_t *priv, xlator_t *this, cdc_info_t *ci,
-                       int (*libz_func)(z_streamp, int),
-                       int flush)
+cdc_flush_libz_buffer(cdc_priv_t *priv, xlator_t *this, cdc_info_t *ci,
+                      int (*libz_func)(z_streamp, int), int flush)
 {
-        int32_t ret = Z_OK;
-        int done = 0;
-        unsigned int deflate_len = 0;
+    int32_t ret = Z_OK;
+    int done = 0;
+    unsigned int deflate_len = 0;
 
-        for (;;) {
-                deflate_len = ci->buffer_size - ci->stream.avail_out;
+    for (;;) {
+        deflate_len = ci->buffer_size - ci->stream.avail_out;
 
-                if (deflate_len != 0) {
-                        CURR_VEC(ci).iov_len = deflate_len;
+        if (deflate_len != 0) {
+            CURR_VEC(ci).iov_len = deflate_len;
 
-                        ret = cdc_alloc_iobuf_and_init_vec (this, priv, ci, 0);
-                        if (ret) {
-                                ret = Z_MEM_ERROR;
-                                break;
-                        }
+            ret = cdc_alloc_iobuf_and_init_vec(this, priv, ci, 0);
+            if (ret) {
+                ret = Z_MEM_ERROR;
+                break;
+            }
 
-                        /* Re-position Zlib output buffer */
-                        cdc_init_zlib_output_stream (priv, ci, 0);
-                }
+            /* Re-position Zlib output buffer */
+            cdc_init_zlib_output_stream(priv, ci, 0);
+        }
 
-                if (done) {
-                        ci->ncount--;
-                        break;
-                }
+        if (done) {
+            ci->ncount--;
+            break;
+        }
 
-                ret = libz_func (&ci->stream, flush);
+        ret = libz_func(&ci->stream, flush);
 
-                if (ret == Z_BUF_ERROR) {
-                        ret = Z_OK;
-                        ci->ncount--;
-                        break;
-                }
+        if (ret == Z_BUF_ERROR) {
+            ret = Z_OK;
+            ci->ncount--;
+            break;
+        }
 
-                done = (ci->stream.avail_out != 0 || ret == Z_STREAM_END);
+        done = (ci->stream.avail_out != 0 || ret == Z_STREAM_END);
 
-                if (ret != Z_OK && ret != Z_STREAM_END)
-                        break;
-        }
+        if (ret != Z_OK && ret != Z_STREAM_END)
+            break;
+    }
 
-        return ret;
+    return ret;
 }
 
 static int32_t
-do_cdc_compress (struct iovec *vec, xlator_t *this, cdc_priv_t *priv,
-                 cdc_info_t *ci)
+do_cdc_compress(struct iovec *vec, xlator_t *this, cdc_priv_t *priv,
+                cdc_info_t *ci)
 {
-        int ret = -1;
-
-        /* Initialize defalte */
-        ret = deflateInit2 (&ci->stream, priv->cdc_level, Z_DEFLATED,
-                            priv->window_size, priv->mem_level,
-                            Z_DEFAULT_STRATEGY);
-
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "unable to init Zlib (retval: %d)", ret);
-                goto out;
-        }
+    int ret = -1;
 
-        ret = cdc_alloc_iobuf_and_init_vec (this, priv, ci, 0);
-        if (ret)
-                goto out;
+    /* Initialize defalte */
+    ret = deflateInit2(&ci->stream, priv->cdc_level, Z_DEFLATED,
+                       priv->window_size, priv->mem_level, Z_DEFAULT_STRATEGY);
 
-        /* setup output buffer */
-        cdc_init_zlib_output_stream (priv, ci, 0);
+    if (ret) {
+        gf_log(this->name, GF_LOG_ERROR, "unable to init Zlib (retval: %d)",
+               ret);
+        goto out;
+    }
 
-        /* setup input buffer */
-        ci->stream.next_in  = (unsigned char *) vec->iov_base;
-        ci->stream.avail_in = vec->iov_len;
+    ret = cdc_alloc_iobuf_and_init_vec(this, priv, ci, 0);
+    if (ret)
+        goto out;
 
-        ci->crc = crc32 (ci->crc, (const Bytef *) vec->iov_base, vec->iov_len);
+    /* setup output buffer */
+    cdc_init_zlib_output_stream(priv, ci, 0);
 
-        gf_log (this->name, GF_LOG_DEBUG, "crc=%lu len=%d buffer_size=%d",
-                ci->crc, ci->stream.avail_in, ci->buffer_size);
+    /* setup input buffer */
+    ci->stream.next_in = (unsigned char *)vec->iov_base;
+    ci->stream.avail_in = vec->iov_len;
 
-        /* compress !! */
-        while (ci->stream.avail_in != 0) {
-                if (ci->stream.avail_out == 0) {
+    ci->crc = crc32(ci->crc, (const Bytef *)vec->iov_base, vec->iov_len);
 
-                        CURR_VEC(ci).iov_len = ci->buffer_size;
+    gf_log(this->name, GF_LOG_DEBUG, "crc=%lu len=%d buffer_size=%d", ci->crc,
+           ci->stream.avail_in, ci->buffer_size);
 
-                        ret = cdc_alloc_iobuf_and_init_vec (this, priv, ci, 0);
-                        if (ret)
-                                break;
+    /* compress !! */
+    while (ci->stream.avail_in != 0) {
+        if (ci->stream.avail_out == 0) {
+            CURR_VEC(ci).iov_len = ci->buffer_size;
 
-                        /* Re-position Zlib output buffer */
-                        cdc_init_zlib_output_stream (priv, ci, 0);
-                }
+            ret = cdc_alloc_iobuf_and_init_vec(this, priv, ci, 0);
+            if (ret)
+                break;
 
-                ret = deflate (&ci->stream, Z_NO_FLUSH);
-                if (ret != Z_OK)
-                        break;
+            /* Re-position Zlib output buffer */
+            cdc_init_zlib_output_stream(priv, ci, 0);
         }
 
- out:
-        return ret;
+        ret = deflate(&ci->stream, Z_NO_FLUSH);
+        if (ret != Z_OK)
+            break;
+    }
+
+out:
+    return ret;
 }
 
 int32_t
-cdc_compress (xlator_t *this, cdc_priv_t *priv, cdc_info_t *ci,
-              dict_t **xdata)
+cdc_compress(xlator_t *this, cdc_priv_t *priv, cdc_info_t *ci, dict_t **xdata)
 {
-        int ret = -1;
-        int i   = 0;
+    int ret = -1;
+    int i = 0;
 
-        ci->iobref = iobref_new ();
-        if (!ci->iobref)
-                goto out;
+    ci->iobref = iobref_new();
+    if (!ci->iobref)
+        goto out;
 
+    if (!*xdata) {
+        *xdata = dict_new();
         if (!*xdata) {
-                *xdata = dict_new ();
-                if (!*xdata) {
-                        gf_log (this->name, GF_LOG_ERROR, "Cannot allocate xdata"
-                                " dict");
-                        goto out;
-                }
-        }
-
-        /* data */
-        for (i = 0; i < ci->count; i++) {
-                ret = do_cdc_compress (&ci->vector[i], this, priv, ci);
-                if (ret != Z_OK)
-                        goto deflate_cleanup_out;
-        }
-
-        /* flush zlib buffer */
-        ret = cdc_flush_libz_buffer (priv, this, ci, deflate, Z_FINISH);
-        if (!(ret == Z_OK || ret == Z_STREAM_END)) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "Compression Error: ret (%d)", ret);
-                ret = -1;
-                goto deflate_cleanup_out;
-        }
-
-        /* trailer */
-        ret = cdc_init_gzip_trailer (this, priv, ci);
-        if (ret)
-                goto deflate_cleanup_out;
-
-        gf_log (this->name, GF_LOG_DEBUG,
-                "Compressed %ld to %ld bytes",
-                ci->stream.total_in, ci->stream.total_out);
-
-        ci->nbytes = ci->stream.total_out + GF_CDC_VALIDATION_SIZE;
-
-        /* set deflated canary value for identification */
-        ret = dict_set_int32 (*xdata, GF_CDC_DEFLATE_CANARY_VAL, 1);
-        if (ret) {
-                /* Send uncompressed data if we can't _tell_ the client
-                 * that deflated data is on it's way. So, we just log
-                 * the faliure and continue as usual.
-                 */
-                 gf_log (this->name, GF_LOG_ERROR,
-                 "Data deflated, but could not set canary"
-                 " value in dict for identification");
+            gf_log(this->name, GF_LOG_ERROR,
+                   "Cannot allocate xdata"
+                   " dict");
+            goto out;
         }
+    }
+
+    /* data */
+    for (i = 0; i < ci->count; i++) {
+        ret = do_cdc_compress(&ci->vector[i], this, priv, ci);
+        if (ret != Z_OK)
+            goto deflate_cleanup_out;
+    }
+
+    /* flush zlib buffer */
+    ret = cdc_flush_libz_buffer(priv, this, ci, deflate, Z_FINISH);
+    if (!(ret == Z_OK || ret == Z_STREAM_END)) {
+        gf_log(this->name, GF_LOG_ERROR, "Compression Error: ret (%d)", ret);
+        ret = -1;
+        goto deflate_cleanup_out;
+    }
+
+    /* trailer */
+    ret = cdc_init_gzip_trailer(this, priv, ci);
+    if (ret)
+        goto deflate_cleanup_out;
+
+    gf_log(this->name, GF_LOG_DEBUG, "Compressed %ld to %ld bytes",
+           ci->stream.total_in, ci->stream.total_out);
+
+    ci->nbytes = ci->stream.total_out + GF_CDC_VALIDATION_SIZE;
+
+    /* set deflated canary value for identification */
+    ret = dict_set_int32(*xdata, GF_CDC_DEFLATE_CANARY_VAL, 1);
+    if (ret) {
+        /* Send uncompressed data if we can't _tell_ the client
+         * that deflated data is on it's way. So, we just log
+         * the failure and continue as usual.
+         */
+        gf_log(this->name, GF_LOG_ERROR,
+               "Data deflated, but could not set canary"
+               " value in dict for identification");
+    }
 
-        /* This is to be used in testing */
-        if ( priv->debug ) {
-                cdc_dump_iovec_to_disk (this, ci, GF_CDC_DEBUG_DUMP_FILE );
-        }
+    /* This is to be used in testing */
+    if (priv->debug) {
+        cdc_dump_iovec_to_disk(this, ci, GF_CDC_DEBUG_DUMP_FILE);
+    }
 
- deflate_cleanup_out:
-        (void) deflateEnd(&ci->stream);
+deflate_cleanup_out:
+    (void)deflateEnd(&ci->stream);
 
- out:
-        return ret;
+out:
+    return ret;
 }
 
-
 /* deflate content is checked by the presence of a canary
  * value in the dict as the key
  */
 static int32_t
-cdc_check_content_for_deflate (dict_t *xdata)
+cdc_check_content_for_deflate(dict_t *xdata)
 {
-        return dict_get (xdata, GF_CDC_DEFLATE_CANARY_VAL) ? -1 : 0;
+    return dict_get(xdata, GF_CDC_DEFLATE_CANARY_VAL) ? -1 : 0;
 }
 
 static unsigned long
-cdc_extract_crc (char *trailer)
+cdc_extract_crc(char *trailer)
 {
-        return cdc_get_long ((unsigned char *) &trailer[0]);
+    return cdc_get_long((unsigned char *)&trailer[0]);
 }
 
 static unsigned long
-cdc_extract_size (char *trailer)
+cdc_extract_size(char *trailer)
 {
-        return cdc_get_long ((unsigned char *) &trailer[4]);
+    return cdc_get_long((unsigned char *)&trailer[4]);
 }
 
 static int32_t
-cdc_validate_inflate (cdc_info_t *ci, unsigned long crc,
-                      unsigned long len)
+cdc_validate_inflate(cdc_info_t *ci, unsigned long crc, unsigned long len)
 {
-        return !((crc == ci->crc)
-                 /* inflated length is hidden inside
-                  * Zlib stream struct */
-                 && (len == ci->stream.total_out));
+    return !((crc == ci->crc)
+             /* inflated length is hidden inside
+              * Zlib stream struct */
+             && (len == ci->stream.total_out));
 }
 
 static int32_t
-do_cdc_decompress (xlator_t *this, cdc_priv_t *priv, cdc_info_t *ci)
+do_cdc_decompress(xlator_t *this, cdc_priv_t *priv, cdc_info_t *ci)
 {
-        int            ret          = -1;
-        int            i            = 0;
-        int            len          = 0;
-        char          *inflte       = NULL;
-        char          *trailer      = NULL;
-        struct iovec   vec          = {0,};
-        unsigned long  computed_crc = 0;
-        unsigned long  computed_len = 0;
-
-        ret = inflateInit2 (&ci->stream, priv->window_size);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "Zlib: Unable to initialize inflate");
-                goto out;
+    int ret = -1;
+    int i = 0;
+    int len = 0;
+    char *inflte = NULL;
+    char *trailer = NULL;
+    struct iovec vec = {
+        0,
+    };
+    unsigned long computed_crc = 0;
+    unsigned long computed_len = 0;
+
+    ret = inflateInit2(&ci->stream, priv->window_size);
+    if (ret) {
+        gf_log(this->name, GF_LOG_ERROR, "Zlib: Unable to initialize inflate");
+        goto out;
+    }
+
+    vec = THIS_VEC(ci, 0);
+
+    trailer = (char *)(((char *)vec.iov_base) + vec.iov_len -
+                       GF_CDC_VALIDATION_SIZE);
+
+    /* CRC of uncompressed data */
+    computed_crc = cdc_extract_crc(trailer);
+
+    /* size of uncomrpessed data */
+    computed_len = cdc_extract_size(trailer);
+
+    gf_log(this->name, GF_LOG_DEBUG, "crc=%lu len=%lu buffer_size=%d",
+           computed_crc, computed_len, ci->buffer_size);
+
+    inflte = vec.iov_base;
+    len = vec.iov_len - GF_CDC_VALIDATION_SIZE;
+
+    /* allocate buffer of the original length of the data */
+    ret = cdc_alloc_iobuf_and_init_vec(this, priv, ci, 0);
+    if (ret)
+        goto out;
+
+    /* setup output buffer */
+    cdc_init_zlib_output_stream(priv, ci, 0);
+
+    /* setup input buffer */
+    ci->stream.next_in = (unsigned char *)inflte;
+    ci->stream.avail_in = len;
+
+    while (ci->stream.avail_in != 0) {
+        if (ci->stream.avail_out == 0) {
+            CURR_VEC(ci).iov_len = ci->buffer_size;
+
+            ret = cdc_alloc_iobuf_and_init_vec(this, priv, ci, 0);
+            if (ret)
+                break;
+
+            /* Re-position Zlib output buffer */
+            cdc_init_zlib_output_stream(priv, ci, 0);
         }
 
-        vec = THIS_VEC(ci, 0);
-
-        trailer = (char *) (((char *) vec.iov_base) + vec.iov_len
-                - GF_CDC_VALIDATION_SIZE);
-
-        /* CRC of uncompressed data */
-        computed_crc = cdc_extract_crc (trailer);
-
-        /* size of uncomrpessed data */
-        computed_len = cdc_extract_size (trailer);
-
-        gf_log (this->name, GF_LOG_DEBUG, "crc=%lu len=%lu buffer_size=%d",
-                computed_crc, computed_len, ci->buffer_size);
-
-        inflte = vec.iov_base ;
-        len = vec.iov_len - GF_CDC_VALIDATION_SIZE;
-
-        /* allocate buffer of the original length of the data */
-        ret = cdc_alloc_iobuf_and_init_vec (this, priv, ci, 0);
-        if (ret)
-                goto out;
-
-        /* setup output buffer */
-        cdc_init_zlib_output_stream (priv, ci, 0);
-
-        /* setup input buffer */
-        ci->stream.next_in = (unsigned char *) inflte;
-        ci->stream.avail_in = len;
-
-        while (ci->stream.avail_in != 0) {
-                if (ci->stream.avail_out == 0) {
-                        CURR_VEC(ci).iov_len = ci->buffer_size;
-
-                        ret = cdc_alloc_iobuf_and_init_vec (this, priv, ci, 0);
-                        if (ret)
-                                break;
-
-                        /* Re-position Zlib output buffer */
-                        cdc_init_zlib_output_stream (priv, ci, 0);
-                }
-
-                ret = inflate (&ci->stream, Z_NO_FLUSH);
-                if (ret == Z_STREAM_ERROR)
-                        break;
-        }
-
-        /* flush zlib buffer */
-        ret = cdc_flush_libz_buffer (priv, this, ci, inflate, Z_SYNC_FLUSH);
-        if (!(ret == Z_OK || ret == Z_STREAM_END)) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "Decompression Error: ret (%d)", ret);
-                ret = -1;
-                goto out;
-        }
-
-        /* compute CRC of the uncompresses data to check for
-         * correctness */
-
-        for (i = 0; i < ci->ncount; i++) {
-                ci->crc = crc32 (ci->crc,
-                                 (const Bytef *) ci->vec[i].iov_base,
-                                 ci->vec[i].iov_len);
-        }
-
-        /* validate inflated data */
-        ret = cdc_validate_inflate (ci, computed_crc, computed_len);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "Checksum or length mismatched in inflated data");
-        }
-
- out:
-        return ret;
+        ret = inflate(&ci->stream, Z_NO_FLUSH);
+        if (ret == Z_STREAM_ERROR)
+            break;
+    }
+
+    /* flush zlib buffer */
+    ret = cdc_flush_libz_buffer(priv, this, ci, inflate, Z_SYNC_FLUSH);
+    if (!(ret == Z_OK || ret == Z_STREAM_END)) {
+        gf_log(this->name, GF_LOG_ERROR, "Decompression Error: ret (%d)", ret);
+        ret = -1;
+        goto out;
+    }
+
+    /* compute CRC of the uncompresses data to check for
+     * correctness */
+
+    for (i = 0; i < ci->ncount; i++) {
+        ci->crc = crc32(ci->crc, (const Bytef *)ci->vec[i].iov_base,
+                        ci->vec[i].iov_len);
+    }
+
+    /* validate inflated data */
+    ret = cdc_validate_inflate(ci, computed_crc, computed_len);
+    if (ret) {
+        gf_log(this->name, GF_LOG_ERROR,
+               "Checksum or length mismatched in inflated data");
+    }
+
+out:
+    return ret;
 }
 
 int32_t
-cdc_decompress (xlator_t *this, cdc_priv_t *priv, cdc_info_t *ci,
-                dict_t *xdata)
+cdc_decompress(xlator_t *this, cdc_priv_t *priv, cdc_info_t *ci, dict_t *xdata)
 {
-        int32_t ret = -1;
-
-        /* check for deflate content */
-        if (!cdc_check_content_for_deflate (xdata)) {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "Content not deflated, passing through ...");
-                goto passthrough_out;
-        }
-
-        ci->iobref = iobref_new ();
-        if (!ci->iobref)
-                goto passthrough_out;
-
-        /* do we need to do this? can we assume that one iovec
-         * will hold per request data every time?
-         *
-         * server/client protocol seems to deal with a single
-         * iovec even if op_ret > 1M. So, it looks ok to
-         * assume that a single iovec will contain all the
-         * data (This saves us a lot from finding the trailer
-         * and the data since it could have been split-up onto
-         * two adjacent iovec's.
-         *
-         * But, in case this translator is loaded above quick-read
-         * for some reason, then it's entirely possible that we get
-         * multiple iovec's...
-         *
-         * This case (handled below) is not tested. (by loading the
-         * xlator below quick-read)
-         */
-
-        /* @@ I_HOPE_THIS_IS_NEVER_HIT */
-        if (ci->count > 1) {
-                gf_log (this->name, GF_LOG_WARNING, "unable to handle"
-                        " multiple iovecs (%d in number)", ci->count);
-                goto inflate_cleanup_out;
-                /* TODO: coallate all iovecs in one */
-        }
-
-        ret = do_cdc_decompress (this, priv, ci);
-        if (ret)
-                goto inflate_cleanup_out;
-
-        ci->nbytes = ci->stream.total_out;
-
-        gf_log (this->name, GF_LOG_DEBUG,
-                "Inflated %ld to %ld bytes",
-                ci->stream.total_in, ci->stream.total_out);
-
- inflate_cleanup_out:
-        (void) inflateEnd (&ci->stream);
-
- passthrough_out:
-        return ret;
+    int32_t ret = -1;
+
+    /* check for deflate content */
+    if (!cdc_check_content_for_deflate(xdata)) {
+        gf_log(this->name, GF_LOG_DEBUG,
+               "Content not deflated, passing through ...");
+        goto passthrough_out;
+    }
+
+    ci->iobref = iobref_new();
+    if (!ci->iobref)
+        goto passthrough_out;
+
+    /* do we need to do this? can we assume that one iovec
+     * will hold per request data every time?
+     *
+     * server/client protocol seems to deal with a single
+     * iovec even if op_ret > 1M. So, it looks ok to
+     * assume that a single iovec will contain all the
+     * data (This saves us a lot from finding the trailer
+     * and the data since it could have been split-up onto
+     * two adjacent iovec's.
+     *
+     * But, in case this translator is loaded above quick-read
+     * for some reason, then it's entirely possible that we get
+     * multiple iovec's...
+     *
+     * This case (handled below) is not tested. (by loading the
+     * xlator below quick-read)
+     */
+
+    /* @@ I_HOPE_THIS_IS_NEVER_HIT */
+    if (ci->count > 1) {
+        gf_log(this->name, GF_LOG_WARNING,
+               "unable to handle"
+               " multiple iovecs (%d in number)",
+               ci->count);
+        goto inflate_cleanup_out;
+        /* TODO: coallate all iovecs in one */
+    }
+
+    ret = do_cdc_decompress(this, priv, ci);
+    if (ret)
+        goto inflate_cleanup_out;
+
+    ci->nbytes = ci->stream.total_out;
+
+    gf_log(this->name, GF_LOG_DEBUG, "Inflated %ld to %ld bytes",
+           ci->stream.total_in, ci->stream.total_out);
+
+inflate_cleanup_out:
+    (void)inflateEnd(&ci->stream);
+
+passthrough_out:
+    return ret;
 }
 
 #endif
diff --git a/xlators/features/compress/src/cdc-mem-types.h b/xlators/features/compress/src/cdc-mem-types.h
index ead2c70ba6e..928afdd2efe 100644
--- a/xlators/features/compress/src/cdc-mem-types.h
+++ b/xlators/features/compress/src/cdc-mem-types.h
@@ -11,13 +11,13 @@
 #ifndef __CDC_MEM_TYPES_H
 #define __CDC_MEM_TYPES_H
 
-#include "mem-types.h"
+#include <glusterfs/mem-types.h>
 
 enum gf_cdc_mem_types {
-        gf_cdc_mt_priv_t         = gf_common_mt_end + 1,
-        gf_cdc_mt_vec_t          = gf_common_mt_end + 2,
-        gf_cdc_mt_gzip_trailer_t = gf_common_mt_end + 3,
-        gf_cdc_mt_end            = gf_common_mt_end + 4,
+    gf_cdc_mt_priv_t = gf_common_mt_end + 1,
+    gf_cdc_mt_vec_t = gf_common_mt_end + 2,
+    gf_cdc_mt_gzip_trailer_t = gf_common_mt_end + 3,
+    gf_cdc_mt_end = gf_common_mt_end + 4,
 };
 
 #endif
diff --git a/xlators/features/compress/src/cdc.c b/xlators/features/compress/src/cdc.c
index 67fc52505e5..b0b51e914ed 100644
--- a/xlators/features/compress/src/cdc.c
+++ b/xlators/features/compress/src/cdc.c
@@ -10,352 +10,339 @@
 
 #include <sys/uio.h>
 
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
-
-#include "xlator.h"
-#include "defaults.h"
-#include "logging.h"
+#include <glusterfs/xlator.h>
+#include <glusterfs/defaults.h>
+#include <glusterfs/logging.h>
 
 #include "cdc.h"
 #include "cdc-mem-types.h"
 
 static void
-cdc_cleanup_iobref (cdc_info_t *ci)
+cdc_cleanup_iobref(cdc_info_t *ci)
 {
-        assert(ci->iobref != NULL);
-        iobref_clear (ci->iobref);
+    assert(ci->iobref != NULL);
+    iobref_clear(ci->iobref);
 }
 
 int32_t
-cdc_readv_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-               int32_t op_ret, int32_t op_errno,
-               struct iovec *vector, int32_t count,
-               struct iatt *stbuf, struct iobref *iobref,
-               dict_t *xdata)
+cdc_readv_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int32_t op_ret,
+              int32_t op_errno, struct iovec *vector, int32_t count,
+              struct iatt *stbuf, struct iobref *iobref, dict_t *xdata)
 {
-        int         ret   = -1;
-        cdc_priv_t *priv  = NULL;
-        cdc_info_t  ci    = {0,};
-
-        GF_VALIDATE_OR_GOTO ("cdc", this, default_out);
-        GF_VALIDATE_OR_GOTO (this->name, frame, default_out);
-
-        priv = this->private;
-
-        if (op_ret <= 0)
-                goto default_out;
-
-        if ( (priv->min_file_size != 0)
-             && (op_ret < priv->min_file_size) )
-                goto default_out;
-
-        ci.count       = count;
-        ci.ibytes      = op_ret;
-        ci.vector      = vector;
-        ci.buf         = NULL;
-        ci.iobref      = NULL;
-        ci.ncount      = 0;
-        ci.crc         = 0;
-        ci.buffer_size = GF_CDC_DEF_BUFFERSIZE;
-
-/* A readv compresses on the server side and decompresses on the client side
- */
-        if (priv->op_mode == GF_CDC_MODE_SERVER) {
-                ret = cdc_compress (this, priv, &ci, &xdata);
-        } else if (priv->op_mode == GF_CDC_MODE_CLIENT) {
-                ret = cdc_decompress (this, priv, &ci, xdata);
-        } else {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "Invalid operation mode (%d)", priv->op_mode);
-        }
-
-        if (ret)
-                goto default_out;
-
-        STACK_UNWIND_STRICT (readv, frame, ci.nbytes, op_errno,
-                             ci.vec, ci.ncount, stbuf, iobref,
-                             xdata);
-        cdc_cleanup_iobref (&ci);
-        return 0;
-
- default_out:
-        STACK_UNWIND_STRICT (readv, frame, op_ret, op_errno,
-                             vector, count, stbuf, iobref, xdata);
-        return 0;
+    int ret = -1;
+    cdc_priv_t *priv = NULL;
+    cdc_info_t ci = {
+        0,
+    };
+
+    GF_VALIDATE_OR_GOTO("cdc", this, default_out);
+    GF_VALIDATE_OR_GOTO(this->name, frame, default_out);
+
+    priv = this->private;
+
+    if (op_ret <= 0)
+        goto default_out;
+
+    if ((priv->min_file_size != 0) && (op_ret < priv->min_file_size))
+        goto default_out;
+
+    ci.count = count;
+    ci.ibytes = op_ret;
+    ci.vector = vector;
+    ci.buf = NULL;
+    ci.iobref = NULL;
+    ci.ncount = 0;
+    ci.crc = 0;
+    ci.buffer_size = GF_CDC_DEF_BUFFERSIZE;
+
+    /* A readv compresses on the server side and decompresses on the client side
+     */
+    if (priv->op_mode == GF_CDC_MODE_SERVER) {
+        ret = cdc_compress(this, priv, &ci, &xdata);
+    } else if (priv->op_mode == GF_CDC_MODE_CLIENT) {
+        ret = cdc_decompress(this, priv, &ci, xdata);
+    } else {
+        gf_log(this->name, GF_LOG_ERROR, "Invalid operation mode (%d)",
+               priv->op_mode);
+    }
+
+    if (ret)
+        goto default_out;
+
+    STACK_UNWIND_STRICT(readv, frame, ci.nbytes, op_errno, ci.vec, ci.ncount,
+                        stbuf, iobref, xdata);
+    cdc_cleanup_iobref(&ci);
+    return 0;
+
+default_out:
+    STACK_UNWIND_STRICT(readv, frame, op_ret, op_errno, vector, count, stbuf,
+                        iobref, xdata);
+    return 0;
 }
 
 int32_t
-cdc_readv (call_frame_t *frame, xlator_t *this,
-           fd_t *fd, size_t size, off_t offset, uint32_t flags,
-           dict_t *xdata)
+cdc_readv(call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
+          off_t offset, uint32_t flags, dict_t *xdata)
 {
-        fop_readv_cbk_t cbk = NULL;
+    fop_readv_cbk_t cbk = NULL;
 
 #ifdef HAVE_LIB_Z
-        cbk = cdc_readv_cbk;
+    cbk = cdc_readv_cbk;
 #else
-        cbk = default_readv_cbk;
+    cbk = default_readv_cbk;
 #endif
-        STACK_WIND (frame, cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->readv,
-                    fd, size, offset, flags, xdata);
-        return 0;
+    STACK_WIND(frame, cbk, FIRST_CHILD(this), FIRST_CHILD(this)->fops->readv,
+               fd, size, offset, flags, xdata);
+    return 0;
 }
 
 int32_t
-cdc_writev_cbk (call_frame_t *frame,
-                void *cookie,
-                xlator_t *this,
-                int32_t op_ret,
-                int32_t op_errno,
-                struct iatt *prebuf,
-                struct iatt *postbuf, dict_t *xdata)
+cdc_writev_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+               int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
+               struct iatt *postbuf, dict_t *xdata)
 {
-
-        STACK_UNWIND_STRICT (writev, frame, op_ret, op_errno, prebuf, postbuf, xdata);
-        return 0;
+    STACK_UNWIND_STRICT(writev, frame, op_ret, op_errno, prebuf, postbuf,
+                        xdata);
+    return 0;
 }
 
 int32_t
-cdc_writev (call_frame_t *frame,
-            xlator_t *this,
-            fd_t *fd,
-            struct iovec *vector,
-            int32_t count,
-            off_t offset,
-            uint32_t flags,
-            struct iobref *iobref, dict_t *xdata)
+cdc_writev(call_frame_t *frame, xlator_t *this, fd_t *fd, struct iovec *vector,
+           int32_t count, off_t offset, uint32_t flags, struct iobref *iobref,
+           dict_t *xdata)
 {
-        int          ret   = -1;
-        cdc_priv_t  *priv  = NULL;
-        cdc_info_t   ci    = {0,};
-        size_t       isize = 0;
-
-        GF_VALIDATE_OR_GOTO ("cdc", this, default_out);
-        GF_VALIDATE_OR_GOTO (this->name, frame, default_out);
-
-        priv = this->private;
-
-        isize =  iov_length(vector, count);
-
-        if (isize <= 0)
-            goto default_out;
-
-        if ( (priv->min_file_size != 0) && (isize < priv->min_file_size) )
-            goto default_out;
-
-        ci.count       = count;
-        ci.ibytes      = isize;
-        ci.vector      = vector;
-        ci.buf         = NULL;
-        ci.iobref      = NULL;
-        ci.ncount      = 0;
-        ci.crc         = 0;
-        ci.buffer_size = GF_CDC_DEF_BUFFERSIZE;
-
-/* A writev compresses on the client side and decompresses on the server side
- */
-	    if (priv->op_mode == GF_CDC_MODE_CLIENT) {
-		    ret = cdc_compress (this, priv, &ci, &xdata);
-	    } else if (priv->op_mode == GF_CDC_MODE_SERVER) {
-		    ret = cdc_decompress (this, priv, &ci, xdata);
-	    } else {
-		    gf_log (this->name, GF_LOG_ERROR, "Invalid operation mode (%d) ", priv->op_mode);
-	    }
-
-	    if (ret)
-		    goto default_out;
-
-	    STACK_WIND (frame,
-                    cdc_writev_cbk,
-                    FIRST_CHILD (this),
-                    FIRST_CHILD (this)->fops->writev,
-                    fd, ci.vec, ci.ncount, offset, flags,
-                    iobref, xdata);
-
-        cdc_cleanup_iobref (&ci);
-        return 0;
-
- default_out:
-        STACK_WIND (frame,
-                    cdc_writev_cbk,
-                    FIRST_CHILD (this),
-                    FIRST_CHILD (this)->fops->writev,
-                    fd, vector, count, offset, flags,
-                    iobref, xdata);
-        return 0;
+    int ret = -1;
+    cdc_priv_t *priv = NULL;
+    cdc_info_t ci = {
+        0,
+    };
+    size_t isize = 0;
+
+    GF_VALIDATE_OR_GOTO("cdc", this, err);
+    GF_VALIDATE_OR_GOTO(this->name, frame, err);
+
+    priv = this->private;
+
+    isize = iov_length(vector, count);
+
+    if (isize <= 0)
+        goto default_out;
+
+    if ((priv->min_file_size != 0) && (isize < priv->min_file_size))
+        goto default_out;
+
+    ci.count = count;
+    ci.ibytes = isize;
+    ci.vector = vector;
+    ci.buf = NULL;
+    ci.iobref = NULL;
+    ci.ncount = 0;
+    ci.crc = 0;
+    ci.buffer_size = GF_CDC_DEF_BUFFERSIZE;
+
+    /* A writev compresses on the client side and decompresses on the server
+     * side
+     */
+    if (priv->op_mode == GF_CDC_MODE_CLIENT) {
+        ret = cdc_compress(this, priv, &ci, &xdata);
+    } else if (priv->op_mode == GF_CDC_MODE_SERVER) {
+        ret = cdc_decompress(this, priv, &ci, xdata);
+    } else {
+        gf_log(this->name, GF_LOG_ERROR, "Invalid operation mode (%d) ",
+               priv->op_mode);
+    }
+
+    if (ret)
+        goto default_out;
+
+    STACK_WIND(frame, cdc_writev_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->writev, fd, ci.vec, ci.ncount, offset,
+               flags, iobref, xdata);
+
+    cdc_cleanup_iobref(&ci);
+    return 0;
+
+default_out:
+    STACK_WIND(frame, cdc_writev_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->writev, fd, vector, count, offset,
+               flags, iobref, xdata);
+    return 0;
+err:
+    STACK_UNWIND_STRICT(writev, frame, -1, EINVAL, NULL, NULL, NULL);
+    return 0;
 }
 
 int32_t
-mem_acct_init (xlator_t *this)
+mem_acct_init(xlator_t *this)
 {
-        int     ret = -1;
-
-        if (!this)
-                return ret;
+    int ret = -1;
 
-        ret = xlator_mem_acct_init (this, gf_cdc_mt_end);
+    if (!this)
+        return ret;
 
-        if (ret != 0) {
-                gf_log(this->name, GF_LOG_ERROR, "Memory accounting init"
-                       "failed");
-                return ret;
-        }
+    ret = xlator_mem_acct_init(this, gf_cdc_mt_end);
 
+    if (ret != 0) {
+        gf_log(this->name, GF_LOG_ERROR,
+               "Memory accounting init"
+               "failed");
         return ret;
+    }
+
+    return ret;
 }
 
 int32_t
-init (xlator_t *this)
+init(xlator_t *this)
 {
-        int         ret      = -1;
-        char       *temp_str = NULL;
-        cdc_priv_t *priv     = NULL;
-
-        GF_VALIDATE_OR_GOTO ("cdc", this, err);
-
-        if (!this->children || this->children->next) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "Need subvolume == 1");
-                goto err;
-        }
-
-        if (!this->parents) {
-                gf_log (this->name, GF_LOG_WARNING,
-                        "Dangling volume. Check volfile");
-        }
-
-        priv = GF_CALLOC (1, sizeof (*priv), gf_cdc_mt_priv_t);
-        if (!priv) {
-                goto err;
-        }
-
-        /* Check if debug mode is turned on */
-        GF_OPTION_INIT ("debug", priv->debug, bool, err);
-        if( priv->debug ) {
-                gf_log (this->name, GF_LOG_DEBUG, "CDC debug option turned on");
-        }
-
-        /* Set Gzip Window Size */
-        GF_OPTION_INIT ("window-size", priv->window_size, int32, err);
-        if ( (priv->window_size > GF_CDC_MAX_WINDOWSIZE)
-             || (priv->window_size < GF_CDC_DEF_WINDOWSIZE) ) {
-                gf_log (this->name, GF_LOG_WARNING,
-                        "Invalid gzip window size (%d), using default",
-                        priv->window_size);
-                priv->window_size = GF_CDC_DEF_WINDOWSIZE;
-        }
-
-        /* Set Gzip (De)Compression Level */
-        GF_OPTION_INIT ("compression-level", priv->cdc_level, int32, err);
-        if ( ((priv->cdc_level < 1) || (priv->cdc_level > 9))
-             && (priv->cdc_level != GF_CDC_DEF_COMPRESSION) ) {
-                gf_log (this->name, GF_LOG_WARNING,
-                        "Invalid gzip (de)compression level (%d),"
-                        " using default", priv->cdc_level);
-                priv->cdc_level = GF_CDC_DEF_COMPRESSION;
-        }
-
-        /* Set Gzip Memory Level */
-        GF_OPTION_INIT ("mem-level", priv->mem_level, int32, err);
-        if ( (priv->mem_level < 1) || (priv->mem_level > 9) ) {
-                gf_log (this->name, GF_LOG_WARNING,
-                        "Invalid gzip memory level, using the default");
-                priv->mem_level = GF_CDC_DEF_MEMLEVEL;
-        }
-
-        /* Set min file size to enable compression */
-        GF_OPTION_INIT ("min-size", priv->min_file_size, int32, err);
-
-        /* Mode of operation - Server/Client */
-        ret = dict_get_str (this->options, "mode", &temp_str);
-        if (ret) {
-                gf_log (this->name, GF_LOG_CRITICAL,
-                        "Operation mode not specified !!");
-                goto err;
-        }
-
-        if (GF_CDC_MODE_IS_CLIENT (temp_str)) {
-                priv->op_mode = GF_CDC_MODE_CLIENT;
-        } else if (GF_CDC_MODE_IS_SERVER (temp_str)) {
-                priv->op_mode = GF_CDC_MODE_SERVER;
-        } else {
-                gf_log (this->name, GF_LOG_CRITICAL,
-                        "Bogus operation mode (%s) specified", temp_str);
-                goto err;
-        }
-
-        this->private = priv;
-        gf_log (this->name, GF_LOG_DEBUG, "CDC xlator loaded in (%s) mode",temp_str);
-        return 0;
-
- err:
-        if (priv)
-                GF_FREE (priv);
-
-        return -1;
+    int ret = -1;
+    char *temp_str = NULL;
+    cdc_priv_t *priv = NULL;
+
+    GF_VALIDATE_OR_GOTO("cdc", this, err);
+
+    if (!this->children || this->children->next) {
+        gf_log(this->name, GF_LOG_ERROR, "Need subvolume == 1");
+        goto err;
+    }
+
+    if (!this->parents) {
+        gf_log(this->name, GF_LOG_WARNING, "Dangling volume. Check volfile");
+    }
+
+    priv = GF_CALLOC(1, sizeof(*priv), gf_cdc_mt_priv_t);
+    if (!priv) {
+        goto err;
+    }
+
+    /* Check if debug mode is turned on */
+    GF_OPTION_INIT("debug", priv->debug, bool, err);
+    if (priv->debug) {
+        gf_log(this->name, GF_LOG_DEBUG, "CDC debug option turned on");
+    }
+
+    /* Set Gzip Window Size */
+    GF_OPTION_INIT("window-size", priv->window_size, int32, err);
+    if ((priv->window_size > GF_CDC_MAX_WINDOWSIZE) ||
+        (priv->window_size < GF_CDC_DEF_WINDOWSIZE)) {
+        gf_log(this->name, GF_LOG_WARNING,
+               "Invalid gzip window size (%d), using default",
+               priv->window_size);
+        priv->window_size = GF_CDC_DEF_WINDOWSIZE;
+    }
+
+    /* Set Gzip (De)Compression Level */
+    GF_OPTION_INIT("compression-level", priv->cdc_level, int32, err);
+    if (((priv->cdc_level < 1) || (priv->cdc_level > 9)) &&
+        (priv->cdc_level != GF_CDC_DEF_COMPRESSION)) {
+        gf_log(this->name, GF_LOG_WARNING,
+               "Invalid gzip (de)compression level (%d),"
+               " using default",
+               priv->cdc_level);
+        priv->cdc_level = GF_CDC_DEF_COMPRESSION;
+    }
+
+    /* Set Gzip Memory Level */
+    GF_OPTION_INIT("mem-level", priv->mem_level, int32, err);
+    if ((priv->mem_level < 1) || (priv->mem_level > 9)) {
+        gf_log(this->name, GF_LOG_WARNING,
+               "Invalid gzip memory level, using the default");
+        priv->mem_level = GF_CDC_DEF_MEMLEVEL;
+    }
+
+    /* Set min file size to enable compression */
+    GF_OPTION_INIT("min-size", priv->min_file_size, int32, err);
+
+    /* Mode of operation - Server/Client */
+    ret = dict_get_str(this->options, "mode", &temp_str);
+    if (ret) {
+        gf_log(this->name, GF_LOG_CRITICAL, "Operation mode not specified !!");
+        goto err;
+    }
+
+    if (GF_CDC_MODE_IS_CLIENT(temp_str)) {
+        priv->op_mode = GF_CDC_MODE_CLIENT;
+    } else if (GF_CDC_MODE_IS_SERVER(temp_str)) {
+        priv->op_mode = GF_CDC_MODE_SERVER;
+    } else {
+        gf_log(this->name, GF_LOG_CRITICAL,
+               "Bogus operation mode (%s) specified", temp_str);
+        goto err;
+    }
+
+    this->private = priv;
+    gf_log(this->name, GF_LOG_DEBUG, "CDC xlator loaded in (%s) mode",
+           temp_str);
+    return 0;
+
+err:
+    if (priv)
+        GF_FREE(priv);
+
+    return -1;
 }
 
 void
-fini (xlator_t *this)
+fini(xlator_t *this)
 {
-        cdc_priv_t *priv = this->private;
+    cdc_priv_t *priv = this->private;
 
-        if (priv)
-                GF_FREE (priv);
-        this->private = NULL;
-        return;
+    if (priv)
+        GF_FREE(priv);
+    this->private = NULL;
+    return;
 }
 
 struct xlator_fops fops = {
-        .readv  = cdc_readv,
-        .writev = cdc_writev,
+    .readv = cdc_readv,
+    .writev = cdc_writev,
 };
 
-struct xlator_cbks cbks = {
-};
+struct xlator_cbks cbks = {};
 
 struct volume_options options[] = {
-        { .key  = {"window-size"},
-          .default_value = "-15",
-          .type = GF_OPTION_TYPE_INT,
-          .description = "Size of the zlib history buffer."
-        },
-        { .key  = {"mem-level"},
-          .default_value = "8",
-          .type = GF_OPTION_TYPE_INT,
-          .description = "Memory allocated for internal compression state. "
-                         "1 uses minimum memory but is slow and reduces "
-                         "compression ratio; memLevel=9 uses maximum memory "
-                         "for optimal speed. The default value is 8."
-        },
-        { .key  = {"compression-level"},
-          .default_value = "-1",
-          .type = GF_OPTION_TYPE_INT,
-          .description = "Compression levels \n"
-                         "0 : no compression, 1 : best speed, \n"
-                         "9 : best compression, -1 : default compression "
-        },
-        { .key  = {"min-size"},
-          .default_value = "0",
-          .type = GF_OPTION_TYPE_INT,
-          .description = "Data is compressed only when its size exceeds this."
-        },
-        { .key  = {"mode"},
-          .value = {"server", "client"},
-          .type = GF_OPTION_TYPE_STR,
-          .description = "Set on the basis of where the xlator is loaded. "
-                         "This option should NOT be configured by user."
-        },
-        { .key = {"debug"},
-          .default_value = "false",
-          .type = GF_OPTION_TYPE_BOOL,
-          .description = "This is used in testing. Will dump compressed data "
-                         "to disk as a gzip file."
-        },
-        { .key  = {NULL}
-        },
+    {.key = {"window-size"},
+     .default_value = "-15",
+     .type = GF_OPTION_TYPE_INT,
+     .description = "Size of the zlib history buffer."},
+    {.key = {"mem-level"},
+     .default_value = "8",
+     .type = GF_OPTION_TYPE_INT,
+     .description = "Memory allocated for internal compression state. "
+                    "1 uses minimum memory but is slow and reduces "
+                    "compression ratio; memLevel=9 uses maximum memory "
+                    "for optimal speed. The default value is 8."},
+    {.key = {"compression-level"},
+     .default_value = "-1",
+     .type = GF_OPTION_TYPE_INT,
+     .description = "Compression levels \n"
+                    "0 : no compression, 1 : best speed, \n"
+                    "9 : best compression, -1 : default compression "},
+    {.key = {"min-size"},
+     .default_value = "0",
+     .type = GF_OPTION_TYPE_INT,
+     .description = "Data is compressed only when its size exceeds this."},
+    {.key = {"mode"},
+     .value = {"server", "client"},
+     .type = GF_OPTION_TYPE_STR,
+     .description = "Set on the basis of where the xlator is loaded. "
+                    "This option should NOT be configured by user."},
+    {.key = {"debug"},
+     .default_value = "false",
+     .type = GF_OPTION_TYPE_BOOL,
+     .description = "This is used in testing. Will dump compressed data "
+                    "to disk as a gzip file."},
+    {.key = {NULL}},
+};
+
+xlator_api_t xlator_api = {
+    .init = init,
+    .fini = fini,
+    .mem_acct_init = mem_acct_init,
+    .op_version = {GD_OP_VERSION_3_9_0},
+    .fops = &fops,
+    .cbks = &cbks,
+    .options = options,
+    .identifier = "cdc",
+    .category = GF_TECH_PREVIEW,
 };
diff --git a/xlators/features/compress/src/cdc.h b/xlators/features/compress/src/cdc.h
index 71f4d2317bb..cb87b06a989 100644
--- a/xlators/features/compress/src/cdc.h
+++ b/xlators/features/compress/src/cdc.h
@@ -15,41 +15,41 @@
 #include "zlib.h"
 #endif
 
-#include "xlator.h"
+#include <glusterfs/xlator.h>
 
 #ifndef MAX_IOVEC
 #define MAX_IOVEC 16
 #endif
 
 typedef struct cdc_priv {
-        int window_size;
-        int mem_level;
-        int cdc_level;
-        int min_file_size;
-        int op_mode;
-        gf_boolean_t debug;
-        gf_lock_t lock;
+    int window_size;
+    int mem_level;
+    int cdc_level;
+    int min_file_size;
+    int op_mode;
+    gf_boolean_t debug;
+    gf_lock_t lock;
 } cdc_priv_t;
 
 typedef struct cdc_info {
-        /* input bits */
-        int            count;
-        int32_t        ibytes;
-        struct iovec  *vector;
-        struct iatt   *buf;
-
-        /* output bits */
-        int            ncount;
-        int            nbytes;
-        int            buffer_size;
-        struct iovec   vec[MAX_IOVEC];
-        struct iobref *iobref;
-
-        /* zlib bits */
+    /* input bits */
+    int count;
+    int32_t ibytes;
+    struct iovec *vector;
+    struct iatt *buf;
+
+    /* output bits */
+    int ncount;
+    int nbytes;
+    int buffer_size;
+    struct iovec vec[MAX_IOVEC];
+    struct iobref *iobref;
+
+    /* zlib bits */
 #ifdef HAVE_LIB_Z
-        z_stream      stream;
+    z_stream stream;
 #endif
-        unsigned long crc;
+    unsigned long crc;
 } cdc_info_t;
 
 #define NVEC(ci) (ci->ncount - 1)
@@ -57,8 +57,8 @@ typedef struct cdc_info {
 #define THIS_VEC(ci, i) ci->vector[i]
 
 /* Gzip defaults */
-#define GF_CDC_DEF_WINDOWSIZE  -15 /* default value */
-#define GF_CDC_MAX_WINDOWSIZE  -8  /* max value     */
+#define GF_CDC_DEF_WINDOWSIZE -15 /* default value */
+#define GF_CDC_MAX_WINDOWSIZE -8  /* max value     */
 
 #ifdef HAVE_LIB_Z
 #define GF_CDC_DEF_COMPRESSION Z_DEFAULT_COMPRESSION
@@ -66,15 +66,15 @@ typedef struct cdc_info {
 #define GF_CDC_DEF_COMPRESSION -1
 #endif
 
-#define GF_CDC_DEF_MEMLEVEL    8
-#define GF_CDC_DEF_BUFFERSIZE  262144 // 256K - default compression buffer size
+#define GF_CDC_DEF_MEMLEVEL 8
+#define GF_CDC_DEF_BUFFERSIZE 262144  // 256K - default compression buffer size
 
 /* Operation mode
  * If xlator is loaded on client, readv decompresses and writev compresses
  * If xlator is loaded on server, readv compresses and writev decompresses
  */
-#define GF_CDC_MODE_CLIENT   0
-#define GF_CDC_MODE_SERVER   1
+#define GF_CDC_MODE_CLIENT 0
+#define GF_CDC_MODE_SERVER 1
 
 /* min size of data to do cmpression
  * 0 == compress even 1byte
@@ -87,21 +87,13 @@ typedef struct cdc_info {
 #define GF_CDC_DEFLATE_CANARY_VAL "deflate"
 #define GF_CDC_DEBUG_DUMP_FILE "/tmp/cdcdump.gz"
 
-#define GF_CDC_MODE_IS_CLIENT(m) \
-        (strcmp (m, "client") == 0)
+#define GF_CDC_MODE_IS_CLIENT(m) (strcmp(m, "client") == 0)
 
-#define GF_CDC_MODE_IS_SERVER(m) \
-        (strcmp (m, "server") == 0)
+#define GF_CDC_MODE_IS_SERVER(m) (strcmp(m, "server") == 0)
 
 int32_t
-cdc_compress (xlator_t *this,
-              cdc_priv_t *priv,
-              cdc_info_t *ci,
-              dict_t **xdata);
+cdc_compress(xlator_t *this, cdc_priv_t *priv, cdc_info_t *ci, dict_t **xdata);
 int32_t
-cdc_decompress (xlator_t *this,
-                cdc_priv_t *priv,
-                cdc_info_t *ci,
-                dict_t *xdata);
+cdc_decompress(xlator_t *this, cdc_priv_t *priv, cdc_info_t *ci, dict_t *xdata);
 
 #endif
diff --git a/xlators/features/filter/Makefile.am b/xlators/features/filter/Makefile.am
deleted file mode 100644
index d471a3f9243..00000000000
--- a/xlators/features/filter/Makefile.am
+++ /dev/null
@@ -1,3 +0,0 @@
-SUBDIRS = src
-
-CLEANFILES = 
diff --git a/xlators/features/filter/src/Makefile.am b/xlators/features/filter/src/Makefile.am
deleted file mode 100644
index d1fda8b0a9d..00000000000
--- a/xlators/features/filter/src/Makefile.am
+++ /dev/null
@@ -1,16 +0,0 @@
-xlator_LTLIBRARIES = filter.la
-xlatordir = $(libdir)/glusterfs/$(PACKAGE_VERSION)/xlator/testing/features
-
-filter_la_LDFLAGS = -module -avoid-version
-
-filter_la_SOURCES = filter.c
-filter_la_LIBADD = $(top_builddir)/libglusterfs/src/libglusterfs.la 
-
-noinst_HEADERS = filter-mem-types.h
-
-AM_CPPFLAGS = $(GF_CPPFLAGS) -I$(top_srcdir)/libglusterfs/src
-
-AM_CFLAGS = -Wall $(GF_CFLAGS)
-
-CLEANFILES = 
-
diff --git a/xlators/features/filter/src/filter.c b/xlators/features/filter/src/filter.c
deleted file mode 100644
index 1d4887b7143..00000000000
--- a/xlators/features/filter/src/filter.c
+++ /dev/null
@@ -1,1734 +0,0 @@
-/*
-   Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
-   This file is part of GlusterFS.
-
-   This file is licensed to you under your choice of the GNU Lesser
-   General Public License, version 3 or any later version (LGPLv3 or
-   later), or the GNU General Public License, version 2 (GPLv2), in all
-   cases as published by the Free Software Foundation.
-*/
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
-
-#include <sys/types.h>
-#include <sys/stat.h>
-#include <unistd.h>
-
-#include "glusterfs.h"
-#include "logging.h"
-#include "dict.h"
-#include "xlator.h"
-#include "filter-mem-types.h"
-
-#define GF_FILTER_NOBODY_UID         65534
-#define GF_FILTER_NOBODY_GID         65534
-#define GF_FILTER_ROOT_UID           0
-#define GF_FILTER_ROOT_GID           0
-
-#define GF_MAXIMUM_FILTERING_ALLOWED 32
-
-/*
-  option root-filtering on (off by default)
-  option translate-uid <uid-range=newuid,uid=newuid>
-  option translate-gid <gid-range=newgid,gid=newgid>
-  option read-only <yes|true>
-  option fixed-uid <uid>
-  option fixed-gid <gid>
-  option filter-uid <uid-range,uid>
-  option filter-gid <gid-range,gid> // not supported yet
-
-*/
-
-struct gf_filter {
-	/* Flags */
-	gf_boolean_t complete_read_only;
-	char fixed_uid_set;
-	char fixed_gid_set;
-	char partial_filter;
-
-	/* Options */
-	/* Mapping/Filtering/Translate whatever you want to call */
-	int translate_num_uid_entries;
-	int translate_num_gid_entries;
-	int translate_input_uid[GF_MAXIMUM_FILTERING_ALLOWED][2];
-	int translate_output_uid[GF_MAXIMUM_FILTERING_ALLOWED];
-	int translate_input_gid[GF_MAXIMUM_FILTERING_ALLOWED][2];
-	int translate_output_gid[GF_MAXIMUM_FILTERING_ALLOWED];
-
-	/* Fixed uid/gid */
-	int fixed_uid;
-	int fixed_gid;
-
-	/* Filter */
-	int filter_num_uid_entries;
-	int filter_num_gid_entries;
-	int filter_input_uid[GF_MAXIMUM_FILTERING_ALLOWED][2];
-	int filter_input_gid[GF_MAXIMUM_FILTERING_ALLOWED][2];
-	
-};
-
-/* update_frame: The main logic of the whole translator.
-   Return values:
-   0: no change
-   // TRANSLATE
-   1: only uid changed 
-   2: only gid changed
-   3: both uid/gid changed
-   // FILTER
-   4: uid in filter range
-   5: gid in filter range  // not supported yet
-   6: complete fs is readonly
-*/
-
-#define GF_FILTER_NO_CHANGE    0
-#define	GF_FILTER_MAP_UID      1
-#define GF_FILTER_MAP_GID      2
-#define GF_FILTER_MAP_BOTH     3
-#define GF_FILTER_FILTER_UID   4
-#define GF_FILTER_FILTER_GID   5
-#define GF_FILTER_RO_FS        6
-
-static int32_t
-update_frame (call_frame_t *frame,
-	      inode_t *inode,
-	      struct gf_filter *filter)
-{
-	uid_t    uid = 0;
-	int32_t  idx = 0;
-	int32_t  ret = 0;
-	int32_t  dictret = 0;
-	uint64_t tmp_uid = 0;
-	
-	for (idx = 0; idx < filter->translate_num_uid_entries; idx++) {
-		if ((frame->root->uid >=filter->translate_input_uid[idx][0]) &&
-		    (frame->root->uid <=filter->translate_input_uid[idx][1])) {
-			dictret = inode_ctx_get (inode, frame->this, &tmp_uid);
-			uid = (uid_t)tmp_uid;
-			if (dictret == 0) {
-				if (frame->root->uid != uid)
-					ret = GF_FILTER_MAP_UID;
-			} else {
-				ret = GF_FILTER_MAP_UID;
-			}
-			break;
-		}
-	}
-	
-	for (idx = 0; idx < filter->translate_num_gid_entries; idx++) {
-		if ((frame->root->gid >=filter->translate_input_gid[idx][0]) &&
-		    (frame->root->gid <=filter->translate_input_gid[idx][1])) {
-			if (ret == GF_FILTER_NO_CHANGE) 
-				ret = GF_FILTER_MAP_GID;
-			else 
-				ret = GF_FILTER_MAP_BOTH;
-			break;
-		}
-	}
-
-
-	if (filter->complete_read_only)
-		return GF_FILTER_RO_FS;
-	
-	if (filter->partial_filter) {
-		dictret = inode_ctx_get (inode, frame->this, &tmp_uid);
-		uid = (uid_t)tmp_uid;
-		if (dictret != -1) {
-			for (idx = 0; idx < filter->filter_num_uid_entries; 
-			     idx++) {
-				if ((uid >=filter->filter_input_uid[idx][0]) &&
-				    (uid <=filter->filter_input_uid[idx][1])) {
-					return GF_FILTER_FILTER_UID;
-				}
-			}
-		}
-	}
-
-	return ret;
-}
-
-/* if 'root' don't change the uid/gid */
-static int32_t
-update_stat (struct iatt *stbuf,
-	     struct gf_filter *filter)
-{
-	int32_t idx = 0;
-	for (idx = 0; idx < filter->translate_num_uid_entries; idx++) {
-		if (stbuf->ia_uid == GF_FILTER_ROOT_UID)
-			continue;
-		if ((stbuf->ia_uid >= filter->translate_input_uid[idx][0]) &&
-		    (stbuf->ia_uid <= filter->translate_input_uid[idx][1])) {
-			stbuf->ia_uid = filter->translate_output_uid[idx];
-			break;
-		}
-	}
-	
-	for (idx = 0; idx < filter->translate_num_gid_entries; idx++) {
-		if (stbuf->ia_gid == GF_FILTER_ROOT_GID)
-			continue;
-		if ((stbuf->ia_gid >= filter->translate_input_gid[idx][0]) &&
-		    (stbuf->ia_gid <= filter->translate_input_gid[idx][1])) {
-			stbuf->ia_gid = filter->translate_output_gid[idx];
-			break;
-		}
-	}
-
-	if (filter->fixed_uid_set) {
-		stbuf->ia_uid = filter->fixed_uid;
-	}
-
-	if (filter->fixed_gid_set) {
-		stbuf->ia_gid = filter->fixed_gid;
-	}
-	
-	return 0;
-}
-
-static int32_t 
-filter_lookup_cbk (call_frame_t *frame,
-		   void *cookie,
-		   xlator_t *this,
-		   int32_t op_ret,
-		   int32_t op_errno,
-		   inode_t *inode,
-		   struct iatt *buf,
-		   dict_t *dict,
-                   struct iatt *postparent)
-{
-	int ret = 0;
-	if (op_ret >= 0) {
-		update_stat (buf, this->private);
-		ret = inode_ctx_put (inode, this, (uint64_t)(long)buf->ia_uid);
-		if (ret == -1) {
-			gf_log (this->name, GF_LOG_ERROR,
-				"couldn't set context");
-		}
-
-		update_stat (postparent, this->private);
-	}
-	STACK_UNWIND (frame, op_ret, op_errno, inode, buf, dict, postparent);
-	return 0;
-}
-
-int32_t 
-filter_lookup (call_frame_t *frame,
-	       xlator_t *this,
-	       loc_t *loc,
-	       dict_t *xattr_req)
-{
-	STACK_WIND (frame,
-		    filter_lookup_cbk,
-		    FIRST_CHILD(this),
-		    FIRST_CHILD(this)->fops->lookup,
-		    loc,
-		    xattr_req);
-	return 0;
-}
-
-
-static int32_t
-filter_stat_cbk (call_frame_t *frame,
-		 void *cookie,
-		 xlator_t *this,
-		 int32_t op_ret,
-		 int32_t op_errno,
-		 struct iatt *buf)
-{
-	if (op_ret >= 0) {
-		update_stat (buf, this->private);
-	}
-	STACK_UNWIND (frame, op_ret, op_errno, buf);
-	return 0;
-}
-
-int32_t
-filter_stat (call_frame_t *frame,
-	     xlator_t *this,
-	     loc_t *loc)
-{
-	STACK_WIND (frame,
-		    filter_stat_cbk,
-		    FIRST_CHILD(this),
-		    FIRST_CHILD(this)->fops->stat,
-		    loc);
-	return 0;
-}
-
-static int32_t
-filter_setattr_cbk (call_frame_t *frame,
-                    void *cookie,
-                    xlator_t *this,
-                    int32_t op_ret,
-                    int32_t op_errno,
-                    struct iatt *preop,
-                    struct iatt *postop)
-{
-	if (op_ret >= 0) {
-		update_stat (preop, this->private);
-		update_stat (postop, this->private);
-	}
-	STACK_UNWIND (frame, op_ret, op_errno, preop, postop);
-	return 0;
-}
-
-int32_t
-filter_setattr (call_frame_t *frame,
-                xlator_t *this,
-                loc_t *loc,
-                struct iatt *stbuf,
-                int32_t valid)
-{
-	int32_t ret = 0;
-	ret = update_frame (frame, loc->inode, this->private);
-	switch (ret) {
-	case GF_FILTER_MAP_UID:
-		if (loc->inode->st_mode & S_IWGRP)
-			break;
-	case GF_FILTER_MAP_BOTH:
-		if (loc->inode->st_mode & S_IWOTH)
-			break;
-		gf_log (this->name, GF_LOG_DEBUG,
-                        "%s: returning permission denied", loc->path);
-		STACK_UNWIND (frame, -1, EPERM, NULL, NULL, NULL);
-		return 0;
-
-	case GF_FILTER_FILTER_UID:
-	case GF_FILTER_FILTER_GID:
-	case GF_FILTER_RO_FS:
-		STACK_UNWIND (frame, -1, EROFS, NULL, NULL);
-		return 0;
-	default:
-		break;
-	}
-
-	STACK_WIND (frame,
-		    filter_setattr_cbk,
-		    FIRST_CHILD(this),
-		    FIRST_CHILD(this)->fops->setattr,
-		    loc,
-		    stbuf, valid);
-	return 0;
-}
-
-static int32_t
-filter_fsetattr_cbk (call_frame_t *frame,
-                     void *cookie,
-                     xlator_t *this,
-                     int32_t op_ret,
-                     int32_t op_errno,
-                     struct iatt *preop,
-                     struct iatt *postop)
-{
-	if (op_ret >= 0) {
-                update_stat (preop, this->private);
-		update_stat (postop, this->private);
-	}
-	STACK_UNWIND (frame,
-		      op_ret,
-		      op_errno,
-		      preop, postop);
-	return 0;
-}
-
-int32_t
-filter_fsetattr (call_frame_t *frame,
-                 xlator_t *this,
-                 fd_t *fd,
-                 struct iatt *stbuf,
-                 int32_t valid)
-{
-	STACK_WIND (frame,
-		    filter_fsetattr_cbk,
-		    FIRST_CHILD(this),
-		    FIRST_CHILD(this)->fops->fsetattr,
-		    fd,
-		    stbuf, valid);
-	return 0;
-}
-
-
-static int32_t
-filter_truncate_cbk (call_frame_t *frame,
-		     void *cookie,
-		     xlator_t *this,
-		     int32_t op_ret,
-		     int32_t op_errno,
-		     struct iatt *prebuf,
-                     struct iatt *postbuf)
-{
-	if (op_ret >= 0) {
-		update_stat (prebuf, this->private);
-		update_stat (postbuf, this->private);
-	}
-	STACK_UNWIND (frame, op_ret, op_errno, prebuf, postbuf);
-	return 0;
-}
-
-int32_t
-filter_truncate (call_frame_t *frame,
-		 xlator_t *this,
-		 loc_t *loc,
-		 off_t offset)
-{
-	int32_t ret = 0;
-	ret = update_frame (frame, loc->inode, this->private);
-	switch (ret) {
-	case GF_FILTER_MAP_UID:
-		if (loc->inode->st_mode & S_IWGRP)
-			break;
-	case GF_FILTER_MAP_BOTH:
-		if (loc->inode->st_mode & S_IWOTH)
-			break;
-		gf_log (this->name, GF_LOG_DEBUG, "%s: returning permission denied", loc->path);
-		STACK_UNWIND (frame, -1, EPERM, NULL, NULL);
-		return 0;
-		
-	case GF_FILTER_FILTER_UID:
-	case GF_FILTER_FILTER_GID:
-	case GF_FILTER_RO_FS:
-		STACK_UNWIND (frame, -1, EROFS, NULL, NULL);
-		return 0;
-	}			
-
-	STACK_WIND (frame,
-		    filter_truncate_cbk,
-		    FIRST_CHILD(this),
-		    FIRST_CHILD(this)->fops->truncate,
-		    loc,
-		    offset);
-	return 0;
-}
-
-static int32_t
-filter_ftruncate_cbk (call_frame_t *frame,
-		      void *cookie,
-		      xlator_t *this,
-		      int32_t op_ret,
-		      int32_t op_errno,
-		      struct iatt *prebuf,
-                      struct iatt *postbuf)
-{
-	if (op_ret >= 0) {
-		update_stat (prebuf, this->private);
-		update_stat (postbuf, this->private);
-	}
-	STACK_UNWIND (frame, op_ret, op_errno, prebuf, postbuf);
-	return 0;
-}
-
-int32_t
-filter_ftruncate (call_frame_t *frame,
-		  xlator_t *this,
-		  fd_t *fd,
-		  off_t offset)
-{
-	STACK_WIND (frame,
-		    filter_ftruncate_cbk,
-		    FIRST_CHILD(this),
-		    FIRST_CHILD(this)->fops->ftruncate,
-		    fd,
-		    offset);
-	return 0;
-}
-
-
-static int32_t
-filter_readlink_cbk (call_frame_t *frame,
-		     void *cookie,
-		     xlator_t *this,
-		     int32_t op_ret,
-		     int32_t op_errno,
-		     const char *path,
-                     struct iatt *sbuf)
-{
-        if (op_ret >= 0)
-                update_stat (sbuf, this->private);
-
-	STACK_UNWIND (frame, op_ret, op_errno, path, sbuf);
-	return 0;
-}
-
-int32_t
-filter_readlink (call_frame_t *frame,
-		 xlator_t *this,
-		 loc_t *loc,
-		 size_t size)
-{
-	int32_t ret = 0;
-	ret = update_frame (frame, loc->inode, this->private);
-	switch (ret) {
-	case GF_FILTER_MAP_UID:
-		if (loc->inode->st_mode & S_IRGRP)
-			break;
-	case GF_FILTER_MAP_BOTH:
-		if (loc->inode->st_mode & S_IROTH)
-			break;
-		gf_log (this->name, GF_LOG_DEBUG, "%s: returning permission denied", loc->path);
-		STACK_UNWIND (frame, -1, EPERM, NULL);
-		return 0;
-	}			
-	STACK_WIND (frame,
-		    filter_readlink_cbk,
-		    FIRST_CHILD(this),
-		    FIRST_CHILD(this)->fops->readlink,
-		    loc,
-		    size);
-	return 0;
-}
-
-
-static int32_t
-filter_mknod_cbk (call_frame_t *frame,
-		  void *cookie,
-		  xlator_t *this,
-		  int32_t op_ret,
-		  int32_t op_errno,
-		  inode_t *inode,
-                  struct iatt *buf,
-                  struct iatt *preparent,
-                  struct iatt *postparent)
-{
-	int ret = 0;
-
-	if (op_ret >= 0) {
-		update_stat (buf, this->private);
-		ret = inode_ctx_put (inode, this, (uint64_t)(long)buf->ia_uid);
-		if (ret == -1) {
-			gf_log (this->name, GF_LOG_ERROR,
-				"couldn't set context");
-		}
-
-		update_stat (preparent, this->private);
-		update_stat (postparent, this->private);
-	}
-	STACK_UNWIND (frame, op_ret, op_errno, inode, buf,
-                      preparent, postparent);
-	return 0;
-}
-
-int32_t
-filter_mknod (call_frame_t *frame,
-	      xlator_t *this,
-	      loc_t *loc,
-	      mode_t mode,
-	      dev_t rdev)
-{
-	int ret = 0;
-	inode_t *parent = loc->parent;
-	ret = update_frame (frame, loc->inode, this->private);
-	switch (ret) {		
-	case GF_FILTER_MAP_UID:
-		if (parent->st_mode & S_IWGRP)
-			break;
-	case GF_FILTER_MAP_BOTH:
-		if (parent->st_mode & S_IWOTH)
-			break;
-		gf_log (this->name, GF_LOG_DEBUG, "%s: returning permission denied", loc->path);
-		STACK_UNWIND (frame, -1, EPERM, NULL, NULL,
-                              NULL, NULL);
-		return 0;
-
-	case GF_FILTER_FILTER_UID:
-	case GF_FILTER_FILTER_GID:
-	case GF_FILTER_RO_FS:
-		STACK_UNWIND (frame, -1, EROFS, NULL, NULL,
-                              NULL, NULL);
-		return 0;
-	}
-	STACK_WIND (frame,
-		    filter_mknod_cbk,
-		    FIRST_CHILD(this),
-		    FIRST_CHILD(this)->fops->mknod,
-		    loc, mode, rdev);
-	return 0;
-}
-
-static int32_t
-filter_mkdir_cbk (call_frame_t *frame,
-		  void *cookie,
-		  xlator_t *this,
-		  int32_t op_ret,
-		  int32_t op_errno,
-		  inode_t *inode,
-                  struct iatt *buf,
-                  struct iatt *preparent,
-                  struct iatt *postparent)
-{
-	int ret = 0;
-	if (op_ret >= 0) {
-		update_stat (buf, this->private);
-		ret = inode_ctx_put (inode, this, (uint64_t)(long)buf->ia_uid);
-		if (ret == -1) {
-			gf_log (this->name, GF_LOG_ERROR,
-				"couldn't set context");
-		}
-
-		update_stat (preparent, this->private);
-		update_stat (postparent, this->private);
-	}
-	STACK_UNWIND (frame, op_ret, op_errno, inode, buf,
-                      preparent, postparent);
-	return 0;
-}
-
-int32_t
-filter_mkdir (call_frame_t *frame,
-	      xlator_t *this,
-	      loc_t *loc,
-	      mode_t mode)
-{
-	int ret = 0;
-	inode_t *parent = loc->parent;
-	ret = update_frame (frame, loc->inode, this->private);
-	switch (ret) {		
-	case GF_FILTER_MAP_UID:
-		if (parent->st_mode & S_IWGRP)
-			break;
-	case GF_FILTER_MAP_BOTH:
-		if (parent->st_mode & S_IWOTH)
-			break;
-		gf_log (this->name, GF_LOG_DEBUG, "%s: returning permission denied", loc->path);
-		STACK_UNWIND (frame, -1, EPERM, NULL, NULL,
-                              NULL, NULL);
-		return 0;
-
-	case GF_FILTER_FILTER_UID:
-	case GF_FILTER_FILTER_GID:
-	case GF_FILTER_RO_FS:
-		STACK_UNWIND (frame, -1, EROFS, NULL, NULL,
-                              NULL, NULL);
-		return 0;
-	}
-	STACK_WIND (frame,
-		    filter_mkdir_cbk,
-		    FIRST_CHILD(this),
-		    FIRST_CHILD(this)->fops->mkdir,
-		    loc, mode);
-	return 0;
-}
-
-static int32_t
-filter_unlink_cbk (call_frame_t *frame,
-		   void *cookie,
-		   xlator_t *this,
-		   int32_t op_ret,
-		   int32_t op_errno,
-                   struct iatt *preparent,
-                   struct iatt *postparent)
-{
-        if (op_ret >= 0) {
-		update_stat (preparent, this->private);
-		update_stat (postparent, this->private);
-        }
-
-	STACK_UNWIND (frame, op_ret, op_errno, preparent, postparent);
-	return 0;
-}
-
-int32_t
-filter_unlink (call_frame_t *frame,
-	       xlator_t *this,
-	       loc_t *loc)
-{
-	int32_t ret = 0;
-	inode_t *parent = loc->parent;
-	if (!parent)
-		parent = inode_parent (loc->inode, 0, NULL);
-	ret = update_frame (frame, loc->inode, this->private);
-	switch (ret) {
-	case GF_FILTER_MAP_UID:
-		if (parent->st_mode & S_IWGRP)
-			break;
-		if (loc->inode->st_mode & S_IWGRP)
-			break;
-	case GF_FILTER_MAP_BOTH:
-		if (parent->st_mode & S_IWOTH)
-			break;
-		if (loc->inode->st_mode & S_IWOTH)
-			break;
-		gf_log (this->name, GF_LOG_DEBUG, "%s: returning permission denied", loc->path);
-		STACK_UNWIND (frame, -1, EPERM, NULL, NULL);
-		return 0;
-	case GF_FILTER_FILTER_UID:
-	case GF_FILTER_FILTER_GID:
-	case GF_FILTER_RO_FS:
-		STACK_UNWIND (frame, -1, EROFS, NULL, NULL);
-		return 0;
-	}
-	STACK_WIND (frame,
-		    filter_unlink_cbk,
-		    FIRST_CHILD(this),
-		    FIRST_CHILD(this)->fops->unlink,
-		    loc);
-	return 0;
-}
-
-static int32_t
-filter_rmdir_cbk (call_frame_t *frame,
-		  void *cookie,
-		  xlator_t *this,
-		  int32_t op_ret,
-		  int32_t op_errno,
-                  struct iatt *preparent,
-                  struct iatt *postparent)
-{
-        if (op_ret >= 0) {
-		update_stat (preparent, this->private);
-		update_stat (postparent, this->private);
-        }
-
-	STACK_UNWIND (frame, op_ret, op_errno, preparent, postparent);
-	return 0;
-}
-
-int32_t
-filter_rmdir (call_frame_t *frame,
-	      xlator_t *this,
-	      loc_t *loc)
-{
-	int32_t ret = 0;
-	inode_t *parent = loc->parent;
-	if (!parent)
-		parent = inode_parent (loc->inode, 0, NULL);
-	ret = update_frame (frame, loc->inode, this->private);
-	switch (ret) {
-	case GF_FILTER_MAP_UID:
-		if (parent->st_mode & S_IWGRP)
-			break;
-		if (loc->inode->st_mode & S_IWGRP)
-			break;
-	case GF_FILTER_MAP_BOTH:
-		if (parent->st_mode & S_IWOTH)
-			break;
-		if (loc->inode->st_mode & S_IWOTH)
-			break;
-		gf_log (this->name, GF_LOG_DEBUG, "%s: returning permission denied", loc->path);
-		STACK_UNWIND (frame, -1, EPERM, NULL, NULL);
-		return 0;
-	case GF_FILTER_FILTER_UID:
-	case GF_FILTER_FILTER_GID:
-	case GF_FILTER_RO_FS:
-                STACK_UNWIND (frame, -1, EROFS, NULL, NULL);
-                return 0;
-	}			
-	STACK_WIND (frame,
-		    filter_rmdir_cbk,
-		    FIRST_CHILD(this),
-		    FIRST_CHILD(this)->fops->rmdir,
-		    loc);
-	return 0;
-}
-
-static int32_t
-filter_symlink_cbk (call_frame_t *frame,
-		    void *cookie,
-		    xlator_t *this,
-		    int32_t op_ret,
-		    int32_t op_errno,
-		    inode_t *inode,
-                    struct iatt *buf,
-                    struct iatt *preparent,
-                    struct iatt *postparent)
-{
-	int ret = 0;
-	if (op_ret >= 0) {
-		update_stat (buf, this->private);
-		ret = inode_ctx_put (inode, this, (uint64_t)(long)buf->ia_uid);
-		if (ret == -1) {
-			gf_log (this->name, GF_LOG_ERROR,
-				"couldn't set context");
-		}
-
-		update_stat (preparent, this->private);
-		update_stat (postparent, this->private);
-	}
-	STACK_UNWIND (frame, op_ret, op_errno, inode, buf,
-                      preparent, postparent);
-	return 0;
-}
-
-int32_t
-filter_symlink (call_frame_t *frame,
-		xlator_t *this,
-		const char *linkpath,
-		loc_t *loc)
-{
-	int ret = 0;
-	inode_t *parent = loc->parent;
-	ret = update_frame (frame, loc->inode, this->private);
-	switch (ret) {		
-	case GF_FILTER_MAP_UID:
-		if (parent->st_mode & S_IWGRP)
-			break;
-	case GF_FILTER_MAP_BOTH:
-		if (parent->st_mode & S_IWOTH)
-			break;
-		gf_log (this->name, GF_LOG_DEBUG, "%s: returning permission denied", loc->path);
-		STACK_UNWIND (frame, -1, EPERM, NULL, NULL,
-                              NULL, NULL);
-		return 0;
-
-	case GF_FILTER_FILTER_UID:
-	case GF_FILTER_FILTER_GID:
-	case GF_FILTER_RO_FS:
-		STACK_UNWIND (frame, -1, EROFS, NULL, NULL,
-                              NULL, NULL);
-		return 0;
-	}
-	STACK_WIND (frame,
-		    filter_symlink_cbk,
-		    FIRST_CHILD(this),
-		    FIRST_CHILD(this)->fops->symlink,
-		    linkpath, loc);
-	return 0;
-}
-
-
-static int32_t
-filter_rename_cbk (call_frame_t *frame,
-		   void *cookie,
-		   xlator_t *this,
-		   int32_t op_ret,
-		   int32_t op_errno,
-		   struct iatt *buf,
-                   struct iatt *preoldparent,
-                   struct iatt *postoldparent,
-                   struct iatt *prenewparent,
-                   struct iatt *postnewparent)
-{
-	if (op_ret >= 0) {
-		update_stat (buf, this->private);
-
-		update_stat (preoldparent, this->private);
-		update_stat (postoldparent, this->private);
-
-		update_stat (prenewparent, this->private);
-		update_stat (postnewparent, this->private);
-	}
-
-	STACK_UNWIND (frame, op_ret, op_errno, buf,
-                      preoldparent, postoldparent,
-                      prenewparent, postnewparent);
-	return 0;
-}
-
-int32_t
-filter_rename (call_frame_t *frame,
-	       xlator_t *this,
-	       loc_t *oldloc,
-	       loc_t *newloc)
-{
-	int32_t ret = 0;
-	inode_t *parent = oldloc->parent;
-	if (!parent)
-		parent = inode_parent (oldloc->inode, 0, NULL);
-	ret = update_frame (frame, oldloc->inode, this->private);
-	switch (ret) {		
-	case GF_FILTER_MAP_UID:
-		if (parent->st_mode & S_IWGRP)
-			break;
-		if (oldloc->inode->st_mode & S_IWGRP)
-			break;
-	case GF_FILTER_MAP_BOTH:
-		if (parent->st_mode & S_IWOTH)
-			break;
-		if (oldloc->inode->st_mode & S_IWOTH)
-			break;
-		gf_log (this->name, GF_LOG_DEBUG, 
-			"%s -> %s: returning permission denied", oldloc->path, newloc->path);
-		STACK_UNWIND (frame, -1, EPERM, NULL,
-                              NULL, NULL,
-                              NULL, NULL);
-		return 0;
-
-	case GF_FILTER_FILTER_UID:
-	case GF_FILTER_FILTER_GID:
-	case GF_FILTER_RO_FS:
-		STACK_UNWIND (frame, -1, EROFS, NULL,
-                              NULL, NULL,
-                              NULL, NULL);
-		return 0;
-	}
-	STACK_WIND (frame,
-		    filter_rename_cbk,
-		    FIRST_CHILD(this),
-		    FIRST_CHILD(this)->fops->rename,
-		    oldloc, newloc);
-	return 0;
-}
-
-
-static int32_t
-filter_link_cbk (call_frame_t *frame,
-		 void *cookie,
-		 xlator_t *this,
-		 int32_t op_ret,
-		 int32_t op_errno,
-		 inode_t *inode,
-                 struct iatt *buf,
-                 struct iatt *preparent,
-                 struct iatt *postparent)
-{
-	int ret = 0;
-	if (op_ret >= 0) {
-		update_stat (buf, this->private);
-		ret = inode_ctx_put (inode, this, (uint64_t)(long)buf->ia_uid);
-		if (ret == -1) {
-			gf_log (this->name, GF_LOG_ERROR,
-				"couldn't set context");
-		}
-
-		update_stat (preparent, this->private);
-		update_stat (postparent, this->private);
-	}
-	STACK_UNWIND (frame, op_ret, op_errno, inode, buf,
-                      preparent, postparent);
-	return 0;
-}
-
-int32_t
-filter_link (call_frame_t *frame,
-	     xlator_t *this,
-	     loc_t *oldloc,
-	     loc_t *newloc)
-{
-	int ret = 0;
-	ret = update_frame (frame, oldloc->inode, this->private);
-	switch (ret) {
-	case GF_FILTER_FILTER_UID:
-	case GF_FILTER_FILTER_GID:
-	case GF_FILTER_RO_FS:
-		STACK_UNWIND (frame, -1, EROFS, NULL, NULL,
-                              NULL, NULL);
-		return 0;
-	}
-	STACK_WIND (frame,
-		    filter_link_cbk,
-		    FIRST_CHILD(this),
-		    FIRST_CHILD(this)->fops->link,
-		    oldloc, newloc);
-	return 0;
-}
-
-
-static int32_t
-filter_create_cbk (call_frame_t *frame,
-		   void *cookie,
-		   xlator_t *this,
-		   int32_t op_ret,
-		   int32_t op_errno,
-		   fd_t *fd,
-		   inode_t *inode,
-		   struct iatt *buf,
-                   struct iatt *preparent,
-                   struct iatt *postparent)
-{
-	int ret = 0;
-	if (op_ret >= 0) {
-		update_stat (buf, this->private);
-		ret = inode_ctx_put (inode, this, (uint64_t)(long)buf->ia_uid);
-		if (ret == -1) {
-			gf_log (this->name, GF_LOG_ERROR,
-				"couldn't set context");
-		}
-		update_stat (preparent, this->private);
-		update_stat (postparent, this->private);
-	}
-	STACK_UNWIND (frame, op_ret, op_errno, fd, inode, buf,
-                      preparent, postparent);
-	return 0;
-}
-
-int32_t
-filter_create (call_frame_t *frame,
-	       xlator_t *this,
-	       loc_t *loc,
-	       int32_t flags,
-	       mode_t mode, fd_t *fd)
-{
-	int ret = 0;
-	inode_t *parent = loc->parent;
-	ret = update_frame (frame, loc->inode, this->private);
-	switch (ret) {		
-	case GF_FILTER_MAP_UID:
-		if (parent->st_mode & S_IWGRP)
-			break;
-	case GF_FILTER_MAP_BOTH:
-		if (parent->st_mode & S_IWOTH)
-			break;
-		gf_log (this->name, GF_LOG_DEBUG, "%s: returning permission denied", loc->path);
-		STACK_UNWIND (frame, -1, EPERM, NULL, NULL, NULL,
-                              NULL, NULL);
-		return 0;
-
-	case GF_FILTER_FILTER_UID:
-	case GF_FILTER_FILTER_GID:
-	case GF_FILTER_RO_FS:
-		STACK_UNWIND (frame, -1, EROFS, NULL, NULL, NULL,
-                              NULL, NULL);
-		return 0;
-	}
-	STACK_WIND (frame, filter_create_cbk,
-		    FIRST_CHILD(this),
-		    FIRST_CHILD(this)->fops->create,
-		    loc, flags, mode, fd);
-	return 0;
-}
-
-static int32_t
-filter_open_cbk (call_frame_t *frame,
-		 void *cookie,
-		 xlator_t *this,
-		 int32_t op_ret,
-		 int32_t op_errno,
-		 fd_t *fd)
-{
-	STACK_UNWIND (frame, op_ret, op_errno, fd);
-	return 0;
-}
-
-int32_t
-filter_open (call_frame_t *frame,
-	     xlator_t *this,
-	     loc_t *loc,
-	     int32_t flags, 
-	     fd_t *fd,
-             int32_t wbflags)
-{
-	int32_t ret = 0;
-	ret = update_frame (frame, loc->inode, this->private);
-	switch (ret) {
-	case GF_FILTER_MAP_UID:
-		if (loc->inode->st_mode & S_IWGRP)
-			break;
-		if (!(((flags & O_ACCMODE) == O_WRONLY)
-                      || ((flags & O_ACCMODE) == O_RDWR))
-		    && (loc->inode->st_mode & S_IRGRP))
-			break;
-	case GF_FILTER_MAP_BOTH:
-		if (loc->inode->st_mode & S_IWOTH)
-				break;
-		if (!(((flags & O_ACCMODE) == O_WRONLY)
-                      || ((flags & O_ACCMODE) == O_RDWR))
-		    && (loc->inode->st_mode & S_IROTH))
-			break;
-		gf_log (this->name, GF_LOG_DEBUG, 
-			"%s: returning permission denied (mode: 0%o, flag=0%o)", 
-			loc->path, loc->inode->st_mode, flags);
-		STACK_UNWIND (frame, -1, EPERM, fd);
-		return 0;
-	case GF_FILTER_FILTER_UID:
-	case GF_FILTER_FILTER_GID:
-	case GF_FILTER_RO_FS:
-		if (!(((flags & O_ACCMODE) == O_WRONLY)
-                      || ((flags & O_ACCMODE) == O_RDWR)))
-			break;
-		STACK_UNWIND (frame, -1, EROFS, NULL);
-		return 0;
-		
-	}
-	STACK_WIND (frame,
-		    filter_open_cbk,
-		    FIRST_CHILD(this),
-		    FIRST_CHILD(this)->fops->open,
-		    loc, flags, fd, wbflags);
-	return 0;
-}
-
-static int32_t
-filter_readv_cbk (call_frame_t *frame,
-		  void *cookie,
-		  xlator_t *this,
-		  int32_t op_ret,
-		  int32_t op_errno,
-		  struct iovec *vector,
-		  int32_t count,
-		  struct iatt *stbuf,
-                  struct iobref *iobref)
-{
-	if (op_ret >= 0) {
-		update_stat (stbuf, this->private);
-	}
-	STACK_UNWIND (frame,
-		      op_ret,
-		      op_errno,
-		      vector,
-		      count,
-		      stbuf,
-                      iobref);
-	return 0;
-}
-
-int32_t
-filter_readv (call_frame_t *frame,
-	      xlator_t *this,
-	      fd_t *fd,
-	      size_t size,
-	      off_t offset)
-{
-	STACK_WIND (frame,
-		    filter_readv_cbk,
-		    FIRST_CHILD(this),
-		    FIRST_CHILD(this)->fops->readv,
-		    fd,
-		    size,
-		    offset);
-	return 0;
-}
-
-
-static int32_t
-filter_writev_cbk (call_frame_t *frame,
-		   void *cookie,
-		   xlator_t *this,
-		   int32_t op_ret,
-		   int32_t op_errno,
-                   struct iatt *prebuf,
-		   struct iatt *postbuf)
-{
-	if (op_ret >= 0) {
-		update_stat (prebuf, this->private);
-		update_stat (postbuf, this->private);
-	}
-	STACK_UNWIND (frame,
-		      op_ret,
-		      op_errno,
-		      prebuf,
-                      postbuf);
-	return 0;
-}
-
-int32_t
-filter_writev (call_frame_t *frame,
-	       xlator_t *this,
-	       fd_t *fd,
-	       struct iovec *vector,
-	       int32_t count,
-	       off_t off,
-               struct iobref *iobref)
-{
-	int32_t ret = 0;
-	ret = update_frame (frame, fd->inode, this->private);
-	switch (ret) {
-	case GF_FILTER_FILTER_UID:
-	case GF_FILTER_FILTER_GID:
-	case GF_FILTER_RO_FS:
-		STACK_UNWIND (frame, -1, EROFS, NULL, NULL);
-		return 0;
-	}
-
-	STACK_WIND (frame,
-		    filter_writev_cbk,
-		    FIRST_CHILD(this),
-		    FIRST_CHILD(this)->fops->writev,
-		    fd,
-		    vector,
-		    count,
-		    off,
-                    iobref);
-	return 0;
-}
-
-static int32_t
-filter_fstat_cbk (call_frame_t *frame,
-		  void *cookie,
-		  xlator_t *this,
-		  int32_t op_ret,
-		  int32_t op_errno,
-		  struct iatt *buf)
-{
-	if (op_ret >= 0) {
-		update_stat (buf, this->private);
-	}
-	STACK_UNWIND (frame,
-		      op_ret,
-		      op_errno,
-		      buf);
-	return 0;
-}
-
-int32_t
-filter_fstat (call_frame_t *frame,
-	      xlator_t *this,
-	      fd_t *fd)
-{
-	STACK_WIND (frame,
-		    filter_fstat_cbk,
-		    FIRST_CHILD(this),
-		    FIRST_CHILD(this)->fops->fstat,
-		    fd);
-	return 0;
-}
-
-static int32_t
-filter_opendir_cbk (call_frame_t *frame,
-		    void *cookie,
-		    xlator_t *this,
-		    int32_t op_ret,
-		    int32_t op_errno,
-		    fd_t *fd)
-{
-	STACK_UNWIND (frame,
-		      op_ret,
-		      op_errno,
-		      fd);
-	return 0;
-}
-
-int32_t
-filter_opendir (call_frame_t *frame,
-		xlator_t *this,
-		loc_t *loc, fd_t *fd)
-{
-	int32_t ret = 0;
-	ret = update_frame (frame, loc->inode, this->private);
-	switch (ret) {
-	case GF_FILTER_MAP_UID:
-		if (loc->inode->st_mode & S_IWGRP)
-			break;
-		if (loc->inode->st_mode & S_IRGRP)
-			break;
-	case GF_FILTER_MAP_BOTH:
-		if (loc->inode->st_mode & S_IWOTH)
-			break;
-		if (loc->inode->st_mode & S_IROTH)
-			break;
-		gf_log (this->name, GF_LOG_DEBUG, "%s: returning permission denied", loc->path);
-		STACK_UNWIND (frame, -1, EPERM, fd);
-		return 0;
-	}			
-	STACK_WIND (frame,
-		    filter_opendir_cbk,
-		    FIRST_CHILD(this),
-		    FIRST_CHILD(this)->fops->opendir,
-		    loc, fd);
-	return 0;
-}
-
-
-static int32_t
-filter_setxattr_cbk (call_frame_t *frame,
-		     void *cookie,
-		     xlator_t *this,
-		     int32_t op_ret,
-		     int32_t op_errno)
-{
-	STACK_UNWIND (frame,
-		      op_ret,
-		      op_errno);
-	return 0;
-}
-
-int32_t
-filter_setxattr (call_frame_t *frame,
-		 xlator_t *this,
-		 loc_t *loc,
-		 dict_t *dict,
-		 int32_t flags)
-{
-
-	int32_t ret = 0;
-	ret = update_frame (frame, loc->inode, this->private);
-	switch (ret) {
-	case GF_FILTER_MAP_UID:
-		if (loc->inode->st_mode & S_IWGRP)
-			break;
-	case GF_FILTER_MAP_BOTH:
-		if (loc->inode->st_mode & S_IWOTH)
-			break;
-		gf_log (this->name, GF_LOG_DEBUG, "%s: returning permission denied", loc->path);
-		STACK_UNWIND (frame, -1, EPERM);
-		return 0;
-	case GF_FILTER_FILTER_UID:
-	case GF_FILTER_FILTER_GID:
-	case GF_FILTER_RO_FS:
-		STACK_UNWIND (frame, -1, EROFS);
-		return 0;
-	}			
-
-	STACK_WIND (frame,
-		    filter_setxattr_cbk,
-		    FIRST_CHILD(this),
-		    FIRST_CHILD(this)->fops->setxattr,
-		    loc,
-		    dict,
-		    flags);
-	return 0;
-}
-
-static int32_t
-filter_getxattr_cbk (call_frame_t *frame,
-		     void *cookie,
-		     xlator_t *this,
-		     int32_t op_ret,
-		     int32_t op_errno,
-		     dict_t *dict)
-{
-	STACK_UNWIND (frame,
-		      op_ret,
-		      op_errno,
-		      dict);
-	return 0;
-}
-
-int32_t
-filter_getxattr (call_frame_t *frame,
-		 xlator_t *this,
-		 loc_t *loc,
-		 const char *name)
-{
-	int32_t ret = 0;
-	ret = update_frame (frame, loc->inode, this->private);
-	switch (ret) {
-	case GF_FILTER_MAP_UID:
-		if (loc->inode->st_mode & S_IRGRP)
-			break;
-	case GF_FILTER_MAP_BOTH:
-		if (loc->inode->st_mode & S_IROTH)
-			break;
-		gf_log (this->name, GF_LOG_DEBUG, "%s: returning permission denied", loc->path);
-		STACK_UNWIND (frame, -1, EPERM, NULL);
-		return 0;
-	}			
-
-	STACK_WIND (frame,
-		    filter_getxattr_cbk,
-		    FIRST_CHILD(this),
-		    FIRST_CHILD(this)->fops->getxattr,
-		    loc,
-		    name);
-	return 0;
-}
-
-static int32_t
-filter_removexattr_cbk (call_frame_t *frame,
-			void *cookie,
-			xlator_t *this,
-			int32_t op_ret,
-			int32_t op_errno)
-{
-	STACK_UNWIND (frame, op_ret, op_errno);
-	return 0;
-}
-
-int32_t
-filter_removexattr (call_frame_t *frame,
-		    xlator_t *this,
-		    loc_t *loc,
-		    const char *name)
-{
-	int32_t ret = 0;
-	ret = update_frame (frame, loc->inode, this->private);
-	switch (ret) {
-	case GF_FILTER_MAP_UID:
-		if (loc->inode->st_mode & S_IWGRP)
-			break;
-	case GF_FILTER_MAP_BOTH:
-		if (loc->inode->st_mode & S_IWOTH)
-			break;
-		gf_log (this->name, GF_LOG_DEBUG, "%s: returning permission denied", loc->path);
-		STACK_UNWIND (frame, -1, EPERM);
-		return 0;
-	case GF_FILTER_FILTER_UID:
-	case GF_FILTER_FILTER_GID:
-	case GF_FILTER_RO_FS:
-		STACK_UNWIND (frame, -1, EROFS);
-		return 0;
-	}			
-
-	STACK_WIND (frame,
-		    filter_removexattr_cbk,
-		    FIRST_CHILD(this),
-		    FIRST_CHILD(this)->fops->removexattr,
-		    loc,
-		    name);
-	return 0;
-}
-
-int32_t
-mem_acct_init (xlator_t *this)
-{
-        int     ret = -1;
-
-        if (!this)
-                return ret;
-
-        ret = xlator_mem_acct_init (this, gf_filter_mt_end + 1);
-        
-        if (ret != 0) {
-                gf_log (this->name, GF_LOG_ERROR, "Memory accounting init"
-                        "failed");
-                return ret;
-        }
-
-        return ret;
-}
-
-int32_t 
-init (xlator_t *this)
-{
-	char *value = NULL;
-	char *tmp_str = NULL;
-	char *tmp_str1 = NULL;
-	char *tmp_str2 = NULL;
-	char *dup_str = NULL;
-	char *input_value_str1 = NULL;
-	char *input_value_str2 = NULL;
-	char *output_value_str = NULL;
-	int32_t input_value = 0;
-	int32_t output_value = 0;
-	data_t *option_data = NULL;
-	struct gf_filter *filter = NULL;
-	gf_boolean_t tmp_bool = 0;
-
-	if (!this->children || this->children->next) {
-		gf_log (this->name,
-			GF_LOG_ERROR,
-			"translator not configured with exactly one child");
-		return -1;
-	}
-	
-	if (!this->parents) {
-		gf_log (this->name, GF_LOG_WARNING,
-			"dangling volume. check volfile ");
-	}
-
-	filter = GF_CALLOC (sizeof (*filter), 1, gf_filter_mt_gf_filter);
-	ERR_ABORT (filter);
-	
-	if (dict_get (this->options, "read-only")) {
-		value = data_to_str (dict_get (this->options, "read-only"));
-		if (gf_string2boolean (value, &filter->complete_read_only) == -1) {
-			gf_log (this->name, GF_LOG_ERROR,
-				"wrong value provided for 'read-only'");
-			return -1;
-		}
-	}
-
-	if (dict_get (this->options, "root-squashing")) {
-		value = data_to_str (dict_get (this->options, "root-squashing"));
-		if (gf_string2boolean (value, &tmp_bool) == -1) {
-			gf_log (this->name, GF_LOG_ERROR,
-				"wrong value provided for 'root-squashing'");
-			return -1;
-		}
-		if (tmp_bool) {
-			filter->translate_num_uid_entries = 1;
-			filter->translate_num_gid_entries = 1;
-			filter->translate_input_uid[0][0] = GF_FILTER_ROOT_UID; /* root */
-			filter->translate_input_uid[0][1] = GF_FILTER_ROOT_UID; /* root */
-			filter->translate_input_gid[0][0] = GF_FILTER_ROOT_GID; /* root */
-			filter->translate_input_gid[0][1] = GF_FILTER_ROOT_GID; /* root */
-			filter->translate_output_uid[0] = GF_FILTER_NOBODY_UID;
-			filter->translate_output_gid[0] = GF_FILTER_NOBODY_GID;
-		}
-	}
-
-	if (dict_get (this->options, "translate-uid")) {
-		option_data = dict_get (this->options, "translate-uid");
-		value = strtok_r (option_data->data, ",", &tmp_str);
-		while (value) {
-			dup_str = gf_strdup (value);
-			input_value_str1 = strtok_r (dup_str, "=", &tmp_str1);
-			if (input_value_str1) {
-				/* Check for n-m */
-				char *temp_string = gf_strdup (input_value_str1);
-				input_value_str2 = strtok_r (temp_string, "-", &tmp_str2);
-				if (gf_string2int (input_value_str2, &input_value) != 0) {
-					gf_log (this->name, GF_LOG_ERROR, 
-						"invalid number format \"%s\"", 
-						input_value_str2);
-					return -1;
-				}
-				filter->translate_input_uid[filter->translate_num_uid_entries][0] = input_value;
-				input_value_str2 = strtok_r (NULL, "-", &tmp_str2);
-				if (input_value_str2) {
-					if (gf_string2int (input_value_str2, &input_value) != 0) {
-						gf_log (this->name, GF_LOG_ERROR, 
-							"invalid number format \"%s\"", 
-							input_value_str2);
-						return -1;
-					}
-				}
-				filter->translate_input_uid[filter->translate_num_uid_entries][1] = input_value;
-				GF_FREE (temp_string);
-				output_value_str = strtok_r (NULL, "=", &tmp_str1);
-				if (output_value_str) {
-					if (gf_string2int (output_value_str, &output_value) != 0) {
-						gf_log (this->name, GF_LOG_ERROR, 
-							"invalid number format \"%s\"", 
-							output_value_str);
-						return -1;
-					}
-				} else {
-					gf_log (this->name, GF_LOG_ERROR, 
-						"mapping string not valid");
-					return -1;
-				}
-			} else {
-				gf_log (this->name, GF_LOG_ERROR, 
-					"mapping string not valid");
-				return -1;
-			}
-			filter->translate_output_uid[filter->translate_num_uid_entries]   = output_value;
-			gf_log (this->name, 
-				GF_LOG_DEBUG, 
-				"pair %d: input uid '%d' will be changed to uid '%d'", 
-				filter->translate_num_uid_entries, input_value, output_value);
-
-			filter->translate_num_uid_entries++;
-			if (filter->translate_num_uid_entries == GF_MAXIMUM_FILTERING_ALLOWED)
-				break;
-			value = strtok_r (NULL, ",", &tmp_str);
-			GF_FREE (dup_str);
-		}
-	}
-
-	tmp_str1 = NULL;
-	tmp_str2 = NULL;
-	tmp_str  = NULL;
-
-	if (dict_get (this->options, "translate-gid")) {
-		option_data = dict_get (this->options, "translate-gid");
-		value = strtok_r (option_data->data, ",", &tmp_str);
-		while (value) {
-			dup_str = gf_strdup (value);
-			input_value_str1 = strtok_r (dup_str, "=", &tmp_str1);
-			if (input_value_str1) {
-				/* Check for n-m */
-				char *temp_string = gf_strdup (input_value_str1);
-				input_value_str2 = strtok_r (temp_string, "-", &tmp_str2);
-				if (gf_string2int (input_value_str2, &input_value) != 0) {
-					gf_log (this->name, GF_LOG_ERROR, 
-						"invalid number format \"%s\"", 
-						input_value_str2);
-					return -1;
-				}
-				filter->translate_input_gid[filter->translate_num_gid_entries][0] = input_value;
-				input_value_str2 = strtok_r (NULL, "-", &tmp_str2);
-				if (input_value_str2) {
-					if (gf_string2int (input_value_str2, &input_value) != 0) {
-						gf_log (this->name, GF_LOG_ERROR, 
-							"invalid number format \"%s\"", 
-							input_value_str2);
-						return -1;
-					}
-				}
-				filter->translate_input_gid[filter->translate_num_gid_entries][1] = input_value;
-				GF_FREE (temp_string);
-				output_value_str = strtok_r (NULL, "=", &tmp_str1);
-				if (output_value_str) {
-					if (gf_string2int (output_value_str, &output_value) != 0) {
-						gf_log (this->name, GF_LOG_ERROR, 
-							"invalid number format \"%s\"", 
-							output_value_str);
-						return -1;
-					}
-				} else {
-					gf_log (this->name, GF_LOG_ERROR, 
-						"translate-gid value not valid");
-					return -1;
-				}
-			} else {
-				gf_log (this->name, GF_LOG_ERROR, 
-					"translate-gid value not valid");
-				return -1;
-			}
-			
-			filter->translate_output_gid[filter->translate_num_gid_entries] = output_value;
-			
-			gf_log (this->name, GF_LOG_DEBUG, 
-				"pair %d: input gid '%d' will be changed to gid '%d'", 
-				filter->translate_num_gid_entries, input_value, output_value);
-			
-			filter->translate_num_gid_entries++;
-			if (filter->translate_num_gid_entries == GF_MAXIMUM_FILTERING_ALLOWED)
-				break;
-			value = strtok_r (NULL, ",", &tmp_str);
-			GF_FREE (dup_str);
-		}
-	}
-
-	tmp_str  = NULL;
-	tmp_str1 = NULL;
-
-	if (dict_get (this->options, "filter-uid")) {
-		option_data = dict_get (this->options, "filter-uid");
-		value = strtok_r (option_data->data, ",", &tmp_str);
-		while (value) {
-			dup_str = gf_strdup (value);
-			/* Check for n-m */
-			input_value_str1 = strtok_r (dup_str, "-", &tmp_str1);
-			if (gf_string2int (input_value_str1, &input_value) != 0) {
-				gf_log (this->name, GF_LOG_ERROR, 
-					"invalid number format \"%s\"", 
-					input_value_str1);
-				return -1;
-			}
-			filter->filter_input_uid[filter->filter_num_uid_entries][0] = input_value;
-			input_value_str1 = strtok_r (NULL, "-", &tmp_str1);
-			if (input_value_str1) {
-				if (gf_string2int (input_value_str1, &input_value) != 0) {
-					gf_log (this->name, GF_LOG_ERROR, 
-						"invalid number format \"%s\"", 
-						input_value_str1);
-					return -1;
-				}
-			}
-			filter->filter_input_uid[filter->filter_num_uid_entries][1] = input_value;
-
-			gf_log (this->name, 
-				GF_LOG_DEBUG, 
-				"filter [%d]: input uid(s) '%s' will be filtered", 
-				filter->filter_num_uid_entries, dup_str);
-			
-			filter->filter_num_uid_entries++;
-			if (filter->filter_num_uid_entries == GF_MAXIMUM_FILTERING_ALLOWED)
-				break;
-			value = strtok_r (NULL, ",", &tmp_str);
-			GF_FREE (dup_str);
-		}
-		filter->partial_filter = 1;
-	}
-
-	tmp_str  = NULL;
-	tmp_str1 = NULL;
-
-	if (dict_get (this->options, "filter-gid")) {
-		option_data = dict_get (this->options, "filter-gid");
-		value = strtok_r (option_data->data, ",", &tmp_str);
-		while (value) {
-			dup_str = gf_strdup (value);
-			/* Check for n-m */
-			input_value_str1 = strtok_r (dup_str, "-", &tmp_str1);
-			if (gf_string2int (input_value_str1, &input_value) != 0) {
-				gf_log (this->name, GF_LOG_ERROR, 
-					"invalid number format \"%s\"", 
-					input_value_str1);
-				return -1;
-			}
-			filter->filter_input_gid[filter->filter_num_gid_entries][0] = input_value;
-			input_value_str1 = strtok_r (NULL, "-", &tmp_str1);
-			if (input_value_str1) {
-				if (gf_string2int (input_value_str1, &input_value) != 0) {
-					gf_log (this->name, GF_LOG_ERROR, 
-						"invalid number format \"%s\"", 
-						input_value_str1);
-					return -1;
-				}
-			}
-			filter->filter_input_gid[filter->filter_num_gid_entries][1] = input_value;
-
-			gf_log (this->name, 
-				GF_LOG_DEBUG, 
-				"filter [%d]: input gid(s) '%s' will be filtered", 
-				filter->filter_num_gid_entries, dup_str);
-			
-			filter->filter_num_gid_entries++;
-			if (filter->filter_num_gid_entries == GF_MAXIMUM_FILTERING_ALLOWED)
-				break;
-			value = strtok_r (NULL, ",", &tmp_str);
-			GF_FREE (dup_str);
-		}
-		gf_log (this->name, GF_LOG_ERROR, "this option is not supported currently.. exiting");
-		return -1;
-		filter->partial_filter = 1;
-	}
-
-	if (dict_get (this->options, "fixed-uid")) {
-		option_data = dict_get (this->options, "fixed-uid");
-		if (gf_string2int (option_data->data, &input_value) != 0) {
-			gf_log (this->name, GF_LOG_ERROR, 
-				"invalid number format \"%s\"", 
-				option_data->data);
-			return -1;
-		}
-		filter->fixed_uid = input_value;
-		filter->fixed_uid_set = 1;
-	}
-
-	if (dict_get (this->options, "fixed-gid")) {
-		option_data = dict_get (this->options, "fixed-gid");
-		if (gf_string2int (option_data->data, &input_value) != 0) {
-			gf_log (this->name, GF_LOG_ERROR, 
-				"invalid number format \"%s\"", 
-				option_data->data);
-			return -1;
-		}
-		filter->fixed_gid = input_value;
-		filter->fixed_gid_set = 1;
-	}
-
-	this->private = filter;
-	return 0;
-}
-
-
-void
-fini (xlator_t *this)
-{
-	struct gf_filter *filter = this->private;
-
-	GF_FREE (filter);
-
-	return;
-}
-
-
-struct xlator_fops fops = {
-	.lookup      = filter_lookup,
-	.stat        = filter_stat,
-	.fstat       = filter_fstat,
-	.readlink    = filter_readlink,
-	.mknod       = filter_mknod,
-	.mkdir       = filter_mkdir,
-	.unlink      = filter_unlink,
-	.rmdir       = filter_rmdir,
-	.symlink     = filter_symlink,
-	.rename      = filter_rename,
-	.link        = filter_link,
-	.truncate    = filter_truncate,
-	.ftruncate   = filter_ftruncate,
-	.create      = filter_create,
-	.open        = filter_open,
-	.readv       = filter_readv,
-	.writev      = filter_writev,
-	.setxattr    = filter_setxattr,
-	.getxattr    = filter_getxattr,
-	.removexattr = filter_removexattr,
-	.opendir     = filter_opendir,
-        .setattr     = filter_setattr,
-        .fsetattr    = filter_fsetattr,
-};
-
-struct xlator_cbks cbks = {
-};
-
-struct volume_options options[] = {
-	{ .key  = { "root-squashing" }, 
-	  .type = GF_OPTION_TYPE_BOOL 
-	},
-	{ .key  = { "read-only" }, 
-	  .type = GF_OPTION_TYPE_BOOL
-	},
-	{ .key  = { "fixed-uid" },  
-	  .type = GF_OPTION_TYPE_INT
-	},
-	{ .key  = { "fixed-gid" },  
-	  .type = GF_OPTION_TYPE_INT
-	},
-	{ .key  = { "translate-uid" },  
-	  .type = GF_OPTION_TYPE_ANY 
-	},
-	{ .key  = { "translate-gid" },  
-	  .type = GF_OPTION_TYPE_ANY
-	},
-	{ .key  = { "filter-uid" },  
-	  .type = GF_OPTION_TYPE_ANY 
-	},
-	{ .key  = { "filter-gid" },  
-	  .type = GF_OPTION_TYPE_ANY 
-	},
-	{ .key = {NULL} },
-};
diff --git a/xlators/features/ganesha/src/Makefile.am b/xlators/features/ganesha/src/Makefile.am
deleted file mode 100644
index d42b68f8440..00000000000
--- a/xlators/features/ganesha/src/Makefile.am
+++ /dev/null
@@ -1,18 +0,0 @@
-xlator_LTLIBRARIES = ganesha.la
-
-xlatordir = $(libdir)/glusterfs/$(PACKAGE_VERSION)/xlator/features
-
-noinst_HEADERS = ganesha.h ganesha-mem-types.h
-
-ganesha_la_LDFLAGS = -module -avoid-version
-
-ganesha_la_SOURCES = ganesha.c
-
-AM_CPPFLAGS = $(GF_CPPFLAGS) -I$(top_srcdir)/libglusterfs/src \
-        -fPIC -D_FILE_OFFSET_BITS=64 -D_GNU_SOURCE -D$(GF_HOST_OS)\
-        -DGANESHA_DIR=\"$(sysconfdir)/ganesha\" \
-        -DGYSNCD_PREFIX=\"$(libexecdir)/glusterfs\"
-
-AM_CFLAGS = -Wall $(GF_CFLAGS)
-
-CLEANFILES =
diff --git a/xlators/features/ganesha/src/ganesha-mem-types.h b/xlators/features/ganesha/src/ganesha-mem-types.h
deleted file mode 100644
index c4976c01afc..00000000000
--- a/xlators/features/ganesha/src/ganesha-mem-types.h
+++ /dev/null
@@ -1,21 +0,0 @@
-/*
-   Copyright (c) 2015 Red Hat, Inc. <http://www.redhat.com>
-   This file is part of GlusterFS.
-
-   This file is licensed to you under your choice of the GNU Lesser
-   General Public License, version 3 or any later version (LGPLv3 or
-   later), or the GNU General Public License, version 2 (GPLv2), in all
-   cases as published by the Free Software Foundation.
-*/
-#ifndef __GANESHA_MEM_TYPES_H__
-#define __GANESHA_MEM_TYPES_H__
-
-
-#include "mem-types.h"
-
-enum gf_ganesha_mem_types_ {
-     gf_ganesha_mt_priv_t = gf_common_mt_end + 1,
-     gf_ganesha_mt_end
-};
-
-#endif
diff --git a/xlators/features/ganesha/src/ganesha.c b/xlators/features/ganesha/src/ganesha.c
deleted file mode 100644
index f034fc00d46..00000000000
--- a/xlators/features/ganesha/src/ganesha.c
+++ /dev/null
@@ -1,85 +0,0 @@
-/*
-   Copyright (c) 2015 Red Hat, Inc. <http://www.redhat.com>
-   This file is part of GlusterFS.
-
-   This file is licensed to you under your choice of the GNU Lesser
-   General Public License, version 3 or any later version (LGPLv3 or
-   later), or the GNU General Public License, version 2 (GPLv2), in all
-   cases as published by the Free Software Foundation.
-*/
-
-
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
-
-#include "ganesha.h"
-#include "ganesha-mem-types.h"
-
-
-
-int32_t
-init (xlator_t *this)
-{
-        int        ret         = -1;
-        ganesha_priv_t   *priv = NULL;
-
-        if (!this->children || this->children->next) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "Need subvolume == 1");
-                goto err;
-        }
-
-        if (!this->parents) {
-                gf_log (this->name, GF_LOG_WARNING,
-                        "Dangling volume. Check volfile");
-                goto err;
-        }
-
-        priv = GF_CALLOC (1, sizeof (*priv), gf_ganesha_mt_priv_t);
-        if (!priv)
-                goto err;
-
-        this->private = priv;
-        ret = 0;
-
-err:
-        return ret;
-}
-
-
-void
-fini (xlator_t *this)
-{
-        ganesha_priv_t *priv = this->private;
-
-        this->private = NULL;
-        if (priv)
-                GF_FREE (priv);
-
-        return;
-}
-
-struct xlator_fops fops = {
-};
-
-struct xlator_cbks cbks = {
-};
-
-struct volume_options options[] = {
-        {
-         .key = {"features.ganesha"},
-         .default_value = "disable",
-         .type =  GF_OPTION_TYPE_BOOL,
-         .description = "enable translator"
-
-        },
-        { .key  = {"ganesha.enable"},
-          .default_value = "off",
-          .type =  GF_OPTION_TYPE_BOOL,
-          .description = "export volume via NFS-Ganesha"
-        },
-        { .key  = {NULL}
-        },
-};
diff --git a/xlators/features/gfid-access/src/Makefile.am b/xlators/features/gfid-access/src/Makefile.am
index db53affaab3..ff95604c4de 100644
--- a/xlators/features/gfid-access/src/Makefile.am
+++ b/xlators/features/gfid-access/src/Makefile.am
@@ -1,14 +1,15 @@
 xlator_LTLIBRARIES = gfid-access.la
 xlatordir = $(libdir)/glusterfs/$(PACKAGE_VERSION)/xlator/features
 
-gfid_access_la_LDFLAGS = -module -avoid-version
+gfid_access_la_LDFLAGS = -module $(GF_XLATOR_DEFAULT_LDFLAGS)
 
 gfid_access_la_SOURCES = gfid-access.c
 gfid_access_la_LIBADD = $(top_builddir)/libglusterfs/src/libglusterfs.la
 
 noinst_HEADERS = gfid-access.h gfid-access-mem-types.h
 
-AM_CPPFLAGS = $(GF_CPPFLAGS) -I$(top_srcdir)/libglusterfs/src
+AM_CPPFLAGS = $(GF_CPPFLAGS) -I$(top_srcdir)/libglusterfs/src \
+	-I$(top_srcdir)/rpc/xdr/src -I$(top_builddir)/rpc/xdr/src
 
 AM_CFLAGS = -Wall $(GF_CFLAGS)
 
diff --git a/xlators/features/gfid-access/src/gfid-access-mem-types.h b/xlators/features/gfid-access/src/gfid-access-mem-types.h
index 168d67b431f..1c4d0b93de2 100644
--- a/xlators/features/gfid-access/src/gfid-access-mem-types.h
+++ b/xlators/features/gfid-access/src/gfid-access-mem-types.h
@@ -11,13 +11,12 @@
 #ifndef _GFID_ACCESS_MEM_TYPES_H
 #define _GFID_ACCESS_MEM_TYPES_H
 
-#include "mem-types.h"
+#include <glusterfs/mem-types.h>
 
 enum gf_changelog_mem_types {
-        gf_gfid_access_mt_priv_t = gf_common_mt_end + 1,
-        gf_gfid_access_mt_gfid_t,
-        gf_gfid_access_mt_end
+    gf_gfid_access_mt_priv_t = gf_common_mt_end + 1,
+    gf_gfid_access_mt_gfid_t,
+    gf_gfid_access_mt_end
 };
 
 #endif
-
diff --git a/xlators/features/gfid-access/src/gfid-access.c b/xlators/features/gfid-access/src/gfid-access.c
index d5054432d76..3fea5672a21 100644
--- a/xlators/features/gfid-access/src/gfid-access.c
+++ b/xlators/features/gfid-access/src/gfid-access.c
@@ -7,1472 +7,1414 @@
    later), or the GNU General Public License, version 2 (GPLv2), in all
    cases as published by the Free Software Foundation.
 */
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
-
 #include "gfid-access.h"
-#include "inode.h"
-#include "byte-order.h"
-#include "statedump.h"
-
+#include <glusterfs/inode.h>
+#include <glusterfs/byte-order.h>
+#include <glusterfs/statedump.h>
 
 int
-ga_valid_inode_loc_copy (loc_t *dst, loc_t *src, xlator_t *this)
+ga_valid_inode_loc_copy(loc_t *dst, loc_t *src, xlator_t *this)
 {
-        int      ret        = 0;
-        uint64_t value      = 0;
-
-        /* if its an entry operation, on the virtual */
-        /* directory inode as parent, we need to handle */
-        /* it properly */
-        ret = loc_copy (dst, src);
-        if (ret < 0)
-                goto out;
-
-        /*
-         * Change ALL virtual inodes with real-inodes in loc
-         */
-        if (dst->parent) {
-                ret = inode_ctx_get (dst->parent, this, &value);
-                if (ret < 0) {
-                        ret = 0; //real-inode
-                        goto out;
-                }
-                inode_unref (dst->parent);
-                dst->parent = inode_ref ((inode_t*)value);
-                gf_uuid_copy (dst->pargfid, dst->parent->gfid);
+    int ret = 0;
+    uint64_t value = 0;
+
+    /* if its an entry operation, on the virtual */
+    /* directory inode as parent, we need to handle */
+    /* it properly */
+    ret = loc_copy(dst, src);
+    if (ret < 0)
+        goto out;
+
+    /*
+     * Change ALL virtual inodes with real-inodes in loc
+     */
+    if (dst->parent) {
+        ret = inode_ctx_get(dst->parent, this, &value);
+        if (ret < 0) {
+            ret = 0;  // real-inode
+            goto out;
         }
+        inode_unref(dst->parent);
+        dst->parent = inode_ref((inode_t *)(uintptr_t)value);
+        gf_uuid_copy(dst->pargfid, dst->parent->gfid);
+    }
 
-        if (dst->inode) {
-                ret = inode_ctx_get (dst->inode, this, &value);
-                if (ret < 0) {
-                        ret = 0; //real-inode
-                        goto out;
-                }
-                inode_unref (dst->inode);
-                dst->inode = inode_ref ((inode_t*)value);
-                gf_uuid_copy (dst->gfid, dst->inode->gfid);
+    if (dst->inode) {
+        ret = inode_ctx_get(dst->inode, this, &value);
+        if (ret < 0) {
+            ret = 0;  // real-inode
+            goto out;
         }
+        inode_unref(dst->inode);
+        dst->inode = inode_ref((inode_t *)(uintptr_t)value);
+        gf_uuid_copy(dst->gfid, dst->inode->gfid);
+    }
 out:
 
-        return ret;
+    return ret;
 }
 
 void
-ga_newfile_args_free (ga_newfile_args_t *args)
+ga_newfile_args_free(ga_newfile_args_t *args)
 {
-        if (!args)
-                goto out;
+    if (!args)
+        goto out;
 
-        GF_FREE (args->bname);
+    GF_FREE(args->bname);
 
-        if (S_ISLNK (args->st_mode) && args->args.symlink.linkpath) {
-                GF_FREE (args->args.symlink.linkpath);
-                args->args.symlink.linkpath = NULL;
-        }
+    if (S_ISLNK(args->st_mode) && args->args.symlink.linkpath) {
+        GF_FREE(args->args.symlink.linkpath);
+        args->args.symlink.linkpath = NULL;
+    }
 
-        mem_put (args);
+    mem_put(args);
 out:
-        return;
+    return;
 }
 
-
 void
-ga_heal_args_free (ga_heal_args_t *args)
+ga_heal_args_free(ga_heal_args_t *args)
 {
-        if (!args)
-                goto out;
+    if (!args)
+        goto out;
 
-        GF_FREE (args->bname);
+    GF_FREE(args->bname);
 
-        mem_put (args);
+    mem_put(args);
 out:
-        return;
+    return;
 }
 
-
 ga_newfile_args_t *
-ga_newfile_parse_args (xlator_t *this, data_t *data)
+ga_newfile_parse_args(xlator_t *this, data_t *data)
 {
-        ga_newfile_args_t *args     = NULL;
-        ga_private_t      *priv     = NULL;
-        int                len      = 0;
-        int                blob_len = 0;
-        int                min_len  = 0;
-        void              *blob     = NULL;
-
-        priv = this->private;
-
-        blob = data->data;
-        blob_len = data->len;
-
-        min_len = sizeof (args->uid) + sizeof (args->gid) + sizeof (args->gfid)
-                + sizeof (args->st_mode) + 2 + 2;
-        if (blob_len < min_len) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "Invalid length: Total length is less "
-                        "than minimum length.");
-                goto err;
+    ga_newfile_args_t *args = NULL;
+    ga_private_t *priv = NULL;
+    int len = 0;
+    int blob_len = 0;
+    int min_len = 0;
+    void *blob = NULL;
+
+    priv = this->private;
+
+    blob = data->data;
+    blob_len = data->len;
+
+    min_len = sizeof(args->uid) + sizeof(args->gid) + sizeof(args->gfid) +
+              sizeof(args->st_mode) + 2 + 2;
+    if (blob_len < min_len) {
+        gf_log(this->name, GF_LOG_ERROR,
+               "Invalid length: Total length is less "
+               "than minimum length.");
+        goto err;
+    }
+
+    args = mem_get0(priv->newfile_args_pool);
+    if (args == NULL)
+        goto err;
+
+    args->uid = ntoh32(*(uint32_t *)blob);
+    blob += sizeof(uint32_t);
+    blob_len -= sizeof(uint32_t);
+
+    args->gid = ntoh32(*(uint32_t *)blob);
+    blob += sizeof(uint32_t);
+    blob_len -= sizeof(uint32_t);
+
+    memcpy(args->gfid, blob, sizeof(args->gfid));
+    blob += sizeof(args->gfid);
+    blob_len -= sizeof(args->gfid);
+
+    args->st_mode = ntoh32(*(uint32_t *)blob);
+    blob += sizeof(uint32_t);
+    blob_len -= sizeof(uint32_t);
+
+    len = strnlen(blob, blob_len);
+    if (len == blob_len) {
+        gf_log(this->name, GF_LOG_ERROR, "gfid: %s. No null byte present.",
+               args->gfid);
+        goto err;
+    }
+
+    args->bname = GF_MALLOC(len + 1, gf_common_mt_char);
+    if (args->bname == NULL)
+        goto err;
+
+    memcpy(args->bname, blob, (len + 1));
+    blob += (len + 1);
+    blob_len -= (len + 1);
+
+    if (S_ISDIR(args->st_mode)) {
+        if (blob_len < sizeof(uint32_t)) {
+            gf_log(this->name, GF_LOG_ERROR, "gfid: %s. Invalid length",
+                   args->gfid);
+            goto err;
         }
-
-        args = mem_get0 (priv->newfile_args_pool);
-        if (args == NULL)
-                goto err;
-
-        args->uid = ntoh32 (*(uint32_t *)blob);
-        blob += sizeof (uint32_t);
-        blob_len -= sizeof (uint32_t);
-
-        args->gid = ntoh32 (*(uint32_t *)blob);
-        blob += sizeof (uint32_t);
-        blob_len -= sizeof (uint32_t);
-
-        memcpy (args->gfid, blob, sizeof (args->gfid));
-        blob += sizeof (args->gfid);
-        blob_len -= sizeof (args->gfid);
-
-        args->st_mode = ntoh32 (*(uint32_t *)blob);
-        blob += sizeof (uint32_t);
-        blob_len -= sizeof (uint32_t);
-
-        len = strnlen (blob, blob_len);
+        args->args.mkdir.mode = ntoh32(*(uint32_t *)blob);
+        blob += sizeof(uint32_t);
+        blob_len -= sizeof(uint32_t);
+
+        if (blob_len < sizeof(uint32_t)) {
+            gf_log(this->name, GF_LOG_ERROR, "gfid: %s. Invalid length",
+                   args->gfid);
+            goto err;
+        }
+        args->args.mkdir.umask = ntoh32(*(uint32_t *)blob);
+        blob_len -= sizeof(uint32_t);
+        if (blob_len < 0) {
+            gf_log(this->name, GF_LOG_ERROR, "gfid: %s. Invalid length",
+                   args->gfid);
+            goto err;
+        }
+    } else if (S_ISLNK(args->st_mode)) {
+        len = strnlen(blob, blob_len);
         if (len == blob_len) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "gfid: %s. No null byte present.",
-                        args->gfid);
-                goto err;
+            gf_log(this->name, GF_LOG_ERROR, "gfid: %s. Invalid length",
+                   args->gfid);
+            goto err;
         }
+        args->args.symlink.linkpath = GF_MALLOC(len + 1, gf_common_mt_char);
+        if (args->args.symlink.linkpath == NULL)
+            goto err;
 
-        args->bname = GF_CALLOC (1, (len + 1), gf_common_mt_char);
-        if (args->bname == NULL)
-                goto err;
-
-        memcpy (args->bname, blob, (len + 1));
-        blob += (len + 1);
+        memcpy(args->args.symlink.linkpath, blob, (len + 1));
         blob_len -= (len + 1);
-
-        if (S_ISDIR (args->st_mode)) {
-                if (blob_len < sizeof (uint32_t)) {
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "gfid: %s. Invalid length",
-                                args->gfid);
-                        goto err;
-                }
-                args->args.mkdir.mode = ntoh32 (*(uint32_t *)blob);
-                blob += sizeof (uint32_t);
-                blob_len -= sizeof (uint32_t);
-
-                if (blob_len < sizeof (uint32_t)) {
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "gfid: %s. Invalid length",
-                                args->gfid);
-                        goto err;
-                }
-                args->args.mkdir.umask = ntoh32 (*(uint32_t *)blob);
-                blob += sizeof (uint32_t);
-                blob_len -= sizeof (uint32_t);
-                if (blob_len < 0) {
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "gfid: %s. Invalid length",
-                                args->gfid);
-                        goto err;
-                }
-        } else if (S_ISLNK (args->st_mode)) {
-                len = strnlen (blob, blob_len);
-                if (len == blob_len) {
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "gfid: %s. Invalid length",
-                                args->gfid);
-                        goto err;
-                }
-                args->args.symlink.linkpath = GF_CALLOC (1, len + 1,
-                                                         gf_common_mt_char);
-                if (args->args.symlink.linkpath == NULL)
-                        goto err;
-
-                memcpy (args->args.symlink.linkpath, blob, (len + 1));
-                blob += (len + 1);
-                blob_len -= (len + 1);
-        } else {
-                if (blob_len < sizeof (uint32_t)) {
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "gfid: %s. Invalid length",
-                                args->gfid);
-                        goto err;
-                }
-                args->args.mknod.mode = ntoh32 (*(uint32_t *)blob);
-                blob += sizeof (uint32_t);
-                blob_len -= sizeof (uint32_t);
-
-                if (blob_len < sizeof (uint32_t)) {
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "gfid: %s. Invalid length",
-                                args->gfid);
-                        goto err;
-                }
-                args->args.mknod.rdev = ntoh32 (*(uint32_t *)blob);
-                blob += sizeof (uint32_t);
-                blob_len -= sizeof (uint32_t);
-
-                if (blob_len < sizeof (uint32_t)) {
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "gfid: %s. Invalid length",
-                                args->gfid);
-                        goto err;
-                }
-                args->args.mknod.umask = ntoh32 (*(uint32_t *)blob);
-                blob += sizeof (uint32_t);
-                blob_len -= sizeof (uint32_t);
+    } else {
+        if (blob_len < sizeof(uint32_t)) {
+            gf_log(this->name, GF_LOG_ERROR, "gfid: %s. Invalid length",
+                   args->gfid);
+            goto err;
         }
-
-        if (blob_len) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "gfid: %s. Invalid length",
-                        args->gfid);
-                goto err;
+        args->args.mknod.mode = ntoh32(*(uint32_t *)blob);
+        blob += sizeof(uint32_t);
+        blob_len -= sizeof(uint32_t);
+
+        if (blob_len < sizeof(uint32_t)) {
+            gf_log(this->name, GF_LOG_ERROR, "gfid: %s. Invalid length",
+                   args->gfid);
+            goto err;
         }
+        args->args.mknod.rdev = ntoh32(*(uint32_t *)blob);
+        blob += sizeof(uint32_t);
+        blob_len -= sizeof(uint32_t);
+
+        if (blob_len < sizeof(uint32_t)) {
+            gf_log(this->name, GF_LOG_ERROR, "gfid: %s. Invalid length",
+                   args->gfid);
+            goto err;
+        }
+        args->args.mknod.umask = ntoh32(*(uint32_t *)blob);
+        blob_len -= sizeof(uint32_t);
+    }
+
+    if (blob_len) {
+        gf_log(this->name, GF_LOG_ERROR, "gfid: %s. Invalid length",
+               args->gfid);
+        goto err;
+    }
 
-        return args;
+    return args;
 
 err:
-        if (args)
-                ga_newfile_args_free (args);
+    if (args)
+        ga_newfile_args_free(args);
 
-        return NULL;
+    return NULL;
 }
 
 ga_heal_args_t *
-ga_heal_parse_args (xlator_t *this, data_t *data)
+ga_heal_parse_args(xlator_t *this, data_t *data)
 {
-        ga_heal_args_t *args     = NULL;
-        ga_private_t   *priv     = NULL;
-        void           *blob     = NULL;
-        int             len      = 0;
-        int             blob_len = 0;
+    ga_heal_args_t *args = NULL;
+    ga_private_t *priv = NULL;
+    void *blob = NULL;
+    int len = 0;
+    int blob_len = 0;
 
-        blob = data->data;
-        blob_len = data->len;
+    blob = data->data;
+    blob_len = data->len;
 
-        priv = this->private;
+    priv = this->private;
 
-        /* bname should at least contain a character */
-        if (blob_len < (sizeof (args->gfid) + 2))
-                goto err;
+    /* bname should at least contain a character */
+    if (blob_len < (sizeof(args->gfid) + 2))
+        goto err;
 
-        args = mem_get0 (priv->heal_args_pool);
-        if (!args)
-                goto err;
+    args = mem_get0(priv->heal_args_pool);
+    if (!args)
+        goto err;
 
-        memcpy (args->gfid, blob, sizeof (args->gfid));
-        blob += sizeof (args->gfid);
-        blob_len -= sizeof (args->gfid);
+    memcpy(args->gfid, blob, sizeof(args->gfid));
+    blob += sizeof(args->gfid);
+    blob_len -= sizeof(args->gfid);
 
-        len = strnlen (blob, blob_len);
-        if (len == blob_len)
-                goto err;
+    len = strnlen(blob, blob_len);
+    if (len == blob_len)
+        goto err;
 
-        args->bname = GF_CALLOC (1, len + 1, gf_common_mt_char);
-        if (!args->bname)
-                goto err;
+    args->bname = GF_MALLOC(len + 1, gf_common_mt_char);
+    if (!args->bname)
+        goto err;
 
-        memcpy (args->bname, blob, len);
-        blob_len -= (len + 1);
+    memcpy(args->bname, blob, len);
+    args->bname[len] = '\0';
+    blob_len -= (len + 1);
 
-        if (blob_len)
-                goto err;
+    if (blob_len)
+        goto err;
 
-        return args;
+    return args;
 
 err:
-        if (args)
-                ga_heal_args_free (args);
+    if (args)
+        ga_heal_args_free(args);
 
-        return NULL;
+    return NULL;
 }
 
 static int32_t
-ga_fill_tmp_loc (loc_t *loc, xlator_t *this, uuid_t gfid,
-                 char *bname, dict_t *xdata, loc_t *new_loc)
+ga_fill_tmp_loc(loc_t *loc, xlator_t *this, uuid_t gfid, char *bname,
+                dict_t *xdata, loc_t *new_loc)
 {
-        int       ret    = -1;
-        uint64_t  value  = 0;
-        inode_t  *parent = NULL;
-        uuid_t *gfid_ptr = NULL;
-
-        parent = loc->inode;
-        ret = inode_ctx_get (loc->inode, this, &value);
-        if (!ret) {
-                parent = (void *)value;
-                if (gf_uuid_is_null (parent->gfid))
-                        parent = loc->inode;
-        }
-
-        /* parent itself should be looked up */
-        gf_uuid_copy (new_loc->pargfid, parent->gfid);
-        new_loc->parent = inode_ref (parent);
-
-        new_loc->inode = inode_grep (parent->table, parent, bname);
-        if (!new_loc->inode) {
-                new_loc->inode = inode_new (parent->table);
-                gf_uuid_copy (new_loc->inode->gfid, gfid);
-        }
-
-        loc_path (new_loc, bname);
-        if (new_loc->path) {
-                new_loc->name = strrchr (new_loc->path, '/');
-                if (new_loc->name)
-                        new_loc->name++;
-        }
-
-        gfid_ptr = GF_CALLOC (1, sizeof(uuid_t), gf_common_mt_uuid_t);
-        if (!gfid_ptr) {
-                ret = -1;
-                goto out;
-        }
-        gf_uuid_copy (*gfid_ptr, gfid);
-        ret = dict_set_dynptr (xdata, "gfid-req", gfid_ptr, sizeof (uuid_t));
-        if (ret < 0)
-                goto out;
-
-        ret = 0;
+    int ret = -1;
+    uint64_t value = 0;
+    inode_t *parent = NULL;
+    unsigned char *gfid_ptr = NULL;
+
+    parent = loc->inode;
+    ret = inode_ctx_get(loc->inode, this, &value);
+    if (!ret) {
+        parent = (void *)(uintptr_t)value;
+        if (gf_uuid_is_null(parent->gfid))
+            parent = loc->inode;
+    }
+
+    /* parent itself should be looked up */
+    gf_uuid_copy(new_loc->pargfid, parent->gfid);
+    new_loc->parent = inode_ref(parent);
+
+    new_loc->inode = inode_grep(parent->table, parent, bname);
+    if (!new_loc->inode) {
+        new_loc->inode = inode_new(parent->table);
+        gf_uuid_copy(new_loc->inode->gfid, gfid);
+    }
+
+    loc_path(new_loc, bname);
+    if (new_loc->path) {
+        new_loc->name = strrchr(new_loc->path, '/');
+        if (new_loc->name)
+            new_loc->name++;
+    }
+
+    gfid_ptr = GF_MALLOC(sizeof(uuid_t), gf_common_mt_uuid_t);
+    if (!gfid_ptr) {
+        ret = -1;
+        goto out;
+    }
+    gf_uuid_copy(gfid_ptr, gfid);
+    ret = dict_set_gfuuid(xdata, "gfid-req", gfid_ptr, false);
+    if (ret < 0)
+        goto out;
+
+    ret = 0;
 
 out:
-        if (ret && gfid_ptr)
-                GF_FREE (gfid_ptr);
-        return ret;
+    if (ret && gfid_ptr)
+        GF_FREE(gfid_ptr);
+    return ret;
 }
 
-
-
 static gf_boolean_t
-__is_gfid_access_dir (uuid_t gfid)
+__is_gfid_access_dir(uuid_t gfid)
 {
-        uuid_t  aux_gfid;
-
-        memset (aux_gfid, 0, 16);
-        aux_gfid[15] = GF_AUX_GFID;
+    static uuid_t aux_gfid = {0, 0, 0, 0, 0, 0, 0, 0,
+                              0, 0, 0, 0, 0, 0, 0, GF_AUX_GFID};
 
-        if (gf_uuid_compare (gfid, aux_gfid) == 0)
-                return _gf_true;
+    if (gf_uuid_compare(gfid, aux_gfid) == 0)
+        return _gf_true;
 
-        return _gf_false;
+    return _gf_false;
 }
 
 int32_t
-ga_forget (xlator_t *this, inode_t *inode)
+ga_forget(xlator_t *this, inode_t *inode)
 {
-        int       ret = -1;
-        uint64_t  value = 0;
-        inode_t  *tmp_inode = NULL;
+    int ret = -1;
+    uint64_t value = 0;
+    inode_t *tmp_inode = NULL;
 
-        ret = inode_ctx_del (inode, this, &value);
-        if (ret)
-                goto out;
+    ret = inode_ctx_del(inode, this, &value);
+    if (ret)
+        goto out;
 
-        tmp_inode = (void *)value;
-        inode_unref (tmp_inode);
+    tmp_inode = (void *)(uintptr_t)value;
+    inode_unref(tmp_inode);
 
 out:
-        return 0;
+    return 0;
 }
 
-
 static int
-ga_heal_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-             int32_t op_ret, int32_t op_errno,
-             inode_t *inode, struct iatt *stat, dict_t *dict,
-             struct iatt *postparent)
-{
-        call_frame_t *orig_frame = NULL;
-
-        orig_frame = frame->local;
-        frame->local = NULL;
-
-        /* don't worry about inode linking and other stuff. They'll happen on
-         * the next lookup.
-         */
-        STACK_DESTROY (frame->root);
-
-        STACK_UNWIND_STRICT (setxattr, orig_frame, op_ret, op_errno, dict);
-
-        return 0;
-}
-
-static int32_t
-ga_newentry_setattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                         int32_t op_ret, int32_t op_errno, struct iatt *statpre,
-                         struct iatt *statpost,
-                         dict_t *xdata)
+ga_heal_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int32_t op_ret,
+            int32_t op_errno, inode_t *inode, struct iatt *stat, dict_t *dict,
+            struct iatt *postparent)
 {
-        ga_local_t *local = NULL;
-
-        local = frame->local;
-        frame->local = NULL;
+    call_frame_t *orig_frame = NULL;
 
-        /* don't worry about inode linking and other stuff. They'll happen on
-         * the next lookup.
-         */
-        STACK_DESTROY (frame->root);
+    orig_frame = frame->local;
+    frame->local = NULL;
 
-        STACK_UNWIND_STRICT (setxattr, local->orig_frame, op_ret,
-                             op_errno, xdata);
+    /* don't worry about inode linking and other stuff. They'll happen on
+     * the next lookup.
+     */
+    STACK_DESTROY(frame->root);
 
-        if (local->xdata)
-                dict_unref (local->xdata);
-        loc_wipe (&local->loc);
-        mem_put (local);
+    STACK_UNWIND_STRICT(setxattr, orig_frame, op_ret, op_errno, dict);
 
-        return 0;
+    return 0;
 }
 
 static int
-ga_newentry_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                 int32_t op_ret, int32_t op_errno,
-                 inode_t *inode, struct iatt *buf,
-                 struct iatt *preparent, struct iatt *postparent,
-                 dict_t *xdata)
+ga_newentry_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                int32_t op_ret, int32_t op_errno, inode_t *inode,
+                struct iatt *buf, struct iatt *preparent,
+                struct iatt *postparent, dict_t *xdata)
 {
-        ga_local_t *local = NULL;
-        struct iatt temp_stat = {0,};
+    ga_local_t *local = NULL;
 
-        local = frame->local;
+    local = frame->local;
 
-        /* no need to proceed if things don't look good here */
-        if (op_ret == -1)
-                goto done;
+    /* don't worry about inode linking and other stuff. They'll happen on
+     * the next lookup.
+     */
+    frame->local = NULL;
+    STACK_DESTROY(frame->root);
 
-        temp_stat.ia_uid = local->uid;
-        temp_stat.ia_gid = local->gid;
+    STACK_UNWIND_STRICT(setxattr, local->orig_frame, op_ret, op_errno, xdata);
 
-        STACK_WIND (frame, ga_newentry_setattr_cbk, FIRST_CHILD (this),
-                    FIRST_CHILD (this)->fops->setattr, &local->loc, &temp_stat,
-                    (GF_SET_ATTR_UID | GF_SET_ATTR_GID), xdata);
+    if (local->xdata)
+        dict_unref(local->xdata);
+    loc_wipe(&local->loc);
+    mem_put(local);
 
-        return 0;
-
-done:
-        /* don't worry about inode linking and other stuff. They'll happen on
-         * the next lookup.
-         */
-        frame->local = NULL;
-        STACK_DESTROY (frame->root);
-
-        STACK_UNWIND_STRICT (setxattr, local->orig_frame, op_ret,
-                             op_errno, xdata);
-
-        if (local->xdata)
-                dict_unref (local->xdata);
-        loc_wipe (&local->loc);
-        mem_put (local);
-
-        return 0;
+    return 0;
 }
 
 static int
-ga_newentry_lookup_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                        int32_t op_ret, int32_t op_errno, inode_t *inode,
-                        struct iatt *stat, dict_t *xdata,
-                        struct iatt *postparent)
+ga_newentry_lookup_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                       int32_t op_ret, int32_t op_errno, inode_t *inode,
+                       struct iatt *stat, dict_t *xdata,
+                       struct iatt *postparent)
 
 {
-        ga_local_t *local = NULL;
+    ga_local_t *local = NULL;
 
-        local = frame->local;
+    local = frame->local;
 
-        if ((op_ret < 0) && ((op_errno != ENOENT) && (op_errno != ESTALE)))
-                goto err;
+    if ((op_ret < 0) && ((op_errno != ENOENT) && (op_errno != ESTALE)))
+        goto err;
 
-        STACK_WIND (frame, ga_newentry_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->mknod, &local->loc, local->mode,
-                    local->rdev, local->umask, local->xdata);
-        return 0;
+    STACK_WIND(frame, ga_newentry_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->mknod, &local->loc, local->mode,
+               local->rdev, local->umask, local->xdata);
+    return 0;
 
 err:
-        frame->local = NULL;
-        STACK_DESTROY (frame->root);
-        STACK_UNWIND_STRICT (setxattr, local->orig_frame, op_ret, op_errno,
-                             xdata);
-        if (local->xdata)
-                dict_unref (local->xdata);
-        loc_wipe (&local->loc);
-        mem_put (local);
-
-        return 0;
+    frame->local = NULL;
+    STACK_DESTROY(frame->root);
+    STACK_UNWIND_STRICT(setxattr, local->orig_frame, op_ret, op_errno, xdata);
+    if (local->xdata)
+        dict_unref(local->xdata);
+    loc_wipe(&local->loc);
+    mem_put(local);
+
+    return 0;
 }
 
 int32_t
-ga_new_entry (call_frame_t *frame, xlator_t *this, loc_t *loc, data_t *data,
-              dict_t *xdata)
+ga_new_entry(call_frame_t *frame, xlator_t *this, loc_t *loc, data_t *data,
+             dict_t *xdata)
 {
-        int                ret       = -1;
-        ga_newfile_args_t *args      = NULL;
-        loc_t              tmp_loc   = {0,};
-        call_frame_t      *new_frame = NULL;
-        ga_local_t        *local     = NULL;
-        uuid_t             gfid      = {0,};
-
-        args = ga_newfile_parse_args (this, data);
-        if (!args)
-                goto out;
-
-        ret = gf_uuid_parse (args->gfid, gfid);
-        if (ret)
-                goto out;
-
-        if (!xdata) {
-                xdata = dict_new ();
-        } else {
-                xdata = dict_ref (xdata);
-        }
-
-        if (!xdata) {
-                ret = -1;
-                goto out;
-        }
-
-        ret = ga_fill_tmp_loc (loc, this, gfid,
-                               args->bname, xdata, &tmp_loc);
-        if (ret)
-                goto out;
-
-        new_frame = copy_frame (frame);
-        if (!new_frame)
-                goto out;
-
-        local = mem_get0 (this->local_pool);
-        local->orig_frame = frame;
-
-        local->uid = args->uid;
-        local->gid = args->gid;
-
-        loc_copy (&local->loc, &tmp_loc);
-
-        new_frame->local = local;
-
-        if (S_ISDIR (args->st_mode)) {
-                STACK_WIND (new_frame, ga_newentry_cbk,
-                            FIRST_CHILD(this), FIRST_CHILD(this)->fops->mkdir,
-                            &tmp_loc, args->args.mkdir.mode,
-                            args->args.mkdir.umask, xdata);
-        } else if (S_ISLNK (args->st_mode)) {
-                STACK_WIND (new_frame, ga_newentry_cbk,
-                            FIRST_CHILD(this), FIRST_CHILD(this)->fops->symlink,
-                            args->args.symlink.linkpath,
-                            &tmp_loc, 0, xdata);
-        } else {
-                /* use 07777 (4 7s) for considering the Sticky bits etc) */
-                ((ga_local_t *)new_frame->local)->mode =
-                     (S_IFMT & args->st_mode) | (07777 & args->args.mknod.mode);
-
-                ((ga_local_t *)new_frame->local)->umask =
-                                                         args->args.mknod.umask;
-                ((ga_local_t *)new_frame->local)->rdev  = args->args.mknod.rdev;
-                ((ga_local_t *)new_frame->local)->xdata = dict_ref (xdata);
-
-                /* send a named lookup, so that dht can cleanup up stale linkto
-                 * files etc.
-                 */
-                STACK_WIND (new_frame, ga_newentry_lookup_cbk,
-                            FIRST_CHILD(this), FIRST_CHILD(this)->fops->lookup,
-                            &tmp_loc, NULL);
-        }
+    int ret = -1;
+    ga_newfile_args_t *args = NULL;
+    loc_t tmp_loc = {
+        0,
+    };
+    call_frame_t *new_frame = NULL;
+    ga_local_t *local = NULL;
+    uuid_t gfid = {
+        0,
+    };
+
+    if (!xdata) {
+        xdata = dict_new();
+    } else {
+        xdata = dict_ref(xdata);
+    }
+
+    if (!xdata) {
+        ret = -1;
+        goto out;
+    }
+
+    args = ga_newfile_parse_args(this, data);
+    if (!args)
+        goto out;
+
+    ret = gf_uuid_parse(args->gfid, gfid);
+    if (ret)
+        goto out;
+
+    ret = ga_fill_tmp_loc(loc, this, gfid, args->bname, xdata, &tmp_loc);
+    if (ret)
+        goto out;
+
+    new_frame = copy_frame(frame);
+    if (!new_frame)
+        goto out;
+
+    local = mem_get0(this->local_pool);
+    local->orig_frame = frame;
+
+    loc_copy(&local->loc, &tmp_loc);
+
+    new_frame->local = local;
+    new_frame->root->uid = args->uid;
+    new_frame->root->gid = args->gid;
+
+    if (S_ISDIR(args->st_mode)) {
+        STACK_WIND(new_frame, ga_newentry_cbk, FIRST_CHILD(this),
+                   FIRST_CHILD(this)->fops->mkdir, &tmp_loc,
+                   args->args.mkdir.mode, args->args.mkdir.umask, xdata);
+    } else if (S_ISLNK(args->st_mode)) {
+        STACK_WIND(new_frame, ga_newentry_cbk, FIRST_CHILD(this),
+                   FIRST_CHILD(this)->fops->symlink,
+                   args->args.symlink.linkpath, &tmp_loc, 0, xdata);
+    } else {
+        /* use 07777 (4 7s) for considering the Sticky bits etc) */
+        ((ga_local_t *)new_frame->local)->mode = (S_IFMT & args->st_mode) |
+                                                 (07777 &
+                                                  args->args.mknod.mode);
+
+        ((ga_local_t *)new_frame->local)->umask = args->args.mknod.umask;
+        ((ga_local_t *)new_frame->local)->rdev = args->args.mknod.rdev;
+        ((ga_local_t *)new_frame->local)->xdata = dict_ref(xdata);
+
+        /* send a named lookup, so that dht can cleanup up stale linkto
+         * files etc.
+         */
+        STACK_WIND(new_frame, ga_newentry_lookup_cbk, FIRST_CHILD(this),
+                   FIRST_CHILD(this)->fops->lookup, &tmp_loc, NULL);
+    }
 
-        ret = 0;
+    ret = 0;
 out:
-        ga_newfile_args_free (args);
+    ga_newfile_args_free(args);
 
-        if (xdata)
-                dict_unref (xdata);
+    if (xdata)
+        dict_unref(xdata);
 
-        loc_wipe (&tmp_loc);
+    loc_wipe(&tmp_loc);
 
-        return ret;
+    return ret;
 }
 
 int32_t
-ga_heal_entry (call_frame_t *frame, xlator_t *this, loc_t *loc, data_t *data,
-               dict_t *xdata)
+ga_heal_entry(call_frame_t *frame, xlator_t *this, loc_t *loc, data_t *data,
+              dict_t *xdata)
 {
-        int             ret       = -1;
-        ga_heal_args_t *args      = NULL;
-        loc_t           tmp_loc   = {0,};
-        call_frame_t   *new_frame = NULL;
-        uuid_t          gfid      = {0,};
-
-        args = ga_heal_parse_args (this, data);
-        if (!args)
-                goto out;
-
-        ret = gf_uuid_parse (args->gfid, gfid);
-        if (ret)
-                goto out;
-
-        if (!xdata)
-                xdata = dict_new ();
-        else
-                xdata = dict_ref (xdata);
-
-        if (!xdata) {
-                ret = -1;
-                goto out;
-        }
-
-        ret = ga_fill_tmp_loc (loc, this, gfid, args->bname,
-                               xdata, &tmp_loc);
-        if (ret)
-                goto out;
-
-        new_frame = copy_frame (frame);
-        if (!new_frame)
-                goto out;
-
-        new_frame->local = (void *)frame;
-
-        STACK_WIND (new_frame, ga_heal_cbk, FIRST_CHILD (this),
-                    FIRST_CHILD(this)->fops->lookup,
-                    &tmp_loc, xdata);
-
-        ret = 0;
+    int ret = -1;
+    ga_heal_args_t *args = NULL;
+    loc_t tmp_loc = {
+        0,
+    };
+    call_frame_t *new_frame = NULL;
+    uuid_t gfid = {
+        0,
+    };
+
+    args = ga_heal_parse_args(this, data);
+    if (!args)
+        goto out;
+
+    ret = gf_uuid_parse(args->gfid, gfid);
+    if (ret)
+        goto out;
+
+    if (!xdata)
+        xdata = dict_new();
+    else
+        xdata = dict_ref(xdata);
+
+    if (!xdata) {
+        ret = -1;
+        goto out;
+    }
+
+    ret = ga_fill_tmp_loc(loc, this, gfid, args->bname, xdata, &tmp_loc);
+    if (ret)
+        goto out;
+
+    new_frame = copy_frame(frame);
+    if (!new_frame)
+        goto out;
+
+    new_frame->local = (void *)frame;
+
+    STACK_WIND(new_frame, ga_heal_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->lookup, &tmp_loc, xdata);
+
+    ret = 0;
 out:
-        if (args)
-                ga_heal_args_free (args);
+    if (args)
+        ga_heal_args_free(args);
 
-        loc_wipe (&tmp_loc);
+    loc_wipe(&tmp_loc);
 
-        if (xdata)
-                dict_unref (xdata);
+    if (xdata)
+        dict_unref(xdata);
 
-        return ret;
+    return ret;
 }
 
 int32_t
-ga_setxattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                 int32_t op_ret, int32_t op_errno,
-                 dict_t *xdata)
+ga_setxattr_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
-        STACK_UNWIND_STRICT (setxattr, frame, op_ret, op_errno, xdata);
-        return 0;
+    STACK_UNWIND_STRICT(setxattr, frame, op_ret, op_errno, xdata);
+    return 0;
 }
 
 int32_t
-ga_setxattr (call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *dict,
-             int32_t flags, dict_t *xdata)
+ga_setxattr(call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *dict,
+            int32_t flags, dict_t *xdata)
 {
-        data_t  *data     = NULL;
-        int      op_errno = ENOMEM;
-        int      ret      = 0;
-        loc_t   ga_loc    = {0, };
-
-        GFID_ACCESS_INODE_OP_CHECK (loc, op_errno, err);
-
-        data = dict_get (dict, GF_FUSE_AUX_GFID_NEWFILE);
-        if (data) {
-                ret = ga_new_entry (frame, this, loc, data, xdata);
-                if (ret)
-                        goto err;
-                return 0;
-        }
+    data_t *data = NULL;
+    int op_errno = ENOMEM;
+    int ret = 0;
+    loc_t ga_loc = {
+        0,
+    };
+
+    GFID_ACCESS_INODE_OP_CHECK(loc, op_errno, err);
+
+    data = dict_get(dict, GF_FUSE_AUX_GFID_NEWFILE);
+    if (data) {
+        ret = ga_new_entry(frame, this, loc, data, xdata);
+        if (ret)
+            goto err;
+        return 0;
+    }
 
-        data = dict_get (dict, GF_FUSE_AUX_GFID_HEAL);
-        if (data) {
-                ret = ga_heal_entry (frame, this, loc, data, xdata);
-                if (ret)
-                        goto err;
-                return 0;
-        }
+    data = dict_get(dict, GF_FUSE_AUX_GFID_HEAL);
+    if (data) {
+        ret = ga_heal_entry(frame, this, loc, data, xdata);
+        if (ret)
+            goto err;
+        return 0;
+    }
 
-        //If the inode is a virtual inode change the inode otherwise perform
-        //the operation on same inode
-        ret = ga_valid_inode_loc_copy (&ga_loc, loc, this);
-        if (ret < 0)
-                goto err;
+    // If the inode is a virtual inode change the inode otherwise perform
+    // the operation on same inode
+    ret = ga_valid_inode_loc_copy(&ga_loc, loc, this);
+    if (ret < 0)
+        goto err;
 
-        STACK_WIND (frame, ga_setxattr_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->setxattr, &ga_loc, dict, flags,
-                    xdata);
+    STACK_WIND(frame, ga_setxattr_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->setxattr, &ga_loc, dict, flags, xdata);
 
-        loc_wipe (&ga_loc);
-        return 0;
+    loc_wipe(&ga_loc);
+    return 0;
 err:
-        STACK_UNWIND_STRICT (setxattr, frame, -1, op_errno, xdata);
-        return 0;
+    STACK_UNWIND_STRICT(setxattr, frame, -1, op_errno, xdata);
+    return 0;
 }
 
-
 int32_t
-ga_virtual_lookup_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                       int32_t op_ret, int32_t op_errno, inode_t *inode,
-                       struct iatt *buf, dict_t *xdata, struct iatt *postparent)
+ga_virtual_lookup_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                      int32_t op_ret, int32_t op_errno, inode_t *inode,
+                      struct iatt *buf, dict_t *xdata, struct iatt *postparent)
 {
-        int             j                       = 0;
-        int             i                       = 0;
-        int             ret                     = 0;
-        uint64_t        temp_ino                = 0;
-        inode_t         *cbk_inode              = NULL;
-        inode_t         *true_inode             = NULL;
-        uuid_t          random_gfid             = {0,};
-        inode_t         *linked_inode           = NULL;
-
-        if (frame->local)
-                cbk_inode = frame->local;
-        else
-                cbk_inode = inode_ref (inode);
-
-        frame->local = NULL;
-        if (op_ret)
-                goto unwind;
-
-        if (!IA_ISDIR (buf->ia_type))
+    int ret = 0;
+    inode_t *cbk_inode = NULL;
+    inode_t *true_inode = NULL;
+    uuid_t random_gfid = {
+        0,
+    };
+    inode_t *linked_inode = NULL;
+
+    if (frame->local)
+        cbk_inode = frame->local;
+    else
+        cbk_inode = inode_ref(inode);
+
+    frame->local = NULL;
+    if (op_ret)
+        goto unwind;
+
+    if (!IA_ISDIR(buf->ia_type))
+        goto unwind;
+
+    /* need to send back a different inode for linking in itable */
+    if (cbk_inode == inode) {
+        /* check if the inode is in the 'itable' or
+           if its just previously discover()'d inode */
+        true_inode = inode_find(inode->table, buf->ia_gfid);
+        if (!true_inode) {
+            /* This unref is for 'inode_ref()' done in beginning.
+               This is needed as cbk_inode is allocated new inode
+               whose unref is taken at the end*/
+            inode_unref(cbk_inode);
+            cbk_inode = inode_new(inode->table);
+
+            if (!cbk_inode) {
+                op_ret = -1;
+                op_errno = ENOMEM;
                 goto unwind;
+            }
+            /* the inode is not present in itable, ie, the actual
+               path is not yet looked up. Use the current inode
+               itself for now */
 
-        /* need to send back a different inode for linking in itable */
-        if (cbk_inode == inode) {
-                /* check if the inode is in the 'itable' or
-                   if its just previously discover()'d inode */
-                true_inode = inode_find (inode->table, buf->ia_gfid);
-                if (!true_inode) {
-                        /* This unref is for 'inode_ref()' done in beginning.
-                           This is needed as cbk_inode is allocated new inode
-                           whose unref is taken at the end*/
-                        inode_unref (cbk_inode);
-                        cbk_inode = inode_new (inode->table);
-
-                        if (!cbk_inode) {
-                                op_ret = -1;
-                                op_errno = ENOMEM;
-                                goto unwind;
-                        }
-                        /* the inode is not present in itable, ie, the actual
-                           path is not yet looked up. Use the current inode
-                           itself for now */
-
-                        linked_inode = inode_link (inode, NULL, NULL, buf);
-                        inode = linked_inode;
-                } else {
-                        /* 'inode_ref()' has been done in inode_find() */
-                        inode = true_inode;
-                }
-
-                ret = inode_ctx_put (cbk_inode, this, (uint64_t)inode);
-                if (ret) {
-                        gf_log (this->name, GF_LOG_WARNING,
-                                "failed to set the inode ctx with"
-                                "the actual inode");
-                        if (inode)
-                                inode_unref (inode);
-                }
-                inode = NULL;
+            linked_inode = inode_link(inode, NULL, NULL, buf);
+            inode = linked_inode;
+        } else {
+            /* 'inode_ref()' has been done in inode_find() */
+            inode = true_inode;
         }
 
-        if (!gf_uuid_is_null (cbk_inode->gfid)) {
-                /* if the previous linked inode is used, use the
-                   same gfid */
-                gf_uuid_copy (random_gfid, cbk_inode->gfid);
-        } else {
-                /* replace the buf->ia_gfid to a random gfid
-                   for directory, for files, what we received is fine */
-                gf_uuid_generate (random_gfid);
+        ret = inode_ctx_put(cbk_inode, this, (uint64_t)(uintptr_t)inode);
+        if (ret) {
+            gf_log(this->name, GF_LOG_WARNING,
+                   "failed to set the inode ctx with"
+                   "the actual inode");
+            if (inode)
+                inode_unref(inode);
         }
+        inode = NULL;
+    }
 
-        gf_uuid_copy (buf->ia_gfid, random_gfid);
+    if (!gf_uuid_is_null(cbk_inode->gfid)) {
+        /* if the previous linked inode is used, use the
+           same gfid */
+        gf_uuid_copy(random_gfid, cbk_inode->gfid);
+    } else {
+        /* replace the buf->ia_gfid to a random gfid
+           for directory, for files, what we received is fine */
+        gf_uuid_generate(random_gfid);
+    }
 
-        for (i = 15; i > (15 - 8); i--) {
-                temp_ino += (uint64_t)(buf->ia_gfid[i]) << j;
-                j += 8;
-        }
-        buf->ia_ino = temp_ino;
+    gf_uuid_copy(buf->ia_gfid, random_gfid);
+
+    buf->ia_ino = gfid_to_ino(buf->ia_gfid);
 
 unwind:
-        /* Lookup on non-existing gfid returns ESTALE.
-           Convert into ENOENT for virtual lookup*/
-        if (op_errno == ESTALE)
-               op_errno = ENOENT;
+    /* Lookup on non-existing gfid returns ESTALE.
+       Convert into ENOENT for virtual lookup*/
+    if (op_errno == ESTALE)
+        op_errno = ENOENT;
 
-        STACK_UNWIND_STRICT (lookup, frame, op_ret, op_errno, cbk_inode, buf,
-                             xdata, postparent);
+    STACK_UNWIND_STRICT(lookup, frame, op_ret, op_errno, cbk_inode, buf, xdata,
+                        postparent);
 
-        /* Also handles inode_unref of frame->local if done in ga_lookup */
-        if (cbk_inode)
-               inode_unref (cbk_inode);
+    /* Also handles inode_unref of frame->local if done in ga_lookup */
+    if (cbk_inode)
+        inode_unref(cbk_inode);
 
-        return 0;
+    return 0;
 }
 
 int32_t
-ga_lookup_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-               int32_t op_ret, int32_t op_errno, inode_t *inode,
-               struct iatt *buf, dict_t *xdata, struct iatt *postparent)
+ga_lookup_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int32_t op_ret,
+              int32_t op_errno, inode_t *inode, struct iatt *buf, dict_t *xdata,
+              struct iatt *postparent)
 {
-        ga_private_t *priv = NULL;
+    ga_private_t *priv = NULL;
 
-        /* if the entry in question is not 'root',
-           then follow the normal path */
-        if (op_ret || !__is_root_gfid(buf->ia_gfid))
-                goto unwind;
+    /* if the entry in question is not 'root',
+       then follow the normal path */
+    if (op_ret || !__is_root_gfid(buf->ia_gfid))
+        goto unwind;
 
-        priv = this->private;
+    priv = this->private;
 
-        /* do we need to copy root stbuf everytime? */
-        /* mostly yes, as we want to have the 'stat' info show latest
-           in every _cbk() */
+    /* do we need to copy root stbuf every time? */
+    /* mostly yes, as we want to have the 'stat' info show latest
+       in every _cbk() */
 
-        /* keep the reference for root stat buf */
-        priv->root_stbuf = *buf;
-        priv->gfiddir_stbuf = priv->root_stbuf;
-        priv->gfiddir_stbuf.ia_gfid[15] = GF_AUX_GFID;
-        priv->gfiddir_stbuf.ia_ino = GF_AUX_GFID;
+    /* keep the reference for root stat buf */
+    priv->root_stbuf = *buf;
+    priv->gfiddir_stbuf = priv->root_stbuf;
+    priv->gfiddir_stbuf.ia_gfid[15] = GF_AUX_GFID;
+    priv->gfiddir_stbuf.ia_ino = GF_AUX_GFID;
 
 unwind:
-        STACK_UNWIND_STRICT (lookup, frame, op_ret, op_errno, inode, buf,
-                             xdata, postparent);
-        return 0;
+    STACK_UNWIND_STRICT(lookup, frame, op_ret, op_errno, inode, buf, xdata,
+                        postparent);
+    return 0;
 }
 
 int32_t
-ga_lookup (call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xdata)
+ga_lookup(call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xdata)
 {
-        ga_private_t *priv     = NULL;
-        int           ret      = -1;
-        uuid_t        tmp_gfid = {0,};
-        loc_t         tmp_loc  = {0,};
-        uint64_t      value    = 0;
-        inode_t      *inode    = NULL;
-        inode_t      *true_inode    = NULL;
-        int32_t       op_errno = ENOENT;
-
-        /* if its discover(), no need for any action here */
-        if (!loc->name)
-                goto wind;
-
-        /* if its revalidate, and inode is not of type directory,
-           proceed with 'wind' */
-        if (loc->inode && loc->inode->ia_type &&
-            !IA_ISDIR (loc->inode->ia_type)) {
-
-                /* a revalidate on ".gfid/<dentry>" is possible, check for it */
-                if (((loc->parent &&
-                      __is_gfid_access_dir (loc->parent->gfid)) ||
-                     __is_gfid_access_dir (loc->pargfid))) {
-
-                        /* here, just send 'loc->gfid' and 'loc->inode' */
-                        tmp_loc.inode = inode_ref (loc->inode);
-                        gf_uuid_copy (tmp_loc.gfid, loc->inode->gfid);
-
-                        STACK_WIND (frame, default_lookup_cbk,
-                                    FIRST_CHILD(this),
-                                    FIRST_CHILD(this)->fops->lookup,
-                                    &tmp_loc, xdata);
-
-                        inode_unref (tmp_loc.inode);
-
-                        return 0;
-                }
-
-                /* not something to bother, continue the flow */
-                goto wind;
-        }
+    ga_private_t *priv = NULL;
+    int ret = -1;
+    uuid_t tmp_gfid = {
+        0,
+    };
+    loc_t tmp_loc = {
+        0,
+    };
+    uint64_t value = 0;
+    inode_t *inode = NULL;
+    inode_t *true_inode = NULL;
+    int32_t op_errno = ENOENT;
+
+    priv = this->private;
+
+    /* Handle nameless lookup on ".gfid" */
+    if (!loc->parent && __is_gfid_access_dir(loc->gfid)) {
+        STACK_UNWIND_STRICT(lookup, frame, 0, 0, loc->inode,
+                            &priv->gfiddir_stbuf, xdata, &priv->root_stbuf);
+        return 0;
+    }
 
-        priv = this->private;
+    /* if its discover(), no need for any action here */
+    if (!loc->name)
+        goto wind;
 
-        /* need to check if the lookup is on virtual dir */
-        if ((loc->name && !strcmp (GF_GFID_DIR, loc->name)) &&
-            ((loc->parent && __is_root_gfid (loc->parent->gfid)) ||
-             __is_root_gfid (loc->pargfid))) {
-                /* this means, the query is on '/.gfid', return the fake stat,
-                   and say success */
+    /* if its revalidate, and inode is not of type directory,
+       proceed with 'wind' */
+    if (loc->inode && loc->inode->ia_type && !IA_ISDIR(loc->inode->ia_type)) {
+        /* a revalidate on ".gfid/<dentry>" is possible, check for it */
+        if (((loc->parent && __is_gfid_access_dir(loc->parent->gfid)) ||
+             __is_gfid_access_dir(loc->pargfid))) {
+            /* here, just send 'loc->gfid' and 'loc->inode' */
+            tmp_loc.inode = inode_ref(loc->inode);
+            gf_uuid_copy(tmp_loc.gfid, loc->inode->gfid);
 
-                STACK_UNWIND_STRICT (lookup, frame, 0, 0, loc->inode,
-                                     &priv->gfiddir_stbuf, xdata,
-                                     &priv->root_stbuf);
-                return 0;
-        }
+            STACK_WIND(frame, default_lookup_cbk, FIRST_CHILD(this),
+                       FIRST_CHILD(this)->fops->lookup, &tmp_loc, xdata);
 
-        /* now, check if the lookup() is on an existing entry,
-           but on gfid-path */
-        if (!((loc->parent && __is_gfid_access_dir (loc->parent->gfid)) ||
-              __is_gfid_access_dir (loc->pargfid))) {
-                if (!loc->parent)
-                        goto wind;
+            inode_unref(tmp_loc.inode);
 
-                ret = inode_ctx_get (loc->parent, this, &value);
-                if (ret)
-                        goto wind;
+            return 0;
+        }
 
-                inode = (inode_t *) value;
+        /* not something to bother, continue the flow */
+        goto wind;
+    }
 
-                ret = loc_copy_overload_parent (&tmp_loc, loc, inode);
-                if (ret)
-                        goto err;
+    /* need to check if the lookup is on virtual dir */
+    if ((loc->name && !strcmp(GF_GFID_DIR, loc->name)) &&
+        ((loc->parent && __is_root_gfid(loc->parent->gfid)) ||
+         __is_root_gfid(loc->pargfid))) {
+        /* this means, the query is on '/.gfid', return the fake stat,
+           and say success */
 
-                STACK_WIND (frame, ga_lookup_cbk, FIRST_CHILD (this),
-                            FIRST_CHILD (this)->fops->lookup, &tmp_loc, xdata);
+        STACK_UNWIND_STRICT(lookup, frame, 0, 0, loc->inode,
+                            &priv->gfiddir_stbuf, xdata, &priv->root_stbuf);
+        return 0;
+    }
 
-                loc_wipe (&tmp_loc);
-                return 0;
-        }
+    /* now, check if the lookup() is on an existing entry,
+       but on gfid-path */
+    if (!((loc->parent && __is_gfid_access_dir(loc->parent->gfid)) ||
+          __is_gfid_access_dir(loc->pargfid))) {
+        if (!loc->parent)
+            goto wind;
 
-        /* make sure the 'basename' is actually a 'canonical-gfid',
-           otherwise, return error */
-        ret = gf_uuid_parse (loc->name, tmp_gfid);
+        ret = inode_ctx_get(loc->parent, this, &value);
         if (ret)
-                goto err;
+            goto wind;
 
-        /* if its fresh lookup, go ahead and send it down, if not,
-           for directory, we need indirection to actual dir inode */
-        if (!(loc->inode && loc->inode->ia_type))
-                goto discover;
+        inode = (inode_t *)(uintptr_t)value;
 
-        /* revalidate on directory */
-        ret = inode_ctx_get (loc->inode, this, &value);
+        ret = loc_copy_overload_parent(&tmp_loc, loc, inode);
         if (ret)
-                goto err;
+            goto err;
 
-        inode = (void *)value;
+        STACK_WIND(frame, ga_lookup_cbk, FIRST_CHILD(this),
+                   FIRST_CHILD(this)->fops->lookup, &tmp_loc, xdata);
 
-        /* valid inode, already looked up, work on that */
-        if (inode->ia_type)
-                goto discover;
-
-        /* check if the inode is in the 'itable' or
-           if its just previously discover()'d inode */
-        true_inode = inode_find (loc->inode->table, tmp_gfid);
-        if (true_inode) {
-                /* time do another lookup and update the context
-                   with proper inode */
-                op_errno = ESTALE;
-                /* 'inode_ref()' done in inode_find */
-                inode_unref (true_inode);
-                goto err;
-        }
+        loc_wipe(&tmp_loc);
+        return 0;
+    }
+
+    /* make sure the 'basename' is actually a 'canonical-gfid',
+       otherwise, return error */
+    ret = gf_uuid_parse(loc->name, tmp_gfid);
+    if (ret)
+        goto err;
+
+    /* if its fresh lookup, go ahead and send it down, if not,
+       for directory, we need indirection to actual dir inode */
+    if (!(loc->inode && loc->inode->ia_type))
+        goto discover;
+
+    /* revalidate on directory */
+    ret = inode_ctx_get(loc->inode, this, &value);
+    if (ret)
+        goto err;
+
+    inode = (void *)(uintptr_t)value;
+
+    /* valid inode, already looked up, work on that */
+    if (inode->ia_type)
+        goto discover;
+
+    /* check if the inode is in the 'itable' or
+       if its just previously discover()'d inode */
+    true_inode = inode_find(loc->inode->table, tmp_gfid);
+    if (true_inode) {
+        /* time do another lookup and update the context
+           with proper inode */
+        op_errno = ESTALE;
+        /* 'inode_ref()' done in inode_find */
+        inode_unref(true_inode);
+        goto err;
+    }
 
 discover:
-        /* for the virtual entries, we don't need to send 'gfid-req' key, as
-           for these entries, we don't want to 'set' a new gfid */
-        if (xdata)
-                dict_del (xdata, "gfid-req");
+    /* for the virtual entries, we don't need to send 'gfid-req' key, as
+       for these entries, we don't want to 'set' a new gfid */
+    if (xdata)
+        dict_del(xdata, "gfid-req");
 
-        gf_uuid_copy (tmp_loc.gfid, tmp_gfid);
+    gf_uuid_copy(tmp_loc.gfid, tmp_gfid);
 
-        /* if revalidate, then we need to have the proper reference */
-        if (inode) {
-                tmp_loc.inode = inode_ref (inode);
-                frame->local = inode_ref (loc->inode);
-        } else {
-                tmp_loc.inode = inode_ref (loc->inode);
-        }
+    /* if revalidate, then we need to have the proper reference */
+    if (inode) {
+        tmp_loc.inode = inode_ref(inode);
+        frame->local = inode_ref(loc->inode);
+    } else {
+        tmp_loc.inode = inode_ref(loc->inode);
+    }
 
-        STACK_WIND (frame, ga_virtual_lookup_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->lookup, &tmp_loc, xdata);
+    STACK_WIND(frame, ga_virtual_lookup_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->lookup, &tmp_loc, xdata);
 
-        inode_unref (tmp_loc.inode);
+    inode_unref(tmp_loc.inode);
 
-        return 0;
+    return 0;
 
 wind:
-        /* used for all the normal lookup path */
-        STACK_WIND (frame, ga_lookup_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->lookup, loc, xdata);
+    /* used for all the normal lookup path */
+    STACK_WIND(frame, ga_lookup_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->lookup, loc, xdata);
 
-        return 0;
+    return 0;
 
 err:
-        STACK_UNWIND_STRICT (lookup, frame, -1, op_errno, loc->inode,
-                             &priv->gfiddir_stbuf, xdata,
-                             &priv->root_stbuf);
-        return 0;
+    STACK_UNWIND_STRICT(lookup, frame, -1, op_errno, loc->inode,
+                        &priv->gfiddir_stbuf, xdata, &priv->root_stbuf);
+    return 0;
 }
 
 int
-ga_mkdir (call_frame_t *frame, xlator_t *this, loc_t *loc, mode_t mode,
-          mode_t umask, dict_t *xdata)
+ga_mkdir(call_frame_t *frame, xlator_t *this, loc_t *loc, mode_t mode,
+         mode_t umask, dict_t *xdata)
 {
-        int op_errno = ENOMEM;
+    int op_errno = ENOMEM;
 
-        GFID_ACCESS_ENTRY_OP_CHECK (loc, op_errno, err);
+    GFID_ACCESS_ENTRY_OP_CHECK(loc, op_errno, err);
 
-        STACK_WIND (frame, default_mkdir_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->mkdir, loc, mode, umask,
-                    xdata);
+    STACK_WIND(frame, default_mkdir_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->mkdir, loc, mode, umask, xdata);
 
-        return 0;
+    return 0;
 
 err:
-        STACK_UNWIND_STRICT (mkdir, frame, -1, op_errno, loc->inode,
-                             NULL, NULL, NULL, xdata);
-        return 0;
+    STACK_UNWIND_STRICT(mkdir, frame, -1, op_errno, loc->inode, NULL, NULL,
+                        NULL, xdata);
+    return 0;
 }
 
-
 int
-ga_create (call_frame_t *frame, xlator_t *this, loc_t *loc, int flags,
-           mode_t mode, mode_t umask, fd_t *fd, dict_t *xdata)
+ga_create(call_frame_t *frame, xlator_t *this, loc_t *loc, int flags,
+          mode_t mode, mode_t umask, fd_t *fd, dict_t *xdata)
 {
-        int op_errno = ENOMEM;
+    int op_errno = ENOMEM;
 
-        GFID_ACCESS_ENTRY_OP_CHECK (loc, op_errno, err);
+    GFID_ACCESS_ENTRY_OP_CHECK(loc, op_errno, err);
 
-        STACK_WIND (frame, default_create_cbk,
-                    FIRST_CHILD(this), FIRST_CHILD(this)->fops->create,
-                    loc, flags, mode, umask, fd, xdata);
-        return 0;
+    STACK_WIND(frame, default_create_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->create, loc, flags, mode, umask, fd,
+               xdata);
+    return 0;
 err:
-        STACK_UNWIND_STRICT (create, frame, -1, op_errno, NULL,
-                             NULL, NULL, NULL, NULL, xdata);
-
-        return 0;
+    STACK_UNWIND_STRICT(create, frame, -1, op_errno, NULL, NULL, NULL, NULL,
+                        NULL, xdata);
 
+    return 0;
 }
 
 int
-ga_symlink (call_frame_t *frame, xlator_t *this, const char *linkname,
-            loc_t *loc, mode_t umask, dict_t *xdata)
+ga_symlink(call_frame_t *frame, xlator_t *this, const char *linkname,
+           loc_t *loc, mode_t umask, dict_t *xdata)
 {
-        int op_errno = ENOMEM;
+    int op_errno = ENOMEM;
 
-        GFID_ACCESS_ENTRY_OP_CHECK (loc, op_errno, err);
+    GFID_ACCESS_ENTRY_OP_CHECK(loc, op_errno, err);
 
-        STACK_WIND (frame, default_symlink_cbk,
-                    FIRST_CHILD(this), FIRST_CHILD(this)->fops->symlink,
-                    linkname, loc, umask, xdata);
-        return 0;
+    STACK_WIND(frame, default_symlink_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->symlink, linkname, loc, umask, xdata);
+    return 0;
 err:
-        STACK_UNWIND_STRICT (symlink, frame, -1, op_errno, NULL,
-                             NULL, NULL, NULL, xdata);
+    STACK_UNWIND_STRICT(symlink, frame, -1, op_errno, NULL, NULL, NULL, NULL,
+                        xdata);
 
-        return 0;
+    return 0;
 }
 
 int
-ga_mknod (call_frame_t *frame, xlator_t *this, loc_t *loc, mode_t mode,
-          dev_t rdev, mode_t umask, dict_t *xdata)
+ga_mknod(call_frame_t *frame, xlator_t *this, loc_t *loc, mode_t mode,
+         dev_t rdev, mode_t umask, dict_t *xdata)
 {
-        int op_errno = ENOMEM;
+    int op_errno = ENOMEM;
 
-        GFID_ACCESS_ENTRY_OP_CHECK (loc, op_errno, err);
+    GFID_ACCESS_ENTRY_OP_CHECK(loc, op_errno, err);
 
-        STACK_WIND (frame, default_mknod_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->mknod, loc, mode, rdev,
-                    umask, xdata);
+    STACK_WIND(frame, default_mknod_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->mknod, loc, mode, rdev, umask, xdata);
 
-        return 0;
+    return 0;
 err:
-        STACK_UNWIND_STRICT (mknod, frame, -1, op_errno, NULL,
-                             NULL, NULL, NULL, xdata);
+    STACK_UNWIND_STRICT(mknod, frame, -1, op_errno, NULL, NULL, NULL, NULL,
+                        xdata);
 
-        return 0;
+    return 0;
 }
 
 int
-ga_rmdir (call_frame_t *frame, xlator_t *this, loc_t *loc, int flag,
-          dict_t *xdata)
+ga_rmdir(call_frame_t *frame, xlator_t *this, loc_t *loc, int flag,
+         dict_t *xdata)
 {
-        int   op_errno = ENOMEM;
-        int   ret      = -1;
-        loc_t ga_loc   = {0, };
+    int op_errno = ENOMEM;
+    int ret = -1;
+    loc_t ga_loc = {
+        0,
+    };
 
-        GFID_ACCESS_ENTRY_OP_CHECK (loc, op_errno, err);
+    GFID_ACCESS_ENTRY_OP_CHECK(loc, op_errno, err);
 
-        ret = ga_valid_inode_loc_copy (&ga_loc, loc, this);
-        if (ret < 0)
-                goto err;
+    ret = ga_valid_inode_loc_copy(&ga_loc, loc, this);
+    if (ret < 0)
+        goto err;
 
-        STACK_WIND (frame, default_rmdir_cbk,
-                    FIRST_CHILD(this), FIRST_CHILD(this)->fops->rmdir,
-                    &ga_loc, flag, xdata);
+    STACK_WIND(frame, default_rmdir_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->rmdir, &ga_loc, flag, xdata);
 
-        loc_wipe (&ga_loc);
-        return 0;
+    loc_wipe(&ga_loc);
+    return 0;
 err:
-        STACK_UNWIND_STRICT (rmdir, frame, -1, op_errno, NULL,
-                             NULL, xdata);
+    STACK_UNWIND_STRICT(rmdir, frame, -1, op_errno, NULL, NULL, xdata);
 
-        return 0;
+    return 0;
 }
 
 int
-ga_unlink (call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t xflag,
-           dict_t *xdata)
+ga_unlink(call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t xflag,
+          dict_t *xdata)
 {
-        int   op_errno = ENOMEM;
-        int   ret      = -1;
-        loc_t ga_loc   = {0, };
+    int op_errno = ENOMEM;
+    int ret = -1;
+    loc_t ga_loc = {
+        0,
+    };
 
-        GFID_ACCESS_ENTRY_OP_CHECK (loc, op_errno, err);
+    GFID_ACCESS_ENTRY_OP_CHECK(loc, op_errno, err);
 
-        ret = ga_valid_inode_loc_copy (&ga_loc, loc, this);
-        if (ret < 0)
-                goto err;
+    ret = ga_valid_inode_loc_copy(&ga_loc, loc, this);
+    if (ret < 0)
+        goto err;
 
-        STACK_WIND (frame, default_unlink_cbk,
-                    FIRST_CHILD(this), FIRST_CHILD(this)->fops->unlink,
-                    &ga_loc, xflag, xdata);
+    STACK_WIND(frame, default_unlink_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->unlink, &ga_loc, xflag, xdata);
 
-        loc_wipe (&ga_loc);
-        return 0;
+    loc_wipe(&ga_loc);
+    return 0;
 err:
-        STACK_UNWIND_STRICT (unlink, frame, -1, op_errno, NULL,
-                             NULL, xdata);
+    STACK_UNWIND_STRICT(unlink, frame, -1, op_errno, NULL, NULL, xdata);
 
-        return 0;
+    return 0;
 }
 
 int
-ga_rename (call_frame_t *frame, xlator_t *this,
-           loc_t *oldloc, loc_t *newloc, dict_t *xdata)
+ga_rename(call_frame_t *frame, xlator_t *this, loc_t *oldloc, loc_t *newloc,
+          dict_t *xdata)
 {
-        int   op_errno  = ENOMEM;
-        int   ret       = 0;
-        loc_t ga_oldloc = {0, };
-        loc_t ga_newloc = {0, };
-
-        GFID_ACCESS_ENTRY_OP_CHECK (oldloc, op_errno, err);
-        GFID_ACCESS_ENTRY_OP_CHECK (newloc, op_errno, err);
-
-        ret = ga_valid_inode_loc_copy (&ga_oldloc, oldloc, this);
-        if (ret < 0)
-                goto err;
-
-        ret = ga_valid_inode_loc_copy (&ga_newloc, newloc, this);
-        if (ret < 0) {
-                loc_wipe (&ga_oldloc);
-                goto err;
-        }
-
-        STACK_WIND (frame, default_rename_cbk,
-                    FIRST_CHILD(this), FIRST_CHILD(this)->fops->rename,
-                    &ga_oldloc, &ga_newloc, xdata);
-
-        loc_wipe (&ga_newloc);
-        loc_wipe (&ga_oldloc);
-        return 0;
+    int op_errno = ENOMEM;
+    int ret = 0;
+    loc_t ga_oldloc = {
+        0,
+    };
+    loc_t ga_newloc = {
+        0,
+    };
+
+    GFID_ACCESS_ENTRY_OP_CHECK(oldloc, op_errno, err);
+    GFID_ACCESS_ENTRY_OP_CHECK(newloc, op_errno, err);
+
+    ret = ga_valid_inode_loc_copy(&ga_oldloc, oldloc, this);
+    if (ret < 0)
+        goto err;
+
+    ret = ga_valid_inode_loc_copy(&ga_newloc, newloc, this);
+    if (ret < 0) {
+        loc_wipe(&ga_oldloc);
+        goto err;
+    }
+
+    STACK_WIND(frame, default_rename_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->rename, &ga_oldloc, &ga_newloc, xdata);
+
+    loc_wipe(&ga_newloc);
+    loc_wipe(&ga_oldloc);
+    return 0;
 err:
-        STACK_UNWIND_STRICT (rename, frame, -1, op_errno, NULL,
-                             NULL, NULL, NULL, NULL, xdata);
+    STACK_UNWIND_STRICT(rename, frame, -1, op_errno, NULL, NULL, NULL, NULL,
+                        NULL, xdata);
 
-        return 0;
+    return 0;
 }
 
-
 int
-ga_link (call_frame_t *frame, xlator_t *this,
-         loc_t *oldloc, loc_t *newloc, dict_t *xdata)
+ga_link(call_frame_t *frame, xlator_t *this, loc_t *oldloc, loc_t *newloc,
+        dict_t *xdata)
 {
-        int   op_errno  = ENOMEM;
-        int   ret       = 0;
-        loc_t ga_oldloc = {0, };
-        loc_t ga_newloc = {0, };
-
-        GFID_ACCESS_ENTRY_OP_CHECK (oldloc, op_errno, err);
-        GFID_ACCESS_ENTRY_OP_CHECK (newloc, op_errno, err);
-
-        ret = ga_valid_inode_loc_copy (&ga_oldloc, oldloc, this);
-        if (ret < 0)
-                goto err;
-
-        ret = ga_valid_inode_loc_copy (&ga_newloc, newloc, this);
-        if (ret < 0) {
-                loc_wipe (&ga_oldloc);
-                goto err;
-        }
-
-        STACK_WIND (frame, default_link_cbk,
-                    FIRST_CHILD(this), FIRST_CHILD(this)->fops->link,
-                    &ga_oldloc, &ga_newloc, xdata);
-
-        loc_wipe (&ga_newloc);
-        loc_wipe (&ga_oldloc);
-        return 0;
+    int op_errno = ENOMEM;
+    int ret = 0;
+    loc_t ga_oldloc = {
+        0,
+    };
+    loc_t ga_newloc = {
+        0,
+    };
+
+    GFID_ACCESS_ENTRY_OP_CHECK(oldloc, op_errno, err);
+    GFID_ACCESS_ENTRY_OP_CHECK(newloc, op_errno, err);
+
+    ret = ga_valid_inode_loc_copy(&ga_oldloc, oldloc, this);
+    if (ret < 0)
+        goto err;
+
+    ret = ga_valid_inode_loc_copy(&ga_newloc, newloc, this);
+    if (ret < 0) {
+        loc_wipe(&ga_oldloc);
+        goto err;
+    }
+
+    STACK_WIND(frame, default_link_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->link, &ga_oldloc, &ga_newloc, xdata);
+
+    loc_wipe(&ga_newloc);
+    loc_wipe(&ga_oldloc);
+    return 0;
 
 err:
-        STACK_UNWIND_STRICT (link, frame, -1, op_errno, NULL,
-                             NULL, NULL, NULL, xdata);
+    STACK_UNWIND_STRICT(link, frame, -1, op_errno, NULL, NULL, NULL, NULL,
+                        xdata);
 
-        return 0;
+    return 0;
 }
 
 int32_t
-ga_opendir (call_frame_t *frame, xlator_t *this, loc_t *loc,
-            fd_t *fd, dict_t *xdata)
+ga_opendir(call_frame_t *frame, xlator_t *this, loc_t *loc, fd_t *fd,
+           dict_t *xdata)
 {
-        int op_errno = ENOMEM;
+    int op_errno = ENOMEM;
 
-        GFID_ACCESS_INODE_OP_CHECK (loc, op_errno, err);
+    GFID_ACCESS_INODE_OP_CHECK(loc, op_errno, err);
 
-        /* also check if the loc->inode itself is virtual
-           inode, if yes, return with failure, mainly because we
-           can't handle all the readdirp and other things on it. */
-        if (inode_ctx_get (loc->inode, this, NULL) == 0) {
-                op_errno = ENOTSUP;
-                goto err;
-        }
+    /* also check if the loc->inode itself is virtual
+       inode, if yes, return with failure, mainly because we
+       can't handle all the readdirp and other things on it. */
+    if (inode_ctx_get(loc->inode, this, NULL) == 0) {
+        op_errno = ENOTSUP;
+        goto err;
+    }
 
-        STACK_WIND (frame, default_opendir_cbk,
-                    FIRST_CHILD(this), FIRST_CHILD(this)->fops->opendir,
-                    loc, fd, xdata);
-        return 0;
+    STACK_WIND(frame, default_opendir_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->opendir, loc, fd, xdata);
+    return 0;
 err:
-        STACK_UNWIND_STRICT (opendir, frame, -1, op_errno, NULL, xdata);
+    STACK_UNWIND_STRICT(opendir, frame, -1, op_errno, NULL, xdata);
 
-        return 0;
+    return 0;
 }
 
 int32_t
-ga_getxattr (call_frame_t *frame, xlator_t *this, loc_t *loc,
-             const char *name, dict_t *xdata)
+ga_getxattr(call_frame_t *frame, xlator_t *this, loc_t *loc, const char *name,
+            dict_t *xdata)
 {
-        int   op_errno = ENOMEM;
-        int   ret      = -1;
-        loc_t ga_loc   = {0, };
+    int op_errno = ENOMEM;
+    int ret = -1;
+    loc_t ga_loc = {
+        0,
+    };
 
-        GFID_ACCESS_INODE_OP_CHECK (loc, op_errno, err);
-        ret = ga_valid_inode_loc_copy (&ga_loc, loc, this);
-        if (ret < 0)
-                goto err;
+    GFID_ACCESS_INODE_OP_CHECK(loc, op_errno, err);
+    ret = ga_valid_inode_loc_copy(&ga_loc, loc, this);
+    if (ret < 0)
+        goto err;
 
-        STACK_WIND (frame, default_getxattr_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->getxattr, &ga_loc, name, xdata);
+    STACK_WIND(frame, default_getxattr_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->getxattr, &ga_loc, name, xdata);
 
-        loc_wipe (&ga_loc);
+    loc_wipe(&ga_loc);
 
-        return 0;
+    return 0;
 err:
-        STACK_UNWIND_STRICT (getxattr, frame, -1, op_errno, NULL, xdata);
+    STACK_UNWIND_STRICT(getxattr, frame, -1, op_errno, NULL, xdata);
 
-        return 0;
+    return 0;
 }
 
 int32_t
-ga_stat (call_frame_t *frame, xlator_t *this, loc_t *loc,
-         dict_t *xdata)
+ga_stat(call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xdata)
 {
-        int          op_errno = ENOMEM;
-        int          ret      = -1;
-        loc_t        ga_loc   = {0, };
-        ga_private_t *priv    = NULL;
-
-        priv = this->private;
-        /* If stat is on ".gfid" itself, do not wind further,
-         * return fake stat and return success.
-         */
-        if (__is_gfid_access_dir(loc->gfid))
-                goto out;
-
-        ret = ga_valid_inode_loc_copy (&ga_loc, loc, this);
-        if (ret < 0)
-                goto err;
-
-        STACK_WIND (frame, default_stat_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->stat, &ga_loc, xdata);
-
-        loc_wipe (&ga_loc);
-        return 0;
+    int op_errno = ENOMEM;
+    int ret = -1;
+    loc_t ga_loc = {
+        0,
+    };
+    ga_private_t *priv = NULL;
+
+    priv = this->private;
+    /* If stat is on ".gfid" itself, do not wind further,
+     * return fake stat and return success.
+     */
+    if (__is_gfid_access_dir(loc->gfid))
+        goto out;
+
+    ret = ga_valid_inode_loc_copy(&ga_loc, loc, this);
+    if (ret < 0)
+        goto err;
+
+    STACK_WIND(frame, default_stat_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->stat, &ga_loc, xdata);
+
+    loc_wipe(&ga_loc);
+    return 0;
 
 err:
-        STACK_UNWIND_STRICT (stat, frame, -1, op_errno, NULL, xdata);
+    STACK_UNWIND_STRICT(stat, frame, -1, op_errno, NULL, xdata);
 
-        return 0;
+    return 0;
 
 out:
-        STACK_UNWIND_STRICT (stat, frame, 0, 0, &priv->gfiddir_stbuf, xdata);
-        return 0;
+    STACK_UNWIND_STRICT(stat, frame, 0, 0, &priv->gfiddir_stbuf, xdata);
+    return 0;
 }
 
 int32_t
-ga_setattr (call_frame_t *frame, xlator_t *this, loc_t *loc,
-            struct iatt *stbuf, int32_t valid,
-            dict_t *xdata)
+ga_setattr(call_frame_t *frame, xlator_t *this, loc_t *loc, struct iatt *stbuf,
+           int32_t valid, dict_t *xdata)
 {
-        int   op_errno = ENOMEM;
-        int   ret      = -1;
-        loc_t ga_loc   = {0, };
-
-        GFID_ACCESS_INODE_OP_CHECK (loc, op_errno, err);
-        ret = ga_valid_inode_loc_copy (&ga_loc, loc, this);
-        if (ret < 0)
-                goto err;
-
-        STACK_WIND (frame, default_setattr_cbk, FIRST_CHILD (this),
-                    FIRST_CHILD (this)->fops->setattr, &ga_loc, stbuf, valid,
-                    xdata);
-
-        loc_wipe (&ga_loc);
-        return 0;
+    int op_errno = ENOMEM;
+    int ret = -1;
+    loc_t ga_loc = {
+        0,
+    };
+
+    GFID_ACCESS_INODE_OP_CHECK(loc, op_errno, err);
+    ret = ga_valid_inode_loc_copy(&ga_loc, loc, this);
+    if (ret < 0)
+        goto err;
+
+    STACK_WIND(frame, default_setattr_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->setattr, &ga_loc, stbuf, valid, xdata);
+
+    loc_wipe(&ga_loc);
+    return 0;
 err:
-        STACK_UNWIND_STRICT (setattr, frame, -1, op_errno, NULL, NULL, xdata);
+    STACK_UNWIND_STRICT(setattr, frame, -1, op_errno, NULL, NULL, xdata);
 
-        return 0;
+    return 0;
 }
 
 int32_t
-ga_removexattr (call_frame_t *frame, xlator_t *this, loc_t *loc,
-                const char *name, dict_t *xdata)
+ga_removexattr(call_frame_t *frame, xlator_t *this, loc_t *loc,
+               const char *name, dict_t *xdata)
 {
-        int   op_errno = ENOMEM;
-        int   ret      = -1;
-        loc_t ga_loc   = {0, };
+    int op_errno = ENOMEM;
+    int ret = -1;
+    loc_t ga_loc = {
+        0,
+    };
 
-        GFID_ACCESS_INODE_OP_CHECK (loc, op_errno, err);
-        ret = ga_valid_inode_loc_copy (&ga_loc, loc, this);
-        if (ret < 0)
-                goto err;
+    GFID_ACCESS_INODE_OP_CHECK(loc, op_errno, err);
+    ret = ga_valid_inode_loc_copy(&ga_loc, loc, this);
+    if (ret < 0)
+        goto err;
 
-        STACK_WIND (frame, default_removexattr_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->removexattr, &ga_loc, name,
-                    xdata);
+    STACK_WIND(frame, default_removexattr_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->removexattr, &ga_loc, name, xdata);
 
-        loc_wipe (&ga_loc);
-        return 0;
+    loc_wipe(&ga_loc);
+    return 0;
 
 err:
-        STACK_UNWIND_STRICT (removexattr, frame, -1, op_errno, xdata);
+    STACK_UNWIND_STRICT(removexattr, frame, -1, op_errno, xdata);
 
-        return 0;
+    return 0;
 }
 
-
 int32_t
-mem_acct_init (xlator_t *this)
+mem_acct_init(xlator_t *this)
 {
-        int     ret = -1;
-
-        if (!this)
-                return ret;
+    int ret = -1;
 
-        ret = xlator_mem_acct_init (this, gf_gfid_access_mt_end + 1);
+    if (!this)
+        return ret;
 
-        if (ret != 0) {
-                gf_log (this->name, GF_LOG_WARNING, "Memory accounting"
-                        " init failed");
-                return ret;
-        }
+    ret = xlator_mem_acct_init(this, gf_gfid_access_mt_end + 1);
 
+    if (ret != 0) {
+        gf_log(this->name, GF_LOG_WARNING,
+               "Memory accounting"
+               " init failed");
         return ret;
+    }
+
+    return ret;
 }
 
 int32_t
-init (xlator_t *this)
+init(xlator_t *this)
 {
-        ga_private_t *priv = NULL;
-        int ret = -1;
-
-        if (!this->children || this->children->next) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "not configured with exactly one child. exiting");
-                goto out;
-        }
-
-        /* This can be the top of graph in certain cases */
-        if (!this->parents) {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "dangling volume. check volfile ");
-        }
-
-        /* TODO: define a mem-type structure */
-        priv = GF_CALLOC (1, sizeof (*priv), gf_gfid_access_mt_priv_t);
-        if (!priv)
-                goto out;
-
-        priv->newfile_args_pool = mem_pool_new (ga_newfile_args_t, 512);
-        if (!priv->newfile_args_pool)
-                goto out;
-
-        priv->heal_args_pool = mem_pool_new (ga_heal_args_t, 512);
-        if (!priv->heal_args_pool)
-                goto out;
-
-        this->local_pool = mem_pool_new (ga_local_t, 16);
-        if (!this->local_pool) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "failed to create local_t's memory pool");
-                goto out;
-        }
-
-        this->private = priv;
-
-        ret = 0;
+    ga_private_t *priv = NULL;
+    int ret = -1;
+
+    if (!this->children || this->children->next) {
+        gf_log(this->name, GF_LOG_ERROR,
+               "not configured with exactly one child. exiting");
+        goto out;
+    }
+
+    /* This can be the top of graph in certain cases */
+    if (!this->parents) {
+        gf_log(this->name, GF_LOG_DEBUG, "dangling volume. check volfile ");
+    }
+
+    /* TODO: define a mem-type structure */
+    priv = GF_CALLOC(1, sizeof(*priv), gf_gfid_access_mt_priv_t);
+    if (!priv)
+        goto out;
+
+    priv->newfile_args_pool = mem_pool_new(ga_newfile_args_t, 512);
+    if (!priv->newfile_args_pool)
+        goto out;
+
+    priv->heal_args_pool = mem_pool_new(ga_heal_args_t, 512);
+    if (!priv->heal_args_pool)
+        goto out;
+
+    this->local_pool = mem_pool_new(ga_local_t, 16);
+    if (!this->local_pool) {
+        gf_log(this->name, GF_LOG_ERROR,
+               "failed to create local_t's memory pool");
+        goto out;
+    }
+
+    this->private = priv;
+
+    ret = 0;
 out:
-        if (ret && priv) {
-                if (priv->newfile_args_pool)
-                        mem_pool_destroy (priv->newfile_args_pool);
-                GF_FREE (priv);
-        }
+    if (ret && priv) {
+        if (priv->newfile_args_pool)
+            mem_pool_destroy(priv->newfile_args_pool);
+        GF_FREE(priv);
+    }
 
-        return ret;
+    return ret;
 }
 
 void
-fini (xlator_t *this)
+fini(xlator_t *this)
 {
-        ga_private_t *priv = NULL;
-        priv = this->private;
-        this->private = NULL;
-
-        if (priv) {
-                if (priv->newfile_args_pool)
-                        mem_pool_destroy (priv->newfile_args_pool);
-                if (priv->heal_args_pool)
-                        mem_pool_destroy (priv->heal_args_pool);
-                GF_FREE (priv);
-        }
-
-        return;
+    ga_private_t *priv = NULL;
+    priv = this->private;
+    this->private = NULL;
+
+    if (priv) {
+        if (priv->newfile_args_pool)
+            mem_pool_destroy(priv->newfile_args_pool);
+        if (priv->heal_args_pool)
+            mem_pool_destroy(priv->heal_args_pool);
+        GF_FREE(priv);
+    }
+
+    return;
 }
 
 int32_t
-ga_dump_inodectx (xlator_t *this, inode_t *inode)
+ga_dump_inodectx(xlator_t *this, inode_t *inode)
 {
-        int       ret = -1;
-        uint64_t  value = 0;
-        inode_t  *tmp_inode = NULL;
-        char      key_prefix[GF_DUMP_MAX_BUF_LEN] = {0, };
-
-        ret = inode_ctx_get (inode, this, &value);
-        if (ret == 0) {
-                tmp_inode = (void*) value;
-                gf_proc_dump_build_key (key_prefix, this->name, "inode");
-                gf_proc_dump_add_section (key_prefix);
-                gf_proc_dump_write ("real-gfid", "%s",
-                                    uuid_utoa (tmp_inode->gfid));
-        }
-
-        return 0;
+    int ret = -1;
+    uint64_t value = 0;
+    inode_t *tmp_inode = NULL;
+    char key_prefix[GF_DUMP_MAX_BUF_LEN] = {
+        0,
+    };
+
+    ret = inode_ctx_get(inode, this, &value);
+    if (ret == 0) {
+        tmp_inode = (void *)(uintptr_t)value;
+        gf_proc_dump_build_key(key_prefix, this->name, "inode");
+        gf_proc_dump_add_section("%s", key_prefix);
+        gf_proc_dump_write("real-gfid", "%s", uuid_utoa(tmp_inode->gfid));
+    }
+
+    return 0;
 }
 
 struct xlator_fops fops = {
-        .lookup = ga_lookup,
-
-        /* entry fops */
-        .mkdir   = ga_mkdir,
-        .mknod   = ga_mknod,
-        .create  = ga_create,
-        .symlink = ga_symlink,
-        .link    = ga_link,
-        .unlink  = ga_unlink,
-        .rmdir   = ga_rmdir,
-        .rename  = ga_rename,
-
-        /* handle any other directory operations here */
-        .opendir  = ga_opendir,
-        .stat     = ga_stat,
-        .setattr  = ga_setattr,
-        .getxattr = ga_getxattr,
-        .removexattr = ga_removexattr,
-
-        /* special fop to handle more entry creations */
-        .setxattr = ga_setxattr,
+    .lookup = ga_lookup,
+
+    /* entry fops */
+    .mkdir = ga_mkdir,
+    .mknod = ga_mknod,
+    .create = ga_create,
+    .symlink = ga_symlink,
+    .link = ga_link,
+    .unlink = ga_unlink,
+    .rmdir = ga_rmdir,
+    .rename = ga_rename,
+
+    /* handle any other directory operations here */
+    .opendir = ga_opendir,
+    .stat = ga_stat,
+    .setattr = ga_setattr,
+    .getxattr = ga_getxattr,
+    .removexattr = ga_removexattr,
+
+    /* special fop to handle more entry creations */
+    .setxattr = ga_setxattr,
 };
 
 struct xlator_cbks cbks = {
-        .forget = ga_forget,
+    .forget = ga_forget,
 };
 
 struct xlator_dumpops dumpops = {
-        .inodectx       = ga_dump_inodectx,
+    .inodectx = ga_dump_inodectx,
 };
 
 struct volume_options options[] = {
-        /* This translator doesn't take any options, or provide any options */
-        { .key  = {NULL} },
+    /* This translator doesn't take any options, or provide any options */
+    {.key = {NULL}},
+};
+
+xlator_api_t xlator_api = {
+    .init = init,
+    .fini = fini,
+    .mem_acct_init = mem_acct_init,
+    .op_version = {1},
+    .fops = &fops,
+    .cbks = &cbks,
+    .options = options,
+    .identifier = "gfid-access",
+    .category = GF_MAINTAINED,
 };
diff --git a/xlators/features/gfid-access/src/gfid-access.h b/xlators/features/gfid-access/src/gfid-access.h
index 2449c5f6835..b1e255e56c0 100644
--- a/xlators/features/gfid-access/src/gfid-access.h
+++ b/xlators/features/gfid-access/src/gfid-access.h
@@ -10,102 +10,97 @@
 #ifndef __GFID_ACCESS_H__
 #define __GFID_ACCESS_H__
 
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
-
-#include "glusterfs.h"
-#include "logging.h"
-#include "dict.h"
-#include "xlator.h"
-#include "defaults.h"
+#include <glusterfs/glusterfs.h>
+#include <glusterfs/logging.h>
+#include <glusterfs/dict.h>
+#include <glusterfs/xlator.h>
+#include <glusterfs/defaults.h>
 #include "gfid-access-mem-types.h"
 
 #define UUID_CANONICAL_FORM_LEN 36
 
 #define GF_FUSE_AUX_GFID_NEWFILE "glusterfs.gfid.newfile"
-#define GF_FUSE_AUX_GFID_HEAL    "glusterfs.gfid.heal"
+#define GF_FUSE_AUX_GFID_HEAL "glusterfs.gfid.heal"
 
 #define GF_GFID_KEY "GLUSTERFS_GFID"
 #define GF_GFID_DIR ".gfid"
 #define GF_AUX_GFID 0xd
 
-#define GFID_ACCESS_ENTRY_OP_CHECK(loc,err,lbl)    do {                 \
-                /* need to check if the lookup is on virtual dir */     \
-                if ((loc->name && !strcmp (GF_GFID_DIR, loc->name)) &&  \
-                    ((loc->parent &&                                    \
-                      __is_root_gfid (loc->parent->gfid)) ||            \
-                      __is_root_gfid (loc->pargfid))) {                 \
-                        err = ENOTSUP;                                  \
-                        goto lbl;                                       \
-                }                                                       \
-                                                                        \
-                /* now, check if the lookup() is on an existing */      \
-                /* entry, but on gfid-path */                           \
-                if ((loc->parent &&                                     \
-                     __is_gfid_access_dir (loc->parent->gfid)) ||       \
-                    __is_gfid_access_dir (loc->pargfid)) {              \
-                        err = EPERM;                                    \
-                        goto lbl;                                       \
-                }                                                       \
-        } while (0)
+#define GFID_ACCESS_ENTRY_OP_CHECK(loc, err, lbl)                              \
+    do {                                                                       \
+        /* need to check if the lookup is on virtual dir */                    \
+        if ((loc->name && !strcmp(GF_GFID_DIR, loc->name)) &&                  \
+            ((loc->parent && __is_root_gfid(loc->parent->gfid)) ||             \
+             __is_root_gfid(loc->pargfid))) {                                  \
+            err = ENOTSUP;                                                     \
+            goto lbl;                                                          \
+        }                                                                      \
+                                                                               \
+        /* now, check if the lookup() is on an existing */                     \
+        /* entry, but on gfid-path */                                          \
+        if ((loc->parent && __is_gfid_access_dir(loc->parent->gfid)) ||        \
+            __is_gfid_access_dir(loc->pargfid)) {                              \
+            err = EPERM;                                                       \
+            goto lbl;                                                          \
+        }                                                                      \
+    } while (0)
 
-#define GFID_ACCESS_INODE_OP_CHECK(loc,err,lbl) do {                    \
-                /*Check if it is on .gfid*/                             \
-                if (__is_gfid_access_dir(loc->gfid)) {  \
-                        err = ENOTSUP;                                  \
-                        goto lbl;                                       \
-                }                                                       \
-        } while (0)
+#define GFID_ACCESS_INODE_OP_CHECK(loc, err, lbl)                              \
+    do {                                                                       \
+        /*Check if it is on .gfid*/                                            \
+        if (__is_gfid_access_dir(loc->gfid)) {                                 \
+            err = ENOTSUP;                                                     \
+            goto lbl;                                                          \
+        }                                                                      \
+    } while (0)
 typedef struct {
-        unsigned int  uid;
-        unsigned int  gid;
-        char          gfid[UUID_CANONICAL_FORM_LEN + 1];
-        unsigned int  st_mode;
-        char         *bname;
+    unsigned int uid;
+    unsigned int gid;
+    char gfid[UUID_CANONICAL_FORM_LEN + 1];
+    unsigned int st_mode;
+    char *bname;
 
-        union {
-                struct _symlink_in {
-                        char     *linkpath;
-                } __attribute__ ((__packed__)) symlink;
+    union {
+        struct _symlink_in {
+            char *linkpath;
+        } __attribute__((__packed__)) symlink;
 
-                struct _mknod_in {
-                        unsigned int mode;
-                        unsigned int rdev;
-                        unsigned int umask;
-                } __attribute__ ((__packed__)) mknod;
+        struct _mknod_in {
+            unsigned int mode;
+            unsigned int rdev;
+            unsigned int umask;
+        } __attribute__((__packed__)) mknod;
 
-                struct _mkdir_in {
-                        unsigned int mode;
-                        unsigned int umask;
-                } __attribute__ ((__packed__)) mkdir;
-        } __attribute__ ((__packed__)) args;
+        struct _mkdir_in {
+            unsigned int mode;
+            unsigned int umask;
+        } __attribute__((__packed__)) mkdir;
+    } __attribute__((__packed__)) args;
 } __attribute__((__packed__)) ga_newfile_args_t;
 
 typedef struct {
-        char      gfid[UUID_CANONICAL_FORM_LEN + 1];
-        char     *bname; /* a null terminated basename */
+    char gfid[UUID_CANONICAL_FORM_LEN + 1];
+    char *bname; /* a null terminated basename */
 } __attribute__((__packed__)) ga_heal_args_t;
 
 struct ga_private {
-        /* root inode's stbuf */
-        struct iatt root_stbuf;
-        struct iatt gfiddir_stbuf;
-        struct mem_pool *newfile_args_pool;
-        struct mem_pool *heal_args_pool;
+    /* root inode's stbuf */
+    struct iatt root_stbuf;
+    struct iatt gfiddir_stbuf;
+    struct mem_pool *newfile_args_pool;
+    struct mem_pool *heal_args_pool;
 };
 typedef struct ga_private ga_private_t;
 
 struct __ga_local {
-        call_frame_t *orig_frame;
-        unsigned int uid;
-        unsigned int gid;
-        loc_t        loc;
-        mode_t       mode;
-        dev_t        rdev;
-        mode_t       umask;
-        dict_t      *xdata;
+    call_frame_t *orig_frame;
+    unsigned int uid;
+    unsigned int gid;
+    loc_t loc;
+    mode_t mode;
+    dev_t rdev;
+    mode_t umask;
+    dict_t *xdata;
 };
 typedef struct __ga_local ga_local_t;
 
diff --git a/xlators/features/glupy/Makefile.am b/xlators/features/glupy/Makefile.am
deleted file mode 100644
index 060429ecf0f..00000000000
--- a/xlators/features/glupy/Makefile.am
+++ /dev/null
@@ -1,3 +0,0 @@
-SUBDIRS = src examples
-
-CLEANFILES =
diff --git a/xlators/features/glupy/doc/README.md b/xlators/features/glupy/doc/README.md
deleted file mode 100644
index 4b8b863ef39..00000000000
--- a/xlators/features/glupy/doc/README.md
+++ /dev/null
@@ -1,44 +0,0 @@
-This is just the very start for a GlusterFS[1] meta-translator that will
-allow translator code to be written in Python.  It's based on the standard
-Python embedding (not extending) techniques, plus a dash of the ctypes module.
-The interface is a pretty minimal adaptation of the dispatches and callbacks
-from the C API[2] to Python, as follows:
-
-* Dispatch functions and callbacks must be defined on an "xlator" class
-  derived from gluster.Translator so that they'll be auto-registered with
-  the C translator during initialization.
-
-* For each dispatch or callback function you want to intercept, you define a
-  Python function using the xxx\_fop\_t or xxx\_cbk\_t decorator.
-
-* The arguments for each operation are different, so you'll need to refer to
-  the C API.  GlusterFS-specific types are used (though only loc\_t is fully
-  defined so far) and type correctness is enforced by ctypes.
-
-* If you do intercept a dispatch function, it is your responsibility to call
-  xxx\_wind (like STACK\_WIND in the C API but operation-specific) to pass
-  the request to the next translator.  If you do not intercept a function, it
-  will default the same way as for C (pass through to the same operation with
-  the same arguments on the first child translator).
-
-* If you intercept a callback function, it is your responsibility to call
-  xxx\_unwind (like STACK\_UNWIND\_STRICT in the C API) to pass the request back
-  to the caller.
-
-So far only the lookup and create operations are handled this way, to support
-the "negative lookup" example.  Now that the basic infrastructure is in place,
-adding more functions should be very quick, though with that much boilerplate I
-might pause to write a code generator.  I also plan to add structure
-definitions and interfaces for some of the utility functions in libglusterfs
-(especially those having to do with inode and fd context) in the fairly near
-future.  Note that you can also use ctypes to get at anything not explicitly
-exposed to Python already.
-
-_If you're coming here because of the Linux Journal article, please note that
-the code has evolved since that was written. The version that matches the
-article is here:_
-
-https://github.com/jdarcy/glupy/tree/4bbae91ba459ea46ef32f2966562492e4ca9187a
-
-[1] http://www.gluster.org
-[2] http://pl.atyp.us/hekafs.org/dist/xlator_api_2.html
diff --git a/xlators/features/glupy/doc/TESTING b/xlators/features/glupy/doc/TESTING
deleted file mode 100644
index e05f17f498f..00000000000
--- a/xlators/features/glupy/doc/TESTING
+++ /dev/null
@@ -1,9 +0,0 @@
-Loading a translator written in Python using the glupy meta translator
--------------------------------------------------------------------------------
-'test.vol' is a simple volfile with the debug-trace Python translator on top
-of a brick. The volfile can be mounted using the following command.
-
-$ glusterfs --debug -f test.vol /path/to/mntpt
-
-If then file operations are performed on the newly mounted file system, log
-output would be printed by the Python translator on the standard output.
diff --git a/xlators/features/glupy/doc/test.vol b/xlators/features/glupy/doc/test.vol
deleted file mode 100644
index 0751a488c1f..00000000000
--- a/xlators/features/glupy/doc/test.vol
+++ /dev/null
@@ -1,10 +0,0 @@
-volume vol-posix
-    type storage/posix
-    option directory /path/to/brick
-end-volume
-
-volume vol-glupy
-    type features/glupy
-    option module-name debug-trace
-    subvolumes vol-posix
-end-volume
diff --git a/xlators/features/glupy/examples/Makefile.am b/xlators/features/glupy/examples/Makefile.am
deleted file mode 100644
index c26abeaafb6..00000000000
--- a/xlators/features/glupy/examples/Makefile.am
+++ /dev/null
@@ -1,5 +0,0 @@
-xlatordir = $(libdir)/glusterfs/$(PACKAGE_VERSION)/xlator/features
-
-glupyexamplesdir = $(xlatordir)/glupy
-
-glupyexamples_PYTHON = negative.py helloworld.py debug-trace.py
diff --git a/xlators/features/glupy/examples/debug-trace.py b/xlators/features/glupy/examples/debug-trace.py
deleted file mode 100644
index 6eef1b58b8f..00000000000
--- a/xlators/features/glupy/examples/debug-trace.py
+++ /dev/null
@@ -1,775 +0,0 @@
-import sys
-import stat
-from uuid import UUID
-from time import strftime, localtime
-from gluster.glupy import *
-
-# This translator was written primarily to test the fop entry point definitions
-# and structure definitions in 'glupy.py'.
-
-# It is similar to the C language debug-trace translator, which logs the
-# arguments passed to the fops and their corresponding cbk functions.
-
-dl.get_id.restype = c_long
-dl.get_id.argtypes = [ POINTER(call_frame_t) ]
-
-dl.get_rootunique.restype = c_uint64
-dl.get_rootunique.argtypes = [ POINTER(call_frame_t) ]
-
-def uuid2str (gfid):
-        return str(UUID(''.join(map("{0:02x}".format, gfid))))
-
-
-def st_mode_from_ia (prot, filetype):
-        st_mode = 0
-        type_bit = 0
-        prot_bit = 0
-
-        if filetype == IA_IFREG:
-                type_bit = stat.S_IFREG
-        elif filetype == IA_IFDIR:
-                type_bit = stat.S_IFDIR
-        elif filetype == IA_IFLNK:
-                type_bit = stat.S_IFLNK
-        elif filetype == IA_IFBLK:
-                type_bit = stat.S_IFBLK
-        elif filetype == IA_IFCHR:
-                type_bit = stat.S_IFCHR
-        elif filetype == IA_IFIFO:
-                type_bit = stat.S_IFIFO
-        elif filetype == IA_IFSOCK:
-                type_bit = stat.S_IFSOCK
-        elif filetype == IA_INVAL:
-                pass
-
-
-        if prot.suid:
-                prot_bit |= stat.S_ISUID
-        if prot.sgid:
-                prot_bit |= stat.S_ISGID
-        if prot.sticky:
-                prot_bit |= stat.S_ISVTX
-
-        if prot.owner.read:
-                prot_bit |= stat.S_IRUSR
-        if prot.owner.write:
-                prot_bit |= stat.S_IWUSR
-        if prot.owner.execn:
-                prot_bit |= stat.S_IXUSR
-
-        if prot.group.read:
-                prot_bit |= stat.S_IRGRP
-        if prot.group.write:
-                prot_bit |= stat.S_IWGRP
-        if prot.group.execn:
-                prot_bit |= stat.S_IXGRP
-
-        if prot.other.read:
-                prot_bit |= stat.S_IROTH
-        if prot.other.write:
-                prot_bit |= stat.S_IWOTH
-        if prot.other.execn:
-                prot_bit |= stat.S_IXOTH
-
-        st_mode = (type_bit | prot_bit)
-
-        return st_mode
-
-
-def trace_stat2str (buf):
-        gfid = uuid2str(buf.contents.ia_gfid)
-        mode = st_mode_from_ia(buf.contents.ia_prot, buf.contents.ia_type)
-        atime_buf = strftime("[%b %d %H:%M:%S]",
-                             localtime(buf.contents.ia_atime))
-        mtime_buf = strftime("[%b %d %H:%M:%S]",
-                             localtime(buf.contents.ia_mtime))
-        ctime_buf = strftime("[%b %d %H:%M:%S]",
-                             localtime(buf.contents.ia_ctime))
-        return ("(gfid={0:s}, ino={1:d}, mode={2:o}, nlink={3:d}, uid ={4:d}, "+
-                "gid ={5:d}, size={6:d}, blocks={7:d}, atime={8:s}, mtime={9:s}, "+
-                "ctime={10:s})").format(gfid, buf.contents.ia_no, mode,
-                                        buf.contents.ia_nlink,
-                                        buf.contents.ia_uid,
-                                        buf.contents.ia_gid,
-                                        buf.contents.ia_size,
-                                        buf.contents.ia_blocks,
-                                        atime_buf, mtime_buf,
-                                        ctime_buf)
-
-class xlator(Translator):
-
-        def __init__(self, c_this):
-                Translator.__init__(self, c_this)
-                self.gfids = {}
-
-        def lookup_fop(self, frame, this, loc, xdata):
-                unique = dl.get_rootunique(frame)
-                key = dl.get_id(frame)
-                gfid = uuid2str(loc.contents.gfid)
-                print("GLUPY TRACE LOOKUP FOP- {0:d}: gfid={1:s}; " +
-                      "path={2:s}").format(unique, gfid, loc.contents.path)
-                self.gfids[key] = gfid
-                dl.wind_lookup(frame, POINTER(xlator_t)(), loc, xdata)
-                return 0
-
-        def lookup_cbk(self, frame, cookie, this, op_ret, op_errno,
-                       inode, buf, xdata, postparent):
-                unique =dl.get_rootunique(frame)
-                key =dl.get_id(frame)
-                if op_ret == 0:
-                        gfid = uuid2str(buf.contents.ia_gfid)
-                        statstr = trace_stat2str(buf)
-                        postparentstr = trace_stat2str(postparent)
-                        print("GLUPY TRACE LOOKUP CBK- {0:d}: gfid={1:s}; "+
-                              "op_ret={2:d}; *buf={3:s}; " +
-                              "*postparent={4:s}").format(unique, gfid,
-                                                          op_ret, statstr,
-                                                          postparentstr)
-                else:
-                        gfid = self.gfids[key]
-                        print("GLUPY TRACE LOOKUP CBK - {0:d}: gfid={1:s};" +
-                              " op_ret={2:d}; op_errno={3:d}").format(unique,
-                                                                      gfid,
-                                                                      op_ret,
-                                                                      op_errno)
-                del self.gfids[key]
-                dl.unwind_lookup(frame, cookie, this, op_ret, op_errno,
-                                 inode, buf, xdata, postparent)
-                return 0
-
-        def create_fop(self, frame, this, loc, flags, mode, umask, fd,
-                       xdata):
-                unique = dl.get_rootunique(frame)
-                gfid = uuid2str(loc.contents.gfid)
-                print("GLUPY TRACE CREATE FOP- {0:d}: gfid={1:s}; path={2:s}; " +
-                      "fd={3:s}; flags=0{4:o}; mode=0{5:o}; " +
-                      "umask=0{6:o}").format(unique, gfid, loc.contents.path,
-                                             fd, flags, mode, umask)
-                dl.wind_create(frame, POINTER(xlator_t)(), loc, flags,mode,
-                               umask, fd, xdata)
-                return 0
-
-        def create_cbk(self, frame, cookie, this, op_ret, op_errno, fd,
-                       inode, buf, preparent, postparent, xdata):
-                unique = dl.get_rootunique(frame)
-                if op_ret >= 0:
-                        gfid = uuid2str(inode.contents.gfid)
-                        statstr = trace_stat2str(buf)
-                        preparentstr = trace_stat2str(preparent)
-                        postparentstr = trace_stat2str(postparent)
-                        print("GLUPY TRACE CREATE CBK- {0:d}: gfid={1:s};" +
-                              " op_ret={2:d}; fd={3:s}; *stbuf={4:s}; " +
-                              "*preparent={5:s};" +
-                              " *postparent={6:s}").format(unique, gfid, op_ret,
-                                                           fd, statstr,
-                                                           preparentstr,
-                                                           postparentstr)
-                else:
-                        print ("GLUPY TRACE CREATE CBK- {0:d}: op_ret={1:d}; " +
-                              "op_errno={2:d}").format(unique, op_ret, op_errno)
-                dl.unwind_create(frame, cookie, this, op_ret, op_errno, fd,
-                                 inode, buf, preparent, postparent, xdata)
-                return 0
-
-        def open_fop(self, frame, this, loc, flags, fd, xdata):
-                unique = dl.get_rootunique(frame)
-                key = dl.get_id(frame)
-                gfid = uuid2str(loc.contents.inode.contents.gfid)
-                print("GLUPY TRACE OPEN FOP- {0:d}: gfid={1:s}; path={2:s}; "+
-                      "flags={3:d}; fd={4:s}").format(unique, gfid,
-                                                      loc.contents.path, flags,
-                                                      fd)
-                self.gfids[key] = gfid
-                dl.wind_open(frame, POINTER(xlator_t)(), loc, flags, fd, xdata)
-                return 0
-
-        def open_cbk(self, frame, cookie, this, op_ret, op_errno, fd, xdata):
-                unique = dl.get_rootunique(frame)
-                key = dl.get_id(frame)
-                gfid = self.gfids[key]
-                print("GLUPY TRACE OPEN CBK- {0:d}: gfid={1:s}; op_ret={2:d}; "
-                      "op_errno={3:d}; *fd={4:s}").format(unique, gfid,
-                                                          op_ret, op_errno, fd)
-                del self.gfids[key]
-                dl.unwind_open(frame, cookie, this, op_ret, op_errno, fd,
-                               xdata)
-                return 0
-
-        def readv_fop(self, frame, this, fd, size, offset, flags, xdata):
-                unique = dl.get_rootunique(frame)
-                key = dl.get_id(frame)
-                gfid = uuid2str(fd.contents.inode.contents.gfid)
-                print("GLUPY TRACE READV FOP- {0:d}: gfid={1:s}; "+
-                      "fd={2:s}; size ={3:d}; offset={4:d}; " +
-                      "flags=0{5:x}").format(unique, gfid, fd, size, offset,
-                                             flags)
-                self.gfids[key] = gfid
-                dl.wind_readv (frame, POINTER(xlator_t)(), fd, size, offset,
-                               flags, xdata)
-                return 0
-
-        def readv_cbk(self, frame, cookie, this, op_ret, op_errno, vector,
-                      count, buf, iobref, xdata):
-                unique = dl.get_rootunique(frame)
-                key = dl.get_id(frame)
-                gfid = self.gfids[key]
-                if op_ret >= 0:
-                        statstr = trace_stat2str(buf)
-                        print("GLUPY TRACE READV CBK- {0:d}: gfid={1:s}, "+
-                              "op_ret={2:d}; *buf={3:s};").format(unique, gfid,
-                                                                  op_ret,
-                                                                  statstr)
-
-                else:
-                        print("GLUPY TRACE READV CBK- {0:d}: gfid={1:s}, "+
-                              "op_ret={2:d}; op_errno={3:d}").format(unique,
-                                                                     gfid,
-                                                                     op_ret,
-                                                                     op_errno)
-                del self.gfids[key]
-                dl.unwind_readv (frame, cookie, this, op_ret, op_errno,
-                                 vector, count, buf, iobref, xdata)
-                return 0
-
-        def writev_fop(self, frame, this, fd, vector, count, offset, flags,
-                       iobref, xdata):
-                unique = dl.get_rootunique(frame)
-                key = dl.get_id(frame)
-                gfid = uuid2str(fd.contents.inode.contents.gfid)
-                print("GLUPY TRACE  WRITEV FOP- {0:d}: gfid={1:s}; " +
-                      "fd={2:s}; count={3:d}; offset={4:d}; " +
-                      "flags=0{5:x}").format(unique, gfid, fd, count, offset,
-                                             flags)
-                self.gfids[key] = gfid
-                dl.wind_writev(frame, POINTER(xlator_t)(), fd, vector, count,
-                               offset, flags, iobref, xdata)
-                return 0
-
-        def writev_cbk(self, frame, cookie, this, op_ret, op_errno, prebuf,
-                       postbuf, xdata):
-                unique = dl.get_rootunique(frame)
-                key = dl.get_id(frame)
-                if op_ret >= 0:
-                        preopstr = trace_stat2str(prebuf)
-                        postopstr = trace_stat2str(postbuf)
-                        print("GLUPY TRACE WRITEV CBK- {0:d}: op_ret={1:d}; " +
-                              "*prebuf={2:s}; " +
-                              "*postbuf={3:s}").format(unique, op_ret, preopstr,
-                                                       postopstr)
-                else:
-                        gfid = self.gfids[key]
-                        print("GLUPY TRACE WRITEV CBK- {0:d}: gfid={1:s}; "+
-                              "op_ret={2:d}; op_errno={3:d}").format(unique,
-                                                                     gfid,
-                                                                     op_ret,
-                                                                     op_errno)
-                del self.gfids[key]
-                dl.unwind_writev (frame, cookie, this, op_ret, op_errno,
-                                  prebuf, postbuf, xdata)
-                return 0
-
-        def opendir_fop(self, frame, this, loc, fd, xdata):
-                unique = dl.get_rootunique(frame)
-                key = dl.get_id(frame)
-                gfid = uuid2str(loc.contents.inode.contents.gfid)
-                print("GLUPY TRACE OPENDIR FOP- {0:d}: gfid={1:s}; path={2:s}; "+
-                      "fd={3:s}").format(unique, gfid, loc.contents.path, fd)
-                self.gfids[key] = gfid
-                dl.wind_opendir(frame, POINTER(xlator_t)(), loc, fd, xdata)
-                return 0
-
-        def opendir_cbk(self, frame, cookie, this, op_ret, op_errno, fd,
-                        xdata):
-                unique = dl.get_rootunique(frame)
-                key = dl.get_id(frame)
-                gfid = self.gfids[key]
-                print("GLUPY TRACE OPENDIR CBK- {0:d}: gfid={1:s}; op_ret={2:d};"+
-                      " op_errno={3:d}; fd={4:s}").format(unique, gfid, op_ret,
-                                                          op_errno, fd)
-                del self.gfids[key]
-                dl.unwind_opendir(frame, cookie, this, op_ret, op_errno,
-                                  fd, xdata)
-                return 0
-
-        def readdir_fop(self, frame, this, fd, size, offset, xdata):
-                unique = dl.get_rootunique(frame)
-                key = dl.get_id(frame)
-                gfid = uuid2str(fd.contents.inode.contents.gfid)
-                print("GLUPY TRACE READDIR FOP- {0:d}:  gfid={1:s}; fd={2:s}; " +
-                      "size={3:d}; offset={4:d}").format(unique, gfid, fd, size,
-                                                         offset)
-                self.gfids[key] = gfid
-                dl.wind_readdir(frame, POINTER(xlator_t)(), fd, size, offset,
-                                xdata)
-                return 0
-
-        def readdir_cbk(self, frame, cookie, this, op_ret, op_errno, buf,
-                        xdata):
-                unique = dl.get_rootunique(frame)
-                key = dl.get_id(frame)
-                gfid = self.gfids[key]
-                print("GLUPY TRACE READDIR CBK- {0:d}: gfid={1:s}; op_ret={2:d};"+
-                      " op_errno={3:d}").format(unique, gfid, op_ret, op_errno)
-                del self.gfids[key]
-                dl.unwind_readdir(frame, cookie, this, op_ret, op_errno, buf,
-                                  xdata)
-                return 0
-
-        def readdirp_fop(self, frame, this, fd, size, offset, dictionary):
-                unique = dl.get_rootunique(frame)
-                key = dl.get_id(frame)
-                gfid = uuid2str(fd.contents.inode.contents.gfid)
-                print("GLUPY TRACE READDIRP FOP- {0:d}: gfid={1:s}; fd={2:s}; "+
-                      " size={3:d}; offset={4:d}").format(unique, gfid, fd, size,
-                                                          offset)
-                self.gfids[key] = gfid
-                dl.wind_readdirp(frame, POINTER(xlator_t)(), fd, size, offset,
-                                 dictionary)
-                return 0
-
-        def readdirp_cbk(self, frame, cookie, this, op_ret, op_errno, buf,
-                         xdata):
-                unique = dl.get_rootunique(frame)
-                key = dl.get_id(frame)
-                gfid = self.gfids[key]
-                print("GLUPY TRACE READDIRP CBK- {0:d}: gfid={1:s}; "+
-                      "op_ret={2:d}; op_errno={3:d}").format(unique, gfid,
-                                                             op_ret, op_errno)
-                del self.gfids[key]
-                dl.unwind_readdirp(frame, cookie, this, op_ret, op_errno, buf,
-                                  xdata)
-                return 0
-
-        def mkdir_fop(self, frame, this, loc, mode, umask, xdata):
-                unique = dl.get_rootunique(frame)
-                gfid = uuid2str(loc.contents.inode.contents.gfid)
-                print("GLUPY TRACE MKDIR FOP- {0:d}: gfid={1:s}; path={2:s}; " +
-                      "mode={3:d}; umask=0{4:o}").format(unique, gfid,
-                                                         loc.contents.path, mode,
-                                                         umask)
-                dl.wind_mkdir(frame, POINTER(xlator_t)(), loc, mode, umask,
-                              xdata)
-                return 0
-
-        def mkdir_cbk(self, frame, cookie, this, op_ret, op_errno, inode, buf,
-                      preparent, postparent,  xdata):
-                unique = dl.get_rootunique(frame)
-                if op_ret == 0:
-                        gfid = uuid2str(inode.contents.gfid)
-                        statstr = trace_stat2str(buf)
-                        preparentstr = trace_stat2str(preparent)
-                        postparentstr = trace_stat2str(postparent)
-                        print("GLUPY TRACE MKDIR CBK- {0:d}: gfid={1:s}; "+
-                              "op_ret={2:d}; *stbuf={3:s}; *prebuf={4:s}; "+
-                              "*postbuf={5:s} ").format(unique, gfid, op_ret,
-                                                        statstr,
-                                                        preparentstr,
-                                                        postparentstr)
-                else:
-                        print("GLUPY TRACE MKDIR CBK- {0:d}:  op_ret={1:d}; "+
-                              "op_errno={2:d}").format(unique, op_ret, op_errno)
-                dl.unwind_mkdir(frame, cookie, this, op_ret, op_errno, inode,
-                                buf, preparent, postparent, xdata)
-                return 0
-
-        def rmdir_fop(self, frame, this, loc, flags, xdata):
-                unique = dl.get_rootunique(frame)
-                key = dl.get_id(frame)
-                gfid = uuid2str(loc.contents.inode.contents.gfid)
-                print("GLUPY TRACE RMDIR FOP- {0:d}: gfid={1:s}; path={2:s}; "+
-                      "flags={3:d}").format(unique, gfid, loc.contents.path,
-                                            flags)
-                self.gfids[key] = gfid
-                dl.wind_rmdir(frame, POINTER(xlator_t)(), loc, flags, xdata)
-                return 0
-
-        def rmdir_cbk(self, frame, cookie, this, op_ret, op_errno, preparent,
-                      postparent, xdata):
-                unique = dl.get_rootunique(frame)
-                key = dl.get_id(frame)
-                gfid = self.gfids[key]
-                if op_ret == 0:
-                        preparentstr = trace_stat2str(preparent)
-                        postparentstr = trace_stat2str(postparent)
-                        print("GLUPY TRACE RMDIR CBK- {0:d}: gfid={1:s}; "+
-                              "op_ret={2:d}; *prebuf={3:s}; "+
-                              "*postbuf={4:s}").format(unique, gfid, op_ret,
-                                                       preparentstr,
-                                                       postparentstr)
-                else:
-                        print("GLUPY TRACE RMDIR CBK- {0:d}: gfid={1:s}; "+
-                              "op_ret={2:d}; op_errno={3:d}").format(unique,
-                                                                     gfid,
-                                                                     op_ret,
-                                                                     op_errno)
-                del self.gfids[key]
-                dl.unwind_rmdir(frame, cookie, this, op_ret, op_errno,
-                                preparent, postparent, xdata)
-                return 0
-
-        def stat_fop(self, frame, this, loc, xdata):
-                unique = dl.get_rootunique(frame)
-                key = dl.get_id(frame)
-                gfid = uuid2str(loc.contents.inode.contents.gfid)
-                print("GLUPY TRACE STAT FOP- {0:d}: gfid={1:s}; " +
-                      " path={2:s}").format(unique, gfid, loc.contents.path)
-                self.gfids[key] = gfid
-                dl.wind_stat(frame, POINTER(xlator_t)(), loc, xdata)
-                return 0
-
-        def stat_cbk(self, frame, cookie, this, op_ret, op_errno, buf,
-                     xdata):
-                unique = dl.get_rootunique(frame)
-                key = dl.get_id(frame)
-                gfid = self.gfids[key]
-                if op_ret == 0:
-                        statstr = trace_stat2str(buf)
-                        print("GLUPY TRACE STAT CBK- {0:d}: gfid={1:s}; "+
-                              "op_ret={2:d};  *buf={3:s};").format(unique,
-                                                                   gfid,
-                                                                   op_ret,
-                                                                   statstr)
-                else:
-                        print("GLUPY TRACE STAT CBK- {0:d}: gfid={1:s}; "+
-                              "op_ret={2:d}; op_errno={3:d}").format(unique,
-                                                                     gfid,
-                                                                     op_ret,
-                                                                     op_errno)
-                del self.gfids[key]
-                dl.unwind_stat(frame, cookie, this, op_ret, op_errno,
-                               buf, xdata)
-                return 0
-
-        def fstat_fop(self, frame, this, fd, xdata):
-                unique = dl.get_rootunique(frame)
-                key = dl.get_id(frame)
-                gfid = uuid2str(fd.contents.inode.contents.gfid)
-                print("GLUPY TRACE FSTAT FOP- {0:d}:  gfid={1:s}; " +
-                      "fd={2:s}").format(unique, gfid, fd)
-                self.gfids[key] = gfid
-                dl.wind_fstat(frame, POINTER(xlator_t)(), fd, xdata)
-                return 0
-
-        def fstat_cbk(self, frame, cookie, this, op_ret, op_errno, buf,
-                      xdata):
-                unique = dl.get_rootunique(frame)
-                key = dl.get_id(frame)
-                gfid = self.gfids[key]
-                if op_ret == 0:
-                        statstr = trace_stat2str(buf)
-                        print("GLUPY TRACE FSTAT CBK- {0:d}: gfid={1:s} "+
-                              " op_ret={2:d}; *buf={3:s}").format(unique,
-                                                                  gfid,
-                                                                  op_ret,
-                                                                  statstr)
-                else:
-                        print("GLUPY TRACE FSTAT CBK- {0:d}: gfid={1:s} "+
-                              "op_ret={2:d}; op_errno={3:d}").format(unique.
-                                                                     gfid,
-                                                                     op_ret,
-                                                                     op_errno)
-                del self.gfids[key]
-                dl.unwind_fstat(frame, cookie, this, op_ret, op_errno,
-                                buf, xdata)
-                return 0
-
-        def statfs_fop(self, frame, this, loc, xdata):
-                unique = dl.get_rootunique(frame)
-                if loc.contents.inode:
-                        gfid = uuid2str(loc.contents.inode.contents.gfid)
-                else:
-                        gfid = "0"
-                print("GLUPY TRACE STATFS FOP- {0:d}: gfid={1:s}; "+
-                      "path={2:s}").format(unique, gfid, loc.contents.path)
-                dl.wind_statfs(frame, POINTER(xlator_t)(), loc, xdata)
-                return 0
-
-        def statfs_cbk(self, frame, cookie, this, op_ret, op_errno, buf,
-                       xdata):
-                unique = dl.get_rootunique(frame)
-                if op_ret == 0:
-                        #TBD: print buf (pointer to an iovec type object)
-                        print("GLUPY TRACE STATFS CBK {0:d}: "+
-                              "op_ret={1:d}").format(unique, op_ret)
-                else:
-                        print("GLUPY TRACE STATFS CBK-  {0:d}"+
-                              "op_ret={1:d}; op_errno={2:d}").format(unique,
-                                                                     op_ret,
-                                                                     op_errno)
-                dl.unwind_statfs(frame, cookie, this, op_ret, op_errno,
-                                 buf, xdata)
-                return 0
-
-        def getxattr_fop(self, frame, this, loc, name, xdata):
-                unique = dl.get_rootunique(frame)
-                key = dl.get_id(frame)
-                gfid = uuid2str(loc.contents.inode.contents.gfid)
-                print("GLUPY TRACE GETXATTR FOP- {0:d}: gfid={1:s}; path={2:s};"+
-                      " name={3:s}").format(unique, gfid, loc.contents.path,
-                                            name)
-                self.gfids[key]=gfid
-                dl.wind_getxattr(frame, POINTER(xlator_t)(), loc, name, xdata)
-                return 0
-
-        def getxattr_cbk(self, frame, cookie, this, op_ret, op_errno,
-                         dictionary, xdata):
-                unique = dl.get_rootunique(frame)
-                key = dl.get_id(frame)
-                gfid = self.gfids[key]
-                print("GLUPY TRACE GETXATTR CBK- {0:d}: gfid={1:s}; "+
-                      "op_ret={2:d}; op_errno={3:d}; "+
-                      " dictionary={4:s}").format(unique, gfid, op_ret, op_errno,
-                                                  dictionary)
-                del self.gfids[key]
-                dl.unwind_getxattr(frame, cookie, this, op_ret, op_errno,
-                                   dictionary, xdata)
-                return 0
-
-        def fgetxattr_fop(self, frame, this, fd, name, xdata):
-                unique = dl.get_rootunique(frame)
-                key = dl.get_id(frame)
-                gfid = uuid2str(fd.contents.inode.contents.gfid)
-                print("GLUPY TRACE FGETXATTR FOP- {0:d}: gfid={1:s}; fd={2:s}; "+
-                      "name={3:s}").format(unique, gfid, fd, name)
-                self.gfids[key] = gfid
-                dl.wind_fgetxattr(frame, POINTER(xlator_t)(), fd, name, xdata)
-                return 0
-
-        def fgetxattr_cbk(self, frame, cookie, this, op_ret, op_errno,
-                          dictionary, xdata):
-                unique = dl.get_rootunique(frame)
-                key = dl.get_id(frame)
-                gfid = self.gfids[key]
-                print("GLUPY TRACE FGETXATTR CBK- {0:d}: gfid={1:s}; "+
-                      "op_ret={2:d}; op_errno={3:d};"+
-                      " dictionary={4:s}").format(unique, gfid, op_ret,
-                                                  op_errno, dictionary)
-                del self.gfids[key]
-                dl.unwind_fgetxattr(frame, cookie, this, op_ret, op_errno,
-                                    dictionary, xdata)
-                return 0
-
-        def setxattr_fop(self, frame, this, loc, dictionary, flags, xdata):
-                unique = dl.get_rootunique(frame)
-                key = dl.get_id(frame)
-                gfid = uuid2str(loc.contents.inode.contents.gfid)
-                print("GLUPY TRACE SETXATTR FOP- {0:d}:  gfid={1:s}; path={2:s};"+
-                      " flags={3:d}").format(unique, gfid, loc.contents.path,
-                                             flags)
-                self.gfids[key] = gfid
-                dl.wind_setxattr(frame, POINTER(xlator_t)(), loc, dictionary,
-                                 flags, xdata)
-                return 0
-
-        def setxattr_cbk(self, frame, cookie, this, op_ret, op_errno, xdata):
-                unique = dl.get_rootunique(frame)
-                key = dl.get_id(frame)
-                gfid = self.gfids[key]
-                print("GLUPY TRACE SETXATTR CBK- {0:d}: gfid={1:s}; "+
-                      "op_ret={2:d}; op_errno={3:d}").format(unique, gfid,
-                                                             op_ret, op_errno)
-                del self.gfids[key]
-                dl.unwind_setxattr(frame, cookie, this, op_ret, op_errno,
-                                   xdata)
-                return 0
-
-        def fsetxattr_fop(self, frame, this, fd, dictionary, flags, xdata):
-                unique = dl.get_rootunique(frame)
-                key = dl.get_id(frame)
-                gfid = uuid2str(fd.contents.inode.contents.gfid)
-                print("GLUPY TRACE FSETXATTR FOP- {0:d}: gfid={1:s}; fd={2:p}; "+
-                      "flags={3:d}").format(unique, gfid, fd, flags)
-                self.gfids[key] = gfid
-                dl.wind_fsetxattr(frame, POINTER(xlator_t)(), fd, dictionary,
-                                  flags, xdata)
-                return 0
-
-        def fsetxattr_cbk(self, frame, cookie, this, op_ret, op_errno, xdata):
-                unique = dl.get_rootunique(frame)
-                key = dl.get_id(frame)
-                gfid = self.gfids[key]
-                print("GLUPY TRACE FSETXATTR CBK- {0:d}: gfid={1:s};  "+
-                      "op_ret={2:d}; op_errno={3:d}").format(unique, gfid,
-                                                             op_ret, op_errno)
-                del self.gfids[key]
-                dl.unwind_fsetxattr(frame, cookie, this, op_ret, op_errno,
-                                   xdata)
-                return 0
-
-        def removexattr_fop(self, frame, this, loc, name, xdata):
-                unique = dl.get_rootunique(frame)
-                key = dl.get_id(frame)
-                gfid = uuid2str(loc.contents.inode.contents.gfid)
-                print("GLUPY TRACE REMOVEXATTR FOP- {0:d}:  gfid={1:s}; "+
-                      "path={2:s}; name={3:s}").format(unique, gfid,
-                                                       loc.contents.path,
-                                                       name)
-                self.gfids[key] = gfid
-                dl.wind_removexattr(frame, POINTER(xlator_t)(), loc, name,
-                                    xdata)
-                return 0
-
-        def removexattr_cbk(self, frame, cookie, this, op_ret, op_errno,
-                            xdata):
-                unique = dl.get_rootunique(frame)
-                key = dl.get_id(frame)
-                gfid = self.gfids[key]
-                print("GLUPY TRACE REMOVEXATTR CBK- {0:d}: gfid={1:s} "+
-                      " op_ret={2:d}; op_errno={3:d}").format(unique, gfid,
-                                                              op_ret, op_errno)
-                del self.gfids[key]
-                dl.unwind_removexattr(frame, cookie, this, op_ret, op_errno,
-                                      xdata)
-                return 0
-
-        def link_fop(self, frame, this, oldloc, newloc, xdata):
-                unique = dl.get_rootunique(frame)
-                key = dl.get_id(frame)
-                if (newloc.contents.inode):
-                        newgfid = uuid2str(newloc.contents.inode.contents.gfid)
-                else:
-                        newgfid = "0"
-                oldgfid = uuid2str(oldloc.contents.inode.contents.gfid)
-                print("GLUPY TRACE LINK FOP-{0:d}: oldgfid={1:s}; oldpath={2:s};"+
-                      "newgfid={3:s};"+
-                      "newpath={4:s}").format(unique, oldgfid,
-                                              oldloc.contents.path,
-                                              newgfid,
-                                              newloc.contents.path)
-                self.gfids[key] =  oldgfid
-                dl.wind_link(frame, POINTER(xlator_t)(), oldloc, newloc,
-                             xdata)
-                return 0
-
-        def link_cbk(self, frame, cookie, this, op_ret, op_errno, inode, buf,
-                     preparent, postparent, xdata):
-                unique = dl.get_rootunique(frame)
-                key = dl.get_id(frame)
-                gfid = self.gfids[key]
-                if op_ret == 0:
-                        statstr = trace_stat2str(buf)
-                        preparentstr = trace_stat2str(preparent)
-                        postparentstr = trace_stat2str(postparent)
-                        print("GLUPY TRACE LINK CBK- {0:d}: op_ret={1:d} "+
-                              "*stbuf={2:s}; *prebuf={3:s}; "+
-                              "*postbuf={4:s} ").format(unique, op_ret, statstr,
-                                                        preparentstr,
-                                                        postparentstr)
-                else:
-                        print("GLUPY TRACE LINK CBK- {0:d}: gfid={1:s}; "+
-                              "op_ret={2:d}; "+
-                              "op_errno={3:d}").format(unique, gfid,
-                                                       op_ret, op_errno)
-                del self.gfids[key]
-                dl.unwind_link(frame, cookie, this, op_ret, op_errno, inode,
-                               buf, preparent, postparent, xdata)
-                return 0
-
-        def unlink_fop(self, frame, this, loc, xflag, xdata):
-                unique = dl.get_rootunique(frame)
-                key = dl.get_id(frame)
-                gfid = uuid2str(loc.contents.inode.contents.gfid)
-                print("GLUPY TRACE UNLINK FOP- {0:d}; gfid={1:s}; path={2:s}; "+
-                      "flag={3:d}").format(unique, gfid, loc.contents.path,
-                                           xflag)
-                self.gfids[key] = gfid
-                dl.wind_unlink(frame, POINTER(xlator_t)(), loc, xflag,
-                               xdata)
-                return 0
-
-        def unlink_cbk(self, frame, cookie, this, op_ret, op_errno,
-                       preparent, postparent, xdata):
-                unique = dl.get_rootunique(frame)
-                key = dl.get_id(frame)
-                gfid = self.gfids[key]
-                if op_ret == 0:
-                        preparentstr = trace_stat2str(preparent)
-                        postparentstr = trace_stat2str(postparent)
-                        print("GLUPY TRACE UNLINK CBK- {0:d}: gfid ={1:s}; "+
-                              "op_ret={2:d}; *prebuf={3:s}; "+
-                              "*postbuf={4:s} ").format(unique, gfid, op_ret,
-                                                        preparentstr,
-                                                        postparentstr)
-                else:
-                        print("GLUPY TRACE UNLINK CBK: {0:d}: gfid ={1:s}; "+
-                              "op_ret={2:d}; "+
-                              "op_errno={3:d}").format(unique, gfid, op_ret,
-                                                       op_errno)
-                del self.gfids[key]
-                dl.unwind_unlink(frame, cookie, this, op_ret, op_errno,
-                                 preparent, postparent, xdata)
-                return 0
-
-        def readlink_fop(self, frame, this, loc, size, xdata):
-                unique = dl.get_rootunique(frame)
-                key = dl.get_id(frame)
-                gfid = uuid2str(loc.contents.inode.contents.gfid)
-                print("GLUPY TRACE READLINK FOP- {0:d}:  gfid={1:s}; path={2:s};"+
-                      " size={3:d}").format(unique, gfid, loc.contents.path,
-                                            size)
-                self.gfids[key] = gfid
-                dl.wind_readlink(frame, POINTER(xlator_t)(), loc, size,
-                               xdata)
-                return 0
-
-        def readlink_cbk(self, frame, cookie, this, op_ret, op_errno,
-                         buf, stbuf, xdata):
-                unique = dl.get_rootunique(frame)
-                key = dl.get_id(frame)
-                gfid  = self.gfids[key]
-                if op_ret == 0:
-                        statstr = trace_stat2str(stbuf)
-                        print("GLUPY TRACE READLINK CBK- {0:d}: gfid={1:s} "+
-                              " op_ret={2:d}; op_errno={3:d}; *prebuf={4:s}; "+
-                              "*postbuf={5:s} ").format(unique, gfid,
-                                                        op_ret, op_errno,
-                                                        buf, statstr)
-                else:
-                        print("GLUPY TRACE READLINK CBK- {0:d}: gfid={1:s} "+
-                              " op_ret={2:d}; op_errno={3:d}").format(unique,
-                                                                      gfid,
-                                                                      op_ret,
-                                                                      op_errno)
-                del self.gfids[key]
-                dl.unwind_readlink(frame, cookie, this, op_ret, op_errno, buf,
-                                   stbuf, xdata)
-                return 0
-
-        def symlink_fop(self, frame, this, linkpath, loc, umask, xdata):
-                unique = dl.get_rootunique(frame)
-                key = dl.get_id(frame)
-                gfid = uuid2str(loc.contents.inode.contents.gfid)
-                print("GLUPY TRACE SYMLINK FOP- {0:d}: gfid={1:s}; "+
-                      "linkpath={2:s}; path={3:s};"+
-                      "umask=0{4:o}").format(unique, gfid, linkpath,
-                                             loc.contents.path, umask)
-                self.gfids[key] = gfid
-                dl.wind_symlink(frame, POINTER(xlator_t)(), linkpath, loc,
-                                umask, xdata)
-                return 0
-
-        def symlink_cbk(self, frame, cookie, this, op_ret, op_errno,
-                        inode, buf, preparent, postparent, xdata):
-                unique = dl.get_rootunique(frame)
-                key = dl.get_id(frame)
-                gfid = self.gfids[key]
-                if op_ret == 0:
-                        statstr = trace_stat2str(buf)
-                        preparentstr = trace_stat2str(preparent)
-                        postparentstr = trace_stat2str(postparent)
-                        print("GLUPY TRACE SYMLINK CBK- {0:d}: gfid={1:s}; "+
-                              "op_ret={2:d}; *stbuf={3:s}; *preparent={4:s}; "+
-                              "*postparent={5:s}").format(unique, gfid,
-                                                          op_ret, statstr,
-                                                          preparentstr,
-                                                          postparentstr)
-                else:
-                        print("GLUPY TRACE SYMLINK CBK- {0:d}: gfid={1:s}; "+
-                              "op_ret={2:d}; op_errno={3:d}").format(unique,
-                                                                     gfid,
-                                                                     op_ret,
-                                                                     op_errno)
-                del self.gfids[key]
-                dl.unwind_symlink(frame, cookie, this, op_ret, op_errno,
-                                  inode, buf, preparent, postparent, xdata)
-                return 0
diff --git a/xlators/features/glupy/examples/helloworld.py b/xlators/features/glupy/examples/helloworld.py
deleted file mode 100644
index b565a4e5bc3..00000000000
--- a/xlators/features/glupy/examples/helloworld.py
+++ /dev/null
@@ -1,19 +0,0 @@
-import sys
-from gluster.glupy import *
-
-class xlator (Translator):
-
-    def __init__(self, c_this):
-        Translator.__init__(self, c_this)
-
-    def lookup_fop(self, frame, this, loc, xdata):
-        print "Python xlator: Hello!"
-        dl.wind_lookup(frame, POINTER(xlator_t)(), loc, xdata)
-        return 0
-
-    def lookup_cbk(self, frame, cookie, this, op_ret, op_errno, inode, buf,
-                   xdata, postparent):
-        print "Python xlator: Hello again!"
-        dl.unwind_lookup(frame, cookie, this, op_ret, op_errno, inode, buf,
-                         xdata, postparent)
-        return 0
diff --git a/xlators/features/glupy/examples/negative.py b/xlators/features/glupy/examples/negative.py
deleted file mode 100644
index e7a4fc07ced..00000000000
--- a/xlators/features/glupy/examples/negative.py
+++ /dev/null
@@ -1,91 +0,0 @@
-import sys
-from uuid import UUID
-from gluster.glupy import *
-
-# Negative-lookup-caching example.  If a file wasn't there the last time we
-# looked, it's probably still not there.  This translator keeps track of
-# those failed lookups for us, and returns ENOENT without needing to pass the
-# call any further for repeated requests.
-
-# If we were doing this for real, we'd need separate caches for each xlator
-# instance.  The easiest way to do this would be to have xlator.__init__
-# "register" each instance in a module-global dict, with the key as the C
-# translator address and the value as the xlator object itself.  For testing
-# and teaching, it's sufficient just to have one cache.  The keys are parent
-# GFIDs, and the entries are lists of names within that parent that we know
-# don't exist.
-cache = {}
-
-# TBD: we need a better way of handling per-request data (frame->local in C).
-dl.get_id.restype = c_long
-dl.get_id.argtypes = [ POINTER(call_frame_t) ]
-
-def uuid2str (gfid):
-    return str(UUID(''.join(map("{0:02x}".format, gfid))))
-
-class xlator (Translator):
-
-    def __init__ (self, c_this):
-        self.requests = {}
-        Translator.__init__(self,c_this)
-
-    def lookup_fop (self, frame, this, loc, xdata):
-        pargfid = uuid2str(loc.contents.pargfid)
-        print "lookup FOP: %s:%s" % (pargfid, loc.contents.name)
-        # Check the cache.
-        if cache.has_key(pargfid):
-            if loc.contents.name in cache[pargfid]:
-                print "short-circuiting for %s:%s" % (pargfid,
-                    loc.contents.name)
-                dl.unwind_lookup(frame,0,this,-1,2,None,None,None,None)
-                return 0
-        key = dl.get_id(frame)
-        self.requests[key] = (pargfid, loc.contents.name[:])
-        # TBD: get real child xl from init, pass it here
-        dl.wind_lookup(frame,POINTER(xlator_t)(),loc,xdata)
-        return 0
-
-    def lookup_cbk (self, frame, cookie, this, op_ret, op_errno, inode, buf,
-                    xdata, postparent):
-        print "lookup CBK: %d (%d)" % (op_ret, op_errno)
-        key = dl.get_id(frame)
-        pargfid, name = self.requests[key]
-        # Update the cache.
-        if op_ret == 0:
-            print "found %s, removing from cache" % name
-            if cache.has_key(pargfid):
-                cache[pargfid].discard(name)
-        elif op_errno == 2:    # ENOENT
-            print "failed to find %s, adding to cache" % name
-            if cache.has_key(pargfid):
-                cache[pargfid].add(name)
-            else:
-                cache[pargfid] = set([name])
-        del self.requests[key]
-        dl.unwind_lookup(frame,cookie,this,op_ret,op_errno,
-                         inode,buf,xdata,postparent)
-        return 0
-
-    def create_fop (self, frame, this, loc, flags, mode, umask, fd, xdata):
-        pargfid = uuid2str(loc.contents.pargfid)
-        print "create FOP: %s:%s" % (pargfid, loc.contents.name)
-        key = dl.get_id(frame)
-        self.requests[key] = (pargfid, loc.contents.name[:])
-        # TBD: get real child xl from init, pass it here
-        dl.wind_create(frame,POINTER(xlator_t)(),loc,flags,mode,umask,fd,xdata)
-        return 0
-
-    def create_cbk (self, frame, cookie, this, op_ret, op_errno, fd, inode,
-                    buf, preparent, postparent, xdata):
-        print "create CBK: %d (%d)" % (op_ret, op_errno)
-        key = dl.get_id(frame)
-        pargfid, name = self.requests[key]
-        # Update the cache.
-        if op_ret == 0:
-            print "created %s, removing from cache" % name
-            if cache.has_key(pargfid):
-                cache[pargfid].discard(name)
-        del self.requests[key]
-        dl.unwind_create(frame,cookie,this,op_ret,op_errno,fd,inode,buf,
-                         preparent,postparent,xdata)
-        return 0
diff --git a/xlators/features/glupy/src/Makefile.am b/xlators/features/glupy/src/Makefile.am
deleted file mode 100644
index 697f0c7e443..00000000000
--- a/xlators/features/glupy/src/Makefile.am
+++ /dev/null
@@ -1,21 +0,0 @@
-xlator_LTLIBRARIES = glupy.la
-
-# Ensure GLUSTER_PYTHON_PATH is passed to glupy.so
-xlatordir = $(libdir)/glusterfs/$(PACKAGE_VERSION)/xlator/features
-glupydir = $(xlatordir)/glupy
-AM_CPPFLAGS = $(PYTHONDEV_CPPFLAGS) $(GF_CPPFLAGS) -I$(top_srcdir)/libglusterfs/src -isystem $(BUILD_PYTHON_INC)
-AM_CFLAGS = $(PYTHONDEV_CFLAGS) -Wall -fno-strict-aliasing -DGLUSTER_PYTHON_PATH=\"$(glupydir)\" -DPATH_GLUSTERFS_GLUPY_MODULE=\"${xlatordir}/glupy${shrext_cmds}\" $(GF_CFLAGS)
-
-# Flags to build glupy.so with
-glupy_la_LDFLAGS = $(PYTHONDEV_LDFLAGS) -module -avoid-version -nostartfiles
-glupy_la_SOURCES = glupy.c
-glupy_la_LIBADD = $(top_builddir)/libglusterfs/src/libglusterfs.la \
-	 -lpthread -l$(BUILD_PYTHON_LIB)
-
-noinst_HEADERS = glupy.h
-
-# Install glupy.py into the Python site-packages area
-pyglupydir = @BUILD_PYTHON_SITE_PACKAGES@/gluster
-pyglupy_PYTHON = glupy.py __init__.py
-
-CLEANFILES =
diff --git a/xlators/features/glupy/src/__init__.py.in b/xlators/features/glupy/src/__init__.py.in
deleted file mode 100644
index e69de29bb2d..00000000000
--- a/xlators/features/glupy/src/__init__.py.in
+++ /dev/null
diff --git a/xlators/features/glupy/src/glupy.c b/xlators/features/glupy/src/glupy.c
deleted file mode 100644
index 7bb88c0a0ae..00000000000
--- a/xlators/features/glupy/src/glupy.c
+++ /dev/null
@@ -1,2501 +0,0 @@
-/*
-   Copyright (c) 2006-2014 Red Hat, Inc. <http://www.redhat.com>
-   This file is part of GlusterFS.
-
-   This file is licensed to you under your choice of the GNU Lesser
-   General Public License, version 3 or any later version (LGPLv3 or
-   later), or the GNU General Public License, version 2 (GPLv2), in all
-   cases as published by the Free Software Foundation.
-*/
-
-#include <ctype.h>
-#include <sys/uio.h>
-#include <Python.h>
-
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
-
-#include "glusterfs.h"
-#include "xlator.h"
-#include "logging.h"
-#include "defaults.h"
-
-#include "glupy.h"
-
-/* UTILITY FUNCTIONS FOR FOP-SPECIFIC CODE */
-
-pthread_key_t gil_init_key;
-
-PyGILState_STATE
-glupy_enter (void)
-{
-        if (!pthread_getspecific(gil_init_key)) {
-                PyEval_ReleaseLock();
-                (void)pthread_setspecific(gil_init_key,(void *)1);
-        }
-
-        return PyGILState_Ensure();
-}
-
-void
-glupy_leave (PyGILState_STATE gstate)
-{
-        PyGILState_Release(gstate);
-}
-
-/* FOP: LOOKUP */
-
-int32_t
-glupy_lookup_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                  int32_t op_ret, int32_t op_errno, inode_t *inode,
-                  struct iatt *buf, dict_t *xdata, struct iatt *postparent)
-{
-        glupy_private_t *priv = this->private;
-        PyGILState_STATE gstate;
-        int32_t ret;
-
-        if (!priv->cbks[GLUPY_LOOKUP]) {
-                goto unwind;
-        }
-
-        gstate = glupy_enter();
-        ret = ((fop_lookup_cbk_t)(priv->cbks[GLUPY_LOOKUP]))(
-                frame, cookie, this, op_ret, op_errno,
-                inode, buf, xdata, postparent);
-        glupy_leave(gstate);
-
-        return ret;
-
-unwind:
-        frame->local = NULL;
-        STACK_UNWIND_STRICT (lookup, frame, op_ret, op_errno, inode, buf,
-                             xdata, postparent);
-        return 0;
-}
-
-int32_t
-glupy_lookup (call_frame_t *frame, xlator_t *this, loc_t *loc,
-              dict_t *xdata)
-{
-        glupy_private_t *priv = this->private;
-        PyGILState_STATE gstate;
-        int32_t ret;
-        static long next_id = 0;
-
-        if (!priv->fops[GLUPY_LOOKUP]) {
-                goto wind;
-        }
-
-        gstate = glupy_enter();
-        frame->local = (void *)++next_id;
-        ret = ((fop_lookup_t)(priv->fops[GLUPY_LOOKUP]))(
-                frame, this, loc, xdata);
-        glupy_leave(gstate);
-
-        return ret;
-
-wind:
-        STACK_WIND (frame, glupy_lookup_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->lookup, loc, xdata);
-        return 0;
-}
-
-void
-wind_lookup (call_frame_t *frame, xlator_t *xl, loc_t *loc, dict_t *xdata)
-{
-        xlator_t        *this = THIS;
-
-        if (!xl || (xl == this)) {
-                xl = FIRST_CHILD(this);
-        }
-
-        STACK_WIND(frame,glupy_lookup_cbk,xl,xl->fops->lookup,loc,xdata);
-}
-
-void
-unwind_lookup (call_frame_t *frame, long cookie, xlator_t *this,
-               int32_t op_ret, int32_t op_errno, inode_t *inode,
-               struct iatt *buf, dict_t *xdata, struct iatt *postparent)
-{
-        frame->local = NULL;
-        STACK_UNWIND_STRICT(lookup,frame,op_ret,op_errno,
-                            inode,buf,xdata,postparent);
-}
-
-void
-set_lookup_fop (long py_this, fop_lookup_t fop)
-{
-        glupy_private_t *priv   = ((xlator_t *)py_this)->private;
-
-        priv->fops[GLUPY_LOOKUP] = (long)fop;
-}
-
-void
-set_lookup_cbk (long py_this, fop_lookup_cbk_t cbk)
-{
-        glupy_private_t *priv   = ((xlator_t *)py_this)->private;
-
-        priv->cbks[GLUPY_LOOKUP] = (long)cbk;
-}
-
-/* FOP: CREATE */
-
-int32_t
-glupy_create_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                  int32_t op_ret, int32_t op_errno, fd_t *fd, inode_t *inode,
-                  struct iatt *buf, struct iatt *preparent,
-                  struct iatt *postparent, dict_t *xdata)
-{
-        glupy_private_t *priv = this->private;
-        PyGILState_STATE gstate;
-        int32_t ret;
-
-        if (!priv->cbks[GLUPY_CREATE]) {
-                goto unwind;
-        }
-
-        gstate = glupy_enter();
-        ret = ((fop_create_cbk_t)(priv->cbks[GLUPY_CREATE]))(
-                frame, cookie, this, op_ret, op_errno,
-                fd, inode, buf, preparent, postparent, xdata);
-        glupy_leave(gstate);
-
-        return ret;
-
-unwind:
-        frame->local = NULL;
-        STACK_UNWIND_STRICT (create, frame, op_ret, op_errno, fd, inode, buf,
-                             preparent, postparent, xdata);
-        return 0;
-}
-
-int32_t
-glupy_create (call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t flags,
-              mode_t mode, mode_t umask, fd_t *fd, dict_t *xdata)
-{
-        glupy_private_t *priv = this->private;
-        PyGILState_STATE gstate;
-        int32_t ret;
-        static long next_id = 0;
-
-        if (!priv->fops[GLUPY_CREATE]) {
-                goto wind;
-        }
-
-        gstate = glupy_enter();
-        frame->local = (void *)++next_id;
-        ret = ((fop_create_t)(priv->fops[GLUPY_CREATE]))(
-                frame, this, loc, flags, mode, umask, fd, xdata);
-        glupy_leave(gstate);
-
-        return ret;
-
-wind:
-        STACK_WIND (frame, glupy_create_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->create, loc, flags, mode, umask,
-                    fd, xdata);
-        return 0;
-}
-
-void
-wind_create (call_frame_t *frame, xlator_t *xl, loc_t *loc, int32_t flags,
-             mode_t mode, mode_t umask, fd_t *fd, dict_t *xdata)
-{
-        xlator_t        *this = THIS;
-
-        if (!xl || (xl == this)) {
-                xl = FIRST_CHILD(this);
-        }
-
-        STACK_WIND (frame, glupy_create_cbk,xl, xl->fops->create,
-                    loc, flags, mode, umask, fd, xdata);
-}
-
-void
-unwind_create (call_frame_t *frame, long cookie, xlator_t *this,
-               int32_t op_ret, int32_t op_errno, fd_t *fd, inode_t *inode,
-               struct iatt *buf, struct iatt *preparent,
-               struct iatt *postparent, dict_t *xdata)
-{
-        frame->local = NULL;
-        STACK_UNWIND_STRICT (create, frame, op_ret, op_errno, fd, inode, buf,
-                             preparent, postparent, xdata);
-}
-
-void
-set_create_fop (long py_this, fop_create_t fop)
-{
-        glupy_private_t *priv   = ((xlator_t *)py_this)->private;
-
-        priv->fops[GLUPY_CREATE] = (long)fop;
-}
-
-void
-set_create_cbk (long py_this, fop_create_cbk_t cbk)
-{
-        glupy_private_t *priv   = ((xlator_t *)py_this)->private;
-
-        priv->cbks[GLUPY_CREATE] = (long)cbk;
-}
-
-/* FOP: OPEN */
-
-int32_t
-glupy_open_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                int32_t op_ret, int32_t op_errno, fd_t *fd, dict_t *xdata)
-{
-        glupy_private_t *priv = this->private;
-        PyGILState_STATE gstate;
-        int32_t ret;
-
-        if (!priv->cbks[GLUPY_OPEN]) {
-                goto unwind;
-        }
-
-        gstate = glupy_enter();
-        ret = ((fop_open_cbk_t)(priv->cbks[GLUPY_OPEN]))(
-                frame, cookie, this, op_ret, op_errno,
-                fd, xdata);
-        glupy_leave(gstate);
-
-        return ret;
-
-unwind:
-        frame->local = NULL;
-        STACK_UNWIND_STRICT (open, frame, op_ret, op_errno, fd, xdata);
-        return 0;
-}
-
-int32_t
-glupy_open (call_frame_t *frame, xlator_t *this, loc_t *loc,
-            int32_t flags, fd_t *fd, dict_t *xdata)
-{
-        glupy_private_t *priv = this->private;
-        PyGILState_STATE gstate;
-        int32_t ret;
-        static long next_id = 0;
-
-        if (!priv->fops[GLUPY_OPEN]) {
-                goto wind;
-        }
-
-        gstate = glupy_enter();
-        frame->local = (void *)++next_id;
-        ret = ((fop_open_t)(priv->fops[GLUPY_OPEN]))(
-                frame, this, loc, flags, fd, xdata);
-        glupy_leave(gstate);
-
-        return ret;
-
-wind:
-        STACK_WIND (frame, glupy_open_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->open, loc, flags, fd, xdata);
-        return 0;
-}
-
-void
-wind_open (call_frame_t *frame, xlator_t *xl, loc_t *loc, int32_t flags,
-           fd_t *fd, dict_t *xdata)
-{
-        xlator_t        *this = THIS;
-
-        if (!xl || (xl == this)) {
-                xl = FIRST_CHILD(this);
-        }
-
-        STACK_WIND (frame, glupy_open_cbk, xl, xl->fops->open, loc, flags,
-                    fd, xdata);
-}
-
-void
-unwind_open (call_frame_t *frame, long cookie, xlator_t *this,
-             int32_t op_ret, int32_t op_errno, fd_t *fd, dict_t *xdata)
-{
-        frame->local = NULL;
-        STACK_UNWIND_STRICT (open, frame, op_ret, op_errno, fd, xdata);
-}
-
-void
-set_open_fop (long py_this, fop_open_t fop)
-{
-        glupy_private_t *priv   = ((xlator_t *)py_this)->private;
-        priv->fops[GLUPY_OPEN] = (long)fop;
-}
-
-void
-set_open_cbk (long py_this, fop_open_cbk_t cbk)
-{
-        glupy_private_t *priv   = ((xlator_t *)py_this)->private;
-        priv->cbks[GLUPY_OPEN] = (long)cbk;
-}
-
-/* FOP: READV */
-
-int32_t
-glupy_readv_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                 int32_t op_ret, int32_t op_errno, struct iovec *vector,
-                 int32_t count, struct iatt *stbuf, struct iobref *iobref,
-                 dict_t *xdata)
-{
-        glupy_private_t *priv = this->private;
-        PyGILState_STATE gstate;
-        int32_t ret;
-
-        if (!priv->cbks[GLUPY_READV]) {
-                goto unwind;
-        }
-
-        gstate = glupy_enter();
-        ret = ((fop_readv_cbk_t)(priv->cbks[GLUPY_READV]))(
-                frame, cookie, this, op_ret, op_errno,
-                vector, count, stbuf, iobref, xdata);
-        glupy_leave(gstate);
-
-        return ret;
-
-unwind:
-        frame->local = NULL;
-        STACK_UNWIND_STRICT (readv, frame, op_ret, op_errno, vector,
-                             count, stbuf, iobref, xdata);
-        return 0;
-}
-
-int32_t
-glupy_readv (call_frame_t *frame, xlator_t *this, fd_t *fd,
-             size_t size, off_t offset, uint32_t flags, dict_t *xdata)
-{
-        glupy_private_t *priv = this->private;
-        PyGILState_STATE gstate;
-        int32_t ret;
-        static long next_id = 0;
-
-        if (!priv->fops[GLUPY_READV]) {
-                goto wind;
-        }
-
-        gstate = glupy_enter();
-        frame->local = (void *)++next_id;
-        ret = ((fop_readv_t)(priv->fops[GLUPY_READV]))(
-                frame, this, fd, size, offset, flags, xdata);
-        glupy_leave(gstate);
-
-        return ret;
-
-wind:
-        STACK_WIND (frame, glupy_readv_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->readv, fd, size, offset,
-                    flags, xdata);
-        return 0;
-}
-
-void
-wind_readv (call_frame_t *frame, xlator_t *xl, fd_t *fd, size_t size,
-            off_t offset, uint32_t flags, dict_t *xdata)
-{
-        xlator_t        *this = THIS;
-
-        if (!xl || (xl == this)) {
-                xl = FIRST_CHILD(this);
-        }
-
-        STACK_WIND (frame, glupy_readv_cbk, xl, xl->fops->readv, fd, size,
-                    offset, flags, xdata);
-}
-
-void
-unwind_readv (call_frame_t *frame, long cookie, xlator_t *this,
-              int32_t op_ret, int32_t op_errno, struct iovec *vector,
-              int32_t count, struct iatt *stbuf, struct iobref *iobref,
-              dict_t *xdata)
-{
-        frame->local = NULL;
-        STACK_UNWIND_STRICT (readv, frame, op_ret, op_errno, vector,
-                             count, stbuf, iobref, xdata);
-}
-
-void
-set_readv_fop (long py_this, fop_readv_t fop)
-{
-        glupy_private_t *priv   = ((xlator_t *)py_this)->private;
-        priv->fops[GLUPY_READV] = (long)fop;
-}
-
-void
-set_readv_cbk (long py_this, fop_readv_cbk_t cbk)
-{
-        glupy_private_t *priv   = ((xlator_t *)py_this)->private;
-        priv->cbks[GLUPY_READV] = (long)cbk;
-}
-
-/* FOP: WRITEV */
-
-int32_t
-glupy_writev_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                  int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
-                  struct iatt *postbuf, dict_t *xdata)
-{
-        glupy_private_t *priv = this->private;
-        PyGILState_STATE gstate;
-        int32_t ret;
-
-        if (!priv->cbks[GLUPY_WRITEV]) {
-                goto unwind;
-        }
-
-        gstate = glupy_enter();
-        ret = ((fop_writev_cbk_t)(priv->cbks[GLUPY_WRITEV]))(
-                frame, cookie, this, op_ret, op_errno,
-                prebuf, postbuf, xdata);
-        glupy_leave(gstate);
-
-        return ret;
-
-unwind:
-        frame->local = NULL;
-        STACK_UNWIND_STRICT (writev, frame, op_ret, op_errno, prebuf,
-                             postbuf, xdata);
-        return 0;
-}
-
-int32_t
-glupy_writev (call_frame_t *frame, xlator_t *this, fd_t *fd,
-              struct iovec *vector, int32_t count, off_t offset,
-              uint32_t flags, struct iobref *iobref, dict_t *xdata)
-{
-        glupy_private_t *priv = this->private;
-        PyGILState_STATE gstate;
-        int32_t ret;
-        static long next_id = 0;
-
-        if (!priv->fops[GLUPY_WRITEV]) {
-                goto wind;
-        }
-
-        gstate = glupy_enter();
-        frame->local = (void *)++next_id;
-        ret = ((fop_writev_t)(priv->fops[GLUPY_WRITEV]))(
-                frame, this, fd, vector, count, offset, flags,
-                iobref, xdata);
-        glupy_leave(gstate);
-
-        return ret;
-
-wind:
-        STACK_WIND (frame, glupy_writev_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->writev, fd, vector, count,
-                    offset, flags, iobref, xdata);
-        return 0;
-}
-
-void
-wind_writev (call_frame_t *frame, xlator_t *xl, fd_t *fd, struct iovec *vector,
-             int32_t count, off_t offset, uint32_t flags, struct iobref *iobref,
-             dict_t *xdata)
-{
-        xlator_t        *this = THIS;
-
-        if (!xl || (xl == this)) {
-                xl = FIRST_CHILD(this);
-        }
-
-        STACK_WIND (frame, glupy_writev_cbk, xl, xl->fops->writev, fd, vector,
-                    count, offset, flags, iobref, xdata);
-}
-
-void
-unwind_writev (call_frame_t *frame, long cookie, xlator_t *this,
-               int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
-               struct iatt *postbuf, dict_t *xdata)
-{
-        frame->local = NULL;
-        STACK_UNWIND_STRICT (writev, frame, op_ret, op_errno, prebuf,
-                             postbuf, xdata);
-}
-
-void
-set_writev_fop (long py_this, fop_writev_t fop)
-{
-        glupy_private_t *priv   = ((xlator_t *)py_this)->private;
-        priv->fops[GLUPY_WRITEV] = (long)fop;
-}
-
-void
-set_writev_cbk (long py_this, fop_writev_cbk_t cbk)
-{
-        glupy_private_t *priv   = ((xlator_t *)py_this)->private;
-        priv->cbks[GLUPY_WRITEV] = (long)cbk;
-}
-
-
-/* FOP: OPENDIR */
-
-int32_t
-glupy_opendir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                   int32_t op_ret, int32_t op_errno, fd_t *fd,
-                   dict_t *xdata)
-{
-        glupy_private_t *priv = this->private;
-        PyGILState_STATE gstate;
-        int32_t ret;
-
-        if (!priv->cbks[GLUPY_OPENDIR]) {
-                goto unwind;
-        }
-
-        gstate = glupy_enter();
-        ret = ((fop_opendir_cbk_t)(priv->cbks[GLUPY_OPENDIR]))(
-                frame, cookie, this, op_ret, op_errno,
-                fd, xdata);
-        glupy_leave(gstate);
-
-        return ret;
-
-unwind:
-        frame->local = NULL;
-        STACK_UNWIND_STRICT (opendir, frame, op_ret, op_errno, fd, xdata);
-        return 0;
-}
-
-int32_t
-glupy_opendir (call_frame_t *frame, xlator_t *this, loc_t *loc,
-               fd_t *fd, dict_t *xdata)
-{
-        glupy_private_t *priv = this->private;
-        PyGILState_STATE gstate;
-        int32_t ret;
-        static long next_id = 0;
-
-        if (!priv->fops[GLUPY_OPENDIR]) {
-                goto wind;
-        }
-
-        gstate = glupy_enter();
-        frame->local = (void *)++next_id;
-        ret = ((fop_opendir_t)(priv->fops[GLUPY_OPENDIR]))(
-                frame, this, loc, fd, xdata);
-        glupy_leave(gstate);
-
-        return ret;
-
-wind:
-        STACK_WIND (frame, glupy_opendir_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->opendir, loc, fd, xdata);
-        return 0;
-}
-
-void
-wind_opendir (call_frame_t *frame, xlator_t *xl, loc_t *loc, fd_t *fd, dict_t *xdata)
-{
-        xlator_t        *this = THIS;
-
-        if (!xl || (xl == this)) {
-                xl = FIRST_CHILD(this);
-        }
-
-        STACK_WIND(frame,glupy_opendir_cbk,xl,xl->fops->opendir,loc,fd,xdata);
-}
-
-void
-unwind_opendir (call_frame_t *frame, long cookie, xlator_t *this,
-                int32_t op_ret, int32_t op_errno, fd_t *fd, dict_t *xdata)
-{
-        frame->local = NULL;
-        STACK_UNWIND_STRICT(opendir,frame,op_ret,op_errno,
-                            fd,xdata);
-}
-
-void
-set_opendir_fop (long py_this, fop_opendir_t fop)
-{
-        glupy_private_t *priv   = ((xlator_t *)py_this)->private;
-
-        priv->fops[GLUPY_OPENDIR] = (long)fop;
-}
-
-void
-set_opendir_cbk (long py_this, fop_opendir_cbk_t cbk)
-{
-        glupy_private_t *priv   = ((xlator_t *)py_this)->private;
-
-        priv->cbks[GLUPY_OPENDIR] = (long)cbk;
-}
-
-/* FOP: READDIR */
-
-int32_t
-glupy_readdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                   int32_t op_ret, int32_t op_errno, gf_dirent_t *entries,
-                   dict_t *xdata)
-{
-        glupy_private_t *priv = this->private;
-        PyGILState_STATE gstate;
-        int32_t ret;
-
-        if (!priv->cbks[GLUPY_READDIR]) {
-                goto unwind;
-        }
-
-        gstate = glupy_enter();
-        ret = ((fop_readdir_cbk_t)(priv->cbks[GLUPY_READDIR]))(
-                frame, cookie, this, op_ret, op_errno,
-                entries,  xdata);
-        glupy_leave(gstate);
-
-        return ret;
-
-unwind:
-        frame->local = NULL;
-        STACK_UNWIND_STRICT (readdir, frame, op_ret, op_errno, entries,
-                             xdata);
-        return 0;
-}
-
-int32_t
-glupy_readdir (call_frame_t *frame, xlator_t *this, fd_t *fd,
-               size_t size, off_t offset, dict_t *xdata)
-{
-        glupy_private_t *priv = this->private;
-        PyGILState_STATE gstate;
-        int32_t ret;
-        static long next_id = 0;
-
-        if (!priv->fops[GLUPY_READDIR]) {
-                goto wind;
-        }
-
-        gstate = glupy_enter();
-        frame->local = (void *)++next_id;
-        ret = ((fop_readdir_t)(priv->fops[GLUPY_READDIR]))(
-                frame, this, fd, size, offset, xdata);
-        glupy_leave(gstate);
-
-        return ret;
-
-wind:
-        STACK_WIND (frame, glupy_readdir_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->readdir,fd, size, offset, xdata);
-        return 0;
-}
-
-void
-wind_readdir(call_frame_t *frame, xlator_t *xl, fd_t *fd, size_t size,
-             off_t offset, dict_t *xdata)
-{
-        xlator_t        *this = THIS;
-
-        if (!xl || (xl == this)) {
-                xl = FIRST_CHILD(this);
-        }
-
-        STACK_WIND(frame,glupy_readdir_cbk,xl,xl->fops->readdir,fd,size,offset,xdata);
-}
-
-void
-unwind_readdir (call_frame_t *frame, long cookie, xlator_t *this,
-                int32_t op_ret, int32_t op_errno, gf_dirent_t *entries,
-                dict_t *xdata)
-{
-        frame->local = NULL;
-        STACK_UNWIND_STRICT(readdir,frame,op_ret,op_errno,
-                            entries, xdata);
-}
-
-void
-set_readdir_fop (long py_this, fop_readdir_t fop)
-{
-        glupy_private_t *priv   = ((xlator_t *)py_this)->private;
-
-        priv->fops[GLUPY_READDIR] = (long)fop;
-}
-
-void
-set_readdir_cbk (long py_this, fop_readdir_cbk_t cbk)
-{
-        glupy_private_t *priv   = ((xlator_t *)py_this)->private;
-
-        priv->cbks[GLUPY_READDIR] = (long)cbk;
-}
-
-
-/* FOP: READDIRP */
-
-int32_t
-glupy_readdirp_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                    int32_t op_ret, int32_t op_errno, gf_dirent_t *entries,
-                    dict_t *xdata)
-{
-        glupy_private_t *priv = this->private;
-        PyGILState_STATE gstate;
-        int32_t ret;
-
-        if (!priv->cbks[GLUPY_READDIRP]) {
-                goto unwind;
-        }
-
-        gstate = glupy_enter();
-        ret = ((fop_readdirp_cbk_t)(priv->cbks[GLUPY_READDIRP]))(
-                frame, cookie, this, op_ret, op_errno,
-                entries,  xdata);
-        glupy_leave(gstate);
-
-        return ret;
-
-unwind:
-        frame->local = NULL;
-        STACK_UNWIND_STRICT (readdirp, frame, op_ret, op_errno, entries,
-                             xdata);
-        return 0;
-}
-
-int32_t
-glupy_readdirp (call_frame_t *frame, xlator_t *this, fd_t *fd,
-                size_t size, off_t offset, dict_t *xdata)
-{
-        glupy_private_t *priv = this->private;
-        PyGILState_STATE gstate;
-        int32_t ret;
-        static long next_id = 0;
-
-        if (!priv->fops[GLUPY_READDIRP]) {
-                goto wind;
-        }
-
-        gstate = glupy_enter();
-        frame->local = (void *)++next_id;
-        ret = ((fop_readdirp_t)(priv->fops[GLUPY_READDIRP]))(
-                frame, this, fd, size, offset, xdata);
-        glupy_leave(gstate);
-
-        return ret;
-
-wind:
-        STACK_WIND (frame, glupy_readdirp_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->readdirp,fd, size, offset, xdata);
-        return 0;
-}
-
-void
-wind_readdirp (call_frame_t *frame, xlator_t *xl, fd_t *fd, size_t size,
-               off_t offset, dict_t *xdata)
-{
-        xlator_t        *this = THIS;
-
-        if (!xl || (xl == this)) {
-                xl = FIRST_CHILD(this);
-        }
-
-        STACK_WIND(frame,glupy_readdirp_cbk,xl,xl->fops->readdirp,fd,size,offset,xdata);
-}
-
-void
-unwind_readdirp (call_frame_t *frame, long cookie, xlator_t *this,
-                 int32_t op_ret, int32_t op_errno, gf_dirent_t *entries,
-                 dict_t *xdata)
-{
-        frame->local = NULL;
-        STACK_UNWIND_STRICT(readdirp,frame,op_ret,op_errno,
-                            entries, xdata);
-}
-
-void
-set_readdirp_fop (long py_this, fop_readdirp_t fop)
-{
-        glupy_private_t *priv   = ((xlator_t *)py_this)->private;
-
-        priv->fops[GLUPY_READDIRP] = (long)fop;
-}
-
-void
-set_readdirp_cbk (long py_this, fop_readdirp_cbk_t cbk)
-{
-        glupy_private_t *priv   = ((xlator_t *)py_this)->private;
-
-        priv->cbks[GLUPY_READDIRP] = (long)cbk;
-}
-
-
-/* FOP:STAT */
-
-int32_t
-glupy_stat_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                int32_t op_ret, int32_t op_errno, struct iatt *buf, dict_t *xdata)
-{
-        glupy_private_t *priv = this->private;
-        PyGILState_STATE gstate;
-        int32_t ret;
-
-        if (!priv->cbks[GLUPY_STAT]) {
-                goto unwind;
-        }
-
-        gstate = glupy_enter();
-        ret = ((fop_stat_cbk_t)(priv->cbks[GLUPY_STAT]))(
-                frame, cookie, this, op_ret, op_errno,
-                buf, xdata);
-        glupy_leave(gstate);
-
-        return ret;
-
-unwind:
-        frame->local = NULL;
-        STACK_UNWIND_STRICT (stat, frame, op_ret, op_errno, buf, xdata);
-        return 0;
-}
-
-int32_t
-glupy_stat (call_frame_t *frame, xlator_t *this, loc_t *loc,
-            dict_t *xdata)
-{
-        glupy_private_t *priv = this->private;
-        PyGILState_STATE gstate;
-        int32_t ret;
-        static long next_id = 0;
-
-        if (!priv->fops[GLUPY_STAT]) {
-                goto wind;
-        }
-
-        gstate = glupy_enter();
-        frame->local = (void *)++next_id;
-        ret = ((fop_stat_t)(priv->fops[GLUPY_STAT]))(
-                frame, this, loc, xdata);
-        glupy_leave(gstate);
-
-        return ret;
-
-wind:
-        STACK_WIND (frame, glupy_stat_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->stat, loc, xdata);
-        return 0;
-}
-
-void
-wind_stat (call_frame_t *frame, xlator_t *xl, loc_t *loc, dict_t *xdata)
-{
-        xlator_t        *this = THIS;
-
-        if (!xl || (xl == this)) {
-                xl = FIRST_CHILD(this);
-        }
-
-        STACK_WIND(frame,glupy_stat_cbk,xl,xl->fops->stat,loc,xdata);
-}
-
-void
-unwind_stat (call_frame_t *frame, long cookie, xlator_t *this,
-             int32_t op_ret, int32_t op_errno, struct iatt *buf,
-             dict_t *xdata)
-{
-        frame->local = NULL;
-        STACK_UNWIND_STRICT(stat,frame,op_ret,op_errno,
-                            buf,xdata);
-}
-
-void
-set_stat_fop (long py_this, fop_stat_t fop)
-{
-        glupy_private_t *priv   = ((xlator_t *)py_this)->private;
-
-        priv->fops[GLUPY_STAT] = (long)fop;
-}
-
-void
-set_stat_cbk (long py_this, fop_stat_cbk_t cbk)
-{
-        glupy_private_t *priv   = ((xlator_t *)py_this)->private;
-
-        priv->cbks[GLUPY_STAT] = (long)cbk;
-}
-
-
-/* FOP: FSTAT */
-
-int32_t
-glupy_fstat_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                 int32_t op_ret, int32_t op_errno, struct iatt *buf, dict_t *xdata)
-{
-        glupy_private_t *priv = this->private;
-        PyGILState_STATE gstate;
-        int32_t ret;
-
-        if (!priv->cbks[GLUPY_FSTAT]) {
-                goto unwind;
-        }
-
-        gstate = glupy_enter();
-        ret = ((fop_fstat_cbk_t)(priv->cbks[GLUPY_FSTAT]))(
-                frame, cookie, this, op_ret, op_errno,
-                buf, xdata);
-        glupy_leave(gstate);
-
-        return ret;
-
-unwind:
-        frame->local = NULL;
-        STACK_UNWIND_STRICT (fstat, frame, op_ret, op_errno, buf, xdata);
-        return 0;
-}
-
-int32_t
-glupy_fstat (call_frame_t *frame, xlator_t *this, fd_t *fd,
-             dict_t *xdata)
-{
-        glupy_private_t *priv = this->private;
-        PyGILState_STATE gstate;
-        int32_t ret;
-        static long next_id = 0;
-
-        if (!priv->fops[GLUPY_FSTAT]) {
-                goto wind;
-        }
-
-        gstate = glupy_enter();
-        frame->local = (void *)++next_id;
-        ret = ((fop_fstat_t)(priv->fops[GLUPY_FSTAT]))(
-                frame, this, fd, xdata);
-        glupy_leave(gstate);
-
-        return ret;
-
-wind:
-        STACK_WIND (frame, glupy_fstat_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->fstat, fd, xdata);
-        return 0;
-}
-
-void
-wind_fstat (call_frame_t *frame, xlator_t *xl, fd_t *fd, dict_t *xdata)
-{
-        xlator_t        *this = THIS;
-
-        if (!xl || (xl == this)) {
-                xl = FIRST_CHILD(this);
-        }
-
-        STACK_WIND(frame,glupy_fstat_cbk,xl,xl->fops->fstat,fd,xdata);
-}
-
-void
-unwind_fstat (call_frame_t *frame, long cookie, xlator_t *this,
-              int32_t op_ret, int32_t op_errno, struct iatt *buf,
-              dict_t *xdata)
-{
-        frame->local = NULL;
-        STACK_UNWIND_STRICT(fstat,frame,op_ret,op_errno,
-                            buf,xdata);
-}
-
-void
-set_fstat_fop (long py_this, fop_fstat_t fop)
-{
-        glupy_private_t *priv   = ((xlator_t *)py_this)->private;
-
-        priv->fops[GLUPY_FSTAT] = (long)fop;
-}
-
-void
-set_fstat_cbk (long py_this, fop_fstat_cbk_t cbk)
-{
-        glupy_private_t *priv   = ((xlator_t *)py_this)->private;
-
-        priv->cbks[GLUPY_FSTAT] = (long)cbk;
-}
-
-/* FOP:STATFS */
-
-int32_t
-glupy_statfs_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                  int32_t op_ret, int32_t op_errno, struct statvfs *buf, dict_t *xdata)
-{
-        glupy_private_t *priv = this->private;
-        PyGILState_STATE gstate;
-        int32_t ret;
-
-        if (!priv->cbks[GLUPY_STATFS]) {
-                goto unwind;
-        }
-
-        gstate = glupy_enter();
-        ret = ((fop_statfs_cbk_t)(priv->cbks[GLUPY_STATFS]))(
-                frame, cookie, this, op_ret, op_errno,
-                buf, xdata);
-        glupy_leave(gstate);
-
-        return ret;
-
-unwind:
-        frame->local = NULL;
-        STACK_UNWIND_STRICT (statfs, frame, op_ret, op_errno, buf, xdata);
-        return 0;
-}
-
-int32_t
-glupy_statfs (call_frame_t *frame, xlator_t *this, loc_t *loc,
-              dict_t *xdata)
-{
-        glupy_private_t *priv = this->private;
-        PyGILState_STATE gstate;
-        int32_t ret;
-        static long next_id = 0;
-
-        if (!priv->fops[GLUPY_STATFS]) {
-                goto wind;
-        }
-
-        gstate = glupy_enter();
-        frame->local = (void *)++next_id;
-        ret = ((fop_statfs_t)(priv->fops[GLUPY_STATFS]))(
-                frame, this, loc, xdata);
-        glupy_leave(gstate);
-
-        return ret;
-
-wind:
-        STACK_WIND (frame, glupy_statfs_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->statfs, loc, xdata);
-        return 0;
-}
-
-void
-wind_statfs (call_frame_t *frame, xlator_t *xl, loc_t *loc, dict_t *xdata)
-{
-        xlator_t        *this = THIS;
-
-        if (!xl || (xl == this)) {
-                xl = FIRST_CHILD(this);
-        }
-
-        STACK_WIND(frame,glupy_statfs_cbk,xl,xl->fops->statfs,loc,xdata);
-}
-
-void
-unwind_statfs (call_frame_t *frame, long cookie, xlator_t *this,
-               int32_t op_ret, int32_t op_errno, struct statvfs *buf,
-               dict_t *xdata)
-{
-        frame->local = NULL;
-        STACK_UNWIND_STRICT(statfs,frame,op_ret,op_errno,
-                            buf,xdata);
-}
-
-void
-set_statfs_fop (long py_this, fop_statfs_t fop)
-{
-        glupy_private_t *priv   = ((xlator_t *)py_this)->private;
-
-        priv->fops[GLUPY_STATFS] = (long)fop;
-}
-
-void
-set_statfs_cbk (long py_this, fop_statfs_cbk_t cbk)
-{
-        glupy_private_t *priv   = ((xlator_t *)py_this)->private;
-
-        priv->cbks[GLUPY_STATFS] = (long)cbk;
-}
-
-
-/* FOP: SETXATTR */
-
-int32_t
-glupy_setxattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                    int32_t op_ret, int32_t op_errno, dict_t *xdata)
-{
-        glupy_private_t *priv = this->private;
-        PyGILState_STATE gstate;
-        int32_t ret;
-
-        if (!priv->cbks[GLUPY_SETXATTR]) {
-                goto unwind;
-        }
-
-        gstate = glupy_enter();
-        ret = ((fop_setxattr_cbk_t)(priv->cbks[GLUPY_SETXATTR]))(
-                frame, cookie, this, op_ret, op_errno,
-                xdata);
-        glupy_leave(gstate);
-
-        return ret;
-
-unwind:
-        frame->local = NULL;
-        STACK_UNWIND_STRICT (setxattr, frame, op_ret, op_errno, xdata);
-        return 0;
-}
-
-int32_t
-glupy_setxattr (call_frame_t *frame, xlator_t *this, loc_t *loc,
-                dict_t *dict, int32_t flags, dict_t *xdata)
-{
-        glupy_private_t *priv = this->private;
-        PyGILState_STATE gstate;
-        int32_t ret;
-        static long next_id = 0;
-
-        if (!priv->fops[GLUPY_SETXATTR]) {
-                goto wind;
-        }
-
-        gstate = glupy_enter();
-        frame->local = (void *)++next_id;
-        ret = ((fop_setxattr_t)(priv->fops[GLUPY_SETXATTR]))(
-                frame, this, loc, dict, flags, xdata);
-        glupy_leave(gstate);
-
-        return ret;
-
-wind:
-        STACK_WIND (frame, glupy_setxattr_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->setxattr, loc, dict,
-                    flags, xdata);
-        return 0;
-}
-
-void
-wind_setxattr (call_frame_t *frame, xlator_t *xl, loc_t *loc,
-               dict_t *dict, int32_t flags, dict_t *xdata)
-{
-        xlator_t        *this = THIS;
-
-        if (!xl || (xl == this)) {
-                xl = FIRST_CHILD(this);
-        }
-
-        STACK_WIND (frame, glupy_setxattr_cbk, xl, xl->fops->setxattr,
-                    loc, dict, flags, xdata);
-}
-
-
-void
-unwind_setxattr (call_frame_t *frame, long cookie, xlator_t *this,
-                 int32_t op_ret, int32_t op_errno, dict_t *xdata)
-{
-        frame->local = NULL;
-        STACK_UNWIND_STRICT (setxattr, frame, op_ret, op_errno, xdata);
-
-}
-
-void
-set_setxattr_fop (long py_this, fop_setxattr_t fop)
-{
-        glupy_private_t *priv   = ((xlator_t *)py_this)->private;
-
-        priv->fops[GLUPY_SETXATTR] = (long)fop;
-}
-
-void
-set_setxattr_cbk (long py_this, fop_setxattr_cbk_t cbk)
-{
-        glupy_private_t *priv   = ((xlator_t *)py_this)->private;
-
-        priv->cbks[GLUPY_SETXATTR] = (long)cbk;
-}
-
-/* FOP: GETXATTR */
-
-int32_t
-glupy_getxattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                    int32_t op_ret, int32_t op_errno, dict_t *dict,
-                    dict_t *xdata)
-{
-        glupy_private_t *priv = this->private;
-        PyGILState_STATE gstate;
-        int32_t ret;
-
-        if (!priv->cbks[GLUPY_GETXATTR]) {
-                goto unwind;
-        }
-
-        gstate = glupy_enter();
-        ret = ((fop_getxattr_cbk_t)(priv->cbks[GLUPY_GETXATTR]))(
-                frame, cookie, this, op_ret, op_errno, dict,
-                xdata);
-        glupy_leave(gstate);
-
-        return ret;
-
-unwind:
-        frame->local = NULL;
-        STACK_UNWIND_STRICT (getxattr, frame, op_ret, op_errno, dict,
-                             xdata);
-        return 0;
-}
-
-int32_t
-glupy_getxattr (call_frame_t *frame, xlator_t *this, loc_t *loc,
-                const char *name, dict_t *xdata)
-{
-        glupy_private_t *priv = this->private;
-        PyGILState_STATE gstate;
-        int32_t ret;
-        static long next_id = 0;
-
-        if (!priv->fops[GLUPY_GETXATTR]) {
-                goto wind;
-        }
-
-        gstate = glupy_enter();
-        frame->local = (void *)++next_id;
-        ret = ((fop_getxattr_t)(priv->fops[GLUPY_GETXATTR]))(
-                frame, this, loc, name, xdata);
-        glupy_leave(gstate);
-
-        return ret;
-
-wind:
-        STACK_WIND (frame, glupy_getxattr_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->getxattr, loc, name,
-                    xdata);
-        return 0;
-}
-
-void
-wind_getxattr (call_frame_t *frame, xlator_t *xl, loc_t *loc,
-               const char *name, dict_t *xdata)
-{
-        xlator_t        *this = THIS;
-
-        if (!xl || (xl == this)) {
-                xl = FIRST_CHILD(this);
-        }
-
-        STACK_WIND (frame, glupy_getxattr_cbk, xl, xl->fops->getxattr,
-                    loc, name, xdata);
-}
-
-
-void
-unwind_getxattr (call_frame_t *frame, long cookie, xlator_t *this,
-                 int32_t op_ret, int32_t op_errno, dict_t *dict,
-                 dict_t *xdata)
-{
-        frame->local = NULL;
-        STACK_UNWIND_STRICT (getxattr, frame, op_ret, op_errno, dict,
-                             xdata);
-
-}
-
-
-void
-set_getxattr_fop (long py_this, fop_getxattr_t fop)
-{
-        glupy_private_t *priv   = ((xlator_t *)py_this)->private;
-
-        priv->fops[GLUPY_GETXATTR] = (long)fop;
-}
-
-
-void
-set_getxattr_cbk (long py_this, fop_getxattr_cbk_t cbk)
-{
-        glupy_private_t *priv   = ((xlator_t *)py_this)->private;
-
-        priv->cbks[GLUPY_GETXATTR] = (long)cbk;
-}
-
-/* FOP: FSETXATTR */
-
-int32_t
-glupy_fsetxattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                     int32_t op_ret, int32_t op_errno, dict_t *xdata)
-{
-        glupy_private_t *priv = this->private;
-        PyGILState_STATE gstate;
-        int32_t ret;
-
-        if (!priv->cbks[GLUPY_FSETXATTR]) {
-                goto unwind;
-        }
-
-        gstate = glupy_enter();
-        ret = ((fop_fsetxattr_cbk_t)(priv->cbks[GLUPY_FSETXATTR]))(
-                frame, cookie, this, op_ret, op_errno,
-                xdata);
-        glupy_leave(gstate);
-
-        return ret;
-
-unwind:
-        frame->local = NULL;
-        STACK_UNWIND_STRICT (fsetxattr, frame, op_ret, op_errno, xdata);
-        return 0;
-}
-
-int32_t
-glupy_fsetxattr (call_frame_t *frame, xlator_t *this, fd_t *fd,
-                 dict_t *dict, int32_t flags, dict_t *xdata)
-{
-        glupy_private_t *priv = this->private;
-        PyGILState_STATE gstate;
-        int32_t ret;
-        static long next_id = 0;
-
-        if (!priv->fops[GLUPY_FSETXATTR]) {
-                goto wind;
-        }
-
-        gstate = glupy_enter();
-        frame->local = (void *)++next_id;
-        ret = ((fop_fsetxattr_t)(priv->fops[GLUPY_FSETXATTR]))(
-                frame, this, fd, dict, flags, xdata);
-        glupy_leave(gstate);
-
-        return ret;
-
-wind:
-        STACK_WIND (frame, glupy_fsetxattr_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->fsetxattr, fd, dict,
-                    flags, xdata);
-        return 0;
-}
-
-void
-wind_fsetxattr (call_frame_t *frame, xlator_t *xl, fd_t *fd,
-                dict_t *dict, int32_t flags, dict_t *xdata)
-{
-        xlator_t        *this = THIS;
-
-        if (!xl || (xl == this)) {
-                xl = FIRST_CHILD(this);
-        }
-
-        STACK_WIND (frame, glupy_fsetxattr_cbk, xl, xl->fops->fsetxattr,
-                    fd, dict, flags, xdata);
-}
-
-
-void
-unwind_fsetxattr (call_frame_t *frame, long cookie, xlator_t *this,
-                  int32_t op_ret, int32_t op_errno, dict_t *xdata)
-{
-        frame->local = NULL;
-        STACK_UNWIND_STRICT (fsetxattr, frame, op_ret, op_errno, xdata);
-
-}
-
-void
-set_fsetxattr_fop (long py_this, fop_fsetxattr_t fop)
-{
-        glupy_private_t *priv   = ((xlator_t *)py_this)->private;
-
-        priv->fops[GLUPY_FSETXATTR] = (long)fop;
-}
-
-void
-set_fsetxattr_cbk (long py_this, fop_fsetxattr_cbk_t cbk)
-{
-        glupy_private_t *priv   = ((xlator_t *)py_this)->private;
-
-        priv->cbks[GLUPY_FSETXATTR] = (long)cbk;
-}
-
-/* FOP: FGETXATTR */
-
-int32_t
-glupy_fgetxattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                     int32_t op_ret, int32_t op_errno, dict_t *dict,
-                     dict_t *xdata)
-{
-        glupy_private_t *priv = this->private;
-        PyGILState_STATE gstate;
-        int32_t ret;
-
-        if (!priv->cbks[GLUPY_FGETXATTR]) {
-                goto unwind;
-        }
-
-        gstate = glupy_enter();
-        ret = ((fop_fgetxattr_cbk_t)(priv->cbks[GLUPY_FGETXATTR]))(
-                frame, cookie, this, op_ret, op_errno, dict,
-                xdata);
-        glupy_leave(gstate);
-
-        return ret;
-
-unwind:
-        frame->local = NULL;
-        STACK_UNWIND_STRICT (fgetxattr, frame, op_ret, op_errno, dict,
-                             xdata);
-        return 0;
-}
-
-int32_t
-glupy_fgetxattr (call_frame_t *frame, xlator_t *this, fd_t *fd,
-                 const char *name, dict_t *xdata)
-{
-        glupy_private_t *priv = this->private;
-        PyGILState_STATE gstate;
-        int32_t ret;
-        static long next_id = 0;
-
-        if (!priv->fops[GLUPY_FGETXATTR]) {
-                goto wind;
-        }
-
-        gstate = glupy_enter();
-        frame->local = (void *)++next_id;
-        ret = ((fop_fgetxattr_t)(priv->fops[GLUPY_FGETXATTR]))(
-                frame, this, fd, name, xdata);
-        glupy_leave(gstate);
-
-        return ret;
-
-wind:
-        STACK_WIND (frame, glupy_fgetxattr_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->fgetxattr, fd, name,
-                    xdata);
-        return 0;
-}
-
-void
-wind_fgetxattr (call_frame_t *frame, xlator_t *xl, fd_t *fd,
-                const char *name, dict_t *xdata)
-{
-        xlator_t        *this = THIS;
-
-        if (!xl || (xl == this)) {
-                xl = FIRST_CHILD(this);
-        }
-
-        STACK_WIND (frame, glupy_fgetxattr_cbk, xl, xl->fops->fgetxattr,
-                    fd, name, xdata);
-}
-
-
-void
-unwind_fgetxattr (call_frame_t *frame, long cookie, xlator_t *this,
-                  int32_t op_ret, int32_t op_errno, dict_t *dict,
-                  dict_t *xdata)
-{
-        frame->local = NULL;
-        STACK_UNWIND_STRICT (fgetxattr, frame, op_ret, op_errno, dict,
-                             xdata);
-
-}
-
-
-void
-set_fgetxattr_fop (long py_this, fop_fgetxattr_t fop)
-{
-        glupy_private_t *priv   = ((xlator_t *)py_this)->private;
-
-        priv->fops[GLUPY_FGETXATTR] = (long)fop;
-}
-
-
-void
-set_fgetxattr_cbk (long py_this, fop_fgetxattr_cbk_t cbk)
-{
-        glupy_private_t *priv   = ((xlator_t *)py_this)->private;
-
-        priv->cbks[GLUPY_FGETXATTR] = (long)cbk;
-}
-
-/* FOP:REMOVEXATTR */
-
-int32_t
-glupy_removexattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                    int32_t op_ret, int32_t op_errno, dict_t *xdata)
-{
-        glupy_private_t *priv = this->private;
-        PyGILState_STATE gstate;
-        int32_t ret;
-
-        if (!priv->cbks[GLUPY_REMOVEXATTR]) {
-                goto unwind;
-        }
-
-        gstate = glupy_enter();
-        ret = ((fop_removexattr_cbk_t)(priv->cbks[GLUPY_REMOVEXATTR]))(
-                frame, cookie, this, op_ret, op_errno, xdata);
-        glupy_leave(gstate);
-
-        return ret;
-
-unwind:
-        frame->local = NULL;
-        STACK_UNWIND_STRICT (removexattr, frame, op_ret, op_errno, xdata);
-        return 0;
-}
-
-int32_t
-glupy_removexattr (call_frame_t *frame, xlator_t *this, loc_t *loc,
-                   const char *name, dict_t *xdata)
-{
-        glupy_private_t *priv = this->private;
-        PyGILState_STATE gstate;
-        int32_t ret;
-        static long next_id = 0;
-
-        if (!priv->fops[GLUPY_REMOVEXATTR]) {
-                goto wind;
-        }
-
-        gstate = glupy_enter();
-        frame->local = (void *)++next_id;
-        ret = ((fop_removexattr_t)(priv->fops[GLUPY_REMOVEXATTR]))(
-                frame, this, loc, name, xdata);
-        glupy_leave(gstate);
-
-        return ret;
-
-wind:
-        STACK_WIND (frame, glupy_removexattr_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->removexattr, loc, name,
-                    xdata);
-        return 0;
-}
-
-void
-wind_removexattr (call_frame_t *frame, xlator_t *xl, loc_t *loc,
-                  const char *name, dict_t *xdata)
-{
-        xlator_t        *this = THIS;
-
-        if (!xl || (xl == this)) {
-                xl = FIRST_CHILD(this);
-        }
-
-        STACK_WIND (frame, glupy_removexattr_cbk, xl, xl->fops->removexattr,
-                    loc, name, xdata);
-}
-
-
-void
-unwind_removexattr (call_frame_t *frame, long cookie, xlator_t *this,
-                    int32_t op_ret, int32_t op_errno, dict_t *xdata)
-{
-        frame->local = NULL;
-        STACK_UNWIND_STRICT (removexattr, frame, op_ret, op_errno, xdata);
-
-}
-
-void
-set_removexattr_fop (long py_this, fop_removexattr_t fop)
-{
-        glupy_private_t *priv   = ((xlator_t *)py_this)->private;
-
-        priv->fops[GLUPY_REMOVEXATTR] = (long)fop;
-}
-
-void
-set_removexattr_cbk (long py_this, fop_removexattr_cbk_t cbk)
-{
-        glupy_private_t *priv   = ((xlator_t *)py_this)->private;
-
-        priv->cbks[GLUPY_REMOVEXATTR] = (long)cbk;
-}
-
-
-/* FOP:FREMOVEXATTR */
-
-int32_t
-glupy_fremovexattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                    int32_t op_ret, int32_t op_errno, dict_t *xdata)
-{
-        glupy_private_t *priv = this->private;
-        PyGILState_STATE gstate;
-        int32_t ret;
-
-        if (!priv->cbks[GLUPY_FREMOVEXATTR]) {
-                goto unwind;
-        }
-
-        gstate = glupy_enter();
-        ret = ((fop_fremovexattr_cbk_t)(priv->cbks[GLUPY_FREMOVEXATTR]))(
-                frame, cookie, this, op_ret, op_errno, xdata);
-        glupy_leave(gstate);
-
-        return ret;
-
-unwind:
-        frame->local = NULL;
-        STACK_UNWIND_STRICT (fremovexattr, frame, op_ret, op_errno, xdata);
-        return 0;
-}
-
-int32_t
-glupy_fremovexattr (call_frame_t *frame, xlator_t *this, fd_t *fd,
-                   const char *name, dict_t *xdata)
-{
-        glupy_private_t *priv = this->private;
-        PyGILState_STATE gstate;
-        int32_t ret;
-        static long next_id = 0;
-
-        if (!priv->fops[GLUPY_FREMOVEXATTR]) {
-                goto wind;
-        }
-
-        gstate = glupy_enter();
-        frame->local = (void *)++next_id;
-        ret = ((fop_fremovexattr_t)(priv->fops[GLUPY_FREMOVEXATTR]))(
-                frame, this, fd, name, xdata);
-        glupy_leave(gstate);
-
-        return ret;
-
-wind:
-        STACK_WIND (frame, glupy_fremovexattr_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->fremovexattr, fd, name,
-                    xdata);
-        return 0;
-}
-
-void
-wind_fremovexattr (call_frame_t *frame, xlator_t *xl, fd_t *fd,
-                   const char *name, dict_t *xdata)
-{
-        xlator_t        *this = THIS;
-
-        if (!xl || (xl == this)) {
-                xl = FIRST_CHILD(this);
-        }
-
-        STACK_WIND (frame, glupy_fremovexattr_cbk, xl, xl->fops->fremovexattr,
-                    fd, name, xdata);
-}
-
-
-void
-unwind_fremovexattr (call_frame_t *frame, long cookie, xlator_t *this,
-                     int32_t op_ret, int32_t op_errno, dict_t *xdata)
-{
-        frame->local = NULL;
-        STACK_UNWIND_STRICT (fremovexattr, frame, op_ret, op_errno, xdata);
-
-}
-
-void
-set_fremovexattr_fop (long py_this, fop_fremovexattr_t fop)
-{
-        glupy_private_t *priv   = ((xlator_t *)py_this)->private;
-
-        priv->fops[GLUPY_FREMOVEXATTR] = (long)fop;
-}
-
-void
-set_fremovexattr_cbk (long py_this, fop_fremovexattr_cbk_t cbk)
-{
-        glupy_private_t *priv   = ((xlator_t *)py_this)->private;
-
-        priv->cbks[GLUPY_FREMOVEXATTR] = (long)cbk;
-}
-
-
-/* FOP: LINK*/
-int32_t
-glupy_link_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                int32_t op_ret, int32_t op_errno, inode_t *inode,
-                struct iatt *buf, struct iatt *preparent,
-                struct iatt *postparent, dict_t *xdata)
-{
-        glupy_private_t *priv = this->private;
-        PyGILState_STATE gstate;
-        int32_t ret;
-
-        if (!priv->cbks[GLUPY_LINK]) {
-                goto unwind;
-        }
-
-        gstate = glupy_enter();
-        ret = ((fop_link_cbk_t)(priv->cbks[GLUPY_LINK]))(
-                frame, cookie, this, op_ret, op_errno,
-                inode, buf, preparent, postparent, xdata);
-        glupy_leave(gstate);
-
-        return ret;
-
-unwind:
-        frame->local = NULL;
-        STACK_UNWIND_STRICT (link, frame, op_ret, op_errno, inode, buf,
-                             preparent, postparent, xdata);
-        return 0;
-}
-
-int32_t
-glupy_link(call_frame_t *frame, xlator_t *this, loc_t *oldloc, loc_t *newloc,
-           dict_t *xdata)
-{
-        glupy_private_t *priv = this->private;
-        PyGILState_STATE gstate;
-        int32_t ret;
-        static long next_id = 0;
-
-        if (!priv->fops[GLUPY_LINK]) {
-                goto wind;
-        }
-
-        gstate = glupy_enter();
-        frame->local = (void *)++next_id;
-        ret = ((fop_link_t)(priv->fops[GLUPY_LINK]))(
-                frame, this, oldloc, newloc, xdata);
-        glupy_leave(gstate);
-
-        return ret;
-
-wind:
-        STACK_WIND (frame, glupy_link_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->link, oldloc, newloc,
-                    xdata);
-        return 0;
-}
-
-void
-wind_link (call_frame_t *frame, xlator_t *xl, loc_t *oldloc, loc_t *newloc,
-           dict_t *xdata)
-{
-        xlator_t        *this = THIS;
-
-        if (!xl || (xl == this)) {
-                xl = FIRST_CHILD(this);
-        }
-
-        STACK_WIND (frame, glupy_link_cbk, xl, xl->fops->link,
-                    oldloc, newloc, xdata);
-}
-
-void
-unwind_link (call_frame_t *frame, long cookie, xlator_t *this,
-             int32_t op_ret, int32_t op_errno, inode_t *inode,
-             struct iatt *buf, struct iatt *preparent,
-             struct iatt *postparent, dict_t *xdata)
-{
-        frame->local = NULL;
-        STACK_UNWIND_STRICT (link, frame, op_ret, op_errno, inode, buf,
-                             preparent, postparent, xdata);
-}
-
-void
-set_link_fop (long py_this, fop_link_t fop)
-{
-        glupy_private_t *priv   = ((xlator_t *)py_this)->private;
-
-        priv->fops[GLUPY_LINK] = (long)fop;
-}
-
-void
-set_link_cbk (long py_this, fop_link_cbk_t cbk)
-{
-        glupy_private_t *priv   = ((xlator_t *)py_this)->private;
-
-        priv->cbks[GLUPY_LINK] = (long)cbk;
-}
-
-/* FOP: SYMLINK*/
-int32_t
-glupy_symlink_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                   int32_t op_ret, int32_t op_errno, inode_t *inode,
-                   struct iatt *buf, struct iatt *preparent,
-                   struct iatt *postparent, dict_t *xdata)
-{
-        glupy_private_t *priv = this->private;
-        PyGILState_STATE gstate;
-        int32_t ret;
-
-        if (!priv->cbks[GLUPY_SYMLINK]) {
-                goto unwind;
-        }
-
-        gstate = glupy_enter();
-        ret = ((fop_symlink_cbk_t)(priv->cbks[GLUPY_SYMLINK]))(
-                frame, cookie, this, op_ret, op_errno,
-                inode, buf, preparent, postparent, xdata);
-        glupy_leave(gstate);
-
-        return ret;
-
-unwind:
-        frame->local = NULL;
-        STACK_UNWIND_STRICT (symlink, frame, op_ret, op_errno, inode, buf,
-                             preparent, postparent, xdata);
-        return 0;
-}
-
-int32_t
-glupy_symlink(call_frame_t *frame, xlator_t *this, const char *linkname,
-              loc_t *loc, mode_t umask, dict_t *xdata)
-{
-        glupy_private_t *priv = this->private;
-        PyGILState_STATE gstate;
-        int32_t ret;
-        static long next_id = 0;
-
-        if (!priv->fops[GLUPY_SYMLINK]) {
-                goto wind;
-        }
-
-        gstate = glupy_enter();
-        frame->local = (void *)++next_id;
-        ret = ((fop_symlink_t)(priv->fops[GLUPY_SYMLINK]))(
-                frame, this, linkname, loc, umask, xdata);
-        glupy_leave(gstate);
-
-        return ret;
-
-wind:
-        STACK_WIND (frame, glupy_symlink_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->symlink, linkname, loc,
-                    umask, xdata);
-        return 0;
-}
-
-void
-wind_symlink (call_frame_t *frame, xlator_t *xl, const char *linkname,
-              loc_t *loc, mode_t umask, dict_t *xdata)
-{
-        xlator_t        *this = THIS;
-
-        if (!xl || (xl == this)) {
-                xl = FIRST_CHILD(this);
-        }
-
-        STACK_WIND (frame, glupy_symlink_cbk, xl, xl->fops->symlink,
-                    linkname, loc, umask, xdata);
-}
-
-void
-unwind_symlink (call_frame_t *frame, long cookie, xlator_t *this,
-                int32_t op_ret, int32_t op_errno, inode_t *inode,
-                struct iatt *buf, struct iatt *preparent,
-                struct iatt *postparent, dict_t *xdata)
-{
-        frame->local = NULL;
-        STACK_UNWIND_STRICT (symlink, frame, op_ret, op_errno, inode, buf,
-                             preparent, postparent, xdata);
-}
-
-void
-set_symlink_fop (long py_this, fop_symlink_t fop)
-{
-        glupy_private_t *priv   = ((xlator_t *)py_this)->private;
-
-        priv->fops[GLUPY_SYMLINK] = (long)fop;
-}
-
-void
-set_symlink_cbk (long py_this, fop_symlink_cbk_t cbk)
-{
-        glupy_private_t *priv   = ((xlator_t *)py_this)->private;
-
-        priv->cbks[GLUPY_SYMLINK] = (long)cbk;
-}
-
-
-/* FOP: READLINK */
-int32_t
-glupy_readlink_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                    int32_t op_ret, int32_t op_errno, const char *path,
-                    struct iatt *buf, dict_t *xdata)
-{
-        glupy_private_t *priv = this->private;
-        PyGILState_STATE gstate;
-        int32_t ret;
-
-        if (!priv->cbks[GLUPY_READLINK]) {
-                goto unwind;
-        }
-
-        gstate = glupy_enter();
-        ret = ((fop_readlink_cbk_t)(priv->cbks[GLUPY_READLINK]))(
-                frame, cookie, this, op_ret, op_errno,
-                path, buf, xdata);
-        glupy_leave(gstate);
-
-        return ret;
-
-unwind:
-        frame->local = NULL;
-        STACK_UNWIND_STRICT (readlink, frame, op_ret, op_errno, path,
-                             buf, xdata);
-        return 0;
-}
-
-int32_t
-glupy_readlink (call_frame_t *frame, xlator_t *this, loc_t *loc,
-                size_t size, dict_t *xdata)
-{
-        glupy_private_t *priv = this->private;
-        PyGILState_STATE gstate;
-        int32_t ret;
-        static long next_id = 0;
-
-        if (!priv->fops[GLUPY_READLINK]) {
-                goto wind;
-        }
-
-        gstate = glupy_enter();
-        frame->local = (void *)++next_id;
-        ret = ((fop_readlink_t)(priv->fops[GLUPY_READLINK]))(
-                frame, this, loc, size, xdata);
-        glupy_leave(gstate);
-
-        return ret;
-
-wind:
-        STACK_WIND (frame, glupy_readlink_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->readlink, loc,
-                    size, xdata);
-        return 0;
-}
-
-void
-wind_readlink (call_frame_t *frame, xlator_t *xl, loc_t *loc,
-               size_t size, dict_t *xdata)
-{
-        xlator_t        *this = THIS;
-
-        if (!xl || (xl == this)) {
-                xl = FIRST_CHILD(this);
-        }
-
-        STACK_WIND (frame, glupy_readlink_cbk, xl, xl->fops->readlink,
-                    loc, size, xdata);
-}
-
-void
-unwind_readlink (call_frame_t *frame, long cookie, xlator_t *this,
-                 int32_t op_ret, int32_t op_errno, const char *path,
-                 struct iatt *buf, dict_t *xdata)
-{
-        frame->local = NULL;
-        STACK_UNWIND_STRICT (readlink, frame, op_ret, op_errno, path, buf,
-                             xdata);
-}
-
-void
-set_readlink_fop (long py_this, fop_readlink_t fop)
-{
-        glupy_private_t *priv   = ((xlator_t *)py_this)->private;
-
-        priv->fops[GLUPY_READLINK] = (long)fop;
-}
-
-void
-set_readlink_cbk (long py_this, fop_readlink_cbk_t cbk)
-{
-        glupy_private_t *priv   = ((xlator_t *)py_this)->private;
-
-        priv->cbks[GLUPY_READLINK] = (long)cbk;
-}
-
-
-/* FOP: UNLINK */
-
-int32_t
-glupy_unlink_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                  int32_t op_ret, int32_t op_errno,
-                  struct iatt *preparent, struct iatt *postparent,
-                  dict_t *xdata)
-{
-        glupy_private_t *priv = this->private;
-        PyGILState_STATE gstate;
-        int32_t ret;
-
-        if (!priv->cbks[GLUPY_UNLINK]) {
-                goto unwind;
-        }
-
-        gstate = glupy_enter();
-        ret = ((fop_unlink_cbk_t)(priv->cbks[GLUPY_UNLINK]))(
-                frame, cookie, this, op_ret, op_errno,
-                preparent, postparent, xdata);
-        glupy_leave(gstate);
-
-        return ret;
-
-unwind:
-        frame->local = NULL;
-        STACK_UNWIND_STRICT (unlink, frame, op_ret, op_errno, preparent,
-                             postparent, xdata);
-        return 0;
-}
-
-int32_t
-glupy_unlink (call_frame_t *frame, xlator_t *this, loc_t *loc,
-              int xflags, dict_t *xdata)
-{
-        glupy_private_t *priv = this->private;
-        PyGILState_STATE gstate;
-        int32_t ret;
-        static long next_id = 0;
-
-        if (!priv->fops[GLUPY_UNLINK]) {
-                goto wind;
-        }
-
-        gstate = glupy_enter();
-        frame->local = (void *)++next_id;
-        ret = ((fop_unlink_t)(priv->fops[GLUPY_UNLINK]))(
-                frame, this, loc, xflags, xdata);
-        glupy_leave(gstate);
-
-        return ret;
-
-wind:
-        STACK_WIND (frame, glupy_unlink_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->unlink, loc,
-                    xflags, xdata);
-        return 0;
-}
-
-void
-wind_unlink (call_frame_t *frame, xlator_t *xl, loc_t *loc,
-             int xflags, dict_t *xdata)
-{
-        xlator_t        *this = THIS;
-
-        if (!xl || (xl == this)) {
-                xl = FIRST_CHILD(this);
-        }
-
-        STACK_WIND (frame, glupy_unlink_cbk, xl, xl->fops->unlink,
-                    loc, xflags, xdata);
-}
-
-void
-unwind_unlink (call_frame_t *frame, long cookie, xlator_t *this,
-               int32_t op_ret, int32_t op_errno,
-               struct iatt *preparent, struct iatt *postparent,
-               dict_t *xdata)
-{
-        frame->local = NULL;
-        STACK_UNWIND_STRICT (unlink, frame, op_ret, op_errno,
-                             preparent, postparent, xdata);
-}
-
-void
-set_unlink_fop (long py_this, fop_unlink_t fop)
-{
-        glupy_private_t *priv   = ((xlator_t *)py_this)->private;
-
-        priv->fops[GLUPY_UNLINK] = (long)fop;
-}
-
-void
-set_unlink_cbk (long py_this, fop_unlink_cbk_t cbk)
-{
-        glupy_private_t *priv   = ((xlator_t *)py_this)->private;
-
-        priv->cbks[GLUPY_UNLINK] = (long)cbk;
-}
-
-
-/* FOP: MKDIR */
-
-int32_t
-glupy_mkdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                 int32_t op_ret, int32_t op_errno, inode_t *inode,
-                 struct iatt *buf, struct iatt *preparent,
-                 struct iatt *postparent, dict_t *xdata)
-{
-        glupy_private_t *priv = this->private;
-        PyGILState_STATE gstate;
-        int32_t ret;
-
-        if (!priv->cbks[GLUPY_MKDIR]) {
-                goto unwind;
-        }
-
-        gstate = glupy_enter();
-        ret = ((fop_mkdir_cbk_t)(priv->cbks[GLUPY_MKDIR]))(
-                frame, cookie, this, op_ret, op_errno,
-                inode, buf, preparent, postparent, xdata);
-        glupy_leave(gstate);
-
-        return ret;
-
-unwind:
-        frame->local = NULL;
-        STACK_UNWIND_STRICT (mkdir, frame, op_ret, op_errno, inode, buf,
-                             preparent, postparent, xdata);
-        return 0;
-}
-
-int32_t
-glupy_mkdir (call_frame_t *frame, xlator_t *this, loc_t *loc, mode_t mode,
-             mode_t umask, dict_t *xdata)
-{
-        glupy_private_t *priv = this->private;
-        PyGILState_STATE gstate;
-        int32_t ret;
-        static long next_id = 0;
-
-        if (!priv->fops[GLUPY_MKDIR]) {
-                goto wind;
-        }
-
-        gstate = glupy_enter();
-        frame->local = (void *)++next_id;
-        ret = ((fop_mkdir_t)(priv->fops[GLUPY_MKDIR]))(
-                frame, this, loc, mode, umask, xdata);
-        glupy_leave(gstate);
-
-        return ret;
-
-wind:
-        STACK_WIND (frame, glupy_mkdir_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->mkdir, loc, mode, umask,
-                    xdata);
-        return 0;
-}
-
-void
-wind_mkdir (call_frame_t *frame, xlator_t *xl, loc_t *loc, mode_t mode,
-            mode_t umask, dict_t *xdata)
-{
-
-        xlator_t        *this = THIS;
-
-        if (!xl || (xl == this)) {
-                xl = FIRST_CHILD(this);
-        }
-
-        STACK_WIND (frame, glupy_mkdir_cbk, xl, xl->fops->mkdir,
-                    loc, mode, umask, xdata);
-}
-
-void
-unwind_mkdir (call_frame_t *frame, long cookie, xlator_t *this,
-              int32_t op_ret, int32_t op_errno, inode_t *inode,
-              struct iatt *buf, struct iatt *preparent,
-              struct iatt *postparent, dict_t *xdata)
-{
-        frame->local = NULL;
-        STACK_UNWIND_STRICT (mkdir, frame, op_ret, op_errno, inode, buf,
-                             preparent, postparent, xdata);
-}
-
-void
-set_mkdir_fop (long py_this, fop_mkdir_t fop)
-{
-        glupy_private_t *priv   = ((xlator_t *)py_this)->private;
-
-        priv->fops[GLUPY_MKDIR] = (long)fop;
-}
-
-void
-set_mkdir_cbk (long py_this, fop_mkdir_cbk_t cbk)
-{
-        glupy_private_t *priv   = ((xlator_t *)py_this)->private;
-
-        priv->cbks[GLUPY_MKDIR] = (long)cbk;
-}
-
-
-/* FOP: RMDIR */
-
-int32_t
-glupy_rmdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                 int32_t op_ret, int32_t op_errno,
-                 struct iatt *preparent, struct iatt *postparent,
-                 dict_t *xdata)
-{
-        glupy_private_t *priv = this->private;
-        PyGILState_STATE gstate;
-        int32_t ret;
-
-        if (!priv->cbks[GLUPY_RMDIR]) {
-                goto unwind;
-        }
-
-        gstate = glupy_enter();
-        ret = ((fop_rmdir_cbk_t)(priv->cbks[GLUPY_RMDIR]))(
-                frame, cookie, this, op_ret, op_errno,
-                preparent, postparent, xdata);
-        glupy_leave(gstate);
-
-        return ret;
-
-unwind:
-        frame->local = NULL;
-        STACK_UNWIND_STRICT (rmdir, frame, op_ret, op_errno, preparent,
-                             postparent, xdata);
-        return 0;
-}
-
-int32_t
-glupy_rmdir (call_frame_t *frame, xlator_t *this, loc_t *loc,
-             int xflags, dict_t *xdata)
-{
-        glupy_private_t *priv = this->private;
-        PyGILState_STATE gstate;
-        int32_t ret;
-        static long next_id = 0;
-
-        if (!priv->fops[GLUPY_RMDIR]) {
-                goto wind;
-        }
-
-        gstate = glupy_enter();
-        frame->local = (void *)++next_id;
-        ret = ((fop_rmdir_t)(priv->fops[GLUPY_RMDIR]))(
-                frame, this, loc, xflags, xdata);
-        glupy_leave(gstate);
-
-        return ret;
-
-wind:
-        STACK_WIND (frame, glupy_rmdir_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->rmdir, loc,
-                    xflags, xdata);
-        return 0;
-}
-
-void
-wind_rmdir (call_frame_t *frame, xlator_t *xl, loc_t *loc,
-            int xflags, dict_t *xdata)
-{
-
-        xlator_t        *this = THIS;
-
-        if (!xl || (xl == this)) {
-                xl = FIRST_CHILD(this);
-        }
-
-        STACK_WIND (frame, glupy_rmdir_cbk, xl, xl->fops->rmdir,
-                    loc, xflags, xdata);
-}
-
-void
-unwind_rmdir (call_frame_t *frame, long cookie, xlator_t *this,
-              int32_t op_ret, int32_t op_errno,
-              struct iatt *preparent, struct iatt *postparent,
-              dict_t *xdata)
-{
-        frame->local = NULL;
-        STACK_UNWIND_STRICT (rmdir, frame, op_ret, op_errno,
-                             preparent, postparent, xdata);
-}
-
-void
-set_rmdir_fop (long py_this, fop_rmdir_t fop)
-{
-        glupy_private_t *priv   = ((xlator_t *)py_this)->private;
-
-        priv->fops[GLUPY_RMDIR] = (long)fop;
-}
-
-void
-set_rmdir_cbk (long py_this, fop_rmdir_cbk_t cbk)
-{
-        glupy_private_t *priv   = ((xlator_t *)py_this)->private;
-
-        priv->cbks[GLUPY_RMDIR] = (long)cbk;
-}
-
-
-/* NON-FOP-SPECIFIC CODE */
-
-
-long
-get_id (call_frame_t *frame)
-{
-        return (long)(frame->local);
-}
-
-uint64_t
-get_rootunique (call_frame_t *frame)
-{
-        return frame->root->unique;
-}
-
-int32_t
-mem_acct_init (xlator_t *this)
-{
-        int     ret = -1;
-
-        if (!this)
-                return ret;
-
-        ret = xlator_mem_acct_init (this, gf_glupy_mt_end);
-
-        if (ret != 0) {
-                gf_log(this->name, GF_LOG_ERROR, "Memory accounting init"
-                       " failed");
-                return ret;
-        }
-
-        return ret;
-}
-
-int32_t
-init (xlator_t *this)
-{
-        glupy_private_t         *priv           = NULL;
-        char                    *module_name    = NULL;
-        PyObject                *py_mod_name    = NULL;
-        PyObject                *py_init_func   = NULL;
-        PyObject                *py_args        = NULL;
-        PyObject                *syspath        = NULL;
-	PyObject                *path           = NULL;
-	PyObject                *error_type     = NULL;
-	PyObject                *error_msg      = NULL;
-	PyObject                *error_bt       = NULL;
-	static gf_boolean_t      py_inited      = _gf_false;
-        void *                   err_cleanup    = &&err_return;
-
-        if (dict_get_str(this->options,"module-name",&module_name) != 0) {
-                gf_log (this->name, GF_LOG_ERROR, "missing module-name");
-                return -1;
-        }
-
-	priv = GF_CALLOC (1, sizeof (glupy_private_t), gf_glupy_mt_priv);
-        if (!priv) {
-                goto *err_cleanup;
-        }
-        this->private = priv;
-        err_cleanup = &&err_free_priv;
-
-	if (!py_inited) {
-	        /* 
-                 * This must be done before Py_Initialize(),
-                 * because it will duplicate the environment,
-                 * and fail to see later environment updates.
-                 */
-	        setenv("PATH_GLUSTERFS_GLUPY_MODULE",
-                       PATH_GLUSTERFS_GLUPY_MODULE, 1);
-
-                Py_Initialize();
-                PyEval_InitThreads();
-
-                (void)pthread_key_create(&gil_init_key,NULL);
-                (void)pthread_setspecific(gil_init_key,(void *)1);
-
-                /* PyEval_InitThreads takes this "for" us.  No thanks. */
-                PyEval_ReleaseLock();
-                py_inited = _gf_true;
-        }
-
-	/* Adjust python's path */
-	syspath = PySys_GetObject("path");
-	path = PyString_FromString(GLUSTER_PYTHON_PATH);
-	PyList_Append(syspath, path);
-	Py_DECREF(path);
-
-	py_mod_name = PyString_FromString(module_name);
-        if (!py_mod_name) {
-                gf_log (this->name, GF_LOG_ERROR, "could not create name");
-                if (PyErr_Occurred()) {
-                        PyErr_Fetch (&error_type, &error_msg, &error_bt);
-                        gf_log (this->name, GF_LOG_ERROR, "Python error: %s",
-                                PyString_AsString(error_msg));
-                }
-                goto *err_cleanup;
-        }
-
-        gf_log (this->name, GF_LOG_DEBUG, "py_mod_name = %s", module_name);
-        priv->py_module = PyImport_Import(py_mod_name);
-        Py_DECREF(py_mod_name);
-        if (!priv->py_module) {
-                gf_log (this->name, GF_LOG_ERROR, "Python import of %s failed",
-                        module_name);
-                if (PyErr_Occurred()) {
-                        PyErr_Fetch (&error_type, &error_msg, &error_bt);
-                        gf_log (this->name, GF_LOG_ERROR, "Python error: %s",
-                                PyString_AsString(error_msg));
-                }
-                goto *err_cleanup;
-        }
-        gf_log (this->name, GF_LOG_INFO, "Import of %s succeeded", module_name);
-        err_cleanup = &&err_deref_module;
-
-        py_init_func = PyObject_GetAttrString(priv->py_module, "xlator");
-        if (!py_init_func || !PyCallable_Check(py_init_func)) {
-                gf_log (this->name, GF_LOG_ERROR, "missing init func");
-                if (PyErr_Occurred()) {
-                        PyErr_Fetch (&error_type, &error_msg, &error_bt);
-                        gf_log (this->name, GF_LOG_ERROR, "Python error: %s",
-                                PyString_AsString(error_msg));
-                }
-                goto *err_cleanup;
-        }
-        err_cleanup = &&err_deref_init;
-
-        py_args = PyTuple_New(1);
-        if (!py_args) {
-                gf_log (this->name, GF_LOG_ERROR, "could not create args");
-                if (PyErr_Occurred()) {
-                        PyErr_Fetch (&error_type, &error_msg, &error_bt);
-                        gf_log (this->name, GF_LOG_ERROR, "Python error: %s",
-                                PyString_AsString(error_msg));
-                }
-                goto *err_cleanup;
-        }
-        PyTuple_SetItem(py_args,0,PyLong_FromLong((long)this));
-
-        /* TBD: pass in list of children */
-        priv->py_xlator = PyObject_CallObject(py_init_func, py_args);
-        Py_DECREF(py_args);
-        if (!priv->py_xlator) {
-                gf_log (this->name, GF_LOG_ERROR, "Python init failed");
-                if (PyErr_Occurred()) {
-                        PyErr_Fetch (&error_type, &error_msg, &error_bt);
-                        gf_log (this->name, GF_LOG_ERROR, "Python error: %s",
-                                PyString_AsString(error_msg));
-                }
-                goto *err_cleanup;
-        }
-        gf_log (this->name, GF_LOG_DEBUG, "init returned %p", priv->py_xlator);
-
-        return 0;
-
-err_deref_init:
-        Py_DECREF(py_init_func);
-err_deref_module:
-        Py_DECREF(priv->py_module);
-err_free_priv:
-        GF_FREE(priv);
-err_return:
-        return -1;
-}
-
-void
-fini (xlator_t *this)
-{
-        glupy_private_t *priv = this->private;
-
-        if (!priv)
-                return;
-        Py_DECREF(priv->py_xlator);
-        Py_DECREF(priv->py_module);
-        this->private = NULL;
-        GF_FREE (priv);
-
-        return;
-}
-
-struct xlator_fops fops = {
-        .lookup       = glupy_lookup,
-        .create       = glupy_create,
-        .open         = glupy_open,
-        .readv        = glupy_readv,
-        .writev       = glupy_writev,
-        .opendir      = glupy_opendir,
-        .readdir      = glupy_readdir,
-        .stat         = glupy_stat,
-        .fstat        = glupy_fstat,
-        .setxattr     = glupy_setxattr,
-        .getxattr     = glupy_getxattr,
-        .fsetxattr    = glupy_fsetxattr,
-        .fgetxattr    = glupy_fgetxattr,
-        .removexattr  = glupy_removexattr,
-        .fremovexattr = glupy_fremovexattr,
-        .link         = glupy_link,
-        .unlink       = glupy_unlink,
-        .readlink     = glupy_readlink,
-        .symlink      = glupy_symlink,
-        .mkdir        = glupy_mkdir,
-        .rmdir        = glupy_rmdir,
-        .statfs       = glupy_statfs,
-        .readdirp     = glupy_readdirp
-};
-
-struct xlator_cbks cbks = {
-};
-
-struct volume_options options[] = {
-        { .key  = {NULL} },
-};
diff --git a/xlators/features/glupy/src/glupy.h b/xlators/features/glupy/src/glupy.h
deleted file mode 100644
index e61d000b889..00000000000
--- a/xlators/features/glupy/src/glupy.h
+++ /dev/null
@@ -1,60 +0,0 @@
-/*
-   Copyright (c) 2006-2014 Red Hat, Inc. <http://www.redhat.com>
-   This file is part of GlusterFS.
-
-   This file is licensed to you under your choice of the GNU Lesser
-   General Public License, version 3 or any later version (LGPLv3 or
-   later), or the GNU General Public License, version 2 (GPLv2), in all
-   cases as published by the Free Software Foundation.
-*/
-
-#ifndef __GLUPY_H__
-#define __GLUPY_H__
-
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
-#include "mem-types.h"
-
-enum {
-        GLUPY_LOOKUP = 0,
-        GLUPY_CREATE,
-        GLUPY_OPEN,
-        GLUPY_READV,
-        GLUPY_WRITEV,
-        GLUPY_OPENDIR,
-        GLUPY_READDIR,
-        GLUPY_READDIRP,
-        GLUPY_STAT,
-        GLUPY_FSTAT,
-        GLUPY_STATFS,
-        GLUPY_SETXATTR,
-        GLUPY_GETXATTR,
-        GLUPY_FSETXATTR,
-        GLUPY_FGETXATTR,
-        GLUPY_REMOVEXATTR,
-        GLUPY_FREMOVEXATTR,
-        GLUPY_LINK,
-        GLUPY_UNLINK,
-        GLUPY_READLINK,
-        GLUPY_SYMLINK,
-        GLUPY_MKNOD,
-        GLUPY_MKDIR,
-        GLUPY_RMDIR,
-        GLUPY_N_FUNCS
-};
-
-typedef struct {
-        PyObject        *py_module;
-        PyObject        *py_xlator;
-        long            fops[GLUPY_N_FUNCS];
-        long            cbks[GLUPY_N_FUNCS];
-} glupy_private_t;
-
-enum gf_glupy_mem_types_ {
-        gf_glupy_mt_priv = gf_common_mt_end + 1,
-        gf_glupy_mt_end
-};
-
-#endif /* __GLUPY_H__ */
diff --git a/xlators/features/glupy/src/glupy.py b/xlators/features/glupy/src/glupy.py
deleted file mode 100644
index b9fc3700fa6..00000000000
--- a/xlators/features/glupy/src/glupy.py
+++ /dev/null
@@ -1,852 +0,0 @@
-##
-## Copyright (c) 2006-2014 Red Hat, Inc. <http://www.redhat.com>
-## This file is part of GlusterFS.
-##
-## This file is licensed to you under your choice of the GNU Lesser
-## General Public License, version 3 or any later version (LGPLv3 or
-## later), or the GNU General Public License, version 2 (GPLv2), in all
-## cases as published by the Free Software Foundation.
-##
-
-import sys
-import os
-from ctypes import *
-
-dl = CDLL(os.getenv("PATH_GLUSTERFS_GLUPY_MODULE", ""),RTLD_GLOBAL)
-
-
-class call_frame_t (Structure):
-        pass
-
-class dev_t (Structure):
-        pass
-
-
-class dict_t (Structure):
-        pass
-
-
-class gf_dirent_t (Structure):
-        pass
-
-
-class iobref_t (Structure):
-	pass
-
-
-class iovec_t (Structure):
-        pass
-
-
-class list_head (Structure):
-        pass
-
-list_head._fields_ = [
-                ("next", POINTER(list_head)),
-                ("prev", POINTER(list_head))
-        ]
-
-
-class rwxperm_t (Structure):
-        _fields_ = [
-                ("read", c_uint8, 1),
-                ("write", c_uint8, 1),
-                ("execn", c_uint8, 1)
-        ]
-
-
-class statvfs_t (Structure):
-        pass
-
-
-class xlator_t (Structure):
-        pass
-
-
-class ia_prot_t (Structure):
-        _fields_ = [
-                ("suid", c_uint8, 1),
-                ("sgid", c_uint8, 1),
-                ("sticky", c_uint8, 1),
-                ("owner", rwxperm_t),
-                ("group", rwxperm_t),
-                ("other", rwxperm_t)
-        ]
-
-# For checking file type.
-(IA_INVAL, IA_IFREG, IA_IFDIR, IA_IFLNK, IA_IFBLK, IA_IFCHR, IA_IFIFO,
- IA_IFSOCK) = xrange(8)
-
-
-class iatt_t (Structure):
-        _fields_ = [
-                ("ia_no", c_uint64),
-                ("ia_gfid", c_ubyte * 16),
-                ("ia_dev", c_uint64),
-                ("ia_type", c_uint),
-                ("ia_prot", ia_prot_t),
-                ("ia_nlink", c_uint32),
-                ("ia_uid", c_uint32),
-                ("ia_gid", c_uint32),
-                ("ia_rdev", c_uint64),
-                ("ia_size", c_uint64),
-                ("ia_blksize", c_uint32),
-                ("ia_blocks", c_uint64),
-                ("ia_atime", c_uint32 ),
-                ("ia_atime_nsec", c_uint32),
-                ("ia_mtime", c_uint32),
-                ("ia_mtime_nsec", c_uint32),
-                ("ia_ctime", c_uint32),
-                ("ia_ctime_nsec", c_uint32)
-        ]
-
-
-class mem_pool (Structure):
-        _fields_ = [
-                ("list", list_head),
-                ("hot_count", c_int),
-                ("cold_count", c_int),
-                ("lock", c_void_p),
-                ("padded_sizeof_type", c_ulong),
-                ("pool", c_void_p),
-                ("pool_end", c_void_p),
-                ("real_sizeof_type", c_int),
-                ("alloc_count", c_uint64),
-                ("pool_misses", c_uint64),
-                ("max_alloc", c_int),
-                ("curr_stdalloc", c_int),
-                ("max_stdalloc", c_int),
-                ("name", c_char_p),
-                ("global_list", list_head)
-        ]
-
-
-class U_ctx_key_inode (Union):
-        _fields_ = [
-                ("key", c_uint64),
-                ("xl_key", POINTER(xlator_t))
-        ]
-
-
-class U_ctx_value1 (Union):
-        _fields_ = [
-                ("value1", c_uint64),
-                ("ptr1", c_void_p)
-        ]
-
-
-class U_ctx_value2 (Union):
-        _fields_ = [
-                ("value2", c_uint64),
-                ("ptr2", c_void_p)
-        ]
-
-class inode_ctx (Structure):
-        _anonymous_ = ("u_key","u_value1","u_value2",)
-        _fields_ = [
-                ("u_key", U_ctx_key_inode),
-                ("u_value1", U_ctx_value1),
-                ("u_value2", U_ctx_value2)
-        ]
-
-class inode_t (Structure):
-        pass
-
-class inode_table_t (Structure):
-        _fields_ = [
-                ("lock", c_void_p),
-                ("hashsize", c_size_t),
-                ("name", c_char_p),
-                ("root", POINTER(inode_t)),
-                ("xl", POINTER(xlator_t)),
-                ("lru_limit", c_uint32),
-                ("inode_hash", POINTER(list_head)),
-                ("name_hash", POINTER(list_head)),
-                ("active", list_head),
-                ("active_size", c_uint32),
-                ("lru", list_head),
-                ("lru_size", c_uint32),
-                ("purge", list_head),
-                ("purge_size", c_uint32),
-                ("inode_pool", POINTER(mem_pool)),
-                ("dentry_pool", POINTER(mem_pool)),
-                ("fd_mem_pool", POINTER(mem_pool))
-        ]
-
-inode_t._fields_ = [
-                ("table", POINTER(inode_table_t)),
-                ("gfid", c_ubyte * 16),
-                ("lock", c_void_p),
-                ("nlookup", c_uint64),
-                ("fd_count", c_uint32),
-                ("ref", c_uint32),
-                ("ia_type", c_uint),
-                ("fd_list", list_head),
-                ("dentry_list", list_head),
-                ("hashv", list_head),
-                ("listv", list_head),
-                ("ctx", POINTER(inode_ctx))
-        ]
-
-
-
-class U_ctx_key_fd (Union):
-        _fields_ = [
-                ("key", c_uint64),
-                ("xl_key", c_void_p)
-        ]
-
-class fd_lk_ctx (Structure):
-        _fields_ = [
-                ("lk_list", list_head),
-                ("ref", c_int),
-                ("lock", c_void_p)
-        ]
-
-class fd_ctx (Structure):
-        _anonymous_ = ("u_key","u_value1")
-        _fields_ = [
-                ("u_key", U_ctx_key_fd),
-                ("u_value1", U_ctx_value1)
-        ]
-
-class fd_t (Structure):
-        _fields_ = [
-                ("pid", c_uint64),
-                ("flags", c_int32),
-                ("refcount", c_int32),
-                ("inode_list", list_head),
-                ("inode", POINTER(inode_t)),
-                ("lock", c_void_p),
-                ("ctx", POINTER(fd_ctx)),
-                ("xl_count", c_int),
-                ("lk_ctx", POINTER(fd_lk_ctx)),
-                ("anonymous", c_uint)
-        ]
-
-class loc_t (Structure):
-        _fields_ = [
-                ("path", c_char_p),
-                ("name", c_char_p),
-                ("inode", POINTER(inode_t)),
-                ("parent", POINTER(inode_t)),
-                ("gfid", c_ubyte * 16),
-                ("pargfid", c_ubyte * 16),
-        ]
-
-
-
-def _init_op (a_class, fop, cbk, wind, unwind):
-        # Decorators, used by translators. We could pass the signatures as
-        # parameters, but it's actually kind of nice to keep them around for
-        # inspection.
-        a_class.fop_type = apply(CFUNCTYPE,a_class.fop_sig)
-        a_class.cbk_type = apply(CFUNCTYPE,a_class.cbk_sig)
-        # Dispatch-function registration.
-        fop.restype = None
-        fop.argtypes = [ c_long, a_class.fop_type ]
-        # Callback-function registration.
-        cbk.restype = None
-        cbk.argtypes = [ c_long, a_class.cbk_type ]
-        # STACK_WIND function.
-        wind.restype = None
-        wind.argtypes = list(a_class.fop_sig[1:])
-        # STACK_UNWIND function.
-        unwind.restype = None
-        unwind.argtypes = list(a_class.cbk_sig[1:])
-
-class OpLookup:
-        fop_sig = (c_int, POINTER(call_frame_t), POINTER(xlator_t),
-                   POINTER(loc_t), POINTER(dict_t))
-        cbk_sig = (c_int, POINTER(call_frame_t), c_long, POINTER(xlator_t),
-                   c_int, c_int, POINTER(inode_t), POINTER(iatt_t),
-                   POINTER(dict_t), POINTER(iatt_t))
-_init_op (OpLookup, dl.set_lookup_fop, dl.set_lookup_cbk,
-                    dl.wind_lookup,    dl.unwind_lookup)
-
-class OpCreate:
-        fop_sig = (c_int, POINTER(call_frame_t), POINTER(xlator_t),
-                   POINTER(loc_t), c_int, c_uint, c_uint, POINTER(fd_t),
-                   POINTER(dict_t))
-        cbk_sig = (c_int, POINTER(call_frame_t), c_long, POINTER(xlator_t),
-                   c_int, c_int, POINTER(fd_t), POINTER(inode_t),
-                   POINTER(iatt_t), POINTER(iatt_t), POINTER(iatt_t),
-                   POINTER(dict_t))
-_init_op (OpCreate, dl.set_create_fop, dl.set_create_cbk,
-                    dl.wind_create,    dl.unwind_create)
-
-class OpOpen:
-        fop_sig = (c_int, POINTER(call_frame_t), POINTER(xlator_t),
-                   POINTER(loc_t), c_int, POINTER(fd_t), POINTER(dict_t))
-        cbk_sig = (c_int, POINTER(call_frame_t), c_long, POINTER(xlator_t),
-                   c_int, c_int, POINTER(fd_t), POINTER(dict_t))
-_init_op (OpOpen, dl.set_open_fop, dl.set_open_cbk,
-                  dl.wind_open,    dl.unwind_open)
-
-class OpReadv:
-        fop_sig = (c_int, POINTER(call_frame_t), POINTER(xlator_t),
-                   POINTER(fd_t), c_size_t, c_long, c_uint32, POINTER(dict_t))
-        cbk_sig = (c_int, POINTER(call_frame_t), c_long, POINTER(xlator_t),
-                   c_int, c_int, POINTER(iovec_t), c_int, POINTER(iatt_t),
-                   POINTER(iobref_t), POINTER(dict_t))
-_init_op (OpReadv, dl.set_readv_fop, dl.set_readv_cbk,
-                   dl.wind_readv,    dl.unwind_readv)
-class OpWritev:
-        fop_sig = (c_int, POINTER(call_frame_t), POINTER(xlator_t),
-                   POINTER(fd_t), POINTER(iovec_t), c_int, c_long, c_uint32,
-                   POINTER(iobref_t), POINTER(dict_t))
-        cbk_sig = (c_int, POINTER(call_frame_t), c_long, POINTER(xlator_t),
-                   c_int, c_int, POINTER(iatt_t), POINTER(iatt_t),
-                   POINTER(dict_t))
-_init_op (OpWritev, dl.set_writev_fop, dl.set_writev_cbk,
-                    dl.wind_writev,    dl.unwind_writev)
-
-class OpOpendir:
-        fop_sig = (c_int, POINTER(call_frame_t), POINTER(xlator_t),
-                   POINTER(loc_t), POINTER(fd_t) ,POINTER(dict_t))
-        cbk_sig = (c_int, POINTER(call_frame_t), c_long, POINTER(xlator_t),
-                   c_int, c_int, POINTER(fd_t), POINTER(dict_t))
-_init_op (OpOpendir, dl.set_opendir_fop, dl.set_opendir_cbk,
-                     dl.wind_opendir,    dl.unwind_opendir)
-
-class OpReaddir:
-        fop_sig = (c_int, POINTER(call_frame_t), POINTER(xlator_t),
-                   POINTER(fd_t), c_size_t, c_long, POINTER(dict_t))
-        cbk_sig = (c_int, POINTER(call_frame_t), c_long, POINTER(xlator_t),
-                   c_int, c_int, POINTER(gf_dirent_t), POINTER(dict_t))
-_init_op (OpReaddir, dl.set_readdir_fop, dl.set_readdir_cbk,
-                     dl.wind_readdir,    dl.unwind_readdir)
-
-class OpReaddirp:
-        fop_sig = (c_int, POINTER(call_frame_t), POINTER(xlator_t),
-                   POINTER(fd_t), c_size_t, c_long, POINTER(dict_t))
-        cbk_sig = (c_int, POINTER(call_frame_t), c_long, POINTER(xlator_t),
-                   c_int, c_int, POINTER(gf_dirent_t), POINTER(dict_t))
-_init_op (OpReaddirp, dl.set_readdirp_fop, dl.set_readdirp_cbk,
-                      dl.wind_readdirp,    dl.unwind_readdirp)
-
-class OpStat:
-        fop_sig = (c_int, POINTER(call_frame_t), POINTER(xlator_t),
-                   POINTER(loc_t), POINTER(dict_t))
-        cbk_sig = (c_int, POINTER(call_frame_t), c_long, POINTER(xlator_t),
-                   c_int, c_int, POINTER(iatt_t), POINTER(dict_t))
-_init_op (OpStat, dl.set_stat_fop, dl.set_stat_cbk,
-                  dl.wind_stat,    dl.unwind_stat)
-
-class OpFstat:
-        fop_sig = (c_int, POINTER(call_frame_t), POINTER(xlator_t),
-                   POINTER(fd_t), POINTER(dict_t))
-        cbk_sig = (c_int, POINTER(call_frame_t), c_long, POINTER(xlator_t),
-                   c_int, c_int, POINTER(iatt_t), POINTER(dict_t))
-_init_op (OpFstat, dl.set_fstat_fop, dl.set_fstat_cbk,
-                   dl.wind_fstat,    dl.unwind_fstat)
-
-class OpStatfs:
-        fop_sig = (c_int, POINTER(call_frame_t), POINTER(xlator_t),
-                   POINTER(loc_t), POINTER(dict_t))
-        cbk_sig = (c_int, POINTER(call_frame_t), c_long, POINTER(xlator_t),
-                   c_int, c_int, POINTER(statvfs_t), POINTER(dict_t))
-_init_op (OpStatfs, dl.set_statfs_fop, dl.set_statfs_cbk,
-                    dl.wind_statfs,    dl.unwind_statfs)
-
-
-class OpSetxattr:
-        fop_sig = (c_int, POINTER(call_frame_t), POINTER(xlator_t),
-                   POINTER(loc_t), POINTER(dict_t), c_int32,
-                   POINTER (dict_t))
-        cbk_sig = (c_int, POINTER(call_frame_t), c_long, POINTER(xlator_t),
-                   c_int, c_int, POINTER(dict_t))
-_init_op (OpSetxattr, dl.set_setxattr_fop, dl.set_setxattr_cbk,
-                      dl.wind_setxattr,    dl.unwind_setxattr)
-
-class OpGetxattr:
-        fop_sig = (c_int, POINTER(call_frame_t), POINTER(xlator_t),
-                   POINTER(loc_t), c_char_p, POINTER (dict_t))
-        cbk_sig = (c_int, POINTER(call_frame_t), c_long, POINTER(xlator_t),
-                   c_int, c_int, POINTER(dict_t), POINTER(dict_t))
-_init_op (OpGetxattr, dl.set_getxattr_fop, dl.set_getxattr_cbk,
-                      dl.wind_getxattr,    dl.unwind_getxattr)
-
-class OpFsetxattr:
-        fop_sig = (c_int, POINTER(call_frame_t), POINTER(xlator_t),
-                   POINTER(fd_t), POINTER(dict_t), c_int32,
-                   POINTER (dict_t))
-        cbk_sig = (c_int, POINTER(call_frame_t), c_long, POINTER(xlator_t),
-                   c_int, c_int, POINTER(dict_t))
-_init_op (OpFsetxattr, dl.set_fsetxattr_fop, dl.set_fsetxattr_cbk,
-                       dl.wind_fsetxattr,    dl.unwind_fsetxattr)
-
-class OpFgetxattr:
-        fop_sig = (c_int, POINTER(call_frame_t), POINTER(xlator_t),
-                   POINTER(fd_t), c_char_p, POINTER (dict_t))
-        cbk_sig = (c_int, POINTER(call_frame_t), c_long, POINTER(xlator_t),
-                   c_int, c_int, POINTER(dict_t), POINTER(dict_t))
-_init_op (OpFgetxattr, dl.set_fgetxattr_fop, dl.set_fgetxattr_cbk,
-                       dl.wind_fgetxattr,    dl.unwind_fgetxattr)
-
-class OpRemovexattr:
-        fop_sig = (c_int, POINTER(call_frame_t), POINTER(xlator_t),
-                   POINTER(loc_t), c_char_p, POINTER(dict_t))
-        cbk_sig = (c_int, POINTER(call_frame_t), c_long, POINTER(xlator_t),
-                   c_int, c_int, POINTER(dict_t))
-_init_op (OpRemovexattr, dl.set_removexattr_fop, dl.set_removexattr_cbk,
-                         dl.wind_removexattr,    dl.unwind_removexattr)
-
-
-class OpFremovexattr:
-        fop_sig = (c_int, POINTER(call_frame_t), POINTER(xlator_t),
-                   POINTER(fd_t), c_char_p, POINTER(dict_t))
-        cbk_sig = (c_int, POINTER(call_frame_t), c_long, POINTER(xlator_t),
-                   c_int, c_int, POINTER(dict_t))
-_init_op (OpFremovexattr, dl.set_fremovexattr_fop, dl.set_fremovexattr_cbk,
-                          dl.wind_fremovexattr,    dl.unwind_fremovexattr)
-
-class OpLink:
-        fop_sig = (c_int, POINTER(call_frame_t), POINTER(xlator_t),
-                   POINTER(loc_t), POINTER(loc_t), POINTER(dict_t))
-        cbk_sig = (c_int, POINTER(call_frame_t), c_long, POINTER(xlator_t),
-                   c_int, c_int, POINTER(inode_t), POINTER(iatt_t),
-                   POINTER(iatt_t), POINTER(iatt_t), POINTER(dict_t))
-_init_op (OpLink, dl.set_link_fop, dl.set_link_cbk,
-                  dl.wind_link,    dl.unwind_link)
-
-class OpSymlink:
-        fop_sig = (c_int, POINTER(call_frame_t), POINTER(xlator_t),
-                   c_char_p, POINTER(loc_t), c_uint, POINTER(dict_t))
-        cbk_sig = (c_int, POINTER(call_frame_t), c_long, POINTER(xlator_t),
-                   c_int, c_int, POINTER(inode_t), POINTER(iatt_t),
-                   POINTER(iatt_t), POINTER(iatt_t), POINTER(dict_t))
-_init_op (OpSymlink, dl.set_symlink_fop, dl.set_symlink_cbk,
-                     dl.wind_symlink,    dl.unwind_symlink)
-
-class OpUnlink:
-        fop_sig = (c_int, POINTER(call_frame_t), POINTER(xlator_t),
-                   POINTER(loc_t), c_int, POINTER(dict_t))
-        cbk_sig = (c_int, POINTER(call_frame_t), c_long, POINTER(xlator_t),
-                   c_int, c_int, POINTER(iatt_t), POINTER(iatt_t),
-                   POINTER(dict_t))
-_init_op (OpUnlink, dl.set_unlink_fop, dl.set_unlink_cbk,
-                    dl.wind_unlink,    dl.unwind_unlink)
-
-class OpReadlink:
-        fop_sig = (c_int, POINTER(call_frame_t), POINTER(xlator_t),
-                   POINTER(loc_t), c_size_t, POINTER(dict_t))
-        cbk_sig = (c_int, POINTER(call_frame_t), c_long, POINTER(xlator_t),
-                   c_int, c_int, c_char_p, POINTER(iatt_t), POINTER(dict_t))
-_init_op (OpReadlink, dl.set_readlink_fop, dl.set_readlink_cbk,
-                      dl.wind_readlink,    dl.unwind_readlink)
-
-class OpMkdir:
-        fop_sig = (c_int, POINTER(call_frame_t), POINTER(xlator_t),
-                   POINTER(loc_t), c_uint, c_uint, POINTER(dict_t))
-        cbk_sig = (c_int, POINTER(call_frame_t), c_long, POINTER(xlator_t),
-                   c_int, c_int, POINTER(inode_t), POINTER(iatt_t),
-                   POINTER(iatt_t), POINTER(iatt_t), POINTER(dict_t))
-_init_op (OpMkdir, dl.set_mkdir_fop, dl.set_mkdir_cbk,
-                   dl.wind_mkdir,    dl.unwind_mkdir)
-
-class OpRmdir:
-        fop_sig = (c_int, POINTER(call_frame_t), POINTER(xlator_t),
-                   POINTER(loc_t), c_int, POINTER(dict_t))
-        cbk_sig = (c_int, POINTER(call_frame_t), c_long, POINTER(xlator_t),
-                   c_int, c_int, POINTER(iatt_t), POINTER(iatt_t),
-                   POINTER(dict_t))
-_init_op (OpRmdir, dl.set_rmdir_fop, dl.set_rmdir_cbk,
-                   dl.wind_rmdir,    dl.unwind_rmdir)
-
-
-class Translator:
-        def __init__ (self, c_this):
-                # This is only here to keep references to the stubs we create,
-                # because ctypes doesn't and glupy.so can't because it doesn't
-                # get a pointer to the actual Python object. It's a dictionary
-                # instead of a list in case we ever allow changing fops/cbks
-                # after initialization and need to look them up.
-                self.stub_refs = {}
-                funcs = dir(self.__class__)
-                if "lookup_fop" in funcs:
-                        @OpLookup.fop_type
-                        def stub (frame, this, loc, xdata, s=self):
-                                return s.lookup_fop (frame, this, loc, xdata)
-                        self.stub_refs["lookup_fop"] = stub
-                        dl.set_lookup_fop(c_this,stub)
-                if "lookup_cbk" in funcs:
-                        @OpLookup.cbk_type
-                        def stub (frame, cookie, this, op_ret, op_errno, inode,
-                                  buf, xdata, postparent, s=self):
-                                return s.lookup_cbk(frame, cookie, this, op_ret,
-                                                    op_errno, inode, buf, xdata,
-                                                    postparent)
-                        self.stub_refs["lookup_cbk"] = stub
-                        dl.set_lookup_cbk(c_this,stub)
-                if "create_fop" in funcs:
-                        @OpCreate.fop_type
-                        def stub (frame, this, loc, flags, mode, umask, fd,
-                                  xdata, s=self):
-                                return s.create_fop (frame, this, loc, flags,
-                                                     mode, umask, fd, xdata)
-                        self.stub_refs["create_fop"] = stub
-                        dl.set_create_fop(c_this,stub)
-                if "create_cbk" in funcs:
-                        @OpCreate.cbk_type
-                        def stub (frame, cookie, this, op_ret, op_errno, fd,
-                                  inode, buf, preparent, postparent, xdata,
-                                  s=self):
-                                return s.create_cbk (frame, cookie, this,
-                                                     op_ret, op_errno, fd,
-                                                     inode, buf, preparent,
-                                                     postparent, xdata)
-                        self.stub_refs["create_cbk"] = stub
-                        dl.set_create_cbk(c_this,stub)
-                if "open_fop" in funcs:
-                        @OpOpen.fop_type
-                        def stub (frame, this, loc, flags, fd,
-                                  xdata, s=self):
-                                return s.open_fop (frame, this, loc, flags,
-                                                   fd, xdata)
-                        self.stub_refs["open_fop"] = stub
-                        dl.set_open_fop(c_this,stub)
-                if "open_cbk" in funcs:
-                        @OpOpen.cbk_type
-                        def stub (frame, cookie, this, op_ret, op_errno, fd,
-                                  xdata, s=self):
-                                return s.open_cbk (frame, cookie, this,
-                                                   op_ret, op_errno, fd,
-                                                   xdata)
-                        self.stub_refs["open_cbk"] = stub
-                        dl.set_open_cbk(c_this,stub)
-                if "readv_fop" in funcs:
-                        @OpReadv.fop_type
-                        def stub (frame, this, fd, size, offset, flags,
-                                  xdata, s=self):
-                                return s.readv_fop (frame, this, fd, size,
-                                                    offset, flags, xdata)
-                        self.stub_refs["readv_fop"] = stub
-                        dl.set_readv_fop(c_this,stub)
-                if "readv_cbk" in funcs:
-                        @OpReadv.cbk_type
-                        def stub (frame, cookie, this, op_ret, op_errno,
-                                  vector, count, stbuf, iobref, xdata,
-                                  s=self):
-                                return s.readv_cbk (frame, cookie, this,
-                                                    op_ret, op_errno, vector,
-                                                    count, stbuf, iobref,
-                                                    xdata)
-                        self.stub_refs["readv_cbk"] = stub
-                        dl.set_readv_cbk(c_this,stub)
-                if "writev_fop" in funcs:
-                        @OpWritev.fop_type
-                        def stub (frame, this, fd, vector, count,
-                                  offset, flags, iobref, xdata, s=self):
-                                return s.writev_fop (frame, this, fd, vector,
-                                                     count, offset, flags,
-                                                     iobref, xdata)
-                        self.stub_refs["writev_fop"] = stub
-                        dl.set_writev_fop(c_this,stub)
-                if "writev_cbk" in funcs:
-                        @OpWritev.cbk_type
-                        def stub (frame, cookie, this, op_ret, op_errno,
-                                  prebuf, postbuf, xdata, s=self):
-                                return s.writev_cbk (frame, cookie, this,
-                                                     op_ret, op_errno, prebuf,
-                                                     postbuf, xdata)
-                        self.stub_refs["writev_cbk"] = stub
-                        dl.set_writev_cbk(c_this,stub)
-                if "opendir_fop" in funcs:
-                        @OpOpendir.fop_type
-                        def stub (frame, this, loc, fd, xdata, s=self):
-                                return s.opendir_fop (frame, this, loc, fd,
-                                                      xdata)
-                        self.stub_refs["opendir_fop"] = stub
-                        dl.set_opendir_fop(c_this,stub)
-                if "opendir_cbk" in funcs:
-                        @OpOpendir.cbk_type
-                        def stub (frame, cookie, this, op_ret, op_errno, fd,
-                                  xdata, s=self):
-                                return s.opendir_cbk(frame, cookie, this,
-                                                     op_ret, op_errno, fd,
-                                                     xdata)
-                        self.stub_refs["opendir_cbk"] = stub
-                        dl.set_opendir_cbk(c_this,stub)
-                if "readdir_fop" in funcs:
-                        @OpReaddir.fop_type
-                        def stub (frame, this, fd, size, offset, xdata, s=self):
-                                return s.readdir_fop (frame, this, fd, size,
-                                                      offset, xdata)
-                        self.stub_refs["readdir_fop"] = stub
-                        dl.set_readdir_fop(c_this,stub)
-                if "readdir_cbk" in funcs:
-                        @OpReaddir.cbk_type
-                        def stub (frame, cookie, this, op_ret, op_errno,
-                                  entries, xdata, s=self):
-                                return s.readdir_cbk(frame, cookie, this,
-                                                     op_ret, op_errno, entries,
-                                                     xdata)
-                        self.stub_refs["readdir_cbk"] = stub
-                        dl.set_readdir_cbk(c_this,stub)
-                if "readdirp_fop" in funcs:
-                        @OpReaddirp.fop_type
-                        def stub (frame, this, fd, size, offset, xdata, s=self):
-                                return s.readdirp_fop (frame, this, fd, size,
-                                                       offset, xdata)
-                        self.stub_refs["readdirp_fop"] = stub
-                        dl.set_readdirp_fop(c_this,stub)
-                if "readdirp_cbk" in funcs:
-                        @OpReaddirp.cbk_type
-                        def stub (frame, cookie, this, op_ret, op_errno,
-                                  entries, xdata, s=self):
-                                return s.readdirp_cbk (frame, cookie, this,
-                                                       op_ret, op_errno,
-                                                       entries, xdata)
-                        self.stub_refs["readdirp_cbk"] = stub
-                        dl.set_readdirp_cbk(c_this,stub)
-                if "stat_fop" in funcs:
-                        @OpStat.fop_type
-                        def stub (frame, this, loc, xdata, s=self):
-                                return s.stat_fop (frame, this, loc, xdata)
-                        self.stub_refs["stat_fop"] = stub
-                        dl.set_stat_fop(c_this,stub)
-                if "stat_cbk" in funcs:
-                        @OpStat.cbk_type
-                        def stub (frame, cookie, this, op_ret, op_errno, buf,
-                                  xdata, s=self):
-                                return s.stat_cbk(frame, cookie, this, op_ret,
-                                                  op_errno, buf, xdata)
-                        self.stub_refs["stat_cbk"] = stub
-                        dl.set_stat_cbk(c_this,stub)
-                if "fstat_fop" in funcs:
-                        @OpFstat.fop_type
-                        def stub (frame, this, fd, xdata, s=self):
-                                return s.fstat_fop (frame, this, fd, xdata)
-                        self.stub_refs["fstat_fop"] = stub
-                        dl.set_fstat_fop(c_this,stub)
-                if "fstat_cbk" in funcs:
-                        @OpFstat.cbk_type
-                        def stub (frame, cookie, this, op_ret, op_errno, buf,
-                                  xdata, s=self):
-                                return s.fstat_cbk(frame, cookie, this, op_ret,
-                                                   op_errno, buf, xdata)
-                        self.stub_refs["fstat_cbk"] = stub
-                        dl.set_fstat_cbk(c_this,stub)
-                if "statfs_fop" in funcs:
-                        @OpStatfs.fop_type
-                        def stub (frame, this, loc, xdata, s=self):
-                                return s.statfs_fop (frame, this, loc, xdata)
-                        self.stub_refs["statfs_fop"] = stub
-                        dl.set_statfs_fop(c_this,stub)
-                if "statfs_cbk" in funcs:
-                        @OpStatfs.cbk_type
-                        def stub (frame, cookie, this, op_ret, op_errno, buf,
-                                  xdata, s=self):
-                                return s.statfs_cbk (frame, cookie, this,
-                                                     op_ret, op_errno, buf,
-                                                     xdata)
-                        self.stub_refs["statfs_cbk"] = stub
-                        dl.set_statfs_cbk(c_this,stub)
-                if "setxattr_fop" in funcs:
-                        @OpSetxattr.fop_type
-                        def stub (frame, this, loc, dictionary, flags, xdata,
-                                  s=self):
-                                return s.setxattr_fop (frame, this, loc,
-                                                       dictionary, flags,
-                                                       xdata)
-                        self.stub_refs["setxattr_fop"] = stub
-                        dl.set_setxattr_fop(c_this,stub)
-                if "setxattr_cbk" in funcs:
-                        @OpSetxattr.cbk_type
-                        def stub (frame, cookie, this, op_ret, op_errno, xdata,
-                                  s=self):
-                                return s.setxattr_cbk(frame, cookie, this,
-                                                      op_ret, op_errno, xdata)
-                        self.stub_refs["setxattr_cbk"] = stub
-                        dl.set_setxattr_cbk(c_this,stub)
-                if "getxattr_fop" in funcs:
-                        @OpGetxattr.fop_type
-                        def stub (frame, this, loc, name, xdata, s=self):
-                                return s.getxattr_fop (frame, this, loc, name,
-                                                       xdata)
-                        self.stub_refs["getxattr_fop"] = stub
-                        dl.set_getxattr_fop(c_this,stub)
-                if "getxattr_cbk" in funcs:
-                        @OpGetxattr.cbk_type
-                        def stub (frame, cookie, this, op_ret, op_errno,
-                                  dictionary, xdata, s=self):
-                                return s.getxattr_cbk(frame, cookie, this,
-                                                      op_ret, op_errno,
-                                                      dictionary, xdata)
-                        self.stub_refs["getxattr_cbk"] = stub
-                        dl.set_getxattr_cbk(c_this,stub)
-                if "fsetxattr_fop" in funcs:
-                        @OpFsetxattr.fop_type
-                        def stub (frame, this, fd, dictionary, flags, xdata,
-                                  s=self):
-                                return s.fsetxattr_fop (frame, this, fd,
-                                                        dictionary, flags,
-                                                        xdata)
-                        self.stub_refs["fsetxattr_fop"] = stub
-                        dl.set_fsetxattr_fop(c_this,stub)
-                if "fsetxattr_cbk" in funcs:
-                        @OpFsetxattr.cbk_type
-                        def stub (frame, cookie, this, op_ret, op_errno, xdata,
-                                  s=self):
-                                return s.fsetxattr_cbk(frame, cookie, this,
-                                                       op_ret, op_errno, xdata)
-                        self.stub_refs["fsetxattr_cbk"] = stub
-                        dl.set_fsetxattr_cbk(c_this,stub)
-                if "fgetxattr_fop" in funcs:
-                        @OpFgetxattr.fop_type
-                        def stub (frame, this, fd, name, xdata, s=self):
-                                return s.fgetxattr_fop (frame, this, fd, name,
-                                                        xdata)
-                        self.stub_refs["fgetxattr_fop"] = stub
-                        dl.set_fgetxattr_fop(c_this,stub)
-                if "fgetxattr_cbk" in funcs:
-                        @OpFgetxattr.cbk_type
-                        def stub (frame, cookie, this, op_ret, op_errno,
-                                  dictionary, xdata, s=self):
-                                return s.fgetxattr_cbk(frame, cookie, this,
-                                                       op_ret, op_errno,
-                                                       dictionary, xdata)
-                        self.stub_refs["fgetxattr_cbk"] = stub
-                        dl.set_fgetxattr_cbk(c_this,stub)
-                if "removexattr_fop" in funcs:
-                        @OpRemovexattr.fop_type
-                        def stub (frame, this, loc, name, xdata, s=self):
-                                return s.removexattr_fop (frame, this, loc,
-                                                          name, xdata)
-                        self.stub_refs["removexattr_fop"] = stub
-                        dl.set_removexattr_fop(c_this,stub)
-                if "removexattr_cbk" in funcs:
-                        @OpRemovexattr.cbk_type
-                        def stub (frame, cookie, this, op_ret, op_errno,
-                                  xdata, s=self):
-                                return s.removexattr_cbk(frame, cookie, this,
-                                                         op_ret, op_errno,
-                                                         xdata)
-                        self.stub_refs["removexattr_cbk"] = stub
-                        dl.set_removexattr_cbk(c_this,stub)
-                if "fremovexattr_fop" in funcs:
-                        @OpFremovexattr.fop_type
-                        def stub (frame, this, fd, name, xdata, s=self):
-                                return s.fremovexattr_fop (frame, this, fd,
-                                                           name, xdata)
-                        self.stub_refs["fremovexattr_fop"] = stub
-                        dl.set_fremovexattr_fop(c_this,stub)
-                if "fremovexattr_cbk" in funcs:
-                        @OpFremovexattr.cbk_type
-                        def stub (frame, cookie, this, op_ret, op_errno,
-                                  xdata, s=self):
-                                return s.fremovexattr_cbk(frame, cookie, this,
-                                                          op_ret, op_errno,
-                                                          xdata)
-                        self.stub_refs["fremovexattr_cbk"] = stub
-                        dl.set_fremovexattr_cbk(c_this,stub)
-                if "link_fop" in funcs:
-                        @OpLink.fop_type
-                        def stub (frame, this, oldloc, newloc,
-                                  xdata, s=self):
-                                return s.link_fop (frame, this, oldloc,
-                                                   newloc, xdata)
-                        self.stub_refs["link_fop"] = stub
-                        dl.set_link_fop(c_this,stub)
-                if "link_cbk" in funcs:
-                        @OpLink.cbk_type
-                        def stub (frame, cookie, this, op_ret, op_errno,
-                                  inode, buf, preparent, postparent, xdata,
-                                  s=self):
-                                return s.link_cbk (frame, cookie, this,
-                                                   op_ret, op_errno, inode,
-                                                   buf, preparent,
-                                                   postparent, xdata)
-                        self.stub_refs["link_cbk"] = stub
-                        dl.set_link_cbk(c_this,stub)
-                if "symlink_fop" in funcs:
-                        @OpSymlink.fop_type
-                        def stub (frame, this, linkname, loc,
-                                  umask, xdata, s=self):
-                                return s.symlink_fop (frame, this, linkname,
-                                                      loc, umask, xdata)
-                        self.stub_refs["symlink_fop"] = stub
-                        dl.set_symlink_fop(c_this,stub)
-                if "symlink_cbk" in funcs:
-                        @OpSymlink.cbk_type
-                        def stub (frame, cookie, this, op_ret, op_errno,
-                                  inode, buf, preparent, postparent, xdata,
-                                  s=self):
-                                return s.symlink_cbk (frame, cookie, this,
-                                                      op_ret, op_errno, inode,
-                                                      buf, preparent,
-                                                      postparent, xdata)
-                        self.stub_refs["symlink_cbk"] = stub
-                        dl.set_symlink_cbk(c_this,stub)
-                if "unlink_fop" in funcs:
-                        @OpUnlink.fop_type
-                        def stub (frame, this, loc, xflags,
-                                  xdata, s=self):
-                                return s.unlink_fop (frame, this, loc,
-                                                     xflags, xdata)
-                        self.stub_refs["unlink_fop"] = stub
-                        dl.set_unlink_fop(c_this,stub)
-                if "unlink_cbk" in funcs:
-                        @OpUnlink.cbk_type
-                        def stub (frame, cookie, this, op_ret, op_errno,
-                                  preparent, postparent, xdata, s=self):
-                                return s.unlink_cbk (frame, cookie, this,
-                                                     op_ret, op_errno,
-                                                     preparent, postparent,
-                                                     xdata)
-                        self.stub_refs["unlink_cbk"] = stub
-                        dl.set_unlink_cbk(c_this,stub)
-                if "readlink_fop" in funcs:
-                        @OpReadlink.fop_type
-                        def stub (frame, this, loc, size,
-                                  xdata, s=self):
-                                return s.readlink_fop (frame, this, loc,
-                                                       size, xdata)
-                        self.stub_refs["readlink_fop"] = stub
-                        dl.set_readlink_fop(c_this,stub)
-                if "readlink_cbk" in funcs:
-                        @OpReadlink.cbk_type
-                        def stub (frame, cookie, this, op_ret, op_errno,
-                                  path, buf, xdata, s=self):
-                                return s.readlink_cbk (frame, cookie, this,
-                                                       op_ret, op_errno,
-                                                       path, buf, xdata)
-                        self.stub_refs["readlink_cbk"] = stub
-                        dl.set_readlink_cbk(c_this,stub)
-                if "mkdir_fop" in funcs:
-                        @OpMkdir.fop_type
-                        def stub (frame, this, loc, mode, umask, xdata,
-                                  s=self):
-                                return s.mkdir_fop (frame, this, loc, mode,
-                                                    umask, xdata)
-                        self.stub_refs["mkdir_fop"] = stub
-                        dl.set_mkdir_fop(c_this,stub)
-                if "mkdir_cbk" in funcs:
-                        @OpMkdir.cbk_type
-                        def stub (frame, cookie, this, op_ret, op_errno, inode,
-                                  buf, preparent, postparent, xdata, s=self):
-                                return s.mkdir_cbk (frame, cookie, this,
-                                                    op_ret, op_errno, inode,
-                                                    buf, preparent,
-                                                    postparent, xdata)
-                        self.stub_refs["mkdir_cbk"] = stub
-                        dl.set_mkdir_cbk(c_this,stub)
-                if "rmdir_fop" in funcs:
-                        @OpRmdir.fop_type
-                        def stub (frame, this, loc, xflags,
-                                  xdata, s=self):
-                                return s.rmdir_fop (frame, this, loc,
-                                                    xflags, xdata)
-                        self.stub_refs["rmdir_fop"] = stub
-                        dl.set_rmdir_fop(c_this,stub)
-                if "rmdir_cbk" in funcs:
-                        @OpRmdir.cbk_type
-                        def stub (frame, cookie, this, op_ret, op_errno,
-                                  preparent, postparent, xdata, s=self):
-                                return s.rmdir_cbk (frame, cookie, this,
-                                                    op_ret, op_errno,
-                                                    preparent, postparent,
-                                                    xdata)
-                        self.stub_refs["rmdir_cbk"] = stub
-                        dl.set_rmdir_cbk(c_this,stub)
diff --git a/xlators/features/glupy/src/setup.py.in b/xlators/features/glupy/src/setup.py.in
deleted file mode 100644
index 611e9695f76..00000000000
--- a/xlators/features/glupy/src/setup.py.in
+++ /dev/null
@@ -1,24 +0,0 @@
-from distutils.core import setup
-
-DESC = """GlusterFS is a distributed file-system capable of scaling to
-several petabytes. It aggregates various storage bricks over Infiniband
-RDMA or TCP/IP interconnect into one large parallel network file system.
-GlusterFS is one of the most sophisticated file systems in terms of
-features and extensibility.  It borrows a powerful concept called
-Translators from GNU Hurd kernel. Much of the code in GlusterFS is in
-user space and easily manageable.
-
-This package contains Glupy, the Python translator interface for GlusterFS."""
-
-setup(
-    name='glusterfs-glupy',
-    version='@PACKAGE_VERSION@',
-    description='Glupy is the Python translator interface for GlusterFS',
-    long_description=DESC,
-    author='Gluster Community',
-    author_email='gluster-devel@gluster.org',
-    license='LGPLv3',
-    url='http://gluster.org/',
-    package_dir={'gluster':''},
-    packages=['gluster']
-)
diff --git a/xlators/features/index/src/Makefile.am b/xlators/features/index/src/Makefile.am
index 73bb8972e70..c71c238c163 100644
--- a/xlators/features/index/src/Makefile.am
+++ b/xlators/features/index/src/Makefile.am
@@ -1,15 +1,17 @@
+if WITH_SERVER
 xlator_LTLIBRARIES = index.la
+endif
 xlatordir = $(libdir)/glusterfs/$(PACKAGE_VERSION)/xlator/features
 
-index_la_LDFLAGS = -module -avoid-version
+index_la_LDFLAGS = -module $(GF_XLATOR_DEFAULT_LDFLAGS)
 
 index_la_SOURCES = index.c
 index_la_LIBADD = $(top_builddir)/libglusterfs/src/libglusterfs.la
 
-noinst_HEADERS = index.h index-mem-types.h
+noinst_HEADERS = index.h index-mem-types.h index-messages.h
 
-AM_CPPFLAGS = $(GF_CPPFLAGS) \
-	-I$(top_srcdir)/libglusterfs/src -I$(top_srcdir)/rpc/xdr/src \
+AM_CPPFLAGS = $(GF_CPPFLAGS) -I$(top_srcdir)/libglusterfs/src \
+	-I$(top_srcdir)/rpc/xdr/src -I$(top_builddir)/rpc/xdr/src \
 	-I$(top_srcdir)/rpc/rpc-lib/src
 
 AM_CFLAGS = -Wall $(GF_CFLAGS)
diff --git a/xlators/features/index/src/index-mem-types.h b/xlators/features/index/src/index-mem-types.h
index 553d492dfbf..58833d0ec9b 100644
--- a/xlators/features/index/src/index-mem-types.h
+++ b/xlators/features/index/src/index-mem-types.h
@@ -8,15 +8,16 @@
    cases as published by the Free Software Foundation.
 */
 
-#ifndef __QUIESCE_MEM_TYPES_H__
-#define __QUIESCE_MEM_TYPES_H__
+#ifndef __INDEX_MEM_TYPES_H__
+#define __INDEX_MEM_TYPES_H__
 
-#include "mem-types.h"
+#include <glusterfs/mem-types.h>
 
 enum gf_index_mem_types_ {
-        gf_index_mt_priv_t = gf_common_mt_end + 1,
-        gf_index_inode_ctx_t = gf_common_mt_end + 2,
-        gf_index_fd_ctx_t = gf_common_mt_end + 3,
-        gf_index_mt_end
+    gf_index_mt_priv_t = gf_common_mt_end + 1,
+    gf_index_inode_ctx_t,
+    gf_index_fd_ctx_t,
+    gf_index_mt_local_t,
+    gf_index_mt_end
 };
 #endif
diff --git a/xlators/features/index/src/index-messages.h b/xlators/features/index/src/index-messages.h
new file mode 100644
index 00000000000..364f17cd34e
--- /dev/null
+++ b/xlators/features/index/src/index-messages.h
@@ -0,0 +1,33 @@
+/*
+ Copyright (c) 2016 Red Hat, Inc. <http://www.redhat.com>
+ This file is part of GlusterFS.
+
+ This file is licensed to you under your choice of the GNU Lesser
+ General Public License, version 3 or any later version (LGPLv3 or
+ later), or the GNU General Public License, version 2 (GPLv2), in all
+ cases as published by the Free Software Foundation.
+ */
+
+#ifndef _INDEX_MESSAGES_H_
+#define _INDEX_MESSAGES_H_
+
+#include <glusterfs/glfs-message-id.h>
+
+/* To add new message IDs, append new identifiers at the end of the list.
+ *
+ * Never remove a message ID. If it's not used anymore, you can rename it or
+ * leave it as it is, but not delete it. This is to prevent reutilization of
+ * IDs by other messages.
+ *
+ * The component name must match one of the entries defined in
+ * glfs-message-id.h.
+ */
+
+GLFS_MSGID(INDEX, INDEX_MSG_INDEX_DIR_CREATE_FAILED,
+           INDEX_MSG_INDEX_READDIR_FAILED, INDEX_MSG_INDEX_ADD_FAILED,
+           INDEX_MSG_INDEX_DEL_FAILED, INDEX_MSG_DICT_SET_FAILED,
+           INDEX_MSG_INODE_CTX_GET_SET_FAILED, INDEX_MSG_INVALID_ARGS,
+           INDEX_MSG_FD_OP_FAILED, INDEX_MSG_WORKER_THREAD_CREATE_FAILED,
+           INDEX_MSG_INVALID_GRAPH);
+
+#endif /* !_INDEX_MESSAGES_H_ */
diff --git a/xlators/features/index/src/index.c b/xlators/features/index/src/index.c
index 0c962bbc6f3..4abb2c73ce5 100644
--- a/xlators/features/index/src/index.c
+++ b/xlators/features/index/src/index.c
@@ -7,1447 +7,2676 @@
    later), or the GNU General Public License, version 2 (GPLv2), in all
    cases as published by the Free Software Foundation.
 */
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
-
 #include "index.h"
-#include "options.h"
+#include <glusterfs/options.h>
 #include "glusterfs3-xdr.h"
-#include "syscall.h"
+#include <glusterfs/syscall.h>
+#include <glusterfs/syncop.h>
+#include <glusterfs/common-utils.h>
+#include "index-messages.h"
+#include <ftw.h>
+#include <libgen.h> /* for dirname() */
+#include <signal.h>
 
 #define XATTROP_SUBDIR "xattrop"
+#define DIRTY_SUBDIR "dirty"
+#define ENTRY_CHANGES_SUBDIR "entry-changes"
 
-call_stub_t *
-__index_dequeue (struct list_head *callstubs)
+struct index_syncop_args {
+    inode_t *parent;
+    gf_dirent_t *entries;
+    char *path;
+};
+
+static char *index_vgfid_xattrs[XATTROP_TYPE_END] = {
+    [XATTROP] = GF_XATTROP_INDEX_GFID,
+    [DIRTY] = GF_XATTROP_DIRTY_GFID,
+    [ENTRY_CHANGES] = GF_XATTROP_ENTRY_CHANGES_GFID};
+
+static char *index_subdirs[XATTROP_TYPE_END] = {
+    [XATTROP] = XATTROP_SUBDIR,
+    [DIRTY] = DIRTY_SUBDIR,
+    [ENTRY_CHANGES] = ENTRY_CHANGES_SUBDIR};
+
+int
+index_get_type_from_vgfid(index_priv_t *priv, uuid_t vgfid)
 {
-        call_stub_t *stub = NULL;
+    int i = 0;
 
-        if (!list_empty (callstubs)) {
-                stub = list_entry (callstubs->next, call_stub_t, list);
-                list_del_init (&stub->list);
-        }
+    for (i = 0; i < XATTROP_TYPE_END; i++) {
+        if (gf_uuid_compare(priv->internal_vgfid[i], vgfid) == 0)
+            return i;
+    }
+    return -1;
+}
 
-        return stub;
+gf_boolean_t
+index_is_virtual_gfid(index_priv_t *priv, uuid_t vgfid)
+{
+    if (index_get_type_from_vgfid(priv, vgfid) < 0)
+        return _gf_false;
+    return _gf_true;
 }
 
-inline static void
-__index_enqueue (struct list_head *callstubs, call_stub_t *stub)
+static int
+__index_inode_ctx_get(inode_t *inode, xlator_t *this, index_inode_ctx_t **ctx)
 {
-        list_add_tail (&stub->list, callstubs);
+    int ret = 0;
+    index_inode_ctx_t *ictx = NULL;
+    uint64_t tmpctx = 0;
+
+    ret = __inode_ctx_get(inode, this, &tmpctx);
+    if (!ret) {
+        ictx = (index_inode_ctx_t *)(long)tmpctx;
+        goto out;
+    }
+    ictx = GF_CALLOC(1, sizeof(*ictx), gf_index_inode_ctx_t);
+    if (!ictx) {
+        ret = -1;
+        goto out;
+    }
+
+    INIT_LIST_HEAD(&ictx->callstubs);
+    ret = __inode_ctx_put(inode, this, (uint64_t)(uintptr_t)ictx);
+    if (ret) {
+        GF_FREE(ictx);
+        ictx = NULL;
+        goto out;
+    }
+out:
+    if (ictx)
+        *ctx = ictx;
+    return ret;
 }
 
-static void
-worker_enqueue (xlator_t *this, call_stub_t *stub)
+static int
+index_inode_ctx_get(inode_t *inode, xlator_t *this, index_inode_ctx_t **ctx)
 {
-        index_priv_t    *priv = NULL;
+    int ret = 0;
 
-        priv = this->private;
-        pthread_mutex_lock (&priv->mutex);
-        {
-                __index_enqueue (&priv->callstubs, stub);
-                pthread_cond_signal (&priv->cond);
-        }
-        pthread_mutex_unlock (&priv->mutex);
+    LOCK(&inode->lock);
+    {
+        ret = __index_inode_ctx_get(inode, this, ctx);
+    }
+    UNLOCK(&inode->lock);
+
+    return ret;
 }
 
-void *
-index_worker (void *data)
-{
-        index_priv_t     *priv = NULL;
-        xlator_t         *this = NULL;
-        call_stub_t      *stub = NULL;
-        int               ret = 0;
-
-        THIS = data;
-        this = data;
-        priv = this->private;
-
-        for (;;) {
-                pthread_mutex_lock (&priv->mutex);
-                {
-                        while (list_empty (&priv->callstubs)) {
-                                ret = pthread_cond_wait (&priv->cond,
-                                                         &priv->mutex);
-                        }
-
-                        stub = __index_dequeue (&priv->callstubs);
-                }
-                pthread_mutex_unlock (&priv->mutex);
+static gf_boolean_t
+index_is_subdir_of_entry_changes(xlator_t *this, inode_t *inode)
+{
+    index_inode_ctx_t *ctx = NULL;
+    int ret = 0;
 
-                if (stub) /* guard against spurious wakeups */
-                        call_resume (stub);
-        }
+    if (!inode)
+        return _gf_false;
 
-        return NULL;
+    ret = index_inode_ctx_get(inode, this, &ctx);
+    if ((ret == 0) && !gf_uuid_is_null(ctx->virtual_pargfid))
+        return _gf_true;
+    return _gf_false;
 }
-int
-__index_inode_ctx_get (inode_t *inode, xlator_t *this, index_inode_ctx_t **ctx)
+
+static int
+index_get_type_from_vgfid_xattr(const char *name)
 {
-        int               ret = 0;
-        index_inode_ctx_t *ictx = NULL;
-        uint64_t          tmpctx = 0;
+    int i = 0;
 
-        ret = __inode_ctx_get (inode, this, &tmpctx);
-        if (!ret) {
-                ictx = (index_inode_ctx_t*) (long) tmpctx;
-                goto out;
-        }
-        ictx = GF_CALLOC (1, sizeof (*ictx), gf_index_inode_ctx_t);
-        if (!ictx) {
-                ret = -1;
-                goto out;
-        }
+    for (i = 0; i < XATTROP_TYPE_END; i++) {
+        if (strcmp(name, index_vgfid_xattrs[i]) == 0)
+            return i;
+    }
+    return -1;
+}
 
-        INIT_LIST_HEAD (&ictx->callstubs);
-        ret = __inode_ctx_put (inode, this, (uint64_t)ictx);
-        if (ret) {
-                GF_FREE (ictx);
-                ictx = NULL;
-                goto out;
-        }
-out:
-        if (ictx)
-                *ctx = ictx;
-        return ret;
+gf_boolean_t
+index_is_fop_on_internal_inode(xlator_t *this, inode_t *inode, uuid_t gfid)
+{
+    index_priv_t *priv = this->private;
+    uuid_t vgfid = {0};
+
+    if (!inode)
+        return _gf_false;
+
+    if (gfid && !gf_uuid_is_null(gfid))
+        gf_uuid_copy(vgfid, gfid);
+    else
+        gf_uuid_copy(vgfid, inode->gfid);
+
+    if (index_is_virtual_gfid(priv, vgfid))
+        return _gf_true;
+    if (index_is_subdir_of_entry_changes(this, inode))
+        return _gf_true;
+    return _gf_false;
 }
 
-int
-index_inode_ctx_get (inode_t *inode, xlator_t *this, index_inode_ctx_t **ctx)
+static gf_boolean_t
+index_is_vgfid_xattr(const char *name)
 {
-        int               ret = 0;
+    if (index_get_type_from_vgfid_xattr(name) < 0)
+        return _gf_false;
+    return _gf_true;
+}
 
-        LOCK (&inode->lock);
-        {
-                ret = __index_inode_ctx_get (inode, this, ctx);
-        }
-        UNLOCK (&inode->lock);
+call_stub_t *
+__index_dequeue(struct list_head *callstubs)
+{
+    call_stub_t *stub = NULL;
+
+    if (!list_empty(callstubs)) {
+        stub = list_entry(callstubs->next, call_stub_t, list);
+        list_del_init(&stub->list);
+    }
 
-        return ret;
+    return stub;
 }
 
 static void
-make_index_dir_path (char *base, const char *subdir,
-                     char *index_dir, size_t len)
+__index_enqueue(struct list_head *callstubs, call_stub_t *stub)
 {
-        snprintf (index_dir, len, "%s/%s", base, subdir);
+    list_add_tail(&stub->list, callstubs);
 }
 
-int
-index_dir_create (xlator_t *this, const char *subdir)
-{
-        int          ret = 0;
-        struct       stat st = {0};
-        char         fullpath[PATH_MAX] = {0};
-        char         path[PATH_MAX] = {0};
-        char         *dir = NULL;
-        index_priv_t *priv = NULL;
-        size_t       len = 0;
-        size_t       pathlen = 0;
-
-        priv = this->private;
-        make_index_dir_path (priv->index_basepath, subdir, fullpath,
-                             sizeof (fullpath));
-        ret = stat (fullpath, &st);
-        if (!ret) {
-                if (!S_ISDIR (st.st_mode))
-                        ret = -2;
-                goto out;
+static void
+worker_enqueue(xlator_t *this, call_stub_t *stub)
+{
+    index_priv_t *priv = NULL;
+
+    priv = this->private;
+    pthread_mutex_lock(&priv->mutex);
+    {
+        __index_enqueue(&priv->callstubs, stub);
+        GF_ATOMIC_INC(priv->stub_cnt);
+        pthread_cond_signal(&priv->cond);
+    }
+    pthread_mutex_unlock(&priv->mutex);
+}
+
+void *
+index_worker(void *data)
+{
+    index_priv_t *priv = NULL;
+    xlator_t *this = NULL;
+    call_stub_t *stub = NULL;
+    gf_boolean_t bye = _gf_false;
+
+    THIS = data;
+    this = data;
+    priv = this->private;
+
+    for (;;) {
+        pthread_mutex_lock(&priv->mutex);
+        {
+            while (list_empty(&priv->callstubs)) {
+                if (priv->down) {
+                    bye = _gf_true; /*Avoid wait*/
+                    break;
+                }
+                (void)pthread_cond_wait(&priv->cond, &priv->mutex);
+                if (priv->down) {
+                    bye = _gf_true;
+                    break;
+                }
+            }
+            if (!bye)
+                stub = __index_dequeue(&priv->callstubs);
+            if (bye) {
+                priv->curr_count--;
+                if (priv->curr_count == 0)
+                    pthread_cond_broadcast(&priv->cond);
+            }
         }
+        pthread_mutex_unlock(&priv->mutex);
 
-        pathlen = strlen (fullpath);
-        if ((pathlen > 1) && fullpath[pathlen - 1] == '/')
-                fullpath[pathlen - 1] = '\0';
-        dir = strchr (fullpath, '/');
-        while (dir) {
-                dir = strchr (dir + 1, '/');
-                if (dir)
-                        len = pathlen - strlen (dir);
-                else
-                        len = pathlen;
-                strncpy (path, fullpath, len);
-                path[len] = '\0';
-                ret = mkdir (path, 0600);
-                if (ret && (errno != EEXIST))
-                        goto out;
+        if (stub) { /* guard against spurious wakeups */
+            call_resume(stub);
+            GF_ATOMIC_DEC(priv->stub_cnt);
         }
-        ret = 0;
+        stub = NULL;
+        if (bye)
+            break;
+    }
+
+    return NULL;
+}
+
+static void
+make_index_dir_path(char *base, const char *subdir, char *index_dir, size_t len)
+{
+    snprintf(index_dir, len, "%s/%s", base, subdir);
+}
+
+int
+index_dir_create(xlator_t *this, const char *subdir)
+{
+    int ret = 0;
+    struct stat st = {0};
+    char fullpath[PATH_MAX] = {0};
+    char path[PATH_MAX] = {0};
+    char *dir = NULL;
+    index_priv_t *priv = NULL;
+    size_t len = 0;
+    size_t pathlen = 0;
+
+    priv = this->private;
+    make_index_dir_path(priv->index_basepath, subdir, fullpath,
+                        sizeof(fullpath));
+    ret = sys_stat(fullpath, &st);
+    if (!ret) {
+        if (!S_ISDIR(st.st_mode))
+            ret = -2;
+        goto out;
+    }
+
+    pathlen = strlen(fullpath);
+    if ((pathlen > 1) && fullpath[pathlen - 1] == '/')
+        fullpath[pathlen - 1] = '\0';
+    dir = strchr(fullpath, '/');
+    while (dir) {
+        dir = strchr(dir + 1, '/');
+        if (dir)
+            len = pathlen - strlen(dir);
+        else
+            len = pathlen;
+        strncpy(path, fullpath, len);
+        path[len] = '\0';
+        ret = sys_mkdir(path, 0600);
+        if (ret && (errno != EEXIST))
+            goto out;
+    }
+    ret = 0;
 out:
-        if (ret == -1) {
-                gf_log (this->name, GF_LOG_ERROR, "%s/%s: Failed to "
-                        "create (%s)", priv->index_basepath, subdir,
-                        strerror (errno));
-        } else if (ret == -2) {
-                gf_log (this->name, GF_LOG_ERROR, "%s/%s: Failed to create, "
-                        "path exists, not a directory ", priv->index_basepath,
-                        subdir);
-        }
-        return ret;
+    if (ret == -1) {
+        gf_msg(this->name, GF_LOG_ERROR, errno,
+               INDEX_MSG_INDEX_DIR_CREATE_FAILED,
+               "%s/%s: Failed to "
+               "create",
+               priv->index_basepath, subdir);
+    } else if (ret == -2) {
+        gf_msg(this->name, GF_LOG_ERROR, ENOTDIR,
+               INDEX_MSG_INDEX_DIR_CREATE_FAILED,
+               "%s/%s: Failed to "
+               "create, path exists, not a directory ",
+               priv->index_basepath, subdir);
+    }
+    return ret;
 }
 
 void
-index_get_index (index_priv_t *priv, uuid_t index)
+index_get_index(index_priv_t *priv, uuid_t index)
 {
-        LOCK (&priv->lock);
-        {
-                gf_uuid_copy (index, priv->index);
-        }
-        UNLOCK (&priv->lock);
+    LOCK(&priv->lock);
+    {
+        gf_uuid_copy(index, priv->index);
+    }
+    UNLOCK(&priv->lock);
 }
 
 void
-index_generate_index (index_priv_t *priv, uuid_t index)
+index_generate_index(index_priv_t *priv, uuid_t index)
 {
-        LOCK (&priv->lock);
-        {
-                //To prevent duplicate generates.
-                //This method fails if number of contending threads is greater
-                //than MAX_LINK count of the fs
-                if (!gf_uuid_compare (priv->index, index))
-                        gf_uuid_generate (priv->index);
-                gf_uuid_copy (index, priv->index);
-        }
-        UNLOCK (&priv->lock);
+    LOCK(&priv->lock);
+    {
+        // To prevent duplicate generates.
+        // This method fails if number of contending threads is greater
+        // than MAX_LINK count of the fs
+        if (!gf_uuid_compare(priv->index, index))
+            gf_uuid_generate(priv->index);
+        gf_uuid_copy(index, priv->index);
+    }
+    UNLOCK(&priv->lock);
 }
 
 static void
-make_index_path (char *base, const char *subdir, uuid_t index,
-                 char *index_path, size_t len)
+make_index_path(char *base, const char *subdir, uuid_t index, char *index_path,
+                size_t len)
 {
-        make_index_dir_path (base, subdir, index_path, len);
-        snprintf (index_path + strlen (index_path), len - strlen (index_path),
-                  "/%s-%s", subdir, uuid_utoa (index));
+    make_index_dir_path(base, subdir, index_path, len);
+    snprintf(index_path + strlen(index_path), len - strlen(index_path),
+             "/%s-%s", subdir, uuid_utoa(index));
 }
 
 static void
-make_gfid_path (char *base, const char *subdir, uuid_t gfid,
-                char *gfid_path, size_t len)
+make_gfid_path(char *base, const char *subdir, uuid_t gfid, char *gfid_path,
+               size_t len)
 {
-        make_index_dir_path (base, subdir, gfid_path, len);
-        snprintf (gfid_path + strlen (gfid_path), len - strlen (gfid_path),
-                  "/%s", uuid_utoa (gfid));
+    make_index_dir_path(base, subdir, gfid_path, len);
+    snprintf(gfid_path + strlen(gfid_path), len - strlen(gfid_path), "/%s",
+             uuid_utoa(gfid));
 }
 
 static void
-make_file_path (char *base, const char *subdir, const char *filename,
-                char *file_path, size_t len)
+make_file_path(char *base, const char *subdir, const char *filename,
+               char *file_path, size_t len)
 {
-        make_index_dir_path (base, subdir, file_path, len);
-        snprintf (file_path + strlen (file_path), len - strlen (file_path),
-                  "/%s", filename);
+    make_index_dir_path(base, subdir, file_path, len);
+    snprintf(file_path + strlen(file_path), len - strlen(file_path), "/%s",
+             filename);
 }
 
 static int
-is_index_file_current (char *filename, uuid_t priv_index)
+is_index_file_current(char *filename, uuid_t priv_index, char *subdir)
 {
-        char *current_index = alloca (strlen ("xattrop-") + GF_UUID_BUF_SIZE);
+    char current_index[GF_UUID_BUF_SIZE + 16] = {
+        0,
+    };
 
-        sprintf (current_index, "xattrop-%s", uuid_utoa(priv_index));
-        return (!strcmp(filename, current_index));
+    snprintf(current_index, sizeof current_index, "%s-%s", subdir,
+             uuid_utoa(priv_index));
+    return (!strcmp(filename, current_index));
 }
 
 static void
-check_delete_stale_index_file (xlator_t *this, char *filename)
+check_delete_stale_index_file(xlator_t *this, char *filename, char *subdir)
 {
-        int             ret = 0;
-        struct stat     st = {0};
-        char            filepath[PATH_MAX] = {0};
-        index_priv_t    *priv = NULL;
+    int ret = 0;
+    struct stat st = {0};
+    char filepath[PATH_MAX] = {0};
+    index_priv_t *priv = NULL;
 
-        priv = this->private;
+    priv = this->private;
 
-        if (is_index_file_current (filename, priv->index))
-                return;
+    if (is_index_file_current(filename, priv->index, subdir))
+        return;
 
-        make_file_path (priv->index_basepath, XATTROP_SUBDIR,
-                        filename, filepath, sizeof (filepath));
-        ret = stat (filepath, &st);
-        if (!ret && st.st_nlink == 1)
-                unlink (filepath);
+    make_file_path(priv->index_basepath, subdir, filename, filepath,
+                   sizeof(filepath));
+    ret = sys_stat(filepath, &st);
+    if (!ret && st.st_nlink == 1)
+        sys_unlink(filepath);
+}
+
+static void
+index_set_link_count(index_priv_t *priv, int64_t count,
+                     index_xattrop_type_t type)
+{
+    switch (type) {
+        case XATTROP:
+            LOCK(&priv->lock);
+            {
+                priv->pending_count = count;
+            }
+            UNLOCK(&priv->lock);
+            break;
+        default:
+            break;
+    }
+}
+
+static void
+index_get_link_count(index_priv_t *priv, int64_t *count,
+                     index_xattrop_type_t type)
+{
+    switch (type) {
+        case XATTROP:
+            LOCK(&priv->lock);
+            {
+                *count = priv->pending_count;
+            }
+            UNLOCK(&priv->lock);
+            break;
+        default:
+            break;
+    }
+}
+
+static void
+index_dec_link_count(index_priv_t *priv, index_xattrop_type_t type)
+{
+    switch (type) {
+        case XATTROP:
+            LOCK(&priv->lock);
+            {
+                priv->pending_count--;
+                if (priv->pending_count == 0)
+                    priv->pending_count--;
+            }
+            UNLOCK(&priv->lock);
+            break;
+        default:
+            break;
+    }
+}
+
+char *
+index_get_subdir_from_type(index_xattrop_type_t type)
+{
+    if (type < XATTROP || type >= XATTROP_TYPE_END)
+        return NULL;
+    return index_subdirs[type];
+}
+
+char *
+index_get_subdir_from_vgfid(index_priv_t *priv, uuid_t vgfid)
+{
+    return index_get_subdir_from_type(index_get_type_from_vgfid(priv, vgfid));
 }
 
 static int
-index_fill_readdir (fd_t *fd, index_fd_ctx_t *fctx, DIR *dir, off_t off,
-                    size_t size, gf_dirent_t *entries)
-{
-        off_t     in_case = -1;
-        off_t     last_off = 0;
-        size_t    filled = 0;
-        int       count = 0;
-        char      entrybuf[sizeof(struct dirent) + 256 + 8];
-        struct dirent  *entry          = NULL;
-        int32_t              this_size      = -1;
-        gf_dirent_t          *this_entry     = NULL;
-        xlator_t             *this = NULL;
-
-        this = THIS;
-        if (!off) {
-                rewinddir (dir);
-        } else {
-                seekdir (dir, off);
+index_fill_readdir(fd_t *fd, index_fd_ctx_t *fctx, DIR *dir, off_t off,
+                   size_t size, gf_dirent_t *entries)
+{
+    off_t in_case = -1;
+    off_t last_off = 0;
+    size_t filled = 0;
+    int count = 0;
+    struct dirent *entry = NULL;
+    struct dirent scratch[2] = {
+        {
+            0,
+        },
+    };
+    int32_t this_size = -1;
+    gf_dirent_t *this_entry = NULL;
+    xlator_t *this = NULL;
+
+    this = THIS;
+    if (!off) {
+        rewinddir(dir);
+    } else {
+        seekdir(dir, off);
 #ifndef GF_LINUX_HOST_OS
-                if ((u_long)telldir(dir) != off && off != fctx->dir_eof) {
-                        gf_log (THIS->name, GF_LOG_ERROR,
-                                "seekdir(0x%llx) failed on dir=%p: "
-				"Invalid argument (offset reused from "
-				"another DIR * structure?)", off, dir);
-                        errno = EINVAL;
-                        count = -1;
-                        goto out;
-                }
-#endif /* GF_LINUX_HOST_OS */
+        if ((u_long)telldir(dir) != off && off != fctx->dir_eof) {
+            gf_msg(THIS->name, GF_LOG_ERROR, EINVAL,
+                   INDEX_MSG_INDEX_READDIR_FAILED,
+                   "seekdir(0x%llx) failed on dir=%p: "
+                   "Invalid argument (offset reused from "
+                   "another DIR * structure?)",
+                   off, dir);
+            errno = EINVAL;
+            count = -1;
+            goto out;
         }
+#endif /* GF_LINUX_HOST_OS */
+    }
 
-        while (filled <= size) {
-                in_case = (u_long)telldir (dir);
+    while (filled <= size) {
+        in_case = (u_long)telldir(dir);
 
-                if (in_case == -1) {
-                        gf_log (THIS->name, GF_LOG_ERROR,
-                                "telldir failed on dir=%p: %s",
-                                dir, strerror (errno));
-                        goto out;
-                }
+        if (in_case == -1) {
+            gf_msg(THIS->name, GF_LOG_ERROR, errno,
+                   INDEX_MSG_INDEX_READDIR_FAILED, "telldir failed on dir=%p",
+                   dir);
+            goto out;
+        }
 
-                errno = 0;
-                entry = NULL;
-                readdir_r (dir, (struct dirent *)entrybuf, &entry);
-
-                if (!entry) {
-                        if (errno == EBADF) {
-                                gf_log (THIS->name, GF_LOG_WARNING,
-                                        "readdir failed on dir=%p: %s",
-                                        dir, strerror (errno));
-                                goto out;
-                        }
-                        break;
-                }
+        errno = 0;
+        entry = sys_readdir(dir, scratch);
+        if (!entry || errno != 0) {
+            if (errno == EBADF) {
+                gf_msg(THIS->name, GF_LOG_WARNING, errno,
+                       INDEX_MSG_INDEX_READDIR_FAILED,
+                       "readdir failed on dir=%p", dir);
+                goto out;
+            }
+            break;
+        }
 
-                if (!strncmp (entry->d_name, XATTROP_SUBDIR"-",
-                              strlen (XATTROP_SUBDIR"-"))) {
-                        check_delete_stale_index_file (this, entry->d_name);
-                        continue;
-                }
+        if (!strncmp(entry->d_name, XATTROP_SUBDIR "-",
+                     strlen(XATTROP_SUBDIR "-"))) {
+            check_delete_stale_index_file(this, entry->d_name, XATTROP_SUBDIR);
+            continue;
+        } else if (!strncmp(entry->d_name, DIRTY_SUBDIR "-",
+                            strlen(DIRTY_SUBDIR "-"))) {
+            check_delete_stale_index_file(this, entry->d_name, DIRTY_SUBDIR);
+            continue;
+        }
 
-                this_size = max (sizeof (gf_dirent_t),
-                                 sizeof (gfs3_dirplist))
-                        + strlen (entry->d_name) + 1;
+        this_size = max(sizeof(gf_dirent_t), sizeof(gfs3_dirplist)) +
+                    strlen(entry->d_name) + 1;
 
-                if (this_size + filled > size) {
-                        seekdir (dir, in_case);
+        if (this_size + filled > size) {
+            seekdir(dir, in_case);
 #ifndef GF_LINUX_HOST_OS
-                        if ((u_long)telldir(dir) != in_case &&
-                            in_case != fctx->dir_eof) {
-				gf_log (THIS->name, GF_LOG_ERROR,
-					"seekdir(0x%llx) failed on dir=%p: "
-					"Invalid argument (offset reused from "
-					"another DIR * structure?)",
-					in_case, dir);
-				errno = EINVAL;
-				count = -1;
-				goto out;
-                        }
+            if ((u_long)telldir(dir) != in_case && in_case != fctx->dir_eof) {
+                gf_msg(THIS->name, GF_LOG_ERROR, EINVAL,
+                       INDEX_MSG_INDEX_READDIR_FAILED,
+                       "seekdir(0x%llx) failed on dir=%p: "
+                       "Invalid argument (offset reused from "
+                       "another DIR * structure?)",
+                       in_case, dir);
+                errno = EINVAL;
+                count = -1;
+                goto out;
+            }
 #endif /* GF_LINUX_HOST_OS */
-                        break;
-                }
-
-                this_entry = gf_dirent_for_name (entry->d_name);
-
-                if (!this_entry) {
-                        gf_log (THIS->name, GF_LOG_ERROR,
-                                "could not create gf_dirent for entry %s: (%s)",
-                                entry->d_name, strerror (errno));
-                        goto out;
-                }
-                /*
-                 * we store the offset of next entry here, which is
-                 * probably not intended, but code using syncop_readdir()
-                 * (glfs-heal.c, afr-self-heald.c, pump.c) rely on it
-                 * for directory read resumption.
-                 */
-                last_off = (u_long)telldir(dir);
-                this_entry->d_off = last_off;
-                this_entry->d_ino = entry->d_ino;
-
-                list_add_tail (&this_entry->list, &entries->list);
-
-                filled += this_size;
-                count ++;
+            break;
         }
 
-        if ((!readdir (dir) && (errno == 0))) {
-                /* Indicate EOF */
-                errno = ENOENT;
-                /* Remember EOF offset for later detection */
-                fctx->dir_eof = last_off;
+        this_entry = gf_dirent_for_name(entry->d_name);
+
+        if (!this_entry) {
+            gf_msg(THIS->name, GF_LOG_ERROR, errno,
+                   INDEX_MSG_INDEX_READDIR_FAILED,
+                   "could not create gf_dirent for entry %s", entry->d_name);
+            goto out;
         }
+        /*
+         * we store the offset of next entry here, which is
+         * probably not intended, but code using syncop_readdir()
+         * (glfs-heal.c, afr-self-heald.c, pump.c) rely on it
+         * for directory read resumption.
+         */
+        last_off = (u_long)telldir(dir);
+        this_entry->d_off = last_off;
+        this_entry->d_ino = entry->d_ino;
+
+        list_add_tail(&this_entry->list, &entries->list);
+
+        filled += this_size;
+        count++;
+    }
+
+    errno = 0;
+
+    if ((!sys_readdir(dir, scratch) && (errno == 0))) {
+        /* Indicate EOF */
+        errno = ENOENT;
+        /* Remember EOF offset for later detection */
+        fctx->dir_eof = last_off;
+    }
 out:
-        return count;
+    return count;
 }
 
 int
-index_add (xlator_t *this, uuid_t gfid, const char *subdir)
-{
-        int32_t           op_errno = 0;
-        char              gfid_path[PATH_MAX] = {0};
-        char              index_path[PATH_MAX] = {0};
-        int               ret = 0;
-        uuid_t            index = {0};
-        index_priv_t      *priv = NULL;
-        struct stat       st = {0};
-        int               fd = 0;
-
-        priv = this->private;
-        GF_ASSERT_AND_GOTO_WITH_ERROR (this->name, !gf_uuid_is_null (gfid),
-                                       out, op_errno, EINVAL);
-
-        make_gfid_path (priv->index_basepath, subdir, gfid,
-                        gfid_path, sizeof (gfid_path));
-
-        ret = stat (gfid_path, &st);
-        if (!ret)
-                goto out;
-        index_get_index (priv, index);
-        make_index_path (priv->index_basepath, subdir,
-                         index, index_path, sizeof (index_path));
-        ret = sys_link (index_path, gfid_path);
-        if (!ret || (errno == EEXIST))  {
-                ret = 0;
-                goto out;
-        }
+index_link_to_base(xlator_t *this, char *fpath, const char *subdir)
+{
+    int ret = 0;
+    int fd = 0;
+    int op_errno = 0;
+    uuid_t index = {0};
+    index_priv_t *priv = this->private;
+    char base[PATH_MAX] = {0};
+
+    index_get_index(priv, index);
+    make_index_path(priv->index_basepath, subdir, index, base, sizeof(base));
+
+    ret = sys_link(base, fpath);
+    if (!ret || (errno == EEXIST)) {
+        ret = 0;
+        goto out;
+    }
 
-        op_errno = errno;
-        if (op_errno == ENOENT) {
-                ret = index_dir_create (this, subdir);
-                if (ret)
-                        goto out;
-        } else if (op_errno == EMLINK) {
-                index_generate_index (priv, index);
-                make_index_path (priv->index_basepath, subdir,
-                                 index, index_path, sizeof (index_path));
-        } else {
-                goto out;
+    op_errno = errno;
+    if (op_errno == ENOENT) {
+        ret = index_dir_create(this, subdir);
+        if (ret) {
+            op_errno = errno;
+            goto out;
         }
+    } else if (op_errno == EMLINK) {
+        index_generate_index(priv, index);
+        make_index_path(priv->index_basepath, subdir, index, base,
+                        sizeof(base));
+    } else {
+        goto out;
+    }
+
+    op_errno = 0;
+    fd = sys_creat(base, 0);
+    if ((fd < 0) && (errno != EEXIST)) {
+        op_errno = errno;
+        gf_msg(this->name, GF_LOG_ERROR, op_errno, INDEX_MSG_INDEX_ADD_FAILED,
+               "%s: Not able to "
+               "create index",
+               fpath);
+        goto out;
+    }
+
+    if (fd >= 0)
+        sys_close(fd);
+
+    ret = sys_link(base, fpath);
+    if (ret && (errno != EEXIST)) {
+        op_errno = errno;
+        gf_msg(this->name, GF_LOG_ERROR, errno, INDEX_MSG_INDEX_ADD_FAILED,
+               "%s: Not able to "
+               "add to index",
+               fpath);
+        goto out;
+    }
+out:
+    return -op_errno;
+}
 
-        fd = creat (index_path, 0);
-        if ((fd < 0) && (errno != EEXIST)) {
-                ret = -1;
-                gf_log (this->name, GF_LOG_ERROR, "%s: Not able to "
-                        "create index (%s)", uuid_utoa (gfid),
-                        strerror (errno));
-                goto out;
-        }
+int
+index_add(xlator_t *this, uuid_t gfid, const char *subdir,
+          index_xattrop_type_t type)
+{
+    char gfid_path[PATH_MAX] = {0};
+    int ret = -1;
+    index_priv_t *priv = NULL;
+    struct stat st = {0};
 
-        if (fd >= 0)
-                close (fd);
+    priv = this->private;
 
-        ret = sys_link (index_path, gfid_path);
-        if (ret && (errno != EEXIST)) {
-                gf_log (this->name, GF_LOG_ERROR, "%s: Not able to "
-                        "add to index (%s)", uuid_utoa (gfid),
-                        strerror (errno));
-                goto out;
-        }
+    if (gf_uuid_is_null(gfid)) {
+        GF_ASSERT(0);
+        goto out;
+    }
 
-        ret = 0;
+    make_gfid_path(priv->index_basepath, subdir, gfid, gfid_path,
+                   sizeof(gfid_path));
+
+    ret = sys_stat(gfid_path, &st);
+    if (!ret)
+        goto out;
+    ret = index_link_to_base(this, gfid_path, subdir);
 out:
-        return ret;
+    return ret;
 }
 
 int
-index_del (xlator_t *this, uuid_t gfid, const char *subdir)
-{
-        int32_t      op_errno __attribute__((unused)) = 0;
-        index_priv_t *priv = NULL;
-        int          ret = 0;
-        char         gfid_path[PATH_MAX] = {0};
-
-        priv = this->private;
-        GF_ASSERT_AND_GOTO_WITH_ERROR (this->name, !gf_uuid_is_null (gfid),
-                                       out, op_errno, EINVAL);
-        make_gfid_path (priv->index_basepath, subdir, gfid,
-                        gfid_path, sizeof (gfid_path));
-        ret = unlink (gfid_path);
-        if (ret && (errno != ENOENT)) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "%s: failed to delete from index (%s)",
-                        gfid_path, strerror (errno));
-                ret = -errno;
-                goto out;
+index_del(xlator_t *this, uuid_t gfid, const char *subdir, int type)
+{
+    int32_t op_errno __attribute__((unused)) = 0;
+    index_priv_t *priv = NULL;
+    int ret = 0;
+    char gfid_path[PATH_MAX] = {0};
+    char rename_dst[PATH_MAX] = {
+        0,
+    };
+    uuid_t uuid;
+
+    priv = this->private;
+    GF_ASSERT_AND_GOTO_WITH_ERROR(this->name, !gf_uuid_is_null(gfid), out,
+                                  op_errno, EINVAL);
+    make_gfid_path(priv->index_basepath, subdir, gfid, gfid_path,
+                   sizeof(gfid_path));
+
+    if ((strcmp(subdir, ENTRY_CHANGES_SUBDIR)) == 0) {
+        ret = sys_rmdir(gfid_path);
+        /* rmdir above could fail with ENOTEMPTY if the indices under
+         * it were created when granular-entry-heal was enabled, whereas
+         * the actual heal that happened was non-granular (or full) in
+         * nature, resulting in name indices getting left out. To
+         * clean up this directory without it affecting the IO path perf,
+         * the directory is renamed to a unique name under
+         * indices/entry-changes. Self-heal will pick up this entry
+         * during crawl and on lookup into the file system figure that
+         * the index is stale and subsequently wipe it out using rmdir().
+         */
+        if ((ret) && (errno == ENOTEMPTY)) {
+            gf_uuid_generate(uuid);
+            make_gfid_path(priv->index_basepath, subdir, uuid, rename_dst,
+                           sizeof(rename_dst));
+            ret = sys_rename(gfid_path, rename_dst);
         }
-        ret = 0;
+    } else {
+        ret = sys_unlink(gfid_path);
+    }
+
+    if (ret && (errno != ENOENT)) {
+        gf_msg(this->name, GF_LOG_ERROR, errno, INDEX_MSG_INDEX_DEL_FAILED,
+               "%s: failed to delete"
+               " from index",
+               gfid_path);
+        ret = -errno;
+        goto out;
+    }
+
+    index_dec_link_count(priv, type);
+    ret = 0;
 out:
-        return ret;
+    return ret;
+}
+
+static gf_boolean_t
+_is_xattr_in_watchlist(dict_t *d, char *k, data_t *v, void *tmp)
+{
+    if (!strncmp(k, tmp, strlen(k)))
+        return _gf_true;
+
+    return _gf_false;
+}
+
+static gf_boolean_t
+is_xattr_in_watchlist(dict_t *this, char *key, data_t *value, void *matchdata)
+{
+    int ret = -1;
+
+    // matchdata is a list of xattrs
+    // key is strncmp'ed with each xattr in matchdata.
+    // ret will be 0 if key pattern is not present in the matchdata
+    // else ret will be count number of xattrs the key pattern-matches with.
+    ret = dict_foreach_match(matchdata, _is_xattr_in_watchlist, key,
+                             dict_null_foreach_fn, NULL);
+
+    if (ret > 0)
+        return _gf_true;
+    return _gf_false;
 }
 
 static int
-_check_key_is_zero_filled (dict_t *d, char *k, data_t *v,
-                           void *tmp)
+index_find_xattr_type(dict_t *d, char *k, data_t *v)
 {
-        if (mem_0filled ((const char*)v->data, v->len)) {
-                /* -1 means, no more iterations, treat as 'break' */
-                return -1;
-        }
+    int idx = -1;
+    index_priv_t *priv = THIS->private;
+
+    if (priv->dirty_watchlist &&
+        is_xattr_in_watchlist(d, k, v, priv->dirty_watchlist))
+        idx = DIRTY;
+    else if (priv->pending_watchlist &&
+             is_xattr_in_watchlist(d, k, v, priv->pending_watchlist))
+        idx = XATTROP;
+
+    return idx;
+}
+
+int
+index_fill_zero_array(dict_t *d, char *k, data_t *v, void *adata)
+{
+    int idx = -1;
+    int *zfilled = adata;
+    // zfilled array contains `state` for all types xattrs.
+    // state : whether the gfid file of this file exists in
+    // corresponding xattr directory or not.
+
+    idx = index_find_xattr_type(d, k, v);
+    if (idx == -1)
         return 0;
+    zfilled[idx] = 0;
+    return 0;
 }
 
-void
-_index_action (xlator_t *this, inode_t *inode, gf_boolean_t zero_xattr)
+static int
+_check_key_is_zero_filled(dict_t *d, char *k, data_t *v, void *tmp)
 {
-        int               ret  = 0;
-        index_inode_ctx_t *ctx = NULL;
+    int *zfilled = tmp;
+    int idx = -1;
 
-        ret = index_inode_ctx_get (inode, this, &ctx);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "Not able to %s %s -> index",
-                        zero_xattr?"del":"add", uuid_utoa (inode->gfid));
-                goto out;
-        }
-        if (zero_xattr) {
-                if (ctx->state == NOTIN)
-                        goto out;
-                ret = index_del (this, inode->gfid, XATTROP_SUBDIR);
-                if (!ret)
-                        ctx->state = NOTIN;
-        } else {
-                if (ctx->state == IN)
-                        goto out;
-                ret = index_add (this, inode->gfid, XATTROP_SUBDIR);
-                if (!ret)
-                        ctx->state = IN;
+    idx = index_find_xattr_type(d, k, v);
+    if (idx == -1)
+        return 0;
+
+    /* Along with checking that the value of a key is zero filled
+     * the key's corresponding index should be assigned
+     * appropriate value.
+     * zfilled[idx] will be 0(false) if value not zero.
+     *              will be 1(true) if value is zero.
+     */
+    if (mem_0filled((const char *)v->data, v->len)) {
+        zfilled[idx] = 0;
+        return 0;
+    }
+
+    /* If zfilled[idx] was previously 0, it means at least
+     * one xattr of its "kind" is non-zero. Keep its value
+     * the same.
+     */
+    if (zfilled[idx])
+        zfilled[idx] = 1;
+    return 0;
+}
+
+int
+index_entry_create(xlator_t *this, inode_t *inode, char *filename)
+{
+    int ret = -1;
+    int op_errno = 0;
+    char pgfid_path[PATH_MAX] = {0};
+    char entry_path[PATH_MAX] = {0};
+    index_priv_t *priv = NULL;
+    index_inode_ctx_t *ctx = NULL;
+    int32_t len = 0;
+
+    priv = this->private;
+
+    GF_ASSERT_AND_GOTO_WITH_ERROR(this->name, !gf_uuid_is_null(inode->gfid),
+                                  out, op_errno, EINVAL);
+    GF_ASSERT_AND_GOTO_WITH_ERROR(this->name, filename, out, op_errno, EINVAL);
+
+    ret = index_inode_ctx_get(inode, this, &ctx);
+    if (ret) {
+        op_errno = EINVAL;
+        gf_msg(this->name, GF_LOG_ERROR, op_errno,
+               INDEX_MSG_INODE_CTX_GET_SET_FAILED,
+               "Not able to get inode ctx for %s", uuid_utoa(inode->gfid));
+        goto out;
+    }
+
+    make_gfid_path(priv->index_basepath, ENTRY_CHANGES_SUBDIR, inode->gfid,
+                   pgfid_path, sizeof(pgfid_path));
+
+    if (ctx->state[ENTRY_CHANGES] != IN) {
+        ret = sys_mkdir(pgfid_path, 0600);
+        if (ret != 0 && errno != EEXIST) {
+            op_errno = errno;
+            goto out;
         }
+        ctx->state[ENTRY_CHANGES] = IN;
+    }
+
+    if (strchr(filename, '/')) {
+        gf_msg(this->name, GF_LOG_ERROR, EINVAL, INDEX_MSG_INDEX_ADD_FAILED,
+               "Got invalid entry (%s) for pargfid path (%s)", filename,
+               pgfid_path);
+        op_errno = EINVAL;
+        goto out;
+    }
+
+    len = snprintf(entry_path, sizeof(entry_path), "%s/%s", pgfid_path,
+                   filename);
+    if ((len < 0) || (len >= sizeof(entry_path))) {
+        op_errno = EINVAL;
+        goto out;
+    }
+
+    op_errno = 0;
+
+    ret = index_link_to_base(this, entry_path, ENTRY_CHANGES_SUBDIR);
 out:
-        return;
+    if (op_errno)
+        ret = -op_errno;
+    return ret;
 }
 
-static gf_boolean_t
-is_xattr_in_watchlist (dict_t *this, char *key, data_t *value, void *matchdata)
+int
+index_entry_delete(xlator_t *this, uuid_t pgfid, char *filename)
 {
-        if (dict_get (matchdata, key))
-                return _gf_true;
+    int ret = 0;
+    int op_errno = 0;
+    char pgfid_path[PATH_MAX] = {0};
+    char entry_path[PATH_MAX] = {0};
+    index_priv_t *priv = NULL;
+    int32_t len = 0;
+
+    priv = this->private;
+
+    GF_ASSERT_AND_GOTO_WITH_ERROR(this->name, !gf_uuid_is_null(pgfid), out,
+                                  op_errno, EINVAL);
+    GF_ASSERT_AND_GOTO_WITH_ERROR(this->name, filename, out, op_errno, EINVAL);
+
+    make_gfid_path(priv->index_basepath, ENTRY_CHANGES_SUBDIR, pgfid,
+                   pgfid_path, sizeof(pgfid_path));
+
+    if (strchr(filename, '/')) {
+        gf_msg(this->name, GF_LOG_ERROR, EINVAL, INDEX_MSG_INDEX_DEL_FAILED,
+               "Got invalid entry (%s) for pargfid path (%s)", filename,
+               pgfid_path);
+        op_errno = EINVAL;
+        goto out;
+    }
+
+    len = snprintf(entry_path, sizeof(entry_path), "%s/%s", pgfid_path,
+                   filename);
+    if ((len < 0) || (len >= sizeof(entry_path))) {
+        op_errno = EINVAL;
+        goto out;
+    }
+
+    ret = sys_unlink(entry_path);
+    if (ret && (errno != ENOENT)) {
+        op_errno = errno;
+        gf_msg(this->name, GF_LOG_ERROR, op_errno, INDEX_MSG_INDEX_DEL_FAILED,
+               "%s: failed to delete from index/entry-changes", entry_path);
+    }
 
-        return _gf_false;
+out:
+    return -op_errno;
+}
+
+int
+index_entry_action(xlator_t *this, inode_t *inode, dict_t *xdata, char *key)
+{
+    int ret = 0;
+    char *filename = NULL;
+
+    ret = dict_get_str(xdata, key, &filename);
+    if (ret != 0) {
+        ret = 0;
+        goto out;
+    }
+
+    if (strcmp(key, GF_XATTROP_ENTRY_IN_KEY) == 0)
+        ret = index_entry_create(this, inode, filename);
+    else if (strcmp(key, GF_XATTROP_ENTRY_OUT_KEY) == 0)
+        ret = index_entry_delete(this, inode->gfid, filename);
+
+out:
+    return ret;
 }
 
 void
-xattrop_index_action (xlator_t *this, inode_t *inode, dict_t *xattr,
-                      dict_match_t match, void *match_data)
+_index_action(xlator_t *this, inode_t *inode, int *zfilled)
+{
+    int ret = 0;
+    int i = 0;
+    index_inode_ctx_t *ctx = NULL;
+    char *subdir = NULL;
+
+    ret = index_inode_ctx_get(inode, this, &ctx);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, EINVAL,
+               INDEX_MSG_INODE_CTX_GET_SET_FAILED,
+               "Not able to get"
+               " inode context for %s.",
+               uuid_utoa(inode->gfid));
+        goto out;
+    }
+
+    for (i = 0; i < XATTROP_TYPE_END; i++) {
+        subdir = index_get_subdir_from_type(i);
+        if (zfilled[i] == 1) {
+            if (ctx->state[i] == NOTIN)
+                continue;
+            ret = index_del(this, inode->gfid, subdir, i);
+            if (!ret)
+                ctx->state[i] = NOTIN;
+        } else if (zfilled[i] == 0) {
+            if (ctx->state[i] == IN)
+                continue;
+            ret = index_add(this, inode->gfid, subdir, i);
+            if (!ret)
+                ctx->state[i] = IN;
+        }
+    }
+out:
+    return;
+}
+
+static void
+index_init_state(xlator_t *this, inode_t *inode, index_inode_ctx_t *ctx,
+                 char *subdir)
 {
-        gf_boolean_t      zero_xattr = _gf_true;
-        int               ret = 0;
+    int ret = -1;
+    char pgfid_path[PATH_MAX] = {0};
+    struct stat st = {0};
+    index_priv_t *priv = NULL;
 
-        ret = dict_foreach_match (xattr, match, match_data,
-                                  _check_key_is_zero_filled, NULL);
-        if (ret == -1)
-                zero_xattr = _gf_false;
-        _index_action (this, inode, zero_xattr);
-        return;
+    priv = this->private;
+
+    make_gfid_path(priv->index_basepath, subdir, inode->gfid, pgfid_path,
+                   sizeof(pgfid_path));
+
+    ret = sys_stat(pgfid_path, &st);
+    if (ret == 0)
+        ctx->state[ENTRY_CHANGES] = IN;
+    else if (ret != 0 && errno == ENOENT)
+        ctx->state[ENTRY_CHANGES] = NOTIN;
+
+    return;
 }
 
-static inline gf_boolean_t
-index_xattrop_track (xlator_t *this, gf_xattrop_flags_t flags, dict_t *dict)
+void
+xattrop_index_action(xlator_t *this, index_local_t *local, dict_t *xattr,
+                     dict_match_t match, void *match_data)
 {
-        index_priv_t *priv = this->private;
+    int ret = 0;
+    int zfilled[XATTROP_TYPE_END] = {
+        0,
+    };
+    int8_t value = 0;
+    char *subdir = NULL;
+    dict_t *req_xdata = NULL;
+    inode_t *inode = NULL;
+    index_inode_ctx_t *ctx = NULL;
+
+    inode = local->inode;
+    req_xdata = local->xdata;
+
+    memset(zfilled, -1, sizeof(zfilled));
+    ret = dict_foreach_match(xattr, match, match_data,
+                             _check_key_is_zero_filled, zfilled);
+    _index_action(this, inode, zfilled);
+
+    if (req_xdata) {
+        ret = index_entry_action(this, inode, req_xdata,
+                                 GF_XATTROP_ENTRY_OUT_KEY);
+
+        ret = dict_get_int8(req_xdata, GF_XATTROP_PURGE_INDEX, &value);
+        if ((ret) || (value == 0))
+            goto out;
+    }
+
+    if (zfilled[XATTROP] != 1)
+        goto out;
+
+    if (inode->ia_type != IA_IFDIR)
+        goto out;
+
+    subdir = index_get_subdir_from_type(ENTRY_CHANGES);
+    ret = index_inode_ctx_get(inode, this, &ctx);
+    if (ctx->state[ENTRY_CHANGES] == UNKNOWN)
+        index_init_state(this, inode, ctx, subdir);
+    if (ctx->state[ENTRY_CHANGES] == IN) {
+        ret = index_del(this, inode->gfid, subdir, ENTRY_CHANGES);
+        ctx->state[ENTRY_CHANGES] = NOTIN;
+    }
 
-        if (flags == GF_XATTROP_ADD_ARRAY)
-                return _gf_true;
+out:
+    return;
+}
 
-        if (flags != GF_XATTROP_ADD_ARRAY64)
-                return _gf_false;
+static gf_boolean_t
+index_xattrop_track(xlator_t *this, gf_xattrop_flags_t flags, dict_t *dict)
+{
+    index_priv_t *priv = this->private;
 
-        if (!priv->xattrop64_watchlist)
-                return _gf_false;
+    if (flags == GF_XATTROP_ADD_ARRAY)
+        return _gf_true;
 
-        if (dict_foreach_match (dict, is_xattr_in_watchlist,
-                                priv->xattrop64_watchlist, dict_null_foreach_fn,
-                                NULL) > 0)
-                return _gf_true;
+    if (flags != GF_XATTROP_ADD_ARRAY64)
+        return _gf_false;
 
+    if (!priv->pending_watchlist)
         return _gf_false;
+
+    if (dict_foreach_match(dict, is_xattr_in_watchlist, priv->pending_watchlist,
+                           dict_null_foreach_fn, NULL) > 0)
+        return _gf_true;
+
+    return _gf_false;
 }
 
 int
-__index_fd_ctx_get (fd_t *fd, xlator_t *this, index_fd_ctx_t **ctx)
+index_inode_path(xlator_t *this, inode_t *inode, char *dirpath, size_t len)
 {
-        int               ret = 0;
-        index_fd_ctx_t    *fctx = NULL;
-        uint64_t          tmpctx = 0;
-        char              index_dir[PATH_MAX] = {0};
-        index_priv_t      *priv = NULL;
-
-        priv = this->private;
-        if (gf_uuid_compare (fd->inode->gfid, priv->xattrop_vgfid)) {
-                ret = -EINVAL;
-                goto out;
-        }
-
-        ret = __fd_ctx_get (fd, this, &tmpctx);
-        if (!ret) {
-                fctx = (index_fd_ctx_t*) (long) tmpctx;
-                goto out;
+    char *subdir = NULL;
+    int ret = 0;
+    index_priv_t *priv = NULL;
+    index_inode_ctx_t *ictx = NULL;
+
+    priv = this->private;
+    if (!index_is_fop_on_internal_inode(this, inode, NULL)) {
+        ret = -EINVAL;
+        goto out;
+    }
+
+    subdir = index_get_subdir_from_vgfid(priv, inode->gfid);
+    if (subdir) {
+        if (len <= strlen(priv->index_basepath) + 1 /*'/'*/ + strlen(subdir)) {
+            ret = -EINVAL;
+            goto out;
         }
-
-        fctx = GF_CALLOC (1, sizeof (*fctx), gf_index_fd_ctx_t);
-        if (!fctx) {
-                ret = -ENOMEM;
-                goto out;
+        make_index_dir_path(priv->index_basepath, subdir, dirpath, len);
+    } else {
+        ret = index_inode_ctx_get(inode, this, &ictx);
+        if (ret)
+            goto out;
+        if (gf_uuid_is_null(ictx->virtual_pargfid)) {
+            ret = -EINVAL;
+            goto out;
         }
-
-        make_index_dir_path (priv->index_basepath, XATTROP_SUBDIR,
-                             index_dir, sizeof (index_dir));
-        fctx->dir = opendir (index_dir);
-        if (!fctx->dir) {
-                ret = -errno;
-                GF_FREE (fctx);
-                fctx = NULL;
-                goto out;
+        make_index_dir_path(priv->index_basepath, ENTRY_CHANGES_SUBDIR, dirpath,
+                            len);
+        if (len <= strlen(dirpath) + 1 /*'/'*/ + SLEN(UUID0_STR)) {
+            ret = -EINVAL;
+            goto out;
         }
-        fctx->dir_eof = -1;
+        strcat(dirpath, "/");
+        strcat(dirpath, uuid_utoa(ictx->virtual_pargfid));
+    }
+out:
+    return ret;
+}
 
-        ret = __fd_ctx_set (fd, this, (uint64_t)(long)fctx);
-        if (ret) {
-                closedir (fctx->dir);
-                GF_FREE (fctx);
-                fctx = NULL;
-                ret = -EINVAL;
-                goto out;
-        }
+int
+__index_fd_ctx_get(fd_t *fd, xlator_t *this, index_fd_ctx_t **ctx)
+{
+    int ret = 0;
+    index_fd_ctx_t *fctx = NULL;
+    uint64_t tmpctx = 0;
+    char dirpath[PATH_MAX] = {0};
+
+    ret = __fd_ctx_get(fd, this, &tmpctx);
+    if (!ret) {
+        fctx = (index_fd_ctx_t *)(long)tmpctx;
+        *ctx = fctx;
+        goto out;
+    }
+
+    ret = index_inode_path(this, fd->inode, dirpath, sizeof(dirpath));
+    if (ret)
+        goto out;
+
+    fctx = GF_CALLOC(1, sizeof(*fctx), gf_index_fd_ctx_t);
+    if (!fctx) {
+        ret = -ENOMEM;
+        goto out;
+    }
+
+    fctx->dir = sys_opendir(dirpath);
+    if (!fctx->dir) {
+        ret = -errno;
+        GF_FREE(fctx);
+        fctx = NULL;
+        goto out;
+    }
+    fctx->dir_eof = -1;
+
+    ret = __fd_ctx_set(fd, this, (uint64_t)(long)fctx);
+    if (ret) {
+        (void)sys_closedir(fctx->dir);
+        GF_FREE(fctx);
+        fctx = NULL;
+        ret = -EINVAL;
+        goto out;
+    }
+    *ctx = fctx;
 out:
-        if (fctx)
-                *ctx = fctx;
-        return ret;
+    return ret;
 }
 
 int
-index_fd_ctx_get (fd_t *fd, xlator_t *this, index_fd_ctx_t **ctx)
+index_fd_ctx_get(fd_t *fd, xlator_t *this, index_fd_ctx_t **ctx)
 {
-        int     ret = 0;
-        LOCK (&fd->lock);
-        {
-                ret = __index_fd_ctx_get (fd, this, ctx);
-        }
-        UNLOCK (&fd->lock);
-        return ret;
+    int ret = 0;
+    LOCK(&fd->lock);
+    {
+        ret = __index_fd_ctx_get(fd, this, ctx);
+    }
+    UNLOCK(&fd->lock);
+    return ret;
 }
 
-//new - Not NULL means start a fop
-//new - NULL means done processing the fop
+// new - Not NULL means start a fop
+// new - NULL means done processing the fop
 void
-index_queue_process (xlator_t *this, inode_t *inode, call_stub_t *new)
+index_queue_process(xlator_t *this, inode_t *inode, call_stub_t *new)
 {
-        call_stub_t       *stub = NULL;
-        index_inode_ctx_t *ctx = NULL;
-        int               ret = 0;
-        call_frame_t      *frame = NULL;
+    call_stub_t *stub = NULL;
+    index_inode_ctx_t *ctx = NULL;
+    int ret = 0;
+    call_frame_t *frame = NULL;
+
+    LOCK(&inode->lock);
+    {
+        ret = __index_inode_ctx_get(inode, this, &ctx);
+        if (ret)
+            goto unlock;
 
-        LOCK (&inode->lock);
-        {
-                ret = __index_inode_ctx_get (inode, this, &ctx);
-                if (ret)
-                        goto unlock;
-
-                if (new) {
-                        __index_enqueue (&ctx->callstubs, new);
-                        new = NULL;
-                } else {
-                        ctx->processing = _gf_false;
-                }
+        if (new) {
+            __index_enqueue(&ctx->callstubs, new);
+            new = NULL;
+        } else {
+            ctx->processing = _gf_false;
+        }
 
-                if (!ctx->processing) {
-                        stub = __index_dequeue (&ctx->callstubs);
-                        if (stub)
-                                ctx->processing = _gf_true;
-                        else
-                                ctx->processing = _gf_false;
-                }
+        if (!ctx->processing) {
+            stub = __index_dequeue(&ctx->callstubs);
+            if (stub)
+                ctx->processing = _gf_true;
+            else
+                ctx->processing = _gf_false;
         }
+    }
 unlock:
-        UNLOCK (&inode->lock);
-
-        if (ret && new) {
-                frame = new->frame;
-                if (new->fop == GF_FOP_XATTROP) {
-                        INDEX_STACK_UNWIND (xattrop, frame, -1, ENOMEM,
-                                            NULL, NULL);
-                } else if (new->fop == GF_FOP_FXATTROP) {
-                        INDEX_STACK_UNWIND (fxattrop, frame, -1, ENOMEM,
-                                            NULL, NULL);
-                }
-                call_stub_destroy (new);
-        } else if (stub) {
-                call_resume (stub);
+    UNLOCK(&inode->lock);
+
+    if (ret && new) {
+        frame = new->frame;
+        if (new->fop == GF_FOP_XATTROP) {
+            INDEX_STACK_UNWIND(xattrop, frame, -1, ENOMEM, NULL, NULL);
+        } else if (new->fop == GF_FOP_FXATTROP) {
+            INDEX_STACK_UNWIND(fxattrop, frame, -1, ENOMEM, NULL, NULL);
         }
-        return;
+        call_stub_destroy(new);
+    } else if (stub) {
+        call_resume(stub);
+    }
+    return;
 }
 
 static int
-xattrop_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-             int32_t op_ret, int32_t op_errno, dict_t *xattr,
-             dict_t *xdata, dict_match_t match, dict_t *matchdata)
+xattrop_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int32_t op_ret,
+            int32_t op_errno, dict_t *xattr, dict_t *xdata, dict_match_t match,
+            dict_t *matchdata)
 {
-        inode_t *inode = NULL;
+    inode_t *inode = NULL;
+    index_local_t *local = NULL;
 
-        inode = inode_ref (frame->local);
-        if (op_ret < 0)
-                goto out;
+    local = frame->local;
+    inode = inode_ref(local->inode);
 
-        xattrop_index_action (this, frame->local, xattr, match, matchdata);
+    if (op_ret < 0)
+        goto out;
+
+    xattrop_index_action(this, local, xattr, match, matchdata);
 out:
-        INDEX_STACK_UNWIND (xattrop, frame, op_ret, op_errno, xattr, xdata);
-        index_queue_process (this, inode, NULL);
-        inode_unref (inode);
+    INDEX_STACK_UNWIND(xattrop, frame, op_ret, op_errno, xattr, xdata);
+    index_queue_process(this, inode, NULL);
+    inode_unref(inode);
 
-        return 0;
+    return 0;
 }
 
 int32_t
-index_xattrop_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                   int32_t op_ret, int32_t op_errno, dict_t *xattr,
-                   dict_t *xdata)
+index_xattrop_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                  int32_t op_ret, int32_t op_errno, dict_t *xattr,
+                  dict_t *xdata)
 {
-        return xattrop_cbk (frame, cookie, this, op_ret, op_errno, xattr, xdata,
-                            dict_match_everything, NULL);
+    index_priv_t *priv = this->private;
+
+    xattrop_cbk(frame, cookie, this, op_ret, op_errno, xattr, xdata,
+                is_xattr_in_watchlist, priv->complete_watchlist);
+    return 0;
 }
 
 int32_t
-index_xattrop64_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                     int32_t op_ret, int32_t op_errno, dict_t *xattr,
-                     dict_t *xdata)
+index_xattrop64_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                    int32_t op_ret, int32_t op_errno, dict_t *xattr,
+                    dict_t *xdata)
 {
-        index_priv_t *priv = this->private;
+    index_priv_t *priv = this->private;
 
-        return xattrop_cbk (frame, cookie, this, op_ret, op_errno, xattr, xdata,
-                            is_xattr_in_watchlist, priv->xattrop64_watchlist);
+    return xattrop_cbk(frame, cookie, this, op_ret, op_errno, xattr, xdata,
+                       is_xattr_in_watchlist, priv->pending_watchlist);
 }
 
-int
-index_xattrop_wrapper (call_frame_t *frame, xlator_t *this, loc_t *loc,
-                       gf_xattrop_flags_t optype, dict_t *xattr, dict_t *xdata)
+void
+index_xattrop_do(call_frame_t *frame, xlator_t *this, loc_t *loc, fd_t *fd,
+                 gf_xattrop_flags_t optype, dict_t *xattr, dict_t *xdata)
 {
-        fop_xattrop_cbk_t cbk = NULL;
-        //In wind phase bring the gfid into index. This way if the brick crashes
-        //just after posix performs xattrop before _cbk reaches index xlator
-        //we will still have the gfid in index.
-        _index_action (this, frame->local, _gf_false);
-
-        if (optype == GF_XATTROP_ADD_ARRAY)
-                cbk = index_xattrop_cbk;
-        else
-                cbk = index_xattrop64_cbk;
-
-        STACK_WIND (frame, cbk, FIRST_CHILD (this),
-                    FIRST_CHILD (this)->fops->xattrop, loc, optype, xattr,
-                    xdata);
-        return 0;
+    int ret = -1;
+    int zfilled[XATTROP_TYPE_END] = {
+        0,
+    };
+    index_local_t *local = NULL;
+    fop_xattrop_cbk_t x_cbk = NULL;
+
+    local = frame->local;
+
+    if (optype == GF_XATTROP_ADD_ARRAY)
+        x_cbk = index_xattrop_cbk;
+    else
+        x_cbk = index_xattrop64_cbk;
+
+    // In wind phase bring the gfid into index. This way if the brick crashes
+    // just after posix performs xattrop before _cbk reaches index xlator
+    // we will still have the gfid in index.
+    memset(zfilled, -1, sizeof(zfilled));
+
+    /* Foreach xattr, set corresponding index of zfilled to 1
+     * zfilled[index] = 1 implies the xattr's value is zero filled
+     * and should be added in its corresponding subdir.
+     *
+     * zfilled should be set to 1 only for those index that
+     * exist in xattr variable. This is to distinguish
+     * between different types of volumes.
+     * For e.g., if the check is not made,
+     * zfilled[DIRTY] is set to 1 for EC volumes,
+     * index file will be tried to create in indices/dirty dir
+     * which doesn't exist for an EC volume.
+     */
+    ret = dict_foreach(xattr, index_fill_zero_array, zfilled);
+
+    _index_action(this, local->inode, zfilled);
+    if (xdata)
+        ret = index_entry_action(this, local->inode, xdata,
+                                 GF_XATTROP_ENTRY_IN_KEY);
+    if (ret < 0) {
+        x_cbk(frame, NULL, this, -1, -ret, NULL, NULL);
+        return;
+    }
+
+    if (loc)
+        STACK_WIND(frame, x_cbk, FIRST_CHILD(this),
+                   FIRST_CHILD(this)->fops->xattrop, loc, optype, xattr, xdata);
+    else
+        STACK_WIND(frame, x_cbk, FIRST_CHILD(this),
+                   FIRST_CHILD(this)->fops->fxattrop, fd, optype, xattr, xdata);
 }
 
 int
-index_fxattrop_wrapper (call_frame_t *frame, xlator_t *this, fd_t *fd,
-                        gf_xattrop_flags_t optype, dict_t *xattr, dict_t *xdata)
+index_xattrop_wrapper(call_frame_t *frame, xlator_t *this, loc_t *loc,
+                      gf_xattrop_flags_t optype, dict_t *xattr, dict_t *xdata)
 {
-        fop_fxattrop_cbk_t cbk = NULL;
-        //In wind phase bring the gfid into index. This way if the brick crashes
-        //just after posix performs xattrop before _cbk reaches index xlator
-        //we will still have the gfid in index.
-        _index_action (this, frame->local, _gf_false);
-
-        if (optype == GF_XATTROP_ADD_ARRAY)
-                cbk = index_xattrop_cbk;
-        else
-                cbk = index_xattrop64_cbk;
+    index_xattrop_do(frame, this, loc, NULL, optype, xattr, xdata);
+    return 0;
+}
 
-        STACK_WIND (frame, cbk, FIRST_CHILD (this),
-                    FIRST_CHILD (this)->fops->fxattrop, fd, optype, xattr,
-                    xdata);
-        return 0;
+int
+index_fxattrop_wrapper(call_frame_t *frame, xlator_t *this, fd_t *fd,
+                       gf_xattrop_flags_t optype, dict_t *xattr, dict_t *xdata)
+{
+    index_xattrop_do(frame, this, NULL, fd, optype, xattr, xdata);
+    return 0;
 }
 
 int32_t
-index_xattrop (call_frame_t *frame, xlator_t *this, loc_t *loc,
-	       gf_xattrop_flags_t flags, dict_t *dict, dict_t *xdata)
+index_xattrop(call_frame_t *frame, xlator_t *this, loc_t *loc,
+              gf_xattrop_flags_t flags, dict_t *dict, dict_t *xdata)
 {
-        call_stub_t     *stub = NULL;
-
-        if (!index_xattrop_track (this, flags, dict))
-                goto out;
-
-        frame->local = inode_ref (loc->inode);
-        stub = fop_xattrop_stub (frame, index_xattrop_wrapper,
-                                 loc, flags, dict, xdata);
-        if (!stub) {
-                INDEX_STACK_UNWIND (xattrop, frame, -1, ENOMEM, NULL, NULL);
-                return 0;
-        }
-
-        index_queue_process (this, loc->inode, stub);
+    call_stub_t *stub = NULL;
+    index_local_t *local = NULL;
+
+    if (!index_xattrop_track(this, flags, dict))
+        goto out;
+
+    local = mem_get0(this->local_pool);
+    if (!local)
+        goto err;
+
+    frame->local = local;
+    local->inode = inode_ref(loc->inode);
+    if (xdata)
+        local->xdata = dict_ref(xdata);
+    stub = fop_xattrop_stub(frame, index_xattrop_wrapper, loc, flags, dict,
+                            xdata);
+
+err:
+    if ((!local) || (!stub)) {
+        INDEX_STACK_UNWIND(xattrop, frame, -1, ENOMEM, NULL, NULL);
         return 0;
+    }
+
+    index_queue_process(this, loc->inode, stub);
+    return 0;
 out:
-        STACK_WIND (frame, default_xattrop_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->xattrop, loc, flags, dict, xdata);
-        return 0;
+    STACK_WIND(frame, default_xattrop_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->xattrop, loc, flags, dict, xdata);
+    return 0;
 }
 
 int32_t
-index_fxattrop (call_frame_t *frame, xlator_t *this, fd_t *fd,
-		gf_xattrop_flags_t flags, dict_t *dict, dict_t *xdata)
+index_fxattrop(call_frame_t *frame, xlator_t *this, fd_t *fd,
+               gf_xattrop_flags_t flags, dict_t *dict, dict_t *xdata)
 {
-        call_stub_t    *stub = NULL;
-
-        if (!index_xattrop_track (this, flags, dict))
-                goto out;
-
-        frame->local = inode_ref (fd->inode);
-        stub = fop_fxattrop_stub (frame, index_fxattrop_wrapper,
-                                  fd, flags, dict, xdata);
-        if (!stub) {
-                INDEX_STACK_UNWIND (fxattrop, frame, -1, ENOMEM, NULL, xdata);
-                return 0;
-        }
-
-        index_queue_process (this, fd->inode, stub);
+    call_stub_t *stub = NULL;
+    index_local_t *local = NULL;
+
+    if (!index_xattrop_track(this, flags, dict))
+        goto out;
+
+    local = mem_get0(this->local_pool);
+    if (!local)
+        goto err;
+
+    frame->local = local;
+    local->inode = inode_ref(fd->inode);
+    if (xdata)
+        local->xdata = dict_ref(xdata);
+    stub = fop_fxattrop_stub(frame, index_fxattrop_wrapper, fd, flags, dict,
+                             xdata);
+
+err:
+    if ((!local) || (!stub)) {
+        INDEX_STACK_UNWIND(fxattrop, frame, -1, ENOMEM, NULL, xdata);
         return 0;
+    }
+
+    index_queue_process(this, fd->inode, stub);
+    return 0;
 out:
-        STACK_WIND (frame, default_fxattrop_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->fxattrop, fd, flags, dict, xdata);
-        return 0;
+    STACK_WIND(frame, default_fxattrop_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->fxattrop, fd, flags, dict, xdata);
+    return 0;
 }
 
 uint64_t
-index_entry_count (xlator_t *this, char *subdir)
+index_entry_count(xlator_t *this, char *subdir)
 {
-	index_priv_t *priv = NULL;
-	char index_dir[PATH_MAX];
-	DIR *dirp = NULL;
-	uint64_t count = 0;
-	struct dirent buf;
-	struct dirent *entry = NULL;
+    uint64_t count = 0;
+    index_priv_t *priv = NULL;
+    DIR *dirp = NULL;
+    struct dirent *entry = NULL;
+    struct dirent scratch[2] = {
+        {
+            0,
+        },
+    };
+    char index_dir[PATH_MAX] = {
+        0,
+    };
+
+    priv = this->private;
 
-	priv = this->private;
+    make_index_dir_path(priv->index_basepath, subdir, index_dir,
+                        sizeof(index_dir));
 
-	make_index_dir_path (priv->index_basepath, subdir,
-			     index_dir, sizeof (index_dir));
+    dirp = sys_opendir(index_dir);
+    if (!dirp)
+        return 0;
 
-	dirp = opendir (index_dir);
-	if (!dirp)
-		return 0;
+    for (;;) {
+        errno = 0;
+        entry = sys_readdir(dirp, scratch);
+        if (!entry || errno != 0)
+            break;
 
-	while (readdir_r (dirp, &buf, &entry) == 0) {
-		if (!entry)
-			break;
-		if (!strcmp (entry->d_name, ".") ||
-		    !strcmp (entry->d_name, ".."))
-			continue;
-                if (!strncmp (entry->d_name, subdir, strlen (subdir)))
-			continue;
-		count++;
-	}
-	closedir (dirp);
+        if (strcmp(entry->d_name, ".") == 0 || strcmp(entry->d_name, "..") == 0)
+            continue;
 
-	return count;
-}
+        if (!strncmp(entry->d_name, subdir, strlen(subdir)))
+            continue;
 
+        count++;
+    }
+
+    (void)sys_closedir(dirp);
+
+    return count;
+}
 
 int32_t
-index_getxattr_wrapper (call_frame_t *frame, xlator_t *this,
-                        loc_t *loc, const char *name, dict_t *xdata)
+index_getxattr_wrapper(call_frame_t *frame, xlator_t *this, loc_t *loc,
+                       const char *name, dict_t *xdata)
 {
-        index_priv_t    *priv = NULL;
-        dict_t          *xattr = NULL;
-        int             ret = 0;
-	uint64_t        count = 0;
+    index_priv_t *priv = NULL;
+    dict_t *xattr = NULL;
+    int ret = 0;
+    int vgfid_type = 0;
+    uint64_t count = 0;
+
+    priv = this->private;
+
+    xattr = dict_new();
+    if (!xattr) {
+        ret = -ENOMEM;
+        goto done;
+    }
+
+    vgfid_type = index_get_type_from_vgfid_xattr(name);
+    if (vgfid_type >= 0) {
+        ret = dict_set_static_bin(xattr, (char *)name,
+                                  priv->internal_vgfid[vgfid_type],
+                                  sizeof(priv->internal_vgfid[vgfid_type]));
+        if (ret) {
+            ret = -EINVAL;
+            gf_msg(this->name, GF_LOG_ERROR, -ret, INDEX_MSG_DICT_SET_FAILED,
+                   "xattrop index "
+                   "gfid set failed");
+            goto done;
+        }
+    }
 
-        priv = this->private;
+    /* TODO: Need to check what kind of link-counts are needed for
+     * ENTRY-CHANGES before refactor of this block with array*/
+    if (strcmp(name, GF_XATTROP_INDEX_COUNT) == 0) {
+        count = index_entry_count(this, XATTROP_SUBDIR);
 
-        xattr = dict_new ();
-        if (!xattr) {
-                ret = -ENOMEM;
-                goto done;
+        ret = dict_set_uint64(xattr, (char *)name, count);
+        if (ret) {
+            ret = -EINVAL;
+            gf_msg(this->name, GF_LOG_ERROR, -ret, INDEX_MSG_DICT_SET_FAILED,
+                   "xattrop index "
+                   "count set failed");
+            goto done;
         }
+    } else if (strcmp(name, GF_XATTROP_DIRTY_COUNT) == 0) {
+        count = index_entry_count(this, DIRTY_SUBDIR);
 
-	if (strcmp (name, GF_XATTROP_INDEX_GFID) == 0) {
-		ret = dict_set_static_bin (xattr, (char*)name, priv->xattrop_vgfid,
-					   sizeof (priv->xattrop_vgfid));
-		if (ret) {
-			ret = -ENOMEM;
-			gf_log (this->name, GF_LOG_ERROR, "xattrop index "
-				"gfid set failed");
-			goto done;
-		}
-	} else if (strcmp (name, GF_XATTROP_INDEX_COUNT) == 0) {
-		count = index_entry_count (this, XATTROP_SUBDIR);
-
-		ret = dict_set_uint64 (xattr, (char *)name, count);
-		if (ret) {
-			ret = -ENOMEM;
-			gf_log (this->name, GF_LOG_ERROR, "xattrop index "
-				"count set failed");
-			goto done;
-		}
-	}
+        ret = dict_set_uint64(xattr, (char *)name, count);
+        if (ret) {
+            ret = -EINVAL;
+            gf_msg(this->name, GF_LOG_ERROR, -ret, INDEX_MSG_DICT_SET_FAILED,
+                   "dirty index "
+                   "count set failed");
+            goto done;
+        }
+    }
 done:
-        if (ret)
-                STACK_UNWIND_STRICT (getxattr, frame, -1, -ret, xattr, xdata);
-        else
-                STACK_UNWIND_STRICT (getxattr, frame, 0, 0, xattr, xdata);
+    if (ret)
+        STACK_UNWIND_STRICT(getxattr, frame, -1, -ret, xattr, NULL);
+    else
+        STACK_UNWIND_STRICT(getxattr, frame, 0, 0, xattr, NULL);
 
-        if (xattr)
-                dict_unref (xattr);
+    if (xattr)
+        dict_unref(xattr);
 
+    return 0;
+}
+
+static int
+index_save_pargfid_for_entry_changes(xlator_t *this, loc_t *loc, char *path)
+{
+    index_priv_t *priv = NULL;
+    index_inode_ctx_t *ctx = NULL;
+    int ret = 0;
+
+    priv = this->private;
+    if (!loc)
+        return -1;
+    if (gf_uuid_compare(loc->pargfid, priv->internal_vgfid[ENTRY_CHANGES]))
         return 0;
+
+    ret = index_inode_ctx_get(loc->inode, this, &ctx);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, EINVAL,
+               INDEX_MSG_INODE_CTX_GET_SET_FAILED,
+               "Unable to get inode context for %s", path);
+        return -EINVAL;
+    }
+    ret = gf_uuid_parse(loc->name, ctx->virtual_pargfid);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, EINVAL,
+               INDEX_MSG_INODE_CTX_GET_SET_FAILED,
+               "Unable to store "
+               "virtual gfid in inode context for %s",
+               path);
+        return -EINVAL;
+    }
+    return 0;
 }
 
 int32_t
-index_lookup_wrapper (call_frame_t *frame, xlator_t *this,
-                      loc_t *loc, dict_t *xattr_req)
-{
-        index_priv_t    *priv = NULL;
-        struct stat     lstatbuf = {0};
-        int             ret = 0;
-        int32_t         op_errno = EINVAL;
-        int32_t         op_ret = -1;
-        char            path[PATH_MAX] = {0};
-        struct iatt     stbuf        = {0, };
-        struct iatt     postparent = {0,};
-        dict_t          *xattr = NULL;
-        gf_boolean_t    is_dir = _gf_false;
-
-        priv = this->private;
-
-        VALIDATE_OR_GOTO (loc, done);
-        if (!gf_uuid_compare (loc->gfid, priv->xattrop_vgfid)) {
-                make_index_dir_path (priv->index_basepath, XATTROP_SUBDIR,
-                                     path, sizeof (path));
-                is_dir = _gf_true;
-        } else if (!gf_uuid_compare (loc->pargfid, priv->xattrop_vgfid)) {
-                make_file_path (priv->index_basepath, XATTROP_SUBDIR,
-                                loc->name, path, sizeof (path));
+index_lookup_wrapper(call_frame_t *frame, xlator_t *this, loc_t *loc,
+                     dict_t *xattr_req)
+{
+    index_priv_t *priv = NULL;
+    struct stat lstatbuf = {0};
+    int ret = 0;
+    int32_t op_errno = EINVAL;
+    int32_t op_ret = -1;
+    uint64_t val = IA_INVAL;
+    char path[PATH_MAX] = {0};
+    struct iatt stbuf = {
+        0,
+    };
+    struct iatt postparent = {
+        0,
+    };
+    dict_t *xattr = NULL;
+    gf_boolean_t is_dir = _gf_false;
+    char *subdir = NULL;
+    loc_t iloc = {0};
+
+    priv = this->private;
+    loc_copy(&iloc, loc);
+
+    VALIDATE_OR_GOTO(loc, done);
+    if (index_is_fop_on_internal_inode(this, loc->parent, loc->pargfid)) {
+        subdir = index_get_subdir_from_vgfid(priv, loc->pargfid);
+        ret = index_inode_path(this, loc->parent, path, sizeof(path));
+        if (ret < 0) {
+            op_errno = -ret;
+            goto done;
         }
+        ret = snprintf(path + strlen(path), PATH_MAX - strlen(path), "/%s",
+                       loc->name);
 
-        ret = lstat (path, &lstatbuf);
+        if ((ret < 0) || (ret > (PATH_MAX - strlen(path)))) {
+            op_errno = EINVAL;
+            op_ret = -1;
+            goto done;
+        }
+
+    } else if (index_is_virtual_gfid(priv, loc->gfid)) {
+        subdir = index_get_subdir_from_vgfid(priv, loc->gfid);
+        make_index_dir_path(priv->index_basepath, subdir, path, sizeof(path));
+        is_dir = _gf_true;
+
+        if ((xattr_req) && (dict_get(xattr_req, GF_INDEX_IA_TYPE_GET_REQ))) {
+            if (0 == strcmp(subdir, index_get_subdir_from_type(ENTRY_CHANGES)))
+                val = IA_IFDIR;
+            else
+                val = IA_IFREG;
+        }
+    } else {
+        if (!inode_is_linked(loc->inode)) {
+            inode_unref(iloc.inode);
+            iloc.inode = inode_find(loc->inode->table, loc->gfid);
+        }
+        ret = index_inode_path(this, iloc.inode, path, sizeof(path));
+        if (ret < 0) {
+            op_errno = -ret;
+            goto done;
+        }
+    }
+    ret = sys_lstat(path, &lstatbuf);
+    if (ret) {
+        gf_msg_debug(this->name, errno, "Stat failed on %s dir ", path);
+        op_errno = errno;
+        goto done;
+    } else if (!S_ISDIR(lstatbuf.st_mode) && is_dir) {
+        op_errno = ENOTDIR;
+        gf_msg_debug(this->name, op_errno,
+                     "Stat failed on %s dir, "
+                     "not a directory",
+                     path);
+        goto done;
+    }
+    xattr = dict_new();
+    if (!xattr) {
+        op_errno = ENOMEM;
+        goto done;
+    }
+
+    if (val != IA_INVAL) {
+        ret = dict_set_uint64(xattr, GF_INDEX_IA_TYPE_GET_RSP, val);
         if (ret) {
-                gf_log (this->name, GF_LOG_DEBUG, "Stat failed on index dir "
-                        "(%s)", strerror (errno));
-                op_errno = errno;
-                goto done;
-        } else if (!S_ISDIR (lstatbuf.st_mode) && is_dir) {
-                gf_log (this->name, GF_LOG_DEBUG, "Stat failed on index dir, "
-                        "not a directory");
-                op_errno = ENOENT;
-                goto done;
+            op_ret = -1;
+            op_errno = -ret;
+            goto done;
         }
-        xattr = dict_new ();
-        if (!xattr) {
-                op_errno = ENOMEM;
-                goto done;
+    }
+
+    iatt_from_stat(&stbuf, &lstatbuf);
+    if (is_dir || inode_is_linked(iloc.inode))
+        loc_gfid(&iloc, stbuf.ia_gfid);
+    else
+        gf_uuid_generate(stbuf.ia_gfid);
+
+    ret = index_save_pargfid_for_entry_changes(this, &iloc, path);
+    if (ret) {
+        op_ret = -1;
+        op_errno = -ret;
+        goto done;
+    }
+
+    stbuf.ia_ino = -1;
+    op_ret = 0;
+done:
+    STACK_UNWIND_STRICT(lookup, frame, op_ret, op_errno,
+                        loc ? loc->inode : NULL, &stbuf, xattr, &postparent);
+    if (xattr)
+        dict_unref(xattr);
+    loc_wipe(&iloc);
+    return 0;
+}
+
+int
+index_get_gfid_type(void *opaque)
+{
+    gf_dirent_t *entry = NULL;
+    xlator_t *this = THIS;
+    struct index_syncop_args *args = opaque;
+    loc_t loc = {0};
+    struct iatt iatt = {0};
+    int ret = 0;
+
+    list_for_each_entry(entry, &args->entries->list, list)
+    {
+        if (strcmp(entry->d_name, ".") == 0 || strcmp(entry->d_name, "..") == 0)
+            continue;
+
+        loc_wipe(&loc);
+
+        entry->d_type = gf_d_type_from_ia_type(IA_INVAL);
+        entry->d_stat.ia_type = IA_INVAL;
+        if (gf_uuid_parse(entry->d_name, loc.gfid))
+            continue;
+
+        loc.inode = inode_find(args->parent->table, loc.gfid);
+        if (loc.inode) {
+            entry->d_stat.ia_type = loc.inode->ia_type;
+            entry->d_type = gf_d_type_from_ia_type(loc.inode->ia_type);
+            continue;
+        }
+        loc.inode = inode_new(args->parent->table);
+        if (!loc.inode)
+            continue;
+        ret = syncop_lookup(FIRST_CHILD(this), &loc, &iatt, 0, 0, 0);
+        if (ret == 0) {
+            entry->d_type = gf_d_type_from_ia_type(iatt.ia_type);
+            entry->d_stat = iatt;
         }
+    }
+    loc_wipe(&loc);
 
-        iatt_from_stat (&stbuf, &lstatbuf);
-        if (is_dir)
-                gf_uuid_copy (stbuf.ia_gfid, priv->xattrop_vgfid);
-        else
-                gf_uuid_generate (stbuf.ia_gfid);
-        stbuf.ia_ino = -1;
-        op_ret = 0;
-done:
-        STACK_UNWIND_STRICT (lookup, frame, op_ret, op_errno,
-                             loc->inode, &stbuf, xattr, &postparent);
-        if (xattr)
-                dict_unref (xattr);
-        return 0;
+    return 0;
 }
 
 int32_t
-index_readdir_wrapper (call_frame_t *frame, xlator_t *this,
-                       fd_t *fd, size_t size, off_t off, dict_t *xdata)
+index_readdir_wrapper(call_frame_t *frame, xlator_t *this, fd_t *fd,
+                      size_t size, off_t off, dict_t *xdata)
 {
-        index_fd_ctx_t       *fctx           = NULL;
-        DIR                  *dir            = NULL;
-        int                   ret            = -1;
-        int32_t               op_ret         = -1;
-        int32_t               op_errno       = 0;
-        int                   count          = 0;
-        gf_dirent_t           entries;
+    index_fd_ctx_t *fctx = NULL;
+    index_priv_t *priv = NULL;
+    DIR *dir = NULL;
+    int ret = -1;
+    int32_t op_ret = -1;
+    int32_t op_errno = 0;
+    int count = 0;
+    gf_dirent_t entries;
+    struct index_syncop_args args = {0};
+
+    priv = this->private;
+    INIT_LIST_HEAD(&entries.list);
+
+    ret = index_fd_ctx_get(fd, this, &fctx);
+    if (ret < 0) {
+        op_errno = -ret;
+        gf_msg(this->name, GF_LOG_WARNING, op_errno, INDEX_MSG_FD_OP_FAILED,
+               "pfd is NULL, fd=%p", fd);
+        goto done;
+    }
+
+    dir = fctx->dir;
+    if (!dir) {
+        op_errno = EINVAL;
+        gf_msg(this->name, GF_LOG_WARNING, op_errno,
+               INDEX_MSG_INDEX_READDIR_FAILED, "dir is NULL for fd=%p", fd);
+        goto done;
+    }
+
+    count = index_fill_readdir(fd, fctx, dir, off, size, &entries);
+
+    /* pick ENOENT to indicate EOF */
+    op_errno = errno;
+    op_ret = count;
+    if (index_is_virtual_gfid(priv, fd->inode->gfid) && xdata &&
+        dict_get(xdata, "get-gfid-type")) {
+        args.parent = fd->inode;
+        args.entries = &entries;
+        ret = synctask_new(this->ctx->env, index_get_gfid_type, NULL, NULL,
+                           &args);
+    }
+done:
+    STACK_UNWIND_STRICT(readdir, frame, op_ret, op_errno, &entries, NULL);
+    gf_dirent_free(&entries);
+    return 0;
+}
 
-        INIT_LIST_HEAD (&entries.list);
+int
+deletion_handler(const char *fpath, const struct stat *sb, int typeflag,
+                 struct FTW *ftwbuf)
+{
+    ia_type_t type = IA_INVAL;
+
+    switch (sb->st_mode & S_IFMT) {
+        case S_IFREG:
+            sys_unlink(fpath);
+            break;
+
+        case S_IFDIR:
+            sys_rmdir(fpath);
+            break;
+        default:
+            type = ia_type_from_st_mode(sb->st_mode);
+            gf_msg(THIS->name, GF_LOG_WARNING, EINVAL, INDEX_MSG_INVALID_ARGS,
+                   "%s neither a regular file nor a directory - type:%s", fpath,
+                   gf_inode_type_to_str(type));
+            break;
+    }
+    return 0;
+}
 
-        ret = index_fd_ctx_get (fd, this, &fctx);
-        if (ret < 0) {
-                gf_log (this->name, GF_LOG_WARNING,
-                        "pfd is NULL, fd=%p", fd);
-                op_errno = -ret;
-                goto done;
-        }
+static int
+index_wipe_index_subdir(void *opaque)
+{
+    struct index_syncop_args *args = opaque;
 
-        dir = fctx->dir;
+    nftw(args->path, deletion_handler, 1, FTW_DEPTH | FTW_PHYS);
+    return 0;
+}
 
-        if (!dir) {
-                gf_log (this->name, GF_LOG_WARNING,
-                        "dir is NULL for fd=%p", fd);
-                op_errno = EINVAL;
-                goto done;
-        }
+static void
+index_get_parent_iatt(struct iatt *parent, char *path, loc_t *loc,
+                      int32_t *op_ret, int32_t *op_errno)
+{
+    int ret = -1;
+    struct stat lstatbuf = {
+        0,
+    };
+
+    ret = sys_lstat(path, &lstatbuf);
+    if (ret < 0) {
+        *op_ret = -1;
+        *op_errno = errno;
+        return;
+    }
 
-        count = index_fill_readdir (fd, fctx, dir, off, size, &entries);
+    iatt_from_stat(parent, &lstatbuf);
+    gf_uuid_copy(parent->ia_gfid, loc->pargfid);
+    parent->ia_ino = -1;
 
-        /* pick ENOENT to indicate EOF */
-        op_errno = errno;
-        op_ret = count;
-done:
-        STACK_UNWIND_STRICT (readdir, frame, op_ret, op_errno, &entries, xdata);
-        gf_dirent_free (&entries);
-        return 0;
+    return;
 }
 
 int
-index_unlink_wrapper (call_frame_t *frame, xlator_t *this, loc_t *loc, int flag,
-                      dict_t *xdata)
-{
-        index_priv_t    *priv = NULL;
-        int32_t         op_ret = 0;
-        int32_t         op_errno = 0;
-        int             ret = 0;
-        struct  iatt    preparent = {0};
-        struct  iatt    postparent = {0};
-        char            index_dir[PATH_MAX] = {0};
-        struct  stat    lstatbuf = {0};
-        uuid_t          gfid = {0};
-
-        priv = this->private;
-        make_index_dir_path (priv->index_basepath, XATTROP_SUBDIR,
-                             index_dir, sizeof (index_dir));
-        ret = lstat (index_dir, &lstatbuf);
+index_rmdir_wrapper(call_frame_t *frame, xlator_t *this, loc_t *loc, int flag,
+                    dict_t *xdata)
+{
+    int ret = 0;
+    int32_t op_ret = 0;
+    int32_t op_errno = 0;
+    char *subdir = NULL;
+    char index_dir[PATH_MAX] = {0};
+    char index_subdir[PATH_MAX] = {0};
+    uuid_t gfid = {0};
+    struct iatt preparent = {0};
+    struct iatt postparent = {0};
+    index_priv_t *priv = NULL;
+    index_xattrop_type_t type = XATTROP_TYPE_UNSET;
+    struct index_syncop_args args = {
+        0,
+    };
+
+    priv = this->private;
+
+    type = index_get_type_from_vgfid(priv, loc->pargfid);
+    subdir = index_get_subdir_from_vgfid(priv, loc->pargfid);
+    make_index_dir_path(priv->index_basepath, subdir, index_dir,
+                        sizeof(index_dir));
+
+    index_get_parent_iatt(&preparent, index_dir, loc, &op_ret, &op_errno);
+    if (op_ret < 0)
+        goto done;
+
+    gf_uuid_parse(loc->name, gfid);
+    make_gfid_path(priv->index_basepath, subdir, gfid, index_subdir,
+                   sizeof(index_subdir));
+
+    if (flag == 0) {
+        ret = index_del(this, gfid, subdir, type);
         if (ret < 0) {
-                op_ret = -1;
-                op_errno = errno;
-                goto done;
+            op_ret = -1;
+            op_errno = -ret;
+            goto done;
         }
+    } else {
+        args.path = index_subdir;
+        ret = synctask_new(this->ctx->env, index_wipe_index_subdir, NULL, NULL,
+                           &args);
+    }
 
-        iatt_from_stat (&preparent, &lstatbuf);
-        gf_uuid_copy (preparent.ia_gfid, priv->xattrop_vgfid);
-        preparent.ia_ino = -1;
-        gf_uuid_parse (loc->name, gfid);
-        ret = index_del (this, gfid, XATTROP_SUBDIR);
-        if (ret < 0) {
-                op_ret = -1;
-                op_errno = -ret;
-                goto done;
+    index_get_parent_iatt(&postparent, index_dir, loc, &op_ret, &op_errno);
+    if (op_ret < 0)
+        goto done;
+
+done:
+    INDEX_STACK_UNWIND(rmdir, frame, op_ret, op_errno, &preparent, &postparent,
+                       xdata);
+    return 0;
+}
+
+int
+index_unlink_wrapper(call_frame_t *frame, xlator_t *this, loc_t *loc, int flag,
+                     dict_t *xdata)
+{
+    index_priv_t *priv = NULL;
+    index_inode_ctx_t *ictx = NULL;
+    int32_t op_ret = 0;
+    int32_t op_errno = 0;
+    int ret = 0;
+    index_xattrop_type_t type = XATTROP_TYPE_UNSET;
+    struct iatt preparent = {0};
+    struct iatt postparent = {0};
+    char index_dir[PATH_MAX] = {0};
+    char filepath[PATH_MAX] = {0};
+    uuid_t gfid = {0};
+    char *subdir = NULL;
+
+    priv = this->private;
+    type = index_get_type_from_vgfid(priv, loc->pargfid);
+    ret = index_inode_path(this, loc->parent, index_dir, sizeof(index_dir));
+    if (ret < 0) {
+        op_ret = -1;
+        op_errno = -ret;
+        goto done;
+    }
+
+    index_get_parent_iatt(&preparent, index_dir, loc, &op_ret, &op_errno);
+    if (op_ret < 0)
+        goto done;
+
+    if (type <= XATTROP_TYPE_UNSET) {
+        ret = index_inode_ctx_get(loc->parent, this, &ictx);
+        if ((ret == 0) && gf_uuid_is_null(ictx->virtual_pargfid)) {
+            ret = -EINVAL;
         }
-        memset (&lstatbuf, 0, sizeof (lstatbuf));
-        ret = lstat (index_dir, &lstatbuf);
-        if (ret < 0) {
-                op_ret = -1;
-                op_errno = errno;
-                goto done;
+        if (ret == 0) {
+            ret = index_entry_delete(this, ictx->virtual_pargfid,
+                                     (char *)loc->name);
         }
-        iatt_from_stat (&postparent, &lstatbuf);
-        gf_uuid_copy (postparent.ia_gfid, priv->xattrop_vgfid);
-        postparent.ia_ino = -1;
+    } else if (type == ENTRY_CHANGES) {
+        make_file_path(priv->index_basepath, ENTRY_CHANGES_SUBDIR,
+                       (char *)loc->name, filepath, sizeof(filepath));
+        ret = sys_unlink(filepath);
+    } else {
+        subdir = index_get_subdir_from_type(type);
+        gf_uuid_parse(loc->name, gfid);
+        ret = index_del(this, gfid, subdir, type);
+    }
+    if (ret < 0) {
+        op_ret = -1;
+        op_errno = -ret;
+        goto done;
+    }
+
+    index_get_parent_iatt(&postparent, index_dir, loc, &op_ret, &op_errno);
+    if (op_ret < 0)
+        goto done;
 done:
-        INDEX_STACK_UNWIND (unlink, frame, op_ret, op_errno, &preparent,
-                            &postparent, xdata);
-        return 0;
+    INDEX_STACK_UNWIND(unlink, frame, op_ret, op_errno, &preparent, &postparent,
+                       xdata);
+    return 0;
 }
 
 int32_t
-index_getxattr (call_frame_t *frame, xlator_t *this,
-                loc_t *loc, const char *name, dict_t *xdata)
+index_getxattr(call_frame_t *frame, xlator_t *this, loc_t *loc,
+               const char *name, dict_t *xdata)
 {
-        call_stub_t     *stub = NULL;
-        index_priv_t    *priv = NULL;
+    call_stub_t *stub = NULL;
 
-        priv = this->private;
+    if (!name ||
+        (!index_is_vgfid_xattr(name) && strcmp(GF_XATTROP_INDEX_COUNT, name) &&
+         strcmp(GF_XATTROP_DIRTY_COUNT, name)))
+        goto out;
 
-        if (!name || (strcmp (GF_XATTROP_INDEX_GFID, name) &&
-		      strcmp (GF_XATTROP_INDEX_COUNT, name)))
-                goto out;
-
-        stub = fop_getxattr_stub (frame, index_getxattr_wrapper, loc, name,
-                                  xdata);
-        if (!stub) {
-                STACK_UNWIND_STRICT (getxattr, frame, -1, ENOMEM, NULL, NULL);
-                return 0;
-        }
-        worker_enqueue (this, stub);
+    stub = fop_getxattr_stub(frame, index_getxattr_wrapper, loc, name, xdata);
+    if (!stub) {
+        STACK_UNWIND_STRICT(getxattr, frame, -1, ENOMEM, NULL, NULL);
         return 0;
+    }
+    worker_enqueue(this, stub);
+    return 0;
 out:
-        STACK_WIND (frame, default_getxattr_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->getxattr, loc, name, xdata);
-        return 0;
+    STACK_WIND(frame, default_getxattr_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->getxattr, loc, name, xdata);
+    return 0;
 }
 
-int32_t
-index_lookup (call_frame_t *frame, xlator_t *this,
-              loc_t *loc, dict_t *xattr_req)
+int64_t
+index_fetch_link_count(xlator_t *this, index_xattrop_type_t type)
 {
-        call_stub_t     *stub = NULL;
-        index_priv_t    *priv = NULL;
+    index_priv_t *priv = this->private;
+    char *subdir = NULL;
+    struct stat lstatbuf = {
+        0,
+    };
+    int ret = -1;
+    int64_t count = -1;
+    DIR *dirp = NULL;
+    struct dirent *entry = NULL;
+    struct dirent scratch[2] = {
+        {
+            0,
+        },
+    };
+    char index_dir[PATH_MAX] = {
+        0,
+    };
+    char index_path[PATH_MAX] = {
+        0,
+    };
+
+    subdir = index_get_subdir_from_type(type);
+    make_index_dir_path(priv->index_basepath, subdir, index_dir,
+                        sizeof(index_dir));
+
+    dirp = sys_opendir(index_dir);
+    if (!dirp)
+        goto out;
+
+    for (;;) {
+        errno = 0;
+        entry = sys_readdir(dirp, scratch);
+        if (!entry || errno != 0) {
+            if (count == -1)
+                count = 0;
+            goto out;
+        }
 
-        priv = this->private;
+        if (strcmp(entry->d_name, ".") == 0 || strcmp(entry->d_name, "..") == 0)
+            continue;
 
-        if (gf_uuid_compare (loc->gfid, priv->xattrop_vgfid) &&
-            gf_uuid_compare (loc->pargfid, priv->xattrop_vgfid))
-                goto normal;
+        make_file_path(priv->index_basepath, subdir, entry->d_name, index_path,
+                       sizeof(index_path));
 
-        stub = fop_lookup_stub (frame, index_lookup_wrapper, loc, xattr_req);
-        if (!stub) {
-                STACK_UNWIND_STRICT (lookup, frame, -1, ENOMEM, loc->inode,
-                                     NULL, NULL, NULL);
-                return 0;
+        ret = sys_lstat(index_path, &lstatbuf);
+        if (ret < 0) {
+            count = -2;
+            continue;
+        } else {
+            count = lstatbuf.st_nlink - 1;
+            if (count == 0)
+                continue;
+            else
+                break;
         }
-        worker_enqueue (this, stub);
-        return 0;
-normal:
-        STACK_WIND (frame, default_lookup_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->lookup, loc, xattr_req);
+    }
+out:
+    if (dirp)
+        (void)sys_closedir(dirp);
+    return count;
+}
 
-        return 0;
+dict_t *
+index_fill_link_count(xlator_t *this, dict_t *xdata)
+{
+    int ret = -1;
+    index_priv_t *priv = NULL;
+    int64_t count = -1;
+
+    priv = this->private;
+    xdata = (xdata) ? dict_ref(xdata) : dict_new();
+    if (!xdata)
+        goto out;
+
+    index_get_link_count(priv, &count, XATTROP);
+    if (count < 0) {
+        count = index_fetch_link_count(this, XATTROP);
+        index_set_link_count(priv, count, XATTROP);
+    }
+
+    if (count == 0) {
+        ret = dict_set_int8(xdata, "link-count", 0);
+        if (ret < 0)
+            gf_msg(this->name, GF_LOG_ERROR, EINVAL, INDEX_MSG_DICT_SET_FAILED,
+                   "Unable to set link-count");
+    } else {
+        ret = dict_set_int8(xdata, "link-count", 1);
+        if (ret < 0)
+            gf_msg(this->name, GF_LOG_ERROR, EINVAL, INDEX_MSG_DICT_SET_FAILED,
+                   "Unable to set link-count");
+    }
+
+out:
+    return xdata;
 }
 
 int32_t
-index_opendir (call_frame_t *frame, xlator_t *this,
-               loc_t *loc, fd_t *fd, dict_t *xdata)
+index_lookup_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                 int32_t op_ret, int32_t op_errno, inode_t *inode,
+                 struct iatt *buf, dict_t *xdata, struct iatt *postparent)
 {
-        index_priv_t    *priv = NULL;
+    xdata = index_fill_link_count(this, xdata);
+    STACK_UNWIND_STRICT(lookup, frame, op_ret, op_errno, inode, buf, xdata,
+                        postparent);
+    if (xdata)
+        dict_unref(xdata);
+    return 0;
+}
 
-        priv = this->private;
-        if (gf_uuid_compare (fd->inode->gfid, priv->xattrop_vgfid))
+int32_t
+index_lookup(call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xattr_req)
+{
+    inode_t *inode = NULL;
+    call_stub_t *stub = NULL;
+    char *flag = NULL;
+    int ret = -1;
+
+    if (!index_is_fop_on_internal_inode(this, loc->parent, loc->pargfid) &&
+        !index_is_fop_on_internal_inode(this, loc->inode, loc->gfid)) {
+        if (!inode_is_linked(loc->inode)) {
+            inode = inode_find(loc->inode->table, loc->gfid);
+            if (!index_is_fop_on_internal_inode(this, inode, loc->gfid)) {
+                inode_unref(inode);
                 goto normal;
+            }
+            inode_unref(inode);
+        } else {
+            goto normal;
+        }
+    }
 
-        frame->local = NULL;
-        STACK_UNWIND_STRICT (opendir, frame, 0, 0, fd, NULL);
+    stub = fop_lookup_stub(frame, index_lookup_wrapper, loc, xattr_req);
+    if (!stub) {
+        STACK_UNWIND_STRICT(lookup, frame, -1, ENOMEM, loc->inode, NULL, NULL,
+                            NULL);
         return 0;
-
+    }
+    worker_enqueue(this, stub);
+    return 0;
 normal:
-        STACK_WIND (frame, default_opendir_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->opendir, loc, fd, xdata);
-        return 0;
+    ret = dict_get_str_sizen(xattr_req, "link-count", &flag);
+    if ((ret == 0) && (strcmp(flag, GF_XATTROP_INDEX_COUNT) == 0)) {
+        STACK_WIND(frame, index_lookup_cbk, FIRST_CHILD(this),
+                   FIRST_CHILD(this)->fops->lookup, loc, xattr_req);
+    } else {
+        STACK_WIND(frame, default_lookup_cbk, FIRST_CHILD(this),
+                   FIRST_CHILD(this)->fops->lookup, loc, xattr_req);
+    }
+
+    return 0;
 }
 
 int32_t
-index_readdir (call_frame_t *frame, xlator_t *this,
-               fd_t *fd, size_t size, off_t off, dict_t *xdata)
+index_fstat_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                int32_t op_ret, int32_t op_errno, struct iatt *buf,
+                dict_t *xdata)
 {
-        call_stub_t     *stub = NULL;
-        index_priv_t    *priv = NULL;
+    xdata = index_fill_link_count(this, xdata);
+    STACK_UNWIND_STRICT(fstat, frame, op_ret, op_errno, buf, xdata);
+    if (xdata)
+        dict_unref(xdata);
+    return 0;
+}
 
-        priv = this->private;
-        if (gf_uuid_compare (fd->inode->gfid, priv->xattrop_vgfid))
-                goto out;
-        stub = fop_readdir_stub (frame, index_readdir_wrapper, fd, size, off,
-                                 xdata);
-        if (!stub) {
-                STACK_UNWIND_STRICT (readdir, frame, -1, ENOMEM, NULL, NULL);
-                return 0;
-        }
-        worker_enqueue (this, stub);
-        return 0;
-out:
-        STACK_WIND (frame, default_readdir_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->readdir, fd, size, off, xdata);
-        return 0;
+int32_t
+index_fstat(call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *xdata)
+{
+    int ret = -1;
+    char *flag = NULL;
+
+    ret = dict_get_str(xdata, "link-count", &flag);
+    if ((ret == 0) && (strcmp(flag, GF_XATTROP_INDEX_COUNT) == 0)) {
+        STACK_WIND(frame, index_fstat_cbk, FIRST_CHILD(this),
+                   FIRST_CHILD(this)->fops->fstat, fd, xdata);
+    } else {
+        STACK_WIND(frame, default_fstat_cbk, FIRST_CHILD(this),
+                   FIRST_CHILD(this)->fops->fstat, fd, xdata);
+    }
+
+    return 0;
 }
 
-int
-index_unlink (call_frame_t *frame, xlator_t *this, loc_t *loc, int xflag,
+int32_t
+index_opendir(call_frame_t *frame, xlator_t *this, loc_t *loc, fd_t *fd,
               dict_t *xdata)
 {
-        call_stub_t     *stub = NULL;
-        index_priv_t    *priv = NULL;
+    if (!index_is_fop_on_internal_inode(this, fd->inode, NULL))
+        goto normal;
 
-        priv = this->private;
-        if (gf_uuid_compare (loc->pargfid, priv->xattrop_vgfid))
-                goto out;
+    frame->local = NULL;
+    STACK_UNWIND_STRICT(opendir, frame, 0, 0, fd, NULL);
+    return 0;
 
-        stub = fop_unlink_stub (frame, index_unlink_wrapper, loc, xflag, xdata);
-        if (!stub) {
-                STACK_UNWIND_STRICT (unlink, frame, -1, ENOMEM, NULL, NULL,
-                                     NULL);
-                return 0;
-        }
-        worker_enqueue (this, stub);
-        return 0;
-out:
-        STACK_WIND (frame, default_unlink_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->unlink, loc, xflag, xdata);
-        return 0;
+normal:
+    STACK_WIND(frame, default_opendir_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->opendir, loc, fd, xdata);
+    return 0;
 }
 
-int
-index_make_xattrop64_watchlist (xlator_t *this, index_priv_t *priv,
-                                char *watchlist)
-{
-        char   *delim         = NULL;
-        char   *dup_watchlist = NULL;
-        char   *key           = NULL;
-        char   *saveptr       = NULL;
-        dict_t *xattrs        = NULL;
-        data_t *dummy         = NULL;
-        int    ret            = 0;
-
-        if (!watchlist)
-                return 0;
-
-        dup_watchlist = gf_strdup (watchlist);
-        if (!dup_watchlist)
-                return -1;
-
-        xattrs = dict_new ();
-        if (!xattrs) {
-                ret = -1;
-                goto out;
-        }
-
-        dummy = int_to_data (1);
-        if (!dummy) {
-                ret = -1;
-                goto out;
-        }
-
-        data_ref (dummy);
-
-        delim = ",";
-        key = strtok_r (dup_watchlist, delim, &saveptr);
-        while (key) {
-                if (strlen (key) == 0) {
-                        ret = -1;
-                        goto out;
-                }
-
-                ret = dict_set (xattrs, key, dummy);
-                if (ret)
-                        goto out;
-
-                key = strtok_r (NULL, delim, &saveptr);
-        }
+int32_t
+index_readdir(call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
+              off_t off, dict_t *xdata)
+{
+    call_stub_t *stub = NULL;
 
-        priv->xattrop64_watchlist = xattrs;
-        xattrs = NULL;
+    if (!index_is_fop_on_internal_inode(this, fd->inode, NULL))
+        goto out;
 
-        ret = 0;
+    stub = fop_readdir_stub(frame, index_readdir_wrapper, fd, size, off, xdata);
+    if (!stub) {
+        STACK_UNWIND_STRICT(readdir, frame, -1, ENOMEM, NULL, NULL);
+        return 0;
+    }
+    worker_enqueue(this, stub);
+    return 0;
 out:
-        if (xattrs)
-                dict_unref (xattrs);
+    STACK_WIND(frame, default_readdir_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->readdir, fd, size, off, xdata);
+    return 0;
+}
 
-        GF_FREE (dup_watchlist);
+int
+index_unlink(call_frame_t *frame, xlator_t *this, loc_t *loc, int xflag,
+             dict_t *xdata)
+{
+    call_stub_t *stub = NULL;
 
-        if (dummy)
-                data_unref (dummy);
+    if (!index_is_fop_on_internal_inode(this, loc->parent, NULL))
+        goto out;
 
-        return ret;
+    stub = fop_unlink_stub(frame, index_unlink_wrapper, loc, xflag, xdata);
+    if (!stub) {
+        STACK_UNWIND_STRICT(unlink, frame, -1, ENOMEM, NULL, NULL, NULL);
+        return 0;
+    }
+    worker_enqueue(this, stub);
+    return 0;
+out:
+    STACK_WIND(frame, default_unlink_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->unlink, loc, xflag, xdata);
+    return 0;
 }
 
-int32_t
-mem_acct_init (xlator_t *this)
+int
+index_rmdir(call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t flags,
+            dict_t *xdata)
 {
-        int     ret = -1;
+    call_stub_t *stub = NULL;
 
-        ret = xlator_mem_acct_init (this, gf_index_mt_end + 1);
+    if (!index_is_fop_on_internal_inode(this, loc->parent, NULL))
+        goto out;
 
-        return ret;
+    stub = fop_rmdir_stub(frame, index_rmdir_wrapper, loc, flags, xdata);
+    if (!stub) {
+        STACK_UNWIND_STRICT(rmdir, frame, -1, ENOMEM, NULL, NULL, NULL);
+        return 0;
+    }
+    worker_enqueue(this, stub);
+    return 0;
+out:
+    STACK_WIND_TAIL(frame, FIRST_CHILD(this), FIRST_CHILD(this)->fops->rmdir,
+                    loc, flags, xdata);
+    return 0;
 }
 
 int
-init (xlator_t *this)
-{
-        int ret = -1;
-        index_priv_t *priv = NULL;
-        pthread_t thread;
-        pthread_attr_t  w_attr;
-        gf_boolean_t    mutex_inited = _gf_false;
-        gf_boolean_t    cond_inited  = _gf_false;
-        gf_boolean_t    attr_inited  = _gf_false;
-        char            *watchlist = NULL;
-
-	if (!this->children || this->children->next) {
-		gf_log (this->name, GF_LOG_ERROR,
-			"'index' not configured with exactly one child");
-                goto out;
-	}
+index_make_xattrop_watchlist(xlator_t *this, index_priv_t *priv,
+                             char *watchlist, index_xattrop_type_t type)
+{
+    char *delim = NULL;
+    char *dup_watchlist = NULL;
+    char *key = NULL;
+    char *saveptr = NULL;
+    dict_t *xattrs = NULL;
+    data_t *dummy = NULL;
+    int ret = 0;
+
+    if (!watchlist)
+        return 0;
 
-	if (!this->parents) {
-		gf_log (this->name, GF_LOG_WARNING,
-			"dangling volume. check volfile ");
-	}
+    dup_watchlist = gf_strdup(watchlist);
+    if (!dup_watchlist)
+        return -1;
+
+    xattrs = dict_new();
+    if (!xattrs) {
+        ret = -1;
+        goto out;
+    }
+
+    dummy = int_to_data(1);
+    if (!dummy) {
+        ret = -1;
+        goto out;
+    }
+
+    data_ref(dummy);
+
+    delim = ",";
+    key = strtok_r(dup_watchlist, delim, &saveptr);
+    while (key) {
+        if (strlen(key) == 0) {
+            ret = -1;
+            goto out;
+        }
 
-        priv = GF_CALLOC (1, sizeof (*priv), gf_index_mt_priv_t);
-        if (!priv)
-                goto out;
+        ret = dict_set(xattrs, key, dummy);
+        if (ret)
+            goto out;
 
-        LOCK_INIT (&priv->lock);
-        if ((ret = pthread_cond_init(&priv->cond, NULL)) != 0) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "pthread_cond_init failed (%d)", ret);
-                goto out;
-        }
-        cond_inited = _gf_true;
+        key = strtok_r(NULL, delim, &saveptr);
+    }
 
-        if ((ret = pthread_mutex_init(&priv->mutex, NULL)) != 0) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "pthread_mutex_init failed (%d)", ret);
+    switch (type) {
+        case DIRTY:
+            priv->dirty_watchlist = dict_copy_with_ref(xattrs,
+                                                       priv->dirty_watchlist);
+            if (!priv->dirty_watchlist) {
+                ret = -1;
                 goto out;
-        }
-        mutex_inited = _gf_true;
-
-        if ((ret = pthread_attr_init (&w_attr)) != 0) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "pthread_attr_init failed (%d)", ret);
+            }
+            break;
+        case XATTROP:
+            priv->pending_watchlist = dict_copy_with_ref(
+                xattrs, priv->pending_watchlist);
+            if (!priv->pending_watchlist) {
+                ret = -1;
                 goto out;
-        }
-        attr_inited = _gf_true;
+            }
+            break;
+        default:
+            break;
+    }
 
-        ret = pthread_attr_setstacksize (&w_attr, INDEX_THREAD_STACK_SIZE);
-        if (ret == EINVAL) {
-                gf_log (this->name, GF_LOG_WARNING,
-                        "Using default thread stack size");
-        }
+    ret = 0;
+out:
+    if (xattrs)
+        dict_unref(xattrs);
 
-        GF_OPTION_INIT ("index-base", priv->index_basepath, path, out);
+    GF_FREE(dup_watchlist);
 
-        GF_OPTION_INIT ("xattrop64-watchlist", watchlist, str, out);
-        ret = index_make_xattrop64_watchlist (this, priv, watchlist);
-        if (ret)
-                goto out;
+    if (dummy)
+        data_unref(dummy);
 
-        gf_uuid_generate (priv->index);
-        gf_uuid_generate (priv->xattrop_vgfid);
-        INIT_LIST_HEAD (&priv->callstubs);
+    return ret;
+}
 
-        this->private = priv;
+int32_t
+mem_acct_init(xlator_t *this)
+{
+    int ret = -1;
 
-        ret = index_dir_create (this, XATTROP_SUBDIR);
-        if (ret < 0)
-                goto out;
+    ret = xlator_mem_acct_init(this, gf_index_mt_end + 1);
 
-        ret = gf_thread_create (&thread, &w_attr, index_worker, this);
-        if (ret) {
-                gf_log (this->name, GF_LOG_WARNING, "Failed to create "
-                        "worker thread, aborting");
-                goto out;
-        }
+    return ret;
+}
 
-        ret = 0;
+int
+init(xlator_t *this)
+{
+    int i = 0;
+    int ret = -1;
+    int64_t count = -1;
+    index_priv_t *priv = NULL;
+    pthread_attr_t w_attr;
+    gf_boolean_t mutex_inited = _gf_false;
+    gf_boolean_t cond_inited = _gf_false;
+    gf_boolean_t attr_inited = _gf_false;
+    char *watchlist = NULL;
+    char *dirtylist = NULL;
+    char *pendinglist = NULL;
+    char *index_base_parent = NULL;
+    char *tmp = NULL;
+
+    if (!this->children || this->children->next) {
+        gf_msg(this->name, GF_LOG_ERROR, EINVAL, INDEX_MSG_INVALID_GRAPH,
+               "'index' not configured with exactly one child");
+        goto out;
+    }
+
+    if (!this->parents) {
+        gf_msg(this->name, GF_LOG_WARNING, EINVAL, INDEX_MSG_INVALID_GRAPH,
+               "dangling volume. check volfile ");
+    }
+
+    priv = GF_CALLOC(1, sizeof(*priv), gf_index_mt_priv_t);
+    if (!priv)
+        goto out;
+
+    LOCK_INIT(&priv->lock);
+    if ((ret = pthread_cond_init(&priv->cond, NULL)) != 0) {
+        gf_msg(this->name, GF_LOG_ERROR, ret, INDEX_MSG_INVALID_ARGS,
+               "pthread_cond_init failed");
+        goto out;
+    }
+    cond_inited = _gf_true;
+
+    if ((ret = pthread_mutex_init(&priv->mutex, NULL)) != 0) {
+        gf_msg(this->name, GF_LOG_ERROR, ret, INDEX_MSG_INVALID_ARGS,
+               "pthread_mutex_init failed");
+        goto out;
+    }
+    mutex_inited = _gf_true;
+
+    if ((ret = pthread_attr_init(&w_attr)) != 0) {
+        gf_msg(this->name, GF_LOG_ERROR, ret, INDEX_MSG_INVALID_ARGS,
+               "pthread_attr_init failed");
+        goto out;
+    }
+    attr_inited = _gf_true;
+
+    ret = pthread_attr_setstacksize(&w_attr, INDEX_THREAD_STACK_SIZE);
+    if (ret == EINVAL) {
+        gf_msg(this->name, GF_LOG_WARNING, ret, INDEX_MSG_INVALID_ARGS,
+               "Using default thread stack size");
+    }
+
+    GF_OPTION_INIT("index-base", priv->index_basepath, path, out);
+    tmp = gf_strdup(priv->index_basepath);
+    index_base_parent = dirname(tmp);
+    if (gf_lstat_dir(index_base_parent, NULL) != 0) {
+        ret = -1;
+        gf_msg(this->name, GF_LOG_ERROR, errno,
+               INDEX_MSG_INDEX_DIR_CREATE_FAILED,
+               "Failed to find parent dir (%s) of index basepath %s.",
+               index_base_parent, priv->index_basepath);
+        goto out;
+    }
+
+    GF_OPTION_INIT("xattrop64-watchlist", watchlist, str, out);
+    ret = index_make_xattrop_watchlist(this, priv, watchlist, XATTROP);
+    if (ret)
+        goto out;
+
+    GF_OPTION_INIT("xattrop-dirty-watchlist", dirtylist, str, out);
+    ret = index_make_xattrop_watchlist(this, priv, dirtylist, DIRTY);
+    if (ret)
+        goto out;
+
+    GF_OPTION_INIT("xattrop-pending-watchlist", pendinglist, str, out);
+    ret = index_make_xattrop_watchlist(this, priv, pendinglist, XATTROP);
+    if (ret)
+        goto out;
+
+    if (priv->dirty_watchlist)
+        priv->complete_watchlist = dict_copy_with_ref(priv->dirty_watchlist,
+                                                      priv->complete_watchlist);
+    if (priv->pending_watchlist)
+        priv->complete_watchlist = dict_copy_with_ref(priv->pending_watchlist,
+                                                      priv->complete_watchlist);
+
+    gf_uuid_generate(priv->index);
+    for (i = 0; i < XATTROP_TYPE_END; i++)
+        gf_uuid_generate(priv->internal_vgfid[i]);
+
+    INIT_LIST_HEAD(&priv->callstubs);
+    GF_ATOMIC_INIT(priv->stub_cnt, 0);
+
+    this->local_pool = mem_pool_new(index_local_t, 64);
+    if (!this->local_pool) {
+        ret = -1;
+        goto out;
+    }
+
+    this->private = priv;
+
+    ret = index_dir_create(this, XATTROP_SUBDIR);
+    if (ret < 0)
+        goto out;
+
+    if (priv->dirty_watchlist) {
+        ret = index_dir_create(this, DIRTY_SUBDIR);
+        if (ret < 0)
+            goto out;
+    }
+
+    ret = index_dir_create(this, ENTRY_CHANGES_SUBDIR);
+    if (ret < 0)
+        goto out;
+
+    /*init indices files counts*/
+    count = index_fetch_link_count(this, XATTROP);
+    index_set_link_count(priv, count, XATTROP);
+    priv->down = _gf_false;
+
+    priv->curr_count = 0;
+    ret = gf_thread_create(&priv->thread, &w_attr, index_worker, this,
+                           "idxwrker");
+    if (ret) {
+        gf_msg(this->name, GF_LOG_WARNING, ret,
+               INDEX_MSG_WORKER_THREAD_CREATE_FAILED,
+               "Failed to create worker thread, aborting");
+        goto out;
+    }
+    priv->curr_count++;
+    ret = 0;
 out:
-        if (ret) {
-                if (cond_inited)
-                        pthread_cond_destroy (&priv->cond);
-                if (mutex_inited)
-                        pthread_mutex_destroy (&priv->mutex);
-                if (priv && priv->xattrop64_watchlist)
-                        dict_unref (priv->xattrop64_watchlist);
-                if (priv)
-                        GF_FREE (priv);
-                this->private = NULL;
-        }
-        if (attr_inited)
-                pthread_attr_destroy (&w_attr);
-        return ret;
+    GF_FREE(tmp);
+
+    if (ret) {
+        if (cond_inited)
+            pthread_cond_destroy(&priv->cond);
+        if (mutex_inited)
+            pthread_mutex_destroy(&priv->mutex);
+        if (priv && priv->dirty_watchlist)
+            dict_unref(priv->dirty_watchlist);
+        if (priv && priv->pending_watchlist)
+            dict_unref(priv->pending_watchlist);
+        if (priv && priv->complete_watchlist)
+            dict_unref(priv->complete_watchlist);
+        if (priv)
+            GF_FREE(priv);
+        this->private = NULL;
+        mem_pool_destroy(this->local_pool);
+        this->local_pool = NULL;
+    }
+
+    if (attr_inited)
+        pthread_attr_destroy(&w_attr);
+    return ret;
 }
 
 void
-fini (xlator_t *this)
+fini(xlator_t *this)
 {
-        index_priv_t *priv = NULL;
-
-        priv = this->private;
-        if (!priv)
-                goto out;
-        this->private = NULL;
-        LOCK_DESTROY (&priv->lock);
-        pthread_cond_destroy (&priv->cond);
-        pthread_mutex_destroy (&priv->mutex);
-        if (priv->xattrop64_watchlist)
-                dict_unref (priv->xattrop64_watchlist);
-        GF_FREE (priv);
+    index_priv_t *priv = NULL;
+
+    priv = this->private;
+    if (!priv)
+        goto out;
+
+    priv->down = _gf_true;
+    pthread_cond_broadcast(&priv->cond);
+    if (priv->thread) {
+        gf_thread_cleanup_xint(priv->thread);
+        priv->thread = 0;
+    }
+    this->private = NULL;
+    LOCK_DESTROY(&priv->lock);
+    pthread_cond_destroy(&priv->cond);
+    pthread_mutex_destroy(&priv->mutex);
+    if (priv->dirty_watchlist)
+        dict_unref(priv->dirty_watchlist);
+    if (priv->pending_watchlist)
+        dict_unref(priv->pending_watchlist);
+    if (priv->complete_watchlist)
+        dict_unref(priv->complete_watchlist);
+    GF_FREE(priv);
+
+    if (this->local_pool) {
+        mem_pool_destroy(this->local_pool);
+        this->local_pool = NULL;
+    }
 out:
-        return;
+    return;
 }
 
 int
-index_forget (xlator_t *this, inode_t *inode)
+index_forget(xlator_t *this, inode_t *inode)
 {
-        uint64_t tmp_cache = 0;
-        if (!inode_ctx_del (inode, this, &tmp_cache))
-                GF_FREE ((index_inode_ctx_t*) (long)tmp_cache);
+    uint64_t tmp_cache = 0;
+    if (!inode_ctx_del(inode, this, &tmp_cache))
+        GF_FREE((index_inode_ctx_t *)(long)tmp_cache);
 
-        return 0;
+    return 0;
 }
 
 int32_t
-index_releasedir (xlator_t *this, fd_t *fd)
+index_releasedir(xlator_t *this, fd_t *fd)
 {
-        index_fd_ctx_t *fctx = NULL;
-        uint64_t        ctx = 0;
-        int             ret = 0;
+    index_fd_ctx_t *fctx = NULL;
+    uint64_t ctx = 0;
+    int ret = 0;
 
-        ret = fd_ctx_del (fd, this, &ctx);
-        if (ret < 0)
-                goto out;
+    ret = fd_ctx_del(fd, this, &ctx);
+    if (ret < 0)
+        goto out;
 
-        fctx = (index_fd_ctx_t*) (long) ctx;
-        if (fctx->dir) {
-                ret = closedir (fctx->dir);
-                if (ret)
-                        gf_log (this->name, GF_LOG_ERROR, "closedir error: %s", strerror (errno));
-        }
+    fctx = (index_fd_ctx_t *)(long)ctx;
+    if (fctx->dir) {
+        ret = sys_closedir(fctx->dir);
+        if (ret)
+            gf_msg(this->name, GF_LOG_ERROR, errno, INDEX_MSG_FD_OP_FAILED,
+                   "closedir error");
+    }
 
-        GF_FREE (fctx);
+    GF_FREE(fctx);
 out:
-        return 0;
+    return 0;
 }
 
 int32_t
-index_release (xlator_t *this, fd_t *fd)
+index_release(xlator_t *this, fd_t *fd)
 {
-        index_fd_ctx_t *fctx = NULL;
-        uint64_t        ctx = 0;
-        int             ret = 0;
+    index_fd_ctx_t *fctx = NULL;
+    uint64_t ctx = 0;
+    int ret = 0;
 
-        ret = fd_ctx_del (fd, this, &ctx);
-        if (ret < 0)
-                goto out;
+    ret = fd_ctx_del(fd, this, &ctx);
+    if (ret < 0)
+        goto out;
 
-        fctx = (index_fd_ctx_t*) (long) ctx;
-        GF_FREE (fctx);
+    fctx = (index_fd_ctx_t *)(long)ctx;
+    GF_FREE(fctx);
 out:
-        return 0;
+    return 0;
 }
 
 int
-notify (xlator_t *this, int event, void *data, ...)
+notify(xlator_t *this, int event, void *data, ...)
 {
-        int     ret = 0;
-        ret = default_notify (this, event, data);
-        return ret;
+    int ret = 0;
+    index_priv_t *priv = NULL;
+    uint64_t stub_cnt = 0;
+    xlator_t *victim = data;
+    struct timespec sleep_till = {
+        0,
+    };
+
+    if (!this)
+        return 0;
+
+    priv = this->private;
+    if (!priv)
+        return 0;
+
+    if ((event == GF_EVENT_PARENT_DOWN) && victim->cleanup_starting) {
+        stub_cnt = GF_ATOMIC_GET(priv->stub_cnt);
+        timespec_now_realtime(&sleep_till);
+        sleep_till.tv_sec += 1;
+
+        /* Wait for draining stub from queue before notify PARENT_DOWN */
+        pthread_mutex_lock(&priv->mutex);
+        {
+            while (stub_cnt) {
+                (void)pthread_cond_timedwait(&priv->cond, &priv->mutex,
+                                             &sleep_till);
+                stub_cnt = GF_ATOMIC_GET(priv->stub_cnt);
+            }
+        }
+        pthread_mutex_unlock(&priv->mutex);
+        gf_log(this->name, GF_LOG_INFO,
+               "Notify GF_EVENT_PARENT_DOWN for brick %s", victim->name);
+    }
+
+    if ((event == GF_EVENT_CHILD_DOWN) && victim->cleanup_starting) {
+        pthread_mutex_lock(&priv->mutex);
+        {
+            priv->down = _gf_true;
+            pthread_cond_broadcast(&priv->cond);
+            while (priv->curr_count)
+                pthread_cond_wait(&priv->cond, &priv->mutex);
+        }
+        pthread_mutex_unlock(&priv->mutex);
+
+        gf_log(this->name, GF_LOG_INFO,
+               "Notify GF_EVENT_CHILD_DOWN for brick %s", victim->name);
+    }
+
+    ret = default_notify(this, event, data);
+    return ret;
 }
 
 struct xlator_fops fops = {
-	.xattrop     = index_xattrop,
-	.fxattrop    = index_fxattrop,
-
-        //interface functions follow
-        .getxattr    = index_getxattr,
-        .lookup      = index_lookup,
-        .opendir     = index_opendir,
-        .readdir     = index_readdir,
-        .unlink      = index_unlink
+    .xattrop = index_xattrop,
+    .fxattrop = index_fxattrop,
+
+    // interface functions follow
+    .getxattr = index_getxattr,
+    .lookup = index_lookup,
+    .opendir = index_opendir,
+    .readdir = index_readdir,
+    .unlink = index_unlink,
+    .rmdir = index_rmdir,
+    .fstat = index_fstat,
 };
 
 struct xlator_dumpops dumpops;
 
-struct xlator_cbks cbks = {
-        .forget         = index_forget,
-        .release        = index_release,
-        .releasedir     = index_releasedir
-};
+struct xlator_cbks cbks = {.forget = index_forget,
+                           .release = index_release,
+                           .releasedir = index_releasedir};
 
 struct volume_options options[] = {
-        { .key  = {"index-base" },
-          .type = GF_OPTION_TYPE_PATH,
-          .description = "path where the index files need to be stored",
-        },
-        { .key  = {"xattrop64-watchlist" },
-          .type = GF_OPTION_TYPE_STR,
-          .description = "Comma separated list of xattrs that are watched",
-        },
-        { .key  = {NULL} },
+    {.key = {"index-base"},
+     .type = GF_OPTION_TYPE_PATH,
+     .description = "path where the index files need to be stored",
+     .default_value = "{{ brick.path }}/.glusterfs/indices"},
+    {.key = {"xattrop64-watchlist"},
+     .type = GF_OPTION_TYPE_STR,
+     .description = "Comma separated list of xattrs that are watched",
+     .default_value = "trusted.ec.dirty"},
+    {.key = {"xattrop-dirty-watchlist"},
+     .type = GF_OPTION_TYPE_STR,
+     .description = "Comma separated list of xattrs that are watched",
+     .default_value = "trusted.afr.dirty"},
+    {.key = {"xattrop-pending-watchlist"},
+     .type = GF_OPTION_TYPE_STR,
+     .description = "Comma separated list of xattrs that are watched",
+     .default_value = "trusted.afr.{{ volume.name }}"},
+    {.key = {NULL}},
+};
+
+xlator_api_t xlator_api = {
+    .init = init,
+    .fini = fini,
+    .notify = notify,
+    .mem_acct_init = mem_acct_init,
+    .op_version = {1}, /* Present from the initial version */
+    .dumpops = &dumpops,
+    .fops = &fops,
+    .cbks = &cbks,
+    .options = options,
+    .identifier = "index",
+    .category = GF_MAINTAINED,
 };
diff --git a/xlators/features/index/src/index.h b/xlators/features/index/src/index.h
index a8dfe067ae1..a2b6e6e2570 100644
--- a/xlators/features/index/src/index.h
+++ b/xlators/features/index/src/index.h
@@ -11,51 +11,76 @@
 #ifndef __INDEX_H__
 #define __INDEX_H__
 
-#include "xlator.h"
-#include "call-stub.h"
-#include "defaults.h"
-#include "byte-order.h"
-#include "common-utils.h"
+#include <glusterfs/xlator.h>
+#include <glusterfs/call-stub.h>
+#include <glusterfs/defaults.h>
+#include <glusterfs/byte-order.h>
+#include <glusterfs/common-utils.h>
 #include "index-mem-types.h"
 
-#define INDEX_THREAD_STACK_SIZE   ((size_t)(1024*1024))
+#define INDEX_THREAD_STACK_SIZE ((size_t)(1024 * 1024))
+
+typedef enum { UNKNOWN, IN, NOTIN } index_state_t;
 
 typedef enum {
-        UNKNOWN,
-        IN,
-        NOTIN
-} index_state_t;
+    XATTROP_TYPE_UNSET = -1,
+    XATTROP,
+    DIRTY,
+    ENTRY_CHANGES,
+    XATTROP_TYPE_END
+} index_xattrop_type_t;
 
 typedef struct index_inode_ctx {
-        gf_boolean_t processing;
-        struct list_head callstubs;
-        index_state_t state;
+    gf_boolean_t processing;
+    struct list_head callstubs;
+    int state[XATTROP_TYPE_END];
+    uuid_t virtual_pargfid; /* virtual gfid of dir under
+                              .glusterfs/indices/entry-changes. */
 } index_inode_ctx_t;
 
 typedef struct index_fd_ctx {
-        DIR *dir;
-        off_t dir_eof;
+    DIR *dir;
+    off_t dir_eof;
 } index_fd_ctx_t;
 
 typedef struct index_priv {
-        char *index_basepath;
-        uuid_t index;
-        gf_lock_t lock;
-        uuid_t xattrop_vgfid;//virtual gfid of the xattrop index dir
-        struct list_head callstubs;
-        pthread_mutex_t mutex;
-        pthread_cond_t  cond;
-        dict_t  *xattrop64_watchlist;
+    char *index_basepath;
+    char *dirty_basepath;
+    uuid_t index;
+    gf_lock_t lock;
+    uuid_t internal_vgfid[XATTROP_TYPE_END];
+    struct list_head callstubs;
+    pthread_mutex_t mutex;
+    pthread_cond_t cond;
+    dict_t *dirty_watchlist;
+    dict_t *pending_watchlist;
+    dict_t *complete_watchlist;
+    int64_t pending_count;
+    pthread_t thread;
+    gf_boolean_t down;
+    gf_atomic_t stub_cnt;
+    int32_t curr_count;
 } index_priv_t;
 
-#define INDEX_STACK_UNWIND(fop, frame, params ...)      \
-do {                                                    \
-        if (frame) {                                    \
-                inode_t *_inode = frame->local;         \
-                frame->local = NULL;                    \
-                inode_unref (_inode);                   \
-        }                                               \
-        STACK_UNWIND_STRICT (fop, frame, params);       \
-} while (0)
+typedef struct index_local {
+    inode_t *inode;
+    dict_t *xdata;
+} index_local_t;
+
+#define INDEX_STACK_UNWIND(fop, frame, params...)                              \
+    do {                                                                       \
+        index_local_t *__local = NULL;                                         \
+        if (frame) {                                                           \
+            __local = frame->local;                                            \
+            frame->local = NULL;                                               \
+        }                                                                      \
+        STACK_UNWIND_STRICT(fop, frame, params);                               \
+        if (__local) {                                                         \
+            inode_unref(__local->inode);                                       \
+            if (__local->xdata)                                                \
+                dict_unref(__local->xdata);                                    \
+            mem_put(__local);                                                  \
+        }                                                                      \
+    } while (0)
 
 #endif
diff --git a/xlators/encryption/crypt/Makefile.am b/xlators/features/leases/Makefile.am
index d471a3f9243..a985f42a877 100644
--- a/xlators/encryption/crypt/Makefile.am
+++ b/xlators/features/leases/Makefile.am
@@ -1,3 +1,3 @@
 SUBDIRS = src
 
-CLEANFILES = 
+CLEANFILES =
diff --git a/xlators/features/leases/src/Makefile.am b/xlators/features/leases/src/Makefile.am
new file mode 100644
index 00000000000..a1aef10e299
--- /dev/null
+++ b/xlators/features/leases/src/Makefile.am
@@ -0,0 +1,20 @@
+if WITH_SERVER
+xlator_LTLIBRARIES = leases.la
+endif
+xlatordir = $(libdir)/glusterfs/$(PACKAGE_VERSION)/xlator/features
+
+leases_la_LDFLAGS = -module $(GF_XLATOR_DEFAULT_LDFLAGS)
+
+leases_la_SOURCES = leases.c leases-internal.c
+
+leases_la_LIBADD = $(top_builddir)/libglusterfs/src/libglusterfs.la
+
+noinst_HEADERS = leases.h leases-mem-types.h leases-messages.h
+
+AM_CPPFLAGS = $(GF_CPPFLAGS) -I$(top_srcdir)/libglusterfs/src \
+	-I$(top_srcdir)/rpc/xdr/src -I$(top_builddir)/rpc/xdr/src \
+	-I$(CONTRIBDIR)/timer-wheel
+
+AM_CFLAGS = -Wall $(GF_CFLAGS)
+
+CLEANFILES =
diff --git a/xlators/features/leases/src/leases-internal.c b/xlators/features/leases/src/leases-internal.c
new file mode 100644
index 00000000000..56dee244281
--- /dev/null
+++ b/xlators/features/leases/src/leases-internal.c
@@ -0,0 +1,1412 @@
+/*
+   Copyright (c) 2015-2016 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
+
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
+
+#ifndef _CONFIG_H
+#define _CONFIG_H
+#include "config.h"
+#endif
+
+#include "leases.h"
+
+/* Mutex locks used in this xlator and their order of acquisition:
+ * Check lease conflict:
+ *         lease_ctx lock
+ *                 add_timer => internal timer locks
+ *         lease_ctx unlock
+ *
+ * Add/remove lease:
+ *         lease_ctx lock
+ *                 add_timer => internal timer locks
+ *                 OR
+ *                 priv lock => Adding/removing to/from the cleanup client list
+ *                 priv unlock
+ *         lease_ctx unlock
+ *
+ * Timer thread:
+ *         Timer internal lock
+ *                 priv lock => By timer handler
+ *                 priv unlock
+ *         Timer internal unlock
+ *
+ * Expired recall cleanup thread:
+ *         priv lock
+ *                 priv condwait
+ *         priv unlock
+ *         lease_ctx lock
+ *                 priv lock
+ *                 priv unlock
+ *         lease_ctx unlock
+ */
+
+/*
+ * Check if lease_lk is enabled
+ * Return Value:
+ * _gf_true  - lease lock option enabled
+ * _gf_false - lease lock option disabled
+ */
+gf_boolean_t
+is_leases_enabled(xlator_t *this)
+{
+    leases_private_t *priv = NULL;
+    gf_boolean_t is_enabled = _gf_false;
+
+    GF_VALIDATE_OR_GOTO("leases", this, out);
+
+    if (this->private) {
+        priv = (leases_private_t *)this->private;
+        is_enabled = priv->leases_enabled;
+    }
+out:
+    return is_enabled;
+}
+
+/*
+ * Get the recall_leaselk_timeout
+ * Return Value:
+ * timeout value(in seconds) set as an option to this xlator.
+ * -1 error case
+ */
+static int32_t
+get_recall_lease_timeout(xlator_t *this)
+{
+    leases_private_t *priv = NULL;
+    int32_t timeout = -1;
+
+    GF_VALIDATE_OR_GOTO("leases", this, out);
+
+    if (this->private) {
+        priv = (leases_private_t *)this->private;
+        timeout = priv->recall_lease_timeout;
+    }
+out:
+    return timeout;
+}
+
+static void
+__dump_leases_info(xlator_t *this, lease_inode_ctx_t *lease_ctx)
+{
+    lease_id_entry_t *lease_entry = NULL;
+    lease_id_entry_t *tmp = NULL;
+
+    GF_VALIDATE_OR_GOTO("leases", this, out);
+    GF_VALIDATE_OR_GOTO("leases", lease_ctx, out);
+
+    gf_msg_debug(this->name, 0,
+                 "Lease held on this inode, lease_type: %d,"
+                 " lease_cnt:%" PRIu64
+                 ", RD lease:%d, RW lease:%d, "
+                 "openfd cnt:%" PRIu64,
+                 lease_ctx->lease_type, lease_ctx->lease_cnt,
+                 lease_ctx->lease_type_cnt[GF_RD_LEASE],
+                 lease_ctx->lease_type_cnt[GF_RW_LEASE], lease_ctx->openfd_cnt);
+
+    list_for_each_entry_safe(lease_entry, tmp, &lease_ctx->lease_id_list,
+                             lease_id_list)
+    {
+        gf_msg_debug(this->name, 0,
+                     "Leases held by client: %s, lease "
+                     "ID:%s, RD lease:%d, RW lease:%d, lease_type: %d, "
+                     "lease_cnt:%" PRIu64,
+                     lease_entry->client_uid, lease_entry->lease_id,
+                     lease_entry->lease_type_cnt[GF_RD_LEASE],
+                     lease_entry->lease_type_cnt[GF_RW_LEASE],
+                     lease_entry->lease_type, lease_entry->lease_cnt);
+    }
+out:
+    return;
+}
+
+static int
+__lease_ctx_set(inode_t *inode, xlator_t *this)
+{
+    lease_inode_ctx_t *inode_ctx = NULL;
+    int ret = -1;
+    uint64_t ctx = 0;
+
+    GF_VALIDATE_OR_GOTO("leases", inode, out);
+    GF_VALIDATE_OR_GOTO("leases", this, out);
+
+    ret = __inode_ctx_get(inode, this, &ctx);
+    if (!ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, LEASE_MSG_INVAL_INODE_CTX,
+               "inode_ctx_get failed");
+        goto out;
+    }
+
+    inode_ctx = GF_CALLOC(1, sizeof(*inode_ctx),
+                          gf_leases_mt_lease_inode_ctx_t);
+    GF_CHECK_ALLOC(inode_ctx, ret, out);
+
+    pthread_mutex_init(&inode_ctx->lock, NULL);
+    INIT_LIST_HEAD(&inode_ctx->lease_id_list);
+    INIT_LIST_HEAD(&inode_ctx->blocked_list);
+
+    inode_ctx->lease_cnt = 0;
+
+    ret = __inode_ctx_set(inode, this, (uint64_t *)inode_ctx);
+    if (ret) {
+        GF_FREE(inode_ctx);
+        gf_msg(this->name, GF_LOG_INFO, 0, LEASE_MSG_INVAL_INODE_CTX,
+               "failed to set inode ctx (%p)", inode);
+    }
+out:
+    return ret;
+}
+
+static lease_inode_ctx_t *
+__lease_ctx_get(inode_t *inode, xlator_t *this)
+{
+    lease_inode_ctx_t *inode_ctx = NULL;
+    uint64_t ctx = 0;
+    int ret = 0;
+
+    GF_VALIDATE_OR_GOTO("leases", inode, out);
+    GF_VALIDATE_OR_GOTO("leases", this, out);
+
+    ret = __inode_ctx_get(inode, this, &ctx);
+    if (ret < 0) {
+        ret = __lease_ctx_set(inode, this);
+        if (ret < 0)
+            goto out;
+
+        ret = __inode_ctx_get(inode, this, &ctx);
+        if (ret < 0) {
+            gf_msg(this->name, GF_LOG_WARNING, 0, LEASE_MSG_INVAL_INODE_CTX,
+                   "failed to get inode ctx (%p)", inode);
+            goto out;
+        }
+    }
+
+    inode_ctx = (lease_inode_ctx_t *)(long)ctx;
+out:
+    return inode_ctx;
+}
+
+lease_inode_ctx_t *
+lease_ctx_get(inode_t *inode, xlator_t *this)
+{
+    lease_inode_ctx_t *inode_ctx = NULL;
+
+    GF_VALIDATE_OR_GOTO("leases", inode, out);
+    GF_VALIDATE_OR_GOTO("leases", this, out);
+
+    LOCK(&inode->lock);
+    {
+        inode_ctx = __lease_ctx_get(inode, this);
+    }
+    UNLOCK(&inode->lock);
+out:
+    return inode_ctx;
+}
+
+static lease_id_entry_t *
+new_lease_id_entry(call_frame_t *frame, const char *lease_id)
+{
+    lease_id_entry_t *lease_entry = NULL;
+
+    GF_VALIDATE_OR_GOTO("leases", frame, out);
+    GF_VALIDATE_OR_GOTO("leases", lease_id, out);
+
+    lease_entry = GF_CALLOC(1, sizeof(*lease_entry),
+                            gf_leases_mt_lease_id_entry_t);
+    if (!lease_entry) {
+        gf_msg(frame->this->name, GF_LOG_ERROR, ENOMEM, LEASE_MSG_NO_MEM,
+               "Memory allocation for lease_entry failed");
+        return NULL;
+    }
+
+    INIT_LIST_HEAD(&lease_entry->lease_id_list);
+    lease_entry->lease_type = NONE;
+    lease_entry->lease_cnt = 0;
+    lease_entry->recall_time = get_recall_lease_timeout(frame->this);
+    lease_entry->client_uid = gf_strdup(frame->root->client->client_uid);
+    if (!lease_entry->client_uid) {
+        gf_msg(frame->this->name, GF_LOG_ERROR, ENOMEM, LEASE_MSG_NO_MEM,
+               "Memory allocation for client_uid failed");
+        GF_FREE(lease_entry);
+        lease_entry = NULL;
+        goto out;
+    }
+
+    memcpy(lease_entry->lease_id, lease_id, LEASE_ID_SIZE);
+out:
+    return lease_entry;
+}
+
+static void
+__destroy_lease_id_entry(lease_id_entry_t *lease_entry)
+{
+    GF_VALIDATE_OR_GOTO("leases", lease_entry, out);
+
+    list_del_init(&lease_entry->lease_id_list);
+    GF_FREE(lease_entry->client_uid);
+    GF_FREE(lease_entry);
+out:
+    return;
+}
+
+static inline gf_boolean_t
+__is_same_lease_id(const char *k1, const char *k2)
+{
+    if (memcmp(k1, k2, strlen(k1)) == 0)
+        return _gf_true;
+
+    return _gf_false;
+}
+
+/* Checks if there are any leases, other than the leases taken
+ * by the given lease_id
+ */
+static gf_boolean_t
+__another_lease_found(lease_inode_ctx_t *lease_ctx, const char *lease_id)
+{
+    lease_id_entry_t *lease_entry = NULL;
+    lease_id_entry_t *tmp = NULL;
+    gf_boolean_t found_lease = _gf_false;
+
+    GF_VALIDATE_OR_GOTO("leases", lease_id, out);
+    GF_VALIDATE_OR_GOTO("leases", lease_ctx, out);
+
+    list_for_each_entry_safe(lease_entry, tmp, &lease_ctx->lease_id_list,
+                             lease_id_list)
+    {
+        if (!__is_same_lease_id(lease_id, lease_entry->lease_id)) {
+            if (lease_entry->lease_cnt > 0) {
+                found_lease = _gf_true;
+                break;
+            }
+        }
+    }
+out:
+    return found_lease;
+}
+
+/* Returns the lease_id_entry for a given lease_id and a given inode.
+ * Return values:
+ * NULL - If no client entry found
+ * lease_id_entry_t* - a pointer to the client entry if found
+ */
+static lease_id_entry_t *
+__get_lease_id_entry(lease_inode_ctx_t *lease_ctx, const char *lease_id)
+{
+    lease_id_entry_t *lease_entry = NULL;
+    lease_id_entry_t *tmp = NULL;
+    lease_id_entry_t *found = NULL;
+
+    GF_VALIDATE_OR_GOTO("leases", lease_id, out);
+    GF_VALIDATE_OR_GOTO("leases", lease_ctx, out);
+
+    list_for_each_entry_safe(lease_entry, tmp, &lease_ctx->lease_id_list,
+                             lease_id_list)
+    {
+        if (__is_same_lease_id(lease_id, lease_entry->lease_id)) {
+            found = lease_entry;
+            gf_msg_debug("leases", 0,
+                         "lease ID entry found "
+                         "Client UID:%s, lease id:%s",
+                         lease_entry->client_uid,
+                         leaseid_utoa(lease_entry->lease_id));
+            break;
+        }
+    }
+out:
+    return found;
+}
+
+/* Returns the lease_id_entry for a given lease_id and a given inode,
+ * if none found creates one.
+ * Return values:
+ * lease_id_entry_t* - a pointer to the client entry
+ */
+static lease_id_entry_t *
+__get_or_new_lease_entry(call_frame_t *frame, const char *lease_id,
+                         lease_inode_ctx_t *lease_ctx)
+{
+    lease_id_entry_t *lease_entry = NULL;
+
+    GF_VALIDATE_OR_GOTO("leases", frame, out);
+    GF_VALIDATE_OR_GOTO("leases", lease_id, out);
+    GF_VALIDATE_OR_GOTO("leases", lease_ctx, out);
+
+    lease_entry = __get_lease_id_entry(lease_ctx, lease_id);
+    if (!lease_entry) { /* create one */
+        lease_entry = new_lease_id_entry(frame, lease_id);
+        if (!lease_entry)
+            goto out;
+
+        list_add_tail(&lease_entry->lease_id_list, &lease_ctx->lease_id_list);
+
+        gf_msg_debug(frame->this->name, 0,
+                     "lease ID entry added,"
+                     " Client UID:%s, lease id:%s",
+                     lease_entry->client_uid,
+                     leaseid_utoa(lease_entry->lease_id));
+    }
+out:
+    return lease_entry;
+}
+
+static lease_inode_t *
+new_lease_inode(inode_t *inode)
+{
+    lease_inode_t *l_inode = GF_MALLOC(sizeof(*l_inode),
+                                       gf_leases_mt_lease_inode_t);
+    if (!l_inode)
+        goto out;
+
+    INIT_LIST_HEAD(&l_inode->list);
+    l_inode->inode = inode_ref(inode);
+out:
+    return l_inode;
+}
+
+static void
+__destroy_lease_inode(lease_inode_t *l_inode)
+{
+    list_del_init(&l_inode->list);
+    inode_unref(l_inode->inode);
+    GF_FREE(l_inode);
+}
+
+static lease_client_t *
+new_lease_client(const char *client_uid)
+{
+    lease_client_t *clnt = GF_MALLOC(sizeof(*clnt),
+                                     gf_leases_mt_lease_client_t);
+    if (!clnt)
+        goto out;
+
+    INIT_LIST_HEAD(&clnt->client_list);
+    INIT_LIST_HEAD(&clnt->inode_list);
+    clnt->client_uid = gf_strdup(client_uid);
+out:
+    return clnt;
+}
+
+static void
+__destroy_lease_client(lease_client_t *clnt)
+{
+    list_del_init(&clnt->inode_list);
+    list_del_init(&clnt->client_list);
+    GF_FREE(clnt);
+
+    return;
+}
+
+static lease_client_t *
+__get_lease_client(xlator_t *this, leases_private_t *priv,
+                   const char *client_uid)
+{
+    lease_client_t *clnt = NULL;
+    lease_client_t *tmp = NULL;
+    lease_client_t *found = NULL;
+
+    list_for_each_entry_safe(clnt, tmp, &priv->client_list, client_list)
+    {
+        if ((strcmp(clnt->client_uid, client_uid) == 0)) {
+            found = clnt;
+            gf_msg_debug(this->name, 0,
+                         "Client:%s already found "
+                         "in the cleanup list",
+                         client_uid);
+            break;
+        }
+    }
+    return found;
+}
+
+static lease_client_t *
+__get_or_new_lease_client(xlator_t *this, leases_private_t *priv,
+                          const char *client_uid)
+{
+    lease_client_t *found = NULL;
+
+    found = __get_lease_client(this, priv, client_uid);
+    if (!found) {
+        found = new_lease_client(client_uid);
+        if (!found)
+            goto out;
+        list_add_tail(&found->client_list, &priv->client_list);
+        gf_msg_debug(this->name, 0,
+                     "Adding a new client:%s entry "
+                     "to the cleanup list",
+                     client_uid);
+    }
+out:
+    return found;
+}
+
+static int
+add_inode_to_client_list(xlator_t *this, inode_t *inode, const char *client_uid)
+{
+    leases_private_t *priv = this->private;
+    lease_client_t *clnt = NULL;
+
+    lease_inode_t *lease_inode = new_lease_inode(inode);
+    if (!lease_inode)
+        return -ENOMEM;
+
+    pthread_mutex_lock(&priv->mutex);
+    {
+        clnt = __get_or_new_lease_client(this, priv, client_uid);
+        if (!clnt) {
+            pthread_mutex_unlock(&priv->mutex);
+            __destroy_lease_inode(lease_inode);
+            return -ENOMEM;
+        }
+        list_add_tail(&clnt->inode_list, &lease_inode->list);
+    }
+    pthread_mutex_unlock(&priv->mutex);
+    gf_msg_debug(this->name, 0,
+                 "Added a new inode:%p to the client(%s) "
+                 "cleanup list, gfid(%s)",
+                 inode, client_uid, uuid_utoa(inode->gfid));
+    return 0;
+}
+
+/* Add lease entry to the corresponding client entry.
+ * Return values:
+ * 0 Success
+ * -1 Failure
+ */
+static int
+__add_lease(call_frame_t *frame, inode_t *inode, lease_inode_ctx_t *lease_ctx,
+            const char *client_uid, struct gf_lease *lease)
+{
+    lease_id_entry_t *lease_entry = NULL;
+    int ret = -1;
+
+    GF_VALIDATE_OR_GOTO("leases", frame, out);
+    GF_VALIDATE_OR_GOTO("leases", client_uid, out);
+    GF_VALIDATE_OR_GOTO("leases", lease_ctx, out);
+    GF_VALIDATE_OR_GOTO("leases", inode, out);
+    GF_VALIDATE_OR_GOTO("leases", lease, out);
+
+    gf_msg_trace(frame->this->name, 0,
+                 "Granting lease lock to client %s with lease id %s"
+                 " on gfid(%s)",
+                 client_uid, leaseid_utoa(lease->lease_id),
+                 uuid_utoa(inode->gfid));
+
+    lease_entry = __get_or_new_lease_entry(frame, lease->lease_id, lease_ctx);
+    if (!lease_entry) {
+        errno = ENOMEM;
+        goto out;
+    }
+
+    lease_entry->lease_type_cnt[lease->lease_type]++;
+    lease_entry->lease_cnt++;
+    lease_entry->lease_type |= lease->lease_type;
+    /* If this is the first lease taken by the client on the file, then
+     * add this inode/file to the client disconnect cleanup list
+     */
+    if (lease_entry->lease_cnt == 1) {
+        add_inode_to_client_list(frame->this, inode, client_uid);
+    }
+
+    lease_ctx->lease_cnt++;
+    lease_ctx->lease_type_cnt[lease->lease_type]++;
+    lease_ctx->lease_type |= lease->lease_type;
+
+    /* Take a ref for the first lock taken on this inode. Corresponding
+     * unref when all the leases are unlocked or during DISCONNECT
+     * Ref is required because the inode on which lease is acquired should
+     * not be deleted when lru cleanup kicks in*/
+    if (lease_ctx->lease_cnt == 1) {
+        lease_ctx->inode = inode_ref(inode);
+    }
+
+    ret = 0;
+out:
+    return ret;
+}
+
+static gf_boolean_t
+__is_clnt_lease_none(const char *client_uid, lease_inode_ctx_t *lease_ctx)
+{
+    gf_boolean_t lease_none = _gf_true;
+    lease_id_entry_t *lease_entry = NULL;
+    lease_id_entry_t *tmp = NULL;
+
+    list_for_each_entry_safe(lease_entry, tmp, &lease_ctx->lease_id_list,
+                             lease_id_list)
+    {
+        if ((strcmp(client_uid, lease_entry->client_uid) == 0) &&
+            (lease_entry->lease_cnt != 0)) {
+            lease_none = _gf_false;
+            break;
+        }
+    }
+
+    return lease_none;
+}
+
+static int
+__remove_inode_from_clnt_list(xlator_t *this, lease_client_t *clnt,
+                              inode_t *inode)
+{
+    int ret = -1;
+    lease_inode_t *l_inode = NULL;
+    lease_inode_t *tmp1 = NULL;
+
+    list_for_each_entry_safe(l_inode, tmp1, &clnt->inode_list, list)
+    {
+        if (l_inode->inode == inode) {
+            __destroy_lease_inode(l_inode);
+            gf_msg_debug(this->name, 0,
+                         "Removed the inode from the client cleanup list");
+            ret = 0;
+        }
+    }
+    /* TODO: Remove the client entry from the cleanup list */
+
+    return ret;
+}
+
+static int
+remove_from_clnt_list(xlator_t *this, const char *client_uid, inode_t *inode)
+{
+    leases_private_t *priv = NULL;
+    int ret = -1;
+    lease_client_t *clnt = NULL;
+
+    priv = this->private;
+    if (!priv)
+        goto out;
+
+    pthread_mutex_lock(&priv->mutex);
+    {
+        clnt = __get_lease_client(this, priv, client_uid);
+        if (!clnt) {
+            pthread_mutex_unlock(&priv->mutex);
+            gf_msg(this->name, GF_LOG_ERROR, 0, LEASE_MSG_CLNT_NOTFOUND,
+                   "There is no client entry found in the cleanup list");
+            goto out;
+        }
+        ret = __remove_inode_from_clnt_list(this, clnt, inode);
+        if (ret) {
+            pthread_mutex_unlock(&priv->mutex);
+            gf_msg(this->name, GF_LOG_ERROR, 0, LEASE_MSG_INODE_NOTFOUND,
+                   "There is no inode entry found in the cleanup list");
+            goto out;
+        }
+    }
+    pthread_mutex_unlock(&priv->mutex);
+out:
+    return ret;
+}
+
+/* Remove lease entry in the corresponding client entry.
+ */
+static int
+__remove_lease(xlator_t *this, inode_t *inode, lease_inode_ctx_t *lease_ctx,
+               const char *client_uid, struct gf_lease *lease)
+{
+    lease_id_entry_t *lease_entry = NULL;
+    int ret = 0;
+    int32_t lease_type = 0;
+    leases_private_t *priv = NULL;
+
+    GF_VALIDATE_OR_GOTO("leases", lease_ctx, out);
+    GF_VALIDATE_OR_GOTO("leases", lease, out);
+
+    priv = this->private;
+
+    gf_msg_trace(this->name, 0,
+                 "Removing lease entry for client: %s, "
+                 "lease type:%d, lease id:%s",
+                 client_uid, lease->lease_type, leaseid_utoa(lease->lease_id));
+
+    /* There could be a race where in server recalled the lease and by the time
+     * client sends lease_unlock request, server may have revoked it. To handle
+     * such cases, if lease doesnt exist treat it as noop and return success.
+     */
+    lease_entry = __get_lease_id_entry(lease_ctx, lease->lease_id);
+    if (!lease_entry) {
+        gf_msg(this->name, GF_LOG_INFO, 0, LEASE_MSG_INVAL_UNLK_LEASE,
+               "Got unlock lease request from client:%s, but has no "
+               "corresponding lock",
+               client_uid);
+        ret = 0;
+        goto out;
+    }
+
+    if (!(lease_entry->lease_type & lease->lease_type)) {
+        gf_msg(this->name, GF_LOG_INFO, 0, LEASE_MSG_INVAL_UNLK_LEASE,
+               "Got unlock lease request from client:%s for an invalid "
+               "lease_type",
+               client_uid);
+        ret = -EINVAL;
+        errno = EINVAL;
+        goto out;
+    }
+    lease_type = lease->lease_type;
+    lease_entry->lease_type_cnt[lease_type]--;
+    lease_entry->lease_cnt--;
+
+    lease_ctx->lease_type_cnt[lease_type]--;
+    lease_ctx->lease_cnt--;
+
+    if (lease_entry->lease_type_cnt[lease_type] == 0)
+        lease_entry->lease_type = lease_entry->lease_type & (~lease_type);
+
+    if (lease_ctx->lease_type_cnt[lease_type] == 0)
+        lease_ctx->lease_type = lease_ctx->lease_type & (~lease_type);
+
+    if (lease_entry->lease_cnt == 0) {
+        if (__is_clnt_lease_none(client_uid, lease_ctx)) {
+            gf_msg_trace(this->name, 0,
+                         "Client(%s) has no leases"
+                         " on gfid (%s), hence removing the inode"
+                         " from the client cleanup list",
+                         client_uid, uuid_utoa(inode->gfid));
+            remove_from_clnt_list(this, client_uid, lease_ctx->inode);
+        }
+        __destroy_lease_id_entry(lease_entry);
+        lease_ctx->blocked_fops_resuming = _gf_true;
+    }
+
+    if (lease_ctx->lease_cnt == 0 && lease_ctx->timer) {
+        ret = gf_tw_del_timer(priv->timer_wheel, lease_ctx->timer);
+        lease_ctx->recall_in_progress = _gf_false;
+        lease_ctx->timer = NULL;
+    }
+out:
+    return ret;
+}
+
+static gf_boolean_t
+__is_lease_grantable(xlator_t *this, lease_inode_ctx_t *lease_ctx,
+                     struct gf_lease *lease, inode_t *inode)
+{
+    uint32_t fd_count = 0;
+    int32_t flags = 0;
+    fd_t *iter_fd = NULL;
+    gf_boolean_t grant = _gf_false;
+    int ret = 0;
+    lease_fd_ctx_t *fd_ctx = NULL;
+    uint64_t ctx = 0;
+
+    GF_VALIDATE_OR_GOTO("leases", lease_ctx, out);
+    GF_VALIDATE_OR_GOTO("leases", lease, out);
+    GF_VALIDATE_OR_GOTO("leases", inode, out);
+
+    if (lease_ctx->recall_in_progress) {
+        gf_msg_debug(this->name, 0,
+                     "Recall in progress, hence "
+                     "failing the lease request");
+        grant = _gf_false;
+        goto out;
+    }
+
+    if (lease_ctx->blocked_fops_resuming) {
+        gf_msg_debug(this->name, 0,
+                     "Previously blocked fops resuming, hence "
+                     "failing the lease request");
+        grant = _gf_false;
+        goto out;
+    }
+
+    LOCK(&inode->lock);
+    {
+        list_for_each_entry(iter_fd, &inode->fd_list, inode_list)
+        {
+            ret = fd_ctx_get(iter_fd, this, &ctx);
+            if (ret < 0) {
+                grant = _gf_false;
+                UNLOCK(&inode->lock);
+                gf_msg(this->name, GF_LOG_ERROR, 0, LEASE_MSG_INVAL_FD_CTX,
+                       "Unable to get fd ctx");
+                goto out;
+            }
+            fd_ctx = (lease_fd_ctx_t *)(long)ctx;
+
+            /* Check for open fd conflict, note that open fds from
+             * the same lease id is not checked for conflict, as it is
+             * lease id based lease.
+             */
+            if (fd_ctx->client_uid != NULL &&
+                !__is_same_lease_id(fd_ctx->lease_id, lease->lease_id)) {
+                fd_count++;
+                flags |= iter_fd->flags;
+            }
+        }
+    }
+    UNLOCK(&inode->lock);
+
+    gf_msg_debug(this->name, 0, "open fd count:%d flags:%d", fd_count, flags);
+
+    __dump_leases_info(this, lease_ctx);
+
+    switch (lease->lease_type) {
+        case GF_RD_LEASE:
+            /* check open fd conflict */
+            if ((fd_count > 0) && ((flags & O_WRONLY) || (flags & O_RDWR))) {
+                grant = _gf_false;
+                break;
+            }
+
+            /* check for conflict with existing leases */
+            if (lease_ctx->lease_type == NONE ||
+                lease_ctx->lease_type == GF_RD_LEASE ||
+                !(__another_lease_found(lease_ctx, lease->lease_id)))
+                grant = _gf_true;
+            else
+                grant = _gf_false;
+            break;
+
+        case GF_RW_LEASE:
+            /* check open fd conflict; conflict if there are any fds open
+             * other than the client on which the lease is requested. */
+            if (fd_count > 0) {
+                grant = _gf_false;
+                break;
+            }
+
+            /* check existing lease conflict */
+            if (lease_ctx->lease_type == NONE ||
+                !(__another_lease_found(lease_ctx, lease->lease_id)))
+                grant = _gf_true;
+            else
+                grant = _gf_false;
+            break;
+
+        default:
+            gf_msg(this->name, GF_LOG_ERROR, EINVAL, LEASE_MSG_INVAL_LEASE_TYPE,
+                   "Invalid lease type specified");
+            break;
+    }
+out:
+    return grant;
+}
+
+static void
+do_blocked_fops(xlator_t *this, lease_inode_ctx_t *lease_ctx)
+{
+    struct list_head wind_list;
+    fop_stub_t *blk_fop = NULL;
+    fop_stub_t *tmp = NULL;
+
+    INIT_LIST_HEAD(&wind_list);
+
+    pthread_mutex_lock(&lease_ctx->lock);
+    {
+        if (!lease_ctx->blocked_fops_resuming) {
+            /* lease_ctx->blocked_fops_resuming will be set
+             * only when the last lease is released. That
+             * is when we need to resume blocked fops and unref
+             * the inode taken in __add_lease (when lease_cnt == 1).
+             * Return otherwise.
+             */
+            pthread_mutex_unlock(&lease_ctx->lock);
+            return;
+        }
+
+        list_for_each_entry_safe(blk_fop, tmp, &lease_ctx->blocked_list, list)
+        {
+            list_del_init(&blk_fop->list);
+            list_add_tail(&blk_fop->list, &wind_list);
+        }
+    }
+    pthread_mutex_unlock(&lease_ctx->lock);
+
+    gf_msg_trace(this->name, 0, "Executing the blocked stubs on gfid(%s)",
+                 uuid_utoa(lease_ctx->inode->gfid));
+    list_for_each_entry_safe(blk_fop, tmp, &wind_list, list)
+    {
+        list_del_init(&blk_fop->list);
+        gf_msg_trace(this->name, 0, "Executing fop:%d", blk_fop->stub->fop);
+        call_resume(blk_fop->stub);
+        GF_FREE(blk_fop);
+    }
+
+    pthread_mutex_lock(&lease_ctx->lock);
+    {
+        lease_ctx->lease_type = NONE;
+        /* unref the inode taken in __add_lease
+         * (when lease_cnt == 1) */
+        lease_ctx->blocked_fops_resuming = _gf_false;
+        inode_unref(lease_ctx->inode);
+        lease_ctx->inode = NULL;
+    }
+    pthread_mutex_unlock(&lease_ctx->lock);
+
+    return;
+}
+
+void
+recall_lease_timer_handler(struct gf_tw_timer_list *timer, void *data,
+                           unsigned long calltime)
+{
+    inode_t *inode = NULL;
+    lease_inode_t *lease_inode = NULL;
+    leases_private_t *priv = NULL;
+    lease_timer_data_t *timer_data = NULL;
+
+    timer_data = data;
+
+    priv = timer_data->this->private;
+    inode = timer_data->inode;
+    lease_inode = new_lease_inode(inode);
+    if (!lease_inode) {
+        errno = ENOMEM;
+        goto out;
+    }
+    pthread_mutex_lock(&priv->mutex);
+    {
+        list_add_tail(&lease_inode->list, &priv->recall_list);
+        pthread_cond_broadcast(&priv->cond);
+    }
+    pthread_mutex_unlock(&priv->mutex);
+out:
+    /* unref the inode_ref taken by timer_data in __recall_lease */
+    inode_unref(timer_data->inode);
+
+    GF_FREE(timer);
+}
+
+static void
+__recall_lease(xlator_t *this, lease_inode_ctx_t *lease_ctx)
+{
+    lease_id_entry_t *lease_entry = NULL;
+    lease_id_entry_t *tmp = NULL;
+    struct gf_upcall up_req = {
+        0,
+    };
+    struct gf_upcall_recall_lease recall_req = {
+        0,
+    };
+    int notify_ret = -1;
+    struct gf_tw_timer_list *timer = NULL;
+    leases_private_t *priv = NULL;
+    lease_timer_data_t *timer_data = NULL;
+    time_t recall_time;
+
+    if (lease_ctx->recall_in_progress) {
+        gf_msg_debug(this->name, 0,
+                     "Lease recall is already in "
+                     "progress, hence not sending another recall");
+        goto out;
+    }
+
+    priv = this->private;
+    recall_time = gf_time();
+    list_for_each_entry_safe(lease_entry, tmp, &lease_ctx->lease_id_list,
+                             lease_id_list)
+    {
+        gf_uuid_copy(up_req.gfid, lease_ctx->inode->gfid);
+        up_req.client_uid = lease_entry->client_uid;
+        up_req.event_type = GF_UPCALL_RECALL_LEASE;
+        up_req.data = &recall_req;
+
+        notify_ret = this->notify(this, GF_EVENT_UPCALL, &up_req);
+        if (notify_ret < 0) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, LEASE_MSG_RECALL_FAIL,
+                   "Recall notification to client: %s failed",
+                   lease_entry->client_uid);
+            /* Do not return from here, continue registering the timer,
+               this is required mostly o keep replicas in sync*/
+        } else {
+            gf_msg_debug(this->name, 0,
+                         "Recall lease (all)"
+                         "notification sent to client %s",
+                         lease_entry->client_uid);
+        }
+
+        lease_ctx->recall_in_progress = _gf_true;
+        lease_entry->recall_time = recall_time;
+    }
+    timer = GF_MALLOC(sizeof(*timer), gf_common_mt_tw_timer_list);
+    if (!timer) {
+        goto out;
+    }
+    timer_data = GF_MALLOC(sizeof(lease_timer_data_t),
+                           gf_leases_mt_timer_data_t);
+    if (!timer_data) {
+        GF_FREE(timer);
+        goto out;
+    }
+
+    timer_data->inode = inode_ref(lease_ctx->inode);
+    timer_data->this = this;
+    timer->data = timer_data;
+
+    INIT_LIST_HEAD(&timer->entry);
+    timer->expires = get_recall_lease_timeout(this);
+    timer->function = recall_lease_timer_handler;
+    lease_ctx->timer = timer;
+    gf_tw_add_timer(priv->timer_wheel, timer);
+    gf_msg_trace(this->name, 0,
+                 "Registering timer "
+                 "%p, after "
+                 "sending recall",
+                 timer);
+out:
+    return;
+}
+
+/* ret = 0; STACK_UNWIND Success
+ * ret = -1; STACK_UNWIND failure
+ */
+int
+process_lease_req(call_frame_t *frame, xlator_t *this, inode_t *inode,
+                  struct gf_lease *lease)
+{
+    int ret = 0;
+    char *client_uid = NULL;
+    lease_inode_ctx_t *lease_ctx = NULL;
+
+    GF_VALIDATE_OR_GOTO("leases", frame, out);
+    GF_VALIDATE_OR_GOTO("leases", this, out);
+    GF_VALIDATE_OR_GOTO("leases", inode, out);
+    GF_VALIDATE_OR_GOTO("leases", lease, out);
+
+    client_uid = frame->root->client->client_uid;
+
+    if (!is_valid_lease_id(lease->lease_id)) {
+        gf_msg(this->name, GF_LOG_ERROR, EINVAL, LEASE_MSG_INVAL_LEASE_ID,
+               "Invalid lease id, from"
+               "client:%s",
+               client_uid);
+        ret = -EINVAL;
+        errno = EINVAL;
+        goto out;
+    }
+
+    lease_ctx = lease_ctx_get(inode, this);
+    if (!lease_ctx) {
+        gf_msg(this->name, GF_LOG_WARNING, ENOMEM, LEASE_MSG_NO_MEM,
+               "Unable to create/get inode ctx, "
+               "inode:%p",
+               inode);
+        ret = -ENOMEM;
+        errno = ENOMEM;
+        goto out;
+    }
+
+    gf_msg_debug(this->name, 0,
+                 "Lease request from client: %s, "
+                 "lease type:%d, lease cmd:%d, lease ID:%s, gfid:%s",
+                 client_uid, lease->lease_type, lease->cmd,
+                 leaseid_utoa(lease->lease_id), uuid_utoa(inode->gfid));
+
+    pthread_mutex_lock(&lease_ctx->lock);
+    {
+        switch (lease->cmd) {
+            case GF_GET_LEASE:
+                lease->lease_type = lease_ctx->lease_type;
+                gf_msg_debug(this->name, 0,
+                             "Get lease, existing lease"
+                             "type: %d",
+                             lease_ctx->lease_type);
+                /*TODO:Should it consider lease id or client_uid?*/
+                break;
+
+            case GF_SET_LEASE:
+                if (__is_lease_grantable(this, lease_ctx, lease, inode)) {
+                    __add_lease(frame, inode, lease_ctx, client_uid, lease);
+                    ret = 0;
+                } else {
+                    gf_msg_debug(this->name, GF_LOG_DEBUG,
+                                 "Not granting the conflicting lease"
+                                 " request from %s on gfid(%s)",
+                                 client_uid, uuid_utoa(inode->gfid));
+                    __recall_lease(this, lease_ctx);
+                    ret = -1;
+                }
+                break;
+            case GF_UNLK_LEASE:
+                ret = __remove_lease(this, inode, lease_ctx, client_uid, lease);
+                if ((ret >= 0) && (lease_ctx->lease_cnt == 0)) {
+                    pthread_mutex_unlock(&lease_ctx->lock);
+                    goto unblock;
+                }
+                break;
+            default:
+                ret = -EINVAL;
+                break;
+        }
+    }
+    pthread_mutex_unlock(&lease_ctx->lock);
+
+    return ret;
+
+unblock:
+    do_blocked_fops(this, lease_ctx);
+out:
+    return ret;
+}
+
+/* ret = 1 conflict
+ * ret = 0 no conflict
+ */
+gf_boolean_t
+__check_lease_conflict(call_frame_t *frame, lease_inode_ctx_t *lease_ctx,
+                       const char *lease_id, gf_boolean_t is_write)
+{
+    gf_lease_types_t lease_type = {
+        0,
+    };
+    gf_boolean_t conflicts = _gf_false;
+    lease_id_entry_t *lease_entry = NULL;
+
+    GF_VALIDATE_OR_GOTO("leases", frame, out);
+    GF_VALIDATE_OR_GOTO("leases", lease_ctx, out);
+
+    lease_type = lease_ctx->lease_type;
+
+    /* If the fop is rename or unlink conflict the lease even if its
+     * from the same client??
+     */
+    if ((frame->root->op == GF_FOP_RENAME) ||
+        (frame->root->op == GF_FOP_UNLINK)) {
+        conflicts = _gf_true;
+        goto recall;
+    }
+
+    /* As internal fops are used to maintain data integrity but do not
+     * make modififications to the client data, no need to conflict with
+     * them.
+     *
+     * @todo: like for locks, even lease state has to be handled by
+     * rebalance or self-heal daemon process. */
+    if (frame->root->pid < 0) {
+        conflicts = _gf_false;
+        goto recall;
+    }
+
+    /* If lease_id is not sent, set conflicts = true if there is
+     * an existing lease */
+    if (!lease_id && (lease_ctx->lease_cnt > 0)) {
+        conflicts = _gf_true;
+        goto recall;
+    }
+
+    switch (lease_type) {
+        case (GF_RW_LEASE | GF_RD_LEASE):
+        case GF_RW_LEASE:
+            lease_entry = __get_lease_id_entry(lease_ctx, lease_id);
+            if (lease_entry && (lease_entry->lease_type & GF_RW_LEASE))
+                conflicts = _gf_false;
+            else
+                conflicts = _gf_true;
+            break;
+        case GF_RD_LEASE:
+            if (is_write && __another_lease_found(lease_ctx, lease_id))
+                conflicts = _gf_true;
+            else
+                conflicts = _gf_false;
+            break;
+        default:
+            break;
+    }
+
+recall:
+    /* If there is a conflict found and recall is not already sent to all
+     * the clients, then send recall to each of the client holding lease.
+     */
+    if (conflicts)
+        __recall_lease(frame->this, lease_ctx);
+out:
+    return conflicts;
+}
+
+/* Return values:
+ * -1 : error, unwind the fop
+ * WIND_FOP: No conflict, wind the fop
+ * BLOCK_FOP: Found a conflicting lease, block the fop
+ */
+int
+check_lease_conflict(call_frame_t *frame, inode_t *inode, const char *lease_id,
+                     uint32_t fop_flags)
+{
+    lease_inode_ctx_t *lease_ctx = NULL;
+    gf_boolean_t is_blocking_fop = _gf_false;
+    gf_boolean_t is_write_fop = _gf_false;
+    gf_boolean_t conflicts = _gf_false;
+    int ret = WIND_FOP;
+
+    lease_ctx = lease_ctx_get(inode, frame->this);
+    if (!lease_ctx) {
+        gf_msg(frame->this->name, GF_LOG_WARNING, ENOMEM, LEASE_MSG_NO_MEM,
+               "Unable to create/get inode ctx");
+        ret = -1;
+        errno = ENOMEM;
+        goto out;
+    }
+
+    is_blocking_fop = ((fop_flags & BLOCKING_FOP) != 0);
+    is_write_fop = ((fop_flags & DATA_MODIFY_FOP) != 0);
+
+    pthread_mutex_lock(&lease_ctx->lock);
+    {
+        if (lease_ctx->lease_type == NONE) {
+            pthread_mutex_unlock(&lease_ctx->lock);
+            gf_msg_debug(frame->this->name, 0,
+                         "No leases found continuing with the"
+                         " fop:%s",
+                         gf_fop_list[frame->root->op]);
+            ret = WIND_FOP;
+            goto out;
+        }
+        conflicts = __check_lease_conflict(frame, lease_ctx, lease_id,
+                                           is_write_fop);
+        if (conflicts) {
+            if (is_blocking_fop) {
+                gf_msg_debug(frame->this->name, 0,
+                             "Fop: %s "
+                             "conflicting existing "
+                             "lease: %d, blocking the"
+                             "fop",
+                             gf_fop_list[frame->root->op],
+                             lease_ctx->lease_type);
+                ret = BLOCK_FOP;
+            } else {
+                gf_msg_debug(frame->this->name, 0,
+                             "Fop: %s "
+                             "conflicting existing "
+                             "lease: %d, sending "
+                             "EAGAIN",
+                             gf_fop_list[frame->root->op],
+                             lease_ctx->lease_type);
+                errno = EAGAIN;
+                ret = -1;
+            }
+        }
+    }
+    pthread_mutex_unlock(&lease_ctx->lock);
+out:
+    return ret;
+}
+
+static int
+remove_clnt_leases(const char *client_uid, inode_t *inode, xlator_t *this)
+{
+    lease_inode_ctx_t *lease_ctx = NULL;
+    lease_id_entry_t *lease_entry = NULL;
+    lease_id_entry_t *tmp = NULL;
+    int ret = 0;
+    int i = 0;
+
+    lease_ctx = lease_ctx_get(inode, this);
+    if (!lease_ctx) {
+        gf_msg(this->name, GF_LOG_WARNING, ENOMEM, LEASE_MSG_INVAL_INODE_CTX,
+               "Unable to create/get inode ctx");
+        ret = -1;
+        errno = ENOMEM;
+        goto out;
+    }
+
+    pthread_mutex_lock(&lease_ctx->lock);
+    {
+        list_for_each_entry_safe(lease_entry, tmp, &lease_ctx->lease_id_list,
+                                 lease_id_list)
+        {
+            if (strcmp(client_uid, lease_entry->client_uid) == 0) {
+                for (i = 0; i < GF_LEASE_MAX_TYPE; i++) {
+                    lease_ctx->lease_type_cnt[i] -= lease_entry
+                                                        ->lease_type_cnt[i];
+                }
+                lease_ctx->lease_cnt -= lease_entry->lease_cnt;
+                __destroy_lease_id_entry(lease_entry);
+                if (lease_ctx->lease_cnt == 0) {
+                    lease_ctx->blocked_fops_resuming = _gf_true;
+                    pthread_mutex_unlock(&lease_ctx->lock);
+                    goto unblock;
+                }
+            }
+        }
+    }
+    pthread_mutex_unlock(&lease_ctx->lock);
+out:
+    return ret;
+
+unblock:
+    do_blocked_fops(this, lease_ctx);
+    return ret;
+}
+
+int
+cleanup_client_leases(xlator_t *this, const char *client_uid)
+{
+    lease_client_t *clnt = NULL;
+    lease_client_t *tmp = NULL;
+    struct list_head cleanup_list = {
+        0,
+    };
+    lease_inode_t *l_inode = NULL;
+    lease_inode_t *tmp1 = NULL;
+    leases_private_t *priv = NULL;
+    int ret = 0;
+
+    priv = this->private;
+    if (!priv) {
+        ret = -1;
+        errno = EINVAL;
+        goto out;
+    }
+
+    INIT_LIST_HEAD(&cleanup_list);
+    pthread_mutex_lock(&priv->mutex);
+    {
+        list_for_each_entry_safe(clnt, tmp, &priv->client_list, client_list)
+        {
+            if ((strcmp(clnt->client_uid, client_uid) == 0)) {
+                list_for_each_entry_safe(l_inode, tmp1, &clnt->inode_list, list)
+                {
+                    list_del_init(&l_inode->list);
+                    list_add_tail(&l_inode->list, &cleanup_list);
+                }
+                __destroy_lease_client(clnt);
+                break;
+            }
+        }
+    }
+    pthread_mutex_unlock(&priv->mutex);
+
+    l_inode = tmp1 = NULL;
+    list_for_each_entry_safe(l_inode, tmp1, &cleanup_list, list)
+    {
+        remove_clnt_leases(client_uid, l_inode->inode, this);
+        __destroy_lease_inode(l_inode);
+    }
+out:
+    return ret;
+}
+
+static void
+__remove_all_leases(xlator_t *this, lease_inode_ctx_t *lease_ctx)
+{
+    int i = 0;
+    lease_id_entry_t *lease_entry = NULL;
+    lease_id_entry_t *tmp = NULL;
+
+    if (lease_ctx->lease_cnt == 0) {
+        /* No leases to remove. Return */
+        return;
+    }
+    __dump_leases_info(this, lease_ctx);
+
+    list_for_each_entry_safe(lease_entry, tmp, &lease_ctx->lease_id_list,
+                             lease_id_list)
+    {
+        lease_entry->lease_cnt = 0;
+        remove_from_clnt_list(this, lease_entry->client_uid, lease_ctx->inode);
+        __destroy_lease_id_entry(lease_entry);
+    }
+    INIT_LIST_HEAD(&lease_ctx->lease_id_list);
+    for (i = 0; i <= GF_LEASE_MAX_TYPE; i++)
+        lease_ctx->lease_type_cnt[i] = 0;
+    lease_ctx->lease_type = 0;
+    lease_ctx->lease_cnt = 0;
+    lease_ctx->recall_in_progress = _gf_false;
+    lease_ctx->timer = NULL;
+    lease_ctx->blocked_fops_resuming = _gf_true;
+
+    /* TODO:
+     * - Mark the corresponding fd bad. Could be done on client side
+     * as a result of recall
+     * - Free the lease_ctx
+     */
+    return;
+}
+
+static int
+remove_all_leases(xlator_t *this, inode_t *inode)
+{
+    lease_inode_ctx_t *lease_ctx = NULL;
+    int ret = 0;
+
+    GF_VALIDATE_OR_GOTO("leases", inode, out);
+
+    lease_ctx = lease_ctx_get(inode, this);
+    if (!lease_ctx) {
+        gf_msg(this->name, GF_LOG_WARNING, ENOMEM, LEASE_MSG_INVAL_INODE_CTX,
+               "Unable to create/get inode ctx");
+        ret = -1;
+        errno = ENOMEM;
+        goto out;
+    }
+
+    pthread_mutex_lock(&lease_ctx->lock);
+    {
+        __remove_all_leases(this, lease_ctx);
+    }
+    pthread_mutex_unlock(&lease_ctx->lock);
+
+    do_blocked_fops(this, lease_ctx);
+out:
+    return ret;
+}
+
+void *
+expired_recall_cleanup(void *data)
+{
+    struct timespec sleep_till = {
+        0,
+    };
+    struct list_head recall_cleanup_list;
+    lease_inode_t *recall_entry = NULL;
+    lease_inode_t *tmp = NULL;
+    leases_private_t *priv = NULL;
+    xlator_t *this = NULL;
+    time_t time_now;
+
+    GF_VALIDATE_OR_GOTO("leases", data, out);
+
+    this = data;
+    priv = this->private;
+
+    gf_msg_debug(this->name, 0, "Started the expired_recall_cleanup thread");
+
+    while (1) {
+        time_now = gf_time();
+        pthread_mutex_lock(&priv->mutex);
+        {
+            if (priv->fini) {
+                pthread_mutex_unlock(&priv->mutex);
+                goto out;
+            }
+            INIT_LIST_HEAD(&recall_cleanup_list);
+            if (list_empty(&priv->recall_list)) {
+                sleep_till.tv_sec = time_now + 600;
+                pthread_cond_timedwait(&priv->cond, &priv->mutex, &sleep_till);
+            }
+            if (!list_empty(&priv->recall_list)) {
+                gf_msg_debug(this->name, 0, "Found expired recalls");
+                list_for_each_entry_safe(recall_entry, tmp, &priv->recall_list,
+                                         list)
+                {
+                    list_del_init(&recall_entry->list);
+                    list_add_tail(&recall_entry->list, &recall_cleanup_list);
+                }
+            }
+        }
+        pthread_mutex_unlock(&priv->mutex);
+
+        recall_entry = tmp = NULL;
+        list_for_each_entry_safe(recall_entry, tmp, &recall_cleanup_list, list)
+        {
+            gf_msg_debug(this->name, 0,
+                         "Recall lease was sent on"
+                         " inode:%p, recall timer has expired"
+                         " and clients haven't unlocked the lease"
+                         " hence cleaning up leases on the inode",
+                         recall_entry->inode);
+            remove_all_leases(this, recall_entry->inode);
+            /* no need to take priv->mutex lock as this entry
+             * reference is removed from global recall list. */
+            __destroy_lease_inode(recall_entry);
+        }
+    }
+
+out:
+    return NULL;
+}
diff --git a/xlators/features/leases/src/leases-mem-types.h b/xlators/features/leases/src/leases-mem-types.h
new file mode 100644
index 00000000000..25664b44156
--- /dev/null
+++ b/xlators/features/leases/src/leases-mem-types.h
@@ -0,0 +1,27 @@
+/*
+   Copyright (c) 2015-2016 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
+
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
+
+#ifndef __LEASES_MEM_TYPES_H__
+#define __LEASES_MEM_TYPES_H__
+
+#include <glusterfs/mem-types.h>
+
+enum gf_leases_mem_types_ {
+    gf_leases_mt_private_t = gf_common_mt_end + 1,
+    gf_leases_mt_lease_client_t,
+    gf_leases_mt_lease_inode_t,
+    gf_leases_mt_fd_ctx_t,
+    gf_leases_mt_lease_inode_ctx_t,
+    gf_leases_mt_lease_id_entry_t,
+    gf_leases_mt_fop_stub_t,
+    gf_leases_mt_timer_data_t,
+    gf_leases_mt_end
+};
+#endif
diff --git a/xlators/features/leases/src/leases-messages.h b/xlators/features/leases/src/leases-messages.h
new file mode 100644
index 00000000000..da696b832de
--- /dev/null
+++ b/xlators/features/leases/src/leases-messages.h
@@ -0,0 +1,33 @@
+/*
+ Copyright (c) 2015-2016 Red Hat, Inc. <http://www.redhat.com>
+ This file is part of GlusterFS.
+
+ This file is licensed to you under your choice of the GNU Lesser
+ General Public License, version 3 or any later version (LGPLv3 or
+ later), or the GNU General Public License, version 2 (GPLv2), in all
+ cases as published by the Free Software Foundation.
+ */
+
+#ifndef _LEASES_MESSAGES_H_
+#define _LEASES_MESSAGES_H_
+
+#include <glusterfs/glfs-message-id.h>
+
+/* To add new message IDs, append new identifiers at the end of the list.
+ *
+ * Never remove a message ID. If it's not used anymore, you can rename it or
+ * leave it as it is, but not delete it. This is to prevent reutilization of
+ * IDs by other messages.
+ *
+ * The component name must match one of the entries defined in
+ * glfs-message-id.h.
+ */
+
+GLFS_MSGID(LEASES, LEASE_MSG_NO_MEM, LEASE_MSG_RECALL_FAIL,
+           LEASE_MSG_INVAL_LEASE_ID, LEASE_MSG_INVAL_UNLK_LEASE,
+           LEASE_MSG_INVAL_INODE_CTX, LEASE_MSG_NOT_ENABLED,
+           LEASE_MSG_NO_TIMER_WHEEL, LEASE_MSG_CLNT_NOTFOUND,
+           LEASE_MSG_INODE_NOTFOUND, LEASE_MSG_INVAL_FD_CTX,
+           LEASE_MSG_INVAL_LEASE_TYPE);
+
+#endif /* !_LEASES_MESSAGES_H_ */
diff --git a/xlators/features/leases/src/leases.c b/xlators/features/leases/src/leases.c
new file mode 100644
index 00000000000..04bee50ba3f
--- /dev/null
+++ b/xlators/features/leases/src/leases.c
@@ -0,0 +1,1168 @@
+/*
+   Copyright (c) 2015-2016 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
+
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
+
+#ifndef _CONFIG_H
+#define _CONFIG_H
+#include "config.h"
+#endif
+
+#include "leases.h"
+
+int32_t
+leases_open_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                int32_t op_ret, int32_t op_errno, fd_t *fd, dict_t *xdata)
+{
+    STACK_UNWIND_STRICT(open, frame, op_ret, op_errno, fd, xdata);
+
+    return 0;
+}
+
+int32_t
+leases_open(call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t flags,
+            fd_t *fd, dict_t *xdata)
+{
+    uint32_t fop_flags = 0;
+    int32_t op_errno = EINVAL;
+    int ret = 0;
+    lease_fd_ctx_t *fd_ctx = NULL;
+    char *lease_id = NULL;
+
+    EXIT_IF_LEASES_OFF(this, out);
+    EXIT_IF_INTERNAL_FOP(frame, xdata, out);
+
+    fd_ctx = GF_CALLOC(1, sizeof(*fd_ctx), gf_leases_mt_fd_ctx_t);
+    if (!fd_ctx) {
+        op_errno = ENOMEM;
+        goto err;
+    }
+
+    fd_ctx->client_uid = gf_strdup(frame->root->client->client_uid);
+    if (!fd_ctx->client_uid) {
+        op_errno = ENOMEM;
+        goto err;
+    }
+
+    GET_FLAGS(frame->root->op, flags);
+    GET_LEASE_ID(xdata, lease_id, frame->root->client->client_uid);
+    if (lease_id != NULL)
+        memcpy(fd_ctx->lease_id, lease_id, LEASE_ID_SIZE);
+    else
+        memset(fd_ctx->lease_id, 0, LEASE_ID_SIZE);
+
+    ret = fd_ctx_set(fd, this, (uint64_t)(uintptr_t)fd_ctx);
+    if (ret) {
+        op_errno = ENOMEM;
+        goto err;
+    }
+
+    ret = check_lease_conflict(frame, fd->inode, lease_id, fop_flags);
+    if (ret < 0)
+        goto err;
+    else if (ret == BLOCK_FOP)
+        goto block;
+    else if (ret == WIND_FOP)
+        goto out;
+
+block:
+    LEASE_BLOCK_FOP(fd->inode, open, frame, this, loc, flags, fd, xdata);
+    return 0;
+
+out:
+    STACK_WIND(frame, leases_open_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->open, loc, flags, fd, xdata);
+    return 0;
+
+err:
+    if (fd_ctx) {
+        GF_FREE(fd_ctx->client_uid);
+        GF_FREE(fd_ctx);
+    }
+
+    STACK_UNWIND_STRICT(open, frame, -1, op_errno, NULL, NULL);
+    return 0;
+}
+
+int32_t
+leases_writev_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int op_ret,
+                  int op_errno, struct iatt *prebuf, struct iatt *postbuf,
+                  dict_t *xdata)
+{
+    STACK_UNWIND_STRICT(writev, frame, op_ret, op_errno, prebuf, postbuf,
+                        xdata);
+
+    return 0;
+}
+
+int32_t
+leases_writev(call_frame_t *frame, xlator_t *this, fd_t *fd,
+              struct iovec *vector, int count, off_t off, uint32_t flags,
+              struct iobref *iobref, dict_t *xdata)
+{
+    uint32_t fop_flags = 0;
+    char *lease_id = NULL;
+    int ret = 0;
+
+    EXIT_IF_LEASES_OFF(this, out);
+    EXIT_IF_INTERNAL_FOP(frame, xdata, out);
+
+    GET_LEASE_ID(xdata, lease_id, frame->root->client->client_uid);
+    GET_FLAGS(frame->root->op, fd->flags);
+
+    ret = check_lease_conflict(frame, fd->inode, lease_id, fop_flags);
+    if (ret < 0)
+        goto err;
+    else if (ret == BLOCK_FOP)
+        goto block;
+    else if (ret == WIND_FOP)
+        goto out;
+
+block:
+    LEASE_BLOCK_FOP(fd->inode, writev, frame, this, fd, vector, count, off,
+                    flags, iobref, xdata);
+    return 0;
+
+out:
+    STACK_WIND(frame, leases_writev_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->writev, fd, vector, count, off, flags,
+               iobref, xdata);
+    return 0;
+
+err:
+    STACK_UNWIND_STRICT(writev, frame, -1, errno, NULL, NULL, NULL);
+    return 0;
+}
+
+int32_t
+leases_readv_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int op_ret,
+                 int op_errno, struct iovec *vector, int count,
+                 struct iatt *stbuf, struct iobref *iobref, dict_t *xdata)
+{
+    STACK_UNWIND_STRICT(readv, frame, op_ret, op_errno, vector, count, stbuf,
+                        iobref, xdata);
+
+    return 0;
+}
+
+int32_t
+leases_readv(call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
+             off_t offset, uint32_t flags, dict_t *xdata)
+{
+    uint32_t fop_flags = 0;
+    char *lease_id = NULL;
+    int ret = 0;
+
+    EXIT_IF_LEASES_OFF(this, out);
+    EXIT_IF_INTERNAL_FOP(frame, xdata, out);
+
+    GET_LEASE_ID(xdata, lease_id, frame->root->client->client_uid);
+    GET_FLAGS(frame->root->op, fd->flags);
+
+    ret = check_lease_conflict(frame, fd->inode, lease_id, fop_flags);
+    if (ret < 0)
+        goto err;
+    else if (ret == BLOCK_FOP)
+        goto block;
+    else if (ret == WIND_FOP)
+        goto out;
+
+block:
+    LEASE_BLOCK_FOP(fd->inode, readv, frame, this, fd, size, offset, flags,
+                    xdata);
+    return 0;
+
+out:
+    STACK_WIND(frame, leases_readv_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->readv, fd, size, offset, flags, xdata);
+    return 0;
+
+err:
+    STACK_UNWIND_STRICT(readv, frame, -1, errno, NULL, 0, NULL, NULL, NULL);
+    return 0;
+}
+
+int32_t
+leases_lk_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int32_t op_ret,
+              int32_t op_errno, struct gf_flock *lock, dict_t *xdata)
+{
+    STACK_UNWIND_STRICT(lk, frame, op_ret, op_errno, lock, xdata);
+
+    return 0;
+}
+
+int32_t
+leases_lk(call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t cmd,
+          struct gf_flock *flock, dict_t *xdata)
+{
+    uint32_t fop_flags = 0;
+    char *lease_id = NULL;
+    int ret = 0;
+
+    EXIT_IF_LEASES_OFF(this, out);
+    EXIT_IF_INTERNAL_FOP(frame, xdata, out);
+
+    GET_LEASE_ID(xdata, lease_id, frame->root->client->client_uid);
+    GET_FLAGS_LK(cmd, flock->l_type, fd->flags);
+
+    ret = check_lease_conflict(frame, fd->inode, lease_id, fop_flags);
+    if (ret < 0)
+        goto err;
+    else if (ret == BLOCK_FOP)
+        goto block;
+    else if (ret == WIND_FOP)
+        goto out;
+
+block:
+    LEASE_BLOCK_FOP(fd->inode, lk, frame, this, fd, cmd, flock, xdata);
+    return 0;
+
+out:
+    STACK_WIND(frame, leases_lk_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->lk, fd, cmd, flock, xdata);
+    return 0;
+
+err:
+    STACK_UNWIND_STRICT(lk, frame, -1, errno, NULL, NULL);
+    return 0;
+}
+
+int32_t
+leases_lease(call_frame_t *frame, xlator_t *this, loc_t *loc,
+             struct gf_lease *lease, dict_t *xdata)
+{
+    int32_t op_errno = 0;
+    int ret = 0;
+    struct gf_lease nullease = {
+        0,
+    };
+    int32_t op_ret = 0;
+
+    EXIT_IF_LEASES_OFF(this, out);
+    EXIT_IF_INTERNAL_FOP(frame, xdata, out);
+
+    ret = process_lease_req(frame, this, loc->inode, lease);
+    if (ret < 0) {
+        op_errno = -ret;
+        op_ret = -1;
+    }
+    goto unwind;
+
+out:
+    gf_msg(this->name, GF_LOG_ERROR, EINVAL, LEASE_MSG_NOT_ENABLED,
+           "\"features/leases\" translator is not enabled. "
+           "You need to enable it for proper functioning of your "
+           "application");
+    op_errno = ENOSYS;
+    op_ret = -1;
+
+unwind:
+    STACK_UNWIND_STRICT(lease, frame, op_ret, op_errno,
+                        (op_errno == ENOSYS) ? &nullease : lease, xdata);
+    return 0;
+}
+
+int32_t
+leases_truncate_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                    int op_ret, int op_errno, struct iatt *prebuf,
+                    struct iatt *postbuf, dict_t *xdata)
+{
+    STACK_UNWIND_STRICT(truncate, frame, op_ret, op_errno, prebuf, postbuf,
+                        xdata);
+
+    return 0;
+}
+
+int32_t
+leases_truncate(call_frame_t *frame, xlator_t *this, loc_t *loc, off_t offset,
+                dict_t *xdata)
+{
+    uint32_t fop_flags = 0;
+    char *lease_id = NULL;
+    int ret = 0;
+
+    EXIT_IF_LEASES_OFF(this, out);
+    EXIT_IF_INTERNAL_FOP(frame, xdata, out);
+
+    GET_LEASE_ID(xdata, lease_id, frame->root->client->client_uid);
+    GET_FLAGS(frame->root->op, 0);
+
+    ret = check_lease_conflict(frame, loc->inode, lease_id, fop_flags);
+    if (ret < 0)
+        goto err;
+    else if (ret == BLOCK_FOP)
+        goto block;
+    else if (ret == WIND_FOP)
+        goto out;
+
+block:
+    LEASE_BLOCK_FOP(loc->inode, truncate, frame, this, loc, offset, xdata);
+    return 0;
+
+out:
+    STACK_WIND(frame, leases_truncate_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->truncate, loc, offset, xdata);
+    return 0;
+
+err:
+    STACK_UNWIND_STRICT(truncate, frame, -1, errno, NULL, NULL, NULL);
+    return 0;
+}
+
+int32_t
+leases_setattr_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                   int op_ret, int op_errno, struct iatt *statpre,
+                   struct iatt *statpost, dict_t *xdata)
+{
+    STACK_UNWIND_STRICT(setattr, frame, op_ret, op_errno, statpre, statpost,
+                        xdata);
+
+    return 0;
+}
+
+int32_t
+leases_setattr(call_frame_t *frame, xlator_t *this, loc_t *loc,
+               struct iatt *stbuf, int32_t valid, dict_t *xdata)
+{
+    uint32_t fop_flags = 0;
+    char *lease_id = NULL;
+    int ret = 0;
+
+    EXIT_IF_LEASES_OFF(this, out);
+    EXIT_IF_INTERNAL_FOP(frame, xdata, out);
+
+    GET_LEASE_ID(xdata, lease_id, frame->root->client->client_uid);
+    GET_FLAGS(frame->root->op, 0);
+
+    ret = check_lease_conflict(frame, loc->inode, lease_id, fop_flags);
+    if (ret < 0)
+        goto err;
+    else if (ret == BLOCK_FOP)
+        goto block;
+    else if (ret == WIND_FOP)
+        goto out;
+
+block:
+    LEASE_BLOCK_FOP(loc->inode, setattr, frame, this, loc, stbuf, valid, xdata);
+    return 0;
+
+out:
+    STACK_WIND(frame, leases_setattr_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->setattr, loc, stbuf, valid, xdata);
+    return 0;
+
+err:
+    STACK_UNWIND_STRICT(setattr, frame, -1, errno, NULL, NULL, NULL);
+    return 0;
+}
+
+int32_t
+leases_rename_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                  int32_t op_ret, int32_t op_errno, struct iatt *stbuf,
+                  struct iatt *preoldparent, struct iatt *postoldparent,
+                  struct iatt *prenewparent, struct iatt *postnewparent,
+                  dict_t *xdata)
+{
+    STACK_UNWIND_STRICT(rename, frame, op_ret, op_errno, stbuf, preoldparent,
+                        postoldparent, prenewparent, postnewparent, xdata);
+
+    return 0;
+}
+
+int32_t
+leases_rename(call_frame_t *frame, xlator_t *this, loc_t *oldloc, loc_t *newloc,
+              dict_t *xdata)
+{
+    uint32_t fop_flags = 0;
+    char *lease_id = NULL;
+    int ret = 0;
+
+    EXIT_IF_LEASES_OFF(this, out);
+    EXIT_IF_INTERNAL_FOP(frame, xdata, out);
+
+    /* should the lease be also checked for newloc */
+    GET_LEASE_ID(xdata, lease_id, frame->root->client->client_uid);
+    GET_FLAGS(frame->root->op, 0);
+
+    ret = check_lease_conflict(frame, oldloc->inode, lease_id, fop_flags);
+    if (ret < 0)
+        goto err;
+    else if (ret == BLOCK_FOP)
+        goto block;
+    else if (ret == WIND_FOP)
+        goto out;
+
+block:
+    LEASE_BLOCK_FOP(oldloc->inode, rename, frame, this, oldloc, newloc, xdata);
+    return 0;
+
+out:
+    STACK_WIND(frame, leases_rename_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->rename, oldloc, newloc, xdata);
+    return 0;
+
+err:
+    STACK_UNWIND_STRICT(rename, frame, -1, errno, NULL, NULL, NULL, NULL, NULL,
+                        NULL);
+    return 0;
+}
+
+int32_t
+leases_unlink_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int op_ret,
+                  int op_errno, struct iatt *preparent, struct iatt *postparent,
+                  dict_t *xdata)
+{
+    STACK_UNWIND_STRICT(unlink, frame, op_ret, op_errno, preparent, postparent,
+                        xdata);
+
+    return 0;
+}
+
+int32_t
+leases_unlink(call_frame_t *frame, xlator_t *this, loc_t *loc, int xflag,
+              dict_t *xdata)
+{
+    uint32_t fop_flags = 0;
+    char *lease_id = NULL;
+    int ret = 0;
+
+    EXIT_IF_LEASES_OFF(this, out);
+    EXIT_IF_INTERNAL_FOP(frame, xdata, out);
+
+    GET_LEASE_ID(xdata, lease_id, frame->root->client->client_uid);
+    GET_FLAGS(frame->root->op, 0);
+
+    ret = check_lease_conflict(frame, loc->inode, lease_id, fop_flags);
+    if (ret < 0)
+        goto err;
+    else if (ret == BLOCK_FOP)
+        goto block;
+    else if (ret == WIND_FOP)
+        goto out;
+
+block:
+    LEASE_BLOCK_FOP(loc->inode, unlink, frame, this, loc, xflag, xdata);
+    return 0;
+
+out:
+    STACK_WIND(frame, leases_unlink_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->unlink, loc, xflag, xdata);
+    return 0;
+
+err:
+    STACK_UNWIND_STRICT(unlink, frame, -1, errno, NULL, NULL, NULL);
+    return 0;
+}
+
+int32_t
+leases_link_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int op_ret,
+                int op_errno, inode_t *inode, struct iatt *stbuf,
+                struct iatt *preparent, struct iatt *postparent, dict_t *xdata)
+{
+    STACK_UNWIND_STRICT(link, frame, op_ret, op_errno, inode, stbuf, preparent,
+                        postparent, xdata);
+
+    return 0;
+}
+
+int32_t
+leases_link(call_frame_t *frame, xlator_t *this, loc_t *oldloc, loc_t *newloc,
+            dict_t *xdata)
+{
+    uint32_t fop_flags = 0;
+    char *lease_id = NULL;
+    int ret = 0;
+
+    EXIT_IF_LEASES_OFF(this, out);
+    EXIT_IF_INTERNAL_FOP(frame, xdata, out);
+
+    GET_LEASE_ID(xdata, lease_id, frame->root->client->client_uid);
+    GET_FLAGS(frame->root->op, 0);
+
+    ret = check_lease_conflict(frame, oldloc->inode, lease_id, fop_flags);
+    if (ret < 0)
+        goto err;
+    else if (ret == BLOCK_FOP)
+        goto block;
+    else if (ret == WIND_FOP)
+        goto out;
+
+block:
+    LEASE_BLOCK_FOP(oldloc->inode, link, frame, this, oldloc, newloc, xdata);
+    return 0;
+out:
+    STACK_WIND(frame, leases_link_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->link, oldloc, newloc, xdata);
+    return 0;
+
+err:
+    STACK_UNWIND_STRICT(link, frame, -1, errno, NULL, NULL, NULL, NULL, NULL);
+    return 0;
+}
+
+int32_t
+leases_create_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int op_ret,
+                  int op_errno, fd_t *fd, inode_t *inode, struct iatt *stbuf,
+                  struct iatt *preparent, struct iatt *postparent,
+                  dict_t *xdata)
+{
+    STACK_UNWIND_STRICT(create, frame, op_ret, op_errno, fd, inode, stbuf,
+                        preparent, postparent, xdata);
+
+    return 0;
+}
+
+int32_t
+leases_create(call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t flags,
+              mode_t mode, mode_t umask, fd_t *fd, dict_t *xdata)
+{
+    uint32_t fop_flags = 0;
+    char *lease_id = NULL;
+    int ret = 0;
+
+    EXIT_IF_LEASES_OFF(this, out);
+    EXIT_IF_INTERNAL_FOP(frame, xdata, out);
+
+    GET_LEASE_ID(xdata, lease_id, frame->root->client->client_uid);
+    GET_FLAGS(frame->root->op, flags);
+
+    ret = check_lease_conflict(frame, fd->inode, lease_id, fop_flags);
+    if (ret < 0)
+        goto err;
+    else if (ret == BLOCK_FOP)
+        goto block;
+    else if (ret == WIND_FOP)
+        goto out;
+
+block:
+    LEASE_BLOCK_FOP(fd->inode, create, frame, this, loc, flags, mode, umask, fd,
+                    xdata);
+    return 0;
+
+out:
+    STACK_WIND(frame, leases_create_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->create, loc, flags, mode, umask, fd,
+               xdata);
+    return 0;
+
+err:
+    STACK_UNWIND_STRICT(create, frame, -1, errno, NULL, NULL, NULL, NULL, NULL,
+                        NULL);
+    return 0;
+}
+
+int32_t
+leases_fsync_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                 int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
+                 struct iatt *postbuf, dict_t *xdata)
+{
+    STACK_UNWIND_STRICT(fsync, frame, op_ret, op_errno, prebuf, postbuf, xdata);
+    return 0;
+}
+
+int32_t
+leases_fsync(call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t flags,
+             dict_t *xdata)
+{
+    uint32_t fop_flags = 0;
+    char *lease_id = NULL;
+    int ret = 0;
+
+    EXIT_IF_LEASES_OFF(this, out);
+    EXIT_IF_INTERNAL_FOP(frame, xdata, out);
+
+    GET_LEASE_ID(xdata, lease_id, frame->root->client->client_uid);
+    GET_FLAGS(frame->root->op, fd->flags);
+
+    ret = check_lease_conflict(frame, fd->inode, lease_id, fop_flags);
+    if (ret < 0)
+        goto err;
+    else if (ret == BLOCK_FOP)
+        goto block;
+    else if (ret == WIND_FOP)
+        goto out;
+
+block:
+    LEASE_BLOCK_FOP(fd->inode, fsync, frame, this, fd, flags, xdata);
+    return 0;
+
+out:
+    STACK_WIND(frame, leases_fsync_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->fsync, fd, flags, xdata);
+    return 0;
+err:
+    STACK_UNWIND_STRICT(fsync, frame, -1, errno, NULL, NULL, NULL);
+    return 0;
+}
+
+int32_t
+leases_ftruncate_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                     int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
+                     struct iatt *postbuf, dict_t *xdata)
+{
+    STACK_UNWIND_STRICT(ftruncate, frame, op_ret, op_errno, prebuf, postbuf,
+                        xdata);
+    return 0;
+}
+
+int32_t
+leases_ftruncate(call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
+                 dict_t *xdata)
+{
+    uint32_t fop_flags = 0;
+    char *lease_id = NULL;
+    int ret = 0;
+
+    EXIT_IF_LEASES_OFF(this, out);
+    EXIT_IF_INTERNAL_FOP(frame, xdata, out);
+
+    GET_LEASE_ID(xdata, lease_id, frame->root->client->client_uid);
+    GET_FLAGS(frame->root->op, 0); /* TODO:fd->flags?*/
+
+    ret = check_lease_conflict(frame, fd->inode, lease_id, fop_flags);
+    if (ret < 0)
+        goto err;
+    else if (ret == BLOCK_FOP)
+        goto block;
+    else if (ret == WIND_FOP)
+        goto out;
+
+block:
+    LEASE_BLOCK_FOP(fd->inode, ftruncate, frame, this, fd, offset, xdata);
+    return 0;
+
+out:
+    STACK_WIND(frame, leases_ftruncate_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->ftruncate, fd, offset, xdata);
+    return 0;
+
+err:
+    STACK_UNWIND_STRICT(ftruncate, frame, -1, errno, NULL, NULL, NULL);
+    return 0;
+}
+
+int32_t
+leases_fsetattr_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                    int32_t op_ret, int32_t op_errno, struct iatt *statpre,
+                    struct iatt *statpost, dict_t *xdata)
+{
+    STACK_UNWIND_STRICT(fsetattr, frame, op_ret, op_errno, statpre, statpost,
+                        xdata);
+    return 0;
+}
+
+int32_t
+leases_fsetattr(call_frame_t *frame, xlator_t *this, fd_t *fd,
+                struct iatt *stbuf, int32_t valid, dict_t *xdata)
+{
+    uint32_t fop_flags = 0;
+    char *lease_id = NULL;
+    int ret = 0;
+
+    EXIT_IF_LEASES_OFF(this, out);
+    EXIT_IF_INTERNAL_FOP(frame, xdata, out);
+
+    GET_LEASE_ID(xdata, lease_id, frame->root->client->client_uid);
+    GET_FLAGS(frame->root->op, fd->flags);
+
+    ret = check_lease_conflict(frame, fd->inode, lease_id, fop_flags);
+    if (ret < 0)
+        goto err;
+    else if (ret == BLOCK_FOP)
+        goto block;
+    else if (ret == WIND_FOP)
+        goto out;
+
+block:
+    LEASE_BLOCK_FOP(fd->inode, fsetattr, frame, this, fd, stbuf, valid, xdata);
+    return 0;
+
+out:
+    STACK_WIND(frame, leases_fsetattr_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->fsetattr, fd, stbuf, valid, xdata);
+    return 0;
+
+err:
+    STACK_UNWIND_STRICT(fsetattr, frame, -1, errno, NULL, NULL, NULL);
+    return 0;
+}
+
+int32_t
+leases_fallocate_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                     int32_t op_ret, int32_t op_errno, struct iatt *pre,
+                     struct iatt *post, dict_t *xdata)
+{
+    STACK_UNWIND_STRICT(fallocate, frame, op_ret, op_errno, pre, post, xdata);
+
+    return 0;
+}
+
+int32_t
+leases_fallocate(call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t mode,
+                 off_t offset, size_t len, dict_t *xdata)
+{
+    uint32_t fop_flags = 0;
+    char *lease_id = NULL;
+    int ret = 0;
+
+    EXIT_IF_LEASES_OFF(this, out);
+    EXIT_IF_INTERNAL_FOP(frame, xdata, out);
+
+    GET_LEASE_ID(xdata, lease_id, frame->root->client->client_uid);
+    GET_FLAGS(frame->root->op, fd->flags);
+
+    ret = check_lease_conflict(frame, fd->inode, lease_id, fop_flags);
+    if (ret < 0)
+        goto err;
+    else if (ret == BLOCK_FOP)
+        goto block;
+    else if (ret == WIND_FOP)
+        goto out;
+
+block:
+    LEASE_BLOCK_FOP(fd->inode, fallocate, frame, this, fd, mode, offset, len,
+                    xdata);
+    return 0;
+
+out:
+    STACK_WIND(frame, leases_fallocate_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->fallocate, fd, mode, offset, len,
+               xdata);
+    return 0;
+
+err:
+    STACK_UNWIND_STRICT(fallocate, frame, -1, errno, NULL, NULL, NULL);
+    return 0;
+}
+
+int32_t
+leases_discard_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                   int32_t op_ret, int32_t op_errno, struct iatt *pre,
+                   struct iatt *post, dict_t *xdata)
+{
+    STACK_UNWIND_STRICT(discard, frame, op_ret, op_errno, pre, post, xdata);
+
+    return 0;
+}
+
+int32_t
+leases_discard(call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
+               size_t len, dict_t *xdata)
+{
+    uint32_t fop_flags = 0;
+    char *lease_id = NULL;
+    int ret = 0;
+
+    EXIT_IF_LEASES_OFF(this, out);
+    EXIT_IF_INTERNAL_FOP(frame, xdata, out);
+
+    GET_LEASE_ID(xdata, lease_id, frame->root->client->client_uid);
+    GET_FLAGS(frame->root->op, fd->flags);
+
+    ret = check_lease_conflict(frame, fd->inode, lease_id, fop_flags);
+    if (ret < 0)
+        goto err;
+    else if (ret == BLOCK_FOP)
+        goto block;
+    else if (ret == WIND_FOP)
+        goto out;
+
+block:
+    LEASE_BLOCK_FOP(fd->inode, discard, frame, this, fd, offset, len, xdata);
+    return 0;
+
+out:
+    STACK_WIND(frame, leases_discard_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->discard, fd, offset, len, xdata);
+    return 0;
+
+err:
+    STACK_UNWIND_STRICT(discard, frame, -1, errno, NULL, NULL, NULL);
+    return 0;
+}
+
+int32_t
+leases_zerofill_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                    int32_t op_ret, int32_t op_errno, struct iatt *pre,
+                    struct iatt *post, dict_t *xdata)
+{
+    STACK_UNWIND_STRICT(zerofill, frame, op_ret, op_errno, pre, post, xdata);
+
+    return 0;
+}
+
+int
+leases_zerofill(call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
+                off_t len, dict_t *xdata)
+{
+    uint32_t fop_flags = 0;
+    char *lease_id = NULL;
+    int ret = 0;
+
+    EXIT_IF_LEASES_OFF(this, out);
+    EXIT_IF_INTERNAL_FOP(frame, xdata, out);
+
+    GET_LEASE_ID(xdata, lease_id, frame->root->client->client_uid);
+    GET_FLAGS(frame->root->op, fd->flags);
+
+    ret = check_lease_conflict(frame, fd->inode, lease_id, fop_flags);
+    if (ret < 0)
+        goto err;
+    else if (ret == BLOCK_FOP)
+        goto block;
+    else if (ret == WIND_FOP)
+        goto out;
+
+block:
+    LEASE_BLOCK_FOP(fd->inode, zerofill, frame, this, fd, offset, len, xdata);
+    return 0;
+
+out:
+    STACK_WIND(frame, leases_zerofill_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->zerofill, fd, offset, len, xdata);
+    return 0;
+
+err:
+    STACK_UNWIND_STRICT(zerofill, frame, -1, errno, NULL, NULL, NULL);
+    return 0;
+}
+
+int
+leases_flush_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                 int32_t op_ret, int32_t op_errno, dict_t *xdata)
+{
+    STACK_UNWIND_STRICT(flush, frame, op_ret, op_errno, xdata);
+
+    return 0;
+}
+
+int
+leases_flush(call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *xdata)
+{
+    uint32_t fop_flags = 0;
+    char *lease_id = NULL;
+    int ret = 0;
+    lease_fd_ctx_t *fd_ctx = NULL;
+    uint64_t ctx = 0;
+
+    EXIT_IF_LEASES_OFF(this, out);
+    EXIT_IF_INTERNAL_FOP(frame, xdata, out);
+
+    GET_LEASE_ID(xdata, lease_id, frame->root->client->client_uid);
+    GET_FLAGS(frame->root->op, fd->flags);
+
+    ret = check_lease_conflict(frame, fd->inode, lease_id, fop_flags);
+    if (ret < 0)
+        goto err;
+    else if (ret == BLOCK_FOP)
+        goto block;
+    else if (ret == WIND_FOP)
+        goto out;
+
+block:
+    LEASE_BLOCK_FOP(fd->inode, flush, frame, this, fd, xdata);
+    return 0;
+
+out:
+    /* *
+     * currently release is not called after the close fop from the
+     * application. Hence lease fd ctx is reset on here.
+     * This is actually not the right way, since flush can be called
+     * not only from the close op.
+     * TODO :
+     *     - Either identify the flush is called from close call on fd from
+     *     from the application.
+     *                      OR
+     *     - Find why release is not called post the last close call
+     */
+    ret = fd_ctx_get(fd, this, &ctx);
+    if (ret == 0) {
+        fd_ctx = (lease_fd_ctx_t *)(long)ctx;
+        if (fd_ctx->client_uid) {
+            GF_FREE(fd_ctx->client_uid);
+            fd_ctx->client_uid = NULL;
+        }
+        memset(fd_ctx->lease_id, 0, LEASE_ID_SIZE);
+    }
+    STACK_WIND(frame, leases_flush_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->flush, fd, xdata);
+    return 0;
+
+err:
+    STACK_UNWIND_STRICT(create, frame, -1, errno, NULL, NULL, NULL, NULL, NULL,
+                        NULL);
+    return 0;
+}
+
+int32_t
+mem_acct_init(xlator_t *this)
+{
+    int ret = -1;
+
+    if (!this)
+        return ret;
+
+    ret = xlator_mem_acct_init(this, gf_leases_mt_end + 1);
+
+    if (ret != 0) {
+        gf_msg(this->name, GF_LOG_WARNING, ENOMEM, LEASE_MSG_NO_MEM,
+               "mem account init failed");
+        return ret;
+    }
+
+    return ret;
+}
+
+static int
+leases_init_priv(xlator_t *this)
+{
+    int ret = 0;
+    leases_private_t *priv = NULL;
+
+    priv = this->private;
+    GF_ASSERT(priv);
+
+    if (!priv->timer_wheel) {
+        priv->timer_wheel = glusterfs_ctx_tw_get(this->ctx);
+        if (!priv->timer_wheel) {
+            ret = -1;
+            goto out;
+        }
+    }
+
+    if (!priv->inited_recall_thr) {
+        ret = gf_thread_create(&priv->recall_thr, NULL, expired_recall_cleanup,
+                               this, "leasercl");
+        if (!ret)
+            priv->inited_recall_thr = _gf_true;
+    }
+
+out:
+    return ret;
+}
+
+int
+reconfigure(xlator_t *this, dict_t *options)
+{
+    leases_private_t *priv = NULL;
+    int ret = -1;
+
+    priv = this->private;
+    GF_ASSERT(priv);
+
+    /* TODO: In case of reconfigure, if its enabling the leases
+     * its not an issue, but if its disabling the leases, there
+     * is more to it, like recall all the existing leases, wait
+     * for unlock of all the leases etc., hence not supporting the
+     * reconfigure for now.
+
+    GF_OPTION_RECONF ("leases", priv->leases_enabled,
+                      options, bool, out);
+
+    if (priv->leases_enabled) {
+            ret = leases_init_priv (this);
+            if (ret)
+                    goto out;
+    }
+    */
+
+    GF_OPTION_RECONF("lease-lock-recall-timeout", priv->recall_lease_timeout,
+                     options, int32, out);
+
+    ret = 0;
+out:
+    return ret;
+}
+
+int
+init(xlator_t *this)
+{
+    int ret = -1;
+    leases_private_t *priv = NULL;
+
+    priv = GF_CALLOC(1, sizeof(*priv), gf_leases_mt_private_t);
+    if (!priv) {
+        gf_msg(this->name, GF_LOG_WARNING, ENOMEM, LEASE_MSG_NO_MEM,
+               "Leases init failed");
+        goto out;
+    }
+
+    GF_OPTION_INIT("leases", priv->leases_enabled, bool, out);
+    GF_OPTION_INIT("lease-lock-recall-timeout", priv->recall_lease_timeout,
+                   int32, out);
+    pthread_mutex_init(&priv->mutex, NULL);
+    INIT_LIST_HEAD(&priv->client_list);
+    INIT_LIST_HEAD(&priv->recall_list);
+
+    this->private = priv;
+
+    if (priv->leases_enabled) {
+        ret = leases_init_priv(this);
+        if (ret)
+            goto out;
+    }
+
+    ret = 0;
+
+out:
+    if (ret) {
+        GF_FREE(priv);
+        this->private = NULL;
+    }
+
+    return ret;
+}
+
+void
+fini(xlator_t *this)
+{
+    leases_private_t *priv = NULL;
+
+    priv = this->private;
+    if (!priv) {
+        return;
+    }
+    this->private = NULL;
+
+    priv->fini = _gf_true;
+    pthread_cond_broadcast(&priv->cond);
+    if (priv->recall_thr) {
+        gf_thread_cleanup_xint(priv->recall_thr);
+        priv->recall_thr = 0;
+        priv->inited_recall_thr = _gf_false;
+    }
+
+    if (priv->timer_wheel) {
+        glusterfs_ctx_tw_put(this->ctx);
+    }
+
+    GF_FREE(priv);
+    return;
+}
+
+static int
+leases_forget(xlator_t *this, inode_t *inode)
+{
+    /* TODO:leases_cleanup_inode_ctx (this, inode); */
+    return 0;
+}
+
+static int
+leases_release(xlator_t *this, fd_t *fd)
+{
+    int ret = -1;
+    uint64_t tmp = 0;
+    lease_fd_ctx_t *fd_ctx = NULL;
+
+    if (fd == NULL) {
+        goto out;
+    }
+
+    gf_log(this->name, GF_LOG_TRACE, "Releasing all leases with fd %p", fd);
+
+    ret = fd_ctx_del(fd, this, &tmp);
+    if (ret) {
+        gf_log(this->name, GF_LOG_DEBUG, "Could not get fdctx");
+        goto out;
+    }
+
+    fd_ctx = (lease_fd_ctx_t *)(long)tmp;
+    if (fd_ctx)
+        GF_FREE(fd_ctx);
+out:
+    return ret;
+}
+
+static int
+leases_clnt_disconnect_cbk(xlator_t *this, client_t *client)
+{
+    int ret = 0;
+
+    EXIT_IF_LEASES_OFF(this, out);
+
+    ret = cleanup_client_leases(this, client->client_uid);
+out:
+    return ret;
+}
+
+struct xlator_fops fops = {
+    /* Metadata modifying fops */
+    .fsetattr = leases_fsetattr,
+    .setattr = leases_setattr,
+
+    /* File Data reading fops */
+    .open = leases_open,
+    .readv = leases_readv,
+
+    /* File Data modifying fops */
+    .truncate = leases_truncate,
+    .ftruncate = leases_ftruncate,
+    .writev = leases_writev,
+    .zerofill = leases_zerofill,
+    .fallocate = leases_fallocate,
+    .discard = leases_discard,
+    .lk = leases_lk,
+    .fsync = leases_fsync,
+    .flush = leases_flush,
+    .lease = leases_lease,
+
+    /* Directory Data modifying fops */
+    .create = leases_create,
+    .rename = leases_rename,
+    .unlink = leases_unlink,
+    .link = leases_link,
+
+#ifdef NOT_SUPPORTED
+    /* internal lk fops */
+    .inodelk = leases_inodelk,
+    .finodelk = leases_finodelk,
+    .entrylk = leases_entrylk,
+    .fentrylk = leases_fentrylk,
+
+    /* Internal special fops*/
+    .xattrop = leases_xattrop,
+    .fxattrop = leases_fxattrop,
+#endif
+};
+
+struct xlator_cbks cbks = {
+    .forget = leases_forget,
+    .release = leases_release,
+    .client_disconnect = leases_clnt_disconnect_cbk,
+};
+
+struct volume_options options[] = {
+    {.key = {"leases"},
+     .type = GF_OPTION_TYPE_BOOL,
+     .default_value = "off",
+     .op_version = {GD_OP_VERSION_3_8_0},
+     .flags = OPT_FLAG_SETTABLE | OPT_FLAG_DOC,
+     .description = "When \"on\", enables leases support"},
+    {.key = {"lease-lock-recall-timeout"},
+     .type = GF_OPTION_TYPE_INT,
+     .default_value = RECALL_LEASE_LK_TIMEOUT,
+     .op_version = {GD_OP_VERSION_3_8_0},
+     .flags = OPT_FLAG_SETTABLE | OPT_FLAG_DOC,
+     .description = "After 'timeout' seconds since the recall_lease"
+                    " request has been sent to the client, the lease lock"
+                    " will be forcefully purged by the server."},
+    {.key = {NULL}},
+};
+
+xlator_api_t xlator_api = {
+    .init = init,
+    .fini = fini,
+    .reconfigure = reconfigure,
+    .mem_acct_init = mem_acct_init,
+    .op_version = {1}, /* Present from the initial version */
+    .fops = &fops,
+    .cbks = &cbks,
+    .options = options,
+    .identifier = "leases",
+    .category = GF_MAINTAINED,
+};
diff --git a/xlators/features/leases/src/leases.h b/xlators/features/leases/src/leases.h
new file mode 100644
index 00000000000..a6e8a6824cc
--- /dev/null
+++ b/xlators/features/leases/src/leases.h
@@ -0,0 +1,259 @@
+/*
+  Copyright (c) 2015-2016 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
+*/
+
+#ifndef _LEASES_H
+#define _LEASES_H
+
+#ifndef _CONFIG_H
+#define _CONFIG_H
+#include "config.h"
+#endif
+
+#include <glusterfs/common-utils.h>
+#include <glusterfs/glusterfs.h>
+#include <glusterfs/xlator.h>
+#include <glusterfs/call-stub.h>
+#include <glusterfs/logging.h>
+#include <glusterfs/client_t.h>
+#include <glusterfs/lkowner.h>
+#include <glusterfs/locking.h>
+#include <glusterfs/upcall-utils.h>
+#include "timer-wheel.h"
+#include "leases-mem-types.h"
+#include "leases-messages.h"
+
+/* The time period for which a client lease lock will be stored after its been
+ * recalled for the first time. */
+#define RECALL_LEASE_LK_TIMEOUT "60"
+
+#define DATA_MODIFY_FOP 0x0001
+#define BLOCKING_FOP 0x0002
+
+#define BLOCK_FOP 0x0001
+#define WIND_FOP 0x0002
+
+#define EXIT_IF_LEASES_OFF(this, label)                                        \
+    do {                                                                       \
+        if (!is_leases_enabled(this))                                          \
+            goto label;                                                        \
+    } while (0)
+
+#define EXIT_IF_INTERNAL_FOP(frame, xdata, label)                              \
+    do {                                                                       \
+        if (frame->root->pid < 0)                                              \
+            goto label;                                                        \
+        if (xdata && dict_get(xdata, GLUSTERFS_INTERNAL_FOP_KEY))              \
+            goto label;                                                        \
+    } while (0)
+
+#define GET_LEASE_ID(xdata, lease_id, client_uid)                              \
+    do {                                                                       \
+        int ret_val = -1;                                                      \
+        ret_val = dict_get_bin(xdata, "lease-id", (void **)&lease_id);         \
+        if (ret_val) {                                                         \
+            ret_val = 0;                                                       \
+            gf_msg_debug("leases", 0, "Lease id is not set for client:%s",     \
+                         client_uid);                                          \
+        }                                                                      \
+    } while (0)
+
+#define GET_FLAGS(fop, fd_flags)                                               \
+    do {                                                                       \
+        if ((fd_flags & (O_WRONLY | O_RDWR)) && fop == GF_FOP_OPEN)            \
+            fop_flags = DATA_MODIFY_FOP;                                       \
+                                                                               \
+        if (fop == GF_FOP_UNLINK || fop == GF_FOP_RENAME ||                    \
+            fop == GF_FOP_TRUNCATE || fop == GF_FOP_FTRUNCATE ||               \
+            fop == GF_FOP_FLUSH || fop == GF_FOP_FSYNC ||                      \
+            fop == GF_FOP_WRITE || fop == GF_FOP_FALLOCATE ||                  \
+            fop == GF_FOP_DISCARD || fop == GF_FOP_ZEROFILL ||                 \
+            fop == GF_FOP_SETATTR || fop == GF_FOP_FSETATTR ||                 \
+            fop == GF_FOP_LINK)                                                \
+            fop_flags = DATA_MODIFY_FOP;                                       \
+                                                                               \
+        if (!(fd_flags & (O_NONBLOCK | O_NDELAY)))                             \
+            fop_flags |= BLOCKING_FOP;                                         \
+                                                                               \
+    } while (0)
+
+#define GET_FLAGS_LK(cmd, l_type, fd_flags)                                    \
+    do {                                                                       \
+        /* TODO: handle F_RESLK_LCK and other glusterfs_lk_recovery_cmds_t */  \
+        if ((cmd == F_SETLKW || cmd == F_SETLKW64 || cmd == F_SETLK ||         \
+             cmd == F_SETLK64) &&                                              \
+            l_type == F_WRLCK)                                                 \
+            fop_flags = DATA_MODIFY_FOP;                                       \
+                                                                               \
+        if (fd_flags & (O_NONBLOCK | O_NDELAY) &&                              \
+            (cmd == F_SETLKW || cmd == F_SETLKW64))                            \
+            fop_flags |= BLOCKING_FOP;                                         \
+                                                                               \
+    } while (0)
+
+#define LEASE_BLOCK_FOP(inode, fop_name, frame, this, params...)               \
+    do {                                                                       \
+        call_stub_t *__stub = NULL;                                            \
+        fop_stub_t *blk_fop = NULL;                                            \
+        lease_inode_ctx_t *lease_ctx = NULL;                                   \
+                                                                               \
+        __stub = fop_##fop_name##_stub(frame, default_##fop_name##_resume,     \
+                                       params);                                \
+        if (!__stub) {                                                         \
+            gf_msg(this->name, GF_LOG_WARNING, ENOMEM, LEASE_MSG_NO_MEM,       \
+                   "Unable to create stub");                                   \
+            ret = -ENOMEM;                                                     \
+            goto __out;                                                        \
+        }                                                                      \
+                                                                               \
+        blk_fop = GF_CALLOC(1, sizeof(*blk_fop), gf_leases_mt_fop_stub_t);     \
+        if (!blk_fop) {                                                        \
+            gf_msg(this->name, GF_LOG_WARNING, ENOMEM, LEASE_MSG_NO_MEM,       \
+                   "Unable to create lease fop stub");                         \
+            ret = -ENOMEM;                                                     \
+            goto __out;                                                        \
+        }                                                                      \
+                                                                               \
+        lease_ctx = lease_ctx_get(inode, this);                                \
+        if (!lease_ctx) {                                                      \
+            gf_msg(this->name, GF_LOG_WARNING, ENOMEM, LEASE_MSG_NO_MEM,       \
+                   "Unable to create/get inode ctx");                          \
+            ret = -ENOMEM;                                                     \
+            goto __out;                                                        \
+        }                                                                      \
+                                                                               \
+        blk_fop->stub = __stub;                                                \
+        pthread_mutex_lock(&lease_ctx->lock);                                  \
+        {                                                                      \
+            /*TODO: If the lease is unlocked btw check lease conflict and      \
+             * by now, then this fop shouldn't be add to the blocked fop       \
+             * list, can use generation number for the same?*/                 \
+            list_add_tail(&blk_fop->list, &lease_ctx->blocked_list);           \
+        }                                                                      \
+        pthread_mutex_unlock(&lease_ctx->lock);                                \
+                                                                               \
+    __out:                                                                     \
+        if (ret < 0) {                                                         \
+            gf_msg(this->name, GF_LOG_WARNING, ENOMEM, LEASE_MSG_NO_MEM,       \
+                   "Unable to create stub for blocking the fop:%s (%s)",       \
+                   gf_fop_list[frame->root->op], strerror(ENOMEM));            \
+            if (__stub != NULL) {                                              \
+                call_stub_destroy(__stub);                                     \
+            }                                                                  \
+            GF_FREE(blk_fop);                                                  \
+            goto err;                                                          \
+        }                                                                      \
+    } while (0)
+
+struct _leases_private {
+    struct list_head client_list;
+    struct list_head recall_list;
+    struct tvec_base *timer_wheel; /* timer wheel where the recall request
+                                      is qued and waits for unlock/expiry */
+    pthread_t recall_thr;
+    pthread_mutex_t mutex;
+    pthread_cond_t cond;
+    int32_t recall_lease_timeout;
+    gf_boolean_t inited_recall_thr;
+    gf_boolean_t fini;
+    gf_boolean_t leases_enabled;
+
+    char _pad[1]; /* manual padding */
+};
+typedef struct _leases_private leases_private_t;
+
+struct _lease_client {
+    char *client_uid;
+    struct list_head client_list;
+    struct list_head inode_list;
+};
+typedef struct _lease_client lease_client_t;
+
+struct _lease_inode {
+    inode_t *inode;
+    struct list_head
+        list; /* This can be part of both inode_list and recall_list */
+};
+typedef struct _lease_inode lease_inode_t;
+
+struct _lease_fd_ctx {
+    char *client_uid;
+    char lease_id[LEASE_ID_SIZE];
+};
+typedef struct _lease_fd_ctx lease_fd_ctx_t;
+
+struct _lease_inode_ctx {
+    struct list_head lease_id_list; /* clients that have taken leases */
+    int lease_type_cnt[GF_LEASE_MAX_TYPE + 1];
+    uint64_t lease_cnt;            /* Total number of leases on this inode */
+    uint64_t openfd_cnt;           /* number of fds open */
+    struct list_head blocked_list; /* List of fops blocked until the
+                                      lease recall is complete */
+    inode_t *inode;                /* this represents the inode on which the
+                                      lock was taken, required mainly during
+                                      disconnect cleanup */
+    struct gf_tw_timer_list *timer;
+    pthread_mutex_t lock;
+    int lease_type;                  /* Types of leases acquired */
+    gf_boolean_t recall_in_progress; /* if lease recall is sent on this inode */
+    gf_boolean_t blocked_fops_resuming; /* if blocked fops are being resumed */
+
+    char _pad[2]; /* manual padding */
+};
+typedef struct _lease_inode_ctx lease_inode_ctx_t;
+
+struct _lease_id_entry {
+    struct list_head lease_id_list;
+    char lease_id[LEASE_ID_SIZE];
+    char *client_uid;                          /* uid of the client that has
+                                                  taken the lease */
+    int lease_type_cnt[GF_LEASE_MAX_TYPE + 1]; /* count of each lease type */
+    uint64_t lease_cnt; /* Number of leases taken under the
+                           given lease id */
+    time_t recall_time; /* time @ which recall was sent */
+    int lease_type;     /* Union of all the leases taken
+                           under the given lease id */
+    char _pad[4];       /* manual padding */
+};
+typedef struct _lease_id_entry lease_id_entry_t;
+
+/* Required? as stub itself will have list */
+struct __fop_stub {
+    struct list_head list;
+    call_stub_t *stub;
+};
+typedef struct __fop_stub fop_stub_t;
+
+struct __lease_timer_data {
+    inode_t *inode;
+    xlator_t *this;
+};
+typedef struct __lease_timer_data lease_timer_data_t;
+
+gf_boolean_t
+is_leases_enabled(xlator_t *this);
+
+lease_inode_ctx_t *
+lease_ctx_get(inode_t *inode, xlator_t *this);
+
+int
+process_lease_req(call_frame_t *frame, xlator_t *this, inode_t *inode,
+                  struct gf_lease *lease);
+
+int
+check_lease_conflict(call_frame_t *frame, inode_t *inode, const char *lease_id,
+                     uint32_t fop_flags);
+
+int
+cleanup_client_leases(xlator_t *this, const char *client_uid);
+
+void *
+expired_recall_cleanup(void *data);
+
+#endif /* _LEASES_H */
diff --git a/xlators/features/locks/src/Makefile.am b/xlators/features/locks/src/Makefile.am
index 0f79731b415..0b174c19d2d 100644
--- a/xlators/features/locks/src/Makefile.am
+++ b/xlators/features/locks/src/Makefile.am
@@ -1,23 +1,29 @@
+if WITH_SERVER
 xlator_LTLIBRARIES = locks.la
+endif
 xlatordir = $(libdir)/glusterfs/$(PACKAGE_VERSION)/xlator/features
 
-locks_la_LDFLAGS = -module -avoid-version
+locks_la_LDFLAGS = -module $(GF_XLATOR_DEFAULT_LDFLAGS)
 
 locks_la_SOURCES = common.c posix.c entrylk.c inodelk.c reservelk.c \
-		   clear.c
+	clear.c
+
 locks_la_LIBADD = $(top_builddir)/libglusterfs/src/libglusterfs.la
 
-noinst_HEADERS = locks.h common.h locks-mem-types.h clear.h
+noinst_HEADERS = locks.h common.h locks-mem-types.h clear.h pl-messages.h
 
-AM_CPPFLAGS = $(GF_CPPFLAGS) -I$(top_srcdir)/libglusterfs/src
+AM_CPPFLAGS = $(GF_CPPFLAGS) -I$(top_srcdir)/libglusterfs/src \
+	-I$(top_srcdir)/rpc/xdr/src -I$(top_builddir)/rpc/xdr/src
 
 
 AM_CFLAGS = -Wall -fno-strict-aliasing $(GF_CFLAGS)
 
 CLEANFILES = 
 
+if WITH_SERVER
 uninstall-local:
 	rm -f $(DESTDIR)$(xlatordir)/posix-locks.so
 
 install-data-hook:
 	ln -sf locks.so $(DESTDIR)$(xlatordir)/posix-locks.so
+endif
diff --git a/xlators/features/locks/src/clear.c b/xlators/features/locks/src/clear.c
index 75593b8988c..ab1eac68a53 100644
--- a/xlators/features/locks/src/clear.c
+++ b/xlators/features/locks/src/clear.c
@@ -12,412 +12,449 @@
 #include <limits.h>
 #include <pthread.h>
 
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
-
-#include "glusterfs.h"
-#include "compat.h"
-#include "xlator.h"
-#include "inode.h"
-#include "logging.h"
-#include "common-utils.h"
+#include <glusterfs/glusterfs.h>
+#include <glusterfs/compat.h>
+#include <glusterfs/xlator.h>
+#include <glusterfs/logging.h>
+#include <glusterfs/common-utils.h>
 
 #include "locks.h"
 #include "common.h"
-#include "statedump.h"
+#include <glusterfs/statedump.h>
 #include "clear.h"
 
+const char *clrlk_type_names[CLRLK_TYPE_MAX] = {
+    [CLRLK_INODE] = "inode",
+    [CLRLK_ENTRY] = "entry",
+    [CLRLK_POSIX] = "posix",
+};
+
 int
-clrlk_get_kind (char *kind)
+clrlk_get_kind(char *kind)
 {
-        char  *clrlk_kinds[CLRLK_KIND_MAX] = {"dummy", "blocked", "granted",
-                                              "all"};
-        int   ret_kind                     = CLRLK_KIND_MAX;
-        int   i                            = 0;
-
-        for (i = CLRLK_BLOCKED; i < CLRLK_KIND_MAX; i++) {
-                if (!strcmp (clrlk_kinds[i], kind)) {
-                        ret_kind = i;
-                        break;
-                }
+    char *clrlk_kinds[CLRLK_KIND_MAX] = {"dummy", "blocked", "granted", "all"};
+    int ret_kind = CLRLK_KIND_MAX;
+    int i = 0;
+
+    for (i = CLRLK_BLOCKED; i < CLRLK_KIND_MAX; i++) {
+        if (!strcmp(clrlk_kinds[i], kind)) {
+            ret_kind = i;
+            break;
         }
+    }
 
-        return ret_kind;
+    return ret_kind;
 }
 
 int
-clrlk_get_type (char *type)
+clrlk_get_type(char *type)
 {
-        char    *clrlk_types[CLRLK_TYPE_MAX]    = {"inode", "entry", "posix"};
-        int     ret_type                        = CLRLK_TYPE_MAX;
-        int     i                               = 0;
-
-        for (i = CLRLK_INODE; i < CLRLK_TYPE_MAX; i++) {
-                if (!strcmp (clrlk_types[i], type)) {
-                        ret_type = i;
-                        break;
-                }
+    char *clrlk_types[CLRLK_TYPE_MAX] = {"inode", "entry", "posix"};
+    int ret_type = CLRLK_TYPE_MAX;
+    int i = 0;
+
+    for (i = CLRLK_INODE; i < CLRLK_TYPE_MAX; i++) {
+        if (!strcmp(clrlk_types[i], type)) {
+            ret_type = i;
+            break;
         }
+    }
 
-        return ret_type;
+    return ret_type;
 }
 
 int
-clrlk_get_lock_range (char *range_str, struct gf_flock *ulock,
-                      gf_boolean_t *chk_range)
+clrlk_get_lock_range(char *range_str, struct gf_flock *ulock,
+                     gf_boolean_t *chk_range)
 {
-        int     ret     =       -1;
-
-        if (!chk_range)
-                goto out;
+    int ret = -1;
 
-        if (!range_str) {
-                ret = 0;
-                *chk_range = _gf_false;
-                goto out;
-        }
-
-        if (sscanf (range_str, "%hd,%"PRId64"-""%"PRId64, &ulock->l_whence,
-                    &ulock->l_start, &ulock->l_len) != 3) {
-                goto out;
-        }
+    if (!chk_range)
+        goto out;
 
+    if (!range_str) {
         ret = 0;
-        *chk_range = _gf_true;
+        *chk_range = _gf_false;
+        goto out;
+    }
+
+    if (sscanf(range_str,
+               "%hd,%" PRId64 "-"
+               "%" PRId64,
+               &ulock->l_whence, &ulock->l_start, &ulock->l_len) != 3) {
+        goto out;
+    }
+
+    ret = 0;
+    *chk_range = _gf_true;
 out:
-        return ret;
+    return ret;
 }
 
 int
-clrlk_parse_args (const char* cmd, clrlk_args *args)
+clrlk_parse_args(const char *cmd, clrlk_args *args)
 {
-        char            *opts           = NULL;
-        char            *cur            = NULL;
-        char            *tok            = NULL;
-        char            *sptr           = NULL;
-        char            *free_ptr       = NULL;
-        char            kw[KW_MAX]     = {[KW_TYPE]     = 't',
-                                          [KW_KIND]     = 'k',
-                                          };
-        int             ret             = -1;
-        int             i               = 0;
-
-        GF_ASSERT (cmd);
-        free_ptr = opts = GF_CALLOC (1, strlen (cmd), gf_common_mt_char);
-        if (!opts)
-                goto out;
-
-        if (sscanf (cmd, GF_XATTR_CLRLK_CMD".%s", opts) < 1) {
-                ret = -1;
-                goto out;
+    char *opts = NULL;
+    char *cur = NULL;
+    char *tok = NULL;
+    char *sptr = NULL;
+    char *free_ptr = NULL;
+    char kw[KW_MAX] = {
+        [KW_TYPE] = 't',
+        [KW_KIND] = 'k',
+    };
+    int ret = -1;
+    int i = 0;
+
+    GF_ASSERT(cmd);
+    free_ptr = opts = GF_CALLOC(1, strlen(cmd), gf_common_mt_char);
+    if (!opts)
+        goto out;
+
+    if (sscanf(cmd, GF_XATTR_CLRLK_CMD ".%s", opts) < 1) {
+        ret = -1;
+        goto out;
+    }
+
+    /*clr_lk_prefix.ttype.kkind.args, args - type specific*/
+    cur = opts;
+    for (i = 0; i < KW_MAX && (tok = strtok_r(cur, ".", &sptr));
+         cur = NULL, i++) {
+        if (tok[0] != kw[i]) {
+            ret = -1;
+            goto out;
         }
-
-        /*clr_lk_prefix.ttype.kkind.args, args - type specific*/
-        cur = opts;
-        for (i = 0; i < KW_MAX && (tok = strtok_r (cur, ".", &sptr));
-             cur = NULL, i++) {
-                if (tok[0] != kw[i]) {
-                        ret = -1;
-                        goto out;
-                }
-                if (i == KW_TYPE)
-                        args->type = clrlk_get_type (tok+1);
-                if (i == KW_KIND)
-                        args->kind = clrlk_get_kind (tok+1);
-        }
-
-        if ((args->type == CLRLK_TYPE_MAX) || (args->kind == CLRLK_KIND_MAX))
-                goto out;
-
-        /*optional args, neither range nor basename can 'legally' contain
-         * "/" in them*/
-        tok = strtok_r (NULL, "/", &sptr);
-        if (tok)
-                args->opts = gf_strdup (tok);
-
-        ret = 0;
+        if (i == KW_TYPE)
+            args->type = clrlk_get_type(tok + 1);
+        if (i == KW_KIND)
+            args->kind = clrlk_get_kind(tok + 1);
+    }
+
+    if ((args->type == CLRLK_TYPE_MAX) || (args->kind == CLRLK_KIND_MAX))
+        goto out;
+
+    /*optional args, neither range nor basename can 'legally' contain
+     * "/" in them*/
+    tok = strtok_r(NULL, "/", &sptr);
+    if (tok)
+        args->opts = gf_strdup(tok);
+
+    ret = 0;
 out:
-        GF_FREE (free_ptr);
-        return ret;
+    GF_FREE(free_ptr);
+    return ret;
 }
 
 int
-clrlk_clear_posixlk (xlator_t *this, pl_inode_t *pl_inode, clrlk_args *args,
-                     int *blkd, int *granted, int *op_errno)
+clrlk_clear_posixlk(xlator_t *this, pl_inode_t *pl_inode, clrlk_args *args,
+                    int *blkd, int *granted, int *op_errno)
 {
-        posix_lock_t            *plock          = NULL;
-        posix_lock_t            *tmp            = NULL;
-        struct gf_flock         ulock           = {0, };
-        int                     ret             = -1;
-        int                     bcount          = 0;
-        int                     gcount          = 0;
-        gf_boolean_t            chk_range       = _gf_false;
-
-        if (clrlk_get_lock_range (args->opts, &ulock, &chk_range)) {
-                *op_errno = EINVAL;
-                goto out;
-        }
-
-        pthread_mutex_lock (&pl_inode->mutex);
+    posix_lock_t *plock = NULL;
+    posix_lock_t *tmp = NULL;
+    struct gf_flock ulock = {
+        0,
+    };
+    int ret = -1;
+    int bcount = 0;
+    int gcount = 0;
+    gf_boolean_t chk_range = _gf_false;
+
+    if (clrlk_get_lock_range(args->opts, &ulock, &chk_range)) {
+        *op_errno = EINVAL;
+        goto out;
+    }
+
+    pthread_mutex_lock(&pl_inode->mutex);
+    {
+        list_for_each_entry_safe(plock, tmp, &pl_inode->ext_list, list)
         {
-                list_for_each_entry_safe (plock, tmp, &pl_inode->ext_list,
-                                          list) {
-                        if ((plock->blocked &&
-                             !(args->kind & CLRLK_BLOCKED)) ||
-                            (!plock->blocked &&
-                             !(args->kind & CLRLK_GRANTED)))
-                                continue;
-
-                        if (chk_range &&
-                            (plock->user_flock.l_whence != ulock.l_whence
-                            || plock->user_flock.l_start != ulock.l_start
-                            || plock->user_flock.l_len != ulock.l_len))
-                                continue;
-
-                        list_del_init (&plock->list);
-                        if (plock->blocked) {
-                                bcount++;
-                                pl_trace_out (this, plock->frame, NULL, NULL,
-                                              F_SETLKW, &plock->user_flock,
-                                              -1, EAGAIN, NULL);
-
-                                STACK_UNWIND_STRICT (lk, plock->frame, -1, EAGAIN,
-                                                     &plock->user_flock, NULL);
-
-                        } else {
-                                gcount++;
-                        }
-                        GF_FREE (plock);
-                }
+            if ((plock->blocked && !(args->kind & CLRLK_BLOCKED)) ||
+                (!plock->blocked && !(args->kind & CLRLK_GRANTED)))
+                continue;
+
+            if (chk_range && (plock->user_flock.l_whence != ulock.l_whence ||
+                              plock->user_flock.l_start != ulock.l_start ||
+                              plock->user_flock.l_len != ulock.l_len))
+                continue;
+
+            list_del_init(&plock->list);
+            if (plock->blocked) {
+                bcount++;
+                pl_trace_out(this, plock->frame, NULL, NULL, F_SETLKW,
+                             &plock->user_flock, -1, EINTR, NULL);
+
+                STACK_UNWIND_STRICT(lk, plock->frame, -1, EINTR,
+                                    &plock->user_flock, NULL);
+
+            } else {
+                gcount++;
+            }
+            __destroy_lock(plock);
         }
-        pthread_mutex_unlock (&pl_inode->mutex);
-        grant_blocked_locks (this, pl_inode);
-        ret = 0;
+    }
+    pthread_mutex_unlock(&pl_inode->mutex);
+    grant_blocked_locks(this, pl_inode);
+    ret = 0;
 out:
-        *blkd    = bcount;
-        *granted = gcount;
-        return ret;
+    *blkd = bcount;
+    *granted = gcount;
+    return ret;
 }
 
 /* Returns 0 on success and -1 on failure */
 int
-clrlk_clear_inodelk (xlator_t *this, pl_inode_t *pl_inode, pl_dom_list_t *dom,
-                     clrlk_args *args, int *blkd, int *granted, int *op_errno)
+clrlk_clear_inodelk(xlator_t *this, pl_inode_t *pl_inode, pl_dom_list_t *dom,
+                    clrlk_args *args, int *blkd, int *granted, int *op_errno)
 {
-        pl_inode_lock_t         *ilock          = NULL;
-        pl_inode_lock_t         *tmp            = NULL;
-        struct gf_flock         ulock           = {0, };
-        int                     ret             = -1;
-        int                     bcount          = 0;
-        int                     gcount          = 0;
-        gf_boolean_t            chk_range       = _gf_false;
-        struct list_head        released;
-
-        INIT_LIST_HEAD (&released);
-        if (clrlk_get_lock_range (args->opts, &ulock, &chk_range)) {
-                *op_errno = EINVAL;
-                goto out;
-        }
-
-        if (args->kind & CLRLK_BLOCKED)
-                goto blkd;
-
-        if (args->kind & CLRLK_GRANTED)
-                goto granted;
+    posix_locks_private_t *priv;
+    pl_inode_lock_t *ilock = NULL;
+    pl_inode_lock_t *tmp = NULL;
+    struct gf_flock ulock = {
+        0,
+    };
+    int ret = -1;
+    int bcount = 0;
+    int gcount = 0;
+    gf_boolean_t chk_range = _gf_false;
+    struct list_head *pcontend = NULL;
+    struct list_head released;
+    struct list_head contend;
+    struct timespec now = {};
+
+    INIT_LIST_HEAD(&released);
+
+    priv = this->private;
+    if (priv->notify_contention) {
+        pcontend = &contend;
+        INIT_LIST_HEAD(pcontend);
+        timespec_now(&now);
+    }
+
+    if (clrlk_get_lock_range(args->opts, &ulock, &chk_range)) {
+        *op_errno = EINVAL;
+        goto out;
+    }
+
+    if (args->kind & CLRLK_BLOCKED)
+        goto blkd;
+
+    if (args->kind & CLRLK_GRANTED)
+        goto granted;
 
 blkd:
-        pthread_mutex_lock (&pl_inode->mutex);
+    pthread_mutex_lock(&pl_inode->mutex);
+    {
+        list_for_each_entry_safe(ilock, tmp, &dom->blocked_inodelks,
+                                 blocked_locks)
         {
-                list_for_each_entry_safe (ilock, tmp, &dom->blocked_inodelks,
-                                          blocked_locks) {
-                        if (chk_range &&
-                            (ilock->user_flock.l_whence != ulock.l_whence
-                            || ilock->user_flock.l_start != ulock.l_start
-                            || ilock->user_flock.l_len != ulock.l_len))
-                                continue;
-
-                        bcount++;
-                        list_del_init (&ilock->blocked_locks);
-                        list_add (&ilock->blocked_locks, &released);
-                }
-        }
-        pthread_mutex_unlock (&pl_inode->mutex);
-
-        list_for_each_entry_safe (ilock, tmp, &released, blocked_locks) {
-                list_del_init (&ilock->blocked_locks);
-                pl_trace_out (this, ilock->frame, NULL, NULL, F_SETLKW,
-                              &ilock->user_flock, -1, EAGAIN,
-                              ilock->volume);
-                STACK_UNWIND_STRICT (inodelk, ilock->frame, -1,
-                                     EAGAIN, NULL);
-                //No need to take lock as the locks are only in one list
-                __pl_inodelk_unref (ilock);
+            if (chk_range && (ilock->user_flock.l_whence != ulock.l_whence ||
+                              ilock->user_flock.l_start != ulock.l_start ||
+                              ilock->user_flock.l_len != ulock.l_len))
+                continue;
+
+            bcount++;
+            list_del_init(&ilock->client_list);
+            list_del_init(&ilock->blocked_locks);
+            list_add(&ilock->blocked_locks, &released);
         }
+    }
+    pthread_mutex_unlock(&pl_inode->mutex);
 
-        if (!(args->kind & CLRLK_GRANTED)) {
-                ret = 0;
-                goto out;
+    if (!list_empty(&released)) {
+        list_for_each_entry_safe(ilock, tmp, &released, blocked_locks)
+        {
+            list_del_init(&ilock->blocked_locks);
+            pl_trace_out(this, ilock->frame, NULL, NULL, F_SETLKW,
+                         &ilock->user_flock, -1, EAGAIN, ilock->volume);
+            STACK_UNWIND_STRICT(inodelk, ilock->frame, -1, EAGAIN, NULL);
+            // No need to take lock as the locks are only in one list
+            __pl_inodelk_unref(ilock);
         }
+    }
+
+    if (!(args->kind & CLRLK_GRANTED)) {
+        ret = 0;
+        goto out;
+    }
 
 granted:
-        pthread_mutex_lock (&pl_inode->mutex);
+    pthread_mutex_lock(&pl_inode->mutex);
+    {
+        list_for_each_entry_safe(ilock, tmp, &dom->inodelk_list, list)
         {
-                list_for_each_entry_safe (ilock, tmp, &dom->inodelk_list,
-                                          list) {
-                        if (chk_range &&
-                            (ilock->user_flock.l_whence != ulock.l_whence
-                            || ilock->user_flock.l_start != ulock.l_start
-                            || ilock->user_flock.l_len != ulock.l_len))
-                                continue;
-
-                        gcount++;
-                        list_del_init (&ilock->list);
-                        list_add (&ilock->list, &released);
-                }
+            if (chk_range && (ilock->user_flock.l_whence != ulock.l_whence ||
+                              ilock->user_flock.l_start != ulock.l_start ||
+                              ilock->user_flock.l_len != ulock.l_len))
+                continue;
+
+            gcount++;
+            list_del_init(&ilock->client_list);
+            list_del_init(&ilock->list);
+            list_add(&ilock->list, &released);
         }
-        pthread_mutex_unlock (&pl_inode->mutex);
+    }
+    pthread_mutex_unlock(&pl_inode->mutex);
 
-        list_for_each_entry_safe (ilock, tmp, &released, list) {
-                list_del_init (&ilock->list);
-                //No need to take lock as the locks are only in one list
-                __pl_inodelk_unref (ilock);
-        }
+    list_for_each_entry_safe(ilock, tmp, &released, list)
+    {
+        list_del_init(&ilock->list);
+        // No need to take lock as the locks are only in one list
+        __pl_inodelk_unref(ilock);
+    }
 
-        ret = 0;
+    ret = 0;
 out:
-        grant_blocked_inode_locks (this, pl_inode, dom);
-        *blkd    = bcount;
-        *granted = gcount;
-        return ret;
+    grant_blocked_inode_locks(this, pl_inode, dom, &now, pcontend);
+    if (pcontend != NULL) {
+        inodelk_contention_notify(this, pcontend);
+    }
+    *blkd = bcount;
+    *granted = gcount;
+    return ret;
 }
 
 /* Returns 0 on success and -1 on failure */
 int
-clrlk_clear_entrylk (xlator_t *this, pl_inode_t *pl_inode, pl_dom_list_t *dom,
-                     clrlk_args *args, int *blkd, int *granted, int *op_errno)
+clrlk_clear_entrylk(xlator_t *this, pl_inode_t *pl_inode, pl_dom_list_t *dom,
+                    clrlk_args *args, int *blkd, int *granted, int *op_errno)
 {
-        pl_entry_lock_t         *elock          = NULL;
-        pl_entry_lock_t         *tmp            = NULL;
-        int                     bcount          = 0;
-        int                     gcount          = 0;
-        int                     ret             = -1;
-        struct list_head        removed;
-        struct list_head        released;
-
-        INIT_LIST_HEAD (&released);
-        if (args->kind & CLRLK_BLOCKED)
-                goto blkd;
-
-        if (args->kind & CLRLK_GRANTED)
-                goto granted;
+    posix_locks_private_t *priv;
+    pl_entry_lock_t *elock = NULL;
+    pl_entry_lock_t *tmp = NULL;
+    int bcount = 0;
+    int gcount = 0;
+    int ret = -1;
+    struct list_head *pcontend = NULL;
+    struct list_head removed;
+    struct list_head released;
+    struct list_head contend;
+    struct timespec now;
+
+    INIT_LIST_HEAD(&released);
+
+    priv = this->private;
+    if (priv->notify_contention) {
+        pcontend = &contend;
+        INIT_LIST_HEAD(pcontend);
+        timespec_now(&now);
+    }
+
+    if (args->kind & CLRLK_BLOCKED)
+        goto blkd;
+
+    if (args->kind & CLRLK_GRANTED)
+        goto granted;
 
 blkd:
-        pthread_mutex_lock (&pl_inode->mutex);
+    pthread_mutex_lock(&pl_inode->mutex);
+    {
+        list_for_each_entry_safe(elock, tmp, &dom->blocked_entrylks,
+                                 blocked_locks)
         {
-                list_for_each_entry_safe (elock, tmp, &dom->blocked_entrylks,
-                                          blocked_locks) {
-                        if (args->opts) {
-                                if (!elock->basename ||
-                                    strcmp (elock->basename, args->opts))
-                                        continue;
-                        }
-
-                        bcount++;
-
-                        list_del_init (&elock->blocked_locks);
-                        list_add_tail (&elock->blocked_locks, &released);
-                }
-        }
-        pthread_mutex_unlock (&pl_inode->mutex);
+            if (args->opts) {
+                if (!elock->basename || strcmp(elock->basename, args->opts))
+                    continue;
+            }
 
-        list_for_each_entry_safe (elock, tmp, &released, blocked_locks) {
-                list_del_init (&elock->blocked_locks);
-                entrylk_trace_out (this, elock->frame, elock->volume, NULL, NULL,
-                                   elock->basename, ENTRYLK_LOCK, elock->type,
-                                   -1, EAGAIN);
-                STACK_UNWIND_STRICT (entrylk, elock->frame, -1, EAGAIN, NULL);
+            bcount++;
 
-		__pl_entrylk_unref (elock);
+            list_del_init(&elock->client_list);
+            list_del_init(&elock->blocked_locks);
+            list_add_tail(&elock->blocked_locks, &released);
         }
+    }
+    pthread_mutex_unlock(&pl_inode->mutex);
+
+    if (!list_empty(&released)) {
+        list_for_each_entry_safe(elock, tmp, &released, blocked_locks)
+        {
+            list_del_init(&elock->blocked_locks);
+            entrylk_trace_out(this, elock->frame, elock->volume, NULL, NULL,
+                              elock->basename, ENTRYLK_LOCK, elock->type, -1,
+                              EAGAIN);
+            STACK_UNWIND_STRICT(entrylk, elock->frame, -1, EAGAIN, NULL);
 
-        if (!(args->kind & CLRLK_GRANTED)) {
-                ret = 0;
-                goto out;
+            __pl_entrylk_unref(elock);
         }
+    }
+
+    if (!(args->kind & CLRLK_GRANTED)) {
+        ret = 0;
+        goto out;
+    }
 
 granted:
-        INIT_LIST_HEAD (&removed);
-        pthread_mutex_lock (&pl_inode->mutex);
+    INIT_LIST_HEAD(&removed);
+    pthread_mutex_lock(&pl_inode->mutex);
+    {
+        list_for_each_entry_safe(elock, tmp, &dom->entrylk_list, domain_list)
         {
-                list_for_each_entry_safe (elock, tmp, &dom->entrylk_list,
-                                          domain_list) {
-                        if (args->opts) {
-                                if (!elock->basename ||
-                                    strcmp (elock->basename, args->opts))
-                                        continue;
-                        }
-
-                        gcount++;
-                        list_del_init (&elock->domain_list);
-                        list_add_tail (&elock->domain_list, &removed);
-
-			__pl_entrylk_unref (elock);
-                }
+            if (args->opts) {
+                if (!elock->basename || strcmp(elock->basename, args->opts))
+                    continue;
+            }
+
+            gcount++;
+            list_del_init(&elock->client_list);
+            list_del_init(&elock->domain_list);
+            list_add_tail(&elock->domain_list, &removed);
+
+            __pl_entrylk_unref(elock);
         }
-        pthread_mutex_unlock (&pl_inode->mutex);
+    }
+    pthread_mutex_unlock(&pl_inode->mutex);
 
-	grant_blocked_entry_locks (this, pl_inode, dom);
+    grant_blocked_entry_locks(this, pl_inode, dom, &now, pcontend);
+    if (pcontend != NULL) {
+        entrylk_contention_notify(this, pcontend);
+    }
 
-        ret = 0;
+    ret = 0;
 out:
-        *blkd    = bcount;
-        *granted = gcount;
-        return ret;
+    *blkd = bcount;
+    *granted = gcount;
+    return ret;
 }
 
 int
-clrlk_clear_lks_in_all_domains (xlator_t *this, pl_inode_t *pl_inode,
-                                clrlk_args *args, int *blkd, int *granted,
-                                int *op_errno)
+clrlk_clear_lks_in_all_domains(xlator_t *this, pl_inode_t *pl_inode,
+                               clrlk_args *args, int *blkd, int *granted,
+                               int *op_errno)
 {
-        pl_dom_list_t   *dom            = NULL;
-        int             ret             = -1;
-        int             tmp_bcount      = 0;
-        int             tmp_gcount      = 0;
-
-        if (list_empty (&pl_inode->dom_list)) {
-                ret = 0;
-                goto out;
-        }
+    pl_dom_list_t *dom = NULL;
+    int ret = -1;
+    int tmp_bcount = 0;
+    int tmp_gcount = 0;
 
-        list_for_each_entry (dom, &pl_inode->dom_list, inode_list) {
-                tmp_bcount = tmp_gcount = 0;
-
-                switch (args->type)
-                {
-                case CLRLK_INODE:
-                        ret = clrlk_clear_inodelk (this, pl_inode, dom, args,
-                                             &tmp_bcount, &tmp_gcount,
-                                             op_errno);
-                        if (ret)
-                                goto out;
-                        break;
-                case CLRLK_ENTRY:
-                        ret = clrlk_clear_entrylk (this, pl_inode, dom, args,
-                                             &tmp_bcount, &tmp_gcount,
-                                             op_errno);
-                        if (ret)
-                                goto out;
-                        break;
-                }
-
-                *blkd    += tmp_bcount;
-                *granted += tmp_gcount;
+    if (list_empty(&pl_inode->dom_list)) {
+        ret = 0;
+        goto out;
+    }
+
+    list_for_each_entry(dom, &pl_inode->dom_list, inode_list)
+    {
+        tmp_bcount = tmp_gcount = 0;
+
+        switch (args->type) {
+            case CLRLK_INODE:
+                ret = clrlk_clear_inodelk(this, pl_inode, dom, args,
+                                          &tmp_bcount, &tmp_gcount, op_errno);
+                if (ret)
+                    goto out;
+                break;
+            case CLRLK_ENTRY:
+                ret = clrlk_clear_entrylk(this, pl_inode, dom, args,
+                                          &tmp_bcount, &tmp_gcount, op_errno);
+                if (ret)
+                    goto out;
+                break;
         }
 
-        ret = 0;
+        *blkd += tmp_bcount;
+        *granted += tmp_gcount;
+    }
+
+    ret = 0;
 out:
-        return ret;
+    return ret;
 }
diff --git a/xlators/features/locks/src/clear.h b/xlators/features/locks/src/clear.h
index 511f3f74ae5..bc118cb1b81 100644
--- a/xlators/features/locks/src/clear.h
+++ b/xlators/features/locks/src/clear.h
@@ -10,67 +10,64 @@
 #ifndef __CLEAR_H__
 #define __CLEAR_H__
 
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
-
-#include "compat-errno.h"
-#include "stack.h"
-#include "call-stub.h"
+#include <glusterfs/compat-errno.h>
+#include <glusterfs/stack.h>
+#include <glusterfs/call-stub.h>
 #include "locks.h"
 
 typedef enum {
-        CLRLK_INODE,
-        CLRLK_ENTRY,
-        CLRLK_POSIX,
-        CLRLK_TYPE_MAX
+    CLRLK_INODE,
+    CLRLK_ENTRY,
+    CLRLK_POSIX,
+    CLRLK_TYPE_MAX
 } clrlk_type;
 
+extern const char *clrlk_type_names[];
+
 typedef enum {
-        CLRLK_BLOCKED = 1,
-        CLRLK_GRANTED,
-        CLRLK_ALL,
-        CLRLK_KIND_MAX
+    CLRLK_BLOCKED = 1,
+    CLRLK_GRANTED,
+    CLRLK_ALL,
+    CLRLK_KIND_MAX
 } clrlk_kind;
 
 typedef enum {
-        KW_TYPE,
-        KW_KIND,
-        /*add new keywords here*/
-        KW_MAX
+    KW_TYPE,
+    KW_KIND,
+    /*add new keywords here*/
+    KW_MAX
 } clrlk_opts;
 
 struct _clrlk_args;
 typedef struct _clrlk_args clrlk_args;
 
 struct _clrlk_args {
-        int type;
-        int kind;
-        char *opts;
+    int type;
+    int kind;
+    char *opts;
 };
 
 int
-clrlk_get__kind (char *kind);
+clrlk_get__kind(char *kind);
 int
-clrlk_get_type (char *type);
+clrlk_get_type(char *type);
 int
-clrlk_get_lock_range (char *range_str, struct gf_flock *ulock,
-                      gf_boolean_t *chk_range);
+clrlk_get_lock_range(char *range_str, struct gf_flock *ulock,
+                     gf_boolean_t *chk_range);
 int
-clrlk_parse_args (const char* cmd, clrlk_args *args);
+clrlk_parse_args(const char *cmd, clrlk_args *args);
 
 int
-clrlk_clear_posixlk (xlator_t *this, pl_inode_t *pl_inode, clrlk_args *args,
-                     int *blkd, int *granted, int *op_errno);
+clrlk_clear_posixlk(xlator_t *this, pl_inode_t *pl_inode, clrlk_args *args,
+                    int *blkd, int *granted, int *op_errno);
 int
-clrlk_clear_inodelk (xlator_t *this, pl_inode_t *pl_inode, pl_dom_list_t *dom,
-                     clrlk_args *args, int *blkd, int *granted, int *op_errno);
+clrlk_clear_inodelk(xlator_t *this, pl_inode_t *pl_inode, pl_dom_list_t *dom,
+                    clrlk_args *args, int *blkd, int *granted, int *op_errno);
 int
-clrlk_clear_entrylk (xlator_t *this, pl_inode_t *pl_inode, pl_dom_list_t *dom,
-                     clrlk_args *args, int *blkd, int *granted, int *op_errno);
+clrlk_clear_entrylk(xlator_t *this, pl_inode_t *pl_inode, pl_dom_list_t *dom,
+                    clrlk_args *args, int *blkd, int *granted, int *op_errno);
 int
-clrlk_clear_lks_in_all_domains (xlator_t *this, pl_inode_t *pl_inode,
-                                clrlk_args *args, int *blkd, int *granted,
-                                int *op_errno);
+clrlk_clear_lks_in_all_domains(xlator_t *this, pl_inode_t *pl_inode,
+                               clrlk_args *args, int *blkd, int *granted,
+                               int *op_errno);
 #endif /* __CLEAR_H__ */
diff --git a/xlators/features/locks/src/common.c b/xlators/features/locks/src/common.c
index 5b1b2f9f446..a2c6be93e03 100644
--- a/xlators/features/locks/src/common.c
+++ b/xlators/features/locks/src/common.c
@@ -1,5 +1,5 @@
 /*
-   Copyright (c) 2006-2012 Red Hat, Inc. <http://www.redhat.com>
+   Copyright (c) 2006-2012, 2015-2016 Red Hat, Inc. <http://www.redhat.com>
    This file is part of GlusterFS.
 
    This file is licensed to you under your choice of the GNU Lesser
@@ -12,724 +12,775 @@
 #include <limits.h>
 #include <pthread.h>
 
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
-
-#include "glusterfs.h"
-#include "compat.h"
-#include "xlator.h"
-#include "inode.h"
-#include "logging.h"
-#include "common-utils.h"
+#include <glusterfs/glusterfs.h>
+#include <glusterfs/compat.h>
+#include <glusterfs/logging.h>
+#include <glusterfs/syncop.h>
 
 #include "locks.h"
 #include "common.h"
 
-
 static int
-__is_lock_grantable (pl_inode_t *pl_inode, posix_lock_t *lock);
+__is_lock_grantable(pl_inode_t *pl_inode, posix_lock_t *lock);
 static void
-__insert_and_merge (pl_inode_t *pl_inode, posix_lock_t *lock);
+__insert_and_merge(pl_inode_t *pl_inode, posix_lock_t *lock);
 static int
-pl_send_prelock_unlock (xlator_t *this, pl_inode_t *pl_inode,
-                        posix_lock_t *old_lock);
+pl_send_prelock_unlock(xlator_t *this, pl_inode_t *pl_inode,
+                       posix_lock_t *old_lock);
 
 static pl_dom_list_t *
-__allocate_domain (const char *volume)
+__allocate_domain(const char *volume)
 {
-        pl_dom_list_t *dom = NULL;
+    pl_dom_list_t *dom = NULL;
 
-        dom = GF_CALLOC (1, sizeof (*dom),
-                         gf_locks_mt_pl_dom_list_t);
-        if (!dom)
-                goto out;
+    dom = GF_CALLOC(1, sizeof(*dom), gf_locks_mt_pl_dom_list_t);
+    if (!dom)
+        goto out;
 
-        dom->domain = gf_strdup(volume);
-        if (!dom->domain)
-                goto out;
+    dom->domain = gf_strdup(volume);
+    if (!dom->domain)
+        goto out;
 
-        gf_log ("posix-locks", GF_LOG_TRACE,
-                "New domain allocated: %s", dom->domain);
+    gf_log("posix-locks", GF_LOG_TRACE, "New domain allocated: %s",
+           dom->domain);
 
-        INIT_LIST_HEAD (&dom->inode_list);
-        INIT_LIST_HEAD (&dom->entrylk_list);
-        INIT_LIST_HEAD (&dom->blocked_entrylks);
-        INIT_LIST_HEAD (&dom->inodelk_list);
-        INIT_LIST_HEAD (&dom->blocked_inodelks);
+    INIT_LIST_HEAD(&dom->inode_list);
+    INIT_LIST_HEAD(&dom->entrylk_list);
+    INIT_LIST_HEAD(&dom->blocked_entrylks);
+    INIT_LIST_HEAD(&dom->inodelk_list);
+    INIT_LIST_HEAD(&dom->blocked_inodelks);
 
 out:
-        if (dom && (NULL == dom->domain)) {
-                GF_FREE (dom);
-                dom = NULL;
-        }
+    if (dom && (NULL == dom->domain)) {
+        GF_FREE(dom);
+        dom = NULL;
+    }
 
-        return dom;
+    return dom;
 }
 
 /* Returns domain for the lock. If domain is not present,
  * allocates a domain and returns it
  */
 pl_dom_list_t *
-get_domain (pl_inode_t *pl_inode, const char *volume)
+get_domain(pl_inode_t *pl_inode, const char *volume)
 {
-        pl_dom_list_t *dom = NULL;
+    pl_dom_list_t *dom = NULL;
 
-        GF_VALIDATE_OR_GOTO ("posix-locks", pl_inode, out);
-        GF_VALIDATE_OR_GOTO ("posix-locks", volume, out);
+    GF_VALIDATE_OR_GOTO("posix-locks", pl_inode, out);
+    GF_VALIDATE_OR_GOTO("posix-locks", volume, out);
 
-        pthread_mutex_lock (&pl_inode->mutex);
+    pthread_mutex_lock(&pl_inode->mutex);
+    {
+        list_for_each_entry(dom, &pl_inode->dom_list, inode_list)
         {
-                list_for_each_entry (dom, &pl_inode->dom_list, inode_list) {
-                        if (strcmp (dom->domain, volume) == 0)
-                                goto unlock;
-                }
-
-                dom = __allocate_domain (volume);
-                if (dom)
-                        list_add (&dom->inode_list, &pl_inode->dom_list);
+            if (strcmp(dom->domain, volume) == 0)
+                goto unlock;
         }
+
+        dom = __allocate_domain(volume);
+        if (dom)
+            list_add(&dom->inode_list, &pl_inode->dom_list);
+    }
 unlock:
-        pthread_mutex_unlock (&pl_inode->mutex);
-        if (dom) {
-                gf_log ("posix-locks", GF_LOG_TRACE, "Domain %s found", volume);
-        } else {
-                gf_log ("posix-locks", GF_LOG_TRACE, "Domain %s not found", volume);
-        }
+    pthread_mutex_unlock(&pl_inode->mutex);
+    if (dom) {
+        gf_log("posix-locks", GF_LOG_TRACE, "Domain %s found", volume);
+    } else {
+        gf_log("posix-locks", GF_LOG_TRACE, "Domain %s not found", volume);
+    }
 out:
-        return dom;
+    return dom;
 }
 
 unsigned long
-fd_to_fdnum (fd_t *fd)
+fd_to_fdnum(fd_t *fd)
 {
-        return ((unsigned long) fd);
+    return ((unsigned long)fd);
 }
 
 fd_t *
-fd_from_fdnum (posix_lock_t *lock)
+fd_from_fdnum(posix_lock_t *lock)
 {
-        return ((fd_t *) lock->fd_num);
+    return ((fd_t *)lock->fd_num);
 }
 
 int
-__pl_inode_is_empty (pl_inode_t *pl_inode)
+__pl_inode_is_empty(pl_inode_t *pl_inode)
 {
-        return (list_empty (&pl_inode->ext_list));
+    return (list_empty(&pl_inode->ext_list));
 }
 
 void
-pl_print_locker (char *str, int size, xlator_t *this, call_frame_t *frame)
+pl_print_locker(char *str, int size, xlator_t *this, call_frame_t *frame)
 {
-        snprintf (str, size, "Pid=%llu, lk-owner=%s, Client=%p, Frame=%llu",
-                  (unsigned long long) frame->root->pid,
-                  lkowner_utoa (&frame->root->lk_owner),
-                  frame->root->client,
-                  (unsigned long long) frame->root->unique);
+    snprintf(str, size, "Pid=%llu, lk-owner=%s, Client=%p, Frame=%llu",
+             (unsigned long long)frame->root->pid,
+             lkowner_utoa(&frame->root->lk_owner), frame->root->client,
+             (unsigned long long)frame->root->unique);
 }
 
-
 void
-pl_print_lockee (char *str, int size, fd_t *fd, loc_t *loc)
+pl_print_lockee(char *str, int size, fd_t *fd, loc_t *loc)
 {
-        inode_t  *inode = NULL;
-        char     *ipath = NULL;
-        int       ret = 0;
+    inode_t *inode = NULL;
+    char *ipath = NULL;
+    int ret = 0;
 
-        if (fd)
-                inode = fd->inode;
-        if (loc)
-                inode = loc->inode;
+    if (fd)
+        inode = fd->inode;
+    if (loc)
+        inode = loc->inode;
 
-        if (!inode) {
-                snprintf (str, size, "<nul>");
-                return;
-        }
+    if (!inode) {
+        snprintf(str, size, "<nul>");
+        return;
+    }
 
-        if (loc && loc->path) {
-                ipath = gf_strdup (loc->path);
-        } else {
-                ret = inode_path (inode, NULL, &ipath);
-                if (ret <= 0)
-                        ipath = NULL;
-        }
+    if (loc && loc->path) {
+        ipath = gf_strdup(loc->path);
+    } else {
+        ret = inode_path(inode, NULL, &ipath);
+        if (ret <= 0)
+            ipath = NULL;
+    }
 
-        snprintf (str, size, "gfid=%s, fd=%p, path=%s",
-                  uuid_utoa (inode->gfid), fd,
-                  ipath ? ipath : "<nul>");
+    snprintf(str, size, "gfid=%s, fd=%p, path=%s", uuid_utoa(inode->gfid), fd,
+             ipath ? ipath : "<nul>");
 
-        GF_FREE (ipath);
+    GF_FREE(ipath);
 }
 
-
 void
-pl_print_lock (char *str, int size, int cmd,
-               struct gf_flock *flock, gf_lkowner_t *owner)
+pl_print_lock(char *str, int size, int cmd, struct gf_flock *flock,
+              gf_lkowner_t *owner)
 {
-        char *cmd_str = NULL;
-        char *type_str = NULL;
+    char *cmd_str = NULL;
+    char *type_str = NULL;
 
-        switch (cmd) {
+    switch (cmd) {
 #if F_GETLK != F_GETLK64
         case F_GETLK64:
 #endif
         case F_GETLK:
-                cmd_str = "GETLK";
-                break;
+            cmd_str = "GETLK";
+            break;
 
 #if F_SETLK != F_SETLK64
         case F_SETLK64:
 #endif
         case F_SETLK:
-                cmd_str = "SETLK";
-                break;
+            cmd_str = "SETLK";
+            break;
 
 #if F_SETLKW != F_SETLKW64
         case F_SETLKW64:
 #endif
         case F_SETLKW:
-                cmd_str = "SETLKW";
-                break;
+            cmd_str = "SETLKW";
+            break;
 
         default:
-                cmd_str = "UNKNOWN";
-                break;
-        }
+            cmd_str = "UNKNOWN";
+            break;
+    }
 
-        switch (flock->l_type) {
+    switch (flock->l_type) {
         case F_RDLCK:
-                type_str = "READ";
-                break;
+            type_str = "READ";
+            break;
         case F_WRLCK:
-                type_str = "WRITE";
-                break;
+            type_str = "WRITE";
+            break;
         case F_UNLCK:
-                type_str = "UNLOCK";
-                break;
+            type_str = "UNLOCK";
+            break;
         default:
-                type_str = "UNKNOWN";
-                break;
-        }
-
-        snprintf (str, size, "lock=FCNTL, cmd=%s, type=%s, "
-                  "start=%llu, len=%llu, pid=%llu, lk-owner=%s",
-                  cmd_str, type_str, (unsigned long long) flock->l_start,
-                  (unsigned long long) flock->l_len,
-                  (unsigned long long) flock->l_pid,
-                  lkowner_utoa (owner));
+            type_str = "UNKNOWN";
+            break;
+    }
+
+    snprintf(str, size,
+             "lock=FCNTL, cmd=%s, type=%s, "
+             "start=%llu, len=%llu, pid=%llu, lk-owner=%s",
+             cmd_str, type_str, (unsigned long long)flock->l_start,
+             (unsigned long long)flock->l_len, (unsigned long long)flock->l_pid,
+             lkowner_utoa(owner));
 }
 
-
 void
-pl_trace_in (xlator_t *this, call_frame_t *frame, fd_t *fd, loc_t *loc,
-             int cmd, struct gf_flock *flock, const char *domain)
+pl_trace_in(xlator_t *this, call_frame_t *frame, fd_t *fd, loc_t *loc, int cmd,
+            struct gf_flock *flock, const char *domain)
 {
-        posix_locks_private_t  *priv = NULL;
-        char                    pl_locker[256];
-        char                    pl_lockee[256];
-        char                    pl_lock[256];
-
-        priv = this->private;
+    posix_locks_private_t *priv = this->private;
+    char pl_locker[256];
+    char pl_lockee[256];
+    char pl_lock[256];
 
-        if (!priv->trace)
-                return;
+    if (!priv->trace)
+        return;
 
-        pl_print_locker (pl_locker, 256, this, frame);
-        pl_print_lockee (pl_lockee, 256, fd, loc);
-        if (domain)
-                pl_print_inodelk (pl_lock, 256, cmd, flock, domain);
-        else
-                pl_print_lock (pl_lock, 256, cmd, flock, &frame->root->lk_owner);
+    pl_print_locker(pl_locker, 256, this, frame);
+    pl_print_lockee(pl_lockee, 256, fd, loc);
+    if (domain)
+        pl_print_inodelk(pl_lock, 256, cmd, flock, domain);
+    else
+        pl_print_lock(pl_lock, 256, cmd, flock, &frame->root->lk_owner);
 
-        gf_log (this->name, GF_LOG_INFO,
-                "[REQUEST] Locker = {%s} Lockee = {%s} Lock = {%s}",
-                pl_locker, pl_lockee, pl_lock);
+    gf_log(this->name, GF_LOG_INFO,
+           "[REQUEST] Locker = {%s} Lockee = {%s} Lock = {%s}", pl_locker,
+           pl_lockee, pl_lock);
 }
 
-
 void
-pl_print_verdict (char *str, int size, int op_ret, int op_errno)
+pl_print_verdict(char *str, int size, int op_ret, int op_errno)
 {
-        char *verdict = NULL;
-
-        if (op_ret == 0) {
-                verdict = "GRANTED";
-        } else {
-                switch (op_errno) {
-                case EAGAIN:
-                        verdict = "TRYAGAIN";
-                        break;
-                default:
-                        verdict = strerror (op_errno);
-                }
+    char *verdict = NULL;
+
+    if (op_ret == 0) {
+        verdict = "GRANTED";
+    } else {
+        switch (op_errno) {
+            case EAGAIN:
+                verdict = "TRYAGAIN";
+                break;
+            default:
+                verdict = strerror(op_errno);
         }
+    }
 
-        snprintf (str, size, "%s", verdict);
+    snprintf(str, size, "%s", verdict);
 }
 
-
 void
-pl_trace_out (xlator_t *this, call_frame_t *frame, fd_t *fd, loc_t *loc,
-              int cmd, struct gf_flock *flock, int op_ret, int op_errno, const char *domain)
+pl_trace_out(xlator_t *this, call_frame_t *frame, fd_t *fd, loc_t *loc, int cmd,
+             struct gf_flock *flock, int op_ret, int op_errno,
+             const char *domain)
 
 {
-        posix_locks_private_t  *priv = NULL;
-        char                    pl_locker[256];
-        char                    pl_lockee[256];
-        char                    pl_lock[256];
-        char                    verdict[32];
+    posix_locks_private_t *priv = NULL;
+    char pl_locker[256];
+    char pl_lockee[256];
+    char pl_lock[256];
+    char verdict[32];
 
-        priv = this->private;
+    priv = this->private;
 
-        if (!priv->trace)
-                return;
+    if (!priv->trace)
+        return;
 
-        pl_print_locker (pl_locker, 256, this, frame);
-        pl_print_lockee (pl_lockee, 256, fd, loc);
-        if (domain)
-                pl_print_inodelk (pl_lock, 256, cmd, flock, domain);
-        else
-                pl_print_lock (pl_lock, 256, cmd, flock, &frame->root->lk_owner);
+    pl_print_locker(pl_locker, 256, this, frame);
+    pl_print_lockee(pl_lockee, 256, fd, loc);
+    if (domain)
+        pl_print_inodelk(pl_lock, 256, cmd, flock, domain);
+    else
+        pl_print_lock(pl_lock, 256, cmd, flock, &frame->root->lk_owner);
 
-        pl_print_verdict (verdict, 32, op_ret, op_errno);
+    pl_print_verdict(verdict, 32, op_ret, op_errno);
 
-        gf_log (this->name, GF_LOG_INFO,
-                "[%s] Locker = {%s} Lockee = {%s} Lock = {%s}",
-                verdict, pl_locker, pl_lockee, pl_lock);
+    gf_log(this->name, GF_LOG_INFO,
+           "[%s] Locker = {%s} Lockee = {%s} Lock = {%s}", verdict, pl_locker,
+           pl_lockee, pl_lock);
 }
 
-
 void
-pl_trace_block (xlator_t *this, call_frame_t *frame, fd_t *fd, loc_t *loc,
-                int cmd, struct gf_flock *flock, const char *domain)
+pl_trace_block(xlator_t *this, call_frame_t *frame, fd_t *fd, loc_t *loc,
+               int cmd, struct gf_flock *flock, const char *domain)
 
 {
-        posix_locks_private_t  *priv = NULL;
-        char                    pl_locker[256];
-        char                    pl_lockee[256];
-        char                    pl_lock[256];
-
-        priv = this->private;
+    posix_locks_private_t *priv = this->private;
+    char pl_locker[256];
+    char pl_lockee[256];
+    char pl_lock[256];
 
-        if (!priv->trace)
-                return;
+    if (!priv->trace)
+        return;
 
-        pl_print_locker (pl_locker, 256, this, frame);
-        pl_print_lockee (pl_lockee, 256, fd, loc);
-        if (domain)
-                pl_print_inodelk (pl_lock, 256, cmd, flock, domain);
-        else
-                pl_print_lock (pl_lock, 256, cmd, flock, &frame->root->lk_owner);
+    pl_print_locker(pl_locker, 256, this, frame);
+    pl_print_lockee(pl_lockee, 256, fd, loc);
+    if (domain)
+        pl_print_inodelk(pl_lock, 256, cmd, flock, domain);
+    else
+        pl_print_lock(pl_lock, 256, cmd, flock, &frame->root->lk_owner);
 
-        gf_log (this->name, GF_LOG_INFO,
-                "[BLOCKED] Locker = {%s} Lockee = {%s} Lock = {%s}",
-                pl_locker, pl_lockee, pl_lock);
+    gf_log(this->name, GF_LOG_INFO,
+           "[BLOCKED] Locker = {%s} Lockee = {%s} Lock = {%s}", pl_locker,
+           pl_lockee, pl_lock);
 }
 
-
 void
-pl_trace_flush (xlator_t *this, call_frame_t *frame, fd_t *fd)
+pl_trace_flush(xlator_t *this, call_frame_t *frame, fd_t *fd)
 {
-        posix_locks_private_t  *priv = NULL;
-        char                    pl_locker[256];
-        char                    pl_lockee[256];
-        pl_inode_t             *pl_inode = NULL;
+    posix_locks_private_t *priv = NULL;
+    char pl_locker[256];
+    char pl_lockee[256];
+    pl_inode_t *pl_inode = NULL;
 
-        priv = this->private;
+    priv = this->private;
 
-        if (!priv->trace)
-                return;
+    if (!priv->trace)
+        return;
 
-        pl_inode = pl_inode_get (this, fd->inode);
+    pl_inode = pl_inode_get(this, fd->inode, NULL);
 
-        if (pl_inode && __pl_inode_is_empty (pl_inode))
-                return;
+    if (pl_inode && __pl_inode_is_empty(pl_inode))
+        return;
 
-        pl_print_locker (pl_locker, 256, this, frame);
-        pl_print_lockee (pl_lockee, 256, fd, NULL);
+    pl_print_locker(pl_locker, 256, this, frame);
+    pl_print_lockee(pl_lockee, 256, fd, NULL);
 
-        gf_log (this->name, GF_LOG_INFO,
-                "[FLUSH] Locker = {%s} Lockee = {%s}",
-                pl_locker, pl_lockee);
+    gf_log(this->name, GF_LOG_INFO, "[FLUSH] Locker = {%s} Lockee = {%s}",
+           pl_locker, pl_lockee);
 }
 
 void
-pl_trace_release (xlator_t *this, fd_t *fd)
+pl_trace_release(xlator_t *this, fd_t *fd)
 {
-        posix_locks_private_t  *priv = NULL;
-        char                    pl_lockee[256];
+    posix_locks_private_t *priv = NULL;
+    char pl_lockee[256];
 
-        priv = this->private;
+    priv = this->private;
 
-        if (!priv->trace)
-                return;
+    if (!priv->trace)
+        return;
 
-        pl_print_lockee (pl_lockee, 256, fd, NULL);
+    pl_print_lockee(pl_lockee, 256, fd, NULL);
 
-        gf_log (this->name, GF_LOG_INFO,
-                "[RELEASE] Lockee = {%s}", pl_lockee);
+    gf_log(this->name, GF_LOG_INFO, "[RELEASE] Lockee = {%s}", pl_lockee);
 }
 
-
 void
-pl_update_refkeeper (xlator_t *this, inode_t *inode)
+pl_update_refkeeper(xlator_t *this, inode_t *inode)
 {
-        pl_inode_t *pl_inode  = NULL;
-        int         is_empty  = 0;
-        int         need_unref = 0;
-        int         need_ref = 0;
+    pl_inode_t *pl_inode = NULL;
+    int is_empty = 0;
+    int need_unref = 0;
+    int need_ref = 0;
 
-        pl_inode = pl_inode_get (this, inode);
+    pl_inode = pl_inode_get(this, inode, NULL);
+    if (!pl_inode)
+        return;
 
-        pthread_mutex_lock (&pl_inode->mutex);
-        {
-                is_empty = __pl_inode_is_empty (pl_inode);
+    pthread_mutex_lock(&pl_inode->mutex);
+    {
+        is_empty = __pl_inode_is_empty(pl_inode);
 
-                if (is_empty && pl_inode->refkeeper) {
-                        need_unref = 1;
-                        pl_inode->refkeeper = NULL;
-                }
+        if (is_empty && pl_inode->refkeeper) {
+            need_unref = 1;
+            pl_inode->refkeeper = NULL;
+        }
 
-                if (!is_empty && !pl_inode->refkeeper) {
-                        need_ref = 1;
-                        pl_inode->refkeeper = inode;
-                }
+        if (!is_empty && !pl_inode->refkeeper) {
+            need_ref = 1;
+            pl_inode->refkeeper = inode;
         }
-        pthread_mutex_unlock (&pl_inode->mutex);
+    }
+    pthread_mutex_unlock(&pl_inode->mutex);
 
-        if (need_unref)
-                inode_unref (inode);
+    if (need_unref)
+        inode_unref(inode);
 
-        if (need_ref)
-                inode_ref (inode);
+    if (need_ref)
+        inode_ref(inode);
 }
 
-
-pl_inode_t *
-pl_inode_get (xlator_t *this, inode_t *inode)
+/* Get lock enforcement info from disk */
+int
+pl_fetch_mlock_info_from_disk(xlator_t *this, pl_inode_t *pl_inode,
+                              pl_local_t *local)
 {
-        uint64_t    tmp_pl_inode = 0;
-        pl_inode_t *pl_inode = NULL;
-        int         ret = 0;
-
-        LOCK (&inode->lock);
-        {
-                ret = __inode_ctx_get (inode, this, &tmp_pl_inode);
-                if (ret == 0) {
-                        pl_inode = (pl_inode_t *)(long)tmp_pl_inode;
-                        goto unlock;
-                }
-                pl_inode = GF_CALLOC (1, sizeof (*pl_inode),
-                                      gf_locks_mt_pl_inode_t);
-                if (!pl_inode) {
-                        goto unlock;
-                }
+    dict_t *xdata_rsp = NULL;
+    int ret = 0;
+    int op_ret = 0;
+
+    if (!local) {
+        return -1;
+    }
+
+    if (local->fd) {
+        op_ret = syncop_fgetxattr(this, local->fd, &xdata_rsp,
+                                  GF_ENFORCE_MANDATORY_LOCK, NULL, NULL);
+    } else {
+        op_ret = syncop_getxattr(this, &local->loc[0], &xdata_rsp,
+                                 GF_ENFORCE_MANDATORY_LOCK, NULL, NULL);
+    }
+
+    pthread_mutex_lock(&pl_inode->mutex);
+    {
+        if (op_ret >= 0) {
+            pl_inode->mlock_enforced = _gf_true;
+            pl_inode->check_mlock_info = _gf_false;
+        } else {
+            gf_msg(this->name, GF_LOG_WARNING, -op_ret, 0,
+                   "getxattr failed with %d", op_ret);
+            pl_inode->mlock_enforced = _gf_false;
+
+            if (-op_ret == ENODATA) {
+                pl_inode->check_mlock_info = _gf_false;
+            } else {
+                pl_inode->check_mlock_info = _gf_true;
+            }
+        }
+    }
+    pthread_mutex_unlock(&pl_inode->mutex);
 
-                gf_log (this->name, GF_LOG_TRACE,
-                        "Allocating new pl inode");
+    return ret;
+}
 
-                pthread_mutex_init (&pl_inode->mutex, NULL);
+pl_inode_t *
+pl_inode_get(xlator_t *this, inode_t *inode, pl_local_t *local)
+{
+    uint64_t tmp_pl_inode = 0;
+    pl_inode_t *pl_inode = NULL;
+    int ret = 0;
+
+    LOCK(&inode->lock);
+    {
+        ret = __inode_ctx_get(inode, this, &tmp_pl_inode);
+        if (ret == 0) {
+            pl_inode = (pl_inode_t *)(long)tmp_pl_inode;
+            goto unlock;
+        }
 
-                INIT_LIST_HEAD (&pl_inode->dom_list);
-                INIT_LIST_HEAD (&pl_inode->ext_list);
-                INIT_LIST_HEAD (&pl_inode->rw_list);
-                INIT_LIST_HEAD (&pl_inode->reservelk_list);
-                INIT_LIST_HEAD (&pl_inode->blocked_reservelks);
-                INIT_LIST_HEAD (&pl_inode->blocked_calls);
-                gf_uuid_copy (pl_inode->gfid, inode->gfid);
+        pl_inode = GF_CALLOC(1, sizeof(*pl_inode), gf_locks_mt_pl_inode_t);
+        if (!pl_inode) {
+            goto unlock;
+        }
 
-                __inode_ctx_put (inode, this, (uint64_t)(long)(pl_inode));
+        gf_log(this->name, GF_LOG_TRACE, "Allocating new pl inode");
+
+        pthread_mutex_init(&pl_inode->mutex, NULL);
+        pthread_cond_init(&pl_inode->check_fop_wind_count, 0);
+
+        INIT_LIST_HEAD(&pl_inode->dom_list);
+        INIT_LIST_HEAD(&pl_inode->ext_list);
+        INIT_LIST_HEAD(&pl_inode->rw_list);
+        INIT_LIST_HEAD(&pl_inode->reservelk_list);
+        INIT_LIST_HEAD(&pl_inode->blocked_reservelks);
+        INIT_LIST_HEAD(&pl_inode->blocked_calls);
+        INIT_LIST_HEAD(&pl_inode->metalk_list);
+        INIT_LIST_HEAD(&pl_inode->queued_locks);
+        INIT_LIST_HEAD(&pl_inode->waiting);
+        gf_uuid_copy(pl_inode->gfid, inode->gfid);
+
+        pl_inode->check_mlock_info = _gf_true;
+        pl_inode->mlock_enforced = _gf_false;
+
+        /* -2 means never looked up. -1 means something went wrong and link
+         * tracking is disabled. */
+        pl_inode->links = -2;
+
+        ret = __inode_ctx_put(inode, this, (uint64_t)(long)(pl_inode));
+        if (ret) {
+            pthread_mutex_destroy(&pl_inode->mutex);
+            GF_FREE(pl_inode);
+            pl_inode = NULL;
+            goto unlock;
         }
+    }
 unlock:
-        UNLOCK (&inode->lock);
+    UNLOCK(&inode->lock);
 
-        return pl_inode;
-}
+    if ((pl_inode != NULL) && pl_is_mandatory_locking_enabled(pl_inode) &&
+        pl_inode->check_mlock_info && local) {
+        /* Note: The lock enforcement information per file can be stored in the
+           attribute flag of stat(x) in posix. With that there won't be a need
+           for doing getxattr post a reboot
+        */
+        pl_fetch_mlock_info_from_disk(this, pl_inode, local);
+    }
 
+    return pl_inode;
+}
 
 /* Create a new posix_lock_t */
 posix_lock_t *
-new_posix_lock (struct gf_flock *flock, client_t *client, pid_t client_pid,
-                gf_lkowner_t *owner, fd_t *fd)
+new_posix_lock(struct gf_flock *flock, client_t *client, pid_t client_pid,
+               gf_lkowner_t *owner, fd_t *fd, uint32_t lk_flags, int blocking,
+               int32_t *op_errno)
 {
-        posix_lock_t *lock = NULL;
+    posix_lock_t *lock = NULL;
 
-        GF_VALIDATE_OR_GOTO ("posix-locks", flock, out);
-        GF_VALIDATE_OR_GOTO ("posix-locks", client, out);
-        GF_VALIDATE_OR_GOTO ("posix-locks", fd, out);
+    GF_VALIDATE_OR_GOTO("posix-locks", flock, out);
+    GF_VALIDATE_OR_GOTO("posix-locks", client, out);
+    GF_VALIDATE_OR_GOTO("posix-locks", fd, out);
 
-        lock = GF_CALLOC (1, sizeof (posix_lock_t),
-                          gf_locks_mt_posix_lock_t);
-        if (!lock) {
-                goto out;
-        }
+    if (!pl_is_lk_owner_valid(owner, client)) {
+        *op_errno = EINVAL;
+        goto out;
+    }
+
+    lock = GF_CALLOC(1, sizeof(posix_lock_t), gf_locks_mt_posix_lock_t);
+    if (!lock) {
+        *op_errno = ENOMEM;
+        goto out;
+    }
+
+    lock->fl_start = flock->l_start;
+    lock->fl_type = flock->l_type;
+
+    if (flock->l_len == 0)
+        lock->fl_end = LLONG_MAX;
+    else
+        lock->fl_end = flock->l_start + flock->l_len - 1;
+
+    lock->client = client;
 
-        lock->fl_start = flock->l_start;
-        lock->fl_type  = flock->l_type;
+    lock->client_uid = gf_strdup(client->client_uid);
+    if (lock->client_uid == NULL) {
+        GF_FREE(lock);
+        lock = NULL;
+        *op_errno = ENOMEM;
+        goto out;
+    }
 
-        if (flock->l_len == 0)
-                lock->fl_end = LLONG_MAX;
-        else
-                lock->fl_end = flock->l_start + flock->l_len - 1;
+    lock->fd_num = fd_to_fdnum(fd);
+    lock->fd = fd;
+    lock->client_pid = client_pid;
+    lock->owner = *owner;
+    lock->lk_flags = lk_flags;
 
-        lock->client     = client;
-        lock->fd_num     = fd_to_fdnum (fd);
-        lock->fd         = fd;
-        lock->client_pid = client_pid;
-        lock->owner      = *owner;
+    lock->blocking = blocking;
+    memcpy(&lock->user_flock, flock, sizeof(lock->user_flock));
 
-        INIT_LIST_HEAD (&lock->list);
+    INIT_LIST_HEAD(&lock->list);
 
 out:
-        return lock;
+    return lock;
 }
 
-
 /* Delete a lock from the inode's lock list */
 void
-__delete_lock (pl_inode_t *pl_inode, posix_lock_t *lock)
+__delete_lock(posix_lock_t *lock)
 {
-        list_del_init (&lock->list);
+    list_del_init(&lock->list);
 }
 
-
 /* Destroy a posix_lock */
 void
-__destroy_lock (posix_lock_t *lock)
+__destroy_lock(posix_lock_t *lock)
 {
-        GF_FREE (lock);
+    GF_FREE(lock->client_uid);
+    GF_FREE(lock);
 }
 
+static posix_lock_t *
+__copy_lock(posix_lock_t *src)
+{
+    posix_lock_t *dst;
+
+    dst = GF_MALLOC(sizeof(posix_lock_t), gf_locks_mt_posix_lock_t);
+    if (dst != NULL) {
+        memcpy(dst, src, sizeof(posix_lock_t));
+        dst->client_uid = gf_strdup(src->client_uid);
+        if (dst->client_uid == NULL) {
+            GF_FREE(dst);
+            dst = NULL;
+        }
+
+        if (dst != NULL)
+            INIT_LIST_HEAD(&dst->list);
+    }
+
+    return dst;
+}
 
 /* Convert a posix_lock to a struct gf_flock */
 void
-posix_lock_to_flock (posix_lock_t *lock, struct gf_flock *flock)
+posix_lock_to_flock(posix_lock_t *lock, struct gf_flock *flock)
 {
-        flock->l_pid   = lock->client_pid;
-        flock->l_type  = lock->fl_type;
-        flock->l_start = lock->fl_start;
-        flock->l_owner = lock->owner;
-
-        if (lock->fl_end == LLONG_MAX)
-                flock->l_len = 0;
-        else
-                flock->l_len = lock->fl_end - lock->fl_start + 1;
+    flock->l_pid = lock->user_flock.l_pid;
+    flock->l_type = lock->fl_type;
+    flock->l_start = lock->fl_start;
+    flock->l_owner = lock->owner;
+
+    if (lock->fl_end == LLONG_MAX)
+        flock->l_len = 0;
+    else
+        flock->l_len = lock->fl_end - lock->fl_start + 1;
 }
 
 /* Insert the lock into the inode's lock list */
 static void
-__insert_lock (pl_inode_t *pl_inode, posix_lock_t *lock)
+__insert_lock(pl_inode_t *pl_inode, posix_lock_t *lock)
 {
-        if (lock->blocked)
-                gettimeofday (&lock->blkd_time, NULL);
-        else
-                gettimeofday (&lock->granted_time, NULL);
-
-        list_add_tail (&lock->list, &pl_inode->ext_list);
+    if (lock->blocked)
+        lock->blkd_time = gf_time();
+    else
+        lock->granted_time = gf_time();
 
-        return;
+    list_add_tail(&lock->list, &pl_inode->ext_list);
 }
 
-
 /* Return true if the locks overlap, false otherwise */
 int
-locks_overlap (posix_lock_t *l1, posix_lock_t *l2)
+locks_overlap(posix_lock_t *l1, posix_lock_t *l2)
 {
-        /*
-           Note:
-           FUSE always gives us absolute offsets, so no need to worry
-           about SEEK_CUR or SEEK_END
-        */
+    /*
+       Note:
+       FUSE always gives us absolute offsets, so no need to worry
+       about SEEK_CUR or SEEK_END
+    */
 
-        return ((l1->fl_end >= l2->fl_start) &&
-                (l2->fl_end >= l1->fl_start));
+    return ((l1->fl_end >= l2->fl_start) && (l2->fl_end >= l1->fl_start));
 }
 
-
 /* Return true if the locks have the same owner */
 int
-same_owner (posix_lock_t *l1, posix_lock_t *l2)
+same_owner(posix_lock_t *l1, posix_lock_t *l2)
 {
-
-        return (is_same_lkowner (&l1->owner, &l2->owner) &&
-                (l1->client == l2->client));
-
+    return (is_same_lkowner(&l1->owner, &l2->owner) &&
+            (l1->client == l2->client));
 }
 
-
 /* Delete all F_UNLCK locks */
 void
-__delete_unlck_locks (pl_inode_t *pl_inode)
+__delete_unlck_locks(pl_inode_t *pl_inode)
 {
-        posix_lock_t *l = NULL;
-        posix_lock_t *tmp = NULL;
-
-        list_for_each_entry_safe (l, tmp, &pl_inode->ext_list, list) {
-                if (l->fl_type == F_UNLCK) {
-                        __delete_lock (pl_inode, l);
-                        __destroy_lock (l);
-                }
+    posix_lock_t *l = NULL;
+    posix_lock_t *tmp = NULL;
+
+    list_for_each_entry_safe(l, tmp, &pl_inode->ext_list, list)
+    {
+        if (l->fl_type == F_UNLCK) {
+            __delete_lock(l);
+            __destroy_lock(l);
         }
+    }
 }
 
-
 /* Add two locks */
 static posix_lock_t *
-add_locks (posix_lock_t *l1, posix_lock_t *l2)
+add_locks(posix_lock_t *l1, posix_lock_t *l2, posix_lock_t *dst)
 {
-        posix_lock_t *sum = NULL;
+    posix_lock_t *sum = NULL;
+
+    sum = __copy_lock(dst);
+    if (!sum)
+        return NULL;
 
-        sum = GF_CALLOC (1, sizeof (posix_lock_t),
-                         gf_locks_mt_posix_lock_t);
-        if (!sum)
-                return NULL;
+    sum->fl_start = min(l1->fl_start, l2->fl_start);
+    sum->fl_end = max(l1->fl_end, l2->fl_end);
 
-        sum->fl_start = min (l1->fl_start, l2->fl_start);
-        sum->fl_end   = max (l1->fl_end, l2->fl_end);
+    posix_lock_to_flock(sum, &sum->user_flock);
 
-        return sum;
+    return sum;
 }
 
 /* Subtract two locks */
 struct _values {
-        posix_lock_t *locks[3];
+    posix_lock_t *locks[3];
 };
 
 /* {big} must always be contained inside {small} */
 static struct _values
-subtract_locks (posix_lock_t *big, posix_lock_t *small)
+subtract_locks(posix_lock_t *big, posix_lock_t *small)
 {
+    struct _values v = {.locks = {0, 0, 0}};
 
-        struct _values v = { .locks = {0, 0, 0} };
-
-        if ((big->fl_start == small->fl_start) &&
-            (big->fl_end   == small->fl_end)) {
-                /* both edges coincide with big */
-                v.locks[0] = GF_CALLOC (1, sizeof (posix_lock_t),
-                                        gf_locks_mt_posix_lock_t);
-                if (!v.locks[0])
-                        goto out;
-                memcpy (v.locks[0], big, sizeof (posix_lock_t));
-                v.locks[0]->fl_type = small->fl_type;
-                goto done;
+    if ((big->fl_start == small->fl_start) && (big->fl_end == small->fl_end)) {
+        /* both edges coincide with big */
+        v.locks[0] = __copy_lock(big);
+        if (!v.locks[0]) {
+            goto out;
         }
 
-        if ((small->fl_start > big->fl_start) &&
-            (small->fl_end   < big->fl_end)) {
-                /* both edges lie inside big */
-                v.locks[0] = GF_CALLOC (1, sizeof (posix_lock_t),
-                                        gf_locks_mt_posix_lock_t);
-                if (!v.locks[0])
-                        goto out;
-
-                v.locks[1] = GF_CALLOC (1, sizeof (posix_lock_t),
-                                        gf_locks_mt_posix_lock_t);
-                if (!v.locks[1])
-                        goto out;
-
-                v.locks[2] = GF_CALLOC (1, sizeof (posix_lock_t),
-                                        gf_locks_mt_posix_lock_t);
-                if (!v.locks[1])
-                        goto out;
-
-                memcpy (v.locks[0], big, sizeof (posix_lock_t));
-                v.locks[0]->fl_end = small->fl_start - 1;
-
-                memcpy (v.locks[1], small, sizeof (posix_lock_t));
-
-                memcpy (v.locks[2], big, sizeof (posix_lock_t));
-                v.locks[2]->fl_start = small->fl_end + 1;
-                goto done;
-
+        v.locks[0]->fl_type = small->fl_type;
+        v.locks[0]->user_flock.l_type = small->fl_type;
+        goto done;
+    }
+
+    if ((small->fl_start > big->fl_start) && (small->fl_end < big->fl_end)) {
+        /* both edges lie inside big */
+        v.locks[0] = __copy_lock(big);
+        v.locks[1] = __copy_lock(small);
+        v.locks[2] = __copy_lock(big);
+        if ((v.locks[0] == NULL) || (v.locks[1] == NULL) ||
+            (v.locks[2] == NULL)) {
+            goto out;
         }
 
-        /* one edge coincides with big */
-        if (small->fl_start == big->fl_start) {
-                v.locks[0] = GF_CALLOC (1, sizeof (posix_lock_t),
-                                        gf_locks_mt_posix_lock_t);
-                if (!v.locks[0])
-                        goto out;
-
-                v.locks[1] = GF_CALLOC (1, sizeof (posix_lock_t),
-                                        gf_locks_mt_posix_lock_t);
-                if (!v.locks[1])
-                        goto out;
-
-                memcpy (v.locks[0], big, sizeof (posix_lock_t));
-                v.locks[0]->fl_start = small->fl_end + 1;
-
-                memcpy (v.locks[1], small, sizeof (posix_lock_t));
-                goto done;
+        v.locks[0]->fl_end = small->fl_start - 1;
+        v.locks[2]->fl_start = small->fl_end + 1;
+        posix_lock_to_flock(v.locks[0], &v.locks[0]->user_flock);
+        posix_lock_to_flock(v.locks[2], &v.locks[2]->user_flock);
+        goto done;
+    }
+
+    /* one edge coincides with big */
+    if (small->fl_start == big->fl_start) {
+        v.locks[0] = __copy_lock(big);
+        v.locks[1] = __copy_lock(small);
+        if ((v.locks[0] == NULL) || (v.locks[1] == NULL)) {
+            goto out;
         }
 
-        if (small->fl_end  == big->fl_end) {
-                v.locks[0] = GF_CALLOC (1, sizeof (posix_lock_t),
-                                        gf_locks_mt_posix_lock_t);
-                if (!v.locks[0])
-                        goto out;
+        v.locks[0]->fl_start = small->fl_end + 1;
+        posix_lock_to_flock(v.locks[0], &v.locks[0]->user_flock);
+        goto done;
+    }
 
-                v.locks[1] = GF_CALLOC (1, sizeof (posix_lock_t),
-                                        gf_locks_mt_posix_lock_t);
-                if (!v.locks[1])
-                        goto out;
-
-                memcpy (v.locks[0], big, sizeof (posix_lock_t));
-                v.locks[0]->fl_end = small->fl_start - 1;
-
-                memcpy (v.locks[1], small, sizeof (posix_lock_t));
-                goto done;
+    if (small->fl_end == big->fl_end) {
+        v.locks[0] = __copy_lock(big);
+        v.locks[1] = __copy_lock(small);
+        if ((v.locks[0] == NULL) || (v.locks[1] == NULL)) {
+            goto out;
         }
 
-        GF_ASSERT (0);
-        gf_log ("posix-locks", GF_LOG_ERROR, "Unexpected case in subtract_locks");
+        v.locks[0]->fl_end = small->fl_start - 1;
+        posix_lock_to_flock(v.locks[0], &v.locks[0]->user_flock);
+        goto done;
+    }
+
+    GF_ASSERT(0);
+    gf_log("posix-locks", GF_LOG_ERROR, "Unexpected case in subtract_locks");
 
 out:
-        if (v.locks[0]) {
-                GF_FREE (v.locks[0]);
-                v.locks[0] = NULL;
-        }
-        if (v.locks[1]) {
-                GF_FREE (v.locks[1]);
-                v.locks[1] = NULL;
-        }
-        if (v.locks[2]) {
-                GF_FREE (v.locks[2]);
-                v.locks[2] = NULL;
-        }
+    if (v.locks[0]) {
+        __destroy_lock(v.locks[0]);
+        v.locks[0] = NULL;
+    }
+    if (v.locks[1]) {
+        __destroy_lock(v.locks[1]);
+        v.locks[1] = NULL;
+    }
+    if (v.locks[2]) {
+        __destroy_lock(v.locks[2]);
+        v.locks[2] = NULL;
+    }
 
 done:
-        return v;
+    return v;
 }
 
 static posix_lock_t *
-first_conflicting_overlap (pl_inode_t *pl_inode, posix_lock_t *lock)
+first_conflicting_overlap(pl_inode_t *pl_inode, posix_lock_t *lock)
 {
-        posix_lock_t *l = NULL;
-        posix_lock_t *conf = NULL;
+    posix_lock_t *l = NULL;
+    posix_lock_t *conf = NULL;
 
-        pthread_mutex_lock (&pl_inode->mutex);
+    pthread_mutex_lock(&pl_inode->mutex);
+    {
+        list_for_each_entry(l, &pl_inode->ext_list, list)
         {
-                list_for_each_entry (l, &pl_inode->ext_list, list) {
-                        if (l->blocked)
-                                continue;
-
-                        if (locks_overlap (l, lock)) {
-                                if (same_owner (l, lock))
-                                        continue;
-
-                                if ((l->fl_type == F_WRLCK) ||
-                                    (lock->fl_type == F_WRLCK)) {
-                                        conf = l;
-                                        goto unlock;
-                                }
-                        }
+            if (l->blocked)
+                continue;
+
+            if (locks_overlap(l, lock)) {
+                if (same_owner(l, lock))
+                    continue;
+
+                if ((l->fl_type == F_WRLCK) || (lock->fl_type == F_WRLCK)) {
+                    conf = l;
+                    goto unlock;
                 }
+            }
         }
+    }
 unlock:
-        pthread_mutex_unlock (&pl_inode->mutex);
+    pthread_mutex_unlock(&pl_inode->mutex);
 
-        return conf;
+    return conf;
 }
 
 /*
@@ -738,351 +789,803 @@ unlock:
   If {begin} is NULL, then start from the beginning of the list
 */
 static posix_lock_t *
-first_overlap (pl_inode_t *pl_inode, posix_lock_t *lock)
+first_overlap(pl_inode_t *pl_inode, posix_lock_t *lock)
 {
-        posix_lock_t *l = NULL;
+    posix_lock_t *l = NULL;
 
-        list_for_each_entry (l, &pl_inode->ext_list, list) {
-                if (l->blocked)
-                        continue;
+    list_for_each_entry(l, &pl_inode->ext_list, list)
+    {
+        if (l->blocked)
+            continue;
 
-                if (locks_overlap (l, lock))
-                        return l;
-        }
+        if (locks_overlap(l, lock))
+            return l;
+    }
 
-        return NULL;
+    return NULL;
 }
 
-
-
 /* Return true if lock is grantable */
 static int
-__is_lock_grantable (pl_inode_t *pl_inode, posix_lock_t *lock)
+__is_lock_grantable(pl_inode_t *pl_inode, posix_lock_t *lock)
 {
-        posix_lock_t *l = NULL;
-        int           ret = 1;
-
-        list_for_each_entry (l, &pl_inode->ext_list, list) {
-                if (!l->blocked && locks_overlap (lock, l)) {
-                        if (((l->fl_type == F_WRLCK)
-                             || (lock->fl_type == F_WRLCK))
-                            && (lock->fl_type != F_UNLCK)
-                            && !same_owner (l, lock)) {
-                                ret = 0;
-                                break;
-                        }
-                }
+    posix_lock_t *l = NULL;
+    int ret = 1;
+
+    list_for_each_entry(l, &pl_inode->ext_list, list)
+    {
+        if (!l->blocked && locks_overlap(lock, l)) {
+            if (((l->fl_type == F_WRLCK) || (lock->fl_type == F_WRLCK)) &&
+                (lock->fl_type != F_UNLCK) && !same_owner(l, lock)) {
+                ret = 0;
+                break;
+            }
         }
-        return ret;
+    }
+    return ret;
 }
 
-
-extern void do_blocked_rw (pl_inode_t *);
-
+extern void
+do_blocked_rw(pl_inode_t *);
 
 static void
-__insert_and_merge (pl_inode_t *pl_inode, posix_lock_t *lock)
+__insert_and_merge(pl_inode_t *pl_inode, posix_lock_t *lock)
 {
-        posix_lock_t  *conf = NULL;
-        posix_lock_t  *t = NULL;
-        posix_lock_t  *sum = NULL;
-        int            i = 0;
-        struct _values v = { .locks = {0, 0, 0} };
-
-        list_for_each_entry_safe (conf, t, &pl_inode->ext_list, list) {
-                if (conf->blocked)
-                        continue;
-                if (!locks_overlap (conf, lock))
-                        continue;
+    posix_lock_t *conf = NULL;
+    posix_lock_t *t = NULL;
+    posix_lock_t *sum = NULL;
+    int i = 0;
+    struct _values v = {.locks = {0, 0, 0}};
+
+    list_for_each_entry_safe(conf, t, &pl_inode->ext_list, list)
+    {
+        if (conf->blocked)
+            continue;
+        if (!locks_overlap(conf, lock))
+            continue;
+
+        if (same_owner(conf, lock)) {
+            if (conf->fl_type == lock->fl_type &&
+                conf->lk_flags == lock->lk_flags) {
+                sum = add_locks(lock, conf, lock);
+
+                __delete_lock(conf);
+                __destroy_lock(conf);
+
+                __destroy_lock(lock);
+                INIT_LIST_HEAD(&sum->list);
+                posix_lock_to_flock(sum, &sum->user_flock);
+                __insert_and_merge(pl_inode, sum);
 
-                if (same_owner (conf, lock)) {
-                        if (conf->fl_type == lock->fl_type) {
-                                sum = add_locks (lock, conf);
+                return;
+            } else {
+                sum = add_locks(lock, conf, conf);
 
-                                sum->fl_type    = lock->fl_type;
-                                sum->client     = lock->client;
-                                sum->fd_num     = lock->fd_num;
-                                sum->client_pid = lock->client_pid;
-                                sum->owner      = lock->owner;
+                v = subtract_locks(sum, lock);
 
-                                __delete_lock (pl_inode, conf);
-                                __destroy_lock (conf);
+                __delete_lock(conf);
+                __destroy_lock(conf);
 
-                                __destroy_lock (lock);
-                                INIT_LIST_HEAD (&sum->list);
-                                posix_lock_to_flock (sum, &sum->user_flock);
-                                __insert_and_merge (pl_inode, sum);
+                __delete_lock(lock);
+                __destroy_lock(lock);
 
-                                return;
-                        } else {
-                                sum = add_locks (lock, conf);
+                __destroy_lock(sum);
 
-                                sum->fl_type    = conf->fl_type;
-                                sum->client     = conf->client;
-                                sum->fd_num     = conf->fd_num;
-                                sum->client_pid = conf->client_pid;
-                                sum->owner      = conf->owner;
+                for (i = 0; i < 3; i++) {
+                    if (!v.locks[i])
+                        continue;
 
-                                v = subtract_locks (sum, lock);
+                    __insert_and_merge(pl_inode, v.locks[i]);
+                }
 
-                                __delete_lock (pl_inode, conf);
-                                __destroy_lock (conf);
+                __delete_unlck_locks(pl_inode);
+                return;
+            }
+        }
 
-                                __delete_lock (pl_inode, lock);
-                                __destroy_lock (lock);
+        if (lock->fl_type == F_UNLCK) {
+            continue;
+        }
 
-                                __destroy_lock (sum);
+        if ((conf->fl_type == F_RDLCK) && (lock->fl_type == F_RDLCK)) {
+            __insert_lock(pl_inode, lock);
+            return;
+        }
+    }
+
+    /* no conflicts, so just insert */
+    if (lock->fl_type != F_UNLCK) {
+        __insert_lock(pl_inode, lock);
+    } else {
+        __destroy_lock(lock);
+    }
+}
 
-                                for (i = 0; i < 3; i++) {
-                                        if (!v.locks[i])
-                                                continue;
+void
+__grant_blocked_locks(xlator_t *this, pl_inode_t *pl_inode,
+                      struct list_head *granted)
+{
+    struct list_head tmp_list;
+    posix_lock_t *l = NULL;
+    posix_lock_t *tmp = NULL;
+    posix_lock_t *conf = NULL;
+
+    INIT_LIST_HEAD(&tmp_list);
+
+    list_for_each_entry_safe(l, tmp, &pl_inode->ext_list, list)
+    {
+        if (l->blocked) {
+            conf = first_overlap(pl_inode, l);
+            if (conf)
+                continue;
+
+            l->blocked = 0;
+            list_move_tail(&l->list, &tmp_list);
+        }
+    }
 
-                                        INIT_LIST_HEAD (&v.locks[i]->list);
-                                        posix_lock_to_flock (v.locks[i],
-                                                       &v.locks[i]->user_flock);
-                                        __insert_and_merge (pl_inode,
-                                                            v.locks[i]);
-                                }
+    list_for_each_entry_safe(l, tmp, &tmp_list, list)
+    {
+        list_del_init(&l->list);
 
-                                __delete_unlck_locks (pl_inode);
-                                return;
-                        }
-                }
+        if (__is_lock_grantable(pl_inode, l)) {
+            conf = GF_CALLOC(1, sizeof(*conf), gf_locks_mt_posix_lock_t);
 
-                if (lock->fl_type == F_UNLCK) {
-                        continue;
-                }
+            if (!conf) {
+                l->blocked = 1;
+                __insert_lock(pl_inode, l);
+                continue;
+            }
 
-                if ((conf->fl_type == F_RDLCK) && (lock->fl_type == F_RDLCK)) {
-                        __insert_lock (pl_inode, lock);
-                        return;
-                }
-        }
+            conf->frame = l->frame;
+            l->frame = NULL;
+
+            posix_lock_to_flock(l, &conf->user_flock);
+
+            gf_log(this->name, GF_LOG_TRACE,
+                   "%s (pid=%d) lk-owner:%s %" PRId64 " - %" PRId64
+                   " => Granted",
+                   l->fl_type == F_UNLCK ? "Unlock" : "Lock", l->client_pid,
+                   lkowner_utoa(&l->owner), l->user_flock.l_start,
+                   l->user_flock.l_len);
 
-        /* no conflicts, so just insert */
-        if (lock->fl_type != F_UNLCK) {
-                __insert_lock (pl_inode, lock);
+            __insert_and_merge(pl_inode, l);
+
+            list_add(&conf->list, granted);
         } else {
-                __destroy_lock (lock);
+            l->blocked = 1;
+            __insert_lock(pl_inode, l);
         }
+    }
 }
 
-
 void
-__grant_blocked_locks (xlator_t *this, pl_inode_t *pl_inode, struct list_head *granted)
+grant_blocked_locks(xlator_t *this, pl_inode_t *pl_inode)
 {
-        struct list_head  tmp_list;
-        posix_lock_t     *l = NULL;
-        posix_lock_t     *tmp = NULL;
-        posix_lock_t     *conf = NULL;
+    struct list_head granted_list;
+    posix_lock_t *tmp = NULL;
+    posix_lock_t *lock = NULL;
+    pl_local_t *local = NULL;
+    INIT_LIST_HEAD(&granted_list);
+
+    pthread_mutex_lock(&pl_inode->mutex);
+    {
+        __grant_blocked_locks(this, pl_inode, &granted_list);
+    }
+    pthread_mutex_unlock(&pl_inode->mutex);
+
+    list_for_each_entry_safe(lock, tmp, &granted_list, list)
+    {
+        list_del_init(&lock->list);
+
+        pl_trace_out(this, lock->frame, NULL, NULL, F_SETLKW, &lock->user_flock,
+                     0, 0, NULL);
+        local = lock->frame->local;
+        PL_STACK_UNWIND_AND_FREE(local, lk, lock->frame, 0, 0,
+                                 &lock->user_flock, NULL);
+        __destroy_lock(lock);
+    }
+
+    return;
+}
 
-        INIT_LIST_HEAD (&tmp_list);
+static int
+pl_send_prelock_unlock(xlator_t *this, pl_inode_t *pl_inode,
+                       posix_lock_t *old_lock)
+{
+    struct gf_flock flock = {
+        0,
+    };
+    posix_lock_t *unlock_lock = NULL;
+    int32_t op_errno = 0;
 
-        list_for_each_entry_safe (l, tmp, &pl_inode->ext_list, list) {
-                if (l->blocked) {
-                        conf = first_overlap (pl_inode, l);
-                        if (conf)
-                                continue;
+    struct list_head granted_list;
+    posix_lock_t *tmp = NULL;
+    posix_lock_t *lock = NULL;
+    pl_local_t *local = NULL;
 
-                        l->blocked = 0;
-                        list_move_tail (&l->list, &tmp_list);
-                }
-        }
+    int ret = -1;
 
-        list_for_each_entry_safe (l, tmp, &tmp_list, list) {
-                list_del_init (&l->list);
+    INIT_LIST_HEAD(&granted_list);
 
-                if (__is_lock_grantable (pl_inode, l)) {
-                        conf = GF_CALLOC (1, sizeof (*conf),
-                                          gf_locks_mt_posix_lock_t);
+    flock.l_type = F_UNLCK;
+    flock.l_whence = old_lock->user_flock.l_whence;
+    flock.l_start = old_lock->user_flock.l_start;
+    flock.l_len = old_lock->user_flock.l_len;
+    flock.l_pid = old_lock->user_flock.l_pid;
 
-                        if (!conf) {
-                                l->blocked = 1;
-                                __insert_lock (pl_inode, l);
-                                continue;
-                        }
+    unlock_lock = new_posix_lock(&flock, old_lock->client, old_lock->client_pid,
+                                 &old_lock->owner, old_lock->fd,
+                                 old_lock->lk_flags, 0, &op_errno);
+    GF_VALIDATE_OR_GOTO(this->name, unlock_lock, out);
+    ret = 0;
 
-                        conf->frame = l->frame;
-                        l->frame = NULL;
+    __insert_and_merge(pl_inode, unlock_lock);
 
-                        posix_lock_to_flock (l, &conf->user_flock);
+    __grant_blocked_locks(this, pl_inode, &granted_list);
 
-                        gf_log (this->name, GF_LOG_TRACE,
-                                "%s (pid=%d) lk-owner:%s %"PRId64" - %"PRId64" => Granted",
-                                l->fl_type == F_UNLCK ? "Unlock" : "Lock",
-                                l->client_pid, lkowner_utoa (&l->owner),
-                                l->user_flock.l_start,
-                                l->user_flock.l_len);
+    list_for_each_entry_safe(lock, tmp, &granted_list, list)
+    {
+        list_del_init(&lock->list);
 
-                        __insert_and_merge (pl_inode, l);
+        pl_trace_out(this, lock->frame, NULL, NULL, F_SETLKW, &lock->user_flock,
+                     0, 0, NULL);
+        local = lock->frame->local;
+        PL_STACK_UNWIND_AND_FREE(local, lk, lock->frame, 0, 0,
+                                 &lock->user_flock, NULL);
+        __destroy_lock(lock);
+    }
 
-                        list_add (&conf->list, granted);
-                } else {
-                        l->blocked = 1;
-                        __insert_lock (pl_inode, l);
-                }
-        }
+out:
+    return ret;
 }
 
+int
+pl_setlk(xlator_t *this, pl_inode_t *pl_inode, posix_lock_t *lock,
+         int can_block)
+{
+    int ret = 0;
 
-void
-grant_blocked_locks (xlator_t *this, pl_inode_t *pl_inode)
+    errno = 0;
+
+    pthread_mutex_lock(&pl_inode->mutex);
+    {
+        /* Send unlock before the actual lock to
+           prevent lock upgrade / downgrade
+           problems only if:
+           - it is a blocking call
+           - it has other conflicting locks
+        */
+
+        if (can_block && !(__is_lock_grantable(pl_inode, lock))) {
+            ret = pl_send_prelock_unlock(this, pl_inode, lock);
+            if (ret)
+                gf_log(this->name, GF_LOG_DEBUG,
+                       "Could not send pre-lock "
+                       "unlock");
+        }
+
+        if (__is_lock_grantable(pl_inode, lock)) {
+            if (pl_metalock_is_active(pl_inode)) {
+                __pl_queue_lock(pl_inode, lock);
+                pthread_mutex_unlock(&pl_inode->mutex);
+                ret = -2;
+                goto out;
+            }
+            gf_log(this->name, GF_LOG_TRACE,
+                   "%s (pid=%d) lk-owner:%s %" PRId64 " - %" PRId64 " => OK",
+                   lock->fl_type == F_UNLCK ? "Unlock" : "Lock",
+                   lock->client_pid, lkowner_utoa(&lock->owner),
+                   lock->user_flock.l_start, lock->user_flock.l_len);
+            __insert_and_merge(pl_inode, lock);
+        } else if (can_block) {
+            if (pl_metalock_is_active(pl_inode)) {
+                __pl_queue_lock(pl_inode, lock);
+                pthread_mutex_unlock(&pl_inode->mutex);
+                ret = -2;
+                goto out;
+            }
+            gf_log(this->name, GF_LOG_TRACE,
+                   "%s (pid=%d) lk-owner:%s %" PRId64 " - %" PRId64
+                   " => Blocked",
+                   lock->fl_type == F_UNLCK ? "Unlock" : "Lock",
+                   lock->client_pid, lkowner_utoa(&lock->owner),
+                   lock->user_flock.l_start, lock->user_flock.l_len);
+
+            pl_trace_block(this, lock->frame, NULL, NULL, F_SETLKW,
+                           &lock->user_flock, NULL);
+
+            lock->blocked = 1;
+            __insert_lock(pl_inode, lock);
+            ret = -1;
+        } else {
+            gf_log(this->name, GF_LOG_TRACE,
+                   "%s (pid=%d) lk-owner:%s %" PRId64 " - %" PRId64 " => NOK",
+                   lock->fl_type == F_UNLCK ? "Unlock" : "Lock",
+                   lock->client_pid, lkowner_utoa(&lock->owner),
+                   lock->user_flock.l_start, lock->user_flock.l_len);
+            errno = EAGAIN;
+            ret = -1;
+        }
+    }
+    pthread_mutex_unlock(&pl_inode->mutex);
+
+    grant_blocked_locks(this, pl_inode);
+
+    do_blocked_rw(pl_inode);
+
+out:
+    return ret;
+}
+
+posix_lock_t *
+pl_getlk(pl_inode_t *pl_inode, posix_lock_t *lock)
 {
-        struct list_head granted_list;
-        posix_lock_t     *tmp = NULL;
-        posix_lock_t     *lock = NULL;
+    posix_lock_t *conf = first_conflicting_overlap(pl_inode, lock);
+    if (conf == NULL) {
+        lock->fl_type = F_UNLCK;
+        return lock;
+    }
+
+    return conf;
+}
 
-        INIT_LIST_HEAD (&granted_list);
+gf_boolean_t
+pl_does_monkey_want_stuck_lock()
+{
+    long int monkey_unlock_rand = 0;
+    long int monkey_unlock_rand_rem = 0;
+
+    /* coverity[DC.WEAK_CRYPTO] */
+    monkey_unlock_rand = random();
+    monkey_unlock_rand_rem = monkey_unlock_rand % 100;
+    if (monkey_unlock_rand_rem == 0)
+        return _gf_true;
+    return _gf_false;
+}
 
-        pthread_mutex_lock (&pl_inode->mutex);
+int
+pl_lock_preempt(pl_inode_t *pl_inode, posix_lock_t *reqlock)
+{
+    posix_lock_t *lock = NULL;
+    posix_lock_t *i = NULL;
+    pl_rw_req_t *rw = NULL;
+    pl_rw_req_t *itr = NULL;
+    struct list_head unwind_blist = {
+        0,
+    };
+    struct list_head unwind_rw_list = {
+        0,
+    };
+    int ret = 0;
+
+    INIT_LIST_HEAD(&unwind_blist);
+    INIT_LIST_HEAD(&unwind_rw_list);
+
+    pthread_mutex_lock(&pl_inode->mutex);
+    {
+        /*
+            - go through the lock list
+            - remove all locks from different owners
+            - same owner locks will be added or substracted based on
+              the new request
+            - add the new lock
+        */
+        list_for_each_entry_safe(lock, i, &pl_inode->ext_list, list)
         {
-                __grant_blocked_locks (this, pl_inode, &granted_list);
+            if (lock->blocked) {
+                list_del_init(&lock->list);
+                list_add(&lock->list, &unwind_blist);
+                continue;
+            }
+
+            if (locks_overlap(lock, reqlock)) {
+                if (same_owner(lock, reqlock))
+                    continue;
+
+                /* remove conflicting locks */
+                list_del_init(&lock->list);
+                __delete_lock(lock);
+                __destroy_lock(lock);
+            }
         }
-        pthread_mutex_unlock (&pl_inode->mutex);
 
-        list_for_each_entry_safe (lock, tmp, &granted_list, list) {
-                list_del_init (&lock->list);
+        __insert_and_merge(pl_inode, reqlock);
 
-                pl_trace_out (this, lock->frame, NULL, NULL, F_SETLKW,
-                              &lock->user_flock, 0, 0, NULL);
+        list_for_each_entry_safe(rw, itr, &pl_inode->rw_list, list)
+        {
+            list_del_init(&rw->list);
+            list_add(&rw->list, &unwind_rw_list);
+        }
+    }
+    pthread_mutex_unlock(&pl_inode->mutex);
+
+    /* unwind blocked locks */
+    list_for_each_entry_safe(lock, i, &unwind_blist, list)
+    {
+        PL_STACK_UNWIND_AND_FREE(((pl_local_t *)lock->frame->local), lk,
+                                 lock->frame, -1, EBUSY, &lock->user_flock,
+                                 NULL);
+        __destroy_lock(lock);
+    }
+
+    /* unwind blocked IOs */
+    list_for_each_entry_safe(rw, itr, &unwind_rw_list, list)
+    {
+        pl_clean_local(rw->stub->frame->local);
+        call_unwind_error(rw->stub, -1, EBUSY);
+    }
+
+    return ret;
+}
 
-                STACK_UNWIND_STRICT (lk, lock->frame, 0, 0,
-                                     &lock->user_flock, NULL);
+/* Return true in case we need to ensure mandatory-locking
+ * semantics under different modes.
+ */
+gf_boolean_t
+pl_is_mandatory_locking_enabled(pl_inode_t *pl_inode)
+{
+    posix_locks_private_t *priv = THIS->private;
 
-                GF_FREE (lock);
-        }
+    if (priv->mandatory_mode == MLK_FILE_BASED && pl_inode->mandatory)
+        return _gf_true;
+    else if (priv->mandatory_mode == MLK_FORCED ||
+             priv->mandatory_mode == MLK_OPTIMAL)
+        return _gf_true;
 
-        return;
+    return _gf_false;
 }
 
-static int
-pl_send_prelock_unlock (xlator_t *this, pl_inode_t *pl_inode,
-                        posix_lock_t *old_lock)
+void
+pl_clean_local(pl_local_t *local)
 {
-        struct gf_flock  flock       = {0,};
-        posix_lock_t *unlock_lock = NULL;
+    if (!local)
+        return;
 
-        struct list_head granted_list;
-        posix_lock_t     *tmp = NULL;
-        posix_lock_t     *lock = NULL;
+    if (local->inodelk_dom_count_req)
+        data_unref(local->inodelk_dom_count_req);
+    loc_wipe(&local->loc[0]);
+    loc_wipe(&local->loc[1]);
+    if (local->fd)
+        fd_unref(local->fd);
+    if (local->inode)
+        inode_unref(local->inode);
+    mem_put(local);
+}
+
+/*
+TODO: detach local initialization from PL_LOCAL_GET_REQUESTS and add it here
+*/
+int
+pl_local_init(call_frame_t *frame, xlator_t *this, loc_t *loc, fd_t *fd)
+{
+    pl_local_t *local = NULL;
+
+    if (!loc && !fd) {
+        return -1;
+    }
+
+    if (!frame->local) {
+        local = mem_get0(this->local_pool);
+        if (!local) {
+            gf_msg(this->name, GF_LOG_ERROR, ENOMEM, 0,
+                   "mem allocation failed");
+            return -1;
+        }
 
-        int ret = -1;
+        local->inode = (loc ? inode_ref(loc->inode) : inode_ref(fd->inode));
 
-        INIT_LIST_HEAD (&granted_list);
+        frame->local = local;
+    }
 
-        flock.l_type   = F_UNLCK;
-        flock.l_whence = old_lock->user_flock.l_whence;
-        flock.l_start  = old_lock->user_flock.l_start;
-        flock.l_len    = old_lock->user_flock.l_len;
+    return 0;
+}
 
+gf_boolean_t
+pl_is_lk_owner_valid(gf_lkowner_t *owner, client_t *client)
+{
+    if (client && (client->opversion < GD_OP_VERSION_7_0)) {
+        return _gf_true;
+    }
+
+    if (is_lk_owner_null(owner)) {
+        return _gf_false;
+    }
+    return _gf_true;
+}
 
-        unlock_lock = new_posix_lock (&flock, old_lock->client,
-                                      old_lock->client_pid, &old_lock->owner,
-                                      old_lock->fd);
-        GF_VALIDATE_OR_GOTO (this->name, unlock_lock, out);
-        ret = 0;
+static int32_t
+pl_inode_from_loc(loc_t *loc, inode_t **pinode)
+{
+    inode_t *inode = NULL;
+    int32_t error = 0;
+
+    if (loc->inode != NULL) {
+        inode = inode_ref(loc->inode);
+        goto done;
+    }
+
+    if (loc->parent == NULL) {
+        error = EINVAL;
+        goto done;
+    }
+
+    if (!gf_uuid_is_null(loc->gfid)) {
+        inode = inode_find(loc->parent->table, loc->gfid);
+        if (inode != NULL) {
+            goto done;
+        }
+    }
 
-        __insert_and_merge (pl_inode, unlock_lock);
+    if (loc->name == NULL) {
+        error = EINVAL;
+        goto done;
+    }
 
-        __grant_blocked_locks (this, pl_inode, &granted_list);
+    inode = inode_grep(loc->parent->table, loc->parent, loc->name);
+    if (inode == NULL) {
+        /* We haven't found any inode. This means that the file doesn't exist
+         * or that even if it exists, we don't have any knowledge about it, so
+         * we don't have locks on it either, which is fine for our purposes. */
+        goto done;
+    }
 
-        list_for_each_entry_safe (lock, tmp, &granted_list, list) {
-                list_del_init (&lock->list);
+done:
+    *pinode = inode;
 
-                pl_trace_out (this, lock->frame, NULL, NULL, F_SETLKW,
-                              &lock->user_flock, 0, 0, NULL);
+    return error;
+}
 
-                STACK_UNWIND_STRICT (lk, lock->frame, 0, 0,
-                                     &lock->user_flock, NULL);
+static gf_boolean_t
+pl_inode_has_owners(xlator_t *xl, client_t *client, pl_inode_t *pl_inode,
+                    struct timespec *now, struct list_head *contend)
+{
+    pl_dom_list_t *dom;
+    pl_inode_lock_t *lock;
+    gf_boolean_t has_owners = _gf_false;
 
-                GF_FREE (lock);
+    list_for_each_entry(dom, &pl_inode->dom_list, inode_list)
+    {
+        list_for_each_entry(lock, &dom->inodelk_list, list)
+        {
+            /* If the lock belongs to the same client, we assume it's related
+             * to the same operation, so we allow the removal to continue. */
+            if (lock->client == client) {
+                continue;
+            }
+            /* If the lock belongs to an internal process, we don't block the
+             * removal. */
+            if (lock->client_pid < 0) {
+                continue;
+            }
+            if (contend == NULL) {
+                return _gf_true;
+            }
+            has_owners = _gf_true;
+            inodelk_contention_notify_check(xl, lock, now, contend);
         }
+    }
 
-out:
-        return ret;
+    return has_owners;
 }
 
-int
-pl_setlk (xlator_t *this, pl_inode_t *pl_inode, posix_lock_t *lock,
-          int can_block)
+int32_t
+pl_inode_remove_prepare(xlator_t *xl, call_frame_t *frame, loc_t *loc,
+                        pl_inode_t **ppl_inode, struct list_head *contend)
 {
-        int              ret = 0;
+    struct timespec now;
+    inode_t *inode;
+    pl_inode_t *pl_inode;
+    int32_t error;
+
+    pl_inode = NULL;
+
+    error = pl_inode_from_loc(loc, &inode);
+    if ((error != 0) || (inode == NULL)) {
+        goto done;
+    }
+
+    pl_inode = pl_inode_get(xl, inode, NULL);
+    if (pl_inode == NULL) {
+        inode_unref(inode);
+        error = ENOMEM;
+        goto done;
+    }
+
+    /* pl_inode_from_loc() already increments ref count for inode, so
+     * we only assign here our reference. */
+    pl_inode->inode = inode;
+
+    timespec_now(&now);
+
+    pthread_mutex_lock(&pl_inode->mutex);
+
+    if (pl_inode->removed) {
+        error = ESTALE;
+        goto unlock;
+    }
+
+    if (pl_inode_has_owners(xl, frame->root->client, pl_inode, &now, contend)) {
+        error = -1;
+        /* We skip the unlock here because the caller must create a stub when
+         * we return -1 and do a call to pl_inode_remove_complete(), which
+         * assumes the lock is still acquired and will release it once
+         * everything else is prepared. */
+        goto done;
+    }
+
+    pl_inode->is_locked = _gf_true;
+    pl_inode->remove_running++;
 
-        errno = 0;
+unlock:
+    pthread_mutex_unlock(&pl_inode->mutex);
 
-        pthread_mutex_lock (&pl_inode->mutex);
-        {
-                /* Send unlock before the actual lock to
-                   prevent lock upgrade / downgrade
-                   problems only if:
-                   - it is a blocking call
-                   - it has other conflicting locks
-                */
-
-                if (can_block &&
-                    !(__is_lock_grantable (pl_inode, lock))) {
-                        ret = pl_send_prelock_unlock (this, pl_inode,
-                                                      lock);
-                        if (ret)
-                                gf_log (this->name, GF_LOG_DEBUG,
-                                "Could not send pre-lock "
-                                        "unlock");
-                }
+done:
+    *ppl_inode = pl_inode;
 
-                if (__is_lock_grantable (pl_inode, lock)) {
-                        gf_log (this->name, GF_LOG_TRACE,
-                                "%s (pid=%d) lk-owner:%s %"PRId64" - %"PRId64" => OK",
-                                lock->fl_type == F_UNLCK ? "Unlock" : "Lock",
-                                lock->client_pid,
-                                lkowner_utoa (&lock->owner),
-                                lock->user_flock.l_start,
-                                lock->user_flock.l_len);
-                        __insert_and_merge (pl_inode, lock);
-                } else if (can_block) {
-                        gf_log (this->name, GF_LOG_TRACE,
-                                "%s (pid=%d) lk-owner:%s %"PRId64" - %"PRId64" => Blocked",
-                                lock->fl_type == F_UNLCK ? "Unlock" : "Lock",
-                                lock->client_pid,
-                                lkowner_utoa (&lock->owner),
-                                lock->user_flock.l_start,
-                                lock->user_flock.l_len);
-                        lock->blocked = 1;
-                        __insert_lock (pl_inode, lock);
-                        ret = -1;
-                } else {
-                        gf_log (this->name, GF_LOG_TRACE,
-                                "%s (pid=%d) lk-owner:%s %"PRId64" - %"PRId64" => NOK",
-                                lock->fl_type == F_UNLCK ? "Unlock" : "Lock",
-                                lock->client_pid,
-                                lkowner_utoa (&lock->owner),
-                                lock->user_flock.l_start,
-                                lock->user_flock.l_len);
-                        errno = EAGAIN;
-                        ret = -1;
-                }
+    return error;
+}
+
+int32_t
+pl_inode_remove_complete(xlator_t *xl, pl_inode_t *pl_inode, call_stub_t *stub,
+                         struct list_head *contend)
+{
+    pl_inode_lock_t *lock;
+    int32_t error = -1;
+
+    if (stub != NULL) {
+        list_add_tail(&stub->list, &pl_inode->waiting);
+        pl_inode->is_locked = _gf_true;
+    } else {
+        error = ENOMEM;
+
+        while (!list_empty(contend)) {
+            lock = list_first_entry(contend, pl_inode_lock_t, list);
+            list_del_init(&lock->list);
+            __pl_inodelk_unref(lock);
         }
-        pthread_mutex_unlock (&pl_inode->mutex);
+    }
 
-        grant_blocked_locks (this, pl_inode);
+    pthread_mutex_unlock(&pl_inode->mutex);
 
-        do_blocked_rw (pl_inode);
+    if (error < 0) {
+        inodelk_contention_notify(xl, contend);
+    }
 
-        return ret;
+    inode_unref(pl_inode->inode);
+
+    return error;
 }
 
+void
+pl_inode_remove_wake(struct list_head *list)
+{
+    call_stub_t *stub;
+
+    while (!list_empty(list)) {
+        stub = list_first_entry(list, call_stub_t, list);
+        list_del_init(&stub->list);
 
-posix_lock_t *
-pl_getlk (pl_inode_t *pl_inode, posix_lock_t *lock)
+        call_resume(stub);
+    }
+}
+
+void
+pl_inode_remove_cbk(xlator_t *xl, pl_inode_t *pl_inode, int32_t error)
 {
-        posix_lock_t *conf = NULL;
+    struct list_head contend, granted;
+    struct timespec now;
+    pl_dom_list_t *dom;
+
+    if (pl_inode == NULL) {
+        return;
+    }
 
-        conf = first_conflicting_overlap (pl_inode, lock);
+    INIT_LIST_HEAD(&contend);
+    INIT_LIST_HEAD(&granted);
+    timespec_now(&now);
 
-        if (conf == NULL) {
-                lock->fl_type = F_UNLCK;
-                return lock;
+    pthread_mutex_lock(&pl_inode->mutex);
+
+    if (error == 0) {
+        if (pl_inode->links >= 0) {
+            pl_inode->links--;
+        }
+        if (pl_inode->links == 0) {
+            pl_inode->removed = _gf_true;
+        }
+    }
+
+    pl_inode->remove_running--;
+
+    if ((pl_inode->remove_running == 0) && list_empty(&pl_inode->waiting)) {
+        pl_inode->is_locked = _gf_false;
+
+        list_for_each_entry(dom, &pl_inode->dom_list, inode_list)
+        {
+            __grant_blocked_inode_locks(xl, pl_inode, &granted, dom, &now,
+                                        &contend);
         }
+    }
+
+    pthread_mutex_unlock(&pl_inode->mutex);
 
-        return conf;
+    unwind_granted_inodes(xl, pl_inode, &granted);
+
+    inodelk_contention_notify(xl, &contend);
+
+    inode_unref(pl_inode->inode);
+}
+
+void
+pl_inode_remove_unlocked(xlator_t *xl, pl_inode_t *pl_inode,
+                         struct list_head *list)
+{
+    call_stub_t *stub, *tmp;
+
+    if (!pl_inode->is_locked) {
+        return;
+    }
+
+    list_for_each_entry_safe(stub, tmp, &pl_inode->waiting, list)
+    {
+        if (!pl_inode_has_owners(xl, stub->frame->root->client, pl_inode, NULL,
+                                 NULL)) {
+            list_move_tail(&stub->list, list);
+        }
+    }
 }
 
+/* This function determines if an inodelk attempt can be done now or it needs
+ * to wait.
+ *
+ * Possible return values:
+ *   < 0: An error occurred. Currently only -ESTALE can be returned if the
+ *        inode has been deleted previously by unlink/rmdir/rename
+ *   = 0: The lock can be attempted.
+ *   > 0: The lock needs to wait because a conflicting remove operation is
+ *        ongoing.
+ */
+int32_t
+pl_inode_remove_inodelk(pl_inode_t *pl_inode, pl_inode_lock_t *lock)
+{
+    pl_dom_list_t *dom;
+    pl_inode_lock_t *ilock;
+
+    /* If the inode has been deleted, we won't allow any lock. */
+    if (pl_inode->removed) {
+        return -ESTALE;
+    }
+
+    /* We only synchronize with locks made for regular operations coming from
+     * the user. Locks done for internal purposes are hard to control and could
+     * lead to long delays or deadlocks quite easily. */
+    if (lock->client_pid < 0) {
+        return 0;
+    }
+    if (!pl_inode->is_locked) {
+        return 0;
+    }
+    if (pl_inode->remove_running > 0) {
+        return 1;
+    }
+
+    list_for_each_entry(dom, &pl_inode->dom_list, inode_list)
+    {
+        list_for_each_entry(ilock, &dom->inodelk_list, list)
+        {
+            /* If a lock from the same client is already granted, we allow this
+             * one to continue. This is necessary to prevent deadlocks when
+             * multiple locks are taken for the same operation.
+             *
+             * On the other side it's unlikely that the same client sends
+             * completely unrelated locks for the same inode.
+             */
+            if (ilock->client == lock->client) {
+                return 0;
+            }
+        }
+    }
+
+    return 1;
+}
diff --git a/xlators/features/locks/src/common.h b/xlators/features/locks/src/common.h
index 5ec630ee857..281223bf3b8 100644
--- a/xlators/features/locks/src/common.h
+++ b/xlators/features/locks/src/common.h
@@ -1,5 +1,5 @@
 /*
-   Copyright (c) 2006-2012 Red Hat, Inc. <http://www.redhat.com>
+   Copyright (c) 2006-2012, 2015-2016 Red Hat, Inc. <http://www.redhat.com>
    This file is part of GlusterFS.
 
    This file is licensed to you under your choice of the GNU Lesser
@@ -10,149 +10,253 @@
 #ifndef __COMMON_H__
 #define __COMMON_H__
 
-#include "lkowner.h"
 /*dump locks format strings */
-#define RANGE_FMT               "type=%s, whence=%hd, start=%llu, len=%llu"
-#define ENTRY_FMT               "type=%s on basename=%s"
-#define DUMP_GEN_FMT            "pid = %llu, owner=%s, client=%p"
-#define GRNTD_AT                "granted at %s"
-#define BLKD_AT                 "blocked at %s"
-#define CONN_ID                 "connection-id=%s"
-#define DUMP_BLKD_FMT           DUMP_GEN_FMT", "CONN_ID", "BLKD_AT
-#define DUMP_GRNTD_FMT          DUMP_GEN_FMT", "CONN_ID", "GRNTD_AT
-#define DUMP_BLKD_GRNTD_FMT     DUMP_GEN_FMT", "CONN_ID", "BLKD_AT", "GRNTD_AT
-
-#define ENTRY_BLKD_FMT          ENTRY_FMT", "DUMP_BLKD_FMT
-#define ENTRY_GRNTD_FMT         ENTRY_FMT", "DUMP_GRNTD_FMT
-#define ENTRY_BLKD_GRNTD_FMT    ENTRY_FMT", "DUMP_BLKD_GRNTD_FMT
-
-#define RANGE_BLKD_FMT          RANGE_FMT", "DUMP_BLKD_FMT
-#define RANGE_GRNTD_FMT         RANGE_FMT", "DUMP_GRNTD_FMT
-#define RANGE_BLKD_GRNTD_FMT    RANGE_FMT", "DUMP_BLKD_GRNTD_FMT
+#define RANGE_FMT "type=%s, whence=%hd, start=%llu, len=%llu"
+#define ENTRY_FMT "type=%s on basename=%s"
+#define DUMP_GEN_FMT "pid = %llu, owner=%s, client=%p"
+#define GRNTD_AT "granted at %s"
+#define BLKD_AT "blocked at %s"
+#define CONN_ID "connection-id=%s"
+#define DUMP_BLKD_FMT DUMP_GEN_FMT ", " CONN_ID ", " BLKD_AT
+#define DUMP_GRNTD_FMT DUMP_GEN_FMT ", " CONN_ID ", " GRNTD_AT
+#define DUMP_BLKD_GRNTD_FMT DUMP_GEN_FMT ", " CONN_ID ", " BLKD_AT ", " GRNTD_AT
+
+#define ENTRY_BLKD_FMT ENTRY_FMT ", " DUMP_BLKD_FMT
+#define ENTRY_GRNTD_FMT ENTRY_FMT ", " DUMP_GRNTD_FMT
+#define ENTRY_BLKD_GRNTD_FMT ENTRY_FMT ", " DUMP_BLKD_GRNTD_FMT
+
+#define RANGE_BLKD_FMT RANGE_FMT ", " DUMP_BLKD_FMT
+#define RANGE_GRNTD_FMT RANGE_FMT ", " DUMP_GRNTD_FMT
+#define RANGE_BLKD_GRNTD_FMT RANGE_FMT ", " DUMP_BLKD_GRNTD_FMT
 
 #define SET_FLOCK_PID(flock, lock) ((flock)->l_pid = lock->client_pid)
 
+#define PL_STACK_UNWIND_AND_FREE(__local, fop, frame, op_ret, params...)       \
+    do {                                                                       \
+        frame->local = NULL;                                                   \
+        STACK_UNWIND_STRICT(fop, frame, op_ret, params);                       \
+        if (__local) {                                                         \
+            if (__local->inodelk_dom_count_req)                                \
+                data_unref(__local->inodelk_dom_count_req);                    \
+            loc_wipe(&__local->loc[0]);                                        \
+            loc_wipe(&__local->loc[1]);                                        \
+            if (__local->fd)                                                   \
+                fd_unref(__local->fd);                                         \
+            if (__local->inode)                                                \
+                inode_unref(__local->inode);                                   \
+            if (__local->xdata) {                                              \
+                dict_unref(__local->xdata);                                    \
+                __local->xdata = NULL;                                         \
+            }                                                                  \
+            mem_put(__local);                                                  \
+        }                                                                      \
+    } while (0)
 
 posix_lock_t *
-new_posix_lock (struct gf_flock *flock, client_t *client, pid_t client_pid,
-                gf_lkowner_t *owner, fd_t *fd);
+new_posix_lock(struct gf_flock *flock, client_t *client, pid_t client_pid,
+               gf_lkowner_t *owner, fd_t *fd, uint32_t lk_flags, int blocking,
+               int32_t *op_errno);
 
 pl_inode_t *
-pl_inode_get (xlator_t *this, inode_t *inode);
+pl_inode_get(xlator_t *this, inode_t *inode, pl_local_t *local);
 
 posix_lock_t *
-pl_getlk (pl_inode_t *inode, posix_lock_t *lock);
+pl_getlk(pl_inode_t *inode, posix_lock_t *lock);
 
 int
-pl_setlk (xlator_t *this, pl_inode_t *inode, posix_lock_t *lock,
-          int can_block);
+pl_setlk(xlator_t *this, pl_inode_t *inode, posix_lock_t *lock, int can_block);
+
+int
+pl_lock_preempt(pl_inode_t *pl_inode, posix_lock_t *reqlock);
 
 void
-grant_blocked_locks (xlator_t *this, pl_inode_t *inode);
+grant_blocked_locks(xlator_t *this, pl_inode_t *inode);
 
 void
-posix_lock_to_flock (posix_lock_t *lock, struct gf_flock *flock);
+posix_lock_to_flock(posix_lock_t *lock, struct gf_flock *flock);
 
 int
-locks_overlap (posix_lock_t *l1, posix_lock_t *l2);
+locks_overlap(posix_lock_t *l1, posix_lock_t *l2);
 
 int
-same_owner (posix_lock_t *l1, posix_lock_t *l2);
+same_owner(posix_lock_t *l1, posix_lock_t *l2);
 
-void __delete_lock (pl_inode_t *, posix_lock_t *);
+void
+__delete_lock(posix_lock_t *);
 
-void __destroy_lock (posix_lock_t *);
+void
+__destroy_lock(posix_lock_t *);
 
 pl_dom_list_t *
-get_domain (pl_inode_t *pl_inode, const char *volume);
+get_domain(pl_inode_t *pl_inode, const char *volume);
 
 void
-grant_blocked_inode_locks (xlator_t *this, pl_inode_t *pl_inode,
-                           pl_dom_list_t *dom);
+grant_blocked_inode_locks(xlator_t *this, pl_inode_t *pl_inode,
+                          pl_dom_list_t *dom, struct timespec *now,
+                          struct list_head *contend);
 
 void
-__delete_inode_lock (pl_inode_lock_t *lock);
+inodelk_contention_notify(xlator_t *this, struct list_head *contend);
 
 void
-__pl_inodelk_unref (pl_inode_lock_t *lock);
+__delete_inode_lock(pl_inode_lock_t *lock);
 
 void
-grant_blocked_entry_locks (xlator_t *this, pl_inode_t *pl_inode,
-                           pl_dom_list_t *dom);
+__pl_inodelk_unref(pl_inode_lock_t *lock);
 
-void pl_update_refkeeper (xlator_t *this, inode_t *inode);
+void
+__grant_blocked_inode_locks(xlator_t *this, pl_inode_t *pl_inode,
+                            struct list_head *granted, pl_dom_list_t *dom,
+                            struct timespec *now, struct list_head *contend);
+
+void
+unwind_granted_inodes(xlator_t *this, pl_inode_t *pl_inode,
+                      struct list_head *granted);
+
+void
+grant_blocked_entry_locks(xlator_t *this, pl_inode_t *pl_inode,
+                          pl_dom_list_t *dom, struct timespec *now,
+                          struct list_head *contend);
+
+void
+entrylk_contention_notify(xlator_t *this, struct list_head *contend);
+
+void
+pl_update_refkeeper(xlator_t *this, inode_t *inode);
 
 int32_t
-__get_inodelk_count (xlator_t *this, pl_inode_t *pl_inode, char *domname);
+__get_inodelk_count(xlator_t *this, pl_inode_t *pl_inode, char *domname);
 int32_t
-get_inodelk_count (xlator_t *this, inode_t *inode, char *domname);
+get_inodelk_count(xlator_t *this, inode_t *inode, char *domname);
 
 int32_t
-__get_entrylk_count (xlator_t *this, pl_inode_t *pl_inode);
+__get_entrylk_count(xlator_t *this, pl_inode_t *pl_inode);
 int32_t
-get_entrylk_count (xlator_t *this, inode_t *inode);
+get_entrylk_count(xlator_t *this, inode_t *inode);
 
-void pl_trace_in (xlator_t *this, call_frame_t *frame, fd_t *fd, loc_t *loc,
-                  int cmd, struct gf_flock *flock, const char *domain);
+void
+pl_trace_in(xlator_t *this, call_frame_t *frame, fd_t *fd, loc_t *loc, int cmd,
+            struct gf_flock *flock, const char *domain);
 
-void pl_trace_out (xlator_t *this, call_frame_t *frame, fd_t *fd, loc_t *loc,
-                   int cmd, struct gf_flock *flock, int op_ret, int op_errno, const char *domain);
+void
+pl_trace_out(xlator_t *this, call_frame_t *frame, fd_t *fd, loc_t *loc, int cmd,
+             struct gf_flock *flock, int op_ret, int op_errno,
+             const char *domain);
 
-void pl_trace_block (xlator_t *this, call_frame_t *frame, fd_t *fd, loc_t *loc,
-                     int cmd, struct gf_flock *flock, const char *domain);
+void
+pl_trace_block(xlator_t *this, call_frame_t *frame, fd_t *fd, loc_t *loc,
+               int cmd, struct gf_flock *flock, const char *domain);
 
-void pl_trace_flush (xlator_t *this, call_frame_t *frame, fd_t *fd);
+void
+pl_trace_flush(xlator_t *this, call_frame_t *frame, fd_t *fd);
 
-void entrylk_trace_in (xlator_t *this, call_frame_t *frame, const char *volume,
-                       fd_t *fd, loc_t *loc, const char *basename,
-                       entrylk_cmd cmd, entrylk_type type);
+void
+entrylk_trace_in(xlator_t *this, call_frame_t *frame, const char *volume,
+                 fd_t *fd, loc_t *loc, const char *basename, entrylk_cmd cmd,
+                 entrylk_type type);
 
-void entrylk_trace_out (xlator_t *this, call_frame_t *frame, const char *volume,
-                        fd_t *fd, loc_t *loc, const char *basename,
-                        entrylk_cmd cmd, entrylk_type type,
-                        int op_ret, int op_errno);
+void
+entrylk_trace_out(xlator_t *this, call_frame_t *frame, const char *volume,
+                  fd_t *fd, loc_t *loc, const char *basename, entrylk_cmd cmd,
+                  entrylk_type type, int op_ret, int op_errno);
 
-void entrylk_trace_block (xlator_t *this, call_frame_t *frame, const char *volume,
-                          fd_t *fd, loc_t *loc, const char *basename,
-                          entrylk_cmd cmd, entrylk_type type);
+void
+entrylk_trace_block(xlator_t *this, call_frame_t *frame, const char *volume,
+                    fd_t *fd, loc_t *loc, const char *basename, entrylk_cmd cmd,
+                    entrylk_type type);
 
 void
-pl_print_verdict (char *str, int size, int op_ret, int op_errno);
+pl_print_verdict(char *str, int size, int op_ret, int op_errno);
 
 void
-pl_print_lockee (char *str, int size, fd_t *fd, loc_t *loc);
+pl_print_lockee(char *str, int size, fd_t *fd, loc_t *loc);
 
 void
-pl_print_locker (char *str, int size, xlator_t *this, call_frame_t *frame);
+pl_print_locker(char *str, int size, xlator_t *this, call_frame_t *frame);
 
 void
-pl_print_inodelk (char *str, int size, int cmd, struct gf_flock *flock, const char *domain);
+pl_print_inodelk(char *str, int size, int cmd, struct gf_flock *flock,
+                 const char *domain);
 
 void
-pl_trace_release (xlator_t *this, fd_t *fd);
+pl_trace_release(xlator_t *this, fd_t *fd);
 
 unsigned long
-fd_to_fdnum (fd_t *fd);
+fd_to_fdnum(fd_t *fd);
 
 fd_t *
-fd_from_fdnum (posix_lock_t *lock);
+fd_from_fdnum(posix_lock_t *lock);
+
+int
+pl_reserve_setlk(xlator_t *this, pl_inode_t *pl_inode, posix_lock_t *lock,
+                 int can_block);
+int
+reservelks_equal(posix_lock_t *l1, posix_lock_t *l2);
 
 int
-pl_reserve_setlk (xlator_t *this, pl_inode_t *pl_inode, posix_lock_t *lock,
-                  int can_block);
+pl_verify_reservelk(xlator_t *this, pl_inode_t *pl_inode, posix_lock_t *lock,
+                    int can_block);
 int
-reservelks_equal (posix_lock_t *l1, posix_lock_t *l2);
+pl_reserve_unlock(xlator_t *this, pl_inode_t *pl_inode, posix_lock_t *reqlock);
+
+int32_t
+check_entrylk_on_basename(xlator_t *this, inode_t *parent, char *basename);
+
+void
+__pl_inodelk_unref(pl_inode_lock_t *lock);
+void
+__pl_entrylk_unref(pl_entry_lock_t *lock);
 
 int
-pl_verify_reservelk (xlator_t *this, pl_inode_t *pl_inode,
-                     posix_lock_t *lock, int can_block);
+pl_metalock_is_active(pl_inode_t *pl_inode);
+
+void
+__pl_queue_lock(pl_inode_t *pl_inode, posix_lock_t *reqlock);
+
+void
+inodelk_contention_notify_check(xlator_t *xl, pl_inode_lock_t *lock,
+                                struct timespec *now,
+                                struct list_head *contend);
+
+void
+entrylk_contention_notify_check(xlator_t *xl, pl_entry_lock_t *lock,
+                                struct timespec *now,
+                                struct list_head *contend);
+
+gf_boolean_t
+pl_does_monkey_want_stuck_lock();
+
+gf_boolean_t
+pl_is_mandatory_locking_enabled(pl_inode_t *pl_inode);
+
+void
+pl_clean_local(pl_local_t *local);
+
 int
-pl_reserve_unlock (xlator_t *this, pl_inode_t *pl_inode, posix_lock_t *reqlock);
+pl_local_init(call_frame_t *frame, xlator_t *this, loc_t *loc, fd_t *fd);
+
+gf_boolean_t
+pl_is_lk_owner_valid(gf_lkowner_t *owner, client_t *client);
+
+int32_t
+pl_inode_remove_prepare(xlator_t *xl, call_frame_t *frame, loc_t *loc,
+                        pl_inode_t **ppl_inode, struct list_head *contend);
+
+int32_t
+pl_inode_remove_complete(xlator_t *xl, pl_inode_t *pl_inode, call_stub_t *stub,
+                         struct list_head *contend);
 
-uint32_t
-check_entrylk_on_basename (xlator_t *this, inode_t *parent, char *basename);
+void
+pl_inode_remove_wake(struct list_head *list);
+
+void
+pl_inode_remove_cbk(xlator_t *xl, pl_inode_t *pl_inode, int32_t error);
 
-void __pl_inodelk_unref (pl_inode_lock_t *lock);
-void __pl_entrylk_unref (pl_entry_lock_t *lock);
+void
+pl_inode_remove_unlocked(xlator_t *xl, pl_inode_t *pl_inode,
+                         struct list_head *list);
+
+int32_t
+pl_inode_remove_inodelk(pl_inode_t *pl_inode, pl_inode_lock_t *lock);
 
 #endif /* __COMMON_H__ */
diff --git a/xlators/features/locks/src/entrylk.c b/xlators/features/locks/src/entrylk.c
index 04ffd6d387b..fd772c850dd 100644
--- a/xlators/features/locks/src/entrylk.c
+++ b/xlators/features/locks/src/entrylk.c
@@ -7,77 +7,77 @@
    later), or the GNU General Public License, version 2 (GPLv2), in all
    cases as published by the Free Software Foundation.
 */
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
-
-#include "glusterfs.h"
-#include "compat.h"
-#include "xlator.h"
-#include "inode.h"
-#include "logging.h"
-#include "common-utils.h"
-#include "list.h"
+#include <glusterfs/glusterfs.h>
+#include <glusterfs/compat.h>
+#include <glusterfs/xlator.h>
+#include <glusterfs/logging.h>
+#include <glusterfs/common-utils.h>
+#include <glusterfs/list.h>
+#include <glusterfs/upcall-utils.h>
 
 #include "locks.h"
+#include "clear.h"
 #include "common.h"
-
+#include "pl-messages.h"
 
 void
-__pl_entrylk_unref (pl_entry_lock_t *lock)
+__pl_entrylk_unref(pl_entry_lock_t *lock)
 {
-        lock->ref--;
-        if (!lock->ref) {
-		GF_FREE ((char *)lock->basename);
-                GF_FREE (lock->connection_id);
-                GF_FREE (lock);
-        }
+    lock->ref--;
+    if (!lock->ref) {
+        GF_FREE((char *)lock->basename);
+        GF_FREE(lock->connection_id);
+        GF_FREE(lock);
+    }
 }
 
-
 static void
-__pl_entrylk_ref (pl_entry_lock_t *lock)
+__pl_entrylk_ref(pl_entry_lock_t *lock)
 {
-        lock->ref++;
+    lock->ref++;
 }
 
-
 static pl_entry_lock_t *
-new_entrylk_lock (pl_inode_t *pinode, const char *basename, entrylk_type type,
-		  const char *domain, call_frame_t *frame, char *conn_id)
+new_entrylk_lock(pl_inode_t *pinode, const char *basename, entrylk_type type,
+                 const char *domain, call_frame_t *frame, char *conn_id,
+                 int32_t *op_errno)
 {
-        pl_entry_lock_t *newlock = NULL;
-
-        newlock = GF_CALLOC (1, sizeof (pl_entry_lock_t),
-                             gf_locks_mt_pl_entry_lock_t);
-        if (!newlock) {
-                goto out;
-        }
-
-        newlock->basename   = basename ? gf_strdup (basename) : NULL;
-        newlock->type       = type;
-        newlock->client     = frame->root->client;
-        newlock->client_pid = frame->root->pid;
-        newlock->volume     = domain;
-        newlock->owner      = frame->root->lk_owner;
-	newlock->frame      = frame;
-        newlock->this       = frame->this;
-
-        if (conn_id) {
-                newlock->connection_id = gf_strdup (conn_id);
-        }
-
-        INIT_LIST_HEAD (&newlock->domain_list);
-        INIT_LIST_HEAD (&newlock->blocked_locks);
-	INIT_LIST_HEAD (&newlock->client_list);
-
-	__pl_entrylk_ref (newlock);
+    pl_entry_lock_t *newlock = NULL;
+
+    if (!pl_is_lk_owner_valid(&frame->root->lk_owner, frame->root->client)) {
+        *op_errno = EINVAL;
+        goto out;
+    }
+
+    newlock = GF_CALLOC(1, sizeof(pl_entry_lock_t),
+                        gf_locks_mt_pl_entry_lock_t);
+    if (!newlock) {
+        *op_errno = ENOMEM;
+        goto out;
+    }
+
+    newlock->basename = basename ? gf_strdup(basename) : NULL;
+    newlock->type = type;
+    newlock->client = frame->root->client;
+    newlock->client_pid = frame->root->pid;
+    newlock->volume = domain;
+    newlock->owner = frame->root->lk_owner;
+    newlock->frame = frame;
+    newlock->this = frame->this;
+
+    if (conn_id) {
+        newlock->connection_id = gf_strdup(conn_id);
+    }
+
+    INIT_LIST_HEAD(&newlock->domain_list);
+    INIT_LIST_HEAD(&newlock->blocked_locks);
+    INIT_LIST_HEAD(&newlock->client_list);
+
+    __pl_entrylk_ref(newlock);
 out:
-        return newlock;
+    return newlock;
 }
 
-
 /**
  * all_names - does a basename represent all names?
  * @basename: name to check
@@ -92,28 +92,220 @@ out:
  */
 
 static int
-names_conflict (const char *n1, const char *n2)
+names_conflict(const char *n1, const char *n2)
 {
-        return all_names (n1) || all_names (n2) || !strcmp (n1, n2);
+    return all_names(n1) || all_names(n2) || !strcmp(n1, n2);
 }
 
-
-static inline int
-__same_entrylk_owner (pl_entry_lock_t *l1, pl_entry_lock_t *l2)
+static int
+__same_entrylk_owner(pl_entry_lock_t *l1, pl_entry_lock_t *l2)
 {
-        return (is_same_lkowner (&l1->owner, &l2->owner) &&
-                (l1->client  == l2->client));
+    return (is_same_lkowner(&l1->owner, &l2->owner) &&
+            (l1->client == l2->client));
 }
 
 /* Just as in inodelk, allow conflicting name locks from same (lk_owner, conn)*/
-static inline int
-__conflicting_entrylks (pl_entry_lock_t *l1, pl_entry_lock_t *l2)
+static int
+__conflicting_entrylks(pl_entry_lock_t *l1, pl_entry_lock_t *l2)
+{
+    if (names_conflict(l1->basename, l2->basename) &&
+        !__same_entrylk_owner(l1, l2))
+        return 1;
+
+    return 0;
+}
+
+/* See comments in inodelk.c for details */
+static inline gf_boolean_t
+__stale_entrylk(xlator_t *this, pl_entry_lock_t *candidate_lock,
+                pl_entry_lock_t *requested_lock, time_t *lock_age_sec)
+{
+    posix_locks_private_t *priv = NULL;
+
+    priv = this->private;
+
+    /* Question: Should we just prune them all given the
+     * chance?  Or just the locks we are attempting to acquire?
+     */
+    if (names_conflict(candidate_lock->basename, requested_lock->basename)) {
+        *lock_age_sec = gf_time() - candidate_lock->granted_time;
+        if (*lock_age_sec > priv->revocation_secs)
+            return _gf_true;
+    }
+    return _gf_false;
+}
+
+/* See comments in inodelk.c for details */
+static gf_boolean_t
+__entrylk_prune_stale(xlator_t *this, pl_inode_t *pinode, pl_dom_list_t *dom,
+                      pl_entry_lock_t *lock)
+{
+    posix_locks_private_t *priv = NULL;
+    pl_entry_lock_t *tmp = NULL;
+    pl_entry_lock_t *lk = NULL;
+    gf_boolean_t revoke_lock = _gf_false;
+    int bcount = 0;
+    int gcount = 0;
+    int op_errno = 0;
+    clrlk_args args;
+    args.opts = NULL;
+    time_t lk_age_sec = 0;
+    uint32_t max_blocked = 0;
+    char *reason_str = NULL;
+
+    priv = this->private;
+    args.type = CLRLK_ENTRY;
+    if (priv->revocation_clear_all == _gf_true)
+        args.kind = CLRLK_ALL;
+    else
+        args.kind = CLRLK_GRANTED;
+
+    if (list_empty(&dom->entrylk_list))
+        goto out;
+
+    pthread_mutex_lock(&pinode->mutex);
+    lock->pinode = pinode;
+    list_for_each_entry_safe(lk, tmp, &dom->entrylk_list, domain_list)
+    {
+        if (__stale_entrylk(this, lk, lock, &lk_age_sec) == _gf_true) {
+            revoke_lock = _gf_true;
+            reason_str = "age";
+            break;
+        }
+    }
+    max_blocked = priv->revocation_max_blocked;
+    if (max_blocked != 0 && revoke_lock == _gf_false) {
+        list_for_each_entry_safe(lk, tmp, &dom->blocked_entrylks, blocked_locks)
+        {
+            max_blocked--;
+            if (max_blocked == 0) {
+                revoke_lock = _gf_true;
+                reason_str = "max blocked";
+                break;
+            }
+        }
+    }
+    pthread_mutex_unlock(&pinode->mutex);
+
+out:
+    if (revoke_lock == _gf_true) {
+        clrlk_clear_entrylk(this, pinode, dom, &args, &bcount, &gcount,
+                            &op_errno);
+        gf_log(this->name, GF_LOG_WARNING,
+               "Lock revocation [reason: %s; gfid: %s; domain: %s; "
+               "age: %ld sec] - Entry lock revoked:  %d granted & %d "
+               "blocked locks cleared",
+               reason_str, uuid_utoa(pinode->gfid), dom->domain, lk_age_sec,
+               gcount, bcount);
+    }
+
+    return revoke_lock;
+}
+
+void
+entrylk_contention_notify_check(xlator_t *this, pl_entry_lock_t *lock,
+                                struct timespec *now, struct list_head *contend)
+{
+    posix_locks_private_t *priv;
+    int64_t elapsed;
+
+    priv = this->private;
+
+    /* If this lock is in a list, it means that we are about to send a
+     * notification for it, so no need to do anything else. */
+    if (!list_empty(&lock->contend)) {
+        return;
+    }
+
+    elapsed = now->tv_sec;
+    elapsed -= lock->contention_time.tv_sec;
+    if (now->tv_nsec < lock->contention_time.tv_nsec) {
+        elapsed--;
+    }
+    if (elapsed < priv->notify_contention_delay) {
+        return;
+    }
+
+    /* All contention notifications will be sent outside of the locked
+     * region. This means that currently granted locks might have already
+     * been unlocked by that time. To avoid the lock or the inode to be
+     * destroyed before we process them, we take an additional reference
+     * on both. */
+    inode_ref(lock->pinode->inode);
+    __pl_entrylk_ref(lock);
+
+    lock->contention_time = *now;
+
+    list_add_tail(&lock->contend, contend);
+}
+
+void
+entrylk_contention_notify(xlator_t *this, struct list_head *contend)
 {
-        if (names_conflict (l1->basename, l2->basename)
-            && !__same_entrylk_owner (l1, l2))
-                return 1;
+    struct gf_upcall up;
+    struct gf_upcall_entrylk_contention lc;
+    pl_entry_lock_t *lock;
+    pl_inode_t *pl_inode;
+    client_t *client;
+    gf_boolean_t notify;
+
+    while (!list_empty(contend)) {
+        lock = list_first_entry(contend, pl_entry_lock_t, contend);
+
+        pl_inode = lock->pinode;
+
+        pthread_mutex_lock(&pl_inode->mutex);
+
+        /* If the lock has already been released, no notification is
+         * sent. We clear the notification time in this case. */
+        notify = !list_empty(&lock->domain_list);
+        if (!notify) {
+            lock->contention_time.tv_sec = 0;
+            lock->contention_time.tv_nsec = 0;
+        } else {
+            lc.type = lock->type;
+            lc.name = lock->basename;
+            lc.pid = lock->client_pid;
+            lc.domain = lock->volume;
+            lc.xdata = NULL;
+
+            gf_uuid_copy(up.gfid, lock->pinode->gfid);
+            client = (client_t *)lock->client;
+            if (client == NULL) {
+                /* A NULL client can be found if the entrylk
+                 * was issued by a server side xlator. */
+                up.client_uid = NULL;
+            } else {
+                up.client_uid = client->client_uid;
+            }
+        }
+
+        pthread_mutex_unlock(&pl_inode->mutex);
+
+        if (notify) {
+            up.event_type = GF_UPCALL_ENTRYLK_CONTENTION;
+            up.data = &lc;
+
+            if (this->notify(this, GF_EVENT_UPCALL, &up) < 0) {
+                gf_msg_debug(this->name, 0,
+                             "Entrylk contention notification "
+                             "failed");
+            } else {
+                gf_msg_debug(this->name, 0,
+                             "Entrylk contention notification "
+                             "sent");
+            }
+        }
+
+        pthread_mutex_lock(&pl_inode->mutex);
+
+        list_del_init(&lock->contend);
+        __pl_entrylk_unref(lock);
 
-        return 0;
+        pthread_mutex_unlock(&pl_inode->mutex);
+
+        inode_unref(pl_inode->inode);
+    }
 }
 
 /**
@@ -123,184 +315,188 @@ __conflicting_entrylks (pl_entry_lock_t *l1, pl_entry_lock_t *l2)
  * @type: type of lock
  */
 static pl_entry_lock_t *
-__entrylk_grantable (pl_dom_list_t *dom, pl_entry_lock_t *lock)
+__entrylk_grantable(xlator_t *this, pl_dom_list_t *dom, pl_entry_lock_t *lock,
+                    struct timespec *now, struct list_head *contend)
 {
-        pl_entry_lock_t *tmp = NULL;
-
-        if (list_empty (&dom->entrylk_list))
-                return NULL;
-
-        list_for_each_entry (tmp, &dom->entrylk_list, domain_list) {
-                if (__conflicting_entrylks (tmp, lock))
-                        return tmp;
+    pl_entry_lock_t *tmp = NULL;
+    pl_entry_lock_t *ret = NULL;
+
+    list_for_each_entry(tmp, &dom->entrylk_list, domain_list)
+    {
+        if (__conflicting_entrylks(tmp, lock)) {
+            if (ret == NULL) {
+                ret = tmp;
+                if (contend == NULL) {
+                    break;
+                }
+            }
+            entrylk_contention_notify_check(this, tmp, now, contend);
         }
+    }
 
-        return NULL;
+    return ret;
 }
 
 static pl_entry_lock_t *
-__blocked_entrylk_conflict (pl_dom_list_t *dom, pl_entry_lock_t *lock)
+__blocked_entrylk_conflict(pl_dom_list_t *dom, pl_entry_lock_t *lock)
 {
-        pl_entry_lock_t *tmp = NULL;
+    pl_entry_lock_t *tmp = NULL;
 
-        if (list_empty (&dom->blocked_entrylks))
-                return NULL;
+    list_for_each_entry(tmp, &dom->blocked_entrylks, blocked_locks)
+    {
+        if (names_conflict(tmp->basename, lock->basename))
+            return lock;
+    }
 
-        list_for_each_entry (tmp, &dom->blocked_entrylks, blocked_locks) {
-                if (names_conflict (tmp->basename, lock->basename))
-                        return lock;
-        }
-
-        return NULL;
+    return NULL;
 }
 
 static int
-__owner_has_lock (pl_dom_list_t *dom, pl_entry_lock_t *newlock)
+__owner_has_lock(pl_dom_list_t *dom, pl_entry_lock_t *newlock)
 {
-        pl_entry_lock_t *lock = NULL;
+    pl_entry_lock_t *lock = NULL;
 
-        list_for_each_entry (lock, &dom->entrylk_list, domain_list) {
-                if (__same_entrylk_owner (lock, newlock))
-                        return 1;
-        }
+    list_for_each_entry(lock, &dom->entrylk_list, domain_list)
+    {
+        if (__same_entrylk_owner(lock, newlock))
+            return 1;
+    }
 
-        list_for_each_entry (lock, &dom->blocked_entrylks, blocked_locks) {
-                if (__same_entrylk_owner (lock, newlock))
-                        return 1;
-        }
+    list_for_each_entry(lock, &dom->blocked_entrylks, blocked_locks)
+    {
+        if (__same_entrylk_owner(lock, newlock))
+            return 1;
+    }
 
-        return 0;
+    return 0;
 }
 
 static int
-names_equal (const char *n1, const char *n2)
+names_equal(const char *n1, const char *n2)
 {
-        return (n1 == NULL && n2 == NULL) || (n1 && n2 && !strcmp (n1, n2));
+    return (n1 == NULL && n2 == NULL) || (n1 && n2 && !strcmp(n1, n2));
 }
 
 void
-pl_print_entrylk (char *str, int size, entrylk_cmd cmd, entrylk_type type,
-                  const char *basename, const char *domain)
+pl_print_entrylk(char *str, int size, entrylk_cmd cmd, entrylk_type type,
+                 const char *basename, const char *domain)
 {
-        char *cmd_str = NULL;
-        char *type_str = NULL;
+    char *cmd_str = NULL;
+    char *type_str = NULL;
 
-        switch (cmd) {
+    switch (cmd) {
         case ENTRYLK_LOCK:
-                cmd_str = "LOCK";
-                break;
+            cmd_str = "LOCK";
+            break;
 
         case ENTRYLK_LOCK_NB:
-                cmd_str = "LOCK_NB";
-                break;
+            cmd_str = "LOCK_NB";
+            break;
 
         case ENTRYLK_UNLOCK:
-                cmd_str = "UNLOCK";
-                break;
+            cmd_str = "UNLOCK";
+            break;
 
         default:
-                cmd_str = "UNKNOWN";
-                break;
-        }
+            cmd_str = "UNKNOWN";
+            break;
+    }
 
-        switch (type) {
+    switch (type) {
         case ENTRYLK_RDLCK:
-                type_str = "READ";
-                break;
+            type_str = "READ";
+            break;
         case ENTRYLK_WRLCK:
-                type_str = "WRITE";
-                break;
+            type_str = "WRITE";
+            break;
         default:
-                type_str = "UNKNOWN";
-                break;
-        }
+            type_str = "UNKNOWN";
+            break;
+    }
 
-        snprintf (str, size, "lock=ENTRYLK, cmd=%s, type=%s, basename=%s, domain: %s",
-                  cmd_str, type_str, basename, domain);
+    snprintf(str, size,
+             "lock=ENTRYLK, cmd=%s, type=%s, basename=%s, domain: %s", cmd_str,
+             type_str, basename, domain);
 }
 
-
 void
-entrylk_trace_in (xlator_t *this, call_frame_t *frame, const char *domain,
-                  fd_t *fd, loc_t *loc, const char *basename,
-                  entrylk_cmd cmd, entrylk_type type)
+entrylk_trace_in(xlator_t *this, call_frame_t *frame, const char *domain,
+                 fd_t *fd, loc_t *loc, const char *basename, entrylk_cmd cmd,
+                 entrylk_type type)
 {
-        posix_locks_private_t  *priv = NULL;
-        char                    pl_locker[256];
-        char                    pl_lockee[256];
-        char                    pl_entrylk[256];
+    posix_locks_private_t *priv = NULL;
+    char pl_locker[256];
+    char pl_lockee[256];
+    char pl_entrylk[256];
 
-        priv = this->private;
+    priv = this->private;
 
-        if (!priv->trace)
-                return;
+    if (!priv->trace)
+        return;
 
-        pl_print_locker (pl_locker, 256, this, frame);
-        pl_print_lockee (pl_lockee, 256, fd, loc);
-        pl_print_entrylk (pl_entrylk, 256, cmd, type, basename, domain);
+    pl_print_locker(pl_locker, 256, this, frame);
+    pl_print_lockee(pl_lockee, 256, fd, loc);
+    pl_print_entrylk(pl_entrylk, 256, cmd, type, basename, domain);
 
-        gf_log (this->name, GF_LOG_INFO,
-                "[REQUEST] Locker = {%s} Lockee = {%s} Lock = {%s}",
-                pl_locker, pl_lockee, pl_entrylk);
+    gf_log(this->name, GF_LOG_INFO,
+           "[REQUEST] Locker = {%s} Lockee = {%s} Lock = {%s}", pl_locker,
+           pl_lockee, pl_entrylk);
 }
 
-
 void
-entrylk_trace_out (xlator_t *this, call_frame_t *frame, const char *domain,
-                   fd_t *fd, loc_t *loc, const char *basename,
-                   entrylk_cmd cmd, entrylk_type type, int op_ret, int op_errno)
+entrylk_trace_out(xlator_t *this, call_frame_t *frame, const char *domain,
+                  fd_t *fd, loc_t *loc, const char *basename, entrylk_cmd cmd,
+                  entrylk_type type, int op_ret, int op_errno)
 {
-        posix_locks_private_t  *priv = NULL;
-        char                    pl_locker[256];
-        char                    pl_lockee[256];
-        char                    pl_entrylk[256];
-        char                    verdict[32];
+    posix_locks_private_t *priv = NULL;
+    char pl_locker[256];
+    char pl_lockee[256];
+    char pl_entrylk[256];
+    char verdict[32];
 
-        priv = this->private;
+    priv = this->private;
 
-        if (!priv->trace)
-                return;
+    if (!priv->trace)
+        return;
 
-        pl_print_locker (pl_locker, 256, this, frame);
-        pl_print_lockee (pl_lockee, 256, fd, loc);
-        pl_print_entrylk (pl_entrylk, 256, cmd, type, basename, domain);
-        pl_print_verdict (verdict, 32, op_ret, op_errno);
+    pl_print_locker(pl_locker, 256, this, frame);
+    pl_print_lockee(pl_lockee, 256, fd, loc);
+    pl_print_entrylk(pl_entrylk, 256, cmd, type, basename, domain);
+    pl_print_verdict(verdict, 32, op_ret, op_errno);
 
-        gf_log (this->name, GF_LOG_INFO,
-                "[%s] Locker = {%s} Lockee = {%s} Lock = {%s}",
-                verdict, pl_locker, pl_lockee, pl_entrylk);
+    gf_log(this->name, GF_LOG_INFO,
+           "[%s] Locker = {%s} Lockee = {%s} Lock = {%s}", verdict, pl_locker,
+           pl_lockee, pl_entrylk);
 }
 
-
 void
-entrylk_trace_block (xlator_t *this, call_frame_t *frame, const char *volume,
-                     fd_t *fd, loc_t *loc, const char *basename,
-                     entrylk_cmd cmd, entrylk_type type)
+entrylk_trace_block(xlator_t *this, call_frame_t *frame, const char *volume,
+                    fd_t *fd, loc_t *loc, const char *basename, entrylk_cmd cmd,
+                    entrylk_type type)
 
 {
-        posix_locks_private_t  *priv = NULL;
-        char                    pl_locker[256];
-        char                    pl_lockee[256];
-        char                    pl_entrylk[256];
+    posix_locks_private_t *priv = NULL;
+    char pl_locker[256];
+    char pl_lockee[256];
+    char pl_entrylk[256];
 
-        priv = this->private;
+    priv = this->private;
 
-        if (!priv->trace)
-                return;
+    if (!priv->trace)
+        return;
 
-        pl_print_locker (pl_locker, 256, this, frame);
-        pl_print_lockee (pl_lockee, 256, fd, loc);
-        pl_print_entrylk (pl_entrylk, 256, cmd, type, basename, volume);
+    pl_print_locker(pl_locker, 256, this, frame);
+    pl_print_lockee(pl_lockee, 256, fd, loc);
+    pl_print_entrylk(pl_entrylk, 256, cmd, type, basename, volume);
 
-        gf_log (this->name, GF_LOG_INFO,
-                "[BLOCKED] Locker = {%s} Lockee = {%s} Lock = {%s}",
-                pl_locker, pl_lockee, pl_entrylk);
+    gf_log(this->name, GF_LOG_INFO,
+           "[BLOCKED] Locker = {%s} Lockee = {%s} Lock = {%s}", pl_locker,
+           pl_lockee, pl_entrylk);
 }
 
 /**
- * __find_most_matching_lock - find the lock struct which most matches in order of:
- *                           lock on the exact basename ||
- *                           an all_names lock
+ * __find_most_matching_lock - find the lock struct which most matches in order
+ * of: lock on the exact basename || an all_names lock
  *
  *
  * @inode: inode in which to look
@@ -308,37 +504,57 @@ entrylk_trace_block (xlator_t *this, call_frame_t *frame, const char *volume,
  */
 
 static pl_entry_lock_t *
-__find_most_matching_lock (pl_dom_list_t *dom, const char *basename)
+__find_most_matching_lock(pl_dom_list_t *dom, const char *basename)
 {
-        pl_entry_lock_t *lock;
-        pl_entry_lock_t *all = NULL;
-        pl_entry_lock_t *exact = NULL;
+    pl_entry_lock_t *lock;
+    pl_entry_lock_t *all = NULL;
+    pl_entry_lock_t *exact = NULL;
 
-        if (list_empty (&dom->entrylk_list))
-                return NULL;
+    if (list_empty(&dom->entrylk_list))
+        return NULL;
 
-        list_for_each_entry (lock, &dom->entrylk_list, domain_list) {
-                if (all_names (lock->basename))
-                        all = lock;
-                else if (names_equal (lock->basename, basename))
-                        exact = lock;
-        }
+    list_for_each_entry(lock, &dom->entrylk_list, domain_list)
+    {
+        if (all_names(lock->basename))
+            all = lock;
+        else if (names_equal(lock->basename, basename))
+            exact = lock;
+    }
 
-        return (exact ? exact : all);
+    return (exact ? exact : all);
 }
 
-static pl_entry_lock_t*
-__find_matching_lock (pl_dom_list_t *dom, pl_entry_lock_t *lock)
+static pl_entry_lock_t *
+__find_matching_lock(pl_dom_list_t *dom, pl_entry_lock_t *lock)
 {
-        pl_entry_lock_t *tmp = NULL;
+    pl_entry_lock_t *tmp = NULL;
+
+    list_for_each_entry(tmp, &dom->entrylk_list, domain_list)
+    {
+        if (names_equal(lock->basename, tmp->basename) &&
+            __same_entrylk_owner(lock, tmp) && (lock->type == tmp->type))
+            return tmp;
+    }
+    return NULL;
+}
 
-        list_for_each_entry (tmp, &dom->entrylk_list, domain_list) {
-                if (names_equal (lock->basename, tmp->basename)
-                    && __same_entrylk_owner (lock, tmp)
-                    && (lock->type == tmp->type))
-                        return tmp;
-        }
-        return NULL;
+static int
+__lock_blocked_add(xlator_t *this, pl_inode_t *pinode, pl_dom_list_t *dom,
+                   pl_entry_lock_t *lock, int nonblock)
+{
+    if (nonblock)
+        goto out;
+
+    lock->blkd_time = gf_time();
+    list_add_tail(&lock->blocked_locks, &dom->blocked_entrylks);
+
+    gf_msg_trace(this->name, 0, "Blocking lock: {pinode=%p, basename=%s}",
+                 pinode, lock->basename);
+
+    entrylk_trace_block(this, lock->frame, NULL, NULL, NULL, lock->basename,
+                        ENTRYLK_LOCK, lock->type);
+out:
+    return -EAGAIN;
 }
 
 /**
@@ -353,63 +569,49 @@ __find_matching_lock (pl_dom_list_t *dom, pl_entry_lock_t *lock)
  */
 
 int
-__lock_entrylk (xlator_t *this, pl_inode_t *pinode, pl_entry_lock_t *lock,
-		int nonblock, pl_dom_list_t *dom)
+__lock_entrylk(xlator_t *this, pl_inode_t *pinode, pl_entry_lock_t *lock,
+               int nonblock, pl_dom_list_t *dom, struct timespec *now,
+               struct list_head *contend)
 {
-        pl_entry_lock_t *conf = NULL;
-        int              ret  = -EAGAIN;
-
-        conf = __entrylk_grantable (dom, lock);
-        if (conf) {
-                ret = -EAGAIN;
-                if (nonblock)
-                        goto out;
-
-                gettimeofday (&lock->blkd_time, NULL);
-                list_add_tail (&lock->blocked_locks, &dom->blocked_entrylks);
-
-                gf_log (this->name, GF_LOG_TRACE,
-                        "Blocking lock: {pinode=%p, basename=%s}",
-                        pinode, lock->basename);
-
-                goto out;
+    pl_entry_lock_t *conf = NULL;
+    int ret = -EAGAIN;
+
+    conf = __entrylk_grantable(this, dom, lock, now, contend);
+    if (conf) {
+        ret = __lock_blocked_add(this, pinode, dom, lock, nonblock);
+        goto out;
+    }
+
+    /* To prevent blocked locks starvation, check if there are any blocked
+     * locks thay may conflict with this lock. If there is then don't grant
+     * the lock. BUT grant the lock if the owner already has lock to allow
+     * nested locks.
+     * Example: SHD from Machine1 takes (gfid, basename=257-length-name)
+     * and is granted.
+     * SHD from machine2 takes (gfid, basename=NULL) and is blocked.
+     * When SHD from Machine1 takes (gfid, basename=NULL) it needs to be
+     * granted, without which self-heal can't progress.
+     * TODO: Find why 'owner_has_lock' is checked even for blocked locks.
+     */
+    if (__blocked_entrylk_conflict(dom, lock) &&
+        !(__owner_has_lock(dom, lock))) {
+        if (nonblock == 0) {
+            gf_log(this->name, GF_LOG_DEBUG,
+                   "Lock is grantable, but blocking to prevent "
+                   "starvation");
         }
 
-        /* To prevent blocked locks starvation, check if there are any blocked
-         * locks thay may conflict with this lock. If there is then don't grant
-         * the lock. BUT grant the lock if the owner already has lock to allow
-         * nested locks.
-         * Example: SHD from Machine1 takes (gfid, basename=257-length-name)
-         * and is granted.
-         * SHD from machine2 takes (gfid, basename=NULL) and is blocked.
-         * When SHD from Machine1 takes (gfid, basename=NULL) it needs to be
-         * granted, without which self-heal can't progress.
-         * TODO: Find why 'owner_has_lock' is checked even for blocked locks.
-         */
-        if (__blocked_entrylk_conflict (dom, lock) && !(__owner_has_lock (dom, lock))) {
-                ret = -EAGAIN;
-                if (nonblock)
-                        goto out;
-
-                gettimeofday (&lock->blkd_time, NULL);
-                list_add_tail (&lock->blocked_locks, &dom->blocked_entrylks);
-
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "Lock is grantable, but blocking to prevent starvation");
-                gf_log (this->name, GF_LOG_TRACE,
-                        "Blocking lock: {pinode=%p, basename=%s}",
-                        pinode, lock->basename);
-
-                goto out;
-        }
+        ret = __lock_blocked_add(this, pinode, dom, lock, nonblock);
+        goto out;
+    }
 
-        __pl_entrylk_ref (lock);
-        gettimeofday (&lock->granted_time, NULL);
-        list_add (&lock->domain_list, &dom->entrylk_list);
+    __pl_entrylk_ref(lock);
+    lock->granted_time = gf_time();
+    list_add(&lock->domain_list, &dom->entrylk_list);
 
-        ret = 0;
+    ret = 0;
 out:
-        return ret;
+    return ret;
 }
 
 /**
@@ -420,291 +622,322 @@ out:
  */
 
 pl_entry_lock_t *
-__unlock_entrylk (pl_dom_list_t *dom, pl_entry_lock_t *lock)
+__unlock_entrylk(pl_dom_list_t *dom, pl_entry_lock_t *lock)
 {
-        pl_entry_lock_t *ret_lock = NULL;
+    pl_entry_lock_t *ret_lock = NULL;
 
-        ret_lock = __find_matching_lock (dom, lock);
+    ret_lock = __find_matching_lock(dom, lock);
 
-        if (ret_lock) {
-                list_del_init (&ret_lock->domain_list);
-        } else {
-                gf_log ("locks", GF_LOG_ERROR, "unlock on %s "
-                        "(type=ENTRYLK_WRLCK) attempted but no matching lock "
-                        "found", lock->basename);
-        }
+    if (ret_lock) {
+        list_del_init(&ret_lock->domain_list);
+    } else {
+        gf_log("locks", GF_LOG_ERROR,
+               "unlock on %s "
+               "(type=ENTRYLK_WRLCK) attempted but no matching lock "
+               "found",
+               lock->basename);
+    }
 
-        return ret_lock;
+    return ret_lock;
 }
 
-uint32_t
-check_entrylk_on_basename (xlator_t *this, inode_t *parent, char *basename)
+int32_t
+check_entrylk_on_basename(xlator_t *this, inode_t *parent, char *basename)
 {
-        uint32_t        entrylk = 0;
-        pl_inode_t      *pinode = 0;
-        pl_dom_list_t   *dom = NULL;
-        pl_entry_lock_t *conf       = NULL;
-
-        pinode = pl_inode_get (this, parent);
-        if (!pinode)
-                goto out;
-        pthread_mutex_lock (&pinode->mutex);
+    int32_t entrylk = 0;
+    pl_dom_list_t *dom = NULL;
+    pl_entry_lock_t *conf = NULL;
+
+    pl_inode_t *pinode = pl_inode_get(this, parent, NULL);
+    if (!pinode)
+        goto out;
+    pthread_mutex_lock(&pinode->mutex);
+    {
+        list_for_each_entry(dom, &pinode->dom_list, inode_list)
         {
-                list_for_each_entry (dom, &pinode->dom_list, inode_list) {
-                        conf = __find_most_matching_lock (dom, basename);
-                        if (conf && conf->basename) {
-                                entrylk = 1;
-                                break;
-                        }
-                }
+            conf = __find_most_matching_lock(dom, basename);
+            if (conf && conf->basename) {
+                entrylk = 1;
+                break;
+            }
         }
-        pthread_mutex_unlock (&pinode->mutex);
+    }
+    pthread_mutex_unlock(&pinode->mutex);
 
 out:
-        return entrylk;
+    return entrylk;
 }
 
 void
-__grant_blocked_entry_locks (xlator_t *this, pl_inode_t *pl_inode,
-                             pl_dom_list_t *dom, struct list_head *granted)
+__grant_blocked_entry_locks(xlator_t *this, pl_inode_t *pl_inode,
+                            pl_dom_list_t *dom, struct list_head *granted,
+                            struct timespec *now, struct list_head *contend)
 {
-        int              bl_ret = 0;
-        pl_entry_lock_t *bl   = NULL;
-        pl_entry_lock_t *tmp  = NULL;
+    int bl_ret = 0;
+    pl_entry_lock_t *bl = NULL;
+    pl_entry_lock_t *tmp = NULL;
 
-        struct list_head blocked_list;
+    struct list_head blocked_list;
 
-        INIT_LIST_HEAD (&blocked_list);
-        list_splice_init (&dom->blocked_entrylks, &blocked_list);
+    INIT_LIST_HEAD(&blocked_list);
+    list_splice_init(&dom->blocked_entrylks, &blocked_list);
 
-        list_for_each_entry_safe (bl, tmp, &blocked_list, blocked_locks) {
+    list_for_each_entry_safe(bl, tmp, &blocked_list, blocked_locks)
+    {
+        list_del_init(&bl->blocked_locks);
 
-                list_del_init (&bl->blocked_locks);
+        bl_ret = __lock_entrylk(bl->this, pl_inode, bl, 0, dom, now, contend);
 
-                bl_ret = __lock_entrylk (bl->this, pl_inode, bl, 0, dom);
-
-                if (bl_ret == 0) {
-                        list_add (&bl->blocked_locks, granted);
-                }
+        if (bl_ret == 0) {
+            list_add_tail(&bl->blocked_locks, granted);
         }
-        return;
+    }
 }
 
 /* Grants locks if possible which are blocked on a lock */
 void
-grant_blocked_entry_locks (xlator_t *this, pl_inode_t *pl_inode,
-			   pl_dom_list_t *dom)
+grant_blocked_entry_locks(xlator_t *this, pl_inode_t *pl_inode,
+                          pl_dom_list_t *dom, struct timespec *now,
+                          struct list_head *contend)
 {
-        struct list_head  granted_list;
-        pl_entry_lock_t  *tmp = NULL;
-        pl_entry_lock_t  *lock = NULL;
-
-        INIT_LIST_HEAD (&granted_list);
-
-        pthread_mutex_lock (&pl_inode->mutex);
+    struct list_head granted_list;
+    pl_entry_lock_t *tmp = NULL;
+    pl_entry_lock_t *lock = NULL;
+
+    INIT_LIST_HEAD(&granted_list);
+
+    pthread_mutex_lock(&pl_inode->mutex);
+    {
+        __grant_blocked_entry_locks(this, pl_inode, dom, &granted_list, now,
+                                    contend);
+    }
+    pthread_mutex_unlock(&pl_inode->mutex);
+
+    list_for_each_entry_safe(lock, tmp, &granted_list, blocked_locks)
+    {
+        entrylk_trace_out(this, lock->frame, NULL, NULL, NULL, lock->basename,
+                          ENTRYLK_LOCK, lock->type, 0, 0);
+
+        STACK_UNWIND_STRICT(entrylk, lock->frame, 0, 0, NULL);
+        lock->frame = NULL;
+    }
+
+    pthread_mutex_lock(&pl_inode->mutex);
+    {
+        list_for_each_entry_safe(lock, tmp, &granted_list, blocked_locks)
         {
-                __grant_blocked_entry_locks (this, pl_inode, dom,
-                                             &granted_list);
+            list_del_init(&lock->blocked_locks);
+            __pl_entrylk_unref(lock);
         }
-        pthread_mutex_unlock (&pl_inode->mutex);
-
-        list_for_each_entry_safe (lock, tmp, &granted_list, blocked_locks) {
-                entrylk_trace_out (this, lock->frame, NULL, NULL, NULL,
-                                   lock->basename, ENTRYLK_LOCK, lock->type,
-                                   0, 0);
-
-                STACK_UNWIND_STRICT (entrylk, lock->frame, 0, 0, NULL);
-		lock->frame = NULL;
-	}
-
-        pthread_mutex_lock (&pl_inode->mutex);
-        {
-		list_for_each_entry_safe (lock, tmp, &granted_list, blocked_locks) {
-			list_del_init (&lock->blocked_locks);
-			__pl_entrylk_unref (lock);
-		}
-	}
-        pthread_mutex_unlock (&pl_inode->mutex);
-
-        return;
+    }
+    pthread_mutex_unlock(&pl_inode->mutex);
 }
 
-
 /* Common entrylk code called by pl_entrylk and pl_fentrylk */
 int
-pl_common_entrylk (call_frame_t *frame, xlator_t *this,
-                   const char *volume, inode_t *inode, const char *basename,
-                   entrylk_cmd cmd, entrylk_type type, loc_t *loc, fd_t *fd,
-                   dict_t *xdata)
-
-{
-        int32_t          op_ret           = -1;
-        int32_t          op_errno         =  0;
-        int              ret              = -1;
-        char             unwind           =  1;
-        GF_UNUSED int    dict_ret         = -1;
-        pl_inode_t      *pinode           =  NULL;
-        pl_entry_lock_t *reqlock          =  NULL;
-        pl_entry_lock_t *unlocked         =  NULL;
-        pl_dom_list_t   *dom              =  NULL;
-        char            *conn_id          =  NULL;
-        pl_ctx_t        *ctx              =  NULL;
-	int              nonblock         =  0;
-        gf_boolean_t     need_inode_unref =  _gf_false;
-
-        if (xdata)
-                dict_ret = dict_get_str (xdata, "connection-id", &conn_id);
-
-        pinode = pl_inode_get (this, inode);
-        if (!pinode) {
-                op_errno = ENOMEM;
-                goto out;
-        }
+pl_common_entrylk(call_frame_t *frame, xlator_t *this, const char *volume,
+                  inode_t *inode, const char *basename, entrylk_cmd cmd,
+                  entrylk_type type, loc_t *loc, fd_t *fd, dict_t *xdata)
 
-	if (frame->root->client) {
-		ctx = pl_ctx_get (frame->root->client, this);
-		if (!ctx) {
-			op_errno = ENOMEM;
-			gf_log (this->name, GF_LOG_INFO, "pl_ctx_get() failed");
-			goto unwind;
-		}
-	}
-
-        dom = get_domain (pinode, volume);
-        if (!dom){
-                op_errno = ENOMEM;
+{
+    int32_t op_ret = -1;
+    int32_t op_errno = 0;
+    int ret = -1;
+    char unwind = 1;
+    GF_UNUSED int dict_ret = -1;
+    pl_inode_t *pinode = NULL;
+    pl_entry_lock_t *reqlock = NULL;
+    pl_entry_lock_t *unlocked = NULL;
+    pl_dom_list_t *dom = NULL;
+    char *conn_id = NULL;
+    pl_ctx_t *ctx = NULL;
+    int nonblock = 0;
+    gf_boolean_t need_inode_unref = _gf_false;
+    posix_locks_private_t *priv = NULL;
+    struct list_head *pcontend = NULL;
+    struct list_head contend;
+    struct timespec now = {};
+
+    priv = this->private;
+
+    if (priv->notify_contention) {
+        pcontend = &contend;
+        INIT_LIST_HEAD(pcontend);
+        timespec_now(&now);
+    }
+
+    if (xdata)
+        dict_ret = dict_get_str(xdata, "connection-id", &conn_id);
+
+    pinode = pl_inode_get(this, inode, NULL);
+    if (!pinode) {
+        op_errno = ENOMEM;
+        goto out;
+    }
+
+    if (frame->root->client) {
+        ctx = pl_ctx_get(frame->root->client, this);
+        if (!ctx) {
+            op_errno = ENOMEM;
+            gf_log(this->name, GF_LOG_INFO, "pl_ctx_get() failed");
+            goto unwind;
+        }
+    }
+
+    dom = get_domain(pinode, volume);
+    if (!dom) {
+        op_errno = ENOMEM;
+        goto out;
+    }
+
+    entrylk_trace_in(this, frame, volume, fd, loc, basename, cmd, type);
+
+    reqlock = new_entrylk_lock(pinode, basename, type, dom->domain, frame,
+                               conn_id, &op_errno);
+    if (!reqlock) {
+        op_ret = -1;
+        goto unwind;
+    }
+
+    /* Ideally, AFTER a successful lock (both blocking and non-blocking) or
+     * an unsuccessful blocking lock operation, the inode needs to be ref'd.
+     *
+     * But doing so might give room to a race where the lock-requesting
+     * client could send a DISCONNECT just before this thread refs the inode
+     * after the locking is done, and the epoll thread could unref the inode
+     * in cleanup which means the inode's refcount would come down to 0, and
+     * the call to pl_forget() at this point destroys @pinode. Now when
+     * the io-thread executing this function tries to access pinode,
+     * it could crash on account of illegal memory access.
+     *
+     * To get around this problem, the inode is ref'd once even before
+     * adding the lock into client_list as a precautionary measure.
+     * This way even if there are DISCONNECTs, there will always be 1 extra
+     * ref on the inode, so @pinode is still alive until after the
+     * current stack unwinds.
+     */
+    pinode->inode = inode_ref(inode);
+    if (priv->revocation_secs != 0) {
+        if (cmd != ENTRYLK_UNLOCK) {
+            __entrylk_prune_stale(this, pinode, dom, reqlock);
+        } else if (priv->monkey_unlocking == _gf_true) {
+            if (pl_does_monkey_want_stuck_lock()) {
+                gf_log(this->name, GF_LOG_WARNING,
+                       "MONKEY LOCKING (forcing stuck lock)!");
+                op_ret = 0;
+                need_inode_unref = _gf_true;
+                pthread_mutex_lock(&pinode->mutex);
+                {
+                    __pl_entrylk_unref(reqlock);
+                }
+                pthread_mutex_unlock(&pinode->mutex);
                 goto out;
+            }
         }
+    }
 
-        entrylk_trace_in (this, frame, volume, fd, loc, basename, cmd, type);
+    switch (cmd) {
+        case ENTRYLK_LOCK_NB:
+            nonblock = 1;
+            /* fall through */
+        case ENTRYLK_LOCK:
+            if (ctx)
+                pthread_mutex_lock(&ctx->lock);
+            pthread_mutex_lock(&pinode->mutex);
+            {
+                reqlock->pinode = pinode;
+
+                ret = __lock_entrylk(this, pinode, reqlock, nonblock, dom, &now,
+                                     pcontend);
+                if (ret == 0) {
+                    reqlock->frame = NULL;
+                    op_ret = 0;
+                } else {
+                    op_errno = -ret;
+                }
 
-        reqlock = new_entrylk_lock (pinode, basename, type, dom->domain, frame,
-				    conn_id);
-        if (!reqlock) {
-                op_ret = -1;
-                op_errno = ENOMEM;
-                goto unwind;
-        }
+                if (ctx && (!ret || !nonblock))
+                    list_add(&reqlock->client_list, &ctx->entrylk_lockers);
 
-        /* Ideally, AFTER a successful lock (both blocking and non-blocking) or
-         * an unsuccessful blocking lock operation, the inode needs to be ref'd.
-         *
-         * But doing so might give room to a race where the lock-requesting
-         * client could send a DISCONNECT just before this thread refs the inode
-         * after the locking is done, and the epoll thread could unref the inode
-         * in cleanup which means the inode's refcount would come down to 0, and
-         * the call to pl_forget() at this point destroys @pinode. Now when
-         * the io-thread executing this function tries to access pinode,
-         * it could crash on account of illegal memory access.
-         *
-         * To get around this problem, the inode is ref'd once even before
-         * adding the lock into client_list as a precautionary measure.
-         * This way even if there are DISCONNECTs, there will always be 1 extra
-         * ref on the inode, so @pinode is still alive until after the
-         * current stack unwinds.
-         */
-        pinode->inode = inode_ref (inode);
-
-        switch (cmd) {
-        case ENTRYLK_LOCK_NB:
-		nonblock = 1;
-		/* fall through */
-	case ENTRYLK_LOCK:
-		if (ctx)
-			pthread_mutex_lock (&ctx->lock);
-                pthread_mutex_lock (&pinode->mutex);
-                {
-			reqlock->pinode = pinode;
-
-                        ret = __lock_entrylk (this, pinode, reqlock, nonblock, dom);
-			if (ret == 0) {
-				reqlock->frame = NULL;
-				op_ret = 0;
-			} else {
-				op_errno = -ret;
-			}
-
-			if (ctx && (!ret || !nonblock))
-				list_add (&reqlock->client_list,
-					  &ctx->entrylk_lockers);
-
-			if (ret == -EAGAIN && !nonblock) {
-				/* blocked */
-				unwind = 0;
-			} else {
-				__pl_entrylk_unref (reqlock);
-			}
-
-                        /* For all but the case where a non-blocking lock
-                         * attempt fails, the extra ref taken before the switch
-                         * block must be negated.
-                         */
-                        if ((ret == -EAGAIN) && (nonblock))
-                                need_inode_unref = _gf_true;
+                if (ret == -EAGAIN && !nonblock) {
+                    /* blocked */
+                    unwind = 0;
+                } else {
+                    __pl_entrylk_unref(reqlock);
                 }
-                pthread_mutex_unlock (&pinode->mutex);
-		if (ctx)
-			pthread_mutex_unlock (&ctx->lock);
-		break;
+
+                /* For all but the case where a non-blocking lock
+                 * attempt fails, the extra ref taken before the switch
+                 * block must be negated.
+                 */
+                if ((ret == -EAGAIN) && (nonblock))
+                    need_inode_unref = _gf_true;
+            }
+            pthread_mutex_unlock(&pinode->mutex);
+            if (ctx)
+                pthread_mutex_unlock(&ctx->lock);
+            break;
 
         case ENTRYLK_UNLOCK:
-		if (ctx)
-			pthread_mutex_lock (&ctx->lock);
-                pthread_mutex_lock (&pinode->mutex);
-                {
-                        /* Irrespective of whether unlock succeeds or not,
-                         * the extra inode ref that was done before the switch
-                         * block must be negated. Towards this,
-                         * @need_inode_unref flag is set unconditionally here.
-                         */
-                        need_inode_unref = _gf_true;
-                        unlocked = __unlock_entrylk (dom, reqlock);
-			if (unlocked) {
-				list_del_init (&unlocked->client_list);
-				__pl_entrylk_unref (unlocked);
-				op_ret = 0;
-			} else {
-				op_errno = EINVAL;
-			}
-			__pl_entrylk_unref (reqlock);
+            if (ctx)
+                pthread_mutex_lock(&ctx->lock);
+            pthread_mutex_lock(&pinode->mutex);
+            {
+                /* Irrespective of whether unlock succeeds or not,
+                 * the extra inode ref that was done before the switch
+                 * block must be negated. Towards this,
+                 * @need_inode_unref flag is set unconditionally here.
+                 */
+                need_inode_unref = _gf_true;
+                unlocked = __unlock_entrylk(dom, reqlock);
+                if (unlocked) {
+                    list_del_init(&unlocked->client_list);
+                    __pl_entrylk_unref(unlocked);
+                    op_ret = 0;
+                } else {
+                    op_errno = EINVAL;
                 }
-                pthread_mutex_unlock (&pinode->mutex);
-		if (ctx)
-			pthread_mutex_unlock (&ctx->lock);
+                __pl_entrylk_unref(reqlock);
+            }
+            pthread_mutex_unlock(&pinode->mutex);
+            if (ctx)
+                pthread_mutex_unlock(&ctx->lock);
 
-		grant_blocked_entry_locks (this, pinode, dom);
+            grant_blocked_entry_locks(this, pinode, dom, &now, pcontend);
 
-                break;
+            break;
 
         default:
-                inode_unref (pinode->inode);
-                gf_log (this->name, GF_LOG_ERROR,
-                        "Unexpected case in entrylk (cmd=%d). Please file"
-                        "a bug report at http://bugs.gluster.com", cmd);
-                goto out;
-        }
-        if (need_inode_unref)
-                inode_unref (pinode->inode);
-
-        /* The following (extra) unref corresponds to the ref that
-         * was done at the time the lock was granted.
-         */
-        if ((cmd == ENTRYLK_UNLOCK) && (op_ret == 0))
-                inode_unref (pinode->inode);
+            need_inode_unref = _gf_true;
+            gf_log(this->name, GF_LOG_ERROR,
+                   "Unexpected case in entrylk (cmd=%d). Please file"
+                   "a bug report at http://bugs.gluster.com",
+                   cmd);
+            goto out;
+    }
+    /* The following (extra) unref corresponds to the ref that
+     * was done at the time the lock was granted.
+     */
+    if ((cmd == ENTRYLK_UNLOCK) && (op_ret == 0))
+        inode_unref(pinode->inode);
 
 out:
 
-        if (unwind) {
-                entrylk_trace_out (this, frame, volume, fd, loc, basename,
-                                   cmd, type, op_ret, op_errno);
-unwind:
-                STACK_UNWIND_STRICT (entrylk, frame, op_ret, op_errno, NULL);
-        } else {
-                entrylk_trace_block (this, frame, volume, fd, loc, basename,
-                                     cmd, type);
-        }
+    if (need_inode_unref)
+        inode_unref(pinode->inode);
+
+    if (unwind) {
+        entrylk_trace_out(this, frame, volume, fd, loc, basename, cmd, type,
+                          op_ret, op_errno);
+    unwind:
+        STACK_UNWIND_STRICT(entrylk, frame, op_ret, op_errno, NULL);
+    }
 
-        return 0;
+    if (pcontend != NULL) {
+        entrylk_contention_notify(this, pcontend);
+    }
+
+    return 0;
 }
 
 /**
@@ -714,17 +947,16 @@ unwind:
  */
 
 int
-pl_entrylk (call_frame_t *frame, xlator_t *this,
-            const char *volume, loc_t *loc, const char *basename,
-            entrylk_cmd cmd, entrylk_type type, dict_t *xdata)
+pl_entrylk(call_frame_t *frame, xlator_t *this, const char *volume, loc_t *loc,
+           const char *basename, entrylk_cmd cmd, entrylk_type type,
+           dict_t *xdata)
 {
-        pl_common_entrylk (frame, this, volume, loc->inode, basename, cmd,
-                           type, loc, NULL, xdata);
+    pl_common_entrylk(frame, this, volume, loc->inode, basename, cmd, type, loc,
+                      NULL, xdata);
 
-        return 0;
+    return 0;
 }
 
-
 /**
  * pl_fentrylk:
  *
@@ -732,176 +964,190 @@ pl_entrylk (call_frame_t *frame, xlator_t *this,
  */
 
 int
-pl_fentrylk (call_frame_t *frame, xlator_t *this,
-             const char *volume, fd_t *fd, const char *basename,
-             entrylk_cmd cmd, entrylk_type type, dict_t *xdata)
+pl_fentrylk(call_frame_t *frame, xlator_t *this, const char *volume, fd_t *fd,
+            const char *basename, entrylk_cmd cmd, entrylk_type type,
+            dict_t *xdata)
 {
-        pl_common_entrylk (frame, this, volume, fd->inode, basename, cmd,
-                           type, NULL, fd, xdata);
+    pl_common_entrylk(frame, this, volume, fd->inode, basename, cmd, type, NULL,
+                      fd, xdata);
 
-        return 0;
+    return 0;
 }
 
-
 static void
-pl_entrylk_log_cleanup (pl_entry_lock_t *lock)
+pl_entrylk_log_cleanup(pl_entry_lock_t *lock)
 {
-	pl_inode_t *pinode = NULL;
+    pl_inode_t *pinode = NULL;
 
-	pinode = lock->pinode;
+    pinode = lock->pinode;
 
-        gf_log (THIS->name, GF_LOG_WARNING,
-                "releasing lock on %s held by "
-                "{client=%p, pid=%"PRId64" lk-owner=%s}",
-                uuid_utoa (pinode->gfid), lock->client,
-                (uint64_t) lock->client_pid, lkowner_utoa (&lock->owner));
+    gf_log(THIS->name, GF_LOG_WARNING,
+           "releasing lock on %s held by "
+           "{client=%p, pid=%" PRId64 " lk-owner=%s}",
+           uuid_utoa(pinode->gfid), lock->client, (uint64_t)lock->client_pid,
+           lkowner_utoa(&lock->owner));
 }
 
-
 /* Release all entrylks from this client */
 int
-pl_entrylk_client_cleanup (xlator_t *this, pl_ctx_t *ctx)
+pl_entrylk_client_cleanup(xlator_t *this, pl_ctx_t *ctx)
 {
-        pl_entry_lock_t *tmp = NULL;
-        pl_entry_lock_t *l = NULL;
-	pl_dom_list_t *dom = NULL;
-        pl_inode_t *pinode = NULL;
-
-        struct list_head released;
-        struct list_head unwind;
-
-        INIT_LIST_HEAD (&released);
-        INIT_LIST_HEAD (&unwind);
-
-	pthread_mutex_lock (&ctx->lock);
+    posix_locks_private_t *priv;
+    pl_entry_lock_t *tmp = NULL;
+    pl_entry_lock_t *l = NULL;
+    pl_dom_list_t *dom = NULL;
+    pl_inode_t *pinode = NULL;
+    struct list_head *pcontend = NULL;
+    struct list_head released;
+    struct list_head unwind;
+    struct list_head contend;
+    struct timespec now = {};
+
+    INIT_LIST_HEAD(&released);
+    INIT_LIST_HEAD(&unwind);
+
+    priv = this->private;
+    if (priv->notify_contention) {
+        pcontend = &contend;
+        INIT_LIST_HEAD(pcontend);
+        timespec_now(&now);
+    }
+
+    pthread_mutex_lock(&ctx->lock);
+    {
+        list_for_each_entry_safe(l, tmp, &ctx->entrylk_lockers, client_list)
         {
-                list_for_each_entry_safe (l, tmp, &ctx->entrylk_lockers,
-					  client_list) {
-                        list_del_init (&l->client_list);
-
-			pl_entrylk_log_cleanup (l);
-
-			pinode = l->pinode;
-
-			pthread_mutex_lock (&pinode->mutex);
-			{
-                        /* If the entrylk object is part of granted list but not
-                         * blocked list, then perform the following actions:
-                         * i.   delete the object from granted list;
-                         * ii.  grant other locks (from other clients) that may
-                         *      have been blocked on this entrylk; and
-                         * iii. unref the object.
-                         *
-                         * If the entrylk object (L1) is part of both granted
-                         * and blocked lists, then this means that a parallel
-                         * unlock on another entrylk (L2 say) may have 'granted'
-                         * L1 and added it to 'granted' list in
-                         * __grant_blocked_entry_locks() (although using the
-                         * 'blocked_locks' member). In that case, the cleanup
-                         * codepath must try and grant other overlapping
-                         * blocked entrylks from other clients, now that L1 is
-                         * out of their way and then unref L1 in the end, and
-                         * leave it to the other thread (the one executing
-                         * unlock codepath) to unwind L1's frame, delete it from
-                         * blocked_locks list, and perform the last unref on L1.
-                         *
-                         * If the entrylk object (L1) is part of blocked list
-                         * only, the cleanup code path must:
-                         * i.   delete it from the blocked_locks list inside
-                         *      this critical section,
-                         * ii.  unwind its frame with EAGAIN,
-                         * iii. try and grant blocked entry locks from other
-                         *      clients that were otherwise grantable, but were
-                         *      blocked to avoid leaving L1 to starve forever.
-                         * iv.  unref the object.
-                         */
-                                if (!list_empty (&l->domain_list)) {
-                                        list_del_init (&l->domain_list);
-                                        list_add_tail (&l->client_list,
-                                                       &released);
-                                } else {
-                                        list_del_init (&l->blocked_locks);
-                                        list_add_tail (&l->client_list,
-                                                       &unwind);
-                                }
-                        }
-			pthread_mutex_unlock (&pinode->mutex);
+            pl_entrylk_log_cleanup(l);
+
+            pinode = l->pinode;
+
+            pthread_mutex_lock(&pinode->mutex);
+            {
+                /* If the entrylk object is part of granted list but not
+                 * blocked list, then perform the following actions:
+                 * i.   delete the object from granted list;
+                 * ii.  grant other locks (from other clients) that may
+                 *      have been blocked on this entrylk; and
+                 * iii. unref the object.
+                 *
+                 * If the entrylk object (L1) is part of both granted
+                 * and blocked lists, then this means that a parallel
+                 * unlock on another entrylk (L2 say) may have 'granted'
+                 * L1 and added it to 'granted' list in
+                 * __grant_blocked_entry_locks() (although using the
+                 * 'blocked_locks' member). In that case, the cleanup
+                 * codepath must try and grant other overlapping
+                 * blocked entrylks from other clients, now that L1 is
+                 * out of their way and then unref L1 in the end, and
+                 * leave it to the other thread (the one executing
+                 * unlock codepath) to unwind L1's frame, delete it from
+                 * blocked_locks list, and perform the last unref on L1.
+                 *
+                 * If the entrylk object (L1) is part of blocked list
+                 * only, the cleanup code path must:
+                 * i.   delete it from the blocked_locks list inside
+                 *      this critical section,
+                 * ii.  unwind its frame with EAGAIN,
+                 * iii. try and grant blocked entry locks from other
+                 *      clients that were otherwise grantable, but were
+                 *      blocked to avoid leaving L1 to starve forever.
+                 * iv.  unref the object.
+                 */
+                list_del_init(&l->client_list);
+
+                if (!list_empty(&l->domain_list)) {
+                    list_del_init(&l->domain_list);
+                    list_add_tail(&l->client_list, &released);
+                } else {
+                    list_del_init(&l->blocked_locks);
+                    list_add_tail(&l->client_list, &unwind);
                 }
-	}
-        pthread_mutex_unlock (&ctx->lock);
+            }
+            pthread_mutex_unlock(&pinode->mutex);
+        }
+    }
+    pthread_mutex_unlock(&ctx->lock);
 
-        list_for_each_entry_safe (l, tmp, &unwind, client_list) {
-                list_del_init (&l->client_list);
+    if (!list_empty(&unwind)) {
+        list_for_each_entry_safe(l, tmp, &unwind, client_list)
+        {
+            list_del_init(&l->client_list);
 
-		if (l->frame)
-			STACK_UNWIND_STRICT (entrylk, l->frame, -1, EAGAIN,
-					     NULL);
-                list_add_tail (&l->client_list, &released);
+            if (l->frame)
+                STACK_UNWIND_STRICT(entrylk, l->frame, -1, EAGAIN, NULL);
+            list_add_tail(&l->client_list, &released);
         }
+    }
 
-        list_for_each_entry_safe (l, tmp, &released, client_list) {
-                list_del_init (&l->client_list);
+    if (!list_empty(&released)) {
+        list_for_each_entry_safe(l, tmp, &released, client_list)
+        {
+            list_del_init(&l->client_list);
 
-		pinode = l->pinode;
+            pinode = l->pinode;
 
-		dom = get_domain (pinode, l->volume);
+            dom = get_domain(pinode, l->volume);
 
-		grant_blocked_entry_locks (this, pinode, dom);
+            grant_blocked_entry_locks(this, pinode, dom, &now, pcontend);
 
-		pthread_mutex_lock (&pinode->mutex);
-		{
-			__pl_entrylk_unref (l);
-		}
-		pthread_mutex_unlock (&pinode->mutex);
-                inode_unref (pinode->inode);
+            pthread_mutex_lock(&pinode->mutex);
+            {
+                __pl_entrylk_unref(l);
+            }
+            pthread_mutex_unlock(&pinode->mutex);
+
+            inode_unref(pinode->inode);
         }
+    }
 
-        return 0;
-}
+    if (pcontend != NULL) {
+        entrylk_contention_notify(this, pcontend);
+    }
 
+    return 0;
+}
 
 int32_t
-__get_entrylk_count (xlator_t *this, pl_inode_t *pl_inode)
+__get_entrylk_count(xlator_t *this, pl_inode_t *pl_inode)
 {
-        int32_t            count = 0;
-        pl_entry_lock_t   *lock  = NULL;
-        pl_dom_list_t     *dom   = NULL;
+    int32_t count = 0;
+    pl_entry_lock_t *lock = NULL;
+    pl_dom_list_t *dom = NULL;
 
-        list_for_each_entry (dom, &pl_inode->dom_list, inode_list) {
-                list_for_each_entry (lock, &dom->entrylk_list, domain_list) {
-                        count++;
-                }
-
-                list_for_each_entry (lock, &dom->blocked_entrylks, blocked_locks) {
-                        count++;
-                }
+    list_for_each_entry(dom, &pl_inode->dom_list, inode_list)
+    {
+        list_for_each_entry(lock, &dom->entrylk_list, domain_list) { count++; }
 
+        list_for_each_entry(lock, &dom->blocked_entrylks, blocked_locks)
+        {
+            count++;
         }
+    }
 
-        return count;
+    return count;
 }
 
 int32_t
-get_entrylk_count (xlator_t *this, inode_t *inode)
+get_entrylk_count(xlator_t *this, inode_t *inode)
 {
-        pl_inode_t   *pl_inode = NULL;
-        uint64_t      tmp_pl_inode = 0;
-        int           ret      = 0;
-        int32_t       count    = 0;
+    pl_inode_t *pl_inode = NULL;
+    uint64_t tmp_pl_inode = 0;
+    int ret = 0;
+    int32_t count = 0;
 
-        ret = inode_ctx_get (inode, this, &tmp_pl_inode);
-        if (ret != 0) {
-                goto out;
-        }
+    ret = inode_ctx_get(inode, this, &tmp_pl_inode);
+    if (ret != 0) {
+        goto out;
+    }
 
-        pl_inode = (pl_inode_t *)(long) tmp_pl_inode;
+    pl_inode = (pl_inode_t *)(long)tmp_pl_inode;
 
-        pthread_mutex_lock (&pl_inode->mutex);
-        {
-                count = __get_entrylk_count (this, pl_inode);
-        }
-        pthread_mutex_unlock (&pl_inode->mutex);
+    pthread_mutex_lock(&pl_inode->mutex);
+    {
+        count = __get_entrylk_count(this, pl_inode);
+    }
+    pthread_mutex_unlock(&pl_inode->mutex);
 
 out:
-        return count;
+    return count;
 }
diff --git a/xlators/features/locks/src/inodelk.c b/xlators/features/locks/src/inodelk.c
index 9860e9f9079..d4e51d6e0a1 100644
--- a/xlators/features/locks/src/inodelk.c
+++ b/xlators/features/locks/src/inodelk.c
@@ -7,893 +7,1168 @@
    later), or the GNU General Public License, version 2 (GPLv2), in all
    cases as published by the Free Software Foundation.
 */
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
-
-#include "glusterfs.h"
-#include "compat.h"
-#include "xlator.h"
-#include "inode.h"
-#include "logging.h"
-#include "common-utils.h"
-#include "list.h"
+#include <glusterfs/glusterfs.h>
+#include <glusterfs/compat.h>
+#include <glusterfs/dict.h>
+#include <glusterfs/logging.h>
+#include <glusterfs/list.h>
+#include <glusterfs/upcall-utils.h>
 
 #include "locks.h"
+#include "clear.h"
 #include "common.h"
 
-inline void
-__delete_inode_lock (pl_inode_lock_t *lock)
+void
+__delete_inode_lock(pl_inode_lock_t *lock)
 {
-        list_del_init (&lock->list);
+    list_del_init(&lock->list);
 }
 
-static inline void
-__pl_inodelk_ref (pl_inode_lock_t *lock)
+static void
+__pl_inodelk_ref(pl_inode_lock_t *lock)
 {
-        lock->ref++;
+    lock->ref++;
 }
 
-inline void
-__pl_inodelk_unref (pl_inode_lock_t *lock)
+void
+__pl_inodelk_unref(pl_inode_lock_t *lock)
 {
-        lock->ref--;
-        if (!lock->ref) {
-                GF_FREE (lock->connection_id);
-                GF_FREE (lock);
-        }
+    lock->ref--;
+    if (!lock->ref) {
+        GF_FREE(lock->connection_id);
+        GF_FREE(lock);
+    }
 }
 
-/* Check if 2 inodelks are conflicting on type. Only 2 shared locks don't conflict */
-static inline int
-inodelk_type_conflict (pl_inode_lock_t *l1, pl_inode_lock_t *l2)
+/* Check if 2 inodelks are conflicting on type. Only 2 shared locks don't
+ * conflict */
+static int
+inodelk_type_conflict(pl_inode_lock_t *l1, pl_inode_lock_t *l2)
 {
-        if (l2->fl_type == F_WRLCK || l1->fl_type == F_WRLCK)
-                return 1;
+    if (l2->fl_type == F_WRLCK || l1->fl_type == F_WRLCK)
+        return 1;
 
-        return 0;
+    return 0;
 }
 
 void
-pl_print_inodelk (char *str, int size, int cmd, struct gf_flock *flock, const char *domain)
+pl_print_inodelk(char *str, int size, int cmd, struct gf_flock *flock,
+                 const char *domain)
 {
-        char *cmd_str = NULL;
-        char *type_str = NULL;
+    char *cmd_str = NULL;
+    char *type_str = NULL;
 
-        switch (cmd) {
+    switch (cmd) {
 #if F_GETLK != F_GETLK64
         case F_GETLK64:
 #endif
         case F_GETLK:
-                cmd_str = "GETLK";
-                break;
+            cmd_str = "GETLK";
+            break;
 
 #if F_SETLK != F_SETLK64
         case F_SETLK64:
 #endif
         case F_SETLK:
-                cmd_str = "SETLK";
-                break;
+            cmd_str = "SETLK";
+            break;
 
 #if F_SETLKW != F_SETLKW64
         case F_SETLKW64:
 #endif
         case F_SETLKW:
-                cmd_str = "SETLKW";
-                break;
+            cmd_str = "SETLKW";
+            break;
 
         default:
-                cmd_str = "UNKNOWN";
-                break;
-        }
+            cmd_str = "UNKNOWN";
+            break;
+    }
 
-        switch (flock->l_type) {
+    switch (flock->l_type) {
         case F_RDLCK:
-                type_str = "READ";
-                break;
+            type_str = "READ";
+            break;
         case F_WRLCK:
-                type_str = "WRITE";
-                break;
+            type_str = "WRITE";
+            break;
         case F_UNLCK:
-                type_str = "UNLOCK";
-                break;
+            type_str = "UNLOCK";
+            break;
         default:
-                type_str = "UNKNOWN";
-                break;
-        }
-
-        snprintf (str, size, "lock=INODELK, cmd=%s, type=%s, "
-                  "domain: %s, start=%llu, len=%llu, pid=%llu",
-                  cmd_str, type_str, domain,
-                  (unsigned long long) flock->l_start,
-                  (unsigned long long) flock->l_len,
-                  (unsigned long long) flock->l_pid);
+            type_str = "UNKNOWN";
+            break;
+    }
+
+    snprintf(str, size,
+             "lock=INODELK, cmd=%s, type=%s, "
+             "domain: %s, start=%llu, len=%llu, pid=%llu",
+             cmd_str, type_str, domain, (unsigned long long)flock->l_start,
+             (unsigned long long)flock->l_len,
+             (unsigned long long)flock->l_pid);
 }
 
 /* Determine if the two inodelks overlap reach other's lock regions */
 static int
-inodelk_overlap (pl_inode_lock_t *l1, pl_inode_lock_t *l2)
+inodelk_overlap(pl_inode_lock_t *l1, pl_inode_lock_t *l2)
 {
-        return ((l1->fl_end >= l2->fl_start) &&
-                (l2->fl_end >= l1->fl_start));
+    return ((l1->fl_end >= l2->fl_start) && (l2->fl_end >= l1->fl_start));
 }
 
 /* Returns true if the 2 inodelks have the same owner */
-static inline int
-same_inodelk_owner (pl_inode_lock_t *l1, pl_inode_lock_t *l2)
+static int
+same_inodelk_owner(pl_inode_lock_t *l1, pl_inode_lock_t *l2)
 {
-        return (is_same_lkowner (&l1->owner, &l2->owner) &&
-                (l1->client == l2->client));
+    return (is_same_lkowner(&l1->owner, &l2->owner) &&
+            (l1->client == l2->client));
 }
 
 /* Returns true if the 2 inodelks conflict with each other */
 static int
-inodelk_conflict (pl_inode_lock_t *l1, pl_inode_lock_t *l2)
+inodelk_conflict(pl_inode_lock_t *l1, pl_inode_lock_t *l2)
 {
-        return (inodelk_overlap (l1, l2) &&
-                inodelk_type_conflict (l1, l2));
+    return (inodelk_overlap(l1, l2) && inodelk_type_conflict(l1, l2));
 }
 
-/* Determine if lock is grantable or not */
-static pl_inode_lock_t *
-__inodelk_grantable (pl_dom_list_t *dom, pl_inode_lock_t *lock)
+/*
+ * Check to see if the candidate lock overlaps/conflicts with the
+ * requested lock.  If so, determine how old the lock is and return
+ * true if it exceeds the configured threshold, false otherwise.
+ */
+static inline gf_boolean_t
+__stale_inodelk(xlator_t *this, pl_inode_lock_t *candidate_lock,
+                pl_inode_lock_t *requested_lock, time_t *lock_age_sec)
 {
-        pl_inode_lock_t *l = NULL;
-        pl_inode_lock_t *ret = NULL;
-        if (list_empty (&dom->inodelk_list))
-                goto out;
-        list_for_each_entry (l, &dom->inodelk_list, list){
-                if (inodelk_conflict (lock, l) &&
-                    !same_inodelk_owner (lock, l)) {
-                        ret = l;
-                        goto out;
-                }
-        }
-out:
-        return ret;
+    posix_locks_private_t *priv = NULL;
+
+    priv = this->private;
+    /* Question: Should we just prune them all given the
+     * chance?  Or just the locks we are attempting to acquire?
+     */
+    if (inodelk_conflict(candidate_lock, requested_lock)) {
+        *lock_age_sec = gf_time() - candidate_lock->granted_time;
+        if (*lock_age_sec > priv->revocation_secs)
+            return _gf_true;
+    }
+    return _gf_false;
 }
 
-static pl_inode_lock_t *
-__blocked_lock_conflict (pl_dom_list_t *dom, pl_inode_lock_t *lock)
+/* Examine any locks held on this inode and potentially revoke the lock
+ * if the age exceeds revocation_secs.  We will clear _only_ those locks
+ * which are granted, and then grant those locks which are blocked.
+ *
+ * Depending on how this patch works in the wild, we may expand this and
+ * introduce a heuristic which clears blocked locks as well if they
+ * are beyond a threshold.
+ */
+static gf_boolean_t
+__inodelk_prune_stale(xlator_t *this, pl_inode_t *pinode, pl_dom_list_t *dom,
+                      pl_inode_lock_t *lock)
 {
-        pl_inode_lock_t *l   = NULL;
-        pl_inode_lock_t *ret = NULL;
-
-        if (list_empty (&dom->blocked_inodelks))
-                return NULL;
+    posix_locks_private_t *priv = NULL;
+    pl_inode_lock_t *tmp = NULL;
+    pl_inode_lock_t *lk = NULL;
+    gf_boolean_t revoke_lock = _gf_false;
+    int bcount = 0;
+    int gcount = 0;
+    int op_errno = 0;
+    clrlk_args args;
+    args.opts = NULL;
+    time_t lk_age_sec = 0;
+    uint32_t max_blocked = 0;
+    char *reason_str = NULL;
+
+    priv = this->private;
+
+    args.type = CLRLK_INODE;
+    if (priv->revocation_clear_all == _gf_true)
+        args.kind = CLRLK_ALL;
+    else
+        args.kind = CLRLK_GRANTED;
+
+    if (list_empty(&dom->inodelk_list))
+        goto out;
+
+    pthread_mutex_lock(&pinode->mutex);
+    list_for_each_entry_safe(lk, tmp, &dom->inodelk_list, list)
+    {
+        if (__stale_inodelk(this, lk, lock, &lk_age_sec) == _gf_true) {
+            revoke_lock = _gf_true;
+            reason_str = "age";
+            break;
+        }
+    }
 
-        list_for_each_entry (l, &dom->blocked_inodelks, blocked_locks) {
-                if (inodelk_conflict (lock, l)) {
-                        ret = l;
-                        goto out;
-                }
+    max_blocked = priv->revocation_max_blocked;
+    if (max_blocked != 0 && revoke_lock == _gf_false) {
+        list_for_each_entry_safe(lk, tmp, &dom->blocked_inodelks, blocked_locks)
+        {
+            max_blocked--;
+            if (max_blocked == 0) {
+                revoke_lock = _gf_true;
+                reason_str = "max blocked";
+                break;
+            }
         }
+    }
+    pthread_mutex_unlock(&pinode->mutex);
 
 out:
-        return ret;
+    if (revoke_lock == _gf_true) {
+        clrlk_clear_inodelk(this, pinode, dom, &args, &bcount, &gcount,
+                            &op_errno);
+        gf_log(this->name, GF_LOG_WARNING,
+               "Lock revocation [reason: %s; gfid: %s; domain: %s; "
+               "age: %ld sec] - Inode lock revoked:  %d granted & %d "
+               "blocked locks cleared",
+               reason_str, uuid_utoa(pinode->gfid), dom->domain, lk_age_sec,
+               gcount, bcount);
+    }
+    return revoke_lock;
 }
 
-static int
-__owner_has_lock (pl_dom_list_t *dom, pl_inode_lock_t *newlock)
+void
+inodelk_contention_notify_check(xlator_t *this, pl_inode_lock_t *lock,
+                                struct timespec *now, struct list_head *contend)
 {
-        pl_inode_lock_t *lock = NULL;
+    posix_locks_private_t *priv;
+    int64_t elapsed;
 
-        list_for_each_entry (lock, &dom->inodelk_list, list) {
-                if (same_inodelk_owner (lock, newlock))
-                        return 1;
-        }
+    priv = this->private;
 
-        list_for_each_entry (lock, &dom->blocked_inodelks, blocked_locks) {
-                if (same_inodelk_owner (lock, newlock))
-                        return 1;
-        }
+    /* If this lock is in a list, it means that we are about to send a
+     * notification for it, so no need to do anything else. */
+    if (!list_empty(&lock->contend)) {
+        return;
+    }
+
+    elapsed = now->tv_sec;
+    elapsed -= lock->contention_time.tv_sec;
+    if (now->tv_nsec < lock->contention_time.tv_nsec) {
+        elapsed--;
+    }
+    if (elapsed < priv->notify_contention_delay) {
+        return;
+    }
 
-        return 0;
-}
+    /* All contention notifications will be sent outside of the locked
+     * region. This means that currently granted locks might have already
+     * been unlocked by that time. To avoid the lock or the inode to be
+     * destroyed before we process them, we take an additional reference
+     * on both. */
+    inode_ref(lock->pl_inode->inode);
+    __pl_inodelk_ref(lock);
 
+    lock->contention_time = *now;
 
-/* Determines if lock can be granted and adds the lock. If the lock
- * is blocking, adds it to the blocked_inodelks list of the domain.
- */
-static int
-__lock_inodelk (xlator_t *this, pl_inode_t *pl_inode, pl_inode_lock_t *lock,
-                int can_block,  pl_dom_list_t *dom)
+    list_add_tail(&lock->contend, contend);
+}
+
+void
+inodelk_contention_notify(xlator_t *this, struct list_head *contend)
 {
-        pl_inode_lock_t *conf = NULL;
-        int ret = -EINVAL;
+    struct gf_upcall up;
+    struct gf_upcall_inodelk_contention lc;
+    pl_inode_lock_t *lock;
+    pl_inode_t *pl_inode;
+    client_t *client;
+    gf_boolean_t notify;
+
+    while (!list_empty(contend)) {
+        lock = list_first_entry(contend, pl_inode_lock_t, contend);
+
+        pl_inode = lock->pl_inode;
+
+        pthread_mutex_lock(&pl_inode->mutex);
+
+        /* If the lock has already been released, no notification is
+         * sent. We clear the notification time in this case. */
+        notify = !list_empty(&lock->list);
+        if (!notify) {
+            lock->contention_time.tv_sec = 0;
+            lock->contention_time.tv_nsec = 0;
+        } else {
+            memcpy(&lc.flock, &lock->user_flock, sizeof(lc.flock));
+            lc.pid = lock->client_pid;
+            lc.domain = lock->volume;
+            lc.xdata = NULL;
+
+            gf_uuid_copy(up.gfid, lock->pl_inode->gfid);
+            client = (client_t *)lock->client;
+            if (client == NULL) {
+                /* A NULL client can be found if the inodelk
+                 * was issued by a server side xlator. */
+                up.client_uid = NULL;
+            } else {
+                up.client_uid = client->client_uid;
+            }
+        }
 
-        conf = __inodelk_grantable (dom, lock);
-        if (conf) {
-                ret = -EAGAIN;
-                if (can_block == 0)
-                        goto out;
+        pthread_mutex_unlock(&pl_inode->mutex);
+
+        if (notify) {
+            up.event_type = GF_UPCALL_INODELK_CONTENTION;
+            up.data = &lc;
+
+            if (this->notify(this, GF_EVENT_UPCALL, &up) < 0) {
+                gf_msg_debug(this->name, 0,
+                             "Inodelk contention notification "
+                             "failed");
+            } else {
+                gf_msg_debug(this->name, 0,
+                             "Inodelk contention notification "
+                             "sent");
+            }
+        }
 
-                gettimeofday (&lock->blkd_time, NULL);
-                list_add_tail (&lock->blocked_locks, &dom->blocked_inodelks);
+        pthread_mutex_lock(&pl_inode->mutex);
 
-                gf_log (this->name, GF_LOG_TRACE,
-                        "%s (pid=%d) lk-owner:%s %"PRId64" - %"PRId64" => Blocked",
-                        lock->fl_type == F_UNLCK ? "Unlock" : "Lock",
-                        lock->client_pid,
-                        lkowner_utoa (&lock->owner),
-                        lock->user_flock.l_start,
-                        lock->user_flock.l_len);
+        list_del_init(&lock->contend);
+        __pl_inodelk_unref(lock);
 
+        pthread_mutex_unlock(&pl_inode->mutex);
 
-                goto out;
+        inode_unref(pl_inode->inode);
+    }
+}
+
+/* Determine if lock is grantable or not */
+static pl_inode_lock_t *
+__inodelk_grantable(xlator_t *this, pl_dom_list_t *dom, pl_inode_lock_t *lock,
+                    struct timespec *now, struct list_head *contend)
+{
+    pl_inode_lock_t *l = NULL;
+    pl_inode_lock_t *ret = NULL;
+
+    list_for_each_entry(l, &dom->inodelk_list, list)
+    {
+        if (inodelk_conflict(lock, l) && !same_inodelk_owner(lock, l)) {
+            if (ret == NULL) {
+                ret = l;
+                if (contend == NULL) {
+                    break;
+                }
+            }
+            inodelk_contention_notify_check(this, l, now, contend);
         }
+    }
 
-        /* To prevent blocked locks starvation, check if there are any blocked
-         * locks thay may conflict with this lock. If there is then don't grant
-         * the lock. BUT grant the lock if the owner already has lock to allow
-         * nested locks.
-         * Example:
-         * SHD from Machine1 takes (gfid, 0-infinity) and is granted.
-         * SHD from machine2 takes (gfid, 0-infinity) and is blocked.
-         * When SHD from Machine1 takes (gfid, 0-128KB) it
-         * needs to be granted, without which the earlier lock on 0-infinity
-         * will not be unlocked by SHD from Machine1.
-         * TODO: Find why 'owner_has_lock' is checked even for blocked locks.
-         */
-        if (__blocked_lock_conflict (dom, lock) && !(__owner_has_lock (dom, lock))) {
-                ret = -EAGAIN;
-                if (can_block == 0)
-                        goto out;
+    return ret;
+}
 
-                gettimeofday (&lock->blkd_time, NULL);
-                list_add_tail (&lock->blocked_locks, &dom->blocked_inodelks);
+static pl_inode_lock_t *
+__blocked_lock_conflict(pl_dom_list_t *dom, pl_inode_lock_t *lock)
+{
+    pl_inode_lock_t *l = NULL;
 
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "Lock is grantable, but blocking to prevent starvation");
-                gf_log (this->name, GF_LOG_TRACE,
-                        "%s (pid=%d) (lk-owner=%s) %"PRId64" - %"PRId64" => Blocked",
-                        lock->fl_type == F_UNLCK ? "Unlock" : "Lock",
-                        lock->client_pid,
-                        lkowner_utoa (&lock->owner),
-                        lock->user_flock.l_start,
-                        lock->user_flock.l_len);
+    list_for_each_entry(l, &dom->blocked_inodelks, blocked_locks)
+    {
+        if (inodelk_conflict(lock, l)) {
+            return l;
+        }
+    }
 
+    return NULL;
+}
 
-                goto out;
-        }
-        __pl_inodelk_ref (lock);
-        gettimeofday (&lock->granted_time, NULL);
-        list_add (&lock->list, &dom->inodelk_list);
+static int
+__owner_has_lock(pl_dom_list_t *dom, pl_inode_lock_t *newlock)
+{
+    pl_inode_lock_t *lock = NULL;
 
-        ret = 0;
+    list_for_each_entry(lock, &dom->inodelk_list, list)
+    {
+        if (same_inodelk_owner(lock, newlock))
+            return 1;
+    }
+
+    list_for_each_entry(lock, &dom->blocked_inodelks, blocked_locks)
+    {
+        if (same_inodelk_owner(lock, newlock))
+            return 1;
+    }
+
+    return 0;
+}
 
+static int
+__lock_blocked_add(xlator_t *this, pl_dom_list_t *dom, pl_inode_lock_t *lock,
+                   int can_block)
+{
+    if (can_block == 0) {
+        goto out;
+    }
+
+    lock->blkd_time = gf_time();
+    list_add_tail(&lock->blocked_locks, &dom->blocked_inodelks);
+
+    gf_msg_trace(this->name, 0,
+                 "%s (pid=%d) (lk-owner=%s) %" PRId64
+                 " - "
+                 "%" PRId64 " => Blocked",
+                 lock->fl_type == F_UNLCK ? "Unlock" : "Lock", lock->client_pid,
+                 lkowner_utoa(&lock->owner), lock->user_flock.l_start,
+                 lock->user_flock.l_len);
+
+    pl_trace_block(this, lock->frame, NULL, NULL, F_SETLKW, &lock->user_flock,
+                   lock->volume);
 out:
+    return -EAGAIN;
+}
+
+/* Determines if lock can be granted and adds the lock. If the lock
+ * is blocking, adds it to the blocked_inodelks list of the domain.
+ */
+static int
+__lock_inodelk(xlator_t *this, pl_inode_t *pl_inode, pl_inode_lock_t *lock,
+               int can_block, pl_dom_list_t *dom, struct timespec *now,
+               struct list_head *contend)
+{
+    pl_inode_lock_t *conf = NULL;
+    int ret;
+
+    ret = pl_inode_remove_inodelk(pl_inode, lock);
+    if (ret < 0) {
         return ret;
+    }
+    if (ret == 0) {
+        conf = __inodelk_grantable(this, dom, lock, now, contend);
+    }
+    if ((ret > 0) || (conf != NULL)) {
+        return __lock_blocked_add(this, dom, lock, can_block);
+    }
+
+    /* To prevent blocked locks starvation, check if there are any blocked
+     * locks thay may conflict with this lock. If there is then don't grant
+     * the lock. BUT grant the lock if the owner already has lock to allow
+     * nested locks.
+     * Example:
+     * SHD from Machine1 takes (gfid, 0-infinity) and is granted.
+     * SHD from machine2 takes (gfid, 0-infinity) and is blocked.
+     * When SHD from Machine1 takes (gfid, 0-128KB) it
+     * needs to be granted, without which the earlier lock on 0-infinity
+     * will not be unlocked by SHD from Machine1.
+     * TODO: Find why 'owner_has_lock' is checked even for blocked locks.
+     */
+    if (__blocked_lock_conflict(dom, lock) && !(__owner_has_lock(dom, lock))) {
+        if (can_block != 0) {
+            gf_log(this->name, GF_LOG_DEBUG,
+                   "Lock is grantable, but blocking to prevent "
+                   "starvation");
+        }
+
+        return __lock_blocked_add(this, dom, lock, can_block);
+    }
+    __pl_inodelk_ref(lock);
+    lock->granted_time = gf_time();
+    list_add(&lock->list, &dom->inodelk_list);
+
+    return 0;
 }
 
 /* Return true if the two inodelks have exactly same lock boundaries */
 static int
-inodelks_equal (pl_inode_lock_t *l1, pl_inode_lock_t *l2)
+inodelks_equal(pl_inode_lock_t *l1, pl_inode_lock_t *l2)
 {
-        if ((l1->fl_start == l2->fl_start) &&
-            (l1->fl_end == l2->fl_end))
-                return 1;
+    if ((l1->fl_start == l2->fl_start) && (l1->fl_end == l2->fl_end))
+        return 1;
 
-        return 0;
+    return 0;
 }
 
-
 static pl_inode_lock_t *
-find_matching_inodelk (pl_inode_lock_t *lock, pl_dom_list_t *dom)
+find_matching_inodelk(pl_inode_lock_t *lock, pl_dom_list_t *dom)
 {
-        pl_inode_lock_t *l = NULL;
-        list_for_each_entry (l, &dom->inodelk_list, list) {
-                if (inodelks_equal (l, lock) &&
-                    same_inodelk_owner (l, lock))
-                        return l;
-        }
-        return NULL;
+    pl_inode_lock_t *l = NULL;
+    list_for_each_entry(l, &dom->inodelk_list, list)
+    {
+        if (inodelks_equal(l, lock) && same_inodelk_owner(l, lock))
+            return l;
+    }
+    return NULL;
 }
 
 /* Set F_UNLCK removes a lock which has the exact same lock boundaries
  * as the UNLCK lock specifies. If such a lock is not found, returns invalid
  */
 static pl_inode_lock_t *
-__inode_unlock_lock (xlator_t *this, pl_inode_lock_t *lock, pl_dom_list_t *dom)
+__inode_unlock_lock(xlator_t *this, pl_inode_lock_t *lock, pl_dom_list_t *dom)
 {
-
-        pl_inode_lock_t *conf = NULL;
-
-        conf = find_matching_inodelk (lock, dom);
-        if (!conf) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        " Matching lock not found for unlock %llu-%llu, by %s "
-                        "on %p", (unsigned long long)lock->fl_start,
-                        (unsigned long long)lock->fl_end,
-                        lkowner_utoa (&lock->owner), lock->client);
-                goto out;
-        }
-        __delete_inode_lock (conf);
-        gf_log (this->name, GF_LOG_DEBUG,
-                " Matching lock found for unlock %llu-%llu, by %s on %p",
-                (unsigned long long)lock->fl_start,
-                (unsigned long long)lock->fl_end, lkowner_utoa (&lock->owner),
-                lock->client);
+    pl_inode_lock_t *conf = NULL;
+    inode_t *inode = NULL;
+
+    inode = lock->pl_inode->inode;
+
+    conf = find_matching_inodelk(lock, dom);
+    if (!conf) {
+        gf_log(this->name, GF_LOG_ERROR,
+               " Matching lock not found for unlock %llu-%llu, by %s "
+               "on %p for gfid:%s",
+               (unsigned long long)lock->fl_start,
+               (unsigned long long)lock->fl_end, lkowner_utoa(&lock->owner),
+               lock->client, inode ? uuid_utoa(inode->gfid) : "UNKNOWN");
+        goto out;
+    }
+    __delete_inode_lock(conf);
+    gf_log(this->name, GF_LOG_DEBUG,
+           " Matching lock found for unlock %llu-%llu, by %s on %p for gfid:%s",
+           (unsigned long long)lock->fl_start, (unsigned long long)lock->fl_end,
+           lkowner_utoa(&lock->owner), lock->client,
+           inode ? uuid_utoa(inode->gfid) : "UNKNOWN");
 
 out:
-        return conf;
+    return conf;
 }
 
-
-static void
-__grant_blocked_inode_locks (xlator_t *this, pl_inode_t *pl_inode,
-                             struct list_head *granted, pl_dom_list_t *dom)
+void
+__grant_blocked_inode_locks(xlator_t *this, pl_inode_t *pl_inode,
+                            struct list_head *granted, pl_dom_list_t *dom,
+                            struct timespec *now, struct list_head *contend)
 {
-        int              bl_ret = 0;
-        pl_inode_lock_t *bl = NULL;
-        pl_inode_lock_t *tmp = NULL;
+    pl_inode_lock_t *bl = NULL;
+    pl_inode_lock_t *tmp = NULL;
 
-        struct list_head blocked_list;
+    struct list_head blocked_list;
 
-        INIT_LIST_HEAD (&blocked_list);
-        list_splice_init (&dom->blocked_inodelks, &blocked_list);
+    INIT_LIST_HEAD(&blocked_list);
+    list_splice_init(&dom->blocked_inodelks, &blocked_list);
 
-        list_for_each_entry_safe (bl, tmp, &blocked_list, blocked_locks) {
+    list_for_each_entry_safe(bl, tmp, &blocked_list, blocked_locks)
+    {
+        list_del_init(&bl->blocked_locks);
 
-                list_del_init (&bl->blocked_locks);
+        bl->status = __lock_inodelk(this, pl_inode, bl, 1, dom, now, contend);
 
-                bl_ret = __lock_inodelk (this, pl_inode, bl, 1, dom);
-
-                if (bl_ret == 0) {
-                        list_add (&bl->blocked_locks, granted);
-                }
+        if (bl->status != -EAGAIN) {
+            list_add_tail(&bl->blocked_locks, granted);
         }
-        return;
+    }
 }
 
-/* Grant all inodelks blocked on a lock */
 void
-grant_blocked_inode_locks (xlator_t *this, pl_inode_t *pl_inode,
-                           pl_dom_list_t *dom)
+unwind_granted_inodes(xlator_t *this, pl_inode_t *pl_inode,
+                      struct list_head *granted)
 {
-        struct list_head granted;
-        pl_inode_lock_t *lock;
-        pl_inode_lock_t *tmp;
-
-        INIT_LIST_HEAD (&granted);
-
-        pthread_mutex_lock (&pl_inode->mutex);
-        {
-                __grant_blocked_inode_locks (this, pl_inode, &granted, dom);
-        }
-        pthread_mutex_unlock (&pl_inode->mutex);
-
-        list_for_each_entry_safe (lock, tmp, &granted, blocked_locks) {
-                gf_log (this->name, GF_LOG_TRACE,
-                        "%s (pid=%d) (lk-owner=%s) %"PRId64" - %"PRId64" => Granted",
-                        lock->fl_type == F_UNLCK ? "Unlock" : "Lock",
-                        lock->client_pid,
-                        lkowner_utoa (&lock->owner),
-                        lock->user_flock.l_start,
-                        lock->user_flock.l_len);
-
-                pl_trace_out (this, lock->frame, NULL, NULL, F_SETLKW,
-                              &lock->user_flock, 0, 0, lock->volume);
-
-                STACK_UNWIND_STRICT (inodelk, lock->frame, 0, 0, NULL);
-		lock->frame = NULL;
+    pl_inode_lock_t *lock;
+    pl_inode_lock_t *tmp;
+    int32_t op_ret;
+    int32_t op_errno;
+
+    list_for_each_entry_safe(lock, tmp, granted, blocked_locks)
+    {
+        if (lock->status == 0) {
+            op_ret = 0;
+            op_errno = 0;
+            gf_log(this->name, GF_LOG_TRACE,
+                   "%s (pid=%d) (lk-owner=%s) %" PRId64 " - %" PRId64
+                   " => Granted",
+                   lock->fl_type == F_UNLCK ? "Unlock" : "Lock",
+                   lock->client_pid, lkowner_utoa(&lock->owner),
+                   lock->user_flock.l_start, lock->user_flock.l_len);
+        } else {
+            op_ret = -1;
+            op_errno = -lock->status;
         }
+        pl_trace_out(this, lock->frame, NULL, NULL, F_SETLKW, &lock->user_flock,
+                     op_ret, op_errno, lock->volume);
+
+        STACK_UNWIND_STRICT(inodelk, lock->frame, op_ret, op_errno, NULL);
+        lock->frame = NULL;
+    }
 
-        pthread_mutex_lock (&pl_inode->mutex);
+    pthread_mutex_lock(&pl_inode->mutex);
+    {
+        list_for_each_entry_safe(lock, tmp, granted, blocked_locks)
         {
-                list_for_each_entry_safe (lock, tmp, &granted, blocked_locks) {
-                        list_del_init (&lock->blocked_locks);
-                        __pl_inodelk_unref (lock);
-                }
+            list_del_init(&lock->blocked_locks);
+            __pl_inodelk_unref(lock);
         }
-        pthread_mutex_unlock (&pl_inode->mutex);
+    }
+    pthread_mutex_unlock(&pl_inode->mutex);
 }
 
+/* Grant all inodelks blocked on a lock */
+void
+grant_blocked_inode_locks(xlator_t *this, pl_inode_t *pl_inode,
+                          pl_dom_list_t *dom, struct timespec *now,
+                          struct list_head *contend)
+{
+    struct list_head granted;
+
+    INIT_LIST_HEAD(&granted);
+
+    pthread_mutex_lock(&pl_inode->mutex);
+    {
+        __grant_blocked_inode_locks(this, pl_inode, &granted, dom, now,
+                                    contend);
+    }
+    pthread_mutex_unlock(&pl_inode->mutex);
+
+    unwind_granted_inodes(this, pl_inode, &granted);
+}
 
 static void
-pl_inodelk_log_cleanup (pl_inode_lock_t *lock)
+pl_inodelk_log_cleanup(pl_inode_lock_t *lock)
 {
-	pl_inode_t *pl_inode = NULL;
+    pl_inode_t *pl_inode = NULL;
 
-	pl_inode = lock->pl_inode;
+    pl_inode = lock->pl_inode;
 
-        gf_log (THIS->name, GF_LOG_WARNING, "releasing lock on %s held by "
-                "{client=%p, pid=%"PRId64" lk-owner=%s}",
-                uuid_utoa (pl_inode->gfid), lock->client,
-                (uint64_t) lock->client_pid, lkowner_utoa (&lock->owner));
+    gf_log(THIS->name, GF_LOG_WARNING,
+           "releasing lock on %s held by "
+           "{client=%p, pid=%" PRId64 " lk-owner=%s}",
+           uuid_utoa(pl_inode->gfid), lock->client, (uint64_t)lock->client_pid,
+           lkowner_utoa(&lock->owner));
 }
 
-
 /* Release all inodelks from this client */
 int
-pl_inodelk_client_cleanup (xlator_t *this, pl_ctx_t *ctx)
+pl_inodelk_client_cleanup(xlator_t *this, pl_ctx_t *ctx)
 {
-        pl_inode_lock_t *tmp = NULL;
-        pl_inode_lock_t *l = NULL;
-	pl_dom_list_t *dom = NULL;
-        pl_inode_t *pl_inode = NULL;
-
-        struct list_head released;
-        struct list_head unwind;
-
-        INIT_LIST_HEAD (&released);
-        INIT_LIST_HEAD (&unwind);
-
-	pthread_mutex_lock (&ctx->lock);
+    posix_locks_private_t *priv;
+    pl_inode_lock_t *tmp = NULL;
+    pl_inode_lock_t *l = NULL;
+    pl_dom_list_t *dom = NULL;
+    pl_inode_t *pl_inode = NULL;
+    struct list_head *pcontend = NULL;
+    struct list_head released;
+    struct list_head unwind;
+    struct list_head contend;
+    struct timespec now = {};
+
+    priv = this->private;
+
+    INIT_LIST_HEAD(&released);
+    INIT_LIST_HEAD(&unwind);
+
+    if (priv->notify_contention) {
+        pcontend = &contend;
+        INIT_LIST_HEAD(pcontend);
+        timespec_now(&now);
+    }
+
+    pthread_mutex_lock(&ctx->lock);
+    {
+        list_for_each_entry_safe(l, tmp, &ctx->inodelk_lockers, client_list)
         {
-                list_for_each_entry_safe (l, tmp, &ctx->inodelk_lockers,
-					  client_list) {
-                        list_del_init (&l->client_list);
-
-			pl_inodelk_log_cleanup (l);
-
-			pl_inode = l->pl_inode;
-
-			pthread_mutex_lock (&pl_inode->mutex);
-			{
-                        /* If the inodelk object is part of granted list but not
-                         * blocked list, then perform the following actions:
-                         * i.   delete the object from granted list;
-                         * ii.  grant other locks (from other clients) that may
-                         *      have been blocked on this inodelk; and
-                         * iii. unref the object.
-                         *
-                         * If the inodelk object (L1) is part of both granted
-                         * and blocked lists, then this means that a parallel
-                         * unlock on another inodelk (L2 say) may have 'granted'
-                         * L1 and added it to 'granted' list in
-                         * __grant_blocked_node_locks() (although using the
-                         * 'blocked_locks' member). In that case, the cleanup
-                         * codepath must try and grant other overlapping
-                         * blocked inodelks from other clients, now that L1 is
-                         * out of their way and then unref L1 in the end, and
-                         * leave it to the other thread (the one executing
-                         * unlock codepath) to unwind L1's frame, delete it from
-                         * blocked_locks list, and perform the last unref on L1.
-                         *
-                         * If the inodelk object (L1) is part of blocked list
-                         * only, the cleanup code path must:
-                         * i.   delete it from the blocked_locks list inside
-                         *      this critical section,
-                         * ii.  unwind its frame with EAGAIN,
-                         * iii. try and grant blocked inode locks from other
-                         *      clients that were otherwise grantable, but just
-                         *      got blocked to avoid leaving L1 to starve
-                         *      forever.
-                         * iv.  unref the object.
-                         */
-                                if (!list_empty (&l->list)) {
-                                        __delete_inode_lock (l);
-                                        list_add_tail (&l->client_list,
-                                                       &released);
-                                } else {
-                                        list_del_init(&l->blocked_locks);
-                                        list_add_tail (&l->client_list,
-                                                       &unwind);
-                                }
-                        }
-			pthread_mutex_unlock (&pl_inode->mutex);
+            pl_inodelk_log_cleanup(l);
+
+            pl_inode = l->pl_inode;
+
+            pthread_mutex_lock(&pl_inode->mutex);
+            {
+                /* If the inodelk object is part of granted list but not
+                 * blocked list, then perform the following actions:
+                 * i.   delete the object from granted list;
+                 * ii.  grant other locks (from other clients) that may
+                 *      have been blocked on this inodelk; and
+                 * iii. unref the object.
+                 *
+                 * If the inodelk object (L1) is part of both granted
+                 * and blocked lists, then this means that a parallel
+                 * unlock on another inodelk (L2 say) may have 'granted'
+                 * L1 and added it to 'granted' list in
+                 * __grant_blocked_inode_locks() (although using the
+                 * 'blocked_locks' member). In that case, the cleanup
+                 * codepath must try and grant other overlapping
+                 * blocked inodelks from other clients, now that L1 is
+                 * out of their way and then unref L1 in the end, and
+                 * leave it to the other thread (the one executing
+                 * unlock codepath) to unwind L1's frame, delete it from
+                 * blocked_locks list, and perform the last unref on L1.
+                 *
+                 * If the inodelk object (L1) is part of blocked list
+                 * only, the cleanup code path must:
+                 * i.   delete it from the blocked_locks list inside
+                 *      this critical section,
+                 * ii.  unwind its frame with EAGAIN,
+                 * iii. try and grant blocked inode locks from other
+                 *      clients that were otherwise grantable, but just
+                 *      got blocked to avoid leaving L1 to starve
+                 *      forever.
+                 * iv.  unref the object.
+                 */
+                list_del_init(&l->client_list);
+
+                if (!list_empty(&l->list)) {
+                    __delete_inode_lock(l);
+                    list_add_tail(&l->client_list, &released);
+                } else {
+                    list_del_init(&l->blocked_locks);
+                    list_add_tail(&l->client_list, &unwind);
                 }
-	}
-        pthread_mutex_unlock (&ctx->lock);
-
-        list_for_each_entry_safe (l, tmp, &unwind, client_list) {
-                list_del_init (&l->client_list);
+            }
+            pthread_mutex_unlock(&pl_inode->mutex);
+        }
+    }
+    pthread_mutex_unlock(&ctx->lock);
 
-                if (l->frame)
-			STACK_UNWIND_STRICT (inodelk, l->frame, -1, EAGAIN,
-					     NULL);
-                list_add_tail (&l->client_list, &released);
+    if (!list_empty(&unwind)) {
+        list_for_each_entry_safe(l, tmp, &unwind, client_list)
+        {
+            list_del_init(&l->client_list);
 
+            if (l->frame)
+                STACK_UNWIND_STRICT(inodelk, l->frame, -1, EAGAIN, NULL);
+            list_add_tail(&l->client_list, &released);
         }
+    }
 
-        list_for_each_entry_safe (l, tmp, &released, client_list) {
-                list_del_init (&l->client_list);
+    if (!list_empty(&released)) {
+        list_for_each_entry_safe(l, tmp, &released, client_list)
+        {
+            list_del_init(&l->client_list);
 
-		pl_inode = l->pl_inode;
+            pl_inode = l->pl_inode;
 
-		dom = get_domain (pl_inode, l->volume);
+            dom = get_domain(pl_inode, l->volume);
 
-		grant_blocked_inode_locks (this, pl_inode, dom);
+            grant_blocked_inode_locks(this, pl_inode, dom, &now, pcontend);
 
-		pthread_mutex_lock (&pl_inode->mutex);
-		{
-			__pl_inodelk_unref (l);
-		}
-		pthread_mutex_unlock (&pl_inode->mutex);
-                inode_unref (pl_inode->inode);
+            pthread_mutex_lock(&pl_inode->mutex);
+            {
+                __pl_inodelk_unref(l);
+            }
+            pthread_mutex_unlock(&pl_inode->mutex);
+            inode_unref(pl_inode->inode);
         }
+    }
 
-        return 0;
-}
+    if (pcontend != NULL) {
+        inodelk_contention_notify(this, pcontend);
+    }
 
+    return 0;
+}
 
 static int
-pl_inode_setlk (xlator_t *this, pl_ctx_t *ctx, pl_inode_t *pl_inode,
-		pl_inode_lock_t *lock, int can_block, pl_dom_list_t *dom,
-                inode_t *inode)
-{
-        int               ret              = -EINVAL;
-        pl_inode_lock_t  *retlock          =  NULL;
-        gf_boolean_t      unref            =  _gf_true;
-        gf_boolean_t      need_inode_unref =  _gf_false;
-        short             fl_type;
-
-	lock->pl_inode = pl_inode;
-        fl_type = lock->fl_type;
-
-        /* Ideally, AFTER a successful lock (both blocking and non-blocking) or
-         * an unsuccessful blocking lock operation, the inode needs to be ref'd.
-         *
-         * But doing so might give room to a race where the lock-requesting
-         * client could send a DISCONNECT just before this thread refs the inode
-         * after the locking is done, and the epoll thread could unref the inode
-         * in cleanup which means the inode's refcount would come down to 0, and
-         * the call to pl_forget() at this point destroys @pl_inode. Now when
-         * the io-thread executing this function tries to access pl_inode,
-         * it could crash on account of illegal memory access.
-         *
-         * To get around this problem, the inode is ref'd once even before
-         * adding the lock into client_list as a precautionary measure.
-         * This way even if there are DISCONNECTs, there will always be 1 extra
-         * ref on the inode, so @pl_inode is still alive until after the
-         * current stack unwinds.
-         */
-        pl_inode->inode = inode_ref (inode);
-
-	if (ctx)
-		pthread_mutex_lock (&ctx->lock);
-        pthread_mutex_lock (&pl_inode->mutex);
-        {
-                if (lock->fl_type != F_UNLCK) {
-                        ret = __lock_inodelk (this, pl_inode, lock, can_block, dom);
-                        if (ret == 0) {
-				lock->frame = NULL;
-                                gf_log (this->name, GF_LOG_TRACE,
-                                        "%s (pid=%d) (lk-owner=%s) %"PRId64" - %"PRId64" => OK",
-                                        lock->fl_type == F_UNLCK ? "Unlock" : "Lock",
-                                        lock->client_pid,
-                                        lkowner_utoa (&lock->owner),
-                                        lock->fl_start,
-                                        lock->fl_end);
-                        } else if (ret == -EAGAIN) {
-                                gf_log (this->name, GF_LOG_TRACE,
-                                        "%s (pid=%d) (lk-owner=%s) %"PRId64" - %"PRId64" => NOK",
-                                        lock->fl_type == F_UNLCK ? "Unlock" : "Lock",
-                                        lock->client_pid,
-                                        lkowner_utoa (&lock->owner),
-                                        lock->user_flock.l_start,
-                                        lock->user_flock.l_len);
-                                if (can_block)
-                                        unref = _gf_false;
-                                /* For all but the case where a non-blocking
-                                 * lock attempt fails, the extra ref taken at
-                                 * the start of this function must be negated.
-                                 */
-                                else
-                                        need_inode_unref = _gf_true;
-                        }
-
-			if (ctx && (!ret || can_block))
-				list_add_tail (&lock->client_list,
-					       &ctx->inodelk_lockers);
-                } else {
-                        /* Irrespective of whether unlock succeeds or not,
-                         * the extra inode ref that was done at the start of
-                         * this function must be negated. Towards this,
-                         * @need_inode_unref flag is set unconditionally here.
-                         */
-                        need_inode_unref = _gf_true;
-                        retlock = __inode_unlock_lock (this, lock, dom);
-                        if (!retlock) {
-                                gf_log (this->name, GF_LOG_DEBUG,
-                                        "Bad Unlock issued on Inode lock");
-                                ret = -EINVAL;
-                                goto out;
-                        }
-			list_del_init (&retlock->client_list);
-			__pl_inodelk_unref (retlock);
-
-                        ret = 0;
+pl_inode_setlk(xlator_t *this, pl_ctx_t *ctx, pl_inode_t *pl_inode,
+               pl_inode_lock_t *lock, int can_block, pl_dom_list_t *dom,
+               inode_t *inode)
+{
+    posix_locks_private_t *priv = NULL;
+    int ret = -EINVAL;
+    pl_inode_lock_t *retlock = NULL;
+    gf_boolean_t unref = _gf_true;
+    gf_boolean_t need_inode_unref = _gf_false;
+    struct list_head *pcontend = NULL;
+    struct list_head contend;
+    struct list_head wake;
+    struct timespec now = {};
+    short fl_type;
+
+    lock->pl_inode = pl_inode;
+    fl_type = lock->fl_type;
+
+    priv = this->private;
+
+    /* Ideally, AFTER a successful lock (both blocking and non-blocking) or
+     * an unsuccessful blocking lock operation, the inode needs to be ref'd.
+     *
+     * But doing so might give room to a race where the lock-requesting
+     * client could send a DISCONNECT just before this thread refs the inode
+     * after the locking is done, and the epoll thread could unref the inode
+     * in cleanup which means the inode's refcount would come down to 0, and
+     * the call to pl_forget() at this point destroys @pl_inode. Now when
+     * the io-thread executing this function tries to access pl_inode,
+     * it could crash on account of illegal memory access.
+     *
+     * To get around this problem, the inode is ref'd once even before
+     * adding the lock into client_list as a precautionary measure.
+     * This way even if there are DISCONNECTs, there will always be 1 extra
+     * ref on the inode, so @pl_inode is still alive until after the
+     * current stack unwinds.
+     */
+    pl_inode->inode = inode_ref(inode);
+
+    if (priv->revocation_secs != 0) {
+        if (lock->fl_type != F_UNLCK) {
+            __inodelk_prune_stale(this, pl_inode, dom, lock);
+        } else if (priv->monkey_unlocking == _gf_true) {
+            if (pl_does_monkey_want_stuck_lock()) {
+                pthread_mutex_lock(&pl_inode->mutex);
+                {
+                    __pl_inodelk_unref(lock);
                 }
-out:
-		if (unref)
-			__pl_inodelk_unref (lock);
-        }
-        pthread_mutex_unlock (&pl_inode->mutex);
-	if (ctx)
-		pthread_mutex_unlock (&ctx->lock);
-
-        if (need_inode_unref)
-                inode_unref (pl_inode->inode);
-
-        /* The following (extra) unref corresponds to the ref that
-         * was done at the time the lock was granted.
-         */
-        if ((fl_type == F_UNLCK) && (ret == 0)) {
-                inode_unref (pl_inode->inode);
-                grant_blocked_inode_locks (this, pl_inode, dom);
+                pthread_mutex_unlock(&pl_inode->mutex);
+                inode_unref(pl_inode->inode);
+                gf_log(this->name, GF_LOG_WARNING,
+                       "MONKEY LOCKING (forcing stuck lock)!");
+                return 0;
+            }
         }
+    }
+
+    if (priv->notify_contention) {
+        pcontend = &contend;
+        INIT_LIST_HEAD(pcontend);
+        timespec_now(&now);
+    }
+
+    INIT_LIST_HEAD(&wake);
+
+    if (ctx)
+        pthread_mutex_lock(&ctx->lock);
+    pthread_mutex_lock(&pl_inode->mutex);
+    {
+        if (lock->fl_type != F_UNLCK) {
+            ret = __lock_inodelk(this, pl_inode, lock, can_block, dom, &now,
+                                 pcontend);
+            if (ret == 0) {
+                lock->frame = NULL;
+                gf_log(this->name, GF_LOG_TRACE,
+                       "%s (pid=%d) (lk-owner=%s) %" PRId64 " - %" PRId64
+                       " => OK",
+                       lock->fl_type == F_UNLCK ? "Unlock" : "Lock",
+                       lock->client_pid, lkowner_utoa(&lock->owner),
+                       lock->fl_start, lock->fl_end);
+            } else if (ret == -EAGAIN) {
+                gf_log(this->name, GF_LOG_TRACE,
+                       "%s (pid=%d) (lk-owner=%s) %" PRId64 " - %" PRId64
+                       " => NOK",
+                       lock->fl_type == F_UNLCK ? "Unlock" : "Lock",
+                       lock->client_pid, lkowner_utoa(&lock->owner),
+                       lock->user_flock.l_start, lock->user_flock.l_len);
+                if (can_block) {
+                    unref = _gf_false;
+                }
+            }
+            /* For all but the case where a non-blocking lock attempt fails
+             * with -EAGAIN, the extra ref taken at the start of this function
+             * must be negated. */
+            need_inode_unref = (ret != 0) && ((ret != -EAGAIN) || !can_block);
+            if (ctx && !need_inode_unref) {
+                list_add_tail(&lock->client_list, &ctx->inodelk_lockers);
+            }
+        } else {
+            /* Irrespective of whether unlock succeeds or not,
+             * the extra inode ref that was done at the start of
+             * this function must be negated. Towards this,
+             * @need_inode_unref flag is set unconditionally here.
+             */
+            need_inode_unref = _gf_true;
+            retlock = __inode_unlock_lock(this, lock, dom);
+            if (!retlock) {
+                gf_log(this->name, GF_LOG_DEBUG,
+                       "Bad Unlock issued on Inode lock");
+                ret = -EINVAL;
+                goto out;
+            }
+            list_del_init(&retlock->client_list);
+            __pl_inodelk_unref(retlock);
 
-        return ret;
+            pl_inode_remove_unlocked(this, pl_inode, &wake);
+
+            ret = 0;
+        }
+    out:
+        if (unref)
+            __pl_inodelk_unref(lock);
+    }
+    pthread_mutex_unlock(&pl_inode->mutex);
+    if (ctx)
+        pthread_mutex_unlock(&ctx->lock);
+
+    pl_inode_remove_wake(&wake);
+
+    /* The following (extra) unref corresponds to the ref that
+     * was done at the time the lock was granted.
+     */
+    if ((fl_type == F_UNLCK) && (ret == 0)) {
+        inode_unref(pl_inode->inode);
+        grant_blocked_inode_locks(this, pl_inode, dom, &now, pcontend);
+    }
+
+    if (need_inode_unref) {
+        inode_unref(pl_inode->inode);
+    }
+
+    if (pcontend != NULL) {
+        inodelk_contention_notify(this, pcontend);
+    }
+
+    return ret;
 }
 
 /* Create a new inode_lock_t */
-pl_inode_lock_t *
-new_inode_lock (struct gf_flock *flock, client_t *client, pid_t client_pid,
-                call_frame_t *frame, xlator_t *this, const char *volume,
-                char *conn_id)
+static pl_inode_lock_t *
+new_inode_lock(struct gf_flock *flock, client_t *client, pid_t client_pid,
+               call_frame_t *frame, xlator_t *this, const char *volume,
+               char *conn_id, int32_t *op_errno)
 
 {
-        pl_inode_lock_t *lock = NULL;
+    pl_inode_lock_t *lock = NULL;
+
+    if (!pl_is_lk_owner_valid(&frame->root->lk_owner, frame->root->client)) {
+        *op_errno = EINVAL;
+        goto out;
+    }
+
+    lock = GF_CALLOC(1, sizeof(*lock), gf_locks_mt_pl_inode_lock_t);
+    if (!lock) {
+        *op_errno = ENOMEM;
+        goto out;
+    }
+
+    lock->fl_start = flock->l_start;
+    lock->fl_type = flock->l_type;
+
+    if (flock->l_len == 0)
+        lock->fl_end = LLONG_MAX;
+    else
+        lock->fl_end = flock->l_start + flock->l_len - 1;
+
+    lock->client = client;
+    lock->client_pid = client_pid;
+    lock->volume = volume;
+    lock->owner = frame->root->lk_owner;
+    lock->frame = frame;
+    lock->this = this;
+
+    if (conn_id) {
+        lock->connection_id = gf_strdup(conn_id);
+    }
+
+    INIT_LIST_HEAD(&lock->list);
+    INIT_LIST_HEAD(&lock->blocked_locks);
+    INIT_LIST_HEAD(&lock->client_list);
+    INIT_LIST_HEAD(&lock->contend);
+    __pl_inodelk_ref(lock);
 
-        lock = GF_CALLOC (1, sizeof (*lock),
-                          gf_locks_mt_pl_inode_lock_t);
-        if (!lock) {
-                return NULL;
-        }
-
-        lock->fl_start = flock->l_start;
-        lock->fl_type  = flock->l_type;
-
-        if (flock->l_len == 0)
-                lock->fl_end = LLONG_MAX;
-        else
-                lock->fl_end = flock->l_start + flock->l_len - 1;
-
-        lock->client     = client;
-        lock->client_pid = client_pid;
-        lock->volume     = volume;
-        lock->owner      = frame->root->lk_owner;
-        lock->frame      = frame;
-        lock->this       = this;
-
-        if (conn_id) {
-                lock->connection_id = gf_strdup (conn_id);
-        }
-
-        INIT_LIST_HEAD (&lock->list);
-        INIT_LIST_HEAD (&lock->blocked_locks);
-	INIT_LIST_HEAD (&lock->client_list);
-        __pl_inodelk_ref (lock);
-
-        return lock;
+out:
+    return lock;
 }
 
 int32_t
-_pl_convert_volume (const char *volume, char **res)
+_pl_convert_volume(const char *volume, char **res)
 {
-        char    *mdata_vol = NULL;
-        int     ret = 0;
+    char *mdata_vol = NULL;
+    int ret = 0;
 
-        mdata_vol = strrchr (volume, ':');
-        //if the volume already ends with :metadata don't bother
-        if (mdata_vol && (strcmp (mdata_vol, ":metadata") == 0))
-                return 0;
-
-        ret = gf_asprintf (res, "%s:metadata", volume);
-        if (ret <= 0)
-                return ENOMEM;
+    mdata_vol = strrchr(volume, ':');
+    // if the volume already ends with :metadata don't bother
+    if (mdata_vol && (strcmp(mdata_vol, ":metadata") == 0))
         return 0;
+
+    ret = gf_asprintf(res, "%s:metadata", volume);
+    if (ret <= 0)
+        return ENOMEM;
+    return 0;
 }
 
 int32_t
-_pl_convert_volume_for_special_range (struct gf_flock *flock,
-                                      const char *volume, char **res)
+_pl_convert_volume_for_special_range(struct gf_flock *flock, const char *volume,
+                                     char **res)
 {
-        int32_t     ret = 0;
+    int32_t ret = 0;
 
-        if ((flock->l_start == LLONG_MAX -1) &&
-            (flock->l_len == 0)) {
-                ret = _pl_convert_volume (volume, res);
-        }
+    if ((flock->l_start == LLONG_MAX - 1) && (flock->l_len == 0)) {
+        ret = _pl_convert_volume(volume, res);
+    }
 
-        return ret;
+    return ret;
 }
 
 /* Common inodelk code called from pl_inodelk and pl_finodelk */
 int
-pl_common_inodelk (call_frame_t *frame, xlator_t *this,
-                   const char *volume, inode_t *inode, int32_t cmd,
-                   struct gf_flock *flock, loc_t *loc, fd_t *fd, dict_t *xdata)
-{
-        int32_t           op_ret     = -1;
-        int32_t           op_errno   = 0;
-        int               ret        = -1;
-        GF_UNUSED int     dict_ret   = -1;
-        int               can_block  = 0;
-        pl_inode_t *      pinode     = NULL;
-        pl_inode_lock_t * reqlock    = NULL;
-        pl_dom_list_t *   dom        = NULL;
-        char             *res        = NULL;
-        char             *res1       = NULL;
-        char             *conn_id    = NULL;
-        pl_ctx_t         *ctx        = NULL;
-
-        if (xdata)
-                dict_ret = dict_get_str (xdata, "connection-id", &conn_id);
-
-        VALIDATE_OR_GOTO (frame, out);
-        VALIDATE_OR_GOTO (inode, unwind);
-        VALIDATE_OR_GOTO (flock, unwind);
-
-        if ((flock->l_start < 0) || (flock->l_len < 0)) {
-                op_errno = EINVAL;
-                goto unwind;
-        }
-
-        op_errno = _pl_convert_volume_for_special_range (flock, volume, &res);
-        if (op_errno)
-                goto unwind;
-        if (res)
-                volume = res;
-
-        pl_trace_in (this, frame, fd, loc, cmd, flock, volume);
-
-	if (frame->root->client) {
-		ctx = pl_ctx_get (frame->root->client, this);
-		if (!ctx) {
-			op_errno = ENOMEM;
-			gf_log (this->name, GF_LOG_INFO, "pl_ctx_get() failed");
-			goto unwind;
-		}
-	}
-
-        pinode = pl_inode_get (this, inode);
-        if (!pinode) {
-                op_errno = ENOMEM;
-                goto unwind;
+pl_common_inodelk(call_frame_t *frame, xlator_t *this, const char *volume,
+                  inode_t *inode, int32_t cmd, struct gf_flock *flock,
+                  loc_t *loc, fd_t *fd, dict_t *xdata)
+{
+    int32_t op_ret = -1;
+    int32_t op_errno = 0;
+    int ret = -1;
+    GF_UNUSED int dict_ret = -1;
+    int can_block = 0;
+    short lock_type = 0;
+    pl_inode_t *pinode = NULL;
+    pl_inode_lock_t *reqlock = NULL;
+    pl_dom_list_t *dom = NULL;
+    char *res = NULL;
+    char *res1 = NULL;
+    char *conn_id = NULL;
+    pl_ctx_t *ctx = NULL;
+
+    if (xdata)
+        dict_ret = dict_get_str(xdata, "connection-id", &conn_id);
+
+    VALIDATE_OR_GOTO(frame, out);
+    VALIDATE_OR_GOTO(inode, unwind);
+    VALIDATE_OR_GOTO(flock, unwind);
+
+    if ((flock->l_start < 0) || (flock->l_len < 0)) {
+        op_errno = EINVAL;
+        goto unwind;
+    }
+
+    op_errno = _pl_convert_volume_for_special_range(flock, volume, &res);
+    if (op_errno)
+        goto unwind;
+    if (res)
+        volume = res;
+
+    pl_trace_in(this, frame, fd, loc, cmd, flock, volume);
+
+    if (frame->root->client) {
+        ctx = pl_ctx_get(frame->root->client, this);
+        if (!ctx) {
+            op_errno = ENOMEM;
+            gf_log(this->name, GF_LOG_INFO, "pl_ctx_get() failed");
+            goto unwind;
         }
+    }
 
-        dom = get_domain (pinode, volume);
-        if (!dom) {
-                op_errno = ENOMEM;
-                goto unwind;
-        }
+    pinode = pl_inode_get(this, inode, NULL);
+    if (!pinode) {
+        op_errno = ENOMEM;
+        goto unwind;
+    }
 
-        reqlock = new_inode_lock (flock, frame->root->client, frame->root->pid,
-                                  frame, this, dom->domain, conn_id);
+    dom = get_domain(pinode, volume);
+    if (!dom) {
+        op_errno = ENOMEM;
+        goto unwind;
+    }
 
-        if (!reqlock) {
-                op_ret = -1;
-                op_errno = ENOMEM;
-                goto unwind;
-        }
+    reqlock = new_inode_lock(flock, frame->root->client, frame->root->pid,
+                             frame, this, dom->domain, conn_id, &op_errno);
 
+    if (!reqlock) {
+        op_ret = -1;
+        goto unwind;
+    }
 
-        switch (cmd) {
+    switch (cmd) {
         case F_SETLKW:
-                can_block = 1;
+            can_block = 1;
 
-                /* fall through */
+            /* fall through */
 
         case F_SETLK:
-                memcpy (&reqlock->user_flock, flock, sizeof (struct gf_flock));
-                ret = pl_inode_setlk (this, ctx, pinode, reqlock, can_block,
-				      dom, inode);
-
-                if (ret < 0) {
-                        if ((can_block) && (F_UNLCK != flock->l_type)) {
-                                pl_trace_block (this, frame, fd, loc,
-                                                cmd, flock, volume);
-                                goto out;
-                        }
-                        gf_log (this->name, GF_LOG_TRACE, "returning EAGAIN");
-                        op_errno = -ret;
-                        goto unwind;
+            lock_type = flock->l_type;
+            memcpy(&reqlock->user_flock, flock, sizeof(struct gf_flock));
+            ret = pl_inode_setlk(this, ctx, pinode, reqlock, can_block, dom,
+                                 inode);
+
+            if (ret < 0) {
+                if (ret == -EAGAIN) {
+                    if (can_block && (F_UNLCK != lock_type)) {
+                        goto out;
+                    }
+                    gf_log(this->name, GF_LOG_TRACE, "returning EAGAIN");
+                } else {
+                    gf_log(this->name, GF_LOG_TRACE, "returning %d", ret);
                 }
-                break;
+                op_errno = -ret;
+                goto unwind;
+            }
+            break;
 
         default:
-                op_errno = ENOTSUP;
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "Lock command F_GETLK not supported for [f]inodelk "
-                        "(cmd=%d)",
-                        cmd);
-                goto unwind;
-        }
+            op_errno = ENOTSUP;
+            gf_log(this->name, GF_LOG_DEBUG,
+                   "Lock command F_GETLK not supported for [f]inodelk "
+                   "(cmd=%d)",
+                   cmd);
+            goto unwind;
+    }
 
-        op_ret = 0;
+    op_ret = 0;
 
 unwind:
-        if (flock != NULL)
-                pl_trace_out (this, frame, fd, loc, cmd, flock, op_ret,
-                              op_errno, volume);
+    if (flock != NULL)
+        pl_trace_out(this, frame, fd, loc, cmd, flock, op_ret, op_errno,
+                     volume);
 
-        STACK_UNWIND_STRICT (inodelk, frame, op_ret, op_errno, NULL);
+    STACK_UNWIND_STRICT(inodelk, frame, op_ret, op_errno, NULL);
 out:
-        GF_FREE (res);
-        GF_FREE (res1);
-        return 0;
+    GF_FREE(res);
+    GF_FREE(res1);
+    return 0;
 }
 
 int
-pl_inodelk (call_frame_t *frame, xlator_t *this,
-            const char *volume, loc_t *loc, int32_t cmd, struct gf_flock *flock,
-            dict_t *xdata)
+pl_inodelk(call_frame_t *frame, xlator_t *this, const char *volume, loc_t *loc,
+           int32_t cmd, struct gf_flock *flock, dict_t *xdata)
 {
-        pl_common_inodelk (frame, this, volume, loc->inode, cmd, flock,
-                           loc, NULL, xdata);
+    pl_common_inodelk(frame, this, volume, loc->inode, cmd, flock, loc, NULL,
+                      xdata);
 
-        return 0;
+    return 0;
 }
 
 int
-pl_finodelk (call_frame_t *frame, xlator_t *this,
-             const char *volume, fd_t *fd, int32_t cmd, struct gf_flock *flock,
-             dict_t *xdata)
+pl_finodelk(call_frame_t *frame, xlator_t *this, const char *volume, fd_t *fd,
+            int32_t cmd, struct gf_flock *flock, dict_t *xdata)
 {
-        pl_common_inodelk (frame, this, volume, fd->inode, cmd, flock,
-                           NULL, fd, xdata);
-
-        return 0;
+    pl_common_inodelk(frame, this, volume, fd->inode, cmd, flock, NULL, fd,
+                      xdata);
 
+    return 0;
 }
 
-static inline int32_t
-__get_inodelk_dom_count (pl_dom_list_t *dom)
+static int32_t
+__get_inodelk_dom_count(pl_dom_list_t *dom)
 {
-        pl_inode_lock_t     *lock   = NULL;
-        int32_t             count   = 0;
-
-        list_for_each_entry (lock, &dom->inodelk_list, list) {
-                count++;
-        }
-        list_for_each_entry (lock, &dom->blocked_inodelks, blocked_locks) {
-                count++;
-        }
-        return count;
+    pl_inode_lock_t *lock = NULL;
+    int32_t count = 0;
+
+    list_for_each_entry(lock, &dom->inodelk_list, list) { count++; }
+    list_for_each_entry(lock, &dom->blocked_inodelks, blocked_locks)
+    {
+        count++;
+    }
+    return count;
 }
 
 /* Returns the no. of locks (blocked/granted) held on a given domain name
  * If @domname is NULL, returns the no. of locks in all the domains present.
  * If @domname is non-NULL and non-existent, returns 0 */
 int32_t
-__get_inodelk_count (xlator_t *this, pl_inode_t *pl_inode, char *domname)
+__get_inodelk_count(xlator_t *this, pl_inode_t *pl_inode, char *domname)
 {
-        int32_t            count  = 0;
-        pl_dom_list_t     *dom    = NULL;
-
-        list_for_each_entry (dom, &pl_inode->dom_list, inode_list) {
-                if (domname) {
-                        if (strcmp (domname, dom->domain) == 0) {
-                                count = __get_inodelk_dom_count (dom);
-                                goto out;
-                        }
-
-                } else {
-                    /* Counting locks from all domains */
-                        count += __get_inodelk_dom_count (dom);
+    int32_t count = 0;
+    pl_dom_list_t *dom = NULL;
+
+    list_for_each_entry(dom, &pl_inode->dom_list, inode_list)
+    {
+        if (domname) {
+            if (strcmp(domname, dom->domain) == 0) {
+                count = __get_inodelk_dom_count(dom);
+                goto out;
+            }
 
-                }
+        } else {
+            /* Counting locks from all domains */
+            count += __get_inodelk_dom_count(dom);
         }
+    }
 
 out:
-        return count;
+    return count;
 }
 
 int32_t
-get_inodelk_count (xlator_t *this, inode_t *inode, char *domname)
+get_inodelk_count(xlator_t *this, inode_t *inode, char *domname)
 {
-        pl_inode_t   *pl_inode = NULL;
-        uint64_t      tmp_pl_inode = 0;
-        int           ret      = 0;
-        int32_t       count    = 0;
+    pl_inode_t *pl_inode = NULL;
+    uint64_t tmp_pl_inode = 0;
+    int ret = 0;
+    int32_t count = 0;
 
-        ret = inode_ctx_get (inode, this, &tmp_pl_inode);
-        if (ret != 0) {
-                goto out;
-        }
+    ret = inode_ctx_get(inode, this, &tmp_pl_inode);
+    if (ret != 0) {
+        goto out;
+    }
 
-        pl_inode = (pl_inode_t *)(long) tmp_pl_inode;
+    pl_inode = (pl_inode_t *)(long)tmp_pl_inode;
 
-        pthread_mutex_lock (&pl_inode->mutex);
-        {
-                count = __get_inodelk_count (this, pl_inode, domname);
-        }
-        pthread_mutex_unlock (&pl_inode->mutex);
+    pthread_mutex_lock(&pl_inode->mutex);
+    {
+        count = __get_inodelk_count(this, pl_inode, domname);
+    }
+    pthread_mutex_unlock(&pl_inode->mutex);
 
 out:
-        return count;
+    return count;
 }
diff --git a/xlators/features/locks/src/locks-mem-types.h b/xlators/features/locks/src/locks-mem-types.h
index 08aeb0a7925..a76605027b3 100644
--- a/xlators/features/locks/src/locks-mem-types.h
+++ b/xlators/features/locks/src/locks-mem-types.h
@@ -11,19 +11,18 @@
 #ifndef __LOCKS_MEM_TYPES_H__
 #define __LOCKS_MEM_TYPES_H__
 
-#include "mem-types.h"
+#include <glusterfs/mem-types.h>
 
 enum gf_locks_mem_types_ {
-        gf_locks_mt_pl_dom_list_t = gf_common_mt_end + 1,
-        gf_locks_mt_pl_inode_t,
-        gf_locks_mt_posix_lock_t,
-        gf_locks_mt_pl_entry_lock_t,
-        gf_locks_mt_pl_inode_lock_t,
-        gf_locks_mt_truncate_ops,
-        gf_locks_mt_pl_rw_req_t,
-        gf_locks_mt_posix_locks_private_t,
-        gf_locks_mt_pl_fdctx_t,
-        gf_locks_mt_end
+    gf_locks_mt_pl_dom_list_t = gf_common_mt_end + 1,
+    gf_locks_mt_pl_inode_t,
+    gf_locks_mt_posix_lock_t,
+    gf_locks_mt_pl_entry_lock_t,
+    gf_locks_mt_pl_inode_lock_t,
+    gf_locks_mt_pl_rw_req_t,
+    gf_locks_mt_posix_locks_private_t,
+    gf_locks_mt_pl_fdctx_t,
+    gf_locks_mt_pl_meta_lock_t,
+    gf_locks_mt_end
 };
 #endif
-
diff --git a/xlators/features/locks/src/locks.h b/xlators/features/locks/src/locks.h
index f761b3d4a00..c868eb494a2 100644
--- a/xlators/features/locks/src/locks.h
+++ b/xlators/features/locks/src/locks.h
@@ -1,5 +1,5 @@
 /*
-   Copyright (c) 2006-2012 Red Hat, Inc. <http://www.redhat.com>
+   Copyright (c) 2006-2012, 2015-2016 Red Hat, Inc. <http://www.redhat.com>
    This file is part of GlusterFS.
 
    This file is licensed to you under your choice of the GNU Lesser
@@ -10,199 +10,283 @@
 #ifndef __POSIX_LOCKS_H__
 #define __POSIX_LOCKS_H__
 
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
-
-#include "compat-errno.h"
-#include "stack.h"
-#include "call-stub.h"
+#include <glusterfs/compat-errno.h>
+#include <glusterfs/stack.h>
+#include <glusterfs/call-stub.h>
 #include "locks-mem-types.h"
-#include "client_t.h"
+#include <glusterfs/client_t.h>
+
+#include <glusterfs/lkowner.h>
 
-#include "lkowner.h"
+typedef enum {
+    MLK_NONE,
+    MLK_FILE_BASED,
+    MLK_FORCED,
+    MLK_OPTIMAL
+} mlk_mode_t; /* defines different mandatory locking modes*/
 
 struct __pl_fd;
 
 struct __posix_lock {
-        struct list_head   list;
+    struct list_head list;
+
+    off_t fl_start;
+    off_t fl_end;
+    uint32_t lk_flags;
+
+    short fl_type;
+    short blocked;              /* waiting to acquire */
+    struct gf_flock user_flock; /* the flock supplied by the user */
+    xlator_t *this;             /* required for blocked locks */
+    unsigned long fd_num;
 
-        short              fl_type;
-        off_t              fl_start;
-        off_t              fl_end;
+    fd_t *fd;
+    call_frame_t *frame;
 
-        short              blocked;    /* waiting to acquire */
-        struct gf_flock    user_flock; /* the flock supplied by the user */
-        xlator_t          *this;       /* required for blocked locks */
-        unsigned long      fd_num;
+    time_t blkd_time;    /* time at which lock was queued into blkd list */
+    time_t granted_time; /* time at which lock was queued into active list */
 
-        fd_t              *fd;
-        call_frame_t      *frame;
+    /* These two together serve to uniquely identify each process
+       across nodes */
 
-        struct timeval     blkd_time;   /*time at which lock was queued into blkd list*/
-        struct timeval     granted_time; /*time at which lock was queued into active list*/
+    void *client; /* to identify client node */
 
-        /* These two together serve to uniquely identify each process
-           across nodes */
+    /* This field uniquely identifies the client the lock belongs to.  As
+     * lock migration is handled by rebalance, the client_t object will be
+     * overwritten by rebalance and can't be deemed as the owner of the
+     * lock on destination. Hence, the below field is migrated from
+     * source to destination by lock_migration_info_t and updated on the
+     * destination. So that on client-server disconnection, server can
+     * cleanup the locks proper;y.  */
 
-        void              *client;     /* to identify client node */
-        gf_lkowner_t       owner;
-        pid_t              client_pid;    /* pid of client process */
+    char *client_uid;
+    gf_lkowner_t owner;
+    pid_t client_pid; /* pid of client process */
+
+    int blocking;
 };
 typedef struct __posix_lock posix_lock_t;
 
 struct __pl_inode_lock {
-        struct list_head   list;
-        struct list_head   blocked_locks; /* list_head pointing to blocked_inodelks */
-        int                ref;
+    struct list_head list;
+    struct list_head blocked_locks; /* list_head pointing to blocked_inodelks */
+    struct list_head contend;       /* list of contending locks */
+    int ref;
+
+    off_t fl_start;
+    off_t fl_end;
 
-        short              fl_type;
-        off_t              fl_start;
-        off_t              fl_end;
+    const char *volume;
 
-        const char        *volume;
+    struct gf_flock user_flock; /* the flock supplied by the user */
+    xlator_t *this;             /* required for blocked locks */
+    struct __pl_inode *pl_inode;
 
-        struct gf_flock    user_flock; /* the flock supplied by the user */
-        xlator_t          *this;       /* required for blocked locks */
-	struct __pl_inode *pl_inode;
+    call_frame_t *frame;
 
-        call_frame_t      *frame;
+    time_t blkd_time;    /* time at which lock was queued into blkd list */
+    time_t granted_time; /* time at which lock was queued into active list */
 
-        struct timeval     blkd_time;   /*time at which lock was queued into blkd list*/
-        struct timeval     granted_time; /*time at which lock was queued into active list*/
+    /*last time at which lock contention was detected and notified*/
+    struct timespec contention_time;
 
-        /* These two together serve to uniquely identify each process
-           across nodes */
+    /* These two together serve to uniquely identify each process
+       across nodes */
 
-        void              *client;     /* to identify client node */
-        gf_lkowner_t       owner;
-        pid_t              client_pid;    /* pid of client process */
+    void *client; /* to identify client node */
+    gf_lkowner_t owner;
+    pid_t client_pid; /* pid of client process */
 
-        char              *connection_id; /* stores the client connection id */
+    char *connection_id; /* stores the client connection id */
 
-	struct list_head   client_list; /* list of all locks from a client */
+    struct list_head client_list; /* list of all locks from a client */
+    short fl_type;
+
+    int32_t status; /* Error code when we try to grant a lock in blocked
+                       state */
 };
 typedef struct __pl_inode_lock pl_inode_lock_t;
 
-struct __pl_rw_req_t {
-        struct list_head      list;
-        call_stub_t          *stub;
-        posix_lock_t          region;
+struct _pl_rw_req {
+    struct list_head list;
+    call_stub_t *stub;
+    posix_lock_t region;
 };
-typedef struct __pl_rw_req_t pl_rw_req_t;
-
-struct __pl_dom_list_t {
-        struct list_head   inode_list;       /* list_head back to pl_inode_t */
-        const char        *domain;
-        struct list_head   entrylk_list;     /* List of entry locks */
-        struct list_head   blocked_entrylks; /* List of all blocked entrylks */
-        struct list_head   inodelk_list;     /* List of inode locks */
-        struct list_head   blocked_inodelks; /* List of all blocked inodelks */
+typedef struct _pl_rw_req pl_rw_req_t;
+
+struct _pl_dom_list {
+    struct list_head inode_list; /* list_head back to pl_inode_t */
+    const char *domain;
+    struct list_head entrylk_list;     /* List of entry locks */
+    struct list_head blocked_entrylks; /* List of all blocked entrylks */
+    struct list_head inodelk_list;     /* List of inode locks */
+    struct list_head blocked_inodelks; /* List of all blocked inodelks */
 };
-typedef struct __pl_dom_list_t pl_dom_list_t;
+typedef struct _pl_dom_list pl_dom_list_t;
 
 struct __entry_lock {
-        struct list_head  domain_list;    /* list_head back to pl_dom_list_t */
-        struct list_head  blocked_locks; /* list_head back to blocked_entrylks */
-	int ref;
+    struct list_head domain_list;   /* list_head back to pl_dom_list_t */
+    struct list_head blocked_locks; /* list_head back to blocked_entrylks */
+    struct list_head contend;       /* list of contending locks */
+    int ref;
+
+    call_frame_t *frame;
+    xlator_t *this;
+    struct __pl_inode *pinode;
 
-        call_frame_t     *frame;
-        xlator_t         *this;
-	struct __pl_inode *pinode;
+    const char *volume;
 
-        const char       *volume;
+    const char *basename;
 
-        const char       *basename;
-        entrylk_type      type;
+    time_t blkd_time;    /* time at which lock was queued into blkd list */
+    time_t granted_time; /* time at which lock was queued into active list */
 
-        struct timeval     blkd_time;   /*time at which lock was queued into blkd list*/
-        struct timeval     granted_time; /*time at which lock was queued into active list*/
+    /*last time at which lock contention was detected and notified*/
+    struct timespec contention_time;
 
-        void             *client;
-        gf_lkowner_t      owner;
-	pid_t             client_pid;    /* pid of client process */
+    void *client;
+    gf_lkowner_t owner;
+    pid_t client_pid; /* pid of client process */
 
-        char             *connection_id; /* stores the client connection id */
+    char *connection_id; /* stores the client connection id */
 
-	struct list_head   client_list; /* list of all locks from a client */
+    struct list_head client_list; /* list of all locks from a client */
+    entrylk_type type;
 };
 typedef struct __entry_lock pl_entry_lock_t;
 
-
 /* The "simulated" inode. This contains a list of all the locks associated
    with this file */
 
 struct __pl_inode {
-        pthread_mutex_t  mutex;
-
-        struct list_head dom_list;       /* list of domains */
-        struct list_head ext_list;       /* list of fcntl locks */
-        struct list_head rw_list;        /* list of waiting r/w requests */
-        struct list_head reservelk_list;        /* list of reservelks */
-        struct list_head blocked_reservelks;        /* list of blocked reservelks */
-        struct list_head blocked_calls;  /* List of blocked lock calls while a reserve is held*/
-        int              mandatory;      /* if mandatory locking is enabled */
-
-        inode_t          *refkeeper;     /* hold refs on an inode while locks are
-                                            held to prevent pruning */
-        uuid_t            gfid;          /* placeholder for gfid of the inode */
-        inode_t          *inode;         /* pointer to be used for ref and unref
-                                            of inode_t as long as there are
-                                            locks on it */
+    pthread_mutex_t mutex;
+
+    struct list_head dom_list;           /* list of domains */
+    struct list_head ext_list;           /* list of fcntl locks */
+    struct list_head rw_list;            /* list of waiting r/w requests */
+    struct list_head reservelk_list;     /* list of reservelks */
+    struct list_head blocked_reservelks; /* list of blocked reservelks */
+    struct list_head blocked_calls;      /* List of blocked lock calls while a
+                                            reserve is held*/
+    struct list_head metalk_list;        /* Meta lock list */
+    struct list_head queued_locks;       /* This is to store the incoming lock
+                                            requests while meta lock is enabled */
+    struct list_head waiting; /* List of pending fops waiting to unlink/rmdir
+                                 the inode. */
+    int mandatory;            /* if mandatory locking is enabled */
+
+    inode_t *refkeeper; /* hold refs on an inode while locks are
+                           held to prevent pruning */
+    uuid_t gfid;        /* placeholder for gfid of the inode */
+    inode_t *inode;     /* pointer to be used for ref and unref
+                           of inode_t as long as there are
+                           locks on it */
+    gf_boolean_t migrated;
+
+    /* Flag to indicate whether to read mlock-enforce xattr from disk */
+    gf_boolean_t check_mlock_info;
+
+    /* Mandatory_lock enforce: IO will be allowed if and only if the lkowner has
+       held the lock.
+
+       Note: An xattr is set on the file to recover this information post
+       reboot. If client does not want mandatory lock to be enforced, then it
+       should remove this xattr explicitly
+    */
+    gf_boolean_t mlock_enforced;
+    /* There are scenarios where mandatory lock is granted but there are IOs
+       pending at posix level. To avoid this before preempting the previous lock
+       owner, we wait for all the fops to be unwound.
+    */
+    int fop_wind_count;
+    pthread_cond_t check_fop_wind_count;
+
+    gf_boolean_t track_fop_wind_count;
+
+    int32_t links;           /* Number of hard links the inode has. */
+    uint32_t remove_running; /* Number of remove operations running. */
+    gf_boolean_t is_locked;  /* Regular locks will be blocked. */
+    gf_boolean_t removed;    /* The inode has been deleted. */
 };
 typedef struct __pl_inode pl_inode_t;
 
+struct __pl_metalk {
+    pthread_mutex_t mutex;
+    /* For pl_inode meta lock list */
+    struct list_head list;
+    /* For pl_ctx_t list */
+    struct list_head client_list;
+    char *client_uid;
+
+    pl_inode_t *pl_inode;
+    int ref;
+};
+typedef struct __pl_metalk pl_meta_lock_t;
 
 typedef struct {
-        gf_boolean_t    mandatory;      /* if mandatory locking is enabled */
-        gf_boolean_t    trace;          /* trace lock requests in and out */
-        char           *brickname;
+    char *brickname;
+    uint32_t revocation_secs;
+    uint32_t revocation_max_blocked;
+    uint32_t notify_contention_delay;
+    mlk_mode_t mandatory_mode; /* holds current mandatory locking mode */
+    gf_boolean_t trace;        /* trace lock requests in and out */
+    gf_boolean_t monkey_unlocking;
+    gf_boolean_t revocation_clear_all;
+    gf_boolean_t notify_contention;
+    gf_boolean_t mlock_enforced;
 } posix_locks_private_t;
 
-
 typedef struct {
-        gf_boolean_t   entrylk_count_req;
-        gf_boolean_t   inodelk_count_req;
-        gf_boolean_t   inodelk_dom_count_req;
-        gf_boolean_t   posixlk_count_req;
-        gf_boolean_t   parent_entrylk_req;
-
-        /* used by {f,}truncate */
-        loc_t  loc;
-        fd_t  *fd;
-        off_t  offset;
-        dict_t *xdata;
-        enum {TRUNCATE, FTRUNCATE} op;
+    data_t *inodelk_dom_count_req;
+
+    dict_t *xdata;
+    loc_t loc[2];
+    fd_t *fd;
+    inode_t *inode;
+    off_t offset;
+    glusterfs_fop_t op;
+    gf_boolean_t entrylk_count_req;
+    gf_boolean_t inodelk_count_req;
+    gf_boolean_t posixlk_count_req;
+    gf_boolean_t parent_entrylk_req;
+    gf_boolean_t multiple_dom_lk_requests;
+    int update_mlock_enforced_flag;
 } pl_local_t;
 
-
 typedef struct {
-        struct list_head locks_list;
+    struct list_head locks_list;
 } pl_fdctx_t;
 
-
 struct _locker {
-        struct list_head  lockers;
-        char             *volume;
-        inode_t          *inode;
-        gf_lkowner_t      owner;
+    struct list_head lockers;
+    char *volume;
+    inode_t *inode;
+    gf_lkowner_t owner;
 };
 
 typedef struct _locks_ctx {
-        pthread_mutex_t      lock;
-        struct list_head     inodelk_lockers;
-        struct list_head     entrylk_lockers;
+    pthread_mutex_t lock;
+    struct list_head inodelk_lockers;
+    struct list_head entrylk_lockers;
+    struct list_head metalk_list;
 } pl_ctx_t;
 
+typedef struct _multi_dom_lk_data {
+    xlator_t *this;
+    inode_t *inode;
+    dict_t *xdata_rsp;
+    gf_boolean_t keep_max;
+} multi_dom_lk_data;
+
+typedef enum { DECREMENT, INCREMENT } pl_count_op_t;
 
 pl_ctx_t *
-pl_ctx_get (client_t *client, xlator_t *xlator);
+pl_ctx_get(client_t *client, xlator_t *xlator);
 
 int
-pl_inodelk_client_cleanup (xlator_t *this, pl_ctx_t *ctx);
+pl_inodelk_client_cleanup(xlator_t *this, pl_ctx_t *ctx);
 
 int
-pl_entrylk_client_cleanup (xlator_t *this, pl_ctx_t *ctx);
+pl_entrylk_client_cleanup(xlator_t *this, pl_ctx_t *ctx);
 
 #endif /* __POSIX_LOCKS_H__ */
diff --git a/xlators/features/locks/src/pl-messages.h b/xlators/features/locks/src/pl-messages.h
new file mode 100644
index 00000000000..e2d3d7ca974
--- /dev/null
+++ b/xlators/features/locks/src/pl-messages.h
@@ -0,0 +1,29 @@
+/*
+  Copyright (c) 2016 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
+*/
+
+#ifndef _PL_MESSAGES_H_
+#define _PL_MESSAGES_H_
+
+#include <glusterfs/glfs-message-id.h>
+
+/* To add new message IDs, append new identifiers at the end of the list.
+ *
+ * Never remove a message ID. If it's not used anymore, you can rename it or
+ * leave it as it is, but not delete it. This is to prevent reutilization of
+ * IDs by other messages.
+ *
+ * The component name must match one of the entries defined in
+ * glfs-message-id.h.
+ */
+
+GLFS_MSGID(PL, PL_MSG_LOCK_NUMBER, PL_MSG_INODELK_CONTENTION_FAILED,
+           PL_MSG_ENTRYLK_CONTENTION_FAILED);
+
+#endif /* !_PL_MESSAGES_H_ */
diff --git a/xlators/features/locks/src/posix.c b/xlators/features/locks/src/posix.c
index d17d0bbe108..cf0ae4c57dd 100644
--- a/xlators/features/locks/src/posix.c
+++ b/xlators/features/locks/src/posix.c
@@ -1,5 +1,5 @@
 /*
-   Copyright (c) 2006-2012 Red Hat, Inc. <http://www.redhat.com>
+   Copyright (c) 2006-2012, 2016 Red Hat, Inc. <http://www.redhat.com>
    This file is part of GlusterFS.
 
    This file is licensed to you under your choice of the GNU Lesser
@@ -12,2736 +12,5084 @@
 #include <limits.h>
 #include <pthread.h>
 
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
-
-#include "glusterfs.h"
-#include "compat.h"
-#include "xlator.h"
-#include "inode.h"
-#include "logging.h"
-#include "common-utils.h"
+#include <glusterfs/compat.h>
+#include <glusterfs/logging.h>
 
 #include "locks.h"
 #include "common.h"
-#include "statedump.h"
+#include <glusterfs/statedump.h>
 #include "clear.h"
-#include "defaults.h"
-#include "syncop.h"
+#include <glusterfs/defaults.h>
+#include <glusterfs/syncop.h>
 
 #ifndef LLONG_MAX
 #define LLONG_MAX LONG_LONG_MAX /* compat with old gcc */
-#endif /* LLONG_MAX */
+#endif                          /* LLONG_MAX */
 
 /* Forward declarations */
 
+void
+do_blocked_rw(pl_inode_t *);
+static int
+__rw_allowable(pl_inode_t *, posix_lock_t *, glusterfs_fop_t);
+static int
+format_brickname(char *);
+int
+pl_lockinfo_get_brickname(xlator_t *, inode_t *, int32_t *);
+static int
+fetch_pathinfo(xlator_t *, inode_t *, int32_t *, char **);
 
-void do_blocked_rw (pl_inode_t *);
-static int __rw_allowable (pl_inode_t *, posix_lock_t *, glusterfs_fop_t);
-static int format_brickname(char *);
-int pl_lockinfo_get_brickname (xlator_t *, inode_t *, int32_t *);
-static int fetch_pathinfo(xlator_t *, inode_t *, int32_t *, char **);
+/*
+ * The client is always requesting data, but older
+ * servers were not returning it. Newer ones are, so
+ * the client is receiving a mix of NULL and non-NULL
+ * xdata in the answers when bricks are of different
+ * versions. This triggers a bug in older clients.
+ * To prevent that, we avoid returning extra xdata to
+ * older clients (making the newer brick to behave as
+ * an old brick).
+ */
+#define PL_STACK_UNWIND_FOR_CLIENT(fop, xdata, frame, op_ret, params...)       \
+    do {                                                                       \
+        pl_local_t *__local = NULL;                                            \
+        if (frame->root->client &&                                             \
+            (frame->root->client->opversion < GD_OP_VERSION_3_10_0)) {         \
+            __local = frame->local;                                            \
+            PL_STACK_UNWIND_AND_FREE(__local, fop, frame, op_ret, params);     \
+        } else {                                                               \
+            PL_STACK_UNWIND(fop, xdata, frame, op_ret, params);                \
+        }                                                                      \
+    } while (0)
+
+#define PL_STACK_UNWIND(fop, xdata, frame, op_ret, params...)                  \
+    do {                                                                       \
+        pl_local_t *__local = NULL;                                            \
+        inode_t *__parent = NULL;                                              \
+        inode_t *__inode = NULL;                                               \
+        char *__name = NULL;                                                   \
+        dict_t *__unref = NULL;                                                \
+        int __i = 0;                                                           \
+        __local = frame->local;                                                \
+        if (op_ret >= 0 && pl_needs_xdata_response(frame->local)) {            \
+            if (xdata)                                                         \
+                dict_ref(xdata);                                               \
+            else                                                               \
+                xdata = dict_new();                                            \
+            if (xdata) {                                                       \
+                __unref = xdata;                                               \
+                while (__local->fd || __local->loc[__i].inode) {               \
+                    pl_get_xdata_rsp_args(__local, #fop, &__parent, &__inode,  \
+                                          &__name, __i);                       \
+                    pl_set_xdata_response(frame->this, __local, __parent,      \
+                                          __inode, __name, xdata, __i > 0);    \
+                    if (__local->fd || __i == 1)                               \
+                        break;                                                 \
+                    __i++;                                                     \
+                }                                                              \
+            }                                                                  \
+        }                                                                      \
+        PL_STACK_UNWIND_AND_FREE(__local, fop, frame, op_ret, params);         \
+        if (__unref)                                                           \
+            dict_unref(__unref);                                               \
+    } while (0)
+
+#define PL_LOCAL_GET_REQUESTS(frame, this, xdata, __fd, __loc, __newloc)       \
+    do {                                                                       \
+        if (pl_has_xdata_requests(xdata)) {                                    \
+            if (!frame->local)                                                 \
+                frame->local = mem_get0(this->local_pool);                     \
+            pl_local_t *__local = frame->local;                                \
+            if (__local) {                                                     \
+                if (__fd) {                                                    \
+                    __local->fd = fd_ref(__fd);                                \
+                    __local->inode = inode_ref(__fd->inode);                   \
+                } else {                                                       \
+                    if (__loc)                                                 \
+                        loc_copy(&__local->loc[0], __loc);                     \
+                    if (__newloc)                                              \
+                        loc_copy(&__local->loc[1], __newloc);                  \
+                    __local->inode = inode_ref(__local->loc[0].inode);         \
+                }                                                              \
+                pl_get_xdata_requests(__local, xdata);                         \
+            }                                                                  \
+        }                                                                      \
+    } while (0)
+
+#define PL_CHECK_LOCK_ENFORCE_KEY(frame, dict, name, this, loc, fd, priv)      \
+    do {                                                                       \
+        if ((dict && (dict_get(dict, GF_ENFORCE_MANDATORY_LOCK))) ||           \
+            (name && (strcmp(name, GF_ENFORCE_MANDATORY_LOCK) == 0))) {        \
+            inode_t *__inode = (loc ? loc->inode : fd->inode);                 \
+            pl_inode_t *__pl_inode = pl_inode_get(this, __inode, NULL);        \
+            if (__pl_inode == NULL) {                                          \
+                op_ret = -1;                                                   \
+                op_errno = ENOMEM;                                             \
+                goto unwind;                                                   \
+            }                                                                  \
+            if (!pl_is_mandatory_locking_enabled(__pl_inode) ||                \
+                !priv->mlock_enforced) {                                       \
+                op_ret = -1;                                                   \
+                gf_msg(this->name, GF_LOG_DEBUG, EINVAL, 0,                    \
+                       "option %s would need mandatory lock to be enabled "    \
+                       "and feature.enforce-mandatory-lock option to be set "  \
+                       "to on",                                                \
+                       GF_ENFORCE_MANDATORY_LOCK);                             \
+                op_errno = EINVAL;                                             \
+                goto unwind;                                                   \
+            }                                                                  \
+                                                                               \
+            op_ret = pl_local_init(frame, this, loc, fd);                      \
+            if (op_ret) {                                                      \
+                op_errno = ENOMEM;                                             \
+                goto unwind;                                                   \
+            }                                                                  \
+                                                                               \
+            ((pl_local_t *)(frame->local))->update_mlock_enforced_flag = 1;    \
+        }                                                                      \
+    } while (0)
+
+#define PL_INODE_REMOVE(_fop, _frame, _xl, _loc1, _loc2, _cont, _cbk,          \
+                        _args...)                                              \
+    ({                                                                         \
+        struct list_head contend;                                              \
+        pl_inode_t *__pl_inode;                                                \
+        call_stub_t *__stub;                                                   \
+        int32_t __error;                                                       \
+        INIT_LIST_HEAD(&contend);                                              \
+        __error = pl_inode_remove_prepare(_xl, _frame, _loc2 ? _loc2 : _loc1,  \
+                                          &__pl_inode, &contend);              \
+        if (__error < 0) {                                                     \
+            __stub = fop_##_fop##_stub(_frame, _cont, ##_args);                \
+            __error = pl_inode_remove_complete(_xl, __pl_inode, __stub,        \
+                                               &contend);                      \
+        } else if (__error == 0) {                                             \
+            PL_LOCAL_GET_REQUESTS(_frame, _xl, xdata, ((fd_t *)NULL), _loc1,   \
+                                  _loc2);                                      \
+            STACK_WIND_COOKIE(_frame, _cbk, __pl_inode, FIRST_CHILD(_xl),      \
+                              FIRST_CHILD(_xl)->fops->_fop, ##_args);          \
+        }                                                                      \
+        __error;                                                               \
+    })
+
+gf_boolean_t
+pl_has_xdata_requests(dict_t *xdata)
+{
+    static char *reqs[] = {GLUSTERFS_ENTRYLK_COUNT,
+                           GLUSTERFS_INODELK_COUNT,
+                           GLUSTERFS_INODELK_DOM_COUNT,
+                           GLUSTERFS_POSIXLK_COUNT,
+                           GLUSTERFS_PARENT_ENTRYLK,
+                           GLUSTERFS_MULTIPLE_DOM_LK_CNT_REQUESTS,
+                           NULL};
+    static int reqs_size[] = {SLEN(GLUSTERFS_ENTRYLK_COUNT),
+                              SLEN(GLUSTERFS_INODELK_COUNT),
+                              SLEN(GLUSTERFS_INODELK_DOM_COUNT),
+                              SLEN(GLUSTERFS_POSIXLK_COUNT),
+                              SLEN(GLUSTERFS_PARENT_ENTRYLK),
+                              SLEN(GLUSTERFS_MULTIPLE_DOM_LK_CNT_REQUESTS),
+                              0};
+    int i = 0;
+
+    if (!xdata)
+        return _gf_false;
+
+    for (i = 0; reqs[i]; i++)
+        if (dict_getn(xdata, reqs[i], reqs_size[i]))
+            return _gf_true;
+
+    return _gf_false;
+}
 
-static pl_fdctx_t *
-pl_new_fdctx ()
+static int
+dict_delete_domain_key(dict_t *dict, char *key, data_t *value, void *data)
 {
-        pl_fdctx_t *fdctx = NULL;
+    dict_del(dict, key);
+    return 0;
+}
 
-        fdctx = GF_CALLOC (1, sizeof (*fdctx),
-                           gf_locks_mt_pl_fdctx_t);
-        GF_VALIDATE_OR_GOTO ("posix-locks", fdctx, out);
+void
+pl_get_xdata_requests(pl_local_t *local, dict_t *xdata)
+{
+    if (!local || !xdata)
+        return;
 
-        INIT_LIST_HEAD (&fdctx->locks_list);
+    GF_ASSERT(local->xdata == NULL);
+    local->xdata = dict_copy_with_ref(xdata, NULL);
+
+    if (dict_get_sizen(xdata, GLUSTERFS_ENTRYLK_COUNT)) {
+        local->entrylk_count_req = 1;
+        dict_del_sizen(xdata, GLUSTERFS_ENTRYLK_COUNT);
+    }
+    if (dict_get_sizen(xdata, GLUSTERFS_INODELK_COUNT)) {
+        local->inodelk_count_req = 1;
+        dict_del_sizen(xdata, GLUSTERFS_INODELK_COUNT);
+    }
+    if (dict_get_sizen(xdata, GLUSTERFS_MULTIPLE_DOM_LK_CNT_REQUESTS)) {
+        local->multiple_dom_lk_requests = 1;
+        dict_del_sizen(xdata, GLUSTERFS_MULTIPLE_DOM_LK_CNT_REQUESTS);
+        dict_foreach_fnmatch(xdata, GLUSTERFS_INODELK_DOM_PREFIX "*",
+                             dict_delete_domain_key, NULL);
+    }
+
+    local->inodelk_dom_count_req = dict_get_sizen(xdata,
+                                                  GLUSTERFS_INODELK_DOM_COUNT);
+    if (local->inodelk_dom_count_req) {
+        data_ref(local->inodelk_dom_count_req);
+        dict_del_sizen(xdata, GLUSTERFS_INODELK_DOM_COUNT);
+    }
+
+    if (dict_get_sizen(xdata, GLUSTERFS_POSIXLK_COUNT)) {
+        local->posixlk_count_req = 1;
+        dict_del_sizen(xdata, GLUSTERFS_POSIXLK_COUNT);
+    }
+
+    if (dict_get_sizen(xdata, GLUSTERFS_PARENT_ENTRYLK)) {
+        local->parent_entrylk_req = 1;
+        dict_del_sizen(xdata, GLUSTERFS_PARENT_ENTRYLK);
+    }
+}
 
-out:
-        return fdctx;
+gf_boolean_t
+pl_needs_xdata_response(pl_local_t *local)
+{
+    if (!local)
+        return _gf_false;
+
+    if (local->parent_entrylk_req || local->entrylk_count_req ||
+        local->inodelk_dom_count_req || local->inodelk_count_req ||
+        local->posixlk_count_req || local->multiple_dom_lk_requests)
+        return _gf_true;
+
+    return _gf_false;
 }
 
-static pl_fdctx_t *
-pl_check_n_create_fdctx (xlator_t *this, fd_t *fd)
+void
+pl_get_xdata_rsp_args(pl_local_t *local, char *fop, inode_t **parent,
+                      inode_t **inode, char **name, int i)
+{
+    if (strcmp(fop, "lookup") == 0) {
+        *parent = local->loc[0].parent;
+        *inode = local->loc[0].inode;
+        *name = (char *)local->loc[0].name;
+    } else {
+        if (local->fd) {
+            *inode = local->fd->inode;
+        } else {
+            *inode = local->loc[i].parent;
+        }
+    }
+}
+
+static inline int
+pl_track_io_fop_count(pl_local_t *local, xlator_t *this, pl_count_op_t op)
 {
-        int         ret   = 0;
-        uint64_t    tmp   = 0;
-        pl_fdctx_t *fdctx = NULL;
+    pl_inode_t *pl_inode = NULL;
 
-        GF_VALIDATE_OR_GOTO ("posix-locks", this, out);
-        GF_VALIDATE_OR_GOTO (this->name, fd, out);
+    if (!local)
+        return -1;
 
-        LOCK (&fd->lock);
-        {
-                ret = __fd_ctx_get (fd, this, &tmp);
-                if ((ret != 0) || (tmp == 0)) {
-                        fdctx = pl_new_fdctx ();
-                        if (fdctx == NULL) {
-                                goto unlock;
-                        }
-                }
+    pl_inode = pl_inode_get(this, local->inode, NULL);
+    if (!pl_inode)
+        return -1;
 
-                ret = __fd_ctx_set (fd, this, (uint64_t)(long)fdctx);
-                if (ret != 0) {
-                        GF_FREE (fdctx);
-                        fdctx = NULL;
-                        gf_log (this->name, GF_LOG_DEBUG,
-                                "failed to set fd ctx");
+    if (pl_inode->mlock_enforced && pl_inode->track_fop_wind_count) {
+        pthread_mutex_lock(&pl_inode->mutex);
+        {
+            if (op == DECREMENT) {
+                pl_inode->fop_wind_count--;
+                /* fop_wind_count can go negative when lock enforcement is
+                 * enabled on unwind path of an IO. Hence the "<" comparision.
+                 */
+                if (pl_inode->fop_wind_count <= 0) {
+                    pthread_cond_broadcast(&pl_inode->check_fop_wind_count);
+                    pl_inode->track_fop_wind_count = _gf_false;
+                    pl_inode->fop_wind_count = 0;
                 }
+            } else {
+                pl_inode->fop_wind_count++;
+            }
         }
-unlock:
-        UNLOCK (&fd->lock);
+        pthread_mutex_unlock(&pl_inode->mutex);
+    }
 
-out:
-        return fdctx;
+    return 0;
 }
 
-int
-pl_truncate_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                 int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
-                 struct iatt *postbuf, dict_t *xdata)
+static int32_t
+__get_posixlk_count(pl_inode_t *pl_inode)
 {
-        pl_local_t *local = NULL;
+    posix_lock_t *lock = NULL;
+    int32_t count = 0;
 
-        local = frame->local;
+    list_for_each_entry(lock, &pl_inode->ext_list, list) { count++; }
 
-        if (local->op == TRUNCATE)
-                loc_wipe (&local->loc);
+    return count;
+}
 
-        if (local->xdata)
-                dict_unref (local->xdata);
-        if (local->fd)
-                fd_unref (local->fd);
+int32_t
+get_posixlk_count(xlator_t *this, inode_t *inode)
+{
+    pl_inode_t *pl_inode = NULL;
+    uint64_t tmp_pl_inode = 0;
+    int32_t count = 0;
 
-        STACK_UNWIND_STRICT (truncate, frame, op_ret, op_errno,
-                             prebuf, postbuf, xdata);
-        return 0;
+    int ret = inode_ctx_get(inode, this, &tmp_pl_inode);
+    if (ret != 0) {
+        goto out;
+    }
+
+    pl_inode = (pl_inode_t *)(long)tmp_pl_inode;
+
+    pthread_mutex_lock(&pl_inode->mutex);
+    {
+        count = __get_posixlk_count(pl_inode);
+    }
+    pthread_mutex_unlock(&pl_inode->mutex);
+
+out:
+    return count;
 }
 
+void
+pl_parent_entrylk_xattr_fill(xlator_t *this, inode_t *parent, char *basename,
+                             dict_t *dict, gf_boolean_t keep_max)
+{
+    int32_t entrylk = 0;
+    int32_t maxcount = -1;
+    int ret = -1;
+
+    if (!parent || !basename)
+        goto out;
+    if (keep_max) {
+        ret = dict_get_int32_sizen(dict, GLUSTERFS_PARENT_ENTRYLK, &maxcount);
+        if (ret < 0)
+            gf_msg_debug(this->name, 0, " Failed to fetch the value for key %s",
+                         GLUSTERFS_PARENT_ENTRYLK);
+    }
+    entrylk = check_entrylk_on_basename(this, parent, basename);
+    if (maxcount >= entrylk)
+        return;
+out:
+    ret = dict_set_int32_sizen(dict, GLUSTERFS_PARENT_ENTRYLK, entrylk);
+    if (ret < 0) {
+        gf_msg_debug(this->name, 0, " dict_set failed on key %s",
+                     GLUSTERFS_PARENT_ENTRYLK);
+    }
+}
 
-static int
-truncate_allowed (pl_inode_t *pl_inode,
-                  client_t *client, pid_t client_pid,
-                  gf_lkowner_t *owner, off_t offset)
+void
+pl_entrylk_xattr_fill(xlator_t *this, inode_t *inode, dict_t *dict,
+                      gf_boolean_t keep_max)
+{
+    int32_t count = 0;
+    int32_t maxcount = -1;
+    int ret = -1;
+
+    if (keep_max) {
+        ret = dict_get_int32_sizen(dict, GLUSTERFS_ENTRYLK_COUNT, &maxcount);
+        if (ret < 0)
+            gf_msg_debug(this->name, 0, " Failed to fetch the value for key %s",
+                         GLUSTERFS_ENTRYLK_COUNT);
+    }
+    count = get_entrylk_count(this, inode);
+    if (maxcount >= count)
+        return;
+
+    ret = dict_set_int32_sizen(dict, GLUSTERFS_ENTRYLK_COUNT, count);
+    if (ret < 0) {
+        gf_msg_debug(this->name, 0, " dict_set failed on key %s",
+                     GLUSTERFS_ENTRYLK_COUNT);
+    }
+}
+
+void
+pl_inodelk_xattr_fill(xlator_t *this, inode_t *inode, dict_t *dict,
+                      char *domname, gf_boolean_t keep_max)
 {
-        posix_lock_t *l = NULL;
-        posix_lock_t  region = {.list = {0, }, };
-        int           ret = 1;
+    int32_t count = 0;
+    int32_t maxcount = -1;
+    int ret = -1;
+
+    if (keep_max) {
+        ret = dict_get_int32_sizen(dict, GLUSTERFS_INODELK_COUNT, &maxcount);
+        if (ret < 0)
+            gf_msg_debug(this->name, 0, " Failed to fetch the value for key %s",
+                         GLUSTERFS_INODELK_COUNT);
+    }
+    count = get_inodelk_count(this, inode, domname);
+    if (maxcount >= count)
+        return;
 
-        region.fl_start   = offset;
-        region.fl_end     = LLONG_MAX;
-        region.client     = client;
-        region.client_pid = client_pid;
-        region.owner      = *owner;
+    ret = dict_set_int32_sizen(dict, GLUSTERFS_INODELK_COUNT, count);
+    if (ret < 0) {
+        gf_msg_debug(this->name, 0,
+                     "Failed to set count for "
+                     "key %s",
+                     GLUSTERFS_INODELK_COUNT);
+    }
 
-        pthread_mutex_lock (&pl_inode->mutex);
-        {
-                list_for_each_entry (l, &pl_inode->ext_list, list) {
-                        if (!l->blocked
-                            && locks_overlap (&region, l)
-                            && !same_owner (&region, l)) {
-                                ret = 0;
-                                gf_log ("posix-locks", GF_LOG_TRACE, "Truncate "
-                                        "allowed");
-                                break;
-                        }
-                }
-        }
-        pthread_mutex_unlock (&pl_inode->mutex);
+    return;
+}
 
-        return ret;
+void
+pl_posixlk_xattr_fill(xlator_t *this, inode_t *inode, dict_t *dict,
+                      gf_boolean_t keep_max)
+{
+    int32_t count = 0;
+    int32_t maxcount = -1;
+    int ret = -1;
+
+    if (keep_max) {
+        ret = dict_get_int32_sizen(dict, GLUSTERFS_POSIXLK_COUNT, &maxcount);
+        if (ret < 0)
+            gf_msg_debug(this->name, 0, " Failed to fetch the value for key %s",
+                         GLUSTERFS_POSIXLK_COUNT);
+    }
+    count = get_posixlk_count(this, inode);
+    if (maxcount >= count)
+        return;
+
+    ret = dict_set_int32_sizen(dict, GLUSTERFS_POSIXLK_COUNT, count);
+    if (ret < 0) {
+        gf_msg_debug(this->name, 0, " dict_set failed on key %s",
+                     GLUSTERFS_POSIXLK_COUNT);
+    }
 }
 
+void
+pl_inodelk_xattr_fill_each(xlator_t *this, inode_t *inode, dict_t *dict,
+                           char *domname, gf_boolean_t keep_max, char *key)
+{
+    int32_t count = 0;
+    int32_t maxcount = -1;
+    int ret = -1;
+
+    if (keep_max) {
+        ret = dict_get_int32(dict, key, &maxcount);
+        if (ret < 0)
+            gf_msg_debug(this->name, 0, " Failed to fetch the value for key %s",
+                         GLUSTERFS_INODELK_COUNT);
+    }
+    count = get_inodelk_count(this, inode, domname);
+    if (maxcount >= count)
+        return;
+
+    ret = dict_set_int32(dict, key, count);
+    if (ret < 0) {
+        gf_msg_debug(this->name, 0,
+                     "Failed to set count for "
+                     "key %s",
+                     key);
+    }
+
+    return;
+}
 
 static int
-truncate_stat_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                   int32_t op_ret, int32_t op_errno, struct iatt *buf,
-                   dict_t *xdata)
+pl_inodelk_xattr_fill_multiple(dict_t *this, char *key, data_t *value,
+                               void *data)
 {
-        posix_locks_private_t *priv = NULL;
-        pl_local_t            *local = NULL;
-        inode_t               *inode = NULL;
-        pl_inode_t            *pl_inode = NULL;
+    multi_dom_lk_data *d = data;
+    char *tmp_key = NULL;
+    char *save_ptr = NULL;
+
+    tmp_key = gf_strdup(key);
+    if (!tmp_key)
+        return -1;
+
+    strtok_r(tmp_key, ":", &save_ptr);
+    if (!*save_ptr) {
+        if (tmp_key)
+            GF_FREE(tmp_key);
+        gf_msg(THIS->name, GF_LOG_ERROR, 0, EINVAL,
+               "Could not tokenize domain string from key %s", key);
+        return -1;
+    }
+
+    pl_inodelk_xattr_fill_each(d->this, d->inode, d->xdata_rsp, save_ptr,
+                               d->keep_max, key);
+    if (tmp_key)
+        GF_FREE(tmp_key);
+
+    return 0;
+}
 
+void
+pl_fill_multiple_dom_lk_requests(xlator_t *this, pl_local_t *local,
+                                 inode_t *inode, dict_t *dict,
+                                 gf_boolean_t keep_max)
+{
+    multi_dom_lk_data data;
 
-        priv = this->private;
-        local = frame->local;
+    data.this = this;
+    data.inode = inode;
+    data.xdata_rsp = dict;
+    data.keep_max = keep_max;
 
-        if (op_ret != 0) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "got error (errno=%d, stderror=%s) from child",
-                        op_errno, strerror (op_errno));
-                goto unwind;
-        }
+    dict_foreach_fnmatch(local->xdata, GLUSTERFS_INODELK_DOM_PREFIX "*",
+                         pl_inodelk_xattr_fill_multiple, &data);
+}
 
-        if (local->op == TRUNCATE)
-                inode = local->loc.inode;
-        else
-                inode = local->fd->inode;
+void
+pl_set_xdata_response(xlator_t *this, pl_local_t *local, inode_t *parent,
+                      inode_t *inode, char *name, dict_t *xdata,
+                      gf_boolean_t max_lock)
+{
+    if (!xdata || !local)
+        return;
 
-        pl_inode = pl_inode_get (this, inode);
-        if (!pl_inode) {
-                op_ret   = -1;
-                op_errno = ENOMEM;
-                goto unwind;
-        }
+    if (local->parent_entrylk_req && parent && name && name[0] != '\0')
+        pl_parent_entrylk_xattr_fill(this, parent, name, xdata, max_lock);
 
-        if (priv->mandatory
-            && pl_inode->mandatory
-            && !truncate_allowed (pl_inode, frame->root->client,
-                                  frame->root->pid, &frame->root->lk_owner,
-                                  local->offset)) {
-                op_ret   = -1;
-                op_errno = EAGAIN;
-                goto unwind;
-        }
+    if (!inode)
+        return;
 
-        switch (local->op) {
-        case TRUNCATE:
-                STACK_WIND (frame, pl_truncate_cbk, FIRST_CHILD (this),
-                            FIRST_CHILD (this)->fops->truncate,
-                            &local->loc, local->offset, local->xdata);
-                break;
-        case FTRUNCATE:
-                STACK_WIND (frame, pl_truncate_cbk, FIRST_CHILD (this),
-                            FIRST_CHILD (this)->fops->ftruncate,
-                            local->fd, local->offset, local->xdata);
-                break;
-        }
+    if (local->entrylk_count_req)
+        pl_entrylk_xattr_fill(this, inode, xdata, max_lock);
 
-        return 0;
+    if (local->inodelk_dom_count_req)
+        pl_inodelk_xattr_fill(this, inode, xdata,
+                              data_to_str(local->inodelk_dom_count_req),
+                              max_lock);
 
-unwind:
-        gf_log (this->name, GF_LOG_ERROR, "truncate failed with ret: %d, "
-                "error: %s", op_ret, strerror (op_errno));
-        if (local->op == TRUNCATE)
-                loc_wipe (&local->loc);
-        if (local->xdata)
-                dict_unref (local->xdata);
-        if (local->fd)
-                fd_unref (local->fd);
-
-        STACK_UNWIND_STRICT (truncate, frame, op_ret, op_errno, buf, NULL, xdata);
-        return 0;
-}
+    if (local->inodelk_count_req)
+        pl_inodelk_xattr_fill(this, inode, xdata, NULL, max_lock);
 
+    if (local->posixlk_count_req)
+        pl_posixlk_xattr_fill(this, inode, xdata, max_lock);
+
+    if (local->multiple_dom_lk_requests)
+        pl_fill_multiple_dom_lk_requests(this, local, inode, xdata, max_lock);
+}
 
+/* Checks whether the region where fop is acting upon conflicts
+ * with existing locks. If there is no conflict function returns
+ * 1 else returns 0 with can_block boolean set accordingly to
+ * indicate block/fail the fop.
+ */
 int
-pl_truncate (call_frame_t *frame, xlator_t *this,
-             loc_t *loc, off_t offset, dict_t *xdata)
+pl_is_fop_allowed(pl_inode_t *pl_inode, posix_lock_t *region, fd_t *fd,
+                  glusterfs_fop_t op, gf_boolean_t *can_block)
 {
-        pl_local_t *local = NULL;
+    int ret = 0;
+
+    if (!__rw_allowable(pl_inode, region, op)) {
+        if (pl_inode->mlock_enforced) {
+            *can_block = _gf_false;
+        } else if ((!fd) || (fd && (fd->flags & O_NONBLOCK))) {
+            gf_log("locks", GF_LOG_TRACE,
+                   "returning EAGAIN"
+                   " because fd is O_NONBLOCK");
+            *can_block = _gf_false;
+        } else {
+            *can_block = _gf_true;
+        }
+    } else {
+        ret = 1;
+    }
 
-        local = mem_get0 (this->local_pool);
-        GF_VALIDATE_OR_GOTO (this->name, local, unwind);
+    return ret;
+}
 
-        local->op         = TRUNCATE;
-        local->offset     = offset;
-        loc_copy (&local->loc, loc);
-        if (xdata)
-                local->xdata = dict_ref (xdata);
+static pl_fdctx_t *
+pl_new_fdctx()
+{
+    pl_fdctx_t *fdctx = GF_MALLOC(sizeof(*fdctx), gf_locks_mt_pl_fdctx_t);
+    GF_VALIDATE_OR_GOTO("posix-locks", fdctx, out);
 
-        frame->local = local;
+    INIT_LIST_HEAD(&fdctx->locks_list);
 
-        STACK_WIND (frame, truncate_stat_cbk, FIRST_CHILD (this),
-                    FIRST_CHILD (this)->fops->stat, loc, NULL);
+out:
+    return fdctx;
+}
 
-        return 0;
+static pl_fdctx_t *
+pl_check_n_create_fdctx(xlator_t *this, fd_t *fd)
+{
+    int ret = 0;
+    uint64_t tmp = 0;
+    pl_fdctx_t *fdctx = NULL;
+
+    GF_VALIDATE_OR_GOTO("posix-locks", this, out);
+    GF_VALIDATE_OR_GOTO(this->name, fd, out);
+
+    LOCK(&fd->lock);
+    {
+        ret = __fd_ctx_get(fd, this, &tmp);
+        if ((ret != 0) || (tmp == 0)) {
+            fdctx = pl_new_fdctx();
+            if (fdctx == NULL) {
+                goto unlock;
+            }
+        }
 
-unwind:
-        gf_log (this->name, GF_LOG_ERROR, "truncate for %s failed with ret: %d, "
-                "error: %s", loc->path, -1, strerror (ENOMEM));
-        STACK_UNWIND_STRICT (truncate, frame, -1, ENOMEM, NULL, NULL, NULL);
+        ret = __fd_ctx_set(fd, this, (uint64_t)(long)fdctx);
+        if (ret != 0) {
+            GF_FREE(fdctx);
+            fdctx = NULL;
+            UNLOCK(&fd->lock);
+            gf_log(this->name, GF_LOG_DEBUG, "failed to set fd ctx");
+            goto out;
+        }
+    }
+unlock:
+    UNLOCK(&fd->lock);
 
-        return 0;
+out:
+    return fdctx;
 }
 
+int32_t
+pl_discard_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+               int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
+               struct iatt *postbuf, dict_t *xdata)
+{
+    pl_track_io_fop_count(frame->local, this, DECREMENT);
+
+    PL_STACK_UNWIND(discard, xdata, frame, op_ret, op_errno, prebuf, postbuf,
+                    xdata);
+    return 0;
+}
 
 int
-pl_ftruncate (call_frame_t *frame, xlator_t *this,
-              fd_t *fd, off_t offset, dict_t *xdata)
+pl_discard_cont(call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
+                size_t len, dict_t *xdata)
 {
-        pl_local_t *local = NULL;
+    pl_track_io_fop_count(frame->local, this, INCREMENT);
+
+    STACK_WIND(frame, pl_discard_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->discard, fd, offset, len, xdata);
+    return 0;
+}
+
+int32_t
+pl_discard(call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
+           size_t len, dict_t *xdata)
+{
+    pl_local_t *local = NULL;
+    pl_inode_t *pl_inode = NULL;
+    pl_rw_req_t *rw = NULL;
+    posix_lock_t region = {
+        .list =
+            {
+                0,
+            },
+    };
+    gf_boolean_t enabled = _gf_false;
+    gf_boolean_t can_block = _gf_true;
+    int op_ret = 0;
+    int op_errno = 0;
+    int allowed = 1;
+
+    GF_VALIDATE_OR_GOTO("locks", this, unwind);
+
+    local = mem_get0(this->local_pool);
+    if (!local) {
+        op_ret = -1;
+        op_errno = ENOMEM;
+        goto unwind;
+    }
+
+    frame->local = local;
+    local->inode = inode_ref(fd->inode);
+    local->fd = fd_ref(fd);
+
+    pl_inode = pl_inode_get(this, fd->inode, local);
+    if (!pl_inode) {
+        op_ret = -1;
+        op_errno = ENOMEM;
+        goto unwind;
+    }
+
+    if (frame->root->pid < 0)
+        enabled = _gf_false;
+    else
+        enabled = pl_is_mandatory_locking_enabled(pl_inode);
+
+    if (enabled) {
+        region.fl_start = offset;
+        region.fl_end = offset + len - 1;
+        region.client = frame->root->client;
+        region.fd_num = fd_to_fdnum(fd);
+        region.client_pid = frame->root->pid;
+        region.owner = frame->root->lk_owner;
+
+        pthread_mutex_lock(&pl_inode->mutex);
+        {
+            allowed = pl_is_fop_allowed(pl_inode, &region, fd, GF_FOP_DISCARD,
+                                        &can_block);
+            if (allowed == 1) {
+                if (pl_inode->mlock_enforced &&
+                    pl_inode->track_fop_wind_count) {
+                    pl_inode->fop_wind_count++;
+                }
+                goto unlock;
+            } else if (!can_block) {
+                op_errno = EAGAIN;
+                op_ret = -1;
+                goto unlock;
+            }
 
-        local = mem_get0 (this->local_pool);
-        GF_VALIDATE_OR_GOTO (this->name, local, unwind);
+            rw = GF_MALLOC(sizeof(*rw), gf_locks_mt_pl_rw_req_t);
+            if (!rw) {
+                op_errno = ENOMEM;
+                op_ret = -1;
+                goto unlock;
+            }
 
-        local->op         = FTRUNCATE;
-        local->offset     = offset;
-        local->fd         = fd_ref (fd);
-        if (xdata)
-                local->xdata = dict_ref (xdata);
+            rw->stub = fop_discard_stub(frame, pl_discard_cont, fd, offset, len,
+                                        xdata);
+            if (!rw->stub) {
+                op_errno = ENOMEM;
+                op_ret = -1;
+                GF_FREE(rw);
+                goto unlock;
+            }
 
-        frame->local = local;
+            rw->region = region;
 
-        STACK_WIND (frame, truncate_stat_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->fstat, fd, xdata);
-        return 0;
+            list_add_tail(&rw->list, &pl_inode->rw_list);
+        }
+    unlock:
+        pthread_mutex_unlock(&pl_inode->mutex);
+    }
 
+    if (allowed == 1)
+        STACK_WIND(frame, pl_discard_cbk, FIRST_CHILD(this),
+                   FIRST_CHILD(this)->fops->discard, fd, offset, len, xdata);
 unwind:
-        gf_log (this->name, GF_LOG_ERROR, "ftruncate failed with ret: %d, "
-                "error: %s", -1, strerror (ENOMEM));
-        STACK_UNWIND_STRICT (ftruncate, frame, -1, ENOMEM, NULL, NULL, NULL);
+    if (op_ret == -1)
+        PL_STACK_UNWIND(discard, xdata, frame, op_ret, op_errno, NULL, NULL,
+                        NULL);
 
-        return 0;
+    return 0;
 }
 
-int
-pl_locks_by_fd (pl_inode_t *pl_inode, fd_t *fd)
+int32_t
+pl_zerofill_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
+                struct iatt *postbuf, dict_t *xdata)
 {
-       posix_lock_t *l = NULL;
-       int found = 0;
+    pl_track_io_fop_count(frame->local, this, DECREMENT);
 
-       pthread_mutex_lock (&pl_inode->mutex);
-       {
+    PL_STACK_UNWIND(zerofill, xdata, frame, op_ret, op_errno, prebuf, postbuf,
+                    xdata);
+    return 0;
+}
 
-               list_for_each_entry (l, &pl_inode->ext_list, list) {
-                       if (l->fd_num == fd_to_fdnum(fd)) {
-                               found = 1;
-                               break;
-                       }
-               }
+int
+pl_zerofill_cont(call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
+                 off_t len, dict_t *xdata)
+{
+    pl_track_io_fop_count(frame->local, this, INCREMENT);
 
-       }
-       pthread_mutex_unlock (&pl_inode->mutex);
-       return found;
+    STACK_WIND(frame, pl_zerofill_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->zerofill, fd, offset, len, xdata);
+    return 0;
 }
 
-static void
-delete_locks_of_fd (xlator_t *this, pl_inode_t *pl_inode, fd_t *fd)
+int32_t
+pl_zerofill(call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
+            off_t len, dict_t *xdata)
 {
-       posix_lock_t *tmp = NULL;
-       posix_lock_t *l = NULL;
+    pl_local_t *local = NULL;
+    pl_inode_t *pl_inode = NULL;
+    pl_rw_req_t *rw = NULL;
+    posix_lock_t region = {
+        .list =
+            {
+                0,
+            },
+    };
+    gf_boolean_t enabled = _gf_false;
+    gf_boolean_t can_block = _gf_true;
+    int op_ret = 0;
+    int op_errno = 0;
+    int allowed = 1;
+
+    GF_VALIDATE_OR_GOTO("locks", this, unwind);
+
+    local = mem_get0(this->local_pool);
+    if (!local) {
+        op_ret = -1;
+        op_errno = ENOMEM;
+        goto unwind;
+    }
+
+    frame->local = local;
+    local->inode = inode_ref(fd->inode);
+    local->fd = fd_ref(fd);
+
+    pl_inode = pl_inode_get(this, fd->inode, local);
+    if (!pl_inode) {
+        op_ret = -1;
+        op_errno = ENOMEM;
+        goto unwind;
+    }
+
+    if (frame->root->pid < 0)
+        enabled = _gf_false;
+    else
+        enabled = pl_is_mandatory_locking_enabled(pl_inode);
+
+    if (enabled) {
+        region.fl_start = offset;
+        region.fl_end = offset + len - 1;
+        region.client = frame->root->client;
+        region.fd_num = fd_to_fdnum(fd);
+        region.client_pid = frame->root->pid;
+        region.owner = frame->root->lk_owner;
+
+        pthread_mutex_lock(&pl_inode->mutex);
+        {
+            allowed = pl_is_fop_allowed(pl_inode, &region, fd, GF_FOP_ZEROFILL,
+                                        &can_block);
+            if (allowed == 1) {
+                if (pl_inode->mlock_enforced &&
+                    pl_inode->track_fop_wind_count) {
+                    pl_inode->fop_wind_count++;
+                }
+                goto unlock;
+            } else if (!can_block) {
+                op_errno = EAGAIN;
+                op_ret = -1;
+                goto unlock;
+            }
 
-       struct list_head blocked_list;
+            rw = GF_MALLOC(sizeof(*rw), gf_locks_mt_pl_rw_req_t);
+            if (!rw) {
+                op_errno = ENOMEM;
+                op_ret = -1;
+                goto unlock;
+            }
 
-       INIT_LIST_HEAD (&blocked_list);
+            rw->stub = fop_zerofill_stub(frame, pl_zerofill_cont, fd, offset,
+                                         len, xdata);
+            if (!rw->stub) {
+                op_errno = ENOMEM;
+                op_ret = -1;
+                GF_FREE(rw);
+                goto unlock;
+            }
 
-       pthread_mutex_lock (&pl_inode->mutex);
-       {
+            rw->region = region;
 
-               list_for_each_entry_safe (l, tmp, &pl_inode->ext_list, list) {
-                       if (l->fd_num == fd_to_fdnum(fd)) {
-                               if (l->blocked) {
-                                       list_move_tail (&l->list, &blocked_list);
-                                       continue;
-                               }
-                               __delete_lock (pl_inode, l);
-                               __destroy_lock (l);
-                       }
-               }
+            list_add_tail(&rw->list, &pl_inode->rw_list);
+        }
+    unlock:
+        pthread_mutex_unlock(&pl_inode->mutex);
+    }
 
-       }
-       pthread_mutex_unlock (&pl_inode->mutex);
+    if (allowed == 1)
+        STACK_WIND(frame, pl_zerofill_cbk, FIRST_CHILD(this),
+                   FIRST_CHILD(this)->fops->zerofill, fd, offset, len, xdata);
+unwind:
+    if (op_ret == -1)
+        PL_STACK_UNWIND(zerofill, xdata, frame, op_ret, op_errno, NULL, NULL,
+                        NULL);
 
-       list_for_each_entry_safe (l, tmp, &blocked_list, list) {
-               list_del_init(&l->list);
-               STACK_UNWIND_STRICT (lk, l->frame, -1, EAGAIN, &l->user_flock,
-                                    NULL);
-               __destroy_lock (l);
-       }
+    return 0;
+}
 
-        grant_blocked_locks (this, pl_inode);
+int
+pl_truncate_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
+                struct iatt *postbuf, dict_t *xdata)
+{
+    pl_local_t *local = frame->local;
 
-        do_blocked_rw (pl_inode);
+    pl_track_io_fop_count(local, this, DECREMENT);
 
+    if (local->op == GF_FOP_TRUNCATE)
+        PL_STACK_UNWIND(truncate, xdata, frame, op_ret, op_errno, prebuf,
+                        postbuf, xdata);
+    else
+        PL_STACK_UNWIND(ftruncate, xdata, frame, op_ret, op_errno, prebuf,
+                        postbuf, xdata);
+    return 0;
 }
 
-static void
-__delete_locks_of_owner (pl_inode_t *pl_inode,
-                         client_t *client, gf_lkowner_t *owner)
-{
-        posix_lock_t *tmp = NULL;
-        posix_lock_t *l = NULL;
-
-        /* TODO: what if it is a blocked lock with pending l->frame */
-
-        list_for_each_entry_safe (l, tmp, &pl_inode->ext_list, list) {
-                if (l->blocked)
-                        continue;
-                if ((l->client == client) &&
-                    is_same_lkowner (&l->owner, owner)) {
-                        gf_log ("posix-locks", GF_LOG_TRACE,
-                                " Flushing lock"
-                                "%s (pid=%d) (lk-owner=%s) %"PRId64" - %"PRId64" state: %s",
-                                l->fl_type == F_UNLCK ? "Unlock" : "Lock",
-                                l->client_pid,
-                                lkowner_utoa (&l->owner),
-                                l->user_flock.l_start,
-                                l->user_flock.l_len,
-                                l->blocked == 1 ? "Blocked" : "Active");
-
-                        __delete_lock (pl_inode, l);
-                        __destroy_lock (l);
-                }
-        }
+int
+pl_ftruncate_cont(call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
+                  dict_t *xdata)
+{
+    pl_track_io_fop_count(frame->local, this, INCREMENT);
 
-        return;
+    STACK_WIND(frame, pl_truncate_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->ftruncate, fd, offset, xdata);
+    return 0;
 }
 
-
-int32_t
-pl_getxattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                 int32_t op_ret, int32_t op_errno, dict_t *dict, dict_t *xdata)
+int
+pl_truncate_cont(call_frame_t *frame, xlator_t *this, loc_t *loc, off_t offset,
+                 dict_t *xdata)
 {
-        STACK_UNWIND_STRICT (getxattr, frame, op_ret, op_errno, dict, xdata);
-        return 0;
+    pl_track_io_fop_count(frame->local, this, INCREMENT);
 
+    STACK_WIND(frame, pl_truncate_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->truncate, loc, offset, xdata);
+    return 0;
 }
 
-int32_t
-pl_getxattr (call_frame_t *frame, xlator_t *this, loc_t *loc,
-             const char *name, dict_t *xdata)
-{
-        int32_t                 op_errno        = EINVAL;
-        int                     op_ret          = -1;
-        int32_t                 bcount          = 0;
-        int32_t                 gcount          = 0;
-        char                    key[PATH_MAX]   = {0, };
-        char                    *lk_summary     = NULL;
-        pl_inode_t              *pl_inode       = NULL;
-        dict_t                  *dict           = NULL;
-        clrlk_args              args            = {0,};
-        char                    *brickname      = NULL;
-
-        if (!name)
-                goto usual;
-
-        if (strncmp (name, GF_XATTR_CLRLK_CMD, strlen (GF_XATTR_CLRLK_CMD)))
-                goto usual;
-
-        if (clrlk_parse_args (name, &args)) {
-                op_errno = EINVAL;
-                goto out;
-        }
+static int
+truncate_stat_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                  int32_t op_ret, int32_t op_errno, struct iatt *buf,
+                  dict_t *xdata)
+{
+    pl_local_t *local = frame->local;
+    inode_t *inode = NULL;
+    pl_inode_t *pl_inode = NULL;
+    pl_rw_req_t *rw = NULL;
+    posix_lock_t region = {
+        .list =
+            {
+                0,
+            },
+    };
+    gf_boolean_t enabled = _gf_false;
+    gf_boolean_t can_block = _gf_true;
+    int allowed = 1;
+
+    GF_VALIDATE_OR_GOTO("locks", this, unwind);
+
+    if (op_ret != 0) {
+        gf_log(this->name, GF_LOG_ERROR,
+               "got error (errno=%d, stderror=%s) from child", op_errno,
+               strerror(op_errno));
+        goto unwind;
+    }
+
+    if (local->op == GF_FOP_TRUNCATE)
+        inode = local->loc[0].inode;
+    else
+        inode = local->fd->inode;
+
+    local->inode = inode_ref(inode);
+
+    pl_inode = pl_inode_get(this, inode, local);
+    if (!pl_inode) {
+        op_ret = -1;
+        op_errno = ENOMEM;
+        goto unwind;
+    }
+
+    if (frame->root->pid < 0)
+        enabled = _gf_false;
+    else
+        enabled = pl_is_mandatory_locking_enabled(pl_inode);
+
+    if (enabled) {
+        region.fl_start = local->offset;
+        region.fl_end = LLONG_MAX;
+        region.client = frame->root->client;
+        region.fd_num = fd_to_fdnum(local->fd);
+        region.client_pid = frame->root->pid;
+        region.owner = frame->root->lk_owner;
+        pthread_mutex_lock(&pl_inode->mutex);
+        {
+            allowed = pl_is_fop_allowed(pl_inode, &region, local->fd, local->op,
+                                        &can_block);
 
-        dict = dict_new ();
-        if (!dict) {
-                op_errno = ENOMEM;
-                goto out;
-        }
+            if (allowed == 1) {
+                if (pl_inode->mlock_enforced &&
+                    pl_inode->track_fop_wind_count) {
+                    pl_inode->fop_wind_count++;
+                }
+                goto unlock;
+            } else if (!can_block) {
+                op_errno = EAGAIN;
+                op_ret = -1;
+                goto unlock;
+            }
 
-        pl_inode = pl_inode_get (this, loc->inode);
-        if (!pl_inode) {
+            rw = GF_MALLOC(sizeof(*rw), gf_locks_mt_pl_rw_req_t);
+            if (!rw) {
                 op_errno = ENOMEM;
-                goto out;
-        }
+                op_ret = -1;
+                goto unlock;
+            }
+
+            if (local->op == GF_FOP_TRUNCATE)
+                rw->stub = fop_truncate_stub(frame, pl_truncate_cont,
+                                             &local->loc[0], local->offset,
+                                             local->xdata);
+            else
+                rw->stub = fop_ftruncate_stub(frame, pl_ftruncate_cont,
+                                              local->fd, local->offset,
+                                              local->xdata);
+            if (!rw->stub) {
+                op_errno = ENOMEM;
+                op_ret = -1;
+                GF_FREE(rw);
+                goto unlock;
+            }
 
-        switch (args.type) {
-                case CLRLK_INODE:
-                case CLRLK_ENTRY:
-                        op_ret = clrlk_clear_lks_in_all_domains (this, pl_inode,
-                                                                 &args, &bcount,
-                                                                 &gcount,
-                                                                 &op_errno);
-                        if (op_ret)
-                                goto out;
-                        break;
-                case CLRLK_POSIX:
-                        op_ret = clrlk_clear_posixlk (this, pl_inode, &args,
-                                                      &bcount, &gcount,
-                                                      &op_errno);
-                        if (op_ret)
-                                goto out;
-                        break;
-                case CLRLK_TYPE_MAX:
-                        op_errno = EINVAL;
-                        goto out;
-        }
+            rw->region = region;
 
-        op_ret = fetch_pathinfo (this, loc->inode, &op_errno, &brickname);
-        if (op_ret) {
-                gf_log (this->name, GF_LOG_WARNING,
-                        "Couldn't get brickname");
-        } else {
-                op_ret = format_brickname(brickname);
-                if (op_ret) {
-                        gf_log (this->name, GF_LOG_WARNING,
-                                "Couldn't format brickname");
-                        GF_FREE(brickname);
-                        brickname = NULL;
-                }
+            list_add_tail(&rw->list, &pl_inode->rw_list);
         }
+    unlock:
+        pthread_mutex_unlock(&pl_inode->mutex);
+    }
 
-        if (!gcount && !bcount) {
-                if (gf_asprintf (&lk_summary, "No locks cleared.") == -1) {
-                        op_ret = -1;
-                        op_errno = ENOMEM;
-                        goto out;
-                }
-        } else if (gf_asprintf (&lk_summary, "%s: %s blocked locks=%d "
-                                "granted locks=%d",
-                                (brickname == NULL)? this->name : brickname,
-                                (args.type == CLRLK_INODE)? "inode":
-                                (args.type == CLRLK_ENTRY)? "entry":
-                                (args.type == CLRLK_POSIX)? "posix": " ",
-                                bcount, gcount) == -1) {
-                op_ret = -1;
-                op_errno = ENOMEM;
-                goto out;
+    if (allowed == 1) {
+        switch (local->op) {
+            case GF_FOP_TRUNCATE:
+                STACK_WIND(frame, pl_truncate_cbk, FIRST_CHILD(this),
+                           FIRST_CHILD(this)->fops->truncate, &local->loc[0],
+                           local->offset, local->xdata);
+                break;
+            case GF_FOP_FTRUNCATE:
+                STACK_WIND(frame, pl_truncate_cbk, FIRST_CHILD(this),
+                           FIRST_CHILD(this)->fops->ftruncate, local->fd,
+                           local->offset, local->xdata);
+                break;
+            default:
+                break;
         }
+    }
+unwind:
+    if (op_ret == -1) {
+        gf_log(this ? this->name : "locks", GF_LOG_ERROR,
+               "truncate failed with "
+               "ret: %d, error: %s",
+               op_ret, strerror(op_errno));
 
-        strncpy (key, name, strlen (name));
-        if (dict_set_dynstr (dict, key, lk_summary)) {
-                op_ret = -1;
-                op_errno = ENOMEM;
-                goto out;
+        switch (local->op) {
+            case GF_FOP_TRUNCATE:
+                PL_STACK_UNWIND(truncate, xdata, frame, op_ret, op_errno, buf,
+                                NULL, xdata);
+                break;
+            case GF_FOP_FTRUNCATE:
+                PL_STACK_UNWIND(ftruncate, xdata, frame, op_ret, op_errno, buf,
+                                NULL, xdata);
+                break;
+            default:
+                break;
         }
+    }
+    return 0;
+}
 
-        op_ret = 0;
-out:
-        GF_FREE(brickname);
-        STACK_UNWIND_STRICT (getxattr, frame, op_ret, op_errno, dict, xdata);
+int
+pl_truncate(call_frame_t *frame, xlator_t *this, loc_t *loc, off_t offset,
+            dict_t *xdata)
+{
+    pl_local_t *local = NULL;
+    int ret = -1;
 
-        GF_FREE (args.opts);
-        if (op_ret && lk_summary)
-                GF_FREE (lk_summary);
-        if (dict)
-                dict_unref (dict);
-        return 0;
+    GF_VALIDATE_OR_GOTO("locks", this, unwind);
 
-usual:
-        STACK_WIND (frame, pl_getxattr_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->getxattr, loc, name, xdata);
-        return 0;
+    local = mem_get0(this->local_pool);
+    GF_VALIDATE_OR_GOTO(this->name, local, unwind);
+
+    local->op = GF_FOP_TRUNCATE;
+    local->offset = offset;
+    loc_copy(&local->loc[0], loc);
+    if (xdata)
+        local->xdata = dict_ref(xdata);
+
+    frame->local = local;
+
+    STACK_WIND(frame, truncate_stat_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->stat, loc, NULL);
+    ret = 0;
+
+unwind:
+    if (ret == -1) {
+        gf_log(this ? this->name : "locks", GF_LOG_ERROR,
+               "truncate on %s failed with"
+               " ret: %d, error: %s",
+               loc->path, -1, strerror(ENOMEM));
+        STACK_UNWIND_STRICT(truncate, frame, -1, ENOMEM, NULL, NULL, NULL);
+    }
+    return 0;
 }
 
-static int
-format_brickname(char *brickname)
+int
+pl_ftruncate(call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
+             dict_t *xdata)
 {
-       int   ret             = -1;
-       char *hostname        = NULL;
-       char *volume          = NULL;
-       char *saveptr         = NULL;
+    pl_local_t *local = NULL;
+    int ret = -1;
 
-       if (!brickname)
-               goto out;
+    GF_VALIDATE_OR_GOTO("locks", this, unwind);
+    local = mem_get0(this->local_pool);
+    GF_VALIDATE_OR_GOTO(this->name, local, unwind);
 
-       strtok_r(brickname, ":", &saveptr);
-       hostname = gf_strdup(strtok_r(NULL, ":", &saveptr));
-       if (hostname == NULL)
-               goto out;
-       volume = gf_strdup(strtok_r(NULL, ".", &saveptr));
-       if (volume == NULL)
-               goto out;
+    local->op = GF_FOP_FTRUNCATE;
+    local->offset = offset;
+    local->fd = fd_ref(fd);
+    if (xdata)
+        local->xdata = dict_ref(xdata);
 
-       sprintf(brickname, "%s:%s", hostname, volume);
+    frame->local = local;
 
-       ret = 0;
-out:
-       GF_FREE(hostname);
-       GF_FREE(volume);
-       return ret;
+    STACK_WIND(frame, truncate_stat_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->fstat, fd, xdata);
+    ret = 0;
+unwind:
+    if (ret == -1) {
+        gf_log(this ? this->name : "locks", GF_LOG_ERROR,
+               "ftruncate failed with"
+               " ret: %d, error: %s",
+               -1, strerror(ENOMEM));
+        STACK_UNWIND_STRICT(ftruncate, frame, -1, ENOMEM, NULL, NULL, NULL);
+    }
+    return 0;
 }
 
-static int
-fetch_pathinfo (xlator_t *this, inode_t *inode, int32_t *op_errno,
-                char **brickname)
+int
+pl_locks_by_fd(pl_inode_t *pl_inode, fd_t *fd)
 {
-        int                    ret       = -1;
-        loc_t                  loc       = {0, };
-        dict_t                *dict      = NULL;
+    posix_lock_t *l = NULL;
+    int found = 0;
 
-        if (!brickname)
-                goto out;
+    pthread_mutex_lock(&pl_inode->mutex);
+    {
+        list_for_each_entry(l, &pl_inode->ext_list, list)
+        {
+            if (l->fd_num == fd_to_fdnum(fd)) {
+                found = 1;
+                break;
+            }
+        }
+    }
+    pthread_mutex_unlock(&pl_inode->mutex);
+    return found;
+}
 
-        if (!op_errno)
-                goto out;
+static void
+delete_locks_of_fd(xlator_t *this, pl_inode_t *pl_inode, fd_t *fd)
+{
+    posix_lock_t *tmp = NULL;
+    posix_lock_t *l = NULL;
 
-        gf_uuid_copy (loc.gfid, inode->gfid);
-        loc.inode = inode_ref (inode);
+    struct list_head blocked_list;
 
-        ret = syncop_getxattr (FIRST_CHILD(this), &loc, &dict,
-                               GF_XATTR_PATHINFO_KEY, NULL, NULL);
-        if (ret < 0) {
-                *op_errno = -ret;
-                ret = -1;
-                goto out;
+    INIT_LIST_HEAD(&blocked_list);
+
+    pthread_mutex_lock(&pl_inode->mutex);
+    {
+        list_for_each_entry_safe(l, tmp, &pl_inode->ext_list, list)
+        {
+            if (l->fd_num == fd_to_fdnum(fd)) {
+                if (l->blocked) {
+                    list_move_tail(&l->list, &blocked_list);
+                    continue;
+                }
+                __delete_lock(l);
+                __destroy_lock(l);
+            }
         }
+    }
+    pthread_mutex_unlock(&pl_inode->mutex);
 
-        ret = dict_get_str (dict, GF_XATTR_PATHINFO_KEY, brickname);
-        if (ret)
-                goto out;
+    list_for_each_entry_safe(l, tmp, &blocked_list, list)
+    {
+        list_del_init(&l->list);
+        STACK_UNWIND_STRICT(lk, l->frame, -1, EAGAIN, &l->user_flock, NULL);
+        __destroy_lock(l);
+    }
 
-        *brickname = gf_strdup(*brickname);
-        if (*brickname == NULL) {
-                ret = -1;
-                goto out;
-        }
+    grant_blocked_locks(this, pl_inode);
 
-        ret = 0;
-out:
-        if (dict != NULL) {
-                dict_unref (dict);
+    do_blocked_rw(pl_inode);
+}
+
+static void
+__delete_locks_of_owner(pl_inode_t *pl_inode, client_t *client,
+                        gf_lkowner_t *owner)
+{
+    posix_lock_t *tmp = NULL;
+    posix_lock_t *l = NULL;
+
+    /* TODO: what if it is a blocked lock with pending l->frame */
+
+    list_for_each_entry_safe(l, tmp, &pl_inode->ext_list, list)
+    {
+        if (l->blocked)
+            continue;
+        if ((l->client == client) && is_same_lkowner(&l->owner, owner)) {
+            gf_log("posix-locks", GF_LOG_TRACE,
+                   " Flushing lock"
+                   "%s (pid=%d) (lk-owner=%s) %" PRId64 " - %" PRId64
+                   " state: %s",
+                   l->fl_type == F_UNLCK ? "Unlock" : "Lock", l->client_pid,
+                   lkowner_utoa(&l->owner), l->user_flock.l_start,
+                   l->user_flock.l_len, l->blocked == 1 ? "Blocked" : "Active");
+
+            __delete_lock(l);
+            __destroy_lock(l);
         }
-        loc_wipe(&loc);
+    }
 
-        return ret;
+    return;
 }
 
+int32_t
+pl_getxattr_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                int32_t op_ret, int32_t op_errno, dict_t *dict, dict_t *xdata)
+{
+    STACK_UNWIND_STRICT(getxattr, frame, op_ret, op_errno, dict, xdata);
+    return 0;
+}
 
-int
-pl_lockinfo_get_brickname (xlator_t *this, inode_t *inode, int32_t *op_errno)
+static int32_t
+pl_getxattr_clrlk(xlator_t *this, const char *name, inode_t *inode,
+                  dict_t **dict, int32_t *op_errno)
 {
-        int                    ret       = -1;
-        posix_locks_private_t *priv      = NULL;
-        char                  *brickname = NULL;
-        char                  *end       = NULL;
-        char                  *tmp       = NULL;
+    int32_t bcount = 0;
+    int32_t gcount = 0;
+    char *key = NULL;
+    char *lk_summary = NULL;
+    pl_inode_t *pl_inode = NULL;
+    clrlk_args args = {
+        0,
+    };
+    char *brickname = NULL;
+    int32_t op_ret = -1;
+
+    *op_errno = EINVAL;
+
+    if (clrlk_parse_args(name, &args)) {
+        *op_errno = EINVAL;
+        goto out;
+    }
+
+    *dict = dict_new();
+    if (!*dict) {
+        *op_errno = ENOMEM;
+        goto out;
+    }
+
+    pl_inode = pl_inode_get(this, inode, NULL);
+    if (!pl_inode) {
+        *op_errno = ENOMEM;
+        goto out;
+    }
+
+    switch (args.type) {
+        case CLRLK_INODE:
+        case CLRLK_ENTRY:
+            op_ret = clrlk_clear_lks_in_all_domains(this, pl_inode, &args,
+                                                    &bcount, &gcount, op_errno);
+            break;
+        case CLRLK_POSIX:
+            op_ret = clrlk_clear_posixlk(this, pl_inode, &args, &bcount,
+                                         &gcount, op_errno);
+            break;
+        default:
+            op_ret = -1;
+            *op_errno = EINVAL;
+    }
+    if (op_ret) {
+        if (args.type >= CLRLK_TYPE_MAX) {
+            gf_log(this->name, GF_LOG_ERROR,
+                   "clear locks: invalid lock type %d", args.type);
+        } else {
+            gf_log(this->name, GF_LOG_ERROR,
+                   "clear locks of type %s failed: %s",
+                   clrlk_type_names[args.type], strerror(*op_errno));
+        }
 
-        priv = this->private;
+        goto out;
+    }
 
-        ret = fetch_pathinfo (this, inode, op_errno, &brickname);
-        if (ret)
-                goto out;
-
-        end = strrchr (brickname, ':');
-        if (!end) {
-                GF_FREE(brickname);
-                ret = -1;
-                goto out;
+    op_ret = fetch_pathinfo(this, inode, op_errno, &brickname);
+    if (op_ret) {
+        gf_log(this->name, GF_LOG_WARNING, "Couldn't get brickname");
+    } else {
+        op_ret = format_brickname(brickname);
+        if (op_ret) {
+            gf_log(this->name, GF_LOG_WARNING, "Couldn't format brickname");
+            GF_FREE(brickname);
+            brickname = NULL;
         }
+    }
 
-        tmp = brickname;
-        brickname = gf_strndup (brickname, (end - brickname));
-        if (brickname == NULL) {
-                ret = -1;
-                goto out;
+    if (!gcount && !bcount) {
+        if (gf_asprintf(&lk_summary, "No locks cleared.") == -1) {
+            op_ret = -1;
+            *op_errno = ENOMEM;
+            goto out;
         }
+    } else if (gf_asprintf(&lk_summary,
+                           "%s: %s blocked locks=%d "
+                           "granted locks=%d",
+                           (brickname == NULL) ? this->name : brickname,
+                           clrlk_type_names[args.type], bcount, gcount) == -1) {
+        op_ret = -1;
+        *op_errno = ENOMEM;
+        goto out;
+    }
+    gf_log(this->name, GF_LOG_DEBUG, "%s", lk_summary);
+
+    key = gf_strdup(name);
+    if (!key) {
+        op_ret = -1;
+        goto out;
+    }
+    if (dict_set_dynstr(*dict, key, lk_summary)) {
+        op_ret = -1;
+        *op_errno = ENOMEM;
+        goto out;
+    }
+
+    op_ret = 0;
 
-        priv->brickname = brickname;
-        ret = 0;
 out:
-        GF_FREE(tmp);
-        return ret;
+    GF_FREE(brickname);
+    GF_FREE(args.opts);
+    GF_FREE(key);
+    if (op_ret) {
+        GF_FREE(lk_summary);
+    }
+
+    return op_ret;
 }
 
-char *
-pl_lockinfo_key (xlator_t *this, inode_t *inode, int32_t *op_errno)
+int32_t
+pl_getxattr(call_frame_t *frame, xlator_t *this, loc_t *loc, const char *name,
+            dict_t *xdata)
 {
-        posix_locks_private_t *priv = NULL;
-        char                  *key  = NULL;
-        int                    ret = 0;
+    int32_t op_errno = EINVAL;
+    int32_t op_ret = -1;
+    dict_t *dict = NULL;
 
-        priv = this->private;
+    if (!name)
+        goto usual;
 
-        if (priv->brickname == NULL) {
-                ret = pl_lockinfo_get_brickname (this, inode, op_errno);
-                if (ret < 0) {
-                        gf_log (this->name, GF_LOG_WARNING,
-                                "cannot get brickname");
-                        goto out;
-                }
-        }
+    if (strncmp(name, GF_XATTR_CLRLK_CMD, SLEN(GF_XATTR_CLRLK_CMD)))
+        goto usual;
 
-        key = priv->brickname;
-out:
-        return key;
+    op_ret = pl_getxattr_clrlk(this, name, loc->inode, &dict, &op_errno);
+
+    STACK_UNWIND_STRICT(getxattr, frame, op_ret, op_errno, dict, xdata);
+
+    if (dict)
+        dict_unref(dict);
+    return 0;
+
+usual:
+    STACK_WIND(frame, pl_getxattr_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->getxattr, loc, name, xdata);
+    return 0;
 }
 
-int32_t
-pl_fgetxattr_handle_lockinfo (xlator_t *this, fd_t *fd,
-                              dict_t *dict, int32_t *op_errno)
+static int
+format_brickname(char *brickname)
 {
-        pl_inode_t    *pl_inode = NULL;
-        char          *key      = NULL, *buf = NULL;
-        int32_t        op_ret   = 0;
-        unsigned long  fdnum    = 0;
-	int32_t        len      = 0;
-        dict_t        *tmp      = NULL;
+    int ret = -1;
+    char *hostname = NULL;
+    char *volume = NULL;
+    char *saveptr = NULL;
 
-        pl_inode = pl_inode_get (this, fd->inode);
+    if (!brickname)
+        goto out;
 
-        if (!pl_inode) {
-                gf_log (this->name, GF_LOG_DEBUG, "Could not get inode.");
-                *op_errno = EBADFD;
-                op_ret = -1;
-                goto out;
-        }
+    strtok_r(brickname, ":", &saveptr);
+    hostname = gf_strdup(strtok_r(NULL, ":", &saveptr));
+    if (hostname == NULL)
+        goto out;
+    volume = gf_strdup(strtok_r(NULL, ".", &saveptr));
+    if (volume == NULL)
+        goto out;
 
-        if (!pl_locks_by_fd (pl_inode, fd)) {
-                op_ret = 0;
-                goto out;
-        }
+    sprintf(brickname, "%s:%s", hostname, volume);
 
-        fdnum = fd_to_fdnum (fd);
+    ret = 0;
+out:
+    GF_FREE(hostname);
+    GF_FREE(volume);
+    return ret;
+}
 
-        key = pl_lockinfo_key (this, fd->inode, op_errno);
-        if (key == NULL) {
-                op_ret = -1;
-                goto out;
-        }
+static int
+fetch_pathinfo(xlator_t *this, inode_t *inode, int32_t *op_errno,
+               char **brickname)
+{
+    int ret = -1;
+    loc_t loc = {
+        0,
+    };
+    dict_t *dict = NULL;
+
+    if (!brickname)
+        goto out;
+
+    if (!op_errno)
+        goto out;
+
+    gf_uuid_copy(loc.gfid, inode->gfid);
+    loc.inode = inode_ref(inode);
+
+    ret = syncop_getxattr(FIRST_CHILD(this), &loc, &dict, GF_XATTR_PATHINFO_KEY,
+                          NULL, NULL);
+    if (ret < 0) {
+        *op_errno = -ret;
+        ret = -1;
+        goto out;
+    }
+
+    ret = dict_get_str_sizen(dict, GF_XATTR_PATHINFO_KEY, brickname);
+    if (ret)
+        goto out;
+
+    *brickname = gf_strdup(*brickname);
+    if (*brickname == NULL) {
+        ret = -1;
+        goto out;
+    }
+
+    ret = 0;
+out:
+    if (dict != NULL) {
+        dict_unref(dict);
+    }
+    loc_wipe(&loc);
 
-        tmp = dict_new ();
-        if (tmp == NULL) {
-                op_ret = -1;
-                *op_errno = ENOMEM;
-                goto out;
-        }
+    return ret;
+}
 
-        op_ret = dict_set_uint64 (tmp, key, fdnum);
-        if (op_ret < 0) {
-                *op_errno = -op_ret;
-                op_ret = -1;
-                gf_log (this->name, GF_LOG_WARNING, "setting lockinfo value "
-                        "(%lu) for fd (ptr:%p inode-gfid:%s) failed (%s)",
-                        fdnum, fd, uuid_utoa (fd->inode->gfid),
-                        strerror (*op_errno));
-                goto out;
-        }
+int
+pl_lockinfo_get_brickname(xlator_t *this, inode_t *inode, int32_t *op_errno)
+{
+    posix_locks_private_t *priv = this->private;
+    char *brickname = NULL;
+    char *end = NULL;
+    char *tmp = NULL;
 
-        len = dict_serialized_length (tmp);
-        if (len < 0) {
-                *op_errno = -op_ret;
-                op_ret = -1;
-                gf_log (this->name, GF_LOG_WARNING,
-                        "dict_serialized_length failed (%s) while handling "
-                        "lockinfo for fd (ptr:%p inode-gfid:%s)",
-                        strerror (*op_errno), fd, uuid_utoa (fd->inode->gfid));
-                goto out;
-        }
+    int ret = fetch_pathinfo(this, inode, op_errno, &brickname);
+    if (ret)
+        goto out;
 
-        buf = GF_CALLOC (1, len, gf_common_mt_char);
-        if (buf == NULL) {
-                op_ret = -1;
-                *op_errno = ENOMEM;
-                goto out;
-        }
+    end = strrchr(brickname, ':');
+    if (!end) {
+        GF_FREE(brickname);
+        ret = -1;
+        goto out;
+    }
+
+    tmp = brickname;
+    brickname = gf_strndup(brickname, (end - brickname));
+    if (brickname == NULL) {
+        ret = -1;
+        goto out;
+    }
+
+    priv->brickname = brickname;
+    ret = 0;
+out:
+    GF_FREE(tmp);
+    return ret;
+}
 
-        op_ret = dict_serialize (tmp, buf);
-        if (op_ret < 0) {
-                *op_errno = -op_ret;
-                op_ret = -1;
-                gf_log (this->name, GF_LOG_WARNING,
-                        "dict_serialize failed (%s) while handling lockinfo "
-                        "for fd (ptr: %p inode-gfid:%s)", strerror (*op_errno),
-                        fd, uuid_utoa (fd->inode->gfid));
-                goto out;
-        }
+char *
+pl_lockinfo_key(xlator_t *this, inode_t *inode, int32_t *op_errno)
+{
+    posix_locks_private_t *priv = this->private;
+    char *key = NULL;
+    int ret = 0;
 
-        op_ret = dict_set_dynptr (dict, GF_XATTR_LOCKINFO_KEY, buf, len);
-        if (op_ret < 0) {
-                *op_errno = -op_ret;
-                op_ret = -1;
-                gf_log (this->name, GF_LOG_WARNING, "setting lockinfo value "
-                        "(%lu) for fd (ptr:%p inode-gfid:%s) failed (%s)",
-                        fdnum, fd, uuid_utoa (fd->inode->gfid),
-                        strerror (*op_errno));
-                goto out;
+    if (priv->brickname == NULL) {
+        ret = pl_lockinfo_get_brickname(this, inode, op_errno);
+        if (ret < 0) {
+            gf_log(this->name, GF_LOG_WARNING, "cannot get brickname");
+            goto out;
         }
+    }
 
-        buf = NULL;
+    key = priv->brickname;
 out:
-        if (tmp != NULL) {
-                dict_unref (tmp);
-        }
+    return key;
+}
 
-        if (buf != NULL) {
-                GF_FREE (buf);
-        }
+int32_t
+pl_fgetxattr_handle_lockinfo(xlator_t *this, fd_t *fd, dict_t *dict,
+                             int32_t *op_errno)
+{
+    char *key = NULL, *buf = NULL;
+    int32_t op_ret = 0;
+    unsigned long fdnum = 0;
+    int32_t len = 0;
+    dict_t *tmp = NULL;
+
+    pl_inode_t *pl_inode = pl_inode_get(this, fd->inode, NULL);
+
+    if (!pl_inode) {
+        gf_log(this->name, GF_LOG_DEBUG, "Could not get inode.");
+        *op_errno = EBADFD;
+        op_ret = -1;
+        goto out;
+    }
+
+    if (!pl_locks_by_fd(pl_inode, fd)) {
+        op_ret = 0;
+        goto out;
+    }
+
+    fdnum = fd_to_fdnum(fd);
+
+    key = pl_lockinfo_key(this, fd->inode, op_errno);
+    if (key == NULL) {
+        op_ret = -1;
+        goto out;
+    }
+
+    tmp = dict_new();
+    if (tmp == NULL) {
+        op_ret = -1;
+        *op_errno = ENOMEM;
+        goto out;
+    }
+
+    op_ret = dict_set_uint64(tmp, key, fdnum);
+    if (op_ret < 0) {
+        *op_errno = -op_ret;
+        op_ret = -1;
+        gf_log(this->name, GF_LOG_WARNING,
+               "setting lockinfo value "
+               "(%lu) for fd (ptr:%p inode-gfid:%s) failed (%s)",
+               fdnum, fd, uuid_utoa(fd->inode->gfid), strerror(*op_errno));
+        goto out;
+    }
+
+    op_ret = dict_allocate_and_serialize(tmp, (char **)&buf,
+                                         (unsigned int *)&len);
+    if (op_ret != 0) {
+        *op_errno = -op_ret;
+        op_ret = -1;
+        gf_log(this->name, GF_LOG_WARNING,
+               "dict_serialized_length failed (%s) while handling "
+               "lockinfo for fd (ptr:%p inode-gfid:%s)",
+               strerror(*op_errno), fd, uuid_utoa(fd->inode->gfid));
+        goto out;
+    }
+
+    op_ret = dict_set_dynptr(dict, GF_XATTR_LOCKINFO_KEY, buf, len);
+    if (op_ret < 0) {
+        *op_errno = -op_ret;
+        op_ret = -1;
+        gf_log(this->name, GF_LOG_WARNING,
+               "setting lockinfo value "
+               "(%lu) for fd (ptr:%p inode-gfid:%s) failed (%s)",
+               fdnum, fd, uuid_utoa(fd->inode->gfid), strerror(*op_errno));
+        goto out;
+    }
+
+    buf = NULL;
+out:
+    if (tmp != NULL) {
+        dict_unref(tmp);
+    }
 
-        return op_ret;
-}
+    if (buf != NULL) {
+        GF_FREE(buf);
+    }
 
+    return op_ret;
+}
 
 int32_t
-pl_fgetxattr (call_frame_t *frame, xlator_t *this, fd_t *fd,
-              const char *name, dict_t *xdata)
+pl_fgetxattr(call_frame_t *frame, xlator_t *this, fd_t *fd, const char *name,
+             dict_t *xdata)
 {
-        int32_t  op_ret = 0, op_errno = 0;
-        dict_t  *dict   = NULL;
-
-        if (!name) {
-                goto usual;
+    int32_t op_ret = 0, op_errno = 0;
+    dict_t *dict = NULL;
+
+    if (!name) {
+        goto usual;
+    }
+
+    if (strcmp(name, GF_XATTR_LOCKINFO_KEY) == 0) {
+        dict = dict_new();
+        if (dict == NULL) {
+            op_ret = -1;
+            op_errno = ENOMEM;
+            goto unwind;
         }
 
-        if (strcmp (name, GF_XATTR_LOCKINFO_KEY) == 0) {
-                dict = dict_new ();
-                if (dict == NULL) {
-                        op_ret = -1;
-                        op_errno = ENOMEM;
-                        goto unwind;
-                }
+        op_ret = pl_fgetxattr_handle_lockinfo(this, fd, dict, &op_errno);
+        if (op_ret < 0) {
+            gf_log(this->name, GF_LOG_WARNING,
+                   "getting lockinfo on fd (ptr:%p inode-gfid:%s) "
+                   "failed (%s)",
+                   fd, uuid_utoa(fd->inode->gfid), strerror(op_errno));
+        }
 
-                op_ret = pl_fgetxattr_handle_lockinfo (this, fd, dict,
-                                                       &op_errno);
-                if (op_ret < 0) {
-                        gf_log (this->name, GF_LOG_WARNING,
-                                "getting lockinfo on fd (ptr:%p inode-gfid:%s) "
-                                "failed (%s)", fd, uuid_utoa (fd->inode->gfid),
-                                strerror (op_errno));
-                }
+        goto unwind;
+    } else if (strncmp(name, GF_XATTR_CLRLK_CMD, SLEN(GF_XATTR_CLRLK_CMD)) ==
+               0) {
+        op_ret = pl_getxattr_clrlk(this, name, fd->inode, &dict, &op_errno);
 
-                goto unwind;
-        } else {
-                goto usual;
-        }
+        goto unwind;
+    } else {
+        goto usual;
+    }
 
 unwind:
-        STACK_UNWIND_STRICT (fgetxattr, frame, op_ret, op_errno, dict, NULL);
-        if (dict != NULL) {
-                dict_unref (dict);
-        }
+    STACK_UNWIND_STRICT(fgetxattr, frame, op_ret, op_errno, dict, NULL);
+    if (dict != NULL) {
+        dict_unref(dict);
+    }
 
-        return 0;
+    return 0;
 
 usual:
-        STACK_WIND (frame, default_fgetxattr_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->fgetxattr, fd, name, xdata);
-        return 0;
+    STACK_WIND(frame, default_fgetxattr_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->fgetxattr, fd, name, xdata);
+    return 0;
 }
 
 int32_t
-pl_migrate_locks (call_frame_t *frame, fd_t *newfd, uint64_t oldfd_num,
-                  int32_t *op_errno)
+pl_migrate_locks(call_frame_t *frame, fd_t *newfd, uint64_t oldfd_num,
+                 int32_t *op_errno)
 {
-        pl_inode_t   *pl_inode  = NULL;
-        uint64_t      newfd_num = 0;
-        posix_lock_t *l         = NULL;
-        int32_t       op_ret    = 0;
-
-        newfd_num = fd_to_fdnum (newfd);
-
-        pl_inode = pl_inode_get (frame->this, newfd->inode);
-        if (pl_inode == NULL) {
-                op_ret = -1;
-                *op_errno = EBADFD;
-                goto out;
-        }
-
-        pthread_mutex_lock (&pl_inode->mutex);
+    posix_lock_t *l = NULL;
+    int32_t op_ret = 0;
+    uint64_t newfd_num = fd_to_fdnum(newfd);
+
+    pl_inode_t *pl_inode = pl_inode_get(frame->this, newfd->inode, NULL);
+    if (pl_inode == NULL) {
+        op_ret = -1;
+        *op_errno = EBADFD;
+        goto out;
+    }
+
+    pthread_mutex_lock(&pl_inode->mutex);
+    {
+        list_for_each_entry(l, &pl_inode->ext_list, list)
         {
-                list_for_each_entry (l, &pl_inode->ext_list, list) {
-                        if (l->fd_num == oldfd_num) {
-                                l->fd_num = newfd_num;
-                                l->client = frame->root->client;
-                        }
-                }
+            if (l->fd_num == oldfd_num) {
+                l->fd_num = newfd_num;
+                l->client = frame->root->client;
+            }
         }
-        pthread_mutex_unlock (&pl_inode->mutex);
+    }
+    pthread_mutex_unlock(&pl_inode->mutex);
 
-        op_ret = 0;
+    op_ret = 0;
 out:
-        return op_ret;
+    return op_ret;
 }
 
 int32_t
-pl_fsetxattr_handle_lockinfo (call_frame_t *frame, fd_t *fd, char *lockinfo_buf,
-                              int len, int32_t *op_errno)
+pl_fsetxattr_handle_lockinfo(call_frame_t *frame, fd_t *fd, char *lockinfo_buf,
+                             int len, int32_t *op_errno)
 {
-        int32_t   op_ret    = -1;
-        dict_t   *lockinfo  = NULL;
-        uint64_t  oldfd_num = 0;
-        char     *key       = NULL;
-
-        lockinfo = dict_new ();
-        if (lockinfo == NULL) {
-                op_ret = -1;
-                *op_errno = ENOMEM;
-                goto out;
-        }
+    int32_t op_ret = -1;
+    uint64_t oldfd_num = 0;
+    char *key = NULL;
+
+    dict_t *lockinfo = dict_new();
+    if (lockinfo == NULL) {
+        op_ret = -1;
+        *op_errno = ENOMEM;
+        goto out;
+    }
+
+    op_ret = dict_unserialize(lockinfo_buf, len, &lockinfo);
+    if (op_ret < 0) {
+        *op_errno = -op_ret;
+        op_ret = -1;
+        goto out;
+    }
+
+    key = pl_lockinfo_key(frame->this, fd->inode, op_errno);
+    if (key == NULL) {
+        op_ret = -1;
+        goto out;
+    }
+
+    op_ret = dict_get_uint64(lockinfo, key, &oldfd_num);
+
+    if (oldfd_num == 0) {
+        op_ret = 0;
+        goto out;
+    }
+
+    op_ret = pl_migrate_locks(frame, fd, oldfd_num, op_errno);
+    if (op_ret < 0) {
+        gf_log(frame->this->name, GF_LOG_WARNING,
+               "migration of locks from oldfd (ptr:%p) to newfd "
+               "(ptr:%p) (inode-gfid:%s)",
+               (void *)(uintptr_t)oldfd_num, fd, uuid_utoa(fd->inode->gfid));
+        goto out;
+    }
 
-        op_ret = dict_unserialize (lockinfo_buf, len, &lockinfo);
-        if (op_ret < 0) {
-                *op_errno = -op_ret;
-                op_ret = -1;
-                goto out;
-        }
+out:
+    dict_unref(lockinfo);
 
-        key = pl_lockinfo_key (frame->this, fd->inode, op_errno);
-        if (key == NULL) {
-                op_ret = -1;
-                goto out;
-        }
+    return op_ret;
+}
 
-        op_ret = dict_get_uint64 (lockinfo, key, &oldfd_num);
+int32_t
+pl_fsetxattr_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                 int32_t op_ret, int32_t op_errno, dict_t *xdata)
+{
+    pl_local_t *local = NULL;
+    pl_inode_t *pl_inode = NULL;
 
-        if (oldfd_num == 0) {
-                op_ret = 0;
-                goto out;
+    local = frame->local;
+    if (local && local->update_mlock_enforced_flag && op_ret != -1) {
+        pl_inode = pl_inode_get(this, local->inode, NULL);
+        if (!pl_inode) {
+            op_ret = -1;
+            op_errno = ENOMEM;
+            goto unwind;
         }
 
-        op_ret = pl_migrate_locks (frame, fd, oldfd_num, op_errno);
-        if (op_ret < 0) {
-                gf_log (frame->this->name, GF_LOG_WARNING,
-                        "migration of locks from oldfd (ptr:%p) to newfd "
-                        "(ptr:%p) (inode-gfid:%s)", (void *)oldfd_num, fd,
-                        uuid_utoa (fd->inode->gfid));
-                goto out;
+        pthread_mutex_lock(&pl_inode->mutex);
+        {
+            pl_inode->mlock_enforced = _gf_true;
+            pl_inode->check_mlock_info = _gf_false;
         }
+        pthread_mutex_unlock(&pl_inode->mutex);
+    }
 
-out:
-        dict_unref (lockinfo);
-
-        return op_ret;
+unwind:
+    PL_STACK_UNWIND_FOR_CLIENT(fsetxattr, xdata, frame, op_ret, op_errno,
+                               xdata);
+    return 0;
 }
 
 int32_t
-pl_fsetxattr (call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *dict,
-              int32_t flags, dict_t *xdata)
+pl_fsetxattr(call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *dict,
+             int32_t flags, dict_t *xdata)
 {
-        int32_t  op_ret       = 0, op_errno = 0;
-        void    *lockinfo_buf = NULL;
-        int      len          = 0;
+    int32_t op_errno = 0;
+    void *lockinfo_buf = NULL;
+    int len = 0;
+    char *name = NULL;
+    posix_locks_private_t *priv = this->private;
+
+    int32_t op_ret = dict_get_ptr_and_len(dict, GF_XATTR_LOCKINFO_KEY,
+                                          &lockinfo_buf, &len);
+    if (lockinfo_buf == NULL) {
+        goto usual;
+    }
+
+    op_ret = pl_fsetxattr_handle_lockinfo(frame, fd, lockinfo_buf, len,
+                                          &op_errno);
+    if (op_ret < 0) {
+        goto unwind;
+    }
 
-        op_ret = dict_get_ptr_and_len (dict, GF_XATTR_LOCKINFO_KEY,
-                                       &lockinfo_buf, &len);
-        if (lockinfo_buf == NULL) {
-                goto usual;
-        }
+usual:
+    PL_LOCAL_GET_REQUESTS(frame, this, xdata, fd, NULL, NULL);
 
-        op_ret = pl_fsetxattr_handle_lockinfo (frame, fd, lockinfo_buf, len,
-                                               &op_errno);
-        if (op_ret < 0) {
-                goto unwind;
-        }
+    PL_CHECK_LOCK_ENFORCE_KEY(frame, dict, name, this, ((loc_t *)NULL), fd,
+                              priv);
 
-usual:
-        STACK_WIND (frame, default_fsetxattr_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->fsetxattr, fd, dict, flags, xdata);
-        return 0;
+    STACK_WIND(frame, pl_fsetxattr_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->fsetxattr, fd, dict, flags, xdata);
+    return 0;
 
 unwind:
-        STACK_UNWIND_STRICT (fsetxattr, frame, op_ret, op_errno, NULL);
-        return 0;
+    PL_STACK_UNWIND_FOR_CLIENT(fsetxattr, xdata, frame, op_ret, op_errno, NULL);
+
+    return 0;
 }
 
 int32_t
-pl_opendir_cbk (call_frame_t *frame,
-                void *cookie,
-                xlator_t *this,
-                int32_t op_ret,
-                int32_t op_errno,
-                fd_t *fd, dict_t *xdata)
+pl_opendir_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+               int32_t op_ret, int32_t op_errno, fd_t *fd, dict_t *xdata)
 {
-        pl_fdctx_t *fdctx = NULL;
+    pl_fdctx_t *fdctx = NULL;
 
-        if (op_ret < 0)
-                goto unwind;
+    if (op_ret < 0)
+        goto unwind;
 
-        fdctx = pl_check_n_create_fdctx (this, fd);
-        if (!fdctx) {
-                op_errno = ENOMEM;
-                op_ret   = -1;
-                goto unwind;
-        }
+    fdctx = pl_check_n_create_fdctx(this, fd);
+    if (!fdctx) {
+        op_errno = ENOMEM;
+        op_ret = -1;
+        goto unwind;
+    }
 
 unwind:
-        STACK_UNWIND_STRICT (opendir,
-                             frame,
-                             op_ret,
-                             op_errno,
-                             fd, xdata);
-        return 0;
+    PL_STACK_UNWIND(opendir, xdata, frame, op_ret, op_errno, fd, xdata);
+
+    return 0;
 }
 
 int32_t
-pl_opendir (call_frame_t *frame, xlator_t *this,
-             loc_t *loc, fd_t *fd, dict_t *xdata)
-{
-        STACK_WIND (frame,
-                    pl_opendir_cbk,
-                    FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->opendir,
-                    loc, fd, xdata);
-        return 0;
-
+pl_opendir(call_frame_t *frame, xlator_t *this, loc_t *loc, fd_t *fd,
+           dict_t *xdata)
+{
+    PL_LOCAL_GET_REQUESTS(frame, this, xdata, fd, NULL, NULL);
+    STACK_WIND(frame, pl_opendir_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->opendir, loc, fd, xdata);
+    return 0;
 }
 
 int
-pl_flush_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-              int32_t op_ret, int32_t op_errno, dict_t *xdata)
+pl_flush_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int32_t op_ret,
+             int32_t op_errno, dict_t *xdata)
 {
-        STACK_UNWIND_STRICT (flush, frame, op_ret, op_errno, xdata);
+    PL_STACK_UNWIND_FOR_CLIENT(flush, xdata, frame, op_ret, op_errno, xdata);
 
-        return 0;
+    return 0;
 }
 
-
 int
-pl_flush (call_frame_t *frame, xlator_t *this,
-          fd_t *fd, dict_t *xdata)
+pl_flush(call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *xdata)
 {
-        pl_inode_t *pl_inode = NULL;
-
-        pl_inode = pl_inode_get (this, fd->inode);
-
-        if (!pl_inode) {
-                gf_log (this->name, GF_LOG_DEBUG, "Could not get inode.");
-                STACK_UNWIND_STRICT (flush, frame, -1, EBADFD, NULL);
-                return 0;
-        }
-
-        pl_trace_flush (this, frame, fd);
-
-        if (frame->root->lk_owner.len == 0) {
-                /* Handle special case when protocol/server sets lk-owner to zero.
-                 * This usually happens due to a client disconnection. Hence, free
-                 * all locks opened with this fd.
-                 */
-                gf_log (this->name, GF_LOG_TRACE,
-                        "Releasing all locks with fd %p", fd);
-                delete_locks_of_fd (this, pl_inode, fd);
-                goto wind;
-
-        }
-        pthread_mutex_lock (&pl_inode->mutex);
-        {
-                __delete_locks_of_owner (pl_inode, frame->root->client,
-                                         &frame->root->lk_owner);
+    pl_inode_t *pl_inode = pl_inode_get(this, fd->inode, NULL);
+    if (!pl_inode) {
+        gf_log(this->name, GF_LOG_DEBUG, "Could not get inode.");
+        STACK_UNWIND_STRICT(flush, frame, -1, EBADFD, NULL);
+        return 0;
+    }
+
+    pthread_mutex_lock(&pl_inode->mutex);
+    {
+        if (pl_inode->migrated) {
+            pthread_mutex_unlock(&pl_inode->mutex);
+            STACK_UNWIND_STRICT(flush, frame, -1, EREMOTE, NULL);
+            return 0;
         }
-        pthread_mutex_unlock (&pl_inode->mutex);
-
-        grant_blocked_locks (this, pl_inode);
-
-        do_blocked_rw (pl_inode);
+    }
+    pthread_mutex_unlock(&pl_inode->mutex);
+
+    pl_trace_flush(this, frame, fd);
+
+    if (frame->root->lk_owner.len == 0) {
+        /* Handle special case when protocol/server sets lk-owner to zero.
+         * This usually happens due to a client disconnection. Hence, free
+         * all locks opened with this fd.
+         */
+        gf_log(this->name, GF_LOG_TRACE, "Releasing all locks with fd %p", fd);
+        delete_locks_of_fd(this, pl_inode, fd);
+        goto wind;
+    }
+    pthread_mutex_lock(&pl_inode->mutex);
+    {
+        __delete_locks_of_owner(pl_inode, frame->root->client,
+                                &frame->root->lk_owner);
+    }
+    pthread_mutex_unlock(&pl_inode->mutex);
+
+    grant_blocked_locks(this, pl_inode);
+
+    do_blocked_rw(pl_inode);
 
 wind:
-        STACK_WIND (frame, pl_flush_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->flush, fd, xdata);
-        return 0;
+    PL_LOCAL_GET_REQUESTS(frame, this, xdata, fd, NULL, NULL);
+    STACK_WIND(frame, pl_flush_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->flush, fd, xdata);
+    return 0;
 }
 
-
 int
-pl_open_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-             int32_t op_ret, int32_t op_errno, fd_t *fd, dict_t *xdata)
+pl_open_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int32_t op_ret,
+            int32_t op_errno, fd_t *fd, dict_t *xdata)
 {
-        pl_fdctx_t *fdctx = NULL;
+    pl_fdctx_t *fdctx = NULL;
 
-        if (op_ret < 0)
-                goto unwind;
+    if (op_ret < 0)
+        goto unwind;
 
-        fdctx = pl_check_n_create_fdctx (this, fd);
-        if (!fdctx) {
-                op_errno = ENOMEM;
-                op_ret   = -1;
-                goto unwind;
-        }
+    fdctx = pl_check_n_create_fdctx(this, fd);
+    if (!fdctx) {
+        op_errno = ENOMEM;
+        op_ret = -1;
+        goto unwind;
+    }
 
 unwind:
-        STACK_UNWIND_STRICT (open, frame, op_ret, op_errno, fd, xdata);
+    STACK_UNWIND_STRICT(open, frame, op_ret, op_errno, fd, xdata);
 
-        return 0;
+    return 0;
 }
 
-
 int
-pl_open (call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t flags,
-         fd_t *fd, dict_t *xdata)
+pl_open(call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t flags,
+        fd_t *fd, dict_t *xdata)
 {
-        STACK_WIND (frame, pl_open_cbk,
-                    FIRST_CHILD(this), FIRST_CHILD(this)->fops->open,
-                    loc, flags, fd, xdata);
+    int op_ret = -1;
+    int op_errno = EINVAL;
+    pl_inode_t *pl_inode = NULL;
+    posix_lock_t *l = NULL;
+    posix_locks_private_t *priv = this->private;
+
+    GF_VALIDATE_OR_GOTO("locks", this, unwind);
+
+    op_ret = 0, op_errno = 0;
+    pl_inode = pl_inode_get(this, fd->inode, NULL);
+    if (!pl_inode) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, ENOMEM, "Could not get inode");
+        op_ret = -1;
+        op_errno = ENOMEM;
+        goto unwind;
+    }
+
+    /* As per design, under forced and file-based mandatory locking modes
+     * it doesn't matter whether inodes's lock list contain advisory or
+     * mandatory type locks. So we just check whether inode's lock list is
+     * empty or not to make sure that no locks are being held for the file.
+     * Whereas under optimal mandatory locking mode, we strictly fail open
+     * if and only if lock list contain mandatory locks.
+     */
+    if (((priv->mandatory_mode == MLK_FILE_BASED) && pl_inode->mandatory) ||
+        priv->mandatory_mode == MLK_FORCED) {
+        if (fd->flags & O_TRUNC) {
+            pthread_mutex_lock(&pl_inode->mutex);
+            {
+                if (!list_empty(&pl_inode->ext_list)) {
+                    op_ret = -1;
+                    op_errno = EAGAIN;
+                }
+            }
+            pthread_mutex_unlock(&pl_inode->mutex);
+        }
+    } else if (priv->mandatory_mode == MLK_OPTIMAL) {
+        if (fd->flags & O_TRUNC) {
+            pthread_mutex_lock(&pl_inode->mutex);
+            {
+                list_for_each_entry(l, &pl_inode->ext_list, list)
+                {
+                    if ((l->lk_flags & GF_LK_MANDATORY)) {
+                        op_ret = -1;
+                        op_errno = EAGAIN;
+                        break;
+                    }
+                }
+            }
+            pthread_mutex_unlock(&pl_inode->mutex);
+        }
+    }
 
-        return 0;
+unwind:
+    if (op_ret == -1)
+        STACK_UNWIND_STRICT(open, frame, op_ret, op_errno, NULL, NULL);
+    else
+        STACK_WIND(frame, pl_open_cbk, FIRST_CHILD(this),
+                   FIRST_CHILD(this)->fops->open, loc, flags, fd, xdata);
+    return 0;
 }
 
-
 int
-pl_create_cbk (call_frame_t *frame, void *cookie,
-               xlator_t *this, int32_t op_ret, int32_t op_errno,
-               fd_t *fd, inode_t *inode, struct iatt *buf,
-               struct iatt *preparent, struct iatt *postparent, dict_t *xdata)
+pl_create_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int32_t op_ret,
+              int32_t op_errno, fd_t *fd, inode_t *inode, struct iatt *buf,
+              struct iatt *preparent, struct iatt *postparent, dict_t *xdata)
 {
-        pl_fdctx_t *fdctx = NULL;
+    pl_fdctx_t *fdctx = NULL;
 
-        if (op_ret < 0)
-                goto unwind;
+    if (op_ret < 0)
+        goto unwind;
 
-        fdctx = pl_check_n_create_fdctx (this, fd);
-        if (!fdctx) {
-                op_errno = ENOMEM;
-                op_ret   = -1;
-                goto unwind;
-        }
+    fdctx = pl_check_n_create_fdctx(this, fd);
+    if (!fdctx) {
+        op_errno = ENOMEM;
+        op_ret = -1;
+        goto unwind;
+    }
 
 unwind:
-        STACK_UNWIND_STRICT (create, frame, op_ret, op_errno, fd, inode, buf,
-                             preparent, postparent, xdata);
+    PL_STACK_UNWIND(create, xdata, frame, op_ret, op_errno, fd, inode, buf,
+                    preparent, postparent, xdata);
 
-        return 0;
+    return 0;
 }
 
-
 int
-pl_create (call_frame_t *frame, xlator_t *this,
-           loc_t *loc, int32_t flags, mode_t mode, mode_t umask, fd_t *fd,
-           dict_t *xdata)
+pl_create(call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t flags,
+          mode_t mode, mode_t umask, fd_t *fd, dict_t *xdata)
 {
-        STACK_WIND (frame, pl_create_cbk,
-                    FIRST_CHILD (this), FIRST_CHILD (this)->fops->create,
-                    loc, flags, mode, umask, fd, xdata);
-        return 0;
-}
+    PL_LOCAL_GET_REQUESTS(frame, this, xdata, fd, NULL, NULL);
 
+    STACK_WIND(frame, pl_create_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->create, loc, flags, mode, umask, fd,
+               xdata);
+    return 0;
+}
 
 int
-pl_readv_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-              int32_t op_ret, int32_t op_errno,
-              struct iovec *vector, int32_t count, struct iatt *stbuf,
-              struct iobref *iobref, dict_t *xdata)
+pl_readv_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int32_t op_ret,
+             int32_t op_errno, struct iovec *vector, int32_t count,
+             struct iatt *stbuf, struct iobref *iobref, dict_t *xdata)
 {
-        STACK_UNWIND_STRICT (readv, frame, op_ret, op_errno,
-                             vector, count, stbuf, iobref, xdata);
+    pl_track_io_fop_count(frame->local, this, DECREMENT);
 
-        return 0;
+    PL_STACK_UNWIND(readv, xdata, frame, op_ret, op_errno, vector, count, stbuf,
+                    iobref, xdata);
+
+    return 0;
 }
 
 int
-pl_writev_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-               int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
-               struct iatt *postbuf, dict_t *xdata)
+pl_writev_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int32_t op_ret,
+              int32_t op_errno, struct iatt *prebuf, struct iatt *postbuf,
+              dict_t *xdata)
 {
-        STACK_UNWIND_STRICT (writev, frame, op_ret, op_errno, prebuf, postbuf,
-                             xdata);
+    pl_track_io_fop_count(frame->local, this, DECREMENT);
 
-        return 0;
-}
+    PL_STACK_UNWIND(writev, xdata, frame, op_ret, op_errno, prebuf, postbuf,
+                    xdata);
 
+    return 0;
+}
 
 void
-do_blocked_rw (pl_inode_t *pl_inode)
+do_blocked_rw(pl_inode_t *pl_inode)
 {
-        struct list_head  wind_list;
-        pl_rw_req_t      *rw = NULL;
-        pl_rw_req_t      *tmp = NULL;
+    struct list_head wind_list;
+    pl_rw_req_t *rw = NULL;
+    pl_rw_req_t *tmp = NULL;
 
-        INIT_LIST_HEAD (&wind_list);
+    INIT_LIST_HEAD(&wind_list);
 
-        pthread_mutex_lock (&pl_inode->mutex);
+    pthread_mutex_lock(&pl_inode->mutex);
+    {
+        list_for_each_entry_safe(rw, tmp, &pl_inode->rw_list, list)
         {
-                list_for_each_entry_safe (rw, tmp, &pl_inode->rw_list, list) {
-                        if (__rw_allowable (pl_inode, &rw->region,
-                                            rw->stub->fop)) {
-                                list_del_init (&rw->list);
-                                list_add_tail (&rw->list, &wind_list);
-                        }
+            if (__rw_allowable(pl_inode, &rw->region, rw->stub->fop)) {
+                list_del_init(&rw->list);
+                list_add_tail(&rw->list, &wind_list);
+                if (pl_inode->mlock_enforced &&
+                    pl_inode->track_fop_wind_count) {
+                    pl_inode->fop_wind_count++;
                 }
+            }
         }
-        pthread_mutex_unlock (&pl_inode->mutex);
+    }
+    pthread_mutex_unlock(&pl_inode->mutex);
 
-        list_for_each_entry_safe (rw, tmp, &wind_list, list) {
-                list_del_init (&rw->list);
-                call_resume (rw->stub);
-                GF_FREE (rw);
-        }
+    list_for_each_entry_safe(rw, tmp, &wind_list, list)
+    {
+        list_del_init(&rw->list);
+        call_resume(rw->stub);
+        GF_FREE(rw);
+    }
 
-        return;
+    return;
 }
 
+/* when mandatory lock is enforced:
+    If an IO request comes on a region which is out of the boundary of the
+    granted mandatory lock, it will be rejected.
+
+    Note: There is no IO blocking with mandatory lock enforced as it may be
+    a stale data from an old client.
+ */
+gf_boolean_t static within_range(posix_lock_t *existing, posix_lock_t *new)
+{
+    if (existing->fl_start <= new->fl_start && existing->fl_end >= new->fl_end)
+        return _gf_true;
+
+    return _gf_false;
+}
 
 static int
-__rw_allowable (pl_inode_t *pl_inode, posix_lock_t *region,
-                glusterfs_fop_t op)
+__rw_allowable(pl_inode_t *pl_inode, posix_lock_t *region, glusterfs_fop_t op)
 {
-        posix_lock_t *l = NULL;
-        int           ret = 1;
+    posix_lock_t *l = NULL;
+    posix_locks_private_t *priv = THIS->private;
+    int ret = 1;
 
-        list_for_each_entry (l, &pl_inode->ext_list, list) {
-                if (locks_overlap (l, region) && !same_owner (l, region)) {
-                        if ((op == GF_FOP_READ) && (l->fl_type != F_WRLCK))
-                                continue;
-                        ret = 0;
-                        break;
+    if (pl_inode->mlock_enforced) {
+        list_for_each_entry(l, &pl_inode->ext_list, list)
+        {
+            /*
+                with lock enforced (fencing) there should not be any blocking
+                lock coexisting.
+            */
+            if (same_owner(l, region)) {
+                /* Should range check be strict for same owner with fencing? */
+                if (locks_overlap(l, region)) {
+                    if (within_range(l, region)) {
+                        return 1;
+                    } else {
+                        /*
+                        Should we allow read fop if it does not fit it in the
+                        range?
+                        if (op == GF_FOP_READ && l->fl_type != F_WRLCK) {
+                            return 1;
+                        }
+                        */
+                        return 0;
+                    }
+                }
+            } else {
+                if (locks_overlap(l, region)) {
+                    /*
+                    with fencing should a read from a different owner be
+                    allowed if the mandatory lock taken is F_RDLCK?
+                    if (op == GF_FOP_READ && l->fl_type != F_WRLCK) {
+                        return 1;
+                    }
+                    */
+                    return 0;
                 }
+            }
         }
 
-        return ret;
-}
+        /* No lock has been taken by this owner */
+        return 0;
+    }
+
+    list_for_each_entry(l, &pl_inode->ext_list, list)
+    {
+        if (!l->blocked && locks_overlap(l, region) && !same_owner(l, region)) {
+            if ((op == GF_FOP_READ) && (l->fl_type != F_WRLCK))
+                continue;
+            /* Check for mandatory lock under optimal
+             * mandatory-locking mode */
+            if (priv->mandatory_mode == MLK_OPTIMAL &&
+                !(l->lk_flags & GF_LK_MANDATORY))
+                continue;
+            ret = 0;
+            break;
+        }
+    }
 
+    return ret;
+}
 
 int
-pl_readv_cont (call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
-               off_t offset, uint32_t flags, dict_t *xdata)
+pl_readv_cont(call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
+              off_t offset, uint32_t flags, dict_t *xdata)
 {
-        STACK_WIND (frame, pl_readv_cbk,
-                    FIRST_CHILD (this), FIRST_CHILD (this)->fops->readv,
-                    fd, size, offset, flags, xdata);
+    pl_track_io_fop_count(frame->local, this, INCREMENT);
 
-        return 0;
-}
+    STACK_WIND(frame, pl_readv_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->readv, fd, size, offset, flags, xdata);
 
+    return 0;
+}
 
 int
-pl_readv (call_frame_t *frame, xlator_t *this,
-          fd_t *fd, size_t size, off_t offset, uint32_t flags, dict_t *xdata)
-{
-        posix_locks_private_t *priv = NULL;
-        pl_inode_t            *pl_inode = NULL;
-        pl_rw_req_t           *rw = NULL;
-        posix_lock_t           region = {.list = {0, }, };
-        int                    op_ret = 0;
-        int                    op_errno = 0;
-        char                   wind_needed = 1;
-
-
-        priv = this->private;
-        pl_inode = pl_inode_get (this, fd->inode);
-
-        if (priv->mandatory && pl_inode->mandatory) {
-                region.fl_start   = offset;
-                region.fl_end     = offset + size - 1;
-                region.client     = frame->root->client;
-                region.fd_num     = fd_to_fdnum(fd);
-                region.client_pid = frame->root->pid;
-                region.owner      = frame->root->lk_owner;
-
-                pthread_mutex_lock (&pl_inode->mutex);
-                {
-                        wind_needed = __rw_allowable (pl_inode, &region,
-                                                      GF_FOP_READ);
-                        if (wind_needed) {
-                                goto unlock;
-                        }
-
-                        if (fd->flags & O_NONBLOCK) {
-                                gf_log (this->name, GF_LOG_TRACE,
-                                        "returning EAGAIN as fd is O_NONBLOCK");
-                                op_errno = EAGAIN;
-                                op_ret = -1;
-                                goto unlock;
-                        }
-
-                        rw = GF_CALLOC (1, sizeof (*rw),
-                                        gf_locks_mt_pl_rw_req_t);
-                        if (!rw) {
-                                op_errno = ENOMEM;
-                                op_ret = -1;
-                                goto unlock;
-                        }
-
-                        rw->stub = fop_readv_stub (frame, pl_readv_cont,
-                                                   fd, size, offset, flags,
-                                                   xdata);
-                        if (!rw->stub) {
-                                op_errno = ENOMEM;
-                                op_ret = -1;
-                                GF_FREE (rw);
-                                goto unlock;
-                        }
+pl_readv(call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
+         off_t offset, uint32_t flags, dict_t *xdata)
+{
+    pl_local_t *local = NULL;
+    pl_inode_t *pl_inode = NULL;
+    pl_rw_req_t *rw = NULL;
+    posix_lock_t region = {
+        .list =
+            {
+                0,
+            },
+    };
+    gf_boolean_t enabled = _gf_false;
+    gf_boolean_t can_block = _gf_true;
+    int op_ret = 0;
+    int op_errno = 0;
+    int allowed = 1;
+
+    GF_VALIDATE_OR_GOTO("locks", this, unwind);
+
+    PL_LOCAL_GET_REQUESTS(frame, this, xdata, fd, NULL, NULL);
+
+    if (!frame->local) {
+        frame->local = mem_get0(this->local_pool);
+        local = frame->local;
+        local->inode = inode_ref(fd->inode);
+        local->fd = fd_ref(fd);
+    }
+
+    pl_inode = pl_inode_get(this, fd->inode, local);
+    if (!pl_inode) {
+        op_ret = -1;
+        op_errno = ENOMEM;
+        goto unwind;
+    }
+
+    if (frame->root->pid < 0)
+        enabled = _gf_false;
+    else
+        enabled = pl_is_mandatory_locking_enabled(pl_inode);
+
+    if (enabled) {
+        region.fl_start = offset;
+        region.fl_end = offset + size - 1;
+        region.client = frame->root->client;
+        region.fd_num = fd_to_fdnum(fd);
+        region.client_pid = frame->root->pid;
+        region.owner = frame->root->lk_owner;
+
+        pthread_mutex_lock(&pl_inode->mutex);
+        {
+            allowed = pl_is_fop_allowed(pl_inode, &region, fd, GF_FOP_READ,
+                                        &can_block);
+            if (allowed == 1) {
+                if (pl_inode->mlock_enforced &&
+                    pl_inode->track_fop_wind_count) {
+                    pl_inode->fop_wind_count++;
+                }
+                goto unlock;
+            } else if (!can_block) {
+                op_errno = EAGAIN;
+                op_ret = -1;
+                goto unlock;
+            }
 
-                        rw->region = region;
+            rw = GF_MALLOC(sizeof(*rw), gf_locks_mt_pl_rw_req_t);
+            if (!rw) {
+                op_errno = ENOMEM;
+                op_ret = -1;
+                goto unlock;
+            }
 
-                        list_add_tail (&rw->list, &pl_inode->rw_list);
-                }
-        unlock:
-                pthread_mutex_unlock (&pl_inode->mutex);
-        }
+            rw->stub = fop_readv_stub(frame, pl_readv_cont, fd, size, offset,
+                                      flags, xdata);
+            if (!rw->stub) {
+                op_errno = ENOMEM;
+                op_ret = -1;
+                GF_FREE(rw);
+                goto unlock;
+            }
 
+            rw->region = region;
 
-        if (wind_needed) {
-                STACK_WIND (frame, pl_readv_cbk,
-                            FIRST_CHILD (this), FIRST_CHILD (this)->fops->readv,
-                            fd, size, offset, flags, xdata);
+            list_add_tail(&rw->list, &pl_inode->rw_list);
         }
+    unlock:
+        pthread_mutex_unlock(&pl_inode->mutex);
+    }
+
+    if (allowed == 1) {
+        STACK_WIND(frame, pl_readv_cbk, FIRST_CHILD(this),
+                   FIRST_CHILD(this)->fops->readv, fd, size, offset, flags,
+                   xdata);
+    }
+unwind:
+    if (op_ret == -1)
+        PL_STACK_UNWIND(readv, xdata, frame, op_ret, op_errno, NULL, 0, NULL,
+                        NULL, NULL);
 
-        if (op_ret == -1)
-                STACK_UNWIND_STRICT (readv, frame, -1, op_errno,
-                                     NULL, 0, NULL, NULL, NULL);
-
-        return 0;
+    return 0;
 }
 
-
 int
-pl_writev_cont (call_frame_t *frame, xlator_t *this, fd_t *fd,
-                struct iovec *vector, int count, off_t offset,
-                uint32_t flags, struct iobref *iobref, dict_t *xdata)
+pl_writev_cont(call_frame_t *frame, xlator_t *this, fd_t *fd,
+               struct iovec *vector, int count, off_t offset, uint32_t flags,
+               struct iobref *iobref, dict_t *xdata)
 {
-        STACK_WIND (frame, pl_writev_cbk,
-                    FIRST_CHILD (this), FIRST_CHILD (this)->fops->writev,
-                    fd, vector, count, offset, flags, iobref, xdata);
+    pl_track_io_fop_count(frame->local, this, INCREMENT);
 
-        return 0;
-}
+    STACK_WIND(frame, pl_writev_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->writev, fd, vector, count, offset,
+               flags, iobref, xdata);
 
+    return 0;
+}
 
 int
-pl_writev (call_frame_t *frame, xlator_t *this, fd_t *fd,
-           struct iovec *vector, int32_t count, off_t offset,
-           uint32_t flags, struct iobref *iobref, dict_t *xdata)
-{
-        posix_locks_private_t *priv = NULL;
-        pl_inode_t            *pl_inode = NULL;
-        pl_rw_req_t           *rw = NULL;
-        posix_lock_t           region = {.list = {0, }, };
-        int                    op_ret = 0;
-        int                    op_errno = 0;
-        char                   wind_needed = 1;
-
-        priv = this->private;
-        pl_inode = pl_inode_get (this, fd->inode);
-
-        if (priv->mandatory && pl_inode->mandatory) {
-                region.fl_start   = offset;
-                region.fl_end     = offset + iov_length (vector, count) - 1;
-                region.client     = frame->root->client;
-                region.fd_num     = fd_to_fdnum(fd);
-                region.client_pid = frame->root->pid;
-                region.owner      = frame->root->lk_owner;
-
-                pthread_mutex_lock (&pl_inode->mutex);
-                {
-                        wind_needed = __rw_allowable (pl_inode, &region,
-                                                      GF_FOP_WRITE);
-                        if (wind_needed)
-                                goto unlock;
-
-                        if (fd->flags & O_NONBLOCK) {
-                                gf_log (this->name, GF_LOG_TRACE,
-                                        "returning EAGAIN because fd is "
-                                        "O_NONBLOCK");
-                                op_errno = EAGAIN;
-                                op_ret = -1;
-                                goto unlock;
-                        }
-
-                        rw = GF_CALLOC (1, sizeof (*rw),
-                                        gf_locks_mt_pl_rw_req_t);
-                        if (!rw) {
-                                op_errno = ENOMEM;
-                                op_ret = -1;
-                                goto unlock;
-                        }
-
-                        rw->stub = fop_writev_stub (frame, pl_writev_cont,
-                                                    fd, vector, count, offset,
-                                                    flags, iobref, xdata);
-                        if (!rw->stub) {
-                                op_errno = ENOMEM;
-                                op_ret = -1;
-                                GF_FREE (rw);
-                                goto unlock;
-                        }
+pl_writev(call_frame_t *frame, xlator_t *this, fd_t *fd, struct iovec *vector,
+          int32_t count, off_t offset, uint32_t flags, struct iobref *iobref,
+          dict_t *xdata)
+{
+    pl_local_t *local = NULL;
+    pl_inode_t *pl_inode = NULL;
+    pl_rw_req_t *rw = NULL;
+    posix_lock_t region = {
+        .list =
+            {
+                0,
+            },
+    };
+    gf_boolean_t enabled = _gf_false;
+    gf_boolean_t can_block = _gf_true;
+    int op_ret = 0;
+    int op_errno = 0;
+    int allowed = 1;
+
+    GF_VALIDATE_OR_GOTO("locks", this, unwind);
+
+    PL_LOCAL_GET_REQUESTS(frame, this, xdata, fd, NULL, NULL);
+
+    if (!frame->local) {
+        frame->local = mem_get0(this->local_pool);
+        local = frame->local;
+        local->inode = inode_ref(fd->inode);
+        local->fd = fd_ref(fd);
+    }
+
+    pl_inode = pl_inode_get(this, fd->inode, local);
+    if (!pl_inode) {
+        op_ret = -1;
+        op_errno = ENOMEM;
+        goto unwind;
+    }
+
+    if (frame->root->pid < 0)
+        enabled = _gf_false;
+    else
+        enabled = pl_is_mandatory_locking_enabled(pl_inode);
+
+    if (enabled) {
+        region.fl_start = offset;
+        region.fl_end = offset + iov_length(vector, count) - 1;
+        region.client = frame->root->client;
+        region.fd_num = fd_to_fdnum(fd);
+        region.client_pid = frame->root->pid;
+        region.owner = frame->root->lk_owner;
+
+        pthread_mutex_lock(&pl_inode->mutex);
+        {
+            allowed = pl_is_fop_allowed(pl_inode, &region, fd, GF_FOP_WRITE,
+                                        &can_block);
+            if (allowed == 1) {
+                if (pl_inode->mlock_enforced &&
+                    pl_inode->track_fop_wind_count) {
+                    pl_inode->fop_wind_count++;
+                }
+                goto unlock;
+            } else if (!can_block) {
+                if (pl_inode->mlock_enforced) {
+                    op_errno = EBUSY;
+                } else {
+                    op_errno = EAGAIN;
+                }
 
-                        rw->region = region;
+                op_ret = -1;
+                goto unlock;
+            }
 
-                        list_add_tail (&rw->list, &pl_inode->rw_list);
-                }
-        unlock:
-                pthread_mutex_unlock (&pl_inode->mutex);
-        }
+            rw = GF_MALLOC(sizeof(*rw), gf_locks_mt_pl_rw_req_t);
+            if (!rw) {
+                op_errno = ENOMEM;
+                op_ret = -1;
+                goto unlock;
+            }
 
+            rw->stub = fop_writev_stub(frame, pl_writev_cont, fd, vector, count,
+                                       offset, flags, iobref, xdata);
+            if (!rw->stub) {
+                op_errno = ENOMEM;
+                op_ret = -1;
+                GF_FREE(rw);
+                goto unlock;
+            }
 
-        if (wind_needed)
-                STACK_WIND (frame, pl_writev_cbk,
-                            FIRST_CHILD (this), FIRST_CHILD (this)->fops->writev,
-                            fd, vector, count, offset, flags, iobref, xdata);
+            rw->region = region;
 
-        if (op_ret == -1)
-                STACK_UNWIND_STRICT (writev, frame, -1, op_errno, NULL, NULL,
-                                     NULL);
+            list_add_tail(&rw->list, &pl_inode->rw_list);
+        }
+    unlock:
+        pthread_mutex_unlock(&pl_inode->mutex);
+    }
+
+    if (allowed == 1) {
+        STACK_WIND(frame, pl_writev_cbk, FIRST_CHILD(this),
+                   FIRST_CHILD(this)->fops->writev, fd, vector, count, offset,
+                   flags, iobref, xdata);
+    }
+unwind:
+    if (op_ret == -1)
+        PL_STACK_UNWIND(writev, xdata, frame, op_ret, op_errno, NULL, NULL,
+                        NULL);
 
-        return 0;
+    return 0;
 }
 
 static int
-__fd_has_locks (pl_inode_t *pl_inode, fd_t *fd)
+__fd_has_locks(pl_inode_t *pl_inode, fd_t *fd)
 {
-        int           found = 0;
-        posix_lock_t *l     = NULL;
+    posix_lock_t *l = NULL;
 
-        list_for_each_entry (l, &pl_inode->ext_list, list) {
-                if (l->fd_num == fd_to_fdnum(fd)) {
-                        found = 1;
-                        break;
-                }
+    list_for_each_entry(l, &pl_inode->ext_list, list)
+    {
+        if (l->fd_num == fd_to_fdnum(fd)) {
+            return 1;
         }
+    }
 
-        return found;
+    return 0;
 }
 
 static posix_lock_t *
-lock_dup (posix_lock_t *lock)
+lock_dup(posix_lock_t *lock)
 {
-        posix_lock_t *new_lock = NULL;
-
-        new_lock = new_posix_lock (&lock->user_flock, lock->client,
-                                   lock->client_pid, &lock->owner,
-                                   (fd_t *)lock->fd_num);
-        return new_lock;
+    int32_t op_errno = 0;
+    return new_posix_lock(&lock->user_flock, lock->client, lock->client_pid,
+                          &lock->owner, (fd_t *)lock->fd_num, lock->lk_flags,
+                          lock->blocking, &op_errno);
 }
 
 static int
-__dup_locks_to_fdctx (pl_inode_t *pl_inode, fd_t *fd,
-                      pl_fdctx_t *fdctx)
-{
-        posix_lock_t *l        = NULL;
-        posix_lock_t *duplock = NULL;
-        int ret = 0;
-
-        list_for_each_entry (l, &pl_inode->ext_list, list) {
-                if (l->fd_num == fd_to_fdnum(fd)) {
-                        duplock = lock_dup (l);
-                        if (!duplock) {
-                                ret = -1;
-                                break;
-                        }
+__dup_locks_to_fdctx(pl_inode_t *pl_inode, fd_t *fd, pl_fdctx_t *fdctx)
+{
+    posix_lock_t *l = NULL;
+    posix_lock_t *duplock = NULL;
+    int ret = 0;
+
+    list_for_each_entry(l, &pl_inode->ext_list, list)
+    {
+        if (l->fd_num == fd_to_fdnum(fd)) {
+            duplock = lock_dup(l);
+            if (!duplock) {
+                ret = -1;
+                break;
+            }
 
-                        list_add_tail (&duplock->list, &fdctx->locks_list);
-                }
+            list_add_tail(&duplock->list, &fdctx->locks_list);
         }
+    }
 
-        return ret;
+    return ret;
 }
 
 static int
-__copy_locks_to_fdctx (pl_inode_t *pl_inode, fd_t *fd,
-                      pl_fdctx_t *fdctx)
+__copy_locks_to_fdctx(pl_inode_t *pl_inode, fd_t *fd, pl_fdctx_t *fdctx)
 {
-        int ret = 0;
-
-        ret = __dup_locks_to_fdctx (pl_inode, fd, fdctx);
-        if (ret)
-                goto out;
-
-out:
-        return ret;
-
+    return __dup_locks_to_fdctx(pl_inode, fd, fdctx);
 }
 
 static void
-pl_mark_eol_lock (posix_lock_t *lock)
+pl_mark_eol_lock(posix_lock_t *lock)
 {
-        lock->user_flock.l_type = GF_LK_EOL;
-        return;
+    lock->user_flock.l_type = GF_LK_EOL;
+    return;
 }
 
 static posix_lock_t *
-__get_next_fdctx_lock (pl_fdctx_t *fdctx)
+__get_next_fdctx_lock(pl_fdctx_t *fdctx)
 {
-        posix_lock_t *lock = NULL;
+    posix_lock_t *lock = NULL;
 
-        GF_ASSERT (fdctx);
+    GF_ASSERT(fdctx);
 
-        if (list_empty (&fdctx->locks_list)) {
-                gf_log (THIS->name, GF_LOG_DEBUG,
-                        "fdctx lock list empty");
-                goto out;
-        }
+    if (list_empty(&fdctx->locks_list)) {
+        gf_log(THIS->name, GF_LOG_DEBUG, "fdctx lock list empty");
+        goto out;
+    }
 
-        lock = list_entry (fdctx->locks_list.next, typeof (*lock),
-                           list);
+    lock = list_entry(fdctx->locks_list.next, typeof(*lock), list);
 
-        GF_ASSERT (lock);
+    GF_ASSERT(lock);
 
-        list_del_init (&lock->list);
+    list_del_init(&lock->list);
 
 out:
-        return lock;
+    return lock;
 }
 
 static int
-__set_next_lock_fd (pl_fdctx_t *fdctx, posix_lock_t *reqlock)
+__set_next_lock_fd(pl_fdctx_t *fdctx, posix_lock_t *reqlock)
 {
-        posix_lock_t *lock  = NULL;
-        int           ret   = 0;
+    posix_lock_t *lock = NULL;
+    int ret = 0;
 
-        GF_ASSERT (fdctx);
+    GF_ASSERT(fdctx);
 
-        lock = __get_next_fdctx_lock (fdctx);
-        if (!lock) {
-                gf_log (THIS->name, GF_LOG_DEBUG,
-                        "marking EOL in reqlock");
-                pl_mark_eol_lock (reqlock);
-                goto out;
-        }
+    lock = __get_next_fdctx_lock(fdctx);
+    if (!lock) {
+        gf_log(THIS->name, GF_LOG_DEBUG, "marking EOL in reqlock");
+        pl_mark_eol_lock(reqlock);
+        goto out;
+    }
 
-        reqlock->user_flock  = lock->user_flock;
-        reqlock->fl_start    = lock->fl_start;
-        reqlock->fl_type     = lock->fl_type;
-        reqlock->fl_end      = lock->fl_end;
-        reqlock->owner       = lock->owner;
+    reqlock->user_flock = lock->user_flock;
+    reqlock->fl_start = lock->fl_start;
+    reqlock->fl_type = lock->fl_type;
+    reqlock->fl_end = lock->fl_end;
+    reqlock->owner = lock->owner;
 
 out:
-        if (lock)
-                __destroy_lock (lock);
+    if (lock)
+        __destroy_lock(lock);
 
-        return ret;
+    return ret;
 }
 
 static int
-pl_getlk_fd (xlator_t *this, pl_inode_t *pl_inode,
-             fd_t *fd, posix_lock_t *reqlock)
+pl_getlk_fd(xlator_t *this, pl_inode_t *pl_inode, fd_t *fd,
+            posix_lock_t *reqlock)
 {
-        uint64_t    tmp   = 0;
-        pl_fdctx_t *fdctx = NULL;
-        int         ret   = 0;
-
-        pthread_mutex_lock (&pl_inode->mutex);
-        {
-                if (!__fd_has_locks (pl_inode, fd)) {
-                        gf_log (this->name, GF_LOG_DEBUG,
-                                "fd=%p has no active locks", fd);
-                        ret = 0;
-                        goto unlock;
-                }
+    uint64_t tmp = 0;
+    pl_fdctx_t *fdctx = NULL;
+    int ret = 0;
+
+    pthread_mutex_lock(&pl_inode->mutex);
+    {
+        if (!__fd_has_locks(pl_inode, fd)) {
+            pthread_mutex_unlock(&pl_inode->mutex);
+            gf_log(this->name, GF_LOG_DEBUG, "fd=%p has no active locks", fd);
+            ret = 0;
+            goto out;
+        }
 
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "There are active locks on fd");
+        gf_log(this->name, GF_LOG_DEBUG, "There are active locks on fd");
 
-                ret = fd_ctx_get (fd, this, &tmp);
-                fdctx = (pl_fdctx_t *)(long) tmp;
+        ret = fd_ctx_get(fd, this, &tmp);
+        fdctx = (pl_fdctx_t *)(long)tmp;
 
-                if (list_empty (&fdctx->locks_list)) {
-                        gf_log (this->name, GF_LOG_TRACE,
-                                "no fdctx -> copying all locks on fd");
+        if (list_empty(&fdctx->locks_list)) {
+            gf_log(this->name, GF_LOG_TRACE,
+                   "no fdctx -> copying all locks on fd");
 
-                        ret = __copy_locks_to_fdctx (pl_inode, fd, fdctx);
-                        if (ret) {
-                                goto unlock;
-                        }
+            ret = __copy_locks_to_fdctx(pl_inode, fd, fdctx);
+            if (ret) {
+                goto unlock;
+            }
 
-                        ret = __set_next_lock_fd (fdctx, reqlock);
+            ret = __set_next_lock_fd(fdctx, reqlock);
 
-                } else {
-                        gf_log (this->name, GF_LOG_TRACE,
-                                "fdctx present -> returning the next lock");
-                        ret = __set_next_lock_fd (fdctx, reqlock);
-                        if (ret) {
-                                gf_log (this->name, GF_LOG_DEBUG,
-                                        "could not get next lock of fd");
-                                goto unlock;
-                        }
-                }
+        } else {
+            gf_log(this->name, GF_LOG_TRACE,
+                   "fdctx present -> returning the next lock");
+            ret = __set_next_lock_fd(fdctx, reqlock);
+            if (ret) {
+                pthread_mutex_unlock(&pl_inode->mutex);
+                gf_log(this->name, GF_LOG_DEBUG,
+                       "could not get next lock of fd");
+                goto out;
+            }
         }
+    }
 
 unlock:
-        pthread_mutex_unlock (&pl_inode->mutex);
-        return ret;
-
+    pthread_mutex_unlock(&pl_inode->mutex);
+out:
+    return ret;
 }
 
 int
-pl_lk (call_frame_t *frame, xlator_t *this,
-       fd_t *fd, int32_t cmd, struct gf_flock *flock, dict_t *xdata)
-{
-        pl_inode_t   *pl_inode   = NULL;
-        int           op_ret     = 0;
-        int           op_errno   = 0;
-        int           can_block  = 0;
-        posix_lock_t *reqlock    = NULL;
-        posix_lock_t *conf       = NULL;
-        int           ret        = 0;
-
-        if ((flock->l_start < 0) || (flock->l_len < 0)) {
-                op_ret = -1;
-                op_errno = EINVAL;
-                goto unwind;
-        }
+pl_metalock_is_active(pl_inode_t *pl_inode)
+{
+    if (list_empty(&pl_inode->metalk_list))
+        return 0;
+    else
+        return 1;
+}
 
-        pl_inode = pl_inode_get (this, fd->inode);
-        if (!pl_inode) {
-                op_ret = -1;
-                op_errno = ENOMEM;
-                goto unwind;
-        }
+void
+__pl_queue_lock(pl_inode_t *pl_inode, posix_lock_t *reqlock)
+{
+    list_add_tail(&reqlock->list, &pl_inode->queued_locks);
+}
 
-        reqlock = new_posix_lock (flock, frame->root->client, frame->root->pid,
-                                  &frame->root->lk_owner, fd);
+int
+pl_lk(call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t cmd,
+      struct gf_flock *flock, dict_t *xdata)
+{
+    pl_inode_t *pl_inode = NULL;
+    int op_ret = 0;
+    int op_errno = 0;
+    int can_block = 0;
+    posix_lock_t *reqlock = NULL;
+    posix_lock_t *conf = NULL;
+    uint32_t lk_flags = 0;
+    posix_locks_private_t *priv = this->private;
+    pl_local_t *local = NULL;
+    short lock_type = 0;
+
+    int ret = dict_get_uint32(xdata, GF_LOCK_MODE, &lk_flags);
+    if (ret == 0) {
+        if (priv->mandatory_mode == MLK_NONE)
+            gf_log(this->name, GF_LOG_DEBUG,
+                   "Lock flags received "
+                   "in a non-mandatory locking environment, "
+                   "continuing");
+        else
+            gf_log(this->name, GF_LOG_DEBUG,
+                   "Lock flags received, "
+                   "continuing");
+    }
+
+    if ((flock->l_start < 0) || ((flock->l_start + flock->l_len) < 0)) {
+        op_ret = -1;
+        op_errno = EINVAL;
+        goto unwind;
+    }
+
+    /* As per 'man 3 fcntl', the value of l_len may be
+     * negative. In such cases, lock request should be
+     * considered for the range starting at 'l_start+l_len'
+     * and ending at 'l_start-1'. Update the fields accordingly.
+     */
+    if (flock->l_len < 0) {
+        flock->l_start += flock->l_len;
+        flock->l_len = labs(flock->l_len);
+    }
+
+    local = mem_get0(this->local_pool);
+    if (!local) {
+        op_ret = -1;
+        op_errno = ENOMEM;
+        goto unwind;
+    } else {
+        frame->local = local;
+        local->fd = fd_ref(fd);
+    }
 
-        if (!reqlock) {
-                op_ret = -1;
-                op_errno = ENOMEM;
-                goto unwind;
-        }
+    pl_inode = pl_inode_get(this, fd->inode, local);
+    if (!pl_inode) {
+        op_ret = -1;
+        op_errno = ENOMEM;
+        goto unwind;
+    }
+
+    reqlock = new_posix_lock(flock, frame->root->client, frame->root->pid,
+                             &frame->root->lk_owner, fd, lk_flags, can_block,
+                             &op_errno);
 
-        pl_trace_in (this, frame, fd, NULL, cmd, flock, NULL);
+    if (!reqlock) {
+        op_ret = -1;
+        goto unwind;
+    }
 
-        switch (cmd) {
+    pl_trace_in(this, frame, fd, NULL, cmd, flock, NULL);
 
+    switch (cmd) {
         case F_RESLK_LCKW:
-                can_block = 1;
+            can_block = 1;
 
-                /* fall through */
+            /* fall through */
         case F_RESLK_LCK:
-                memcpy (&reqlock->user_flock, flock, sizeof (struct gf_flock));
-                reqlock->frame = frame;
-                reqlock->this = this;
+            reqlock->frame = frame;
+            reqlock->this = this;
 
-                ret = pl_reserve_setlk (this, pl_inode, reqlock,
-                                        can_block);
-                if (ret < 0) {
-                        if (can_block)
-                                goto out;
+            ret = pl_reserve_setlk(this, pl_inode, reqlock, can_block);
+            if (ret < 0) {
+                if (can_block)
+                    goto out;
 
-                        op_ret = -1;
-                        op_errno = -ret;
-                        __destroy_lock (reqlock);
-                        goto unwind;
-                }
-                /* Finally a getlk and return the call */
-                conf = pl_getlk (pl_inode, reqlock);
-                if (conf)
-                        posix_lock_to_flock (conf, flock);
-                break;
+                op_ret = -1;
+                op_errno = -ret;
+                __destroy_lock(reqlock);
+                goto unwind;
+            }
+            /* Finally a getlk and return the call */
+            conf = pl_getlk(pl_inode, reqlock);
+            if (conf)
+                posix_lock_to_flock(conf, flock);
+            break;
 
         case F_RESLK_UNLCK:
-                reqlock->frame = frame;
-                reqlock->this = this;
-                ret = pl_reserve_unlock (this, pl_inode, reqlock);
-                if (ret < 0) {
-                        op_ret = -1;
-                        op_errno = -ret;
-                }
-                __destroy_lock (reqlock);
-                goto unwind;
+            reqlock->frame = frame;
+            reqlock->this = this;
+            ret = pl_reserve_unlock(this, pl_inode, reqlock);
+            if (ret < 0) {
+                op_ret = -1;
+                op_errno = -ret;
+            }
+            __destroy_lock(reqlock);
+            goto unwind;
 
-                break;
+            break;
 
         case F_GETLK_FD:
-                reqlock->frame = frame;
-                reqlock->this = this;
-                ret = pl_verify_reservelk (this, pl_inode, reqlock, can_block);
-                GF_ASSERT (ret >= 0);
-
-                ret = pl_getlk_fd (this, pl_inode, fd, reqlock);
-                if (ret < 0) {
-                        gf_log (this->name, GF_LOG_DEBUG,
-                                "getting locks on fd failed");
-                        op_ret = -1;
-                        op_errno = ENOLCK;
-                        goto unwind;
-                }
+            reqlock->frame = frame;
+            reqlock->this = this;
+            ret = pl_verify_reservelk(this, pl_inode, reqlock, can_block);
+            GF_ASSERT(ret >= 0);
+
+            ret = pl_getlk_fd(this, pl_inode, fd, reqlock);
+            if (ret < 0) {
+                gf_log(this->name, GF_LOG_DEBUG, "getting locks on fd failed");
+                op_ret = -1;
+                op_errno = ENOLCK;
+                goto unwind;
+            }
 
-                gf_log (this->name, GF_LOG_TRACE,
-                        "Replying with a lock on fd for healing");
+            gf_log(this->name, GF_LOG_TRACE,
+                   "Replying with a lock on fd for healing");
 
-                posix_lock_to_flock (reqlock, flock);
-                __destroy_lock (reqlock);
+            posix_lock_to_flock(reqlock, flock);
+            __destroy_lock(reqlock);
 
-                break;
+            break;
 
 #if F_GETLK != F_GETLK64
         case F_GETLK64:
 #endif
         case F_GETLK:
-                conf = pl_getlk (pl_inode, reqlock);
-                posix_lock_to_flock (conf, flock);
-                __destroy_lock (reqlock);
+            conf = pl_getlk(pl_inode, reqlock);
+            posix_lock_to_flock(conf, flock);
+            __destroy_lock(reqlock);
 
-                break;
+            break;
 
 #if F_SETLKW != F_SETLKW64
         case F_SETLKW64:
 #endif
         case F_SETLKW:
-                can_block = 1;
-                reqlock->frame  = frame;
-                reqlock->this   = this;
-
-                /* fall through */
+            can_block = 1;
+            reqlock->frame = frame;
+            reqlock->this = this;
+            reqlock->blocking = can_block;
+            /* fall through */
 
 #if F_SETLK != F_SETLK64
         case F_SETLK64:
 #endif
         case F_SETLK:
-                memcpy (&reqlock->user_flock, flock, sizeof (struct gf_flock));
-                ret = pl_verify_reservelk (this, pl_inode, reqlock, can_block);
-                if (ret < 0) {
-                        gf_log (this->name, GF_LOG_TRACE,
-                                "Lock blocked due to conflicting reserve lock");
-                        goto out;
+            reqlock->frame = frame;
+            reqlock->this = this;
+            lock_type = flock->l_type;
+
+            pthread_mutex_lock(&pl_inode->mutex);
+            {
+                if (pl_inode->migrated) {
+                    op_errno = EREMOTE;
+                    pthread_mutex_unlock(&pl_inode->mutex);
+                    STACK_UNWIND_STRICT(lk, frame, -1, op_errno, flock, xdata);
+
+                    __destroy_lock(reqlock);
+                    goto out;
                 }
-                ret = pl_setlk (this, pl_inode, reqlock,
-                                can_block);
+            }
+            pthread_mutex_unlock(&pl_inode->mutex);
 
+            ret = pl_verify_reservelk(this, pl_inode, reqlock, can_block);
+            if (ret < 0) {
+                gf_log(this->name, GF_LOG_TRACE,
+                       "Lock blocked due to conflicting reserve lock");
+                goto out;
+            }
+
+            if (reqlock->fl_type != F_UNLCK && pl_inode->mlock_enforced) {
+                ret = pl_lock_preempt(pl_inode, reqlock);
                 if (ret == -1) {
-                        if ((can_block) && (F_UNLCK != flock->l_type)) {
-                                pl_trace_block (this, frame, fd, NULL, cmd, flock, NULL);
-                                goto out;
-                        }
-                        gf_log (this->name, GF_LOG_DEBUG, "returning EAGAIN");
-                        op_ret = -1;
-                        op_errno = EAGAIN;
-                        __destroy_lock (reqlock);
-
-                } else if ((0 == ret) && (F_UNLCK == flock->l_type)) {
-                        /* For NLM's last "unlock on fd" detection */
-                        if (pl_locks_by_fd (pl_inode, fd))
-                                flock->l_type = F_RDLCK;
-                        else
-                                flock->l_type = F_UNLCK;
+                    gf_log(this->name, GF_LOG_ERROR, "lock preempt failed");
+                    op_ret = -1;
+                    op_errno = EAGAIN;
+                    __destroy_lock(reqlock);
+                    goto out;
                 }
-        }
 
-unwind:
-        pl_trace_out (this, frame, fd, NULL, cmd, flock, op_ret, op_errno, NULL);
-        pl_update_refkeeper (this, fd->inode);
+                pl_trace_block(this, frame, fd, NULL, cmd, flock, NULL);
+                goto unwind;
+            }
 
+            ret = pl_setlk(this, pl_inode, reqlock, can_block);
+            if (ret == -1) {
+                if ((can_block) && (F_UNLCK != lock_type)) {
+                    goto out;
+                }
+                gf_log(this->name, GF_LOG_DEBUG, "returning EAGAIN");
+                op_ret = -1;
+                op_errno = EAGAIN;
+                __destroy_lock(reqlock);
+            } else if (ret == -2) {
+                goto out;
+            } else if ((0 == ret) && (F_UNLCK == flock->l_type)) {
+                /* For NLM's last "unlock on fd" detection */
+                if (pl_locks_by_fd(pl_inode, fd))
+                    flock->l_type = F_RDLCK;
+                else
+                    flock->l_type = F_UNLCK;
+            }
+    }
 
-        STACK_UNWIND_STRICT (lk, frame, op_ret, op_errno, flock, xdata);
+unwind:
+    pl_trace_out(this, frame, fd, NULL, cmd, flock, op_ret, op_errno, NULL);
+    pl_update_refkeeper(this, fd->inode);
+
+    PL_STACK_UNWIND(lk, xdata, frame, op_ret, op_errno, flock, xdata);
 out:
-        return 0;
+    return 0;
 }
 
-
 /* TODO: this function just logs, no action required?? */
 int
-pl_forget (xlator_t *this,
-           inode_t *inode)
+pl_forget(xlator_t *this, inode_t *inode)
 {
-        pl_inode_t   *pl_inode = NULL;
+    pl_inode_t *pl_inode = NULL;
 
-        posix_lock_t *ext_tmp = NULL;
-        posix_lock_t *ext_l   = NULL;
-        struct list_head posixlks_released;
+    posix_lock_t *ext_tmp = NULL;
+    posix_lock_t *ext_l = NULL;
+    struct list_head posixlks_released;
 
-        pl_inode_lock_t *ino_tmp = NULL;
-        pl_inode_lock_t *ino_l   = NULL;
-        struct list_head inodelks_released;
+    pl_inode_lock_t *ino_tmp = NULL;
+    pl_inode_lock_t *ino_l = NULL;
+    struct list_head inodelks_released;
 
-        pl_rw_req_t *rw_tmp = NULL;
-        pl_rw_req_t *rw_req = NULL;
+    pl_rw_req_t *rw_tmp = NULL;
+    pl_rw_req_t *rw_req = NULL;
 
-        pl_entry_lock_t *entry_tmp = NULL;
-        pl_entry_lock_t *entry_l   = NULL;
-        struct list_head entrylks_released;
+    pl_entry_lock_t *entry_tmp = NULL;
+    pl_entry_lock_t *entry_l = NULL;
+    struct list_head entrylks_released;
 
-        pl_dom_list_t *dom = NULL;
-        pl_dom_list_t *dom_tmp = NULL;
+    pl_dom_list_t *dom = NULL;
+    pl_dom_list_t *dom_tmp = NULL;
 
-        INIT_LIST_HEAD (&posixlks_released);
-        INIT_LIST_HEAD (&inodelks_released);
-        INIT_LIST_HEAD (&entrylks_released);
+    INIT_LIST_HEAD(&posixlks_released);
+    INIT_LIST_HEAD(&inodelks_released);
+    INIT_LIST_HEAD(&entrylks_released);
 
-        pl_inode = pl_inode_get (this, inode);
+    pl_inode = pl_inode_get(this, inode, NULL);
+    if (!pl_inode)
+        return 0;
 
-        pthread_mutex_lock (&pl_inode->mutex);
-        {
+    pthread_mutex_lock(&pl_inode->mutex);
+    {
+        if (!list_empty(&pl_inode->rw_list)) {
+            gf_log(this->name, GF_LOG_WARNING,
+                   "Pending R/W requests found, releasing.");
+
+            list_for_each_entry_safe(rw_req, rw_tmp, &pl_inode->rw_list, list)
+            {
+                list_del(&rw_req->list);
+                call_stub_destroy(rw_req->stub);
+                GF_FREE(rw_req);
+            }
+        }
 
-                if (!list_empty (&pl_inode->rw_list)) {
-                        gf_log (this->name, GF_LOG_WARNING,
-                                "Pending R/W requests found, releasing.");
+        if (!list_empty(&pl_inode->ext_list)) {
+            gf_log(this->name, GF_LOG_WARNING,
+                   "Pending fcntl locks found, releasing.");
 
-                        list_for_each_entry_safe (rw_req, rw_tmp, &pl_inode->rw_list,
-                                                  list) {
+            list_for_each_entry_safe(ext_l, ext_tmp, &pl_inode->ext_list, list)
+            {
+                __delete_lock(ext_l);
+                if (ext_l->blocked) {
+                    list_add_tail(&ext_l->list, &posixlks_released);
+                    continue;
+                }
+                __destroy_lock(ext_l);
+            }
+        }
 
-                                list_del (&rw_req->list);
-                                GF_FREE (rw_req);
-                        }
+        list_for_each_entry_safe(dom, dom_tmp, &pl_inode->dom_list, inode_list)
+        {
+            if (!list_empty(&dom->inodelk_list)) {
+                gf_log(this->name, GF_LOG_WARNING,
+                       "Pending inode locks found, releasing.");
+
+                list_for_each_entry_safe(ino_l, ino_tmp, &dom->inodelk_list,
+                                         list)
+                {
+                    __delete_inode_lock(ino_l);
+                    __pl_inodelk_unref(ino_l);
                 }
 
-                if (!list_empty (&pl_inode->ext_list)) {
-                        gf_log (this->name, GF_LOG_WARNING,
-                                "Pending fcntl locks found, releasing.");
+                list_splice_init(&dom->blocked_inodelks, &inodelks_released);
+            }
+            if (!list_empty(&dom->entrylk_list)) {
+                gf_log(this->name, GF_LOG_WARNING,
+                       "Pending entry locks found, releasing.");
 
-                        list_for_each_entry_safe (ext_l, ext_tmp, &pl_inode->ext_list,
-                                                  list) {
+                list_for_each_entry_safe(entry_l, entry_tmp, &dom->entrylk_list,
+                                         domain_list)
+                {
+                    list_del_init(&entry_l->domain_list);
 
-                                __delete_lock (pl_inode, ext_l);
-                                if (ext_l->blocked) {
-                                        list_add_tail (&ext_l->list, &posixlks_released);
-                                        continue;
-                                }
-                                __destroy_lock (ext_l);
-                        }
+                    GF_FREE((char *)entry_l->basename);
+                    GF_FREE(entry_l->connection_id);
+                    GF_FREE(entry_l);
                 }
 
+                list_splice_init(&dom->blocked_entrylks, &entrylks_released);
+            }
 
-                list_for_each_entry_safe (dom, dom_tmp, &pl_inode->dom_list, inode_list) {
+            list_del(&dom->inode_list);
+            gf_log("posix-locks", GF_LOG_TRACE, " Cleaning up domain: %s",
+                   dom->domain);
+            GF_FREE((char *)(dom->domain));
+            GF_FREE(dom);
+        }
+    }
+    pthread_mutex_unlock(&pl_inode->mutex);
 
-                        if (!list_empty (&dom->inodelk_list)) {
-                                gf_log (this->name, GF_LOG_WARNING,
-                                        "Pending inode locks found, releasing.");
+    if (!list_empty(&posixlks_released)) {
+        list_for_each_entry_safe(ext_l, ext_tmp, &posixlks_released, list)
+        {
+            STACK_UNWIND_STRICT(lk, ext_l->frame, -1, 0, &ext_l->user_flock,
+                                NULL);
+            __destroy_lock(ext_l);
+        }
+    }
 
-                                list_for_each_entry_safe (ino_l, ino_tmp, &dom->inodelk_list, list) {
-                                        __delete_inode_lock (ino_l);
-                                        __pl_inodelk_unref (ino_l);
-                                }
+    if (!list_empty(&inodelks_released)) {
+        list_for_each_entry_safe(ino_l, ino_tmp, &inodelks_released,
+                                 blocked_locks)
+        {
+            STACK_UNWIND_STRICT(inodelk, ino_l->frame, -1, 0, NULL);
+            __pl_inodelk_unref(ino_l);
+        }
+    }
 
-                                list_splice_init (&dom->blocked_inodelks, &inodelks_released);
+    if (!list_empty(&entrylks_released)) {
+        list_for_each_entry_safe(entry_l, entry_tmp, &entrylks_released,
+                                 blocked_locks)
+        {
+            STACK_UNWIND_STRICT(entrylk, entry_l->frame, -1, 0, NULL);
+            GF_FREE((char *)entry_l->basename);
+            GF_FREE(entry_l->connection_id);
+            GF_FREE(entry_l);
+        }
+    }
 
+    pthread_mutex_destroy(&pl_inode->mutex);
 
-                        }
-                        if (!list_empty (&dom->entrylk_list)) {
-                                gf_log (this->name, GF_LOG_WARNING,
-                                        "Pending entry locks found, releasing.");
+    GF_FREE(pl_inode);
 
-                                list_for_each_entry_safe (entry_l, entry_tmp, &dom->entrylk_list, domain_list) {
-                                        list_del_init (&entry_l->domain_list);
+    return 0;
+}
 
-                                        GF_FREE ((char *)entry_l->basename);
-                                        GF_FREE (entry_l->connection_id);
-                                        GF_FREE (entry_l);
-                                }
+int
+pl_release(xlator_t *this, fd_t *fd)
+{
+    pl_inode_t *pl_inode = NULL;
+    uint64_t tmp_pl_inode = 0;
+    int ret = -1;
+    uint64_t tmp = 0;
+    pl_fdctx_t *fdctx = NULL;
 
-                                list_splice_init (&dom->blocked_entrylks, &entrylks_released);
-                        }
+    if (fd == NULL) {
+        goto out;
+    }
 
-                        list_del (&dom->inode_list);
-                        gf_log ("posix-locks", GF_LOG_TRACE,
-                                " Cleaning up domain: %s", dom->domain);
-                        GF_FREE ((char *)(dom->domain));
-                        GF_FREE (dom);
-                }
+    ret = inode_ctx_get(fd->inode, this, &tmp_pl_inode);
+    if (ret != 0)
+        goto clean;
 
-        }
-        pthread_mutex_unlock (&pl_inode->mutex);
+    pl_inode = (pl_inode_t *)(long)tmp_pl_inode;
 
-        list_for_each_entry_safe (ext_l, ext_tmp, &posixlks_released, list) {
+    pl_trace_release(this, fd);
 
-                STACK_UNWIND_STRICT (lk, ext_l->frame, -1, 0,
-                                     &ext_l->user_flock, NULL);
-                __destroy_lock (ext_l);
-        }
+    gf_log(this->name, GF_LOG_TRACE, "Releasing all locks with fd %p", fd);
 
-        list_for_each_entry_safe (ino_l, ino_tmp, &inodelks_released, blocked_locks) {
+    delete_locks_of_fd(this, pl_inode, fd);
+    pl_update_refkeeper(this, fd->inode);
 
-                STACK_UNWIND_STRICT (inodelk, ino_l->frame, -1, 0, NULL);
-                __pl_inodelk_unref (ino_l);
-        }
+clean:
+    ret = fd_ctx_del(fd, this, &tmp);
+    if (ret) {
+        gf_log(this->name, GF_LOG_DEBUG, "Could not get fdctx");
+        goto out;
+    }
 
-        list_for_each_entry_safe (entry_l, entry_tmp, &entrylks_released, blocked_locks) {
+    fdctx = (pl_fdctx_t *)(long)tmp;
 
-                STACK_UNWIND_STRICT (entrylk, entry_l->frame, -1, 0, NULL);
-                GF_FREE ((char *)entry_l->basename);
-                GF_FREE (entry_l->connection_id);
-                GF_FREE (entry_l);
+    GF_FREE(fdctx);
+out:
+    return ret;
+}
 
-        }
+int
+pl_releasedir(xlator_t *this, fd_t *fd)
+{
+    int ret = -1;
+    uint64_t tmp = 0;
+    pl_fdctx_t *fdctx = NULL;
 
-        GF_FREE (pl_inode);
+    if (fd == NULL) {
+        goto out;
+    }
 
-        return 0;
+    ret = fd_ctx_del(fd, this, &tmp);
+    if (ret) {
+        gf_log(this->name, GF_LOG_DEBUG, "Could not get fdctx");
+        goto out;
+    }
+
+    fdctx = (pl_fdctx_t *)(long)tmp;
+
+    GF_FREE(fdctx);
+out:
+    return ret;
 }
 
-int
-pl_release (xlator_t *this, fd_t *fd)
+static int32_t
+pl_request_link_count(dict_t **pxdata)
 {
-        pl_inode_t *pl_inode     = NULL;
-        uint64_t    tmp_pl_inode = 0;
-        int         ret          = -1;
-        uint64_t    tmp          = 0;
-        pl_fdctx_t *fdctx        = NULL;
+    dict_t *xdata;
 
-        if (fd == NULL) {
-                goto out;
+    xdata = *pxdata;
+    if (xdata == NULL) {
+        xdata = dict_new();
+        if (xdata == NULL) {
+            return ENOMEM;
         }
+    } else {
+        dict_ref(xdata);
+    }
 
-        ret = inode_ctx_get (fd->inode, this, &tmp_pl_inode);
-        if (ret != 0)
-                goto out;
+    if (dict_set_uint32(xdata, GET_LINK_COUNT, 0) != 0) {
+        dict_unref(xdata);
+        return ENOMEM;
+    }
 
-        pl_inode = (pl_inode_t *)(long)tmp_pl_inode;
+    *pxdata = xdata;
 
-        pl_trace_release (this, fd);
+    return 0;
+}
 
-        gf_log (this->name, GF_LOG_TRACE,
-                "Releasing all locks with fd %p", fd);
+static int32_t
+pl_check_link_count(dict_t *xdata)
+{
+    int32_t count;
 
-        delete_locks_of_fd (this, pl_inode, fd);
-        pl_update_refkeeper (this, fd->inode);
+    /* In case we are unable to read the link count from xdata, we take a
+     * conservative approach and return -2, which will prevent the inode from
+     * being considered deleted. In fact it will cause link tracking for this
+     * inode to be disabled completely to avoid races. */
 
-        ret = fd_ctx_del (fd, this, &tmp);
-        if (ret) {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "Could not get fdctx");
-                goto out;
-        }
+    if (xdata == NULL) {
+        return -2;
+    }
 
-        fdctx = (pl_fdctx_t *)(long)tmp;
+    if (dict_get_int32(xdata, GET_LINK_COUNT, &count) != 0) {
+        return -2;
+    }
 
-        GF_FREE (fdctx);
-out:
-        return ret;
+    return count;
 }
 
-int
-pl_releasedir (xlator_t *this, fd_t *fd)
+int32_t
+pl_lookup_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int32_t op_ret,
+              int32_t op_errno, inode_t *inode, struct iatt *buf, dict_t *xdata,
+              struct iatt *postparent)
 {
-        int         ret          = -1;
-        uint64_t    tmp          = 0;
-        pl_fdctx_t *fdctx        = NULL;
+    pl_inode_t *pl_inode;
 
-        if (fd == NULL) {
-                goto out;
+    if (op_ret >= 0) {
+        pl_inode = pl_inode_get(this, inode, NULL);
+        if (pl_inode == NULL) {
+            PL_STACK_UNWIND(lookup, xdata, frame, -1, ENOMEM, NULL, NULL, NULL,
+                            NULL);
+            return 0;
         }
 
-        ret = fd_ctx_del (fd, this, &tmp);
-        if (ret) {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "Could not get fdctx");
-                goto out;
+        pthread_mutex_lock(&pl_inode->mutex);
+
+        /* We only update the link count if we previously didn't know it.
+         * Doing it always can lead to races since lookup is not executed
+         * atomically most of the times. */
+        if (pl_inode->links == -2) {
+            pl_inode->links = pl_check_link_count(xdata);
+            if (buf->ia_type == IA_IFDIR) {
+                /* Directories have at least 2 links. To avoid special handling
+                 * for directories, we simply decrement the value here to make
+                 * them equivalent to regular files. */
+                pl_inode->links--;
+            }
         }
 
-        fdctx = (pl_fdctx_t *)(long)tmp;
+        pthread_mutex_unlock(&pl_inode->mutex);
+    }
 
-        GF_FREE (fdctx);
-out:
-        return ret;
+    PL_STACK_UNWIND(lookup, xdata, frame, op_ret, op_errno, inode, buf, xdata,
+                    postparent);
+    return 0;
 }
 
 int32_t
-__get_posixlk_count (xlator_t *this, pl_inode_t *pl_inode)
+pl_lookup(call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xdata)
 {
-        posix_lock_t *lock   = NULL;
-        int32_t       count  = 0;
-
-        list_for_each_entry (lock, &pl_inode->ext_list, list) {
-
-                count++;
-        }
+    int32_t error;
+
+    error = pl_request_link_count(&xdata);
+    if (error == 0) {
+        PL_LOCAL_GET_REQUESTS(frame, this, xdata, ((fd_t *)NULL), loc, NULL);
+        STACK_WIND(frame, pl_lookup_cbk, FIRST_CHILD(this),
+                   FIRST_CHILD(this)->fops->lookup, loc, xdata);
+        dict_unref(xdata);
+    } else {
+        STACK_UNWIND_STRICT(lookup, frame, -1, error, NULL, NULL, NULL, NULL);
+    }
+    return 0;
+}
 
-        return count;
+int32_t
+pl_fstat_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int32_t op_ret,
+             int32_t op_errno, struct iatt *buf, dict_t *xdata)
+{
+    PL_STACK_UNWIND(fstat, xdata, frame, op_ret, op_errno, buf, xdata);
+    return 0;
 }
 
 int32_t
-get_posixlk_count (xlator_t *this, inode_t *inode)
+pl_fstat(call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *xdata)
 {
-        pl_inode_t   *pl_inode = NULL;
-        uint64_t      tmp_pl_inode = 0;
-        int           ret      = 0;
-        int32_t       count    = 0;
+    PL_LOCAL_GET_REQUESTS(frame, this, xdata, fd, NULL, NULL);
+    STACK_WIND(frame, pl_fstat_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->fstat, fd, xdata);
+    return 0;
+}
 
-        ret = inode_ctx_get (inode, this, &tmp_pl_inode);
-        if (ret != 0) {
-                goto out;
-        }
+int
+pl_readdirp_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int op_ret,
+                int op_errno, gf_dirent_t *entries, dict_t *xdata)
+{
+    pl_local_t *local = NULL;
+    gf_dirent_t *entry = NULL;
 
-        pl_inode = (pl_inode_t *)(long) tmp_pl_inode;
+    if (op_ret <= 0)
+        goto unwind;
 
-        pthread_mutex_lock (&pl_inode->mutex);
-        {
-                count =__get_posixlk_count (this, pl_inode);
-        }
-        pthread_mutex_unlock (&pl_inode->mutex);
+    local = frame->local;
+    if (!local)
+        goto unwind;
 
-out:
-        return count;
+    list_for_each_entry(entry, &entries->list, list)
+    {
+        pl_set_xdata_response(this, local, local->fd->inode, entry->inode,
+                              entry->d_name, entry->dict, 0);
+    }
+
+unwind:
+    PL_STACK_UNWIND(readdirp, xdata, frame, op_ret, op_errno, entries, xdata);
+
+    return 0;
 }
 
-void
-pl_parent_entrylk_xattr_fill (xlator_t *this, inode_t *parent,
-                              char *basename, dict_t *dict)
+int
+pl_readdirp(call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
+            off_t offset, dict_t *xdata)
 {
-        uint32_t         entrylk = 0;
-        int             ret     = -1;
+    PL_LOCAL_GET_REQUESTS(frame, this, xdata, fd, NULL, NULL);
+    STACK_WIND(frame, pl_readdirp_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->readdirp, fd, size, offset, xdata);
+
+    return 0;
+}
+
+lock_migration_info_t *
+gf_mig_info_for_lock(posix_lock_t *lock)
+{
+    lock_migration_info_t *new = GF_MALLOC(sizeof(lock_migration_info_t),
+                                           gf_common_mt_lock_mig);
+    if (new == NULL) {
+        goto out;
+    }
+
+    INIT_LIST_HEAD(&new->list);
+
+    posix_lock_to_flock(lock, &new->flock);
+
+    new->lk_flags = lock->lk_flags;
+
+    new->client_uid = gf_strdup(lock->client_uid);
 
-        if (!parent || !basename || !strlen (basename))
-                goto out;
-        entrylk = check_entrylk_on_basename (this, parent, basename);
 out:
-        ret = dict_set_uint32 (dict, GLUSTERFS_PARENT_ENTRYLK, entrylk);
-        if (ret < 0) {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        " dict_set failed on key %s", GLUSTERFS_PARENT_ENTRYLK);
-        }
+    return new;
 }
 
-void
-pl_entrylk_xattr_fill (xlator_t *this, inode_t *inode,
-                       dict_t *dict)
+int
+pl_fill_active_locks(pl_inode_t *pl_inode, lock_migration_info_t *lmi)
 {
-        int32_t     count = 0;
-        int         ret   = -1;
+    posix_lock_t *temp = NULL;
+    lock_migration_info_t *newlock = NULL;
+    int count = 0;
+
+    pthread_mutex_lock(&pl_inode->mutex);
+    {
+        if (list_empty(&pl_inode->ext_list)) {
+            count = 0;
+            goto unlock;
+        }
 
-        count = get_entrylk_count (this, inode);
-        ret = dict_set_int32 (dict, GLUSTERFS_ENTRYLK_COUNT, count);
-        if (ret < 0) {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        " dict_set failed on key %s", GLUSTERFS_ENTRYLK_COUNT);
+        list_for_each_entry(temp, &pl_inode->ext_list, list)
+        {
+            if (temp->blocked)
+                continue;
+
+            newlock = gf_mig_info_for_lock(temp);
+            if (!newlock) {
+                pthread_mutex_unlock(&pl_inode->mutex);
+                gf_msg(THIS->name, GF_LOG_ERROR, 0, 0, "lock_dup failed");
+                count = -1;
+                goto out;
+            }
+
+            list_add_tail(&newlock->list, &lmi->list);
+            count++;
         }
+    }
 
+unlock:
+    pthread_mutex_unlock(&pl_inode->mutex);
+out:
+    return count;
 }
 
-void
-pl_inodelk_xattr_fill (xlator_t *this, inode_t *inode, dict_t *dict,
-                       gf_boolean_t per_dom)
+/* This function reads only active locks */
+static int
+pl_getactivelk(call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xdata)
 {
-        int32_t     count = 0;
-        int         ret   = -1;
-        char        *domname = NULL;
+    pl_inode_t *pl_inode = NULL;
+    lock_migration_info_t locks;
+    int op_ret = 0;
+    int op_errno = 0;
+    int count = 0;
 
+    INIT_LIST_HEAD(&locks.list);
 
-        if (per_dom){
-                ret = dict_get_str (dict, GLUSTERFS_INODELK_DOM_COUNT,
-                                    &domname);
-                if (ret) {
-                        gf_log (this->name, GF_LOG_ERROR, "Failed to get "
-                                "value for key %s",GLUSTERFS_INODELK_DOM_COUNT);
-                        goto out;
-                }
-        }
+    pl_inode = pl_inode_get(this, loc->inode, NULL);
+    if (!pl_inode) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, 0, "pl_inode_get failed");
 
-        count = get_inodelk_count (this, inode, domname);
+        op_ret = -1;
+        op_errno = ENOMEM;
+        goto out;
+    }
 
-        ret = dict_set_int32 (dict, GLUSTERFS_INODELK_COUNT, count);
-        if (ret < 0) {
-                gf_log (this->name, GF_LOG_DEBUG, "Failed to set count for "
-                        "key %s", GLUSTERFS_INODELK_COUNT);
-        }
+    count = pl_fill_active_locks(pl_inode, &locks);
+
+    op_ret = count;
 
 out:
-        return;
+    STACK_UNWIND_STRICT(getactivelk, frame, op_ret, op_errno, &locks, NULL);
+
+    gf_free_mig_locks(&locks);
+
+    return 0;
 }
 
 void
-pl_posixlk_xattr_fill (xlator_t *this, inode_t *inode,
-                       dict_t *dict)
+pl_metalk_unref(pl_meta_lock_t *lock)
 {
-        int32_t     count = 0;
-        int         ret   = -1;
+    lock->ref--;
+    if (!lock->ref) {
+        GF_FREE(lock->client_uid);
+        GF_FREE(lock);
+    }
+}
 
-        count = get_posixlk_count (this, inode);
-        ret = dict_set_int32 (dict, GLUSTERFS_POSIXLK_COUNT, count);
-        if (ret < 0) {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        " dict_set failed on key %s", GLUSTERFS_POSIXLK_COUNT);
-        }
+void
+__pl_metalk_ref(pl_meta_lock_t *lock)
+{
+    lock->ref++;
+}
 
+pl_meta_lock_t *
+new_meta_lock(call_frame_t *frame, xlator_t *this)
+{
+    pl_meta_lock_t *lock = GF_CALLOC(1, sizeof(*lock),
+                                     gf_locks_mt_pl_meta_lock_t);
+
+    if (!lock) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, ENOMEM,
+               "mem allocation"
+               " failed for meta lock");
+        goto out;
+    }
+
+    INIT_LIST_HEAD(&lock->list);
+    INIT_LIST_HEAD(&lock->client_list);
+
+    lock->client_uid = gf_strdup(frame->root->client->client_uid);
+    if (!lock->client_uid) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, ENOMEM,
+               "mem allocation"
+               " failed for client_uid");
+        GF_FREE(lock);
+        lock = NULL;
+        goto out;
+    }
+
+    __pl_metalk_ref(lock);
+out:
+    return lock;
 }
 
-int32_t
-pl_lookup_cbk (call_frame_t *frame,
-                    void *cookie,
-                    xlator_t *this,
-                    int32_t op_ret,
-                    int32_t op_errno,
-                    inode_t *inode,
-                    struct iatt *buf,
-                    dict_t *xdata,
-                    struct iatt *postparent)
+int
+pl_insert_metalk(pl_inode_t *pl_inode, pl_ctx_t *ctx, pl_meta_lock_t *lock)
 {
-        pl_local_t *local = NULL;
+    int ret = 0;
 
-        GF_VALIDATE_OR_GOTO (this->name, frame->local, out);
+    if (!pl_inode || !ctx || !lock) {
+        gf_msg(THIS->name, GF_LOG_INFO, 0, 0, "NULL parameter");
+        ret = -1;
+        goto out;
+    }
 
-        if (op_ret)
-                goto out;
+    lock->pl_inode = pl_inode;
 
-        local = frame->local;
+    /* refer function pl_inode_setlk for more info for this ref.
+     * This should be unrefed on meta-unlock triggered by rebalance or
+     * in cleanup with client disconnect*/
+    /*TODO: unref this in  cleanup code for disconnect and meta-unlock*/
+    pl_inode->inode = inode_ref(pl_inode->inode);
 
-        if (local->parent_entrylk_req)
-                pl_parent_entrylk_xattr_fill (this, local->loc.parent,
-                                              (char*)local->loc.name, xdata);
-        if (local->entrylk_count_req)
-                pl_entrylk_xattr_fill (this, inode, xdata);
-        if (local->inodelk_count_req)
-                pl_inodelk_xattr_fill (this, inode, xdata, _gf_false);
-        if (local->inodelk_dom_count_req)
-                pl_inodelk_xattr_fill (this, inode, xdata, _gf_true);
-        if (local->posixlk_count_req)
-                pl_posixlk_xattr_fill (this, inode, xdata);
+    /* NOTE:In case of a client-server disconnect we need to cleanup metalk.
+     * Hence, adding the metalk to pl_ctx_t as well. The mutex lock order
+     * should always be on ctx and then on pl_inode*/
 
+    pthread_mutex_lock(&ctx->lock);
+    {
+        pthread_mutex_lock(&pl_inode->mutex);
+        {
+            list_add_tail(&lock->list, &pl_inode->metalk_list);
+        }
+        pthread_mutex_unlock(&pl_inode->mutex);
+
+        list_add_tail(&lock->client_list, &ctx->metalk_list);
+    }
+    pthread_mutex_unlock(&ctx->lock);
 
 out:
-        local = frame->local;
-        frame->local = NULL;
-
-        if (local != NULL) {
-                loc_wipe (&local->loc);
-                mem_put (local);
-        }
-
-        STACK_UNWIND_STRICT (
-                     lookup,
-                     frame,
-                     op_ret,
-                     op_errno,
-                     inode,
-                     buf,
-                     xdata,
-                     postparent);
-        return 0;
+    return ret;
 }
 
 int32_t
-pl_lookup (call_frame_t *frame,
-           xlator_t *this,
-           loc_t *loc,
-           dict_t *xdata)
+pl_metalk(call_frame_t *frame, xlator_t *this, inode_t *inode)
 {
-        pl_local_t *local  = NULL;
-        int         ret    = -1;
+    pl_inode_t *pl_inode = NULL;
+    int ret = 0;
+    pl_meta_lock_t *reqlk = NULL;
+    pl_ctx_t *ctx = NULL;
+
+    pl_inode = pl_inode_get(this, inode, NULL);
+    if (!pl_inode) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, ENOMEM,
+               "pl_inode mem allocation failedd");
+
+        ret = -1;
+        goto out;
+    }
+
+    /* Non rebalance process trying to do metalock */
+    if (frame->root->pid != GF_CLIENT_PID_DEFRAG) {
+        ret = -1;
+        goto out;
+    }
+
+    /* Note: In the current scheme of glusterfs where lock migration is
+     * experimental, (ideally) the rebalance process which is migrating
+     * the file should request for a metalock. Hence, the metalock count
+     * should not be more than one for an inode. In future, if there is a
+     * need for meta-lock from other clients, the following block can be
+     * removed.
+     *
+     * Since pl_metalk is called as part of setxattr operation, any client
+     * process(non-rebalance) residing outside trusted network can exhaust
+     * memory of the server node by issuing setxattr repetitively on the
+     * metalock key. The following code makes sure that more than
+     * one metalock cannot be granted on an inode*/
+    pthread_mutex_lock(&pl_inode->mutex);
+    {
+        if (pl_metalock_is_active(pl_inode)) {
+            ret = -1;
+        }
+    }
+    pthread_mutex_unlock(&pl_inode->mutex);
+
+    if (ret == -1) {
+        gf_msg(this->name, GF_LOG_WARNING, EINVAL, 0,
+               "More than one meta-lock cannot be granted on"
+               " the inode");
+        goto out;
+    }
+
+    if (frame->root->client) {
+        ctx = pl_ctx_get(frame->root->client, this);
+        if (!ctx) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, 0, "pl_ctx_get failed");
+
+            ret = -1;
+            goto out;
+        }
+    } else {
+        gf_msg(this->name, GF_LOG_INFO, 0, 0,
+               "frame-root-client "
+               "is NULL");
+
+        ret = -1;
+        goto out;
+    }
+
+    reqlk = new_meta_lock(frame, this);
+    if (!reqlk) {
+        ret = -1;
+        goto out;
+    }
+
+    ret = pl_insert_metalk(pl_inode, ctx, reqlk);
+    if (ret < 0) {
+        pl_metalk_unref(reqlk);
+    }
 
-        VALIDATE_OR_GOTO (frame, out);
-        VALIDATE_OR_GOTO (this, out);
-        VALIDATE_OR_GOTO (loc, out);
+out:
+    return ret;
+}
 
-        local = mem_get0 (this->local_pool);
-        GF_VALIDATE_OR_GOTO (this->name, local, out);
+static void
+__unwind_queued_locks(pl_inode_t *pl_inode, struct list_head *tmp_list)
+{
+    if (list_empty(&pl_inode->queued_locks))
+        return;
 
-        if (xdata) {
-                if (dict_get (xdata, GLUSTERFS_ENTRYLK_COUNT))
-                        local->entrylk_count_req = 1;
-                if (dict_get (xdata, GLUSTERFS_INODELK_COUNT))
-                        local->inodelk_count_req = 1;
-                if (dict_get (xdata, GLUSTERFS_INODELK_DOM_COUNT))
-                        local->inodelk_dom_count_req = 1;
-                if (dict_get (xdata, GLUSTERFS_POSIXLK_COUNT))
-                        local->posixlk_count_req = 1;
-                if (dict_get (xdata, GLUSTERFS_PARENT_ENTRYLK))
-                        local->parent_entrylk_req = 1;
-        }
+    list_splice_init(&pl_inode->queued_locks, tmp_list);
+}
 
-        frame->local = local;
-        loc_copy (&local->loc, loc);
-
-        STACK_WIND (frame,
-                    pl_lookup_cbk,
-                    FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->lookup,
-                    loc, xdata);
-        ret = 0;
-out:
-        if (ret == -1)
-                STACK_UNWIND_STRICT (lookup, frame, -1, 0, NULL,
-                                     NULL, NULL, NULL);
+static void
+__unwind_blocked_locks(pl_inode_t *pl_inode, struct list_head *tmp_list)
+{
+    posix_lock_t *lock = NULL;
+    posix_lock_t *tmp = NULL;
 
-        return 0;
+    if (list_empty(&pl_inode->ext_list))
+        return;
+
+    list_for_each_entry_safe(lock, tmp, &pl_inode->ext_list, list)
+    {
+        if (!lock->blocking)
+            continue;
+
+        list_del_init(&lock->list);
+        list_add_tail(&lock->list, tmp_list);
+    }
 }
+
 int
-pl_readdirp_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                 int op_ret, int op_errno, gf_dirent_t *entries, dict_t *xdata)
+pl_metaunlock(call_frame_t *frame, xlator_t *this, inode_t *inode, dict_t *dict)
 {
-        pl_local_t *local  = NULL;
-        gf_dirent_t *entry = NULL;
+    pl_inode_t *pl_inode = NULL;
+    int ret = 0;
+    pl_meta_lock_t *meta_lock = NULL;
+    pl_meta_lock_t *tmp_metalk = NULL;
+    pl_ctx_t *ctx = NULL;
+    posix_lock_t *posix_lock = NULL;
+    posix_lock_t *tmp_posixlk = NULL;
+    struct list_head tmp_posixlk_list;
+
+    INIT_LIST_HEAD(&tmp_posixlk_list);
+
+    if (frame->root->client) {
+        ctx = pl_ctx_get(frame->root->client, this);
+        if (!ctx) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, 0, "pl_ctx_get failed");
+
+            ret = -1;
+            goto out;
+        }
+    } else {
+        gf_msg(this->name, GF_LOG_ERROR, 0, 0,
+               "frame-root-client is "
+               "NULL");
+        ret = -1;
+        goto out;
+    }
+
+    pl_inode = pl_inode_get(this, inode, NULL);
+    if (!pl_inode) {
+        ret = -1;
+        goto out;
+    }
+
+    pthread_mutex_lock(&ctx->lock);
+    {
+        pthread_mutex_lock(&pl_inode->mutex);
+        {
+            /* Unwind queued locks regardless of migration status */
+            __unwind_queued_locks(pl_inode, &tmp_posixlk_list);
 
-        local = frame->local;
+            /* Unwind blocked locks only for successful migration */
+            if (dict_get_sizen(dict, "status")) {
+                /* unwind all blocked locks */
+                __unwind_blocked_locks(pl_inode, &tmp_posixlk_list);
+            }
 
-        if (op_ret <= 0)
-                goto unwind;
+            /* unlock metalk */
+            /* if this list is empty then pl_inode->metalk_list
+             * should be empty too. meta lock should in all cases
+             * be added/removed from both pl_ctx_t and pl_inode */
+
+            if (list_empty(&ctx->metalk_list))
+                goto unlock;
+
+            list_for_each_entry_safe(meta_lock, tmp_metalk, &ctx->metalk_list,
+                                     client_list)
+            {
+                list_del_init(&meta_lock->client_list);
+
+                pl_inode = meta_lock->pl_inode;
+
+                list_del_init(&meta_lock->list);
 
-        list_for_each_entry (entry, &entries->list, list) {
-                if (local->entrylk_count_req)
-                        pl_entrylk_xattr_fill (this, entry->inode, entry->dict);
-                if (local->inodelk_count_req)
-                        pl_inodelk_xattr_fill (this, entry->inode, entry->dict,
-                                               _gf_false);
-                if (local->inodelk_dom_count_req)
-                        pl_inodelk_xattr_fill (this, entry->inode, entry->dict,
-                                               _gf_true);
-                if (local->posixlk_count_req)
-                        pl_posixlk_xattr_fill (this, entry->inode, entry->dict);
+                pl_metalk_unref(meta_lock);
+
+                /* The corresponding ref is taken in
+                 * pl_insert_metalk*/
+                inode_unref(pl_inode->inode);
+            }
+
+            if (dict_get_sizen(dict, "status"))
+                pl_inode->migrated = _gf_true;
+            else
+                pl_inode->migrated = _gf_false;
         }
+    unlock:
 
-unwind:
-        frame->local = NULL;
-        STACK_UNWIND_STRICT (readdirp, frame, op_ret, op_errno, entries, xdata);
+        pthread_mutex_unlock(&pl_inode->mutex);
+    }
+    pthread_mutex_unlock(&ctx->lock);
+
+out:
+    list_for_each_entry_safe(posix_lock, tmp_posixlk, &tmp_posixlk_list, list)
+    {
+        list_del_init(&posix_lock->list);
 
-        if (local)
-                mem_put (local);
+        STACK_UNWIND_STRICT(lk, posix_lock->frame, -1, EREMOTE,
+                            &posix_lock->user_flock, NULL);
 
-        return 0;
+        __destroy_lock(posix_lock);
+    }
+
+    return ret;
 }
 
-int
-pl_readdirp (call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
-             off_t offset, dict_t *dict)
+int32_t
+pl_setxattr_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
-        pl_local_t *local  = NULL;
-
-        local = mem_get0 (this->local_pool);
-        GF_VALIDATE_OR_GOTO (this->name, local, out);
+    pl_local_t *local = NULL;
+    pl_inode_t *pl_inode = NULL;
+    local = frame->local;
+    if (local && local->update_mlock_enforced_flag && op_ret != -1) {
+        pl_inode = pl_inode_get(this, local->inode, NULL);
+        if (!pl_inode) {
+            op_ret = -1;
+            op_errno = ENOMEM;
+            goto unwind;
+        }
 
-        if (dict) {
-                if (dict_get (dict, GLUSTERFS_ENTRYLK_COUNT))
-                        local->entrylk_count_req = 1;
-                if (dict_get (dict, GLUSTERFS_INODELK_COUNT))
-                        local->inodelk_count_req = 1;
-                if (dict_get (dict, GLUSTERFS_INODELK_DOM_COUNT))
-                        local->inodelk_dom_count_req = 1;
-                if (dict_get (dict, GLUSTERFS_POSIXLK_COUNT))
-                        local->posixlk_count_req = 1;
+        pthread_mutex_lock(&pl_inode->mutex);
+        {
+            while (pl_inode->fop_wind_count > 0) {
+                gf_msg(this->name, GF_LOG_INFO, 0, 0,
+                       "waiting for existing fops (count %d) to drain for "
+                       "gfid %s",
+                       pl_inode->fop_wind_count, uuid_utoa(pl_inode->gfid));
+                pthread_cond_wait(&pl_inode->check_fop_wind_count,
+                                  &pl_inode->mutex);
+            }
+            pl_inode->mlock_enforced = _gf_true;
+            pl_inode->check_mlock_info = _gf_false;
         }
+        pthread_mutex_unlock(&pl_inode->mutex);
+    }
 
-        frame->local = local;
+unwind:
+    PL_STACK_UNWIND_FOR_CLIENT(setxattr, xdata, frame, op_ret, op_errno, xdata);
+    return 0;
+}
 
-        STACK_WIND (frame, pl_readdirp_cbk,
-                    FIRST_CHILD(this), FIRST_CHILD(this)->fops->readdirp,
-                    fd, size, offset, dict);
+int32_t
+pl_setxattr(call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *dict,
+            int flags, dict_t *xdata)
+{
+    int op_ret = 0;
+    int op_errno = EINVAL;
+    dict_t *xdata_rsp = NULL;
+    char *name = NULL;
+    posix_locks_private_t *priv = this->private;
 
-        return 0;
-out:
-        STACK_UNWIND_STRICT (readdirp, frame, -1, ENOMEM, NULL, NULL);
-        return 0;
-}
+    PL_LOCAL_GET_REQUESTS(frame, this, xdata, ((fd_t *)NULL), loc, NULL);
 
+    if (dict_get_sizen(dict, GF_META_LOCK_KEY)) {
+        op_ret = pl_metalk(frame, this, loc->inode);
+
+    } else if (dict_get_sizen(dict, GF_META_UNLOCK_KEY)) {
+        op_ret = pl_metaunlock(frame, this, loc->inode, dict);
+    } else {
+        goto usual;
+    }
+
+    PL_STACK_UNWIND_FOR_CLIENT(setxattr, xdata_rsp, frame, op_ret, op_errno,
+                               xdata_rsp);
+    return 0;
+
+usual:
+    PL_CHECK_LOCK_ENFORCE_KEY(frame, dict, name, this, loc, ((fd_t *)NULL),
+                              priv);
+
+    STACK_WIND(frame, pl_setxattr_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->setxattr, loc, dict, flags, xdata);
+    return 0;
+
+unwind:
+    PL_STACK_UNWIND_FOR_CLIENT(setxattr, xdata, frame, op_ret, op_errno, xdata);
+
+    return 0;
+}
 
 void
-pl_dump_lock (char *str, int size, struct gf_flock *flock,
-              gf_lkowner_t *owner, void *trans, char *conn_id,
-              time_t *granted_time, time_t *blkd_time, gf_boolean_t active)
-{
-        char  *type_str    = NULL;
-        char   granted[256] = {0,};
-        char   blocked[256] = {0,};
-
-        if (granted_time)
-                gf_time_fmt (granted, sizeof (granted), *granted_time,
-                             gf_timefmt_FT);
-        if (blkd_time)
-                gf_time_fmt (blocked, sizeof (blocked), *blkd_time,
-                             gf_timefmt_FT);
-        switch (flock->l_type) {
+pl_dump_lock(char *str, int size, struct gf_flock *flock, gf_lkowner_t *owner,
+             void *trans, char *conn_id, time_t *granted_time,
+             time_t *blkd_time, gf_boolean_t active)
+{
+    char *type_str = NULL;
+    char granted[GF_TIMESTR_SIZE] = {
+        0,
+    };
+    char blocked[GF_TIMESTR_SIZE] = {
+        0,
+    };
+
+    if (granted_time)
+        gf_time_fmt(granted, sizeof(granted), *granted_time, gf_timefmt_FT);
+    if (blkd_time)
+        gf_time_fmt(blocked, sizeof(blocked), *blkd_time, gf_timefmt_FT);
+    switch (flock->l_type) {
         case F_RDLCK:
-                type_str = "READ";
-                break;
+            type_str = "READ";
+            break;
         case F_WRLCK:
-                type_str = "WRITE";
-                break;
+            type_str = "WRITE";
+            break;
         case F_UNLCK:
-                type_str = "UNLOCK";
-                break;
+            type_str = "UNLOCK";
+            break;
         default:
-                type_str = "UNKNOWN";
-                break;
+            type_str = "UNKNOWN";
+            break;
+    }
+
+    if (active) {
+        if (blkd_time && *blkd_time == 0) {
+            snprintf(str, size, RANGE_GRNTD_FMT, type_str, flock->l_whence,
+                     (unsigned long long)flock->l_start,
+                     (unsigned long long)flock->l_len,
+                     (unsigned long long)flock->l_pid, lkowner_utoa(owner),
+                     trans, conn_id, granted);
+        } else {
+            snprintf(str, size, RANGE_BLKD_GRNTD_FMT, type_str, flock->l_whence,
+                     (unsigned long long)flock->l_start,
+                     (unsigned long long)flock->l_len,
+                     (unsigned long long)flock->l_pid, lkowner_utoa(owner),
+                     trans, conn_id, blocked, granted);
         }
+    } else {
+        snprintf(str, size, RANGE_BLKD_FMT, type_str, flock->l_whence,
+                 (unsigned long long)flock->l_start,
+                 (unsigned long long)flock->l_len,
+                 (unsigned long long)flock->l_pid, lkowner_utoa(owner), trans,
+                 conn_id, blocked);
+    }
+}
 
-        if (active) {
-                if (blkd_time && *blkd_time == 0) {
-                        snprintf (str, size, RANGE_GRNTD_FMT,
-                                  type_str, flock->l_whence,
-                                  (unsigned long long) flock->l_start,
-                                  (unsigned long long) flock->l_len,
-                                  (unsigned long long) flock->l_pid,
-                                  lkowner_utoa (owner), trans, conn_id,
-                                  granted);
-                } else {
-                        snprintf (str, size, RANGE_BLKD_GRNTD_FMT,
-                                  type_str, flock->l_whence,
-                                  (unsigned long long) flock->l_start,
-                                  (unsigned long long) flock->l_len,
-                                  (unsigned long long) flock->l_pid,
-                                  lkowner_utoa (owner), trans, conn_id,
-                                  blocked, granted);
-                }
-        } else {
-                snprintf (str, size, RANGE_BLKD_FMT,
-                          type_str, flock->l_whence,
-                          (unsigned long long) flock->l_start,
-                          (unsigned long long) flock->l_len,
-                          (unsigned long long) flock->l_pid,
-                          lkowner_utoa (owner), trans, conn_id, blocked);
+void
+__dump_entrylks(pl_inode_t *pl_inode)
+{
+    pl_dom_list_t *dom = NULL;
+    pl_entry_lock_t *lock = NULL;
+    char blocked[GF_TIMESTR_SIZE] = {
+        0,
+    };
+    char granted[GF_TIMESTR_SIZE] = {
+        0,
+    };
+    int count = 0;
+    char key[GF_DUMP_MAX_BUF_LEN] = {
+        0,
+    };
+    char *k = "xlator.feature.locks.lock-dump.domain.entrylk";
+
+    char tmp[4098];
+
+    list_for_each_entry(dom, &pl_inode->dom_list, inode_list)
+    {
+        count = 0;
+
+        gf_proc_dump_build_key(key, "lock-dump.domain", "domain");
+        gf_proc_dump_write(key, "%s", dom->domain);
+
+        list_for_each_entry(lock, &dom->entrylk_list, domain_list)
+        {
+            gf_time_fmt(granted, sizeof(granted), lock->granted_time,
+                        gf_timefmt_FT);
+            gf_proc_dump_build_key(key, k, "entrylk[%d](ACTIVE)", count);
+            if (lock->blkd_time == 0) {
+                snprintf(tmp, sizeof(tmp), ENTRY_GRNTD_FMT,
+                         lock->type == ENTRYLK_RDLCK ? "ENTRYLK_RDLCK"
+                                                     : "ENTRYLK_WRLCK",
+                         lock->basename, (unsigned long long)lock->client_pid,
+                         lkowner_utoa(&lock->owner), lock->client,
+                         lock->connection_id, granted);
+            } else {
+                gf_time_fmt(blocked, sizeof(blocked), lock->blkd_time,
+                            gf_timefmt_FT);
+                snprintf(tmp, sizeof(tmp), ENTRY_BLKD_GRNTD_FMT,
+                         lock->type == ENTRYLK_RDLCK ? "ENTRYLK_RDLCK"
+                                                     : "ENTRYLK_WRLCK",
+                         lock->basename, (unsigned long long)lock->client_pid,
+                         lkowner_utoa(&lock->owner), lock->client,
+                         lock->connection_id, blocked, granted);
+            }
+
+            gf_proc_dump_write(key, "%s", tmp);
+
+            count++;
         }
 
+        list_for_each_entry(lock, &dom->blocked_entrylks, blocked_locks)
+        {
+            gf_time_fmt(blocked, sizeof(blocked), lock->blkd_time,
+                        gf_timefmt_FT);
+
+            gf_proc_dump_build_key(key, k, "entrylk[%d](BLOCKED)", count);
+            snprintf(
+                tmp, sizeof(tmp), ENTRY_BLKD_FMT,
+                lock->type == ENTRYLK_RDLCK ? "ENTRYLK_RDLCK" : "ENTRYLK_WRLCK",
+                lock->basename, (unsigned long long)lock->client_pid,
+                lkowner_utoa(&lock->owner), lock->client, lock->connection_id,
+                blocked);
+
+            gf_proc_dump_write(key, "%s", tmp);
+
+            count++;
+        }
+    }
 }
 
 void
-__dump_entrylks (pl_inode_t *pl_inode)
-{
-        pl_dom_list_t   *dom  = NULL;
-        pl_entry_lock_t *lock = NULL;
-        char             blocked[256] = {0,};
-        char             granted[256] = {0,};
-        int              count = 0;
-        char             key[GF_DUMP_MAX_BUF_LEN] = {0,};
-        char            *k = "xlator.feature.locks.lock-dump.domain.entrylk";
-
-        char tmp[4098];
-
-        list_for_each_entry (dom, &pl_inode->dom_list, inode_list) {
-
-                count = 0;
-
-                gf_proc_dump_build_key(key,
-                                       "lock-dump.domain",
-                                       "domain");
-                gf_proc_dump_write(key, "%s", dom->domain);
-
-                list_for_each_entry (lock, &dom->entrylk_list, domain_list) {
-
-                        gf_time_fmt (granted, sizeof (granted),
-                                     lock->granted_time.tv_sec, gf_timefmt_FT);
-                        gf_proc_dump_build_key(key, k,
-                                               "entrylk[%d](ACTIVE)", count );
-                        if (lock->blkd_time.tv_sec == 0) {
-                                snprintf (tmp, sizeof (tmp), ENTRY_GRNTD_FMT,
-                                          lock->type == ENTRYLK_RDLCK ?
-                                          "ENTRYLK_RDLCK" : "ENTRYLK_WRLCK",
-                                          lock->basename,
-                                          (unsigned long long) lock->client_pid,
-                                          lkowner_utoa (&lock->owner),
-                                          lock->client,
-                                          lock->connection_id, granted);
-                        } else {
-                                gf_time_fmt (blocked, sizeof (blocked),
-                                             lock->blkd_time.tv_sec,
-                                             gf_timefmt_FT);
-                                snprintf (tmp, sizeof (tmp),
-                                          ENTRY_BLKD_GRNTD_FMT,
-                                          lock->type == ENTRYLK_RDLCK ?
-                                          "ENTRYLK_RDLCK" : "ENTRYLK_WRLCK",
-                                          lock->basename,
-                                          (unsigned long long) lock->client_pid,
-                                          lkowner_utoa (&lock->owner),
-                                          lock->client,
-                                          lock->connection_id,
-                                          blocked, granted);
-                        }
+dump_entrylks(pl_inode_t *pl_inode)
+{
+    pthread_mutex_lock(&pl_inode->mutex);
+    {
+        __dump_entrylks(pl_inode);
+    }
+    pthread_mutex_unlock(&pl_inode->mutex);
+}
 
-                        gf_proc_dump_write(key, tmp);
+void
+__dump_inodelks(pl_inode_t *pl_inode)
+{
+    pl_dom_list_t *dom = NULL;
+    pl_inode_lock_t *lock = NULL;
+    int count = 0;
+    char key[GF_DUMP_MAX_BUF_LEN];
 
-                        count++;
-                }
+    char tmp[4098];
 
-                list_for_each_entry (lock, &dom->blocked_entrylks,
-                                     blocked_locks) {
+    list_for_each_entry(dom, &pl_inode->dom_list, inode_list)
+    {
+        count = 0;
 
-                        gf_time_fmt (blocked, sizeof (blocked),
-                                     lock->blkd_time.tv_sec, gf_timefmt_FT);
+        gf_proc_dump_build_key(key, "lock-dump.domain", "domain");
+        gf_proc_dump_write(key, "%s", dom->domain);
 
-                        gf_proc_dump_build_key(key, k,
-                                               "entrylk[%d](BLOCKED)", count );
-                        snprintf (tmp, sizeof (tmp), ENTRY_BLKD_FMT,
-                                  lock->type == ENTRYLK_RDLCK ?
-                                  "ENTRYLK_RDLCK" : "ENTRYLK_WRLCK",
-                                  lock->basename,
-                                  (unsigned long long) lock->client_pid,
-                                  lkowner_utoa (&lock->owner), lock->client,
-                                  lock->connection_id, blocked);
+        list_for_each_entry(lock, &dom->inodelk_list, list)
+        {
+            gf_proc_dump_build_key(key, "inodelk", "inodelk[%d](ACTIVE)",
+                                   count);
 
-                        gf_proc_dump_write(key, tmp);
+            SET_FLOCK_PID(&lock->user_flock, lock);
+            pl_dump_lock(tmp, sizeof(tmp), &lock->user_flock, &lock->owner,
+                         lock->client, lock->connection_id, &lock->granted_time,
+                         &lock->blkd_time, _gf_true);
+            gf_proc_dump_write(key, "%s", tmp);
 
-                        count++;
-                }
+            count++;
+        }
 
+        list_for_each_entry(lock, &dom->blocked_inodelks, blocked_locks)
+        {
+            gf_proc_dump_build_key(key, "inodelk", "inodelk[%d](BLOCKED)",
+                                   count);
+            SET_FLOCK_PID(&lock->user_flock, lock);
+            pl_dump_lock(tmp, sizeof(tmp), &lock->user_flock, &lock->owner,
+                         lock->client, lock->connection_id, 0, &lock->blkd_time,
+                         _gf_false);
+            gf_proc_dump_write(key, "%s", tmp);
+
+            count++;
         }
+    }
 }
 
 void
-dump_entrylks (pl_inode_t *pl_inode)
+dump_inodelks(pl_inode_t *pl_inode)
 {
-        pthread_mutex_lock (&pl_inode->mutex);
-        {
-                __dump_entrylks (pl_inode);
-        }
-        pthread_mutex_unlock (&pl_inode->mutex);
+    pthread_mutex_lock(&pl_inode->mutex);
+    {
+        __dump_inodelks(pl_inode);
+    }
+    pthread_mutex_unlock(&pl_inode->mutex);
+}
 
+void
+__dump_posixlks(pl_inode_t *pl_inode)
+{
+    posix_lock_t *lock = NULL;
+    int count = 0;
+    char key[GF_DUMP_MAX_BUF_LEN];
+
+    char tmp[4098];
+
+    list_for_each_entry(lock, &pl_inode->ext_list, list)
+    {
+        SET_FLOCK_PID(&lock->user_flock, lock);
+        gf_proc_dump_build_key(key, "posixlk", "posixlk[%d](%s)", count,
+                               lock->blocked ? "BLOCKED" : "ACTIVE");
+        pl_dump_lock(tmp, sizeof(tmp), &lock->user_flock, &lock->owner,
+                     lock->client, lock->client_uid, &lock->granted_time,
+                     &lock->blkd_time, (lock->blocked) ? _gf_false : _gf_true);
+        gf_proc_dump_write(key, "%s", tmp);
+
+        count++;
+    }
 }
 
 void
-__dump_inodelks (pl_inode_t *pl_inode)
+dump_posixlks(pl_inode_t *pl_inode)
 {
-        pl_dom_list_t   *dom  = NULL;
-        pl_inode_lock_t *lock = NULL;
-        int             count = 0;
-        char            key[GF_DUMP_MAX_BUF_LEN];
+    pthread_mutex_lock(&pl_inode->mutex);
+    {
+        __dump_posixlks(pl_inode);
+    }
+    pthread_mutex_unlock(&pl_inode->mutex);
+}
 
-        char tmp[4098];
+int32_t
+pl_dump_inode_priv(xlator_t *this, inode_t *inode)
+{
+    int ret = -1;
+    uint64_t tmp_pl_inode = 0;
+    pl_inode_t *pl_inode = NULL;
+    char *pathname = NULL;
+    gf_boolean_t section_added = _gf_false;
+
+    int count = 0;
+
+    if (!inode) {
+        errno = EINVAL;
+        goto out;
+    }
+
+    ret = TRY_LOCK(&inode->lock);
+    if (ret)
+        goto out;
+    {
+        ret = __inode_ctx_get(inode, this, &tmp_pl_inode);
+        if (ret)
+            goto unlock;
+    }
+unlock:
+    UNLOCK(&inode->lock);
+    if (ret)
+        goto out;
+
+    pl_inode = (pl_inode_t *)(long)tmp_pl_inode;
+    if (!pl_inode) {
+        ret = -1;
+        goto out;
+    }
+
+    gf_proc_dump_add_section("xlator.features.locks.%s.inode", this->name);
+    section_added = _gf_true;
+
+    /*We are safe to call __inode_path since we have the
+     * inode->table->lock */
+    __inode_path(inode, NULL, &pathname);
+    if (pathname)
+        gf_proc_dump_write("path", "%s", pathname);
+
+    gf_proc_dump_write("mandatory", "%d", pl_inode->mandatory);
+
+    ret = pthread_mutex_trylock(&pl_inode->mutex);
+    if (ret)
+        goto out;
+    {
+        count = __get_entrylk_count(this, pl_inode);
+        if (count) {
+            gf_proc_dump_write("entrylk-count", "%d", count);
+            __dump_entrylks(pl_inode);
+        }
 
-        list_for_each_entry (dom, &pl_inode->dom_list, inode_list) {
+        count = __get_inodelk_count(this, pl_inode, NULL);
+        if (count) {
+            gf_proc_dump_write("inodelk-count", "%d", count);
+            __dump_inodelks(pl_inode);
+        }
 
-                count = 0;
+        count = __get_posixlk_count(pl_inode);
+        if (count) {
+            gf_proc_dump_write("posixlk-count", "%d", count);
+            __dump_posixlks(pl_inode);
+        }
 
-                gf_proc_dump_build_key(key,
-                                       "lock-dump.domain",
-                                       "domain");
-                gf_proc_dump_write(key, "%s", dom->domain);
+        gf_proc_dump_write("links", "%d", pl_inode->links);
+        gf_proc_dump_write("removes_pending", "%u", pl_inode->remove_running);
+        gf_proc_dump_write("removed", "%u", pl_inode->removed);
+    }
+    pthread_mutex_unlock(&pl_inode->mutex);
 
-                list_for_each_entry (lock, &dom->inodelk_list, list) {
+out:
+    GF_FREE(pathname);
+
+    if (ret && inode) {
+        if (!section_added)
+            gf_proc_dump_add_section(
+                "xlator.features.locks.%s."
+                "inode",
+                this->name);
+        gf_proc_dump_write("Unable to print lock state",
+                           "(Lock "
+                           "acquisition failure) %s",
+                           uuid_utoa(inode->gfid));
+    }
+    return ret;
+}
 
-                        gf_proc_dump_build_key(key,
-                                               "inodelk",
-                                               "inodelk[%d](ACTIVE)",count );
+int32_t
+mem_acct_init(xlator_t *this)
+{
+    int ret = -1;
 
-                        SET_FLOCK_PID (&lock->user_flock, lock);
-                        pl_dump_lock (tmp, sizeof (tmp), &lock->user_flock,
-                                      &lock->owner,
-                                      lock->client, lock->connection_id,
-                                      &lock->granted_time.tv_sec,
-                                      &lock->blkd_time.tv_sec,
-                                      _gf_true);
-                        gf_proc_dump_write(key, tmp);
+    if (!this)
+        return ret;
 
-                        count++;
-                }
+    ret = xlator_mem_acct_init(this, gf_locks_mt_end + 1);
 
-                list_for_each_entry (lock, &dom->blocked_inodelks, blocked_locks) {
+    if (ret != 0) {
+        gf_log(this->name, GF_LOG_ERROR,
+               "Memory accounting init"
+               "failed");
+        return ret;
+    }
 
-                        gf_proc_dump_build_key(key,
-                                               "inodelk",
-                                               "inodelk[%d](BLOCKED)",count );
-                        SET_FLOCK_PID (&lock->user_flock, lock);
-                        pl_dump_lock (tmp, sizeof (tmp), &lock->user_flock,
-                                      &lock->owner,
-                                      lock->client, lock->connection_id,
-                                      0, &lock->blkd_time.tv_sec,
-                                      _gf_false);
-                        gf_proc_dump_write(key, tmp);
+    return ret;
+}
 
-                        count++;
-                }
+pl_ctx_t *
+pl_ctx_get(client_t *client, xlator_t *xlator)
+{
+    void *tmp = NULL;
+    pl_ctx_t *ctx = NULL;
+    pl_ctx_t *setted_ctx = NULL;
 
-        }
+    client_ctx_get(client, xlator, &tmp);
+
+    ctx = tmp;
+
+    if (ctx != NULL)
+        goto out;
 
+    ctx = GF_CALLOC(1, sizeof(pl_ctx_t), gf_locks_mt_posix_lock_t);
+
+    if (ctx == NULL)
+        goto out;
+
+    pthread_mutex_init(&ctx->lock, NULL);
+    INIT_LIST_HEAD(&ctx->inodelk_lockers);
+    INIT_LIST_HEAD(&ctx->entrylk_lockers);
+    INIT_LIST_HEAD(&ctx->metalk_list);
+
+    setted_ctx = client_ctx_set(client, xlator, ctx);
+    if (ctx != setted_ctx) {
+        pthread_mutex_destroy(&ctx->lock);
+        GF_FREE(ctx);
+        ctx = setted_ctx;
+    }
+out:
+    return ctx;
 }
 
-void
-dump_inodelks (pl_inode_t *pl_inode)
+int
+pl_metalk_client_cleanup(xlator_t *this, pl_ctx_t *ctx)
 {
-        pthread_mutex_lock (&pl_inode->mutex);
+    pl_meta_lock_t *meta_lock = NULL;
+    pl_meta_lock_t *tmp_metalk = NULL;
+    pl_inode_t *pl_inode = NULL;
+    posix_lock_t *posix_lock = NULL;
+    posix_lock_t *tmp_posixlk = NULL;
+    struct list_head tmp_posixlk_list;
+
+    INIT_LIST_HEAD(&tmp_posixlk_list);
+
+    pthread_mutex_lock(&ctx->lock);
+    {
+        /* if this list is empty then pl_inode->metalk_list should be
+         * empty too. meta lock should in all cases be added/removed
+         * from both pl_ctx_t and pl_inode */
+        if (list_empty(&ctx->metalk_list))
+            goto unlock;
+
+        list_for_each_entry_safe(meta_lock, tmp_metalk, &ctx->metalk_list,
+                                 client_list)
         {
-                __dump_inodelks (pl_inode);
+            list_del_init(&meta_lock->client_list);
+
+            pl_inode = meta_lock->pl_inode;
+
+            pthread_mutex_lock(&pl_inode->mutex);
+
+            {
+                /* Since the migration status is unknown here
+                 * unwind all queued and blocked locks to check
+                 * migration status and find the correct
+                 * destination */
+                __unwind_queued_locks(pl_inode, &tmp_posixlk_list);
+
+                __unwind_blocked_locks(pl_inode, &tmp_posixlk_list);
+
+                list_del_init(&meta_lock->list);
+
+                pl_metalk_unref(meta_lock);
+            }
+            pthread_mutex_unlock(&pl_inode->mutex);
+
+            /* The corresponding ref is taken in
+             * pl_insert_metalk*/
+            inode_unref(pl_inode->inode);
         }
-        pthread_mutex_unlock (&pl_inode->mutex);
+    }
+
+unlock:
+    pthread_mutex_unlock(&ctx->lock);
+
+    list_for_each_entry_safe(posix_lock, tmp_posixlk, &tmp_posixlk_list, list)
+    {
+        list_del_init(&posix_lock->list);
+
+        STACK_UNWIND_STRICT(lk, posix_lock->frame, -1, EREMOTE,
+                            &posix_lock->user_flock, NULL);
 
+        __destroy_lock(posix_lock);
+    }
+    return 0;
 }
 
-void
-__dump_posixlks (pl_inode_t *pl_inode)
+static int
+pl_client_disconnect_cbk(xlator_t *this, client_t *client)
 {
-        posix_lock_t    *lock = NULL;
-        int             count = 0;
-        char            key[GF_DUMP_MAX_BUF_LEN];
+    pl_ctx_t *pl_ctx = pl_ctx_get(client, this);
+    if (pl_ctx) {
+        pl_inodelk_client_cleanup(this, pl_ctx);
+        pl_entrylk_client_cleanup(this, pl_ctx);
+        pl_metalk_client_cleanup(this, pl_ctx);
+    }
+
+    return 0;
+}
 
-        char tmp[4098];
+static int
+pl_client_destroy_cbk(xlator_t *this, client_t *client)
+{
+    void *tmp = NULL;
+    pl_ctx_t *pl_ctx = NULL;
 
-      list_for_each_entry (lock, &pl_inode->ext_list, list) {
+    pl_client_disconnect_cbk(this, client);
 
-              SET_FLOCK_PID (&lock->user_flock, lock);
-              gf_proc_dump_build_key(key,
-                                     "posixlk",
-                                     "posixlk[%d](%s)",
-                                     count,
-                                     lock->blocked ? "BLOCKED" : "ACTIVE");
-              pl_dump_lock (tmp, sizeof (tmp), &lock->user_flock,
-                            &lock->owner, lock->client, NULL,
-                            &lock->granted_time.tv_sec, &lock->blkd_time.tv_sec,
-                            (lock->blocked)? _gf_false: _gf_true);
-              gf_proc_dump_write(key, tmp);
+    client_ctx_del(client, this, &tmp);
 
-              count++;
-        }
+    if (tmp == NULL)
+        return 0;
+
+    pl_ctx = tmp;
+
+    GF_ASSERT(list_empty(&pl_ctx->inodelk_lockers));
+    GF_ASSERT(list_empty(&pl_ctx->entrylk_lockers));
+
+    pthread_mutex_destroy(&pl_ctx->lock);
+    GF_FREE(pl_ctx);
+
+    return 0;
+}
+
+int
+reconfigure(xlator_t *this, dict_t *options)
+{
+    posix_locks_private_t *priv = this->private;
+    int ret = -1;
+    char *tmp_str = NULL;
+
+    GF_OPTION_RECONF("trace", priv->trace, options, bool, out);
+
+    GF_OPTION_RECONF("monkey-unlocking", priv->monkey_unlocking, options, bool,
+                     out);
+
+    GF_OPTION_RECONF("revocation-secs", priv->revocation_secs, options, uint32,
+                     out);
+
+    GF_OPTION_RECONF("revocation-clear-all", priv->revocation_clear_all,
+                     options, bool, out);
+
+    GF_OPTION_RECONF("revocation-max-blocked", priv->revocation_max_blocked,
+                     options, uint32, out);
+
+    GF_OPTION_RECONF("notify-contention", priv->notify_contention, options,
+                     bool, out);
+
+    GF_OPTION_RECONF("notify-contention-delay", priv->notify_contention_delay,
+                     options, uint32, out);
+
+    GF_OPTION_RECONF("mandatory-locking", tmp_str, options, str, out);
+
+    GF_OPTION_RECONF("enforce-mandatory-lock", priv->mlock_enforced, options,
+                     bool, out);
+
+    if (!strcmp(tmp_str, "forced"))
+        priv->mandatory_mode = MLK_FORCED;
+    else if (!strcmp(tmp_str, "file"))
+        priv->mandatory_mode = MLK_FILE_BASED;
+    else if (!strcmp(tmp_str, "optimal"))
+        priv->mandatory_mode = MLK_OPTIMAL;
+    else
+        priv->mandatory_mode = MLK_NONE;
+
+    ret = 0;
+
+out:
+    return ret;
+}
+
+int
+init(xlator_t *this)
+{
+    posix_locks_private_t *priv = NULL;
+    xlator_list_t *trav = NULL;
+    char *tmp_str = NULL;
+    int ret = -1;
+
+    if (!this->children || this->children->next) {
+        gf_log(this->name, GF_LOG_CRITICAL,
+               "FATAL: posix-locks should have exactly one child");
+        goto out;
+    }
+
+    if (!this->parents) {
+        gf_log(this->name, GF_LOG_WARNING,
+               "Volume is dangling. Please check the volume file.");
+    }
+
+    trav = this->children;
+    while (trav->xlator->children)
+        trav = trav->xlator->children;
+
+    if (strncmp("storage/", trav->xlator->type, 8)) {
+        gf_log(this->name, GF_LOG_CRITICAL,
+               "'locks' translator is not loaded over a storage "
+               "translator");
+        goto out;
+    }
+
+    priv = GF_CALLOC(1, sizeof(*priv), gf_locks_mt_posix_locks_private_t);
+
+    GF_OPTION_INIT("mandatory-locking", tmp_str, str, out);
+    if (!strcmp(tmp_str, "forced"))
+        priv->mandatory_mode = MLK_FORCED;
+    else if (!strcmp(tmp_str, "file"))
+        priv->mandatory_mode = MLK_FILE_BASED;
+    else if (!strcmp(tmp_str, "optimal"))
+        priv->mandatory_mode = MLK_OPTIMAL;
+    else
+        priv->mandatory_mode = MLK_NONE;
+
+    tmp_str = NULL;
+
+    GF_OPTION_INIT("trace", priv->trace, bool, out);
+
+    GF_OPTION_INIT("monkey-unlocking", priv->monkey_unlocking, bool, out);
+
+    GF_OPTION_INIT("revocation-secs", priv->revocation_secs, uint32, out);
+
+    GF_OPTION_INIT("revocation-clear-all", priv->revocation_clear_all, bool,
+                   out);
+
+    GF_OPTION_INIT("revocation-max-blocked", priv->revocation_max_blocked,
+                   uint32, out);
+
+    GF_OPTION_INIT("notify-contention", priv->notify_contention, bool, out);
+
+    GF_OPTION_INIT("notify-contention-delay", priv->notify_contention_delay,
+                   uint32, out);
+
+    GF_OPTION_INIT("enforce-mandatory-lock", priv->mlock_enforced, bool, out);
+
+    this->local_pool = mem_pool_new(pl_local_t, 32);
+    if (!this->local_pool) {
+        ret = -1;
+        gf_log(this->name, GF_LOG_ERROR,
+               "failed to create local_t's memory pool");
+        goto out;
+    }
+
+    this->private = priv;
+    ret = 0;
+
+out:
+    if (ret) {
+        GF_FREE(priv);
+    }
+    return ret;
 }
 
 void
-dump_posixlks (pl_inode_t *pl_inode)
+fini(xlator_t *this)
 {
-        pthread_mutex_lock (&pl_inode->mutex);
-        {
-                __dump_posixlks (pl_inode);
-        }
-        pthread_mutex_unlock (&pl_inode->mutex);
+    posix_locks_private_t *priv = this->private;
+    if (!priv)
+        return;
+    this->private = NULL;
+    if (this->local_pool) {
+        mem_pool_destroy(this->local_pool);
+        this->local_pool = NULL;
+    }
+    GF_FREE(priv->brickname);
+    GF_FREE(priv);
+
+    return;
+}
+
+int
+pl_inodelk(call_frame_t *frame, xlator_t *this, const char *volume, loc_t *loc,
+           int32_t cmd, struct gf_flock *flock, dict_t *xdata);
+
+int
+pl_finodelk(call_frame_t *frame, xlator_t *this, const char *volume, fd_t *fd,
+            int32_t cmd, struct gf_flock *flock, dict_t *xdata);
 
+int
+pl_entrylk(call_frame_t *frame, xlator_t *this, const char *volume, loc_t *loc,
+           const char *basename, entrylk_cmd cmd, entrylk_type type,
+           dict_t *xdata);
+
+int
+pl_fentrylk(call_frame_t *frame, xlator_t *this, const char *volume, fd_t *fd,
+            const char *basename, entrylk_cmd cmd, entrylk_type type,
+            dict_t *xdata);
+
+int32_t
+pl_rename_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int32_t op_ret,
+              int32_t op_errno, struct iatt *buf, struct iatt *preoldparent,
+              struct iatt *postoldparent, struct iatt *prenewparent,
+              struct iatt *postnewparent, dict_t *xdata)
+{
+    pl_inode_remove_cbk(this, cookie, op_ret < 0 ? op_errno : 0);
+
+    PL_STACK_UNWIND(rename, xdata, frame, op_ret, op_errno, buf, preoldparent,
+                    postoldparent, prenewparent, postnewparent, xdata);
+
+    return 0;
 }
 
 int32_t
-pl_dump_inode_priv (xlator_t *this, inode_t *inode)
+pl_rename(call_frame_t *frame, xlator_t *this, loc_t *oldloc, loc_t *newloc,
+          dict_t *xdata)
 {
+    int32_t error;
 
-        int             ret = -1;
-        uint64_t        tmp_pl_inode = 0;
-        pl_inode_t      *pl_inode = NULL;
-        char            *pathname = NULL;
-        gf_boolean_t    section_added = _gf_false;
+    error = PL_INODE_REMOVE(rename, frame, this, oldloc, newloc, pl_rename,
+                            pl_rename_cbk, oldloc, newloc, xdata);
+    if (error > 0) {
+        STACK_UNWIND_STRICT(rename, frame, -1, error, NULL, NULL, NULL, NULL,
+                            NULL, NULL);
+    }
 
-        int count      = 0;
+    return 0;
+}
 
-        if (!inode) {
-                errno = EINVAL;
-                goto out;
+posix_lock_t *
+gf_lkmig_info_to_posix_lock(call_frame_t *frame, lock_migration_info_t *lmi)
+{
+    posix_lock_t *lock = GF_CALLOC(1, sizeof(posix_lock_t),
+                                   gf_locks_mt_posix_lock_t);
+    if (!lock)
+        goto out;
+
+    lock->fl_start = lmi->flock.l_start;
+    lock->fl_type = lmi->flock.l_type;
+
+    if (lmi->flock.l_len == 0)
+        lock->fl_end = LLONG_MAX;
+    else
+        lock->fl_end = lmi->flock.l_start + lmi->flock.l_len - 1;
+
+    lock->client = frame->root->client;
+
+    lock->lk_flags = lmi->lk_flags;
+
+    lock->client_uid = gf_strdup(lmi->client_uid);
+    if (lock->client_uid == NULL) {
+        GF_FREE(lock);
+        lock = NULL;
+        goto out;
+    }
+
+    lock->client_pid = lmi->flock.l_pid;
+    lock->owner = lmi->flock.l_owner;
+
+    INIT_LIST_HEAD(&lock->list);
+
+out:
+    return lock;
+}
+
+/* This function is supposed to write the active locks from the source brick(in
+ * rebalance context) and write here. Hence, will add the locks directly to the
+ * pl_inode->ext_list*/
+int
+pl_write_active_locks(call_frame_t *frame, pl_inode_t *pl_inode,
+                      lock_migration_info_t *locklist)
+{
+    posix_lock_t *newlock = NULL;
+    lock_migration_info_t *temp = NULL;
+    int ret = 0;
+
+    pthread_mutex_lock(&pl_inode->mutex);
+    {
+        /* Just making sure the activelk list is empty. Should not
+         * happen though*/
+        if (!list_empty(&pl_inode->ext_list)) {
+            pthread_mutex_unlock(&pl_inode->mutex);
+            gf_msg(THIS->name, GF_LOG_ERROR, 0, 0, "invalid locks found");
+
+            ret = -1;
+            goto out;
         }
 
-        ret = TRY_LOCK (&inode->lock);
-        if (ret)
-                goto out;
-        {
-                ret = __inode_ctx_get (inode, this, &tmp_pl_inode);
-                if (ret)
-                        goto unlock;
+        /* This list also should not be empty */
+        if (list_empty(&locklist->list)) {
+            pthread_mutex_unlock(&pl_inode->mutex);
+            gf_msg(THIS->name, GF_LOG_ERROR, 0, 0, "empty lock list");
+
+            ret = -1;
+            goto out;
         }
-unlock:
-        UNLOCK (&inode->lock);
-        if (ret)
-                goto out;
 
-        pl_inode = (pl_inode_t *)(long)tmp_pl_inode;
-        if (!pl_inode) {
+        list_for_each_entry(temp, &locklist->list, list)
+        {
+            newlock = gf_lkmig_info_to_posix_lock(frame, temp);
+            if (!newlock) {
+                pthread_mutex_unlock(&pl_inode->mutex);
+                gf_msg(THIS->name, GF_LOG_ERROR, 0, 0,
+                       "mem allocation failed for newlock");
+
                 ret = -1;
                 goto out;
+            }
+            list_add_tail(&newlock->list, &pl_inode->ext_list);
         }
+    }
+    /*TODO: What if few lock add failed with ENOMEM. Should the already
+     *      added locks be clearted */
+    pthread_mutex_unlock(&pl_inode->mutex);
+out:
+    return ret;
+}
 
-        gf_proc_dump_add_section("xlator.features.locks.%s.inode", this->name);
-        section_added = _gf_true;
+static int
+pl_setactivelk(call_frame_t *frame, xlator_t *this, loc_t *loc,
+               lock_migration_info_t *locklist, dict_t *xdata)
+{
+    int op_ret = 0;
+    int op_errno = 0;
+    int ret = 0;
 
-        /*We are safe to call __inode_path since we have the
-         * inode->table->lock */
-        __inode_path (inode, NULL, &pathname);
-        if (pathname)
-                gf_proc_dump_write ("path", "%s", pathname);
+    pl_inode_t *pl_inode = pl_inode_get(this, loc->inode, NULL);
+    if (!pl_inode) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, 0, "pl_inode_get failed");
 
-        gf_proc_dump_write("mandatory", "%d", pl_inode->mandatory);
+        op_ret = -1;
+        op_errno = ENOMEM;
+        goto out;
+    }
+    ret = pl_write_active_locks(frame, pl_inode, locklist);
 
-        ret = pthread_mutex_trylock (&pl_inode->mutex);
-        if (ret)
-                goto out;
-        {
-                count = __get_entrylk_count (this, pl_inode);
-                if (count) {
-                        gf_proc_dump_write("entrylk-count", "%d", count);
-                        __dump_entrylks (pl_inode);
-                }
+    op_ret = ret;
 
-                count = __get_inodelk_count (this, pl_inode, NULL);
-                if (count) {
-                        gf_proc_dump_write("inodelk-count", "%d", count);
-                        __dump_inodelks (pl_inode);
-                }
+out:
+    STACK_UNWIND_STRICT(setactivelk, frame, op_ret, op_errno, NULL);
 
-                count = __get_posixlk_count (this, pl_inode);
-                if (count) {
-                        gf_proc_dump_write("posixlk-count", "%d", count);
-                        __dump_posixlks (pl_inode);
-                }
-        }
-        pthread_mutex_unlock (&pl_inode->mutex);
+    return 0;
+}
 
-out:
-        GF_FREE (pathname);
+int32_t
+pl_unlink_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int32_t op_ret,
+              int32_t op_errno, struct iatt *preparent, struct iatt *postparent,
+              dict_t *xdata)
+{
+    pl_inode_remove_cbk(this, cookie, op_ret < 0 ? op_errno : 0);
 
-        if (ret && inode) {
-                if (!section_added)
-                        gf_proc_dump_add_section ("xlator.features.locks.%s."
-                                                  "inode", this->name);
-                gf_proc_dump_write ("Unable to print lock state", "(Lock "
-                                    "acquisition failure) %s",
-                                    uuid_utoa (inode->gfid));
-        }
-        return ret;
+    PL_STACK_UNWIND(unlink, xdata, frame, op_ret, op_errno, preparent,
+                    postparent, xdata);
+
+    return 0;
 }
 
 int32_t
-mem_acct_init (xlator_t *this)
+pl_unlink(call_frame_t *frame, xlator_t *this, loc_t *loc, int xflag,
+          dict_t *xdata)
 {
-        int     ret = -1;
+    int32_t error;
 
-        if (!this)
-                return ret;
+    error = PL_INODE_REMOVE(unlink, frame, this, loc, NULL, pl_unlink,
+                            pl_unlink_cbk, loc, xflag, xdata);
+    if (error > 0) {
+        STACK_UNWIND_STRICT(unlink, frame, -1, error, NULL, NULL, NULL);
+    }
 
-        ret = xlator_mem_acct_init (this, gf_locks_mt_end + 1);
+    return 0;
+}
 
-        if (ret != 0) {
-                gf_log (this->name, GF_LOG_ERROR, "Memory accounting init"
-                                "failed");
-                return ret;
-        }
+int32_t
+pl_mkdir_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int32_t op_ret,
+             int32_t op_errno, inode_t *inode, struct iatt *buf,
+             struct iatt *preparent, struct iatt *postparent, dict_t *xdata)
+{
+    PL_STACK_UNWIND_FOR_CLIENT(mkdir, xdata, frame, op_ret, op_errno, inode,
+                               buf, preparent, postparent, xdata);
+    return 0;
+}
 
-        return ret;
+int
+pl_mkdir(call_frame_t *frame, xlator_t *this, loc_t *loc, mode_t mode,
+         mode_t umask, dict_t *xdata)
+{
+    PL_LOCAL_GET_REQUESTS(frame, this, xdata, ((fd_t *)NULL), loc, NULL);
+    STACK_WIND(frame, pl_mkdir_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->mkdir, loc, mode, umask, xdata);
+    return 0;
 }
 
+int32_t
+pl_stat_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int32_t op_ret,
+            int32_t op_errno, struct iatt *buf, dict_t *xdata)
+{
+    PL_STACK_UNWIND_FOR_CLIENT(stat, xdata, frame, op_ret, op_errno, buf,
+                               xdata);
+    return 0;
+}
 
-pl_ctx_t*
-pl_ctx_get (client_t *client, xlator_t *xlator)
+int
+pl_stat(call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xdata)
 {
-        void *tmp = NULL;
-        pl_ctx_t *ctx = NULL;
+    PL_LOCAL_GET_REQUESTS(frame, this, xdata, ((fd_t *)NULL), loc, NULL);
+    STACK_WIND(frame, pl_stat_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->stat, loc, xdata);
+    return 0;
+}
 
-        client_ctx_get (client, xlator, &tmp);
+int32_t
+pl_mknod_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int32_t op_ret,
+             int32_t op_errno, inode_t *inode, struct iatt *buf,
+             struct iatt *preparent, struct iatt *postparent, dict_t *xdata)
+{
+    PL_STACK_UNWIND_FOR_CLIENT(mknod, xdata, frame, op_ret, op_errno, inode,
+                               buf, preparent, postparent, xdata);
+    return 0;
+}
 
-        ctx = tmp;
+int
+pl_mknod(call_frame_t *frame, xlator_t *this, loc_t *loc, mode_t mode,
+         dev_t rdev, mode_t umask, dict_t *xdata)
+{
+    PL_LOCAL_GET_REQUESTS(frame, this, xdata, ((fd_t *)NULL), loc, NULL);
+    STACK_WIND(frame, pl_mknod_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->mknod, loc, mode, rdev, umask, xdata);
+    return 0;
+}
 
-        if (ctx != NULL)
-                goto out;
+int32_t
+pl_rmdir_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int32_t op_ret,
+             int32_t op_errno, struct iatt *preparent, struct iatt *postparent,
+             dict_t *xdata)
+{
+    pl_inode_remove_cbk(this, cookie, op_ret < 0 ? op_errno : 0);
 
-        ctx = GF_CALLOC (1, sizeof (pl_ctx_t), gf_locks_mt_posix_lock_t);
+    PL_STACK_UNWIND_FOR_CLIENT(rmdir, xdata, frame, op_ret, op_errno, preparent,
+                               postparent, xdata);
 
-        if (ctx == NULL)
-                goto out;
+    return 0;
+}
 
-        pthread_mutex_init (&ctx->lock, NULL);
-	INIT_LIST_HEAD (&ctx->inodelk_lockers);
-	INIT_LIST_HEAD (&ctx->entrylk_lockers);
+int
+pl_rmdir(call_frame_t *frame, xlator_t *this, loc_t *loc, int xflags,
+         dict_t *xdata)
+{
+    int32_t error;
 
-        if (client_ctx_set (client, xlator, ctx) != 0) {
-                pthread_mutex_destroy (&ctx->lock);
-                GF_FREE (ctx);
-                ctx = NULL;
-        }
-out:
-        return ctx;
+    error = PL_INODE_REMOVE(rmdir, frame, this, loc, NULL, pl_rmdir,
+                            pl_rmdir_cbk, loc, xflags, xdata);
+    if (error > 0) {
+        STACK_UNWIND_STRICT(rmdir, frame, -1, error, NULL, NULL, NULL);
+    }
+
+    return 0;
 }
 
+int32_t
+pl_symlink_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+               int32_t op_ret, int32_t op_errno, inode_t *inode,
+               struct iatt *buf, struct iatt *preparent,
+               struct iatt *postparent, dict_t *xdata)
+{
+    PL_STACK_UNWIND_FOR_CLIENT(symlink, xdata, frame, op_ret, op_errno, inode,
+                               buf, preparent, postparent, xdata);
+    return 0;
+}
 
-static int
-pl_client_disconnect_cbk (xlator_t *this, client_t *client)
+int
+pl_symlink(call_frame_t *frame, xlator_t *this, const char *linkname,
+           loc_t *loc, mode_t umask, dict_t *xdata)
+{
+    PL_LOCAL_GET_REQUESTS(frame, this, xdata, ((fd_t *)NULL), loc, NULL);
+    STACK_WIND(frame, pl_symlink_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->symlink, linkname, loc, umask, xdata);
+    return 0;
+}
+
+int32_t
+pl_link_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int32_t op_ret,
+            int32_t op_errno, inode_t *inode, struct iatt *buf,
+            struct iatt *preparent, struct iatt *postparent, dict_t *xdata)
 {
-        pl_ctx_t *pl_ctx = NULL;
+    pl_inode_t *pl_inode = (pl_inode_t *)cookie;
 
-	pl_ctx = pl_ctx_get (client, this);
+    if (op_ret >= 0) {
+        pthread_mutex_lock(&pl_inode->mutex);
 
-	pl_inodelk_client_cleanup (this, pl_ctx);
+        /* TODO: can happen pl_inode->links == 0 ? */
+        if (pl_inode->links >= 0) {
+            pl_inode->links++;
+        }
 
-	pl_entrylk_client_cleanup (this, pl_ctx);
+        pthread_mutex_unlock(&pl_inode->mutex);
+    }
 
-	return 0;
+    PL_STACK_UNWIND_FOR_CLIENT(link, xdata, frame, op_ret, op_errno, inode, buf,
+                               preparent, postparent, xdata);
+    return 0;
 }
 
-
-static int
-pl_client_destroy_cbk (xlator_t *this, client_t *client)
+int
+pl_link(call_frame_t *frame, xlator_t *this, loc_t *oldloc, loc_t *newloc,
+        dict_t *xdata)
 {
-        void     *tmp    = NULL;
-        pl_ctx_t *pl_ctx = NULL;
+    pl_inode_t *pl_inode;
 
-	pl_client_disconnect_cbk (this, client);
+    pl_inode = pl_inode_get(this, oldloc->inode, NULL);
+    if (pl_inode == NULL) {
+        STACK_UNWIND_STRICT(link, frame, -1, ENOMEM, NULL, NULL, NULL, NULL,
+                            NULL);
+        return 0;
+    }
+
+    PL_LOCAL_GET_REQUESTS(frame, this, xdata, ((fd_t *)NULL), oldloc, newloc);
+    STACK_WIND_COOKIE(frame, pl_link_cbk, pl_inode, FIRST_CHILD(this),
+                      FIRST_CHILD(this)->fops->link, oldloc, newloc, xdata);
+    return 0;
+}
+
+int32_t
+pl_fsync_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int32_t op_ret,
+             int32_t op_errno, struct iatt *prebuf, struct iatt *postbuf,
+             dict_t *xdata)
+{
+    PL_STACK_UNWIND_FOR_CLIENT(fsync, xdata, frame, op_ret, op_errno, prebuf,
+                               postbuf, xdata);
+    return 0;
+}
 
-        client_ctx_del (client, this, &tmp);
+int
+pl_fsync(call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t datasync,
+         dict_t *xdata)
+{
+    PL_LOCAL_GET_REQUESTS(frame, this, xdata, fd, NULL, NULL);
+    STACK_WIND(frame, pl_fsync_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->fsync, fd, datasync, xdata);
+    return 0;
+}
 
-        if (tmp == NULL)
-                return 0;
+int32_t
+pl_readdir_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+               int32_t op_ret, int32_t op_errno, gf_dirent_t *entries,
+               dict_t *xdata)
+{
+    PL_STACK_UNWIND_FOR_CLIENT(readdir, xdata, frame, op_ret, op_errno, entries,
+                               xdata);
+    return 0;
+}
 
-        pl_ctx = tmp;
+int
+pl_readdir(call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
+           off_t offset, dict_t *xdata)
+{
+    PL_LOCAL_GET_REQUESTS(frame, this, xdata, fd, NULL, NULL);
+    STACK_WIND(frame, pl_readdir_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->readdir, fd, size, offset, xdata);
+    return 0;
+}
 
-	GF_ASSERT (list_empty(&pl_ctx->inodelk_lockers));
-	GF_ASSERT (list_empty(&pl_ctx->entrylk_lockers));
+int32_t
+pl_fsyncdir_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                int32_t op_ret, int32_t op_errno, dict_t *xdata)
+{
+    PL_STACK_UNWIND_FOR_CLIENT(fsyncdir, xdata, frame, op_ret, op_errno, xdata);
+    return 0;
+}
 
-	pthread_mutex_destroy (&pl_ctx->lock);
-        GF_FREE (pl_ctx);
+int
+pl_fsyncdir(call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t datasync,
+            dict_t *xdata)
+{
+    PL_LOCAL_GET_REQUESTS(frame, this, xdata, fd, NULL, NULL);
+    STACK_WIND(frame, pl_fsyncdir_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->fsyncdir, fd, datasync, xdata);
+    return 0;
+}
 
-        return 0;
+int32_t
+pl_statfs_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int32_t op_ret,
+              int32_t op_errno, struct statvfs *buf, dict_t *xdata)
+{
+    PL_STACK_UNWIND_FOR_CLIENT(statfs, xdata, frame, op_ret, op_errno, buf,
+                               xdata);
+    return 0;
 }
 
 int
-reconfigure (xlator_t *this, dict_t *options)
+pl_statfs(call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xdata)
 {
-        posix_locks_private_t *priv = NULL;
-        int                   ret = -1;
+    PL_LOCAL_GET_REQUESTS(frame, this, xdata, ((fd_t *)NULL), loc, NULL);
+    STACK_WIND(frame, pl_statfs_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->statfs, loc, xdata);
+    return 0;
+}
 
-        priv = this->private;
+int32_t
+pl_removexattr_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                   int32_t op_ret, int32_t op_errno, dict_t *xdata)
+{
+    pl_local_t *local = NULL;
+    pl_inode_t *pl_inode = NULL;
 
-        GF_OPTION_RECONF ("trace", priv->trace, options, bool, out);
+    local = frame->local;
+    if (local && local->update_mlock_enforced_flag && op_ret != -1) {
+        pl_inode = pl_inode_get(this, local->inode, NULL);
+        if (!pl_inode) {
+            op_ret = -1;
+            op_errno = ENOMEM;
+            goto unwind;
+        }
 
-        ret = 0;
-out:
-        return ret;
+        pthread_mutex_lock(&pl_inode->mutex);
+        {
+            pl_inode->mlock_enforced = _gf_false;
+            pl_inode->check_mlock_info = _gf_false;
+            pl_inode->track_fop_wind_count = _gf_true;
+        }
+        pthread_mutex_unlock(&pl_inode->mutex);
+    }
+
+unwind:
+    PL_STACK_UNWIND_FOR_CLIENT(removexattr, xdata, frame, op_ret, op_errno,
+                               xdata);
+    return 0;
 }
 
 int
-init (xlator_t *this)
+pl_removexattr(call_frame_t *frame, xlator_t *this, loc_t *loc,
+               const char *name, dict_t *xdata)
 {
-        posix_locks_private_t *priv = NULL;
-        xlator_list_t         *trav = NULL;
-        data_t                *mandatory = NULL;
-        int                   ret = -1;
+    int op_ret = 0;
+    int op_errno = EINVAL;
+    posix_locks_private_t *priv = this->private;
 
-        if (!this->children || this->children->next) {
-                gf_log (this->name, GF_LOG_CRITICAL,
-                        "FATAL: posix-locks should have exactly one child");
-                goto out;
+    PL_LOCAL_GET_REQUESTS(frame, this, xdata, ((fd_t *)NULL), loc, NULL);
+
+    PL_CHECK_LOCK_ENFORCE_KEY(frame, ((dict_t *)NULL), name, this, loc,
+                              ((fd_t *)NULL), priv);
+
+    STACK_WIND(frame, pl_removexattr_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->removexattr, loc, name, xdata);
+    return 0;
+
+unwind:
+    PL_STACK_UNWIND_FOR_CLIENT(removexattr, xdata, frame, op_ret, op_errno,
+                               NULL);
+
+    return 0;
+}
+
+int32_t
+pl_fremovexattr_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                    int32_t op_ret, int32_t op_errno, dict_t *xdata)
+{
+    pl_local_t *local = NULL;
+    pl_inode_t *pl_inode = NULL;
+
+    local = frame->local;
+    if (local && local->update_mlock_enforced_flag && op_ret != -1) {
+        pl_inode = pl_inode_get(this, local->inode, NULL);
+        if (!pl_inode) {
+            op_ret = -1;
+            op_errno = ENOMEM;
+            goto unwind;
         }
 
-        if (!this->parents) {
-                gf_log (this->name, GF_LOG_WARNING,
-                        "Volume is dangling. Please check the volume file.");
+        pthread_mutex_lock(&pl_inode->mutex);
+        {
+            pl_inode->mlock_enforced = _gf_false;
+            pl_inode->check_mlock_info = _gf_false;
         }
+        pthread_mutex_unlock(&pl_inode->mutex);
+    }
 
-        trav = this->children;
-        while (trav->xlator->children)
-                trav = trav->xlator->children;
+unwind:
+    PL_STACK_UNWIND_FOR_CLIENT(fremovexattr, xdata, frame, op_ret, op_errno,
+                               xdata);
+    return 0;
+}
 
-        if (strncmp ("storage/", trav->xlator->type, 8)) {
-                gf_log (this->name, GF_LOG_CRITICAL,
-                        "'locks' translator is not loaded over a storage "
-                        "translator");
-                goto out;
-        }
+int
+pl_fremovexattr(call_frame_t *frame, xlator_t *this, fd_t *fd, const char *name,
+                dict_t *xdata)
+{
+    int op_ret = -1;
+    int op_errno = EINVAL;
+    posix_locks_private_t *priv = this->private;
 
-        priv = GF_CALLOC (1, sizeof (*priv),
-                          gf_locks_mt_posix_locks_private_t);
+    PL_LOCAL_GET_REQUESTS(frame, this, xdata, fd, NULL, NULL);
 
-        mandatory = dict_get (this->options, "mandatory-locks");
-        if (mandatory)
-                gf_log (this->name, GF_LOG_WARNING,
-                        "mandatory locks not supported in this minor release.");
+    PL_CHECK_LOCK_ENFORCE_KEY(frame, ((dict_t *)NULL), name, this,
+                              ((loc_t *)NULL), fd, priv);
 
-        GF_OPTION_INIT ("trace", priv->trace, bool, out);
+    STACK_WIND(frame, pl_fremovexattr_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->fremovexattr, fd, name, xdata);
+    return 0;
 
-        this->local_pool = mem_pool_new (pl_local_t, 32);
-        if (!this->local_pool) {
-                ret = -1;
-                gf_log (this->name, GF_LOG_ERROR,
-                        "failed to create local_t's memory pool");
-                goto out;
-        }
+unwind:
+    PL_STACK_UNWIND_FOR_CLIENT(fremovexattr, xdata, frame, op_ret, op_errno,
+                               NULL);
+    return 0;
+}
 
-        this->private = priv;
-        ret = 0;
+int32_t
+pl_rchecksum_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                 int32_t op_ret, int32_t op_errno, uint32_t weak_cksum,
+                 uint8_t *strong_cksum, dict_t *xdata)
+{
+    PL_STACK_UNWIND_FOR_CLIENT(rchecksum, xdata, frame, op_ret, op_errno,
+                               weak_cksum, strong_cksum, xdata);
+    return 0;
+}
 
-out:
-        if (ret) {
-                GF_FREE (priv);
-        }
-        return ret;
+int
+pl_rchecksum(call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
+             int32_t len, dict_t *xdata)
+{
+    PL_LOCAL_GET_REQUESTS(frame, this, xdata, fd, NULL, NULL);
+    STACK_WIND(frame, pl_rchecksum_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->rchecksum, fd, offset, len, xdata);
+    return 0;
 }
 
+int32_t
+pl_xattrop_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+               int32_t op_ret, int32_t op_errno, dict_t *dict, dict_t *xdata)
+{
+    PL_STACK_UNWIND_FOR_CLIENT(xattrop, xdata, frame, op_ret, op_errno, dict,
+                               xdata);
+    return 0;
+}
 
 int
-fini (xlator_t *this)
+pl_xattrop(call_frame_t *frame, xlator_t *this, loc_t *loc,
+           gf_xattrop_flags_t optype, dict_t *xattr, dict_t *xdata)
 {
-        posix_locks_private_t *priv = NULL;
+    PL_LOCAL_GET_REQUESTS(frame, this, xdata, ((fd_t *)NULL), loc, NULL);
+    STACK_WIND(frame, pl_xattrop_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->xattrop, loc, optype, xattr, xdata);
+    return 0;
+}
 
-        priv = this->private;
-        if (!priv)
-                return 0;
-        this->private = NULL;
-        GF_FREE (priv->brickname);
-        GF_FREE (priv);
+int32_t
+pl_fxattrop_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                int32_t op_ret, int32_t op_errno, dict_t *dict, dict_t *xdata)
+{
+    PL_STACK_UNWIND_FOR_CLIENT(fxattrop, xdata, frame, op_ret, op_errno, dict,
+                               xdata);
+    return 0;
+}
 
-        return 0;
+int
+pl_fxattrop(call_frame_t *frame, xlator_t *this, fd_t *fd,
+            gf_xattrop_flags_t optype, dict_t *xattr, dict_t *xdata)
+{
+    PL_LOCAL_GET_REQUESTS(frame, this, xdata, fd, NULL, NULL);
+    STACK_WIND(frame, pl_fxattrop_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->fxattrop, fd, optype, xattr, xdata);
+    return 0;
 }
 
+int32_t
+pl_setattr_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+               int32_t op_ret, int32_t op_errno, struct iatt *statpre,
+               struct iatt *statpost, dict_t *xdata)
+{
+    PL_STACK_UNWIND_FOR_CLIENT(setattr, xdata, frame, op_ret, op_errno, statpre,
+                               statpost, xdata);
+    return 0;
+}
 
 int
-pl_inodelk (call_frame_t *frame, xlator_t *this,
-            const char *volume, loc_t *loc, int32_t cmd, struct gf_flock *flock,
-            dict_t *xdata);
+pl_setattr(call_frame_t *frame, xlator_t *this, loc_t *loc, struct iatt *stbuf,
+           int32_t valid, dict_t *xdata)
+{
+    PL_LOCAL_GET_REQUESTS(frame, this, xdata, ((fd_t *)NULL), loc, NULL);
+    STACK_WIND(frame, pl_setattr_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->setattr, loc, stbuf, valid, xdata);
+    return 0;
+}
+
+int32_t
+pl_fsetattr_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                int32_t op_ret, int32_t op_errno, struct iatt *statpre,
+                struct iatt *statpost, dict_t *xdata)
+{
+    PL_STACK_UNWIND_FOR_CLIENT(fsetattr, xdata, frame, op_ret, op_errno,
+                               statpre, statpost, xdata);
+    return 0;
+}
 
 int
-pl_finodelk (call_frame_t *frame, xlator_t *this,
-             const char *volume, fd_t *fd, int32_t cmd, struct gf_flock *flock,
-             dict_t *xdata);
+pl_fsetattr(call_frame_t *frame, xlator_t *this, fd_t *fd, struct iatt *stbuf,
+            int32_t valid, dict_t *xdata)
+{
+    PL_LOCAL_GET_REQUESTS(frame, this, xdata, fd, NULL, NULL);
+    STACK_WIND(frame, pl_fsetattr_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->fsetattr, fd, stbuf, valid, xdata);
+    return 0;
+}
+
+int32_t
+pl_fallocate_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                 int32_t op_ret, int32_t op_errno, struct iatt *pre,
+                 struct iatt *post, dict_t *xdata)
+{
+    PL_STACK_UNWIND_FOR_CLIENT(fallocate, xdata, frame, op_ret, op_errno, pre,
+                               post, xdata);
+    return 0;
+}
 
 int
-pl_entrylk (call_frame_t *frame, xlator_t *this,
-            const char *volume, loc_t *loc, const char *basename,
-            entrylk_cmd cmd, entrylk_type type, dict_t *xdata);
+pl_fallocate(call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t keep_size,
+             off_t offset, size_t len, dict_t *xdata)
+{
+    PL_LOCAL_GET_REQUESTS(frame, this, xdata, fd, NULL, NULL);
+    STACK_WIND(frame, pl_fallocate_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->fallocate, fd, keep_size, offset, len,
+               xdata);
+    return 0;
+}
+
+int32_t
+pl_readlink_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                int32_t op_ret, int32_t op_errno, const char *path,
+                struct iatt *buf, dict_t *xdata)
+{
+    PL_STACK_UNWIND_FOR_CLIENT(readlink, xdata, frame, op_ret, op_errno, path,
+                               buf, xdata);
+    return 0;
+}
 
 int
-pl_fentrylk (call_frame_t *frame, xlator_t *this,
-             const char *volume, fd_t *fd, const char *basename,
-             entrylk_cmd cmd, entrylk_type type, dict_t *xdata);
+pl_readlink(call_frame_t *frame, xlator_t *this, loc_t *loc, size_t size,
+            dict_t *xdata)
+{
+    PL_LOCAL_GET_REQUESTS(frame, this, xdata, ((fd_t *)NULL), loc, NULL);
+    STACK_WIND(frame, pl_readlink_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->readlink, loc, size, xdata);
+    return 0;
+}
+
+int32_t
+pl_access_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int32_t op_ret,
+              int32_t op_errno, dict_t *xdata)
+{
+    PL_STACK_UNWIND_FOR_CLIENT(access, xdata, frame, op_ret, op_errno, xdata);
+    return 0;
+}
+
+int
+pl_access(call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t mask,
+          dict_t *xdata)
+{
+    PL_LOCAL_GET_REQUESTS(frame, this, xdata, ((fd_t *)NULL), loc, NULL);
+    STACK_WIND(frame, pl_access_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->access, loc, mask, xdata);
+    return 0;
+}
+
+int32_t
+pl_seek_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int32_t op_ret,
+            int32_t op_errno, off_t offset, dict_t *xdata)
+{
+    PL_STACK_UNWIND_FOR_CLIENT(seek, xdata, frame, op_ret, op_errno, offset,
+                               xdata);
+    return 0;
+}
+
+int32_t
+pl_seek(call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
+        gf_seek_what_t what, dict_t *xdata)
+{
+    PL_LOCAL_GET_REQUESTS(frame, this, xdata, fd, NULL, NULL);
+    STACK_WIND(frame, pl_seek_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->seek, fd, offset, what, xdata);
+    return 0;
+}
 
 struct xlator_fops fops = {
-        .lookup      = pl_lookup,
-        .create      = pl_create,
-        .truncate    = pl_truncate,
-        .ftruncate   = pl_ftruncate,
-        .open        = pl_open,
-        .readv       = pl_readv,
-        .writev      = pl_writev,
-        .lk          = pl_lk,
-        .inodelk     = pl_inodelk,
-        .finodelk    = pl_finodelk,
-        .entrylk     = pl_entrylk,
-        .fentrylk    = pl_fentrylk,
-        .flush       = pl_flush,
-        .opendir     = pl_opendir,
-        .readdirp    = pl_readdirp,
-        .getxattr    = pl_getxattr,
-        .fgetxattr   = pl_fgetxattr,
-        .fsetxattr   = pl_fsetxattr,
+    .lookup = pl_lookup,
+    .create = pl_create,
+    .fstat = pl_fstat,
+    .truncate = pl_truncate,
+    .ftruncate = pl_ftruncate,
+    .discard = pl_discard,
+    .zerofill = pl_zerofill,
+    .open = pl_open,
+    .readv = pl_readv,
+    .writev = pl_writev,
+    .lk = pl_lk,
+    .inodelk = pl_inodelk,
+    .finodelk = pl_finodelk,
+    .entrylk = pl_entrylk,
+    .fentrylk = pl_fentrylk,
+    .flush = pl_flush,
+    .opendir = pl_opendir,
+    .readdirp = pl_readdirp,
+    .setxattr = pl_setxattr,
+    .fsetxattr = pl_fsetxattr,
+    .getxattr = pl_getxattr,
+    .fgetxattr = pl_fgetxattr,
+    .removexattr = pl_removexattr,
+    .fremovexattr = pl_fremovexattr,
+    .rename = pl_rename,
+    .getactivelk = pl_getactivelk,
+    .setactivelk = pl_setactivelk,
+    .unlink = pl_unlink,
+    .access = pl_access,
+    .readlink = pl_readlink,
+    .fallocate = pl_fallocate,
+    .fsetattr = pl_fsetattr,
+    .setattr = pl_setattr,
+    .fxattrop = pl_fxattrop,
+    .xattrop = pl_xattrop,
+    .rchecksum = pl_rchecksum,
+    .statfs = pl_statfs,
+    .fsyncdir = pl_fsyncdir,
+    .readdir = pl_readdir,
+    .symlink = pl_symlink,
+    .link = pl_link,
+    .rmdir = pl_rmdir,
+    .mknod = pl_mknod,
+    .stat = pl_stat,
+    .seek = pl_seek,
 };
 
 struct xlator_dumpops dumpops = {
-        .inodectx    = pl_dump_inode_priv,
+    .inodectx = pl_dump_inode_priv,
 };
 
 struct xlator_cbks cbks = {
-        .forget            = pl_forget,
-        .release           = pl_release,
-        .releasedir        = pl_releasedir,
-        .client_destroy    = pl_client_destroy_cbk,
-        .client_disconnect = pl_client_disconnect_cbk,
+    .forget = pl_forget,
+    .release = pl_release,
+    .releasedir = pl_releasedir,
+    .client_destroy = pl_client_destroy_cbk,
+    .client_disconnect = pl_client_disconnect_cbk,
 };
 
-
 struct volume_options options[] = {
-        { .key  = { "mandatory-locks", "mandatory" },
-          .type = GF_OPTION_TYPE_BOOL
-        },
-        { .key  = { "trace" },
-          .type = GF_OPTION_TYPE_BOOL
-        },
-        { .key = {NULL} },
+    {.key = {"mandatory-locking"},
+     .type = GF_OPTION_TYPE_STR,
+     .default_value = "off",
+     .op_version = {GD_OP_VERSION_3_8_0},
+     .flags = OPT_FLAG_SETTABLE | OPT_FLAG_DOC,
+     .tags = {"locks"},
+     .description = "Specifies the mandatory-locking mode. Valid options "
+                    "are 'file' to use linux style mandatory locks, "
+                    "'forced' to use volume strictly under mandatory lock "
+                    "semantics only and 'optimal' to treat advisory and "
+                    "mandatory locks separately on their own."},
+    {.key = {"trace"},
+     .type = GF_OPTION_TYPE_BOOL,
+     .default_value = "off",
+     .op_version = {GD_OP_VERSION_3_7_0},
+     .flags = OPT_FLAG_SETTABLE | OPT_FLAG_DOC,
+     .tags = {"locks"},
+     .description = "Trace the different lock requests "
+                    "to logs."},
+    {.key = {"monkey-unlocking"},
+     .type = GF_OPTION_TYPE_BOOL,
+     .default_value = "false",
+     .op_version = {GD_OP_VERSION_3_9_0},
+     .flags = OPT_FLAG_SETTABLE,
+     .tags = {"locks"},
+     .description = "Ignore a random number of unlock requests.  Useful "
+                    "for testing/creating robust lock recovery mechanisms."},
+    {
+        .key = {"revocation-secs"},
+        .type = GF_OPTION_TYPE_INT,
+        .min = 0,
+        .max = INT_MAX,
+        .default_value = "0",
+        .op_version = {GD_OP_VERSION_3_9_0},
+        .flags = OPT_FLAG_SETTABLE | OPT_FLAG_DOC,
+        .tags = {"locks"},
+        .description = "Maximum time a lock can be taken out, before"
+                       "being revoked.",
+    },
+    {
+        .key = {"revocation-clear-all"},
+        .type = GF_OPTION_TYPE_BOOL,
+        .default_value = "false",
+        .op_version = {GD_OP_VERSION_3_9_0},
+        .flags = OPT_FLAG_SETTABLE | OPT_FLAG_DOC,
+        .tags = {"locks"},
+        .description = "If set to true, will revoke BOTH granted and blocked "
+                       "(pending) lock requests if a revocation threshold is "
+                       "hit.",
+    },
+    {.key = {"revocation-max-blocked"},
+     .type = GF_OPTION_TYPE_INT,
+     .min = 0,
+     .max = INT_MAX,
+     .default_value = "0",
+     .op_version = {GD_OP_VERSION_3_9_0},
+     .flags = OPT_FLAG_SETTABLE | OPT_FLAG_DOC,
+     .tags = {"locks"},
+     .description = "A number of blocked lock requests after which a lock "
+                    "will be revoked to allow the others to proceed.  Can "
+                    "be used in conjunction w/ revocation-clear-all."},
+    {.key = {"notify-contention"},
+     .type = GF_OPTION_TYPE_BOOL,
+     .default_value = "yes",
+     .flags = OPT_FLAG_SETTABLE | OPT_FLAG_DOC,
+     .op_version = {GD_OP_VERSION_4_0_0},
+     .tags = {"locks", "contention"},
+     .description = "When this option is enabled and a lock request "
+                    "conflicts with a currently granted lock, an upcall "
+                    "notification will be sent to the current owner of "
+                    "the lock to request it to be released as soon as "
+                    "possible."},
+    {.key = {"notify-contention-delay"},
+     .type = GF_OPTION_TYPE_INT,
+     .min = 0, /* An upcall notification is sent every time a conflict is
+                * detected. */
+     .max = 60,
+     .default_value = "5",
+     .flags = OPT_FLAG_SETTABLE | OPT_FLAG_DOC,
+     .op_version = {GD_OP_VERSION_4_0_0},
+     .tags = {"locks", "contention", "timeout"},
+     .description = "This value determines the minimum amount of time "
+                    "(in seconds) between upcall contention notifications "
+                    "on the same inode. If multiple lock requests are "
+                    "received during this period, only one upcall will "
+                    "be sent."},
+    {.key = {"enforce-mandatory-lock"},
+     .type = GF_OPTION_TYPE_BOOL,
+     .default_value = "off",
+     .flags = OPT_FLAG_SETTABLE,
+     .op_version = {GD_OP_VERSION_6_0},
+     .description = "option to enable lock enforcement"},
+    {.key = {NULL}},
+};
+
+xlator_api_t xlator_api = {
+    .init = init,
+    .fini = fini,
+    .reconfigure = reconfigure,
+    .mem_acct_init = mem_acct_init,
+    .op_version = {1}, /* Present from the initial version */
+    .dumpops = &dumpops,
+    .fops = &fops,
+    .cbks = &cbks,
+    .options = options,
+    .identifier = "locks",
+    .category = GF_MAINTAINED,
 };
diff --git a/xlators/features/locks/src/reservelk.c b/xlators/features/locks/src/reservelk.c
index 11abd26d85f..604691fd887 100644
--- a/xlators/features/locks/src/reservelk.c
+++ b/xlators/features/locks/src/reservelk.c
@@ -7,437 +7,376 @@
    later), or the GNU General Public License, version 2 (GPLv2), in all
    cases as published by the Free Software Foundation.
 */
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
-
-#include "glusterfs.h"
-#include "compat.h"
-#include "xlator.h"
-#include "inode.h"
-#include "logging.h"
-#include "common-utils.h"
-#include "list.h"
+#include <glusterfs/glusterfs.h>
+#include <glusterfs/compat.h>
+#include <glusterfs/xlator.h>
+#include <glusterfs/logging.h>
+#include <glusterfs/common-utils.h>
+#include <glusterfs/list.h>
 
 #include "locks.h"
 #include "common.h"
 
-void
-__delete_reserve_lock (posix_lock_t *lock)
-{
-        list_del (&lock->list);
-}
-
-void
-__destroy_reserve_lock (posix_lock_t *lock)
-{
-        GF_FREE (lock);
-}
-
 /* Return true if the two reservelks have exactly same lock boundaries */
 int
-reservelks_equal (posix_lock_t *l1, posix_lock_t *l2)
+reservelks_equal(posix_lock_t *l1, posix_lock_t *l2)
 {
-        if ((l1->fl_start == l2->fl_start) &&
-            (l1->fl_end == l2->fl_end))
-                return 1;
+    if ((l1->fl_start == l2->fl_start) && (l1->fl_end == l2->fl_end))
+        return 1;
 
-        return 0;
+    return 0;
 }
 
 /* Determine if lock is grantable or not */
 static posix_lock_t *
-__reservelk_grantable (pl_inode_t *pl_inode, posix_lock_t *lock)
+__reservelk_grantable(pl_inode_t *pl_inode, posix_lock_t *lock)
 {
-        xlator_t     *this     = NULL;
-        posix_lock_t *l        = NULL;
-        posix_lock_t *ret_lock = NULL;
-
-        this = THIS;
-
-        if (list_empty (&pl_inode->reservelk_list)) {
-                gf_log (this->name, GF_LOG_TRACE,
-                        "No reservelks in list");
-                goto out;
-        }
-        list_for_each_entry (l, &pl_inode->reservelk_list, list){
-                if (reservelks_equal (lock, l)) {
-                        ret_lock = l;
-                        break;
-                }
+    xlator_t *this = THIS;
+    posix_lock_t *l = NULL;
+    posix_lock_t *ret_lock = NULL;
+
+    if (list_empty(&pl_inode->reservelk_list)) {
+        gf_log(this->name, GF_LOG_TRACE, "No reservelks in list");
+        goto out;
+    }
+    list_for_each_entry(l, &pl_inode->reservelk_list, list)
+    {
+        if (reservelks_equal(lock, l)) {
+            ret_lock = l;
+            break;
         }
+    }
 out:
-        return ret_lock;
+    return ret_lock;
 }
 
-static inline int
-__same_owner_reservelk (posix_lock_t *l1, posix_lock_t *l2)
+static int
+__same_owner_reservelk(posix_lock_t *l1, posix_lock_t *l2)
 {
-        return (is_same_lkowner (&l1->owner, &l2->owner));
-
+    return (is_same_lkowner(&l1->owner, &l2->owner));
 }
 
 static posix_lock_t *
-__matching_reservelk (pl_inode_t *pl_inode, posix_lock_t *lock)
+__matching_reservelk(pl_inode_t *pl_inode, posix_lock_t *lock)
 {
-        posix_lock_t *l = NULL;
+    posix_lock_t *l = NULL;
 
-        if (list_empty (&pl_inode->reservelk_list)) {
-            gf_log ("posix-locks", GF_LOG_TRACE,
-                    "reservelk list empty");
-            return NULL;
-        }
+    if (list_empty(&pl_inode->reservelk_list)) {
+        gf_log("posix-locks", GF_LOG_TRACE, "reservelk list empty");
+        return NULL;
+    }
 
-        list_for_each_entry (l, &pl_inode->reservelk_list, list) {
-                if (reservelks_equal (l, lock)) {
-                        gf_log ("posix-locks", GF_LOG_TRACE,
-                                "equal reservelk found");
-                        break;
-                }
+    list_for_each_entry(l, &pl_inode->reservelk_list, list)
+    {
+        if (reservelks_equal(l, lock)) {
+            gf_log("posix-locks", GF_LOG_TRACE, "equal reservelk found");
+            break;
         }
+    }
 
-        return l;
+    return l;
 }
 
 static int
-__reservelk_conflict (xlator_t *this, pl_inode_t *pl_inode,
-                      posix_lock_t *lock)
+__reservelk_conflict(xlator_t *this, pl_inode_t *pl_inode, posix_lock_t *lock)
 {
-        posix_lock_t *conf = NULL;
-        int ret = 0;
-
-        conf = __matching_reservelk (pl_inode, lock);
-        if (conf) {
-                gf_log (this->name, GF_LOG_TRACE,
-                        "Matching reservelk found");
-                if (__same_owner_reservelk (lock, conf)) {
-                        list_del_init (&conf->list);
-                        gf_log (this->name, GF_LOG_TRACE,
-                                "Removing the matching reservelk for setlk to progress");
-                        GF_FREE (conf);
-                        ret = 0;
-                } else {
-                        gf_log (this->name, GF_LOG_TRACE,
-                                "Conflicting reservelk found");
-                        ret = 1;
-                }
-
+    int ret = 0;
+
+    posix_lock_t *conf = __matching_reservelk(pl_inode, lock);
+    if (conf) {
+        gf_log(this->name, GF_LOG_TRACE, "Matching reservelk found");
+        if (__same_owner_reservelk(lock, conf)) {
+            list_del_init(&conf->list);
+            gf_log(this->name, GF_LOG_TRACE,
+                   "Removing the matching reservelk for setlk to progress");
+            __destroy_lock(conf);
+            ret = 0;
+        } else {
+            gf_log(this->name, GF_LOG_TRACE, "Conflicting reservelk found");
+            ret = 1;
         }
-        return ret;
-
+    }
+    return ret;
 }
 
 int
-pl_verify_reservelk (xlator_t *this, pl_inode_t *pl_inode,
-                     posix_lock_t *lock, int can_block)
+pl_verify_reservelk(xlator_t *this, pl_inode_t *pl_inode, posix_lock_t *lock,
+                    const int can_block)
 {
-        int ret = 0;
-
-        pthread_mutex_lock (&pl_inode->mutex);
-        {
-                if (__reservelk_conflict (this, pl_inode, lock)) {
-                        gf_log (this->name, GF_LOG_TRACE,
-                                "Found conflicting reservelk. Blocking until reservelk is unlocked.");
-                        lock->blocked = can_block;
-                        list_add_tail (&lock->list, &pl_inode->blocked_calls);
-                        ret = -1;
-                        goto unlock;
-                }
-
-                gf_log (this->name, GF_LOG_TRACE,
-                        "no conflicting reservelk found. Call continuing");
-                ret = 0;
-
+    int ret = 0;
+
+    pthread_mutex_lock(&pl_inode->mutex);
+    {
+        if (__reservelk_conflict(this, pl_inode, lock)) {
+            lock->blocked = can_block;
+            list_add_tail(&lock->list, &pl_inode->blocked_calls);
+            pthread_mutex_unlock(&pl_inode->mutex);
+            gf_log(this->name, GF_LOG_TRACE,
+                   "Found conflicting reservelk. Blocking until reservelk is "
+                   "unlocked.");
+            ret = -1;
+            goto out;
         }
-unlock:
-        pthread_mutex_unlock (&pl_inode->mutex);
-
-        return ret;
-
+    }
+    pthread_mutex_unlock(&pl_inode->mutex);
+    gf_log(this->name, GF_LOG_TRACE,
+           "no conflicting reservelk found. Call continuing");
+    ret = 0;
+out:
+    return ret;
 }
 
-
 /* Determines if lock can be granted and adds the lock. If the lock
  * is blocking, adds it to the blocked_reservelks.
  */
 static int
-__lock_reservelk (xlator_t *this, pl_inode_t *pl_inode, posix_lock_t *lock,
-                  int can_block)
+__lock_reservelk(xlator_t *this, pl_inode_t *pl_inode, posix_lock_t *lock,
+                 const int can_block)
 {
-        posix_lock_t *conf = NULL;
-        int ret = -EINVAL;
-
-        conf = __reservelk_grantable (pl_inode, lock);
-        if (conf){
-                ret = -EAGAIN;
-                if (can_block == 0)
-                        goto out;
+    int ret = -EINVAL;
 
-                list_add_tail (&lock->list, &pl_inode->blocked_reservelks);
+    posix_lock_t *conf = __reservelk_grantable(pl_inode, lock);
+    if (conf) {
+        ret = -EAGAIN;
+        if (can_block == 0)
+            goto out;
 
-                gf_log (this->name, GF_LOG_TRACE,
-                        "%s (pid=%d) lk-owner:%s %"PRId64" - %"PRId64" => Blocked",
-                        lock->fl_type == F_UNLCK ? "Unlock" : "Lock",
-                        lock->client_pid,
-                        lkowner_utoa (&lock->owner),
-                        lock->user_flock.l_start,
-                        lock->user_flock.l_len);
+        list_add_tail(&lock->list, &pl_inode->blocked_reservelks);
 
+        gf_log(this->name, GF_LOG_TRACE,
+               "%s (pid=%d) lk-owner:%s %" PRId64 " - %" PRId64 " => Blocked",
+               lock->fl_type == F_UNLCK ? "Unlock" : "Lock", lock->client_pid,
+               lkowner_utoa(&lock->owner), lock->user_flock.l_start,
+               lock->user_flock.l_len);
 
-                goto out;
-        }
+        goto out;
+    }
 
-        list_add (&lock->list, &pl_inode->reservelk_list);
+    list_add(&lock->list, &pl_inode->reservelk_list);
 
-        ret = 0;
+    ret = 0;
 
 out:
-        return ret;
+    return ret;
 }
 
 static posix_lock_t *
-find_matching_reservelk (posix_lock_t *lock, pl_inode_t *pl_inode)
+find_matching_reservelk(posix_lock_t *lock, pl_inode_t *pl_inode)
 {
-        posix_lock_t *l = NULL;
-        list_for_each_entry (l, &pl_inode->reservelk_list, list) {
-                if (reservelks_equal (l, lock))
-                        return l;
-        }
-        return NULL;
+    posix_lock_t *l = NULL;
+    list_for_each_entry(l, &pl_inode->reservelk_list, list)
+    {
+        if (reservelks_equal(l, lock))
+            return l;
+    }
+    return NULL;
 }
 
 /* Set F_UNLCK removes a lock which has the exact same lock boundaries
  * as the UNLCK lock specifies. If such a lock is not found, returns invalid
  */
 static posix_lock_t *
-__reserve_unlock_lock (xlator_t *this, posix_lock_t *lock, pl_inode_t *pl_inode)
+__reserve_unlock_lock(xlator_t *this, posix_lock_t *lock, pl_inode_t *pl_inode)
 {
-
-        posix_lock_t *conf = NULL;
-
-        conf = find_matching_reservelk (lock, pl_inode);
-        if (!conf) {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        " Matching lock not found for unlock");
-                goto out;
-        }
-        __delete_reserve_lock (conf);
-        gf_log (this->name, GF_LOG_DEBUG,
-                " Matching lock found for unlock");
+    posix_lock_t *conf = find_matching_reservelk(lock, pl_inode);
+    if (!conf) {
+        gf_log(this->name, GF_LOG_DEBUG, " Matching lock not found for unlock");
+        goto out;
+    }
+    __delete_lock(conf);
+    gf_log(this->name, GF_LOG_DEBUG, " Matching lock found for unlock");
 
 out:
-        return conf;
-
-
+    return conf;
 }
 
 static void
-__grant_blocked_reserve_locks (xlator_t *this, pl_inode_t *pl_inode,
-                               struct list_head *granted)
+__grant_blocked_reserve_locks(xlator_t *this, pl_inode_t *pl_inode,
+                              struct list_head *granted)
 {
-        int              bl_ret = 0;
-        posix_lock_t *bl = NULL;
-        posix_lock_t *tmp = NULL;
-
-        struct list_head blocked_list;
+    int bl_ret = 0;
+    posix_lock_t *bl = NULL;
+    posix_lock_t *tmp = NULL;
 
-        INIT_LIST_HEAD (&blocked_list);
-        list_splice_init (&pl_inode->blocked_reservelks, &blocked_list);
+    struct list_head blocked_list;
 
-        list_for_each_entry_safe (bl, tmp, &blocked_list, list) {
+    INIT_LIST_HEAD(&blocked_list);
+    list_splice_init(&pl_inode->blocked_reservelks, &blocked_list);
 
-                list_del_init (&bl->list);
+    list_for_each_entry_safe(bl, tmp, &blocked_list, list)
+    {
+        list_del_init(&bl->list);
 
-                bl_ret = __lock_reservelk (this, pl_inode, bl, 1);
+        bl_ret = __lock_reservelk(this, pl_inode, bl, 1);
 
-                if (bl_ret == 0) {
-                        list_add (&bl->list, granted);
-                }
+        if (bl_ret == 0) {
+            list_add(&bl->list, granted);
         }
-        return;
+    }
+    return;
 }
 
 /* Grant all reservelks blocked on lock(s) */
 void
-grant_blocked_reserve_locks (xlator_t *this, pl_inode_t *pl_inode)
+grant_blocked_reserve_locks(xlator_t *this, pl_inode_t *pl_inode)
 {
-        struct list_head granted;
-        posix_lock_t *lock = NULL;
-        posix_lock_t *tmp = NULL;
+    struct list_head granted;
+    posix_lock_t *lock = NULL;
+    posix_lock_t *tmp = NULL;
 
-        INIT_LIST_HEAD (&granted);
-
-        if (list_empty (&pl_inode->blocked_reservelks)) {
-                gf_log (this->name, GF_LOG_TRACE,
-                        "No blocked locks to be granted");
-                return;
-        }
-
-        pthread_mutex_lock (&pl_inode->mutex);
-        {
-                __grant_blocked_reserve_locks (this, pl_inode, &granted);
-        }
-        pthread_mutex_unlock (&pl_inode->mutex);
-
-        list_for_each_entry_safe (lock, tmp, &granted, list) {
-                gf_log (this->name, GF_LOG_TRACE,
-                        "%s (pid=%d) (lk-owner=%s) %"PRId64" - %"PRId64" => Granted",
-                        lock->fl_type == F_UNLCK ? "Unlock" : "Lock",
-                        lock->client_pid,
-                        lkowner_utoa (&lock->owner),
-                        lock->user_flock.l_start,
-                        lock->user_flock.l_len);
-
-                STACK_UNWIND_STRICT (lk, lock->frame, 0, 0, &lock->user_flock,
-                                     NULL);
-        }
+    INIT_LIST_HEAD(&granted);
 
+    if (list_empty(&pl_inode->blocked_reservelks)) {
+        gf_log(this->name, GF_LOG_TRACE, "No blocked locks to be granted");
+        return;
+    }
+
+    pthread_mutex_lock(&pl_inode->mutex);
+    {
+        __grant_blocked_reserve_locks(this, pl_inode, &granted);
+    }
+    pthread_mutex_unlock(&pl_inode->mutex);
+
+    list_for_each_entry_safe(lock, tmp, &granted, list)
+    {
+        gf_log(this->name, GF_LOG_TRACE,
+               "%s (pid=%d) (lk-owner=%s) %" PRId64 " - %" PRId64 " => Granted",
+               lock->fl_type == F_UNLCK ? "Unlock" : "Lock", lock->client_pid,
+               lkowner_utoa(&lock->owner), lock->user_flock.l_start,
+               lock->user_flock.l_len);
+
+        STACK_UNWIND_STRICT(lk, lock->frame, 0, 0, &lock->user_flock, NULL);
+    }
 }
 
 static void
-__grant_blocked_lock_calls (xlator_t *this, pl_inode_t *pl_inode,
-                            struct list_head *granted)
+__grant_blocked_lock_calls(xlator_t *this, pl_inode_t *pl_inode,
+                           struct list_head *granted)
 {
-        int              bl_ret = 0;
-        posix_lock_t *bl = NULL;
-        posix_lock_t *tmp = NULL;
+    int bl_ret = 0;
+    posix_lock_t *bl = NULL;
+    posix_lock_t *tmp = NULL;
 
-        struct list_head blocked_list;
+    struct list_head blocked_list;
 
-        INIT_LIST_HEAD (&blocked_list);
-        list_splice_init (&pl_inode->blocked_reservelks, &blocked_list);
+    INIT_LIST_HEAD(&blocked_list);
+    list_splice_init(&pl_inode->blocked_reservelks, &blocked_list);
 
-        list_for_each_entry_safe (bl, tmp, &blocked_list, list) {
+    list_for_each_entry_safe(bl, tmp, &blocked_list, list)
+    {
+        list_del_init(&bl->list);
 
-                list_del_init (&bl->list);
+        bl_ret = pl_verify_reservelk(this, pl_inode, bl, bl->blocked);
 
-                bl_ret = pl_verify_reservelk (this, pl_inode, bl, bl->blocked);
-
-                if (bl_ret == 0) {
-                        list_add_tail (&bl->list, granted);
-                }
+        if (bl_ret == 0) {
+            list_add_tail(&bl->list, granted);
         }
-        return;
+    }
+    return;
 }
 
 void
-grant_blocked_lock_calls (xlator_t *this, pl_inode_t *pl_inode)
+grant_blocked_lock_calls(xlator_t *this, pl_inode_t *pl_inode)
 {
-        struct list_head granted;
-        posix_lock_t *lock = NULL;
-        posix_lock_t *tmp = NULL;
-        fd_t         *fd  = NULL;
-
-        int can_block = 0;
-        int32_t cmd = 0;
-        int ret = 0;
-
-        if (list_empty (&pl_inode->blocked_calls)) {
-                gf_log (this->name, GF_LOG_TRACE,
-                        "No blocked lock calls to be granted");
-                return;
-        }
+    struct list_head granted;
+    posix_lock_t *lock = NULL;
+    posix_lock_t *tmp = NULL;
+    fd_t *fd = NULL;
 
-        pthread_mutex_lock (&pl_inode->mutex);
-        {
-                __grant_blocked_lock_calls (this, pl_inode, &granted);
-        }
-        pthread_mutex_unlock (&pl_inode->mutex);
-
-        list_for_each_entry_safe (lock, tmp, &granted, list) {
-                fd = fd_from_fdnum (lock);
-
-                if (lock->blocked) {
-                        can_block = 1;
-                        cmd = F_SETLKW;
-                }
-                else
-                        cmd = F_SETLK;
-
-                lock->blocked = 0;
-                ret = pl_setlk (this, pl_inode, lock, can_block);
-                if (ret == -1) {
-                        if (can_block) {
-                                pl_trace_block (this, lock->frame, fd, NULL,
-                                                cmd, &lock->user_flock, NULL);
-                                continue;
-                        } else {
-                                gf_log (this->name, GF_LOG_DEBUG, "returning EAGAIN");
-                                pl_trace_out (this, lock->frame, fd, NULL, cmd,
-                                              &lock->user_flock, -1, EAGAIN, NULL);
-                                pl_update_refkeeper (this, fd->inode);
-                                STACK_UNWIND_STRICT (lk, lock->frame, -1,
-                                                     EAGAIN, &lock->user_flock,
-                                                     NULL);
-                                __destroy_lock (lock);
-                        }
-                }
+    int can_block = 0;
+    int32_t cmd = 0;
+    int ret = 0;
 
+    if (list_empty(&pl_inode->blocked_calls)) {
+        gf_log(this->name, GF_LOG_TRACE, "No blocked lock calls to be granted");
+        return;
+    }
+
+    pthread_mutex_lock(&pl_inode->mutex);
+    {
+        __grant_blocked_lock_calls(this, pl_inode, &granted);
+    }
+    pthread_mutex_unlock(&pl_inode->mutex);
+
+    list_for_each_entry_safe(lock, tmp, &granted, list)
+    {
+        fd = fd_from_fdnum(lock);
+
+        if (lock->blocked) {
+            can_block = 1;
+            cmd = F_SETLKW;
+        } else
+            cmd = F_SETLK;
+
+        lock->blocked = 0;
+        ret = pl_setlk(this, pl_inode, lock, can_block);
+        if (ret == -1) {
+            if (can_block) {
+                continue;
+            } else {
+                gf_log(this->name, GF_LOG_DEBUG, "returning EAGAIN");
+                pl_trace_out(this, lock->frame, fd, NULL, cmd,
+                             &lock->user_flock, -1, EAGAIN, NULL);
+                pl_update_refkeeper(this, fd->inode);
+                STACK_UNWIND_STRICT(lk, lock->frame, -1, EAGAIN,
+                                    &lock->user_flock, NULL);
+                __destroy_lock(lock);
+            }
         }
-
+    }
 }
 
-
 int
-pl_reserve_unlock (xlator_t *this, pl_inode_t *pl_inode, posix_lock_t *lock)
+pl_reserve_unlock(xlator_t *this, pl_inode_t *pl_inode, posix_lock_t *lock)
 {
-        posix_lock_t *retlock = NULL;
-        int ret = -1;
-
-        pthread_mutex_lock (&pl_inode->mutex);
-        {
-                retlock = __reserve_unlock_lock (this, lock, pl_inode);
-                if (!retlock) {
-                        gf_log (this->name, GF_LOG_DEBUG,
-                                "Bad Unlock issued on Inode lock");
-                        ret = -EINVAL;
-                        goto out;
-                }
-
-                gf_log (this->name, GF_LOG_TRACE,
-                        "Reservelk Unlock successful");
-                __destroy_reserve_lock (retlock);
-                ret = 0;
+    posix_lock_t *retlock = NULL;
+    int ret = -1;
+
+    pthread_mutex_lock(&pl_inode->mutex);
+    {
+        retlock = __reserve_unlock_lock(this, lock, pl_inode);
+        if (!retlock) {
+            pthread_mutex_unlock(&pl_inode->mutex);
+            gf_log(this->name, GF_LOG_DEBUG, "Bad Unlock issued on Inode lock");
+            ret = -EINVAL;
+            goto out;
         }
-out:
-        pthread_mutex_unlock (&pl_inode->mutex);
-
-        grant_blocked_reserve_locks (this, pl_inode);
-        grant_blocked_lock_calls (this, pl_inode);
 
-        return ret;
+        gf_log(this->name, GF_LOG_TRACE, "Reservelk Unlock successful");
+        __destroy_lock(retlock);
+        ret = 0;
+    }
+    pthread_mutex_unlock(&pl_inode->mutex);
+out:
+    grant_blocked_reserve_locks(this, pl_inode);
+    grant_blocked_lock_calls(this, pl_inode);
 
+    return ret;
 }
 
 int
-pl_reserve_setlk (xlator_t *this, pl_inode_t *pl_inode, posix_lock_t *lock,
-                  int can_block)
+pl_reserve_setlk(xlator_t *this, pl_inode_t *pl_inode, posix_lock_t *lock,
+                 int can_block)
 {
-        int ret = -EINVAL;
-
-        pthread_mutex_lock (&pl_inode->mutex);
-        {
-
-                        ret = __lock_reservelk (this, pl_inode, lock, can_block);
-                        if (ret < 0)
-                                gf_log (this->name, GF_LOG_TRACE,
-                                        "%s (pid=%d) (lk-owner=%s) %"PRId64" - %"PRId64" => NOK",
-                                        lock->fl_type == F_UNLCK ? "Unlock" : "Lock",
-                                        lock->client_pid,
-                                        lkowner_utoa (&lock->owner),
-                                        lock->user_flock.l_start,
-                                        lock->user_flock.l_len);
-                        else
-                                gf_log (this->name, GF_LOG_TRACE,
-                                        "%s (pid=%d) (lk-owner=%s) %"PRId64" - %"PRId64" => OK",
-                                        lock->fl_type == F_UNLCK ? "Unlock" : "Lock",
-                                        lock->client_pid,
-                                        lkowner_utoa (&lock->owner),
-                                        lock->fl_start,
-                                        lock->fl_end);
-
-        }
-        pthread_mutex_unlock (&pl_inode->mutex);
-        return ret;
+    int ret = -EINVAL;
+
+    pthread_mutex_lock(&pl_inode->mutex);
+    {
+        ret = __lock_reservelk(this, pl_inode, lock, can_block);
+    }
+    pthread_mutex_unlock(&pl_inode->mutex);
+
+    if (ret < 0)
+        gf_log(this->name, GF_LOG_TRACE,
+               "%s (pid=%d) (lk-owner=%s) %" PRId64 " - %" PRId64 " => NOK",
+               lock->fl_type == F_UNLCK ? "Unlock" : "Lock", lock->client_pid,
+               lkowner_utoa(&lock->owner), lock->user_flock.l_start,
+               lock->user_flock.l_len);
+    else
+        gf_log(this->name, GF_LOG_TRACE,
+               "%s (pid=%d) (lk-owner=%s) %" PRId64 " - %" PRId64 " => OK",
+               lock->fl_type == F_UNLCK ? "Unlock" : "Lock", lock->client_pid,
+               lkowner_utoa(&lock->owner), lock->fl_start, lock->fl_end);
+
+    return ret;
 }
diff --git a/xlators/features/locks/tests/unit-test.c b/xlators/features/locks/tests/unit-test.c
index d2cca32dec3..d285b12b5aa 100644
--- a/xlators/features/locks/tests/unit-test.c
+++ b/xlators/features/locks/tests/unit-test.c
@@ -7,59 +7,71 @@
    later), or the GNU General Public License, version 2 (GPLv2), in all
    cases as published by the Free Software Foundation.
 */
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
-
-#include "glusterfs.h"
-#include "compat.h"
-#include "xlator.h"
-#include "inode.h"
-#include "logging.h"
-#include "common-utils.h"
-#include "list.h"
+#include <glusterfs/glusterfs.h>
+#include <glusterfs/compat.h>
+#include <glusterfs/xlator.h>
+#include <glusterfs/logging.h>
+#include <glusterfs/common-utils.h>
+#include <glusterfs/list.h>
 
 #include "locks.h"
 #include "common.h"
 
-#define expect(cond) if (!(cond)) { goto out; }
+#define expect(cond)                                                           \
+    if (!(cond)) {                                                             \
+        goto out;                                                              \
+    }
 
-extern int lock_name (pl_inode_t *, const char *, entrylk_type);
-extern int unlock_name (pl_inode_t *, const char *, entrylk_type);
+extern int
+lock_name(pl_inode_t *, const char *, entrylk_type);
+extern int
+unlock_name(pl_inode_t *, const char *, entrylk_type);
 
-int main (int argc, char **argv)
+int
+main(int argc, char **argv)
 {
-	int ret = 1;
-	int r = -1;
+    int ret = 1;
+    int r = -1;
+
+    pl_inode_t *pinode = CALLOC(sizeof(pl_inode_t), 1);
+    pthread_mutex_init(&pinode->dir_lock_mutex, NULL);
+    INIT_LIST_HEAD(&pinode->gf_dir_locks);
 
-	pl_inode_t *pinode = CALLOC (sizeof (pl_inode_t), 1);
-	pthread_mutex_init (&pinode->dir_lock_mutex, NULL);
-	INIT_LIST_HEAD (&pinode->gf_dir_locks);
+    r = lock_name(pinode, NULL, ENTRYLK_WRLCK);
+    expect(r == 0);
+    {
+        r = lock_name(pinode, "foo", ENTRYLK_WRLCK);
+        expect(r == -EAGAIN);
+    }
+    r = unlock_name(pinode, NULL, ENTRYLK_WRLCK);
+    expect(r == 0);
 
-	r = lock_name (pinode, NULL, ENTRYLK_WRLCK); expect (r == 0);
-	{
-		r = lock_name (pinode, "foo", ENTRYLK_WRLCK); expect (r == -EAGAIN);
-	}
-	r = unlock_name (pinode, NULL, ENTRYLK_WRLCK); expect (r == 0);
+    r = lock_name(pinode, "foo", ENTRYLK_RDLCK);
+    expect(r == 0);
+    {
+        r = lock_name(pinode, "foo", ENTRYLK_RDLCK);
+        expect(r == 0);
+        {
+            r = lock_name(pinode, "foo", ENTRYLK_WRLCK);
+            expect(r == -EAGAIN);
+        }
+        r = unlock_name(pinode, "foo", ENTRYLK_RDLCK);
+        expect(r == 0);
+    }
+    r = unlock_name(pinode, "foo", ENTRYLK_RDLCK);
+    expect(r == 0);
 
-	r = lock_name (pinode, "foo", ENTRYLK_RDLCK); expect (r == 0);
-	{
-		r = lock_name (pinode, "foo", ENTRYLK_RDLCK); expect (r == 0);
-		{
-			r = lock_name (pinode, "foo", ENTRYLK_WRLCK); expect (r == -EAGAIN);
-		}
-		r = unlock_name (pinode, "foo", ENTRYLK_RDLCK); expect (r == 0);
-	}
-	r = unlock_name (pinode, "foo", ENTRYLK_RDLCK); expect (r == 0);
-	
-	r = lock_name (pinode, "foo", ENTRYLK_WRLCK); expect (r == 0);
-	r = unlock_name (pinode, "foo", ENTRYLK_WRLCK); expect (r == 0);
+    r = lock_name(pinode, "foo", ENTRYLK_WRLCK);
+    expect(r == 0);
+    r = unlock_name(pinode, "foo", ENTRYLK_WRLCK);
+    expect(r == 0);
 
-	r = lock_name (pinode, "baz", ENTRYLK_WRLCK); expect (r == 0);
-	r = lock_name (pinode, "baz", ENTRYLK_RDLCK); expect (r == -EAGAIN);
+    r = lock_name(pinode, "baz", ENTRYLK_WRLCK);
+    expect(r == 0);
+    r = lock_name(pinode, "baz", ENTRYLK_RDLCK);
+    expect(r == -EAGAIN);
 
-	ret = 0;
+    ret = 0;
 out:
-	return ret;
+    return ret;
 }
diff --git a/xlators/features/mac-compat/Makefile.am b/xlators/features/mac-compat/Makefile.am
deleted file mode 100644
index d471a3f9243..00000000000
--- a/xlators/features/mac-compat/Makefile.am
+++ /dev/null
@@ -1,3 +0,0 @@
-SUBDIRS = src
-
-CLEANFILES = 
diff --git a/xlators/features/mac-compat/src/Makefile.am b/xlators/features/mac-compat/src/Makefile.am
deleted file mode 100644
index 42ed350e93e..00000000000
--- a/xlators/features/mac-compat/src/Makefile.am
+++ /dev/null
@@ -1,15 +0,0 @@
-xlator_LTLIBRARIES = mac-compat.la
-xlatordir = $(libdir)/glusterfs/$(PACKAGE_VERSION)/xlator/features
-
-mac_compat_la_LDFLAGS = -module -avoid-version
-
-mac_compat_la_SOURCES = mac-compat.c
-mac_compat_la_LIBADD = $(top_builddir)/libglusterfs/src/libglusterfs.la
-
-noinst_HEADERS = mac-compat.h
-
-AM_CPPFLAGS = $(GF_CPPFLAGS) -I$(top_srcdir)/libglusterfs/src
-
-AM_CFLAGS = -Wall $(GF_CFLAGS)
-
-CLEANFILES =
diff --git a/xlators/features/mac-compat/src/mac-compat.c b/xlators/features/mac-compat/src/mac-compat.c
deleted file mode 100644
index 0eaf563e889..00000000000
--- a/xlators/features/mac-compat/src/mac-compat.c
+++ /dev/null
@@ -1,349 +0,0 @@
-/*
-   Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
-   This file is part of GlusterFS.
-
-   This file is licensed to you under your choice of the GNU Lesser
-   General Public License, version 3 or any later version (LGPLv3 or
-   later), or the GNU General Public License, version 2 (GPLv2), in all
-   cases as published by the Free Software Foundation.
-*/
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
-
-#include "xlator.h"
-#include "defaults.h"
-#include "compat-errno.h"
-#include "syscall.h"
-#include "mem-pool.h"
-#include "mac-compat.h"
-
-static int
-dict_key_remove_namespace(dict_t *dict, char *key, data_t *value, void *data)
-{
-  /*
-    char buffer[3*value->len+1];
-    int index = 0;
-    for (index = 0; index < value->len; index++)
-    sprintf(buffer+3*index, " %02x", value->data[index]);
-  */
-        xlator_t *this = (xlator_t *) data;
-        if (strncmp(key, "user.", 5) == 0) {
-                dict_set (dict, key + 5, value);
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "remove_namespace_dict: %s -> %s ", key, key + 5);
-                dict_del (dict, key);
-        }
-        return 0;
-}
-
-int32_t
-maccomp_getxattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                      int32_t op_ret, int32_t op_errno, dict_t *dict,
-                      dict_t *xdata)
-{
-        intptr_t ax = (intptr_t)this->private;
-        int i = 0;
-
-        gf_log (this->name, GF_LOG_DEBUG,
-                "getxattr_cbk: dict %p private: %p xdata %p ", dict,
-                this->private, xdata);
-
-        if (dict) {
-                dict_foreach(dict, dict_key_remove_namespace, this);
-        }
-        else {
-                // TODO: we expect dict to exist here, don't know why this
-                // this is needed
-                dict = dict_new();
-        }
-        gf_log (this->name, GF_LOG_DEBUG,
-                "getxattr_cbk: dict %p ax: %ld op_ret %d op_err %d  ", dict, ax,
-                op_ret, op_errno);
-        if ((ax == GF_XATTR_ALL && op_ret >= 0) || ax != GF_XATTR_NONE) {
-                op_ret = op_errno = 0;
-                for (i = 0; i < GF_XATTR_ALL; i++) {
-                        if (dict_get (dict, apple_xattr_name[i]))
-                                continue;
-                        /* set dummy data */
-                        gf_log (this->name, GF_LOG_DEBUG,
-                                "getxattr_cbk: setting dummy data %p, %s", dict,
-                                apple_xattr_name[i]);
-                        if (dict_set (dict, apple_xattr_name[i],
-                                      bin_to_data ((void *)apple_xattr_value[i],
-                                                   apple_xattr_len[i])) == -1) {
-                                op_ret = -1;
-                                op_errno = ENOATTR;
-
-                                break;
-                        }
-                }
-         }
-        STACK_UNWIND_STRICT (getxattr, frame, op_ret, op_errno, dict, xdata);
-        return 0;
-}
-
-
-static
-int prepend_xattr_user_namespace(dict_t *dict, char *key, data_t *value, void *obj)
-{
-        xlator_t *this = (xlator_t *) obj;
-        dict_t *newdict = (dict_t *) this->private;
-        char *newkey = NULL;
-        gf_add_prefix(XATTR_USER_PREFIX, key, &newkey);
-        key = newkey;
-        dict_set(newdict, (char *)key, value);
-        if (newkey)
-                GF_FREE(newkey);
-        return 0;
-}
-
-intptr_t
-check_name(const char *name, char **newkey)
-{
-        intptr_t ax = GF_XATTR_NONE;
-        if (name) {
-                int i = 0;
-                for (i = 0; i < GF_XATTR_ALL; i++) {
-                        if (strcmp (apple_xattr_name[i], name) == 0) {
-                                ax = i;
-                                break;
-                        }
-                }
-                gf_add_prefix("user.", name, newkey);
-        } else
-                ax = GF_XATTR_ALL;
-        return ax;
-}
-
-int32_t
-maccomp_getxattr (call_frame_t *frame, xlator_t *this, loc_t *loc,
-                  const char *name, dict_t *xdata)
-{
-        char *newkey = NULL;
-        this->private = (void *) check_name(name, &newkey);
-
-        gf_log (this->name, GF_LOG_DEBUG,
-                "getxattr: name %s private: %p xdata %p ", name,
-                this->private, xdata);
-        STACK_WIND (frame, maccomp_getxattr_cbk,
-                    FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->getxattr,
-                    loc, newkey, xdata);
-        return 0;
-}
-
-
-int32_t
-maccomp_fgetxattr (call_frame_t *frame, xlator_t *this, fd_t *fd,
-                   const char *name, dict_t *xdata)
-{
-        char *newkey = NULL;
-        this->private = (void *) check_name(name, &newkey);
-
-        STACK_WIND (frame, maccomp_getxattr_cbk,
-                    FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->fgetxattr,
-                    fd, newkey, xdata);
-        GF_FREE(newkey);
-        return 0;
-}
-
-int32_t
-maccomp_setxattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                      int32_t op_ret, int32_t op_errno, dict_t *xdata)
-{
-        intptr_t ax = (intptr_t)this->private;
-
-        if (op_ret == -1 && ax != GF_XATTR_NONE)
-                op_ret = op_errno = 0;
-        gf_log (this->name, GF_LOG_DEBUG,
-                "setxattr_cbk op_ret %d op_errno %d private: %p xdata %p ",
-                op_ret, op_errno, this->private, xdata);
-        STACK_UNWIND_STRICT (setxattr, frame, op_ret, op_errno, xdata);
-        return 0;
-}
-
-int32_t
-maccomp_setattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                     int32_t op_ret, int32_t op_errno, struct iatt *iatt1,
-                     struct iatt *iattr2, dict_t *xdata)
-{
-        gf_log (this->name, GF_LOG_DEBUG,
-                "setattr_cbk op_ret %d op_errno %d private: %p xdata %p ",
-                op_ret, op_errno, this->private, xdata);
-        STACK_UNWIND_STRICT (setattr, frame, op_ret, op_errno,
-                             iatt1, iattr2, xdata);
-        return 0;
-}
-
-int map_flags(int flags)
-{
-        /* DARWIN has different defines on XATTR_ flags.
-           There do not seem to be a POSIX standard
-           Parse any other flags over.
-           NOFOLLOW is always true on Linux and Darwin
-        */
-        int linux_flags = flags & ~(GF_XATTR_CREATE | GF_XATTR_REPLACE | XATTR_REPLACE);
-        if (XATTR_CREATE & flags)
-                linux_flags |= GF_XATTR_CREATE;
-        if (XATTR_REPLACE & flags)
-                linux_flags |= GF_XATTR_REPLACE;
-        return linux_flags;
-}
-
-int32_t
-maccomp_fremovexattr (call_frame_t *frame, xlator_t *this, fd_t *fd,
-                   const char *name, dict_t *xdata)
-{
-        char *newkey = NULL;
-
-        this->private = (void *) check_name(name, &newkey);
-
-        STACK_WIND (frame, default_fremovexattr_cbk,
-                    FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->fremovexattr,
-                    fd, newkey, xdata);
-        GF_FREE(newkey);
-        return 0;
-}
-
-int32_t
-maccomp_setxattr (call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *dict,
-                  int32_t flags, dict_t *xdata)
-{
-        intptr_t ax = GF_XATTR_NONE;
-        int i = 0;
-
-        for (i = 0; i < GF_XATTR_ALL; i++) {
-                if (dict_get (dict, apple_xattr_name[i])) {
-                        ax = i;
-
-                        break;
-                }
-        }
-        dict_t *newdict = dict_new();
-        this->private = (void *) newdict;
-        dict_foreach(dict, prepend_xattr_user_namespace, this);
-
-        this->private = (void *)ax;
-        int linux_flags = map_flags(flags);
-        gf_log (this->name, GF_LOG_DEBUG,
-                "setxattr flags: %d -> %d dict %p private: %p xdata %p ",
-                flags, linux_flags, dict, this->private, xdata);
-        STACK_WIND (frame, maccomp_setxattr_cbk,
-                    FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->setxattr,
-                    loc, newdict, linux_flags, xdata);
-        dict_unref(newdict);
-        return 0;
-}
-
-int32_t
-maccomp_setattr (call_frame_t *frame, xlator_t *this, loc_t *loc, struct iatt *iattr,
-                  int32_t flags, dict_t *xdata)
-{
-        gf_log (this->name, GF_LOG_DEBUG,
-                "setattr iattr %p private: %p xdata %p ",
-                iattr, this->private, xdata);
-        STACK_WIND (frame, maccomp_setattr_cbk,
-                    FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->setattr,
-                    loc, iattr, flags, xdata);
-        return 0;
-}
-
-int32_t
-maccomp_removexattr (call_frame_t *frame, xlator_t *this, loc_t *loc,
-                     const char *name, dict_t *xdata)
-{
-        char *newkey = NULL;
-        this->private = (void *) check_name(name, &newkey);
-
-        STACK_WIND (frame, default_removexattr_cbk,
-                    FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->removexattr,
-                    loc, newkey, xdata);
-
-        gf_log (this->name, GF_LOG_TRACE,
-                "removeattr name %p private: %p xdata %p ",
-                name, this->private, xdata);
-        GF_FREE(newkey);
-        return 0;
-
-}
-
-int32_t
-maccomp_fsetxattr (call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *dict,
-                   int32_t flags, dict_t *xdata)
-{
-        intptr_t ax = GF_XATTR_NONE;
-        int i = 0;
-
-        for (i = 0; i < GF_XATTR_ALL; i++) {
-                if (dict_get (dict, apple_xattr_name[i])) {
-                        ax = i;
-
-                        break;
-                }
-        }
-
-        dict_t *newdict = dict_new();
-        this->private = (void *) newdict;
-        dict_foreach(dict, prepend_xattr_user_namespace, this);
-
-        this->private = (void *)ax;
-        int linux_flags = map_flags(flags);
-        gf_log (this->name, GF_LOG_DEBUG,
-                "fsetxattr flags: %d -> %d dict %p private: %p xdata %p ",
-                flags, linux_flags, dict, this->private, xdata);
-        STACK_WIND (frame, maccomp_setxattr_cbk,
-                    FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->fsetxattr,
-                    fd, newdict, linux_flags, xdata);
-        dict_unref(newdict);
-        return 0;
-}
-
-
-int32_t
-init (xlator_t *this)
-{
-        if (!this->children || this->children->next) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "translator not configured with exactly one child");
-                return -1;
-        }
-
-        if (!this->parents) {
-                gf_log (this->name, GF_LOG_WARNING,
-                        "dangling volume. check volfile ");
-        }
-
-        return 0;
-}
-
-
-void
-fini (xlator_t *this)
-{
-        return;
-}
-
-
-struct xlator_fops fops = {
-        .getxattr       = maccomp_getxattr,
-        .fgetxattr      = maccomp_fgetxattr,
-        .setxattr       = maccomp_setxattr,
-        .setattr        = maccomp_setattr,
-        .fsetxattr      = maccomp_fsetxattr,
-        .removexattr    = maccomp_removexattr,
-        .fremovexattr   = maccomp_fremovexattr,
-};
-
-struct xlator_cbks cbks;
-
-struct volume_options options[] = {
-        { .key = {NULL} },
-};
diff --git a/xlators/features/mac-compat/src/mac-compat.h b/xlators/features/mac-compat/src/mac-compat.h
deleted file mode 100644
index b033ca0e4d8..00000000000
--- a/xlators/features/mac-compat/src/mac-compat.h
+++ /dev/null
@@ -1,41 +0,0 @@
-/*
-   Copyright (c) 2014 Red Hat, Inc. <http://www.redhat.com>
-   This file is part of GlusterFS.
-
-   This file is licensed to you under your choice of the GNU Lesser
-   General Public License, version 3 or any later version (LGPLv3 or
-   later), or the GNU General Public License, version 2 (GPLv2), in all
-   cases as published by the Free Software Foundation.
-*/
-
-#ifndef __MAC_COMPAT_H__
-#define __MAC_COMPAT_H__
-
-enum apple_xattr {
-        GF_FINDER_INFO_XATTR,
-        GF_RESOURCE_FORK_XATTR,
-        GF_XATTR_ALL,
-        GF_XATTR_NONE
-};
-
-static char *apple_xattr_name[] = {
-        [GF_FINDER_INFO_XATTR]   = "com.apple.FinderInfo",
-        [GF_RESOURCE_FORK_XATTR] = "com.apple.ResourceFork"
-};
-
-static const char *apple_xattr_value[] = {
-        [GF_FINDER_INFO_XATTR]   =
-        /* 1 2 3 4 5 6 7 8 */
-         "\0\0\0\0\0\0\0\0"
-         "\0\0\0\0\0\0\0\0"
-         "\0\0\0\0\0\0\0\0"
-         "\0\0\0\0\0\0\0\0",
-        [GF_RESOURCE_FORK_XATTR] = ""
-};
-
-static int32_t apple_xattr_len[] = {
-        [GF_FINDER_INFO_XATTR]   = 32,
-        [GF_RESOURCE_FORK_XATTR] = 1
-};
-
-#endif /* __MAC_COMPAT_H__ */
diff --git a/xlators/features/marker/src/Makefile.am b/xlators/features/marker/src/Makefile.am
index a7c67647218..58056b36511 100644
--- a/xlators/features/marker/src/Makefile.am
+++ b/xlators/features/marker/src/Makefile.am
@@ -1,14 +1,21 @@
+if WITH_SERVER
 xlator_LTLIBRARIES = marker.la
+endif
 xlatordir = $(libdir)/glusterfs/$(PACKAGE_VERSION)/xlator/features
 
-marker_la_LDFLAGS = -module -avoid-version
+marker_la_LDFLAGS = -module $(GF_XLATOR_DEFAULT_LDFLAGS)
+
+marker_la_SOURCES = marker.c marker-quota.c marker-quota-helper.c \
+	marker-common.c
 
-marker_la_SOURCES = marker.c marker-quota.c marker-quota-helper.c marker-common.c
 marker_la_LIBADD = $(top_builddir)/libglusterfs/src/libglusterfs.la
 
-noinst_HEADERS = marker-mem-types.h marker.h marker-quota.h marker-quota-helper.h marker-common.h $(top_builddir)/xlators/lib/src/libxlator.h
+noinst_HEADERS = marker-mem-types.h marker.h marker-quota.h \
+	marker-quota-helper.h marker-common.h \
+	$(top_builddir)/xlators/lib/src/libxlator.h
 
 AM_CPPFLAGS = $(GF_CPPFLAGS) -I$(top_srcdir)/libglusterfs/src \
+	-I$(top_srcdir)/rpc/xdr/src -I$(top_builddir)/rpc/xdr/src \
 	-I$(top_srcdir)/xlators/lib/src
 
 AM_CFLAGS = -Wall -fno-strict-aliasing $(GF_CFLAGS)
diff --git a/xlators/features/marker/src/marker-common.c b/xlators/features/marker/src/marker-common.c
index 84a718add97..9c9047005d6 100644
--- a/xlators/features/marker/src/marker-common.c
+++ b/xlators/features/marker/src/marker-common.c
@@ -7,63 +7,51 @@
    later), or the GNU General Public License, version 2 (GPLv2), in all
    cases as published by the Free Software Foundation.
 */
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
 #include <fnmatch.h>
 #include "marker-common.h"
 
 marker_inode_ctx_t *
-marker_inode_ctx_new ()
+marker_inode_ctx_new()
 {
-        marker_inode_ctx_t *ctx = NULL;
+    marker_inode_ctx_t *ctx = NULL;
 
-        ctx = GF_CALLOC (1, sizeof (marker_inode_ctx_t),
-                         gf_marker_mt_marker_inode_ctx_t);
-        if (ctx == NULL)
-                goto out;
+    ctx = GF_CALLOC(1, sizeof(marker_inode_ctx_t),
+                    gf_marker_mt_marker_inode_ctx_t);
+    if (ctx == NULL)
+        goto out;
 
-        ctx->quota_ctx = NULL;
+    ctx->quota_ctx = NULL;
 out:
-        return ctx;
+    return ctx;
 }
 
 int32_t
-marker_force_inode_ctx_get (inode_t *inode, xlator_t *this,
-                            marker_inode_ctx_t **ctx)
+marker_force_inode_ctx_get(inode_t *inode, xlator_t *this,
+                           marker_inode_ctx_t **ctx)
 {
-        int32_t  ret     = -1;
-        uint64_t ctx_int = 0;
-
-        LOCK (&inode->lock);
-        {
-                ret = __inode_ctx_get (inode, this, &ctx_int);
-                if (ret == 0)
-                        *ctx = (marker_inode_ctx_t *) (unsigned long)ctx_int;
-                else {
-                        *ctx = marker_inode_ctx_new ();
-                        if (*ctx == NULL)
-                                goto unlock;
-
-                        ret = __inode_ctx_put (inode, this,
-                                               (uint64_t )(unsigned long) *ctx);
-                        if (ret == -1) {
-                                GF_FREE (*ctx);
-                                goto unlock;
-                        }
-                        ret = 0;
-                }
+    int32_t ret = -1;
+    uint64_t ctx_int = 0;
+
+    LOCK(&inode->lock);
+    {
+        ret = __inode_ctx_get(inode, this, &ctx_int);
+        if (ret == 0)
+            *ctx = (marker_inode_ctx_t *)(unsigned long)ctx_int;
+        else {
+            *ctx = marker_inode_ctx_new();
+            if (*ctx == NULL)
+                goto unlock;
+
+            ret = __inode_ctx_put(inode, this, (uint64_t)(unsigned long)*ctx);
+            if (ret == -1) {
+                GF_FREE(*ctx);
+                goto unlock;
+            }
+            ret = 0;
         }
-unlock: UNLOCK (&inode->lock);
+    }
+unlock:
+    UNLOCK(&inode->lock);
 
-        return ret;
-}
-
-int
-marker_filter_quota_xattr (dict_t *dict, char *key,
-                           data_t *value, void *data)
-{
-        dict_del (dict, key);
-        return 0;
+    return ret;
 }
diff --git a/xlators/features/marker/src/marker-common.h b/xlators/features/marker/src/marker-common.h
index 23dd846cb0a..7f8cffe7d35 100644
--- a/xlators/features/marker/src/marker-common.h
+++ b/xlators/features/marker/src/marker-common.h
@@ -10,18 +10,10 @@
 #ifndef _MARKER_COMMON_H
 #define _MARKER_COMMON_H
 
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
-
-#include "inode.h"
-#include "xlator.h"
+#include <glusterfs/xlator.h>
 #include "marker.h"
 
 int32_t
-marker_force_inode_ctx_get (inode_t *, xlator_t *, marker_inode_ctx_t **);
+marker_force_inode_ctx_get(inode_t *, xlator_t *, marker_inode_ctx_t **);
 
-int
-marker_filter_quota_xattr (dict_t *, char *, data_t *, void *);
 #endif
diff --git a/xlators/features/marker/src/marker-mem-types.h b/xlators/features/marker/src/marker-mem-types.h
index dc5ad16ed76..aedfdb4a1b7 100644
--- a/xlators/features/marker/src/marker-mem-types.h
+++ b/xlators/features/marker/src/marker-mem-types.h
@@ -10,18 +10,19 @@
 #ifndef __MARKER_MEM_TYPES_H__
 #define __MARKER_MEM_TYPES_H__
 
-#include "mem-types.h"
+#include <glusterfs/mem-types.h>
 
 enum gf_marker_mem_types_ {
-        gf_marker_mt_marker_conf_t = gf_common_mt_end + 1,
-        gf_marker_mt_loc_t,
-        gf_marker_mt_volume_mark,
-        gf_marker_mt_int64_t,
-        gf_marker_mt_quota_inode_ctx_t,
-        gf_marker_mt_marker_inode_ctx_t,
-        gf_marker_mt_inode_contribution_t,
-        gf_marker_mt_quota_meta_t,
-        gf_marker_mt_quota_synctask_t,
-        gf_marker_mt_end
+    /* Those are used by ALLOCATE_OR_GOTO macro */
+    gf_marker_mt_marker_conf_t = gf_common_mt_end + 1,
+    gf_marker_mt_loc_t,
+    gf_marker_mt_volume_mark,
+    gf_marker_mt_int64_t,
+    gf_marker_mt_quota_inode_ctx_t,
+    gf_marker_mt_marker_inode_ctx_t,
+    gf_marker_mt_inode_contribution_t,
+    gf_marker_mt_quota_meta_t,
+    gf_marker_mt_quota_synctask_t,
+    gf_marker_mt_end
 };
 #endif
diff --git a/xlators/features/marker/src/marker-quota-helper.c b/xlators/features/marker/src/marker-quota-helper.c
index 67801c86403..ecd85d67b2b 100644
--- a/xlators/features/marker/src/marker-quota-helper.c
+++ b/xlators/features/marker/src/marker-quota-helper.c
@@ -7,429 +7,374 @@
    later), or the GNU General Public License, version 2 (GPLv2), in all
    cases as published by the Free Software Foundation.
 */
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
-
-#include "locking.h"
+#include <glusterfs/locking.h>
 #include "marker-quota.h"
 #include "marker-common.h"
 #include "marker-quota-helper.h"
 #include "marker-mem-types.h"
 
 int
-mq_loc_fill (loc_t *loc, inode_t *inode, inode_t *parent, char *path)
+mq_loc_fill(loc_t *loc, inode_t *inode, inode_t *parent, char *path)
 {
-        int ret = -1;
+    int ret = -1;
 
-        GF_VALIDATE_OR_GOTO ("marker", loc, out);
-        GF_VALIDATE_OR_GOTO ("marker", inode, out);
-        GF_VALIDATE_OR_GOTO ("marker", path, out);
-        /* Not checking for parent because while filling
-         * loc of root, parent will be NULL
-         */
+    GF_VALIDATE_OR_GOTO("marker", loc, out);
+    GF_VALIDATE_OR_GOTO("marker", inode, out);
+    GF_VALIDATE_OR_GOTO("marker", path, out);
+    /* Not checking for parent because while filling
+     * loc of root, parent will be NULL
+     */
 
-        if (inode) {
-                loc->inode = inode_ref (inode);
-        }
+    if (inode) {
+        loc->inode = inode_ref(inode);
+    }
 
-        if (parent)
-                loc->parent = inode_ref (parent);
+    if (parent)
+        loc->parent = inode_ref(parent);
 
-        if (!gf_uuid_is_null (inode->gfid))
-                gf_uuid_copy (loc->gfid, inode->gfid);
+    if (!gf_uuid_is_null(inode->gfid))
+        gf_uuid_copy(loc->gfid, inode->gfid);
 
-        loc->path = gf_strdup (path);
-        if (!loc->path) {
-                gf_log ("loc fill", GF_LOG_ERROR, "strdup failed");
-                goto out;
-        }
+    loc->path = gf_strdup(path);
+    if (!loc->path) {
+        gf_log("loc fill", GF_LOG_ERROR, "strdup failed");
+        goto out;
+    }
 
-        loc->name = strrchr (loc->path, '/');
-        if (loc->name)
-                loc->name++;
-        else
-                goto out;
+    loc->name = strrchr(loc->path, '/');
+    if (loc->name)
+        loc->name++;
+    else
+        goto out;
 
-        ret = 0;
+    ret = 0;
 
 out:
-        if (ret < 0)
-                loc_wipe (loc);
+    if (ret < 0)
+        loc_wipe(loc);
 
-        return ret;
+    return ret;
 }
 
-
 int32_t
-mq_inode_loc_fill (const char *parent_gfid, inode_t *inode, loc_t *loc)
+mq_inode_loc_fill(const char *parent_gfid, inode_t *inode, loc_t *loc)
 {
-        char            *resolvedpath = NULL;
-        inode_t         *parent       = NULL;
-        int              ret          = -1;
+    char *resolvedpath = NULL;
+    inode_t *parent = NULL;
+    quota_inode_ctx_t *ctx = NULL;
+    xlator_t *this = NULL;
+    int ret = -1;
+
+    this = THIS;
+
+    if (inode == NULL) {
+        gf_log_callingfn("marker", GF_LOG_ERROR,
+                         "loc fill failed, "
+                         "inode is NULL");
+        return ret;
+    }
 
-        if ((!inode) || (!loc))
-                return ret;
+    if (loc == NULL)
+        return ret;
 
-        if ((inode) && __is_root_gfid (inode->gfid)) {
-                loc->parent = NULL;
-                goto ignore_parent;
-        }
+    if ((inode) && __is_root_gfid(inode->gfid)) {
+        loc->parent = NULL;
+        goto ignore_parent;
+    }
 
-        if (parent_gfid == NULL)
-                parent = inode_parent (inode, 0, NULL);
-        else
-                parent = inode_find (inode->table,
-                                     (unsigned char *) parent_gfid);
+    if (parent_gfid == NULL)
+        parent = inode_parent(inode, 0, NULL);
+    else
+        parent = inode_find(inode->table, (unsigned char *)parent_gfid);
 
-        if (parent == NULL) {
-                gf_log ("marker", GF_LOG_ERROR, "parent is NULL for %s",
-                        uuid_utoa(inode->gfid));
-                goto err;
-        }
+    if (parent == NULL) {
+        gf_log("marker", GF_LOG_ERROR, "parent is NULL for %s",
+               uuid_utoa(inode->gfid));
+        goto err;
+    }
 
 ignore_parent:
-        ret = inode_path (inode, NULL, &resolvedpath);
-        if (ret < 0) {
-                gf_log ("marker", GF_LOG_ERROR, "failed to resolve path for %s",
-                        uuid_utoa(inode->gfid));
-                goto err;
-        }
-
-        ret = mq_loc_fill (loc, inode, parent, resolvedpath);
-        if (ret < 0)
-                goto err;
+    ret = inode_path(inode, NULL, &resolvedpath);
+    if (ret < 0) {
+        gf_log("marker", GF_LOG_ERROR, "failed to resolve path for %s",
+               uuid_utoa(inode->gfid));
+        goto err;
+    }
+
+    ret = mq_loc_fill(loc, inode, parent, resolvedpath);
+    if (ret < 0)
+        goto err;
+
+    ret = mq_inode_ctx_get(inode, this, &ctx);
+    if (ret < 0 || ctx == NULL)
+        ctx = mq_inode_ctx_new(inode, this);
+    if (ctx == NULL) {
+        gf_log(this->name, GF_LOG_WARNING,
+               "mq_inode_ctx_new "
+               "failed for %s",
+               uuid_utoa(inode->gfid));
+        ret = -1;
+        goto err;
+    }
+    ret = 0;
 
 err:
-        if (parent)
-                inode_unref (parent);
+    if (parent)
+        inode_unref(parent);
 
-        GF_FREE (resolvedpath);
+    GF_FREE(resolvedpath);
 
-        return ret;
+    return ret;
 }
 
-
 quota_inode_ctx_t *
-mq_alloc_inode_ctx ()
-{
-        int32_t                  ret    = -1;
-        quota_inode_ctx_t       *ctx    = NULL;
-
-        QUOTA_ALLOC (ctx, quota_inode_ctx_t, ret);
-        if (ret == -1)
-                goto out;
-
-        ctx->size = 0;
-        ctx->dirty = 0;
-        ctx->updation_status = _gf_false;
-        LOCK_INIT (&ctx->lock);
-        INIT_LIST_HEAD (&ctx->contribution_head);
-out:
-        return ctx;
-}
-
-inode_contribution_t *
-mq_get_contribution_node (inode_t *inode, quota_inode_ctx_t *ctx)
+mq_alloc_inode_ctx()
 {
-        inode_contribution_t    *contri = NULL;
-        inode_contribution_t    *temp   = NULL;
-
-        if (!inode || !ctx)
-                goto out;
-
-        list_for_each_entry (temp, &ctx->contribution_head, contri_list) {
-                if (gf_uuid_compare (temp->gfid, inode->gfid) == 0) {
-                        contri = temp;
-                        goto out;
-                }
-        }
+    int32_t ret = -1;
+    quota_inode_ctx_t *ctx = NULL;
+
+    QUOTA_ALLOC(ctx, quota_inode_ctx_t, ret);
+    if (ret == -1)
+        goto out;
+
+    ctx->size = 0;
+    ctx->dirty = 0;
+    ctx->updation_status = _gf_false;
+    LOCK_INIT(&ctx->lock);
+    INIT_LIST_HEAD(&ctx->contribution_head);
 out:
-        return contri;
+    return ctx;
 }
 
-
-int32_t
-mq_delete_contribution_node (dict_t *dict, char *key,
-                             inode_contribution_t *contribution)
+static void
+mq_contri_fini(inode_contribution_t *contri)
 {
-        if (dict_get (dict, key) != NULL)
-                goto out;
-
-        QUOTA_FREE_CONTRIBUTION_NODE (contribution);
-out:
-        return 0;
+    LOCK_DESTROY(&contri->lock);
+    GF_FREE(contri);
 }
 
-
 inode_contribution_t *
-__mq_add_new_contribution_node (xlator_t *this, quota_inode_ctx_t *ctx,
-                                loc_t *loc)
+mq_contri_init(inode_t *inode)
 {
-        int32_t               ret          = 0;
-        inode_contribution_t *contribution = NULL;
-
-        if (!loc->parent) {
-                if (!gf_uuid_is_null (loc->pargfid))
-                        loc->parent = inode_find (loc->inode->table,
-                                                  loc->pargfid);
-
-                if (!loc->parent)
-                        loc->parent = inode_parent (loc->inode, loc->pargfid,
-                                                    loc->name);
-                if (!loc->parent)
-                        goto out;
-        }
-
-        list_for_each_entry (contribution, &ctx->contribution_head,
-                             contri_list) {
-                if (loc->parent &&
-                    gf_uuid_compare (contribution->gfid, loc->parent->gfid) == 0) {
-                        goto out;
-                }
-        }
+    inode_contribution_t *contri = NULL;
+    int32_t ret = 0;
 
-        QUOTA_ALLOC (contribution, inode_contribution_t, ret);
-        if (ret == -1)
-                goto out;
+    QUOTA_ALLOC(contri, inode_contribution_t, ret);
+    if (ret == -1)
+        goto out;
 
-        contribution->contribution = 0;
+    GF_REF_INIT(contri, mq_contri_fini);
 
-        gf_uuid_copy (contribution->gfid, loc->parent->gfid);
+    contri->contribution = 0;
+    contri->file_count = 0;
+    contri->dir_count = 0;
+    gf_uuid_copy(contri->gfid, inode->gfid);
 
-	LOCK_INIT (&contribution->lock);
-        INIT_LIST_HEAD (&contribution->contri_list);
-
-        list_add_tail (&contribution->contri_list, &ctx->contribution_head);
+    LOCK_INIT(&contri->lock);
+    INIT_LIST_HEAD(&contri->contri_list);
 
 out:
-        return contribution;
+    return contri;
 }
 
-
 inode_contribution_t *
-mq_add_new_contribution_node (xlator_t *this, quota_inode_ctx_t *ctx,
-                              loc_t *loc)
+mq_get_contribution_node(inode_t *inode, quota_inode_ctx_t *ctx)
 {
-        inode_contribution_t *contribution = NULL;
+    inode_contribution_t *contri = NULL;
+    inode_contribution_t *temp = NULL;
 
-        if ((ctx == NULL) || (loc == NULL))
-                return NULL;
+    if (!inode || !ctx)
+        goto out;
 
-        if (((loc->path) && (strcmp (loc->path, "/") == 0))
-            || (!loc->path && gf_uuid_is_null (loc->pargfid)))
-                return NULL;
+    LOCK(&ctx->lock);
+    {
+        if (list_empty(&ctx->contribution_head))
+            goto unlock;
 
-        LOCK (&ctx->lock);
+        list_for_each_entry(temp, &ctx->contribution_head, contri_list)
         {
-                contribution = __mq_add_new_contribution_node (this, ctx, loc);
+            if (gf_uuid_compare(temp->gfid, inode->gfid) == 0) {
+                contri = temp;
+                GF_REF_GET(contri);
+                break;
+            }
         }
-        UNLOCK (&ctx->lock);
+    }
+unlock:
+    UNLOCK(&ctx->lock);
 
-        return contribution;
+out:
+    return contri;
 }
 
-
-int32_t
-mq_dict_set_contribution (xlator_t *this, dict_t *dict, loc_t *loc,
-                          uuid_t gfid, char *contri_key)
+inode_contribution_t *
+__mq_add_new_contribution_node(xlator_t *this, quota_inode_ctx_t *ctx,
+                               loc_t *loc)
 {
-        int32_t ret                  = -1;
-        char    key[CONTRI_KEY_MAX]  = {0, };
-
-        GF_VALIDATE_OR_GOTO ("marker", this, out);
-        GF_VALIDATE_OR_GOTO ("marker", dict, out);
-        GF_VALIDATE_OR_GOTO ("marker", loc, out);
-
-        if (gfid && !gf_uuid_is_null(gfid)) {
-                GET_CONTRI_KEY (key, gfid, ret);
-        } else if (loc->parent) {
-                GET_CONTRI_KEY (key, loc->parent->gfid, ret);
-        } else {
-                /* nameless lookup, fetch contributions to all parents */
-                GET_CONTRI_KEY (key, NULL, ret);
+    inode_contribution_t *contribution = NULL;
+
+    if (!loc->parent) {
+        if (!gf_uuid_is_null(loc->pargfid))
+            loc->parent = inode_find(loc->inode->table, loc->pargfid);
+
+        if (!loc->parent)
+            loc->parent = inode_parent(loc->inode, loc->pargfid, loc->name);
+        if (!loc->parent)
+            goto out;
+    }
+
+    list_for_each_entry(contribution, &ctx->contribution_head, contri_list)
+    {
+        if (loc->parent &&
+            gf_uuid_compare(contribution->gfid, loc->parent->gfid) == 0) {
+            goto out;
         }
+    }
 
-        if (ret < 0)
-                goto out;
-
-        ret = dict_set_int64 (dict, key, 0);
-        if (ret < 0)
-                goto out;
+    contribution = mq_contri_init(loc->parent);
+    if (contribution == NULL)
+        goto out;
 
-        if (contri_key)
-                strncpy (contri_key, key, CONTRI_KEY_MAX);
+    list_add_tail(&contribution->contri_list, &ctx->contribution_head);
 
 out:
-        if (ret < 0)
-                gf_log_callingfn (this->name, GF_LOG_ERROR, "dict set failed");
-
-        return ret;
+    return contribution;
 }
 
-
-int32_t
-mq_inode_ctx_get (inode_t *inode, xlator_t *this,
-                  quota_inode_ctx_t **ctx)
+inode_contribution_t *
+mq_add_new_contribution_node(xlator_t *this, quota_inode_ctx_t *ctx, loc_t *loc)
 {
-        int32_t             ret      = -1;
-        uint64_t            ctx_int  = 0;
-        marker_inode_ctx_t *mark_ctx = NULL;
-
-        GF_VALIDATE_OR_GOTO ("marker", inode, out);
-        GF_VALIDATE_OR_GOTO ("marker", this, out);
-        GF_VALIDATE_OR_GOTO ("marker", ctx, out);
-
-        ret = inode_ctx_get (inode, this, &ctx_int);
-        if (ret < 0) {
-                ret = -1;
-                *ctx = NULL;
-                goto out;
-        }
+    inode_contribution_t *contribution = NULL;
 
-        mark_ctx = (marker_inode_ctx_t *) (unsigned long)ctx_int;
-        if (mark_ctx->quota_ctx == NULL) {
-                ret = -1;
-                goto out;
-        }
+    if ((ctx == NULL) || (loc == NULL))
+        return NULL;
 
-        *ctx = mark_ctx->quota_ctx;
+    if (((loc->path) && (strcmp(loc->path, "/") == 0)) ||
+        (!loc->path && gf_uuid_is_null(loc->pargfid)))
+        return NULL;
 
-        ret = 0;
+    LOCK(&ctx->lock);
+    {
+        contribution = __mq_add_new_contribution_node(this, ctx, loc);
+        if (contribution)
+            GF_REF_GET(contribution);
+    }
+    UNLOCK(&ctx->lock);
 
-out:
-        return ret;
+    return contribution;
 }
 
-
-quota_inode_ctx_t *
-__mq_inode_ctx_new (inode_t *inode, xlator_t *this)
+int32_t
+mq_dict_set_contribution(xlator_t *this, dict_t *dict, loc_t *loc, uuid_t gfid,
+                         char *contri_key)
 {
-        int32_t               ret        = -1;
-        quota_inode_ctx_t    *quota_ctx  = NULL;
-        marker_inode_ctx_t   *mark_ctx   = NULL;
-
-        ret = marker_force_inode_ctx_get (inode, this, &mark_ctx);
-        if (ret < 0) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "marker_force_inode_ctx_get() failed");
-                goto out;
-        }
-
-        LOCK (&inode->lock);
-        {
-                if (mark_ctx->quota_ctx == NULL) {
-                        quota_ctx = mq_alloc_inode_ctx ();
-                        if (quota_ctx == NULL) {
-                                ret = -1;
-                                goto unlock;
-                        }
-                        mark_ctx->quota_ctx = quota_ctx;
-                } else {
-                        quota_ctx = mark_ctx->quota_ctx;
-                }
-
-                ret = 0;
+    int32_t ret = -1;
+    char key[QUOTA_KEY_MAX] = {
+        0,
+    };
+
+    GF_VALIDATE_OR_GOTO("marker", this, out);
+    GF_VALIDATE_OR_GOTO("marker", dict, out);
+    GF_VALIDATE_OR_GOTO("marker", loc, out);
+
+    if (gfid && !gf_uuid_is_null(gfid)) {
+        GET_CONTRI_KEY(this, key, gfid, ret);
+    } else if (loc->parent) {
+        GET_CONTRI_KEY(this, key, loc->parent->gfid, ret);
+    } else {
+        /* nameless lookup, fetch contributions to all parents */
+        GET_CONTRI_KEY(this, key, NULL, ret);
+    }
+
+    if (ret < 0)
+        goto out;
+
+    ret = dict_set_int64(dict, key, 0);
+    if (ret < 0)
+        goto out;
+
+    if (contri_key)
+        if (snprintf(contri_key, QUOTA_KEY_MAX, "%s", key) >= QUOTA_KEY_MAX) {
+            ret = -1;
+            goto out;
         }
-unlock:
-        UNLOCK (&inode->lock);
-out:
-        return quota_ctx;
-}
-
-
-quota_inode_ctx_t *
-mq_inode_ctx_new (inode_t * inode, xlator_t *this)
-{
-        return __mq_inode_ctx_new (inode, this);
-}
-
-quota_local_t *
-mq_local_new ()
-{
-        quota_local_t  *local   = NULL;
-
-        local = mem_get0 (THIS->local_pool);
-        if (!local)
-                goto out;
-
-        local->ref = 1;
-        LOCK_INIT (&local->lock);
-
-        local->ctx = NULL;
-        local->contri = NULL;
 
 out:
-        return local;
-}
-
-quota_local_t *
-mq_local_ref (quota_local_t *local)
-{
-        LOCK (&local->lock);
-        {
-                local->ref ++;
-        }
-        UNLOCK (&local->lock);
+    if (ret < 0)
+        gf_log_callingfn(this ? this->name : "Marker", GF_LOG_ERROR,
+                         "dict set failed");
 
-        return local;
+    return ret;
 }
 
-
 int32_t
-mq_local_unref (xlator_t *this, quota_local_t *local)
+mq_inode_ctx_get(inode_t *inode, xlator_t *this, quota_inode_ctx_t **ctx)
 {
-        int32_t ref = 0;
-        if (local == NULL)
-                goto out;
+    int32_t ret = -1;
+    uint64_t ctx_int = 0;
+    marker_inode_ctx_t *mark_ctx = NULL;
 
-        QUOTA_SAFE_DECREMENT (&local->lock, local->ref, ref);
+    GF_VALIDATE_OR_GOTO("marker", inode, out);
+    GF_VALIDATE_OR_GOTO("marker", this, out);
+    GF_VALIDATE_OR_GOTO("marker", ctx, out);
 
-        if (ref != 0)
-                goto out;
+    ret = inode_ctx_get(inode, this, &ctx_int);
+    if (ret < 0) {
+        ret = -1;
+        *ctx = NULL;
+        goto out;
+    }
 
-        if (local->fd != NULL)
-                fd_unref (local->fd);
+    mark_ctx = (marker_inode_ctx_t *)(unsigned long)ctx_int;
+    if (mark_ctx->quota_ctx == NULL) {
+        ret = -1;
+        goto out;
+    }
 
-        loc_wipe (&local->loc);
+    *ctx = mark_ctx->quota_ctx;
 
-        loc_wipe (&local->parent_loc);
+    ret = 0;
 
-        LOCK_DESTROY (&local->lock);
-
-        mem_put (local);
 out:
-        return 0;
+    return ret;
 }
 
-
-inode_contribution_t *
-mq_get_contribution_from_loc (xlator_t *this, loc_t *loc)
+quota_inode_ctx_t *
+__mq_inode_ctx_new(inode_t *inode, xlator_t *this)
 {
-        int32_t               ret          = 0;
-        quota_inode_ctx_t    *ctx          = NULL;
-        inode_contribution_t *contribution = NULL;
-
-        ret = mq_inode_ctx_get (loc->inode, this, &ctx);
-        if (ret < 0) {
-                gf_log_callingfn (this->name, GF_LOG_WARNING,
-                                  "cannot get marker-quota context from inode "
-                                  "(gfid:%s, path:%s)",
-                                  uuid_utoa (loc->inode->gfid), loc->path);
-                goto err;
+    int32_t ret = -1;
+    quota_inode_ctx_t *quota_ctx = NULL;
+    marker_inode_ctx_t *mark_ctx = NULL;
+
+    ret = marker_force_inode_ctx_get(inode, this, &mark_ctx);
+    if (ret < 0) {
+        gf_log(this->name, GF_LOG_ERROR, "marker_force_inode_ctx_get() failed");
+        goto out;
+    }
+
+    LOCK(&inode->lock);
+    {
+        if (mark_ctx->quota_ctx == NULL) {
+            quota_ctx = mq_alloc_inode_ctx();
+            if (quota_ctx == NULL) {
+                ret = -1;
+                goto unlock;
+            }
+            mark_ctx->quota_ctx = quota_ctx;
+        } else {
+            quota_ctx = mark_ctx->quota_ctx;
         }
 
-        contribution = mq_get_contribution_node (loc->parent, ctx);
-        if (contribution == NULL) {
-                gf_log_callingfn (this->name, GF_LOG_WARNING,
-                                  "inode (gfid:%s, path:%s) has "
-                                  "no contribution towards parent (gfid:%s)",
-                                  uuid_utoa (loc->inode->gfid),
-                                  loc->path, uuid_utoa (loc->parent->gfid));
-                goto err;
-        }
+        ret = 0;
+    }
+unlock:
+    UNLOCK(&inode->lock);
+out:
+    return quota_ctx;
+}
 
-err:
-        return contribution;
+quota_inode_ctx_t *
+mq_inode_ctx_new(inode_t *inode, xlator_t *this)
+{
+    return __mq_inode_ctx_new(inode, this);
 }
diff --git a/xlators/features/marker/src/marker-quota-helper.h b/xlators/features/marker/src/marker-quota-helper.h
index 161413debfa..d4091dd2180 100644
--- a/xlators/features/marker/src/marker-quota-helper.h
+++ b/xlators/features/marker/src/marker-quota-helper.h
@@ -11,66 +11,56 @@
 #ifndef _MARKER_QUOTA_HELPER_H
 #define _MARKER_QUOTA_HELPER_H
 
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
-
 #include "marker.h"
 
-#define QUOTA_FREE_CONTRIBUTION_NODE(_contribution)     \
-        do {                                            \
-                list_del (&_contribution->contri_list); \
-                GF_FREE (_contribution);                \
-        } while (0)
-
-#define QUOTA_SAFE_INCREMENT(lock, var)                 \
-        do {                                            \
-                LOCK (lock);                            \
-                        var ++;                         \
-                UNLOCK (lock);                          \
-        } while (0)
-
-#define QUOTA_SAFE_DECREMENT(lock, var, value)  \
-        do {                                    \
-                LOCK (lock);                    \
-                {                               \
-                      value = --var;            \
-                }                               \
-                UNLOCK (lock);                  \
-        } while (0)
+#define QUOTA_FREE_CONTRIBUTION_NODE(ctx, _contribution)                       \
+    do {                                                                       \
+        LOCK(&ctx->lock);                                                      \
+        {                                                                      \
+            list_del_init(&_contribution->contri_list);                        \
+            GF_REF_PUT(_contribution);                                         \
+        }                                                                      \
+        UNLOCK(&ctx->lock);                                                    \
+    } while (0)
+
+#define QUOTA_SAFE_INCREMENT(lock, var)                                        \
+    do {                                                                       \
+        LOCK(lock);                                                            \
+        var++;                                                                 \
+        UNLOCK(lock);                                                          \
+    } while (0)
+
+#define QUOTA_SAFE_DECREMENT(lock, var, value)                                 \
+    do {                                                                       \
+        LOCK(lock);                                                            \
+        {                                                                      \
+            value = --var;                                                     \
+        }                                                                      \
+        UNLOCK(lock);                                                          \
+    } while (0)
 
 inode_contribution_t *
-mq_add_new_contribution_node (xlator_t *, quota_inode_ctx_t *, loc_t *);
+mq_add_new_contribution_node(xlator_t *, quota_inode_ctx_t *, loc_t *);
 
 int32_t
-mq_dict_set_contribution (xlator_t *, dict_t *, loc_t *, uuid_t, char *);
+mq_dict_set_contribution(xlator_t *, dict_t *, loc_t *, uuid_t, char *);
 
 quota_inode_ctx_t *
-mq_inode_ctx_new (inode_t *, xlator_t *);
+mq_inode_ctx_new(inode_t *, xlator_t *);
 
 int32_t
-mq_inode_ctx_get (inode_t *, xlator_t *, quota_inode_ctx_t **);
+mq_inode_ctx_get(inode_t *, xlator_t *, quota_inode_ctx_t **);
 
 int32_t
-mq_delete_contribution_node (dict_t *, char *, inode_contribution_t *);
-
-int32_t
-mq_inode_loc_fill (const char *, inode_t *, loc_t *);
-
-quota_local_t *
-mq_local_new ();
-
-quota_local_t *
-mq_local_ref (quota_local_t *);
+mq_delete_contribution_node(dict_t *, char *, inode_contribution_t *);
 
 int32_t
-mq_local_unref (xlator_t *, quota_local_t *);
+mq_inode_loc_fill(const char *, inode_t *, loc_t *);
 
 inode_contribution_t *
-mq_get_contribution_node (inode_t *, quota_inode_ctx_t *);
+mq_contri_init(inode_t *inode);
 
 inode_contribution_t *
-mq_get_contribution_from_loc (xlator_t *this, loc_t *loc);
+mq_get_contribution_node(inode_t *, quota_inode_ctx_t *);
 
 #endif
diff --git a/xlators/features/marker/src/marker-quota.c b/xlators/features/marker/src/marker-quota.c
index c6cf470e5a4..3de2ea1c92c 100644
--- a/xlators/features/marker/src/marker-quota.c
+++ b/xlators/features/marker/src/marker-quota.c
@@ -7,4006 +7,2291 @@
    later), or the GNU General Public License, version 2 (GPLv2), in all
    cases as published by the Free Software Foundation.
 */
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
-
-#include "dict.h"
-#include "xlator.h"
-#include "defaults.h"
+#include <glusterfs/dict.h>
+#include <glusterfs/xlator.h>
+#include <glusterfs/defaults.h>
 #include "libxlator.h"
-#include "common-utils.h"
-#include "byte-order.h"
+#include <glusterfs/common-utils.h>
+#include <glusterfs/byte-order.h>
 #include "marker-quota.h"
 #include "marker-quota-helper.h"
-#include "syncop.h"
-#include "quota-common-utils.h"
+#include <glusterfs/syncop.h>
+#include <glusterfs/quota-common-utils.h>
 
 int
-mq_loc_copy (loc_t *dst, loc_t *src)
+mq_loc_copy(loc_t *dst, loc_t *src)
 {
-        int ret = -1;
+    int ret = -1;
 
-        GF_VALIDATE_OR_GOTO ("marker", dst, out);
-        GF_VALIDATE_OR_GOTO ("marker", src, out);
+    GF_VALIDATE_OR_GOTO("marker", dst, out);
+    GF_VALIDATE_OR_GOTO("marker", src, out);
 
-        if (src->inode == NULL ||
-            ((src->parent == NULL) && (gf_uuid_is_null (src->pargfid))
-             && !__is_root_gfid (src->inode->gfid))) {
-                gf_log ("marker", GF_LOG_WARNING,
-                        "src loc is not valid");
-                goto out;
-        }
+    if (src->inode == NULL ||
+        ((src->parent == NULL) && (gf_uuid_is_null(src->pargfid)) &&
+         !__is_root_gfid(src->inode->gfid))) {
+        gf_log("marker", GF_LOG_WARNING, "src loc is not valid");
+        goto out;
+    }
 
-        ret = loc_copy (dst, src);
+    ret = loc_copy(dst, src);
 out:
-        return ret;
+    return ret;
 }
 
-int32_t
-mq_get_local_err (quota_local_t *local,
-                  int32_t *val)
+static void
+mq_set_ctx_status(quota_inode_ctx_t *ctx, gf_boolean_t *flag,
+                  gf_boolean_t status)
 {
-        int32_t ret = -1;
-
-        GF_VALIDATE_OR_GOTO ("marker", local, out);
-        GF_VALIDATE_OR_GOTO ("marker", val, out);
-
-        LOCK (&local->lock);
-        {
-                *val = local->err;
-        }
-        UNLOCK (&local->lock);
-
-        ret = 0;
-out:
-        return ret;
+    LOCK(&ctx->lock);
+    {
+        *flag = status;
+    }
+    UNLOCK(&ctx->lock);
 }
 
-int32_t
-mq_get_ctx_updation_status (quota_inode_ctx_t *ctx,
-                            gf_boolean_t *status)
+static void
+mq_test_and_set_ctx_status(quota_inode_ctx_t *ctx, gf_boolean_t *flag,
+                           gf_boolean_t *status)
 {
-        int32_t   ret = -1;
-
-        GF_VALIDATE_OR_GOTO ("marker", ctx, out);
-        GF_VALIDATE_OR_GOTO ("marker", status, out);
-
-        LOCK (&ctx->lock);
-        {
-                *status = ctx->updation_status;
-        }
-        UNLOCK (&ctx->lock);
+    gf_boolean_t temp = _gf_false;
 
-        ret = 0;
-out:
-        return ret;
+    LOCK(&ctx->lock);
+    {
+        temp = *status;
+        *status = *flag;
+        *flag = temp;
+    }
+    UNLOCK(&ctx->lock);
 }
 
-
-int32_t
-mq_set_ctx_updation_status (quota_inode_ctx_t *ctx,
-                            gf_boolean_t status)
+static void
+mq_get_ctx_status(quota_inode_ctx_t *ctx, gf_boolean_t *flag,
+                  gf_boolean_t *status)
 {
-        int32_t   ret = -1;
-
-        if (ctx == NULL)
-                goto out;
-
-        LOCK (&ctx->lock);
-        {
-                ctx->updation_status = status;
-        }
-        UNLOCK (&ctx->lock);
-
-        ret = 0;
-out:
-        return ret;
+    LOCK(&ctx->lock);
+    {
+        *status = *flag;
+    }
+    UNLOCK(&ctx->lock);
 }
 
 int32_t
-mq_test_and_set_ctx_updation_status (quota_inode_ctx_t *ctx,
-                                     gf_boolean_t *status)
+mq_get_ctx_updation_status(quota_inode_ctx_t *ctx, gf_boolean_t *status)
 {
-        int32_t         ret     = -1;
-        gf_boolean_t    temp    = _gf_false;
-
-        GF_VALIDATE_OR_GOTO ("marker", ctx, out);
-        GF_VALIDATE_OR_GOTO ("marker", status, out);
-
-        LOCK (&ctx->lock);
-        {
-                temp = *status;
-                *status = ctx->updation_status;
-                ctx->updation_status = temp;
-        }
-        UNLOCK (&ctx->lock);
+    GF_VALIDATE_OR_GOTO("marker", ctx, out);
+    GF_VALIDATE_OR_GOTO("marker", status, out);
 
-        ret = 0;
+    mq_get_ctx_status(ctx, &ctx->updation_status, status);
+    return 0;
 out:
-        return ret;
+    return -1;
 }
 
-void
-mq_assign_lk_owner (xlator_t *this, call_frame_t *frame)
-{
-        marker_conf_t *conf     = NULL;
-        uint64_t       lk_owner = 0;
-
-        conf = this->private;
-
-        LOCK (&conf->lock);
-        {
-                if (++conf->quota_lk_owner == 0) {
-                        ++conf->quota_lk_owner;
-                }
-
-                lk_owner = conf->quota_lk_owner;
-        }
-        UNLOCK (&conf->lock);
-
-        set_lk_owner_from_uint64 (&frame->root->lk_owner, lk_owner);
-
-        return;
-}
-
-
 int32_t
-mq_loc_fill_from_name (xlator_t *this, loc_t *newloc, loc_t *oldloc,
-                       uint64_t ino, char *name)
+mq_set_ctx_updation_status(quota_inode_ctx_t *ctx, gf_boolean_t status)
 {
-        int32_t   ret  = -1;
-        int32_t   len  = 0;
-        char     *path = NULL;
-
-        GF_VALIDATE_OR_GOTO ("marker", this, out);
-        GF_VALIDATE_OR_GOTO ("marker", newloc, out);
-        GF_VALIDATE_OR_GOTO ("marker", oldloc, out);
-        GF_VALIDATE_OR_GOTO ("marker", name, out);
-
-        newloc->inode = inode_new (oldloc->inode->table);
-
-        if (!newloc->inode) {
-                ret = -1;
-                goto out;
-        }
-
-        newloc->parent = inode_ref (oldloc->inode);
-        gf_uuid_copy (newloc->pargfid, oldloc->inode->gfid);
+    GF_VALIDATE_OR_GOTO("marker", ctx, out);
 
-        if (!oldloc->path) {
-                ret = loc_path (oldloc, NULL);
-                if (ret == -1)
-                        goto out;
-        }
-
-        len = strlen (oldloc->path);
-
-        if (oldloc->path [len - 1] == '/')
-                ret = gf_asprintf ((char **) &path, "%s%s",
-                                   oldloc->path, name);
-        else
-                ret = gf_asprintf ((char **) &path, "%s/%s",
-                                   oldloc->path, name);
-
-        if (ret < 0)
-                goto out;
-
-        newloc->path = path;
-
-        newloc->name = strrchr (newloc->path, '/');
-
-        if (newloc->name)
-                newloc->name++;
-
-        gf_log (this->name, GF_LOG_DEBUG, "path = %s name =%s",
-                newloc->path, newloc->name);
+    mq_set_ctx_status(ctx, &ctx->updation_status, status);
+    return 0;
 out:
-        return ret;
-}
-
-int32_t
-mq_dirty_inode_updation_done (call_frame_t *frame, void *cookie, xlator_t *this,
-                              int32_t op_ret, int32_t op_errno, dict_t *xdata)
-{
-        QUOTA_STACK_DESTROY (frame, this);
-
-        return 0;
+    return -1;
 }
 
 int32_t
-mq_release_lock_on_dirty_inode (call_frame_t *frame, void *cookie, xlator_t *this,
-                                int32_t op_ret, int32_t op_errno, dict_t *xdata)
+mq_test_and_set_ctx_updation_status(quota_inode_ctx_t *ctx,
+                                    gf_boolean_t *status)
 {
-        struct gf_flock   lock  = {0, };
-        quota_local_t    *local = NULL;
-        loc_t             loc = {0, };
-	int               ret = -1;
+    GF_VALIDATE_OR_GOTO("marker", ctx, out);
+    GF_VALIDATE_OR_GOTO("marker", status, out);
 
-        local = frame->local;
-
-        if (op_ret == -1) {
-                local->err = -1;
-
-                mq_dirty_inode_updation_done (frame, NULL, this, 0, 0, NULL);
-
-                return 0;
-        }
-
-        if (op_ret == 0)
-                local->ctx->dirty = 0;
-
-        lock.l_type   = F_UNLCK;
-        lock.l_whence = SEEK_SET;
-        lock.l_start  = 0;
-        lock.l_len    = 0;
-        lock.l_pid    = 0;
-
-        ret = loc_copy (&loc, &local->loc);
-	if (ret == -1) {
-                local->err = -1;
-                frame->local = NULL;
-                mq_dirty_inode_updation_done (frame, NULL, this, 0, 0, NULL);
-                return 0;
-        }
-
-        if (local->loc.inode == NULL) {
-                gf_log (this->name, GF_LOG_WARNING,
-                        "Inode is NULL, so can't stackwind.");
-                goto out;
-        }
-
-        STACK_WIND (frame,
-                    mq_dirty_inode_updation_done,
-                    FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->inodelk,
-                    this->name, &loc, F_SETLKW, &lock, NULL);
-
-        loc_wipe (&loc);
-
-        return 0;
+    mq_test_and_set_ctx_status(ctx, &ctx->updation_status, status);
+    return 0;
 out:
-        mq_dirty_inode_updation_done (frame, NULL, this, -1, 0, NULL);
-
-        return 0;
+    return -1;
 }
 
 int32_t
-mq_mark_inode_undirty (call_frame_t *frame, void *cookie, xlator_t *this,
-                       int32_t op_ret, int32_t op_errno, dict_t *dict,
-                       dict_t *xdata)
+mq_set_ctx_create_status(quota_inode_ctx_t *ctx, gf_boolean_t status)
 {
-        int32_t        ret     = -1;
-        int64_t       *size    = NULL;
-        dict_t        *newdict = NULL;
-        quota_local_t *local   = NULL;
-
-        local = (quota_local_t *) frame->local;
-
-        if (op_ret == -1)
-                goto err;
-
-        if (!dict)
-                goto wind;
-
-        ret = dict_get_bin (dict, QUOTA_SIZE_KEY, (void **) &size);
-        if (ret)
-                goto wind;
+    GF_VALIDATE_OR_GOTO("marker", ctx, out);
 
-        LOCK (&local->ctx->lock);
-        {
-                local->ctx->size = ntoh64 (*size);
-        }
-        UNLOCK (&local->ctx->lock);
-
-wind:
-        newdict = dict_new ();
-        if (!newdict)
-                goto err;
-
-        ret = dict_set_int8 (newdict, QUOTA_DIRTY_KEY, 0);
-        if (ret)
-                goto err;
-
-        if (gf_uuid_is_null (local->loc.gfid))
-                gf_uuid_copy (local->loc.gfid, local->loc.inode->gfid);
-
-        GF_UUID_ASSERT (local->loc.gfid);
-        STACK_WIND (frame, mq_release_lock_on_dirty_inode,
-                    FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->setxattr,
-                    &local->loc, newdict, 0, NULL);
-        ret = 0;
-
-err:
-        if (op_ret == -1 || ret == -1) {
-                local->err = -1;
-
-                mq_release_lock_on_dirty_inode (frame, NULL, this, 0, 0, NULL);
-        }
-
-        if (newdict)
-                dict_unref (newdict);
-
-        return 0;
+    mq_set_ctx_status(ctx, &ctx->create_status, status);
+    return 0;
+out:
+    return -1;
 }
 
 int32_t
-mq_update_size_xattr (call_frame_t *frame, void *cookie, xlator_t *this,
-                      int32_t op_ret, int32_t op_errno, inode_t *inode,
-                      struct iatt *buf, dict_t *dict, struct iatt *postparent)
+mq_test_and_set_ctx_create_status(quota_inode_ctx_t *ctx, gf_boolean_t *status)
 {
-        int32_t          ret      = -1;
-        dict_t          *new_dict = NULL;
-        int64_t         *size     = NULL;
-        int64_t         *delta    = NULL;
-        quota_local_t   *local    = NULL;
-
-        local = frame->local;
-
-        if (op_ret == -1)
-                goto err;
-
-        if (dict == NULL) {
-                gf_log (this->name, GF_LOG_WARNING,
-                        "Dict is null while updating the size xattr %s",
-                        local->loc.path?local->loc.path:"");
-                goto err;
-        }
-
-        ret = dict_get_bin (dict, QUOTA_SIZE_KEY, (void **) &size);
-        if (!size) {
-                gf_log (this->name, GF_LOG_WARNING,
-                        "failed to get the size, %s",
-                        local->loc.path?local->loc.path:"");
-                goto err;
-        }
-
-        QUOTA_ALLOC_OR_GOTO (delta, int64_t, ret, err);
+    GF_VALIDATE_OR_GOTO("marker", ctx, out);
+    GF_VALIDATE_OR_GOTO("marker", status, out);
 
-        *delta = hton64 (local->sum - ntoh64 (*size));
-
-        gf_log (this->name, GF_LOG_DEBUG, "calculated size = %"PRId64", "
-                "original size = %"PRIu64
-                " path = %s diff = %"PRIu64, local->sum, ntoh64 (*size),
-                local->loc.path, ntoh64 (*delta));
-
-        new_dict = dict_new ();
-        if (!new_dict) {
-		errno = ENOMEM;
-		goto err;
-	}
-
-        ret = dict_set_bin (new_dict, QUOTA_SIZE_KEY, delta, 8);
-        if (ret)
-                goto err;
-
-        if (gf_uuid_is_null (local->loc.gfid))
-                gf_uuid_copy (local->loc.gfid, buf->ia_gfid);
-
-        GF_UUID_ASSERT (local->loc.gfid);
-
-        STACK_WIND (frame, mq_mark_inode_undirty, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->xattrop, &local->loc,
-                    GF_XATTROP_ADD_ARRAY64, new_dict, NULL);
-
-        ret = 0;
-
-err:
-        if (op_ret == -1 || ret == -1) {
-                local->err = -1;
-                mq_release_lock_on_dirty_inode (frame, NULL, this, 0, 0, NULL);
-        }
-
-        if (new_dict)
-                dict_unref (new_dict);
-
-        return 0;
+    mq_test_and_set_ctx_status(ctx, &ctx->create_status, status);
+    return 0;
+out:
+    return -1;
 }
 
-int32_t
-mq_test_and_set_local_err(quota_local_t *local,
-                          int32_t *val)
+static void
+mq_set_ctx_dirty_status(quota_inode_ctx_t *ctx, gf_boolean_t status)
 {
-        int     tmp = 0;
-        int32_t ret = -1;
-
-        GF_VALIDATE_OR_GOTO ("marker", local, out);
-        GF_VALIDATE_OR_GOTO ("marker", val, out);
+    GF_VALIDATE_OR_GOTO("marker", ctx, out);
 
-        LOCK (&local->lock);
-        {
-                tmp = local->err;
-                local->err = *val;
-                *val = tmp;
-        }
-        UNLOCK (&local->lock);
-
-        ret = 0;
+    mq_set_ctx_status(ctx, &ctx->dirty_status, status);
 out:
-        return ret;
+    return;
 }
 
-int32_t
-mq_get_dirty_inode_size (call_frame_t *frame, xlator_t *this)
-{
-        int32_t        ret   = -1;
-        dict_t        *dict  = NULL;
-        quota_local_t *local = NULL;
-
-        local = (quota_local_t *) frame->local;
-
-        dict = dict_new ();
-        if (!dict) {
-                ret = -1;
-                goto err;
-        }
-
-        ret = dict_set_int64 (dict, QUOTA_SIZE_KEY, 0);
-        if (ret)
-                goto err;
-
-        if (gf_uuid_is_null (local->loc.gfid))
-                gf_uuid_copy (local->loc.gfid, local->loc.inode->gfid);
-
-        GF_UUID_ASSERT (local->loc.gfid);
+int
+mq_build_ancestry(xlator_t *this, loc_t *loc)
+{
+    int32_t ret = -1;
+    fd_t *fd = NULL;
+    gf_dirent_t entries;
+    gf_dirent_t *entry = NULL;
+    dict_t *xdata = NULL;
+    inode_t *tmp_parent = NULL;
+    inode_t *tmp_inode = NULL;
+    inode_t *linked_inode = NULL;
+    quota_inode_ctx_t *ctx = NULL;
+
+    INIT_LIST_HEAD(&entries.list);
+
+    xdata = dict_new();
+    if (xdata == NULL) {
+        gf_log(this->name, GF_LOG_ERROR, "dict_new failed");
+        ret = -ENOMEM;
+        goto out;
+    }
+
+    ret = dict_set_int8(xdata, GET_ANCESTRY_DENTRY_KEY, 1);
+    if (ret < 0)
+        goto out;
+
+    fd = fd_anonymous(loc->inode);
+    if (fd == NULL) {
+        gf_log(this->name, GF_LOG_ERROR, "fd creation failed");
+        ret = -ENOMEM;
+        goto out;
+    }
+
+    fd_bind(fd);
+
+    ret = syncop_readdirp(this, fd, 131072, 0, &entries, xdata, NULL);
+    if (ret < 0) {
+        gf_log(this->name,
+               (-ret == ENOENT || -ret == ESTALE) ? GF_LOG_DEBUG : GF_LOG_ERROR,
+               "readdirp failed "
+               "for %s: %s",
+               loc->path, strerror(-ret));
+        goto out;
+    }
+
+    if (list_empty(&entries.list)) {
+        ret = -1;
+        goto out;
+    }
+
+    list_for_each_entry(entry, &entries.list, list)
+    {
+        if (__is_root_gfid(entry->inode->gfid)) {
+            /* The list contains a sub-list for each possible path
+             * to the target inode. Each sub-list starts with the
+             * root entry of the tree and is followed by the child
+             * entries for a particular path to the target entry.
+             * The root entry is an implied sub-list delimiter,
+             * as it denotes we have started processing a new path.
+             * Reset the parent pointer and continue
+             */
+
+            tmp_parent = NULL;
+        } else {
+            linked_inode = inode_link(entry->inode, tmp_parent, entry->d_name,
+                                      &entry->d_stat);
+            if (linked_inode) {
+                tmp_inode = entry->inode;
+                entry->inode = linked_inode;
+                inode_unref(tmp_inode);
+            } else {
+                gf_log(this->name, GF_LOG_ERROR, "inode link failed");
+                ret = -EINVAL;
+                goto out;
+            }
+        }
+
+        ctx = mq_inode_ctx_new(entry->inode, this);
+        if (ctx == NULL) {
+            gf_log(this->name, GF_LOG_WARNING,
+                   "mq_inode_ctx_new "
+                   "failed for %s",
+                   uuid_utoa(entry->inode->gfid));
+            ret = -ENOMEM;
+            goto out;
+        }
+
+        /* For non-directory, posix_get_ancestry_non_directory returns
+         * all hard-links that are represented by nodes adjacent to
+         * each other in the dentry-list.
+         * (Unlike the directory case where adjacent nodes either have
+         *  a parent/child relationship or belong to different paths).
+         */
+        if (entry->inode->ia_type == IA_IFDIR)
+            tmp_parent = entry->inode;
+    }
+
+    if (loc->parent)
+        inode_unref(loc->parent);
+
+    loc->parent = inode_parent(loc->inode, 0, NULL);
+    if (loc->parent == NULL) {
+        ret = -1;
+        goto out;
+    }
+
+    ret = 0;
 
-        STACK_WIND (frame, mq_update_size_xattr, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->lookup, &local->loc, dict);
-        ret =0;
+out:
+    gf_dirent_free(&entries);
 
-err:
-        if (ret) {
-                local->err = -1;
+    if (fd)
+        fd_unref(fd);
 
-                mq_release_lock_on_dirty_inode (frame, NULL, this, 0, 0, NULL);
-        }
+    if (xdata)
+        dict_unref(xdata);
 
-        if (dict)
-                dict_unref (dict);
-
-        return 0;
+    return ret;
 }
 
-int32_t
-mq_get_child_contribution (call_frame_t *frame,
-                           void *cookie,
-                           xlator_t *this,
-                           int32_t op_ret,
-                           int32_t op_errno,
-                           inode_t *inode,
-                           struct iatt *buf,
-                           dict_t *dict,
-                           struct iatt *postparent)
-{
-        int32_t        ret                           = -1;
-        int32_t        val                           = 0;
-        char           contri_key[CONTRI_KEY_MAX]    = {0, };
-        int64_t       *contri                        = NULL;
-        quota_local_t *local                         = NULL;
-
-        local = frame->local;
-
-        frame->local = NULL;
-
-        QUOTA_STACK_DESTROY (frame, this);
-
-        if (op_ret == -1) {
-                gf_log (this->name, GF_LOG_ERROR, "%s",
-                        strerror (op_errno));
-                val = -2;
-                if (!mq_test_and_set_local_err (local, &val) &&
-                    val != -2)
-                        mq_release_lock_on_dirty_inode (local->frame, NULL,
-                                                        this, 0, 0, NULL);
-
-                goto exit;
-        }
-
-        ret = mq_get_local_err (local, &val);
-        if (!ret && val == -2)
-                goto exit;
-
-        GET_CONTRI_KEY (contri_key, local->loc.inode->gfid, ret);
-        if (ret < 0)
-                goto out;
-
-        if (!dict)
-                goto out;
-
-        if (dict_get_bin (dict, contri_key, (void **) &contri) == 0)
-                local->sum += ntoh64 (*contri);
-
+/* This function should be used only in inspect_directory and inspect_file
+ * function to heal quota xattrs.
+ * Inode quota feature is introduced in 3.7.
+ * If gluster setup is upgraded from 3.6 to 3.7, there can be a
+ * getxattr and setxattr spikes with quota heal as inode quota is missing.
+ * So this wrapper function is to avoid xattrs spikes during upgrade.
+ * This function returns success even is inode-quota xattrs are missing and
+ * hence no healing performed.
+ */
+static int32_t
+_quota_dict_get_meta(xlator_t *this, dict_t *dict, char *key, const int keylen,
+                     quota_meta_t *meta, ia_type_t ia_type,
+                     gf_boolean_t add_delta)
+{
+    int32_t ret = 0;
+    marker_conf_t *priv = NULL;
+
+    priv = this->private;
+
+    ret = quota_dict_get_inode_meta(dict, key, keylen, meta);
+    if (ret == -2 && (priv->feature_enabled & GF_INODE_QUOTA) == 0) {
+        /* quota_dict_get_inode_meta returns -2 if
+         * inode quota xattrs are not present.
+         * if inode quota self heal is turned off,
+         * then we should skip healing inode quotas
+         */
+
+        gf_log(this->name, GF_LOG_DEBUG,
+               "inode quota disabled. "
+               "inode quota self heal will not be performed");
+        ret = 0;
+        if (add_delta) {
+            if (ia_type == IA_IFDIR)
+                meta->dir_count = 1;
+            else
+                meta->file_count = 1;
+        }
+    }
+
+    return ret;
+}
+
+int32_t
+quota_dict_set_size_meta(xlator_t *this, dict_t *dict, const quota_meta_t *meta)
+{
+    int32_t ret = -ENOMEM;
+    quota_meta_t *value = NULL;
+    char size_key[QUOTA_KEY_MAX] = {
+        0,
+    };
+
+    value = GF_MALLOC(2 * sizeof(quota_meta_t), gf_common_quota_meta_t);
+    if (value == NULL) {
+        goto out;
+    }
+    value[0].size = hton64(meta->size);
+    value[0].file_count = hton64(meta->file_count);
+    value[0].dir_count = hton64(meta->dir_count);
+
+    value[1].size = 0;
+    value[1].file_count = 0;
+    value[1].dir_count = hton64(1);
+
+    GET_SIZE_KEY(this, size_key, ret);
+    if (ret < 0)
+        goto out;
+    ret = dict_set_bin(dict, size_key, value, (sizeof(quota_meta_t) * 2));
+    if (ret < 0) {
+        gf_log_callingfn("quota", GF_LOG_ERROR, "dict set failed");
+        GF_FREE(value);
+    }
 out:
-        LOCK (&local->lock);
-        {
-                val = --local->dentry_child_count;
-        }
-        UNLOCK (&local->lock);
-
-        if (val == 0) {
-                mq_dirty_inode_readdir (local->frame, NULL, this,
-                                        0, 0, NULL, NULL);
-        }
-        mq_local_unref (this, local);
-
-        return 0;
-exit:
-        mq_local_unref (this, local);
-        return 0;
+    return ret;
 }
 
-int32_t
-mq_readdir_cbk (call_frame_t *frame,
-                void *cookie,
-                xlator_t *this,
-                int32_t op_ret,
-                int32_t op_errno,
-                gf_dirent_t *entries, dict_t *xdata)
+void
+mq_compute_delta(quota_meta_t *delta, const quota_meta_t *op1,
+                 const quota_meta_t *op2)
 {
-        char           contri_key[CONTRI_KEY_MAX]    = {0, };
-        int32_t        ret                           = 0;
-        int32_t        val                           = 0;
-        off_t          offset                        = 0;
-        int32_t        count                         = 0;
-        dict_t        *dict                          = NULL;
-        quota_local_t *local                         = NULL;
-        gf_dirent_t   *entry                         = NULL;
-        call_frame_t  *newframe                      = NULL;
-        loc_t          loc                           = {0, };
-
-        local = mq_local_ref (frame->local);
-
-        if (op_ret == -1) {
-                gf_log (this->name, GF_LOG_TRACE,
-                        "readdir failed %s", strerror (op_errno));
-                local->err = -1;
-
-                mq_release_lock_on_dirty_inode (frame, NULL, this, 0, 0, NULL);
-
-                goto end;
-        } else if (op_ret == 0) {
-                mq_get_dirty_inode_size (frame, this);
-
-                goto end;
-        }
-
-        local->dentry_child_count =  0;
-
-        list_for_each_entry (entry, (&entries->list), list) {
-                gf_log (this->name, GF_LOG_DEBUG, "entry  = %s", entry->d_name);
-
-                if ((!strcmp (entry->d_name, ".")) || (!strcmp (entry->d_name,
-                                                                ".."))) {
-                        gf_log (this->name, GF_LOG_DEBUG, "entry  = %s",
-                                entry->d_name);
-                        continue;
-                }
-
-                offset = entry->d_off;
-                count++;
-        }
-
-        if (count == 0) {
-                mq_get_dirty_inode_size (frame, this);
-                goto end;
-
-        }
-
-        local->frame = frame;
-
-        LOCK (&local->lock);
-        {
-                local->dentry_child_count = count;
-                local->d_off = offset;
-        }
-        UNLOCK (&local->lock);
-
-
-        list_for_each_entry (entry, (&entries->list), list) {
-                gf_log (this->name, GF_LOG_DEBUG, "entry  = %s", entry->d_name);
-
-                if ((!strcmp (entry->d_name, ".")) || (!strcmp (entry->d_name,
-                                                                ".."))) {
-                        gf_log (this->name, GF_LOG_DEBUG, "entry  = %s",
-                                entry->d_name);
-                        continue;
-                }
-
-                ret = mq_loc_fill_from_name (this, &loc, &local->loc,
-                                             entry->d_ino, entry->d_name);
-                if (ret < 0) {
-                        gf_log (this->name, GF_LOG_WARNING, "Couldn't build "
-                                "loc for %s/%s, returning from updation of "
-                                "quota attributes",
-                                uuid_utoa (local->loc.inode->gfid),
-                                entry->d_name);
-                        goto out;
-                }
-
-                ret = 0;
-
-                LOCK (&local->lock);
-                {
-                        if (local->err != -2) {
-                                newframe = copy_frame (frame);
-                                if (!newframe) {
-                                        ret = -1;
-                                }
-                        } else
-                                ret = -1;
-                }
-                UNLOCK (&local->lock);
-
-                if (ret == -1)
-                        goto out;
-
-                newframe->local = mq_local_ref (local);
-
-                dict = dict_new ();
-                if (!dict) {
-                        ret = -1;
-                        goto out;
-                }
-
-                GET_CONTRI_KEY (contri_key, local->loc.inode->gfid, ret);
-                if (ret < 0)
-                        goto out;
-
-                ret = dict_set_int64 (dict, contri_key, 0);
-                if (ret)
-                        goto out;
-
-                STACK_WIND (newframe,
-                            mq_get_child_contribution,
-                            FIRST_CHILD(this),
-                            FIRST_CHILD(this)->fops->lookup,
-                            &loc, dict);
-
-                offset = entry->d_off;
-
-                loc_wipe (&loc);
-
-                newframe = NULL;
-
-        out:
-                if (dict) {
-                        dict_unref (dict);
-                        dict = NULL;
-                }
-
-                if (ret) {
-                        val = -2;
-                        mq_test_and_set_local_err (local, &val);
-
-                        if (newframe) {
-                                newframe->local = NULL;
-                                mq_local_unref(this, local);
-                                QUOTA_STACK_DESTROY (newframe, this);
-                        }
-
-                        break;
-                }
-        }
-
-        if (ret && val != -2) {
-                mq_release_lock_on_dirty_inode (frame, NULL, this, 0, 0, NULL);
-        }
-end:
-        mq_local_unref (this, local);
-
-        return 0;
+    delta->size = op1->size - op2->size;
+    delta->file_count = op1->file_count - op2->file_count;
+    delta->dir_count = op1->dir_count - op2->dir_count;
 }
 
-int32_t
-mq_dirty_inode_readdir (call_frame_t *frame,
-                        void *cookie,
-                        xlator_t *this,
-                        int32_t op_ret,
-                        int32_t op_errno,
-                        fd_t *fd, dict_t *xdata)
+void
+mq_add_meta(quota_meta_t *dst, const quota_meta_t *src)
 {
-        quota_local_t *local = NULL;
-
-        local = frame->local;
-
-        if (op_ret == -1) {
-                local->err = -1;
-                mq_release_lock_on_dirty_inode (frame, NULL, this, 0, 0, NULL);
-                return 0;
-        }
-
-        if (local->fd == NULL)
-                local->fd = fd_ref (fd);
-
-        STACK_WIND (frame,
-                    mq_readdir_cbk,
-                    FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->readdir,
-                    local->fd, READDIR_BUF, local->d_off, xdata);
-
-        return 0;
+    dst->size += src->size;
+    dst->file_count += src->file_count;
+    dst->dir_count += src->dir_count;
 }
 
-int32_t
-mq_check_if_still_dirty (call_frame_t *frame,
-                         void *cookie,
-                         xlator_t *this,
-                         int32_t op_ret,
-                         int32_t op_errno,
-                         inode_t *inode,
-                         struct iatt *buf,
-                         dict_t *dict,
-                         struct iatt *postparent)
-{
-        int8_t           dirty          = -1;
-        int32_t          ret            = -1;
-        fd_t            *fd             = NULL;
-        quota_local_t   *local          = NULL;
-
-        local = frame->local;
-
-        if (op_ret == -1) {
-                gf_log (this->name, GF_LOG_ERROR, "failed to get "
-                        "the dirty xattr for %s", local->loc.path);
-                goto err;
-        }
-
-        if (!dict) {
-                ret = -1;
-                goto err;
-        }
-
-        ret = dict_get_int8 (dict, QUOTA_DIRTY_KEY, &dirty);
-        if (ret)
-                goto err;
-
-        //the inode is not dirty anymore
-        if (dirty == 0) {
-                mq_release_lock_on_dirty_inode (frame, NULL, this, 0, 0, NULL);
-
-                return 0;
-        }
-
-        fd = fd_create (local->loc.inode, frame->root->pid);
-
-        local->d_off = 0;
-
-        if (gf_uuid_is_null (local->loc.gfid))
-                gf_uuid_copy (local->loc.gfid, buf->ia_gfid);
-
-        GF_UUID_ASSERT (local->loc.gfid);
-        STACK_WIND(frame,
-                   mq_dirty_inode_readdir,
-                   FIRST_CHILD(this),
-                   FIRST_CHILD(this)->fops->opendir,
-                   &local->loc, fd, NULL);
+void
+mq_sub_meta(quota_meta_t *dst, const quota_meta_t *src)
+{
+    if (src == NULL) {
+        dst->size = -dst->size;
+        dst->file_count = -dst->file_count;
+        dst->dir_count = -dst->dir_count;
+    } else {
+        dst->size = src->size - dst->size;
+        dst->file_count = src->file_count - dst->file_count;
+        dst->dir_count = src->dir_count - dst->dir_count;
+    }
+}
+
+int32_t
+mq_are_xattrs_set(xlator_t *this, loc_t *loc, gf_boolean_t *contri_set,
+                  gf_boolean_t *size_set)
+{
+    int32_t ret = -1;
+    char contri_key[QUOTA_KEY_MAX] = {
+        0,
+    };
+    char size_key[QUOTA_KEY_MAX] = {
+        0,
+    };
+    quota_meta_t meta = {
+        0,
+    };
+    struct iatt stbuf = {
+        0,
+    };
+    dict_t *dict = NULL;
+    dict_t *rsp_dict = NULL;
+
+    dict = dict_new();
+    if (dict == NULL) {
+        gf_log(this->name, GF_LOG_ERROR, "dict_new failed");
+        goto out;
+    }
+
+    ret = mq_req_xattr(this, loc, dict, contri_key, size_key);
+    if (ret < 0)
+        goto out;
+
+    ret = syncop_lookup(FIRST_CHILD(this), loc, &stbuf, NULL, dict, &rsp_dict);
+    if (ret < 0) {
+        gf_log_callingfn(
+            this->name,
+            (-ret == ENOENT || -ret == ESTALE) ? GF_LOG_DEBUG : GF_LOG_ERROR,
+            "lookup failed "
+            "for %s: %s",
+            loc->path, strerror(-ret));
+        goto out;
+    }
+
+    if (rsp_dict == NULL)
+        goto out;
+
+    *contri_set = _gf_true;
+    *size_set = _gf_true;
+    if (loc->inode->ia_type == IA_IFDIR) {
+        ret = quota_dict_get_inode_meta(rsp_dict, size_key, strlen(size_key),
+                                        &meta);
+        if (ret < 0 || meta.dir_count == 0)
+            *size_set = _gf_false;
+    }
+
+    if (!loc_is_root(loc)) {
+        ret = quota_dict_get_inode_meta(rsp_dict, contri_key,
+                                        strlen(contri_key), &meta);
+        if (ret < 0)
+            *contri_set = _gf_false;
+    }
 
-        ret = 0;
+    ret = 0;
+out:
+    if (dict)
+        dict_unref(dict);
 
-err:
-        if (op_ret == -1 || ret == -1) {
-                local->err = -1;
-                mq_release_lock_on_dirty_inode (frame, NULL, this, 0, 0, NULL);
-        }
+    if (rsp_dict)
+        dict_unref(rsp_dict);
 
-        if (fd != NULL) {
-                fd_unref (fd);
-        }
-
-        return 0;
+    return ret;
 }
 
 int32_t
-mq_get_dirty_xattr (call_frame_t *frame, void *cookie, xlator_t *this,
-                    int32_t op_ret, int32_t op_errno, dict_t *xdata)
+mq_create_size_xattrs(xlator_t *this, quota_inode_ctx_t *ctx, loc_t *loc)
 {
-        int32_t        ret       = -1;
-        dict_t        *xattr_req = NULL;
-        quota_local_t *local     = NULL;
-
-        if (op_ret == -1) {
-                mq_dirty_inode_updation_done (frame, NULL, this, 0, 0, NULL);
-                return 0;
-        }
-
-        local = frame->local;
-
-        xattr_req = dict_new ();
-        if (xattr_req == NULL) {
-                ret = -1;
-                goto err;
-        }
-
-        ret = dict_set_int8 (xattr_req, QUOTA_DIRTY_KEY, 0);
-        if (ret)
-                goto err;
+    int32_t ret = -1;
+    quota_meta_t size = {
+        0,
+    };
+    dict_t *dict = NULL;
 
-        if (gf_uuid_is_null (local->loc.gfid))
-                gf_uuid_copy (local->loc.gfid, local->loc.inode->gfid);
+    GF_VALIDATE_OR_GOTO("marker", loc, out);
+    GF_VALIDATE_OR_GOTO("marker", loc->inode, out);
 
-        if (gf_uuid_is_null (local->loc.gfid)) {
-                ret = -1;
-                goto err;
-        }
-
-        STACK_WIND (frame,
-                    mq_check_if_still_dirty,
-                    FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->lookup,
-                    &local->loc,
-                    xattr_req);
+    if (loc->inode->ia_type != IA_IFDIR) {
         ret = 0;
+        goto out;
+    }
+
+    dict = dict_new();
+    if (!dict) {
+        gf_log(this->name, GF_LOG_ERROR, "dict_new failed");
+        ret = -1;
+        goto out;
+    }
+
+    ret = quota_dict_set_size_meta(this, dict, &size);
+    if (ret < 0)
+        goto out;
+
+    ret = syncop_xattrop(FIRST_CHILD(this), loc,
+                         GF_XATTROP_ADD_ARRAY64_WITH_DEFAULT, dict, NULL, NULL,
+                         NULL);
+
+    if (ret < 0) {
+        gf_log_callingfn(
+            this->name,
+            (-ret == ENOENT || -ret == ESTALE) ? GF_LOG_DEBUG : GF_LOG_ERROR,
+            "xattrop failed "
+            "for %s: %s",
+            loc->path, strerror(-ret));
+        goto out;
+    }
 
-err:
-        if (ret) {
-                local->err = -1;
-                mq_release_lock_on_dirty_inode(frame, NULL, this, 0, 0, NULL);
-        }
-
-        if (xattr_req)
-                dict_unref (xattr_req);
+out:
+    if (dict)
+        dict_unref(dict);
 
-        return 0;
+    return ret;
 }
 
-/* return 1 when dirty updation started
- * 0 other wise
- */
 int32_t
-mq_update_dirty_inode (xlator_t *this, loc_t *loc, quota_inode_ctx_t *ctx,
-                       inode_contribution_t *contribution)
+mq_lock(xlator_t *this, loc_t *loc, short l_type)
 {
-        int32_t          ret        = -1;
-        quota_local_t   *local      = NULL;
-        gf_boolean_t     status     = _gf_false;
-        struct gf_flock  lock       = {0, };
-        call_frame_t    *frame      = NULL;
-
-        ret = mq_get_ctx_updation_status (ctx, &status);
-        if (ret == -1 || status == _gf_true) {
-                ret = 0;
-                goto out;
-        }
-
-        frame = create_frame (this, this->ctx->pool);
-        if (frame == NULL) {
-                ret = -1;
-                goto out;
-        }
-
-        mq_assign_lk_owner (this, frame);
+    struct gf_flock lock = {
+        0,
+    };
+    int32_t ret = -1;
 
-        local = mq_local_new ();
-        if (local == NULL)
-                goto fr_destroy;
+    GF_VALIDATE_OR_GOTO("marker", loc, out);
+    GF_VALIDATE_OR_GOTO("marker", loc->inode, out);
 
-        frame->local = local;
-        ret = mq_loc_copy (&local->loc, loc);
-        if (ret < 0)
-                goto fr_destroy;
-
-        local->ctx = ctx;
-
-        local->contri = contribution;
+    gf_log(this->name, GF_LOG_DEBUG, "set lock type %d on %s", l_type,
+           loc->path);
 
-        lock.l_type = F_WRLCK;
-        lock.l_whence = SEEK_SET;
-        lock.l_start = 0;
-        lock.l_len = 0;
+    lock.l_len = 0;
+    lock.l_start = 0;
+    lock.l_type = l_type;
+    lock.l_whence = SEEK_SET;
 
-        if (local->loc.inode == NULL) {
-                ret = -1;
-                gf_log (this->name, GF_LOG_WARNING,
-                        "Inode is NULL, so can't stackwind.");
-                goto fr_destroy;
-        }
+    ret = syncop_inodelk(FIRST_CHILD(this), this->name, loc, F_SETLKW, &lock,
+                         NULL, NULL);
+    if (ret < 0)
+        gf_log_callingfn(
+            this->name,
+            (-ret == ENOENT || -ret == ESTALE) ? GF_LOG_DEBUG : GF_LOG_ERROR,
+            "inodelk failed "
+            "for %s: %s",
+            loc->path, strerror(-ret));
 
-        STACK_WIND (frame,
-                    mq_get_dirty_xattr,
-                    FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->inodelk,
-                    this->name, &local->loc, F_SETLKW, &lock, NULL);
-        return 1;
-
-fr_destroy:
-        QUOTA_STACK_DESTROY (frame, this);
 out:
 
-        return 0;
+    return ret;
 }
 
-
 int32_t
-mq_inode_creation_done (call_frame_t *frame, void *cookie, xlator_t *this,
-                        int32_t op_ret, int32_t op_errno, dict_t *xdata)
+mq_get_dirty(xlator_t *this, loc_t *loc, int32_t *dirty)
 {
-        quota_local_t *local = NULL;
+    int32_t ret = -1;
+    int8_t value = 0;
+    dict_t *dict = NULL;
+    dict_t *rsp_dict = NULL;
+    struct iatt stbuf = {
+        0,
+    };
 
-        if (frame == NULL)
-                return 0;
+    dict = dict_new();
+    if (dict == NULL) {
+        gf_log(this->name, GF_LOG_ERROR, "dict_new failed");
+        goto out;
+    }
 
-        local = frame->local;
+    ret = dict_set_int64(dict, QUOTA_DIRTY_KEY, 0);
+    if (ret < 0) {
+        gf_log(this->name, GF_LOG_WARNING, "dict set failed");
+        goto out;
+    }
 
-        if (local != NULL) {
-                mq_initiate_quota_txn (this, &local->loc);
-        }
+    ret = syncop_lookup(FIRST_CHILD(this), loc, &stbuf, NULL, dict, &rsp_dict);
+    if (ret < 0) {
+        gf_log_callingfn(
+            this->name,
+            (-ret == ENOENT || -ret == ESTALE) ? GF_LOG_DEBUG : GF_LOG_ERROR,
+            "lookup failed "
+            "for %s: %s",
+            loc->path, strerror(-ret));
+        goto out;
+    }
 
-        QUOTA_STACK_DESTROY (frame, this);
+    ret = dict_get_int8(rsp_dict, QUOTA_DIRTY_KEY, &value);
+    if (ret < 0)
+        goto out;
 
-        return 0;
-}
+    *dirty = value;
 
+out:
+    if (dict)
+        dict_unref(dict);
+
+    if (rsp_dict)
+        dict_unref(rsp_dict);
+
+    return ret;
+}
+
+int32_t
+mq_get_set_dirty(xlator_t *this, loc_t *loc, int32_t dirty, int32_t *prev_dirty)
+{
+    int32_t ret = -1;
+    int8_t value = 0;
+    quota_inode_ctx_t *ctx = NULL;
+    dict_t *dict = NULL;
+    dict_t *rsp_dict = NULL;
+
+    GF_VALIDATE_OR_GOTO("marker", loc, out);
+    GF_VALIDATE_OR_GOTO("marker", loc->inode, out);
+    GF_VALIDATE_OR_GOTO("marker", prev_dirty, out);
+
+    ret = mq_inode_ctx_get(loc->inode, this, &ctx);
+    if (ret < 0) {
+        gf_log(this->name, GF_LOG_ERROR,
+               "failed to get inode ctx for "
+               "%s",
+               loc->path);
+        goto out;
+    }
+
+    dict = dict_new();
+    if (!dict) {
+        gf_log(this->name, GF_LOG_ERROR, "dict_new failed");
+        ret = -1;
+        goto out;
+    }
+
+    ret = dict_set_int8(dict, QUOTA_DIRTY_KEY, dirty);
+    if (ret < 0) {
+        gf_log(this->name, GF_LOG_ERROR, "dict_set failed");
+        goto out;
+    }
+
+    ret = syncop_xattrop(FIRST_CHILD(this), loc, GF_XATTROP_GET_AND_SET, dict,
+                         NULL, NULL, &rsp_dict);
+    if (ret < 0) {
+        gf_log_callingfn(
+            this->name,
+            (-ret == ENOENT || -ret == ESTALE) ? GF_LOG_DEBUG : GF_LOG_ERROR,
+            "xattrop failed "
+            "for %s: %s",
+            loc->path, strerror(-ret));
+        goto out;
+    }
+
+    *prev_dirty = 0;
+    if (rsp_dict) {
+        ret = dict_get_int8(rsp_dict, QUOTA_DIRTY_KEY, &value);
+        if (ret == 0)
+            *prev_dirty = value;
+    }
+
+    LOCK(&ctx->lock);
+    {
+        ctx->dirty = dirty;
+    }
+    UNLOCK(&ctx->lock);
+    ret = 0;
+out:
+    if (dict)
+        dict_unref(dict);
 
-int32_t
-mq_xattr_creation_release_lock (call_frame_t *frame, void *cookie,
-                                xlator_t *this, int32_t op_ret,
-                                int32_t op_errno, dict_t *xdata)
-{
-        struct gf_flock  lock  = {0, };
-        quota_local_t   *local = NULL;
-
-        local = frame->local;
-
-        lock.l_type   = F_UNLCK;
-        lock.l_whence = SEEK_SET;
-        lock.l_start  = 0;
-        lock.l_len    = 0;
-        lock.l_pid    = 0;
-
-        STACK_WIND (frame,
-                    mq_inode_creation_done,
-                    FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->inodelk,
-                    this->name, &local->loc,
-                    F_SETLKW, &lock, NULL);
+    if (rsp_dict)
+        dict_unref(rsp_dict);
 
-        return 0;
+    return ret;
 }
 
-
 int32_t
-mq_create_dirty_xattr (call_frame_t *frame, void *cookie, xlator_t *this,
-                       int32_t op_ret, int32_t op_errno, dict_t *dict,
-                       dict_t *xdata)
+mq_mark_dirty(xlator_t *this, loc_t *loc, int32_t dirty)
 {
-        int32_t          ret       = -1;
-        dict_t          *newdict   = NULL;
-        quota_local_t   *local     = NULL;
+    int32_t ret = -1;
+    dict_t *dict = NULL;
+    quota_inode_ctx_t *ctx = NULL;
 
-        if (op_ret < 0) {
-                goto err;
-        }
-
-        local = frame->local;
-
-        if (local->loc.inode->ia_type == IA_IFDIR) {
-                newdict = dict_new ();
-                if (!newdict) {
-                        goto err;
-                }
-
-                ret = dict_set_int8 (newdict, QUOTA_DIRTY_KEY, 0);
-                if (ret == -1) {
-                        goto err;
-                }
-
-                gf_uuid_copy (local->loc.gfid, local->loc.inode->gfid);
-                GF_UUID_ASSERT (local->loc.gfid);
-
-                STACK_WIND (frame, mq_xattr_creation_release_lock,
-                            FIRST_CHILD(this),
-                            FIRST_CHILD(this)->fops->setxattr,
-                            &local->loc, newdict, 0, NULL);
-        } else {
-                mq_xattr_creation_release_lock (frame, NULL, this, 0, 0, NULL);
-        }
+    GF_VALIDATE_OR_GOTO("marker", loc, out);
+    GF_VALIDATE_OR_GOTO("marker", loc->inode, out);
 
+    ret = mq_inode_ctx_get(loc->inode, this, &ctx);
+    if (ret < 0) {
+        gf_log(this->name, GF_LOG_ERROR,
+               "failed to get inode ctx for "
+               "%s",
+               loc->path);
         ret = 0;
+        goto out;
+    }
+
+    dict = dict_new();
+    if (!dict) {
+        ret = -1;
+        gf_log(this->name, GF_LOG_ERROR, "dict_new failed");
+        goto out;
+    }
+
+    ret = dict_set_int8(dict, QUOTA_DIRTY_KEY, dirty);
+    if (ret < 0) {
+        gf_log(this->name, GF_LOG_ERROR, "dict_set failed");
+        goto out;
+    }
+
+    ret = syncop_setxattr(FIRST_CHILD(this), loc, dict, 0, NULL, NULL);
+    if (ret < 0) {
+        gf_log_callingfn(
+            this->name,
+            (-ret == ENOENT || -ret == ESTALE) ? GF_LOG_DEBUG : GF_LOG_ERROR,
+            "setxattr dirty = %d "
+            "failed for %s: %s",
+            dirty, loc->path, strerror(-ret));
+        goto out;
+    }
+
+    LOCK(&ctx->lock);
+    {
+        ctx->dirty = dirty;
+    }
+    UNLOCK(&ctx->lock);
 
-err:
-        if (ret < 0) {
-                mq_xattr_creation_release_lock (frame, NULL, this, 0, 0, NULL);
-        }
-
-        if (newdict != NULL)
-                dict_unref (newdict);
-
-        return 0;
-}
-
-
-int32_t
-mq_create_xattr (xlator_t *this, call_frame_t *frame)
-{
-        int32_t               ret                  = 0;
-        int64_t              *value                = NULL;
-        int64_t              *size                 = NULL;
-        dict_t               *dict                 = NULL;
-        char                  key[CONTRI_KEY_MAX]  = {0, };
-        quota_local_t        *local                = NULL;
-        quota_inode_ctx_t    *ctx                  = NULL;
-        inode_contribution_t *contri               = NULL;
-
-        if (frame == NULL || this == NULL)
-                return 0;
-
-        local = frame->local;
-
-        ret = mq_inode_ctx_get (local->loc.inode, this, &ctx);
+out:
+    if (dict)
+        dict_unref(dict);
+
+    return ret;
+}
+
+int32_t
+_mq_get_metadata(xlator_t *this, loc_t *loc, quota_meta_t *contri,
+                 quota_meta_t *size, uuid_t contri_gfid)
+{
+    int32_t ret = -1;
+    quota_meta_t meta = {
+        0,
+    };
+    char contri_key[QUOTA_KEY_MAX] = {
+        0,
+    };
+    char size_key[QUOTA_KEY_MAX] = {
+        0,
+    };
+    int keylen = 0;
+    dict_t *dict = NULL;
+    dict_t *rsp_dict = NULL;
+    struct iatt stbuf = {
+        0,
+    };
+
+    GF_VALIDATE_OR_GOTO("marker", loc, out);
+    GF_VALIDATE_OR_GOTO("marker", loc->inode, out);
+
+    if (size == NULL && contri == NULL)
+        goto out;
+
+    dict = dict_new();
+    if (dict == NULL) {
+        gf_log(this->name, GF_LOG_ERROR, "dict_new failed");
+        goto out;
+    }
+
+    if (size && loc->inode->ia_type == IA_IFDIR) {
+        GET_SIZE_KEY(this, size_key, keylen);
+        if (keylen < 0)
+            goto out;
+        ret = dict_set_int64(dict, size_key, 0);
         if (ret < 0) {
-                ctx = mq_inode_ctx_new (local->loc.inode, this);
-                if (ctx == NULL) {
-                        gf_log (this->name, GF_LOG_WARNING,
-                                "mq_inode_ctx_new failed");
-                        ret = -1;
-                        goto out;
-                }
-        }
-
-        dict = dict_new ();
-        if (!dict)
-                goto out;
-
-        if (local->loc.inode->ia_type == IA_IFDIR) {
-                QUOTA_ALLOC_OR_GOTO (size, int64_t, ret, err);
-                ret = dict_set_bin (dict, QUOTA_SIZE_KEY, size, 8);
-                if (ret < 0)
-                        goto free_size;
+            gf_log(this->name, GF_LOG_ERROR, "dict_set failed.");
+            goto out;
         }
+    }
 
-        if ((local->loc.path && strcmp (local->loc.path, "/") != 0)
-            || (local->loc.inode && !gf_uuid_is_null (local->loc.inode->gfid) &&
-                !__is_root_gfid (local->loc.inode->gfid))
-            || (!gf_uuid_is_null (local->loc.gfid)
-                && !__is_root_gfid (local->loc.gfid))) {
-                contri = mq_add_new_contribution_node (this, ctx, &local->loc);
-                if (contri == NULL)
-                        goto err;
-
-                QUOTA_ALLOC_OR_GOTO (value, int64_t, ret, err);
-                GET_CONTRI_KEY (key, local->loc.parent->gfid, ret);
-
-                ret = dict_set_bin (dict, key, value, 8);
-                if (ret < 0)
-                        goto free_value;
-        }
-
-        if (gf_uuid_is_null (local->loc.gfid)) {
-                ret = -1;
+    if (contri && !loc_is_root(loc)) {
+        ret = mq_dict_set_contribution(this, dict, loc, contri_gfid,
+                                       contri_key);
+        if (ret < 0)
+            goto out;
+    }
+
+    ret = syncop_lookup(FIRST_CHILD(this), loc, &stbuf, NULL, dict, &rsp_dict);
+    if (ret < 0) {
+        gf_log_callingfn(
+            this->name,
+            (-ret == ENOENT || -ret == ESTALE) ? GF_LOG_DEBUG : GF_LOG_ERROR,
+            "lookup failed "
+            "for %s: %s",
+            loc->path, strerror(-ret));
+        goto out;
+    }
+
+    if (size) {
+        if (loc->inode->ia_type == IA_IFDIR) {
+            ret = quota_dict_get_meta(rsp_dict, size_key, keylen, &meta);
+            if (ret < 0) {
+                gf_log(this->name, GF_LOG_ERROR, "dict_get failed.");
                 goto out;
-        }
-
-        STACK_WIND (frame, mq_create_dirty_xattr, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->xattrop, &local->loc,
-                    GF_XATTROP_ADD_ARRAY64, dict, NULL);
-        ret = 0;
+            }
 
-free_size:
-        if (ret < 0) {
-                GF_FREE (size);
+            size->size = meta.size;
+            size->file_count = meta.file_count;
+            size->dir_count = meta.dir_count;
+        } else {
+            size->size = stbuf.ia_blocks * 512;
+            size->file_count = 1;
+            size->dir_count = 0;
         }
+    }
 
-free_value:
+    if (contri && !loc_is_root(loc)) {
+        ret = quota_dict_get_meta(rsp_dict, contri_key, strlen(contri_key),
+                                  &meta);
         if (ret < 0) {
-                GF_FREE (value);
+            contri->size = 0;
+            contri->file_count = 0;
+            contri->dir_count = 0;
+        } else {
+            contri->size = meta.size;
+            contri->file_count = meta.file_count;
+            contri->dir_count = meta.dir_count;
         }
+    }
 
-err:
-        dict_unref (dict);
+    ret = 0;
 
 out:
-        if (ret < 0) {
-                mq_xattr_creation_release_lock (frame, NULL, this, 0, 0, NULL);
-        }
-
-        return 0;
-}
+    if (dict)
+        dict_unref(dict);
 
+    if (rsp_dict)
+        dict_unref(rsp_dict);
 
-int32_t
-mq_check_n_set_inode_xattr (call_frame_t *frame, void *cookie,
-                            xlator_t *this, int32_t op_ret, int32_t op_errno,
-                            inode_t *inode, struct iatt *buf, dict_t *dict,
-                            struct iatt *postparent)
-{
-        quota_local_t        *local                      = NULL;
-        int64_t              *size                       = NULL;
-        int64_t              *contri                     = NULL;
-        int8_t                dirty                      = 0;
-        int32_t               ret                        = 0;
-        char                  contri_key[CONTRI_KEY_MAX] = {0, };
-
-        if (op_ret < 0) {
-                goto out;
-        }
-
-        local = frame->local;
-
-        ret = dict_get_bin (dict, QUOTA_SIZE_KEY, (void **) &size);
-        if (ret < 0)
-                goto create_xattr;
-
-        ret = dict_get_int8 (dict, QUOTA_DIRTY_KEY, &dirty);
-        if (ret < 0)
-                goto create_xattr;
-
-        //check contribution xattr if not root
-        if ((local->loc.path && strcmp (local->loc.path, "/") != 0)
-            || (!gf_uuid_is_null (local->loc.gfid)
-                && !__is_root_gfid (local->loc.gfid))
-            || (local->loc.inode
-                && !gf_uuid_is_null (local->loc.inode->gfid)
-                && !__is_root_gfid (local->loc.inode->gfid))) {
-                GET_CONTRI_KEY (contri_key, local->loc.parent->gfid, ret);
-                if (ret < 0)
-                        goto out;
-
-                ret = dict_get_bin (dict, contri_key, (void **) &contri);
-                if (ret < 0)
-                        goto create_xattr;
-        }
-
-out:
-        mq_xattr_creation_release_lock (frame, NULL, this, 0, 0, NULL);
-        return 0;
-
-create_xattr:
-        if (gf_uuid_is_null (local->loc.gfid)) {
-                gf_uuid_copy (local->loc.gfid, buf->ia_gfid);
-        }
-
-        mq_create_xattr (this, frame);
-        return 0;
+    return ret;
 }
 
-
 int32_t
-mq_get_xattr (call_frame_t *frame, void *cookie, xlator_t *this,
-              int32_t op_ret, int32_t op_errno, dict_t *xdata)
+mq_get_metadata(xlator_t *this, loc_t *loc, quota_meta_t *contri,
+                quota_meta_t *size, quota_inode_ctx_t *ctx,
+                inode_contribution_t *contribution)
 {
-        dict_t        *xattr_req = NULL;
-        quota_local_t *local     = NULL;
-        int32_t        ret       = 0;
-
-        if (op_ret < 0) {
-                goto lock_err;
-        }
+    int32_t ret = -1;
 
-        local = frame->local;
+    GF_VALIDATE_OR_GOTO("marker", loc, out);
+    GF_VALIDATE_OR_GOTO("marker", loc->inode, out);
+    GF_VALIDATE_OR_GOTO("marker", ctx, out);
+    GF_VALIDATE_OR_GOTO("marker", contribution, out);
 
-        xattr_req = dict_new ();
-        if (xattr_req == NULL) {
-                goto err;
-        }
-
-        ret = mq_req_xattr (this, &local->loc, xattr_req, NULL);
-        if (ret < 0) {
-                gf_log (this->name, GF_LOG_WARNING, "cannot request xattr");
-                goto err;
-        }
-
-        if (gf_uuid_is_null (local->loc.gfid))
-                gf_uuid_copy (local->loc.gfid, local->loc.inode->gfid);
-
-        if (gf_uuid_is_null (local->loc.gfid)) {
-                ret = -1;
-                goto err;
-        }
-
-        STACK_WIND (frame, mq_check_n_set_inode_xattr, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->lookup, &local->loc, xattr_req);
-
-        dict_unref (xattr_req);
-
-        return 0;
-
-err:
-        mq_xattr_creation_release_lock (frame, NULL, this, 0, 0, NULL);
-
-        if (xattr_req)
-                dict_unref (xattr_req);
-        return 0;
-
-lock_err:
-        mq_inode_creation_done (frame, NULL, this, 0, 0, NULL);
-        return 0;
-}
-
-
-int32_t
-mq_set_inode_xattr (xlator_t *this, loc_t *loc)
-{
-        struct gf_flock  lock  = {0, };
-        quota_local_t   *local = NULL;
-        int32_t          ret   = 0;
-        call_frame_t    *frame = NULL;
+    if (size == NULL && contri == NULL) {
+        ret = 0;
+        goto out;
+    }
 
-        frame = create_frame (this, this->ctx->pool);
-        if (!frame) {
-                ret = -1;
-                goto err;
-        }
+    ret = _mq_get_metadata(this, loc, contri, size, contribution->gfid);
+    if (ret < 0)
+        goto out;
 
-        local = mq_local_new ();
-        if (local == NULL) {
-                goto err;
+    if (size) {
+        LOCK(&ctx->lock);
+        {
+            ctx->size = size->size;
+            ctx->file_count = size->file_count;
+            ctx->dir_count = size->dir_count;
         }
+        UNLOCK(&ctx->lock);
+    }
 
-        frame->local = local;
-
-        ret = loc_copy (&local->loc, loc);
-        if (ret < 0) {
-                goto err;
+    if (contri) {
+        LOCK(&contribution->lock);
+        {
+            contribution->contribution = contri->size;
+            contribution->file_count = contri->file_count;
+            contribution->dir_count = contri->dir_count;
         }
+        UNLOCK(&contribution->lock);
+    }
 
-        frame->local = local;
-
-        lock.l_len    = 0;
-        lock.l_start  = 0;
-        lock.l_type   = F_WRLCK;
-        lock.l_whence = SEEK_SET;
-
-        STACK_WIND (frame,
-                    mq_get_xattr,
-                    FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->inodelk,
-                    this->name, &local->loc, F_SETLKW, &lock, NULL);
-
-        return 0;
-
-err:
-        if (frame)
-                QUOTA_STACK_DESTROY (frame, this);
-
-        return 0;
+out:
+    return ret;
 }
 
-
 int32_t
-mq_get_parent_inode_local (xlator_t *this, quota_local_t *local)
+mq_get_delta(xlator_t *this, loc_t *loc, quota_meta_t *delta,
+             quota_inode_ctx_t *ctx, inode_contribution_t *contribution)
 {
-        int32_t            ret = -1;
-        quota_inode_ctx_t *ctx = NULL;
-        inode_contribution_t *contribution = NULL;
-
-        GF_VALIDATE_OR_GOTO ("marker", this, out);
-        GF_VALIDATE_OR_GOTO ("marker", local, out);
-
-        local->contri = NULL;
-
-        loc_wipe (&local->loc);
-
-        ret = mq_loc_copy (&local->loc, &local->parent_loc);
-        if (ret < 0) {
-                gf_log_callingfn (this->name, GF_LOG_WARNING,
-                        "loc copy failed");
-                goto out;
-        }
-
-        loc_wipe (&local->parent_loc);
+    int32_t ret = -1;
+    quota_meta_t size = {
+        0,
+    };
+    quota_meta_t contri = {
+        0,
+    };
 
-        ret = mq_inode_loc_fill (NULL, local->loc.parent,
-                                 &local->parent_loc);
-        if (ret < 0) {
-                gf_log_callingfn (this->name, GF_LOG_WARNING,
-                        "failed to build parent loc of %s",
-                        local->loc.path);
-                goto out;
-        }
+    GF_VALIDATE_OR_GOTO("marker", loc, out);
+    GF_VALIDATE_OR_GOTO("marker", loc->inode, out);
+    GF_VALIDATE_OR_GOTO("marker", ctx, out);
+    GF_VALIDATE_OR_GOTO("marker", contribution, out);
 
-        ret = mq_inode_ctx_get (local->loc.inode, this, &ctx);
-        if (ret < 0) {
-                gf_log_callingfn (this->name, GF_LOG_WARNING,
-                                  "inode ctx get failed");
-                goto out;
-        }
+    ret = mq_get_metadata(this, loc, &contri, &size, ctx, contribution);
+    if (ret < 0)
+        goto out;
 
-        local->ctx = ctx;
+    mq_compute_delta(delta, &size, &contri);
 
-        if (list_empty (&ctx->contribution_head)) {
-                gf_log_callingfn (this->name, GF_LOG_ERROR,
-                        "contribution node list is empty");
-                ret = -1;
-                goto out;
-        }
-
-        /* Earlier we used to get the next entry in the list maintained
-           by the context. In a good situation it works. i.e the next
-           parent in the directory hierarchy for this path is obtained.
-
-           But consider the below situation:
-           mount-point: /mnt/point
-           quota enabled directories within mount point: /a, /b, /c
-
-           Now when some file (file1) in the directory /c is written some data,
-           then to update the directories, marker has to get the contribution
-           object for the parent inode, i.e /c.
-           Beefore, it was being done by
-           local->contri = (inode_contribution_t *) ctx->contribution_head.next;
-           It works in the normal situations. But suppose /c is moved to /b.
-           Now /b's contribution object is added to the end of the list of
-           parents that the file file1 within /b/c is maintaining. Now if
-           the file /b/c/file1 is copied to /b/c/new, to update the parent in
-           the order c, b and / we cannot go to the next element in the list,
-           as in this case the next contribution object would be / and /b's
-           contribution will be at the end of the list. So get the proper
-           parent's contribution, by searching the entire list.
-        */
-        contribution = mq_get_contribution_node (local->loc.parent, ctx);
-        GF_ASSERT (contribution != NULL);
-        local->contri = contribution;
-
-        ret = 0;
 out:
-        return ret;
+    return ret;
 }
 
-
 int32_t
-mq_xattr_updation_done (call_frame_t *frame,
-                        void *cookie,
-                        xlator_t *this,
-                        int32_t op_ret,
-                        int32_t op_errno,
-                        dict_t *dict, dict_t *xdata)
+mq_remove_contri(xlator_t *this, loc_t *loc, quota_inode_ctx_t *ctx,
+                 inode_contribution_t *contri, quota_meta_t *delta,
+                 uint32_t nlink)
 {
-        QUOTA_STACK_DESTROY (frame, this);
-        return 0;
-}
+    int32_t ret = -1;
+    char contri_key[QUOTA_KEY_MAX] = {
+        0,
+    };
 
-
-int32_t
-mq_inodelk_cbk (call_frame_t *frame, void *cookie,
-                xlator_t *this, int32_t op_ret, int32_t op_errno, dict_t *xdata)
-{
-        int32_t         ret    = 0;
-        gf_boolean_t    status = _gf_false;
-        quota_local_t  *local  = NULL;
-
-        local = frame->local;
-
-        if (op_ret == -1 || local->err) {
-                if (op_ret == -1) {
-                        gf_log (this->name, GF_LOG_DEBUG,
-                                "unlocking failed on path (%s)(%s)",
-                                local->parent_loc.path, strerror (op_errno));
-                }
-                mq_xattr_updation_done (frame, NULL, this, 0, 0, NULL, NULL);
-
-                return 0;
-        }
-
-        gf_log (this->name, GF_LOG_DEBUG,
-                "inodelk released on %s", local->parent_loc.path);
-
-        if ((strcmp (local->parent_loc.path, "/") == 0)
-            || (local->delta == 0)) {
-                mq_xattr_updation_done (frame, NULL, this, 0, 0, NULL, NULL);
+    if (nlink == 1) {
+        /*File was a last link and has been deleted */
+        ret = 0;
+        goto done;
+    }
+
+    GET_CONTRI_KEY(this, contri_key, contri->gfid, ret);
+    if (ret < 0) {
+        gf_log(this->name, GF_LOG_ERROR,
+               "get contri_key "
+               "failed for %s",
+               uuid_utoa(contri->gfid));
+        goto out;
+    }
+
+    ret = syncop_removexattr(FIRST_CHILD(this), loc, contri_key, 0, NULL);
+    if (ret < 0) {
+        if (-ret == ENOENT || -ret == ESTALE || -ret == ENODATA ||
+            -ret == ENOATTR) {
+            /* Remove contri in done when unlink operation is
+             * performed, so return success on ENOENT/ESTSLE
+             * rename operation removes xattr earlier,
+             * so return success on ENODATA
+             */
+            ret = 0;
         } else {
-                ret = mq_get_parent_inode_local (this, local);
-                if (ret < 0) {
-                        mq_xattr_updation_done (frame, NULL, this, 0, 0, NULL,
-                                                NULL);
-                        goto out;
-                }
-                status = _gf_true;
-
-                ret = mq_test_and_set_ctx_updation_status (local->ctx, &status);
-                if (ret == 0 && status == _gf_false) {
-                        mq_get_lock_on_parent (frame, this);
-                } else {
-                        mq_xattr_updation_done (frame, NULL, this, 0, 0, NULL,
-                                                NULL);
-                }
+            gf_log_callingfn(this->name, GF_LOG_ERROR,
+                             "removexattr %s failed for %s: %s", contri_key,
+                             loc->path, strerror(-ret));
+            goto out;
         }
-out:
-        return 0;
-}
-
+    }
 
-//now release lock on the parent inode
-int32_t
-mq_release_parent_lock (call_frame_t *frame, void *cookie,
-                        xlator_t *this, int32_t op_ret,
-                        int32_t op_errno, dict_t *xdata)
-{
-        int32_t            ret      = 0;
-        quota_local_t     *local    = NULL;
-        quota_inode_ctx_t *ctx      = NULL;
-        struct gf_flock    lock     = {0, };
-
-        local = frame->local;
-
-        if (local->err != 0) {
-                gf_log_callingfn (this->name,
-                                  (local->err == ENOENT) ? GF_LOG_DEBUG
-                                  : GF_LOG_WARNING,
-                                  "An operation during quota updation "
-                                  "of path (%s) failed (%s)", local->loc.path,
-                                  strerror (local->err));
-        }
+done:
+    LOCK(&contri->lock);
+    {
+        contri->contribution += delta->size;
+        contri->file_count += delta->file_count;
+        contri->dir_count += delta->dir_count;
+    }
+    UNLOCK(&contri->lock);
 
-        ret = mq_inode_ctx_get (local->parent_loc.inode, this, &ctx);
-        if (ret < 0)
-                goto wind;
+    ret = 0;
 
-        LOCK (&ctx->lock);
-        {
-                ctx->dirty = 0;
-        }
-        UNLOCK (&ctx->lock);
-
-        if (local->parent_loc.inode == NULL) {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "Invalid parent inode.");
-                goto err;
-        }
+out:
+    QUOTA_FREE_CONTRIBUTION_NODE(ctx, contri);
 
-wind:
-        lock.l_type   = F_UNLCK;
-        lock.l_whence = SEEK_SET;
-        lock.l_start  = 0;
-        lock.l_len    = 0;
-        lock.l_pid    = 0;
-
-        STACK_WIND (frame,
-                    mq_inodelk_cbk,
-                    FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->inodelk,
-                    this->name, &local->parent_loc,
-                    F_SETLKW, &lock, NULL);
-
-        return 0;
-err:
-        mq_xattr_updation_done (frame, NULL, this,
-                                0, 0 , NULL, NULL);
-        return 0;
+    return ret;
 }
 
-
 int32_t
-mq_mark_undirty (call_frame_t *frame,
-                 void *cookie,
-                 xlator_t *this,
-                 int32_t op_ret,
-                 int32_t op_errno,
-                 dict_t *dict, dict_t *xdata)
+mq_update_contri(xlator_t *this, loc_t *loc, inode_contribution_t *contri,
+                 quota_meta_t *delta)
 {
-        int32_t            ret          = -1;
-        int64_t           *size         = NULL;
-        dict_t            *newdict      = NULL;
-        quota_local_t     *local        = NULL;
-        quota_inode_ctx_t *ctx          = NULL;
-
-        local = frame->local;
-
-        if (op_ret == -1) {
-                gf_log (this->name, GF_LOG_WARNING, "%s occurred while"
-                        " updating the size of %s", strerror (op_errno),
-                        local->parent_loc.path);
-
-                goto err;
-        }
-
-        //update the size of the parent inode
-        if (dict != NULL) {
-                ret = mq_inode_ctx_get (local->parent_loc.inode, this, &ctx);
-                if (ret < 0) {
-                        op_errno = EINVAL;
-                        goto err;
-                }
-
-                ret = dict_get_bin (dict, QUOTA_SIZE_KEY, (void **) &size);
-                if (ret < 0) {
-                        op_errno = EINVAL;
-                        goto err;
-                }
-
-                LOCK (&ctx->lock);
-                {
-                        if (size)
-                                ctx->size = ntoh64 (*size);
-                        gf_log (this->name, GF_LOG_DEBUG, "%s %"PRId64,
-                                local->parent_loc.path, ctx->size);
-                }
-                UNLOCK (&ctx->lock);
-        }
-
-        newdict = dict_new ();
-        if (!newdict) {
-                op_errno = ENOMEM;
-                goto err;
-        }
-
-        ret = dict_set_int8 (newdict, QUOTA_DIRTY_KEY, 0);
-
-        if (ret == -1) {
-                op_errno = -ret;
-                goto err;
-        }
-
-        gf_uuid_copy (local->parent_loc.gfid, local->parent_loc.inode->gfid);
-        GF_UUID_ASSERT (local->parent_loc.gfid);
+    int32_t ret = -1;
+    char contri_key[QUOTA_KEY_MAX] = {
+        0,
+    };
+    dict_t *dict = NULL;
 
-        STACK_WIND (frame, mq_release_parent_lock,
-                    FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->setxattr,
-                    &local->parent_loc, newdict, 0, NULL);
+    GF_VALIDATE_OR_GOTO("marker", loc, out);
+    GF_VALIDATE_OR_GOTO("marker", loc->inode, out);
+    GF_VALIDATE_OR_GOTO("marker", delta, out);
+    GF_VALIDATE_OR_GOTO("marker", contri, out);
 
+    if (quota_meta_is_null(delta)) {
         ret = 0;
-err:
-        if (op_ret == -1 || ret == -1) {
-                local->err = op_errno;
+        goto out;
+    }
+
+    dict = dict_new();
+    if (!dict) {
+        gf_log(this->name, GF_LOG_ERROR, "dict_new failed");
+        ret = -1;
+        goto out;
+    }
+
+    GET_CONTRI_KEY(this, contri_key, contri->gfid, ret);
+    if (ret < 0) {
+        gf_log(this->name, GF_LOG_ERROR,
+               "get contri_key "
+               "failed for %s",
+               uuid_utoa(contri->gfid));
+        goto out;
+    }
+
+    ret = quota_dict_set_meta(dict, contri_key, delta, loc->inode->ia_type);
+    if (ret < 0)
+        goto out;
+
+    ret = syncop_xattrop(FIRST_CHILD(this), loc, GF_XATTROP_ADD_ARRAY64, dict,
+                         NULL, NULL, NULL);
+    if (ret < 0) {
+        gf_log_callingfn(
+            this->name,
+            (-ret == ENOENT || -ret == ESTALE) ? GF_LOG_DEBUG : GF_LOG_ERROR,
+            "xattrop failed "
+            "for %s: %s",
+            loc->path, strerror(-ret));
+        goto out;
+    }
+
+    LOCK(&contri->lock);
+    {
+        contri->contribution += delta->size;
+        contri->file_count += delta->file_count;
+        contri->dir_count += delta->dir_count;
+    }
+    UNLOCK(&contri->lock);
 
-                mq_release_parent_lock (frame, NULL, this, 0, 0, NULL);
-        }
-
-        if (newdict)
-                dict_unref (newdict);
+out:
+    if (dict)
+        dict_unref(dict);
 
-        return 0;
+    return ret;
 }
 
-
 int32_t
-mq_update_parent_size (call_frame_t *frame,
-                       void *cookie,
-                       xlator_t *this,
-                       int32_t op_ret,
-                       int32_t op_errno,
-                       dict_t *dict, dict_t *xdata)
+mq_update_size(xlator_t *this, loc_t *loc, quota_meta_t *delta)
 {
-        int64_t             *size       = NULL;
-        int32_t              ret        = -1;
-        dict_t              *newdict    = NULL;
-        quota_local_t       *local      = NULL;
-        quota_inode_ctx_t   *ctx        = NULL;
-
-        local = frame->local;
-
-        if (op_ret == -1) {
-                gf_log (this->name, ((op_errno == ENOENT) ? GF_LOG_DEBUG :
-                                     GF_LOG_WARNING),
-                        "xattrop call failed: %s", strerror (op_errno));
-
-                goto err;
-        }
-
-        LOCK (&local->contri->lock);
-        {
-                local->contri->contribution += local->delta;
-        }
-        UNLOCK (&local->contri->lock);
-
-        gf_log_callingfn (this->name, GF_LOG_DEBUG, "path: %s size: %"PRId64
-                          " contribution:%"PRId64,
-                          local->loc.path, local->ctx->size,
-                          local->contri->contribution);
-
-        if (dict == NULL) {
-                op_errno = EINVAL;
-                goto err;
-        }
-
-        ret = mq_inode_ctx_get (local->parent_loc.inode, this, &ctx);
-        if (ret < 0) {
-                op_errno = EINVAL;
-                goto err;
-        }
+    int32_t ret = -1;
+    quota_inode_ctx_t *ctx = NULL;
+    dict_t *dict = NULL;
 
-        newdict = dict_new ();
-        if (!newdict) {
-                op_errno = ENOMEM;
-                ret = -1;
-                goto err;
-        }
-
-        QUOTA_ALLOC_OR_GOTO (size, int64_t, ret, err);
+    GF_VALIDATE_OR_GOTO("marker", loc, out);
+    GF_VALIDATE_OR_GOTO("marker", loc->inode, out);
+    GF_VALIDATE_OR_GOTO("marker", delta, out);
 
-        *size = hton64 (local->delta);
-
-        ret = dict_set_bin (newdict, QUOTA_SIZE_KEY, size, 8);
-        if (ret < 0) {
-                op_errno = -ret;
-                goto err;
-        }
-
-        if (gf_uuid_is_null (local->parent_loc.gfid))
-                        gf_uuid_copy (local->parent_loc.gfid,
-                                   local->parent_loc.inode->gfid);
-        GF_UUID_ASSERT (local->parent_loc.gfid);
-
-        STACK_WIND (frame,
-                    mq_mark_undirty,
-                    FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->xattrop,
-                    &local->parent_loc,
-                    GF_XATTROP_ADD_ARRAY64,
-                    newdict, NULL);
+    if (quota_meta_is_null(delta)) {
         ret = 0;
-err:
-        if (op_ret == -1 || ret < 0) {
-                local->err = op_errno;
-                mq_release_parent_lock (frame, NULL, this, 0, 0, NULL);
-        }
+        goto out;
+    }
+
+    ret = mq_inode_ctx_get(loc->inode, this, &ctx);
+    if (ret < 0) {
+        gf_log(this->name, GF_LOG_ERROR,
+               "failed to get inode ctx for "
+               "%s",
+               loc->path);
+        goto out;
+    }
+
+    dict = dict_new();
+    if (!dict) {
+        gf_log(this->name, GF_LOG_ERROR, "dict_new failed");
+        ret = -1;
+        goto out;
+    }
+
+    ret = quota_dict_set_size_meta(this, dict, delta);
+    if (ret < 0)
+        goto out;
+
+    ret = syncop_xattrop(FIRST_CHILD(this), loc,
+                         GF_XATTROP_ADD_ARRAY64_WITH_DEFAULT, dict, NULL, NULL,
+                         NULL);
+    if (ret < 0) {
+        gf_log_callingfn(
+            this->name,
+            (-ret == ENOENT || -ret == ESTALE) ? GF_LOG_DEBUG : GF_LOG_ERROR,
+            "xattrop failed "
+            "for %s: %s",
+            loc->path, strerror(-ret));
+        goto out;
+    }
+
+    LOCK(&ctx->lock);
+    {
+        ctx->size += delta->size;
+        ctx->file_count += delta->file_count;
+        if (ctx->dir_count == 0)
+            ctx->dir_count += delta->dir_count + 1;
+        else
+            ctx->dir_count += delta->dir_count;
+    }
+    UNLOCK(&ctx->lock);
 
-        if (newdict)
-                dict_unref (newdict);
+out:
+    if (dict)
+        dict_unref(dict);
 
-        return 0;
+    return ret;
 }
 
-int32_t
-mq_update_inode_contribution (call_frame_t *frame, void *cookie,
-                              xlator_t *this, int32_t op_ret,
-                              int32_t op_errno, inode_t *inode,
-                              struct iatt *buf, dict_t *dict,
-                              struct iatt *postparent)
+int
+mq_synctask_cleanup(int ret, call_frame_t *frame, void *opaque)
 {
-        int32_t               ret                         = -1;
-        int64_t              *size                        = NULL;
-        int64_t               size_int                    = 0;
-        int64_t               contri_int                  = 0;
-        int64_t              *contri                      = NULL;
-        int64_t              *delta                       = NULL;
-        char                  contri_key[CONTRI_KEY_MAX]  = {0, };
-        dict_t               *newdict                     = NULL;
-        quota_local_t        *local                       = NULL;
-        quota_inode_ctx_t    *ctx                         = NULL;
-        inode_contribution_t *contribution                = NULL;
-
-        local = frame->local;
-
-        if (op_ret == -1) {
-                gf_log (this->name, ((op_errno == ENOENT) ? GF_LOG_DEBUG :
-                                     GF_LOG_WARNING),
-                        "failed to get size and contribution of path (%s)(%s)",
-                        local->loc.path, strerror (op_errno));
-                goto err;
-        }
+    quota_synctask_t *args = NULL;
 
-        ctx = local->ctx;
-        contribution = local->contri;
+    GF_ASSERT(opaque);
 
-        //prepare to update size & contribution of the inode
-        GET_CONTRI_KEY (contri_key, contribution->gfid, ret);
-        if (ret == -1) {
-                op_errno = ENOMEM;
-                goto err;
-        }
+    args = (quota_synctask_t *)opaque;
+    loc_wipe(&args->loc);
 
-        LOCK (&ctx->lock);
-        {
-                if (local->loc.inode->ia_type == IA_IFDIR ) {
-                        ret = dict_get_bin (dict, QUOTA_SIZE_KEY,
-                                            (void **) &size);
-                        if (ret < 0) {
-                                op_errno = EINVAL;
-                                goto unlock;
-                        }
-
-                        ctx->size = ntoh64 (*size);
-                } else
-                        ctx->size = buf->ia_blocks * 512;
-
-                size_int = ctx->size;
-        }
-unlock:
-        UNLOCK  (&ctx->lock);
+    if (args->stub)
+        call_resume(args->stub);
 
-        if (ret < 0) {
-                goto err;
-        }
-
-        ret = dict_get_bin (dict, contri_key, (void **) &contri);
+    if (!args->is_static)
+        GF_FREE(args);
 
-        LOCK (&contribution->lock);
-        {
-                if (ret < 0)
-                        contribution->contribution = 0;
-                else
-                        contribution->contribution = ntoh64 (*contri);
-
-                contri_int = contribution->contribution;
-        }
-        UNLOCK (&contribution->lock);
-
-        gf_log (this->name, GF_LOG_DEBUG, "%s %"PRId64 " %"PRId64,
-                local->loc.path, size_int, contri_int);
-
-        local->delta = size_int - contri_int;
-
-        if (local->delta == 0) {
-                mq_mark_undirty (frame, NULL, this, 0, 0, NULL, NULL);
-                return 0;
-        }
-
-        newdict = dict_new ();
-        if (newdict == NULL) {
-                op_errno = ENOMEM;
-                ret = -1;
-                goto err;
-        }
-
-        QUOTA_ALLOC_OR_GOTO (delta, int64_t, ret, err);
-
-        *delta = hton64 (local->delta);
-
-        ret = dict_set_bin (newdict, contri_key, delta, 8);
-        if (ret < 0) {
-                op_errno = -ret;
-                ret = -1;
-                goto err;
-        }
-
-        if (gf_uuid_is_null (local->loc.gfid))
-                gf_uuid_copy (local->loc.gfid, buf->ia_gfid);
-
-        GF_UUID_ASSERT (local->loc.gfid);
-
-        STACK_WIND (frame,
-                    mq_update_parent_size,
-                    FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->xattrop,
-                    &local->loc,
-                    GF_XATTROP_ADD_ARRAY64,
-                    newdict, NULL);
-        ret = 0;
-
-err:
-        if (op_ret == -1 || ret < 0) {
-                local->err = op_errno;
-
-                mq_release_parent_lock (frame, NULL, this, 0, 0, NULL);
-        }
-
-        if (newdict)
-                dict_unref (newdict);
-
-        return 0;
+    return 0;
 }
 
-int32_t
-mq_fetch_child_size_and_contri (call_frame_t *frame, void *cookie,
-                                xlator_t *this, int32_t op_ret,
-                                int32_t op_errno, dict_t *xdata)
-{
-        int32_t            ret                         = -1;
-        char               contri_key[CONTRI_KEY_MAX]  = {0, };
-        dict_t            *newdict                     = NULL;
-        quota_local_t     *local                       = NULL;
-        quota_inode_ctx_t *ctx                         = NULL;
-
-        local = frame->local;
-
-        if (op_ret == -1) {
-                gf_log (this->name, (op_errno == ENOENT) ? GF_LOG_DEBUG
-                        : GF_LOG_WARNING,
-                        "couldnt mark inode corresponding to path (%s) dirty "
-                        "(%s)", local->parent_loc.path, strerror (op_errno));
-                goto err;
-        }
-
-        VALIDATE_OR_GOTO (local->ctx, err);
-        VALIDATE_OR_GOTO (local->contri, err);
-
-        gf_log (this->name, GF_LOG_DEBUG, "%s marked dirty",
-                local->parent_loc.path);
-
-        //update parent ctx
-        ret = mq_inode_ctx_get (local->parent_loc.inode, this, &ctx);
-        if (ret == -1) {
-                op_errno = EINVAL;
-                goto err;
-        }
-
-        LOCK (&ctx->lock);
-        {
-                ctx->dirty = 1;
-        }
-        UNLOCK (&ctx->lock);
-
-        newdict = dict_new ();
-        if (newdict == NULL) {
-                op_errno = ENOMEM;
-                goto err;
-        }
-
-        if (local->loc.inode->ia_type == IA_IFDIR) {
-                ret = dict_set_int64 (newdict, QUOTA_SIZE_KEY, 0);
-                if (ret < 0) {
-                        gf_log (this->name, GF_LOG_WARNING,
-                                "dict_set failed.");
-                        goto err;
-                }
-        }
-
-        GET_CONTRI_KEY (contri_key, local->contri->gfid, ret);
-        if (ret < 0) {
-                op_errno = ENOMEM;
-                goto err;
-        }
-
-        ret = dict_set_int64 (newdict, contri_key, 0);
-        if (ret < 0) {
-                gf_log (this->name, GF_LOG_WARNING,
-                        "dict_set failed.");
-                goto err;
-        }
-
-        mq_set_ctx_updation_status (local->ctx, _gf_false);
-
-        if (gf_uuid_is_null (local->loc.gfid))
-                gf_uuid_copy (local->loc.gfid, local->loc.inode->gfid);
-
-        GF_UUID_ASSERT (local->loc.gfid);
-
-        STACK_WIND (frame, mq_update_inode_contribution, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->lookup, &local->loc, newdict);
-
-        ret = 0;
-
-err:
-        if ((op_ret == -1) || (ret < 0)) {
-                local->err = op_errno;
-
-                mq_set_ctx_updation_status (local->ctx, _gf_false);
-
-                mq_release_parent_lock (frame, NULL, this, 0, 0, NULL);
+int
+mq_synctask1(xlator_t *this, synctask_fn_t task, gf_boolean_t spawn, loc_t *loc,
+             quota_meta_t *contri, uint32_t nlink, call_stub_t *stub)
+{
+    int32_t ret = -1;
+    quota_synctask_t *args = NULL;
+    quota_synctask_t static_args = {
+        0,
+    };
+
+    if (spawn) {
+        QUOTA_ALLOC_OR_GOTO(args, quota_synctask_t, ret, out);
+        args->is_static = _gf_false;
+    } else {
+        args = &static_args;
+        args->is_static = _gf_true;
+    }
+
+    args->this = this;
+    args->stub = stub;
+    loc_copy(&args->loc, loc);
+    args->ia_nlink = nlink;
+
+    if (contri) {
+        args->contri = *contri;
+    } else {
+        args->contri.size = -1;
+        args->contri.file_count = -1;
+        args->contri.dir_count = -1;
+    }
+
+    if (spawn) {
+        ret = synctask_new1(this->ctx->env, 1024 * 16, task,
+                            mq_synctask_cleanup, NULL, args);
+        if (ret) {
+            gf_log(this->name, GF_LOG_ERROR,
+                   "Failed to spawn "
+                   "new synctask");
+            mq_synctask_cleanup(ret, NULL, args);
         }
+    } else {
+        ret = task(args);
+        mq_synctask_cleanup(ret, NULL, args);
+    }
 
-        if (newdict)
-                dict_unref (newdict);
-
-        return 0;
+out:
+    return ret;
 }
 
-int32_t
-mq_markdirty (call_frame_t *frame, void *cookie,
-              xlator_t *this, int32_t op_ret, int32_t op_errno, dict_t *xdata)
+int
+mq_synctask(xlator_t *this, synctask_fn_t task, gf_boolean_t spawn, loc_t *loc)
 {
-        int32_t        ret      = -1;
-        dict_t        *dict     = NULL;
-        quota_local_t *local    = NULL;
-
-        local = frame->local;
-
-        if (op_ret == -1){
-                gf_log (this->name, (op_errno == ENOENT) ? GF_LOG_DEBUG
-                        : GF_LOG_WARNING, "acquiring locks failed on %s (%s)",
-                        local->parent_loc.path, strerror (op_errno));
-
-                local->err = op_errno;
-
-                mq_set_ctx_updation_status (local->ctx, _gf_false);
-
-                mq_inodelk_cbk (frame, NULL, this, 0, 0, NULL);
-
-                return 0;
-        }
-
-        gf_log (this->name, GF_LOG_TRACE,
-                "inodelk succeeded on  %s", local->parent_loc.path);
-
-        dict = dict_new ();
-        if (!dict) {
-                ret = -1;
-                goto err;
-        }
-
-        ret = dict_set_int8 (dict, QUOTA_DIRTY_KEY, 1);
-        if (ret == -1)
-                goto err;
-
-        gf_uuid_copy (local->parent_loc.gfid,
-                   local->parent_loc.inode->gfid);
-        GF_UUID_ASSERT (local->parent_loc.gfid);
-
-        STACK_WIND (frame, mq_fetch_child_size_and_contri,
-                    FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->setxattr,
-                    &local->parent_loc, dict, 0, NULL);
-
-        ret = 0;
-err:
-        if (ret == -1) {
-                local->err = 1;
-
-                mq_set_ctx_updation_status (local->ctx, _gf_false);
-
-                mq_release_parent_lock (frame, NULL, this, 0, 0, NULL);
-        }
-
-        if (dict)
-                dict_unref (dict);
-
-        return 0;
+    return mq_synctask1(this, task, spawn, loc, NULL, -1, NULL);
 }
 
-
 int32_t
-mq_get_lock_on_parent (call_frame_t *frame, xlator_t *this)
+mq_prevalidate_txn(xlator_t *this, loc_t *origin_loc, loc_t *loc,
+                   quota_inode_ctx_t **ctx, struct iatt *buf)
 {
-        struct gf_flock  lock  = {0, };
-        quota_local_t   *local = NULL;
+    int32_t ret = -1;
+    quota_inode_ctx_t *ctxtmp = NULL;
 
-        GF_VALIDATE_OR_GOTO ("marker", frame, fr_destroy);
+    if (buf) {
+        if (buf->ia_type == IA_IFREG && IS_DHT_LINKFILE_MODE(buf))
+            goto out;
 
-        local = frame->local;
-        gf_log (this->name, GF_LOG_DEBUG, "taking lock on %s",
-                local->parent_loc.path);
+        if (buf->ia_type != IA_IFREG && buf->ia_type != IA_IFLNK &&
+            buf->ia_type != IA_IFDIR)
+            goto out;
+    }
 
-        if (local->parent_loc.inode == NULL) {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "parent inode is not valid, aborting "
-                        "transaction.");
-                goto fr_destroy;
-        }
+    if (origin_loc == NULL || origin_loc->inode == NULL ||
+        gf_uuid_is_null(origin_loc->inode->gfid))
+        goto out;
 
-        lock.l_len    = 0;
-        lock.l_start  = 0;
-        lock.l_type   = F_WRLCK;
-        lock.l_whence = SEEK_SET;
+    loc_copy(loc, origin_loc);
 
-        STACK_WIND (frame,
-                    mq_markdirty,
-                    FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->inodelk,
-                    this->name, &local->parent_loc, F_SETLKW, &lock, NULL);
+    if (gf_uuid_is_null(loc->gfid))
+        gf_uuid_copy(loc->gfid, loc->inode->gfid);
 
-        return 0;
+    if (!loc_is_root(loc) && loc->parent == NULL)
+        loc->parent = inode_parent(loc->inode, 0, NULL);
 
-fr_destroy:
-        QUOTA_STACK_DESTROY (frame, this);
+    ret = mq_inode_ctx_get(loc->inode, this, &ctxtmp);
+    if (ret < 0) {
+        gf_log_callingfn(this->name, GF_LOG_WARNING,
+                         "inode ctx for "
+                         "is NULL for %s",
+                         loc->path);
+        goto out;
+    }
+    if (ctx)
+        *ctx = ctxtmp;
 
-        return -1;
+    ret = 0;
+out:
+    return ret;
 }
 
 int
-mq_prepare_txn_frame (xlator_t *this, loc_t *loc,
-                      quota_inode_ctx_t *ctx,
-                      inode_contribution_t *contri,
-                      call_frame_t **new_frame)
-{
-        call_frame_t    *frame = NULL;
-        int              ret   = -1;
-        quota_local_t   *local = NULL;
-
-        if (!this || !loc || !new_frame)
-                goto err;
-
-        frame = create_frame (this, this->ctx->pool);
-        if (frame == NULL)
-                goto err;
-
-        mq_assign_lk_owner (this, frame);
-
-        local = mq_local_new ();
-        if (local == NULL)
-                goto fr_destroy;
+mq_create_xattrs_task(void *opaque)
+{
+    int32_t ret = -1;
+    gf_boolean_t locked = _gf_false;
+    gf_boolean_t contri_set = _gf_false;
+    gf_boolean_t size_set = _gf_false;
+    gf_boolean_t need_txn = _gf_false;
+    quota_synctask_t *args = NULL;
+    quota_inode_ctx_t *ctx = NULL;
+    xlator_t *this = NULL;
+    loc_t *loc = NULL;
+    gf_boolean_t status = _gf_false;
+
+    GF_ASSERT(opaque);
+
+    args = (quota_synctask_t *)opaque;
+    loc = &args->loc;
+    this = args->this;
+    THIS = this;
+
+    ret = mq_inode_ctx_get(loc->inode, this, &ctx);
+    if (ret < 0) {
+        gf_log(this->name, GF_LOG_WARNING,
+               "Failed to"
+               "get inode ctx, aborting quota create txn");
+        goto out;
+    }
+
+    if (loc->inode->ia_type == IA_IFDIR) {
+        /* lock not required for files */
+        ret = mq_lock(this, loc, F_WRLCK);
+        if (ret < 0)
+            goto out;
+        locked = _gf_true;
+    }
 
-        frame->local = local;
+    ret = mq_are_xattrs_set(this, loc, &contri_set, &size_set);
+    if (ret < 0 || (contri_set && size_set))
+        goto out;
 
-        ret = mq_loc_copy (&local->loc, loc);
-        if (ret < 0)
-                goto fr_destroy;
+    mq_set_ctx_create_status(ctx, _gf_false);
+    status = _gf_true;
 
-        ret = mq_inode_loc_fill (NULL, local->loc.parent,
-                                 &local->parent_loc);
+    if (loc->inode->ia_type == IA_IFDIR && size_set == _gf_false) {
+        ret = mq_create_size_xattrs(this, ctx, loc);
         if (ret < 0)
-                goto fr_destroy;
+            goto out;
+    }
 
-        local->ctx = ctx;
-        local->contri = contri;
+    need_txn = _gf_true;
+out:
+    if (locked)
+        ret = mq_lock(this, loc, F_UNLCK);
 
-        ret = 0;
-        *new_frame = frame;
+    if (status == _gf_false)
+        mq_set_ctx_create_status(ctx, _gf_false);
 
-        return ret;
+    if (need_txn)
+        ret = mq_initiate_quota_blocking_txn(this, loc, NULL);
 
-fr_destroy:
-        QUOTA_STACK_DESTROY (frame, this);
-err:
-        return ret;
+    return ret;
 }
 
-int
-mq_start_quota_txn (xlator_t *this, loc_t *loc,
-                    quota_inode_ctx_t *ctx,
-                    inode_contribution_t *contri)
+static int
+_mq_create_xattrs_txn(xlator_t *this, loc_t *origin_loc, struct iatt *buf,
+                      gf_boolean_t spawn)
 {
-        int32_t        ret      = -1;
-        call_frame_t  *frame    = NULL;
-
-        ret = mq_prepare_txn_frame (this, loc, ctx,
-                                    contri, &frame);
-        if (ret)
-                goto err;
-
-        ret = mq_get_lock_on_parent (frame, this);
-        if (ret == -1)
-                goto err;
+    int32_t ret = -1;
+    quota_inode_ctx_t *ctx = NULL;
+    gf_boolean_t status = _gf_true;
+    loc_t loc = {
+        0,
+    };
+    inode_contribution_t *contribution = NULL;
 
-        return 0;
+    ret = mq_prevalidate_txn(this, origin_loc, &loc, &ctx, buf);
+    if (ret < 0)
+        goto out;
 
-err:
-        mq_set_ctx_updation_status (ctx, _gf_false);
+    ret = mq_test_and_set_ctx_create_status(ctx, &status);
+    if (ret < 0 || status == _gf_true)
+        goto out;
 
-        return -1;
-}
-
-void
-mq_compute_delta (quota_meta_t *delta, const quota_meta_t *op1,
-                  const quota_meta_t *op2)
-{
-        delta->size       = op1->size - op2->size;
-        delta->file_count = op1->file_count - op2->file_count;
-        delta->dir_count  = op1->dir_count - op2->dir_count;
-}
-
-void
-mq_add_meta (quota_meta_t *dst, const quota_meta_t *src)
-{
-        dst->size       += src->size;
-        dst->file_count += src->file_count;
-        dst->dir_count  += src->dir_count;
-}
-
-void
-mq_sub_meta (quota_meta_t *dst, const quota_meta_t *src)
-{
-        if (src == NULL) {
-                dst->size       = -dst->size;
-                dst->file_count = -dst->file_count;
-                dst->dir_count  = -dst->dir_count;
+    if (!loc_is_root(&loc) && loc.parent) {
+        contribution = mq_add_new_contribution_node(this, ctx, &loc);
+        if (contribution == NULL) {
+            gf_log(this->name, GF_LOG_WARNING,
+                   "cannot add a new contribution node "
+                   "(%s)",
+                   uuid_utoa(loc.gfid));
+            ret = -1;
+            goto out;
         } else {
-                dst->size       = src->size - dst->size;
-                dst->file_count = src->file_count - dst->file_count;
-                dst->dir_count  = src->dir_count - dst->dir_count;
+            GF_REF_PUT(contribution);
         }
-}
+    }
 
-gf_boolean_t
-quota_meta_is_null (const quota_meta_t *meta)
-{
-        if (meta->size == 0 &&
-            meta->file_count == 0 &&
-            meta->dir_count == 0)
-                return _gf_true;
+    ret = mq_synctask(this, mq_create_xattrs_task, spawn, &loc);
+out:
+    if (ret < 0 && status == _gf_false)
+        mq_set_ctx_create_status(ctx, _gf_false);
 
-        return _gf_false;
+    loc_wipe(&loc);
+    return ret;
 }
 
-int32_t
-mq_are_xattrs_set (xlator_t *this, loc_t *loc, gf_boolean_t *result,
-                   gf_boolean_t *objects)
+int
+mq_create_xattrs_txn(xlator_t *this, loc_t *loc, struct iatt *buf)
 {
-        int32_t        ret                         = -1;
-        char           contri_key[CONTRI_KEY_MAX]  = {0, };
-        quota_meta_t   meta                        = {0, };
-        struct iatt    stbuf                       = {0,};
-        dict_t        *dict                        = NULL;
-        dict_t        *rsp_dict                    = NULL;
-
-        dict = dict_new ();
-        if (dict == NULL) {
-                gf_log (this->name, GF_LOG_ERROR, "dict_new failed");
-                goto out;
-        }
+    int32_t ret = -1;
 
-        ret = mq_req_xattr (this, loc, dict, contri_key);
-        if (ret < 0)
-                goto out;
-
-        ret = syncop_lookup (FIRST_CHILD(this), loc, &stbuf, NULL,
-                             dict, &rsp_dict);
-        if (ret < 0) {
-                gf_log (this->name, (-ret == ENOENT || -ret == ESTALE)
-                        ? GF_LOG_DEBUG:GF_LOG_ERROR, "lookup failed "
-                        "for %s: %s", loc->path, strerror (-ret));
-                goto out;
-        }
-
-        if (rsp_dict == NULL) {
-                *result = _gf_false;
-                goto out;
-        }
-
-        *result = _gf_true;
-        if (loc->inode->ia_type == IA_IFDIR) {
-                ret = quota_dict_get_meta (rsp_dict, QUOTA_SIZE_KEY, &meta);
-                if (ret < 0 || meta.dir_count == 0) {
-                        ret = 0;
-                        *result = _gf_false;
-                        goto out;
-                }
-                *objects = _gf_true;
-        }
-
-        if (!loc_is_root(loc) && !dict_get (rsp_dict, contri_key)) {
-                *result = _gf_false;
-                goto out;
-        }
+    GF_VALIDATE_OR_GOTO("marker", loc, out);
+    GF_VALIDATE_OR_GOTO("marker", loc->inode, out);
 
+    ret = _mq_create_xattrs_txn(this, loc, buf, _gf_true);
 out:
-        if (dict)
-                dict_unref (dict);
-
-        if (rsp_dict)
-                dict_unref (rsp_dict);
-
-        return ret;
-}
-
-int32_t
-mq_create_xattrs (xlator_t *this, loc_t *loc, gf_boolean_t objects)
-{
-        quota_meta_t           size                 = {0, };
-        quota_meta_t           contri               = {0, };
-        int32_t                ret                  = -1;
-        char                   key[CONTRI_KEY_MAX]  = {0, };
-        dict_t                *dict                 = NULL;
-        quota_inode_ctx_t     *ctx                  = NULL;
-        inode_contribution_t  *contribution         = NULL;
-
-        GF_VALIDATE_OR_GOTO ("marker", loc, out);
-        GF_VALIDATE_OR_GOTO ("marker", loc->inode, out);
-
-        ret = mq_inode_ctx_get (loc->inode, this, &ctx);
+    return ret;
+}
+
+int32_t
+mq_reduce_parent_size_task(void *opaque)
+{
+    int32_t ret = -1;
+    int32_t prev_dirty = 0;
+    quota_inode_ctx_t *ctx = NULL;
+    quota_inode_ctx_t *parent_ctx = NULL;
+    inode_contribution_t *contribution = NULL;
+    quota_meta_t delta = {
+        0,
+    };
+    quota_meta_t contri = {
+        0,
+    };
+    loc_t parent_loc = {
+        0,
+    };
+    gf_boolean_t locked = _gf_false;
+    gf_boolean_t dirty = _gf_false;
+    quota_synctask_t *args = NULL;
+    xlator_t *this = NULL;
+    loc_t *loc = NULL;
+    gf_boolean_t remove_xattr = _gf_true;
+    uint32_t nlink = 0;
+
+    GF_ASSERT(opaque);
+
+    args = (quota_synctask_t *)opaque;
+    loc = &args->loc;
+    contri = args->contri;
+    nlink = args->ia_nlink;
+    this = args->this;
+    THIS = this;
+
+    ret = mq_inode_loc_fill(NULL, loc->parent, &parent_loc);
+    if (ret < 0) {
+        gf_log(this->name, GF_LOG_ERROR,
+               "parent_loc fill failed for "
+               "child inode %s: ",
+               uuid_utoa(loc->inode->gfid));
+        goto out;
+    }
+
+    ret = mq_lock(this, &parent_loc, F_WRLCK);
+    if (ret < 0)
+        goto out;
+    locked = _gf_true;
+
+    if (contri.size >= 0) {
+        /* contri parameter is supplied only for rename operation.
+         * remove xattr is alreday performed, we need to skip
+         * removexattr for rename operation
+         */
+        remove_xattr = _gf_false;
+        delta.size = contri.size;
+        delta.file_count = contri.file_count;
+        delta.dir_count = contri.dir_count;
+    } else {
+        remove_xattr = _gf_true;
+
+        ret = mq_inode_ctx_get(loc->inode, this, &ctx);
         if (ret < 0) {
-                ctx = mq_inode_ctx_new (loc->inode, this);
-                if (ctx == NULL) {
-                        gf_log (this->name, GF_LOG_WARNING,
-                                "mq_inode_ctx_new failed");
-                        ret = -1;
-                        goto out;
-                }
+            gf_log_callingfn(this->name, GF_LOG_WARNING,
+                             "ctx for"
+                             " the node %s is NULL",
+                             loc->path);
+            goto out;
         }
 
-        dict = dict_new ();
-        if (!dict) {
-                gf_log (this->name, GF_LOG_ERROR, "dict_new failed");
-                ret = -1;
-                goto out;
-        }
-
-        if (loc->inode->ia_type == IA_IFDIR) {
-                if (objects == _gf_false) {
-                        /* Initial object count of a directory is 1 */
-                        size.dir_count = 1;
-                }
-                ret = quota_dict_set_meta (dict, QUOTA_SIZE_KEY, &size,
-                                           IA_IFDIR);
-                if (ret < 0)
-                        goto out;
-        }
-
-        if (!loc_is_root (loc)) {
-                contribution = mq_add_new_contribution_node (this, ctx, loc);
-                if (contribution == NULL) {
-                        ret = -1;
-                        goto out;
-                }
-
-                GET_CONTRI_KEY (key, contribution->gfid, ret);
-                ret = quota_dict_set_meta (dict, key, &contri,
-                                           loc->inode->ia_type);
-                if (ret < 0)
-                        goto out;
+        contribution = mq_get_contribution_node(loc->parent, ctx);
+        if (contribution == NULL) {
+            ret = -1;
+            gf_log(this->name, GF_LOG_DEBUG,
+                   "contribution for the node %s is NULL", loc->path);
+            goto out;
         }
 
-        ret = syncop_xattrop(FIRST_CHILD(this), loc, GF_XATTROP_ADD_ARRAY64,
-                             dict, NULL, NULL);
-
-        if (ret < 0) {
-                gf_log (this->name, (-ret == ENOENT || -ret == ESTALE)
-                        ? GF_LOG_DEBUG:GF_LOG_ERROR, "xattrop failed "
-                        "for %s: %s", loc->path, strerror (-ret));
-                goto out;
+        LOCK(&contribution->lock);
+        {
+            delta.size = contribution->contribution;
+            delta.file_count = contribution->file_count;
+            delta.dir_count = contribution->dir_count;
         }
+        UNLOCK(&contribution->lock);
+    }
 
-out:
-        if (dict)
-                dict_unref (dict);
-
-        return ret;
-}
-
-int32_t
-mq_lock (xlator_t *this, loc_t *loc, short l_type)
-{
-        struct gf_flock  lock  = {0, };
-        int32_t          ret   = -1;
-
-        GF_VALIDATE_OR_GOTO ("marker", loc, out);
-        GF_VALIDATE_OR_GOTO ("marker", loc->inode, out);
-
-        gf_log (this->name, GF_LOG_DEBUG, "set lock type %d on %s",
-                l_type, loc->path);
+    ret = mq_get_set_dirty(this, &parent_loc, 1, &prev_dirty);
+    if (ret < 0)
+        goto out;
+    dirty = _gf_true;
 
-        lock.l_len    = 0;
-        lock.l_start  = 0;
-        lock.l_type   = l_type;
-        lock.l_whence = SEEK_SET;
+    mq_sub_meta(&delta, NULL);
 
-        ret = syncop_inodelk (FIRST_CHILD(this), this->name, loc, F_SETLKW,
-                              &lock, NULL, NULL);
+    if (remove_xattr) {
+        ret = mq_remove_contri(this, loc, ctx, contribution, &delta, nlink);
         if (ret < 0)
-                gf_log (this->name, (-ret == ENOENT || -ret == ESTALE)
-                        ? GF_LOG_DEBUG:GF_LOG_ERROR, "inodelk failed "
-                        "for %s: %s", loc->path, strerror (-ret));
-
-out:
-
-        return ret;
-}
-
-int32_t
-mq_get_dirty (xlator_t *this, loc_t *loc, int32_t *dirty)
-{
-        int32_t        ret              = -1;
-        int8_t         value            = 0;
-        dict_t        *dict             = NULL;
-        dict_t        *rsp_dict         = NULL;
-        struct iatt    stbuf            = {0,};
-
-        dict = dict_new ();
-        if (dict == NULL) {
-                gf_log (this->name, GF_LOG_ERROR, "dict_new failed");
-                goto out;
-        }
+            goto out;
+    }
 
-        ret = dict_set_int64 (dict, QUOTA_DIRTY_KEY, 0);
-        if (ret < 0) {
-                gf_log (this->name, GF_LOG_WARNING, "dict set failed");
-                goto out;
-        }
+    if (quota_meta_is_null(&delta))
+        goto out;
 
-        ret = syncop_lookup (FIRST_CHILD(this), loc, &stbuf, NULL,
-                             dict, &rsp_dict);
-        if (ret < 0) {
-                gf_log (this->name, (-ret == ENOENT || -ret == ESTALE)
-                        ? GF_LOG_DEBUG:GF_LOG_ERROR, "lookup failed "
-                        "for %s: %s", loc->path, strerror (-ret));
-                goto out;
-        }
-
-        ret = dict_get_int8 (rsp_dict, QUOTA_DIRTY_KEY, &value);
-        if (ret < 0)
-                goto out;
-
-        *dirty = value;
+    ret = mq_update_size(this, &parent_loc, &delta);
+    if (ret < 0)
+        goto out;
 
 out:
-        if (dict)
-                dict_unref (dict);
-
-        if (rsp_dict)
-                dict_unref (rsp_dict);
-
-        return ret;
-}
-
-int32_t
-mq_mark_dirty (xlator_t *this, loc_t *loc, int32_t dirty)
-{
-        int32_t            ret      = -1;
-        dict_t            *dict     = NULL;
-        quota_inode_ctx_t *ctx      = NULL;
-
-        GF_VALIDATE_OR_GOTO ("marker", loc, out);
-        GF_VALIDATE_OR_GOTO ("marker", loc->inode, out);
-
-        dict = dict_new ();
-        if (!dict) {
-                gf_log (this->name, GF_LOG_ERROR, "dict_new failed");
-                goto out;
+    if (dirty) {
+        if (ret < 0 || prev_dirty) {
+            /* On failure clear dirty status flag.
+             * In the next lookup inspect_directory_xattr
+             * can set the status flag and fix the
+             * dirty directory.
+             * Do the same if dir was dirty before
+             * the txn
+             */
+            ret = mq_inode_ctx_get(parent_loc.inode, this, &parent_ctx);
+            if (ret == 0)
+                mq_set_ctx_dirty_status(parent_ctx, _gf_false);
+        } else {
+            ret = mq_mark_dirty(this, &parent_loc, 0);
         }
+    }
 
-        ret = dict_set_int8 (dict, QUOTA_DIRTY_KEY, dirty);
-        if (ret < 0) {
-                gf_log (this->name, GF_LOG_ERROR, "dict_set failed");
-                goto out;
-        }
+    if (locked)
+        ret = mq_lock(this, &parent_loc, F_UNLCK);
 
-        ret = syncop_setxattr (FIRST_CHILD(this), loc, dict, 0, NULL, NULL);
-        if (ret < 0) {
-                gf_log (this->name, (-ret == ENOENT || -ret == ESTALE)
-                        ? GF_LOG_DEBUG:GF_LOG_ERROR, "setxattr dirty = %d "
-                        "failed for %s: %s", dirty, loc->path, strerror (-ret));
-                goto out;
-        }
+    if (ret >= 0)
+        ret = mq_initiate_quota_blocking_txn(this, &parent_loc, NULL);
 
-        ret = mq_inode_ctx_get (loc->inode, this, &ctx);
-        if (ret < 0) {
-                gf_log (this->name, GF_LOG_ERROR, "failed to get inode ctx for "
-                        "%s", loc->path);
-                goto out;
-        }
+    loc_wipe(&parent_loc);
 
-        LOCK (&ctx->lock);
-        {
-                ctx->dirty = dirty;
-        }
-        UNLOCK (&ctx->lock);
+    if (contribution)
+        GF_REF_PUT(contribution);
 
-out:
-        if (dict)
-                dict_unref (dict);
-
-        return ret;
+    return ret;
 }
 
 int32_t
-_mq_get_metadata (xlator_t *this, loc_t *loc, quota_meta_t *contri,
-                  quota_meta_t *size, uuid_t contri_gfid)
+mq_reduce_parent_size_txn(xlator_t *this, loc_t *origin_loc,
+                          quota_meta_t *contri, uint32_t nlink,
+                          call_stub_t *stub)
 {
-        int32_t            ret                         = -1;
-        quota_meta_t       meta                        = {0, };
-        char               contri_key[CONTRI_KEY_MAX]  = {0, };
-        dict_t            *dict                        = NULL;
-        dict_t            *rsp_dict                    = NULL;
-        struct iatt        stbuf                       = {0,};
+    int32_t ret = -1;
+    loc_t loc = {
+        0,
+    };
+    gf_boolean_t resume_stub = _gf_true;
 
-        GF_VALIDATE_OR_GOTO ("marker", loc, out);
-        GF_VALIDATE_OR_GOTO ("marker", loc->inode, out);
+    GF_VALIDATE_OR_GOTO("marker", this, out);
+    GF_VALIDATE_OR_GOTO("marker", origin_loc, out);
 
-        if (size == NULL && contri == NULL)
-                goto out;
-
-        dict = dict_new ();
-        if (dict == NULL) {
-                gf_log (this->name, GF_LOG_ERROR, "dict_new failed");
-                goto out;
-        }
-
-        if (size && loc->inode->ia_type == IA_IFDIR) {
-                ret = dict_set_int64 (dict, QUOTA_SIZE_KEY, 0);
-                if (ret < 0) {
-                        gf_log (this->name, GF_LOG_ERROR, "dict_set failed.");
-                        goto out;
-                }
-        }
-
-        if (contri && !loc_is_root(loc)) {
-                ret = mq_dict_set_contribution (this, dict, loc, contri_gfid,
-                                                contri_key);
-                if (ret < 0)
-                        goto out;
-        }
-
-        ret = syncop_lookup (FIRST_CHILD(this), loc, &stbuf, NULL,
-                             dict, &rsp_dict);
-        if (ret < 0) {
-                gf_log (this->name, (-ret == ENOENT || -ret == ESTALE)
-                        ? GF_LOG_DEBUG:GF_LOG_ERROR, "lookup failed "
-                        "for %s: %s", loc->path, strerror (-ret));
-                goto out;
-        }
-
-        if (size) {
-                if (loc->inode->ia_type == IA_IFDIR) {
-                        ret = quota_dict_get_meta (rsp_dict, QUOTA_SIZE_KEY,
-                                                   &meta);
-                        if (ret < 0) {
-                                gf_log (this->name, GF_LOG_ERROR,
-                                        "dict_get failed.");
-                                goto out;
-                        }
-
-                        size->size = meta.size;
-                        size->file_count = meta.file_count;
-                        size->dir_count = meta.dir_count;
-                } else {
-                        size->size = stbuf.ia_blocks * 512;
-                        size->file_count = 1;
-                        size->dir_count = 0;
-                }
-        }
-
-        if (contri && !loc_is_root(loc)) {
-                ret = quota_dict_get_meta (rsp_dict, contri_key, &meta);
-                if (ret < 0) {
-                        contri->size = 0;
-                        contri->file_count = 0;
-                        contri->dir_count = 0;
-                } else {
-                        contri->size = meta.size;
-                        contri->file_count = meta.file_count;
-                        contri->dir_count = meta.dir_count;
-                }
-        }
+    ret = mq_prevalidate_txn(this, origin_loc, &loc, NULL, NULL);
+    if (ret < 0)
+        goto out;
 
+    if (loc_is_root(&loc)) {
         ret = 0;
+        goto out;
+    }
 
+    resume_stub = _gf_false;
+    ret = mq_synctask1(this, mq_reduce_parent_size_task, _gf_true, &loc, contri,
+                       nlink, stub);
 out:
-        if (dict)
-                dict_unref (dict);
-
-        if (rsp_dict)
-                dict_unref (rsp_dict);
-
-        return ret;
-}
-
-int32_t
-mq_get_metadata (xlator_t *this, loc_t *loc, quota_meta_t *contri,
-                 quota_meta_t *size, quota_inode_ctx_t *ctx,
-                 inode_contribution_t *contribution)
-{
-        int32_t         ret      = -1;
-
-        GF_VALIDATE_OR_GOTO ("marker", loc, out);
-        GF_VALIDATE_OR_GOTO ("marker", loc->inode, out);
-        GF_VALIDATE_OR_GOTO ("marker", ctx, out);
-        GF_VALIDATE_OR_GOTO ("marker", contribution, out);
+    loc_wipe(&loc);
 
-        if (size == NULL && contri == NULL) {
-                ret = 0;
-                goto out;
-        }
-
-        ret = _mq_get_metadata (this, loc, contri, size, contribution->gfid);
-        if (ret < 0) {
-                gf_log_callingfn (this->name, GF_LOG_ERROR, "Failed to get "
-                                  "metadata for %s", loc->path);
-                goto out;
-        }
+    if (resume_stub && stub)
+        call_resume(stub);
 
-        if (size) {
-                LOCK (&ctx->lock);
-                {
-                        ctx->size = size->size;
-                        ctx->file_count = size->file_count;
-                        ctx->dir_count = size->dir_count;
-                }
-                UNLOCK  (&ctx->lock);
-        }
+    if (ret)
+        gf_log_callingfn(this ? this->name : "Marker", GF_LOG_ERROR,
+                         "mq_reduce_parent_size_txn failed");
 
-        if (contri) {
-                LOCK (&contribution->lock);
-                {
-                        contribution->contribution = contri->size;
-                        contribution->file_count = contri->file_count;
-                        contribution->dir_count = contri->dir_count;
-                }
-                UNLOCK (&contribution->lock);
-        }
-
-out:
-        return ret;
+    return ret;
 }
 
-int32_t
-mq_get_size (xlator_t *this, loc_t *loc, quota_meta_t *size)
-{
-        int32_t         ret      = -1;
-
-        ret = _mq_get_metadata (this, loc, NULL, size, 0);
+int
+mq_initiate_quota_task(void *opaque)
+{
+    int32_t ret = -1;
+    int32_t prev_dirty = 0;
+    loc_t child_loc = {
+        0,
+    };
+    loc_t parent_loc = {
+        0,
+    };
+    gf_boolean_t locked = _gf_false;
+    gf_boolean_t dirty = _gf_false;
+    gf_boolean_t status = _gf_false;
+    quota_meta_t delta = {
+        0,
+    };
+    quota_synctask_t *args = NULL;
+    xlator_t *this = NULL;
+    loc_t *loc = NULL;
+    inode_contribution_t *contri = NULL;
+    quota_inode_ctx_t *ctx = NULL;
+    quota_inode_ctx_t *parent_ctx = NULL;
+    inode_t *tmp_parent = NULL;
+
+    GF_VALIDATE_OR_GOTO("marker", opaque, out);
+
+    args = (quota_synctask_t *)opaque;
+    loc = &args->loc;
+    this = args->this;
+
+    GF_VALIDATE_OR_GOTO("marker", this, out);
+    THIS = this;
+
+    GF_VALIDATE_OR_GOTO(this->name, loc, out);
+    GF_VALIDATE_OR_GOTO(this->name, loc->inode, out);
+
+    ret = mq_loc_copy(&child_loc, loc);
+    if (ret < 0) {
+        gf_log(this->name, GF_LOG_ERROR, "loc copy failed");
+        goto out;
+    }
+
+    while (!__is_root_gfid(child_loc.gfid)) {
+        ret = mq_inode_ctx_get(child_loc.inode, this, &ctx);
         if (ret < 0) {
-                gf_log_callingfn (this->name, GF_LOG_ERROR, "Failed to get "
-                                  "metadata for %s", loc->path);
+            gf_log(this->name, GF_LOG_WARNING,
+                   "inode ctx get failed for %s, "
+                   "aborting update txn",
+                   child_loc.path);
+            goto out;
+        }
+
+        /* To improve performance, abort current transaction
+         * if one is already in progress for same inode
+         */
+        if (status == _gf_true) {
+            /* status will already set before txn start,
+             * so it should not be set in first
+             * loop iteration
+             */
+            ret = mq_test_and_set_ctx_updation_status(ctx, &status);
+            if (ret < 0 || status == _gf_true)
+                goto out;
+        }
+
+        if (child_loc.parent == NULL) {
+            ret = mq_build_ancestry(this, &child_loc);
+            if (ret < 0 || child_loc.parent == NULL) {
+                /* If application performs parallel remove
+                 * operations on same set of files/directories
+                 * then we may get ENOENT/ESTALE
+                 */
+                gf_log(this->name,
+                       (-ret == ENOENT || -ret == ESTALE) ? GF_LOG_DEBUG
+                                                          : GF_LOG_ERROR,
+                       "build ancestry failed for inode %s",
+                       uuid_utoa(child_loc.inode->gfid));
+                ret = -1;
                 goto out;
+            }
         }
 
-out:
-        return ret;
-}
-
-int32_t
-mq_get_contri (xlator_t *this, loc_t *loc, quota_meta_t *contri,
-               uuid_t contri_gfid)
-{
-        int32_t         ret      = -1;
-
-        ret = _mq_get_metadata (this, loc, contri, NULL, contri_gfid);
+        ret = mq_inode_loc_fill(NULL, child_loc.parent, &parent_loc);
         if (ret < 0) {
-                gf_log_callingfn (this->name, GF_LOG_ERROR, "Failed to get "
-                                  "metadata for %s", loc->path);
-                goto out;
+            gf_log(this->name, GF_LOG_ERROR,
+                   "parent_loc fill "
+                   "failed for child inode %s: ",
+                   uuid_utoa(child_loc.inode->gfid));
+            goto out;
         }
 
-out:
-        return ret;
-}
-
-int32_t
-mq_get_delta (xlator_t *this, loc_t *loc, quota_meta_t *delta,
-              quota_inode_ctx_t *ctx, inode_contribution_t *contribution)
-{
-        int32_t         ret      = -1;
-        quota_meta_t    size     = {0, };
-        quota_meta_t    contri   = {0, };
-
-        GF_VALIDATE_OR_GOTO ("marker", loc, out);
-        GF_VALIDATE_OR_GOTO ("marker", loc->inode, out);
-        GF_VALIDATE_OR_GOTO ("marker", ctx, out);
-        GF_VALIDATE_OR_GOTO ("marker", contribution, out);
-
-        ret = mq_get_metadata (this, loc, &contri, &size, ctx, contribution);
+        ret = mq_lock(this, &parent_loc, F_WRLCK);
         if (ret < 0)
-                goto out;
-
-        mq_compute_delta (delta, &size, &contri);
-
-out:
-        return ret;
-}
-
-int32_t
-mq_remove_contri (xlator_t *this, loc_t *loc, inode_contribution_t *contri)
-{
-        int32_t              ret                         = -1;
-        char                 contri_key[CONTRI_KEY_MAX]  = {0, };
-
-        GET_CONTRI_KEY (contri_key, contri->gfid, ret);
-        if (ret < 0) {
-                gf_log (this->name, GF_LOG_ERROR, "get contri_key "
-                        "failed for %s", uuid_utoa(contri->gfid));
-                goto out;
-        }
-
-        ret = syncop_removexattr (FIRST_CHILD(this), loc, contri_key, 0, NULL);
-        if (ret < 0) {
-                if (-ret == ENOENT || -ret == ESTALE) {
-                        /* Remove contri in done when unlink operation is
-                         * performed, so return success on ENOENT/ESTSLE
-                         */
-                        ret = 0;
-                } else {
-                        gf_log (this->name, GF_LOG_ERROR, "removexattr %s "
-                                "failed for %s: %s", contri_key, loc->path,
-                                strerror (-ret));
-                        goto out;
-                }
-        }
-
-        LOCK (&contri->lock);
-        {
-                contri->contribution = 0;
-                contri->file_count = 0;
-                contri->dir_count = 0;
-        }
-        UNLOCK (&contri->lock);
-
-        ret = 0;
-out:
-
-        return ret;
-}
+            goto out;
+        locked = _gf_true;
 
-int32_t
-mq_update_contri (xlator_t *this, loc_t *loc, inode_contribution_t *contri,
-                  quota_meta_t *delta)
-{
-        int32_t              ret                         = -1;
-        char                 contri_key[CONTRI_KEY_MAX]  = {0, };
-        dict_t              *dict                        = NULL;
+        mq_set_ctx_updation_status(ctx, _gf_false);
+        status = _gf_true;
 
-        GF_VALIDATE_OR_GOTO ("marker", loc, out);
-        GF_VALIDATE_OR_GOTO ("marker", loc->inode, out);
-        GF_VALIDATE_OR_GOTO ("marker", delta, out);
-        GF_VALIDATE_OR_GOTO ("marker", contri, out);
+        /* Contribution node can be NULL in below scenarios and
+           create if needed:
 
-        if (quota_meta_is_null (delta)) {
-                ret = 0;
-                goto out;
-        }
+           Scenario 1)
+           In this case create a new contribution node
+           Suppose hard link for a file f1 present in a directory d1 is
+           created in the directory d2 (as f2). Now, since d2's
+           contribution is not there in f1's inode ctx, d2's
+           contribution xattr won't be created and will create problems
+           for quota operations.
 
-        dict = dict_new ();
-        if (!dict) {
-                gf_log (this->name, GF_LOG_ERROR, "dict_new failed");
+           Don't create contribution if parent has been changed after
+           taking a lock, this can happen when rename is performed
+           and writes is still in-progress for the same file
+
+           Scenario 2)
+           When a rename operation is performed, contribution node
+           for olp path will be removed.
+
+           Create contribution node only if oldparent is same as
+           newparent.
+           Consider below example
+           1) rename FOP invoked on file 'x'
+           2) write is still in progress for file 'x'
+           3) rename takes a lock on old-parent
+           4) write-update txn blocked on old-parent to acquire lock
+           5) in rename_cbk, contri xattrs are removed and contribution
+              is deleted and lock is released
+           6) now write-update txn gets the lock and updates the
+              wrong parent as it was holding lock on old parent
+              so validate parent once the lock is acquired
+
+             For more information on this problem, please see
+             doc for marker_rename in file marker.c
+        */
+        contri = mq_get_contribution_node(child_loc.parent, ctx);
+        if (contri == NULL) {
+            tmp_parent = inode_parent(child_loc.inode, 0, NULL);
+            if (tmp_parent == NULL) {
+                /* This can happen if application performs
+                 * parallel remove operations on same set
+                 * of files/directories
+                 */
+                gf_log(this->name, GF_LOG_WARNING,
+                       "parent is "
+                       "NULL for inode %s",
+                       uuid_utoa(child_loc.inode->gfid));
                 ret = -1;
                 goto out;
-        }
-
-        GET_CONTRI_KEY (contri_key, contri->gfid, ret);
-        if (ret < 0) {
-                gf_log (this->name, GF_LOG_ERROR, "get contri_key "
-                        "failed for %s", uuid_utoa(contri->gfid));
-                goto out;
-        }
-
-        ret = quota_dict_set_meta (dict, contri_key, delta,
-                                   loc->inode->ia_type);
-        if (ret < 0)
-                goto out;
-
-        ret = syncop_xattrop(FIRST_CHILD(this), loc, GF_XATTROP_ADD_ARRAY64,
-                             dict, NULL, NULL);
-        if (ret < 0) {
-                gf_log (this->name, (-ret == ENOENT || -ret == ESTALE)
-                        ? GF_LOG_DEBUG:GF_LOG_ERROR, "xattrop failed "
-                        "for %s: %s", loc->path, strerror (-ret));
-                goto out;
-        }
-
-        LOCK (&contri->lock);
-        {
-                contri->contribution += delta->size;
-                contri->file_count += delta->file_count;
-                contri->dir_count += delta->dir_count;
-        }
-        UNLOCK (&contri->lock);
-
-out:
-        if (dict)
-                dict_unref (dict);
-
-        return ret;
-}
-
-int32_t
-mq_update_size (xlator_t *this, loc_t *loc, quota_meta_t *delta)
-{
-        int32_t              ret              = -1;
-        quota_inode_ctx_t   *ctx              = NULL;
-        dict_t              *dict             = NULL;
-
-        GF_VALIDATE_OR_GOTO ("marker", loc, out);
-        GF_VALIDATE_OR_GOTO ("marker", loc->inode, out);
-        GF_VALIDATE_OR_GOTO ("marker", delta, out);
-
-        if (quota_meta_is_null (delta)) {
+            }
+            if (gf_uuid_compare(tmp_parent->gfid, parent_loc.gfid)) {
+                /* abort txn if parent has changed */
                 ret = 0;
                 goto out;
-        }
+            }
 
-        ret = mq_inode_ctx_get (loc->inode, this, &ctx);
-        if (ret < 0) {
-                gf_log (this->name, GF_LOG_ERROR, "failed to get inode ctx for "
-                        "%s", loc->path);
-                goto out;
-        }
+            inode_unref(tmp_parent);
+            tmp_parent = NULL;
 
-        dict = dict_new ();
-        if (!dict) {
-                gf_log (this->name, GF_LOG_ERROR, "dict_new failed");
+            contri = mq_add_new_contribution_node(this, ctx, &child_loc);
+            if (contri == NULL) {
+                gf_log(this->name, GF_LOG_ERROR,
+                       "Failed to "
+                       "create contribution node for %s, "
+                       "abort update txn",
+                       child_loc.path);
                 ret = -1;
                 goto out;
+            }
         }
 
-        ret = quota_dict_set_meta (dict, QUOTA_SIZE_KEY, delta,
-                                   loc->inode->ia_type);
+        ret = mq_get_delta(this, &child_loc, &delta, ctx, contri);
         if (ret < 0)
-                goto out;
-
-        ret = syncop_xattrop(FIRST_CHILD(this), loc, GF_XATTROP_ADD_ARRAY64,
-                             dict, NULL, NULL);
-        if (ret < 0) {
-                gf_log (this->name, (-ret == ENOENT || -ret == ESTALE)
-                        ? GF_LOG_DEBUG:GF_LOG_ERROR, "xattrop failed "
-                        "for %s: %s", loc->path, strerror (-ret));
-                goto out;
-        }
-
-        LOCK (&ctx->lock);
-        {
-                ctx->size += delta->size;
-                ctx->file_count += delta->file_count;
-                ctx->dir_count += delta->dir_count;
-        }
-        UNLOCK (&ctx->lock);
-
-out:
-        if (dict)
-                dict_unref (dict);
-
-        return ret;
-}
-
-int
-mq_synctask_cleanup (int ret, call_frame_t *frame, void *opaque)
-{
-        quota_synctask_t       *args         = NULL;
+            goto out;
 
-        GF_ASSERT (opaque);
+        if (quota_meta_is_null(&delta))
+            goto out;
 
-        args = (quota_synctask_t *) opaque;
-        loc_wipe (&args->loc);
-        if (args->dict)
-                dict_unref (args->dict);
-
-        if (!args->is_static)
-                GF_FREE (args);
-
-        return 0;
-}
-
-int
-mq_synctask (xlator_t *this, synctask_fn_t task, gf_boolean_t spawn, loc_t *loc,
-             dict_t *dict, struct iatt *buf, int64_t contri)
-{
-        int32_t              ret         = -1;
-        quota_synctask_t    *args        = NULL;
-        quota_synctask_t     static_args = {0, };
-
-        if (spawn) {
-                QUOTA_ALLOC_OR_GOTO (args, quota_synctask_t, ret, out);
-                args->is_static = _gf_false;
-        } else {
-                args = &static_args;
-                args->is_static = _gf_true;
-        }
-
-        args->this = this;
-        loc_copy (&args->loc, loc);
-        args->contri = contri;
-        if (dict)
-                args->dict = dict_ref (dict);
-        if (buf)
-                args->buf = *buf;
-
-
-        if (spawn) {
-                ret = synctask_new (this->ctx->env, task, mq_synctask_cleanup,
-                                    NULL, args);
-                if (ret) {
-                        gf_log (this->name, GF_LOG_ERROR, "Failed to spawn "
-                                "new synctask");
-                        mq_synctask_cleanup (ret, NULL, args);
-                }
-        } else {
-                ret = task (args);
-                mq_synctask_cleanup (ret, NULL, args);
-        }
+        ret = mq_get_set_dirty(this, &parent_loc, 1, &prev_dirty);
+        if (ret < 0)
+            goto out;
+        dirty = _gf_true;
 
-out:
-        return ret;
-}
+        ret = mq_update_contri(this, &child_loc, contri, &delta);
+        if (ret < 0)
+            goto out;
 
-int
-mq_start_quota_txn_v2 (xlator_t *this, loc_t *loc, quota_inode_ctx_t *ctx,
-                       inode_contribution_t *contri)
-{
-        int32_t            ret        = -1;
-        loc_t              child_loc  = {0,};
-        loc_t              parent_loc = {0,};
-        gf_boolean_t       locked     = _gf_false;
-        gf_boolean_t       dirty      = _gf_false;
-        gf_boolean_t       status     = _gf_true;
-        quota_meta_t       delta      = {0, };
-
-        GF_VALIDATE_OR_GOTO ("marker", loc, out);
-        GF_VALIDATE_OR_GOTO ("marker", loc->inode, out);
-        GF_VALIDATE_OR_GOTO ("marker", ctx, out);
-        GF_VALIDATE_OR_GOTO ("marker", contri, out);
-
-        ret = mq_loc_copy (&child_loc, loc);
+        ret = mq_update_size(this, &parent_loc, &delta);
         if (ret < 0) {
-                gf_log (this->name, GF_LOG_ERROR, "loc copy failed");
-                goto out;
-        }
-
-        if (gf_uuid_is_null (child_loc.gfid))
-                gf_uuid_copy (child_loc.gfid, child_loc.inode->gfid);
-
-        if (gf_uuid_is_null (child_loc.gfid)) {
-                ret = -1;
-                gf_log (this->name, GF_LOG_DEBUG, "UUID is null for %s",
-                        child_loc.path);
-                goto out;
+            gf_log(this->name, GF_LOG_DEBUG,
+                   "rollback "
+                   "contri updation");
+            mq_sub_meta(&delta, NULL);
+            mq_update_contri(this, &child_loc, contri, &delta);
+            goto out;
         }
 
-        while (!__is_root_gfid (child_loc.gfid)) {
-                /* To improve performance, abort current transaction
-                 * if one is already in progress for same inode
-                 */
-                ret = mq_test_and_set_ctx_updation_status (ctx, &status);
-                if (ret < 0 || status == _gf_true)
-                        goto out;
-
-                ret = mq_inode_loc_fill (NULL, child_loc.parent, &parent_loc);
-                if (ret < 0) {
-                        gf_log (this->name, GF_LOG_ERROR, "loc fill failed");
-                        goto out;
-                }
-
-                ret = mq_lock (this, &parent_loc, F_WRLCK);
-                if (ret < 0)
-                        goto out;
-                locked = _gf_true;
-
-                mq_set_ctx_updation_status (ctx, _gf_false);
-                status = _gf_true;
-
-                ret = mq_get_delta (this, &child_loc, &delta, ctx, contri);
-                if (ret < 0)
-                        goto out;
-
-                if (quota_meta_is_null (&delta))
-                        goto out;
-
-                ret = mq_mark_dirty (this, &parent_loc, 1);
-                if (ret < 0)
-                        goto out;
-                dirty = _gf_true;
-
-                ret = mq_update_contri (this, &child_loc, contri, &delta);
-                if (ret < 0) {
-                        gf_log (this->name, GF_LOG_ERROR, "contri "
-                                "update failed for %s", child_loc.path);
-                        goto out;
-                }
-
-                ret = mq_update_size (this, &parent_loc, &delta);
-                if (ret < 0) {
-                        gf_log (this->name, GF_LOG_WARNING, "rollback "
-                                "contri updation");
-                        mq_sub_meta (&delta, NULL);
-                        mq_update_contri (this, &child_loc, contri, &delta);
-                        goto out;
-                }
-
-                ret = mq_mark_dirty (this, &parent_loc, 0);
-                dirty = _gf_false;
-
-                ret = mq_lock (this, &parent_loc, F_UNLCK);
-                locked = _gf_false;
-
-                if (__is_root_gfid (parent_loc.gfid))
-                        break;
-
-                /* Repeate above steps upwards till the root */
-                loc_wipe (&child_loc);
-                ret = mq_loc_copy (&child_loc, &parent_loc);
-                if (ret < 0)
-                        goto out;
-                loc_wipe (&parent_loc);
-
-                ret = mq_inode_ctx_get (child_loc.inode, this, &ctx);
-                if (ret < 0)
-                        goto out;
-
-                if (list_empty (&ctx->contribution_head)) {
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "contribution node list is empty (%s)",
-                                uuid_utoa(child_loc.inode->gfid));
-                        ret = -1;
-                        goto out;
-                }
-                contri = mq_get_contribution_node (child_loc.parent, ctx);
-                GF_ASSERT (contri != NULL);
+        if (prev_dirty == 0) {
+            ret = mq_mark_dirty(this, &parent_loc, 0);
+        } else {
+            ret = mq_inode_ctx_get(parent_loc.inode, this, &parent_ctx);
+            if (ret == 0)
+                mq_set_ctx_dirty_status(parent_ctx, _gf_false);
         }
+        dirty = _gf_false;
+        prev_dirty = 0;
 
-out:
-        if (ret >= 0 && dirty)
-                ret = mq_mark_dirty (this, &parent_loc, 0);
-
-        if (locked)
-                ret = mq_lock (this, &parent_loc, F_UNLCK);
-
-        if (status == _gf_false)
-                mq_set_ctx_updation_status (ctx, _gf_false);
-
-        loc_wipe (&child_loc);
-        loc_wipe (&parent_loc);
-
-        return ret;
-}
+        ret = mq_lock(this, &parent_loc, F_UNLCK);
+        locked = _gf_false;
 
-int
-mq_create_xattrs_task (void *opaque)
-{
-        int32_t                  ret        = -1;
-        gf_boolean_t             locked     = _gf_false;
-        gf_boolean_t             xattrs_set = _gf_false;
-        gf_boolean_t             objects    = _gf_false;
-        gf_boolean_t             need_txn   = _gf_false;
-        quota_synctask_t        *args       = NULL;
-        xlator_t                *this       = NULL;
-        loc_t                   *loc        = NULL;
-
-        GF_ASSERT (opaque);
-
-        args = (quota_synctask_t *) opaque;
-        loc = &args->loc;
-        this = args->this;
-        THIS = this;
-
-        if (gf_uuid_is_null (loc->gfid))
-                gf_uuid_copy (loc->gfid, loc->inode->gfid);
-
-        if (gf_uuid_is_null (loc->gfid)) {
-                ret = -1;
-                gf_log (this->name, GF_LOG_DEBUG, "UUID is null for %s",
-                        loc->path);
-                goto out;
-        }
+        if (__is_root_gfid(parent_loc.gfid))
+            break;
 
-        ret = mq_lock (this, loc, F_WRLCK);
+        /* Repeate above steps upwards till the root */
+        loc_wipe(&child_loc);
+        ret = mq_loc_copy(&child_loc, &parent_loc);
         if (ret < 0)
-                goto out;
-        locked = _gf_true;
+            goto out;
 
-        ret = mq_are_xattrs_set (this, loc, &xattrs_set, &objects);
-        if (ret < 0 || xattrs_set)
-                goto out;
+        loc_wipe(&parent_loc);
+        GF_REF_PUT(contri);
+        contri = NULL;
+    }
 
-        ret = mq_create_xattrs (this, loc, objects);
-        if (ret < 0)
-                goto out;
-
-        need_txn = _gf_true;
 out:
-        if (locked)
-                ret = mq_lock (this, loc, F_UNLCK);
-
-        if (need_txn)
-                ret = mq_initiate_quota_blocking_txn (this, loc);
+    if ((dirty) && (ret < 0)) {
+        /* On failure clear dirty status flag.
+         * In the next lookup inspect_directory_xattr
+         * can set the status flag and fix the
+         * dirty directory.
+         * Do the same if the dir was dirty before
+         * txn
+         */
+        ret = mq_inode_ctx_get(parent_loc.inode, this, &parent_ctx);
+        if (ret == 0)
+            mq_set_ctx_dirty_status(parent_ctx, _gf_false);
+    }
 
-        return ret;
-}
-
-int
-mq_create_xattrs_txn (xlator_t *this, loc_t *loc)
-{
-        int32_t                  ret        = -1;
+    if (locked)
+        ret = mq_lock(this, &parent_loc, F_UNLCK);
 
-        GF_VALIDATE_OR_GOTO ("marker", loc, out);
-        GF_VALIDATE_OR_GOTO ("marker", loc->inode, out);
+    if (ctx && status == _gf_false)
+        mq_set_ctx_updation_status(ctx, _gf_false);
 
-        ret = mq_synctask (this, mq_create_xattrs_task, _gf_true, loc, NULL,
-                           NULL, 0);
-out:
-        return ret;
-}
+    loc_wipe(&child_loc);
+    loc_wipe(&parent_loc);
 
-int
-mq_create_xattrs_blocking_txn (xlator_t *this, loc_t *loc)
-{
-        int32_t                  ret        = -1;
+    if (tmp_parent)
+        inode_unref(tmp_parent);
 
-        GF_VALIDATE_OR_GOTO ("marker", loc, out);
-        GF_VALIDATE_OR_GOTO ("marker", loc->inode, out);
+    if (contri)
+        GF_REF_PUT(contri);
 
-        ret = mq_synctask (this, mq_create_xattrs_task, _gf_false, loc, NULL,
-                           NULL, 0);
-out:
-        return ret;
+    return 0;
 }
 
-int32_t
-mq_reduce_parent_size_task (void *opaque)
+int
+_mq_initiate_quota_txn(xlator_t *this, loc_t *origin_loc, struct iatt *buf,
+                       gf_boolean_t spawn)
 {
-        int32_t                  ret           = -1;
-        quota_inode_ctx_t       *ctx           = NULL;
-        inode_contribution_t    *contribution  = NULL;
-        quota_meta_t             delta         = {0, };
-        loc_t                    parent_loc    = {0,};
-        gf_boolean_t             locked        = _gf_false;
-        gf_boolean_t             dirty         = _gf_false;
-        quota_synctask_t        *args          = NULL;
-        xlator_t                *this          = NULL;
-        loc_t                   *loc           = NULL;
-        int64_t                  contri        = 0;
-
-        GF_ASSERT (opaque);
-
-        args = (quota_synctask_t *) opaque;
-        loc = &args->loc;
-        contri = args->contri;
-        this = args->this;
-        THIS = this;
-
-        ret = mq_inode_ctx_get (loc->inode, this, &ctx);
-        if (ret < 0) {
-                gf_log_callingfn (this->name, GF_LOG_WARNING, "ctx for"
-                                  " the node %s is NULL", loc->path);
-                goto out;
-        }
-
-        contribution = mq_get_contribution_node (loc->parent, ctx);
-        if (contribution == NULL) {
-                ret = -1;
-                gf_log_callingfn (this->name, GF_LOG_WARNING,
-                                  "contribution for the node %s is NULL",
-                                  loc->path);
-                goto out;
-        }
+    int32_t ret = -1;
+    quota_inode_ctx_t *ctx = NULL;
+    gf_boolean_t status = _gf_true;
+    loc_t loc = {
+        0,
+    };
 
-        if (contri >= 0) {
-                /* contri paramater is supplied only for rename operation */
-                delta.size = contri;
-                delta.file_count = 1;
-                delta.dir_count = 0;
-        }
+    ret = mq_prevalidate_txn(this, origin_loc, &loc, &ctx, buf);
+    if (ret < 0)
+        goto out;
 
-        ret = mq_inode_loc_fill (NULL, loc->parent, &parent_loc);
-        if (ret < 0) {
-                gf_log (this->name, GF_LOG_ERROR, "loc fill failed");
-                goto out;
-        }
-
-        ret = mq_lock (this, &parent_loc, F_WRLCK);
-        if (ret < 0)
-                goto out;
-        locked = _gf_true;
-
-        if (contri < 0) {
-                LOCK (&contribution->lock);
-                {
-                        delta.size = contribution->contribution;
-                        delta.file_count = contribution->file_count;
-                        delta.dir_count = contribution->dir_count;
-                }
-                UNLOCK (&contribution->lock);
-        }
-
-        /* TODO: Handle handlinks with better approach
-           Iterating dentry_list without a lock is not a good idea
-        if (loc->inode->ia_type != IA_IFDIR) {
-                list_for_each_entry (dentry, &inode->dentry_list, inode_list) {
-                        if (loc->parent == dentry->parent) {
-                                * If the file has another link within the same
-                                * directory, we should not be reducing the size
-                                * of parent
-                                *
-                                delta = 0;
-                                idelta = 0;
-                                break;
-                        }
-                }
-        }
-        */
-
-        if (quota_meta_is_null (&delta))
-                goto out;
-
-        ret = mq_mark_dirty (this, &parent_loc, 1);
-        if (ret < 0)
-                goto out;
-        dirty = _gf_true;
+    if (loc_is_root(&loc)) {
+        ret = 0;
+        goto out;
+    }
 
-        ret = mq_remove_contri (this, loc, contribution);
-        if (ret < 0)
-                goto out;
+    ret = mq_test_and_set_ctx_updation_status(ctx, &status);
+    if (ret < 0 || status == _gf_true)
+        goto out;
 
-        mq_sub_meta (&delta, NULL);
-        ret = mq_update_size (this, &parent_loc, &delta);
-        if (ret < 0)
-                goto out;
+    ret = mq_synctask(this, mq_initiate_quota_task, spawn, &loc);
 
 out:
-        if (dirty && ret >= 0)
-                ret = mq_mark_dirty (this, &parent_loc, 0);
-
-        if (locked)
-                ret = mq_lock (this, &parent_loc, F_UNLCK);
-
-        if (ret >= 0)
-                ret = mq_initiate_quota_blocking_txn (this, &parent_loc);
-
-        loc_wipe (&parent_loc);
+    if (ret < 0 && status == _gf_false)
+        mq_set_ctx_updation_status(ctx, _gf_false);
 
-        return ret;
-}
-
-int32_t
-mq_reduce_parent_size_txn (xlator_t *this, loc_t *loc, int64_t contri)
-{
-        int32_t                  ret           = -1;
-
-        GF_VALIDATE_OR_GOTO ("marker", this, out);
-        GF_VALIDATE_OR_GOTO ("marker", loc, out);
-        GF_VALIDATE_OR_GOTO ("marker", loc->inode, out);
-
-        ret = mq_synctask (this, mq_reduce_parent_size_task, _gf_true, loc,
-                           NULL, NULL, contri);
-out:
-        return ret;
+    loc_wipe(&loc);
+    return ret;
 }
 
 int
-mq_initiate_quota_task (void *opaque)
+mq_initiate_quota_txn(xlator_t *this, loc_t *loc, struct iatt *buf)
 {
-        int32_t                 ret          = -1;
-        quota_inode_ctx_t      *ctx          = NULL;
-        inode_contribution_t   *contribution = NULL;
-        quota_synctask_t       *args         = NULL;
-        xlator_t               *this         = NULL;
-        loc_t                  *loc          = NULL;
-
-        GF_ASSERT (opaque);
-
-        args = (quota_synctask_t *) opaque;
-        loc = &args->loc;
-        this = args->this;
-        THIS = this;
-
-        ret = mq_inode_ctx_get (loc->inode, this, &ctx);
-        if (ret == -1) {
-                gf_log (this->name, GF_LOG_WARNING,
-                        "inode ctx get failed, aborting quota txn");
-                ret = -1;
-                goto out;
-        }
+    int32_t ret = -1;
 
-        /* Create the contribution node if its absent. Is it right to
-           assume that if the contribution node is not there, then
-           create one and proceed instead of returning?
-           Reason for this assumption is for hard links. Suppose
-           hard link for a file f1 present in a directory d1 is
-           created in the directory d2 (as f2). Now, since d2's
-           contribution is not there in f1's inode ctx, d2's
-           contribution xattr wont be created and will create problems
-           for quota operations.
-        */
-        contribution = mq_get_contribution_node (loc->parent, ctx);
-        if (!contribution) {
-                if (!loc_is_root(loc))
-                        gf_log_callingfn (this->name, GF_LOG_TRACE,
-                                          "contribution node for the "
-                                          "path (%s) with parent (%s) "
-                                          "not found", loc->path,
-                                          loc->parent ?
-                                          uuid_utoa (loc->parent->gfid) :
-                                          NULL);
-
-                contribution = mq_add_new_contribution_node (this, ctx, loc);
-                if (!contribution) {
-                        if (!loc_is_root(loc))
-                                gf_log_callingfn (this->name, GF_LOG_WARNING,
-                                                  "could not allocate "
-                                                  " contribution node for (%s) "
-                                                  "parent: (%s)", loc->path,
-                                                  loc->parent ?
-                                                 uuid_utoa (loc->parent->gfid) :
-                                                  NULL);
-                        goto out;
-                }
-        }
+    GF_VALIDATE_OR_GOTO("marker", this, out);
+    GF_VALIDATE_OR_GOTO("marker", loc, out);
+    GF_VALIDATE_OR_GOTO("marker", loc->inode, out);
 
-        mq_start_quota_txn_v2 (this, loc, ctx, contribution);
-
-        ret = 0;
+    ret = _mq_initiate_quota_txn(this, loc, buf, _gf_true);
 out:
-        return ret;
+    return ret;
 }
 
 int
-mq_initiate_quota_txn (xlator_t *this, loc_t *loc)
+mq_initiate_quota_blocking_txn(xlator_t *this, loc_t *loc, struct iatt *buf)
 {
-        int32_t                 ret          = -1;
+    int32_t ret = -1;
 
-        GF_VALIDATE_OR_GOTO ("marker", this, out);
-        GF_VALIDATE_OR_GOTO ("marker", loc, out);
-        GF_VALIDATE_OR_GOTO ("marker", loc->inode, out);
+    GF_VALIDATE_OR_GOTO("marker", this, out);
+    GF_VALIDATE_OR_GOTO("marker", loc, out);
+    GF_VALIDATE_OR_GOTO("marker", loc->inode, out);
 
-        ret = mq_synctask (this, mq_initiate_quota_task, _gf_true, loc, NULL,
-                           NULL, 0);
+    ret = _mq_initiate_quota_txn(this, loc, buf, _gf_false);
 out:
-        return ret;
+    return ret;
 }
 
 int
-mq_initiate_quota_blocking_txn (xlator_t *this, loc_t *loc)
-{
-        int32_t                 ret          = -1;
-
-        GF_VALIDATE_OR_GOTO ("marker", this, out);
-        GF_VALIDATE_OR_GOTO ("marker", loc, out);
-        GF_VALIDATE_OR_GOTO ("marker", loc->inode, out);
-
-        ret = mq_synctask (this, mq_initiate_quota_task, _gf_false, loc, NULL,
-                           NULL, 0);
-out:
-        return ret;
-}
-
-/* return 1 when dirty updation is performed
- * return 0 other wise
- */
-int32_t
-mq_update_dirty_inode_v2 (xlator_t *this, loc_t *loc, quota_inode_ctx_t *ctx,
-                          inode_contribution_t *contribution)
-{
-        int32_t          ret            = -1;
-        fd_t            *fd             = NULL;
-        off_t            offset         = 0;
-        loc_t            child_loc      = {0, };
-        gf_dirent_t      entries;
-        gf_dirent_t     *entry          = NULL;
-        gf_boolean_t     status         = _gf_true;
-        gf_boolean_t     locked         = _gf_false;
-        gf_boolean_t     free_entries   = _gf_false;
-        gf_boolean_t     updated        = _gf_false;
-        int32_t          dirty          = 0;
-        quota_meta_t     contri         = {0, };
-        quota_meta_t     size           = {0, };
-        quota_meta_t     contri_sum     = {0, };
-        quota_meta_t     delta          = {0, };
-
-        GF_VALIDATE_OR_GOTO ("marker", loc, out);
-        GF_VALIDATE_OR_GOTO ("marker", loc->inode, out);
-
-        ret = mq_get_ctx_updation_status (ctx, &status);
-        if (ret == -1 || status == _gf_true) {
-                ret = 0;
-                goto out;
+mq_update_dirty_inode_task(void *opaque)
+{
+    int32_t ret = -1;
+    fd_t *fd = NULL;
+    off_t offset = 0;
+    gf_dirent_t entries;
+    gf_dirent_t *entry = NULL;
+    gf_boolean_t locked = _gf_false;
+    gf_boolean_t updated = _gf_false;
+    int32_t dirty = 0;
+    quota_meta_t contri = {
+        0,
+    };
+    quota_meta_t size = {
+        0,
+    };
+    quota_meta_t contri_sum = {
+        0,
+    };
+    quota_meta_t delta = {
+        0,
+    };
+    quota_synctask_t *args = NULL;
+    xlator_t *this = NULL;
+    loc_t *loc = NULL;
+    quota_inode_ctx_t *ctx = NULL;
+    dict_t *xdata = NULL;
+    char contri_key[QUOTA_KEY_MAX] = {
+        0,
+    };
+    int keylen = 0;
+
+    GF_ASSERT(opaque);
+
+    args = (quota_synctask_t *)opaque;
+    loc = &args->loc;
+    this = args->this;
+    THIS = this;
+    INIT_LIST_HEAD(&entries.list);
+
+    ret = mq_inode_ctx_get(loc->inode, this, &ctx);
+    if (ret < 0)
+        goto out;
+
+    GET_CONTRI_KEY(this, contri_key, loc->gfid, keylen);
+    if (keylen < 0) {
+        ret = keylen;
+        goto out;
+    }
+
+    xdata = dict_new();
+    if (xdata == NULL) {
+        gf_log(this->name, GF_LOG_ERROR, "dict_new failed");
+        ret = -1;
+        goto out;
+    }
+
+    ret = dict_set_int64(xdata, contri_key, 0);
+    if (ret < 0) {
+        gf_log(this->name, GF_LOG_ERROR, "dict_set failed");
+        goto out;
+    }
+
+    ret = mq_lock(this, loc, F_WRLCK);
+    if (ret < 0)
+        goto out;
+    locked = _gf_true;
+
+    ret = mq_get_dirty(this, loc, &dirty);
+    if (ret < 0 || dirty == 0) {
+        ret = 0;
+        goto out;
+    }
+
+    fd = fd_create(loc->inode, 0);
+    if (!fd) {
+        gf_log(this->name, GF_LOG_ERROR, "Failed to create fd");
+        ret = -1;
+        goto out;
+    }
+
+    ret = syncop_opendir(this, loc, fd, NULL, NULL);
+    if (ret < 0) {
+        gf_log(this->name,
+               (-ret == ENOENT || -ret == ESTALE) ? GF_LOG_DEBUG : GF_LOG_ERROR,
+               "opendir failed "
+               "for %s: %s",
+               loc->path, strerror(-ret));
+        goto out;
+    }
+
+    fd_bind(fd);
+    while ((ret = syncop_readdirp(this, fd, 131072, offset, &entries, xdata,
+                                  NULL)) != 0) {
+        if (ret < 0) {
+            gf_log(this->name,
+                   (-ret == ENOENT || -ret == ESTALE) ? GF_LOG_DEBUG
+                                                      : GF_LOG_ERROR,
+                   "readdirp failed "
+                   "for %s: %s",
+                   loc->path, strerror(-ret));
+            goto out;
         }
 
-        if (gf_uuid_is_null (loc->gfid))
-                gf_uuid_copy (loc->gfid, loc->inode->gfid);
+        if (list_empty(&entries.list))
+            break;
 
-        if (gf_uuid_is_null (loc->gfid)) {
-                ret = -1;
-                gf_log (this->name, GF_LOG_DEBUG, "UUID is null for %s",
-                        loc->path);
-                goto out;
-        }
+        list_for_each_entry(entry, &entries.list, list)
+        {
+            offset = entry->d_off;
 
-        ret = mq_lock (this, loc, F_WRLCK);
-        if (ret < 0)
-                goto out;
-        locked = _gf_true;
+            if (!strcmp(entry->d_name, ".") || !strcmp(entry->d_name, ".."))
+                continue;
 
-        ret = mq_get_dirty (this, loc, &dirty);
-        if (ret < 0 || dirty == 0) {
-                ret = 0;
-                goto out;
-        }
+            memset(&contri, 0, sizeof(contri));
+            quota_dict_get_meta(entry->dict, contri_key, keylen, &contri);
+            if (quota_meta_is_null(&contri))
+                continue;
 
-        fd = fd_create (loc->inode, 0);
-        if (!fd) {
-                gf_log (this->name, GF_LOG_ERROR, "Failed to create fd");
-                ret = -1;
-                goto out;
+            mq_add_meta(&contri_sum, &contri);
         }
 
-        ret = syncop_opendir (this, loc, fd, NULL, NULL);
-        if (ret < 0) {
-                gf_log (this->name, (-ret == ENOENT || -ret == ESTALE)
-                        ? GF_LOG_DEBUG:GF_LOG_ERROR, "opendir failed "
-                        "for %s: %s", loc->path, strerror (-ret));
-                goto out;
-        }
+        gf_dirent_free(&entries);
+    }
+    /* Inculde for self */
+    contri_sum.dir_count++;
 
-        INIT_LIST_HEAD (&entries.list);
-        while ((ret = syncop_readdirp (this, fd, 131072, offset, &entries,
-                                       NULL, NULL)) != 0) {
-                if (ret < 0) {
-                        gf_log (this->name, (-ret == ENOENT || -ret == ESTALE)
-                                ? GF_LOG_DEBUG:GF_LOG_ERROR, "readdirp failed "
-                                "for %s: %s", loc->path, strerror (-ret));
-                        goto out;
-                }
-
-                if (list_empty (&entries.list))
-                        break;
-
-                free_entries = _gf_true;
-                list_for_each_entry (entry, &entries.list, list) {
-                        offset = entry->d_off;
-
-                        if (!strcmp (entry->d_name, ".") ||
-                            !strcmp (entry->d_name, ".."))
-                                continue;
-
-                        ret = loc_build_child (&child_loc, loc, entry->d_name);
-                        if (ret < 0) {
-                                gf_log (this->name, GF_LOG_WARNING,
-                                        "Couldn't build loc for %s/%s "
-                                        "returning from updation of dirty "
-                                        "inode", loc->path, entry->d_name);
-                                goto out;
-                        }
-
-                        ret = mq_get_contri (this, &child_loc, &contri,
-                                             loc->gfid);
-                        if (ret < 0)
-                                goto out;
-
-                        mq_add_meta (&contri_sum, &contri);
-                        loc_wipe (&child_loc);
-                }
-
-                gf_dirent_free (&entries);
-                free_entries = _gf_false;
-        }
-        /* Inculde for self */
-        contri_sum.dir_count++;
+    ret = _mq_get_metadata(this, loc, NULL, &size, 0);
+    if (ret < 0)
+        goto out;
 
-        ret = mq_get_size (this, loc, &size);
-        if (ret < 0)
-                goto out;
+    mq_compute_delta(&delta, &contri_sum, &size);
 
-        mq_compute_delta (&delta, &contri_sum, &size);
+    if (quota_meta_is_null(&delta))
+        goto out;
 
-        if (quota_meta_is_null (&delta))
-                goto out;
+    gf_log(this->name, GF_LOG_INFO,
+           "calculated size = %" PRId64 ", original size = %" PRIu64
+           ", diff = %" PRIu64 ", path = %s ",
+           contri_sum.size, size.size, delta.size, loc->path);
 
-        gf_log (this->name, GF_LOG_INFO, "calculated size = %"PRId64
-                ", original size = %"PRIu64 ", diff = %"PRIu64
-                ", path = %s ", contri_sum.size, size.size, delta.size,
-                loc->path);
+    gf_log(this->name, GF_LOG_INFO,
+           "calculated f_count = %" PRId64 ", original f_count = %" PRIu64
+           ", diff = %" PRIu64 ", path = %s ",
+           contri_sum.file_count, size.file_count, delta.file_count, loc->path);
 
-        gf_log (this->name, GF_LOG_INFO, "calculated f_count = %"PRId64
-                ", original f_count = %"PRIu64 ", diff = %"PRIu64
-                ", path = %s ", contri_sum.file_count, size.file_count,
-                delta.file_count, loc->path);
+    gf_log(this->name, GF_LOG_INFO,
+           "calculated d_count = %" PRId64 ", original d_count = %" PRIu64
+           ", diff = %" PRIu64 ", path = %s ",
+           contri_sum.dir_count, size.dir_count, delta.dir_count, loc->path);
 
-        gf_log (this->name, GF_LOG_INFO, "calculated d_count = %"PRId64
-                ", original d_count = %"PRIu64 ", diff = %"PRIu64
-                ", path = %s ", contri_sum.dir_count, size.dir_count,
-                delta.dir_count, loc->path);
+    ret = mq_update_size(this, loc, &delta);
+    if (ret < 0)
+        goto out;
 
-
-        ret = mq_update_size (this, loc, &delta);
-        if (ret < 0)
-                goto out;
-
-        updated = _gf_true;
+    updated = _gf_true;
 
 out:
-        if (free_entries)
-                gf_dirent_free (&entries);
+    gf_dirent_free(&entries);
 
-        if (fd)
-                fd_unref (fd);
+    if (fd)
+        fd_unref(fd);
 
-        if (ret >= 0 && dirty)
-                mq_mark_dirty (this, loc, 0);
+    if (xdata)
+        dict_unref(xdata);
 
-        if (locked)
-                mq_lock (this, loc, F_UNLCK);
+    if (ret < 0) {
+        /* On failure clear dirty status flag.
+         * In the next lookup inspect_directory_xattr
+         * can set the status flag and fix the
+         * dirty directory
+         */
+        if (ctx)
+            mq_set_ctx_dirty_status(ctx, _gf_false);
+    } else if (dirty) {
+        mq_mark_dirty(this, loc, 0);
+    }
 
-        if (status == _gf_false)
-                mq_set_ctx_updation_status (ctx, _gf_false);
+    if (locked)
+        mq_lock(this, loc, F_UNLCK);
 
-        loc_wipe(&child_loc);
+    if (updated)
+        mq_initiate_quota_blocking_txn(this, loc, NULL);
 
-        if (updated)
-                return 1;
-        else
-                return 0;
+    return ret;
 }
 
 int32_t
-mq_inspect_directory_xattr_task (void *opaque)
+mq_update_dirty_inode_txn(xlator_t *this, loc_t *loc, quota_inode_ctx_t *ctx)
 {
-        int32_t               ret                          = 0;
-        int8_t                dirty                        = -1;
-        quota_meta_t          size                         = {0, };
-        quota_meta_t          contri                       = {0, };
-        quota_meta_t          delta                        = {0, };
-        char                  contri_key[CONTRI_KEY_MAX]   = {0, };
-        quota_inode_ctx_t    *ctx                          = NULL;
-        inode_contribution_t *contribution                 = NULL;
-        quota_synctask_t     *args                         = NULL;
-        xlator_t             *this                         = NULL;
-        loc_t                *loc                          = NULL;
-        dict_t               *dict                         = NULL;
-
-        GF_ASSERT (opaque);
-
-        args = (quota_synctask_t *) opaque;
-        loc = &args->loc;
-        dict = args->dict;
-        this = args->this;
-        THIS = this;
-
-        ret = mq_inode_ctx_get (loc->inode, this, &ctx);
-        if (ret < 0) {
-                ctx = mq_inode_ctx_new (loc->inode, this);
-                if (ctx == NULL) {
-                        gf_log (this->name, GF_LOG_WARNING,
-                                "mq_inode_ctx_new failed");
-                        ret = -1;
-                        goto err;
-                }
-        }
+    int32_t ret = -1;
+    gf_boolean_t status = _gf_true;
 
-        if (!loc_is_root(loc)) {
-                contribution = mq_add_new_contribution_node (this, ctx, loc);
-                if (contribution == NULL) {
-                        if (!gf_uuid_is_null (loc->inode->gfid))
-                                gf_log (this->name, GF_LOG_DEBUG,
-                                        "cannot add a new contribution node "
-                                        "(%s)", uuid_utoa (loc->inode->gfid));
-                        ret = -1;
-                        goto err;
-                }
-        }
+    GF_VALIDATE_OR_GOTO("marker", loc, out);
+    GF_VALIDATE_OR_GOTO("marker", loc->inode, out);
 
-        ret = dict_get_int8 (dict, QUOTA_DIRTY_KEY, &dirty);
-        if (ret < 0)
-                goto out;
+    mq_test_and_set_ctx_status(ctx, &ctx->dirty_status, &status);
+    if (status == _gf_true)
+        goto out;
 
-        ret = quota_dict_get_meta (dict, QUOTA_SIZE_KEY, &size);
+    ret = mq_synctask(this, mq_update_dirty_inode_task, _gf_true, loc);
+out:
+    if (ret < 0 && status == _gf_false)
+        mq_set_ctx_dirty_status(ctx, _gf_false);
+
+    return ret;
+}
+
+int32_t
+mq_inspect_directory_xattr(xlator_t *this, quota_inode_ctx_t *ctx,
+                           inode_contribution_t *contribution, loc_t *loc,
+                           dict_t *dict)
+{
+    int32_t ret = -1;
+    int8_t dirty = -1;
+    quota_meta_t size = {
+        0,
+    };
+    quota_meta_t contri = {
+        0,
+    };
+    quota_meta_t delta = {
+        0,
+    };
+    char contri_key[QUOTA_KEY_MAX] = {
+        0,
+    };
+    char size_key[QUOTA_KEY_MAX] = {
+        0,
+    };
+    int keylen = 0;
+    gf_boolean_t status = _gf_false;
+
+    ret = dict_get_int8(dict, QUOTA_DIRTY_KEY, &dirty);
+    if (ret < 0) {
+        /* dirty is set only on the first file write operation
+         * so ignore this error
+         */
+        ret = 0;
+        dirty = 0;
+    }
+
+    GET_SIZE_KEY(this, size_key, keylen);
+    if (keylen < 0) {
+        ret = -1;
+        goto out;
+    }
+    ret = _quota_dict_get_meta(this, dict, size_key, keylen, &size, IA_IFDIR,
+                               _gf_false);
+    if (ret < 0)
+        goto create_xattr;
+
+    if (!contribution)
+        goto create_xattr;
+
+    if (!loc_is_root(loc)) {
+        GET_CONTRI_KEY(this, contri_key, contribution->gfid, keylen);
+        if (keylen < 0) {
+            ret = -1;
+            goto out;
+        }
+        ret = _quota_dict_get_meta(this, dict, contri_key, keylen, &contri,
+                                   IA_IFDIR, _gf_false);
         if (ret < 0)
-                goto out;
+            goto create_xattr;
 
-        if (!loc_is_root(loc)) {
-                GET_CONTRI_KEY (contri_key, contribution->gfid, ret);
-                if (ret < 0)
-                        goto err;
-
-                ret = quota_dict_get_meta (dict, contri_key, &contri);
-                if (ret < 0)
-                        goto out;
-
-                LOCK (&contribution->lock);
-                {
-                        contribution->contribution = contri.size;
-                        contribution->file_count = contri.file_count;
-                        contribution->dir_count = contri.dir_count;
-                }
-                UNLOCK (&contribution->lock);
-        }
-
-        LOCK (&ctx->lock);
+        LOCK(&contribution->lock);
         {
-                ctx->size = size.size;
-                ctx->file_count = size.file_count;
-                ctx->dir_count = size.dir_count;
-                ctx->dirty = dirty;
-        }
-        UNLOCK (&ctx->lock);
-
-        mq_compute_delta (&delta, &size, &contri);
-
-        if (dirty)
-                ret = mq_update_dirty_inode_v2 (this, loc, ctx, contribution);
-
-        if ((!dirty || ret == 1) &&
-            !loc_is_root(loc) &&
-            !quota_meta_is_null (&delta))
-                mq_initiate_quota_blocking_txn (this, loc);
-
+            contribution->contribution = contri.size;
+            contribution->file_count = contri.file_count;
+            contribution->dir_count = contri.dir_count;
+        }
+        UNLOCK(&contribution->lock);
+    }
+
+    LOCK(&ctx->lock);
+    {
+        ctx->size = size.size;
+        ctx->file_count = size.file_count;
+        ctx->dir_count = size.dir_count;
+        ctx->dirty = dirty;
+    }
+    UNLOCK(&ctx->lock);
+
+    ret = mq_get_ctx_updation_status(ctx, &status);
+    if (ret < 0 || status == _gf_true) {
+        /* If the update txn is in progress abort inspection */
         ret = 0;
-out:
-        if (ret < 0)
-                ret = mq_create_xattrs_blocking_txn (this, loc);
+        goto out;
+    }
 
-err:
-        return ret;
-}
+    mq_compute_delta(&delta, &size, &contri);
 
-int32_t
-mq_inspect_directory_xattr_txn (xlator_t *this, loc_t *loc, dict_t *dict,
-                                struct iatt buf)
-{
-        int32_t   ret = -1;
+    if (dirty) {
+        ret = mq_update_dirty_inode_txn(this, loc, ctx);
+        goto out;
+    }
 
-        ret = mq_synctask (this, mq_inspect_directory_xattr_task, _gf_true,
-                           loc, dict, &buf, 0);
+    if (!loc_is_root(loc) && !quota_meta_is_null(&delta))
+        mq_initiate_quota_txn(this, loc, NULL);
 
-        return ret;
-}
+    ret = 0;
+    goto out;
 
-int32_t
-mq_inspect_file_xattr_task (void *opaque)
-{
-        int32_t               ret                          = -1;
-        quota_meta_t          size                         = {0, };
-        quota_meta_t          contri                       = {0, };
-        quota_meta_t          delta                        = {0, };
-        char                  contri_key[CONTRI_KEY_MAX]   = {0, };
-        quota_inode_ctx_t    *ctx                          = NULL;
-        inode_contribution_t *contribution                 = NULL;
-        quota_synctask_t     *args                         = NULL;
-        xlator_t             *this                         = NULL;
-        loc_t                *loc                          = NULL;
-        dict_t               *dict                         = NULL;
-        struct iatt           buf                          = {0,};
-
-        GF_ASSERT (opaque);
-
-        args = (quota_synctask_t *) opaque;
-        loc = &args->loc;
-        dict = args->dict;
-        buf = args->buf;
-        this = args->this;
-        THIS = this;
-
-        ret = mq_inode_ctx_get (loc->inode, this, &ctx);
-        if (ret < 0) {
-                ctx = mq_inode_ctx_new (loc->inode, this);
-                if (ctx == NULL) {
-                        gf_log (this->name, GF_LOG_WARNING,
-                                "mq_inode_ctx_new failed");
-                        ret = -1;
-                        goto out;
-                }
-        }
-
-        contribution = mq_add_new_contribution_node (this, ctx, loc);
-        if (contribution == NULL) {
-                gf_log_callingfn (this->name, GF_LOG_DEBUG, "cannot allocate "
-                                  "contribution node (path:%s)", loc->path);
-                ret = -1;
-                goto out;
-        }
+create_xattr:
+    if (ret < 0)
+        ret = mq_create_xattrs_txn(this, loc, NULL);
 
-        LOCK (&ctx->lock);
+out:
+    return ret;
+}
+
+int32_t
+mq_inspect_file_xattr(xlator_t *this, quota_inode_ctx_t *ctx,
+                      inode_contribution_t *contribution, loc_t *loc,
+                      dict_t *dict, struct iatt *buf)
+{
+    int32_t ret = -1;
+    quota_meta_t size = {
+        0,
+    };
+    quota_meta_t contri = {
+        0,
+    };
+    quota_meta_t delta = {
+        0,
+    };
+    char contri_key[QUOTA_KEY_MAX] = {
+        0,
+    };
+    int keylen = 0;
+    gf_boolean_t status = _gf_false;
+
+    if (!buf || !contribution || !ctx)
+        goto out;
+
+    LOCK(&ctx->lock);
+    {
+        ctx->size = 512 * buf->ia_blocks;
+        ctx->file_count = 1;
+        ctx->dir_count = 0;
+
+        size.size = ctx->size;
+        size.file_count = ctx->file_count;
+        size.dir_count = ctx->dir_count;
+    }
+    UNLOCK(&ctx->lock);
+
+    GET_CONTRI_KEY(this, contri_key, contribution->gfid, keylen);
+    if (keylen < 0) {
+        ret = -1;
+        goto out;
+    }
+
+    ret = _quota_dict_get_meta(this, dict, contri_key, keylen, &contri,
+                               IA_IFREG, _gf_true);
+    if (ret < 0) {
+        ret = mq_create_xattrs_txn(this, loc, NULL);
+    } else {
+        LOCK(&contribution->lock);
         {
-                ctx->size = 512 * buf.ia_blocks;
-                ctx->file_count = 1;
-                ctx->dir_count = 0;
-
-                size.size = ctx->size;
-                size.file_count = ctx->file_count;
-                size.dir_count = ctx->dir_count;
+            contribution->contribution = contri.size;
+            contribution->file_count = contri.file_count;
+            contribution->dir_count = contri.dir_count;
         }
-        UNLOCK (&ctx->lock);
-
-        list_for_each_entry (contribution, &ctx->contribution_head,
-                             contri_list) {
-
-                GET_CONTRI_KEY (contri_key, contribution->gfid, ret);
-                if (ret < 0)
-                        continue;
-
-                ret = quota_dict_get_meta (dict, contri_key, &contri);
-                if (ret < 0) {
-                        ret = mq_create_xattrs_blocking_txn (this, loc);
-                } else {
-                        LOCK (&contribution->lock);
-                        {
-                                contribution->contribution = contri.size;
-                                contribution->file_count = contri.file_count;
-                                contribution->dir_count = contri.dir_count;
-                        }
-                        UNLOCK (&contribution->lock);
-
-                        mq_compute_delta (&delta, &size, &contri);
-                        if (!quota_meta_is_null (&delta)) {
-                                mq_initiate_quota_blocking_txn (this, loc);
-                                /* TODO: revist this code when fixing hardlinks
-                                 */
-                                break;
-                        }
-                }
-
-                /* TODO: loc->parent might need to be assigned to corresponding
-                 * contribution inode. We need to handle hard links here
-                */
-        }
-
-out:
-
-        return ret;
-}
-
-int32_t
-mq_inspect_file_xattr_txn (xlator_t *this, loc_t *loc, dict_t *dict,
-                           struct iatt buf)
-{
-        int32_t   ret = -1;
-
-        ret = mq_synctask (this, mq_inspect_file_xattr_task, _gf_true,
-                           loc, dict, &buf, 0);
+        UNLOCK(&contribution->lock);
 
-        return ret;
-}
-
-int32_t
-mq_xattr_state (xlator_t *this, loc_t *loc, dict_t *dict, struct iatt buf)
-{
-        if (((buf.ia_type == IA_IFREG) && !dht_is_linkfile (&buf, dict))
-            || (buf.ia_type == IA_IFLNK))  {
-                mq_inspect_file_xattr_txn (this, loc, dict, buf);
-        } else if (buf.ia_type == IA_IFDIR)
-                mq_inspect_directory_xattr_txn (this, loc, dict, buf);
-
-        return 0;
-}
-
-int32_t
-mq_req_xattr (xlator_t *this, loc_t *loc, dict_t *dict,
-              char *contri_key)
-{
-        int32_t               ret       = -1;
-
-        GF_VALIDATE_OR_GOTO ("marker", this, out);
-        GF_VALIDATE_OR_GOTO ("marker", loc, out);
-        GF_VALIDATE_OR_GOTO ("marker", dict, out);
-
-        if (!loc_is_root(loc)) {
-                ret = mq_dict_set_contribution (this, dict, loc, NULL,
-                                                contri_key);
-                if (ret < 0)
-                        goto out;
+        ret = mq_get_ctx_updation_status(ctx, &status);
+        if (ret < 0 || status == _gf_true) {
+            /* If the update txn is in progress abort inspection */
+            ret = 0;
+            goto out;
         }
 
-        ret = dict_set_uint64 (dict, QUOTA_SIZE_KEY, 0);
-        if (ret < 0)
-                goto out;
-
-        ret = dict_set_int8 (dict, QUOTA_DIRTY_KEY, 0);
+        mq_compute_delta(&delta, &size, &contri);
+        if (!quota_meta_is_null(&delta))
+            mq_initiate_quota_txn(this, loc, NULL);
+    }
+    /* TODO: revist this code when fixing hardlinks */
 
 out:
-        if (ret < 0)
-                gf_log_callingfn (this->name, GF_LOG_ERROR, "dict set failed");
-
-        return ret;
+    return ret;
 }
 
-
 int32_t
-mq_removexattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                    int32_t op_ret, int32_t op_errno, dict_t *xdata)
+mq_xattr_state(xlator_t *this, loc_t *origin_loc, dict_t *dict,
+               struct iatt *buf)
 {
-        QUOTA_STACK_DESTROY (frame, this);
+    int32_t ret = -1;
+    quota_inode_ctx_t *ctx = NULL;
+    loc_t loc = {
+        0,
+    };
+    inode_contribution_t *contribution = NULL;
 
-        return 0;
-}
+    ret = mq_prevalidate_txn(this, origin_loc, &loc, &ctx, buf);
+    if (ret < 0 || loc.parent == NULL)
+        goto out;
 
-int32_t
-_mq_inode_remove_done (call_frame_t *frame, void *cookie, xlator_t *this,
-                       int32_t op_ret, int32_t op_errno, dict_t *xdata)
-{
-        int32_t        ret                           = 0;
-        char           contri_key[CONTRI_KEY_MAX]    = {0, };
-        quota_local_t *local                         = NULL;
-        inode_t       *inode                         = NULL;
-        dentry_t      *tmp                           = NULL;
-        gf_boolean_t  last_dentry                    = _gf_true;
-        loc_t         loc                            = {0, };
-        dentry_t      *other_dentry                  = NULL;
-        gf_boolean_t  remove                         = _gf_false;
-
-        local = (quota_local_t *) frame->local;
-
-        if (op_ret == -1 || local->err == -1) {
-                mq_removexattr_cbk (frame, NULL, this, -1, 0, NULL);
-                return 0;
-        }
-
-        frame->local = NULL;
-
-        GET_CONTRI_KEY (contri_key, local->contri->gfid, ret);
-
-        if (!local->loc.inode)
-                inode = inode_grep (local->loc.parent->table, local->loc.parent,
-                                    local->loc.name);
+    if (!loc_is_root(&loc)) {
+        contribution = mq_add_new_contribution_node(this, ctx, &loc);
+        if (contribution == NULL) {
+            if (!gf_uuid_is_null(loc.inode->gfid))
+                gf_log(this->name, GF_LOG_WARNING,
+                       "cannot add a new contribution node "
+                       "(%s)",
+                       uuid_utoa(loc.gfid));
+            ret = -1;
+            goto out;
+        }
+        if (buf->ia_type == IA_IFDIR)
+            mq_inspect_directory_xattr(this, ctx, contribution, &loc, dict);
         else
-                inode = inode_ref (local->loc.inode);
-
-        /* Suppose there are 2 directories dir1 and dir2. Quota limit is set on
-           both the directories. There is a file (f1) in dir1. A hark link is
-           created for that file inside the directory dir2 (say f2). Now one
-           more xattr is set in the inode as a new hard link is created in a
-           separate directory.
-           i.e trusted.glusterfs.quota.<gfid of dir2>.contri=<contribution>
-
-           Now when the hardlink f2 is removed, then the new xattr added (i.e
-           the xattr indicating its contribution to ITS parent directory) should
-           be removed (IFF there is not another hardlink for that file in the
-           same directory).
-
-           To do that upon getting unlink first check whether any other hard
-           links for the same inode exists in the same directory. If so do not
-           do anything and proceed for quota transaction.
-           Otherwise, if the removed entry was the only link for that inode
-           within that directory, then get another dentry for the inode
-           (by traversing the list of dentries for the inode) and using the
-           the dentry's parent and name, send removexattr so that the xattr
-           is removed.
-
-           If it is not done, then if the volume is restarted or the brick
-           process is restarted, then wrong quota usage will be shown for the
-           directory dir2.
-        */
-        if (inode) {
-                tmp = NULL;
-                list_for_each_entry (tmp, &inode->dentry_list, inode_list) {
-                        if (local->loc.parent == tmp->parent) {
-                                if (strcmp (local->loc.name, local->loc.name)) {
-                                        last_dentry = _gf_false;
-                                        break;
-                                }
-                        }
-                }
-                remove = last_dentry;
-        }
+            mq_inspect_file_xattr(this, ctx, contribution, &loc, dict, buf);
+    } else {
+        mq_inspect_directory_xattr(this, ctx, 0, &loc, dict);
+    }
 
-        if (remove) {
-                if (!other_dentry)    {
-                        list_for_each_entry (tmp, &inode->dentry_list,
-                                             inode_list) {
-                                if (local->loc.parent != tmp->parent) {
-                                        other_dentry = tmp;
-                                        break;
-                                }
-                        }
-                }
-
-                if (!other_dentry)
-                        mq_removexattr_cbk (frame, NULL, this, 0, 0, NULL);
-                else {
-                        loc.parent = inode_ref (other_dentry->parent);
-                        loc.name = gf_strdup (other_dentry->name);
-                        gf_uuid_copy (loc.pargfid , other_dentry->parent->gfid);
-                        loc.inode = inode_ref (inode);
-                        gf_uuid_copy (loc.gfid, inode->gfid);
-                        inode_path (other_dentry->parent, other_dentry->name,
-                                    (char **)&loc.path);
-
-                        STACK_WIND (frame, mq_removexattr_cbk,
-                                    FIRST_CHILD(this),
-                                    FIRST_CHILD(this)->fops->removexattr,
-                                    &loc, contri_key, NULL);
-                }
-        } else
-                mq_removexattr_cbk (frame, NULL, this, 0, 0, NULL);
-
-        ret = 0;
-
-        if (strcmp (local->parent_loc.path, "/") != 0) {
-                ret = mq_get_parent_inode_local (this, local);
-                if (ret < 0)
-                        goto out;
-
-                mq_start_quota_txn (this, &local->loc, local->ctx, local->contri);
-        }
 out:
-        mq_local_unref (this, local);
-
-        loc_wipe (&loc);
-        inode_unref (inode);
-        return 0;
-}
-
-int32_t
-mq_inode_remove_done (call_frame_t *frame, void *cookie, xlator_t *this,
-                      int32_t op_ret, int32_t op_errno, dict_t *dict,
-                      dict_t *xdata)
-{
-        int32_t            ret   = -1;
-        struct gf_flock    lock  = {0, };
-        quota_inode_ctx_t *ctx   = NULL;
-        quota_local_t     *local = NULL;
-        int64_t contribution = 0;
+    loc_wipe(&loc);
 
-        local = frame->local;
-        if (op_ret == -1)
-                local->err = -1;
+    if (contribution)
+        GF_REF_PUT(contribution);
 
-        ret = mq_inode_ctx_get (local->parent_loc.inode, this, &ctx);
-
-        LOCK (&local->contri->lock);
-        {
-                contribution = local->contri->contribution;
-        }
-        UNLOCK (&local->contri->lock);
-
-        if (contribution == local->size) {
-                if (ret == 0) {
-                        LOCK (&ctx->lock);
-                        {
-                                ctx->size -= contribution;
-                        }
-                        UNLOCK (&ctx->lock);
-
-                        LOCK (&local->contri->lock);
-                        {
-                                local->contri->contribution = 0;
-                        }
-                        UNLOCK (&local->contri->lock);
-                }
-        }
-
-        lock.l_type   = F_UNLCK;
-        lock.l_whence = SEEK_SET;
-        lock.l_start  = 0;
-        lock.l_len    = 0;
-        lock.l_pid    = 0;
-
-        STACK_WIND (frame,
-                    _mq_inode_remove_done,
-                    FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->inodelk,
-                    this->name, &local->parent_loc,
-                    F_SETLKW, &lock, NULL);
-        return 0;
+    return ret;
 }
 
 int32_t
-mq_reduce_parent_size_xattr (call_frame_t *frame, void *cookie, xlator_t *this,
-                             int32_t op_ret, int32_t op_errno, dict_t *xdata)
+mq_req_xattr(xlator_t *this, loc_t *loc, dict_t *dict, char *contri_key,
+             char *size_key)
 {
-        int32_t                  ret               = -1;
-        int64_t                 *size              = NULL;
-        dict_t                  *dict              = NULL;
-        quota_local_t           *local             = NULL;
-
-        local = frame->local;
-        if (op_ret == -1) {
-                gf_log (this->name, GF_LOG_WARNING,
-                        "inodelk set failed on %s", local->parent_loc.path);
-                QUOTA_STACK_DESTROY (frame, this);
-                return 0;
-        }
+    int32_t ret = -1;
+    char key[QUOTA_KEY_MAX] = {
+        0,
+    };
 
-        VALIDATE_OR_GOTO (local->contri, err);
+    GF_VALIDATE_OR_GOTO("marker", this, out);
+    GF_VALIDATE_OR_GOTO("marker", loc, out);
+    GF_VALIDATE_OR_GOTO("marker", dict, out);
 
-        dict = dict_new ();
-        if (dict == NULL) {
-                gf_log (this->name, GF_LOG_ERROR, "dict_new failed");
-                ret = -1;
-                goto err;
-        }
-
-        QUOTA_ALLOC_OR_GOTO (size, int64_t, ret, err);
-
-        *size = hton64 (-local->size);
-
-        ret = dict_set_bin (dict, QUOTA_SIZE_KEY, size, 8);
+    if (!loc_is_root(loc)) {
+        ret = mq_dict_set_contribution(this, dict, loc, NULL, contri_key);
         if (ret < 0)
-                goto err;
-
-        gf_uuid_copy (local->parent_loc.gfid,
-                   local->parent_loc.inode->gfid);
-        GF_UUID_ASSERT (local->parent_loc.gfid);
-
-        STACK_WIND (frame, mq_inode_remove_done, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->xattrop, &local->parent_loc,
-                    GF_XATTROP_ADD_ARRAY64, dict, NULL);
-        dict_unref (dict);
-        return 0;
-
-err:
-        local->err = 1;
-        mq_inode_remove_done (frame, NULL, this, -1, 0, NULL, NULL);
-        if (dict)
-                dict_unref (dict);
-        return 0;
-}
-
-int32_t
-mq_reduce_parent_size (xlator_t *this, loc_t *loc, int64_t contri)
-{
-        int32_t                  ret           = -1;
-        struct gf_flock          lock          = {0,};
-        call_frame_t            *frame         = NULL;
-        quota_local_t           *local         = NULL;
-        quota_inode_ctx_t       *ctx           = NULL;
-        inode_contribution_t    *contribution  = NULL;
-
-        GF_VALIDATE_OR_GOTO ("marker", this, out);
-        GF_VALIDATE_OR_GOTO ("marker", loc, out);
-
-        ret = mq_inode_ctx_get (loc->inode, this, &ctx);
-        if (ret < 0)
-                goto out;
-
-        contribution = mq_get_contribution_node (loc->parent, ctx);
-        if (contribution == NULL) {
-                gf_log_callingfn (this->name, GF_LOG_WARNING, "contribution for"
-                                  " the node %s is NULL", loc->path);
-                goto out;
-        }
-
-        local = mq_local_new ();
-        if (local == NULL) {
-                ret = -1;
-                goto out;
-        }
-
-        if (contri >= 0) {
-                local->size = contri;
-        } else {
-                LOCK (&contribution->lock);
-                {
-                        local->size = contribution->contribution;
-                }
-                UNLOCK (&contribution->lock);
-        }
-
-        if (local->size == 0) {
-                gf_log_callingfn (this->name, GF_LOG_TRACE,
-                                  "local->size is 0 " "path: (%s)", loc->path);
-                ret = 0;
-                goto out;
-        }
-
-        ret = mq_loc_copy (&local->loc, loc);
-        if (ret < 0)
-                goto out;
-
-        local->ctx = ctx;
-        local->contri = contribution;
-
-        ret = mq_inode_loc_fill (NULL, loc->parent, &local->parent_loc);
-        if (ret < 0) {
-                gf_log_callingfn (this->name, GF_LOG_INFO, "building parent loc"
-                                  " failed. (gfid: %s)",
-                                  uuid_utoa (loc->parent->gfid));
-                goto out;
-        }
+            goto out;
+    }
 
-        frame = create_frame (this, this->ctx->pool);
-        if (!frame) {
-                ret = -1;
-                goto out;
+    GET_SIZE_KEY(this, key, ret);
+    if (ret < 0)
+        goto out;
+    if (size_key)
+        if (snprintf(size_key, QUOTA_KEY_MAX, "%s", key) >= QUOTA_KEY_MAX) {
+            ret = -1;
+            goto out;
         }
 
-        mq_assign_lk_owner (this, frame);
+    ret = dict_set_uint64(dict, key, 0);
+    if (ret < 0)
+        goto out;
 
-        frame->local = local;
-
-        lock.l_len    = 0;
-        lock.l_start  = 0;
-        lock.l_type   = F_WRLCK;
-        lock.l_whence = SEEK_SET;
-
-        if (local->parent_loc.inode == NULL) {
-                ret = -1;
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "Inode is NULL, so can't stackwind.");
-                goto out;
-        }
-
-        STACK_WIND (frame,
-                    mq_reduce_parent_size_xattr,
-                    FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->inodelk,
-                    this->name, &local->parent_loc, F_SETLKW, &lock, NULL);
-        local = NULL;
-        ret = 0;
+    ret = dict_set_int8(dict, QUOTA_DIRTY_KEY, 0);
 
 out:
-        if (local != NULL)
-                mq_local_unref (this, local);
-
-        return ret;
-}
-
-
-int32_t
-init_quota_priv (xlator_t *this)
-{
-        return 0;
+    if (ret < 0)
+        gf_log_callingfn(this ? this->name : "Marker", GF_LOG_ERROR,
+                         "dict set failed");
+    return ret;
 }
 
-
 int32_t
-mq_rename_update_newpath (xlator_t *this, loc_t *loc)
+mq_forget(xlator_t *this, quota_inode_ctx_t *ctx)
 {
-        int32_t               ret          = -1;
-        quota_inode_ctx_t    *ctx          = NULL;
-        inode_contribution_t *contribution = NULL;
+    inode_contribution_t *contri = NULL;
+    inode_contribution_t *next = NULL;
 
-        GF_VALIDATE_OR_GOTO ("marker", this, out);
-        GF_VALIDATE_OR_GOTO ("marker", loc, out);
-        GF_VALIDATE_OR_GOTO ("marker", loc->inode, out);
+    GF_VALIDATE_OR_GOTO("marker", this, out);
+    GF_VALIDATE_OR_GOTO("marker", ctx, out);
 
-        ret = mq_inode_ctx_get (loc->inode, this, &ctx);
-        if (ret < 0)
-                goto out;
-
-        contribution = mq_add_new_contribution_node (this, ctx, loc);
-        if (contribution == NULL) {
-                ret = -1;
-                goto out;
-        }
-
-        mq_initiate_quota_txn (this, loc);
-out:
-        return ret;
-}
-
-int32_t
-mq_forget (xlator_t *this, quota_inode_ctx_t *ctx)
-{
-        inode_contribution_t *contri = NULL;
-        inode_contribution_t *next   = NULL;
-
-        GF_VALIDATE_OR_GOTO ("marker", this, out);
-        GF_VALIDATE_OR_GOTO ("marker", ctx, out);
-
-        list_for_each_entry_safe (contri, next, &ctx->contribution_head,
-                                  contri_list) {
-                list_del (&contri->contri_list);
-                GF_FREE (contri);
-        }
+    list_for_each_entry_safe(contri, next, &ctx->contribution_head, contri_list)
+    {
+        list_del_init(&contri->contri_list);
+        GF_REF_PUT(contri);
+    }
 
-        LOCK_DESTROY (&ctx->lock);
-        GF_FREE (ctx);
+    LOCK_DESTROY(&ctx->lock);
+    GF_FREE(ctx);
 out:
-        return 0;
+    return 0;
 }
diff --git a/xlators/features/marker/src/marker-quota.h b/xlators/features/marker/src/marker-quota.h
index a81db7fa684..4bbf6878b22 100644
--- a/xlators/features/marker/src/marker-quota.h
+++ b/xlators/features/marker/src/marker-quota.h
@@ -10,149 +10,131 @@
 #ifndef _MARKER_QUOTA_H
 #define _MARKER_QUOTA_H
 
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
-
-#include "xlator.h"
+#include <glusterfs/xlator.h>
 #include "marker-mem-types.h"
+#include <glusterfs/refcount.h>
+#include <glusterfs/quota-common-utils.h>
+#include <glusterfs/call-stub.h>
 
 #define QUOTA_XATTR_PREFIX "trusted.glusterfs"
 #define QUOTA_DIRTY_KEY "trusted.glusterfs.quota.dirty"
 
-#define CONTRIBUTION  "contri"
-#define CONTRI_KEY_MAX 512
+#define CONTRIBUTION "contri"
+#define QUOTA_KEY_MAX 512
 #define READDIR_BUF 4096
 
-
-#define QUOTA_STACK_DESTROY(_frame, _this)              \
-        do {                                            \
-                quota_local_t *_local = NULL;           \
-                _local = _frame->local;                 \
-                _frame->local = NULL;                   \
-                STACK_DESTROY (_frame->root);           \
-                mq_local_unref (_this, _local);         \
-        } while (0)
-
-
-#define QUOTA_ALLOC(var, type, ret)                     \
-        do {                                            \
-                ret = 0;                                \
-                var = GF_CALLOC (sizeof (type), 1,      \
-                                 gf_marker_mt_##type);  \
-                if (!var) {                             \
-                        ret = -1;                       \
-                }                                       \
-        } while (0);
-
-#define QUOTA_ALLOC_OR_GOTO(var, type, ret, label)      \
-        do {                                            \
-                var = GF_CALLOC (sizeof (type), 1,      \
-                                 gf_marker_mt_##type);  \
-                if (!var) {                             \
-                        gf_log ("", GF_LOG_ERROR,       \
-                                "out of memory");       \
-                        ret = -1;                       \
-                        goto label;                     \
-                }                                       \
-                ret = 0;                                \
-        } while (0);
-
-#define GET_CONTRI_KEY(var, _gfid, _ret)                                  \
-        do {                                                              \
-                if (_gfid != NULL) {                                      \
-                        char _gfid_unparsed[40];                          \
-                        gf_uuid_unparse (_gfid, _gfid_unparsed);          \
-                        _ret = snprintf (var, CONTRI_KEY_MAX,             \
-                                         QUOTA_XATTR_PREFIX               \
-                                         ".%s.%s." CONTRIBUTION, "quota", \
-                                         _gfid_unparsed);                 \
-                } else {                                                  \
-                        _ret = snprintf (var, CONTRI_KEY_MAX,             \
-                                         QUOTA_XATTR_PREFIX               \
-                                         ".%s.." CONTRIBUTION, "quota");  \
-                }                                                         \
-        } while (0)
-
-#define QUOTA_SAFE_INCREMENT(lock, var)         \
-        do {                                    \
-                LOCK (lock);                    \
-                var ++;                         \
-                UNLOCK (lock);                  \
-        } while (0)
+#define QUOTA_ALLOC(var, type, ret)                                            \
+    do {                                                                       \
+        ret = 0;                                                               \
+        var = GF_CALLOC(sizeof(type), 1, gf_marker_mt_##type);                 \
+        if (!var) {                                                            \
+            ret = -1;                                                          \
+        }                                                                      \
+    } while (0);
+
+#define QUOTA_ALLOC_OR_GOTO(var, type, ret, label)                             \
+    do {                                                                       \
+        var = GF_CALLOC(sizeof(type), 1, gf_marker_mt_##type);                 \
+        if (!var) {                                                            \
+            gf_log("", GF_LOG_ERROR, "out of memory");                         \
+            ret = -1;                                                          \
+            goto label;                                                        \
+        }                                                                      \
+        ret = 0;                                                               \
+    } while (0);
+
+#define GET_QUOTA_KEY(_this, var, key, _ret)                                   \
+    do {                                                                       \
+        marker_conf_t *_priv = _this->private;                                 \
+        if (_priv->version > 0)                                                \
+            _ret = snprintf(var, QUOTA_KEY_MAX, "%s.%d", key, _priv->version); \
+        else                                                                   \
+            _ret = snprintf(var, QUOTA_KEY_MAX, "%s", key);                    \
+    } while (0)
+
+#define GET_CONTRI_KEY(_this, var, _gfid, _ret)                                \
+    do {                                                                       \
+        char _tmp_var[QUOTA_KEY_MAX] = {                                       \
+            0,                                                                 \
+        };                                                                     \
+        if (_gfid != NULL) {                                                   \
+            char _gfid_unparsed[40];                                           \
+            gf_uuid_unparse(_gfid, _gfid_unparsed);                            \
+            _ret = snprintf(_tmp_var, QUOTA_KEY_MAX,                           \
+                            QUOTA_XATTR_PREFIX ".%s.%s." CONTRIBUTION,         \
+                            "quota", _gfid_unparsed);                          \
+        } else {                                                               \
+            _ret = snprintf(_tmp_var, QUOTA_KEY_MAX,                           \
+                            QUOTA_XATTR_PREFIX ".%s.." CONTRIBUTION, "quota"); \
+        }                                                                      \
+        GET_QUOTA_KEY(_this, var, _tmp_var, _ret);                             \
+    } while (0)
+
+#define GET_SIZE_KEY(_this, var, _ret)                                         \
+    {                                                                          \
+        GET_QUOTA_KEY(_this, var, QUOTA_SIZE_KEY, _ret);                       \
+    }
+
+#define QUOTA_SAFE_INCREMENT(lock, var)                                        \
+    do {                                                                       \
+        LOCK(lock);                                                            \
+        var++;                                                                 \
+        UNLOCK(lock);                                                          \
+    } while (0)
 
 struct quota_inode_ctx {
-        int64_t                size;
-        int64_t                file_count;
-        int64_t                dir_count;
-        int8_t                 dirty;
-        gf_boolean_t           updation_status;
-        gf_lock_t              lock;
-        struct list_head       contribution_head;
+    int64_t size;
+    int64_t file_count;
+    int64_t dir_count;
+    int8_t dirty;
+    gf_boolean_t create_status;
+    gf_boolean_t updation_status;
+    gf_boolean_t dirty_status;
+    gf_lock_t lock;
+    struct list_head contribution_head;
 };
 typedef struct quota_inode_ctx quota_inode_ctx_t;
 
 struct quota_synctask {
-        xlator_t      *this;
-        loc_t          loc;
-        dict_t        *dict;
-        struct iatt    buf;
-        int64_t        contri;
-        gf_boolean_t   is_static;
+    xlator_t *this;
+    loc_t loc;
+    quota_meta_t contri;
+    gf_boolean_t is_static;
+    uint32_t ia_nlink;
+    call_stub_t *stub;
 };
 typedef struct quota_synctask quota_synctask_t;
 
 struct inode_contribution {
-        struct list_head contri_list;
-        int64_t          contribution;
-        int64_t          file_count;
-        int64_t          dir_count;
-        uuid_t           gfid;
-  gf_lock_t lock;
+    struct list_head contri_list;
+    int64_t contribution;
+    int64_t file_count;
+    int64_t dir_count;
+    uuid_t gfid;
+    gf_lock_t lock;
+    GF_REF_DECL;
 };
 typedef struct inode_contribution inode_contribution_t;
 
 int32_t
-mq_get_lock_on_parent (call_frame_t *, xlator_t *);
-
-int32_t
-mq_req_xattr (xlator_t *, loc_t *, dict_t *, char *);
-
-int32_t
-init_quota_priv (xlator_t *);
-
-int32_t
-mq_xattr_state (xlator_t *, loc_t *, dict_t *, struct iatt);
+mq_req_xattr(xlator_t *, loc_t *, dict_t *, char *, char *);
 
 int32_t
-mq_set_inode_xattr (xlator_t *, loc_t *);
+mq_xattr_state(xlator_t *, loc_t *, dict_t *, struct iatt *);
 
 int
-mq_initiate_quota_txn (xlator_t *, loc_t *);
+mq_initiate_quota_txn(xlator_t *, loc_t *, struct iatt *);
 
 int
-mq_initiate_quota_blocking_txn (xlator_t *, loc_t *);
+mq_initiate_quota_blocking_txn(xlator_t *, loc_t *, struct iatt *);
 
 int
-mq_create_xattrs_txn (xlator_t *this, loc_t *loc);
-
-int32_t
-mq_dirty_inode_readdir (call_frame_t *, void *, xlator_t *,
-                        int32_t, int32_t, fd_t *, dict_t *);
-
-int32_t
-mq_reduce_parent_size (xlator_t *, loc_t *, int64_t);
-
-int32_t
-mq_reduce_parent_size_txn (xlator_t *, loc_t *, int64_t);
-
-int32_t
-mq_rename_update_newpath (xlator_t *, loc_t *);
+mq_create_xattrs_txn(xlator_t *this, loc_t *loc, struct iatt *buf);
 
 int32_t
-mq_inspect_file_xattr (xlator_t *this, loc_t *loc, dict_t *dict, struct iatt buf);
+mq_reduce_parent_size_txn(xlator_t *, loc_t *, quota_meta_t *, uint32_t nlink,
+                          call_stub_t *stub);
 
 int32_t
-mq_forget (xlator_t *, quota_inode_ctx_t *);
+mq_forget(xlator_t *, quota_inode_ctx_t *);
 #endif
diff --git a/xlators/features/marker/src/marker.c b/xlators/features/marker/src/marker.c
index fe7c634e344..1375ccc498c 100644
--- a/xlators/features/marker/src/marker.c
+++ b/xlators/features/marker/src/marker.c
@@ -7,2096 +7,2315 @@
    later), or the GNU General Public License, version 2 (GPLv2), in all
    cases as published by the Free Software Foundation.
 */
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
-
-#include "xlator.h"
-#include "defaults.h"
+#include <glusterfs/xlator.h>
+#include <glusterfs/defaults.h>
 #include "libxlator.h"
 #include "marker.h"
 #include "marker-mem-types.h"
 #include "marker-quota.h"
 #include "marker-quota-helper.h"
 #include "marker-common.h"
-#include "byte-order.h"
-#include "syncop.h"
+#include <glusterfs/byte-order.h>
+#include <glusterfs/syncop.h>
+#include <glusterfs/syscall.h>
 
 #include <fnmatch.h>
 
 #define _GF_UID_GID_CHANGED 1
 
-static char   *quota_external_xattrs[] = {
-        QUOTA_SIZE_KEY,
-        QUOTA_LIMIT_KEY,
-        QUOTA_LIMIT_OBJECTS_KEY,
-        NULL,
+static char *mq_ext_xattrs[] = {
+    QUOTA_SIZE_KEY,
+    QUOTA_LIMIT_KEY,
+    QUOTA_LIMIT_OBJECTS_KEY,
+    NULL,
 };
 
 void
-fini (xlator_t *this);
+fini(xlator_t *this);
 
 int32_t
-marker_start_setxattr (call_frame_t *, xlator_t *);
+marker_start_setxattr(call_frame_t *, xlator_t *);
+
+/* When client/quotad request for quota xattrs,
+ * replace the key-name by adding the version number
+ * in end of the key-name.
+ * In the cbk, result value of xattrs for original
+ * key-name.
+ * Below function marker_key_replace_with_ver and
+ * marker_key_set_ver is used for setting/removing
+ * version for the key-name
+ */
+int
+marker_key_replace_with_ver(xlator_t *this, dict_t *dict)
+{
+    int ret = -1;
+    int i = 0;
+    marker_conf_t *priv = NULL;
+    char key[QUOTA_KEY_MAX] = {
+        0,
+    };
+
+    priv = this->private;
+
+    if (dict == NULL || priv->version <= 0) {
+        ret = 0;
+        goto out;
+    }
+
+    for (i = 0; mq_ext_xattrs[i]; i++) {
+        if (dict_get(dict, mq_ext_xattrs[i])) {
+            GET_QUOTA_KEY(this, key, mq_ext_xattrs[i], ret);
+            if (ret < 0)
+                goto out;
+
+            ret = dict_set(dict, key, dict_get(dict, mq_ext_xattrs[i]));
+            if (ret < 0)
+                goto out;
+
+            dict_del(dict, mq_ext_xattrs[i]);
+        }
+    }
+
+    ret = 0;
+
+out:
+    return ret;
+}
+
+int
+marker_key_set_ver(xlator_t *this, dict_t *dict)
+{
+    int ret = -1;
+    int i = -1;
+    marker_conf_t *priv = NULL;
+    char key[QUOTA_KEY_MAX] = {
+        0,
+    };
+
+    priv = this->private;
+
+    if (dict == NULL || priv->version <= 0) {
+        ret = 0;
+        goto out;
+    }
+
+    for (i = 0; mq_ext_xattrs[i]; i++) {
+        GET_QUOTA_KEY(this, key, mq_ext_xattrs[i], ret);
+        if (ret < 0)
+            goto out;
+
+        if (dict_get(dict, key))
+            dict_set(dict, mq_ext_xattrs[i], dict_get(dict, key));
+    }
+
+    ret = 0;
+out:
+    return ret;
+}
 
 marker_local_t *
-marker_local_ref (marker_local_t *local)
+marker_local_ref(marker_local_t *local)
 {
-        GF_VALIDATE_OR_GOTO ("marker", local, err);
+    GF_VALIDATE_OR_GOTO("marker", local, err);
 
-        LOCK (&local->lock);
-        {
-                local->ref++;
-        }
-        UNLOCK (&local->lock);
+    LOCK(&local->lock);
+    {
+        local->ref++;
+    }
+    UNLOCK(&local->lock);
 
-        return local;
+    return local;
 err:
-        return NULL;
+    return NULL;
 }
 
 int
-marker_loc_fill (loc_t *loc, inode_t *inode, inode_t *parent, char *path)
+marker_loc_fill(loc_t *loc, inode_t *inode, inode_t *parent, char *path)
 {
-        int     ret = -1;
+    int ret = -1;
 
-        if (!loc)
-                return ret;
+    if (!loc)
+        return ret;
 
-        if (inode) {
-                loc->inode = inode_ref (inode);
-                if (gf_uuid_is_null (loc->gfid)) {
-                        gf_uuid_copy (loc->gfid, loc->inode->gfid);
-                }
+    if (inode) {
+        loc->inode = inode_ref(inode);
+        if (gf_uuid_is_null(loc->gfid)) {
+            gf_uuid_copy(loc->gfid, loc->inode->gfid);
         }
+    }
 
-        if (parent)
-                loc->parent = inode_ref (parent);
-
-        if (path) {
-                loc->path = gf_strdup (path);
-                if (!loc->path) {
-                        gf_log ("loc fill", GF_LOG_ERROR, "strdup failed");
-                        goto loc_wipe;
-                }
-
-                loc->name = strrchr (loc->path, '/');
-                if (loc->name)
-                        loc->name++;
+    if (parent)
+        loc->parent = inode_ref(parent);
+
+    if (path) {
+        loc->path = gf_strdup(path);
+        if (!loc->path) {
+            gf_log("loc fill", GF_LOG_ERROR, "strdup failed");
+            goto loc_wipe;
         }
 
-        ret = 0;
+        loc->name = strrchr(loc->path, '/');
+        if (loc->name)
+            loc->name++;
+    }
+
+    ret = 0;
 loc_wipe:
-        if (ret < 0)
-                loc_wipe (loc);
+    if (ret < 0)
+        loc_wipe(loc);
 
-        return ret;
+    return ret;
 }
 
 int
-marker_inode_loc_fill (inode_t *inode, char *name, loc_t *loc)
+_marker_inode_loc_fill(inode_t *inode, inode_t *parent, char *name, loc_t *loc)
 {
-        char            *resolvedpath = NULL;
-        int              ret          = -1;
-	inode_t         *parent = NULL;
+    char *resolvedpath = NULL;
+    int ret = -1;
+    gf_boolean_t free_parent = _gf_false;
 
-        if ((!inode) || (!loc))
-                return ret;
+    if ((!inode) || (!loc))
+        return ret;
 
-	parent = inode_parent (inode, NULL, NULL);
+    if (parent && name)
+        ret = inode_path(parent, name, &resolvedpath);
+    else
+        ret = inode_path(inode, NULL, &resolvedpath);
+    if (ret < 0)
+        goto err;
 
-        ret = inode_path (inode, name, &resolvedpath);
-        if (ret < 0)
-                goto err;
+    if (parent == NULL) {
+        parent = inode_parent(inode, NULL, NULL);
+        free_parent = _gf_true;
+    }
 
-        ret = marker_loc_fill (loc, inode, parent, resolvedpath);
-        if (ret < 0)
-                goto err;
+    ret = marker_loc_fill(loc, inode, parent, resolvedpath);
+    if (ret < 0)
+        goto err;
 
 err:
-	if (parent)
-		inode_unref (parent);
+    if (free_parent)
+        inode_unref(parent);
 
-        GF_FREE (resolvedpath);
+    GF_FREE(resolvedpath);
 
-        return ret;
+    return ret;
+}
+
+int
+marker_inode_loc_fill(inode_t *inode, loc_t *loc)
+{
+    return _marker_inode_loc_fill(inode, NULL, NULL, loc);
 }
 
 int32_t
-marker_trav_parent (marker_local_t *local)
+marker_trav_parent(marker_local_t *local)
 {
-        int32_t ret = 0;
-        loc_t   loc = {0, };
-        inode_t *parent = NULL;
-        int8_t  need_unref = 0;
+    int32_t ret = 0;
+    loc_t loc = {
+        0,
+    };
+    inode_t *parent = NULL;
+    int8_t need_unref = 0;
 
-        if (!local->loc.parent) {
-                parent = inode_parent (local->loc.inode, NULL, NULL);
-                if (parent)
-                        need_unref = 1;
-        } else
-                parent = local->loc.parent;
+    if (!local->loc.parent) {
+        parent = inode_parent(local->loc.inode, NULL, NULL);
+        if (parent)
+            need_unref = 1;
+    } else
+        parent = local->loc.parent;
 
-        ret = marker_inode_loc_fill (parent, NULL, &loc);
+    ret = marker_inode_loc_fill(parent, &loc);
 
-        if (ret < 0) {
-                ret = -1;
-                goto out;
-        }
+    if (ret < 0) {
+        ret = -1;
+        goto out;
+    }
 
-        loc_wipe (&local->loc);
+    loc_wipe(&local->loc);
 
-        local->loc = loc;
+    local->loc = loc;
 out:
-        if (need_unref)
-                inode_unref (parent);
+    if (need_unref)
+        inode_unref(parent);
 
-        return ret;
+    return ret;
 }
 
-int32_t
-marker_error_handler (xlator_t *this, marker_local_t *local, int32_t op_errno)
+void
+marker_error_handler(xlator_t *this, marker_local_t *local, int32_t op_errno)
 {
-         marker_conf_t *priv = NULL;
-         const char    *path = NULL;
-
-         priv = (marker_conf_t *) this->private;
-         path = local
-                 ? (local->loc.path
-                    ? local->loc.path : uuid_utoa(local->loc.gfid))
-                 : "<nul>";
-
-         gf_log (this->name, GF_LOG_CRITICAL,
-                 "Indexing gone corrupt at %s (reason: %s)."
-                 " Geo-replication slave content needs to be revalidated",
-                 path, strerror (op_errno));
-        unlink (priv->timestamp_file);
+    marker_conf_t *priv = (marker_conf_t *)this->private;
+    const char *path = local ? ((local->loc.path) ? local->loc.path
+                                                  : uuid_utoa(local->loc.gfid))
+                             : "<nul>";
 
-        return 0;
+    gf_log(this->name, GF_LOG_CRITICAL,
+           "Indexing gone corrupt at %s (reason: %s)."
+           " Geo-replication slave content needs to be revalidated",
+           path, strerror(op_errno));
+    sys_unlink(priv->timestamp_file);
 }
 
 int32_t
-marker_local_unref (marker_local_t *local)
+marker_local_unref(marker_local_t *local)
 {
-        int32_t   var = 0;
-
-        if (local == NULL)
-                return -1;
-
-        LOCK (&local->lock);
-        {
-                var = --local->ref;
-        }
-        UNLOCK (&local->lock);
-
-        if (var != 0)
-                goto out;
+    int32_t var = 0;
 
-        loc_wipe (&local->loc);
-        loc_wipe (&local->parent_loc);
-        if (local->xdata)
-                dict_unref (local->xdata);
+    if (local == NULL)
+        return -1;
 
-        if (local->oplocal) {
-                marker_local_unref (local->oplocal);
-                local->oplocal = NULL;
-        }
-        mem_put (local);
+    LOCK(&local->lock);
+    {
+        var = --local->ref;
+    }
+    UNLOCK(&local->lock);
+
+    if (var != 0)
+        goto out;
+
+    loc_wipe(&local->loc);
+    loc_wipe(&local->parent_loc);
+    if (local->xdata)
+        dict_unref(local->xdata);
+
+    if (local->lk_frame) {
+        STACK_DESTROY(local->lk_frame->root);
+        local->lk_frame = NULL;
+    }
+
+    if (local->oplocal) {
+        marker_local_unref(local->oplocal);
+        local->oplocal = NULL;
+    }
+    mem_put(local);
 out:
-        return 0;
+    return 0;
 }
 
 int32_t
-stat_stampfile (xlator_t *this, marker_conf_t *priv,
-                struct volume_mark **status)
+stat_stampfile(xlator_t *this, marker_conf_t *priv, struct volume_mark **status)
 {
-        struct stat          buf      = {0, };
-        struct volume_mark  *vol_mark = NULL;
+    struct stat buf = {
+        0,
+    };
+    struct volume_mark *vol_mark = NULL;
 
-        vol_mark = GF_CALLOC (sizeof (struct volume_mark), 1,
-                              gf_marker_mt_volume_mark);
+    vol_mark = GF_CALLOC(sizeof(struct volume_mark), 1,
+                         gf_marker_mt_volume_mark);
 
-        vol_mark->major = 1;
-        vol_mark->minor = 0;
+    vol_mark->major = 1;
+    vol_mark->minor = 0;
 
-        GF_ASSERT (sizeof (priv->volume_uuid_bin) == 16);
-        memcpy (vol_mark->uuid, priv->volume_uuid_bin, 16);
+    GF_ASSERT(sizeof(priv->volume_uuid_bin) == 16);
+    memcpy(vol_mark->uuid, priv->volume_uuid_bin, 16);
 
-        if (stat (priv->timestamp_file, &buf) != -1) {
-                vol_mark->retval = 0;
-                vol_mark->sec = htonl (buf.st_mtime);
-                vol_mark->usec = htonl (ST_MTIM_NSEC (&buf)/1000);
-        } else
-                vol_mark->retval = 1;
+    if (sys_stat(priv->timestamp_file, &buf) != -1) {
+        vol_mark->retval = 0;
+        vol_mark->sec = htonl(buf.st_mtime);
+        vol_mark->usec = htonl(ST_MTIM_NSEC(&buf) / 1000);
+    } else
+        vol_mark->retval = 1;
 
-        *status = vol_mark;
+    *status = vol_mark;
 
-        return 0;
+    return 0;
 }
 
 int32_t
-marker_getxattr_stampfile_cbk (call_frame_t *frame, xlator_t *this,
-                               const char *name, struct volume_mark *vol_mark,
-                               dict_t *xdata)
+marker_getxattr_stampfile_cbk(call_frame_t *frame, xlator_t *this,
+                              const char *name, struct volume_mark *vol_mark,
+                              dict_t *xdata)
 {
-        int32_t   ret  = -1;
-        dict_t   *dict = NULL;
+    int32_t ret = -1;
+    dict_t *dict = NULL;
 
-        if (vol_mark == NULL){
-                STACK_UNWIND_STRICT (getxattr, frame, -1, ENOMEM, NULL, NULL);
+    if (vol_mark == NULL) {
+        STACK_UNWIND_STRICT(getxattr, frame, -1, ENOMEM, NULL, NULL);
 
-                goto out;
-        }
+        goto out;
+    }
 
-        dict = dict_new ();
+    dict = dict_new();
 
-        ret = dict_set_bin (dict, (char *)name, vol_mark,
-                            sizeof (struct volume_mark));
-        if (ret)
-                gf_log (this->name, GF_LOG_WARNING, "failed to set key %s",
-                        name);
+    ret = dict_set_bin(dict, (char *)name, vol_mark,
+                       sizeof(struct volume_mark));
+    if (ret) {
+        GF_FREE(vol_mark);
+        gf_log(this->name, GF_LOG_WARNING, "failed to set key %s", name);
+    }
 
-        STACK_UNWIND_STRICT (getxattr, frame, 0, 0, dict, xdata);
+    STACK_UNWIND_STRICT(getxattr, frame, 0, 0, dict, xdata);
 
-        if (dict)
-                dict_unref (dict);
+    if (dict)
+        dict_unref(dict);
 out:
-        return 0;
+    return 0;
 }
 
 gf_boolean_t
-call_from_special_client (call_frame_t *frame, xlator_t *this, const char *name)
+call_from_special_client(call_frame_t *frame, xlator_t *this, const char *name)
 {
-        struct volume_mark     *vol_mark   = NULL;
-        marker_conf_t          *priv       = NULL;
-        gf_boolean_t           is_true     = _gf_true;
+    struct volume_mark *vol_mark = NULL;
+    marker_conf_t *priv = NULL;
+    gf_boolean_t is_true = _gf_true;
 
-        priv = (marker_conf_t *)this->private;
+    priv = (marker_conf_t *)this->private;
 
-        if (frame->root->pid != GF_CLIENT_PID_GSYNCD || name == NULL ||
-            strcmp (name, MARKER_XATTR_PREFIX "." VOLUME_MARK) != 0) {
-                is_true = _gf_false;
-                goto out;
-        }
+    if (frame->root->pid != GF_CLIENT_PID_GSYNCD || name == NULL ||
+        strcmp(name, MARKER_XATTR_PREFIX "." VOLUME_MARK) != 0) {
+        is_true = _gf_false;
+        goto out;
+    }
 
-        stat_stampfile (this, priv, &vol_mark);
+    stat_stampfile(this, priv, &vol_mark);
 
-        marker_getxattr_stampfile_cbk (frame, this, name, vol_mark, NULL);
+    marker_getxattr_stampfile_cbk(frame, this, name, vol_mark, NULL);
 out:
-        return is_true;
+    return is_true;
 }
 
 static gf_boolean_t
-_is_quota_internal_xattr (dict_t *d, char *k, data_t *v, void *data)
+_is_quota_internal_xattr(dict_t *d, char *k, data_t *v, void *data)
 {
-        int     i = 0;
-        char    **external_xattrs = data;
+    int i = 0;
+    char **external_xattrs = data;
 
-        for (i = 0; external_xattrs && external_xattrs[i]; i++) {
-                if (strcmp (k, external_xattrs[i]) == 0)
-                        return _gf_false;
-        }
+    for (i = 0; external_xattrs && external_xattrs[i]; i++) {
+        if (strcmp(k, external_xattrs[i]) == 0)
+            return _gf_false;
+    }
 
-        if (fnmatch ("trusted.glusterfs.quota*", k, 0) == 0)
-                return _gf_true;
+    if (fnmatch("trusted.glusterfs.quota*", k, 0) == 0)
+        return _gf_true;
 
-        /* It would be nice if posix filters pgfid xattrs. But since marker
-         * also takes up responsibility to clean these up, adding the filtering
-         * here (Check 'quota_xattr_cleaner')
-         */
-        if (fnmatch (PGFID_XATTR_KEY_PREFIX"*", k, 0) == 0)
-                return _gf_true;
+    /* It would be nice if posix filters pgfid xattrs. But since marker
+     * also takes up responsibility to clean these up, adding the filtering
+     * here (Check 'quota_xattr_cleaner')
+     */
+    if (fnmatch(PGFID_XATTR_KEY_PREFIX "*", k, 0) == 0)
+        return _gf_true;
 
-        return _gf_false;
+    return _gf_false;
 }
 
 static void
-marker_filter_internal_xattrs (xlator_t *this, dict_t *xattrs)
+marker_filter_internal_xattrs(xlator_t *this, dict_t *xattrs)
 {
-        marker_conf_t *priv   = NULL;
-        char         **ext    = NULL;
+    marker_conf_t *priv = NULL;
+    char **ext = NULL;
 
-        priv = this->private;
-        if (priv->feature_enabled & GF_QUOTA)
-                ext = quota_external_xattrs;
+    priv = this->private;
+    if (priv->feature_enabled & GF_QUOTA)
+        ext = mq_ext_xattrs;
 
-        dict_foreach_match (xattrs, _is_quota_internal_xattr, ext,
-                            dict_remove_foreach_fn, NULL);
-        return;
+    dict_foreach_match(xattrs, _is_quota_internal_xattr, ext,
+                       dict_remove_foreach_fn, NULL);
 }
 
-int32_t
-marker_getxattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                     int32_t op_ret, int32_t op_errno, dict_t *dict,
-                     dict_t *xdata)
-{
-        marker_local_t *local = NULL;
-        local = frame->local;
-
-
-        if (cookie) {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "Filtering the quota extended attributes");
-
-                /* If the getxattr is from a non special client, then do not
-                   copy the quota related xattrs (except the quota limit key
-                   i.e trusted.glusterfs.quota.limit-set which has been set by
-                   glusterd on the directory on which quota limit is set.) for
-                   directories. Let the healing of xattrs happen upon lookup.
-                   NOTE: setting of trusted.glusterfs.quota.limit-set as of now
-                   happens from glusterd. It should be moved to quotad. Also
-                   trusted.glusterfs.quota.limit-set is set on directory which
-                   is permanent till quota is removed on that directory or limit
-                   is changed. So let that xattr be healed by other xlators
-                   properly whenever directory healing is done.
-                */
-                /*
-                 * Except limit-set xattr, rest of the xattrs are maintained
-                 * by quota xlator. Don't expose them to other xlators.
-                 * This filter makes sure quota xattrs are not healed as part of
-                 * metadata self-heal
-                 */
-                marker_filter_internal_xattrs (frame->this, dict);
-        }
+static void
+marker_filter_gsyncd_xattrs(call_frame_t *frame, xlator_t *this, dict_t *xattrs)
+{
+    marker_conf_t *priv = NULL;
 
-        frame->local = NULL;
-        STACK_UNWIND_STRICT (getxattr, frame, op_ret, op_errno, dict, xdata);
-        marker_local_unref (local);
-        return 0;
+    priv = this->private;
+    GF_ASSERT(priv);
+    GF_ASSERT(frame);
+
+    if (xattrs && frame->root->pid != GF_CLIENT_PID_GSYNCD) {
+        GF_REMOVE_INTERNAL_XATTR(GF_XATTR_XTIME_PATTERN, xattrs);
+    }
+    return;
 }
 
 int32_t
-marker_getxattr (call_frame_t *frame, xlator_t *this, loc_t *loc,
-                 const char *name, dict_t *xdata)
-{
-        gf_boolean_t    is_true  = _gf_false;
-        marker_conf_t   *priv    = NULL;
-        unsigned long   cookie   = 0;
-        marker_local_t  *local   = NULL;
+marker_getxattr_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                    int32_t op_ret, int32_t op_errno, dict_t *dict,
+                    dict_t *xdata)
+{
+    int32_t ret = -1;
+    if (op_ret < 0)
+        goto unwind;
+
+    ret = marker_key_set_ver(this, dict);
+    if (ret < 0) {
+        op_ret = -1;
+        op_errno = ENOMEM;
+        goto unwind;
+    }
+
+    if (cookie) {
+        gf_log(this->name, GF_LOG_DEBUG,
+               "Filtering the quota extended attributes");
+
+        /* If the getxattr is from a non special client, then do not
+           copy the quota related xattrs (except the quota limit key
+           i.e trusted.glusterfs.quota.limit-set which has been set by
+           glusterd on the directory on which quota limit is set.) for
+           directories. Let the healing of xattrs happen upon lookup.
+           NOTE: setting of trusted.glusterfs.quota.limit-set as of now
+           happens from glusterd. It should be moved to quotad. Also
+           trusted.glusterfs.quota.limit-set is set on directory which
+           is permanent till quota is removed on that directory or limit
+           is changed. So let that xattr be healed by other xlators
+           properly whenever directory healing is done.
+        */
+        /*
+         * Except limit-set xattr, rest of the xattrs are maintained
+         * by quota xlator. Don't expose them to other xlators.
+         * This filter makes sure quota xattrs are not healed as part of
+         * metadata self-heal
+         */
+        marker_filter_internal_xattrs(frame->this, dict);
+    }
 
-        priv = this->private;
+    /* Filter gsyncd xtime xattr for non gsyncd clients */
+    marker_filter_gsyncd_xattrs(frame, frame->this, dict);
 
-        frame->local = mem_get0 (this->local_pool);
-        local = frame->local;
-        if (local == NULL)
+unwind:
+    MARKER_STACK_UNWIND(getxattr, frame, op_ret, op_errno, dict, xdata);
+    return 0;
+}
+
+int32_t
+marker_getxattr(call_frame_t *frame, xlator_t *this, loc_t *loc,
+                const char *name, dict_t *xdata)
+{
+    gf_boolean_t is_true = _gf_false;
+    marker_conf_t *priv = NULL;
+    unsigned long cookie = 0;
+    marker_local_t *local = NULL;
+    char key[QUOTA_KEY_MAX] = {
+        0,
+    };
+    int32_t ret = -1;
+    int32_t i = 0;
+
+    priv = this->private;
+
+    if (name) {
+        for (i = 0; mq_ext_xattrs[i]; i++) {
+            if (strcmp(name, mq_ext_xattrs[i]))
+                continue;
+
+            GET_QUOTA_KEY(this, key, mq_ext_xattrs[i], ret);
+            if (ret < 0)
                 goto out;
+            name = key;
+            break;
+        }
+    }
+
+    frame->local = mem_get0(this->local_pool);
+    local = frame->local;
+    if (local == NULL)
+        goto out;
+
+    MARKER_INIT_LOCAL(frame, local);
+
+    if ((loc_copy(&local->loc, loc)) < 0)
+        goto out;
+
+    gf_log(this->name, GF_LOG_DEBUG, "USER:PID = %d", frame->root->pid);
 
-        MARKER_INIT_LOCAL (frame, local);
-
-        if ((loc_copy (&local->loc, loc)) < 0)
-		goto out;
-
-        gf_log (this->name, GF_LOG_DEBUG, "USER:PID = %d", frame->root->pid);
-
-        if (priv && priv->feature_enabled & GF_XTIME)
-                is_true = call_from_special_client (frame, this, name);
-
-        if (is_true == _gf_false) {
-                if (name == NULL) {
-                        /* Signifies that marker translator
-                         * has to filter the quota's xattr's,
-                         * this is to prevent afr from performing
-                         * self healing on marker-quota xattrs'
-                         */
-                        cookie = 1;
-                }
-                STACK_WIND_COOKIE (frame, marker_getxattr_cbk,
-				   (void *)cookie,
-                                   FIRST_CHILD(this),
-                                   FIRST_CHILD(this)->fops->getxattr,
-				   loc, name, xdata);
+    if (priv && priv->feature_enabled & GF_XTIME)
+        is_true = call_from_special_client(frame, this, name);
+
+    if (is_true == _gf_false) {
+        if (name == NULL) {
+            /* Signifies that marker translator
+             * has to filter the quota's xattr's,
+             * this is to prevent afr from performing
+             * self healing on marker-quota xattrs'
+             */
+            cookie = 1;
         }
+        STACK_WIND_COOKIE(frame, marker_getxattr_cbk, (void *)cookie,
+                          FIRST_CHILD(this), FIRST_CHILD(this)->fops->getxattr,
+                          loc, name, xdata);
+    }
 
-        return 0;
+    return 0;
 out:
-        frame->local = NULL;
-        STACK_UNWIND_STRICT (getxattr, frame, -1, ENOMEM, NULL, NULL);
-        marker_local_unref (local);
-        return 0;
+    MARKER_STACK_UNWIND(getxattr, frame, -1, ENOMEM, NULL, NULL);
+    return 0;
 }
 
-
 int32_t
-marker_setxattr_done (call_frame_t *frame)
+marker_setxattr_done(call_frame_t *frame)
 {
-        marker_local_t *local = NULL;
+    marker_local_t *local = NULL;
 
-        local = (marker_local_t *) frame->local;
+    local = (marker_local_t *)frame->local;
 
-        frame->local = NULL;
+    frame->local = NULL;
 
-        STACK_DESTROY (frame->root);
+    STACK_DESTROY(frame->root);
 
-        marker_local_unref (local);
+    marker_local_unref(local);
 
-        return 0;
+    return 0;
 }
 
 int
-marker_specific_setxattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                              int32_t op_ret, int32_t op_errno, dict_t *xdata)
+marker_specific_setxattr_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                             int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
-        int32_t         ret   = 0;
-        int32_t         done  = 0;
-        marker_local_t *local = NULL;
+    int32_t ret = 0;
+    int32_t done = 1;
+    marker_local_t *local = NULL;
 
-        local = (marker_local_t*) frame->local;
+    local = (marker_local_t *)frame->local;
 
-        if (op_ret == -1 && op_errno == ENOSPC) {
-                marker_error_handler (this, local, op_errno);
-                done = 1;
-                goto out;
-        }
+    if (op_ret == -1 && op_errno == ENOSPC) {
+        marker_error_handler(this, local, op_errno);
+        goto out;
+    }
 
-        if (local) {
-                if (local->loc.path && strcmp (local->loc.path, "/") == 0) {
-                        done = 1;
-                        goto out;
-                }
-                if (__is_root_gfid (local->loc.gfid)) {
-                        done = 1;
-                        goto out;
-                }
+    if (local) {
+        if (local->loc.path && strcmp(local->loc.path, "/") == 0) {
+            goto out;
         }
-
-        ret = marker_trav_parent (local);
-
-        if (ret == -1) {
-                gf_log (this->name, GF_LOG_DEBUG, "Error occurred "
-                        "while traversing to the parent, stopping marker");
-
-                done = 1;
-
-                goto out;
+        if (__is_root_gfid(local->loc.gfid)) {
+            goto out;
         }
+    }
+
+    ret = (local) ? marker_trav_parent(local) : -1;
 
-        marker_start_setxattr (frame, this);
+    if (ret == -1) {
+        gf_log(this->name, GF_LOG_DEBUG,
+               "Error occurred "
+               "while traversing to the parent, stopping marker");
+        goto out;
+    }
 
+    marker_start_setxattr(frame, this);
+    done = 0;
 out:
-        if (done) {
-                marker_setxattr_done (frame);
-        }
+    if (done) {
+        marker_setxattr_done(frame);
+    }
 
-        return 0;
+    return 0;
 }
 
 int32_t
-marker_start_setxattr (call_frame_t *frame, xlator_t *this)
+marker_start_setxattr(call_frame_t *frame, xlator_t *this)
 {
-        int32_t          ret   = -1;
-        dict_t          *dict  = NULL;
-        marker_local_t  *local = NULL;
-        marker_conf_t   *priv  = NULL;
+    int32_t ret = -1;
+    dict_t *dict = NULL;
+    marker_local_t *local = NULL;
+    marker_conf_t *priv = NULL;
 
-        priv = this->private;
+    priv = this->private;
 
-        local = (marker_local_t*) frame->local;
+    local = (marker_local_t *)frame->local;
 
-        if (!local)
-                goto out;
+    if (!local)
+        goto out;
 
-        dict = dict_new ();
+    dict = dict_new();
 
-        if (!dict)
-                goto out;
+    if (!dict)
+        goto out;
 
-        if (local->loc.inode && gf_uuid_is_null (local->loc.gfid))
-                gf_uuid_copy (local->loc.gfid, local->loc.inode->gfid);
+    if (local->loc.inode && gf_uuid_is_null(local->loc.gfid))
+        gf_uuid_copy(local->loc.gfid, local->loc.inode->gfid);
 
-        GF_UUID_ASSERT (local->loc.gfid);
+    GF_UUID_ASSERT(local->loc.gfid);
 
-        ret = dict_set_static_bin (dict, priv->marker_xattr,
-                                   (void *)local->timebuf, 8);
-        if (ret) {
-                gf_log (this->name, GF_LOG_WARNING,
-                        "failed to set marker xattr (%s)", local->loc.path);
-                goto out;
-        }
+    ret = dict_set_static_bin(dict, priv->marker_xattr, (void *)local->timebuf,
+                              8);
+    if (ret) {
+        gf_log(this->name, GF_LOG_WARNING, "failed to set marker xattr (%s)",
+               local->loc.path);
+        goto out;
+    }
 
-        STACK_WIND (frame, marker_specific_setxattr_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->setxattr, &local->loc, dict, 0,
-                    NULL);
+    STACK_WIND(frame, marker_specific_setxattr_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->setxattr, &local->loc, dict, 0, NULL);
 
-        ret = 0;
+    ret = 0;
 out:
-        if (dict)
-                dict_unref (dict);
+    if (dict)
+        dict_unref(dict);
 
-        return ret;
+    return ret;
 }
 
 void
-marker_gettimeofday (marker_local_t *local)
+marker_gettimeofday(marker_local_t *local)
 {
-        struct timeval tv = {0, };
+    struct timeval tv = {
+        0,
+    };
 
-        gettimeofday (&tv, NULL);
+    gettimeofday(&tv, NULL);
 
-        local->timebuf [0] = htonl (tv.tv_sec);
-        local->timebuf [1] = htonl (tv.tv_usec);
+    local->timebuf[0] = htonl(tv.tv_sec);
+    local->timebuf[1] = htonl(tv.tv_usec);
 
-        return;
+    return;
 }
 
 int32_t
-marker_create_frame (xlator_t *this, marker_local_t *local)
+marker_create_frame(xlator_t *this, marker_local_t *local)
 {
-        call_frame_t *frame = NULL;
+    call_frame_t *frame = NULL;
 
-        frame = create_frame (this, this->ctx->pool);
+    frame = create_frame(this, this->ctx->pool);
 
-        if (!frame)
-                return -1;
+    if (!frame)
+        return -1;
 
-        frame->local = (void *) local;
+    frame->local = (void *)local;
 
-        marker_start_setxattr (frame, this);
+    marker_start_setxattr(frame, this);
 
-        return 0;
+    return 0;
 }
 
 int32_t
-marker_xtime_update_marks (xlator_t *this, marker_local_t *local)
+marker_xtime_update_marks(xlator_t *this, marker_local_t *local)
 {
-        marker_conf_t *priv = NULL;
+    marker_conf_t *priv = NULL;
 
-        GF_VALIDATE_OR_GOTO ("marker", this, out);
-        GF_VALIDATE_OR_GOTO (this->name, local, out);
+    GF_VALIDATE_OR_GOTO("marker", this, out);
+    GF_VALIDATE_OR_GOTO(this->name, local, out);
 
-        priv = this->private;
+    priv = this->private;
 
-        if ((local->pid == GF_CLIENT_PID_GSYNCD
-             && !(priv->feature_enabled & GF_XTIME_GSYNC_FORCE))
-            || (local->pid == GF_CLIENT_PID_DEFRAG))
-                goto out;
+    if ((local->pid == GF_CLIENT_PID_GSYNCD &&
+         !(priv->feature_enabled & GF_XTIME_GSYNC_FORCE)) ||
+        (local->pid == GF_CLIENT_PID_DEFRAG))
+        goto out;
 
-        marker_gettimeofday (local);
+    marker_gettimeofday(local);
 
-        marker_local_ref (local);
+    marker_local_ref(local);
 
-        marker_create_frame (this, local);
+    marker_create_frame(this, local);
 out:
-        return 0;
+    return 0;
 }
 
-
 int32_t
-marker_mkdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                  int32_t op_ret, int32_t op_errno, inode_t *inode,
-                  struct iatt *buf, struct iatt *preparent,
-                  struct iatt *postparent, dict_t *xdata)
+marker_mkdir_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                 int32_t op_ret, int32_t op_errno, inode_t *inode,
+                 struct iatt *buf, struct iatt *preparent,
+                 struct iatt *postparent, dict_t *xdata)
 {
-        marker_conf_t      *priv    = NULL;
-        marker_local_t     *local   = NULL;
+    marker_conf_t *priv = NULL;
+    marker_local_t *local = NULL;
+    quota_inode_ctx_t *ctx = NULL;
 
-        if (op_ret == -1) {
-                gf_log (this->name, GF_LOG_TRACE, "error occurred "
-                        "while creating directory %s", strerror (op_errno));
-        }
+    if (op_ret == -1) {
+        gf_log(this->name, GF_LOG_TRACE,
+               "error occurred "
+               "while creating directory %s",
+               strerror(op_errno));
+    }
 
-        local = (marker_local_t *) frame->local;
+    local = (marker_local_t *)frame->local;
 
-        frame->local = NULL;
+    frame->local = NULL;
+    priv = this->private;
 
-        STACK_UNWIND_STRICT (mkdir, frame, op_ret, op_errno, inode,
-                             buf, preparent, postparent, xdata);
+    if (op_ret >= 0 && inode && (priv->feature_enabled & GF_QUOTA)) {
+        ctx = mq_inode_ctx_new(inode, this);
+        if (ctx == NULL) {
+            gf_log(this->name, GF_LOG_WARNING,
+                   "mq_inode_ctx_new "
+                   "failed for %s",
+                   uuid_utoa(inode->gfid));
+            op_ret = -1;
+            op_errno = ENOMEM;
+        }
+    }
 
-        if (op_ret == -1 || local == NULL)
-                goto out;
+    STACK_UNWIND_STRICT(mkdir, frame, op_ret, op_errno, inode, buf, preparent,
+                        postparent, xdata);
 
-        if (gf_uuid_is_null (local->loc.gfid))
-                gf_uuid_copy (local->loc.gfid, buf->ia_gfid);
+    if (op_ret == -1 || local == NULL)
+        goto out;
 
-        priv = this->private;
+    if (gf_uuid_is_null(local->loc.gfid))
+        gf_uuid_copy(local->loc.gfid, buf->ia_gfid);
 
-        if (priv->feature_enabled & GF_QUOTA)
-                mq_create_xattrs_txn (this, &local->loc);
+    if (priv->feature_enabled & GF_QUOTA)
+        mq_create_xattrs_txn(this, &local->loc, NULL);
 
-        if (priv->feature_enabled & GF_XTIME)
-                marker_xtime_update_marks (this, local);
+    if (priv->feature_enabled & GF_XTIME)
+        marker_xtime_update_marks(this, local);
 
 out:
-        marker_local_unref (local);
+    marker_local_unref(local);
 
-        return 0;
+    return 0;
 }
 
 int
-marker_mkdir (call_frame_t *frame, xlator_t *this, loc_t *loc, mode_t mode,
-              mode_t umask, dict_t *xdata)
+marker_mkdir(call_frame_t *frame, xlator_t *this, loc_t *loc, mode_t mode,
+             mode_t umask, dict_t *xdata)
 {
-        int32_t          ret   = 0;
-        marker_local_t  *local = NULL;
-        marker_conf_t   *priv  = NULL;
+    int32_t ret = 0;
+    marker_local_t *local = NULL;
+    marker_conf_t *priv = NULL;
 
-        priv = this->private;
+    priv = this->private;
 
-        if (priv->feature_enabled == 0)
-                goto wind;
+    if (priv->feature_enabled == 0)
+        goto wind;
 
-        local = mem_get0 (this->local_pool);
+    local = mem_get0(this->local_pool);
 
-        MARKER_INIT_LOCAL (frame, local);
+    MARKER_INIT_LOCAL(frame, local);
 
-        ret = loc_copy (&local->loc, loc);
+    ret = loc_copy(&local->loc, loc);
 
-        if (ret == -1)
-                goto err;
+    if (ret == -1)
+        goto err;
 wind:
-        STACK_WIND (frame, marker_mkdir_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->mkdir, loc, mode, umask, xdata);
+    STACK_WIND(frame, marker_mkdir_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->mkdir, loc, mode, umask, xdata);
 
-        return 0;
+    return 0;
 err:
-        STACK_UNWIND_STRICT (mkdir, frame, -1, ENOMEM, NULL,
-                             NULL, NULL, NULL, NULL);
-        return 0;
-}
+    MARKER_STACK_UNWIND(mkdir, frame, -1, ENOMEM, NULL, NULL, NULL, NULL, NULL);
 
+    return 0;
+}
 
 int32_t
-marker_create_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                   int32_t op_ret, int32_t op_errno, fd_t *fd, inode_t *inode,
-                   struct iatt *buf, struct iatt *preparent,
-                   struct iatt *postparent, dict_t *xdata)
+marker_create_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                  int32_t op_ret, int32_t op_errno, fd_t *fd, inode_t *inode,
+                  struct iatt *buf, struct iatt *preparent,
+                  struct iatt *postparent, dict_t *xdata)
 {
-        marker_local_t     *local   = NULL;
-        marker_conf_t      *priv    = NULL;
+    marker_local_t *local = NULL;
+    marker_conf_t *priv = NULL;
+    quota_inode_ctx_t *ctx = NULL;
 
-        if (op_ret == -1) {
-                gf_log (this->name, GF_LOG_TRACE, "error occurred "
-                        "while creating file %s", strerror (op_errno));
-        }
+    if (op_ret == -1) {
+        gf_log(this->name, GF_LOG_TRACE,
+               "error occurred "
+               "while creating file %s",
+               strerror(op_errno));
+    }
 
-        local = (marker_local_t *) frame->local;
+    local = (marker_local_t *)frame->local;
 
-        frame->local = NULL;
+    frame->local = NULL;
+    priv = this->private;
 
-        STACK_UNWIND_STRICT (create, frame, op_ret, op_errno, fd, inode, buf,
-                             preparent, postparent, xdata);
+    if (op_ret >= 0 && inode && (priv->feature_enabled & GF_QUOTA)) {
+        ctx = mq_inode_ctx_new(inode, this);
+        if (ctx == NULL) {
+            gf_log(this->name, GF_LOG_WARNING,
+                   "mq_inode_ctx_new "
+                   "failed for %s",
+                   uuid_utoa(inode->gfid));
+            op_ret = -1;
+            op_errno = ENOMEM;
+        }
+    }
 
-        if (op_ret == -1 || local == NULL)
-                goto out;
+    STACK_UNWIND_STRICT(create, frame, op_ret, op_errno, fd, inode, buf,
+                        preparent, postparent, xdata);
 
-        if (gf_uuid_is_null (local->loc.gfid))
-                gf_uuid_copy (local->loc.gfid, buf->ia_gfid);
+    if (op_ret == -1 || local == NULL)
+        goto out;
 
-        priv = this->private;
+    if (gf_uuid_is_null(local->loc.gfid))
+        gf_uuid_copy(local->loc.gfid, buf->ia_gfid);
 
-        if (priv->feature_enabled & GF_QUOTA)
-                mq_create_xattrs_txn (this, &local->loc);
+    if (priv->feature_enabled & GF_QUOTA)
+        mq_create_xattrs_txn(this, &local->loc, buf);
 
-        if (priv->feature_enabled & GF_XTIME)
-                marker_xtime_update_marks (this, local);
+    if (priv->feature_enabled & GF_XTIME)
+        marker_xtime_update_marks(this, local);
 
 out:
-        marker_local_unref (local);
+    marker_local_unref(local);
 
-        return 0;
+    return 0;
 }
 
 int32_t
-marker_create (call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t flags,
-               mode_t mode, mode_t umask, fd_t *fd, dict_t *xdata)
+marker_create(call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t flags,
+              mode_t mode, mode_t umask, fd_t *fd, dict_t *xdata)
 {
-        int32_t          ret   = 0;
-        marker_local_t  *local = NULL;
-        marker_conf_t   *priv  = NULL;
+    int32_t ret = 0;
+    marker_local_t *local = NULL;
+    marker_conf_t *priv = NULL;
 
-        priv = this->private;
+    priv = this->private;
 
-        if (priv->feature_enabled == 0)
-                goto wind;
+    if (priv->feature_enabled == 0)
+        goto wind;
 
-        local = mem_get0 (this->local_pool);
+    local = mem_get0(this->local_pool);
 
-        MARKER_INIT_LOCAL (frame, local);
+    MARKER_INIT_LOCAL(frame, local);
 
-        ret = loc_copy (&local->loc, loc);
+    ret = loc_copy(&local->loc, loc);
 
-        if (ret == -1)
-                goto err;
+    if (ret == -1)
+        goto err;
 wind:
-        STACK_WIND (frame, marker_create_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->create, loc, flags, mode, umask,
-                    fd, xdata);
-        return 0;
+    STACK_WIND(frame, marker_create_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->create, loc, flags, mode, umask, fd,
+               xdata);
+    return 0;
 err:
-        STACK_UNWIND_STRICT (create, frame, -1, ENOMEM, NULL, NULL, NULL, NULL,
-                             NULL, NULL);
+    MARKER_STACK_UNWIND(create, frame, -1, ENOMEM, NULL, NULL, NULL, NULL, NULL,
+                        NULL);
 
-        return 0;
+    return 0;
 }
 
-
 int32_t
-marker_writev_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                   int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
-                   struct iatt *postbuf, dict_t *xdata)
+marker_writev_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                  int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
+                  struct iatt *postbuf, dict_t *xdata)
 {
-        marker_conf_t      *priv    = NULL;
-        marker_local_t     *local   = NULL;
+    marker_conf_t *priv = NULL;
+    marker_local_t *local = NULL;
 
-        if (op_ret == -1) {
-                gf_log (this->name, GF_LOG_TRACE, "error occurred "
-                        "while write, %s", strerror (op_errno));
-        }
+    if (op_ret == -1) {
+        gf_log(this->name, GF_LOG_TRACE,
+               "error occurred "
+               "while write, %s",
+               strerror(op_errno));
+    }
 
-        local = (marker_local_t *) frame->local;
+    local = (marker_local_t *)frame->local;
 
-        frame->local = NULL;
+    frame->local = NULL;
 
-        STACK_UNWIND_STRICT (writev, frame, op_ret, op_errno, prebuf, postbuf,
-                             xdata);
+    STACK_UNWIND_STRICT(writev, frame, op_ret, op_errno, prebuf, postbuf,
+                        xdata);
 
-        if (op_ret == -1 || local == NULL)
-                goto out;
+    if (op_ret == -1 || local == NULL)
+        goto out;
 
-        priv = this->private;
+    priv = this->private;
 
-        if (priv->feature_enabled & GF_QUOTA)
-                mq_initiate_quota_txn (this, &local->loc);
+    if (priv->feature_enabled & GF_QUOTA)
+        mq_initiate_quota_txn(this, &local->loc, postbuf);
 
-        if (priv->feature_enabled & GF_XTIME)
-                marker_xtime_update_marks (this, local);
+    if (priv->feature_enabled & GF_XTIME)
+        marker_xtime_update_marks(this, local);
 
 out:
-        marker_local_unref (local);
+    marker_local_unref(local);
 
-        return 0;
+    return 0;
 }
 
 int32_t
-marker_writev (call_frame_t *frame,
-               xlator_t *this,
-               fd_t *fd,
-               struct iovec *vector,
-               int32_t count,
-               off_t offset, uint32_t flags,
-               struct iobref *iobref, dict_t *xdata)
+marker_writev(call_frame_t *frame, xlator_t *this, fd_t *fd,
+              struct iovec *vector, int32_t count, off_t offset, uint32_t flags,
+              struct iobref *iobref, dict_t *xdata)
 {
-        int32_t          ret   = 0;
-        marker_local_t  *local = NULL;
-        marker_conf_t   *priv  = NULL;
+    int32_t ret = 0;
+    marker_local_t *local = NULL;
+    marker_conf_t *priv = NULL;
 
-        priv = this->private;
+    priv = this->private;
 
-        if (priv->feature_enabled == 0)
-                goto wind;
+    if (priv->feature_enabled == 0)
+        goto wind;
 
-        local = mem_get0 (this->local_pool);
+    local = mem_get0(this->local_pool);
 
-        MARKER_INIT_LOCAL (frame, local);
+    MARKER_INIT_LOCAL(frame, local);
 
-        ret = marker_inode_loc_fill (fd->inode, NULL, &local->loc);
+    ret = marker_inode_loc_fill(fd->inode, &local->loc);
 
-        if (ret == -1)
-                goto err;
+    if (ret == -1)
+        goto err;
 wind:
-        STACK_WIND (frame, marker_writev_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->writev, fd, vector, count, offset,
-                    flags, iobref, xdata);
-        return 0;
+    STACK_WIND(frame, marker_writev_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->writev, fd, vector, count, offset,
+               flags, iobref, xdata);
+    return 0;
 err:
-        STACK_UNWIND_STRICT (writev, frame, -1, ENOMEM, NULL, NULL, NULL);
+    MARKER_STACK_UNWIND(writev, frame, -1, ENOMEM, NULL, NULL, NULL);
 
-        return 0;
+    return 0;
 }
 
-
 int32_t
-marker_rmdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                  int32_t op_ret, int32_t op_errno, struct iatt *preparent,
-                  struct iatt *postparent, dict_t *xdata)
+marker_rmdir_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                 int32_t op_ret, int32_t op_errno, struct iatt *preparent,
+                 struct iatt *postparent, dict_t *xdata)
 {
-        marker_conf_t      *priv    = NULL;
-        marker_local_t     *local   = NULL;
+    marker_conf_t *priv = NULL;
+    marker_local_t *local = NULL;
+    call_stub_t *stub = NULL;
 
-        if (op_ret == -1) {
-                gf_log (this->name, GF_LOG_TRACE, "error occurred "
-                        "rmdir %s", strerror (op_errno));
-        }
+    if (op_ret == -1) {
+        gf_log(this->name, GF_LOG_TRACE,
+               "error occurred "
+               "rmdir %s",
+               strerror(op_errno));
+    }
 
-        local = (marker_local_t *) frame->local;
+    local = (marker_local_t *)frame->local;
 
-        frame->local = NULL;
+    frame->local = NULL;
+    priv = this->private;
 
-        STACK_UNWIND_STRICT (rmdir, frame, op_ret, op_errno, preparent,
-                             postparent, xdata);
+    if (op_ret == -1 || local == NULL)
+        goto out;
 
-        if (op_ret == -1 || local == NULL)
-                goto out;
+    if (priv->feature_enabled & GF_XTIME)
+        marker_xtime_update_marks(this, local);
 
-        priv = this->private;
+    if (priv->feature_enabled & GF_QUOTA) {
+        /* If a 'rm -rf' is performed by a client, rmdir can be faster
+           than marker background mq_reduce_parent_size_txn.
+           In this case, as part of rmdir parent child association
+           will be removed in the server protocol.
+           This can lead to mq_reduce_parent_size_txn failures.
 
-        if (priv->feature_enabled & GF_QUOTA)
-                mq_reduce_parent_size_txn (this, &local->loc, -1);
+           So perform mq_reduce_parent_size_txn in foreground
+           and unwind to server once txn is complete
+         */
+
+        stub = fop_rmdir_cbk_stub(frame, default_rmdir_cbk, op_ret, op_errno,
+                                  preparent, postparent, xdata);
+        mq_reduce_parent_size_txn(this, &local->loc, NULL, 1, stub);
+
+        if (stub) {
+            marker_local_unref(local);
+            return 0;
+        }
+    }
 
-        if (priv->feature_enabled & GF_XTIME)
-                marker_xtime_update_marks (this, local);
 out:
-        marker_local_unref (local);
+    STACK_UNWIND_STRICT(rmdir, frame, op_ret, op_errno, preparent, postparent,
+                        xdata);
 
-        return 0;
+    marker_local_unref(local);
+
+    return 0;
 }
 
 int32_t
-marker_rmdir (call_frame_t *frame, xlator_t *this, loc_t *loc, int flags,
-              dict_t *xdata)
+marker_rmdir(call_frame_t *frame, xlator_t *this, loc_t *loc, int flags,
+             dict_t *xdata)
 {
-        int32_t          ret   = 0;
-        marker_local_t  *local = NULL;
-        marker_conf_t   *priv  = NULL;
+    int32_t ret = 0;
+    marker_local_t *local = NULL;
+    marker_conf_t *priv = NULL;
 
-        priv = this->private;
+    priv = this->private;
 
-        if (priv->feature_enabled == 0)
-                goto wind;
+    if (priv->feature_enabled == 0)
+        goto wind;
 
-        local = mem_get0 (this->local_pool);
+    local = mem_get0(this->local_pool);
 
-        MARKER_INIT_LOCAL (frame, local);
+    MARKER_INIT_LOCAL(frame, local);
 
-        ret = loc_copy (&local->loc, loc);
+    ret = loc_copy(&local->loc, loc);
 
-        if (ret == -1)
-                goto err;
+    if (ret == -1)
+        goto err;
 wind:
-        STACK_WIND (frame, marker_rmdir_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->rmdir, loc, flags, xdata);
-        return 0;
+    STACK_WIND(frame, marker_rmdir_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->rmdir, loc, flags, xdata);
+    return 0;
 err:
-        STACK_UNWIND_STRICT (rmdir, frame, -1, ENOMEM, NULL, NULL, NULL);
+    MARKER_STACK_UNWIND(rmdir, frame, -1, ENOMEM, NULL, NULL, NULL);
 
-        return 0;
+    return 0;
 }
 
-
 int32_t
-marker_unlink_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                   int32_t op_ret, int32_t op_errno, struct iatt *preparent,
-                   struct iatt *postparent, dict_t *xdata)
+marker_unlink_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                  int32_t op_ret, int32_t op_errno, struct iatt *preparent,
+                  struct iatt *postparent, dict_t *xdata)
 {
-        marker_conf_t      *priv    = NULL;
-        marker_local_t     *local   = NULL;
+    marker_conf_t *priv = NULL;
+    marker_local_t *local = NULL;
+    uint32_t nlink = -1;
+    GF_UNUSED int32_t ret = 0;
+    call_stub_t *stub = NULL;
 
-        if (op_ret == -1) {
-                gf_log (this->name, GF_LOG_TRACE,
-                        "%s occurred in unlink", strerror (op_errno));
-        }
+    if (op_ret == -1) {
+        gf_log(this->name, GF_LOG_TRACE, "%s occurred in unlink",
+               strerror(op_errno));
+    }
 
-        local = (marker_local_t *) frame->local;
+    local = (marker_local_t *)frame->local;
 
-        frame->local = NULL;
+    frame->local = NULL;
+    priv = this->private;
 
-        STACK_UNWIND_STRICT (unlink, frame, op_ret, op_errno, preparent,
-                             postparent, xdata);
+    if (op_ret == -1 || local == NULL)
+        goto out;
 
-        if (op_ret == -1 || local == NULL)
-                goto out;
+    if (priv->feature_enabled & GF_XTIME)
+        marker_xtime_update_marks(this, local);
 
-        priv = this->private;
+    if (priv->feature_enabled & GF_QUOTA) {
+        if (local->skip_txn)
+            goto out;
 
-        if (priv->feature_enabled & GF_QUOTA) {
-                if (!local->skip_txn)
-                        mq_reduce_parent_size_txn (this, &local->loc, -1);
+        if (xdata) {
+            ret = dict_get_uint32(xdata, GF_RESPONSE_LINK_COUNT_XDATA, &nlink);
+            if (ret) {
+                gf_log(this->name, GF_LOG_TRACE, "dict get failed %s ",
+                       strerror(-ret));
+            }
+        }
+
+        /* If a 'rm -rf' is performed by a client, unlink can be faster
+           than marker background mq_reduce_parent_size_txn.
+           In this case, as part of unlink parent child association
+           will be removed in the server protocol.
+           This can lead to mq_reduce_parent_size_txn failures.
+
+          So perform mq_reduce_parent_size_txn in foreground
+          and unwind to server once txn is complete
+        */
+
+        stub = fop_unlink_cbk_stub(frame, default_unlink_cbk, op_ret, op_errno,
+                                   preparent, postparent, xdata);
+        mq_reduce_parent_size_txn(this, &local->loc, NULL, nlink, stub);
+
+        if (stub) {
+            marker_local_unref(local);
+            return 0;
         }
+    }
 
-        if (priv->feature_enabled & GF_XTIME)
-                marker_xtime_update_marks (this, local);
 out:
-        marker_local_unref (local);
+    STACK_UNWIND_STRICT(unlink, frame, op_ret, op_errno, preparent, postparent,
+                        xdata);
 
-        return 0;
-}
+    marker_local_unref(local);
 
+    return 0;
+}
 
 int32_t
-marker_unlink (call_frame_t *frame, xlator_t *this, loc_t *loc, int xflag,
-               dict_t *xdata)
+marker_unlink(call_frame_t *frame, xlator_t *this, loc_t *loc, int xflag,
+              dict_t *xdata)
 {
-        int32_t          ret   = 0;
-        marker_local_t  *local = NULL;
-        marker_conf_t   *priv  = NULL;
+    int32_t ret = 0;
+    marker_local_t *local = NULL;
+    marker_conf_t *priv = NULL;
+    gf_boolean_t dict_free = _gf_false;
 
-        priv = this->private;
+    priv = this->private;
 
-        if (priv->feature_enabled == 0)
-                goto unlink_wind;
+    if (priv->feature_enabled == 0)
+        goto unlink_wind;
 
-        local = mem_get0 (this->local_pool);
-        local->xflag = xflag;
-        if (xdata)
-                local->xdata = dict_ref (xdata);
-        MARKER_INIT_LOCAL (frame, local);
+    local = mem_get0(this->local_pool);
+    local->xflag = xflag;
+    if (xdata)
+        local->xdata = dict_ref(xdata);
+    MARKER_INIT_LOCAL(frame, local);
 
-        ret = loc_copy (&local->loc, loc);
+    ret = loc_copy(&local->loc, loc);
 
-        if (ret == -1)
-                goto err;
+    if (ret == -1)
+        goto err;
 
-        if (xdata && dict_get (xdata, GLUSTERFS_MARKER_DONT_ACCOUNT_KEY)) {
-                local->skip_txn = 1;
-                goto unlink_wind;
-        }
+    if (xdata && dict_get(xdata, GLUSTERFS_MARKER_DONT_ACCOUNT_KEY)) {
+        local->skip_txn = 1;
+        goto unlink_wind;
+    }
+
+    if (xdata == NULL) {
+        xdata = dict_new();
+        dict_free = _gf_true;
+    }
+
+    ret = dict_set_int32(xdata, GF_REQUEST_LINK_COUNT_XDATA, 1);
+    if (ret < 0)
+        goto err;
 
 unlink_wind:
-        STACK_WIND (frame, marker_unlink_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->unlink, loc, xflag, xdata);
-        return 0;
+    STACK_WIND(frame, marker_unlink_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->unlink, loc, xflag, xdata);
+    goto out;
+
 err:
-        frame->local = NULL;
-        STACK_UNWIND_STRICT (unlink, frame, -1, ENOMEM, NULL, NULL, NULL);
-        marker_local_unref (local);
-        return 0;
-}
+    MARKER_STACK_UNWIND(unlink, frame, -1, ENOMEM, NULL, NULL, NULL);
 
+out:
+    if (dict_free)
+        dict_unref(xdata);
+    return 0;
+}
 
 int32_t
-marker_link_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                 int32_t op_ret, int32_t op_errno, inode_t *inode,
-                 struct iatt *buf, struct iatt *preparent,
-                 struct iatt *postparent, dict_t *xdata)
+marker_link_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                int32_t op_ret, int32_t op_errno, inode_t *inode,
+                struct iatt *buf, struct iatt *preparent,
+                struct iatt *postparent, dict_t *xdata)
 {
-        marker_local_t     *local   = NULL;
-        marker_conf_t      *priv    = NULL;
-
-        if (op_ret == -1) {
-                gf_log (this->name, GF_LOG_TRACE, "%s occurred while "
-                        "linking a file ", strerror (op_errno));
-        }
+    marker_local_t *local = NULL;
+    marker_conf_t *priv = NULL;
 
-        local = (marker_local_t *) frame->local;
+    if (op_ret == -1) {
+        gf_log(this->name, GF_LOG_TRACE,
+               "%s occurred while "
+               "linking a file ",
+               strerror(op_errno));
+    }
 
-        frame->local = NULL;
+    local = (marker_local_t *)frame->local;
 
-        STACK_UNWIND_STRICT (link, frame, op_ret, op_errno, inode, buf,
-                             preparent, postparent, xdata);
+    frame->local = NULL;
 
-        if (op_ret == -1 || local == NULL)
-                goto out;
+    STACK_UNWIND_STRICT(link, frame, op_ret, op_errno, inode, buf, preparent,
+                        postparent, xdata);
 
-        priv = this->private;
+    if (op_ret == -1 || local == NULL)
+        goto out;
 
-        if (priv->feature_enabled & GF_QUOTA) {
-                if (!local->skip_txn)
-                        mq_create_xattrs_txn (this, &local->loc);
-        }
+    priv = this->private;
 
+    if (priv->feature_enabled & GF_QUOTA) {
+        if (!local->skip_txn)
+            mq_create_xattrs_txn(this, &local->loc, buf);
+    }
 
-        if (priv->feature_enabled & GF_XTIME)
-                marker_xtime_update_marks (this, local);
+    if (priv->feature_enabled & GF_XTIME)
+        marker_xtime_update_marks(this, local);
 out:
-        marker_local_unref (local);
+    marker_local_unref(local);
 
-        return 0;
+    return 0;
 }
 
 int32_t
-marker_link (call_frame_t *frame, xlator_t *this, loc_t *oldloc, loc_t *newloc,
-             dict_t *xdata)
+marker_link(call_frame_t *frame, xlator_t *this, loc_t *oldloc, loc_t *newloc,
+            dict_t *xdata)
 {
-        int32_t          ret   = 0;
-        marker_local_t  *local = NULL;
-        marker_conf_t   *priv  = NULL;
+    int32_t ret = 0;
+    marker_local_t *local = NULL;
+    marker_conf_t *priv = NULL;
 
-        priv = this->private;
+    priv = this->private;
 
-        if (priv->feature_enabled == 0)
-                goto wind;
+    if (priv->feature_enabled == 0)
+        goto wind;
 
-        local = mem_get0 (this->local_pool);
+    local = mem_get0(this->local_pool);
 
-        MARKER_INIT_LOCAL (frame, local);
+    MARKER_INIT_LOCAL(frame, local);
 
-        ret = loc_copy (&local->loc, newloc);
+    ret = loc_copy(&local->loc, newloc);
 
-        if (ret == -1)
-                goto err;
+    if (ret == -1)
+        goto err;
 
-        if (xdata && dict_get (xdata, GLUSTERFS_MARKER_DONT_ACCOUNT_KEY))
-                local->skip_txn = 1;
+    if (xdata && dict_get(xdata, GLUSTERFS_MARKER_DONT_ACCOUNT_KEY))
+        local->skip_txn = 1;
 wind:
-        STACK_WIND (frame, marker_link_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->link, oldloc, newloc, xdata);
-        return 0;
+    STACK_WIND(frame, marker_link_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->link, oldloc, newloc, xdata);
+    return 0;
 err:
-        STACK_UNWIND_STRICT (link, frame, -1, ENOMEM, NULL, NULL, NULL, NULL,
-                             NULL);
+    MARKER_STACK_UNWIND(link, frame, -1, ENOMEM, NULL, NULL, NULL, NULL, NULL);
 
-        return 0;
+    return 0;
 }
 
-
 int32_t
-marker_rename_done (call_frame_t *frame, void *cookie, xlator_t *this,
-                    int32_t op_ret, int32_t op_errno, dict_t *xdata)
+marker_rename_done(call_frame_t *frame, void *cookie, xlator_t *this,
+                   int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
-        marker_local_t *local  = NULL, *oplocal = NULL;
-        loc_t           newloc = {0, };
-        marker_conf_t  *priv   = NULL;
-
-        local = frame->local;
-        oplocal = local->oplocal;
-
-        priv = this->private;
-
-        frame->local = NULL;
-
-        if (op_ret < 0) {
-                if (local->err == 0) {
-                        local->err =  op_errno ? op_errno : EINVAL;
-                }
+    marker_local_t *local = NULL, *oplocal = NULL;
+    loc_t newloc = {
+        0,
+    };
+    marker_conf_t *priv = NULL;
 
-                gf_log (this->name, GF_LOG_WARNING,
-                        "inodelk (UNLOCK) failed on path:%s (gfid:%s) (%s)",
-                        local->parent_loc.path,
-                        uuid_utoa (local->parent_loc.inode->gfid),
-                        strerror (op_errno));
-        }
+    local = frame->local;
+    oplocal = local->oplocal;
 
-        if (local->stub != NULL) {
-                call_resume (local->stub);
-                local->stub = NULL;
-        } else if (local->err != 0) {
-                STACK_UNWIND_STRICT (rename, frame, -1, local->err, NULL, NULL,
-                                     NULL, NULL, NULL, NULL);
-        } else {
-                gf_log (this->name, GF_LOG_CRITICAL,
-                        "continuation stub to unwind the call is absent, hence "
-                        "call will be hung (call-stack id = %"PRIu64")",
-                        frame->root->unique);
-        }
+    priv = this->private;
 
-        mq_reduce_parent_size_txn (this, &oplocal->loc, oplocal->contribution);
+    frame->local = NULL;
 
-        if (local->loc.inode != NULL) {
-                mq_reduce_parent_size_txn (this, &local->loc,
-                                           local->contribution);
-        }
+    if (op_ret < 0) {
+        gf_log(this->name, GF_LOG_WARNING,
+               "inodelk (UNLOCK) failed on path:%s (gfid:%s) (%s)",
+               oplocal->parent_loc.path,
+               uuid_utoa(oplocal->parent_loc.inode->gfid), strerror(op_errno));
+    }
 
-        newloc.inode = inode_ref (oplocal->loc.inode);
-        newloc.path = gf_strdup (local->loc.path);
-        newloc.name = strrchr (newloc.path, '/');
-        if (newloc.name)
-                newloc.name++;
-        newloc.parent = inode_ref (local->loc.parent);
+    if (local->err != 0)
+        goto err;
 
-        mq_create_xattrs_txn (this, &newloc);
+    mq_reduce_parent_size_txn(this, &oplocal->loc, &oplocal->contribution, -1,
+                              NULL);
 
-        loc_wipe (&newloc);
+    if (local->loc.inode != NULL) {
+        /* If destination file exits before rename, it would have
+         * been unlinked while renaming a file
+         */
+        mq_reduce_parent_size_txn(this, &local->loc, NULL, local->ia_nlink,
+                                  NULL);
+    }
+
+    newloc.inode = inode_ref(oplocal->loc.inode);
+    newloc.path = gf_strdup(local->loc.path);
+    newloc.name = strrchr(newloc.path, '/');
+    if (newloc.name)
+        newloc.name++;
+    newloc.parent = inode_ref(local->loc.parent);
+
+    mq_create_xattrs_txn(this, &newloc, &local->buf);
+
+    loc_wipe(&newloc);
+
+    if (priv->feature_enabled & GF_XTIME) {
+        if (!local->loc.inode)
+            local->loc.inode = inode_ref(oplocal->loc.inode);
+        // update marks on oldpath
+        gf_uuid_copy(local->loc.gfid, oplocal->loc.inode->gfid);
+        marker_xtime_update_marks(this, oplocal);
+        marker_xtime_update_marks(this, local);
+    }
 
-        if (priv->feature_enabled & GF_XTIME) {
-                //update marks on oldpath
-                gf_uuid_copy (local->loc.gfid, oplocal->loc.inode->gfid);
-                marker_xtime_update_marks (this, oplocal);
-                marker_xtime_update_marks (this, local);
-        }
+err:
+    marker_local_unref(local);
+    marker_local_unref(oplocal);
 
-        marker_local_unref (local);
-        marker_local_unref (oplocal);
-        return 0;
+    return 0;
 }
 
-
-int32_t
-marker_rename_release_newp_lock (call_frame_t *frame, void *cookie,
-                                 xlator_t *this, int32_t op_ret,
-                                 int32_t op_errno, dict_t *xdata)
+void
+marker_rename_release_oldp_lock(marker_local_t *local, xlator_t *this)
 {
-        marker_local_t  *local = NULL, *oplocal = NULL;
-        struct gf_flock  lock  = {0, };
+    marker_local_t *oplocal = NULL;
+    call_frame_t *lk_frame = NULL;
+    struct gf_flock lock = {
+        0,
+    };
 
-        local = frame->local;
-        oplocal = local->oplocal;
-
-        if (op_ret < 0) {
-                if (local->err == 0) {
-                        local->err = op_errno ? op_errno : EINVAL;
-                }
+    oplocal = local->oplocal;
+    lk_frame = local->lk_frame;
 
-                gf_log (this->name, GF_LOG_WARNING,
-                        "inodelk (UNLOCK) failed on %s (gfid:%s) (%s)",
-                        oplocal->parent_loc.path,
-                        uuid_utoa (oplocal->parent_loc.inode->gfid),
-                        strerror (op_errno));
-        }
+    if (lk_frame == NULL)
+        goto err;
 
-        if (local->next_lock_on == NULL) {
-                marker_rename_done (frame, NULL, this, 0, 0, NULL);
-                goto out;
-        }
+    lock.l_type = F_UNLCK;
+    lock.l_whence = SEEK_SET;
+    lock.l_start = 0;
+    lock.l_len = 0;
+    lock.l_pid = 0;
 
-        lock.l_type   = F_UNLCK;
-        lock.l_whence = SEEK_SET;
-        lock.l_start  = 0;
-        lock.l_len    = 0;
-        lock.l_pid    = 0;
+    STACK_WIND(lk_frame, marker_rename_done, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->inodelk, this->name,
+               &oplocal->parent_loc, F_SETLKW, &lock, NULL);
 
-        STACK_WIND (frame,
-                    marker_rename_done,
-                    FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->inodelk,
-                    this->name, &local->parent_loc, F_SETLKW, &lock, NULL);
+    return;
 
-out:
-        return 0;
+err:
+    marker_local_unref(local);
+    marker_local_unref(oplocal);
 }
 
-
 int32_t
-marker_rename_release_oldp_lock (call_frame_t *frame, void *cookie,
-                                 xlator_t *this, int32_t op_ret,
-                                 int32_t op_errno, dict_t *xdata)
+marker_rename_unwind(call_frame_t *frame, void *cookie, xlator_t *this,
+                     int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
-        marker_local_t  *local = NULL, *oplocal = NULL;
-        struct gf_flock  lock  = {0, };
-
-        local = frame->local;
-        oplocal = local->oplocal;
-
-        if ((op_ret < 0) && (op_errno != ENOATTR) && (op_errno != ENODATA)) {
-                local->err = op_errno;
-        }
-
-        //Reset frame uid and gid if set.
-        if (cookie == (void *) _GF_UID_GID_CHANGED)
-                MARKER_RESET_UID_GID (frame, frame->root, local);
-
-        lock.l_type   = F_UNLCK;
-        lock.l_whence = SEEK_SET;
-        lock.l_start  = 0;
-        lock.l_len    = 0;
-        lock.l_pid    = 0;
-
-        STACK_WIND (frame,
-                    marker_rename_release_newp_lock,
-                    FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->inodelk,
-                    this->name, &oplocal->parent_loc, F_SETLKW, &lock, NULL);
-        return 0;
-}
-
-
-int32_t
-marker_rename_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                   int32_t op_ret, int32_t op_errno, struct iatt *buf,
-                   struct iatt *preoldparent, struct iatt *postoldparent,
-                   struct iatt *prenewparent, struct iatt *postnewparent,
-                   dict_t *xdata)
-{
-        marker_conf_t  *priv                            = NULL;
-        marker_local_t *local                           = NULL;
-        marker_local_t *oplocal                         = NULL;
-        call_stub_t    *stub                            = NULL;
-        int32_t         ret                             = 0;
-        char            contri_key[CONTRI_KEY_MAX]      = {0, };
-        loc_t           newloc                          = {0, };
-
-        local = (marker_local_t *) frame->local;
+    marker_local_t *local = NULL;
+    marker_local_t *oplocal = NULL;
+    quota_inode_ctx_t *ctx = NULL;
+    inode_contribution_t *contri = NULL;
 
-        if (local != NULL) {
-                oplocal = local->oplocal;
-        }
-
-        priv = this->private;
-
-        if (op_ret < 0) {
-                if (local != NULL) {
-                        local->err = op_errno;
-                }
-
-                gf_log (this->name, GF_LOG_TRACE, "%s occurred while "
-                        "renaming a file ", strerror (op_errno));
-        }
-
-        if (priv->feature_enabled & GF_QUOTA) {
-                if ((op_ret < 0) || (local == NULL)) {
-                        goto quota_err;
-                }
-
-                stub = fop_rename_cbk_stub (frame, default_rename_cbk, op_ret,
-                                            op_errno, buf, preoldparent,
-                                            postoldparent, prenewparent,
-                                            postnewparent, xdata);
-                if (stub == NULL) {
-                        local->err = ENOMEM;
-                        goto quota_err;
-                }
-
-                local->stub = stub;
-
-                GET_CONTRI_KEY (contri_key, oplocal->loc.parent->gfid, ret);
-                if (ret < 0) {
-                        local->err = ENOMEM;
-                        goto quota_err;
-                }
-
-                /* Removexattr requires uid and gid to be 0,
-                 * reset them in the callback.
-                 */
-                MARKER_SET_UID_GID (frame, local, frame->root);
-
-                newloc.inode = inode_ref (oplocal->loc.inode);
-                newloc.path = gf_strdup (local->loc.path);
-                newloc.name = strrchr (newloc.path, '/');
-                if (newloc.name)
-                        newloc.name++;
-                newloc.parent = inode_ref (local->loc.parent);
-                gf_uuid_copy (newloc.gfid, oplocal->loc.inode->gfid);
-
-                STACK_WIND_COOKIE (frame, marker_rename_release_oldp_lock,
-                                   frame->cookie, FIRST_CHILD(this),
-                                   FIRST_CHILD(this)->fops->removexattr,
-                                   &newloc, contri_key, NULL);
-
-                loc_wipe (&newloc);
-        } else {
-                frame->local = NULL;
-
-                STACK_UNWIND_STRICT (rename, frame, op_ret, op_errno, buf,
-                                     preoldparent, postoldparent, prenewparent,
-                                     postnewparent, xdata);
-
-                if ((op_ret < 0) || (local == NULL)) {
-                        goto out;
-                }
-
-                if (priv->feature_enabled & GF_XTIME) {
-                        //update marks on oldpath
-                        gf_uuid_copy (local->loc.gfid, oplocal->loc.inode->gfid);
-                        marker_xtime_update_marks (this, oplocal);
-                        marker_xtime_update_marks (this, local);
-                }
-        }
+    local = frame->local;
+    oplocal = local->oplocal;
+    frame->local = NULL;
 
-out:
-        if (!(priv->feature_enabled & GF_QUOTA)) {
-                marker_local_unref (local);
-                marker_local_unref (oplocal);
-        }
+    // Reset frame uid and gid if set.
+    if (cookie == (void *)_GF_UID_GID_CHANGED)
+        MARKER_RESET_UID_GID(frame, frame->root, local);
 
-        return 0;
+    if (op_ret < 0)
+        local->err = op_errno ? op_errno : EINVAL;
 
-quota_err:
-        marker_rename_release_oldp_lock (frame, NULL, this, 0, 0, NULL);
-        return 0;
+    if (local->stub != NULL) {
+        /* Remove contribution node from in-memory even if
+         * remove-xattr has failed as the rename is already performed
+         * if local->stub is set, which means rename was successful
+         */
+        (void)mq_inode_ctx_get(oplocal->loc.inode, this, &ctx);
+        if (ctx) {
+            contri = mq_get_contribution_node(oplocal->loc.parent, ctx);
+            if (contri) {
+                QUOTA_FREE_CONTRIBUTION_NODE(ctx, contri);
+                GF_REF_PUT(contri);
+            }
+        }
+
+        call_resume(local->stub);
+        local->stub = NULL;
+        local->err = 0;
+    } else if (local->err != 0) {
+        STACK_UNWIND_STRICT(rename, frame, -1, local->err, NULL, NULL, NULL,
+                            NULL, NULL, NULL);
+    } else {
+        gf_log(this->name, GF_LOG_CRITICAL,
+               "continuation stub to unwind the call is absent, hence "
+               "call will be hung (call-stack id = %" PRIu64 ")",
+               frame->root->unique);
+    }
+
+    /* If there are in-progress writes on old-path when during rename
+     * operation, update txn will update the wrong path if lock
+     * is released before rename unwind.
+     * So release lock only after rename unwind
+     */
+    marker_rename_release_oldp_lock(local, this);
+
+    return 0;
 }
 
-
 int32_t
-marker_do_rename (call_frame_t *frame, void *cookie, xlator_t *this,
-                  int32_t op_ret, int32_t op_errno, dict_t *dict, dict_t *xdata)
+marker_rename_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                  int32_t op_ret, int32_t op_errno, struct iatt *buf,
+                  struct iatt *preoldparent, struct iatt *postoldparent,
+                  struct iatt *prenewparent, struct iatt *postnewparent,
+                  dict_t *xdata)
 {
-        marker_local_t       *local                      = NULL;
-        marker_local_t       *oplocal                    = NULL;
-        char                  contri_key[CONTRI_KEY_MAX] = {0, };
-        int32_t               ret                        = 0;
-        int64_t              *contribution               = 0;
-
-        local = frame->local;
+    marker_conf_t *priv = NULL;
+    marker_local_t *local = NULL;
+    marker_local_t *oplocal = NULL;
+    call_stub_t *stub = NULL;
+    int32_t ret = 0;
+    char contri_key[QUOTA_KEY_MAX] = {
+        0,
+    };
+    loc_t newloc = {
+        0,
+    };
+
+    local = (marker_local_t *)frame->local;
+
+    if (local != NULL) {
         oplocal = local->oplocal;
+    }
 
-        //Reset frame uid and gid if set.
-        if (cookie == (void *) _GF_UID_GID_CHANGED)
-                MARKER_RESET_UID_GID (frame, frame->root, local);
-
-        if ((op_ret < 0) && (op_errno != ENOATTR) && (op_errno != ENODATA)) {
-                local->err = op_errno ? op_errno : EINVAL;
-                gf_log (this->name, GF_LOG_WARNING,
-                        "fetching contribution values from %s (gfid:%s) "
-                        "failed (%s)", local->loc.path,
-                        uuid_utoa (local->loc.inode->gfid),
-                        strerror (op_errno));
-                goto err;
-        }
+    priv = this->private;
 
-        if (local->loc.inode != NULL) {
-                GET_CONTRI_KEY (contri_key, local->loc.parent->gfid, ret);
-                if (ret < 0) {
-                        local->err = errno ? errno : ENOMEM;
-                        goto err;
-                }
-
-                if (dict_get_bin (dict, contri_key,
-                                  (void **) &contribution) == 0) {
-                        local->contribution = ntoh64 (*contribution);
-                }
+    if (op_ret < 0) {
+        if (local != NULL) {
+            local->err = op_errno;
         }
 
-        STACK_WIND (frame, marker_rename_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->rename, &oplocal->loc,
-                    &local->loc, NULL);
-
-        return 0;
-
-err:
-        marker_rename_release_oldp_lock (frame, NULL, this, 0, 0, NULL);
-        return 0;
-}
-
-
-int32_t
-marker_get_newpath_contribution (call_frame_t *frame, void *cookie,
-                                 xlator_t *this, int32_t op_ret,
-                                 int32_t op_errno, dict_t *dict, dict_t *xdata)
-{
-        marker_local_t *local                      = NULL;
-        marker_local_t *oplocal                    = NULL;
-        char            contri_key[CONTRI_KEY_MAX] = {0, };
-        int32_t         ret                        = 0;
-        int64_t        *contribution               = 0;
+        gf_log(this->name, GF_LOG_TRACE,
+               "%s occurred while "
+               "renaming a file ",
+               strerror(op_errno));
+    }
 
-        local = frame->local;
-        oplocal = local->oplocal;
-
-        //Reset frame uid and gid if set.
-        if (cookie == (void *) _GF_UID_GID_CHANGED)
-                MARKER_RESET_UID_GID (frame, frame->root, local);
-
-        if ((op_ret < 0) && (op_errno != ENOATTR) && (op_errno != ENODATA)) {
-                local->err = op_errno ? op_errno : EINVAL;
-                gf_log (this->name, GF_LOG_WARNING,
-                        "fetching contribution values from %s (gfid:%s) "
-                        "failed (%s)", oplocal->loc.path,
-                        uuid_utoa (oplocal->loc.inode->gfid),
-                        strerror (op_errno));
-                goto err;
+    if (priv->feature_enabled & GF_QUOTA) {
+        if ((op_ret < 0) || (local == NULL)) {
+            goto quota_err;
         }
 
-        GET_CONTRI_KEY (contri_key, oplocal->loc.parent->gfid, ret);
-        if (ret < 0) {
-                local->err = errno ? errno : ENOMEM;
-                goto err;
-        }
+        local->ia_nlink = 0;
+        if (xdata)
+            ret = dict_get_uint32(xdata, GF_RESPONSE_LINK_COUNT_XDATA,
+                                  &local->ia_nlink);
 
-        if (dict_get_bin (dict, contri_key, (void **) &contribution) == 0)
-                oplocal->contribution = ntoh64 (*contribution);
-
-        if (local->loc.inode != NULL) {
-                GET_CONTRI_KEY (contri_key, local->loc.parent->gfid, ret);
-                if (ret < 0) {
-                        local->err = errno ? errno : ENOMEM;
-                        goto err;
-                }
-
-                /* getxattr requires uid and gid to be 0,
-                 * reset them in the callback.
-                 */
-                MARKER_SET_UID_GID (frame, local, frame->root);
-                if (gf_uuid_is_null (local->loc.gfid))
-                        gf_uuid_copy (local->loc.gfid, local->loc.inode->gfid);
-
-                GF_UUID_ASSERT (local->loc.gfid);
-
-                STACK_WIND_COOKIE (frame, marker_do_rename,
-                                   frame->cookie, FIRST_CHILD(this),
-                                   FIRST_CHILD(this)->fops->getxattr,
-                                   &local->loc, contri_key, NULL);
-        } else {
-                marker_do_rename (frame, NULL, this, 0, 0, NULL, NULL);
+        local->buf = *buf;
+        stub = fop_rename_cbk_stub(frame, default_rename_cbk, op_ret, op_errno,
+                                   buf, preoldparent, postoldparent,
+                                   prenewparent, postnewparent, xdata);
+        if (stub == NULL) {
+            local->err = ENOMEM;
+            goto quota_err;
         }
 
-        return 0;
-err:
-        marker_rename_release_oldp_lock (frame, NULL, this, 0, 0, NULL);
-        return 0;
-}
-
-
-int32_t
-marker_get_oldpath_contribution (call_frame_t *frame, void *cookie,
-                                 xlator_t *this, int32_t op_ret,
-                                 int32_t op_errno, dict_t *xdata)
-{
-        marker_local_t *local                      = NULL;
-        marker_local_t *oplocal                    = NULL;
-        char            contri_key[CONTRI_KEY_MAX] = {0, };
-        int32_t         ret                        = 0;
-
-        local = frame->local;
-        oplocal = local->oplocal;
+        local->stub = stub;
 
-        if (op_ret < 0) {
-                local->err = op_errno ? op_errno : EINVAL;
-                gf_log (this->name, GF_LOG_WARNING,
-                        "cannot hold inodelk on %s (gfid:%s) (%s)",
-                        local->next_lock_on->path,
-                        uuid_utoa (local->next_lock_on->inode->gfid),
-                        strerror (op_errno));
-                goto lock_err;
-        }
-
-        GET_CONTRI_KEY (contri_key, oplocal->loc.parent->gfid, ret);
+        GET_CONTRI_KEY(this, contri_key, oplocal->loc.parent->gfid, ret);
         if (ret < 0) {
-                local->err = errno ? errno : ENOMEM;
-                goto quota_err;
+            local->err = ENOMEM;
+            goto quota_err;
         }
 
-        /* getxattr requires uid and gid to be 0,
+        /* Removexattr requires uid and gid to be 0,
          * reset them in the callback.
          */
-        MARKER_SET_UID_GID (frame, local, frame->root);
+        MARKER_SET_UID_GID(frame, local, frame->root);
 
-        if (gf_uuid_is_null (oplocal->loc.gfid))
-                        gf_uuid_copy (oplocal->loc.gfid,
-                                   oplocal->loc.inode->gfid);
+        newloc.inode = inode_ref(oplocal->loc.inode);
+        newloc.path = gf_strdup(local->loc.path);
+        newloc.name = strrchr(newloc.path, '/');
+        if (newloc.name)
+            newloc.name++;
+        newloc.parent = inode_ref(local->loc.parent);
+        gf_uuid_copy(newloc.gfid, oplocal->loc.inode->gfid);
 
-        GF_UUID_ASSERT (oplocal->loc.gfid);
+        STACK_WIND_COOKIE(
+            frame, marker_rename_unwind, frame->cookie, FIRST_CHILD(this),
+            FIRST_CHILD(this)->fops->removexattr, &newloc, contri_key, NULL);
 
-        STACK_WIND_COOKIE (frame, marker_get_newpath_contribution,
-                           frame->cookie, FIRST_CHILD(this),
-                           FIRST_CHILD(this)->fops->getxattr,
-                           &oplocal->loc, contri_key, NULL);
-        return 0;
+        loc_wipe(&newloc);
+    } else {
+        frame->local = NULL;
 
-quota_err:
-        marker_rename_release_oldp_lock (frame, NULL, this, 0, 0, NULL);
-        return 0;
+        STACK_UNWIND_STRICT(rename, frame, op_ret, op_errno, buf, preoldparent,
+                            postoldparent, prenewparent, postnewparent, xdata);
 
-lock_err:
-        if ((local->next_lock_on == NULL)
-            || (local->next_lock_on == &local->parent_loc)) {
-                local->next_lock_on = NULL;
-                marker_rename_release_oldp_lock (frame, NULL, this, 0, 0, NULL);
-        } else {
-                marker_rename_release_newp_lock (frame, NULL, this, 0, 0, NULL);
+        if ((op_ret < 0) || (local == NULL)) {
+            goto out;
         }
 
-        return 0;
-}
-
-
-int32_t
-marker_rename_inodelk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                           int32_t op_ret, int32_t op_errno, dict_t *xdata)
-{
-        marker_local_t  *local = NULL, *oplocal = NULL;
-        loc_t           *loc   = NULL;
-        struct gf_flock  lock  = {0, };
+        if (priv->feature_enabled & GF_XTIME) {
+            // update marks on oldpath
+            if (!local->loc.inode)
+                local->loc.inode = inode_ref(oplocal->loc.inode);
+            gf_uuid_copy(local->loc.gfid, oplocal->loc.inode->gfid);
+            marker_xtime_update_marks(this, oplocal);
+            marker_xtime_update_marks(this, local);
+        }
+    }
 
-        local = frame->local;
-        oplocal = local->oplocal;
+out:
+    if (!(priv->feature_enabled & GF_QUOTA)) {
+        marker_local_unref(local);
+        marker_local_unref(oplocal);
+    }
 
-        if (op_ret < 0) {
-                if (local->next_lock_on != &oplocal->parent_loc) {
-                        loc = &oplocal->parent_loc;
-                } else {
-                        loc = &local->parent_loc;
-                }
-
-                local->err = op_errno ? op_errno : EINVAL;
-                gf_log (this->name, GF_LOG_WARNING,
-                        "cannot hold inodelk on %s (gfid:%s) (%s)",
-                        loc->path, uuid_utoa (loc->inode->gfid),
-                        strerror (op_errno));
-                goto err;
-        }
+    return 0;
 
-        if (local->next_lock_on != NULL) {
-                lock.l_len    = 0;
-                lock.l_start  = 0;
-                lock.l_type   = F_WRLCK;
-                lock.l_whence = SEEK_SET;
-
-                STACK_WIND (frame,
-                            marker_get_oldpath_contribution,
-                            FIRST_CHILD(this),
-                            FIRST_CHILD(this)->fops->inodelk,
-                            this->name, local->next_lock_on,
-                            F_SETLKW, &lock, NULL);
-        } else {
-                marker_get_oldpath_contribution (frame, 0, this, 0, 0, NULL);
-        }
+quota_err:
+    marker_rename_unwind(frame, NULL, this, 0, 0, NULL);
+    return 0;
+}
 
-        return 0;
+int32_t
+marker_do_rename(call_frame_t *frame, void *cookie, xlator_t *this,
+                 int32_t op_ret, int32_t op_errno, dict_t *dict, dict_t *xdata)
+{
+    marker_local_t *local = NULL;
+    marker_local_t *oplocal = NULL;
+    char contri_key[QUOTA_KEY_MAX] = {
+        0,
+    };
+    int keylen = 0;
+    quota_meta_t contribution = {
+        0,
+    };
+
+    local = frame->local;
+    oplocal = local->oplocal;
+
+    // Reset frame uid and gid if set.
+    if (cookie == (void *)_GF_UID_GID_CHANGED)
+        MARKER_RESET_UID_GID(frame, frame->root, local);
+
+    if ((op_ret < 0) && (op_errno != ENOATTR) && (op_errno != ENODATA)) {
+        local->err = op_errno ? op_errno : EINVAL;
+        gf_log(this->name, GF_LOG_WARNING,
+               "fetching contribution values from %s (gfid:%s) "
+               "failed (%s)",
+               oplocal->loc.path, uuid_utoa(oplocal->loc.inode->gfid),
+               strerror(op_errno));
+        goto err;
+    }
+
+    GET_CONTRI_KEY(this, contri_key, oplocal->loc.parent->gfid, keylen);
+    if (keylen < 0) {
+        local->err = errno ? errno : ENOMEM;
+        goto err;
+    }
+    quota_dict_get_meta(dict, contri_key, keylen, &contribution);
+    oplocal->contribution = contribution;
+
+    STACK_WIND(frame, marker_rename_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->rename, &oplocal->loc, &local->loc,
+               local->xdata);
+
+    return 0;
 
 err:
-        marker_rename_done (frame, NULL, this, 0, 0, NULL);
-        return 0;
+    marker_rename_unwind(frame, NULL, this, 0, 0, NULL);
+    return 0;
 }
 
-
 int32_t
-marker_rename (call_frame_t *frame, xlator_t *this, loc_t *oldloc,
-               loc_t *newloc, dict_t *xdata)
+marker_get_oldpath_contribution(call_frame_t *lk_frame, void *cookie,
+                                xlator_t *this, int32_t op_ret,
+                                int32_t op_errno, dict_t *xdata)
+{
+    call_frame_t *frame = NULL;
+    marker_local_t *local = NULL;
+    marker_local_t *oplocal = NULL;
+    char contri_key[QUOTA_KEY_MAX] = {
+        0,
+    };
+    int32_t ret = 0;
+
+    local = lk_frame->local;
+    oplocal = local->oplocal;
+    frame = local->frame;
+
+    if (op_ret < 0) {
+        local->err = op_errno ? op_errno : EINVAL;
+        gf_log(this->name, GF_LOG_WARNING,
+               "cannot hold inodelk on %s (gfid:%s) (%s)", oplocal->loc.path,
+               uuid_utoa(oplocal->loc.inode->gfid), strerror(op_errno));
+        if (local->lk_frame) {
+            STACK_DESTROY(local->lk_frame->root);
+            local->lk_frame = NULL;
+        }
+        goto err;
+    }
+
+    GET_CONTRI_KEY(this, contri_key, oplocal->loc.parent->gfid, ret);
+    if (ret < 0) {
+        local->err = errno ? errno : ENOMEM;
+        goto err;
+    }
+
+    /* getxattr requires uid and gid to be 0,
+     * reset them in the callback.
+     */
+    MARKER_SET_UID_GID(frame, local, frame->root);
+
+    if (gf_uuid_is_null(oplocal->loc.gfid))
+        gf_uuid_copy(oplocal->loc.gfid, oplocal->loc.inode->gfid);
+
+    GF_UUID_ASSERT(oplocal->loc.gfid);
+
+    STACK_WIND_COOKIE(frame, marker_do_rename, frame->cookie, FIRST_CHILD(this),
+                      FIRST_CHILD(this)->fops->getxattr, &oplocal->loc,
+                      contri_key, NULL);
+
+    return 0;
+err:
+    marker_rename_unwind(frame, NULL, this, 0, 0, NULL);
+    return 0;
+}
+
+/* For a marker_rename FOP, following is the algorithm used for Quota
+ * accounting. The use-case considered is:
+ * 1. rename (src, dst)
+ * 2. both src and dst exist
+ * 3. there are parallel operations on src and dst (lets say through fds
+ *    opened on them before rename was initiated).
+ *
+ * PS: We've not thought through whether this algo works in the presence of
+ *     hardlinks to src and/or dst.
+ *
+ * Algorithm:
+ * ==========
+ *
+ * 1) set inodelk on src-parent
+ *    As part of rename operation, parent can change for the file.
+ *    We need to remove contribution (both on disk xattr and in-memory one)
+ *    to src-parent (and its ancestors) and add the contribution to dst-parent
+ *    (and its ancestors). While we are doing these operations, contribution of
+ *    the file/directory shouldn't be changing as we want to be sure that
+ *      a) what we subtract from src-parent is exactly what we add to dst-parent
+ *      b) we should subtract from src-parent exactly what we contributed to
+ *         src-parent
+ *    So, We hold a lock on src-parent to block any parallel transcations on
+ *    src-inode (since that's the one which survives rename).
+ *
+ *    If there are any parallel transactions on dst-inode they keep succeeding
+ *    till the association of dst-inode with dst-parent is broken because of an
+ *    inode_rename after unwind of rename fop from marker. Only after unwind
+ *    (and hence inode_rename), we delete and subtract the contribution of
+ *    dst-inode to dst-parent. That way we are making sure we subtract exactly
+ *    what dst-inode contributed to dst-parent.
+ *
+ * 2) lookup contribution to src-parent on src-inode.
+ *    We need to save the contribution info for use at step-8.
+ *
+ * 3) wind rename
+ *    Perform rename on disk
+ *
+ * 4) remove xattr on src-loc
+ *    After rename, parent can change, so
+ *    need to remove xattrs storing contribution to src-parent.
+ *
+ * 5) remove contribution node corresponding to src-parent from the in-memory
+ *    list.
+ *    After rename, contri gfid can change and we have
+ *    also removed xattr from file.
+ *    We need to remove in-memory contribution node to prevent updations to
+ *    src-parent even after a successful rename
+ *
+ * 6) unwind rename
+ *    This will ensure that rename is done in the server
+ *    inode table. An inode_rename disassociates src-inode from src-parent and
+ *    associates it with dst-parent. It also disassociates dst-inode from
+ *    dst-parent. After inode_rename, inode_parent on src-inode will give
+ *    dst-parent and inode_parent on dst-inode will return NULL (assuming
+ *    dst-inode doesn't have any hardlinks).
+ *
+ * 7) release inodelk on src-parent
+ *    Lock on src-parent should be released only after
+ *    rename on disk, remove xattr and rename_unwind (and hence inode_rename)
+ *    operations. If lock is released before inode_rename, a parallel
+ *    transaction on src-inode can still update src-parent (as inode_parent on
+ *    src-inode can still return src-parent). This would make the
+ *    contribution from src-inode to src-parent stored in step-2 stale.
+ *
+ * 8) Initiate mq_reduce_parent_size_txn on src-parent to remove contribution
+ *    of src-inode to src-parent. We use the contribution stored in step-2.
+ *    Since, we had acquired the lock on src-parent all along step-2 through
+ *    inode_rename, we can be sure that a parallel transaction wouldn't have
+ *    added a delta to src-parent.
+ *
+ * 9) Initiate mq_reduce_parent_size_txn on dst-parent if dst-inode exists.
+ *    The size reduced from dst-parent and its ancestors is the
+ *    size stored as contribution to dst-parent in dst-inode.
+ *    If the destination file had existed, rename will unlink the
+ *    destination file as part of its operation.
+ *    We need to reduce the size on the dest parent similarly to
+ *    unlink. Since, we are initiating reduce-parent-size transaction after
+ *    inode_rename, we can be sure that a parallel transaction wouldn't add
+ *    delta to dst-parent while we are reducing the contribution of dst-inode
+ *    from its ancestors before rename.
+ *
+ * 10) create contribution xattr to dst-parent on src-inode.
+ */
+int32_t
+marker_rename(call_frame_t *frame, xlator_t *this, loc_t *oldloc, loc_t *newloc,
+              dict_t *xdata)
 {
-        int32_t         ret              = 0;
-        marker_local_t *local            = NULL;
-        marker_local_t *oplocal          = NULL;
-        marker_conf_t  *priv             = NULL;
-        struct gf_flock lock             = {0, };
-        loc_t          *lock_on          = NULL;
+    int32_t ret = 0;
+    marker_local_t *local = NULL;
+    marker_local_t *oplocal = NULL;
+    marker_conf_t *priv = NULL;
+    struct gf_flock lock = {
+        0,
+    };
 
-        priv = this->private;
+    priv = this->private;
 
-        if (priv->feature_enabled == 0)
-                goto rename_wind;
+    if (priv->feature_enabled == 0)
+        goto rename_wind;
 
-        local = mem_get0 (this->local_pool);
+    local = mem_get0(this->local_pool);
 
-        MARKER_INIT_LOCAL (frame, local);
+    MARKER_INIT_LOCAL(frame, local);
 
-        oplocal = mem_get0 (this->local_pool);
+    oplocal = mem_get0(this->local_pool);
 
-        MARKER_INIT_LOCAL (frame, oplocal);
+    MARKER_INIT_LOCAL(frame, oplocal);
 
-        frame->local = local;
+    frame->local = local;
 
-        local->oplocal = marker_local_ref (oplocal);
+    local->oplocal = marker_local_ref(oplocal);
 
-        ret = loc_copy (&local->loc, newloc);
-        if (ret < 0)
-                goto err;
+    ret = loc_copy(&local->loc, newloc);
+    if (ret < 0)
+        goto err;
 
-        ret = loc_copy (&oplocal->loc, oldloc);
-        if (ret < 0)
-                goto err;
+    ret = loc_copy(&oplocal->loc, oldloc);
+    if (ret < 0)
+        goto err;
 
-        if (!(priv->feature_enabled & GF_QUOTA)) {
-                goto rename_wind;
-        }
+    if (!(priv->feature_enabled & GF_QUOTA)) {
+        goto rename_wind;
+    }
 
-        ret = mq_inode_loc_fill (NULL, newloc->parent, &local->parent_loc);
-        if (ret < 0)
-                goto err;
+    ret = mq_inode_loc_fill(NULL, newloc->parent, &local->parent_loc);
+    if (ret < 0)
+        goto err;
 
-        ret = mq_inode_loc_fill (NULL, oldloc->parent, &oplocal->parent_loc);
-        if (ret < 0)
-                goto err;
+    ret = mq_inode_loc_fill(NULL, oldloc->parent, &oplocal->parent_loc);
+    if (ret < 0)
+        goto err;
 
-        if ((newloc->inode != NULL) && (newloc->parent != oldloc->parent)
-            && (gf_uuid_compare (newloc->parent->gfid,
-                              oldloc->parent->gfid) < 0)) {
-                lock_on = &local->parent_loc;
-                local->next_lock_on = &oplocal->parent_loc;
-        } else {
-                lock_on = &oplocal->parent_loc;
-                if ((newloc->inode != NULL) && (newloc->parent
-                                                != oldloc->parent)) {
-                        local->next_lock_on = &local->parent_loc;
-                }
-        }
+    lock.l_len = 0;
+    lock.l_start = 0;
+    lock.l_type = F_WRLCK;
+    lock.l_whence = SEEK_SET;
 
-        lock.l_len    = 0;
-        lock.l_start  = 0;
-        lock.l_type   = F_WRLCK;
-        lock.l_whence = SEEK_SET;
+    local->xdata = xdata ? dict_ref(xdata) : dict_new();
+    ret = dict_set_int32(local->xdata, GF_REQUEST_LINK_COUNT_XDATA, 1);
+    if (ret < 0)
+        goto err;
 
-        STACK_WIND (frame,
-                    marker_rename_inodelk_cbk,
-                    FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->inodelk,
-                    this->name, lock_on,
-                    F_SETLKW, &lock, NULL);
+    local->frame = frame;
+    local->lk_frame = create_frame(this, this->ctx->pool);
+    if (local->lk_frame == NULL)
+        goto err;
 
-        return 0;
+    local->lk_frame->root->uid = 0;
+    local->lk_frame->root->gid = 0;
+    local->lk_frame->local = local;
+    set_lk_owner_from_ptr(&local->lk_frame->root->lk_owner,
+                          local->lk_frame->root);
+
+    STACK_WIND(local->lk_frame, marker_get_oldpath_contribution,
+               FIRST_CHILD(this), FIRST_CHILD(this)->fops->inodelk, this->name,
+               &oplocal->parent_loc, F_SETLKW, &lock, NULL);
+
+    return 0;
 
 rename_wind:
-        STACK_WIND (frame, marker_rename_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->rename, oldloc, newloc, xdata);
+    STACK_WIND(frame, marker_rename_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->rename, oldloc, newloc, xdata);
 
-        return 0;
+    return 0;
 err:
-        STACK_UNWIND_STRICT (rename, frame, -1, ENOMEM, NULL,
-                             NULL, NULL, NULL, NULL, NULL);
+    MARKER_STACK_UNWIND(rename, frame, -1, ENOMEM, NULL, NULL, NULL, NULL, NULL,
+                        NULL);
+    marker_local_unref(oplocal);
 
-        return 0;
+    return 0;
 }
 
-
 int32_t
-marker_truncate_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                     int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
-                     struct iatt *postbuf, dict_t *xdata)
-{
-        marker_local_t     *local   = NULL;
-        marker_conf_t      *priv    = NULL;
-
-        if (op_ret == -1) {
-                gf_log (this->name, GF_LOG_TRACE, "%s occurred while "
-                        "truncating a file ", strerror (op_errno));
-        }
-
-        local = (marker_local_t *) frame->local;
-
-        frame->local = NULL;
-
-        STACK_UNWIND_STRICT (truncate, frame, op_ret, op_errno, prebuf,
-                             postbuf, xdata);
-
-        if (op_ret == -1 || local == NULL)
-                goto out;
-
-        priv = this->private;
-
-        if (priv->feature_enabled & GF_QUOTA)
-                mq_initiate_quota_txn (this, &local->loc);
+marker_truncate_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                    int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
+                    struct iatt *postbuf, dict_t *xdata)
+{
+    marker_local_t *local = NULL;
+    marker_conf_t *priv = NULL;
+
+    if (op_ret == -1) {
+        gf_log(this->name, GF_LOG_TRACE,
+               "%s occurred while "
+               "truncating a file ",
+               strerror(op_errno));
+    }
+
+    local = (marker_local_t *)frame->local;
+
+    frame->local = NULL;
+
+    STACK_UNWIND_STRICT(truncate, frame, op_ret, op_errno, prebuf, postbuf,
+                        xdata);
+
+    if (op_ret == -1 || local == NULL)
+        goto out;
+
+    priv = this->private;
+
+    if (priv->feature_enabled & GF_QUOTA) {
+        /* DHT Rebalance process, at the end of migration will
+         * first make the src file as a linkto file and then
+         * truncate the file. By doing a truncate after making the
+         * src file as linkto file, the contri which is already
+         * accounted is left over.
+         * So, we need to account for the linkto file when a truncate
+         * happens, thereby updating the contri properly.
+         * By passing NULL for postbuf, mq_prevalidate does not check
+         * for linkto file.
+         * Same happens with ftruncate as well.
+         */
+        if (postbuf && IS_DHT_LINKFILE_MODE(postbuf))
+            mq_initiate_quota_txn(this, &local->loc, NULL);
+        else
+            mq_initiate_quota_txn(this, &local->loc, postbuf);
+    }
 
-        if (priv->feature_enabled & GF_XTIME)
-                marker_xtime_update_marks (this, local);
+    if (priv->feature_enabled & GF_XTIME)
+        marker_xtime_update_marks(this, local);
 
 out:
-        marker_local_unref (local);
+    marker_local_unref(local);
 
-        return 0;
+    return 0;
 }
 
 int32_t
-marker_truncate (call_frame_t *frame, xlator_t *this, loc_t *loc, off_t offset,
-                 dict_t *xdata)
+marker_truncate(call_frame_t *frame, xlator_t *this, loc_t *loc, off_t offset,
+                dict_t *xdata)
 {
-        int32_t          ret   = 0;
-        marker_local_t  *local = NULL;
-        marker_conf_t   *priv  = NULL;
+    int32_t ret = 0;
+    marker_local_t *local = NULL;
+    marker_conf_t *priv = NULL;
 
-        priv = this->private;
+    priv = this->private;
 
-        if (priv->feature_enabled == 0)
-                goto wind;
+    if (priv->feature_enabled == 0)
+        goto wind;
 
-        local = mem_get0 (this->local_pool);
+    local = mem_get0(this->local_pool);
 
-        MARKER_INIT_LOCAL (frame, local);
+    MARKER_INIT_LOCAL(frame, local);
 
-        ret = loc_copy (&local->loc, loc);
+    ret = loc_copy(&local->loc, loc);
 
-        if (ret == -1)
-                goto err;
+    if (ret == -1)
+        goto err;
 wind:
-        STACK_WIND (frame, marker_truncate_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->truncate, loc, offset, xdata);
-        return 0;
+    STACK_WIND(frame, marker_truncate_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->truncate, loc, offset, xdata);
+    return 0;
 err:
-        STACK_UNWIND_STRICT (truncate, frame, -1, ENOMEM, NULL, NULL, NULL);
+    MARKER_STACK_UNWIND(truncate, frame, -1, ENOMEM, NULL, NULL, NULL);
 
-        return 0;
+    return 0;
 }
 
-
 int32_t
-marker_ftruncate_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                      int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
-                      struct iatt *postbuf, dict_t *xdata)
+marker_ftruncate_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                     int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
+                     struct iatt *postbuf, dict_t *xdata)
 {
-        marker_local_t     *local   = NULL;
-        marker_conf_t      *priv    = NULL;
+    marker_local_t *local = NULL;
+    marker_conf_t *priv = NULL;
 
-        if (op_ret == -1) {
-                gf_log (this->name, GF_LOG_TRACE, "%s occurred while "
-                        "truncating a file ", strerror (op_errno));
-        }
+    if (op_ret == -1) {
+        gf_log(this->name, GF_LOG_TRACE,
+               "%s occurred while "
+               "truncating a file ",
+               strerror(op_errno));
+    }
 
-        local = (marker_local_t *) frame->local;
+    local = (marker_local_t *)frame->local;
 
-        frame->local = NULL;
+    frame->local = NULL;
 
-        STACK_UNWIND_STRICT (ftruncate, frame, op_ret, op_errno, prebuf,
-                             postbuf, xdata);
+    STACK_UNWIND_STRICT(ftruncate, frame, op_ret, op_errno, prebuf, postbuf,
+                        xdata);
 
-        if (op_ret == -1 || local == NULL)
-                goto out;
+    if (op_ret == -1 || local == NULL)
+        goto out;
 
-        priv = this->private;
+    priv = this->private;
 
-        if (priv->feature_enabled & GF_QUOTA)
-                mq_initiate_quota_txn (this, &local->loc);
+    if (priv->feature_enabled & GF_QUOTA) {
+        if (postbuf && IS_DHT_LINKFILE_MODE(postbuf))
+            mq_initiate_quota_txn(this, &local->loc, NULL);
+        else
+            mq_initiate_quota_txn(this, &local->loc, postbuf);
+    }
 
-        if (priv->feature_enabled & GF_XTIME)
-                marker_xtime_update_marks (this, local);
+    if (priv->feature_enabled & GF_XTIME)
+        marker_xtime_update_marks(this, local);
 out:
-        marker_local_unref (local);
+    marker_local_unref(local);
 
-        return 0;
+    return 0;
 }
 
 int32_t
-marker_ftruncate (call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
-                  dict_t *xdata)
+marker_ftruncate(call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
+                 dict_t *xdata)
 {
-        int32_t          ret   = 0;
-        marker_local_t  *local = NULL;
-        marker_conf_t   *priv  = NULL;
+    int32_t ret = 0;
+    marker_local_t *local = NULL;
+    marker_conf_t *priv = NULL;
 
-        priv = this->private;
+    priv = this->private;
 
-        if (priv->feature_enabled == 0)
-                goto wind;
+    if (priv->feature_enabled == 0)
+        goto wind;
 
-        local = mem_get0 (this->local_pool);
+    local = mem_get0(this->local_pool);
 
-        MARKER_INIT_LOCAL (frame, local);
+    MARKER_INIT_LOCAL(frame, local);
 
-        ret = marker_inode_loc_fill (fd->inode, NULL, &local->loc);
+    ret = marker_inode_loc_fill(fd->inode, &local->loc);
 
-        if (ret == -1)
-                goto err;
+    if (ret == -1)
+        goto err;
 wind:
-        STACK_WIND (frame, marker_ftruncate_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->ftruncate, fd, offset, xdata);
-        return 0;
+    STACK_WIND(frame, marker_ftruncate_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->ftruncate, fd, offset, xdata);
+    return 0;
 err:
-        STACK_UNWIND_STRICT (ftruncate, frame, -1, ENOMEM, NULL, NULL, NULL);
+    MARKER_STACK_UNWIND(ftruncate, frame, -1, ENOMEM, NULL, NULL, NULL);
 
-        return 0;
+    return 0;
 }
 
-
 int32_t
-marker_symlink_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                    int32_t op_ret, int32_t op_errno, inode_t *inode,
-                    struct iatt *buf, struct iatt *preparent,
-                    struct iatt *postparent, dict_t *xdata)
+marker_symlink_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                   int32_t op_ret, int32_t op_errno, inode_t *inode,
+                   struct iatt *buf, struct iatt *preparent,
+                   struct iatt *postparent, dict_t *xdata)
 {
-        marker_conf_t      *priv    = NULL;
-        marker_local_t     *local   = NULL;
+    marker_conf_t *priv = NULL;
+    marker_local_t *local = NULL;
+    quota_inode_ctx_t *ctx = NULL;
 
-        if (op_ret == -1) {
-                gf_log (this->name, GF_LOG_TRACE, "%s occurred while "
-                        "creating symlinks ", strerror (op_errno));
-        }
+    if (op_ret == -1) {
+        gf_log(this->name, GF_LOG_TRACE,
+               "%s occurred while "
+               "creating symlinks ",
+               strerror(op_errno));
+    }
 
-        local = (marker_local_t *) frame->local;
+    local = (marker_local_t *)frame->local;
 
-        frame->local = NULL;
+    frame->local = NULL;
+    priv = this->private;
 
-        STACK_UNWIND_STRICT (symlink, frame, op_ret, op_errno, inode, buf,
-                             preparent, postparent, xdata);
+    if (op_ret >= 0 && inode && (priv->feature_enabled & GF_QUOTA)) {
+        ctx = mq_inode_ctx_new(inode, this);
+        if (ctx == NULL) {
+            gf_log(this->name, GF_LOG_WARNING,
+                   "mq_inode_ctx_new "
+                   "failed for %s",
+                   uuid_utoa(inode->gfid));
+            op_ret = -1;
+            op_errno = ENOMEM;
+        }
+    }
 
-        if (op_ret == -1 || local == NULL)
-                goto out;
+    STACK_UNWIND_STRICT(symlink, frame, op_ret, op_errno, inode, buf, preparent,
+                        postparent, xdata);
 
-        if (gf_uuid_is_null (local->loc.gfid))
-                gf_uuid_copy (local->loc.gfid, buf->ia_gfid);
+    if (op_ret == -1 || local == NULL)
+        goto out;
 
-        priv = this->private;
+    if (gf_uuid_is_null(local->loc.gfid))
+        gf_uuid_copy(local->loc.gfid, buf->ia_gfid);
 
-        if (priv->feature_enabled & GF_QUOTA) {
-                mq_create_xattrs_txn (this, &local->loc);
-        }
+    if (priv->feature_enabled & GF_QUOTA) {
+        mq_create_xattrs_txn(this, &local->loc, buf);
+    }
 
-        if (priv->feature_enabled & GF_XTIME)
-                marker_xtime_update_marks (this, local);
+    if (priv->feature_enabled & GF_XTIME)
+        marker_xtime_update_marks(this, local);
 out:
-        marker_local_unref (local);
+    marker_local_unref(local);
 
-        return 0;
+    return 0;
 }
 
 int
-marker_symlink (call_frame_t *frame, xlator_t *this, const char *linkpath,
-                loc_t *loc, mode_t umask, dict_t *xdata)
+marker_symlink(call_frame_t *frame, xlator_t *this, const char *linkpath,
+               loc_t *loc, mode_t umask, dict_t *xdata)
 {
-        int32_t          ret   = 0;
-        marker_local_t  *local = NULL;
-        marker_conf_t   *priv  = NULL;
+    int32_t ret = 0;
+    marker_local_t *local = NULL;
+    marker_conf_t *priv = NULL;
 
-        priv = this->private;
+    priv = this->private;
 
-        if (priv->feature_enabled == 0)
-                goto wind;
+    if (priv->feature_enabled == 0)
+        goto wind;
 
-        local = mem_get0 (this->local_pool);
+    local = mem_get0(this->local_pool);
 
-        MARKER_INIT_LOCAL (frame, local);
+    MARKER_INIT_LOCAL(frame, local);
 
-        ret = loc_copy (&local->loc, loc);
+    ret = loc_copy(&local->loc, loc);
 
-        if (ret == -1)
-                goto err;
+    if (ret == -1)
+        goto err;
 wind:
-        STACK_WIND (frame, marker_symlink_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->symlink, linkpath, loc, umask,
-                    xdata);
-        return 0;
+    STACK_WIND(frame, marker_symlink_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->symlink, linkpath, loc, umask, xdata);
+    return 0;
 err:
-        STACK_UNWIND_STRICT (symlink, frame, -1, ENOMEM, NULL,
-                             NULL, NULL, NULL, NULL);
-        return 0;
-}
+    MARKER_STACK_UNWIND(symlink, frame, -1, ENOMEM, NULL, NULL, NULL, NULL,
+                        NULL);
 
+    return 0;
+}
 
 int32_t
-marker_mknod_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                  int32_t op_ret, int32_t op_errno, inode_t *inode,
-                  struct iatt *buf, struct iatt *preparent,
-                  struct iatt *postparent, dict_t *xdata)
+marker_mknod_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                 int32_t op_ret, int32_t op_errno, inode_t *inode,
+                 struct iatt *buf, struct iatt *preparent,
+                 struct iatt *postparent, dict_t *xdata)
 {
-        marker_local_t     *local   = NULL;
-        marker_conf_t      *priv    = NULL;
+    marker_local_t *local = NULL;
+    marker_conf_t *priv = NULL;
+    quota_inode_ctx_t *ctx = NULL;
 
-        if (op_ret == -1) {
-                gf_log (this->name, GF_LOG_TRACE, "%s occurred with "
-                        "mknod ", strerror (op_errno));
-        }
+    if (op_ret == -1) {
+        gf_log(this->name, GF_LOG_TRACE,
+               "%s occurred with "
+               "mknod ",
+               strerror(op_errno));
+    }
 
-        local = (marker_local_t *) frame->local;
+    local = (marker_local_t *)frame->local;
 
-        frame->local = NULL;
+    frame->local = NULL;
+    priv = this->private;
 
-        STACK_UNWIND_STRICT (mknod, frame, op_ret, op_errno, inode,
-                             buf, preparent, postparent, xdata);
+    if (op_ret >= 0 && inode && (priv->feature_enabled & GF_QUOTA)) {
+        ctx = mq_inode_ctx_new(inode, this);
+        if (ctx == NULL) {
+            gf_log(this->name, GF_LOG_WARNING,
+                   "mq_inode_ctx_new "
+                   "failed for %s",
+                   uuid_utoa(inode->gfid));
+            op_ret = -1;
+            op_errno = ENOMEM;
+        }
+    }
 
-        if (op_ret == -1 ||  local == NULL)
-                goto out;
+    STACK_UNWIND_STRICT(mknod, frame, op_ret, op_errno, inode, buf, preparent,
+                        postparent, xdata);
 
-        if (gf_uuid_is_null (local->loc.gfid))
-                gf_uuid_copy (local->loc.gfid, buf->ia_gfid);
+    if (op_ret == -1 || local == NULL)
+        goto out;
 
-        priv = this->private;
+    if (gf_uuid_is_null(local->loc.gfid))
+        gf_uuid_copy(local->loc.gfid, buf->ia_gfid);
 
-        if ((priv->feature_enabled & GF_QUOTA) && (S_ISREG (local->mode))) {
-                mq_create_xattrs_txn (this, &local->loc);
-        }
+    if ((priv->feature_enabled & GF_QUOTA) && (S_ISREG(local->mode))) {
+        mq_create_xattrs_txn(this, &local->loc, buf);
+    }
 
-        if (priv->feature_enabled & GF_XTIME)
-                marker_xtime_update_marks (this, local);
+    if (priv->feature_enabled & GF_XTIME)
+        marker_xtime_update_marks(this, local);
 out:
-        marker_local_unref (local);
+    marker_local_unref(local);
 
-        return 0;
+    return 0;
 }
 
 int
-marker_mknod (call_frame_t *frame, xlator_t *this, loc_t *loc, mode_t mode,
-              dev_t rdev, mode_t umask, dict_t *xdata)
+marker_mknod(call_frame_t *frame, xlator_t *this, loc_t *loc, mode_t mode,
+             dev_t rdev, mode_t umask, dict_t *xdata)
 {
-        int32_t          ret   = 0;
-        marker_local_t  *local = NULL;
-        marker_conf_t   *priv  = NULL;
+    int32_t ret = 0;
+    marker_local_t *local = NULL;
+    marker_conf_t *priv = NULL;
 
-        priv = this->private;
+    priv = this->private;
 
-        if (priv->feature_enabled == 0)
-                goto wind;
+    if (priv->feature_enabled == 0)
+        goto wind;
 
-        local = mem_get0 (this->local_pool);
+    local = mem_get0(this->local_pool);
 
-        MARKER_INIT_LOCAL (frame, local);
+    MARKER_INIT_LOCAL(frame, local);
 
-        ret = loc_copy (&local->loc, loc);
+    ret = loc_copy(&local->loc, loc);
 
-        local->mode = mode;
+    local->mode = mode;
 
-        if (ret == -1)
-                goto err;
+    if (ret == -1)
+        goto err;
 wind:
-        STACK_WIND (frame, marker_mknod_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->mknod, loc, mode, rdev, umask,
-                    xdata);
-        return 0;
+    STACK_WIND(frame, marker_mknod_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->mknod, loc, mode, rdev, umask, xdata);
+    return 0;
 err:
-        STACK_UNWIND_STRICT (mknod, frame, -1, ENOMEM, NULL,
-                             NULL, NULL, NULL, NULL);
-        return 0;
-}
+    MARKER_STACK_UNWIND(mknod, frame, -1, ENOMEM, NULL, NULL, NULL, NULL, NULL);
 
+    return 0;
+}
 
 int32_t
-marker_fallocate_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                      int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
-                      struct iatt *postbuf, dict_t *xdata)
+marker_fallocate_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                     int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
+                     struct iatt *postbuf, dict_t *xdata)
 {
-        marker_local_t     *local   = NULL;
-        marker_conf_t      *priv    = NULL;
+    marker_local_t *local = NULL;
+    marker_conf_t *priv = NULL;
 
-        if (op_ret == -1) {
-                gf_log (this->name, GF_LOG_TRACE, "%s occurred while "
-                        "fallocating a file ", strerror (op_errno));
-        }
+    if (op_ret == -1) {
+        gf_log(this->name, GF_LOG_TRACE,
+               "%s occurred while "
+               "fallocating a file ",
+               strerror(op_errno));
+    }
 
-        local = (marker_local_t *) frame->local;
+    local = (marker_local_t *)frame->local;
 
-        frame->local = NULL;
+    frame->local = NULL;
 
-        STACK_UNWIND_STRICT (fallocate, frame, op_ret, op_errno, prebuf,
-                             postbuf, xdata);
+    STACK_UNWIND_STRICT(fallocate, frame, op_ret, op_errno, prebuf, postbuf,
+                        xdata);
 
-        if (op_ret == -1 || local == NULL)
-                goto out;
+    if (op_ret == -1 || local == NULL)
+        goto out;
 
-        priv = this->private;
+    priv = this->private;
 
-        if (priv->feature_enabled & GF_QUOTA)
-                mq_initiate_quota_txn (this, &local->loc);
+    if (priv->feature_enabled & GF_QUOTA)
+        mq_initiate_quota_txn(this, &local->loc, postbuf);
 
-        if (priv->feature_enabled & GF_XTIME)
-                marker_xtime_update_marks (this, local);
+    if (priv->feature_enabled & GF_XTIME)
+        marker_xtime_update_marks(this, local);
 out:
-        marker_local_unref (local);
+    marker_local_unref(local);
 
-        return 0;
+    return 0;
 }
 
 int32_t
 marker_fallocate(call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t mode,
-		 off_t offset, size_t len, dict_t *xdata)
+                 off_t offset, size_t len, dict_t *xdata)
 {
-        int32_t          ret   = 0;
-        marker_local_t  *local = NULL;
-        marker_conf_t   *priv  = NULL;
+    int32_t ret = 0;
+    marker_local_t *local = NULL;
+    marker_conf_t *priv = NULL;
 
-        priv = this->private;
+    priv = this->private;
 
-        if (priv->feature_enabled == 0)
-                goto wind;
+    if (priv->feature_enabled == 0)
+        goto wind;
 
-        local = mem_get0 (this->local_pool);
+    local = mem_get0(this->local_pool);
 
-        MARKER_INIT_LOCAL (frame, local);
+    MARKER_INIT_LOCAL(frame, local);
 
-        ret = marker_inode_loc_fill (fd->inode, NULL, &local->loc);
+    ret = marker_inode_loc_fill(fd->inode, &local->loc);
 
-        if (ret == -1)
-                goto err;
+    if (ret == -1)
+        goto err;
 wind:
-        STACK_WIND (frame, marker_fallocate_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->fallocate, fd, mode, offset, len,
-		    xdata);
-        return 0;
+    STACK_WIND(frame, marker_fallocate_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->fallocate, fd, mode, offset, len,
+               xdata);
+    return 0;
 err:
-        STACK_UNWIND_STRICT (fallocate, frame, -1, ENOMEM, NULL, NULL, NULL);
+    MARKER_STACK_UNWIND(fallocate, frame, -1, ENOMEM, NULL, NULL, NULL);
 
-        return 0;
+    return 0;
 }
 
-
 int32_t
 marker_discard_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
                    int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
                    struct iatt *postbuf, dict_t *xdata)
 {
-        marker_local_t     *local   = NULL;
-        marker_conf_t      *priv    = NULL;
+    marker_local_t *local = NULL;
+    marker_conf_t *priv = NULL;
 
-        if (op_ret == -1) {
-                gf_log (this->name, GF_LOG_TRACE, "%s occurred during discard",
-                        strerror (op_errno));
-        }
+    if (op_ret == -1) {
+        gf_log(this->name, GF_LOG_TRACE, "%s occurred during discard",
+               strerror(op_errno));
+    }
 
-        local = (marker_local_t *) frame->local;
+    local = (marker_local_t *)frame->local;
 
-        frame->local = NULL;
+    frame->local = NULL;
 
-        STACK_UNWIND_STRICT (discard, frame, op_ret, op_errno, prebuf,
-                             postbuf, xdata);
+    STACK_UNWIND_STRICT(discard, frame, op_ret, op_errno, prebuf, postbuf,
+                        xdata);
 
-        if (op_ret == -1 || local == NULL)
-                goto out;
+    if (op_ret == -1 || local == NULL)
+        goto out;
 
-        priv = this->private;
+    priv = this->private;
 
-        if (priv->feature_enabled & GF_QUOTA)
-                mq_initiate_quota_txn (this, &local->loc);
+    if (priv->feature_enabled & GF_QUOTA)
+        mq_initiate_quota_txn(this, &local->loc, postbuf);
 
-        if (priv->feature_enabled & GF_XTIME)
-                marker_xtime_update_marks (this, local);
+    if (priv->feature_enabled & GF_XTIME)
+        marker_xtime_update_marks(this, local);
 out:
-        marker_local_unref (local);
+    marker_local_unref(local);
 
-        return 0;
+    return 0;
 }
 
 int32_t
 marker_discard(call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
-	       size_t len, dict_t *xdata)
+               size_t len, dict_t *xdata)
 {
-        int32_t          ret   = 0;
-        marker_local_t  *local = NULL;
-        marker_conf_t   *priv  = NULL;
+    int32_t ret = 0;
+    marker_local_t *local = NULL;
+    marker_conf_t *priv = NULL;
 
-        priv = this->private;
+    priv = this->private;
 
-        if (priv->feature_enabled == 0)
-                goto wind;
+    if (priv->feature_enabled == 0)
+        goto wind;
 
-        local = mem_get0 (this->local_pool);
+    local = mem_get0(this->local_pool);
 
-        MARKER_INIT_LOCAL (frame, local);
+    MARKER_INIT_LOCAL(frame, local);
 
-        ret = marker_inode_loc_fill (fd->inode, NULL, &local->loc);
+    ret = marker_inode_loc_fill(fd->inode, &local->loc);
 
-        if (ret == -1)
-                goto err;
+    if (ret == -1)
+        goto err;
 wind:
-        STACK_WIND (frame, marker_discard_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->discard, fd, offset, len, xdata);
-        return 0;
+    STACK_WIND(frame, marker_discard_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->discard, fd, offset, len, xdata);
+    return 0;
 err:
-        STACK_UNWIND_STRICT (discard, frame, -1, ENOMEM, NULL, NULL, NULL);
+    MARKER_STACK_UNWIND(discard, frame, -1, ENOMEM, NULL, NULL, NULL);
 
-        return 0;
+    return 0;
 }
 
 int32_t
 marker_zerofill_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
-                   int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
-                   struct iatt *postbuf, dict_t *xdata)
+                    int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
+                    struct iatt *postbuf, dict_t *xdata)
 {
-        marker_local_t     *local   = NULL;
-        marker_conf_t      *priv    = NULL;
+    marker_local_t *local = NULL;
+    marker_conf_t *priv = NULL;
 
-        if (op_ret == -1) {
-                gf_log (this->name, GF_LOG_TRACE, "%s occurred during zerofill",
-                        strerror (op_errno));
-        }
+    if (op_ret == -1) {
+        gf_log(this->name, GF_LOG_TRACE, "%s occurred during zerofill",
+               strerror(op_errno));
+    }
 
-        local = (marker_local_t *) frame->local;
+    local = (marker_local_t *)frame->local;
 
-        frame->local = NULL;
+    frame->local = NULL;
 
-        STACK_UNWIND_STRICT (zerofill, frame, op_ret, op_errno, prebuf,
-                             postbuf, xdata);
+    STACK_UNWIND_STRICT(zerofill, frame, op_ret, op_errno, prebuf, postbuf,
+                        xdata);
 
-        if (op_ret == -1 || local == NULL)
-                goto out;
+    if (op_ret == -1 || local == NULL)
+        goto out;
 
-        priv = this->private;
+    priv = this->private;
 
-        if (priv->feature_enabled & GF_QUOTA)
-                mq_initiate_quota_txn (this, &local->loc);
+    if (priv->feature_enabled & GF_QUOTA)
+        mq_initiate_quota_txn(this, &local->loc, postbuf);
 
-        if (priv->feature_enabled & GF_XTIME)
-                marker_xtime_update_marks (this, local);
+    if (priv->feature_enabled & GF_XTIME)
+        marker_xtime_update_marks(this, local);
 out:
-        marker_local_unref (local);
+    marker_local_unref(local);
 
-        return 0;
+    return 0;
 }
 
 int32_t
 marker_zerofill(call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
-               off_t len, dict_t *xdata)
+                off_t len, dict_t *xdata)
 {
-        int32_t          ret   = 0;
-        marker_local_t  *local = NULL;
-        marker_conf_t   *priv  = NULL;
+    int32_t ret = 0;
+    marker_local_t *local = NULL;
+    marker_conf_t *priv = NULL;
 
-        priv = this->private;
+    priv = this->private;
 
-        if (priv->feature_enabled == 0)
-                goto wind;
+    if (priv->feature_enabled == 0)
+        goto wind;
 
-        local = mem_get0 (this->local_pool);
+    local = mem_get0(this->local_pool);
 
-        MARKER_INIT_LOCAL (frame, local);
+    MARKER_INIT_LOCAL(frame, local);
 
-        ret = marker_inode_loc_fill (fd->inode, NULL, &local->loc);
+    ret = marker_inode_loc_fill(fd->inode, &local->loc);
 
-        if (ret == -1)
-                goto err;
+    if (ret == -1)
+        goto err;
 wind:
-        STACK_WIND (frame, marker_zerofill_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->zerofill, fd, offset, len, xdata);
-        return 0;
+    STACK_WIND(frame, marker_zerofill_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->zerofill, fd, offset, len, xdata);
+    return 0;
 err:
-        STACK_UNWIND_STRICT (zerofill, frame, -1, ENOMEM, NULL, NULL, NULL);
+    MARKER_STACK_UNWIND(zerofill, frame, -1, ENOMEM, NULL, NULL, NULL);
 
-        return 0;
+    return 0;
 }
 
-
 /* when a call from the special client is received on
  * key trusted.glusterfs.volume-mark with value "RESET"
  * or if the value is 0length, update the change the
@@ -2104,1103 +2323,1246 @@ err:
  * timestamp file.
  */
 int32_t
-call_from_sp_client_to_reset_tmfile (call_frame_t *frame,
-                                     xlator_t     *this,
-                                     dict_t       *dict)
+call_from_sp_client_to_reset_tmfile(call_frame_t *frame, xlator_t *this,
+                                    dict_t *dict)
 {
-        int32_t          fd       = 0;
-        int32_t          op_ret   = 0;
-        int32_t          op_errno = 0;
-        data_t          *data     = NULL;
-        marker_conf_t   *priv     = NULL;
+    int32_t fd = 0;
+    int32_t op_ret = 0;
+    int32_t op_errno = 0;
+    data_t *data = NULL;
+    marker_conf_t *priv = NULL;
 
-        if (frame == NULL || this == NULL || dict == NULL)
-                return -1;
+    if (frame == NULL || this == NULL || dict == NULL)
+        return -1;
 
-        priv = this->private;
+    priv = this->private;
 
-        data = dict_get (dict, "trusted.glusterfs.volume-mark");
-        if (data == NULL)
-                return -1;
+    data = dict_get(dict, "trusted.glusterfs.volume-mark");
+    if (data == NULL)
+        return -1;
 
-        if (frame->root->pid != GF_CLIENT_PID_GSYNCD) {
-                op_ret = -1;
-                op_errno = EPERM;
+    if (frame->root->pid != GF_CLIENT_PID_GSYNCD) {
+        op_ret = -1;
+        op_errno = EPERM;
 
-                goto out;
+        goto out;
+    }
+
+    if (data->len == 0 ||
+        (data->len == 5 && memcmp(data->data, "RESET", 5) == 0)) {
+        fd = open(priv->timestamp_file, O_WRONLY | O_TRUNC);
+        if (fd != -1) {
+            /* TODO check  whether the O_TRUNC would update the
+             * timestamps on a zero length file on all machies.
+             */
+            sys_close(fd);
         }
 
-        if (data->len == 0 || (data->len == 5 &&
-                               memcmp (data->data, "RESET", 5) == 0)) {
-                fd = open (priv->timestamp_file, O_WRONLY|O_TRUNC);
-                if (fd != -1) {
-                        /* TODO check  whether the O_TRUNC would update the
-                         * timestamps on a zero length file on all machies.
-                         */
-                        close (fd);
-                }
-
-                if (fd != -1 || errno == ENOENT) {
-                        op_ret = 0;
-                        op_errno = 0;
-                } else {
-                        op_ret = -1;
-                        op_errno = errno;
-                }
+        if (fd != -1 || errno == ENOENT) {
+            op_ret = 0;
+            op_errno = 0;
         } else {
-                op_ret = -1;
-                op_errno = EINVAL;
+            op_ret = -1;
+            op_errno = errno;
         }
+    } else {
+        op_ret = -1;
+        op_errno = EINVAL;
+    }
 out:
-        STACK_UNWIND_STRICT (setxattr, frame, op_ret, op_errno, NULL);
+    STACK_UNWIND_STRICT(setxattr, frame, op_ret, op_errno, NULL);
 
-        return 0;
+    return 0;
 }
 
-
 int32_t
-marker_setxattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                     int32_t op_ret, int32_t op_errno, dict_t *xdata)
+marker_setxattr_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                    int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
-        marker_local_t     *local   = NULL;
-        marker_conf_t      *priv    = NULL;
+    marker_local_t *local = NULL;
+    marker_conf_t *priv = NULL;
 
-        if (op_ret == -1) {
-                gf_log (this->name, GF_LOG_TRACE, "%s occurred in "
-                        "setxattr ", strerror (op_errno));
-        }
+    if (op_ret == -1) {
+        gf_log(this->name, GF_LOG_TRACE,
+               "%s occurred in "
+               "setxattr ",
+               strerror(op_errno));
+    }
 
-        local = (marker_local_t *) frame->local;
+    local = (marker_local_t *)frame->local;
 
-        frame->local = NULL;
+    frame->local = NULL;
 
-        STACK_UNWIND_STRICT (setxattr, frame, op_ret, op_errno, xdata);
+    STACK_UNWIND_STRICT(setxattr, frame, op_ret, op_errno, xdata);
 
-        if (op_ret == -1 || local == NULL)
-                goto out;
+    if (op_ret == -1 || local == NULL)
+        goto out;
 
-        priv = this->private;
+    priv = this->private;
 
-        if (priv->feature_enabled & GF_XTIME)
-                marker_xtime_update_marks (this, local);
+    if (priv->feature_enabled & GF_XTIME)
+        marker_xtime_update_marks(this, local);
 out:
-        marker_local_unref (local);
+    marker_local_unref(local);
 
-        return 0;
+    return 0;
 }
 
 int
-remove_quota_keys (dict_t *dict, char *k, data_t *v, void *data)
-{
-	call_frame_t 	*frame = data;
-	marker_local_t 	*local = frame->local;
-	xlator_t 	*this = frame->this;
-	int 		ret = -1;
-
-	ret = syncop_removexattr (FIRST_CHILD (this), &local->loc, k, 0, NULL);
-	if (ret) {
-		gf_log (this->name, GF_LOG_ERROR, "%s: Failed to remove "
-			"extended attribute: %s", local->loc.path, k);
-                return -1;
-	}
-	return 0;
+remove_quota_keys(dict_t *dict, char *k, data_t *v, void *data)
+{
+    call_frame_t *frame = data;
+    marker_local_t *local = frame->local;
+    xlator_t *this = frame->this;
+    marker_conf_t *priv = NULL;
+    char ver_str[NAME_MAX] = {
+        0,
+    };
+    char *dot = NULL;
+    int ret = -1;
+
+    priv = this->private;
+
+    /* If quota is enabled immediately after disable.
+     * quota healing starts creating new xattrs
+     * before completing the cleanup operation.
+     * So we should check if the xattr is the new.
+     * Do not remove xattr if its xattr
+     * version is same as current version
+     */
+    if ((priv->feature_enabled & GF_QUOTA) && priv->version > 0) {
+        snprintf(ver_str, sizeof(ver_str), ".%d", priv->version);
+        dot = strrchr(k, '.');
+        if (dot && !strcmp(dot, ver_str))
+            return 0;
+    }
+
+    ret = syncop_removexattr(FIRST_CHILD(this), &local->loc, k, 0, NULL);
+    if (ret) {
+        gf_log(this->name, GF_LOG_ERROR,
+               "%s: Failed to remove "
+               "extended attribute: %s",
+               local->loc.path, k);
+        return -1;
+    }
+    return 0;
 }
 
 int
-quota_xattr_cleaner_cbk (int ret, call_frame_t *frame, void *args)
+quota_xattr_cleaner_cbk(int ret, call_frame_t *frame, void *args)
 {
-        dict_t *xdata = args;
-        int op_ret = -1;
-        int op_errno = 0;
-	marker_local_t *local = NULL;
-
-	local = frame->local;
-	frame->local = NULL;
+    dict_t *xdata = args;
+    int op_ret = -1;
+    int op_errno = 0;
 
-        op_ret   = (ret < 0)? -1: 0;
-        op_errno = -ret;
+    op_ret = (ret < 0) ? -1 : 0;
+    op_errno = -ret;
 
-        STACK_UNWIND_STRICT (setxattr, frame, op_ret, op_errno, xdata);
-	marker_local_unref (local);
-        return ret;
+    MARKER_STACK_UNWIND(setxattr, frame, op_ret, op_errno, xdata);
+    return ret;
 }
 
 int
-quota_xattr_cleaner (void *args)
-{
-        struct synctask *task  = NULL;
-        call_frame_t    *frame = NULL;
-        xlator_t        *this  = NULL;
-        marker_local_t   *local = NULL;
-        dict_t          *xdata = NULL;
-        int             ret    = -1;
-
-        task = synctask_get ();
-        if (!task)
-                goto out;
-
-        frame = task->frame;
-        this  = frame->this;
-        local = frame->local;
-
-        ret = syncop_listxattr (FIRST_CHILD(this), &local->loc, &xdata, NULL,
-                                NULL);
-        if (ret == -1) {
-                ret = -errno;
-                goto out;
-        }
-
-	ret = dict_foreach_fnmatch (xdata, "trusted.glusterfs.quota.*",
-                                    remove_quota_keys, frame);
-        if (ret == -1) {
-                ret = -errno;
-                goto out;
-        }
-	ret = dict_foreach_fnmatch (xdata, PGFID_XATTR_KEY_PREFIX"*",
-                                    remove_quota_keys, frame);
-        if (ret == -1) {
-                ret = -errno;
-                goto out;
-        }
-
-        ret = 0;
+quota_xattr_cleaner(void *args)
+{
+    struct synctask *task = NULL;
+    call_frame_t *frame = NULL;
+    xlator_t *this = NULL;
+    marker_local_t *local = NULL;
+    dict_t *xdata = NULL;
+    int ret = -1;
+
+    task = synctask_get();
+    if (!task)
+        goto out;
+
+    frame = task->frame;
+    this = frame->this;
+    local = frame->local;
+
+    ret = syncop_listxattr(FIRST_CHILD(this), &local->loc, &xdata, NULL, NULL);
+    if (ret == -1) {
+        ret = -errno;
+        goto out;
+    }
+
+    ret = dict_foreach_fnmatch(xdata, "trusted.glusterfs.quota.*",
+                               remove_quota_keys, frame);
+    if (ret == -1) {
+        ret = -errno;
+        goto out;
+    }
+    ret = dict_foreach_fnmatch(xdata, PGFID_XATTR_KEY_PREFIX "*",
+                               remove_quota_keys, frame);
+    if (ret == -1) {
+        ret = -errno;
+        goto out;
+    }
+
+    ret = 0;
 out:
-        if (xdata)
-                dict_unref (xdata);
+    if (xdata)
+        dict_unref(xdata);
 
-        return ret;
+    return ret;
 }
 
 int
-marker_do_xattr_cleanup (call_frame_t *frame, xlator_t *this, dict_t *xdata,
+marker_do_xattr_cleanup(call_frame_t *frame, xlator_t *this, dict_t *xdata,
                         loc_t *loc)
 {
-        int           ret       = -1;
-        marker_local_t *local    = NULL;
+    int ret = -1;
+    marker_local_t *local = NULL;
 
-        local = mem_get0 (this->local_pool);
-	if (!local)
-		goto out;
+    local = mem_get0(this->local_pool);
+    if (!local)
+        goto out;
 
-        MARKER_INIT_LOCAL (frame, local);
+    MARKER_INIT_LOCAL(frame, local);
 
-        loc_copy (&local->loc, loc);
-        ret = synctask_new (this->ctx->env, quota_xattr_cleaner,
-			    quota_xattr_cleaner_cbk, frame, xdata);
-        if (ret) {
-		gf_log (this->name, GF_LOG_ERROR, "Failed to create synctask "
-			"for cleaning up quota extended attributes");
-                goto out;
-	}
+    loc_copy(&local->loc, loc);
+    ret = synctask_new(this->ctx->env, quota_xattr_cleaner,
+                       quota_xattr_cleaner_cbk, frame, xdata);
+    if (ret) {
+        gf_log(this->name, GF_LOG_ERROR,
+               "Failed to create synctask "
+               "for cleaning up quota extended attributes");
+        goto out;
+    }
 
-        ret = 0;
+    ret = 0;
 out:
-        if (ret) {
-		frame->local = NULL;
-                STACK_UNWIND_STRICT (setxattr, frame, -1, ENOMEM, xdata);
-		marker_local_unref (local);
-        }
-        return ret;
+    if (ret)
+        MARKER_STACK_UNWIND(setxattr, frame, -1, ENOMEM, xdata);
+
+    return ret;
 }
 
-static inline gf_boolean_t
-marker_xattr_cleanup_cmd (dict_t *dict)
+static gf_boolean_t
+marker_xattr_cleanup_cmd(dict_t *dict)
 {
-        return (dict_get (dict, VIRTUAL_QUOTA_XATTR_CLEANUP_KEY) != NULL);
+    return (dict_get(dict, VIRTUAL_QUOTA_XATTR_CLEANUP_KEY) != NULL);
 }
 
 int32_t
-marker_setxattr (call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *dict,
-                 int32_t flags, dict_t *xdata)
+marker_setxattr(call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *dict,
+                int32_t flags, dict_t *xdata)
 {
-        int32_t          ret            = 0;
-        marker_local_t  *local          = NULL;
-        marker_conf_t   *priv           = NULL;
-        int              op_errno       = ENOMEM;
-
-        priv = this->private;
-
-        if (marker_xattr_cleanup_cmd (dict)) {
-                if (frame->root->uid != 0 || frame->root->gid != 0) {
-                        op_errno = EPERM;
-                        ret = -1;
-                        goto err;
-                }
-
-                /* The following function does the cleanup and then unwinds the
-                 * corresponding call*/
-                loc_path (loc, NULL);
-                marker_do_xattr_cleanup (frame, this, xdata, loc);
-                return 0;
+    int32_t ret = 0;
+    marker_local_t *local = NULL;
+    marker_conf_t *priv = NULL;
+    int op_errno = ENOMEM;
+
+    priv = this->private;
+
+    if (marker_xattr_cleanup_cmd(dict)) {
+        if (frame->root->uid != 0 || frame->root->gid != 0) {
+            op_errno = EPERM;
+            ret = -1;
+            goto err;
         }
 
-        if (priv->feature_enabled == 0)
-                goto wind;
+        /* The following function does the cleanup and then unwinds the
+         * corresponding call*/
+        loc_path(loc, NULL);
+        marker_do_xattr_cleanup(frame, this, xdata, loc);
+        return 0;
+    }
+
+    ret = marker_key_replace_with_ver(this, dict);
+    if (ret < 0)
+        goto err;
 
-        ret = call_from_sp_client_to_reset_tmfile (frame, this, dict);
-        if (ret == 0)
-                return 0;
+    if (priv->feature_enabled == 0)
+        goto wind;
 
-        local = mem_get0 (this->local_pool);
+    ret = call_from_sp_client_to_reset_tmfile(frame, this, dict);
+    if (ret == 0)
+        return 0;
 
-        MARKER_INIT_LOCAL (frame, local);
+    local = mem_get0(this->local_pool);
 
-        ret = loc_copy (&local->loc, loc);
+    MARKER_INIT_LOCAL(frame, local);
 
-        if (ret == -1)
-                goto err;
+    ret = loc_copy(&local->loc, loc);
+
+    if (ret == -1)
+        goto err;
 wind:
-        STACK_WIND (frame, marker_setxattr_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->setxattr, loc, dict, flags, xdata);
-        return 0;
+    STACK_WIND(frame, marker_setxattr_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->setxattr, loc, dict, flags, xdata);
+    return 0;
 err:
-        STACK_UNWIND_STRICT (setxattr, frame, -1, op_errno, NULL);
+    MARKER_STACK_UNWIND(setxattr, frame, -1, op_errno, NULL);
 
-        return 0;
+    return 0;
 }
 
-
 int32_t
-marker_fsetxattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                      int32_t op_ret, int32_t op_errno, dict_t *xdata)
+marker_fsetxattr_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                     int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
-        marker_local_t     *local   = NULL;
-        marker_conf_t      *priv    = NULL;
+    marker_local_t *local = NULL;
+    marker_conf_t *priv = NULL;
 
-        if (op_ret == -1) {
-                gf_log (this->name, GF_LOG_TRACE, "%s occurred in "
-                        "fsetxattr", strerror (op_errno));
-        }
+    if (op_ret == -1) {
+        gf_log(this->name, GF_LOG_TRACE,
+               "%s occurred in "
+               "fsetxattr",
+               strerror(op_errno));
+    }
 
-        local = (marker_local_t *) frame->local;
+    local = (marker_local_t *)frame->local;
 
-        frame->local = NULL;
+    frame->local = NULL;
 
-        STACK_UNWIND_STRICT (fsetxattr, frame, op_ret, op_errno, xdata);
+    STACK_UNWIND_STRICT(fsetxattr, frame, op_ret, op_errno, xdata);
 
-        if (op_ret == -1 || local == NULL)
-                goto out;
+    if (op_ret == -1 || local == NULL)
+        goto out;
 
-        priv = this->private;
+    priv = this->private;
 
-        if (priv->feature_enabled & GF_XTIME)
-                marker_xtime_update_marks (this, local);
+    if (priv->feature_enabled & GF_XTIME)
+        marker_xtime_update_marks(this, local);
 out:
-        marker_local_unref (local);
+    marker_local_unref(local);
 
-        return 0;
+    return 0;
 }
 
 int32_t
-marker_fsetxattr (call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *dict,
-                  int32_t flags, dict_t *xdata)
+marker_fsetxattr(call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *dict,
+                 int32_t flags, dict_t *xdata)
 {
-        int32_t          ret   = 0;
-        marker_local_t  *local = NULL;
-        marker_conf_t   *priv  = NULL;
+    int32_t ret = 0;
+    marker_local_t *local = NULL;
+    marker_conf_t *priv = NULL;
 
-        priv = this->private;
+    priv = this->private;
 
-        if (priv->feature_enabled == 0)
-                goto wind;
+    if (priv->feature_enabled == 0)
+        goto wind;
 
-        ret = call_from_sp_client_to_reset_tmfile (frame, this, dict);
-        if (ret == 0)
-                return 0;
+    ret = call_from_sp_client_to_reset_tmfile(frame, this, dict);
+    if (ret == 0)
+        return 0;
 
-        local = mem_get0 (this->local_pool);
+    local = mem_get0(this->local_pool);
 
-        MARKER_INIT_LOCAL (frame, local);
+    MARKER_INIT_LOCAL(frame, local);
 
-        ret = marker_inode_loc_fill (fd->inode, NULL, &local->loc);
+    ret = marker_inode_loc_fill(fd->inode, &local->loc);
 
-        if (ret == -1)
-                goto err;
+    if (ret == -1)
+        goto err;
 wind:
-        STACK_WIND (frame, marker_fsetxattr_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->fsetxattr, fd, dict, flags, xdata);
-        return 0;
+    STACK_WIND(frame, marker_fsetxattr_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->fsetxattr, fd, dict, flags, xdata);
+    return 0;
 err:
-        STACK_UNWIND_STRICT (fsetxattr, frame, -1, ENOMEM, NULL);
+    MARKER_STACK_UNWIND(fsetxattr, frame, -1, ENOMEM, NULL);
 
-        return 0;
+    return 0;
 }
 
-
 int32_t
-marker_fsetattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                     int32_t op_ret, int32_t op_errno, struct iatt *statpre,
-                     struct iatt *statpost, dict_t *xdata)
+marker_fsetattr_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                    int32_t op_ret, int32_t op_errno, struct iatt *statpre,
+                    struct iatt *statpost, dict_t *xdata)
 {
-        marker_local_t     *local   = NULL;
-        marker_conf_t      *priv    = NULL;
+    marker_local_t *local = NULL;
+    marker_conf_t *priv = NULL;
 
-        if (op_ret == -1) {
-                gf_log (this->name, GF_LOG_TRACE, "%s occurred in "
-                        "fsetattr ", strerror (op_errno));
-        }
+    if (op_ret == -1) {
+        gf_log(this->name, GF_LOG_TRACE,
+               "%s occurred in "
+               "fsetattr ",
+               strerror(op_errno));
+    }
 
-        local = (marker_local_t *) frame->local;
+    local = (marker_local_t *)frame->local;
 
-        frame->local = NULL;
+    frame->local = NULL;
 
-        STACK_UNWIND_STRICT (fsetattr, frame, op_ret, op_errno, statpre,
-                             statpost, xdata);
+    STACK_UNWIND_STRICT(fsetattr, frame, op_ret, op_errno, statpre, statpost,
+                        xdata);
 
-        if (op_ret == -1 || local == NULL)
-                goto out;
+    if (op_ret == -1 || local == NULL)
+        goto out;
 
-        priv = this->private;
+    priv = this->private;
 
-        if (priv->feature_enabled & GF_XTIME)
-                marker_xtime_update_marks (this, local);
+    if (priv->feature_enabled & GF_XTIME)
+        marker_xtime_update_marks(this, local);
 out:
-        marker_local_unref (local);
+    marker_local_unref(local);
 
-        return 0;
+    return 0;
 }
 
-
 int32_t
-marker_fsetattr (call_frame_t *frame, xlator_t *this, fd_t *fd,
-                 struct iatt *stbuf, int32_t valid, dict_t *xdata)
+marker_fsetattr(call_frame_t *frame, xlator_t *this, fd_t *fd,
+                struct iatt *stbuf, int32_t valid, dict_t *xdata)
 {
-        int32_t          ret   = 0;
-        marker_local_t  *local = NULL;
-        marker_conf_t   *priv  = NULL;
+    int32_t ret = 0;
+    marker_local_t *local = NULL;
+    marker_conf_t *priv = NULL;
 
-        priv = this->private;
+    priv = this->private;
 
-        if (priv->feature_enabled == 0)
-                goto wind;
+    if (priv->feature_enabled == 0)
+        goto wind;
 
-        local = mem_get0 (this->local_pool);
+    local = mem_get0(this->local_pool);
 
-        MARKER_INIT_LOCAL (frame, local);
+    MARKER_INIT_LOCAL(frame, local);
 
-        ret = marker_inode_loc_fill (fd->inode, NULL, &local->loc);
+    ret = marker_inode_loc_fill(fd->inode, &local->loc);
 
-        if (ret == -1)
-                goto err;
+    if (ret == -1)
+        goto err;
 wind:
-        STACK_WIND (frame, marker_fsetattr_cbk, FIRST_CHILD (this),
-                    FIRST_CHILD (this)->fops->fsetattr, fd, stbuf, valid, xdata);
-        return 0;
+    STACK_WIND(frame, marker_fsetattr_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->fsetattr, fd, stbuf, valid, xdata);
+    return 0;
 err:
-        STACK_UNWIND_STRICT (fsetattr, frame, -1, ENOMEM, NULL, NULL, NULL);
+    MARKER_STACK_UNWIND(fsetattr, frame, -1, ENOMEM, NULL, NULL, NULL);
 
-        return 0;
+    return 0;
 }
 
-
 int32_t
-marker_setattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                    int32_t op_ret, int32_t op_errno, struct iatt *statpre,
-                    struct iatt *statpost, dict_t *xdata)
+marker_setattr_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                   int32_t op_ret, int32_t op_errno, struct iatt *statpre,
+                   struct iatt *statpost, dict_t *xdata)
 {
-        marker_local_t     *local   = NULL;
-        marker_conf_t      *priv    = NULL;
+    marker_local_t *local = NULL;
+    marker_conf_t *priv = NULL;
 
-        local = (marker_local_t *) frame->local;
+    local = (marker_local_t *)frame->local;
 
-        frame->local = NULL;
+    frame->local = NULL;
 
-        if (op_ret == -1) {
-                gf_log (this->name, GF_LOG_TRACE,
-                        "%s occurred during setattr of %s",
-                        strerror (op_errno),
-                        (local ? local->loc.path : "<nul>"));
-        }
+    if (op_ret == -1) {
+        gf_log(this->name, GF_LOG_TRACE, "%s occurred during setattr of %s",
+               strerror(op_errno), (local ? local->loc.path : "<nul>"));
+    }
 
-        STACK_UNWIND_STRICT (setattr, frame, op_ret, op_errno, statpre,
-                             statpost, xdata);
+    STACK_UNWIND_STRICT(setattr, frame, op_ret, op_errno, statpre, statpost,
+                        xdata);
 
-        if (op_ret == -1 || local == NULL)
-                goto out;
+    if (op_ret == -1 || local == NULL)
+        goto out;
 
-        priv = this->private;
+    priv = this->private;
 
-        if (priv->feature_enabled & GF_XTIME)
-                marker_xtime_update_marks (this, local);
+    if (priv->feature_enabled & GF_XTIME)
+        marker_xtime_update_marks(this, local);
 out:
-        marker_local_unref (local);
+    marker_local_unref(local);
 
-        return 0;
+    return 0;
 }
 
 int32_t
-marker_setattr (call_frame_t *frame, xlator_t *this, loc_t *loc,
-                struct iatt *stbuf, int32_t valid, dict_t *xdata)
+marker_setattr(call_frame_t *frame, xlator_t *this, loc_t *loc,
+               struct iatt *stbuf, int32_t valid, dict_t *xdata)
 {
-        int32_t          ret   = 0;
-        marker_local_t  *local = NULL;
-        marker_conf_t   *priv  = NULL;
+    int32_t ret = 0;
+    marker_local_t *local = NULL;
+    marker_conf_t *priv = NULL;
 
-        priv = this->private;
+    priv = this->private;
 
-        if (priv->feature_enabled == 0)
-                goto wind;
+    if (priv->feature_enabled == 0)
+        goto wind;
 
-        local = mem_get0 (this->local_pool);
+    local = mem_get0(this->local_pool);
 
-        MARKER_INIT_LOCAL (frame, local);
+    MARKER_INIT_LOCAL(frame, local);
 
-        ret = loc_copy (&local->loc, loc);
+    ret = loc_copy(&local->loc, loc);
 
-        if (ret == -1)
-                goto err;
+    if (ret == -1)
+        goto err;
 wind:
-        STACK_WIND (frame, marker_setattr_cbk, FIRST_CHILD (this),
-                    FIRST_CHILD (this)->fops->setattr, loc, stbuf, valid, xdata);
-        return 0;
+    STACK_WIND(frame, marker_setattr_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->setattr, loc, stbuf, valid, xdata);
+    return 0;
 err:
-        STACK_UNWIND_STRICT (setattr, frame, -1, ENOMEM, NULL, NULL, NULL);
+    MARKER_STACK_UNWIND(setattr, frame, -1, ENOMEM, NULL, NULL, NULL);
 
-        return 0;
+    return 0;
 }
 
-
 int32_t
-marker_removexattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                        int32_t op_ret, int32_t op_errno, dict_t *xdata)
+marker_removexattr_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                       int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
-        marker_local_t     *local   = NULL;
-        marker_conf_t      *priv    = NULL;
+    marker_local_t *local = NULL;
+    marker_conf_t *priv = NULL;
 
-        if (op_ret == -1) {
-                gf_log (this->name, GF_LOG_TRACE,
-                                "%s occurred while "
-                                "removing extended attribute",
-                                     strerror (op_errno));
-        }
+    if (op_ret == -1) {
+        gf_log(this->name, GF_LOG_TRACE,
+               "%s occurred while "
+               "removing extended attribute",
+               strerror(op_errno));
+    }
 
-        local = (marker_local_t *) frame->local;
+    local = (marker_local_t *)frame->local;
 
-        frame->local = NULL;
+    frame->local = NULL;
 
-        STACK_UNWIND_STRICT (removexattr, frame, op_ret, op_errno, xdata);
+    STACK_UNWIND_STRICT(removexattr, frame, op_ret, op_errno, xdata);
 
-        if (op_ret == -1 || local == NULL)
-                goto out;
+    if (op_ret == -1 || local == NULL)
+        goto out;
 
-        priv = this->private;
+    priv = this->private;
 
-        if (priv->feature_enabled & GF_XTIME)
-                marker_xtime_update_marks (this, local);
+    if (priv->feature_enabled & GF_XTIME)
+        marker_xtime_update_marks(this, local);
 out:
-        marker_local_unref (local);
+    marker_local_unref(local);
 
-        return 0;
+    return 0;
 }
 
 int32_t
-marker_removexattr (call_frame_t *frame, xlator_t *this, loc_t *loc,
-                    const char *name, dict_t *xdata)
-{
-        int32_t          ret   = 0;
-        marker_local_t  *local = NULL;
-        marker_conf_t   *priv  = NULL;
-
-        priv = this->private;
+marker_removexattr(call_frame_t *frame, xlator_t *this, loc_t *loc,
+                   const char *name, dict_t *xdata)
+{
+    int32_t ret = -1;
+    int32_t i = 0;
+    marker_local_t *local = NULL;
+    marker_conf_t *priv = NULL;
+    char key[QUOTA_KEY_MAX] = {
+        0,
+    };
+
+    priv = this->private;
+
+    if (name) {
+        for (i = 0; mq_ext_xattrs[i]; i++) {
+            if (strcmp(name, mq_ext_xattrs[i]))
+                continue;
+
+            GET_QUOTA_KEY(this, key, mq_ext_xattrs[i], ret);
+            if (ret < 0)
+                goto err;
+            name = key;
+            break;
+        }
+    }
 
-        if (priv->feature_enabled == 0)
-                goto wind;
+    if (priv->feature_enabled == 0)
+        goto wind;
 
-        local = mem_get0 (this->local_pool);
+    local = mem_get0(this->local_pool);
 
-        MARKER_INIT_LOCAL (frame, local);
+    MARKER_INIT_LOCAL(frame, local);
 
-        ret = loc_copy (&local->loc, loc);
+    ret = loc_copy(&local->loc, loc);
 
-        if (ret == -1)
-                goto err;
+    if (ret == -1)
+        goto err;
 wind:
-        STACK_WIND (frame, marker_removexattr_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->removexattr, loc, name, xdata);
-        return 0;
+    STACK_WIND(frame, marker_removexattr_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->removexattr, loc, name, xdata);
+    return 0;
 err:
-        STACK_UNWIND_STRICT (removexattr, frame, -1, ENOMEM, NULL);
+    MARKER_STACK_UNWIND(removexattr, frame, -1, ENOMEM, NULL);
 
-        return 0;
+    return 0;
 }
 
 static gf_boolean_t
-__has_quota_xattrs (dict_t *xattrs)
+__has_quota_xattrs(dict_t *xattrs)
 {
-        if (dict_foreach_match (xattrs, _is_quota_internal_xattr, NULL,
-                                dict_null_foreach_fn, NULL) > 0)
-                return _gf_true;
+    if (dict_foreach_match(xattrs, _is_quota_internal_xattr, NULL,
+                           dict_null_foreach_fn, NULL) > 0)
+        return _gf_true;
 
-        return _gf_false;
+    return _gf_false;
 }
 
 int32_t
-marker_lookup_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                   int32_t op_ret, int32_t op_errno, inode_t *inode,
-                   struct iatt *buf, dict_t *dict, struct iatt *postparent)
-{
-        marker_conf_t  *priv    = NULL;
-        marker_local_t *local   = NULL;
-        dict_t         *xattrs  = NULL;
-        priv = this->private;
-
-        if (op_ret == -1) {
-                gf_log (this->name, GF_LOG_TRACE, "lookup failed with %s",
-                        strerror (op_errno));
+marker_lookup_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                  int32_t op_ret, int32_t op_errno, inode_t *inode,
+                  struct iatt *buf, dict_t *dict, struct iatt *postparent)
+{
+    marker_conf_t *priv = NULL;
+    marker_local_t *local = NULL;
+    dict_t *xattrs = NULL;
+    quota_inode_ctx_t *ctx = NULL;
+    int32_t ret = -1;
+
+    priv = this->private;
+    local = (marker_local_t *)frame->local;
+    frame->local = NULL;
+
+    if (op_ret == -1) {
+        gf_log(this->name, GF_LOG_TRACE, "lookup failed with %s",
+               strerror(op_errno));
+        goto unwind;
+    }
+
+    ret = marker_key_set_ver(this, dict);
+    if (ret < 0) {
+        op_ret = -1;
+        op_errno = ENOMEM;
+        goto unwind;
+    }
+
+    if (dict && __has_quota_xattrs(dict)) {
+        xattrs = dict_copy_with_ref(dict, NULL);
+        if (!xattrs) {
+            op_ret = -1;
+            op_errno = ENOMEM;
+        } else {
+            marker_filter_internal_xattrs(this, xattrs);
         }
+    } else if (dict) {
+        xattrs = dict_ref(dict);
+    }
 
-        if (dict && __has_quota_xattrs (dict)) {
-                xattrs = dict_copy_with_ref (dict, NULL);
-                if (!xattrs) {
-                        op_ret = -1;
-                        op_errno = ENOMEM;
-                } else {
-                        marker_filter_internal_xattrs (this, xattrs);
-                }
-        } else if (dict) {
-                xattrs = dict_ref (dict);
+    if (op_ret >= 0 && inode && (priv->feature_enabled & GF_QUOTA)) {
+        ctx = mq_inode_ctx_new(inode, this);
+        if (ctx == NULL) {
+            gf_log(this->name, GF_LOG_WARNING,
+                   "mq_inode_ctx_new "
+                   "failed for %s",
+                   uuid_utoa(inode->gfid));
+            op_ret = -1;
+            op_errno = ENOMEM;
         }
+    }
 
-        local = (marker_local_t *) frame->local;
-
-        frame->local = NULL;
-
-        STACK_UNWIND_STRICT (lookup, frame, op_ret, op_errno, inode, buf,
-                             xattrs, postparent);
-
-        if (op_ret == -1 || local == NULL)
-                goto out;
+unwind:
+    STACK_UNWIND_STRICT(lookup, frame, op_ret, op_errno, inode, buf, xattrs,
+                        postparent);
 
-        /* copy the gfid from the stat structure instead of inode,
-         * since if the lookup is fresh lookup, then the inode
-         * would have not yet linked to the inode table which happens
-         * in protocol/server.
-         */
-        if (gf_uuid_is_null (local->loc.gfid))
-                gf_uuid_copy (local->loc.gfid, buf->ia_gfid);
+    if (op_ret == -1 || local == NULL)
+        goto out;
 
+    /* copy the gfid from the stat structure instead of inode,
+     * since if the lookup is fresh lookup, then the inode
+     * would have not yet linked to the inode table which happens
+     * in protocol/server.
+     */
+    if (gf_uuid_is_null(local->loc.gfid))
+        gf_uuid_copy(local->loc.gfid, buf->ia_gfid);
 
-        if (priv->feature_enabled & GF_QUOTA) {
-                mq_xattr_state (this, &local->loc, dict, *buf);
-        }
+    if (priv->feature_enabled & GF_QUOTA) {
+        mq_xattr_state(this, &local->loc, dict, buf);
+    }
 
 out:
-        marker_local_unref (local);
-        if (xattrs)
-                dict_unref (xattrs);
+    marker_local_unref(local);
+    if (xattrs)
+        dict_unref(xattrs);
 
-        return 0;
+    return 0;
 }
 
 int32_t
-marker_lookup (call_frame_t *frame, xlator_t *this,
-               loc_t *loc, dict_t *xattr_req)
+marker_lookup(call_frame_t *frame, xlator_t *this, loc_t *loc,
+              dict_t *xattr_req)
 {
-        int32_t         ret     = 0;
-        marker_local_t *local   = NULL;
-        marker_conf_t  *priv    = NULL;
+    int32_t ret = 0;
+    marker_local_t *local = NULL;
+    marker_conf_t *priv = NULL;
 
-        priv = this->private;
+    priv = this->private;
 
-        if (priv->feature_enabled == 0)
-                goto wind;
+    xattr_req = xattr_req ? dict_ref(xattr_req) : dict_new();
+    if (!xattr_req)
+        goto err;
 
-        local = mem_get0 (this->local_pool);
-        if (local == NULL)
-                goto err;
+    ret = marker_key_replace_with_ver(this, xattr_req);
+    if (ret < 0)
+        goto err;
 
-        MARKER_INIT_LOCAL (frame, local);
+    if (priv->feature_enabled == 0)
+        goto wind;
 
-        ret = loc_copy (&local->loc, loc);
-        if (ret == -1)
-                goto err;
+    local = mem_get0(this->local_pool);
+    if (local == NULL)
+        goto err;
+
+    MARKER_INIT_LOCAL(frame, local);
+
+    ret = loc_copy(&local->loc, loc);
+    if (ret == -1)
+        goto err;
+
+    if ((priv->feature_enabled & GF_QUOTA))
+        mq_req_xattr(this, loc, xattr_req, NULL, NULL);
 
-        if ((priv->feature_enabled & GF_QUOTA) && xattr_req)
-                mq_req_xattr (this, loc, xattr_req, NULL);
 wind:
-        STACK_WIND (frame, marker_lookup_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->lookup, loc, xattr_req);
-        return 0;
+    STACK_WIND(frame, marker_lookup_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->lookup, loc, xattr_req);
+
+    dict_unref(xattr_req);
+
+    return 0;
 err:
-        STACK_UNWIND_STRICT (lookup, frame, -1, ENOMEM, NULL, NULL, NULL, NULL);
+    MARKER_STACK_UNWIND(lookup, frame, -1, ENOMEM, NULL, NULL, NULL, NULL);
 
-        return 0;
-}
+    if (xattr_req)
+        dict_unref(xattr_req);
 
+    return 0;
+}
 
 int
-marker_build_ancestry_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                           int op_ret, int op_errno, gf_dirent_t *entries,
-                           dict_t *xdata)
+marker_build_ancestry_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                          int op_ret, int op_errno, gf_dirent_t *entries,
+                          dict_t *xdata)
 {
-        gf_dirent_t *entry = NULL;
-        loc_t        loc    = {0, };
-        inode_t     *parent = NULL;
-        int          ret    = -1;
-
-        if ((op_ret <= 0) || (entries == NULL)) {
-                goto out;
-        }
-
+    gf_dirent_t *entry = NULL;
+    quota_inode_ctx_t *ctx = NULL;
+    int ret = -1;
 
-        list_for_each_entry (entry, &entries->list, list) {
-                if (entry->inode == entry->inode->table->root) {
-                        inode_unref (parent);
-                        parent = NULL;
-                }
+    if ((op_ret <= 0) || (entries == NULL)) {
+        goto out;
+    }
 
-                ret = marker_inode_loc_fill (entry->inode,
-                                             entry->d_name, &loc);
-                if (ret) {
-                        gf_log (this->name, GF_LOG_WARNING, "Couldn't build "
-                                "loc for %s/%s",
-                                parent? uuid_utoa (parent->gfid): NULL,
-                                entry->d_name);
-                        continue;
-                }
+    list_for_each_entry(entry, &entries->list, list)
+    {
+        if (entry->inode == NULL)
+            continue;
 
-                mq_xattr_state (this, &loc, entry->dict, entry->d_stat);
-
-                inode_unref (parent);
-                parent = inode_ref (entry->inode);
-                loc_wipe (&loc);
+        ret = marker_key_set_ver(this, entry->dict);
+        if (ret < 0) {
+            op_ret = -1;
+            op_errno = ENOMEM;
+            break;
         }
 
-        if (parent)
-                inode_unref (parent);
+        ctx = mq_inode_ctx_new(entry->inode, this);
+        if (ctx == NULL)
+            gf_log(this->name, GF_LOG_WARNING,
+                   "mq_inode_ctx_new "
+                   "failed for %s",
+                   uuid_utoa(entry->inode->gfid));
+    }
 
 out:
-        STACK_UNWIND_STRICT (readdirp, frame, op_ret, op_errno, entries, xdata);
-        return 0;
+    STACK_UNWIND_STRICT(readdirp, frame, op_ret, op_errno, entries, xdata);
+    return 0;
 }
 
 int
-marker_readdirp_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                     int op_ret, int op_errno, gf_dirent_t *entries,
-                     dict_t *xdata)
-{
-        gf_dirent_t    *entry = NULL;
-        marker_conf_t  *priv  = NULL;
-        marker_local_t *local = NULL;
-        loc_t           loc   = {0, };
-        int             ret   = -1;
-        char           *resolvedpath = NULL;
+marker_readdirp_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                    int op_ret, int op_errno, gf_dirent_t *entries,
+                    dict_t *xdata)
+{
+    gf_dirent_t *entry = NULL;
+    marker_conf_t *priv = NULL;
+    marker_local_t *local = NULL;
+    loc_t loc = {
+        0,
+    };
+    int ret = -1;
+    char *resolvedpath = NULL;
+    quota_inode_ctx_t *ctx = NULL;
+
+    if (op_ret <= 0)
+        goto unwind;
+
+    priv = this->private;
+    local = frame->local;
+
+    if (!(priv->feature_enabled & GF_QUOTA) || (local == NULL)) {
+        goto unwind;
+    }
+
+    list_for_each_entry(entry, &entries->list, list)
+    {
+        if ((strcmp(entry->d_name, ".") == 0) ||
+            (strcmp(entry->d_name, "..") == 0) || entry->inode == NULL)
+            continue;
+
+        loc.parent = inode_ref(local->loc.inode);
+        loc.inode = inode_ref(entry->inode);
+        ret = inode_path(loc.parent, entry->d_name, &resolvedpath);
+        if (ret < 0) {
+            gf_log(this->name, GF_LOG_ERROR,
+                   "failed to get the "
+                   "path for the entry %s",
+                   entry->d_name);
+            loc_wipe(&loc);
+            continue;
+        }
 
-        if (op_ret <= 0)
-                goto unwind;
+        loc.path = resolvedpath;
+        resolvedpath = NULL;
 
-        priv = this->private;
-        local = frame->local;
+        ctx = mq_inode_ctx_new(loc.inode, this);
+        if (ctx == NULL)
+            gf_log(this->name, GF_LOG_WARNING,
+                   "mq_inode_ctx_new "
+                   "failed for %s",
+                   uuid_utoa(loc.inode->gfid));
 
-        if (!(priv->feature_enabled & GF_QUOTA) || (local == NULL)) {
-                goto unwind;
-        }
+        mq_xattr_state(this, &loc, entry->dict, &entry->d_stat);
+        loc_wipe(&loc);
 
-        list_for_each_entry (entry, &entries->list, list) {
-                if ((strcmp (entry->d_name, ".") == 0) ||
-                    (strcmp (entry->d_name, "..") == 0))
-                        continue;
-
-                loc.parent = inode_ref (local->loc.inode);
-                loc.inode = inode_ref (entry->inode);
-                ret = inode_path (loc.parent, entry->d_name, &resolvedpath);
-                if (ret < 0) {
-                        gf_log (this->name, GF_LOG_ERROR, "failed to get the "
-                                "path for the entry %s", entry->d_name);
-                        loc_wipe (&loc);
-                        continue;
-                }
-
-                loc.path = gf_strdup (resolvedpath);
-                if (!loc.path) {
-                        gf_log (this->name, GF_LOG_ERROR, "strdup of path "
-                                "failed for the entry %s (path: %s)",
-                                entry->d_name, resolvedpath);
-                        loc_wipe (&loc);
-                        continue;
-                }
-
-                mq_xattr_state (this, &loc, entry->dict, entry->d_stat);
-
-                loc_wipe (&loc);
-                GF_FREE (resolvedpath);
-                resolvedpath = NULL;
+        ret = marker_key_set_ver(this, entry->dict);
+        if (ret < 0) {
+            op_ret = -1;
+            op_errno = ENOMEM;
+            goto unwind;
         }
+    }
 
 unwind:
-        local = frame->local;
-        frame->local = NULL;
+    MARKER_STACK_UNWIND(readdirp, frame, op_ret, op_errno, entries, xdata);
 
-        STACK_UNWIND_STRICT (readdirp, frame, op_ret, op_errno, entries, xdata);
-        marker_local_unref (local);
-
-        return 0;
+    return 0;
 }
 
 int
-marker_readdirp (call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
-                 off_t offset, dict_t *dict)
-{
-        marker_conf_t  *priv  = NULL;
-        loc_t           loc   = {0, };
-        marker_local_t *local = NULL;
-
-        priv = this->private;
-
-        dict = dict ? dict_ref(dict) : dict_new();
-        if (!dict)
-                goto unwind;
-
-        if (dict_get (dict, GET_ANCESTRY_DENTRY_KEY)) {
-                STACK_WIND (frame, marker_build_ancestry_cbk,
-                            FIRST_CHILD(this),
-                            FIRST_CHILD(this)->fops->readdirp,
-                            fd, size, offset, dict);
-        } else {
-                if (priv->feature_enabled & GF_QUOTA) {
-                        local = mem_get0 (this->local_pool);
-
-                        MARKER_INIT_LOCAL (frame, local);
+marker_readdirp(call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
+                off_t offset, dict_t *dict)
+{
+    marker_conf_t *priv = NULL;
+    loc_t loc = {
+        0,
+    };
+    marker_local_t *local = NULL;
+    int ret = -1;
+
+    priv = this->private;
+
+    dict = dict ? dict_ref(dict) : dict_new();
+    if (!dict)
+        goto unwind;
+
+    ret = marker_key_replace_with_ver(this, dict);
+    if (ret < 0)
+        goto unwind;
+
+    if (dict_get(dict, GET_ANCESTRY_DENTRY_KEY)) {
+        STACK_WIND(frame, marker_build_ancestry_cbk, FIRST_CHILD(this),
+                   FIRST_CHILD(this)->fops->readdirp, fd, size, offset, dict);
+    } else {
+        if (priv->feature_enabled & GF_QUOTA) {
+            local = mem_get0(this->local_pool);
 
-                        loc.parent = local->loc.inode = inode_ref (fd->inode);
+            MARKER_INIT_LOCAL(frame, local);
 
-                        mq_req_xattr (this, &loc, dict, NULL);
-                }
+            loc.parent = local->loc.inode = inode_ref(fd->inode);
 
-                STACK_WIND (frame, marker_readdirp_cbk,
-                            FIRST_CHILD(this),
-                            FIRST_CHILD(this)->fops->readdirp,
-                            fd, size, offset, dict);
+            mq_req_xattr(this, &loc, dict, NULL, NULL);
         }
 
-        dict_unref (dict);
-        return 0;
+        STACK_WIND(frame, marker_readdirp_cbk, FIRST_CHILD(this),
+                   FIRST_CHILD(this)->fops->readdirp, fd, size, offset, dict);
+    }
+
+    dict_unref(dict);
+    return 0;
 unwind:
-        STACK_UNWIND_STRICT (readdirp, frame, -1, ENOMEM, NULL, NULL);
-        return 0;
+    MARKER_STACK_UNWIND(readdirp, frame, -1, ENOMEM, NULL, NULL);
+    return 0;
 }
 
 int32_t
-mem_acct_init (xlator_t *this)
+mem_acct_init(xlator_t *this)
 {
-        int     ret = -1;
+    int ret = -1;
 
-        if (!this)
-                return ret;
-
-        ret = xlator_mem_acct_init (this, gf_marker_mt_end + 1);
+    if (!this)
+        return ret;
 
-        if (ret != 0) {
-                gf_log(this->name, GF_LOG_ERROR, "Memory accounting init"
-                       " failed");
-                return ret;
-        }
+    ret = xlator_mem_acct_init(this, gf_marker_mt_end + 1);
 
+    if (ret != 0) {
+        gf_log(this->name, GF_LOG_ERROR,
+               "Memory accounting init"
+               " failed");
         return ret;
-}
+    }
 
+    return ret;
+}
 
 int32_t
-init_xtime_priv (xlator_t *this, dict_t *options)
+init_xtime_priv(xlator_t *this, dict_t *options)
 {
-        data_t          *data    = NULL;
-        int32_t          ret     = -1;
-        marker_conf_t   *priv    = NULL;
+    int32_t ret = -1;
+    marker_conf_t *priv = NULL;
+    char *tmp_opt = NULL;
 
-        GF_VALIDATE_OR_GOTO ("marker", this, out);
-        GF_VALIDATE_OR_GOTO (this->name, options, out);
-        GF_VALIDATE_OR_GOTO (this->name, this->private, out);
+    GF_VALIDATE_OR_GOTO("marker", this, out);
+    GF_VALIDATE_OR_GOTO(this->name, options, out);
+    GF_VALIDATE_OR_GOTO(this->name, this->private, out);
 
-        priv = this->private;
+    priv = this->private;
 
-        if((data = dict_get (options, VOLUME_UUID)) != NULL) {
-                priv->volume_uuid = data->data;
+    ret = dict_get_str(options, "volume-uuid", &tmp_opt);
 
-                ret = gf_uuid_parse (priv->volume_uuid, priv->volume_uuid_bin);
-                if (ret == -1) {
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "invalid volume uuid %s", priv->volume_uuid);
-                        goto out;
-                }
+    if (ret) {
+        priv->volume_uuid = NULL;
+        tmp_opt = "";
 
-                ret = gf_asprintf (& (priv->marker_xattr), "%s.%s.%s",
-                                   MARKER_XATTR_PREFIX, priv->volume_uuid,
-                                   XTIME);
+        gf_log(this->name, GF_LOG_ERROR,
+               "please specify the volume-uuid"
+               "in the translator options");
 
-                if (ret == -1){
-                        priv->marker_xattr = NULL;
-                        goto out;
-                }
+        return -1;
+    }
+    gf_asprintf(&priv->volume_uuid, "%s", tmp_opt);
 
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "volume-uuid = %s", priv->volume_uuid);
-        } else {
-                priv->volume_uuid = NULL;
+    ret = gf_uuid_parse(priv->volume_uuid, priv->volume_uuid_bin);
 
-                gf_log (this->name, GF_LOG_ERROR,
-                        "please specify the volume-uuid"
-                        "in the translator options");
+    if (ret == -1) {
+        gf_log(this->name, GF_LOG_ERROR, "invalid volume uuid %s",
+               priv->volume_uuid);
+        goto out;
+    }
 
-                return -1;
-        }
+    ret = gf_asprintf(&(priv->marker_xattr), "%s.%s.%s", MARKER_XATTR_PREFIX,
+                      priv->volume_uuid, XTIME);
 
-        if ((data = dict_get (options, TIMESTAMP_FILE)) != NULL) {
-                priv->timestamp_file = data->data;
+    if (ret == -1) {
+        priv->marker_xattr = NULL;
+        goto out;
+    }
 
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "the timestamp-file is = %s",
-                        priv->timestamp_file);
+    gf_log(this->name, GF_LOG_DEBUG, "volume-uuid = %s", priv->volume_uuid);
 
-        } else {
-                priv->timestamp_file = NULL;
+    ret = dict_get_str(options, "timestamp-file", &tmp_opt);
+    if (ret) {
+        priv->timestamp_file = NULL;
+        tmp_opt = "";
 
-                gf_log (this->name, GF_LOG_ERROR,
-                        "please specify the timestamp-file"
-                        "in the translator options");
+        gf_log(this->name, GF_LOG_ERROR,
+               "please specify the timestamp-file"
+               "in the translator options");
 
-                goto out;
-        }
+        goto out;
+    }
 
-        ret = 0;
+    ret = gf_asprintf(&priv->timestamp_file, "%s", tmp_opt);
+    if (ret == -1) {
+        priv->timestamp_file = NULL;
+        goto out;
+    }
+
+    gf_log(this->name, GF_LOG_DEBUG, "the timestamp-file is = %s",
+           priv->timestamp_file);
+
+    ret = 0;
 out:
-        return ret;
+    return ret;
 }
 
 void
-marker_xtime_priv_cleanup (xlator_t *this)
+marker_xtime_priv_cleanup(xlator_t *this)
 {
-        marker_conf_t *priv = NULL;
+    marker_conf_t *priv = NULL;
 
-        GF_VALIDATE_OR_GOTO ("marker", this, out);
+    GF_VALIDATE_OR_GOTO("marker", this, out);
 
-        priv = (marker_conf_t *) this->private;
+    priv = (marker_conf_t *)this->private;
 
-        GF_VALIDATE_OR_GOTO (this->name, priv, out);
+    GF_VALIDATE_OR_GOTO(this->name, priv, out);
 
-        GF_FREE (priv->volume_uuid);
+    GF_FREE(priv->volume_uuid);
 
-        GF_FREE (priv->timestamp_file);
+    GF_FREE(priv->timestamp_file);
 
-        GF_FREE (priv->marker_xattr);
+    GF_FREE(priv->marker_xattr);
 out:
-        return;
+    return;
 }
 
 void
-marker_priv_cleanup (xlator_t *this)
+marker_priv_cleanup(xlator_t *this)
 {
-        marker_conf_t *priv = NULL;
+    marker_conf_t *priv = NULL;
 
-        GF_VALIDATE_OR_GOTO ("marker", this, out);
+    GF_VALIDATE_OR_GOTO("marker", this, out);
 
-        priv = (marker_conf_t *) this->private;
+    priv = (marker_conf_t *)this->private;
 
-        GF_VALIDATE_OR_GOTO (this->name, priv, out);
+    GF_VALIDATE_OR_GOTO(this->name, priv, out);
 
-        marker_xtime_priv_cleanup (this);
+    marker_xtime_priv_cleanup(this);
 
-        LOCK_DESTROY (&priv->lock);
+    LOCK_DESTROY(&priv->lock);
+
+    GF_FREE(priv);
+
+    if (this->local_pool) {
+        mem_pool_destroy(this->local_pool);
+        this->local_pool = NULL;
+    }
 
-        GF_FREE (priv);
 out:
-        return;
+    return;
 }
 
 int32_t
-reconfigure (xlator_t *this, dict_t *options)
-{
-        int32_t         ret     = 0;
-        data_t         *data    = NULL;
-        gf_boolean_t    flag    = _gf_false;
-        marker_conf_t  *priv    = NULL;
-
-        GF_ASSERT (this);
-        GF_ASSERT (this->private);
-
-        priv = this->private;
-
-        priv->feature_enabled = 0;
-
-        GF_VALIDATE_OR_GOTO (this->name, options, out);
-
-        data = dict_get (options, "quota");
-        if (data) {
-                ret = gf_string2boolean (data->data, &flag);
-                if (ret == 0 && flag == _gf_true) {
-                        ret = init_quota_priv (this);
-                        if (ret < 0) {
-                                gf_log (this->name, GF_LOG_WARNING,
-                                        "failed to initialize quota private");
-                        } else {
-                                priv->feature_enabled |= GF_QUOTA;
-                        }
-                }
-        }
-
-        data = dict_get (options, "xtime");
-        if (data) {
-                ret = gf_string2boolean (data->data, &flag);
-                if (ret == 0 && flag == _gf_true) {
-                        marker_xtime_priv_cleanup (this);
-
-                        ret = init_xtime_priv (this, options);
-                        if (ret < 0) {
-                                gf_log (this->name, GF_LOG_WARNING,
-                                        "failed to initialize xtime private, "
-                                        "xtime updation will fail");
-                        } else {
-                                priv->feature_enabled |= GF_XTIME;
-                                data = dict_get (options, "gsync-force-xtime");
-                                if (!data)
-                                        goto out;
-                                ret = gf_string2boolean (data->data, &flag);
-                                if (ret == 0 && flag)
-                                        priv->feature_enabled |= GF_XTIME_GSYNC_FORCE;
-                        }
-                }
-        }
+reconfigure(xlator_t *this, dict_t *options)
+{
+    int32_t ret = 0;
+    data_t *data = NULL;
+    gf_boolean_t flag = _gf_false;
+    marker_conf_t *priv = NULL;
+    int32_t version = 0;
+
+    GF_ASSERT(this);
+    GF_ASSERT(this->private);
+
+    priv = this->private;
+
+    priv->feature_enabled = 0;
+
+    GF_VALIDATE_OR_GOTO(this->name, options, out);
+
+    data = dict_get(options, "quota");
+    if (data) {
+        ret = gf_string2boolean(data->data, &flag);
+        if (ret == 0 && flag == _gf_true)
+            priv->feature_enabled |= GF_QUOTA;
+    }
+
+    data = dict_get(options, "inode-quota");
+    if (data) {
+        ret = gf_string2boolean(data->data, &flag);
+        if (ret == 0 && flag == _gf_true)
+            priv->feature_enabled |= GF_INODE_QUOTA;
+    }
+
+    data = dict_get(options, "quota-version");
+    if (data)
+        ret = gf_string2int32(data->data, &version);
+
+    if (priv->feature_enabled) {
+        if (version >= 0)
+            priv->version = version;
+        else
+            gf_log(this->name, GF_LOG_ERROR,
+                   "Invalid quota "
+                   "version %d",
+                   priv->version);
+    }
+
+    data = dict_get(options, "xtime");
+    if (data) {
+        ret = gf_string2boolean(data->data, &flag);
+        if (ret == 0 && flag == _gf_true) {
+            marker_xtime_priv_cleanup(this);
+
+            ret = init_xtime_priv(this, options);
+            if (ret < 0) {
+                gf_log(this->name, GF_LOG_WARNING,
+                       "failed to initialize xtime private, "
+                       "xtime updation will fail");
+            } else {
+                priv->feature_enabled |= GF_XTIME;
+                data = dict_get(options, "gsync-force-xtime");
+                if (!data)
+                    goto out;
+                ret = gf_string2boolean(data->data, &flag);
+                if (ret == 0 && flag)
+                    priv->feature_enabled |= GF_XTIME_GSYNC_FORCE;
+            }
+        }
+    }
 out:
-        return ret;
+    return ret;
 }
 
-
 int32_t
-init (xlator_t *this)
-{
-        dict_t        *options = NULL;
-        data_t        *data    = NULL;
-        int32_t        ret     = 0;
-        gf_boolean_t   flag    = _gf_false;
-        marker_conf_t *priv    = NULL;
-
-        if (!this->children) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "marker translator needs subvolume defined.");
-                return -1;
-        }
+init(xlator_t *this)
+{
+    dict_t *options = NULL;
+    data_t *data = NULL;
+    int32_t ret = 0;
+    gf_boolean_t flag = _gf_false;
+    marker_conf_t *priv = NULL;
 
-        if (!this->parents) {
-                gf_log (this->name, GF_LOG_WARNING,
-                        "Volume is dangling.");
-                return -1;
-        }
+    if (!this->children) {
+        gf_log(this->name, GF_LOG_ERROR,
+               "marker translator needs subvolume defined.");
+        return -1;
+    }
 
-        options = this->options;
+    if (!this->parents) {
+        gf_log(this->name, GF_LOG_WARNING, "Volume is dangling.");
+        return -1;
+    }
 
-        ALLOCATE_OR_GOTO (this->private, marker_conf_t, err);
+    options = this->options;
 
-        priv = this->private;
+    ALLOCATE_OR_GOTO(this->private, marker_conf_t, err);
 
-        priv->feature_enabled = 0;
+    priv = this->private;
 
-        LOCK_INIT (&priv->lock);
+    priv->feature_enabled = 0;
+    priv->version = 0;
 
-        data = dict_get (options, "quota");
-        if (data) {
-                ret = gf_string2boolean (data->data, &flag);
-                if (ret == 0 && flag == _gf_true) {
-                        ret = init_quota_priv (this);
-                        if (ret < 0)
-                                goto err;
+    LOCK_INIT(&priv->lock);
 
-                        priv->feature_enabled |= GF_QUOTA;
-                }
-        }
+    data = dict_get(options, "quota");
+    if (data) {
+        ret = gf_string2boolean(data->data, &flag);
+        if (ret == 0 && flag == _gf_true)
+            priv->feature_enabled |= GF_QUOTA;
+    }
 
-        data = dict_get (options, "xtime");
-        if (data) {
-                ret = gf_string2boolean (data->data, &flag);
-                if (ret == 0 && flag == _gf_true) {
-                        ret = init_xtime_priv (this, options);
-                        if (ret < 0)
-                                goto err;
-
-                        priv->feature_enabled |= GF_XTIME;
-                        data = dict_get (options, "gsync-force-xtime");
-                        if (!data)
-                                goto cont;
-                        ret = gf_string2boolean (data->data, &flag);
-                        if (ret == 0 && flag)
-                                priv->feature_enabled |= GF_XTIME_GSYNC_FORCE;
-                }
-        }
+    data = dict_get(options, "inode-quota");
+    if (data) {
+        ret = gf_string2boolean(data->data, &flag);
+        if (ret == 0 && flag == _gf_true)
+            priv->feature_enabled |= GF_INODE_QUOTA;
+    }
+
+    data = dict_get(options, "quota-version");
+    if (data)
+        ret = gf_string2int32(data->data, &priv->version);
 
- cont:
-        this->local_pool = mem_pool_new (marker_local_t, 128);
-        if (!this->local_pool) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "failed to create local_t's memory pool");
+    if ((ret == 0) && priv->feature_enabled && priv->version < 0) {
+        gf_log(this->name, GF_LOG_ERROR, "Invalid quota version %d",
+               priv->version);
+        goto err;
+    }
+
+    data = dict_get(options, "xtime");
+    if (data) {
+        ret = gf_string2boolean(data->data, &flag);
+        if (ret == 0 && flag == _gf_true) {
+            ret = init_xtime_priv(this, options);
+            if (ret < 0)
                 goto err;
-        }
 
-        return 0;
+            priv->feature_enabled |= GF_XTIME;
+            data = dict_get(options, "gsync-force-xtime");
+            if (!data)
+                goto cont;
+            ret = gf_string2boolean(data->data, &flag);
+            if (ret == 0 && flag)
+                priv->feature_enabled |= GF_XTIME_GSYNC_FORCE;
+        }
+    }
+
+cont:
+    this->local_pool = mem_pool_new(marker_local_t, 128);
+    if (!this->local_pool) {
+        gf_log(this->name, GF_LOG_ERROR,
+               "failed to create local_t's memory pool");
+        goto err;
+    }
+
+    return 0;
 err:
-        marker_priv_cleanup (this);
+    marker_priv_cleanup(this);
 
-        return -1;
+    return -1;
 }
 
 int32_t
-marker_forget (xlator_t *this, inode_t *inode)
+marker_forget(xlator_t *this, inode_t *inode)
 {
-        marker_inode_ctx_t *ctx   = NULL;
-        uint64_t            value = 0;
+    marker_inode_ctx_t *ctx = NULL;
+    uint64_t value = 0;
 
-        if (inode_ctx_del (inode, this, &value) != 0)
-                goto out;
+    if (inode_ctx_del(inode, this, &value) != 0)
+        goto out;
 
-        ctx = (marker_inode_ctx_t *)(unsigned long)value;
-        if (ctx == NULL) {
-                goto out;
-        }
+    ctx = (marker_inode_ctx_t *)(unsigned long)value;
+    if (ctx == NULL) {
+        goto out;
+    }
 
-        mq_forget (this, ctx->quota_ctx);
+    mq_forget(this, ctx->quota_ctx);
 
-        GF_FREE (ctx);
+    GF_FREE(ctx);
 out:
-        return 0;
+    return 0;
 }
 
 void
-fini (xlator_t *this)
+fini(xlator_t *this)
 {
-        marker_priv_cleanup (this);
+    marker_priv_cleanup(this);
 }
 
 struct xlator_fops fops = {
-        .lookup      = marker_lookup,
-        .create      = marker_create,
-        .mkdir       = marker_mkdir,
-        .writev      = marker_writev,
-        .truncate    = marker_truncate,
-        .ftruncate   = marker_ftruncate,
-        .symlink     = marker_symlink,
-        .link        = marker_link,
-        .unlink      = marker_unlink,
-        .rmdir       = marker_rmdir,
-        .rename      = marker_rename,
-        .mknod       = marker_mknod,
-        .setxattr    = marker_setxattr,
-        .fsetxattr   = marker_fsetxattr,
-        .setattr     = marker_setattr,
-        .fsetattr    = marker_fsetattr,
-        .removexattr = marker_removexattr,
-        .getxattr    = marker_getxattr,
-        .readdirp    = marker_readdirp,
-	.fallocate   = marker_fallocate,
-	.discard     = marker_discard,
-        .zerofill    = marker_zerofill,
+    .lookup = marker_lookup,
+    .create = marker_create,
+    .mkdir = marker_mkdir,
+    .writev = marker_writev,
+    .truncate = marker_truncate,
+    .ftruncate = marker_ftruncate,
+    .symlink = marker_symlink,
+    .link = marker_link,
+    .unlink = marker_unlink,
+    .rmdir = marker_rmdir,
+    .rename = marker_rename,
+    .mknod = marker_mknod,
+    .setxattr = marker_setxattr,
+    .fsetxattr = marker_fsetxattr,
+    .setattr = marker_setattr,
+    .fsetattr = marker_fsetattr,
+    .removexattr = marker_removexattr,
+    .getxattr = marker_getxattr,
+    .readdirp = marker_readdirp,
+    .fallocate = marker_fallocate,
+    .discard = marker_discard,
+    .zerofill = marker_zerofill,
 };
 
-struct xlator_cbks cbks = {
-        .forget = marker_forget
-};
+struct xlator_cbks cbks = {.forget = marker_forget};
 
 struct volume_options options[] = {
-        {.key = {"volume-uuid"}},
-        {.key = {"timestamp-file"}},
-        {.key = {"quota"}},
-        {.key = {"xtime"}},
-        {.key = {"gsync-force-xtime"}},
-        {.key = {NULL}}
+    {.key = {"volume-uuid"}, .default_value = "{{ volume.id }}"},
+    {.key = {"timestamp-file"}},
+    {
+        .key = {"quota"},
+        .op_version = {1},
+        .flags = OPT_FLAG_NONE,
+        .tags = {},
+    },
+    {
+        .key = {"inode-quota"},
+        .op_version = {1},
+        .flags = OPT_FLAG_NONE,
+        .tags = {},
+    },
+    {
+        .key = {"xtime"},
+        .op_version = {1},
+        .flags = OPT_FLAG_SETTABLE | OPT_FLAG_FORCE,
+        .tags = {},
+    },
+    {
+        .key = {"gsync-force-xtime"},
+        .op_version = {2},
+        .flags = OPT_FLAG_SETTABLE | OPT_FLAG_FORCE,
+        .tags = {},
+    },
+    {
+        .key = {"quota-version"},
+        .flags = OPT_FLAG_NONE,
+    },
+    {.key = {NULL}}};
+
+xlator_api_t xlator_api = {
+    .init = init,
+    .fini = fini,
+    .reconfigure = reconfigure,
+    .mem_acct_init = mem_acct_init,
+    .op_version = {1}, /* Present from the initial version */
+    .fops = &fops,
+    .cbks = &cbks,
+    .options = options,
+    .identifier = "marker",
+    .category = GF_MAINTAINED,
 };
diff --git a/xlators/features/marker/src/marker.h b/xlators/features/marker/src/marker.h
index b2d58d23f6c..4821094c14b 100644
--- a/xlators/features/marker/src/marker.h
+++ b/xlators/features/marker/src/marker.h
@@ -10,129 +10,138 @@
 #ifndef _MARKER_H
 #define _MARKER_H
 
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
-
 #include "marker-quota.h"
-#include "xlator.h"
-#include "defaults.h"
-#include "compat-uuid.h"
-#include "call-stub.h"
+#include <glusterfs/xlator.h>
+#include <glusterfs/defaults.h>
+#include <glusterfs/compat-uuid.h>
+#include <glusterfs/call-stub.h>
 
 #define MARKER_XATTR_PREFIX "trusted.glusterfs"
-#define XTIME               "xtime"
-#define VOLUME_MARK         "volume-mark"
-#define VOLUME_UUID         "volume-uuid"
-#define TIMESTAMP_FILE      "timestamp-file"
+#define XTIME "xtime"
+#define VOLUME_MARK "volume-mark"
+#define VOLUME_UUID "volume-uuid"
+#define TIMESTAMP_FILE "timestamp-file"
 
 enum {
-        GF_QUOTA             = 1,
-        GF_XTIME             = 2,
-        GF_XTIME_GSYNC_FORCE = 4,
+    GF_QUOTA = 1,
+    GF_XTIME = 2,
+    GF_XTIME_GSYNC_FORCE = 4,
+    GF_INODE_QUOTA = 8,
 };
 
 /*initialize the local variable*/
-#define MARKER_INIT_LOCAL(_frame,_local) do {                   \
-                _frame->local = _local;                         \
-                _local->pid = _frame->root->pid;                \
-                memset (&_local->loc, 0, sizeof (loc_t));       \
-                _local->ref = 1;                                \
-                _local->uid = -1;                               \
-                _local->gid = -1;                               \
-                LOCK_INIT (&_local->lock);                      \
-                _local->oplocal = NULL;                         \
-        } while (0)
+#define MARKER_INIT_LOCAL(_frame, _local)                                      \
+    do {                                                                       \
+        _frame->local = _local;                                                \
+        _local->pid = _frame->root->pid;                                       \
+        memset(&_local->loc, 0, sizeof(loc_t));                                \
+        _local->ref = 1;                                                       \
+        _local->uid = -1;                                                      \
+        _local->gid = -1;                                                      \
+        LOCK_INIT(&_local->lock);                                              \
+        _local->oplocal = NULL;                                                \
+    } while (0)
 
 /* try alloc and if it fails, goto label */
-#define ALLOCATE_OR_GOTO(var, type, label) do {                  \
-                var = GF_CALLOC (sizeof (type), 1,               \
-                                 gf_marker_mt_##type);           \
-                if (!var) {                                      \
-                        gf_log (this->name, GF_LOG_ERROR,        \
-                                "out of memory :(");             \
-                        goto label;                              \
-                }                                                \
-        } while (0)
-
-#define _MARKER_SET_UID_GID(dest, src)          \
-        do {                                    \
-                if (src->uid != -1 &&           \
-                    src->gid != -1) {           \
-                        dest->uid = src->uid;   \
-                        dest->gid = src->gid;   \
-                }                               \
-        } while (0)
-
-#define MARKER_SET_UID_GID(frame, dest, src)                    \
-        do {                                                    \
-                _MARKER_SET_UID_GID (dest, src);                \
-                frame->root->uid = 0;                           \
-                frame->root->gid = 0;                           \
-                frame->cookie = (void *) _GF_UID_GID_CHANGED;   \
-        } while (0)
-
-#define MARKER_RESET_UID_GID(frame, dest, src)                  \
-        do {                                                    \
-                _MARKER_SET_UID_GID (dest, src);                \
-                frame->cookie = NULL;                           \
-        } while (0)
-
-struct marker_local{
-        uint32_t        timebuf[2];
-        pid_t           pid;
-        loc_t           loc;
-        loc_t           parent_loc;
-        loc_t          *next_lock_on;
-        uid_t           uid;
-        gid_t           gid;
-        int32_t         ref;
-        int32_t         ia_nlink;
-        gf_lock_t       lock;
-        mode_t          mode;
-        int32_t         err;
-        call_stub_t    *stub;
-        int64_t         contribution;
-        struct marker_local *oplocal;
-
-        /* marker quota specific */
-        int64_t delta;
-        int64_t d_off;
-        int64_t sum;
-        int64_t size;
-        int32_t hl_count;
-        int32_t dentry_child_count;
-
-        fd_t         *fd;
-        call_frame_t *frame;
-
-        quota_inode_ctx_t    *ctx;
-        inode_contribution_t *contri;
-
-        int xflag;
-        dict_t *xdata;
-        gf_boolean_t skip_txn;
+#define ALLOCATE_OR_GOTO(var, type, label)                                     \
+    do {                                                                       \
+        var = GF_CALLOC(sizeof(type), 1, gf_marker_mt_##type);                 \
+        if (!var) {                                                            \
+            gf_log(this->name, GF_LOG_ERROR, "out of memory :(");              \
+            goto label;                                                        \
+        }                                                                      \
+    } while (0)
+
+#define _MARKER_SET_UID_GID(dest, src)                                         \
+    do {                                                                       \
+        if (src->uid != -1 && src->gid != -1) {                                \
+            dest->uid = src->uid;                                              \
+            dest->gid = src->gid;                                              \
+        }                                                                      \
+    } while (0)
+
+#define MARKER_SET_UID_GID(frame, dest, src)                                   \
+    do {                                                                       \
+        _MARKER_SET_UID_GID(dest, src);                                        \
+        frame->root->uid = 0;                                                  \
+        frame->root->gid = 0;                                                  \
+        frame->cookie = (void *)_GF_UID_GID_CHANGED;                           \
+    } while (0)
+
+#define MARKER_RESET_UID_GID(frame, dest, src)                                 \
+    do {                                                                       \
+        _MARKER_SET_UID_GID(dest, src);                                        \
+        frame->cookie = NULL;                                                  \
+    } while (0)
+
+#define MARKER_STACK_UNWIND(fop, frame, params...)                             \
+    do {                                                                       \
+        quota_local_t *_local = NULL;                                          \
+        if (frame) {                                                           \
+            _local = frame->local;                                             \
+            frame->local = NULL;                                               \
+        }                                                                      \
+        STACK_UNWIND_STRICT(fop, frame, params);                               \
+        if (_local)                                                            \
+            marker_local_unref(_local);                                        \
+    } while (0)
+
+struct marker_local {
+    uint32_t timebuf[2];
+    pid_t pid;
+    loc_t loc;
+    loc_t parent_loc;
+    uid_t uid;
+    gid_t gid;
+    int32_t ref;
+    uint32_t ia_nlink;
+    struct iatt buf;
+    gf_lock_t lock;
+    mode_t mode;
+    int32_t err;
+    call_stub_t *stub;
+    call_frame_t *lk_frame;
+    quota_meta_t contribution;
+    struct marker_local *oplocal;
+
+    /* marker quota specific */
+    int64_t delta;
+    int64_t d_off;
+    int64_t sum;
+    int64_t size;
+    int32_t hl_count;
+    int32_t dentry_child_count;
+
+    fd_t *fd;
+    call_frame_t *frame;
+
+    quota_inode_ctx_t *ctx;
+    inode_contribution_t *contri;
+
+    int xflag;
+    dict_t *xdata;
+    gf_boolean_t skip_txn;
 };
 typedef struct marker_local marker_local_t;
 
 #define quota_local_t marker_local_t
 
 struct marker_inode_ctx {
-        struct quota_inode_ctx *quota_ctx;
+    struct quota_inode_ctx *quota_ctx;
 };
 typedef struct marker_inode_ctx marker_inode_ctx_t;
 
-struct marker_conf{
-        char         feature_enabled;
-        char        *size_key;
-        char        *dirty_key;
-        char        *volume_uuid;
-        uuid_t      volume_uuid_bin;
-        char        *timestamp_file;
-        char        *marker_xattr;
-        uint64_t     quota_lk_owner;
-        gf_lock_t    lock;
+struct marker_conf {
+    char feature_enabled;
+    char *size_key;
+    char *dirty_key;
+    char *volume_uuid;
+    uuid_t volume_uuid_bin;
+    char *timestamp_file;
+    char *marker_xattr;
+    uint64_t quota_lk_owner;
+    gf_lock_t lock;
+    int32_t version;
 };
 typedef struct marker_conf marker_conf_t;
 
diff --git a/xlators/cluster/stripe/Makefile.am b/xlators/features/metadisp/Makefile.am
index d471a3f9243..a985f42a877 100644
--- a/xlators/cluster/stripe/Makefile.am
+++ b/xlators/features/metadisp/Makefile.am
@@ -1,3 +1,3 @@
 SUBDIRS = src
 
-CLEANFILES = 
+CLEANFILES =
diff --git a/xlators/features/metadisp/src/Makefile.am b/xlators/features/metadisp/src/Makefile.am
new file mode 100644
index 00000000000..1520ad8c424
--- /dev/null
+++ b/xlators/features/metadisp/src/Makefile.am
@@ -0,0 +1,38 @@
+noinst_PYTHON = gen-fops.py
+
+EXTRA_DIST = fops-tmpl.c
+
+xlator_LTLIBRARIES = metadisp.la
+xlatordir = $(libdir)/glusterfs/$(PACKAGE_VERSION)/xlator/features
+
+nodist_metadisp_la_SOURCES = fops.c
+
+BUILT_SOURCES = fops.c
+
+metadisp_la_LDFLAGS = -module $(GF_XLATOR_DEFAULT_LDFLAGS)
+
+metadisp_la_SOURCES = metadisp.c \
+		metadisp-unlink.c \
+		metadisp-stat.c \
+		metadisp-lookup.c \
+		metadisp-readdir.c \
+		metadisp-create.c \
+		metadisp-open.c \
+		metadisp-fsync.c \
+		metadisp-setattr.c \
+		backend.c
+
+metadisp_la_LIBADD = $(top_builddir)/libglusterfs/src/libglusterfs.la
+
+noinst_HEADERS = metadisp.h metadisp-fops.h
+
+AM_CPPFLAGS = $(GF_CPPFLAGS) -I$(top_srcdir)/libglusterfs/src \
+	-I$(top_srcdir)/rpc/xdr/src -I$(top_builddir)/rpc/xdr/src
+
+AM_CFLAGS = -Wall $(GF_CFLAGS)
+
+fops.c: fops-tmpl.c $(top_srcdir)/libglusterfs/src/generator.py gen-fops.py
+	PYTHONPATH=$(top_srcdir)/libglusterfs/src \
+	$(PYTHON) $(srcdir)/gen-fops.py $(srcdir)/fops-tmpl.c > $@
+
+CLEANFILES = $(nodist_metadisp_la_SOURCES)
diff --git a/xlators/features/metadisp/src/backend.c b/xlators/features/metadisp/src/backend.c
new file mode 100644
index 00000000000..ee2c25bfaa7
--- /dev/null
+++ b/xlators/features/metadisp/src/backend.c
@@ -0,0 +1,45 @@
+#define GFID_STR_LEN 37
+
+#include "metadisp.h"
+
+/*
+ * backend.c
+ *
+ * functions responsible for converting user-facing paths to backend-style
+ * "/$GFID" paths.
+ */
+
+int32_t
+build_backend_loc(uuid_t gfid, loc_t *src_loc, loc_t *dst_loc)
+{
+    static uuid_t root = {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1};
+    char gfid_buf[GFID_STR_LEN + 1] = {
+        0,
+    };
+    char *path = NULL;
+
+    GF_VALIDATE_OR_GOTO("metadisp", src_loc, out);
+    GF_VALIDATE_OR_GOTO("metadisp", dst_loc, out);
+
+    loc_copy(dst_loc, src_loc);
+    memcpy(dst_loc->pargfid, root, sizeof(root));
+    GF_FREE((char *)dst_loc->path);  // we are overwriting path so nuke
+                                     // whatever loc_copy gave us
+
+    uuid_utoa_r(gfid, gfid_buf);
+
+    path = GF_CALLOC(GFID_STR_LEN + 1, sizeof(char),
+                     gf_common_mt_char);  // freed via loc_wipe
+
+    path[0] = '/';
+    strncpy(path + 1, gfid_buf, GFID_STR_LEN);
+    path[GFID_STR_LEN] = 0;
+    dst_loc->path = path;
+    if (src_loc->name)
+        dst_loc->name = strrchr(dst_loc->path, '/');
+    if (dst_loc->name)
+        dst_loc->name++;
+    return 0;
+out:
+    return -1;
+}
diff --git a/xlators/features/metadisp/src/fops-tmpl.c b/xlators/features/metadisp/src/fops-tmpl.c
new file mode 100644
index 00000000000..4385b7dd5b7
--- /dev/null
+++ b/xlators/features/metadisp/src/fops-tmpl.c
@@ -0,0 +1,10 @@
+#ifndef _CONFIG_H
+#define _CONFIG_H
+#include "config.h"
+#endif
+
+#include <glusterfs/xlator.h>
+#include "metadisp.h"
+#include "metadisp-fops.h"
+
+#pragma generate
diff --git a/xlators/features/metadisp/src/gen-fops.py b/xlators/features/metadisp/src/gen-fops.py
new file mode 100644
index 00000000000..8b5e120fdec
--- /dev/null
+++ b/xlators/features/metadisp/src/gen-fops.py
@@ -0,0 +1,160 @@
+#!/usr/bin/python
+
+import sys
+from generator import fop_subs, generate
+
+FN_METADATA_CHILD_GENERIC = """
+int32_t
+metadisp_@NAME@ (call_frame_t *frame, xlator_t *this,
+                 @LONG_ARGS@)
+{
+  METADISP_TRACE("@NAME@ metadata");
+  STACK_WIND (frame, default_@NAME@_cbk,
+              METADATA_CHILD(this), METADATA_CHILD(this)->fops->@NAME@,
+              @SHORT_ARGS@);
+  return 0;
+}
+"""
+
+FN_GENERIC_TEMPLATE = """
+int32_t
+metadisp_@NAME@ (call_frame_t *frame, xlator_t *this,
+                          @LONG_ARGS@)
+{
+  METADISP_TRACE("@NAME@ generic");
+  STACK_WIND (frame, default_@NAME@_cbk,
+                          DATA_CHILD(this), DATA_CHILD(this)->fops->@NAME@,
+                          @SHORT_ARGS@);
+  return 0;
+}
+"""
+
+FN_DATAFD_TEMPLATE = """
+int32_t
+metadisp_@NAME@ (call_frame_t *frame, xlator_t *this,
+                          @LONG_ARGS@)
+{
+  METADISP_TRACE("@NAME@ datafd");
+  xlator_t *child = NULL;
+  child = DATA_CHILD(this);
+  STACK_WIND (frame, default_@NAME@_cbk,
+                          child, child->fops->@NAME@,
+                          @SHORT_ARGS@);
+  return 0;
+}
+"""
+
+FN_DATALOC_TEMPLATE = """
+int32_t
+metadisp_@NAME@ (call_frame_t *frame, xlator_t *this,
+                          @LONG_ARGS@)
+{
+  METADISP_TRACE("@NAME@ dataloc");
+  loc_t backend_loc = {
+      0,
+  };
+  if (build_backend_loc(loc->gfid, loc, &backend_loc)) {
+      goto unwind;
+  }
+  xlator_t *child = NULL;
+  child = DATA_CHILD(this);
+  STACK_WIND (frame, default_@NAME@_cbk,
+                          child, child->fops->@NAME@,
+                          @SHORT_ARGS@);
+  return 0;
+
+unwind:
+  STACK_UNWIND_STRICT(lookup, frame, -1, EINVAL, NULL, NULL, NULL, NULL);
+  return 0;
+}
+"""
+
+FOPS_LINE_TEMPLATE = "\t.@NAME@ = metadisp_@NAME@,"
+
+skipped = [
+    "readdir",
+    "readdirp",
+    "lookup",
+    "fsync",
+    "stat",
+    "open",
+    "create",
+    "unlink",
+    "setattr",
+    # TODO: implement "inodelk",
+]
+
+
+def gen_fops():
+    done = skipped
+
+    #
+    # these are fops that wind to the DATA_CHILD
+    #
+    # NOTE: re-written in order from google doc:
+    #          https://docs.google.com/document/d/1KEwVtSNvDhs4qb63gWx2ulCp5GJjge77NGJk4p_Ms4Q
+    for name in [
+        "writev",
+        "readv",
+        "ftruncate",
+        "zerofill",
+        "discard",
+        "seek",
+        "fstat",
+    ]:
+        done = done + [name]
+        print(generate(FN_DATAFD_TEMPLATE, name, fop_subs))
+
+    for name in ["truncate"]:
+        done = done + [name]
+        print(generate(FN_DATALOC_TEMPLATE, name, fop_subs))
+
+    # these are fops that operate solely on dentries, folders,
+    # or extended attributes. Therefore, they must always
+    # wind to METADATA_CHILD and should never perform
+    # any path rewriting
+    #
+    # NOTE: re-written in order from google doc:
+    #          https://docs.google.com/document/d/1KEwVtSNvDhs4qb63gWx2ulCp5GJjge77NGJk4p_Ms4Q
+    for name in [
+        "mkdir",
+        "symlink",
+        "link",
+        "rename",
+        "mknod",
+        "opendir",
+        # "readdir,  # special-cased
+        # "readdirp, # special-cased
+        "fsyncdir",
+        # "setattr", # special-cased
+        "readlink",
+        "fentrylk",
+        "access",
+        # TODO: these wind to both,
+        # data for backend-attributes and metadata for the rest
+        "xattrop",
+        "setxattr",
+        "getxattr",
+        "removexattr",
+        "fgetxattr",
+        "fsetxattr",
+        "fremovexattr",
+    ]:
+
+        done = done + [name]
+        print(generate(FN_METADATA_CHILD_GENERIC, name, fop_subs))
+
+    print("struct xlator_fops fops = {")
+    for name in done:
+        print(generate(FOPS_LINE_TEMPLATE, name, fop_subs))
+
+    print("};")
+
+
+for l in open(sys.argv[1], "r").readlines():
+    if l.find("#pragma generate") != -1:
+        print("/* BEGIN GENERATED CODE - DO NOT MODIFY */")
+        gen_fops()
+        print("/* END GENERATED CODE */")
+    else:
+        print(l[:-1])
diff --git a/xlators/features/metadisp/src/metadisp-create.c b/xlators/features/metadisp/src/metadisp-create.c
new file mode 100644
index 00000000000..f8c9798dd59
--- /dev/null
+++ b/xlators/features/metadisp/src/metadisp-create.c
@@ -0,0 +1,101 @@
+#include "metadisp.h"
+#include <glusterfs/call-stub.h>
+
+/**
+ * Create, like stat, is a two-step process. We send a create
+ * to the METADATA_CHILD, then send another create to the DATA_CHILD.
+ *
+ * We do the metadata child first to ensure that the ACLs are enforced.
+ */
+
+int32_t
+metadisp_create_dentry_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                           int32_t op_ret, int32_t op_errno, fd_t *fd,
+                           inode_t *inode, struct iatt *buf,
+                           struct iatt *preparent, struct iatt *postparent,
+                           dict_t *xdata)
+{
+    STACK_UNWIND_STRICT(create, frame, op_ret, op_errno, fd, inode, buf,
+                        preparent, postparent, xdata);
+    return 0;
+}
+
+int32_t
+metadisp_create_resume(call_frame_t *frame, xlator_t *this, loc_t *loc,
+                       int32_t flags, mode_t mode, mode_t umask, fd_t *fd,
+                       dict_t *xdata)
+{
+    // create the backend data inode
+    STACK_WIND(frame, metadisp_create_dentry_cbk, DATA_CHILD(this),
+               DATA_CHILD(this)->fops->create, loc, flags, mode, umask, fd,
+               xdata);
+    return 0;
+}
+
+int32_t
+metadisp_create_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                    int32_t op_ret, int32_t op_errno, fd_t *fd, inode_t *inode,
+                    struct iatt *buf, struct iatt *preparent,
+                    struct iatt *postparent, dict_t *xdata)
+{
+    METADISP_TRACE("%d %d", op_ret, op_errno);
+    call_stub_t *stub = cookie;
+    if (op_ret != 0) {
+        STACK_UNWIND_STRICT(create, frame, op_ret, op_errno, fd, inode, buf,
+                            preparent, postparent, xdata);
+        return 0;
+    }
+
+    if (stub == NULL) {
+        goto unwind;
+    }
+
+    if (stub->poison) {
+        call_stub_destroy(stub);
+        return 0;
+    }
+
+    call_resume(stub);
+    return 0;
+
+unwind:
+    STACK_UNWIND_STRICT(create, frame, -1, EINVAL, NULL, NULL, NULL, NULL, NULL,
+                        NULL);
+    return 0;
+}
+
+int32_t
+metadisp_create(call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t flags,
+                mode_t mode, mode_t umask, fd_t *fd, dict_t *xdata)
+{
+    METADISP_TRACE(".");
+
+    loc_t backend_loc = {
+        0,
+    };
+    call_stub_t *stub = NULL;
+    uuid_t *gfid_req = NULL;
+
+    RESOLVE_GFID_REQ(xdata, gfid_req, out);
+
+    if (build_backend_loc(*gfid_req, loc, &backend_loc)) {
+        goto unwind;
+    }
+
+    frame->local = loc;
+
+    stub = fop_create_stub(frame, metadisp_create_resume, &backend_loc, flags,
+                           mode, umask, fd, xdata);
+
+    STACK_WIND_COOKIE(frame, metadisp_create_cbk, stub, METADATA_CHILD(this),
+                      METADATA_CHILD(this)->fops->create, loc, flags, mode,
+                      umask, fd, xdata);
+    return 0;
+
+unwind:
+    STACK_UNWIND_STRICT(create, frame, -1, EINVAL, NULL, NULL, NULL, NULL, NULL,
+                        NULL);
+    return 0;
+out:
+    return -1;
+}
diff --git a/xlators/features/metadisp/src/metadisp-fops.h b/xlators/features/metadisp/src/metadisp-fops.h
new file mode 100644
index 00000000000..56dd427cf34
--- /dev/null
+++ b/xlators/features/metadisp/src/metadisp-fops.h
@@ -0,0 +1,51 @@
+#ifndef GF_METADISP_FOPS_H_
+#define GF_METADISP_FOPS_H_
+
+#include <glusterfs/xlator.h>
+#include <glusterfs/dict.h>
+#include <glusterfs/glusterfs.h>
+
+#include <sys/types.h>
+
+/* fops in here are defined in their own file. Every other fop is just defined
+ * inline of fops.c */
+
+int
+metadisp_readdir(call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
+                 off_t off, dict_t *xdata);
+
+int
+metadisp_readdirp(call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
+                  off_t off, dict_t *dict);
+
+int
+metadisp_lookup(call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xdata);
+
+int
+metadisp_create(call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t flags,
+                mode_t mode, mode_t umask, fd_t *fd, dict_t *xdata);
+
+int
+metadisp_open(call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t flags,
+              fd_t *fd, dict_t *xdata);
+
+int
+metadisp_stat(call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xdata);
+
+int
+metadisp_inodelk(call_frame_t *frame, xlator_t *this, const char *volume,
+                 loc_t *loc, int32_t cmd, struct gf_flock *lock, dict_t *xdata);
+
+int
+metadisp_fsync(call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t flags,
+               dict_t *xdata);
+
+int
+metadisp_unlink(call_frame_t *frame, xlator_t *this, loc_t *loc, int xflag,
+                dict_t *xdata);
+
+int
+metadisp_setattr(call_frame_t *frame, xlator_t *this, loc_t *loc,
+                 struct iatt *stbuf, int32_t valid, dict_t *xdata);
+
+#endif
diff --git a/xlators/features/metadisp/src/metadisp-fsync.c b/xlators/features/metadisp/src/metadisp-fsync.c
new file mode 100644
index 00000000000..2e46fa84eac
--- /dev/null
+++ b/xlators/features/metadisp/src/metadisp-fsync.c
@@ -0,0 +1,54 @@
+
+#include "metadisp.h"
+#include <glusterfs/call-stub.h>
+
+int32_t
+metadisp_fsync_resume(call_frame_t *frame, xlator_t *this, fd_t *fd,
+                      int32_t flags, dict_t *xdata)
+{
+    STACK_WIND(frame, default_fsync_cbk, DATA_CHILD(this),
+               DATA_CHILD(this)->fops->fsync, fd, flags, xdata);
+    return 0;
+}
+
+int32_t
+metadisp_fsync_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                   int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
+                   struct iatt *postbuf, dict_t *xdata)
+{
+    call_stub_t *stub = NULL;
+    if (cookie) {
+        stub = cookie;
+    }
+
+    if (op_ret != 0) {
+        goto unwind;
+    }
+
+    if (stub->poison) {
+        call_stub_destroy(stub);
+        stub = NULL;
+        return 0;
+    }
+
+    call_resume(stub);
+    return 0;
+
+unwind:
+    if (stub) {
+        call_stub_destroy(stub);
+    }
+    STACK_UNWIND_STRICT(fsync, frame, op_ret, op_errno, prebuf, postbuf, xdata);
+    return 0;
+}
+
+int32_t
+metadisp_fsync(call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t flags,
+               dict_t *xdata)
+{
+    call_stub_t *stub = NULL;
+    stub = fop_fsync_stub(frame, metadisp_fsync_resume, fd, flags, xdata);
+    STACK_WIND_COOKIE(frame, metadisp_fsync_cbk, stub, METADATA_CHILD(this),
+                      METADATA_CHILD(this)->fops->fsync, fd, flags, xdata);
+    return 0;
+}
diff --git a/xlators/features/metadisp/src/metadisp-lookup.c b/xlators/features/metadisp/src/metadisp-lookup.c
new file mode 100644
index 00000000000..27d90c9f746
--- /dev/null
+++ b/xlators/features/metadisp/src/metadisp-lookup.c
@@ -0,0 +1,90 @@
+#include "metadisp.h"
+#include <glusterfs/call-stub.h>
+
+/**
+ * Lookup, like stat, is a two-step process for grabbing the metadata details
+ * as well as the data details.
+ */
+
+int32_t
+metadisp_backend_lookup_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                            int32_t op_ret, int32_t op_errno, inode_t *inode,
+                            struct iatt *buf, dict_t *xdata,
+                            struct iatt *postparent)
+{
+    METADISP_TRACE("backend_lookup_cbk");
+    if (op_errno == ENOENT) {
+        op_errno = ENODATA;
+        op_ret = -1;
+    }
+    STACK_UNWIND_STRICT(lookup, frame, op_ret, op_errno, inode, buf, xdata,
+                        postparent);
+    return 0;
+}
+
+int32_t
+metadisp_backend_lookup_resume(call_frame_t *frame, xlator_t *this, loc_t *loc,
+                               dict_t *xdata)
+{
+    METADISP_TRACE("backend_lookup_resume");
+    loc_t backend_loc = {
+        0,
+    };
+    if (build_backend_loc(loc->gfid, loc, &backend_loc)) {
+        goto unwind;
+    }
+
+    STACK_WIND(frame, metadisp_backend_lookup_cbk, DATA_CHILD(this),
+               DATA_CHILD(this)->fops->lookup, &backend_loc, xdata);
+    return 0;
+
+unwind:
+    STACK_UNWIND_STRICT(lookup, frame, -1, EINVAL, NULL, NULL, NULL, NULL);
+    return 0;
+}
+
+int32_t
+metadisp_lookup_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                    int32_t op_ret, int32_t op_errno, inode_t *inode,
+                    struct iatt *buf, dict_t *xdata, struct iatt *postparent)
+{
+    METADISP_TRACE("%d %d", op_ret, op_errno);
+    call_stub_t *stub = NULL;
+    stub = cookie;
+
+    if (op_ret != 0) {
+        goto unwind;
+    }
+
+    if (!IA_ISREG(buf->ia_type)) {
+        goto unwind;
+    } else if (!stub) {
+        op_errno = EINVAL;
+        goto unwind;
+    }
+
+    METADISP_TRACE("resuming stub");
+
+    // memcpy(stub->args.loc.gfid, buf->ia_gfid, sizeof(uuid_t));
+    call_resume(stub);
+    return 0;
+unwind:
+    METADISP_TRACE("unwinding %d %d", op_ret, op_errno);
+    STACK_UNWIND_STRICT(lookup, frame, op_ret, op_errno, inode, buf, xdata,
+                        postparent);
+    if (stub) {
+        call_stub_destroy(stub);
+    }
+    return 0;
+}
+
+int32_t
+metadisp_lookup(call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xdata)
+{
+    METADISP_TRACE("lookup");
+    call_stub_t *stub = NULL;
+    stub = fop_lookup_stub(frame, metadisp_backend_lookup_resume, loc, xdata);
+    STACK_WIND_COOKIE(frame, metadisp_lookup_cbk, stub, METADATA_CHILD(this),
+                      METADATA_CHILD(this)->fops->lookup, loc, xdata);
+    return 0;
+}
diff --git a/xlators/features/metadisp/src/metadisp-open.c b/xlators/features/metadisp/src/metadisp-open.c
new file mode 100644
index 00000000000..64814afe636
--- /dev/null
+++ b/xlators/features/metadisp/src/metadisp-open.c
@@ -0,0 +1,70 @@
+#include <glusterfs/call-stub.h>
+#include "metadisp.h"
+
+int32_t
+metadisp_open_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                  int32_t op_ret, int32_t op_errno, fd_t *fd, dict_t *xdata)
+{
+    METADISP_TRACE("got open results %d %d", op_ret, op_errno);
+
+    call_stub_t *stub = NULL;
+    if (cookie) {
+        stub = cookie;
+    }
+
+    if (op_ret != 0) {
+        goto unwind;
+    }
+
+    if (!stub) {
+        goto unwind;
+    }
+
+    if (stub->poison) {
+        call_stub_destroy(stub);
+        stub = NULL;
+        return 0;
+    }
+
+    call_resume(stub);
+    return 0;
+
+unwind:
+    if (stub) {
+        call_stub_destroy(stub);
+    }
+    STACK_UNWIND_STRICT(open, frame, op_ret, op_errno, fd, xdata);
+    return 0;
+}
+
+int32_t
+metadisp_open_resume(call_frame_t *frame, xlator_t *this, loc_t *loc,
+                     int32_t flags, fd_t *fd, dict_t *xdata)
+{
+    STACK_WIND_COOKIE(frame, metadisp_open_cbk, NULL, DATA_CHILD(this),
+                      DATA_CHILD(this)->fops->open, loc, flags, fd, xdata);
+    return 0;
+}
+
+int32_t
+metadisp_open(call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t flags,
+              fd_t *fd, dict_t *xdata)
+{
+    call_stub_t *stub = NULL;
+    loc_t backend_loc = {
+        0,
+    };
+
+    if (build_backend_loc(loc->gfid, loc, &backend_loc)) {
+        goto unwind;
+    }
+
+    stub = fop_open_stub(frame, metadisp_open_resume, &backend_loc, flags, fd,
+                         xdata);
+    STACK_WIND_COOKIE(frame, metadisp_open_cbk, stub, METADATA_CHILD(this),
+                      METADATA_CHILD(this)->fops->open, loc, flags, fd, xdata);
+    return 0;
+unwind:
+    STACK_UNWIND_STRICT(open, frame, -1, EINVAL, NULL, NULL);
+    return 0;
+}
diff --git a/xlators/features/metadisp/src/metadisp-readdir.c b/xlators/features/metadisp/src/metadisp-readdir.c
new file mode 100644
index 00000000000..5f840b1e88f
--- /dev/null
+++ b/xlators/features/metadisp/src/metadisp-readdir.c
@@ -0,0 +1,65 @@
+#include "metadisp.h"
+
+/**
+ * With a change to the posix xlator, readdir and readdirp are shockingly
+ * simple.
+ *
+ * The issue with separating the backend data of the files
+ * with the metadata is that readdirs must now read from multiple sources
+ * to coalesce the directory entries.
+ *
+ * The way we do this is to tell the METADATA_CHILD that when it's
+ * running readdirp, each file entry should have a stat wound to
+ * 'stat-source-of-truth'.
+ *
+ * see metadisp_stat for how it handles winds _from_posix.
+ */
+
+int32_t
+metadisp_readdir(call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
+                 off_t off, dict_t *xdata)
+{
+    METADISP_TRACE(".");
+    /*
+     * Always use readdirp, even if the original was readdir. Why? Because NFS.
+     * There are multiple translations between Gluster, UNIX, and NFS stat
+     * structures in that path. One of them uses the type etc. from the stat
+     * structure, which is only filled in by readdirp. If we use readdir, the
+     * entries do actually go all the way back to the client and are visible in
+     * getdents, but then the readdir throws them away because of the
+     * uninitialized type.
+     */
+    GF_UNUSED int32_t ret;
+    if (!xdata) {
+        xdata = dict_new();
+    }
+
+    // ret = dict_set_int32 (xdata, "list-xattr", 1);
+
+    // I'm my own source of truth!
+    ret = dict_set_static_ptr(xdata, "stat-source-of-truth", (void *)this);
+
+    STACK_WIND(frame, default_readdirp_cbk, METADATA_CHILD(this),
+               METADATA_CHILD(this)->fops->readdirp, fd, size, off, xdata);
+
+    return 0;
+}
+
+int32_t
+metadisp_readdirp(call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
+                  off_t off, dict_t *xdata)
+{
+    METADISP_TRACE(".");
+    if (!xdata) {
+        xdata = dict_new();
+    }
+    GF_UNUSED int32_t ret;
+    // ret = dict_set_int32 (xdata, "list-xattr", 1);
+
+    // I'm my own source of truth!
+    ret = dict_set_static_ptr(xdata, "stat-source-of-truth", (void *)this);
+
+    STACK_WIND(frame, default_readdirp_cbk, METADATA_CHILD(this),
+               METADATA_CHILD(this)->fops->readdirp, fd, size, off, xdata);
+    return 0;
+}
diff --git a/xlators/features/metadisp/src/metadisp-setattr.c b/xlators/features/metadisp/src/metadisp-setattr.c
new file mode 100644
index 00000000000..6991cf644f3
--- /dev/null
+++ b/xlators/features/metadisp/src/metadisp-setattr.c
@@ -0,0 +1,90 @@
+#include "metadisp.h"
+#include <glusterfs/call-stub.h>
+
+int32_t
+metadisp_backend_setattr_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                             int32_t op_ret, int32_t op_errno,
+                             struct iatt *statpre, struct iatt *statpost,
+                             dict_t *xdata)
+
+{
+    METADISP_TRACE("backend_setattr_cbk");
+    if (op_errno == ENOENT) {
+        op_errno = ENODATA;
+        op_ret = -1;
+    }
+    STACK_UNWIND_STRICT(setattr, frame, op_ret, op_errno, statpre, statpost,
+                        xdata);
+    return 0;
+}
+
+int32_t
+metadisp_backend_setattr_resume(call_frame_t *frame, xlator_t *this, loc_t *loc,
+                                struct iatt *stbuf, int32_t valid,
+                                dict_t *xdata)
+
+{
+    METADISP_TRACE("backend_setattr_resume");
+    loc_t backend_loc = {
+        0,
+    };
+    if (build_backend_loc(loc->gfid, loc, &backend_loc)) {
+        goto unwind;
+    }
+
+    STACK_WIND(frame, metadisp_backend_setattr_cbk, DATA_CHILD(this),
+               DATA_CHILD(this)->fops->setattr, &backend_loc, stbuf, valid,
+               xdata);
+    return 0;
+
+unwind:
+    STACK_UNWIND_STRICT(setattr, frame, -1, EINVAL, NULL, NULL, NULL);
+    return 0;
+}
+
+int32_t
+metadisp_setattr_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                     int32_t op_ret, int32_t op_errno, struct iatt *statpre,
+                     struct iatt *statpost, dict_t *xdata)
+{
+    METADISP_TRACE("%d %d", op_ret, op_errno);
+    call_stub_t *stub = NULL;
+    stub = cookie;
+
+    if (op_ret != 0) {
+        goto unwind;
+    }
+
+    if (!IA_ISREG(statpost->ia_type)) {
+        goto unwind;
+    } else if (!stub) {
+        op_errno = EINVAL;
+        goto unwind;
+    }
+
+    METADISP_TRACE("resuming stub");
+    call_resume(stub);
+    return 0;
+unwind:
+    METADISP_TRACE("unwinding %d %d", op_ret, op_errno);
+    STACK_UNWIND_STRICT(setattr, frame, op_ret, op_errno, statpre, statpost,
+                        xdata);
+    if (stub) {
+        call_stub_destroy(stub);
+    }
+    return 0;
+}
+
+int32_t
+metadisp_setattr(call_frame_t *frame, xlator_t *this, loc_t *loc,
+                 struct iatt *stbuf, int32_t valid, dict_t *xdata)
+{
+    METADISP_TRACE("setattr");
+    call_stub_t *stub = NULL;
+    stub = fop_setattr_stub(frame, metadisp_backend_setattr_resume, loc, stbuf,
+                            valid, xdata);
+    STACK_WIND_COOKIE(frame, metadisp_setattr_cbk, stub, METADATA_CHILD(this),
+                      METADATA_CHILD(this)->fops->setattr, loc, stbuf, valid,
+                      xdata);
+    return 0;
+}
diff --git a/xlators/features/metadisp/src/metadisp-stat.c b/xlators/features/metadisp/src/metadisp-stat.c
new file mode 100644
index 00000000000..b06d0dbcddd
--- /dev/null
+++ b/xlators/features/metadisp/src/metadisp-stat.c
@@ -0,0 +1,124 @@
+#include "metadisp.h"
+#include <glusterfs/call-stub.h>
+
+/**
+ * The stat flow in METADISP is complicated because we must
+ * do ensure a few things:
+ *    1. stat, on the path within the metadata layer,
+ *       MUST get the backend FD of the data layer.
+ *        --- we wind to the metadata layer, then the data layer.
+ *
+ *    2. the metadata layer MUST be able to ask the data
+ *       layer for stat information.
+ *        --- this is 'syncop-internal-from-posix'
+ *
+ *    3. when the metadata exists BUT the data is missing,
+ *       we MUST mark the backend file as bad and heal it.
+ */
+
+int32_t
+metadisp_stat_backend_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                          int32_t op_ret, int32_t op_errno, struct iatt *buf,
+                          dict_t *xdata)
+{
+    METADISP_TRACE("got backend stat results %d %d", op_ret, op_errno);
+    if (op_errno == ENOENT) {
+        STACK_UNWIND_STRICT(open, frame, -1, ENODATA, NULL, NULL);
+        return 0;
+    }
+    STACK_UNWIND_STRICT(stat, frame, op_ret, op_errno, buf, xdata);
+    return 0;
+}
+
+int32_t
+metadisp_stat_resume(call_frame_t *frame, xlator_t *this, loc_t *loc,
+                     dict_t *xdata)
+{
+    METADISP_TRACE("winding stat to path %s", loc->path);
+    if (gf_uuid_is_null(loc->gfid)) {
+        METADISP_TRACE("bad object, sending EUCLEAN");
+        STACK_UNWIND_STRICT(open, frame, -1, EUCLEAN, NULL, NULL);
+        return 0;
+    }
+
+    STACK_WIND(frame, metadisp_stat_backend_cbk, SECOND_CHILD(this),
+               SECOND_CHILD(this)->fops->stat, loc, xdata);
+    return 0;
+}
+
+int32_t
+metadisp_stat_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                  int32_t op_ret, int32_t op_errno, struct iatt *buf,
+                  dict_t *xdata)
+{
+    call_stub_t *stub = NULL;
+
+    METADISP_TRACE("got stat results %d %d", op_ret, op_errno);
+
+    if (cookie) {
+        stub = cookie;
+    }
+
+    if (op_ret != 0) {
+        goto unwind;
+    }
+
+    // only use the stub for the files
+    if (!IA_ISREG(buf->ia_type)) {
+        goto unwind;
+    }
+
+    if (stub->poison) {
+        call_stub_destroy(stub);
+        stub = NULL;
+        return 0;
+    }
+
+    call_resume(stub);
+    return 0;
+
+unwind:
+    if (stub) {
+        call_stub_destroy(stub);
+    }
+    STACK_UNWIND_STRICT(stat, frame, op_ret, op_errno, buf, xdata);
+    return 0;
+}
+
+int32_t
+metadisp_stat(call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xdata)
+{
+    call_stub_t *stub = NULL;
+    int32_t ret = 0;
+    loc_t backend_loc = {
+        0,
+    };
+    METADISP_FILTER_ROOT(stat, loc, xdata);
+
+    if (build_backend_loc(loc->gfid, loc, &backend_loc)) {
+        goto unwind;
+    }
+
+    if (dict_get_int32(xdata, "syncop-internal-from-posix", &ret) == 0) {
+        // if we've just been sent a stat from posix, then we know
+        // that we must send down a stat for a file to the second child.
+        //
+        // that means we can skip the stat for the first child and just
+        // send to the data disk.
+        METADISP_TRACE("got syncop-internal-from-posix");
+        STACK_WIND(frame, default_stat_cbk, DATA_CHILD(this),
+                   DATA_CHILD(this)->fops->stat, &backend_loc, xdata);
+        return 0;
+    }
+
+    // we do not know if the request is for a file, folder, etc. wind
+    // to first child to find out.
+    stub = fop_stat_stub(frame, metadisp_stat_resume, &backend_loc, xdata);
+    METADISP_TRACE("winding stat to first child %s", loc->path);
+    STACK_WIND_COOKIE(frame, metadisp_stat_cbk, stub, METADATA_CHILD(this),
+                      METADATA_CHILD(this)->fops->stat, loc, xdata);
+    return 0;
+unwind:
+    STACK_UNWIND_STRICT(stat, frame, -1, EINVAL, NULL, NULL);
+    return 0;
+}
diff --git a/xlators/features/metadisp/src/metadisp-unlink.c b/xlators/features/metadisp/src/metadisp-unlink.c
new file mode 100644
index 00000000000..1f6a8eb35ce
--- /dev/null
+++ b/xlators/features/metadisp/src/metadisp-unlink.c
@@ -0,0 +1,160 @@
+
+#include "metadisp.h"
+#include <glusterfs/call-stub.h>
+
+/**
+ * The unlink flow in metadisp is complicated because we must
+ * do ensure that UNLINK causes both the metadata objects
+ * to get removed and the data objects to get removed.
+ */
+
+int32_t
+metadisp_unlink_resume(call_frame_t *frame, xlator_t *this, loc_t *loc,
+                       int xflag, dict_t *xdata)
+{
+    METADISP_TRACE("winding backend unlink to path %s", loc->path);
+    STACK_WIND(frame, default_unlink_cbk, DATA_CHILD(this),
+               DATA_CHILD(this)->fops->unlink, loc, xflag, xdata);
+    return 0;
+}
+
+int32_t
+metadisp_unlink_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                    int32_t op_ret, int32_t op_errno, struct iatt *preparent,
+                    struct iatt *postparent, dict_t *xdata)
+{
+    METADISP_TRACE(". %d %d", op_ret, op_errno);
+
+    int ret = 0;
+    call_stub_t *stub = NULL;
+    int nlink = 0;
+
+    if (cookie) {
+        stub = cookie;
+    }
+
+    if (op_ret != 0) {
+        goto unwind;
+    }
+
+    if (stub->poison) {
+        call_stub_destroy(stub);
+        stub = NULL;
+        return 0;
+    }
+
+    ret = dict_get_uint32(xdata, GF_RESPONSE_LINK_COUNT_XDATA, &nlink);
+    if (ret != 0) {
+        op_errno = EINVAL;
+        op_ret = -1;
+        goto unwind;
+    }
+    METADISP_TRACE("frontend hardlink count %d %d", ret, nlink);
+    if (nlink > 1) {
+        goto unwind;
+    }
+
+    call_resume(stub);
+    return 0;
+
+unwind:
+    if (stub) {
+        call_stub_destroy(stub);
+    }
+    STACK_UNWIND_STRICT(unlink, frame, op_ret, op_errno, preparent, postparent,
+                        xdata);
+    return 0;
+}
+
+int32_t
+metadisp_unlink_lookup_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                           int32_t op_ret, int32_t op_errno, inode_t *inode,
+                           struct iatt *buf, dict_t *xdata,
+                           struct iatt *postparent)
+{
+    call_stub_t *stub = NULL;
+
+    if (cookie) {
+        stub = cookie;
+    }
+
+    if (op_ret != 0) {
+        goto unwind;
+    }
+
+    // fail fast on empty gfid so we don't loop forever
+    if (gf_uuid_is_null(buf->ia_gfid)) {
+        op_ret = -1;
+        op_errno = ENODATA;
+        goto unwind;
+    }
+
+    // fill gfid since the stub is incomplete
+    memcpy(stub->args.loc.gfid, buf->ia_gfid, sizeof(uuid_t));
+    memcpy(stub->args.loc.pargfid, postparent->ia_gfid, sizeof(uuid_t));
+
+    if (stub->poison) {
+        call_stub_destroy(stub);
+        stub = NULL;
+        return 0;
+    }
+
+    call_resume(stub);
+    return 0;
+
+unwind:
+    if (stub) {
+        call_stub_destroy(stub);
+    }
+    STACK_UNWIND_STRICT(unlink, frame, op_ret, op_errno, NULL, NULL, NULL);
+    return 0;
+}
+
+int32_t
+metadisp_unlink(call_frame_t *frame, xlator_t *this, loc_t *loc, int xflag,
+                dict_t *xdata)
+{
+    call_stub_t *stub = NULL;
+    loc_t backend_loc = {
+        0,
+    };
+
+    if (gf_uuid_is_null(loc->gfid)) {
+        METADISP_TRACE("winding lookup for unlink to path %s", loc->path);
+
+        // loop back to ourselves after a lookup
+        stub = fop_unlink_stub(frame, metadisp_unlink, loc, xflag, xdata);
+        STACK_WIND_COOKIE(frame, metadisp_unlink_lookup_cbk, stub,
+                          METADATA_CHILD(this),
+                          METADATA_CHILD(this)->fops->lookup, loc, xdata);
+        return 0;
+    }
+
+    if (build_backend_loc(loc->gfid, loc, &backend_loc)) {
+        goto unwind;
+    }
+
+    //
+    // ensure we get the link count on the unlink response, so we can
+    // account for hardlinks before winding to the backend.
+    // NOTE:
+    //   multiple xlators use GF_REQUEST_LINK_COUNT_XDATA. confirmation
+    //   is needed to ensure that multiple requests will work in the same
+    //   xlator stack.
+    //
+    if (!xdata) {
+        xdata = dict_new();
+    }
+    dict_set_int32(xdata, GF_REQUEST_LINK_COUNT_XDATA, 1);
+
+    METADISP_TRACE("winding frontend unlink to path %s", loc->path);
+    stub = fop_unlink_stub(frame, metadisp_unlink_resume, &backend_loc, xflag,
+                           xdata);
+
+    STACK_WIND_COOKIE(frame, metadisp_unlink_cbk, stub, METADATA_CHILD(this),
+                      METADATA_CHILD(this)->fops->unlink, loc, xflag, xdata);
+    return 0;
+unwind:
+    STACK_UNWIND_STRICT(unlink, frame, -1, EINVAL, NULL, NULL, NULL);
+    return 0;
+}
diff --git a/xlators/features/metadisp/src/metadisp.c b/xlators/features/metadisp/src/metadisp.c
new file mode 100644
index 00000000000..3c8f150cebc
--- /dev/null
+++ b/xlators/features/metadisp/src/metadisp.c
@@ -0,0 +1,46 @@
+#include <glusterfs/call-stub.h>
+
+#include "metadisp.h"
+#include "metadisp-fops.h"
+
+int32_t
+init(xlator_t *this)
+{
+    if (!this->children) {
+        gf_log(this->name, GF_LOG_ERROR,
+               "not configured with children. exiting");
+        return -1;
+    }
+
+    if (!this->parents) {
+        gf_log(this->name, GF_LOG_WARNING, "dangling volume. check volfile ");
+    }
+
+    return 0;
+}
+
+void
+fini(xlator_t *this)
+{
+    return;
+}
+
+/* defined in fops.c */
+struct xlator_fops fops;
+
+struct xlator_cbks cbks = {};
+
+struct volume_options options[] = {
+    {.key = {NULL}},
+};
+
+xlator_api_t xlator_api = {
+    .init = init,
+    .fini = fini,
+    .fops = &fops,
+    .cbks = &cbks,
+    .options = options,
+    .op_version = {1},
+    .identifier = "metadisp",
+    .category = GF_EXPERIMENTAL,
+};
diff --git a/xlators/features/metadisp/src/metadisp.h b/xlators/features/metadisp/src/metadisp.h
new file mode 100644
index 00000000000..c8fd7a13c04
--- /dev/null
+++ b/xlators/features/metadisp/src/metadisp.h
@@ -0,0 +1,45 @@
+/*
+   Copyright (c) 2013 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
+
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
+#ifndef GF_METADISP_H_
+#define GF_METADISP_H_
+
+#include <glusterfs/glusterfs.h>
+#include <glusterfs/logging.h>
+#include <glusterfs/dict.h>
+#include <glusterfs/xlator.h>
+#include <glusterfs/defaults.h>
+
+#define METADATA_CHILD(_this) FIRST_CHILD(_this)
+#define DATA_CHILD(_this) SECOND_CHILD(_this)
+
+int32_t
+build_backend_loc(uuid_t gfid, loc_t *src_loc, loc_t *dst_loc);
+
+#define METADISP_TRACE(_args...) gf_log("metadisp", GF_LOG_INFO, _args)
+
+#define METADISP_FILTER_ROOT(_op, _args...)                                    \
+    if (strcmp(loc->path, "/") == 0) {                                         \
+        STACK_WIND(frame, default_##_op##_cbk, METADATA_CHILD(this),           \
+                   METADATA_CHILD(this)->fops->_op, _args);                    \
+        return 0;                                                              \
+    }
+
+#define METADISP_FILTER_ROOT_BY_GFID(_op, _gfid, _args...)                     \
+    if (__is_root_gfid(_gfid)) {                                               \
+        STACK_WIND(frame, default_##_op##_cbk, METADATA_CHILD(this),           \
+                   METADATA_CHILD(this)->fops->_op, _args);                    \
+        return 0;                                                              \
+    }
+
+#define RESOLVE_GFID_REQ(_dict, _dest, _lbl)                                   \
+    VALIDATE_OR_GOTO(dict_get_ptr(_dict, "gfid-req", (void **)&_dest) == 0,    \
+                     _lbl)
+
+#endif /* __TEMPLATE_H__ */
diff --git a/xlators/features/namespace/Makefile.am b/xlators/features/namespace/Makefile.am
new file mode 100644
index 00000000000..a985f42a877
--- /dev/null
+++ b/xlators/features/namespace/Makefile.am
@@ -0,0 +1,3 @@
+SUBDIRS = src
+
+CLEANFILES =
diff --git a/xlators/features/namespace/src/Makefile.am b/xlators/features/namespace/src/Makefile.am
new file mode 100644
index 00000000000..e355d42cf4e
--- /dev/null
+++ b/xlators/features/namespace/src/Makefile.am
@@ -0,0 +1,17 @@
+xlator_LTLIBRARIES = namespace.la
+xlatordir = $(libdir)/glusterfs/$(PACKAGE_VERSION)/xlator/features
+
+namespace_la_LDFLAGS = -module $(GF_XLATOR_DEFAULT_LDFLAGS)
+
+namespace_la_SOURCES = namespace.c
+namespace_la_LIBADD = $(top_builddir)/libglusterfs/src/libglusterfs.la
+
+noinst_HEADERS = namespace.h
+
+AM_CPPFLAGS = $(GF_CPPFLAGS) -I$(top_srcdir)/libglusterfs/src \
+	-I$(top_srcdir)/rpc/xdr/src -I$(top_builddir)/rpc/xdr/src \
+	-I$(top_srcdir)/xlators/lib/src
+
+AM_CFLAGS = -Wall $(GF_CFLAGS)
+
+CLEANFILES =
diff --git a/xlators/features/namespace/src/namespace.c b/xlators/features/namespace/src/namespace.c
new file mode 100644
index 00000000000..86c5ebee900
--- /dev/null
+++ b/xlators/features/namespace/src/namespace.c
@@ -0,0 +1,1344 @@
+/*
+ * Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
+ * This file is part of GlusterFS.
+ *
+ * This file is licensed to you under your choice of the GNU Lesser
+ * General Public License, version 3 or any later version (LGPLv3 or
+ * later), or the GNU General Public License, version 2 (GPLv2), in all
+ * cases as published by the Free Software Foundation.
+ *
+ * xlators/features/namespace:
+ *      This translator tags each request with a namespace hash,
+ *      which then can be used in later translators to track and
+ *      throttle fops per namespace.
+ */
+
+#include <sys/types.h>
+
+#include <glusterfs/defaults.h>
+#include <glusterfs/hashfn.h>
+#include <glusterfs/logging.h>
+#include "namespace.h"
+
+/* Return codes for common path parsing functions. */
+enum _path_parse_result {
+    PATH_PARSE_RESULT_NO_PATH = 0,
+    PATH_PARSE_RESULT_FOUND = 1,
+    PATH_PARSE_RESULT_IS_GFID = 2,
+};
+
+typedef enum _path_parse_result path_parse_result_t;
+
+/* Clean up an ns_local struct. Wipe a loc (its inode is ref'd, so we're good.)
+ */
+static inline void
+ns_local_cleanup(ns_local_t *local)
+{
+    if (!local) {
+        return;
+    }
+
+    loc_wipe(&local->loc);
+    GF_FREE(local);
+}
+
+/* Create a new ns_local. We ref the inode, fake a new loc struct, and stash
+ * the stub given to us. */
+static inline ns_local_t *
+ns_local_new(call_stub_t *stub, inode_t *inode)
+{
+    ns_local_t *local = NULL;
+    loc_t loc = {
+        0,
+    };
+
+    if (!stub || !inode) {
+        goto out;
+    }
+
+    local = GF_CALLOC(1, sizeof(ns_local_t), 0);
+    if (local == NULL) {
+        goto out;
+    }
+
+    /* Set up a fake loc_t struct to give to the getxattr call. */
+    gf_uuid_copy(loc.gfid, inode->gfid);
+    loc.inode = inode_ref(inode);
+
+    /* If for some reason inode_ref() fails, then just give up. */
+    if (!loc.inode) {
+        GF_FREE(local);
+        goto out;
+    }
+
+    local->stub = stub;
+    local->loc = loc;
+
+out:
+    return local;
+}
+
+/* Try parsing a path string. If the path string is a GFID, then return
+ * with PATH_PARSE_RESULT_IS_GFID. If we have no namespace (i.e. '/') then
+ * return PATH_PARSE_RESULT_NO_PATH and set the hash to 1. Otherwise, hash the
+ * namespace and store it in the info struct. */
+static path_parse_result_t
+parse_path(ns_info_t *info, const char *path)
+{
+    int len = 0;
+    const char *ns_begin = path;
+    const char *ns_end = NULL;
+
+    if (!path || strlen(path) == 0) {
+        return PATH_PARSE_RESULT_NO_PATH;
+    }
+
+    if (path[0] == '<') {
+        return PATH_PARSE_RESULT_IS_GFID;
+    }
+
+    /* Right now we only want the top-level directory, so
+     * skip the initial '/' and read until the next '/'. */
+    while (*ns_begin == '/') {
+        ns_begin++;
+    }
+
+    /* ns_end will point to the next '/' or NULL if there is no delimiting
+     * '/' (i.e. "/directory" or the top level "/") */
+    ns_end = strchr(ns_begin, '/');
+    len = ns_end ? (ns_end - ns_begin) : strlen(ns_begin);
+
+    if (len != 0) {
+        info->hash = SuperFastHash(ns_begin, len);
+    } else {
+        /* If our substring is empty, then we can hash '/' instead.
+         * '/' is used in the namespace config for the top-level
+         * namespace. */
+        info->hash = SuperFastHash("/", 1);
+    }
+
+    info->found = _gf_true;
+    return PATH_PARSE_RESULT_FOUND;
+}
+
+/* Cache namespace info stored in the stack (info) into the inode. */
+static int
+ns_inode_ctx_put(inode_t *inode, xlator_t *this, ns_info_t *info)
+{
+    ns_info_t *cached_ns_info = NULL;
+    uint64_t ns_as_64 = 0;
+    int ret = -1;
+
+    if (!inode || !this) {
+        gf_log(this ? this->name : "namespace", GF_LOG_WARNING,
+               "Need a valid inode and xlator to cache ns_info.");
+        ret = -1;
+        goto out;
+    }
+
+    cached_ns_info = GF_CALLOC(1, sizeof(ns_info_t), 0);
+
+    /* If we've run out of memory, then return ENOMEM. */
+    if (cached_ns_info == NULL) {
+        gf_log(this->name, GF_LOG_WARNING, "No memory to cache ns_info.");
+        ret = -(ENOMEM);
+        goto out;
+    }
+
+    *cached_ns_info = *info;
+    ns_as_64 = (uint64_t)(uintptr_t)cached_ns_info;
+
+    ret = inode_ctx_put(inode, this, ns_as_64);
+
+    if (ret) {
+        goto out;
+    }
+
+    ret = 0;
+out:
+    if (ret && cached_ns_info) {
+        GF_FREE(cached_ns_info);
+    }
+
+    return ret;
+}
+
+/* Retrieve namespace info cached in the inode into the stack for use in later
+ * translators. */
+static int
+ns_inode_ctx_get(inode_t *inode, xlator_t *this, ns_info_t *info)
+{
+    ns_info_t *cached_ns_info = NULL;
+    uint64_t ns_as_64 = 0;
+    int ret = -1;
+
+    if (!inode) {
+        ret = -ENOENT;
+        goto out;
+    }
+
+    ret = inode_ctx_get(inode, this, &ns_as_64);
+
+    if (!ret) {
+        cached_ns_info = (ns_info_t *)(uintptr_t)ns_as_64;
+        *info = *cached_ns_info;
+    }
+
+out:
+    return ret;
+}
+
+/* This callback is the top of the unwind path of our attempt to get the path
+ * manually from the posix translator. We'll try to parse the path returned
+ * if it exists, then cache the hash if possible. Then just return to the
+ * default stub that we provide in the local, since there's nothing else to do
+ * once we've gotten the namespace hash. */
+int32_t
+get_path_resume_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                    int32_t op_ret, int32_t op_errno, dict_t *dict,
+                    dict_t *xdata)
+{
+    path_parse_result_t ret = PATH_PARSE_RESULT_NO_PATH;
+    call_frame_t *resume_frame = NULL;
+    ns_local_t *local = NULL;
+    call_stub_t *stub = NULL;
+    ns_info_t *info = NULL;
+    char *path = NULL;
+
+    GF_VALIDATE_OR_GOTO(this->name, frame, out);
+    local = frame->local;
+
+    GF_VALIDATE_OR_GOTO(this->name, local, out);
+    stub = local->stub;
+
+    GF_VALIDATE_OR_GOTO(this->name, stub, out);
+    /* Get the ns_info from the frame that we will eventually resume,
+     * not the frame that we're going to destroy (frame). */
+    resume_frame = stub->frame;
+
+    GF_VALIDATE_OR_GOTO(this->name, resume_frame, out);
+    GF_VALIDATE_OR_GOTO(this->name, resume_frame->root, out);
+    info = &resume_frame->root->ns_info;
+
+    GF_VALIDATE_OR_GOTO(this->name, dict, out);
+
+    /* If we get a value back for the GET_ANCESTRY_PATH_KEY, then we
+     * try to access it and parse it like a path. */
+    if (!op_ret && !dict_get_str(dict, GET_ANCESTRY_PATH_KEY, &path)) {
+        gf_log(this->name, GF_LOG_DEBUG, "G>P %s retrieved path %s",
+               uuid_utoa(local->loc.gfid), path);
+        /* Now let's parse a path, finally. */
+        ret = parse_path(info, path);
+    }
+
+    if (ret == PATH_PARSE_RESULT_FOUND) {
+        /* If we finally found namespace, then stash it. */
+        ns_inode_ctx_put(local->loc.inode, this, info);
+
+        gf_log(this->name, GF_LOG_DEBUG, "G>P %s %10u namespace found %s",
+               uuid_utoa(local->loc.inode->gfid), info->hash, path);
+    } else if (ret == PATH_PARSE_RESULT_NO_PATH) {
+        gf_log(this->name, GF_LOG_WARNING, "G>P %s has no path",
+               uuid_utoa(local->loc.inode->gfid));
+    } else if (ret == PATH_PARSE_RESULT_IS_GFID) {
+        gf_log(this->name, GF_LOG_WARNING,
+               "G>P %s winding failed, still have gfid",
+               uuid_utoa(local->loc.inode->gfid));
+    }
+
+out:
+    /* Make sure to clean up local finally. */
+
+    if (frame) {
+        frame->local = NULL;
+        STACK_DESTROY(frame->root);
+    }
+
+    if (local) {
+        ns_local_cleanup(local);
+    }
+
+    if (stub) {
+        call_resume(stub);
+    }
+
+    return 0;
+}
+
+/* This function tries first to set a namespace based on the information that
+ * it can retrieve from an `loc_t`. This includes first looking for a cached
+ * namespace in the inode, then trying to parse the path string in the `loc_t`
+ * struct. If this fails, then it will try to call inode_path. */
+static path_parse_result_t
+set_ns_from_loc(const char *fn, call_frame_t *frame, xlator_t *this, loc_t *loc)
+{
+    path_parse_result_t ret = PATH_PARSE_RESULT_NO_PATH;
+    ns_private_t *priv = (ns_private_t *)this->private;
+    ns_info_t *info = &frame->root->ns_info;
+    char *path = NULL;
+
+    info->hash = 0;
+    info->found = _gf_false;
+
+    if (!priv->tag_namespaces) {
+        return ret;
+    }
+
+    /* This is our first pass at trying to get a path. Try getting
+     * from the inode context, then from the loc's path itself. */
+    if (!loc || !loc->path || !loc->inode) {
+        ret = PATH_PARSE_RESULT_NO_PATH;
+    } else if (!ns_inode_ctx_get(loc->inode, this, info)) {
+        ret = PATH_PARSE_RESULT_FOUND;
+    } else {
+        ret = parse_path(info, loc->path);
+        gf_log(this->name, GF_LOG_DEBUG, "%s: LOC retrieved path %s", fn,
+               loc->path);
+
+        if (ret == PATH_PARSE_RESULT_FOUND) {
+            ns_inode_ctx_put(loc->inode, this, info);
+        }
+    }
+
+    /* Keep trying by calling inode_path next, making sure to copy
+    the loc's gfid into its inode if necessary. */
+    if (ret == PATH_PARSE_RESULT_IS_GFID) {
+        if (gf_uuid_is_null(loc->inode->gfid)) {
+            gf_uuid_copy(loc->inode->gfid, loc->gfid);
+        }
+
+        if (inode_path(loc->inode, NULL, &path) >= 0 && path) {
+            ret = parse_path(info, loc->path);
+            gf_log(this->name, GF_LOG_DEBUG, "%s: LOC retrieved path %s", fn,
+                   path);
+
+            if (ret == PATH_PARSE_RESULT_FOUND) {
+                ns_inode_ctx_put(loc->inode, this, info);
+            }
+        }
+
+        if (path) {
+            GF_FREE(path);
+        }
+    }
+
+    /* Report our status, and if we have a GFID, we'll eventually try a
+     * GET_ANCESTRY_PATH_KEY wind when we return from this function. */
+    if (ret == PATH_PARSE_RESULT_FOUND) {
+        gf_log(this->name, GF_LOG_DEBUG,
+               "%s: LOC %s %10u namespace found for %s", fn,
+               uuid_utoa(loc->inode->gfid), info->hash, loc->path);
+    } else if (ret == PATH_PARSE_RESULT_NO_PATH) {
+        gf_log(this->name, GF_LOG_WARNING, "%s: LOC has no path", fn);
+    } else if (ret == PATH_PARSE_RESULT_IS_GFID) {
+        /* Make sure to copy the inode's gfid for the eventual wind. */
+        if (gf_uuid_is_null(loc->inode->gfid)) {
+            gf_uuid_copy(loc->inode->gfid, loc->gfid);
+        }
+
+        gf_log(this->name, GF_LOG_DEBUG, "%s: LOC %s winding, looking for path",
+               fn, uuid_utoa(loc->inode->gfid));
+    }
+
+    return ret;
+}
+
+/* This function tries first to set a namespace based on the information that
+ * it can retrieve from an `fd_t`. This includes first looking for a cached
+ * namespace in the inode, then trying to call inode_path manually. */
+static path_parse_result_t
+set_ns_from_fd(const char *fn, call_frame_t *frame, xlator_t *this, fd_t *fd)
+{
+    path_parse_result_t ret = PATH_PARSE_RESULT_NO_PATH;
+    ns_private_t *priv = (ns_private_t *)this->private;
+    ns_info_t *info = &frame->root->ns_info;
+    char *path = NULL;
+
+    info->hash = 0;
+    info->found = _gf_false;
+
+    if (!priv->tag_namespaces) {
+        return ret;
+    }
+
+    /* This is our first pass at trying to get a path. Try getting
+     * from the inode context, then inode_path. */
+    if (!fd || !fd->inode) {
+        ret = PATH_PARSE_RESULT_NO_PATH;
+    } else if (!ns_inode_ctx_get(fd->inode, this, info)) {
+        ret = PATH_PARSE_RESULT_FOUND;
+    } else if (inode_path(fd->inode, NULL, &path) >= 0 && path) {
+        ret = parse_path(info, path);
+        gf_log(this->name, GF_LOG_DEBUG, "%s: FD  retrieved path %s", fn, path);
+
+        if (ret == PATH_PARSE_RESULT_FOUND) {
+            ns_inode_ctx_put(fd->inode, this, info);
+        }
+    }
+
+    if (path) {
+        GF_FREE(path);
+    }
+
+    /* Report our status, and if we have a GFID, we'll eventually try a
+     * GET_ANCESTRY_PATH_KEY wind when we return from this function. */
+    if (ret == PATH_PARSE_RESULT_FOUND) {
+        gf_log(this->name, GF_LOG_DEBUG, "%s: FD  %s %10u namespace found", fn,
+               uuid_utoa(fd->inode->gfid), info->hash);
+    } else if (ret == PATH_PARSE_RESULT_NO_PATH) {
+        gf_log(this->name, GF_LOG_WARNING, "%s: FD  has no path", fn);
+    } else if (ret == PATH_PARSE_RESULT_IS_GFID) {
+        gf_log(this->name, GF_LOG_DEBUG, "%s: FD  %s winding, looking for path",
+               fn, uuid_utoa(fd->inode->gfid));
+    }
+
+    return ret;
+}
+
+/* This macro does the work of winding down a call of `getxattr` in the case
+ * that we have to retrieve the path manually. It assumes that there is a label
+ * called `wind` and the existence of several basic variables (frame, this),
+ * but otherwise is general enough for any fop (fd- or loc-based.) */
+#define GET_ANCESTRY_PATH_WIND(fop, inode, args...)                            \
+    do {                                                                       \
+        ns_info_t *info = &frame->root->ns_info;                               \
+        call_frame_t *new_frame = NULL;                                        \
+        ns_local_t *local = NULL;                                              \
+        call_stub_t *stub = NULL;                                              \
+                                                                               \
+        gf_log(this->name, GF_LOG_DEBUG, "    %s winding, looking for path",   \
+               uuid_utoa(inode->gfid));                                        \
+                                                                               \
+        new_frame = create_frame(this, this->ctx->pool);                       \
+        if (!new_frame) {                                                      \
+            gf_log(this->name, GF_LOG_ERROR,                                   \
+                   "Cannot allocate new call frame.");                         \
+            goto wind;                                                         \
+        }                                                                      \
+                                                                               \
+        stub = fop_##fop##_stub(frame, default_##fop, args);                   \
+        if (!stub) {                                                           \
+            gf_log(this->name, GF_LOG_ERROR,                                   \
+                   "Cannot allocate function stub.");                          \
+            goto wind;                                                         \
+        }                                                                      \
+                                                                               \
+        new_frame->root->uid = 0;                                              \
+        new_frame->root->gid = 0;                                              \
+        /* Put a phony "not found" NS info into this call. */                  \
+        new_frame->root->ns_info = *info;                                      \
+                                                                               \
+        local = ns_local_new(stub, inode);                                     \
+        if (!local) {                                                          \
+            gf_log(this->name, GF_LOG_ERROR,                                   \
+                   "Cannot allocate function local.");                         \
+            goto wind;                                                         \
+        }                                                                      \
+                                                                               \
+        new_frame->local = local;                                              \
+        /* After allocating a new frame, a call stub (to                       \
+         * resume our current fop), and a local variables                      \
+         * struct (for our loc to getxattr and our resume                      \
+         * stub), call getxattr and unwind to get_path_resume_cbk.             \
+         */                                                                    \
+        STACK_WIND(new_frame, get_path_resume_cbk, FIRST_CHILD(this),          \
+                   FIRST_CHILD(this)->fops->getxattr, &local->loc,             \
+                   GET_ANCESTRY_PATH_KEY, NULL);                               \
+    } while (0)
+
+int32_t
+ns_rmdir(call_frame_t *frame, xlator_t *this, loc_t *loc, int xflags,
+         dict_t *xdata)
+{
+    path_parse_result_t ret = set_ns_from_loc(__FUNCTION__, frame, this, loc);
+
+    if (ret == PATH_PARSE_RESULT_IS_GFID) {
+        GET_ANCESTRY_PATH_WIND(rmdir, loc->inode, loc, xflags, xdata);
+        return 0;
+    }
+wind:
+    STACK_WIND(frame, default_rmdir_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->rmdir, loc, xflags, xdata);
+    return 0;
+}
+
+int32_t
+ns_unlink(call_frame_t *frame, xlator_t *this, loc_t *loc, int xflags,
+          dict_t *xdata)
+{
+    path_parse_result_t ret = set_ns_from_loc(__FUNCTION__, frame, this, loc);
+
+    if (ret == PATH_PARSE_RESULT_IS_GFID) {
+        GET_ANCESTRY_PATH_WIND(unlink, loc->inode, loc, xflags, xdata);
+        return 0;
+    }
+wind:
+    STACK_WIND(frame, default_unlink_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->unlink, loc, xflags, xdata);
+    return 0;
+}
+
+int32_t
+ns_rename(call_frame_t *frame, xlator_t *this, loc_t *oldloc, loc_t *newloc,
+          dict_t *xdata)
+{
+    path_parse_result_t ret = set_ns_from_loc(__FUNCTION__, frame, this,
+                                              newloc);
+
+    if (ret == PATH_PARSE_RESULT_IS_GFID) {
+        GET_ANCESTRY_PATH_WIND(rename, newloc->inode, oldloc, newloc, xdata);
+        return 0;
+    }
+wind:
+    STACK_WIND(frame, default_rename_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->rename, oldloc, newloc, xdata);
+    return 0;
+}
+
+int32_t
+ns_link(call_frame_t *frame, xlator_t *this, loc_t *oldloc, loc_t *newloc,
+        dict_t *xdata)
+{
+    path_parse_result_t ret = set_ns_from_loc(__FUNCTION__, frame, this,
+                                              newloc);
+
+    if (ret == PATH_PARSE_RESULT_IS_GFID) {
+        GET_ANCESTRY_PATH_WIND(link, newloc->inode, oldloc, newloc, xdata);
+        return 0;
+    }
+wind:
+    STACK_WIND(frame, default_link_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->link, oldloc, newloc, xdata);
+    return 0;
+}
+
+int32_t
+ns_mkdir(call_frame_t *frame, xlator_t *this, loc_t *loc, mode_t mode,
+         mode_t umask, dict_t *xdata)
+{
+    path_parse_result_t ret = set_ns_from_loc(__FUNCTION__, frame, this, loc);
+
+    if (ret == PATH_PARSE_RESULT_IS_GFID) {
+        GET_ANCESTRY_PATH_WIND(mkdir, loc->inode, loc, mode, umask, xdata);
+        return 0;
+    }
+wind:
+    STACK_WIND(frame, default_mkdir_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->mkdir, loc, mode, umask, xdata);
+    return 0;
+}
+
+int32_t
+ns_symlink(call_frame_t *frame, xlator_t *this, const char *linkname,
+           loc_t *loc, mode_t umask, dict_t *xdata)
+{
+    path_parse_result_t ret = set_ns_from_loc(__FUNCTION__, frame, this, loc);
+
+    if (ret == PATH_PARSE_RESULT_IS_GFID) {
+        GET_ANCESTRY_PATH_WIND(symlink, loc->inode, linkname, loc, umask,
+                               xdata);
+        return 0;
+    }
+wind:
+    STACK_WIND(frame, default_symlink_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->symlink, linkname, loc, umask, xdata);
+    return 0;
+}
+
+int32_t
+ns_mknod(call_frame_t *frame, xlator_t *this, loc_t *loc, mode_t mode,
+         dev_t dev, mode_t umask, dict_t *xdata)
+{
+    path_parse_result_t ret = set_ns_from_loc(__FUNCTION__, frame, this, loc);
+
+    if (ret == PATH_PARSE_RESULT_IS_GFID) {
+        GET_ANCESTRY_PATH_WIND(mknod, loc->inode, loc, mode, dev, umask, xdata);
+        return 0;
+    }
+wind:
+    STACK_WIND(frame, default_mknod_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->mknod, loc, mode, dev, umask, xdata);
+    return 0;
+}
+
+int32_t
+ns_create(call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t flags,
+          mode_t mode, mode_t umask, fd_t *fd, dict_t *xdata)
+{
+    path_parse_result_t ret = set_ns_from_loc(__FUNCTION__, frame, this, loc);
+    if (ret == PATH_PARSE_RESULT_IS_GFID) {
+        GET_ANCESTRY_PATH_WIND(create, loc->inode, loc, flags, mode, umask, fd,
+                               xdata);
+        return 0;
+    }
+wind:
+    STACK_WIND(frame, default_create_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->create, loc, flags, mode, umask, fd,
+               xdata);
+    return 0;
+}
+
+int32_t
+ns_fsetattr(call_frame_t *frame, xlator_t *this, fd_t *fd, struct iatt *stbuf,
+            int32_t valid, dict_t *xdata)
+{
+    path_parse_result_t ret = set_ns_from_fd(__FUNCTION__, frame, this, fd);
+
+    if (ret == PATH_PARSE_RESULT_IS_GFID) {
+        GET_ANCESTRY_PATH_WIND(fsetattr, fd->inode, fd, stbuf, valid, xdata);
+        return 0;
+    }
+wind:
+    STACK_WIND(frame, default_fsetattr_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->fsetattr, fd, stbuf, valid, xdata);
+    return 0;
+}
+
+int32_t
+ns_setattr(call_frame_t *frame, xlator_t *this, loc_t *loc, struct iatt *stbuf,
+           int32_t valid, dict_t *xdata)
+{
+    path_parse_result_t ret = set_ns_from_loc(__FUNCTION__, frame, this, loc);
+
+    if (ret == PATH_PARSE_RESULT_IS_GFID) {
+        GET_ANCESTRY_PATH_WIND(setattr, loc->inode, loc, stbuf, valid, xdata);
+        return 0;
+    }
+wind:
+    STACK_WIND(frame, default_setattr_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->setattr, loc, stbuf, valid, xdata);
+    return 0;
+}
+
+int32_t
+ns_fremovexattr(call_frame_t *frame, xlator_t *this, fd_t *fd, const char *name,
+                dict_t *xdata)
+{
+    path_parse_result_t ret = set_ns_from_fd(__FUNCTION__, frame, this, fd);
+
+    if (ret == PATH_PARSE_RESULT_IS_GFID) {
+        GET_ANCESTRY_PATH_WIND(fremovexattr, fd->inode, fd, name, xdata);
+        return 0;
+    }
+wind:
+    STACK_WIND(frame, default_fremovexattr_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->fremovexattr, fd, name, xdata);
+    return 0;
+}
+
+int32_t
+ns_removexattr(call_frame_t *frame, xlator_t *this, loc_t *loc,
+               const char *name, dict_t *xdata)
+{
+    path_parse_result_t ret = set_ns_from_loc(__FUNCTION__, frame, this, loc);
+
+    if (ret == PATH_PARSE_RESULT_IS_GFID) {
+        GET_ANCESTRY_PATH_WIND(removexattr, loc->inode, loc, name, xdata);
+        return 0;
+    }
+wind:
+    STACK_WIND(frame, default_removexattr_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->removexattr, loc, name, xdata);
+    return 0;
+}
+
+int32_t
+ns_setxattr(call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *dict,
+            int32_t flags, dict_t *xdata)
+{
+    path_parse_result_t ret = set_ns_from_loc(__FUNCTION__, frame, this, loc);
+
+    if (ret == PATH_PARSE_RESULT_IS_GFID) {
+        GET_ANCESTRY_PATH_WIND(setxattr, loc->inode, loc, dict, flags, xdata);
+        return 0;
+    }
+wind:
+    STACK_WIND(frame, default_setxattr_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->setxattr, loc, dict, flags, xdata);
+    return 0;
+}
+
+int32_t
+ns_fsetxattr(call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *dict,
+             int32_t flags, dict_t *xdata)
+{
+    path_parse_result_t ret = set_ns_from_fd(__FUNCTION__, frame, this, fd);
+
+    if (ret == PATH_PARSE_RESULT_IS_GFID) {
+        GET_ANCESTRY_PATH_WIND(fsetxattr, fd->inode, fd, dict, flags, xdata);
+        return 0;
+    }
+wind:
+    STACK_WIND(frame, default_fsetxattr_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->fsetxattr, fd, dict, flags, xdata);
+    return 0;
+}
+
+int32_t
+ns_truncate(call_frame_t *frame, xlator_t *this, loc_t *loc, off_t offset,
+            dict_t *xdata)
+{
+    path_parse_result_t ret = set_ns_from_loc(__FUNCTION__, frame, this, loc);
+
+    if (ret == PATH_PARSE_RESULT_IS_GFID) {
+        GET_ANCESTRY_PATH_WIND(truncate, loc->inode, loc, offset, xdata);
+        return 0;
+    }
+wind:
+    STACK_WIND(frame, default_truncate_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->truncate, loc, offset, xdata);
+    return 0;
+}
+
+int32_t
+ns_ftruncate(call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
+             dict_t *xdata)
+{
+    path_parse_result_t ret = set_ns_from_fd(__FUNCTION__, frame, this, fd);
+
+    if (ret == PATH_PARSE_RESULT_IS_GFID) {
+        GET_ANCESTRY_PATH_WIND(ftruncate, fd->inode, fd, offset, xdata);
+        return 0;
+    }
+wind:
+    STACK_WIND(frame, default_ftruncate_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->ftruncate, fd, offset, xdata);
+    return 0;
+}
+
+int32_t
+ns_writev(call_frame_t *frame, xlator_t *this, fd_t *fd, struct iovec *vector,
+          int32_t count, off_t offset, uint32_t flags, struct iobref *iobref,
+          dict_t *xdata)
+{
+    path_parse_result_t ret = set_ns_from_fd(__FUNCTION__, frame, this, fd);
+
+    if (ret == PATH_PARSE_RESULT_IS_GFID) {
+        GET_ANCESTRY_PATH_WIND(writev, fd->inode, fd, vector, count, offset,
+                               flags, iobref, xdata);
+        return 0;
+    }
+wind:
+    STACK_WIND(frame, default_writev_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->writev, fd, vector, count, offset,
+               flags, iobref, xdata);
+    return 0;
+}
+
+int32_t
+ns_lookup(call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xdata)
+{
+    path_parse_result_t ret = set_ns_from_loc(__FUNCTION__, frame, this, loc);
+
+    if (ret == PATH_PARSE_RESULT_IS_GFID) {
+        GET_ANCESTRY_PATH_WIND(lookup, loc->inode, loc, xdata);
+        return 0;
+    }
+wind:
+    STACK_WIND(frame, default_lookup_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->lookup, loc, xdata);
+    return 0;
+}
+
+int32_t
+ns_stat(call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xdata)
+{
+    path_parse_result_t ret = set_ns_from_loc(__FUNCTION__, frame, this, loc);
+
+    if (ret == PATH_PARSE_RESULT_IS_GFID) {
+        GET_ANCESTRY_PATH_WIND(stat, loc->inode, loc, xdata);
+        return 0;
+    }
+wind:
+    STACK_WIND(frame, default_stat_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->stat, loc, xdata);
+    return 0;
+}
+
+int32_t
+ns_fstat(call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *xdata)
+{
+    path_parse_result_t ret = set_ns_from_fd(__FUNCTION__, frame, this, fd);
+
+    if (ret == PATH_PARSE_RESULT_IS_GFID) {
+        GET_ANCESTRY_PATH_WIND(fstat, fd->inode, fd, xdata);
+        return 0;
+    }
+wind:
+    STACK_WIND(frame, default_fstat_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->fstat, fd, xdata);
+    return 0;
+}
+
+int32_t
+ns_readlink(call_frame_t *frame, xlator_t *this, loc_t *loc, size_t size,
+            dict_t *xdata)
+{
+    path_parse_result_t ret = set_ns_from_loc(__FUNCTION__, frame, this, loc);
+
+    if (ret == PATH_PARSE_RESULT_IS_GFID) {
+        GET_ANCESTRY_PATH_WIND(readlink, loc->inode, loc, size, xdata);
+        return 0;
+    }
+wind:
+    STACK_WIND(frame, default_readlink_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->readlink, loc, size, xdata);
+    return 0;
+}
+
+int32_t
+ns_access(call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t mask,
+          dict_t *xdata)
+{
+    path_parse_result_t ret = set_ns_from_loc(__FUNCTION__, frame, this, loc);
+
+    if (ret == PATH_PARSE_RESULT_IS_GFID) {
+        GET_ANCESTRY_PATH_WIND(access, loc->inode, loc, mask, xdata);
+        return 0;
+    }
+wind:
+    STACK_WIND(frame, default_access_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->access, loc, mask, xdata);
+    return 0;
+}
+
+int32_t
+ns_open(call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t flags,
+        fd_t *fd, dict_t *xdata)
+{
+    path_parse_result_t ret = set_ns_from_fd(__FUNCTION__, frame, this, fd);
+
+    if (ret == PATH_PARSE_RESULT_IS_GFID) {
+        GET_ANCESTRY_PATH_WIND(open, fd->inode, loc, flags, fd, xdata);
+        return 0;
+    }
+wind:
+    STACK_WIND(frame, default_open_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->open, loc, flags, fd, xdata);
+    return 0;
+}
+
+int32_t
+ns_readv(call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
+         off_t offset, uint32_t flags, dict_t *xdata)
+{
+    path_parse_result_t ret = set_ns_from_fd(__FUNCTION__, frame, this, fd);
+
+    if (ret == PATH_PARSE_RESULT_IS_GFID) {
+        GET_ANCESTRY_PATH_WIND(readv, fd->inode, fd, size, offset, flags,
+                               xdata);
+        return 0;
+    }
+wind:
+    STACK_WIND(frame, default_readv_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->readv, fd, size, offset, flags, xdata);
+    return 0;
+}
+
+int32_t
+ns_flush(call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *xdata)
+{
+    path_parse_result_t ret = set_ns_from_fd(__FUNCTION__, frame, this, fd);
+
+    if (ret == PATH_PARSE_RESULT_IS_GFID) {
+        GET_ANCESTRY_PATH_WIND(flush, fd->inode, fd, xdata);
+        return 0;
+    }
+wind:
+    STACK_WIND(frame, default_flush_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->flush, fd, xdata);
+    return 0;
+}
+
+int32_t
+ns_fsync(call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t datasync,
+         dict_t *xdata)
+{
+    path_parse_result_t ret = set_ns_from_fd(__FUNCTION__, frame, this, fd);
+
+    if (ret == PATH_PARSE_RESULT_IS_GFID) {
+        GET_ANCESTRY_PATH_WIND(fsync, fd->inode, fd, datasync, xdata);
+        return 0;
+    }
+wind:
+    STACK_WIND(frame, default_fsync_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->fsync, fd, datasync, xdata);
+    return 0;
+}
+
+int32_t
+ns_opendir(call_frame_t *frame, xlator_t *this, loc_t *loc, fd_t *fd,
+           dict_t *xdata)
+{
+    path_parse_result_t ret = set_ns_from_loc(__FUNCTION__, frame, this, loc);
+
+    if (ret == PATH_PARSE_RESULT_IS_GFID) {
+        GET_ANCESTRY_PATH_WIND(opendir, loc->inode, loc, fd, xdata);
+        return 0;
+    }
+wind:
+    STACK_WIND(frame, default_opendir_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->opendir, loc, fd, xdata);
+    return 0;
+}
+
+int32_t
+ns_fsyncdir(call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t datasync,
+            dict_t *xdata)
+
+{
+    path_parse_result_t ret = set_ns_from_fd(__FUNCTION__, frame, this, fd);
+
+    if (ret == PATH_PARSE_RESULT_IS_GFID) {
+        GET_ANCESTRY_PATH_WIND(fsyncdir, fd->inode, fd, datasync, xdata);
+        return 0;
+    }
+wind:
+    STACK_WIND(frame, default_fsyncdir_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->fsyncdir, fd, datasync, xdata);
+    return 0;
+}
+
+int32_t
+ns_rchecksum(call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
+             int32_t len, dict_t *xdata)
+{
+    path_parse_result_t ret = set_ns_from_fd(__FUNCTION__, frame, this, fd);
+
+    if (ret == PATH_PARSE_RESULT_IS_GFID) {
+        GET_ANCESTRY_PATH_WIND(rchecksum, fd->inode, fd, offset, len, xdata);
+        return 0;
+    }
+wind:
+    STACK_WIND(frame, default_rchecksum_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->rchecksum, fd, offset, len, xdata);
+    return 0;
+}
+
+int32_t
+ns_statfs(call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xdata)
+{
+    path_parse_result_t ret = set_ns_from_loc(__FUNCTION__, frame, this, loc);
+
+    if (ret == PATH_PARSE_RESULT_IS_GFID) {
+        GET_ANCESTRY_PATH_WIND(statfs, loc->inode, loc, xdata);
+        return 0;
+    }
+wind:
+    STACK_WIND(frame, default_statfs_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->statfs, loc, xdata);
+    return 0;
+}
+
+int32_t
+ns_inodelk(call_frame_t *frame, xlator_t *this, const char *volume, loc_t *loc,
+           int32_t cmd, struct gf_flock *flock, dict_t *xdata)
+{
+    path_parse_result_t ret = set_ns_from_loc(__FUNCTION__, frame, this, loc);
+
+    if (ret == PATH_PARSE_RESULT_IS_GFID) {
+        GET_ANCESTRY_PATH_WIND(inodelk, loc->inode, volume, loc, cmd, flock,
+                               xdata);
+        return 0;
+    }
+wind:
+    STACK_WIND(frame, default_inodelk_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->inodelk, volume, loc, cmd, flock,
+               xdata);
+    return 0;
+}
+
+int32_t
+ns_finodelk(call_frame_t *frame, xlator_t *this, const char *volume, fd_t *fd,
+            int32_t cmd, struct gf_flock *flock, dict_t *xdata)
+{
+    path_parse_result_t ret = set_ns_from_fd(__FUNCTION__, frame, this, fd);
+
+    if (ret == PATH_PARSE_RESULT_IS_GFID) {
+        GET_ANCESTRY_PATH_WIND(finodelk, fd->inode, volume, fd, cmd, flock,
+                               xdata);
+        return 0;
+    }
+wind:
+    STACK_WIND(frame, default_finodelk_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->finodelk, volume, fd, cmd, flock,
+               xdata);
+    return 0;
+}
+
+int32_t
+ns_entrylk(call_frame_t *frame, xlator_t *this, const char *volume, loc_t *loc,
+           const char *basename, entrylk_cmd cmd, entrylk_type type,
+           dict_t *xdata)
+{
+    path_parse_result_t ret = set_ns_from_loc(__FUNCTION__, frame, this, loc);
+
+    if (ret == PATH_PARSE_RESULT_IS_GFID) {
+        GET_ANCESTRY_PATH_WIND(entrylk, loc->inode, volume, loc, basename, cmd,
+                               type, xdata);
+        return 0;
+    }
+wind:
+    STACK_WIND(frame, default_entrylk_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->entrylk, volume, loc, basename, cmd,
+               type, xdata);
+    return 0;
+}
+
+int32_t
+ns_fentrylk(call_frame_t *frame, xlator_t *this, const char *volume, fd_t *fd,
+            const char *basename, entrylk_cmd cmd, entrylk_type type,
+            dict_t *xdata)
+{
+    path_parse_result_t ret = set_ns_from_fd(__FUNCTION__, frame, this, fd);
+
+    if (ret == PATH_PARSE_RESULT_IS_GFID) {
+        GET_ANCESTRY_PATH_WIND(fentrylk, fd->inode, volume, fd, basename, cmd,
+                               type, xdata);
+        return 0;
+    }
+wind:
+    STACK_WIND(frame, default_fentrylk_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->fentrylk, volume, fd, basename, cmd,
+               type, xdata);
+    return 0;
+}
+
+int32_t
+ns_fgetxattr(call_frame_t *frame, xlator_t *this, fd_t *fd, const char *name,
+             dict_t *xdata)
+{
+    path_parse_result_t ret = set_ns_from_fd(__FUNCTION__, frame, this, fd);
+
+    if (ret == PATH_PARSE_RESULT_IS_GFID) {
+        GET_ANCESTRY_PATH_WIND(fgetxattr, fd->inode, fd, name, xdata);
+        return 0;
+    }
+wind:
+    STACK_WIND(frame, default_fgetxattr_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->fgetxattr, fd, name, xdata);
+    return 0;
+}
+
+int32_t
+ns_getxattr(call_frame_t *frame, xlator_t *this, loc_t *loc, const char *name,
+            dict_t *xdata)
+{
+    path_parse_result_t ret = set_ns_from_loc(__FUNCTION__, frame, this, loc);
+
+    if (ret == PATH_PARSE_RESULT_IS_GFID) {
+        GET_ANCESTRY_PATH_WIND(getxattr, loc->inode, loc, name, xdata);
+        return 0;
+    }
+wind:
+    STACK_WIND(frame, default_getxattr_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->getxattr, loc, name, xdata);
+    return 0;
+}
+
+int32_t
+ns_lk(call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t cmd,
+      struct gf_flock *flock, dict_t *xdata)
+{
+    path_parse_result_t ret = set_ns_from_fd(__FUNCTION__, frame, this, fd);
+
+    if (ret == PATH_PARSE_RESULT_IS_GFID) {
+        GET_ANCESTRY_PATH_WIND(lk, fd->inode, fd, cmd, flock, xdata);
+        return 0;
+    }
+wind:
+    STACK_WIND(frame, default_lk_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->lk, fd, cmd, flock, xdata);
+    return 0;
+}
+
+int32_t
+ns_readdir(call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
+           off_t offset, dict_t *xdata)
+{
+    path_parse_result_t ret = set_ns_from_fd(__FUNCTION__, frame, this, fd);
+
+    if (ret == PATH_PARSE_RESULT_IS_GFID) {
+        GET_ANCESTRY_PATH_WIND(readdir, fd->inode, fd, size, offset, xdata);
+        return 0;
+    }
+wind:
+    STACK_WIND(frame, default_readdir_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->readdir, fd, size, offset, xdata);
+
+    return 0;
+}
+
+int32_t
+ns_readdirp(call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
+            off_t offset, dict_t *dict)
+{
+    path_parse_result_t ret = set_ns_from_fd(__FUNCTION__, frame, this, fd);
+
+    if (ret == PATH_PARSE_RESULT_IS_GFID) {
+        GET_ANCESTRY_PATH_WIND(readdirp, fd->inode, fd, size, offset, dict);
+        return 0;
+    }
+wind:
+    STACK_WIND(frame, default_readdirp_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->readdirp, fd, size, offset, dict);
+    return 0;
+}
+
+int32_t
+ns_xattrop(call_frame_t *frame, xlator_t *this, loc_t *loc,
+           gf_xattrop_flags_t flags, dict_t *dict, dict_t *xdata)
+{
+    path_parse_result_t ret = set_ns_from_loc(__FUNCTION__, frame, this, loc);
+
+    if (ret == PATH_PARSE_RESULT_IS_GFID) {
+        GET_ANCESTRY_PATH_WIND(xattrop, loc->inode, loc, flags, dict, xdata);
+        return 0;
+    }
+wind:
+    STACK_WIND(frame, default_xattrop_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->xattrop, loc, flags, dict, xdata);
+
+    return 0;
+}
+
+int32_t
+ns_fxattrop(call_frame_t *frame, xlator_t *this, fd_t *fd,
+            gf_xattrop_flags_t flags, dict_t *dict, dict_t *xdata)
+{
+    path_parse_result_t ret = set_ns_from_fd(__FUNCTION__, frame, this, fd);
+
+    if (ret == PATH_PARSE_RESULT_IS_GFID) {
+        GET_ANCESTRY_PATH_WIND(fxattrop, fd->inode, fd, flags, dict, xdata);
+        return 0;
+    }
+wind:
+    STACK_WIND(frame, default_fxattrop_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->fxattrop, fd, flags, dict, xdata);
+
+    return 0;
+}
+
+int32_t
+ns_getspec(call_frame_t *frame, xlator_t *this, const char *key, int32_t flag)
+{
+    STACK_WIND(frame, default_getspec_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->getspec, key, flag);
+    return 0;
+}
+
+int32_t
+ns_fallocate(call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t keep_size,
+             off_t offset, size_t len, dict_t *xdata)
+{
+    path_parse_result_t ret = set_ns_from_fd(__FUNCTION__, frame, this, fd);
+
+    if (ret == PATH_PARSE_RESULT_IS_GFID) {
+        GET_ANCESTRY_PATH_WIND(fallocate, fd->inode, fd, keep_size, offset, len,
+                               xdata);
+        return 0;
+    }
+wind:
+    STACK_WIND(frame, default_fallocate_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->fallocate, fd, keep_size, offset, len,
+               xdata);
+    return 0;
+}
+
+int32_t
+ns_discard(call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
+           size_t len, dict_t *xdata)
+{
+    path_parse_result_t ret = set_ns_from_fd(__FUNCTION__, frame, this, fd);
+
+    if (ret == PATH_PARSE_RESULT_IS_GFID) {
+        GET_ANCESTRY_PATH_WIND(discard, fd->inode, fd, offset, len, xdata);
+        return 0;
+    }
+wind:
+    STACK_WIND(frame, default_discard_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->discard, fd, offset, len, xdata);
+    return 0;
+}
+
+int32_t
+ns_zerofill(call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
+            off_t len, dict_t *xdata)
+{
+    path_parse_result_t ret = set_ns_from_fd(__FUNCTION__, frame, this, fd);
+
+    if (ret == PATH_PARSE_RESULT_IS_GFID) {
+        GET_ANCESTRY_PATH_WIND(zerofill, fd->inode, fd, offset, len, xdata);
+        return 0;
+    }
+wind:
+    STACK_WIND(frame, default_zerofill_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->zerofill, fd, offset, len, xdata);
+    return 0;
+}
+
+int
+ns_forget(xlator_t *this, inode_t *inode)
+{
+    uint64_t ns_as_64 = 0;
+    ns_info_t *info = NULL;
+
+    inode_ctx_del(inode, this, &ns_as_64);
+
+    if (!ns_as_64) {
+        return 0;
+    }
+
+    info = (ns_info_t *)(uintptr_t)ns_as_64;
+    GF_FREE(info);
+
+    return 0;
+}
+
+int32_t
+init(xlator_t *this)
+{
+    int32_t ret = -1;
+    ns_private_t *priv = NULL;
+
+    GF_VALIDATE_OR_GOTO(GF_NAMESPACE, this, out);
+
+    if (!this->children || this->children->next) {
+        gf_log(this->name, GF_LOG_ERROR,
+               "translator needs a single subvolume.");
+        goto out;
+    }
+
+    if (!this->parents) {
+        gf_log(this->name, GF_LOG_ERROR,
+               "dangling volume. please check volfile.");
+        goto out;
+    }
+
+    priv = GF_CALLOC(1, sizeof(ns_private_t), 0);
+
+    if (!priv) {
+        gf_log(this->name, GF_LOG_ERROR, "Can't allocate ns_priv structure.");
+        goto out;
+    }
+
+    GF_OPTION_INIT("tag-namespaces", priv->tag_namespaces, bool, out);
+
+    gf_log(this->name, GF_LOG_INFO, "Namespace xlator loaded");
+    this->private = priv;
+    ret = 0;
+
+out:
+    if (ret) {
+        GF_FREE(priv);
+    }
+
+    return ret;
+}
+
+void
+fini(xlator_t *this)
+{
+    GF_FREE(this->private);
+}
+
+int
+reconfigure(xlator_t *this, dict_t *options)
+{
+    int ret = -1;
+    ns_private_t *priv = NULL;
+
+    GF_VALIDATE_OR_GOTO(this->name, this->private, out);
+    GF_VALIDATE_OR_GOTO(this->name, options, out);
+
+    priv = (ns_private_t *)this->private;
+
+    GF_OPTION_RECONF("tag-namespaces", priv->tag_namespaces, options, bool,
+                     out);
+
+    ret = 0;
+out:
+    return ret;
+}
+
+struct xlator_fops fops = {
+    .lookup = ns_lookup,
+    .stat = ns_stat,
+    .fstat = ns_fstat,
+    .truncate = ns_truncate,
+    .ftruncate = ns_ftruncate,
+    .access = ns_access,
+    .readlink = ns_readlink,
+    .mknod = ns_mknod,
+    .mkdir = ns_mkdir,
+    .unlink = ns_unlink,
+    .rmdir = ns_rmdir,
+    .symlink = ns_symlink,
+    .rename = ns_rename,
+    .link = ns_link,
+    .create = ns_create,
+    .open = ns_open,
+    .readv = ns_readv,
+    .writev = ns_writev,
+    .flush = ns_flush,
+    .fsync = ns_fsync,
+    .opendir = ns_opendir,
+    .readdir = ns_readdir,
+    .readdirp = ns_readdirp,
+    .fsyncdir = ns_fsyncdir,
+    .statfs = ns_statfs,
+    .setxattr = ns_setxattr,
+    .getxattr = ns_getxattr,
+    .fsetxattr = ns_fsetxattr,
+    .fgetxattr = ns_fgetxattr,
+    .removexattr = ns_removexattr,
+    .fremovexattr = ns_fremovexattr,
+    .lk = ns_lk,
+    .inodelk = ns_inodelk,
+    .finodelk = ns_finodelk,
+    .entrylk = ns_entrylk,
+    .fentrylk = ns_fentrylk,
+    .rchecksum = ns_rchecksum,
+    .xattrop = ns_xattrop,
+    .fxattrop = ns_fxattrop,
+    .setattr = ns_setattr,
+    .fsetattr = ns_fsetattr,
+    .getspec = ns_getspec,
+    .fallocate = ns_fallocate,
+    .discard = ns_discard,
+    .zerofill = ns_zerofill,
+};
+
+struct xlator_cbks cbks = {
+    .forget = ns_forget,
+};
+
+struct xlator_dumpops dumpops;
+
+struct volume_options options[] = {
+    {
+        .key = {"tag-namespaces"},
+        .type = GF_OPTION_TYPE_BOOL,
+        .default_value = "off",
+        .description = "This option enables this translator's functionality "
+                       "that tags every fop with a namespace hash for later "
+                       "throttling, stats collection, logging, etc.",
+        .op_version = {GD_OP_VERSION_4_1_0},
+        .tags = {"namespace"},
+        .flags = OPT_FLAG_SETTABLE | OPT_FLAG_CLIENT_OPT | OPT_FLAG_DOC,
+    },
+    {.key = {NULL}},
+};
+
+xlator_api_t xlator_api = {
+    .init = init,
+    .fini = fini,
+    .reconfigure = reconfigure,
+    .op_version = {GD_OP_VERSION_3_12_0},
+    .dumpops = &dumpops,
+    .fops = &fops,
+    .cbks = &cbks,
+    .options = options,
+    .identifier = "namespace",
+    .category = GF_TECH_PREVIEW,
+};
diff --git a/xlators/features/namespace/src/namespace.h b/xlators/features/namespace/src/namespace.h
new file mode 100644
index 00000000000..3a9b84d6426
--- /dev/null
+++ b/xlators/features/namespace/src/namespace.h
@@ -0,0 +1,23 @@
+#ifndef __NAMESPACE_H__
+#define __NAMESPACE_H__
+
+#ifndef _CONFIG_H
+#define _CONFIG_H
+#include "config.h"
+#endif
+
+#include <glusterfs/xlator.h>
+#include <glusterfs/call-stub.h>
+
+#define GF_NAMESPACE "namespace"
+
+typedef struct {
+    gf_boolean_t tag_namespaces;
+} ns_private_t;
+
+typedef struct {
+    loc_t loc;         /* We store a "fake" loc_t for the getxattr wind. */
+    call_stub_t *stub; /* A stub back to the function we're resuming. */
+} ns_local_t;
+
+#endif /* __NAMESPACE_H__ */
diff --git a/xlators/features/path-convertor/Makefile.am b/xlators/features/path-convertor/Makefile.am
deleted file mode 100644
index d471a3f9243..00000000000
--- a/xlators/features/path-convertor/Makefile.am
+++ /dev/null
@@ -1,3 +0,0 @@
-SUBDIRS = src
-
-CLEANFILES = 
diff --git a/xlators/features/path-convertor/src/Makefile.am b/xlators/features/path-convertor/src/Makefile.am
deleted file mode 100644
index 393a7bd089c..00000000000
--- a/xlators/features/path-convertor/src/Makefile.am
+++ /dev/null
@@ -1,15 +0,0 @@
-
-xlator_LTLIBRARIES = path-converter.la
-xlatordir = $(libdir)/glusterfs/$(PACKAGE_VERSION)/xlator/testing/features
-
-path_converter_la_LDFLAGS = -module -avoid-version 
-
-path_converter_la_SOURCES = path.c
-path_converter_la_LIBADD = $(top_builddir)/libglusterfs/src/libglusterfs.la 
-
-AM_CPPFLAGS = $(GF_CPPFLAGS) -I$(top_srcdir)/libglusterfs/src
-
-AM_CFLAGS = -Wall $(GF_CFLAGS)
-
-CLEANFILES = 
-
diff --git a/xlators/features/path-convertor/src/path-mem-types.h b/xlators/features/path-convertor/src/path-mem-types.h
deleted file mode 100644
index 77ada8d537a..00000000000
--- a/xlators/features/path-convertor/src/path-mem-types.h
+++ /dev/null
@@ -1,22 +0,0 @@
-/*
-   Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
-   This file is part of GlusterFS.
-
-   This file is licensed to you under your choice of the GNU Lesser
-   General Public License, version 3 or any later version (LGPLv3 or
-   later), or the GNU General Public License, version 2 (GPLv2), in all
-   cases as published by the Free Software Foundation.
-*/
-#ifndef __PATH_MEM_TYPES_H__
-#define __PATH_MEM_TYPES_H__
-
-#include "mem-types.h"
-
-enum gf_path_mem_types_ {
-        gf_path_mt_path_private_t = gf_common_mt_end + 1,
-        gf_path_mt_char,
-        gf_path_mt_regex_t,
-        gf_path_mt_end
-};
-#endif
-
diff --git a/xlators/features/path-convertor/src/path.c b/xlators/features/path-convertor/src/path.c
deleted file mode 100644
index 5c52e0a8d53..00000000000
--- a/xlators/features/path-convertor/src/path.c
+++ /dev/null
@@ -1,1228 +0,0 @@
-/*
-   Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
-   This file is part of GlusterFS.
-
-   This file is licensed to you under your choice of the GNU Lesser
-   General Public License, version 3 or any later version (LGPLv3 or
-   later), or the GNU General Public License, version 2 (GPLv2), in all
-   cases as published by the Free Software Foundation.
-*/
-/* TODO: add gf_log to all the cases returning errors */
-
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
-
-/**
- * xlators/features/path-translator:
- *    This translator converts the path it gets into user specified targets.
- */
-
-#include <sys/types.h>
-#include <regex.h>
-#include <time.h>
-#include <errno.h>
-#include "glusterfs.h"
-#include "xlator.h"
-#include "path-mem-types.h"
-
-typedef struct path_private
-{
-	int32_t this_len;
-	int32_t start_off;
-	int32_t end_off;
-	char *this;
-	char *that;
-	char *path;
-	regex_t *preg;
-} path_private_t;
-
-static char *
-name_this_to_that (xlator_t *xl, const char *path, const char *name)
-{
-	path_private_t *priv = xl->private;
-	char priv_path[PATH_MAX] = {0,};
-	char *tmp_name = NULL;
-	int32_t path_len = strlen (path);
-	int32_t name_len = strlen (name) - ZR_FILE_CONTENT_STRLEN;
-	int32_t total_len = path_len + name_len;
-	int32_t i = 0, j = 0;
-
-	if (path_len >= priv->end_off)
-		return (char *)name;
-
-	if (priv->end_off && (total_len > priv->end_off)) {
-		j = priv->start_off;
-		tmp_name = GF_CALLOC (1, (total_len +
-                                          ZR_FILE_CONTENT_STRLEN),
-                                      gf_path_mt_char);
-		ERR_ABORT (tmp_name);
-
-		/* Get the complete path for the file first */
-		strcpy (tmp_name, path);
-		strcat (tmp_name, name + ZR_FILE_CONTENT_STRLEN);
-
-		strncpy (priv_path, tmp_name, priv->start_off);
-		for (i = priv->start_off; i < priv->end_off; i++) {
-			if (tmp_name[i] == '/')
-				continue;
-			priv_path[j++] = tmp_name[i];
-		}
-		memcpy ((priv_path + j), 
-			(tmp_name + priv->end_off), 
-			(total_len - priv->end_off));
-		priv_path[(total_len - (priv->end_off - j))] = '\0';
-
-		strcpy (tmp_name, ZR_FILE_CONTENT_STR);
-		strcat (tmp_name, priv_path);
-
-		return tmp_name;
-	}
-
-	return (char *)name;
-}
-
-/* This function should return 
- *  NULL - 
- *  converted path - if path match
- *  same path - if it doesn't match
- */
-static char *
-path_this_to_that (xlator_t *xl, const char *path)
-{
-	path_private_t *priv = xl->private;
-	char *priv_path = NULL;
-	int32_t path_len = strlen (path);
-	int32_t i = 0, j = 0;
-
-	if (priv->end_off && (path_len > priv->start_off)) {
-		priv_path = GF_CALLOC (1, path_len, gf_path_mt_char);
-		ERR_ABORT (priv_path);
-
-		if (priv->start_off && (path_len > priv->start_off))
-			memcpy (priv_path, path, priv->start_off);
-		if (path_len > priv->end_off) {
-			j = priv->start_off;
-			for (i = priv->start_off; i < priv->end_off; i++) {
-				if (path[i] == '/')
-					continue;
-				priv_path[j++] = path[i];
-			}
-			memcpy ((priv_path + j), 
-				(path + priv->end_off), 
-				(path_len - priv->end_off));
-			priv_path[(path_len - (priv->end_off - j))] = '\0';
-		}
-		return priv_path;
-	}
-	return (char *)path;
-}
-
-int32_t 
-path_create_cbk (call_frame_t *frame,
-		 void *cookie,
-		 xlator_t *this,
-		 int32_t op_ret,
-		 int32_t op_errno,
-		 fd_t *fd,
-		 inode_t *inode,
-		 struct iatt *buf,
-                 struct iatt *preparent,
-                 struct iatt *postparent)
-{
-	STACK_UNWIND (frame, op_ret, op_errno, fd, inode, buf);
-	return 0;
-}
-
-int32_t 
-path_open_cbk (call_frame_t *frame,
-	       void *cookie,
-	       xlator_t *this,
-	       int32_t op_ret,
-	       int32_t op_errno,
-	       fd_t *fd)
-{
-	STACK_UNWIND (frame, op_ret, op_errno, fd);
-	return 0;
-}
-
-int32_t 
-path_getdents_cbk (call_frame_t *frame,
-		   void *cookie,
-		   xlator_t *this,
-		   int32_t op_ret,
-		   int32_t op_errno,
-		   dir_entry_t *entries,
-		   int32_t count)
-{
-	STACK_UNWIND (frame, op_ret, op_errno, entries, count);
-	return 0;
-}
-
-int32_t 
-path_readdir_cbk (call_frame_t *frame,
-		  void *cookie,
-		  xlator_t *this,
-		  int32_t op_ret,
-		  int32_t op_errno,
-		  gf_dirent_t *buf)
-{
-	STACK_UNWIND (frame, op_ret, op_errno, buf);
-	return 0;
-}
-
-
-int32_t 
-path_readlink_cbk (call_frame_t *frame,
-		   void *cookie,
-		   xlator_t *this,
-		   int32_t op_ret,
-		   int32_t op_errno,
-		   const char *buf,
-                   struct iatt *sbuf)
-{
-	STACK_UNWIND (frame, op_ret, op_errno, buf, sbuf);
-	return 0;
-}
-
-int32_t 
-path_lookup_cbk (call_frame_t *frame,
-		 void *cookie,
-		 xlator_t *this,
-		 int32_t op_ret,
-		 int32_t op_errno,
-		 inode_t *inode,
-		 struct iatt *buf,
-		 dict_t *xattr,
-                 struct iatt *postparent)
-{
-	STACK_UNWIND (frame, op_ret, op_errno, inode, buf, xattr);
-	return 0;
-}
-
-
-int32_t 
-path_symlink_cbk (call_frame_t *frame,
-		  void *cookie,
-		  xlator_t *this,
-		  int32_t op_ret,
-		  int32_t op_errno,
-		  inode_t *inode,
-                  struct iatt *buf,
-                  struct iatt *preparent,
-                  struct iatt *postparent)
-{
-	STACK_UNWIND (frame, op_ret, op_errno, inode, buf);
-	return 0;
-}
-
-int32_t 
-path_mknod_cbk (call_frame_t *frame,
-		void *cookie,
-		xlator_t *this,
-		int32_t op_ret,
-		int32_t op_errno,
-		inode_t *inode,
-                struct iatt *buf,
-                struct iatt *preparent,
-                struct iatt *postparent)
-{
-	STACK_UNWIND (frame, op_ret, op_errno, inode, buf);
-	return 0;
-}
-  
-
-int32_t 
-path_mkdir_cbk (call_frame_t *frame,
-		void *cookie,
-		xlator_t *this,
-		int32_t op_ret,
-		int32_t op_errno,
-		inode_t *inode,
-                struct iatt *buf,
-                struct iatt *preparent,
-                struct iatt *postparent)
-{
-	STACK_UNWIND (frame, op_ret, op_errno, inode, buf);
-	return 0;
-}
-  
-int32_t 
-path_link_cbk (call_frame_t *frame,
-	       void *cookie,
-	       xlator_t *this,
-	       int32_t op_ret,
-	       int32_t op_errno,
-	       inode_t *inode,
-               struct iatt *buf,
-               struct iatt *preparent,
-               struct iatt *postparent)
-{
-	STACK_UNWIND (frame, op_ret, op_errno, inode, buf);
-	return 0;
-}
-
-int32_t 
-path_opendir_cbk (call_frame_t *frame,
-		  void *cookie,
-		  xlator_t *this,
-		  int32_t op_ret,
-		  int32_t op_errno,
-		  fd_t *fd)
-{
-	STACK_UNWIND (frame, op_ret, op_errno, fd);
-	return 0;
-}
-
-
-int32_t
-path_rename_buf_cbk (call_frame_t *frame,
-		     void *cookie,
-		     xlator_t *this,
-		     int32_t op_ret,
-		     int32_t op_errno,
-		     struct iatt *buf,
-                     struct iatt *preoldparent,
-                     struct iatt *postoldparent,
-                     struct iatt *prenewparent,
-                     struct iatt *postnewparent)
-{
-	STACK_UNWIND (frame, op_ret, op_errno, buf);
-	return 0;
-}
-
-
-
-int32_t
-path_common_buf_cbk (call_frame_t *frame,
-		     void *cookie,
-		     xlator_t *this,
-		     int32_t op_ret,
-		     int32_t op_errno,
-		     struct iatt *buf)
-{
-	STACK_UNWIND (frame, op_ret, op_errno, buf);
-	return 0;
-}
-
-int32_t 
-path_common_dict_cbk (call_frame_t *frame,
-		      void *cookie,
-		      xlator_t *this,
-		      int32_t op_ret,
-		      int32_t op_errno,
-		      dict_t *dict)
-{
-	STACK_UNWIND (frame, op_ret, op_errno, dict);
-	return 0;
-}
-
-int32_t 
-path_common_remove_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                        int32_t op_ret, int32_t op_errno,struct iatt *preparent,
-                        struct iatt *postparent)
-{
-	STACK_UNWIND (frame, op_ret, op_errno);
-	return 0;
-}
-
-int32_t
-path_truncate_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                   int32_t op_ret, int32_t op_errno,struct iatt *prebuf,
-                   struct iatt *postbuf)
-{
-	STACK_UNWIND (frame, op_ret, op_errno, prebuf, postbuf);
-	return 0;
-}
-
-
-int32_t
-path_common_cbk (call_frame_t *frame,
-		 void *cookie,
-		 xlator_t *this,
-		 int32_t op_ret,
-		 int32_t op_errno)
-{
-	STACK_UNWIND (frame, op_ret, op_errno);
-	return 0;
-}
-
-/* */
-int32_t 
-path_lookup (call_frame_t *frame,
-	     xlator_t *this,
-	     loc_t *loc,
-	     dict_t *xattr_req)
-{
-	char *loc_path = (char *)loc->path;
-	char *tmp_path = NULL;
-	
-	if (!(tmp_path = path_this_to_that (this, loc->path))) {
-		STACK_UNWIND (frame, -1, ENOENT, NULL, NULL);
-		return 0;
-	}
-	loc->path = tmp_path;
-
-	STACK_WIND (frame, path_lookup_cbk,
-		    FIRST_CHILD(this), 
-		    FIRST_CHILD(this)->fops->lookup, 
-		    loc, xattr_req);
-
-	loc->path = loc_path;	
-	if (tmp_path != loc_path)
-		GF_FREE (tmp_path);
-
-	return 0;
-}
-
-int32_t 
-path_stat (call_frame_t *frame,
-	   xlator_t *this,
-	   loc_t *loc)
-{
-	char *loc_path = (char *)loc->path;
-	char *tmp_path = NULL;
-	
-	if (!(tmp_path = path_this_to_that (this, loc->path))) {
-		STACK_UNWIND (frame, -1, ENOENT, NULL, NULL);
-		return 0;
-	}
-	loc->path = tmp_path;
-
-	STACK_WIND (frame, 
-		    path_common_buf_cbk, 
-		    FIRST_CHILD(this), 
-		    FIRST_CHILD(this)->fops->stat, 
-		    loc);
-  
-	loc->path = loc_path;	
-	if (tmp_path != loc_path)
-		GF_FREE (tmp_path);
-
-	return 0;
-}
-
-int32_t 
-path_readlink (call_frame_t *frame,
-	       xlator_t *this,
-	       loc_t *loc,
-	       size_t size)
-{
-	char *loc_path = (char *)loc->path;
-	char *tmp_path = NULL;
-	
-	if (!(tmp_path = path_this_to_that (this, loc->path))) {
-		STACK_UNWIND (frame, -1, ENOENT, NULL, NULL);
-		return 0;
-	}
-	loc->path = tmp_path;
-
-	STACK_WIND (frame, 
-		    path_readlink_cbk,
-		    FIRST_CHILD(this), 
-		    FIRST_CHILD(this)->fops->readlink, 
-		    loc, 
-		    size);
-  
-	loc->path = loc_path;	
-	if (tmp_path != loc_path)
-		GF_FREE (tmp_path);
-
-	return 0;
-}
-
-int32_t 
-path_mknod (call_frame_t *frame,
-	    xlator_t *this,
-	    loc_t *loc,
-	    mode_t mode,
-	    dev_t dev)
-{
-	char *loc_path = (char *)loc->path;
-	char *tmp_path = NULL;
-	
-	if (!(tmp_path = path_this_to_that (this, loc->path))) {
-		STACK_UNWIND (frame, -1, ENOENT, NULL, NULL);
-		return 0;
-	}
-	loc->path = tmp_path;
-
-	STACK_WIND (frame, 
-		    path_mknod_cbk,
-		    FIRST_CHILD(this), 
-		    FIRST_CHILD(this)->fops->mknod, 
-		    loc, 
-		    mode, 
-		    dev);
-
-	loc->path = loc_path;	
-	if (tmp_path != loc_path)
-		GF_FREE (tmp_path);
-
-	return 0;
-}
-
-int32_t 
-path_mkdir (call_frame_t *frame,
-	    xlator_t *this,
-	    loc_t *loc,
-	    mode_t mode)
-{
-	char *loc_path = (char *)loc->path;
-	char *tmp_path = NULL;
-	
-	if (!(tmp_path = path_this_to_that (this, loc->path))) {
-		STACK_UNWIND (frame, -1, ENOENT, NULL, NULL);
-		return 0;
-	}
-	loc->path = tmp_path;
-
-	STACK_WIND (frame, 
-		    path_mkdir_cbk,
-		    FIRST_CHILD(this), 
-		    FIRST_CHILD(this)->fops->mkdir, 
-		    loc, 
-		    mode);
-
-	loc->path = loc_path;	
-	if (tmp_path != loc_path)
-		GF_FREE (tmp_path);
-
-	return 0;
-}
-
-int32_t 
-path_unlink (call_frame_t *frame,
-	     xlator_t *this,
-	     loc_t *loc)
-{
-	char *loc_path = (char *)loc->path;
-	char *tmp_path = NULL;
-	
-	if (!(tmp_path = path_this_to_that (this, loc->path))) {
-		STACK_UNWIND (frame, -1, ENOENT, NULL, NULL);
-		return 0;
-	}
-	loc->path = tmp_path;
-
-	STACK_WIND (frame, 
-		    path_common_remove_cbk,
-		    FIRST_CHILD(this), 
-		    FIRST_CHILD(this)->fops->unlink, 
-		    loc);
-
-	loc->path = loc_path;	
-	if (tmp_path != loc_path)
-		GF_FREE (tmp_path);
-
-	return 0;
-}
-
-int32_t 
-path_rmdir (call_frame_t *frame,
-	    xlator_t *this,
-	    loc_t *loc)
-{
-	char *loc_path = (char *)loc->path;
-	char *tmp_path = NULL;
-	
-	if (!(tmp_path = path_this_to_that (this, loc->path))) {
-		STACK_UNWIND (frame, -1, ENOENT, NULL, NULL);
-		return 0;
-	}
-	loc->path = tmp_path;
-
-	STACK_WIND (frame, 
-		    path_common_remove_cbk,
-		    FIRST_CHILD(this), 
-		    FIRST_CHILD(this)->fops->rmdir, 
-		    loc);
-  
-	loc->path = loc_path;	
-	if (tmp_path != loc_path)
-		GF_FREE (tmp_path);
-
-	return 0;
-}
-
-int32_t 
-path_symlink (call_frame_t *frame,
-	      xlator_t *this,
-	      const char *linkpath,
-	      loc_t *loc)
-{
-	char *loc_path = (char *)loc->path;
-	char *tmp_path = NULL;
-	
-	if (!(tmp_path = path_this_to_that (this, loc->path))) {
-		STACK_UNWIND (frame, -1, ENOENT, NULL, NULL);
-		return 0;
-	}
-	loc->path = tmp_path;
-
-	STACK_WIND (frame, 
-		    path_symlink_cbk,
-		    FIRST_CHILD(this), 
-		    FIRST_CHILD(this)->fops->symlink, 
-		    linkpath,
-		    loc);
-  
-	loc->path = loc_path;	
-	if (tmp_path != loc_path)
-		GF_FREE (tmp_path);
-
-	return 0;
-}
-
-int32_t 
-path_rename (call_frame_t *frame,
-	     xlator_t *this,
-	     loc_t *oldloc,
-	     loc_t *newloc)
-{  
-	char *oldloc_path = (char *)oldloc->path;
-	char *tmp_oldloc_path = NULL;
-
-	char *newloc_path = (char *)newloc->path;
-	char *tmp_newloc_path = NULL;
-	
-	if (!(tmp_oldloc_path = path_this_to_that (this, oldloc->path))) {
-		STACK_UNWIND (frame, -1, ENOENT, NULL, NULL);
-		return 0;
-	}
-	oldloc->path = tmp_oldloc_path;
-
-	if (!(tmp_newloc_path = path_this_to_that (this, newloc->path))) {
-		STACK_UNWIND (frame, -1, ENOENT, NULL, NULL);
-		return 0;
-	}
-	newloc->path = tmp_newloc_path;
-
-	STACK_WIND (frame, 
-		    path_rename_buf_cbk,
-		    FIRST_CHILD(this), 
-		    FIRST_CHILD(this)->fops->rename, 
-		    oldloc,
-		    newloc);
-  
-	oldloc->path = oldloc_path;	
-	if (tmp_oldloc_path != oldloc_path)
-		GF_FREE (tmp_oldloc_path);
-
-	newloc->path = newloc_path;	
-	if (tmp_newloc_path != newloc_path)
-		GF_FREE (tmp_newloc_path);
-
-	return 0;
-}
-
-int32_t 
-path_link (call_frame_t *frame,
-	   xlator_t *this,
-	   loc_t *oldloc,
-	   loc_t *newloc)
-{
-	char *oldloc_path = (char *)oldloc->path;
-	char *tmp_oldloc_path = NULL;
-
-	char *newloc_path = (char *)newloc->path;
-	char *tmp_newloc_path = NULL;
-	
-	if (!(tmp_oldloc_path = path_this_to_that (this, oldloc->path))) {
-		STACK_UNWIND (frame, -1, ENOENT, NULL, NULL);
-		return 0;
-	}
-	oldloc->path = tmp_oldloc_path;
-
-	if (!(tmp_newloc_path = path_this_to_that (this, newloc->path))) {
-		STACK_UNWIND (frame, -1, ENOENT, NULL, NULL);
-		return 0;
-	}
-	newloc->path = tmp_newloc_path;
-
-	STACK_WIND (frame, 
-		    path_link_cbk, 
-		    FIRST_CHILD(this), 
-		    FIRST_CHILD(this)->fops->link, 
-		    oldloc, 
-		    newloc);
-
-	oldloc->path = oldloc_path;	
-	if (tmp_oldloc_path != oldloc_path)
-		GF_FREE (tmp_oldloc_path);
-
-	newloc->path = newloc_path;	
-	if (tmp_newloc_path != newloc_path)
-		GF_FREE (tmp_newloc_path);
-
-	return 0;
-}
-
-int32_t 
-path_setattr_cbk (call_frame_t *frame,
-                  void *cookie,
-                  xlator_t *this,
-                  int32_t op_ret,
-                  int32_t op_errno,
-                  struct iatt *preop,
-                  struct iatt *postop)
-{
-	STACK_UNWIND (frame, op_ret, op_errno, preop, postop);
-	return 0;
-}
-
-int32_t
-path_setattr (call_frame_t *frame,
-              xlator_t *this,
-              loc_t *loc,
-              struct iatt *stbuf,
-              int32_t valid)
-{
-	char *loc_path = (char *)loc->path;
-	char *tmp_path = NULL;
-
-	if (!(tmp_path = path_this_to_that (this, loc->path))) {
-		STACK_UNWIND (frame, -1, ENOENT, NULL, NULL);
-		return 0;
-	}
-	loc->path = tmp_path;
-
-	STACK_WIND (frame,
-		    path_setattr_cbk,
-		    FIRST_CHILD(this),
-		    FIRST_CHILD(this)->fops->setattr,
-		    loc,
-		    stbuf, valid);
-
-	loc->path = loc_path;
-	if (tmp_path != loc_path)
-		GF_FREE (tmp_path);
-
-	return 0;
-}
-
-
-int32_t 
-path_truncate (call_frame_t *frame,
-	       xlator_t *this,
-	       loc_t *loc,
-	       off_t offset)
-{
-	char *loc_path = (char *)loc->path;
-	char *tmp_path = NULL;
-	
-	if (!(tmp_path = path_this_to_that (this, loc->path))) {
-		STACK_UNWIND (frame, -1, ENOENT, NULL, NULL);
-		return 0;
-	}
-	loc->path = tmp_path;
-
-	STACK_WIND (frame, 
-		    path_truncate_cbk,
-		    FIRST_CHILD(this), 
-		    FIRST_CHILD(this)->fops->truncate, 
-		    loc, 
-		    offset);
-  
-	loc->path = loc_path;	
-	if (tmp_path != loc_path)
-		GF_FREE (tmp_path);
-
-	return 0;
-}
-
-
-int32_t 
-path_open (call_frame_t *frame,
-	   xlator_t *this,
-	   loc_t *loc,
-	   int32_t flags,
-	   fd_t *fd,
-           int32_t wbflags)
-{
-	char *loc_path = (char *)loc->path;
-	char *tmp_path = NULL;
-	
-	if (!(tmp_path = path_this_to_that (this, loc->path))) {
-		STACK_UNWIND (frame, -1, ENOENT, NULL, NULL);
-		return 0;
-	}
-	loc->path = tmp_path;
-
-	STACK_WIND (frame, 
-		    path_open_cbk, 
-		    FIRST_CHILD(this), 
-		    FIRST_CHILD(this)->fops->open, 
-		    loc, 
-		    flags,
-		    fd,
-                    wbflags);
-
-	loc->path = loc_path;	
-	if (tmp_path != loc_path)
-		GF_FREE (tmp_path);
-
-	return 0;
-}
-
-int32_t 
-path_create (call_frame_t *frame,
-	     xlator_t *this,
-	     loc_t *loc,
-	     int32_t flags,
-	     mode_t mode,
-	     fd_t *fd)
-{
-	char *loc_path = (char *)loc->path;
-	char *tmp_path = NULL;
-	
-	if (!(tmp_path = path_this_to_that (this, loc->path))) {
-		STACK_UNWIND (frame, -1, ENOENT, NULL, NULL);
-		return 0;
-	}
-	loc->path = tmp_path;
-
-	STACK_WIND (frame, 
-		    path_create_cbk, 
-		    FIRST_CHILD(this), 
-		    FIRST_CHILD(this)->fops->create, 
-		    loc, 
-		    flags,
-		    mode,
-		    fd);
-
-	loc->path = loc_path;	
-	if (tmp_path != loc_path)
-		GF_FREE (tmp_path);
-
-	return 0;
-}
-
-int32_t 
-path_setxattr (call_frame_t *frame,
-	       xlator_t *this,
-	       loc_t *loc,
-	       dict_t *dict,
-	       int32_t flags)
-{
-	char *tmp_name = NULL;
-	data_pair_t *trav = dict->members_list;
-	char *loc_path = (char *)loc->path;
-	char *tmp_path = NULL;
-	
-	if (!(tmp_path = path_this_to_that (this, loc->path))) {
-		STACK_UNWIND (frame, -1, ENOENT, NULL, NULL);
-		return 0;
-	}
-	loc->path = tmp_path;
-
-	if (ZR_FILE_CONTENT_REQUEST(trav->key)) {
-		tmp_name = name_this_to_that (this, loc->path, trav->key);
-		if (tmp_name != trav->key) {
-			trav->key = tmp_name;
-		} else {
-			tmp_name = NULL;
-		}
-	}
-
-	STACK_WIND (frame, 
-		    path_common_cbk, 
-		    FIRST_CHILD(this), 
-		    FIRST_CHILD(this)->fops->setxattr, 
-		    loc, 
-		    dict,
-		    flags);
-
-	loc->path = loc_path;	
-	if (tmp_path != loc_path)
-		GF_FREE (tmp_path);
-
-	GF_FREE (tmp_name);
-
-	return 0;
-}
-
-int32_t 
-path_getxattr (call_frame_t *frame,
-	       xlator_t *this,
-	       loc_t *loc,
-	       const char *name)
-{
-	char *tmp_name = (char *)name;
-	char *loc_path = (char *)loc->path;
-	char *tmp_path = NULL;
-	
-	if (!(tmp_path = path_this_to_that (this, loc->path))) {
-		STACK_UNWIND (frame, -1, ENOENT, NULL, NULL);
-		return 0;
-	}
-	loc->path = tmp_path;
-
-	if (ZR_FILE_CONTENT_REQUEST(name)) {
-		tmp_name = name_this_to_that (this, loc->path, name);
-	}
-
-	STACK_WIND (frame, 
-		    path_common_dict_cbk, 
-		    FIRST_CHILD(this), 
-		    FIRST_CHILD(this)->fops->getxattr,
-		    loc, 
-		    tmp_name);
-
-	loc->path = loc_path;	
-	if (tmp_path != loc_path)
-		GF_FREE (tmp_path);
-
-	if (tmp_name != name)
-		GF_FREE (tmp_name);
-
-	return 0;
-}
-
-int32_t 
-path_removexattr (call_frame_t *frame,
-		  xlator_t *this,
-		  loc_t *loc,
-		  const char *name)
-{
-	char *tmp_name = (char *)name;
-	char *loc_path = (char *)loc->path;
-	char *tmp_path = NULL;
-	
-	if (!(tmp_path = path_this_to_that (this, loc->path))) {
-		STACK_UNWIND (frame, -1, ENOENT, NULL, NULL);
-		return 0;
-	}
-	loc->path = tmp_path;
-
-	if (ZR_FILE_CONTENT_REQUEST(name)) {
-		tmp_name = name_this_to_that (this, loc->path, name);
-	}
-
-	STACK_WIND (frame, 
-		    path_common_cbk, 
-		    FIRST_CHILD(this), 
-		    FIRST_CHILD(this)->fops->removexattr, 
-		    loc, 
-		    tmp_name);
-
-	loc->path = loc_path;	
-	if (tmp_path != loc_path)
-		GF_FREE (tmp_path);
-
-	if (tmp_name != name)
-		GF_FREE (tmp_name);
-
-	return 0;
-}
-
-int32_t 
-path_opendir (call_frame_t *frame,
-	      xlator_t *this,
-	      loc_t *loc,
-	      fd_t *fd)
-{
-	char *loc_path = (char *)loc->path;
-	char *tmp_path = NULL;
-	
-	if (!(tmp_path = path_this_to_that (this, loc->path))) {
-		STACK_UNWIND (frame, -1, ENOENT, NULL, NULL);
-		return 0;
-	}
-	loc->path = tmp_path;
-
-	STACK_WIND (frame, 
-		    path_opendir_cbk, 
-		    FIRST_CHILD(this), 
-		    FIRST_CHILD(this)->fops->opendir, 
-		    loc, 
-		    fd);
-
-	loc->path = loc_path;	
-	if (tmp_path != loc_path)
-		GF_FREE (tmp_path);
-
-	return 0;
-}
-
-int32_t 
-path_access (call_frame_t *frame,
-	     xlator_t *this,
-	     loc_t *loc,
-	     int32_t mask)
-{
-	char *loc_path = (char *)loc->path;
-	char *tmp_path = NULL;
-	
-	if (!(tmp_path = path_this_to_that (this, loc->path))) {
-		STACK_UNWIND (frame, -1, ENOENT, NULL, NULL);
-		return 0;
-	}
-	loc->path = tmp_path;
-
-	STACK_WIND (frame, 
-		    path_common_cbk, 
-		    FIRST_CHILD(this), 
-		    FIRST_CHILD(this)->fops->access, 
-		    loc, 
-		    mask);
-
-	loc->path = loc_path;
-	if (tmp_path != loc_path)
-		GF_FREE (tmp_path);
-
-	return 0;
-}
-
-int32_t
-path_checksum_cbk (call_frame_t *frame,
-		   void *cookie,
-		   xlator_t *this,
-		   int32_t op_ret,
-		   int32_t op_errno,
-		   uint8_t *fchecksum,
-		   uint8_t *dchecksum)
-{
-	STACK_UNWIND (frame, op_ret, op_errno, fchecksum, dchecksum);
-	return 0;
-}
-
-int32_t
-path_checksum (call_frame_t *frame,
-	       xlator_t *this,
-	       loc_t *loc,
-	       int32_t flag)
-{
-	char *loc_path = (char *)loc->path;
-	char *tmp_path = NULL;
-	
-	if (!(tmp_path = path_this_to_that (this, loc->path))) {
-		STACK_UNWIND (frame, -1, ENOENT, NULL, NULL);
-		return 0;
-	}
-	loc->path = tmp_path;
-
-	STACK_WIND (frame,
-		    path_checksum_cbk,
-		    FIRST_CHILD(this), 
-		    FIRST_CHILD(this)->fops->checksum, 
-		    loc, 
-		    flag);
-
-	loc->path = loc_path;	
-	if (tmp_path != loc_path)
-		GF_FREE (tmp_path);
-
-	return 0;
-}
-
-
-int32_t
-path_entrylk (call_frame_t *frame, xlator_t *this,
-	      const char *volume, loc_t *loc, const char *basename,
-	      entrylk_cmd cmd, entrylk_type type)
-{
-	char *loc_path = (char *)loc->path;
-	char *tmp_path = NULL;
-	
-	if (!(tmp_path = path_this_to_that (this, loc->path))) {
-		STACK_UNWIND (frame, -1, ENOENT, NULL, NULL);
-		return 0;
-	}
-	loc->path = tmp_path;
-
-	STACK_WIND (frame, path_common_cbk,
-		    FIRST_CHILD(this),
-		    FIRST_CHILD(this)->fops->entrylk,
-		    volume, loc, basename, cmd, type);
-
-	loc->path = loc_path;	
-	if (tmp_path != loc_path)
-		GF_FREE (tmp_path);
-
-	return 0;
-}
-
-int32_t
-path_inodelk (call_frame_t *frame, xlator_t *this,
-              const char *volume, loc_t *loc, int32_t cmd, struct gf_flock *lock)
-{
-	char *loc_path = (char *)loc->path;
-	char *tmp_path = NULL;
-	
-	if (!(tmp_path = path_this_to_that (this, loc->path))) {
-		STACK_UNWIND (frame, -1, ENOENT, NULL, NULL);
-		return 0;
-	}
-	loc->path = tmp_path;
-
-	STACK_WIND (frame,
-		    path_common_cbk,
-		    FIRST_CHILD(this),
-		    FIRST_CHILD(this)->fops->inodelk,
-		    volume, loc, cmd, lock);
-
-	loc->path = loc_path;	
-	if (tmp_path != loc_path)
-		GF_FREE (tmp_path);
-
-	return 0;
-}
-
-
-int32_t
-path_xattrop (call_frame_t *frame,
-	      xlator_t *this,
-	      loc_t *loc,
-	      gf_xattrop_flags_t flags,
-	      dict_t *dict)
-{
-	char *loc_path = (char *)loc->path;
-	char *tmp_path = NULL;
-	
-	if (!(tmp_path = path_this_to_that (this, loc->path))) {
-		STACK_UNWIND (frame, -1, ENOENT, NULL, NULL);
-		return 0;
-	}
-	loc->path = tmp_path;
-
-	STACK_WIND (frame,
-		    path_common_dict_cbk,
-		    FIRST_CHILD(this),
-		    FIRST_CHILD(this)->fops->xattrop,
-		    loc,
-		    flags,
-		    dict);
-
-	loc->path = loc_path;	
-	if (tmp_path != loc_path)
-		GF_FREE (tmp_path);
-
-	return 0;
-}
-
-int32_t
-mem_acct_init (xlator_t *this)
-{
-        int     ret = -1;
-
-        if (!this)
-                return ret;
-
-        ret = xlator_mem_acct_init (this, gf_path_mt_end + 1);
-        
-        if (ret != 0) {
-                gf_log (this->name, GF_LOG_ERROR, "Memory accounting init"
-                        "failed");
-                return ret;
-        }
-
-        return ret;
-}
-
-int32_t 
-init (xlator_t *this)
-{
-	dict_t *options = this->options;
-	path_private_t *priv = NULL;
-
-	if (!this->children || this->children->next) {
-		gf_log (this->name, GF_LOG_ERROR, 
-			"path translator requires exactly one subvolume");
-		return -1;
-	}
-    
-	if (!this->parents) {
-		gf_log (this->name, GF_LOG_WARNING,
-			"dangling volume. check volfile ");
-	}
-  
-	priv = GF_CALLOC (1, sizeof (*priv), gf_path_mt_path_private_t);
-	ERR_ABORT (priv);
-	if (dict_get (options, "start-offset")) {
-		priv->start_off = data_to_int32 (dict_get (options, 
-							   "start-offset"));
-	}
-	if (dict_get (options, "end-offset")) {
-		priv->end_off = data_to_int32 (dict_get (options, 
-							 "end-offset"));
-	}
-
-	if (dict_get (options, "regex")) {
-		int32_t ret = 0;
-		priv->preg = GF_CALLOC (1, sizeof (regex_t),
-                                        gf_path_mt_regex_t);
-		ERR_ABORT (priv->preg);
-		ret = regcomp (priv->preg, 
-			       data_to_str (dict_get (options, "regex")), 
-			       REG_EXTENDED);
-		if (ret) {
-			gf_log (this->name, GF_LOG_ERROR, 
-				"Failed to compile the 'option regex'");
-			GF_FREE (priv);
-			return -1;
-		}
-		if (dict_get (options, "replace-with")) {
-			priv->that = data_to_str (dict_get (options, 
-							    "replace-with"));
-		} else {
-			priv->that = "";
-		}
-	}
-
-	this->private = priv;
-	return 0;
-}
-
-void
-fini (xlator_t *this)
-{
-	return;
-}
-
-struct xlator_fops fops = {
-	.stat        = path_stat,
-	.readlink    = path_readlink,
-	.mknod       = path_mknod,
-	.mkdir       = path_mkdir,
-	.unlink      = path_unlink,
-	.rmdir       = path_rmdir,
-	.symlink     = path_symlink,
-	.rename      = path_rename,
-	.link        = path_link,
-	.truncate    = path_truncate,
-	.open        = path_open,
-	.setxattr    = path_setxattr,
-	.getxattr    = path_getxattr,
-	.removexattr = path_removexattr,
-	.opendir     = path_opendir,
-	.access      = path_access,
-	.create      = path_create,
-	.lookup      = path_lookup,
-	.checksum    = path_checksum,
-	.xattrop     = path_xattrop,
-	.entrylk     = path_entrylk,
-	.inodelk     = path_inodelk,
-        .setattr     = path_setattr,
-};
-
-struct xlator_cbks cbks = {
-};
-
-struct volume_options options[] = { 
-	{ .key  = {"start-offset"}, 
-	  .type = GF_OPTION_TYPE_INT, 
-	  .min  = 0, 
-	  .max  = 4095 
-	},
-	{ .key  = {"end-offset"}, 
-	  .type = GF_OPTION_TYPE_INT, 
-	  .min  = 1, 
-	  .max  = 4096 
-	},
-	{ .key  = {"replace-with"}, 
-	  .type = GF_OPTION_TYPE_ANY 
-	},
-	{ .key  = {NULL} },
-};
diff --git a/xlators/features/protect/Makefile.am b/xlators/features/protect/Makefile.am
deleted file mode 100644
index d471a3f9243..00000000000
--- a/xlators/features/protect/Makefile.am
+++ /dev/null
@@ -1,3 +0,0 @@
-SUBDIRS = src
-
-CLEANFILES = 
diff --git a/xlators/features/protect/src/Makefile.am b/xlators/features/protect/src/Makefile.am
deleted file mode 100644
index fdfed4b219e..00000000000
--- a/xlators/features/protect/src/Makefile.am
+++ /dev/null
@@ -1,21 +0,0 @@
-xlator_LTLIBRARIES = prot_dht.la prot_client.la prot_server.la
-
-xlatordir = $(libdir)/glusterfs/$(PACKAGE_VERSION)/xlator/features
-
-prot_dht_la_LDFLAGS = -module -avoid-version
-prot_dht_la_SOURCES = prot_dht.c
-prot_dht_la_LIBADD  = $(top_builddir)/libglusterfs/src/libglusterfs.la
-
-prot_client_la_LDFLAGS = -module -avoid-version
-prot_client_la_SOURCES = prot_client.c
-prot_client_la_LIBADD  = $(top_builddir)/libglusterfs/src/libglusterfs.la
-
-prot_server_la_LDFLAGS = -module -avoid-version
-prot_server_la_SOURCES = prot_server.c
-prot_server_la_LIBADD  = $(top_builddir)/libglusterfs/src/libglusterfs.la
-
-AM_CPPFLAGS = $(GF_CPPFLAGS) -I$(top_srcdir)/libglusterfs/src \
-	 -I$(CONTRIBDIR)/libexecinfo
-AM_CFLAGS = -Wall $(GF_CFLAGS)
-
-CLEANFILES =
diff --git a/xlators/features/protect/src/prot_client.c b/xlators/features/protect/src/prot_client.c
deleted file mode 100644
index 500c772bedd..00000000000
--- a/xlators/features/protect/src/prot_client.c
+++ /dev/null
@@ -1,219 +0,0 @@
-/*
-   Copyright (c) 2013 Red Hat, Inc. <http://www.redhat.com>
-   This file is part of GlusterFS.
-
-   This file is licensed to you under your choice of the GNU Lesser
-   General Public License, version 3 or any later version (LGPLv3 or
-   later), or the GNU General Public License, version 2 (GPLv2), in all
-   cases as published by the Free Software Foundation.
-*/
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
-
-#include "xlator.h"
-#include "defaults.h"
-
-#ifdef HAVE_BACKTRACE
-#include <execinfo.h>
-#else
-#include "execinfo_compat.h"
-#endif
-
-#define NUM_FRAMES 20
-
-static char PROTECT_KEY[] = "trusted.glusterfs.protect";
-
-enum {
-        PROT_ACT_NONE = 0,
-        PROT_ACT_LOG,
-        PROT_ACT_REJECT,
-};
-
-void
-pcli_print_trace (char *name, call_frame_t *frame)
-{
-        void    *frames[NUM_FRAMES];
-        char    **symbols;
-        int     size;
-        int     i;
-
-        gf_log (name, GF_LOG_INFO, "Translator stack:");
-        while (frame) {
-                gf_log (name, GF_LOG_INFO, "%s (%s)",
-                        frame->wind_from, frame->this->name);
-                frame = frame->next;
-        }
-
-        size = backtrace (frames, NUM_FRAMES);
-        if (size <= 0) {
-                return;
-        }
-        symbols = backtrace_symbols (frames, size);
-        if (!symbols) {
-                return;
-        }
-
-        gf_log (name, GF_LOG_INFO, "Processor stack:");
-        for (i = 0; i < size; ++i) {
-                gf_log (name, GF_LOG_INFO, "%s", symbols[i]);
-        }
-        free (symbols);
-}
-
-int32_t
-pcli_rename (call_frame_t *frame, xlator_t *this, loc_t *oldloc,
-                loc_t *newloc, dict_t *xdata)
-{
-        uint64_t        value;
-
-        if (newloc->parent == oldloc->parent) {
-                gf_log (this->name, GF_LOG_DEBUG, "rename in same directory");
-                goto simple_unwind;
-        }
-        if (!oldloc->parent) {
-                goto simple_unwind;
-        }
-        if (inode_ctx_get (oldloc->parent, this, &value) != 0) {
-                goto simple_unwind;
-        }
-
-        if (value != PROT_ACT_NONE) {
-                gf_log (this->name, GF_LOG_WARNING,
-                        "got rename for protected %s", oldloc->path);
-                pcli_print_trace (this->name, frame->next);
-                if (value == PROT_ACT_REJECT) {
-                        STACK_UNWIND_STRICT (rename, frame, -1, EPERM,
-                                             NULL, NULL, NULL, NULL, NULL,
-                                             xdata);
-                        return 0;
-                }
-        }
-
-simple_unwind:
-        STACK_WIND_TAIL (frame, FIRST_CHILD(this),
-                         FIRST_CHILD(this)->fops->rename, oldloc, newloc,
-                         xdata);
-        return 0;
-}
-
-int32_t
-pcli_setxattr (call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *dict,
-               int32_t flags, dict_t *xdata)
-{
-        data_t          *data;
-        uint64_t        value;
-
-        /*
-         * We can't use dict_get_str and strcmp here, because the value comes
-         * directly from the user and might not be NUL-terminated (it would
-         * be if we had set it ourselves.
-         */
-
-        data = dict_get(dict,PROTECT_KEY);
-        if (!data) {
-                goto simple_wind;
-        }
-
-        if (dict->count > 1) {
-                gf_log (this->name, GF_LOG_WARNING,
-                        "attempted to mix %s with other keys", PROTECT_KEY);
-                goto simple_wind;
-        }
-
-        gf_log (this->name, GF_LOG_DEBUG, "got %s request", PROTECT_KEY);
-        if (!strncmp(data->data,"log",data->len)) {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "logging removals on %s", loc->path);
-                value = PROT_ACT_LOG;
-        }
-        else if (!strncmp(data->data,"reject",data->len)) {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "rejecting removals on %s", loc->path);
-                value = PROT_ACT_REJECT;
-        }
-        else {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "removing protection on %s", loc->path);
-                value = PROT_ACT_NONE;
-        }
-        /* Right now the value doesn't matter - just the presence. */
-        if (inode_ctx_set(loc->inode,this,&value) != 0) {
-                gf_log (this->name, GF_LOG_WARNING,
-                        "failed to set protection status for %s", loc->path);
-        }
-        STACK_UNWIND_STRICT (setxattr, frame, 0, 0, NULL);
-        return 0;
-
-simple_wind:
-        STACK_WIND_TAIL (frame,
-                         FIRST_CHILD(this), FIRST_CHILD(this)->fops->setxattr,
-                         loc, dict, flags, xdata);
-        return 0;
-}
-
-int32_t
-pcli_unlink (call_frame_t *frame, xlator_t *this, loc_t *loc, int xflag,
-             dict_t *xdata)
-{
-        uint64_t        value;
-
-        if (!loc->parent || (inode_ctx_get(loc->parent,this,&value) != 0)) {
-                goto simple_unwind;
-        }
-
-        if (value != PROT_ACT_NONE) {
-                gf_log (this->name, GF_LOG_WARNING,
-                        "got unlink for protected %s", loc->path);
-                pcli_print_trace(this->name,frame->next);
-                if (value == PROT_ACT_REJECT) {
-                        STACK_UNWIND_STRICT (unlink, frame, -1, EPERM,
-                                             NULL, NULL, NULL);
-                        return 0;
-                }
-        }
-
-simple_unwind:
-        STACK_WIND_TAIL (frame, FIRST_CHILD(this),
-                         FIRST_CHILD(this)->fops->unlink, loc, xflag, xdata);
-        return 0;
-}
-
-int32_t
-init (xlator_t *this)
-{
-        if (!this->children || this->children->next) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "translator not configured with exactly one child");
-                return -1;
-        }
-
-        if (!this->parents) {
-                gf_log (this->name, GF_LOG_WARNING,
-                        "dangling volume. check volfile ");
-        }
-
-        return 0;
-}
-
-
-void
-fini (xlator_t *this)
-{
-        return;
-}
-
-
-struct xlator_fops fops = {
-        .rename         = pcli_rename,
-        .setxattr       = pcli_setxattr,
-        .unlink         = pcli_unlink,
-};
-
-struct xlator_cbks cbks = {
-};
-
-struct volume_options options[] = {
-        { .key = {NULL} },
-};
diff --git a/xlators/features/protect/src/prot_dht.c b/xlators/features/protect/src/prot_dht.c
deleted file mode 100644
index feec6ffd69c..00000000000
--- a/xlators/features/protect/src/prot_dht.c
+++ /dev/null
@@ -1,168 +0,0 @@
-/*
-   Copyright (c) 2013 Red Hat, Inc. <http://www.redhat.com>
-   This file is part of GlusterFS.
-
-   This file is licensed to you under your choice of the GNU Lesser
-   General Public License, version 3 or any later version (LGPLv3 or
-   later), or the GNU General Public License, version 2 (GPLv2), in all
-   cases as published by the Free Software Foundation.
-*/
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
-
-#include "xlator.h"
-#include "defaults.h"
-
-enum gf_pdht_mem_types_ {
-        gf_pdht_mt_coord_t = gf_common_mt_end + 1,
-        gf_pdht_mt_end
-};
-
-typedef struct {
-        pthread_mutex_t         lock;
-        uint16_t                refs;
-        int32_t                 op_ret;
-        int32_t                 op_errno;
-        dict_t                  *xdata;
-} pdht_coord_t;
-
-static char PROTECT_KEY[] = "trusted.glusterfs.protect";
-
-void
-pdht_unref_and_unlock (call_frame_t *frame, xlator_t *this,
-                       pdht_coord_t *coord)
-{
-        gf_boolean_t    should_unwind;
-
-        should_unwind = (--(coord->refs) == 0);
-        pthread_mutex_unlock(&coord->lock);
-
-        if (should_unwind) {
-                STACK_UNWIND_STRICT (setxattr, frame, 
-                                     coord->op_ret, coord->op_errno,
-                                     coord->xdata);
-                if (coord->xdata) {
-                        dict_unref(coord->xdata);
-                }
-                GF_FREE(coord);
-        }
-}
-
-int32_t
-pdht_recurse_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                  int32_t op_ret, int32_t op_errno, dict_t *xdata)
-{
-        pdht_coord_t    *coord = cookie;
-
-        pthread_mutex_lock(&coord->lock);
-        if (op_ret) {
-                coord->op_ret = op_ret;
-                coord->op_errno = op_errno;
-        }
-        if (xdata) {
-                if (coord->xdata) {
-                        dict_unref(coord->xdata);
-                }
-                coord->xdata = dict_ref(xdata);
-        }
-        pdht_unref_and_unlock(frame,this,coord);
-                
-        return 0;
-}
-
-void
-pdht_recurse (call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *dict,
-              int32_t flags, dict_t *xdata, xlator_t *xl, pdht_coord_t *coord)
-{
-        xlator_list_t   *iter;
-
-        if (!strcmp(xl->type,"features/prot_client")) {
-                pthread_mutex_lock(&coord->lock);
-                ++(coord->refs);
-                pthread_mutex_unlock(&coord->lock);
-                STACK_WIND_COOKIE (frame, pdht_recurse_cbk, coord, xl,
-                                   xl->fops->setxattr, loc, dict, flags, xdata);
-        }
-
-        else for (iter = xl->children; iter; iter = iter->next) {
-                pdht_recurse (frame, this, loc, dict, flags, xdata,
-                              iter->xlator, coord);
-        }
-}
-
-int32_t
-pdht_setxattr (call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *dict,
-               int32_t flags, dict_t *xdata)
-{
-        pdht_coord_t    *coord;
-
-        if (!dict_get(dict,PROTECT_KEY)) {
-                goto simple_wind;
-        }
-
-        if (dict->count > 1) {
-                gf_log (this->name, GF_LOG_WARNING,
-                        "attempted to mix %s with other keys", PROTECT_KEY);
-                goto simple_wind;
-        }
-
-        coord = GF_CALLOC(1,sizeof(*coord),gf_pdht_mt_coord_t);
-        if (!coord) {
-                gf_log (this->name, GF_LOG_WARNING, "allocation failed");
-                goto simple_wind;
-        }
-
-        pthread_mutex_init(&coord->lock,NULL);
-        coord->refs = 1;
-        coord->op_ret = 0;
-        coord->xdata = NULL;
-
-        pdht_recurse(frame,this,loc,dict,flags,xdata,this,coord);
-        pthread_mutex_lock(&coord->lock);
-        pdht_unref_and_unlock(frame,this,coord);
-
-        return 0;
-
-simple_wind:
-        STACK_WIND_TAIL (frame,
-                         FIRST_CHILD(this), FIRST_CHILD(this)->fops->setxattr,
-                         loc, dict, flags, xdata);
-        return 0;
-}
-
-int32_t
-init (xlator_t *this)
-{
-	if (!this->children || this->children->next) {
-		gf_log (this->name, GF_LOG_ERROR,
-			"translator not configured with exactly one child");
-		return -1;
-	}
-
-	if (!this->parents) {
-		gf_log (this->name, GF_LOG_WARNING,
-			"dangling volume. check volfile ");
-	}
-
-	return 0;
-}
-
-
-void
-fini (xlator_t *this)
-{
-	return;
-}
-
-struct xlator_fops fops = {
-        .setxattr = pdht_setxattr,
-};
-
-struct xlator_cbks cbks = {
-};
-
-struct volume_options options[] = {
-	{ .key = {NULL} },
-};
diff --git a/xlators/features/protect/src/prot_server.c b/xlators/features/protect/src/prot_server.c
deleted file mode 100644
index beaee0889b7..00000000000
--- a/xlators/features/protect/src/prot_server.c
+++ /dev/null
@@ -1,51 +0,0 @@
-/*
-   Copyright (c) 2013 Red Hat, Inc. <http://www.redhat.com>
-   This file is part of GlusterFS.
-
-   This file is licensed to you under your choice of the GNU Lesser
-   General Public License, version 3 or any later version (LGPLv3 or
-   later), or the GNU General Public License, version 2 (GPLv2), in all
-   cases as published by the Free Software Foundation.
-*/
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
-
-#include "xlator.h"
-#include "defaults.h"
-
-int32_t
-init (xlator_t *this)
-{
-	if (!this->children || this->children->next) {
-		gf_log (this->name, GF_LOG_ERROR,
-			"translator not configured with exactly one child");
-		return -1;
-	}
-
-	if (!this->parents) {
-		gf_log (this->name, GF_LOG_WARNING,
-			"dangling volume. check volfile ");
-	}
-
-	return 0;
-}
-
-
-void
-fini (xlator_t *this)
-{
-	return;
-}
-
-
-struct xlator_fops fops = {
-};
-
-struct xlator_cbks cbks = {
-};
-
-struct volume_options options[] = {
-	{ .key = {NULL} },
-};
diff --git a/xlators/features/qemu-block/src/Makefile.am b/xlators/features/qemu-block/src/Makefile.am
deleted file mode 100644
index e32fc54605b..00000000000
--- a/xlators/features/qemu-block/src/Makefile.am
+++ /dev/null
@@ -1,156 +0,0 @@
-if ENABLE_QEMU_BLOCK
-xlator_LTLIBRARIES = qemu-block.la
-xlatordir = $(libdir)/glusterfs/$(PACKAGE_VERSION)/xlator/features
-
-qemu_block_la_LDFLAGS = -module -avoid-version
-qemu_block_la_LIBADD = $(top_builddir)/libglusterfs/src/libglusterfs.la \
-	$(GLIB_LIBS) $(ZLIB_LIBS)
-
-qemu_block_la_SOURCES_qemu = \
-	$(CONTRIBDIR)/qemu/qemu-coroutine.c \
-	$(CONTRIBDIR)/qemu/qemu-coroutine-lock.c \
-	$(CONTRIBDIR)/qemu/qemu-coroutine-sleep.c \
-	$(CONTRIBDIR)/qemu/coroutine-ucontext.c \
-	$(CONTRIBDIR)/qemu/block.c \
-	$(CONTRIBDIR)/qemu/nop-symbols.c
-
-qemu_block_la_SOURCES_qemu_util = \
-	$(CONTRIBDIR)/qemu/util/aes.c \
-	$(CONTRIBDIR)/qemu/util/bitmap.c \
-	$(CONTRIBDIR)/qemu/util/bitops.c \
-	$(CONTRIBDIR)/qemu/util/cutils.c \
-	$(CONTRIBDIR)/qemu/util/error.c \
-	$(CONTRIBDIR)/qemu/util/hbitmap.c \
-	$(CONTRIBDIR)/qemu/util/iov.c \
-	$(CONTRIBDIR)/qemu/util/module.c \
-	$(CONTRIBDIR)/qemu/util/oslib-posix.c \
-	$(CONTRIBDIR)/qemu/util/qemu-option.c \
-	$(CONTRIBDIR)/qemu/util/qemu-error.c \
-	$(CONTRIBDIR)/qemu/util/qemu-thread-posix.c \
-	$(CONTRIBDIR)/qemu/util/unicode.c \
-	$(CONTRIBDIR)/qemu/util/hexdump.c
-
-qemu_block_la_SOURCES_qemu_block = \
-	$(CONTRIBDIR)/qemu/block/snapshot.c \
-	$(CONTRIBDIR)/qemu/block/qcow2-cache.c \
-	$(CONTRIBDIR)/qemu/block/qcow2-cluster.c \
-	$(CONTRIBDIR)/qemu/block/qcow2-refcount.c \
-	$(CONTRIBDIR)/qemu/block/qcow2-snapshot.c \
-	$(CONTRIBDIR)/qemu/block/qcow2.c \
-	$(CONTRIBDIR)/qemu/block/qed-check.c \
-	$(CONTRIBDIR)/qemu/block/qed-cluster.c \
-	$(CONTRIBDIR)/qemu/block/qed-gencb.c \
-	$(CONTRIBDIR)/qemu/block/qed-l2-cache.c \
-	$(CONTRIBDIR)/qemu/block/qed-table.c \
-	$(CONTRIBDIR)/qemu/block/qed.c
-
-qemu_block_la_SOURCES_qemu_qobject = \
-	$(CONTRIBDIR)/qemu/qobject/json-lexer.c \
-	$(CONTRIBDIR)/qemu/qobject/json-parser.c \
-	$(CONTRIBDIR)/qemu/qobject/json-streamer.c \
-	$(CONTRIBDIR)/qemu/qobject/qbool.c \
-	$(CONTRIBDIR)/qemu/qobject/qdict.c \
-	$(CONTRIBDIR)/qemu/qobject/qerror.c \
-	$(CONTRIBDIR)/qemu/qobject/qfloat.c \
-	$(CONTRIBDIR)/qemu/qobject/qint.c \
-	$(CONTRIBDIR)/qemu/qobject/qjson.c \
-	$(CONTRIBDIR)/qemu/qobject/qlist.c \
-	$(CONTRIBDIR)/qemu/qobject/qstring.c
-
-qemu_block_la_SOURCES = \
-	$(qemu_block_la_SOURCES_qemu) \
-	$(qemu_block_la_SOURCES_qemu_util) \
-	$(qemu_block_la_SOURCES_qemu_block) \
-	$(qemu_block_la_SOURCES_qemu_qobject) \
-	bdrv-xlator.c \
-	coroutine-synctask.c \
-	bh-syncop.c \
-	monitor-logging.c \
-	clock-timer.c \
-	qemu-block.c \
-	qb-coroutines.c
-
-noinst_HEADERS_qemu = \
-	$(CONTRIBDIR)/qemu/config-host.h \
-	$(CONTRIBDIR)/qemu/qapi-types.h \
-	$(CONTRIBDIR)/qemu/qmp-commands.h \
-	$(CONTRIBDIR)/qemu/trace/generated-tracers.h \
-	$(CONTRIBDIR)/qemu/include/config.h \
-	$(CONTRIBDIR)/qemu/include/glib-compat.h \
-	$(CONTRIBDIR)/qemu/include/qemu-common.h \
-	$(CONTRIBDIR)/qemu/include/trace.h \
-	$(CONTRIBDIR)/qemu/include/block/coroutine.h \
-	$(CONTRIBDIR)/qemu/include/block/aio.h \
-	$(CONTRIBDIR)/qemu/include/block/block.h \
-	$(CONTRIBDIR)/qemu/include/block/block_int.h \
-	$(CONTRIBDIR)/qemu/include/block/blockjob.h \
-	$(CONTRIBDIR)/qemu/include/block/coroutine.h \
-	$(CONTRIBDIR)/qemu/include/block/coroutine_int.h \
-	$(CONTRIBDIR)/qemu/include/block/snapshot.h \
-	$(CONTRIBDIR)/qemu/include/exec/cpu-common.h \
-	$(CONTRIBDIR)/qemu/include/exec/hwaddr.h \
-	$(CONTRIBDIR)/qemu/include/exec/poison.h \
-	$(CONTRIBDIR)/qemu/include/fpu/softfloat.h \
-	$(CONTRIBDIR)/qemu/include/migration/migration.h \
-	$(CONTRIBDIR)/qemu/include/migration/qemu-file.h \
-	$(CONTRIBDIR)/qemu/include/migration/vmstate.h \
-	$(CONTRIBDIR)/qemu/include/monitor/monitor.h \
-	$(CONTRIBDIR)/qemu/include/monitor/readline.h \
-	$(CONTRIBDIR)/qemu/include/qapi/error.h \
-	$(CONTRIBDIR)/qemu/include/qapi/qmp/json-lexer.h \
-	$(CONTRIBDIR)/qemu/include/qapi/qmp/json-parser.h \
-	$(CONTRIBDIR)/qemu/include/qapi/qmp/json-streamer.h \
-	$(CONTRIBDIR)/qemu/include/qapi/qmp/qbool.h \
-	$(CONTRIBDIR)/qemu/include/qapi/qmp/qdict.h \
-	$(CONTRIBDIR)/qemu/include/qapi/qmp/qerror.h \
-	$(CONTRIBDIR)/qemu/include/qapi/qmp/qfloat.h \
-	$(CONTRIBDIR)/qemu/include/qapi/qmp/qint.h \
-	$(CONTRIBDIR)/qemu/include/qapi/qmp/qjson.h \
-	$(CONTRIBDIR)/qemu/include/qapi/qmp/qlist.h \
-	$(CONTRIBDIR)/qemu/include/qapi/qmp/qobject.h \
-	$(CONTRIBDIR)/qemu/include/qapi/qmp/qstring.h \
-	$(CONTRIBDIR)/qemu/include/qapi/qmp/types.h \
-	$(CONTRIBDIR)/qemu/include/qemu/aes.h \
-	$(CONTRIBDIR)/qemu/include/qemu/atomic.h \
-	$(CONTRIBDIR)/qemu/include/qemu/bitmap.h \
-	$(CONTRIBDIR)/qemu/include/qemu/bitops.h \
-	$(CONTRIBDIR)/qemu/include/qemu/bswap.h \
-	$(CONTRIBDIR)/qemu/include/qemu/compiler.h \
-	$(CONTRIBDIR)/qemu/include/qemu/error-report.h \
-	$(CONTRIBDIR)/qemu/include/qemu/event_notifier.h \
-	$(CONTRIBDIR)/qemu/include/qemu/hbitmap.h \
-	$(CONTRIBDIR)/qemu/include/qemu/host-utils.h \
-	$(CONTRIBDIR)/qemu/include/qemu/iov.h \
-	$(CONTRIBDIR)/qemu/include/qemu/main-loop.h \
-	$(CONTRIBDIR)/qemu/include/qemu/module.h \
-	$(CONTRIBDIR)/qemu/include/qemu/notify.h \
-	$(CONTRIBDIR)/qemu/include/qemu/option.h \
-	$(CONTRIBDIR)/qemu/include/qemu/option_int.h \
-	$(CONTRIBDIR)/qemu/include/qemu/osdep.h \
-	$(CONTRIBDIR)/qemu/include/qemu/queue.h \
-	$(CONTRIBDIR)/qemu/include/qemu/sockets.h \
-	$(CONTRIBDIR)/qemu/include/qemu/thread-posix.h \
-	$(CONTRIBDIR)/qemu/include/qemu/thread.h \
-	$(CONTRIBDIR)/qemu/include/qemu/timer.h \
-	$(CONTRIBDIR)/qemu/include/qemu/typedefs.h \
-	$(CONTRIBDIR)/qemu/include/sysemu/sysemu.h \
-	$(CONTRIBDIR)/qemu/include/sysemu/os-posix.h \
-	$(CONTRIBDIR)/qemu/block/qcow2.h \
-	$(CONTRIBDIR)/qemu/block/qed.h
-
-noinst_HEADERS = \
-	$(noinst_HEADERS_qemu) \
-	qemu-block.h \
-	qemu-block-memory-types.h \
-	qb-coroutines.h
-
-AM_CPPFLAGS = $(GF_CPPFLAGS) -I$(top_srcdir)/libglusterfs/src \
-            -I$(CONTRIBDIR)/qemu \
-            -I$(CONTRIBDIR)/qemu/include \
-            -DGLUSTER_XLATOR
-
-AM_CFLAGS = -fno-strict-aliasing -Wall $(GF_CFLAGS) $(GLIB_CFLAGS)
-
-CLEANFILES =
-
-endif
diff --git a/xlators/features/qemu-block/src/bdrv-xlator.c b/xlators/features/qemu-block/src/bdrv-xlator.c
deleted file mode 100644
index 5dbe292d6d7..00000000000
--- a/xlators/features/qemu-block/src/bdrv-xlator.c
+++ /dev/null
@@ -1,391 +0,0 @@
-/*
-  Copyright (c) 2013 Red Hat, Inc. <http://www.redhat.com>
-  This file is part of GlusterFS.
-
-  This file is licensed to you under your choice of the GNU Lesser
-  General Public License, version 3 or any later version (LGPLv3 or
-  later), or the GNU General Public License, version 2 (GPLv2), in all
-  cases as published by the Free Software Foundation.
-*/
-
-
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
-
-#include "inode.h"
-#include "syncop.h"
-#include "qemu-block.h"
-#include "block/block_int.h"
-
-typedef struct BDRVGlusterState {
-	inode_t *inode;
-} BDRVGlusterState;
-
-static QemuOptsList runtime_opts = {
-	.name = "gluster",
-	.head = QTAILQ_HEAD_INITIALIZER(runtime_opts.head),
-	.desc = {
-		{
-			.name = "filename",
-			.type = QEMU_OPT_STRING,
-			.help = "GFID of file",
-		},
-		{ /* end of list */ }
-	},
-};
-
-inode_t *
-qb_inode_from_filename (const char *filename)
-{
-	const char *iptr = NULL;
-	inode_t *inode = NULL;
-
-	iptr = filename + 17;
-	sscanf (iptr, "%p", &inode);
-
-	return inode;
-}
-
-
-int
-qb_inode_to_filename (inode_t *inode, char *filename, int size)
-{
-	return snprintf (filename, size, "gluster://inodep:%p", inode);
-}
-
-
-static fd_t *
-fd_from_bs (BlockDriverState *bs)
-{
-	BDRVGlusterState *s = bs->opaque;
-
-	return fd_anonymous (s->inode);
-}
-
-
-static int
-qemu_gluster_open (BlockDriverState *bs, QDict *options, int bdrv_flags)
-{
-	inode_t *inode = NULL;
-	BDRVGlusterState *s = bs->opaque;
-	QemuOpts *opts = NULL;
-	Error *local_err = NULL;
-	const char *filename = NULL;
-	char gfid_str[128];
-	int ret;
-	qb_conf_t *conf = THIS->private;
-
-	opts = qemu_opts_create_nofail(&runtime_opts);
-	qemu_opts_absorb_qdict(opts, options, &local_err);
-	if (error_is_set(&local_err)) {
-		qerror_report_err(local_err);
-		error_free(local_err);
-		return -EINVAL;
-	}
-
-	filename = qemu_opt_get(opts, "filename");
-
-	/*
-	 * gfid:<gfid> format means we're opening a backing image.
-	 */
-	ret = sscanf(filename, "gluster://gfid:%s", gfid_str);
-	if (ret) {
-		loc_t loc = {0,};
-		struct iatt buf = {0,};
-		uuid_t gfid;
-
-		gf_uuid_parse(gfid_str, gfid);
-
-		loc.inode = inode_find(conf->root_inode->table, gfid);
-		if (!loc.inode) {
-			loc.inode = inode_new(conf->root_inode->table);
-			gf_uuid_copy(loc.inode->gfid, gfid);
-		}
-
-		gf_uuid_copy(loc.gfid, loc.inode->gfid);
-		ret = syncop_lookup(FIRST_CHILD(THIS), &loc, &buf, NULL,
-                                    NULL, NULL);
-		if (ret) {
-			loc_wipe(&loc);
-			return ret;
-		}
-
-		s->inode = inode_ref(loc.inode);
-		loc_wipe(&loc);
-	} else {
-		inode = qb_inode_from_filename (filename);
-		if (!inode)
-			return -EINVAL;
-
-		s->inode = inode_ref(inode);
-	}
-
-	return 0;
-}
-
-
-static int
-qemu_gluster_create (const char *filename, QEMUOptionParameter *options)
-{
-	uint64_t total_size = 0;
-	inode_t *inode = NULL;
-	fd_t *fd = NULL;
-	struct iatt stat = {0, };
-	int ret = 0;
-
-	inode = qb_inode_from_filename (filename);
-	if (!inode)
-		return -EINVAL;
-
-	while (options && options->name) {
-		if (!strcmp(options->name, BLOCK_OPT_SIZE)) {
-			total_size = options->value.n / BDRV_SECTOR_SIZE;
-		}
-		options++;
-	}
-
-	fd = fd_anonymous (inode);
-	if (!fd)
-		return -ENOMEM;
-
-	ret = syncop_fstat (FIRST_CHILD(THIS), fd, &stat, NULL, NULL);
-	if (ret) {
-		fd_unref (fd);
-		return ret;
-	}
-
-	if (stat.ia_size) {
-		/* format ONLY if the filesize is 0 bytes */
-		fd_unref (fd);
-		return -EFBIG;
-	}
-
-	if (total_size) {
-		ret = syncop_ftruncate (FIRST_CHILD(THIS), fd, total_size,
-                                        NULL, NULL);
-		if (ret) {
-			fd_unref (fd);
-			return ret;
-		}
-	}
-
-	fd_unref (fd);
-	return 0;
-}
-
-
-static int
-qemu_gluster_co_readv (BlockDriverState *bs, int64_t sector_num, int nb_sectors,
-		       QEMUIOVector *qiov)
-{
-	fd_t *fd = NULL;
-	off_t offset = 0;
-	size_t size = 0;
-	struct iovec *iov = NULL;
-	int count = 0;
-	struct iobref *iobref = NULL;
-	int ret = 0;
-
-	fd = fd_from_bs (bs);
-	if (!fd)
-		return -EIO;
-
-	offset = sector_num * BDRV_SECTOR_SIZE;
-	size = nb_sectors * BDRV_SECTOR_SIZE;
-
-	ret = syncop_readv (FIRST_CHILD(THIS), fd, size, offset, 0,
-			    &iov, &count, &iobref, NULL, NULL);
-	if (ret < 0)
-		goto out;
-
-	iov_copy (qiov->iov, qiov->niov, iov, count); /* *choke!* */
-
-out:
-	GF_FREE (iov);
-	if (iobref)
-		iobref_unref (iobref);
-	fd_unref (fd);
-	return ret;
-}
-
-
-static int
-qemu_gluster_co_writev (BlockDriverState *bs, int64_t sector_num, int nb_sectors,
-			QEMUIOVector *qiov)
-{
-	fd_t *fd = NULL;
-	off_t offset = 0;
-	size_t size = 0;
-	struct iobref *iobref = NULL;
-	struct iobuf *iobuf = NULL;
-	struct iovec iov = {0, };
-	int ret = -ENOMEM;
-
-	fd = fd_from_bs (bs);
-	if (!fd)
-		return -EIO;
-
-	offset = sector_num * BDRV_SECTOR_SIZE;
-	size = nb_sectors * BDRV_SECTOR_SIZE;
-
-	iobuf = iobuf_get2 (THIS->ctx->iobuf_pool, size);
-	if (!iobuf)
-		goto out;
-
-	iobref = iobref_new ();
-	if (!iobref) {
-		goto out;
-	}
-
-	iobref_add (iobref, iobuf);
-
-	iov_unload (iobuf_ptr (iobuf), qiov->iov, qiov->niov); /* *choke!* */
-
-	iov.iov_base = iobuf_ptr (iobuf);
-	iov.iov_len = size;
-
-	ret = syncop_writev (FIRST_CHILD(THIS), fd, &iov, 1, offset, iobref, 0,
-                             NULL, NULL);
-
-out:
-	if (iobuf)
-		iobuf_unref (iobuf);
-	if (iobref)
-		iobref_unref (iobref);
-	fd_unref (fd);
-	return ret;
-}
-
-
-static int
-qemu_gluster_co_flush (BlockDriverState *bs)
-{
-	fd_t *fd = NULL;
-	int ret = 0;
-
-	fd = fd_from_bs (bs);
-
-	ret = syncop_flush (FIRST_CHILD(THIS), fd, NULL, NULL);
-
-	fd_unref (fd);
-
-	return ret;
-}
-
-
-static int
-qemu_gluster_co_fsync (BlockDriverState *bs)
-{
-	fd_t *fd = NULL;
-	int ret = 0;
-
-	fd = fd_from_bs (bs);
-
-	ret = syncop_fsync (FIRST_CHILD(THIS), fd, 0, NULL, NULL);
-
-	fd_unref (fd);
-
-	return ret;
-}
-
-
-static int
-qemu_gluster_truncate (BlockDriverState *bs, int64_t offset)
-{
-	fd_t *fd = NULL;
-	int ret = 0;
-
-	fd = fd_from_bs (bs);
-
-	ret = syncop_ftruncate (FIRST_CHILD(THIS), fd, offset, NULL, NULL);
-
-	fd_unref (fd);
-
-	return ret;
-}
-
-
-static int64_t
-qemu_gluster_getlength (BlockDriverState *bs)
-{
-	fd_t *fd = NULL;
-	int ret = 0;
-	struct iatt iatt = {0, };
-
-	fd = fd_from_bs (bs);
-
-	ret = syncop_fstat (FIRST_CHILD(THIS), fd, &iatt, NULL, NULL);
-	if (ret < 0)
-		return -1;
-
-	return iatt.ia_size;
-}
-
-
-static int64_t
-qemu_gluster_allocated_file_size (BlockDriverState *bs)
-{
-	fd_t *fd = NULL;
-	int ret = 0;
-	struct iatt iatt = {0, };
-
-	fd = fd_from_bs (bs);
-
-	ret = syncop_fstat (FIRST_CHILD(THIS), fd, &iatt, NULL, NULL);
-	if (ret < 0)
-		return -1;
-
-	return iatt.ia_blocks * 512;
-}
-
-
-static void
-qemu_gluster_close (BlockDriverState *bs)
-{
-	BDRVGlusterState *s = NULL;
-
-	s = bs->opaque;
-
-	inode_unref (s->inode);
-
-	return;
-}
-
-
-static QEMUOptionParameter qemu_gluster_create_options[] = {
-	{
-		.name = BLOCK_OPT_SIZE,
-		.type = OPT_SIZE,
-		.help = "Virtual disk size"
-	},
-	{ NULL }
-};
-
-
-static BlockDriver bdrv_gluster = {
-	.format_name                  = "gluster",
-	.protocol_name                = "gluster",
-	.instance_size                = sizeof(BDRVGlusterState),
-	.bdrv_file_open               = qemu_gluster_open,
-	.bdrv_close                   = qemu_gluster_close,
-	.bdrv_create                  = qemu_gluster_create,
-	.bdrv_getlength               = qemu_gluster_getlength,
-	.bdrv_get_allocated_file_size = qemu_gluster_allocated_file_size,
-	.bdrv_co_readv                = qemu_gluster_co_readv,
-	.bdrv_co_writev               = qemu_gluster_co_writev,
-	.bdrv_co_flush_to_os          = qemu_gluster_co_flush,
-	.bdrv_co_flush_to_disk        = qemu_gluster_co_fsync,
-	.bdrv_truncate                = qemu_gluster_truncate,
-	.create_options               = qemu_gluster_create_options,
-};
-
-
-static void bdrv_gluster_init(void)
-{
-	bdrv_register(&bdrv_gluster);
-}
-
-
-block_init(bdrv_gluster_init);
diff --git a/xlators/features/qemu-block/src/bh-syncop.c b/xlators/features/qemu-block/src/bh-syncop.c
deleted file mode 100644
index e8686f6d4ba..00000000000
--- a/xlators/features/qemu-block/src/bh-syncop.c
+++ /dev/null
@@ -1,48 +0,0 @@
-/*
-  Copyright (c) 2013 Red Hat, Inc. <http://www.redhat.com>
-  This file is part of GlusterFS.
-
-  This file is licensed to you under your choice of the GNU Lesser
-  General Public License, version 3 or any later version (LGPLv3 or
-  later), or the GNU General Public License, version 2 (GPLv2), in all
-  cases as published by the Free Software Foundation.
-*/
-
-
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
-
-#include "glusterfs.h"
-#include "logging.h"
-#include "dict.h"
-#include "xlator.h"
-#include "syncop.h"
-#include "qemu-block-memory-types.h"
-
-#include "block/aio.h"
-
-void
-qemu_bh_schedule (QEMUBH *bh)
-{
-	return;
-}
-
-void
-qemu_bh_cancel (QEMUBH *bh)
-{
-	return;
-}
-
-void
-qemu_bh_delete (QEMUBH *bh)
-{
-
-}
-
-QEMUBH *
-qemu_bh_new (QEMUBHFunc *cb, void *opaque)
-{
-	return NULL;
-}
diff --git a/xlators/features/qemu-block/src/clock-timer.c b/xlators/features/qemu-block/src/clock-timer.c
deleted file mode 100644
index fcbec6ad1cd..00000000000
--- a/xlators/features/qemu-block/src/clock-timer.c
+++ /dev/null
@@ -1,60 +0,0 @@
-/*
-  Copyright (c) 2013 Red Hat, Inc. <http://www.redhat.com>
-  This file is part of GlusterFS.
-
-  This file is licensed to you under your choice of the GNU Lesser
-  General Public License, version 3 or any later version (LGPLv3 or
-  later), or the GNU General Public License, version 2 (GPLv2), in all
-  cases as published by the Free Software Foundation.
-*/
-
-
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
-
-#include "glusterfs.h"
-#include "logging.h"
-#include "dict.h"
-#include "xlator.h"
-#include "syncop.h"
-#include "qemu-block-memory-types.h"
-
-#include "qemu/timer.h"
-
-QEMUClock *vm_clock;
-int use_rt_clock = 0;
-
-QEMUTimer *qemu_new_timer (QEMUClock *clock, int scale,
-			   QEMUTimerCB *cb, void *opaque)
-{
-	return NULL;
-}
-
-int64_t qemu_get_clock_ns (QEMUClock *clock)
-{
-	return 0;
-}
-
-void qemu_mod_timer (QEMUTimer *ts, int64_t expire_time)
-{
-	return;
-}
-
-void qemu_free_timer (QEMUTimer *ts)
-{
-
-}
-
-void qemu_del_timer (QEMUTimer *ts)
-{
-
-}
-
-bool qemu_aio_wait()
-{
-	synctask_wake (synctask_get());
-	synctask_yield (synctask_get());
-	return 0;
-}
diff --git a/xlators/features/qemu-block/src/coroutine-synctask.c b/xlators/features/qemu-block/src/coroutine-synctask.c
deleted file mode 100644
index e43988a953f..00000000000
--- a/xlators/features/qemu-block/src/coroutine-synctask.c
+++ /dev/null
@@ -1,116 +0,0 @@
-/*
-  Copyright (c) 2013 Red Hat, Inc. <http://www.redhat.com>
-  This file is part of GlusterFS.
-
-  This file is licensed to you under your choice of the GNU Lesser
-  General Public License, version 3 or any later version (LGPLv3 or
-  later), or the GNU General Public License, version 2 (GPLv2), in all
-  cases as published by the Free Software Foundation.
-*/
-
-
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
-
-#include "glusterfs.h"
-#include "logging.h"
-#include "dict.h"
-#include "xlator.h"
-#include "syncop.h"
-#include "qemu-block-memory-types.h"
-
-#include "qemu-block.h"
-
-/*
- * This code serves as the bridge from the main glusterfs context to the qemu
- * coroutine context via synctask. We create a single threaded syncenv with a
- * single synctask responsible for processing a queue of coroutines. The qemu
- * code invoked from within the synctask function handlers uses the ucontext
- * coroutine implementation and scheduling logic internal to qemu. This
- * effectively donates a thread of execution to qemu and its internal coroutine
- * management.
- *
- * NOTE: The existence of concurrent synctasks has proven quite racy with regard
- * to qemu coroutine management, particularly related to the lifecycle
- * differences with top-level synctasks and internally created coroutines and
- * interactions with qemu-internal queues (and locks, in turn). We explicitly
- * disallow this scenario, via the queue, until it is more well supported.
- */
-
-static struct {
-	struct list_head	queue;
-	gf_lock_t		lock;
-	struct synctask		*task;
-} qb_co;
-
-static void
-init_qbco()
-{
-	INIT_LIST_HEAD(&qb_co.queue);
-	LOCK_INIT(&qb_co.lock);
-}
-
-static int
-synctask_nop_cbk (int ret, call_frame_t *frame, void *opaque)
-{
-	return 0;
-}
-
-static int
-qb_synctask_wrap (void *opaque)
-{
-	qb_local_t *qb_local, *tmp;
-
-	LOCK(&qb_co.lock);
-
-	while (!list_empty(&qb_co.queue)) {
-		list_for_each_entry_safe(qb_local, tmp, &qb_co.queue, list) {
-			list_del_init(&qb_local->list);
-			break;
-		}
-
-		UNLOCK(&qb_co.lock);
-
-		qb_local->synctask_fn(qb_local);
-		/* qb_local is now unwound and gone! */
-
-		LOCK(&qb_co.lock);
-	}
-
-	qb_co.task = NULL;
-
-	UNLOCK(&qb_co.lock);
-
-	return 0;
-}
-
-int
-qb_coroutine (call_frame_t *frame, synctask_fn_t fn)
-{
-	qb_local_t *qb_local = NULL;
-	qb_conf_t *qb_conf = NULL;
-	static int init = 0;
-
-	qb_local = frame->local;
-	qb_local->synctask_fn = fn;
-	qb_conf = frame->this->private;
-
-	if (!init) {
-		init = 1;
-		init_qbco();
-	}
-
-	LOCK(&qb_co.lock);
-
-	if (!qb_co.task)
-		qb_co.task = synctask_create(qb_conf->env, qb_synctask_wrap,
-					     synctask_nop_cbk, frame, NULL);
-
-	list_add_tail(&qb_local->list, &qb_co.queue);
-
-	UNLOCK(&qb_co.lock);
-
-	return 0;
-}
diff --git a/xlators/features/qemu-block/src/monitor-logging.c b/xlators/features/qemu-block/src/monitor-logging.c
deleted file mode 100644
index d37c37f0f29..00000000000
--- a/xlators/features/qemu-block/src/monitor-logging.c
+++ /dev/null
@@ -1,50 +0,0 @@
-/*
-  Copyright (c) 2013 Red Hat, Inc. <http://www.redhat.com>
-  This file is part of GlusterFS.
-
-  This file is licensed to you under your choice of the GNU Lesser
-  General Public License, version 3 or any later version (LGPLv3 or
-  later), or the GNU General Public License, version 2 (GPLv2), in all
-  cases as published by the Free Software Foundation.
-*/
-
-
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
-
-#include "glusterfs.h"
-#include "logging.h"
-#include "dict.h"
-#include "xlator.h"
-#include "qemu-block-memory-types.h"
-
-#include "block/block_int.h"
-
-Monitor *cur_mon;
-
-int
-monitor_cur_is_qmp()
-{
-	/* No QMP support here */
-	return 0;
-}
-
-void
-monitor_set_error (Monitor *mon, QError *qerror)
-{
-	/* NOP here */
-	return;
-}
-
-
-void
-monitor_vprintf(Monitor *mon, const char *fmt, va_list ap)
-{
-	char buf[4096];
-
-	vsnprintf(buf, sizeof(buf), fmt, ap);
-
-	gf_log (THIS->name, GF_LOG_ERROR, "%s", buf);
-}
diff --git a/xlators/features/qemu-block/src/qb-coroutines.c b/xlators/features/qemu-block/src/qb-coroutines.c
deleted file mode 100644
index 4457576314e..00000000000
--- a/xlators/features/qemu-block/src/qb-coroutines.c
+++ /dev/null
@@ -1,667 +0,0 @@
-/*
-  Copyright (c) 2013 Red Hat, Inc. <http://www.redhat.com>
-  This file is part of GlusterFS.
-
-  This file is licensed to you under your choice of the GNU Lesser
-  General Public License, version 3 or any later version (LGPLv3 or
-  later), or the GNU General Public License, version 2 (GPLv2), in all
-  cases as published by the Free Software Foundation.
-*/
-
-
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
-
-#include "glusterfs.h"
-#include "logging.h"
-#include "dict.h"
-#include "xlator.h"
-#include "inode.h"
-#include "call-stub.h"
-#include "defaults.h"
-#include "qemu-block-memory-types.h"
-#include "qemu-block.h"
-#include "qb-coroutines.h"
-
-
-int
-qb_format_and_resume (void *opaque)
-{
-	qb_local_t *local = NULL;
-	call_frame_t *frame = NULL;
-	call_stub_t *stub = NULL;
-	inode_t *inode = NULL;
-	char filename[64];
-	char base_filename[128];
-	int use_base = 0;
-	qb_inode_t *qb_inode = NULL;
-	Error *local_err = NULL;
-	fd_t *fd = NULL;
-	dict_t *xattr = NULL;
-	qb_conf_t *qb_conf = NULL;
-	int ret = -1;
-
-	local = opaque;
-	frame = local->frame;
-	stub = local->stub;
-	inode = local->inode;
-	qb_conf = frame->this->private;
-
-	qb_inode_to_filename (inode, filename, 64);
-
-	qb_inode = qb_inode_ctx_get (frame->this, inode);
-
-	/*
-	 * See if the caller specified a backing image.
-	 */
-	if (!gf_uuid_is_null(qb_inode->backing_gfid) || qb_inode->backing_fname) {
-		loc_t loc = {0,};
-		char gfid_str[64];
-		struct iatt buf;
-
-		if (!gf_uuid_is_null(qb_inode->backing_gfid)) {
-			loc.inode = inode_find(qb_conf->root_inode->table,
-					qb_inode->backing_gfid);
-			if (!loc.inode) {
-				loc.inode = inode_new(qb_conf->root_inode->table);
-				gf_uuid_copy(loc.inode->gfid,
-					qb_inode->backing_gfid);
-			}
-			gf_uuid_copy(loc.gfid, loc.inode->gfid);
-		} else if (qb_inode->backing_fname) {
-			loc.inode = inode_new(qb_conf->root_inode->table);
-			loc.name = qb_inode->backing_fname;
-			loc.parent = inode_parent(inode, NULL, NULL);
-			loc_path(&loc, loc.name);
-		}
-
-		/*
-		 * Lookup the backing image. Verify existence and/or get the
-		 * gfid if we don't already have it.
-		 */
-		ret = syncop_lookup(FIRST_CHILD(frame->this), &loc, &buf, NULL,
-                                    NULL, NULL);
-		GF_FREE(qb_inode->backing_fname);
-		if (ret) {
-			loc_wipe(&loc);
-			ret = -ret;
-			goto err;
-		}
-
-		gf_uuid_copy(qb_inode->backing_gfid, buf.ia_gfid);
-		loc_wipe(&loc);
-
-		/*
-		 * We pass the filename of the backing image into the qemu block
-		 * subsystem as the associated gfid. This is embedded into the
-		 * clone image and passed along to the gluster bdrv backend when
-		 * the block subsystem needs to operate on the backing image on
-		 * behalf of the clone.
-		 */
-		gf_uuid_unparse(qb_inode->backing_gfid, gfid_str);
-		snprintf(base_filename, sizeof(base_filename),
-			 "gluster://gfid:%s", gfid_str);
-		use_base = 1;
-	}
-
-	bdrv_img_create (filename, qb_inode->fmt,
-			 use_base ? base_filename : NULL, 0, 0, qb_inode->size,
-			 0, &local_err, true);
-
-	if (error_is_set (&local_err)) {
-		gf_log (frame->this->name, GF_LOG_ERROR, "%s",
-			error_get_pretty (local_err));
-		error_free (local_err);
-		QB_STUB_UNWIND (stub, -1, EIO);
-		return 0;
-	}
-
-	fd = fd_anonymous (inode);
-	if (!fd) {
-		gf_log (frame->this->name, GF_LOG_ERROR,
-			"could not create anonymous fd for %s",
-			uuid_utoa (inode->gfid));
-		QB_STUB_UNWIND (stub, -1, ENOMEM);
-		return 0;
-	}
-
-	xattr = dict_new ();
-	if (!xattr) {
-		gf_log (frame->this->name, GF_LOG_ERROR,
-			"could not allocate xattr dict for %s",
-			uuid_utoa (inode->gfid));
-		QB_STUB_UNWIND (stub, -1, ENOMEM);
-		fd_unref (fd);
-		return 0;
-	}
-
-	ret = dict_set_str (xattr, qb_conf->qb_xattr_key, local->fmt);
-	if (ret) {
-		gf_log (frame->this->name, GF_LOG_ERROR,
-			"could not dict_set for %s",
-			uuid_utoa (inode->gfid));
-		QB_STUB_UNWIND (stub, -1, ENOMEM);
-		fd_unref (fd);
-		dict_unref (xattr);
-		return 0;
-	}
-
-	ret = syncop_fsetxattr (FIRST_CHILD(THIS), fd, xattr, 0, NULL, NULL);
-	if (ret) {
-		gf_log (frame->this->name, GF_LOG_ERROR,
-			"failed to setxattr for %s",
-			uuid_utoa (inode->gfid));
-		QB_STUB_UNWIND (stub, -1, -ret);
-		fd_unref (fd);
-		dict_unref (xattr);
-		return 0;
-	}
-
-	fd_unref (fd);
-	dict_unref (xattr);
-
-	QB_STUB_UNWIND (stub, 0, 0);
-
-	return 0;
-
-err:
-	QB_STUB_UNWIND(stub, -1, ret);
-	return 0;
-}
-
-
-static BlockDriverState *
-qb_bs_create (inode_t *inode, const char *fmt)
-{
-	char filename[64];
-	BlockDriverState *bs = NULL;
-	BlockDriver *drv = NULL;
-	int op_errno = 0;
-	int ret = 0;
-
-	bs = bdrv_new (uuid_utoa (inode->gfid));
-	if (!bs) {
-		op_errno = ENOMEM;
-		gf_log (THIS->name, GF_LOG_ERROR,
-			"could not allocate @bdrv for gfid:%s",
-			uuid_utoa (inode->gfid));
-		goto err;
-	}
-
-	drv = bdrv_find_format (fmt);
-	if (!drv) {
-		op_errno = EINVAL;
-		gf_log (THIS->name, GF_LOG_ERROR,
-			"Unknown file format: %s for gfid:%s",
-			fmt, uuid_utoa (inode->gfid));
-		goto err;
-	}
-
-	qb_inode_to_filename (inode, filename, 64);
-
-	ret = bdrv_open (bs, filename, NULL, BDRV_O_RDWR, drv);
-	if (ret < 0) {
-		op_errno = -ret;
-		gf_log (THIS->name, GF_LOG_ERROR,
-			"Unable to bdrv_open() gfid:%s (%s)",
-			uuid_utoa (inode->gfid), strerror (op_errno));
-		goto err;
-	}
-
-	return bs;
-err:
-	errno = op_errno;
-	return NULL;
-}
-
-
-int
-qb_co_open (void *opaque)
-{
-	qb_local_t *local = NULL;
-	call_frame_t *frame = NULL;
-	call_stub_t *stub = NULL;
-	inode_t *inode = NULL;
-	qb_inode_t *qb_inode = NULL;
-
-	local = opaque;
-	frame = local->frame;
-	stub = local->stub;
-	inode = local->inode;
-
-	qb_inode = qb_inode_ctx_get (frame->this, inode);
-	if (!qb_inode->bs) {
-		/* FIXME: we need locks around this when
-		   enabling multithreaded syncop/coroutine
-		   for qemu-block
-		*/
-
-		qb_inode->bs = qb_bs_create (inode, qb_inode->fmt);
-		if (!qb_inode->bs) {
-			QB_STUB_UNWIND (stub, -1, errno);
-			return 0;
-		}
-	}
-	qb_inode->refcnt++;
-
-	QB_STUB_RESUME (stub);
-
-	return 0;
-}
-
-
-int
-qb_co_writev (void *opaque)
-{
-	qb_local_t *local = NULL;
-	call_frame_t *frame = NULL;
-	call_stub_t *stub = NULL;
-	inode_t *inode = NULL;
-	qb_inode_t *qb_inode = NULL;
-	QEMUIOVector qiov = {0, };
-	int ret = 0;
-
-	local = opaque;
-	frame = local->frame;
-	stub = local->stub;
-	inode = local->inode;
-
-	qb_inode = qb_inode_ctx_get (frame->this, inode);
-	if (!qb_inode->bs) {
-		/* FIXME: we need locks around this when
-		   enabling multithreaded syncop/coroutine
-		   for qemu-block
-		*/
-
-		qb_inode->bs = qb_bs_create (inode, qb_inode->fmt);
-		if (!qb_inode->bs) {
-			QB_STUB_UNWIND (stub, -1, errno);
-			return 0;
-		}
-	}
-
-	qemu_iovec_init_external (&qiov, stub->args.vector, stub->args.count);
-
-	ret = bdrv_pwritev (qb_inode->bs, stub->args.offset, &qiov);
-
-	if (ret < 0) {
-		QB_STUB_UNWIND (stub, -1, -ret);
-	} else {
-		QB_STUB_UNWIND (stub, ret, 0);
-	}
-
-	return 0;
-}
-
-
-int
-qb_co_readv (void *opaque)
-{
-	qb_local_t *local = NULL;
-	call_frame_t *frame = NULL;
-	call_stub_t *stub = NULL;
-	inode_t *inode = NULL;
-	qb_inode_t *qb_inode = NULL;
-	struct iobuf *iobuf = NULL;
-	struct iobref *iobref = NULL;
-	struct iovec iov = {0, };
-	int ret = 0;
-
-	local = opaque;
-	frame = local->frame;
-	stub = local->stub;
-	inode = local->inode;
-
-	qb_inode = qb_inode_ctx_get (frame->this, inode);
-	if (!qb_inode->bs) {
-		/* FIXME: we need locks around this when
-		   enabling multithreaded syncop/coroutine
-		   for qemu-block
-		*/
-
-		qb_inode->bs = qb_bs_create (inode, qb_inode->fmt);
-		if (!qb_inode->bs) {
-			QB_STUB_UNWIND (stub, -1, errno);
-			return 0;
-		}
-	}
-
-	if (stub->args.offset >= qb_inode->size) {
-		QB_STUB_UNWIND (stub, 0, 0);
-		return 0;
-	}
-
-	iobuf = iobuf_get2 (frame->this->ctx->iobuf_pool, stub->args.size);
-	if (!iobuf) {
-		QB_STUB_UNWIND (stub, -1, ENOMEM);
-		return 0;
-	}
-
-	iobref = iobref_new ();
-	if (!iobref) {
-		QB_STUB_UNWIND (stub, -1, ENOMEM);
-		iobuf_unref (iobuf);
-		return 0;
-	}
-
-	if (iobref_add (iobref, iobuf) < 0) {
-		iobuf_unref (iobuf);
-		iobref_unref (iobref);
-		QB_STUB_UNWIND (stub, -1, ENOMEM);
-		return 0;
-	}
-
-	ret = bdrv_pread (qb_inode->bs, stub->args.offset, iobuf_ptr (iobuf),
-			  stub->args.size);
-
-	if (ret < 0) {
-		QB_STUB_UNWIND (stub, -1, -ret);
-		iobref_unref (iobref);
-		return 0;
-	}
-
-	iov.iov_base = iobuf_ptr (iobuf);
-	iov.iov_len = ret;
-
-	stub->args_cbk.vector = iov_dup (&iov, 1);
-	stub->args_cbk.count = 1;
-	stub->args_cbk.iobref = iobref;
-
-	QB_STUB_UNWIND (stub, ret, 0);
-
-	return 0;
-}
-
-
-int
-qb_co_fsync (void *opaque)
-{
-	qb_local_t *local = NULL;
-	call_frame_t *frame = NULL;
-	call_stub_t *stub = NULL;
-	inode_t *inode = NULL;
-	qb_inode_t *qb_inode = NULL;
-	int ret = 0;
-
-	local = opaque;
-	frame = local->frame;
-	stub = local->stub;
-	inode = local->inode;
-
-	qb_inode = qb_inode_ctx_get (frame->this, inode);
-	if (!qb_inode->bs) {
-		/* FIXME: we need locks around this when
-		   enabling multithreaded syncop/coroutine
-		   for qemu-block
-		*/
-
-		qb_inode->bs = qb_bs_create (inode, qb_inode->fmt);
-		if (!qb_inode->bs) {
-			QB_STUB_UNWIND (stub, -1, errno);
-			return 0;
-		}
-	}
-
-	ret = bdrv_flush (qb_inode->bs);
-
-	if (ret < 0) {
-		QB_STUB_UNWIND (stub, -1, -ret);
-	} else {
-		QB_STUB_UNWIND (stub, ret, 0);
-	}
-
-	return 0;
-}
-
-
-static void
-qb_update_size_xattr (xlator_t *this, fd_t *fd, const char *fmt, off_t offset)
-{
-	char val[QB_XATTR_VAL_MAX];
-	qb_conf_t *qb_conf = NULL;
-	dict_t *xattr = NULL;
-
-	qb_conf = this->private;
-
-	snprintf (val, QB_XATTR_VAL_MAX, "%s:%llu",
-		  fmt, (long long unsigned) offset);
-
-	xattr = dict_new ();
-	if (!xattr)
-		return;
-
-	if (dict_set_str (xattr, qb_conf->qb_xattr_key, val) != 0) {
-		dict_unref (xattr);
-		return;
-	}
-
-	syncop_fsetxattr (FIRST_CHILD(this), fd, xattr, 0, NULL, NULL);
-	dict_unref (xattr);
-}
-
-
-int
-qb_co_truncate (void *opaque)
-{
-	qb_local_t *local = NULL;
-	call_frame_t *frame = NULL;
-	call_stub_t *stub = NULL;
-	inode_t *inode = NULL;
-	qb_inode_t *qb_inode = NULL;
-	int ret = 0;
-	off_t offset = 0;
-	xlator_t *this = NULL;
-
-	this = THIS;
-
-	local = opaque;
-	frame = local->frame;
-	stub = local->stub;
-	inode = local->inode;
-
-	qb_inode = qb_inode_ctx_get (frame->this, inode);
-	if (!qb_inode->bs) {
-		/* FIXME: we need locks around this when
-		   enabling multithreaded syncop/coroutine
-		   for qemu-block
-		*/
-
-		qb_inode->bs = qb_bs_create (inode, qb_inode->fmt);
-		if (!qb_inode->bs) {
-			QB_STUB_UNWIND (stub, -1, errno);
-			return 0;
-		}
-	}
-
-	ret = syncop_fstat (FIRST_CHILD(this), local->fd,
-                            &stub->args_cbk.prestat, NULL, NULL);
-        if (ret < 0)
-                goto out;
-	stub->args_cbk.prestat.ia_size = qb_inode->size;
-
-	ret = bdrv_truncate (qb_inode->bs, stub->args.offset);
-	if (ret < 0)
-		goto out;
-
-	offset = bdrv_getlength (qb_inode->bs);
-
-	qb_inode->size = offset;
-
-	ret = syncop_fstat (FIRST_CHILD(this), local->fd,
-                            &stub->args_cbk.poststat, NULL, NULL);
-        if (ret < 0)
-                goto out;
-	stub->args_cbk.poststat.ia_size = qb_inode->size;
-
-	qb_update_size_xattr (this, local->fd, qb_inode->fmt, qb_inode->size);
-
-out:
-	if (ret < 0) {
-		QB_STUB_UNWIND (stub, -1, -ret);
-	} else {
-		QB_STUB_UNWIND (stub, ret, 0);
-	}
-
-	return 0;
-}
-
-
-int
-qb_co_close (void *opaque)
-{
-	qb_local_t *local = NULL;
-	call_frame_t *frame = NULL;
-	inode_t *inode = NULL;
-	qb_inode_t *qb_inode = NULL;
-	BlockDriverState *bs = NULL;
-
-	local = opaque;
-	inode = local->inode;
-
-	qb_inode = qb_inode_ctx_get (THIS, inode);
-
-	if (!--qb_inode->refcnt) {
-		bs = qb_inode->bs;
-		qb_inode->bs = NULL;
-		bdrv_delete (bs);
-	}
-
-	frame = local->frame;
-	frame->local = NULL;
-	qb_local_free (THIS, local);
-	STACK_DESTROY (frame->root);
-
-	return 0;
-}
-
-
-int
-qb_snapshot_create (void *opaque)
-{
-	qb_local_t *local = NULL;
-	call_frame_t *frame = NULL;
-	call_stub_t *stub = NULL;
-	inode_t *inode = NULL;
-	qb_inode_t *qb_inode = NULL;
-	QEMUSnapshotInfo sn;
-	struct timeval tv = {0, };
-	int ret = 0;
-
-	local = opaque;
-	frame = local->frame;
-	stub = local->stub;
-	inode = local->inode;
-
-	qb_inode = qb_inode_ctx_get (frame->this, inode);
-	if (!qb_inode->bs) {
-		/* FIXME: we need locks around this when
-		   enabling multithreaded syncop/coroutine
-		   for qemu-block
-		*/
-
-		qb_inode->bs = qb_bs_create (inode, qb_inode->fmt);
-		if (!qb_inode->bs) {
-			QB_STUB_UNWIND (stub, -1, errno);
-			return 0;
-		}
-	}
-
-	memset (&sn, 0, sizeof (sn));
-        pstrcpy (sn.name, sizeof(sn.name), local->name);
-        gettimeofday (&tv, NULL);
-        sn.date_sec = tv.tv_sec;
-        sn.date_nsec = tv.tv_usec * 1000;
-
-        ret = bdrv_snapshot_create (qb_inode->bs, &sn);
-	if (ret < 0) {
-		QB_STUB_UNWIND (stub, -1, -ret);
-	} else {
-		QB_STUB_UNWIND (stub, ret, 0);
-	}
-
-	return 0;
-}
-
-
-int
-qb_snapshot_delete (void *opaque)
-{
-	qb_local_t *local = NULL;
-	call_frame_t *frame = NULL;
-	call_stub_t *stub = NULL;
-	inode_t *inode = NULL;
-	qb_inode_t *qb_inode = NULL;
-	int ret = 0;
-
-	local = opaque;
-	frame = local->frame;
-	stub = local->stub;
-	inode = local->inode;
-
-	qb_inode = qb_inode_ctx_get (frame->this, inode);
-	if (!qb_inode->bs) {
-		/* FIXME: we need locks around this when
-		   enabling multithreaded syncop/coroutine
-		   for qemu-block
-		*/
-
-		qb_inode->bs = qb_bs_create (inode, qb_inode->fmt);
-		if (!qb_inode->bs) {
-			QB_STUB_UNWIND (stub, -1, errno);
-			return 0;
-		}
-	}
-
-	ret = bdrv_snapshot_delete (qb_inode->bs, local->name);
-
-	if (ret < 0) {
-		QB_STUB_UNWIND (stub, -1, -ret);
-	} else {
-		QB_STUB_UNWIND (stub, ret, 0);
-	}
-
-	return 0;
-}
-
-
-int
-qb_snapshot_goto (void *opaque)
-{
-	qb_local_t *local = NULL;
-	call_frame_t *frame = NULL;
-	call_stub_t *stub = NULL;
-	inode_t *inode = NULL;
-	qb_inode_t *qb_inode = NULL;
-	int ret = 0;
-
-	local = opaque;
-	frame = local->frame;
-	stub = local->stub;
-	inode = local->inode;
-
-	qb_inode = qb_inode_ctx_get (frame->this, inode);
-	if (!qb_inode->bs) {
-		/* FIXME: we need locks around this when
-		   enabling multithreaded syncop/coroutine
-		   for qemu-block
-		*/
-
-		qb_inode->bs = qb_bs_create (inode, qb_inode->fmt);
-		if (!qb_inode->bs) {
-			QB_STUB_UNWIND (stub, -1, errno);
-			return 0;
-		}
-	}
-
-	ret = bdrv_snapshot_goto (qb_inode->bs, local->name);
-
-	if (ret < 0) {
-		QB_STUB_UNWIND (stub, -1, -ret);
-	} else {
-		QB_STUB_UNWIND (stub, ret, 0);
-	}
-
-	return 0;
-}
diff --git a/xlators/features/qemu-block/src/qb-coroutines.h b/xlators/features/qemu-block/src/qb-coroutines.h
deleted file mode 100644
index 583319f3b06..00000000000
--- a/xlators/features/qemu-block/src/qb-coroutines.h
+++ /dev/null
@@ -1,30 +0,0 @@
-/*
-  Copyright (c) 2013 Red Hat, Inc. <http://www.redhat.com>
-  This file is part of GlusterFS.
-
-  This file is licensed to you under your choice of the GNU Lesser
-  General Public License, version 3 or any later version (LGPLv3 or
-  later), or the GNU General Public License, version 2 (GPLv2), in all
-  cases as published by the Free Software Foundation.
-*/
-
-#ifndef __QB_COROUTINES_H
-#define __QB_COROUTINES_H
-
-#include "syncop.h"
-#include "call-stub.h"
-#include "block/block_int.h"
-#include "monitor/monitor.h"
-
-int qb_format_and_resume (void *opaque);
-int qb_snapshot_create (void *opaque);
-int qb_snapshot_delete (void *opaque);
-int qb_snapshot_goto (void *opaque);
-int qb_co_open (void *opaque);
-int qb_co_close (void *opaque);
-int qb_co_writev (void *opaque);
-int qb_co_readv (void *opaque);
-int qb_co_fsync (void *opaque);
-int qb_co_truncate (void *opaque);
-
-#endif /* __QB_COROUTINES_H */
diff --git a/xlators/features/qemu-block/src/qemu-block-memory-types.h b/xlators/features/qemu-block/src/qemu-block-memory-types.h
deleted file mode 100644
index 267b3893fed..00000000000
--- a/xlators/features/qemu-block/src/qemu-block-memory-types.h
+++ /dev/null
@@ -1,25 +0,0 @@
-/*
-  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
-  This file is part of GlusterFS.
-
-  This file is licensed to you under your choice of the GNU Lesser
-  General Public License, version 3 or any later version (LGPLv3 or
-  later), or the GNU General Public License, version 2 (GPLv2), in all
-  cases as published by the Free Software Foundation.
-*/
-
-
-#ifndef __QB_MEM_TYPES_H__
-#define __QB_MEM_TYPES_H__
-
-#include "mem-types.h"
-
-enum gf_qb_mem_types_ {
-        gf_qb_mt_qb_conf_t   = gf_common_mt_end + 1,
-        gf_qb_mt_qb_inode_t,
-        gf_qb_mt_qb_local_t,
-	gf_qb_mt_coroutinesynctask_t,
-        gf_qb_mt_end
-};
-#endif
-
diff --git a/xlators/features/qemu-block/src/qemu-block.c b/xlators/features/qemu-block/src/qemu-block.c
deleted file mode 100644
index 8cc78ff165c..00000000000
--- a/xlators/features/qemu-block/src/qemu-block.c
+++ /dev/null
@@ -1,1139 +0,0 @@
-/*
-  Copyright (c) 2013 Red Hat, Inc. <http://www.redhat.com>
-  This file is part of GlusterFS.
-
-  This file is licensed to you under your choice of the GNU Lesser
-  General Public License, version 3 or any later version (LGPLv3 or
-  later), or the GNU General Public License, version 2 (GPLv2), in all
-  cases as published by the Free Software Foundation.
-*/
-
-
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
-
-#include "glusterfs.h"
-#include "logging.h"
-#include "dict.h"
-#include "xlator.h"
-#include "inode.h"
-#include "call-stub.h"
-#include "defaults.h"
-#include "qemu-block-memory-types.h"
-#include "qemu-block.h"
-#include "qb-coroutines.h"
-
-
-qb_inode_t *
-__qb_inode_ctx_get (xlator_t *this, inode_t *inode)
-{
-        uint64_t    value    = 0;
-        qb_inode_t *qb_inode = NULL;
-
-        __inode_ctx_get (inode, this, &value);
-        qb_inode = (qb_inode_t *)(unsigned long) value;
-
-        return qb_inode;
-}
-
-
-qb_inode_t *
-qb_inode_ctx_get (xlator_t *this, inode_t *inode)
-{
-        qb_inode_t *qb_inode = NULL;
-
-        LOCK (&inode->lock);
-        {
-                qb_inode = __qb_inode_ctx_get (this, inode);
-        }
-        UNLOCK (&inode->lock);
-
-        return qb_inode;
-}
-
-
-qb_inode_t *
-qb_inode_ctx_del (xlator_t *this, inode_t *inode)
-{
-        uint64_t    value    = 0;
-        qb_inode_t *qb_inode = NULL;
-
-        inode_ctx_del (inode, this, &value);
-        qb_inode = (qb_inode_t *)(unsigned long) value;
-
-        return qb_inode;
-}
-
-
-int
-qb_inode_cleanup (xlator_t *this, inode_t *inode, int warn)
-{
-	qb_inode_t *qb_inode = NULL;
-
-	qb_inode = qb_inode_ctx_del (this, inode);
-
-	if (!qb_inode)
-		return 0;
-
-	if (warn)
-		gf_log (this->name, GF_LOG_WARNING,
-			"inode %s no longer block formatted",
-			uuid_utoa (inode->gfid));
-
-	/* free (qb_inode->bs); */
-
-	GF_FREE (qb_inode);
-
-	return 0;
-}
-
-
-int
-qb_iatt_fixup (xlator_t *this, inode_t *inode, struct iatt *iatt)
-{
-	qb_inode_t *qb_inode = NULL;
-
-	qb_inode = qb_inode_ctx_get (this, inode);
-	if (!qb_inode)
-		return 0;
-
-	iatt->ia_size = qb_inode->size;
-
-	return 0;
-}
-
-
-int
-qb_format_extract (xlator_t *this, char *format, inode_t *inode)
-{
-	char       *s, *save;
-	uint64_t    size = 0;
-	char        fmt[QB_XATTR_VAL_MAX+1] = {0, };
-	qb_inode_t *qb_inode = NULL;
-	char *formatstr = NULL;
-	uuid_t gfid = {0,};
-	char gfid_str[64] = {0,};
-	int ret;
-
-	strncpy(fmt, format, QB_XATTR_VAL_MAX);
-
-	s = strtok_r(fmt, ":", &save);
-	if (!s)
-		goto invalid;
-	formatstr = gf_strdup(s);
-
-	s = strtok_r(NULL, ":", &save);
-	if (!s)
-		goto invalid;
-	if (gf_string2bytesize (s, &size))
-		goto invalid;
-	if (!size)
-		goto invalid;
-
-	s = strtok_r(NULL, "\0", &save);
-	if (s && !strncmp(s, "<gfid:", strlen("<gfid:"))) {
-		/*
-		 * Check for valid gfid backing image specifier.
-		 */
-		if (strlen(s) + 1 > sizeof(gfid_str))
-			goto invalid;
-		ret = sscanf(s, "<gfid:%[^>]s", gfid_str);
-		if (ret == 1) {
-			ret = gf_uuid_parse(gfid_str, gfid);
-			if (ret < 0)
-				goto invalid;
-		}
-	}
-
-	qb_inode = qb_inode_ctx_get (this, inode);
-	if (!qb_inode)
-		qb_inode = GF_CALLOC (1, sizeof (*qb_inode),
-				      gf_qb_mt_qb_inode_t);
-	if (!qb_inode) {
-		GF_FREE(formatstr);
-		return ENOMEM;
-	}
-
-	strncpy(qb_inode->fmt, formatstr, QB_XATTR_VAL_MAX);
-	qb_inode->size = size;
-
-	/*
-	 * If a backing gfid was not specified, interpret any remaining bytes
-	 * associated with a backing image as a filename local to the parent
-	 * directory. The format processing will validate further.
-	 */
-	if (!gf_uuid_is_null(gfid))
-		gf_uuid_copy(qb_inode->backing_gfid, gfid);
-	else if (s)
-		qb_inode->backing_fname = gf_strdup(s);
-
-	inode_ctx_set (inode, this, (void *)&qb_inode);
-
-	GF_FREE(formatstr);
-
-	return 0;
-
-invalid:
-	GF_FREE(formatstr);
-
-	gf_log (this->name, GF_LOG_WARNING,
-		"invalid format '%s' in inode %s", format,
-		uuid_utoa (inode->gfid));
-	return EINVAL;
-}
-
-
-void
-qb_local_free (xlator_t *this, qb_local_t *local)
-{
-	if (local->inode)
-		inode_unref (local->inode);
-	if (local->fd)
-		fd_unref (local->fd);
-	GF_FREE (local);
-}
-
-
-int
-qb_local_init (call_frame_t *frame)
-{
-	qb_local_t *qb_local = NULL;
-
-	qb_local = GF_CALLOC (1, sizeof (*qb_local), gf_qb_mt_qb_local_t);
-	if (!qb_local)
-		return -1;
-	INIT_LIST_HEAD(&qb_local->list);
-
-	qb_local->frame = frame;
-	frame->local = qb_local;
-
-	return 0;
-}
-
-
-int
-qb_lookup_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-	       int op_ret, int op_errno, inode_t *inode, struct iatt *buf,
-	       dict_t *xdata, struct iatt *postparent)
-{
-	char *format = NULL;
-	qb_conf_t *conf = NULL;
-
-	conf = this->private;
-
-	if (op_ret == -1)
-		goto out;
-
-	/*
-	 * Cache the root inode for dealing with backing images. The format
-	 * coroutine and the gluster qemu backend driver both use the root inode
-	 * table to verify and/or redirect I/O to the backing image via
-	 * anonymous fd's.
-	 */
-	if (!conf->root_inode && __is_root_gfid(inode->gfid))
-		conf->root_inode = inode_ref(inode);
-
-	if (!xdata)
-		goto out;
-
-	if (dict_get_str (xdata, conf->qb_xattr_key, &format))
-		goto out;
-
-	if (!format) {
-		qb_inode_cleanup (this, inode, 1);
-		goto out;
-	}
-
-	op_errno = qb_format_extract (this, format, inode);
-	if (op_errno)
-		op_ret = -1;
-
-	qb_iatt_fixup (this, inode, buf);
-out:
-	QB_STACK_UNWIND (lookup, frame, op_ret, op_errno, inode, buf,
-			 xdata, postparent);
-	return 0;
-}
-
-
-int
-qb_lookup (call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xdata)
-{
-	qb_conf_t *conf = NULL;
-
-	conf = this->private;
-
-	xdata = xdata ? dict_ref (xdata) : dict_new ();
-
-	if (!xdata)
-		goto enomem;
-
-	if (dict_set_int32 (xdata, conf->qb_xattr_key, 0))
-		goto enomem;
-
-	STACK_WIND (frame, qb_lookup_cbk, FIRST_CHILD(this),
-		    FIRST_CHILD(this)->fops->lookup, loc, xdata);
-	dict_unref (xdata);
-	return 0;
-enomem:
-	QB_STACK_UNWIND (lookup, frame, -1, ENOMEM, 0, 0, 0, 0);
-	if (xdata)
-		dict_unref (xdata);
-	return 0;
-}
-
-
-int
-qb_setxattr_format (call_frame_t *frame, xlator_t *this, call_stub_t *stub,
-		    dict_t *xattr, inode_t *inode)
-{
-	char *format = NULL;
-	int op_errno = 0;
-	qb_local_t *qb_local = NULL;
-	data_t *data = NULL;
-	qb_inode_t *qb_inode;
-
-	if (!(data = dict_get (xattr, "trusted.glusterfs.block-format"))) {
-		QB_STUB_RESUME (stub);
-		return 0;
-	}
-
-	format = alloca (data->len + 1);
-	memcpy (format, data->data, data->len);
-	format[data->len] = 0;
-
-	op_errno = qb_format_extract (this, format, inode);
-	if (op_errno) {
-		QB_STUB_UNWIND (stub, -1, op_errno);
-		return 0;
-	}
-	qb_inode = qb_inode_ctx_get(this, inode);
-
-	qb_local = frame->local;
-
-	qb_local->stub = stub;
-	qb_local->inode = inode_ref (inode);
-
-	snprintf(qb_local->fmt, QB_XATTR_VAL_MAX, "%s:%" PRId64, qb_inode->fmt,
-		 qb_inode->size);
-
-	qb_coroutine (frame, qb_format_and_resume);
-
-	return 0;
-}
-
-
-int
-qb_setxattr_snapshot_create (call_frame_t *frame, xlator_t *this,
-			     call_stub_t *stub, dict_t *xattr, inode_t *inode)
-{
-	qb_local_t *qb_local = NULL;
-	char *name = NULL;
-	data_t *data = NULL;
-
-	if (!(data = dict_get (xattr, "trusted.glusterfs.block-snapshot-create"))) {
-		QB_STUB_RESUME (stub);
-		return 0;
-	}
-
-	name = alloca (data->len + 1);
-	memcpy (name, data->data, data->len);
-	name[data->len] = 0;
-
-	qb_local = frame->local;
-
-	qb_local->stub = stub;
-	qb_local->inode = inode_ref (inode);
-	strncpy (qb_local->name, name, 128);
-
-	qb_coroutine (frame, qb_snapshot_create);
-
-	return 0;
-}
-
-
-int
-qb_setxattr_snapshot_delete (call_frame_t *frame, xlator_t *this,
-			     call_stub_t *stub, dict_t *xattr, inode_t *inode)
-{
-	qb_local_t *qb_local = NULL;
-	char *name = NULL;
-	data_t *data = NULL;
-
-	if (!(data = dict_get (xattr, "trusted.glusterfs.block-snapshot-delete"))) {
-		QB_STUB_RESUME (stub);
-		return 0;
-	}
-
-	name = alloca (data->len + 1);
-	memcpy (name, data->data, data->len);
-	name[data->len] = 0;
-
-	qb_local = frame->local;
-
-	qb_local->stub = stub;
-	qb_local->inode = inode_ref (inode);
-	strncpy (qb_local->name, name, 128);
-
-	qb_coroutine (frame, qb_snapshot_delete);
-
-	return 0;
-}
-
-int
-qb_setxattr_snapshot_goto (call_frame_t *frame, xlator_t *this,
-			   call_stub_t *stub, dict_t *xattr, inode_t *inode)
-{
-	qb_local_t *qb_local = NULL;
-	char *name = NULL;
-	data_t *data = NULL;
-
-	if (!(data = dict_get (xattr, "trusted.glusterfs.block-snapshot-goto"))) {
-		QB_STUB_RESUME (stub);
-		return 0;
-	}
-
-	name = alloca (data->len + 1);
-	memcpy (name, data->data, data->len);
-	name[data->len] = 0;
-
-	qb_local = frame->local;
-
-	qb_local->stub = stub;
-	qb_local->inode = inode_ref (inode);
-	strncpy (qb_local->name, name, 128);
-
-	qb_coroutine (frame, qb_snapshot_goto);
-
-	return 0;
-}
-
-
-int
-qb_setxattr_common (call_frame_t *frame, xlator_t *this, call_stub_t *stub,
-		    dict_t *xattr, inode_t *inode)
-{
-
-	if (dict_get (xattr, "trusted.glusterfs.block-format")) {
-		qb_setxattr_format (frame, this, stub, xattr, inode);
-		return 0;
-	}
-
-	if (dict_get (xattr, "trusted.glusterfs.block-snapshot-create")) {
-		qb_setxattr_snapshot_create (frame, this, stub, xattr, inode);
-		return 0;
-	}
-
-	if (dict_get (xattr, "trusted.glusterfs.block-snapshot-delete")) {
-		qb_setxattr_snapshot_delete (frame, this, stub, xattr, inode);
-		return 0;
-	}
-
-	if (dict_get (xattr, "trusted.glusterfs.block-snapshot-goto")) {
-		qb_setxattr_snapshot_goto (frame, this, stub, xattr, inode);
-		return 0;
-	}
-
-	QB_STUB_RESUME (stub);
-
-	return 0;
-}
-
-
-int
-qb_setxattr (call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xattr,
-	     int flags, dict_t *xdata)
-{
-	call_stub_t *stub = NULL;
-
-	if (qb_local_init (frame) != 0)
-		goto enomem;
-
-	stub = fop_setxattr_stub (frame, default_setxattr_resume, loc, xattr,
-				  flags, xdata);
-	if (!stub)
-		goto enomem;
-
-	qb_setxattr_common (frame, this, stub, xattr, loc->inode);
-
-	return 0;
-enomem:
-	QB_STACK_UNWIND (setxattr, frame, -1, ENOMEM, 0);
-	return 0;
-}
-
-
-int
-qb_fsetxattr (call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *xattr,
-	      int flags, dict_t *xdata)
-{
-	call_stub_t *stub = NULL;
-
-	if (qb_local_init (frame) != 0)
-		goto enomem;
-
-	stub = fop_fsetxattr_stub (frame, default_fsetxattr_resume, fd, xattr,
-				   flags, xdata);
-	if (!stub)
-		goto enomem;
-
-	qb_setxattr_common (frame, this, stub, xattr, fd->inode);
-
-	return 0;
-enomem:
-	QB_STACK_UNWIND (fsetxattr, frame, -1, ENOMEM, 0);
-	return 0;
-}
-
-
-int
-qb_open_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-	     int op_ret, int op_errno, fd_t *fd, dict_t *xdata)
-{
-	call_stub_t *stub = NULL;
-	qb_local_t *qb_local = NULL;
-
-	qb_local = frame->local;
-
-	if (op_ret < 0)
-		goto unwind;
-
-	if (!qb_inode_ctx_get (this, qb_local->inode))
-		goto unwind;
-
-	stub = fop_open_cbk_stub (frame, NULL, op_ret, op_errno, fd, xdata);
-	if (!stub) {
-		op_ret = -1;
-		op_errno = ENOMEM;
-		goto unwind;
-	}
-
-	qb_local->stub = stub;
-
-	qb_coroutine (frame, qb_co_open);
-
-	return 0;
-unwind:
-	QB_STACK_UNWIND (open, frame, op_ret, op_errno, fd, xdata);
-	return 0;
-}
-
-
-int
-qb_open (call_frame_t *frame, xlator_t *this, loc_t *loc, int flags,
-	 fd_t *fd, dict_t *xdata)
-{
-	qb_local_t *qb_local = NULL;
-	qb_inode_t *qb_inode = NULL;
-
-	qb_inode = qb_inode_ctx_get (this, loc->inode);
-	if (!qb_inode) {
-		STACK_WIND (frame, default_open_cbk, FIRST_CHILD(this),
-			    FIRST_CHILD(this)->fops->open, loc, flags, fd,
-			    xdata);
-		return 0;
-	}
-
-	if (qb_local_init (frame) != 0)
-		goto enomem;
-
-	qb_local = frame->local;
-
-	qb_local->inode = inode_ref (loc->inode);
-	qb_local->fd = fd_ref (fd);
-
-	STACK_WIND (frame, qb_open_cbk, FIRST_CHILD(this),
-		    FIRST_CHILD(this)->fops->open, loc, flags, fd, xdata);
-	return 0;
-enomem:
-	QB_STACK_UNWIND (open, frame, -1, ENOMEM, 0, 0);
-	return 0;
-}
-
-
-int
-qb_writev (call_frame_t *frame, xlator_t *this, fd_t *fd, struct iovec *vector,
-	   int count, off_t offset, uint32_t flags, struct iobref *iobref,
-	   dict_t *xdata)
-{
-	qb_local_t *qb_local = NULL;
-	qb_inode_t *qb_inode = NULL;
-
-	qb_inode = qb_inode_ctx_get (this, fd->inode);
-	if (!qb_inode) {
-		STACK_WIND (frame, default_writev_cbk, FIRST_CHILD(this),
-			    FIRST_CHILD(this)->fops->writev, fd, vector, count,
-			    offset, flags, iobref, xdata);
-		return 0;
-	}
-
-	if (qb_local_init (frame) != 0)
-		goto enomem;
-
-	qb_local = frame->local;
-
-	qb_local->inode = inode_ref (fd->inode);
-	qb_local->fd = fd_ref (fd);
-
-	qb_local->stub = fop_writev_stub (frame, NULL, fd, vector, count,
-					  offset, flags, iobref, xdata);
-	if (!qb_local->stub)
-		goto enomem;
-
-	qb_coroutine (frame, qb_co_writev);
-
-	return 0;
-enomem:
-	QB_STACK_UNWIND (writev, frame, -1, ENOMEM, 0, 0, 0);
-	return 0;
-}
-
-
-int
-qb_readv (call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
-	  off_t offset, uint32_t flags, dict_t *xdata)
-{
-	qb_local_t *qb_local = NULL;
-	qb_inode_t *qb_inode = NULL;
-
-	qb_inode = qb_inode_ctx_get (this, fd->inode);
-	if (!qb_inode) {
-		STACK_WIND (frame, default_readv_cbk, FIRST_CHILD(this),
-			    FIRST_CHILD(this)->fops->readv, fd, size, offset,
-			    flags, xdata);
-		return 0;
-	}
-
-	if (qb_local_init (frame) != 0)
-		goto enomem;
-
-	qb_local = frame->local;
-
-	qb_local->inode = inode_ref (fd->inode);
-	qb_local->fd = fd_ref (fd);
-
-	qb_local->stub = fop_readv_stub (frame, NULL, fd, size, offset,
-					  flags, xdata);
-	if (!qb_local->stub)
-		goto enomem;
-
-	qb_coroutine (frame, qb_co_readv);
-
-	return 0;
-enomem:
-	QB_STACK_UNWIND (readv, frame, -1, ENOMEM, 0, 0, 0, 0, 0);
-	return 0;
-}
-
-
-int
-qb_fsync (call_frame_t *frame, xlator_t *this, fd_t *fd, int dsync,
-	  dict_t *xdata)
-{
-	qb_local_t *qb_local = NULL;
-	qb_inode_t *qb_inode = NULL;
-
-	qb_inode = qb_inode_ctx_get (this, fd->inode);
-	if (!qb_inode) {
-		STACK_WIND (frame, default_fsync_cbk, FIRST_CHILD(this),
-			    FIRST_CHILD(this)->fops->fsync, fd, dsync, xdata);
-		return 0;
-	}
-
-	if (qb_local_init (frame) != 0)
-		goto enomem;
-
-	qb_local = frame->local;
-
-	qb_local->inode = inode_ref (fd->inode);
-	qb_local->fd = fd_ref (fd);
-
-	qb_local->stub = fop_fsync_stub (frame, NULL, fd, dsync, xdata);
-
-	if (!qb_local->stub)
-		goto enomem;
-
-	qb_coroutine (frame, qb_co_fsync);
-
-	return 0;
-enomem:
-	QB_STACK_UNWIND (fsync, frame, -1, ENOMEM, 0, 0, 0);
-	return 0;
-}
-
-
-int
-qb_flush (call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *xdata)
-{
-	qb_local_t *qb_local = NULL;
-	qb_inode_t *qb_inode = NULL;
-
-	qb_inode = qb_inode_ctx_get (this, fd->inode);
-	if (!qb_inode) {
-		STACK_WIND (frame, default_flush_cbk, FIRST_CHILD(this),
-			    FIRST_CHILD(this)->fops->flush, fd, xdata);
-		return 0;
-	}
-
-	if (qb_local_init (frame) != 0)
-		goto enomem;
-
-	qb_local = frame->local;
-
-	qb_local->inode = inode_ref (fd->inode);
-	qb_local->fd = fd_ref (fd);
-
-	qb_local->stub = fop_flush_stub (frame, NULL, fd, xdata);
-
-	if (!qb_local->stub)
-		goto enomem;
-
-	qb_coroutine (frame, qb_co_fsync);
-
-	return 0;
-enomem:
-	QB_STACK_UNWIND (flush, frame, -1, ENOMEM, 0);
-	return 0;
-}
-
-static int32_t
-qb_readdirp_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
-		int32_t op_ret, int32_t op_errno, gf_dirent_t *entries,
-		dict_t *xdata)
-{
-	qb_conf_t *conf = this->private;
-	gf_dirent_t *entry;
-	char *format;
-
-	list_for_each_entry(entry, &entries->list, list) {
-		if (!entry->inode || !entry->dict)
-			continue;
-
-		format = NULL;
-		if (dict_get_str(entry->dict, conf->qb_xattr_key, &format))
-			continue;
-
-		if (!format) {
-			qb_inode_cleanup(this, entry->inode, 1);
-			continue;
-		}
-
-		if (qb_format_extract(this, format, entry->inode))
-			continue;
-
-		qb_iatt_fixup(this, entry->inode, &entry->d_stat);
-	}
-
-	STACK_UNWIND_STRICT(readdirp, frame, op_ret, op_errno, entries, xdata);
-	return 0;
-}
-
-static int32_t
-qb_readdirp(call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
-	    off_t off, dict_t *xdata)
-{
-	qb_conf_t *conf = this->private;
-
-	xdata = xdata ? dict_ref(xdata) : dict_new();
-	if (!xdata)
-		goto enomem;
-
-	if (dict_set_int32 (xdata, conf->qb_xattr_key, 0))
-		goto enomem;
-
-	STACK_WIND(frame, qb_readdirp_cbk, FIRST_CHILD(this),
-		   FIRST_CHILD(this)->fops->readdirp, fd, size, off, xdata);
-
-	dict_unref(xdata);
-	return 0;
-
-enomem:
-	QB_STACK_UNWIND(readdirp, frame, -1, ENOMEM, NULL, NULL);
-	if (xdata)
-		dict_unref(xdata);
-	return 0;
-}
-
-int
-qb_truncate (call_frame_t *frame, xlator_t *this, loc_t *loc, off_t offset,
-	     dict_t *xdata)
-{
-	qb_local_t *qb_local = NULL;
-	qb_inode_t *qb_inode = NULL;
-
-	qb_inode = qb_inode_ctx_get (this, loc->inode);
-	if (!qb_inode) {
-		STACK_WIND (frame, default_truncate_cbk, FIRST_CHILD(this),
-			    FIRST_CHILD(this)->fops->truncate, loc, offset,
-			    xdata);
-		return 0;
-	}
-
-	if (qb_local_init (frame) != 0)
-		goto enomem;
-
-	qb_local = frame->local;
-
-	qb_local->inode = inode_ref (loc->inode);
-	qb_local->fd = fd_anonymous (loc->inode);
-
-	qb_local->stub = fop_truncate_stub (frame, NULL, loc, offset, xdata);
-
-	if (!qb_local->stub)
-		goto enomem;
-
-	qb_coroutine (frame, qb_co_truncate);
-
-	return 0;
-enomem:
-	QB_STACK_UNWIND (truncate, frame, -1, ENOMEM, 0, 0, 0);
-	return 0;
-}
-
-
-int
-qb_ftruncate (call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
-	     dict_t *xdata)
-{
-	qb_local_t *qb_local = NULL;
-	qb_inode_t *qb_inode = NULL;
-
-	qb_inode = qb_inode_ctx_get (this, fd->inode);
-	if (!qb_inode) {
-		STACK_WIND (frame, default_ftruncate_cbk, FIRST_CHILD(this),
-			    FIRST_CHILD(this)->fops->ftruncate, fd, offset,
-			    xdata);
-		return 0;
-	}
-
-	if (qb_local_init (frame) != 0)
-		goto enomem;
-
-	qb_local = frame->local;
-
-	qb_local->inode = inode_ref (fd->inode);
-	qb_local->fd = fd_ref (fd);
-
-	qb_local->stub = fop_ftruncate_stub (frame, NULL, fd, offset, xdata);
-
-	if (!qb_local->stub)
-		goto enomem;
-
-	qb_coroutine (frame, qb_co_truncate);
-
-	return 0;
-enomem:
-	QB_STACK_UNWIND (ftruncate, frame, -1, ENOMEM, 0, 0, 0);
-	return 0;
-}
-
-
-int
-qb_stat_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-	     int op_ret, int op_errno, struct iatt *iatt, dict_t *xdata)
-{
-	inode_t *inode = NULL;
-
-	inode = frame->local;
-	frame->local = NULL;
-
-	if (inode) {
-		qb_iatt_fixup (this, inode, iatt);
-		inode_unref (inode);
-	}
-
-	QB_STACK_UNWIND (stat, frame, op_ret, op_errno, iatt, xdata);
-
-	return 0;
-}
-
-int
-qb_stat (call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xdata)
-{
-	if (qb_inode_ctx_get (this, loc->inode))
-		frame->local = inode_ref (loc->inode);
-
-	STACK_WIND (frame, qb_stat_cbk, FIRST_CHILD(this),
-		    FIRST_CHILD(this)->fops->stat, loc, xdata);
-	return 0;
-}
-
-
-int
-qb_fstat_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-	      int op_ret, int op_errno, struct iatt *iatt, dict_t *xdata)
-{
-	inode_t *inode = NULL;
-
-	inode = frame->local;
-	frame->local = NULL;
-
-	if (inode) {
-		qb_iatt_fixup (this, inode, iatt);
-		inode_unref (inode);
-	}
-
-	QB_STACK_UNWIND (fstat, frame, op_ret, op_errno, iatt, xdata);
-
-	return 0;
-}
-
-
-int
-qb_fstat (call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *xdata)
-{
-	if (qb_inode_ctx_get (this, fd->inode))
-		frame->local = inode_ref (fd->inode);
-
-	STACK_WIND (frame, qb_fstat_cbk, FIRST_CHILD(this),
-		    FIRST_CHILD(this)->fops->fstat, fd, xdata);
-	return 0;
-}
-
-
-int
-qb_setattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-		int op_ret, int op_errno, struct iatt *pre, struct iatt *post,
-		dict_t *xdata)
-{
-	inode_t *inode = NULL;
-
-	inode = frame->local;
-	frame->local = NULL;
-
-	if (inode) {
-		qb_iatt_fixup (this, inode, pre);
-		qb_iatt_fixup (this, inode, post);
-		inode_unref (inode);
-	}
-
-	QB_STACK_UNWIND (setattr, frame, op_ret, op_errno, pre, post, xdata);
-
-	return 0;
-}
-
-
-int
-qb_setattr (call_frame_t *frame, xlator_t *this, loc_t *loc, struct iatt *buf,
-	    int valid, dict_t *xdata)
-{
-	if (qb_inode_ctx_get (this, loc->inode))
-		frame->local = inode_ref (loc->inode);
-
-	STACK_WIND (frame, qb_setattr_cbk, FIRST_CHILD(this),
-		    FIRST_CHILD(this)->fops->setattr, loc, buf, valid, xdata);
-	return 0;
-}
-
-
-int
-qb_fsetattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-		 int op_ret, int op_errno, struct iatt *pre, struct iatt *post,
-		 dict_t *xdata)
-{
-	inode_t *inode = NULL;
-
-	inode = frame->local;
-	frame->local = NULL;
-
-	if (inode) {
-		qb_iatt_fixup (this, inode, pre);
-		qb_iatt_fixup (this, inode, post);
-		inode_unref (inode);
-	}
-
-	QB_STACK_UNWIND (fsetattr, frame, op_ret, op_errno, pre, post, xdata);
-
-	return 0;
-}
-
-
-int
-qb_fsetattr (call_frame_t *frame, xlator_t *this, fd_t *fd, struct iatt *buf,
-	     int valid, dict_t *xdata)
-{
-	if (qb_inode_ctx_get (this, fd->inode))
-		frame->local = inode_ref (fd->inode);
-
-	STACK_WIND (frame, qb_setattr_cbk, FIRST_CHILD(this),
-		    FIRST_CHILD(this)->fops->fsetattr, fd, buf, valid, xdata);
-	return 0;
-}
-
-
-int
-qb_forget (xlator_t *this, inode_t *inode)
-{
-	return qb_inode_cleanup (this, inode, 0);
-}
-
-
-int
-qb_release (xlator_t *this, fd_t *fd)
-{
-	call_frame_t *frame = NULL;
-
-	frame = create_frame (this, this->ctx->pool);
-	if (!frame) {
-		gf_log (this->name, GF_LOG_ERROR,
-			"Could not allocate frame. "
-			"Leaking QEMU BlockDriverState");
-		return -1;
-	}
-
-	if (qb_local_init (frame) != 0) {
-		gf_log (this->name, GF_LOG_ERROR,
-			"Could not allocate local. "
-			"Leaking QEMU BlockDriverState");
-		STACK_DESTROY (frame->root);
-		return -1;
-	}
-
-	if (qb_coroutine (frame, qb_co_close) != 0) {
-		gf_log (this->name, GF_LOG_ERROR,
-			"Could not allocate coroutine. "
-			"Leaking QEMU BlockDriverState");
-		qb_local_free (this, frame->local);
-		frame->local = NULL;
-		STACK_DESTROY (frame->root);
-	}
-
-	return 0;
-}
-
-int
-mem_acct_init (xlator_t *this)
-{
-        int     ret = -1;
-
-        ret = xlator_mem_acct_init (this, gf_qb_mt_end + 1);
-
-        if (ret)
-                gf_log (this->name, GF_LOG_ERROR, "Memory accounting init "
-                        "failed");
-        return ret;
-}
-
-
-int
-reconfigure (xlator_t *this, dict_t *options)
-{
-	return 0;
-}
-
-
-int
-init (xlator_t *this)
-{
-        qb_conf_t *conf    = NULL;
-        int32_t    ret     = -1;
-	static int bdrv_inited = 0;
-
-        if (!this->children || this->children->next) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "FATAL: qemu-block (%s) not configured with exactly "
-                        "one child", this->name);
-                goto out;
-        }
-
-        conf = GF_CALLOC (1, sizeof (*conf), gf_qb_mt_qb_conf_t);
-        if (!conf)
-                goto out;
-
-        /* configure 'option window-size <size>' */
-        GF_OPTION_INIT ("default-password", conf->default_password, str, out);
-
-	/* qemu coroutines use "co_mutex" for synchronizing among themselves.
-	   However "co_mutex" itself is not threadsafe if the coroutine framework
-	   is multithreaded (which usually is not). However synctasks are
-	   fundamentally multithreaded, so for now create a syncenv which has
-	   scaling limits set to max 1 thread so that the qemu coroutines can
-	   execute "safely".
-
-	   Future work: provide an implementation of "co_mutex" which is
-	   threadsafe and use the global multithreaded ctx->env syncenv.
-	*/
-	conf->env = syncenv_new (0, 1, 1);
-
-        this->private = conf;
-
-        ret = 0;
-
-	snprintf (conf->qb_xattr_key, QB_XATTR_KEY_MAX, QB_XATTR_KEY_FMT,
-		  this->name);
-
-	cur_mon = (void *) 1;
-
-	if (!bdrv_inited) {
-		bdrv_init ();
-		bdrv_inited = 1;
-	}
-
-out:
-        if (ret)
-                GF_FREE (conf);
-
-        return ret;
-}
-
-
-void
-fini (xlator_t *this)
-{
-        qb_conf_t *conf = NULL;
-
-        conf = this->private;
-
-        this->private = NULL;
-
-	if (conf->root_inode)
-		inode_unref(conf->root_inode);
-        GF_FREE (conf);
-
-	return;
-}
-
-
-struct xlator_fops fops = {
-	.lookup      = qb_lookup,
-	.fsetxattr   = qb_fsetxattr,
-	.setxattr    = qb_setxattr,
-	.open        = qb_open,
-        .writev      = qb_writev,
-        .readv       = qb_readv,
-        .fsync       = qb_fsync,
-        .truncate    = qb_truncate,
-        .ftruncate   = qb_ftruncate,
-        .stat        = qb_stat,
-        .fstat       = qb_fstat,
-	.setattr     = qb_setattr,
-	.fsetattr    = qb_fsetattr,
-        .flush       = qb_flush,
-/*
-	.getxattr    = qb_getxattr,
-	.fgetxattr   = qb_fgetxattr
-*/
-	.readdirp    = qb_readdirp,
-};
-
-
-struct xlator_cbks cbks = {
-        .forget   = qb_forget,
-	.release  = qb_release,
-};
-
-
-struct xlator_dumpops dumpops = {
-};
-
-
-struct volume_options options[] = {
-        { .key  = {"default-password"},
-          .type = GF_OPTION_TYPE_STR,
-          .default_value = "",
-          .description = "Default password for the AES encrypted block images."
-        },
-        { .key = {NULL} },
-};
diff --git a/xlators/features/qemu-block/src/qemu-block.h b/xlators/features/qemu-block/src/qemu-block.h
deleted file mode 100644
index 21cdcec2613..00000000000
--- a/xlators/features/qemu-block/src/qemu-block.h
+++ /dev/null
@@ -1,109 +0,0 @@
-/*
-  Copyright (c) 2013 Red Hat, Inc. <http://www.redhat.com>
-  This file is part of GlusterFS.
-
-  This file is licensed to you under your choice of the GNU Lesser
-  General Public License, version 3 or any later version (LGPLv3 or
-  later), or the GNU General Public License, version 2 (GPLv2), in all
-  cases as published by the Free Software Foundation.
-*/
-
-#ifndef __QEMU_BLOCK_H
-#define __QEMU_BLOCK_H
-
-#include "syncop.h"
-#include "call-stub.h"
-#include "block/block_int.h"
-#include "monitor/monitor.h"
-
-/* QB_XATTR_KEY_FMT is the on-disk xattr stored in the inode which
-   indicates that the file must be "interpreted" by the block format
-   logic. The value of the key is of the pattern:
-
-   "format:virtual_size"
-
-   e.g
-
-   "qcow2:20GB" or "qed:100GB"
-
-   The format and virtual size are colon separated. The format is
-   a case sensitive string which qemu recognizes. virtual_size is
-   specified as a size which glusterfs recognizes as size (i.e.,
-   value accepted by gf_string2bytesize())
-*/
-#define QB_XATTR_KEY_FMT "trusted.glusterfs.%s.format"
-
-#define QB_XATTR_KEY_MAX 64
-
-#define QB_XATTR_VAL_MAX 64
-
-
-typedef struct qb_inode {
-	char fmt[QB_XATTR_VAL_MAX]; /* this is only the format, not "format:size" */
-	uint64_t size; /* virtual size in bytes */
-	BlockDriverState *bs;
-	int refcnt;
-	uuid_t backing_gfid;
-	char *backing_fname;
-} qb_inode_t;
-
-
-typedef struct qb_conf {
-	Monitor *mon;
-	struct syncenv *env;
-	char qb_xattr_key[QB_XATTR_KEY_MAX];
-	char *default_password;
-	inode_t *root_inode;
-} qb_conf_t;
-
-
-typedef struct qb_local {
-	call_frame_t      *frame; /* backpointer */
-	call_stub_t       *stub;
-	inode_t           *inode;
-	fd_t              *fd;
-	char               fmt[QB_XATTR_VAL_MAX+1];
-	char               name[256];
-	synctask_fn_t      synctask_fn;
-	struct list_head   list;
-} qb_local_t;
-
-void qb_local_free (xlator_t *this, qb_local_t *local);
-int qb_coroutine (call_frame_t *frame, synctask_fn_t fn);
-inode_t *qb_inode_from_filename (const char *filename);
-int qb_inode_to_filename (inode_t *inode, char *filename, int size);
-int qb_format_extract (xlator_t *this, char *format, inode_t *inode);
-
-qb_inode_t *qb_inode_ctx_get (xlator_t *this, inode_t *inode);
-
-#define QB_STACK_UNWIND(typ, frame, args ...) do {	\
-	qb_local_t *__local = frame->local;		\
-	xlator_t *__this = frame->this;			\
-							\
-	frame->local = NULL;				\
-	STACK_UNWIND_STRICT (typ, frame, args);		\
-	if (__local)					\
-		qb_local_free (__this, __local);	\
-	} while (0)
-
-#define QB_STUB_UNWIND(stub, op_ret, op_errno) do {	\
-	qb_local_t *__local = stub->frame->local;	\
-	xlator_t *__this = stub->frame->this;		\
-							\
-	stub->frame->local = NULL;			\
-	call_unwind_error (stub, op_ret, op_errno);	\
-	if (__local)					\
-		qb_local_free (__this, __local);	\
-	} while (0)
-
-#define QB_STUB_RESUME(stub_errno) do {			\
-	qb_local_t *__local = stub->frame->local;	\
-	xlator_t *__this = stub->frame->this;		\
-							\
-	stub->frame->local = NULL;			\
-	call_resume (stub);				\
-	if (__local)					\
-		qb_local_free (__this, __local);	\
-	} while (0)
-
-#endif /* !__QEMU_BLOCK_H */
diff --git a/xlators/features/quiesce/src/Makefile.am b/xlators/features/quiesce/src/Makefile.am
index 15e46629e78..74ea999c045 100644
--- a/xlators/features/quiesce/src/Makefile.am
+++ b/xlators/features/quiesce/src/Makefile.am
@@ -1,14 +1,15 @@
 xlator_LTLIBRARIES = quiesce.la
 xlatordir = $(libdir)/glusterfs/$(PACKAGE_VERSION)/xlator/features
 
-quiesce_la_LDFLAGS = -module -avoid-version
+quiesce_la_LDFLAGS = -module $(GF_XLATOR_DEFAULT_LDFLAGS)
 
 quiesce_la_SOURCES = quiesce.c
 quiesce_la_LIBADD = $(top_builddir)/libglusterfs/src/libglusterfs.la
 
-noinst_HEADERS = quiesce.h quiesce-mem-types.h
+noinst_HEADERS = quiesce.h quiesce-mem-types.h quiesce-messages.h
 
-AM_CPPFLAGS = $(GF_CPPFLAGS) -I$(top_srcdir)/libglusterfs/src
+AM_CPPFLAGS = $(GF_CPPFLAGS) -I$(top_srcdir)/libglusterfs/src \
+	-I$(top_srcdir)/rpc/xdr/src -I$(top_builddir)/rpc/xdr/src
 
 AM_CFLAGS = -Wall $(GF_CFLAGS)
 
diff --git a/xlators/features/quiesce/src/quiesce-mem-types.h b/xlators/features/quiesce/src/quiesce-mem-types.h
index 6e582f424ea..416456b13af 100644
--- a/xlators/features/quiesce/src/quiesce-mem-types.h
+++ b/xlators/features/quiesce/src/quiesce-mem-types.h
@@ -11,10 +11,11 @@
 #ifndef __QUIESCE_MEM_TYPES_H__
 #define __QUIESCE_MEM_TYPES_H__
 
-#include "mem-types.h"
+#include <glusterfs/mem-types.h>
 
 enum gf_quiesce_mem_types_ {
-        gf_quiesce_mt_priv_t = gf_common_mt_end + 1,
-        gf_quiesce_mt_end
+    gf_quiesce_mt_priv_t = gf_common_mt_end + 1,
+    gf_quiesce_mt_failover_hosts,
+    gf_quiesce_mt_end
 };
 #endif
diff --git a/xlators/features/quiesce/src/quiesce-messages.h b/xlators/features/quiesce/src/quiesce-messages.h
new file mode 100644
index 00000000000..32ffd409807
--- /dev/null
+++ b/xlators/features/quiesce/src/quiesce-messages.h
@@ -0,0 +1,28 @@
+/*
+ *   Copyright (c) 2016 Red Hat, Inc. <http://www.redhat.com>
+ *   This file is part of GlusterFS.
+ *
+ *   This file is licensed to you under your choice of the GNU Lesser
+ *   General Public License, version 3 or any later version (LGPLv3 or
+ *   later), or the GNU General Public License, version 2 (GPLv2), in all
+ *   cases as published by the Free Software Foundation.
+ */
+
+#ifndef __QUIESCE_MESSAGES_H__
+#define __QUIESCE_MESSAGES_H__
+
+#include <glusterfs/glfs-message-id.h>
+
+/* To add new message IDs, append new identifiers at the end of the list.
+ *
+ * Never remove a message ID. If it's not used anymore, you can rename it or
+ * leave it as it is, but not delete it. This is to prevent reutilization of
+ * IDs by other messages.
+ *
+ * The component name must match one of the entries defined in
+ * glfs-message-id.h.
+ */
+
+GLFS_MSGID(QUIESCE, QUIESCE_MSG_INVAL_HOST, QUIESCE_MSG_FAILOVER_FAILED);
+
+#endif /* __NL_CACHE_MESSAGES_H__ */
diff --git a/xlators/features/quiesce/src/quiesce.c b/xlators/features/quiesce/src/quiesce.c
index 24c7dc6ed31..0e5eb60a16f 100644
--- a/xlators/features/quiesce/src/quiesce.c
+++ b/xlators/features/quiesce/src/quiesce.c
@@ -7,669 +7,815 @@
    later), or the GNU General Public License, version 2 (GPLv2), in all
    cases as published by the Free Software Foundation.
 */
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
-
 #include "quiesce.h"
-#include "defaults.h"
-#include "call-stub.h"
+#include <glusterfs/defaults.h>
+#include <glusterfs/call-stub.h>
 
 /* TODO: */
 /* Think about 'writev/_*_lk/setattr/xattrop/' fops to do re-transmittion */
 
+void
+gf_quiesce_timeout(void *data);
 
 /* Quiesce Specific Functions */
 void
-gf_quiesce_local_wipe (xlator_t *this, quiesce_local_t *local)
+gf_quiesce_local_wipe(xlator_t *this, quiesce_local_t *local)
 {
-        if (!local || !this || !this->private)
-                return;
+    if (!local || !this || !this->private)
+        return;
 
-        if (local->loc.inode)
-                loc_wipe (&local->loc);
-        if (local->fd)
-                fd_unref (local->fd);
-        GF_FREE (local->name);
-        GF_FREE (local->volname);
-        if (local->dict)
-                dict_unref (local->dict);
-        if (local->iobref)
-                iobref_unref (local->iobref);
-        GF_FREE (local->vector);
+    if (local->loc.inode)
+        loc_wipe(&local->loc);
+    if (local->fd)
+        fd_unref(local->fd);
+    GF_FREE(local->name);
+    GF_FREE(local->volname);
+    if (local->dict)
+        dict_unref(local->dict);
+    if (local->iobref)
+        iobref_unref(local->iobref);
+    GF_FREE(local->vector);
 
-        mem_put (local);
+    mem_put(local);
 }
 
-call_stub_t *
-gf_quiesce_dequeue (xlator_t *this)
+void
+__gf_quiesce_start_timer(xlator_t *this, quiesce_priv_t *priv)
 {
-        call_stub_t  *stub = NULL;
-        quiesce_priv_t *priv = NULL;
+    struct timespec timeout = {
+        0,
+    };
 
-        priv = this->private;
+    if (!priv->timer) {
+        timeout.tv_sec = priv->timeout;
+        timeout.tv_nsec = 0;
 
-        if (!priv || list_empty (&priv->req))
-                return NULL;
-
-        LOCK (&priv->lock);
-        {
-                stub = list_entry (priv->req.next, call_stub_t, list);
-                list_del_init (&stub->list);
-                priv->queue_size--;
+        priv->timer = gf_timer_call_after(this->ctx, timeout,
+                                          gf_quiesce_timeout, (void *)this);
+        if (priv->timer == NULL) {
+            gf_log(this->name, GF_LOG_ERROR, "Cannot create timer");
         }
-        UNLOCK (&priv->lock);
+    }
+}
+
+static void
+__gf_quiesce_cleanup_failover_hosts(xlator_t *this, quiesce_priv_t *priv)
+{
+    quiesce_failover_hosts_t *tmp = NULL;
+    quiesce_failover_hosts_t *failover_host = NULL;
 
-        return stub;
+    list_for_each_entry_safe(failover_host, tmp, &priv->failover_list, list)
+    {
+        GF_FREE(failover_host->addr);
+        list_del(&failover_host->list);
+        GF_FREE(failover_host);
+    }
+    return;
 }
 
-void *
-gf_quiesce_dequeue_start (void *data)
+void
+gf_quiesce_populate_failover_hosts(xlator_t *this, quiesce_priv_t *priv,
+                                   const char *value)
+{
+    char *dup_val = NULL;
+    char *addr_tok = NULL;
+    char *save_ptr = NULL;
+    quiesce_failover_hosts_t *failover_host = NULL;
+
+    if (!value)
+        goto out;
+
+    dup_val = gf_strdup(value);
+    if (!dup_val)
+        goto out;
+
+    addr_tok = strtok_r(dup_val, ",", &save_ptr);
+    LOCK(&priv->lock);
+    {
+        if (!list_empty(&priv->failover_list))
+            __gf_quiesce_cleanup_failover_hosts(this, priv);
+
+        while (addr_tok) {
+            if (!valid_internet_address(addr_tok, _gf_true, _gf_false)) {
+                gf_msg(this->name, GF_LOG_INFO, 0, QUIESCE_MSG_INVAL_HOST,
+                       "Specified "
+                       "invalid internet address:%s",
+                       addr_tok);
+                continue;
+            }
+            failover_host = GF_CALLOC(1, sizeof(*failover_host),
+                                      gf_quiesce_mt_failover_hosts);
+            failover_host->addr = gf_strdup(addr_tok);
+            INIT_LIST_HEAD(&failover_host->list);
+            list_add(&failover_host->list, &priv->failover_list);
+            addr_tok = strtok_r(NULL, ",", &save_ptr);
+        }
+    }
+    UNLOCK(&priv->lock);
+    GF_FREE(dup_val);
+out:
+    return;
+}
+
+int32_t
+gf_quiesce_failover_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                        int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
-        xlator_t       *this = NULL;
-        quiesce_priv_t *priv = NULL;
-        call_stub_t    *stub = NULL;
+    quiesce_priv_t *priv = NULL;
 
-        this = data;
-        priv = this->private;
-        THIS = this;
+    if (op_ret < 0) {
+        /* Failure here doesn't mean the failover to another host didn't
+         * succeed, we will know if failover succeeds or not by the
+         * CHILD_UP/CHILD_DOWN event. A failure here indicates something
+         * went wrong with the submission of failover command, hence
+         * just abort the failover attempts without retrying with other
+         * hosts.
+         */
+        gf_msg(this->name, GF_LOG_INFO, op_errno, QUIESCE_MSG_FAILOVER_FAILED,
+               "Initiating failover to host:%s failed:", (char *)cookie);
+    }
 
-        while (!list_empty (&priv->req)) {
-                stub = gf_quiesce_dequeue (this);
-                if (stub) {
-                        call_resume (stub);
-                }
-        }
+    GF_FREE(cookie);
+    STACK_DESTROY(frame->root);
 
-        return 0;
+    priv = this->private;
+    __gf_quiesce_start_timer(this, priv);
+
+    return 0;
 }
 
+int
+__gf_quiesce_perform_failover(xlator_t *this)
+{
+    int ret = 0;
+    call_frame_t *frame = NULL;
+    dict_t *dict = NULL;
+    quiesce_priv_t *priv = NULL;
+    quiesce_failover_hosts_t *failover_host = NULL;
+    quiesce_failover_hosts_t *host = NULL;
+
+    priv = this->private;
+
+    if (priv->pass_through) {
+        gf_msg_trace(this->name, 0,
+                     "child is up, hence not "
+                     "performing any failover");
+        goto out;
+    }
+
+    list_for_each_entry(failover_host, &priv->failover_list, list)
+    {
+        if (failover_host->tried == 0) {
+            host = failover_host;
+            failover_host->tried = 1;
+            break;
+        }
+    }
+    if (!host) {
+        /*TODO: Keep trying until any of the gfproxy comes back up.
+                Currently it tries failing over once for each host,
+                if it doesn't succeed then returns error to mount point
+           list_for_each_entry (failover_host,
+                        &priv->failover_list, list) {
+                failover_host->tried = 0;
+        }*/
+        gf_msg_debug(this->name, 0,
+                     "all the failover hosts have "
+                     "been tried and looks like didn't succeed");
+        ret = -1;
+        goto out;
+    }
+
+    frame = create_frame(this, this->ctx->pool);
+    if (!frame) {
+        gf_msg_debug(this->name, 0, "failed to create the frame");
+        ret = -1;
+        goto out;
+    }
+
+    dict = dict_new();
+
+    ret = dict_set_dynstr(dict, CLIENT_CMD_CONNECT, gf_strdup(host->addr));
+
+    gf_msg_trace(this->name, 0, "Initiating failover to:%s", host->addr);
+
+    STACK_WIND_COOKIE(frame, gf_quiesce_failover_cbk, NULL, FIRST_CHILD(this),
+                      FIRST_CHILD(this)->fops->setxattr, NULL, dict, 0, NULL);
+out:
 
-void
-gf_quiesce_timeout (void *data)
+    if (dict)
+        dict_unref(dict);
+
+    return ret;
+}
+
+call_stub_t *
+gf_quiesce_dequeue(xlator_t *this)
 {
-        xlator_t       *this = NULL;
-        quiesce_priv_t *priv = NULL;
+    call_stub_t *stub = NULL;
+    quiesce_priv_t *priv = NULL;
 
-        this = data;
-        priv = this->private;
-        THIS = this;
+    priv = this->private;
 
-        LOCK (&priv->lock);
-        {
-                priv->pass_through = _gf_true;
-        }
-        UNLOCK (&priv->lock);
+    if (!priv || list_empty(&priv->req))
+        return NULL;
 
-        gf_quiesce_dequeue_start (this);
+    LOCK(&priv->lock);
+    {
+        stub = list_entry(priv->req.next, call_stub_t, list);
+        list_del_init(&stub->list);
+        priv->queue_size--;
+    }
+    UNLOCK(&priv->lock);
 
-        return;
+    return stub;
 }
 
-void
-gf_quiesce_enqueue (xlator_t *this, call_stub_t *stub)
+void *
+gf_quiesce_dequeue_start(void *data)
 {
-        quiesce_priv_t *priv    = NULL;
-        struct timespec timeout = {0,};
+    xlator_t *this = NULL;
+    quiesce_priv_t *priv = NULL;
+    call_stub_t *stub = NULL;
 
-        priv = this->private;
-        if (!priv) {
-                gf_log_callingfn (this->name, GF_LOG_ERROR,
-                                  "this->private == NULL");
-                return;
-        }
+    this = data;
+    priv = this->private;
+    THIS = this;
 
-        LOCK (&priv->lock);
-        {
-                list_add_tail (&stub->list, &priv->req);
-                priv->queue_size++;
+    while (!list_empty(&priv->req)) {
+        stub = gf_quiesce_dequeue(this);
+        if (stub) {
+            call_resume(stub);
         }
-        UNLOCK (&priv->lock);
+    }
+
+    return 0;
+}
+
+void
+gf_quiesce_timeout(void *data)
+{
+    xlator_t *this = NULL;
+    quiesce_priv_t *priv = NULL;
+    int ret = -1;
 
-        if (!priv->timer) {
-                timeout.tv_sec = 20;
-                timeout.tv_nsec = 0;
+    this = data;
+    priv = this->private;
+    THIS = this;
 
-                priv->timer = gf_timer_call_after (this->ctx,
-                                                   timeout,
-                                                   gf_quiesce_timeout,
-                                                   (void *) this);
+    LOCK(&priv->lock);
+    {
+        priv->timer = NULL;
+        if (priv->pass_through) {
+            UNLOCK(&priv->lock);
+            goto out;
         }
+        ret = __gf_quiesce_perform_failover(THIS);
+    }
+    UNLOCK(&priv->lock);
 
-        return;
+    if (ret < 0) {
+        priv->pass_through = _gf_true;
+        gf_quiesce_dequeue_start(this);
+    }
+
+out:
+    return;
 }
 
+void
+gf_quiesce_enqueue(xlator_t *this, call_stub_t *stub)
+{
+    quiesce_priv_t *priv = NULL;
 
+    priv = this->private;
+    if (!priv) {
+        gf_log_callingfn(this->name, GF_LOG_ERROR, "this->private == NULL");
+        return;
+    }
+
+    LOCK(&priv->lock);
+    {
+        list_add_tail(&stub->list, &priv->req);
+        priv->queue_size++;
+        __gf_quiesce_start_timer(this, priv);
+    }
+    UNLOCK(&priv->lock);
+
+    return;
+}
 
 /* _CBK function section */
 
 int32_t
-quiesce_lookup_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                    int32_t op_ret, int32_t op_errno, inode_t *inode,
-                    struct iatt *buf, dict_t *dict, struct iatt *postparent)
+quiesce_lookup_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                   int32_t op_ret, int32_t op_errno, inode_t *inode,
+                   struct iatt *buf, dict_t *dict, struct iatt *postparent)
 {
-        call_stub_t    *stub = NULL;
-        quiesce_local_t *local = NULL;
+    call_stub_t *stub = NULL;
+    quiesce_local_t *local = NULL;
 
-        local = frame->local;
-        frame->local = NULL;
-        if ((op_ret == -1) && (op_errno == ENOTCONN)) {
-                /* Re-transmit (by putting in the queue) */
-                stub = fop_lookup_stub (frame, default_lookup_resume,
-                                        &local->loc, local->dict);
-                if (!stub) {
-                        STACK_UNWIND_STRICT (lookup, frame, -1, ENOMEM,
-                                             NULL, NULL, NULL, NULL);
-                        goto out;
-                }
-
-                gf_quiesce_enqueue (this, stub);
-                goto out;
+    local = frame->local;
+    frame->local = NULL;
+    if ((op_ret == -1) && (op_errno == ENOTCONN)) {
+        /* Re-transmit (by putting in the queue) */
+        stub = fop_lookup_stub(frame, default_lookup_resume, &local->loc,
+                               local->dict);
+        if (!stub) {
+            STACK_UNWIND_STRICT(lookup, frame, -1, ENOMEM, NULL, NULL, NULL,
+                                NULL);
+            goto out;
         }
 
-        STACK_UNWIND_STRICT (lookup, frame, op_ret, op_errno, inode, buf,
-                             dict, postparent);
+        gf_quiesce_enqueue(this, stub);
+        goto out;
+    }
+
+    STACK_UNWIND_STRICT(lookup, frame, op_ret, op_errno, inode, buf, dict,
+                        postparent);
 out:
-        gf_quiesce_local_wipe (this, local);
+    gf_quiesce_local_wipe(this, local);
 
-        return 0;
+    return 0;
 }
 
 int32_t
-quiesce_stat_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                  int32_t op_ret, int32_t op_errno, struct iatt *buf,
-                  dict_t *xdata)
+quiesce_stat_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                 int32_t op_ret, int32_t op_errno, struct iatt *buf,
+                 dict_t *xdata)
 {
-        call_stub_t    *stub = NULL;
-        quiesce_local_t *local = NULL;
-
-        local = frame->local;
-        frame->local = NULL;
-        if ((op_ret == -1) && (op_errno == ENOTCONN)) {
-                /* Re-transmit (by putting in the queue) */
-                stub = fop_stat_stub (frame, default_stat_resume,
-                                      &local->loc, xdata);
-                if (!stub) {
-                        STACK_UNWIND_STRICT (stat, frame, -1, ENOMEM,
-                                             NULL, NULL);
-                        goto out;
-                }
+    call_stub_t *stub = NULL;
+    quiesce_local_t *local = NULL;
 
-                gf_quiesce_enqueue (this, stub);
-                goto out;
+    local = frame->local;
+    frame->local = NULL;
+    if ((op_ret == -1) && (op_errno == ENOTCONN)) {
+        /* Re-transmit (by putting in the queue) */
+        stub = fop_stat_stub(frame, default_stat_resume, &local->loc, xdata);
+        if (!stub) {
+            STACK_UNWIND_STRICT(stat, frame, -1, ENOMEM, NULL, NULL);
+            goto out;
         }
 
-        STACK_UNWIND_STRICT (stat, frame, op_ret, op_errno, buf, xdata);
+        gf_quiesce_enqueue(this, stub);
+        goto out;
+    }
+
+    STACK_UNWIND_STRICT(stat, frame, op_ret, op_errno, buf, xdata);
 out:
-        gf_quiesce_local_wipe (this, local);
+    gf_quiesce_local_wipe(this, local);
 
-        return 0;
+    return 0;
 }
 
 int32_t
-quiesce_access_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                    int32_t op_ret, int32_t op_errno, dict_t *xdata)
+quiesce_access_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                   int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
-        call_stub_t    *stub = NULL;
-        quiesce_local_t *local = NULL;
-
-        local = frame->local;
-        frame->local = NULL;
-        if ((op_ret == -1) && (op_errno == ENOTCONN)) {
-                /* Re-transmit (by putting in the queue) */
-                stub = fop_access_stub (frame, default_access_resume,
-                                        &local->loc, local->flag, xdata);
-                if (!stub) {
-                        STACK_UNWIND_STRICT (access, frame, -1, ENOMEM, NULL);
-                        goto out;
-                }
+    call_stub_t *stub = NULL;
+    quiesce_local_t *local = NULL;
 
-                gf_quiesce_enqueue (this, stub);
-                goto out;
+    local = frame->local;
+    frame->local = NULL;
+    if ((op_ret == -1) && (op_errno == ENOTCONN)) {
+        /* Re-transmit (by putting in the queue) */
+        stub = fop_access_stub(frame, default_access_resume, &local->loc,
+                               local->flag, xdata);
+        if (!stub) {
+            STACK_UNWIND_STRICT(access, frame, -1, ENOMEM, NULL);
+            goto out;
         }
 
-        STACK_UNWIND_STRICT (access, frame, op_ret, op_errno, xdata);
+        gf_quiesce_enqueue(this, stub);
+        goto out;
+    }
+
+    STACK_UNWIND_STRICT(access, frame, op_ret, op_errno, xdata);
 out:
-        gf_quiesce_local_wipe (this, local);
+    gf_quiesce_local_wipe(this, local);
 
-        return 0;
+    return 0;
 }
 
 int32_t
-quiesce_readlink_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                      int32_t op_ret, int32_t op_errno, const char *path,
-                      struct iatt *buf, dict_t *xdata)
+quiesce_readlink_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                     int32_t op_ret, int32_t op_errno, const char *path,
+                     struct iatt *buf, dict_t *xdata)
 {
-        call_stub_t    *stub = NULL;
-        quiesce_local_t *local = NULL;
-
-        local = frame->local;
-        frame->local = NULL;
-        if ((op_ret == -1) && (op_errno == ENOTCONN)) {
-                /* Re-transmit (by putting in the queue) */
-                stub = fop_readlink_stub (frame, default_readlink_resume,
-                                          &local->loc, local->size, xdata);
-                if (!stub) {
-                        STACK_UNWIND_STRICT (readlink, frame, -1, ENOMEM,
-                                             NULL, NULL, NULL);
-                        goto out;
-                }
+    call_stub_t *stub = NULL;
+    quiesce_local_t *local = NULL;
 
-                gf_quiesce_enqueue (this, stub);
-                goto out;
+    local = frame->local;
+    frame->local = NULL;
+    if ((op_ret == -1) && (op_errno == ENOTCONN)) {
+        /* Re-transmit (by putting in the queue) */
+        stub = fop_readlink_stub(frame, default_readlink_resume, &local->loc,
+                                 local->size, xdata);
+        if (!stub) {
+            STACK_UNWIND_STRICT(readlink, frame, -1, ENOMEM, NULL, NULL, NULL);
+            goto out;
         }
 
-        STACK_UNWIND_STRICT (readlink, frame, op_ret, op_errno, path, buf, xdata);
+        gf_quiesce_enqueue(this, stub);
+        goto out;
+    }
+
+    STACK_UNWIND_STRICT(readlink, frame, op_ret, op_errno, path, buf, xdata);
 out:
-        gf_quiesce_local_wipe (this, local);
+    gf_quiesce_local_wipe(this, local);
 
-        return 0;
+    return 0;
 }
 
 int32_t
-quiesce_open_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                  int32_t op_ret, int32_t op_errno, fd_t *fd, dict_t *xdata)
+quiesce_open_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                 int32_t op_ret, int32_t op_errno, fd_t *fd, dict_t *xdata)
 {
-        call_stub_t    *stub = NULL;
-        quiesce_local_t *local = NULL;
+    call_stub_t *stub = NULL;
+    quiesce_local_t *local = NULL;
 
-        local = frame->local;
-        frame->local = NULL;
-        if ((op_ret == -1) && (op_errno == ENOTCONN)) {
-                /* Re-transmit (by putting in the queue) */
-                stub = fop_open_stub (frame, default_open_resume,
-                                      &local->loc, local->flag, local->fd,
-                                      xdata);
-                if (!stub) {
-                        STACK_UNWIND_STRICT (open, frame, -1, ENOMEM,
-                                             NULL, NULL);
-                        goto out;
-                }
-
-                gf_quiesce_enqueue (this, stub);
-                goto out;
+    local = frame->local;
+    frame->local = NULL;
+    if ((op_ret == -1) && (op_errno == ENOTCONN)) {
+        /* Re-transmit (by putting in the queue) */
+        stub = fop_open_stub(frame, default_open_resume, &local->loc,
+                             local->flag, local->fd, xdata);
+        if (!stub) {
+            STACK_UNWIND_STRICT(open, frame, -1, ENOMEM, NULL, NULL);
+            goto out;
         }
 
-        STACK_UNWIND_STRICT (open, frame, op_ret, op_errno, fd, xdata);
+        gf_quiesce_enqueue(this, stub);
+        goto out;
+    }
+
+    STACK_UNWIND_STRICT(open, frame, op_ret, op_errno, fd, xdata);
 out:
-        gf_quiesce_local_wipe (this, local);
+    gf_quiesce_local_wipe(this, local);
 
-        return 0;
+    return 0;
 }
 
 int32_t
-quiesce_readv_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                   int32_t op_ret, int32_t op_errno, struct iovec *vector,
-                   int32_t count, struct iatt *stbuf, struct iobref *iobref, dict_t *xdata)
+quiesce_readv_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                  int32_t op_ret, int32_t op_errno, struct iovec *vector,
+                  int32_t count, struct iatt *stbuf, struct iobref *iobref,
+                  dict_t *xdata)
 {
-        call_stub_t    *stub = NULL;
-        quiesce_local_t *local = NULL;
-
-        local = frame->local;
-        frame->local = NULL;
-        if ((op_ret == -1) && (op_errno == ENOTCONN)) {
-                /* Re-transmit (by putting in the queue) */
-                stub = fop_readv_stub (frame, default_readv_resume,
-                                       local->fd, local->size, local->offset,
-                                       local->io_flag, xdata);
-                if (!stub) {
-                        STACK_UNWIND_STRICT (readv, frame, -1, ENOMEM,
-                                             NULL, 0, NULL, NULL, NULL);
-                        goto out;
-                }
+    call_stub_t *stub = NULL;
+    quiesce_local_t *local = NULL;
 
-                gf_quiesce_enqueue (this, stub);
-                goto out;
+    local = frame->local;
+    frame->local = NULL;
+    if ((op_ret == -1) && (op_errno == ENOTCONN)) {
+        /* Re-transmit (by putting in the queue) */
+        stub = fop_readv_stub(frame, default_readv_resume, local->fd,
+                              local->size, local->offset, local->io_flag,
+                              xdata);
+        if (!stub) {
+            STACK_UNWIND_STRICT(readv, frame, -1, ENOMEM, NULL, 0, NULL, NULL,
+                                NULL);
+            goto out;
         }
 
-        STACK_UNWIND_STRICT (readv, frame, op_ret, op_errno, vector, count,
-                             stbuf, iobref, xdata);
+        gf_quiesce_enqueue(this, stub);
+        goto out;
+    }
+
+    STACK_UNWIND_STRICT(readv, frame, op_ret, op_errno, vector, count, stbuf,
+                        iobref, xdata);
 out:
-        gf_quiesce_local_wipe (this, local);
+    gf_quiesce_local_wipe(this, local);
 
-        return 0;
+    return 0;
 }
 
 int32_t
-quiesce_flush_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                   int32_t op_ret, int32_t op_errno, dict_t *xdata)
+quiesce_flush_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                  int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
-        call_stub_t    *stub = NULL;
-        quiesce_local_t *local = NULL;
+    call_stub_t *stub = NULL;
+    quiesce_local_t *local = NULL;
 
-        local = frame->local;
-        frame->local = NULL;
-        if ((op_ret == -1) && (op_errno == ENOTCONN)) {
-                /* Re-transmit (by putting in the queue) */
-                stub = fop_flush_stub (frame, default_flush_resume,
-                                       local->fd, xdata);
-                if (!stub) {
-                        STACK_UNWIND_STRICT (flush, frame, -1, ENOMEM, NULL);
-                        goto out;
-                }
-
-                gf_quiesce_enqueue (this, stub);
-                goto out;
+    local = frame->local;
+    frame->local = NULL;
+    if ((op_ret == -1) && (op_errno == ENOTCONN)) {
+        /* Re-transmit (by putting in the queue) */
+        stub = fop_flush_stub(frame, default_flush_resume, local->fd, xdata);
+        if (!stub) {
+            STACK_UNWIND_STRICT(flush, frame, -1, ENOMEM, NULL);
+            goto out;
         }
 
-        STACK_UNWIND_STRICT (flush, frame, op_ret, op_errno, xdata);
+        gf_quiesce_enqueue(this, stub);
+        goto out;
+    }
+
+    STACK_UNWIND_STRICT(flush, frame, op_ret, op_errno, xdata);
 out:
-        gf_quiesce_local_wipe (this, local);
+    gf_quiesce_local_wipe(this, local);
 
-        return 0;
+    return 0;
 }
 
-
-
 int32_t
-quiesce_fsync_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                   int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
-                   struct iatt *postbuf, dict_t *xdata)
+quiesce_fsync_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                  int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
+                  struct iatt *postbuf, dict_t *xdata)
 {
-        call_stub_t    *stub = NULL;
-        quiesce_local_t *local = NULL;
+    call_stub_t *stub = NULL;
+    quiesce_local_t *local = NULL;
 
-        local = frame->local;
-        frame->local = NULL;
-        if ((op_ret == -1) && (op_errno == ENOTCONN)) {
-                /* Re-transmit (by putting in the queue) */
-                stub = fop_fsync_stub (frame, default_fsync_resume,
-                                       local->fd, local->flag, xdata);
-                if (!stub) {
-                        STACK_UNWIND_STRICT (fsync, frame, -1, ENOMEM,
-                                             NULL, NULL, NULL);
-                        goto out;
-                }
-
-                gf_quiesce_enqueue (this, stub);
-                goto out;
+    local = frame->local;
+    frame->local = NULL;
+    if ((op_ret == -1) && (op_errno == ENOTCONN)) {
+        /* Re-transmit (by putting in the queue) */
+        stub = fop_fsync_stub(frame, default_fsync_resume, local->fd,
+                              local->flag, xdata);
+        if (!stub) {
+            STACK_UNWIND_STRICT(fsync, frame, -1, ENOMEM, NULL, NULL, NULL);
+            goto out;
         }
 
-        STACK_UNWIND_STRICT (fsync, frame, op_ret, op_errno, prebuf, postbuf, xdata);
+        gf_quiesce_enqueue(this, stub);
+        goto out;
+    }
+
+    STACK_UNWIND_STRICT(fsync, frame, op_ret, op_errno, prebuf, postbuf, xdata);
 out:
-        gf_quiesce_local_wipe (this, local);
+    gf_quiesce_local_wipe(this, local);
 
-        return 0;
+    return 0;
 }
 
 int32_t
-quiesce_fstat_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                   int32_t op_ret, int32_t op_errno, struct iatt *buf, dict_t *xdata)
+quiesce_fstat_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                  int32_t op_ret, int32_t op_errno, struct iatt *buf,
+                  dict_t *xdata)
 {
-        call_stub_t    *stub = NULL;
-        quiesce_local_t *local = NULL;
-
-        local = frame->local;
-        frame->local = NULL;
-        if ((op_ret == -1) && (op_errno == ENOTCONN)) {
-                /* Re-transmit (by putting in the queue) */
-                stub = fop_fstat_stub (frame, default_fstat_resume,
-                                       local->fd, xdata);
-                if (!stub) {
-                        STACK_UNWIND_STRICT (fstat, frame, -1, ENOMEM,
-                                             NULL, NULL);
-                        goto out;
-                }
+    call_stub_t *stub = NULL;
+    quiesce_local_t *local = NULL;
 
-                gf_quiesce_enqueue (this, stub);
-                goto out;
+    local = frame->local;
+    frame->local = NULL;
+    if ((op_ret == -1) && (op_errno == ENOTCONN)) {
+        /* Re-transmit (by putting in the queue) */
+        stub = fop_fstat_stub(frame, default_fstat_resume, local->fd, xdata);
+        if (!stub) {
+            STACK_UNWIND_STRICT(fstat, frame, -1, ENOMEM, NULL, NULL);
+            goto out;
         }
 
-        STACK_UNWIND_STRICT (fstat, frame, op_ret, op_errno, buf, xdata);
+        gf_quiesce_enqueue(this, stub);
+        goto out;
+    }
+
+    STACK_UNWIND_STRICT(fstat, frame, op_ret, op_errno, buf, xdata);
 out:
-        gf_quiesce_local_wipe (this, local);
+    gf_quiesce_local_wipe(this, local);
 
-        return 0;
+    return 0;
 }
 
 int32_t
-quiesce_opendir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                     int32_t op_ret, int32_t op_errno, fd_t *fd, dict_t *xdata)
+quiesce_opendir_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                    int32_t op_ret, int32_t op_errno, fd_t *fd, dict_t *xdata)
 {
-        call_stub_t    *stub = NULL;
-        quiesce_local_t *local = NULL;
+    call_stub_t *stub = NULL;
+    quiesce_local_t *local = NULL;
 
-        local = frame->local;
-        frame->local = NULL;
-        if ((op_ret == -1) && (op_errno == ENOTCONN)) {
-                /* Re-transmit (by putting in the queue) */
-                stub = fop_opendir_stub (frame, default_opendir_resume,
-                                         &local->loc, local->fd, xdata);
-                if (!stub) {
-                        STACK_UNWIND_STRICT (opendir, frame, -1, ENOMEM,
-                                             NULL, NULL);
-                        goto out;
-                }
-
-                gf_quiesce_enqueue (this, stub);
-                goto out;
+    local = frame->local;
+    frame->local = NULL;
+    if ((op_ret == -1) && (op_errno == ENOTCONN)) {
+        /* Re-transmit (by putting in the queue) */
+        stub = fop_opendir_stub(frame, default_opendir_resume, &local->loc,
+                                local->fd, xdata);
+        if (!stub) {
+            STACK_UNWIND_STRICT(opendir, frame, -1, ENOMEM, NULL, NULL);
+            goto out;
         }
 
-        STACK_UNWIND_STRICT (opendir, frame, op_ret, op_errno, fd, xdata);
+        gf_quiesce_enqueue(this, stub);
+        goto out;
+    }
+
+    STACK_UNWIND_STRICT(opendir, frame, op_ret, op_errno, fd, xdata);
 out:
-        gf_quiesce_local_wipe (this, local);
+    gf_quiesce_local_wipe(this, local);
 
-        return 0;
+    return 0;
 }
 
 int32_t
-quiesce_fsyncdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                      int32_t op_ret, int32_t op_errno, dict_t *xdata)
+quiesce_fsyncdir_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                     int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
-        call_stub_t    *stub = NULL;
-        quiesce_local_t *local = NULL;
+    call_stub_t *stub = NULL;
+    quiesce_local_t *local = NULL;
 
-        local = frame->local;
-        frame->local = NULL;
-        if ((op_ret == -1) && (op_errno == ENOTCONN)) {
-                /* Re-transmit (by putting in the queue) */
-                stub = fop_fsyncdir_stub (frame, default_fsyncdir_resume,
-                                          local->fd, local->flag, xdata);
-                if (!stub) {
-                        STACK_UNWIND_STRICT (fsyncdir, frame, -1, ENOMEM, NULL);
-                        goto out;
-                }
-
-                gf_quiesce_enqueue (this, stub);
-                goto out;
+    local = frame->local;
+    frame->local = NULL;
+    if ((op_ret == -1) && (op_errno == ENOTCONN)) {
+        /* Re-transmit (by putting in the queue) */
+        stub = fop_fsyncdir_stub(frame, default_fsyncdir_resume, local->fd,
+                                 local->flag, xdata);
+        if (!stub) {
+            STACK_UNWIND_STRICT(fsyncdir, frame, -1, ENOMEM, NULL);
+            goto out;
         }
 
-        STACK_UNWIND_STRICT (fsyncdir, frame, op_ret, op_errno, xdata);
+        gf_quiesce_enqueue(this, stub);
+        goto out;
+    }
+
+    STACK_UNWIND_STRICT(fsyncdir, frame, op_ret, op_errno, xdata);
 out:
-        gf_quiesce_local_wipe (this, local);
+    gf_quiesce_local_wipe(this, local);
 
-        return 0;
+    return 0;
 }
 
 int32_t
-quiesce_statfs_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                    int32_t op_ret, int32_t op_errno, struct statvfs *buf, dict_t *xdata)
+quiesce_statfs_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                   int32_t op_ret, int32_t op_errno, struct statvfs *buf,
+                   dict_t *xdata)
 {
-        call_stub_t    *stub = NULL;
-        quiesce_local_t *local = NULL;
+    call_stub_t *stub = NULL;
+    quiesce_local_t *local = NULL;
 
-        local = frame->local;
-        frame->local = NULL;
-        if ((op_ret == -1) && (op_errno == ENOTCONN)) {
-                /* Re-transmit (by putting in the queue) */
-                stub = fop_statfs_stub (frame, default_statfs_resume,
-                                        &local->loc, xdata);
-                if (!stub) {
-                        STACK_UNWIND_STRICT (statfs, frame, -1, ENOMEM,
-                                             NULL, NULL);
-                        goto out;
-                }
-
-                gf_quiesce_enqueue (this, stub);
-                goto out;
+    local = frame->local;
+    frame->local = NULL;
+    if ((op_ret == -1) && (op_errno == ENOTCONN)) {
+        /* Re-transmit (by putting in the queue) */
+        stub = fop_statfs_stub(frame, default_statfs_resume, &local->loc,
+                               xdata);
+        if (!stub) {
+            STACK_UNWIND_STRICT(statfs, frame, -1, ENOMEM, NULL, NULL);
+            goto out;
         }
 
-        STACK_UNWIND_STRICT (statfs, frame, op_ret, op_errno, buf, xdata);
+        gf_quiesce_enqueue(this, stub);
+        goto out;
+    }
+
+    STACK_UNWIND_STRICT(statfs, frame, op_ret, op_errno, buf, xdata);
 out:
-        gf_quiesce_local_wipe (this, local);
+    gf_quiesce_local_wipe(this, local);
 
-        return 0;
+    return 0;
 }
 
 int32_t
-quiesce_fgetxattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                       int32_t op_ret, int32_t op_errno, dict_t *dict, dict_t *xdata)
+quiesce_fgetxattr_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                      int32_t op_ret, int32_t op_errno, dict_t *dict,
+                      dict_t *xdata)
 {
-        call_stub_t    *stub = NULL;
-        quiesce_local_t *local = NULL;
-
-        local = frame->local;
-        frame->local = NULL;
-        if ((op_ret == -1) && (op_errno == ENOTCONN)) {
-                /* Re-transmit (by putting in the queue) */
-                stub = fop_fgetxattr_stub (frame, default_fgetxattr_resume,
-                                           local->fd, local->name, xdata);
-                if (!stub) {
-                        STACK_UNWIND_STRICT (fgetxattr, frame, -1, ENOMEM,
-                                             NULL, NULL);
-                        goto out;
-                }
+    call_stub_t *stub = NULL;
+    quiesce_local_t *local = NULL;
 
-                gf_quiesce_enqueue (this, stub);
-                goto out;
+    local = frame->local;
+    frame->local = NULL;
+    if ((op_ret == -1) && (op_errno == ENOTCONN)) {
+        /* Re-transmit (by putting in the queue) */
+        stub = fop_fgetxattr_stub(frame, default_fgetxattr_resume, local->fd,
+                                  local->name, xdata);
+        if (!stub) {
+            STACK_UNWIND_STRICT(fgetxattr, frame, -1, ENOMEM, NULL, NULL);
+            goto out;
         }
 
-        STACK_UNWIND_STRICT (fgetxattr, frame, op_ret, op_errno, dict, xdata);
+        gf_quiesce_enqueue(this, stub);
+        goto out;
+    }
+
+    STACK_UNWIND_STRICT(fgetxattr, frame, op_ret, op_errno, dict, xdata);
 out:
-        gf_quiesce_local_wipe (this, local);
+    gf_quiesce_local_wipe(this, local);
 
-        return 0;
+    return 0;
 }
 
-
 int32_t
-quiesce_getxattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                      int32_t op_ret, int32_t op_errno, dict_t *dict, dict_t *xdata)
+quiesce_getxattr_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                     int32_t op_ret, int32_t op_errno, dict_t *dict,
+                     dict_t *xdata)
 {
-        call_stub_t    *stub = NULL;
-        quiesce_local_t *local = NULL;
+    call_stub_t *stub = NULL;
+    quiesce_local_t *local = NULL;
 
-        local = frame->local;
-        frame->local = NULL;
-        if ((op_ret == -1) && (op_errno == ENOTCONN)) {
-                /* Re-transmit (by putting in the queue) */
-                stub = fop_getxattr_stub (frame, default_getxattr_resume,
-                                          &local->loc, local->name, xdata);
-                if (!stub) {
-                        STACK_UNWIND_STRICT (getxattr, frame, -1, ENOMEM,
-                                             NULL, NULL);
-                        goto out;
-                }
-
-                gf_quiesce_enqueue (this, stub);
-                goto out;
+    local = frame->local;
+    frame->local = NULL;
+    if ((op_ret == -1) && (op_errno == ENOTCONN)) {
+        /* Re-transmit (by putting in the queue) */
+        stub = fop_getxattr_stub(frame, default_getxattr_resume, &local->loc,
+                                 local->name, xdata);
+        if (!stub) {
+            STACK_UNWIND_STRICT(getxattr, frame, -1, ENOMEM, NULL, NULL);
+            goto out;
         }
 
-        STACK_UNWIND_STRICT (getxattr, frame, op_ret, op_errno, dict, xdata);
+        gf_quiesce_enqueue(this, stub);
+        goto out;
+    }
+
+    STACK_UNWIND_STRICT(getxattr, frame, op_ret, op_errno, dict, xdata);
 out:
-        gf_quiesce_local_wipe (this, local);
+    gf_quiesce_local_wipe(this, local);
 
-        return 0;
+    return 0;
 }
 
-
 int32_t
-quiesce_rchecksum_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                       int32_t op_ret, int32_t op_errno, uint32_t weak_checksum,
-                       uint8_t *strong_checksum, dict_t *xdata)
+quiesce_rchecksum_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                      int32_t op_ret, int32_t op_errno, uint32_t weak_checksum,
+                      uint8_t *strong_checksum, dict_t *xdata)
 {
-        call_stub_t    *stub = NULL;
-        quiesce_local_t *local = NULL;
+    call_stub_t *stub = NULL;
+    quiesce_local_t *local = NULL;
 
-        local = frame->local;
-        frame->local = NULL;
-        if ((op_ret == -1) && (op_errno == ENOTCONN)) {
-                /* Re-transmit (by putting in the queue) */
-                stub = fop_rchecksum_stub (frame, default_rchecksum_resume,
-                                           local->fd, local->offset, local->flag, xdata);
-                if (!stub) {
-                        STACK_UNWIND_STRICT (rchecksum, frame, -1, ENOMEM,
-                                             0, NULL, NULL);
-                        goto out;
-                }
-
-                gf_quiesce_enqueue (this, stub);
-                goto out;
+    local = frame->local;
+    frame->local = NULL;
+    if ((op_ret == -1) && (op_errno == ENOTCONN)) {
+        /* Re-transmit (by putting in the queue) */
+        stub = fop_rchecksum_stub(frame, default_rchecksum_resume, local->fd,
+                                  local->offset, local->flag, xdata);
+        if (!stub) {
+            STACK_UNWIND_STRICT(rchecksum, frame, -1, ENOMEM, 0, NULL, NULL);
+            goto out;
         }
 
-        STACK_UNWIND_STRICT (rchecksum, frame, op_ret, op_errno, weak_checksum,
-                             strong_checksum, xdata);
+        gf_quiesce_enqueue(this, stub);
+        goto out;
+    }
+
+    STACK_UNWIND_STRICT(rchecksum, frame, op_ret, op_errno, weak_checksum,
+                        strong_checksum, xdata);
 out:
-        gf_quiesce_local_wipe (this, local);
+    gf_quiesce_local_wipe(this, local);
 
-        return 0;
+    return 0;
 }
 
-
 int32_t
-quiesce_readdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                     int32_t op_ret, int32_t op_errno, gf_dirent_t *entries, dict_t *xdata)
+quiesce_readdir_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                    int32_t op_ret, int32_t op_errno, gf_dirent_t *entries,
+                    dict_t *xdata)
 {
-        call_stub_t    *stub = NULL;
-        quiesce_local_t *local = NULL;
+    call_stub_t *stub = NULL;
+    quiesce_local_t *local = NULL;
 
-        local = frame->local;
-        frame->local = NULL;
-        if ((op_ret == -1) && (op_errno == ENOTCONN)) {
-                /* Re-transmit (by putting in the queue) */
-                stub = fop_readdir_stub (frame, default_readdir_resume,
-                                         local->fd, local->size, local->offset, xdata);
-                if (!stub) {
-                        STACK_UNWIND_STRICT (readdir, frame, -1, ENOMEM,
-                                             NULL, NULL);
-                        goto out;
-                }
-
-                gf_quiesce_enqueue (this, stub);
-                goto out;
+    local = frame->local;
+    frame->local = NULL;
+    if ((op_ret == -1) && (op_errno == ENOTCONN)) {
+        /* Re-transmit (by putting in the queue) */
+        stub = fop_readdir_stub(frame, default_readdir_resume, local->fd,
+                                local->size, local->offset, xdata);
+        if (!stub) {
+            STACK_UNWIND_STRICT(readdir, frame, -1, ENOMEM, NULL, NULL);
+            goto out;
         }
 
-        STACK_UNWIND_STRICT (readdir, frame, op_ret, op_errno, entries, xdata);
+        gf_quiesce_enqueue(this, stub);
+        goto out;
+    }
+
+    STACK_UNWIND_STRICT(readdir, frame, op_ret, op_errno, entries, xdata);
 out:
-        gf_quiesce_local_wipe (this, local);
+    gf_quiesce_local_wipe(this, local);
 
-        return 0;
+    return 0;
 }
 
-
 int32_t
-quiesce_readdirp_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                      int32_t op_ret, int32_t op_errno, gf_dirent_t *entries, dict_t *xdata)
+quiesce_readdirp_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                     int32_t op_ret, int32_t op_errno, gf_dirent_t *entries,
+                     dict_t *xdata)
 {
-        call_stub_t    *stub = NULL;
-        quiesce_local_t *local = NULL;
+    call_stub_t *stub = NULL;
+    quiesce_local_t *local = NULL;
 
-        local = frame->local;
-        frame->local = NULL;
-        if ((op_ret == -1) && (op_errno == ENOTCONN)) {
-                /* Re-transmit (by putting in the queue) */
-                stub = fop_readdirp_stub (frame, default_readdirp_resume,
-                                          local->fd, local->size, local->offset,
-                                          local->dict);
-                if (!stub) {
-                        STACK_UNWIND_STRICT (readdirp, frame, -1, ENOMEM,
-                                             NULL, NULL);
-                        goto out;
-                }
-
-                gf_quiesce_enqueue (this, stub);
-                goto out;
+    local = frame->local;
+    frame->local = NULL;
+    if ((op_ret == -1) && (op_errno == ENOTCONN)) {
+        /* Re-transmit (by putting in the queue) */
+        stub = fop_readdirp_stub(frame, default_readdirp_resume, local->fd,
+                                 local->size, local->offset, local->dict);
+        if (!stub) {
+            STACK_UNWIND_STRICT(readdirp, frame, -1, ENOMEM, NULL, NULL);
+            goto out;
         }
 
-        STACK_UNWIND_STRICT (readdirp, frame, op_ret, op_errno, entries, xdata);
+        gf_quiesce_enqueue(this, stub);
+        goto out;
+    }
+
+    STACK_UNWIND_STRICT(readdirp, frame, op_ret, op_errno, entries, xdata);
 out:
-        gf_quiesce_local_wipe (this, local);
+    gf_quiesce_local_wipe(this, local);
 
-        return 0;
+    return 0;
 }
 
-
 #if 0
 
 int32_t
@@ -1015,1596 +1161,1544 @@ out:
 
 #endif /* if 0 */
 
-
 /* FOP */
 
 /* No retransmittion */
 
 int32_t
-quiesce_removexattr (call_frame_t *frame,
-		     xlator_t *this,
-		     loc_t *loc,
-		     const char *name, dict_t *xdata)
+quiesce_removexattr(call_frame_t *frame, xlator_t *this, loc_t *loc,
+                    const char *name, dict_t *xdata)
 {
-	quiesce_priv_t *priv = NULL;
-        call_stub_t    *stub = NULL;
+    quiesce_priv_t *priv = NULL;
+    call_stub_t *stub = NULL;
 
-        priv = this->private;
+    priv = this->private;
 
-        if (priv->pass_through) {
-                STACK_WIND (frame,
-                            default_removexattr_cbk,
-                            FIRST_CHILD(this),
-                            FIRST_CHILD(this)->fops->removexattr,
-                            loc,
-                            name, xdata);
-                return 0;
-        }
+    if (priv->pass_through) {
+        STACK_WIND(frame, default_removexattr_cbk, FIRST_CHILD(this),
+                   FIRST_CHILD(this)->fops->removexattr, loc, name, xdata);
+        return 0;
+    }
 
-        stub = fop_removexattr_stub (frame, default_removexattr_resume,
-                                     loc, name, xdata);
-        if (!stub) {
-                STACK_UNWIND_STRICT (removexattr, frame, -1, ENOMEM, NULL);
-                return 0;
-        }
+    stub = fop_removexattr_stub(frame, default_removexattr_resume, loc, name,
+                                xdata);
+    if (!stub) {
+        STACK_UNWIND_STRICT(removexattr, frame, -1, ENOMEM, NULL);
+        return 0;
+    }
 
-        gf_quiesce_enqueue (this, stub);
+    gf_quiesce_enqueue(this, stub);
 
-        return 0;
+    return 0;
 }
 
 int32_t
-quiesce_truncate (call_frame_t *frame,
-		  xlator_t *this,
-		  loc_t *loc,
-		  off_t offset, dict_t *xdata)
+quiesce_fremovexattr(call_frame_t *frame, xlator_t *this, fd_t *fd,
+                     const char *name, dict_t *xdata)
 {
-	quiesce_priv_t *priv = NULL;
-        call_stub_t    *stub = NULL;
+    quiesce_priv_t *priv = NULL;
+    call_stub_t *stub = NULL;
 
-        priv = this->private;
+    priv = this->private;
 
-        if (priv->pass_through) {
-                STACK_WIND (frame,
-                            default_truncate_cbk,
-                            FIRST_CHILD(this),
-                            FIRST_CHILD(this)->fops->truncate,
-                            loc,
-                            offset, xdata);
-                return 0;
-        }
+    if (priv->pass_through) {
+        STACK_WIND(frame, default_fremovexattr_cbk, FIRST_CHILD(this),
+                   FIRST_CHILD(this)->fops->fremovexattr, fd, name, xdata);
+        return 0;
+    }
 
-        stub = fop_truncate_stub (frame, default_truncate_resume, loc, offset, xdata);
-        if (!stub) {
-                STACK_UNWIND_STRICT (truncate, frame, -1, ENOMEM, NULL, NULL, NULL);
-                return 0;
-        }
+    stub = fop_fremovexattr_stub(frame, default_fremovexattr_resume, fd, name,
+                                 xdata);
+    if (!stub) {
+        STACK_UNWIND_STRICT(fremovexattr, frame, -1, ENOMEM, NULL);
+        return 0;
+    }
 
-        gf_quiesce_enqueue (this, stub);
+    gf_quiesce_enqueue(this, stub);
 
-        return 0;
+    return 0;
 }
 
 int32_t
-quiesce_fsetxattr (call_frame_t *frame,
-                   xlator_t *this,
-                   fd_t *fd,
-                   dict_t *dict,
-                   int32_t flags, dict_t *xdata)
+quiesce_truncate(call_frame_t *frame, xlator_t *this, loc_t *loc, off_t offset,
+                 dict_t *xdata)
 {
-	quiesce_priv_t *priv = NULL;
-        call_stub_t    *stub = NULL;
+    quiesce_priv_t *priv = NULL;
+    call_stub_t *stub = NULL;
 
-        priv = this->private;
+    priv = this->private;
 
-        if (priv->pass_through) {
-                STACK_WIND (frame,
-                            default_fsetxattr_cbk,
-                            FIRST_CHILD(this),
-                            FIRST_CHILD(this)->fops->fsetxattr,
-                            fd,
-                            dict,
-                            flags, xdata);
-                return 0;
-        }
+    if (priv->pass_through) {
+        STACK_WIND(frame, default_truncate_cbk, FIRST_CHILD(this),
+                   FIRST_CHILD(this)->fops->truncate, loc, offset, xdata);
+        return 0;
+    }
 
-        stub = fop_fsetxattr_stub (frame, default_fsetxattr_resume,
-                                   fd, dict, flags, xdata);
-        if (!stub) {
-                STACK_UNWIND_STRICT (fsetxattr, frame, -1, ENOMEM, NULL);
-                return 0;
-        }
+    stub = fop_truncate_stub(frame, default_truncate_resume, loc, offset,
+                             xdata);
+    if (!stub) {
+        STACK_UNWIND_STRICT(truncate, frame, -1, ENOMEM, NULL, NULL, NULL);
+        return 0;
+    }
 
-        gf_quiesce_enqueue (this, stub);
+    gf_quiesce_enqueue(this, stub);
 
-        return 0;
+    return 0;
 }
 
 int32_t
-quiesce_setxattr (call_frame_t *frame,
-		  xlator_t *this,
-		  loc_t *loc,
-		  dict_t *dict,
-		  int32_t flags, dict_t *xdata)
+quiesce_fsetxattr(call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *dict,
+                  int32_t flags, dict_t *xdata)
 {
-	quiesce_priv_t *priv = NULL;
-        call_stub_t    *stub = NULL;
+    quiesce_priv_t *priv = NULL;
+    call_stub_t *stub = NULL;
 
-        priv = this->private;
+    priv = this->private;
 
-        if (priv->pass_through) {
-                STACK_WIND (frame,
-                            default_setxattr_cbk,
-                            FIRST_CHILD(this),
-                            FIRST_CHILD(this)->fops->setxattr,
-                            loc,
-                            dict,
-                            flags, xdata);
-                return 0;
-        }
+    if (priv->pass_through) {
+        STACK_WIND(frame, default_fsetxattr_cbk, FIRST_CHILD(this),
+                   FIRST_CHILD(this)->fops->fsetxattr, fd, dict, flags, xdata);
+        return 0;
+    }
 
-        stub = fop_setxattr_stub (frame, default_setxattr_resume,
-                                  loc, dict, flags, xdata);
-        if (!stub) {
-                STACK_UNWIND_STRICT (setxattr, frame, -1, ENOMEM, NULL);
-                return 0;
-        }
+    stub = fop_fsetxattr_stub(frame, default_fsetxattr_resume, fd, dict, flags,
+                              xdata);
+    if (!stub) {
+        STACK_UNWIND_STRICT(fsetxattr, frame, -1, ENOMEM, NULL);
+        return 0;
+    }
 
-        gf_quiesce_enqueue (this, stub);
+    gf_quiesce_enqueue(this, stub);
 
-        return 0;
+    return 0;
 }
 
 int32_t
-quiesce_create (call_frame_t *frame, xlator_t *this,
-		loc_t *loc, int32_t flags, mode_t mode,
-                mode_t umask, fd_t *fd, dict_t *xdata)
+quiesce_setxattr(call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *dict,
+                 int32_t flags, dict_t *xdata)
 {
-	quiesce_priv_t *priv = NULL;
-        call_stub_t    *stub = NULL;
+    quiesce_priv_t *priv = NULL;
+    call_stub_t *stub = NULL;
 
-        priv = this->private;
+    priv = this->private;
 
-        if (priv->pass_through) {
-                /* Don't send O_APPEND below, as write() re-transmittions can
-                   fail with O_APPEND */
-                STACK_WIND (frame, default_create_cbk,
-                            FIRST_CHILD(this),
-                            FIRST_CHILD(this)->fops->create,
-                            loc, (flags & ~O_APPEND), mode, umask, fd, xdata);
-                return 0;
-        }
+    if (priv->pass_through) {
+        STACK_WIND(frame, default_setxattr_cbk, FIRST_CHILD(this),
+                   FIRST_CHILD(this)->fops->setxattr, loc, dict, flags, xdata);
+        return 0;
+    }
 
-        stub = fop_create_stub (frame, default_create_resume,
-                                loc, (flags & ~O_APPEND), mode, umask, fd, xdata);
-        if (!stub) {
-                STACK_UNWIND_STRICT (create, frame, -1, ENOMEM,
-                                     NULL, NULL, NULL, NULL, NULL, NULL);
-                return 0;
-        }
+    stub = fop_setxattr_stub(frame, default_setxattr_resume, loc, dict, flags,
+                             xdata);
+    if (!stub) {
+        STACK_UNWIND_STRICT(setxattr, frame, -1, ENOMEM, NULL);
+        return 0;
+    }
 
-        gf_quiesce_enqueue (this, stub);
+    gf_quiesce_enqueue(this, stub);
 
-        return 0;
+    return 0;
 }
 
 int32_t
-quiesce_link (call_frame_t *frame,
-	      xlator_t *this,
-	      loc_t *oldloc,
-	      loc_t *newloc, dict_t *xdata)
+quiesce_create(call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t flags,
+               mode_t mode, mode_t umask, fd_t *fd, dict_t *xdata)
 {
-	quiesce_priv_t *priv = NULL;
-        call_stub_t    *stub = NULL;
+    quiesce_priv_t *priv = NULL;
+    call_stub_t *stub = NULL;
 
-        priv = this->private;
+    priv = this->private;
 
-        if (priv->pass_through) {
-                STACK_WIND (frame,
-                            default_link_cbk,
-                            FIRST_CHILD(this),
-                            FIRST_CHILD(this)->fops->link,
-                            oldloc, newloc, xdata);
-                return 0;
-        }
+    if (priv->pass_through) {
+        /* Don't send O_APPEND below, as write() re-transmittions can
+           fail with O_APPEND */
+        STACK_WIND(frame, default_create_cbk, FIRST_CHILD(this),
+                   FIRST_CHILD(this)->fops->create, loc, (flags & ~O_APPEND),
+                   mode, umask, fd, xdata);
+        return 0;
+    }
 
-        stub = fop_link_stub (frame, default_link_resume, oldloc, newloc, xdata);
-        if (!stub) {
-                STACK_UNWIND_STRICT (link, frame, -1, ENOMEM,
-                                     NULL, NULL, NULL, NULL, NULL);
-                return 0;
-        }
+    stub = fop_create_stub(frame, default_create_resume, loc,
+                           (flags & ~O_APPEND), mode, umask, fd, xdata);
+    if (!stub) {
+        STACK_UNWIND_STRICT(create, frame, -1, ENOMEM, NULL, NULL, NULL, NULL,
+                            NULL, NULL);
+        return 0;
+    }
 
-        gf_quiesce_enqueue (this, stub);
+    gf_quiesce_enqueue(this, stub);
 
-        return 0;
+    return 0;
 }
 
 int32_t
-quiesce_rename (call_frame_t *frame,
-		xlator_t *this,
-		loc_t *oldloc,
-		loc_t *newloc, dict_t *xdata)
+quiesce_link(call_frame_t *frame, xlator_t *this, loc_t *oldloc, loc_t *newloc,
+             dict_t *xdata)
 {
-	quiesce_priv_t *priv = NULL;
-        call_stub_t    *stub = NULL;
+    quiesce_priv_t *priv = NULL;
+    call_stub_t *stub = NULL;
 
-        priv = this->private;
+    priv = this->private;
 
-        if (priv->pass_through) {
-                STACK_WIND (frame,
-                            default_rename_cbk,
-                            FIRST_CHILD(this),
-                            FIRST_CHILD(this)->fops->rename,
-                            oldloc, newloc, xdata);
-                return 0;
-        }
+    if (priv->pass_through) {
+        STACK_WIND(frame, default_link_cbk, FIRST_CHILD(this),
+                   FIRST_CHILD(this)->fops->link, oldloc, newloc, xdata);
+        return 0;
+    }
 
-        stub = fop_rename_stub (frame, default_rename_resume, oldloc, newloc, xdata);
-        if (!stub) {
-                STACK_UNWIND_STRICT (rename, frame, -1, ENOMEM,
-                                     NULL, NULL, NULL, NULL, NULL, NULL);
-                return 0;
-        }
+    stub = fop_link_stub(frame, default_link_resume, oldloc, newloc, xdata);
+    if (!stub) {
+        STACK_UNWIND_STRICT(link, frame, -1, ENOMEM, NULL, NULL, NULL, NULL,
+                            NULL);
+        return 0;
+    }
 
-        gf_quiesce_enqueue (this, stub);
+    gf_quiesce_enqueue(this, stub);
 
-        return 0;
+    return 0;
 }
 
+int32_t
+quiesce_rename(call_frame_t *frame, xlator_t *this, loc_t *oldloc,
+               loc_t *newloc, dict_t *xdata)
+{
+    quiesce_priv_t *priv = NULL;
+    call_stub_t *stub = NULL;
+
+    priv = this->private;
+
+    if (priv->pass_through) {
+        STACK_WIND(frame, default_rename_cbk, FIRST_CHILD(this),
+                   FIRST_CHILD(this)->fops->rename, oldloc, newloc, xdata);
+        return 0;
+    }
+
+    stub = fop_rename_stub(frame, default_rename_resume, oldloc, newloc, xdata);
+    if (!stub) {
+        STACK_UNWIND_STRICT(rename, frame, -1, ENOMEM, NULL, NULL, NULL, NULL,
+                            NULL, NULL);
+        return 0;
+    }
+
+    gf_quiesce_enqueue(this, stub);
+
+    return 0;
+}
 
 int
-quiesce_symlink (call_frame_t *frame, xlator_t *this,
-		 const char *linkpath, loc_t *loc, mode_t umask, dict_t *xdata)
+quiesce_symlink(call_frame_t *frame, xlator_t *this, const char *linkpath,
+                loc_t *loc, mode_t umask, dict_t *xdata)
 {
-	quiesce_priv_t *priv = NULL;
-        call_stub_t    *stub = NULL;
+    quiesce_priv_t *priv = NULL;
+    call_stub_t *stub = NULL;
 
-        priv = this->private;
+    priv = this->private;
 
-        if (priv->pass_through) {
-                STACK_WIND (frame, default_symlink_cbk,
-                            FIRST_CHILD(this),
-                            FIRST_CHILD(this)->fops->symlink,
-                            linkpath, loc, umask, xdata);
-                return 0;
-        }
+    if (priv->pass_through) {
+        STACK_WIND(frame, default_symlink_cbk, FIRST_CHILD(this),
+                   FIRST_CHILD(this)->fops->symlink, linkpath, loc, umask,
+                   xdata);
+        return 0;
+    }
 
-        stub = fop_symlink_stub (frame, default_symlink_resume,
-                                 linkpath, loc, umask, xdata);
-        if (!stub) {
-                STACK_UNWIND_STRICT (symlink, frame, -1, ENOMEM,
-                                     NULL, NULL, NULL, NULL, NULL);
-                return 0;
-        }
+    stub = fop_symlink_stub(frame, default_symlink_resume, linkpath, loc, umask,
+                            xdata);
+    if (!stub) {
+        STACK_UNWIND_STRICT(symlink, frame, -1, ENOMEM, NULL, NULL, NULL, NULL,
+                            NULL);
+        return 0;
+    }
 
-        gf_quiesce_enqueue (this, stub);
+    gf_quiesce_enqueue(this, stub);
 
-        return 0;
+    return 0;
 }
 
-
 int
-quiesce_rmdir (call_frame_t *frame, xlator_t *this, loc_t *loc, int flags, dict_t *xdata)
+quiesce_rmdir(call_frame_t *frame, xlator_t *this, loc_t *loc, int flags,
+              dict_t *xdata)
 {
-	quiesce_priv_t *priv = NULL;
-        call_stub_t    *stub = NULL;
+    quiesce_priv_t *priv = NULL;
+    call_stub_t *stub = NULL;
 
-        priv = this->private;
+    priv = this->private;
 
-        if (priv->pass_through) {
-                STACK_WIND (frame, default_rmdir_cbk,
-                            FIRST_CHILD(this),
-                            FIRST_CHILD(this)->fops->rmdir,
-                            loc, flags, xdata);
-                return 0;
-        }
+    if (priv->pass_through) {
+        STACK_WIND(frame, default_rmdir_cbk, FIRST_CHILD(this),
+                   FIRST_CHILD(this)->fops->rmdir, loc, flags, xdata);
+        return 0;
+    }
 
-        stub = fop_rmdir_stub (frame, default_rmdir_resume, loc, flags, xdata);
-        if (!stub) {
-                STACK_UNWIND_STRICT (rmdir, frame, -1, ENOMEM, NULL, NULL, NULL);
-                return 0;
-        }
+    stub = fop_rmdir_stub(frame, default_rmdir_resume, loc, flags, xdata);
+    if (!stub) {
+        STACK_UNWIND_STRICT(rmdir, frame, -1, ENOMEM, NULL, NULL, NULL);
+        return 0;
+    }
 
-        gf_quiesce_enqueue (this, stub);
+    gf_quiesce_enqueue(this, stub);
 
-        return 0;
+    return 0;
 }
 
 int32_t
-quiesce_unlink (call_frame_t *frame,
-		xlator_t *this,
-		loc_t *loc, int xflag, dict_t *xdata)
+quiesce_unlink(call_frame_t *frame, xlator_t *this, loc_t *loc, int xflag,
+               dict_t *xdata)
 {
-	quiesce_priv_t *priv = NULL;
-        call_stub_t    *stub = NULL;
+    quiesce_priv_t *priv = NULL;
+    call_stub_t *stub = NULL;
 
-        priv = this->private;
+    priv = this->private;
 
-        if (priv->pass_through) {
-                STACK_WIND (frame,
-                            default_unlink_cbk,
-                            FIRST_CHILD(this),
-                            FIRST_CHILD(this)->fops->unlink,
-                            loc, xflag, xdata);
-                return 0;
-        }
+    if (priv->pass_through) {
+        STACK_WIND(frame, default_unlink_cbk, FIRST_CHILD(this),
+                   FIRST_CHILD(this)->fops->unlink, loc, xflag, xdata);
+        return 0;
+    }
 
-        stub = fop_unlink_stub (frame, default_unlink_resume, loc, xflag, xdata);
-        if (!stub) {
-                STACK_UNWIND_STRICT (unlink, frame, -1, ENOMEM, NULL, NULL, NULL);
-                return 0;
-        }
+    stub = fop_unlink_stub(frame, default_unlink_resume, loc, xflag, xdata);
+    if (!stub) {
+        STACK_UNWIND_STRICT(unlink, frame, -1, ENOMEM, NULL, NULL, NULL);
+        return 0;
+    }
 
-        gf_quiesce_enqueue (this, stub);
+    gf_quiesce_enqueue(this, stub);
 
-        return 0;
+    return 0;
 }
 
 int
-quiesce_mkdir (call_frame_t *frame, xlator_t *this,
-	       loc_t *loc, mode_t mode, mode_t umask, dict_t *xdata)
+quiesce_mkdir(call_frame_t *frame, xlator_t *this, loc_t *loc, mode_t mode,
+              mode_t umask, dict_t *xdata)
 {
-	quiesce_priv_t *priv = NULL;
-        call_stub_t    *stub = NULL;
+    quiesce_priv_t *priv = NULL;
+    call_stub_t *stub = NULL;
 
-        priv = this->private;
+    priv = this->private;
 
-        if (priv->pass_through) {
-                STACK_WIND (frame, default_mkdir_cbk,
-                            FIRST_CHILD(this),
-                            FIRST_CHILD(this)->fops->mkdir,
-                            loc, mode, umask, xdata);
-                return 0;
-        }
+    if (priv->pass_through) {
+        STACK_WIND(frame, default_mkdir_cbk, FIRST_CHILD(this),
+                   FIRST_CHILD(this)->fops->mkdir, loc, mode, umask, xdata);
+        return 0;
+    }
 
-        stub = fop_mkdir_stub (frame, default_mkdir_resume,
-                               loc, mode, umask, xdata);
-        if (!stub) {
-                STACK_UNWIND_STRICT (mkdir, frame, -1, ENOMEM,
-                                     NULL, NULL, NULL, NULL, NULL);
-                return 0;
-        }
+    stub = fop_mkdir_stub(frame, default_mkdir_resume, loc, mode, umask, xdata);
+    if (!stub) {
+        STACK_UNWIND_STRICT(mkdir, frame, -1, ENOMEM, NULL, NULL, NULL, NULL,
+                            NULL);
+        return 0;
+    }
 
-        gf_quiesce_enqueue (this, stub);
+    gf_quiesce_enqueue(this, stub);
 
-        return 0;
+    return 0;
 }
 
-
 int
-quiesce_mknod (call_frame_t *frame, xlator_t *this,
-	       loc_t *loc, mode_t mode, dev_t rdev, mode_t umask, dict_t *xdata)
+quiesce_mknod(call_frame_t *frame, xlator_t *this, loc_t *loc, mode_t mode,
+              dev_t rdev, mode_t umask, dict_t *xdata)
 {
-	quiesce_priv_t *priv = NULL;
-        call_stub_t    *stub = NULL;
+    quiesce_priv_t *priv = NULL;
+    call_stub_t *stub = NULL;
 
-        priv = this->private;
+    priv = this->private;
 
-        if (priv->pass_through) {
-                STACK_WIND (frame, default_mknod_cbk,
-                            FIRST_CHILD(this),
-                            FIRST_CHILD(this)->fops->mknod,
-                            loc, mode, rdev, umask, xdata);
-                return 0;
-        }
+    if (priv->pass_through) {
+        STACK_WIND(frame, default_mknod_cbk, FIRST_CHILD(this),
+                   FIRST_CHILD(this)->fops->mknod, loc, mode, rdev, umask,
+                   xdata);
+        return 0;
+    }
 
-        stub = fop_mknod_stub (frame, default_mknod_resume,
-                               loc, mode, rdev, umask, xdata);
-        if (!stub) {
-                STACK_UNWIND_STRICT (mknod, frame, -1, ENOMEM,
-                                     NULL, NULL, NULL, NULL, NULL);
-                return 0;
-        }
+    stub = fop_mknod_stub(frame, default_mknod_resume, loc, mode, rdev, umask,
+                          xdata);
+    if (!stub) {
+        STACK_UNWIND_STRICT(mknod, frame, -1, ENOMEM, NULL, NULL, NULL, NULL,
+                            NULL);
+        return 0;
+    }
 
-        gf_quiesce_enqueue (this, stub);
+    gf_quiesce_enqueue(this, stub);
 
-        return 0;
+    return 0;
 }
 
 int32_t
-quiesce_ftruncate (call_frame_t *frame,
-		   xlator_t *this,
-		   fd_t *fd,
-		   off_t offset, dict_t *xdata)
+quiesce_ftruncate(call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
+                  dict_t *xdata)
 {
-	quiesce_priv_t *priv = NULL;
-        call_stub_t    *stub = NULL;
+    quiesce_priv_t *priv = NULL;
+    call_stub_t *stub = NULL;
 
-        priv = this->private;
+    priv = this->private;
 
-        if (priv->pass_through) {
-                STACK_WIND (frame,
-                            default_ftruncate_cbk,
-                            FIRST_CHILD(this),
-                            FIRST_CHILD(this)->fops->ftruncate,
-                            fd,
-                            offset, xdata);
-                return 0;
-        }
+    if (priv->pass_through) {
+        STACK_WIND(frame, default_ftruncate_cbk, FIRST_CHILD(this),
+                   FIRST_CHILD(this)->fops->ftruncate, fd, offset, xdata);
+        return 0;
+    }
 
-        stub = fop_ftruncate_stub (frame, default_ftruncate_resume, fd, offset, xdata);
-        if (!stub) {
-                STACK_UNWIND_STRICT (ftruncate, frame, -1, ENOMEM, NULL, NULL, NULL);
-                return 0;
-        }
+    stub = fop_ftruncate_stub(frame, default_ftruncate_resume, fd, offset,
+                              xdata);
+    if (!stub) {
+        STACK_UNWIND_STRICT(ftruncate, frame, -1, ENOMEM, NULL, NULL, NULL);
+        return 0;
+    }
 
-        gf_quiesce_enqueue (this, stub);
+    gf_quiesce_enqueue(this, stub);
 
-        return 0;
+    return 0;
 }
 
 /* Re-transmittion */
 
 int32_t
-quiesce_readlink (call_frame_t *frame,
-		  xlator_t *this,
-		  loc_t *loc,
-		  size_t size, dict_t *xdata)
+quiesce_readlink(call_frame_t *frame, xlator_t *this, loc_t *loc, size_t size,
+                 dict_t *xdata)
 {
-	quiesce_priv_t *priv = NULL;
-        call_stub_t    *stub = NULL;
-        quiesce_local_t *local = NULL;
+    quiesce_priv_t *priv = NULL;
+    call_stub_t *stub = NULL;
+    quiesce_local_t *local = NULL;
 
-        priv = this->private;
+    priv = this->private;
 
-        if (priv && priv->pass_through) {
-                local = mem_get0 (priv->local_pool);
-                loc_dup (loc, &local->loc);
-                local->size = size;
-                frame->local = local;
-
-                STACK_WIND (frame,
-                            quiesce_readlink_cbk,
-                            FIRST_CHILD(this),
-                            FIRST_CHILD(this)->fops->readlink,
-                            loc,
-                            size, xdata);
-                return 0;
-        }
+    if (priv && priv->pass_through) {
+        local = mem_get0(priv->local_pool);
+        loc_dup(loc, &local->loc);
+        local->size = size;
+        frame->local = local;
 
-        stub = fop_readlink_stub (frame, default_readlink_resume, loc, size, xdata);
-        if (!stub) {
-                STACK_UNWIND_STRICT (readlink, frame, -1, ENOMEM, NULL, NULL, NULL);
-                return 0;
-        }
-
-        gf_quiesce_enqueue (this, stub);
+        STACK_WIND(frame, quiesce_readlink_cbk, FIRST_CHILD(this),
+                   FIRST_CHILD(this)->fops->readlink, loc, size, xdata);
+        return 0;
+    }
 
+    stub = fop_readlink_stub(frame, default_readlink_resume, loc, size, xdata);
+    if (!stub) {
+        STACK_UNWIND_STRICT(readlink, frame, -1, ENOMEM, NULL, NULL, NULL);
         return 0;
-}
+    }
 
+    gf_quiesce_enqueue(this, stub);
+
+    return 0;
+}
 
 int32_t
-quiesce_access (call_frame_t *frame,
-		xlator_t *this,
-		loc_t *loc,
-		int32_t mask, dict_t *xdata)
+quiesce_access(call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t mask,
+               dict_t *xdata)
 {
-	quiesce_priv_t *priv = NULL;
-        call_stub_t    *stub = NULL;
-        quiesce_local_t *local = NULL;
-
-        priv = this->private;
+    quiesce_priv_t *priv = NULL;
+    call_stub_t *stub = NULL;
+    quiesce_local_t *local = NULL;
 
-        if (priv && priv->pass_through) {
-                local = mem_get0 (priv->local_pool);
-                loc_dup (loc, &local->loc);
-                local->flag = mask;
-                frame->local = local;
-
-                STACK_WIND (frame,
-                            quiesce_access_cbk,
-                            FIRST_CHILD(this),
-                            FIRST_CHILD(this)->fops->access,
-                            loc,
-                            mask, xdata);
-                return 0;
-        }
+    priv = this->private;
 
-        stub = fop_access_stub (frame, default_access_resume, loc, mask, xdata);
-        if (!stub) {
-                STACK_UNWIND_STRICT (access, frame, -1, ENOMEM, NULL);
-                return 0;
-        }
+    if (priv && priv->pass_through) {
+        local = mem_get0(priv->local_pool);
+        loc_dup(loc, &local->loc);
+        local->flag = mask;
+        frame->local = local;
 
-        gf_quiesce_enqueue (this, stub);
+        STACK_WIND(frame, quiesce_access_cbk, FIRST_CHILD(this),
+                   FIRST_CHILD(this)->fops->access, loc, mask, xdata);
+        return 0;
+    }
 
+    stub = fop_access_stub(frame, default_access_resume, loc, mask, xdata);
+    if (!stub) {
+        STACK_UNWIND_STRICT(access, frame, -1, ENOMEM, NULL);
         return 0;
+    }
+
+    gf_quiesce_enqueue(this, stub);
+
+    return 0;
 }
 
 int32_t
-quiesce_fgetxattr (call_frame_t *frame,
-                   xlator_t *this,
-                   fd_t *fd,
-                   const char *name, dict_t *xdata)
+quiesce_fgetxattr(call_frame_t *frame, xlator_t *this, fd_t *fd,
+                  const char *name, dict_t *xdata)
 {
-        quiesce_priv_t *priv = NULL;
-        call_stub_t    *stub = NULL;
-        quiesce_local_t *local = NULL;
+    quiesce_priv_t *priv = NULL;
+    call_stub_t *stub = NULL;
+    quiesce_local_t *local = NULL;
 
-        priv = this->private;
+    priv = this->private;
 
-        if (priv && priv->pass_through) {
-                local = mem_get0 (priv->local_pool);
-                local->fd = fd_ref (fd);
-                if (name)
-                        local->name = gf_strdup (name);
-
-                frame->local = local;
-
-                STACK_WIND (frame,
-                            quiesce_fgetxattr_cbk,
-                            FIRST_CHILD(this),
-                            FIRST_CHILD(this)->fops->fgetxattr,
-                            fd,
-                            name, xdata);
-                return 0;
-        }
+    if (priv && priv->pass_through) {
+        local = mem_get0(priv->local_pool);
+        local->fd = fd_ref(fd);
+        if (name)
+            local->name = gf_strdup(name);
 
-        stub = fop_fgetxattr_stub (frame, default_fgetxattr_resume, fd, name, xdata);
-        if (!stub) {
-                STACK_UNWIND_STRICT (fgetxattr, frame, -1, ENOMEM, NULL, NULL);
-                return 0;
-        }
+        frame->local = local;
 
-        gf_quiesce_enqueue (this, stub);
+        STACK_WIND(frame, quiesce_fgetxattr_cbk, FIRST_CHILD(this),
+                   FIRST_CHILD(this)->fops->fgetxattr, fd, name, xdata);
+        return 0;
+    }
 
+    stub = fop_fgetxattr_stub(frame, default_fgetxattr_resume, fd, name, xdata);
+    if (!stub) {
+        STACK_UNWIND_STRICT(fgetxattr, frame, -1, ENOMEM, NULL, NULL);
         return 0;
+    }
+
+    gf_quiesce_enqueue(this, stub);
+
+    return 0;
 }
 
 int32_t
-quiesce_statfs (call_frame_t *frame,
-		xlator_t *this,
-		loc_t *loc, dict_t *xdata)
+quiesce_statfs(call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xdata)
 {
-	quiesce_priv_t *priv = NULL;
-        call_stub_t    *stub = NULL;
-        quiesce_local_t *local = NULL;
+    quiesce_priv_t *priv = NULL;
+    call_stub_t *stub = NULL;
+    quiesce_local_t *local = NULL;
 
-        priv = this->private;
+    priv = this->private;
 
-        if (priv && priv->pass_through) {
-                local = mem_get0 (priv->local_pool);
-                loc_dup (loc, &local->loc);
-                frame->local = local;
-
-                STACK_WIND (frame,
-                            quiesce_statfs_cbk,
-                            FIRST_CHILD(this),
-                            FIRST_CHILD(this)->fops->statfs,
-                            loc, xdata);
-                return 0;
-        }
-
-        stub = fop_statfs_stub (frame, default_statfs_resume, loc, xdata);
-        if (!stub) {
-                STACK_UNWIND_STRICT (statfs, frame, -1, ENOMEM, NULL, NULL);
-                return 0;
-        }
+    if (priv && priv->pass_through) {
+        local = mem_get0(priv->local_pool);
+        loc_dup(loc, &local->loc);
+        frame->local = local;
 
-        gf_quiesce_enqueue (this, stub);
+        STACK_WIND(frame, quiesce_statfs_cbk, FIRST_CHILD(this),
+                   FIRST_CHILD(this)->fops->statfs, loc, xdata);
+        return 0;
+    }
 
+    stub = fop_statfs_stub(frame, default_statfs_resume, loc, xdata);
+    if (!stub) {
+        STACK_UNWIND_STRICT(statfs, frame, -1, ENOMEM, NULL, NULL);
         return 0;
+    }
+
+    gf_quiesce_enqueue(this, stub);
+
+    return 0;
 }
 
 int32_t
-quiesce_fsyncdir (call_frame_t *frame,
-		  xlator_t *this,
-		  fd_t *fd,
-		  int32_t flags, dict_t *xdata)
+quiesce_fsyncdir(call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t flags,
+                 dict_t *xdata)
 {
-	quiesce_priv_t *priv = NULL;
-        call_stub_t    *stub = NULL;
-        quiesce_local_t *local = NULL;
+    quiesce_priv_t *priv = NULL;
+    call_stub_t *stub = NULL;
+    quiesce_local_t *local = NULL;
 
-        priv = this->private;
+    priv = this->private;
 
-        if (priv && priv->pass_through) {
-                local = mem_get0 (priv->local_pool);
-                local->fd = fd_ref (fd);
-                local->flag = flags;
-                frame->local = local;
-
-                STACK_WIND (frame,
-                            quiesce_fsyncdir_cbk,
-                            FIRST_CHILD(this),
-                            FIRST_CHILD(this)->fops->fsyncdir,
-                            fd,
-                            flags, xdata);
-                return 0;
-        }
-
-        stub = fop_fsyncdir_stub (frame, default_fsyncdir_resume, fd, flags, xdata);
-        if (!stub) {
-                STACK_UNWIND_STRICT (fsyncdir, frame, -1, ENOMEM, NULL);
-                return 0;
-        }
+    if (priv && priv->pass_through) {
+        local = mem_get0(priv->local_pool);
+        local->fd = fd_ref(fd);
+        local->flag = flags;
+        frame->local = local;
 
-        gf_quiesce_enqueue (this, stub);
+        STACK_WIND(frame, quiesce_fsyncdir_cbk, FIRST_CHILD(this),
+                   FIRST_CHILD(this)->fops->fsyncdir, fd, flags, xdata);
+        return 0;
+    }
 
+    stub = fop_fsyncdir_stub(frame, default_fsyncdir_resume, fd, flags, xdata);
+    if (!stub) {
+        STACK_UNWIND_STRICT(fsyncdir, frame, -1, ENOMEM, NULL);
         return 0;
+    }
+
+    gf_quiesce_enqueue(this, stub);
+
+    return 0;
 }
 
 int32_t
-quiesce_opendir (call_frame_t *frame,
-		 xlator_t *this,
-		 loc_t *loc, fd_t *fd, dict_t *xdata)
+quiesce_opendir(call_frame_t *frame, xlator_t *this, loc_t *loc, fd_t *fd,
+                dict_t *xdata)
 {
-	quiesce_priv_t *priv = NULL;
-        call_stub_t    *stub = NULL;
-        quiesce_local_t *local = NULL;
+    quiesce_priv_t *priv = NULL;
+    call_stub_t *stub = NULL;
+    quiesce_local_t *local = NULL;
 
-        priv = this->private;
+    priv = this->private;
 
-        if (priv && priv->pass_through) {
-                local = mem_get0 (priv->local_pool);
-                loc_dup (loc, &local->loc);
-                local->fd = fd_ref (fd);
-                frame->local = local;
-
-                STACK_WIND (frame,
-                            quiesce_opendir_cbk,
-                            FIRST_CHILD(this),
-                            FIRST_CHILD(this)->fops->opendir,
-                            loc, fd, xdata);
-                return 0;
-        }
+    if (priv && priv->pass_through) {
+        local = mem_get0(priv->local_pool);
+        loc_dup(loc, &local->loc);
+        local->fd = fd_ref(fd);
+        frame->local = local;
 
-        stub = fop_opendir_stub (frame, default_opendir_resume, loc, fd, xdata);
-        if (!stub) {
-                STACK_UNWIND_STRICT (opendir, frame, -1, ENOMEM, NULL, NULL);
-                return 0;
-        }
-
-        gf_quiesce_enqueue (this, stub);
+        STACK_WIND(frame, quiesce_opendir_cbk, FIRST_CHILD(this),
+                   FIRST_CHILD(this)->fops->opendir, loc, fd, xdata);
+        return 0;
+    }
 
+    stub = fop_opendir_stub(frame, default_opendir_resume, loc, fd, xdata);
+    if (!stub) {
+        STACK_UNWIND_STRICT(opendir, frame, -1, ENOMEM, NULL, NULL);
         return 0;
+    }
+
+    gf_quiesce_enqueue(this, stub);
+
+    return 0;
 }
 
 int32_t
-quiesce_fstat (call_frame_t *frame,
-	       xlator_t *this,
-	       fd_t *fd, dict_t *xdata)
+quiesce_fstat(call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *xdata)
 {
-	quiesce_priv_t *priv = NULL;
-        call_stub_t    *stub = NULL;
-        quiesce_local_t *local = NULL;
+    quiesce_priv_t *priv = NULL;
+    call_stub_t *stub = NULL;
+    quiesce_local_t *local = NULL;
 
-        priv = this->private;
-
-        if (priv && priv->pass_through) {
-                local = mem_get0 (priv->local_pool);
-                local->fd = fd_ref (fd);
-                frame->local = local;
-
-                STACK_WIND (frame,
-                            quiesce_fstat_cbk,
-                            FIRST_CHILD(this),
-                            FIRST_CHILD(this)->fops->fstat,
-                            fd, xdata);
-                return 0;
-        }
+    priv = this->private;
 
-        stub = fop_fstat_stub (frame, default_fstat_resume, fd, xdata);
-        if (!stub) {
-                STACK_UNWIND_STRICT (fstat, frame, -1, ENOMEM, NULL, NULL);
-                return 0;
-        }
+    if (priv && priv->pass_through) {
+        local = mem_get0(priv->local_pool);
+        local->fd = fd_ref(fd);
+        frame->local = local;
 
-        gf_quiesce_enqueue (this, stub);
+        STACK_WIND(frame, quiesce_fstat_cbk, FIRST_CHILD(this),
+                   FIRST_CHILD(this)->fops->fstat, fd, xdata);
+        return 0;
+    }
 
+    stub = fop_fstat_stub(frame, default_fstat_resume, fd, xdata);
+    if (!stub) {
+        STACK_UNWIND_STRICT(fstat, frame, -1, ENOMEM, NULL, NULL);
         return 0;
+    }
+
+    gf_quiesce_enqueue(this, stub);
+
+    return 0;
 }
 
 int32_t
-quiesce_fsync (call_frame_t *frame,
-	       xlator_t *this,
-	       fd_t *fd,
-	       int32_t flags, dict_t *xdata)
+quiesce_fsync(call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t flags,
+              dict_t *xdata)
 {
-	quiesce_priv_t *priv = NULL;
-        call_stub_t    *stub = NULL;
-        quiesce_local_t *local = NULL;
+    quiesce_priv_t *priv = NULL;
+    call_stub_t *stub = NULL;
+    quiesce_local_t *local = NULL;
 
-        priv = this->private;
+    priv = this->private;
 
-        if (priv && priv->pass_through) {
-                local = mem_get0 (priv->local_pool);
-                local->fd = fd_ref (fd);
-                local->flag = flags;
-                frame->local = local;
-
-                STACK_WIND (frame,
-                            quiesce_fsync_cbk,
-                            FIRST_CHILD(this),
-                            FIRST_CHILD(this)->fops->fsync,
-                            fd,
-                            flags, xdata);
-                return 0;
-        }
+    if (priv && priv->pass_through) {
+        local = mem_get0(priv->local_pool);
+        local->fd = fd_ref(fd);
+        local->flag = flags;
+        frame->local = local;
 
-        stub = fop_fsync_stub (frame, default_fsync_resume, fd, flags, xdata);
-        if (!stub) {
-                STACK_UNWIND_STRICT (fsync, frame, -1, ENOMEM, NULL, NULL, NULL);
-                return 0;
-        }
-
-        gf_quiesce_enqueue (this, stub);
+        STACK_WIND(frame, quiesce_fsync_cbk, FIRST_CHILD(this),
+                   FIRST_CHILD(this)->fops->fsync, fd, flags, xdata);
+        return 0;
+    }
 
+    stub = fop_fsync_stub(frame, default_fsync_resume, fd, flags, xdata);
+    if (!stub) {
+        STACK_UNWIND_STRICT(fsync, frame, -1, ENOMEM, NULL, NULL, NULL);
         return 0;
+    }
+
+    gf_quiesce_enqueue(this, stub);
+
+    return 0;
 }
 
 int32_t
-quiesce_flush (call_frame_t *frame,
-	       xlator_t *this,
-	       fd_t *fd, dict_t *xdata)
+quiesce_flush(call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *xdata)
 {
-	quiesce_priv_t *priv = NULL;
-        call_stub_t    *stub = NULL;
-        quiesce_local_t *local = NULL;
-
-        priv = this->private;
+    quiesce_priv_t *priv = NULL;
+    call_stub_t *stub = NULL;
+    quiesce_local_t *local = NULL;
 
-        if (priv && priv->pass_through) {
-                local = mem_get0 (priv->local_pool);
-                local->fd = fd_ref (fd);
-                frame->local = local;
-
-                STACK_WIND (frame,
-                            quiesce_flush_cbk,
-                            FIRST_CHILD(this),
-                            FIRST_CHILD(this)->fops->flush,
-                            fd, xdata);
-                return 0;
-        }
+    priv = this->private;
 
-        stub = fop_flush_stub (frame, default_flush_resume, fd, xdata);
-        if (!stub) {
-                STACK_UNWIND_STRICT (flush, frame, -1, ENOMEM, NULL);
-                return 0;
-        }
+    if (priv && priv->pass_through) {
+        local = mem_get0(priv->local_pool);
+        local->fd = fd_ref(fd);
+        frame->local = local;
 
-        gf_quiesce_enqueue (this, stub);
+        STACK_WIND(frame, quiesce_flush_cbk, FIRST_CHILD(this),
+                   FIRST_CHILD(this)->fops->flush, fd, xdata);
+        return 0;
+    }
 
+    stub = fop_flush_stub(frame, default_flush_resume, fd, xdata);
+    if (!stub) {
+        STACK_UNWIND_STRICT(flush, frame, -1, ENOMEM, NULL);
         return 0;
+    }
+
+    gf_quiesce_enqueue(this, stub);
+
+    return 0;
 }
 
 int32_t
-quiesce_writev (call_frame_t *frame,
-		xlator_t *this,
-		fd_t *fd,
-		struct iovec *vector,
-		int32_t count,
-		off_t off, uint32_t flags,
-                struct iobref *iobref, dict_t *xdata)
+quiesce_writev(call_frame_t *frame, xlator_t *this, fd_t *fd,
+               struct iovec *vector, int32_t count, off_t off, uint32_t flags,
+               struct iobref *iobref, dict_t *xdata)
 {
-	quiesce_priv_t *priv = NULL;
-        call_stub_t    *stub = NULL;
+    quiesce_priv_t *priv = NULL;
+    call_stub_t *stub = NULL;
 
-        priv = this->private;
+    priv = this->private;
 
-        if (priv && priv->pass_through) {
-                STACK_WIND (frame,
-                            default_writev_cbk,
-                            FIRST_CHILD(this),
-                            FIRST_CHILD(this)->fops->writev,
-                            fd,
-                            vector,
-                            count,
-                            off, flags,
-                            iobref, xdata);
-                return 0;
-        }
+    if (priv && priv->pass_through) {
+        STACK_WIND(frame, default_writev_cbk, FIRST_CHILD(this),
+                   FIRST_CHILD(this)->fops->writev, fd, vector, count, off,
+                   flags, iobref, xdata);
+        return 0;
+    }
 
-        stub = fop_writev_stub (frame, default_writev_resume,
-                                fd, vector, count, off, flags, iobref, xdata);
-        if (!stub) {
-                STACK_UNWIND_STRICT (writev, frame, -1, ENOMEM, NULL, NULL, NULL);
-                return 0;
-        }
+    stub = fop_writev_stub(frame, default_writev_resume, fd, vector, count, off,
+                           flags, iobref, xdata);
+    if (!stub) {
+        STACK_UNWIND_STRICT(writev, frame, -1, ENOMEM, NULL, NULL, NULL);
+        return 0;
+    }
 
-        gf_quiesce_enqueue (this, stub);
+    gf_quiesce_enqueue(this, stub);
 
-        return 0;
+    return 0;
 }
 
 int32_t
-quiesce_readv (call_frame_t *frame,
-	       xlator_t *this,
-	       fd_t *fd,
-	       size_t size,
-	       off_t offset, uint32_t flags, dict_t *xdata)
+quiesce_readv(call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
+              off_t offset, uint32_t flags, dict_t *xdata)
 {
-	quiesce_priv_t *priv = NULL;
-        call_stub_t    *stub = NULL;
-        quiesce_local_t *local = NULL;
-
-        priv = this->private;
+    quiesce_priv_t *priv = NULL;
+    call_stub_t *stub = NULL;
+    quiesce_local_t *local = NULL;
 
-        if (priv && priv->pass_through) {
-                local = mem_get0 (priv->local_pool);
-                local->fd = fd_ref (fd);
-                local->size = size;
-                local->offset = offset;
-                local->io_flag = flags;
-                frame->local = local;
-
-                STACK_WIND (frame,
-                            quiesce_readv_cbk,
-                            FIRST_CHILD(this),
-                            FIRST_CHILD(this)->fops->readv,
-                            fd,
-                            size,
-                            offset, flags, xdata);
-                return 0;
-        }
+    priv = this->private;
 
-        stub = fop_readv_stub (frame, default_readv_resume, fd, size, offset,
-                               flags, xdata);
-        if (!stub) {
-                STACK_UNWIND_STRICT (readv, frame, -1, ENOMEM,
-                                     NULL, 0, NULL, NULL, NULL);
-                return 0;
-        }
+    if (priv && priv->pass_through) {
+        local = mem_get0(priv->local_pool);
+        local->fd = fd_ref(fd);
+        local->size = size;
+        local->offset = offset;
+        local->io_flag = flags;
+        frame->local = local;
 
-        gf_quiesce_enqueue (this, stub);
+        STACK_WIND(frame, quiesce_readv_cbk, FIRST_CHILD(this),
+                   FIRST_CHILD(this)->fops->readv, fd, size, offset, flags,
+                   xdata);
+        return 0;
+    }
 
+    stub = fop_readv_stub(frame, default_readv_resume, fd, size, offset, flags,
+                          xdata);
+    if (!stub) {
+        STACK_UNWIND_STRICT(readv, frame, -1, ENOMEM, NULL, 0, NULL, NULL,
+                            NULL);
         return 0;
-}
+    }
+
+    gf_quiesce_enqueue(this, stub);
 
+    return 0;
+}
 
 int32_t
-quiesce_open (call_frame_t *frame,
-	      xlator_t *this,
-	      loc_t *loc,
-	      int32_t flags, fd_t *fd,
-              dict_t *xdata)
+quiesce_open(call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t flags,
+             fd_t *fd, dict_t *xdata)
 {
-	quiesce_priv_t *priv = NULL;
-        call_stub_t    *stub = NULL;
-        quiesce_local_t *local = NULL;
+    quiesce_priv_t *priv = NULL;
+    call_stub_t *stub = NULL;
+    quiesce_local_t *local = NULL;
 
-        priv = this->private;
+    priv = this->private;
 
-        if (priv && priv->pass_through) {
-                local = mem_get0 (priv->local_pool);
-                loc_dup (loc, &local->loc);
-                local->fd = fd_ref (fd);
-
-                /* Don't send O_APPEND below, as write() re-transmittions can
-                   fail with O_APPEND */
-                local->flag = (flags & ~O_APPEND);
-                frame->local = local;
-
-                STACK_WIND (frame,
-                            quiesce_open_cbk,
-                            FIRST_CHILD(this),
-                            FIRST_CHILD(this)->fops->open,
-                            loc, (flags & ~O_APPEND), fd, xdata);
-                return 0;
-        }
+    if (priv && priv->pass_through) {
+        local = mem_get0(priv->local_pool);
+        loc_dup(loc, &local->loc);
+        local->fd = fd_ref(fd);
 
-        stub = fop_open_stub (frame, default_open_resume, loc,
-                              (flags & ~O_APPEND), fd, xdata);
-        if (!stub) {
-                STACK_UNWIND_STRICT (open, frame, -1, ENOMEM, NULL, NULL);
-                return 0;
-        }
+        /* Don't send O_APPEND below, as write() re-transmittions can
+           fail with O_APPEND */
+        local->flag = (flags & ~O_APPEND);
+        frame->local = local;
 
-        gf_quiesce_enqueue (this, stub);
+        STACK_WIND(frame, quiesce_open_cbk, FIRST_CHILD(this),
+                   FIRST_CHILD(this)->fops->open, loc, (flags & ~O_APPEND), fd,
+                   xdata);
+        return 0;
+    }
 
+    stub = fop_open_stub(frame, default_open_resume, loc, (flags & ~O_APPEND),
+                         fd, xdata);
+    if (!stub) {
+        STACK_UNWIND_STRICT(open, frame, -1, ENOMEM, NULL, NULL);
         return 0;
+    }
+
+    gf_quiesce_enqueue(this, stub);
+
+    return 0;
 }
 
 int32_t
-quiesce_getxattr (call_frame_t *frame,
-		  xlator_t *this,
-		  loc_t *loc,
-		  const char *name, dict_t *xdata)
+quiesce_getxattr(call_frame_t *frame, xlator_t *this, loc_t *loc,
+                 const char *name, dict_t *xdata)
 {
-	quiesce_priv_t *priv = NULL;
-        call_stub_t    *stub = NULL;
-        quiesce_local_t *local = NULL;
+    quiesce_priv_t *priv = NULL;
+    call_stub_t *stub = NULL;
+    quiesce_local_t *local = NULL;
 
-        priv = this->private;
+    priv = this->private;
 
-        if (priv && priv->pass_through) {
-                local = mem_get0 (priv->local_pool);
-                loc_dup (loc, &local->loc);
-                if (name)
-                        local->name = gf_strdup (name);
-
-                frame->local = local;
-
-                STACK_WIND (frame,
-                            quiesce_getxattr_cbk,
-                            FIRST_CHILD(this),
-                            FIRST_CHILD(this)->fops->getxattr,
-                            loc,
-                            name, xdata);
-                return 0;
-        }
+    if (priv && priv->pass_through) {
+        local = mem_get0(priv->local_pool);
+        loc_dup(loc, &local->loc);
+        if (name)
+            local->name = gf_strdup(name);
 
-        stub = fop_getxattr_stub (frame, default_getxattr_resume, loc, name, xdata);
-        if (!stub) {
-                STACK_UNWIND_STRICT (getxattr, frame, -1, ENOMEM, NULL, NULL);
-                return 0;
-        }
+        frame->local = local;
 
-        gf_quiesce_enqueue (this, stub);
+        STACK_WIND(frame, quiesce_getxattr_cbk, FIRST_CHILD(this),
+                   FIRST_CHILD(this)->fops->getxattr, loc, name, xdata);
+        return 0;
+    }
 
+    stub = fop_getxattr_stub(frame, default_getxattr_resume, loc, name, xdata);
+    if (!stub) {
+        STACK_UNWIND_STRICT(getxattr, frame, -1, ENOMEM, NULL, NULL);
         return 0;
-}
+    }
 
+    gf_quiesce_enqueue(this, stub);
+
+    return 0;
+}
 
 int32_t
-quiesce_xattrop (call_frame_t *frame,
-		 xlator_t *this,
-		 loc_t *loc,
-		 gf_xattrop_flags_t flags,
-		 dict_t *dict, dict_t *xdata)
+quiesce_xattrop(call_frame_t *frame, xlator_t *this, loc_t *loc,
+                gf_xattrop_flags_t flags, dict_t *dict, dict_t *xdata)
 {
-	quiesce_priv_t *priv = NULL;
-        call_stub_t    *stub = NULL;
+    quiesce_priv_t *priv = NULL;
+    call_stub_t *stub = NULL;
 
-        priv = this->private;
+    priv = this->private;
 
-        if (priv && priv->pass_through) {
-                STACK_WIND (frame,
-                            default_xattrop_cbk,
-                            FIRST_CHILD(this),
-                            FIRST_CHILD(this)->fops->xattrop,
-                            loc,
-                            flags,
-                            dict, xdata);
-                return 0;
-        }
+    if (priv && priv->pass_through) {
+        STACK_WIND(frame, default_xattrop_cbk, FIRST_CHILD(this),
+                   FIRST_CHILD(this)->fops->xattrop, loc, flags, dict, xdata);
+        return 0;
+    }
 
-        stub = fop_xattrop_stub (frame, default_xattrop_resume,
-                                 loc, flags, dict, xdata);
-        if (!stub) {
-                STACK_UNWIND_STRICT (xattrop, frame, -1, ENOMEM, NULL, NULL);
-                return 0;
-        }
+    stub = fop_xattrop_stub(frame, default_xattrop_resume, loc, flags, dict,
+                            xdata);
+    if (!stub) {
+        STACK_UNWIND_STRICT(xattrop, frame, -1, ENOMEM, NULL, NULL);
+        return 0;
+    }
 
-        gf_quiesce_enqueue (this, stub);
+    gf_quiesce_enqueue(this, stub);
 
-        return 0;
+    return 0;
 }
 
 int32_t
-quiesce_fxattrop (call_frame_t *frame,
-		  xlator_t *this,
-		  fd_t *fd,
-		  gf_xattrop_flags_t flags,
-		  dict_t *dict, dict_t *xdata)
+quiesce_fxattrop(call_frame_t *frame, xlator_t *this, fd_t *fd,
+                 gf_xattrop_flags_t flags, dict_t *dict, dict_t *xdata)
 {
-	quiesce_priv_t *priv = NULL;
-        call_stub_t    *stub = NULL;
+    quiesce_priv_t *priv = NULL;
+    call_stub_t *stub = NULL;
 
-        priv = this->private;
+    priv = this->private;
 
-        if (priv && priv->pass_through) {
-                STACK_WIND (frame,
-                            default_fxattrop_cbk,
-                            FIRST_CHILD(this),
-                            FIRST_CHILD(this)->fops->fxattrop,
-                            fd,
-                            flags,
-                            dict, xdata);
-                return 0;
-        }
+    if (priv && priv->pass_through) {
+        STACK_WIND(frame, default_fxattrop_cbk, FIRST_CHILD(this),
+                   FIRST_CHILD(this)->fops->fxattrop, fd, flags, dict, xdata);
+        return 0;
+    }
 
-        stub = fop_fxattrop_stub (frame, default_fxattrop_resume,
-                                  fd, flags, dict, xdata);
-        if (!stub) {
-                STACK_UNWIND_STRICT (fxattrop, frame, -1, ENOMEM, NULL, NULL);
-                return 0;
-        }
+    stub = fop_fxattrop_stub(frame, default_fxattrop_resume, fd, flags, dict,
+                             xdata);
+    if (!stub) {
+        STACK_UNWIND_STRICT(fxattrop, frame, -1, ENOMEM, NULL, NULL);
+        return 0;
+    }
+
+    gf_quiesce_enqueue(this, stub);
+
+    return 0;
+}
+
+int32_t
+quiesce_lk(call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t cmd,
+           struct gf_flock *lock, dict_t *xdata)
+{
+    quiesce_priv_t *priv = NULL;
+    call_stub_t *stub = NULL;
 
-        gf_quiesce_enqueue (this, stub);
+    priv = this->private;
 
+    if (priv && priv->pass_through) {
+        STACK_WIND(frame, default_lk_cbk, FIRST_CHILD(this),
+                   FIRST_CHILD(this)->fops->lk, fd, cmd, lock, xdata);
         return 0;
+    }
+
+    stub = fop_lk_stub(frame, default_lk_resume, fd, cmd, lock, xdata);
+    if (!stub) {
+        STACK_UNWIND_STRICT(lk, frame, -1, ENOMEM, NULL, NULL);
+        return 0;
+    }
+
+    gf_quiesce_enqueue(this, stub);
+
+    return 0;
 }
 
 int32_t
-quiesce_lk (call_frame_t *frame,
-	    xlator_t *this,
-	    fd_t *fd,
-	    int32_t cmd,
-	    struct gf_flock *lock, dict_t *xdata)
+quiesce_inodelk(call_frame_t *frame, xlator_t *this, const char *volume,
+                loc_t *loc, int32_t cmd, struct gf_flock *lock, dict_t *xdata)
 {
-	quiesce_priv_t *priv = NULL;
-        call_stub_t    *stub = NULL;
+    quiesce_priv_t *priv = NULL;
+    call_stub_t *stub = NULL;
 
-        priv = this->private;
+    priv = this->private;
 
-        if (priv && priv->pass_through) {
-                STACK_WIND (frame,
-                            default_lk_cbk,
-                            FIRST_CHILD(this),
-                            FIRST_CHILD(this)->fops->lk,
-                            fd,
-                            cmd,
+    if (priv && priv->pass_through) {
+        STACK_WIND(frame, default_inodelk_cbk, FIRST_CHILD(this),
+                   FIRST_CHILD(this)->fops->inodelk, volume, loc, cmd, lock,
+                   xdata);
+        return 0;
+    }
+
+    stub = fop_inodelk_stub(frame, default_inodelk_resume, volume, loc, cmd,
                             lock, xdata);
-                return 0;
-        }
+    if (!stub) {
+        STACK_UNWIND_STRICT(inodelk, frame, -1, ENOMEM, NULL);
+        return 0;
+    }
 
-        stub = fop_lk_stub (frame, default_lk_resume, fd, cmd, lock, xdata);
-        if (!stub) {
-                STACK_UNWIND_STRICT (lk, frame, -1, ENOMEM, NULL, NULL);
-                return 0;
-        }
+    gf_quiesce_enqueue(this, stub);
 
-        gf_quiesce_enqueue (this, stub);
+    return 0;
+}
+
+int32_t
+quiesce_finodelk(call_frame_t *frame, xlator_t *this, const char *volume,
+                 fd_t *fd, int32_t cmd, struct gf_flock *lock, dict_t *xdata)
+{
+    quiesce_priv_t *priv = NULL;
+    call_stub_t *stub = NULL;
+
+    priv = this->private;
 
+    if (priv && priv->pass_through) {
+        STACK_WIND(frame, default_finodelk_cbk, FIRST_CHILD(this),
+                   FIRST_CHILD(this)->fops->finodelk, volume, fd, cmd, lock,
+                   xdata);
         return 0;
-}
+    }
 
+    stub = fop_finodelk_stub(frame, default_finodelk_resume, volume, fd, cmd,
+                             lock, xdata);
+    if (!stub) {
+        STACK_UNWIND_STRICT(finodelk, frame, -1, ENOMEM, NULL);
+        return 0;
+    }
+
+    gf_quiesce_enqueue(this, stub);
+
+    return 0;
+}
 
 int32_t
-quiesce_inodelk (call_frame_t *frame, xlator_t *this,
-		 const char *volume, loc_t *loc, int32_t cmd,
-                 struct gf_flock *lock, dict_t *xdata)
+quiesce_entrylk(call_frame_t *frame, xlator_t *this, const char *volume,
+                loc_t *loc, const char *basename, entrylk_cmd cmd,
+                entrylk_type type, dict_t *xdata)
 {
-	quiesce_priv_t *priv = NULL;
-        call_stub_t    *stub = NULL;
+    quiesce_priv_t *priv = NULL;
+    call_stub_t *stub = NULL;
 
-        priv = this->private;
+    priv = this->private;
 
-        if (priv && priv->pass_through) {
-                STACK_WIND (frame,
-                            default_inodelk_cbk,
-                            FIRST_CHILD(this),
-                            FIRST_CHILD(this)->fops->inodelk,
-                            volume, loc, cmd, lock, xdata);
-                return 0;
-        }
+    if (priv && priv->pass_through) {
+        STACK_WIND(frame, default_entrylk_cbk, FIRST_CHILD(this),
+                   FIRST_CHILD(this)->fops->entrylk, volume, loc, basename, cmd,
+                   type, xdata);
+        return 0;
+    }
 
-        stub = fop_inodelk_stub (frame, default_inodelk_resume,
-                                 volume, loc, cmd, lock, xdata);
-        if (!stub) {
-                STACK_UNWIND_STRICT (inodelk, frame, -1, ENOMEM, NULL);
-                return 0;
-        }
+    stub = fop_entrylk_stub(frame, default_entrylk_resume, volume, loc,
+                            basename, cmd, type, xdata);
+    if (!stub) {
+        STACK_UNWIND_STRICT(entrylk, frame, -1, ENOMEM, NULL);
+        return 0;
+    }
 
-        gf_quiesce_enqueue (this, stub);
+    gf_quiesce_enqueue(this, stub);
 
-        return 0;
+    return 0;
 }
 
 int32_t
-quiesce_finodelk (call_frame_t *frame, xlator_t *this,
-		  const char *volume, fd_t *fd, int32_t cmd, struct gf_flock *lock, dict_t *xdata)
+quiesce_fentrylk(call_frame_t *frame, xlator_t *this, const char *volume,
+                 fd_t *fd, const char *basename, entrylk_cmd cmd,
+                 entrylk_type type, dict_t *xdata)
 {
-	quiesce_priv_t *priv = NULL;
-        call_stub_t    *stub = NULL;
+    quiesce_priv_t *priv = NULL;
+    call_stub_t *stub = NULL;
 
-        priv = this->private;
+    priv = this->private;
 
-        if (priv && priv->pass_through) {
-                STACK_WIND (frame,
-                            default_finodelk_cbk,
-                            FIRST_CHILD(this),
-                            FIRST_CHILD(this)->fops->finodelk,
-                            volume, fd, cmd, lock, xdata);
-                return 0;
-        }
+    if (priv && priv->pass_through) {
+        STACK_WIND(frame, default_fentrylk_cbk, FIRST_CHILD(this),
+                   FIRST_CHILD(this)->fops->fentrylk, volume, fd, basename, cmd,
+                   type, xdata);
+        return 0;
+    }
 
-        stub = fop_finodelk_stub (frame, default_finodelk_resume,
-                                  volume, fd, cmd, lock, xdata);
-        if (!stub) {
-                STACK_UNWIND_STRICT (finodelk, frame, -1, ENOMEM, NULL);
-                return 0;
-        }
+    stub = fop_fentrylk_stub(frame, default_fentrylk_resume, volume, fd,
+                             basename, cmd, type, xdata);
+    if (!stub) {
+        STACK_UNWIND_STRICT(fentrylk, frame, -1, ENOMEM, NULL);
+        return 0;
+    }
 
-        gf_quiesce_enqueue (this, stub);
+    gf_quiesce_enqueue(this, stub);
 
-        return 0;
+    return 0;
 }
 
 int32_t
-quiesce_entrylk (call_frame_t *frame, xlator_t *this,
-		 const char *volume, loc_t *loc, const char *basename,
-		 entrylk_cmd cmd, entrylk_type type, dict_t *xdata)
+quiesce_rchecksum(call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
+                  int32_t len, dict_t *xdata)
 {
-	quiesce_priv_t *priv = NULL;
-        call_stub_t    *stub = NULL;
-
-        priv = this->private;
+    quiesce_priv_t *priv = NULL;
+    call_stub_t *stub = NULL;
+    quiesce_local_t *local = NULL;
 
-        if (priv && priv->pass_through) {
-                STACK_WIND (frame, default_entrylk_cbk,
-                            FIRST_CHILD(this),
-                            FIRST_CHILD(this)->fops->entrylk,
-                            volume, loc, basename, cmd, type, xdata);
-                return 0;
-        }
+    priv = this->private;
 
-        stub = fop_entrylk_stub (frame, default_entrylk_resume,
-                                 volume, loc, basename, cmd, type, xdata);
-        if (!stub) {
-                STACK_UNWIND_STRICT (entrylk, frame, -1, ENOMEM, NULL);
-                return 0;
-        }
+    if (priv && priv->pass_through) {
+        local = mem_get0(priv->local_pool);
+        local->fd = fd_ref(fd);
+        local->offset = offset;
+        local->flag = len;
+        frame->local = local;
 
-        gf_quiesce_enqueue (this, stub);
+        STACK_WIND(frame, quiesce_rchecksum_cbk, FIRST_CHILD(this),
+                   FIRST_CHILD(this)->fops->rchecksum, fd, offset, len, xdata);
+        return 0;
+    }
 
+    stub = fop_rchecksum_stub(frame, default_rchecksum_resume, fd, offset, len,
+                              xdata);
+    if (!stub) {
+        STACK_UNWIND_STRICT(rchecksum, frame, -1, ENOMEM, 0, NULL, NULL);
         return 0;
+    }
+
+    gf_quiesce_enqueue(this, stub);
+
+    return 0;
 }
 
 int32_t
-quiesce_fentrylk (call_frame_t *frame, xlator_t *this,
-		  const char *volume, fd_t *fd, const char *basename,
-		  entrylk_cmd cmd, entrylk_type type, dict_t *xdata)
+quiesce_readdir(call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
+                off_t off, dict_t *xdata)
 {
-	quiesce_priv_t *priv = NULL;
-        call_stub_t    *stub = NULL;
+    quiesce_priv_t *priv = NULL;
+    call_stub_t *stub = NULL;
+    quiesce_local_t *local = NULL;
 
-        priv = this->private;
+    priv = this->private;
 
-        if (priv && priv->pass_through) {
-                STACK_WIND (frame, default_fentrylk_cbk,
-                            FIRST_CHILD(this),
-                            FIRST_CHILD(this)->fops->fentrylk,
-                            volume, fd, basename, cmd, type, xdata);
-                return 0;
-        }
+    if (priv && priv->pass_through) {
+        local = mem_get0(priv->local_pool);
+        local->fd = fd_ref(fd);
+        local->size = size;
+        local->offset = off;
+        frame->local = local;
 
-        stub = fop_fentrylk_stub (frame, default_fentrylk_resume,
-                                  volume, fd, basename, cmd, type, xdata);
-        if (!stub) {
-                STACK_UNWIND_STRICT (fentrylk, frame, -1, ENOMEM, NULL);
-                return 0;
-        }
-
-        gf_quiesce_enqueue (this, stub);
+        STACK_WIND(frame, quiesce_readdir_cbk, FIRST_CHILD(this),
+                   FIRST_CHILD(this)->fops->readdir, fd, size, off, xdata);
+        return 0;
+    }
 
+    stub = fop_readdir_stub(frame, default_readdir_resume, fd, size, off,
+                            xdata);
+    if (!stub) {
+        STACK_UNWIND_STRICT(readdir, frame, -1, ENOMEM, NULL, NULL);
         return 0;
+    }
+
+    gf_quiesce_enqueue(this, stub);
+
+    return 0;
 }
 
 int32_t
-quiesce_rchecksum (call_frame_t *frame,
-                   xlator_t *this,
-                   fd_t *fd, off_t offset,
-                   int32_t len, dict_t *xdata)
+quiesce_readdirp(call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
+                 off_t off, dict_t *dict)
 {
-	quiesce_priv_t *priv = NULL;
-        call_stub_t    *stub = NULL;
-        quiesce_local_t *local = NULL;
-
-        priv = this->private;
+    quiesce_priv_t *priv = NULL;
+    call_stub_t *stub = NULL;
+    quiesce_local_t *local = NULL;
 
-        if (priv && priv->pass_through) {
-                local = mem_get0 (priv->local_pool);
-                local->fd = fd_ref (fd);
-                local->offset = offset;
-                local->flag = len;
-                frame->local = local;
-
-                STACK_WIND (frame,
-                            quiesce_rchecksum_cbk,
-                            FIRST_CHILD(this),
-                            FIRST_CHILD(this)->fops->rchecksum,
-                            fd, offset, len, xdata);
-                return 0;
-        }
+    priv = this->private;
 
-        stub = fop_rchecksum_stub (frame, default_rchecksum_resume,
-                                   fd, offset, len, xdata);
-        if (!stub) {
-                STACK_UNWIND_STRICT (rchecksum, frame, -1, ENOMEM, 0, NULL, NULL);
-                return 0;
-        }
+    if (priv && priv->pass_through) {
+        local = mem_get0(priv->local_pool);
+        local->fd = fd_ref(fd);
+        local->size = size;
+        local->offset = off;
+        local->dict = dict_ref(dict);
+        frame->local = local;
 
-        gf_quiesce_enqueue (this, stub);
+        STACK_WIND(frame, quiesce_readdirp_cbk, FIRST_CHILD(this),
+                   FIRST_CHILD(this)->fops->readdirp, fd, size, off, dict);
+        return 0;
+    }
 
+    stub = fop_readdirp_stub(frame, default_readdirp_resume, fd, size, off,
+                             dict);
+    if (!stub) {
+        STACK_UNWIND_STRICT(readdirp, frame, -1, ENOMEM, NULL, NULL);
         return 0;
-}
+    }
 
+    gf_quiesce_enqueue(this, stub);
+
+    return 0;
+}
 
 int32_t
-quiesce_readdir (call_frame_t *frame,
-		 xlator_t *this,
-		 fd_t *fd,
-		 size_t size,
-		 off_t off, dict_t *xdata)
+quiesce_setattr(call_frame_t *frame, xlator_t *this, loc_t *loc,
+                struct iatt *stbuf, int32_t valid, dict_t *xdata)
 {
-	quiesce_priv_t *priv = NULL;
-        call_stub_t    *stub = NULL;
-        quiesce_local_t *local = NULL;
+    quiesce_priv_t *priv = NULL;
+    call_stub_t *stub = NULL;
 
-        priv = this->private;
+    priv = this->private;
 
-        if (priv && priv->pass_through) {
-                local = mem_get0 (priv->local_pool);
-                local->fd = fd_ref (fd);
-                local->size = size;
-                local->offset = off;
-                frame->local = local;
-
-                STACK_WIND (frame,
-                            quiesce_readdir_cbk,
-                            FIRST_CHILD(this),
-                            FIRST_CHILD(this)->fops->readdir,
-                            fd, size, off, xdata);
-                return 0;
-        }
+    if (priv && priv->pass_through) {
+        STACK_WIND(frame, default_setattr_cbk, FIRST_CHILD(this),
+                   FIRST_CHILD(this)->fops->setattr, loc, stbuf, valid, xdata);
+        return 0;
+    }
 
-        stub = fop_readdir_stub (frame, default_readdir_resume, fd, size, off, xdata);
-        if (!stub) {
-                STACK_UNWIND_STRICT (readdir, frame, -1, ENOMEM, NULL, NULL);
-                return 0;
-        }
+    stub = fop_setattr_stub(frame, default_setattr_resume, loc, stbuf, valid,
+                            xdata);
+    if (!stub) {
+        STACK_UNWIND_STRICT(setattr, frame, -1, ENOMEM, NULL, NULL, NULL);
+        return 0;
+    }
 
-        gf_quiesce_enqueue (this, stub);
+    gf_quiesce_enqueue(this, stub);
 
-        return 0;
+    return 0;
 }
 
-
 int32_t
-quiesce_readdirp (call_frame_t *frame,
-		  xlator_t *this,
-		  fd_t *fd,
-		  size_t size,
-		  off_t off, dict_t *dict)
+quiesce_stat(call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xdata)
 {
-	quiesce_priv_t *priv = NULL;
-        call_stub_t    *stub = NULL;
-        quiesce_local_t *local = NULL;
-
-        priv = this->private;
+    quiesce_priv_t *priv = NULL;
+    call_stub_t *stub = NULL;
+    quiesce_local_t *local = NULL;
 
-        if (priv && priv->pass_through) {
-                local = mem_get0 (priv->local_pool);
-                local->fd = fd_ref (fd);
-                local->size = size;
-                local->offset = off;
-                local->dict = dict_ref (dict);
-                frame->local = local;
-
-                STACK_WIND (frame,
-                            quiesce_readdirp_cbk,
-                            FIRST_CHILD(this),
-                            FIRST_CHILD(this)->fops->readdirp,
-                            fd, size, off, dict);
-                return 0;
-        }
+    priv = this->private;
 
-        stub = fop_readdirp_stub (frame, default_readdirp_resume, fd, size,
-                                  off, dict);
-        if (!stub) {
-                STACK_UNWIND_STRICT (readdirp, frame, -1, ENOMEM, NULL, NULL);
-                return 0;
-        }
+    if (priv && priv->pass_through) {
+        local = mem_get0(priv->local_pool);
+        loc_dup(loc, &local->loc);
+        frame->local = local;
 
-        gf_quiesce_enqueue (this, stub);
+        STACK_WIND(frame, quiesce_stat_cbk, FIRST_CHILD(this),
+                   FIRST_CHILD(this)->fops->stat, loc, xdata);
+        return 0;
+    }
 
+    stub = fop_stat_stub(frame, default_stat_resume, loc, xdata);
+    if (!stub) {
+        STACK_UNWIND_STRICT(stat, frame, -1, ENOMEM, NULL, NULL);
         return 0;
+    }
+
+    gf_quiesce_enqueue(this, stub);
+
+    return 0;
 }
 
 int32_t
-quiesce_setattr (call_frame_t *frame,
-                 xlator_t *this,
-                 loc_t *loc,
-                 struct iatt *stbuf,
-                 int32_t valid, dict_t *xdata)
+quiesce_lookup(call_frame_t *frame, xlator_t *this, loc_t *loc,
+               dict_t *xattr_req)
 {
-	quiesce_priv_t *priv = NULL;
-        call_stub_t    *stub = NULL;
+    quiesce_priv_t *priv = NULL;
+    call_stub_t *stub = NULL;
+    quiesce_local_t *local = NULL;
 
-        priv = this->private;
-
-        if (priv && priv->pass_through) {
-                STACK_WIND (frame,
-                            default_setattr_cbk,
-                            FIRST_CHILD (this),
-                            FIRST_CHILD (this)->fops->setattr,
-                            loc, stbuf, valid, xdata);
-                return 0;
-        }
+    priv = this->private;
 
-        stub = fop_setattr_stub (frame, default_setattr_resume,
-                                   loc, stbuf, valid, xdata);
-        if (!stub) {
-                STACK_UNWIND_STRICT (setattr, frame, -1, ENOMEM, NULL, NULL, NULL);
-                return 0;
-        }
+    if (priv && priv->pass_through) {
+        local = mem_get0(priv->local_pool);
+        loc_dup(loc, &local->loc);
+        local->dict = dict_ref(xattr_req);
+        frame->local = local;
 
-        gf_quiesce_enqueue (this, stub);
+        STACK_WIND(frame, quiesce_lookup_cbk, FIRST_CHILD(this),
+                   FIRST_CHILD(this)->fops->lookup, loc, xattr_req);
+        return 0;
+    }
 
+    stub = fop_lookup_stub(frame, default_lookup_resume, loc, xattr_req);
+    if (!stub) {
+        STACK_UNWIND_STRICT(lookup, frame, -1, ENOMEM, NULL, NULL, NULL, NULL);
         return 0;
-}
+    }
+
+    gf_quiesce_enqueue(this, stub);
 
+    return 0;
+}
 
 int32_t
-quiesce_stat (call_frame_t *frame,
-	      xlator_t *this,
-	      loc_t *loc, dict_t *xdata)
+quiesce_fsetattr(call_frame_t *frame, xlator_t *this, fd_t *fd,
+                 struct iatt *stbuf, int32_t valid, dict_t *xdata)
 {
-	quiesce_priv_t *priv = NULL;
-        call_stub_t    *stub = NULL;
-        quiesce_local_t *local = NULL;
+    quiesce_priv_t *priv = NULL;
+    call_stub_t *stub = NULL;
 
-        priv = this->private;
+    priv = this->private;
 
-        if (priv && priv->pass_through) {
-                local = mem_get0 (priv->local_pool);
-                loc_dup (loc, &local->loc);
-                frame->local = local;
-
-                STACK_WIND (frame,
-                            quiesce_stat_cbk,
-                            FIRST_CHILD(this),
-                            FIRST_CHILD(this)->fops->stat,
-                            loc, xdata);
-                return 0;
-        }
+    if (priv && priv->pass_through) {
+        STACK_WIND(frame, default_fsetattr_cbk, FIRST_CHILD(this),
+                   FIRST_CHILD(this)->fops->fsetattr, fd, stbuf, valid, xdata);
+        return 0;
+    }
 
-        stub = fop_stat_stub (frame, default_stat_resume, loc, xdata);
-        if (!stub) {
-                STACK_UNWIND_STRICT (stat, frame, -1, ENOMEM, NULL, NULL);
-                return 0;
-        }
+    stub = fop_fsetattr_stub(frame, default_fsetattr_resume, fd, stbuf, valid,
+                             xdata);
+    if (!stub) {
+        STACK_UNWIND_STRICT(fsetattr, frame, -1, ENOMEM, NULL, NULL, NULL);
+        return 0;
+    }
 
-        gf_quiesce_enqueue (this, stub);
+    gf_quiesce_enqueue(this, stub);
 
-        return 0;
+    return 0;
 }
 
 int32_t
-quiesce_lookup (call_frame_t *frame,
-		xlator_t *this,
-		loc_t *loc,
-		dict_t *xattr_req)
+quiesce_fallocate(call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t mode,
+                  off_t offset, size_t len, dict_t *xdata)
 {
-	quiesce_priv_t *priv = NULL;
-        call_stub_t    *stub = NULL;
-        quiesce_local_t *local = NULL;
+    quiesce_priv_t *priv = NULL;
+    call_stub_t *stub = NULL;
 
-        priv = this->private;
+    priv = this->private;
 
-        if (priv && priv->pass_through) {
-                local = mem_get0 (priv->local_pool);
-                loc_dup (loc, &local->loc);
-                local->dict = dict_ref (xattr_req);
-                frame->local = local;
-
-                STACK_WIND (frame,
-                            quiesce_lookup_cbk,
-                            FIRST_CHILD(this),
-                            FIRST_CHILD(this)->fops->lookup,
-                            loc, xattr_req);
-                return 0;
-        }
+    if (priv && priv->pass_through) {
+        STACK_WIND(frame, default_fallocate_cbk, FIRST_CHILD(this),
+                   FIRST_CHILD(this)->fops->fallocate, fd, mode, offset, len,
+                   xdata);
+        return 0;
+    }
+
+    stub = fop_fallocate_stub(frame, default_fallocate_resume, fd, mode, offset,
+                              len, xdata);
+    if (!stub) {
+        STACK_UNWIND_STRICT(fallocate, frame, -1, ENOMEM, NULL, NULL, NULL);
+        return 0;
+    }
 
-        stub = fop_lookup_stub (frame, default_lookup_resume, loc, xattr_req);
+    gf_quiesce_enqueue(this, stub);
+
+    return 0;
+}
+
+int
+quiesce_seek_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                 int32_t op_ret, int32_t op_errno, off_t offset, dict_t *xdata)
+{
+    call_stub_t *stub = NULL;
+    quiesce_local_t *local = NULL;
+
+    local = frame->local;
+    frame->local = NULL;
+    if ((op_ret == -1) && (op_errno == ENOTCONN)) {
+        /* Re-transmit (by putting in the queue) */
+        stub = fop_seek_stub(frame, default_seek_resume, local->fd,
+                             local->offset, local->what, xdata);
         if (!stub) {
-                STACK_UNWIND_STRICT (lookup, frame, -1, ENOMEM,
-                                     NULL, NULL, NULL, NULL);
-                return 0;
+            STACK_UNWIND_STRICT(seek, frame, -1, ENOMEM, 0, NULL);
+            goto out;
         }
 
-        gf_quiesce_enqueue (this, stub);
+        gf_quiesce_enqueue(this, stub);
+        goto out;
+    }
 
-        return 0;
+    STACK_UNWIND_STRICT(seek, frame, op_ret, op_errno, offset, xdata);
+out:
+    gf_quiesce_local_wipe(this, local);
+
+    return 0;
 }
 
-int32_t
-quiesce_fsetattr (call_frame_t *frame,
-                  xlator_t *this,
-                  fd_t *fd,
-                  struct iatt *stbuf,
-                  int32_t valid, dict_t *xdata)
+int
+quiesce_seek(call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
+             gf_seek_what_t what, dict_t *xdata)
 {
-	quiesce_priv_t *priv = NULL;
-        call_stub_t    *stub = NULL;
+    quiesce_priv_t *priv = NULL;
+    call_stub_t *stub = NULL;
+    quiesce_local_t *local = NULL;
 
-        priv = this->private;
+    priv = this->private;
 
-        if (priv && priv->pass_through) {
-                STACK_WIND (frame,
-                            default_fsetattr_cbk,
-                            FIRST_CHILD (this),
-                            FIRST_CHILD (this)->fops->fsetattr,
-                            fd, stbuf, valid, xdata);
-                return 0;
-        }
+    if (priv && priv->pass_through) {
+        local = mem_get0(priv->local_pool);
+        local->fd = fd_ref(fd);
+        local->offset = offset;
+        local->what = what;
 
-        stub = fop_fsetattr_stub (frame, default_fsetattr_resume,
-                                  fd, stbuf, valid, xdata);
-        if (!stub) {
-                STACK_UNWIND_STRICT (fsetattr, frame, -1, ENOMEM, NULL, NULL, NULL);
-                return 0;
-        }
+        frame->local = local;
 
-        gf_quiesce_enqueue (this, stub);
+        STACK_WIND(frame, quiesce_seek_cbk, FIRST_CHILD(this),
+                   FIRST_CHILD(this)->fops->seek, fd, offset, what, xdata);
+        return 0;
+    }
 
+    stub = fop_seek_stub(frame, default_seek_resume, fd, offset, what, xdata);
+    if (!stub) {
+        STACK_UNWIND_STRICT(seek, frame, -1, ENOMEM, 0, NULL);
         return 0;
+    }
+
+    gf_quiesce_enqueue(this, stub);
+
+    return 0;
 }
 
 int32_t
-mem_acct_init (xlator_t *this)
+mem_acct_init(xlator_t *this)
 {
-        int     ret = -1;
+    int ret = -1;
 
-        ret = xlator_mem_acct_init (this, gf_quiesce_mt_end + 1);
+    ret = xlator_mem_acct_init(this, gf_quiesce_mt_end + 1);
 
-        return ret;
+    return ret;
 }
 
 int
-init (xlator_t *this)
+reconfigure(xlator_t *this, dict_t *options)
 {
-        int ret = -1;
-        quiesce_priv_t *priv = NULL;
+    int32_t ret = -1;
+    quiesce_priv_t *priv = NULL;
 
-	if (!this->children || this->children->next) {
-		gf_log (this->name, GF_LOG_ERROR,
-			"'quiesce' not configured with exactly one child");
-                goto out;
-	}
+    priv = this->private;
 
-	if (!this->parents) {
-		gf_log (this->name, GF_LOG_WARNING,
-			"dangling volume. check volfile ");
-	}
+    GF_OPTION_RECONF("timeout", priv->timeout, options, time, out);
+    GF_OPTION_RECONF("failover-hosts", priv->failover_hosts, options, str, out);
+    gf_quiesce_populate_failover_hosts(this, priv, priv->failover_hosts);
 
-        priv = GF_CALLOC (1, sizeof (*priv), gf_quiesce_mt_priv_t);
-        if (!priv)
-                goto out;
+    ret = 0;
+out:
+    return ret;
+}
+
+int
+init(xlator_t *this)
+{
+    int ret = -1;
+    quiesce_priv_t *priv = NULL;
 
-        priv->local_pool =  mem_pool_new (quiesce_local_t,
-                                          GF_FOPS_EXPECTED_IN_PARALLEL);
+    if (!this->children || this->children->next) {
+        gf_log(this->name, GF_LOG_ERROR,
+               "'quiesce' not configured with exactly one child");
+        goto out;
+    }
 
-        LOCK_INIT (&priv->lock);
-        priv->pass_through = _gf_false;
+    if (!this->parents) {
+        gf_log(this->name, GF_LOG_WARNING, "dangling volume. check volfile ");
+    }
 
-        INIT_LIST_HEAD (&priv->req);
+    priv = GF_CALLOC(1, sizeof(*priv), gf_quiesce_mt_priv_t);
+    if (!priv)
+        goto out;
 
-        this->private = priv;
-        ret = 0;
+    INIT_LIST_HEAD(&priv->failover_list);
+
+    GF_OPTION_INIT("timeout", priv->timeout, time, out);
+    GF_OPTION_INIT("failover-hosts", priv->failover_hosts, str, out);
+    gf_quiesce_populate_failover_hosts(this, priv, priv->failover_hosts);
+
+    priv->local_pool = mem_pool_new(quiesce_local_t,
+                                    GF_FOPS_EXPECTED_IN_PARALLEL);
+
+    LOCK_INIT(&priv->lock);
+    priv->pass_through = _gf_false;
+
+    INIT_LIST_HEAD(&priv->req);
+
+    this->private = priv;
+    ret = 0;
 out:
-        return ret;
+    return ret;
 }
 
 void
-fini (xlator_t *this)
+fini(xlator_t *this)
 {
-        quiesce_priv_t *priv = NULL;
+    quiesce_priv_t *priv = NULL;
 
-        priv = this->private;
-        if (!priv)
-                goto out;
-        this->private = NULL;
+    priv = this->private;
+    if (!priv)
+        goto out;
+    this->private = NULL;
 
-        mem_pool_destroy (priv->local_pool);
-        LOCK_DESTROY (&priv->lock);
-        GF_FREE (priv);
+    mem_pool_destroy(priv->local_pool);
+    priv->local_pool = NULL;
+    LOCK_DESTROY(&priv->lock);
+    GF_FREE(priv);
 out:
-        return;
+    return;
 }
 
 int
-notify (xlator_t *this, int event, void *data, ...)
-{
-        int             ret     = 0;
-        quiesce_priv_t *priv    = NULL;
-        struct timespec timeout = {0,};
-
-        priv = this->private;
-        if (!priv)
-                goto out;
-
-        switch (event) {
-        case GF_EVENT_CHILD_UP:
-        {
-                ret = pthread_create (&priv->thr, NULL, gf_quiesce_dequeue_start,
-                                      this);
-                if (ret) {
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "failed to create the quiesce-dequeue thread");
-                }
-
-                LOCK (&priv->lock);
-                {
-                        priv->pass_through = _gf_true;
-                }
-                UNLOCK (&priv->lock);
-                break;
+notify(xlator_t *this, int event, void *data, ...)
+{
+    int ret = 0;
+    quiesce_priv_t *priv = NULL;
+
+    priv = this->private;
+    if (!priv)
+        goto out;
+
+    switch (event) {
+        case GF_EVENT_CHILD_UP: {
+            ret = gf_thread_create(&priv->thr, NULL, gf_quiesce_dequeue_start,
+                                   this, "quiesce");
+            if (ret) {
+                gf_log(this->name, GF_LOG_ERROR,
+                       "failed to create the quiesce-dequeue thread");
+            }
+
+            LOCK(&priv->lock);
+            {
+                priv->pass_through = _gf_true;
+            }
+            UNLOCK(&priv->lock);
+            break;
         }
         case GF_EVENT_CHILD_DOWN:
-                LOCK (&priv->lock);
-                {
-                        priv->pass_through = _gf_false;
-                }
-                UNLOCK (&priv->lock);
-
-                if (priv->timer)
-                        break;
-                timeout.tv_sec = 20;
-                timeout.tv_nsec = 0;
-
-                priv->timer = gf_timer_call_after (this->ctx,
-                                                   timeout,
-                                                   gf_quiesce_timeout,
-                                                   (void *) this);
-
-                if (priv->timer == NULL) {
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "Cannot create timer");
-                }
-
-                break;
+            LOCK(&priv->lock);
+            {
+                priv->pass_through = _gf_false;
+                __gf_quiesce_start_timer(this, priv);
+            }
+            UNLOCK(&priv->lock);
+            break;
         default:
-                break;
-        }
+            break;
+    }
 
-        ret = default_notify (this, event, data);
+    ret = default_notify(this, event, data);
 out:
-        return ret;
+    return ret;
 }
 
-
 struct xlator_fops fops = {
-        /* write/modifying fops */
-	.mknod       = quiesce_mknod,
-	.create      = quiesce_create,
-	.truncate    = quiesce_truncate,
-	.ftruncate   = quiesce_ftruncate,
-	.setxattr    = quiesce_setxattr,
-	.removexattr = quiesce_removexattr,
-	.symlink     = quiesce_symlink,
-	.unlink      = quiesce_unlink,
-	.link        = quiesce_link,
-	.mkdir       = quiesce_mkdir,
-	.rmdir       = quiesce_rmdir,
-	.rename      = quiesce_rename,
-
-        /* The below calls are known to change state, hence
-           re-transmittion is not advised */
-	.lk          = quiesce_lk,
-	.inodelk     = quiesce_inodelk,
-	.finodelk    = quiesce_finodelk,
-	.entrylk     = quiesce_entrylk,
-	.fentrylk    = quiesce_fentrylk,
-	.xattrop     = quiesce_xattrop,
-	.fxattrop    = quiesce_fxattrop,
-        .setattr     = quiesce_setattr,
-        .fsetattr    = quiesce_fsetattr,
-
-        /* Special case, re-transmittion is not harmful *
-         * as offset is properly sent from above layers */
-        /* TODO: not re-transmitted as of now */
-	.writev      = quiesce_writev,
-
-        /* re-transmittable fops */
-	.lookup      = quiesce_lookup,
-	.stat        = quiesce_stat,
-	.fstat       = quiesce_fstat,
-	.access      = quiesce_access,
-	.readlink    = quiesce_readlink,
-	.getxattr    = quiesce_getxattr,
-	.open        = quiesce_open,
-	.readv       = quiesce_readv,
-	.flush       = quiesce_flush,
-	.fsync       = quiesce_fsync,
-	.statfs      = quiesce_statfs,
-	.opendir     = quiesce_opendir,
-	.readdir     = quiesce_readdir,
-	.readdirp    = quiesce_readdirp,
-	.fsyncdir    = quiesce_fsyncdir,
-
+    /* write/modifying fops */
+    .mknod = quiesce_mknod,
+    .create = quiesce_create,
+    .truncate = quiesce_truncate,
+    .ftruncate = quiesce_ftruncate,
+    .setxattr = quiesce_setxattr,
+    .fsetxattr = quiesce_fsetxattr,
+    .removexattr = quiesce_removexattr,
+    .fremovexattr = quiesce_fremovexattr,
+    .symlink = quiesce_symlink,
+    .unlink = quiesce_unlink,
+    .link = quiesce_link,
+    .mkdir = quiesce_mkdir,
+    .rmdir = quiesce_rmdir,
+    .rename = quiesce_rename,
+    .fallocate = quiesce_fallocate,
+
+    /* The below calls are known to change state, hence
+       re-transmittion is not advised */
+    .lk = quiesce_lk,
+    .inodelk = quiesce_inodelk,
+    .finodelk = quiesce_finodelk,
+    .entrylk = quiesce_entrylk,
+    .fentrylk = quiesce_fentrylk,
+    .xattrop = quiesce_xattrop,
+    .fxattrop = quiesce_fxattrop,
+    .setattr = quiesce_setattr,
+    .fsetattr = quiesce_fsetattr,
+
+    /* Special case, re-transmittion is not harmful *
+     * as offset is properly sent from above layers */
+    /* TODO: not re-transmitted as of now */
+    .writev = quiesce_writev,
+
+    /* re-transmittable fops */
+    .lookup = quiesce_lookup,
+    .stat = quiesce_stat,
+    .fstat = quiesce_fstat,
+    .access = quiesce_access,
+    .readlink = quiesce_readlink,
+    .getxattr = quiesce_getxattr,
+    .fgetxattr = quiesce_fgetxattr,
+    .open = quiesce_open,
+    .readv = quiesce_readv,
+    .flush = quiesce_flush,
+    .fsync = quiesce_fsync,
+    .statfs = quiesce_statfs,
+    .opendir = quiesce_opendir,
+    .readdir = quiesce_readdir,
+    .readdirp = quiesce_readdirp,
+    .fsyncdir = quiesce_fsyncdir,
+    .seek = quiesce_seek,
 };
 
 struct xlator_dumpops dumpops;
 
-
 struct xlator_cbks cbks;
 
-
 struct volume_options options[] = {
-	{ .key  = {NULL} },
+    {
+        .key = {"timeout"},
+        .type = GF_OPTION_TYPE_TIME,
+        .default_value = "45",
+        .description =
+            "After 'timeout' seconds since the time 'quiesce' "
+            "option was set to \"!pass-through\", acknowledgements to file "
+            "operations are no longer quiesced and previously "
+            "quiesced acknowledgements are sent to the application",
+        .op_version = {GD_OP_VERSION_4_0_0},
+        .flags = OPT_FLAG_CLIENT_OPT | OPT_FLAG_SETTABLE | OPT_FLAG_DOC,
+    },
+    {.key = {"failover-hosts"},
+     .type = GF_OPTION_TYPE_INTERNET_ADDRESS_LIST,
+     .op_version = {GD_OP_VERSION_4_0_0},
+     .flags = OPT_FLAG_CLIENT_OPT | OPT_FLAG_SETTABLE | OPT_FLAG_DOC,
+     .description = "It is a comma separated list of hostname/IP "
+                    "addresses. It Specifies the list of hosts where "
+                    "the gfproxy daemons are running, to which the "
+                    "the thin clients can failover to."},
+    {.key = {NULL}},
+};
+
+xlator_api_t xlator_api = {
+    .init = init,
+    .fini = fini,
+    .notify = notify,
+    .reconfigure = reconfigure,
+    .mem_acct_init = mem_acct_init,
+    .op_version = {GD_OP_VERSION_3_12_0},
+    .dumpops = &dumpops,
+    .fops = &fops,
+    .cbks = &cbks,
+    .options = options,
+    .identifier = "quiesce",
+    .category = GF_TECH_PREVIEW,
 };
diff --git a/xlators/features/quiesce/src/quiesce.h b/xlators/features/quiesce/src/quiesce.h
index 878ed77e928..6ab2af40a56 100644
--- a/xlators/features/quiesce/src/quiesce.h
+++ b/xlators/features/quiesce/src/quiesce.h
@@ -12,40 +12,54 @@
 #define __QUIESCE_H__
 
 #include "quiesce-mem-types.h"
-#include "xlator.h"
-#include "timer.h"
+#include "quiesce-messages.h"
+#include <glusterfs/xlator.h>
+#include <glusterfs/timer.h>
 
 #define GF_FOPS_EXPECTED_IN_PARALLEL 512
 
 typedef struct {
-        gf_timer_t       *timer;
-        gf_boolean_t      pass_through;
-        gf_lock_t         lock;
-        struct list_head  req;
-        int               queue_size;
-        pthread_t         thr;
-        struct mem_pool  *local_pool;
+    struct list_head list;
+    char *addr;
+    gf_boolean_t tried; /* indicates attempted connecting */
+} quiesce_failover_hosts_t;
+
+typedef struct {
+    gf_timer_t *timer;
+    gf_boolean_t pass_through;
+    gf_lock_t lock;
+    struct list_head req;
+    int queue_size;
+    pthread_t thr;
+    struct mem_pool *local_pool;
+    uint32_t timeout;
+    char *failover_hosts;
+    struct list_head failover_list;
 } quiesce_priv_t;
 
 typedef struct {
-        fd_t               *fd;
-        char               *name;
-        char               *volname;
-        loc_t               loc;
-        off_t               size;
-        off_t               offset;
-        mode_t              mode;
-        int32_t             flag;
-        struct iatt         stbuf;
-        struct iovec       *vector;
-        struct iobref      *iobref;
-        dict_t             *dict;
-        struct gf_flock     flock;
-        entrylk_cmd         cmd;
-        entrylk_type        type;
-        gf_xattrop_flags_t  xattrop_flags;
-        int32_t             wbflags;
-        uint32_t            io_flag;
+    fd_t *fd;
+    char *name;
+    char *volname;
+    loc_t loc;
+    off_t size;
+    off_t offset;
+    mode_t mode;
+    int32_t flag;
+    struct iatt stbuf;
+    struct iovec *vector;
+    struct iobref *iobref;
+    dict_t *dict;
+    struct gf_flock flock;
+    entrylk_cmd cmd;
+    entrylk_type type;
+    gf_xattrop_flags_t xattrop_flags;
+    int32_t wbflags;
+    uint32_t io_flag;
+    /* for fallocate */
+    size_t len;
+    /* for lseek */
+    gf_seek_what_t what;
 } quiesce_local_t;
 
 #endif
diff --git a/xlators/features/quota/src/Makefile.am b/xlators/features/quota/src/Makefile.am
index 7165adc59ef..1c2dcef0ca3 100644
--- a/xlators/features/quota/src/Makefile.am
+++ b/xlators/features/quota/src/Makefile.am
@@ -1,22 +1,29 @@
+if WITH_SERVER
 xlator_LTLIBRARIES = quota.la quotad.la
+endif
 xlatordir = $(libdir)/glusterfs/$(PACKAGE_VERSION)/xlator/features
 
-quota_la_LDFLAGS = -module -avoid-version
-quotad_la_LDFLAGS = -module -avoid-version
+quota_la_LDFLAGS = -module $(GF_XLATOR_DEFAULT_LDFLAGS)
+quotad_la_LDFLAGS = -module $(GF_XLATOR_DEFAULT_LDFLAGS)
 
 quota_la_SOURCES = quota.c quota-enforcer-client.c
-quota_la_LIBADD = $(top_builddir)/libglusterfs/src/libglusterfs.la
+quota_la_LIBADD = $(top_builddir)/libglusterfs/src/libglusterfs.la \
+                  $(top_builddir)/rpc/xdr/src/libgfxdr.la \
+                  $(top_builddir)/rpc/rpc-lib/src/libgfrpc.la
 
 quotad_la_SOURCES = quotad.c quotad-helpers.c quotad-aggregator.c
-quotad_la_LIBADD = $(top_builddir)/libglusterfs/src/libglusterfs.la
+quotad_la_LIBADD = $(top_builddir)/libglusterfs/src/libglusterfs.la \
+                   $(top_builddir)/rpc/xdr/src/libgfxdr.la \
+                   $(top_builddir)/rpc/rpc-lib/src/libgfrpc.la
 
-noinst_HEADERS = quota-mem-types.h quota.h quotad-aggregator.h quotad-helpers.h
+noinst_HEADERS = quota-mem-types.h quota.h quotad-aggregator.h \
+	quotad-helpers.h quota-messages.h
 
 AM_CPPFLAGS = $(GF_CPPFLAGS) -I$(top_srcdir)/libglusterfs/src \
-	-I$(top_srcdir)/xlators/cluster/dht/src -I$(top_srcdir)/rpc/xdr/src/ \
-	-I$(top_srcdir)/rpc/rpc-lib/src
+	-I$(top_srcdir)/rpc/xdr/src/ -I$(top_builddir)/rpc/xdr/src/ \
+	-I$(top_srcdir)/rpc/rpc-lib/src \
+	-I$(top_srcdir)/xlators/cluster/dht/src
 
 AM_CFLAGS = -Wall $(GF_CFLAGS)
 
 CLEANFILES =
-
diff --git a/xlators/features/quota/src/quota-enforcer-client.c b/xlators/features/quota/src/quota-enforcer-client.c
index 01cc4472664..480d64ade27 100644
--- a/xlators/features/quota/src/quota-enforcer-client.c
+++ b/xlators/features/quota/src/quota-enforcer-client.c
@@ -28,379 +28,476 @@
 #include <semaphore.h>
 #include <errno.h>
 
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
-
 #ifdef HAVE_MALLOC_H
 #include <malloc.h>
 #endif
 
-#ifdef HAVE_MALLOC_STATS
-#ifdef DEBUG
-#include <mcheck.h>
-#endif
-#endif
-
 #include "quota.h"
+#include "quota-messages.h"
 
 extern struct rpc_clnt_program quota_enforcer_clnt;
 
 int32_t
-quota_validate_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                    int32_t op_ret, int32_t op_errno, inode_t *inode,
-                    struct iatt *buf, dict_t *xdata, struct iatt *postparent);
+quota_validate_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                   int32_t op_ret, int32_t op_errno, inode_t *inode,
+                   struct iatt *buf, dict_t *xdata, struct iatt *postparent);
 
 int
-quota_enforcer_submit_request (void *req, call_frame_t *frame,
-                               rpc_clnt_prog_t *prog,
-                               int procnum, struct iobref *iobref,
-                               xlator_t *this, fop_cbk_fn_t cbkfn,
-                               xdrproc_t xdrproc)
+quota_enforcer_submit_request(void *req, call_frame_t *frame,
+                              rpc_clnt_prog_t *prog, int procnum,
+                              struct iobref *iobref, xlator_t *this,
+                              fop_cbk_fn_t cbkfn, xdrproc_t xdrproc)
 {
-        int           ret        = -1;
-        int           count      = 0;
-        struct iovec  iov        = {0, };
-        struct iobuf *iobuf      = NULL;
-        char          new_iobref = 0;
-        ssize_t       xdr_size   = 0;
-        quota_priv_t *priv       = NULL;
-
-        GF_ASSERT (this);
-
-        priv = this->private;
-
-        if (req) {
-                xdr_size = xdr_sizeof (xdrproc, req);
-                iobuf = iobuf_get2 (this->ctx->iobuf_pool, xdr_size);
-                if (!iobuf) {
-                        goto out;
-                }
-
-                if (!iobref) {
-                        iobref = iobref_new ();
-                        if (!iobref) {
-                                goto out;
-                        }
-
-                        new_iobref = 1;
-                }
-
-                iobref_add (iobref, iobuf);
-
-                iov.iov_base = iobuf->ptr;
-                iov.iov_len  = iobuf_size (iobuf);
-
-                /* Create the xdr payload */
-                ret = xdr_serialize_generic (iov, req, xdrproc);
-                if (ret == -1) {
-                        goto out;
-                }
-                iov.iov_len = ret;
-                count = 1;
+    int ret = -1;
+    int count = 0;
+    struct iovec iov = {
+        0,
+    };
+    struct iobuf *iobuf = NULL;
+    char new_iobref = 0;
+    ssize_t xdr_size = 0;
+    quota_priv_t *priv = NULL;
+
+    GF_ASSERT(this);
+
+    priv = this->private;
+
+    if (req) {
+        xdr_size = xdr_sizeof(xdrproc, req);
+        iobuf = iobuf_get2(this->ctx->iobuf_pool, xdr_size);
+        if (!iobuf) {
+            goto out;
+        }
+
+        if (!iobref) {
+            iobref = iobref_new();
+            if (!iobref) {
+                goto out;
+            }
+
+            new_iobref = 1;
+        }
+
+        iobref_add(iobref, iobuf);
+
+        iov.iov_base = iobuf->ptr;
+        iov.iov_len = iobuf_size(iobuf);
+
+        /* Create the xdr payload */
+        ret = xdr_serialize_generic(iov, req, xdrproc);
+        if (ret == -1) {
+            goto out;
         }
+        iov.iov_len = ret;
+        count = 1;
+    }
 
-        /* Send the msg */
-        ret = rpc_clnt_submit (priv->rpc_clnt, prog, procnum, cbkfn,
-                               &iov, count,
-                               NULL, 0, iobref, frame, NULL, 0, NULL, 0, NULL);
-        ret = 0;
+    /* Send the msg */
+    ret = rpc_clnt_submit(priv->rpc_clnt, prog, procnum, cbkfn, &iov, count,
+                          NULL, 0, iobref, frame, NULL, 0, NULL, 0, NULL);
+    ret = 0;
 
 out:
-        if (new_iobref)
-                iobref_unref (iobref);
-        if (iobuf)
-                iobuf_unref (iobuf);
+    if (new_iobref)
+        iobref_unref(iobref);
+    if (iobuf)
+        iobuf_unref(iobuf);
 
-        return ret;
+    return ret;
 }
 
 int
-quota_enforcer_lookup_cbk (struct rpc_req *req, struct iovec *iov,
-                           int count, void *myframe)
+quota_enforcer_lookup_cbk(struct rpc_req *req, struct iovec *iov, int count,
+                          void *myframe)
 {
-        quota_local_t    *local      = NULL;
-        call_frame_t     *frame      = NULL;
-        int               ret        = 0;
-        gfs3_lookup_rsp   rsp        = {0,};
-        struct iatt       stbuf      = {0,};
-        struct iatt       postparent = {0,};
-        int               op_errno   = EINVAL;
-        dict_t           *xdata      = NULL;
-        inode_t          *inode      = NULL;
-        xlator_t         *this       = NULL;
-
-        this = THIS;
-
-        frame = myframe;
-        local = frame->local;
-        inode = local->validate_loc.inode;
-
-        if (-1 == req->rpc_status) {
-                rsp.op_ret   = -1;
-                op_errno = ENOTCONN;
-                goto out;
-        }
+    quota_local_t *local = NULL;
+    call_frame_t *frame = NULL;
+    int ret = 0;
+    gfs3_lookup_rsp rsp = {
+        0,
+    };
+    struct iatt stbuf = {
+        0,
+    };
+    struct iatt postparent = {
+        0,
+    };
+    int op_errno = EINVAL;
+    dict_t *xdata = NULL;
+    inode_t *inode = NULL;
+    xlator_t *this = NULL;
+    quota_priv_t *priv = NULL;
+    struct timespec retry_delay = {
+        0,
+    };
+    gf_timer_t *timer = NULL;
+
+    this = THIS;
+
+    frame = myframe;
+    local = frame->local;
+    inode = local->validate_loc.inode;
+    priv = this->private;
+
+    if (-1 == req->rpc_status) {
+        rsp.op_ret = -1;
+        op_errno = ENOTCONN;
+        goto out;
+    }
+
+    ret = xdr_to_generic(*iov, &rsp, (xdrproc_t)xdr_gfs3_lookup_rsp);
+    if (ret < 0) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, Q_MSG_XDR_DECODING_FAILED,
+               "XDR decoding failed");
+        rsp.op_ret = -1;
+        op_errno = EINVAL;
+        goto out;
+    }
 
-        ret = xdr_to_generic (*iov, &rsp, (xdrproc_t)xdr_gfs3_lookup_rsp);
-        if (ret < 0) {
-                gf_log (this->name, GF_LOG_ERROR, "XDR decoding failed");
-                rsp.op_ret   = -1;
-                op_errno = EINVAL;
-                goto out;
-        }
+    op_errno = gf_error_to_errno(rsp.op_errno);
+    gf_stat_to_iatt(&rsp.postparent, &postparent);
 
-        op_errno = gf_error_to_errno (rsp.op_errno);
-        gf_stat_to_iatt (&rsp.postparent, &postparent);
+    if (rsp.op_ret == -1)
+        goto out;
 
-        if (rsp.op_ret == -1)
-                goto out;
+    rsp.op_ret = -1;
+    gf_stat_to_iatt(&rsp.stat, &stbuf);
 
+    GF_PROTOCOL_DICT_UNSERIALIZE(frame->this, xdata, (rsp.xdata.xdata_val),
+                                 (rsp.xdata.xdata_len), rsp.op_ret, op_errno,
+                                 out);
+
+    if ((!gf_uuid_is_null(inode->gfid)) &&
+        (gf_uuid_compare(stbuf.ia_gfid, inode->gfid) != 0)) {
+        gf_msg_debug(frame->this->name, ESTALE, "gfid changed for %s",
+                     local->validate_loc.path);
         rsp.op_ret = -1;
-        gf_stat_to_iatt (&rsp.stat, &stbuf);
-
-        GF_PROTOCOL_DICT_UNSERIALIZE (frame->this, xdata, (rsp.xdata.xdata_val),
-                                      (rsp.xdata.xdata_len), rsp.op_ret,
-                                      op_errno, out);
-
-        if ((!gf_uuid_is_null (inode->gfid))
-            && (gf_uuid_compare (stbuf.ia_gfid, inode->gfid) != 0)) {
-                gf_log (frame->this->name, GF_LOG_DEBUG,
-                        "gfid changed for %s", local->validate_loc.path);
-                rsp.op_ret = -1;
-                op_errno = ESTALE;
-                goto out;
-        }
+        op_errno = ESTALE;
+        goto out;
+    }
 
-        rsp.op_ret = 0;
+    rsp.op_ret = 0;
 
 out:
-        rsp.op_errno = op_errno;
-        if (rsp.op_ret == -1) {
-                /* any error other than ENOENT */
-                if (rsp.op_errno != ENOENT)
-                        gf_log (this->name, GF_LOG_WARNING,
-                                "remote operation failed: %s. Path: %s (%s)",
-                                strerror (rsp.op_errno),
-                                local->validate_loc.path,
-                                loc_gfid_utoa (&local->validate_loc));
-                else
-                        gf_log (this->name, GF_LOG_TRACE,
-                                "not found on remote node");
+    rsp.op_errno = op_errno;
+
+    /* We need to retry connecting to quotad on ENOTCONN error.
+     * Suppose if there are two volumes vol1 and vol2,
+     * and quota is enabled and limit is set on vol1.
+     * Now if IO is happening on vol1 and quota is enabled/disabled
+     * on vol2, quotad gets restarted and client will receive
+     * ENOTCONN in the IO path of vol1
+     */
+    if (rsp.op_ret == -1 && rsp.op_errno == ENOTCONN) {
+        if (local->quotad_conn_retry >= 12) {
+            priv->quotad_conn_status = 1;
+            gf_log(this->name, GF_LOG_WARNING,
+                   "failed to connect "
+                   "to quotad after retry count %d)",
+                   local->quotad_conn_retry);
+        } else {
+            local->quotad_conn_retry++;
+        }
 
+        if (priv->quotad_conn_status == 0) {
+            /* retry connecting after 5secs for 12 retries
+             * (up to 60sec).
+             */
+            gf_log(this->name, GF_LOG_DEBUG,
+                   "retry connecting to "
+                   "quotad (retry count %d)",
+                   local->quotad_conn_retry);
+
+            retry_delay.tv_sec = 5;
+            retry_delay.tv_nsec = 0;
+            timer = gf_timer_call_after(this->ctx, retry_delay,
+                                        _quota_enforcer_lookup, (void *)frame);
+            if (timer == NULL) {
+                gf_log(this->name, GF_LOG_WARNING,
+                       "failed to "
+                       "set quota_enforcer_lookup with timer");
+            } else {
+                goto clean;
+            }
         }
+    } else {
+        priv->quotad_conn_status = 0;
+    }
+
+    if (rsp.op_ret == -1) {
+        /* any error other than ENOENT */
+        if (rsp.op_errno != ENOENT)
+            gf_msg(
+                this->name, GF_LOG_WARNING, rsp.op_errno, Q_MSG_LOOKUP_FAILED,
+                "Getting cluster-wide size of directory failed "
+                "(path: %s gfid:%s)",
+                local->validate_loc.path, loc_gfid_utoa(&local->validate_loc));
+        else
+            gf_msg_trace(this->name, ENOENT, "not found on remote node");
+
+    } else if (local->quotad_conn_retry) {
+        gf_log(this->name, GF_LOG_DEBUG,
+               "connected to quotad after "
+               "retry count %d",
+               local->quotad_conn_retry);
+    }
 
-        local->validate_cbk (frame, NULL, this, rsp.op_ret, rsp.op_errno, inode,
-                             &stbuf, xdata, &postparent);
+    local->validate_cbk(frame, NULL, this, rsp.op_ret, rsp.op_errno, inode,
+                        &stbuf, xdata, &postparent);
 
-        if (xdata)
-                dict_unref (xdata);
+clean:
+    if (xdata)
+        dict_unref(xdata);
 
-        free (rsp.xdata.xdata_val);
+    free(rsp.xdata.xdata_val);
 
-        return 0;
+    return 0;
 }
 
-int
-quota_enforcer_lookup (call_frame_t *frame, xlator_t *this, loc_t *loc,
-                       dict_t *xdata, fop_lookup_cbk_t validate_cbk)
+void
+_quota_enforcer_lookup(void *data)
 {
-        quota_local_t          *local      = NULL;
-        gfs3_lookup_req         req        = {{0,},};
-        int                     ret        = 0;
-        int                     op_errno   = ESTALE;
-        quota_priv_t           *priv       = NULL;
-
-        if (!frame || !this || !loc)
-                goto unwind;
-
-        local = frame->local;
-        local->validate_cbk = validate_cbk;
-
-        priv = this->private;
+    quota_local_t *local = NULL;
+    gfs3_lookup_req req = {
+        {
+            0,
+        },
+    };
+    int ret = 0;
+    int op_errno = ESTALE;
+    quota_priv_t *priv = NULL;
+    call_frame_t *frame = NULL;
+    loc_t *loc = NULL;
+    xlator_t *this = NULL;
+    char *dir_path = NULL;
+
+    frame = data;
+    local = frame->local;
+    this = local->this;
+    loc = &local->validate_loc;
+
+    priv = this->private;
+
+    if (!(loc && loc->inode))
+        goto unwind;
+
+    if (!gf_uuid_is_null(loc->inode->gfid))
+        memcpy(req.gfid, loc->inode->gfid, 16);
+    else
+        memcpy(req.gfid, loc->gfid, 16);
+
+    if (local->validate_xdata) {
+        GF_PROTOCOL_DICT_SERIALIZE(this, local->validate_xdata,
+                                   (&req.xdata.xdata_val), req.xdata.xdata_len,
+                                   op_errno, unwind);
+    }
+
+    if (loc->name)
+        req.bname = (char *)loc->name;
+    else
+        req.bname = "";
+
+    if (loc->path)
+        dir_path = (char *)loc->path;
+    else
+        dir_path = "";
+
+    ret = quota_enforcer_submit_request(
+        &req, frame, priv->quota_enforcer, GF_AGGREGATOR_LOOKUP, NULL, this,
+        quota_enforcer_lookup_cbk, (xdrproc_t)xdr_gfs3_lookup_req);
+
+    if (ret) {
+        gf_msg(this->name, GF_LOG_WARNING, 0, Q_MSG_RPC_SUBMIT_FAILED,
+               "Couldn't send the request to "
+               "fetch cluster wide size of directory (path:%s gfid:%s)",
+               dir_path, req.gfid);
+    }
+
+    GF_FREE(req.xdata.xdata_val);
+
+    return;
 
-        if (!(loc && loc->inode))
-                goto unwind;
+unwind:
+    local->validate_cbk(frame, NULL, this, -1, op_errno, NULL, NULL, NULL,
+                        NULL);
 
-        if (!gf_uuid_is_null (loc->inode->gfid))
-                memcpy (req.gfid, loc->inode->gfid, 16);
-        else
-                memcpy (req.gfid, loc->gfid, 16);
+    GF_FREE(req.xdata.xdata_val);
 
-        if (xdata) {
-                GF_PROTOCOL_DICT_SERIALIZE (this, xdata,
-                                            (&req.xdata.xdata_val),
-                                            req.xdata.xdata_len,
-                                            op_errno, unwind);
-        }
+    return;
+}
 
-        if (loc->name)
-                req.bname = (char *)loc->name;
-        else
-                req.bname = "";
+int
+quota_enforcer_lookup(call_frame_t *frame, xlator_t *this, dict_t *xdata,
+                      fop_lookup_cbk_t validate_cbk)
+{
+    quota_local_t *local = NULL;
 
-        ret = quota_enforcer_submit_request (&req, frame,
-                                             priv->quota_enforcer,
-                                             GF_AGGREGATOR_LOOKUP,
-                                             NULL, this,
-                                             quota_enforcer_lookup_cbk,
-                                             (xdrproc_t)xdr_gfs3_lookup_req);
+    if (!frame || !this)
+        goto unwind;
 
-        if (ret) {
-                gf_log (this->name, GF_LOG_WARNING, "failed to send the fop");
-        }
+    local = frame->local;
+    local->this = this;
+    local->validate_cbk = validate_cbk;
+    local->validate_xdata = dict_ref(xdata);
 
-        GF_FREE (req.xdata.xdata_val);
+    _quota_enforcer_lookup(frame);
 
-        return 0;
+    return 0;
 
 unwind:
-        validate_cbk (frame, NULL, this, -1, op_errno, NULL, NULL, NULL, NULL);
+    validate_cbk(frame, NULL, this, -1, ESTALE, NULL, NULL, NULL, NULL);
 
-        GF_FREE (req.xdata.xdata_val);
-
-        return 0;
+    return 0;
 }
 
 int
-quota_enforcer_notify (struct rpc_clnt *rpc, void *mydata,
-                       rpc_clnt_event_t event, void *data)
+quota_enforcer_notify(struct rpc_clnt *rpc, void *mydata,
+                      rpc_clnt_event_t event, void *data)
 {
-        xlator_t                *this = NULL;
-        int                     ret = 0;
-
-        this = mydata;
-
-        switch (event) {
-        case RPC_CLNT_CONNECT:
-        {
-                gf_log (this->name, GF_LOG_TRACE, "got RPC_CLNT_CONNECT");
-                break;
+    xlator_t *this = NULL;
+    int ret = 0;
+    quota_priv_t *priv = NULL;
+
+    this = mydata;
+    priv = this->private;
+    switch (event) {
+        case RPC_CLNT_CONNECT: {
+            pthread_mutex_lock(&priv->conn_mutex);
+            {
+                priv->conn_status = _gf_true;
+            }
+            pthread_mutex_unlock(&priv->conn_mutex);
+            gf_msg_trace(this->name, 0, "got RPC_CLNT_CONNECT");
+            break;
         }
 
-        case RPC_CLNT_DISCONNECT:
-        {
-                gf_log (this->name, GF_LOG_TRACE, "got RPC_CLNT_DISCONNECT");
-                break;
+        case RPC_CLNT_DISCONNECT: {
+            pthread_mutex_lock(&priv->conn_mutex);
+            {
+                priv->conn_status = _gf_false;
+                pthread_cond_signal(&priv->conn_cond);
+            }
+            pthread_mutex_unlock(&priv->conn_mutex);
+            gf_msg_trace(this->name, 0, "got RPC_CLNT_DISCONNECT");
+            break;
         }
 
         default:
-                gf_log (this->name, GF_LOG_TRACE,
-                        "got some other RPC event %d", event);
-                ret = 0;
-                break;
-        }
+            gf_msg_trace(this->name, 0, "got some other RPC event %d", event);
+            ret = 0;
+            break;
+    }
 
-        return ret;
+    return ret;
 }
 
 int
-quota_enforcer_blocking_connect (rpc_clnt_t *rpc)
+quota_enforcer_blocking_connect(rpc_clnt_t *rpc)
 {
-        dict_t *options = NULL;
-        int     ret     = -1;
+    dict_t *options = NULL;
+    int ret = -1;
 
-        options = dict_new ();
-        if (options == NULL)
-                goto out;
+    options = dict_new();
+    if (options == NULL)
+        goto out;
 
-        ret = dict_set_str (options, "non-blocking-io", "no");
-        if (ret)
-                goto out;
+    ret = dict_set_sizen_str_sizen(options, "non-blocking-io", "no");
+    if (ret)
+        goto out;
 
-        rpc->conn.trans->reconfigure (rpc->conn.trans, options);
+    rpc->conn.trans->reconfigure(rpc->conn.trans, options);
 
-        rpc_clnt_start (rpc);
+    rpc_clnt_start(rpc);
 
-        ret = dict_set_str (options, "non-blocking-io", "yes");
-        if (ret)
-                goto out;
+    ret = dict_set_sizen_str_sizen(options, "non-blocking-io", "yes");
+    if (ret)
+        goto out;
 
-        rpc->conn.trans->reconfigure (rpc->conn.trans, options);
+    rpc->conn.trans->reconfigure(rpc->conn.trans, options);
 
-        ret = 0;
+    ret = 0;
 out:
-        if (options)
-                dict_unref (options);
+    if (options)
+        dict_unref(options);
 
-        return ret;
+    return ret;
 }
 
-//Returns a started rpc_clnt. Creates a new rpc_clnt if quota_priv doesn't have
-//one already
+// Returns a started rpc_clnt. Creates a new rpc_clnt if quota_priv doesn't have
+// one already
 struct rpc_clnt *
-quota_enforcer_init (xlator_t *this, dict_t *options)
+quota_enforcer_init(xlator_t *this, dict_t *options)
 {
-        struct rpc_clnt *rpc  = NULL;
-        quota_priv_t    *priv = NULL;
-        int              ret  = -1;
+    struct rpc_clnt *rpc = NULL;
+    quota_priv_t *priv = NULL;
+    int ret = -1;
 
-        priv = this->private;
+    priv = this->private;
 
-        LOCK (&priv->lock);
-        {
-                if (priv->rpc_clnt) {
-                        ret = 0;
-                        rpc = priv->rpc_clnt;
-                }
+    LOCK(&priv->lock);
+    {
+        if (priv->rpc_clnt) {
+            ret = 0;
+            rpc = priv->rpc_clnt;
         }
-        UNLOCK (&priv->lock);
+    }
+    UNLOCK(&priv->lock);
 
-        if (rpc)
-                goto out;
+    if (rpc)
+        goto out;
 
-        priv->quota_enforcer = &quota_enforcer_clnt;
+    priv->quota_enforcer = &quota_enforcer_clnt;
 
-        ret = dict_set_str (options, "transport.address-family", "unix");
-        if (ret)
-                goto out;
+    ret = dict_set_sizen_str_sizen(options, "transport.address-family", "unix");
+    if (ret)
+        goto out;
 
-        ret = dict_set_str (options, "transport-type", "socket");
-        if (ret)
-                goto out;
+    ret = dict_set_sizen_str_sizen(options, "transport-type", "socket");
+    if (ret)
+        goto out;
 
-        ret = dict_set_str (options, "transport.socket.connect-path",
-                            "/var/run/gluster/quotad.socket");
-        if (ret)
-                goto out;
+    ret = dict_set_sizen_str_sizen(options, "transport.socket.connect-path",
+                                   "/var/run/gluster/quotad.socket");
+    if (ret)
+        goto out;
 
-        rpc = rpc_clnt_new (options, this->ctx, this->name, 16);
-        if (!rpc) {
-                ret = -1;
-                goto out;
-        }
+    rpc = rpc_clnt_new(options, this, this->name, 16);
+    if (!rpc) {
+        ret = -1;
+        goto out;
+    }
 
-        ret = rpc_clnt_register_notify (rpc, quota_enforcer_notify, this);
-        if (ret) {
-                gf_log ("cli", GF_LOG_ERROR, "failed to register notify");
-                goto out;
-        }
+    ret = rpc_clnt_register_notify(rpc, quota_enforcer_notify, this);
+    if (ret) {
+        gf_msg("quota", GF_LOG_ERROR, 0, Q_MSG_RPCCLNT_REGISTER_NOTIFY_FAILED,
+               "failed to register notify");
+        goto out;
+    }
 
-        ret = quota_enforcer_blocking_connect (rpc);
-        if (ret)
-                goto out;
+    ret = quota_enforcer_blocking_connect(rpc);
+    if (ret)
+        goto out;
 
-        ret = 0;
+    ret = 0;
 out:
-        if (ret) {
-                if (rpc)
-                        rpc_clnt_unref (rpc);
-                rpc = NULL;
-        }
+    if (ret) {
+        if (rpc)
+            rpc_clnt_unref(rpc);
+        rpc = NULL;
+    }
 
-        return rpc;
+    return rpc;
 }
 
 struct rpc_clnt_procedure quota_enforcer_actors[GF_AGGREGATOR_MAXVALUE] = {
-        [GF_AGGREGATOR_NULL]     = {"NULL", NULL},
-        [GF_AGGREGATOR_LOOKUP]   = {"LOOKUP", NULL},
+    [GF_AGGREGATOR_NULL] = {"NULL", NULL},
+    [GF_AGGREGATOR_LOOKUP] = {"LOOKUP", NULL},
 };
 
 struct rpc_clnt_program quota_enforcer_clnt = {
-        .progname  = "Quota enforcer",
-        .prognum   = GLUSTER_AGGREGATOR_PROGRAM,
-        .progver   = GLUSTER_AGGREGATOR_VERSION,
-        .numproc   = GF_AGGREGATOR_MAXVALUE,
-        .proctable = quota_enforcer_actors,
+    .progname = "Quota enforcer",
+    .prognum = GLUSTER_AGGREGATOR_PROGRAM,
+    .progver = GLUSTER_AGGREGATOR_VERSION,
+    .numproc = GF_AGGREGATOR_MAXVALUE,
+    .proctable = quota_enforcer_actors,
 };
diff --git a/xlators/features/quota/src/quota-mem-types.h b/xlators/features/quota/src/quota-mem-types.h
index 97d9165681f..782a7de96bb 100644
--- a/xlators/features/quota/src/quota-mem-types.h
+++ b/xlators/features/quota/src/quota-mem-types.h
@@ -10,21 +10,21 @@
 #ifndef __QUOTA_MEM_TYPES_H__
 #define __QUOTA_MEM_TYPES_H__
 
-#include "mem-types.h"
+#include <glusterfs/mem-types.h>
 
 enum gf_quota_mem_types_ {
-        gf_quota_mt_quota_priv_t = gf_common_mt_end + 1,
-        gf_quota_mt_quota_inode_ctx_t,
-        gf_quota_mt_loc_t,
-        gf_quota_mt_char,
-        gf_quota_mt_int64_t,
-        gf_quota_mt_int32_t,
-        gf_quota_mt_limits_t,
-        gf_quota_mt_quota_dentry_t,
-        gf_quota_mt_quota_limits_level_t,
-        gf_quota_mt_qd_vols_conf_t,
-        gf_quota_mt_aggregator_state_t,
-        gf_quota_mt_end
+    /* Those are used by QUOTA_ALLOC_OR_GOTO macro */
+    gf_quota_mt_quota_priv_t = gf_common_mt_end + 1,
+    gf_quota_mt_quota_inode_ctx_t,
+    gf_quota_mt_loc_t,
+    gf_quota_mt_char,
+    gf_quota_mt_int64_t,
+    gf_quota_mt_int32_t,
+    gf_quota_mt_limits_t,
+    gf_quota_mt_quota_dentry_t,
+    gf_quota_mt_quota_limits_level_t,
+    gf_quota_mt_qd_vols_conf_t,
+    gf_quota_mt_aggregator_state_t,
+    gf_quota_mt_end
 };
 #endif
-
diff --git a/xlators/features/quota/src/quota-messages.h b/xlators/features/quota/src/quota-messages.h
new file mode 100644
index 00000000000..d434ed75e76
--- /dev/null
+++ b/xlators/features/quota/src/quota-messages.h
@@ -0,0 +1,39 @@
+/*
+  Copyright (c) 2013 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
+*/
+
+#ifndef _QUOTA_MESSAGES_H_
+#define _QUOTA_MESSAGES_H_
+
+#include <glusterfs/glfs-message-id.h>
+
+/* To add new message IDs, append new identifiers at the end of the list.
+ *
+ * Never remove a message ID. If it's not used anymore, you can rename it or
+ * leave it as it is, but not delete it. This is to prevent reutilization of
+ * IDs by other messages.
+ *
+ * The component name must match one of the entries defined in
+ * glfs-message-id.h.
+ */
+
+GLFS_MSGID(QUOTA, Q_MSG_ENFORCEMENT_FAILED, Q_MSG_ENOMEM, Q_MSG_PARENT_NULL,
+           Q_MSG_CROSSED_SOFT_LIMIT, Q_MSG_QUOTA_ENFORCER_RPC_INIT_FAILED,
+           Q_MSG_REMOTE_OPERATION_FAILED, Q_MSG_FAILED_TO_SEND_FOP,
+           Q_MSG_INVALID_VOLFILE, Q_MSG_INODE_PARENT_NOT_FOUND,
+           Q_MSG_XDR_DECODE_ERROR, Q_MSG_DICT_UNSERIALIZE_FAIL,
+           Q_MSG_DICT_SERIALIZE_FAIL, Q_MSG_RPCSVC_INIT_FAILED,
+           Q_MSG_RPCSVC_LISTENER_CREATION_FAILED, Q_MSG_RPCSVC_REGISTER_FAILED,
+           Q_MSG_XDR_DECODING_FAILED, Q_MSG_RPCCLNT_REGISTER_NOTIFY_FAILED,
+           Q_MSG_ANCESTRY_BUILD_FAILED, Q_MSG_SIZE_KEY_MISSING,
+           Q_MSG_INODE_CTX_GET_FAILED, Q_MSG_INODE_CTX_SET_FAILED,
+           Q_MSG_LOOKUP_FAILED, Q_MSG_RPC_SUBMIT_FAILED,
+           Q_MSG_ENFORCEMENT_SKIPPED, Q_MSG_INTERNAL_FOP_KEY_MISSING);
+
+#endif /* !_QUOTA_MESSAGES_H_ */
diff --git a/xlators/features/quota/src/quota.c b/xlators/features/quota/src/quota.c
index 61a68e2464f..18df9ae6d19 100644
--- a/xlators/features/quota/src/quota.c
+++ b/xlators/features/quota/src/quota.c
@@ -7,4530 +7,4636 @@
    later), or the GNU General Public License, version 2 (GPLv2), in all
    cases as published by the Free Software Foundation.
 */
-#include <fnmatch.h>
 
 #include "quota.h"
-#include "common-utils.h"
-#include "defaults.h"
-#include "statedump.h"
-#include "quota-common-utils.h"
+#include <glusterfs/statedump.h>
+#include "quota-messages.h"
+#include <glusterfs/events.h>
 
 struct volume_options options[];
 
 static int32_t
-__quota_init_inode_ctx (inode_t *inode, xlator_t *this,
-                        quota_inode_ctx_t **context)
+__quota_init_inode_ctx(inode_t *inode, xlator_t *this,
+                       quota_inode_ctx_t **context)
 {
-        int32_t            ret  = -1;
-        quota_inode_ctx_t *ctx  = NULL;
+    int32_t ret = -1;
+    quota_inode_ctx_t *ctx = NULL;
 
-        if (inode == NULL) {
-                goto out;
-        }
+    if (inode == NULL) {
+        goto out;
+    }
 
-        QUOTA_ALLOC_OR_GOTO (ctx, quota_inode_ctx_t, out);
+    QUOTA_ALLOC_OR_GOTO(ctx, quota_inode_ctx_t, out);
 
-        LOCK_INIT(&ctx->lock);
+    LOCK_INIT(&ctx->lock);
 
-        if (context != NULL) {
-                *context = ctx;
-        }
+    if (context != NULL) {
+        *context = ctx;
+    }
 
-        INIT_LIST_HEAD (&ctx->parents);
+    INIT_LIST_HEAD(&ctx->parents);
 
-        ret = __inode_ctx_put (inode, this, (uint64_t )(long)ctx);
-        if (ret == -1) {
-                gf_log (this->name, GF_LOG_WARNING,
-                        "cannot set quota context in inode (gfid:%s)",
-                        uuid_utoa (inode->gfid));
-        }
+    ret = __inode_ctx_put(inode, this, (uint64_t)(long)ctx);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_WARNING, 0, Q_MSG_INODE_CTX_SET_FAILED,
+               "cannot set quota context "
+               "in inode (gfid:%s)",
+               uuid_utoa(inode->gfid));
+        GF_FREE(ctx);
+    }
 out:
-        return ret;
+    return ret;
 }
 
-
 static int32_t
-quota_inode_ctx_get (inode_t *inode, xlator_t *this,
-                     quota_inode_ctx_t **ctx, char create_if_absent)
+quota_inode_ctx_get(inode_t *inode, xlator_t *this, quota_inode_ctx_t **ctx,
+                    char create_if_absent)
 {
-        int32_t  ret = 0;
-        uint64_t ctx_int;
+    int32_t ret = 0;
+    uint64_t ctx_int;
 
-        LOCK (&inode->lock);
-        {
-                ret = __inode_ctx_get (inode, this, &ctx_int);
+    LOCK(&inode->lock);
+    {
+        ret = __inode_ctx_get(inode, this, &ctx_int);
 
-                if ((ret == 0) && (ctx != NULL)) {
-                        *ctx = (quota_inode_ctx_t *) (unsigned long)ctx_int;
-                } else if (create_if_absent) {
-                        ret = __quota_init_inode_ctx (inode, this, ctx);
-                }
+        if ((ret == 0) && (ctx != NULL)) {
+            *ctx = (quota_inode_ctx_t *)(unsigned long)ctx_int;
+        } else if (create_if_absent) {
+            ret = __quota_init_inode_ctx(inode, this, ctx);
         }
-        UNLOCK (&inode->lock);
+    }
+    UNLOCK(&inode->lock);
 
-        return ret;
+    return ret;
 }
 
 int
-quota_loc_fill (loc_t *loc, inode_t *inode, inode_t *parent, char *path)
+quota_loc_fill(loc_t *loc, inode_t *inode, inode_t *parent, char *path)
 {
-        int ret = -1;
+    int ret = -1;
 
-        if (!loc || (inode == NULL))
-                return ret;
+    if (!loc || (inode == NULL))
+        return ret;
 
-        if (inode) {
-                loc->inode = inode_ref (inode);
-                gf_uuid_copy (loc->gfid, inode->gfid);
-        }
+    if (inode) {
+        loc->inode = inode_ref(inode);
+        gf_uuid_copy(loc->gfid, inode->gfid);
+    }
 
-        if (parent) {
-                loc->parent = inode_ref (parent);
-        }
+    if (parent) {
+        loc->parent = inode_ref(parent);
+    }
 
-        if (path != NULL) {
-                loc->path = gf_strdup (path);
+    if (path != NULL) {
+        loc->path = gf_strdup(path);
 
-                loc->name = strrchr (loc->path, '/');
-                if (loc->name) {
-                        loc->name++;
-                }
+        loc->name = strrchr(loc->path, '/');
+        if (loc->name) {
+            loc->name++;
         }
+    }
 
-        ret = 0;
+    ret = 0;
 
-        return ret;
+    return ret;
 }
 
-
 int
-quota_inode_loc_fill (inode_t *inode, loc_t *loc)
+quota_inode_loc_fill(inode_t *inode, loc_t *loc)
 {
-        char            *resolvedpath = NULL;
-        inode_t         *parent       = NULL;
-        int              ret          = -1;
-        xlator_t        *this         = NULL;
+    char *resolvedpath = NULL;
+    inode_t *parent = NULL;
+    int ret = -1;
+    xlator_t *this = NULL;
 
-        if ((!inode) || (!loc)) {
-                return ret;
-        }
+    if ((!inode) || (!loc)) {
+        return ret;
+    }
 
-        this = THIS;
+    this = THIS;
 
-        if ((inode) && __is_root_gfid (inode->gfid)) {
-                loc->parent = NULL;
-                goto ignore_parent;
-        }
+    if ((inode) && __is_root_gfid(inode->gfid)) {
+        loc->parent = NULL;
+        goto ignore_parent;
+    }
 
-        parent = inode_parent (inode, 0, NULL);
-        if (!parent) {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "cannot find parent for inode (gfid:%s)",
-                        uuid_utoa (inode->gfid));
-        }
+    parent = inode_parent(inode, 0, NULL);
+    if (!parent) {
+        gf_msg_debug(this->name, 0,
+                     "cannot find parent for "
+                     "inode (gfid:%s)",
+                     uuid_utoa(inode->gfid));
+    }
 
 ignore_parent:
-        ret = inode_path (inode, NULL, &resolvedpath);
-        if (ret < 0) {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "cannot construct path for inode (gfid:%s)",
-                        uuid_utoa (inode->gfid));
-        }
-
-        ret = quota_loc_fill (loc, inode, parent, resolvedpath);
-        if (ret < 0) {
-                gf_log (this->name, GF_LOG_WARNING, "cannot fill loc");
-                goto err;
-        }
+    ret = inode_path(inode, NULL, &resolvedpath);
+    if (ret < 0) {
+        gf_msg_debug(this->name, 0,
+                     "cannot construct path for "
+                     "inode (gfid:%s)",
+                     uuid_utoa(inode->gfid));
+    }
+
+    ret = quota_loc_fill(loc, inode, parent, resolvedpath);
+    if (ret < 0) {
+        gf_msg(this->name, GF_LOG_WARNING, ENOMEM, Q_MSG_ENOMEM,
+               "cannot fill loc");
+        goto err;
+    }
 
 err:
-        if (parent) {
-                inode_unref (parent);
-        }
+    if (parent) {
+        inode_unref(parent);
+    }
 
-        GF_FREE (resolvedpath);
+    GF_FREE(resolvedpath);
 
-        return ret;
+    return ret;
 }
 
-
 int32_t
-quota_local_cleanup (xlator_t *this, quota_local_t *local)
+quota_local_cleanup(quota_local_t *local)
 {
-        if (local == NULL) {
-                goto out;
-        }
+    if (local == NULL) {
+        goto out;
+    }
 
-        loc_wipe (&local->loc);
-        loc_wipe (&local->newloc);
-        loc_wipe (&local->oldloc);
-        loc_wipe (&local->validate_loc);
+    loc_wipe(&local->loc);
+    loc_wipe(&local->newloc);
+    loc_wipe(&local->oldloc);
+    loc_wipe(&local->validate_loc);
 
-        inode_unref (local->inode);
+    inode_unref(local->inode);
 
-        if (local->xdata)
-                dict_unref (local->xdata);
+    if (local->xdata)
+        dict_unref(local->xdata);
 
-        if (local->stub)
-                call_stub_destroy (local->stub);
+    if (local->validate_xdata)
+        dict_unref(local->validate_xdata);
 
-        LOCK_DESTROY (&local->lock);
+    if (local->stub)
+        call_stub_destroy(local->stub);
 
-        mem_put (local);
+    LOCK_DESTROY(&local->lock);
+
+    mem_put(local);
 out:
-        return 0;
+    return 0;
 }
 
-
-static inline quota_local_t *
-quota_local_new ()
+static quota_local_t *
+quota_local_new()
 {
-        quota_local_t *local = NULL;
-        local = mem_get0 (THIS->local_pool);
-        if (local == NULL)
-                goto out;
+    quota_local_t *local = NULL;
+    local = mem_get0(THIS->local_pool);
+    if (local == NULL)
+        goto out;
 
-        LOCK_INIT (&local->lock);
-        local->space_available = -1;
+    LOCK_INIT(&local->lock);
+    local->space_available = -1;
 
 out:
-        return local;
+    return local;
 }
 
-
 quota_dentry_t *
-__quota_dentry_new (quota_inode_ctx_t *ctx, char *name, uuid_t par)
+__quota_dentry_new(quota_inode_ctx_t *ctx, char *name, uuid_t par)
 {
-        quota_dentry_t    *dentry = NULL;
-        GF_UNUSED int32_t  ret    = 0;
+    quota_dentry_t *dentry = NULL;
+    GF_UNUSED int32_t ret = 0;
 
-        QUOTA_ALLOC_OR_GOTO (dentry, quota_dentry_t, err);
+    QUOTA_ALLOC_OR_GOTO(dentry, quota_dentry_t, err);
 
-        INIT_LIST_HEAD (&dentry->next);
+    INIT_LIST_HEAD(&dentry->next);
 
-        dentry->name = gf_strdup (name);
-        if (dentry->name == NULL) {
-                GF_FREE (dentry);
-                dentry = NULL;
-                goto err;
-        }
+    dentry->name = gf_strdup(name);
+    if (dentry->name == NULL) {
+        GF_FREE(dentry);
+        dentry = NULL;
+        goto err;
+    }
 
-        gf_uuid_copy (dentry->par, par);
+    gf_uuid_copy(dentry->par, par);
 
-        if (ctx != NULL)
-                list_add_tail (&dentry->next, &ctx->parents);
+    if (ctx != NULL)
+        list_add_tail(&dentry->next, &ctx->parents);
 
 err:
-        return dentry;
+    return dentry;
 }
 
-
 void
-__quota_dentry_free (quota_dentry_t *dentry)
+__quota_dentry_free(quota_dentry_t *dentry)
 {
-        if (dentry == NULL) {
-                goto out;
-        }
+    if (dentry == NULL) {
+        goto out;
+    }
 
-        list_del_init (&dentry->next);
+    list_del_init(&dentry->next);
 
-        GF_FREE (dentry->name);
-        GF_FREE (dentry);
+    GF_FREE(dentry->name);
+    GF_FREE(dentry);
 out:
-        return;
+    return;
 }
 
-static inline inode_t*
-__quota_inode_parent (inode_t *inode, uuid_t pargfid, const char *name)
+void
+__quota_dentry_del(quota_inode_ctx_t *ctx, const char *name, uuid_t par)
 {
-        inode_t *parent            = NULL;
+    quota_dentry_t *dentry = NULL;
+    quota_dentry_t *tmp = NULL;
 
-        parent = inode_parent (inode, pargfid, name);
-        inode_unref (inode);
-        return parent;
+    list_for_each_entry_safe(dentry, tmp, &ctx->parents, next)
+    {
+        if ((strcmp(dentry->name, name) == 0) &&
+            (gf_uuid_compare(dentry->par, par) == 0)) {
+            __quota_dentry_free(dentry);
+            break;
+        }
+    }
 }
 
-static inline inode_t*
-quota_inode_parent (inode_t *inode, uuid_t pargfid, const char *name)
+void
+quota_dentry_del(quota_inode_ctx_t *ctx, const char *name, uuid_t par)
 {
-        inode_t *parent            = NULL;
+    LOCK(&ctx->lock);
+    {
+        __quota_dentry_del(ctx, name, par);
+    }
+    UNLOCK(&ctx->lock);
+}
 
-        parent = __quota_inode_parent (inode, pargfid, name);
-        if (!parent)
-                gf_log_callingfn (THIS->name, GF_LOG_ERROR, "Failed to find "
-                                  "ancestor for inode (%s)",
-                                  uuid_utoa(inode->gfid));
+static inode_t *
+__quota_inode_parent(inode_t *inode, uuid_t pargfid, const char *name)
+{
+    inode_t *parent = NULL;
 
-        return parent;
+    parent = inode_parent(inode, pargfid, name);
+    inode_unref(inode);
+    return parent;
 }
 
-int32_t
-quota_inode_depth (inode_t *inode)
+static inode_t *
+quota_inode_parent(inode_t *inode, uuid_t pargfid, const char *name)
 {
-        int      depth             = 0;
-        inode_t *cur_inode         = NULL;
-
-        cur_inode = inode_ref (inode);
-        while (cur_inode && !__is_root_gfid (cur_inode->gfid)) {
-                depth++;
-                cur_inode = quota_inode_parent (cur_inode, 0 , NULL);
-                if (!cur_inode)
-                        depth = -1;
-        }
+    inode_t *parent = NULL;
 
-        if (cur_inode)
-                inode_unref (cur_inode);
+    parent = __quota_inode_parent(inode, pargfid, name);
+    if (!parent)
+        gf_msg_callingfn(THIS->name, GF_LOG_ERROR, 0, Q_MSG_PARENT_NULL,
+                         "Failed to find "
+                         "ancestor for inode (%s)",
+                         uuid_utoa(inode->gfid));
 
-        return depth;
+    return parent;
 }
 
-int32_t quota_find_common_ancestor (inode_t *inode1, inode_t *inode2,
-                                    uuid_t *common_ancestor)
+int32_t
+quota_inode_depth(inode_t *inode)
 {
-        int32_t         depth1         = 0;
-        int32_t         depth2         = 0;
-        int32_t         ret            = -1;
-        inode_t        *cur_inode1     = NULL;
-        inode_t        *cur_inode2     = NULL;
-
-        depth1 = quota_inode_depth (inode1);
-        if (depth1 < 0)
-                goto out;
-
-        depth2 = quota_inode_depth (inode2);
-        if (depth2 < 0)
-                goto out;
-
-        cur_inode1 = inode_ref (inode1);
-        cur_inode2 = inode_ref (inode2);
+    int depth = 0;
+    inode_t *cur_inode = NULL;
 
-        while (cur_inode1 && depth1 > depth2) {
-                cur_inode1 = quota_inode_parent (cur_inode1, 0 , NULL);
-                depth1--;
-        }
+    cur_inode = inode_ref(inode);
+    while (cur_inode && !__is_root_gfid(cur_inode->gfid)) {
+        depth++;
+        cur_inode = quota_inode_parent(cur_inode, 0, NULL);
+        if (!cur_inode)
+            depth = -1;
+    }
 
-        while (cur_inode2 && depth2 > depth1) {
-                cur_inode2 = quota_inode_parent (cur_inode2, 0 , NULL);
-                depth2--;
-        }
+    if (cur_inode)
+        inode_unref(cur_inode);
 
-        while (depth1 && cur_inode1 && cur_inode2 && cur_inode1 != cur_inode2) {
-                cur_inode1 = quota_inode_parent (cur_inode1, 0 , NULL);
-                cur_inode2 = quota_inode_parent (cur_inode2, 0 , NULL);
-                depth1--;
-        }
+    return depth;
+}
 
-        if (cur_inode1 && cur_inode2) {
-                gf_uuid_copy (*common_ancestor, cur_inode1->gfid);
-                ret = 0;
-        }
+int32_t
+quota_find_common_ancestor(inode_t *inode1, inode_t *inode2,
+                           uuid_t *common_ancestor)
+{
+    int32_t depth1 = 0;
+    int32_t depth2 = 0;
+    int32_t ret = -1;
+    inode_t *cur_inode1 = NULL;
+    inode_t *cur_inode2 = NULL;
+
+    depth1 = quota_inode_depth(inode1);
+    if (depth1 < 0)
+        goto out;
+
+    depth2 = quota_inode_depth(inode2);
+    if (depth2 < 0)
+        goto out;
+
+    cur_inode1 = inode_ref(inode1);
+    cur_inode2 = inode_ref(inode2);
+
+    while (cur_inode1 && depth1 > depth2) {
+        cur_inode1 = quota_inode_parent(cur_inode1, 0, NULL);
+        depth1--;
+    }
+
+    while (cur_inode2 && depth2 > depth1) {
+        cur_inode2 = quota_inode_parent(cur_inode2, 0, NULL);
+        depth2--;
+    }
+
+    while (depth1 && cur_inode1 && cur_inode2 && cur_inode1 != cur_inode2) {
+        cur_inode1 = quota_inode_parent(cur_inode1, 0, NULL);
+        cur_inode2 = quota_inode_parent(cur_inode2, 0, NULL);
+        depth1--;
+    }
+
+    if (cur_inode1 && cur_inode2) {
+        gf_uuid_copy(*common_ancestor, cur_inode1->gfid);
+        ret = 0;
+    }
 out:
-        if (cur_inode1)
-                inode_unref (cur_inode1);
+    if (cur_inode1)
+        inode_unref(cur_inode1);
 
-        if (cur_inode2)
-                inode_unref (cur_inode2);
+    if (cur_inode2)
+        inode_unref(cur_inode2);
 
-        return ret;
- }
+    return ret;
+}
 
 void
-check_ancestory_continue (struct list_head *parents, inode_t *inode,
-                          int32_t op_ret, int32_t op_errno, void *data)
+check_ancestory_continue(struct list_head *parents, inode_t *inode,
+                         int32_t op_ret, int32_t op_errno, void *data)
 {
-        call_frame_t   *frame        = NULL;
-        quota_local_t  *local        = NULL;
-        uint32_t        link_count   = 0;
-
-        frame = data;
-        local = frame->local;
-
-        if (parents && list_empty (parents)) {
-                gf_log (THIS->name, GF_LOG_WARNING,
-                        "Couldn't build ancestry for inode (gfid:%s). "
-                        "Without knowing ancestors till root, quota "
-                        "cannot be enforced. "
-                        "Hence, failing fop with EIO",
-                        uuid_utoa (inode->gfid));
-                op_errno = EIO;
-                op_ret = -1;
-        }
-
-        LOCK (&local->lock);
-        {
-                link_count = --local->link_count;
-                if (op_ret < 0) {
-                        local->op_ret = op_ret;
-                        local->op_errno = op_errno;
-                }
+    call_frame_t *frame = NULL;
+    quota_local_t *local = NULL;
+    uint32_t link_count = 0;
+
+    frame = data;
+    local = frame->local;
+
+    if (parents && list_empty(parents)) {
+        gf_msg(THIS->name, GF_LOG_WARNING, EIO, Q_MSG_ANCESTRY_BUILD_FAILED,
+               "Couldn't build ancestry for inode (gfid:%s). "
+               "Without knowing ancestors till root, quota "
+               "cannot be enforced. "
+               "Hence, failing fop with EIO",
+               uuid_utoa(inode->gfid));
+        op_errno = EIO;
+        op_ret = -1;
+    }
+
+    LOCK(&local->lock);
+    {
+        link_count = --local->link_count;
+        if (op_ret < 0) {
+            local->op_ret = op_ret;
+            local->op_errno = op_errno;
         }
-        UNLOCK (&local->lock);
+    }
+    UNLOCK(&local->lock);
 
-        if (link_count == 0)
-                local->fop_continue_cbk (frame);
+    if (link_count == 0)
+        local->fop_continue_cbk(frame);
 }
 
 void
-check_ancestory (call_frame_t *frame, inode_t *inode)
+check_ancestory(call_frame_t *frame, inode_t *inode)
 {
-        inode_t *cur_inode = NULL;
-        inode_t *parent    = NULL;
-
-        cur_inode = inode_ref (inode);
-        while (cur_inode && !__is_root_gfid (cur_inode->gfid)) {
-                parent = inode_parent (cur_inode, 0, NULL);
-                if (!parent) {
-                        quota_build_ancestry (cur_inode,
-                                              check_ancestory_continue, frame);
-                        return;
-                }
-                inode_unref (cur_inode);
-                cur_inode = parent;
-        }
+    inode_t *cur_inode = NULL;
+    inode_t *parent = NULL;
 
-        if (cur_inode) {
-                inode_unref (cur_inode);
-                check_ancestory_continue (NULL, NULL, 0, 0, frame);
-        } else {
-                check_ancestory_continue (NULL, NULL, -1, ESTALE, frame);
-        }
+    cur_inode = inode_ref(inode);
+    while (cur_inode && !__is_root_gfid(cur_inode->gfid)) {
+        parent = inode_parent(cur_inode, 0, NULL);
+        if (!parent) {
+            quota_build_ancestry(cur_inode, check_ancestory_continue, frame);
+            inode_unref(cur_inode);
+            return;
+        }
+        inode_unref(cur_inode);
+        cur_inode = parent;
+    }
+
+    if (cur_inode) {
+        inode_unref(cur_inode);
+        check_ancestory_continue(NULL, NULL, 0, 0, frame);
+    } else {
+        check_ancestory_continue(NULL, NULL, -1, ESTALE, frame);
+    }
 }
 
 void
-check_ancestory_2_cbk (struct list_head *parents, inode_t *inode,
-                       int32_t op_ret, int32_t op_errno, void *data)
+check_ancestory_2_cbk(struct list_head *parents, inode_t *inode, int32_t op_ret,
+                      int32_t op_errno, void *data)
 {
-        inode_t            *this_inode  = NULL;
-        quota_inode_ctx_t  *ctx         = NULL;
+    inode_t *this_inode = NULL;
+    quota_inode_ctx_t *ctx = NULL;
 
-        this_inode = data;
+    this_inode = data;
 
-        if (op_ret < 0)
-                goto out;
+    if (op_ret < 0)
+        goto out;
 
-        if (parents == NULL || list_empty (parents)) {
-                gf_log (THIS->name, GF_LOG_WARNING,
-                        "Couldn't build ancestry for inode (gfid:%s). "
-                        "Without knowing ancestors till root, quota "
-                        "cannot be enforced.",
-                        uuid_utoa (this_inode->gfid));
-                goto out;
-        }
+    if (parents == NULL || list_empty(parents)) {
+        gf_msg(THIS->name, GF_LOG_WARNING, 0, Q_MSG_ENFORCEMENT_FAILED,
+               "Couldn't build ancestry for inode (gfid:%s). "
+               "Without knowing ancestors till root, quota "
+               "cannot be enforced.",
+               uuid_utoa(this_inode->gfid));
+        goto out;
+    }
 
-        quota_inode_ctx_get (this_inode, THIS, &ctx, 0);
-        if (ctx)
-                ctx->ancestry_built = _gf_true;
+    quota_inode_ctx_get(this_inode, THIS, &ctx, 0);
+    if (ctx)
+        ctx->ancestry_built = _gf_true;
 
 out:
-        inode_unref (this_inode);
+    inode_unref(this_inode);
 }
 
 void
-check_ancestory_2 (xlator_t *this, quota_local_t *local, inode_t *inode)
+check_ancestory_2(xlator_t *this, quota_local_t *local, inode_t *inode)
 {
-        inode_t            *cur_inode    = NULL;
-        inode_t            *parent       = NULL;
-        quota_inode_ctx_t  *ctx          = NULL;
-        char               *name         = NULL;
-        uuid_t              pgfid        = {0};
-
-        name = (char *) local->loc.name;
-        if (local->loc.parent) {
-                gf_uuid_copy (pgfid, local->loc.parent->gfid);
-                parent = local->loc.parent;
-        }
-
-        cur_inode = inode_ref (inode);
-        while (cur_inode && !__is_root_gfid (cur_inode->gfid)) {
-                quota_inode_ctx_get (cur_inode, this, &ctx, 0);
-                /* build ancestry is required only on the first lookup,
-                 * so stop crawling when the inode_ctx is set for an inode
-                 */
-                if (ctx && ctx->ancestry_built)
-                        goto setctx;
-
-                parent = inode_parent (cur_inode, pgfid, name);
-                if (!parent) {
-                        quota_build_ancestry (cur_inode, check_ancestory_2_cbk,
-                                              inode_ref (inode));
-                        goto out;
-                }
+    inode_t *cur_inode = NULL;
+    inode_t *parent = NULL;
+    quota_inode_ctx_t *ctx = NULL;
+    char *name = NULL;
+    uuid_t pgfid = {0};
+
+    name = (char *)local->loc.name;
+    if (local->loc.parent) {
+        gf_uuid_copy(pgfid, local->loc.parent->gfid);
+    }
+
+    cur_inode = inode_ref(inode);
+    while (cur_inode && !__is_root_gfid(cur_inode->gfid)) {
+        quota_inode_ctx_get(cur_inode, this, &ctx, 0);
+        /* build ancestry is required only on the first lookup,
+         * so stop crawling when the inode_ctx is set for an inode
+         */
+        if (ctx && ctx->ancestry_built)
+            goto setctx;
 
-                if (name != NULL) {
-                        name = NULL;
-                        gf_uuid_clear (pgfid);
-                }
+        parent = inode_parent(cur_inode, pgfid, name);
+        if (!parent) {
+            quota_build_ancestry(cur_inode, check_ancestory_2_cbk,
+                                 inode_ref(inode));
+            goto out;
+        }
 
-                inode_unref (cur_inode);
-                cur_inode = parent;
+        if (name != NULL) {
+            name = NULL;
+            gf_uuid_clear(pgfid);
         }
 
+        inode_unref(cur_inode);
+        cur_inode = parent;
+    }
+
 setctx:
-        if (cur_inode && cur_inode != inode) {
-                quota_inode_ctx_get (inode, this, &ctx, 0);
-                if (ctx)
-                        ctx->ancestry_built = _gf_true;
-        }
+    if (cur_inode && cur_inode != inode) {
+        quota_inode_ctx_get(inode, this, &ctx, 0);
+        if (ctx)
+            ctx->ancestry_built = _gf_true;
+    }
 out:
-        if (cur_inode)
-                inode_unref (cur_inode);
+    if (cur_inode)
+        inode_unref(cur_inode);
 }
 
-static inline void
-quota_link_count_decrement (quota_local_t *local)
+static void
+quota_link_count_decrement(call_frame_t *frame)
 {
-        call_stub_t *stub       = NULL;
-        int          link_count = -1;
+    call_frame_t *tmpframe = NULL;
+    quota_local_t *local = NULL;
+    call_stub_t *stub = NULL;
+    int link_count = -1;
+
+    local = frame->local;
+    if (local && local->par_frame) {
+        local = local->par_frame->local;
+        tmpframe = frame;
+    }
+
+    if (local == NULL)
+        goto out;
+
+    LOCK(&local->lock);
+    {
+        link_count = --local->link_count;
+        if (link_count == 0) {
+            stub = local->stub;
+            local->stub = NULL;
+        }
+    }
+    UNLOCK(&local->lock);
+
+    if (stub != NULL) {
+        call_resume(stub);
+    }
 
-        if (local == NULL)
-                goto out;
+out:
+    if (tmpframe) {
+        local = tmpframe->local;
+        tmpframe->local = NULL;
 
-        LOCK (&local->lock);
-        {
-                link_count = --local->link_count;
-                if (link_count == 0) {
-                        stub = local->stub;
-                        local->stub = NULL;
-                }
-        }
-        UNLOCK (&local->lock);
+        STACK_DESTROY(frame->root);
+        if (local)
+            quota_local_cleanup(local);
+    }
 
-        if (stub != NULL) {
-                call_resume (stub);
-        }
-out:
-        return;
+    return;
 }
 
-static inline void
-quota_handle_validate_error (quota_local_t *local, int32_t op_ret,
-                             int32_t op_errno)
+static void
+quota_handle_validate_error(call_frame_t *frame, int32_t op_ret,
+                            int32_t op_errno)
 {
-        if (local == NULL)
-                goto out;
+    quota_local_t *local;
+
+    local = frame->local;
+    if (local && local->par_frame)
+        local = local->par_frame->local;
+
+    if (local == NULL)
+        goto out;
 
-        LOCK (&local->lock);
+    if (op_ret < 0) {
+        LOCK(&local->lock);
         {
-                if (op_ret < 0) {
-                        local->op_ret = op_ret;
-                        local->op_errno = op_errno;
-                }
+            local->op_ret = op_ret;
+            local->op_errno = op_errno;
         }
-        UNLOCK (&local->lock);
-
-        /* we abort checking limits on this path to root */
-        quota_link_count_decrement (local);
+        UNLOCK(&local->lock);
+    }
+    /* we abort checking limits on this path to root */
+    quota_link_count_decrement(frame);
 out:
-        return;
+    return;
 }
 
 int32_t
-quota_validate_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                    int32_t op_ret, int32_t op_errno, inode_t *inode,
-                    struct iatt *buf, dict_t *xdata, struct iatt *postparent)
+quota_validate_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                   int32_t op_ret, int32_t op_errno, inode_t *inode,
+                   struct iatt *buf, dict_t *xdata, struct iatt *postparent)
 {
-        quota_local_t     *local      = NULL;
-        int32_t            ret        = 0;
-        quota_inode_ctx_t *ctx        = NULL;
-        int64_t           *object_size = 0;
-        uint64_t           value      = 0;
-        data_t            *data       = NULL;
-        quota_meta_t       size       = {0,};
-
-        local = frame->local;
-
-        if (op_ret < 0) {
-                goto unwind;
-        }
-
-        GF_ASSERT (local);
-        GF_ASSERT (frame);
-        GF_VALIDATE_OR_GOTO_WITH_ERROR ("quota", this, unwind, op_errno,
-                                        EINVAL);
-        GF_VALIDATE_OR_GOTO_WITH_ERROR (this->name, xdata, unwind, op_errno,
-                                        EINVAL);
-
-        ret = inode_ctx_get (local->validate_loc.inode, this, &value);
-
-        ctx = (quota_inode_ctx_t *)(unsigned long)value;
-        if ((ret == -1) || (ctx == NULL)) {
-                gf_log (this->name, GF_LOG_WARNING,
-                        "quota context is not present in inode (gfid:%s)",
-                        uuid_utoa (local->validate_loc.inode->gfid));
-                op_errno = EINVAL;
-                goto unwind;
-        }
-
-        ret = quota_dict_get_meta (xdata, QUOTA_SIZE_KEY, &size);
-        if (ret == -1) {
-                gf_log (this->name, GF_LOG_WARNING, "dict get failed "
-                        "on quota size");
-                op_errno = EINVAL;
-        }
-
-        local->just_validated = 1; /* so that we don't go into infinite
-                                    * loop of validation and checking
-                                    * limit when timeout is zero.
-                                    */
-        LOCK (&ctx->lock);
-        {
-                ctx->size = size.size;
-                ctx->file_count = size.file_count;
-                ctx->dir_count = size.dir_count;
-                gettimeofday (&ctx->tv, NULL);
-        }
-        UNLOCK (&ctx->lock);
-
-        quota_check_limit (frame, local->validate_loc.inode, this, NULL, NULL);
-        return 0;
+    quota_local_t *local = NULL;
+    int32_t ret = 0;
+    quota_inode_ctx_t *ctx = NULL;
+    uint64_t value = 0;
+    quota_meta_t size = {
+        0,
+    };
+
+    local = frame->local;
+
+    if (op_ret < 0) {
+        goto unwind;
+    }
+
+    GF_ASSERT(local);
+    GF_ASSERT(frame);
+    GF_VALIDATE_OR_GOTO_WITH_ERROR("quota", this, unwind, op_errno, EINVAL);
+    GF_VALIDATE_OR_GOTO_WITH_ERROR(this->name, xdata, unwind, op_errno, EINVAL);
+
+    ret = inode_ctx_get(local->validate_loc.inode, this, &value);
+
+    ctx = (quota_inode_ctx_t *)(unsigned long)value;
+    if ((ret == -1) || (ctx == NULL)) {
+        gf_msg(this->name, GF_LOG_WARNING, EINVAL, Q_MSG_INODE_CTX_GET_FAILED,
+               "quota context is"
+               " not present in  inode (gfid:%s)",
+               uuid_utoa(local->validate_loc.inode->gfid));
+        op_errno = EINVAL;
+        goto unwind;
+    }
+
+    ret = quota_dict_get_meta(xdata, QUOTA_SIZE_KEY, SLEN(QUOTA_SIZE_KEY),
+                              &size);
+    if (ret == -1) {
+        gf_msg(this->name, GF_LOG_WARNING, EINVAL, Q_MSG_SIZE_KEY_MISSING,
+               "quota size key not present "
+               "in dict");
+        op_errno = EINVAL;
+    }
+
+    local->just_validated = 1; /* so that we don't go into infinite
+                                * loop of validation and checking
+                                * limit when timeout is zero.
+                                */
+    LOCK(&ctx->lock);
+    {
+        ctx->size = size.size;
+        ctx->validate_time = gf_time();
+        ctx->file_count = size.file_count;
+        ctx->dir_count = size.dir_count;
+    }
+    UNLOCK(&ctx->lock);
+
+    quota_check_limit(frame, local->validate_loc.inode, this);
+    return 0;
 
 unwind:
-        quota_handle_validate_error (local, op_ret, op_errno);
-        return 0;
+    quota_handle_validate_error(frame, op_ret, op_errno);
+    return 0;
 }
 
-
-static inline uint64_t
-quota_time_elapsed (struct timeval *now, struct timeval *then)
+static inline gf_boolean_t
+quota_timeout(time_t t, uint32_t timeout)
 {
-        return (now->tv_sec - then->tv_sec);
+    return (gf_time() - t) >= timeout;
 }
 
-
-int32_t
-quota_timeout (struct timeval *tv, int32_t timeout)
+/* Return: 1 if new entry added
+ *         0 no entry added
+ *        -1 on errors
+ */
+static int32_t
+quota_add_parent(struct list_head *list, char *name, uuid_t pgfid)
 {
-        struct timeval now       = {0,};
-        int32_t        timed_out = 0;
-
-        gettimeofday (&now, NULL);
+    quota_dentry_t *entry = NULL;
+    gf_boolean_t found = _gf_false;
+    int ret = 0;
 
-        if (quota_time_elapsed (&now, tv) >= timeout) {
-                timed_out = 1;
+    if (!list_empty(list)) {
+        list_for_each_entry(entry, list, next)
+        {
+            if (gf_uuid_compare(pgfid, entry->par) == 0) {
+                found = _gf_true;
+                goto out;
+            }
         }
+    }
+
+    entry = __quota_dentry_new(NULL, name, pgfid);
+    if (entry)
+        list_add_tail(&entry->next, list);
+    else
+        ret = -1;
 
-        return timed_out;
+out:
+    if (found)
+        return 0;
+    else if (ret == 0)
+        return 1;
+    else
+        return -1;
 }
 
-static inline void
-quota_add_parent (quota_dentry_t *dentry, struct list_head *list)
+/* This function iterates the parent list in inode
+ * context and add unique parent to the list
+ * Returns number of dentry added to the list, or -1 on errors
+ */
+static int32_t
+quota_add_parents_from_ctx(quota_inode_ctx_t *ctx, struct list_head *list)
 {
-        quota_dentry_t *entry = NULL;
-        gf_boolean_t    found = _gf_false;
+    int ret = 0;
+    quota_dentry_t *dentry = NULL;
+    int32_t count = 0;
 
-        if ((dentry == NULL) || (list == NULL)) {
-                goto out;
-        }
+    if (ctx == NULL || list == NULL)
+        goto out;
 
-        list_for_each_entry (entry, list, next) {
-                if (gf_uuid_compare (dentry->par, entry->par) == 0) {
-                        found = _gf_true;
-                        goto out;
-                }
+    LOCK(&ctx->lock);
+    {
+        list_for_each_entry(dentry, &ctx->parents, next)
+        {
+            ret = quota_add_parent(list, dentry->name, dentry->par);
+            if (ret == 1)
+                count++;
+            else if (ret == -1)
+                break;
         }
-
-        list_add_tail (&dentry->next, list);
+    }
+    UNLOCK(&ctx->lock);
 
 out:
-        return;
+    return (ret == -1) ? -1 : count;
 }
 
 int32_t
-quota_build_ancestry_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                          int32_t op_ret, int32_t op_errno,
-                          gf_dirent_t *entries, dict_t *xdata)
+quota_build_ancestry_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                         int32_t op_ret, int32_t op_errno, gf_dirent_t *entries,
+                         dict_t *xdata)
 {
-        inode_t           *parent  = NULL, *tmp_parent = NULL;
-        gf_dirent_t       *entry   = NULL;
-        loc_t              loc     = {0, };
-        quota_dentry_t    *dentry  = NULL, *tmp = NULL;
-        quota_inode_ctx_t *ctx     = NULL;
-        struct list_head   parents = {0, };
-        quota_local_t     *local   = NULL;
-
-        INIT_LIST_HEAD (&parents);
-
-        local = frame->local;
-        frame->local = NULL;
-
-        if (op_ret < 0)
-                goto err;
-
-        parent = inode_parent (local->loc.inode, 0, NULL);
-        if (parent == NULL) {
-                gf_log (this->name, GF_LOG_WARNING, "parent is NULL");
-                op_errno = EINVAL;
-                goto err;
-        }
-
-        if ((op_ret > 0) && (entries != NULL)) {
-                list_for_each_entry (entry, &entries->list, list) {
-                        if (__is_root_gfid (entry->inode->gfid)) {
-                                /* The list contains a sub-list for each
-                                 * possible path to the target inode. Each
-                                 * sub-list starts with the root entry of the
-                                 * tree and is followed by the child entries
-                                 * for a particular path to the target entry.
-                                 * The root entry is an implied sub-list
-                                 * delimiter, as it denotes we have started
-                                 * processing a new path. Reset the parent
-                                 * pointer and continue
-                                 */
-
-                                tmp_parent = NULL;
-                        }
-
-                        gf_uuid_copy (loc.gfid, entry->d_stat.ia_gfid);
-
-                        loc.inode = inode_ref (entry->inode);
-                        loc.parent = inode_ref (tmp_parent);
-                        loc.name = entry->d_name;
-
-                        quota_fill_inodectx (this, entry->inode, entry->dict,
-                                             &loc, &entry->d_stat, &op_errno);
-
-                        tmp_parent = entry->inode;
-
-                        loc_wipe (&loc);
-                }
-        }
+    inode_t *parent = NULL;
+    inode_t *tmp_parent = NULL;
+    inode_t *linked_inode = NULL;
+    inode_t *tmp_inode = NULL;
+    gf_dirent_t *entry = NULL;
+    loc_t loc = {
+        0,
+    };
+    quota_dentry_t *dentry = NULL;
+    quota_dentry_t *tmp = NULL;
+    quota_inode_ctx_t *ctx = NULL;
+    struct list_head parents;
+    quota_local_t *local = NULL;
+    int ret;
+
+    INIT_LIST_HEAD(&parents);
+
+    local = frame->local;
+    frame->local = NULL;
+
+    if (op_ret < 0)
+        goto err;
+
+    if ((op_ret > 0) && (entries != NULL)) {
+        list_for_each_entry(entry, &entries->list, list)
+        {
+            if (__is_root_gfid(entry->inode->gfid)) {
+                /* The list contains a sub-list for each
+                 * possible path to the target inode. Each
+                 * sub-list starts with the root entry of the
+                 * tree and is followed by the child entries
+                 * for a particular path to the target entry.
+                 * The root entry is an implied sub-list
+                 * delimiter, as it denotes we have started
+                 * processing a new path. Reset the parent
+                 * pointer and continue
+                 */
 
-        quota_inode_ctx_get (local->loc.inode, this, &ctx, 0);
-
-        if (ctx != NULL) {
-                LOCK (&ctx->lock);
-                {
-                        list_for_each_entry (dentry, &ctx->parents, next) {
-                                /* we built ancestry for a non-directory */
-                                tmp = __quota_dentry_new (NULL, dentry->name,
-                                                          dentry->par);
-                                quota_add_parent (tmp, &parents);
-
-                                if (list_empty (&tmp->next)) {
-                                        __quota_dentry_free (tmp);
-                                        tmp = NULL;
-                                }
-                        }
+                tmp_parent = NULL;
+            } else {
+                /* For a non-root entry, link this inode */
+                linked_inode = inode_link(entry->inode, tmp_parent,
+                                          entry->d_name, &entry->d_stat);
+                if (linked_inode) {
+                    tmp_inode = entry->inode;
+                    entry->inode = linked_inode;
+                    inode_unref(tmp_inode);
+                } else {
+                    gf_msg(this->name, GF_LOG_WARNING, EINVAL,
+                           Q_MSG_PARENT_NULL, "inode link failed");
+                    op_errno = EINVAL;
+                    goto err;
                 }
-                UNLOCK (&ctx->lock);
+            }
+
+            gf_uuid_copy(loc.gfid, entry->d_stat.ia_gfid);
+
+            loc.inode = inode_ref(entry->inode);
+            loc.parent = inode_ref(tmp_parent);
+            loc.name = entry->d_name;
+
+            quota_fill_inodectx(this, entry->inode, entry->dict, &loc,
+                                &entry->d_stat, &op_errno);
+
+            /* For non-directory, posix_get_ancestry_non_directory
+             * returns all hard-links that are represented by nodes
+             * adjacent to each other in the dentry-list.
+             * (Unlike the directory case where adjacent nodes
+             * either have a parent/child relationship or belong to
+             * different paths).
+             */
+            if (entry->inode->ia_type == IA_IFDIR)
+                tmp_parent = entry->inode;
+
+            loc_wipe(&loc);
+        }
+    }
+
+    parent = inode_parent(local->loc.inode, 0, NULL);
+    if (parent == NULL) {
+        gf_msg(this->name, GF_LOG_WARNING, EINVAL, Q_MSG_PARENT_NULL,
+               "parent is NULL");
+        op_errno = EINVAL;
+        goto err;
+    }
+
+    quota_inode_ctx_get(local->loc.inode, this, &ctx, 0);
+
+    ret = quota_add_parents_from_ctx(ctx, &parents);
+    if (ret == -1) {
+        op_errno = errno;
+        goto err;
+    }
+
+    if (list_empty(&parents)) {
+        /* we built ancestry for a directory */
+        list_for_each_entry(entry, &entries->list, list)
+        {
+            if (entry->inode == local->loc.inode)
+                break;
         }
 
-        if (list_empty (&parents)) {
-                /* we built ancestry for a directory */
-                list_for_each_entry (entry, &entries->list, list) {
-                        if (entry->inode == local->loc.inode)
-                                break;
-                }
+        /* Getting assertion here, need to investigate
+           comment for now
+           GF_ASSERT (&entry->list != &entries->list);
+        */
 
-                GF_ASSERT (&entry->list != &entries->list);
-
-                tmp = __quota_dentry_new (NULL, entry->d_name, parent->gfid);
-                quota_add_parent (tmp, &parents);
+        ret = quota_add_parent(&parents, entry->d_name, parent->gfid);
+        if (ret == -1) {
+            op_errno = errno;
+            goto err;
         }
+    }
 
-        local->ancestry_cbk (&parents, local->loc.inode, 0, 0,
-                             local->ancestry_data);
-        goto cleanup;
+    local->ancestry_cbk(&parents, local->loc.inode, 0, 0, local->ancestry_data);
+    goto cleanup;
 
 err:
-        local->ancestry_cbk (NULL, NULL, -1, op_errno, local->ancestry_data);
+    local->ancestry_cbk(NULL, NULL, -1, op_errno, local->ancestry_data);
 
 cleanup:
-        STACK_DESTROY (frame->root);
-        quota_local_cleanup (this, local);
+    STACK_DESTROY(frame->root);
+    quota_local_cleanup(local);
 
-        if (parent != NULL) {
-                inode_unref (parent);
-                parent = NULL;
-        }
+    if (parent != NULL) {
+        inode_unref(parent);
+        parent = NULL;
+    }
 
-        list_for_each_entry_safe (dentry, tmp, &parents, next) {
-                __quota_dentry_free (dentry);
+    if (!list_empty(&parents)) {
+        list_for_each_entry_safe(dentry, tmp, &parents, next)
+        {
+            __quota_dentry_free(dentry);
         }
+    }
 
-        return 0;
+    return 0;
 }
 
 int
-quota_build_ancestry (inode_t *inode, quota_ancestry_built_t ancestry_cbk,
-                      void *data)
+quota_build_ancestry(inode_t *inode, quota_ancestry_built_t ancestry_cbk,
+                     void *data)
 {
-        fd_t          *fd        = NULL;
-        quota_local_t *local     = NULL;
-        call_frame_t  *new_frame = NULL;
-        int            op_errno  = ENOMEM;
-        int            op_ret    = -1;
-        xlator_t      *this      = NULL;
-        dict_t        *xdata_req = NULL;
-
-        this = THIS;
-
-        xdata_req = dict_new ();
-        if (xdata_req == NULL)
-                goto err;
-
-        fd = fd_anonymous (inode);
-        if (fd == NULL)
-                goto err;
-
-        new_frame = create_frame (this, this->ctx->pool);
-        if (new_frame == NULL)
-                goto err;
-
-        local = quota_local_new ();
-        if (local == NULL)
-                goto err;
-
-        new_frame->root->uid = new_frame->root->gid = 0;
-        new_frame->local = local;
-        local->ancestry_cbk = ancestry_cbk;
-        local->ancestry_data = data;
-        local->loc.inode = inode_ref (inode);
-
-        op_ret = dict_set_int8 (xdata_req, QUOTA_LIMIT_KEY, 1);
-        if (op_ret < 0) {
-                op_errno = -op_ret;
-                goto err;
-        }
-
-        op_ret = dict_set_int8 (xdata_req, GET_ANCESTRY_DENTRY_KEY, 1);
-        if (op_ret < 0) {
-                op_errno = -op_ret;
-                goto err;
-        }
-
-        /* This would ask posix layer to construct dentry chain till root
-         * We don't need to do a opendir, we can use the anonymous fd
-         * here for  the readidrp.
-         * avoiding opendir also reduces the window size where another FOP
-         * can be executed before completion of build ancestry
-         */
-        STACK_WIND (new_frame, quota_build_ancestry_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->readdirp, fd, 0, 0, xdata_req);
-
-        op_ret = 0;
+    fd_t *fd = NULL;
+    quota_local_t *local = NULL;
+    call_frame_t *new_frame = NULL;
+    int op_errno = ENOMEM;
+    int op_ret = -1;
+    xlator_t *this = NULL;
+    dict_t *xdata_req = NULL;
+
+    this = THIS;
+
+    xdata_req = dict_new();
+    if (xdata_req == NULL)
+        goto err;
+
+    fd = fd_anonymous(inode);
+    if (fd == NULL)
+        goto err;
+
+    new_frame = create_frame(this, this->ctx->pool);
+    if (new_frame == NULL)
+        goto err;
+
+    local = quota_local_new();
+    if (local == NULL)
+        goto err;
+
+    new_frame->root->uid = new_frame->root->gid = 0;
+    new_frame->local = local;
+    local->ancestry_cbk = ancestry_cbk;
+    local->ancestry_data = data;
+    local->loc.inode = inode_ref(inode);
+
+    op_ret = dict_set_int8(xdata_req, QUOTA_LIMIT_KEY, 1);
+    if (op_ret < 0) {
+        op_errno = -op_ret;
+        goto err;
+    }
+
+    op_ret = dict_set_int8(xdata_req, QUOTA_LIMIT_OBJECTS_KEY, 1);
+    if (op_ret < 0) {
+        op_errno = -op_ret;
+        goto err;
+    }
+
+    op_ret = dict_set_int8(xdata_req, GET_ANCESTRY_DENTRY_KEY, 1);
+    if (op_ret < 0) {
+        op_errno = -op_ret;
+        goto err;
+    }
+
+    /* This would ask posix layer to construct dentry chain till root
+     * We don't need to do a opendir, we can use the anonymous fd
+     * here for  the readidrp.
+     * avoiding opendir also reduces the window size where another FOP
+     * can be executed before completion of build ancestry
+     */
+    STACK_WIND(new_frame, quota_build_ancestry_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->readdirp, fd, 0, 0, xdata_req);
+
+    op_ret = 0;
 
 err:
-        if (fd)
-                fd_unref (fd);
-
-        if (xdata_req)
-                dict_unref (xdata_req);
+    if (fd)
+        fd_unref(fd);
 
-        if (op_ret < 0) {
-                ancestry_cbk (NULL, NULL, -1, op_errno, data);
+    if (xdata_req)
+        dict_unref(xdata_req);
 
-                if (new_frame) {
-                        local = new_frame->local;
-                        new_frame->local = NULL;
-                        STACK_DESTROY (new_frame->root);
-                }
+    if (op_ret < 0) {
+        ancestry_cbk(NULL, NULL, -1, op_errno, data);
 
-                if (local)
-                        quota_local_cleanup (this, local);
+        if (new_frame) {
+            local = new_frame->local;
+            new_frame->local = NULL;
+            STACK_DESTROY(new_frame->root);
         }
 
-        return 0;
+        if (local)
+            quota_local_cleanup(local);
+    }
+
+    return 0;
 }
 
 int
-quota_validate (call_frame_t *frame, inode_t *inode, xlator_t *this,
-                fop_lookup_cbk_t cbk_fn)
+quota_validate(call_frame_t *frame, inode_t *inode, xlator_t *this,
+               fop_lookup_cbk_t cbk_fn)
 {
-        quota_local_t     *local = NULL;
-        int                ret   = 0;
-        dict_t            *xdata = NULL;
-        quota_priv_t      *priv  = NULL;
-
-        local = frame->local;
-        priv = this->private;
-
-        LOCK (&local->lock);
-        {
-                loc_wipe (&local->validate_loc);
-
-                ret = quota_inode_loc_fill (inode, &local->validate_loc);
-                if (ret < 0) {
-                        gf_log (this->name, GF_LOG_WARNING,
-                                "cannot fill loc for inode (gfid:%s), hence "
-                                "aborting quota-checks and continuing with fop",
-                                uuid_utoa (inode->gfid));
-                }
-        }
-        UNLOCK (&local->lock);
+    quota_local_t *local = NULL;
+    int ret = 0;
+    dict_t *xdata = NULL;
+    quota_priv_t *priv = NULL;
 
-        if (ret < 0) {
-                ret = -ENOMEM;
-                goto err;
-        }
+    local = frame->local;
+    priv = this->private;
 
-        xdata = dict_new ();
-        if (xdata == NULL) {
-                ret = -ENOMEM;
-                goto err;
-        }
+    LOCK(&local->lock);
+    {
+        loc_wipe(&local->validate_loc);
 
-        ret = dict_set_int8 (xdata, QUOTA_SIZE_KEY, 1);
+        ret = quota_inode_loc_fill(inode, &local->validate_loc);
         if (ret < 0) {
-                gf_log (this->name, GF_LOG_WARNING, "dict set failed");
-                ret = -ENOMEM;
-                goto err;
-        }
-
-        ret = dict_set_str (xdata, "volume-uuid", priv->volume_uuid);
-        if (ret < 0) {
-                gf_log (this->name, GF_LOG_WARNING, "dict set failed");
-                ret = -ENOMEM;
-                goto err;
-        }
-
-        ret = quota_enforcer_lookup (frame, this, &local->validate_loc, xdata,
-                                     cbk_fn);
-        if (ret < 0) {
-                ret = -ENOTCONN;
-                goto err;
-        }
-
-        ret = 0;
+            gf_msg(this->name, GF_LOG_WARNING, ENOMEM, Q_MSG_ENFORCEMENT_FAILED,
+                   "cannot fill loc for inode (gfid:%s), hence "
+                   "aborting quota-checks and continuing with fop",
+                   uuid_utoa(inode->gfid));
+        }
+    }
+    UNLOCK(&local->lock);
+
+    if (ret < 0) {
+        ret = -ENOMEM;
+        goto err;
+    }
+
+    xdata = dict_new();
+    if (xdata == NULL) {
+        ret = -ENOMEM;
+        goto err;
+    }
+
+    ret = dict_set_int8(xdata, QUOTA_SIZE_KEY, 1);
+    if (ret < 0) {
+        gf_msg(this->name, GF_LOG_WARNING, ENOMEM, Q_MSG_ENOMEM,
+               "dict set failed");
+        ret = -ENOMEM;
+        goto err;
+    }
+
+    ret = dict_set_str(xdata, "volume-uuid", priv->volume_uuid);
+    if (ret < 0) {
+        gf_msg(this->name, GF_LOG_WARNING, ENOMEM, Q_MSG_ENOMEM,
+               "dict set failed");
+        ret = -ENOMEM;
+        goto err;
+    }
+
+    ret = quota_enforcer_lookup(frame, this, xdata, cbk_fn);
+    if (ret < 0) {
+        ret = -ENOTCONN;
+        goto err;
+    }
+
+    ret = 0;
 err:
-        if (xdata)
-                dict_unref (xdata);
+    if (xdata)
+        dict_unref(xdata);
 
-        return ret;
+    return ret;
 }
 
 void
-quota_check_limit_continuation (struct list_head *parents, inode_t *inode,
-                                int32_t op_ret, int32_t op_errno, void *data)
+quota_check_limit_continuation(struct list_head *parents, inode_t *inode,
+                               int32_t op_ret, int32_t op_errno, void *data)
 {
-        call_frame_t   *frame        = NULL;
-        xlator_t       *this         = NULL;
-        quota_local_t  *local        = NULL;
-        quota_dentry_t *entry        = NULL;
-        inode_t        *parent       = NULL;
-        int             parent_count = 0;
-
-        frame = data;
-        local = frame->local;
-        this = THIS;
-
-        if ((op_ret < 0) || list_empty (parents)) {
-                if (op_ret >= 0) {
-                        gf_log (this->name, GF_LOG_WARNING,
-                                "Couldn't build ancestry for inode (gfid:%s). "
-                                "Without knowing ancestors till root, quota "
-                                "cannot be enforced. "
-                                "Hence, failing fop with EIO",
-                                uuid_utoa (inode->gfid));
-                        op_errno = EIO;
-                }
-
-                quota_handle_validate_error (local, -1, op_errno);
-                goto out;
-        }
-
-        list_for_each_entry (entry, parents, next) {
-                parent_count++;
-        }
-
-        LOCK (&local->lock);
+    call_frame_t *frame = NULL;
+    xlator_t *this = NULL;
+    quota_local_t *local = NULL;
+    quota_local_t *par_local = NULL;
+    quota_dentry_t *entry = NULL;
+    inode_t *parent = NULL;
+    int parent_count = 0;
+
+    frame = data;
+    local = frame->local;
+    this = THIS;
+
+    if (local->par_frame)
+        par_local = local->par_frame->local;
+    else
+        par_local = local;
+
+    if ((op_ret < 0) || list_empty(parents)) {
+        if (op_ret >= 0) {
+            gf_msg(this->name, GF_LOG_WARNING, EIO, Q_MSG_ANCESTRY_BUILD_FAILED,
+                   "Couldn't build ancestry for inode (gfid:%s). "
+                   "Without knowing ancestors till root, quota"
+                   "cannot be enforced. "
+                   "Hence, failing fop with EIO",
+                   uuid_utoa(inode->gfid));
+            op_errno = EIO;
+        }
+
+        quota_handle_validate_error(frame, -1, op_errno);
+        goto out;
+    }
+
+    list_for_each_entry(entry, parents, next) { parent_count++; }
+
+    LOCK(&par_local->lock);
+    {
+        par_local->link_count += (parent_count - 1);
+    }
+    UNLOCK(&par_local->lock);
+
+    if (local->par_frame) {
+        list_for_each_entry(entry, parents, next)
         {
-                local->link_count += (parent_count - 1);
+            parent = inode_find(inode->table, entry->par);
+            quota_check_limit(frame, parent, this);
+            inode_unref(parent);
         }
-        UNLOCK (&local->lock);
-
-        list_for_each_entry (entry, parents, next) {
-                parent = inode_find (inode->table, entry->par);
-
-                quota_check_limit (frame, parent, this, NULL, NULL);
+    } else {
+        list_for_each_entry(entry, parents, next)
+        {
+            parent = do_quota_check_limit(frame, inode, this, entry, _gf_true);
+            if (parent)
+                inode_unref(parent);
+            else
+                quota_link_count_decrement(frame);
         }
+    }
 
 out:
-        return;
+    return;
 }
 
 int32_t
-quota_check_object_limit (call_frame_t *frame, quota_inode_ctx_t *ctx,
-                          quota_priv_t *priv, inode_t *_inode, xlator_t *this,
-                          int32_t *op_errno, int just_validated,
-                          quota_local_t *local, gf_boolean_t *skip_check)
+quota_check_object_limit(call_frame_t *frame, quota_inode_ctx_t *ctx,
+                         quota_priv_t *priv, inode_t *_inode, xlator_t *this,
+                         int32_t *op_errno, int just_validated,
+                         quota_local_t *local, gf_boolean_t *skip_check)
 {
-        int32_t         ret                     = -1;
-        uint32_t        timeout                 =  0;
-        char            need_validate           =  0;
-        gf_boolean_t    hard_limit_exceeded     =  0;
-        int64_t         object_aggr_count       =  0;
-
-        GF_ASSERT (frame);
-        GF_ASSERT (priv);
-        GF_ASSERT (_inode);
-        GF_ASSERT (this);
-        GF_ASSERT (local);
-
-        if (ctx != NULL && (ctx->object_hard_lim > 0 ||
-                            ctx->object_soft_lim)) {
-                LOCK (&ctx->lock);
-                {
-                        timeout = priv->soft_timeout;
-
-                        object_aggr_count = ctx->file_count +
-                                            ctx->dir_count + 1;
-                        if (((ctx->object_soft_lim >= 0)
-                             && (object_aggr_count) >
-                             ctx->object_soft_lim)) {
-                                timeout = priv->hard_timeout;
-                        }
-
-                        if (!just_validated
-                            && quota_timeout (&ctx->tv, timeout)) {
-                                need_validate = 1;
-                        } else if ((object_aggr_count) >
-                                  ctx->object_hard_lim) {
-                                hard_limit_exceeded = 1;
-                        }
-                }
-                UNLOCK (&ctx->lock);
-
-                if (need_validate && *skip_check != _gf_true) {
-                        *skip_check = _gf_true;
-                        ret = quota_validate (frame, _inode, this,
-                                              quota_validate_cbk);
-                        if (ret < 0) {
-                                *op_errno = -ret;
-                                *skip_check = _gf_false;
-                        }
-                        goto out;
-                }
+    int32_t ret = -1;
+    uint32_t timeout = 0;
+    char need_validate = 0;
+    gf_boolean_t hard_limit_exceeded = 0;
+    int64_t object_aggr_count = 0;
+
+    GF_ASSERT(frame);
+    GF_ASSERT(priv);
+    GF_ASSERT(_inode);
+    GF_ASSERT(this);
+    GF_ASSERT(local);
+
+    if (ctx != NULL && (ctx->object_hard_lim > 0 || ctx->object_soft_lim)) {
+        LOCK(&ctx->lock);
+        {
+            timeout = priv->soft_timeout;
 
-                if (hard_limit_exceeded) {
-                        local->op_ret = -1;
-                        local->op_errno = EDQUOT;
-                        *op_errno = EDQUOT;
-                }
+            object_aggr_count = ctx->file_count + ctx->dir_count + 1;
+            if (((ctx->object_soft_lim >= 0) &&
+                 (object_aggr_count) > ctx->object_soft_lim)) {
+                timeout = priv->hard_timeout;
+            }
 
-                /*We log usage only if quota limit is configured on
-                   that inode
-                */
-                quota_log_usage (this, ctx, _inode, 0);
+            if (!just_validated && quota_timeout(ctx->validate_time, timeout)) {
+                need_validate = 1;
+            } else if ((object_aggr_count) > ctx->object_hard_lim) {
+                hard_limit_exceeded = 1;
+            }
         }
+        UNLOCK(&ctx->lock);
 
-        ret = 0;
+        if (need_validate && *skip_check != _gf_true) {
+            *skip_check = _gf_true;
+            ret = quota_validate(frame, _inode, this, quota_validate_cbk);
+            if (ret < 0) {
+                *op_errno = -ret;
+                *skip_check = _gf_false;
+            }
+            goto out;
+        }
+
+        if (hard_limit_exceeded) {
+            local->op_ret = -1;
+            local->op_errno = EDQUOT;
+            *op_errno = EDQUOT;
+            goto out;
+        }
+
+        /*We log usage only if quota limit is configured on
+           that inode
+        */
+        quota_log_usage(this, ctx, _inode, 0);
+    }
+
+    ret = 0;
 
 out:
-        return ret;
+    return ret;
 }
 
-
 int32_t
-quota_check_size_limit (call_frame_t *frame, quota_inode_ctx_t *ctx,
-                          quota_priv_t *priv, inode_t *_inode, xlator_t *this,
-                          int32_t *op_errno, int just_validated, int64_t delta,
-                          quota_local_t *local, gf_boolean_t *skip_check)
+quota_check_size_limit(call_frame_t *frame, quota_inode_ctx_t *ctx,
+                       quota_priv_t *priv, inode_t *_inode, xlator_t *this,
+                       int32_t *op_errno, int just_validated, int64_t delta,
+                       quota_local_t *local, gf_boolean_t *skip_check)
 {
-        int32_t         ret                     = -1;
-        uint32_t        timeout                 =  0;
-        char            need_validate           =  0;
-        gf_boolean_t    hard_limit_exceeded     =  0;
-        int64_t         space_available         =  0;
-        int64_t         wouldbe_size            =  0;
-
-        GF_ASSERT (frame);
-        GF_ASSERT (priv);
-        GF_ASSERT (_inode);
-        GF_ASSERT (this);
-        GF_ASSERT (local);
-
-        if (ctx != NULL && (ctx->hard_lim > 0 || ctx->soft_lim > 0)) {
-                wouldbe_size = ctx->size + delta;
-
-                LOCK (&ctx->lock);
-                {
-                        timeout = priv->soft_timeout;
-
-                        if ((ctx->soft_lim >= 0)
-                            && (wouldbe_size > ctx->soft_lim)) {
-                                timeout = priv->hard_timeout;
-                        }
-
-                        if (!just_validated
-                            && quota_timeout (&ctx->tv, timeout)) {
-                                need_validate = 1;
-                        } else if (wouldbe_size >= ctx->hard_lim) {
-                                hard_limit_exceeded = 1;
-                        }
-                }
-                UNLOCK (&ctx->lock);
-
-                if (need_validate && *skip_check != _gf_true) {
-                        *skip_check = _gf_true;
-                        ret = quota_validate (frame, _inode, this,
-                                              quota_validate_cbk);
-                        if (ret < 0) {
-                                *op_errno = -ret;
-                                *skip_check = _gf_false;
-                        }
-                        goto out;
-                }
+    int32_t ret = -1;
+    uint32_t timeout = 0;
+    char need_validate = 0;
+    gf_boolean_t hard_limit_exceeded = 0;
+    int64_t space_available = 0;
+    int64_t wouldbe_size = 0;
+
+    GF_ASSERT(frame);
+    GF_ASSERT(priv);
+    GF_ASSERT(_inode);
+    GF_ASSERT(this);
+    GF_ASSERT(local);
+
+    if (ctx != NULL && (ctx->hard_lim > 0 || ctx->soft_lim > 0)) {
+        wouldbe_size = ctx->size + delta;
+
+        LOCK(&ctx->lock);
+        {
+            timeout = priv->soft_timeout;
 
-                if (hard_limit_exceeded) {
-                        local->op_ret = -1;
-                        local->op_errno = EDQUOT;
+            if ((ctx->soft_lim >= 0) && (wouldbe_size > ctx->soft_lim)) {
+                timeout = priv->hard_timeout;
+            }
+
+            if (!just_validated && quota_timeout(ctx->validate_time, timeout)) {
+                need_validate = 1;
+            } else if (wouldbe_size >= ctx->hard_lim) {
+                hard_limit_exceeded = 1;
+            }
+        }
+        UNLOCK(&ctx->lock);
 
-                        space_available = ctx->hard_lim - ctx->size;
+        if (need_validate && *skip_check != _gf_true) {
+            *skip_check = _gf_true;
+            ret = quota_validate(frame, _inode, this, quota_validate_cbk);
+            if (ret < 0) {
+                *op_errno = -ret;
+                *skip_check = _gf_false;
+            }
+            goto out;
+        }
 
-                        if (space_available < 0)
-                                space_available = 0;
+        if (hard_limit_exceeded) {
+            local->op_ret = -1;
+            local->op_errno = EDQUOT;
 
-                        if ((local->space_available < 0)
-                            || (local->space_available
-                                > space_available)){
-                                local->space_available
-                                        = space_available;
+            space_available = ctx->hard_lim - ctx->size;
 
-                        }
+            if (space_available < 0)
+                space_available = 0;
 
-                        if (space_available == 0) {
-                                *op_errno = EDQUOT;
-                                goto out;
-                        }
-                }
+            if ((local->space_available < 0) ||
+                (local->space_available > space_available)) {
+                local->space_available = space_available;
+            }
 
-                /* We log usage only if quota limit is configured on
-                   that inode. */
-                quota_log_usage (this, ctx, _inode, delta);
+            if (space_available == 0) {
+                *op_errno = EDQUOT;
+                goto out;
+            }
         }
 
-        ret = 0;
+        /* We log usage only if quota limit is configured on
+           that inode. */
+        quota_log_usage(this, ctx, _inode, delta);
+    }
+
+    ret = 0;
 out:
-        return ret;
+    return ret;
 }
 
-
 int32_t
-quota_check_limit (call_frame_t *frame, inode_t *inode, xlator_t *this,
-                   char *name, uuid_t par)
+quota_check_limit(call_frame_t *frame, inode_t *inode, xlator_t *this)
 {
-        int32_t            ret                 = -1, op_errno = EINVAL;
-        inode_t           *_inode              = NULL, *parent = NULL;
-        quota_inode_ctx_t *ctx                 = NULL;
-        quota_priv_t      *priv                = NULL;
-        quota_local_t     *local               = NULL;
-        char               need_validate       = 0;
-        char               just_validated      = 0;
-        gf_boolean_t       hard_limit_exceeded = 0;
-        int64_t            delta               = 0;
-        uint64_t           value               = 0;
-        uuid_t             trav_uuid           = {0,};
-        gf_boolean_t       skip_check          = _gf_false;
-
-        GF_VALIDATE_OR_GOTO ("quota", this, err);
-        GF_VALIDATE_OR_GOTO (this->name, frame, err);
-        GF_VALIDATE_OR_GOTO (this->name, inode, err);
-
-        local  = frame->local;
-        GF_VALIDATE_OR_GOTO (this->name, local, err);
-
-        delta = local->delta;
-
-        GF_VALIDATE_OR_GOTO (this->name, local->stub, err);
-        /* Allow all the trusted clients
-         * Don't block the gluster internal processes like rebalance, gsyncd,
-         * self heal etc from the disk quotas.
-         *
-         * Method: Allow all the clients with PID negative. This is by the
-         * assumption that any kernel assigned pid doesn't have the negative
-         * number.
-         */
-        if (0 > frame->root->pid) {
-                ret = 0;
-                quota_link_count_decrement (local);
-                goto done;
-        }
+    int32_t ret = -1, op_errno = EINVAL;
+    inode_t *_inode = NULL, *parent = NULL;
+    quota_inode_ctx_t *ctx = NULL;
+    quota_priv_t *priv = NULL;
+    quota_local_t *local = NULL;
+    quota_local_t *par_local = NULL;
+    char just_validated = 0;
+    int64_t delta = 0;
+    int8_t object_delta = 0;
+    uint64_t value = 0;
+    gf_boolean_t skip_check = _gf_false;
+
+    GF_VALIDATE_OR_GOTO("quota", this, err);
+    GF_VALIDATE_OR_GOTO(this->name, frame, err);
+    GF_VALIDATE_OR_GOTO(this->name, inode, err);
+
+    local = frame->local;
+    GF_VALIDATE_OR_GOTO(this->name, local, err);
+
+    if (local->par_frame) {
+        par_local = local->par_frame->local;
+        GF_VALIDATE_OR_GOTO(this->name, par_local, err);
+    } else {
+        par_local = local;
+    }
+
+    delta = par_local->delta;
+    object_delta = par_local->object_delta;
+
+    GF_VALIDATE_OR_GOTO(this->name, par_local->stub, err);
+    /* Allow all the trusted clients
+     * Don't block the gluster internal processes like rebalance, gsyncd,
+     * self heal etc from the disk quotas.
+     *
+     * Method: Allow all the clients with PID negative. This is by the
+     * assumption that any kernel assigned pid doesn't have the negative
+     * number.
+     */
+    if (0 > frame->root->pid) {
+        ret = 0;
+        quota_link_count_decrement(frame);
+        goto done;
+    }
 
-        priv = this->private;
+    priv = this->private;
 
-        inode_ctx_get (inode, this, &value);
-        ctx = (quota_inode_ctx_t *)(unsigned long)value;
+    inode_ctx_get(inode, this, &value);
+    ctx = (quota_inode_ctx_t *)(unsigned long)value;
 
-        _inode = inode_ref (inode);
+    _inode = inode_ref(inode);
 
-        LOCK (&local->lock);
-        {
-                just_validated = local->just_validated;
-                local->just_validated = 0;
-        }
-        UNLOCK (&local->lock);
+    LOCK(&local->lock);
+    {
+        just_validated = local->just_validated;
+        local->just_validated = 0;
+    }
+    UNLOCK(&local->lock);
 
-        if ( par != NULL ) {
-                gf_uuid_copy (trav_uuid, par);
+    do {
+        /* In a rename operation, enforce should be stopped at common
+           ancestor */
+        if (!gf_uuid_is_null(par_local->common_ancestor) &&
+            !gf_uuid_compare(_inode->gfid, par_local->common_ancestor)) {
+            quota_link_count_decrement(frame);
+            break;
         }
 
-        do {
-                /* In a rename operation, enforce should be stopped at common
-                   ancestor */
-                if (!gf_uuid_is_null (local->common_ancestor) &&
-                    !gf_uuid_compare (_inode->gfid, local->common_ancestor)) {
-                        quota_link_count_decrement (local);
-                        break;
-                }
+        if (object_delta <= 0)
+            goto skip_check_object_limit;
 
-                ret = quota_check_object_limit (frame, ctx, priv, _inode, this,
-                                                &op_errno, just_validated,
-                                                local, &skip_check);
-                if (skip_check == _gf_true)
-                        goto done;
+        ret = quota_check_object_limit(frame, ctx, priv, _inode, this,
+                                       &op_errno, just_validated, par_local,
+                                       &skip_check);
+        if (skip_check == _gf_true)
+            goto done;
 
-                if (ret) {
-                        gf_log (this->name, GF_LOG_ERROR, "Failed to check "
-                                "quota object limit");
-                        goto err;
-                }
+        if (ret) {
+            if (op_errno != EDQUOT)
+                gf_msg(this->name, GF_LOG_ERROR, 0, Q_MSG_ENFORCEMENT_FAILED,
+                       "Failed to "
+                       "check quota object limit");
+            goto err;
+        }
 
-                ret = quota_check_size_limit (frame, ctx, priv, _inode, this,
-                                              &op_errno, just_validated, delta,
-                                              local, &skip_check);
-                if (skip_check == _gf_true)
-                        goto done;
+    skip_check_object_limit:
+        ret = quota_check_size_limit(frame, ctx, priv, _inode, this, &op_errno,
+                                     just_validated, delta, par_local,
+                                     &skip_check);
+        if (skip_check == _gf_true)
+            goto done;
 
-                if (ret) {
-                        gf_log (this->name, GF_LOG_ERROR, "Failed to check "
-                                "quota size limit");
-                        goto err;
-                }
+        if (ret) {
+            if (op_errno != EDQUOT)
+                gf_msg(this->name, GF_LOG_ERROR, 0, Q_MSG_ENFORCEMENT_FAILED,
+                       "Failed to "
+                       "check quota size limit");
+            goto err;
+        }
 
-                if (__is_root_gfid (_inode->gfid)) {
-                        quota_link_count_decrement (local);
-                        break;
-                }
+        if (__is_root_gfid(_inode->gfid)) {
+            quota_link_count_decrement(frame);
+            break;
+        }
 
-                parent = inode_parent (_inode, trav_uuid, name);
+        parent = inode_parent(_inode, 0, NULL);
+        if (parent == NULL) {
+            ret = quota_build_ancestry(_inode, quota_check_limit_continuation,
+                                       frame);
+            if (ret < 0) {
+                op_errno = -ret;
+                goto err;
+            }
 
-                if (name != NULL) {
-                        name = NULL;
-                        gf_uuid_clear (trav_uuid);
-                }
+            break;
+        }
 
-                if (parent == NULL) {
-                        ret = quota_build_ancestry (_inode,
-                                                    quota_check_limit_continuation,
-                                                    frame);
-                        if (ret < 0) {
-                                op_errno = -ret;
-                                goto err;
-                        }
+        inode_unref(_inode);
+        _inode = parent;
+        just_validated = 0;
 
-                        break;
-                }
+        value = 0;
+        inode_ctx_get(_inode, this, &value);
+        ctx = (quota_inode_ctx_t *)(unsigned long)value;
+    } while (1);
 
-                inode_unref (_inode);
-                _inode = parent;
-                just_validated = 0;
+done:
+    if (_inode != NULL) {
+        inode_unref(_inode);
+        _inode = NULL;
+    }
+    return 0;
 
-                value = 0;
-                inode_ctx_get (_inode, this, &value);
-                ctx = (quota_inode_ctx_t *)(unsigned long)value;
-        } while (1);
+err:
+    quota_handle_validate_error(frame, -1, op_errno);
 
+    inode_unref(_inode);
+    return 0;
+}
 
-done:
-        if (_inode != NULL) {
-                inode_unref (_inode);
-                _inode = NULL;
+inode_t *
+do_quota_check_limit(call_frame_t *frame, inode_t *inode, xlator_t *this,
+                     quota_dentry_t *dentry, gf_boolean_t force)
+{
+    int32_t ret = -1;
+    inode_t *parent = NULL;
+    call_frame_t *new_frame = NULL;
+    quota_local_t *new_local = NULL;
+
+    parent = inode_parent(inode, dentry->par, dentry->name);
+    if (parent == NULL) {
+        if (force)
+            parent = inode_find(inode->table, dentry->par);
+        else
+            goto out;
+    }
+    if (parent == NULL)
+        goto out;
+
+    new_frame = copy_frame(frame);
+    if (new_frame == NULL)
+        goto out;
+
+    new_local = quota_local_new();
+    if (new_local == NULL)
+        goto out;
+
+    new_frame->local = new_local;
+    new_local->par_frame = frame;
+
+    quota_check_limit(new_frame, parent, this);
+
+    ret = 0;
+out:
+    if (ret < 0) {
+        if (parent) {
+            /* Caller should decrement link_count, in case parent is
+             * NULL
+             */
+            quota_handle_validate_error(frame, -1, ENOMEM);
         }
-        return 0;
 
-err:
-        quota_handle_validate_error (local, -1, op_errno);
+        if (new_frame) {
+            new_frame->local = NULL;
+            STACK_DESTROY(new_frame->root);
+        }
+    }
 
-        inode_unref (_inode);
-        return 0;
+    return parent;
 }
 
-static inline int
-quota_get_limits (xlator_t *this, dict_t *dict, int64_t *hard_lim,
-                  int64_t *soft_lim, int64_t *object_hard_limit,
-                  int64_t *object_soft_limit)
+static int
+quota_get_limits(xlator_t *this, dict_t *dict, int64_t *hard_lim,
+                 int64_t *soft_lim, int64_t *object_hard_limit,
+                 int64_t *object_soft_limit)
 {
-        quota_limits_t *limit             = NULL;
-        quota_limits_t *object_limit      = NULL;
-        quota_priv_t   *priv              = NULL;
-        int64_t         soft_lim_percent  = 0;
-        int64_t        *ptr               = NULL;
-        int             ret               = 0;
-
-        if ((this == NULL) || (dict == NULL) || (hard_lim == NULL)
-            || (soft_lim == NULL))
-                goto out;
+    quota_limits_t *limit = NULL;
+    quota_limits_t *object_limit = NULL;
+    quota_priv_t *priv = NULL;
+    int64_t soft_lim_percent = 0;
+    int64_t *ptr = NULL;
+    int ret = 0;
 
-        priv = this->private;
+    if ((this == NULL) || (dict == NULL) || (hard_lim == NULL) ||
+        (soft_lim == NULL))
+        goto out;
 
-        ret = dict_get_bin (dict, QUOTA_LIMIT_KEY, (void **) &ptr);
-        limit = (quota_limits_t *)ptr;
+    priv = this->private;
 
-        if (limit) {
-                *hard_lim = ntoh64 (limit->hl);
-                soft_lim_percent = ntoh64 (limit->sl);
-        }
+    ret = dict_get_bin(dict, QUOTA_LIMIT_KEY, (void **)&ptr);
+    limit = (quota_limits_t *)ptr;
 
-        if (soft_lim_percent < 0) {
-                soft_lim_percent = priv->default_soft_lim;
-        }
+    if (limit) {
+        *hard_lim = ntoh64(limit->hl);
+        soft_lim_percent = ntoh64(limit->sl);
+    }
 
-        if ((*hard_lim > 0) && (soft_lim_percent > 0)) {
-                *soft_lim = (soft_lim_percent * (*hard_lim))/100;
-        }
+    if (soft_lim_percent < 0) {
+        soft_lim_percent = priv->default_soft_lim;
+    }
 
-        ret = dict_get_bin (dict, QUOTA_LIMIT_OBJECTS_KEY, (void **) &ptr);
-        if (ret)
-                return 0;
-        object_limit = (quota_limits_t *)ptr;
+    if ((*hard_lim > 0) && (soft_lim_percent > 0)) {
+        *soft_lim = (soft_lim_percent * (*hard_lim)) / 100;
+    }
 
-        if (object_limit) {
-                *object_hard_limit = ntoh64 (object_limit->hl);
-                 soft_lim_percent = ntoh64 (object_limit->sl);
-        }
+    ret = dict_get_bin(dict, QUOTA_LIMIT_OBJECTS_KEY, (void **)&ptr);
+    if (ret)
+        return 0;
+    object_limit = (quota_limits_t *)ptr;
 
-        if (soft_lim_percent < 0) {
-                soft_lim_percent = priv->default_soft_lim;
-        }
+    if (object_limit) {
+        *object_hard_limit = ntoh64(object_limit->hl);
+        soft_lim_percent = ntoh64(object_limit->sl);
+    }
 
-        if ((*object_hard_limit > 0) && (soft_lim_percent > 0)) {
-                *object_soft_limit = (soft_lim_percent *
-                                     (*object_hard_limit))/100;
-        }
+    if (soft_lim_percent < 0) {
+        soft_lim_percent = priv->default_soft_lim;
+    }
+
+    if ((*object_hard_limit > 0) && (soft_lim_percent > 0)) {
+        *object_soft_limit = (soft_lim_percent * (*object_hard_limit)) / 100;
+    }
 
 out:
-        return 0;
+    return 0;
 }
 
 int
-quota_fill_inodectx (xlator_t *this, inode_t *inode, dict_t *dict,
-                     loc_t *loc, struct iatt *buf, int32_t *op_errno)
+quota_fill_inodectx(xlator_t *this, inode_t *inode, dict_t *dict, loc_t *loc,
+                    struct iatt *buf, int32_t *op_errno)
 {
-        int32_t            ret                  = -1;
-        char               found                = 0;
-        quota_inode_ctx_t *ctx                  = NULL;
-        quota_dentry_t    *dentry               = NULL;
-        uint64_t           value                = 0;
-        int64_t            hard_lim             = 0;
-        int64_t            soft_lim             = 0;
-        int64_t            object_hard_limit    = 0;
-        int64_t            object_soft_limit    = 0;
-
-        quota_get_limits (this, dict, &hard_lim, &soft_lim, &object_hard_limit,
-                          &object_soft_limit);
-
-        inode_ctx_get (inode, this, &value);
-        ctx = (quota_inode_ctx_t *)(unsigned long)value;
-
-        if ((((ctx == NULL) || (ctx->hard_lim == hard_lim))
-             && (hard_lim < 0) && !QUOTA_REG_OR_LNK_FILE (buf->ia_type))) {
-                ret = 0;
-                goto out;
+    int32_t ret = -1;
+    char found = 0;
+    quota_inode_ctx_t *ctx = NULL;
+    quota_dentry_t *dentry = NULL;
+    uint64_t value = 0;
+    int64_t hard_lim = 0;
+    int64_t soft_lim = 0;
+    int64_t object_hard_limit = 0;
+    int64_t object_soft_limit = 0;
+
+    quota_get_limits(this, dict, &hard_lim, &soft_lim, &object_hard_limit,
+                     &object_soft_limit);
+
+    inode_ctx_get(inode, this, &value);
+    ctx = (quota_inode_ctx_t *)(unsigned long)value;
+
+    if ((((ctx == NULL) || (ctx->hard_lim == hard_lim)) && (hard_lim < 0) &&
+         !QUOTA_REG_OR_LNK_FILE(buf->ia_type))) {
+        ret = 0;
+        goto out;
+    }
+
+    ret = quota_inode_ctx_get(inode, this, &ctx, 1);
+    if ((ret == -1) || (ctx == NULL)) {
+        gf_msg(this->name, GF_LOG_WARNING, ENOMEM, Q_MSG_INODE_CTX_GET_FAILED,
+               "cannot create quota "
+               "context in inode(gfid:%s)",
+               uuid_utoa(inode->gfid));
+        ret = -1;
+        *op_errno = ENOMEM;
+        goto out;
+    }
+
+    LOCK(&ctx->lock);
+    {
+        ctx->hard_lim = hard_lim;
+        ctx->soft_lim = soft_lim;
+        ctx->object_hard_lim = object_hard_limit;
+        ctx->object_soft_lim = object_soft_limit;
+
+        ctx->buf = *buf;
+
+        if (!QUOTA_REG_OR_LNK_FILE(buf->ia_type)) {
+            goto unlock;
+        }
+
+        /* do nothing if it is a nameless lookup */
+        if (loc->name == NULL || !loc->parent)
+            goto unlock;
+
+        list_for_each_entry(dentry, &ctx->parents, next)
+        {
+            if ((strcmp(dentry->name, loc->name) == 0) &&
+                (gf_uuid_compare(loc->parent->gfid, dentry->par) == 0)) {
+                found = 1;
+                break;
+            }
         }
 
-        ret = quota_inode_ctx_get (inode, this, &ctx, 1);
-        if ((ret == -1) || (ctx == NULL)) {
-                gf_log (this->name, GF_LOG_WARNING, "cannot create quota "
-                        "context in inode(gfid:%s)",
-                        uuid_utoa (inode->gfid));
+        if (!found) {
+            dentry = __quota_dentry_new(ctx, (char *)loc->name,
+                                        loc->parent->gfid);
+            if (dentry == NULL) {
+                /*
+                  gf_msg (this->name, GF_LOG_WARNING, ENOMEM,
+                          Q_MSG_ENOMEM,
+                          "cannot create a new dentry (par:%"
+-                                          PRId64", name:%s) for inode(ino:%"
+-                                          PRId64", gfid:%s)",
+-                                          uuid_utoa (local->loc.inode->gfid));
+                */
                 ret = -1;
                 *op_errno = ENOMEM;
-                goto out;
+                goto unlock;
+            }
         }
+    }
+unlock:
+    UNLOCK(&ctx->lock);
 
-        LOCK (&ctx->lock);
-        {
-                ctx->hard_lim = hard_lim;
-                ctx->soft_lim = soft_lim;
-                ctx->object_hard_lim = object_hard_limit;
-                ctx->object_soft_lim = object_soft_limit;
-
-                ctx->buf = *buf;
+out:
+    return ret;
+}
 
-                if (!QUOTA_REG_OR_LNK_FILE (buf->ia_type)) {
-                        goto unlock;
-                }
+/*
+ * return _gf_true if enforcement is needed and _gf_false otherwise
+ */
+gf_boolean_t
+should_quota_enforce(xlator_t *this, dict_t *dict, glusterfs_fop_t fop)
+{
+    int ret = 0;
 
-                /* do nothing if it is a nameless lookup */
-                if (loc->name == NULL || !loc->parent)
-                        goto unlock;
-
-                list_for_each_entry (dentry, &ctx->parents, next) {
-                        if ((strcmp (dentry->name, loc->name) == 0) &&
-                            (gf_uuid_compare (loc->parent->gfid,
-                                           dentry->par) == 0)) {
-                                found = 1;
-                                break;
-                        }
-                }
-
-                if (!found) {
-                        dentry = __quota_dentry_new (ctx,
-                                                     (char *)loc->name,
-                                                     loc->parent->gfid);
-                        if (dentry == NULL) {
-                                /*
-                                  gf_log (this->name, GF_LOG_WARNING,
-                                        "cannot create a new dentry (par:%"
-                                        PRId64", name:%s) for inode(ino:%"
-                                        PRId64", gfid:%s)",
-                                        uuid_utoa (local->loc.inode->gfid));
-                                */
-                                ret = -1;
-                                *op_errno = ENOMEM;
-                                goto unlock;
-                        }
-                }
-        }
-unlock:
-        UNLOCK (&ctx->lock);
+    ret = dict_check_flag(dict, GF_INTERNAL_CTX_KEY, GF_DHT_HEAL_DIR);
 
+    if (fop == GF_FOP_MKDIR && ret == DICT_FLAG_SET) {
+        return _gf_false;
+    } else if (ret == -ENOENT) {
+        gf_msg(this->name, GF_LOG_DEBUG, EINVAL, Q_MSG_INTERNAL_FOP_KEY_MISSING,
+               "No internal fop context present");
+        goto out;
+    }
 out:
-        return ret;
+    return _gf_true;
 }
 
 int32_t
-quota_lookup_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                  int32_t op_ret, int32_t op_errno, inode_t *inode,
-                  struct iatt *buf, dict_t *dict, struct iatt *postparent)
+quota_lookup_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                 int32_t op_ret, int32_t op_errno, inode_t *inode,
+                 struct iatt *buf, dict_t *dict, struct iatt *postparent)
 {
-        quota_local_t      *local        = NULL;
-        int32_t             ret          = 0;
-        inode_t            *this_inode   = NULL;
+    quota_local_t *local = NULL;
+    inode_t *this_inode = NULL;
 
-        local = frame->local;
-        frame->local = NULL;
+    local = frame->local;
+    frame->local = NULL;
 
-        if (op_ret >= 0 && inode) {
-                this_inode = inode_ref (inode);
+    if (op_ret >= 0 && inode) {
+        this_inode = inode_ref(inode);
 
-                op_ret = quota_fill_inodectx (this, inode, dict, &local->loc,
-                                              buf, &op_errno);
-                if (op_ret < 0)
-                        op_errno = ENOMEM;
-        }
+        op_ret = quota_fill_inodectx(this, inode, dict, &local->loc, buf,
+                                     &op_errno);
+        if (op_ret < 0)
+            op_errno = ENOMEM;
+    }
 
-        QUOTA_STACK_UNWIND (lookup, frame, op_ret, op_errno, inode, buf,
-                            dict, postparent);
+    QUOTA_STACK_UNWIND(lookup, frame, op_ret, op_errno, inode, buf, dict,
+                       postparent);
 
-        if (op_ret < 0 || this_inode == NULL || gf_uuid_is_null(this_inode->gfid))
-                goto out;
+    if (op_ret < 0 || this_inode == NULL || gf_uuid_is_null(this_inode->gfid))
+        goto out;
 
-        check_ancestory_2 (this, local, this_inode);
+    check_ancestory_2(this, local, this_inode);
 
 out:
-        if (this_inode)
-                inode_unref (this_inode);
+    if (this_inode)
+        inode_unref(this_inode);
 
-        quota_local_cleanup (this, local);
+    quota_local_cleanup(local);
 
-        return 0;
+    return 0;
 }
 
 int32_t
-quota_lookup (call_frame_t *frame, xlator_t *this, loc_t *loc,
-              dict_t *xattr_req)
+quota_lookup(call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xattr_req)
 {
-        quota_priv_t  *priv             = NULL;
-        int32_t        ret              = -1;
-        quota_local_t *local            = NULL;
+    quota_priv_t *priv = NULL;
+    int32_t ret = -1;
+    quota_local_t *local = NULL;
 
-        priv = this->private;
+    priv = this->private;
 
-        WIND_IF_QUOTAOFF (priv->is_quota_on, off);
+    WIND_IF_QUOTAOFF(priv->is_quota_on, off);
 
-        xattr_req = xattr_req ? dict_ref(xattr_req) : dict_new();
-        if (!xattr_req)
-                goto err;
+    xattr_req = xattr_req ? dict_ref(xattr_req) : dict_new();
+    if (!xattr_req)
+        goto err;
 
-        local = quota_local_new ();
-        if (local == NULL) {
-                goto err;
-        }
+    local = quota_local_new();
+    if (local == NULL) {
+        goto err;
+    }
 
-        frame->local = local;
-        loc_copy (&local->loc, loc);
+    frame->local = local;
+    loc_copy(&local->loc, loc);
 
-        ret = dict_set_int8 (xattr_req, QUOTA_LIMIT_KEY, 1);
-        if (ret < 0) {
-                gf_log (this->name, GF_LOG_WARNING,
-                        "dict set of key for hard-limit failed");
-                goto err;
-        }
+    ret = dict_set_int8(xattr_req, QUOTA_LIMIT_KEY, 1);
+    if (ret < 0) {
+        gf_msg(this->name, GF_LOG_WARNING, ENOMEM, Q_MSG_ENOMEM,
+               "dict set of key for "
+               "hard-limit failed");
+        goto err;
+    }
 
-        ret = dict_set_int8 (xattr_req, QUOTA_LIMIT_OBJECTS_KEY, 1);
-        if (ret < 0) {
-                gf_log (this->name, GF_LOG_WARNING,
-                        "dict set of key for quota object limit failed");
-                goto err;
-        }
+    ret = dict_set_int8(xattr_req, QUOTA_LIMIT_OBJECTS_KEY, 1);
+    if (ret < 0) {
+        gf_msg(this->name, GF_LOG_WARNING, ENOMEM, Q_MSG_ENOMEM,
+               "dict set of key for quota object limit failed");
+        goto err;
+    }
 
-        STACK_WIND (frame, quota_lookup_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->lookup, loc, xattr_req);
+    STACK_WIND(frame, quota_lookup_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->lookup, loc, xattr_req);
 
-        ret = 0;
+    ret = 0;
 
 err:
-        if (xattr_req)
-                dict_unref (xattr_req);
+    if (xattr_req)
+        dict_unref(xattr_req);
 
-        if (ret < 0) {
-                QUOTA_STACK_UNWIND (lookup, frame, -1, ENOMEM,
-                                    NULL, NULL, NULL, NULL);
-        }
+    if (ret < 0) {
+        QUOTA_STACK_UNWIND(lookup, frame, -1, ENOMEM, NULL, NULL, NULL, NULL);
+    }
 
-        return 0;
+    return 0;
 
 off:
-        STACK_WIND_TAIL (frame, FIRST_CHILD(this),
-                         FIRST_CHILD(this)->fops->lookup, loc, xattr_req);
-        return 0;
+    STACK_WIND_TAIL(frame, FIRST_CHILD(this), FIRST_CHILD(this)->fops->lookup,
+                    loc, xattr_req);
+    return 0;
 }
 
 int32_t
-quota_writev_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                  int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
-                  struct iatt *postbuf, dict_t *xdata)
+quota_writev_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                 int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
+                 struct iatt *postbuf, dict_t *xdata)
 {
-        int32_t                  ret            = 0;
-        uint64_t                 ctx_int        = 0;
-        quota_inode_ctx_t       *ctx            = NULL;
-        quota_local_t           *local          = NULL;
-
-        local = frame->local;
-
-        if ((op_ret < 0) || (local == NULL)) {
-                goto out;
-        }
-
-        ret = inode_ctx_get (local->loc.inode, this, &ctx_int);
-        if (ret) {
-                gf_log (this->name, GF_LOG_WARNING,
-                        "%s: failed to get the context", local->loc.path);
-                goto out;
-        }
-
-        ctx = (quota_inode_ctx_t *)(unsigned long) ctx_int;
-
-        if (ctx == NULL) {
-                gf_log (this->name, GF_LOG_WARNING,
-                        "quota context not set in %s (gfid:%s)",
-                        local->loc.path, uuid_utoa (local->loc.inode->gfid));
-                goto out;
-        }
-
-        LOCK (&ctx->lock);
-        {
-                ctx->buf = *postbuf;
-        }
-        UNLOCK (&ctx->lock);
+    int32_t ret = 0;
+    uint64_t ctx_int = 0;
+    quota_inode_ctx_t *ctx = NULL;
+    quota_local_t *local = NULL;
+
+    local = frame->local;
+
+    if ((op_ret < 0) || (local == NULL) || (postbuf == NULL)) {
+        goto out;
+    }
+
+    ret = inode_ctx_get(local->loc.inode, this, &ctx_int);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_WARNING, 0, Q_MSG_INODE_CTX_GET_FAILED,
+               "%s: failed to get the "
+               "context",
+               local->loc.path);
+        goto out;
+    }
+
+    ctx = (quota_inode_ctx_t *)(unsigned long)ctx_int;
+
+    if (ctx == NULL) {
+        gf_msg(this->name, GF_LOG_WARNING, 0, Q_MSG_INODE_CTX_GET_FAILED,
+               "quota context not set in %s (gfid:%s)", local->loc.path,
+               uuid_utoa(local->loc.inode->gfid));
+        goto out;
+    }
+
+    LOCK(&ctx->lock);
+    {
+        ctx->buf = *postbuf;
+    }
+    UNLOCK(&ctx->lock);
 
 out:
-        QUOTA_STACK_UNWIND (writev, frame, op_ret, op_errno, prebuf, postbuf,
-                            xdata);
+    QUOTA_STACK_UNWIND(writev, frame, op_ret, op_errno, prebuf, postbuf, xdata);
 
-        return 0;
+    return 0;
 }
 
+static int gf_quota_enforcer_log;
 
 int32_t
-quota_writev_helper (call_frame_t *frame, xlator_t *this, fd_t *fd,
-                     struct iovec *vector, int32_t count, off_t off,
-                     uint32_t flags, struct iobref *iobref, dict_t *xdata)
+quota_writev_helper(call_frame_t *frame, xlator_t *this, fd_t *fd,
+                    struct iovec *vector, int32_t count, off_t off,
+                    uint32_t flags, struct iobref *iobref, dict_t *xdata)
 {
-        quota_local_t *local      = NULL;
-        int32_t        op_errno   = EINVAL;
-        quota_priv_t  *priv       = NULL;
-        struct iovec  *new_vector = NULL;
-        int32_t        new_count  = 0;
-
-        priv = this->private;
-
-        local = frame->local;
-        if (local == NULL) {
-                gf_log (this->name, GF_LOG_WARNING, "local is NULL");
-                goto unwind;
-        }
+    quota_local_t *local = NULL;
+    int32_t op_errno = EINVAL;
+    struct iovec *new_vector = NULL;
+    int32_t new_count = 0;
 
-        if (local->op_ret == -1) {
-                op_errno = local->op_errno;
+    local = frame->local;
 
-                if ((op_errno == EDQUOT) && (local->space_available > 0)) {
-                        new_count = iov_subset (vector, count, 0,
-                                                local->space_available, NULL);
+    GF_VALIDATE_OR_GOTO("quota", local, unwind);
 
-                        new_vector = GF_CALLOC (new_count,
-                                                sizeof (struct iovec),
-                                                gf_common_mt_iovec);
-                        if (new_vector == NULL) {
-                                local->op_ret = -1;
-                                local->op_errno = ENOMEM;
-                                goto unwind;
-                        }
-
-                        new_count = iov_subset (vector, count, 0,
-                                                local->space_available,
-                                                new_vector);
+    if (local->op_ret == -1) {
+        op_errno = local->op_errno;
 
-                        vector = new_vector;
-                        count = new_count;
-                } else {
-                        goto unwind;
-                }
+        if ((op_errno == EDQUOT) && (local->space_available > 0)) {
+            new_count = iov_subset(vector, count, 0, local->space_available,
+                                   &new_vector, 0);
+            if (new_count < 0) {
+                local->op_ret = -1;
+                local->op_errno = ENOMEM;
+                goto unwind;
+            }
+
+            vector = new_vector;
+            count = new_count;
+        } else if (op_errno == ENOENT || op_errno == ESTALE) {
+            /* We may get ENOENT/ESTALE in case of below scenario
+             *     fd = open file.txt
+             *     unlink file.txt
+             *     write on fd
+             * Here build_ancestry can fail as the file is removed.
+             * For now ignore ENOENT/ESTALE with writes on active fd
+             * We need to re-visit this code once we understand
+             * how other file-system behave in this scenario
+             */
+            gf_msg_debug(this->name, 0,
+                         "quota enforcer failed "
+                         "with ENOENT/ESTALE on %s, cannot check "
+                         "quota limits and allowing writes",
+                         uuid_utoa(fd->inode->gfid));
+        } else if ((op_errno == EINVAL) &&
+                   !inode_parent(local->loc.inode, 0, NULL)) {
+            /* We may get INVAL with parent == NULL,
+             * in case of below scenario
+             *     1. enable quota
+             *     2. glusterfsd stop/start
+             *     3. nameless lookup
+             *     4. write on fd
+             * Here build_ancestry can fail as the file's pgfid
+             * is't exist.
+             * For now ignore EINVAL with writes on active fd
+             * untils the pgfid is created at name lookup
+             */
+            GF_LOG_OCCASIONALLY(gf_quota_enforcer_log, this->name,
+                                GF_LOG_CRITICAL,
+                                "Quota cannot be enforced as "
+                                "parent is not available and writes are being "
+                                "allowed without checking whether they are "
+                                "within quota limits. This can happen if Quota "
+                                "crawl is not complete. If crawl has been "
+                                "completed, please file a bug.");
+        } else {
+            goto unwind;
         }
+    }
 
-        STACK_WIND (frame, quota_writev_cbk,
-                    FIRST_CHILD(this), FIRST_CHILD(this)->fops->writev, fd,
-                    vector, count, off, flags, iobref, xdata);
+    STACK_WIND(frame, quota_writev_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->writev, fd, vector, count, off, flags,
+               iobref, xdata);
 
-        if (new_vector != NULL)
-                GF_FREE (new_vector);
+    if (new_vector != NULL)
+        GF_FREE(new_vector);
 
-        return 0;
+    return 0;
 
 unwind:
-        QUOTA_STACK_UNWIND (writev, frame, -1, op_errno, NULL, NULL, NULL);
-        return 0;
+    QUOTA_STACK_UNWIND(writev, frame, -1, op_errno, NULL, NULL, NULL);
+    return 0;
 }
 
-
 int32_t
-quota_writev (call_frame_t *frame, xlator_t *this, fd_t *fd,
-              struct iovec *vector, int32_t count, off_t off,
-              uint32_t flags, struct iobref *iobref, dict_t *xdata)
+quota_writev(call_frame_t *frame, xlator_t *this, fd_t *fd,
+             struct iovec *vector, int32_t count, off_t off, uint32_t flags,
+             struct iobref *iobref, dict_t *xdata)
 {
-        quota_priv_t      *priv    = NULL;
-        int32_t            ret     = -1, op_errno = EINVAL;
-        int32_t            parents = 0;
-        uint64_t           size    = 0;
-        quota_local_t     *local   = NULL;
-        quota_inode_ctx_t *ctx     = NULL;
-        quota_dentry_t    *dentry  = NULL, *tmp = NULL;
-        call_stub_t       *stub    = NULL;
-        struct list_head   head    = {0, };
-
-        priv = this->private;
-
-        WIND_IF_QUOTAOFF (priv->is_quota_on, off);
-
-        INIT_LIST_HEAD (&head);
-
-        GF_ASSERT (frame);
-        GF_VALIDATE_OR_GOTO ("quota", this, unwind);
-        GF_VALIDATE_OR_GOTO (this->name, fd, unwind);
-
-        local = quota_local_new ();
-        if (local == NULL) {
-                goto unwind;
-        }
-
-        frame->local = local;
-        local->loc.inode = inode_ref (fd->inode);
-
-        ret = quota_inode_ctx_get (fd->inode, this, &ctx, 0);
-        if (ctx == NULL) {
-                gf_log (this->name, GF_LOG_DEBUG, "quota context is NULL on "
-                        "inode (%s). "
-                        "If quota is not enabled recently and crawler has "
-                        "finished crawling, its an error",
-                        uuid_utoa (fd->inode->gfid));
-        }
-
-        stub = fop_writev_stub (frame, quota_writev_helper, fd, vector, count,
-                                off, flags, iobref, xdata);
-        if (stub == NULL) {
-                op_errno = ENOMEM;
-                goto unwind;
-        }
+    quota_priv_t *priv = NULL;
+    int32_t op_errno = EINVAL;
+    int32_t parents = 0;
+    int32_t fail_count = 0;
+    uint64_t size = 0;
+    quota_local_t *local = NULL;
+    quota_inode_ctx_t *ctx = NULL;
+    quota_dentry_t *dentry = NULL, *tmp = NULL;
+    call_stub_t *stub = NULL;
+    struct list_head head;
+    inode_t *par_inode = NULL;
+
+    priv = this->private;
+
+    WIND_IF_QUOTAOFF(priv->is_quota_on, off);
+
+    INIT_LIST_HEAD(&head);
+
+    GF_ASSERT(frame);
+    GF_VALIDATE_OR_GOTO("quota", this, unwind);
+    GF_VALIDATE_OR_GOTO(this->name, fd, unwind);
+
+    local = quota_local_new();
+    if (local == NULL) {
+        goto unwind;
+    }
+
+    frame->local = local;
+    local->loc.inode = inode_ref(fd->inode);
+
+    (void)quota_inode_ctx_get(fd->inode, this, &ctx, 0);
+    if (ctx == NULL) {
+        gf_msg_debug(this->name, 0,
+                     "quota context is NULL on inode"
+                     " (%s). If quota is not enabled recently and "
+                     "crawler has finished crawling, its an error",
+                     uuid_utoa(fd->inode->gfid));
+    }
+
+    stub = fop_writev_stub(frame, quota_writev_helper, fd, vector, count, off,
+                           flags, iobref, xdata);
+    if (stub == NULL) {
+        op_errno = ENOMEM;
+        goto unwind;
+    }
+
+    priv = this->private;
+    GF_VALIDATE_OR_GOTO(this->name, priv, unwind);
+
+    parents = quota_add_parents_from_ctx(ctx, &head);
+    if (parents == -1) {
+        op_errno = errno;
+        goto unwind;
+    }
+
+    size = iov_length(vector, count);
+
+    LOCK(&local->lock);
+    {
+        local->delta = size;
+        local->object_delta = 0;
+        local->link_count = (parents != 0) ? parents : 1;
+        local->stub = stub;
+    }
+    UNLOCK(&local->lock);
 
-        priv = this->private;
-        GF_VALIDATE_OR_GOTO (this->name, priv, unwind);
-
-        size = iov_length (vector, count);
-        if (ctx != NULL) {
-                LOCK (&ctx->lock);
-                {
-                        list_for_each_entry (dentry, &ctx->parents, next) {
-                                tmp = __quota_dentry_new (NULL, dentry->name,
-                                                          dentry->par);
-                                list_add_tail (&tmp->next, &head);
-                                parents++;
-                        }
+    if (parents == 0) {
+        /* nameless lookup on this inode, allow quota to reconstruct
+         * ancestry as part of check_limit.
+         */
+        quota_check_limit(frame, fd->inode, this);
+    } else {
+        list_for_each_entry_safe(dentry, tmp, &head, next)
+        {
+            par_inode = do_quota_check_limit(frame, fd->inode, this, dentry,
+                                             _gf_false);
+            if (par_inode == NULL) {
+                if (ctx) {
+                    /* remove stale entry from inode ctx */
+                    quota_dentry_del(ctx, dentry->name, dentry->par);
+                    parents--;
+                    fail_count++;
                 }
-                UNLOCK (&ctx->lock);
+            } else {
+                inode_unref(par_inode);
+            }
+            __quota_dentry_free(dentry);
         }
 
-        LOCK (&local->lock);
-        {
-                local->delta = size;
-                local->link_count = (parents != 0) ? parents : 1;
-                local->stub = stub;
+        if (parents == 0) {
+            LOCK(&local->lock);
+            {
+                local->link_count++;
+            }
+            UNLOCK(&local->lock);
+            quota_check_limit(frame, fd->inode, this);
         }
-        UNLOCK (&local->lock);
 
-        if (parents == 0) {
-                /* nameless lookup on this inode, allow quota to reconstruct
-                 * ancestry as part of check_limit.
-                 */
-                quota_check_limit (frame, fd->inode, this, NULL, NULL);
-        } else {
-                list_for_each_entry_safe (dentry, tmp, &head, next) {
-                        quota_check_limit (frame, fd->inode, this, dentry->name,
-                                           dentry->par);
-                        __quota_dentry_free (dentry);
-                }
+        while (fail_count != 0) {
+            quota_link_count_decrement(frame);
+            fail_count--;
         }
+    }
 
-        return 0;
+    return 0;
 
 unwind:
-        QUOTA_STACK_UNWIND (writev, frame, -1, op_errno, NULL, NULL, NULL);
-        return 0;
+    QUOTA_STACK_UNWIND(writev, frame, -1, op_errno, NULL, NULL, NULL);
+    return 0;
 
 off:
-        STACK_WIND_TAIL (frame, FIRST_CHILD(this),
-                         FIRST_CHILD(this)->fops->writev, fd,
-                         vector, count, off, flags, iobref, xdata);
-        return 0;
+    STACK_WIND_TAIL(frame, FIRST_CHILD(this), FIRST_CHILD(this)->fops->writev,
+                    fd, vector, count, off, flags, iobref, xdata);
+    return 0;
 }
 
-
 int32_t
-quota_mkdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                 int32_t op_ret, int32_t op_errno, inode_t *inode,
-                 struct iatt *buf, struct iatt *preparent,
-                 struct iatt *postparent, dict_t *xdata)
+quota_mkdir_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                int32_t op_ret, int32_t op_errno, inode_t *inode,
+                struct iatt *buf, struct iatt *preparent,
+                struct iatt *postparent, dict_t *xdata)
 {
-        QUOTA_STACK_UNWIND (mkdir, frame, op_ret, op_errno, inode,
-                            buf, preparent, postparent, xdata);
-        return 0;
+    QUOTA_STACK_UNWIND(mkdir, frame, op_ret, op_errno, inode, buf, preparent,
+                       postparent, xdata);
+    return 0;
 }
 
-
 int32_t
-quota_mkdir_helper (call_frame_t *frame, xlator_t *this, loc_t *loc,
-                    mode_t mode, mode_t umask, dict_t *xdata)
+quota_mkdir_helper(call_frame_t *frame, xlator_t *this, loc_t *loc, mode_t mode,
+                   mode_t umask, dict_t *xdata)
 {
-        quota_local_t *local    = NULL;
-        int32_t        op_errno = EINVAL;
+    quota_local_t *local = NULL;
+    int32_t op_errno = EINVAL;
 
-        local = frame->local;
-        if (local == NULL) {
-                gf_log (this->name, GF_LOG_WARNING, "local is NULL");
-                goto unwind;
-        }
+    local = frame->local;
 
-        op_errno = local->op_errno;
+    GF_VALIDATE_OR_GOTO("quota", local, unwind);
 
-        if (local->op_ret == -1) {
-                goto unwind;
-        }
+    op_errno = local->op_errno;
 
-        STACK_WIND (frame, quota_mkdir_cbk,
-                    FIRST_CHILD(this), FIRST_CHILD(this)->fops->mkdir, loc,
-                    mode, umask, xdata);
+    if (local->op_ret == -1) {
+        goto unwind;
+    }
 
-        return 0;
+    STACK_WIND(frame, quota_mkdir_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->mkdir, loc, mode, umask, xdata);
+
+    return 0;
 
 unwind:
-        QUOTA_STACK_UNWIND (mkdir, frame, -1, op_errno, NULL, NULL,
-                            NULL, NULL, NULL);
-        return 0;
+    QUOTA_STACK_UNWIND(mkdir, frame, -1, op_errno, NULL, NULL, NULL, NULL,
+                       NULL);
+    return 0;
 }
 
-
 int32_t
-quota_mkdir (call_frame_t *frame, xlator_t *this, loc_t *loc, mode_t mode,
-             mode_t umask, dict_t *xdata)
+quota_mkdir(call_frame_t *frame, xlator_t *this, loc_t *loc, mode_t mode,
+            mode_t umask, dict_t *xdata)
 {
-        quota_priv_t  *priv  = NULL;
-        int32_t        ret   = 0, op_errno = 0;
-        quota_local_t *local = NULL;
-        call_stub_t   *stub  = NULL;
-
-        priv = this->private;
-
-        WIND_IF_QUOTAOFF (priv->is_quota_on, off);
-
-        local = quota_local_new ();
-        if (local == NULL) {
-                op_errno = ENOMEM;
-                goto err;
-        }
-
-        frame->local = local;
-
-        ret = loc_copy (&local->loc, loc);
-        if (ret) {
-                op_errno = ENOMEM;
-                gf_log (this->name, GF_LOG_WARNING, "loc_copy failed");
-                goto err;
-        }
-
-        stub = fop_mkdir_stub (frame, quota_mkdir_helper, loc, mode, umask,
-                               xdata);
-        if (stub == NULL) {
-                op_errno = ENOMEM;
-                goto err;
-        }
-
-        LOCK (&local->lock);
-        {
-                local->stub = stub;
-                local->delta = 0;
-                local->link_count = 1;
-        }
-        UNLOCK (&local->lock);
+    quota_priv_t *priv = NULL;
+    int32_t ret = 0, op_errno = 0;
+    quota_local_t *local = NULL;
+    call_stub_t *stub = NULL;
+
+    priv = this->private;
+    WIND_IF_QUOTAOFF(priv->is_quota_on, off);
+
+    if (!should_quota_enforce(this, xdata, GF_FOP_MKDIR)) {
+        gf_msg(this->name, GF_LOG_DEBUG, 0, Q_MSG_ENFORCEMENT_SKIPPED,
+               "Enforcement has been skipped(internal fop).");
+        goto off;
+    }
+
+    local = quota_local_new();
+    if (local == NULL) {
+        op_errno = ENOMEM;
+        goto err;
+    }
+
+    frame->local = local;
+
+    ret = loc_copy(&local->loc, loc);
+    if (ret) {
+        op_errno = ENOMEM;
+        gf_msg(this->name, GF_LOG_WARNING, ENOMEM, Q_MSG_ENOMEM,
+               "loc_copy failed");
+        goto err;
+    }
+
+    stub = fop_mkdir_stub(frame, quota_mkdir_helper, loc, mode, umask, xdata);
+    if (stub == NULL) {
+        op_errno = ENOMEM;
+        goto err;
+    }
+
+    LOCK(&local->lock);
+    {
+        local->stub = stub;
+        local->delta = 0;
+        local->object_delta = 1;
+        local->link_count = 1;
+    }
+    UNLOCK(&local->lock);
 
-        quota_check_limit (frame, loc->parent, this, NULL, NULL);
-        return 0;
+    quota_check_limit(frame, loc->parent, this);
+    return 0;
 
 err:
-        QUOTA_STACK_UNWIND (mkdir, frame, -1, op_errno, NULL, NULL, NULL,
-                            NULL, NULL);
+    QUOTA_STACK_UNWIND(mkdir, frame, -1, op_errno, NULL, NULL, NULL, NULL,
+                       NULL);
 
-        return 0;
+    return 0;
 
 off:
-        STACK_WIND_TAIL (frame, FIRST_CHILD(this),
-                         FIRST_CHILD(this)->fops->mkdir,
-                         loc, mode, umask, xdata);
+    STACK_WIND_TAIL(frame, FIRST_CHILD(this), FIRST_CHILD(this)->fops->mkdir,
+                    loc, mode, umask, xdata);
 
-        return 0;
+    return 0;
 }
 
-
 int32_t
-quota_create_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                  int32_t op_ret, int32_t op_errno, fd_t *fd, inode_t *inode,
-                  struct iatt *buf, struct iatt *preparent,
-                  struct iatt *postparent, dict_t *xdata)
+quota_create_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                 int32_t op_ret, int32_t op_errno, fd_t *fd, inode_t *inode,
+                 struct iatt *buf, struct iatt *preparent,
+                 struct iatt *postparent, dict_t *xdata)
 {
-        int32_t            ret    = -1;
-        quota_local_t     *local  = NULL;
-        quota_inode_ctx_t *ctx    = NULL;
-        quota_dentry_t    *dentry = NULL;
-
-        local = frame->local;
-        if (op_ret < 0) {
-                goto unwind;
-        }
-
-        ret = quota_inode_ctx_get (inode, this, &ctx, 1);
-        if ((ret == -1) || (ctx == NULL)) {
-                gf_log (this->name, GF_LOG_WARNING, "cannot create quota "
-                        "context in inode(gfid:%s)",
-                        uuid_utoa (inode->gfid));
-                op_ret = -1;
-                op_errno = ENOMEM;
-                goto unwind;
-        }
-
-        LOCK (&ctx->lock);
-        {
-                ctx->buf = *buf;
-
-                dentry = __quota_dentry_new (ctx, (char *)local->loc.name,
-                                             local->loc.parent->gfid);
-                if (dentry == NULL) {
-                        gf_log (this->name, GF_LOG_WARNING,
-                                "cannot create a new dentry (name:%s) for "
-                                "inode(gfid:%s)", local->loc.name,
-                                uuid_utoa (local->loc.inode->gfid));
-                        op_ret = -1;
-                        op_errno = ENOMEM;
-                        goto unlock;
-                }
-        }
+    int32_t ret = -1;
+    quota_local_t *local = NULL;
+    quota_inode_ctx_t *ctx = NULL;
+    quota_dentry_t *dentry = NULL;
+
+    local = frame->local;
+    if (op_ret < 0) {
+        goto unwind;
+    }
+
+    ret = quota_inode_ctx_get(inode, this, &ctx, 1);
+    if ((ret == -1) || (ctx == NULL)) {
+        gf_msg(this->name, GF_LOG_WARNING, ENOMEM, Q_MSG_INODE_CTX_GET_FAILED,
+               "cannot create quota "
+               "context in inode(gfid:%s)",
+               uuid_utoa(inode->gfid));
+        op_ret = -1;
+        op_errno = ENOMEM;
+        goto unwind;
+    }
+
+    LOCK(&ctx->lock);
+    {
+        ctx->buf = *buf;
+
+        dentry = __quota_dentry_new(ctx, (char *)local->loc.name,
+                                    local->loc.parent->gfid);
+        if (dentry == NULL) {
+            gf_msg(this->name, GF_LOG_WARNING, ENOMEM, Q_MSG_ENOMEM,
+                   "cannot create a new dentry "
+                   "(name:%s) for inode(gfid:%s)",
+                   local->loc.name, uuid_utoa(local->loc.inode->gfid));
+            op_ret = -1;
+            op_errno = ENOMEM;
+            goto unlock;
+        }
+    }
 unlock:
-        UNLOCK (&ctx->lock);
+    UNLOCK(&ctx->lock);
 
 unwind:
-        QUOTA_STACK_UNWIND (create, frame, op_ret, op_errno, fd, inode, buf,
-                            preparent, postparent, xdata);
-        return 0;
+    QUOTA_STACK_UNWIND(create, frame, op_ret, op_errno, fd, inode, buf,
+                       preparent, postparent, xdata);
+    return 0;
 }
 
-
 int32_t
-quota_create_helper (call_frame_t *frame, xlator_t *this, loc_t *loc,
-                     int32_t flags, mode_t mode, mode_t umask, fd_t *fd,
-                     dict_t *xdata)
+quota_create_helper(call_frame_t *frame, xlator_t *this, loc_t *loc,
+                    int32_t flags, mode_t mode, mode_t umask, fd_t *fd,
+                    dict_t *xdata)
 {
-        quota_local_t *local    = NULL;
-        int32_t        op_errno = EINVAL;
-        quota_priv_t  *priv     = NULL;
+    quota_local_t *local = NULL;
+    int32_t op_errno = EINVAL;
 
-        local = frame->local;
+    local = frame->local;
 
-        priv = this->private;
-
-        if (local == NULL) {
-                gf_log (this->name, GF_LOG_WARNING, "local is NULL");
-                goto unwind;
-        }
-
-        if (local->op_ret == -1) {
-                op_errno = local->op_errno;
-                goto unwind;
-        }
+    GF_VALIDATE_OR_GOTO("quota", local, unwind);
 
+    if (local->op_ret == -1) {
+        op_errno = local->op_errno;
+        goto unwind;
+    }
 
-        STACK_WIND (frame, quota_create_cbk,
-                    FIRST_CHILD (this), FIRST_CHILD (this)->fops->create, loc,
-                    flags, mode, umask, fd, xdata);
-        return 0;
+    STACK_WIND(frame, quota_create_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->create, loc, flags, mode, umask, fd,
+               xdata);
+    return 0;
 
 unwind:
-        QUOTA_STACK_UNWIND (create, frame, -1, op_errno, NULL, NULL,
-                            NULL, NULL, NULL, NULL);
-        return 0;
+    QUOTA_STACK_UNWIND(create, frame, -1, op_errno, NULL, NULL, NULL, NULL,
+                       NULL, NULL);
+    return 0;
 }
 
-
 int32_t
-quota_create (call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t flags,
-              mode_t mode, mode_t umask, fd_t *fd, dict_t *xdata)
+quota_create(call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t flags,
+             mode_t mode, mode_t umask, fd_t *fd, dict_t *xdata)
 {
-        quota_priv_t  *priv     = NULL;
-        int32_t        ret      = -1;
-        quota_local_t *local    = NULL;
-        int32_t        op_errno = 0;
-        call_stub_t   *stub     = NULL;
-
-        priv = this->private;
-
-        WIND_IF_QUOTAOFF (priv->is_quota_on, off);
-        QUOTA_WIND_FOR_INTERNAL_FOP (xdata, off);
-
-        local = quota_local_new ();
-        if (local == NULL) {
-                op_errno = ENOMEM;
-                goto err;
-        }
-
-        frame->local = local;
-
-        ret = loc_copy (&local->loc, loc);
-        if (ret) {
-                gf_log (this->name, GF_LOG_WARNING, "loc_copy failed");
-                op_errno = ENOMEM;
-                goto err;
-        }
-
-        stub = fop_create_stub (frame, quota_create_helper, loc, flags, mode,
-                                umask, fd, xdata);
-        if (stub == NULL) {
-                goto err;
-        }
-
-        LOCK (&local->lock);
-        {
-                local->link_count = 1;
-                local->stub = stub;
-                local->delta = 0;
-        }
-        UNLOCK (&local->lock);
+    quota_priv_t *priv = NULL;
+    int32_t ret = -1;
+    quota_local_t *local = NULL;
+    int32_t op_errno = 0;
+    call_stub_t *stub = NULL;
+
+    priv = this->private;
+
+    WIND_IF_QUOTAOFF(priv->is_quota_on, off);
+    QUOTA_WIND_FOR_INTERNAL_FOP(xdata, off);
+
+    local = quota_local_new();
+    if (local == NULL) {
+        op_errno = ENOMEM;
+        goto err;
+    }
+
+    frame->local = local;
+
+    ret = loc_copy(&local->loc, loc);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_WARNING, ENOMEM, Q_MSG_ENOMEM,
+               "loc_copy failed");
+        op_errno = ENOMEM;
+        goto err;
+    }
+
+    stub = fop_create_stub(frame, quota_create_helper, loc, flags, mode, umask,
+                           fd, xdata);
+    if (stub == NULL) {
+        goto err;
+    }
+
+    LOCK(&local->lock);
+    {
+        local->link_count = 1;
+        local->stub = stub;
+        local->delta = 0;
+        local->object_delta = 1;
+    }
+    UNLOCK(&local->lock);
 
-        quota_check_limit (frame, loc->parent, this, NULL, NULL);
-        return 0;
+    quota_check_limit(frame, loc->parent, this);
+    return 0;
 err:
-        QUOTA_STACK_UNWIND (create, frame, -1, op_errno, NULL, NULL, NULL,
-                            NULL, NULL, NULL);
+    QUOTA_STACK_UNWIND(create, frame, -1, op_errno, NULL, NULL, NULL, NULL,
+                       NULL, NULL);
 
-        return 0;
+    return 0;
 
 off:
-        STACK_WIND_TAIL (frame, FIRST_CHILD (this),
-                         FIRST_CHILD (this)->fops->create, loc,
-                         flags, mode, umask, fd, xdata);
-        return 0;
+    STACK_WIND_TAIL(frame, FIRST_CHILD(this), FIRST_CHILD(this)->fops->create,
+                    loc, flags, mode, umask, fd, xdata);
+    return 0;
 }
 
-
 int32_t
-quota_unlink_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                  int32_t op_ret, int32_t op_errno, struct iatt *preparent,
-                  struct iatt *postparent, dict_t *xdata)
+quota_unlink_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                 int32_t op_ret, int32_t op_errno, struct iatt *preparent,
+                 struct iatt *postparent, dict_t *xdata)
 {
-        quota_local_t     *local = NULL;
-        quota_inode_ctx_t *ctx   = NULL;
-        uint64_t           value = 0;
-        quota_dentry_t    *dentry = NULL;
-        quota_dentry_t    *old_dentry = NULL;
+    quota_local_t *local = NULL;
+    quota_inode_ctx_t *ctx = NULL;
+    uint64_t value = 0;
 
-        if (op_ret < 0) {
-                goto out;
-        }
+    if (op_ret < 0) {
+        goto out;
+    }
 
-        local = (quota_local_t *) frame->local;
+    local = (quota_local_t *)frame->local;
 
-        inode_ctx_get (local->loc.inode, this, &value);
-        ctx = (quota_inode_ctx_t *)(unsigned long)value;
+    inode_ctx_get(local->loc.inode, this, &value);
+    ctx = (quota_inode_ctx_t *)(unsigned long)value;
 
-        if (ctx == NULL) {
-                gf_log (this->name, GF_LOG_WARNING,
-                        "quota context not set in inode (gfid:%s)",
-                        uuid_utoa (local->loc.inode->gfid));
-                goto out;
-        }
+    if (ctx == NULL) {
+        gf_msg(this->name, GF_LOG_INFO, EINVAL, Q_MSG_INODE_CTX_GET_FAILED,
+               "quota context not set inode (gfid:%s)",
+               uuid_utoa(local->loc.gfid));
+        goto out;
+    }
 
-        LOCK (&ctx->lock);
-        {
-                list_for_each_entry (dentry, &ctx->parents, next) {
-                        if ((strcmp (dentry->name, local->loc.name) == 0) &&
-                            (gf_uuid_compare (local->loc.parent->gfid,
-                                           dentry->par) == 0)) {
-                                old_dentry = dentry;
-                                break;
-                        }
-                }
-                if (old_dentry)
-                        __quota_dentry_free (old_dentry);
-        }
-        UNLOCK (&ctx->lock);
+    quota_dentry_del(ctx, local->loc.name, local->loc.parent->gfid);
 
 out:
-        QUOTA_STACK_UNWIND (unlink, frame, op_ret, op_errno, preparent,
-                            postparent, xdata);
-        return 0;
+    QUOTA_STACK_UNWIND(unlink, frame, op_ret, op_errno, preparent, postparent,
+                       xdata);
+    return 0;
 }
 
-
 int32_t
-quota_unlink (call_frame_t *frame, xlator_t *this, loc_t *loc, int xflag,
-              dict_t *xdata)
+quota_unlink(call_frame_t *frame, xlator_t *this, loc_t *loc, int xflag,
+             dict_t *xdata)
 {
-        quota_priv_t       *priv        = NULL;
-        int32_t        ret = -1;
-        quota_local_t *local = NULL;
+    quota_priv_t *priv = NULL;
+    int32_t ret = -1;
+    quota_local_t *local = NULL;
 
-        priv = this->private;
+    priv = this->private;
 
-        WIND_IF_QUOTAOFF (priv->is_quota_on, off);
+    WIND_IF_QUOTAOFF(priv->is_quota_on, off);
 
-        local = quota_local_new ();
-        if (local == NULL) {
-                goto err;
-        }
+    local = quota_local_new();
+    if (local == NULL) {
+        goto err;
+    }
 
-        frame->local = local;
+    frame->local = local;
 
-        ret = loc_copy (&local->loc, loc);
-        if (ret) {
-                gf_log (this->name, GF_LOG_WARNING, "loc_copy failed");
-                goto err;
-        }
+    ret = loc_copy(&local->loc, loc);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_WARNING, ENOMEM, Q_MSG_ENOMEM,
+               "loc_copy failed");
+        goto err;
+    }
 
-        STACK_WIND (frame, quota_unlink_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->unlink, loc, xflag, xdata);
+    STACK_WIND(frame, quota_unlink_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->unlink, loc, xflag, xdata);
 
-        ret = 0;
+    ret = 0;
 
 err:
-        if (ret == -1) {
-                QUOTA_STACK_UNWIND (unlink, frame, -1, 0, NULL, NULL, NULL);
-        }
+    if (ret == -1) {
+        QUOTA_STACK_UNWIND(unlink, frame, -1, 0, NULL, NULL, NULL);
+    }
 
-        return 0;
+    return 0;
 
 off:
-        STACK_WIND_TAIL (frame, FIRST_CHILD(this),
-                         FIRST_CHILD(this)->fops->unlink, loc, xflag, xdata);
-        return 0;
+    STACK_WIND_TAIL(frame, FIRST_CHILD(this), FIRST_CHILD(this)->fops->unlink,
+                    loc, xflag, xdata);
+    return 0;
 }
 
-
 int32_t
-quota_link_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                int32_t op_ret, int32_t op_errno, inode_t *inode,
-                struct iatt *buf, struct iatt *preparent,
-                struct iatt *postparent, dict_t *xdata)
+quota_link_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+               int32_t op_ret, int32_t op_errno, inode_t *inode,
+               struct iatt *buf, struct iatt *preparent,
+               struct iatt *postparent, dict_t *xdata)
 {
-        int32_t               ret          = -1;
-        quota_local_t        *local        = NULL;
-        quota_inode_ctx_t    *ctx          = NULL;
-        quota_dentry_t       *dentry       = NULL;
-        char                  found        = 0;
-
-        if (op_ret < 0) {
-                goto out;
-        }
-
-        local = (quota_local_t *) frame->local;
-
-        ret = quota_inode_ctx_get (inode, this, &ctx, 0);
-        if ((ret == -1) || (ctx == NULL)) {
-                gf_log (this->name, GF_LOG_DEBUG, "quota context is NULL on "
-                        "inode (%s). "
-                        "If quota is not enabled recently and crawler has "
-                        "finished crawling, its an error",
-                        uuid_utoa (inode->gfid));
-                goto out;
-        }
-
-        LOCK (&ctx->lock);
+    int32_t ret = -1;
+    quota_local_t *local = NULL;
+    quota_inode_ctx_t *ctx = NULL;
+    quota_dentry_t *dentry = NULL;
+    char found = 0;
+
+    if (op_ret < 0) {
+        goto out;
+    }
+
+    local = (quota_local_t *)frame->local;
+
+    ret = quota_inode_ctx_get(inode, this, &ctx, 0);
+    if ((ret == -1) || (ctx == NULL)) {
+        gf_msg_debug(this->name, 0,
+                     "quota context is NULL on inode"
+                     " (%s). If quota is not enabled recently and "
+                     "crawler has finished crawling, its an error",
+                     uuid_utoa(inode->gfid));
+        goto out;
+    }
+
+    LOCK(&ctx->lock);
+    {
+        list_for_each_entry(dentry, &ctx->parents, next)
         {
-                list_for_each_entry (dentry, &ctx->parents, next) {
-                        if ((strcmp (dentry->name, local->loc.name) == 0) &&
-                            (gf_uuid_compare (local->loc.parent->gfid,
-                                           dentry->par) == 0)) {
-                                found = 1;
-                                gf_log (this->name, GF_LOG_WARNING,
-                                        "new entry being linked (name:%s) for "
-                                        "inode (gfid:%s) is already present "
-                                        "in inode-dentry-list", dentry->name,
-                                        uuid_utoa (local->loc.inode->gfid));
-                                break;
-                        }
-                }
-
-                if (!found) {
-                        dentry = __quota_dentry_new (ctx,
-                                                     (char *)local->loc.name,
-                                                     local->loc.parent->gfid);
-                        if (dentry == NULL) {
-                                gf_log (this->name, GF_LOG_WARNING,
-                                        "cannot create a new dentry (name:%s) "
-                                        "for inode(gfid:%s)", local->loc.name,
-                                        uuid_utoa (local->loc.inode->gfid));
-                                op_ret = -1;
-                                op_errno = ENOMEM;
-                                goto unlock;
-                        }
-                }
-
-                ctx->buf = *buf;
+            if ((strcmp(dentry->name, local->loc.name) == 0) &&
+                (gf_uuid_compare(local->loc.parent->gfid, dentry->par) == 0)) {
+                found = 1;
+
+                gf_msg_debug(this->name, 0,
+                             "new entry being"
+                             " linked (name:%s) for inode "
+                             "(gfid:%s) is already present "
+                             "in inode-dentry-list",
+                             dentry->name, uuid_utoa(local->loc.inode->gfid));
+                break;
+            }
+        }
+
+        if (!found) {
+            dentry = __quota_dentry_new(ctx, (char *)local->loc.name,
+                                        local->loc.parent->gfid);
+            if (dentry == NULL) {
+                gf_msg(this->name, GF_LOG_WARNING, ENOMEM, Q_MSG_ENOMEM,
+                       "cannot create a new dentry (name:%s)"
+                       "for inode(gfid:%s)",
+                       local->loc.name, uuid_utoa(local->loc.inode->gfid));
+                op_ret = -1;
+                op_errno = ENOMEM;
+                goto unlock;
+            }
         }
+
+        ctx->buf = *buf;
+    }
 unlock:
-        UNLOCK (&ctx->lock);
+    UNLOCK(&ctx->lock);
 
 out:
-        QUOTA_STACK_UNWIND (link, frame, op_ret, op_errno, inode, buf,
-                            preparent, postparent, xdata);
+    QUOTA_STACK_UNWIND(link, frame, op_ret, op_errno, inode, buf, preparent,
+                       postparent, xdata);
 
-        return 0;
+    return 0;
 }
 
-
 int32_t
-quota_link_helper (call_frame_t *frame, xlator_t *this, loc_t *oldloc,
-                   loc_t *newloc, dict_t *xdata)
+quota_link_helper(call_frame_t *frame, xlator_t *this, loc_t *oldloc,
+                  loc_t *newloc, dict_t *xdata)
 {
-        quota_local_t *local    = NULL;
-        int32_t        op_errno = EINVAL;
-        quota_priv_t  *priv     = NULL;
+    quota_local_t *local = NULL;
+    int32_t op_errno = EINVAL;
 
-        priv = this->private;
+    local = frame->local;
 
-        local = frame->local;
-        if (local == NULL) {
-                gf_log (this->name, GF_LOG_WARNING, "local is NULL");
-                goto unwind;
-        }
+    GF_VALIDATE_OR_GOTO("quota", local, unwind);
 
-        op_errno = local->op_errno;
+    op_errno = local->op_errno;
 
-        if (local->op_ret == -1) {
-                goto unwind;
-        }
+    if (local->op_ret == -1) {
+        goto unwind;
+    }
 
-        STACK_WIND (frame, quota_link_cbk,
-                    FIRST_CHILD(this), FIRST_CHILD(this)->fops->link, oldloc,
-                    newloc, xdata);
-        return 0;
+    STACK_WIND(frame, quota_link_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->link, oldloc, newloc, xdata);
+    return 0;
 
 unwind:
-        QUOTA_STACK_UNWIND (link, frame, -1, op_errno, NULL, NULL,
-                            NULL, NULL, NULL);
-        return 0;
+    QUOTA_STACK_UNWIND(link, frame, -1, op_errno, NULL, NULL, NULL, NULL, NULL);
+    return 0;
 }
 
 void
-quota_link_continue (call_frame_t *frame)
+quota_link_continue(call_frame_t *frame)
 {
-        int32_t            ret               = -1;
-        int32_t            op_errno          = EIO;
-        quota_local_t     *local             = NULL;
-        uuid_t             common_ancestor   = {0};
-        xlator_t          *this              = NULL;
-        quota_inode_ctx_t *ctx               = NULL;
-        inode_t           *src_parent        = NULL;
-        inode_t           *dst_parent        = NULL;
-
-        local = frame->local;
-        this = THIS;
-
-        if (local->op_ret < 0) {
-                op_errno = local->op_errno;
-                goto err;
-        }
-
-        if (local->xdata &&
-            dict_get (local->xdata, GLUSTERFS_INTERNAL_FOP_KEY)) {
-                /* Treat link as rename, crawl upwards only till common ancestor
-                 */
-                ret = quota_find_common_ancestor (local->oldloc.inode,
-                                                  local->newloc.parent,
-                                                  &common_ancestor);
-                if (ret < 0 || gf_uuid_is_null(common_ancestor)) {
-                        gf_log (this->name, GF_LOG_ERROR, "failed to get "
-                                "common_ancestor for %s and %s",
-                                local->oldloc.path, local->newloc.path);
-                        op_errno = ESTALE;
-                        goto err;
-                }
-        } else {
-                /* Treat link as a new file.
-                 * TODO: Currently marker accounts twice for the links created
-                 * across directories.
-                 * This needs re-vist if marker accounts only once
-                 * for the links created across directories
-                 */
-                src_parent = inode_parent (local->oldloc.inode, 0, NULL);
-                dst_parent = inode_parent (local->newloc.inode, 0, NULL);
-
-                /* No need to check quota limit if src and dst parents are same
-                 */
-                if (src_parent == dst_parent ||
-                    gf_uuid_compare (src_parent->gfid, dst_parent->gfid) == 0) {
-                        inode_unref (src_parent);
-                        inode_unref (dst_parent);
-                        goto off;
-                }
-
-                inode_unref (src_parent);
-                inode_unref (dst_parent);
-        }
+    int32_t ret = -1;
+    int32_t op_errno = EIO;
+    quota_local_t *local = NULL;
+    uuid_t common_ancestor = {0};
+    xlator_t *this = NULL;
+    quota_inode_ctx_t *ctx = NULL;
+    inode_t *src_parent = NULL;
+    inode_t *dst_parent = NULL;
+
+    local = frame->local;
+    this = THIS;
+
+    if (local->op_ret < 0) {
+        op_errno = local->op_errno;
+        goto err;
+    }
 
-        quota_inode_ctx_get (local->oldloc.inode, this, &ctx, 0);
-        if (ctx == NULL) {
-                gf_log (this->name, GF_LOG_DEBUG, "quota context is NULL on "
-                        "inode (%s). "
-                        "If quota is not enabled recently and crawler has "
-                        "finished crawling, its an error",
-                        uuid_utoa (local->oldloc.inode->gfid));
-        }
+    if (local->xdata && dict_get(local->xdata, GLUSTERFS_INTERNAL_FOP_KEY)) {
+        /* Treat link as rename, crawl upwards only till common ancestor
+         */
+        ret = quota_find_common_ancestor(
+            local->oldloc.inode, local->newloc.parent, &common_ancestor);
+        if (ret < 0 || gf_uuid_is_null(common_ancestor)) {
+            gf_msg(this->name, GF_LOG_ERROR, ESTALE,
+                   Q_MSG_ANCESTRY_BUILD_FAILED,
+                   "failed to get "
+                   "common_ancestor for %s and %s",
+                   local->oldloc.path, local->newloc.path);
+            op_errno = ESTALE;
+            goto err;
+        }
+    } else {
+        /* Treat link as a new file.
+         * TODO: Currently marker accounts twice for the links created
+         * across directories.
+         * This needs re-visit if marker accounts only once
+         * for the links created across directories
+         */
+        if (local->oldloc.parent)
+            src_parent = inode_ref(local->oldloc.parent);
+        else
+            src_parent = inode_parent(local->oldloc.inode, 0, NULL);
+        dst_parent = local->newloc.parent;
 
-        LOCK (&local->lock);
-        {
-                local->link_count = 1;
-                local->delta = (ctx != NULL) ? ctx->buf.ia_blocks * 512 : 0;
-                gf_uuid_copy (local->common_ancestor, common_ancestor);
-        }
-        UNLOCK (&local->lock);
+        /* No need to check quota limit if src and dst parents are same
+         */
+        if (src_parent == dst_parent ||
+            gf_uuid_compare(src_parent->gfid, dst_parent->gfid) == 0) {
+            inode_unref(src_parent);
+            goto wind;
+        }
+
+        inode_unref(src_parent);
+    }
+
+    quota_inode_ctx_get(local->oldloc.inode, this, &ctx, 0);
+    if (ctx == NULL) {
+        gf_msg_debug(this->name, 0,
+                     "quota context is NULL on inode"
+                     " (%s). If quota is not enabled recently and "
+                     "crawler has finished crawling, its an error",
+                     uuid_utoa(local->oldloc.inode->gfid));
+    }
+
+    LOCK(&local->lock);
+    {
+        local->link_count = 1;
+        local->delta = (ctx != NULL) ? ctx->buf.ia_blocks * 512 : 0;
+        local->object_delta = 1;
+        gf_uuid_copy(local->common_ancestor, common_ancestor);
+    }
+    UNLOCK(&local->lock);
 
-        quota_check_limit (frame, local->newloc.parent, this, NULL, NULL);
-        return;
+    quota_check_limit(frame, local->newloc.parent, this);
+    return;
 
 err:
-        QUOTA_STACK_UNWIND (link, frame, -1, op_errno, NULL, NULL,
-                            NULL, NULL, NULL);
-        return;
-
-off:
-        frame->local = NULL;
-
-        STACK_WIND_TAIL (frame, FIRST_CHILD(this),
-                         FIRST_CHILD(this)->fops->link, &(local->oldloc),
-                         &(local->newloc), local->xdata);
-
-        quota_local_cleanup (this, local);
-        return;
+    QUOTA_STACK_UNWIND(link, frame, -1, op_errno, NULL, NULL, NULL, NULL, NULL);
+    return;
+
+wind:
+    STACK_WIND(frame, quota_link_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->link, &(local->oldloc),
+               &(local->newloc), local->xdata);
+    return;
 }
 
 int32_t
-quota_link (call_frame_t *frame, xlator_t *this, loc_t *oldloc, loc_t *newloc,
-            dict_t *xdata)
+quota_link(call_frame_t *frame, xlator_t *this, loc_t *oldloc, loc_t *newloc,
+           dict_t *xdata)
 {
-        quota_priv_t      *priv     = NULL;
-        int32_t            ret      = -1;
-        int32_t            op_errno = ENOMEM;
-        quota_local_t     *local    = NULL;
-        call_stub_t       *stub     = NULL;
-
-        priv = this->private;
-
-        WIND_IF_QUOTAOFF (priv->is_quota_on, off);
-
-        /* No need to check quota limit if src and dst parents are same */
-        if (oldloc->parent && newloc->parent &&
-            !gf_uuid_compare(oldloc->parent->gfid, newloc->parent->gfid)) {
-                gf_log (this->name, GF_LOG_DEBUG, "link %s -> %s are "
-                        "in the same directory, so skip check limit",
-                        oldloc->path, newloc->path);
-                goto off;
-        }
-
-        local = quota_local_new ();
-        if (local == NULL) {
-                goto err;
-        }
-
-        frame->local = (void *) local;
-
-        if (xdata)
-                local->xdata = dict_ref (xdata);
-
-        ret = loc_copy (&local->loc, newloc);
-        if (ret == -1) {
-                gf_log (this->name, GF_LOG_WARNING, "loc_copy failed");
-                goto err;
-        }
-
-        ret = loc_copy (&local->oldloc, oldloc);
-        if (ret < 0) {
-                gf_log (this->name, GF_LOG_WARNING, "loc_copy failed");
-                goto err;
-        }
-
-        ret = loc_copy (&local->newloc, newloc);
-        if (ret < 0) {
-                gf_log (this->name, GF_LOG_WARNING, "loc_copy failed");
-                goto err;
-        }
+    quota_priv_t *priv = NULL;
+    int32_t ret = -1;
+    int32_t op_errno = ENOMEM;
+    quota_local_t *local = NULL;
+    call_stub_t *stub = NULL;
+
+    priv = this->private;
+
+    WIND_IF_QUOTAOFF(priv->is_quota_on, off);
+
+    local = quota_local_new();
+    if (local == NULL) {
+        goto err;
+    }
+
+    frame->local = (void *)local;
+
+    if (xdata)
+        local->xdata = dict_ref(xdata);
+
+    ret = loc_copy(&local->loc, newloc);
+    if (ret == -1) {
+        gf_msg(this->name, GF_LOG_WARNING, ENOMEM, Q_MSG_ENOMEM,
+               "loc_copy failed");
+        goto err;
+    }
+
+    ret = loc_copy(&local->oldloc, oldloc);
+    if (ret < 0) {
+        gf_msg(this->name, GF_LOG_WARNING, ENOMEM, Q_MSG_ENOMEM,
+               "loc_copy failed");
+        goto err;
+    }
+
+    ret = loc_copy(&local->newloc, newloc);
+    if (ret < 0) {
+        gf_msg(this->name, GF_LOG_WARNING, ENOMEM, Q_MSG_ENOMEM,
+               "loc_copy failed");
+        goto err;
+    }
+
+    /* No need to check quota limit if src and dst parents are same */
+    if (oldloc->parent && newloc->parent &&
+        !gf_uuid_compare(oldloc->parent->gfid, newloc->parent->gfid)) {
+        gf_msg_debug(this->name, GF_LOG_DEBUG,
+                     "link %s -> %s are "
+                     "in the same directory, so skip check limit",
+                     oldloc->path, newloc->path);
+        goto wind;
+    }
+
+    stub = fop_link_stub(frame, quota_link_helper, oldloc, newloc, xdata);
+    if (stub == NULL) {
+        goto err;
+    }
+
+    LOCK(&local->lock);
+    {
+        local->link_count = 2;
+        local->fop_continue_cbk = quota_link_continue;
+        local->stub = stub;
+    }
+    UNLOCK(&local->lock);
 
-        stub = fop_link_stub (frame, quota_link_helper, oldloc, newloc, xdata);
-        if (stub == NULL) {
-                goto err;
-        }
+    check_ancestory(frame, newloc->parent);
 
-        LOCK (&local->lock);
-        {
-                local->link_count = 2;
-                local->fop_continue_cbk = quota_link_continue;
-                local->stub = stub;
-        }
-        UNLOCK (&local->lock);
+    /* source parent can be NULL, so do check_ancestry on a file */
+    if (oldloc->parent)
+        check_ancestory(frame, oldloc->parent);
+    else
+        check_ancestory(frame, oldloc->inode);
 
-        check_ancestory (frame, newloc->parent);
-
-        /* source parent can be NULL, so do check_ancestory on a file */
-        check_ancestory (frame, oldloc->inode);
-        return 0;
+    return 0;
 
 err:
-        QUOTA_STACK_UNWIND (link, frame, -1, op_errno, NULL, NULL,
-                            NULL, NULL, NULL);
-        return 0;
+    QUOTA_STACK_UNWIND(link, frame, -1, op_errno, NULL, NULL, NULL, NULL, NULL);
+    return 0;
 
 off:
-        STACK_WIND_TAIL (frame, FIRST_CHILD(this),
-                         FIRST_CHILD(this)->fops->link, oldloc,
-                         newloc, xdata);
-        return 0;
+    STACK_WIND_TAIL(frame, FIRST_CHILD(this), FIRST_CHILD(this)->fops->link,
+                    oldloc, newloc, xdata);
+    return 0;
+
+wind:
+    STACK_WIND(frame, quota_link_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->link, oldloc, newloc, xdata);
+    return 0;
 }
 
-
 int32_t
-quota_rename_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                  int32_t op_ret, int32_t op_errno, struct iatt *buf,
-                  struct iatt *preoldparent, struct iatt *postoldparent,
-                  struct iatt *prenewparent, struct iatt *postnewparent,
-                  dict_t *xdata)
+quota_rename_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                 int32_t op_ret, int32_t op_errno, struct iatt *buf,
+                 struct iatt *preoldparent, struct iatt *postoldparent,
+                 struct iatt *prenewparent, struct iatt *postnewparent,
+                 dict_t *xdata)
 {
-        int32_t               ret              = -1;
-        int64_t               size             = 0;
-        quota_local_t        *local            = NULL;
-        quota_inode_ctx_t    *ctx              = NULL;
-        quota_dentry_t       *old_dentry       = NULL, *dentry = NULL;
-        char                  new_dentry_found = 0;
+    int32_t ret = -1;
+    quota_local_t *local = NULL;
+    quota_inode_ctx_t *ctx = NULL;
+    quota_dentry_t *old_dentry = NULL, *dentry = NULL;
+    char new_dentry_found = 0;
 
-        if (op_ret < 0) {
-                goto out;
-        }
+    if (op_ret < 0) {
+        goto out;
+    }
 
-        local = frame->local;
-        if (local == NULL) {
-                op_ret = -1;
-                op_errno = EINVAL;
-                gf_log (this->name, GF_LOG_WARNING, "local is NULL");
-                goto out;
-        }
+    local = frame->local;
 
-        if (QUOTA_REG_OR_LNK_FILE (local->oldloc.inode->ia_type)) {
-                size = buf->ia_blocks * 512;
-        }
+    GF_VALIDATE_OR_GOTO("quota", local, out);
 
-        if (!QUOTA_REG_OR_LNK_FILE (local->oldloc.inode->ia_type)) {
-                goto out;
-        }
+    if (!QUOTA_REG_OR_LNK_FILE(local->oldloc.inode->ia_type))
+        goto out;
 
-        ret = quota_inode_ctx_get (local->oldloc.inode, this, &ctx, 0);
-        if ((ret == -1) || (ctx == NULL)) {
-                gf_log (this->name, GF_LOG_DEBUG, "quota context is NULL on "
-                        "inode (%s). "
-                        "If quota is not enabled recently and crawler has "
-                        "finished crawling, its an error",
-                        uuid_utoa (local->oldloc.inode->gfid));
+    ret = quota_inode_ctx_get(local->oldloc.inode, this, &ctx, 0);
+    if ((ret == -1) || (ctx == NULL)) {
+        gf_msg_debug(this->name, 0,
+                     "quota context is NULL on inode"
+                     " (%s). If quota is not enabled recently and "
+                     "crawler has finished crawling, its an error",
+                     uuid_utoa(local->oldloc.inode->gfid));
 
-                goto out;
-        }
+        goto out;
+    }
 
-        LOCK (&ctx->lock);
+    LOCK(&ctx->lock);
+    {
+        list_for_each_entry(dentry, &ctx->parents, next)
         {
-                list_for_each_entry (dentry, &ctx->parents, next) {
-                        if ((strcmp (dentry->name, local->oldloc.name) == 0) &&
-                            (gf_uuid_compare (local->oldloc.parent->gfid,
-                                           dentry->par) == 0)) {
-                                old_dentry = dentry;
-                        } else if ((strcmp (dentry->name,
-                                            local->newloc.name) == 0) &&
-                                   (gf_uuid_compare (local->newloc.parent->gfid,
-                                                  dentry->par) == 0)) {
-                                new_dentry_found = 1;
-                                gf_log (this->name, GF_LOG_WARNING,
-                                        "new entry being linked (name:%s) for "
-                                        "inode (gfid:%s) is already present "
-                                        "in inode-dentry-list", dentry->name,
-                                        uuid_utoa (local->oldloc.inode->gfid));
-                        }
-
-                        if (old_dentry && new_dentry_found)
-                                break;
-                }
-
-                if (old_dentry != NULL) {
-                        __quota_dentry_free (old_dentry);
-                } else {
-                        gf_log (this->name, GF_LOG_WARNING,
-                                "dentry corresponding to the path just renamed "
-                                "(name:%s) is not present", local->oldloc.name);
-                }
-
-                if (!new_dentry_found) {
-                        dentry = __quota_dentry_new (ctx,
-                                                     (char *)local->newloc.name,
-                                                     local->newloc.parent->gfid);
-                        if (dentry == NULL) {
-                                gf_log (this->name, GF_LOG_WARNING,
-                                        "cannot create a new dentry (name:%s) "
-                                        "for inode(gfid:%s)",
-                                        local->newloc.name,
-                                        uuid_utoa (local->oldloc.inode->gfid));
-                                op_ret = -1;
-                                op_errno = ENOMEM;
-                                goto unlock;
-                        }
-                }
-
-                ctx->buf = *buf;
+            if ((strcmp(dentry->name, local->oldloc.name) == 0) &&
+                (gf_uuid_compare(local->oldloc.parent->gfid, dentry->par) ==
+                 0)) {
+                old_dentry = dentry;
+            } else if ((strcmp(dentry->name, local->newloc.name) == 0) &&
+                       (gf_uuid_compare(local->newloc.parent->gfid,
+                                        dentry->par) == 0)) {
+                new_dentry_found = 1;
+                gf_msg_debug(this->name, 0,
+                             "new entry being "
+                             "linked (name:%s) for inode (gfid:%s) "
+                             "is in inode-dentry-list",
+                             dentry->name,
+                             uuid_utoa(local->oldloc.inode->gfid));
+            }
+
+            if (old_dentry && new_dentry_found)
+                break;
+        }
+
+        if (old_dentry != NULL) {
+            __quota_dentry_free(old_dentry);
+        } else {
+            gf_msg_debug(this->name, 0,
+                         "dentry corresponding"
+                         "the path just renamed (name:%s) is not"
+                         " present",
+                         local->oldloc.name);
+        }
+
+        if (!new_dentry_found) {
+            dentry = __quota_dentry_new(ctx, (char *)local->newloc.name,
+                                        local->newloc.parent->gfid);
+            if (dentry == NULL) {
+                gf_msg(this->name, GF_LOG_WARNING, ENOMEM, Q_MSG_ENOMEM,
+                       "cannot create a new dentry (name:%s) "
+                       "for inode(gfid:%s)",
+                       local->newloc.name,
+                       uuid_utoa(local->newloc.inode->gfid));
+                op_ret = -1;
+                op_errno = ENOMEM;
+                goto unlock;
+            }
         }
+
+        ctx->buf = *buf;
+    }
 unlock:
-        UNLOCK (&ctx->lock);
+    UNLOCK(&ctx->lock);
 
 out:
-        QUOTA_STACK_UNWIND (rename, frame, op_ret, op_errno, buf, preoldparent,
-                            postoldparent, prenewparent, postnewparent, xdata);
+    QUOTA_STACK_UNWIND(rename, frame, op_ret, op_errno, buf, preoldparent,
+                       postoldparent, prenewparent, postnewparent, xdata);
 
-        return 0;
+    return 0;
 }
 
-
 int32_t
-quota_rename_helper (call_frame_t *frame, xlator_t *this, loc_t *oldloc,
-                     loc_t *newloc, dict_t *xdata)
+quota_rename_helper(call_frame_t *frame, xlator_t *this, loc_t *oldloc,
+                    loc_t *newloc, dict_t *xdata)
 {
-        quota_local_t *local    = NULL;
-        int32_t        op_errno = EINVAL;
-        quota_priv_t  *priv     = NULL;
+    quota_local_t *local = NULL;
+    int32_t op_errno = EINVAL;
 
-        priv = this->private;
+    local = frame->local;
 
-        local = frame->local;
-        if (local == NULL) {
-                gf_log (this->name, GF_LOG_WARNING, "local is NULL");
-                goto unwind;
-        }
+    GF_VALIDATE_OR_GOTO("quota", local, unwind);
 
-        op_errno = local->op_errno;
+    op_errno = local->op_errno;
 
-        if (local->op_ret == -1) {
-                goto unwind;
-        }
+    if (local->op_ret == -1) {
+        goto unwind;
+    }
 
-        STACK_WIND (frame, quota_rename_cbk,
-                    FIRST_CHILD(this), FIRST_CHILD(this)->fops->rename, oldloc,
-                    newloc, xdata);
+    STACK_WIND(frame, quota_rename_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->rename, oldloc, newloc, xdata);
 
-        return 0;
+    return 0;
 
 unwind:
-        QUOTA_STACK_UNWIND (rename, frame, -1, op_errno, NULL, NULL,
-                            NULL, NULL, NULL, NULL);
-        return 0;
+    QUOTA_STACK_UNWIND(rename, frame, -1, op_errno, NULL, NULL, NULL, NULL,
+                       NULL, NULL);
+    return 0;
 }
 
-
 static int32_t
-quota_rename_get_size_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                           int32_t op_ret, int32_t op_errno, inode_t *inode,
-                           struct iatt *buf, dict_t *xdata,
-                           struct iatt *postparent)
+quota_rename_get_size_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                          int32_t op_ret, int32_t op_errno, inode_t *inode,
+                          struct iatt *buf, dict_t *xdata,
+                          struct iatt *postparent)
 {
-        quota_local_t     *local      = NULL;
-        int32_t            ret        = 0;
-        int64_t           *size       = 0;
-
-        GF_ASSERT (frame);
-        GF_VALIDATE_OR_GOTO_WITH_ERROR ("quota", this, out, op_errno,
-                                        EINVAL);
-        GF_VALIDATE_OR_GOTO_WITH_ERROR (this->name, xdata, out, op_errno,
-                                        EINVAL);
-        local = frame->local;
-        GF_ASSERT (local);
-        local->link_count = 1;
-
-        if (op_ret < 0)
-                goto out;
-
-
-        ret = dict_get_bin (xdata, QUOTA_SIZE_KEY, (void **) &size);
-        if (ret < 0) {
-                gf_log (this->name, GF_LOG_WARNING,
-                        "size key not present in dict");
-                op_errno = EINVAL;
-                goto out;
-        }
-        local->delta = ntoh64 (*size);
-        quota_check_limit (frame, local->newloc.parent, this,
-                           NULL, NULL);
-        return 0;
+    quota_local_t *local = NULL;
+    int32_t ret = 0;
+    int64_t *size = 0;
+
+    GF_ASSERT(frame);
+    GF_VALIDATE_OR_GOTO_WITH_ERROR("quota", this, out, op_errno, EINVAL);
+    GF_VALIDATE_OR_GOTO_WITH_ERROR(this->name, xdata, out, op_errno, EINVAL);
+    local = frame->local;
+    GF_ASSERT(local);
+    local->link_count = 1;
+
+    if (op_ret < 0)
+        goto out;
+
+    ret = dict_get_bin(xdata, QUOTA_SIZE_KEY, (void **)&size);
+    if (ret < 0) {
+        gf_msg(this->name, GF_LOG_WARNING, EINVAL, Q_MSG_SIZE_KEY_MISSING,
+               "size key not present in dict");
+        op_errno = EINVAL;
+        goto out;
+    }
+    local->delta = ntoh64(*size);
+    local->object_delta = 1;
+    quota_check_limit(frame, local->newloc.parent, this);
+    return 0;
 
 out:
-        quota_handle_validate_error (local, -1, op_errno);
-        return 0;
+    quota_handle_validate_error(frame, -1, op_errno);
+    return 0;
 }
 
 void
-quota_rename_continue (call_frame_t *frame)
+quota_rename_continue(call_frame_t *frame)
 {
-        int32_t            ret               = -1;
-        int32_t            op_errno          = EIO;
-        quota_local_t     *local             = NULL;
-        uuid_t             common_ancestor   = {0};
-        xlator_t          *this              = NULL;
-        quota_inode_ctx_t *ctx               = NULL;
-
-        local = frame->local;
-        this = THIS;
-
-        if (local->op_ret < 0) {
-                op_errno = local->op_errno;
-                goto err;
-        }
+    int32_t ret = -1;
+    int32_t op_errno = EIO;
+    quota_local_t *local = NULL;
+    uuid_t common_ancestor = {0};
+    xlator_t *this = NULL;
+    quota_inode_ctx_t *ctx = NULL;
 
-        ret = quota_find_common_ancestor (local->oldloc.parent,
-                                          local->newloc.parent,
-                                          &common_ancestor);
-        if (ret < 0 || gf_uuid_is_null(common_ancestor)) {
-                gf_log (this->name, GF_LOG_ERROR, "failed to get "
-                        "common_ancestor for %s and %s",
-                        local->oldloc.path, local->newloc.path);
-                op_errno = ESTALE;
-                goto err;
-        }
-
-        LOCK (&local->lock);
-        {
-                local->link_count = 1;
-                gf_uuid_copy (local->common_ancestor, common_ancestor);
-        }
-        UNLOCK (&local->lock);
-
-        if (QUOTA_REG_OR_LNK_FILE (local->oldloc.inode->ia_type)) {
-                ret = quota_inode_ctx_get (local->oldloc.inode, this, &ctx, 0);
-                if (ctx == NULL) {
-                        gf_log (this->name, GF_LOG_WARNING,
-                                "quota context not set in inode (gfid:%s), "
-                                "considering file size as zero while enforcing "
-                                "quota on new ancestry",
-                                uuid_utoa (local->oldloc.inode->gfid));
-                        local->delta = 0;
-                } else {
+    local = frame->local;
+    this = THIS;
 
-                    /* FIXME: We need to account for the size occupied by this
-                     * inode on the target directory. To avoid double
-                     * accounting, we need to modify enforcer to perform
-                     * quota_check_limit only uptil the least common ancestor
-                     * directory inode*/
+    if (local->op_ret < 0) {
+        op_errno = local->op_errno;
+        goto err;
+    }
+
+    ret = quota_find_common_ancestor(local->oldloc.parent, local->newloc.parent,
+                                     &common_ancestor);
+    if (ret < 0 || gf_uuid_is_null(common_ancestor)) {
+        gf_msg(this->name, GF_LOG_ERROR, ESTALE, Q_MSG_ANCESTRY_BUILD_FAILED,
+               "failed to get "
+               "common_ancestor for %s and %s",
+               local->oldloc.path, local->newloc.path);
+        op_errno = ESTALE;
+        goto err;
+    }
+
+    LOCK(&local->lock);
+    {
+        local->link_count = 1;
+        gf_uuid_copy(local->common_ancestor, common_ancestor);
+    }
+    UNLOCK(&local->lock);
 
-                    /* FIXME: The following code assumes that regular files and
-                     * linkfiles are present, in their entirety, in a single
-                     * brick. This *assumption is invalid in the case of
-                     * stripe.*/
+    if (QUOTA_REG_OR_LNK_FILE(local->oldloc.inode->ia_type)) {
+        ret = quota_inode_ctx_get(local->oldloc.inode, this, &ctx, 0);
+        if (ctx == NULL) {
+            gf_msg(this->name, GF_LOG_WARNING, 0, Q_MSG_INODE_CTX_GET_FAILED,
+                   "quota context not set in inode (gfid:%s), "
+                   "considering file size as zero while enforcing "
+                   "quota on new ancestry",
+                   uuid_utoa(local->oldloc.inode->gfid));
+
+            local->delta = 0;
+            local->object_delta = 1;
+        } else {
+            /* FIXME: We need to account for the size occupied by
+             * this inode on the target directory. To avoid double
+             * accounting, we need to modify enforcer to perform
+             * quota_check_limit only up till the least common
+             * ancestor directory inode*/
 
-                    local->delta = ctx->buf.ia_blocks * 512;
-                }
+            /* FIXME: The following code assumes that regular files
+             * and link files are present, in their entirety, in a
+             * single brick. This *assumption is invalid in the
+             * case of stripe.*/
 
-        } else if (IA_ISDIR (local->oldloc.inode->ia_type)) {
-                        ret = quota_validate (frame, local->oldloc.inode, this,
-                                              quota_rename_get_size_cbk);
-                        if (ret){
-                                op_errno = -ret;
-                                goto err;
-                        }
+            local->delta = ctx->buf.ia_blocks * 512;
+            local->object_delta = 1;
+        }
 
-                        return;
+    } else if (IA_ISDIR(local->oldloc.inode->ia_type)) {
+        ret = quota_validate(frame, local->oldloc.inode, this,
+                             quota_rename_get_size_cbk);
+        if (ret) {
+            op_errno = -ret;
+            goto err;
         }
 
-        quota_check_limit (frame, local->newloc.parent, this, NULL, NULL);
         return;
+    }
 
-err:
-        QUOTA_STACK_UNWIND (rename, frame, -1, op_errno, NULL,
-                            NULL, NULL, NULL, NULL, NULL);
-        return;
+    quota_check_limit(frame, local->newloc.parent, this);
+    return;
 
+err:
+    QUOTA_STACK_UNWIND(rename, frame, -1, op_errno, NULL, NULL, NULL, NULL,
+                       NULL, NULL);
+    return;
 }
 
 int32_t
-quota_rename (call_frame_t *frame, xlator_t *this, loc_t *oldloc,
-              loc_t *newloc, dict_t *xdata)
+quota_rename(call_frame_t *frame, xlator_t *this, loc_t *oldloc, loc_t *newloc,
+             dict_t *xdata)
 {
-        quota_priv_t      *priv              = NULL;
-        int32_t            ret               = -1;
-        int32_t            op_errno          = ENOMEM;
-        quota_local_t     *local             = NULL;
-        call_stub_t       *stub              = NULL;
-
-        priv = this->private;
-
-        WIND_IF_QUOTAOFF (priv->is_quota_on, off);
-
-        /* No need to check quota limit if src and dst parents are same */
-        if (oldloc->parent && newloc->parent &&
-            !gf_uuid_compare(oldloc->parent->gfid, newloc->parent->gfid)) {
-                gf_log (this->name, GF_LOG_DEBUG, "rename %s -> %s are "
-                        "in the same directory, so skip check limit",
-                        oldloc->path, newloc->path);
-                goto off;
-        }
-
-        local = quota_local_new ();
-        if (local == NULL) {
-                goto err;
-        }
-
-        frame->local = local;
-
-        ret = loc_copy (&local->oldloc, oldloc);
-        if (ret < 0) {
-                gf_log (this->name, GF_LOG_WARNING, "loc_copy failed");
-                goto err;
-        }
-
-        ret = loc_copy (&local->newloc, newloc);
-        if (ret < 0) {
-                gf_log (this->name, GF_LOG_WARNING, "loc_copy failed");
-                goto err;
-        }
-
-        stub = fop_rename_stub (frame, quota_rename_helper, oldloc, newloc,
-                                xdata);
-        if (stub == NULL) {
-                goto err;
-        }
-
-        LOCK (&local->lock);
-        {
-                /* link_count here tell how many check_ancestory should be done
-                 * before continuing the FOP
-                 */
-                local->link_count = 2;
-                local->stub = stub;
-                local->fop_continue_cbk = quota_rename_continue;
-        }
-        UNLOCK (&local->lock);
+    quota_priv_t *priv = NULL;
+    int32_t ret = -1;
+    int32_t op_errno = ENOMEM;
+    quota_local_t *local = NULL;
+    call_stub_t *stub = NULL;
+
+    priv = this->private;
+
+    WIND_IF_QUOTAOFF(priv->is_quota_on, off);
+
+    local = quota_local_new();
+    if (local == NULL) {
+        goto err;
+    }
+
+    frame->local = local;
+
+    ret = loc_copy(&local->oldloc, oldloc);
+    if (ret < 0) {
+        gf_msg(this->name, GF_LOG_WARNING, ENOMEM, Q_MSG_ENOMEM,
+               "loc_copy failed");
+        goto err;
+    }
+
+    ret = loc_copy(&local->newloc, newloc);
+    if (ret < 0) {
+        gf_msg(this->name, GF_LOG_WARNING, ENOMEM, Q_MSG_ENOMEM,
+               "loc_copy failed");
+        goto err;
+    }
+
+    /* No need to check quota limit if src and dst parents are same */
+    if (oldloc->parent && newloc->parent &&
+        !gf_uuid_compare(oldloc->parent->gfid, newloc->parent->gfid)) {
+        gf_msg_debug(this->name, 0,
+                     "rename %s -> %s are "
+                     "in the same directory, so skip check limit",
+                     oldloc->path, newloc->path);
+        goto wind;
+    }
+
+    stub = fop_rename_stub(frame, quota_rename_helper, oldloc, newloc, xdata);
+    if (stub == NULL) {
+        goto err;
+    }
+
+    LOCK(&local->lock);
+    {
+        /* link_count here tell how many check_ancestry should be done
+         * before continuing the FOP
+         */
+        local->link_count = 2;
+        local->stub = stub;
+        local->fop_continue_cbk = quota_rename_continue;
+    }
+    UNLOCK(&local->lock);
 
-        check_ancestory (frame, newloc->parent);
-        check_ancestory (frame, oldloc->parent);
-        return 0;
+    check_ancestory(frame, newloc->parent);
+    check_ancestory(frame, oldloc->parent);
+    return 0;
 
 err:
-        QUOTA_STACK_UNWIND (rename, frame, -1, op_errno, NULL,
-                            NULL, NULL, NULL, NULL, NULL);
-        return 0;
+    QUOTA_STACK_UNWIND(rename, frame, -1, op_errno, NULL, NULL, NULL, NULL,
+                       NULL, NULL);
+    return 0;
 
 off:
-        STACK_WIND_TAIL (frame, FIRST_CHILD(this),
-                         FIRST_CHILD(this)->fops->rename, oldloc,
-                         newloc, xdata);
-
-        return 0;
+    STACK_WIND_TAIL(frame, FIRST_CHILD(this), FIRST_CHILD(this)->fops->rename,
+                    oldloc, newloc, xdata);
+    return 0;
+
+wind:
+    STACK_WIND(frame, quota_rename_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->rename, oldloc, newloc, xdata);
+    return 0;
 }
 
-
 int32_t
-quota_symlink_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                   int32_t op_ret, int32_t op_errno, inode_t *inode,
-                   struct iatt *buf, struct iatt *preparent,
-                   struct iatt *postparent, dict_t *xdata)
+quota_symlink_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                  int32_t op_ret, int32_t op_errno, inode_t *inode,
+                  struct iatt *buf, struct iatt *preparent,
+                  struct iatt *postparent, dict_t *xdata)
 {
-        quota_local_t     *local  = NULL;
-        quota_inode_ctx_t *ctx    = NULL;
-        quota_dentry_t    *dentry = NULL;
-
-        if (op_ret < 0) {
-                goto out;
-        }
-
-        local = frame->local;
-
-        quota_inode_ctx_get (local->loc.inode, this, &ctx, 1);
-        if (ctx == NULL) {
-                gf_log (this->name, GF_LOG_DEBUG, "quota context is NULL on "
-                        "inode (%s). "
-                        "If quota is not enabled recently and crawler has "
-                        "finished crawling, its an error",
-                        uuid_utoa (local->loc.inode->gfid));
-
-                goto out;
-        }
-
-        LOCK (&ctx->lock);
-        {
-                ctx->buf = *buf;
-
-                dentry = __quota_dentry_new (ctx, (char *)local->loc.name,
-                                             local->loc.parent->gfid);
-                if (dentry == NULL) {
-                        gf_log (this->name, GF_LOG_WARNING,
-                                "cannot create a new dentry (name:%s) for "
-                                "inode(gfid:%s)", local->loc.name,
-                                uuid_utoa (local->loc.inode->gfid));
-                        op_ret = -1;
-                        op_errno = ENOMEM;
-                }
+    quota_local_t *local = NULL;
+    quota_inode_ctx_t *ctx = NULL;
+    quota_dentry_t *dentry = NULL;
+    int32_t ret = -1;
+
+    if (op_ret < 0) {
+        goto out;
+    }
+
+    local = frame->local;
+
+    ret = quota_inode_ctx_get(local->loc.inode, this, &ctx, 1);
+    if ((ret == -1) || (ctx == NULL)) {
+        gf_msg_debug(this->name, 0,
+                     "quota context is NULL on inode"
+                     " (%s). If quota is not enabled recently and "
+                     "crawler has finished crawling, its an error",
+                     uuid_utoa(local->loc.inode->gfid));
+
+        goto out;
+    }
+
+    LOCK(&ctx->lock);
+    {
+        ctx->buf = *buf;
+
+        dentry = __quota_dentry_new(ctx, (char *)local->loc.name,
+                                    local->loc.parent->gfid);
+        if (dentry == NULL) {
+            gf_msg(this->name, GF_LOG_WARNING, ENOMEM, Q_MSG_ENOMEM,
+                   "cannot create "
+                   "a new dentry (name:%s) for inode(gfid:%s)",
+                   local->loc.name, uuid_utoa(local->loc.inode->gfid));
+            op_ret = -1;
+            op_errno = ENOMEM;
         }
-        UNLOCK (&ctx->lock);
+    }
+    UNLOCK(&ctx->lock);
 
 out:
-        QUOTA_STACK_UNWIND (symlink, frame, op_ret, op_errno, inode, buf,
-                            preparent, postparent, xdata);
+    QUOTA_STACK_UNWIND(symlink, frame, op_ret, op_errno, inode, buf, preparent,
+                       postparent, xdata);
 
-        return 0;
+    return 0;
 }
 
-
 int
-quota_symlink_helper (call_frame_t *frame, xlator_t *this, const char *linkpath,
-                      loc_t *loc, mode_t umask, dict_t *xdata)
+quota_symlink_helper(call_frame_t *frame, xlator_t *this, const char *linkpath,
+                     loc_t *loc, mode_t umask, dict_t *xdata)
 {
-        quota_local_t *local    = NULL;
-        int32_t        op_errno = EINVAL;
-        quota_priv_t  *priv     = NULL;
+    quota_local_t *local = NULL;
+    int32_t op_errno = EINVAL;
 
-        local = frame->local;
-        if (local == NULL) {
-                gf_log (this->name, GF_LOG_WARNING, "local is NULL");
-                goto unwind;
-        }
+    local = frame->local;
 
-        priv = this->private;
+    GF_VALIDATE_OR_GOTO("quota", local, unwind);
 
-        if (local->op_ret == -1) {
-                op_errno = local->op_errno;
-                goto unwind;
-        }
+    if (local->op_ret == -1) {
+        op_errno = local->op_errno;
+        goto unwind;
+    }
 
-        STACK_WIND (frame, quota_symlink_cbk,
-                    FIRST_CHILD(this), FIRST_CHILD(this)->fops->symlink,
-                    linkpath, loc, umask, xdata);
-        return 0;
+    STACK_WIND(frame, quota_symlink_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->symlink, linkpath, loc, umask, xdata);
+    return 0;
 
 unwind:
-        QUOTA_STACK_UNWIND (symlink, frame, -1, op_errno, NULL, NULL,
-                            NULL, NULL, NULL);
-        return 0;
+    QUOTA_STACK_UNWIND(symlink, frame, -1, op_errno, NULL, NULL, NULL, NULL,
+                       NULL);
+    return 0;
 }
 
-
 int
-quota_symlink (call_frame_t *frame, xlator_t *this, const char *linkpath,
-               loc_t *loc, mode_t umask, dict_t *xdata)
+quota_symlink(call_frame_t *frame, xlator_t *this, const char *linkpath,
+              loc_t *loc, mode_t umask, dict_t *xdata)
 {
-        quota_priv_t  *priv     = NULL;
-        int32_t        ret      = -1;
-        int32_t        op_errno = ENOMEM;
-        quota_local_t *local    = NULL;
-        call_stub_t   *stub     = NULL;
+    quota_priv_t *priv = NULL;
+    int32_t ret = -1;
+    int32_t op_errno = ENOMEM;
+    quota_local_t *local = NULL;
+    call_stub_t *stub = NULL;
 
-        priv = this->private;
+    priv = this->private;
 
-        WIND_IF_QUOTAOFF (priv->is_quota_on, off);
+    WIND_IF_QUOTAOFF(priv->is_quota_on, off);
 
-        local = quota_local_new ();
-        if (local == NULL) {
-                goto err;
-        }
+    local = quota_local_new();
+    if (local == NULL) {
+        goto err;
+    }
 
-        frame->local = local;
+    frame->local = local;
 
-        ret = loc_copy (&local->loc, loc);
-        if (ret < 0) {
-                gf_log (this->name, GF_LOG_WARNING, "loc_copy failed");
-                goto err;
-        }
+    ret = loc_copy(&local->loc, loc);
+    if (ret < 0) {
+        gf_msg(this->name, GF_LOG_WARNING, ENOMEM, Q_MSG_ENOMEM,
+               "loc_copy failed");
+        goto err;
+    }
 
-        stub = fop_symlink_stub (frame, quota_symlink_helper, linkpath, loc,
-                                 umask, xdata);
-        if (stub == NULL) {
-                goto err;
-        }
+    stub = fop_symlink_stub(frame, quota_symlink_helper, linkpath, loc, umask,
+                            xdata);
+    if (stub == NULL) {
+        goto err;
+    }
 
-        LOCK (&local->lock);
-        {
-                local->stub = stub;
-                local->delta = strlen (linkpath);
-                local->link_count = 1;
-        }
-        UNLOCK (&local->lock);
+    LOCK(&local->lock);
+    {
+        local->stub = stub;
+        local->delta = strlen(linkpath);
+        local->object_delta = 1;
+        local->link_count = 1;
+    }
+    UNLOCK(&local->lock);
 
-        quota_check_limit (frame, loc->parent, this, NULL, NULL);
-        return 0;
+    quota_check_limit(frame, loc->parent, this);
+    return 0;
 
 err:
-        QUOTA_STACK_UNWIND (symlink, frame, -1, op_errno, NULL, NULL, NULL,
-                            NULL, NULL);
+    QUOTA_STACK_UNWIND(symlink, frame, -1, op_errno, NULL, NULL, NULL, NULL,
+                       NULL);
 
-        return 0;
+    return 0;
 
 off:
-        STACK_WIND_TAIL (frame, FIRST_CHILD(this),
-                         FIRST_CHILD(this)->fops->symlink,
-                         linkpath, loc, umask, xdata);
-        return 0;
+    STACK_WIND_TAIL(frame, FIRST_CHILD(this), FIRST_CHILD(this)->fops->symlink,
+                    linkpath, loc, umask, xdata);
+    return 0;
 }
 
-
 int32_t
-quota_truncate_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                    int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
-                    struct iatt *postbuf, dict_t *xdata)
+quota_truncate_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                   int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
+                   struct iatt *postbuf, dict_t *xdata)
 {
-        quota_local_t     *local = NULL;
-        quota_inode_ctx_t *ctx   = NULL;
+    quota_local_t *local = NULL;
+    quota_inode_ctx_t *ctx = NULL;
 
-        if (op_ret < 0) {
-                goto out;
-        }
+    if (op_ret < 0) {
+        goto out;
+    }
 
-        local = frame->local;
-        if (local == NULL) {
-                gf_log (this->name, GF_LOG_WARNING, "local is NULL");
-                goto out;
-        }
+    local = frame->local;
 
-        quota_inode_ctx_get (local->loc.inode, this, &ctx, 0);
-        if (ctx == NULL) {
-                gf_log (this->name, GF_LOG_DEBUG, "quota context is NULL on "
-                        "inode (%s). "
-                        "If quota is not enabled recently and crawler has "
-                        "finished crawling, its an error",
-                        uuid_utoa (local->loc.inode->gfid));
-                goto out;
-        }
+    GF_VALIDATE_OR_GOTO("quota", local, out);
 
-        LOCK (&ctx->lock);
-        {
-                ctx->buf = *postbuf;
-        }
-        UNLOCK (&ctx->lock);
+    quota_inode_ctx_get(local->loc.inode, this, &ctx, 0);
+    if (ctx == NULL) {
+        gf_msg_debug(this->name, 0,
+                     "quota context is NULL on inode"
+                     " (%s). If quota is not enabled recently and "
+                     "crawler has finished crawling, its an error",
+                     uuid_utoa(local->loc.inode->gfid));
+        goto out;
+    }
+
+    LOCK(&ctx->lock);
+    {
+        ctx->buf = *postbuf;
+    }
+    UNLOCK(&ctx->lock);
 
 out:
-        QUOTA_STACK_UNWIND (truncate, frame, op_ret, op_errno, prebuf,
-                            postbuf, xdata);
-        return 0;
+    QUOTA_STACK_UNWIND(truncate, frame, op_ret, op_errno, prebuf, postbuf,
+                       xdata);
+    return 0;
 }
 
-
 int32_t
-quota_truncate (call_frame_t *frame, xlator_t *this, loc_t *loc, off_t offset,
-                dict_t *xdata)
+quota_truncate(call_frame_t *frame, xlator_t *this, loc_t *loc, off_t offset,
+               dict_t *xdata)
 {
-        quota_priv_t       *priv        = NULL;
-        int32_t          ret   = -1;
-        quota_local_t   *local = NULL;
+    quota_priv_t *priv = NULL;
+    int32_t ret = -1;
+    quota_local_t *local = NULL;
 
-        priv = this->private;
+    priv = this->private;
 
-        WIND_IF_QUOTAOFF (priv->is_quota_on, off);
+    WIND_IF_QUOTAOFF(priv->is_quota_on, off);
 
-        local = quota_local_new ();
-        if (local == NULL) {
-                goto err;
-        }
+    local = quota_local_new();
+    if (local == NULL) {
+        goto err;
+    }
 
-        frame->local = local;
+    frame->local = local;
 
-        ret =  loc_copy (&local->loc, loc);
-        if (ret < 0) {
-                gf_log (this->name, GF_LOG_WARNING, "loc_copy failed");
-                goto err;
-        }
+    ret = loc_copy(&local->loc, loc);
+    if (ret < 0) {
+        gf_msg(this->name, GF_LOG_WARNING, ENOMEM, Q_MSG_ENOMEM,
+               "loc_copy failed");
+        goto err;
+    }
 
-        STACK_WIND (frame, quota_truncate_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->truncate, loc, offset, xdata);
+    STACK_WIND(frame, quota_truncate_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->truncate, loc, offset, xdata);
 
-        return 0;
+    return 0;
 
 err:
-        QUOTA_STACK_UNWIND (truncate, frame, -1, ENOMEM, NULL, NULL, NULL);
+    QUOTA_STACK_UNWIND(truncate, frame, -1, ENOMEM, NULL, NULL, NULL);
 
-        return 0;
+    return 0;
 off:
-        STACK_WIND_TAIL (frame, FIRST_CHILD(this),
-                         FIRST_CHILD(this)->fops->truncate, loc, offset, xdata);
-        return 0;
+    STACK_WIND_TAIL(frame, FIRST_CHILD(this), FIRST_CHILD(this)->fops->truncate,
+                    loc, offset, xdata);
+    return 0;
 }
 
-
 int32_t
-quota_ftruncate_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                     int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
-                     struct iatt *postbuf, dict_t *xdata)
+quota_ftruncate_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                    int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
+                    struct iatt *postbuf, dict_t *xdata)
 {
-        quota_local_t     *local = NULL;
-        quota_inode_ctx_t *ctx   = NULL;
+    quota_local_t *local = NULL;
+    quota_inode_ctx_t *ctx = NULL;
 
-        if (op_ret < 0) {
-                goto out;
-        }
+    if (op_ret < 0) {
+        goto out;
+    }
 
-        local = frame->local;
-        if (local == NULL) {
-                gf_log (this->name, GF_LOG_WARNING, "local is NULL");
-                goto out;
-        }
+    local = frame->local;
 
-        quota_inode_ctx_get (local->loc.inode, this, &ctx, 0);
-        if (ctx == NULL) {
-                gf_log (this->name, GF_LOG_DEBUG, "quota context is NULL on "
-                        "inode (%s). "
-                        "If quota is not enabled recently and crawler has "
-                        "finished crawling, its an error",
-                        uuid_utoa (local->loc.inode->gfid));
-                goto out;
-        }
+    GF_VALIDATE_OR_GOTO("quota", local, out);
 
-        LOCK (&ctx->lock);
-        {
-                ctx->buf = *postbuf;
-        }
-        UNLOCK (&ctx->lock);
+    quota_inode_ctx_get(local->loc.inode, this, &ctx, 0);
+    if (ctx == NULL) {
+        gf_msg_debug(this->name, 0,
+                     "quota context is NULL on inode"
+                     " (%s). If quota is not enabled recently and "
+                     "crawler has finished crawling, its an error",
+                     uuid_utoa(local->loc.inode->gfid));
+        goto out;
+    }
+
+    LOCK(&ctx->lock);
+    {
+        ctx->buf = *postbuf;
+    }
+    UNLOCK(&ctx->lock);
 
 out:
-        QUOTA_STACK_UNWIND (ftruncate, frame, op_ret, op_errno, prebuf,
-                            postbuf, xdata);
-        return 0;
+    QUOTA_STACK_UNWIND(ftruncate, frame, op_ret, op_errno, prebuf, postbuf,
+                       xdata);
+    return 0;
 }
 
-
 int32_t
-quota_ftruncate (call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
-                 dict_t *xdata)
+quota_ftruncate(call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
+                dict_t *xdata)
 {
-        quota_priv_t       *priv        = NULL;
-        quota_local_t   *local = NULL;
+    quota_priv_t *priv = NULL;
+    quota_local_t *local = NULL;
 
-        priv = this->private;
+    priv = this->private;
 
-        WIND_IF_QUOTAOFF (priv->is_quota_on, off);
+    WIND_IF_QUOTAOFF(priv->is_quota_on, off);
 
-        local = quota_local_new ();
-        if (local == NULL)
-                goto err;
+    local = quota_local_new();
+    if (local == NULL)
+        goto err;
 
-        frame->local = local;
+    frame->local = local;
 
-        local->loc.inode = inode_ref (fd->inode);
+    local->loc.inode = inode_ref(fd->inode);
 
-        STACK_WIND (frame, quota_ftruncate_cbk,
-                    FIRST_CHILD(this), FIRST_CHILD(this)->fops->ftruncate, fd,
-                    offset, xdata);
+    STACK_WIND(frame, quota_ftruncate_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->ftruncate, fd, offset, xdata);
 
-        return 0;
+    return 0;
 err:
-        QUOTA_STACK_UNWIND (ftruncate, frame, -1, ENOMEM, NULL, NULL, NULL);
+    QUOTA_STACK_UNWIND(ftruncate, frame, -1, ENOMEM, NULL, NULL, NULL);
 
-        return 0;
+    return 0;
 
 off:
-        STACK_WIND_TAIL (frame, FIRST_CHILD(this),
-                         FIRST_CHILD(this)->fops->ftruncate, fd,
-                         offset, xdata);
-        return 0;
+    STACK_WIND_TAIL(frame, FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->ftruncate, fd, offset, xdata);
+    return 0;
 }
 
-
-int32_t
-quota_send_dir_limit_to_cli (call_frame_t *frame, xlator_t *this,
-                             inode_t *inode, const char *name)
+static int32_t
+quota_send_dir_limit_to_cli(call_frame_t *frame, xlator_t *this, inode_t *inode,
+                            const char *name, const int namelen)
 {
-        int32_t            ret               = 0;
-        char               dir_limit [1024]  = {0, };
-        dict_t            *dict              = NULL;
-        quota_inode_ctx_t *ctx               = NULL;
-        uint64_t           value             = 0;
-        quota_priv_t      *priv              = NULL;
-
-        priv = this->private;
-        if (!priv->is_quota_on) {
-                snprintf (dir_limit, 1024, "Quota is disabled please turn on");
-                goto dict_set;
-        }
-
-        ret = inode_ctx_get (inode, this, &value);
-        if (ret < 0)
-                goto out;
-
-        ctx = (quota_inode_ctx_t *)(unsigned long)value;
-        snprintf (dir_limit, 1024, "%"PRId64",%"PRId64, ctx->size,
-                  ctx->hard_lim);
+    int32_t ret = 0;
+    int dir_limit_len = 0;
+    char dir_limit[64] = {
+        0,
+    };
+    dict_t *dict = NULL;
+    quota_inode_ctx_t *ctx = NULL;
+    uint64_t value = 0;
+    quota_priv_t *priv = NULL;
+
+    priv = this->private;
+    if (!priv->is_quota_on) {
+        dir_limit_len = snprintf(dir_limit, sizeof(dir_limit),
+                                 "Quota is disabled please turn on");
+        goto dict_set;
+    }
+
+    ret = inode_ctx_get(inode, this, &value);
+    if (ret < 0)
+        goto out;
+
+    ctx = (quota_inode_ctx_t *)(unsigned long)value;
+    dir_limit_len = snprintf(dir_limit, sizeof(dir_limit),
+                             "%" PRId64 ",%" PRId64, ctx->size, ctx->hard_lim);
 
 dict_set:
-        dict = dict_new ();
-        if (dict == NULL) {
-                ret = -1;
-                goto out;
-        }
+    dict = dict_new();
+    if (dict == NULL) {
+        ret = -1;
+        goto out;
+    }
 
-        ret = dict_set_str (dict, (char *) name, dir_limit);
-        if (ret < 0)
-                goto out;
+    ret = dict_set_nstrn(dict, (char *)name, namelen, dir_limit, dir_limit_len);
+    if (ret < 0)
+        goto out;
 
-        gf_log (this->name, GF_LOG_DEBUG, "str = %s", dir_limit);
+    gf_msg_debug(this->name, 0, "str = %s", dir_limit);
 
-        QUOTA_STACK_UNWIND (getxattr, frame, 0, 0, dict, NULL);
+    QUOTA_STACK_UNWIND(getxattr, frame, 0, 0, dict, NULL);
 
-        ret = 0;
+    ret = 0;
 
 out:
-        if (dict)
-                dict_unref (dict);
-        return ret;
+    if (dict)
+        dict_unref(dict);
+    return ret;
 }
 
-
 int32_t
-quota_fgetxattr (call_frame_t *frame, xlator_t *this, fd_t *fd,
-                 const char *name, dict_t *xdata)
+quota_fgetxattr(call_frame_t *frame, xlator_t *this, fd_t *fd, const char *name,
+                dict_t *xdata)
 {
-        int32_t ret     = 0;
+    int32_t ret = 0;
 
-        if (name && strcasecmp (name, "trusted.limit.list") == 0) {
-                ret = quota_send_dir_limit_to_cli (frame, this, fd->inode,
-                                                   name);
-                if (ret == 0) {
-                        return 0;
-                }
+    if (name && strcasecmp(name, "trusted.limit.list") == 0) {
+        ret = quota_send_dir_limit_to_cli(frame, this, fd->inode,
+                                          "trusted.limit.list",
+                                          SLEN("trusted.limit.list"));
+        if (ret == 0) {
+            return 0;
         }
+    }
 
-        STACK_WIND (frame, default_fgetxattr_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->fgetxattr, fd, name, xdata);
-        return 0;
+    STACK_WIND(frame, default_fgetxattr_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->fgetxattr, fd, name, xdata);
+    return 0;
 }
 
-
 int32_t
-quota_getxattr (call_frame_t *frame, xlator_t *this, loc_t *loc,
-                const char *name, dict_t *xdata)
+quota_getxattr(call_frame_t *frame, xlator_t *this, loc_t *loc,
+               const char *name, dict_t *xdata)
 {
-        int32_t ret     = 0;
-
-        if ((name != NULL) && strcasecmp (name, "trusted.limit.list") == 0) {
-                ret = quota_send_dir_limit_to_cli (frame, this, loc->inode,
-                                                   name);
-                if (ret == 0)
-                        return 0;
-        }
-
-        STACK_WIND (frame, default_getxattr_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->getxattr, loc, name, xdata);
-        return 0;
+    int32_t ret = 0;
+
+    if ((name != NULL) && strcasecmp(name, "trusted.limit.list") == 0) {
+        ret = quota_send_dir_limit_to_cli(frame, this, loc->inode,
+                                          "trusted.limit.list",
+                                          SLEN("trusted.limit.list"));
+        if (ret == 0)
+            return 0;
+    }
+
+    STACK_WIND(frame, default_getxattr_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->getxattr, loc, name, xdata);
+    return 0;
 }
 
-
 int32_t
-quota_stat_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                int32_t op_ret, int32_t op_errno, struct iatt *buf,
-                dict_t *xdata)
+quota_stat_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+               int32_t op_ret, int32_t op_errno, struct iatt *buf,
+               dict_t *xdata)
 {
-        quota_local_t     *local = NULL;
-        quota_inode_ctx_t *ctx   = NULL;
+    quota_local_t *local = NULL;
+    quota_inode_ctx_t *ctx = NULL;
 
-        if (op_ret < 0) {
-                goto out;
-        }
+    if (op_ret < 0) {
+        goto out;
+    }
 
-        local = frame->local;
-        if (local == NULL) {
-                gf_log (this->name, GF_LOG_WARNING, "local is NULL");
-                goto out;
-        }
+    local = frame->local;
 
-        quota_inode_ctx_get (local->loc.inode, this, &ctx, 0);
-        if (ctx == NULL) {
-                if (!IA_ISDIR (buf->ia_type)) {
-                        gf_log (this->name, GF_LOG_DEBUG,
-                                "quota context is NULL on "
-                                "inode (%s). "
-                                "If quota is not enabled recently and crawler "
-                                "has finished crawling, its an error",
-                                uuid_utoa (local->loc.inode->gfid));
-                }
+    GF_VALIDATE_OR_GOTO("quota", local, out);
 
-                goto out;
+    quota_inode_ctx_get(local->loc.inode, this, &ctx, 0);
+    if (ctx == NULL) {
+        if (!IA_ISDIR(buf->ia_type)) {
+            gf_msg_debug(this->name, 0,
+                         "quota context is NULL on inode"
+                         " (%s). If quota is not enabled recently and "
+                         "crawler has finished crawling, its an error",
+                         uuid_utoa(local->loc.inode->gfid));
         }
 
-        LOCK (&ctx->lock);
-        {
-                if (buf)
-                        ctx->buf = *buf;
-        }
-        UNLOCK (&ctx->lock);
+        goto out;
+    }
+
+    if (buf) {
+        LOCK(&ctx->lock);
+        ctx->buf = *buf;
+        UNLOCK(&ctx->lock);
+    }
 
 out:
-        QUOTA_STACK_UNWIND (stat, frame, op_ret, op_errno, buf, xdata);
-        return 0;
+    QUOTA_STACK_UNWIND(stat, frame, op_ret, op_errno, buf, xdata);
+    return 0;
 }
 
-
 int32_t
-quota_stat (call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xdata)
+quota_stat(call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xdata)
 {
-        quota_priv_t       *priv        = NULL;
-        quota_local_t *local = NULL;
-        int32_t        ret   = -1;
+    quota_priv_t *priv = NULL;
+    quota_local_t *local = NULL;
+    int32_t ret = -1;
 
-        priv = this->private;
+    priv = this->private;
 
-        WIND_IF_QUOTAOFF (priv->is_quota_on, off);
+    WIND_IF_QUOTAOFF(priv->is_quota_on, off);
 
-        local = quota_local_new ();
-        if (local == NULL) {
-                goto unwind;
-        }
+    local = quota_local_new();
+    if (local == NULL) {
+        goto unwind;
+    }
 
-        frame->local = local;
-        ret = loc_copy (&local->loc, loc);
-        if (ret < 0) {
-                gf_log (this->name, GF_LOG_WARNING, "loc_copy failed");
-                goto unwind;
-        }
+    frame->local = local;
+    ret = loc_copy(&local->loc, loc);
+    if (ret < 0) {
+        gf_msg(this->name, GF_LOG_WARNING, ENOMEM, Q_MSG_ENOMEM,
+               "loc_copy failed");
+        goto unwind;
+    }
 
-        STACK_WIND (frame, quota_stat_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->stat, loc,
-                    xdata);
-        return 0;
+    STACK_WIND(frame, quota_stat_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->stat, loc, xdata);
+    return 0;
 
 unwind:
-        QUOTA_STACK_UNWIND (stat, frame, -1, ENOMEM, NULL, NULL);
-        return 0;
+    QUOTA_STACK_UNWIND(stat, frame, -1, ENOMEM, NULL, NULL);
+    return 0;
 
 off:
-        STACK_WIND_TAIL (frame, FIRST_CHILD(this),
-                         FIRST_CHILD(this)->fops->stat, loc,
-                         xdata);
-        return 0;
+    STACK_WIND_TAIL(frame, FIRST_CHILD(this), FIRST_CHILD(this)->fops->stat,
+                    loc, xdata);
+    return 0;
 }
 
-
 int32_t
-quota_fstat_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                 int32_t op_ret, int32_t op_errno, struct iatt *buf,
-                 dict_t *xdata)
+quota_fstat_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                int32_t op_ret, int32_t op_errno, struct iatt *buf,
+                dict_t *xdata)
 {
-        quota_local_t     *local = NULL;
-        quota_inode_ctx_t *ctx   = NULL;
+    quota_local_t *local = NULL;
+    quota_inode_ctx_t *ctx = NULL;
 
-        if (op_ret < 0) {
-                goto out;
-        }
+    if (op_ret < 0) {
+        goto out;
+    }
 
-        local = frame->local;
-        if (local == NULL) {
-                gf_log (this->name, GF_LOG_WARNING, "local is NULL");
-                goto out;
-        }
+    local = frame->local;
 
-        quota_inode_ctx_get (local->loc.inode, this, &ctx, 0);
-        if (ctx == NULL) {
-                if (!IA_ISDIR (buf->ia_type)) {
-                        gf_log (this->name, GF_LOG_DEBUG,
-                                "quota context is NULL on "
-                                "inode (%s). "
-                                "If quota is not enabled recently and crawler "
-                                "has finished crawling, its an error",
-                                uuid_utoa (local->loc.inode->gfid));
-                }
+    GF_VALIDATE_OR_GOTO("quota", local, out);
 
-                goto out;
+    quota_inode_ctx_get(local->loc.inode, this, &ctx, 0);
+    if (ctx == NULL) {
+        if (!IA_ISDIR(buf->ia_type)) {
+            gf_msg_debug(this->name, 0,
+                         "quota context is NULL on inode"
+                         " (%s). If quota is not enabled recently and "
+                         "crawler has finished crawling, its an error",
+                         uuid_utoa(local->loc.inode->gfid));
         }
 
-        LOCK (&ctx->lock);
-        {
-                if (buf)
-                        ctx->buf = *buf;
-        }
-        UNLOCK (&ctx->lock);
+        goto out;
+    }
+
+    if (buf) {
+        LOCK(&ctx->lock);
+        ctx->buf = *buf;
+        UNLOCK(&ctx->lock);
+    }
 
 out:
-        QUOTA_STACK_UNWIND (fstat, frame, op_ret, op_errno, buf, xdata);
-        return 0;
+    QUOTA_STACK_UNWIND(fstat, frame, op_ret, op_errno, buf, xdata);
+    return 0;
 }
 
-
 int32_t
-quota_fstat (call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *xdata)
+quota_fstat(call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *xdata)
 {
-        quota_priv_t       *priv        = NULL;
-        quota_local_t *local = NULL;
+    quota_priv_t *priv = NULL;
+    quota_local_t *local = NULL;
 
-        priv = this->private;
+    priv = this->private;
 
-        WIND_IF_QUOTAOFF (priv->is_quota_on, off);
+    WIND_IF_QUOTAOFF(priv->is_quota_on, off);
 
-        local = quota_local_new ();
-        if (local == NULL) {
-                goto unwind;
-        }
+    local = quota_local_new();
+    if (local == NULL) {
+        goto unwind;
+    }
 
-        frame->local = local;
+    frame->local = local;
 
-        local->loc.inode = inode_ref (fd->inode);
+    local->loc.inode = inode_ref(fd->inode);
 
-        STACK_WIND (frame, quota_fstat_cbk,
-                    FIRST_CHILD(this), FIRST_CHILD(this)->fops->fstat, fd,
-                    xdata);
-        return 0;
+    STACK_WIND(frame, quota_fstat_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->fstat, fd, xdata);
+    return 0;
 
 unwind:
-        QUOTA_STACK_UNWIND (fstat, frame, -1, ENOMEM, NULL, NULL);
-        return 0;
+    QUOTA_STACK_UNWIND(fstat, frame, -1, ENOMEM, NULL, NULL);
+    return 0;
 
 off:
-        STACK_WIND_TAIL (frame, FIRST_CHILD(this),
-                         FIRST_CHILD(this)->fops->fstat, fd,
-                         xdata);
-        return 0;
+    STACK_WIND_TAIL(frame, FIRST_CHILD(this), FIRST_CHILD(this)->fops->fstat,
+                    fd, xdata);
+    return 0;
 }
 
-
 int32_t
-quota_readlink_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                    int32_t op_ret, int32_t op_errno, const char *path,
-                    struct iatt *buf, dict_t *xdata)
+quota_readlink_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                   int32_t op_ret, int32_t op_errno, const char *path,
+                   struct iatt *buf, dict_t *xdata)
 {
-        quota_local_t     *local = NULL;
-        quota_inode_ctx_t *ctx   = NULL;
+    quota_local_t *local = NULL;
+    quota_inode_ctx_t *ctx = NULL;
 
-        if (op_ret < 0) {
-                goto out;
-        }
+    if (op_ret < 0) {
+        goto out;
+    }
 
-        local = frame->local;
-        if (local == NULL) {
-                gf_log (this->name, GF_LOG_WARNING, "local is NULL");
-                goto out;
-        }
+    local = frame->local;
 
-        quota_inode_ctx_get (local->loc.inode, this, &ctx, 0);
-        if (ctx == NULL) {
-                gf_log (this->name, GF_LOG_DEBUG, "quota context is NULL on "
-                        "inode (%s). "
-                        "If quota is not enabled recently and crawler has "
-                        "finished crawling, its an error",
-                        uuid_utoa (local->loc.inode->gfid));
-                goto out;
-        }
+    GF_VALIDATE_OR_GOTO("quota", local, out);
 
-        LOCK (&ctx->lock);
-        {
-                ctx->buf = *buf;
-        }
-        UNLOCK (&ctx->lock);
+    quota_inode_ctx_get(local->loc.inode, this, &ctx, 0);
+    if (ctx == NULL) {
+        gf_msg_debug(this->name, 0,
+                     "quota context is NULL on inode"
+                     " (%s). If quota is not enabled recently and "
+                     "crawler has finished crawling, its an error",
+                     uuid_utoa(local->loc.inode->gfid));
+        goto out;
+    }
+
+    LOCK(&ctx->lock);
+    {
+        ctx->buf = *buf;
+    }
+    UNLOCK(&ctx->lock);
 
 out:
-        QUOTA_STACK_UNWIND (readlink, frame, op_ret, op_errno, path, buf,
-                            xdata);
-        return 0;
+    QUOTA_STACK_UNWIND(readlink, frame, op_ret, op_errno, path, buf, xdata);
+    return 0;
 }
 
-
 int32_t
-quota_readlink (call_frame_t *frame, xlator_t *this, loc_t *loc, size_t size,
-                dict_t *xdata)
+quota_readlink(call_frame_t *frame, xlator_t *this, loc_t *loc, size_t size,
+               dict_t *xdata)
 {
-        quota_priv_t       *priv        = NULL;
-        quota_local_t *local = NULL;
-        int32_t        ret   = -1;
+    quota_priv_t *priv = NULL;
+    quota_local_t *local = NULL;
+    int32_t ret = -1;
 
-        priv = this->private;
+    priv = this->private;
 
-        WIND_IF_QUOTAOFF (priv->is_quota_on, off);
+    WIND_IF_QUOTAOFF(priv->is_quota_on, off);
 
-        local = quota_local_new ();
-        if (local == NULL) {
-                goto unwind;
-        }
+    local = quota_local_new();
+    if (local == NULL) {
+        goto unwind;
+    }
 
-        frame->local = local;
+    frame->local = local;
 
-        ret = loc_copy (&local->loc, loc);
-        if (ret < 0) {
-                gf_log (this->name, GF_LOG_WARNING, "loc_copy failed");
-                goto unwind;
-        }
+    ret = loc_copy(&local->loc, loc);
+    if (ret < 0) {
+        gf_msg(this->name, GF_LOG_WARNING, ENOMEM, Q_MSG_ENOMEM,
+               "loc_copy failed");
+        goto unwind;
+    }
 
-        STACK_WIND (frame, quota_readlink_cbk,
-                    FIRST_CHILD(this), FIRST_CHILD(this)->fops->readlink, loc,
-                    size, xdata);
-        return 0;
+    STACK_WIND(frame, quota_readlink_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->readlink, loc, size, xdata);
+    return 0;
 
 unwind:
-        QUOTA_STACK_UNWIND (readlink, frame, -1, ENOMEM, NULL, NULL, NULL);
-        return 0;
+    QUOTA_STACK_UNWIND(readlink, frame, -1, ENOMEM, NULL, NULL, NULL);
+    return 0;
 
 off:
-        STACK_WIND_TAIL (frame, FIRST_CHILD(this),
-                         FIRST_CHILD(this)->fops->readlink, loc,
-                         size, xdata);
-        return 0;
+    STACK_WIND_TAIL(frame, FIRST_CHILD(this), FIRST_CHILD(this)->fops->readlink,
+                    loc, size, xdata);
+    return 0;
 }
 
-
 int32_t
-quota_readv_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                 int32_t op_ret, int32_t op_errno, struct iovec *vector,
-                 int32_t count, struct iatt *buf, struct iobref *iobref,
-                 dict_t *xdata)
+quota_readv_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                int32_t op_ret, int32_t op_errno, struct iovec *vector,
+                int32_t count, struct iatt *buf, struct iobref *iobref,
+                dict_t *xdata)
 {
-        quota_local_t     *local = NULL;
-        quota_inode_ctx_t *ctx   = NULL;
+    quota_local_t *local = NULL;
+    quota_inode_ctx_t *ctx = NULL;
 
-        if (op_ret < 0) {
-                goto out;
-        }
+    if (op_ret < 0) {
+        goto out;
+    }
 
-        local = frame->local;
-        if (local == NULL) {
-                gf_log (this->name, GF_LOG_WARNING, "local is NULL");
-                goto out;
-        }
+    local = frame->local;
 
-        quota_inode_ctx_get (local->loc.inode, this, &ctx, 0);
-        if (ctx == NULL) {
-                gf_log (this->name, GF_LOG_DEBUG, "quota context is NULL on "
-                        "inode (%s). "
-                        "If quota is not enabled recently and crawler has "
-                        "finished crawling, its an error",
-                        uuid_utoa (local->loc.inode->gfid));
-                goto out;
-        }
+    GF_VALIDATE_OR_GOTO("quota", local, out);
 
-        LOCK (&ctx->lock);
-        {
-                ctx->buf = *buf;
-        }
-        UNLOCK (&ctx->lock);
+    quota_inode_ctx_get(local->loc.inode, this, &ctx, 0);
+    if (ctx == NULL) {
+        gf_msg_debug(this->name, 0,
+                     "quota context is NULL on inode"
+                     " (%s). If quota is not enabled recently and "
+                     "crawler has finished crawling, its an error",
+                     uuid_utoa(local->loc.inode->gfid));
+        goto out;
+    }
+
+    LOCK(&ctx->lock);
+    {
+        ctx->buf = *buf;
+    }
+    UNLOCK(&ctx->lock);
 
 out:
-        QUOTA_STACK_UNWIND (readv, frame, op_ret, op_errno, vector, count,
-                            buf, iobref, xdata);
-        return 0;
+    QUOTA_STACK_UNWIND(readv, frame, op_ret, op_errno, vector, count, buf,
+                       iobref, xdata);
+    return 0;
 }
 
-
 int32_t
-quota_readv (call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
-             off_t offset, uint32_t flags, dict_t *xdata)
+quota_readv(call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
+            off_t offset, uint32_t flags, dict_t *xdata)
 {
-        quota_priv_t       *priv        = NULL;
-        quota_local_t *local = NULL;
+    quota_priv_t *priv = NULL;
+    quota_local_t *local = NULL;
 
-        priv = this->private;
+    priv = this->private;
 
-        WIND_IF_QUOTAOFF (priv->is_quota_on, off);
+    WIND_IF_QUOTAOFF(priv->is_quota_on, off);
 
-        local = quota_local_new ();
-        if (local == NULL) {
-                goto unwind;
-        }
+    local = quota_local_new();
+    if (local == NULL) {
+        goto unwind;
+    }
 
-        frame->local = local;
+    frame->local = local;
 
-        local->loc.inode = inode_ref (fd->inode);
+    local->loc.inode = inode_ref(fd->inode);
 
-        STACK_WIND (frame, quota_readv_cbk,
-                    FIRST_CHILD(this), FIRST_CHILD(this)->fops->readv, fd,
-                    size, offset, flags, xdata);
-        return 0;
+    STACK_WIND(frame, quota_readv_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->readv, fd, size, offset, flags, xdata);
+    return 0;
 
 unwind:
-        QUOTA_STACK_UNWIND (readv, frame, -1, ENOMEM, NULL, -1, NULL, NULL,
-                            NULL);
-        return 0;
+    QUOTA_STACK_UNWIND(readv, frame, -1, ENOMEM, NULL, -1, NULL, NULL, NULL);
+    return 0;
 
 off:
-        STACK_WIND_TAIL (frame, FIRST_CHILD(this),
-                         FIRST_CHILD(this)->fops->readv, fd,
-                         size, offset, flags, xdata);
-        return 0;
+    STACK_WIND_TAIL(frame, FIRST_CHILD(this), FIRST_CHILD(this)->fops->readv,
+                    fd, size, offset, flags, xdata);
+    return 0;
 }
 
-
 int32_t
-quota_fsync_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                 int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
-                 struct iatt *postbuf, dict_t *xdata)
+quota_fsync_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
+                struct iatt *postbuf, dict_t *xdata)
 {
-        quota_local_t     *local = NULL;
-        quota_inode_ctx_t *ctx   = NULL;
+    quota_local_t *local = NULL;
+    quota_inode_ctx_t *ctx = NULL;
 
-        if (op_ret < 0) {
-                goto out;
-        }
+    if (op_ret < 0) {
+        goto out;
+    }
 
-        local = frame->local;
-        if (local == NULL) {
-                gf_log (this->name, GF_LOG_WARNING, "local is NULL");
-                goto out;
-        }
+    local = frame->local;
 
-        quota_inode_ctx_get (local->loc.inode, this, &ctx, 0);
-        if (ctx == NULL) {
-                gf_log (this->name, GF_LOG_DEBUG, "quota context is NULL on "
-                        "inode (%s). "
-                        "If quota is not enabled recently and crawler has "
-                        "finished crawling, its an error",
-                        uuid_utoa (local->loc.inode->gfid));
-                goto out;
-        }
+    GF_VALIDATE_OR_GOTO("quota", local, out);
 
-        LOCK (&ctx->lock);
-        {
-                ctx->buf = *postbuf;
-        }
-        UNLOCK (&ctx->lock);
+    quota_inode_ctx_get(local->loc.inode, this, &ctx, 0);
+    if (ctx == NULL) {
+        gf_msg_debug(this->name, 0,
+                     "quota context is NULL on inode"
+                     " (%s). If quota is not enabled recently and "
+                     "crawler has finished crawling, its an error",
+                     uuid_utoa(local->loc.inode->gfid));
+        goto out;
+    }
+
+    LOCK(&ctx->lock);
+    {
+        ctx->buf = *postbuf;
+    }
+    UNLOCK(&ctx->lock);
 
 out:
-        QUOTA_STACK_UNWIND (fsync, frame, op_ret, op_errno, prebuf, postbuf,
-                            xdata);
-        return 0;
+    QUOTA_STACK_UNWIND(fsync, frame, op_ret, op_errno, prebuf, postbuf, xdata);
+    return 0;
 }
 
-
 int32_t
-quota_fsync (call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t flags,
-             dict_t *xdata)
+quota_fsync(call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t flags,
+            dict_t *xdata)
 {
-        quota_priv_t       *priv        = NULL;
-        quota_local_t *local = NULL;
+    quota_priv_t *priv = NULL;
+    quota_local_t *local = NULL;
 
-        priv = this->private;
+    priv = this->private;
 
-        WIND_IF_QUOTAOFF (priv->is_quota_on, off);
+    WIND_IF_QUOTAOFF(priv->is_quota_on, off);
 
-        local = quota_local_new ();
-        if (local == NULL) {
-                goto unwind;
-        }
+    local = quota_local_new();
+    if (local == NULL) {
+        goto unwind;
+    }
 
-        local->loc.inode = inode_ref (fd->inode);
+    local->loc.inode = inode_ref(fd->inode);
 
-        frame->local = local;
+    frame->local = local;
 
-        STACK_WIND (frame, quota_fsync_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->fsync, fd,
-                    flags, xdata);
-        return 0;
+    STACK_WIND(frame, quota_fsync_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->fsync, fd, flags, xdata);
+    return 0;
 
 unwind:
-        QUOTA_STACK_UNWIND (fsync, frame, -1, ENOMEM, NULL, NULL, NULL);
-        return 0;
+    QUOTA_STACK_UNWIND(fsync, frame, -1, ENOMEM, NULL, NULL, NULL);
+    return 0;
 
 off:
-        STACK_WIND_TAIL (frame, FIRST_CHILD(this),
-                         FIRST_CHILD(this)->fops->fsync, fd,
-                         flags, xdata);
-        return 0;
+    STACK_WIND_TAIL(frame, FIRST_CHILD(this), FIRST_CHILD(this)->fops->fsync,
+                    fd, flags, xdata);
+    return 0;
 }
 
-
 int32_t
-quota_setattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                   int32_t op_ret, int32_t op_errno, struct iatt *statpre,
-                   struct iatt *statpost, dict_t *xdata)
+quota_setattr_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                  int32_t op_ret, int32_t op_errno, struct iatt *statpre,
+                  struct iatt *statpost, dict_t *xdata)
 {
-        quota_local_t     *local = NULL;
-        quota_inode_ctx_t *ctx   = NULL;
+    quota_local_t *local = NULL;
+    quota_inode_ctx_t *ctx = NULL;
 
-        if (op_ret < 0) {
-                goto out;
-        }
+    if (op_ret < 0) {
+        goto out;
+    }
 
-        local = frame->local;
-        if (local == NULL) {
-                gf_log (this->name, GF_LOG_WARNING, "local is NULL");
-                goto out;
-        }
+    local = frame->local;
 
-        quota_inode_ctx_get (local->loc.inode, this, &ctx, 0);
-        if (ctx == NULL) {
-                if (!IA_ISDIR (statpost->ia_type)) {
-                        gf_log (this->name, GF_LOG_DEBUG, "quota context is "
-                                "NULL on inode (%s). "
-                                "If quota is not enabled recently and crawler "
-                                "has finished crawling, its an error",
-                                uuid_utoa (local->loc.inode->gfid));
-                }
+    GF_VALIDATE_OR_GOTO("quota", local, out);
 
-                goto out;
+    quota_inode_ctx_get(local->loc.inode, this, &ctx, 0);
+    if (ctx == NULL) {
+        if (!IA_ISDIR(statpost->ia_type)) {
+            gf_msg_debug(this->name, 0,
+                         "quota context is NULL on inode"
+                         " (%s). If quota is not enabled recently and "
+                         "crawler has finished crawling, its an error",
+                         uuid_utoa(local->loc.inode->gfid));
         }
 
-        LOCK (&ctx->lock);
-        {
-                if (statpost)
-                        ctx->buf = *statpost;
-        }
-        UNLOCK (&ctx->lock);
+        goto out;
+    }
+
+    if (statpost) {
+        LOCK(&ctx->lock);
+        ctx->buf = *statpost;
+        UNLOCK(&ctx->lock);
+    }
 
 out:
-        QUOTA_STACK_UNWIND (setattr, frame, op_ret, op_errno, statpre,
-                            statpost, xdata);
-        return 0;
+    QUOTA_STACK_UNWIND(setattr, frame, op_ret, op_errno, statpre, statpost,
+                       xdata);
+    return 0;
 }
 
-
 int32_t
-quota_setattr (call_frame_t *frame, xlator_t *this, loc_t *loc,
-               struct iatt *stbuf, int32_t valid, dict_t *xdata)
+quota_setattr(call_frame_t *frame, xlator_t *this, loc_t *loc,
+              struct iatt *stbuf, int32_t valid, dict_t *xdata)
 {
-        quota_priv_t       *priv        = NULL;
-        quota_local_t *local = NULL;
-        int32_t        ret   = -1;
+    quota_priv_t *priv = NULL;
+    quota_local_t *local = NULL;
+    int32_t ret = -1;
 
-        priv = this->private;
+    priv = this->private;
 
-        WIND_IF_QUOTAOFF (priv->is_quota_on, off);
+    WIND_IF_QUOTAOFF(priv->is_quota_on, off);
 
-        local = quota_local_new ();
-        if (local == NULL) {
-                goto unwind;
-        }
+    local = quota_local_new();
+    if (local == NULL) {
+        goto unwind;
+    }
 
-        frame->local = local;
+    frame->local = local;
 
-        ret = loc_copy (&local->loc, loc);
-        if (ret < 0) {
-                gf_log (this->name, GF_LOG_WARNING, "loc_copy failed");
-                goto unwind;
-        }
+    ret = loc_copy(&local->loc, loc);
+    if (ret < 0) {
+        gf_msg(this->name, GF_LOG_WARNING, ENOMEM, Q_MSG_ENOMEM,
+               "loc_copy failed");
+        goto unwind;
+    }
 
-        STACK_WIND (frame, quota_setattr_cbk, FIRST_CHILD (this),
-                    FIRST_CHILD (this)->fops->setattr, loc,
-                    stbuf, valid, xdata);
-        return 0;
+    STACK_WIND(frame, quota_setattr_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->setattr, loc, stbuf, valid, xdata);
+    return 0;
 
 unwind:
-        QUOTA_STACK_UNWIND (setattr, frame, -1, ENOMEM, NULL, NULL, NULL);
-        return 0;
+    QUOTA_STACK_UNWIND(setattr, frame, -1, ENOMEM, NULL, NULL, NULL);
+    return 0;
 
 off:
-        STACK_WIND_TAIL (frame, FIRST_CHILD (this),
-                         FIRST_CHILD (this)->fops->setattr, loc,
-                         stbuf, valid, xdata);
-        return 0;
+    STACK_WIND_TAIL(frame, FIRST_CHILD(this), FIRST_CHILD(this)->fops->setattr,
+                    loc, stbuf, valid, xdata);
+    return 0;
 }
 
-
 int32_t
-quota_fsetattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                    int32_t op_ret, int32_t op_errno, struct iatt *statpre,
-                    struct iatt *statpost, dict_t *xdata)
+quota_fsetattr_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                   int32_t op_ret, int32_t op_errno, struct iatt *statpre,
+                   struct iatt *statpost, dict_t *xdata)
 {
-        quota_local_t     *local = NULL;
-        quota_inode_ctx_t *ctx   = NULL;
+    quota_local_t *local = NULL;
+    quota_inode_ctx_t *ctx = NULL;
 
-        if (op_ret < 0) {
-                goto out;
-        }
+    if (op_ret < 0) {
+        goto out;
+    }
 
-        local = frame->local;
-        if (local == NULL) {
-                gf_log (this->name, GF_LOG_WARNING, "local is NULL");
-                goto out;
-        }
+    local = frame->local;
 
-        quota_inode_ctx_get (local->loc.inode, this, &ctx, 0);
-        if (ctx == NULL) {
-                if (!IA_ISDIR (statpost->ia_type)) {
-                        gf_log (this->name, GF_LOG_DEBUG, "quota context is "
-                                "NULL on inode (%s). "
-                                "If quota is not enabled recently and crawler "
-                                "has finished crawling, its an error",
-                                uuid_utoa (local->loc.inode->gfid));
-                }
+    GF_VALIDATE_OR_GOTO("quota", local, out);
 
-                goto out;
+    quota_inode_ctx_get(local->loc.inode, this, &ctx, 0);
+    if (ctx == NULL) {
+        if (!IA_ISDIR(statpost->ia_type)) {
+            gf_msg_debug(this->name, 0,
+                         "quota context is NULL on inode"
+                         " (%s). If quota is not enabled recently and "
+                         "crawler has finished crawling, its an error",
+                         uuid_utoa(local->loc.inode->gfid));
         }
 
-        LOCK (&ctx->lock);
-        {
-                ctx->buf = *statpost;
-        }
-        UNLOCK (&ctx->lock);
+        goto out;
+    }
+
+    LOCK(&ctx->lock);
+    {
+        ctx->buf = *statpost;
+    }
+    UNLOCK(&ctx->lock);
 
 out:
-        QUOTA_STACK_UNWIND (fsetattr, frame, op_ret, op_errno, statpre,
-                            statpost, xdata);
-        return 0;
+    QUOTA_STACK_UNWIND(fsetattr, frame, op_ret, op_errno, statpre, statpost,
+                       xdata);
+    return 0;
 }
 
-
 int32_t
-quota_fsetattr (call_frame_t *frame, xlator_t *this, fd_t *fd,
-                struct iatt *stbuf, int32_t valid, dict_t *xdata)
+quota_fsetattr(call_frame_t *frame, xlator_t *this, fd_t *fd,
+               struct iatt *stbuf, int32_t valid, dict_t *xdata)
 {
-        quota_priv_t       *priv        = NULL;
-        quota_local_t *local = NULL;
+    quota_priv_t *priv = NULL;
+    quota_local_t *local = NULL;
 
-        priv = this->private;
+    priv = this->private;
 
-        WIND_IF_QUOTAOFF (priv->is_quota_on, off);
+    WIND_IF_QUOTAOFF(priv->is_quota_on, off);
 
-        local = quota_local_new ();
-        if (local == NULL) {
-                goto unwind;
-        }
+    local = quota_local_new();
+    if (local == NULL) {
+        goto unwind;
+    }
 
-        frame->local = local;
+    frame->local = local;
 
-        local->loc.inode = inode_ref (fd->inode);
+    local->loc.inode = inode_ref(fd->inode);
 
-        STACK_WIND (frame, quota_fsetattr_cbk,
-                    FIRST_CHILD (this), FIRST_CHILD (this)->fops->fsetattr, fd,
-                    stbuf, valid, xdata);
-        return 0;
+    STACK_WIND(frame, quota_fsetattr_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->fsetattr, fd, stbuf, valid, xdata);
+    return 0;
 
 unwind:
-        QUOTA_STACK_UNWIND (fsetattr, frame, -1, ENOMEM, NULL, NULL, NULL);
-        return 0;
+    QUOTA_STACK_UNWIND(fsetattr, frame, -1, ENOMEM, NULL, NULL, NULL);
+    return 0;
 
 off:
-        STACK_WIND_TAIL (frame, FIRST_CHILD (this),
-                         FIRST_CHILD (this)->fops->fsetattr, fd,
-                         stbuf, valid, xdata);
-        return 0;
+    STACK_WIND_TAIL(frame, FIRST_CHILD(this), FIRST_CHILD(this)->fops->fsetattr,
+                    fd, stbuf, valid, xdata);
+    return 0;
 }
 
-
 int32_t
-quota_mknod_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                 int32_t op_ret, int32_t op_errno, inode_t *inode,
-                 struct iatt *buf, struct iatt *preparent,
-                 struct iatt *postparent, dict_t *xdata)
+quota_mknod_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                int32_t op_ret, int32_t op_errno, inode_t *inode,
+                struct iatt *buf, struct iatt *preparent,
+                struct iatt *postparent, dict_t *xdata)
 {
-        int32_t            ret    = -1;
-        quota_local_t     *local  = NULL;
-        quota_inode_ctx_t *ctx    = NULL;
-        quota_dentry_t    *dentry = NULL;
-
-        local = frame->local;
-        if (op_ret < 0) {
-                goto unwind;
-        }
-
-        ret = quota_inode_ctx_get (inode, this, &ctx, 1);
-        if ((ret == -1) || (ctx == NULL)) {
-                gf_log (this->name, GF_LOG_WARNING, "cannot create quota "
-                        "context in inode (gfid:%s)", uuid_utoa (inode->gfid));
-                op_ret = -1;
-                op_errno = ENOMEM;
-                goto unwind;
-        }
-
-        LOCK (&ctx->lock);
-        {
-                ctx->buf = *buf;
-
-                dentry = __quota_dentry_new (ctx, (char *)local->loc.name,
-                                             local->loc.parent->gfid);
-                if (dentry == NULL) {
-                        gf_log (this->name, GF_LOG_WARNING,
-                                "cannot create a new dentry (name:%s) for "
-                                "inode(gfid:%s)", local->loc.name,
-                                uuid_utoa (local->loc.inode->gfid));
-                        op_ret = -1;
-                        op_errno = ENOMEM;
-                        goto unlock;
-                }
-        }
+    int32_t ret = -1;
+    quota_local_t *local = NULL;
+    quota_inode_ctx_t *ctx = NULL;
+    quota_dentry_t *dentry = NULL;
+
+    local = frame->local;
+    if (op_ret < 0) {
+        goto unwind;
+    }
+
+    ret = quota_inode_ctx_get(inode, this, &ctx, 1);
+    if ((ret == -1) || (ctx == NULL)) {
+        gf_msg(this->name, GF_LOG_WARNING, 0, Q_MSG_INODE_CTX_GET_FAILED,
+               "cannot create quota context in "
+               "inode(gfid:%s)",
+               uuid_utoa(inode->gfid));
+        op_ret = -1;
+        op_errno = ENOMEM;
+        goto unwind;
+    }
+
+    LOCK(&ctx->lock);
+    {
+        ctx->buf = *buf;
+
+        dentry = __quota_dentry_new(ctx, (char *)local->loc.name,
+                                    local->loc.parent->gfid);
+        if (dentry == NULL) {
+            gf_msg(this->name, GF_LOG_WARNING, ENOMEM, Q_MSG_ENOMEM,
+                   "cannot create a new dentry "
+                   "(name:%s) for inode(gfid:%s)",
+                   local->loc.name, uuid_utoa(local->loc.inode->gfid));
+            op_ret = -1;
+            op_errno = ENOMEM;
+            goto unlock;
+        }
+    }
 unlock:
-        UNLOCK (&ctx->lock);
+    UNLOCK(&ctx->lock);
 
 unwind:
-        QUOTA_STACK_UNWIND (mknod, frame, op_ret, op_errno, inode,
-                            buf, preparent, postparent, xdata);
-        return 0;
+    QUOTA_STACK_UNWIND(mknod, frame, op_ret, op_errno, inode, buf, preparent,
+                       postparent, xdata);
+    return 0;
 }
 
-
 int
-quota_mknod_helper (call_frame_t *frame, xlator_t *this, loc_t *loc,
-                    mode_t mode, dev_t rdev, mode_t umask, dict_t *xdata)
+quota_mknod_helper(call_frame_t *frame, xlator_t *this, loc_t *loc, mode_t mode,
+                   dev_t rdev, mode_t umask, dict_t *xdata)
 {
-        quota_local_t *local    = NULL;
-        int32_t        op_errno = EINVAL;
-        quota_priv_t  *priv     = NULL;
+    quota_local_t *local = NULL;
+    int32_t op_errno = EINVAL;
 
-        local = frame->local;
-        if (local == NULL) {
-                gf_log (this->name, GF_LOG_WARNING, "local is NULL");
-                goto unwind;
-        }
+    local = frame->local;
 
-        priv = this->private;
+    GF_VALIDATE_OR_GOTO("quota", local, unwind);
 
-        if (local->op_ret == -1) {
-                op_errno = local->op_errno;
-                goto unwind;
-        }
+    if (local->op_ret == -1) {
+        op_errno = local->op_errno;
+        goto unwind;
+    }
 
-        STACK_WIND (frame, quota_mknod_cbk,
-                    FIRST_CHILD(this), FIRST_CHILD(this)->fops->mknod, loc,
-                    mode, rdev, umask, xdata);
+    STACK_WIND(frame, quota_mknod_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->mknod, loc, mode, rdev, umask, xdata);
 
-        return 0;
+    return 0;
 
 unwind:
-        QUOTA_STACK_UNWIND (mknod, frame, -1, op_errno, NULL, NULL,
-                            NULL, NULL, NULL);
-        return 0;
+    QUOTA_STACK_UNWIND(mknod, frame, -1, op_errno, NULL, NULL, NULL, NULL,
+                       NULL);
+    return 0;
 }
 
-
 int
-quota_mknod (call_frame_t *frame, xlator_t *this, loc_t *loc, mode_t mode,
-             dev_t rdev, mode_t umask, dict_t *xdata)
+quota_mknod(call_frame_t *frame, xlator_t *this, loc_t *loc, mode_t mode,
+            dev_t rdev, mode_t umask, dict_t *xdata)
 {
-        quota_priv_t  *priv     = NULL;
-        int32_t        ret      = -1;
-        quota_local_t *local    = NULL;
-        call_stub_t   *stub     = NULL;
-
-        priv = this->private;
-
-        WIND_IF_QUOTAOFF (priv->is_quota_on, off);
-        QUOTA_WIND_FOR_INTERNAL_FOP (xdata, off);
-
-        local = quota_local_new ();
-        if (local == NULL) {
-                goto err;
-        }
-
-        frame->local = local;
-
-        ret = loc_copy (&local->loc, loc);
-        if (ret) {
-                gf_log (this->name, GF_LOG_WARNING, "loc_copy failed");
-                goto err;
-        }
-
-        stub = fop_mknod_stub (frame, quota_mknod_helper, loc, mode, rdev,
-                               umask, xdata);
-        if (stub == NULL) {
-                goto err;
-        }
-
-        LOCK (&local->lock);
-        {
-                local->link_count = 1;
-                local->stub = stub;
-                local->delta = 0;
-        }
-        UNLOCK (&local->lock);
+    quota_priv_t *priv = NULL;
+    int32_t ret = -1;
+    quota_local_t *local = NULL;
+    call_stub_t *stub = NULL;
+
+    priv = this->private;
+
+    WIND_IF_QUOTAOFF(priv->is_quota_on, off);
+    QUOTA_WIND_FOR_INTERNAL_FOP(xdata, off);
+
+    local = quota_local_new();
+    if (local == NULL) {
+        goto err;
+    }
+
+    frame->local = local;
+
+    ret = loc_copy(&local->loc, loc);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_WARNING, ENOMEM, Q_MSG_ENOMEM,
+               "loc_copy failed");
+        goto err;
+    }
+
+    stub = fop_mknod_stub(frame, quota_mknod_helper, loc, mode, rdev, umask,
+                          xdata);
+    if (stub == NULL) {
+        goto err;
+    }
+
+    LOCK(&local->lock);
+    {
+        local->link_count = 1;
+        local->stub = stub;
+        local->delta = 0;
+        local->object_delta = 1;
+    }
+    UNLOCK(&local->lock);
 
-        quota_check_limit (frame, loc->parent, this, NULL, NULL);
-        return 0;
+    quota_check_limit(frame, loc->parent, this);
+    return 0;
 
 err:
-        QUOTA_STACK_UNWIND (mknod, frame, -1, ENOMEM, NULL, NULL, NULL, NULL,
-                            NULL);
-        return 0;
+    QUOTA_STACK_UNWIND(mknod, frame, -1, ENOMEM, NULL, NULL, NULL, NULL, NULL);
+    return 0;
 
 off:
-        STACK_WIND_TAIL (frame, FIRST_CHILD(this),
-                         FIRST_CHILD(this)->fops->mknod, loc,
-                         mode, rdev, umask, xdata);
-        return 0;
+    STACK_WIND_TAIL(frame, FIRST_CHILD(this), FIRST_CHILD(this)->fops->mknod,
+                    loc, mode, rdev, umask, xdata);
+    return 0;
 }
 
 int
-quota_setxattr_cbk (call_frame_t *frame, void *cookie,
-                    xlator_t *this, int op_ret, int op_errno, dict_t *xdata)
+quota_setxattr_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                   int op_ret, int op_errno, dict_t *xdata)
 {
-        quota_local_t     *local = NULL;
-        quota_inode_ctx_t *ctx   = NULL;
-        int                ret   = 0;
-
-        if (op_ret < 0) {
-                goto out;
-        }
-
-        local = frame->local;
-        if (!local)
-                goto out;
-
-        ret = quota_inode_ctx_get (local->loc.inode, this, &ctx, 1);
-        if ((ret < 0) || (ctx == NULL)) {
-                op_errno = -1;
-                goto out;
-        }
-
-        LOCK (&ctx->lock);
-        {
-                ctx->hard_lim = local->limit.hl;
-                ctx->soft_lim = local->limit.sl;
-                ctx->object_hard_lim = local->object_limit.hl;
-                ctx->object_soft_lim = local->object_limit.sl;
-        }
-        UNLOCK (&ctx->lock);
+    quota_local_t *local = NULL;
+    quota_inode_ctx_t *ctx = NULL;
+    int ret = 0;
+
+    if (op_ret < 0) {
+        goto out;
+    }
+
+    local = frame->local;
+    if (!local)
+        goto out;
+
+    ret = quota_inode_ctx_get(local->loc.inode, this, &ctx, 1);
+    if ((ret < 0) || (ctx == NULL)) {
+        op_errno = -1;
+        goto out;
+    }
+
+    LOCK(&ctx->lock);
+    {
+        ctx->hard_lim = local->limit.hl;
+        ctx->soft_lim = local->limit.sl;
+        ctx->object_hard_lim = local->object_limit.hl;
+        ctx->object_soft_lim = local->object_limit.sl;
+    }
+    UNLOCK(&ctx->lock);
 
 out:
-        QUOTA_STACK_UNWIND (setxattr, frame, op_ret, op_errno, xdata);
-        return 0;
+    QUOTA_STACK_UNWIND(setxattr, frame, op_ret, op_errno, xdata);
+    return 0;
 }
 
 int
-quota_setxattr (call_frame_t *frame, xlator_t *this,
-                loc_t *loc, dict_t *dict, int flags, dict_t *xdata)
+quota_setxattr(call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *dict,
+               int flags, dict_t *xdata)
 {
-        quota_priv_t    *priv                   = NULL;
-        int             op_errno                = EINVAL;
-        int             op_ret                  = -1;
-        int64_t         hard_lim                = -1;
-        int64_t         soft_lim                = -1;
-        int64_t         object_hard_limit       = -1;
-        int64_t         object_soft_limit       = -1;
-        quota_local_t   *local                  = NULL;
-
-        priv = this->private;
-
-        WIND_IF_QUOTAOFF (priv->is_quota_on, off);
-
-        VALIDATE_OR_GOTO (frame, err);
-        VALIDATE_OR_GOTO (this, err);
-        VALIDATE_OR_GOTO (loc, err);
-
-        if (frame->root->pid >= 0) {
-                GF_IF_INTERNAL_XATTR_GOTO ("trusted.glusterfs.quota*", dict,
-                                           op_errno, err);
-                GF_IF_INTERNAL_XATTR_GOTO ("trusted.pgfid*", dict, op_errno,
-                                           err);
-        }
-
-        quota_get_limits (this, dict, &hard_lim, &soft_lim, &object_hard_limit,
-                          &object_soft_limit);
-
-        if (hard_lim > 0 || object_hard_limit > 0) {
-                local = quota_local_new ();
-                if (local == NULL) {
-                        op_errno = ENOMEM;
-                        goto err;
-                }
-                frame->local = local;
-                loc_copy (&local->loc, loc);
-        }
-
-        if (hard_lim > 0) {
-                local->limit.hl = hard_lim;
-                local->limit.sl = soft_lim;
-        }
-
-        if (object_hard_limit > 0) {
-                local->object_limit.hl = object_hard_limit;
-                local->object_limit.sl = object_soft_limit;
+    quota_priv_t *priv = NULL;
+    int op_errno = EINVAL;
+    int op_ret = -1;
+    int64_t hard_lim = -1;
+    int64_t soft_lim = -1;
+    int64_t object_hard_limit = -1;
+    int64_t object_soft_limit = -1;
+    quota_local_t *local = NULL;
+    gf_boolean_t internal_fop = _gf_false;
+
+    priv = this->private;
+
+    WIND_IF_QUOTAOFF(priv->is_quota_on, off);
+
+    VALIDATE_OR_GOTO(frame, err);
+    VALIDATE_OR_GOTO(this, err);
+    VALIDATE_OR_GOTO(loc, err);
+
+    if (xdata && dict_get_sizen(xdata, GLUSTERFS_INTERNAL_FOP_KEY))
+        internal_fop = _gf_true;
+
+    if (frame->root->pid >= 0 && internal_fop == _gf_false) {
+        GF_IF_INTERNAL_XATTR_GOTO("trusted.glusterfs.quota*", dict, op_errno,
+                                  err);
+        GF_IF_INTERNAL_XATTR_GOTO("trusted.pgfid*", dict, op_errno, err);
+    }
+
+    quota_get_limits(this, dict, &hard_lim, &soft_lim, &object_hard_limit,
+                     &object_soft_limit);
+
+    if (hard_lim > 0 || object_hard_limit > 0) {
+        local = quota_local_new();
+        if (local == NULL) {
+            op_errno = ENOMEM;
+            goto err;
         }
-
-        STACK_WIND (frame, quota_setxattr_cbk,
-                    FIRST_CHILD(this), FIRST_CHILD(this)->fops->setxattr, loc,
-                    dict, flags, xdata);
-        return 0;
+        frame->local = local;
+        loc_copy(&local->loc, loc);
+    }
+
+    if (hard_lim > 0) {
+        local->limit.hl = hard_lim;
+        local->limit.sl = soft_lim;
+    }
+
+    if (object_hard_limit > 0) {
+        local->object_limit.hl = object_hard_limit;
+        local->object_limit.sl = object_soft_limit;
+    }
+
+    STACK_WIND(frame, quota_setxattr_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->setxattr, loc, dict, flags, xdata);
+    return 0;
 err:
-        QUOTA_STACK_UNWIND (setxattr, frame, op_ret, op_errno, NULL);
-        return 0;
+    QUOTA_STACK_UNWIND(setxattr, frame, op_ret, op_errno, NULL);
+    return 0;
 
 off:
-        STACK_WIND_TAIL (frame, FIRST_CHILD(this),
-                         FIRST_CHILD(this)->fops->setxattr, loc,
-                         dict, flags, xdata);
-        return 0;
+    STACK_WIND_TAIL(frame, FIRST_CHILD(this), FIRST_CHILD(this)->fops->setxattr,
+                    loc, dict, flags, xdata);
+    return 0;
 }
 
 int
-quota_fsetxattr_cbk (call_frame_t *frame, void *cookie,
-                     xlator_t *this, int op_ret, int op_errno, dict_t *xdata)
+quota_fsetxattr_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                    int op_ret, int op_errno, dict_t *xdata)
 {
-        quota_inode_ctx_t *ctx   = NULL;
-        quota_local_t     *local = NULL;
-
-        if (op_ret < 0)
-                goto out;
-
-        local = frame->local;
-        if (!local)
-                goto out;
-
-        op_ret = quota_inode_ctx_get (local->loc.inode, this, &ctx, 1);
-        if ((op_ret < 0) || (ctx == NULL)) {
-                op_errno = ENOMEM;
-                goto out;
-        }
-
-        LOCK (&ctx->lock);
-        {
-                ctx->hard_lim = local->limit.hl;
-                ctx->soft_lim = local->limit.sl;
-                ctx->object_hard_lim = local->object_limit.hl;
-                ctx->object_soft_lim = local->object_limit.sl;
-        }
-        UNLOCK (&ctx->lock);
+    quota_inode_ctx_t *ctx = NULL;
+    quota_local_t *local = NULL;
+
+    if (op_ret < 0)
+        goto out;
+
+    local = frame->local;
+    if (!local)
+        goto out;
+
+    op_ret = quota_inode_ctx_get(local->loc.inode, this, &ctx, 1);
+    if ((op_ret < 0) || (ctx == NULL)) {
+        op_errno = ENOMEM;
+        goto out;
+    }
+
+    LOCK(&ctx->lock);
+    {
+        ctx->hard_lim = local->limit.hl;
+        ctx->soft_lim = local->limit.sl;
+        ctx->object_hard_lim = local->object_limit.hl;
+        ctx->object_soft_lim = local->object_limit.sl;
+    }
+    UNLOCK(&ctx->lock);
 
 out:
-        QUOTA_STACK_UNWIND (fsetxattr, frame, op_ret, op_errno, xdata);
-        return 0;
+    QUOTA_STACK_UNWIND(fsetxattr, frame, op_ret, op_errno, xdata);
+    return 0;
 }
 
 int
-quota_fsetxattr (call_frame_t *frame, xlator_t *this, fd_t *fd,
-                 dict_t *dict, int flags, dict_t *xdata)
+quota_fsetxattr(call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *dict,
+                int flags, dict_t *xdata)
 {
-        quota_priv_t    *priv                   = NULL;
-        int32_t         op_ret                  = -1;
-        int32_t         op_errno                = EINVAL;
-        quota_local_t   *local                  = NULL;
-        int64_t         hard_lim                = -1;
-        int64_t         soft_lim                = -1;
-        int64_t         object_hard_limit       = -1;
-        int64_t         object_soft_limit       = -1;
-
-        priv = this->private;
-
-        WIND_IF_QUOTAOFF (priv->is_quota_on, off);
-
-        VALIDATE_OR_GOTO (frame, err);
-        VALIDATE_OR_GOTO (this, err);
-        VALIDATE_OR_GOTO (fd, err);
-
-        if (0 <= frame->root->pid) {
-                GF_IF_INTERNAL_XATTR_GOTO ("trusted.glusterfs.quota*",
-                                            dict, op_errno, err);
-                GF_IF_INTERNAL_XATTR_GOTO ("trusted.pgfid*", dict,
-                                            op_errno, err);
-        }
-
-        quota_get_limits (this, dict, &hard_lim, &soft_lim, &object_hard_limit,
-                          &object_soft_limit);
-
-        if (hard_lim > 0 || object_hard_limit > 0) {
-                local = quota_local_new ();
-                if (local == NULL) {
-                        op_errno = ENOMEM;
-                        goto err;
-                }
-                frame->local = local;
-                local->loc.inode = inode_ref (fd->inode);
-        }
-
-        if (hard_lim > 0) {
-                local->limit.hl = hard_lim;
-                local->limit.sl = soft_lim;
-        }
-
-        if (object_hard_limit > 0) {
-                local->object_limit.hl = object_hard_limit;
-                local->object_limit.sl = object_soft_limit;
+    quota_priv_t *priv = NULL;
+    int32_t op_ret = -1;
+    int32_t op_errno = EINVAL;
+    quota_local_t *local = NULL;
+    int64_t hard_lim = -1;
+    int64_t soft_lim = -1;
+    int64_t object_hard_limit = -1;
+    int64_t object_soft_limit = -1;
+
+    priv = this->private;
+
+    WIND_IF_QUOTAOFF(priv->is_quota_on, off);
+
+    VALIDATE_OR_GOTO(frame, err);
+    VALIDATE_OR_GOTO(this, err);
+    VALIDATE_OR_GOTO(fd, err);
+
+    if (0 <= frame->root->pid) {
+        GF_IF_INTERNAL_XATTR_GOTO("trusted.glusterfs.quota*", dict, op_errno,
+                                  err);
+        GF_IF_INTERNAL_XATTR_GOTO("trusted.pgfid*", dict, op_errno, err);
+    }
+
+    quota_get_limits(this, dict, &hard_lim, &soft_lim, &object_hard_limit,
+                     &object_soft_limit);
+
+    if (hard_lim > 0 || object_hard_limit > 0) {
+        local = quota_local_new();
+        if (local == NULL) {
+            op_errno = ENOMEM;
+            goto err;
         }
-
-        STACK_WIND (frame, quota_fsetxattr_cbk,
-                    FIRST_CHILD(this), FIRST_CHILD(this)->fops->fsetxattr, fd,
-                    dict, flags, xdata);
-        return 0;
+        frame->local = local;
+        local->loc.inode = inode_ref(fd->inode);
+    }
+
+    if (hard_lim > 0) {
+        local->limit.hl = hard_lim;
+        local->limit.sl = soft_lim;
+    }
+
+    if (object_hard_limit > 0) {
+        local->object_limit.hl = object_hard_limit;
+        local->object_limit.sl = object_soft_limit;
+    }
+
+    STACK_WIND(frame, quota_fsetxattr_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->fsetxattr, fd, dict, flags, xdata);
+    return 0;
 err:
-        QUOTA_STACK_UNWIND (fsetxattr, frame, op_ret, op_errno, NULL);
-        return 0;
+    QUOTA_STACK_UNWIND(fsetxattr, frame, op_ret, op_errno, NULL);
+    return 0;
 
 off:
-        STACK_WIND_TAIL (frame, FIRST_CHILD(this),
-                         FIRST_CHILD(this)->fops->fsetxattr, fd,
-                         dict, flags, xdata);
-        return 0;
+    STACK_WIND_TAIL(frame, FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->fsetxattr, fd, dict, flags, xdata);
+    return 0;
 }
 
-
 int
-quota_removexattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                       int32_t op_ret, int32_t op_errno, dict_t *xdata)
+quota_removexattr_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                      int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
-        QUOTA_STACK_UNWIND (removexattr, frame, op_ret, op_errno, xdata);
-        return 0;
+    QUOTA_STACK_UNWIND(removexattr, frame, op_ret, op_errno, xdata);
+    return 0;
 }
 
 int
-quota_removexattr (call_frame_t *frame, xlator_t *this,
-                   loc_t *loc, const char *name, dict_t *xdata)
+quota_removexattr(call_frame_t *frame, xlator_t *this, loc_t *loc,
+                  const char *name, dict_t *xdata)
 {
-        quota_priv_t       *priv        = NULL;
-        int32_t         op_errno = EINVAL;
+    quota_priv_t *priv = NULL;
+    int32_t op_errno = EINVAL;
 
-        priv = this->private;
+    priv = this->private;
 
-        WIND_IF_QUOTAOFF (priv->is_quota_on, off);
+    WIND_IF_QUOTAOFF(priv->is_quota_on, off);
 
-        VALIDATE_OR_GOTO (this, err);
+    VALIDATE_OR_GOTO(this, err);
 
-        /* all quota xattrs can be cleaned up by doing setxattr on special key.
-         * Hence its ok that we don't allow removexattr on quota keys here.
-         */
-        if (frame->root->pid >= 0) {
-                GF_IF_NATIVE_XATTR_GOTO ("trusted.glusterfs.quota*",
-                                         name, op_errno, err);
-                GF_IF_NATIVE_XATTR_GOTO ("trusted.pgfid*", name,
-                                         op_errno, err);
-        }
+    /* all quota xattrs can be cleaned up by doing setxattr on special key.
+     * Hence its ok that we don't allow removexattr on quota keys here.
+     */
+    if (frame->root->pid >= 0) {
+        GF_IF_NATIVE_XATTR_GOTO("trusted.glusterfs.quota*", name, op_errno,
+                                err);
+        GF_IF_NATIVE_XATTR_GOTO("trusted.pgfid*", name, op_errno, err);
+    }
 
-        VALIDATE_OR_GOTO (frame, err);
-        VALIDATE_OR_GOTO (loc, err);
+    VALIDATE_OR_GOTO(frame, err);
+    VALIDATE_OR_GOTO(loc, err);
 
-        STACK_WIND (frame, quota_removexattr_cbk,
-                    FIRST_CHILD(this), FIRST_CHILD(this)->fops->removexattr,
-                    loc, name, xdata);
-        return 0;
+    STACK_WIND(frame, quota_removexattr_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->removexattr, loc, name, xdata);
+    return 0;
 
 err:
-        QUOTA_STACK_UNWIND (removexattr, frame, -1,  op_errno, NULL);
-        return 0;
+    QUOTA_STACK_UNWIND(removexattr, frame, -1, op_errno, NULL);
+    return 0;
 
 off:
-        STACK_WIND_TAIL (frame, FIRST_CHILD(this),
-                         FIRST_CHILD(this)->fops->removexattr,
-                         loc, name, xdata);
-        return 0;
+    STACK_WIND_TAIL(frame, FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->removexattr, loc, name, xdata);
+    return 0;
 }
 
-
 int
-quota_fremovexattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                        int32_t op_ret, int32_t op_errno, dict_t *xdata)
+quota_fremovexattr_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                       int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
-        QUOTA_STACK_UNWIND (fremovexattr, frame, op_ret, op_errno, xdata);
-        return 0;
+    QUOTA_STACK_UNWIND(fremovexattr, frame, op_ret, op_errno, xdata);
+    return 0;
 }
 
 int
-quota_fremovexattr (call_frame_t *frame, xlator_t *this,
-                    fd_t *fd, const char *name, dict_t *xdata)
+quota_fremovexattr(call_frame_t *frame, xlator_t *this, fd_t *fd,
+                   const char *name, dict_t *xdata)
 {
-        quota_priv_t       *priv        = NULL;
-        int32_t         op_ret   = -1;
-        int32_t         op_errno = EINVAL;
-
-        priv = this->private;
-
-        WIND_IF_QUOTAOFF (priv->is_quota_on, off);
-
-        VALIDATE_OR_GOTO (frame, err);
-        VALIDATE_OR_GOTO (this, err);
-        VALIDATE_OR_GOTO (fd, err);
-
-        if (frame->root->pid >= 0) {
-                GF_IF_NATIVE_XATTR_GOTO ("trusted.glusterfs.quota*",
-                                         name, op_errno, err);
-                GF_IF_NATIVE_XATTR_GOTO ("trusted.pgfid*", name,
-                                         op_errno, err);
-        }
-        STACK_WIND (frame, quota_fremovexattr_cbk,
-                    FIRST_CHILD(this), FIRST_CHILD(this)->fops->fremovexattr,
-                    fd, name, xdata);
-        return 0;
+    quota_priv_t *priv = NULL;
+    int32_t op_ret = -1;
+    int32_t op_errno = EINVAL;
+
+    priv = this->private;
+
+    WIND_IF_QUOTAOFF(priv->is_quota_on, off);
+
+    VALIDATE_OR_GOTO(frame, err);
+    VALIDATE_OR_GOTO(this, err);
+    VALIDATE_OR_GOTO(fd, err);
+
+    if (frame->root->pid >= 0) {
+        GF_IF_NATIVE_XATTR_GOTO("trusted.glusterfs.quota*", name, op_errno,
+                                err);
+        GF_IF_NATIVE_XATTR_GOTO("trusted.pgfid*", name, op_errno, err);
+    }
+    STACK_WIND(frame, quota_fremovexattr_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->fremovexattr, fd, name, xdata);
+    return 0;
 err:
-        QUOTA_STACK_UNWIND (fremovexattr, frame, op_ret, op_errno, NULL);
-        return 0;
+    QUOTA_STACK_UNWIND(fremovexattr, frame, op_ret, op_errno, NULL);
+    return 0;
 
 off:
-        STACK_WIND_TAIL (frame, FIRST_CHILD(this),
-                         FIRST_CHILD(this)->fops->fremovexattr,
-                         fd, name, xdata);
-        return 0;
+    STACK_WIND_TAIL(frame, FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->fremovexattr, fd, name, xdata);
+    return 0;
 }
 
-
 int32_t
-quota_statfs_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                  int32_t op_ret, int32_t op_errno, struct statvfs *buf,
-                  dict_t *xdata)
+quota_statfs_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                 int32_t op_ret, int32_t op_errno, struct statvfs *buf,
+                 dict_t *xdata)
 {
-	inode_t           *inode        = NULL;
-        uint64_t           value        = 0;
-	int64_t            usage        = -1;
-	int64_t            avail        = -1;
-        int64_t            blocks       = 0;
-        quota_inode_ctx_t *ctx          = NULL;
-        int                ret          = 0;
-        quota_local_t     *local        = frame->local;
-
-        inode = cookie;
-
-        /* This fop will fail mostly in case of client disconnect's,
-         * which is already logged. Hence, not logging here */
-        if (op_ret == -1)
-                goto unwind;
-	/*
-	 * We should never get here unless quota_statfs (below) sent us a
-	 * cookie, and it would only do so if the value was non-NULL.  This
-	 * check is therefore just routine defensive coding.
-	 */
-	if (!inode) {
-		gf_log(this->name,GF_LOG_WARNING,
-		       "null inode, cannot adjust for quota");
-		goto unwind;
-	}
-
-        inode_ctx_get (inode, this, &value);
-        ctx = (quota_inode_ctx_t *)(unsigned long)value;
-        if (!ctx || ctx->hard_lim <= 0)
-                goto unwind;
-
-        { /* statfs is adjusted in this code block */
-                usage = (ctx->size) / buf->f_bsize;
-
-                blocks = ctx->hard_lim / buf->f_bsize;
-                buf->f_blocks = blocks;
-
-                avail = buf->f_blocks - usage;
-                avail = max (avail, 0);
-
-                buf->f_bfree = avail;
-                /*
-                 * We have to assume that the total assigned quota
-                 * won't cause us to dip into the reserved space,
-                 * because dealing with the overcommitted cases is
-                 * just too hairy (especially when different bricks
-                 * might be using different reserved percentages and
-                 * such).
-                 */
-                buf->f_bavail = buf->f_bfree;
-        }
+    inode_t *inode = NULL;
+    uint64_t value = 0;
+    int64_t usage = -1;
+    int64_t avail = -1;
+    int64_t blocks = 0;
+    quota_inode_ctx_t *ctx = NULL;
+    int ret = 0;
+
+    inode = cookie;
+
+    /* This fop will fail mostly in case of client disconnect,
+     * which is already logged. Hence, not logging here */
+    if (op_ret == -1)
+        goto unwind;
+    /*
+     * We should never get here unless quota_statfs (below) sent us a
+     * cookie, and it would only do so if the value was non-NULL.  This
+     * check is therefore just routine defensive coding.
+     */
+
+    GF_VALIDATE_OR_GOTO("quota", inode, unwind);
+
+    inode_ctx_get(inode, this, &value);
+    ctx = (quota_inode_ctx_t *)(unsigned long)value;
+    if (!ctx || ctx->hard_lim <= 0)
+        goto unwind;
+
+    { /* statfs is adjusted in this code block */
+        usage = (ctx->size) / buf->f_bsize;
+
+        blocks = ctx->hard_lim / buf->f_bsize;
+        buf->f_blocks = blocks;
+
+        avail = buf->f_blocks - usage;
+        avail = max(avail, 0);
+
+        buf->f_bfree = avail;
+        /*
+         * We have to assume that the total assigned quota
+         * won't cause us to dip into the reserved space,
+         * because dealing with the overcommitted cases is
+         * just too hairy (especially when different bricks
+         * might be using different reserved percentages and
+         * such).
+         */
+        buf->f_bavail = buf->f_bfree;
+    }
 
-        xdata = xdata ? dict_ref(xdata) : dict_new();
-        if (!xdata)
-                goto unwind;
+    xdata = xdata ? dict_ref(xdata) : dict_new();
+    if (!xdata)
+        goto unwind;
 
-        ret = dict_set_int8 (xdata, "quota-deem-statfs", 1);
-        if (-1 == ret)
-                gf_log (this->name, GF_LOG_ERROR, "Dict set failed, "
-                        "deem-statfs option may have no effect");
+    ret = dict_set_int8(xdata, "quota-deem-statfs", 1);
+    if (-1 == ret)
+        gf_msg(this->name, GF_LOG_ERROR, ENOMEM, Q_MSG_ENOMEM,
+               "Dict set failed, deem-statfs option may "
+               "have no effect");
 
 unwind:
-        QUOTA_STACK_UNWIND (statfs, frame, op_ret, op_errno, buf, xdata);
+    QUOTA_STACK_UNWIND(statfs, frame, op_ret, op_errno, buf, xdata);
 
-        if (xdata)
-                dict_unref (xdata);
+    if (xdata)
+        dict_unref(xdata);
 
-        return 0;
+    return 0;
 }
 
-
 int32_t
-quota_statfs_helper (call_frame_t *frame, xlator_t *this, loc_t *loc,
-                     dict_t *xdata)
+quota_statfs_helper(call_frame_t *frame, xlator_t *this, loc_t *loc,
+                    dict_t *xdata)
 {
-        quota_local_t   *local          = frame->local;
-        int              op_errno       = EINVAL;
+    quota_local_t *local = frame->local;
+    int op_errno = EINVAL;
 
-        GF_VALIDATE_OR_GOTO ("quota", local, err);
+    GF_VALIDATE_OR_GOTO("quota", local, err);
 
-        if (-1 == local->op_ret) {
-                op_errno = local->op_errno;
-                goto err;
-        }
+    if (-1 == local->op_ret) {
+        op_errno = local->op_errno;
+        goto err;
+    }
 
-        STACK_WIND_COOKIE (frame, quota_statfs_cbk, local->inode,
-                           FIRST_CHILD(this),
-                           FIRST_CHILD(this)->fops->statfs, loc, xdata);
-        return 0;
+    STACK_WIND_COOKIE(frame, quota_statfs_cbk, local->inode, FIRST_CHILD(this),
+                      FIRST_CHILD(this)->fops->statfs, loc, xdata);
+    return 0;
 err:
-        QUOTA_STACK_UNWIND (statfs, frame, -1, op_errno, NULL, NULL);
+    QUOTA_STACK_UNWIND(statfs, frame, -1, op_errno, NULL, NULL);
 
-        return 0;
+    return 0;
 }
 
 int32_t
-quota_statfs_validate_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                           int32_t op_ret, int32_t op_errno, inode_t *inode,
-                           struct iatt *buf, dict_t *xdata,
-                           struct iatt *postparent)
+quota_statfs_validate_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                          int32_t op_ret, int32_t op_errno, inode_t *inode,
+                          struct iatt *buf, dict_t *xdata,
+                          struct iatt *postparent)
 {
-        quota_local_t     *local      = NULL;
-        int32_t            ret        = 0;
-        quota_inode_ctx_t *ctx        = NULL;
-        uint64_t           value      = 0;
-        data_t            *data       = NULL;
-        quota_meta_t       size       = {0,};
-
-        local = frame->local;
-
-        if (op_ret < 0)
-                goto resume;
-
-        GF_ASSERT (local);
-        GF_ASSERT (frame);
-        GF_VALIDATE_OR_GOTO_WITH_ERROR ("quota", this, resume, op_errno,
-                                        EINVAL);
-        GF_VALIDATE_OR_GOTO_WITH_ERROR (this->name, xdata, resume, op_errno,
-                                        EINVAL);
-
-        ret = inode_ctx_get (local->validate_loc.inode, this, &value);
-
-        ctx = (quota_inode_ctx_t *)(unsigned long)value;
-        if ((ret == -1) || (ctx == NULL)) {
-                gf_log (this->name, GF_LOG_WARNING,
-                        "quota context is not present in inode (gfid:%s)",
-                        uuid_utoa (local->validate_loc.inode->gfid));
-                op_errno = EINVAL;
-                goto resume;
-        }
-
-        ret = quota_dict_get_meta (xdata, QUOTA_SIZE_KEY, &size);
-        if (ret == -1) {
-                gf_log (this->name, GF_LOG_WARNING, "dict get failed "
-                        "on quota size");
-                op_errno = EINVAL;
-        }
-
-        LOCK (&ctx->lock);
-        {
-                ctx->size = size.size;
-                ctx->file_count = size.file_count;
-                ctx->dir_count = size.dir_count;
-                gettimeofday (&ctx->tv, NULL);
-        }
-        UNLOCK (&ctx->lock);
+    quota_local_t *local = NULL;
+    int32_t ret = 0;
+    quota_inode_ctx_t *ctx = NULL;
+    uint64_t value = 0;
+    quota_meta_t size = {
+        0,
+    };
+
+    local = frame->local;
+
+    if (op_ret < 0)
+        goto resume;
+
+    GF_ASSERT(local);
+    GF_ASSERT(frame);
+    GF_VALIDATE_OR_GOTO_WITH_ERROR("quota", this, resume, op_errno, EINVAL);
+    GF_VALIDATE_OR_GOTO_WITH_ERROR(this->name, xdata, resume, op_errno, EINVAL);
+
+    ret = inode_ctx_get(local->validate_loc.inode, this, &value);
+
+    ctx = (quota_inode_ctx_t *)(unsigned long)value;
+    if ((ret == -1) || (ctx == NULL)) {
+        gf_msg(this->name, GF_LOG_WARNING, EINVAL, Q_MSG_INODE_CTX_GET_FAILED,
+               "quota context is not present in inode (gfid:%s)",
+               uuid_utoa(local->validate_loc.inode->gfid));
+        op_errno = EINVAL;
+        goto resume;
+    }
+
+    ret = quota_dict_get_meta(xdata, QUOTA_SIZE_KEY, SLEN(QUOTA_SIZE_KEY),
+                              &size);
+    if (ret == -1) {
+        gf_msg(this->name, GF_LOG_WARNING, EINVAL, Q_MSG_SIZE_KEY_MISSING,
+               "size key not present in "
+               "dict");
+        op_errno = EINVAL;
+    }
+
+    LOCK(&ctx->lock);
+    {
+        ctx->size = size.size;
+        ctx->validate_time = gf_time();
+        ctx->file_count = size.file_count;
+        ctx->dir_count = size.dir_count;
+    }
+    UNLOCK(&ctx->lock);
 
 resume:
-        quota_link_count_decrement (local);
-        return 0;
+    local->op_errno = op_errno;
+    quota_link_count_decrement(frame);
+    return 0;
 }
 
 void
-quota_get_limit_dir_continuation (struct list_head *parents, inode_t *inode,
-                                  int32_t op_ret, int32_t op_errno, void *data)
+quota_get_limit_dir_continuation(struct list_head *parents, inode_t *inode,
+                                 int32_t op_ret, int32_t op_errno, void *data)
 {
-        call_frame_t   *frame        = NULL;
-        xlator_t       *this         = NULL;
-        quota_local_t  *local        = NULL;
-        quota_dentry_t *entry        = NULL;
-        inode_t        *parent       = NULL;
-
-        frame = data;
-        local = frame->local;
-        this = THIS;
-
-        if ((op_ret < 0) || list_empty (parents)) {
-                if (op_ret >= 0) {
-                        gf_log (this->name, GF_LOG_WARNING,
-                                "Couldn't build ancestry for inode (gfid:%s). "
-                                "Without knowing ancestors till root, quota "
-                                "cannot be enforced. "
-                                "Hence, failing fop with EIO",
-                                uuid_utoa (inode->gfid));
-                        op_errno = EIO;
-                }
+    call_frame_t *frame = NULL;
+    xlator_t *this = NULL;
+    quota_dentry_t *entry = NULL;
+    inode_t *parent = NULL;
 
-                quota_handle_validate_error (local, -1, op_errno);
-                goto out;
+    frame = data;
+    this = THIS;
+
+    if ((op_ret < 0) || list_empty(parents)) {
+        if (op_ret >= 0) {
+            gf_msg(this->name, GF_LOG_WARNING, EIO, Q_MSG_ANCESTRY_BUILD_FAILED,
+                   "Couldn't build ancestry for inode (gfid:%s). "
+                   "Without knowing ancestors till root, quota "
+                   "cannot be enforced. "
+                   "Hence, failing fop with EIO",
+                   uuid_utoa(inode->gfid));
+            op_errno = EIO;
         }
 
-        entry = list_entry (parents, quota_dentry_t, next);
-        parent = inode_find (inode->table, entry->par);
+        quota_handle_validate_error(frame, -1, op_errno);
+        goto out;
+    }
+
+    entry = list_entry(parents, quota_dentry_t, next);
+    parent = inode_find(inode->table, entry->par);
 
-        quota_get_limit_dir (frame, parent, this);
+    quota_get_limit_dir(frame, parent, this);
 
-        inode_unref (parent);
+    inode_unref(parent);
 out:
-        return;
+    return;
 }
 
 void
-quota_statfs_continue (call_frame_t *frame, xlator_t *this, inode_t *inode)
+quota_statfs_continue(call_frame_t *frame, xlator_t *this, inode_t *inode)
 {
-        call_stub_t     *stub           = NULL;
-        quota_local_t   *local          = frame->local;
-        int              ret            = -1;
-
-        stub = fop_statfs_stub (frame, quota_statfs_helper,
-                                &local->loc, local->xdata);
-        if (!stub)
-                goto err;
-
-        LOCK (&local->lock);
-        {
-                local->inode = inode_ref (inode);
-                local->link_count = 1;
-                local->stub = stub;
-        }
-        UNLOCK (&local->lock);
-
-        ret = quota_validate (frame, local->inode, this,
-                              quota_statfs_validate_cbk);
-        if (0 > ret)
-                quota_handle_validate_error (local, -1, -ret);
-        return;
-
-err:
-        QUOTA_STACK_UNWIND (statfs, frame, -1, ENOMEM, NULL, NULL);
-
-        return;
+    quota_local_t *local = frame->local;
+    int ret = -1;
+
+    LOCK(&local->lock);
+    {
+        local->inode = inode_ref(inode);
+    }
+    UNLOCK(&local->lock);
+
+    ret = quota_validate(frame, local->inode, this, quota_statfs_validate_cbk);
+    if (0 > ret)
+        quota_handle_validate_error(frame, -1, -ret);
 }
 
 void
-quota_get_limit_dir (call_frame_t *frame, inode_t *cur_inode, xlator_t *this)
+quota_get_limit_dir(call_frame_t *frame, inode_t *cur_inode, xlator_t *this)
 {
-        inode_t                *inode           = NULL;
-        inode_t                *parent          = NULL;
-        uint64_t                value           = 0;
-        quota_inode_ctx_t      *ctx             = NULL;
-        int                     ret             = -1;
-        quota_local_t          *local           = frame->local;
-
-        if (!cur_inode)
-                goto out;
+    inode_t *inode = NULL;
+    inode_t *parent = NULL;
+    uint64_t value = 0;
+    quota_inode_ctx_t *ctx = NULL;
+    quota_local_t *local = frame->local;
 
-        inode = inode_ref (cur_inode);
-        while (inode) {
-                value = 0;
-                inode_ctx_get (inode, this, &value);
+    if (!cur_inode)
+        goto out;
 
-                if (value) {
-                        ctx = (quota_inode_ctx_t *)(unsigned long)value;
-                        if (ctx->hard_lim > 0)
-                                break;
-                }
+    inode = inode_ref(cur_inode);
+    while (inode) {
+        value = 0;
+        inode_ctx_get(inode, this, &value);
 
-                if (__is_root_gfid (inode->gfid))
-                        goto off;
+        if (value) {
+            ctx = (quota_inode_ctx_t *)(unsigned long)value;
+            if (ctx->hard_lim > 0)
+                break;
+        }
 
-                parent = inode_parent (inode, 0, NULL);
-                if (!parent) {
-                        ret = quota_build_ancestry
-                               (inode, quota_get_limit_dir_continuation,
-                                (void *)frame);
-                        goto out;
-                }
+        if (__is_root_gfid(inode->gfid))
+            goto off;
 
-                inode_unref (inode);
-                inode = parent;
+        parent = inode_parent(inode, 0, NULL);
+        if (!parent) {
+            (void)quota_build_ancestry(inode, quota_get_limit_dir_continuation,
+                                       frame);
+            goto out;
         }
 
-        quota_statfs_continue (frame, this, inode);
-        inode_unref (inode);
-        return;
+        inode_unref(inode);
+        inode = parent;
+    }
+
+    quota_statfs_continue(frame, this, inode);
+    inode_unref(inode);
+    return;
 
 off:
-        gf_log (this->name, GF_LOG_DEBUG,
-                "No limit set on the inode or it's parents.");
+    gf_msg_debug(this->name, 0, "No limit set on the inode or it's parents.");
 
-        QUOTA_STACK_WIND_TAIL (frame, FIRST_CHILD(this),
-                               FIRST_CHILD(this)->fops->statfs,
-                               &local->loc, local->xdata);
+    QUOTA_STACK_WIND_TAIL(frame, FIRST_CHILD(this),
+                          FIRST_CHILD(this)->fops->statfs, &local->loc,
+                          local->xdata);
 out:
-        inode_unref (inode);
+    inode_unref(inode);
 
-        return;
+    return;
 }
 
 int32_t
-quota_statfs (call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xdata)
+quota_statfs(call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xdata)
 {
-        int              op_errno            = 0;
-        int              ret                 = -1;
-        int8_t           ignore_deem_statfs  = 0;
-        quota_priv_t    *priv                = NULL;
-        quota_local_t   *local               = NULL;
+    int op_errno = 0;
+    int ret = -1;
+    int8_t ignore_deem_statfs = 0;
+    quota_priv_t *priv = NULL;
+    quota_local_t *local = NULL;
+    call_stub_t *stub = NULL;
 
-        priv = this->private;
-        GF_ASSERT (loc);
+    priv = this->private;
+    GF_ASSERT(loc);
 
-        WIND_IF_QUOTAOFF (priv->is_quota_on, off);
-
-        ret = dict_get_int8 (xdata, GF_INTERNAL_IGNORE_DEEM_STATFS,
-                             &ignore_deem_statfs);
-        ret = 0;
+    WIND_IF_QUOTAOFF(priv->is_quota_on, off);
 
-        if (ignore_deem_statfs)
-                goto off;
+    ret = dict_get_int8(xdata, GF_INTERNAL_IGNORE_DEEM_STATFS,
+                        &ignore_deem_statfs);
+    ret = 0;
 
-	if (priv->consider_statfs && loc->inode) {
-                local = quota_local_new ();
-                if (!local) {
-                        op_errno = ENOMEM;
-                        goto err;
-                }
+    if (ignore_deem_statfs)
+        goto off;
 
-                ret = loc_copy (&local->loc, loc);
-                if (-1 == ret) {
-                        op_errno = ENOMEM;
-                        goto err;
-                }
+    if (priv->consider_statfs && loc->inode) {
+        local = quota_local_new();
+        if (!local) {
+            op_errno = ENOMEM;
+            goto err;
+        }
+        frame->local = local;
 
-                if (xdata)
-                        local->xdata = dict_ref (xdata);
+        ret = loc_copy(&local->loc, loc);
+        if (-1 == ret) {
+            op_errno = ENOMEM;
+            goto err;
+        }
 
-                local->link_count = 1;
+        if (xdata)
+            local->xdata = dict_ref(xdata);
 
-                frame->local = local;
+        stub = fop_statfs_stub(frame, quota_statfs_helper, &local->loc,
+                               local->xdata);
+        if (!stub) {
+            op_errno = ENOMEM;
+            goto err;
+        }
 
-                quota_get_limit_dir (frame, loc->inode, this);
+        LOCK(&local->lock);
+        {
+            local->link_count = 1;
+            local->stub = stub;
+        }
+        UNLOCK(&local->lock);
 
-                return 0;
-	}
+        quota_get_limit_dir(frame, loc->inode, this);
 
-        /*
-         * We have to make sure that we never get to quota_statfs_cbk
-         * with a cookie that points to something other than an inode,
-         * which is exactly what would happen with STACK_UNWIND using
-         * that as a callback.  Therefore, use default_statfs_cbk in
-         * this case instead.
-         *
-         * Also if the option deem-statfs is not set to "on" don't
-         * bother calculating quota limit on / in statfs_cbk.
-         */
-        if (priv->consider_statfs)
-                gf_log (this->name, GF_LOG_ERROR,
-                        "Missing inode, can't adjust for quota");
+        return 0;
+    }
 
+    /*
+     * We have to make sure that we never get to quota_statfs_cbk
+     * with a cookie that points to something other than an inode,
+     * which is exactly what would happen with STACK_UNWIND using
+     * that as a callback.  Therefore, use default_statfs_cbk in
+     * this case instead.
+     *
+     * Also if the option deem-statfs is not set to "on" don't
+     * bother calculating quota limit on / in statfs_cbk.
+     */
+    if (priv->consider_statfs)
+        gf_log(this->name, GF_LOG_ERROR,
+               "Missing inode, can't adjust for quota");
 
 off:
-        STACK_WIND_TAIL (frame, FIRST_CHILD(this),
-                         FIRST_CHILD(this)->fops->statfs, loc, xdata);
-        return 0;
+    STACK_WIND_TAIL(frame, FIRST_CHILD(this), FIRST_CHILD(this)->fops->statfs,
+                    loc, xdata);
+    return 0;
 
 err:
-        QUOTA_STACK_UNWIND (statfs, frame, -1, op_errno, NULL, NULL);
+    QUOTA_STACK_UNWIND(statfs, frame, -1, op_errno, NULL, NULL);
 
-        return 0;
+    return 0;
 }
 
 int
-quota_readdirp_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                    int op_ret, int op_errno, gf_dirent_t *entries,
-                    dict_t *xdata)
+quota_readdirp_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                   int op_ret, int op_errno, gf_dirent_t *entries,
+                   dict_t *xdata)
 {
-        gf_dirent_t   *entry = NULL;
-        quota_local_t *local = NULL;
-        loc_t          loc   = {0, };
+    gf_dirent_t *entry = NULL;
+    quota_local_t *local = NULL;
+    loc_t loc = {
+        0,
+    };
 
-        if (op_ret <= 0)
-                goto unwind;
+    if (op_ret <= 0)
+        goto unwind;
 
-        local = frame->local;
+    local = frame->local;
 
-        list_for_each_entry (entry, &entries->list, list) {
-                if ((strcmp (entry->d_name, ".") == 0)
-                    || (strcmp (entry->d_name, "..") == 0))
-                        continue;
+    list_for_each_entry(entry, &entries->list, list)
+    {
+        if ((strcmp(entry->d_name, ".") == 0) ||
+            (strcmp(entry->d_name, "..") == 0) || entry->inode == NULL)
+            continue;
 
-                gf_uuid_copy (loc.gfid, entry->d_stat.ia_gfid);
-                loc.inode = inode_ref (entry->inode);
-                loc.parent = inode_ref (local->loc.inode);
-                gf_uuid_copy (loc.pargfid, loc.parent->gfid);
-                loc.name = entry->d_name;
+        gf_uuid_copy(loc.gfid, entry->d_stat.ia_gfid);
+        loc.inode = inode_ref(entry->inode);
+        loc.parent = inode_ref(local->loc.inode);
+        gf_uuid_copy(loc.pargfid, loc.parent->gfid);
+        loc.name = entry->d_name;
 
-                quota_fill_inodectx (this, entry->inode, entry->dict,
-                                     &loc, &entry->d_stat, &op_errno);
+        quota_fill_inodectx(this, entry->inode, entry->dict, &loc,
+                            &entry->d_stat, &op_errno);
 
-                loc_wipe (&loc);
-        }
+        loc_wipe(&loc);
+    }
 
 unwind:
-        QUOTA_STACK_UNWIND (readdirp, frame, op_ret, op_errno, entries, xdata);
+    QUOTA_STACK_UNWIND(readdirp, frame, op_ret, op_errno, entries, xdata);
 
-        return 0;
+    return 0;
 }
 
 int
-quota_readdirp (call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
-                off_t offset, dict_t *dict)
+quota_readdirp(call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
+               off_t offset, dict_t *dict)
 {
-        quota_priv_t  *priv     = NULL;
-        int            ret      = 0;
-        gf_boolean_t   new_dict = _gf_false;
-        quota_local_t *local    = NULL;
+    quota_priv_t *priv = NULL;
+    int ret = 0;
+    gf_boolean_t new_dict = _gf_false;
+    quota_local_t *local = NULL;
 
-        priv = this->private;
+    priv = this->private;
 
-        WIND_IF_QUOTAOFF (priv->is_quota_on, off);
+    WIND_IF_QUOTAOFF(priv->is_quota_on, off);
 
-        local = quota_local_new ();
+    local = quota_local_new();
 
-        if (local == NULL) {
-                goto err;
-        }
+    if (local == NULL) {
+        goto err;
+    }
 
-        frame->local = local;
+    frame->local = local;
 
-        local->loc.inode = inode_ref (fd->inode);
+    local->loc.inode = inode_ref(fd->inode);
 
-        if (dict == NULL) {
-                dict = dict_new ();
-                new_dict = _gf_true;
-        }
+    if (dict == NULL) {
+        dict = dict_new();
+        new_dict = _gf_true;
+    }
 
-        if (dict) {
-                ret = dict_set_int8 (dict, QUOTA_LIMIT_KEY, 1);
-                if (ret < 0) {
-                        gf_log (this->name, GF_LOG_WARNING,
-                                "dict set of key for hard-limit failed");
-                        goto err;
-                }
+    if (dict) {
+        ret = dict_set_int8(dict, QUOTA_LIMIT_KEY, 1);
+        if (ret < 0) {
+            gf_msg(this->name, GF_LOG_WARNING, ENOMEM, Q_MSG_ENOMEM,
+                   "dict set of key for hard-limit");
+            goto err;
         }
+    }
 
-        if (dict) {
-                ret = dict_set_int8 (dict, QUOTA_LIMIT_OBJECTS_KEY, 1);
-                if (ret < 0) {
-                        gf_log (this->name, GF_LOG_WARNING,
-                                "dict set of key for hard-limit failed");
-                        goto err;
-                }
+    if (dict) {
+        ret = dict_set_int8(dict, QUOTA_LIMIT_OBJECTS_KEY, 1);
+        if (ret < 0) {
+            gf_msg(this->name, GF_LOG_WARNING, ENOMEM, Q_MSG_ENOMEM,
+                   "dict set of key for hard-limit "
+                   "failed");
+            goto err;
         }
+    }
 
-        STACK_WIND (frame, quota_readdirp_cbk,
-                    FIRST_CHILD(this), FIRST_CHILD(this)->fops->readdirp, fd,
-                    size, offset, dict);
+    STACK_WIND(frame, quota_readdirp_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->readdirp, fd, size, offset, dict);
 
-        if (new_dict) {
-                dict_unref (dict);
-        }
+    if (new_dict) {
+        dict_unref(dict);
+    }
 
-        return 0;
+    return 0;
 err:
-        STACK_UNWIND_STRICT (readdirp, frame, -1, EINVAL, NULL, NULL);
+    STACK_UNWIND_STRICT(readdirp, frame, -1, EINVAL, NULL, NULL);
 
-        if (new_dict) {
-                dict_unref (dict);
-        }
+    if (new_dict) {
+        dict_unref(dict);
+    }
 
-        return 0;
+    return 0;
 
 off:
-        STACK_WIND_TAIL (frame, FIRST_CHILD(this),
-                         FIRST_CHILD(this)->fops->readdirp, fd,
-                         size, offset, dict);
-        return 0;
+    STACK_WIND_TAIL(frame, FIRST_CHILD(this), FIRST_CHILD(this)->fops->readdirp,
+                    fd, size, offset, dict);
+    return 0;
 }
 
 int32_t
@@ -4538,551 +4644,693 @@ quota_fallocate_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
                     int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
                     struct iatt *postbuf, dict_t *xdata)
 {
-        int32_t                  ret            = 0;
-        uint64_t                 ctx_int        = 0;
-        quota_inode_ctx_t       *ctx            = NULL;
-        quota_local_t           *local          = NULL;
-
-        local = frame->local;
-
-        if ((op_ret < 0) || (local == NULL)) {
-                goto out;
-        }
-
-        ret = inode_ctx_get (local->loc.inode, this, &ctx_int);
-        if (ret) {
-                gf_log (this->name, GF_LOG_WARNING,
-                        "%s: failed to get the context", local->loc.path);
-                goto out;
-        }
-
-        ctx = (quota_inode_ctx_t *)(unsigned long) ctx_int;
-
-        if (ctx == NULL) {
-                gf_log (this->name, GF_LOG_WARNING,
-                        "quota context not set in %s (gfid:%s)",
-                        local->loc.path, uuid_utoa (local->loc.inode->gfid));
-                goto out;
-        }
-
-        LOCK (&ctx->lock);
-        {
-                ctx->buf = *postbuf;
-        }
-        UNLOCK (&ctx->lock);
+    int32_t ret = 0;
+    uint64_t ctx_int = 0;
+    quota_inode_ctx_t *ctx = NULL;
+    quota_local_t *local = NULL;
+
+    local = frame->local;
+
+    if ((op_ret < 0) || (local == NULL)) {
+        goto out;
+    }
+
+    ret = inode_ctx_get(local->loc.inode, this, &ctx_int);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_WARNING, 0, Q_MSG_INODE_CTX_GET_FAILED,
+               "%s: failed to get the context", local->loc.path);
+        goto out;
+    }
+
+    ctx = (quota_inode_ctx_t *)(unsigned long)ctx_int;
+
+    if (ctx == NULL) {
+        gf_msg(this->name, GF_LOG_WARNING, 0, Q_MSG_INODE_CTX_GET_FAILED,
+               "quota context not set in %s (gfid:%s)", local->loc.path,
+               uuid_utoa(local->loc.inode->gfid));
+        goto out;
+    }
+
+    LOCK(&ctx->lock);
+    {
+        ctx->buf = *postbuf;
+    }
+    UNLOCK(&ctx->lock);
 
 out:
-        QUOTA_STACK_UNWIND (fallocate, frame, op_ret, op_errno, prebuf, postbuf,
-                            xdata);
+    QUOTA_STACK_UNWIND(fallocate, frame, op_ret, op_errno, prebuf, postbuf,
+                       xdata);
 
-        return 0;
+    return 0;
 }
 
-
 int32_t
-quota_fallocate_helper (call_frame_t *frame, xlator_t *this, fd_t *fd,
-                        int32_t mode, off_t offset, size_t len, dict_t *xdata)
+quota_fallocate_helper(call_frame_t *frame, xlator_t *this, fd_t *fd,
+                       int32_t mode, off_t offset, size_t len, dict_t *xdata)
 {
-        quota_local_t *local    = NULL;
-        int32_t        op_errno = EINVAL;
-        quota_priv_t  *priv     = NULL;
+    quota_local_t *local = NULL;
+    int32_t op_errno = EINVAL;
 
-        local = frame->local;
-        if (local == NULL) {
-                gf_log (this->name, GF_LOG_WARNING, "local is NULL");
-                goto unwind;
-        }
+    local = frame->local;
 
-        priv = this->private;
+    GF_VALIDATE_OR_GOTO("quota", local, unwind);
 
-        if (local->op_ret == -1) {
-                op_errno = local->op_errno;
-                goto unwind;
+    if (local->op_ret == -1) {
+        op_errno = local->op_errno;
+        if (op_errno == ENOENT || op_errno == ESTALE) {
+            /* We may get ENOENT/ESTALE in case of below scenario
+             *     fd = open file.txt
+             *     unlink file.txt
+             *     fallocate on fd
+             * Here build_ancestry can fail as the file is removed.
+             * For now ignore ENOENT/ESTALE on active fd
+             * We need to re-visit this code once we understand
+             * how other file-system behave in this scenario
+             */
+            gf_msg_debug(this->name, 0,
+                         "quota enforcer failed "
+                         "with ENOENT/ESTALE on %s, cannot check "
+                         "quota limits and allowing fallocate",
+                         uuid_utoa(fd->inode->gfid));
+        } else {
+            goto unwind;
         }
+    }
 
-        STACK_WIND (frame, quota_fallocate_cbk,
-                    FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->fallocate, fd, mode, offset, len,
-		    xdata);
-        return 0;
+    STACK_WIND(frame, quota_fallocate_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->fallocate, fd, mode, offset, len,
+               xdata);
+    return 0;
 
 unwind:
-        QUOTA_STACK_UNWIND (fallocate, frame, -1, op_errno, NULL, NULL, NULL);
-        return 0;
+    QUOTA_STACK_UNWIND(fallocate, frame, -1, op_errno, NULL, NULL, NULL);
+    return 0;
 }
 
-
 int32_t
 quota_fallocate(call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t mode,
-		off_t offset, size_t len, dict_t *xdata)
+                off_t offset, size_t len, dict_t *xdata)
 {
-        int32_t            ret     = -1, op_errno = EINVAL;
-        int32_t            parents = 0;
-        quota_local_t     *local   = NULL;
-        quota_inode_ctx_t *ctx     = NULL;
-        quota_priv_t      *priv    = NULL;
-        quota_dentry_t    *dentry  = NULL;
-        call_stub_t       *stub    = NULL;
-
-        priv = this->private;
-        GF_VALIDATE_OR_GOTO (this->name, priv, unwind);
-
-        WIND_IF_QUOTAOFF (priv->is_quota_on, off);
-
-        GF_ASSERT (frame);
-        GF_VALIDATE_OR_GOTO ("quota", this, unwind);
-        GF_VALIDATE_OR_GOTO (this->name, fd, unwind);
-
-        local = quota_local_new ();
-        if (local == NULL) {
-                goto unwind;
-        }
-
-        frame->local = local;
-        local->loc.inode = inode_ref (fd->inode);
-
-        ret = quota_inode_ctx_get (fd->inode, this, &ctx, 0);
-        if (ctx == NULL) {
-                gf_log (this->name, GF_LOG_DEBUG, "quota context is "
-                        "NULL on inode (%s). "
-                        "If quota is not enabled recently and crawler "
-                        "has finished crawling, its an error",
-                        uuid_utoa (local->loc.inode->gfid));
-        }
-
-        stub = fop_fallocate_stub(frame, quota_fallocate_helper, fd, mode,
-                                  offset, len, xdata);
-        if (stub == NULL) {
-                op_errno = ENOMEM;
-                goto unwind;
+    int32_t op_errno = EINVAL;
+    int32_t parents = 0;
+    int32_t fail_count = 0;
+    quota_local_t *local = NULL;
+    quota_inode_ctx_t *ctx = NULL;
+    quota_priv_t *priv = NULL;
+    quota_dentry_t *dentry = NULL;
+    quota_dentry_t *tmp = NULL;
+    call_stub_t *stub = NULL;
+    struct list_head head = {
+        0,
+    };
+    inode_t *par_inode = NULL;
+
+    priv = this->private;
+    GF_VALIDATE_OR_GOTO(this->name, priv, unwind);
+
+    WIND_IF_QUOTAOFF(priv->is_quota_on, off);
+
+    INIT_LIST_HEAD(&head);
+
+    GF_ASSERT(frame);
+    GF_VALIDATE_OR_GOTO("quota", this, unwind);
+    GF_VALIDATE_OR_GOTO(this->name, fd, unwind);
+
+    local = quota_local_new();
+    if (local == NULL) {
+        goto unwind;
+    }
+
+    frame->local = local;
+    local->loc.inode = inode_ref(fd->inode);
+
+    (void)quota_inode_ctx_get(fd->inode, this, &ctx, 0);
+    if (ctx == NULL) {
+        gf_msg_debug(this->name, 0,
+                     "quota context is NULL on inode"
+                     " (%s). If quota is not enabled recently and "
+                     "crawler has finished crawling, its an error",
+                     uuid_utoa(local->loc.inode->gfid));
+    }
+
+    stub = fop_fallocate_stub(frame, quota_fallocate_helper, fd, mode, offset,
+                              len, xdata);
+    if (stub == NULL) {
+        op_errno = ENOMEM;
+        goto unwind;
+    }
+
+    priv = this->private;
+    GF_VALIDATE_OR_GOTO(this->name, priv, unwind);
+
+    parents = quota_add_parents_from_ctx(ctx, &head);
+    if (parents == -1) {
+        op_errno = errno;
+        goto unwind;
+    }
+
+    /*
+     * Note that by using len as the delta we're assuming the range from
+     * offset to offset+len has not already been allocated. This can result
+     * in ENOSPC errors attempting to allocate an already allocated range.
+     */
+    local->delta = len;
+    local->object_delta = 0;
+    local->stub = stub;
+    local->link_count = parents;
+
+    if (parents == 0) {
+        local->link_count = 1;
+        quota_check_limit(frame, fd->inode, this);
+    } else {
+        list_for_each_entry_safe(dentry, tmp, &head, next)
+        {
+            par_inode = do_quota_check_limit(frame, fd->inode, this, dentry,
+                                             _gf_false);
+            if (par_inode == NULL) {
+                /* remove stale entry from inode_ctx */
+                quota_dentry_del(ctx, dentry->name, dentry->par);
+                parents--;
+                fail_count++;
+            } else {
+                inode_unref(par_inode);
+            }
+            __quota_dentry_free(dentry);
         }
 
-        priv = this->private;
-        GF_VALIDATE_OR_GOTO (this->name, priv, unwind);
-
-        if (ctx != NULL) {
-                LOCK (&ctx->lock);
-                {
-                        list_for_each_entry (dentry, &ctx->parents, next) {
-                                parents++;
-                        }
-                }
-                UNLOCK (&ctx->lock);
+        if (parents == 0) {
+            LOCK(&local->lock);
+            {
+                local->link_count++;
+            }
+            UNLOCK(&local->lock);
+            quota_check_limit(frame, fd->inode, this);
         }
 
-	/*
-	 * Note that by using len as the delta we're assuming the range from
-	 * offset to offset+len has not already been allocated. This can result
-	 * in ENOSPC errors attempting to allocate an already allocated range.
-	 */
-        local->delta = len;
-        local->stub = stub;
-        local->link_count = parents;
-
-        if (parents == 0) {
-                local->link_count = 1;
-                quota_check_limit (frame, fd->inode, this, NULL, NULL);
-        } else {
-                list_for_each_entry (dentry, &ctx->parents, next) {
-                        quota_check_limit (frame, fd->inode, this, dentry->name,
-                                           dentry->par);
-                }
+        while (fail_count != 0) {
+            quota_link_count_decrement(frame);
+            fail_count--;
         }
+    }
 
-        return 0;
+    return 0;
 
 unwind:
-        QUOTA_STACK_UNWIND (fallocate, frame, -1, op_errno, NULL, NULL, NULL);
-        return 0;
+    QUOTA_STACK_UNWIND(fallocate, frame, -1, op_errno, NULL, NULL, NULL);
+    return 0;
 
 off:
-        STACK_WIND_TAIL (frame, FIRST_CHILD(this),
-                         FIRST_CHILD(this)->fops->fallocate, fd, mode, offset,
-                         len, xdata);
-        return 0;
+    STACK_WIND_TAIL(frame, FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->fallocate, fd, mode, offset, len,
+                    xdata);
+    return 0;
 }
 
 void
-quota_log_helper (char **usage_str, int64_t cur_size, inode_t *inode,
-                  char **path, struct timeval *cur_time)
+quota_log_helper(char **usage_str, int64_t cur_size, inode_t *inode,
+                 char **path, time_t *cur_time)
 {
-        xlator_t        *this   = THIS;
+    xlator_t *this = THIS;
 
-        if (!usage_str || !inode || !path || !cur_time) {
-                gf_log (this->name, GF_LOG_ERROR, "Received null argument");
-                return;
-        }
+    if (!usage_str || !inode || !path || !cur_time) {
+        gf_log(this->name, GF_LOG_ERROR, "Received null argument");
+        return;
+    }
 
-        *usage_str = gf_uint64_2human_readable (cur_size);
-        if (!(*usage_str))
-                gf_log (this->name, GF_LOG_ERROR, "integer to string "
-                        "conversion failed Reason:\"Cannot allocate memory\"");
+    *usage_str = gf_uint64_2human_readable(cur_size);
+    if (!(*usage_str))
+        gf_msg(this->name, GF_LOG_ERROR, ENOMEM, Q_MSG_ENOMEM,
+               "integer to string conversion failed Reason"
+               ":\"Cannot allocate memory\"");
 
-        inode_path (inode, NULL, path);
-        if (!(*path))
-                *path = uuid_utoa (inode->gfid);
+    inode_path(inode, NULL, path);
+    if (!(*path))
+        *path = uuid_utoa(inode->gfid);
 
-        gettimeofday (cur_time, NULL);
+    *cur_time = gf_time();
 }
 
 /* Logs if
-*  i.   Usage crossed soft limit
-*  ii.  Usage above soft limit and alert-time elapsed
-*/
+ *  i.   Usage crossed soft limit
+ *  ii.  Usage above soft limit and alert-time elapsed
+ */
 void
-quota_log_usage (xlator_t *this, quota_inode_ctx_t *ctx, inode_t *inode,
-                 int64_t delta)
+quota_log_usage(xlator_t *this, quota_inode_ctx_t *ctx, inode_t *inode,
+                int64_t delta)
 {
-        struct timeval           cur_time       = {0,};
-        char                    *usage_str      = NULL;
-        char                    *path           = NULL;
-        int64_t                  cur_size       = 0;
-        quota_priv_t            *priv           = NULL;
+    time_t cur_time = 0;
+    char *usage_str = NULL;
+    char *path = NULL;
+    int64_t cur_size = 0;
+    quota_priv_t *priv = NULL;
 
-        priv = this->private;
-        cur_size = ctx->size + delta;
+    priv = this->private;
+    cur_size = ctx->size + delta;
 
-        if ((ctx->soft_lim <= 0) || cur_size < ctx->soft_lim)
-                return;
+    if ((ctx->soft_lim <= 0) || cur_size < ctx->soft_lim)
+        return;
 
-        /* Usage crossed/reached soft limit */
-        if (DID_REACH_LIMIT (ctx->soft_lim, ctx->size, cur_size)) {
+    /* Usage crossed/reached soft limit */
+    if (DID_REACH_LIMIT(ctx->soft_lim, ctx->size, cur_size)) {
+        quota_log_helper(&usage_str, cur_size, inode, &path, &cur_time);
 
-                quota_log_helper (&usage_str, cur_size, inode,
-                                  &path, &cur_time);
+        gf_msg(this->name, GF_LOG_ALERT, 0, Q_MSG_CROSSED_SOFT_LIMIT,
+               "Usage crossed soft limit: "
+               "%s used by %s",
+               usage_str, path);
 
-                gf_log (this->name, GF_LOG_ALERT, "Usage crossed "
-                        "soft limit: %s used by %s", usage_str, path);
-                ctx->prev_log = cur_time;
-        }
-        /* Usage is above soft limit */
-        else if (cur_size > ctx->soft_lim &&
-                           quota_timeout (&ctx->prev_log, priv->log_timeout)) {
+        gf_event(EVENT_QUOTA_CROSSED_SOFT_LIMIT,
+                 "Usage=%s;volume=%s;"
+                 "path=%s",
+                 usage_str, priv->volume_uuid, path);
 
-                quota_log_helper (&usage_str, cur_size, inode,
-                                  &path, &cur_time);
+        ctx->prev_log_time = cur_time;
 
-                gf_log (this->name, GF_LOG_ALERT, "Usage is above "
-                        "soft limit: %s used by %s", usage_str, path);
-                ctx->prev_log = cur_time;
-        }
+    }
+    /* Usage is above soft limit */
+    else if (cur_size > ctx->soft_lim &&
+             quota_timeout(ctx->prev_log_time, priv->log_timeout)) {
+        quota_log_helper(&usage_str, cur_size, inode, &path, &cur_time);
+
+        gf_msg(this->name, GF_LOG_ALERT, 0, Q_MSG_CROSSED_SOFT_LIMIT,
+               "Usage is above soft limit: %s used by %s", usage_str, path);
 
-        if (usage_str)
-                GF_FREE (usage_str);
+        gf_event(EVENT_QUOTA_CROSSED_SOFT_LIMIT,
+                 "Usage=%s;volume=%s;"
+                 "path=%s",
+                 usage_str, priv->volume_uuid, path);
+
+        ctx->prev_log_time = cur_time;
+    }
+
+    if (path)
+        GF_FREE(path);
+
+    if (usage_str)
+        GF_FREE(usage_str);
 }
 
 int32_t
-mem_acct_init (xlator_t *this)
+mem_acct_init(xlator_t *this)
 {
-        int     ret = -1;
+    int ret = -1;
 
-        if (!this)
-                return ret;
-
-        ret = xlator_mem_acct_init (this, gf_quota_mt_end + 1);
+    if (!this)
+        return ret;
 
-        if (ret != 0) {
-                gf_log (this->name, GF_LOG_WARNING, "Memory accounting"
-                        "init failed");
-                return ret;
-        }
+    ret = xlator_mem_acct_init(this, gf_quota_mt_end + 1);
 
+    if (ret != 0) {
+        gf_msg(this->name, GF_LOG_WARNING, ENOMEM, Q_MSG_ENOMEM,
+               "Memory accounting init failed");
         return ret;
-}
+    }
 
+    return ret;
+}
 
 int32_t
-quota_forget (xlator_t *this, inode_t *inode)
+quota_forget(xlator_t *this, inode_t *inode)
 {
-        int32_t               ret     = 0;
-        uint64_t              ctx_int = 0;
-        quota_inode_ctx_t    *ctx     = NULL;
-        quota_dentry_t       *dentry  = NULL, *tmp;
+    int32_t ret = 0;
+    uint64_t ctx_int = 0;
+    quota_inode_ctx_t *ctx = NULL;
+    quota_dentry_t *dentry = NULL, *tmp;
 
-        ret = inode_ctx_del (inode, this, &ctx_int);
+    ret = inode_ctx_del(inode, this, &ctx_int);
 
-        if (ret < 0) {
-                return 0;
-        }
+    if (ret < 0) {
+        return 0;
+    }
 
-        ctx = (quota_inode_ctx_t *) (long)ctx_int;
+    ctx = (quota_inode_ctx_t *)(long)ctx_int;
 
-        LOCK (&ctx->lock);
+    LOCK(&ctx->lock);
+    {
+        list_for_each_entry_safe(dentry, tmp, &ctx->parents, next)
         {
-                list_for_each_entry_safe (dentry, tmp, &ctx->parents, next) {
-                        __quota_dentry_free (dentry);
-                }
+            __quota_dentry_free(dentry);
         }
-        UNLOCK (&ctx->lock);
+    }
+    UNLOCK(&ctx->lock);
 
-        LOCK_DESTROY (&ctx->lock);
+    LOCK_DESTROY(&ctx->lock);
 
-        GF_FREE (ctx);
+    GF_FREE(ctx);
 
-        return 0;
+    return 0;
 }
 
-int32_t
-init (xlator_t *this)
+int
+notify(xlator_t *this, int event, void *data, ...)
 {
-        int32_t       ret  = -1;
-        quota_priv_t *priv = NULL;
-        rpc_clnt_t   *rpc  = NULL;
-
-        if ((this->children == NULL)
-            || this->children->next) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "FATAL: quota (%s) not configured with "
-                        "exactly one child", this->name);
-                return -1;
-        }
-
-        if (this->parents == NULL) {
-                gf_log (this->name, GF_LOG_WARNING,
-                        "dangling volume. check volfile");
-        }
-
-        QUOTA_ALLOC_OR_GOTO (priv, quota_priv_t, err);
-
-        LOCK_INIT (&priv->lock);
-
-        this->private = priv;
-
-        GF_OPTION_INIT ("deem-statfs", priv->consider_statfs, bool, err);
-        GF_OPTION_INIT ("server-quota", priv->is_quota_on, bool, err);
-        GF_OPTION_INIT ("default-soft-limit", priv->default_soft_lim, percent,
-                        err);
-        GF_OPTION_INIT ("soft-timeout", priv->soft_timeout, time, err);
-        GF_OPTION_INIT ("hard-timeout", priv->hard_timeout, time, err);
-        GF_OPTION_INIT ("alert-time", priv->log_timeout, time, err);
-        GF_OPTION_INIT ("volume-uuid", priv->volume_uuid, str, err);
-
-        this->local_pool = mem_pool_new (quota_local_t, 64);
-        if (!this->local_pool) {
-                ret = -1;
-                gf_log (this->name, GF_LOG_ERROR,
-                        "failed to create local_t's memory pool");
-                goto err;
+    quota_priv_t *priv = NULL;
+    int ret = 0;
+    rpc_clnt_t *rpc = NULL;
+    gf_boolean_t conn_status = _gf_true;
+    xlator_t *victim = data;
+
+    priv = this->private;
+    if (!priv || !priv->is_quota_on)
+        goto out;
+
+    if (event == GF_EVENT_PARENT_DOWN) {
+        rpc = priv->rpc_clnt;
+        if (rpc) {
+            rpc_clnt_disable(rpc);
+            pthread_mutex_lock(&priv->conn_mutex);
+            {
+                conn_status = priv->conn_status;
+                while (conn_status) {
+                    (void)pthread_cond_wait(&priv->conn_cond,
+                                            &priv->conn_mutex);
+                    conn_status = priv->conn_status;
+                }
+            }
+            pthread_mutex_unlock(&priv->conn_mutex);
+            gf_log(this->name, GF_LOG_INFO,
+                   "Notify GF_EVENT_PARENT_DOWN for brick %s", victim->name);
         }
+    }
 
-        if (priv->is_quota_on) {
-                rpc = quota_enforcer_init (this, this->options);
-                if (rpc == NULL) {
-                        ret = -1;
-                        gf_log (this->name, GF_LOG_WARNING,
-                                "quota enforcer rpc init failed");
-                        goto err;
-                }
+out:
+    ret = default_notify(this, event, data);
+    return ret;
+}
 
-                LOCK (&priv->lock);
-                {
-                        priv->rpc_clnt = rpc;
-                }
-                UNLOCK (&priv->lock);
+int32_t
+init(xlator_t *this)
+{
+    int32_t ret = -1;
+    quota_priv_t *priv = NULL;
+    rpc_clnt_t *rpc = NULL;
+
+    if ((this->children == NULL) || this->children->next) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, Q_MSG_INVALID_VOLFILE,
+               "FATAL: quota (%s) not configured with "
+               "exactly one child",
+               this->name);
+        return -1;
+    }
+
+    if (this->parents == NULL) {
+        gf_msg(this->name, GF_LOG_WARNING, 0, Q_MSG_INVALID_VOLFILE,
+               "dangling volume. check volfile");
+    }
+
+    QUOTA_ALLOC_OR_GOTO(priv, quota_priv_t, err);
+
+    LOCK_INIT(&priv->lock);
+
+    this->private = priv;
+
+    GF_OPTION_INIT("deem-statfs", priv->consider_statfs, bool, err);
+    GF_OPTION_INIT("server-quota", priv->is_quota_on, bool, err);
+    GF_OPTION_INIT("default-soft-limit", priv->default_soft_lim, percent, err);
+    GF_OPTION_INIT("soft-timeout", priv->soft_timeout, time, err);
+    GF_OPTION_INIT("hard-timeout", priv->hard_timeout, time, err);
+    GF_OPTION_INIT("alert-time", priv->log_timeout, time, err);
+    GF_OPTION_INIT("volume-uuid", priv->volume_uuid, str, err);
+
+    this->local_pool = mem_pool_new(quota_local_t, 64);
+    if (!this->local_pool) {
+        ret = -1;
+        gf_msg(this->name, GF_LOG_ERROR, ENOMEM, Q_MSG_ENOMEM,
+               "failed to create local_t's memory pool");
+        goto err;
+    }
+
+    pthread_mutex_init(&priv->conn_mutex, NULL);
+    pthread_cond_init(&priv->conn_cond, NULL);
+    priv->conn_status = _gf_false;
+
+    if (priv->is_quota_on) {
+        rpc = quota_enforcer_init(this, this->options);
+        if (rpc == NULL) {
+            ret = -1;
+            gf_msg(this->name, GF_LOG_WARNING, 0,
+                   Q_MSG_QUOTA_ENFORCER_RPC_INIT_FAILED,
+                   "quota enforcer rpc init failed");
+            goto err;
+        }
+
+        LOCK(&priv->lock);
+        {
+            priv->rpc_clnt = rpc;
         }
+        UNLOCK(&priv->lock);
+    }
 
-        ret = 0;
+    ret = 0;
 err:
-        return ret;
+    return ret;
 }
 
 int
-reconfigure (xlator_t *this, dict_t *options)
+reconfigure(xlator_t *this, dict_t *options)
 {
-        int32_t       ret      = -1;
-        quota_priv_t *priv     = NULL;
-        gf_boolean_t  quota_on = _gf_false;
-        rpc_clnt_t   *rpc      = NULL;
-
-        priv = this->private;
-
-        GF_OPTION_RECONF ("deem-statfs", priv->consider_statfs, options, bool,
-                          out);
-        GF_OPTION_RECONF ("server-quota", quota_on, options, bool,
-                          out);
-        GF_OPTION_RECONF ("default-soft-limit", priv->default_soft_lim,
-                          options, percent, out);
-        GF_OPTION_RECONF ("alert-time", priv->log_timeout, options,
-                          time, out);
-        GF_OPTION_RECONF ("soft-timeout", priv->soft_timeout, options,
-                          time, out);
-        GF_OPTION_RECONF ("hard-timeout", priv->hard_timeout, options,
-                          time, out);
-
-        if (quota_on) {
-                priv->rpc_clnt = quota_enforcer_init (this,
-                                                      this->options);
-                if (priv->rpc_clnt == NULL) {
-                        ret = -1;
-                        gf_log (this->name, GF_LOG_WARNING,
-                                "quota enforcer rpc init failed");
-                        goto out;
-                }
+    int32_t ret = -1;
+    quota_priv_t *priv = NULL;
+    gf_boolean_t quota_on = _gf_false;
+    rpc_clnt_t *rpc = NULL;
+
+    priv = this->private;
+
+    GF_OPTION_RECONF("deem-statfs", priv->consider_statfs, options, bool, out);
+    GF_OPTION_RECONF("server-quota", quota_on, options, bool, out);
+    GF_OPTION_RECONF("default-soft-limit", priv->default_soft_lim, options,
+                     percent, out);
+    GF_OPTION_RECONF("alert-time", priv->log_timeout, options, time, out);
+    GF_OPTION_RECONF("soft-timeout", priv->soft_timeout, options, time, out);
+    GF_OPTION_RECONF("hard-timeout", priv->hard_timeout, options, time, out);
+
+    if (quota_on) {
+        priv->rpc_clnt = quota_enforcer_init(this, this->options);
+        if (priv->rpc_clnt == NULL) {
+            ret = -1;
+            gf_msg(this->name, GF_LOG_WARNING, 0,
+                   Q_MSG_QUOTA_ENFORCER_RPC_INIT_FAILED,
+                   "quota enforcer rpc init failed");
+            goto out;
+        }
+
+    } else {
+        LOCK(&priv->lock);
+        {
+            rpc = priv->rpc_clnt;
+            priv->rpc_clnt = NULL;
+        }
+        UNLOCK(&priv->lock);
 
-        } else {
-                LOCK (&priv->lock);
-                {
-                        rpc = priv->rpc_clnt;
-                        priv->rpc_clnt = NULL;
-                }
-                UNLOCK (&priv->lock);
-
-                if (rpc != NULL) {
-                        // Quotad is shutdown when there is no started volume
-                        // which has quota enabled. So, we should disable the
-                        // enforcer client when quota is disabled on a volume,
-                        // to avoid spurious reconnect attempts to a service
-                        // (quotad), that is known to be down.
-                        rpc_clnt_unref (rpc);
-                }
+        if (rpc != NULL) {
+            // Quotad is shutdown when there is no started volume
+            // which has quota enabled. So, we should disable the
+            // enforcer client when quota is disabled on a volume,
+            // to avoid spurious reconnect attempts to a service
+            // (quotad), that is known to be down.
+            rpc_clnt_unref(rpc);
         }
+    }
 
-        priv->is_quota_on = quota_on;
+    priv->is_quota_on = quota_on;
 
-        ret = 0;
+    ret = 0;
 out:
-        return ret;
+    return ret;
 }
 
 int32_t
-quota_priv_dump (xlator_t *this)
+quota_priv_dump(xlator_t *this)
 {
-        quota_priv_t *priv = NULL;
-        int32_t       ret  = -1;
-
-
-        GF_ASSERT (this);
-
-        priv = this->private;
-
-        gf_proc_dump_add_section ("xlators.features.quota.priv", this->name);
-
-        ret = TRY_LOCK (&priv->lock);
-        if (ret)
-             goto out;
-        else {
-                gf_proc_dump_write("soft-timeout", "%d", priv->soft_timeout);
-                gf_proc_dump_write("hard-timeout", "%d", priv->hard_timeout);
-                gf_proc_dump_write("alert-time", "%d", priv->log_timeout);
-                gf_proc_dump_write("quota-on", "%d", priv->is_quota_on);
-                gf_proc_dump_write("statfs", "%d", priv->consider_statfs);
-                gf_proc_dump_write("volume-uuid", "%s", priv->volume_uuid);
-                gf_proc_dump_write("validation-count", "%ld",
-                                    priv->validation_count);
-        }
-        UNLOCK (&priv->lock);
+    quota_priv_t *priv = NULL;
+    int32_t ret = -1;
+
+    GF_ASSERT(this);
+
+    priv = this->private;
+    if (!priv)
+        goto out;
+
+    gf_proc_dump_add_section("xlators.features.quota.priv");
+
+    ret = TRY_LOCK(&priv->lock);
+    if (ret)
+        goto out;
+    else {
+        gf_proc_dump_write("soft-timeout", "%u", priv->soft_timeout);
+        gf_proc_dump_write("hard-timeout", "%u", priv->hard_timeout);
+        gf_proc_dump_write("alert-time", "%u", priv->log_timeout);
+        gf_proc_dump_write("quota-on", "%d", priv->is_quota_on);
+        gf_proc_dump_write("statfs", "%d", priv->consider_statfs);
+        gf_proc_dump_write("volume-uuid", "%s", priv->volume_uuid);
+        gf_proc_dump_write("validation-count", "%" PRIu64,
+                           priv->validation_count);
+    }
+    UNLOCK(&priv->lock);
 
 out:
-        return 0;
+    return 0;
 }
 
 void
-fini (xlator_t *this)
+fini(xlator_t *this)
 {
+    quota_priv_t *priv = NULL;
+    rpc_clnt_t *rpc = NULL;
+
+    priv = this->private;
+    if (!priv)
         return;
+    rpc = priv->rpc_clnt;
+    priv->rpc_clnt = NULL;
+    if (rpc) {
+        rpc_clnt_connection_cleanup(&rpc->conn);
+        rpc_clnt_unref(rpc);
+    }
+
+    this->private = NULL;
+    LOCK_DESTROY(&priv->lock);
+    pthread_mutex_destroy(&priv->conn_mutex);
+    pthread_cond_destroy(&priv->conn_cond);
+
+    GF_FREE(priv);
+    if (this->local_pool) {
+        mem_pool_destroy(this->local_pool);
+        this->local_pool = NULL;
+    }
+    return;
 }
 
-
 struct xlator_fops fops = {
-        .statfs       = quota_statfs,
-        .lookup       = quota_lookup,
-        .writev       = quota_writev,
-        .create       = quota_create,
-        .mkdir        = quota_mkdir,
-        .truncate     = quota_truncate,
-        .ftruncate    = quota_ftruncate,
-        .unlink       = quota_unlink,
-        .symlink      = quota_symlink,
-        .link         = quota_link,
-        .rename       = quota_rename,
-        .getxattr     = quota_getxattr,
-        .fgetxattr    = quota_fgetxattr,
-        .stat         = quota_stat,
-        .fstat        = quota_fstat,
-        .readlink     = quota_readlink,
-        .readv        = quota_readv,
-        .fsync        = quota_fsync,
-        .setattr      = quota_setattr,
-        .fsetattr     = quota_fsetattr,
-        .mknod        = quota_mknod,
-        .setxattr     = quota_setxattr,
-        .fsetxattr    = quota_fsetxattr,
-        .removexattr  = quota_removexattr,
-        .fremovexattr = quota_fremovexattr,
-        .readdirp     = quota_readdirp,
-	.fallocate    = quota_fallocate,
+    .statfs = quota_statfs,
+    .lookup = quota_lookup,
+    .writev = quota_writev,
+    .create = quota_create,
+    .mkdir = quota_mkdir,
+    .truncate = quota_truncate,
+    .ftruncate = quota_ftruncate,
+    .unlink = quota_unlink,
+    .symlink = quota_symlink,
+    .link = quota_link,
+    .rename = quota_rename,
+    .getxattr = quota_getxattr,
+    .fgetxattr = quota_fgetxattr,
+    .stat = quota_stat,
+    .fstat = quota_fstat,
+    .readlink = quota_readlink,
+    .readv = quota_readv,
+    .fsync = quota_fsync,
+    .setattr = quota_setattr,
+    .fsetattr = quota_fsetattr,
+    .mknod = quota_mknod,
+    .setxattr = quota_setxattr,
+    .fsetxattr = quota_fsetxattr,
+    .removexattr = quota_removexattr,
+    .fremovexattr = quota_fremovexattr,
+    .readdirp = quota_readdirp,
+    .fallocate = quota_fallocate,
 };
 
-struct xlator_cbks cbks = {
-        .forget = quota_forget
-};
+struct xlator_cbks cbks = {.forget = quota_forget};
 
 struct xlator_dumpops dumpops = {
-        .priv    = quota_priv_dump,
+    .priv = quota_priv_dump,
 };
 struct volume_options options[] = {
-        {.key = {"limit-set"}},
-        {.key = {"deem-statfs"},
-         .type = GF_OPTION_TYPE_BOOL,
-         .default_value = "off",
-         .description = "If set to on, it takes quota limits into"
-                        "consideration while estimating fs size. (df command)"
-                        " (Default is off)."
-        },
-        {.key = {"server-quota"},
-         .type = GF_OPTION_TYPE_BOOL,
-         .default_value = "off",
-         .description = "Skip the quota enforcement if the feature is"
-                        " not turned on. This is not a user exposed option."
-        },
-        {.key = {"default-soft-limit"},
-         .type = GF_OPTION_TYPE_PERCENT,
-         .default_value = "80%",
-        },
-        {.key = {"soft-timeout"},
-         .type = GF_OPTION_TYPE_TIME,
-         .min = 0,
-         .max = 1800,
-         .default_value = "60",
-         .description = "quota caches the directory sizes on client. "
-                        "soft-timeout indicates the timeout for the validity of"
-                        " cache before soft-limit has been crossed."
-        },
-        {.key = {"hard-timeout"},
-         .type = GF_OPTION_TYPE_TIME,
-         .min = 0,
-         .max = 60,
-         .default_value = "5",
-         .description = "quota caches the directory sizes on client. "
-                        "hard-timeout indicates the timeout for the validity of"
-                        " cache after soft-limit has been crossed."
-        },
-        { .key   = {"username"},
-          .type  = GF_OPTION_TYPE_ANY,
-        },
-        { .key   = {"password"},
-          .type  = GF_OPTION_TYPE_ANY,
-        },
-        { .key   = {"transport-type"},
-          .value = {"tcp", "socket", "ib-verbs", "unix", "ib-sdp",
-                    "tcp/client", "ib-verbs/client", "rdma"},
-          .type  = GF_OPTION_TYPE_STR,
-        },
-        { .key   = {"remote-host"},
-          .type  = GF_OPTION_TYPE_INTERNET_ADDRESS,
-        },
-        { .key   = {"remote-port"},
-          .type  = GF_OPTION_TYPE_INT,
-        },
-        { .key  = {"volume-uuid"},
-          .type = GF_OPTION_TYPE_STR,
-          .description = "uuid of the volume this brick is part of."
-        },
-        { .key  = {"alert-time"},
-          .type = GF_OPTION_TYPE_TIME,
-          .min = 0,
-          .max = 7*86400,
-          .default_value = "86400",
-        },
-        {.key = {NULL}}
+    {
+        .key = {"enable"},
+        .type = GF_OPTION_TYPE_BOOL,
+        .default_value = "off",
+        .description = "enable is the volume option that can be used "
+                       "to turn on quota.",
+        .op_version = {1},
+        .flags = OPT_FLAG_SETTABLE | OPT_FLAG_DOC,
+        .level = OPT_STATUS_BASIC,
+        .tags = {},
+    },
+    {
+        .key = {"deem-statfs"},
+        .type = GF_OPTION_TYPE_BOOL,
+        .default_value = "on",
+        .description = "If set to on, it takes quota limits into"
+                       " consideration while estimating fs size. (df command)"
+                       " (Default is on).",
+        .op_version = {2},
+        .flags = OPT_FLAG_SETTABLE | OPT_FLAG_DOC,
+        .tags = {},
+    },
+    {
+        .key = {"server-quota"},
+        .type = GF_OPTION_TYPE_BOOL,
+        .default_value = "off",
+        .description = "Skip the quota enforcement if the feature is"
+                       " not turned on. This is not a user exposed option.",
+        .flags = OPT_FLAG_NONE,
+    },
+    {
+        .key = {"default-soft-limit"},
+        .type = GF_OPTION_TYPE_PERCENT,
+        .default_value = "80%",
+        .op_version = {3},
+        .description = "Soft limit is expressed as a proportion of hard limit."
+                       " Default-soft-limit is the proportion used when the "
+                       " user does not supply any soft limit value.",
+        .flags = OPT_FLAG_SETTABLE | OPT_FLAG_DOC,
+        .tags = {},
+    },
+    {
+        .key = {"soft-timeout"},
+        .type = GF_OPTION_TYPE_TIME,
+        .min = 0,
+        .max = 1800,
+        .default_value = "60",
+        .description = "quota caches the directory sizes on client. "
+                       "soft-timeout indicates the timeout for the validity of"
+                       " cache before soft-limit has been crossed.",
+        .op_version = {3},
+        .flags = OPT_FLAG_SETTABLE | OPT_FLAG_DOC,
+        .tags = {},
+    },
+    {
+        .key = {"hard-timeout"},
+        .type = GF_OPTION_TYPE_TIME,
+        .min = 0,
+        .max = 60,
+        .default_value = "5",
+        .description = "quota caches the directory sizes on client. "
+                       "hard-timeout indicates the timeout for the validity of"
+                       " cache after soft-limit has been crossed.",
+        .op_version = {3},
+        .flags = OPT_FLAG_SETTABLE | OPT_FLAG_DOC,
+        .tags = {},
+    },
+    {.key = {"volume-uuid"},
+     .type = GF_OPTION_TYPE_STR,
+     .default_value = "{{ volume.id }}",
+     .description = "uuid of the volume this brick is part of."},
+    {
+        .key = {"alert-time"},
+        .type = GF_OPTION_TYPE_TIME,
+        .min = 0,
+        .max = 7 * 86400,
+        .default_value = "86400",
+        .op_version = {3},
+        .flags = OPT_FLAG_SETTABLE | OPT_FLAG_DOC,
+        .description = "Frequency of limit breach messages in log.",
+        .tags = {},
+    },
+    {.key = {NULL}}};
+
+xlator_api_t xlator_api = {
+    .init = init,
+    .fini = fini,
+    .notify = notify,
+    .reconfigure = reconfigure,
+    .mem_acct_init = mem_acct_init,
+    .op_version = {1}, /* Present from the initial version */
+    .fops = &fops,
+    .cbks = &cbks,
+    .options = options,
+    .identifier = "quota",
+    .category = GF_MAINTAINED,
 };
diff --git a/xlators/features/quota/src/quota.h b/xlators/features/quota/src/quota.h
index 183f8c1c42e..0395d78c9ef 100644
--- a/xlators/features/quota/src/quota.h
+++ b/xlators/features/quota/src/quota.h
@@ -7,268 +7,260 @@
    later), or the GNU General Public License, version 2 (GPLv2), in all
    cases as published by the Free Software Foundation.
 */
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
-
 #ifndef _QUOTA_H
 #define _QUOTA_H
 
-#include "xlator.h"
-#include "call-stub.h"
-#include "defaults.h"
-#include "common-utils.h"
+#include <glusterfs/call-stub.h>
 #include "quota-mem-types.h"
-#include "glusterfs.h"
-#include "compat.h"
-#include "logging.h"
-#include "dict.h"
-#include "stack.h"
-#include "event.h"
-#include "globals.h"
+#include <glusterfs/glusterfs.h>
+#include <glusterfs/compat.h>
+#include <glusterfs/logging.h>
+#include <glusterfs/dict.h>
+#include <glusterfs/gf-event.h>
 #include "rpcsvc.h"
 #include "rpc-clnt.h"
-#include "byte-order.h"
+#include <glusterfs/byte-order.h>
 #include "glusterfs3-xdr.h"
 #include "glusterfs3.h"
 #include "xdr-generic.h"
-#include "compat-errno.h"
+#include <glusterfs/compat-errno.h>
 #include "protocol-common.h"
-#include "quota-common-utils.h"
+#include <glusterfs/quota-common-utils.h>
+#include "quota-messages.h"
 
-#define DIRTY                   "dirty"
-#define SIZE                    "size"
-#define CONTRIBUTION            "contri"
-#define VAL_LENGTH              8
-#define READDIR_BUF             4096
+#define DIRTY "dirty"
+#define SIZE "size"
+#define CONTRIBUTION "contri"
+#define VAL_LENGTH 8
+#define READDIR_BUF 4096
 
 #ifndef UUID_CANONICAL_FORM_LEN
 #define UUID_CANONICAL_FORM_LEN 36
 #endif
 
-#define WIND_IF_QUOTAOFF(is_quota_on, label)     \
-        if (!is_quota_on)                       \
-                goto label;
-
-#define QUOTA_WIND_FOR_INTERNAL_FOP(xdata, label)                          \
-        do {                                                               \
-                if (xdata && dict_get (xdata, GLUSTERFS_INTERNAL_FOP_KEY)) \
-                goto label;                                                \
-        } while (0)
-
-#define DID_REACH_LIMIT(lim, prev_size, cur_size)               \
-        ((cur_size) >= (lim) && (prev_size) < (lim))
-
-#define QUOTA_SAFE_INCREMENT(lock, var)         \
-        do {                                    \
-                LOCK (lock);                    \
-                var ++;                         \
-                UNLOCK (lock);                  \
-        } while (0)
-
-#define QUOTA_SAFE_DECREMENT(lock, var)         \
-        do {                                    \
-                LOCK (lock);                    \
-                var --;                         \
-                UNLOCK (lock);                  \
-        } while (0)
-
-#define QUOTA_ALLOC_OR_GOTO(var, type, label)           \
-        do {                                            \
-                var = GF_CALLOC (sizeof (type), 1,      \
-                                 gf_quota_mt_##type);   \
-                if (!var) {                             \
-                        gf_log ("", GF_LOG_ERROR,       \
-                                "out of memory");    \
-                        ret = -1;                       \
-                        goto label;                     \
-                }                                       \
-        } while (0);
-
-#define QUOTA_STACK_WIND_TAIL(frame, params...)                         \
-        do {                                                            \
-                quota_local_t *_local = NULL;                           \
-                xlator_t      *_this  = NULL;                           \
-                                                                        \
-                if (frame) {                                            \
-                        _local = frame->local;                          \
-                        _this  = frame->this;                           \
-                        frame->local = NULL;                            \
-                }                                                       \
-                                                                        \
-                STACK_WIND_TAIL (frame, params);                        \
-                                                                        \
-                if (_local)                                             \
-                        quota_local_cleanup (_this, _local);            \
-        } while (0)
-
-#define QUOTA_STACK_UNWIND(fop, frame, params...)                       \
-        do {                                                            \
-                quota_local_t *_local = NULL;                           \
-                xlator_t      *_this  = NULL;                           \
-                if (frame) {                                            \
-                        _local = frame->local;                          \
-                        _this  = frame->this;                           \
-                        frame->local = NULL;                            \
-                }                                                       \
-                STACK_UNWIND_STRICT (fop, frame, params);               \
-                quota_local_cleanup (_this, _local);                    \
-        } while (0)
-
-#define QUOTA_FREE_CONTRIBUTION_NODE(_contribution)     \
-        do {                                            \
-                list_del (&_contribution->contri_list); \
-                GF_FREE (_contribution);                \
-        } while (0)
-
-#define GET_CONTRI_KEY(var, _vol_name, _gfid, _ret)             \
-        do {                                                    \
-                char _gfid_unparsed[40];                        \
-                if (_gfid != NULL) {                            \
-                        gf_uuid_unparse (_gfid, _gfid_unparsed);\
-                        _ret = gf_asprintf (var, QUOTA_XATTR_PREFIX     \
-                                            "%s.%s." CONTRIBUTION,      \
-                                            _vol_name, _gfid_unparsed); \
-                } else {                                                \
-                        _ret = gf_asprintf (var, QUOTA_XATTR_PREFIX     \
-                                            "%s.." CONTRIBUTION,       \
-                                            _vol_name);             \
-                }                                                       \
-         } while (0)
-
-
-#define GET_CONTRI_KEY_OR_GOTO(var, _vol_name, _gfid, label)    \
-        do {                                                    \
-                GET_CONTRI_KEY(var, _vol_name, _gfid, ret);     \
-                if (ret == -1)                                  \
-                        goto label;                             \
-        } while (0)
-
-#define GET_DIRTY_KEY_OR_GOTO(var, _vol_name, label)            \
-        do {                                                    \
-                ret = gf_asprintf (var, QUOTA_XATTR_PREFIX      \
-                                   "%s." DIRTY, _vol_name);     \
-                if (ret == -1)                                  \
-                        goto label;                             \
-        } while (0)
-
-#define QUOTA_REG_OR_LNK_FILE(ia_type)  \
-    (IA_ISREG (ia_type) || IA_ISLNK (ia_type))
-
-
+#define WIND_IF_QUOTAOFF(is_quota_on, label)                                   \
+    if (!is_quota_on)                                                          \
+        goto label;
+
+#define QUOTA_WIND_FOR_INTERNAL_FOP(xdata, label)                              \
+    do {                                                                       \
+        if (xdata && dict_get_sizen(xdata, GLUSTERFS_INTERNAL_FOP_KEY))        \
+            goto label;                                                        \
+    } while (0)
+
+#define DID_REACH_LIMIT(lim, prev_size, cur_size)                              \
+    ((cur_size) >= (lim) && (prev_size) < (lim))
+
+#define QUOTA_SAFE_INCREMENT(lock, var)                                        \
+    do {                                                                       \
+        LOCK(lock);                                                            \
+        var++;                                                                 \
+        UNLOCK(lock);                                                          \
+    } while (0)
+
+#define QUOTA_SAFE_DECREMENT(lock, var)                                        \
+    do {                                                                       \
+        LOCK(lock);                                                            \
+        var--;                                                                 \
+        UNLOCK(lock);                                                          \
+    } while (0)
+
+#define QUOTA_ALLOC_OR_GOTO(var, type, label)                                  \
+    do {                                                                       \
+        var = GF_CALLOC(sizeof(type), 1, gf_quota_mt_##type);                  \
+        if (!var) {                                                            \
+            gf_msg("", GF_LOG_ERROR, ENOMEM, Q_MSG_ENOMEM, "out of memory");   \
+            ret = -1;                                                          \
+            goto label;                                                        \
+        }                                                                      \
+    } while (0);
+
+#define QUOTA_STACK_WIND_TAIL(frame, params...)                                \
+    do {                                                                       \
+        quota_local_t *_local = NULL;                                          \
+                                                                               \
+        if (frame) {                                                           \
+            _local = frame->local;                                             \
+            frame->local = NULL;                                               \
+        }                                                                      \
+                                                                               \
+        STACK_WIND_TAIL(frame, params);                                        \
+                                                                               \
+        if (_local)                                                            \
+            quota_local_cleanup(_local);                                       \
+    } while (0)
+
+#define QUOTA_STACK_UNWIND(fop, frame, params...)                              \
+    do {                                                                       \
+        quota_local_t *_local = NULL;                                          \
+        if (frame) {                                                           \
+            _local = frame->local;                                             \
+            frame->local = NULL;                                               \
+        }                                                                      \
+        STACK_UNWIND_STRICT(fop, frame, params);                               \
+        quota_local_cleanup(_local);                                           \
+    } while (0)
+
+#define QUOTA_FREE_CONTRIBUTION_NODE(_contribution)                            \
+    do {                                                                       \
+        list_del(&_contribution->contri_list);                                 \
+        GF_FREE(_contribution);                                                \
+    } while (0)
+
+#define GET_CONTRI_KEY(var, _vol_name, _gfid, _ret)                            \
+    do {                                                                       \
+        char _gfid_unparsed[40];                                               \
+        if (_gfid != NULL) {                                                   \
+            gf_uuid_unparse(_gfid, _gfid_unparsed);                            \
+            _ret = gf_asprintf(var, QUOTA_XATTR_PREFIX "%s.%s." CONTRIBUTION,  \
+                               _vol_name, _gfid_unparsed);                     \
+        } else {                                                               \
+            _ret = gf_asprintf(var, QUOTA_XATTR_PREFIX "%s.." CONTRIBUTION,    \
+                               _vol_name);                                     \
+        }                                                                      \
+    } while (0)
+
+#define GET_CONTRI_KEY_OR_GOTO(var, _vol_name, _gfid, label)                   \
+    do {                                                                       \
+        GET_CONTRI_KEY(var, _vol_name, _gfid, ret);                            \
+        if (ret == -1)                                                         \
+            goto label;                                                        \
+    } while (0)
+
+#define GET_DIRTY_KEY_OR_GOTO(var, _vol_name, label)                           \
+    do {                                                                       \
+        ret = gf_asprintf(var, QUOTA_XATTR_PREFIX "%s." DIRTY, _vol_name);     \
+        if (ret == -1)                                                         \
+            goto label;                                                        \
+    } while (0)
+
+#define QUOTA_REG_OR_LNK_FILE(ia_type) (IA_ISREG(ia_type) || IA_ISLNK(ia_type))
 
 struct quota_dentry {
-        char            *name;
-        uuid_t           par;
-        struct list_head next;
+    char *name;
+    uuid_t par;
+    struct list_head next;
 };
 typedef struct quota_dentry quota_dentry_t;
 
 struct quota_inode_ctx {
-        int64_t          size;
-        int64_t          hard_lim;
-        int64_t          soft_lim;
-        int64_t          file_count;
-        int64_t          dir_count;
-        int64_t          object_hard_lim;
-        int64_t          object_soft_lim;
-        struct iatt      buf;
-        struct list_head parents;
-        struct timeval   tv;
-        struct timeval   prev_log;
-        gf_boolean_t     ancestry_built;
-        gf_lock_t        lock;
+    int64_t size;
+    int64_t hard_lim;
+    int64_t soft_lim;
+    int64_t file_count;
+    int64_t dir_count;
+    int64_t object_hard_lim;
+    int64_t object_soft_lim;
+    struct iatt buf;
+    struct list_head parents;
+    time_t validate_time;
+    time_t prev_log_time;
+    gf_boolean_t ancestry_built;
+    gf_lock_t lock;
 };
 typedef struct quota_inode_ctx quota_inode_ctx_t;
 
-typedef void
-(*quota_ancestry_built_t) (struct list_head *parents, inode_t *inode,
-                           int32_t op_ret, int32_t op_errno, void *data);
+typedef void (*quota_ancestry_built_t)(struct list_head *parents,
+                                       inode_t *inode, int32_t op_ret,
+                                       int32_t op_errno, void *data);
 
-typedef void
-(*quota_fop_continue_t) (call_frame_t *frame);
+typedef void (*quota_fop_continue_t)(call_frame_t *frame);
 
 struct quota_local {
-        gf_lock_t               lock;
-        uint32_t                validate_count;
-        uint32_t                link_count;
-        loc_t                   loc;
-        loc_t                   oldloc;
-        loc_t                   newloc;
-        loc_t                   validate_loc;
-        int64_t                 delta;
-        int32_t                 op_ret;
-        int32_t                 op_errno;
-        int64_t                 size;
-        char                    just_validated;
-        fop_lookup_cbk_t        validate_cbk;
-        quota_fop_continue_t    fop_continue_cbk;
-        inode_t                *inode;
-        uuid_t                  common_ancestor; /* Used by quota_rename */
-        call_stub_t            *stub;
-        struct iobref          *iobref;
-        quota_limits_t          limit;
-        quota_limits_t          object_limit;
-        int64_t                 space_available;
-        quota_ancestry_built_t  ancestry_cbk;
-        void                   *ancestry_data;
-        dict_t                 *xdata;
+    gf_lock_t lock;
+    uint32_t link_count;
+    loc_t loc;
+    loc_t oldloc;
+    loc_t newloc;
+    loc_t validate_loc;
+    int64_t delta;
+    int8_t object_delta;
+    int32_t op_ret;
+    int32_t op_errno;
+    int64_t size;
+    char just_validated;
+    fop_lookup_cbk_t validate_cbk;
+    quota_fop_continue_t fop_continue_cbk;
+    inode_t *inode;
+    uuid_t common_ancestor; /* Used by quota_rename */
+    call_stub_t *stub;
+    struct iobref *iobref;
+    quota_limits_t limit;
+    quota_limits_t object_limit;
+    int64_t space_available;
+    quota_ancestry_built_t ancestry_cbk;
+    void *ancestry_data;
+    dict_t *xdata;
+    dict_t *validate_xdata;
+    int32_t quotad_conn_retry;
+    xlator_t *this;
+    call_frame_t *par_frame;
 };
-typedef struct quota_local      quota_local_t;
+typedef struct quota_local quota_local_t;
 
 struct quota_priv {
-        uint32_t                soft_timeout;
-        uint32_t                hard_timeout;
-        uint32_t               log_timeout;
-        double                 default_soft_lim;
-        gf_boolean_t           is_quota_on;
-        gf_boolean_t           consider_statfs;
-        gf_lock_t              lock;
-        rpc_clnt_prog_t       *quota_enforcer;
-        struct rpcsvc_program *quotad_aggregator;
-        struct rpc_clnt       *rpc_clnt;
-        rpcsvc_t              *rpcsvc;
-        inode_table_t         *itable;
-        char                  *volume_uuid;
-        uint64_t               validation_count;
+    /* FIXME: consider time_t for timeouts. */
+    uint32_t soft_timeout;
+    uint32_t hard_timeout;
+    uint32_t log_timeout;
+    double default_soft_lim;
+    gf_boolean_t is_quota_on;
+    gf_boolean_t consider_statfs;
+    gf_lock_t lock;
+    rpc_clnt_prog_t *quota_enforcer;
+    struct rpcsvc_program *quotad_aggregator;
+    struct rpc_clnt *rpc_clnt;
+    rpcsvc_t *rpcsvc;
+    inode_table_t *itable;
+    char *volume_uuid;
+    uint64_t validation_count;
+    int32_t quotad_conn_status;
+    pthread_mutex_t conn_mutex;
+    pthread_cond_t conn_cond;
+    gf_boolean_t conn_status;
 };
-typedef struct quota_priv      quota_priv_t;
+typedef struct quota_priv quota_priv_t;
 
 int
-quota_enforcer_lookup (call_frame_t *frame, xlator_t *this, loc_t *loc,
-                       dict_t *xdata, fop_lookup_cbk_t cbk);
+quota_enforcer_lookup(call_frame_t *frame, xlator_t *this, dict_t *xdata,
+                      fop_lookup_cbk_t cbk);
+
+void
+_quota_enforcer_lookup(void *data);
+
 struct rpc_clnt *
-quota_enforcer_init (xlator_t *this, dict_t *options);
+quota_enforcer_init(xlator_t *this, dict_t *options);
 
 void
-quota_log_usage (xlator_t *this, quota_inode_ctx_t *ctx, inode_t *inode,
-                 int64_t delta);
+quota_log_usage(xlator_t *this, quota_inode_ctx_t *ctx, inode_t *inode,
+                int64_t delta);
 
 int
-quota_build_ancestry (inode_t *inode, quota_ancestry_built_t ancestry_cbk,
-                      void *data);
+quota_build_ancestry(inode_t *inode, quota_ancestry_built_t ancestry_cbk,
+                     void *data);
 
 void
-quota_get_limit_dir (call_frame_t *frame, inode_t *cur_inode, xlator_t *this);
+quota_get_limit_dir(call_frame_t *frame, inode_t *cur_inode, xlator_t *this);
 
 int32_t
-quota_check_limit (call_frame_t *frame, inode_t *inode, xlator_t *this,
-                   char *name, uuid_t par);
+quota_check_limit(call_frame_t *frame, inode_t *inode, xlator_t *this);
 
+inode_t *
+do_quota_check_limit(call_frame_t *frame, inode_t *inode, xlator_t *this,
+                     quota_dentry_t *dentry, gf_boolean_t force);
 int
-quota_fill_inodectx (xlator_t *this, inode_t *inode, dict_t *dict,
-                     loc_t *loc, struct iatt *buf, int32_t *op_errno);
+quota_fill_inodectx(xlator_t *this, inode_t *inode, dict_t *dict, loc_t *loc,
+                    struct iatt *buf, int32_t *op_errno);
 
 int32_t
-quota_check_size_limit (call_frame_t *frame, quota_inode_ctx_t *ctx,
-                          quota_priv_t *priv, inode_t *_inode, xlator_t *this,
-                          int32_t *op_errno, int just_validated, int64_t delta,
-                          quota_local_t *local, gf_boolean_t *skip_check);
+quota_check_size_limit(call_frame_t *frame, quota_inode_ctx_t *ctx,
+                       quota_priv_t *priv, inode_t *_inode, xlator_t *this,
+                       int32_t *op_errno, int just_validated, int64_t delta,
+                       quota_local_t *local, gf_boolean_t *skip_check);
 
 int32_t
-quota_check_object_limit (call_frame_t *frame, quota_inode_ctx_t *ctx,
-                          quota_priv_t *priv, inode_t *_inode, xlator_t *this,
-                          int32_t *op_errno, int just_validated,
-                          quota_local_t *local, gf_boolean_t *skip_check);
+quota_check_object_limit(call_frame_t *frame, quota_inode_ctx_t *ctx,
+                         quota_priv_t *priv, inode_t *_inode, xlator_t *this,
+                         int32_t *op_errno, int just_validated,
+                         quota_local_t *local, gf_boolean_t *skip_check);
 #endif
diff --git a/xlators/features/quota/src/quotad-aggregator.c b/xlators/features/quota/src/quotad-aggregator.c
index 6f2f6e8a7b0..75d47867b5b 100644
--- a/xlators/features/quota/src/quotad-aggregator.c
+++ b/xlators/features/quota/src/quotad-aggregator.c
@@ -13,424 +13,482 @@
 #include "quotad-helpers.h"
 #include "quotad-aggregator.h"
 
-struct rpcsvc_program quotad_aggregator_prog;
+static char *qd_ext_xattrs[] = {
+    QUOTA_SIZE_KEY,
+    QUOTA_LIMIT_KEY,
+    QUOTA_LIMIT_OBJECTS_KEY,
+    NULL,
+};
+
+static struct rpcsvc_program quotad_aggregator_prog;
 
 struct iobuf *
-quotad_serialize_reply (rpcsvc_request_t *req, void *arg, struct iovec *outmsg,
-                        xdrproc_t xdrproc)
+quotad_serialize_reply(rpcsvc_request_t *req, void *arg, struct iovec *outmsg,
+                       xdrproc_t xdrproc)
 {
-        struct iobuf *iob      = NULL;
-        ssize_t       retlen   = 0;
-        ssize_t       xdr_size = 0;
-
-        GF_VALIDATE_OR_GOTO ("server", req, ret);
-
-        /* First, get the io buffer into which the reply in arg will
-         * be serialized.
+    struct iobuf *iob = NULL;
+    ssize_t retlen = 0;
+    ssize_t xdr_size = 0;
+
+    GF_VALIDATE_OR_GOTO("server", req, ret);
+
+    /* First, get the io buffer into which the reply in arg will
+     * be serialized.
+     */
+    if (arg && xdrproc) {
+        xdr_size = xdr_sizeof(xdrproc, arg);
+        iob = iobuf_get2(req->svc->ctx->iobuf_pool, xdr_size);
+        if (!iob) {
+            gf_log_callingfn(THIS->name, GF_LOG_ERROR, "Failed to get iobuf");
+            goto ret;
+        };
+
+        iobuf_to_iovec(iob, outmsg);
+        /* Use the given serializer to translate the given C structure
+         * in arg to XDR format which will be written into the buffer
+         * in outmsg.
          */
-        if (arg && xdrproc) {
-                xdr_size = xdr_sizeof (xdrproc, arg);
-                iob = iobuf_get2 (req->svc->ctx->iobuf_pool, xdr_size);
-                if (!iob) {
-                        gf_log_callingfn (THIS->name, GF_LOG_ERROR,
-                                          "Failed to get iobuf");
-                        goto ret;
-                };
-
-                iobuf_to_iovec (iob, outmsg);
-                /* Use the given serializer to translate the give C structure in arg
-                 * to XDR format which will be written into the buffer in outmsg.
-                 */
-                /* retlen is used to received the error since size_t is unsigned and we
-                 * need -1 for error notification during encoding.
-                 */
-
-                retlen = xdr_serialize_generic (*outmsg, arg, xdrproc);
-                if (retlen == -1) {
-                        /* Failed to Encode 'GlusterFS' msg in RPC is not exactly
-                           failure of RPC return values.. client should get
-                           notified about this, so there are no missing frames */
-                        gf_log_callingfn ("", GF_LOG_ERROR, "Failed to encode message");
-                        req->rpc_err = GARBAGE_ARGS;
-                        retlen = 0;
-                }
+        /* retlen is used to received the error since size_t is unsigned and we
+         * need -1 for error notification during encoding.
+         */
+
+        retlen = xdr_serialize_generic(*outmsg, arg, xdrproc);
+        if (retlen == -1) {
+            /* Failed to Encode 'GlusterFS' msg in RPC is not exactly
+               failure of RPC return values.. Client should get
+               notified about this, so there are no missing frames */
+            gf_log_callingfn("", GF_LOG_ERROR, "Failed to encode message");
+            req->rpc_err = GARBAGE_ARGS;
+            retlen = 0;
         }
-        outmsg->iov_len = retlen;
+    }
+    outmsg->iov_len = retlen;
 ret:
-        return iob;
+    return iob;
 }
 
 int
-quotad_aggregator_submit_reply (call_frame_t *frame, rpcsvc_request_t *req,
-                                void *arg, struct iovec *payload,
-                                int payloadcount, struct iobref *iobref,
-                                xdrproc_t xdrproc)
+quotad_aggregator_submit_reply(call_frame_t *frame, rpcsvc_request_t *req,
+                               void *arg, struct iovec *payload,
+                               int payloadcount, struct iobref *iobref,
+                               xdrproc_t xdrproc)
 {
-        struct iobuf              *iob        = NULL;
-        int                        ret        = -1;
-        struct iovec               rsp        = {0,};
-        quotad_aggregator_state_t *state      = NULL;
-        char                       new_iobref = 0;
+    struct iobuf *iob = NULL;
+    int ret = -1;
+    struct iovec rsp = {
+        0,
+    };
+    quotad_aggregator_state_t *state = NULL;
+    char new_iobref = 0;
 
-        GF_VALIDATE_OR_GOTO ("server", req, ret);
+    GF_VALIDATE_OR_GOTO("server", req, ret);
 
-        if (frame) {
-                state = frame->root->state;
-                frame->local = NULL;
-        }
+    if (frame) {
+        state = frame->root->state;
+        frame->local = NULL;
+    }
 
+    if (!iobref) {
+        iobref = iobref_new();
         if (!iobref) {
-                iobref = iobref_new ();
-                if (!iobref) {
-                        goto ret;
-                }
-
-                new_iobref = 1;
+            goto ret;
         }
 
-        iob = quotad_serialize_reply (req, arg, &rsp, xdrproc);
-        if (!iob) {
-                gf_log ("", GF_LOG_ERROR, "Failed to serialize reply");
-                goto ret;
-        }
+        new_iobref = 1;
+    }
+
+    iob = quotad_serialize_reply(req, arg, &rsp, xdrproc);
+    if (!iob) {
+        gf_msg("", GF_LOG_ERROR, 0, Q_MSG_DICT_SERIALIZE_FAIL,
+               "Failed to serialize reply");
+        goto ret;
+    }
 
-        iobref_add (iobref, iob);
+    iobref_add(iobref, iob);
 
-        ret = rpcsvc_submit_generic (req, &rsp, 1, payload, payloadcount,
-                                     iobref);
+    ret = rpcsvc_submit_generic(req, &rsp, 1, payload, payloadcount, iobref);
 
-        iobuf_unref (iob);
+    iobuf_unref(iob);
 
-        ret = 0;
+    ret = 0;
 ret:
-        if (state) {
-                quotad_aggregator_free_state (state);
-        }
+    if (state) {
+        quotad_aggregator_free_state(state);
+    }
 
-        if (frame)
-                STACK_DESTROY (frame->root);
+    if (frame)
+        STACK_DESTROY(frame->root);
 
-        if (new_iobref) {
-                iobref_unref (iobref);
-        }
+    if (new_iobref) {
+        iobref_unref(iobref);
+    }
 
-        return ret;
+    return ret;
 }
 
 int
-quotad_aggregator_getlimit_cbk (xlator_t *this, call_frame_t *frame,
-                                void *lookup_rsp)
+quotad_aggregator_getlimit_cbk(xlator_t *this, call_frame_t *frame,
+                               void *lookup_rsp)
 {
-        gfs3_lookup_rsp            *rsp    = lookup_rsp;
-        gf_cli_rsp                 cli_rsp = {0,};
-        dict_t                     *xdata  = NULL;
-        quotad_aggregator_state_t  *state  = NULL;
-        int                         ret    = -1;
-        int                         type   = 0;
-
-        GF_PROTOCOL_DICT_UNSERIALIZE (frame->this, xdata,
-                                      (rsp->xdata.xdata_val),
-                                      (rsp->xdata.xdata_len), rsp->op_ret,
-                                      rsp->op_errno, out);
-
-        if (xdata) {
-                state = frame->root->state;
-                ret = dict_get_int32 (state->xdata, "type", &type);
-                if (ret < 0)
-                        goto out;
-
-                ret = dict_set_int32 (xdata, "type", type);
-                if (ret < 0)
-                        goto out;
-        }
+    gfs3_lookup_rsp *rsp = lookup_rsp;
+    gf_cli_rsp cli_rsp = {
+        0,
+    };
+    dict_t *xdata = NULL;
+    quotad_aggregator_state_t *state = NULL;
+    int ret = -1;
+    int type = 0;
+
+    if (!rsp || (rsp->op_ret == -1))
+        goto reply;
+
+    GF_PROTOCOL_DICT_UNSERIALIZE(frame->this, xdata, (rsp->xdata.xdata_val),
+                                 (rsp->xdata.xdata_len), rsp->op_ret,
+                                 rsp->op_errno, out);
+
+    if (xdata) {
+        state = frame->root->state;
+        ret = dict_get_int32n(state->req_xdata, "type", SLEN("type"), &type);
+        if (ret < 0)
+            goto out;
+
+        ret = dict_set_int32_sizen(xdata, "type", type);
+        if (ret < 0)
+            goto out;
+    }
 
-        ret = 0;
+    ret = 0;
 out:
-        rsp->op_ret = ret;
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "failed to unserialize "
-                        "nameless lookup rsp");
-                goto reply;
-        }
-        cli_rsp.op_ret = rsp->op_ret;
-        cli_rsp.op_errno = rsp->op_errno;
-        cli_rsp.op_errstr = "";
-        if (xdata) {
-                GF_PROTOCOL_DICT_SERIALIZE (frame->this, xdata,
-                                            (&cli_rsp.dict.dict_val),
-                                            (cli_rsp.dict.dict_len),
-                                            cli_rsp.op_errno, reply);
-        }
+    rsp->op_ret = ret;
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, Q_MSG_DICT_UNSERIALIZE_FAIL,
+               "failed to unserialize "
+               "nameless lookup rsp");
+        goto reply;
+    }
+    cli_rsp.op_ret = rsp->op_ret;
+    cli_rsp.op_errno = rsp->op_errno;
+    cli_rsp.op_errstr = "";
+    if (xdata) {
+        GF_PROTOCOL_DICT_SERIALIZE(frame->this, xdata, (&cli_rsp.dict.dict_val),
+                                   (cli_rsp.dict.dict_len), cli_rsp.op_errno,
+                                   reply);
+    }
 
 reply:
-        quotad_aggregator_submit_reply (frame, frame->local, (void*)&cli_rsp, NULL, 0,
-                                        NULL, (xdrproc_t)xdr_gf_cli_rsp);
+    quotad_aggregator_submit_reply(frame, (frame) ? frame->local : NULL,
+                                   (void *)&cli_rsp, NULL, 0, NULL,
+                                   (xdrproc_t)xdr_gf_cli_rsp);
 
-        dict_unref (xdata);
-        GF_FREE (cli_rsp.dict.dict_val);
-        return 0;
+    dict_unref(xdata);
+    GF_FREE(cli_rsp.dict.dict_val);
+    return 0;
 }
 
 int
-quotad_aggregator_getlimit (rpcsvc_request_t *req)
+quotad_aggregator_getlimit(rpcsvc_request_t *req)
 {
-        call_frame_t              *frame = NULL;
-        gf_cli_req                 cli_req = {{0}, };
-        gf_cli_rsp                 cli_rsp = {0};
-        gfs3_lookup_req            args  = {{0,},};
-        gfs3_lookup_rsp            rsp   = {0,};
-        quotad_aggregator_state_t *state = NULL;
-        xlator_t                  *this  = NULL;
-        dict_t                    *dict  = NULL;
-        int                        ret   = -1, op_errno = 0;
-        char                      *gfid_str = NULL;
-        uuid_t                     gfid = {0};
-
-        GF_VALIDATE_OR_GOTO ("quotad-aggregator", req, err);
-
-        this = THIS;
-
-        ret = xdr_to_generic (req->msg[0], &cli_req, (xdrproc_t)xdr_gf_cli_req);
-        if (ret < 0)  {
-                //failed to decode msg;
-                gf_log ("", GF_LOG_ERROR, "xdr decoding error");
-                req->rpc_err = GARBAGE_ARGS;
-                goto err;
-        }
-
-        if (cli_req.dict.dict_len) {
-                dict = dict_new ();
-                ret = dict_unserialize (cli_req.dict.dict_val,
-                                        cli_req.dict.dict_len, &dict);
-                if (ret < 0) {
-                        gf_log (this->name, GF_LOG_ERROR, "Failed to "
-                                "unserialize req-buffer to dictionary");
-                        goto err;
-                }
-        }
-
-        ret = dict_get_str (dict, "gfid", &gfid_str);
-        if (ret) {
-                goto err;
-        }
-
-        gf_uuid_parse ((const char*)gfid_str, gfid);
-
-        frame = quotad_aggregator_get_frame_from_req (req);
-        if (frame == NULL) {
-                rsp.op_errno = ENOMEM;
-                goto err;
-        }
-        state = frame->root->state;
-        state->xdata = dict;
-
-        ret = dict_set_int32 (state->xdata, QUOTA_LIMIT_KEY, 42);
-        if (ret)
-                goto err;
-
-        ret = dict_set_int32 (state->xdata, QUOTA_LIMIT_OBJECTS_KEY, 42);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "Failed to set "
-                        "QUOTA_LIMIT_OBJECTS_KEY");
-                goto err;
-        }
-
-        ret = dict_set_int32 (state->xdata, QUOTA_SIZE_KEY, 42);
-        if (ret)
-                goto err;
-
-        ret = dict_set_int32 (state->xdata, GET_ANCESTRY_PATH_KEY, 42);
-        if (ret)
-                goto err;
-
-        memcpy (&args.gfid, &gfid, 16);
-
-        args.bname           = alloca (req->msg[0].iov_len);
-        args.xdata.xdata_val = alloca (req->msg[0].iov_len);
-
-        ret = qd_nameless_lookup (this, frame, &args, state->xdata,
-                                  quotad_aggregator_getlimit_cbk);
-        if (ret) {
-                rsp.op_errno = ret;
-                goto err;
+    call_frame_t *frame = NULL;
+    gf_cli_req cli_req = {
+        {0},
+    };
+    gf_cli_rsp cli_rsp = {0};
+    quotad_aggregator_state_t *state = NULL;
+    xlator_t *this = NULL;
+    dict_t *dict = NULL;
+    int ret = -1, op_errno = 0;
+    char *gfid_str = NULL;
+    uuid_t gfid = {0};
+    char *volume_uuid = NULL;
+
+    GF_VALIDATE_OR_GOTO("quotad-aggregator", req, err);
+
+    this = THIS;
+
+    cli_req.dict.dict_val = alloca(req->msg[0].iov_len);
+
+    ret = xdr_to_generic(req->msg[0], &cli_req, (xdrproc_t)xdr_gf_cli_req);
+    if (ret < 0) {
+        // failed to decode msg;
+        gf_msg("this->name", GF_LOG_ERROR, 0, Q_MSG_XDR_DECODE_ERROR,
+               "xdr decoding error");
+        req->rpc_err = GARBAGE_ARGS;
+        goto err;
+    }
+
+    if (cli_req.dict.dict_len) {
+        dict = dict_new();
+        ret = dict_unserialize(cli_req.dict.dict_val, cli_req.dict.dict_len,
+                               &dict);
+        if (ret < 0) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, Q_MSG_DICT_UNSERIALIZE_FAIL,
+                   "Failed to unserialize req-buffer to "
+                   "dictionary");
+            goto err;
         }
-
-        return ret;
+    }
+
+    ret = dict_get_strn(dict, "gfid", SLEN("gfid"), &gfid_str);
+    if (ret) {
+        goto err;
+    }
+
+    ret = dict_get_strn(dict, "volume-uuid", SLEN("volume-uuid"), &volume_uuid);
+    if (ret) {
+        goto err;
+    }
+
+    gf_uuid_parse((const char *)gfid_str, gfid);
+
+    frame = quotad_aggregator_get_frame_from_req(req);
+    if (frame == NULL) {
+        cli_rsp.op_errno = ENOMEM;
+        goto errx;
+    }
+    state = frame->root->state;
+    state->req_xdata = dict;
+    state->xdata = dict_new();
+    dict = NULL;
+
+    ret = dict_set_int32_sizen(state->xdata, QUOTA_LIMIT_KEY, 42);
+    if (ret)
+        goto err;
+
+    ret = dict_set_int32_sizen(state->xdata, QUOTA_LIMIT_OBJECTS_KEY, 42);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, ENOMEM, Q_MSG_ENOMEM,
+               "Failed to set QUOTA_LIMIT_OBJECTS_KEY");
+        goto err;
+    }
+
+    ret = dict_set_int32_sizen(state->xdata, QUOTA_SIZE_KEY, 42);
+    if (ret)
+        goto err;
+
+    ret = dict_set_int32_sizen(state->xdata, GET_ANCESTRY_PATH_KEY, 42);
+    if (ret)
+        goto err;
+
+    ret = qd_nameless_lookup(this, frame, (char *)gfid, state->xdata,
+                             volume_uuid, quotad_aggregator_getlimit_cbk);
+    if (ret) {
+        cli_rsp.op_errno = ret;
+        goto errx;
+    }
+
+    return ret;
 
 err:
-        cli_rsp.op_ret = -1;
-        cli_rsp.op_errno = op_errno;
-        cli_rsp.op_errstr = "";
-
-        quotad_aggregator_getlimit_cbk (this, frame, &cli_rsp);
-        if (dict)
-                dict_unref (dict);
-
-        return ret;
+    cli_rsp.op_errno = op_errno;
+errx:
+    cli_rsp.op_ret = -1;
+    cli_rsp.op_errstr = "";
+
+    quotad_aggregator_getlimit_cbk(this, frame, &cli_rsp);
+    if (dict)
+        dict_unref(dict);
+    return ret;
 }
 
 int
-quotad_aggregator_lookup_cbk (xlator_t *this, call_frame_t *frame,
-                              void *rsp)
+quotad_aggregator_lookup_cbk(xlator_t *this, call_frame_t *frame, void *rsp)
 {
-        quotad_aggregator_submit_reply (frame, frame->local, rsp, NULL, 0, NULL,
-                                        (xdrproc_t)xdr_gfs3_lookup_rsp);
+    quotad_aggregator_submit_reply(frame, frame ? frame->local : NULL, rsp,
+                                   NULL, 0, NULL,
+                                   (xdrproc_t)xdr_gfs3_lookup_rsp);
 
-        return 0;
+    return 0;
 }
 
-
 int
-quotad_aggregator_lookup (rpcsvc_request_t *req)
+quotad_aggregator_lookup(rpcsvc_request_t *req)
 {
-        call_frame_t              *frame = NULL;
-        gfs3_lookup_req            args  = {{0,},};
-        int                        ret   = -1, op_errno = 0;
-        gfs3_lookup_rsp            rsp   = {0,};
-        quotad_aggregator_state_t *state = NULL;
-        xlator_t                  *this  = NULL;
-
-        GF_VALIDATE_OR_GOTO ("quotad-aggregator", req, err);
-
-        this = THIS;
-
-        args.bname           = alloca (req->msg[0].iov_len);
-        args.xdata.xdata_val = alloca (req->msg[0].iov_len);
-
-        ret = xdr_to_generic (req->msg[0], &args,
-                              (xdrproc_t)xdr_gfs3_lookup_req);
-        if (ret < 0) {
-                rsp.op_errno = EINVAL;
-                goto err;
-        }
-
-        frame = quotad_aggregator_get_frame_from_req (req);
-        if (frame == NULL) {
-                rsp.op_errno = ENOMEM;
+    call_frame_t *frame = NULL;
+    gfs3_lookup_req args = {
+        {
+            0,
+        },
+    };
+    int i = 0, ret = -1, op_errno = 0;
+    gfs3_lookup_rsp rsp = {
+        0,
+    };
+    quotad_aggregator_state_t *state = NULL;
+    xlator_t *this = NULL;
+    dict_t *dict = NULL;
+    char *volume_uuid = NULL;
+
+    GF_VALIDATE_OR_GOTO("quotad-aggregator", req, err);
+
+    this = THIS;
+
+    args.bname = alloca(req->msg[0].iov_len);
+    args.xdata.xdata_val = alloca(req->msg[0].iov_len);
+
+    ret = xdr_to_generic(req->msg[0], &args, (xdrproc_t)xdr_gfs3_lookup_req);
+    if (ret < 0) {
+        rsp.op_errno = EINVAL;
+        goto err;
+    }
+
+    frame = quotad_aggregator_get_frame_from_req(req);
+    if (frame == NULL) {
+        rsp.op_errno = ENOMEM;
+        goto err;
+    }
+
+    state = frame->root->state;
+
+    GF_PROTOCOL_DICT_UNSERIALIZE(this, dict, (args.xdata.xdata_val),
+                                 (args.xdata.xdata_len), ret, op_errno, err);
+
+    ret = dict_get_str(dict, "volume-uuid", &volume_uuid);
+    if (ret) {
+        goto err;
+    }
+
+    state->xdata = dict_new();
+
+    for (i = 0; qd_ext_xattrs[i]; i++) {
+        if (dict_get(dict, qd_ext_xattrs[i])) {
+            ret = dict_set_uint32(state->xdata, qd_ext_xattrs[i], 1);
+            if (ret < 0)
                 goto err;
         }
+    }
 
-        state = frame->root->state;
-
-        GF_PROTOCOL_DICT_UNSERIALIZE (this, state->xdata,
-                                      (args.xdata.xdata_val),
-                                      (args.xdata.xdata_len), ret,
-                                      op_errno, err);
-
+    ret = qd_nameless_lookup(this, frame, args.gfid, state->xdata, volume_uuid,
+                             quotad_aggregator_lookup_cbk);
+    if (ret) {
+        rsp.op_errno = ret;
+        goto err;
+    }
 
-        ret = qd_nameless_lookup (this, frame, &args, state->xdata,
-                                  quotad_aggregator_lookup_cbk);
-        if (ret) {
-                rsp.op_errno = ret;
-                goto err;
-        }
+    if (dict)
+        dict_unref(dict);
 
-        return ret;
+    return ret;
 
 err:
-        rsp.op_ret = -1;
-        rsp.op_errno = op_errno;
+    rsp.op_ret = -1;
+    rsp.op_errno = op_errno;
+
+    quotad_aggregator_lookup_cbk(this, frame, &rsp);
+    if (dict)
+        dict_unref(dict);
 
-        quotad_aggregator_lookup_cbk (this, frame, &rsp);
-        return ret;
+    return ret;
 }
 
 int
-quotad_aggregator_rpc_notify (rpcsvc_t *rpc, void *xl, rpcsvc_event_t event,
-                              void *data)
+quotad_aggregator_rpc_notify(rpcsvc_t *rpc, void *xl, rpcsvc_event_t event,
+                             void *data)
 {
-        if (!xl || !data) {
-                gf_log_callingfn ("server", GF_LOG_WARNING,
-                                  "Calling rpc_notify without initializing");
-                goto out;
-        }
+    if (!xl || !data) {
+        gf_log_callingfn("server", GF_LOG_WARNING,
+                         "Calling rpc_notify without initializing");
+        goto out;
+    }
 
-        switch (event) {
+    switch (event) {
         case RPCSVC_EVENT_ACCEPT:
-                break;
+            break;
 
         case RPCSVC_EVENT_DISCONNECT:
-                break;
+            break;
 
         default:
-                break;
-        }
+            break;
+    }
 
 out:
-        return 0;
+    return 0;
 }
 
 int
-quotad_aggregator_init (xlator_t *this)
+quotad_aggregator_init(xlator_t *this)
 {
-        quota_priv_t *priv = NULL;
-        int           ret  = -1;
-
-        priv = this->private;
-
-        ret = dict_set_str (this->options, "transport.address-family", "unix");
-        if (ret)
-                goto out;
-
-        ret = dict_set_str (this->options, "transport-type", "socket");
-        if (ret)
-                goto out;
-
-        ret = dict_set_str (this->options, "transport.socket.listen-path",
-                            "/var/run/gluster/quotad.socket");
-        if (ret)
-                goto out;
-
-        /* RPC related */
-        priv->rpcsvc = rpcsvc_init (this, this->ctx, this->options, 0);
-        if (priv->rpcsvc == NULL) {
-                gf_log (this->name, GF_LOG_WARNING,
-                        "creation of rpcsvc failed");
-                ret = -1;
-                goto out;
-        }
+    quota_priv_t *priv = NULL;
+    int ret = -1;
 
-        ret = rpcsvc_create_listeners (priv->rpcsvc, this->options,
-                                       this->name);
-        if (ret < 1) {
-                gf_log (this->name, GF_LOG_WARNING,
-                        "creation of listener failed");
-                ret = -1;
-                goto out;
-        }
-
-        priv->quotad_aggregator = &quotad_aggregator_prog;
-        quotad_aggregator_prog.options = this->options;
-
-        ret = rpcsvc_program_register (priv->rpcsvc, &quotad_aggregator_prog);
-        if (ret) {
-                gf_log (this->name, GF_LOG_WARNING,
-                        "registration of program (name:%s, prognum:%d, "
-                        "progver:%d) failed", quotad_aggregator_prog.progname,
-                        quotad_aggregator_prog.prognum,
-                        quotad_aggregator_prog.progver);
-                goto out;
-        }
+    priv = this->private;
 
-        ret = 0;
+    if (priv->rpcsvc) {
+        /* Listener already created */
+        return 0;
+    }
+
+    ret = dict_set_nstrn(this->options, "transport.address-family",
+                         SLEN("transport.address-family"), "unix",
+                         SLEN("unix"));
+    if (ret)
+        goto out;
+
+    ret = dict_set_nstrn(this->options, "transport-type",
+                         SLEN("transport-type"), "socket", SLEN("socket"));
+    if (ret)
+        goto out;
+
+    ret = dict_set_nstrn(this->options, "transport.socket.listen-path",
+                         SLEN("transport.socket.listen-path"),
+                         "/var/run/gluster/quotad.socket",
+                         SLEN("/var/run/gluster/quotad.socket"));
+    if (ret)
+        goto out;
+
+    /* RPC related */
+    priv->rpcsvc = rpcsvc_init(this, this->ctx, this->options, 0);
+    if (priv->rpcsvc == NULL) {
+        gf_msg(this->name, GF_LOG_WARNING, 0, Q_MSG_RPCSVC_INIT_FAILED,
+               "creation of rpcsvc failed");
+        ret = -1;
+        goto out;
+    }
+
+    ret = rpcsvc_create_listeners(priv->rpcsvc, this->options, this->name);
+    if (ret < 1) {
+        gf_msg(this->name, GF_LOG_WARNING, 0,
+               Q_MSG_RPCSVC_LISTENER_CREATION_FAILED,
+               "creation of listener failed");
+        ret = -1;
+        goto out;
+    }
+
+    priv->quotad_aggregator = &quotad_aggregator_prog;
+    quotad_aggregator_prog.options = this->options;
+
+    ret = rpcsvc_program_register(priv->rpcsvc, &quotad_aggregator_prog,
+                                  _gf_false);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_WARNING, 0, Q_MSG_RPCSVC_REGISTER_FAILED,
+               "registration of program (name:%s, prognum:%d, "
+               "progver:%d) failed",
+               quotad_aggregator_prog.progname, quotad_aggregator_prog.prognum,
+               quotad_aggregator_prog.progver);
+        goto out;
+    }
+
+    ret = 0;
 out:
-        return ret;
+    if (ret && priv->rpcsvc) {
+        GF_FREE(priv->rpcsvc);
+        priv->rpcsvc = NULL;
+    }
+
+    return ret;
 }
 
-rpcsvc_actor_t quotad_aggregator_actors[GF_AGGREGATOR_MAXVALUE] = {
-        [GF_AGGREGATOR_NULL]     = {"NULL", GF_AGGREGATOR_NULL, NULL, NULL, 0,
-                                    DRC_NA},
-        [GF_AGGREGATOR_LOOKUP]   = {"LOOKUP", GF_AGGREGATOR_NULL,
-                                    quotad_aggregator_lookup, NULL, 0, DRC_NA},
-        [GF_AGGREGATOR_GETLIMIT] = {"GETLIMIT", GF_AGGREGATOR_GETLIMIT,
-                                   quotad_aggregator_getlimit, NULL, 0, DRC_NA},
+static rpcsvc_actor_t quotad_aggregator_actors[GF_AGGREGATOR_MAXVALUE] = {
+    [GF_AGGREGATOR_NULL] = {"NULL", NULL, NULL, GF_AGGREGATOR_NULL, DRC_NA, 0},
+    [GF_AGGREGATOR_LOOKUP] = {"LOOKUP", quotad_aggregator_lookup, NULL,
+                              GF_AGGREGATOR_NULL, DRC_NA, 0},
+    [GF_AGGREGATOR_GETLIMIT] = {"GETLIMIT", quotad_aggregator_getlimit, NULL,
+                                GF_AGGREGATOR_GETLIMIT, DRC_NA, 0},
 };
 
-
-struct rpcsvc_program quotad_aggregator_prog = {
-        .progname  = "GlusterFS 3.3",
-        .prognum   = GLUSTER_AGGREGATOR_PROGRAM,
-        .progver   = GLUSTER_AGGREGATOR_VERSION,
-        .numactors = GF_AGGREGATOR_MAXVALUE,
-        .actors    = quotad_aggregator_actors
-};
+static struct rpcsvc_program quotad_aggregator_prog = {
+    .progname = "GlusterFS 3.3",
+    .prognum = GLUSTER_AGGREGATOR_PROGRAM,
+    .progver = GLUSTER_AGGREGATOR_VERSION,
+    .numactors = GF_AGGREGATOR_MAXVALUE,
+    .actors = quotad_aggregator_actors};
diff --git a/xlators/features/quota/src/quotad-aggregator.h b/xlators/features/quota/src/quotad-aggregator.h
index 5ddea5b3c46..706592c7d50 100644
--- a/xlators/features/quota/src/quotad-aggregator.h
+++ b/xlators/features/quota/src/quotad-aggregator.h
@@ -12,26 +12,27 @@
 #define _QUOTAD_AGGREGATOR_H
 
 #include "quota.h"
-#include "stack.h"
+#include <glusterfs/stack.h>
 #include "glusterfs3-xdr.h"
-#include "inode.h"
+#include <glusterfs/inode.h>
 
 typedef struct {
-        void          *pool;
-        xlator_t      *this;
-	xlator_t      *active_subvol;
-        inode_table_t *itable;
-        loc_t          loc;
-        dict_t        *xdata;
+    void *pool;
+    xlator_t *this;
+    xlator_t *active_subvol;
+    inode_table_t *itable;
+    loc_t loc;
+    dict_t *xdata;
+    dict_t *req_xdata;
 } quotad_aggregator_state_t;
 
-typedef int (*quotad_aggregator_lookup_cbk_t) (xlator_t *this,
-                                               call_frame_t *frame,
-                                               void *rsp);
+typedef int (*quotad_aggregator_lookup_cbk_t)(xlator_t *this,
+                                              call_frame_t *frame, void *rsp);
 int
-qd_nameless_lookup (xlator_t *this, call_frame_t *frame, gfs3_lookup_req *req,
-                    dict_t *xdata, quotad_aggregator_lookup_cbk_t lookup_cbk);
+qd_nameless_lookup(xlator_t *this, call_frame_t *frame, char *gfid,
+                   dict_t *xdata, char *volume_uuid,
+                   quotad_aggregator_lookup_cbk_t lookup_cbk);
 int
-quotad_aggregator_init (xlator_t *this);
+quotad_aggregator_init(xlator_t *this);
 
 #endif
diff --git a/xlators/features/quota/src/quotad-helpers.c b/xlators/features/quota/src/quotad-helpers.c
index 9a98ab17a24..51ff1d7e98d 100644
--- a/xlators/features/quota/src/quotad-helpers.c
+++ b/xlators/features/quota/src/quotad-helpers.c
@@ -11,97 +11,97 @@
 #include "quotad-helpers.h"
 
 quotad_aggregator_state_t *
-get_quotad_aggregator_state (xlator_t *this, rpcsvc_request_t *req)
+get_quotad_aggregator_state(xlator_t *this, rpcsvc_request_t *req)
 {
-        quotad_aggregator_state_t *state         = NULL;
-	xlator_t                  *active_subvol = NULL;
-        quota_priv_t              *priv          = NULL;
+    quotad_aggregator_state_t *state = NULL;
+    xlator_t *active_subvol = NULL;
+    quota_priv_t *priv = NULL;
 
-        state = (void *)GF_CALLOC (1, sizeof (*state),
-                                   gf_quota_mt_aggregator_state_t);
-        if (!state)
-                return NULL;
+    state = (void *)GF_CALLOC(1, sizeof(*state),
+                              gf_quota_mt_aggregator_state_t);
+    if (!state)
+        return NULL;
 
-	state->this = THIS;
-        priv = this->private;
+    state->this = THIS;
+    priv = this->private;
 
-        LOCK (&priv->lock);
-        {
-                active_subvol = state->active_subvol = FIRST_CHILD (this);
-        }
-        UNLOCK (&priv->lock);
+    LOCK(&priv->lock);
+    {
+        active_subvol = state->active_subvol = FIRST_CHILD(this);
+    }
+    UNLOCK(&priv->lock);
 
-        if (active_subvol->itable == NULL)
-                active_subvol->itable = inode_table_new (4096, active_subvol);
+    if (active_subvol->itable == NULL)
+        active_subvol->itable = inode_table_new(4096, active_subvol);
 
-	state->itable = active_subvol->itable;
+    state->itable = active_subvol->itable;
 
-        state->pool = this->ctx->pool;
+    state->pool = this->ctx->pool;
 
-        return state;
+    return state;
 }
 
 void
-quotad_aggregator_free_state (quotad_aggregator_state_t *state)
+quotad_aggregator_free_state(quotad_aggregator_state_t *state)
 {
-        if (state->xdata)
-                dict_unref (state->xdata);
+    if (state->xdata)
+        dict_unref(state->xdata);
 
-        GF_FREE (state);
+    if (state->req_xdata)
+        dict_unref(state->req_xdata);
+
+    GF_FREE(state);
 }
 
 call_frame_t *
-quotad_aggregator_alloc_frame (rpcsvc_request_t *req)
+quotad_aggregator_alloc_frame(rpcsvc_request_t *req)
 {
-        call_frame_t              *frame = NULL;
-        quotad_aggregator_state_t *state = NULL;
-        xlator_t                  *this  = NULL;
+    call_frame_t *frame = NULL;
+    quotad_aggregator_state_t *state = NULL;
+    xlator_t *this = NULL;
 
-        GF_VALIDATE_OR_GOTO ("server", req, out);
-        GF_VALIDATE_OR_GOTO ("server", req->trans, out);
-        GF_VALIDATE_OR_GOTO ("server", req->svc, out);
-        GF_VALIDATE_OR_GOTO ("server", req->svc->ctx, out);
+    GF_VALIDATE_OR_GOTO("server", req, out);
+    GF_VALIDATE_OR_GOTO("server", req->trans, out);
+    GF_VALIDATE_OR_GOTO("server", req->svc, out);
+    GF_VALIDATE_OR_GOTO("server", req->svc->ctx, out);
 
-        this = req->svc->mydata;
+    this = req->svc->xl;
 
-        frame = create_frame (this, req->svc->ctx->pool);
-        if (!frame)
-                goto out;
+    frame = create_frame(this, req->svc->ctx->pool);
+    if (!frame)
+        goto out;
 
-        state = get_quotad_aggregator_state (this, req);
-        if (!state)
-                goto out;
+    state = get_quotad_aggregator_state(this, req);
+    if (!state)
+        goto out;
 
-        frame->root->state = state;
-        frame->root->unique = 0;
+    frame->root->state = state;
 
-        frame->this = this;
+    frame->this = this;
 out:
-        return frame;
+    return frame;
 }
 
 call_frame_t *
-quotad_aggregator_get_frame_from_req (rpcsvc_request_t *req)
+quotad_aggregator_get_frame_from_req(rpcsvc_request_t *req)
 {
-        call_frame_t *frame  = NULL;
-
-        GF_VALIDATE_OR_GOTO ("server", req, out);
+    call_frame_t *frame = NULL;
 
-        frame = quotad_aggregator_alloc_frame (req);
-        if (!frame)
-                goto out;
+    GF_VALIDATE_OR_GOTO("server", req, out);
 
-        frame->root->op       = req->procnum;
+    frame = quotad_aggregator_alloc_frame(req);
+    if (!frame)
+        goto out;
 
-        frame->root->unique   = req->xid;
+    frame->root->op = req->procnum;
 
-        frame->root->uid      = req->uid;
-        frame->root->gid      = req->gid;
-        frame->root->pid      = req->pid;
+    frame->root->uid = req->uid;
+    frame->root->gid = req->gid;
+    frame->root->pid = req->pid;
 
-        frame->root->lk_owner = req->lk_owner;
+    frame->root->lk_owner = req->lk_owner;
 
-        frame->local = req;
+    frame->local = req;
 out:
-        return frame;
+    return frame;
 }
diff --git a/xlators/features/quota/src/quotad-helpers.h b/xlators/features/quota/src/quotad-helpers.h
index a10fb7fa82a..bcb39fe845e 100644
--- a/xlators/features/quota/src/quotad-helpers.h
+++ b/xlators/features/quota/src/quotad-helpers.h
@@ -16,9 +16,9 @@
 #include "quotad-aggregator.h"
 
 void
-quotad_aggregator_free_state (quotad_aggregator_state_t *state);
+quotad_aggregator_free_state(quotad_aggregator_state_t *state);
 
 call_frame_t *
-quotad_aggregator_get_frame_from_req (rpcsvc_request_t *req);
+quotad_aggregator_get_frame_from_req(rpcsvc_request_t *req);
 
 #endif
diff --git a/xlators/features/quota/src/quotad.c b/xlators/features/quota/src/quotad.c
index 243b943e986..643f25c9c2a 100644
--- a/xlators/features/quota/src/quotad.c
+++ b/xlators/features/quota/src/quotad.c
@@ -9,202 +9,237 @@
 */
 #include "quota.h"
 #include "quotad-aggregator.h"
-#include "common-utils.h"
 
-int32_t
-mem_acct_init (xlator_t *this)
+int
+qd_notify(xlator_t *this, int32_t event, void *data, ...)
 {
-        int     ret = -1;
+    switch (event) {
+        case GF_EVENT_PARENT_UP:
+            quotad_aggregator_init(this);
+    }
+
+    default_notify(this, event, data);
+    return 0;
+}
 
-        if (!this)
-                return ret;
+int32_t
+mem_acct_init(xlator_t *this)
+{
+    int ret = -1;
 
-        ret = xlator_mem_acct_init (this, gf_quota_mt_end + 1);
+    if (!this)
+        return ret;
 
-        if (0 != ret) {
-                gf_log (this->name, GF_LOG_WARNING, "Memory accounting "
-                        "init failed");
-                return ret;
-        }
+    ret = xlator_mem_acct_init(this, gf_quota_mt_end + 1);
 
+    if (0 != ret) {
+        gf_log(this->name, GF_LOG_WARNING,
+               "Memory accounting "
+               "init failed");
         return ret;
+    }
+
+    return ret;
 }
 
 int32_t
-qd_lookup_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-               int32_t op_ret, int32_t op_errno, inode_t *inode,
-               struct iatt *buf, dict_t *xdata, struct iatt *postparent)
+qd_lookup_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int32_t op_ret,
+              int32_t op_errno, inode_t *inode, struct iatt *buf, dict_t *xdata,
+              struct iatt *postparent)
 {
-        quotad_aggregator_lookup_cbk_t  lookup_cbk = NULL;
-        gfs3_lookup_rsp                 rsp = {0, };
+    quotad_aggregator_lookup_cbk_t lookup_cbk = NULL;
+    gfs3_lookup_rsp rsp = {
+        0,
+    };
 
-        lookup_cbk = cookie;
+    lookup_cbk = cookie;
 
-        rsp.op_ret = op_ret;
-        rsp.op_errno = op_errno;
+    rsp.op_ret = op_ret;
+    rsp.op_errno = op_errno;
 
-        gf_stat_from_iatt (&rsp.postparent, postparent);
+    gf_stat_from_iatt(&rsp.postparent, postparent);
 
-        GF_PROTOCOL_DICT_SERIALIZE (this, xdata, (&rsp.xdata.xdata_val),
-                                    rsp.xdata.xdata_len, rsp.op_errno, out);
+    GF_PROTOCOL_DICT_SERIALIZE(this, xdata, (&rsp.xdata.xdata_val),
+                               rsp.xdata.xdata_len, rsp.op_errno, out);
 
-        gf_stat_from_iatt (&rsp.stat, buf);
+    gf_stat_from_iatt(&rsp.stat, buf);
 
 out:
-        lookup_cbk (this, frame, &rsp);
+    lookup_cbk(this, frame, &rsp);
 
-        GF_FREE (rsp.xdata.xdata_val);
+    GF_FREE(rsp.xdata.xdata_val);
 
-        inode_unref (inode);
+    inode_unref(inode);
 
-        return 0;
+    return 0;
 }
 
 xlator_t *
-qd_find_subvol (xlator_t *this, char *volume_uuid)
+qd_find_subvol(xlator_t *this, char *volume_uuid)
 {
-        xlator_list_t *child  = NULL;
-        xlator_t      *subvol = NULL;
-        char           key[1024];
-        char          *optstr = NULL;
-
-        if (!this || !volume_uuid)
-                goto out;
-
-        for (child = this->children; child; child = child->next) {
-                snprintf(key, 1024, "%s.volume-id", child->xlator->name);
-                if (dict_get_str(this->options, key, &optstr) < 0)
-                        continue;
-
-                if (strcmp (optstr, volume_uuid) == 0) {
-                        subvol = child->xlator;
-                        break;
-                }
+    xlator_list_t *child = NULL;
+    xlator_t *subvol = NULL;
+    char key[1024];
+    int keylen = 0;
+    char *optstr = NULL;
+
+    if (!this || !volume_uuid)
+        goto out;
+
+    for (child = this->children; child; child = child->next) {
+        keylen = snprintf(key, sizeof(key), "%s.volume-id",
+                          child->xlator->name);
+        if (dict_get_strn(this->options, key, keylen, &optstr) < 0)
+            continue;
+
+        if (strcmp(optstr, volume_uuid) == 0) {
+            subvol = child->xlator;
+            break;
         }
+    }
 
 out:
-        return subvol;
+    return subvol;
 }
 
 int
-qd_nameless_lookup (xlator_t *this, call_frame_t *frame, gfs3_lookup_req *req,
-                    dict_t *xdata, quotad_aggregator_lookup_cbk_t lookup_cbk)
+qd_nameless_lookup(xlator_t *this, call_frame_t *frame, char *gfid,
+                   dict_t *xdata, char *volume_uuid,
+                   quotad_aggregator_lookup_cbk_t lookup_cbk)
 {
-        gfs3_lookup_rsp            rsp         = {0, };
-        int                        op_errno    = 0, ret = -1;
-        loc_t                      loc         = {0, };
-        quotad_aggregator_state_t *state       = NULL;
-        quota_priv_t              *priv        = NULL;
-        xlator_t                  *subvol      = NULL;
-        char                      *volume_uuid = NULL;
-
-        priv = this->private;
-        state = frame->root->state;
-
-        frame->root->op = GF_FOP_LOOKUP;
-
-        loc.inode = inode_new (state->itable);
-        if (loc.inode == NULL) {
-                op_errno = ENOMEM;
-                goto out;
-        }
-
-        memcpy (loc.gfid, req->gfid, 16);
-
-        ret = dict_get_str (xdata, "volume-uuid", &volume_uuid);
-        if (ret < 0) {
-                op_errno = EINVAL;
-                goto out;
-        }
-
-        subvol = qd_find_subvol (this, volume_uuid);
-        if (subvol == NULL) {
-                op_errno = EINVAL;
-                goto out;
-        }
-
-        STACK_WIND_COOKIE (frame, qd_lookup_cbk, lookup_cbk, subvol,
-                           subvol->fops->lookup, &loc, xdata);
-        return 0;
+    gfs3_lookup_rsp rsp = {
+        0,
+    };
+    int op_errno = 0, ret = -1;
+    loc_t loc = {
+        0,
+    };
+    quotad_aggregator_state_t *state = NULL;
+    xlator_t *subvol = NULL;
+
+    state = frame->root->state;
+
+    frame->root->op = GF_FOP_LOOKUP;
+
+    loc.inode = inode_new(state->itable);
+    if (loc.inode == NULL) {
+        op_errno = ENOMEM;
+        goto out;
+    }
+
+    memcpy(loc.gfid, gfid, 16);
+
+    ret = dict_set_int8(xdata, QUOTA_READ_ONLY_KEY, 1);
+    if (ret < 0) {
+        gf_msg(this->name, GF_LOG_WARNING, ENOMEM, Q_MSG_ENOMEM,
+               "dict set failed");
+        ret = -ENOMEM;
+        goto out;
+    }
+
+    subvol = qd_find_subvol(this, volume_uuid);
+    if (subvol == NULL) {
+        op_errno = EINVAL;
+        goto out;
+    }
+
+    STACK_WIND_COOKIE(frame, qd_lookup_cbk, lookup_cbk, subvol,
+                      subvol->fops->lookup, &loc, xdata);
+    return 0;
 
 out:
-        rsp.op_ret = -1;
-        rsp.op_errno = op_errno;
+    rsp.op_ret = -1;
+    rsp.op_errno = op_errno;
 
-        lookup_cbk (this, frame, &rsp);
+    lookup_cbk(this, frame, &rsp);
 
-        inode_unref (loc.inode);
-        return 0;
+    inode_unref(loc.inode);
+    return 0;
 }
 
 int
-qd_reconfigure (xlator_t *this, dict_t *options)
+qd_reconfigure(xlator_t *this, dict_t *options)
 {
-        /* As of now quotad is restarted upon alteration of volfile */
-        return 0;
+    /* As of now quotad is restarted upon alteration of volfile */
+    return 0;
 }
 
 void
-qd_fini (xlator_t *this)
+qd_fini(xlator_t *this)
 {
-        return;
+    quota_priv_t *priv = NULL;
+
+    if (this == NULL || this->private == NULL)
+        goto out;
+
+    priv = this->private;
+
+    if (priv->rpcsvc) {
+        GF_FREE(priv->rpcsvc);
+        priv->rpcsvc = NULL;
+    }
+
+    GF_FREE(priv);
+
+out:
+    return;
 }
 
 int32_t
-qd_init (xlator_t *this)
+qd_init(xlator_t *this)
 {
-        int32_t          ret            = -1;
-        quota_priv_t    *priv           = NULL;
-
-        if (NULL == this->children) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "FATAL: quota (%s) not configured for min of 1 child",
-                        this->name);
-                ret = -1;
-                goto err;
-        }
+    int32_t ret = -1;
+    quota_priv_t *priv = NULL;
 
-        QUOTA_ALLOC_OR_GOTO (priv, quota_priv_t, err);
-        LOCK_INIT (&priv->lock);
+    if (NULL == this->children) {
+        gf_log(this->name, GF_LOG_ERROR,
+               "FATAL: quota (%s) not configured for min of 1 child",
+               this->name);
+        ret = -1;
+        goto err;
+    }
 
-        this->private = priv;
+    QUOTA_ALLOC_OR_GOTO(priv, quota_priv_t, err);
+    LOCK_INIT(&priv->lock);
 
-        ret = quotad_aggregator_init (this);
-        if (ret < 0)
-                goto err;
+    this->private = priv;
 
-        ret = 0;
+    ret = 0;
 err:
-        if (ret) {
-                GF_FREE (priv);
-        }
-        return ret;
+    if (ret) {
+        GF_FREE(priv);
+    }
+    return ret;
 }
 
-class_methods_t class_methods = {
-        .init           = qd_init,
-        .fini           = qd_fini,
-        .reconfigure    = qd_reconfigure,
-};
+struct xlator_fops fops = {};
 
-struct xlator_fops fops = {
-};
+struct xlator_cbks cbks = {};
 
-struct xlator_cbks cbks = {
+struct volume_options options[] = {
+    {.key = {"transport-type"},
+     .value = {"rpc", "rpc-over-rdma", "tcp", "socket", "ib-verbs", "unix",
+               "ib-sdp", "tcp/server", "ib-verbs/server", "rdma",
+               "rdma*([ \t]),*([ \t])socket", "rdma*([ \t]),*([ \t])tcp",
+               "tcp*([ \t]),*([ \t])rdma", "socket*([ \t]),*([ \t])rdma"},
+     .type = GF_OPTION_TYPE_STR},
+    {
+        .key = {"transport.*"},
+        .type = GF_OPTION_TYPE_ANY,
+    },
+    {.key = {NULL}},
 };
 
-struct volume_options options[] = {
-        { .key   = {"transport-type"},
-          .value = {"rpc", "rpc-over-rdma", "tcp", "socket", "ib-verbs",
-                    "unix", "ib-sdp", "tcp/server", "ib-verbs/server", "rdma",
-                    "rdma*([ \t]),*([ \t])socket",
-                    "rdma*([ \t]),*([ \t])tcp",
-                    "tcp*([ \t]),*([ \t])rdma",
-                    "socket*([ \t]),*([ \t])rdma"},
-          .type  = GF_OPTION_TYPE_STR
-        },
-        { .key   = {"transport.*"},
-          .type  = GF_OPTION_TYPE_ANY,
-        },
-        {.key = {NULL}}
+xlator_api_t xlator_api = {
+    .init = qd_init,
+    .fini = qd_fini,
+    .reconfigure = qd_reconfigure,
+    .notify = qd_notify,
+    .mem_acct_init = mem_acct_init,
+    .op_version = {1},
+    .fops = &fops,
+    .cbks = &cbks,
+    .options = options,
+    .identifier = "quotad",
+    .category = GF_MAINTAINED,
 };
diff --git a/xlators/features/read-only/src/Makefile.am b/xlators/features/read-only/src/Makefile.am
index 209e4208af0..e4a2017ef0d 100644
--- a/xlators/features/read-only/src/Makefile.am
+++ b/xlators/features/read-only/src/Makefile.am
@@ -2,19 +2,20 @@ xlator_LTLIBRARIES = read-only.la worm.la
 
 xlatordir = $(libdir)/glusterfs/$(PACKAGE_VERSION)/xlator/features
 
-noinst_HEADERS = read-only.h read-only-mem-types.h read-only-common.h
+noinst_HEADERS = read-only.h read-only-mem-types.h read-only-common.h worm-helper.h
 
-read_only_la_LDFLAGS = -module -avoid-version
+read_only_la_LDFLAGS = -module $(GF_XLATOR_DEFAULT_LDFLAGS)
 
 read_only_la_SOURCES = read-only.c read-only-common.c
 read_only_la_LIBADD = $(top_builddir)/libglusterfs/src/libglusterfs.la 
 
-worm_la_LDFLAGS = -module -avoid-version
+worm_la_LDFLAGS = -module $(GF_XLATOR_DEFAULT_LDFLAGS)
 
-worm_la_SOURCES = read-only-common.c worm.c
+worm_la_SOURCES = read-only-common.c worm-helper.c worm.c
 worm_la_LIBADD = $(top_builddir)/libglusterfs/src/libglusterfs.la
 
-AM_CPPFLAGS = $(GF_CPPFLAGS) -I$(top_srcdir)/libglusterfs/src
+AM_CPPFLAGS = $(GF_CPPFLAGS) -I$(top_srcdir)/libglusterfs/src \
+	-I$(top_srcdir)/rpc/xdr/src -I$(top_builddir)/rpc/xdr/src
 
 AM_CFLAGS = -Wall $(GF_CFLAGS)
 
diff --git a/xlators/features/read-only/src/read-only-common.c b/xlators/features/read-only/src/read-only-common.c
index 39937d106c5..9640e7e3eee 100644
--- a/xlators/features/read-only/src/read-only-common.c
+++ b/xlators/features/read-only/src/read-only-common.c
@@ -7,406 +7,400 @@
    later), or the GNU General Public License, version 2 (GPLv2), in all
    cases as published by the Free Software Foundation.
 */
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
-
 #include "read-only.h"
 #include "read-only-mem-types.h"
-#include "defaults.h"
+#include <glusterfs/defaults.h>
 
 gf_boolean_t
-is_readonly_or_worm_enabled (xlator_t *this)
+is_readonly_or_worm_enabled(call_frame_t *frame, xlator_t *this)
 {
-        read_only_priv_t  *priv                     = NULL;
-        gf_boolean_t       readonly_or_worm_enabled = _gf_false;
+    read_only_priv_t *priv = NULL;
+    gf_boolean_t readonly_or_worm_enabled = _gf_false;
+
+    priv = this->private;
+    GF_ASSERT(priv);
 
-        priv = this->private;
-        GF_ASSERT (priv);
+    readonly_or_worm_enabled = priv->readonly_or_worm_enabled;
 
-        readonly_or_worm_enabled = priv->readonly_or_worm_enabled;
+    if (frame->root->pid < GF_CLIENT_PID_MAX)
+        readonly_or_worm_enabled = _gf_false;
+
+    return readonly_or_worm_enabled;
+}
 
-        return readonly_or_worm_enabled;
+static int
+_check_key_is_zero_filled(dict_t *d, char *k, data_t *v, void *tmp)
+{
+    if (mem_0filled((const char *)v->data, v->len)) {
+        /* -1 means, no more iterations, treat as 'break' */
+        return -1;
+    }
+    return 0;
 }
 
 int32_t
-ro_xattrop (call_frame_t *frame, xlator_t *this, loc_t *loc,
-            gf_xattrop_flags_t flags, dict_t *dict, dict_t *xdata)
+ro_xattrop(call_frame_t *frame, xlator_t *this, loc_t *loc,
+           gf_xattrop_flags_t flags, dict_t *dict, dict_t *xdata)
 {
-        if (is_readonly_or_worm_enabled (this))
-                STACK_UNWIND_STRICT (xattrop, frame, -1, EROFS, NULL, xdata);
-        else
-                STACK_WIND_TAIL (frame, FIRST_CHILD (this),
-                                 FIRST_CHILD(this)->fops->xattrop,
-                                 loc, flags, dict, xdata);
-        return 0;
+    gf_boolean_t allzero = _gf_false;
+    int ret = 0;
+
+    ret = dict_foreach(dict, _check_key_is_zero_filled, NULL);
+    if (ret == 0)
+        allzero = _gf_true;
+
+    if (is_readonly_or_worm_enabled(frame, this) && !allzero)
+        STACK_UNWIND_STRICT(xattrop, frame, -1, EROFS, NULL, xdata);
+    else
+        STACK_WIND_TAIL(frame, FIRST_CHILD(this),
+                        FIRST_CHILD(this)->fops->xattrop, loc, flags, dict,
+                        xdata);
+    return 0;
 }
 
 int32_t
-ro_fxattrop (call_frame_t *frame, xlator_t *this,
-             fd_t *fd, gf_xattrop_flags_t flags, dict_t *dict, dict_t *xdata)
+ro_fxattrop(call_frame_t *frame, xlator_t *this, fd_t *fd,
+            gf_xattrop_flags_t flags, dict_t *dict, dict_t *xdata)
 {
-        if (is_readonly_or_worm_enabled (this))
-                STACK_UNWIND_STRICT (fxattrop, frame, -1, EROFS, NULL, xdata);
-        else
-                STACK_WIND_TAIL (frame, FIRST_CHILD (this),
-                                 FIRST_CHILD(this)->fops->fxattrop,
-                                 fd, flags, dict, xdata);
+    gf_boolean_t allzero = _gf_false;
+    int ret = 0;
 
-        return 0;
+    ret = dict_foreach(dict, _check_key_is_zero_filled, NULL);
+    if (ret == 0)
+        allzero = _gf_true;
+
+    if (is_readonly_or_worm_enabled(frame, this) && !allzero)
+        STACK_UNWIND_STRICT(fxattrop, frame, -1, EROFS, NULL, xdata);
+    else
+        STACK_WIND_TAIL(frame, FIRST_CHILD(this),
+                        FIRST_CHILD(this)->fops->fxattrop, fd, flags, dict,
+                        xdata);
+
+    return 0;
 }
 
 int32_t
-ro_entrylk (call_frame_t *frame, xlator_t *this, const char *volume,
-            loc_t *loc, const char *basename, entrylk_cmd cmd,
-            entrylk_type type, dict_t *xdata)
+ro_entrylk(call_frame_t *frame, xlator_t *this, const char *volume, loc_t *loc,
+           const char *basename, entrylk_cmd cmd, entrylk_type type,
+           dict_t *xdata)
 {
-        if (is_readonly_or_worm_enabled (this))
-                STACK_UNWIND_STRICT (entrylk, frame, -1, EROFS, xdata);
-        else
-                STACK_WIND_TAIL (frame, FIRST_CHILD (this),
-                                 FIRST_CHILD(this)->fops->entrylk,
-                                 volume, loc, basename, cmd, type, xdata);
+    STACK_WIND_TAIL(frame, FIRST_CHILD(this), FIRST_CHILD(this)->fops->entrylk,
+                    volume, loc, basename, cmd, type, xdata);
 
-        return 0;
+    return 0;
 }
 
 int32_t
-ro_fentrylk (call_frame_t *frame, xlator_t *this, const char *volume,
-             fd_t *fd, const char *basename, entrylk_cmd cmd, entrylk_type type,
-             dict_t *xdata)
+ro_fentrylk(call_frame_t *frame, xlator_t *this, const char *volume, fd_t *fd,
+            const char *basename, entrylk_cmd cmd, entrylk_type type,
+            dict_t *xdata)
 {
-        if (is_readonly_or_worm_enabled (this))
-                STACK_UNWIND_STRICT (fentrylk, frame, -1, EROFS, xdata);
-        else
-                STACK_WIND_TAIL (frame, FIRST_CHILD (this),
-                                 FIRST_CHILD(this)->fops->fentrylk,
-                                 volume, fd, basename, cmd, type, xdata);
+    STACK_WIND_TAIL(frame, FIRST_CHILD(this), FIRST_CHILD(this)->fops->fentrylk,
+                    volume, fd, basename, cmd, type, xdata);
 
-        return 0;
+    return 0;
 }
 
 int32_t
-ro_inodelk (call_frame_t *frame, xlator_t *this, const char *volume,
-            loc_t *loc, int32_t cmd, struct gf_flock *lock, dict_t *xdata)
+ro_inodelk(call_frame_t *frame, xlator_t *this, const char *volume, loc_t *loc,
+           int32_t cmd, struct gf_flock *lock, dict_t *xdata)
 {
-        if (is_readonly_or_worm_enabled (this))
-                STACK_UNWIND_STRICT (inodelk, frame, -1, EROFS, xdata);
-        else
-                STACK_WIND_TAIL (frame, FIRST_CHILD (this),
-                                 FIRST_CHILD(this)->fops->inodelk,
-                                 volume, loc, cmd, lock, xdata);
+    STACK_WIND_TAIL(frame, FIRST_CHILD(this), FIRST_CHILD(this)->fops->inodelk,
+                    volume, loc, cmd, lock, xdata);
 
-        return 0;
+    return 0;
 }
 
 int32_t
-ro_finodelk (call_frame_t *frame, xlator_t *this, const char *volume,
-             fd_t *fd, int32_t cmd, struct gf_flock *lock, dict_t *xdata)
+ro_finodelk(call_frame_t *frame, xlator_t *this, const char *volume, fd_t *fd,
+            int32_t cmd, struct gf_flock *lock, dict_t *xdata)
 {
-        if (is_readonly_or_worm_enabled (this))
-                STACK_UNWIND_STRICT (finodelk, frame, -1, EROFS, xdata);
-        else
-                STACK_WIND_TAIL (frame, FIRST_CHILD (this),
-                                 FIRST_CHILD(this)->fops->finodelk,
-                                 volume, fd, cmd, lock, xdata);
+    STACK_WIND_TAIL(frame, FIRST_CHILD(this), FIRST_CHILD(this)->fops->finodelk,
+                    volume, fd, cmd, lock, xdata);
 
-        return 0;
+    return 0;
 }
 
 int32_t
-ro_lk (call_frame_t *frame, xlator_t *this, fd_t *fd, int cmd,
-       struct gf_flock *flock, dict_t *xdata)
+ro_lk(call_frame_t *frame, xlator_t *this, fd_t *fd, int cmd,
+      struct gf_flock *flock, dict_t *xdata)
 {
-        if (is_readonly_or_worm_enabled (this))
-                STACK_UNWIND_STRICT (lk, frame, -1, EROFS, NULL, xdata);
-        else
-                STACK_WIND_TAIL (frame, FIRST_CHILD (this),
-                                 FIRST_CHILD(this)->fops->lk, fd, cmd, flock,
-                                 xdata);
+    STACK_WIND_TAIL(frame, FIRST_CHILD(this), FIRST_CHILD(this)->fops->lk, fd,
+                    cmd, flock, xdata);
 
-        return 0;
+    return 0;
 }
 
 int32_t
-ro_setattr (call_frame_t *frame, xlator_t *this, loc_t *loc,
-            struct iatt *stbuf, int32_t valid, dict_t *xdata)
+ro_setattr(call_frame_t *frame, xlator_t *this, loc_t *loc, struct iatt *stbuf,
+           int32_t valid, dict_t *xdata)
 {
-        if (is_readonly_or_worm_enabled (this))
-                STACK_UNWIND_STRICT (setattr, frame, -1, EROFS, NULL, NULL,
-                                     xdata);
-        else
-                STACK_WIND_TAIL (frame, FIRST_CHILD (this),
-                                 FIRST_CHILD(this)->fops->setattr, loc, stbuf,
-                                 valid, xdata);
+    if (is_readonly_or_worm_enabled(frame, this))
+        STACK_UNWIND_STRICT(setattr, frame, -1, EROFS, NULL, NULL, xdata);
+    else
+        STACK_WIND_TAIL(frame, FIRST_CHILD(this),
+                        FIRST_CHILD(this)->fops->setattr, loc, stbuf, valid,
+                        xdata);
+
+    return 0;
+}
 
-        return 0;
+int32_t
+ro_fsetattr(call_frame_t *frame, xlator_t *this, fd_t *fd, struct iatt *stbuf,
+            int32_t valid, dict_t *xdata)
+{
+    if (is_readonly_or_worm_enabled(frame, this))
+        STACK_UNWIND_STRICT(fsetattr, frame, -1, EROFS, NULL, NULL, xdata);
+    else
+        STACK_WIND_TAIL(frame, FIRST_CHILD(this),
+                        FIRST_CHILD(this)->fops->fsetattr, fd, stbuf, valid,
+                        xdata);
+
+    return 0;
 }
 
 int32_t
-ro_fsetattr (call_frame_t *frame, xlator_t *this, fd_t *fd,
-             struct iatt *stbuf, int32_t valid, dict_t *xdata)
+ro_truncate(call_frame_t *frame, xlator_t *this, loc_t *loc, off_t offset,
+            dict_t *xdata)
 {
-        if (is_readonly_or_worm_enabled (this))
-                STACK_UNWIND_STRICT (fsetattr, frame, -1, EROFS, NULL, NULL,
-                                     xdata);
-        else
-                STACK_WIND_TAIL (frame, FIRST_CHILD (this),
-                                 FIRST_CHILD(this)->fops->fsetattr, fd, stbuf,
-                                 valid, xdata);
+    if (is_readonly_or_worm_enabled(frame, this))
+        STACK_UNWIND_STRICT(truncate, frame, -1, EROFS, NULL, NULL, xdata);
+    else
+        STACK_WIND_TAIL(frame, FIRST_CHILD(this),
+                        FIRST_CHILD(this)->fops->truncate, loc, offset, xdata);
 
-        return 0;
+    return 0;
 }
 
-
 int32_t
-ro_truncate (call_frame_t *frame, xlator_t *this, loc_t *loc, off_t offset, dict_t *xdata)
+ro_ftruncate(call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
+             dict_t *xdata)
 {
-        if (is_readonly_or_worm_enabled (this))
-                STACK_UNWIND_STRICT (truncate, frame, -1, EROFS, NULL, NULL,
-                                     xdata);
-        else
-                STACK_WIND_TAIL (frame, FIRST_CHILD (this),
-                                 FIRST_CHILD(this)->fops->truncate, loc, offset,
-                                 xdata);
-
-	return 0;
+    if (is_readonly_or_worm_enabled(frame, this))
+        STACK_UNWIND_STRICT(ftruncate, frame, -1, EROFS, NULL, NULL, xdata);
+    else
+        STACK_WIND_TAIL(frame, FIRST_CHILD(this),
+                        FIRST_CHILD(this)->fops->ftruncate, fd, offset, xdata);
+
+    return 0;
 }
 
 int32_t
-ro_ftruncate (call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset, dict_t *xdata)
+ro_fallocate(call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t mode,
+             off_t offset, size_t len, dict_t *xdata)
 {
-        if (is_readonly_or_worm_enabled (this))
-                STACK_UNWIND_STRICT (ftruncate, frame, -1, EROFS, NULL, NULL,
-                                     xdata);
-        else
-                STACK_WIND_TAIL (frame, FIRST_CHILD (this),
-                                 FIRST_CHILD(this)->fops->ftruncate, fd, offset,
-                                 xdata);
-
-	return 0;
+    if (is_readonly_or_worm_enabled(frame, this))
+        STACK_UNWIND_STRICT(fallocate, frame, -1, EROFS, NULL, NULL, xdata);
+    else
+        STACK_WIND_TAIL(frame, FIRST_CHILD(this),
+                        FIRST_CHILD(this)->fops->fallocate, fd, mode, offset,
+                        len, xdata);
+    return 0;
 }
 
 int
-ro_mknod (call_frame_t *frame, xlator_t *this, loc_t *loc, mode_t mode,
-          dev_t rdev, mode_t umask, dict_t *xdata)
+ro_mknod(call_frame_t *frame, xlator_t *this, loc_t *loc, mode_t mode,
+         dev_t rdev, mode_t umask, dict_t *xdata)
 {
-        if (is_readonly_or_worm_enabled (this))
-                STACK_UNWIND_STRICT (mknod, frame, -1, EROFS, NULL, NULL, NULL,
-                                     NULL, xdata);
-        else
-                STACK_WIND_TAIL (frame, FIRST_CHILD (this),
-                                 FIRST_CHILD(this)->fops->mknod, loc, mode,
-                                 rdev, umask, xdata);
-
-	return 0;
+    if (is_readonly_or_worm_enabled(frame, this))
+        STACK_UNWIND_STRICT(mknod, frame, -1, EROFS, NULL, NULL, NULL, NULL,
+                            xdata);
+    else
+        STACK_WIND_TAIL(frame, FIRST_CHILD(this),
+                        FIRST_CHILD(this)->fops->mknod, loc, mode, rdev, umask,
+                        xdata);
+
+    return 0;
 }
 
-
 int
-ro_mkdir (call_frame_t *frame, xlator_t *this, loc_t *loc, mode_t mode,
-          mode_t umask, dict_t *xdata)
+ro_mkdir(call_frame_t *frame, xlator_t *this, loc_t *loc, mode_t mode,
+         mode_t umask, dict_t *xdata)
 {
-        if (is_readonly_or_worm_enabled (this))
-                STACK_UNWIND_STRICT (mkdir, frame, -1, EROFS, NULL, NULL, NULL,
-                                     NULL, xdata);
-        else
-                STACK_WIND_TAIL (frame, FIRST_CHILD (this),
-                                 FIRST_CHILD(this)->fops->mkdir, loc, mode,
-                                 umask, xdata);
-
-        return 0;
+    if (is_readonly_or_worm_enabled(frame, this))
+        STACK_UNWIND_STRICT(mkdir, frame, -1, EROFS, NULL, NULL, NULL, NULL,
+                            xdata);
+    else
+        STACK_WIND_TAIL(frame, FIRST_CHILD(this),
+                        FIRST_CHILD(this)->fops->mkdir, loc, mode, umask,
+                        xdata);
+
+    return 0;
 }
 
 int32_t
-ro_unlink (call_frame_t *frame, xlator_t *this, loc_t *loc, int xflag,
-           dict_t *xdata)
+ro_unlink(call_frame_t *frame, xlator_t *this, loc_t *loc, int xflag,
+          dict_t *xdata)
 {
-        if (is_readonly_or_worm_enabled (this))
-                STACK_UNWIND_STRICT (unlink, frame, -1, EROFS, NULL, NULL,
-                                     xdata);
-        else
-                STACK_WIND_TAIL (frame, FIRST_CHILD (this),
-                                 FIRST_CHILD(this)->fops->unlink, loc, xflag,
-                                 xdata);
+    if (is_readonly_or_worm_enabled(frame, this))
+        STACK_UNWIND_STRICT(unlink, frame, -1, EROFS, NULL, NULL, xdata);
+    else
+        STACK_WIND_TAIL(frame, FIRST_CHILD(this),
+                        FIRST_CHILD(this)->fops->unlink, loc, xflag, xdata);
 
-        return 0;
+    return 0;
 }
 
-
 int
-ro_rmdir (call_frame_t *frame, xlator_t *this, loc_t *loc, int flags,
-          dict_t *xdata)
+ro_rmdir(call_frame_t *frame, xlator_t *this, loc_t *loc, int flags,
+         dict_t *xdata)
 {
-        if (is_readonly_or_worm_enabled (this))
-                STACK_UNWIND_STRICT (rmdir, frame, -1, EROFS, NULL, NULL,
-                                     xdata);
-        else
-                STACK_WIND_TAIL (frame, FIRST_CHILD (this),
-                                 FIRST_CHILD(this)->fops->rmdir, loc, flags,
-                                 xdata);
+    if (is_readonly_or_worm_enabled(frame, this))
+        STACK_UNWIND_STRICT(rmdir, frame, -1, EROFS, NULL, NULL, xdata);
+    else
+        STACK_WIND_TAIL(frame, FIRST_CHILD(this),
+                        FIRST_CHILD(this)->fops->rmdir, loc, flags, xdata);
 
-        return 0;
+    return 0;
 }
 
-
 int
-ro_symlink (call_frame_t *frame, xlator_t *this, const char *linkpath,
-            loc_t *loc, mode_t umask, dict_t *xdata)
+ro_symlink(call_frame_t *frame, xlator_t *this, const char *linkpath,
+           loc_t *loc, mode_t umask, dict_t *xdata)
 {
-        if (is_readonly_or_worm_enabled (this))
-                STACK_UNWIND_STRICT (symlink, frame, -1, EROFS, NULL, NULL,
-                                     NULL, NULL, xdata);
-        else
-                STACK_WIND_TAIL (frame, FIRST_CHILD (this),
-                                 FIRST_CHILD(this)->fops->symlink, linkpath,
-                                 loc, umask, xdata);
-
-        return 0;
+    if (is_readonly_or_worm_enabled(frame, this))
+        STACK_UNWIND_STRICT(symlink, frame, -1, EROFS, NULL, NULL, NULL, NULL,
+                            xdata);
+    else
+        STACK_WIND_TAIL(frame, FIRST_CHILD(this),
+                        FIRST_CHILD(this)->fops->symlink, linkpath, loc, umask,
+                        xdata);
+
+    return 0;
 }
 
-
-
 int32_t
-ro_rename (call_frame_t *frame, xlator_t *this, loc_t *oldloc, loc_t *newloc,
-           dict_t *xdata)
+ro_rename(call_frame_t *frame, xlator_t *this, loc_t *oldloc, loc_t *newloc,
+          dict_t *xdata)
 {
-        if (is_readonly_or_worm_enabled (this))
-                STACK_UNWIND_STRICT (rename, frame, -1, EROFS, NULL, NULL, NULL,
-                                     NULL, NULL, xdata);
-        else
-                STACK_WIND_TAIL (frame, FIRST_CHILD (this),
-                                 FIRST_CHILD(this)->fops->rename, oldloc,
-                                 newloc, xdata);
-
-        return 0;
+    if (is_readonly_or_worm_enabled(frame, this))
+        STACK_UNWIND_STRICT(rename, frame, -1, EROFS, NULL, NULL, NULL, NULL,
+                            NULL, xdata);
+    else
+        STACK_WIND_TAIL(frame, FIRST_CHILD(this),
+                        FIRST_CHILD(this)->fops->rename, oldloc, newloc, xdata);
+
+    return 0;
 }
 
-
 int32_t
-ro_link (call_frame_t *frame, xlator_t *this, loc_t *oldloc, loc_t *newloc, dict_t *xdata)
+ro_link(call_frame_t *frame, xlator_t *this, loc_t *oldloc, loc_t *newloc,
+        dict_t *xdata)
 {
-        if (is_readonly_or_worm_enabled (this))
-                STACK_UNWIND_STRICT (link, frame, -1, EROFS, NULL, NULL, NULL,
-                                     NULL, xdata);
-        else
-                STACK_WIND_TAIL (frame, FIRST_CHILD (this),
-                                 FIRST_CHILD(this)->fops->link, oldloc, newloc,
-                                 xdata);
-
-        return 0;
+    if (is_readonly_or_worm_enabled(frame, this))
+        STACK_UNWIND_STRICT(link, frame, -1, EROFS, NULL, NULL, NULL, NULL,
+                            xdata);
+    else
+        STACK_WIND_TAIL(frame, FIRST_CHILD(this), FIRST_CHILD(this)->fops->link,
+                        oldloc, newloc, xdata);
+
+    return 0;
 }
 
 int32_t
-ro_create (call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t flags,
-           mode_t mode, mode_t umask, fd_t *fd, dict_t *xdata)
+ro_create(call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t flags,
+          mode_t mode, mode_t umask, fd_t *fd, dict_t *xdata)
 {
-        if (is_readonly_or_worm_enabled (this))
-                STACK_UNWIND_STRICT (create, frame, -1, EROFS, NULL, NULL, NULL,
-                                     NULL, NULL, xdata);
-        else
-                STACK_WIND_TAIL (frame, FIRST_CHILD (this),
-                                 FIRST_CHILD(this)->fops->create, loc, flags,
-                                 mode, umask, fd, xdata);
-
-        return 0;
+    if (is_readonly_or_worm_enabled(frame, this))
+        STACK_UNWIND_STRICT(create, frame, -1, EROFS, NULL, NULL, NULL, NULL,
+                            NULL, xdata);
+    else
+        STACK_WIND_TAIL(frame, FIRST_CHILD(this),
+                        FIRST_CHILD(this)->fops->create, loc, flags, mode,
+                        umask, fd, xdata);
+
+    return 0;
 }
 
-
 static int32_t
-ro_open_cbk (call_frame_t *frame, void *cookie, xlator_t *this, int32_t op_ret,
-             int32_t op_errno, fd_t *fd, dict_t *xdata)
+ro_open_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int32_t op_ret,
+            int32_t op_errno, fd_t *fd, dict_t *xdata)
 {
-        STACK_UNWIND_STRICT (open, frame, op_ret, op_errno, fd, xdata);
-        return 0;
+    STACK_UNWIND_STRICT(open, frame, op_ret, op_errno, fd, xdata);
+    return 0;
 }
 
 int32_t
-ro_open (call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t flags,
-         fd_t *fd, dict_t *xdata)
+ro_open(call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t flags,
+        fd_t *fd, dict_t *xdata)
 {
-        if (is_readonly_or_worm_enabled (this) &&
-            (((flags & O_ACCMODE) == O_WRONLY) ||
-              ((flags & O_ACCMODE) == O_RDWR))) {
-                STACK_UNWIND_STRICT (open, frame, -1, EROFS, NULL, xdata);
-                return 0;
-	}
-
-	STACK_WIND (frame, ro_open_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->open, loc, flags, fd, xdata);
-	return 0;
+    if (is_readonly_or_worm_enabled(frame, this) &&
+        (((flags & O_ACCMODE) == O_WRONLY) ||
+         ((flags & O_ACCMODE) == O_RDWR))) {
+        STACK_UNWIND_STRICT(open, frame, -1, EROFS, NULL, xdata);
+        return 0;
+    }
+
+    STACK_WIND(frame, ro_open_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->open, loc, flags, fd, xdata);
+    return 0;
 }
 
 int32_t
-ro_fsetxattr (call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *dict,
-              int32_t flags, dict_t *xdata)
+ro_fsetxattr(call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *dict,
+             int32_t flags, dict_t *xdata)
 {
-        if (is_readonly_or_worm_enabled (this))
-                STACK_UNWIND_STRICT (fsetxattr, frame, -1, EROFS, xdata);
-        else
-                STACK_WIND_TAIL (frame, FIRST_CHILD (this),
-                                 FIRST_CHILD(this)->fops->fsetxattr, fd, dict,
-                                 flags, xdata);
-
-        return 0;
+    if (is_readonly_or_worm_enabled(frame, this))
+        STACK_UNWIND_STRICT(fsetxattr, frame, -1, EROFS, xdata);
+    else
+        STACK_WIND_TAIL(frame, FIRST_CHILD(this),
+                        FIRST_CHILD(this)->fops->fsetxattr, fd, dict, flags,
+                        xdata);
+
+    return 0;
 }
 
 int32_t
-ro_fsyncdir (call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t flags,
-             dict_t *xdata)
+ro_fsyncdir(call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t flags,
+            dict_t *xdata)
 {
-        if (is_readonly_or_worm_enabled (this))
-                STACK_UNWIND_STRICT (fsyncdir, frame, -1, EROFS, xdata);
-        else
-                STACK_WIND_TAIL (frame, FIRST_CHILD (this),
-                                 FIRST_CHILD(this)->fops->fsyncdir, fd, flags,
-                                 xdata);
-
-	return 0;
+    if (is_readonly_or_worm_enabled(frame, this))
+        STACK_UNWIND_STRICT(fsyncdir, frame, -1, EROFS, xdata);
+    else
+        STACK_WIND_TAIL(frame, FIRST_CHILD(this),
+                        FIRST_CHILD(this)->fops->fsyncdir, fd, flags, xdata);
+
+    return 0;
 }
 
 int32_t
-ro_writev (call_frame_t *frame, xlator_t *this, fd_t *fd, struct iovec *vector,
-           int32_t count, off_t off, uint32_t flags, struct iobref *iobref,
-           dict_t *xdata)
+ro_writev(call_frame_t *frame, xlator_t *this, fd_t *fd, struct iovec *vector,
+          int32_t count, off_t off, uint32_t flags, struct iobref *iobref,
+          dict_t *xdata)
 {
-        if (is_readonly_or_worm_enabled (this))
-                STACK_UNWIND_STRICT (writev, frame, -1, EROFS, NULL, NULL,
-                                     xdata);
-        else
-                STACK_WIND_TAIL (frame, FIRST_CHILD (this),
-                                 FIRST_CHILD(this)->fops->writev, fd, vector,
-                                 count, off, flags, iobref, xdata);
-
-	return 0;
+    if (is_readonly_or_worm_enabled(frame, this))
+        STACK_UNWIND_STRICT(writev, frame, -1, EROFS, NULL, NULL, xdata);
+    else
+        STACK_WIND_TAIL(frame, FIRST_CHILD(this),
+                        FIRST_CHILD(this)->fops->writev, fd, vector, count, off,
+                        flags, iobref, xdata);
+
+    return 0;
 }
 
-
 int32_t
-ro_setxattr (call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *dict,
-             int32_t flags, dict_t *xdata)
+ro_setxattr(call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *dict,
+            int32_t flags, dict_t *xdata)
 {
-        if (is_readonly_or_worm_enabled (this))
-                STACK_UNWIND_STRICT (setxattr, frame, -1, EROFS, xdata);
-        else
-                STACK_WIND_TAIL (frame, FIRST_CHILD (this),
-                                 FIRST_CHILD(this)->fops->setxattr, loc, dict,
-                                 flags, xdata);
-
-	return 0;
+    if (is_readonly_or_worm_enabled(frame, this))
+        STACK_UNWIND_STRICT(setxattr, frame, -1, EROFS, xdata);
+    else
+        STACK_WIND_TAIL(frame, FIRST_CHILD(this),
+                        FIRST_CHILD(this)->fops->setxattr, loc, dict, flags,
+                        xdata);
+
+    return 0;
 }
 
 int32_t
-ro_removexattr (call_frame_t *frame, xlator_t *this, loc_t *loc,
-                const char *name, dict_t *xdata)
+ro_removexattr(call_frame_t *frame, xlator_t *this, loc_t *loc,
+               const char *name, dict_t *xdata)
 {
-        if (is_readonly_or_worm_enabled (this))
-                STACK_UNWIND_STRICT (removexattr, frame, -1, EROFS, xdata);
-        else
-                STACK_WIND_TAIL (frame, FIRST_CHILD (this),
-                                 FIRST_CHILD(this)->fops->removexattr, loc,
-                                 name, xdata);
+    if (is_readonly_or_worm_enabled(frame, this))
+        STACK_UNWIND_STRICT(removexattr, frame, -1, EROFS, xdata);
+    else
+        STACK_WIND_TAIL(frame, FIRST_CHILD(this),
+                        FIRST_CHILD(this)->fops->removexattr, loc, name, xdata);
 
-        return 0;
+    return 0;
 }
diff --git a/xlators/features/read-only/src/read-only-common.h b/xlators/features/read-only/src/read-only-common.h
index c99ba0310bb..5561961ffa2 100644
--- a/xlators/features/read-only/src/read-only-common.h
+++ b/xlators/features/read-only/src/read-only-common.h
@@ -7,112 +7,115 @@
    later), or the GNU General Public License, version 2 (GPLv2), in all
    cases as published by the Free Software Foundation.
 */
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
-
-#include "xlator.h"
-#include "defaults.h"
+#include <glusterfs/xlator.h>
+#include <glusterfs/defaults.h>
 
 gf_boolean_t
-is_readonly_or_worm_enabled (xlator_t *this);
+is_readonly_or_worm_enabled(call_frame_t *frame, xlator_t *this);
 
 int32_t
-ro_xattrop (call_frame_t *frame, xlator_t *this, loc_t *loc,
-            gf_xattrop_flags_t flags, dict_t *dict, dict_t *xdata);
+ro_xattrop(call_frame_t *frame, xlator_t *this, loc_t *loc,
+           gf_xattrop_flags_t flags, dict_t *dict, dict_t *xdata);
 
 int32_t
-ro_fxattrop (call_frame_t *frame, xlator_t *this,
-             fd_t *fd, gf_xattrop_flags_t flags, dict_t *dict, dict_t *xdata);
+ro_fxattrop(call_frame_t *frame, xlator_t *this, fd_t *fd,
+            gf_xattrop_flags_t flags, dict_t *dict, dict_t *xdata);
 
 int32_t
-ro_entrylk (call_frame_t *frame, xlator_t *this, const char *volume,
-            loc_t *loc, const char *basename, entrylk_cmd cmd,
-            entrylk_type type, dict_t *xdata);
+ro_entrylk(call_frame_t *frame, xlator_t *this, const char *volume, loc_t *loc,
+           const char *basename, entrylk_cmd cmd, entrylk_type type,
+           dict_t *xdata);
 
 int32_t
-ro_fentrylk (call_frame_t *frame, xlator_t *this, const char *volume,
-             fd_t *fd, const char *basename, entrylk_cmd cmd, entrylk_type
-             type, dict_t *xdata);
+ro_fentrylk(call_frame_t *frame, xlator_t *this, const char *volume, fd_t *fd,
+            const char *basename, entrylk_cmd cmd, entrylk_type type,
+            dict_t *xdata);
 
 int32_t
-ro_inodelk (call_frame_t *frame, xlator_t *this, const char *volume,
-            loc_t *loc, int32_t cmd, struct gf_flock *lock, dict_t *xdata);
+ro_inodelk(call_frame_t *frame, xlator_t *this, const char *volume, loc_t *loc,
+           int32_t cmd, struct gf_flock *lock, dict_t *xdata);
 
 int32_t
-ro_finodelk (call_frame_t *frame, xlator_t *this, const char *volume,
-             fd_t *fd, int32_t cmd, struct gf_flock *lock, dict_t *xdata);
+ro_finodelk(call_frame_t *frame, xlator_t *this, const char *volume, fd_t *fd,
+            int32_t cmd, struct gf_flock *lock, dict_t *xdata);
 
 int32_t
-ro_lk (call_frame_t *frame, xlator_t *this, fd_t *fd, int cmd,
-       struct gf_flock *flock, dict_t *xdata);
+ro_lk(call_frame_t *frame, xlator_t *this, fd_t *fd, int cmd,
+      struct gf_flock *flock, dict_t *xdata);
 
 int32_t
-ro_setattr (call_frame_t *frame, xlator_t *this, loc_t *loc,
-            struct iatt *stbuf, int32_t valid, dict_t *xdata);
+ro_setattr(call_frame_t *frame, xlator_t *this, loc_t *loc, struct iatt *stbuf,
+           int32_t valid, dict_t *xdata);
 
 int32_t
-ro_fsetattr (call_frame_t *frame, xlator_t *this, fd_t *fd,
-             struct iatt *stbuf, int32_t valid, dict_t *xdata);
-
+ro_fsetattr(call_frame_t *frame, xlator_t *this, fd_t *fd, struct iatt *stbuf,
+            int32_t valid, dict_t *xdata);
 
 int32_t
-ro_truncate (call_frame_t *frame, xlator_t *this, loc_t *loc, off_t offset, dict_t *xdata);
+ro_truncate(call_frame_t *frame, xlator_t *this, loc_t *loc, off_t offset,
+            dict_t *xdata);
 
 int32_t
-ro_ftruncate (call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset, dict_t *xdata);
+ro_ftruncate(call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
+             dict_t *xdata);
 
 int
-ro_mknod (call_frame_t *frame, xlator_t *this, loc_t *loc, mode_t mode,
-          dev_t rdev, mode_t umask, dict_t *xdata);
+ro_mknod(call_frame_t *frame, xlator_t *this, loc_t *loc, mode_t mode,
+         dev_t rdev, mode_t umask, dict_t *xdata);
 
 int
-ro_mkdir (call_frame_t *frame, xlator_t *this, loc_t *loc, mode_t mode,
-          mode_t umask, dict_t *xdata);
+ro_mkdir(call_frame_t *frame, xlator_t *this, loc_t *loc, mode_t mode,
+         mode_t umask, dict_t *xdata);
 
 int32_t
-ro_unlink (call_frame_t *frame, xlator_t *this, loc_t *loc, int xflag,
-           dict_t *xdata);
-
-int
-ro_rmdir (call_frame_t *frame, xlator_t *this, loc_t *loc, int flags,
+ro_unlink(call_frame_t *frame, xlator_t *this, loc_t *loc, int xflag,
           dict_t *xdata);
 
+int
+ro_rmdir(call_frame_t *frame, xlator_t *this, loc_t *loc, int flags,
+         dict_t *xdata);
 
 int
-ro_symlink (call_frame_t *frame, xlator_t *this, const char *linkpath,
-            loc_t *loc, mode_t umask, dict_t *xdata);
+ro_symlink(call_frame_t *frame, xlator_t *this, const char *linkpath,
+           loc_t *loc, mode_t umask, dict_t *xdata);
 
 int32_t
-ro_rename (call_frame_t *frame, xlator_t *this, loc_t *oldloc, loc_t *newloc, dict_t *xdata);
+ro_rename(call_frame_t *frame, xlator_t *this, loc_t *oldloc, loc_t *newloc,
+          dict_t *xdata);
 
 int32_t
-ro_link (call_frame_t *frame, xlator_t *this, loc_t *oldloc, loc_t *newloc, dict_t *xdata);
+ro_link(call_frame_t *frame, xlator_t *this, loc_t *oldloc, loc_t *newloc,
+        dict_t *xdata);
 
 int32_t
-ro_create (call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t flags,
-           mode_t mode, mode_t umask, fd_t *fd, dict_t *xdata);
+ro_create(call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t flags,
+          mode_t mode, mode_t umask, fd_t *fd, dict_t *xdata);
 
 int32_t
-ro_open (call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t flags,
-         fd_t *fd, dict_t *xdata);
+ro_open(call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t flags,
+        fd_t *fd, dict_t *xdata);
 
 int32_t
-ro_fsetxattr (call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *dict,
-              int32_t flags, dict_t *xdata);
+ro_fsetxattr(call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *dict,
+             int32_t flags, dict_t *xdata);
 
 int32_t
-ro_fsyncdir (call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t flags, dict_t *xdata);
+ro_fsyncdir(call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t flags,
+            dict_t *xdata);
 
 int32_t
-ro_writev (call_frame_t *frame, xlator_t *this, fd_t *fd, struct iovec *vector,
-           int32_t count, off_t off, uint32_t flags, struct iobref *iobref, dict_t *xdata);
+ro_writev(call_frame_t *frame, xlator_t *this, fd_t *fd, struct iovec *vector,
+          int32_t count, off_t off, uint32_t flags, struct iobref *iobref,
+          dict_t *xdata);
 
 int32_t
-ro_setxattr (call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *dict,
-             int32_t flags, dict_t *xdata);
+ro_setxattr(call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *dict,
+            int32_t flags, dict_t *xdata);
+
+int32_t
+ro_removexattr(call_frame_t *frame, xlator_t *this, loc_t *loc,
+               const char *name, dict_t *xdata);
 
 int32_t
-ro_removexattr (call_frame_t *frame, xlator_t *this, loc_t *loc,
-                const char *name, dict_t *xdata);
+ro_fallocate(call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t mode,
+             off_t offset, size_t len, dict_t *xdata);
diff --git a/xlators/features/read-only/src/read-only-mem-types.h b/xlators/features/read-only/src/read-only-mem-types.h
index 940700a017d..c67d6c02cd0 100644
--- a/xlators/features/read-only/src/read-only-mem-types.h
+++ b/xlators/features/read-only/src/read-only-mem-types.h
@@ -11,10 +11,10 @@
 #ifndef __READONLY_MEM_TYPES_H__
 #define __READONLY_MEM_TYPES_H__
 
-#include "mem-types.h"
+#include <glusterfs/mem-types.h>
 
 enum gf_read_only_mem_types_ {
-        gf_read_only_mt_priv_t = gf_common_mt_end + 1,
-        gf_read_only_mt_end
+    gf_read_only_mt_priv_t = gf_common_mt_end + 1,
+    gf_read_only_mt_end
 };
 #endif
diff --git a/xlators/features/read-only/src/read-only.c b/xlators/features/read-only/src/read-only.c
index 173c4f51e2a..48654998e63 100644
--- a/xlators/features/read-only/src/read-only.c
+++ b/xlators/features/read-only/src/read-only.c
@@ -7,129 +7,138 @@
    later), or the GNU General Public License, version 2 (GPLv2), in all
    cases as published by the Free Software Foundation.
 */
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
-
-#include "defaults.h"
 #include "read-only-common.h"
 #include "read-only-mem-types.h"
 #include "read-only.h"
 
 int32_t
-mem_acct_init (xlator_t *this)
+mem_acct_init(xlator_t *this)
 {
-        int     ret = -1;
+    int ret = -1;
 
-        ret = xlator_mem_acct_init (this, gf_read_only_mt_end + 1);
-        if (ret)
-                gf_log (this->name, GF_LOG_ERROR, "Memory accounting "
-                        "initialization failed.");
+    ret = xlator_mem_acct_init(this, gf_read_only_mt_end + 1);
+    if (ret)
+        gf_log(this->name, GF_LOG_ERROR,
+               "Memory accounting "
+               "initialization failed.");
 
-        return ret;
+    return ret;
 }
 
 int32_t
-init (xlator_t *this)
+init(xlator_t *this)
 {
-	int                     ret     = -1;
-        read_only_priv_t       *priv    = NULL;
+    int ret = -1;
+    read_only_priv_t *priv = NULL;
+
+    if (!this->children || this->children->next) {
+        gf_log(this->name, GF_LOG_ERROR,
+               "translator not configured with exactly one child");
+        return -1;
+    }
 
-        if (!this->children || this->children->next) {
-		gf_log (this->name, GF_LOG_ERROR,
-			"translator not configured with exactly one child");
-		return -1;
-	}
+    if (!this->parents) {
+        gf_log(this->name, GF_LOG_WARNING, "dangling volume. check volfile ");
+    }
 
-	if (!this->parents) {
-		gf_log (this->name, GF_LOG_WARNING,
-			"dangling volume. check volfile ");
-	}
+    priv = GF_CALLOC(1, sizeof(*priv), gf_read_only_mt_priv_t);
+    if (!priv)
+        goto out;
 
-        priv = GF_CALLOC (1, sizeof (*priv), gf_read_only_mt_priv_t);
-        if (!priv)
-                goto out;
+    this->private = priv;
 
-        GF_OPTION_INIT ("read-only", priv->readonly_or_worm_enabled, bool, out);
+    GF_OPTION_INIT("read-only", priv->readonly_or_worm_enabled, bool, out);
 
-        this->private = priv;
-        ret = 0;
+    ret = 0;
 out:
-        return ret;
+    return ret;
 }
 
 int
-reconfigure (xlator_t *this, dict_t *options)
+reconfigure(xlator_t *this, dict_t *options)
 {
-        read_only_priv_t  *priv             = NULL;
-        int                ret              = -1;
-        gf_boolean_t       readonly_or_worm_enabled = _gf_false;
+    read_only_priv_t *priv = NULL;
+    int ret = -1;
+    gf_boolean_t readonly_or_worm_enabled = _gf_false;
 
-        priv = this->private;
-        GF_ASSERT (priv);
+    priv = this->private;
+    GF_ASSERT(priv);
 
-        GF_OPTION_RECONF ("read-only", readonly_or_worm_enabled, options, bool,
-                          out);
-        priv->readonly_or_worm_enabled = readonly_or_worm_enabled;
-        ret = 0;
+    GF_OPTION_RECONF("read-only", readonly_or_worm_enabled, options, bool, out);
+    priv->readonly_or_worm_enabled = readonly_or_worm_enabled;
+    ret = 0;
 out:
-        gf_log (this->name, GF_LOG_DEBUG, "returning %d", ret);
-        return ret;
+    gf_log(this->name, GF_LOG_DEBUG, "returning %d", ret);
+    return ret;
 }
 
 void
-fini (xlator_t *this)
+fini(xlator_t *this)
 {
-        read_only_priv_t         *priv    = NULL;
+    read_only_priv_t *priv = NULL;
 
-        priv = this->private;
-        if (!priv)
-                return;
+    priv = this->private;
+    if (!priv)
+        return;
 
-        this->private = NULL;
-        GF_FREE (priv);
+    this->private = NULL;
+    GF_FREE(priv);
 
-        return;
+    return;
 }
 
-
 struct xlator_fops fops = {
-        .mknod       = ro_mknod,
-        .mkdir       = ro_mkdir,
-        .unlink      = ro_unlink,
-        .rmdir       = ro_rmdir,
-        .symlink     = ro_symlink,
-        .rename      = ro_rename,
-        .link        = ro_link,
-        .truncate    = ro_truncate,
-        .open        = ro_open,
-        .writev      = ro_writev,
-        .setxattr    = ro_setxattr,
-        .fsetxattr   = ro_fsetxattr,
-        .removexattr = ro_removexattr,
-        .fsyncdir    = ro_fsyncdir,
-        .ftruncate   = ro_ftruncate,
-        .create      = ro_create,
-        .setattr     = ro_setattr,
-        .fsetattr    = ro_fsetattr,
-        .xattrop     = ro_xattrop,
-        .fxattrop    = ro_fxattrop,
-        .inodelk     = ro_inodelk,
-        .finodelk    = ro_finodelk,
-        .entrylk     = ro_entrylk,
-        .fentrylk    = ro_fentrylk,
-        .lk          = ro_lk,
+    .mknod = ro_mknod,
+    .mkdir = ro_mkdir,
+    .unlink = ro_unlink,
+    .rmdir = ro_rmdir,
+    .symlink = ro_symlink,
+    .rename = ro_rename,
+    .link = ro_link,
+    .truncate = ro_truncate,
+    .open = ro_open,
+    .writev = ro_writev,
+    .setxattr = ro_setxattr,
+    .fsetxattr = ro_fsetxattr,
+    .removexattr = ro_removexattr,
+    .fsyncdir = ro_fsyncdir,
+    .ftruncate = ro_ftruncate,
+    .create = ro_create,
+    .setattr = ro_setattr,
+    .fsetattr = ro_fsetattr,
+    .xattrop = ro_xattrop,
+    .fxattrop = ro_fxattrop,
+    .inodelk = ro_inodelk,
+    .finodelk = ro_finodelk,
+    .entrylk = ro_entrylk,
+    .fentrylk = ro_fentrylk,
+    .lk = ro_lk,
+    .fallocate = ro_fallocate,
 };
 
-struct xlator_cbks cbks = {
-};
+struct xlator_cbks cbks = {};
 
 struct volume_options options[] = {
-	{ .key  = {"read-only"},
-          .type = GF_OPTION_TYPE_BOOL,
-          .default_value = "off",
-          .description = "When \"on\", makes a volume read-only. It is turned "
-                         "\"off\" by default."
-        },
+    {.key = {"read-only"},
+     .type = GF_OPTION_TYPE_BOOL,
+     .default_value = "off",
+     /*.validate_fn = validate_boolean,*/
+     .op_version = {1},
+     .flags = OPT_FLAG_SETTABLE,
+     .description = "When \"on\", makes a volume read-only. It is turned "
+                    "\"off\" by default."},
+    {.key = {NULL}},
+};
+
+xlator_api_t xlator_api = {
+    .init = init,
+    .fini = fini,
+    .reconfigure = reconfigure,
+    .mem_acct_init = mem_acct_init,
+    .op_version = {1}, /* Present from the initial version */
+    .fops = &fops,
+    .cbks = &cbks,
+    .options = options,
+    .identifier = "read-only",
+    .category = GF_TECH_PREVIEW,
 };
diff --git a/xlators/features/read-only/src/read-only.h b/xlators/features/read-only/src/read-only.h
index 8e7e1b68081..aced5d3c577 100644
--- a/xlators/features/read-only/src/read-only.h
+++ b/xlators/features/read-only/src/read-only.h
@@ -11,11 +11,27 @@
 #ifndef __READONLY_H__
 #define __READONLY_H__
 
-#include "read-only-mem-types.h"
-#include "xlator.h"
+#include <stdint.h>               // for uint64_t, uint8_t
+#include <sys/time.h>             // for time_t
+#include "glusterfs/glusterfs.h"  // for gf_boolean_t
 
 typedef struct {
-        gf_boolean_t      readonly_or_worm_enabled;
+    uint8_t worm : 1;
+    uint8_t retain : 1;
+    uint8_t legal_hold : 1;
+    uint8_t ret_mode : 1;
+    int64_t ret_period;
+    int64_t auto_commit_period;
+} worm_reten_state_t;
+
+typedef struct {
+    gf_boolean_t readonly_or_worm_enabled;
+    gf_boolean_t worm_file;
+    gf_boolean_t worm_files_deletable;
+    int64_t reten_period;
+    int64_t com_period;
+    int reten_mode;
+    time_t start_time;
 } read_only_priv_t;
 
 #endif
diff --git a/xlators/features/read-only/src/worm-helper.c b/xlators/features/read-only/src/worm-helper.c
new file mode 100644
index 00000000000..df45f2a940b
--- /dev/null
+++ b/xlators/features/read-only/src/worm-helper.c
@@ -0,0 +1,395 @@
+/*
+   Copyright (c) 2016 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
+
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
+#include "read-only-mem-types.h"
+#include "read-only.h"
+#include <glusterfs/xlator.h>
+#include <glusterfs/syncop.h>
+#include "worm-helper.h"
+
+/*Function to check whether file is read-only.
+ * The input *stbuf contains the attributes of the file, which is used to check
+ * the write protection bits for all the users of the file.
+ * Return true if all the write bits are disabled,false otherwise*/
+gf_boolean_t
+gf_worm_write_disabled(struct iatt *stbuf)
+{
+    gf_boolean_t ret = _gf_false;
+
+    GF_VALIDATE_OR_GOTO("worm", stbuf, out);
+
+    if (stbuf->ia_prot.owner.write == 0 && stbuf->ia_prot.group.write == 0 &&
+        stbuf->ia_prot.other.write == 0)
+        ret = _gf_true;
+out:
+    return ret;
+}
+
+int32_t
+worm_init_state(xlator_t *this, gf_boolean_t fop_with_fd, void *file_ptr)
+{
+    int ret = -1;
+    uint64_t start_time = 0;
+    dict_t *dict = NULL;
+
+    GF_VALIDATE_OR_GOTO("worm", this, out);
+    GF_VALIDATE_OR_GOTO(this->name, file_ptr, out);
+
+    start_time = gf_time();
+    dict = dict_new();
+    if (!dict) {
+        gf_log(this->name, GF_LOG_ERROR, "Error creating the dict");
+        goto out;
+    }
+    ret = dict_set_uint64(dict, "trusted.start_time", start_time);
+    if (ret) {
+        gf_log(this->name, GF_LOG_ERROR, "Error in setting the dict");
+        goto out;
+    }
+    if (fop_with_fd)
+        ret = syncop_fsetxattr(this, (fd_t *)file_ptr, dict, 0, NULL, NULL);
+    else
+        ret = syncop_setxattr(this, (loc_t *)file_ptr, dict, 0, NULL, NULL);
+out:
+    if (dict)
+        dict_unref(dict);
+    return ret;
+}
+
+/*Function to set the retention state for a file.
+ * It loads the WORM/Retention state into the retention_state pointer.*/
+int32_t
+worm_set_state(xlator_t *this, gf_boolean_t fop_with_fd, void *file_ptr,
+               worm_reten_state_t *retention_state, struct iatt *stbuf)
+{
+    read_only_priv_t *priv = NULL;
+    struct iatt stpre = {
+        0,
+    };
+    int ret = -1;
+
+    GF_VALIDATE_OR_GOTO("worm", this, out);
+    GF_VALIDATE_OR_GOTO(this->name, file_ptr, out);
+    GF_VALIDATE_OR_GOTO(this->name, retention_state, out);
+    GF_VALIDATE_OR_GOTO(this->name, stbuf, out);
+
+    priv = this->private;
+    GF_ASSERT(priv);
+    retention_state->worm = 1;
+    retention_state->retain = 1;
+    retention_state->legal_hold = 0;
+    retention_state->ret_mode = priv->reten_mode;
+    retention_state->ret_period = priv->reten_period;
+    retention_state->auto_commit_period = priv->com_period;
+    if (fop_with_fd)
+        ret = syncop_fstat(this, (fd_t *)file_ptr, &stpre, NULL, NULL);
+    else
+        ret = syncop_stat(this, (loc_t *)file_ptr, &stpre, NULL, NULL);
+    if (ret)
+        goto out;
+    stbuf->ia_mtime = stpre.ia_mtime;
+    stbuf->ia_atime = gf_time() + retention_state->ret_period;
+
+    if (fop_with_fd)
+        ret = syncop_fsetattr(this, (fd_t *)file_ptr, stbuf, GF_SET_ATTR_ATIME,
+                              NULL, NULL, NULL, NULL);
+    else
+        ret = syncop_setattr(this, (loc_t *)file_ptr, stbuf, GF_SET_ATTR_ATIME,
+                             NULL, NULL, NULL, NULL);
+    if (ret)
+        goto out;
+
+    ret = gf_worm_set_xattr(this, retention_state, fop_with_fd, file_ptr);
+    if (ret) {
+        gf_log(this->name, GF_LOG_ERROR, "Error setting xattr");
+        goto out;
+    }
+    ret = 0;
+out:
+    return ret;
+}
+
+/*This function gets the state of the WORM/Retention xattr and loads it in the
+ * dict pointer.*/
+int32_t
+worm_get_state(xlator_t *this, gf_boolean_t fop_with_fd, void *file_ptr,
+               worm_reten_state_t *reten_state)
+{
+    dict_t *dict = NULL;
+    char *val = NULL;
+    int ret = -1;
+
+    GF_VALIDATE_OR_GOTO("worm", this, out);
+    GF_VALIDATE_OR_GOTO(this->name, file_ptr, out);
+    GF_VALIDATE_OR_GOTO(this->name, reten_state, out);
+
+    if (fop_with_fd)
+        ret = syncop_fgetxattr(this, (fd_t *)file_ptr, &dict,
+                               "trusted.reten_state", NULL, NULL);
+    else
+        ret = syncop_getxattr(this, (loc_t *)file_ptr, &dict,
+                              "trusted.reten_state", NULL, NULL);
+    if (ret < 0 || !dict) {
+        ret = -1;
+        goto out;
+    }
+    ret = dict_get_str(dict, "trusted.reten_state", &val);
+    if (ret) {
+        ret = -2;
+        gf_log(this->name, GF_LOG_ERROR, "Empty val");
+    }
+    gf_worm_deserialize_state(val, reten_state);
+out:
+    if (dict)
+        dict_unref(dict);
+    return ret;
+}
+
+/*Function to lookup the current state of the WORM/Retention profile.
+ * Based on the retain value and the access time of the file, the transition
+ * from WORM/Retention to WORM is made.*/
+void
+gf_worm_state_lookup(xlator_t *this, gf_boolean_t fop_with_fd, void *file_ptr,
+                     worm_reten_state_t *reten_state, struct iatt *stbuf)
+{
+    int ret = -1;
+
+    GF_VALIDATE_OR_GOTO("worm", this, out);
+    GF_VALIDATE_OR_GOTO(this->name, file_ptr, out);
+    GF_VALIDATE_OR_GOTO(this->name, reten_state, out);
+    GF_VALIDATE_OR_GOTO(this->name, stbuf, out);
+
+    stbuf->ia_atime -= reten_state->ret_period;
+    reten_state->retain = 0;
+    reten_state->ret_period = 0;
+    reten_state->auto_commit_period = 0;
+    ret = gf_worm_set_xattr(this, reten_state, fop_with_fd, file_ptr);
+    if (ret) {
+        gf_log(this->name, GF_LOG_ERROR, "Error setting xattr");
+        goto out;
+    }
+
+    if (fop_with_fd)
+        ret = syncop_fsetattr(this, (fd_t *)file_ptr, stbuf, GF_SET_ATTR_ATIME,
+                              NULL, NULL, NULL, NULL);
+    else
+        ret = syncop_setattr(this, (loc_t *)file_ptr, stbuf, GF_SET_ATTR_ATIME,
+                             NULL, NULL, NULL, NULL);
+    if (ret)
+        goto out;
+    gf_log(this->name, GF_LOG_INFO, "Retention state reset");
+out:
+    return;
+}
+
+/*This function serializes and stores the WORM/Retention state of a file in an
+ * uint64_t variable by setting the bits using the bitwise operations.*/
+void
+gf_worm_serialize_state(worm_reten_state_t *reten_state, char *val)
+{
+    uint32_t state = 0;
+
+    GF_VALIDATE_OR_GOTO("worm", reten_state, out);
+    GF_VALIDATE_OR_GOTO("worm", val, out);
+
+    state |= reten_state->worm << 0;
+    state |= reten_state->retain << 1;
+    state |= reten_state->legal_hold << 2;
+    state |= reten_state->ret_mode << 3;
+    sprintf(val, "%d/%" PRIu64 "/%" PRIu64, state, reten_state->ret_period,
+            reten_state->auto_commit_period);
+
+out:
+    return;
+}
+
+/*This function deserializes the data stored in the xattr of the file and loads
+ * the value to the reten_state structure.*/
+void
+gf_worm_deserialize_state(char *val, worm_reten_state_t *reten_state)
+{
+    char *token = NULL;
+    uint32_t state = 0;
+
+    GF_VALIDATE_OR_GOTO("worm", val, out);
+    GF_VALIDATE_OR_GOTO("worm", reten_state, out);
+
+    token = strtok(val, "/");
+    state = atoi(token);
+    reten_state->worm = (state >> 0) & 1;
+    reten_state->retain = (state >> 1) & 1;
+    reten_state->legal_hold = (state >> 2) & 1;
+    reten_state->ret_mode = (state >> 3) & 1;
+    token = strtok(NULL, "/");
+    reten_state->ret_period = atoi(token);
+    token = strtok(NULL, "/");
+    reten_state->auto_commit_period = atoi(token);
+
+out:
+    return;
+}
+
+/*Function to set the xattr for a file.
+ * If the xattr is already present then it will replace that.*/
+int32_t
+gf_worm_set_xattr(xlator_t *this, worm_reten_state_t *reten_state,
+                  gf_boolean_t fop_with_fd, void *file_ptr)
+{
+    char val[100] = "";
+    int ret = -1;
+    dict_t *dict = NULL;
+
+    GF_VALIDATE_OR_GOTO("worm", this, out);
+    GF_VALIDATE_OR_GOTO(this->name, reten_state, out);
+    GF_VALIDATE_OR_GOTO(this->name, file_ptr, out);
+
+    gf_worm_serialize_state(reten_state, val);
+    dict = dict_new();
+    if (!dict) {
+        gf_log(this->name, GF_LOG_ERROR, "Error creating the dict");
+        goto out;
+    }
+    ret = dict_set_str(dict, "trusted.reten_state", val);
+    if (ret) {
+        gf_log(this->name, GF_LOG_ERROR, "Error in setting the dict");
+        goto out;
+    }
+    if (fop_with_fd)
+        ret = syncop_fsetxattr(this, (fd_t *)file_ptr, dict, 0, NULL, NULL);
+    else
+        ret = syncop_setxattr(this, (loc_t *)file_ptr, dict, 0, NULL, NULL);
+out:
+    if (dict)
+        dict_unref(dict);
+    return ret;
+}
+
+/*This function checks whether a file's timeout is happened for the state
+ * transition and if yes, then it will do the transition from the current state
+ * to the appropriate state. It also decides whether to continue or to block
+ * the FOP.
+ * Return:
+ * 0 : If the FOP should continue i.e., if the file is not in the WORM-Retained
+ *     state or if the FOP is unlink and the file is not in the Retained state.
+ * 1: If the FOP sholud block i.e., if the file is in WORM-Retained/WORM state.
+ * 2: Blocks the FOP if any operation fails while doing the state transition or
+ *    fails to get the state of the file.*/
+int
+gf_worm_state_transition(xlator_t *this, gf_boolean_t fop_with_fd,
+                         void *file_ptr, glusterfs_fop_t op)
+{
+    int op_errno = EROFS;
+    int ret = -1;
+    time_t now = 0;
+    uint64_t com_period = 0;
+    uint64_t start_time = 0;
+    dict_t *dict = NULL;
+    worm_reten_state_t reten_state = {
+        0,
+    };
+    read_only_priv_t *priv = NULL;
+    struct iatt stbuf = {
+        0,
+    };
+
+    priv = this->private;
+    GF_ASSERT(priv);
+
+    if (fop_with_fd)
+        ret = syncop_fgetxattr(this, (fd_t *)file_ptr, &dict,
+                               "trusted.start_time", NULL, NULL);
+    else
+        ret = syncop_getxattr(this, (loc_t *)file_ptr, &dict,
+                              "trusted.start_time", NULL, NULL);
+    if (ret < 0 || !dict) {
+        op_errno = ret;
+        gf_msg(this->name, GF_LOG_ERROR, -ret, 0, "Error getting xattr");
+        goto out;
+    }
+    ret = dict_get_uint64(dict, "trusted.start_time", &start_time);
+    if (ret) {
+        op_errno = ret;
+        gf_msg(this->name, GF_LOG_ERROR, -ret, 0, "Error getting start time");
+        goto out;
+    }
+
+    com_period = priv->com_period;
+    if (fop_with_fd)
+        ret = syncop_fstat(this, (fd_t *)file_ptr, &stbuf, NULL, NULL);
+    else
+        ret = syncop_stat(this, (loc_t *)file_ptr, &stbuf, NULL, NULL);
+    if (ret) {
+        op_errno = ret;
+        gf_msg(this->name, GF_LOG_ERROR, -ret, 0, "Error getting file stat");
+        goto out;
+    }
+
+    ret = worm_get_state(this, fop_with_fd, file_ptr, &reten_state);
+    if (ret == -2) {
+        op_errno = ret;
+        gf_msg(this->name, GF_LOG_ERROR, -ret, 0,
+               "Error getting worm/retention state");
+        goto out;
+    }
+
+    now = gf_time();
+
+    if (ret == -1 && (now - start_time) >= com_period) {
+        if ((now - stbuf.ia_mtime) >= com_period) {
+            ret = worm_set_state(this, fop_with_fd, file_ptr, &reten_state,
+                                 &stbuf);
+            if (ret) {
+                op_errno = ret;
+                gf_msg(this->name, GF_LOG_ERROR, -ret, 0,
+                       "Error setting worm/retention state");
+                goto out;
+            }
+            goto out;
+        } else {
+            op_errno = 0;
+            goto out;
+        }
+    } else if (ret == -1 && (now - start_time) < com_period) {
+        op_errno = 0;
+        goto out;
+    } else if (reten_state.retain && ((now >= stbuf.ia_atime))) {
+        gf_worm_state_lookup(this, fop_with_fd, file_ptr, &reten_state, &stbuf);
+    }
+    if (reten_state.worm && !reten_state.retain && priv->worm_files_deletable &&
+        op == GF_FOP_UNLINK) {
+        op_errno = 0;
+        goto out;
+    }
+
+out:
+    if (dict)
+        dict_unref(dict);
+    return op_errno;
+}
+
+/*Function to check whether a file is independently WORMed (i.e., file level
+ * WORM is set on the file). */
+int32_t
+is_wormfile(xlator_t *this, gf_boolean_t fop_with_fd, void *file_ptr)
+{
+    int ret = -1;
+    dict_t *dict = NULL;
+
+    if (fop_with_fd)
+        ret = syncop_fgetxattr(this, (fd_t *)file_ptr, &dict,
+                               "trusted.worm_file", NULL, NULL);
+    else
+        ret = syncop_getxattr(this, (loc_t *)file_ptr, &dict,
+                              "trusted.worm_file", NULL, NULL);
+    if (dict) {
+        ret = 0;
+        dict_unref(dict);
+    }
+    return ret;
+}
diff --git a/xlators/features/read-only/src/worm-helper.h b/xlators/features/read-only/src/worm-helper.h
new file mode 100644
index 00000000000..b42f8d2b40c
--- /dev/null
+++ b/xlators/features/read-only/src/worm-helper.h
@@ -0,0 +1,44 @@
+/*
+   Copyright (c) 2016 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
+
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
+
+gf_boolean_t
+gf_worm_write_disabled(struct iatt *stbuf);
+
+int32_t
+worm_init_state(xlator_t *this, gf_boolean_t fop_with_fd, void *file_ptr);
+
+int32_t
+worm_set_state(xlator_t *this, gf_boolean_t fop_with_fd, void *file_ptr,
+               worm_reten_state_t *retention_state, struct iatt *stbuf);
+
+int32_t
+worm_get_state(xlator_t *this, gf_boolean_t fop_with_fd, void *file_ptr,
+               worm_reten_state_t *reten_state);
+
+void
+gf_worm_state_lookup(xlator_t *this, gf_boolean_t fop_with_fd, void *file_ptr,
+                     worm_reten_state_t *reten_state, struct iatt *stbuf);
+
+void
+gf_worm_serialize_state(worm_reten_state_t *reten_state, char *val);
+
+void
+gf_worm_deserialize_state(char *val, worm_reten_state_t *reten_state);
+
+int32_t
+gf_worm_set_xattr(xlator_t *this, worm_reten_state_t *reten_state,
+                  gf_boolean_t fop_with_fd, void *file_ptr);
+
+int
+gf_worm_state_transition(xlator_t *this, gf_boolean_t fop_with_fd,
+                         void *file_ptr, glusterfs_fop_t op);
+
+int32_t
+is_wormfile(xlator_t *this, gf_boolean_t fop_with_fd, void *file_ptr);
diff --git a/xlators/features/read-only/src/worm.c b/xlators/features/read-only/src/worm.c
index f62ffa3f6f3..1cc5526d5cd 100644
--- a/xlators/features/read-only/src/worm.c
+++ b/xlators/features/read-only/src/worm.c
@@ -1,5 +1,5 @@
 /*
-   Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
+   Copyright (c) 2008-2012, 2016 Red Hat, Inc. <http://www.redhat.com>
    This file is part of GlusterFS.
 
    This file is licensed to you under your choice of the GNU Lesser
@@ -7,143 +7,716 @@
    later), or the GNU General Public License, version 2 (GPLv2), in all
    cases as published by the Free Software Foundation.
 */
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
-
-#include "xlator.h"
-#include "defaults.h"
+#include <glusterfs/xlator.h>
+#include <glusterfs/defaults.h>
 #include "read-only-common.h"
 #include "read-only-mem-types.h"
 #include "read-only.h"
+#include <glusterfs/syncop.h>
+#include "worm-helper.h"
 
 int32_t
-mem_acct_init (xlator_t *this)
+mem_acct_init(xlator_t *this)
 {
-        int     ret = -1;
+    int ret = -1;
 
-        ret = xlator_mem_acct_init (this, gf_read_only_mt_end + 1);
-        if (ret)
-                gf_log (this->name, GF_LOG_ERROR, "Memory accounting "
-                        "initialization failed.");
+    ret = xlator_mem_acct_init(this, gf_read_only_mt_end + 1);
+    if (ret)
+        gf_log(this->name, GF_LOG_ERROR,
+               "Memory accounting "
+               "initialization failed.");
 
-        return ret;
+    return ret;
 }
 
 static int32_t
-worm_open_cbk (call_frame_t *frame, void *cookie, xlator_t *this, int32_t op_ret,
-               int32_t op_errno, fd_t *fd, dict_t *xdata)
+worm_open(call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t flags,
+          fd_t *fd, dict_t *xdata)
 {
-        STACK_UNWIND_STRICT (open, frame, op_ret, op_errno, fd, xdata);
+    if (is_readonly_or_worm_enabled(frame, this) &&
+        (flags & (O_WRONLY | O_RDWR | O_APPEND | O_TRUNC))) {
+        STACK_UNWIND_STRICT(open, frame, -1, EROFS, NULL, NULL);
         return 0;
+    }
+
+    STACK_WIND_TAIL(frame, FIRST_CHILD(this), FIRST_CHILD(this)->fops->open,
+                    loc, flags, fd, xdata);
+    return 0;
 }
 
-int32_t
-worm_open (call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t flags,
-           fd_t *fd, dict_t *xdata)
+static int32_t
+worm_link(call_frame_t *frame, xlator_t *this, loc_t *oldloc, loc_t *newloc,
+          dict_t *xdata)
+{
+    int op_errno = EROFS;
+    read_only_priv_t *priv = NULL;
+
+    priv = this->private;
+    GF_ASSERT(priv);
+    if (is_readonly_or_worm_enabled(frame, this))
+        goto out;
+    if (!priv->worm_file || (frame->root->pid < 0)) {
+        op_errno = 0;
+        goto out;
+    }
+
+    gf_uuid_copy(oldloc->gfid, oldloc->inode->gfid);
+    if (is_wormfile(this, _gf_false, oldloc)) {
+        op_errno = 0;
+        goto out;
+    }
+    op_errno = gf_worm_state_transition(this, _gf_false, oldloc, GF_FOP_LINK);
+
+out:
+    if (op_errno) {
+        if (op_errno < 0)
+            op_errno = EROFS;
+        STACK_UNWIND_STRICT(link, frame, -1, op_errno, NULL, NULL, NULL, NULL,
+                            NULL);
+    } else
+        STACK_WIND_TAIL(frame, FIRST_CHILD(this), FIRST_CHILD(this)->fops->link,
+                        oldloc, newloc, xdata);
+    return 0;
+}
+
+static int32_t
+worm_unlink(call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t flags,
+            dict_t *xdata)
+{
+    int op_errno = EROFS;
+    read_only_priv_t *priv = NULL;
+
+    priv = this->private;
+    GF_ASSERT(priv);
+    if (is_readonly_or_worm_enabled(frame, this)) {
+        goto out;
+    }
+    if (!priv->worm_file || (frame->root->pid < 0)) {
+        op_errno = 0;
+        goto out;
+    }
+
+    gf_uuid_copy(loc->gfid, loc->inode->gfid);
+    if (is_wormfile(this, _gf_false, loc)) {
+        op_errno = 0;
+        goto out;
+    }
+    op_errno = gf_worm_state_transition(this, _gf_false, loc, GF_FOP_UNLINK);
+out:
+    if (op_errno) {
+        if (op_errno < 0)
+            op_errno = EROFS;
+        STACK_UNWIND_STRICT(unlink, frame, -1, op_errno, NULL, NULL, NULL);
+    } else
+        STACK_WIND_TAIL(frame, FIRST_CHILD(this),
+                        FIRST_CHILD(this)->fops->unlink, loc, flags, xdata);
+    return 0;
+}
+
+static int32_t
+worm_rename(call_frame_t *frame, xlator_t *this, loc_t *oldloc, loc_t *newloc,
+            dict_t *xdata)
 {
-        if (is_readonly_or_worm_enabled (this) &&
-            ((((flags & O_ACCMODE) == O_WRONLY) ||
-              ((flags & O_ACCMODE) == O_RDWR)) &&
-              !(flags & O_APPEND))) {
-                STACK_UNWIND_STRICT (open, frame, -1, EROFS, NULL, NULL);
-                return 0;
+    int op_errno = EROFS;
+    read_only_priv_t *priv = NULL;
+
+    priv = this->private;
+    GF_ASSERT(priv);
+    if (is_readonly_or_worm_enabled(frame, this))
+        goto out;
+    if (!priv->worm_file || (frame->root->pid < 0)) {
+        op_errno = 0;
+        goto out;
+    }
+
+    gf_uuid_copy(oldloc->gfid, oldloc->inode->gfid);
+    if (is_wormfile(this, _gf_false, oldloc)) {
+        op_errno = 0;
+        goto check_newloc;
+    }
+    op_errno = gf_worm_state_transition(this, _gf_false, oldloc, GF_FOP_RENAME);
+
+    if (op_errno == 0) {
+    check_newloc:
+        if (newloc->inode != NULL) {
+            gf_uuid_copy(newloc->gfid, newloc->inode->gfid);
+            if (is_wormfile(this, _gf_false, newloc)) {
+                op_errno = 0;
+                goto out;
+            }
+            op_errno = gf_worm_state_transition(this, _gf_false, newloc,
+                                                GF_FOP_RENAME);
         }
+    }
 
-        STACK_WIND (frame, worm_open_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->open, loc, flags, fd, xdata);
-        return 0;
+out:
+    if (op_errno) {
+        if (op_errno < 0)
+            op_errno = EROFS;
+        STACK_UNWIND_STRICT(rename, frame, -1, op_errno, NULL, NULL, NULL, NULL,
+                            NULL, NULL);
+    } else
+        STACK_WIND_TAIL(frame, FIRST_CHILD(this),
+                        FIRST_CHILD(this)->fops->rename, oldloc, newloc, xdata);
+    return 0;
 }
 
-int32_t
-init (xlator_t *this)
+static int32_t
+worm_truncate(call_frame_t *frame, xlator_t *this, loc_t *loc, off_t offset,
+              dict_t *xdata)
 {
-        int                     ret     = -1;
-        read_only_priv_t       *priv    = NULL;
+    int op_errno = EROFS;
+    read_only_priv_t *priv = NULL;
 
-        if (!this->children || this->children->next) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "translator not configured with exactly one child");
-                return -1;
+    priv = this->private;
+    GF_ASSERT(priv);
+    if (is_readonly_or_worm_enabled(frame, this))
+        goto out;
+    if (!priv->worm_file || (frame->root->pid < 0)) {
+        op_errno = 0;
+        goto out;
+    }
+
+    if (is_wormfile(this, _gf_false, loc)) {
+        op_errno = 0;
+        goto out;
+    }
+    op_errno = gf_worm_state_transition(this, _gf_false, loc, GF_FOP_TRUNCATE);
+
+out:
+    if (op_errno) {
+        if (op_errno < 0)
+            op_errno = EROFS;
+        STACK_UNWIND_STRICT(truncate, frame, -1, op_errno, NULL, NULL, NULL);
+    } else
+        STACK_WIND_TAIL(frame, FIRST_CHILD(this),
+                        FIRST_CHILD(this)->fops->truncate, loc, offset, xdata);
+    return 0;
+}
+
+static int32_t
+worm_ftruncate(call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
+               dict_t *xdata)
+{
+    int op_errno = EROFS;
+    read_only_priv_t *priv = NULL;
+
+    priv = this->private;
+    GF_ASSERT(priv);
+    if (is_readonly_or_worm_enabled(frame, this))
+        goto out;
+    if (!priv->worm_file || (frame->root->pid < 0)) {
+        op_errno = 0;
+        goto out;
+    }
+
+    if (is_wormfile(this, _gf_true, fd)) {
+        op_errno = 0;
+        goto out;
+    }
+    op_errno = gf_worm_state_transition(this, _gf_true, fd, GF_FOP_FTRUNCATE);
+
+out:
+    if (op_errno) {
+        if (op_errno < 0)
+            op_errno = EROFS;
+        STACK_UNWIND_STRICT(ftruncate, frame, -1, op_errno, NULL, NULL, NULL);
+    } else
+        STACK_WIND_TAIL(frame, FIRST_CHILD(this),
+                        FIRST_CHILD(this)->fops->ftruncate, fd, offset, xdata);
+    return 0;
+}
+
+static int32_t
+worm_setattr(call_frame_t *frame, xlator_t *this, loc_t *loc,
+             struct iatt *stbuf, int32_t valid, dict_t *xdata)
+{
+    gf_boolean_t rd_only = _gf_false;
+    worm_reten_state_t reten_state = {
+        0,
+    };
+    struct iatt stpre = {
+        0,
+    };
+    read_only_priv_t *priv = NULL;
+    int op_errno = EROFS;
+    int ret = -1;
+
+    priv = this->private;
+    GF_ASSERT(priv);
+    if (!priv->worm_file) {
+        op_errno = 0;
+        goto out;
+    }
+
+    if (is_wormfile(this, _gf_false, loc)) {
+        op_errno = 0;
+        goto out;
+    }
+    if (valid & GF_SET_ATTR_MODE) {
+        rd_only = gf_worm_write_disabled(stbuf);
+        if (!rd_only) {
+            op_errno = 0;
+            goto out;
         }
 
-        if (!this->parents) {
-                gf_log (this->name, GF_LOG_WARNING,
-                        "dangling volume. check volfile ");
+        ret = worm_set_state(this, _gf_false, loc, &reten_state, stbuf);
+        if (ret) {
+            gf_log(this->name, GF_LOG_ERROR, "Error setting worm state");
+            goto out;
+        }
+    } else if (valid & GF_SET_ATTR_ATIME) {
+        ret = worm_get_state(this, _gf_false, loc, &reten_state);
+        if (ret) {
+            op_errno = 0;
+            goto out;
         }
+        if (reten_state.retain) {
+            ret = syncop_stat(this, loc, &stpre, NULL, NULL);
+            if (ret)
+                goto out;
+            if (reten_state.ret_mode == 0) {
+                if (stbuf->ia_atime < stpre.ia_mtime) {
+                    gf_log(this->name, GF_LOG_ERROR,
+                           "Cannot set atime less than "
+                           "the mtime for a WORM-Retained "
+                           "file");
+                    goto out;
+                }
+            } else {
+                if (stbuf->ia_atime < stpre.ia_atime) {
+                    gf_log(this->name, GF_LOG_ERROR,
+                           "Cannot decrease the atime of a"
+                           " WORM-Retained file in "
+                           "Enterprise mode");
+                    goto out;
+                }
+            }
+            reten_state.ret_period = reten_state.ret_period + stbuf->ia_atime -
+                                     stpre.ia_atime;
+            ret = gf_worm_set_xattr(this, &reten_state, _gf_false, loc);
+            if (ret) {
+                goto out;
+            }
+            stbuf->ia_mtime = stpre.ia_mtime;
+        }
+    }
+    op_errno = 0;
+
+out:
+    if (op_errno)
+        STACK_UNWIND_STRICT(setattr, frame, -1, EROFS, NULL, NULL, NULL);
+    else
+        STACK_WIND_TAIL(frame, FIRST_CHILD(this),
+                        FIRST_CHILD(this)->fops->setattr, loc, stbuf, valid,
+                        xdata);
+    return 0;
+}
+
+static int32_t
+worm_fsetattr(call_frame_t *frame, xlator_t *this, fd_t *fd, struct iatt *stbuf,
+              int32_t valid, dict_t *xdata)
+{
+    gf_boolean_t rd_only = _gf_false;
+    worm_reten_state_t reten_state = {
+        0,
+    };
+    struct iatt stpre = {
+        0,
+    };
+    read_only_priv_t *priv = NULL;
+    int op_errno = EROFS;
+    int ret = -1;
 
-        priv = GF_CALLOC (1, sizeof (*priv), gf_read_only_mt_priv_t);
-        if (!priv)
+    priv = this->private;
+    GF_ASSERT(priv);
+    if (!priv->worm_file) {
+        op_errno = 0;
+        goto out;
+    }
+
+    if (is_wormfile(this, _gf_true, fd)) {
+        op_errno = 0;
+        goto out;
+    }
+    if (valid & GF_SET_ATTR_MODE) {
+        rd_only = gf_worm_write_disabled(stbuf);
+        if (!rd_only) {
+            op_errno = 0;
+            goto out;
+        }
+
+        ret = worm_set_state(this, _gf_true, fd, &reten_state, stbuf);
+        if (ret) {
+            gf_log(this->name, GF_LOG_ERROR, "Error setting worm state");
+            goto out;
+        }
+    } else if (valid & GF_SET_ATTR_ATIME) {
+        ret = worm_get_state(this, _gf_true, fd, &reten_state);
+        if (ret) {
+            op_errno = 0;
+            goto out;
+        }
+        if (reten_state.retain) {
+            ret = syncop_fstat(this, fd, &stpre, NULL, NULL);
+            if (ret)
                 goto out;
+            if (reten_state.ret_mode == 0) {
+                if (stbuf->ia_atime < stpre.ia_mtime) {
+                    gf_log(this->name, GF_LOG_ERROR,
+                           "Cannot set atime less than "
+                           "the mtime for a WORM-Retained "
+                           "file");
+                    goto out;
+                }
+            } else {
+                if (stbuf->ia_atime < stpre.ia_atime) {
+                    gf_log(this->name, GF_LOG_ERROR,
+                           "Cannot decrease the atime of a"
+                           " WORM-Retained file in "
+                           "Enterprise mode");
+                    goto out;
+                }
+            }
+            reten_state.ret_period = reten_state.ret_period + stbuf->ia_atime -
+                                     stpre.ia_atime;
+            ret = gf_worm_set_xattr(this, &reten_state, _gf_true, fd);
+            if (ret) {
+                goto out;
+            }
 
-        GF_OPTION_INIT ("worm", priv->readonly_or_worm_enabled, bool, out);
+            stbuf->ia_mtime = stpre.ia_mtime;
+        }
+    }
+    op_errno = 0;
 
-        this->private = priv;
-        ret = 0;
 out:
-        return ret;
+    if (op_errno)
+        STACK_UNWIND_STRICT(fsetattr, frame, -1, op_errno, NULL, NULL, NULL);
+    else
+        STACK_WIND_TAIL(frame, FIRST_CHILD(this),
+                        FIRST_CHILD(this)->fops->fsetattr, fd, stbuf, valid,
+                        xdata);
+    return 0;
 }
 
-int
-reconfigure (xlator_t *this, dict_t *options)
+static int32_t
+worm_writev(call_frame_t *frame, xlator_t *this, fd_t *fd, struct iovec *vector,
+            int32_t count, off_t offset, uint32_t flags, struct iobref *iobref,
+            dict_t *xdata)
 {
-        read_only_priv_t  *priv                     = NULL;
-        int                ret                      = -1;
-        gf_boolean_t       readonly_or_worm_enabled = _gf_false;
+    read_only_priv_t *priv = NULL;
+    int op_errno = EROFS;
 
-        priv = this->private;
-        GF_ASSERT (priv);
+    priv = this->private;
+    GF_ASSERT(priv);
+    if (!priv->worm_file || (frame->root->pid < 0)) {
+        op_errno = 0;
+        goto out;
+    }
+    if (is_wormfile(this, _gf_true, fd)) {
+        op_errno = 0;
+        goto out;
+    }
+    op_errno = gf_worm_state_transition(this, _gf_true, fd, GF_FOP_WRITE);
 
-        GF_OPTION_RECONF ("worm", readonly_or_worm_enabled, options, bool, out);
+out:
+    if (op_errno) {
+        if (op_errno < 0)
+            op_errno = EROFS;
+        STACK_UNWIND_STRICT(writev, frame, -1, op_errno, NULL, NULL, NULL);
+    } else
+        STACK_WIND_TAIL(frame, FIRST_CHILD(this),
+                        FIRST_CHILD(this)->fops->writev, fd, vector, count,
+                        offset, flags, iobref, xdata);
+    return 0;
+}
+
+static int32_t
+worm_create_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                int32_t op_ret, int32_t op_errno, fd_t *fd, inode_t *inode,
+                struct iatt *buf, struct iatt *preparent,
+                struct iatt *postparent, dict_t *xdata)
+{
+    int ret = 0;
+    read_only_priv_t *priv = NULL;
+    // In case of an error exit because fd can be NULL and this would
+    // cause an segfault when performing fsetxattr . We explicitly
+    // unwind to avoid future problems
+    if (op_ret < 0) {
+        goto out;
+    }
+
+    priv = this->private;
+    GF_ASSERT(priv);
+    if (priv->worm_file) {
+        ret = fd_ctx_set(fd, this, 1);
+        if (ret) {
+            gf_log(this->name, GF_LOG_ERROR,
+                   "Failed to set the fd ctx "
+                   "for gfid:%s . Worm feature may not work for the gfid",
+                   uuid_utoa(inode->gfid));
+        }
+        ret = worm_init_state(this, _gf_true, fd);
+        if (ret) {
+            gf_log(this->name, GF_LOG_ERROR, "Error initializing state");
+        }
+    }
 
-        priv->readonly_or_worm_enabled = readonly_or_worm_enabled;
-        ret = 0;
 out:
-        gf_log (this->name, GF_LOG_DEBUG, "returning %d", ret);
-        return ret;
+    STACK_UNWIND_STRICT(create, frame, op_ret, op_errno, fd, inode, buf,
+                        preparent, postparent, xdata);
+    return ret;
 }
 
-void
-fini (xlator_t *this)
+static int32_t
+worm_create(call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t flags,
+            mode_t mode, mode_t umask, fd_t *fd, dict_t *xdata)
+{
+    STACK_WIND(frame, worm_create_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->create, loc, flags, mode, umask, fd,
+               xdata);
+    return 0;
+}
+
+static void
+set_reten_mode(read_only_priv_t *priv, char *reten_mode)
+{
+    if (strcmp(reten_mode, "relax") == 0)
+        priv->reten_mode = 0;
+    else
+        priv->reten_mode = 1;
+}
+
+int32_t
+init(xlator_t *this)
+{
+    int ret = -1;
+    read_only_priv_t *priv = NULL;
+    char *reten_mode = NULL;
+
+    if (!this->children || this->children->next) {
+        gf_log(this->name, GF_LOG_ERROR,
+               "translator not configured with exactly one child");
+        return -1;
+    }
+
+    if (!this->parents) {
+        gf_log(this->name, GF_LOG_WARNING, "dangling volume. check volfile ");
+    }
+
+    this->local_pool = mem_pool_new(read_only_priv_t, 64);
+    if (!this->local_pool) {
+        ret = -1;
+        gf_log(this->name, GF_LOG_ERROR,
+               "failed to create read_only_priv_t's memory pool");
+        goto out;
+    }
+
+    priv = mem_get0(this->local_pool);
+    if (!priv) {
+        gf_log(this->name, GF_LOG_ERROR, "Error allocating priv");
+        goto out;
+    }
+
+    this->private = priv;
+
+    GF_OPTION_INIT("worm", priv->readonly_or_worm_enabled, bool, out);
+    GF_OPTION_INIT("worm-file-level", priv->worm_file, bool, out);
+    GF_OPTION_INIT("default-retention-period", priv->reten_period, int64, out);
+    GF_OPTION_INIT("auto-commit-period", priv->com_period, int64, out);
+    GF_OPTION_INIT("retention-mode", reten_mode, str, out);
+    set_reten_mode(priv, reten_mode);
+    GF_OPTION_INIT("worm-files-deletable", priv->worm_files_deletable, bool,
+                   out);
+
+    ret = 0;
+out:
+    return ret;
+}
+
+int
+reconfigure(xlator_t *this, dict_t *options)
 {
-        read_only_priv_t         *priv    = NULL;
+    read_only_priv_t *priv = NULL;
+    char *reten_mode = NULL;
+    int ret = -1;
 
-        priv = this->private;
-        if (!priv)
-                return;
+    priv = this->private;
+    GF_ASSERT(priv);
 
-        this->private = NULL;
-        GF_FREE (priv);
+    GF_OPTION_RECONF("worm", priv->readonly_or_worm_enabled, options, bool,
+                     out);
+    GF_OPTION_RECONF("worm-file-level", priv->worm_file, options, bool, out);
+    GF_OPTION_RECONF("default-retention-period", priv->reten_period, options,
+                     int64, out);
+    GF_OPTION_RECONF("retention-mode", reten_mode, options, str, out);
+    set_reten_mode(priv, reten_mode);
+    GF_OPTION_RECONF("auto-commit-period", priv->com_period, options, int64,
+                     out);
+    GF_OPTION_RECONF("worm-files-deletable", priv->worm_files_deletable,
+                     options, bool, out);
+    ret = 0;
+out:
+    gf_log(this->name, GF_LOG_DEBUG, "returning %d", ret);
+    return ret;
+}
 
-        return;
+void
+fini(xlator_t *this)
+{
+    read_only_priv_t *priv = NULL;
+
+    priv = this->private;
+    if (!priv)
+        goto out;
+    mem_put(priv);
+    this->private = NULL;
+    mem_pool_destroy(this->local_pool);
+    this->local_pool = NULL;
+out:
+    return;
 }
 
 struct xlator_fops fops = {
-        .open        = worm_open,
-
-        .unlink      = ro_unlink,
-        .rmdir       = ro_rmdir,
-        .rename      = ro_rename,
-        .truncate    = ro_truncate,
-        .removexattr = ro_removexattr,
-        .fsyncdir    = ro_fsyncdir,
-        .xattrop     = ro_xattrop,
-        .inodelk     = ro_inodelk,
-        .finodelk    = ro_finodelk,
-        .entrylk     = ro_entrylk,
-        .fentrylk    = ro_fentrylk,
-        .lk          = ro_lk,
+    .open = worm_open,
+    .writev = worm_writev,
+    .setattr = worm_setattr,
+    .fsetattr = worm_fsetattr,
+    .rename = worm_rename,
+    .link = worm_link,
+    .unlink = worm_unlink,
+    .truncate = worm_truncate,
+    .ftruncate = worm_ftruncate,
+    .create = worm_create,
+
+    .rmdir = ro_rmdir,
+    .removexattr = ro_removexattr,
+    .fsyncdir = ro_fsyncdir,
+    .xattrop = ro_xattrop,
+    .inodelk = ro_inodelk,
+    .finodelk = ro_finodelk,
+    .entrylk = ro_entrylk,
+    .fentrylk = ro_fentrylk,
+    .lk = ro_lk,
 };
 
-struct xlator_cbks cbks;
+int32_t
+worm_release(xlator_t *this, fd_t *fd)
+{
+    dict_t *dict = NULL;
+    int ret = -1;
+    dict = dict_new();
+    uint64_t value = 0;
+    loc_t loc = {
+        0,
+    };
+    read_only_priv_t *priv = NULL;
+    priv = this->private;
+
+    if (priv->worm_file) {
+        if (!dict) {
+            gf_log(this->name, GF_LOG_ERROR, "Error creating the dict");
+            goto out;
+        }
+
+        ret = fd_ctx_get(fd, this, &value);
+        if (ret) {
+            gf_log(this->name, GF_LOG_DEBUG, "Failed to get the fd ctx");
+        }
+        if (!value) {
+            goto out;
+        }
+
+        ret = dict_set_int8(dict, "trusted.worm_file", 1);
+        if (ret) {
+            gf_log(this->name, GF_LOG_ERROR,
+                   "Error in setting "
+                   "the dict");
+            goto out;
+        }
+
+        loc.inode = inode_ref(fd->inode);
+        gf_uuid_copy(loc.gfid, fd->inode->gfid);
+        ret = syncop_setxattr(this, &loc, dict, 0, NULL, NULL);
+        if (ret) {
+            gf_log(this->name, GF_LOG_ERROR, "Error setting xattr");
+            goto out;
+        }
+
+        gf_worm_state_transition(this, _gf_false, &loc, GF_FOP_WRITE);
+    }
+
+out:
+    loc_wipe(&loc);
+    if (dict)
+        dict_unref(dict);
+    return 0;
+}
+
+struct xlator_cbks cbks = {
+    .release = worm_release,
+};
 
 struct volume_options options[] = {
-        { .key  = {"worm"},
-          .type = GF_OPTION_TYPE_BOOL,
-          .default_value = "off",
-          .description = "When \"on\", makes a volume get write once read many "
-                         " feature. It is turned \"off\" by default."
-        },
+    {.key = {"worm"},
+     .type = GF_OPTION_TYPE_BOOL,
+     .default_value = "off",
+     /*.validate_fn = validate_boolean,*/
+     .op_version = {2},
+     .flags = OPT_FLAG_SETTABLE,
+     .description = "When \"on\", makes a volume get write once read many "
+                    " feature. It is turned \"off\" by default."},
+    {.key = {"worm-file-level"},
+     .type = GF_OPTION_TYPE_BOOL,
+     .default_value = "off",
+     /*.validate_fn = validate_boolean,*/
+     .op_version = {GD_OP_VERSION_3_8_0},
+     .flags = OPT_FLAG_SETTABLE,
+     .description = "When \"on\", activates the file level worm. "
+                    "It is turned \"off\" by default."},
+    {.key = {"worm-files-deletable"},
+     .type = GF_OPTION_TYPE_BOOL,
+     .default_value = "on",
+     /*.validate_fn = validate_boolean,*/
+     .op_version = {GD_OP_VERSION_3_13_0},
+     .flags = OPT_FLAG_SETTABLE,
+     .description = "When \"off\", doesn't allow the Worm files"
+                    "to be deleted. It is turned \"on\" by default."},
+    {.key = {"default-retention-period"},
+     .type = GF_OPTION_TYPE_TIME,
+     .default_value = "120",
+     /*.validate_fn = validate_worm_period,*/
+     .op_version = {GD_OP_VERSION_3_8_0},
+     .flags = OPT_FLAG_SETTABLE,
+     .description = "The default retention period for the files."},
+    {.key = {"retention-mode"},
+     .type = GF_OPTION_TYPE_STR,
+     .default_value = "relax",
+     /*.validate_fn = validate_reten_mode,*/
+     .op_version = {GD_OP_VERSION_3_8_0},
+     .flags = OPT_FLAG_SETTABLE,
+     .description = "The mode of retention (relax/enterprise). "
+                    "It is relax by default."},
+    {.key = {"auto-commit-period"},
+     .type = GF_OPTION_TYPE_TIME,
+     .default_value = "180",
+     /*.validate_fn = validate_worm_period,*/
+     .op_version = {GD_OP_VERSION_3_8_0},
+     .flags = OPT_FLAG_SETTABLE,
+     .description = "Auto commit period for the files."},
+    {.key = {NULL}},
 };
 
+xlator_api_t xlator_api = {
+    .init = init,
+    .fini = fini,
+    .reconfigure = reconfigure,
+    .mem_acct_init = mem_acct_init,
+    .op_version = {1}, /* Present from the initial version */
+    .fops = &fops,
+    .cbks = &cbks,
+    .options = options,
+    .identifier = "worm",
+    .category = GF_TECH_PREVIEW,
+};
diff --git a/xlators/features/sdfs/Makefile.am b/xlators/features/sdfs/Makefile.am
new file mode 100644
index 00000000000..a985f42a877
--- /dev/null
+++ b/xlators/features/sdfs/Makefile.am
@@ -0,0 +1,3 @@
+SUBDIRS = src
+
+CLEANFILES =
diff --git a/xlators/features/sdfs/src/Makefile.am b/xlators/features/sdfs/src/Makefile.am
new file mode 100644
index 00000000000..6118d46ad22
--- /dev/null
+++ b/xlators/features/sdfs/src/Makefile.am
@@ -0,0 +1,19 @@
+if WITH_SERVER
+xlator_LTLIBRARIES = sdfs.la
+endif
+xlatordir = $(libdir)/glusterfs/$(PACKAGE_VERSION)/xlator/features
+
+sdfs_la_LDFLAGS = -module $(GF_XLATOR_DEFAULT_LDFLAGS)
+
+sdfs_la_SOURCES = sdfs.c
+sdfs_la_LIBADD = $(top_builddir)/libglusterfs/src/libglusterfs.la
+
+noinst_HEADERS = sdfs.h sdfs-messages.h $(top_builddir)/xlators/lib/src/libxlator.h
+
+AM_CPPFLAGS = $(GF_CPPFLAGS) -I$(top_srcdir)/libglusterfs/src \
+                                 -I$(top_srcdir)/xlators/lib/src \
+        -I$(top_srcdir)/rpc/xdr/src/ -I$(top_builddir)/rpc/xdr/src/
+
+AM_CFLAGS = -Wall -fno-strict-aliasing $(GF_CFLAGS)
+
+CLEANFILES =
diff --git a/xlators/features/sdfs/src/sdfs-messages.h b/xlators/features/sdfs/src/sdfs-messages.h
new file mode 100644
index 00000000000..3053efa8935
--- /dev/null
+++ b/xlators/features/sdfs/src/sdfs-messages.h
@@ -0,0 +1,67 @@
+/*
+ Copyright (c) 2016 Red Hat, Inc. <http://www.redhat.com>
+ This file is part of GlusterFS.
+
+ This file is licensed to you under your choice of the GNU Lesser
+ General Public License, version 3 or any later version (LGPLv3 or
+ later), or the GNU General Public License, version 2 (GPLv2), in all
+ cases as published by the Free Software Foundation.
+ */
+
+#ifndef _DFS_MESSAGES_H_
+#define _DFS_MESSAGES_H_
+
+#include <glusterfs/glfs-message-id.h>
+
+/* file bit-rot-bitd-messages.h
+ * brief SDFS log-message IDs and their descriptions
+ */
+
+/* NOTE: Rules for message additions
+ * 1) Each instance of a message is _better_ left with a unique message ID, even
+ *    if the message format is the same. Reasoning is that, if the message
+ *    format needs to change in one instance, the other instances are not
+ *    impacted or the new change does not change the ID of the instance being
+ *    modified.
+ * 2) Addition of a message,
+ *       - Should increment the GLFS_NUM_MESSAGES
+ *       - Append to the list of messages defined, towards the end
+ *       - Retain macro naming as glfs_msg_X (for redability across developers)
+ * NOTE: Rules for message format modifications
+ * 3) Check acorss the code if the message ID macro in question is reused
+ *    anywhere. If reused then then the modifications should ensure correctness
+ *    everywhere, or needs a new message ID as (1) above was not adhered to. If
+ *    not used anywhere, proceed with the required modification.
+ * NOTE: Rules for message deletion
+ * 4) Check (3) and if used anywhere else, then cannot be deleted. If not used
+ *    anywhere, then can be deleted, but will leave a hole by design, as
+ *    addition rules specify modification to the end of the list and not filling
+ *    holes.
+ */
+
+#define GLFS_SDFS_BASE GLFS_MSGID_COMP_SDFS
+#define GLFS_SDFS_NUM_MESSAGES 2
+#define GLFS_MSGID_END (GLFS_SDFS_BASE + GLFS_SDFS_NUM_MESSAGES + 1)
+/* Messaged with message IDs */
+#define glfs_msg_start_x GLFS_DFS_BASE, "Invalid: Start of messages"
+/*------------*/
+
+#define SDFS_MSG_ENTRYLK_ERROR (GLFS_SDFS_BASE + 1)
+/*!
+ * @messageid
+ * @diagnosis
+ * @recommendedaction
+ *
+ */
+
+#define SDFS_MSG_MKDIR_ERROR (GLFS_SDFS_BASE + 2)
+/*!
+ * @messageid
+ * @diagnosis
+ * @recommendedaction
+ *
+ */
+/*------------*/
+
+#define glfs_msg_end_x GLFS_MSGID_END, "Invalid: End of messages"
+#endif /* !_SDFS_MESSAGES_H_ */
diff --git a/xlators/features/sdfs/src/sdfs.c b/xlators/features/sdfs/src/sdfs.c
new file mode 100644
index 00000000000..aaf13f0852e
--- /dev/null
+++ b/xlators/features/sdfs/src/sdfs.c
@@ -0,0 +1,1479 @@
+/*
+   Copyright (c) 2016 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
+
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
+#include <libgen.h>
+#include "sdfs.h"
+
+static int
+sdfs_frame_return(call_frame_t *frame)
+{
+    sdfs_local_t *local = NULL;
+
+    if (!frame)
+        return -1;
+
+    local = frame->local;
+
+    return GF_ATOMIC_DEC(local->call_cnt);
+}
+
+static void
+sdfs_lock_free(sdfs_entry_lock_t *entrylk)
+{
+    if (entrylk == NULL)
+        goto out;
+
+    loc_wipe(&entrylk->parent_loc);
+    GF_FREE(entrylk->basename);
+
+out:
+    return;
+}
+
+static void
+sdfs_lock_array_free(sdfs_lock_t *lock)
+{
+    sdfs_entry_lock_t *entrylk = NULL;
+    int i = 0;
+
+    if (lock == NULL)
+        goto out;
+
+    for (i = 0; i < lock->lock_count; i++) {
+        entrylk = &lock->entrylk[i];
+        sdfs_lock_free(entrylk);
+    }
+
+out:
+    return;
+}
+
+static void
+sdfs_local_cleanup(sdfs_local_t *local)
+{
+    if (!local)
+        return;
+
+    loc_wipe(&local->loc);
+    loc_wipe(&local->parent_loc);
+
+    if (local->stub) {
+        call_stub_destroy(local->stub);
+        local->stub = NULL;
+    }
+
+    sdfs_lock_array_free(local->lock);
+    GF_FREE(local->lock);
+
+    mem_put(local);
+}
+
+static int
+sdfs_build_parent_loc(loc_t *parent, loc_t *child)
+{
+    int ret = -1;
+    char *path = NULL;
+
+    if (!child->parent) {
+        goto out;
+    }
+    parent->inode = inode_ref(child->parent);
+    path = gf_strdup(child->path);
+    if (!path) {
+        ret = -ENOMEM;
+        goto out;
+    }
+
+    parent->path = dirname(path);
+    if (!parent->path) {
+        goto out;
+    }
+
+    gf_uuid_copy(parent->gfid, child->pargfid);
+    return 0;
+
+out:
+    GF_FREE(path);
+    return ret;
+}
+
+static sdfs_local_t *
+sdfs_local_init(call_frame_t *frame, xlator_t *this)
+{
+    sdfs_local_t *local = NULL;
+
+    local = mem_get0(this->local_pool);
+    if (!local)
+        goto out;
+
+    frame->local = local;
+out:
+    return local;
+}
+
+static int
+sdfs_get_new_frame_common(call_frame_t *frame, call_frame_t **new_frame)
+{
+    int ret = -1;
+    sdfs_local_t *local = NULL;
+    client_t *client = NULL;
+
+    *new_frame = copy_frame(frame);
+    if (!*new_frame) {
+        goto err;
+    }
+
+    client = frame->root->client;
+    gf_client_ref(client);
+    (*new_frame)->root->client = client;
+
+    local = sdfs_local_init(*new_frame, THIS);
+    if (!local) {
+        goto err;
+    }
+
+    local->main_frame = frame;
+    /*Set unique lk-owner for the fop*/
+    set_lk_owner_from_ptr(&(*new_frame)->root->lk_owner, (*new_frame)->root);
+
+    ret = 0;
+err:
+    if ((ret == -1) && (*new_frame)) {
+        SDFS_STACK_DESTROY((*new_frame));
+        *new_frame = NULL;
+    }
+
+    return ret;
+}
+
+static int
+sdfs_get_new_frame(call_frame_t *frame, loc_t *loc, call_frame_t **new_frame)
+{
+    int ret = -1;
+    sdfs_local_t *local = NULL;
+
+    ret = sdfs_get_new_frame_common(frame, new_frame);
+    if (ret < 0) {
+        goto err;
+    }
+
+    local = (*new_frame)->local;
+
+    ret = sdfs_build_parent_loc(&local->parent_loc, loc);
+    if (ret) {
+        goto err;
+    }
+
+    ret = loc_copy(&local->loc, loc);
+    if (ret == -1) {
+        goto err;
+    }
+
+    ret = 0;
+err:
+    if (ret && (*new_frame)) {
+        SDFS_STACK_DESTROY((*new_frame));
+        *new_frame = NULL;
+        ret = -1;
+    }
+
+    return ret;
+}
+
+static int
+sdfs_get_new_frame_readdirp(call_frame_t *frame, fd_t *fd,
+                            call_frame_t **new_frame)
+{
+    int ret = -1;
+    sdfs_local_t *local = NULL;
+
+    ret = sdfs_get_new_frame_common(frame, new_frame);
+    if (ret < 0) {
+        goto err;
+    }
+
+    local = (*new_frame)->local;
+    local->parent_loc.inode = inode_ref(fd->inode);
+    gf_uuid_copy(local->parent_loc.gfid, fd->inode->gfid);
+
+    ret = 0;
+err:
+    return ret;
+}
+
+int
+sdfs_entrylk_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                 int32_t op_ret, int32_t op_errno, dict_t *xdata)
+{
+    sdfs_local_t *local = NULL;
+    call_stub_t *stub = NULL;
+
+    local = frame->local;
+
+    local->op_ret = op_ret;
+    local->op_errno = op_errno;
+
+    if (local->stub) {
+        stub = local->stub;
+        local->stub = NULL;
+        call_resume(stub);
+    } else {
+        if (op_ret < 0)
+            gf_msg(this->name, GF_LOG_ERROR, 0, SDFS_MSG_ENTRYLK_ERROR,
+                   "Unlocking entry lock failed for %s", local->loc.name);
+
+        SDFS_STACK_DESTROY(frame);
+    }
+
+    return 0;
+}
+
+int
+sdfs_mkdir_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+               int32_t op_ret, int32_t op_errno, inode_t *inode,
+               struct iatt *stbuf, struct iatt *preparent,
+               struct iatt *postparent, dict_t *xdata)
+{
+    sdfs_local_t *local = NULL;
+
+    local = frame->local;
+
+    STACK_UNWIND_STRICT(mkdir, local->main_frame, op_ret, op_errno, inode,
+                        stbuf, preparent, postparent, xdata);
+
+    local->main_frame = NULL;
+    STACK_WIND(frame, sdfs_entrylk_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->entrylk, this->name, &local->parent_loc,
+               local->loc.name, ENTRYLK_UNLOCK, ENTRYLK_WRLCK, xdata);
+    return 0;
+}
+
+int
+sdfs_mkdir_helper(call_frame_t *frame, xlator_t *this, loc_t *loc, mode_t mode,
+                  mode_t umask, dict_t *xdata)
+{
+    sdfs_local_t *local = NULL;
+    char gfid[GF_UUID_BUF_SIZE] = {0};
+    int op_errno = -1;
+
+    local = frame->local;
+
+    gf_uuid_unparse(loc->pargfid, gfid);
+
+    if (local->op_ret < 0) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, SDFS_MSG_ENTRYLK_ERROR,
+               "Acquiring entry lock failed for directory %s "
+               "with parent gfid %s",
+               local->loc.name, gfid);
+        op_errno = local->op_errno;
+        goto err;
+    }
+
+    STACK_WIND(frame, sdfs_mkdir_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->mkdir, loc, mode, umask, xdata);
+
+    return 0;
+err:
+    STACK_UNWIND_STRICT(mkdir, local->main_frame, -1, op_errno, NULL, NULL,
+                        NULL, NULL, NULL);
+
+    local->main_frame = NULL;
+    SDFS_STACK_DESTROY(frame);
+    return 0;
+}
+
+int
+sdfs_mkdir(call_frame_t *frame, xlator_t *this, loc_t *loc, mode_t mode,
+           mode_t umask, dict_t *xdata)
+{
+    sdfs_local_t *local = NULL;
+    call_frame_t *new_frame = NULL;
+    call_stub_t *stub = NULL;
+    int op_errno = 0;
+
+    if (-1 == sdfs_get_new_frame(frame, loc, &new_frame)) {
+        op_errno = ENOMEM;
+        goto err;
+    }
+
+    stub = fop_mkdir_stub(new_frame, sdfs_mkdir_helper, loc, mode, umask,
+                          xdata);
+    if (!stub) {
+        op_errno = ENOMEM;
+        goto err;
+    }
+
+    local = new_frame->local;
+    local->stub = stub;
+
+    STACK_WIND(new_frame, sdfs_entrylk_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->entrylk, this->name, &local->parent_loc,
+               local->loc.name, ENTRYLK_LOCK, ENTRYLK_WRLCK, xdata);
+
+    return 0;
+err:
+    STACK_UNWIND_STRICT(mkdir, frame, -1, op_errno, NULL, NULL, NULL, NULL,
+                        NULL);
+
+    if (new_frame)
+        SDFS_STACK_DESTROY(new_frame);
+
+    return 0;
+}
+
+int
+sdfs_rmdir_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+               int32_t op_ret, int32_t op_errno, struct iatt *preparent,
+               struct iatt *postparent, dict_t *xdata)
+{
+    sdfs_local_t *local = NULL;
+
+    local = frame->local;
+
+    STACK_UNWIND_STRICT(rmdir, local->main_frame, op_ret, op_errno, preparent,
+                        postparent, xdata);
+
+    local->main_frame = NULL;
+    STACK_WIND(frame, sdfs_entrylk_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->entrylk, this->name, &local->parent_loc,
+               local->loc.name, ENTRYLK_UNLOCK, ENTRYLK_WRLCK, xdata);
+    return 0;
+}
+
+int
+sdfs_rmdir_helper(call_frame_t *frame, xlator_t *this, loc_t *loc, int flags,
+                  dict_t *xdata)
+{
+    sdfs_local_t *local = NULL;
+    char gfid[GF_UUID_BUF_SIZE] = {0};
+
+    local = frame->local;
+
+    gf_uuid_unparse(loc->pargfid, gfid);
+
+    if (local->op_ret < 0) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, SDFS_MSG_ENTRYLK_ERROR,
+               "Acquiring entry lock failed for directory %s "
+               "with parent gfid %s",
+               local->loc.name, gfid);
+        goto err;
+    }
+
+    STACK_WIND(frame, sdfs_rmdir_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->rmdir, loc, flags, xdata);
+
+    return 0;
+err:
+    STACK_UNWIND_STRICT(rmdir, local->main_frame, -1, local->op_errno, NULL,
+                        NULL, NULL);
+
+    local->main_frame = NULL;
+    SDFS_STACK_DESTROY(frame);
+    return 0;
+}
+
+int
+sdfs_rmdir(call_frame_t *frame, xlator_t *this, loc_t *loc, int flags,
+           dict_t *xdata)
+{
+    sdfs_local_t *local = NULL;
+    call_frame_t *new_frame = NULL;
+    call_stub_t *stub = NULL;
+    int op_errno = 0;
+
+    if (-1 == sdfs_get_new_frame(frame, loc, &new_frame)) {
+        op_errno = ENOMEM;
+        goto err;
+    }
+
+    stub = fop_rmdir_stub(new_frame, sdfs_rmdir_helper, loc, flags, xdata);
+    if (!stub) {
+        op_errno = ENOMEM;
+        goto err;
+    }
+
+    local = new_frame->local;
+    local->stub = stub;
+
+    STACK_WIND(new_frame, sdfs_entrylk_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->entrylk, this->name, &local->parent_loc,
+               local->loc.name, ENTRYLK_LOCK, ENTRYLK_WRLCK, xdata);
+
+    return 0;
+err:
+    STACK_UNWIND_STRICT(rmdir, frame, -1, op_errno, NULL, NULL, NULL);
+
+    if (new_frame)
+        SDFS_STACK_DESTROY(new_frame);
+
+    return 0;
+}
+
+int
+sdfs_create_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                int32_t op_ret, int32_t op_errno, fd_t *fd, inode_t *inode,
+                struct iatt *stbuf, struct iatt *preparent,
+                struct iatt *postparent, dict_t *xdata)
+{
+    sdfs_local_t *local = NULL;
+
+    local = frame->local;
+
+    STACK_UNWIND_STRICT(create, local->main_frame, op_ret, op_errno, fd, inode,
+                        stbuf, preparent, postparent, xdata);
+
+    local->main_frame = NULL;
+    STACK_WIND(frame, sdfs_entrylk_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->entrylk, this->name, &local->parent_loc,
+               local->loc.name, ENTRYLK_UNLOCK, ENTRYLK_WRLCK, xdata);
+    return 0;
+}
+
+int
+sdfs_create_helper(call_frame_t *frame, xlator_t *this, loc_t *loc,
+                   int32_t flags, mode_t mode, mode_t umask, fd_t *fd,
+                   dict_t *xdata)
+{
+    sdfs_local_t *local = NULL;
+    char gfid[GF_UUID_BUF_SIZE] = {0};
+
+    local = frame->local;
+
+    gf_uuid_unparse(loc->pargfid, gfid);
+
+    if (local->op_ret < 0) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, SDFS_MSG_ENTRYLK_ERROR,
+               "Acquiring entry lock failed for directory %s "
+               "with parent gfid %s",
+               local->loc.name, gfid);
+        goto err;
+    }
+
+    STACK_WIND(frame, sdfs_create_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->create, loc, flags, mode, umask, fd,
+               xdata);
+
+    return 0;
+err:
+    STACK_UNWIND_STRICT(create, local->main_frame, -1, local->op_errno, NULL,
+                        NULL, NULL, NULL, NULL, NULL);
+
+    local->main_frame = NULL;
+    SDFS_STACK_DESTROY(frame);
+    return 0;
+}
+
+int
+sdfs_create(call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t flags,
+            mode_t mode, mode_t umask, fd_t *fd, dict_t *xdata)
+{
+    sdfs_local_t *local = NULL;
+    call_frame_t *new_frame = NULL;
+    call_stub_t *stub = NULL;
+    int op_errno = 0;
+
+    if (-1 == sdfs_get_new_frame(frame, loc, &new_frame)) {
+        op_errno = ENOMEM;
+        goto err;
+    }
+
+    stub = fop_create_stub(new_frame, sdfs_create_helper, loc, flags, mode,
+                           umask, fd, xdata);
+    if (!stub) {
+        op_errno = ENOMEM;
+        goto err;
+    }
+
+    local = new_frame->local;
+    local->stub = stub;
+
+    STACK_WIND(new_frame, sdfs_entrylk_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->entrylk, this->name, &local->parent_loc,
+               local->loc.name, ENTRYLK_LOCK, ENTRYLK_WRLCK, xdata);
+
+    return 0;
+err:
+    STACK_UNWIND_STRICT(create, frame, -1, op_errno, NULL, NULL, NULL, NULL,
+                        NULL, NULL);
+
+    if (new_frame)
+        SDFS_STACK_DESTROY(new_frame);
+
+    return 0;
+}
+
+int
+sdfs_unlink_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                int32_t op_ret, int32_t op_errno, struct iatt *preparent,
+                struct iatt *postparent, dict_t *xdata)
+{
+    sdfs_local_t *local = NULL;
+
+    local = frame->local;
+
+    STACK_UNWIND_STRICT(unlink, local->main_frame, op_ret, op_errno, preparent,
+                        postparent, xdata);
+
+    local->main_frame = NULL;
+    STACK_WIND(frame, sdfs_entrylk_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->entrylk, this->name, &local->parent_loc,
+               local->loc.name, ENTRYLK_UNLOCK, ENTRYLK_WRLCK, xdata);
+    return 0;
+}
+
+int
+sdfs_unlink_helper(call_frame_t *frame, xlator_t *this, loc_t *loc, int flags,
+                   dict_t *xdata)
+{
+    sdfs_local_t *local = NULL;
+    char gfid[GF_UUID_BUF_SIZE] = {0};
+
+    local = frame->local;
+
+    gf_uuid_unparse(loc->pargfid, gfid);
+
+    if (local->op_ret < 0) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, SDFS_MSG_ENTRYLK_ERROR,
+               "Acquiring entry lock failed for directory %s "
+               "with parent gfid %s",
+               local->loc.name, gfid);
+        goto err;
+    }
+
+    STACK_WIND(frame, sdfs_unlink_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->unlink, loc, flags, xdata);
+
+    return 0;
+err:
+    STACK_UNWIND_STRICT(unlink, local->main_frame, -1, local->op_errno, NULL,
+                        NULL, NULL);
+
+    local->main_frame = NULL;
+    SDFS_STACK_DESTROY(frame);
+    return 0;
+}
+
+int
+sdfs_unlink(call_frame_t *frame, xlator_t *this, loc_t *loc, int flags,
+            dict_t *xdata)
+{
+    sdfs_local_t *local = NULL;
+    call_frame_t *new_frame = NULL;
+    call_stub_t *stub = NULL;
+    int op_errno = 0;
+
+    if (-1 == sdfs_get_new_frame(frame, loc, &new_frame)) {
+        op_errno = ENOMEM;
+        goto err;
+    }
+
+    stub = fop_unlink_stub(new_frame, sdfs_unlink_helper, loc, flags, xdata);
+    if (!stub) {
+        op_errno = ENOMEM;
+        goto err;
+    }
+
+    local = new_frame->local;
+    local->stub = stub;
+
+    STACK_WIND(new_frame, sdfs_entrylk_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->entrylk, this->name, &local->parent_loc,
+               local->loc.name, ENTRYLK_LOCK, ENTRYLK_WRLCK, xdata);
+
+    return 0;
+err:
+    STACK_UNWIND_STRICT(unlink, frame, -1, op_errno, NULL, NULL, NULL);
+
+    if (new_frame)
+        SDFS_STACK_DESTROY(new_frame);
+
+    return 0;
+}
+
+int
+sdfs_symlink_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                 int32_t op_ret, int32_t op_errno, inode_t *inode,
+                 struct iatt *stbuf, struct iatt *preparent,
+                 struct iatt *postparent, dict_t *xdata)
+{
+    sdfs_local_t *local = NULL;
+
+    local = frame->local;
+
+    STACK_UNWIND_STRICT(link, local->main_frame, op_ret, op_errno, inode, stbuf,
+                        preparent, postparent, xdata);
+
+    local->main_frame = NULL;
+    STACK_WIND(frame, sdfs_entrylk_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->entrylk, this->name, &local->parent_loc,
+               local->loc.name, ENTRYLK_UNLOCK, ENTRYLK_WRLCK, xdata);
+    return 0;
+}
+
+int
+sdfs_symlink_helper(call_frame_t *frame, xlator_t *this, const char *linkname,
+                    loc_t *loc, mode_t umask, dict_t *xdata)
+{
+    sdfs_local_t *local = NULL;
+    char gfid[GF_UUID_BUF_SIZE] = {0};
+
+    local = frame->local;
+
+    gf_uuid_unparse(loc->pargfid, gfid);
+
+    if (local->op_ret < 0) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, SDFS_MSG_ENTRYLK_ERROR,
+               "Acquiring entry lock failed for directory %s "
+               "with parent gfid %s",
+               local->loc.name, gfid);
+        goto err;
+    }
+
+    STACK_WIND(frame, sdfs_symlink_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->symlink, linkname, loc, umask, xdata);
+
+    return 0;
+err:
+    STACK_UNWIND_STRICT(link, local->main_frame, -1, local->op_errno, NULL,
+                        NULL, NULL, NULL, NULL);
+
+    local->main_frame = NULL;
+    SDFS_STACK_DESTROY(frame);
+    return 0;
+}
+
+int
+sdfs_symlink(call_frame_t *frame, xlator_t *this, const char *linkname,
+             loc_t *loc, mode_t umask, dict_t *xdata)
+{
+    sdfs_local_t *local = NULL;
+    call_frame_t *new_frame = NULL;
+    call_stub_t *stub = NULL;
+    int op_errno = 0;
+
+    if (-1 == sdfs_get_new_frame(frame, loc, &new_frame)) {
+        op_errno = ENOMEM;
+        goto err;
+    }
+
+    stub = fop_symlink_stub(new_frame, sdfs_symlink_helper, linkname, loc,
+                            umask, xdata);
+    if (!stub) {
+        op_errno = ENOMEM;
+        goto err;
+    }
+
+    local = new_frame->local;
+    local->stub = stub;
+
+    STACK_WIND(new_frame, sdfs_entrylk_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->entrylk, this->name, &local->parent_loc,
+               local->loc.name, ENTRYLK_LOCK, ENTRYLK_WRLCK, xdata);
+
+    return 0;
+err:
+    STACK_UNWIND_STRICT(link, frame, -1, op_errno, NULL, NULL, NULL, NULL,
+                        NULL);
+
+    if (new_frame)
+        SDFS_STACK_DESTROY(new_frame);
+
+    return 0;
+}
+
+int
+sdfs_common_entrylk_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                        int32_t op_ret, int32_t op_errno, dict_t *xdata)
+{
+    sdfs_local_t *local = NULL;
+    int this_call_cnt = 0;
+    int lk_index = 0;
+    sdfs_lock_t *locks = NULL;
+    call_stub_t *stub = NULL;
+
+    local = frame->local;
+    locks = local->lock;
+    lk_index = (long)cookie;
+
+    if (op_ret < 0) {
+        local->op_ret = op_ret;
+        local->op_errno = op_errno;
+    } else {
+        locks->entrylk->locked[lk_index] = _gf_true;
+    }
+
+    this_call_cnt = sdfs_frame_return(frame);
+    if (this_call_cnt > 0) {
+        gf_log(this->name, GF_LOG_DEBUG,
+               "As there are more callcnt (%d) returning without WIND",
+               this_call_cnt);
+        return 0;
+    }
+
+    if (local->stub) {
+        stub = local->stub;
+        local->stub = NULL;
+        call_resume(stub);
+    } else {
+        if (local->op_ret < 0)
+            gf_msg(this->name, GF_LOG_ERROR, 0, SDFS_MSG_ENTRYLK_ERROR,
+                   "unlocking entry lock failed ");
+        SDFS_STACK_DESTROY(frame);
+    }
+
+    return 0;
+}
+
+int
+sdfs_link_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int32_t op_ret,
+              int32_t op_errno, inode_t *inode, struct iatt *stbuf,
+              struct iatt *preparent, struct iatt *postparent, dict_t *xdata)
+{
+    sdfs_local_t *local = NULL;
+    sdfs_lock_t *lock = NULL;
+    int i = 0;
+    int lock_count = 0;
+
+    local = frame->local;
+    lock = local->lock;
+
+    STACK_UNWIND_STRICT(link, local->main_frame, op_ret, op_errno, inode, stbuf,
+                        preparent, postparent, xdata);
+
+    local->main_frame = NULL;
+    lock_count = lock->lock_count;
+    for (i = 0; i < lock_count; i++) {
+        STACK_WIND_COOKIE(frame, sdfs_common_entrylk_cbk, (void *)(long)i,
+                          FIRST_CHILD(this), FIRST_CHILD(this)->fops->entrylk,
+                          this->name, &lock->entrylk[i].parent_loc,
+                          lock->entrylk[i].basename, ENTRYLK_UNLOCK,
+                          ENTRYLK_WRLCK, xdata);
+    }
+
+    return 0;
+}
+
+int
+sdfs_link_helper(call_frame_t *frame, xlator_t *this, loc_t *oldloc,
+                 loc_t *newloc, dict_t *xdata)
+{
+    sdfs_local_t *local = NULL;
+    sdfs_lock_t *locks = NULL;
+    gf_boolean_t stack_destroy = _gf_true;
+    int lock_count = 0;
+    int i = 0;
+
+    local = frame->local;
+    locks = local->lock;
+
+    if (local->op_ret < 0) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, SDFS_MSG_ENTRYLK_ERROR,
+               "Acquiring entry lock failed");
+        goto err;
+    }
+
+    STACK_WIND(frame, sdfs_link_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->link, oldloc, newloc, xdata);
+
+    return 0;
+err:
+    STACK_UNWIND_STRICT(link, local->main_frame, -1, local->op_errno, NULL,
+                        NULL, NULL, NULL, NULL);
+
+    local->main_frame = NULL;
+    for (i = 0; i < locks->lock_count && locks->entrylk->locked[i]; i++) {
+        lock_count++;
+    }
+    GF_ATOMIC_INIT(local->call_cnt, lock_count);
+
+    for (i = 0; i < lock_count; i++) {
+        if (!locks->entrylk->locked[i]) {
+            lock_count++;
+            continue;
+        }
+
+        stack_destroy = _gf_false;
+        STACK_WIND(frame, sdfs_common_entrylk_cbk, FIRST_CHILD(this),
+                   FIRST_CHILD(this)->fops->entrylk, this->name,
+                   &locks->entrylk[i].parent_loc, locks->entrylk[i].basename,
+                   ENTRYLK_UNLOCK, ENTRYLK_WRLCK, xdata);
+    }
+
+    if (stack_destroy)
+        SDFS_STACK_DESTROY(frame);
+
+    return 0;
+}
+
+static int
+sdfs_init_entry_lock(sdfs_entry_lock_t *lock, loc_t *loc)
+{
+    int ret = 0;
+
+    ret = sdfs_build_parent_loc(&lock->parent_loc, loc);
+    if (ret)
+        return -1;
+
+    lock->basename = gf_strdup(loc->name);
+    if (!lock->basename)
+        return -1;
+
+    return 0;
+}
+
+int
+sdfs_entry_lock_cmp(const void *l1, const void *l2)
+{
+    const sdfs_entry_lock_t *r1 = l1;
+    const sdfs_entry_lock_t *r2 = l2;
+    int ret = 0;
+    uuid_t gfid1 = {0};
+    uuid_t gfid2 = {0};
+
+    loc_gfid((loc_t *)&r1->parent_loc, gfid1);
+    loc_gfid((loc_t *)&r2->parent_loc, gfid2);
+    ret = gf_uuid_compare(gfid1, gfid2);
+    /*Entrylks with NULL basename are the 'smallest'*/
+    if (ret == 0) {
+        if (!r1->basename)
+            return -1;
+        if (!r2->basename)
+            return 1;
+        ret = strcmp(r1->basename, r2->basename);
+    }
+
+    if (ret <= 0)
+        return -1;
+    else
+        return 1;
+}
+
+int
+sdfs_link(call_frame_t *frame, xlator_t *this, loc_t *oldloc, loc_t *newloc,
+          dict_t *xdata)
+{
+    sdfs_local_t *local = NULL;
+    call_frame_t *new_frame = NULL;
+    call_stub_t *stub = NULL;
+    sdfs_lock_t *lock = NULL;
+    client_t *client = NULL;
+    int ret = 0;
+    int op_errno = ENOMEM;
+
+    new_frame = copy_frame(frame);
+    if (!new_frame) {
+        op_errno = ENOMEM;
+        goto err;
+    }
+    /*Set unique lk-owner for the fop*/
+    set_lk_owner_from_ptr(&new_frame->root->lk_owner, new_frame->root);
+
+    gf_client_ref(client);
+    new_frame->root->client = client;
+    local = sdfs_local_init(new_frame, this);
+    if (!local) {
+        op_errno = ENOMEM;
+        goto err;
+    }
+
+    local->main_frame = frame;
+
+    lock = GF_CALLOC(1, sizeof(*lock), gf_common_mt_char);
+    if (!lock)
+        goto err;
+
+    local->lock = lock;
+
+    ret = sdfs_init_entry_lock(&lock->entrylk[0], newloc);
+    if (ret)
+        goto err;
+
+    ++lock->lock_count;
+
+    local->lock = lock;
+    GF_ATOMIC_INIT(local->call_cnt, lock->lock_count);
+
+    ret = loc_copy(&local->loc, newloc);
+    if (ret == -1) {
+        op_errno = ENOMEM;
+        goto err;
+    }
+
+    stub = fop_link_stub(new_frame, sdfs_link_helper, oldloc, newloc, xdata);
+    if (!stub) {
+        op_errno = ENOMEM;
+        goto err;
+    }
+
+    local->stub = stub;
+
+    STACK_WIND_COOKIE(new_frame, sdfs_common_entrylk_cbk, 0, FIRST_CHILD(this),
+                      FIRST_CHILD(this)->fops->entrylk, this->name,
+                      &lock->entrylk[0].parent_loc, lock->entrylk[0].basename,
+                      ENTRYLK_LOCK, ENTRYLK_WRLCK, xdata);
+
+    return 0;
+err:
+
+    STACK_UNWIND_STRICT(link, frame, -1, op_errno, NULL, NULL, NULL, NULL,
+                        NULL);
+
+    if (new_frame)
+        SDFS_STACK_DESTROY(new_frame);
+
+    return 0;
+}
+
+int
+sdfs_mknod_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+               int32_t op_ret, int32_t op_errno, inode_t *inode,
+               struct iatt *stbuf, struct iatt *preparent,
+               struct iatt *postparent, dict_t *xdata)
+{
+    sdfs_local_t *local = NULL;
+
+    local = frame->local;
+
+    STACK_UNWIND_STRICT(mknod, local->main_frame, op_ret, op_errno, inode,
+                        stbuf, preparent, postparent, xdata);
+
+    local->main_frame = NULL;
+    STACK_WIND(frame, sdfs_entrylk_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->entrylk, this->name, &local->parent_loc,
+               local->loc.name, ENTRYLK_UNLOCK, ENTRYLK_WRLCK, xdata);
+    return 0;
+}
+
+int
+sdfs_mknod_helper(call_frame_t *frame, xlator_t *this, loc_t *loc, mode_t mode,
+                  dev_t rdev, mode_t umask, dict_t *xdata)
+{
+    sdfs_local_t *local = NULL;
+    char gfid[GF_UUID_BUF_SIZE] = {0};
+
+    local = frame->local;
+
+    gf_uuid_unparse(loc->pargfid, gfid);
+
+    if (local->op_ret < 0) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, SDFS_MSG_ENTRYLK_ERROR,
+               "Acquiring entry lock failed for directory %s "
+               "with parent gfid %s",
+               local->loc.name, gfid);
+        goto err;
+    }
+
+    STACK_WIND(frame, sdfs_mknod_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->mknod, loc, mode, rdev, umask, xdata);
+
+    return 0;
+err:
+    STACK_UNWIND_STRICT(mknod, local->main_frame, -1, local->op_errno, NULL,
+                        NULL, NULL, NULL, NULL);
+
+    local->main_frame = NULL;
+    SDFS_STACK_DESTROY(frame);
+    return 0;
+}
+
+int
+sdfs_mknod(call_frame_t *frame, xlator_t *this, loc_t *loc, mode_t mode,
+           dev_t rdev, mode_t umask, dict_t *xdata)
+{
+    sdfs_local_t *local = NULL;
+    call_frame_t *new_frame = NULL;
+    call_stub_t *stub = NULL;
+    int op_errno = 0;
+
+    if (-1 == sdfs_get_new_frame(frame, loc, &new_frame)) {
+        op_errno = ENOMEM;
+        goto err;
+    }
+
+    stub = fop_mknod_stub(new_frame, sdfs_mknod_helper, loc, mode, rdev, umask,
+                          xdata);
+    if (!stub) {
+        op_errno = ENOMEM;
+        goto err;
+    }
+
+    local = new_frame->local;
+    local->stub = stub;
+
+    STACK_WIND(new_frame, sdfs_entrylk_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->entrylk, this->name, &local->parent_loc,
+               local->loc.name, ENTRYLK_LOCK, ENTRYLK_WRLCK, xdata);
+
+    return 0;
+err:
+    STACK_UNWIND_STRICT(mknod, frame, -1, op_errno, NULL, NULL, NULL, NULL,
+                        NULL);
+
+    if (new_frame)
+        SDFS_STACK_DESTROY(new_frame);
+
+    return 0;
+}
+
+int
+sdfs_rename_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                int32_t op_ret, int32_t op_errno, struct iatt *stbuf,
+                struct iatt *preoldparent, struct iatt *postoldparent,
+                struct iatt *prenewparent, struct iatt *postnewparent,
+                dict_t *xdata)
+{
+    sdfs_local_t *local = NULL;
+    sdfs_lock_t *lock = NULL;
+    int i = 0;
+    int call_cnt = 0;
+
+    local = frame->local;
+    lock = local->lock;
+    GF_ATOMIC_INIT(local->call_cnt, lock->lock_count);
+
+    STACK_UNWIND_STRICT(rename, local->main_frame, op_ret, op_errno, stbuf,
+                        preoldparent, postoldparent, prenewparent,
+                        postnewparent, xdata);
+
+    local->main_frame = NULL;
+    call_cnt = GF_ATOMIC_GET(local->call_cnt);
+
+    for (i = 0; i < call_cnt; i++) {
+        STACK_WIND_COOKIE(frame, sdfs_common_entrylk_cbk, (void *)(long)i,
+                          FIRST_CHILD(this), FIRST_CHILD(this)->fops->entrylk,
+                          this->name, &lock->entrylk[i].parent_loc,
+                          lock->entrylk[i].basename, ENTRYLK_UNLOCK,
+                          ENTRYLK_WRLCK, xdata);
+    }
+
+    return 0;
+}
+
+int
+sdfs_rename_helper(call_frame_t *frame, xlator_t *this, loc_t *oldloc,
+                   loc_t *newloc, dict_t *xdata)
+{
+    sdfs_local_t *local = NULL;
+    sdfs_lock_t *lock = NULL;
+    gf_boolean_t stack_destroy = _gf_true;
+    int lock_count = 0;
+    int i = 0;
+
+    local = frame->local;
+    lock = local->lock;
+
+    if (local->op_ret < 0) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, SDFS_MSG_ENTRYLK_ERROR,
+               "Acquiring entry lock failed ");
+        goto err;
+    }
+
+    STACK_WIND(frame, sdfs_rename_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->rename, oldloc, newloc, xdata);
+
+    return 0;
+
+err:
+    STACK_UNWIND_STRICT(rename, local->main_frame, -1, local->op_errno, NULL,
+                        NULL, NULL, NULL, NULL, NULL);
+
+    local->main_frame = NULL;
+    for (i = 0; i < lock->lock_count && lock->entrylk->locked[i]; i++) {
+        lock_count++;
+    }
+    GF_ATOMIC_INIT(local->call_cnt, lock_count);
+
+    for (i = 0; i < lock_count; i++) {
+        if (!lock->entrylk->locked[i]) {
+            lock_count++;
+            continue;
+        }
+        stack_destroy = _gf_false;
+        STACK_WIND(frame, sdfs_common_entrylk_cbk, FIRST_CHILD(this),
+                   FIRST_CHILD(this)->fops->entrylk, this->name,
+                   &lock->entrylk[i].parent_loc, lock->entrylk[i].basename,
+                   ENTRYLK_UNLOCK, ENTRYLK_WRLCK, xdata);
+    }
+
+    if (stack_destroy)
+        SDFS_STACK_DESTROY(frame);
+
+    return 0;
+}
+
+int
+sdfs_rename(call_frame_t *frame, xlator_t *this, loc_t *oldloc, loc_t *newloc,
+            dict_t *xdata)
+{
+    sdfs_local_t *local = NULL;
+    sdfs_lock_t *lock = NULL;
+    call_frame_t *new_frame = NULL;
+    call_stub_t *stub = NULL;
+    client_t *client = NULL;
+    int ret = 0;
+    int op_errno = ENOMEM;
+    int i = 0;
+    int call_cnt = 0;
+
+    new_frame = copy_frame(frame);
+    if (!new_frame) {
+        op_errno = ENOMEM;
+        goto err;
+    }
+    /*Set unique lk-owner for the fop*/
+    set_lk_owner_from_ptr(&new_frame->root->lk_owner, new_frame->root);
+
+    gf_client_ref(client);
+    new_frame->root->client = client;
+    local = sdfs_local_init(new_frame, this);
+    if (!local) {
+        op_errno = ENOMEM;
+        goto err;
+    }
+
+    local->main_frame = frame;
+
+    lock = GF_CALLOC(1, sizeof(*lock), gf_common_mt_char);
+    if (!lock)
+        goto err;
+
+    local->lock = lock;
+
+    ret = sdfs_init_entry_lock(&lock->entrylk[0], oldloc);
+    if (ret)
+        goto err;
+    lock->entrylk->locked[0] = _gf_false;
+
+    ++lock->lock_count;
+
+    ret = sdfs_init_entry_lock(&lock->entrylk[1], newloc);
+    if (ret)
+        goto err;
+    lock->entrylk->locked[1] = _gf_false;
+
+    ++lock->lock_count;
+
+    qsort(lock->entrylk, lock->lock_count, sizeof(*lock->entrylk),
+          sdfs_entry_lock_cmp);
+
+    local->lock = lock;
+    GF_ATOMIC_INIT(local->call_cnt, lock->lock_count);
+
+    stub = fop_rename_stub(new_frame, sdfs_rename_helper, oldloc, newloc,
+                           xdata);
+    if (!stub) {
+        op_errno = ENOMEM;
+        goto err;
+    }
+
+    local->stub = stub;
+    call_cnt = GF_ATOMIC_GET(local->call_cnt);
+    for (i = 0; i < call_cnt; i++) {
+        STACK_WIND_COOKIE(new_frame, sdfs_common_entrylk_cbk, (void *)(long)i,
+                          FIRST_CHILD(this), FIRST_CHILD(this)->fops->entrylk,
+                          this->name, &lock->entrylk[i].parent_loc,
+                          lock->entrylk[i].basename, ENTRYLK_LOCK,
+                          ENTRYLK_WRLCK, xdata);
+    }
+
+    return 0;
+err:
+
+    STACK_UNWIND_STRICT(rename, frame, -1, op_errno, NULL, NULL, NULL, NULL,
+                        NULL, NULL);
+
+    if (new_frame)
+        SDFS_STACK_DESTROY(new_frame);
+
+    return 0;
+}
+
+int
+sdfs_lookup_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                int32_t op_ret, int32_t op_errno, inode_t *inode,
+                struct iatt *stbuf, dict_t *xdata, struct iatt *postparent)
+{
+    sdfs_local_t *local = NULL;
+
+    local = frame->local;
+
+    if (!local->loc.parent) {
+        sdfs_local_cleanup(local);
+        frame->local = NULL;
+        STACK_UNWIND_STRICT(lookup, frame, op_ret, op_errno, inode, stbuf,
+                            xdata, postparent);
+        return 0;
+    }
+
+    STACK_UNWIND_STRICT(lookup, local->main_frame, op_ret, op_errno, inode,
+                        stbuf, xdata, postparent);
+
+    local->main_frame = NULL;
+    STACK_WIND(frame, sdfs_entrylk_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->entrylk, this->name, &local->parent_loc,
+               local->loc.name, ENTRYLK_UNLOCK, ENTRYLK_RDLCK, xdata);
+    return 0;
+}
+
+int
+sdfs_lookup_helper(call_frame_t *frame, xlator_t *this, loc_t *loc,
+                   dict_t *xdata)
+{
+    sdfs_local_t *local = NULL;
+    char gfid[GF_UUID_BUF_SIZE] = {0};
+
+    local = frame->local;
+
+    gf_uuid_unparse(loc->pargfid, gfid);
+
+    if (local->op_ret < 0) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, SDFS_MSG_ENTRYLK_ERROR,
+               "Acquiring entry lock failed for directory %s "
+               "with parent gfid %s",
+               local->loc.name, gfid);
+        goto err;
+    }
+
+    STACK_WIND(frame, sdfs_lookup_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->lookup, loc, xdata);
+
+    return 0;
+err:
+    STACK_UNWIND_STRICT(lookup, local->main_frame, -1, local->op_errno, NULL,
+                        NULL, NULL, NULL);
+    local->main_frame = NULL;
+
+    SDFS_STACK_DESTROY(frame);
+    return 0;
+}
+
+int
+sdfs_lookup(call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xdata)
+{
+    sdfs_local_t *local = NULL;
+    call_frame_t *new_frame = NULL;
+    call_stub_t *stub = NULL;
+    int op_errno = 0;
+
+    if (!loc->parent) {
+        local = sdfs_local_init(frame, this);
+        if (!local) {
+            op_errno = ENOMEM;
+            goto err;
+        }
+
+        STACK_WIND_TAIL(frame, FIRST_CHILD(this),
+                        FIRST_CHILD(this)->fops->lookup, loc, xdata);
+        return 0;
+    }
+
+    if (-1 == sdfs_get_new_frame(frame, loc, &new_frame)) {
+        op_errno = ENOMEM;
+        goto err;
+    }
+
+    stub = fop_lookup_stub(new_frame, sdfs_lookup_helper, loc, xdata);
+    if (!stub) {
+        op_errno = ENOMEM;
+        goto err;
+    }
+
+    local = new_frame->local;
+    local->stub = stub;
+
+    STACK_WIND(new_frame, sdfs_entrylk_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->entrylk, this->name, &local->parent_loc,
+               local->loc.name, ENTRYLK_LOCK, ENTRYLK_RDLCK, xdata);
+
+    return 0;
+
+err:
+    STACK_UNWIND_STRICT(lookup, frame, -1, op_errno, NULL, NULL, NULL, NULL);
+
+    if (new_frame)
+        SDFS_STACK_DESTROY(new_frame);
+
+    return 0;
+}
+
+int32_t
+sdfs_readdirp_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                  int32_t op_ret, int32_t op_errno, gf_dirent_t *entries,
+                  dict_t *xdata)
+{
+    sdfs_local_t *local = NULL;
+
+    local = frame->local;
+    STACK_UNWIND_STRICT(readdirp, local->main_frame, op_ret, op_errno, entries,
+                        xdata);
+
+    local->main_frame = NULL;
+    STACK_WIND(frame, sdfs_entrylk_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->entrylk, this->name, &local->parent_loc,
+               NULL, ENTRYLK_UNLOCK, ENTRYLK_RDLCK, xdata);
+    return 0;
+}
+
+int32_t
+sdfs_readdirp_helper(call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
+                     off_t off, dict_t *xdata)
+{
+    sdfs_local_t *local = NULL;
+    char gfid[GF_UUID_BUF_SIZE] = {0};
+
+    local = frame->local;
+
+    gf_uuid_unparse(fd->inode->gfid, gfid);
+
+    if (local->op_ret < 0) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, SDFS_MSG_ENTRYLK_ERROR,
+               "Acquiring entry lock failed for directory %s "
+               "with parent gfid %s",
+               local->loc.name, gfid);
+        goto err;
+    }
+
+    STACK_WIND(frame, sdfs_readdirp_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->readdirp, fd, size, off, xdata);
+
+    return 0;
+err:
+    STACK_UNWIND_STRICT(readdirp, local->main_frame, -1, local->op_errno, NULL,
+                        NULL);
+
+    local->main_frame = NULL;
+
+    SDFS_STACK_DESTROY(frame);
+    return 0;
+}
+
+int32_t
+sdfs_readdirp(call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
+              off_t off, dict_t *xdata)
+{
+    sdfs_local_t *local = NULL;
+    call_frame_t *new_frame = NULL;
+    call_stub_t *stub = NULL;
+    int op_errno = 0;
+
+    if (-1 == sdfs_get_new_frame_readdirp(frame, fd, &new_frame)) {
+        op_errno = ENOMEM;
+        goto err;
+    }
+
+    stub = fop_readdirp_stub(new_frame, sdfs_readdirp_helper, fd, size, off,
+                             xdata);
+    if (!stub) {
+        op_errno = ENOMEM;
+        goto err;
+    }
+
+    local = new_frame->local;
+    local->stub = stub;
+
+    STACK_WIND(new_frame, sdfs_entrylk_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->entrylk, this->name, &local->parent_loc,
+               NULL, ENTRYLK_LOCK, ENTRYLK_RDLCK, xdata);
+
+    return 0;
+
+err:
+    STACK_UNWIND_STRICT(readdirp, frame, -1, op_errno, NULL, NULL);
+
+    if (new_frame)
+        SDFS_STACK_DESTROY(new_frame);
+
+    return 0;
+}
+
+int
+init(xlator_t *this)
+{
+    int ret = -1;
+
+    if (!this->children || this->children->next) {
+        gf_log(this->name, GF_LOG_ERROR,
+               "'dentry-fop-serializer' not configured with exactly one child");
+        goto out;
+    }
+
+    if (!this->parents) {
+        gf_log(this->name, GF_LOG_WARNING, "dangling volume. check volfile ");
+    }
+
+    this->local_pool = mem_pool_new(sdfs_local_t, 512);
+    if (!this->local_pool) {
+        goto out;
+    }
+
+    GF_OPTION_INIT("pass-through", this->pass_through, bool, out);
+
+    ret = 0;
+
+out:
+    return ret;
+}
+
+int
+reconfigure(xlator_t *this, dict_t *options)
+{
+    int ret = -1;
+
+    GF_OPTION_RECONF("pass-through", this->pass_through, options, bool, out);
+
+    ret = 0;
+out:
+    return ret;
+}
+
+void
+fini(xlator_t *this)
+{
+    mem_pool_destroy(this->local_pool);
+    this->local_pool = NULL;
+    return;
+}
+
+struct xlator_fops fops = {
+    .mkdir = sdfs_mkdir,
+    .rmdir = sdfs_rmdir,
+    .create = sdfs_create,
+    .unlink = sdfs_unlink,
+    .symlink = sdfs_symlink,
+    .link = sdfs_link,
+    .mknod = sdfs_mknod,
+    .rename = sdfs_rename,
+    .lookup = sdfs_lookup,
+    .readdirp = sdfs_readdirp,
+};
+
+struct xlator_cbks cbks;
+
+struct volume_options options[] = {
+    {.key = {"pass-through"},
+     .type = GF_OPTION_TYPE_BOOL,
+     .default_value = "true",
+     .op_version = {GD_OP_VERSION_4_1_0},
+     .flags = OPT_FLAG_SETTABLE | OPT_FLAG_DOC | OPT_FLAG_CLIENT_OPT,
+     .tags = {"sdfs"},
+     .description = "Enable/Disable dentry serialize functionality"},
+    {.key = {NULL}},
+};
+
+xlator_api_t xlator_api = {
+    .init = init,
+    .fini = fini,
+    .reconfigure = reconfigure,
+    .op_version = {GD_OP_VERSION_4_0_0},
+    .fops = &fops,
+    .cbks = &cbks,
+    .options = options,
+    .identifier = "sdfs",
+    .category = GF_TECH_PREVIEW,
+};
diff --git a/xlators/features/sdfs/src/sdfs.h b/xlators/features/sdfs/src/sdfs.h
new file mode 100644
index 00000000000..dded5a2d7fc
--- /dev/null
+++ b/xlators/features/sdfs/src/sdfs.h
@@ -0,0 +1,49 @@
+/*
+   Copyright (c) 2016 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
+
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
+
+#include <glusterfs/xlator.h>
+#include <glusterfs/call-stub.h>
+#include "sdfs-messages.h"
+#include <glusterfs/atomic.h>
+
+#define SDFS_LOCK_COUNT_MAX 2
+
+typedef struct {
+    loc_t parent_loc;
+    char *basename;
+    int locked[SDFS_LOCK_COUNT_MAX];
+} sdfs_entry_lock_t;
+
+typedef struct {
+    sdfs_entry_lock_t entrylk[SDFS_LOCK_COUNT_MAX];
+    int lock_count;
+} sdfs_lock_t;
+
+struct sdfs_local {
+    call_frame_t *main_frame;
+    loc_t loc;
+    loc_t parent_loc;
+    call_stub_t *stub;
+    sdfs_lock_t *lock;
+    int op_ret;
+    int op_errno;
+    gf_atomic_t call_cnt;
+};
+typedef struct sdfs_local sdfs_local_t;
+
+#define SDFS_STACK_DESTROY(frame)                                              \
+    do {                                                                       \
+        sdfs_local_t *__local = NULL;                                          \
+        __local = frame->local;                                                \
+        frame->local = NULL;                                                   \
+        gf_client_unref(frame->root->client);                                  \
+        STACK_DESTROY(frame->root);                                            \
+        sdfs_local_cleanup(__local);                                           \
+    } while (0)
diff --git a/xlators/features/selinux/Makefile.am b/xlators/features/selinux/Makefile.am
new file mode 100644
index 00000000000..a985f42a877
--- /dev/null
+++ b/xlators/features/selinux/Makefile.am
@@ -0,0 +1,3 @@
+SUBDIRS = src
+
+CLEANFILES =
diff --git a/xlators/features/selinux/src/Makefile.am b/xlators/features/selinux/src/Makefile.am
new file mode 100644
index 00000000000..4f1e5e149b3
--- /dev/null
+++ b/xlators/features/selinux/src/Makefile.am
@@ -0,0 +1,20 @@
+if WITH_SERVER
+xlator_LTLIBRARIES = selinux.la
+endif
+xlatordir = $(libdir)/glusterfs/$(PACKAGE_VERSION)/xlator/features
+
+selinux_la_LDFLAGS = -module $(GF_XLATOR_DEFAULT_LDFLAGS)
+
+selinux_la_SOURCES = selinux.c
+
+selinux_la_LIBADD = $(top_builddir)/libglusterfs/src/libglusterfs.la
+
+noinst_HEADERS = selinux.h selinux-messages.h selinux-mem-types.h
+
+AM_CPPFLAGS = $(GF_CPPFLAGS) -I$(top_srcdir)/libglusterfs/src \
+	-I$(top_srcdir)/rpc/xdr/src -I$(top_builddir)/rpc/xdr/src
+
+AM_CFLAGS = -Wall $(GF_CFLAGS)
+
+CLEANFILES =
+
diff --git a/xlators/features/filter/src/filter-mem-types.h b/xlators/features/selinux/src/selinux-mem-types.h
index 47a17249b8d..553e59e5a9d 100644
--- a/xlators/features/filter/src/filter-mem-types.h
+++ b/xlators/features/selinux/src/selinux-mem-types.h
@@ -1,5 +1,5 @@
 /*
-   Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
+   Copyright (c) 2017 Red Hat, Inc. <http://www.redhat.com>
    This file is part of GlusterFS.
 
    This file is licensed to you under your choice of the GNU Lesser
@@ -7,14 +7,13 @@
    later), or the GNU General Public License, version 2 (GPLv2), in all
    cases as published by the Free Software Foundation.
 */
-#ifndef __FILTER_MEM_TYPES_H__
-#define __FILTER_MEM_TYPES_H__
+#ifndef __SELINUX_MEM_TYPES_H__
+#define __SELINUX_MEM_TYPES_H__
 
-#include "mem-types.h"
+#include <glusterfs/mem-types.h>
 
-enum gf_filter_mem_types_ {
-        gf_filter_mt_gf_filter = gf_common_mt_end + 1,
-        gf_filter_mt_end
+enum gf_selinux_mem_types_ {
+    gf_selinux_mt_selinux_priv_t = gf_common_mt_end + 1,
+    gf_selinux_mt_end
 };
 #endif
-
diff --git a/xlators/features/selinux/src/selinux-messages.h b/xlators/features/selinux/src/selinux-messages.h
new file mode 100644
index 00000000000..f49a54f956c
--- /dev/null
+++ b/xlators/features/selinux/src/selinux-messages.h
@@ -0,0 +1,30 @@
+/*
+  Copyright (c) 2017 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
+*/
+
+#ifndef _SELINUX_MESSAGES_H__
+#define _SELINUX_MESSAGES_H__
+
+#include <glusterfs/glfs-message-id.h>
+
+/* To add new message IDs, append new identifiers at the end of the list.
+ *
+ * Never remove a message ID. If it's not used anymore, you can rename it or
+ * leave it as it is, but not delete it. This is to prevent reutilization of
+ * IDs by other messages.
+ *
+ * The component name must match one of the entries defined in
+ * glfs-message-id.h.
+ */
+
+GLFS_MSGID(SL, SL_MSG_INVALID_VOLFILE, SL_MSG_ENOMEM,
+           SL_MSG_MEM_ACCT_INIT_FAILED, SL_MSG_SELINUX_GLUSTER_XATTR_MISSING,
+           SL_MSG_SELINUX_XATTR_MISSING);
+
+#endif /*_SELINUX_MESSAGES_H */
diff --git a/xlators/features/selinux/src/selinux.c b/xlators/features/selinux/src/selinux.c
new file mode 100644
index 00000000000..9b1b4b55e1a
--- /dev/null
+++ b/xlators/features/selinux/src/selinux.c
@@ -0,0 +1,323 @@
+/*
+   Copyright (c) 2017 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
+
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
+
+#include <glusterfs/xlator.h>
+
+#include "selinux.h"
+#include "selinux-messages.h"
+#include "selinux-mem-types.h"
+#include <glusterfs/compat-errno.h>
+
+static int
+selinux_fgetxattr_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                      int op_ret, int op_errno, dict_t *dict, dict_t *xdata)
+{
+    int ret = 0;
+    char *name = cookie;
+
+    if (op_errno == 0 && dict && name &&
+        (!strcmp(name, SELINUX_GLUSTER_XATTR))) {
+        ret = dict_rename_key(dict, SELINUX_GLUSTER_XATTR, SELINUX_XATTR);
+        if (ret < 0)
+            gf_msg(this->name, GF_LOG_ERROR, op_errno,
+                   SL_MSG_SELINUX_GLUSTER_XATTR_MISSING,
+                   "getxattr failed for %s", SELINUX_XATTR);
+    }
+
+    STACK_UNWIND_STRICT(fgetxattr, frame, op_ret, op_errno, dict, xdata);
+    return ret;
+}
+
+static int
+selinux_fgetxattr(call_frame_t *frame, xlator_t *this, fd_t *fd,
+                  const char *name, dict_t *xdata)
+{
+    selinux_priv_t *priv = NULL;
+    int32_t op_ret = -1;
+    int32_t op_errno = EINVAL;
+    char *xattr_name = (char *)name;
+
+    priv = this->private;
+
+    GF_VALIDATE_OR_GOTO("selinux", priv, err);
+
+    /* name can be NULL for listxattr calls */
+    if (!priv->selinux_enabled || !name)
+        goto off;
+
+    if (strcmp(name, SELINUX_XATTR) == 0)
+        xattr_name = SELINUX_GLUSTER_XATTR;
+
+off:
+    STACK_WIND_COOKIE(frame, selinux_fgetxattr_cbk, xattr_name,
+                      FIRST_CHILD(this), FIRST_CHILD(this)->fops->fgetxattr, fd,
+                      xattr_name, xdata);
+    return 0;
+err:
+    STACK_UNWIND_STRICT(fgetxattr, frame, op_ret, op_errno, NULL, xdata);
+
+    return 0;
+}
+
+static int
+selinux_getxattr_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                     int op_ret, int op_errno, dict_t *dict, dict_t *xdata)
+{
+    int ret = 0;
+    char *name = cookie;
+
+    if (op_errno == 0 && dict && name &&
+        (!strcmp(name, SELINUX_GLUSTER_XATTR))) {
+        ret = dict_rename_key(dict, SELINUX_GLUSTER_XATTR, SELINUX_XATTR);
+        if (ret < 0)
+            gf_msg(this->name, GF_LOG_ERROR, op_errno,
+                   SL_MSG_SELINUX_GLUSTER_XATTR_MISSING,
+                   "getxattr failed for %s", SELINUX_XATTR);
+    }
+
+    STACK_UNWIND_STRICT(getxattr, frame, op_ret, op_errno, dict, xdata);
+
+    return 0;
+}
+
+static int
+selinux_getxattr(call_frame_t *frame, xlator_t *this, loc_t *loc,
+                 const char *name, dict_t *xdata)
+{
+    selinux_priv_t *priv = NULL;
+    int32_t op_ret = -1;
+    int32_t op_errno = EINVAL;
+    char *xattr_name = (char *)name;
+
+    priv = this->private;
+
+    GF_VALIDATE_OR_GOTO("selinux", priv, err);
+
+    /* name can be NULL for listxattr calls */
+    if (!priv->selinux_enabled || !name)
+        goto off;
+
+    if (strcmp(name, SELINUX_XATTR) == 0)
+        xattr_name = SELINUX_GLUSTER_XATTR;
+
+off:
+    STACK_WIND_COOKIE(frame, selinux_getxattr_cbk, xattr_name,
+                      FIRST_CHILD(this), FIRST_CHILD(this)->fops->getxattr, loc,
+                      xattr_name, xdata);
+    return 0;
+err:
+    STACK_UNWIND_STRICT(getxattr, frame, op_ret, op_errno, NULL, xdata);
+    return 0;
+}
+
+static int
+selinux_fsetxattr_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                      int op_ret, int op_errno, dict_t *xdata)
+{
+    STACK_UNWIND_STRICT(fsetxattr, frame, op_ret, op_errno, xdata);
+    return 0;
+}
+
+static int
+selinux_fsetxattr(call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *dict,
+                  int flags, dict_t *xdata)
+{
+    selinux_priv_t *priv = NULL;
+    int32_t op_ret = -1;
+    int32_t op_errno = EINVAL;
+    int32_t ret = -1;
+
+    priv = this->private;
+
+    GF_VALIDATE_OR_GOTO("selinux", priv, err);
+
+    if (!priv->selinux_enabled && !dict)
+        goto off;
+
+    ret = dict_rename_key(dict, SELINUX_XATTR, SELINUX_GLUSTER_XATTR);
+    if (ret < 0 && ret != -ENODATA)
+        goto err;
+
+off:
+    STACK_WIND(frame, selinux_fsetxattr_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->fsetxattr, fd, dict, flags, xdata);
+
+    return 0;
+err:
+    STACK_UNWIND_STRICT(fsetxattr, frame, op_ret, op_errno, xdata);
+    return 0;
+}
+
+static int
+selinux_setxattr_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                     int op_ret, int op_errno, dict_t *xdata)
+{
+    STACK_UNWIND_STRICT(setxattr, frame, op_ret, op_errno, xdata);
+    return 0;
+}
+
+static int
+selinux_setxattr(call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *dict,
+                 int flags, dict_t *xdata)
+{
+    selinux_priv_t *priv = NULL;
+    int32_t op_ret = -1;
+    int32_t op_errno = EINVAL;
+    int32_t ret = -1;
+
+    priv = this->private;
+
+    GF_VALIDATE_OR_GOTO("selinux", priv, err);
+
+    if (!priv->selinux_enabled && !dict)
+        goto off;
+
+    ret = dict_rename_key(dict, SELINUX_XATTR, SELINUX_GLUSTER_XATTR);
+    if (ret < 0 && ret != -ENODATA)
+        goto err;
+
+off:
+    STACK_WIND(frame, selinux_setxattr_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->setxattr, loc, dict, flags, xdata);
+    return 0;
+err:
+    STACK_UNWIND_STRICT(setxattr, frame, op_ret, op_errno, xdata);
+    return 0;
+}
+
+int32_t
+mem_acct_init(xlator_t *this)
+{
+    int ret = -1;
+
+    GF_VALIDATE_OR_GOTO("selinux", this, out);
+
+    ret = xlator_mem_acct_init(this, gf_selinux_mt_end + 1);
+
+    if (ret != 0) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, SL_MSG_MEM_ACCT_INIT_FAILED,
+               "Memory accounting init failed");
+        return ret;
+    }
+out:
+    return ret;
+}
+
+int32_t
+init(xlator_t *this)
+{
+    int32_t ret = -1;
+    selinux_priv_t *priv = NULL;
+
+    GF_VALIDATE_OR_GOTO("selinux", this, out);
+
+    if (!this->children || this->children->next) {
+        gf_msg(this->name, GF_LOG_WARNING, 0, SL_MSG_INVALID_VOLFILE,
+               "Error: SELinux (%s) not configured with exactly one "
+               "child",
+               this->name);
+        return -1;
+    }
+
+    if (this->parents == NULL) {
+        gf_msg(this->name, GF_LOG_WARNING, 0, SL_MSG_INVALID_VOLFILE,
+               "Dangling volume. Please check the volfile");
+    }
+
+    priv = GF_CALLOC(1, sizeof(*priv), gf_selinux_mt_selinux_priv_t);
+    if (!priv) {
+        gf_log(this->name, GF_LOG_ERROR, "out of memory");
+        goto out;
+    }
+
+    GF_OPTION_INIT("selinux", priv->selinux_enabled, bool, out);
+
+    this->local_pool = mem_pool_new(selinux_priv_t, 64);
+    if (!this->local_pool) {
+        gf_msg(this->name, GF_LOG_ERROR, ENOMEM, SL_MSG_ENOMEM,
+               "Failed to create local_t's memory pool");
+        goto out;
+    }
+
+    this->private = (void *)priv;
+    ret = 0;
+out:
+    if (ret) {
+        GF_FREE(priv);
+        mem_pool_destroy(this->local_pool);
+        this->local_pool = NULL;
+    }
+    return ret;
+}
+
+int
+reconfigure(xlator_t *this, dict_t *options)
+{
+    int32_t ret = -1;
+    selinux_priv_t *priv = NULL;
+
+    priv = this->private;
+
+    GF_OPTION_RECONF("selinux", priv->selinux_enabled, options, bool, out);
+
+    ret = 0;
+out:
+    return ret;
+}
+
+void
+fini(xlator_t *this)
+{
+    selinux_priv_t *priv = NULL;
+
+    priv = this->private;
+    GF_FREE(priv);
+
+    mem_pool_destroy(this->local_pool);
+    this->local_pool = NULL;
+
+    return;
+}
+
+struct xlator_fops fops = {
+    .getxattr = selinux_getxattr,
+    .fgetxattr = selinux_fgetxattr,
+    .setxattr = selinux_setxattr,
+    .fsetxattr = selinux_fsetxattr,
+};
+
+struct xlator_cbks cbks = {};
+
+struct volume_options options[] = {
+    {
+        .key = {"selinux"},
+        .type = GF_OPTION_TYPE_BOOL,
+        .default_value = "on",
+        .description = "Enable/disable selinux translator",
+        .op_version = {GD_OP_VERSION_3_11_0},
+        .flags = OPT_FLAG_SETTABLE,
+        .tags = {"security", "linux"},
+    },
+    {
+        .key = {NULL},
+    }};
+
+xlator_api_t xlator_api = {
+    .init = init,
+    .fini = fini,
+    .reconfigure = reconfigure,
+    .mem_acct_init = mem_acct_init,
+    .op_version = {1}, /* Present from the initial version */
+    .fops = &fops,
+    .cbks = &cbks,
+    .options = options,
+    .identifier = "selinux",
+    .category = GF_MAINTAINED,
+};
diff --git a/xlators/features/selinux/src/selinux.h b/xlators/features/selinux/src/selinux.h
new file mode 100644
index 00000000000..1bbdad3bb36
--- /dev/null
+++ b/xlators/features/selinux/src/selinux.h
@@ -0,0 +1,24 @@
+/*
+   Copyright (c) 2017 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
+
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
+#ifndef __SELINUX_H__
+#define __SELINUX_H__
+
+#include <glusterfs/common-utils.h>
+
+#define SELINUX_XATTR "security.selinux"
+#define SELINUX_GLUSTER_XATTR "trusted.glusterfs.selinux"
+
+struct selinux_priv {
+    gf_boolean_t selinux_enabled;
+};
+
+typedef struct selinux_priv selinux_priv_t;
+
+#endif
diff --git a/xlators/features/shard/src/Makefile.am b/xlators/features/shard/src/Makefile.am
index b589cafe924..bf5700d4bcc 100644
--- a/xlators/features/shard/src/Makefile.am
+++ b/xlators/features/shard/src/Makefile.am
@@ -1,15 +1,16 @@
 xlator_LTLIBRARIES = shard.la
 xlatordir = $(libdir)/glusterfs/$(PACKAGE_VERSION)/xlator/features
 
-shard_la_LDFLAGS = -module -avoid-version
+shard_la_LDFLAGS = -module $(GF_XLATOR_DEFAULT_LDFLAGS)
 
 shard_la_SOURCES = shard.c
 
 shard_la_LIBADD = $(top_builddir)/libglusterfs/src/libglusterfs.la
 
-noinst_HEADERS = shard.h shard-mem-types.h
+noinst_HEADERS = shard.h shard-mem-types.h shard-messages.h
 
-AM_CPPFLAGS = $(GF_CPPFLAGS) -I$(top_srcdir)/libglusterfs/src
+AM_CPPFLAGS = $(GF_CPPFLAGS) -I$(top_srcdir)/libglusterfs/src \
+	-I$(top_srcdir)/rpc/xdr/src -I$(top_builddir)/rpc/xdr/src
 
 AM_CFLAGS = -Wall $(GF_CFLAGS)
 
diff --git a/xlators/features/shard/src/shard-mem-types.h b/xlators/features/shard/src/shard-mem-types.h
index 5a043278b7d..1fe7e2e2798 100644
--- a/xlators/features/shard/src/shard-mem-types.h
+++ b/xlators/features/shard/src/shard-mem-types.h
@@ -10,14 +10,15 @@
 #ifndef __SHARD_MEM_TYPES_H__
 #define __SHARD_MEM_TYPES_H__
 
-#include "mem-types.h"
+#include <glusterfs/mem-types.h>
 
 enum gf_shard_mem_types_ {
-        gf_shard_mt_priv_t = gf_common_mt_end + 1,
-        gf_shard_mt_inode_list,
-        gf_shard_mt_inode_ctx_t,
-        gf_shard_mt_iovec,
-        gf_shard_mt_uint64_t,
-        gf_shard_mt_end
+    gf_shard_mt_priv_t = gf_common_mt_end + 1,
+    gf_shard_mt_inode_list,
+    gf_shard_mt_inode_ctx_t,
+    gf_shard_mt_iovec,
+    gf_shard_mt_int64_t,
+    gf_shard_mt_uint64_t,
+    gf_shard_mt_end
 };
 #endif
diff --git a/xlators/features/shard/src/shard-messages.h b/xlators/features/shard/src/shard-messages.h
new file mode 100644
index 00000000000..2d0867eb136
--- /dev/null
+++ b/xlators/features/shard/src/shard-messages.h
@@ -0,0 +1,39 @@
+/*
+ Copyright (c) 2015 Red Hat, Inc. <http://www.redhat.com>
+ This file is part of GlusterFS.
+
+ This file is licensed to you under your choice of the GNU Lesser
+ General Public License, version 3 or any later version (LGPLv3 or
+ later), or the GNU General Public License, version 2 (GPLv2), in all
+ cases as published by the Free Software Foundation.
+ */
+
+#ifndef _SHARD_MESSAGES_H_
+#define _SHARD_MESSAGES_H_
+
+#include <glusterfs/glfs-message-id.h>
+
+/* To add new message IDs, append new identifiers at the end of the list.
+ *
+ * Never remove a message ID. If it's not used anymore, you can rename it or
+ * leave it as it is, but not delete it. This is to prevent reutilization of
+ * IDs by other messages.
+ *
+ * The component name must match one of the entries defined in
+ * glfs-message-id.h.
+ */
+
+GLFS_MSGID(SHARD, SHARD_MSG_BASE_FILE_LOOKUP_FAILED, SHARD_MSG_DICT_OP_FAILED,
+           SHARD_MSG_DOT_SHARD_NODIR, SHARD_MSG_FD_CTX_SET_FAILED,
+           SHARD_MSG_INODE_CTX_GET_FAILED, SHARD_MSG_INODE_CTX_SET_FAILED,
+           SHARD_MSG_INODE_PATH_FAILED, SHARD_MSG_INTERNAL_XATTR_MISSING,
+           SHARD_MSG_INVALID_VOLFILE, SHARD_MSG_LOOKUP_SHARD_FAILED,
+           SHARD_MSG_MEM_ACCT_INIT_FAILED, SHARD_MSG_NULL_THIS,
+           SHARD_MSG_SIZE_SET_FAILED, SHARD_MSG_STAT_FAILED,
+           SHARD_MSG_TRUNCATE_LAST_SHARD_FAILED,
+           SHARD_MSG_UPDATE_FILE_SIZE_FAILED, SHARD_MSG_FOP_NOT_SUPPORTED,
+           SHARD_MSG_INVALID_FOP, SHARD_MSG_MEMALLOC_FAILED,
+           SHARD_MSG_FOP_FAILED, SHARD_MSG_SHARDS_DELETION_FAILED,
+           SHARD_MSG_SHARD_DELETION_COMPLETED);
+
+#endif /* !_SHARD_MESSAGES_H_ */
diff --git a/xlators/features/shard/src/shard.c b/xlators/features/shard/src/shard.c
index 506c22e2400..e5f93063943 100644
--- a/xlators/features/shard/src/shard.c
+++ b/xlators/features/shard/src/shard.c
@@ -12,2006 +12,7371 @@
 
 #include "shard.h"
 #include "shard-mem-types.h"
-#include "byte-order.h"
-#include "defaults.h"
+#include <glusterfs/byte-order.h>
+#include <glusterfs/defaults.h>
+#include <glusterfs/statedump.h>
 
 static gf_boolean_t
-__is_shard_dir (uuid_t gfid)
+__is_shard_dir(uuid_t gfid)
 {
-        shard_priv_t  *priv = THIS->private;
+    shard_priv_t *priv = THIS->private;
 
-        if (gf_uuid_compare (gfid, priv->dot_shard_gfid) == 0)
-                return _gf_true;
+    if (gf_uuid_compare(gfid, priv->dot_shard_gfid) == 0)
+        return _gf_true;
 
-        return _gf_false;
+    return _gf_false;
+}
+
+static gf_boolean_t
+__is_gsyncd_on_shard_dir(call_frame_t *frame, loc_t *loc)
+{
+    if (frame->root->pid == GF_CLIENT_PID_GSYNCD &&
+        (__is_shard_dir(loc->pargfid) ||
+         (loc->parent && __is_shard_dir(loc->parent->gfid))))
+        return _gf_true;
+
+    return _gf_false;
 }
 
 void
-shard_make_block_bname (int block_num, uuid_t gfid, char *buf, size_t len)
+shard_make_block_bname(int block_num, uuid_t gfid, char *buf, size_t len)
 {
-        char gfid_str[GF_UUID_BUF_SIZE] = {0,};
+    char gfid_str[GF_UUID_BUF_SIZE] = {
+        0,
+    };
 
-        gf_uuid_unparse (gfid, gfid_str);
-        snprintf (buf, len, "%s.%d", gfid_str, block_num);
+    gf_uuid_unparse(gfid, gfid_str);
+    snprintf(buf, len, "%s.%d", gfid_str, block_num);
 }
 
 void
-shard_make_block_abspath (int block_num, uuid_t gfid, char *filepath,
-                          size_t len)
+shard_make_block_abspath(int block_num, uuid_t gfid, char *filepath, size_t len)
 {
-        char gfid_str[GF_UUID_BUF_SIZE] = {0,};
+    char gfid_str[GF_UUID_BUF_SIZE] = {
+        0,
+    };
 
-        gf_uuid_unparse (gfid, gfid_str);
-        snprintf (filepath, len, "/%s/%s.%d", GF_SHARD_DIR, gfid_str,
-                  block_num);
+    gf_uuid_unparse(gfid, gfid_str);
+    snprintf(filepath, len, "/%s/%s.%d", GF_SHARD_DIR, gfid_str, block_num);
 }
 
 int
-__shard_inode_ctx_get (inode_t *inode, xlator_t *this, shard_inode_ctx_t **ctx)
+__shard_inode_ctx_get(inode_t *inode, xlator_t *this, shard_inode_ctx_t **ctx)
 {
-        int                 ret      = -1;
-        uint64_t            ctx_uint = 0;
-        shard_inode_ctx_t  *ctx_p    = NULL;
-
-        ret = __inode_ctx_get (inode, this, &ctx_uint);
-        if (ret == 0) {
-                *ctx = (shard_inode_ctx_t *) ctx_uint;
-                return ret;
-        }
+    int ret = -1;
+    uint64_t ctx_uint = 0;
+    shard_inode_ctx_t *ctx_p = NULL;
 
-        ctx_p = GF_CALLOC (1, sizeof (*ctx_p), gf_shard_mt_inode_ctx_t);
-        if (!ctx_p)
-                return ret;
+    ret = __inode_ctx_get(inode, this, &ctx_uint);
+    if (ret == 0) {
+        *ctx = (shard_inode_ctx_t *)(uintptr_t)ctx_uint;
+        return ret;
+    }
 
-        ret = __inode_ctx_set (inode, this, (uint64_t *)&ctx_p);
-        if (ret < 0) {
-                GF_FREE (ctx_p);
-                return ret;
-        }
+    ctx_p = GF_CALLOC(1, sizeof(*ctx_p), gf_shard_mt_inode_ctx_t);
+    if (!ctx_p)
+        return ret;
 
-        *ctx = ctx_p;
+    INIT_LIST_HEAD(&ctx_p->ilist);
+    INIT_LIST_HEAD(&ctx_p->to_fsync_list);
 
+    ctx_uint = (uint64_t)(uintptr_t)ctx_p;
+    ret = __inode_ctx_set(inode, this, &ctx_uint);
+    if (ret < 0) {
+        GF_FREE(ctx_p);
         return ret;
+    }
+
+    *ctx = ctx_p;
+
+    return ret;
 }
 
+int
+shard_inode_ctx_get(inode_t *inode, xlator_t *this, shard_inode_ctx_t **ctx)
+{
+    int ret = 0;
+
+    LOCK(&inode->lock);
+    {
+        ret = __shard_inode_ctx_get(inode, this, ctx);
+    }
+    UNLOCK(&inode->lock);
+
+    return ret;
+}
 
 int
-__shard_inode_ctx_set (inode_t *inode, xlator_t *this,
-                       shard_inode_ctx_t *ctx_in)
+__shard_inode_ctx_set(inode_t *inode, xlator_t *this, struct iatt *stbuf,
+                      uint64_t block_size, int32_t valid)
 {
-        int                 ret = -1;
-        shard_inode_ctx_t  *ctx = NULL;
+    int ret = -1;
+    shard_inode_ctx_t *ctx = NULL;
 
-        ret = __shard_inode_ctx_get (inode, this, &ctx);
-        if (ret)
-                return ret;
+    ret = __shard_inode_ctx_get(inode, this, &ctx);
+    if (ret)
+        return ret;
 
-        ctx->block_size = ctx_in->block_size;
-        ctx->mode = ctx_in->mode;
-        ctx->rdev = ctx_in->rdev;
+    if (valid & SHARD_MASK_BLOCK_SIZE)
+        ctx->block_size = block_size;
 
-        return 0;
+    if (valid & SHARD_MASK_PROT)
+        ctx->stat.ia_prot = stbuf->ia_prot;
+
+    if (valid & SHARD_MASK_NLINK)
+        ctx->stat.ia_nlink = stbuf->ia_nlink;
+
+    if (valid & SHARD_MASK_UID)
+        ctx->stat.ia_uid = stbuf->ia_uid;
+
+    if (valid & SHARD_MASK_GID)
+        ctx->stat.ia_gid = stbuf->ia_gid;
+
+    if (valid & SHARD_MASK_SIZE)
+        ctx->stat.ia_size = stbuf->ia_size;
+
+    if (valid & SHARD_MASK_BLOCKS)
+        ctx->stat.ia_blocks = stbuf->ia_blocks;
+
+    if (valid & SHARD_MASK_TIMES) {
+        SHARD_TIME_UPDATE(ctx->stat.ia_mtime, ctx->stat.ia_mtime_nsec,
+                          stbuf->ia_mtime, stbuf->ia_mtime_nsec);
+        SHARD_TIME_UPDATE(ctx->stat.ia_ctime, ctx->stat.ia_ctime_nsec,
+                          stbuf->ia_ctime, stbuf->ia_ctime_nsec);
+        SHARD_TIME_UPDATE(ctx->stat.ia_atime, ctx->stat.ia_atime_nsec,
+                          stbuf->ia_atime, stbuf->ia_atime_nsec);
+    }
+
+    if (valid & SHARD_MASK_OTHERS) {
+        ctx->stat.ia_ino = stbuf->ia_ino;
+        gf_uuid_copy(ctx->stat.ia_gfid, stbuf->ia_gfid);
+        ctx->stat.ia_dev = stbuf->ia_dev;
+        ctx->stat.ia_type = stbuf->ia_type;
+        ctx->stat.ia_rdev = stbuf->ia_rdev;
+        ctx->stat.ia_blksize = stbuf->ia_blksize;
+    }
+
+    if (valid & SHARD_MASK_REFRESH_RESET)
+        ctx->refresh = _gf_false;
+
+    return 0;
 }
 
 int
-shard_inode_ctx_set_all (inode_t *inode, xlator_t *this,
-                         shard_inode_ctx_t *ctx_in)
+shard_inode_ctx_set(inode_t *inode, xlator_t *this, struct iatt *stbuf,
+                    uint64_t block_size, int32_t valid)
 {
-        int ret = -1;
+    int ret = -1;
 
-        LOCK (&inode->lock);
-        {
-                ret = __shard_inode_ctx_set (inode, this, ctx_in);
-        }
-        UNLOCK (&inode->lock);
+    LOCK(&inode->lock);
+    {
+        ret = __shard_inode_ctx_set(inode, this, stbuf, block_size, valid);
+    }
+    UNLOCK(&inode->lock);
+
+    return ret;
+}
 
+int
+__shard_inode_ctx_set_refresh_flag(inode_t *inode, xlator_t *this)
+{
+    int ret = -1;
+    shard_inode_ctx_t *ctx = NULL;
+
+    ret = __shard_inode_ctx_get(inode, this, &ctx);
+    if (ret)
         return ret;
 
+    ctx->refresh = _gf_true;
+
+    return 0;
+}
+int
+shard_inode_ctx_set_refresh_flag(inode_t *inode, xlator_t *this)
+{
+    int ret = -1;
+
+    LOCK(&inode->lock);
+    {
+        ret = __shard_inode_ctx_set_refresh_flag(inode, this);
+    }
+    UNLOCK(&inode->lock);
+
+    return ret;
 }
 
 int
-__shard_inode_ctx_get_block_size (inode_t *inode, xlator_t *this,
-                                  uint64_t *block_size)
+__shard_inode_ctx_mark_dir_refreshed(inode_t *inode, xlator_t *this)
 {
-        int                 ret      = -1;
-        uint64_t            ctx_uint = 0;
-        shard_inode_ctx_t  *ctx      = NULL;
+    int ret = -1;
+    shard_inode_ctx_t *ctx = NULL;
+
+    ret = __shard_inode_ctx_get(inode, this, &ctx);
+    if (ret)
+        return ret;
 
-        ret = __inode_ctx_get (inode, this, &ctx_uint);
-        if (ret < 0)
-                return ret;
+    ctx->refreshed = _gf_true;
+    return 0;
+}
 
-        ctx = (shard_inode_ctx_t *) ctx_uint;
+int
+shard_inode_ctx_mark_dir_refreshed(inode_t *inode, xlator_t *this)
+{
+    int ret = -1;
 
-        *block_size = ctx->block_size;
+    LOCK(&inode->lock);
+    {
+        ret = __shard_inode_ctx_mark_dir_refreshed(inode, this);
+    }
+    UNLOCK(&inode->lock);
 
-        return 0;
+    return ret;
 }
 
 int
-shard_inode_ctx_get_block_size (inode_t *inode, xlator_t *this,
-                                uint64_t *block_size)
+__shard_inode_ctx_add_to_fsync_list(inode_t *base_inode, xlator_t *this,
+                                    inode_t *shard_inode)
 {
-        int ret = -1;
+    int ret = -1;
+    shard_inode_ctx_t *base_ictx = NULL;
+    shard_inode_ctx_t *shard_ictx = NULL;
 
-        LOCK (&inode->lock);
-        {
-                ret = __shard_inode_ctx_get_block_size (inode, this,
-                                                        block_size);
-        }
-        UNLOCK (&inode->lock);
+    ret = __shard_inode_ctx_get(base_inode, this, &base_ictx);
+    if (ret)
+        return ret;
 
+    ret = __shard_inode_ctx_get(shard_inode, this, &shard_ictx);
+    if (ret)
         return ret;
+
+    if (shard_ictx->fsync_needed) {
+        shard_ictx->fsync_needed++;
+        return 1;
+    }
+
+    list_add_tail(&shard_ictx->to_fsync_list, &base_ictx->to_fsync_list);
+    shard_ictx->inode = shard_inode;
+    shard_ictx->fsync_needed++;
+    base_ictx->fsync_count++;
+    shard_ictx->base_inode = base_inode;
+
+    return 0;
 }
 
 int
-__shard_inode_ctx_get_all (inode_t *inode, xlator_t *this,
-                           shard_inode_ctx_t *ctx_out)
+shard_inode_ctx_add_to_fsync_list(inode_t *base_inode, xlator_t *this,
+                                  inode_t *shard_inode)
 {
-        int                 ret      = -1;
-        uint64_t            ctx_uint = 0;
-        shard_inode_ctx_t  *ctx      = NULL;
+    int ret = -1;
+
+    /* This ref acts as a refkeepr on the base inode. We
+     * need to keep this inode alive as it holds the head
+     * of the to_fsync_list.
+     */
+    inode_ref(base_inode);
+    inode_ref(shard_inode);
+
+    LOCK(&base_inode->lock);
+    LOCK(&shard_inode->lock);
+    {
+        ret = __shard_inode_ctx_add_to_fsync_list(base_inode, this,
+                                                  shard_inode);
+    }
+    UNLOCK(&shard_inode->lock);
+    UNLOCK(&base_inode->lock);
+
+    /* Unref the base inode corresponding to the ref above, if the shard is
+     * found to be already part of the fsync list.
+     */
+    if (ret != 0) {
+        inode_unref(base_inode);
+        inode_unref(shard_inode);
+    }
+    return ret;
+}
 
-        ret = __inode_ctx_get (inode, this, &ctx_uint);
-        if (ret < 0)
-                return ret;
+gf_boolean_t
+__shard_inode_ctx_needs_lookup(inode_t *inode, xlator_t *this)
+{
+    int ret = -1;
+    shard_inode_ctx_t *ctx = NULL;
 
-        ctx = (shard_inode_ctx_t *) ctx_uint;
+    ret = __shard_inode_ctx_get(inode, this, &ctx);
+    /* If inode ctx get fails, better to err on the side of caution and
+     * try again? Unless the failure is due to mem-allocation.
+     */
+    if (ret)
+        return _gf_true;
 
-        ctx_out->block_size = ctx->block_size;
-        ctx_out->mode = ctx->mode;
-        ctx_out->rdev = ctx->rdev;
+    return !ctx->refreshed;
+}
 
-        return 0;
+gf_boolean_t
+shard_inode_ctx_needs_lookup(inode_t *inode, xlator_t *this)
+{
+    gf_boolean_t flag = _gf_false;
+
+    LOCK(&inode->lock);
+    {
+        flag = __shard_inode_ctx_needs_lookup(inode, this);
+    }
+    UNLOCK(&inode->lock);
+
+    return flag;
+}
+int
+__shard_inode_ctx_invalidate(inode_t *inode, xlator_t *this, struct iatt *stbuf)
+{
+    int ret = -1;
+    shard_inode_ctx_t *ctx = NULL;
+
+    ret = __shard_inode_ctx_get(inode, this, &ctx);
+    if (ret)
+        return ret;
+
+    if ((stbuf->ia_size != ctx->stat.ia_size) ||
+        (stbuf->ia_blocks != ctx->stat.ia_blocks))
+        ctx->refresh = _gf_true;
+
+    return 0;
 }
 
 int
-shard_inode_ctx_get_all (inode_t *inode, xlator_t *this,
-                         shard_inode_ctx_t *ctx_out)
+shard_inode_ctx_invalidate(inode_t *inode, xlator_t *this, struct iatt *stbuf)
 {
-        int ret = -1;
+    int ret = -1;
 
-        LOCK (&inode->lock);
-        {
-                ret = __shard_inode_ctx_get_all (inode, this, ctx_out);
-        }
-        UNLOCK (&inode->lock);
+    LOCK(&inode->lock);
+    {
+        ret = __shard_inode_ctx_invalidate(inode, this, stbuf);
+    }
+    UNLOCK(&inode->lock);
+
+    return ret;
+}
+
+int
+__shard_inode_ctx_get_block_size(inode_t *inode, xlator_t *this,
+                                 uint64_t *block_size)
+{
+    int ret = -1;
+    uint64_t ctx_uint = 0;
+    shard_inode_ctx_t *ctx = NULL;
 
+    ret = __inode_ctx_get(inode, this, &ctx_uint);
+    if (ret < 0)
         return ret;
+
+    ctx = (shard_inode_ctx_t *)(uintptr_t)ctx_uint;
+
+    *block_size = ctx->block_size;
+
+    return 0;
 }
 
-void
-shard_local_wipe (shard_local_t *local)
+int
+shard_inode_ctx_get_block_size(inode_t *inode, xlator_t *this,
+                               uint64_t *block_size)
 {
-        int  i     = 0;
-        int  count = 0;
+    int ret = -1;
 
-        count = local->num_blocks;
+    LOCK(&inode->lock);
+    {
+        ret = __shard_inode_ctx_get_block_size(inode, this, block_size);
+    }
+    UNLOCK(&inode->lock);
 
-        loc_wipe (&local->loc);
-        loc_wipe (&local->dot_shard_loc);
+    return ret;
+}
 
-        if (local->fd)
-                fd_unref (local->fd);
+int
+__shard_inode_ctx_get_fsync_count(inode_t *inode, xlator_t *this,
+                                  int *fsync_count)
+{
+    int ret = -1;
+    uint64_t ctx_uint = 0;
+    shard_inode_ctx_t *ctx = NULL;
 
-        if (local->xattr_req)
-                dict_unref (local->xattr_req);
-        if (local->xattr_rsp)
-                dict_unref (local->xattr_rsp);
+    ret = __inode_ctx_get(inode, this, &ctx_uint);
+    if (ret < 0)
+        return ret;
 
-        for (i = 0; i < count; i++) {
-                if (local->inode_list[i])
-                        inode_unref (local->inode_list[i]);
-        }
+    ctx = (shard_inode_ctx_t *)(uintptr_t)ctx_uint;
 
-        GF_FREE (local->inode_list);
+    *fsync_count = ctx->fsync_needed;
 
-        GF_FREE (local->vector);
-        if (local->iobref)
-                iobref_unref (local->iobref);
+    return 0;
 }
 
 int
-shard_modify_size_and_block_count (struct iatt *stbuf, dict_t *dict)
+shard_inode_ctx_get_fsync_count(inode_t *inode, xlator_t *this,
+                                int *fsync_count)
 {
-        int                  ret       = -1;
-        void                *size_attr = NULL;
-        uint64_t             size_array[4];
+    int ret = -1;
 
-        ret = dict_get_ptr (dict, GF_XATTR_SHARD_FILE_SIZE, &size_attr);
-        if (ret) {
-                gf_log (THIS->name, GF_LOG_ERROR, "Failed to get "
-                        GF_XATTR_SHARD_FILE_SIZE " for %s",
-                        uuid_utoa (stbuf->ia_gfid));
-                return ret;
-        }
+    LOCK(&inode->lock);
+    {
+        ret = __shard_inode_ctx_get_fsync_count(inode, this, fsync_count);
+    }
+    UNLOCK(&inode->lock);
 
-        memcpy (size_array, size_attr, sizeof (size_array));
+    return ret;
+}
+int
+__shard_inode_ctx_get_all(inode_t *inode, xlator_t *this,
+                          shard_inode_ctx_t *ctx_out)
+{
+    int ret = -1;
+    uint64_t ctx_uint = 0;
+    shard_inode_ctx_t *ctx = NULL;
 
-        stbuf->ia_size = ntoh64 (size_array[0]);
-        stbuf->ia_blocks = ntoh64 (size_array[2]);
+    ret = __inode_ctx_get(inode, this, &ctx_uint);
+    if (ret < 0)
+        return ret;
 
-        return 0;
+    ctx = (shard_inode_ctx_t *)(uintptr_t)ctx_uint;
+
+    memcpy(ctx_out, ctx, sizeof(shard_inode_ctx_t));
+    return 0;
 }
 
 int
-shard_call_count_return (call_frame_t *frame)
+shard_inode_ctx_get_all(inode_t *inode, xlator_t *this,
+                        shard_inode_ctx_t *ctx_out)
 {
-        int             call_count = 0;
-        shard_local_t  *local      = NULL;
+    int ret = -1;
 
-        local = frame->local;
+    LOCK(&inode->lock);
+    {
+        ret = __shard_inode_ctx_get_all(inode, this, ctx_out);
+    }
+    UNLOCK(&inode->lock);
 
-        LOCK (&frame->lock);
-        {
-                call_count = --local->call_count;
-        }
-        UNLOCK (&frame->lock);
+    return ret;
+}
 
-        return call_count;
+int
+__shard_inode_ctx_fill_iatt_from_cache(inode_t *inode, xlator_t *this,
+                                       struct iatt *buf,
+                                       gf_boolean_t *need_refresh)
+{
+    int ret = -1;
+    uint64_t ctx_uint = 0;
+    shard_inode_ctx_t *ctx = NULL;
+
+    ret = __inode_ctx_get(inode, this, &ctx_uint);
+    if (ret < 0)
+        return ret;
+
+    ctx = (shard_inode_ctx_t *)(uintptr_t)ctx_uint;
+
+    if (ctx->refresh == _gf_false)
+        *buf = ctx->stat;
+    else
+        *need_refresh = _gf_true;
+
+    return 0;
 }
 
 int
-shard_lookup_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                  int32_t op_ret, int32_t op_errno, inode_t *inode,
-                  struct iatt *buf, dict_t *xdata, struct iatt *postparent)
+shard_inode_ctx_fill_iatt_from_cache(inode_t *inode, xlator_t *this,
+                                     struct iatt *buf,
+                                     gf_boolean_t *need_refresh)
 {
-        int                ret     = 0;
-        uint64_t           size    = 0;
-        shard_inode_ctx_t  ctx_tmp = {0,};
+    int ret = -1;
 
-        if (op_ret < 0)
-                goto unwind;
+    LOCK(&inode->lock);
+    {
+        ret = __shard_inode_ctx_fill_iatt_from_cache(inode, this, buf,
+                                                     need_refresh);
+    }
+    UNLOCK(&inode->lock);
 
-        if (IA_ISDIR (buf->ia_type))
-                goto unwind;
+    return ret;
+}
 
-        ret = dict_get_uint64 (xdata, GF_XATTR_SHARD_BLOCK_SIZE, &size);
-        if (!ret) {
-                ctx_tmp.block_size = ntoh64 (size);
-                ctx_tmp.mode = st_mode_from_ia (buf->ia_prot, buf->ia_type);
-                ctx_tmp.rdev = buf->ia_rdev;
-        }
-        /* else it is assumed that the file was created prior to enabling
-         * sharding on the volume.
-         */
+void
+shard_local_wipe(shard_local_t *local)
+{
+    int i = 0;
+    int count = 0;
+
+    count = local->num_blocks;
+
+    syncbarrier_destroy(&local->barrier);
+    loc_wipe(&local->loc);
+    loc_wipe(&local->dot_shard_loc);
+    loc_wipe(&local->dot_shard_rm_loc);
+    loc_wipe(&local->loc2);
+    loc_wipe(&local->tmp_loc);
+    loc_wipe(&local->int_inodelk.loc);
+    loc_wipe(&local->int_entrylk.loc);
+    loc_wipe(&local->newloc);
+
+    if (local->name)
+        GF_FREE(local->name);
+
+    if (local->int_entrylk.basename)
+        GF_FREE(local->int_entrylk.basename);
+    if (local->fd)
+        fd_unref(local->fd);
+
+    if (local->xattr_req)
+        dict_unref(local->xattr_req);
+    if (local->xattr_rsp)
+        dict_unref(local->xattr_rsp);
+
+    for (i = 0; i < count; i++) {
+        if (!local->inode_list)
+            break;
+
+        if (local->inode_list[i])
+            inode_unref(local->inode_list[i]);
+    }
+
+    GF_FREE(local->inode_list);
+
+    GF_FREE(local->vector);
+    if (local->iobref)
+        iobref_unref(local->iobref);
+    if (local->list_inited)
+        gf_dirent_free(&local->entries_head);
+    if (local->inodelk_frame)
+        SHARD_STACK_DESTROY(local->inodelk_frame);
+    if (local->entrylk_frame)
+        SHARD_STACK_DESTROY(local->entrylk_frame);
+}
 
-        ret = shard_inode_ctx_set_all (inode, this, &ctx_tmp);
-        if (ret)
-                gf_log (this->name, GF_LOG_WARNING, "Failed to set inode ctx "
-                        "for %s", uuid_utoa (buf->ia_gfid));
+int
+shard_modify_size_and_block_count(struct iatt *stbuf, dict_t *dict)
+{
+    int ret = -1;
+    void *size_attr = NULL;
+    uint64_t size_array[4];
+
+    ret = dict_get_ptr(dict, GF_XATTR_SHARD_FILE_SIZE, &size_attr);
+    if (ret) {
+        gf_msg_callingfn(THIS->name, GF_LOG_ERROR, 0,
+                         SHARD_MSG_INTERNAL_XATTR_MISSING,
+                         "Failed to "
+                         "get " GF_XATTR_SHARD_FILE_SIZE " for %s",
+                         uuid_utoa(stbuf->ia_gfid));
+        return ret;
+    }
 
-        /* To-Do: return the call with aggregated values of ia_size and
-         * ia_blocks
-         */
+    memcpy(size_array, size_attr, sizeof(size_array));
 
-unwind:
-        SHARD_STACK_UNWIND (lookup, frame, op_ret, op_errno, inode, buf,
-                            xdata, postparent);
-        return 0;
+    stbuf->ia_size = ntoh64(size_array[0]);
+    stbuf->ia_blocks = ntoh64(size_array[2]);
+
+    return 0;
 }
 
 int
-shard_lookup (call_frame_t *frame, xlator_t *this, loc_t *loc,
-              dict_t *xattr_req)
+shard_call_count_return(call_frame_t *frame)
 {
-        int             ret        = -1;
-        int32_t         op_errno   = ENOMEM;
-        uint64_t        block_size = 0;
-        shard_local_t  *local      = NULL;
+    int call_count = 0;
+    shard_local_t *local = NULL;
 
-        SHARD_ENTRY_FOP_CHECK (loc, op_errno, err);
+    local = frame->local;
 
-        local = mem_get0 (this->local_pool);
-        if (!local)
-                goto err;
+    LOCK(&frame->lock);
+    {
+        call_count = --local->call_count;
+    }
+    UNLOCK(&frame->lock);
 
-        frame->local = local;
+    return call_count;
+}
 
-        loc_copy (&local->loc, loc);
+static char *
+shard_internal_dir_string(shard_internal_dir_type_t type)
+{
+    char *str = NULL;
+
+    switch (type) {
+        case SHARD_INTERNAL_DIR_DOT_SHARD:
+            str = GF_SHARD_DIR;
+            break;
+        case SHARD_INTERNAL_DIR_DOT_SHARD_REMOVE_ME:
+            str = GF_SHARD_REMOVE_ME_DIR;
+            break;
+        default:
+            break;
+    }
+    return str;
+}
 
-        local->xattr_req = xattr_req ? dict_ref (xattr_req) : dict_new ();
-        if (!local->xattr_req)
-                goto err;
+static int
+shard_init_internal_dir_loc(xlator_t *this, shard_local_t *local,
+                            shard_internal_dir_type_t type)
+{
+    int ret = -1;
+    char *bname = NULL;
+    inode_t *parent = NULL;
+    loc_t *internal_dir_loc = NULL;
+    shard_priv_t *priv = NULL;
+
+    priv = this->private;
+    if (!local)
+        return -1;
+
+    switch (type) {
+        case SHARD_INTERNAL_DIR_DOT_SHARD:
+            internal_dir_loc = &local->dot_shard_loc;
+            bname = GF_SHARD_DIR;
+            parent = inode_ref(this->itable->root);
+            break;
+        case SHARD_INTERNAL_DIR_DOT_SHARD_REMOVE_ME:
+            internal_dir_loc = &local->dot_shard_rm_loc;
+            bname = GF_SHARD_REMOVE_ME_DIR;
+            parent = inode_ref(priv->dot_shard_inode);
+            break;
+        default:
+            break;
+    }
+
+    internal_dir_loc->inode = inode_new(this->itable);
+    internal_dir_loc->parent = parent;
+    ret = inode_path(internal_dir_loc->parent, bname,
+                     (char **)&internal_dir_loc->path);
+    if (ret < 0 || !(internal_dir_loc->inode)) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, SHARD_MSG_INODE_PATH_FAILED,
+               "Inode path failed on %s", bname);
+        goto out;
+    }
+
+    internal_dir_loc->name = strrchr(internal_dir_loc->path, '/');
+    if (internal_dir_loc->name)
+        internal_dir_loc->name++;
+
+    ret = 0;
+out:
+    return ret;
+}
 
-        if (shard_inode_ctx_get_block_size (loc->inode, this, &block_size)) {
-                ret = dict_set_uint64 (local->xattr_req,
-                                       GF_XATTR_SHARD_BLOCK_SIZE, 0);
-                if (ret) {
-                        gf_log (this->name, GF_LOG_WARNING, "Failed to set dict"
-                                " value: key:%s for path %s",
-                                GF_XATTR_SHARD_BLOCK_SIZE, loc->path);
-                        goto err;
+inode_t *
+__shard_update_shards_inode_list(inode_t *linked_inode, xlator_t *this,
+                                 inode_t *base_inode, int block_num,
+                                 uuid_t gfid)
+{
+    char block_bname[256] = {
+        0,
+    };
+    inode_t *lru_inode = NULL;
+    shard_priv_t *priv = NULL;
+    shard_inode_ctx_t *ctx = NULL;
+    shard_inode_ctx_t *lru_inode_ctx = NULL;
+    shard_inode_ctx_t *lru_base_inode_ctx = NULL;
+    inode_t *fsync_inode = NULL;
+    inode_t *lru_base_inode = NULL;
+    gf_boolean_t do_fsync = _gf_false;
+
+    priv = this->private;
+
+    shard_inode_ctx_get(linked_inode, this, &ctx);
+
+    if (list_empty(&ctx->ilist)) {
+        if (priv->inode_count + 1 <= priv->lru_limit) {
+            /* If this inode was linked here for the first time (indicated
+             * by empty list), and if there is still space in the priv list,
+             * add this ctx to the tail of the list.
+             */
+            /* For as long as an inode is in lru list, we try to
+             * keep it alive by holding a ref on it.
+             */
+            inode_ref(linked_inode);
+            if (base_inode)
+                gf_uuid_copy(ctx->base_gfid, base_inode->gfid);
+            else
+                gf_uuid_copy(ctx->base_gfid, gfid);
+            ctx->block_num = block_num;
+            list_add_tail(&ctx->ilist, &priv->ilist_head);
+            priv->inode_count++;
+            ctx->base_inode = inode_ref(base_inode);
+        } else {
+            /*If on the other hand there is no available slot for this inode
+             * in the list, delete the lru inode from the head of the list,
+             * unlink it. And in its place add this new inode into the list.
+             */
+            lru_inode_ctx = list_first_entry(&priv->ilist_head,
+                                             shard_inode_ctx_t, ilist);
+            GF_ASSERT(lru_inode_ctx->block_num > 0);
+            lru_base_inode = lru_inode_ctx->base_inode;
+            list_del_init(&lru_inode_ctx->ilist);
+            lru_inode = inode_find(linked_inode->table,
+                                   lru_inode_ctx->stat.ia_gfid);
+            /* If the lru inode was part of the pending-fsync list,
+             * the base inode needs to be unref'd, the lru inode
+             * deleted from fsync list and fsync'd in a new frame,
+             * and then unlinked in memory and forgotten.
+             */
+            if (!lru_base_inode)
+                goto after_fsync_check;
+            LOCK(&lru_base_inode->lock);
+            LOCK(&lru_inode->lock);
+            {
+                if (!list_empty(&lru_inode_ctx->to_fsync_list)) {
+                    list_del_init(&lru_inode_ctx->to_fsync_list);
+                    lru_inode_ctx->fsync_needed = 0;
+                    do_fsync = _gf_true;
+                    __shard_inode_ctx_get(lru_base_inode, this,
+                                          &lru_base_inode_ctx);
+                    lru_base_inode_ctx->fsync_count--;
                 }
+            }
+            UNLOCK(&lru_inode->lock);
+            UNLOCK(&lru_base_inode->lock);
+
+        after_fsync_check:
+            if (!do_fsync) {
+                shard_make_block_bname(lru_inode_ctx->block_num,
+                                       lru_inode_ctx->base_gfid, block_bname,
+                                       sizeof(block_bname));
+                /* The following unref corresponds to the ref held at
+                 * the time the shard was added to the lru list.
+                 */
+                inode_unref(lru_inode);
+                inode_unlink(lru_inode, priv->dot_shard_inode, block_bname);
+                inode_forget(lru_inode, 0);
+            } else {
+                /* The following unref corresponds to the ref
+                 * held when the shard was added to fsync list.
+                 */
+                inode_unref(lru_inode);
+                fsync_inode = lru_inode;
+                if (lru_base_inode)
+                    inode_unref(lru_base_inode);
+            }
+            /* The following unref corresponds to the ref
+             * held by inode_find() above.
+             */
+            inode_unref(lru_inode);
+
+            /* The following unref corresponds to the ref held on the base shard
+             * at the time of adding shard inode to lru list
+             */
+            if (lru_base_inode)
+                inode_unref(lru_base_inode);
+
+            /* For as long as an inode is in lru list, we try to
+             * keep it alive by holding a ref on it.
+             */
+            inode_ref(linked_inode);
+            if (base_inode)
+                gf_uuid_copy(ctx->base_gfid, base_inode->gfid);
+            else
+                gf_uuid_copy(ctx->base_gfid, gfid);
+            ctx->block_num = block_num;
+            ctx->base_inode = inode_ref(base_inode);
+            list_add_tail(&ctx->ilist, &priv->ilist_head);
+        }
+    } else {
+        /* If this is not the first time this inode is being operated on, move
+         * it to the most recently used end of the list.
+         */
+        list_move_tail(&ctx->ilist, &priv->ilist_head);
+    }
+    return fsync_inode;
+}
+
+int
+shard_common_failure_unwind(glusterfs_fop_t fop, call_frame_t *frame,
+                            int32_t op_ret, int32_t op_errno)
+{
+    switch (fop) {
+        case GF_FOP_LOOKUP:
+            SHARD_STACK_UNWIND(lookup, frame, op_ret, op_errno, NULL, NULL,
+                               NULL, NULL);
+            break;
+        case GF_FOP_STAT:
+            SHARD_STACK_UNWIND(stat, frame, op_ret, op_errno, NULL, NULL);
+            break;
+        case GF_FOP_FSTAT:
+            SHARD_STACK_UNWIND(fstat, frame, op_ret, op_errno, NULL, NULL);
+            break;
+        case GF_FOP_TRUNCATE:
+            SHARD_STACK_UNWIND(truncate, frame, op_ret, op_errno, NULL, NULL,
+                               NULL);
+            break;
+        case GF_FOP_FTRUNCATE:
+            SHARD_STACK_UNWIND(ftruncate, frame, op_ret, op_errno, NULL, NULL,
+                               NULL);
+            break;
+        case GF_FOP_MKNOD:
+            SHARD_STACK_UNWIND(mknod, frame, op_ret, op_errno, NULL, NULL, NULL,
+                               NULL, NULL);
+            break;
+        case GF_FOP_LINK:
+            SHARD_STACK_UNWIND(link, frame, op_ret, op_errno, NULL, NULL, NULL,
+                               NULL, NULL);
+            break;
+        case GF_FOP_CREATE:
+            SHARD_STACK_UNWIND(create, frame, op_ret, op_errno, NULL, NULL,
+                               NULL, NULL, NULL, NULL);
+            break;
+        case GF_FOP_UNLINK:
+            SHARD_STACK_UNWIND(unlink, frame, op_ret, op_errno, NULL, NULL,
+                               NULL);
+            break;
+        case GF_FOP_RENAME:
+            SHARD_STACK_UNWIND(rename, frame, op_ret, op_errno, NULL, NULL,
+                               NULL, NULL, NULL, NULL);
+            break;
+        case GF_FOP_WRITE:
+            SHARD_STACK_UNWIND(writev, frame, op_ret, op_errno, NULL, NULL,
+                               NULL);
+            break;
+        case GF_FOP_FALLOCATE:
+            SHARD_STACK_UNWIND(fallocate, frame, op_ret, op_errno, NULL, NULL,
+                               NULL);
+            break;
+        case GF_FOP_ZEROFILL:
+            SHARD_STACK_UNWIND(zerofill, frame, op_ret, op_errno, NULL, NULL,
+                               NULL);
+            break;
+        case GF_FOP_DISCARD:
+            SHARD_STACK_UNWIND(discard, frame, op_ret, op_errno, NULL, NULL,
+                               NULL);
+            break;
+        case GF_FOP_READ:
+            SHARD_STACK_UNWIND(readv, frame, op_ret, op_errno, NULL, -1, NULL,
+                               NULL, NULL);
+            break;
+        case GF_FOP_FSYNC:
+            SHARD_STACK_UNWIND(fsync, frame, op_ret, op_errno, NULL, NULL,
+                               NULL);
+            break;
+        case GF_FOP_REMOVEXATTR:
+            SHARD_STACK_UNWIND(removexattr, frame, op_ret, op_errno, NULL);
+            break;
+        case GF_FOP_FREMOVEXATTR:
+            SHARD_STACK_UNWIND(fremovexattr, frame, op_ret, op_errno, NULL);
+            break;
+        case GF_FOP_FGETXATTR:
+            SHARD_STACK_UNWIND(fgetxattr, frame, op_ret, op_errno, NULL, NULL);
+            break;
+        case GF_FOP_GETXATTR:
+            SHARD_STACK_UNWIND(getxattr, frame, op_ret, op_errno, NULL, NULL);
+            break;
+        case GF_FOP_FSETXATTR:
+            SHARD_STACK_UNWIND(fsetxattr, frame, op_ret, op_errno, NULL);
+            break;
+        case GF_FOP_SETXATTR:
+            SHARD_STACK_UNWIND(setxattr, frame, op_ret, op_errno, NULL);
+            break;
+        case GF_FOP_SETATTR:
+            SHARD_STACK_UNWIND(setattr, frame, op_ret, op_errno, NULL, NULL,
+                               NULL);
+            break;
+        case GF_FOP_FSETATTR:
+            SHARD_STACK_UNWIND(fsetattr, frame, op_ret, op_errno, NULL, NULL,
+                               NULL);
+            break;
+        case GF_FOP_SEEK:
+            SHARD_STACK_UNWIND(seek, frame, op_ret, op_errno, 0, NULL);
+            break;
+        default:
+            gf_msg(THIS->name, GF_LOG_WARNING, 0, SHARD_MSG_INVALID_FOP,
+                   "Invalid fop id = %d", fop);
+            break;
+    }
+    return 0;
+}
+
+int
+shard_common_inode_write_success_unwind(glusterfs_fop_t fop,
+                                        call_frame_t *frame, int32_t op_ret)
+{
+    shard_local_t *local = frame->local;
+
+    /* the below 3 variables are required because, in SHARD_STACK_UNWIND()
+       macro, there is a check for local being null. So many static analyzers
+       backtrace the code with assumption of possible (local == NULL) case,
+       and complains for below lines. By handling it like below, we overcome
+       the warnings */
+
+    struct iatt *prebuf = ((local) ? &local->prebuf : NULL);
+    struct iatt *postbuf = ((local) ? &local->postbuf : NULL);
+    dict_t *xattr_rsp = ((local) ? local->xattr_rsp : NULL);
+
+    switch (fop) {
+        case GF_FOP_WRITE:
+            SHARD_STACK_UNWIND(writev, frame, op_ret, 0, prebuf, postbuf,
+                               xattr_rsp);
+            break;
+        case GF_FOP_FALLOCATE:
+            SHARD_STACK_UNWIND(fallocate, frame, op_ret, 0, prebuf, postbuf,
+                               xattr_rsp);
+            break;
+        case GF_FOP_ZEROFILL:
+            SHARD_STACK_UNWIND(zerofill, frame, op_ret, 0, prebuf, postbuf,
+                               xattr_rsp);
+            break;
+        case GF_FOP_DISCARD:
+            SHARD_STACK_UNWIND(discard, frame, op_ret, 0, prebuf, postbuf,
+                               xattr_rsp);
+            break;
+        default:
+            gf_msg(THIS->name, GF_LOG_WARNING, 0, SHARD_MSG_INVALID_FOP,
+                   "Invalid fop id = %d", fop);
+            break;
+    }
+    return 0;
+}
+
+int
+shard_evicted_inode_fsync_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                              int32_t op_ret, int32_t op_errno,
+                              struct iatt *prebuf, struct iatt *postbuf,
+                              dict_t *xdata)
+{
+    char block_bname[256] = {
+        0,
+    };
+    fd_t *anon_fd = cookie;
+    inode_t *shard_inode = NULL;
+    shard_inode_ctx_t *ctx = NULL;
+    shard_priv_t *priv = NULL;
+
+    priv = this->private;
+
+    if (anon_fd == NULL || op_ret < 0) {
+        gf_msg(this->name, GF_LOG_WARNING, op_errno, SHARD_MSG_MEMALLOC_FAILED,
+               "fsync failed on shard");
+        goto out;
+    }
+    shard_inode = anon_fd->inode;
+
+    LOCK(&priv->lock);
+    LOCK(&shard_inode->lock);
+    {
+        __shard_inode_ctx_get(shard_inode, this, &ctx);
+        if ((list_empty(&ctx->to_fsync_list)) && (list_empty(&ctx->ilist))) {
+            shard_make_block_bname(ctx->block_num, shard_inode->gfid,
+                                   block_bname, sizeof(block_bname));
+            inode_unlink(shard_inode, priv->dot_shard_inode, block_bname);
+            /* The following unref corresponds to the ref held by
+             * inode_link() at the time the shard was created or
+             * looked up
+             */
+            inode_unref(shard_inode);
+            inode_forget(shard_inode, 0);
         }
+    }
+    UNLOCK(&shard_inode->lock);
+    UNLOCK(&priv->lock);
 
-        STACK_WIND (frame, shard_lookup_cbk, FIRST_CHILD (this),
-                    FIRST_CHILD (this)->fops->lookup, loc, local->xattr_req);
+out:
+    if (anon_fd)
+        fd_unref(anon_fd);
+    STACK_DESTROY(frame->root);
+    return 0;
+}
 
-        return 0;
+int
+shard_initiate_evicted_inode_fsync(xlator_t *this, inode_t *inode)
+{
+    fd_t *anon_fd = NULL;
+    call_frame_t *fsync_frame = NULL;
+
+    fsync_frame = create_frame(this, this->ctx->pool);
+    if (!fsync_frame) {
+        gf_msg(this->name, GF_LOG_WARNING, ENOMEM, SHARD_MSG_MEMALLOC_FAILED,
+               "Failed to create new frame "
+               "to fsync shard");
+        return -1;
+    }
+
+    anon_fd = fd_anonymous(inode);
+    if (!anon_fd) {
+        gf_msg(this->name, GF_LOG_WARNING, ENOMEM, SHARD_MSG_MEMALLOC_FAILED,
+               "Failed to create anon fd to"
+               " fsync shard");
+        STACK_DESTROY(fsync_frame->root);
+        return -1;
+    }
+
+    STACK_WIND_COOKIE(fsync_frame, shard_evicted_inode_fsync_cbk, anon_fd,
+                      FIRST_CHILD(this), FIRST_CHILD(this)->fops->fsync,
+                      anon_fd, 1, NULL);
+    return 0;
+}
+
+int
+shard_common_inode_write_post_lookup_shards_handler(call_frame_t *frame,
+                                                    xlator_t *this);
 
+int
+shard_common_resolve_shards(call_frame_t *frame, xlator_t *this,
+                            shard_post_resolve_fop_handler_t post_res_handler)
+{
+    int i = -1;
+    uint32_t shard_idx_iter = 0;
+    char path[PATH_MAX] = {
+        0,
+    };
+    uuid_t gfid = {
+        0,
+    };
+    inode_t *inode = NULL;
+    inode_t *res_inode = NULL;
+    inode_t *fsync_inode = NULL;
+    shard_priv_t *priv = NULL;
+    shard_local_t *local = NULL;
+    uint64_t resolve_count = 0;
+
+    priv = this->private;
+    local = frame->local;
+    local->call_count = 0;
+    shard_idx_iter = local->first_block;
+    res_inode = local->resolver_base_inode;
+
+    if ((local->op_ret < 0) || (local->resolve_not))
+        goto out;
+
+    /* If this prealloc FOP is for fresh file creation, then the size of the
+     * file will be 0. Then there will be no shards associated with this file.
+     * So we can skip the lookup process for the shards which do not exists
+     * and directly issue mknod to crete shards.
+     *
+     * In case the prealloc fop is to extend the preallocated file to bigger
+     * size then just lookup and populate inodes of existing shards and
+     * update the create count
+     */
+    if (local->fop == GF_FOP_FALLOCATE) {
+        if (!local->prebuf.ia_size) {
+            local->inode_list[0] = inode_ref(res_inode);
+            local->create_count = local->last_block;
+            shard_common_inode_write_post_lookup_shards_handler(frame, this);
+            return 0;
+        }
+        if (local->prebuf.ia_size < local->total_size)
+            local->create_count = local->last_block -
+                                  ((local->prebuf.ia_size - 1) /
+                                   local->block_size);
+    }
+
+    resolve_count = local->last_block - local->create_count;
+
+    if (res_inode)
+        gf_uuid_copy(gfid, res_inode->gfid);
+    else
+        gf_uuid_copy(gfid, local->base_gfid);
+
+    while (shard_idx_iter <= resolve_count) {
+        i++;
+        if (shard_idx_iter == 0) {
+            local->inode_list[i] = inode_ref(res_inode);
+            shard_idx_iter++;
+            continue;
+        }
 
-err:
-        SHARD_STACK_UNWIND (lookup, frame, -1, op_errno, NULL, NULL,
-                             NULL, NULL);
-        return 0;
+        shard_make_block_abspath(shard_idx_iter, gfid, path, sizeof(path));
+
+        inode = NULL;
+        inode = inode_resolve(this->itable, path);
+        if (inode) {
+            gf_msg_debug(this->name, 0,
+                         "Shard %d already "
+                         "present. gfid=%s. Saving inode for future.",
+                         shard_idx_iter, uuid_utoa(inode->gfid));
+            local->inode_list[i] = inode;
+            /* Let the ref on the inodes that are already present
+             * in inode table still be held so that they don't get
+             * forgotten by the time the fop reaches the actual
+             * write stage.
+             */
+            LOCK(&priv->lock);
+            {
+                fsync_inode = __shard_update_shards_inode_list(
+                    inode, this, res_inode, shard_idx_iter, gfid);
+            }
+            UNLOCK(&priv->lock);
+            shard_idx_iter++;
+            if (fsync_inode)
+                shard_initiate_evicted_inode_fsync(this, fsync_inode);
+            continue;
+        } else {
+            local->call_count++;
+            shard_idx_iter++;
+        }
+    }
+out:
+    post_res_handler(frame, this);
+    return 0;
+}
 
+int
+shard_update_file_size_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                           int32_t op_ret, int32_t op_errno, dict_t *dict,
+                           dict_t *xdata)
+{
+    inode_t *inode = NULL;
+    shard_local_t *local = NULL;
+
+    local = frame->local;
+
+    if ((local->fd) && (local->fd->inode))
+        inode = local->fd->inode;
+    else if (local->loc.inode)
+        inode = local->loc.inode;
+
+    if (op_ret < 0) {
+        gf_msg(this->name, GF_LOG_ERROR, op_errno,
+               SHARD_MSG_UPDATE_FILE_SIZE_FAILED,
+               "Update to file size"
+               " xattr failed on %s",
+               uuid_utoa(inode->gfid));
+        local->op_ret = op_ret;
+        local->op_errno = op_errno;
+        goto err;
+    }
+
+    if (shard_modify_size_and_block_count(&local->postbuf, dict)) {
+        local->op_ret = -1;
+        local->op_errno = ENOMEM;
+        goto err;
+    }
+err:
+    local->post_update_size_handler(frame, this);
+    return 0;
 }
 
 int
-shard_stat_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                int32_t op_ret, int32_t op_errno, struct iatt *buf,
-                dict_t *xdata)
+shard_set_size_attrs(int64_t size, int64_t block_count, int64_t **size_attr_p)
 {
-        /* To-Do: Update ia_size and ia_blocks in @buf before presenting it
-         * to the parent.
-         */
+    int ret = -1;
+    int64_t *size_attr = NULL;
 
-        SHARD_STACK_UNWIND (stat, frame, op_ret, op_errno, buf, xdata);
-        return 0;
+    if (!size_attr_p)
+        goto out;
+
+    size_attr = GF_CALLOC(4, sizeof(int64_t), gf_shard_mt_int64_t);
+    if (!size_attr)
+        goto out;
+
+    size_attr[0] = hton64(size);
+    /* As sharding evolves, it _may_ be necessary to embed more pieces of
+     * information within the same xattr. So allocating slots for them in
+     * advance. For now, only bytes 0-63 and 128-191 which would make up the
+     * current size and block count respectively of the file are valid.
+     */
+    size_attr[2] = hton64(block_count);
+
+    *size_attr_p = size_attr;
+
+    ret = 0;
+out:
+    return ret;
 }
 
 int
-shard_stat (call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xdata)
+shard_update_file_size(call_frame_t *frame, xlator_t *this, fd_t *fd,
+                       loc_t *loc, shard_post_update_size_fop_handler_t handler)
 {
-        STACK_WIND (frame, shard_stat_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->stat, loc, xdata);
+    int ret = -1;
+    int64_t *size_attr = NULL;
+    int64_t delta_blocks = 0;
+    inode_t *inode = NULL;
+    shard_local_t *local = NULL;
+    dict_t *xattr_req = NULL;
+
+    local = frame->local;
+    local->post_update_size_handler = handler;
+
+    xattr_req = dict_new();
+    if (!xattr_req) {
+        local->op_ret = -1;
+        local->op_errno = ENOMEM;
+        goto out;
+    }
+
+    if (fd)
+        inode = fd->inode;
+    else
+        inode = loc->inode;
+
+    /* If both size and block count have not changed, then skip the xattrop.
+     */
+    delta_blocks = GF_ATOMIC_GET(local->delta_blocks);
+    if ((local->delta_size + local->hole_size == 0) && (delta_blocks == 0)) {
+        goto out;
+    }
+
+    ret = shard_set_size_attrs(local->delta_size + local->hole_size,
+                               delta_blocks, &size_attr);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, SHARD_MSG_SIZE_SET_FAILED,
+               "Failed to set size attrs for %s", uuid_utoa(inode->gfid));
+        local->op_ret = -1;
+        local->op_errno = ENOMEM;
+        goto out;
+    }
+
+    ret = dict_set_bin(xattr_req, GF_XATTR_SHARD_FILE_SIZE, size_attr, 8 * 4);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, SHARD_MSG_DICT_OP_FAILED,
+               "Failed to set key %s into dict. gfid=%s",
+               GF_XATTR_SHARD_FILE_SIZE, uuid_utoa(inode->gfid));
+        GF_FREE(size_attr);
+        local->op_ret = -1;
+        local->op_errno = ENOMEM;
+        goto out;
+    }
+
+    if (fd)
+        STACK_WIND(frame, shard_update_file_size_cbk, FIRST_CHILD(this),
+                   FIRST_CHILD(this)->fops->fxattrop, fd,
+                   GF_XATTROP_ADD_ARRAY64, xattr_req, NULL);
+    else
+        STACK_WIND(frame, shard_update_file_size_cbk, FIRST_CHILD(this),
+                   FIRST_CHILD(this)->fops->xattrop, loc,
+                   GF_XATTROP_ADD_ARRAY64, xattr_req, NULL);
+
+    dict_unref(xattr_req);
+    return 0;
 
-        return 0;
+out:
+    if (xattr_req)
+        dict_unref(xattr_req);
+    handler(frame, this);
+    return 0;
+}
+
+static inode_t *
+shard_link_internal_dir_inode(shard_local_t *local, inode_t *inode,
+                              struct iatt *buf, shard_internal_dir_type_t type)
+{
+    inode_t *linked_inode = NULL;
+    shard_priv_t *priv = NULL;
+    char *bname = NULL;
+    inode_t **priv_inode = NULL;
+    inode_t *parent = NULL;
+
+    priv = THIS->private;
+
+    switch (type) {
+        case SHARD_INTERNAL_DIR_DOT_SHARD:
+            bname = GF_SHARD_DIR;
+            priv_inode = &priv->dot_shard_inode;
+            parent = inode->table->root;
+            break;
+        case SHARD_INTERNAL_DIR_DOT_SHARD_REMOVE_ME:
+            bname = GF_SHARD_REMOVE_ME_DIR;
+            priv_inode = &priv->dot_shard_rm_inode;
+            parent = priv->dot_shard_inode;
+            break;
+        default:
+            break;
+    }
+
+    linked_inode = inode_link(inode, parent, bname, buf);
+    inode_lookup(linked_inode);
+    *priv_inode = linked_inode;
+    return linked_inode;
 }
 
 int
-shard_fstat_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                 int32_t op_ret, int32_t op_errno, struct iatt *buf,
-                 dict_t *xdata)
+shard_refresh_internal_dir_cbk(call_frame_t *frame, void *cookie,
+                               xlator_t *this, int32_t op_ret, int32_t op_errno,
+                               inode_t *inode, struct iatt *buf, dict_t *xdata,
+                               struct iatt *postparent)
 {
-        /* To-Do: Update ia_size and ia_blocks in @buf before presenting it
-         * to the parent.
-         */
-        SHARD_STACK_UNWIND (fstat, frame, op_ret, op_errno, buf, xdata);
-        return 0;
+    shard_local_t *local = NULL;
+    inode_t *linked_inode = NULL;
+    shard_internal_dir_type_t type = (shard_internal_dir_type_t)cookie;
+
+    local = frame->local;
+
+    if (op_ret) {
+        local->op_ret = op_ret;
+        local->op_errno = op_errno;
+        goto out;
+    }
+
+    /* To-Do: Fix refcount increment per call to
+     * shard_link_internal_dir_inode().
+     */
+    linked_inode = shard_link_internal_dir_inode(local, inode, buf, type);
+    shard_inode_ctx_mark_dir_refreshed(linked_inode, this);
+out:
+    shard_common_resolve_shards(frame, this, local->post_res_handler);
+    return 0;
 }
 
 int
-shard_fstat (call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *xdata)
+shard_refresh_internal_dir(call_frame_t *frame, xlator_t *this,
+                           shard_internal_dir_type_t type)
 {
-        STACK_WIND (frame, shard_fstat_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->fstat, fd, xdata);
-        return 0;
+    loc_t loc = {
+        0,
+    };
+    inode_t *inode = NULL;
+    shard_priv_t *priv = NULL;
+    shard_local_t *local = NULL;
+    uuid_t gfid = {
+        0,
+    };
+
+    local = frame->local;
+    priv = this->private;
+
+    switch (type) {
+        case SHARD_INTERNAL_DIR_DOT_SHARD:
+            gf_uuid_copy(gfid, priv->dot_shard_gfid);
+            break;
+        case SHARD_INTERNAL_DIR_DOT_SHARD_REMOVE_ME:
+            gf_uuid_copy(gfid, priv->dot_shard_rm_gfid);
+            break;
+        default:
+            break;
+    }
+
+    inode = inode_find(this->itable, gfid);
+
+    if (!shard_inode_ctx_needs_lookup(inode, this)) {
+        local->op_ret = 0;
+        goto out;
+    }
+
+    /* Plain assignment because the ref is already taken above through
+     * call to inode_find()
+     */
+    loc.inode = inode;
+    gf_uuid_copy(loc.gfid, gfid);
+
+    STACK_WIND_COOKIE(frame, shard_refresh_internal_dir_cbk, (void *)(long)type,
+                      FIRST_CHILD(this), FIRST_CHILD(this)->fops->lookup, &loc,
+                      NULL);
+    loc_wipe(&loc);
+
+    return 0;
+
+out:
+    shard_common_resolve_shards(frame, this, local->post_res_handler);
+    return 0;
 }
 
 int
-shard_truncate (call_frame_t *frame, xlator_t *this, loc_t *loc, off_t offset,
-                dict_t *xdata)
+shard_lookup_internal_dir_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                              int32_t op_ret, int32_t op_errno, inode_t *inode,
+                              struct iatt *buf, dict_t *xdata,
+                              struct iatt *postparent)
 {
-        /*TBD - once the ability to determine the size/number of shards for a
-         * file in place, this can be filled.
-         */
-        SHARD_STACK_UNWIND (truncate, frame, -1, ENOTCONN, NULL, NULL, NULL);
-        return 0;
+    inode_t *link_inode = NULL;
+    shard_local_t *local = NULL;
+    shard_internal_dir_type_t type = (shard_internal_dir_type_t)cookie;
+
+    local = frame->local;
+
+    if (op_ret) {
+        local->op_ret = op_ret;
+        local->op_errno = op_errno;
+        goto unwind;
+    }
+
+    if (!IA_ISDIR(buf->ia_type)) {
+        gf_msg(this->name, GF_LOG_CRITICAL, 0, SHARD_MSG_DOT_SHARD_NODIR,
+               "%s already exists and "
+               "is not a directory. Please remove it from all bricks "
+               "and try again",
+               shard_internal_dir_string(type));
+        local->op_ret = -1;
+        local->op_errno = EIO;
+        goto unwind;
+    }
+
+    link_inode = shard_link_internal_dir_inode(local, inode, buf, type);
+    if (link_inode != inode) {
+        shard_refresh_internal_dir(frame, this, type);
+    } else {
+        shard_inode_ctx_mark_dir_refreshed(link_inode, this);
+        shard_common_resolve_shards(frame, this, local->post_res_handler);
+    }
+    return 0;
+
+unwind:
+    local->post_res_handler(frame, this);
+    return 0;
 }
 
 int
-shard_ftruncate (call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
-                 dict_t *xdata)
+shard_lookup_internal_dir(call_frame_t *frame, xlator_t *this,
+                          shard_post_resolve_fop_handler_t post_res_handler,
+                          shard_internal_dir_type_t type)
 {
-        /*TBD - once the ability to determine the size/number of shards for a
-         * file in place, this can be filled.
+    int ret = -1;
+    dict_t *xattr_req = NULL;
+    shard_priv_t *priv = NULL;
+    shard_local_t *local = NULL;
+    uuid_t *gfid = NULL;
+    loc_t *loc = NULL;
+    gf_boolean_t free_gfid = _gf_true;
+
+    local = frame->local;
+    priv = this->private;
+    local->post_res_handler = post_res_handler;
+
+    gfid = GF_MALLOC(sizeof(uuid_t), gf_common_mt_uuid_t);
+    if (!gfid)
+        goto err;
+
+    xattr_req = dict_new();
+    if (!xattr_req) {
+        local->op_ret = -1;
+        local->op_errno = ENOMEM;
+        goto err;
+    }
+
+    switch (type) {
+        case SHARD_INTERNAL_DIR_DOT_SHARD:
+            gf_uuid_copy(*gfid, priv->dot_shard_gfid);
+            loc = &local->dot_shard_loc;
+            break;
+        case SHARD_INTERNAL_DIR_DOT_SHARD_REMOVE_ME:
+            gf_uuid_copy(*gfid, priv->dot_shard_rm_gfid);
+            loc = &local->dot_shard_rm_loc;
+            break;
+        default:
+            bzero(*gfid, sizeof(uuid_t));
+            break;
+    }
+
+    ret = dict_set_gfuuid(xattr_req, "gfid-req", *gfid, false);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, SHARD_MSG_DICT_OP_FAILED,
+               "Failed to set gfid of %s into dict",
+               shard_internal_dir_string(type));
+        local->op_ret = -1;
+        local->op_errno = ENOMEM;
+        goto err;
+    } else {
+        free_gfid = _gf_false;
+    }
+
+    STACK_WIND_COOKIE(frame, shard_lookup_internal_dir_cbk, (void *)(long)type,
+                      FIRST_CHILD(this), FIRST_CHILD(this)->fops->lookup, loc,
+                      xattr_req);
+
+    dict_unref(xattr_req);
+    return 0;
+
+err:
+    if (xattr_req)
+        dict_unref(xattr_req);
+    if (free_gfid)
+        GF_FREE(gfid);
+    post_res_handler(frame, this);
+    return 0;
+}
+
+static void
+shard_inode_ctx_update(inode_t *inode, xlator_t *this, dict_t *xdata,
+                       struct iatt *buf)
+{
+    int ret = 0;
+    uint64_t size = 0;
+    void *bsize = NULL;
+
+    if (shard_inode_ctx_get_block_size(inode, this, &size)) {
+        /* Fresh lookup */
+        ret = dict_get_ptr(xdata, GF_XATTR_SHARD_BLOCK_SIZE, &bsize);
+        if (!ret)
+            size = ntoh64(*((uint64_t *)bsize));
+        /* If the file is sharded, set its block size, otherwise just
+         * set 0.
          */
-        SHARD_STACK_UNWIND (ftruncate, frame, -1, ENOTCONN, NULL, NULL, NULL);
+
+        shard_inode_ctx_set(inode, this, buf, size, SHARD_MASK_BLOCK_SIZE);
+    }
+    /* If the file is sharded, also set the remaining attributes,
+     * except for ia_size and ia_blocks.
+     */
+    if (size) {
+        shard_inode_ctx_set(inode, this, buf, 0, SHARD_LOOKUP_MASK);
+        (void)shard_inode_ctx_invalidate(inode, this, buf);
+    }
+}
+
+int
+shard_delete_shards(void *opaque);
+
+int
+shard_delete_shards_cbk(int ret, call_frame_t *frame, void *data);
+
+int
+shard_start_background_deletion(xlator_t *this)
+{
+    int ret = 0;
+    gf_boolean_t i_cleanup = _gf_true;
+    shard_priv_t *priv = NULL;
+    call_frame_t *cleanup_frame = NULL;
+
+    priv = this->private;
+
+    LOCK(&priv->lock);
+    {
+        switch (priv->bg_del_state) {
+            case SHARD_BG_DELETION_NONE:
+                i_cleanup = _gf_true;
+                priv->bg_del_state = SHARD_BG_DELETION_LAUNCHING;
+                break;
+            case SHARD_BG_DELETION_LAUNCHING:
+                i_cleanup = _gf_false;
+                break;
+            case SHARD_BG_DELETION_IN_PROGRESS:
+                priv->bg_del_state = SHARD_BG_DELETION_LAUNCHING;
+                i_cleanup = _gf_false;
+                break;
+            default:
+                break;
+        }
+    }
+    UNLOCK(&priv->lock);
+    if (!i_cleanup)
         return 0;
+
+    cleanup_frame = create_frame(this, this->ctx->pool);
+    if (!cleanup_frame) {
+        gf_msg(this->name, GF_LOG_WARNING, ENOMEM, SHARD_MSG_MEMALLOC_FAILED,
+               "Failed to create "
+               "new frame to delete shards");
+        ret = -ENOMEM;
+        goto err;
+    }
+
+    set_lk_owner_from_ptr(&cleanup_frame->root->lk_owner, cleanup_frame->root);
+
+    ret = synctask_new(this->ctx->env, shard_delete_shards,
+                       shard_delete_shards_cbk, cleanup_frame, cleanup_frame);
+    if (ret < 0) {
+        gf_msg(this->name, GF_LOG_WARNING, errno,
+               SHARD_MSG_SHARDS_DELETION_FAILED,
+               "failed to create task to do background "
+               "cleanup of shards");
+        STACK_DESTROY(cleanup_frame->root);
+        goto err;
+    }
+    return 0;
+
+err:
+    LOCK(&priv->lock);
+    {
+        priv->bg_del_state = SHARD_BG_DELETION_NONE;
+    }
+    UNLOCK(&priv->lock);
+    return ret;
 }
 
 int
-shard_mknod_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+shard_lookup_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
                  int32_t op_ret, int32_t op_errno, inode_t *inode,
-                 struct iatt *buf, struct iatt *preparent,
-                 struct iatt *postparent, dict_t *xdata)
+                 struct iatt *buf, dict_t *xdata, struct iatt *postparent)
 {
-        int                ret     = -1;
-        shard_local_t     *local   = NULL;
-        shard_inode_ctx_t  ctx_tmp = {0,};
-
-        local = frame->local;
+    int ret = -1;
+    shard_priv_t *priv = NULL;
+    gf_boolean_t i_start_cleanup = _gf_false;
+
+    priv = this->private;
+
+    if (op_ret < 0)
+        goto unwind;
+
+    if (IA_ISDIR(buf->ia_type))
+        goto unwind;
+
+    /* Also, if the file is sharded, get the file size and block cnt xattr,
+     * and store them in the stbuf appropriately.
+     */
+
+    if (dict_get(xdata, GF_XATTR_SHARD_FILE_SIZE) &&
+        frame->root->pid != GF_CLIENT_PID_GSYNCD)
+        shard_modify_size_and_block_count(buf, xdata);
+
+    /* If this was a fresh lookup, there are two possibilities:
+     * 1) If the file is sharded (indicated by the presence of block size
+     *    xattr), store this block size, along with rdev and mode in its
+     *    inode ctx.
+     * 2) If the file is not sharded, store size along with rdev and mode
+     *    (which are anyway don't cares) in inode ctx. Since @ctx_tmp is
+     *    already initialised to all zeroes, nothing more needs to be done.
+     */
+
+    (void)shard_inode_ctx_update(inode, this, xdata, buf);
+
+    LOCK(&priv->lock);
+    {
+        if (priv->first_lookup_done == _gf_false) {
+            priv->first_lookup_done = _gf_true;
+            i_start_cleanup = _gf_true;
+        }
+    }
+    UNLOCK(&priv->lock);
 
-        if (op_ret == -1)
-                goto unwind;
+    if (!i_start_cleanup)
+        goto unwind;
 
-        ctx_tmp.block_size = ntoh64 (local->block_size);
-        ctx_tmp.mode = st_mode_from_ia (buf->ia_prot, buf->ia_type);
-        ctx_tmp.rdev = buf->ia_rdev;
-        ret = shard_inode_ctx_set_all (inode, this, &ctx_tmp);
-        if (ret)
-                gf_log (this->name, GF_LOG_WARNING, "Failed to set inode ctx "
-                        "for %s", uuid_utoa (inode->gfid));
+    ret = shard_start_background_deletion(this);
+    if (ret < 0) {
+        LOCK(&priv->lock);
+        {
+            priv->first_lookup_done = _gf_false;
+        }
+        UNLOCK(&priv->lock);
+    }
 
 unwind:
-        SHARD_STACK_UNWIND (mknod, frame, op_ret, op_errno, inode, buf,
-                            preparent, postparent, xdata);
+    SHARD_STACK_UNWIND(lookup, frame, op_ret, op_errno, inode, buf, xdata,
+                       postparent);
+    return 0;
+}
 
-        return 0;
+int
+shard_lookup(call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xattr_req)
+{
+    int ret = -1;
+    int32_t op_errno = ENOMEM;
+    uint64_t block_size = 0;
+    shard_local_t *local = NULL;
+
+    this->itable = loc->inode->table;
+    if ((frame->root->pid != GF_CLIENT_PID_GSYNCD) &&
+        (frame->root->pid != GF_CLIENT_PID_GLFS_HEAL)) {
+        SHARD_ENTRY_FOP_CHECK(loc, op_errno, err);
+    }
+
+    local = mem_get0(this->local_pool);
+    if (!local)
+        goto err;
+
+    frame->local = local;
+
+    loc_copy(&local->loc, loc);
+
+    local->xattr_req = xattr_req ? dict_ref(xattr_req) : dict_new();
+    if (!local->xattr_req)
+        goto err;
+
+    if (shard_inode_ctx_get_block_size(loc->inode, this, &block_size)) {
+        ret = dict_set_uint64(local->xattr_req, GF_XATTR_SHARD_BLOCK_SIZE, 0);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_WARNING, 0, SHARD_MSG_DICT_OP_FAILED,
+                   "Failed to set dict"
+                   " value: key:%s for path %s",
+                   GF_XATTR_SHARD_BLOCK_SIZE, loc->path);
+            goto err;
+        }
+    }
+
+    if (frame->root->pid != GF_CLIENT_PID_GSYNCD) {
+        ret = dict_set_uint64(local->xattr_req, GF_XATTR_SHARD_FILE_SIZE,
+                              8 * 4);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_WARNING, 0, SHARD_MSG_DICT_OP_FAILED,
+                   "Failed to set dict value: key:%s for path %s.",
+                   GF_XATTR_SHARD_FILE_SIZE, loc->path);
+            goto err;
+        }
+    }
+
+    if ((xattr_req) && (dict_get(xattr_req, GF_CONTENT_KEY)))
+        dict_del(xattr_req, GF_CONTENT_KEY);
+
+    STACK_WIND(frame, shard_lookup_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->lookup, loc, local->xattr_req);
+    return 0;
+err:
+    shard_common_failure_unwind(GF_FOP_LOOKUP, frame, -1, op_errno);
+    return 0;
 }
 
 int
-shard_set_size_attrs (uint64_t size, uint64_t block_count,
-                      uint64_t **size_attr_p)
+shard_set_iattr_invoke_post_handler(call_frame_t *frame, xlator_t *this,
+                                    inode_t *inode, int32_t op_ret,
+                                    int32_t op_errno, struct iatt *buf,
+                                    dict_t *xdata)
 {
-        int             ret       = -1;
-        uint64_t       *size_attr = NULL;
+    int ret = -1;
+    int32_t mask = SHARD_INODE_WRITE_MASK;
+    shard_local_t *local = frame->local;
+    shard_inode_ctx_t ctx = {
+        0,
+    };
+
+    if (op_ret < 0) {
+        gf_msg(this->name, GF_LOG_ERROR, op_errno,
+               SHARD_MSG_BASE_FILE_LOOKUP_FAILED,
+               "Lookup on base file"
+               " failed : %s",
+               uuid_utoa(inode->gfid));
+        local->op_ret = op_ret;
+        local->op_errno = op_errno;
+        goto unwind;
+    }
+
+    local->prebuf = *buf;
+    if (shard_modify_size_and_block_count(&local->prebuf, xdata)) {
+        local->op_ret = -1;
+        local->op_errno = EINVAL;
+        goto unwind;
+    }
+
+    if (shard_inode_ctx_get_all(inode, this, &ctx))
+        mask = SHARD_ALL_MASK;
+
+    ret = shard_inode_ctx_set(inode, this, &local->prebuf, 0,
+                              (mask | SHARD_MASK_REFRESH_RESET));
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, SHARD_MSG_INODE_CTX_SET_FAILED, 0,
+               "Failed to set inode"
+               " write params into inode ctx for %s",
+               uuid_utoa(buf->ia_gfid));
+        local->op_ret = -1;
+        local->op_errno = ENOMEM;
+        goto unwind;
+    }
 
-        if (!size_attr_p)
-                goto out;
+unwind:
+    local->handler(frame, this);
+    return 0;
+}
 
-        size_attr = GF_CALLOC (4, sizeof (uint64_t), gf_shard_mt_uint64_t);
-        if (!size_attr)
-                goto out;
+int
+shard_fstat_base_file_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                          int32_t op_ret, int32_t op_errno, struct iatt *buf,
+                          dict_t *xdata)
+{
+    shard_local_t *local = frame->local;
 
-        size_attr[0] = hton64 (size);
-        /* As sharding evolves, it _may_ be necessary to embed more pieces of
-         * information within the same xattr. So allocating slots for them in
-         * advance. For now, only bytes 0-63 and 128-191 which would make up the
-         * current size and block count respectively of the file are valid.
-         */
-        size_attr[2] = hton64 (block_count);
+    shard_set_iattr_invoke_post_handler(frame, this, local->fd->inode, op_ret,
+                                        op_errno, buf, xdata);
+    return 0;
+}
 
-        *size_attr_p = size_attr;
+int
+shard_lookup_base_file_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                           int32_t op_ret, int32_t op_errno, inode_t *inode,
+                           struct iatt *buf, dict_t *xdata,
+                           struct iatt *postparent)
+{
+    /* In case of op_ret < 0, inode passed to this function will be NULL
+       ex: in case of op_errno = ENOENT. So refer prefilled inode data
+       which is part of local.
+       Note: Reassigning/overriding the inode passed to this cbk with inode
+       which is part of *struct shard_local_t* won't cause any issue as
+       both inodes have same reference/address as of the inode passed */
+    inode = ((shard_local_t *)frame->local)->loc.inode;
+
+    shard_set_iattr_invoke_post_handler(frame, this, inode, op_ret, op_errno,
+                                        buf, xdata);
+    return 0;
+}
+
+/* This function decides whether to make file based lookup or
+ * fd based lookup (fstat) depending on the 3rd and 4th arg.
+ * If fd != NULL and loc == NULL then call is for fstat
+ * If fd == NULL and loc != NULL then call is for file based
+ * lookup. Please pass args based on the requirement.
+ */
+int
+shard_refresh_base_file(call_frame_t *frame, xlator_t *this, loc_t *loc,
+                        fd_t *fd, shard_post_fop_handler_t handler)
+{
+    int ret = -1;
+    inode_t *inode = NULL;
+    shard_local_t *local = NULL;
+    dict_t *xattr_req = NULL;
+    gf_boolean_t need_refresh = _gf_false;
+
+    local = frame->local;
+    local->handler = handler;
+    inode = fd ? fd->inode : loc->inode;
+
+    ret = shard_inode_ctx_fill_iatt_from_cache(inode, this, &local->prebuf,
+                                               &need_refresh);
+    /* By this time, inode ctx should have been created either in create,
+     * mknod, readdirp or lookup. If not it is a bug!
+     */
+    if ((ret == 0) && (need_refresh == _gf_false)) {
+        gf_msg_debug(this->name, 0,
+                     "Skipping lookup on base file: %s"
+                     "Serving prebuf off the inode ctx cache",
+                     uuid_utoa(inode->gfid));
+        goto out;
+    }
+
+    xattr_req = dict_new();
+    if (!xattr_req) {
+        local->op_ret = -1;
+        local->op_errno = ENOMEM;
+        goto out;
+    }
+
+    SHARD_MD_READ_FOP_INIT_REQ_DICT(this, xattr_req, inode->gfid, local, out);
+
+    if (fd)
+        STACK_WIND(frame, shard_fstat_base_file_cbk, FIRST_CHILD(this),
+                   FIRST_CHILD(this)->fops->fstat, fd, xattr_req);
+    else
+        STACK_WIND(frame, shard_lookup_base_file_cbk, FIRST_CHILD(this),
+                   FIRST_CHILD(this)->fops->lookup, loc, xattr_req);
+
+    dict_unref(xattr_req);
+    return 0;
 
-        ret = 0;
 out:
-        return ret;
+    if (xattr_req)
+        dict_unref(xattr_req);
+    handler(frame, this);
+    return 0;
 }
 
 int
-shard_mknod (call_frame_t *frame, xlator_t *this, loc_t *loc, mode_t mode,
-             dev_t rdev, mode_t umask, dict_t *xdata)
+shard_post_fstat_handler(call_frame_t *frame, xlator_t *this)
 {
-        shard_local_t  *local      = NULL;
+    shard_local_t *local = NULL;
 
-        local = mem_get0 (this->local_pool);
-        if (!local)
-                goto err;
+    local = frame->local;
 
-        frame->local = local;
+    if (local->op_ret >= 0)
+        shard_inode_ctx_set(local->fd->inode, this, &local->prebuf, 0,
+                            SHARD_LOOKUP_MASK);
 
-        SHARD_INODE_CREATE_INIT (this, local, xdata, loc, err);
+    SHARD_STACK_UNWIND(fstat, frame, local->op_ret, local->op_errno,
+                       &local->prebuf, local->xattr_rsp);
+    return 0;
+}
 
-        STACK_WIND (frame, shard_mknod_cbk, FIRST_CHILD (this),
-                    FIRST_CHILD(this)->fops->mknod, loc, mode, rdev, umask,
-                    xdata);
-        return 0;
+int
+shard_post_stat_handler(call_frame_t *frame, xlator_t *this)
+{
+    shard_local_t *local = NULL;
 
-err:
-        SHARD_STACK_UNWIND (mknod, frame, -1, ENOMEM, NULL, NULL, NULL,
-                            NULL, NULL);
-        return 0;
+    local = frame->local;
+
+    if (local->op_ret >= 0)
+        shard_inode_ctx_set(local->loc.inode, this, &local->prebuf, 0,
+                            SHARD_LOOKUP_MASK);
 
+    SHARD_STACK_UNWIND(stat, frame, local->op_ret, local->op_errno,
+                       &local->prebuf, local->xattr_rsp);
+    return 0;
 }
 
 int
-shard_unlink_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                  int32_t op_ret, int32_t op_errno, struct iatt *preparent,
-                  struct iatt *postparent, dict_t *xdata)
+shard_common_stat_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                      int32_t op_ret, int32_t op_errno, struct iatt *buf,
+                      dict_t *xdata)
 {
-        /* To-Do: Unlink all the shards associated with the file */
+    inode_t *inode = NULL;
+    shard_local_t *local = NULL;
+
+    local = frame->local;
+
+    if (op_ret < 0) {
+        gf_msg(this->name, GF_LOG_ERROR, op_errno, SHARD_MSG_STAT_FAILED,
+               "stat failed: %s",
+               local->fd ? uuid_utoa(local->fd->inode->gfid)
+                         : uuid_utoa((local->loc.inode)->gfid));
+        local->op_ret = op_ret;
+        local->op_errno = op_errno;
+        goto unwind;
+    }
+
+    local->prebuf = *buf;
+    if (shard_modify_size_and_block_count(&local->prebuf, xdata)) {
+        local->op_ret = -1;
+        local->op_errno = EINVAL;
+        goto unwind;
+    }
+    local->xattr_rsp = dict_ref(xdata);
+
+    if (local->loc.inode)
+        inode = local->loc.inode;
+    else
+        inode = local->fd->inode;
+
+    shard_inode_ctx_invalidate(inode, this, &local->prebuf);
 
-        SHARD_STACK_UNWIND (unlink, frame, op_ret, op_errno,  preparent,
-                            postparent, xdata);
+unwind:
+    local->handler(frame, this);
+    return 0;
+}
+
+int
+shard_stat(call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xdata)
+{
+    int ret = -1;
+    uint64_t block_size = 0;
+    shard_local_t *local = NULL;
 
+    if ((IA_ISDIR(loc->inode->ia_type)) || (IA_ISLNK(loc->inode->ia_type))) {
+        STACK_WIND(frame, default_stat_cbk, FIRST_CHILD(this),
+                   FIRST_CHILD(this)->fops->stat, loc, xdata);
+        return 0;
+    }
+
+    ret = shard_inode_ctx_get_block_size(loc->inode, this, &block_size);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, SHARD_MSG_INODE_CTX_GET_FAILED,
+               "Failed to get block "
+               "size from inode ctx of %s",
+               uuid_utoa(loc->inode->gfid));
+        goto err;
+    }
+
+    if (!block_size || frame->root->pid == GF_CLIENT_PID_GSYNCD) {
+        STACK_WIND(frame, default_stat_cbk, FIRST_CHILD(this),
+                   FIRST_CHILD(this)->fops->stat, loc, xdata);
         return 0;
+    }
+
+    local = mem_get0(this->local_pool);
+    if (!local)
+        goto err;
+
+    frame->local = local;
+
+    local->handler = shard_post_stat_handler;
+    loc_copy(&local->loc, loc);
+    local->xattr_req = (xdata) ? dict_ref(xdata) : dict_new();
+    if (!local->xattr_req)
+        goto err;
+
+    SHARD_MD_READ_FOP_INIT_REQ_DICT(this, local->xattr_req, local->loc.gfid,
+                                    local, err);
+
+    STACK_WIND(frame, shard_common_stat_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->stat, loc, local->xattr_req);
+    return 0;
+err:
+    shard_common_failure_unwind(GF_FOP_STAT, frame, -1, ENOMEM);
+    return 0;
 }
 
 int
-shard_unlink (call_frame_t *frame, xlator_t *this, loc_t *loc, int xflag,
-              dict_t *xdata)
+shard_fstat(call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *xdata)
 {
-        int             op_errno = ENOMEM;
-        shard_local_t  *local    = NULL;
+    int ret = -1;
+    uint64_t block_size = 0;
+    shard_local_t *local = NULL;
 
-        local = mem_get0 (this->local_pool);
-        if (!local)
-                goto err;
+    if ((IA_ISDIR(fd->inode->ia_type)) || (IA_ISLNK(fd->inode->ia_type))) {
+        STACK_WIND(frame, default_fstat_cbk, FIRST_CHILD(this),
+                   FIRST_CHILD(this)->fops->fstat, fd, xdata);
+        return 0;
+    }
+
+    ret = shard_inode_ctx_get_block_size(fd->inode, this, &block_size);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, SHARD_MSG_INODE_CTX_GET_FAILED,
+               "Failed to get block "
+               "size from inode ctx of %s",
+               uuid_utoa(fd->inode->gfid));
+        goto err;
+    }
+
+    if (!block_size || frame->root->pid == GF_CLIENT_PID_GSYNCD) {
+        STACK_WIND(frame, default_fstat_cbk, FIRST_CHILD(this),
+                   FIRST_CHILD(this)->fops->fstat, fd, xdata);
+        return 0;
+    }
 
-        frame->local = local;
+    if (!this->itable)
+        this->itable = fd->inode->table;
 
-        loc_copy (&local->loc, loc);
-        local->xflag = xflag;
+    local = mem_get0(this->local_pool);
+    if (!local)
+        goto err;
 
-        if (xdata)
-                local->xattr_req = dict_ref (xdata);
+    frame->local = local;
 
-        STACK_WIND (frame, shard_unlink_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->unlink, loc, xflag, xdata);
+    local->handler = shard_post_fstat_handler;
+    local->fd = fd_ref(fd);
+    local->xattr_req = (xdata) ? dict_ref(xdata) : dict_new();
+    if (!local->xattr_req)
+        goto err;
 
-        return 0;
+    SHARD_MD_READ_FOP_INIT_REQ_DICT(this, local->xattr_req, fd->inode->gfid,
+                                    local, err);
+
+    STACK_WIND(frame, shard_common_stat_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->fstat, fd, local->xattr_req);
+    return 0;
 err:
-        SHARD_STACK_UNWIND (unlink, frame, -1, op_errno, NULL, NULL, NULL);
+    shard_common_failure_unwind(GF_FOP_FSTAT, frame, -1, ENOMEM);
+    return 0;
+}
+
+int
+shard_post_update_size_truncate_handler(call_frame_t *frame, xlator_t *this)
+{
+    shard_local_t *local = NULL;
+
+    local = frame->local;
+
+    if (local->fop == GF_FOP_TRUNCATE)
+        SHARD_STACK_UNWIND(truncate, frame, local->op_ret, local->op_errno,
+                           &local->prebuf, &local->postbuf, NULL);
+    else
+        SHARD_STACK_UNWIND(ftruncate, frame, local->op_ret, local->op_errno,
+                           &local->prebuf, &local->postbuf, NULL);
+    return 0;
+}
+
+int
+shard_truncate_last_shard_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                              int32_t op_ret, int32_t op_errno,
+                              struct iatt *prebuf, struct iatt *postbuf,
+                              dict_t *xdata)
+{
+    inode_t *inode = NULL;
+    int64_t delta_blocks = 0;
+    shard_local_t *local = NULL;
+
+    local = frame->local;
+
+    SHARD_UNSET_ROOT_FS_ID(frame, local);
+
+    inode = (local->fop == GF_FOP_TRUNCATE) ? local->loc.inode
+                                            : local->fd->inode;
+    if (op_ret < 0) {
+        gf_msg(this->name, GF_LOG_ERROR, op_errno,
+               SHARD_MSG_TRUNCATE_LAST_SHARD_FAILED,
+               "truncate on last"
+               " shard failed : %s",
+               uuid_utoa(inode->gfid));
+        local->op_ret = op_ret;
+        local->op_errno = op_errno;
+        goto err;
+    }
+
+    local->postbuf.ia_size = local->offset;
+    /* Let the delta be negative. We want xattrop to do subtraction */
+    local->delta_size = local->postbuf.ia_size - local->prebuf.ia_size;
+    delta_blocks = GF_ATOMIC_ADD(local->delta_blocks,
+                                 postbuf->ia_blocks - prebuf->ia_blocks);
+    GF_ASSERT(delta_blocks <= 0);
+    local->postbuf.ia_blocks += delta_blocks;
+    local->hole_size = 0;
+
+    shard_inode_ctx_set(inode, this, &local->postbuf, 0, SHARD_MASK_TIMES);
+    shard_update_file_size(frame, this, NULL, &local->loc,
+                           shard_post_update_size_truncate_handler);
+    return 0;
+err:
+    shard_common_failure_unwind(local->fop, frame, local->op_ret,
+                                local->op_errno);
+    return 0;
+}
+
+int
+shard_truncate_last_shard(call_frame_t *frame, xlator_t *this, inode_t *inode)
+{
+    size_t last_shard_size_after = 0;
+    loc_t loc = {
+        0,
+    };
+    shard_local_t *local = NULL;
+
+    local = frame->local;
+
+    /* A NULL inode could be due to the fact that the last shard which
+     * needs to be truncated does not exist due to it lying in a hole
+     * region. So the only thing left to do in that case would be an
+     * update to file size xattr.
+     */
+    if (!inode) {
+        gf_msg_debug(this->name, 0,
+                     "Last shard to be truncated absent in backend: %" PRIu64
+                     " of gfid %s. Directly proceeding to update file size",
+                     local->first_block, uuid_utoa(local->loc.inode->gfid));
+        shard_update_file_size(frame, this, NULL, &local->loc,
+                               shard_post_update_size_truncate_handler);
         return 0;
+    }
+
+    SHARD_SET_ROOT_FS_ID(frame, local);
 
+    loc.inode = inode_ref(inode);
+    gf_uuid_copy(loc.gfid, inode->gfid);
+
+    last_shard_size_after = (local->offset % local->block_size);
+
+    STACK_WIND(frame, shard_truncate_last_shard_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->truncate, &loc, last_shard_size_after,
+               NULL);
+    loc_wipe(&loc);
+    return 0;
 }
 
+void
+shard_unlink_block_inode(shard_local_t *local, int shard_block_num);
 
 int
-shard_rename_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                  int32_t op_ret, int32_t op_errno, struct iatt *buf,
-                  struct iatt *preoldparent, struct iatt *postoldparent,
-                  struct iatt *prenewparent, struct iatt *postnewparent,
-                  dict_t *xdata)
+shard_truncate_htol_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                        int32_t op_ret, int32_t op_errno,
+                        struct iatt *preparent, struct iatt *postparent,
+                        dict_t *xdata)
+{
+    int ret = 0;
+    int call_count = 0;
+    int shard_block_num = (long)cookie;
+    uint64_t block_count = 0;
+    shard_local_t *local = NULL;
+
+    local = frame->local;
+
+    if (op_ret < 0) {
+        local->op_ret = op_ret;
+        local->op_errno = op_errno;
+        goto done;
+    }
+    ret = dict_get_uint64(xdata, GF_GET_FILE_BLOCK_COUNT, &block_count);
+    if (!ret) {
+        GF_ATOMIC_SUB(local->delta_blocks, block_count);
+    } else {
+        /* dict_get failed possibly due to a heterogeneous cluster? */
+        gf_msg(this->name, GF_LOG_WARNING, 0, SHARD_MSG_DICT_OP_FAILED,
+               "Failed to get key %s from dict during truncate of gfid %s",
+               GF_GET_FILE_BLOCK_COUNT,
+               uuid_utoa(local->resolver_base_inode->gfid));
+    }
+
+    shard_unlink_block_inode(local, shard_block_num);
+done:
+    call_count = shard_call_count_return(frame);
+    if (call_count == 0) {
+        SHARD_UNSET_ROOT_FS_ID(frame, local);
+        shard_truncate_last_shard(frame, this, local->inode_list[0]);
+    }
+    return 0;
+}
+
+int
+shard_truncate_htol(call_frame_t *frame, xlator_t *this, inode_t *inode)
 {
-        /* To-Do: When both src and dst names exist, and src is renamed to
-         * dst, all the shards associated with the dst file must be unlinked.
+    int i = 1;
+    int ret = -1;
+    int call_count = 0;
+    uint32_t cur_block = 0;
+    uint32_t last_block = 0;
+    char path[PATH_MAX] = {
+        0,
+    };
+    char *bname = NULL;
+    loc_t loc = {
+        0,
+    };
+    gf_boolean_t wind_failed = _gf_false;
+    shard_local_t *local = NULL;
+    shard_priv_t *priv = NULL;
+    dict_t *xdata_req = NULL;
+
+    local = frame->local;
+    priv = this->private;
+
+    cur_block = local->first_block + 1;
+    last_block = local->last_block;
+
+    /* Determine call count */
+    for (i = 1; i < local->num_blocks; i++) {
+        if (!local->inode_list[i])
+            continue;
+        call_count++;
+    }
+
+    if (!call_count) {
+        /* Call count = 0 implies that all of the shards that need to be
+         * unlinked do not exist. So shard xlator would now proceed to
+         * do the final truncate + size updates.
          */
-        SHARD_STACK_UNWIND (rename, frame, op_ret, op_errno, buf,
-                            preoldparent, postoldparent, prenewparent,
-                            postnewparent, xdata);
+        gf_msg_debug(this->name, 0,
+                     "Shards to be unlinked as part of "
+                     "truncate absent in backend: %s. Directly "
+                     "proceeding to update file size",
+                     uuid_utoa(inode->gfid));
+        local->postbuf.ia_size = local->offset;
+        local->postbuf.ia_blocks = local->prebuf.ia_blocks;
+        local->delta_size = local->postbuf.ia_size - local->prebuf.ia_size;
+        GF_ATOMIC_INIT(local->delta_blocks, 0);
+        local->hole_size = 0;
+        shard_update_file_size(frame, this, local->fd, &local->loc,
+                               shard_post_update_size_truncate_handler);
         return 0;
+    }
+
+    local->call_count = call_count;
+    i = 1;
+    xdata_req = dict_new();
+    if (!xdata_req) {
+        shard_common_failure_unwind(local->fop, frame, -1, ENOMEM);
+        return 0;
+    }
+    ret = dict_set_uint64(xdata_req, GF_GET_FILE_BLOCK_COUNT, 8 * 8);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_WARNING, 0, SHARD_MSG_DICT_OP_FAILED,
+               "Failed to set key %s into dict during truncate of %s",
+               GF_GET_FILE_BLOCK_COUNT,
+               uuid_utoa(local->resolver_base_inode->gfid));
+        dict_unref(xdata_req);
+        shard_common_failure_unwind(local->fop, frame, -1, ENOMEM);
+        return 0;
+    }
+
+    SHARD_SET_ROOT_FS_ID(frame, local);
+    while (cur_block <= last_block) {
+        if (!local->inode_list[i]) {
+            cur_block++;
+            i++;
+            continue;
+        }
+        if (wind_failed) {
+            shard_truncate_htol_cbk(frame, (void *)(long)cur_block, this, -1,
+                                    ENOMEM, NULL, NULL, NULL);
+            goto next;
+        }
+
+        shard_make_block_abspath(cur_block, inode->gfid, path, sizeof(path));
+        bname = strrchr(path, '/') + 1;
+        loc.parent = inode_ref(priv->dot_shard_inode);
+        ret = inode_path(loc.parent, bname, (char **)&(loc.path));
+        if (ret < 0) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, SHARD_MSG_INODE_PATH_FAILED,
+                   "Inode path failed"
+                   " on %s. Base file gfid = %s",
+                   bname, uuid_utoa(inode->gfid));
+            local->op_ret = -1;
+            local->op_errno = ENOMEM;
+            loc_wipe(&loc);
+            wind_failed = _gf_true;
+            shard_truncate_htol_cbk(frame, (void *)(long)cur_block, this, -1,
+                                    ENOMEM, NULL, NULL, NULL);
+            goto next;
+        }
+        loc.name = strrchr(loc.path, '/');
+        if (loc.name)
+            loc.name++;
+        loc.inode = inode_ref(local->inode_list[i]);
+
+        STACK_WIND_COOKIE(frame, shard_truncate_htol_cbk,
+                          (void *)(long)cur_block, FIRST_CHILD(this),
+                          FIRST_CHILD(this)->fops->unlink, &loc, 0, xdata_req);
+        loc_wipe(&loc);
+    next:
+        i++;
+        cur_block++;
+        if (!--call_count)
+            break;
+    }
+    dict_unref(xdata_req);
+    return 0;
 }
 
 int
-shard_rename (call_frame_t *frame, xlator_t *this, loc_t *oldloc, loc_t *newloc,
-              dict_t *xdata)
+shard_truncate_do(call_frame_t *frame, xlator_t *this)
 {
-        int32_t         op_errno = ENOMEM;
-        shard_local_t  *local    = NULL;
+    shard_local_t *local = NULL;
 
-        local = mem_get0 (this->local_pool);
-        if (!local)
-                goto err;
+    local = frame->local;
 
-        frame->local = local;
-
-        STACK_WIND (frame, shard_rename_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->rename, oldloc, newloc, xdata);
+    if (local->num_blocks == 1) {
+        /* This means that there are no shards to be unlinked.
+         * The fop boils down to truncating the last shard, updating
+         * the size and unwinding.
+         */
+        shard_truncate_last_shard(frame, this, local->inode_list[0]);
         return 0;
+    } else {
+        shard_truncate_htol(frame, this, local->loc.inode);
+    }
+    return 0;
+}
 
-err:
-        SHARD_STACK_UNWIND (rename, frame, -1, op_errno, NULL, NULL, NULL,
-                             NULL, NULL, NULL);
+int
+shard_post_lookup_shards_truncate_handler(call_frame_t *frame, xlator_t *this)
+{
+    shard_local_t *local = NULL;
+
+    local = frame->local;
+
+    if (local->op_ret < 0) {
+        shard_common_failure_unwind(local->fop, frame, local->op_ret,
+                                    local->op_errno);
         return 0;
+    }
 
+    shard_truncate_do(frame, this);
+    return 0;
 }
 
+void
+shard_link_block_inode(shard_local_t *local, int block_num, inode_t *inode,
+                       struct iatt *buf)
+{
+    int list_index = 0;
+    char block_bname[256] = {
+        0,
+    };
+    uuid_t gfid = {
+        0,
+    };
+    inode_t *linked_inode = NULL;
+    xlator_t *this = NULL;
+    inode_t *fsync_inode = NULL;
+    shard_priv_t *priv = NULL;
+    inode_t *base_inode = NULL;
+
+    this = THIS;
+    priv = this->private;
+    if (local->loc.inode) {
+        gf_uuid_copy(gfid, local->loc.inode->gfid);
+        base_inode = local->loc.inode;
+    } else if (local->resolver_base_inode) {
+        gf_uuid_copy(gfid, local->resolver_base_inode->gfid);
+        base_inode = local->resolver_base_inode;
+    } else {
+        gf_uuid_copy(gfid, local->base_gfid);
+    }
+
+    shard_make_block_bname(block_num, gfid, block_bname, sizeof(block_bname));
+
+    shard_inode_ctx_set(inode, this, buf, 0, SHARD_LOOKUP_MASK);
+    linked_inode = inode_link(inode, priv->dot_shard_inode, block_bname, buf);
+    inode_lookup(linked_inode);
+    list_index = block_num - local->first_block;
+    local->inode_list[list_index] = linked_inode;
+
+    LOCK(&priv->lock);
+    {
+        fsync_inode = __shard_update_shards_inode_list(
+            linked_inode, this, base_inode, block_num, gfid);
+    }
+    UNLOCK(&priv->lock);
+    if (fsync_inode)
+        shard_initiate_evicted_inode_fsync(this, fsync_inode);
+}
 
 int
-shard_create_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                  int32_t op_ret, int32_t op_errno, fd_t *fd, inode_t *inode,
-                  struct iatt *stbuf, struct iatt *preparent,
-                  struct iatt *postparent, dict_t *xdata)
+shard_common_lookup_shards_cbk(call_frame_t *frame, void *cookie,
+                               xlator_t *this, int32_t op_ret, int32_t op_errno,
+                               inode_t *inode, struct iatt *buf, dict_t *xdata,
+                               struct iatt *postparent)
 {
-        int             ret        = -1;
-        shard_local_t  *local      = NULL;
-        shard_inode_ctx_t ctx_tmp = {0,};
+    int call_count = 0;
+    int shard_block_num = (long)cookie;
+    uuid_t gfid = {
+        0,
+    };
+    shard_local_t *local = NULL;
+
+    local = frame->local;
+    if (local->resolver_base_inode)
+        gf_uuid_copy(gfid, local->resolver_base_inode->gfid);
+    else
+        gf_uuid_copy(gfid, local->base_gfid);
+
+    if (op_ret < 0) {
+        /* Ignore absence of shards in the backend in truncate fop. */
+        switch (local->fop) {
+            case GF_FOP_TRUNCATE:
+            case GF_FOP_FTRUNCATE:
+            case GF_FOP_RENAME:
+            case GF_FOP_UNLINK:
+                if (op_errno == ENOENT)
+                    goto done;
+                break;
+            case GF_FOP_WRITE:
+            case GF_FOP_READ:
+            case GF_FOP_ZEROFILL:
+            case GF_FOP_DISCARD:
+            case GF_FOP_FALLOCATE:
+                if ((!local->first_lookup_done) && (op_errno == ENOENT)) {
+                    LOCK(&frame->lock);
+                    {
+                        local->create_count++;
+                    }
+                    UNLOCK(&frame->lock);
+                    goto done;
+                }
+                break;
+            default:
+                break;
+        }
 
-        local = frame->local;
+        /* else */
+        gf_msg(this->name, GF_LOG_ERROR, op_errno,
+               SHARD_MSG_LOOKUP_SHARD_FAILED,
+               "Lookup on shard %d "
+               "failed. Base file gfid = %s",
+               shard_block_num, uuid_utoa(gfid));
+        local->op_ret = op_ret;
+        local->op_errno = op_errno;
+        goto done;
+    }
 
-        if (op_ret == -1)
-                goto unwind;
+    shard_link_block_inode(local, shard_block_num, inode, buf);
 
-        ctx_tmp.block_size = ntoh64 (local->block_size);
-        ctx_tmp.mode = st_mode_from_ia (stbuf->ia_prot, stbuf->ia_type);
-        ctx_tmp.rdev = stbuf->ia_rdev;
-        ret = shard_inode_ctx_set_all (inode, this, &ctx_tmp);
-        if (ret) {
-                gf_log (this->name, GF_LOG_WARNING, "Failed to set block size "
-                        "for %s in inode ctx", uuid_utoa (inode->gfid));
-                goto unwind;
+done:
+    if (local->lookup_shards_barriered) {
+        syncbarrier_wake(&local->barrier);
+        return 0;
+    } else {
+        call_count = shard_call_count_return(frame);
+        if (call_count == 0) {
+            if (!local->first_lookup_done)
+                local->first_lookup_done = _gf_true;
+            local->pls_fop_handler(frame, this);
         }
+    }
+    return 0;
+}
 
-unwind:
-        SHARD_STACK_UNWIND (create, frame, op_ret, op_errno, fd, inode, stbuf,
-                            preparent, postparent, xdata);
-        return 0;
+dict_t *
+shard_create_gfid_dict(dict_t *dict)
+{
+    int ret = 0;
+    dict_t *new = NULL;
+    unsigned char *gfid = NULL;
+
+    new = dict_copy_with_ref(dict, NULL);
+    if (!new)
+        return NULL;
+
+    gfid = GF_MALLOC(sizeof(uuid_t), gf_common_mt_char);
+    if (!gfid) {
+        ret = -1;
+        goto out;
+    }
+
+    gf_uuid_generate(gfid);
+
+    ret = dict_set_gfuuid(new, "gfid-req", gfid, false);
+
+out:
+    if (ret) {
+        dict_unref(new);
+        new = NULL;
+        GF_FREE(gfid);
+    }
+
+    return new;
 }
 
 int
-shard_create (call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t flags,
-              mode_t mode, mode_t umask, fd_t *fd, dict_t *xdata)
+shard_common_lookup_shards(call_frame_t *frame, xlator_t *this, inode_t *inode,
+                           shard_post_lookup_shards_fop_handler_t handler)
 {
-        shard_local_t  *local      = NULL;
+    int i = 0;
+    int ret = 0;
+    int count = 0;
+    int call_count = 0;
+    int32_t shard_idx_iter = 0;
+    int lookup_count = 0;
+    char path[PATH_MAX] = {
+        0,
+    };
+    char *bname = NULL;
+    uuid_t gfid = {
+        0,
+    };
+    loc_t loc = {
+        0,
+    };
+    shard_local_t *local = NULL;
+    shard_priv_t *priv = NULL;
+    gf_boolean_t wind_failed = _gf_false;
+    dict_t *xattr_req = NULL;
+
+    priv = this->private;
+    local = frame->local;
+    count = call_count = local->call_count;
+    shard_idx_iter = local->first_block;
+    lookup_count = local->last_block - local->create_count;
+    local->pls_fop_handler = handler;
+    if (local->lookup_shards_barriered)
+        local->barrier.waitfor = local->call_count;
+
+    if (inode)
+        gf_uuid_copy(gfid, inode->gfid);
+    else
+        gf_uuid_copy(gfid, local->base_gfid);
+
+    while (shard_idx_iter <= lookup_count) {
+        if (local->inode_list[i]) {
+            i++;
+            shard_idx_iter++;
+            continue;
+        }
+
+        if (wind_failed) {
+            shard_common_lookup_shards_cbk(frame, (void *)(long)shard_idx_iter,
+                                           this, -1, ENOMEM, NULL, NULL, NULL,
+                                           NULL);
+            goto next;
+        }
 
-        local = mem_get0 (this->local_pool);
-        if (!local)
-                goto err;
+        shard_make_block_abspath(shard_idx_iter, gfid, path, sizeof(path));
+
+        bname = strrchr(path, '/') + 1;
+        loc.inode = inode_new(this->itable);
+        loc.parent = inode_ref(priv->dot_shard_inode);
+        gf_uuid_copy(loc.pargfid, priv->dot_shard_gfid);
+        ret = inode_path(loc.parent, bname, (char **)&(loc.path));
+        if (ret < 0 || !(loc.inode)) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, SHARD_MSG_INODE_PATH_FAILED,
+                   "Inode path failed"
+                   " on %s, base file gfid = %s",
+                   bname, uuid_utoa(gfid));
+            local->op_ret = -1;
+            local->op_errno = ENOMEM;
+            loc_wipe(&loc);
+            wind_failed = _gf_true;
+            shard_common_lookup_shards_cbk(frame, (void *)(long)shard_idx_iter,
+                                           this, -1, ENOMEM, NULL, NULL, NULL,
+                                           NULL);
+            goto next;
+        }
 
-        frame->local = local;
+        loc.name = strrchr(loc.path, '/');
+        if (loc.name)
+            loc.name++;
+
+        xattr_req = shard_create_gfid_dict(local->xattr_req);
+        if (!xattr_req) {
+            local->op_ret = -1;
+            local->op_errno = ENOMEM;
+            wind_failed = _gf_true;
+            loc_wipe(&loc);
+            shard_common_lookup_shards_cbk(frame, (void *)(long)shard_idx_iter,
+                                           this, -1, ENOMEM, NULL, NULL, NULL,
+                                           NULL);
+            goto next;
+        }
 
-        SHARD_INODE_CREATE_INIT (this, local, xdata, loc, err);
+        STACK_WIND_COOKIE(frame, shard_common_lookup_shards_cbk,
+                          (void *)(long)shard_idx_iter, FIRST_CHILD(this),
+                          FIRST_CHILD(this)->fops->lookup, &loc, xattr_req);
+        loc_wipe(&loc);
+        dict_unref(xattr_req);
+    next:
+        shard_idx_iter++;
+        i++;
+
+        if (!--call_count)
+            break;
+    }
+    if (local->lookup_shards_barriered) {
+        syncbarrier_wait(&local->barrier, count);
+        local->pls_fop_handler(frame, this);
+    }
+    return 0;
+}
 
-        STACK_WIND (frame, shard_create_cbk, FIRST_CHILD (this),
-                    FIRST_CHILD(this)->fops->create, loc, flags, mode, umask,
-                    fd, xdata);
+int
+shard_post_resolve_truncate_handler(call_frame_t *frame, xlator_t *this)
+{
+    shard_local_t *local = NULL;
+
+    local = frame->local;
+
+    if (local->op_ret < 0) {
+        if (local->op_errno == ENOENT) {
+            /* If lookup on /.shard fails with ENOENT, it means that
+             * the file was 0-byte in size but truncated sometime in
+             * the past to a higher size which is reflected in the
+             * size xattr, and now being truncated to a lower size.
+             * In this case, the only thing that needs to be done is
+             * to update the size xattr of the file and unwind.
+             */
+            local->first_block = local->last_block = 0;
+            local->num_blocks = 1;
+            local->call_count = 0;
+            local->op_ret = 0;
+            local->postbuf.ia_size = local->offset;
+            shard_update_file_size(frame, this, local->fd, &local->loc,
+                                   shard_post_update_size_truncate_handler);
+            return 0;
+        } else {
+            shard_common_failure_unwind(local->fop, frame, local->op_ret,
+                                        local->op_errno);
+            return 0;
+        }
+    }
+
+    if (!local->call_count)
+        shard_truncate_do(frame, this);
+    else
+        shard_common_lookup_shards(frame, this, local->loc.inode,
+                                   shard_post_lookup_shards_truncate_handler);
+
+    return 0;
+}
+
+int
+shard_truncate_begin(call_frame_t *frame, xlator_t *this)
+{
+    int ret = 0;
+    shard_local_t *local = NULL;
+    shard_priv_t *priv = NULL;
+
+    priv = this->private;
+    local = frame->local;
+
+    /* First participant block here is the lowest numbered block that would
+     * hold the last byte of the file post successful truncation.
+     * Last participant block is the block that contains the last byte in
+     * the current state of the file.
+     * If (first block == last_block):
+     *         then that means that the file only needs truncation of the
+     *         first (or last since both are same) block.
+     * Else
+     *         if (new_size % block_size == 0)
+     *                 then that means there is no truncate to be done with
+     *                 only shards from first_block + 1 through the last
+     *                 block needing to be unlinked.
+     *         else
+     *                 both truncate of the first block and unlink of the
+     *                 remaining shards until end of file is required.
+     */
+    local->first_block = (local->offset == 0)
+                             ? 0
+                             : get_lowest_block(local->offset - 1,
+                                                local->block_size);
+    local->last_block = get_highest_block(0, local->prebuf.ia_size,
+                                          local->block_size);
+
+    local->num_blocks = local->last_block - local->first_block + 1;
+    GF_ASSERT(local->num_blocks > 0);
+    local->resolver_base_inode = (local->fop == GF_FOP_TRUNCATE)
+                                     ? local->loc.inode
+                                     : local->fd->inode;
+
+    if ((local->first_block == 0) && (local->num_blocks == 1)) {
+        if (local->fop == GF_FOP_TRUNCATE)
+            STACK_WIND(frame, shard_truncate_last_shard_cbk, FIRST_CHILD(this),
+                       FIRST_CHILD(this)->fops->truncate, &local->loc,
+                       local->offset, local->xattr_req);
+        else
+            STACK_WIND(frame, shard_truncate_last_shard_cbk, FIRST_CHILD(this),
+                       FIRST_CHILD(this)->fops->ftruncate, local->fd,
+                       local->offset, local->xattr_req);
         return 0;
+    }
+
+    local->inode_list = GF_CALLOC(local->num_blocks, sizeof(inode_t *),
+                                  gf_shard_mt_inode_list);
+    if (!local->inode_list)
+        goto err;
+
+    local->dot_shard_loc.inode = inode_find(this->itable, priv->dot_shard_gfid);
+    if (!local->dot_shard_loc.inode) {
+        ret = shard_init_internal_dir_loc(this, local,
+                                          SHARD_INTERNAL_DIR_DOT_SHARD);
+        if (ret)
+            goto err;
+        shard_lookup_internal_dir(frame, this,
+                                  shard_post_resolve_truncate_handler,
+                                  SHARD_INTERNAL_DIR_DOT_SHARD);
+    } else {
+        local->post_res_handler = shard_post_resolve_truncate_handler;
+        shard_refresh_internal_dir(frame, this, SHARD_INTERNAL_DIR_DOT_SHARD);
+    }
+    return 0;
 
 err:
-        SHARD_STACK_UNWIND (create, frame, -1, ENOMEM, NULL, NULL, NULL,
-                             NULL, NULL, NULL);
+    shard_common_failure_unwind(local->fop, frame, -1, ENOMEM);
+    return 0;
+}
+
+int
+shard_post_lookup_truncate_handler(call_frame_t *frame, xlator_t *this)
+{
+    shard_local_t *local = NULL;
+    struct iatt tmp_stbuf = {
+        0,
+    };
+
+    local = frame->local;
+
+    if (local->op_ret < 0) {
+        shard_common_failure_unwind(local->fop, frame, local->op_ret,
+                                    local->op_errno);
         return 0;
+    }
 
+    local->postbuf = tmp_stbuf = local->prebuf;
+
+    if (local->prebuf.ia_size == local->offset) {
+        /* If the file size is same as requested size, unwind the call
+         * immediately.
+         */
+        if (local->fop == GF_FOP_TRUNCATE)
+            SHARD_STACK_UNWIND(truncate, frame, 0, 0, &local->prebuf,
+                               &local->postbuf, NULL);
+        else
+            SHARD_STACK_UNWIND(ftruncate, frame, 0, 0, &local->prebuf,
+                               &local->postbuf, NULL);
+    } else if (local->offset > local->prebuf.ia_size) {
+        /* If the truncate is from a lower to a higher size, set the
+         * new size xattr and unwind.
+         */
+        local->hole_size = local->offset - local->prebuf.ia_size;
+        local->delta_size = 0;
+        GF_ATOMIC_INIT(local->delta_blocks, 0);
+        local->postbuf.ia_size = local->offset;
+        tmp_stbuf.ia_size = local->offset;
+        shard_inode_ctx_set(local->loc.inode, this, &tmp_stbuf, 0,
+                            SHARD_INODE_WRITE_MASK);
+        shard_update_file_size(frame, this, NULL, &local->loc,
+                               shard_post_update_size_truncate_handler);
+    } else {
+        /* ... else
+         * i.   unlink all shards that need to be unlinked.
+         * ii.  truncate the last of the shards.
+         * iii. update the new size using setxattr.
+         * and unwind the fop.
+         */
+        local->hole_size = 0;
+        local->delta_size = (local->offset - local->prebuf.ia_size);
+        GF_ATOMIC_INIT(local->delta_blocks, 0);
+        tmp_stbuf.ia_size = local->offset;
+        shard_inode_ctx_set(local->loc.inode, this, &tmp_stbuf, 0,
+                            SHARD_INODE_WRITE_MASK);
+        shard_truncate_begin(frame, this);
+    }
+    return 0;
 }
 
+/* TO-DO:
+ * Fix updates to size and block count with racing write(s) and truncate(s).
+ */
+
 int
-shard_open_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                int32_t op_ret, int32_t op_errno, fd_t *fd, dict_t *xdata)
+shard_truncate(call_frame_t *frame, xlator_t *this, loc_t *loc, off_t offset,
+               dict_t *xdata)
 {
-        /* To-Do: Handle open with O_TRUNC under locks */
-        SHARD_STACK_UNWIND (open, frame, op_ret, op_errno, fd, xdata);
+    int ret = -1;
+    uint64_t block_size = 0;
+    shard_local_t *local = NULL;
+
+    ret = shard_inode_ctx_get_block_size(loc->inode, this, &block_size);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, SHARD_MSG_INODE_CTX_GET_FAILED,
+               "Failed to get block "
+               "size from inode ctx of %s",
+               uuid_utoa(loc->inode->gfid));
+        goto err;
+    }
+
+    if (!block_size || frame->root->pid == GF_CLIENT_PID_GSYNCD) {
+        STACK_WIND(frame, default_truncate_cbk, FIRST_CHILD(this),
+                   FIRST_CHILD(this)->fops->truncate, loc, offset, xdata);
         return 0;
+    }
+
+    if (!this->itable)
+        this->itable = loc->inode->table;
+
+    local = mem_get0(this->local_pool);
+    if (!local)
+        goto err;
+
+    frame->local = local;
+
+    ret = syncbarrier_init(&local->barrier);
+    if (ret)
+        goto err;
+    loc_copy(&local->loc, loc);
+    local->offset = offset;
+    local->block_size = block_size;
+    local->fop = GF_FOP_TRUNCATE;
+    local->xattr_req = (xdata) ? dict_ref(xdata) : dict_new();
+    if (!local->xattr_req)
+        goto err;
+    local->resolver_base_inode = loc->inode;
+    GF_ATOMIC_INIT(local->delta_blocks, 0);
+
+    shard_refresh_base_file(frame, this, &local->loc, NULL,
+                            shard_post_lookup_truncate_handler);
+    return 0;
+
+err:
+    shard_common_failure_unwind(GF_FOP_TRUNCATE, frame, -1, ENOMEM);
+    return 0;
 }
 
 int
-shard_open (call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t flags,
-            fd_t *fd, dict_t *xdata)
+shard_ftruncate(call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
+                dict_t *xdata)
 {
-        STACK_WIND (frame, shard_open_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->open, loc, flags, fd, xdata);
+    int ret = -1;
+    uint64_t block_size = 0;
+    shard_local_t *local = NULL;
+
+    ret = shard_inode_ctx_get_block_size(fd->inode, this, &block_size);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, SHARD_MSG_INODE_CTX_GET_FAILED,
+               "Failed to get block "
+               "size from inode ctx of %s",
+               uuid_utoa(fd->inode->gfid));
+        goto err;
+    }
+
+    if (!block_size || frame->root->pid == GF_CLIENT_PID_GSYNCD) {
+        STACK_WIND(frame, default_ftruncate_cbk, FIRST_CHILD(this),
+                   FIRST_CHILD(this)->fops->ftruncate, fd, offset, xdata);
         return 0;
+    }
+
+    if (!this->itable)
+        this->itable = fd->inode->table;
+
+    local = mem_get0(this->local_pool);
+    if (!local)
+        goto err;
+
+    frame->local = local;
+    ret = syncbarrier_init(&local->barrier);
+    if (ret)
+        goto err;
+    local->fd = fd_ref(fd);
+    local->offset = offset;
+    local->block_size = block_size;
+    local->xattr_req = (xdata) ? dict_ref(xdata) : dict_new();
+    if (!local->xattr_req)
+        goto err;
+    local->fop = GF_FOP_FTRUNCATE;
+
+    local->loc.inode = inode_ref(fd->inode);
+    gf_uuid_copy(local->loc.gfid, fd->inode->gfid);
+    local->resolver_base_inode = fd->inode;
+    GF_ATOMIC_INIT(local->delta_blocks, 0);
+
+    shard_refresh_base_file(frame, this, NULL, fd,
+                            shard_post_lookup_truncate_handler);
+    return 0;
+err:
+    shard_common_failure_unwind(GF_FOP_FTRUNCATE, frame, -1, ENOMEM);
+    return 0;
 }
 
-/* - Incomplete - TBD */
 int
-shard_readv (call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
-             off_t offset, uint32_t flags, dict_t *xdata)
+shard_mknod_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                int32_t op_ret, int32_t op_errno, inode_t *inode,
+                struct iatt *buf, struct iatt *preparent,
+                struct iatt *postparent, dict_t *xdata)
 {
-/*
-        int             i                       = 0;
-        int32_t         op_errno                = ENOMEM;
-        uint64_t        block_size              = 0;
-        int             highest_block           = 0;
-        int             num_blocks              = 0;
-        int             cur_block               = 0;
-        char            shard_abspath[PATH_MAX] = {0,};
-        off_t           cur_offset              = 0;
-        size_t          total_size              = 0;
-        fd_t           *cur_fd                  = NULL;
-        inode_t        *inode                   = NULL;
-        shard_local_t  *local                   = NULL;
-
-        ret = shard_inode_ctx_get_block_size (fd->inode, this, &block_size);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "Failed to get inode ctx for "
-                        "%s", uuid_utoa(fd->inode->gfid));
-                goto out;
-        }
+    int ret = -1;
+    shard_local_t *local = NULL;
 
-        if (!block_size) {
-                STACK_WIND (frame, default_readv_cbk, FIRST_CHILD(this),
-                            FIRST_CHILD(this)->fops->readv, fd, size, offset,
-                            flags, xdata);
-                return 0;
-        }
+    local = frame->local;
 
-        local = mem_get0 (this->local_pool);
-        if (!local)
-                goto err;
-
-        frame->local = local;
-
-        local->block_size = block_size;
-        local->offset = offset;
-        local->len = size;
-        local->first_block = get_lowest_block (offset, block_size);
-        highest_block = get_highest_block (offset, size, block_size);
-        num_blocks = local->num_blocks = highest_block - local->first_block + 1;
-
-        while (num_blocks--) {
-                cur_fd = (local->first_block == 0) ? fd_ref (fd) :
-                         fd_anonymous (inode);
-                cur_offset = (cur_block == local->first_block) ?
-                              get_shard_offset(offset, block_size):0;
-                STACK_WIND_COOKIE (frame, shard_readv_cbk, cur_fd,
-                                   FIRST_CHILD(this),
-                                   FIRST_CHILD(this)->fops->readv, cur_fd,
-                                   cur_size, cur_offset, flags, xdata);
-        }
+    if (op_ret == -1)
+        goto unwind;
 
-        return 0;
+    ret = shard_inode_ctx_set(inode, this, buf, local->block_size,
+                              SHARD_ALL_MASK);
+    if (ret)
+        gf_msg(this->name, GF_LOG_WARNING, 0, SHARD_MSG_INODE_CTX_SET_FAILED,
+               "Failed to set inode "
+               "ctx for %s",
+               uuid_utoa(inode->gfid));
 
+unwind:
+    SHARD_STACK_UNWIND(mknod, frame, op_ret, op_errno, inode, buf, preparent,
+                       postparent, xdata);
+
+    return 0;
+}
+
+int
+shard_mknod(call_frame_t *frame, xlator_t *this, loc_t *loc, mode_t mode,
+            dev_t rdev, mode_t umask, dict_t *xdata)
+{
+    shard_priv_t *priv = NULL;
+    shard_local_t *local = NULL;
+
+    priv = this->private;
+    local = mem_get0(this->local_pool);
+    if (!local)
+        goto err;
+
+    frame->local = local;
+    local->block_size = priv->block_size;
+    if (!__is_gsyncd_on_shard_dir(frame, loc)) {
+        SHARD_INODE_CREATE_INIT(this, local->block_size, xdata, loc, 0, 0, err);
+    }
+
+    STACK_WIND(frame, shard_mknod_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->mknod, loc, mode, rdev, umask, xdata);
+    return 0;
 err:
-        SHARD_STACK_UNWIND (readv, frame, -1, op_errno, NULL, 0, NULL, NULL,
-                            NULL);
-*/
-        SHARD_STACK_UNWIND (readv, frame, -1, ENOTCONN, NULL, 0, NULL, NULL,
-                            NULL);
-        return 0;
+    shard_common_failure_unwind(GF_FOP_MKNOD, frame, -1, ENOMEM);
+    return 0;
+}
+
+int32_t
+shard_link_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+               int32_t op_ret, int32_t op_errno, inode_t *inode,
+               struct iatt *buf, struct iatt *preparent,
+               struct iatt *postparent, dict_t *xdata)
+{
+    shard_local_t *local = NULL;
+
+    local = frame->local;
+    if (op_ret < 0)
+        goto err;
 
+    shard_inode_ctx_set(inode, this, buf, 0,
+                        SHARD_MASK_NLINK | SHARD_MASK_TIMES);
+    buf->ia_size = local->prebuf.ia_size;
+    buf->ia_blocks = local->prebuf.ia_blocks;
+
+    SHARD_STACK_UNWIND(link, frame, op_ret, op_errno, inode, buf, preparent,
+                       postparent, xdata);
+    return 0;
+err:
+    shard_common_failure_unwind(GF_FOP_LINK, frame, op_ret, op_errno);
+    return 0;
 }
 
 int
-shard_update_file_size_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                            int32_t op_ret, int32_t op_errno, dict_t *xdata)
+shard_post_lookup_link_handler(call_frame_t *frame, xlator_t *this)
 {
-        shard_local_t *local = NULL;
+    shard_local_t *local = NULL;
 
-        local = frame->local;
+    local = frame->local;
 
-        if (op_ret < 0) {
-                local->op_ret = op_ret;
-                local->op_errno = op_errno;
-                goto err;
-        }
+    if (local->op_ret < 0) {
+        SHARD_STACK_UNWIND(link, frame, local->op_ret, local->op_errno, NULL,
+                           NULL, NULL, NULL, NULL);
+        return 0;
+    }
+
+    STACK_WIND(frame, shard_link_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->link, &local->loc, &local->loc2,
+               local->xattr_req);
+    return 0;
+}
 
-        SHARD_STACK_UNWIND (writev, frame, local->written_size, local->op_errno,
-                            &local->prebuf, &local->postbuf, local->xattr_rsp);
+int32_t
+shard_link(call_frame_t *frame, xlator_t *this, loc_t *oldloc, loc_t *newloc,
+           dict_t *xdata)
+{
+    int ret = -1;
+    uint64_t block_size = 0;
+    shard_local_t *local = NULL;
+
+    ret = shard_inode_ctx_get_block_size(oldloc->inode, this, &block_size);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, SHARD_MSG_INODE_CTX_GET_FAILED,
+               "Failed to get block "
+               "size from inode ctx of %s",
+               uuid_utoa(oldloc->inode->gfid));
+        goto err;
+    }
+
+    if (!block_size) {
+        STACK_WIND_TAIL(frame, FIRST_CHILD(this), FIRST_CHILD(this)->fops->link,
+                        oldloc, newloc, xdata);
         return 0;
+    }
+
+    if (!this->itable)
+        this->itable = oldloc->inode->table;
 
+    local = mem_get0(this->local_pool);
+    if (!local)
+        goto err;
+
+    frame->local = local;
+
+    loc_copy(&local->loc, oldloc);
+    loc_copy(&local->loc2, newloc);
+    local->xattr_req = (xdata) ? dict_ref(xdata) : dict_new();
+    if (!local->xattr_req)
+        goto err;
+
+    shard_refresh_base_file(frame, this, &local->loc, NULL,
+                            shard_post_lookup_link_handler);
+    return 0;
 err:
-        SHARD_STACK_UNWIND (writev, frame, -1, local->op_errno, NULL,
-                            NULL, NULL);
+    shard_common_failure_unwind(GF_FOP_LINK, frame, -1, ENOMEM);
+    return 0;
+}
+
+int
+shard_unlink_shards_do(call_frame_t *frame, xlator_t *this, inode_t *inode);
+
+int
+shard_post_lookup_shards_unlink_handler(call_frame_t *frame, xlator_t *this)
+{
+    shard_local_t *local = NULL;
+    uuid_t gfid = {
+        0,
+    };
+
+    local = frame->local;
+
+    if (local->resolver_base_inode)
+        gf_uuid_copy(gfid, local->resolver_base_inode->gfid);
+    else
+        gf_uuid_copy(gfid, local->base_gfid);
+
+    if ((local->op_ret < 0) && (local->op_errno != ENOENT)) {
+        gf_msg(this->name, GF_LOG_ERROR, local->op_errno, SHARD_MSG_FOP_FAILED,
+               "failed to delete shards of %s", uuid_utoa(gfid));
         return 0;
+    }
+    local->op_ret = 0;
+    local->op_errno = 0;
+
+    shard_unlink_shards_do(frame, this, local->resolver_base_inode);
+    return 0;
 }
 
 int
-shard_update_file_size (call_frame_t *frame, xlator_t *this)
+shard_post_resolve_unlink_handler(call_frame_t *frame, xlator_t *this)
 {
-        int            ret       = -1;
-        uint64_t      *size_attr = NULL;
-        fd_t          *fd        = NULL;
-        shard_local_t *local     = NULL;
-        dict_t        *xattr_req = NULL;
+    shard_local_t *local = NULL;
 
-        local = frame->local;
-        fd = local->fd;
+    local = frame->local;
+    local->lookup_shards_barriered = _gf_true;
 
-        xattr_req = dict_new ();
-        if (!xattr_req)
-                goto err;
+    if (!local->call_count)
+        shard_unlink_shards_do(frame, this, local->resolver_base_inode);
+    else
+        shard_common_lookup_shards(frame, this, local->resolver_base_inode,
+                                   shard_post_lookup_shards_unlink_handler);
+    return 0;
+}
 
-        ret = shard_set_size_attrs (local->postbuf.ia_size,
-                                    local->postbuf.ia_blocks, &size_attr);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "Failed to set size attrs for"
-                        " %s", uuid_utoa (fd->inode->gfid));
-                goto err;
+void
+shard_unlink_block_inode(shard_local_t *local, int shard_block_num)
+{
+    char block_bname[256] = {
+        0,
+    };
+    uuid_t gfid = {
+        0,
+    };
+    inode_t *inode = NULL;
+    inode_t *base_inode = NULL;
+    xlator_t *this = NULL;
+    shard_priv_t *priv = NULL;
+    shard_inode_ctx_t *ctx = NULL;
+    shard_inode_ctx_t *base_ictx = NULL;
+    int unref_base_inode = 0;
+    int unref_shard_inode = 0;
+
+    this = THIS;
+    priv = this->private;
+
+    inode = local->inode_list[shard_block_num - local->first_block];
+    shard_inode_ctx_get(inode, this, &ctx);
+    base_inode = ctx->base_inode;
+    if (base_inode)
+        gf_uuid_copy(gfid, base_inode->gfid);
+    else
+        gf_uuid_copy(gfid, ctx->base_gfid);
+    shard_make_block_bname(shard_block_num, gfid, block_bname,
+                           sizeof(block_bname));
+
+    LOCK(&priv->lock);
+    if (base_inode)
+        LOCK(&base_inode->lock);
+    LOCK(&inode->lock);
+    {
+        __shard_inode_ctx_get(inode, this, &ctx);
+        if (!list_empty(&ctx->ilist)) {
+            list_del_init(&ctx->ilist);
+            priv->inode_count--;
+            unref_base_inode++;
+            unref_shard_inode++;
+            GF_ASSERT(priv->inode_count >= 0);
         }
-
-        ret = dict_set_bin (xattr_req, GF_XATTR_SHARD_FILE_SIZE, size_attr,
-                            8 * 4);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "Failed to set key %s into "
-                        "dict. gfid=%s", GF_XATTR_SHARD_FILE_SIZE,
-                        uuid_utoa (fd->inode->gfid));
-                GF_FREE (size_attr);
-                goto err;
+        if (ctx->fsync_needed) {
+            unref_base_inode++;
+            unref_shard_inode++;
+            list_del_init(&ctx->to_fsync_list);
+            if (base_inode) {
+                __shard_inode_ctx_get(base_inode, this, &base_ictx);
+                base_ictx->fsync_count--;
+            }
         }
+    }
+    UNLOCK(&inode->lock);
+    if (base_inode)
+        UNLOCK(&base_inode->lock);
+
+    inode_unlink(inode, priv->dot_shard_inode, block_bname);
+    inode_ref_reduce_by_n(inode, unref_shard_inode);
+    inode_forget(inode, 0);
+
+    if (base_inode && unref_base_inode)
+        inode_ref_reduce_by_n(base_inode, unref_base_inode);
+    UNLOCK(&priv->lock);
+}
 
-        STACK_WIND (frame, shard_update_file_size_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->fsetxattr, fd, xattr_req, 0, NULL);
+int
+shard_rename_cbk(call_frame_t *frame, xlator_t *this)
+{
+    shard_local_t *local = NULL;
 
-        dict_unref (xattr_req);
-        return 0;
+    local = frame->local;
 
-err:
-        if (xattr_req)
-                dict_unref (xattr_req);
-        SHARD_STACK_UNWIND (writev, frame, -1, ENOMEM, NULL, NULL, NULL);
+    SHARD_STACK_UNWIND(rename, frame, local->op_ret, local->op_errno,
+                       &local->prebuf, &local->preoldparent,
+                       &local->postoldparent, &local->prenewparent,
+                       &local->postnewparent, local->xattr_rsp);
+    return 0;
+}
+
+int32_t
+shard_unlink_cbk(call_frame_t *frame, xlator_t *this)
+{
+    shard_local_t *local = frame->local;
+
+    SHARD_STACK_UNWIND(unlink, frame, local->op_ret, local->op_errno,
+                       &local->preoldparent, &local->postoldparent,
+                       local->xattr_rsp);
+    return 0;
+}
+
+int
+shard_unlink_shards_do_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                           int32_t op_ret, int32_t op_errno,
+                           struct iatt *preparent, struct iatt *postparent,
+                           dict_t *xdata)
+{
+    int shard_block_num = (long)cookie;
+    shard_local_t *local = NULL;
+
+    local = frame->local;
+
+    if (op_ret < 0) {
+        local->op_ret = op_ret;
+        local->op_errno = op_errno;
+        goto done;
+    }
+
+    shard_unlink_block_inode(local, shard_block_num);
+done:
+    syncbarrier_wake(&local->barrier);
+    return 0;
+}
+
+int
+shard_unlink_shards_do(call_frame_t *frame, xlator_t *this, inode_t *inode)
+{
+    int i = 0;
+    int ret = -1;
+    int count = 0;
+    uint32_t cur_block = 0;
+    uint32_t cur_block_idx = 0; /*this is idx into inode_list[] array */
+    char *bname = NULL;
+    char path[PATH_MAX] = {
+        0,
+    };
+    uuid_t gfid = {
+        0,
+    };
+    loc_t loc = {
+        0,
+    };
+    gf_boolean_t wind_failed = _gf_false;
+    shard_local_t *local = NULL;
+    shard_priv_t *priv = NULL;
+
+    priv = this->private;
+    local = frame->local;
+
+    if (inode)
+        gf_uuid_copy(gfid, inode->gfid);
+    else
+        gf_uuid_copy(gfid, local->base_gfid);
+
+    for (i = 0; i < local->num_blocks; i++) {
+        if (!local->inode_list[i])
+            continue;
+        count++;
+    }
+
+    if (!count) {
+        /* callcount = 0 implies that all of the shards that need to be
+         * unlinked are non-existent (in other words the file is full of
+         * holes).
+         */
+        gf_msg_debug(this->name, 0,
+                     "All shards that need to be "
+                     "unlinked are non-existent: %s",
+                     uuid_utoa(gfid));
         return 0;
+    }
 
+    SHARD_SET_ROOT_FS_ID(frame, local);
+    local->barrier.waitfor = count;
+    cur_block = cur_block_idx + local->first_block;
+
+    while (cur_block_idx < local->num_blocks) {
+        if (!local->inode_list[cur_block_idx])
+            goto next;
+
+        if (wind_failed) {
+            shard_unlink_shards_do_cbk(frame, (void *)(long)cur_block, this, -1,
+                                       ENOMEM, NULL, NULL, NULL);
+            goto next;
+        }
+
+        shard_make_block_abspath(cur_block, gfid, path, sizeof(path));
+        bname = strrchr(path, '/') + 1;
+        loc.parent = inode_ref(priv->dot_shard_inode);
+        ret = inode_path(loc.parent, bname, (char **)&(loc.path));
+        if (ret < 0) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, SHARD_MSG_INODE_PATH_FAILED,
+                   "Inode path failed"
+                   " on %s, base file gfid = %s",
+                   bname, uuid_utoa(gfid));
+            local->op_ret = -1;
+            local->op_errno = ENOMEM;
+            loc_wipe(&loc);
+            wind_failed = _gf_true;
+            shard_unlink_shards_do_cbk(frame, (void *)(long)cur_block, this, -1,
+                                       ENOMEM, NULL, NULL, NULL);
+            goto next;
+        }
+
+        loc.name = strrchr(loc.path, '/');
+        if (loc.name)
+            loc.name++;
+        loc.inode = inode_ref(local->inode_list[cur_block_idx]);
+
+        STACK_WIND_COOKIE(frame, shard_unlink_shards_do_cbk,
+                          (void *)(long)cur_block, FIRST_CHILD(this),
+                          FIRST_CHILD(this)->fops->unlink, &loc, local->xflag,
+                          local->xattr_req);
+        loc_wipe(&loc);
+    next:
+        cur_block++;
+        cur_block_idx++;
+    }
+    syncbarrier_wait(&local->barrier, count);
+    SHARD_UNSET_ROOT_FS_ID(frame, local);
+    return 0;
 }
 
 int
-shard_writev_do_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                     int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
-                     struct iatt *postbuf, dict_t *xdata)
+shard_regulated_shards_deletion(call_frame_t *cleanup_frame, xlator_t *this,
+                                int now, int first_block, gf_dirent_t *entry)
 {
-        int             call_count = 0;
-        fd_t           *anon_fd    = cookie;
-        shard_local_t  *local      = NULL;
+    int i = 0;
+    int ret = 0;
+    shard_local_t *local = NULL;
+    uuid_t gfid = {
+        0,
+    };
+
+    local = cleanup_frame->local;
+
+    local->inode_list = GF_CALLOC(now, sizeof(inode_t *),
+                                  gf_shard_mt_inode_list);
+    if (!local->inode_list)
+        return -ENOMEM;
+
+    local->first_block = first_block;
+    local->last_block = first_block + now - 1;
+    local->num_blocks = now;
+    gf_uuid_parse(entry->d_name, gfid);
+    gf_uuid_copy(local->base_gfid, gfid);
+    local->resolver_base_inode = inode_find(this->itable, gfid);
+    local->call_count = 0;
+    ret = syncbarrier_init(&local->barrier);
+    if (ret) {
+        GF_FREE(local->inode_list);
+        local->inode_list = NULL;
+        inode_unref(local->resolver_base_inode);
+        local->resolver_base_inode = NULL;
+        return -errno;
+    }
+    shard_common_resolve_shards(cleanup_frame, this,
+                                shard_post_resolve_unlink_handler);
+
+    for (i = 0; i < local->num_blocks; i++) {
+        if (local->inode_list[i])
+            inode_unref(local->inode_list[i]);
+    }
+    GF_FREE(local->inode_list);
+    local->inode_list = NULL;
+    if (local->op_ret)
+        ret = -local->op_errno;
+    syncbarrier_destroy(&local->barrier);
+    inode_unref(local->resolver_base_inode);
+    local->resolver_base_inode = NULL;
+    STACK_RESET(cleanup_frame->root);
+    return ret;
+}
 
-        local = frame->local;
+int
+__shard_delete_shards_of_entry(call_frame_t *cleanup_frame, xlator_t *this,
+                               gf_dirent_t *entry, inode_t *inode)
+{
+    int ret = 0;
+    int shard_count = 0;
+    int first_block = 0;
+    int now = 0;
+    uint64_t size = 0;
+    uint64_t block_size = 0;
+    uint64_t size_array[4] = {
+        0,
+    };
+    void *bsize = NULL;
+    void *size_attr = NULL;
+    dict_t *xattr_rsp = NULL;
+    loc_t loc = {
+        0,
+    };
+    shard_local_t *local = NULL;
+    shard_priv_t *priv = NULL;
+
+    priv = this->private;
+    local = cleanup_frame->local;
+    ret = dict_reset(local->xattr_req);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_WARNING, 0, SHARD_MSG_DICT_OP_FAILED,
+               "Failed to reset dict");
+        ret = -ENOMEM;
+        goto err;
+    }
+
+    ret = dict_set_uint64(local->xattr_req, GF_XATTR_SHARD_BLOCK_SIZE, 0);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_WARNING, 0, SHARD_MSG_DICT_OP_FAILED,
+               "Failed to set dict value: key:%s", GF_XATTR_SHARD_BLOCK_SIZE);
+        ret = -ENOMEM;
+        goto err;
+    }
+
+    ret = dict_set_uint64(local->xattr_req, GF_XATTR_SHARD_FILE_SIZE, 8 * 4);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_WARNING, 0, SHARD_MSG_DICT_OP_FAILED,
+               "Failed to set dict value: key:%s", GF_XATTR_SHARD_FILE_SIZE);
+        ret = -ENOMEM;
+        goto err;
+    }
+
+    loc.inode = inode_ref(inode);
+    loc.parent = inode_ref(priv->dot_shard_rm_inode);
+    ret = inode_path(loc.parent, entry->d_name, (char **)&(loc.path));
+    if (ret < 0) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, SHARD_MSG_INODE_PATH_FAILED,
+               "Inode path  failed on %s", entry->d_name);
+        ret = -ENOMEM;
+        goto err;
+    }
+
+    loc.name = strrchr(loc.path, '/');
+    if (loc.name)
+        loc.name++;
+    ret = syncop_lookup(FIRST_CHILD(this), &loc, NULL, NULL, local->xattr_req,
+                        &xattr_rsp);
+    if (ret)
+        goto err;
+
+    ret = dict_get_ptr(xattr_rsp, GF_XATTR_SHARD_BLOCK_SIZE, &bsize);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, SHARD_MSG_DICT_OP_FAILED,
+               "Failed to get dict value: key:%s", GF_XATTR_SHARD_BLOCK_SIZE);
+        goto err;
+    }
+    block_size = ntoh64(*((uint64_t *)bsize));
+
+    ret = dict_get_ptr(xattr_rsp, GF_XATTR_SHARD_FILE_SIZE, &size_attr);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, SHARD_MSG_DICT_OP_FAILED,
+               "Failed to get dict value: key:%s", GF_XATTR_SHARD_FILE_SIZE);
+        goto err;
+    }
+
+    memcpy(size_array, size_attr, sizeof(size_array));
+    size = ntoh64(size_array[0]);
+
+    shard_count = (size / block_size) - 1;
+    if (shard_count < 0) {
+        gf_msg_debug(this->name, 0,
+                     "Size of %s hasn't grown beyond "
+                     "its shard-block-size. Nothing to delete. "
+                     "Returning",
+                     entry->d_name);
+        /* File size < shard-block-size, so nothing to delete */
+        ret = 0;
+        goto delete_marker;
+    }
+    if ((size % block_size) > 0)
+        shard_count++;
+
+    if (shard_count == 0) {
+        gf_msg_debug(this->name, 0,
+                     "Size of %s is exactly equal to "
+                     "its shard-block-size. Nothing to delete. "
+                     "Returning",
+                     entry->d_name);
+        ret = 0;
+        goto delete_marker;
+    }
+    gf_msg_debug(this->name, 0,
+                 "base file = %s, "
+                 "shard-block-size=%" PRIu64 ", file-size=%" PRIu64
+                 ", "
+                 "shard_count=%d",
+                 entry->d_name, block_size, size, shard_count);
+
+    /* Perform a gfid-based lookup to see if gfid corresponding to marker
+     * file's base name exists.
+     */
+    loc_wipe(&loc);
+    loc.inode = inode_new(this->itable);
+    if (!loc.inode) {
+        ret = -ENOMEM;
+        goto err;
+    }
+    gf_uuid_parse(entry->d_name, loc.gfid);
+    ret = syncop_lookup(FIRST_CHILD(this), &loc, NULL, NULL, NULL, NULL);
+    if (!ret) {
+        gf_msg_debug(this->name, 0,
+                     "Base shard corresponding to gfid "
+                     "%s is present. Skipping shard deletion. "
+                     "Returning",
+                     entry->d_name);
+        ret = 0;
+        goto delete_marker;
+    }
 
-        if (op_ret < 0) {
-                local->op_ret = op_ret;
-                local->op_errno = op_errno;
+    first_block = 1;
+
+    while (shard_count) {
+        if (shard_count < local->deletion_rate) {
+            now = shard_count;
+            shard_count = 0;
         } else {
-                local->written_size += op_ret;
-                local->postbuf.ia_blocks += (postbuf->ia_blocks -
-                                             prebuf->ia_blocks);
-                local->postbuf.ia_size += (postbuf->ia_size - prebuf->ia_size);
+            now = local->deletion_rate;
+            shard_count -= local->deletion_rate;
         }
 
-        if (anon_fd)
-                fd_unref (anon_fd);
+        gf_msg_debug(this->name, 0,
+                     "deleting %d shards starting from "
+                     "block %d of gfid %s",
+                     now, first_block, entry->d_name);
+        ret = shard_regulated_shards_deletion(cleanup_frame, this, now,
+                                              first_block, entry);
+        if (ret)
+            goto err;
+        first_block += now;
+    }
+
+delete_marker:
+    loc_wipe(&loc);
+    loc.inode = inode_ref(inode);
+    loc.parent = inode_ref(priv->dot_shard_rm_inode);
+    ret = inode_path(loc.parent, entry->d_name, (char **)&(loc.path));
+    if (ret < 0) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, SHARD_MSG_INODE_PATH_FAILED,
+               "Inode path  failed on %s", entry->d_name);
+        ret = -ENOMEM;
+        goto err;
+    }
+    loc.name = strrchr(loc.path, '/');
+    if (loc.name)
+        loc.name++;
+    ret = syncop_unlink(FIRST_CHILD(this), &loc, NULL, NULL);
+    if (ret)
+        gf_msg(this->name, GF_LOG_ERROR, 0, SHARD_MSG_SHARDS_DELETION_FAILED,
+               "Failed to delete %s "
+               "from /%s",
+               entry->d_name, GF_SHARD_REMOVE_ME_DIR);
+err:
+    if (xattr_rsp)
+        dict_unref(xattr_rsp);
+    loc_wipe(&loc);
+    return ret;
+}
 
-        call_count = shard_call_count_return (frame);
-        if (call_count == 0) {
-                if (local->op_ret < 0) {
-                        SHARD_STACK_UNWIND (writev, frame, local->op_ret,
-                                            local->op_errno, NULL, NULL, NULL);
-                } else {
-                        if (xdata)
-                                local->xattr_rsp = dict_ref (xdata);
-                        shard_update_file_size (frame, this);
-                }
+int
+shard_delete_shards_of_entry(call_frame_t *cleanup_frame, xlator_t *this,
+                             gf_dirent_t *entry, inode_t *inode)
+{
+    int ret = -1;
+    loc_t loc = {
+        0,
+    };
+    shard_priv_t *priv = NULL;
+
+    priv = this->private;
+    loc.inode = inode_ref(priv->dot_shard_rm_inode);
+
+    ret = syncop_entrylk(FIRST_CHILD(this), this->name, &loc, entry->d_name,
+                         ENTRYLK_LOCK_NB, ENTRYLK_WRLCK, NULL, NULL);
+    if (ret < 0) {
+        if (ret == -EAGAIN) {
+            ret = 0;
         }
+        goto out;
+    }
+    {
+        ret = __shard_delete_shards_of_entry(cleanup_frame, this, entry, inode);
+    }
+    syncop_entrylk(FIRST_CHILD(this), this->name, &loc, entry->d_name,
+                   ENTRYLK_UNLOCK, ENTRYLK_WRLCK, NULL, NULL);
+out:
+    loc_wipe(&loc);
+    return ret;
+}
 
-        return 0;
+int
+shard_delete_shards_cbk(int ret, call_frame_t *frame, void *data)
+{
+    SHARD_STACK_DESTROY(frame);
+    return 0;
 }
 
 int
-shard_writev_do (call_frame_t *frame, xlator_t *this)
-{
-        int             i                 = 0;
-        int             count             = 0;
-        int             call_count        = 0;
-        int             last_block        = 0;
-        uint32_t        cur_block         = 0;
-        fd_t           *fd                = NULL;
-        fd_t           *anon_fd           = NULL;
-        shard_local_t  *local             = NULL;
-        struct iovec   *vec               = NULL;
-        gf_boolean_t    wind_failed       = _gf_false;
-        off_t           orig_offset       = 0;
-        off_t           shard_offset      = 0;
-        off_t           vec_offset        = 0;
-        size_t          remaining_size    = 0;
-        size_t          write_size        = 0;
-
-        local = frame->local;
-        fd = local->fd;
-
-        orig_offset = local->offset;
-        remaining_size = local->total_size;
-        cur_block = local->first_block;
-        local->call_count = call_count = local->num_blocks;
-        last_block = local->last_block;
-
-        while (cur_block <= last_block) {
-                if (wind_failed) {
-                        shard_writev_do_cbk (frame, (void *) (long) 0, this, -1,
-                                             ENOMEM, NULL, NULL, NULL);
-                        goto next;
-                }
+shard_resolve_internal_dir(xlator_t *this, shard_local_t *local,
+                           shard_internal_dir_type_t type)
+{
+    int ret = 0;
+    char *bname = NULL;
+    loc_t *loc = NULL;
+    shard_priv_t *priv = NULL;
+    uuid_t gfid = {
+        0,
+    };
+    struct iatt stbuf = {
+        0,
+    };
+
+    priv = this->private;
+
+    switch (type) {
+        case SHARD_INTERNAL_DIR_DOT_SHARD:
+            loc = &local->dot_shard_loc;
+            gf_uuid_copy(gfid, priv->dot_shard_gfid);
+            bname = GF_SHARD_DIR;
+            break;
+        case SHARD_INTERNAL_DIR_DOT_SHARD_REMOVE_ME:
+            loc = &local->dot_shard_rm_loc;
+            gf_uuid_copy(gfid, priv->dot_shard_rm_gfid);
+            bname = GF_SHARD_REMOVE_ME_DIR;
+            break;
+        default:
+            break;
+    }
+
+    loc->inode = inode_find(this->itable, gfid);
+    if (!loc->inode) {
+        ret = shard_init_internal_dir_loc(this, local, type);
+        if (ret)
+            goto err;
+        ret = dict_reset(local->xattr_req);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_WARNING, 0, SHARD_MSG_DICT_OP_FAILED,
+                   "Failed to reset "
+                   "dict");
+            ret = -ENOMEM;
+            goto err;
+        }
+        ret = dict_set_gfuuid(local->xattr_req, "gfid-req", gfid, true);
+        ret = syncop_lookup(FIRST_CHILD(this), loc, &stbuf, NULL,
+                            local->xattr_req, NULL);
+        if (ret < 0) {
+            if (ret != -ENOENT)
+                gf_msg(this->name, GF_LOG_ERROR, -ret,
+                       SHARD_MSG_SHARDS_DELETION_FAILED,
+                       "Lookup on %s failed, exiting", bname);
+            goto err;
+        } else {
+            shard_link_internal_dir_inode(local, loc->inode, &stbuf, type);
+        }
+    }
+    ret = 0;
+err:
+    return ret;
+}
 
-                shard_offset = orig_offset % local->block_size;
-                write_size = local->block_size - shard_offset;
-                if (write_size > remaining_size)
-                        write_size = remaining_size;
-
-                remaining_size -= write_size;
-
-                count = iov_subset (local->vector, local->count, vec_offset,
-                                    vec_offset + write_size, NULL);
-
-                vec = GF_CALLOC (count, sizeof (struct iovec),
-                                 gf_shard_mt_iovec);
-                if (!vec) {
-                        local->op_ret = -1;
-                        local->op_errno = ENOMEM;
-                        wind_failed = _gf_true;
-                        GF_FREE (vec);
-                        shard_writev_do_cbk (frame, (void *) (long) 0, this, -1,
-                                             ENOMEM, NULL, NULL, NULL);
-                        goto next;
-                }
+int
+shard_lookup_marker_entry(xlator_t *this, shard_local_t *local,
+                          gf_dirent_t *entry)
+{
+    int ret = 0;
+    loc_t loc = {
+        0,
+    };
+
+    loc.inode = inode_new(this->itable);
+    if (!loc.inode) {
+        ret = -ENOMEM;
+        goto err;
+    }
+    loc.parent = inode_ref(local->fd->inode);
+
+    ret = inode_path(loc.parent, entry->d_name, (char **)&(loc.path));
+    if (ret < 0) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, SHARD_MSG_INODE_PATH_FAILED,
+               "Inode path failed on %s", entry->d_name);
+        ret = -ENOMEM;
+        goto err;
+    }
+
+    loc.name = strrchr(loc.path, '/');
+    if (loc.name)
+        loc.name++;
+
+    ret = syncop_lookup(FIRST_CHILD(this), &loc, NULL, NULL, NULL, NULL);
+    if (ret < 0) {
+        goto err;
+    }
+    entry->inode = inode_ref(loc.inode);
+    ret = 0;
+err:
+    loc_wipe(&loc);
+    return ret;
+}
 
-                count = iov_subset (local->vector, local->count, vec_offset,
-                                    vec_offset + write_size, vec);
-
-                if (cur_block == 0) {
-                        anon_fd = fd_ref (fd);
-                } else {
-                        anon_fd = fd_anonymous (local->inode_list[i]);
-                        if (!anon_fd) {
-                                local->op_ret = -1;
-                                local->op_errno = ENOMEM;
-                                wind_failed = _gf_true;
-                                GF_FREE (vec);
-                                shard_writev_do_cbk (frame,
-                                                     (void *) (long) anon_fd,
-                                                     this, -1, ENOMEM, NULL,
-                                                     NULL, NULL);
-                                goto next;
-                        }
+int
+shard_delete_shards(void *opaque)
+{
+    int ret = 0;
+    off_t offset = 0;
+    loc_t loc = {
+        0,
+    };
+    inode_t *link_inode = NULL;
+    xlator_t *this = NULL;
+    shard_priv_t *priv = NULL;
+    shard_local_t *local = NULL;
+    gf_dirent_t entries;
+    gf_dirent_t *entry = NULL;
+    call_frame_t *cleanup_frame = NULL;
+    gf_boolean_t done = _gf_false;
+
+    this = THIS;
+    priv = this->private;
+    INIT_LIST_HEAD(&entries.list);
+
+    cleanup_frame = opaque;
+
+    local = mem_get0(this->local_pool);
+    if (!local) {
+        gf_msg(this->name, GF_LOG_WARNING, ENOMEM, SHARD_MSG_MEMALLOC_FAILED,
+               "Failed to create local to "
+               "delete shards");
+        ret = -ENOMEM;
+        goto err;
+    }
+    cleanup_frame->local = local;
+    local->fop = GF_FOP_UNLINK;
+
+    local->xattr_req = dict_new();
+    if (!local->xattr_req) {
+        ret = -ENOMEM;
+        goto err;
+    }
+    local->deletion_rate = priv->deletion_rate;
+
+    ret = shard_resolve_internal_dir(this, local, SHARD_INTERNAL_DIR_DOT_SHARD);
+    if (ret == -ENOENT) {
+        gf_msg_debug(this->name, 0,
+                     ".shard absent. Nothing to"
+                     " delete. Exiting");
+        ret = 0;
+        goto err;
+    } else if (ret < 0) {
+        goto err;
+    }
+
+    ret = shard_resolve_internal_dir(this, local,
+                                     SHARD_INTERNAL_DIR_DOT_SHARD_REMOVE_ME);
+    if (ret == -ENOENT) {
+        gf_msg_debug(this->name, 0,
+                     ".remove_me absent. "
+                     "Nothing to delete. Exiting");
+        ret = 0;
+        goto err;
+    } else if (ret < 0) {
+        goto err;
+    }
+
+    local->fd = fd_anonymous(local->dot_shard_rm_loc.inode);
+    if (!local->fd) {
+        ret = -ENOMEM;
+        goto err;
+    }
+
+    for (;;) {
+        offset = 0;
+        LOCK(&priv->lock);
+        {
+            if (priv->bg_del_state == SHARD_BG_DELETION_LAUNCHING) {
+                priv->bg_del_state = SHARD_BG_DELETION_IN_PROGRESS;
+            } else if (priv->bg_del_state == SHARD_BG_DELETION_IN_PROGRESS) {
+                priv->bg_del_state = SHARD_BG_DELETION_NONE;
+                done = _gf_true;
+            }
+        }
+        UNLOCK(&priv->lock);
+        if (done)
+            break;
+        while (
+            (ret = syncop_readdirp(FIRST_CHILD(this), local->fd, 131072, offset,
+                                   &entries, local->xattr_req, NULL))) {
+            if (ret > 0)
+                ret = 0;
+            list_for_each_entry(entry, &entries.list, list)
+            {
+                offset = entry->d_off;
+
+                if (!strcmp(entry->d_name, ".") || !strcmp(entry->d_name, ".."))
+                    continue;
+
+                if (!entry->inode) {
+                    ret = shard_lookup_marker_entry(this, local, entry);
+                    if (ret < 0)
+                        continue;
+                }
+                link_inode = inode_link(entry->inode, local->fd->inode,
+                                        entry->d_name, &entry->d_stat);
+
+                gf_msg_debug(this->name, 0,
+                             "Initiating deletion of "
+                             "shards of gfid %s",
+                             entry->d_name);
+                ret = shard_delete_shards_of_entry(cleanup_frame, this, entry,
+                                                   link_inode);
+                inode_unlink(link_inode, local->fd->inode, entry->d_name);
+                inode_unref(link_inode);
+                if (ret) {
+                    gf_msg(this->name, GF_LOG_ERROR, -ret,
+                           SHARD_MSG_SHARDS_DELETION_FAILED,
+                           "Failed to clean up shards of gfid %s",
+                           entry->d_name);
+                    continue;
                 }
+                gf_msg(this->name, GF_LOG_INFO, 0,
+                       SHARD_MSG_SHARD_DELETION_COMPLETED,
+                       "Deleted "
+                       "shards of gfid=%s from backend",
+                       entry->d_name);
+            }
+            gf_dirent_free(&entries);
+            if (ret)
+                break;
+        }
+    }
+    ret = 0;
+    loc_wipe(&loc);
+    return ret;
+
+err:
+    LOCK(&priv->lock);
+    {
+        priv->bg_del_state = SHARD_BG_DELETION_NONE;
+    }
+    UNLOCK(&priv->lock);
+    loc_wipe(&loc);
+    return ret;
+}
+
+int
+shard_unlock_inodelk_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                         int32_t op_ret, int32_t op_errno, dict_t *xdata)
+{
+    if (op_ret)
+        gf_msg(this->name, GF_LOG_ERROR, op_errno, SHARD_MSG_FOP_FAILED,
+               "Unlock failed. Please check brick logs for "
+               "more details");
+    SHARD_STACK_DESTROY(frame);
+    return 0;
+}
 
-                STACK_WIND_COOKIE (frame, shard_writev_do_cbk, anon_fd,
-                                   FIRST_CHILD(this),
-                                   FIRST_CHILD(this)->fops->writev, anon_fd,
-                                   vec, count, shard_offset, local->flags,
-                                   local->iobref, local->xattr_req);
-                GF_FREE (vec);
-                vec = NULL;
-                orig_offset += write_size;
-                vec_offset += write_size;
-next:
-                cur_block++;
-                i++;
-                call_count--;
+int
+shard_unlock_inodelk(call_frame_t *frame, xlator_t *this)
+{
+    loc_t *loc = NULL;
+    call_frame_t *lk_frame = NULL;
+    shard_local_t *local = NULL;
+    shard_local_t *lk_local = NULL;
+    shard_inodelk_t *lock = NULL;
+
+    local = frame->local;
+    lk_frame = local->inodelk_frame;
+    lk_local = lk_frame->local;
+    local->inodelk_frame = NULL;
+    loc = &local->int_inodelk.loc;
+    lock = &lk_local->int_inodelk;
+    lock->flock.l_type = F_UNLCK;
+
+    STACK_WIND(lk_frame, shard_unlock_inodelk_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->inodelk, lock->domain, loc, F_SETLK,
+               &lock->flock, NULL);
+    local->int_inodelk.acquired_lock = _gf_false;
+    return 0;
+}
+
+int
+shard_rename_src_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                     int32_t op_ret, int32_t op_errno, struct iatt *buf,
+                     struct iatt *preoldparent, struct iatt *postoldparent,
+                     struct iatt *prenewparent, struct iatt *postnewparent,
+                     dict_t *xdata);
+int
+shard_rename_src_base_file(call_frame_t *frame, xlator_t *this)
+{
+    int ret = 0;
+    loc_t *dst_loc = NULL;
+    loc_t tmp_loc = {
+        0,
+    };
+    shard_local_t *local = frame->local;
+
+    if (local->dst_block_size) {
+        tmp_loc.parent = inode_ref(local->loc2.parent);
+        ret = inode_path(tmp_loc.parent, local->loc2.name,
+                         (char **)&tmp_loc.path);
+        if (ret < 0) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, SHARD_MSG_INODE_PATH_FAILED,
+                   "Inode path failed"
+                   " on pargfid=%s bname=%s",
+                   uuid_utoa(tmp_loc.parent->gfid), local->loc2.name);
+            local->op_ret = -1;
+            local->op_errno = ENOMEM;
+            goto err;
         }
-        return 0;
+
+        tmp_loc.name = strrchr(tmp_loc.path, '/');
+        if (tmp_loc.name)
+            tmp_loc.name++;
+        dst_loc = &tmp_loc;
+    } else {
+        dst_loc = &local->loc2;
+    }
+
+    /* To-Do: Request open-fd count on dst base file */
+    STACK_WIND(frame, shard_rename_src_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->rename, &local->loc, dst_loc,
+               local->xattr_req);
+    loc_wipe(&tmp_loc);
+    return 0;
+err:
+    loc_wipe(&tmp_loc);
+    shard_common_failure_unwind(local->fop, frame, local->op_ret,
+                                local->op_errno);
+    return 0;
 }
 
 int
-shard_lookup_base_file_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                            int32_t op_ret, int32_t op_errno, inode_t *inode,
-                            struct iatt *buf, dict_t *xdata,
-                            struct iatt *postparent)
+shard_unlink_base_file(call_frame_t *frame, xlator_t *this);
+
+int
+shard_set_size_attrs_on_marker_file_cbk(call_frame_t *frame, void *cookie,
+                                        xlator_t *this, int32_t op_ret,
+                                        int32_t op_errno, dict_t *dict,
+                                        dict_t *xdata)
 {
-        shard_local_t     *local  = NULL;
+    shard_priv_t *priv = NULL;
+    shard_local_t *local = NULL;
+
+    priv = this->private;
+    local = frame->local;
+    if (op_ret < 0) {
+        gf_msg(this->name, GF_LOG_ERROR, op_errno, SHARD_MSG_FOP_FAILED,
+               "Xattrop on marker file failed "
+               "while performing %s; entry gfid=%s",
+               gf_fop_string(local->fop), local->newloc.name);
+        goto err;
+    }
+
+    inode_unlink(local->newloc.inode, priv->dot_shard_rm_inode,
+                 local->newloc.name);
+
+    if (local->fop == GF_FOP_UNLINK)
+        shard_unlink_base_file(frame, this);
+    else if (local->fop == GF_FOP_RENAME)
+        shard_rename_src_base_file(frame, this);
+    return 0;
+err:
+    shard_common_failure_unwind(local->fop, frame, op_ret, op_errno);
+    return 0;
+}
 
-        local = frame->local;
+int
+shard_set_size_attrs_on_marker_file(call_frame_t *frame, xlator_t *this)
+{
+    int op_errno = ENOMEM;
+    uint64_t bs = 0;
+    dict_t *xdata = NULL;
+    shard_local_t *local = NULL;
+
+    local = frame->local;
+    xdata = dict_new();
+    if (!xdata)
+        goto err;
+
+    if (local->fop == GF_FOP_UNLINK)
+        bs = local->block_size;
+    else if (local->fop == GF_FOP_RENAME)
+        bs = local->dst_block_size;
+    SHARD_INODE_CREATE_INIT(this, bs, xdata, &local->newloc,
+                            local->prebuf.ia_size, 0, err);
+    STACK_WIND(frame, shard_set_size_attrs_on_marker_file_cbk,
+               FIRST_CHILD(this), FIRST_CHILD(this)->fops->xattrop,
+               &local->newloc, GF_XATTROP_GET_AND_SET, xdata, NULL);
+    dict_unref(xdata);
+    return 0;
+err:
+    if (xdata)
+        dict_unref(xdata);
+    shard_common_failure_unwind(local->fop, frame, -1, op_errno);
+    return 0;
+}
 
-        if (op_ret < 0)
-                goto unwind;
+int
+shard_lookup_marker_file_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                             int32_t op_ret, int32_t op_errno, inode_t *inode,
+                             struct iatt *buf, dict_t *xdata,
+                             struct iatt *postparent)
+{
+    inode_t *linked_inode = NULL;
+    shard_priv_t *priv = NULL;
+    shard_local_t *local = NULL;
+
+    local = frame->local;
+    priv = this->private;
+
+    if (op_ret < 0) {
+        gf_msg(this->name, GF_LOG_ERROR, op_errno, SHARD_MSG_FOP_FAILED,
+               "Lookup on marker file failed "
+               "while performing %s; entry gfid=%s",
+               gf_fop_string(local->fop), local->newloc.name);
+        goto err;
+    }
+
+    linked_inode = inode_link(inode, priv->dot_shard_rm_inode,
+                              local->newloc.name, buf);
+    inode_unref(local->newloc.inode);
+    local->newloc.inode = linked_inode;
+    shard_set_size_attrs_on_marker_file(frame, this);
+    return 0;
+err:
+    shard_common_failure_unwind(local->fop, frame, op_ret, op_errno);
+    return 0;
+}
 
-        local->prebuf = *buf;
-        if (shard_modify_size_and_block_count (&local->prebuf, xdata)) {
-                op_ret = -1;
-                op_errno = EINVAL;
-                goto unwind;
+int
+shard_lookup_marker_file(call_frame_t *frame, xlator_t *this)
+{
+    int op_errno = ENOMEM;
+    dict_t *xattr_req = NULL;
+    shard_local_t *local = NULL;
+
+    local = frame->local;
+
+    xattr_req = shard_create_gfid_dict(local->xattr_req);
+    if (!xattr_req)
+        goto err;
+
+    STACK_WIND(frame, shard_lookup_marker_file_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->lookup, &local->newloc, xattr_req);
+    dict_unref(xattr_req);
+    return 0;
+err:
+    shard_common_failure_unwind(local->fop, frame, -1, op_errno);
+    return 0;
+}
+
+int
+shard_create_marker_file_under_remove_me_cbk(
+    call_frame_t *frame, void *cookie, xlator_t *this, int32_t op_ret,
+    int32_t op_errno, inode_t *inode, struct iatt *buf, struct iatt *preparent,
+    struct iatt *postparent, dict_t *xdata)
+{
+    inode_t *linked_inode = NULL;
+    shard_priv_t *priv = NULL;
+    shard_local_t *local = NULL;
+
+    local = frame->local;
+    priv = this->private;
+
+    SHARD_UNSET_ROOT_FS_ID(frame, local);
+    if (op_ret < 0) {
+        if ((op_errno != EEXIST) && (op_errno != ENODATA)) {
+            local->op_ret = op_ret;
+            local->op_errno = op_errno;
+            gf_msg(this->name, GF_LOG_ERROR, op_errno, SHARD_MSG_FOP_FAILED,
+                   "Marker file creation "
+                   "failed while performing %s; entry gfid=%s",
+                   gf_fop_string(local->fop), local->newloc.name);
+            goto err;
+        } else {
+            shard_lookup_marker_file(frame, this);
+            return 0;
         }
+    }
+
+    linked_inode = inode_link(inode, priv->dot_shard_rm_inode,
+                              local->newloc.name, buf);
+    inode_unref(local->newloc.inode);
+    local->newloc.inode = linked_inode;
+
+    if (local->fop == GF_FOP_UNLINK)
+        shard_unlink_base_file(frame, this);
+    else if (local->fop == GF_FOP_RENAME)
+        shard_rename_src_base_file(frame, this);
+    return 0;
+err:
+    shard_common_failure_unwind(local->fop, frame, -1, local->op_errno);
+    return 0;
+}
 
-        local->postbuf = local->prebuf;
+int
+shard_create_marker_file_under_remove_me(call_frame_t *frame, xlator_t *this,
+                                         loc_t *loc)
+{
+    int ret = 0;
+    int op_errno = ENOMEM;
+    uint64_t bs = 0;
+    char g1[64] = {
+        0,
+    };
+    char g2[64] = {
+        0,
+    };
+    dict_t *xattr_req = NULL;
+    shard_priv_t *priv = NULL;
+    shard_local_t *local = NULL;
+
+    priv = this->private;
+    local = frame->local;
+
+    SHARD_SET_ROOT_FS_ID(frame, local);
+
+    xattr_req = shard_create_gfid_dict(local->xattr_req);
+    if (!xattr_req)
+        goto err;
+
+    local->newloc.inode = inode_new(this->itable);
+    local->newloc.parent = inode_ref(priv->dot_shard_rm_inode);
+    ret = inode_path(local->newloc.parent, uuid_utoa(loc->inode->gfid),
+                     (char **)&local->newloc.path);
+    if (ret < 0) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, SHARD_MSG_INODE_PATH_FAILED,
+               "Inode path failed on "
+               "pargfid=%s bname=%s",
+               uuid_utoa_r(priv->dot_shard_rm_gfid, g1),
+               uuid_utoa_r(loc->inode->gfid, g2));
+        goto err;
+    }
+    local->newloc.name = strrchr(local->newloc.path, '/');
+    if (local->newloc.name)
+        local->newloc.name++;
+
+    if (local->fop == GF_FOP_UNLINK)
+        bs = local->block_size;
+    else if (local->fop == GF_FOP_RENAME)
+        bs = local->dst_block_size;
+
+    SHARD_INODE_CREATE_INIT(this, bs, xattr_req, &local->newloc,
+                            local->prebuf.ia_size, 0, err);
+
+    STACK_WIND(frame, shard_create_marker_file_under_remove_me_cbk,
+               FIRST_CHILD(this), FIRST_CHILD(this)->fops->mknod,
+               &local->newloc, 0, 0, 0644, xattr_req);
+    dict_unref(xattr_req);
+    return 0;
 
-        shard_writev_do (frame, this);
+err:
+    if (xattr_req)
+        dict_unref(xattr_req);
+    shard_create_marker_file_under_remove_me_cbk(frame, 0, this, -1, op_errno,
+                                                 NULL, NULL, NULL, NULL, NULL);
+    return 0;
+}
 
-        return 0;
+int
+shard_unlock_entrylk(call_frame_t *frame, xlator_t *this);
 
-unwind:
-        SHARD_STACK_UNWIND (writev, frame, op_ret, op_errno, NULL, NULL, NULL);
-        return 0;
+int
+shard_unlink_base_file_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                           int32_t op_ret, int32_t op_errno,
+                           struct iatt *preparent, struct iatt *postparent,
+                           dict_t *xdata)
+{
+    int ret = 0;
+    shard_local_t *local = NULL;
+
+    local = frame->local;
+
+    if (op_ret < 0) {
+        local->op_ret = op_ret;
+        local->op_errno = op_errno;
+    } else {
+        shard_inode_ctx_set_refresh_flag(local->int_inodelk.loc.inode, this);
+        local->preoldparent = *preparent;
+        local->postoldparent = *postparent;
+        if (xdata)
+            local->xattr_rsp = dict_ref(xdata);
+        if (local->cleanup_required)
+            shard_start_background_deletion(this);
+    }
+
+    if (local->entrylk_frame) {
+        ret = shard_unlock_entrylk(frame, this);
+        if (ret < 0) {
+            local->op_ret = -1;
+            local->op_errno = -ret;
+        }
+    }
 
+    ret = shard_unlock_inodelk(frame, this);
+    if (ret < 0) {
+        local->op_ret = -1;
+        local->op_errno = -ret;
+    }
+
+    shard_unlink_cbk(frame, this);
+    return 0;
 }
 
 int
-shard_lookup_base_file (call_frame_t *frame, xlator_t *this)
+shard_unlink_base_file(call_frame_t *frame, xlator_t *this)
 {
-        fd_t               *fd    = NULL;
-        shard_local_t      *local = NULL;
-        dict_t             *xattr_req = NULL;
+    shard_local_t *local = frame->local;
+
+    /* To-Do: Request open-fd count on base file */
+    STACK_WIND(frame, shard_unlink_base_file_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->unlink, &local->loc, local->xflag,
+               local->xattr_req);
+    return 0;
+}
 
-        local = frame->local;
-        fd = local->fd;
+int
+shard_unlock_entrylk_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                         int32_t op_ret, int32_t op_errno, dict_t *xdata)
+{
+    if (op_ret)
+        gf_msg(this->name, GF_LOG_ERROR, op_errno, SHARD_MSG_FOP_FAILED,
+               "Unlock failed. Please check brick logs for "
+               "more details");
+    SHARD_STACK_DESTROY(frame);
+    return 0;
+}
 
-        xattr_req = dict_new ();
-        if (!xattr_req)
-                goto err;
+int
+shard_unlock_entrylk(call_frame_t *frame, xlator_t *this)
+{
+    loc_t *loc = NULL;
+    call_frame_t *lk_frame = NULL;
+    shard_local_t *local = NULL;
+    shard_local_t *lk_local = NULL;
+    shard_entrylk_t *lock = NULL;
+
+    local = frame->local;
+    lk_frame = local->entrylk_frame;
+    lk_local = lk_frame->local;
+    local->entrylk_frame = NULL;
+    lock = &lk_local->int_entrylk;
+    loc = &lock->loc;
+
+    STACK_WIND(lk_frame, shard_unlock_entrylk_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->entrylk, this->name, loc,
+               lk_local->int_entrylk.basename, ENTRYLK_UNLOCK, ENTRYLK_WRLCK,
+               NULL);
+    local->int_entrylk.acquired_lock = _gf_false;
+    return 0;
+}
 
-        local->loc.inode = inode_new (fd->inode->table);
-        gf_uuid_copy (local->loc.gfid, fd->inode->gfid);
+int
+shard_post_entrylk_fop_handler(call_frame_t *frame, xlator_t *this)
+{
+    shard_local_t *local = NULL;
+
+    local = frame->local;
+
+    switch (local->fop) {
+        case GF_FOP_UNLINK:
+        case GF_FOP_RENAME:
+            shard_create_marker_file_under_remove_me(frame, this,
+                                                     &local->int_inodelk.loc);
+            break;
+        default:
+            gf_msg(this->name, GF_LOG_WARNING, 0, SHARD_MSG_INVALID_FOP,
+                   "post-entrylk handler not defined. This case should not"
+                   " be hit");
+            break;
+    }
+    return 0;
+}
 
-        SHARD_MD_READ_FOP_INIT_REQ_DICT (this, xattr_req, fd->inode->gfid, err);
+int
+shard_acquire_entrylk_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                          int32_t op_ret, int32_t op_errno, dict_t *xdata)
+{
+    call_frame_t *main_frame = NULL;
+    shard_local_t *local = NULL;
+    shard_local_t *main_local = NULL;
 
-        STACK_WIND (frame, shard_lookup_base_file_cbk, FIRST_CHILD (this),
-                    FIRST_CHILD(this)->fops->lookup, &local->loc,
-                    xattr_req);
+    local = frame->local;
+    main_frame = local->main_frame;
+    main_local = main_frame->local;
 
-        dict_unref (xattr_req);
+    if (local->op_ret < 0) {
+        shard_common_failure_unwind(main_local->fop, main_frame, op_ret,
+                                    op_errno);
         return 0;
+    }
+    main_local->int_entrylk.acquired_lock = _gf_true;
+    shard_post_entrylk_fop_handler(main_frame, this);
+    return 0;
+}
 
+int
+shard_acquire_entrylk(call_frame_t *frame, xlator_t *this, inode_t *inode,
+                      uuid_t gfid)
+{
+    char gfid_str[GF_UUID_BUF_SIZE] = {
+        0,
+    };
+    shard_local_t *local = NULL;
+    shard_local_t *entrylk_local = NULL;
+    shard_entrylk_t *int_entrylk = NULL;
+    call_frame_t *entrylk_frame = NULL;
+
+    local = frame->local;
+    entrylk_frame = create_frame(this, this->ctx->pool);
+    if (!entrylk_frame) {
+        gf_msg(this->name, GF_LOG_WARNING, ENOMEM, SHARD_MSG_MEMALLOC_FAILED,
+               "Failed to create new frame "
+               "to lock marker file");
+        goto err;
+    }
+
+    entrylk_local = mem_get0(this->local_pool);
+    if (!entrylk_local) {
+        STACK_DESTROY(entrylk_frame->root);
+        goto err;
+    }
+
+    entrylk_frame->local = entrylk_local;
+    entrylk_local->main_frame = frame;
+    int_entrylk = &entrylk_local->int_entrylk;
+
+    int_entrylk->loc.inode = inode_ref(inode);
+    set_lk_owner_from_ptr(&entrylk_frame->root->lk_owner, entrylk_frame->root);
+    local->entrylk_frame = entrylk_frame;
+    gf_uuid_unparse(gfid, gfid_str);
+    int_entrylk->basename = gf_strdup(gfid_str);
+
+    STACK_WIND(entrylk_frame, shard_acquire_entrylk_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->entrylk, this->name, &int_entrylk->loc,
+               int_entrylk->basename, ENTRYLK_LOCK, ENTRYLK_WRLCK, NULL);
+    return 0;
 err:
-        if (xattr_req)
-                dict_unref (xattr_req);
-        SHARD_STACK_UNWIND (writev, frame, -1, ENOMEM, NULL, NULL, NULL);
+    shard_common_failure_unwind(local->fop, frame, -1, ENOMEM);
+    return 0;
+}
+
+int
+shard_post_lookup_base_shard_rm_handler(call_frame_t *frame, xlator_t *this)
+{
+    shard_local_t *local = NULL;
+    shard_priv_t *priv = NULL;
+
+    priv = this->private;
+    local = frame->local;
+
+    if (local->op_ret < 0) {
+        shard_common_failure_unwind(local->fop, frame, -1, local->op_errno);
         return 0;
+    }
+
+    if (local->prebuf.ia_nlink > 1) {
+        gf_msg_debug(this->name, 0,
+                     "link count on %s > 1:%d, "
+                     "performing rename()/unlink()",
+                     local->int_inodelk.loc.path, local->prebuf.ia_nlink);
+        if (local->fop == GF_FOP_RENAME)
+            shard_rename_src_base_file(frame, this);
+        else if (local->fop == GF_FOP_UNLINK)
+            shard_unlink_base_file(frame, this);
+    } else {
+        gf_msg_debug(this->name, 0,
+                     "link count on %s = 1, creating "
+                     "file under .remove_me",
+                     local->int_inodelk.loc.path);
+        local->cleanup_required = _gf_true;
+        shard_acquire_entrylk(frame, this, priv->dot_shard_rm_inode,
+                              local->prebuf.ia_gfid);
+    }
+    return 0;
+}
+
+int
+shard_post_inodelk_fop_handler(call_frame_t *frame, xlator_t *this)
+{
+    shard_local_t *local = NULL;
+
+    local = frame->local;
+
+    switch (local->fop) {
+        case GF_FOP_UNLINK:
+        case GF_FOP_RENAME:
+            shard_refresh_base_file(frame, this, &local->int_inodelk.loc, NULL,
+                                    shard_post_lookup_base_shard_rm_handler);
+            break;
+        default:
+            gf_msg(this->name, GF_LOG_WARNING, 0, SHARD_MSG_INVALID_FOP,
+                   "post-inodelk handler not defined. This case should not"
+                   " be hit");
+            break;
+    }
+    return 0;
+}
 
+int
+shard_acquire_inodelk_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                          int32_t op_ret, int32_t op_errno, dict_t *xdata)
+{
+    call_frame_t *main_frame = NULL;
+    shard_local_t *local = NULL;
+    shard_local_t *main_local = NULL;
+
+    local = frame->local;
+    main_frame = local->main_frame;
+    main_local = main_frame->local;
+
+    if (local->op_ret < 0) {
+        shard_common_failure_unwind(main_local->fop, main_frame, op_ret,
+                                    op_errno);
+        return 0;
+    }
+    main_local->int_inodelk.acquired_lock = _gf_true;
+    shard_post_inodelk_fop_handler(main_frame, this);
+    return 0;
 }
 
-void
-shard_link_block_inode (shard_local_t *local, int block_num, inode_t *inode,
-                        struct iatt *buf)
+int
+shard_acquire_inodelk(call_frame_t *frame, xlator_t *this, loc_t *loc)
 {
-        char            block_bname[256] = {0,};
-        inode_t        *linked_inode     = NULL;
-        shard_priv_t   *priv             = NULL;
+    call_frame_t *lk_frame = NULL;
+    shard_local_t *local = NULL;
+    shard_local_t *lk_local = NULL;
+    shard_inodelk_t *int_inodelk = NULL;
+
+    local = frame->local;
+    lk_frame = create_frame(this, this->ctx->pool);
+    if (!lk_frame) {
+        gf_msg(this->name, GF_LOG_WARNING, ENOMEM, SHARD_MSG_MEMALLOC_FAILED,
+               "Failed to create new frame "
+               "to lock base shard");
+        goto err;
+    }
+    lk_local = mem_get0(this->local_pool);
+    if (!lk_local) {
+        STACK_DESTROY(lk_frame->root);
+        goto err;
+    }
+
+    lk_frame->local = lk_local;
+    lk_local->main_frame = frame;
+    int_inodelk = &lk_local->int_inodelk;
+
+    int_inodelk->flock.l_len = 0;
+    int_inodelk->flock.l_start = 0;
+    int_inodelk->domain = this->name;
+    int_inodelk->flock.l_type = F_WRLCK;
+    loc_copy(&local->int_inodelk.loc, loc);
+    set_lk_owner_from_ptr(&lk_frame->root->lk_owner, lk_frame->root);
+    local->inodelk_frame = lk_frame;
+
+    STACK_WIND(lk_frame, shard_acquire_inodelk_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->inodelk, int_inodelk->domain,
+               &local->int_inodelk.loc, F_SETLKW, &int_inodelk->flock, NULL);
+    return 0;
+err:
+    shard_common_failure_unwind(local->fop, frame, -1, ENOMEM);
+    return 0;
+}
 
-        priv = THIS->private;
+int
+shard_post_mkdir_rm_handler(call_frame_t *frame, xlator_t *this)
+{
+    loc_t *loc = NULL;
+    shard_local_t *local = NULL;
 
-        shard_make_block_bname (block_num, local->fd->inode->gfid, block_bname,
-                                sizeof (block_bname));
+    local = frame->local;
 
-        linked_inode = inode_link (inode, priv->dot_shard_inode, block_bname,
-                                   buf);
-        inode_lookup (linked_inode);
-        local->inode_list[block_num - local->first_block] = linked_inode;
-        /* Defer unref'ing the inodes until write is complete to prevent
-         * them from getting purged. These inodes are unref'd in the event of
-         * a failure or after successfull fop completion in shard_local_wipe().
-         */
+    if (local->op_ret < 0) {
+        shard_common_failure_unwind(local->fop, frame, -1, local->op_errno);
+        return 0;
+    }
+    if (local->fop == GF_FOP_UNLINK)
+        loc = &local->loc;
+    else if (local->fop == GF_FOP_RENAME)
+        loc = &local->loc2;
+    shard_acquire_inodelk(frame, this, loc);
+    return 0;
 }
 
 int
-shard_writev_lookup_shards_cbk (call_frame_t *frame, void *cookie,
-                                xlator_t *this, int32_t op_ret,
-                                int32_t op_errno, inode_t *inode,
-                                struct iatt *buf, dict_t *xdata,
-                                struct iatt *postparent)
+shard_mkdir_internal_dir(call_frame_t *frame, xlator_t *this,
+                         shard_post_resolve_fop_handler_t handler,
+                         shard_internal_dir_type_t type);
+int
+shard_pre_mkdir_rm_handler(call_frame_t *frame, xlator_t *this)
 {
-        int             call_count      = 0;
-        int             shard_block_num = (long) cookie;
-        shard_local_t  *local           = NULL;
+    shard_local_t *local = NULL;
 
-        local = frame->local;
+    local = frame->local;
 
-        if (op_ret < 0) {
-                local->op_ret = op_ret;
-                local->op_errno = op_errno;
-                goto done;
+    if (local->op_ret < 0) {
+        shard_common_failure_unwind(local->fop, frame, -1, local->op_errno);
+        return 0;
+    }
+    shard_mkdir_internal_dir(frame, this, shard_post_mkdir_rm_handler,
+                             SHARD_INTERNAL_DIR_DOT_SHARD_REMOVE_ME);
+    return 0;
+}
+
+void
+shard_begin_rm_resolution(call_frame_t *frame, xlator_t *this)
+{
+    shard_priv_t *priv = NULL;
+    shard_local_t *local = NULL;
+
+    priv = this->private;
+    local = frame->local;
+
+    local->dot_shard_rm_loc.inode = inode_find(this->itable,
+                                               priv->dot_shard_rm_gfid);
+    if (!local->dot_shard_rm_loc.inode) {
+        local->dot_shard_loc.inode = inode_find(this->itable,
+                                                priv->dot_shard_gfid);
+        if (!local->dot_shard_loc.inode) {
+            shard_mkdir_internal_dir(frame, this, shard_pre_mkdir_rm_handler,
+                                     SHARD_INTERNAL_DIR_DOT_SHARD);
+        } else {
+            local->post_res_handler = shard_pre_mkdir_rm_handler;
+            shard_refresh_internal_dir(frame, this,
+                                       SHARD_INTERNAL_DIR_DOT_SHARD);
         }
+    } else {
+        local->post_res_handler = shard_post_mkdir_rm_handler;
+        shard_refresh_internal_dir(frame, this,
+                                   SHARD_INTERNAL_DIR_DOT_SHARD_REMOVE_ME);
+    }
+}
 
-        shard_link_block_inode (local, shard_block_num, inode, buf);
+int
+shard_unlink(call_frame_t *frame, xlator_t *this, loc_t *loc, int xflag,
+             dict_t *xdata)
+{
+    int ret = -1;
+    uint64_t block_size = 0;
+    shard_local_t *local = NULL;
+
+    ret = shard_inode_ctx_get_block_size(loc->inode, this, &block_size);
+    if ((ret) && (!IA_ISLNK(loc->inode->ia_type))) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, SHARD_MSG_INODE_CTX_GET_FAILED,
+               "Failed to get block "
+               "size from inode ctx of %s",
+               uuid_utoa(loc->inode->gfid));
+        goto err;
+    }
+
+    if (!block_size || frame->root->pid == GF_CLIENT_PID_GSYNCD) {
+        STACK_WIND(frame, default_unlink_cbk, FIRST_CHILD(this),
+                   FIRST_CHILD(this)->fops->unlink, loc, xflag, xdata);
+        return 0;
+    }
+
+    local = mem_get0(this->local_pool);
+    if (!local)
+        goto err;
+
+    frame->local = local;
+
+    loc_copy(&local->loc, loc);
+    local->xflag = xflag;
+    local->xattr_req = (xdata) ? dict_ref(xdata) : dict_new();
+    local->block_size = block_size;
+    local->resolver_base_inode = loc->inode;
+    local->fop = GF_FOP_UNLINK;
+    if (!this->itable)
+        this->itable = (local->loc.inode)->table;
+
+    local->resolve_not = _gf_true;
+    shard_begin_rm_resolution(frame, this);
+    return 0;
+err:
+    shard_common_failure_unwind(GF_FOP_UNLINK, frame, -1, ENOMEM);
+    return 0;
+}
 
-done:
-        call_count = shard_call_count_return (frame);
-        if (call_count == 0) {
-                if (local->op_ret < 0)
-                        goto unwind;
-                else
-                        shard_lookup_base_file (frame, this);
+int
+shard_post_rename_lookup_handler(call_frame_t *frame, xlator_t *this)
+{
+    shard_rename_cbk(frame, this);
+    return 0;
+}
+
+int
+shard_rename_src_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                     int32_t op_ret, int32_t op_errno, struct iatt *buf,
+                     struct iatt *preoldparent, struct iatt *postoldparent,
+                     struct iatt *prenewparent, struct iatt *postnewparent,
+                     dict_t *xdata)
+{
+    int ret = 0;
+    shard_local_t *local = NULL;
+
+    local = frame->local;
+
+    if (op_ret < 0) {
+        local->op_ret = op_ret;
+        local->op_errno = op_errno;
+        goto err;
+    }
+    /* Set ctx->refresh to TRUE to force a lookup on disk when
+     * shard_lookup_base_file() is called next to refresh the hard link
+     * count in ctx. Note that this is applicable only to the case where
+     * the rename dst is already existent and sharded.
+     */
+    if ((local->dst_block_size) && (!local->cleanup_required))
+        shard_inode_ctx_set_refresh_flag(local->int_inodelk.loc.inode, this);
+
+    local->prebuf = *buf;
+    local->preoldparent = *preoldparent;
+    local->postoldparent = *postoldparent;
+    local->prenewparent = *prenewparent;
+    local->postnewparent = *postnewparent;
+    if (xdata)
+        local->xattr_rsp = dict_ref(xdata);
+
+    if (local->dst_block_size) {
+        if (local->entrylk_frame) {
+            ret = shard_unlock_entrylk(frame, this);
+            if (ret < 0) {
+                local->op_ret = -1;
+                local->op_errno = -ret;
+            }
         }
+
+        ret = shard_unlock_inodelk(frame, this);
+        if (ret < 0) {
+            local->op_ret = -1;
+            local->op_errno = -ret;
+            goto err;
+        }
+        if (local->cleanup_required)
+            shard_start_background_deletion(this);
+    }
+
+    /* Now the base file of src, if sharded, is looked up to gather ia_size
+     * and ia_blocks.*/
+    if (local->block_size) {
+        local->tmp_loc.inode = inode_new(this->itable);
+        gf_uuid_copy(local->tmp_loc.gfid, (local->loc.inode)->gfid);
+        shard_refresh_base_file(frame, this, &local->tmp_loc, NULL,
+                                shard_post_rename_lookup_handler);
+    } else {
+        shard_rename_cbk(frame, this);
+    }
+    return 0;
+err:
+    shard_common_failure_unwind(local->fop, frame, local->op_ret,
+                                local->op_errno);
+    return 0;
+}
+
+int
+shard_post_lookup_dst_base_file_handler(call_frame_t *frame, xlator_t *this)
+{
+    shard_local_t *local = NULL;
+
+    local = frame->local;
+
+    if (local->op_ret < 0) {
+        shard_common_failure_unwind(local->fop, frame, local->op_ret,
+                                    local->op_errno);
         return 0;
+    }
+
+    /* Save dst base file attributes into postbuf so the information is not
+     * lost when it is overwritten after lookup on base file of src in
+     * shard_lookup_base_file_cbk().
+     */
+    local->postbuf = local->prebuf;
+    shard_rename_src_base_file(frame, this);
+    return 0;
+}
 
-unwind:
-        SHARD_STACK_UNWIND (writev, frame, local->op_ret, local->op_errno, NULL,
-                            NULL, NULL);
+int
+shard_rename(call_frame_t *frame, xlator_t *this, loc_t *oldloc, loc_t *newloc,
+             dict_t *xdata)
+{
+    int ret = -1;
+    uint64_t block_size = 0;
+    uint64_t dst_block_size = 0;
+    shard_local_t *local = NULL;
+
+    if (IA_ISDIR(oldloc->inode->ia_type)) {
+        STACK_WIND(frame, default_rename_cbk, FIRST_CHILD(this),
+                   FIRST_CHILD(this)->fops->rename, oldloc, newloc, xdata);
         return 0;
+    }
+
+    ret = shard_inode_ctx_get_block_size(oldloc->inode, this, &block_size);
+    if ((ret) && (!IA_ISLNK(oldloc->inode->ia_type))) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, SHARD_MSG_INODE_CTX_GET_FAILED,
+               "Failed to get block "
+               "size from inode ctx of %s",
+               uuid_utoa(oldloc->inode->gfid));
+        goto err;
+    }
+
+    if (newloc->inode)
+        ret = shard_inode_ctx_get_block_size(newloc->inode, this,
+                                             &dst_block_size);
+
+    /* The following stack_wind covers the case where:
+     * a. the src file is not sharded and dst doesn't exist, OR
+     * b. the src and dst both exist but are not sharded.
+     */
+    if (((!block_size) && (!dst_block_size)) ||
+        frame->root->pid == GF_CLIENT_PID_GSYNCD) {
+        STACK_WIND(frame, default_rename_cbk, FIRST_CHILD(this),
+                   FIRST_CHILD(this)->fops->rename, oldloc, newloc, xdata);
+        return 0;
+    }
+
+    local = mem_get0(this->local_pool);
+    if (!local)
+        goto err;
+
+    frame->local = local;
+    loc_copy(&local->loc, oldloc);
+    loc_copy(&local->loc2, newloc);
+    local->resolver_base_inode = newloc->inode;
+    local->fop = GF_FOP_RENAME;
+    local->xattr_req = (xdata) ? dict_ref(xdata) : dict_new();
+    if (!local->xattr_req)
+        goto err;
+
+    local->block_size = block_size;
+    local->dst_block_size = dst_block_size;
+    if (!this->itable)
+        this->itable = (local->loc.inode)->table;
+    local->resolve_not = _gf_true;
+
+    /* The following if-block covers the case where the dst file exists
+     * and is sharded.
+     */
+    if (local->dst_block_size) {
+        shard_begin_rm_resolution(frame, this);
+    } else {
+        /* The following block covers the case where the dst either doesn't
+         * exist or is NOT sharded but the src is sharded. In this case, shard
+         * xlator would go ahead and rename src to dst. Once done, it would also
+         * lookup the base shard of src to get the ia_size and ia_blocks xattr
+         * values.
+         */
+        shard_rename_src_base_file(frame, this);
+    }
+    return 0;
+
+err:
+    shard_common_failure_unwind(GF_FOP_RENAME, frame, -1, ENOMEM);
+    return 0;
 }
 
-dict_t*
-shard_create_gfid_dict (dict_t *dict)
+int
+shard_create_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                 int32_t op_ret, int32_t op_errno, fd_t *fd, inode_t *inode,
+                 struct iatt *stbuf, struct iatt *preparent,
+                 struct iatt *postparent, dict_t *xdata)
 {
-        int    ret   = 0;
-        dict_t *new  = NULL;
-        uuid_t *gfid = NULL;
+    int ret = -1;
+    shard_local_t *local = NULL;
 
-        new = dict_copy_with_ref (dict, NULL);
-        if (!new)
-                return NULL;
+    local = frame->local;
 
-        gfid = GF_CALLOC (1, sizeof (uuid_t), gf_common_mt_char);
-        if (!gfid) {
-                ret = -1;
-                goto out;
-        }
+    if (op_ret == -1)
+        goto unwind;
+
+    ret = shard_inode_ctx_set(inode, this, stbuf, local->block_size,
+                              SHARD_ALL_MASK);
+    if (ret)
+        gf_msg(this->name, GF_LOG_WARNING, 0, SHARD_MSG_INODE_CTX_SET_FAILED,
+               "Failed to set inode "
+               "ctx for %s",
+               uuid_utoa(inode->gfid));
 
-        gf_uuid_generate (*gfid);
+unwind:
+    SHARD_STACK_UNWIND(create, frame, op_ret, op_errno, fd, inode, stbuf,
+                       preparent, postparent, xdata);
+    return 0;
+}
 
-        ret = dict_set_dynptr (new, "gfid-req", gfid, sizeof (uuid_t));
+int
+shard_create(call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t flags,
+             mode_t mode, mode_t umask, fd_t *fd, dict_t *xdata)
+{
+    shard_priv_t *priv = NULL;
+    shard_local_t *local = NULL;
+
+    priv = this->private;
+    local = mem_get0(this->local_pool);
+    if (!local)
+        goto err;
+
+    frame->local = local;
+    local->block_size = priv->block_size;
+
+    if (!__is_gsyncd_on_shard_dir(frame, loc)) {
+        SHARD_INODE_CREATE_INIT(this, local->block_size, xdata, loc, 0, 0, err);
+    }
+
+    STACK_WIND(frame, shard_create_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->create, loc, flags, mode, umask, fd,
+               xdata);
+    return 0;
+err:
+    shard_common_failure_unwind(GF_FOP_CREATE, frame, -1, ENOMEM);
+    return 0;
+}
+
+int
+shard_open_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+               int32_t op_ret, int32_t op_errno, fd_t *fd, dict_t *xdata)
+{
+    /* To-Do: Handle open with O_TRUNC under locks */
+    SHARD_STACK_UNWIND(open, frame, op_ret, op_errno, fd, xdata);
+    return 0;
+}
+
+int
+shard_open(call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t flags,
+           fd_t *fd, dict_t *xdata)
+{
+    STACK_WIND(frame, shard_open_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->open, loc, flags, fd, xdata);
+    return 0;
+}
+
+int
+shard_readv_do_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                   int32_t op_ret, int32_t op_errno, struct iovec *vector,
+                   int32_t count, struct iatt *stbuf, struct iobref *iobref,
+                   dict_t *xdata)
+{
+    int i = 0;
+    int call_count = 0;
+    void *address = NULL;
+    uint64_t block_num = 0;
+    off_t off = 0;
+    struct iovec vec = {
+        0,
+    };
+    shard_local_t *local = NULL;
+    fd_t *anon_fd = cookie;
+    shard_inode_ctx_t *ctx = NULL;
+
+    local = frame->local;
+
+    /* If shard has already seen a failure here before, there is no point
+     * in aggregating subsequent reads, so just go to out.
+     */
+    if (local->op_ret < 0)
+        goto out;
+
+    if (op_ret < 0) {
+        local->op_ret = op_ret;
+        local->op_errno = op_errno;
+        goto out;
+    }
+
+    if (local->op_ret >= 0)
+        local->op_ret += op_ret;
+
+    shard_inode_ctx_get(anon_fd->inode, this, &ctx);
+    block_num = ctx->block_num;
+
+    if (block_num == local->first_block) {
+        address = local->iobuf->ptr;
+    } else {
+        /* else
+         * address to start writing to = beginning of buffer +
+         *                    number of bytes until end of first block +
+         *                    + block_size times number of blocks
+         *                    between the current block and the first
+         */
+        address = (char *)local->iobuf->ptr +
+                  (local->block_size - (local->offset % local->block_size)) +
+                  ((block_num - local->first_block - 1) * local->block_size);
+    }
+
+    for (i = 0; i < count; i++) {
+        address = (char *)address + off;
+        memcpy(address, vector[i].iov_base, vector[i].iov_len);
+        off += vector[i].iov_len;
+    }
 
 out:
-        if (ret) {
-                dict_unref (new);
-                new = NULL;
-                GF_FREE (gfid);
+    if (anon_fd)
+        fd_unref(anon_fd);
+    call_count = shard_call_count_return(frame);
+    if (call_count == 0) {
+        SHARD_UNSET_ROOT_FS_ID(frame, local);
+        if (local->op_ret < 0) {
+            shard_common_failure_unwind(GF_FOP_READ, frame, local->op_ret,
+                                        local->op_errno);
+        } else {
+            if (xdata)
+                local->xattr_rsp = dict_ref(xdata);
+            vec.iov_base = local->iobuf->ptr;
+            if (local->offset + local->req_size > local->prebuf.ia_size)
+                local->total_size = local->prebuf.ia_size - local->offset;
+            vec.iov_len = local->total_size;
+            local->op_ret = local->total_size;
+            SHARD_STACK_UNWIND(readv, frame, local->op_ret, local->op_errno,
+                               &vec, 1, &local->prebuf, local->iobref,
+                               local->xattr_rsp);
+            return 0;
         }
+    }
 
-        return new;
-}
-
-int
-shard_writev_lookup_shards (call_frame_t *frame, xlator_t *this)
-{
-        int            i              = 0;
-        int            ret            = 0;
-        int            call_count     = 0;
-        int32_t        shard_idx_iter = 0;
-        int            last_block     = 0;
-        char           path[PATH_MAX] = {0,};
-        char          *bname          = NULL;
-        loc_t          loc            = {0,};
-        fd_t          *fd             = NULL;
-        shard_local_t *local          = NULL;
-        shard_priv_t  *priv           = NULL;
-        gf_boolean_t   wind_failed    = _gf_false;
-        dict_t        *xattr_req      = NULL;
-
-        priv = this->private;
-        local = frame->local;
-        fd = local->fd;
-        call_count = local->call_count;
-        shard_idx_iter = local->first_block;
-        last_block = local->last_block;
-
-        while (shard_idx_iter <= last_block) {
-                if (local->inode_list[i]) {
-                        i++;
-                        shard_idx_iter++;
-                        continue;
-                }
+    return 0;
+}
 
-                if (wind_failed) {
-                        shard_writev_lookup_shards_cbk (frame,
-                                                 (void *) (long) shard_idx_iter,
-                                                        this, -1, ENOMEM, NULL,
-                                                        NULL, NULL, NULL);
-                        goto next;
-                }
+int
+shard_readv_do(call_frame_t *frame, xlator_t *this)
+{
+    int i = 0;
+    int call_count = 0;
+    int last_block = 0;
+    int cur_block = 0;
+    off_t orig_offset = 0;
+    off_t shard_offset = 0;
+    size_t read_size = 0;
+    size_t remaining_size = 0;
+    fd_t *fd = NULL;
+    fd_t *anon_fd = NULL;
+    shard_local_t *local = NULL;
+    gf_boolean_t wind_failed = _gf_false;
+
+    local = frame->local;
+    fd = local->fd;
+
+    orig_offset = local->offset;
+    cur_block = local->first_block;
+    last_block = local->last_block;
+    remaining_size = local->total_size;
+    local->call_count = call_count = local->num_blocks;
+
+    SHARD_SET_ROOT_FS_ID(frame, local);
+
+    if (fd->flags & O_DIRECT)
+        local->flags = O_DIRECT;
+
+    while (cur_block <= last_block) {
+        if (wind_failed) {
+            shard_readv_do_cbk(frame, (void *)(long)0, this, -1, ENOMEM, NULL,
+                               0, NULL, NULL, NULL);
+            goto next;
+        }
 
-                shard_make_block_abspath (shard_idx_iter, fd->inode->gfid,
-                                          path, sizeof(path));
-
-                bname = strrchr (path, '/') + 1;
-                loc.inode = inode_new (this->itable);
-                loc.parent = inode_ref (priv->dot_shard_inode);
-                ret = inode_path (loc.parent, bname, (char **) &(loc.path));
-                if (ret < 0) {
-                        gf_log (this->name, GF_LOG_ERROR, "Inode path failed on"
-                                " %s", bname);
-                        local->op_ret = -1;
-                        local->op_errno = ENOMEM;
-                        loc_wipe (&loc);
-                        shard_writev_lookup_shards_cbk (frame,
-                                                 (void *) (long) shard_idx_iter,
-                                                        this, -1, ENOMEM, NULL,
-                                                        NULL, NULL, NULL);
-                        goto next;
-                }
+        shard_offset = orig_offset % local->block_size;
+        read_size = local->block_size - shard_offset;
+        if (read_size > remaining_size)
+            read_size = remaining_size;
 
-                loc.name = strrchr (loc.path, '/');
-                if (loc.name)
-                        loc.name++;
-
-                xattr_req = shard_create_gfid_dict (local->xattr_req);
-                if (!xattr_req) {
-                        local->op_ret = -1;
-                        local->op_errno = ENOMEM;
-                        wind_failed = _gf_true;
-                        loc_wipe (&loc);
-                        shard_writev_lookup_shards_cbk (frame,
-                                                 (void *) (long) shard_idx_iter,
-                                                        this, -1, ENOMEM, NULL,
-                                                        NULL, NULL, NULL);
-                        goto next;
-                }
+        remaining_size -= read_size;
 
-                STACK_WIND_COOKIE (frame, shard_writev_lookup_shards_cbk,
-                                   (void *) (long) shard_idx_iter,
-                                   FIRST_CHILD(this),
-                                   FIRST_CHILD(this)->fops->lookup, &loc,
-                                   xattr_req);
-                loc_wipe (&loc);
-                dict_unref (xattr_req);
-next:
-                shard_idx_iter++;
-                i++;
-
-                if (!--call_count)
-                        break;
+        if (cur_block == 0) {
+            anon_fd = fd_ref(fd);
+        } else {
+            anon_fd = fd_anonymous(local->inode_list[i]);
+            if (!anon_fd) {
+                local->op_ret = -1;
+                local->op_errno = ENOMEM;
+                wind_failed = _gf_true;
+                shard_readv_do_cbk(frame, (void *)(long)anon_fd, this, -1,
+                                   ENOMEM, NULL, 0, NULL, NULL, NULL);
+                goto next;
+            }
         }
 
-        return 0;
+        STACK_WIND_COOKIE(frame, shard_readv_do_cbk, anon_fd, FIRST_CHILD(this),
+                          FIRST_CHILD(this)->fops->readv, anon_fd, read_size,
+                          shard_offset, local->flags, local->xattr_req);
+
+        orig_offset += read_size;
+    next:
+        cur_block++;
+        i++;
+        call_count--;
+    }
+    return 0;
 }
 
 int
-shard_writev_mknod_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                        int32_t op_ret, int32_t op_errno, inode_t *inode,
-                        struct iatt *buf, struct iatt *preparent,
-                        struct iatt *postparent, dict_t *xdata)
+shard_common_mknod_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                       int32_t op_ret, int32_t op_errno, inode_t *inode,
+                       struct iatt *buf, struct iatt *preparent,
+                       struct iatt *postparent, dict_t *xdata)
 {
-        int             shard_block_num  = (long) cookie;
-        int             call_count       = 0;
-        shard_local_t  *local            = NULL;
+    int shard_block_num = (long)cookie;
+    int call_count = 0;
+    shard_local_t *local = NULL;
+
+    local = frame->local;
+
+    if (op_ret < 0) {
+        if (op_errno == EEXIST) {
+            LOCK(&frame->lock);
+            {
+                local->eexist_count++;
+            }
+            UNLOCK(&frame->lock);
+        } else {
+            local->op_ret = op_ret;
+            local->op_errno = op_errno;
+        }
+        gf_msg_debug(this->name, 0,
+                     "mknod of shard %d "
+                     "failed: %s",
+                     shard_block_num, strerror(op_errno));
+        goto done;
+    }
 
-        local = frame->local;
+    shard_link_block_inode(local, shard_block_num, inode, buf);
 
-        if (op_ret < 0) {
-                if (op_errno == EEXIST) {
-                        local->eexist_count++;
-                } else {
-                        local->op_ret = op_ret;
-                        local->op_errno = op_errno;
-                }
-                gf_log (this->name, GF_LOG_DEBUG, "SHARD WRITEV: mknod of "
-                        "shard %d failed: %s", shard_block_num,
-                        strerror (op_errno));
-                goto done;
+done:
+    call_count = shard_call_count_return(frame);
+    if (call_count == 0) {
+        SHARD_UNSET_ROOT_FS_ID(frame, local);
+        local->create_count = 0;
+        local->post_mknod_handler(frame, this);
+    }
+
+    return 0;
+}
+
+int
+shard_common_resume_mknod(call_frame_t *frame, xlator_t *this,
+                          shard_post_mknod_fop_handler_t post_mknod_handler)
+{
+    int i = 0;
+    int shard_idx_iter = 0;
+    int last_block = 0;
+    int ret = 0;
+    int call_count = 0;
+    char path[PATH_MAX] = {
+        0,
+    };
+    mode_t mode = 0;
+    char *bname = NULL;
+    shard_priv_t *priv = NULL;
+    shard_inode_ctx_t ctx_tmp = {
+        0,
+    };
+    shard_local_t *local = NULL;
+    gf_boolean_t wind_failed = _gf_false;
+    fd_t *fd = NULL;
+    loc_t loc = {
+        0,
+    };
+    dict_t *xattr_req = NULL;
+
+    local = frame->local;
+    priv = this->private;
+    fd = local->fd;
+    shard_idx_iter = local->first_block;
+    last_block = local->last_block;
+    call_count = local->call_count = local->create_count;
+    local->post_mknod_handler = post_mknod_handler;
+
+    SHARD_SET_ROOT_FS_ID(frame, local);
+
+    ret = shard_inode_ctx_get_all(fd->inode, this, &ctx_tmp);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, SHARD_MSG_INODE_CTX_GET_FAILED,
+               "Failed to get inode "
+               "ctx for %s",
+               uuid_utoa(fd->inode->gfid));
+        local->op_ret = -1;
+        local->op_errno = ENOMEM;
+        goto err;
+    }
+    mode = st_mode_from_ia(ctx_tmp.stat.ia_prot, ctx_tmp.stat.ia_type);
+
+    while (shard_idx_iter <= last_block) {
+        if (local->inode_list[i]) {
+            shard_idx_iter++;
+            i++;
+            continue;
         }
 
-        shard_link_block_inode (local, shard_block_num, inode, buf);
+        if (wind_failed) {
+            shard_common_mknod_cbk(frame, (void *)(long)shard_idx_iter, this,
+                                   -1, ENOMEM, NULL, NULL, NULL, NULL, NULL);
+            goto next;
+        }
 
-done:
-        call_count = shard_call_count_return (frame);
-        if (call_count == 0) {
-                if (local->op_ret < 0) {
-                        goto unwind;
-                } else {
-                        if (!local->eexist_count) {
-                                shard_lookup_base_file (frame, this);
-                        } else {
-                                local->call_count = local->eexist_count;
-                                shard_writev_lookup_shards (frame, this);
-                        }
-                }
+        shard_make_block_abspath(shard_idx_iter, fd->inode->gfid, path,
+                                 sizeof(path));
+
+        xattr_req = shard_create_gfid_dict(local->xattr_req);
+        if (!xattr_req) {
+            local->op_ret = -1;
+            local->op_errno = ENOMEM;
+            wind_failed = _gf_true;
+            shard_common_mknod_cbk(frame, (void *)(long)shard_idx_iter, this,
+                                   -1, ENOMEM, NULL, NULL, NULL, NULL, NULL);
+            goto next;
         }
+
+        bname = strrchr(path, '/') + 1;
+        loc.inode = inode_new(this->itable);
+        loc.parent = inode_ref(priv->dot_shard_inode);
+        ret = inode_path(loc.parent, bname, (char **)&(loc.path));
+        if (ret < 0 || !(loc.inode)) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, SHARD_MSG_INODE_PATH_FAILED,
+                   "Inode path failed"
+                   "on %s, base file gfid = %s",
+                   bname, uuid_utoa(fd->inode->gfid));
+            local->op_ret = -1;
+            local->op_errno = ENOMEM;
+            wind_failed = _gf_true;
+            loc_wipe(&loc);
+            dict_unref(xattr_req);
+            shard_common_mknod_cbk(frame, (void *)(long)shard_idx_iter, this,
+                                   -1, ENOMEM, NULL, NULL, NULL, NULL, NULL);
+            goto next;
+        }
+
+        loc.name = strrchr(loc.path, '/');
+        if (loc.name)
+            loc.name++;
+
+        STACK_WIND_COOKIE(frame, shard_common_mknod_cbk,
+                          (void *)(long)shard_idx_iter, FIRST_CHILD(this),
+                          FIRST_CHILD(this)->fops->mknod, &loc, mode,
+                          ctx_tmp.stat.ia_rdev, 0, xattr_req);
+        loc_wipe(&loc);
+        dict_unref(xattr_req);
+
+    next:
+        shard_idx_iter++;
+        i++;
+        if (!--call_count)
+            break;
+    }
+
+    return 0;
+err:
+    /*
+     * This block is for handling failure in shard_inode_ctx_get_all().
+     * Failures in the while-loop are handled within the loop.
+     */
+    SHARD_UNSET_ROOT_FS_ID(frame, local);
+    post_mknod_handler(frame, this);
+    return 0;
+}
+
+int
+shard_post_mknod_readv_handler(call_frame_t *frame, xlator_t *this);
+
+int
+shard_post_lookup_shards_readv_handler(call_frame_t *frame, xlator_t *this)
+{
+    shard_local_t *local = NULL;
+
+    local = frame->local;
+
+    if (local->op_ret < 0) {
+        shard_common_failure_unwind(GF_FOP_READ, frame, local->op_ret,
+                                    local->op_errno);
         return 0;
+    }
 
-unwind:
-        SHARD_STACK_UNWIND (writev, frame, local->op_ret, local->op_errno, NULL,
-                            NULL, NULL);
+    if (local->create_count) {
+        shard_common_resume_mknod(frame, this, shard_post_mknod_readv_handler);
+    } else {
+        shard_readv_do(frame, this);
+    }
+
+    return 0;
+}
+
+int
+shard_post_mknod_readv_handler(call_frame_t *frame, xlator_t *this)
+{
+    shard_local_t *local = NULL;
+
+    local = frame->local;
+
+    if (local->op_ret < 0) {
+        shard_common_failure_unwind(GF_FOP_READ, frame, local->op_ret,
+                                    local->op_errno);
         return 0;
+    }
+
+    if (!local->eexist_count) {
+        shard_readv_do(frame, this);
+    } else {
+        local->call_count = local->eexist_count;
+        shard_common_lookup_shards(frame, this, local->loc.inode,
+                                   shard_post_lookup_shards_readv_handler);
+    }
+    return 0;
 }
 
 int
-shard_writev_resume_mknod (call_frame_t *frame, xlator_t *this)
-{
-        int                 i              = 0;
-        int                 shard_idx_iter = 0;
-        int                 last_block     = 0;
-        int                 ret            = 0;
-        int                 call_count     = 0;
-        char                path[PATH_MAX] = {0,};
-        char               *bname          = NULL;
-        shard_priv_t       *priv           = NULL;
-        shard_inode_ctx_t   ctx_tmp        = {0,};
-        shard_local_t      *local          = NULL;
-        gf_boolean_t        wind_failed    = _gf_false;
-        fd_t               *fd             = NULL;
-        loc_t               loc            = {0,};
-        dict_t             *xattr_req      = NULL;
-
-        local = frame->local;
-        priv = this->private;
-        fd = local->fd;
-        shard_idx_iter = local->first_block;
-        last_block = local->last_block;
-        call_count = local->call_count;
-
-        ret = shard_inode_ctx_get_all (fd->inode, this, &ctx_tmp);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "Failed to get inode ctx for"
-                        " %s", uuid_utoa (fd->inode->gfid));
-                goto err;
-        }
+shard_post_resolve_readv_handler(call_frame_t *frame, xlator_t *this)
+{
+    shard_local_t *local = NULL;
 
-        while (shard_idx_iter <= last_block) {
-                if (local->inode_list[i]) {
-                        shard_idx_iter++;
-                        i++;
-                        continue;
-                }
+    local = frame->local;
 
-                if (wind_failed) {
-                        shard_writev_mknod_cbk (frame,
-                                                (void *) (long) shard_idx_iter,
-                                                this, -1, ENOMEM, NULL, NULL,
-                                                NULL, NULL, NULL);
-                        goto next;
-                }
+    if (local->op_ret < 0) {
+        if (local->op_errno != ENOENT) {
+            shard_common_failure_unwind(GF_FOP_READ, frame, local->op_ret,
+                                        local->op_errno);
+            return 0;
+        } else {
+            struct iovec vec = {
+                0,
+            };
+
+            vec.iov_base = local->iobuf->ptr;
+            vec.iov_len = local->total_size;
+            local->op_ret = local->total_size;
+            SHARD_STACK_UNWIND(readv, frame, local->op_ret, 0, &vec, 1,
+                               &local->prebuf, local->iobref, NULL);
+            return 0;
+        }
+    }
 
-                shard_make_block_abspath (shard_idx_iter, fd->inode->gfid,
-                                          path, sizeof(path));
+    if (local->call_count) {
+        shard_common_lookup_shards(frame, this, local->resolver_base_inode,
+                                   shard_post_lookup_shards_readv_handler);
+    } else {
+        shard_readv_do(frame, this);
+    }
 
-                xattr_req = shard_create_gfid_dict (local->xattr_req);
-                if (!xattr_req) {
-                        local->op_ret = -1;
-                        local->op_errno = ENOMEM;
-                        wind_failed = _gf_true;
-                        shard_writev_mknod_cbk (frame,
-                                                (void *) (long) shard_idx_iter,
-                                                this, -1, ENOMEM, NULL, NULL,
-                                                NULL, NULL, NULL);
-                        goto next;
-                }
+    return 0;
+}
 
-                bname = strrchr (path, '/') + 1;
-                loc.inode = inode_new (this->itable);
-                loc.parent = inode_ref (priv->dot_shard_inode);
-                ret = inode_path (loc.parent, bname,
-                                       (char **) &(loc.path));
-                if (ret < 0) {
-                        gf_log (this->name, GF_LOG_ERROR, "Inode path failed on"
-                                " %s", bname);
-                        local->op_ret = -1;
-                        local->op_errno = ENOMEM;
-                        wind_failed = _gf_true;
-                        loc_wipe (&loc);
-                        dict_unref (xattr_req);
-                        shard_writev_mknod_cbk (frame,
-                                                (void *) (long) shard_idx_iter,
-                                                this, -1, ENOMEM, NULL, NULL,
-                                                NULL, NULL, NULL);
-                        goto next;
-                }
+int
+shard_post_lookup_readv_handler(call_frame_t *frame, xlator_t *this)
+{
+    int ret = 0;
+    struct iobuf *iobuf = NULL;
+    shard_local_t *local = NULL;
+    shard_priv_t *priv = NULL;
 
-                loc.name = strrchr (loc.path, '/');
-                if (loc.name)
-                        loc.name++;
-
-                STACK_WIND_COOKIE (frame, shard_writev_mknod_cbk,
-                                   (void *) (long) shard_idx_iter,
-                                   FIRST_CHILD(this),
-                                   FIRST_CHILD(this)->fops->mknod, &loc,
-                                   ctx_tmp.mode, ctx_tmp.rdev, 0, xattr_req);
-                loc_wipe (&loc);
-                dict_unref (xattr_req);
-
-next:
-                shard_idx_iter++;
-                i++;
-                if (!--call_count)
-                        break;
-        }
+    priv = this->private;
+    local = frame->local;
+
+    if (local->op_ret < 0) {
+        shard_common_failure_unwind(GF_FOP_READ, frame, local->op_ret,
+                                    local->op_errno);
+        return 0;
+    }
 
+    if (local->offset >= local->prebuf.ia_size) {
+        /* If the read is being performed past the end of the file,
+         * unwind the FOP with 0 bytes read as status.
+         */
+        struct iovec vec = {
+            0,
+        };
+
+        iobuf = iobuf_get2(this->ctx->iobuf_pool, local->req_size);
+        if (!iobuf)
+            goto err;
+
+        vec.iov_base = iobuf->ptr;
+        vec.iov_len = 0;
+        local->iobref = iobref_new();
+        iobref_add(local->iobref, iobuf);
+        iobuf_unref(iobuf);
+
+        SHARD_STACK_UNWIND(readv, frame, 0, 0, &vec, 1, &local->prebuf,
+                           local->iobref, NULL);
         return 0;
+    }
+
+    local->first_block = get_lowest_block(local->offset, local->block_size);
+
+    local->total_size = local->req_size;
+
+    local->last_block = get_highest_block(local->offset, local->total_size,
+                                          local->block_size);
+
+    local->num_blocks = local->last_block - local->first_block + 1;
+    GF_ASSERT(local->num_blocks > 0);
+    local->resolver_base_inode = local->loc.inode;
+
+    local->inode_list = GF_CALLOC(local->num_blocks, sizeof(inode_t *),
+                                  gf_shard_mt_inode_list);
+    if (!local->inode_list)
+        goto err;
+
+    iobuf = iobuf_get2(this->ctx->iobuf_pool, local->total_size);
+    if (!iobuf)
+        goto err;
+
+    local->iobref = iobref_new();
+    if (!local->iobref) {
+        iobuf_unref(iobuf);
+        goto err;
+    }
+
+    if (iobref_add(local->iobref, iobuf) != 0) {
+        iobuf_unref(iobuf);
+        goto err;
+    }
+
+    memset(iobuf->ptr, 0, local->total_size);
+    iobuf_unref(iobuf);
+    local->iobuf = iobuf;
+
+    local->dot_shard_loc.inode = inode_find(this->itable, priv->dot_shard_gfid);
+    if (!local->dot_shard_loc.inode) {
+        ret = shard_init_internal_dir_loc(this, local,
+                                          SHARD_INTERNAL_DIR_DOT_SHARD);
+        if (ret)
+            goto err;
+        shard_lookup_internal_dir(frame, this, shard_post_resolve_readv_handler,
+                                  SHARD_INTERNAL_DIR_DOT_SHARD);
+    } else {
+        local->post_res_handler = shard_post_resolve_readv_handler;
+        shard_refresh_internal_dir(frame, this, SHARD_INTERNAL_DIR_DOT_SHARD);
+    }
+    return 0;
 err:
-        /*
-         * This block is for handling failure in shard_inode_ctx_get_all().
-         * Failures in the while-loop are handled within the loop.
+    shard_common_failure_unwind(GF_FOP_READ, frame, -1, ENOMEM);
+    return 0;
+}
+
+int
+shard_readv(call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
+            off_t offset, uint32_t flags, dict_t *xdata)
+{
+    int ret = 0;
+    uint64_t block_size = 0;
+    shard_local_t *local = NULL;
+
+    ret = shard_inode_ctx_get_block_size(fd->inode, this, &block_size);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, SHARD_MSG_INODE_CTX_GET_FAILED,
+               "Failed to get block "
+               "size for %s from its inode ctx",
+               uuid_utoa(fd->inode->gfid));
+        goto err;
+    }
+
+    if (!block_size || frame->root->pid == GF_CLIENT_PID_GSYNCD) {
+        /* block_size = 0 means that the file was created before
+         * sharding was enabled on the volume.
          */
-        SHARD_STACK_UNWIND (writev, frame, -1, ENOMEM, NULL, NULL, NULL);
+        STACK_WIND(frame, default_readv_cbk, FIRST_CHILD(this),
+                   FIRST_CHILD(this)->fops->readv, fd, size, offset, flags,
+                   xdata);
         return 0;
+    }
+
+    if (!this->itable)
+        this->itable = fd->inode->table;
+
+    local = mem_get0(this->local_pool);
+    if (!local)
+        goto err;
+
+    frame->local = local;
+
+    ret = syncbarrier_init(&local->barrier);
+    if (ret)
+        goto err;
+    local->fd = fd_ref(fd);
+    local->block_size = block_size;
+    local->offset = offset;
+    local->req_size = size;
+    local->flags = flags;
+    local->fop = GF_FOP_READ;
+    local->xattr_req = (xdata) ? dict_ref(xdata) : dict_new();
+    if (!local->xattr_req)
+        goto err;
+
+    local->loc.inode = inode_ref(fd->inode);
+    gf_uuid_copy(local->loc.gfid, fd->inode->gfid);
+
+    shard_refresh_base_file(frame, this, NULL, fd,
+                            shard_post_lookup_readv_handler);
+    return 0;
+err:
+    shard_common_failure_unwind(GF_FOP_READ, frame, -1, ENOMEM);
+    return 0;
 }
 
 int
-shard_common_resolve_shards (call_frame_t *frame, xlator_t *this)
+shard_common_inode_write_post_update_size_handler(call_frame_t *frame,
+                                                  xlator_t *this)
 {
-        int            i              = -1;
-        uint32_t       shard_idx_iter = 0;
-        char           path[PATH_MAX] = {0,};
-        fd_t          *fd             = NULL;
-        inode_t       *inode          = NULL;
-        shard_local_t *local          = NULL;
+    shard_local_t *local = NULL;
+
+    local = frame->local;
+
+    if (local->op_ret < 0) {
+        shard_common_failure_unwind(local->fop, frame, local->op_ret,
+                                    local->op_errno);
+    } else {
+        shard_common_inode_write_success_unwind(local->fop, frame,
+                                                local->written_size);
+    }
+    return 0;
+}
 
-        local = frame->local;
-        fd = local->fd;
-        shard_idx_iter = local->first_block;
+static gf_boolean_t
+shard_is_appending_write(shard_local_t *local)
+{
+    if (local->fop != GF_FOP_WRITE)
+        return _gf_false;
+    if (local->flags & O_APPEND)
+        return _gf_true;
+    if (local->fd->flags & O_APPEND)
+        return _gf_true;
+    return _gf_false;
+}
 
-        while (shard_idx_iter <= local->last_block) {
-                i++;
-                if (shard_idx_iter == 0) {
-                        local->inode_list[i] = inode_ref (fd->inode);
-                        shard_idx_iter++;
-                        continue;
-                }
+int
+__shard_get_delta_size_from_inode_ctx(shard_local_t *local, inode_t *inode,
+                                      xlator_t *this)
+{
+    int ret = -1;
+    uint64_t ctx_uint = 0;
+    shard_inode_ctx_t *ctx = NULL;
 
-                shard_make_block_abspath (shard_idx_iter, fd->inode->gfid,
-                                          path, sizeof(path));
-
-                inode = NULL;
-                inode = inode_resolve (this->itable, path);
-                if (inode) {
-                        gf_log (this->name, GF_LOG_DEBUG, "Shard %d already "
-                                "present. gfid=%s. Saving inode for future.",
-                                shard_idx_iter, uuid_utoa(inode->gfid));
-                        shard_idx_iter++;
-                        local->inode_list[i] = inode;
-                        /* Let the ref on the inodes that are already present
-                         * in inode table still be held so that they don't get
-                         * forgotten by the time the fop reaches the actual
-                         * write stage.
-                         */
-                         continue;
-                } else {
-                        local->call_count++;
-                        shard_idx_iter++;
-                }
-        }
+    ret = __inode_ctx_get(inode, this, &ctx_uint);
+    if (ret < 0)
+        return ret;
 
-        if (local->call_count)
-                shard_writev_resume_mknod (frame, this);
-        else
-                shard_lookup_base_file (frame, this);
+    ctx = (shard_inode_ctx_t *)(uintptr_t)ctx_uint;
 
-        return 0;
+    if (shard_is_appending_write(local)) {
+        local->delta_size = local->total_size;
+    } else if (local->offset + local->total_size > ctx->stat.ia_size) {
+        local->delta_size = (local->offset + local->total_size) -
+                            ctx->stat.ia_size;
+    } else {
+        local->delta_size = 0;
+    }
+    ctx->stat.ia_size += (local->delta_size);
+    local->postbuf = ctx->stat;
+
+    return 0;
 }
 
-static void
-shard_link_dot_shard_inode (shard_local_t *local, inode_t *inode,
-                            struct iatt *buf)
+int
+shard_get_delta_size_from_inode_ctx(shard_local_t *local, inode_t *inode,
+                                    xlator_t *this)
 {
-        inode_t       *linked_inode = NULL;
-        shard_priv_t  *priv         = NULL;
+    int ret = -1;
 
-        priv = THIS->private;
+    LOCK(&inode->lock);
+    {
+        ret = __shard_get_delta_size_from_inode_ctx(local, inode, this);
+    }
+    UNLOCK(&inode->lock);
 
-        linked_inode = inode_link (inode, local->dot_shard_loc.parent,
-                                   local->dot_shard_loc.name, buf);
-        inode_lookup (linked_inode);
-        priv->dot_shard_inode = linked_inode;
+    return ret;
 }
 
 int
-shard_lookup_dot_shard_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                            int32_t op_ret, int32_t op_errno, inode_t *inode,
-                            struct iatt *buf, dict_t *xdata,
-                            struct iatt *postparent)
+shard_common_inode_write_do_cbk(call_frame_t *frame, void *cookie,
+                                xlator_t *this, int32_t op_ret,
+                                int32_t op_errno, struct iatt *pre,
+                                struct iatt *post, dict_t *xdata)
 {
-        shard_local_t *local = NULL;
-
-        local = frame->local;
+    int call_count = 0;
+    fd_t *anon_fd = cookie;
+    shard_local_t *local = NULL;
+    glusterfs_fop_t fop = 0;
 
-        if (op_ret)
-                goto unwind;
+    local = frame->local;
+    fop = local->fop;
 
-        if (!IA_ISDIR (buf->ia_type)) {
-                gf_log (this->name, GF_LOG_CRITICAL, "/.shard already exists "
-                        "and is not a directory. Please remove /.shard from all"
-                        " bricks and try again");
-                op_errno = EIO;
-                goto unwind;
+    LOCK(&frame->lock);
+    {
+        if (op_ret < 0) {
+            local->op_ret = op_ret;
+            local->op_errno = op_errno;
+        } else {
+            local->written_size += op_ret;
+            GF_ATOMIC_ADD(local->delta_blocks,
+                          post->ia_blocks - pre->ia_blocks);
+            local->delta_size += (post->ia_size - pre->ia_size);
+            shard_inode_ctx_set(local->fd->inode, this, post, 0,
+                                SHARD_MASK_TIMES);
+            if (local->fd->inode != anon_fd->inode)
+                shard_inode_ctx_add_to_fsync_list(local->fd->inode, this,
+                                                  anon_fd->inode);
         }
+    }
+    UNLOCK(&frame->lock);
+
+    if (anon_fd)
+        fd_unref(anon_fd);
+
+    call_count = shard_call_count_return(frame);
+    if (call_count == 0) {
+        SHARD_UNSET_ROOT_FS_ID(frame, local);
+        if (local->op_ret < 0) {
+            shard_common_failure_unwind(fop, frame, local->op_ret,
+                                        local->op_errno);
+        } else {
+            shard_get_delta_size_from_inode_ctx(local, local->fd->inode, this);
+            local->hole_size = 0;
+            if (xdata)
+                local->xattr_rsp = dict_ref(xdata);
+            shard_update_file_size(
+                frame, this, local->fd, NULL,
+                shard_common_inode_write_post_update_size_handler);
+        }
+    }
 
-        shard_link_dot_shard_inode (local, inode, buf);
-        shard_common_resolve_shards (frame, this);
-        return 0;
+    return 0;
+}
 
-unwind:
-        SHARD_STACK_UNWIND (writev, frame, -1, op_errno, NULL, NULL, NULL);
-        return 0;
+int
+shard_common_inode_write_wind(call_frame_t *frame, xlator_t *this, fd_t *fd,
+                              struct iovec *vec, int count, off_t shard_offset,
+                              size_t size)
+{
+    shard_local_t *local = NULL;
+
+    local = frame->local;
+
+    switch (local->fop) {
+        case GF_FOP_WRITE:
+            STACK_WIND_COOKIE(
+                frame, shard_common_inode_write_do_cbk, fd, FIRST_CHILD(this),
+                FIRST_CHILD(this)->fops->writev, fd, vec, count, shard_offset,
+                local->flags, local->iobref, local->xattr_req);
+            break;
+        case GF_FOP_FALLOCATE:
+            STACK_WIND_COOKIE(
+                frame, shard_common_inode_write_do_cbk, fd, FIRST_CHILD(this),
+                FIRST_CHILD(this)->fops->fallocate, fd, local->flags,
+                shard_offset, size, local->xattr_req);
+            break;
+        case GF_FOP_ZEROFILL:
+            STACK_WIND_COOKIE(frame, shard_common_inode_write_do_cbk, fd,
+                              FIRST_CHILD(this),
+                              FIRST_CHILD(this)->fops->zerofill, fd,
+                              shard_offset, size, local->xattr_req);
+            break;
+        case GF_FOP_DISCARD:
+            STACK_WIND_COOKIE(frame, shard_common_inode_write_do_cbk, fd,
+                              FIRST_CHILD(this),
+                              FIRST_CHILD(this)->fops->discard, fd,
+                              shard_offset, size, local->xattr_req);
+            break;
+        default:
+            gf_msg(this->name, GF_LOG_WARNING, 0, SHARD_MSG_INVALID_FOP,
+                   "Invalid fop id = %d", local->fop);
+            break;
+    }
+    return 0;
 }
 
 int
-shard_lookup_dot_shard (call_frame_t *frame, xlator_t *this)
+shard_common_inode_write_do(call_frame_t *frame, xlator_t *this)
 {
-        int                 ret       = -1;
-        dict_t             *xattr_req = NULL;
-        shard_priv_t       *priv      = NULL;
-        shard_local_t      *local     = NULL;
+    int i = 0;
+    int count = 0;
+    int call_count = 0;
+    int last_block = 0;
+    uint32_t cur_block = 0;
+    fd_t *fd = NULL;
+    fd_t *anon_fd = NULL;
+    shard_local_t *local = NULL;
+    struct iovec *vec = NULL;
+    gf_boolean_t wind_failed = _gf_false;
+    gf_boolean_t odirect = _gf_false;
+    off_t orig_offset = 0;
+    off_t shard_offset = 0;
+    off_t vec_offset = 0;
+    size_t remaining_size = 0;
+    size_t shard_write_size = 0;
+
+    local = frame->local;
+    fd = local->fd;
+
+    orig_offset = local->offset;
+    remaining_size = local->total_size;
+    cur_block = local->first_block;
+    local->call_count = call_count = local->num_blocks;
+    last_block = local->last_block;
+
+    SHARD_SET_ROOT_FS_ID(frame, local);
+
+    if (dict_set_uint32(local->xattr_req, GLUSTERFS_WRITE_UPDATE_ATOMIC, 4)) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, SHARD_MSG_DICT_OP_FAILED,
+               "Failed to set " GLUSTERFS_WRITE_UPDATE_ATOMIC
+               " into "
+               "dict: %s",
+               uuid_utoa(fd->inode->gfid));
+        local->op_ret = -1;
+        local->op_errno = ENOMEM;
+        local->call_count = 1;
+        shard_common_inode_write_do_cbk(frame, (void *)(long)0, this, -1,
+                                        ENOMEM, NULL, NULL, NULL);
+        return 0;
+    }
 
-        local = frame->local;
-        priv = this->private;
+    if ((fd->flags & O_DIRECT) && (local->fop == GF_FOP_WRITE))
+        odirect = _gf_true;
 
-        xattr_req = dict_new ();
-        if (!xattr_req)
-                goto err;
+    while (cur_block <= last_block) {
+        if (wind_failed) {
+            shard_common_inode_write_do_cbk(frame, (void *)(long)0, this, -1,
+                                            ENOMEM, NULL, NULL, NULL);
+            goto next;
+        }
 
-        ret = dict_set_static_bin (xattr_req, "gfid-req", priv->dot_shard_gfid,
-                                   16);
-        if (!ret) {
-                gf_log (this->name, GF_LOG_ERROR, "Failed to set gfid of "
-                        "/.shard into dict");
-                goto err;
+        shard_offset = orig_offset % local->block_size;
+        shard_write_size = local->block_size - shard_offset;
+        if (shard_write_size > remaining_size)
+            shard_write_size = remaining_size;
+
+        remaining_size -= shard_write_size;
+
+        if (local->fop == GF_FOP_WRITE) {
+            vec = NULL;
+            count = iov_subset(local->vector, local->count, vec_offset,
+                               shard_write_size, &vec, 0);
+            if (count < 0) {
+                local->op_ret = -1;
+                local->op_errno = ENOMEM;
+                wind_failed = _gf_true;
+                shard_common_inode_write_do_cbk(frame, (void *)(long)0, this,
+                                                -1, ENOMEM, NULL, NULL, NULL);
+                goto next;
+            }
         }
 
-        STACK_WIND (frame, shard_lookup_dot_shard_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->lookup, &local->dot_shard_loc,
-                    xattr_req);
-        dict_unref (xattr_req);
+        if (cur_block == 0) {
+            anon_fd = fd_ref(fd);
+        } else {
+            anon_fd = fd_anonymous(local->inode_list[i]);
+            if (!anon_fd) {
+                local->op_ret = -1;
+                local->op_errno = ENOMEM;
+                wind_failed = _gf_true;
+                GF_FREE(vec);
+                shard_common_inode_write_do_cbk(frame, (void *)(long)anon_fd,
+                                                this, -1, ENOMEM, NULL, NULL,
+                                                NULL);
+                goto next;
+            }
+
+            if (local->fop == GF_FOP_WRITE) {
+                if (odirect)
+                    local->flags = O_DIRECT;
+                else
+                    local->flags = GF_ANON_FD_FLAGS;
+            }
+        }
+
+        shard_common_inode_write_wind(frame, this, anon_fd, vec, count,
+                                      shard_offset, shard_write_size);
+        if (vec)
+            vec_offset += shard_write_size;
+        orig_offset += shard_write_size;
+        GF_FREE(vec);
+        vec = NULL;
+    next:
+        cur_block++;
+        i++;
+        call_count--;
+    }
+    return 0;
+}
+
+int
+shard_common_inode_write_post_mknod_handler(call_frame_t *frame,
+                                            xlator_t *this);
+
+int
+shard_common_inode_write_post_lookup_shards_handler(call_frame_t *frame,
+                                                    xlator_t *this)
+{
+    shard_local_t *local = NULL;
+
+    local = frame->local;
+
+    if (local->op_ret < 0) {
+        shard_common_failure_unwind(local->fop, frame, local->op_ret,
+                                    local->op_errno);
         return 0;
+    }
 
-err:
-        if (xattr_req)
-                dict_unref (xattr_req);
-        SHARD_STACK_UNWIND (writev, frame, -1, ENOMEM, NULL, NULL, NULL);
+    if (local->create_count) {
+        shard_common_resume_mknod(frame, this,
+                                  shard_common_inode_write_post_mknod_handler);
+    } else {
+        shard_common_inode_write_do(frame, this);
+    }
+
+    return 0;
+}
+
+int
+shard_common_inode_write_post_mknod_handler(call_frame_t *frame, xlator_t *this)
+{
+    shard_local_t *local = NULL;
+
+    local = frame->local;
+
+    if (local->op_ret < 0) {
+        shard_common_failure_unwind(local->fop, frame, local->op_ret,
+                                    local->op_errno);
         return 0;
+    }
+
+    if (!local->eexist_count) {
+        shard_common_inode_write_do(frame, this);
+    } else {
+        local->call_count = local->eexist_count;
+        shard_common_lookup_shards(
+            frame, this, local->loc.inode,
+            shard_common_inode_write_post_lookup_shards_handler);
+    }
+
+    return 0;
 }
 
 int
-shard_writev_mkdir_dot_shard_cbk (call_frame_t *frame, void *cookie,
-                                  xlator_t *this, int32_t op_ret,
-                                  int32_t op_errno, inode_t *inode,
-                                  struct iatt *buf, struct iatt *preparent,
-                                  struct iatt *postparent, dict_t *xdata)
+shard_common_inode_write_post_resolve_handler(call_frame_t *frame,
+                                              xlator_t *this)
 {
-        shard_local_t *local = NULL;
+    shard_local_t *local = NULL;
 
-        local = frame->local;
+    local = frame->local;
 
-        if (op_ret == -1) {
-                if (op_errno != EEXIST) {
-                        goto unwind;
-                } else {
-                        gf_log (this->name, GF_LOG_DEBUG, "mkdir on /.shard "
-                                "failed with EEXIST. Attempting lookup now");
-                        shard_lookup_dot_shard (frame, this);
-                        return 0;
-                }
-        }
+    if (local->op_ret < 0) {
+        shard_common_failure_unwind(local->fop, frame, local->op_ret,
+                                    local->op_errno);
+        return 0;
+    }
+
+    if (local->call_count) {
+        shard_common_lookup_shards(
+            frame, this, local->resolver_base_inode,
+            shard_common_inode_write_post_lookup_shards_handler);
+    } else if (local->create_count) {
+        shard_common_inode_write_post_lookup_shards_handler(frame, this);
+    } else {
+        shard_common_inode_write_do(frame, this);
+    }
+
+    return 0;
+}
+
+int
+shard_common_inode_write_post_lookup_handler(call_frame_t *frame,
+                                             xlator_t *this)
+{
+    shard_local_t *local = frame->local;
+    shard_priv_t *priv = this->private;
 
-        shard_link_dot_shard_inode (local, inode, buf);
-        shard_common_resolve_shards (frame, this);
+    if (local->op_ret < 0) {
+        shard_common_failure_unwind(local->fop, frame, local->op_ret,
+                                    local->op_errno);
         return 0;
+    }
+
+    local->postbuf = local->prebuf;
+
+    /*Adjust offset to EOF so that correct shard is chosen for append*/
+    if (shard_is_appending_write(local))
+        local->offset = local->prebuf.ia_size;
+
+    local->first_block = get_lowest_block(local->offset, local->block_size);
+    local->last_block = get_highest_block(local->offset, local->total_size,
+                                          local->block_size);
+    local->num_blocks = local->last_block - local->first_block + 1;
+    GF_ASSERT(local->num_blocks > 0);
+    local->inode_list = GF_CALLOC(local->num_blocks, sizeof(inode_t *),
+                                  gf_shard_mt_inode_list);
+    if (!local->inode_list) {
+        shard_common_failure_unwind(local->fop, frame, -1, ENOMEM);
+        return 0;
+    }
+
+    gf_msg_trace(this->name, 0,
+                 "%s: gfid=%s first_block=%" PRIu64
+                 " "
+                 "last_block=%" PRIu64 " num_blocks=%" PRIu64 " offset=%" PRId64
+                 " total_size=%zu flags=%" PRId32 "",
+                 gf_fop_list[local->fop],
+                 uuid_utoa(local->resolver_base_inode->gfid),
+                 local->first_block, local->last_block, local->num_blocks,
+                 local->offset, local->total_size, local->flags);
+
+    local->dot_shard_loc.inode = inode_find(this->itable, priv->dot_shard_gfid);
+
+    if (!local->dot_shard_loc.inode) {
+        /*change handler*/
+        shard_mkdir_internal_dir(frame, this,
+                                 shard_common_inode_write_post_resolve_handler,
+                                 SHARD_INTERNAL_DIR_DOT_SHARD);
+    } else {
+        /*change handler*/
+        local->post_res_handler = shard_common_inode_write_post_resolve_handler;
+        shard_refresh_internal_dir(frame, this, SHARD_INTERNAL_DIR_DOT_SHARD);
+    }
+    return 0;
+}
+
+int
+shard_mkdir_internal_dir_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                             int32_t op_ret, int32_t op_errno, inode_t *inode,
+                             struct iatt *buf, struct iatt *preparent,
+                             struct iatt *postparent, dict_t *xdata)
+{
+    inode_t *link_inode = NULL;
+    shard_local_t *local = NULL;
+    shard_internal_dir_type_t type = (shard_internal_dir_type_t)cookie;
+
+    local = frame->local;
 
+    SHARD_UNSET_ROOT_FS_ID(frame, local);
+
+    if (op_ret == -1) {
+        if (op_errno != EEXIST) {
+            local->op_ret = op_ret;
+            local->op_errno = op_errno;
+            goto unwind;
+        } else {
+            gf_msg_debug(this->name, 0,
+                         "mkdir on %s failed "
+                         "with EEXIST. Attempting lookup now",
+                         shard_internal_dir_string(type));
+            shard_lookup_internal_dir(frame, this, local->post_res_handler,
+                                      type);
+            return 0;
+        }
+    }
+
+    link_inode = shard_link_internal_dir_inode(local, inode, buf, type);
+    if (link_inode != inode) {
+        shard_refresh_internal_dir(frame, this, type);
+    } else {
+        shard_inode_ctx_mark_dir_refreshed(link_inode, this);
+        shard_common_resolve_shards(frame, this, local->post_res_handler);
+    }
+    return 0;
 unwind:
-        SHARD_STACK_UNWIND (writev, frame, -1, op_errno, NULL, NULL, NULL);
-        return 0;
+    shard_common_resolve_shards(frame, this, local->post_res_handler);
+    return 0;
 }
 
 int
-shard_writev_mkdir_dot_shard (call_frame_t *frame, xlator_t *this)
+shard_mkdir_internal_dir(call_frame_t *frame, xlator_t *this,
+                         shard_post_resolve_fop_handler_t handler,
+                         shard_internal_dir_type_t type)
 {
-        int             ret           = -1;
-        shard_local_t  *local         = NULL;
-        shard_priv_t   *priv          = NULL;
-        loc_t          *dot_shard_loc = NULL;
-        dict_t         *xattr_req     = NULL;
+    int ret = -1;
+    shard_local_t *local = NULL;
+    shard_priv_t *priv = NULL;
+    dict_t *xattr_req = NULL;
+    uuid_t *gfid = NULL;
+    loc_t *loc = NULL;
+    gf_boolean_t free_gfid = _gf_true;
+
+    local = frame->local;
+    priv = this->private;
+
+    local->post_res_handler = handler;
+    gfid = GF_MALLOC(sizeof(uuid_t), gf_common_mt_uuid_t);
+    if (!gfid)
+        goto err;
+
+    switch (type) {
+        case SHARD_INTERNAL_DIR_DOT_SHARD:
+            gf_uuid_copy(*gfid, priv->dot_shard_gfid);
+            loc = &local->dot_shard_loc;
+            break;
+        case SHARD_INTERNAL_DIR_DOT_SHARD_REMOVE_ME:
+            gf_uuid_copy(*gfid, priv->dot_shard_rm_gfid);
+            loc = &local->dot_shard_rm_loc;
+            break;
+        default:
+            bzero(*gfid, sizeof(uuid_t));
+            break;
+    }
+
+    xattr_req = dict_new();
+    if (!xattr_req)
+        goto err;
+
+    ret = shard_init_internal_dir_loc(this, local, type);
+    if (ret)
+        goto err;
+
+    ret = dict_set_gfuuid(xattr_req, "gfid-req", *gfid, false);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, SHARD_MSG_DICT_OP_FAILED,
+               "Failed to set gfid-req for %s",
+               shard_internal_dir_string(type));
+        goto err;
+    } else {
+        free_gfid = _gf_false;
+    }
+
+    SHARD_SET_ROOT_FS_ID(frame, local);
+
+    STACK_WIND_COOKIE(frame, shard_mkdir_internal_dir_cbk, (void *)(long)type,
+                      FIRST_CHILD(this), FIRST_CHILD(this)->fops->mkdir, loc,
+                      0755, 0, xattr_req);
+    dict_unref(xattr_req);
+    return 0;
 
-        local = frame->local;
-        priv = this->private;
+err:
+    if (xattr_req)
+        dict_unref(xattr_req);
+    local->op_ret = -1;
+    local->op_errno = ENOMEM;
+    if (free_gfid)
+        GF_FREE(gfid);
+    handler(frame, this);
+    return 0;
+}
 
-        xattr_req = dict_new ();
-        if (!xattr_req)
-                goto err;
+int
+shard_flush_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                int32_t op_ret, int32_t op_errno, dict_t *xdata)
+{
+    /* To-Do: Wind flush on all shards of the file */
+    SHARD_STACK_UNWIND(flush, frame, op_ret, op_errno, xdata);
+    return 0;
+}
 
-        dot_shard_loc = &local->dot_shard_loc;
+int
+shard_flush(call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *xdata)
+{
+    STACK_WIND(frame, shard_flush_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->flush, fd, xdata);
+    return 0;
+}
 
-        dot_shard_loc->inode = inode_new (this->itable);
-        dot_shard_loc->parent = inode_ref (this->itable->root);
-        ret = inode_path (dot_shard_loc->parent, GF_SHARD_DIR,
-                                          (char **)&dot_shard_loc->path);
-        if (ret < 0) {
-                gf_log (this->name, GF_LOG_ERROR, "Inode path failed on"
-                        " %s", GF_SHARD_DIR);
-                goto err;
+int
+__shard_get_timestamps_from_inode_ctx(shard_local_t *local, inode_t *inode,
+                                      xlator_t *this)
+{
+    int ret = -1;
+    uint64_t ctx_uint = 0;
+    shard_inode_ctx_t *ctx = NULL;
+
+    ret = __inode_ctx_get(inode, this, &ctx_uint);
+    if (ret < 0)
+        return ret;
+
+    ctx = (shard_inode_ctx_t *)(uintptr_t)ctx_uint;
+
+    local->postbuf.ia_ctime = ctx->stat.ia_ctime;
+    local->postbuf.ia_ctime_nsec = ctx->stat.ia_ctime_nsec;
+    local->postbuf.ia_atime = ctx->stat.ia_atime;
+    local->postbuf.ia_atime_nsec = ctx->stat.ia_atime_nsec;
+    local->postbuf.ia_mtime = ctx->stat.ia_mtime;
+    local->postbuf.ia_mtime_nsec = ctx->stat.ia_mtime_nsec;
+
+    return 0;
+}
+
+int
+shard_get_timestamps_from_inode_ctx(shard_local_t *local, inode_t *inode,
+                                    xlator_t *this)
+{
+    int ret = 0;
+
+    LOCK(&inode->lock);
+    {
+        ret = __shard_get_timestamps_from_inode_ctx(local, inode, this);
+    }
+    UNLOCK(&inode->lock);
+
+    return ret;
+}
+
+int
+shard_fsync_shards_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                       int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
+                       struct iatt *postbuf, dict_t *xdata)
+{
+    int call_count = 0;
+    uint64_t fsync_count = 0;
+    fd_t *anon_fd = cookie;
+    shard_local_t *local = NULL;
+    shard_inode_ctx_t *ctx = NULL;
+    shard_inode_ctx_t *base_ictx = NULL;
+    inode_t *base_inode = NULL;
+    gf_boolean_t unref_shard_inode = _gf_false;
+
+    local = frame->local;
+    base_inode = local->fd->inode;
+
+    if (local->op_ret < 0)
+        goto out;
+
+    LOCK(&frame->lock);
+    {
+        if (op_ret < 0) {
+            local->op_ret = op_ret;
+            local->op_errno = op_errno;
+            UNLOCK(&frame->lock);
+            goto out;
+        }
+        shard_inode_ctx_set(local->fd->inode, this, postbuf, 0,
+                            SHARD_MASK_TIMES);
+    }
+    UNLOCK(&frame->lock);
+    fd_ctx_get(anon_fd, this, &fsync_count);
+out:
+    if (anon_fd && (base_inode != anon_fd->inode)) {
+        LOCK(&base_inode->lock);
+        LOCK(&anon_fd->inode->lock);
+        {
+            __shard_inode_ctx_get(anon_fd->inode, this, &ctx);
+            __shard_inode_ctx_get(base_inode, this, &base_ictx);
+            if (op_ret == 0)
+                ctx->fsync_needed -= fsync_count;
+            GF_ASSERT(ctx->fsync_needed >= 0);
+            if (ctx->fsync_needed != 0) {
+                list_add_tail(&ctx->to_fsync_list, &base_ictx->to_fsync_list);
+                base_ictx->fsync_count++;
+            } else {
+                unref_shard_inode = _gf_true;
+            }
         }
+        UNLOCK(&anon_fd->inode->lock);
+        UNLOCK(&base_inode->lock);
+    }
 
-        dot_shard_loc->name = strrchr (dot_shard_loc->path, '/');
-        if (dot_shard_loc->name)
-                dot_shard_loc->name++;
+    if (unref_shard_inode)
+        inode_unref(anon_fd->inode);
+    if (anon_fd)
+        fd_unref(anon_fd);
 
-        ret = dict_set_static_bin (xattr_req, "gfid-req", priv->dot_shard_gfid,
-                                   16);
+    call_count = shard_call_count_return(frame);
+    if (call_count != 0)
+        return 0;
+
+    if (local->op_ret < 0) {
+        shard_common_failure_unwind(GF_FOP_FSYNC, frame, local->op_ret,
+                                    local->op_errno);
+    } else {
+        shard_get_timestamps_from_inode_ctx(local, base_inode, this);
+        SHARD_STACK_UNWIND(fsync, frame, local->op_ret, local->op_errno,
+                           &local->prebuf, &local->postbuf, local->xattr_rsp);
+    }
+    return 0;
+}
+
+int
+shard_post_lookup_fsync_handler(call_frame_t *frame, xlator_t *this)
+{
+    int ret = 0;
+    int call_count = 0;
+    int fsync_count = 0;
+    fd_t *anon_fd = NULL;
+    inode_t *base_inode = NULL;
+    shard_local_t *local = NULL;
+    shard_inode_ctx_t *ctx = NULL;
+    shard_inode_ctx_t *iter = NULL;
+    struct list_head copy = {
+        0,
+    };
+    shard_inode_ctx_t *tmp = NULL;
+
+    local = frame->local;
+    base_inode = local->fd->inode;
+    local->postbuf = local->prebuf;
+    INIT_LIST_HEAD(&copy);
+
+    if (local->op_ret < 0) {
+        shard_common_failure_unwind(GF_FOP_FSYNC, frame, local->op_ret,
+                                    local->op_errno);
+        return 0;
+    }
+
+    LOCK(&base_inode->lock);
+    {
+        __shard_inode_ctx_get(base_inode, this, &ctx);
+        list_splice_init(&ctx->to_fsync_list, &copy);
+        call_count = ctx->fsync_count;
+        ctx->fsync_count = 0;
+    }
+    UNLOCK(&base_inode->lock);
+
+    local->call_count = ++call_count;
+
+    /* Send fsync() on the base shard first */
+    anon_fd = fd_ref(local->fd);
+    STACK_WIND_COOKIE(frame, shard_fsync_shards_cbk, anon_fd, FIRST_CHILD(this),
+                      FIRST_CHILD(this)->fops->fsync, anon_fd, local->datasync,
+                      local->xattr_req);
+    call_count--;
+    anon_fd = NULL;
+
+    list_for_each_entry_safe(iter, tmp, &copy, to_fsync_list)
+    {
+        list_del_init(&iter->to_fsync_list);
+        fsync_count = 0;
+        shard_inode_ctx_get_fsync_count(iter->inode, this, &fsync_count);
+        GF_ASSERT(fsync_count > 0);
+        anon_fd = fd_anonymous(iter->inode);
+        if (!anon_fd) {
+            local->op_ret = -1;
+            local->op_errno = ENOMEM;
+            gf_msg(this->name, GF_LOG_WARNING, ENOMEM,
+                   SHARD_MSG_MEMALLOC_FAILED,
+                   "Failed to create "
+                   "anon fd to fsync shard");
+            shard_fsync_shards_cbk(frame, (void *)(long)anon_fd, this, -1,
+                                   ENOMEM, NULL, NULL, NULL);
+            continue;
+        }
+
+        ret = fd_ctx_set(anon_fd, this, fsync_count);
         if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "Failed to set gfid-req for "
-                        "/.shard");
-                goto err;
+            gf_msg(this->name, GF_LOG_ERROR, 0, SHARD_MSG_FD_CTX_SET_FAILED,
+                   "Failed to set fd "
+                   "ctx for shard inode gfid=%s",
+                   uuid_utoa(iter->inode->gfid));
+            local->op_ret = -1;
+            local->op_errno = ENOMEM;
+            shard_fsync_shards_cbk(frame, (void *)(long)anon_fd, this, -1,
+                                   ENOMEM, NULL, NULL, NULL);
+            continue;
         }
+        STACK_WIND_COOKIE(frame, shard_fsync_shards_cbk, anon_fd,
+                          FIRST_CHILD(this), FIRST_CHILD(this)->fops->fsync,
+                          anon_fd, local->datasync, local->xattr_req);
+        call_count--;
+    }
 
-        STACK_WIND (frame, shard_writev_mkdir_dot_shard_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->mkdir, &local->dot_shard_loc,
-                    0755, 0, xattr_req);
-        dict_unref (xattr_req);
+    return 0;
+}
+
+int
+shard_fsync(call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t datasync,
+            dict_t *xdata)
+{
+    int ret = 0;
+    uint64_t block_size = 0;
+    shard_local_t *local = NULL;
+
+    ret = shard_inode_ctx_get_block_size(fd->inode, this, &block_size);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, SHARD_MSG_INODE_CTX_GET_FAILED,
+               "Failed to get block "
+               "size for %s from its inode ctx",
+               uuid_utoa(fd->inode->gfid));
+        goto err;
+    }
+
+    if (!block_size || frame->root->pid == GF_CLIENT_PID_GSYNCD) {
+        STACK_WIND(frame, default_fsync_cbk, FIRST_CHILD(this),
+                   FIRST_CHILD(this)->fops->fsync, fd, datasync, xdata);
         return 0;
+    }
+
+    if (!this->itable)
+        this->itable = fd->inode->table;
+
+    local = mem_get0(this->local_pool);
+    if (!local)
+        goto err;
+
+    frame->local = local;
+
+    local->fd = fd_ref(fd);
+    local->fop = GF_FOP_FSYNC;
+    local->datasync = datasync;
+    local->xattr_req = (xdata) ? dict_ref(xdata) : dict_new();
+    if (!local->xattr_req)
+        goto err;
 
+    local->loc.inode = inode_ref(fd->inode);
+    gf_uuid_copy(local->loc.gfid, fd->inode->gfid);
+
+    shard_refresh_base_file(frame, this, NULL, fd,
+                            shard_post_lookup_fsync_handler);
+    return 0;
 err:
-        if (xattr_req)
-                dict_unref (xattr_req);
-        SHARD_STACK_UNWIND (writev, frame, -1, ENOMEM, NULL, NULL, NULL);
-        return 0;
+    shard_common_failure_unwind(GF_FOP_FSYNC, frame, -1, ENOMEM);
+    return 0;
 }
 
 int
-shard_writev (call_frame_t *frame, xlator_t *this, fd_t *fd,
-              struct iovec *vector, int32_t count, off_t offset, uint32_t flags,
-              struct iobref *iobref, dict_t *xdata)
+shard_readdir_past_dot_shard_cbk(call_frame_t *frame, void *cookie,
+                                 xlator_t *this, int32_t op_ret,
+                                 int32_t op_errno, gf_dirent_t *orig_entries,
+                                 dict_t *xdata)
 {
-        int             ret            = 0;
-        int             i              = 0;
-        uint64_t        block_size     = 0;
-        shard_local_t  *local          = NULL;
-        shard_priv_t   *priv           = NULL;
+    gf_dirent_t *entry = NULL;
+    gf_dirent_t *tmp = NULL;
+    shard_local_t *local = NULL;
 
-        priv = this->private;
+    local = frame->local;
 
-        ret = shard_inode_ctx_get_block_size (fd->inode, this, &block_size);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "Failed to get block size "
-                        "for %s from its inode ctx",
-                        uuid_utoa (fd->inode->gfid));
-                goto out;
-        }
+    if (op_ret < 0)
+        goto unwind;
 
-        if (!block_size) {
-                /* block_size = 0 means that the file was created before
-                 * sharding was enabled on the volume.
-                 */
-                STACK_WIND (frame, default_writev_cbk,
-                            FIRST_CHILD(this), FIRST_CHILD(this)->fops->writev,
-                            fd, vector, count, offset, flags, iobref, xdata);
-                return 0;
+    list_for_each_entry_safe(entry, tmp, (&orig_entries->list), list)
+    {
+        list_del_init(&entry->list);
+        list_add_tail(&entry->list, &local->entries_head.list);
+
+        if (!entry->dict)
+            continue;
+
+        if (IA_ISDIR(entry->d_stat.ia_type))
+            continue;
+
+        if (dict_get(entry->dict, GF_XATTR_SHARD_FILE_SIZE))
+            shard_modify_size_and_block_count(&entry->d_stat, entry->dict);
+        if (!entry->inode)
+            continue;
+
+        shard_inode_ctx_update(entry->inode, this, entry->dict, &entry->d_stat);
+    }
+    local->op_ret += op_ret;
+
+unwind:
+    if (local->fop == GF_FOP_READDIR)
+        SHARD_STACK_UNWIND(readdir, frame, local->op_ret, local->op_errno,
+                           &local->entries_head, xdata);
+    else
+        SHARD_STACK_UNWIND(readdirp, frame, op_ret, op_errno,
+                           &local->entries_head, xdata);
+    return 0;
+}
+
+int32_t
+shard_readdir_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                  int32_t op_ret, int32_t op_errno, gf_dirent_t *orig_entries,
+                  dict_t *xdata)
+{
+    fd_t *fd = NULL;
+    gf_dirent_t *entry = NULL;
+    gf_dirent_t *tmp = NULL;
+    shard_local_t *local = NULL;
+    gf_boolean_t last_entry = _gf_false;
+
+    local = frame->local;
+    fd = local->fd;
+
+    if (op_ret < 0)
+        goto unwind;
+
+    list_for_each_entry_safe(entry, tmp, (&orig_entries->list), list)
+    {
+        if (last_entry)
+            last_entry = _gf_false;
+
+        if (__is_root_gfid(fd->inode->gfid) &&
+            !(strcmp(entry->d_name, GF_SHARD_DIR))) {
+            local->offset = entry->d_off;
+            op_ret--;
+            last_entry = _gf_true;
+            continue;
         }
 
-        if (!this->itable)
-                this->itable = fd->inode->table;
+        list_del_init(&entry->list);
+        list_add_tail(&entry->list, &local->entries_head.list);
 
-        local = mem_get0 (this->local_pool);
-        if (!local)
-                goto out;
+        if (!entry->dict)
+            continue;
 
-        frame->local = local;
+        if (IA_ISDIR(entry->d_stat.ia_type))
+            continue;
 
-        local->xattr_req = (xdata) ? dict_ref (xdata) : dict_new ();
-        if (!local->xattr_req)
-                goto out;
+        if (dict_get(entry->dict, GF_XATTR_SHARD_FILE_SIZE) &&
+            frame->root->pid != GF_CLIENT_PID_GSYNCD)
+            shard_modify_size_and_block_count(&entry->d_stat, entry->dict);
 
-        local->vector = iov_dup (vector, count);
-        if (!local->vector)
-                goto out;
+        if (!entry->inode)
+            continue;
 
-        for (i = 0; i < count; i++)
-                local->total_size += vector[i].iov_len;
+        shard_inode_ctx_update(entry->inode, this, entry->dict, &entry->d_stat);
+    }
 
-        local->count = count;
-        local->offset = offset;
-        local->flags = flags;
-        local->iobref = iobref_ref (iobref);
-        local->fd = fd_ref (fd);
-        local->block_size = block_size;
-        local->first_block = get_lowest_block (offset, local->block_size);
-        local->last_block = get_highest_block (offset, local->total_size,
-                                               local->block_size);
-        local->num_blocks = local->last_block - local->first_block + 1;
-        local->inode_list = GF_CALLOC (local->num_blocks, sizeof (inode_t *),
-                                       gf_shard_mt_inode_list);
-        if (!local->inode_list)
-                goto out;
-
-        gf_log (this->name, GF_LOG_TRACE, "gfid=%s first_block=%"PRIu32" "
-                "last_block=%"PRIu32" num_blocks=%"PRIu32" offset=%"PRId64" "
-                "total_size=%lu", uuid_utoa (fd->inode->gfid),
-                local->first_block, local->last_block, local->num_blocks,
-                offset, local->total_size);
-
-        local->dot_shard_loc.inode = inode_find (this->itable,
-                                                 priv->dot_shard_gfid);
-        if (!local->dot_shard_loc.inode)
-                shard_writev_mkdir_dot_shard (frame, this);
-        else
-                shard_common_resolve_shards (frame, this);
+    local->op_ret = op_ret;
 
+    if (last_entry) {
+        if (local->fop == GF_FOP_READDIR)
+            STACK_WIND(frame, shard_readdir_past_dot_shard_cbk,
+                       FIRST_CHILD(this), FIRST_CHILD(this)->fops->readdir,
+                       local->fd, local->readdir_size, local->offset,
+                       local->xattr_req);
+        else
+            STACK_WIND(frame, shard_readdir_past_dot_shard_cbk,
+                       FIRST_CHILD(this), FIRST_CHILD(this)->fops->readdirp,
+                       local->fd, local->readdir_size, local->offset,
+                       local->xattr_req);
         return 0;
-out:
-        SHARD_STACK_UNWIND (writev, frame, -1, ENOMEM, NULL, NULL, NULL);
-        return 0;
+    }
+
+unwind:
+    if (local->fop == GF_FOP_READDIR)
+        SHARD_STACK_UNWIND(readdir, frame, op_ret, op_errno,
+                           &local->entries_head, xdata);
+    else
+        SHARD_STACK_UNWIND(readdirp, frame, op_ret, op_errno,
+                           &local->entries_head, xdata);
+    return 0;
 }
 
 int
-shard_flush_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                 int32_t op_ret, int32_t op_errno, dict_t *xdata)
+shard_readdir_do(call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
+                 off_t offset, int whichop, dict_t *xdata)
 {
-        /* To-Do: Wind flush on all shards of the file */
-        SHARD_STACK_UNWIND (flush, frame, op_ret, op_errno, xdata);
-        return 0;
+    int ret = 0;
+    shard_local_t *local = NULL;
+
+    local = mem_get0(this->local_pool);
+    if (!local) {
+        goto err;
+    }
+
+    frame->local = local;
+
+    local->fd = fd_ref(fd);
+    local->fop = whichop;
+    local->readdir_size = size;
+    INIT_LIST_HEAD(&local->entries_head.list);
+    local->list_inited = _gf_true;
+
+    if (whichop == GF_FOP_READDIR) {
+        STACK_WIND(frame, shard_readdir_cbk, FIRST_CHILD(this),
+                   FIRST_CHILD(this)->fops->readdir, fd, size, offset, xdata);
+    } else {
+        local->xattr_req = (xdata) ? dict_ref(xdata) : dict_new();
+        SHARD_MD_READ_FOP_INIT_REQ_DICT(this, local->xattr_req, fd->inode->gfid,
+                                        local, err);
+        ret = dict_set_uint64(local->xattr_req, GF_XATTR_SHARD_BLOCK_SIZE, 0);
+        if (ret) {
+            gf_log(this->name, GF_LOG_WARNING,
+                   "Failed to set "
+                   "dict value: key:%s, directory gfid=%s",
+                   GF_XATTR_SHARD_BLOCK_SIZE, uuid_utoa(fd->inode->gfid));
+            goto err;
+        }
+
+        STACK_WIND(frame, shard_readdir_cbk, FIRST_CHILD(this),
+                   FIRST_CHILD(this)->fops->readdirp, fd, size, offset,
+                   local->xattr_req);
+    }
+
+    return 0;
+
+err:
+    STACK_UNWIND_STRICT(readdir, frame, -1, ENOMEM, NULL, NULL);
+    return 0;
 }
 
-int
-shard_flush (call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *xdata)
+int32_t
+shard_readdir(call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
+              off_t offset, dict_t *xdata)
 {
-        STACK_WIND (frame, shard_flush_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->flush, fd, xdata);
-        return 0;
+    shard_readdir_do(frame, this, fd, size, offset, GF_FOP_READDIR, xdata);
+    return 0;
 }
 
-int
-shard_fsync_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                 int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
-                 struct iatt *postbuf, dict_t *xdata)
+int32_t
+shard_readdirp(call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
+               off_t offset, dict_t *xdata)
 {
-        /* To-Do: Wind fsync on all shards of the file */
-        SHARD_STACK_UNWIND (fsync, frame, op_ret, op_errno, prebuf, postbuf,
-                            xdata);
-        return 0;
+    shard_readdir_do(frame, this, fd, size, offset, GF_FOP_READDIRP, xdata);
+    return 0;
 }
 
-int
-shard_fsync (call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t datasync,
-             dict_t *xdata)
+int32_t
+shard_modify_and_set_iatt_in_dict(dict_t *xdata, shard_local_t *local,
+                                  char *key)
 {
-        STACK_WIND (frame, shard_fsync_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->fsync, fd, datasync, xdata);
+    int ret = 0;
+    struct iatt *tmpbuf = NULL;
+    struct iatt *stbuf = NULL;
+    data_t *data = NULL;
+
+    if (!xdata)
+        return 0;
+
+    data = dict_get(xdata, key);
+    if (!data)
         return 0;
+
+    tmpbuf = data_to_iatt(data, key);
+    stbuf = GF_MALLOC(sizeof(struct iatt), gf_common_mt_char);
+    if (stbuf == NULL) {
+        local->op_ret = -1;
+        local->op_errno = ENOMEM;
+        goto err;
+    }
+    *stbuf = *tmpbuf;
+    stbuf->ia_size = local->prebuf.ia_size;
+    stbuf->ia_blocks = local->prebuf.ia_blocks;
+    ret = dict_set_iatt(xdata, key, stbuf, false);
+    if (ret < 0) {
+        local->op_ret = -1;
+        local->op_errno = ENOMEM;
+        goto err;
+    }
+    return 0;
+
+err:
+    GF_FREE(stbuf);
+    return -1;
 }
 
 int32_t
-shard_readdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                   int32_t op_ret, int32_t op_errno, gf_dirent_t *entries,
-                   dict_t *xdata)
+shard_common_remove_xattr_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                              int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
-        fd_t          *fd    = NULL;
-        gf_dirent_t   *entry = NULL;
-        gf_dirent_t   *tmp   = NULL;
-        shard_local_t *local = NULL;
-        gf_dirent_t    skipped;
+    int ret = -1;
+    shard_local_t *local = NULL;
 
-        INIT_LIST_HEAD (&skipped.list);
+    local = frame->local;
 
-        local = frame->local;
-        fd = local->fd;
+    if (op_ret < 0) {
+        local->op_ret = op_ret;
+        local->op_errno = op_errno;
+        goto err;
+    }
 
-        if (op_ret < 0)
-                goto unwind;
+    ret = shard_modify_and_set_iatt_in_dict(xdata, local, GF_PRESTAT);
+    if (ret < 0)
+        goto err;
 
-        list_for_each_entry_safe (entry, tmp, &entries->list, list) {
-                if (__is_root_gfid (fd->inode->gfid) &&
-                    !(strcmp (entry->d_name, GF_SHARD_DIR))) {
-                        list_del_init (&entry->list);
-                        list_add_tail (&entry->list, &skipped.list);
-                        break;
-                }
-        }
+    ret = shard_modify_and_set_iatt_in_dict(xdata, local, GF_POSTSTAT);
+    if (ret < 0)
+        goto err;
 
-unwind:
-        SHARD_STACK_UNWIND (readdir, frame, op_ret, op_errno, entries, xdata);
-        gf_dirent_free (&skipped);
-        return 0;
-}
+    if (local->fd)
+        SHARD_STACK_UNWIND(fremovexattr, frame, local->op_ret, local->op_errno,
+                           xdata);
+    else
+        SHARD_STACK_UNWIND(removexattr, frame, local->op_ret, local->op_errno,
+                           xdata);
+    return 0;
 
+err:
+    shard_common_failure_unwind(local->fop, frame, local->op_ret,
+                                local->op_errno);
+    return 0;
+}
 
-int
-shard_readdir_do (call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
-               off_t offset, int whichop, dict_t *xdata)
+int32_t
+shard_post_lookup_remove_xattr_handler(call_frame_t *frame, xlator_t *this)
 {
-        int             op_errno = ENOMEM;
-        shard_local_t  *local    = NULL;
+    shard_local_t *local = NULL;
 
-        local = mem_get0 (this->local_pool);
-        if (!local)
-                goto err;
+    local = frame->local;
 
-        frame->local = local;
-
-        local->fd = fd_ref (fd);
+    if (local->op_ret < 0) {
+        shard_common_failure_unwind(local->fop, frame, local->op_ret,
+                                    local->op_errno);
+        return 0;
+    }
+
+    if (local->fd)
+        STACK_WIND(frame, shard_common_remove_xattr_cbk, FIRST_CHILD(this),
+                   FIRST_CHILD(this)->fops->fremovexattr, local->fd,
+                   local->name, local->xattr_req);
+    else
+        STACK_WIND(frame, shard_common_remove_xattr_cbk, FIRST_CHILD(this),
+                   FIRST_CHILD(this)->fops->removexattr, &local->loc,
+                   local->name, local->xattr_req);
+    return 0;
+}
 
-        if (whichop == GF_FOP_READDIR)
-                STACK_WIND (frame, shard_readdir_cbk, FIRST_CHILD(this),
-                            FIRST_CHILD(this)->fops->readdir, fd, size, offset,
+int32_t
+shard_common_remove_xattr(call_frame_t *frame, xlator_t *this,
+                          glusterfs_fop_t fop, loc_t *loc, fd_t *fd,
+                          const char *name, dict_t *xdata)
+{
+    int ret = -1;
+    int op_errno = ENOMEM;
+    uint64_t block_size = 0;
+    shard_local_t *local = NULL;
+    inode_t *inode = loc ? loc->inode : fd->inode;
+
+    if ((IA_ISDIR(inode->ia_type)) || (IA_ISLNK(inode->ia_type))) {
+        if (loc)
+            STACK_WIND_TAIL(frame, FIRST_CHILD(this),
+                            FIRST_CHILD(this)->fops->removexattr, loc, name,
                             xdata);
         else
-                STACK_WIND (frame, shard_readdir_cbk, FIRST_CHILD(this),
-                            FIRST_CHILD(this)->fops->readdirp, fd, size, offset,
+            STACK_WIND_TAIL(frame, FIRST_CHILD(this),
+                            FIRST_CHILD(this)->fops->fremovexattr, fd, name,
                             xdata);
-
         return 0;
-
-err:
-        STACK_UNWIND_STRICT (readdir, frame, -1, op_errno, NULL, NULL);
+    }
+
+    /* If shard's special xattrs are attempted to be removed,
+     * fail the fop with EPERM (except if the client is gsyncd).
+     */
+    if (frame->root->pid != GF_CLIENT_PID_GSYNCD) {
+        GF_IF_NATIVE_XATTR_GOTO(SHARD_XATTR_PREFIX "*", name, op_errno, err);
+    }
+
+    /* Repeat the same check for bulk-removexattr */
+    if (xdata && (frame->root->pid != GF_CLIENT_PID_GSYNCD)) {
+        dict_del(xdata, GF_XATTR_SHARD_BLOCK_SIZE);
+        dict_del(xdata, GF_XATTR_SHARD_FILE_SIZE);
+    }
+
+    ret = shard_inode_ctx_get_block_size(inode, this, &block_size);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, SHARD_MSG_INODE_CTX_GET_FAILED,
+               "Failed to get block size from inode ctx of %s",
+               uuid_utoa(inode->gfid));
+        goto err;
+    }
+
+    if (!block_size || frame->root->pid == GF_CLIENT_PID_GSYNCD) {
+        if (loc)
+            STACK_WIND_TAIL(frame, FIRST_CHILD(this),
+                            FIRST_CHILD(this)->fops->removexattr, loc, name,
+                            xdata);
+        else
+            STACK_WIND_TAIL(frame, FIRST_CHILD(this),
+                            FIRST_CHILD(this)->fops->fremovexattr, fd, name,
+                            xdata);
         return 0;
+    }
+
+    local = mem_get0(this->local_pool);
+    if (!local)
+        goto err;
+
+    frame->local = local;
+    local->fop = fop;
+    if (loc) {
+        if (loc_copy(&local->loc, loc) != 0)
+            goto err;
+    }
+
+    if (fd) {
+        local->fd = fd_ref(fd);
+        local->loc.inode = inode_ref(fd->inode);
+        gf_uuid_copy(local->loc.gfid, fd->inode->gfid);
+    }
+
+    if (name) {
+        local->name = gf_strdup(name);
+        if (!local->name)
+            goto err;
+    }
+
+    if (xdata)
+        local->xattr_req = dict_ref(xdata);
+
+    shard_refresh_base_file(frame, this, loc, fd,
+                            shard_post_lookup_remove_xattr_handler);
+    return 0;
+err:
+    shard_common_failure_unwind(fop, frame, -1, op_errno);
+    return 0;
+}
 
+int32_t
+shard_removexattr(call_frame_t *frame, xlator_t *this, loc_t *loc,
+                  const char *name, dict_t *xdata)
+{
+    shard_common_remove_xattr(frame, this, GF_FOP_REMOVEXATTR, loc, NULL, name,
+                              xdata);
+    return 0;
 }
 
+int32_t
+shard_fremovexattr(call_frame_t *frame, xlator_t *this, fd_t *fd,
+                   const char *name, dict_t *xdata)
+{
+    shard_common_remove_xattr(frame, this, GF_FOP_FREMOVEXATTR, NULL, fd, name,
+                              xdata);
+    return 0;
+}
 
 int32_t
-shard_readdir (call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
-               off_t offset, dict_t *xdata)
+shard_fgetxattr_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                    int32_t op_ret, int32_t op_errno, dict_t *dict,
+                    dict_t *xdata)
 {
-        shard_readdir_do (frame, this, fd, size, offset, GF_FOP_READDIR, xdata);
-        return 0;
+    if (op_ret < 0)
+        goto unwind;
+
+    if (dict && (frame->root->pid != GF_CLIENT_PID_GSYNCD)) {
+        dict_del(dict, GF_XATTR_SHARD_BLOCK_SIZE);
+        dict_del(dict, GF_XATTR_SHARD_FILE_SIZE);
+    }
+
+unwind:
+    SHARD_STACK_UNWIND(fgetxattr, frame, op_ret, op_errno, dict, xdata);
+    return 0;
 }
 
+int32_t
+shard_fgetxattr(call_frame_t *frame, xlator_t *this, fd_t *fd, const char *name,
+                dict_t *xdata)
+{
+    int op_errno = EINVAL;
+
+    if ((frame->root->pid != GF_CLIENT_PID_GSYNCD) && (name) &&
+        (!strncmp(name, SHARD_XATTR_PREFIX, SLEN(SHARD_XATTR_PREFIX)))) {
+        op_errno = ENODATA;
+        goto out;
+    }
+
+    STACK_WIND(frame, shard_fgetxattr_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->fgetxattr, fd, name, xdata);
+    return 0;
+out:
+    shard_common_failure_unwind(GF_FOP_FGETXATTR, frame, -1, op_errno);
+    return 0;
+}
+
+int32_t
+shard_getxattr_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                   int32_t op_ret, int32_t op_errno, dict_t *dict,
+                   dict_t *xdata)
+{
+    if (op_ret < 0)
+        goto unwind;
+
+    if (dict && (frame->root->pid != GF_CLIENT_PID_GSYNCD)) {
+        dict_del(dict, GF_XATTR_SHARD_BLOCK_SIZE);
+        dict_del(dict, GF_XATTR_SHARD_FILE_SIZE);
+    }
+
+unwind:
+    SHARD_STACK_UNWIND(getxattr, frame, op_ret, op_errno, dict, xdata);
+    return 0;
+}
+
+int32_t
+shard_getxattr(call_frame_t *frame, xlator_t *this, loc_t *loc,
+               const char *name, dict_t *xdata)
+{
+    int op_errno = EINVAL;
+
+    if ((frame->root->pid != GF_CLIENT_PID_GSYNCD) && (name) &&
+        (!strncmp(name, SHARD_XATTR_PREFIX, sizeof(SHARD_XATTR_PREFIX) - 1))) {
+        op_errno = ENODATA;
+        goto out;
+    }
+
+    STACK_WIND(frame, shard_getxattr_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->getxattr, loc, name, xdata);
+    return 0;
+out:
+    shard_common_failure_unwind(GF_FOP_GETXATTR, frame, -1, op_errno);
+    return 0;
+}
 
 int32_t
-shard_readdirp (call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
-                off_t offset, dict_t *xdata)
+shard_common_set_xattr_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                           int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
-        shard_readdir_do (frame, this, fd, size, offset, GF_FOP_READDIRP,
-                          xdata);
+    int ret = -1;
+    shard_local_t *local = NULL;
+
+    local = frame->local;
+
+    if (op_ret < 0) {
+        local->op_ret = op_ret;
+        local->op_errno = op_errno;
+        goto err;
+    }
+
+    ret = shard_modify_and_set_iatt_in_dict(xdata, local, GF_PRESTAT);
+    if (ret < 0)
+        goto err;
+
+    ret = shard_modify_and_set_iatt_in_dict(xdata, local, GF_POSTSTAT);
+    if (ret < 0)
+        goto err;
+
+    if (local->fd)
+        SHARD_STACK_UNWIND(fsetxattr, frame, local->op_ret, local->op_errno,
+                           xdata);
+    else
+        SHARD_STACK_UNWIND(setxattr, frame, local->op_ret, local->op_errno,
+                           xdata);
+    return 0;
+
+err:
+    shard_common_failure_unwind(local->fop, frame, local->op_ret,
+                                local->op_errno);
+    return 0;
+}
+
+int32_t
+shard_post_lookup_set_xattr_handler(call_frame_t *frame, xlator_t *this)
+{
+    shard_local_t *local = NULL;
+
+    local = frame->local;
+
+    if (local->op_ret < 0) {
+        shard_common_failure_unwind(local->fop, frame, local->op_ret,
+                                    local->op_errno);
         return 0;
+    }
+
+    if (local->fd)
+        STACK_WIND(frame, shard_common_set_xattr_cbk, FIRST_CHILD(this),
+                   FIRST_CHILD(this)->fops->fsetxattr, local->fd,
+                   local->xattr_req, local->flags, local->xattr_rsp);
+    else
+        STACK_WIND(frame, shard_common_set_xattr_cbk, FIRST_CHILD(this),
+                   FIRST_CHILD(this)->fops->setxattr, &local->loc,
+                   local->xattr_req, local->flags, local->xattr_rsp);
+    return 0;
 }
 
-int
-shard_setattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                   int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
-                   struct iatt *postbuf, dict_t *xdata)
+int32_t
+shard_common_set_xattr(call_frame_t *frame, xlator_t *this, glusterfs_fop_t fop,
+                       loc_t *loc, fd_t *fd, dict_t *dict, int32_t flags,
+                       dict_t *xdata)
 {
-        /* To-Do: Call fsetattr on all shards. */
-        SHARD_STACK_UNWIND (setattr, frame, op_ret, op_errno, prebuf, postbuf,
+    int ret = -1;
+    int op_errno = ENOMEM;
+    uint64_t block_size = 0;
+    shard_local_t *local = NULL;
+    inode_t *inode = loc ? loc->inode : fd->inode;
+
+    if ((IA_ISDIR(inode->ia_type)) || (IA_ISLNK(inode->ia_type))) {
+        if (loc)
+            STACK_WIND_TAIL(frame, FIRST_CHILD(this),
+                            FIRST_CHILD(this)->fops->setxattr, loc, dict, flags,
+                            xdata);
+        else
+            STACK_WIND_TAIL(frame, FIRST_CHILD(this),
+                            FIRST_CHILD(this)->fops->fsetxattr, fd, dict, flags,
                             xdata);
         return 0;
+    }
+
+    /* Sharded or not, if shard's special xattrs are attempted to be set,
+     * fail the fop with EPERM (except if the client is gsyncd.
+     */
+    if (frame->root->pid != GF_CLIENT_PID_GSYNCD) {
+        GF_IF_INTERNAL_XATTR_GOTO(SHARD_XATTR_PREFIX "*", dict, op_errno, err);
+    }
+
+    ret = shard_inode_ctx_get_block_size(inode, this, &block_size);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, SHARD_MSG_INODE_CTX_GET_FAILED,
+               "Failed to get block size from inode ctx of %s",
+               uuid_utoa(inode->gfid));
+        goto err;
+    }
+
+    if (!block_size || frame->root->pid == GF_CLIENT_PID_GSYNCD) {
+        if (loc)
+            STACK_WIND_TAIL(frame, FIRST_CHILD(this),
+                            FIRST_CHILD(this)->fops->setxattr, loc, dict, flags,
+                            xdata);
+        else
+            STACK_WIND_TAIL(frame, FIRST_CHILD(this),
+                            FIRST_CHILD(this)->fops->fsetxattr, fd, dict, flags,
+                            xdata);
+        return 0;
+    }
+
+    local = mem_get0(this->local_pool);
+    if (!local)
+        goto err;
+
+    frame->local = local;
+    local->fop = fop;
+    if (loc) {
+        if (loc_copy(&local->loc, loc) != 0)
+            goto err;
+    }
+
+    if (fd) {
+        local->fd = fd_ref(fd);
+        local->loc.inode = inode_ref(fd->inode);
+        gf_uuid_copy(local->loc.gfid, fd->inode->gfid);
+    }
+    local->flags = flags;
+    /* Reusing local->xattr_req and local->xattr_rsp to store the setxattr dict
+     * and the xdata dict
+     */
+    if (dict)
+        local->xattr_req = dict_ref(dict);
+    if (xdata)
+        local->xattr_rsp = dict_ref(xdata);
+
+    shard_refresh_base_file(frame, this, loc, fd,
+                            shard_post_lookup_set_xattr_handler);
+    return 0;
+err:
+    shard_common_failure_unwind(fop, frame, -1, op_errno);
+    return 0;
 }
 
-int
-shard_setattr (call_frame_t *frame, xlator_t *this, loc_t *loc,
-               struct iatt *stbuf, int32_t valid, dict_t *xdata)
+int32_t
+shard_fsetxattr(call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *dict,
+                int32_t flags, dict_t *xdata)
 {
-        STACK_WIND (frame, shard_setattr_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->setattr, loc, stbuf, valid, xdata);
+    shard_common_set_xattr(frame, this, GF_FOP_FSETXATTR, NULL, fd, dict, flags,
+                           xdata);
+    return 0;
+}
 
-        return 0;
+int32_t
+shard_setxattr(call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *dict,
+               int32_t flags, dict_t *xdata)
+{
+    shard_common_set_xattr(frame, this, GF_FOP_SETXATTR, loc, NULL, dict, flags,
+                           xdata);
+    return 0;
 }
 
 int
-shard_fsetattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                   int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
-                   struct iatt *postbuf, dict_t *xdata)
+shard_post_setattr_handler(call_frame_t *frame, xlator_t *this)
 {
-        /* To-Do: Call fsetattr on all shards. */
-        SHARD_STACK_UNWIND (fsetattr, frame, op_ret, op_errno, prebuf, postbuf,
-                            xdata);
-        return 0;
+    shard_local_t *local = NULL;
+
+    local = frame->local;
+
+    if (local->fop == GF_FOP_SETATTR) {
+        if (local->op_ret >= 0)
+            shard_inode_ctx_set(local->loc.inode, this, &local->postbuf, 0,
+                                SHARD_LOOKUP_MASK);
+        SHARD_STACK_UNWIND(setattr, frame, local->op_ret, local->op_errno,
+                           &local->prebuf, &local->postbuf, local->xattr_rsp);
+    } else if (local->fop == GF_FOP_FSETATTR) {
+        if (local->op_ret >= 0)
+            shard_inode_ctx_set(local->fd->inode, this, &local->postbuf, 0,
+                                SHARD_LOOKUP_MASK);
+        SHARD_STACK_UNWIND(fsetattr, frame, local->op_ret, local->op_errno,
+                           &local->prebuf, &local->postbuf, local->xattr_rsp);
+    }
+
+    return 0;
 }
 
 int
-shard_fsetattr (call_frame_t *frame, xlator_t *this, fd_t *fd,
-                struct iatt *stbuf, int32_t valid, dict_t *xdata)
+shard_common_setattr_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                         int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
+                         struct iatt *postbuf, dict_t *xdata)
 {
-        STACK_WIND (frame, shard_fsetattr_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->fsetattr, fd, stbuf, valid, xdata);
+    shard_local_t *local = NULL;
+
+    local = frame->local;
+
+    if (op_ret < 0) {
+        local->op_ret = op_ret;
+        local->op_errno = op_errno;
+        goto unwind;
+    }
+
+    local->prebuf = *prebuf;
+    if (shard_modify_size_and_block_count(&local->prebuf, xdata)) {
+        local->op_ret = -1;
+        local->op_errno = EINVAL;
+        goto unwind;
+    }
+    if (xdata)
+        local->xattr_rsp = dict_ref(xdata);
+    local->postbuf = *postbuf;
+    local->postbuf.ia_size = local->prebuf.ia_size;
+    local->postbuf.ia_blocks = local->prebuf.ia_blocks;
 
-        return 0;
+unwind:
+    local->handler(frame, this);
+    return 0;
 }
 
 int
-shard_fallocate (call_frame_t *frame, xlator_t *this, fd_t *fd,
-                 int32_t keep_size, off_t offset, size_t len, dict_t *xdata)
+shard_setattr(call_frame_t *frame, xlator_t *this, loc_t *loc,
+              struct iatt *stbuf, int32_t valid, dict_t *xdata)
 {
-        /* TBD */
-        SHARD_STACK_UNWIND (fallocate, frame, -1, ENOTCONN, NULL, NULL, NULL);
+    int ret = -1;
+    uint64_t block_size = 0;
+    shard_local_t *local = NULL;
+
+    if ((IA_ISDIR(loc->inode->ia_type)) || (IA_ISLNK(loc->inode->ia_type))) {
+        STACK_WIND(frame, default_setattr_cbk, FIRST_CHILD(this),
+                   FIRST_CHILD(this)->fops->setattr, loc, stbuf, valid, xdata);
         return 0;
+    }
+
+    ret = shard_inode_ctx_get_block_size(loc->inode, this, &block_size);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, SHARD_MSG_INODE_CTX_GET_FAILED,
+               "Failed to get block size from inode ctx of %s",
+               uuid_utoa(loc->inode->gfid));
+        goto err;
+    }
+
+    if (!block_size || frame->root->pid == GF_CLIENT_PID_GSYNCD) {
+        STACK_WIND(frame, default_setattr_cbk, FIRST_CHILD(this),
+                   FIRST_CHILD(this)->fops->setattr, loc, stbuf, valid, xdata);
+        return 0;
+    }
+
+    local = mem_get0(this->local_pool);
+    if (!local)
+        goto err;
+
+    frame->local = local;
+
+    local->handler = shard_post_setattr_handler;
+    local->xattr_req = (xdata) ? dict_ref(xdata) : dict_new();
+    if (!local->xattr_req)
+        goto err;
+    local->fop = GF_FOP_SETATTR;
+    loc_copy(&local->loc, loc);
+
+    SHARD_MD_READ_FOP_INIT_REQ_DICT(this, local->xattr_req, local->loc.gfid,
+                                    local, err);
+
+    STACK_WIND(frame, shard_common_setattr_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->setattr, loc, stbuf, valid,
+               local->xattr_req);
+    return 0;
+err:
+    shard_common_failure_unwind(GF_FOP_SETATTR, frame, -1, ENOMEM);
+    return 0;
 }
 
 int
-shard_discard (call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
-              size_t len, dict_t *xdata)
+shard_fsetattr(call_frame_t *frame, xlator_t *this, fd_t *fd,
+               struct iatt *stbuf, int32_t valid, dict_t *xdata)
 {
-        /* TBD */
-        SHARD_STACK_UNWIND (discard, frame, -1, ENOTCONN, NULL, NULL, NULL);
+    int ret = -1;
+    uint64_t block_size = 0;
+    shard_local_t *local = NULL;
+
+    if ((IA_ISDIR(fd->inode->ia_type)) || (IA_ISLNK(fd->inode->ia_type))) {
+        STACK_WIND(frame, default_fsetattr_cbk, FIRST_CHILD(this),
+                   FIRST_CHILD(this)->fops->fsetattr, fd, stbuf, valid, xdata);
         return 0;
+    }
+
+    ret = shard_inode_ctx_get_block_size(fd->inode, this, &block_size);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, SHARD_MSG_INODE_CTX_GET_FAILED,
+               "Failed to get block size from inode ctx of %s",
+               uuid_utoa(fd->inode->gfid));
+        goto err;
+    }
+
+    if (!block_size || frame->root->pid == GF_CLIENT_PID_GSYNCD) {
+        STACK_WIND(frame, default_fsetattr_cbk, FIRST_CHILD(this),
+                   FIRST_CHILD(this)->fops->fsetattr, fd, stbuf, valid, xdata);
+        return 0;
+    }
+
+    if (!this->itable)
+        this->itable = fd->inode->table;
+
+    local = mem_get0(this->local_pool);
+    if (!local)
+        goto err;
+
+    frame->local = local;
+
+    local->handler = shard_post_setattr_handler;
+    local->xattr_req = (xdata) ? dict_ref(xdata) : dict_new();
+    if (!local->xattr_req)
+        goto err;
+    local->fop = GF_FOP_FSETATTR;
+    local->fd = fd_ref(fd);
+
+    SHARD_MD_READ_FOP_INIT_REQ_DICT(this, local->xattr_req, fd->inode->gfid,
+                                    local, err);
+
+    STACK_WIND(frame, shard_common_setattr_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->fsetattr, fd, stbuf, valid,
+               local->xattr_req);
+    return 0;
+err:
+    shard_common_failure_unwind(GF_FOP_FSETATTR, frame, -1, ENOMEM);
+    return 0;
 }
 
 int
-shard_zerofill (call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
-                off_t len, dict_t *xdata)
+shard_common_inode_write_begin(call_frame_t *frame, xlator_t *this,
+                               glusterfs_fop_t fop, fd_t *fd,
+                               struct iovec *vector, int32_t count,
+                               off_t offset, uint32_t flags, size_t len,
+                               struct iobref *iobref, dict_t *xdata)
 {
-        /* TBD */
-        SHARD_STACK_UNWIND (zerofill, frame, -1, ENOTCONN, NULL, NULL, NULL);
+    int ret = 0;
+    int i = 0;
+    uint64_t block_size = 0;
+    shard_local_t *local = NULL;
+
+    ret = shard_inode_ctx_get_block_size(fd->inode, this, &block_size);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, SHARD_MSG_INODE_CTX_GET_FAILED,
+               "Failed to get block "
+               "size for %s from its inode ctx",
+               uuid_utoa(fd->inode->gfid));
+        goto out;
+    }
+
+    if (!block_size || frame->root->pid == GF_CLIENT_PID_GSYNCD) {
+        /* block_size = 0 means that the file was created before
+         * sharding was enabled on the volume.
+         */
+        switch (fop) {
+            case GF_FOP_WRITE:
+                STACK_WIND_TAIL(frame, FIRST_CHILD(this),
+                                FIRST_CHILD(this)->fops->writev, fd, vector,
+                                count, offset, flags, iobref, xdata);
+                break;
+            case GF_FOP_FALLOCATE:
+                STACK_WIND_TAIL(frame, FIRST_CHILD(this),
+                                FIRST_CHILD(this)->fops->fallocate, fd, flags,
+                                offset, len, xdata);
+                break;
+            case GF_FOP_ZEROFILL:
+                STACK_WIND_TAIL(frame, FIRST_CHILD(this),
+                                FIRST_CHILD(this)->fops->zerofill, fd, offset,
+                                len, xdata);
+                break;
+            case GF_FOP_DISCARD:
+                STACK_WIND_TAIL(frame, FIRST_CHILD(this),
+                                FIRST_CHILD(this)->fops->discard, fd, offset,
+                                len, xdata);
+                break;
+            default:
+                gf_msg(this->name, GF_LOG_WARNING, 0, SHARD_MSG_INVALID_FOP,
+                       "Invalid fop id = %d", fop);
+                break;
+        }
         return 0;
-}
+    }
 
-int32_t
-mem_acct_init (xlator_t *this)
-{
-        int     ret = -1;
+    if (!this->itable)
+        this->itable = fd->inode->table;
 
-        if (!this)
-                return ret;
+    local = mem_get0(this->local_pool);
+    if (!local)
+        goto out;
 
-        ret = xlator_mem_acct_init (this, gf_shard_mt_end + 1);
+    frame->local = local;
 
-        if (ret != 0) {
-                gf_log(this->name, GF_LOG_ERROR, "Memory accounting init"
-                       "failed");
-                return ret;
-        }
+    ret = syncbarrier_init(&local->barrier);
+    if (ret)
+        goto out;
+    local->xattr_req = (xdata) ? dict_ref(xdata) : dict_new();
+    if (!local->xattr_req)
+        goto out;
 
-        return ret;
+    if (vector) {
+        local->vector = iov_dup(vector, count);
+        if (!local->vector)
+            goto out;
+        for (i = 0; i < count; i++)
+            local->total_size += vector[i].iov_len;
+        local->count = count;
+    } else {
+        local->total_size = len;
+    }
+
+    local->fop = fop;
+    local->offset = offset;
+    local->flags = flags;
+    if (iobref)
+        local->iobref = iobref_ref(iobref);
+    local->fd = fd_ref(fd);
+    local->block_size = block_size;
+    local->resolver_base_inode = local->fd->inode;
+    GF_ATOMIC_INIT(local->delta_blocks, 0);
+
+    local->loc.inode = inode_ref(fd->inode);
+    gf_uuid_copy(local->loc.gfid, fd->inode->gfid);
+
+    shard_refresh_base_file(frame, this, NULL, fd,
+                            shard_common_inode_write_post_lookup_handler);
+    return 0;
+out:
+    shard_common_failure_unwind(fop, frame, -1, ENOMEM);
+    return 0;
 }
 
 int
-init (xlator_t *this)
+shard_writev(call_frame_t *frame, xlator_t *this, fd_t *fd,
+             struct iovec *vector, int32_t count, off_t offset, uint32_t flags,
+             struct iobref *iobref, dict_t *xdata)
 {
-        int           ret  = -1;
-        shard_priv_t *priv = NULL;
+    shard_common_inode_write_begin(frame, this, GF_FOP_WRITE, fd, vector, count,
+                                   offset, flags, 0, iobref, xdata);
+    return 0;
+}
 
-        if (!this) {
-                gf_log ("shard", GF_LOG_ERROR, "this is NULL. init() failed");
-                goto out;
-        }
+int
+shard_fallocate(call_frame_t *frame, xlator_t *this, fd_t *fd,
+                int32_t keep_size, off_t offset, size_t len, dict_t *xdata)
+{
+    if ((keep_size != 0) && (keep_size != FALLOC_FL_ZERO_RANGE) &&
+        (keep_size != (FALLOC_FL_KEEP_SIZE | FALLOC_FL_PUNCH_HOLE)))
+        goto out;
 
-        if (!this->parents) {
-                gf_log (this->name, GF_LOG_ERROR, "Dangling volume. "
-                        "Check volfile");
-                goto out;
-        }
+    shard_common_inode_write_begin(frame, this, GF_FOP_FALLOCATE, fd, NULL, 0,
+                                   offset, keep_size, len, NULL, xdata);
+    return 0;
+out:
+    shard_common_failure_unwind(GF_FOP_FALLOCATE, frame, -1, ENOTSUP);
+    return 0;
+}
 
-        if (!this->children || this->children->next) {
-                gf_log (this->name, GF_LOG_ERROR, "shard not configured with "
-                        "exactly one sub-volume. Check volfile");
-                goto out;
-        }
+int
+shard_zerofill(call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
+               off_t len, dict_t *xdata)
+{
+    shard_common_inode_write_begin(frame, this, GF_FOP_ZEROFILL, fd, NULL, 0,
+                                   offset, 0, len, NULL, xdata);
+    return 0;
+}
 
-        priv = GF_CALLOC (1, sizeof (shard_priv_t), gf_shard_mt_priv_t);
-        if (!priv)
-                goto out;
+int
+shard_discard(call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
+              size_t len, dict_t *xdata)
+{
+    shard_common_inode_write_begin(frame, this, GF_FOP_DISCARD, fd, NULL, 0,
+                                   offset, 0, len, NULL, xdata);
+    return 0;
+}
 
-        GF_OPTION_INIT ("shard-block-size", priv->block_size, size_uint64, out);
+int32_t
+shard_seek(call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
+           gf_seek_what_t what, dict_t *xdata)
+{
+    /* TBD */
+    gf_msg(this->name, GF_LOG_INFO, ENOTSUP, SHARD_MSG_FOP_NOT_SUPPORTED,
+           "seek called on %s.", uuid_utoa(fd->inode->gfid));
+    shard_common_failure_unwind(GF_FOP_SEEK, frame, -1, ENOTSUP);
+    return 0;
+}
 
-        this->local_pool = mem_pool_new (shard_local_t, 128);
-        if (!this->local_pool) {
-                ret = -1;
-                gf_log (this->name, GF_LOG_ERROR, "Failed to allocate locals "
-                        "from mempool");
-                goto out;
-        }
-        gf_uuid_parse (SHARD_ROOT_GFID, priv->dot_shard_gfid);
+int32_t
+mem_acct_init(xlator_t *this)
+{
+    int ret = -1;
 
-        this->private = priv;
-        ret = 0;
-out:
-        if  (ret) {
-                GF_FREE (priv);
-                mem_pool_destroy (this->local_pool);
-        }
+    if (!this)
+        return ret;
+
+    ret = xlator_mem_acct_init(this, gf_shard_mt_end + 1);
 
+    if (ret != 0) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, SHARD_MSG_MEM_ACCT_INIT_FAILED,
+               "Memory accounting init"
+               "failed");
         return ret;
+    }
 
+    return ret;
+}
+
+int
+init(xlator_t *this)
+{
+    int ret = -1;
+    shard_priv_t *priv = NULL;
+
+    if (!this) {
+        gf_msg("shard", GF_LOG_ERROR, 0, SHARD_MSG_NULL_THIS,
+               "this is NULL. init() failed");
+        return -1;
+    }
+
+    if (!this->parents) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, SHARD_MSG_INVALID_VOLFILE,
+               "Dangling volume. Check volfile");
+        goto out;
+    }
+
+    if (!this->children || this->children->next) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, SHARD_MSG_INVALID_VOLFILE,
+               "shard not configured with exactly one sub-volume. "
+               "Check volfile");
+        goto out;
+    }
+
+    priv = GF_CALLOC(1, sizeof(shard_priv_t), gf_shard_mt_priv_t);
+    if (!priv)
+        goto out;
+
+    GF_OPTION_INIT("shard-block-size", priv->block_size, size_uint64, out);
+
+    GF_OPTION_INIT("shard-deletion-rate", priv->deletion_rate, uint32, out);
+
+    GF_OPTION_INIT("shard-lru-limit", priv->lru_limit, uint64, out);
+
+    this->local_pool = mem_pool_new(shard_local_t, 128);
+    if (!this->local_pool) {
+        ret = -1;
+        goto out;
+    }
+    gf_uuid_parse(SHARD_ROOT_GFID, priv->dot_shard_gfid);
+    gf_uuid_parse(DOT_SHARD_REMOVE_ME_GFID, priv->dot_shard_rm_gfid);
+
+    this->private = priv;
+    LOCK_INIT(&priv->lock);
+    INIT_LIST_HEAD(&priv->ilist_head);
+    ret = 0;
+out:
+    if (ret) {
+        GF_FREE(priv);
+        mem_pool_destroy(this->local_pool);
+    }
+
+    return ret;
 }
 
 void
-fini (xlator_t *this)
+fini(xlator_t *this)
 {
-        shard_priv_t *priv = NULL;
+    shard_priv_t *priv = NULL;
 
-        GF_VALIDATE_OR_GOTO ("shard", this, out);
+    GF_VALIDATE_OR_GOTO("shard", this, out);
 
-        mem_pool_destroy (this->local_pool);
-        this->local_pool = NULL;
+    /*Itable was not created by shard, hence setting to NULL.*/
+    this->itable = NULL;
 
-        priv = this->private;
-        if (!priv)
-                goto out;
+    mem_pool_destroy(this->local_pool);
+    this->local_pool = NULL;
 
-        this->private = NULL;
-        GF_FREE (priv);
+    priv = this->private;
+    if (!priv)
+        goto out;
+
+    this->private = NULL;
+    LOCK_DESTROY(&priv->lock);
+    GF_FREE(priv);
 
 out:
-        return;
+    return;
 }
 
 int
-reconfigure (xlator_t *this, dict_t *options)
+reconfigure(xlator_t *this, dict_t *options)
 {
-        int            ret  = -1;
-        shard_priv_t  *priv = NULL;
+    int ret = -1;
+    shard_priv_t *priv = NULL;
 
-        priv = this->private;
+    priv = this->private;
 
-        GF_OPTION_RECONF ("shard-block-size", priv->block_size, options, size,
-                          out);
+    GF_OPTION_RECONF("shard-block-size", priv->block_size, options, size, out);
 
-        ret = 0;
+    GF_OPTION_RECONF("shard-deletion-rate", priv->deletion_rate, options,
+                     uint32, out);
+    ret = 0;
 
 out:
-        return ret;
+    return ret;
 }
 
 int
-shard_forget (xlator_t *this, inode_t *inode)
+shard_forget(xlator_t *this, inode_t *inode)
 {
-        uint64_t            ctx_uint = 0;
-        shard_inode_ctx_t  *ctx      = NULL;
+    uint64_t ctx_uint = 0;
+    shard_inode_ctx_t *ctx = NULL;
+    shard_priv_t *priv = NULL;
 
-        inode_ctx_del (inode, this, &ctx_uint);
-        if (!ctx_uint)
-                return 0;
+    priv = this->private;
+    if (!priv)
+        return 0;
 
-        ctx = (shard_inode_ctx_t *)ctx_uint;
+    inode_ctx_del(inode, this, &ctx_uint);
+    if (!ctx_uint)
+        return 0;
 
-        GF_FREE (ctx);
+    ctx = (shard_inode_ctx_t *)(uintptr_t)ctx_uint;
 
-        return 0;
+    /* When LRU limit reaches inode will be forcefully removed from the
+     * table, inode needs to be removed from LRU of shard as well.
+     */
+    if (!list_empty(&ctx->ilist)) {
+        LOCK(&priv->lock);
+        {
+            list_del_init(&ctx->ilist);
+            priv->inode_count--;
+        }
+        UNLOCK(&priv->lock);
+    }
+    GF_FREE(ctx);
+
+    return 0;
 }
 
 int
-shard_release (xlator_t *this, fd_t *fd)
+shard_release(xlator_t *this, fd_t *fd)
 {
-        /* TBD */
-        return 0;
+    /* TBD */
+    return 0;
 }
 
 int
-shard_priv_dump (xlator_t *this)
+shard_priv_dump(xlator_t *this)
 {
-        /* TBD */
-        return 0;
+    shard_priv_t *priv = NULL;
+    char key_prefix[GF_DUMP_MAX_BUF_LEN] = {
+        0,
+    };
+    char *str = NULL;
+
+    priv = this->private;
+
+    snprintf(key_prefix, GF_DUMP_MAX_BUF_LEN, "%s.%s", this->type, this->name);
+    gf_proc_dump_add_section("%s", key_prefix);
+    str = gf_uint64_2human_readable(priv->block_size);
+    gf_proc_dump_write("shard-block-size", "%s", str);
+    gf_proc_dump_write("inode-count", "%d", priv->inode_count);
+    gf_proc_dump_write("ilist_head", "%p", &priv->ilist_head);
+    gf_proc_dump_write("lru-max-limit", "%" PRIu64, priv->lru_limit);
+
+    GF_FREE(str);
+
+    return 0;
 }
 
 int
-shard_releasedir (xlator_t *this, fd_t *fd)
+shard_releasedir(xlator_t *this, fd_t *fd)
 {
-        return 0;
+    return 0;
 }
 
 struct xlator_fops fops = {
-        .lookup      = shard_lookup,
-        .open        = shard_open,
-        .flush       = shard_flush,
-        .fsync       = shard_fsync,
-        .stat        = shard_stat,
-        .fstat       = shard_fstat,
-        .readv       = shard_readv,
-        .writev      = shard_writev,
-        .truncate    = shard_truncate,
-        .ftruncate   = shard_ftruncate,
-        .setattr     = shard_setattr,
-        .fsetattr    = shard_fsetattr,
-        .fallocate   = shard_fallocate,
-        .discard     = shard_discard,
-        .zerofill    = shard_zerofill,
-        .readdir     = shard_readdir,
-        .readdirp    = shard_readdirp,
-        .create      = shard_create,
-        .mknod       = shard_mknod,
-        .unlink      = shard_unlink,
-        .rename      = shard_rename,
+    .lookup = shard_lookup,
+    .open = shard_open,
+    .flush = shard_flush,
+    .fsync = shard_fsync,
+    .stat = shard_stat,
+    .fstat = shard_fstat,
+    .getxattr = shard_getxattr,
+    .fgetxattr = shard_fgetxattr,
+    .readv = shard_readv,
+    .writev = shard_writev,
+    .truncate = shard_truncate,
+    .ftruncate = shard_ftruncate,
+    .setxattr = shard_setxattr,
+    .fsetxattr = shard_fsetxattr,
+    .setattr = shard_setattr,
+    .fsetattr = shard_fsetattr,
+    .removexattr = shard_removexattr,
+    .fremovexattr = shard_fremovexattr,
+    .fallocate = shard_fallocate,
+    .discard = shard_discard,
+    .zerofill = shard_zerofill,
+    .readdir = shard_readdir,
+    .readdirp = shard_readdirp,
+    .create = shard_create,
+    .mknod = shard_mknod,
+    .link = shard_link,
+    .unlink = shard_unlink,
+    .rename = shard_rename,
+    .seek = shard_seek,
 };
 
 struct xlator_cbks cbks = {
-        .forget  = shard_forget,
-        .release = shard_release,
-        .releasedir = shard_releasedir,
+    .forget = shard_forget,
+    .release = shard_release,
+    .releasedir = shard_releasedir,
 };
 
 struct xlator_dumpops dumpops = {
-        .priv = shard_priv_dump,
+    .priv = shard_priv_dump,
 };
 
 struct volume_options options[] = {
-        {  .key = {"shard-block-size"},
-           .type = GF_OPTION_TYPE_SIZET,
-           .default_value = "4MB",
-           .min = SHARD_MIN_BLOCK_SIZE,
-           .max = SHARD_MAX_BLOCK_SIZE,
-           .description = "The size unit used to break a file into multiple "
-                          "chunks",
-        },
-        { .key = {NULL} },
+    {
+        .key = {"shard"},
+        .type = GF_OPTION_TYPE_BOOL,
+        .default_value = "off",
+        .description = "enable/disable shard",
+        .op_version = {GD_OP_VERSION_6_0},
+        .flags = OPT_FLAG_SETTABLE,
+    },
+    {
+        .key = {"shard-block-size"},
+        .type = GF_OPTION_TYPE_SIZET,
+        .op_version = {GD_OP_VERSION_3_7_0},
+        .flags = OPT_FLAG_SETTABLE | OPT_FLAG_CLIENT_OPT | OPT_FLAG_DOC,
+        .tags = {"shard"},
+        .default_value = "64MB",
+        .min = SHARD_MIN_BLOCK_SIZE,
+        .max = SHARD_MAX_BLOCK_SIZE,
+        .description = "The size unit used to break a file into multiple "
+                       "chunks",
+    },
+    {
+        .key = {"shard-deletion-rate"},
+        .type = GF_OPTION_TYPE_INT,
+        .op_version = {GD_OP_VERSION_5_0},
+        .flags = OPT_FLAG_SETTABLE | OPT_FLAG_CLIENT_OPT | OPT_FLAG_DOC,
+        .tags = {"shard"},
+        .default_value = "100",
+        .min = 100,
+        .max = INT_MAX,
+        .description = "The number of shards to send deletes on at a time",
+    },
+    {
+        .key = {"shard-lru-limit"},
+        .type = GF_OPTION_TYPE_INT,
+        .op_version = {GD_OP_VERSION_5_0},
+        .flags = OPT_FLAG_SETTABLE | OPT_FLAG_CLIENT_OPT,
+        .tags = {"shard"},
+        .default_value = "16384",
+        .min = 20,
+        .max = INT_MAX,
+        .description = "The number of resolved shard inodes to keep in "
+                       "memory. A higher number means shards that are "
+                       "resolved will remain in memory longer, avoiding "
+                       "frequent lookups on them when they participate in "
+                       "file operations. The option also has a bearing on "
+                       "amount of memory consumed by these inodes and their "
+                       "internal metadata",
+    },
+    {.key = {NULL}},
+};
+
+xlator_api_t xlator_api = {
+    .init = init,
+    .fini = fini,
+    .reconfigure = reconfigure,
+    .mem_acct_init = mem_acct_init,
+    .op_version = {1}, /* Present from the initial version */
+    .dumpops = &dumpops,
+    .fops = &fops,
+    .cbks = &cbks,
+    .options = options,
+    .identifier = "shard",
+    .category = GF_MAINTAINED,
 };
diff --git a/xlators/features/shard/src/shard.h b/xlators/features/shard/src/shard.h
index 0eeabc51f5e..4fe181b64d5 100644
--- a/xlators/features/shard/src/shard.h
+++ b/xlators/features/shard/src/shard.h
@@ -8,161 +8,341 @@
   cases as published by the Free Software Foundation.
 */
 
-
 #ifndef __SHARD_H__
 #define __SHARD_H__
 
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
-
-#include "xlator.h"
-#include "compat-errno.h"
+#include <glusterfs/xlator.h>
+#include <glusterfs/compat-errno.h>
+#include "shard-messages.h"
+#include <glusterfs/syncop.h>
 
 #define GF_SHARD_DIR ".shard"
-#define SHARD_MIN_BLOCK_SIZE  (4 * GF_UNIT_MB)
-#define SHARD_MAX_BLOCK_SIZE  (4 * GF_UNIT_TB)
+#define GF_SHARD_REMOVE_ME_DIR ".remove_me"
+#define SHARD_MIN_BLOCK_SIZE (4 * GF_UNIT_MB)
+#define SHARD_MAX_BLOCK_SIZE (4 * GF_UNIT_TB)
+#define SHARD_XATTR_PREFIX "trusted.glusterfs.shard."
 #define GF_XATTR_SHARD_BLOCK_SIZE "trusted.glusterfs.shard.block-size"
-#define GF_XATTR_SHARD_FILE_SIZE  "trusted.glusterfs.shard.file-size"
-#define SHARD_ROOT_GFID "be318638-e8a0-4c6d-977d-7a937aa84806"
-#define SHARD_INODE_LRU_LIMIT 4096
-
-#define get_lowest_block(off, shard_size) (off / shard_size)
-#define get_highest_block(off, len, shard_size) ((off+len-1) / shard_size)
-
-#define SHARD_ENTRY_FOP_CHECK(loc, op_errno, label) do {               \
-        if ((loc->name && !strcmp (GF_SHARD_DIR, loc->name)) &&        \
-            (((loc->parent) &&                                          \
-            __is_root_gfid (loc->parent->gfid)) ||                     \
-            __is_root_gfid (loc->pargfid))) {                           \
-                    op_errno = EPERM;                                  \
-                    goto label;                                        \
-        }                                                              \
-                                                                       \
-        if ((loc->parent &&                                            \
-            __is_shard_dir (loc->parent->gfid)) ||                     \
-            __is_shard_dir (loc->pargfid)) {                           \
-                    op_errno = EPERM;                                  \
-                    goto label;                                        \
-        }                                                              \
-} while (0)
-
-#define SHARD_INODE_OP_CHECK(gfid, err, label) do {                    \
-        if (__is_shard_dir(gfid)) {                                    \
-                err = EPERM;                                           \
-                goto label;                                            \
-        }                                                              \
-} while (0)
-
-#define SHARD_STACK_UNWIND(fop, frame, params ...) do {       \
-        shard_local_t *__local = NULL;                         \
-        if (frame) {                                           \
-                __local = frame->local;                        \
-                frame->local = NULL;                           \
-        }                                                      \
-        STACK_UNWIND_STRICT (fop, frame, params);              \
-        if (__local) {                                         \
-                shard_local_wipe (__local);                    \
-                mem_put (__local);                             \
-        }                                                      \
-} while (0)
-
-
-#define SHARD_INODE_CREATE_INIT(this, local, xattr_req, loc, label) do {      \
-        int            __ret       = -1;                                      \
-        uint64_t      *__size_attr = NULL;                                    \
-        shard_priv_t  *__priv      = NULL;                                    \
-                                                                              \
-        __priv = this->private;                                               \
-                                                                              \
-        local->block_size = hton64 (__priv->block_size);                      \
-        __ret = dict_set_static_bin (xattr_req, GF_XATTR_SHARD_BLOCK_SIZE,    \
-                                     &local->block_size,                      \
-                                     sizeof (local->block_size));             \
-        if (__ret) {                                                          \
-                gf_log (this->name, GF_LOG_WARNING, "Failed to set key: %s "  \
-                        "on path %s", GF_XATTR_SHARD_BLOCK_SIZE, loc->path);  \
-                goto label;                                                   \
-        }                                                                     \
-                                                                              \
-        __ret = shard_set_size_attrs (0, 0, &__size_attr);                    \
-        if (__ret)                                                            \
-                goto label;                                                   \
-                                                                              \
-        __ret = dict_set_bin (xattr_req, GF_XATTR_SHARD_FILE_SIZE,            \
-                              __size_attr, 8 * 4);                            \
-        if (__ret) {                                                          \
-                gf_log (this->name, GF_LOG_WARNING, "Failed to set key: %s "  \
-                        "on path %s", GF_XATTR_SHARD_FILE_SIZE, loc->path);   \
-                GF_FREE (__size_attr);                                        \
-                goto label;                                                   \
-        }                                                                     \
-} while (0)
-
-
-#define SHARD_MD_READ_FOP_INIT_REQ_DICT(this, xattr_req, gfid, label) do {    \
-        int __ret = -1;                                                       \
-                                                                              \
-        __ret = dict_set_uint64 (xattr_req, GF_XATTR_SHARD_FILE_SIZE, 8 * 4); \
-        if (__ret) {                                                          \
-                gf_log (this->name, GF_LOG_WARNING, "Failed to set dict"      \
-                        " value: key:%s for %s.", GF_XATTR_SHARD_FILE_SIZE,   \
-                        uuid_utoa (gfid));                                    \
-                goto label;                                                   \
-        }                                                                     \
-} while (0)
+/**
+ *  Bit masks for the valid flag, which is used while updating ctx
+ **/
+#define SHARD_MASK_BLOCK_SIZE (1 << 0)
+#define SHARD_MASK_PROT (1 << 1)
+#define SHARD_MASK_NLINK (1 << 2)
+#define SHARD_MASK_UID (1 << 3)
+#define SHARD_MASK_GID (1 << 4)
+#define SHARD_MASK_SIZE (1 << 6)
+#define SHARD_MASK_BLOCKS (1 << 7)
+#define SHARD_MASK_TIMES (1 << 8)
+#define SHARD_MASK_OTHERS (1 << 9)
+#define SHARD_MASK_REFRESH_RESET (1 << 10)
+
+#define SHARD_INODE_WRITE_MASK                                                 \
+    (SHARD_MASK_SIZE | SHARD_MASK_BLOCKS | SHARD_MASK_TIMES)
+
+#define SHARD_LOOKUP_MASK                                                      \
+    (SHARD_MASK_PROT | SHARD_MASK_NLINK | SHARD_MASK_UID | SHARD_MASK_GID |    \
+     SHARD_MASK_TIMES | SHARD_MASK_OTHERS)
+
+#define SHARD_ALL_MASK                                                         \
+    (SHARD_MASK_BLOCK_SIZE | SHARD_MASK_PROT | SHARD_MASK_NLINK |              \
+     SHARD_MASK_UID | SHARD_MASK_GID | SHARD_MASK_SIZE | SHARD_MASK_BLOCKS |   \
+     SHARD_MASK_TIMES | SHARD_MASK_OTHERS)
+
+#define get_lowest_block(off, shard_size) ((off) / (shard_size))
+#define get_highest_block(off, len, shard_size)                                \
+    (((((off) + (len)) == 0) ? 0 : ((off) + (len)-1)) / (shard_size))
+
+int
+shard_unlock_inodelk(call_frame_t *frame, xlator_t *this);
+
+int
+shard_unlock_entrylk(call_frame_t *frame, xlator_t *this);
+
+#define SHARD_ENTRY_FOP_CHECK(loc, op_errno, label)                            \
+    do {                                                                       \
+        if ((loc->name && !strcmp(GF_SHARD_DIR, loc->name)) &&                 \
+            (((loc->parent) && __is_root_gfid(loc->parent->gfid)) ||           \
+             __is_root_gfid(loc->pargfid))) {                                  \
+            op_errno = EPERM;                                                  \
+            goto label;                                                        \
+        }                                                                      \
+                                                                               \
+        if ((loc->parent && __is_shard_dir(loc->parent->gfid)) ||              \
+            __is_shard_dir(loc->pargfid)) {                                    \
+            op_errno = EPERM;                                                  \
+            goto label;                                                        \
+        }                                                                      \
+    } while (0)
+
+#define SHARD_INODE_OP_CHECK(gfid, err, label)                                 \
+    do {                                                                       \
+        if (__is_shard_dir(gfid)) {                                            \
+            err = EPERM;                                                       \
+            goto label;                                                        \
+        }                                                                      \
+    } while (0)
+
+#define SHARD_STACK_UNWIND(fop, frame, params...)                              \
+    do {                                                                       \
+        shard_local_t *__local = NULL;                                         \
+        if (frame) {                                                           \
+            __local = frame->local;                                            \
+            if (__local && __local->int_inodelk.acquired_lock)                 \
+                shard_unlock_inodelk(frame, frame->this);                      \
+            if (__local && __local->int_entrylk.acquired_lock)                 \
+                shard_unlock_entrylk(frame, frame->this);                      \
+            frame->local = NULL;                                               \
+        }                                                                      \
+        STACK_UNWIND_STRICT(fop, frame, params);                               \
+        if (__local) {                                                         \
+            shard_local_wipe(__local);                                         \
+            mem_put(__local);                                                  \
+        }                                                                      \
+    } while (0)
+
+#define SHARD_STACK_DESTROY(frame)                                             \
+    do {                                                                       \
+        shard_local_t *__local = NULL;                                         \
+        __local = frame->local;                                                \
+        frame->local = NULL;                                                   \
+        STACK_DESTROY(frame->root);                                            \
+        if (__local) {                                                         \
+            shard_local_wipe(__local);                                         \
+            mem_put(__local);                                                  \
+        }                                                                      \
+    } while (0);
 
+#define SHARD_INODE_CREATE_INIT(this, block_size, xattr_req, loc, size,        \
+                                block_count, label)                            \
+    do {                                                                       \
+        int __ret = -1;                                                        \
+        int64_t *__size_attr = NULL;                                           \
+        uint64_t *__bs = 0;                                                    \
+                                                                               \
+        __bs = GF_MALLOC(sizeof(uint64_t), gf_shard_mt_uint64_t);              \
+        if (!__bs)                                                             \
+            goto label;                                                        \
+        *__bs = hton64(block_size);                                            \
+        __ret = dict_set_bin(xattr_req, GF_XATTR_SHARD_BLOCK_SIZE, __bs,       \
+                             sizeof(*__bs));                                   \
+        if (__ret) {                                                           \
+            gf_msg(this->name, GF_LOG_WARNING, 0, SHARD_MSG_DICT_OP_FAILED,    \
+                   "Failed to set key: %s "                                    \
+                   "on path %s",                                               \
+                   GF_XATTR_SHARD_BLOCK_SIZE, (loc)->path);                    \
+            GF_FREE(__bs);                                                     \
+            goto label;                                                        \
+        }                                                                      \
+                                                                               \
+        __ret = shard_set_size_attrs(size, block_count, &__size_attr);         \
+        if (__ret)                                                             \
+            goto label;                                                        \
+                                                                               \
+        __ret = dict_set_bin(xattr_req, GF_XATTR_SHARD_FILE_SIZE, __size_attr, \
+                             8 * 4);                                           \
+        if (__ret) {                                                           \
+            gf_msg(this->name, GF_LOG_WARNING, 0, SHARD_MSG_DICT_OP_FAILED,    \
+                   "Failed to set key: %s "                                    \
+                   "on path %s",                                               \
+                   GF_XATTR_SHARD_FILE_SIZE, (loc)->path);                     \
+            GF_FREE(__size_attr);                                              \
+            goto label;                                                        \
+        }                                                                      \
+    } while (0)
+
+#define SHARD_MD_READ_FOP_INIT_REQ_DICT(this, dict, gfid, local, label)        \
+    do {                                                                       \
+        int __ret = -1;                                                        \
+                                                                               \
+        __ret = dict_set_uint64(dict, GF_XATTR_SHARD_FILE_SIZE, 8 * 4);        \
+        if (__ret) {                                                           \
+            local->op_ret = -1;                                                \
+            local->op_errno = ENOMEM;                                          \
+            gf_msg(this->name, GF_LOG_WARNING, 0, SHARD_MSG_DICT_OP_FAILED,    \
+                   "Failed to set dict value:"                                 \
+                   " key:%s for %s.",                                          \
+                   GF_XATTR_SHARD_FILE_SIZE, uuid_utoa(gfid));                 \
+            goto label;                                                        \
+        }                                                                      \
+    } while (0)
+
+#define SHARD_SET_ROOT_FS_ID(frame, local)                                     \
+    do {                                                                       \
+        if (!local->is_set_fsid) {                                             \
+            local->uid = frame->root->uid;                                     \
+            local->gid = frame->root->gid;                                     \
+            frame->root->uid = 0;                                              \
+            frame->root->gid = 0;                                              \
+            local->is_set_fsid = _gf_true;                                     \
+        }                                                                      \
+    } while (0)
+
+#define SHARD_UNSET_ROOT_FS_ID(frame, local)                                   \
+    do {                                                                       \
+        if (local->is_set_fsid) {                                              \
+            frame->root->uid = local->uid;                                     \
+            frame->root->gid = local->gid;                                     \
+            local->is_set_fsid = _gf_false;                                    \
+        }                                                                      \
+    } while (0)
+
+#define SHARD_TIME_UPDATE(ctx_sec, ctx_nsec, new_sec, new_nsec)                \
+    do {                                                                       \
+        if (ctx_sec == new_sec)                                                \
+            ctx_nsec = new_nsec = max(new_nsec, ctx_nsec);                     \
+        else if (ctx_sec > new_sec) {                                          \
+            new_sec = ctx_sec;                                                 \
+            new_nsec = ctx_nsec;                                               \
+        } else {                                                               \
+            ctx_sec = new_sec;                                                 \
+            ctx_nsec = new_nsec;                                               \
+        }                                                                      \
+    } while (0)
+
+typedef enum {
+    SHARD_BG_DELETION_NONE = 0,
+    SHARD_BG_DELETION_LAUNCHING,
+    SHARD_BG_DELETION_IN_PROGRESS,
+} shard_bg_deletion_state_t;
+
+/* rm = "remove me" */
 
 typedef struct shard_priv {
-        uint64_t block_size;
-        uuid_t dot_shard_gfid;
-        inode_t *dot_shard_inode;
+    uint64_t block_size;
+    uuid_t dot_shard_gfid;
+    uuid_t dot_shard_rm_gfid;
+    inode_t *dot_shard_inode;
+    inode_t *dot_shard_rm_inode;
+    gf_lock_t lock;
+    int inode_count;
+    struct list_head ilist_head;
+    uint32_t deletion_rate;
+    shard_bg_deletion_state_t bg_del_state;
+    gf_boolean_t first_lookup_done;
+    uint64_t lru_limit;
 } shard_priv_t;
 
 typedef struct {
-        loc_t *loc;
-        short type;
-        char *domain;
-} shard_lock_t;
+    loc_t loc;
+    char *domain;
+    struct gf_flock flock;
+    gf_boolean_t acquired_lock;
+} shard_inodelk_t;
+
+typedef struct {
+    loc_t loc;
+    char *domain;
+    char *basename;
+    entrylk_cmd cmd;
+    entrylk_type type;
+    gf_boolean_t acquired_lock;
+} shard_entrylk_t;
+
+typedef int32_t (*shard_post_fop_handler_t)(call_frame_t *frame,
+                                            xlator_t *this);
+typedef int32_t (*shard_post_resolve_fop_handler_t)(call_frame_t *frame,
+                                                    xlator_t *this);
+typedef int32_t (*shard_post_lookup_shards_fop_handler_t)(call_frame_t *frame,
+                                                          xlator_t *this);
+
+typedef int32_t (*shard_post_mknod_fop_handler_t)(call_frame_t *frame,
+                                                  xlator_t *this);
+
+typedef int32_t (*shard_post_update_size_fop_handler_t)(call_frame_t *frame,
+                                                        xlator_t *this);
 
 typedef struct shard_local {
-        int op_ret;
-        int op_errno;
-        int first_block;
-        int last_block;
-        int num_blocks;
-        int call_count;
-        int eexist_count;
-        int xflag;
-        int count;
-        uint32_t flags;
-        uint64_t block_size;
-        off_t offset;
-        size_t total_size;
-        size_t written_size;
-        loc_t loc;
-        loc_t dot_shard_loc;
-        fd_t *fd;
-        dict_t *xattr_req;
-        dict_t *xattr_rsp;
-        inode_t **inode_list;
-        struct iatt prebuf;
-        struct iatt postbuf;
-        struct iovec *vector;
-        struct iobref *iobref;
-        struct {
-                int lock_count;
-                fop_inodelk_cbk_t inodelk_cbk;
-                shard_lock_t *shard_lock;
-        } lock;
+    int op_ret;
+    int op_errno;
+    uint64_t first_block;
+    uint64_t last_block;
+    uint64_t num_blocks;
+    int call_count;
+    int eexist_count;
+    int create_count;
+    int xflag;
+    int count;
+    uint32_t flags;
+    uint32_t uid;
+    uint32_t gid;
+    uint64_t block_size;
+    uint64_t dst_block_size;
+    int32_t datasync;
+    off_t offset;
+    size_t total_size;
+    size_t written_size;
+    size_t hole_size;
+    size_t req_size;
+    size_t readdir_size;
+    int64_t delta_size;
+    gf_atomic_t delta_blocks;
+    loc_t loc;
+    loc_t dot_shard_loc;
+    loc_t dot_shard_rm_loc;
+    loc_t loc2;
+    loc_t tmp_loc;
+    fd_t *fd;
+    dict_t *xattr_req;
+    dict_t *xattr_rsp;
+    inode_t **inode_list;
+    glusterfs_fop_t fop;
+    struct iatt prebuf;
+    struct iatt postbuf;
+    struct iatt preoldparent;
+    struct iatt postoldparent;
+    struct iatt prenewparent;
+    struct iatt postnewparent;
+    struct iovec *vector;
+    struct iobref *iobref;
+    struct iobuf *iobuf;
+    gf_dirent_t entries_head;
+    gf_boolean_t is_set_fsid;
+    gf_boolean_t list_inited;
+    shard_post_fop_handler_t handler;
+    shard_post_lookup_shards_fop_handler_t pls_fop_handler;
+    shard_post_resolve_fop_handler_t post_res_handler;
+    shard_post_mknod_fop_handler_t post_mknod_handler;
+    shard_post_update_size_fop_handler_t post_update_size_handler;
+    shard_inodelk_t int_inodelk;
+    shard_entrylk_t int_entrylk;
+    inode_t *resolver_base_inode;
+    gf_boolean_t first_lookup_done;
+    syncbarrier_t barrier;
+    gf_boolean_t lookup_shards_barriered;
+    gf_boolean_t unlink_shards_barriered;
+    gf_boolean_t resolve_not;
+    loc_t newloc;
+    call_frame_t *main_frame;
+    call_frame_t *inodelk_frame;
+    call_frame_t *entrylk_frame;
+    uint32_t deletion_rate;
+    gf_boolean_t cleanup_required;
+    uuid_t base_gfid;
+    char *name;
 } shard_local_t;
 
 typedef struct shard_inode_ctx {
-        uint32_t rdev;
-        uint64_t block_size; /* The block size with which this inode is
-                                sharded */
-        mode_t mode;
+    uint64_t block_size; /* The block size with which this inode is
+                            sharded */
+    struct iatt stat;
+    gf_boolean_t refresh;
+    /* The following members of inode ctx will be applicable only to the
+     * individual shards' ctx and never the base file ctx.
+     */
+    struct list_head ilist;
+    uuid_t base_gfid;
+    int block_num;
+    gf_boolean_t refreshed;
+    struct list_head to_fsync_list;
+    int fsync_needed;
+    inode_t *inode;
+    int fsync_count;
+    inode_t *base_inode;
 } shard_inode_ctx_t;
 
+typedef enum {
+    SHARD_INTERNAL_DIR_DOT_SHARD = 1,
+    SHARD_INTERNAL_DIR_DOT_SHARD_REMOVE_ME,
+} shard_internal_dir_type_t;
+
 #endif /* __SHARD_H__ */
diff --git a/xlators/features/snapview-client/src/Makefile.am b/xlators/features/snapview-client/src/Makefile.am
index 8a3f6fe4ea2..fa08656c537 100644
--- a/xlators/features/snapview-client/src/Makefile.am
+++ b/xlators/features/snapview-client/src/Makefile.am
@@ -1,14 +1,15 @@
 xlator_LTLIBRARIES = snapview-client.la
 xlatordir = $(libdir)/glusterfs/$(PACKAGE_VERSION)/xlator/features
 
-snapview_client_la_LDFLAGS = -module -avoid-version
+snapview_client_la_LDFLAGS = -module $(GF_XLATOR_DEFAULT_LDFLAGS)
 
 snapview_client_la_SOURCES = snapview-client.c
 snapview_client_la_LIBADD = $(top_builddir)/libglusterfs/src/libglusterfs.la
 
-noinst_HEADERS = snapview-client.h snapview-client-mem-types.h
+noinst_HEADERS = snapview-client.h snapview-client-mem-types.h snapview-client-messages.h
 
-AM_CPPFLAGS = $(GF_CPPFLAGS) -I$(top_srcdir)/libglusterfs/src
+AM_CPPFLAGS = $(GF_CPPFLAGS) -I$(top_srcdir)/libglusterfs/src \
+	-I$(top_srcdir)/rpc/xdr/src -I$(top_builddir)/rpc/xdr/src
 
 AM_CFLAGS = -Wall $(GF_CFLAGS)
 
diff --git a/xlators/features/snapview-client/src/snapview-client-mem-types.h b/xlators/features/snapview-client/src/snapview-client-mem-types.h
index 1a0158d950e..3c3ab555a55 100644
--- a/xlators/features/snapview-client/src/snapview-client-mem-types.h
+++ b/xlators/features/snapview-client/src/snapview-client-mem-types.h
@@ -11,14 +11,14 @@
 #ifndef _SVC_MEM_TYPES_H
 #define _SVC_MEM_TYPES_H
 
-#include "mem-types.h"
+#include <glusterfs/mem-types.h>
 
 enum svc_mem_types {
-        gf_svc_mt_svc_private_t = gf_common_mt_end + 1,
-        gf_svc_mt_svc_local_t,
-        gf_svc_mt_svc_inode_t,
-        gf_svc_mt_svc_fd_t,
-        gf_svc_mt_end
+    gf_svc_mt_svc_private_t = gf_common_mt_end + 1,
+    gf_svc_mt_svc_local_t,
+    gf_svc_mt_svc_inode_t,
+    gf_svc_mt_svc_fd_t,
+    gf_svc_mt_end
 };
 
 #endif
diff --git a/xlators/features/snapview-client/src/snapview-client-messages.h b/xlators/features/snapview-client/src/snapview-client-messages.h
new file mode 100644
index 00000000000..c02fb154930
--- /dev/null
+++ b/xlators/features/snapview-client/src/snapview-client-messages.h
@@ -0,0 +1,71 @@
+/*
+ Copyright (c) 2018 Red Hat, Inc. <http://www.redhat.com>
+ This file is part of GlusterFS.
+
+ This file is licensed to you under your choice of the GNU Lesser
+ General Public License, version 3 or any later version (LGPLv3 or
+ later), or the GNU General Public License, version 2 (GPLv2), in all
+ cases as published by the Free Software Foundation.
+ */
+
+#ifndef _SNAPVIEW_CLIENT_MESSAGES_H_
+#define _SNAPVIEW_CLIENT_MESSAGES_H_
+
+#include <glusterfs/glfs-message-id.h>
+
+/* To add new message IDs, append new identifiers at the end of the list.
+ *
+ * Never remove a message ID. If it's not used anymore, you can rename it or
+ * leave it as it is, but not delete it. This is to prevent reutilization of
+ * IDs by other messages.
+ *
+ * The component name must match one of the entries defined in
+ * glfs-message-id.h.
+ */
+
+GLFS_MSGID(SNAPVIEW_CLIENT, SVC_MSG_NO_MEMORY, SVC_MSG_MEM_ACNT_FAILED,
+           SVC_MSG_SET_INODE_CONTEXT_FAILED, SVC_MSG_GET_INODE_CONTEXT_FAILED,
+           SVC_MSG_DELETE_INODE_CONTEXT_FAILED, SVC_MSG_SET_FD_CONTEXT_FAILED,
+           SVC_MSG_GET_FD_CONTEXT_FAILED, SVC_MSG_DICT_SET_FAILED,
+           SVC_MSG_SUBVOLUME_NULL, SVC_MSG_NO_CHILD_FOR_XLATOR,
+           SVC_MSG_XLATOR_CHILDREN_WRONG, SVC_MSG_NORMAL_GRAPH_LOOKUP_FAIL,
+           SVC_MSG_SNAPVIEW_GRAPH_LOOKUP_FAIL, SVC_MSG_OPENDIR_SPECIAL_DIR,
+           SVC_MSG_RENAME_SNAPSHOT_ENTRY, SVC_MSG_LINK_SNAPSHOT_ENTRY,
+           SVC_MSG_COPY_ENTRY_POINT_FAILED, SVC_MSG_ENTRY_POINT_SPECIAL_DIR,
+           SVC_MSG_STR_LEN, SVC_MSG_INVALID_ENTRY_POINT, SVC_MSG_NULL_PRIV,
+           SVC_MSG_PRIV_DESTROY_FAILED, SVC_MSG_ALLOC_FD_FAILED,
+           SVC_MSG_ALLOC_INODE_FAILED, SVC_MSG_NULL_SPECIAL_DIR,
+           SVC_MSG_MEM_POOL_GET_FAILED);
+
+#define SVC_MSG_ALLOC_FD_FAILED_STR "failed to allocate new fd context"
+#define SVC_MSG_SET_FD_CONTEXT_FAILED_STR "failed to set fd context"
+#define SVC_MSG_STR_LEN_STR                                                    \
+    "destination buffer size is less than the length of entry point name"
+#define SVC_MSG_NORMAL_GRAPH_LOOKUP_FAIL_STR "lookup failed on normal graph"
+#define SVC_MSG_SNAPVIEW_GRAPH_LOOKUP_FAIL_STR "lookup failed on snapview graph"
+#define SVC_MSG_SET_INODE_CONTEXT_FAILED_STR "failed to set inode context"
+#define SVC_MSG_NO_MEMORY_STR "failed to allocate memory"
+#define SVC_MSG_COPY_ENTRY_POINT_FAILED_STR                                    \
+    "failed to copy the entry point string"
+#define SVC_MSG_GET_FD_CONTEXT_FAILED_STR "fd context not found"
+#define SVC_MSG_GET_INODE_CONTEXT_FAILED_STR "failed to get inode context"
+#define SVC_MSG_ALLOC_INODE_FAILED_STR "failed to allocate new inode"
+#define SVC_MSG_DICT_SET_FAILED_STR "failed to set dict"
+#define SVC_MSG_RENAME_SNAPSHOT_ENTRY_STR                                      \
+    "rename happening on a entry residing in snapshot"
+#define SVC_MSG_DELETE_INODE_CONTEXT_FAILED_STR "failed to delete inode context"
+#define SVC_MSG_NULL_PRIV_STR "priv NULL"
+#define SVC_MSG_INVALID_ENTRY_POINT_STR "not a valid entry point"
+#define SVC_MSG_MEM_ACNT_FAILED_STR "Memory accouting init failed"
+#define SVC_MSG_NO_CHILD_FOR_XLATOR_STR "configured without any child"
+#define SVC_MSG_XLATOR_CHILDREN_WRONG_STR                                      \
+    "snap-view-client has got wrong subvolumes. It can have only 2"
+#define SVC_MSG_ENTRY_POINT_SPECIAL_DIR_STR                                    \
+    "entry point directory cannot be part of special directory"
+#define SVC_MSG_NULL_SPECIAL_DIR_STR "null special directory"
+#define SVC_MSG_MEM_POOL_GET_FAILED_STR                                        \
+    "could not get mem pool for frame->local"
+#define SVC_MSG_PRIV_DESTROY_FAILED_STR "failed to destroy private"
+#define SVC_MSG_LINK_SNAPSHOT_ENTRY_STR                                        \
+    "link happening on a entry residin gin snapshot"
+#endif /* !_SNAPVIEW_CLIENT_MESSAGES_H_ */
diff --git a/xlators/features/snapview-client/src/snapview-client.c b/xlators/features/snapview-client/src/snapview-client.c
index 8351922af24..486c5179d5b 100644
--- a/xlators/features/snapview-client/src/snapview-client.c
+++ b/xlators/features/snapview-client/src/snapview-client.c
@@ -1,623 +1,713 @@
- /*
-   Copyright (c) 2014 Red Hat, Inc. <http://www.redhat.com>
-   This file is part of GlusterFS.
-
-   This file is licensed to you under your choice of the GNU Lesser
-   General Public License, version 3 or any later version (LGPLv3 or
-   later), or the GNU General Public License, version 2 (GPLv2), in all
-   cases as published by the Free Software Foundation.
-*/
+/*
+  Copyright (c) 2014 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
 
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
+*/
 
 #include "snapview-client.h"
-#include "inode.h"
-#include "byte-order.h"
+#include <glusterfs/inode.h>
+#include <glusterfs/byte-order.h>
 
-
-void
-svc_local_free (svc_local_t *local)
+static void
+svc_local_free(svc_local_t *local)
 {
-        if (local) {
-                loc_wipe (&local->loc);
-                if (local->fd)
-                        fd_unref (local->fd);
-                if (local->xdata)
-                        dict_unref (local->xdata);
-                mem_put (local);
-        }
+    if (local) {
+        loc_wipe(&local->loc);
+        if (local->fd)
+            fd_unref(local->fd);
+        if (local->xdata)
+            dict_unref(local->xdata);
+        mem_put(local);
+    }
 }
 
-xlator_t *
-svc_get_subvolume (xlator_t *this, int inode_type)
+static xlator_t *
+svc_get_subvolume(xlator_t *this, int inode_type)
 {
-        xlator_t *subvolume = NULL;
+    xlator_t *subvolume = NULL;
 
-        GF_VALIDATE_OR_GOTO ("snapview-client", this, out);
+    GF_VALIDATE_OR_GOTO("snapview-client", this, out);
 
-        if (inode_type == VIRTUAL_INODE)
-                subvolume = SECOND_CHILD (this);
-        else
-                subvolume = FIRST_CHILD (this);
+    if (inode_type == VIRTUAL_INODE)
+        subvolume = SECOND_CHILD(this);
+    else
+        subvolume = FIRST_CHILD(this);
 
 out:
-        return subvolume;
+    return subvolume;
 }
 
-int32_t
-__svc_inode_ctx_set (xlator_t *this, inode_t *inode, int inode_type)
+static int32_t
+__svc_inode_ctx_set(xlator_t *this, inode_t *inode, int inode_type)
 {
-        uint64_t    value = 0;
-        int32_t       ret = -1;
+    uint64_t value = 0;
+    int32_t ret = -1;
 
-        GF_VALIDATE_OR_GOTO ("snapview-client", this, out);
-        GF_VALIDATE_OR_GOTO (this->name, inode, out);
+    GF_VALIDATE_OR_GOTO("snapview-client", this, out);
+    GF_VALIDATE_OR_GOTO(this->name, inode, out);
 
-        value = inode_type;
+    value = inode_type;
 
-        ret = __inode_ctx_set (inode, this, &value);
+    ret = __inode_ctx_set(inode, this, &value);
 
 out:
-        return ret;
+    return ret;
 }
 
-int
-__svc_inode_ctx_get (xlator_t *this, inode_t *inode, int *inode_type)
+static int
+__svc_inode_ctx_get(xlator_t *this, inode_t *inode, int *inode_type)
 {
-        uint64_t    value      = 0;
-        int         ret        = -1;
+    uint64_t value = 0;
+    int ret = -1;
 
-        GF_VALIDATE_OR_GOTO ("snapview-client", this, out);
-        GF_VALIDATE_OR_GOTO (this->name, inode, out);
+    GF_VALIDATE_OR_GOTO("snapview-client", this, out);
+    GF_VALIDATE_OR_GOTO(this->name, inode, out);
 
-        ret = __inode_ctx_get (inode, this, &value);
-        if (ret < 0)
-                goto out;
+    ret = __inode_ctx_get(inode, this, &value);
+    if (ret < 0)
+        goto out;
 
-        *inode_type = (int)(value);
+    *inode_type = (int)(value);
 
 out:
-        return ret;
+    return ret;
 }
 
-int
-svc_inode_ctx_get (xlator_t *this, inode_t *inode, int *inode_type)
+static int
+svc_inode_ctx_get(xlator_t *this, inode_t *inode, int *inode_type)
 {
-        int          ret        = -1;
+    int ret = -1;
 
-        GF_VALIDATE_OR_GOTO ("snapview-client", this, out);
-        GF_VALIDATE_OR_GOTO (this->name, inode, out);
+    GF_VALIDATE_OR_GOTO("snapview-client", this, out);
+    GF_VALIDATE_OR_GOTO(this->name, inode, out);
 
-        LOCK (&inode->lock);
-        {
-                ret = __svc_inode_ctx_get (this, inode, inode_type);
-        }
-        UNLOCK (&inode->lock);
+    LOCK(&inode->lock);
+    {
+        ret = __svc_inode_ctx_get(this, inode, inode_type);
+    }
+    UNLOCK(&inode->lock);
 
 out:
-        return ret;
+    return ret;
 }
 
-int32_t
-svc_inode_ctx_set (xlator_t *this, inode_t *inode, int inode_type)
+static int32_t
+svc_inode_ctx_set(xlator_t *this, inode_t *inode, int inode_type)
 {
-        int32_t   ret = -1;
+    int32_t ret = -1;
 
-        GF_VALIDATE_OR_GOTO ("snapview-client", this, out);
-        GF_VALIDATE_OR_GOTO (this->name, inode, out);
+    GF_VALIDATE_OR_GOTO("snapview-client", this, out);
+    GF_VALIDATE_OR_GOTO(this->name, inode, out);
 
-        LOCK (&inode->lock);
-        {
-                ret = __svc_inode_ctx_set (this, inode, inode_type);
-        }
-        UNLOCK (&inode->lock);
+    LOCK(&inode->lock);
+    {
+        ret = __svc_inode_ctx_set(this, inode, inode_type);
+    }
+    UNLOCK(&inode->lock);
 
 out:
-        return ret;
+    return ret;
 }
 
-svc_fd_t *
-svc_fd_new (void)
+static svc_fd_t *
+svc_fd_new(void)
 {
-        svc_fd_t    *svc_fd = NULL;
+    svc_fd_t *svc_fd = NULL;
 
-        svc_fd = GF_CALLOC (1, sizeof (*svc_fd), gf_svc_mt_svc_fd_t);
+    svc_fd = GF_CALLOC(1, sizeof(*svc_fd), gf_svc_mt_svc_fd_t);
 
-        return svc_fd;
+    return svc_fd;
 }
 
-svc_fd_t *
-__svc_fd_ctx_get (xlator_t *this, fd_t *fd)
+static svc_fd_t *
+__svc_fd_ctx_get(xlator_t *this, fd_t *fd)
 {
-        svc_fd_t *svc_fd = NULL;
-        uint64_t  value  = 0;
-        int       ret    = -1;
+    svc_fd_t *svc_fd = NULL;
+    uint64_t value = 0;
+    int ret = -1;
 
-        GF_VALIDATE_OR_GOTO ("snapview-client", this, out);
-        GF_VALIDATE_OR_GOTO (this->name, fd, out);
+    GF_VALIDATE_OR_GOTO("snapview-client", this, out);
+    GF_VALIDATE_OR_GOTO(this->name, fd, out);
 
-        ret = __fd_ctx_get (fd, this, &value);
-        if (ret)
-                return NULL;
+    ret = __fd_ctx_get(fd, this, &value);
+    if (ret)
+        return NULL;
 
-        svc_fd = (svc_fd_t *) ((long) value);
+    svc_fd = (svc_fd_t *)((long)value);
 
 out:
-        return svc_fd;
+    return svc_fd;
 }
 
-svc_fd_t *
-svc_fd_ctx_get (xlator_t *this, fd_t *fd)
+static svc_fd_t *
+svc_fd_ctx_get(xlator_t *this, fd_t *fd)
 {
-        svc_fd_t *svc_fd = NULL;
+    svc_fd_t *svc_fd = NULL;
 
-        GF_VALIDATE_OR_GOTO ("snapview-client", this, out);
-        GF_VALIDATE_OR_GOTO (this->name, fd, out);
+    GF_VALIDATE_OR_GOTO("snapview-client", this, out);
+    GF_VALIDATE_OR_GOTO(this->name, fd, out);
 
-        LOCK (&fd->lock);
-        {
-                svc_fd = __svc_fd_ctx_get (this, fd);
-        }
-        UNLOCK (&fd->lock);
+    LOCK(&fd->lock);
+    {
+        svc_fd = __svc_fd_ctx_get(this, fd);
+    }
+    UNLOCK(&fd->lock);
 
 out:
-        return svc_fd;
+    return svc_fd;
 }
 
-int
-__svc_fd_ctx_set (xlator_t *this, fd_t *fd, svc_fd_t *svc_fd)
+static int
+__svc_fd_ctx_set(xlator_t *this, fd_t *fd, svc_fd_t *svc_fd)
 {
-        uint64_t    value = 0;
-        int         ret   = -1;
+    uint64_t value = 0;
+    int ret = -1;
 
-        GF_VALIDATE_OR_GOTO ("snapview-client", this, out);
-        GF_VALIDATE_OR_GOTO (this->name, fd, out);
-        GF_VALIDATE_OR_GOTO (this->name, svc_fd, out);
+    GF_VALIDATE_OR_GOTO("snapview-client", this, out);
+    GF_VALIDATE_OR_GOTO(this->name, fd, out);
+    GF_VALIDATE_OR_GOTO(this->name, svc_fd, out);
 
-        value = (uint64_t)(long) svc_fd;
+    value = (uint64_t)(long)svc_fd;
 
-        ret = __fd_ctx_set (fd, this, value);
+    ret = __fd_ctx_set(fd, this, value);
 
 out:
-        return ret;
+    return ret;
 }
 
-int32_t
-svc_fd_ctx_set (xlator_t *this, fd_t *fd, svc_fd_t *svc_fd)
+static svc_fd_t *
+__svc_fd_ctx_get_or_new(xlator_t *this, fd_t *fd)
 {
-        int32_t    ret = -1;
+    svc_fd_t *svc_fd = NULL;
+    int ret = -1;
+    inode_t *inode = NULL;
 
-        GF_VALIDATE_OR_GOTO ("snapview-client", this, out);
-        GF_VALIDATE_OR_GOTO (this->name, fd, out);
-        GF_VALIDATE_OR_GOTO (this->name, svc_fd, out);
+    GF_VALIDATE_OR_GOTO("snapview-client", this, out);
+    GF_VALIDATE_OR_GOTO(this->name, fd, out);
 
-        LOCK (&fd->lock);
-        {
-                ret = __svc_fd_ctx_set (this, fd, svc_fd);
-        }
-        UNLOCK (&fd->lock);
+    inode = fd->inode;
+    svc_fd = __svc_fd_ctx_get(this, fd);
+    if (svc_fd) {
+        ret = 0;
+        goto out;
+    }
+
+    svc_fd = svc_fd_new();
+    if (!svc_fd) {
+        gf_smsg(this->name, GF_LOG_ERROR, ENOMEM, SVC_MSG_ALLOC_FD_FAILED,
+                "gfid=%s", uuid_utoa(inode->gfid), NULL);
+        goto out;
+    }
+
+    ret = __svc_fd_ctx_set(this, fd, svc_fd);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_ERROR, 0, SVC_MSG_SET_FD_CONTEXT_FAILED,
+                "gfid=%s", uuid_utoa(inode->gfid), NULL);
+        ret = -1;
+    }
 
 out:
-        return ret;
+    if (ret) {
+        GF_FREE(svc_fd);
+        svc_fd = NULL;
+    }
+
+    return svc_fd;
 }
 
-svc_fd_t *
-__svc_fd_ctx_get_or_new (xlator_t *this, fd_t *fd)
+static svc_fd_t *
+svc_fd_ctx_get_or_new(xlator_t *this, fd_t *fd)
 {
-        svc_fd_t        *svc_fd    = NULL;
-        int              ret       = -1;
-        inode_t         *inode     = NULL;
-
-        GF_VALIDATE_OR_GOTO ("snapview-client", this, out);
-        GF_VALIDATE_OR_GOTO (this->name, fd, out);
+    svc_fd_t *svc_fd = NULL;
 
-        inode = fd->inode;
-        svc_fd = __svc_fd_ctx_get (this, fd);
-        if (svc_fd) {
-                ret = 0;
-                goto out;
-        }
+    GF_VALIDATE_OR_GOTO("snapview-client", this, out);
+    GF_VALIDATE_OR_GOTO(this->name, fd, out);
 
-        svc_fd = svc_fd_new ();
-        if (!svc_fd) {
-                gf_log (this->name, GF_LOG_ERROR, "failed to allocate new fd "
-                        "context for gfid %s", uuid_utoa (inode->gfid));
-                goto out;
-        }
-
-        ret = __svc_fd_ctx_set (this, fd, svc_fd);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "failed to set fd context "
-                        "for gfid %s", uuid_utoa (inode->gfid));
-                ret = -1;
-        }
+    LOCK(&fd->lock);
+    {
+        svc_fd = __svc_fd_ctx_get_or_new(this, fd);
+    }
+    UNLOCK(&fd->lock);
 
 out:
-        if (ret) {
-                GF_FREE (svc_fd);
-                svc_fd = NULL;
-        }
-
-        return svc_fd;
+    return svc_fd;
 }
 
-svc_fd_t *
-svc_fd_ctx_get_or_new (xlator_t *this, fd_t *fd)
+/**
+ * @this: xlator
+ * @entry_point: pointer to the buffer provided by consumer
+ *
+ * This function is mainly for copying the entry point name
+ * (stored as string in priv->path) to a buffer point to by
+ * @entry_point within the lock. It is for the consumer to
+ * allocate the memory for the buffer.
+ *
+ * This function is called by all the functions (or fops)
+ * who need to use priv->path for avoiding the race.
+ * For example, either in lookup or in any other fop,
+ * while priv->path is being accessed, a reconfigure can
+ * happen to change priv->path. This ensures that, a lock
+ * is taken before accessing priv->path.
+ **/
+int
+gf_svc_get_entry_point(xlator_t *this, char *entry_point, size_t dest_size)
 {
-        svc_fd_t  *svc_fd = NULL;
+    int ret = -1;
+    svc_private_t *priv = NULL;
 
-        GF_VALIDATE_OR_GOTO ("snapview-client", this, out);
-        GF_VALIDATE_OR_GOTO (this->name, fd, out);
+    GF_VALIDATE_OR_GOTO("snapview-client", this, out);
+    GF_VALIDATE_OR_GOTO(this->name, entry_point, out);
+
+    priv = this->private;
 
-        LOCK (&fd->lock);
-        {
-                svc_fd = __svc_fd_ctx_get_or_new (this, fd);
+    LOCK(&priv->lock);
+    {
+        if (dest_size <= strlen(priv->path)) {
+            gf_smsg(this->name, GF_LOG_ERROR, 0, SVC_MSG_STR_LEN,
+                    "dest-size=%zu", dest_size, "priv-path-len=%zu",
+                    strlen(priv->path), "path=%s", priv->path, NULL);
+        } else {
+            snprintf(entry_point, dest_size, "%s", priv->path);
+            ret = 0;
         }
-        UNLOCK (&fd->lock);
+    }
+    UNLOCK(&priv->lock);
 
 out:
-        return svc_fd;
+    return ret;
 }
 
-
-int32_t
-svc_lookup_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                int32_t op_ret, int32_t op_errno, inode_t *inode,
-                struct iatt *buf, dict_t *xdata, struct iatt *postparent)
+static int32_t
+gf_svc_lookup_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                  int32_t op_ret, int32_t op_errno, inode_t *inode,
+                  struct iatt *buf, dict_t *xdata, struct iatt *postparent)
 {
-        svc_local_t     *local       = NULL;
-        inode_t         *parent      = NULL;
-        xlator_t        *subvolume   = NULL;
-        gf_boolean_t     do_unwind   = _gf_true;
-        int              inode_type  = -1;
-        int              parent_type = -1;
-        int              ret         = -1;
-
-        local = frame->local;
-        subvolume = local->subvolume;
-        if (!subvolume) {
-                gf_log_callingfn (this->name, GF_LOG_ERROR, "path: %s, "
-                                  "gfid: %s ", local->loc.path,
-                                  inode?uuid_utoa (inode->gfid):"");
-                GF_ASSERT (0);
-        }
-
-        /* There is a possibility that, the client process just came online
-           and does not have the inode on which the lookup came. In that case,
-           the fresh inode created from fuse for the lookup fop, wont have
-           the inode context set without which svc cannot decide where to
-           STACK_WIND to. So by default it decides to send the fop to the
-           regular subvolume (i.e first child of the xlator). If lookup fails
-           on the regular volume, then there is a possibility that the lookup
-           is happening on a virtual inode (i.e history data residing in snaps).
-           So if lookup fails with ENOENT and the inode context is not there,
-           then send the lookup to the 2nd child of svc.
-
-           If there are any changes in volfile/client-restarted then inode-ctx
-           is lost. In this case if nameless lookup fails with ESTALE,
-           then send the lookup to the 2nd child of svc.
-        */
-        if (op_ret) {
-                if (subvolume == FIRST_CHILD (this)) {
-                        gf_log (this->name,
-                                (op_errno == ENOENT || op_errno == ESTALE)
-                                ? GF_LOG_DEBUG:GF_LOG_ERROR,
-                                "Lookup failed on normal graph with error %s",
-                                strerror (op_errno));
-                } else {
-                        gf_log (this->name,
-                                (op_errno == ENOENT || op_errno == ESTALE)
-                                ? GF_LOG_DEBUG:GF_LOG_ERROR,
-                                "Lookup failed on snapview graph with error %s",
-                                strerror (op_errno));
-                }
-
-                if ((op_errno == ENOENT || op_errno == ESTALE) &&
-                    !gf_uuid_is_null (local->loc.gfid)) {
-                        ret = svc_inode_ctx_get (this, inode, &inode_type);
-                        if (ret < 0 && subvolume == FIRST_CHILD (this)) {
-                                gf_log (this->name, GF_LOG_DEBUG,
-                                        "Lookup on normal graph failed. "
-                                        "Sending lookup to snapview-server");
-
-                                subvolume = SECOND_CHILD (this);
-                                local->subvolume = subvolume;
-                                STACK_WIND (frame, svc_lookup_cbk, subvolume,
-                                            subvolume->fops->lookup,
-                                            &local->loc, xdata);
-                                do_unwind = _gf_false;
-                        }
-                }
-
-                goto out;
-        }
-
-        if (local->loc.parent)
-                parent = inode_ref (local->loc.parent);
-        else {
-                parent = inode_parent (inode, NULL, NULL);
-                if (!parent && !gf_uuid_is_null (local->loc.pargfid)) {
-                        parent = inode_find (inode->table,
-                                             local->loc.pargfid);
-                }
+    svc_local_t *local = NULL;
+    xlator_t *subvolume = NULL;
+    gf_boolean_t do_unwind = _gf_true;
+    int inode_type = -1;
+    int ret = -1;
+
+    local = frame->local;
+    subvolume = local->subvolume;
+    if (!subvolume) {
+        gf_msg_callingfn(this->name, GF_LOG_ERROR, 0, SVC_MSG_SUBVOLUME_NULL,
+                         "path: %s gfid: %s ", local->loc.path,
+                         inode ? uuid_utoa(inode->gfid) : "");
+        GF_ASSERT(0);
+    }
+
+    /* There is a possibility that, the client process just came online
+       and does not have the inode on which the lookup came. In that case,
+       the fresh inode created from fuse for the lookup fop, won't have
+       the inode context set without which svc cannot decide where to
+       STACK_WIND to. So by default it decides to send the fop to the
+       regular subvolume (i.e first child of the xlator). If lookup fails
+       on the regular volume, then there is a possibility that the lookup
+       is happening on a virtual inode (i.e history data residing in snaps).
+       So if lookup fails with ENOENT and the inode context is not there,
+       then send the lookup to the 2nd child of svc.
+
+       If there are any changes in volfile/client-restarted then inode-ctx
+       is lost. In this case if nameless lookup fails with ESTALE,
+       then send the lookup to the 2nd child of svc.
+    */
+    if (op_ret) {
+        if (subvolume == FIRST_CHILD(this)) {
+            gf_smsg(this->name,
+                    (op_errno == ENOENT || op_errno == ESTALE) ? GF_LOG_DEBUG
+                                                               : GF_LOG_ERROR,
+                    op_errno, SVC_MSG_NORMAL_GRAPH_LOOKUP_FAIL, "error=%s",
+                    strerror(op_errno), NULL);
+        } else {
+            gf_smsg(this->name,
+                    (op_errno == ENOENT || op_errno == ESTALE) ? GF_LOG_DEBUG
+                                                               : GF_LOG_ERROR,
+                    op_errno, SVC_MSG_SNAPVIEW_GRAPH_LOOKUP_FAIL, "error=%s",
+                    strerror(op_errno), NULL);
+            goto out;
+        }
+
+        if ((op_errno == ENOENT || op_errno == ESTALE) &&
+            !gf_uuid_is_null(local->loc.gfid)) {
+            if (inode != NULL)
+                ret = svc_inode_ctx_get(this, inode, &inode_type);
+
+            if (ret < 0 || inode == NULL) {
+                gf_msg_debug(this->name, 0,
+                             "Lookup on normal graph failed. "
+                             " Sending lookup to snapview-server");
+                subvolume = SECOND_CHILD(this);
+                local->subvolume = subvolume;
+                STACK_WIND(frame, gf_svc_lookup_cbk, subvolume,
+                           subvolume->fops->lookup, &local->loc, xdata);
+                do_unwind = _gf_false;
+            }
         }
 
-        if (!__is_root_gfid (buf->ia_gfid) && parent) {
-                ret = svc_inode_ctx_get (this, parent, &parent_type);
-                if (ret < 0) {
-                        op_ret = -1;
-                        op_errno = EINVAL;
-                        gf_log (this->name, GF_LOG_WARNING,
-                                "Error fetching parent context");
-                        goto out;
-                }
-        }
+        goto out;
+    }
 
-        if (subvolume == FIRST_CHILD (this))
-                inode_type = NORMAL_INODE;
-        else
-                inode_type = VIRTUAL_INODE;
+    if (subvolume == FIRST_CHILD(this))
+        inode_type = NORMAL_INODE;
+    else
+        inode_type = VIRTUAL_INODE;
 
-        ret = svc_inode_ctx_set (this, inode, inode_type);
-        if (ret)
-                gf_log (this->name, GF_LOG_ERROR, "failed to set inode type"
-                        "into the context");
+    ret = svc_inode_ctx_set(this, inode, inode_type);
+    if (ret)
+        gf_smsg(this->name, GF_LOG_ERROR, 0, SVC_MSG_SET_INODE_CONTEXT_FAILED,
+                "gfid=%s", uuid_utoa(inode->gfid), NULL);
 
 out:
-        if (do_unwind) {
-                SVC_STACK_UNWIND (lookup, frame, op_ret, op_errno, inode, buf,
-                                  xdata, postparent);
-        }
+    if (do_unwind) {
+        SVC_STACK_UNWIND(lookup, frame, op_ret, op_errno, inode, buf, xdata,
+                         postparent);
+    }
 
-        if (parent)
-                inode_unref (parent);
-
-        return 0;
+    return 0;
 }
 
-int32_t
-svc_lookup (call_frame_t *frame, xlator_t *this, loc_t *loc,
-            dict_t *xdata)
+static int32_t
+gf_svc_lookup(call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xdata)
 {
-        int32_t        ret         =   -1;
-        svc_local_t   *local       = NULL;
-        xlator_t      *subvolume   = NULL;
-        int            op_ret      = -1;
-        int            op_errno    = EINVAL;
-        inode_t       *parent      = NULL;
-        svc_private_t *priv        = NULL;
-        dict_t        *new_xdata   = NULL;
-        int            inode_type  = -1;
-        int            parent_type = -1;
-        gf_boolean_t   wind        = _gf_false;
-
-        GF_VALIDATE_OR_GOTO ("svc", this, out);
-        GF_VALIDATE_OR_GOTO (this->name, this->private, out);
-        GF_VALIDATE_OR_GOTO (this->name, frame, out);
-        GF_VALIDATE_OR_GOTO (this->name, loc, out);
-        GF_VALIDATE_OR_GOTO (this->name, loc->inode, out);
-
-        priv = this->private;
-
-        ret = svc_inode_ctx_get (this, loc->inode, &inode_type);
-        if (!__is_root_gfid (loc->gfid)) {
-                if (loc->parent) {
-                        parent = inode_ref (loc->parent);
-                        ret = svc_inode_ctx_get (this, loc->parent,
-                                                 &parent_type);
-                } else {
-                        parent = inode_parent (loc->inode, loc->pargfid, NULL);
-                        if (parent)
-                                ret = svc_inode_ctx_get (this, parent,
-                                                         &parent_type);
-                }
-        }
-
-        local = mem_get0 (this->local_pool);
-        if (!local) {
-                gf_log (this->name, GF_LOG_ERROR, "failed to allocate local");
-                op_ret = -1;
-                op_errno = ENOMEM;
-                goto out;
-        }
-
-        frame->local = local;
-        loc_copy (&local->loc, loc);
-
-        if (__is_root_gfid (loc->inode->gfid)) {
-                subvolume = FIRST_CHILD (this);
-                GF_ASSERT (subvolume);
-                local->subvolume = subvolume;
-                wind = _gf_true;
-                goto out;
-        }
-
-        /* nfs sends nameless lookups directly using the gfid. In that case
-           loc->name will be NULL. So check if loc->name is NULL. If so, then
-           try to get the subvolume using inode context. But if the inode has
-           not been looked up yet, then send the lookup call to the first
-           subvolume.
-        */
-
-        if (!loc->name) {
-                if (gf_uuid_is_null (loc->inode->gfid)) {
-                        subvolume = FIRST_CHILD (this);
-                        local->subvolume = subvolume;
-                        wind = _gf_true;
-                        goto out;
-                } else {
-                        if (inode_type >= 0)
-                                subvolume = svc_get_subvolume (this,
-                                                               inode_type);
-                        else
-                                subvolume = FIRST_CHILD (this);
-                        local->subvolume = subvolume;
-                        wind = _gf_true;
-                        goto out;
-                }
-        }
-
-        if (strcmp (loc->name, priv->path)) {
-                if (parent_type == NORMAL_INODE) {
-                        subvolume = FIRST_CHILD (this);
-                        local->subvolume = subvolume;
-                } else {
-                        subvolume = SECOND_CHILD (this);
-                        local->subvolume = subvolume;
-                }
+    int32_t ret = -1;
+    svc_local_t *local = NULL;
+    xlator_t *subvolume = NULL;
+    int op_ret = -1;
+    int op_errno = EINVAL;
+    inode_t *parent = NULL;
+    dict_t *new_xdata = NULL;
+    int inode_type = -1;
+    int parent_type = -1;
+    gf_boolean_t wind = _gf_false;
+    char entry_point[NAME_MAX + 1] = {
+        0,
+    };
+
+    GF_VALIDATE_OR_GOTO("svc", this, out);
+    GF_VALIDATE_OR_GOTO(this->name, frame, out);
+    GF_VALIDATE_OR_GOTO(this->name, loc, out);
+    GF_VALIDATE_OR_GOTO(this->name, loc->inode, out);
+
+    ret = svc_inode_ctx_get(this, loc->inode, &inode_type);
+    if (!__is_root_gfid(loc->gfid)) {
+        if (loc->parent) {
+            parent = inode_ref(loc->parent);
+            ret = svc_inode_ctx_get(this, loc->parent, &parent_type);
         } else {
-                subvolume = SECOND_CHILD (this);
-                local->subvolume = subvolume;
-                if (parent_type == NORMAL_INODE) {
-                        /* Indication of whether the lookup is happening on the
-                           entry point or not, to the snapview-server.
-                        */
-                        SVC_ENTRY_POINT_SET (this, xdata, op_ret, op_errno,
-                                             new_xdata, priv, ret, out);
-                }
+            parent = inode_parent(loc->inode, loc->pargfid, NULL);
+            if (parent)
+                ret = svc_inode_ctx_get(this, parent, &parent_type);
+        }
+    }
+
+    local = mem_get0(this->local_pool);
+    if (!local) {
+        op_ret = -1;
+        op_errno = ENOMEM;
+        gf_smsg(this->name, GF_LOG_ERROR, op_errno, SVC_MSG_NO_MEMORY, NULL);
+        goto out;
+    }
+
+    frame->local = local;
+    loc_copy(&local->loc, loc);
+
+    if (__is_root_gfid(loc->inode->gfid)) {
+        subvolume = FIRST_CHILD(this);
+        GF_ASSERT(subvolume);
+        local->subvolume = subvolume;
+        wind = _gf_true;
+        goto out;
+    }
+
+    /* nfs sends nameless lookups directly using the gfid. In that case
+       loc->name will be NULL. So check if loc->name is NULL. If so, then
+       try to get the subvolume using inode context. But if the inode has
+       not been looked up yet, then send the lookup call to the first
+       subvolume.
+    */
+
+    if (!loc->name) {
+        if (gf_uuid_is_null(loc->inode->gfid)) {
+            subvolume = FIRST_CHILD(this);
+            local->subvolume = subvolume;
+            wind = _gf_true;
+            goto out;
+        } else {
+            if (inode_type >= 0)
+                subvolume = svc_get_subvolume(this, inode_type);
+            else
+                subvolume = FIRST_CHILD(this);
+            local->subvolume = subvolume;
+            wind = _gf_true;
+            goto out;
+        }
+    }
+
+    if (gf_svc_get_entry_point(this, entry_point, sizeof(entry_point))) {
+        gf_smsg(this->name, GF_LOG_WARNING, op_errno,
+                SVC_MSG_COPY_ENTRY_POINT_FAILED, NULL);
+        goto out;
+    }
+
+    if (strcmp(loc->name, entry_point)) {
+        if (parent_type == VIRTUAL_INODE) {
+            subvolume = SECOND_CHILD(this);
+        } else {
+            /*
+             * Either parent type is normal graph, or the parent
+             * type is uncertain.
+             */
+            subvolume = FIRST_CHILD(this);
+        }
+        local->subvolume = subvolume;
+    } else {
+        subvolume = SECOND_CHILD(this);
+        local->subvolume = subvolume;
+        if (parent_type == NORMAL_INODE) {
+            /* Indication of whether the lookup is happening on the
+               entry point or not, to the snapview-server.
+            */
+            SVC_ENTRY_POINT_SET(this, xdata, op_ret, op_errno, new_xdata, ret,
+                                out);
         }
+    }
 
-        wind = _gf_true;
+    wind = _gf_true;
 
 out:
-        if (wind)
-                STACK_WIND (frame, svc_lookup_cbk,
-                            subvolume, subvolume->fops->lookup, loc, xdata);
-        else
-                SVC_STACK_UNWIND (lookup, frame, op_ret, op_errno, NULL,
-                                  NULL, NULL, NULL);
-        if (new_xdata)
-                dict_unref (new_xdata);
+    if (wind)
+        STACK_WIND(frame, gf_svc_lookup_cbk, subvolume, subvolume->fops->lookup,
+                   loc, xdata);
+    else
+        SVC_STACK_UNWIND(lookup, frame, op_ret, op_errno, NULL, NULL, NULL,
+                         NULL);
+    if (new_xdata)
+        dict_unref(new_xdata);
+
+    if (parent)
+        inode_unref(parent);
+
+    return 0;
+}
 
-        if (parent)
-                inode_unref (parent);
+static int32_t
+gf_svc_statfs(call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xdata)
+{
+    xlator_t *subvolume = NULL;
+    int32_t ret = -1;
+    int inode_type = -1;
+    int32_t op_ret = -1;
+    int32_t op_errno = EINVAL;
+    gf_boolean_t wind = _gf_false;
+    svc_private_t *priv = NULL;
+    const char *path = NULL;
+    int path_len = -1;
+    int snap_len = -1;
+    loc_t root_loc = {
+        0,
+    };
+    loc_t *temp_loc = NULL;
+    char entry_point[NAME_MAX + 1] = {
+        0,
+    };
+
+    GF_VALIDATE_OR_GOTO("svc", this, out);
+    GF_VALIDATE_OR_GOTO(this->name, frame, out);
+    GF_VALIDATE_OR_GOTO(this->name, loc, out);
+    GF_VALIDATE_OR_GOTO(this->name, loc->inode, out);
+
+    priv = this->private;
+    SVC_GET_SUBVOL_FROM_CTX(this, op_ret, op_errno, inode_type, ret, loc->inode,
+                            subvolume, out);
+    path_len = strlen(loc->path);
+    snap_len = strlen(priv->path);
+    temp_loc = loc;
+
+    if (path_len >= snap_len && inode_type == VIRTUAL_INODE) {
+        path = &loc->path[path_len - snap_len];
+        if (gf_svc_get_entry_point(this, entry_point, sizeof(entry_point))) {
+            gf_smsg(this->name, GF_LOG_WARNING, op_errno,
+                    SVC_MSG_COPY_ENTRY_POINT_FAILED, NULL);
+            goto out;
+        }
+
+        if (!strcmp(path, entry_point)) {
+            /*
+             * statfs call for virtual snap directory.
+             * Sent the fops to parent volume by removing
+             * virtual directory from path
+             */
+            subvolume = FIRST_CHILD(this);
+            root_loc.path = gf_strdup("/");
+            gf_uuid_clear(root_loc.gfid);
+            root_loc.gfid[15] = 1;
+            root_loc.inode = inode_ref(loc->inode->table->root);
+            temp_loc = &root_loc;
+        }
+    }
+
+    STACK_WIND_TAIL(frame, subvolume, subvolume->fops->statfs, temp_loc, xdata);
+    if (temp_loc == &root_loc)
+        loc_wipe(temp_loc);
+
+    wind = _gf_true;
+out:
+    if (!wind)
+        SVC_STACK_UNWIND(statfs, frame, op_ret, op_errno, NULL, NULL);
+    return 0;
+}
 
-        return 0;
+static int32_t
+gf_svc_stat_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                int32_t op_ret, int32_t op_errno, struct iatt *buf,
+                dict_t *xdata)
+{
+    /* TODO: FIX ME
+     * Consider a testcase:
+     * #mount -t nfs host1:/vol1 /mnt
+     * #ls /mnt
+     * #ls /mnt/.snaps (As expected this fails)
+     * #gluster volume set vol1 features.uss enable
+     * Now `ls /mnt/.snaps` should work, but fails with No such file or
+     * directory. This is because NFS client (gNFS) caches the list of files
+     * in a directory. This cache is updated if there are any changes in the
+     * directory attributes. So, one way to solve this problem is to change
+     * 'ctime' attribute when USS is enabled as below.
+     *
+     * if (op_ret == 0 && IA_ISDIR(buf->ia_type))
+     *     buf->ia_ctime_nsec++;
+     *
+     * But this is not the ideal solution as applications see the unexpected
+     * ctime change causing failures.
+     */
+
+    SVC_STACK_UNWIND(stat, frame, op_ret, op_errno, buf, xdata);
+    return 0;
 }
 
 /* should all the fops be handled like lookup is supposed to be
    handled? i.e just based on inode type decide where the call should
    be sent and in the call back update the contexts.
 */
-int32_t
-svc_stat (call_frame_t *frame, xlator_t *this, loc_t *loc,
-          dict_t *xdata)
+static int32_t
+gf_svc_stat(call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xdata)
 {
-        int32_t      ret        = -1;
-        int          inode_type = -1;
-        xlator_t    *subvolume  = NULL;
-        int32_t      op_ret     = -1;
-        int32_t      op_errno   = EINVAL;
-        gf_boolean_t wind       = _gf_false;
+    int32_t ret = -1;
+    int inode_type = -1;
+    xlator_t *subvolume = NULL;
+    int32_t op_ret = -1;
+    int32_t op_errno = EINVAL;
+    gf_boolean_t wind = _gf_false;
 
-        GF_VALIDATE_OR_GOTO ("svc", this, out);
-        GF_VALIDATE_OR_GOTO (this->name, frame, out);
-        GF_VALIDATE_OR_GOTO (this->name, loc, out);
-        GF_VALIDATE_OR_GOTO (this->name, loc->inode, out);
+    GF_VALIDATE_OR_GOTO("svc", this, out);
+    GF_VALIDATE_OR_GOTO(this->name, frame, out);
+    GF_VALIDATE_OR_GOTO(this->name, loc, out);
+    GF_VALIDATE_OR_GOTO(this->name, loc->inode, out);
 
-        SVC_GET_SUBVOL_FROM_CTX (this, op_ret, op_errno, inode_type, ret,
-                                 loc->inode, subvolume, out);
+    SVC_GET_SUBVOL_FROM_CTX(this, op_ret, op_errno, inode_type, ret, loc->inode,
+                            subvolume, out);
 
-        STACK_WIND_TAIL (frame,subvolume, subvolume->fops->stat, loc, xdata);
+    STACK_WIND(frame, gf_svc_stat_cbk, subvolume, subvolume->fops->stat, loc,
+               xdata);
 
-        wind = _gf_true;
+    wind = _gf_true;
 
 out:
-        if (!wind)
-                SVC_STACK_UNWIND (stat, frame, op_ret, op_errno,
-                                  NULL, NULL);
-        return 0;
+    if (!wind)
+        SVC_STACK_UNWIND(stat, frame, op_ret, op_errno, NULL, NULL);
+    return 0;
 }
 
-int32_t
-svc_fstat (call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *xdata)
+static int32_t
+gf_svc_fstat(call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *xdata)
 {
-        int32_t      ret        = -1;
-        int          inode_type = -1;
-        xlator_t    *subvolume  = NULL;
-        int32_t      op_ret     = -1;
-        int32_t      op_errno   = EINVAL;
-        gf_boolean_t wind       = _gf_false;
+    int32_t ret = -1;
+    int inode_type = -1;
+    xlator_t *subvolume = NULL;
+    int32_t op_ret = -1;
+    int32_t op_errno = EINVAL;
+    gf_boolean_t wind = _gf_false;
 
-        GF_VALIDATE_OR_GOTO ("svc", this, out);
-        GF_VALIDATE_OR_GOTO (this->name, frame, out);
-        GF_VALIDATE_OR_GOTO (this->name, fd, out);
-        GF_VALIDATE_OR_GOTO (this->name, fd->inode, out);
+    GF_VALIDATE_OR_GOTO("svc", this, out);
+    GF_VALIDATE_OR_GOTO(this->name, frame, out);
+    GF_VALIDATE_OR_GOTO(this->name, fd, out);
+    GF_VALIDATE_OR_GOTO(this->name, fd->inode, out);
 
-        SVC_GET_SUBVOL_FROM_CTX (this, op_ret, op_errno, inode_type, ret,
-                                 fd->inode, subvolume, out);
+    SVC_GET_SUBVOL_FROM_CTX(this, op_ret, op_errno, inode_type, ret, fd->inode,
+                            subvolume, out);
 
-        STACK_WIND_TAIL (frame, subvolume, subvolume->fops->fstat, fd, xdata);
+    STACK_WIND_TAIL(frame, subvolume, subvolume->fops->fstat, fd, xdata);
 
-        wind = _gf_true;
+    wind = _gf_true;
 
 out:
-        if (!wind)
-                SVC_STACK_UNWIND (fstat, frame, op_ret, op_errno, NULL, NULL);
+    if (!wind)
+        SVC_STACK_UNWIND(fstat, frame, op_ret, op_errno, NULL, NULL);
 
-        return ret;
+    return ret;
 }
 
-int32_t
-svc_opendir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                 int32_t op_ret, int32_t op_errno, fd_t *fd, dict_t *xdata)
+static int32_t
+gf_svc_opendir_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                   int32_t op_ret, int32_t op_errno, fd_t *fd, dict_t *xdata)
 {
-        svc_fd_t        *svc_fd          = NULL;
-        svc_local_t     *local           = NULL;
-        svc_private_t   *priv            = NULL;
-        gf_boolean_t     special_dir     = _gf_false;
-        char             path[PATH_MAX]  = {0, };
-
-        GF_VALIDATE_OR_GOTO ("snapview-client", this, out);
-        GF_VALIDATE_OR_GOTO (this->name, this->private, out);
-
-        if (op_ret)
-                goto out;
+    svc_fd_t *svc_fd = NULL;
+    svc_local_t *local = NULL;
+    svc_private_t *priv = NULL;
+    gf_boolean_t special_dir = _gf_false;
+    char path[PATH_MAX] = {
+        0,
+    };
+
+    GF_VALIDATE_OR_GOTO("snapview-client", this, out);
+    GF_VALIDATE_OR_GOTO(this->name, this->private, out);
+
+    if (op_ret)
+        goto out;
+
+    priv = this->private;
+    local = frame->local;
+
+    if (local->subvolume == FIRST_CHILD(this) && priv->special_dir &&
+        strcmp(priv->special_dir, "")) {
+        if (!__is_root_gfid(fd->inode->gfid))
+            snprintf(path, sizeof(path), "%s/.", priv->special_dir);
+        else
+            snprintf(path, sizeof(path), "/.");
 
-        priv = this->private;
-        local = frame->local;
-
-        if (local->subvolume == FIRST_CHILD (this) && priv->special_dir
-            && strcmp (priv->special_dir, "")) {
-                if (!__is_root_gfid (fd->inode->gfid))
-                        snprintf (path, sizeof (path), "%s/.",
-                                  priv->special_dir);
-                else
-                        snprintf (path, sizeof (path), "/.");
-
-                if (!strcmp (local->loc.path, priv->special_dir) ||
-                    !strcmp (local->loc.path, path)) {
-                        gf_log_callingfn (this->name, GF_LOG_DEBUG,
-                                          "got opendir on special "
-                                          "directory %s (%s)", path,
-                                          uuid_utoa (fd->inode->gfid));
-                        special_dir = _gf_true;
-                }
+        if (!strcmp(local->loc.path, priv->special_dir) ||
+            !strcmp(local->loc.path, path)) {
+            gf_msg_debug(this->name, 0,
+                         "got opendir on special directory"
+                         " %s (gfid: %s)",
+                         path, uuid_utoa(fd->inode->gfid));
+            special_dir = _gf_true;
         }
+    }
 
-        if (special_dir) {
-                svc_fd = svc_fd_ctx_get_or_new (this, fd);
-                if (!svc_fd) {
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "fd context not found for %s",
-                                uuid_utoa (fd->inode->gfid));
-                        goto out;
-                }
-
-                svc_fd->last_offset = -1;
-                svc_fd->special_dir = special_dir;
+    if (special_dir) {
+        svc_fd = svc_fd_ctx_get_or_new(this, fd);
+        if (!svc_fd) {
+            gf_smsg(this->name, GF_LOG_ERROR, 0, SVC_MSG_GET_FD_CONTEXT_FAILED,
+                    "gfid=%s", uuid_utoa(fd->inode->gfid), NULL);
+            goto out;
         }
 
+        svc_fd->last_offset = -1;
+        svc_fd->special_dir = special_dir;
+    }
+
 out:
-        STACK_UNWIND_STRICT (opendir, frame, op_ret, op_errno, fd, xdata);
+    STACK_UNWIND_STRICT(opendir, frame, op_ret, op_errno, fd, xdata);
 
-        return 0;
+    return 0;
 }
 
-
 /* If the inode represents a directory which is actually
    present in a snapshot, then opendir on that directory
    should be sent to the snap-view-server which opens
@@ -628,99 +718,99 @@ out:
    will not be any corresponding fd opened on svs and
    svc has to do things that open-behind is doing.
 */
-int32_t
-svc_opendir (call_frame_t *frame, xlator_t *this, loc_t *loc, fd_t *fd,
-             dict_t *xdata)
+static int32_t
+gf_svc_opendir(call_frame_t *frame, xlator_t *this, loc_t *loc, fd_t *fd,
+               dict_t *xdata)
 {
-        int32_t        ret        = -1;
-        int            inode_type = -1;
-        xlator_t      *subvolume  = NULL;
-        int            op_ret     = -1;
-        int            op_errno   = EINVAL;
-        gf_boolean_t   wind       = _gf_false;
-        svc_local_t   *local      = NULL;
-
-        GF_VALIDATE_OR_GOTO ("svc", this, out);
-        GF_VALIDATE_OR_GOTO (this->name, frame, out);
-        GF_VALIDATE_OR_GOTO (this->name, loc, out);
-        GF_VALIDATE_OR_GOTO (this->name, loc->inode, out);
-        GF_VALIDATE_OR_GOTO (this->name, fd, out);
-
-        local = mem_get0 (this->local_pool);
-        if (!local) {
-                gf_log (this->name, GF_LOG_ERROR, "failed to allocate memory "
-                        "for local (path: %s, gfid: %s)", loc->path,
-                        uuid_utoa (fd->inode->gfid));
-                op_errno = ENOMEM;
-                goto out;
-        }
-
-        SVC_GET_SUBVOL_FROM_CTX (this, op_ret, op_errno, inode_type, ret,
-                                 loc->inode, subvolume, out);
-
-        loc_copy (&local->loc, loc);
-        local->subvolume = subvolume;
-        frame->local = local;
-
-        STACK_WIND (frame, svc_opendir_cbk, subvolume, subvolume->fops->opendir,
-                    loc, fd, xdata);
-
-        wind = _gf_true;
+    int32_t ret = -1;
+    int inode_type = -1;
+    xlator_t *subvolume = NULL;
+    int op_ret = -1;
+    int op_errno = EINVAL;
+    gf_boolean_t wind = _gf_false;
+    svc_local_t *local = NULL;
+
+    GF_VALIDATE_OR_GOTO("svc", this, out);
+    GF_VALIDATE_OR_GOTO(this->name, frame, out);
+    GF_VALIDATE_OR_GOTO(this->name, loc, out);
+    GF_VALIDATE_OR_GOTO(this->name, loc->inode, out);
+    GF_VALIDATE_OR_GOTO(this->name, fd, out);
+
+    local = mem_get0(this->local_pool);
+    if (!local) {
+        op_errno = ENOMEM;
+        gf_smsg(this->name, GF_LOG_ERROR, op_errno, SVC_MSG_NO_MEMORY,
+                "path=%s", loc->path, "gfid=%s", uuid_utoa(fd->inode->gfid),
+                NULL);
+        goto out;
+    }
+    loc_copy(&local->loc, loc);
+    frame->local = local;
+
+    SVC_GET_SUBVOL_FROM_CTX(this, op_ret, op_errno, inode_type, ret, loc->inode,
+                            subvolume, out);
+    local->subvolume = subvolume;
+
+    STACK_WIND(frame, gf_svc_opendir_cbk, subvolume, subvolume->fops->opendir,
+               loc, fd, xdata);
+
+    wind = _gf_true;
 
 out:
-        if (!wind)
-                SVC_STACK_UNWIND (opendir, frame, op_ret, op_errno, NULL, NULL);
+    if (!wind)
+        SVC_STACK_UNWIND(opendir, frame, op_ret, op_errno, NULL, NULL);
 
-        return 0;
+    return 0;
 }
 
-int32_t
-svc_setattr (call_frame_t *frame, xlator_t *this, loc_t *loc,
-             struct iatt *stbuf, int32_t valid, dict_t *xdata)
+static int32_t
+gf_svc_setattr(call_frame_t *frame, xlator_t *this, loc_t *loc,
+               struct iatt *stbuf, int32_t valid, dict_t *xdata)
 {
-        int32_t      ret        = -1;
-        int          inode_type = -1;
-        int          op_ret     = -1;
-        int          op_errno   = EINVAL;
-        gf_boolean_t wind       = _gf_false;
-
-        GF_VALIDATE_OR_GOTO ("svc", this, out);
-        GF_VALIDATE_OR_GOTO (this->name, frame, out);
-        GF_VALIDATE_OR_GOTO (this->name, loc, out);
-        GF_VALIDATE_OR_GOTO (this->name, loc->inode, out);
-
-        ret = svc_inode_ctx_get (this, loc->inode, &inode_type);
-        if (ret < 0) {
-                op_ret = -1;
-                op_errno = EINVAL;
-                gf_log (this->name, GF_LOG_ERROR, "failed to get the inode "
-                        "context for %s (gfid: %s)", loc->path,
-                        uuid_utoa (loc->inode->gfid));
-                goto out;
-        }
-
-        if (inode_type == NORMAL_INODE) {
-                STACK_WIND_TAIL (frame, FIRST_CHILD (this),
-                                 FIRST_CHILD (this)->fops->setattr, loc, stbuf,
-                                 valid, xdata);
-        } else {
-                op_ret = -1;
-                op_errno = EROFS;
-                goto out;
-        }
-
-        wind = _gf_true;
+    int32_t ret = -1;
+    int inode_type = -1;
+    int op_ret = -1;
+    int op_errno = EINVAL;
+    gf_boolean_t wind = _gf_false;
+
+    GF_VALIDATE_OR_GOTO("svc", this, out);
+    GF_VALIDATE_OR_GOTO(this->name, frame, out);
+    GF_VALIDATE_OR_GOTO(this->name, loc, out);
+    GF_VALIDATE_OR_GOTO(this->name, loc->inode, out);
+
+    ret = svc_inode_ctx_get(this, loc->inode, &inode_type);
+    if (ret < 0) {
+        op_ret = -1;
+        op_errno = EINVAL;
+        gf_smsg(this->name, GF_LOG_ERROR, op_errno,
+                SVC_MSG_GET_INODE_CONTEXT_FAILED, "path=%s", loc->path,
+                "gfid= %s", uuid_utoa(loc->inode->gfid), NULL);
+        goto out;
+    }
+
+    if (inode_type == NORMAL_INODE) {
+        STACK_WIND_TAIL(frame, FIRST_CHILD(this),
+                        FIRST_CHILD(this)->fops->setattr, loc, stbuf, valid,
+                        xdata);
+    } else {
+        op_ret = -1;
+        op_errno = EROFS;
+        goto out;
+    }
+
+    wind = _gf_true;
 
 out:
-        if (!wind)
-                SVC_STACK_UNWIND (setattr, frame, op_ret, op_errno,
-                                  NULL, NULL, NULL);
-        return 0;
+    if (!wind)
+        SVC_STACK_UNWIND(setattr, frame, op_ret, op_errno, NULL, NULL, NULL);
+    return 0;
 }
 
-int32_t
-svc_fsetattr (call_frame_t *frame, xlator_t *this, fd_t *fd, struct iatt *stbuf,
-             int32_t valid, dict_t *xdata)
+/* XXX: This function is currently not used. Remove "#if 0" when required */
+#if 0
+static int32_t
+gf_svc_fsetattr (call_frame_t *frame, xlator_t *this, fd_t *fd,
+                 struct iatt *stbuf, int32_t valid, dict_t *xdata)
 {
         int32_t      ret        = -1;
         int          inode_type = -1;
@@ -735,10 +825,12 @@ svc_fsetattr (call_frame_t *frame, xlator_t *this, fd_t *fd, struct iatt *stbuf,
 
         ret = svc_inode_ctx_get (this, fd->inode, &inode_type);
         if (ret < 0) {
-                gf_log (this->name, GF_LOG_ERROR, "failed to get the inode "
-                        "context for %s", uuid_utoa (fd->inode->gfid));
                 op_ret = -1;
                 op_errno = EINVAL;
+                gf_msg (this->name, GF_LOG_ERROR, op_errno,
+                        SVC_MSG_GET_INODE_CONTEXT_FAILED, "failed to "
+                        "get the inode context for %s",
+                        uuid_utoa (fd->inode->gfid));
                 goto out;
         }
 
@@ -760,90 +852,98 @@ out:
                                   NULL, NULL, NULL);
         return 0;
 }
+#endif /* gf_svc_fsetattr() is not used */
 
-int32_t
-svc_getxattr (call_frame_t *frame, xlator_t *this, loc_t *loc, const char *name,
-              dict_t *xdata)
+static int32_t
+gf_svc_getxattr(call_frame_t *frame, xlator_t *this, loc_t *loc,
+                const char *name, dict_t *xdata)
 {
-        int32_t          ret                    = -1;
-        int              inode_type             = -1;
-        xlator_t        *subvolume              = NULL;
-        int              op_ret                 = -1;
-        int              op_errno               = EINVAL;
-        gf_boolean_t     wind                   = _gf_false;
-        svc_private_t   *priv                   = NULL;
-        char             attrname[PATH_MAX]     = "";
-        char             attrval[64]            = "";
-        dict_t          *dict                   = NULL;
+    int32_t ret = -1;
+    int inode_type = -1;
+    xlator_t *subvolume = NULL;
+    int op_ret = -1;
+    int op_errno = EINVAL;
+    gf_boolean_t wind = _gf_false;
+    svc_private_t *priv = NULL;
+    char attrname[PATH_MAX] = "";
+    char attrval[64] = "";
+    dict_t *dict = NULL;
+    char entry_point[NAME_MAX + 1] = {
+        0,
+    };
+
+    GF_VALIDATE_OR_GOTO("svc", this, out);
+    GF_VALIDATE_OR_GOTO(this->name, frame, out);
+    GF_VALIDATE_OR_GOTO(this->name, loc, out);
+    GF_VALIDATE_OR_GOTO(this->name, loc->inode, out);
+    priv = this->private;
+    GF_VALIDATE_OR_GOTO(this->name, priv, out);
+
+    /*
+     * Samba sends this special key for case insensitive
+     * filename check. This request comes with a parent
+     * path and with a special key GF_XATTR_GET_REAL_FILENAME_KEY.
+     * e.g. "glusterfs.get_real_filename:.snaps".
+     * If the name variable matches this key then we have
+     * to send back .snaps as the real filename.
+     */
+    if (!name)
+        goto stack_wind;
+
+    sscanf(name, "%[^:]:%[^@]", attrname, attrval);
+    strcat(attrname, ":");
+
+    if (!strcmp(attrname, GF_XATTR_GET_REAL_FILENAME_KEY)) {
+        if (gf_svc_get_entry_point(this, entry_point, sizeof(entry_point))) {
+            gf_smsg(this->name, GF_LOG_WARNING, op_errno,
+                    SVC_MSG_COPY_ENTRY_POINT_FAILED, NULL);
+            goto out;
+        }
+
+        if (!strcasecmp(attrval, entry_point)) {
+            dict = dict_new();
+            if (NULL == dict) {
+                op_errno = ENOMEM;
+                goto out;
+            }
 
-        GF_VALIDATE_OR_GOTO ("svc", this, out);
-        GF_VALIDATE_OR_GOTO (this->name, frame, out);
-        GF_VALIDATE_OR_GOTO (this->name, loc, out);
-        GF_VALIDATE_OR_GOTO (this->name, loc->inode, out);
-        priv = this->private;
-        GF_VALIDATE_OR_GOTO (this->name, priv, out);
-
-        /*
-         * Samba sends this special key for case insensitive
-         * filename check. This request comes with a parent
-         * path and with a special key GF_XATTR_GET_REAL_FILENAME_KEY.
-         * e.g. "glusterfs.get_real_filename:.snaps".
-         * If the name variable matches this key then we have
-         * to send back .snaps as the real filename.
-         */
-        if (!name)
-                goto stack_wind;
-
-        sscanf (name, "%[^:]:%[^@]", attrname, attrval);
-        strcat (attrname, ":");
-
-        if (!strcmp (attrname, GF_XATTR_GET_REAL_FILENAME_KEY)) {
-                if (!strcasecmp (attrval, priv->path)) {
-                        dict = dict_new ();
-                        if (NULL == dict) {
-                                op_errno = ENOMEM;
-                                goto out;
-                        }
-
-                        ret = dict_set_dynstr_with_alloc (dict,
-                                        (char *)name,
-                                        priv->path);
-
-                        if (ret) {
-                                op_errno = ENOMEM;
-                                dict_unref (dict);
-                                goto out;
-                        }
-
-                        op_errno = 0;
-                        op_ret = strlen (priv->path) + 1;
-                        /* We should return from here */
-                        goto out;
-                }
+            ret = dict_set_dynstr_with_alloc(dict, (char *)name, entry_point);
+
+            if (ret) {
+                op_errno = ENOMEM;
+                goto out;
+            }
+
+            op_errno = 0;
+            op_ret = strlen(entry_point) + 1;
+            /* We should return from here */
+            goto out;
         }
+    }
 stack_wind:
-        SVC_GET_SUBVOL_FROM_CTX (this, op_ret, op_errno, inode_type, ret,
-                                 loc->inode, subvolume, out);
+    SVC_GET_SUBVOL_FROM_CTX(this, op_ret, op_errno, inode_type, ret, loc->inode,
+                            subvolume, out);
 
-        STACK_WIND_TAIL (frame, subvolume, subvolume->fops->getxattr, loc, name,
-                         xdata);
+    STACK_WIND_TAIL(frame, subvolume, subvolume->fops->getxattr, loc, name,
+                    xdata);
 
-        wind = _gf_true;
+    wind = _gf_true;
 
 out:
-        if (!wind)
-                SVC_STACK_UNWIND (getxattr, frame, op_ret, op_errno,
-                                  dict, NULL);
+    if (!wind)
+        SVC_STACK_UNWIND(getxattr, frame, op_ret, op_errno, dict, NULL);
 
-        if (dict)
-                dict_unref (dict);
+    if (dict)
+        dict_unref(dict);
 
-        return 0;
+    return 0;
 }
 
-int32_t
-svc_fgetxattr (call_frame_t *frame, xlator_t *this, fd_t *fd, const char *name,
-              dict_t *xdata)
+/* XXX: This function is currently not used. Mark it '#if 0' when required */
+#if 0
+static int32_t
+gf_svc_fgetxattr (call_frame_t *frame, xlator_t *this, fd_t *fd,
+                  const char *name, dict_t *xdata)
 {
         int32_t       ret        = -1;
         int           inode_type = -1;
@@ -871,664 +971,737 @@ out:
                                   NULL, NULL);
         return 0;
 }
+#endif /* gf_svc_fgetxattr() is not used */
 
-int32_t
-svc_setxattr (call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *dict,
-              int32_t flags, dict_t *xdata)
+static int32_t
+gf_svc_setxattr(call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *dict,
+                int32_t flags, dict_t *xdata)
 {
-        int32_t      ret         = -1;
-        int          inode_type  = -1;
-        int          op_ret      = -1;
-        int          op_errno    = EINVAL;
-        gf_boolean_t wind        = _gf_false;
-
-        GF_VALIDATE_OR_GOTO ("svc", this, out);
-        GF_VALIDATE_OR_GOTO (this->name, frame, out);
-        GF_VALIDATE_OR_GOTO (this->name, loc, out);
-        GF_VALIDATE_OR_GOTO (this->name, loc->inode, out);
-
-        ret = svc_inode_ctx_get (this, loc->inode, &inode_type);
-        if (ret < 0) {
-                gf_log (this->name, GF_LOG_ERROR, "failed to get inode context "
-                        "for %s (gfid: %s)", loc->name,
-                        uuid_utoa (loc->inode->gfid));
-                op_ret = -1;
-                op_errno = EINVAL;
-                goto out;
-        }
-
-        if (inode_type == NORMAL_INODE) {
-                STACK_WIND_TAIL (frame, FIRST_CHILD (this),
-                                 FIRST_CHILD (this)->fops->setxattr, loc, dict,
-                                 flags, xdata);
-        } else {
-                op_ret = -1;
-                op_errno = EROFS;
-                goto out;
-        }
-
-        wind = _gf_true;
+    int32_t ret = -1;
+    int inode_type = -1;
+    int op_ret = -1;
+    int op_errno = EINVAL;
+    gf_boolean_t wind = _gf_false;
+
+    GF_VALIDATE_OR_GOTO("svc", this, out);
+    GF_VALIDATE_OR_GOTO(this->name, frame, out);
+    GF_VALIDATE_OR_GOTO(this->name, loc, out);
+    GF_VALIDATE_OR_GOTO(this->name, loc->inode, out);
+
+    ret = svc_inode_ctx_get(this, loc->inode, &inode_type);
+    if (ret < 0) {
+        op_ret = -1;
+        op_errno = EINVAL;
+        gf_smsg(this->name, GF_LOG_ERROR, op_errno,
+                SVC_MSG_GET_INODE_CONTEXT_FAILED, "name=%s", loc->name,
+                "gfid=%s", uuid_utoa(loc->inode->gfid), NULL);
+        goto out;
+    }
+
+    if (inode_type == NORMAL_INODE) {
+        STACK_WIND_TAIL(frame, FIRST_CHILD(this),
+                        FIRST_CHILD(this)->fops->setxattr, loc, dict, flags,
+                        xdata);
+    } else {
+        op_ret = -1;
+        op_errno = EROFS;
+        goto out;
+    }
+
+    wind = _gf_true;
 
 out:
-        if (!wind)
-                SVC_STACK_UNWIND (setxattr, frame, op_ret, op_errno,
-                                  NULL);
+    if (!wind)
+        SVC_STACK_UNWIND(setxattr, frame, op_ret, op_errno, NULL);
 
-        return 0;
+    return 0;
 }
 
-int32_t
-svc_fsetxattr (call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *dict,
-              int32_t flags, dict_t *xdata)
+static int32_t
+gf_svc_fsetxattr(call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *dict,
+                 int32_t flags, dict_t *xdata)
 {
-        int32_t      ret        = -1;
-        int          inode_type = -1;
-        int          op_ret     = -1;
-        int          op_errno   = EINVAL;
-        gf_boolean_t wind       = _gf_false;
-
-        GF_VALIDATE_OR_GOTO ("svc", this, out);
-        GF_VALIDATE_OR_GOTO (this->name, frame, out);
-        GF_VALIDATE_OR_GOTO (this->name, fd, out);
-        GF_VALIDATE_OR_GOTO (this->name, fd->inode, out);
-
-        ret = svc_inode_ctx_get (this, fd->inode, &inode_type);
-        if (ret < 0) {
-                gf_log (this->name, GF_LOG_ERROR, "failed to get inode context "
-                        "for %s", uuid_utoa (fd->inode->gfid));
-                op_ret = -1;
-                op_errno = EINVAL;
-                goto out;
-        }
-
-        if (inode_type == NORMAL_INODE) {
-                STACK_WIND_TAIL (frame, FIRST_CHILD (this),
-                                 FIRST_CHILD (this)->fops->fsetxattr, fd, dict,
-                                 flags, xdata);
-        } else {
-                op_ret = -1;
-                op_errno = EROFS;
-                goto out;
-        }
-
-        wind = _gf_true;
+    int32_t ret = -1;
+    int inode_type = -1;
+    int op_ret = -1;
+    int op_errno = EINVAL;
+    gf_boolean_t wind = _gf_false;
+
+    GF_VALIDATE_OR_GOTO("svc", this, out);
+    GF_VALIDATE_OR_GOTO(this->name, frame, out);
+    GF_VALIDATE_OR_GOTO(this->name, fd, out);
+    GF_VALIDATE_OR_GOTO(this->name, fd->inode, out);
+
+    ret = svc_inode_ctx_get(this, fd->inode, &inode_type);
+    if (ret < 0) {
+        op_ret = -1;
+        op_errno = EINVAL;
+        gf_smsg(this->name, GF_LOG_ERROR, op_errno,
+                SVC_MSG_GET_INODE_CONTEXT_FAILED, "gfid=%s",
+                uuid_utoa(fd->inode->gfid), NULL);
+        goto out;
+    }
+
+    if (inode_type == NORMAL_INODE) {
+        STACK_WIND_TAIL(frame, FIRST_CHILD(this),
+                        FIRST_CHILD(this)->fops->fsetxattr, fd, dict, flags,
+                        xdata);
+    } else {
+        op_ret = -1;
+        op_errno = EROFS;
+        goto out;
+    }
+
+    wind = _gf_true;
 
 out:
-        if (!wind)
-                STACK_UNWIND_STRICT (fsetxattr, frame, op_ret, op_errno,
-                                     NULL);
+    if (!wind)
+        STACK_UNWIND_STRICT(fsetxattr, frame, op_ret, op_errno, NULL);
 
-        return 0;
+    return 0;
 }
 
-int32_t
-svc_rmdir (call_frame_t *frame, xlator_t *this, loc_t *loc, int flags,
-           dict_t *xdata)
+static int32_t
+gf_svc_rmdir(call_frame_t *frame, xlator_t *this, loc_t *loc, int flags,
+             dict_t *xdata)
 {
-        int          inode_type = -1;
-        int          ret        = -1;
-        int          op_ret     = -1;
-        int          op_errno   = EINVAL;
-        gf_boolean_t wind       = _gf_false;
-
-        GF_VALIDATE_OR_GOTO ("svc", this, out);
-        GF_VALIDATE_OR_GOTO (this->name, frame, out);
-        GF_VALIDATE_OR_GOTO (this->name, loc, out);
-        GF_VALIDATE_OR_GOTO (this->name, loc->inode, out);
-
-        ret = svc_inode_ctx_get (this, loc->inode, &inode_type);
-        if (ret < 0) {
-                gf_log (this->name, GF_LOG_ERROR, "failed to get the inode "
-                        "context for %s (gfid: %s)", loc->name,
-                        uuid_utoa (loc->inode->gfid));
-                op_ret = -1;
-                op_errno = EINVAL;
-                goto out;
-        }
-
-        if (inode_type == NORMAL_INODE) {
-                STACK_WIND_TAIL (frame, FIRST_CHILD (this),
-                                 FIRST_CHILD (this)->fops->rmdir, loc, flags,
-                                 xdata);
-        } else {
-                op_ret = -1;
-                op_errno = EROFS;
-                goto out;
-        }
-
-        wind = _gf_true;
+    int inode_type = -1;
+    int ret = -1;
+    int op_ret = -1;
+    int op_errno = EINVAL;
+    gf_boolean_t wind = _gf_false;
+
+    GF_VALIDATE_OR_GOTO("svc", this, out);
+    GF_VALIDATE_OR_GOTO(this->name, frame, out);
+    GF_VALIDATE_OR_GOTO(this->name, loc, out);
+    GF_VALIDATE_OR_GOTO(this->name, loc->inode, out);
+
+    ret = svc_inode_ctx_get(this, loc->inode, &inode_type);
+    if (ret < 0) {
+        op_ret = -1;
+        op_errno = EINVAL;
+        gf_smsg(this->name, GF_LOG_ERROR, op_errno,
+                SVC_MSG_GET_INODE_CONTEXT_FAILED, "name=%s", loc->name,
+                "gfid=%s", uuid_utoa(loc->inode->gfid), NULL);
+        goto out;
+    }
+
+    if (inode_type == NORMAL_INODE) {
+        STACK_WIND_TAIL(frame, FIRST_CHILD(this),
+                        FIRST_CHILD(this)->fops->rmdir, loc, flags, xdata);
+    } else {
+        op_ret = -1;
+        op_errno = EROFS;
+        goto out;
+    }
+
+    wind = _gf_true;
 
 out:
-        if (!wind)
-                SVC_STACK_UNWIND (rmdir, frame, op_ret, op_errno,
-                                  NULL, NULL, NULL);
-        return 0;
+    if (!wind)
+        SVC_STACK_UNWIND(rmdir, frame, op_ret, op_errno, NULL, NULL, NULL);
+    return 0;
 }
 
-int32_t
-svc_mkdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-               int32_t op_ret, int32_t op_errno, inode_t *inode,
-               struct iatt *buf, struct iatt *preparent,
-               struct iatt *postparent, dict_t *xdata)
+static int32_t
+gf_svc_mkdir_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                 int32_t op_ret, int32_t op_errno, inode_t *inode,
+                 struct iatt *buf, struct iatt *preparent,
+                 struct iatt *postparent, dict_t *xdata)
 {
-        int inode_type = -1;
-        int ret        = -1;
-
-        if (op_ret < 0)
-                goto out;
+    int inode_type = -1;
+    int ret = -1;
 
-        inode_type = NORMAL_INODE;
-        ret = svc_inode_ctx_set (this, inode, inode_type);
-        if (ret)
-                gf_log (this->name, GF_LOG_ERROR, "failed to set inode "
-                        "context");
+    if (op_ret < 0)
+        goto out;
 
+    inode_type = NORMAL_INODE;
+    ret = svc_inode_ctx_set(this, inode, inode_type);
+    if (ret)
+        gf_smsg(this->name, GF_LOG_ERROR, 0, SVC_MSG_SET_INODE_CONTEXT_FAILED,
+                NULL);
 
 out:
-        SVC_STACK_UNWIND (mkdir, frame, op_ret, op_errno, inode,
-                          buf, preparent, postparent, xdata);
-        return 0;
+    SVC_STACK_UNWIND(mkdir, frame, op_ret, op_errno, inode, buf, preparent,
+                     postparent, xdata);
+    return 0;
 }
 
-int32_t
-svc_mkdir (call_frame_t *frame, xlator_t *this, loc_t *loc, mode_t mode,
-           mode_t umask, dict_t *xdata)
+static int32_t
+gf_svc_mkdir(call_frame_t *frame, xlator_t *this, loc_t *loc, mode_t mode,
+             mode_t umask, dict_t *xdata)
 {
-        int            parent_type = -1;
-        int            ret         = -1;
-        int            op_ret      = -1;
-        int            op_errno    = EINVAL;
-        svc_private_t *priv        = NULL;
-        gf_boolean_t   wind        = _gf_false;
-
-        GF_VALIDATE_OR_GOTO ("svc", this, out);
-        GF_VALIDATE_OR_GOTO (this->name, this->private, out);
-        GF_VALIDATE_OR_GOTO (this->name, frame, out);
-        GF_VALIDATE_OR_GOTO (this->name, loc, out);
-        GF_VALIDATE_OR_GOTO (this->name, loc->inode, out);
-
-        priv = this->private;
-
-        ret = svc_inode_ctx_get (this, loc->parent, &parent_type);
-        if (ret < 0) {
-                gf_log (this->name, GF_LOG_ERROR, "failed to get the inode "
-                        "context for %s", uuid_utoa (loc->parent->gfid));
-                op_ret = -1;
-                op_errno = EINVAL;
-                goto out;
-        }
-
-        if (strcmp (loc->name, priv->path) && parent_type == NORMAL_INODE) {
-                STACK_WIND (frame, svc_mkdir_cbk, FIRST_CHILD (this),
-                            FIRST_CHILD (this)->fops->mkdir, loc, mode,
-                            umask, xdata);
-        } else {
-                op_ret = -1;
-                op_errno = EROFS;
-                goto out;
-        }
-
-        wind = _gf_true;
+    int parent_type = -1;
+    int ret = -1;
+    int op_ret = -1;
+    int op_errno = EINVAL;
+    gf_boolean_t wind = _gf_false;
+    char entry_point[NAME_MAX + 1] = {
+        0,
+    };
+
+    GF_VALIDATE_OR_GOTO("svc", this, out);
+    GF_VALIDATE_OR_GOTO(this->name, frame, out);
+    GF_VALIDATE_OR_GOTO(this->name, loc, out);
+    GF_VALIDATE_OR_GOTO(this->name, loc->inode, out);
+
+    ret = svc_inode_ctx_get(this, loc->parent, &parent_type);
+    if (ret < 0) {
+        op_ret = -1;
+        op_errno = EINVAL;
+        gf_smsg(this->name, GF_LOG_ERROR, op_errno,
+                SVC_MSG_GET_INODE_CONTEXT_FAILED, "gfid=%s",
+                uuid_utoa(loc->parent->gfid), NULL);
+        goto out;
+    }
+
+    if (gf_svc_get_entry_point(this, entry_point, sizeof(entry_point))) {
+        gf_smsg(this->name, GF_LOG_WARNING, op_errno,
+                SVC_MSG_COPY_ENTRY_POINT_FAILED, NULL);
+        goto out;
+    }
+
+    if (strcmp(loc->name, entry_point) && parent_type == NORMAL_INODE) {
+        STACK_WIND(frame, gf_svc_mkdir_cbk, FIRST_CHILD(this),
+                   FIRST_CHILD(this)->fops->mkdir, loc, mode, umask, xdata);
+    } else {
+        op_ret = -1;
+        op_errno = EROFS;
+        goto out;
+    }
+
+    wind = _gf_true;
 
 out:
-        if (!wind)
-                SVC_STACK_UNWIND (mkdir, frame, op_ret, op_errno, NULL, NULL,
-                                  NULL, NULL, NULL);
-        return 0;
+    if (!wind)
+        SVC_STACK_UNWIND(mkdir, frame, op_ret, op_errno, NULL, NULL, NULL, NULL,
+                         NULL);
+    return 0;
 }
 
-int32_t
-svc_mknod_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-               int32_t op_ret, int32_t op_errno, inode_t *inode,
-               struct iatt *buf, struct iatt *preparent,
-               struct iatt *postparent, dict_t *xdata)
+static int32_t
+gf_svc_mknod_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                 int32_t op_ret, int32_t op_errno, inode_t *inode,
+                 struct iatt *buf, struct iatt *preparent,
+                 struct iatt *postparent, dict_t *xdata)
 {
-        int inode_type = -1;
-        int ret        = -1;
+    int inode_type = -1;
+    int ret = -1;
 
-        if (op_ret < 0)
-                goto out;
+    if (op_ret < 0)
+        goto out;
 
-        inode_type = NORMAL_INODE;
-        ret = svc_inode_ctx_set (this, inode, inode_type);
-        if (ret)
-                gf_log (this->name, GF_LOG_ERROR, "failed to set inode "
-                        "context");
+    inode_type = NORMAL_INODE;
+    ret = svc_inode_ctx_set(this, inode, inode_type);
+    if (ret)
+        gf_smsg(this->name, GF_LOG_ERROR, 0, SVC_MSG_SET_INODE_CONTEXT_FAILED,
+                NULL);
 
 out:
-        SVC_STACK_UNWIND (mknod, frame, op_ret, op_errno, inode,
-                          buf, preparent, postparent, xdata);
-        return 0;
+    SVC_STACK_UNWIND(mknod, frame, op_ret, op_errno, inode, buf, preparent,
+                     postparent, xdata);
+    return 0;
 }
 
-int32_t
-svc_mknod (call_frame_t *frame, xlator_t *this, loc_t *loc, mode_t mode,
-           dev_t rdev, mode_t umask, dict_t *xdata)
+static int32_t
+gf_svc_mknod(call_frame_t *frame, xlator_t *this, loc_t *loc, mode_t mode,
+             dev_t rdev, mode_t umask, dict_t *xdata)
 {
-        int            parent_type = -1;
-        int            ret         = -1;
-        int            op_ret      = -1;
-        int            op_errno    = EINVAL;
-        svc_private_t *priv        = NULL;
-        gf_boolean_t   wind        = _gf_false;
-
-        GF_VALIDATE_OR_GOTO ("svc", this, out);
-        GF_VALIDATE_OR_GOTO (this->name, this->private, out);
-        GF_VALIDATE_OR_GOTO (this->name, frame, out);
-        GF_VALIDATE_OR_GOTO (this->name, loc, out);
-        GF_VALIDATE_OR_GOTO (this->name, loc->inode, out);
-
-        priv = this->private;
-
-        ret = svc_inode_ctx_get (this, loc->parent, &parent_type);
-        if (ret < 0) {
-                gf_log (this->name, GF_LOG_ERROR, "failed to get the inode "
-                        "context for %s", uuid_utoa (loc->parent->gfid));
-                op_ret = -1;
-                op_errno = EINVAL;
-                goto out;
-        }
-
-        if (strcmp (loc->name, priv->path) && parent_type == NORMAL_INODE) {
-                STACK_WIND (frame, svc_mknod_cbk, FIRST_CHILD (this),
-                            FIRST_CHILD (this)->fops->mknod, loc, mode,
-                            rdev, umask, xdata);
-        } else {
-                op_ret = -1;
-                op_errno = EROFS;
-                goto out;
-        }
-
-        wind = _gf_true;
+    int parent_type = -1;
+    int ret = -1;
+    int op_ret = -1;
+    int op_errno = EINVAL;
+    gf_boolean_t wind = _gf_false;
+    char entry_point[NAME_MAX + 1] = {
+        0,
+    };
+
+    GF_VALIDATE_OR_GOTO("svc", this, out);
+    GF_VALIDATE_OR_GOTO(this->name, frame, out);
+    GF_VALIDATE_OR_GOTO(this->name, loc, out);
+    GF_VALIDATE_OR_GOTO(this->name, loc->inode, out);
+
+    ret = svc_inode_ctx_get(this, loc->parent, &parent_type);
+    if (ret < 0) {
+        op_ret = -1;
+        op_errno = EINVAL;
+        gf_smsg(this->name, GF_LOG_ERROR, op_errno,
+                SVC_MSG_GET_INODE_CONTEXT_FAILED, "gfid=%s",
+                uuid_utoa(loc->parent->gfid), NULL);
+        goto out;
+    }
+
+    if (gf_svc_get_entry_point(this, entry_point, sizeof(entry_point))) {
+        gf_smsg(this->name, GF_LOG_WARNING, op_errno,
+                SVC_MSG_COPY_ENTRY_POINT_FAILED, NULL);
+        goto out;
+    }
+
+    if (strcmp(loc->name, entry_point) && parent_type == NORMAL_INODE) {
+        STACK_WIND(frame, gf_svc_mknod_cbk, FIRST_CHILD(this),
+                   FIRST_CHILD(this)->fops->mknod, loc, mode, rdev, umask,
+                   xdata);
+    } else {
+        op_ret = -1;
+        op_errno = EROFS;
+        goto out;
+    }
+
+    wind = _gf_true;
 
 out:
-        if (!wind)
-                SVC_STACK_UNWIND (mknod, frame, op_ret, op_errno, NULL, NULL,
-                                  NULL, NULL, NULL);
-        return 0;
+    if (!wind)
+        SVC_STACK_UNWIND(mknod, frame, op_ret, op_errno, NULL, NULL, NULL, NULL,
+                         NULL);
+    return 0;
 }
 
 /* If the flags of the open call contain O_WRONLY or O_RDWR and the inode is
    a virtual inode, then unwind the call back with EROFS. Otherwise simply
    STACK_WIND the call to the first child of svc xlator.
 */
-int32_t
-svc_open (call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t flags,
-          fd_t *fd, dict_t *xdata)
+static int32_t
+gf_svc_open(call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t flags,
+            fd_t *fd, dict_t *xdata)
 {
-        xlator_t    *subvolume  = NULL;
-        int          inode_type = -1;
-        int          op_ret     = -1;
-        int          op_errno   = EINVAL;
-        int          ret        = -1;
-        gf_boolean_t wind       = _gf_false;
-
-        GF_VALIDATE_OR_GOTO ("svc", this, out);
-        GF_VALIDATE_OR_GOTO (this->name, frame, out);
-        GF_VALIDATE_OR_GOTO (this->name, loc, out);
-        GF_VALIDATE_OR_GOTO (this->name, loc->inode, out);
-        GF_VALIDATE_OR_GOTO (this->name, fd, out);
-
-        /* Another way is to STACK_WIND to normal subvolume, if inode
-           type is not there in the context. If the file actually resides
-           in snapshots, then ENOENT would be returned. Needs more analysis.
-        */
-        SVC_GET_SUBVOL_FROM_CTX (this, op_ret, op_errno, inode_type, ret,
-                                 loc->inode, subvolume, out);
-
-        if (((flags & O_ACCMODE) == O_WRONLY) ||
-            ((flags & O_ACCMODE) == O_RDWR)) {
-                if (subvolume != FIRST_CHILD (this)) {
-                        op_ret = -1;
-                        op_errno = EINVAL;
-                        goto out;
-                }
-        }
-
-        STACK_WIND_TAIL (frame, subvolume, subvolume->fops->open, loc,
-                         flags, fd, xdata);
-
-        wind = _gf_true;
+    xlator_t *subvolume = NULL;
+    int inode_type = -1;
+    int op_ret = -1;
+    int op_errno = EINVAL;
+    int ret = -1;
+    gf_boolean_t wind = _gf_false;
+
+    GF_VALIDATE_OR_GOTO("svc", this, out);
+    GF_VALIDATE_OR_GOTO(this->name, frame, out);
+    GF_VALIDATE_OR_GOTO(this->name, loc, out);
+    GF_VALIDATE_OR_GOTO(this->name, loc->inode, out);
+    GF_VALIDATE_OR_GOTO(this->name, fd, out);
+
+    /* Another way is to STACK_WIND to normal subvolume, if inode
+       type is not there in the context. If the file actually resides
+       in snapshots, then ENOENT would be returned. Needs more analysis.
+    */
+    SVC_GET_SUBVOL_FROM_CTX(this, op_ret, op_errno, inode_type, ret, loc->inode,
+                            subvolume, out);
+
+    if (((flags & O_ACCMODE) == O_WRONLY) || ((flags & O_ACCMODE) == O_RDWR)) {
+        if (subvolume != FIRST_CHILD(this)) {
+            op_ret = -1;
+            op_errno = EINVAL;
+            goto out;
+        }
+    }
+
+    STACK_WIND_TAIL(frame, subvolume, subvolume->fops->open, loc, flags, fd,
+                    xdata);
+
+    wind = _gf_true;
 
 out:
-        if (!wind)
-                SVC_STACK_UNWIND (open, frame, op_ret, op_errno, NULL,
-                                  NULL);
-        return 0;
+    if (!wind)
+        SVC_STACK_UNWIND(open, frame, op_ret, op_errno, NULL, NULL);
+    return 0;
 }
 
-int32_t
-svc_create_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                int32_t op_ret, int32_t op_errno, fd_t *fd, inode_t *inode,
-                struct iatt *stbuf, struct iatt *preparent,
-                struct iatt *postparent, dict_t *xdata)
+static int32_t
+gf_svc_create_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                  int32_t op_ret, int32_t op_errno, fd_t *fd, inode_t *inode,
+                  struct iatt *stbuf, struct iatt *preparent,
+                  struct iatt *postparent, dict_t *xdata)
 {
-        int  inode_type = -1;
-        int  ret        = -1;
+    int inode_type = -1;
+    int ret = -1;
 
-        if (op_ret < 0)
-                goto out;
+    if (op_ret < 0)
+        goto out;
 
-        inode_type = NORMAL_INODE;
-        ret = svc_inode_ctx_set (this, inode, inode_type);
-        if (ret)
-                gf_log (this->name, GF_LOG_ERROR, "failed to set inode "
-                        "context");
+    inode_type = NORMAL_INODE;
+    ret = svc_inode_ctx_set(this, inode, inode_type);
+    if (ret)
+        gf_smsg(this->name, GF_LOG_ERROR, 0, SVC_MSG_SET_INODE_CONTEXT_FAILED,
+                NULL);
 
 out:
-        SVC_STACK_UNWIND (create, frame, op_ret, op_errno, fd,
-                          inode, stbuf, preparent, postparent, xdata);
+    SVC_STACK_UNWIND(create, frame, op_ret, op_errno, fd, inode, stbuf,
+                     preparent, postparent, xdata);
 
-        return 0;
+    return 0;
 }
 
-int32_t
-svc_create (call_frame_t *frame, xlator_t *this,
-            loc_t *loc, int32_t flags, mode_t mode,
-            mode_t umask, fd_t *fd, dict_t *xdata)
+static int32_t
+gf_svc_create(call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t flags,
+              mode_t mode, mode_t umask, fd_t *fd, dict_t *xdata)
 {
-        int            parent_type = -1;
-        int            ret         = -1;
-        int            op_ret      = -1;
-        int            op_errno    = EINVAL;
-        svc_private_t *priv        = NULL;
-        gf_boolean_t   wind        = _gf_false;
-
-        GF_VALIDATE_OR_GOTO ("svc", this, out);
-        GF_VALIDATE_OR_GOTO (this->name, this->private, out);
-        GF_VALIDATE_OR_GOTO (this->name, frame, out);
-        GF_VALIDATE_OR_GOTO (this->name, loc, out);
-        GF_VALIDATE_OR_GOTO (this->name, loc->inode, out);
-        GF_VALIDATE_OR_GOTO (this->name, fd, out);
-
-        priv = this->private;
-
-        ret = svc_inode_ctx_get (this, loc->parent, &parent_type);
-        if (ret < 0) {
-                gf_log (this->name, GF_LOG_ERROR, "failed to get the inode "
-                        "context for %s", uuid_utoa (loc->parent->gfid));
-                op_ret = -1;
-                op_errno = EINVAL;
-                goto out;
-        }
-
-        if (strcmp (loc->name, priv->path) && parent_type == NORMAL_INODE) {
-                STACK_WIND (frame, svc_create_cbk, FIRST_CHILD (this),
-                            FIRST_CHILD (this)->fops->create, loc, flags,
-                            mode, umask, fd, xdata);
-        } else {
-                op_ret = -1;
-                op_errno = EROFS;
-                goto out;
-        }
-
-        wind = _gf_true;
+    int parent_type = -1;
+    int ret = -1;
+    int op_ret = -1;
+    int op_errno = EINVAL;
+    gf_boolean_t wind = _gf_false;
+    char entry_point[NAME_MAX + 1] = {
+        0,
+    };
+
+    GF_VALIDATE_OR_GOTO("svc", this, out);
+    GF_VALIDATE_OR_GOTO(this->name, frame, out);
+    GF_VALIDATE_OR_GOTO(this->name, loc, out);
+    GF_VALIDATE_OR_GOTO(this->name, loc->inode, out);
+    GF_VALIDATE_OR_GOTO(this->name, fd, out);
+
+    ret = svc_inode_ctx_get(this, loc->parent, &parent_type);
+    if (ret < 0) {
+        op_ret = -1;
+        op_errno = EINVAL;
+        gf_smsg(this->name, GF_LOG_ERROR, op_errno,
+                SVC_MSG_GET_INODE_CONTEXT_FAILED, "gfid=%s",
+                uuid_utoa(loc->parent->gfid), NULL);
+        goto out;
+    }
+
+    if (gf_svc_get_entry_point(this, entry_point, sizeof(entry_point))) {
+        gf_smsg(this->name, GF_LOG_WARNING, op_errno,
+                SVC_MSG_COPY_ENTRY_POINT_FAILED, NULL);
+        goto out;
+    }
+
+    if (strcmp(loc->name, entry_point) && parent_type == NORMAL_INODE) {
+        STACK_WIND(frame, gf_svc_create_cbk, FIRST_CHILD(this),
+                   FIRST_CHILD(this)->fops->create, loc, flags, mode, umask, fd,
+                   xdata);
+    } else {
+        op_ret = -1;
+        op_errno = EROFS;
+        goto out;
+    }
+
+    wind = _gf_true;
 
 out:
-        if (!wind)
-                SVC_STACK_UNWIND (create, frame, op_ret, op_errno,
-                                  NULL, NULL, NULL, NULL, NULL, NULL);
-        return 0;
+    if (!wind)
+        SVC_STACK_UNWIND(create, frame, op_ret, op_errno, NULL, NULL, NULL,
+                         NULL, NULL, NULL);
+    return 0;
 }
 
-int32_t
-svc_symlink_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                     int32_t op_ret, int32_t op_errno, inode_t *inode,
-                     struct iatt *buf, struct iatt *preparent,
-                     struct iatt *postparent, dict_t *xdata)
+static int32_t
+gf_svc_symlink_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                   int32_t op_ret, int32_t op_errno, inode_t *inode,
+                   struct iatt *buf, struct iatt *preparent,
+                   struct iatt *postparent, dict_t *xdata)
 {
-        int inode_type = -1;
-        int ret        = -1;
+    int inode_type = -1;
+    int ret = -1;
 
-        if (op_ret < 0)
-                goto out;
+    if (op_ret < 0)
+        goto out;
 
-        inode_type = NORMAL_INODE;
-        ret = svc_inode_ctx_set (this, inode, inode_type);
-        if (ret)
-                gf_log (this->name, GF_LOG_ERROR, "failed to set inode "
-                        "context");
+    inode_type = NORMAL_INODE;
+    ret = svc_inode_ctx_set(this, inode, inode_type);
+    if (ret)
+        gf_smsg(this->name, GF_LOG_ERROR, 0, SVC_MSG_SET_INODE_CONTEXT_FAILED,
+                NULL);
 
 out:
-        SVC_STACK_UNWIND (symlink, frame, op_ret, op_errno, inode,
-                          buf, preparent, postparent, xdata);
+    SVC_STACK_UNWIND(symlink, frame, op_ret, op_errno, inode, buf, preparent,
+                     postparent, xdata);
 
-        return 0;
+    return 0;
 }
 
-int32_t
-svc_symlink (call_frame_t *frame, xlator_t *this, const char *linkpath,
-             loc_t *loc, mode_t umask, dict_t *xdata)
+static int32_t
+gf_svc_symlink(call_frame_t *frame, xlator_t *this, const char *linkpath,
+               loc_t *loc, mode_t umask, dict_t *xdata)
 {
-        int            parent_type = -1;
-        int            op_ret      = -1;
-        int            op_errno    = EINVAL;
-        int            ret         = -1;
-        svc_private_t *priv        = NULL;
-        gf_boolean_t   wind        = _gf_false;
-
-        GF_VALIDATE_OR_GOTO ("svc", this, out);
-        GF_VALIDATE_OR_GOTO (this->name, frame, out);
-        GF_VALIDATE_OR_GOTO (this->name, this->private, out);
-        GF_VALIDATE_OR_GOTO (this->name, loc, out);
-        GF_VALIDATE_OR_GOTO (this->name, loc->inode, out);
-
-        priv = this->private;
-
-        ret = svc_inode_ctx_get (this, loc->parent, &parent_type);
-        if (ret < 0) {
-                gf_log (this->name, GF_LOG_ERROR, "failed to get the inode "
-                        "context for %s", uuid_utoa (loc->parent->gfid));
-                op_ret = -1;
-                op_errno = EINVAL;
-                goto out;
-        }
-
-        if (strcmp (loc->name, priv->path) && parent_type == NORMAL_INODE) {
-                STACK_WIND (frame, svc_symlink_cbk, FIRST_CHILD (this),
-                            FIRST_CHILD (this)->fops->symlink, linkpath, loc,
-                            umask, xdata);
-        } else {
-                op_ret = -1;
-                op_errno = EROFS;
-                goto out;
-        }
-
-        wind = _gf_true;
+    int parent_type = -1;
+    int op_ret = -1;
+    int op_errno = EINVAL;
+    int ret = -1;
+    gf_boolean_t wind = _gf_false;
+    char entry_point[NAME_MAX + 1] = {
+        0,
+    };
+
+    GF_VALIDATE_OR_GOTO("svc", this, out);
+    GF_VALIDATE_OR_GOTO(this->name, frame, out);
+    GF_VALIDATE_OR_GOTO(this->name, loc, out);
+    GF_VALIDATE_OR_GOTO(this->name, loc->inode, out);
+
+    ret = svc_inode_ctx_get(this, loc->parent, &parent_type);
+    if (ret < 0) {
+        op_ret = -1;
+        op_errno = EINVAL;
+        gf_smsg(this->name, GF_LOG_ERROR, op_errno,
+                SVC_MSG_GET_INODE_CONTEXT_FAILED, "gfid=%s",
+                uuid_utoa(loc->parent->gfid), NULL);
+        goto out;
+    }
+
+    if (gf_svc_get_entry_point(this, entry_point, sizeof(entry_point))) {
+        gf_smsg(this->name, GF_LOG_WARNING, op_errno,
+                SVC_MSG_COPY_ENTRY_POINT_FAILED, NULL);
+        goto out;
+    }
+
+    if (strcmp(loc->name, entry_point) && parent_type == NORMAL_INODE) {
+        STACK_WIND(frame, gf_svc_symlink_cbk, FIRST_CHILD(this),
+                   FIRST_CHILD(this)->fops->symlink, linkpath, loc, umask,
+                   xdata);
+    } else {
+        op_ret = -1;
+        op_errno = EROFS;
+        goto out;
+    }
+
+    wind = _gf_true;
 
 out:
-        if (!wind)
-                SVC_STACK_UNWIND (symlink, frame, op_ret, op_errno,
-                                  NULL, NULL, NULL, NULL, NULL);
-        return 0;
+    if (!wind)
+        SVC_STACK_UNWIND(symlink, frame, op_ret, op_errno, NULL, NULL, NULL,
+                         NULL, NULL);
+    return 0;
 }
 
-int32_t
-svc_unlink (call_frame_t *frame, xlator_t *this, loc_t *loc, int flags,
-            dict_t *xdata)
+static int32_t
+gf_svc_unlink(call_frame_t *frame, xlator_t *this, loc_t *loc, int flags,
+              dict_t *xdata)
 {
-        int            inode_type   = -1;
-        int            op_ret       = -1;
-        int            op_errno     = EINVAL;
-        int            ret          = -1;
-        gf_boolean_t   wind         = _gf_false;
-
-        GF_VALIDATE_OR_GOTO ("svc", this, out);
-        GF_VALIDATE_OR_GOTO (this->name, frame, out);
-        GF_VALIDATE_OR_GOTO (this->name, loc, out);
-        GF_VALIDATE_OR_GOTO (this->name, loc->inode, out);
-
-        ret = svc_inode_ctx_get (this, loc->inode, &inode_type);
-        if (ret < 0) {
-                gf_log (this->name, GF_LOG_ERROR, "failed to get the inode "
-                        "context for %s", uuid_utoa (loc->parent->gfid));
-                op_ret = -1;
-                op_errno = EINVAL;
-                goto out;
-        }
-
-        if (inode_type == NORMAL_INODE) {
-                STACK_WIND_TAIL (frame, FIRST_CHILD (this),
-                                 FIRST_CHILD (this)->fops->unlink, loc, flags,
-                                 xdata);
-        } else {
-                op_ret = -1;
-                op_errno = EROFS;
-                goto out;
-        }
-
-        wind = _gf_true;
+    int inode_type = -1;
+    int op_ret = -1;
+    int op_errno = EINVAL;
+    int ret = -1;
+    gf_boolean_t wind = _gf_false;
+
+    GF_VALIDATE_OR_GOTO("svc", this, out);
+    GF_VALIDATE_OR_GOTO(this->name, frame, out);
+    GF_VALIDATE_OR_GOTO(this->name, loc, out);
+    GF_VALIDATE_OR_GOTO(this->name, loc->inode, out);
+
+    ret = svc_inode_ctx_get(this, loc->inode, &inode_type);
+    if (ret < 0) {
+        op_ret = -1;
+        op_errno = EINVAL;
+        gf_smsg(this->name, GF_LOG_ERROR, op_errno,
+                SVC_MSG_GET_INODE_CONTEXT_FAILED, "gfid=%s",
+                uuid_utoa(loc->parent->gfid), NULL);
+        goto out;
+    }
+
+    if (inode_type == NORMAL_INODE) {
+        STACK_WIND_TAIL(frame, FIRST_CHILD(this),
+                        FIRST_CHILD(this)->fops->unlink, loc, flags, xdata);
+    } else {
+        op_ret = -1;
+        op_errno = EROFS;
+        goto out;
+    }
+
+    wind = _gf_true;
 
 out:
-        if (!wind)
-                SVC_STACK_UNWIND (unlink, frame, op_ret, op_errno, NULL, NULL,
-                                  NULL);
-        return 0;
+    if (!wind)
+        SVC_STACK_UNWIND(unlink, frame, op_ret, op_errno, NULL, NULL, NULL);
+    return 0;
 }
 
-int32_t
-svc_readv (call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
-           off_t offset, uint32_t flags, dict_t *xdata)
+static int32_t
+gf_svc_readv(call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
+             off_t offset, uint32_t flags, dict_t *xdata)
 {
-        int           inode_type = -1;
-        xlator_t     *subvolume  = NULL;
-        int           ret        = -1;
-        int           op_ret     = -1;
-        int           op_errno   = EINVAL;
-        gf_boolean_t  wind       = _gf_false;
+    int inode_type = -1;
+    xlator_t *subvolume = NULL;
+    int ret = -1;
+    int op_ret = -1;
+    int op_errno = EINVAL;
+    gf_boolean_t wind = _gf_false;
 
-        GF_VALIDATE_OR_GOTO ("svc", this, out);
-        GF_VALIDATE_OR_GOTO (this->name, frame, out);
-        GF_VALIDATE_OR_GOTO (this->name, fd, out);
-        GF_VALIDATE_OR_GOTO (this->name, fd->inode, out);
+    GF_VALIDATE_OR_GOTO("svc", this, out);
+    GF_VALIDATE_OR_GOTO(this->name, frame, out);
+    GF_VALIDATE_OR_GOTO(this->name, fd, out);
+    GF_VALIDATE_OR_GOTO(this->name, fd->inode, out);
 
-        SVC_GET_SUBVOL_FROM_CTX (this, op_ret, op_errno, inode_type, ret,
-                                 fd->inode, subvolume, out);
+    SVC_GET_SUBVOL_FROM_CTX(this, op_ret, op_errno, inode_type, ret, fd->inode,
+                            subvolume, out);
 
-        STACK_WIND_TAIL (frame, subvolume, subvolume->fops->readv,
-                         fd, size, offset, flags, xdata);
+    STACK_WIND_TAIL(frame, subvolume, subvolume->fops->readv, fd, size, offset,
+                    flags, xdata);
 
-        wind = _gf_true;
+    wind = _gf_true;
 
 out:
-        if (!wind)
-                SVC_STACK_UNWIND (readv, frame, op_ret, op_errno, NULL, 0, NULL,
-                                  NULL, NULL);
-        return 0;
+    if (!wind)
+        SVC_STACK_UNWIND(readv, frame, op_ret, op_errno, NULL, 0, NULL, NULL,
+                         NULL);
+    return 0;
 }
 
-int32_t
-svc_readlink (call_frame_t *frame, xlator_t *this,
-              loc_t *loc, size_t size, dict_t *xdata)
+static int32_t
+gf_svc_readlink(call_frame_t *frame, xlator_t *this, loc_t *loc, size_t size,
+                dict_t *xdata)
 {
-        int              inode_type = -1;
-        xlator_t        *subvolume  = NULL;
-        int              ret        = -1;
-        int              op_ret     = -1;
-        int              op_errno   = EINVAL;
-        gf_boolean_t     wind       = _gf_false;
+    int inode_type = -1;
+    xlator_t *subvolume = NULL;
+    int ret = -1;
+    int op_ret = -1;
+    int op_errno = EINVAL;
+    gf_boolean_t wind = _gf_false;
 
-        GF_VALIDATE_OR_GOTO ("svc", this, out);
-        GF_VALIDATE_OR_GOTO (this->name, frame, out);
-        GF_VALIDATE_OR_GOTO (this->name, loc, out);
-        GF_VALIDATE_OR_GOTO (this->name, loc->inode, out);
+    GF_VALIDATE_OR_GOTO("svc", this, out);
+    GF_VALIDATE_OR_GOTO(this->name, frame, out);
+    GF_VALIDATE_OR_GOTO(this->name, loc, out);
+    GF_VALIDATE_OR_GOTO(this->name, loc->inode, out);
 
-        SVC_GET_SUBVOL_FROM_CTX (this, op_ret, op_errno, inode_type, ret,
-                                 loc->inode, subvolume, out);
+    SVC_GET_SUBVOL_FROM_CTX(this, op_ret, op_errno, inode_type, ret, loc->inode,
+                            subvolume, out);
 
-        STACK_WIND_TAIL (frame, subvolume, subvolume->fops->readlink, loc, size,
-                         xdata);
+    STACK_WIND_TAIL(frame, subvolume, subvolume->fops->readlink, loc, size,
+                    xdata);
 
-        wind = _gf_true;
+    wind = _gf_true;
 
 out:
-        if (!wind)
-                STACK_UNWIND_STRICT (readlink, frame, op_ret, op_errno, NULL, NULL,
-                                     NULL);
-        return 0;
+    if (!wind)
+        STACK_UNWIND_STRICT(readlink, frame, op_ret, op_errno, NULL, NULL,
+                            NULL);
+    return 0;
 }
 
-int32_t
-svc_access (call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t mask,
-            dict_t *xdata)
+static int32_t
+gf_svc_access(call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t mask,
+              dict_t *xdata)
 {
-        int            ret        = -1;
-        int            inode_type = -1;
-        xlator_t      *subvolume  = NULL;
-        int            op_ret     = -1;
-        int            op_errno   = EINVAL;
-        gf_boolean_t   wind       = _gf_false;
+    int ret = -1;
+    int inode_type = -1;
+    xlator_t *subvolume = NULL;
+    int op_ret = -1;
+    int op_errno = EINVAL;
+    gf_boolean_t wind = _gf_false;
 
-        GF_VALIDATE_OR_GOTO ("svc", this, out);
-        GF_VALIDATE_OR_GOTO (this->name, frame, out);
-        GF_VALIDATE_OR_GOTO (this->name, loc, out);
-        GF_VALIDATE_OR_GOTO (this->name, loc->inode, out);
+    GF_VALIDATE_OR_GOTO("svc", this, out);
+    GF_VALIDATE_OR_GOTO(this->name, frame, out);
+    GF_VALIDATE_OR_GOTO(this->name, loc, out);
+    GF_VALIDATE_OR_GOTO(this->name, loc->inode, out);
 
-        SVC_GET_SUBVOL_FROM_CTX (this, op_ret, op_errno, inode_type, ret,
-                                 loc->inode, subvolume, out);
+    SVC_GET_SUBVOL_FROM_CTX(this, op_ret, op_errno, inode_type, ret, loc->inode,
+                            subvolume, out);
 
-        STACK_WIND_TAIL (frame, subvolume, subvolume->fops->access, loc, mask,
-                         xdata);
+    STACK_WIND_TAIL(frame, subvolume, subvolume->fops->access, loc, mask,
+                    xdata);
 
-        wind = _gf_true;
+    wind = _gf_true;
 
 out:
-        if (!wind)
-                SVC_STACK_UNWIND (access, frame, op_ret, op_errno, NULL);
+    if (!wind)
+        SVC_STACK_UNWIND(access, frame, op_ret, op_errno, NULL);
 
-        return 0;
+    return 0;
 }
 
 int32_t
-svc_readdir (call_frame_t *frame, xlator_t *this, fd_t *fd,
-             size_t size, off_t off,
-             dict_t *xdata)
+gf_svc_readdir_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                   int32_t op_ret, int32_t op_errno, gf_dirent_t *entries,
+                   dict_t *xdata)
 {
-        int           inode_type = -1;
-        xlator_t     *subvolume  = NULL;
-        int           ret        = -1;
-        int           op_ret     = -1;
-        int           op_errno   = EINVAL;
-        gf_boolean_t  wind       = _gf_false;
-        svc_fd_t     *svc_fd     = NULL;
-        gf_dirent_t   entries;
-
-        INIT_LIST_HEAD (&entries);
-
-        GF_VALIDATE_OR_GOTO ("svc", this, out);
-        GF_VALIDATE_OR_GOTO (this->name, frame, out);
-        GF_VALIDATE_OR_GOTO (this->name, fd, out);
-        GF_VALIDATE_OR_GOTO (this->name, fd->inode, out);
+    gf_dirent_t *entry = NULL;
+    gf_dirent_t *tmpentry = NULL;
+    svc_local_t *local = NULL;
+    char entry_point[NAME_MAX + 1] = {
+        0,
+    };
+
+    if (op_ret < 0)
+        goto out;
+
+    local = frame->local;
+
+    /* If .snaps pre-exists, then it should not be listed
+     * in the NORMAL INODE directory when USS is enabled,
+     * so filter the .snaps entry if exists.
+     * However it is OK to list .snaps in VIRTUAL world
+     */
+    if (local->subvolume != FIRST_CHILD(this))
+        goto out;
+
+    /*
+     * Better to goto out if getting the entry point
+     * fails. We might end up sending the directory
+     * entry for the snapview entry point in the readdir
+     * response. But, the intention is to avoid the race
+     * condition where priv->path is being changed in
+     * reconfigure while this is accessing it.
+     */
+    if (gf_svc_get_entry_point(this, entry_point, sizeof(entry_point))) {
+        gf_smsg(this->name, GF_LOG_WARNING, op_errno,
+                SVC_MSG_COPY_ENTRY_POINT_FAILED, NULL);
+        goto out;
+    }
+
+    list_for_each_entry_safe(entry, tmpentry, &entries->list, list)
+    {
+        if (strcmp(entry_point, entry->d_name) == 0)
+            gf_dirent_entry_free(entry);
+    }
 
-        svc_fd = svc_fd_ctx_get_or_new (this,  fd);
-        if (!svc_fd)
-                gf_log (this->name, GF_LOG_ERROR, "failed to get the fd "
-                        "context for the inode %s",
-                        uuid_utoa (fd->inode->gfid));
-        else {
-                if (svc_fd->entry_point_handled && off == svc_fd->last_offset) {
-                        op_ret = 0;
-                        op_errno = ENOENT;
-                        goto out;
-                }
-        }
-
-        SVC_GET_SUBVOL_FROM_CTX (this, op_ret, op_errno, inode_type, ret,
-                                 fd->inode, subvolume, out);
-
-        STACK_WIND_TAIL (frame, subvolume, subvolume->fops->readdir, fd, size,
-                         off, xdata);
+out:
+    SVC_STACK_UNWIND(readdir, frame, op_ret, op_errno, entries, xdata);
+    return 0;
+}
 
-        wind = _gf_true;
+static int32_t
+gf_svc_readdir(call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
+               off_t off, dict_t *xdata)
+{
+    int inode_type = -1;
+    xlator_t *subvolume = NULL;
+    svc_local_t *local = NULL;
+    int ret = -1;
+    int op_ret = -1;
+    int op_errno = EINVAL;
+    gf_boolean_t wind = _gf_false;
+    svc_fd_t *svc_fd = NULL;
+    gf_dirent_t entries;
+
+    INIT_LIST_HEAD(&entries);
+
+    GF_VALIDATE_OR_GOTO("svc", this, out);
+    GF_VALIDATE_OR_GOTO(this->name, frame, out);
+    GF_VALIDATE_OR_GOTO(this->name, fd, out);
+    GF_VALIDATE_OR_GOTO(this->name, fd->inode, out);
+
+    svc_fd = svc_fd_ctx_get_or_new(this, fd);
+    if (!svc_fd)
+        gf_smsg(this->name, GF_LOG_ERROR, 0, SVC_MSG_GET_FD_CONTEXT_FAILED,
+                "gfid=%s", uuid_utoa(fd->inode->gfid), NULL);
+    else {
+        if (svc_fd->entry_point_handled && off == svc_fd->last_offset) {
+            op_ret = 0;
+            op_errno = ENOENT;
+            goto out;
+        }
+    }
+
+    SVC_GET_SUBVOL_FROM_CTX(this, op_ret, op_errno, inode_type, ret, fd->inode,
+                            subvolume, out);
+
+    local = mem_get0(this->local_pool);
+    if (!local) {
+        gf_smsg(this->name, GF_LOG_ERROR, op_errno, SVC_MSG_NO_MEMORY,
+                "inode-gfid=%s", uuid_utoa(fd->inode->gfid), NULL);
+        goto out;
+    }
+    local->subvolume = subvolume;
+    frame->local = local;
+
+    STACK_WIND(frame, gf_svc_readdir_cbk, subvolume, subvolume->fops->readdir,
+               fd, size, off, xdata);
+
+    wind = _gf_true;
 
 out:
-        if (!wind)
-                SVC_STACK_UNWIND (readdir, frame, op_ret, op_errno, &entries,
-                                  NULL);
+    if (!wind)
+        SVC_STACK_UNWIND(readdir, frame, op_ret, op_errno, &entries, NULL);
 
-        gf_dirent_free (&entries);
+    gf_dirent_free(&entries);
 
-        return 0;
+    return 0;
 }
 
 /*
@@ -1555,801 +1728,1064 @@ out:
  * windows.
  */
 
-int32_t
-svc_readdirp_lookup_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                         int32_t op_ret, int32_t op_errno, inode_t *inode,
-                         struct iatt *buf, dict_t *xdata,
-                         struct iatt *postparent)
+static int32_t
+gf_svc_readdirp_lookup_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                           int32_t op_ret, int32_t op_errno, inode_t *inode,
+                           struct iatt *buf, dict_t *xdata,
+                           struct iatt *postparent)
 {
-        gf_dirent_t    entries;
-        gf_dirent_t   *entry      = NULL;
-        svc_private_t *private    = NULL;
-        svc_fd_t      *svc_fd     = NULL;
-        svc_local_t   *local      = NULL;
-        int            inode_type = -1;
-        int            ret        = -1;
-
-        GF_VALIDATE_OR_GOTO ("snapview-client", this, out);
-        GF_VALIDATE_OR_GOTO (this->name, this->private, out);
-
-        private = this->private;
-        INIT_LIST_HEAD (&entries.list);
-
-        local = frame->local;
-
-        if (op_ret) {
-                op_ret = 0;
-                op_errno = ENOENT;
-                goto out;
-        }
-
-        svc_fd = svc_fd_ctx_get (this, local->fd);
-        if (!svc_fd) {
-                gf_log (this->name, GF_LOG_ERROR, "failed to get the fd "
-                        "context for the inode %s",
-                        uuid_utoa (local->fd->inode->gfid));
-                op_ret = 0;
-                op_errno = ENOENT;
-                goto out;
-        }
-
-        entry = gf_dirent_for_name (private->path);
-        if (!entry) {
-                gf_log (this->name, GF_LOG_ERROR, "failed to allocate memory "
-                        "for the entry %s", private->path);
-                op_ret = 0;
-                op_errno = ENOMEM;
-                goto out;
-        }
-
-        entry->inode = inode_ref (inode);
-        entry->d_off = svc_fd->last_offset + 22;
-        entry->d_ino = buf->ia_ino;
-        entry->d_type = DT_DIR;
-        entry->d_stat = *buf;
-        inode_type = VIRTUAL_INODE;
-        ret = svc_inode_ctx_set (this, entry->inode, inode_type);
-        if (ret)
-                gf_log (this->name, GF_LOG_ERROR, "failed to set the inode "
-                        "context");
-
-        list_add_tail (&entry->list, &entries.list);
-        op_ret = 1;
-        svc_fd->last_offset = entry->d_off;
-        svc_fd->entry_point_handled = _gf_true;
+    gf_dirent_t entries;
+    gf_dirent_t *entry = NULL;
+    svc_fd_t *svc_fd = NULL;
+    svc_local_t *local = NULL;
+    int inode_type = -1;
+    int ret = -1;
+    char entry_point[NAME_MAX + 1] = {
+        0,
+    };
+
+    GF_VALIDATE_OR_GOTO("snapview-client", this, out);
+
+    INIT_LIST_HEAD(&entries.list);
+
+    local = frame->local;
+
+    if (op_ret) {
+        if (op_errno == ESTALE && !local->revalidate) {
+            local->revalidate = 1;
+            ret = gf_svc_special_dir_revalidate_lookup(frame, this, xdata);
+
+            if (!ret)
+                return 0;
+        }
+        op_ret = 0;
+        op_errno = ENOENT;
+        goto out;
+    }
+
+    svc_fd = svc_fd_ctx_get(this, local->fd);
+    if (!svc_fd) {
+        gf_smsg(this->name, GF_LOG_ERROR, 0, SVC_MSG_GET_FD_CONTEXT_FAILED,
+                "gfid=%s", uuid_utoa(local->fd->inode->gfid), NULL);
+        op_ret = 0;
+        op_errno = ENOENT;
+        goto out;
+    }
+
+    if (gf_svc_get_entry_point(this, entry_point, sizeof(entry_point))) {
+        gf_smsg(this->name, GF_LOG_WARNING, 0, SVC_MSG_COPY_ENTRY_POINT_FAILED,
+                NULL);
+        op_ret = 0;
+        op_errno = ENOENT;
+        goto out;
+    }
+
+    entry = gf_dirent_for_name(entry_point);
+    if (!entry) {
+        gf_smsg(this->name, GF_LOG_ERROR, 0, SVC_MSG_NO_MEMORY,
+                "entry-point=%s", entry_point, NULL);
+        op_ret = 0;
+        op_errno = ENOMEM;
+        goto out;
+    }
+
+    entry->inode = inode_ref(inode);
+    entry->d_off = svc_fd->last_offset + 22;
+    entry->d_ino = buf->ia_ino;
+    entry->d_type = DT_DIR;
+    entry->d_stat = *buf;
+    inode_type = VIRTUAL_INODE;
+    ret = svc_inode_ctx_set(this, entry->inode, inode_type);
+    if (ret)
+        gf_smsg(this->name, GF_LOG_ERROR, 0, SVC_MSG_SET_INODE_CONTEXT_FAILED,
+                "entry-name=%s", entry->d_name, NULL);
+
+    list_add_tail(&entry->list, &entries.list);
+    op_ret = 1;
+    svc_fd->last_offset = entry->d_off;
+    svc_fd->entry_point_handled = _gf_true;
 
 out:
-        SVC_STACK_UNWIND (readdirp, frame, op_ret, op_errno, &entries,
-                          local->xdata);
+    SVC_STACK_UNWIND(readdirp, frame, op_ret, op_errno, &entries,
+                     local ? local->xdata : NULL);
 
-        gf_dirent_free (&entries);
+    gf_dirent_free(&entries);
 
-        return 0;
+    return 0;
 }
 
-gf_boolean_t
-svc_readdir_on_special_dir (call_frame_t *frame, void *cookie, xlator_t *this,
-                            int32_t op_ret, int32_t op_errno,
-                            gf_dirent_t *entries, dict_t *xdata)
+int
+gf_svc_special_dir_revalidate_lookup(call_frame_t *frame, xlator_t *this,
+                                     dict_t *xdata)
 {
-        svc_local_t   *local      = NULL;
-        svc_private_t *private    = NULL;
-        inode_t       *inode      = NULL;
-        fd_t          *fd         = NULL;
-        char          *path       = NULL;
-        loc_t         *loc        = NULL;
-        dict_t        *tmp_xdata  = NULL;
-        int            ret        = -1;
-        gf_boolean_t   unwind     = _gf_true;
-        svc_fd_t      *svc_fd     = NULL;
-
-        GF_VALIDATE_OR_GOTO ("snapview-client", this, out);
-        GF_VALIDATE_OR_GOTO (this->name, this->private, out);
-
-        private = this->private;
-        local = frame->local;
-
-        loc = &local->loc;
-        fd = local->fd;
-        svc_fd = svc_fd_ctx_get (this, fd);
-        if (!svc_fd) {
-                gf_log (this->name, GF_LOG_ERROR, "failed to get the fd "
-                        "context for the inode %s",
-                        uuid_utoa (fd->inode->gfid));
-                goto out;
-        }
-
-        /*
-         * check if its end of readdir operation from posix, if special_dir
-         * option is set, if readdir is done on special directory and if
-         * readdirp is from normal regular graph.
-         */
-
-        if (!private->show_entry_point)
-                goto out;
-
-        if (op_ret == 0 && op_errno == ENOENT && private->special_dir &&
-            strcmp (private->special_dir, "") && svc_fd->special_dir &&
-            local->subvolume == FIRST_CHILD (this)) {
-                inode = inode_grep (fd->inode->table, fd->inode,
-                                    private->path);
-                if (!inode) {
-                        inode = inode_new (fd->inode->table);
-                        if (!inode) {
-                                gf_log (this->name, GF_LOG_ERROR, "failed to "
-                                        "allocate new inode");
-                                goto out;
-                        }
-                }
-
-                gf_uuid_copy (local->loc.pargfid, fd->inode->gfid);
-                gf_uuid_copy (local->loc.gfid, inode->gfid);
-                if (gf_uuid_is_null (inode->gfid))
-                        ret = inode_path (fd->inode, private->path, &path);
-                else
-                        ret = inode_path (inode, NULL, &path);
-
-                if (ret < 0)
-                        goto out;
-                loc->path = gf_strdup (path);
-                if (loc->path) {
-                        if (!loc->name ||
-                            (loc->name && !strcmp (loc->name, ""))) {
-                                loc->name = strrchr (loc->path, '/');
-                                if (loc->name)
-                                        loc->name++;
-                        }
-                }
-
-                loc->inode = inode;
-                loc->parent = inode_ref (fd->inode);
-                tmp_xdata = dict_new ();
-                if (!tmp_xdata)
-                        goto out;
-                ret = dict_set_str (tmp_xdata, "entry-point", "true");
-                if (ret) {
-                        gf_log (this->name, GF_LOG_ERROR, "failed to set dict");
-                        goto out;
-                }
-
-                local->cookie = cookie;
-                local->xdata = dict_ref (xdata);
-                STACK_WIND (frame, svc_readdirp_lookup_cbk,
-                            SECOND_CHILD (this),
-                            SECOND_CHILD (this)->fops->lookup, loc, tmp_xdata);
-                unwind = _gf_false;
-        }
-
+    svc_local_t *local = NULL;
+    loc_t *loc = NULL;
+    dict_t *tmp_xdata = NULL;
+    char *path = NULL;
+    int ret = -1;
+    char entry_point[NAME_MAX + 1] = {
+        0,
+    };
+
+    GF_VALIDATE_OR_GOTO("snapview-client", this, out);
+
+    local = frame->local;
+    loc = &local->loc;
+
+    if (local->xdata) {
+        dict_unref(local->xdata);
+        local->xdata = NULL;
+    }
+
+    if (xdata)
+        local->xdata = dict_ref(xdata);
+
+    inode_unref(loc->inode);
+    loc->inode = inode_new(loc->parent->table);
+    if (!loc->inode) {
+        gf_smsg(this->name, GF_LOG_ERROR, ENOMEM, SVC_MSG_ALLOC_INODE_FAILED,
+                NULL);
+        goto out;
+    }
+
+    if (gf_svc_get_entry_point(this, entry_point, sizeof(entry_point))) {
+        gf_smsg(this->name, GF_LOG_WARNING, 0, SVC_MSG_COPY_ENTRY_POINT_FAILED,
+                NULL);
+        goto out;
+    }
+
+    gf_uuid_copy(local->loc.gfid, loc->inode->gfid);
+    ret = inode_path(loc->parent, entry_point, &path);
+    if (ret < 0)
+        goto out;
+
+    if (loc->path)
+        GF_FREE((char *)loc->path);
+
+    loc->path = gf_strdup(path);
+    if (loc->path) {
+        if (!loc->name || (loc->name && !strcmp(loc->name, ""))) {
+            loc->name = strrchr(loc->path, '/');
+            if (loc->name)
+                loc->name++;
+        }
+    } else
+        loc->path = NULL;
+
+    tmp_xdata = dict_new();
+    if (!tmp_xdata) {
+        ret = -1;
+        goto out;
+    }
+
+    ret = dict_set_str(tmp_xdata, "entry-point", "true");
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_ERROR, 0, SVC_MSG_DICT_SET_FAILED, NULL);
+        goto out;
+    }
+
+    STACK_WIND(frame, gf_svc_readdirp_lookup_cbk, SECOND_CHILD(this),
+               SECOND_CHILD(this)->fops->lookup, loc, tmp_xdata);
 out:
-        if (tmp_xdata)
-                dict_unref (tmp_xdata);
+    if (tmp_xdata)
+        dict_unref(tmp_xdata);
 
-        GF_FREE (path);
-        return unwind;
+    GF_FREE(path);
+    return ret;
 }
 
-int32_t
-svc_readdirp_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                  int32_t op_ret, int32_t op_errno,
-                  gf_dirent_t *entries, dict_t *xdata)
+static gf_boolean_t
+gf_svc_readdir_on_special_dir(call_frame_t *frame, void *cookie, xlator_t *this,
+                              int32_t op_ret, int32_t op_errno,
+                              gf_dirent_t *entries, dict_t *xdata)
 {
-        gf_dirent_t   *entry      = NULL;
-        svc_local_t   *local      = NULL;
-        gf_boolean_t   real       = _gf_true;
-        int            inode_type = -1;
-        int            ret        = -1;
-        svc_fd_t      *svc_fd     = NULL;
-        gf_boolean_t   unwind     = _gf_true;
-
-        GF_VALIDATE_OR_GOTO ("snapview-client", this, out);
-
-        if (op_ret < 0)
+    svc_local_t *local = NULL;
+    svc_private_t *private = NULL;
+    inode_t *inode = NULL;
+    fd_t *fd = NULL;
+    char *path = NULL;
+    loc_t *loc = NULL;
+    dict_t *tmp_xdata = NULL;
+    int ret = -1;
+    gf_boolean_t unwind = _gf_true;
+    svc_fd_t *svc_fd = NULL;
+    char entry_point[NAME_MAX + 1] = {
+        0,
+    };
+
+    GF_VALIDATE_OR_GOTO("snapview-client", this, out);
+    GF_VALIDATE_OR_GOTO(this->name, this->private, out);
+
+   private
+    = this->private;
+    local = frame->local;
+
+    loc = &local->loc;
+    fd = local->fd;
+    svc_fd = svc_fd_ctx_get(this, fd);
+    if (!svc_fd) {
+        gf_smsg(this->name, GF_LOG_ERROR, 0, SVC_MSG_GET_FD_CONTEXT_FAILED,
+                "gfid=%s", uuid_utoa(fd->inode->gfid), NULL);
+        goto out;
+    }
+
+    /*
+     * check if its end of readdir operation from posix, if special_dir
+     * option is set, if readdir is done on special directory and if
+     * readdirp is from normal regular graph.
+     */
+
+    if (!private->show_entry_point)
+        goto out;
+
+    if (op_ret == 0 && op_errno == ENOENT && private->special_dir &&
+        strcmp(private->special_dir, "") && svc_fd->special_dir &&
+        local->subvolume == FIRST_CHILD(this)) {
+        if (gf_svc_get_entry_point(this, entry_point, sizeof(entry_point))) {
+            gf_smsg(this->name, GF_LOG_WARNING, 0,
+                    SVC_MSG_GET_FD_CONTEXT_FAILED, NULL);
+            goto out;
+        }
+
+        inode = inode_grep(fd->inode->table, fd->inode, entry_point);
+        if (!inode) {
+            inode = inode_new(fd->inode->table);
+            if (!inode) {
+                gf_smsg(this->name, GF_LOG_ERROR, 0, SVC_MSG_ALLOC_INODE_FAILED,
+                        NULL);
                 goto out;
+            }
+        }
 
-        local = frame->local;
+        gf_uuid_copy(local->loc.pargfid, fd->inode->gfid);
+        gf_uuid_copy(local->loc.gfid, inode->gfid);
+        if (gf_uuid_is_null(inode->gfid))
+            ret = inode_path(fd->inode, entry_point, &path);
+        else
+            ret = inode_path(inode, NULL, &path);
 
-        svc_fd = svc_fd_ctx_get (this, local->fd);
-        if (!svc_fd) {
-                gf_log (this->name, GF_LOG_WARNING, "failed to get the fd "
-                        "context for the gfid %s",
-                        uuid_utoa (local->fd->inode->gfid));
+        if (ret < 0)
+            goto out;
+        loc->path = gf_strdup(path);
+        if (loc->path) {
+            if (!loc->name || (loc->name && !strcmp(loc->name, ""))) {
+                loc->name = strrchr(loc->path, '/');
+                if (loc->name)
+                    loc->name++;
+            }
+        }
+
+        loc->inode = inode;
+        loc->parent = inode_ref(fd->inode);
+        tmp_xdata = dict_new();
+        if (!tmp_xdata)
+            goto out;
+        ret = dict_set_str(tmp_xdata, "entry-point", "true");
+        if (ret) {
+            gf_smsg(this->name, GF_LOG_ERROR, 0, SVC_MSG_DICT_SET_FAILED, NULL);
+            goto out;
         }
 
-        if (local->subvolume == FIRST_CHILD (this))
-                real = _gf_true;
-        else
-                real = _gf_false;
-
-        list_for_each_entry (entry, &entries->list, list) {
-                if (!entry->inode)
-                        continue;
-
-                if (real)
-                        inode_type = NORMAL_INODE;
-                else
-                        inode_type = VIRTUAL_INODE;
-
-                ret = svc_inode_ctx_set (this, entry->inode, inode_type);
-                if (ret)
-                        gf_log (this->name, GF_LOG_ERROR, "failed to set inode "
-                                "context");
-                if (svc_fd)
-                        svc_fd->last_offset = entry->d_off;
+        local->cookie = cookie;
+        if (local->xdata) {
+            dict_unref(local->xdata);
+            local->xdata = NULL;
         }
+        if (xdata)
+            local->xdata = dict_ref(xdata);
 
-        unwind = svc_readdir_on_special_dir (frame, cookie, this, op_ret,
-                                             op_errno, entries, xdata);
+        STACK_WIND(frame, gf_svc_readdirp_lookup_cbk, SECOND_CHILD(this),
+                   SECOND_CHILD(this)->fops->lookup, loc, tmp_xdata);
+        unwind = _gf_false;
+    }
 
 out:
-        if (unwind)
-                SVC_STACK_UNWIND (readdirp, frame, op_ret, op_errno, entries,
-                                  xdata);
+    if (tmp_xdata)
+        dict_unref(tmp_xdata);
 
-        return 0;
+    GF_FREE(path);
+    return unwind;
 }
 
-int32_t
-svc_readdirp (call_frame_t *frame, xlator_t *this, fd_t *fd,
-              size_t size, off_t off,
-              dict_t *xdata)
+static int32_t
+gf_svc_readdirp_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                    int32_t op_ret, int32_t op_errno, gf_dirent_t *entries,
+                    dict_t *xdata)
 {
-        int            inode_type = -1;
-        xlator_t      *subvolume  = NULL;
-        svc_local_t   *local      = NULL;
-        int            ret        = -1;
-        int            op_ret     = -1;
-        int            op_errno   = EINVAL;
-        gf_boolean_t   wind       = _gf_false;
-        svc_fd_t      *svc_fd     = NULL;
-        gf_dirent_t    entries;
-
-        INIT_LIST_HEAD (&entries.list);
-
-        GF_VALIDATE_OR_GOTO ("svc", this, out);
-        GF_VALIDATE_OR_GOTO (this->name, frame, out);
-        GF_VALIDATE_OR_GOTO (this->name, fd, out);
-        GF_VALIDATE_OR_GOTO (this->name, fd->inode, out);
-
-        local = mem_get0 (this->local_pool);
-        if (!local) {
-                gf_log (this->name, GF_LOG_ERROR, "failed to allocate local");
-                op_errno = ENOMEM;
-                goto out;
-        }
+    gf_dirent_t *entry = NULL;
+    gf_dirent_t *tmpentry = NULL;
+    svc_local_t *local = NULL;
+    int inode_type = -1;
+    int ret = -1;
+    svc_fd_t *svc_fd = NULL;
+    gf_boolean_t unwind = _gf_true;
+    char entry_point[NAME_MAX + 1] = {
+        0,
+    };
+
+    if (op_ret < 0)
+        goto out;
+
+    GF_VALIDATE_OR_GOTO("snapview-client", this, out);
+
+    local = frame->local;
+
+    svc_fd = svc_fd_ctx_get(this, local->fd);
+    if (!svc_fd) {
+        gf_smsg(this->name, GF_LOG_WARNING, 0, SVC_MSG_GET_FD_CONTEXT_FAILED,
+                "gfid=%s", uuid_utoa(local->fd->inode->gfid), NULL);
+    }
+
+    if (local->subvolume == FIRST_CHILD(this))
+        inode_type = NORMAL_INODE;
+    else
+        inode_type = VIRTUAL_INODE;
 
-        /*
-         * This is mainly for samba shares (or windows clients). As part of
-         * readdirp on the directory used as samba share, the entry point
-         * directory would have been added at the end. So when a new readdirp
-         * request comes, we have to check if the entry point has been handled
-         * or not in readdirp. That information and the offset used for it
-         * is remembered in fd context. If it has been handled, then simply
-         * unwind indication end of readdir operation.
+    /*
+     * Better to goto out and return whatever is there in the
+     * readdirp response (even if the readdir response contains
+     * a directory entry for the snapshot entry point). Otherwise
+     * if we ignore the error, then there is a chance of race
+     * condition where, priv->path is changed in reconfigure
+     */
+    if (gf_svc_get_entry_point(this, entry_point, sizeof(entry_point))) {
+        gf_smsg(this->name, GF_LOG_WARNING, 0, SVC_MSG_COPY_ENTRY_POINT_FAILED,
+                NULL);
+        goto out;
+    }
+
+    list_for_each_entry_safe(entry, tmpentry, &entries->list, list)
+    {
+        /* If .snaps pre-exists, then it should not be listed
+         * in the NORMAL INODE directory when USS is enabled,
+         * so filter the .snaps entry if exists.
+         * However it is OK to list .snaps in VIRTUAL world
          */
-        svc_fd = svc_fd_ctx_get_or_new (this,  fd);
-        if (!svc_fd)
-                gf_log (this->name, GF_LOG_ERROR, "failed to get the fd "
-                        "context for the inode %s",
-                        uuid_utoa (fd->inode->gfid));
-        else {
-                if (svc_fd->entry_point_handled && off == svc_fd->last_offset) {
-                        op_ret = 0;
-                        op_errno = ENOENT;
-                        goto out;
-                }
+        if (inode_type == NORMAL_INODE && !strcmp(entry_point, entry->d_name)) {
+            gf_dirent_entry_free(entry);
+            continue;
         }
 
-        SVC_GET_SUBVOL_FROM_CTX (this, op_ret, op_errno, inode_type, ret,
-                                 fd->inode, subvolume, out);
+        if (!entry->inode)
+            continue;
 
-        local->subvolume = subvolume;
-        local->fd = fd_ref (fd);
-        frame->local = local;
+        ret = svc_inode_ctx_set(this, entry->inode, inode_type);
+        if (ret)
+            gf_smsg(this->name, GF_LOG_ERROR, 0,
+                    SVC_MSG_SET_INODE_CONTEXT_FAILED, NULL);
+        if (svc_fd)
+            svc_fd->last_offset = entry->d_off;
+    }
 
-        STACK_WIND (frame, svc_readdirp_cbk, subvolume,
-                    subvolume->fops->readdirp, fd, size, off, xdata);
+    unwind = gf_svc_readdir_on_special_dir(frame, cookie, this, op_ret,
+                                           op_errno, entries, xdata);
 
-        wind = _gf_true;
+out:
+    if (unwind)
+        SVC_STACK_UNWIND(readdirp, frame, op_ret, op_errno, entries, xdata);
+
+    return 0;
+}
+
+static int32_t
+gf_svc_readdirp(call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
+                off_t off, dict_t *xdata)
+{
+    int inode_type = -1;
+    xlator_t *subvolume = NULL;
+    svc_local_t *local = NULL;
+    int ret = -1;
+    int op_ret = -1;
+    int op_errno = EINVAL;
+    gf_boolean_t wind = _gf_false;
+    svc_fd_t *svc_fd = NULL;
+    gf_dirent_t entries;
+
+    INIT_LIST_HEAD(&entries.list);
+
+    GF_VALIDATE_OR_GOTO("svc", this, out);
+    GF_VALIDATE_OR_GOTO(this->name, frame, out);
+    GF_VALIDATE_OR_GOTO(this->name, fd, out);
+    GF_VALIDATE_OR_GOTO(this->name, fd->inode, out);
+
+    local = mem_get0(this->local_pool);
+    if (!local) {
+        op_errno = ENOMEM;
+        gf_smsg(this->name, GF_LOG_ERROR, op_errno, SVC_MSG_NO_MEMORY, NULL);
+        goto out;
+    }
+
+    /*
+     * This is mainly for samba shares (or windows clients). As part of
+     * readdirp on the directory used as samba share, the entry point
+     * directory would have been added at the end. So when a new readdirp
+     * request comes, we have to check if the entry point has been handled
+     * or not in readdirp. That information and the offset used for it
+     * is remembered in fd context. If it has been handled, then simply
+     * unwind indication end of readdir operation.
+     */
+    svc_fd = svc_fd_ctx_get_or_new(this, fd);
+    if (!svc_fd)
+        gf_smsg(this->name, GF_LOG_ERROR, 0, SVC_MSG_GET_FD_CONTEXT_FAILED,
+                "gfid=%s", uuid_utoa(fd->inode->gfid), NULL);
+    else {
+        if (svc_fd->entry_point_handled && off == svc_fd->last_offset) {
+            op_ret = 0;
+            op_errno = ENOENT;
+            goto out;
+        }
+    }
+
+    SVC_GET_SUBVOL_FROM_CTX(this, op_ret, op_errno, inode_type, ret, fd->inode,
+                            subvolume, out);
+
+    local->subvolume = subvolume;
+    local->fd = fd_ref(fd);
+    frame->local = local;
+
+    STACK_WIND(frame, gf_svc_readdirp_cbk, subvolume, subvolume->fops->readdirp,
+               fd, size, off, xdata);
+
+    wind = _gf_true;
 
 out:
-        if (!wind)
-                SVC_STACK_UNWIND (readdirp, frame, op_ret, op_errno, &entries,
-                                  NULL);
+    if (!wind)
+        SVC_STACK_UNWIND(readdirp, frame, op_ret, op_errno, &entries, NULL);
 
-        gf_dirent_free (&entries);
+    gf_dirent_free(&entries);
 
-        return 0;
+    return 0;
 }
 
 /* Renaming the entries from or to snapshots is not allowed as the snapshots
    are read-only.
 */
-int32_t
-svc_rename (call_frame_t *frame, xlator_t *this, loc_t *oldloc,
-            loc_t *newloc, dict_t *xdata)
+static int32_t
+gf_svc_rename(call_frame_t *frame, xlator_t *this, loc_t *oldloc, loc_t *newloc,
+              dict_t *xdata)
 {
-        int          src_inode_type  = -1;
-        int          dst_inode_type  = -1;
-        int          dst_parent_type = -1;
-        int32_t      op_ret          = -1;
-        int32_t      op_errno        = 0;
-        int32_t      ret             = -1;
-        gf_boolean_t wind            = _gf_false;
-
-        GF_VALIDATE_OR_GOTO ("svc", this, out);
-        GF_VALIDATE_OR_GOTO (this->name, frame, out);
-        GF_VALIDATE_OR_GOTO (this->name, oldloc, out);
-        GF_VALIDATE_OR_GOTO (this->name, oldloc->inode, out);
-        GF_VALIDATE_OR_GOTO (this->name, newloc, out);
-
-        ret = svc_inode_ctx_get (this, oldloc->inode, &src_inode_type);
-        if (ret < 0) {
-                gf_log (this->name, GF_LOG_ERROR, "failed to get the inode "
-                        "context for the inode %s",
-                        uuid_utoa (oldloc->inode->gfid));
-                op_ret = -1;
-                op_errno = EINVAL;
-                goto out;
-        }
-
-        if (src_inode_type == VIRTUAL_INODE) {
-                gf_log (this->name, GF_LOG_ERROR, "rename happening on a entry"
-                        " %s residing in snapshot", oldloc->name);
-                op_ret = -1;
-                op_errno = EROFS;
-                goto out;
+    int src_inode_type = -1;
+    int dst_inode_type = -1;
+    int dst_parent_type = -1;
+    int32_t op_ret = -1;
+    int32_t op_errno = 0;
+    int32_t ret = -1;
+    gf_boolean_t wind = _gf_false;
+
+    GF_VALIDATE_OR_GOTO("svc", this, out);
+    GF_VALIDATE_OR_GOTO(this->name, frame, out);
+    GF_VALIDATE_OR_GOTO(this->name, oldloc, out);
+    GF_VALIDATE_OR_GOTO(this->name, oldloc->inode, out);
+    GF_VALIDATE_OR_GOTO(this->name, newloc, out);
+
+    ret = svc_inode_ctx_get(this, oldloc->inode, &src_inode_type);
+    if (ret < 0) {
+        op_ret = -1;
+        op_errno = EINVAL;
+        gf_smsg(this->name, GF_LOG_ERROR, op_errno,
+                SVC_MSG_GET_INODE_CONTEXT_FAILED, "gfid=%s",
+                uuid_utoa(oldloc->inode->gfid), NULL);
+        goto out;
+    }
+
+    if (src_inode_type == VIRTUAL_INODE) {
+        op_ret = -1;
+        op_errno = EROFS;
+        gf_smsg(this->name, GF_LOG_ERROR, op_errno,
+                SVC_MSG_RENAME_SNAPSHOT_ENTRY, "name=%s", oldloc->name, NULL);
+        goto out;
+    }
+
+    if (newloc->inode) {
+        ret = svc_inode_ctx_get(this, newloc->inode, &dst_inode_type);
+        if (!ret && dst_inode_type == VIRTUAL_INODE) {
+            op_ret = -1;
+            op_errno = EROFS;
+            gf_smsg(this->name, GF_LOG_ERROR, op_errno,
+                    SVC_MSG_RENAME_SNAPSHOT_ENTRY, "oldloc-name=%s",
+                    oldloc->name, "newloc-name=%s", newloc->name, NULL);
+            goto out;
+        }
+    }
+
+    if (dst_inode_type < 0) {
+        ret = svc_inode_ctx_get(this, newloc->parent, &dst_parent_type);
+        if (!ret && dst_parent_type == VIRTUAL_INODE) {
+            op_ret = -1;
+            op_errno = EROFS;
+            gf_smsg(this->name, GF_LOG_ERROR, op_errno,
+                    SVC_MSG_RENAME_SNAPSHOT_ENTRY, "oldloc-name=%s",
+                    oldloc->name, "newloc-name=%s", newloc->name, NULL);
+            goto out;
         }
+    }
 
-        if (newloc->inode) {
-                ret = svc_inode_ctx_get (this, newloc->inode, &dst_inode_type);
-                if (!ret && dst_inode_type == VIRTUAL_INODE) {
-                        gf_log (this->name, GF_LOG_ERROR, "rename of %s "
-                                "happening to a entry %s residing in snapshot",
-                                oldloc->name, newloc->name);
-                        op_ret = -1;
-                        op_errno = EROFS;
-                        goto out;
-                }
-        }
+    STACK_WIND_TAIL(frame, FIRST_CHILD(this), FIRST_CHILD(this)->fops->rename,
+                    oldloc, newloc, xdata);
 
-        if (dst_inode_type < 0) {
-                ret = svc_inode_ctx_get (this, newloc->parent,
-                                         &dst_parent_type);
-                if (!ret && dst_parent_type == VIRTUAL_INODE) {
-                        gf_log (this->name, GF_LOG_ERROR, "rename of %s "
-                                "happening to a entry %s residing in snapshot",
-                                oldloc->name, newloc->name);
-                        op_ret = -1;
-                        op_errno = EROFS;
-                        goto out;
-                }
-        }
-
-        STACK_WIND_TAIL (frame, FIRST_CHILD (this),
-                         FIRST_CHILD (this)->fops->rename, oldloc, newloc,
-                         xdata);
-
-        wind = _gf_true;
+    wind = _gf_true;
 
 out:
-        if (!wind)
-                SVC_STACK_UNWIND (rename, frame, op_ret, op_errno, NULL,
-                                  NULL, NULL, NULL, NULL, NULL);
-        return 0;
+    if (!wind)
+        SVC_STACK_UNWIND(rename, frame, op_ret, op_errno, NULL, NULL, NULL,
+                         NULL, NULL, NULL);
+    return 0;
 }
 
 /* Creating hardlinks for the files from the snapshot is not allowed as it
    will be equivalent of creating hardlinks across different filesystems.
-   And so is vise versa.
+   And so is vice versa.
 */
-int32_t
-svc_link (call_frame_t *frame, xlator_t *this, loc_t *oldloc, loc_t *newloc,
-          dict_t *xdata)
+static int32_t
+gf_svc_link(call_frame_t *frame, xlator_t *this, loc_t *oldloc, loc_t *newloc,
+            dict_t *xdata)
 {
-        int          src_inode_type  = -1;
-        int          dst_parent_type = -1;
-        int32_t      op_ret          = -1;
-        int32_t      op_errno        = 0;
-        int32_t      ret             = -1;
-        gf_boolean_t wind            = _gf_false;
-
-        GF_VALIDATE_OR_GOTO ("svc", this, out);
-        GF_VALIDATE_OR_GOTO (this->name, frame, out);
-        GF_VALIDATE_OR_GOTO (this->name, oldloc, out);
-        GF_VALIDATE_OR_GOTO (this->name, oldloc->inode, out);
-        GF_VALIDATE_OR_GOTO (this->name, newloc, out);
-
-        ret = svc_inode_ctx_get (this, oldloc->inode, &src_inode_type);
-        if (!ret && src_inode_type == VIRTUAL_INODE) {
-                gf_log (this->name, GF_LOG_ERROR, "rename happening on a entry"
-                        " %s residing in snapshot", oldloc->name);
-                op_ret = -1;
-                op_errno = EROFS;
-                goto out;
-        }
-
-        ret = svc_inode_ctx_get (this, newloc->parent, &dst_parent_type);
-        if (!ret && dst_parent_type == VIRTUAL_INODE) {
-                gf_log (this->name, GF_LOG_ERROR, "rename of %s "
-                        "happening to a entry %s residing in snapshot",
-                        oldloc->name, newloc->name);
-                op_ret = -1;
-                op_errno = EROFS;
-                goto out;
-        }
-
-        STACK_WIND_TAIL (frame, FIRST_CHILD (this),
-                         FIRST_CHILD (this)->fops->link, oldloc, newloc, xdata);
-
-        wind = _gf_true;
+    int src_inode_type = -1;
+    int dst_parent_type = -1;
+    int32_t op_ret = -1;
+    int32_t op_errno = 0;
+    int32_t ret = -1;
+    gf_boolean_t wind = _gf_false;
+
+    GF_VALIDATE_OR_GOTO("svc", this, out);
+    GF_VALIDATE_OR_GOTO(this->name, frame, out);
+    GF_VALIDATE_OR_GOTO(this->name, oldloc, out);
+    GF_VALIDATE_OR_GOTO(this->name, oldloc->inode, out);
+    GF_VALIDATE_OR_GOTO(this->name, newloc, out);
+
+    ret = svc_inode_ctx_get(this, oldloc->inode, &src_inode_type);
+    if (!ret && src_inode_type == VIRTUAL_INODE) {
+        op_ret = -1;
+        op_errno = EROFS;
+        gf_smsg(this->name, GF_LOG_ERROR, op_errno, SVC_MSG_LINK_SNAPSHOT_ENTRY,
+                "oldloc-name=%s", oldloc->name, NULL);
+        goto out;
+    }
+
+    ret = svc_inode_ctx_get(this, newloc->parent, &dst_parent_type);
+    if (!ret && dst_parent_type == VIRTUAL_INODE) {
+        op_ret = -1;
+        op_errno = EROFS;
+        gf_smsg(this->name, GF_LOG_ERROR, op_errno, SVC_MSG_LINK_SNAPSHOT_ENTRY,
+                "oldloc-name=%s", oldloc->name, "newloc-name=%s", newloc->name,
+                NULL);
+        goto out;
+    }
+
+    STACK_WIND_TAIL(frame, FIRST_CHILD(this), FIRST_CHILD(this)->fops->link,
+                    oldloc, newloc, xdata);
+
+    wind = _gf_true;
 
 out:
-        if (!wind)
-                SVC_STACK_UNWIND (link, frame, op_ret, op_errno,
-                                  NULL, NULL, NULL, NULL, NULL);
-        return 0;
+    if (!wind)
+        SVC_STACK_UNWIND(link, frame, op_ret, op_errno, NULL, NULL, NULL, NULL,
+                         NULL);
+    return 0;
 }
 
-int32_t
-svc_removexattr (call_frame_t *frame, xlator_t *this, loc_t *loc,
-                 const char *name, dict_t *xdata)
+static int32_t
+gf_svc_removexattr(call_frame_t *frame, xlator_t *this, loc_t *loc,
+                   const char *name, dict_t *xdata)
 {
-        int          ret        = -1;
-        int          inode_type = -1;
-        int          op_ret     = -1;
-        int          op_errno   = EINVAL;
-        gf_boolean_t wind       = _gf_false;
+    int ret = -1;
+    int inode_type = -1;
+    int op_ret = -1;
+    int op_errno = EINVAL;
+    gf_boolean_t wind = _gf_false;
+
+    GF_VALIDATE_OR_GOTO("svc", this, out);
+    GF_VALIDATE_OR_GOTO(this->name, frame, out);
+    GF_VALIDATE_OR_GOTO(this->name, loc, out);
+    GF_VALIDATE_OR_GOTO(this->name, loc->inode, out);
+
+    ret = svc_inode_ctx_get(this, loc->inode, &inode_type);
+    if (ret < 0) {
+        op_ret = -1;
+        op_errno = EINVAL;
+        gf_smsg(this->name, GF_LOG_ERROR, op_errno,
+                SVC_MSG_GET_INODE_CONTEXT_FAILED, "path=%s", loc->path,
+                "gfid=%s", uuid_utoa(loc->inode->gfid), NULL);
+        goto out;
+    }
+
+    if (inode_type == NORMAL_INODE) {
+        STACK_WIND_TAIL(frame, FIRST_CHILD(this),
+                        FIRST_CHILD(this)->fops->removexattr, loc, name, xdata);
+    } else {
+        op_ret = -1;
+        op_errno = EROFS;
+        goto out;
+    }
+
+    wind = _gf_true;
 
-        GF_VALIDATE_OR_GOTO ("svc", this, out);
-        GF_VALIDATE_OR_GOTO (this->name, frame, out);
-        GF_VALIDATE_OR_GOTO (this->name, loc, out);
-        GF_VALIDATE_OR_GOTO (this->name, loc->inode, out);
-
-        ret = svc_inode_ctx_get (this, loc->inode, &inode_type);
-        if (ret < 0) {
-                gf_log (this->name, GF_LOG_ERROR, "failed to get te inode "
-                        "context for %s (gfid: %s)", loc->path,
-                        uuid_utoa (loc->inode->gfid));
-                op_ret = -1;
-                op_errno = EINVAL;
-                goto out;
-        }
+out:
+    if (!wind)
+        SVC_STACK_UNWIND(removexattr, frame, op_ret, op_errno, NULL);
 
-        if (inode_type == NORMAL_INODE) {
-                STACK_WIND_TAIL (frame, FIRST_CHILD (this),
-                                 FIRST_CHILD (this)->fops->removexattr, loc,
-                                 name, xdata);
-        } else {
-                op_ret = -1;
-                op_errno = EROFS;
-                goto out;
-        }
+    return 0;
+}
 
-        wind = _gf_true;
+static int
+gf_svc_fsync(call_frame_t *frame, xlator_t *this, fd_t *fd, int datasync,
+             dict_t *xdata)
+{
+    int inode_type = -1;
+    int ret = -1;
+    int op_ret = -1;
+    int op_errno = EINVAL;
+    gf_boolean_t wind = _gf_false;
+
+    GF_VALIDATE_OR_GOTO("svc", this, out);
+    GF_VALIDATE_OR_GOTO(this->name, frame, out);
+    GF_VALIDATE_OR_GOTO(this->name, fd, out);
+    GF_VALIDATE_OR_GOTO(this->name, fd->inode, out);
+
+    ret = svc_inode_ctx_get(this, fd->inode, &inode_type);
+    if (ret < 0) {
+        op_ret = -1;
+        op_errno = EINVAL;
+        gf_smsg(this->name, GF_LOG_ERROR, op_errno,
+                SVC_MSG_GET_INODE_CONTEXT_FAILED, "gfid=%s",
+                uuid_utoa(fd->inode->gfid), NULL);
+        goto out;
+    }
+
+    if (inode_type == NORMAL_INODE) {
+        STACK_WIND_TAIL(frame, FIRST_CHILD(this),
+                        FIRST_CHILD(this)->fops->fsync, fd, datasync, xdata);
+    } else {
+        op_ret = -1;
+        op_errno = EROFS;
+        goto out;
+    }
+
+    wind = _gf_true;
 
 out:
-        if (!wind)
-                SVC_STACK_UNWIND (removexattr, frame, op_ret, op_errno,
-                                  NULL);
+    if (!wind)
+        SVC_STACK_UNWIND(fsync, frame, op_ret, op_errno, NULL, NULL, NULL);
 
-        return 0;
+    return 0;
 }
 
-int
-svc_fsync (call_frame_t *frame, xlator_t *this, fd_t *fd, int datasync,
-           dict_t *xdata)
+static int32_t
+gf_svc_flush(call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *xdata)
 {
-        int             inode_type = -1;
-        int             ret        = -1;
-        int             op_ret     = -1;
-        int             op_errno   = EINVAL;
-        gf_boolean_t    wind       = _gf_false;
+    int32_t op_ret = -1;
+    int32_t op_errno = 0;
+    int ret = -1;
+    int inode_type = -1;
+    xlator_t *subvolume = NULL;
+    gf_boolean_t wind = _gf_false;
 
-        GF_VALIDATE_OR_GOTO ("svc", this, out);
-        GF_VALIDATE_OR_GOTO (this->name, frame, out);
-        GF_VALIDATE_OR_GOTO (this->name, fd, out);
-        GF_VALIDATE_OR_GOTO (this->name, fd->inode, out);
+    GF_VALIDATE_OR_GOTO("svc", this, out);
+    GF_VALIDATE_OR_GOTO(this->name, frame, out);
+    GF_VALIDATE_OR_GOTO(this->name, fd, out);
+    GF_VALIDATE_OR_GOTO(this->name, fd->inode, out);
 
-        ret = svc_inode_ctx_get (this, fd->inode, &inode_type);
-        if (ret < 0) {
-                gf_log (this->name, GF_LOG_ERROR, "failed to get inode context "
-                        "for %s", uuid_utoa (fd->inode->gfid));
-                op_ret = -1;
-                op_errno = EINVAL;
-                goto out;
-        }
+    SVC_GET_SUBVOL_FROM_CTX(this, op_ret, op_errno, inode_type, ret, fd->inode,
+                            subvolume, out);
 
-        if (inode_type == NORMAL_INODE) {
-                STACK_WIND_TAIL (frame, FIRST_CHILD (this),
-                                 FIRST_CHILD (this)->fops->fsync, fd, datasync,
-                                 xdata);
-        } else {
-                op_ret = -1;
-                op_errno = EROFS;
-                goto out;
-        }
+    STACK_WIND_TAIL(frame, subvolume, subvolume->fops->flush, fd, xdata);
 
-        wind = _gf_true;
+    wind = _gf_true;
 
 out:
-        if (!wind)
-                SVC_STACK_UNWIND (fsync, frame, op_ret, op_errno, NULL, NULL,
-                                  NULL);
+    if (!wind)
+        SVC_STACK_UNWIND(flush, frame, op_ret, op_errno, NULL);
 
-        return 0;
+    return 0;
 }
 
-int32_t
-svc_flush (call_frame_t *frame, xlator_t *this,
-           fd_t *fd, dict_t *xdata)
+static int32_t
+gf_svc_releasedir(xlator_t *this, fd_t *fd)
 {
-        int32_t           op_ret     = -1;
-        int32_t           op_errno   = 0;
-        int               ret        = -1;
-        int               inode_type = -1;
-        xlator_t         *subvolume  = NULL;
-        gf_boolean_t      wind       = _gf_false;
-
-        GF_VALIDATE_OR_GOTO ("svc", this, out);
-        GF_VALIDATE_OR_GOTO (this->name, frame, out);
-        GF_VALIDATE_OR_GOTO (this->name, fd, out);
-        GF_VALIDATE_OR_GOTO (this->name, fd->inode, out);
+    svc_fd_t *sfd = NULL;
+    uint64_t tmp_pfd = 0;
+    int ret = 0;
 
-        SVC_GET_SUBVOL_FROM_CTX (this, op_ret, op_errno, inode_type, ret,
-                                 fd->inode, subvolume, out);
+    GF_VALIDATE_OR_GOTO("snapview-client", this, out);
+    GF_VALIDATE_OR_GOTO(this->name, fd, out);
 
-        STACK_WIND_TAIL (frame, subvolume, subvolume->fops->flush, fd, xdata);
+    ret = fd_ctx_del(fd, this, &tmp_pfd);
+    if (ret < 0) {
+        gf_msg_debug(this->name, 0, "pfd from fd=%p is NULL", fd);
+        goto out;
+    }
 
-        wind = _gf_true;
+    GF_FREE(sfd);
 
 out:
-        if (!wind)
-                SVC_STACK_UNWIND (flush, frame, op_ret, op_errno, NULL);
-
-        return 0;
+    return 0;
 }
 
-int32_t
-svc_releasedir (xlator_t *this, fd_t *fd)
+static int32_t
+gf_svc_forget(xlator_t *this, inode_t *inode)
 {
-        svc_fd_t *sfd      = NULL;
-        uint64_t          tmp_pfd  = 0;
-        int               ret      = 0;
-
-        GF_VALIDATE_OR_GOTO ("snapview-client", this, out);
-        GF_VALIDATE_OR_GOTO (this->name, fd, out);
+    int ret = -1;
+    uint64_t value = 0;
 
-        ret = fd_ctx_del (fd, this, &tmp_pfd);
-        if (ret < 0) {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "pfd from fd=%p is NULL", fd);
-                goto out;
-        }
+    GF_VALIDATE_OR_GOTO("svc", this, out);
+    GF_VALIDATE_OR_GOTO(this->name, inode, out);
 
-        GF_FREE (sfd);
+    ret = inode_ctx_del(inode, this, &value);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_ERROR, 0,
+                SVC_MSG_DELETE_INODE_CONTEXT_FAILED, "gfid=%s",
+                uuid_utoa(inode->gfid), NULL);
+        goto out;
+    }
 
 out:
-        return 0;
+    return 0;
 }
 
-int32_t
-svc_forget (xlator_t *this, inode_t *inode)
+static int
+gf_svc_priv_destroy(xlator_t *this, svc_private_t *priv)
 {
-        int            ret      = -1;
-        uint64_t       value    = 0;
+    int ret = -1;
 
-        GF_VALIDATE_OR_GOTO ("svc", this, out);
-        GF_VALIDATE_OR_GOTO (this->name, inode, out);
+    if (!priv) {
+        gf_smsg(this->name, GF_LOG_WARNING, 0, SVC_MSG_NULL_PRIV, NULL);
+        goto out;
+    }
 
-        ret = inode_ctx_del (inode, this, &value);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "failed to delete inode "
-                        "context for %s", uuid_utoa (inode->gfid));
-                goto out;
-        }
+    GF_FREE(priv->path);
+    GF_FREE(priv->special_dir);
+
+    LOCK_DESTROY(&priv->lock);
+
+    GF_FREE(priv);
+
+    if (this->local_pool) {
+        mem_pool_destroy(this->local_pool);
+        this->local_pool = NULL;
+    }
+
+    ret = 0;
 
 out:
-        return 0;
+    return ret;
 }
 
+/**
+ * ** NOTE **:
+ * =============
+ * The option "snapdir-entry-path" is NOT reconfigurable.
+ * That option as of now is only for the consumption of
+ * samba, where, it needs to tell glusterfs about the
+ * directory that is shared with windows client for the
+ * access. Now, in windows-explorer (GUI) interface, for
+ * the directory shared, the entry point to the snapshot
+ * world (snapshot-directory option) should be visible,
+ * atleast as a hidden entry. For that to happen, glusterfs
+ * has to send that entry in the readdir response coming on
+ * the directory used as the smb share. Therefore, samba,
+ * while initializing the gluster volume (via gfapi) sets
+ * the xlator option "snapdir-entry-path" to the directory
+ * which is to be shared with windows (check the file
+ * vfs_glusterfs.c from samba source code). So to avoid
+ * problems with smb access, not allowing snapdir-entry-path
+ * option to be configurable. That option is for those
+ * consumers who know what they are doing.
+ **/
 int
-reconfigure (xlator_t *this, dict_t *options)
+reconfigure(xlator_t *this, dict_t *options)
 {
-        svc_private_t  *priv = NULL;
-
-        priv = this->private;
+    svc_private_t *priv = NULL;
+    char *path = NULL;
+    gf_boolean_t show_entry_point = _gf_false;
+    char *tmp = NULL;
+
+    priv = this->private;
+
+    GF_OPTION_RECONF("snapshot-directory", path, options, str, out);
+    if (!path || (strlen(path) > NAME_MAX) || path[0] != '.') {
+        gf_smsg(this->name, GF_LOG_ERROR, 0, SVC_MSG_INVALID_ENTRY_POINT,
+                "path=%s", path, NULL);
+        goto out;
+    }
+
+    GF_OPTION_RECONF("show-snapshot-directory", show_entry_point, options, bool,
+                     out);
+
+    /*
+     * The assumption now is that priv->path is an allocated memory (either
+     * in init or in a previous reconfigure).
+     * So, the intention here is to preserve the older contents of the option
+     * until the new option's value has been completely stored in the priv.
+     * So, do this.
+     *  - Store the pointer of priv->path in a temporary pointer.
+     *  - Allocate new memory for the new value of the option that is just
+     *    obtained from the above call to GF_OPTION_RECONF.
+     *  - If the above allocation fails, again set the pointer from priv
+     *    to the address stored in tmp. i.e. the previous value.
+     *  - If the allocation succeeds, then free the tmp pointer.
+     * WARNING: Before changing the allocation and freeing logic of
+     *          priv->path, always check the init function to see how
+     *          priv->path is set. Take decisions accordingly. As of now,
+     *          the assumption is that, the string elements of private
+     *          structure of snapview-client are allocated (either in
+     *          init or here in reconfugure).
+     */
+    LOCK(&priv->lock);
+    {
+        tmp = priv->path;
+        priv->path = NULL;
+        priv->path = gf_strdup(path);
+        if (!priv->path) {
+            gf_log(this->name, GF_LOG_ERROR,
+                   "failed to reconfigure snapshot-directory option to %s",
+                   path);
+            priv->path = tmp;
+        } else {
+            GF_FREE(tmp);
+            tmp = NULL;
+        }
 
-        GF_OPTION_RECONF ("snapshot-directory", priv->path, options, str, out);
-        GF_OPTION_RECONF ("show-snapshot-directory", priv->show_entry_point,
-                          options, bool, out);
+        priv->show_entry_point = show_entry_point;
+    }
+    UNLOCK(&priv->lock);
 
 out:
-        return 0;
+    return 0;
 }
 
 int32_t
-mem_acct_init (xlator_t *this)
+mem_acct_init(xlator_t *this)
 {
-        int32_t     ret = -1;
+    int32_t ret = -1;
 
-        if (!this)
-                return ret;
+    if (!this)
+        return ret;
 
-        ret = xlator_mem_acct_init (this, gf_svc_mt_end + 1);
+    ret = xlator_mem_acct_init(this, gf_svc_mt_end + 1);
 
-        if (ret != 0) {
-                gf_log (this->name, GF_LOG_WARNING, "Memory accounting"
-                        " init failed");
-                return ret;
-        }
+    if (ret != 0) {
+        gf_smsg(this->name, GF_LOG_WARNING, 0, SVC_MSG_MEM_ACNT_FAILED, NULL);
+    }
 
-        return ret;
+    return ret;
 }
 
 int32_t
-init (xlator_t *this)
+init(xlator_t *this)
 {
-        svc_private_t *private  = NULL;
-        int            ret      = -1;
-        int            children = 0;
-        xlator_list_t *xl       = NULL;
-
-        if (!this->children) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "configured without any child");
-                goto out;
-        }
-
-        xl = this->children;
-        while (xl) {
-                children++;
-                xl = xl->next;
-        }
+    svc_private_t *private = NULL;
+    int ret = -1;
+    int children = 0;
+    xlator_list_t *xl = NULL;
+    char *path = NULL;
+    char *special_dir = NULL;
+
+    if (!this->children) {
+        gf_smsg(this->name, GF_LOG_ERROR, 0, SVC_MSG_NO_CHILD_FOR_XLATOR, NULL);
+        goto out;
+    }
+
+    xl = this->children;
+    while (xl) {
+        children++;
+        xl = xl->next;
+    }
+
+    if (children != 2) {
+        gf_smsg(this->name, GF_LOG_ERROR, 0, SVC_MSG_XLATOR_CHILDREN_WRONG,
+                "subvol-num=%d", children, NULL);
+        goto out;
+    }
+
+    /* This can be the top of graph in certain cases */
+    if (!this->parents) {
+        gf_msg_debug(this->name, 0,
+                     "dangling volume. Check "
+                     "volfile");
+    }
+
+   private
+    = GF_CALLOC(1, sizeof(*private), gf_svc_mt_svc_private_t);
+    if (!private)
+        goto out;
+
+    LOCK_INIT(&private->lock);
+
+    GF_OPTION_INIT("snapshot-directory", path, str, out);
+    if (!path || (strlen(path) > NAME_MAX) || path[0] != '.') {
+        gf_smsg(this->name, GF_LOG_ERROR, 0, SVC_MSG_INVALID_ENTRY_POINT,
+                "path=%s", path, NULL);
+        goto out;
+    }
+
+   private
+    ->path = gf_strdup(path);
+    if (!private->path) {
+        gf_smsg(this->name, GF_LOG_ERROR, 0, SVC_MSG_NO_MEMORY,
+                "entry-point-path=%s", path, NULL);
+        goto out;
+    }
+
+    GF_OPTION_INIT("snapdir-entry-path", special_dir, str, out);
+    if (!special_dir || strstr(special_dir, path)) {
+        if (special_dir)
+            gf_smsg(this->name, GF_LOG_ERROR, 0,
+                    SVC_MSG_ENTRY_POINT_SPECIAL_DIR, "path=%s", path,
+                    "special-dir=%s", special_dir);
+        else
+            gf_smsg(this->name, GF_LOG_ERROR, 0, SVC_MSG_NULL_SPECIAL_DIR,
+                    NULL);
+        goto out;
+    }
 
-        if (children != 2) {
-                gf_log (this->name, GF_LOG_ERROR, "snap-view-client has got "
-                        "%d subvolumes. It can have only 2 subvolumes.",
-                        children);
-                goto out;
-        }
+   private
+    ->special_dir = gf_strdup(special_dir);
+    if (!private->special_dir) {
+        gf_smsg(this->name, GF_LOG_ERROR, 0, SVC_MSG_NO_MEMORY,
+                "special-directory=%s", special_dir, NULL);
+        goto out;
+    }
 
-        /* This can be the top of graph in certain cases */
-        if (!this->parents) {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "dangling volume. check volfile ");
-        }
+    GF_OPTION_INIT("show-snapshot-directory", private->show_entry_point, bool,
+                   out);
 
-        private = GF_CALLOC (1, sizeof (*private), gf_svc_mt_svc_private_t);
-        if (!private)
-                goto out;
+    this->local_pool = mem_pool_new(svc_local_t, 128);
+    if (!this->local_pool) {
+        gf_smsg(this->name, GF_LOG_ERROR, 0, SVC_MSG_MEM_POOL_GET_FAILED, NULL);
+        goto out;
+    }
 
-        GF_OPTION_INIT ("snapshot-directory", private->path, str, out);
-        GF_OPTION_INIT ("snapdir-entry-path", private->special_dir, str,
-                        out);
-        GF_OPTION_INIT ("show-snapshot-directory", private->show_entry_point,
-                        bool, out);
-
-        if (strstr (private->special_dir, private->path)) {
-                gf_log (this->name, GF_LOG_ERROR, "entry point directory "
-                        "cannot be part of the special directory");
-                GF_FREE (private->special_dir);
-                private->special_dir = NULL;
-                goto out;
-        }
-
-        this->private = private;
-        this->local_pool = mem_pool_new (svc_local_t, 128);
-        if (!this->local_pool) {
-                gf_log (this->name, GF_LOG_ERROR, "could not get mem pool for "
-                        "frame->local");
-                goto out;
-        }
+    this->private = private;
 
-        ret = 0;
+    ret = 0;
 
 out:
-        if (ret)
-                GF_FREE (private);
+    if (ret)
+        (void)gf_svc_priv_destroy(this, private);
 
-        return ret;
+    return ret;
 }
 
 void
-fini (xlator_t *this)
+fini(xlator_t *this)
 {
-        svc_private_t *priv = NULL;
+    svc_private_t *priv = NULL;
 
-        if (!this)
-                return;
+    if (!this)
+        return;
 
-        priv = this->private;
-        if (!priv)
-                return;
+    priv = this->private;
+    if (!priv)
+        return;
 
-        this->private = NULL;
+    /*
+     * Just log the failure and go ahead to
+     * set this->priv to NULL.
+     */
+    if (gf_svc_priv_destroy(this, priv))
+        gf_smsg(this->name, GF_LOG_WARNING, 0, SVC_MSG_PRIV_DESTROY_FAILED,
+                NULL);
 
-        GF_FREE (priv);
+    this->private = NULL;
 
-        return;
+    return;
 }
 
 int
-notify (xlator_t *this, int event, void *data, ...)
+notify(xlator_t *this, int event, void *data, ...)
 {
-        xlator_t       *subvol     = NULL;
-        int             ret        = 0;
-
-        subvol = data;
-
-        /* As there are two subvolumes in snapview-client, there is
-         * a possibility that the regular subvolume is still down and
-         * snapd subvolume come up first. So if we don't handle this situation
-         * CHILD_UP event will be propagated upwards to fuse when
-         * regular subvolume is still down.
-         * This can cause data unavailable for the application.
-         * So for now send notifications up only for regular subvolume.
-         *
-         * TODO: In future if required we may need to handle
-         * notifications from virtual subvolume
-         */
-        if (subvol != SECOND_CHILD (this))
-                ret = default_notify (this, event, data);
-
-        return ret;
+    xlator_t *subvol = NULL;
+    int ret = 0;
+
+    subvol = data;
+
+    /* As there are two subvolumes in snapview-client, there is
+     * a possibility that the regular subvolume is still down and
+     * snapd subvolume come up first. So if we don't handle this situation
+     * CHILD_UP event will be propagated upwards to fuse when
+     * regular subvolume is still down.
+     * This can cause data unavailable for the application.
+     * So for now send notifications up only for regular subvolume.
+     *
+     * TODO: In future if required we may need to handle
+     * notifications from virtual subvolume
+     */
+    if (subvol != SECOND_CHILD(this))
+        ret = default_notify(this, event, data);
+
+    return ret;
 }
 
 struct xlator_fops fops = {
-        .lookup        = svc_lookup,
-        .opendir       = svc_opendir,
-        .stat          = svc_stat,
-        .fstat         = svc_fstat,
-        .rmdir         = svc_rmdir,
-        .rename        = svc_rename,
-        .mkdir         = svc_mkdir,
-        .open          = svc_open,
-        .unlink        = svc_unlink,
-        .setattr       = svc_setattr,
-        .getxattr      = svc_getxattr,
-        .setxattr      = svc_setxattr,
-        .fsetxattr     = svc_fsetxattr,
-        .readv         = svc_readv,
-        .readdir       = svc_readdir,
-        .readdirp      = svc_readdirp,
-        .create        = svc_create,
-        .readlink      = svc_readlink,
-        .mknod         = svc_mknod,
-        .symlink       = svc_symlink,
-        .flush         = svc_flush,
-        .link          = svc_link,
-        .access        = svc_access,
-        .removexattr   = svc_removexattr,
-        .fsync         = svc_fsync,
+    .lookup = gf_svc_lookup,
+    .opendir = gf_svc_opendir,
+    .stat = gf_svc_stat,
+    .fstat = gf_svc_fstat,
+    .statfs = gf_svc_statfs,
+    .rmdir = gf_svc_rmdir,
+    .rename = gf_svc_rename,
+    .mkdir = gf_svc_mkdir,
+    .open = gf_svc_open,
+    .unlink = gf_svc_unlink,
+    .setattr = gf_svc_setattr,
+    .getxattr = gf_svc_getxattr,
+    .setxattr = gf_svc_setxattr,
+    .fsetxattr = gf_svc_fsetxattr,
+    .readv = gf_svc_readv,
+    .readdir = gf_svc_readdir,
+    .readdirp = gf_svc_readdirp,
+    .create = gf_svc_create,
+    .readlink = gf_svc_readlink,
+    .mknod = gf_svc_mknod,
+    .symlink = gf_svc_symlink,
+    .flush = gf_svc_flush,
+    .link = gf_svc_link,
+    .access = gf_svc_access,
+    .removexattr = gf_svc_removexattr,
+    .fsync = gf_svc_fsync,
 };
 
 struct xlator_cbks cbks = {
-        .forget = svc_forget,
-        .releasedir = svc_releasedir,
+    .forget = gf_svc_forget,
+    .releasedir = gf_svc_releasedir,
 };
 
 struct volume_options options[] = {
-        { .key = {"snapshot-directory"},
-          .type = GF_OPTION_TYPE_STR,
-          .default_value = ".snaps",
-        },
-        { .key = {"snapdir-entry-path"},
-          .type = GF_OPTION_TYPE_STR,
-          .description = "An option to set the path of a directory on which "
-                         "when readdir comes, dentry for the snapshot-directory"
-                         " should be created and added in the readdir response",
-          .default_value = "",
-        },
-        { .key = {"show-snapshot-directory"},
-          .type = GF_OPTION_TYPE_BOOL,
-          .description = "If this option is set, and the option "
-                         "\"snapdir-entry-path\" is set (which is set by samba "
-                         "vfs plugin for glusterfs, then send the entry point "
-                         "when readdir comes on the snapdir-entry-path",
-          .default_value = "off",
-        },
-        { .key  = {NULL} },
+    {
+        .key = {"snapshot-directory"},
+        .type = GF_OPTION_TYPE_STR,
+        .default_value = ".snaps",
+    },
+    {
+        .key = {"snapdir-entry-path"},
+        .type = GF_OPTION_TYPE_STR,
+        .description = "An option to set the path of a directory on which "
+                       "when readdir comes, dentry for the snapshot-directory"
+                       " should be created and added in the readdir response",
+        .default_value = "",
+    },
+    {
+        .key = {"show-snapshot-directory"},
+        .type = GF_OPTION_TYPE_BOOL,
+        .description = "If this option is set, and the option "
+                       "\"snapdir-entry-path\" is set (which is set by samba "
+                       "vfs plugin for glusterfs, then send the entry point "
+                       "when readdir comes on the snapdir-entry-path",
+        .default_value = "off",
+    },
+    {.key = {NULL}},
+};
+
+xlator_api_t xlator_api = {
+    .init = init,
+    .fini = fini,
+    .notify = notify,
+    .reconfigure = reconfigure,
+    .mem_acct_init = mem_acct_init,
+    .op_version = {1},
+    .fops = &fops,
+    .cbks = &cbks,
+    .options = options,
+    .identifier = "snapview-client",
+    .category = GF_MAINTAINED,
 };
diff --git a/xlators/features/snapview-client/src/snapview-client.h b/xlators/features/snapview-client/src/snapview-client.h
index 9973458884b..166116a439d 100644
--- a/xlators/features/snapview-client/src/snapview-client.h
+++ b/xlators/features/snapview-client/src/snapview-client.h
@@ -1,126 +1,101 @@
- /*
-   Copyright (c) 2014 Red Hat, Inc. <http://www.redhat.com>
-   This file is part of GlusterFS.
-
-   This file is licensed to you under your choice of the GNU Lesser
-   General Public License, version 3 or any later version (LGPLv3 or
-   later), or the GNU General Public License, version 2 (GPLv2), in all
-   cases as published by the Free Software Foundation.
+/*
+  Copyright (c) 2014 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 #ifndef __SNAP_VIEW_CLIENT_H__
 #define __SNAP_VIEW_CLIENT_H__
 
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
-
-#include "glusterfs.h"
-#include "logging.h"
-#include "dict.h"
-#include "xlator.h"
-#include "defaults.h"
+#include <glusterfs/glusterfs.h>
+#include <glusterfs/logging.h>
+#include <glusterfs/dict.h>
+#include <glusterfs/xlator.h>
+#include <glusterfs/defaults.h>
 #include "snapview-client-mem-types.h"
+#include "snapview-client-messages.h"
 
 struct __svc_local {
-        loc_t loc;
-        xlator_t *subvolume;
-        fd_t     *fd;
-        void *cookie;
-        dict_t *xdata;
+    loc_t loc;
+    xlator_t *subvolume;
+    fd_t *fd;
+    void *cookie;
+    dict_t *xdata;
+    uint16_t revalidate;
 };
 typedef struct __svc_local svc_local_t;
 
-void
-svc_local_free (svc_local_t *local);
-
-#define SVC_STACK_UNWIND(fop, frame, params ...) do {           \
-                svc_local_t *__local = NULL;                    \
-                if (frame) {                                    \
-                        __local      = frame->local;            \
-                        frame->local = NULL;                    \
-                }                                               \
-                STACK_UNWIND_STRICT (fop, frame, params);       \
-                svc_local_free (__local);                       \
-        } while (0)
-
-#define SVC_ENTRY_POINT_SET(this, xdata, op_ret, op_errno, new_xdata,   \
-                            priv, ret, label)                           \
-        do {                                                            \
-                if (!xdata) {                                           \
-                        xdata = new_xdata = dict_new ();                \
-                        if (!new_xdata) {                               \
-                                gf_log (this->name, GF_LOG_ERROR,       \
-                                        "failed to allocate new dict"); \
-                                op_ret = -1;                            \
-                                op_errno = ENOMEM;                      \
-                                goto label;                             \
-                        }                                               \
-                }                                                       \
-                ret = dict_set_str (xdata, "entry-point", "true");      \
-                if (ret) {                                              \
-                        gf_log (this->name, GF_LOG_ERROR,               \
-                                "failed to set dict");                  \
-                        op_ret = -1;                                    \
-                        op_errno = ENOMEM;                              \
-                        goto label;                                     \
-                }                                                       \
-        } while (0);
-
-#define SVC_GET_SUBVOL_FROM_CTX(this, op_ret, op_errno, inode_type, ret, \
-                                inode, subvolume, label)                \
-        do {                                                            \
-                ret = svc_inode_ctx_get (this, inode, &inode_type);     \
-                if (ret < 0) {                                          \
-                        gf_log (this->name, GF_LOG_ERROR,               \
-                                "inode context not found for gfid %s",  \
-                                uuid_utoa (inode->gfid));               \
-                        op_ret = -1;                                    \
-                        op_errno = EINVAL;                              \
-                        goto label;                                     \
-                }                                                       \
-                                                                        \
-                subvolume = svc_get_subvolume (this, inode_type);       \
-        }  while (0);
+#define SVC_STACK_UNWIND(fop, frame, params...)                                \
+    do {                                                                       \
+        svc_local_t *__local = NULL;                                           \
+        if (frame) {                                                           \
+            __local = frame->local;                                            \
+            frame->local = NULL;                                               \
+        }                                                                      \
+        STACK_UNWIND_STRICT(fop, frame, params);                               \
+        svc_local_free(__local);                                               \
+    } while (0)
+
+#define SVC_ENTRY_POINT_SET(this, xdata, op_ret, op_errno, new_xdata, ret,     \
+                            label)                                             \
+    do {                                                                       \
+        if (!xdata) {                                                          \
+            xdata = new_xdata = dict_new();                                    \
+            if (!new_xdata) {                                                  \
+                gf_log(this->name, GF_LOG_ERROR,                               \
+                       "failed to allocate new dict");                         \
+                op_ret = -1;                                                   \
+                op_errno = ENOMEM;                                             \
+                goto label;                                                    \
+            }                                                                  \
+        }                                                                      \
+        ret = dict_set_str(xdata, "entry-point", "true");                      \
+        if (ret) {                                                             \
+            gf_log(this->name, GF_LOG_ERROR, "failed to set dict");            \
+            op_ret = -1;                                                       \
+            op_errno = ENOMEM;                                                 \
+            goto label;                                                        \
+        }                                                                      \
+    } while (0);
+
+#define SVC_GET_SUBVOL_FROM_CTX(this, op_ret, op_errno, inode_type, ret,       \
+                                inode, subvolume, label)                       \
+    do {                                                                       \
+        ret = svc_inode_ctx_get(this, inode, &inode_type);                     \
+        if (ret < 0) {                                                         \
+            gf_log(this->name, GF_LOG_ERROR,                                   \
+                   "inode context not found for gfid %s",                      \
+                   uuid_utoa(inode->gfid));                                    \
+            op_ret = -1;                                                       \
+            op_errno = EINVAL;                                                 \
+            goto label;                                                        \
+        }                                                                      \
+                                                                               \
+        subvolume = svc_get_subvolume(this, inode_type);                       \
+    } while (0);
 
 struct svc_private {
-        char *path;
-        char *special_dir; /* needed for samba */
-        gf_boolean_t show_entry_point;
+    char *path;
+    char *special_dir; /* needed for samba */
+    gf_boolean_t show_entry_point;
+    gf_lock_t lock; /* mainly to guard private->path */
 };
 typedef struct svc_private svc_private_t;
 
 struct svc_fd {
-        off_t last_offset;
-        gf_boolean_t entry_point_handled;
-        gf_boolean_t special_dir;
+    off_t last_offset;
+    gf_boolean_t entry_point_handled;
+    gf_boolean_t special_dir;
 };
 typedef struct svc_fd svc_fd_t;
 
-typedef enum {
-        NORMAL_INODE = 1,
-        VIRTUAL_INODE
-} inode_type_t;
-
-void svc_local_free (svc_local_t *local);
-
-xlator_t *
-svc_get_subvolume (xlator_t *this, int inode_type);
-
-int
-__svc_inode_ctx_get (xlator_t *this, inode_t *inode, int *inode_type);
+typedef enum { NORMAL_INODE = 1, VIRTUAL_INODE } inode_type_t;
 
 int
-svc_inode_ctx_get (xlator_t *this, inode_t *inode, int *inode_type);
-
-int32_t
-svc_inode_ctx_set (xlator_t *this, inode_t *inode, int inode_type);
-
-void
-svc_local_free (svc_local_t *local);
+gf_svc_special_dir_revalidate_lookup(call_frame_t *frame, xlator_t *this,
+                                     dict_t *xdata);
 
-gf_boolean_t
-svc_readdir_on_special_dir (call_frame_t *frame, void *cookie, xlator_t *this,
-                            int32_t op_ret, int32_t op_errno,
-                            gf_dirent_t *entries, dict_t *xdata);
 #endif /* __SNAP_VIEW_CLIENT_H__ */
diff --git a/xlators/features/snapview-server/src/Makefile.am b/xlators/features/snapview-server/src/Makefile.am
index df58d7bef71..2935f138a4c 100644
--- a/xlators/features/snapview-server/src/Makefile.am
+++ b/xlators/features/snapview-server/src/Makefile.am
@@ -1,21 +1,24 @@
+if WITH_SERVER
 xlator_LTLIBRARIES = snapview-server.la
+endif
 xlatordir = $(libdir)/glusterfs/$(PACKAGE_VERSION)/xlator/features
 
-snapview_server_la_LDFLAGS = -module -avoid-version
+snapview_server_la_LDFLAGS = -module $(GF_XLATOR_DEFAULT_LDFLAGS)
 
-snapview_server_la_SOURCES = snapview-server.c snapview-server-mgmt.c snapview-server-helpers.c
-snapview_server_la_LIBADD = $(top_builddir)/libglusterfs/src/libglusterfs.la\
-                            $(top_builddir)/api/src/libgfapi.la\
-                            $(RLLIBS) $(top_builddir)/rpc/xdr/src/libgfxdr.la \
-                            $(top_builddir)/rpc/rpc-lib/src/libgfrpc.la
+snapview_server_la_SOURCES = snapview-server.c snapview-server-mgmt.c \
+	snapview-server-helpers.c
 
-noinst_HEADERS = snapview-server.h snapview-server-mem-types.h
+snapview_server_la_LIBADD = $(top_builddir)/libglusterfs/src/libglusterfs.la \
+	$(top_builddir)/api/src/libgfapi.la \
+	$(RLLIBS) $(top_builddir)/rpc/xdr/src/libgfxdr.la \
+	$(top_builddir)/rpc/rpc-lib/src/libgfrpc.la
+
+noinst_HEADERS = snapview-server.h snapview-server-mem-types.h snapview-server-messages.h
 
 AM_CPPFLAGS = $(GF_CPPFLAGS) -I$(top_srcdir)/libglusterfs/src \
-               -I$(top_srcdir)/api/src \
-               -I$(top_srcdir)/rpc/rpc-lib/src \
-               -I$(top_srcdir)/rpc/xdr/src \
-	       -DDATADIR=\"$(localstatedir)\"
+	-I$(top_srcdir)/api/src -I$(top_srcdir)/rpc/rpc-lib/src \
+	-I$(top_srcdir)/rpc/xdr/src -I$(top_builddir)/rpc/xdr/src \
+	-DDATADIR=\"$(localstatedir)\"
 
 AM_CFLAGS = -Wall $(GF_CFLAGS)
 
diff --git a/xlators/features/snapview-server/src/snapview-server-helpers.c b/xlators/features/snapview-server/src/snapview-server-helpers.c
index 92ffdf512ef..62c1ddac49c 100644
--- a/xlators/features/snapview-server/src/snapview-server-helpers.c
+++ b/xlators/features/snapview-server/src/snapview-server-helpers.c
@@ -7,588 +7,709 @@
    later), or the GNU General Public License, version 2 (GPLv2), in all
    cases as published by the Free Software Foundation.
 */
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
-
 #include "snapview-server.h"
 #include "snapview-server-mem-types.h"
 
-#include "xlator.h"
+#include <glusterfs/xlator.h>
 #include "rpc-clnt.h"
 #include "xdr-generic.h"
 #include "protocol-common.h"
 #include <pthread.h>
 
-
 int
-__svs_inode_ctx_set (xlator_t *this, inode_t *inode, svs_inode_t *svs_inode)
+__svs_inode_ctx_set(xlator_t *this, inode_t *inode, svs_inode_t *svs_inode)
 {
-        uint64_t    value = 0;
-        int         ret   = -1;
+    uint64_t value = 0;
+    int ret = -1;
 
-        GF_VALIDATE_OR_GOTO ("snapview-server", this, out);
-        GF_VALIDATE_OR_GOTO (this->name, inode, out);
-        GF_VALIDATE_OR_GOTO (this->name, svs_inode, out);
+    GF_VALIDATE_OR_GOTO("snapview-server", this, out);
+    GF_VALIDATE_OR_GOTO(this->name, inode, out);
+    GF_VALIDATE_OR_GOTO(this->name, svs_inode, out);
 
-        value = (uint64_t)(long) svs_inode;
+    value = (uint64_t)(long)svs_inode;
 
-        ret = __inode_ctx_set (inode, this, &value);
+    ret = __inode_ctx_set(inode, this, &value);
 
 out:
-        return ret;
+    return ret;
 }
 
 svs_inode_t *
-__svs_inode_ctx_get (xlator_t *this, inode_t *inode)
+__svs_inode_ctx_get(xlator_t *this, inode_t *inode)
 {
-        svs_inode_t *svs_inode = NULL;
-        uint64_t     value     = 0;
-        int          ret       = -1;
+    svs_inode_t *svs_inode = NULL;
+    uint64_t value = 0;
+    int ret = -1;
 
-        GF_VALIDATE_OR_GOTO ("snapview-server", this, out);
-        GF_VALIDATE_OR_GOTO (this->name, inode, out);
+    GF_VALIDATE_OR_GOTO("snapview-server", this, out);
+    GF_VALIDATE_OR_GOTO(this->name, inode, out);
 
-        ret = __inode_ctx_get (inode, this, &value);
-        if (ret)
-                goto out;
+    ret = __inode_ctx_get(inode, this, &value);
+    if (ret)
+        goto out;
 
-        svs_inode = (svs_inode_t *) ((long) value);
+    svs_inode = (svs_inode_t *)((long)value);
 
 out:
-        return svs_inode;
+    return svs_inode;
 }
 
 svs_inode_t *
-svs_inode_ctx_get (xlator_t *this, inode_t *inode)
+svs_inode_ctx_get(xlator_t *this, inode_t *inode)
 {
-        svs_inode_t *svs_inode = NULL;
+    svs_inode_t *svs_inode = NULL;
 
-        GF_VALIDATE_OR_GOTO ("snapview-server", this, out);
-        GF_VALIDATE_OR_GOTO (this->name, inode, out);
+    GF_VALIDATE_OR_GOTO("snapview-server", this, out);
+    GF_VALIDATE_OR_GOTO(this->name, inode, out);
 
-        LOCK (&inode->lock);
-        {
-                svs_inode = __svs_inode_ctx_get (this, inode);
-        }
-        UNLOCK (&inode->lock);
+    LOCK(&inode->lock);
+    {
+        svs_inode = __svs_inode_ctx_get(this, inode);
+    }
+    UNLOCK(&inode->lock);
 
 out:
-        return svs_inode;
+    return svs_inode;
 }
 
 int32_t
-svs_inode_ctx_set (xlator_t *this, inode_t *inode, svs_inode_t *svs_inode)
+svs_inode_ctx_set(xlator_t *this, inode_t *inode, svs_inode_t *svs_inode)
 {
-        int32_t    ret = -1;
+    int32_t ret = -1;
 
-        GF_VALIDATE_OR_GOTO ("snapview-server", this, out);
-        GF_VALIDATE_OR_GOTO (this->name, inode, out);
-        GF_VALIDATE_OR_GOTO (this->name, svs_inode, out);
+    GF_VALIDATE_OR_GOTO("snapview-server", this, out);
+    GF_VALIDATE_OR_GOTO(this->name, inode, out);
+    GF_VALIDATE_OR_GOTO(this->name, svs_inode, out);
 
-        LOCK (&inode->lock);
-        {
-                ret = __svs_inode_ctx_set (this, inode, svs_inode);
-        }
-        UNLOCK (&inode->lock);
+    LOCK(&inode->lock);
+    {
+        ret = __svs_inode_ctx_set(this, inode, svs_inode);
+    }
+    UNLOCK(&inode->lock);
 
 out:
-        return ret;
+    return ret;
 }
 
 svs_inode_t *
-svs_inode_new (void)
+svs_inode_new(void)
 {
-        svs_inode_t    *svs_inode = NULL;
+    svs_inode_t *svs_inode = NULL;
 
-        svs_inode = GF_CALLOC (1, sizeof (*svs_inode), gf_svs_mt_svs_inode_t);
+    svs_inode = GF_CALLOC(1, sizeof(*svs_inode), gf_svs_mt_svs_inode_t);
 
-        return svs_inode;
+    return svs_inode;
 }
 
 svs_inode_t *
-svs_inode_ctx_get_or_new (xlator_t *this, inode_t *inode)
+svs_inode_ctx_get_or_new(xlator_t *this, inode_t *inode)
 {
-        svs_inode_t   *svs_inode = NULL;
-        int            ret       = -1;
-
-        GF_VALIDATE_OR_GOTO ("snapview-server", this, out);
-        GF_VALIDATE_OR_GOTO (this->name, inode, out);
-
-        LOCK (&inode->lock);
-        {
-                svs_inode = __svs_inode_ctx_get (this, inode);
-                if (!svs_inode) {
-                        svs_inode = svs_inode_new ();
-                        if (svs_inode) {
-                                ret = __svs_inode_ctx_set (this, inode,
-                                                           svs_inode);
-                                if (ret) {
-                                        GF_FREE (svs_inode);
-                                        svs_inode = NULL;
-                                }
-                        }
+    svs_inode_t *svs_inode = NULL;
+    int ret = -1;
+
+    GF_VALIDATE_OR_GOTO("snapview-server", this, out);
+    GF_VALIDATE_OR_GOTO(this->name, inode, out);
+
+    LOCK(&inode->lock);
+    {
+        svs_inode = __svs_inode_ctx_get(this, inode);
+        if (!svs_inode) {
+            svs_inode = svs_inode_new();
+            if (svs_inode) {
+                ret = __svs_inode_ctx_set(this, inode, svs_inode);
+                if (ret) {
+                    GF_FREE(svs_inode);
+                    svs_inode = NULL;
                 }
+            }
         }
-        UNLOCK (&inode->lock);
+    }
+    UNLOCK(&inode->lock);
 
 out:
-        return svs_inode;
+    return svs_inode;
 }
 
 svs_fd_t *
-svs_fd_new (void)
+svs_fd_new(void)
 {
-        svs_fd_t    *svs_fd = NULL;
+    svs_fd_t *svs_fd = NULL;
 
-        svs_fd = GF_CALLOC (1, sizeof (*svs_fd), gf_svs_mt_svs_fd_t);
+    svs_fd = GF_CALLOC(1, sizeof(*svs_fd), gf_svs_mt_svs_fd_t);
 
-        return svs_fd;
+    return svs_fd;
 }
 
 int
-__svs_fd_ctx_set (xlator_t *this, fd_t *fd, svs_fd_t *svs_fd)
+__svs_fd_ctx_set(xlator_t *this, fd_t *fd, svs_fd_t *svs_fd)
 {
-        uint64_t    value = 0;
-        int         ret   = -1;
+    uint64_t value = 0;
+    int ret = -1;
 
-        GF_VALIDATE_OR_GOTO ("snapview-server", this, out);
-        GF_VALIDATE_OR_GOTO (this->name, fd, out);
-        GF_VALIDATE_OR_GOTO (this->name, svs_fd, out);
+    GF_VALIDATE_OR_GOTO("snapview-server", this, out);
+    GF_VALIDATE_OR_GOTO(this->name, fd, out);
+    GF_VALIDATE_OR_GOTO(this->name, svs_fd, out);
 
-        value = (uint64_t)(long) svs_fd;
+    value = (uint64_t)(long)svs_fd;
 
-        ret = __fd_ctx_set (fd, this, value);
+    ret = __fd_ctx_set(fd, this, value);
 
 out:
-        return ret;
+    return ret;
 }
 
 svs_fd_t *
-__svs_fd_ctx_get (xlator_t *this, fd_t *fd)
+__svs_fd_ctx_get(xlator_t *this, fd_t *fd)
 {
-        svs_fd_t *svs_fd = NULL;
-        uint64_t  value  = 0;
-        int       ret    = -1;
+    svs_fd_t *svs_fd = NULL;
+    uint64_t value = 0;
+    int ret = -1;
 
-        GF_VALIDATE_OR_GOTO ("snapview-server", this, out);
-        GF_VALIDATE_OR_GOTO (this->name, fd, out);
+    GF_VALIDATE_OR_GOTO("snapview-server", this, out);
+    GF_VALIDATE_OR_GOTO(this->name, fd, out);
 
-        ret = __fd_ctx_get (fd, this, &value);
-        if (ret)
-                return NULL;
+    ret = __fd_ctx_get(fd, this, &value);
+    if (ret)
+        return NULL;
 
-        svs_fd = (svs_fd_t *) ((long) value);
+    svs_fd = (svs_fd_t *)((long)value);
 
 out:
-        return svs_fd;
+    return svs_fd;
 }
 
 svs_fd_t *
-svs_fd_ctx_get (xlator_t *this, fd_t *fd)
+svs_fd_ctx_get(xlator_t *this, fd_t *fd)
 {
-        svs_fd_t *svs_fd = NULL;
+    svs_fd_t *svs_fd = NULL;
 
-        GF_VALIDATE_OR_GOTO ("snapview-server", this, out);
-        GF_VALIDATE_OR_GOTO (this->name, fd, out);
+    GF_VALIDATE_OR_GOTO("snapview-server", this, out);
+    GF_VALIDATE_OR_GOTO(this->name, fd, out);
 
-        LOCK (&fd->lock);
-        {
-                svs_fd = __svs_fd_ctx_get (this, fd);
-        }
-        UNLOCK (&fd->lock);
+    LOCK(&fd->lock);
+    {
+        svs_fd = __svs_fd_ctx_get(this, fd);
+    }
+    UNLOCK(&fd->lock);
 
 out:
-        return svs_fd;
+    return svs_fd;
 }
 
 int32_t
-svs_fd_ctx_set (xlator_t *this, fd_t *fd, svs_fd_t *svs_fd)
+svs_fd_ctx_set(xlator_t *this, fd_t *fd, svs_fd_t *svs_fd)
 {
-        int32_t    ret = -1;
+    int32_t ret = -1;
 
-        GF_VALIDATE_OR_GOTO ("snapview-server", this, out);
-        GF_VALIDATE_OR_GOTO (this->name, fd, out);
-        GF_VALIDATE_OR_GOTO (this->name, svs_fd, out);
+    GF_VALIDATE_OR_GOTO("snapview-server", this, out);
+    GF_VALIDATE_OR_GOTO(this->name, fd, out);
+    GF_VALIDATE_OR_GOTO(this->name, svs_fd, out);
 
-        LOCK (&fd->lock);
-        {
-                ret = __svs_fd_ctx_set (this, fd, svs_fd);
-        }
-        UNLOCK (&fd->lock);
+    LOCK(&fd->lock);
+    {
+        ret = __svs_fd_ctx_set(this, fd, svs_fd);
+    }
+    UNLOCK(&fd->lock);
 
 out:
-        return ret;
+    return ret;
 }
 
 svs_fd_t *
-__svs_fd_ctx_get_or_new (xlator_t *this, fd_t *fd)
+__svs_fd_ctx_get_or_new(xlator_t *this, fd_t *fd)
 {
-        svs_fd_t        *svs_fd    = NULL;
-        int              ret       = -1;
-        glfs_t          *fs        = NULL;
-        glfs_object_t   *object    = NULL;
-        svs_inode_t     *inode_ctx = NULL;
-        glfs_fd_t       *glfd      = NULL;
-        inode_t         *inode     = NULL;
-
-        GF_VALIDATE_OR_GOTO ("snapview-server", this, out);
-        GF_VALIDATE_OR_GOTO (this->name, fd, out);
-
-        inode = fd->inode;
-        svs_fd = __svs_fd_ctx_get (this, fd);
-        if (svs_fd) {
-                ret = 0;
-                goto out;
+    svs_fd_t *svs_fd = NULL;
+    int ret = -1;
+    glfs_t *fs = NULL;
+    glfs_object_t *object = NULL;
+    svs_inode_t *inode_ctx = NULL;
+    glfs_fd_t *glfd = NULL;
+    inode_t *inode = NULL;
+
+    GF_VALIDATE_OR_GOTO("snapview-server", this, out);
+    GF_VALIDATE_OR_GOTO(this->name, fd, out);
+
+    inode = fd->inode;
+    svs_fd = __svs_fd_ctx_get(this, fd);
+    if (svs_fd) {
+        ret = 0;
+        goto out;
+    }
+
+    svs_fd = svs_fd_new();
+    if (!svs_fd) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, SVS_MSG_NEW_FD_CTX_FAILED,
+               "failed to allocate new fd "
+               "context for gfid %s",
+               uuid_utoa(inode->gfid));
+        goto out;
+    }
+
+    if (fd_is_anonymous(fd)) {
+        inode_ctx = svs_inode_ctx_get(this, inode);
+        if (!inode_ctx) {
+            gf_msg(this->name, GF_LOG_ERROR, 0,
+                   SVS_MSG_GET_INODE_CONTEXT_FAILED,
+                   "failed to get inode "
+                   "context for %s",
+                   uuid_utoa(inode->gfid));
+            goto out;
         }
 
-        svs_fd = svs_fd_new ();
-        if (!svs_fd) {
-                gf_log (this->name, GF_LOG_ERROR, "failed to allocate new fd "
-                        "context for gfid %s", uuid_utoa (inode->gfid));
+        fs = inode_ctx->fs;
+        object = inode_ctx->object;
+
+        if (inode->ia_type == IA_IFDIR) {
+            glfd = glfs_h_opendir(fs, object);
+            if (!glfd) {
+                gf_msg(this->name, GF_LOG_ERROR, errno, SVS_MSG_OPENDIR_FAILED,
+                       "failed to "
+                       "open the directory %s",
+                       uuid_utoa(inode->gfid));
                 goto out;
+            }
         }
 
-        if (fd_is_anonymous (fd)) {
-                inode_ctx = svs_inode_ctx_get (this, inode);
-                if (!inode_ctx) {
-                        gf_log (this->name, GF_LOG_ERROR, "failed to get inode "
-                                "context for %s", uuid_utoa (inode->gfid));
-                        goto out;
-                }
-
-                fs = inode_ctx->fs;
-                object = inode_ctx->object;
-
-                if (inode->ia_type == IA_IFDIR) {
-                        glfd = glfs_h_opendir (fs, object);
-                        if (!glfd) {
-                                gf_log (this->name, GF_LOG_ERROR, "failed to "
-                                        "open the directory %s",
-                                        uuid_utoa (inode->gfid));
-                                goto out;
-                        }
-                }
-
-                if (inode->ia_type == IA_IFREG) {
-                        glfd = glfs_h_open (fs, object, O_RDONLY|O_LARGEFILE);
-                        if (!glfd) {
-                                gf_log (this->name, GF_LOG_ERROR, "failed to "
-                                        "open the file %s",
-                                        uuid_utoa (inode->gfid));
-                                goto out;
-                        }
-                }
-
-                svs_fd->fd = glfd;
+        if (inode->ia_type == IA_IFREG) {
+            glfd = glfs_h_open(fs, object, O_RDONLY | O_LARGEFILE);
+            if (!glfd) {
+                gf_msg(this->name, GF_LOG_ERROR, errno, SVS_MSG_OPEN_FAILED,
+                       "failed to "
+                       "open the file %s",
+                       uuid_utoa(inode->gfid));
+                goto out;
+            }
         }
 
-        ret = __svs_fd_ctx_set (this, fd, svs_fd);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "failed to set fd context "
-                        "for gfid %s", uuid_utoa (inode->gfid));
-                if (svs_fd->fd) {
-                        if (inode->ia_type == IA_IFDIR) {
-                                ret = glfs_closedir (svs_fd->fd);
-                                if (ret)
-                                        gf_log (this->name, GF_LOG_ERROR,
-                                                "failed to close the fd for %s",
-                                                uuid_utoa (inode->gfid));
-                        }
-                        if (inode->ia_type == IA_IFREG) {
-                                ret = glfs_close (svs_fd->fd);
-                                if (ret)
-                                        gf_log (this->name, GF_LOG_ERROR,
-                                                "failed to close the fd for %s",
-                                                uuid_utoa (inode->gfid));
-                        }
-                }
-                ret = -1;
+        svs_fd->fd = glfd;
+    }
+
+    ret = __svs_fd_ctx_set(this, fd, svs_fd);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, SVS_MSG_SET_FD_CONTEXT_FAILED,
+               "failed to set fd context "
+               "for gfid %s",
+               uuid_utoa(inode->gfid));
+        if (svs_fd->fd) {
+            if (inode->ia_type == IA_IFDIR) {
+                ret = glfs_closedir(svs_fd->fd);
+                if (ret)
+                    gf_msg(this->name, GF_LOG_ERROR, errno,
+                           SVS_MSG_CLOSEDIR_FAILED,
+                           "failed to close the fd for %s",
+                           uuid_utoa(inode->gfid));
+            }
+            if (inode->ia_type == IA_IFREG) {
+                ret = glfs_close(svs_fd->fd);
+                if (ret)
+                    gf_msg(this->name, GF_LOG_ERROR, 0, SVS_MSG_CLOSE_FAILED,
+                           "failed to close the fd for %s",
+                           uuid_utoa(inode->gfid));
+            }
         }
+        ret = -1;
+    }
 
 out:
-        if (ret) {
-                GF_FREE (svs_fd);
-                svs_fd = NULL;
-        }
+    if (ret) {
+        GF_FREE(svs_fd);
+        svs_fd = NULL;
+    }
 
-        return svs_fd;
+    return svs_fd;
 }
 
 svs_fd_t *
-svs_fd_ctx_get_or_new (xlator_t *this, fd_t *fd)
+svs_fd_ctx_get_or_new(xlator_t *this, fd_t *fd)
 {
-        svs_fd_t  *svs_fd = NULL;
+    svs_fd_t *svs_fd = NULL;
 
-        GF_VALIDATE_OR_GOTO ("snapview-server", this, out);
-        GF_VALIDATE_OR_GOTO (this->name, fd, out);
+    GF_VALIDATE_OR_GOTO("snapview-server", this, out);
+    GF_VALIDATE_OR_GOTO(this->name, fd, out);
 
-        LOCK (&fd->lock);
-        {
-                svs_fd = __svs_fd_ctx_get_or_new (this, fd);
-        }
-        UNLOCK (&fd->lock);
+    LOCK(&fd->lock);
+    {
+        svs_fd = __svs_fd_ctx_get_or_new(this, fd);
+    }
+    UNLOCK(&fd->lock);
+
+out:
+    return svs_fd;
+}
+
+int
+svs_uuid_generate(xlator_t *this, uuid_t gfid, char *snapname,
+                  uuid_t origin_gfid)
+{
+    char ino_string[NAME_MAX + 32] = "";
+    uuid_t tmp = {
+        0,
+    };
+    int ret = -1;
+
+    GF_VALIDATE_OR_GOTO("snapview-server", this, out);
+    GF_VALIDATE_OR_GOTO(this->name, snapname, out);
+
+    (void)snprintf(ino_string, sizeof(ino_string), "%s%s", snapname,
+                   uuid_utoa(origin_gfid));
+
+    if (gf_gfid_generate_from_xxh64(tmp, ino_string)) {
+        gf_msg(this->name, GF_LOG_WARNING, 0, SVS_MSG_GFID_GEN_FAILED,
+               "failed to generate "
+               "gfid for object with actual gfid of %s "
+               "(snapname: %s, key: %s)",
+               uuid_utoa(origin_gfid), snapname, ino_string);
+        goto out;
+    }
+
+    gf_uuid_copy(gfid, tmp);
+
+    ret = 0;
+
+    gf_msg_debug(this->name, 0, "gfid generated is %s ", uuid_utoa(gfid));
 
 out:
-        return svs_fd;
+    return ret;
 }
 
 void
-svs_fill_ino_from_gfid (struct iatt *buf)
+svs_fill_ino_from_gfid(struct iatt *buf)
 {
-        uint64_t  temp_ino = 0;
-        int       j        = 0;
-        int       i        = 0;
-        xlator_t *this     = NULL;
+    xlator_t *this = NULL;
 
-        this = THIS;
+    this = THIS;
 
-        GF_VALIDATE_OR_GOTO ("snapview-server", this, out);
-        GF_VALIDATE_OR_GOTO (this->name, buf, out);
+    GF_VALIDATE_OR_GOTO("snapview-server", this, out);
+    GF_VALIDATE_OR_GOTO(this->name, buf, out);
 
-        /* consider least significant 8 bytes of value out of gfid */
-        if (gf_uuid_is_null (buf->ia_gfid)) {
-                buf->ia_ino = -1;
-                goto out;
-        }
-        for (i = 15; i > (15 - 8); i--) {
-                temp_ino += (uint64_t)(buf->ia_gfid[i]) << j;
-                j += 8;
-        }
-        buf->ia_ino = temp_ino;
+    /* consider least significant 8 bytes of value out of gfid */
+    if (gf_uuid_is_null(buf->ia_gfid)) {
+        buf->ia_ino = -1;
+        goto out;
+    }
+
+    buf->ia_ino = gfid_to_ino(buf->ia_gfid);
 out:
-        return;
+    return;
 }
 
 void
-svs_iatt_fill (uuid_t gfid, struct iatt *buf)
+svs_iatt_fill(uuid_t gfid, struct iatt *buf)
 {
-        struct timeval  tv    = {0, };
-        xlator_t       *this  = NULL;
+    struct timeval tv = {
+        0,
+    };
+    xlator_t *this = NULL;
 
-        this = THIS;
+    this = THIS;
 
-        GF_VALIDATE_OR_GOTO ("snapview-server", this, out);
-        GF_VALIDATE_OR_GOTO (this->name, buf, out);
+    GF_VALIDATE_OR_GOTO("snapview-server", this, out);
+    GF_VALIDATE_OR_GOTO(this->name, buf, out);
 
-        buf->ia_type = IA_IFDIR;
-        buf->ia_uid  = 0;
-        buf->ia_gid  = 0;
-        buf->ia_size = 0;
-        buf->ia_nlink = 2;
-        buf->ia_blocks = 8;
-        buf->ia_size = 4096;
+    buf->ia_type = IA_IFDIR;
+    buf->ia_uid = 0;
+    buf->ia_gid = 0;
+    buf->ia_size = 0;
+    buf->ia_nlink = 2;
+    buf->ia_blocks = 8;
+    buf->ia_size = 4096;
 
-        gf_uuid_copy (buf->ia_gfid, gfid);
-        svs_fill_ino_from_gfid (buf);
+    gf_uuid_copy(buf->ia_gfid, gfid);
+    svs_fill_ino_from_gfid(buf);
 
-        buf->ia_prot = ia_prot_from_st_mode (0755);
+    buf->ia_prot = ia_prot_from_st_mode(0755);
 
-        gettimeofday (&tv, 0);
+    gettimeofday(&tv, 0);
 
-        buf->ia_mtime = buf->ia_atime = buf->ia_ctime = tv.tv_sec;
-        buf->ia_mtime_nsec = buf->ia_atime_nsec = buf->ia_ctime_nsec =
-                (tv.tv_usec * 1000);
+    buf->ia_mtime = buf->ia_atime = buf->ia_ctime = tv.tv_sec;
+    buf->ia_mtime_nsec = buf->ia_atime_nsec = buf->ia_ctime_nsec = (tv.tv_usec *
+                                                                    1000);
 
 out:
-        return;
+    return;
 }
 
 /* priv->snaplist_lock should be held before calling this function */
 snap_dirent_t *
-__svs_get_snap_dirent (xlator_t *this, const char *name)
+__svs_get_snap_dirent(xlator_t *this, const char *name)
 {
-        svs_private_t      *private     = NULL;
-        int                 i           = 0;
-        snap_dirent_t      *dirents     = NULL;
-        snap_dirent_t      *tmp_dirent  = NULL;
-        snap_dirent_t      *dirent      = NULL;
-
-        private = this->private;
-
-        dirents = private->dirents;
-        if (!dirents) {
-                goto out;
-        }
-
-        tmp_dirent = dirents;
-        for (i = 0; i < private->num_snaps; i++) {
-                if (!strcmp (tmp_dirent->name, name)) {
-                        dirent = tmp_dirent;
-                        break;
-                }
-                tmp_dirent++;
+    svs_private_t *private = NULL;
+    int i = 0;
+    snap_dirent_t *dirents = NULL;
+    snap_dirent_t *tmp_dirent = NULL;
+    snap_dirent_t *dirent = NULL;
+
+   private
+    = this->private;
+
+    dirents = private->dirents;
+    if (!dirents) {
+        goto out;
+    }
+
+    tmp_dirent = dirents;
+    for (i = 0; i < private->num_snaps; i++) {
+        if (!strcmp(tmp_dirent->name, name)) {
+            dirent = tmp_dirent;
+            break;
         }
+        tmp_dirent++;
+    }
 
- out:
-        return dirent;
+out:
+    return dirent;
 }
 
 glfs_t *
-__svs_initialise_snapshot_volume (xlator_t *this, const char *name,
-                                  int32_t *op_errno)
+__svs_initialise_snapshot_volume(xlator_t *this, const char *name,
+                                 int32_t *op_errno)
 {
-        svs_private_t      *priv                = NULL;
-        int32_t             ret                 = -1;
-        int32_t             local_errno         = ESTALE;
-        snap_dirent_t      *dirent              = NULL;
-        char                volname[PATH_MAX]   = {0, };
-        glfs_t             *fs                  = NULL;
-        int                 loglevel            = GF_LOG_INFO;
-        char                logfile[PATH_MAX]   = {0, };
-
-        GF_VALIDATE_OR_GOTO ("snapview-server", this, out);
-        GF_VALIDATE_OR_GOTO (this->name, this->private, out);
-        GF_VALIDATE_OR_GOTO (this->name, name, out);
-
-        priv = this->private;
-
-        dirent = __svs_get_snap_dirent (this, name);
-        if (!dirent) {
-                gf_log (this->name, GF_LOG_DEBUG, "snap entry for "
-                        "name %s not found", name);
-                local_errno = ENOENT;
-                goto out;
+    svs_private_t *priv = NULL;
+    int32_t ret = -1;
+    int32_t local_errno = ESTALE;
+    snap_dirent_t *dirent = NULL;
+    char volname[PATH_MAX] = {
+        0,
+    };
+    glfs_t *fs = NULL;
+    int loglevel = GF_LOG_INFO;
+    char logfile[PATH_MAX] = {
+        0,
+    };
+    char *volfile_server = NULL;
+
+    GF_VALIDATE_OR_GOTO("snapview-server", this, out);
+    GF_VALIDATE_OR_GOTO(this->name, this->private, out);
+    GF_VALIDATE_OR_GOTO(this->name, name, out);
+
+    priv = this->private;
+
+    dirent = __svs_get_snap_dirent(this, name);
+    if (!dirent) {
+        gf_msg_debug(this->name, 0,
+                     "snap entry for "
+                     "name %s not found",
+                     name);
+        local_errno = ENOENT;
+        goto out;
+    }
+
+    if (dirent->fs) {
+        ret = 0;
+        fs = dirent->fs;
+        goto out;
+    }
+
+    snprintf(volname, sizeof(volname), "/snaps/%s/%s/%s", dirent->name,
+             dirent->snap_volname, dirent->snap_volname);
+
+    fs = glfs_new(volname);
+    if (!fs) {
+        local_errno = ENOMEM;
+        gf_msg(this->name, GF_LOG_ERROR, local_errno, SVS_MSG_GLFS_NEW_FAILED,
+               "glfs instance for snap volume %s "
+               "failed",
+               dirent->name);
+        goto out;
+    }
+
+    /*
+     * Before, localhost was used as the volfile server. But, with that
+     * method, accessing snapshots started giving ENOENT error if a
+     * specific bind address is mentioned in the glusterd volume file.
+     * Check the bug https://bugzilla.redhat.com/show_bug.cgi?id=1725211.
+     * So, the new method is tried below, where, snapview-server first
+     * uses the volfile server used by the snapd (obtained from the
+     * command line arguments saved in the global context of the process).
+     * If the volfile server in global context is NULL, then localhost
+     * is tried (like before).
+     */
+    if (this->ctx->cmd_args.volfile_server) {
+        volfile_server = gf_strdup(this->ctx->cmd_args.volfile_server);
+        if (!volfile_server) {
+            gf_msg(this->name, GF_LOG_WARNING, ENOMEM,
+                   SVS_MSG_VOLFILE_SERVER_GET_FAIL,
+                   "failed to copy volfile server %s. ",
+                   this->ctx->cmd_args.volfile_server);
+            ret = -1;
+            goto out;
         }
-
-        if (dirent->fs) {
-                ret = 0;
-                fs = dirent->fs;
-                goto out;
+    } else {
+        gf_msg(this->name, GF_LOG_WARNING, ENOMEM,
+               SVS_MSG_VOLFILE_SERVER_GET_FAIL,
+               "volfile server is NULL in cmd args. "
+               "Trying with localhost");
+        volfile_server = gf_strdup("localhost");
+        if (!volfile_server) {
+            gf_msg(this->name, GF_LOG_WARNING, ENOMEM,
+                   SVS_MSG_VOLFILE_SERVER_GET_FAIL,
+                   "failed to copy volfile server localhost.");
+            ret = -1;
+            goto out;
         }
+    }
+
+    ret = glfs_set_volfile_server(fs, "tcp", volfile_server, 24007);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, local_errno,
+               SVS_MSG_SET_VOLFILE_SERVR_FAILED,
+               "setting the "
+               "volfile server %s for snap volume %s "
+               "failed",
+               volfile_server, dirent->name);
+        goto out;
+    }
+
+    snprintf(logfile, sizeof(logfile),
+             DEFAULT_SVD_LOG_FILE_DIRECTORY "/snaps/%s/%s-%s.log",
+             priv->volname, name, dirent->uuid);
+
+    ret = glfs_set_logging(fs, logfile, loglevel);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, local_errno,
+               SVS_MSG_SET_LOGGING_FAILED,
+               "failed to set the "
+               "log file path");
+        goto out;
+    }
+
+    ret = glfs_init(fs);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, local_errno, SVS_MSG_GLFS_INIT_FAILED,
+               "initing the "
+               "fs for %s failed",
+               dirent->name);
+        goto out;
+    }
+
+    ret = 0;
 
-        snprintf (volname, sizeof (volname), "/snaps/%s/%s",
-                  dirent->name, dirent->snap_volname);
+out:
+    if (ret) {
+        if (op_errno)
+            *op_errno = local_errno;
 
+        if (fs)
+            glfs_fini(fs);
+        fs = NULL;
+    }
 
-        fs = glfs_new (volname);
-        if (!fs) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "glfs instance for snap volume %s "
-                        "failed", dirent->name);
-                local_errno = ENOMEM;
-                goto out;
-        }
+    if (fs) {
+        dirent->fs = fs;
+    }
 
-        ret = glfs_set_volfile_server (fs, "tcp", "localhost",
-                                       24007);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "setting the "
-                        "volfile server for snap volume %s "
-                        "failed", dirent->name);
-                goto out;
-        }
+    GF_FREE(volfile_server);
+    return fs;
+}
 
-        snprintf (logfile, sizeof (logfile),
-                  DEFAULT_SVD_LOG_FILE_DIRECTORY "/snaps/%s/%s-%s.log",
-                  priv->volname, name, dirent->uuid);
+glfs_t *
+svs_initialise_snapshot_volume(xlator_t *this, const char *name,
+                               int32_t *op_errno)
+{
+    glfs_t *fs = NULL;
+    svs_private_t *priv = NULL;
 
-        ret = glfs_set_logging(fs, logfile, loglevel);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "failed to set the "
-                        "log file path");
-                goto out;
-        }
+    GF_VALIDATE_OR_GOTO("snapview-server", this, out);
+    GF_VALIDATE_OR_GOTO(this->name, this->private, out);
+    GF_VALIDATE_OR_GOTO(this->name, name, out);
 
-        ret = glfs_init (fs);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "initing the "
-                        "fs for %s failed", dirent->name);
-                goto out;
-        }
+    priv = this->private;
 
-        ret = 0;
+    LOCK(&priv->snaplist_lock);
+    {
+        fs = __svs_initialise_snapshot_volume(this, name, op_errno);
+    }
+    UNLOCK(&priv->snaplist_lock);
 
 out:
-        if (ret) {
-                if (op_errno)
-                        *op_errno = local_errno;
 
-                if (fs)
-                        glfs_fini (fs);
-                fs = NULL;
-        }
+    return fs;
+}
+
+snap_dirent_t *
+svs_get_latest_snap_entry(xlator_t *this)
+{
+    svs_private_t *priv = NULL;
+    snap_dirent_t *dirents = NULL;
+    snap_dirent_t *dirent = NULL;
 
-        if (fs) {
-                dirent->fs = fs;
+    GF_VALIDATE_OR_GOTO("svs", this, out);
+
+    priv = this->private;
+
+    LOCK(&priv->snaplist_lock);
+    {
+        dirents = priv->dirents;
+        if (!dirents) {
+            goto unlock;
         }
+        if (priv->num_snaps)
+            dirent = &dirents[priv->num_snaps - 1];
+    }
+unlock:
+    UNLOCK(&priv->snaplist_lock);
 
-        return fs;
+out:
+    return dirent;
 }
 
 glfs_t *
-svs_initialise_snapshot_volume (xlator_t *this, const char *name,
-                                int32_t *op_errno)
+svs_get_latest_snapshot(xlator_t *this)
 {
-        glfs_t             *fs   = NULL;
-        svs_private_t      *priv = NULL;
+    glfs_t *fs = NULL;
+    snap_dirent_t *dirent = NULL;
+    svs_private_t *priv = NULL;
 
-        GF_VALIDATE_OR_GOTO ("snapview-server", this, out);
-        GF_VALIDATE_OR_GOTO (this->name, this->private, out);
-        GF_VALIDATE_OR_GOTO (this->name, name, out);
+    GF_VALIDATE_OR_GOTO("svs", this, out);
+    priv = this->private;
 
-        priv = this->private;
+    dirent = svs_get_latest_snap_entry(this);
 
-        LOCK (&priv->snaplist_lock);
+    if (dirent) {
+        LOCK(&priv->snaplist_lock);
         {
-                fs = __svs_initialise_snapshot_volume (this, name, op_errno);
+            fs = dirent->fs;
         }
-        UNLOCK (&priv->snaplist_lock);
-
+        UNLOCK(&priv->snaplist_lock);
+    }
 
 out:
-
-        return fs;
+    return fs;
 }
 
-snap_dirent_t *
-svs_get_latest_snap_entry (xlator_t *this)
+glfs_t *
+svs_inode_ctx_glfs_mapping(xlator_t *this, svs_inode_t *inode_ctx)
 {
-        svs_private_t *priv       = NULL;
-        snap_dirent_t *dirents    = NULL;
-        snap_dirent_t *dirent     = NULL;
+    glfs_t *fs = NULL;
 
-        GF_VALIDATE_OR_GOTO ("svs", this, out);
+    GF_VALIDATE_OR_GOTO("svs", this, out);
+    GF_VALIDATE_OR_GOTO(this->name, inode_ctx, out);
 
-        priv = this->private;
+    fs = inode_ctx->fs;
 
-        LOCK (&priv->snaplist_lock);
-        {
-                dirents = priv->dirents;
-                if (!dirents) {
-                        goto unlock;
-                }
-                if (priv->num_snaps)
-                        dirent = &dirents[priv->num_snaps - 1];
-        }
-unlock:
-        UNLOCK (&priv->snaplist_lock);
+    SVS_CHECK_VALID_SNAPSHOT_HANDLE(fs, this);
 
 out:
-        return dirent;
+    return fs;
 }
 
 glfs_t *
-svs_get_latest_snapshot (xlator_t *this)
+svs_inode_glfs_mapping(xlator_t *this, inode_t *inode)
 {
-        glfs_t        *fs         = NULL;
-        snap_dirent_t *dirent     = NULL;
-        svs_private_t *priv       = NULL;
-
-        GF_VALIDATE_OR_GOTO ("svs", this, out);
-        priv = this->private;
+    svs_inode_t *inode_ctx = NULL;
+    glfs_t *fs = NULL;
 
-        dirent = svs_get_latest_snap_entry (this);
+    inode_ctx = svs_inode_ctx_get(this, inode);
+    if (!inode_ctx) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, SVS_MSG_GET_INODE_CONTEXT_FAILED,
+               "inode context not found for"
+               " the inode %s",
+               uuid_utoa(inode->gfid));
+        goto out;
+    }
 
-        if (dirent) {
-                LOCK (&priv->snaplist_lock);
-                {
-                        fs = dirent->fs;
-                }
-                UNLOCK (&priv->snaplist_lock);
-        }
+    fs = svs_inode_ctx_glfs_mapping(this, inode_ctx);
 
 out:
-        return fs;
+    return fs;
 }
diff --git a/xlators/features/snapview-server/src/snapview-server-mem-types.h b/xlators/features/snapview-server/src/snapview-server-mem-types.h
index a8035165000..63456b85323 100644
--- a/xlators/features/snapview-server/src/snapview-server-mem-types.h
+++ b/xlators/features/snapview-server/src/snapview-server-mem-types.h
@@ -11,16 +11,15 @@
 #ifndef __SNAP_VIEW_MEM_TYPES_H
 #define __SNAP_VIEW_MEM_TYPES_H
 
-#include "mem-types.h"
+#include <glusterfs/mem-types.h>
 
 enum snapview_mem_types {
-        gf_svs_mt_priv_t = gf_common_mt_end + 1,
-        gf_svs_mt_svs_inode_t,
-        gf_svs_mt_dirents_t,
-        gf_svs_mt_svs_fd_t,
-        gf_svs_mt_snaplist_t,
-        gf_svs_mt_end
+    gf_svs_mt_priv_t = gf_common_mt_end + 1,
+    gf_svs_mt_svs_inode_t,
+    gf_svs_mt_dirents_t,
+    gf_svs_mt_svs_fd_t,
+    gf_svs_mt_snaplist_t,
+    gf_svs_mt_end
 };
 
 #endif
-
diff --git a/xlators/features/snapview-server/src/snapview-server-messages.h b/xlators/features/snapview-server/src/snapview-server-messages.h
new file mode 100644
index 00000000000..f634ab5d2b0
--- /dev/null
+++ b/xlators/features/snapview-server/src/snapview-server-messages.h
@@ -0,0 +1,54 @@
+/*
+ Copyright (c) 2018 Red Hat, Inc. <http://www.redhat.com>
+ This file is part of GlusterFS.
+
+ This file is licensed to you under your choice of the GNU Lesser
+ General Public License, version 3 or any later version (LGPLv3 or
+ later), or the GNU General Public License, version 2 (GPLv2), in all
+ cases as published by the Free Software Foundation.
+ */
+
+#ifndef _SNAPVIEW_SERVER_MESSAGES_H_
+#define _SNAPVIEW_SERVER_MESSAGES_H_
+
+#include <glusterfs/glfs-message-id.h>
+
+/* To add new message IDs, append new identifiers at the end of the list.
+ *
+ * Never remove a message ID. If it's not used anymore, you can rename it or
+ * leave it as it is, but not delete it. This is to prevent reutilization of
+ * IDs by other messages.
+ *
+ * The component name must match one of the entries defined in
+ * glfs-message-id.h.
+ */
+
+GLFS_MSGID(SNAPVIEW_SERVER, SVS_MSG_NO_MEMORY, SVS_MSG_MEM_ACNT_FAILED,
+           SVS_MSG_NULL_GFID, SVS_MSG_GET_LATEST_SNAP_FAILED,
+           SVS_MSG_INVALID_GLFS_CTX, SVS_MSG_LOCK_DESTROY_FAILED,
+           SVS_MSG_SNAPSHOT_LIST_CHANGED, SVS_MSG_MGMT_INIT_FAILED,
+           SVS_MSG_GET_SNAPSHOT_LIST_FAILED, SVS_MSG_GET_GLFS_H_OBJECT_FAILED,
+           SVS_MSG_PARENT_CTX_OR_NAME_NULL, SVS_MSG_SET_INODE_CONTEXT_FAILED,
+           SVS_MSG_GET_INODE_CONTEXT_FAILED, SVS_MSG_NEW_INODE_CTX_FAILED,
+           SVS_MSG_DELETE_INODE_CONTEXT_FAILED, SVS_MSG_SET_FD_CONTEXT_FAILED,
+           SVS_MSG_NEW_FD_CTX_FAILED, SVS_MSG_DELETE_FD_CTX_FAILED,
+           SVS_MSG_GETXATTR_FAILED, SVS_MSG_LISTXATTR_FAILED,
+           SVS_MSG_RELEASEDIR_FAILED, SVS_MSG_RELEASE_FAILED,
+           SVS_MSG_TELLDIR_FAILED, SVS_MSG_STAT_FAILED, SVS_MSG_STATFS_FAILED,
+           SVS_MSG_OPEN_FAILED, SVS_MSG_READ_FAILED, SVS_MSG_READLINK_FAILED,
+           SVS_MSG_ACCESS_FAILED, SVS_MSG_GET_FD_CONTEXT_FAILED,
+           SVS_MSG_DICT_SET_FAILED, SVS_MSG_OPENDIR_FAILED,
+           SVS_MSG_FS_INSTANCE_INVALID, SVS_MSG_SETFSUID_FAIL,
+           SVS_MSG_SETFSGID_FAIL, SVS_MSG_SETFSGRPS_FAIL,
+           SVS_MSG_BUILD_TRNSPRT_OPT_FAILED, SVS_MSG_RPC_INIT_FAILED,
+           SVS_MSG_REG_NOTIFY_FAILED, SVS_MSG_REG_CBK_PRGM_FAILED,
+           SVS_MSG_RPC_CLNT_START_FAILED, SVS_MSG_XDR_PAYLOAD_FAILED,
+           SVS_MSG_NULL_CTX, SVS_MSG_RPC_CALL_FAILED, SVS_MSG_XDR_DECODE_FAILED,
+           SVS_MSG_RSP_DICT_EMPTY, SVS_MSG_DICT_GET_FAILED,
+           SVS_MSG_SNAP_LIST_REFRESH_FAILED, SVS_MSG_RPC_REQ_FAILED,
+           SVS_MSG_CLOSEDIR_FAILED, SVS_MSG_CLOSE_FAILED,
+           SVS_MSG_GFID_GEN_FAILED, SVS_MSG_GLFS_NEW_FAILED,
+           SVS_MSG_SET_VOLFILE_SERVR_FAILED, SVS_MSG_SET_LOGGING_FAILED,
+           SVS_MSG_VOLFILE_SERVER_GET_FAIL, SVS_MSG_GLFS_INIT_FAILED);
+
+#endif /* !_SNAPVIEW_CLIENT_MESSAGES_H_ */
diff --git a/xlators/features/snapview-server/src/snapview-server-mgmt.c b/xlators/features/snapview-server/src/snapview-server-mgmt.c
index 0fe3687e204..ecf31c3b880 100644
--- a/xlators/features/snapview-server/src/snapview-server-mgmt.c
+++ b/xlators/features/snapview-server/src/snapview-server-mgmt.c
@@ -7,475 +7,518 @@
    later), or the GNU General Public License, version 2 (GPLv2), in all
    cases as published by the Free Software Foundation.
 */
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
-
 #include "snapview-server.h"
 #include "snapview-server-mem-types.h"
 #include <pthread.h>
 
 int
-mgmt_cbk_snap (struct rpc_clnt *rpc, void *mydata, void *data)
+mgmt_cbk_snap(struct rpc_clnt *rpc, void *mydata, void *data)
 {
-        xlator_t *this = NULL;
+    xlator_t *this = NULL;
 
-        this = mydata;
-        GF_ASSERT (this);
+    this = mydata;
+    GF_ASSERT(this);
 
-        gf_log ("mgmt", GF_LOG_INFO, "list of snapshots changed");
+    gf_msg("mgmt", GF_LOG_INFO, 0, SVS_MSG_SNAPSHOT_LIST_CHANGED,
+           "list of snapshots changed");
 
-        svs_get_snapshot_list (this);
-        return 0;
+    svs_get_snapshot_list(this);
+    return 0;
 }
 
-rpcclnt_cb_actor_t svs_cbk_actors[GF_CBK_MAXVALUE] = {
-        [GF_CBK_GET_SNAPS] = {"GETSNAPS", GF_CBK_GET_SNAPS, mgmt_cbk_snap},
+static rpcclnt_cb_actor_t svs_cbk_actors[GF_CBK_MAXVALUE] = {
+    [GF_CBK_GET_SNAPS] = {"GETSNAPS", mgmt_cbk_snap, GF_CBK_GET_SNAPS},
 };
 
-struct rpcclnt_cb_program svs_cbk_prog = {
-        .progname  = "GlusterFS Callback",
-        .prognum   = GLUSTER_CBK_PROGRAM,
-        .progver   = GLUSTER_CBK_VERSION,
-        .actors    = svs_cbk_actors,
-        .numactors = GF_CBK_MAXVALUE,
+static struct rpcclnt_cb_program svs_cbk_prog = {
+    .progname = "GlusterFS Callback",
+    .prognum = GLUSTER_CBK_PROGRAM,
+    .progver = GLUSTER_CBK_VERSION,
+    .actors = svs_cbk_actors,
+    .numactors = GF_CBK_MAXVALUE,
 };
 
-char *clnt_handshake_procs[GF_HNDSK_MAXVALUE] = {
-        [GF_HNDSK_NULL]         = "NULL",
-        [GF_HNDSK_EVENT_NOTIFY] = "EVENTNOTIFY",
+static char *clnt_handshake_procs[GF_HNDSK_MAXVALUE] = {
+    [GF_HNDSK_NULL] = "NULL",
+    [GF_HNDSK_EVENT_NOTIFY] = "EVENTNOTIFY",
 };
 
-rpc_clnt_prog_t svs_clnt_handshake_prog = {
-        .progname  = "GlusterFS Handshake",
-        .prognum   = GLUSTER_HNDSK_PROGRAM,
-        .progver   = GLUSTER_HNDSK_VERSION,
-        .procnames = clnt_handshake_procs,
+static rpc_clnt_prog_t svs_clnt_handshake_prog = {
+    .progname = "GlusterFS Handshake",
+    .prognum = GLUSTER_HNDSK_PROGRAM,
+    .progver = GLUSTER_HNDSK_VERSION,
+    .procnames = clnt_handshake_procs,
 };
 
-int
-svs_mgmt_init (xlator_t *this)
+static int
+svs_rpc_notify(struct rpc_clnt *rpc, void *mydata, rpc_clnt_event_t event,
+               void *data)
 {
-        int              ret      = -1;
-        svs_private_t   *priv     = NULL;
-        dict_t          *options  = NULL;
-        int              port     = GF_DEFAULT_BASE_PORT;
-        char            *host     = NULL;
-        cmd_args_t      *cmd_args = NULL;
-        glusterfs_ctx_t *ctx      = NULL;
-
-        GF_VALIDATE_OR_GOTO ("snapview-server", this, out);
-        GF_VALIDATE_OR_GOTO (this->name, this->private, out);
-        GF_VALIDATE_OR_GOTO (this->name, this->ctx, out);
-
-        priv = this->private;
-
-        ctx = this->ctx;
-        cmd_args = &ctx->cmd_args;
-
-        host = "localhost";
-        if (cmd_args->volfile_server)
-                host = cmd_args->volfile_server;
-
-        ret = rpc_transport_inet_options_build (&options, host, port);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "failed to build the "
-                        "transport options");
-                goto out;
-        }
-
-        priv->rpc = rpc_clnt_new (options, this->ctx, this->name, 8);
-        if (!priv->rpc) {
-                gf_log (this->name, GF_LOG_ERROR, "failed to initialize RPC");
-                goto out;
-        }
-
-        ret = rpcclnt_cbk_program_register (priv->rpc, &svs_cbk_prog,
-                                            this);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "failed to register callback program");
-                goto out;
-        }
-
-        ret = rpc_clnt_start (priv->rpc);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "failed to start the rpc "
-                        "client");
-                goto out;
-        }
-
-        ret = 0;
+    xlator_t *this = NULL;
+    int ret = 0;
+
+    this = mydata;
+
+    switch (event) {
+        case RPC_CLNT_CONNECT:
+            ret = svs_get_snapshot_list(this);
+            if (ret) {
+                gf_msg(this->name, GF_LOG_ERROR, EINVAL,
+                       SVS_MSG_GET_SNAPSHOT_LIST_FAILED,
+                       "Error in refreshing the snaplist "
+                       "infrastructure");
+                ret = -1;
+            }
+            break;
+        default:
+            break;
+    }
+    return ret;
+}
 
-        gf_log (this->name, GF_LOG_DEBUG, "svs mgmt init successful");
+int
+svs_mgmt_init(xlator_t *this)
+{
+    int ret = -1;
+    svs_private_t *priv = NULL;
+    dict_t *options = NULL;
+    int port = GF_DEFAULT_BASE_PORT;
+    char *host = NULL;
+    cmd_args_t *cmd_args = NULL;
+    glusterfs_ctx_t *ctx = NULL;
+    xlator_cmdline_option_t *opt = NULL;
+
+    GF_VALIDATE_OR_GOTO("snapview-server", this, out);
+    GF_VALIDATE_OR_GOTO(this->name, this->private, out);
+    GF_VALIDATE_OR_GOTO(this->name, this->ctx, out);
+
+    priv = this->private;
+
+    ctx = this->ctx;
+    cmd_args = &ctx->cmd_args;
+
+    host = "localhost";
+    if (cmd_args->volfile_server)
+        host = cmd_args->volfile_server;
+
+    options = dict_new();
+    if (!options)
+        goto out;
+
+    opt = find_xlator_option_in_cmd_args_t("address-family", cmd_args);
+    ret = rpc_transport_inet_options_build(options, host, port,
+                                           (opt != NULL ? opt->value : NULL));
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, SVS_MSG_BUILD_TRNSPRT_OPT_FAILED,
+               "failed to build the "
+               "transport options");
+        goto out;
+    }
+
+    priv->rpc = rpc_clnt_new(options, this, this->name, 8);
+    if (!priv->rpc) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, SVS_MSG_RPC_INIT_FAILED,
+               "failed to initialize RPC");
+        goto out;
+    }
+
+    ret = rpc_clnt_register_notify(priv->rpc, svs_rpc_notify, this);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_WARNING, 0, SVS_MSG_REG_NOTIFY_FAILED,
+               "failed to register notify function");
+        goto out;
+    }
+
+    ret = rpcclnt_cbk_program_register(priv->rpc, &svs_cbk_prog, this);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, SVS_MSG_REG_CBK_PRGM_FAILED,
+               "failed to register callback program");
+        goto out;
+    }
+
+    ret = rpc_clnt_start(priv->rpc);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, SVS_MSG_RPC_CLNT_START_FAILED,
+               "failed to start the rpc "
+               "client");
+        goto out;
+    }
+
+    ret = 0;
+
+    gf_msg_debug(this->name, 0, "svs mgmt init successful");
 
 out:
-        if (ret)
-                if (priv) {
-                        rpc_clnt_connection_cleanup (&priv->rpc->conn);
-                        rpc_clnt_unref (priv->rpc);
-                        priv->rpc = NULL;
-                }
+    if (options)
+        dict_unref(options);
+    if (ret)
+        if (priv) {
+            rpc_clnt_connection_cleanup(&priv->rpc->conn);
+            rpc_clnt_unref(priv->rpc);
+            priv->rpc = NULL;
+        }
 
-        return ret;
+    return ret;
 }
 
 int
-svs_mgmt_submit_request (void *req, call_frame_t *frame,
-                         glusterfs_ctx_t *ctx,
-                         rpc_clnt_prog_t *prog, int procnum,
-                         fop_cbk_fn_t cbkfn, xdrproc_t xdrproc)
+svs_mgmt_submit_request(void *req, call_frame_t *frame, glusterfs_ctx_t *ctx,
+                        rpc_clnt_prog_t *prog, int procnum, fop_cbk_fn_t cbkfn,
+                        xdrproc_t xdrproc)
 {
-        int                     ret        = -1;
-        int                     count      = 0;
-        struct iovec            iov        = {0, };
-        struct iobuf            *iobuf     = NULL;
-        struct iobref           *iobref    = NULL;
-        ssize_t                 xdr_size   = 0;
-
-        GF_VALIDATE_OR_GOTO ("snapview-server", frame, out);
-        GF_VALIDATE_OR_GOTO ("snapview-server", req, out);
-        GF_VALIDATE_OR_GOTO ("snapview-server", ctx, out);
-        GF_VALIDATE_OR_GOTO ("snapview-server", prog, out);
-
-        GF_ASSERT (frame->this);
-
-        iobref = iobref_new ();
-        if (!iobref) {
-                goto out;
+    int ret = -1;
+    int count = 0;
+    struct iovec iov = {
+        0,
+    };
+    struct iobuf *iobuf = NULL;
+    struct iobref *iobref = NULL;
+    ssize_t xdr_size = 0;
+
+    GF_VALIDATE_OR_GOTO("snapview-server", frame, out);
+    GF_VALIDATE_OR_GOTO("snapview-server", req, out);
+    GF_VALIDATE_OR_GOTO("snapview-server", ctx, out);
+    GF_VALIDATE_OR_GOTO("snapview-server", prog, out);
+
+    GF_ASSERT(frame->this);
+
+    iobref = iobref_new();
+    if (!iobref) {
+        gf_msg(frame->this->name, GF_LOG_WARNING, ENOMEM, SVS_MSG_NO_MEMORY,
+               "failed to allocate "
+               "new iobref");
+        goto out;
+    }
+
+    if (req) {
+        xdr_size = xdr_sizeof(xdrproc, req);
+
+        iobuf = iobuf_get2(ctx->iobuf_pool, xdr_size);
+        if (!iobuf) {
+            goto out;
         }
 
-        if (req) {
-                xdr_size = xdr_sizeof (xdrproc, req);
-
-                iobuf = iobuf_get2 (ctx->iobuf_pool, xdr_size);
-                if (!iobuf) {
-                        goto out;
-                }
-
-                iobref_add (iobref, iobuf);
+        iobref_add(iobref, iobuf);
 
-                iov.iov_base = iobuf->ptr;
-                iov.iov_len  = iobuf_pagesize (iobuf);
+        iov.iov_base = iobuf->ptr;
+        iov.iov_len = iobuf_pagesize(iobuf);
 
-                /* Create the xdr payload */
-                ret = xdr_serialize_generic (iov, req, xdrproc);
-                if (ret == -1) {
-                        gf_log (frame->this->name, GF_LOG_WARNING,
-                                "Failed to create XDR payload");
-                        goto out;
-                }
-                iov.iov_len = ret;
-                count = 1;
+        /* Create the xdr payload */
+        ret = xdr_serialize_generic(iov, req, xdrproc);
+        if (ret == -1) {
+            gf_msg(frame->this->name, GF_LOG_WARNING, 0,
+                   SVS_MSG_XDR_PAYLOAD_FAILED, "Failed to create XDR payload");
+            goto out;
         }
+        iov.iov_len = ret;
+        count = 1;
+    }
 
-        ret = rpc_clnt_submit (ctx->mgmt, prog, procnum, cbkfn,
-                               &iov, count,
-                               NULL, 0, iobref, frame, NULL, 0, NULL, 0, NULL);
+    ret = rpc_clnt_submit(ctx->mgmt, prog, procnum, cbkfn, &iov, count, NULL, 0,
+                          iobref, frame, NULL, 0, NULL, 0, NULL);
 
 out:
-        if (iobref)
-                iobref_unref (iobref);
+    if (iobref)
+        iobref_unref(iobref);
 
-        if (iobuf)
-                iobuf_unref (iobuf);
-        return ret;
+    if (iobuf)
+        iobuf_unref(iobuf);
+    return ret;
 }
 
-
 int
-mgmt_get_snapinfo_cbk (struct rpc_req *req, struct iovec *iov,
-                       int count, void *myframe)
+mgmt_get_snapinfo_cbk(struct rpc_req *req, struct iovec *iov, int count,
+                      void *myframe)
 {
-        gf_getsnap_name_uuid_rsp        rsp             = {0,};
-        call_frame_t                    *frame          = NULL;
-        glusterfs_ctx_t                 *ctx            = NULL;
-        int                             ret             = -1;
-        dict_t                          *dict           = NULL;
-        char                            key[1024]       = {0};
-        int                             snapcount       = 0;
-        svs_private_t                   *priv           = NULL;
-        xlator_t                        *this           = NULL;
-        int                             i               = 0;
-        int                             j               = 0;
-        char                            *value          = NULL;
-        snap_dirent_t                   *dirents        = NULL;
-        snap_dirent_t                   *old_dirents    = NULL;
-        int                             oldcount        = 0;
-
-        GF_VALIDATE_OR_GOTO ("snapview-server", req, error_out);
-        GF_VALIDATE_OR_GOTO ("snapview-server", myframe, error_out);
-        GF_VALIDATE_OR_GOTO ("snapview-server", iov, error_out);
-
-        frame       = myframe;
-        this        = frame->this;
-        ctx         = frame->this->ctx;
-        priv        = this->private;
-        old_dirents = priv->dirents;
-
-        if (!ctx) {
-                gf_log (frame->this->name, GF_LOG_ERROR, "NULL context");
-                errno = EINVAL;
-                goto out;
+    gf_getsnap_name_uuid_rsp rsp = {
+        0,
+    };
+    call_frame_t *frame = NULL;
+    glusterfs_ctx_t *ctx = NULL;
+    int ret = -1;
+    dict_t *dict = NULL;
+    char key[32] = {0};
+    int len;
+    int snapcount = 0;
+    svs_private_t *priv = NULL;
+    xlator_t *this = NULL;
+    int i = 0;
+    int j = 0;
+    char *value = NULL;
+    snap_dirent_t *dirents = NULL;
+    snap_dirent_t *old_dirents = NULL;
+    int oldcount = 0;
+
+    GF_VALIDATE_OR_GOTO("snapview-server", req, error_out);
+    GF_VALIDATE_OR_GOTO("snapview-server", myframe, error_out);
+    GF_VALIDATE_OR_GOTO("snapview-server", iov, error_out);
+
+    frame = myframe;
+    this = frame->this;
+    ctx = frame->this->ctx;
+    priv = this->private;
+
+    if (!ctx) {
+        errno = EINVAL;
+        gf_msg(frame->this->name, GF_LOG_ERROR, errno, SVS_MSG_NULL_CTX,
+               "NULL context");
+        goto out;
+    }
+
+    if (-1 == req->rpc_status) {
+        errno = EINVAL;
+        gf_msg(frame->this->name, GF_LOG_ERROR, errno, SVS_MSG_RPC_CALL_FAILED,
+               "RPC call is not successful");
+        goto out;
+    }
+
+    ret = xdr_to_generic(*iov, &rsp, (xdrproc_t)xdr_gf_getsnap_name_uuid_rsp);
+    if (ret < 0) {
+        gf_msg(frame->this->name, GF_LOG_ERROR, 0, SVS_MSG_XDR_DECODE_FAILED,
+               "Failed to decode xdr response, rsp.op_ret = %d", rsp.op_ret);
+        goto out;
+    }
+
+    if (rsp.op_ret == -1) {
+        errno = rsp.op_errno;
+        ret = -1;
+        goto out;
+    }
+
+    if (!rsp.dict.dict_len) {
+        ret = -1;
+        errno = EINVAL;
+        gf_msg(frame->this->name, GF_LOG_ERROR, errno, SVS_MSG_RSP_DICT_EMPTY,
+               "Response dict is not populated");
+        goto out;
+    }
+
+    dict = dict_new();
+    if (!dict) {
+        ret = -1;
+        errno = ENOMEM;
+        goto out;
+    }
+
+    ret = dict_unserialize(rsp.dict.dict_val, rsp.dict.dict_len, &dict);
+    if (ret) {
+        errno = EINVAL;
+        gf_msg(frame->this->name, GF_LOG_ERROR, errno,
+               LG_MSG_DICT_UNSERIAL_FAILED, "Failed to unserialize dictionary");
+        goto out;
+    }
+
+    ret = dict_get_int32(dict, "snap-count", (int32_t *)&snapcount);
+    if (ret) {
+        errno = EINVAL;
+        ret = -1;
+        gf_msg(this->name, GF_LOG_ERROR, errno, SVS_MSG_DICT_GET_FAILED,
+               "Error retrieving snapcount");
+        goto out;
+    }
+
+    if (snapcount > 0) {
+        /* first time we are fetching snap list */
+        dirents = GF_CALLOC(snapcount, sizeof(snap_dirent_t),
+                            gf_svs_mt_dirents_t);
+        if (!dirents) {
+            errno = ENOMEM;
+            ret = -1;
+            gf_msg(frame->this->name, GF_LOG_ERROR, errno, SVS_MSG_NO_MEMORY,
+                   "Unable to allocate memory");
+            goto out;
         }
+    }
 
-        if (-1 == req->rpc_status) {
-                gf_log (frame->this->name, GF_LOG_ERROR,
-                        "RPC call is not successful");
-                errno = EINVAL;
-                goto out;
-        }
-
-        ret = xdr_to_generic (*iov, &rsp,
-                              (xdrproc_t)xdr_gf_getsnap_name_uuid_rsp);
-        if (ret < 0) {
-                gf_log (frame->this->name, GF_LOG_ERROR,
-                        "Failed to decode xdr response, rsp.op_ret = %d",
-                        rsp.op_ret);
-                goto out;
-        }
-
-        if (rsp.op_ret == -1) {
-                errno = rsp.op_errno;
-                ret = -1;
-                goto out;
-        }
-
-        if (!rsp.dict.dict_len) {
-                gf_log (frame->this->name, GF_LOG_ERROR,
-                        "Response dict is not populated");
-                ret = -1;
-                errno = EINVAL;
-                goto out;
+    for (i = 0; i < snapcount; i++) {
+        len = snprintf(key, sizeof(key), "snap-volname.%d", i + 1);
+        ret = dict_get_strn(dict, key, len, &value);
+        if (ret) {
+            errno = EINVAL;
+            ret = -1;
+            gf_msg(this->name, GF_LOG_ERROR, errno, SVS_MSG_DICT_GET_FAILED,
+                   "Error retrieving snap volname %d", i + 1);
+            goto out;
         }
 
-        dict = dict_new ();
-        if (!dict) {
-                ret = -1;
-                errno = ENOMEM;
-                goto out;
-        }
+        strncpy(dirents[i].snap_volname, value,
+                sizeof(dirents[i].snap_volname));
 
-        ret = dict_unserialize (rsp.dict.dict_val, rsp.dict.dict_len, &dict);
+        len = snprintf(key, sizeof(key), "snap-id.%d", i + 1);
+        ret = dict_get_strn(dict, key, len, &value);
         if (ret) {
-                gf_log (frame->this->name, GF_LOG_ERROR,
-                        "Failed to unserialize dictionary");
-                errno = EINVAL;
-                goto out;
+            errno = EINVAL;
+            ret = -1;
+            gf_msg(this->name, GF_LOG_ERROR, errno, SVS_MSG_DICT_GET_FAILED,
+                   "Error retrieving snap uuid %d", i + 1);
+            goto out;
         }
+        strncpy(dirents[i].uuid, value, sizeof(dirents[i].uuid));
 
-        ret = dict_get_int32 (dict, "snap-count", (int32_t*)&snapcount);
+        len = snprintf(key, sizeof(key), "snapname.%d", i + 1);
+        ret = dict_get_strn(dict, key, len, &value);
         if (ret) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "Error retrieving snapcount");
-                        errno = EINVAL;
-                        ret = -1;
-                        goto out;
+            errno = EINVAL;
+            ret = -1;
+            gf_msg(this->name, GF_LOG_ERROR, errno, SVS_MSG_DICT_GET_FAILED,
+                   "Error retrieving snap name %d", i + 1);
+            goto out;
         }
-
-        if (snapcount > 0) {
-                /* first time we are fetching snap list */
-                dirents = GF_CALLOC (snapcount, sizeof (snap_dirent_t),
-                                     gf_svs_mt_dirents_t);
-                if (!dirents) {
-                        gf_log (frame->this->name, GF_LOG_ERROR,
-                                "Unable to allocate memory");
-                        errno = ENOMEM;
-                        ret = -1;
-                        goto out;
-                }
-        }
-
-        for (i = 0; i < snapcount; i++) {
-                snprintf (key, sizeof (key), "snap-volname.%d", i+1);
-                ret = dict_get_str (dict, key, &value);
-                if (ret) {
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "Error retrieving snap volname %d",
-                                i+1);
-                        errno = EINVAL;
-                        ret = -1;
-                        goto out;
-                }
-
-                strncpy (dirents[i].snap_volname, value,
-                         sizeof (dirents[i].snap_volname));
-
-                snprintf (key, sizeof (key), "snap-id.%d", i+1);
-                ret = dict_get_str (dict, key, &value);
-                if (ret) {
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "Error retrieving snap uuid %d", i+1);
-                        errno = EINVAL;
-                        ret = -1;
-                        goto out;
-                }
-                strncpy (dirents[i].uuid, value,
-                         sizeof (dirents[i].uuid));
-
-                snprintf (key, sizeof (key), "snapname.%d", i+1);
-                ret = dict_get_str (dict, key, &value);
-                if (ret) {
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "Error retrieving snap name %d", i+1);
-                        errno = EINVAL;
-                        ret = -1;
-                        goto out;
-                }
-                strncpy (dirents[i].name, value,
-                         sizeof (dirents[i].name));
-        }
-
-        /*
-         * Got the new snap list populated in dirents
-         * The new snap list is either a subset or a superset of
-         * the existing snaplist old_dirents which has priv->num_snaps
-         * number of entries.
-         *
-         * If subset, then clean up the fs for entries which are
-         * no longer relevant.
-         *
-         * For other overlapping entries set the fs for new dirents
-         * entries which have a fs assigned already in old_dirents
-         *
-         * We do this as we don't want to do new glfs_init()s repeatedly
-         * as the dirents entries for snapshot volumes get repatedly
-         * cleaned up and allocated. And if we don't then that will lead
-         * to memleaks
-         */
-
-        LOCK (&priv->snaplist_lock);
-        {
-                oldcount = priv->num_snaps;
-                for (i = 0; i < priv->num_snaps; i++) {
-                        for (j = 0; j < snapcount; j++) {
-                                if ((!strcmp (old_dirents[i].name,
-                                              dirents[j].name)) &&
-                                    (!strcmp (old_dirents[i].uuid,
-                                              dirents[j].uuid)))    {
-                                        dirents[j].fs = old_dirents[i].fs;
-                                        old_dirents[i].fs = NULL;
-                                        break;
-                                }
-                        }
+        strncpy(dirents[i].name, value, sizeof(dirents[i].name));
+    }
+
+    /*
+     * Got the new snap list populated in dirents
+     * The new snap list is either a subset or a superset of
+     * the existing snaplist old_dirents which has priv->num_snaps
+     * number of entries.
+     *
+     * If subset, then clean up the fs for entries which are
+     * no longer relevant.
+     *
+     * For other overlapping entries set the fs for new dirents
+     * entries which have a fs assigned already in old_dirents
+     *
+     * We do this as we don't want to do new glfs_init()s repeatedly
+     * as the dirents entries for snapshot volumes get repatedly
+     * cleaned up and allocated. And if we don't then that will lead
+     * to memleaks
+     */
+
+    LOCK(&priv->snaplist_lock);
+    {
+        oldcount = priv->num_snaps;
+        old_dirents = priv->dirents;
+        for (i = 0; i < priv->num_snaps; i++) {
+            for (j = 0; j < snapcount; j++) {
+                if ((!strcmp(old_dirents[i].name, dirents[j].name)) &&
+                    (!strcmp(old_dirents[i].uuid, dirents[j].uuid))) {
+                    dirents[j].fs = old_dirents[i].fs;
+                    old_dirents[i].fs = NULL;
+                    break;
                 }
-
-                priv->dirents = dirents;
-                priv->num_snaps = snapcount;
+            }
         }
-        UNLOCK (&priv->snaplist_lock);
 
-        if (old_dirents) {
-                for (i = 0; i < oldcount; i++) {
-                        if (old_dirents[i].fs)
-                                glfs_fini (old_dirents[i].fs);
-                }
+        priv->dirents = dirents;
+        priv->num_snaps = snapcount;
+    }
+    UNLOCK(&priv->snaplist_lock);
+
+    if (old_dirents) {
+        for (i = 0; i < oldcount; i++) {
+            if (old_dirents[i].fs)
+                gf_msg_debug(this->name, 0,
+                             "calling glfs_fini on "
+                             "name: %s, snap_volname: %s, uuid: %s",
+                             old_dirents[i].name, old_dirents[i].snap_volname,
+                             old_dirents[i].uuid);
+            glfs_fini(old_dirents[i].fs);
         }
+    }
 
-        GF_FREE (old_dirents);
+    GF_FREE(old_dirents);
 
-        ret = 0;
+    ret = 0;
 
 out:
-        if (dict) {
-                dict_unref (dict);
-        }
-        free (rsp.dict.dict_val);
-        free (rsp.op_errstr);
+    if (dict) {
+        dict_unref(dict);
+    }
+    free(rsp.dict.dict_val);
+    free(rsp.op_errstr);
 
-        if (ret && dirents) {
-                gf_log (this->name, GF_LOG_WARNING,
-                        "Could not update dirents with refreshed snap list");
-                GF_FREE (dirents);
-        }
+    if (ret && dirents) {
+        gf_msg(this->name, GF_LOG_WARNING, 0, SVS_MSG_SNAP_LIST_REFRESH_FAILED,
+               "Could not update dirents with refreshed snap list");
+        GF_FREE(dirents);
+    }
 
-        if (myframe)
-                SVS_STACK_DESTROY (myframe);
+    if (myframe)
+        SVS_STACK_DESTROY(myframe);
 
 error_out:
-        return ret;
+    return ret;
 }
 
 int
-svs_get_snapshot_list (xlator_t *this)
+svs_get_snapshot_list(xlator_t *this)
 {
-        gf_getsnap_name_uuid_req        req             = {{0,}};
-        int                             ret             = -1;
-        dict_t                          *dict           = NULL;
-        glusterfs_ctx_t                 *ctx            = NULL;
-        call_frame_t                    *frame          = NULL;
-        svs_private_t                   *priv           = NULL;
-        gf_boolean_t                    frame_cleanup   = _gf_true;
-
-        GF_VALIDATE_OR_GOTO ("snapview-server", this, out);
-
-        ctx  = this->ctx;
-        if (!ctx) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "ctx is NULL");
-                goto out;
-        }
-
-        frame = create_frame (this, ctx->pool);
-        if (!frame) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "Error allocating frame");
-                goto out;
-        }
-
-        priv = this->private;
-
-        dict = dict_new ();
-        if (!dict) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "Error allocating dictionary");
-                goto out;
-        }
-
-        ret = dict_set_str (dict, "volname", priv->volname);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "Error setting volname in dict");
-                goto out;
-        }
-
-        ret = dict_allocate_and_serialize (dict, &req.dict.dict_val,
-                                           &req.dict.dict_len);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "Failed to serialize dictionary");
-                ret = -1;
-                goto out;
-        }
-
-        ret = svs_mgmt_submit_request (&req, frame, ctx,
-                                       &svs_clnt_handshake_prog,
-                                       GF_HNDSK_GET_SNAPSHOT_INFO,
-                                       mgmt_get_snapinfo_cbk,
-                                       (xdrproc_t)xdr_gf_getsnap_name_uuid_req);
-
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "Error sending snapshot names RPC request");
-        }
-
-        frame_cleanup = _gf_false;
+    gf_getsnap_name_uuid_req req = {{
+        0,
+    }};
+    int ret = -1;
+    dict_t *dict = NULL;
+    glusterfs_ctx_t *ctx = NULL;
+    call_frame_t *frame = NULL;
+    svs_private_t *priv = NULL;
+    gf_boolean_t frame_cleanup = _gf_true;
+
+    GF_VALIDATE_OR_GOTO("snapview-server", this, out);
+
+    ctx = this->ctx;
+    if (!ctx) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, SVS_MSG_NULL_CTX, "ctx is NULL");
+        goto out;
+    }
+
+    frame = create_frame(this, ctx->pool);
+    if (!frame) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, LG_MSG_FRAME_ERROR,
+               "Error allocating frame");
+        goto out;
+    }
+
+    priv = this->private;
+
+    dict = dict_new();
+    if (!dict) {
+        gf_msg(this->name, GF_LOG_ERROR, ENOMEM, SVS_MSG_NO_MEMORY,
+               "Error allocating dictionary");
+        goto out;
+    }
+
+    ret = dict_set_str(dict, "volname", priv->volname);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, SVS_MSG_DICT_SET_FAILED,
+               "Error setting volname in dict");
+        goto out;
+    }
+
+    ret = dict_allocate_and_serialize(dict, &req.dict.dict_val,
+                                      &req.dict.dict_len);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, LG_MSG_DICT_UNSERIAL_FAILED,
+               "Failed to serialize dictionary");
+        ret = -1;
+        goto out;
+    }
+
+    ret = svs_mgmt_submit_request(
+        &req, frame, ctx, &svs_clnt_handshake_prog, GF_HNDSK_GET_SNAPSHOT_INFO,
+        mgmt_get_snapinfo_cbk, (xdrproc_t)xdr_gf_getsnap_name_uuid_req);
+
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, SVS_MSG_RPC_REQ_FAILED,
+               "Error sending snapshot names RPC request");
+    }
+
+    frame_cleanup = _gf_false;
 
 out:
-        if (dict) {
-                dict_unref (dict);
-        }
-        GF_FREE (req.dict.dict_val);
-
-        if (frame_cleanup && frame) {
-                /*
-                 * Destroy the frame if we encountered an error
-                 * Else we need to clean it up in
-                 * mgmt_get_snapinfo_cbk
-                 */
-                SVS_STACK_DESTROY (frame);
-        }
+    if (dict) {
+        dict_unref(dict);
+    }
+    GF_FREE(req.dict.dict_val);
+
+    if (frame_cleanup && frame) {
+        /*
+         * Destroy the frame if we encountered an error
+         * Else we need to clean it up in
+         * mgmt_get_snapinfo_cbk
+         */
+        SVS_STACK_DESTROY(frame);
+    }
 
-        return ret;
+    return ret;
 }
diff --git a/xlators/features/snapview-server/src/snapview-server.c b/xlators/features/snapview-server/src/snapview-server.c
index 035dd54d1f9..76cccae5914 100644
--- a/xlators/features/snapview-server/src/snapview-server.c
+++ b/xlators/features/snapview-server/src/snapview-server.c
@@ -7,85 +7,121 @@
    later), or the GNU General Public License, version 2 (GPLv2), in all
    cases as published by the Free Software Foundation.
 */
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
-
 #include "snapview-server.h"
 #include "snapview-server-mem-types.h"
-#include "compat-errno.h"
+#include <glusterfs/compat-errno.h>
 
-#include "xlator.h"
+#include <glusterfs/xlator.h>
 #include "rpc-clnt.h"
 #include "xdr-generic.h"
 #include "protocol-common.h"
-#include "syscall.h"
+#include <glusterfs/syscall.h>
 #include <pthread.h>
 
+#include "glfs-internal.h"
+
+int
+gf_setcredentials(uid_t *uid, gid_t *gid, uint16_t ngrps, uint32_t *groups)
+{
+    int ret = 0;
+
+    if (uid) {
+        ret = glfs_setfsuid(*uid);
+        if (ret != 0) {
+            gf_msg("snapview-server", GF_LOG_ERROR, 0, SVS_MSG_SETFSUID_FAIL,
+                   "failed to set uid "
+                   "%u in thread context",
+                   *uid);
+            return ret;
+        }
+    }
+    if (gid) {
+        ret = glfs_setfsgid(*gid);
+        if (ret != 0) {
+            gf_msg("snapview-server", GF_LOG_ERROR, 0, SVS_MSG_SETFSGID_FAIL,
+                   "failed to set gid "
+                   "%u in thread context",
+                   *gid);
+            return ret;
+        }
+    }
+
+    if (ngrps != 0 && groups) {
+        ret = glfs_setfsgroups(ngrps, groups);
+        if (ret != 0) {
+            gf_msg("snapview-server", GF_LOG_ERROR, 0, SVS_MSG_SETFSGRPS_FAIL,
+                   "failed to set "
+                   "groups in thread context");
+            return ret;
+        }
+    }
+    return 0;
+}
 
 int32_t
-svs_lookup_entry_point (xlator_t *this, loc_t *loc, inode_t *parent,
-                        struct iatt *buf, struct iatt *postparent,
-                        int32_t *op_errno)
+svs_lookup_entry_point(xlator_t *this, loc_t *loc, inode_t *parent,
+                       struct iatt *buf, struct iatt *postparent,
+                       int32_t *op_errno)
 {
-        uuid_t         gfid;
-        svs_inode_t   *inode_ctx = NULL;
-        int            op_ret    = -1;
-
-        GF_VALIDATE_OR_GOTO ("snapview-server", this, out);
-        GF_VALIDATE_OR_GOTO (this->name, loc, out);
-        GF_VALIDATE_OR_GOTO (this->name, loc->inode, out);
-        GF_VALIDATE_OR_GOTO (this->name, buf, out);
-        GF_VALIDATE_OR_GOTO (this->name, postparent, out);
-
-        if (gf_uuid_is_null (loc->inode->gfid)) {
-                gf_uuid_generate (gfid);
-                svs_iatt_fill (gfid, buf);
-
-                /* Here the inode context of the entry point directory
-                   is filled with just the type of the inode and the gfid
-                   of the parent from where the entry point was entered.
-                   The glfs object and the fs instance will be NULL.
-                */
-                if (parent)
-                        svs_iatt_fill (parent->gfid, postparent);
-                else {
-                        svs_iatt_fill (buf->ia_gfid, postparent);
-                }
+    uuid_t gfid;
+    svs_inode_t *inode_ctx = NULL;
+    int op_ret = -1;
+
+    GF_VALIDATE_OR_GOTO("snapview-server", this, out);
+    GF_VALIDATE_OR_GOTO(this->name, loc, out);
+    GF_VALIDATE_OR_GOTO(this->name, loc->inode, out);
+    GF_VALIDATE_OR_GOTO(this->name, buf, out);
+    GF_VALIDATE_OR_GOTO(this->name, postparent, out);
+
+    if (gf_uuid_is_null(loc->inode->gfid)) {
+        gf_uuid_generate(gfid);
+        svs_iatt_fill(gfid, buf);
+
+        /* Here the inode context of the entry point directory
+           is filled with just the type of the inode and the gfid
+           of the parent from where the entry point was entered.
+           The glfs object and the fs instance will be NULL.
+        */
+        if (parent)
+            svs_iatt_fill(parent->gfid, postparent);
+        else {
+            svs_iatt_fill(buf->ia_gfid, postparent);
+        }
 
-                inode_ctx = svs_inode_ctx_get_or_new (this, loc->inode);
-                if (!inode_ctx) {
-                        gf_log (this->name, GF_LOG_ERROR, "failed to "
-                                "allocate inode context for entry point "
-                                "directory");
-                        op_ret = -1;
-                        *op_errno = ENOMEM;
-                        goto out;
-                }
-                gf_uuid_copy (inode_ctx->pargfid, loc->pargfid);
-                memcpy (&inode_ctx->buf, buf, sizeof (*buf));
-                inode_ctx->type = SNAP_VIEW_ENTRY_POINT_INODE;
+        inode_ctx = svs_inode_ctx_get_or_new(this, loc->inode);
+        if (!inode_ctx) {
+            op_ret = -1;
+            *op_errno = ENOMEM;
+            gf_msg(this->name, GF_LOG_ERROR, *op_errno,
+                   SVS_MSG_NEW_INODE_CTX_FAILED,
+                   "failed to "
+                   "allocate inode context for entry point "
+                   "directory");
+            goto out;
+        }
+
+        gf_uuid_copy(inode_ctx->pargfid, loc->pargfid);
+        memcpy(&inode_ctx->buf, buf, sizeof(*buf));
+        inode_ctx->type = SNAP_VIEW_ENTRY_POINT_INODE;
+    } else {
+        inode_ctx = svs_inode_ctx_get(this, loc->inode);
+        if (inode_ctx) {
+            memcpy(buf, &inode_ctx->buf, sizeof(*buf));
+            svs_iatt_fill(inode_ctx->pargfid, postparent);
         } else {
-                if (inode_ctx) {
-                        memcpy (buf, &inode_ctx->buf, sizeof (*buf));
-                        svs_iatt_fill (inode_ctx->pargfid, postparent);
-                } else {
-                        svs_iatt_fill (loc->inode->gfid, buf);
-                        if (parent)
-                                svs_iatt_fill (parent->gfid,
-                                               postparent);
-                        else {
-                                svs_iatt_fill (loc->inode->gfid,
-                                               postparent);
-                        }
-                }
+            svs_iatt_fill(loc->inode->gfid, buf);
+            if (parent)
+                svs_iatt_fill(parent->gfid, postparent);
+            else {
+                svs_iatt_fill(loc->inode->gfid, postparent);
+            }
         }
+    }
 
-        op_ret = 0;
+    op_ret = 0;
 
 out:
-        return op_ret;
+    return op_ret;
 }
 
 /* When lookup comes from client and the protocol/server tries to resolve
@@ -108,80 +144,87 @@ out:
    snapshot is referred and a random gfid is not generated.
 */
 int32_t
-svs_lookup_gfid (xlator_t *this, loc_t *loc, struct iatt *buf,
-                 struct iatt *postparent, int32_t *op_errno)
+svs_lookup_gfid(xlator_t *this, loc_t *loc, struct iatt *buf,
+                struct iatt *postparent, int32_t *op_errno)
 {
-        int32_t         op_ret                          = -1;
-        unsigned char   handle_obj[GFAPI_HANDLE_LENGTH] = {0, };
-        glfs_t         *fs                              = NULL;
-        glfs_object_t  *object                          = NULL;
-        struct stat     statbuf                         = {0, };
-        svs_inode_t    *inode_ctx                       = NULL;
-
-        GF_VALIDATE_OR_GOTO ("snapview-server", this, out);
-        GF_VALIDATE_OR_GOTO (this->name, loc, out);
-        GF_VALIDATE_OR_GOTO (this->name, loc->inode, out);
-        GF_VALIDATE_OR_GOTO (this->name, buf, out);
-        GF_VALIDATE_OR_GOTO (this->name, postparent, out);
-
-        if (gf_uuid_is_null (loc->gfid) && gf_uuid_is_null (loc->inode->gfid)) {
-                gf_log (this->name, GF_LOG_ERROR, "gfid is NULL");
-                goto out;
-        }
-
-        if (!gf_uuid_is_null (loc->inode->gfid))
-                memcpy (handle_obj, loc->inode->gfid,
-                        GFAPI_HANDLE_LENGTH);
-        else
-                memcpy (handle_obj, loc->gfid,
-                        GFAPI_HANDLE_LENGTH);
-
-        fs = svs_get_latest_snapshot (this);
-        if (!fs) {
-                gf_log (this->name, GF_LOG_ERROR, "failed to get the latest "
-                        "snapshot");
-                op_ret = -1;
-                *op_errno = EINVAL;
-                goto out;
-        }
-
-
-        object = glfs_h_create_from_handle (fs, handle_obj, GFAPI_HANDLE_LENGTH,
-                                            &statbuf);
-        if (!object) {
-                gf_log (this->name, GF_LOG_ERROR, "failed to do lookup and get "
-                        "the handle on the snapshot %s (path: %s, gfid: %s)",
-                        loc->name, loc->path, uuid_utoa (loc->gfid));
-                op_ret = -1;
-                *op_errno = ESTALE;
-                goto out;
-        }
-
-        inode_ctx = svs_inode_ctx_get_or_new (this, loc->inode);
-        if (!inode_ctx) {
-                gf_log (this->name, GF_LOG_ERROR, "failed to allocate inode "
-                        "context");
-                op_ret = -1;
-                *op_errno = ENOMEM;
-                goto out;
-        }
-
-        iatt_from_stat (buf, &statbuf);
-        if (!gf_uuid_is_null (loc->gfid))
-                gf_uuid_copy (buf->ia_gfid, loc->gfid);
-        else
-                gf_uuid_copy (buf->ia_gfid, loc->inode->gfid);
-
-        inode_ctx->type = SNAP_VIEW_VIRTUAL_INODE;
-        inode_ctx->fs = fs;
-        inode_ctx->object = object;
-        memcpy (&inode_ctx->buf, buf, sizeof (*buf));
-        svs_iatt_fill (buf->ia_gfid, postparent);
-
-        op_ret = 0;
+    int32_t op_ret = -1;
+    unsigned char handle_obj[GFAPI_HANDLE_LENGTH] = {
+        0,
+    };
+    glfs_t *fs = NULL;
+    glfs_object_t *object = NULL;
+    struct stat statbuf = {
+        0,
+    };
+    svs_inode_t *inode_ctx = NULL;
+
+    GF_VALIDATE_OR_GOTO("snapview-server", this, out);
+    GF_VALIDATE_OR_GOTO(this->name, loc, out);
+    GF_VALIDATE_OR_GOTO(this->name, loc->inode, out);
+    GF_VALIDATE_OR_GOTO(this->name, buf, out);
+    GF_VALIDATE_OR_GOTO(this->name, postparent, out);
+
+    if (gf_uuid_is_null(loc->gfid) && gf_uuid_is_null(loc->inode->gfid)) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, SVS_MSG_NULL_GFID, "gfid is NULL");
+        goto out;
+    }
+
+    if (!gf_uuid_is_null(loc->inode->gfid))
+        memcpy(handle_obj, loc->inode->gfid, GFAPI_HANDLE_LENGTH);
+    else
+        memcpy(handle_obj, loc->gfid, GFAPI_HANDLE_LENGTH);
+
+    fs = svs_get_latest_snapshot(this);
+    if (!fs) {
+        op_ret = -1;
+        *op_errno = EINVAL;
+        gf_msg(this->name, GF_LOG_ERROR, *op_errno,
+               SVS_MSG_GET_LATEST_SNAP_FAILED,
+               "failed to get the latest "
+               "snapshot");
+        goto out;
+    }
+
+    object = glfs_h_create_from_handle(fs, handle_obj, GFAPI_HANDLE_LENGTH,
+                                       &statbuf);
+    if (!object) {
+        op_ret = -1;
+        *op_errno = ESTALE;
+        gf_msg(this->name, GF_LOG_ERROR, *op_errno,
+               SVS_MSG_GET_GLFS_H_OBJECT_FAILED,
+               "failed to do lookup and get "
+               "the handle on the snapshot %s (path: %s, gfid: %s)",
+               loc->name, loc->path, uuid_utoa(loc->gfid));
+        goto out;
+    }
+
+    inode_ctx = svs_inode_ctx_get_or_new(this, loc->inode);
+    if (!inode_ctx) {
+        op_ret = -1;
+        *op_errno = ENOMEM;
+        gf_msg(this->name, GF_LOG_ERROR, *op_errno,
+               SVS_MSG_NEW_INODE_CTX_FAILED,
+               "failed to allocate inode "
+               "context");
+        goto out;
+    }
+
+    iatt_from_stat(buf, &statbuf);
+    if (!gf_uuid_is_null(loc->gfid))
+        gf_uuid_copy(buf->ia_gfid, loc->gfid);
+    else
+        gf_uuid_copy(buf->ia_gfid, loc->inode->gfid);
+
+    inode_ctx->type = SNAP_VIEW_VIRTUAL_INODE;
+    inode_ctx->fs = fs;
+    inode_ctx->object = object;
+    memcpy(&inode_ctx->buf, buf, sizeof(*buf));
+    svs_iatt_fill(buf->ia_gfid, postparent);
+
+    op_ret = 0;
 
 out:
-        return op_ret;
+    return op_ret;
 }
 
 /* If the parent is an entry point inode, then create the handle for the
@@ -192,149 +235,213 @@ out:
    parent's context
 */
 int32_t
-svs_lookup_snapshot (xlator_t *this, loc_t *loc, struct iatt *buf,
-                     struct iatt *postparent, inode_t *parent,
-                     svs_inode_t *parent_ctx, int32_t *op_errno)
+svs_lookup_snapshot(xlator_t *this, loc_t *loc, struct iatt *buf,
+                    struct iatt *postparent, inode_t *parent,
+                    svs_inode_t *parent_ctx, int32_t *op_errno)
 {
-        int32_t         op_ret                          = -1;
-        unsigned char   handle_obj[GFAPI_HANDLE_LENGTH] = {0, };
-        glfs_t         *fs                              = NULL;
-        glfs_object_t  *object                          = NULL;
-        struct stat     statbuf                         = {0, };
-        svs_inode_t    *inode_ctx                       = NULL;
-        uuid_t          gfid;
-
-        GF_VALIDATE_OR_GOTO ("snapview-server", this, out);
-        GF_VALIDATE_OR_GOTO (this->name, loc, out);
-        GF_VALIDATE_OR_GOTO (this->name, loc->inode, out);
-        GF_VALIDATE_OR_GOTO (this->name, buf, out);
-        GF_VALIDATE_OR_GOTO (this->name, postparent, out);
-        GF_VALIDATE_OR_GOTO (this->name, parent_ctx, out);
-        GF_VALIDATE_OR_GOTO (this->name, parent, out);
-
-        fs = svs_initialise_snapshot_volume (this, loc->name, op_errno);
-        if (!fs) {
-                gf_log (this->name, GF_LOG_DEBUG, "failed to "
-                        "create the fs instance for snap %s",
-                        loc->name);
-                *op_errno = ENOENT;
-                op_ret = -1;
-                goto out;
-        }
-
-        memcpy (handle_obj, parent_ctx->pargfid,
-                GFAPI_HANDLE_LENGTH);
-        object = glfs_h_create_from_handle (fs, handle_obj, GFAPI_HANDLE_LENGTH,
-                                            &statbuf);
-        if (!object) {
-                gf_log (this->name, GF_LOG_DEBUG, "failed to do lookup and "
-                        "get the handle on the snapshot %s", loc->name);
-                op_ret = -1;
-                *op_errno = errno;
-                goto out;
-        }
-
-        inode_ctx = svs_inode_ctx_get_or_new (this, loc->inode);
-        if (!inode_ctx) {
-                gf_log (this->name, GF_LOG_ERROR, "failed to "
-                        "allocate inode context");
-                op_ret = -1;
-                *op_errno = ENOMEM;
-                goto out;
-        }
+    int32_t op_ret = -1;
+    unsigned char handle_obj[GFAPI_HANDLE_LENGTH] = {
+        0,
+    };
+    glfs_t *fs = NULL;
+    glfs_object_t *object = NULL;
+    struct stat statbuf = {
+        0,
+    };
+    svs_inode_t *inode_ctx = NULL;
+    uuid_t gfid;
+
+    GF_VALIDATE_OR_GOTO("snapview-server", this, out);
+    GF_VALIDATE_OR_GOTO(this->name, loc, out);
+    GF_VALIDATE_OR_GOTO(this->name, loc->inode, out);
+    GF_VALIDATE_OR_GOTO(this->name, buf, out);
+    GF_VALIDATE_OR_GOTO(this->name, postparent, out);
+    GF_VALIDATE_OR_GOTO(this->name, parent_ctx, out);
+    GF_VALIDATE_OR_GOTO(this->name, parent, out);
+
+    fs = svs_initialise_snapshot_volume(this, loc->name, op_errno);
+    if (!fs) {
+        gf_msg_debug(this->name, 0,
+                     "failed to create "
+                     "the fs instance for snap %s",
+                     loc->name);
+        *op_errno = ENOENT;
+        op_ret = -1;
+        goto out;
+    }
+
+    memcpy(handle_obj, parent_ctx->pargfid, GFAPI_HANDLE_LENGTH);
+    object = glfs_h_create_from_handle(fs, handle_obj, GFAPI_HANDLE_LENGTH,
+                                       &statbuf);
+    if (!object) {
+        op_ret = -1;
+        *op_errno = errno;
+        /* Should this be in warning or error mode? */
+        gf_msg_debug(this->name, 0,
+                     "failed to do lookup and "
+                     "get the handle on the snapshot %s",
+                     loc->name);
+        goto out;
+    }
+
+    inode_ctx = svs_inode_ctx_get_or_new(this, loc->inode);
+    if (!inode_ctx) {
+        op_ret = -1;
+        *op_errno = ENOMEM;
+        gf_msg(this->name, GF_LOG_ERROR, *op_errno,
+               SVS_MSG_NEW_INODE_CTX_FAILED,
+               "failed to allocate "
+               "inode context");
+        goto out;
+    }
+
+    if (gf_uuid_is_null(loc->gfid) && gf_uuid_is_null(loc->inode->gfid))
+        gf_uuid_generate(gfid);
+    else {
+        if (!gf_uuid_is_null(loc->inode->gfid))
+            gf_uuid_copy(gfid, loc->inode->gfid);
+        else
+            gf_uuid_copy(gfid, loc->gfid);
+    }
+    iatt_from_stat(buf, &statbuf);
+    gf_uuid_copy(buf->ia_gfid, gfid);
+    svs_fill_ino_from_gfid(buf);
+    inode_ctx->type = SNAP_VIEW_SNAPSHOT_INODE;
+    inode_ctx->fs = fs;
+    inode_ctx->object = object;
+    memcpy(&inode_ctx->buf, buf, sizeof(*buf));
+    svs_iatt_fill(parent->gfid, postparent);
+
+    SVS_STRDUP(inode_ctx->snapname, loc->name);
+    if (!inode_ctx->snapname) {
+        op_ret = -1;
+        *op_errno = ENOMEM;
+        goto out;
+    }
+    op_ret = 0;
 
-        if (gf_uuid_is_null (loc->gfid) &&
-            gf_uuid_is_null (loc->inode->gfid))
-                gf_uuid_generate (gfid);
-        else {
-                if (!gf_uuid_is_null (loc->inode->gfid))
-                        gf_uuid_copy (gfid, loc->inode->gfid);
-                else
-                        gf_uuid_copy (gfid, loc->gfid);
-        }
-        iatt_from_stat (buf, &statbuf);
-        gf_uuid_copy (buf->ia_gfid, gfid);
-        svs_fill_ino_from_gfid (buf);
-        inode_ctx->type = SNAP_VIEW_SNAPSHOT_INODE;
-        inode_ctx->fs = fs;
-        inode_ctx->object = object;
-        memcpy (&inode_ctx->buf, buf, sizeof (*buf));
-        svs_iatt_fill (parent->gfid, postparent);
+out:
+    if (op_ret) {
+        if (object)
+            glfs_h_close(object);
 
-        op_ret = 0;
+        if (inode_ctx)
+            inode_ctx->object = NULL;
+    }
 
-out:
-        return op_ret;
+    return op_ret;
 }
 
 /* Both parent and entry are from snapshot world */
 int32_t
-svs_lookup_entry (xlator_t *this, loc_t *loc, struct iatt *buf,
-                  struct iatt *postparent, inode_t *parent,
-                  svs_inode_t *parent_ctx, int32_t *op_errno)
+svs_lookup_entry(xlator_t *this, loc_t *loc, struct iatt *buf,
+                 struct iatt *postparent, inode_t *parent,
+                 svs_inode_t *parent_ctx, int32_t *op_errno)
 {
-        int32_t         op_ret                          = -1;
-        glfs_t         *fs                              = NULL;
-        glfs_object_t  *object                          = NULL;
-        struct stat     statbuf                         = {0, };
-        svs_inode_t    *inode_ctx                       = NULL;
-        glfs_object_t  *parent_object                   = NULL;
-        uuid_t          gfid;
-
-        GF_VALIDATE_OR_GOTO ("snapview-server", this, out);
-        GF_VALIDATE_OR_GOTO (this->name, loc, out);
-        GF_VALIDATE_OR_GOTO (this->name, loc->inode, out);
-        GF_VALIDATE_OR_GOTO (this->name, buf, out);
-        GF_VALIDATE_OR_GOTO (this->name, postparent, out);
-        GF_VALIDATE_OR_GOTO (this->name, parent_ctx, out);
-        GF_VALIDATE_OR_GOTO (this->name, parent, out);
-
-        parent_object = parent_ctx->object;
-        fs = parent_ctx->fs;
-
-        object = glfs_h_lookupat (fs, parent_object, loc->name,
-                                  &statbuf);
-        if (!object) {
-                gf_log (this->name, GF_LOG_DEBUG, "failed to do lookup and "
-                        "get the handle for entry %s (path: %s)", loc->name,
-                        loc->path);
-                op_ret = -1;
-                *op_errno = errno;
-                goto out;
-        }
+    int32_t op_ret = -1;
+    glfs_t *fs = NULL;
+    glfs_object_t *object = NULL;
+    struct stat statbuf = {
+        0,
+    };
+    svs_inode_t *inode_ctx = NULL;
+    glfs_object_t *parent_object = NULL;
+    uuid_t gfid = {
+        0,
+    };
+
+    GF_VALIDATE_OR_GOTO("snapview-server", this, out);
+    GF_VALIDATE_OR_GOTO(this->name, loc, out);
+    GF_VALIDATE_OR_GOTO(this->name, loc->inode, out);
+    GF_VALIDATE_OR_GOTO(this->name, buf, out);
+    GF_VALIDATE_OR_GOTO(this->name, postparent, out);
+    GF_VALIDATE_OR_GOTO(this->name, parent_ctx, out);
+    GF_VALIDATE_OR_GOTO(this->name, parent, out);
+
+    parent_object = parent_ctx->object;
+    fs = parent_ctx->fs;
+
+    object = glfs_h_lookupat(fs, parent_object, loc->name, &statbuf, 0);
+    if (!object) {
+        /* should this be in WARNING or ERROR mode? */
+        gf_msg_debug(this->name, 0,
+                     "failed to do lookup and "
+                     "get the handle for entry %s (path: %s)",
+                     loc->name, loc->path);
+        op_ret = -1;
+        *op_errno = errno;
+        goto out;
+    }
+
+    if (gf_uuid_is_null(object->gfid)) {
+        /* should this be in WARNING or ERROR mode? */
+        gf_msg_debug(this->name, 0,
+                     "gfid from glfs handle is "
+                     "NULL for entry %s (path: %s)",
+                     loc->name, loc->path);
+        op_ret = -1;
+        *op_errno = errno;
+        goto out;
+    }
+
+    inode_ctx = svs_inode_ctx_get_or_new(this, loc->inode);
+    if (!inode_ctx) {
+        op_ret = -1;
+        *op_errno = ENOMEM;
+        gf_msg(this->name, GF_LOG_ERROR, *op_errno,
+               SVS_MSG_NEW_INODE_CTX_FAILED,
+               "failed to allocate "
+               "inode context");
+        goto out;
+    }
+
+    if (gf_uuid_is_null(loc->gfid) && gf_uuid_is_null(loc->inode->gfid)) {
+        if (svs_uuid_generate(this, gfid, parent_ctx->snapname, object->gfid)) {
+            /*
+             * should op_errno be something else such as
+             * EINVAL or ESTALE?
+             */
+            op_ret = -1;
+            *op_errno = EIO;
+            goto out;
+        }
+    } else {
+        if (!gf_uuid_is_null(loc->inode->gfid))
+            gf_uuid_copy(gfid, loc->inode->gfid);
+        else
+            gf_uuid_copy(gfid, loc->gfid);
+    }
 
-        inode_ctx = svs_inode_ctx_get_or_new (this, loc->inode);
-        if (!inode_ctx) {
-                gf_log (this->name, GF_LOG_ERROR, "failed to "
-                        "allocate inode context");
-                op_ret = -1;
-                *op_errno = ENOMEM;
-                goto out;
-        }
+    iatt_from_stat(buf, &statbuf);
+    gf_uuid_copy(buf->ia_gfid, gfid);
+    svs_fill_ino_from_gfid(buf);
+    inode_ctx->type = SNAP_VIEW_VIRTUAL_INODE;
+    inode_ctx->fs = fs;
+    inode_ctx->object = object;
+    memcpy(&inode_ctx->buf, buf, sizeof(*buf));
+    svs_iatt_fill(parent->gfid, postparent);
 
-        if (gf_uuid_is_null (loc->gfid) &&
-            gf_uuid_is_null (loc->inode->gfid))
-                gf_uuid_generate (gfid);
-        else {
-                if (!gf_uuid_is_null (loc->inode->gfid))
-                        gf_uuid_copy (gfid, loc->inode->gfid);
-                else
-                        gf_uuid_copy (gfid, loc->gfid);
+    if (IA_ISDIR(buf->ia_type)) {
+        SVS_STRDUP(inode_ctx->snapname, parent_ctx->snapname);
+        if (!inode_ctx->snapname) {
+            op_ret = -1;
+            *op_errno = ENOMEM;
+            goto out;
         }
+    }
 
-        iatt_from_stat (buf, &statbuf);
-        gf_uuid_copy (buf->ia_gfid, gfid);
-        svs_fill_ino_from_gfid (buf);
-        inode_ctx->type = SNAP_VIEW_VIRTUAL_INODE;
-        inode_ctx->fs = fs;
-        inode_ctx->object = object;
-        memcpy (&inode_ctx->buf, buf, sizeof (*buf));
-        svs_iatt_fill (parent->gfid, postparent);
-
-        op_ret = 0;
+    op_ret = 0;
 
 out:
-        return op_ret;
+    if (op_ret) {
+        if (object)
+            glfs_h_close(object);
+
+        if (inode_ctx)
+            inode_ctx->object = NULL;
+    }
+
+    return op_ret;
 }
 
 /* inode context is there means lookup has come on an object which was
@@ -363,344 +470,352 @@ out:
      world
 */
 int32_t
-svs_revalidate (xlator_t *this, loc_t *loc, inode_t *parent,
-                svs_inode_t *inode_ctx, svs_inode_t *parent_ctx,
-                struct iatt *buf, struct iatt *postparent, int32_t *op_errno)
+svs_revalidate(xlator_t *this, loc_t *loc, inode_t *parent,
+               svs_inode_t *inode_ctx, svs_inode_t *parent_ctx,
+               struct iatt *buf, struct iatt *postparent, int32_t *op_errno)
 {
-        int32_t         op_ret                          = -1;
-        int             ret                             = -1;
-        char            tmp_uuid[64]                    = {0, };
-        glfs_t         *fs                              = NULL;
-        glfs_object_t  *object                          = NULL;
-
-        GF_VALIDATE_OR_GOTO ("snapview-server", this, out);
-        GF_VALIDATE_OR_GOTO (this->name, buf, out);
-        GF_VALIDATE_OR_GOTO (this->name, postparent, out);
-        GF_VALIDATE_OR_GOTO (this->name, inode_ctx, out);
-
-        if (inode_ctx->type == SNAP_VIEW_ENTRY_POINT_INODE) {
-                svs_iatt_fill (loc->inode->gfid, buf);
+    int32_t op_ret = -1;
+    int ret = -1;
+    char tmp_uuid[64] = {
+        0,
+    };
+    glfs_t *fs = NULL;
+
+    GF_VALIDATE_OR_GOTO("snapview-server", this, out);
+    GF_VALIDATE_OR_GOTO(this->name, buf, out);
+    GF_VALIDATE_OR_GOTO(this->name, postparent, out);
+    GF_VALIDATE_OR_GOTO(this->name, inode_ctx, out);
+
+    if (inode_ctx->type == SNAP_VIEW_ENTRY_POINT_INODE) {
+        svs_iatt_fill(loc->inode->gfid, buf);
+        if (parent)
+            svs_iatt_fill(parent->gfid, postparent);
+        else
+            svs_iatt_fill(loc->inode->gfid, postparent);
+        op_ret = 0;
+        goto out;
+    } else {
+        /* Though fs and object are present in the inode context, its
+         * better to check if fs is valid or not before doing anything.
+         * Its for the protection from the following operations.
+         * 1) Create a file on the glusterfs mount point
+         * 2) Create a snapshot (say "snap1")
+         * 3) Access the contents of the snapshot
+         * 4) Delete the file from the mount point
+         * 5) Delete the snapshot "snap1"
+         * 6) Create a new snapshot "snap1"
+         *
+         * Now accessing the new snapshot "snap1" gives problems.
+         * Because the inode and dentry created for snap1 would not be
+         * deleted upon the deletion of the snapshot (as deletion of
+         * snapshot is a gluster cli operation, not a fop). So next time
+         * upon creation of a new snap with same name, the previous
+         * inode and dentry itself will be used. But the inode context
+         * contains old information about the glfs_t instance and the
+         * handle in the gfapi world. Thus the glfs_t instance should
+         * be checked before accessing. If its wrong, then right
+         * instance should be obtained by doing the lookup.
+         */
+        if (inode_ctx->fs && inode_ctx->object) {
+            fs = inode_ctx->fs;
+            SVS_CHECK_VALID_SNAPSHOT_HANDLE(fs, this);
+            if (fs) {
+                memcpy(buf, &inode_ctx->buf, sizeof(*buf));
                 if (parent)
-                        svs_iatt_fill (parent->gfid,
-                                       postparent);
+                    svs_iatt_fill(parent->gfid, postparent);
                 else
-                        svs_iatt_fill (loc->inode->gfid, postparent);
+                    svs_iatt_fill(buf->ia_gfid, postparent);
                 op_ret = 0;
                 goto out;
-        } else {
-                /* Though fs and object are present in the inode context, its
-                 * better to check if fs is valid or not before doing anything.
-                 * Its for the protection from the following operations.
-                 * 1) Create a file on the glusterfs mount point
-                 * 2) Create a snapshot (say "snap1")
-                 * 3) Access the contents of the snapshot
-                 * 4) Delete the file from the mount point
-                 * 5) Delete the snapshot "snap1"
-                 * 6) Create a new snapshot "snap1"
-                 *
-                 * Now accessing the new snapshot "snap1" gives problems.
-                 * Because the inode and dentry created for snap1 would not be
-                 * deleted upon the deletion of the snapshot (as deletion of
-                 * snapshot is a gluster cli operation, not a fop). So next time
-                 * upon creation of a new snap with same name, the previous
-                 * inode and dentry itself will be used. But the inode context
-                 * contains old information about the glfs_t instance and the
-                 * handle in the gfapi world. Thus the glfs_t instance should
-                 * be checked before accessing. If its wrong, then right
-                 * instance should be obtained by doing the lookup.
-                 */
-                if (inode_ctx->fs && inode_ctx->object) {
-                        fs = inode_ctx->fs;
-                        object = inode_ctx->object;
-                        SVS_CHECK_VALID_SNAPSHOT_HANDLE(fs, this);
-                        if (fs) {
-                                memcpy (buf, &inode_ctx->buf, sizeof (*buf));
-                                if (parent)
-                                        svs_iatt_fill (parent->gfid,
-                                                       postparent);
-                                else
-                                        svs_iatt_fill (buf->ia_gfid,
-                                                       postparent);
-                                op_ret = 0;
-                                goto out;
-                        } else {
-                                inode_ctx->fs = NULL;
-                                inode_ctx->object = NULL;
-                                ret = svs_get_handle (this, loc, inode_ctx,
-                                                      op_errno);
-                                if (ret) {
-                                        gf_log (this->name, GF_LOG_ERROR,
-                                                "failed to get the handle for "
-                                                "%s (gfid %s)", loc->path,
-                                                uuid_utoa_r (loc->inode->gfid,
-                                                             tmp_uuid));
-                                        op_ret = -1;
-                                        goto out;
-                                }
-                        }
+            } else {
+                inode_ctx->fs = NULL;
+                inode_ctx->object = NULL;
+                ret = svs_get_handle(this, loc, inode_ctx, op_errno);
+                if (ret) {
+                    gf_msg(this->name, GF_LOG_ERROR, *op_errno,
+                           SVS_MSG_GET_GLFS_H_OBJECT_FAILED,
+                           "failed to get the handle for "
+                           "%s (gfid %s)",
+                           loc->path, uuid_utoa_r(loc->inode->gfid, tmp_uuid));
+                    op_ret = -1;
+                    goto out;
                 }
+            }
+        }
 
-                /* To send the lookup to gfapi world, both the name of the
-                   entry as well as the parent context is needed.
-                */
-                if (!loc->name || !parent_ctx) {
-                        *op_errno = ESTALE;
-                        gf_log (this->name, GF_LOG_ERROR, "%s is NULL",
-                                loc->name?"parent context":"loc->name");
-                        goto out;
-                }
+        /* To send the lookup to gfapi world, both the name of the
+           entry as well as the parent context is needed.
+        */
+        if (!loc->name || !parent_ctx) {
+            *op_errno = ESTALE;
+            gf_msg(this->name, GF_LOG_ERROR, *op_errno,
+                   SVS_MSG_PARENT_CTX_OR_NAME_NULL, "%s is NULL",
+                   loc->name ? "parent context" : "loc->name");
+            goto out;
+        }
 
-                if (parent_ctx->type == SNAP_VIEW_ENTRY_POINT_INODE)
-                        op_ret = svs_lookup_snapshot (this, loc, buf,
-                                                      postparent, parent,
-                                                      parent_ctx, op_errno);
-                else
-                        op_ret = svs_lookup_entry (this, loc, buf, postparent,
-                                                   parent, parent_ctx,
-                                                   op_errno);
+        if (parent_ctx->type == SNAP_VIEW_ENTRY_POINT_INODE)
+            op_ret = svs_lookup_snapshot(this, loc, buf, postparent, parent,
+                                         parent_ctx, op_errno);
+        else
+            op_ret = svs_lookup_entry(this, loc, buf, postparent, parent,
+                                      parent_ctx, op_errno);
 
-                goto out;
-        }
+        goto out;
+    }
 
 out:
-        return op_ret;
+    return op_ret;
 }
 
 int32_t
-svs_lookup (call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xdata)
+svs_lookup(call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xdata)
 {
-        struct iatt    buf                            = {0, };
-        int32_t        op_ret                         = -1;
-        int32_t        op_errno                       = EINVAL;
-        struct iatt    postparent                     = {0,};
-        svs_inode_t   *inode_ctx                      = NULL;
-        svs_inode_t   *parent_ctx                     = NULL;
-        int32_t        ret                            = -1;
-        svs_private_t *private                        = NULL;
-        inode_t       *parent                         = NULL;
-        snap_dirent_t *dirent                         = NULL;
-        gf_boolean_t   entry_point_key                = _gf_false;
-        gf_boolean_t   entry_point                    = _gf_false;
-
-        GF_VALIDATE_OR_GOTO ("svs", this, out);
-        GF_VALIDATE_OR_GOTO (this->name, this->private, out);
-        GF_VALIDATE_OR_GOTO (this->name, frame, out);
-        GF_VALIDATE_OR_GOTO (this->name, loc, out);
-        GF_VALIDATE_OR_GOTO (this->name, loc->inode, out);
-
-        private = this->private;
-
-        /* For lookups sent on inodes (i.e not parent inode + basename, but
-           direct inode itself which usually is a nameless lookup or revalidate
-           on the inode), loc->name will not be there. Get it from path if
-           it is there.
-           This is the difference between nameless lookup and revalidate lookup
-           on an inode:
-           nameless lookup: loc->path contains gfid and strrchr on it fails
-           revalidate lookup: loc->path contains the entry name of the inode
-                              and strrchr gives the name of the entry from path
-        */
-        if (loc->path) {
-                if (!loc->name || (loc->name && !strcmp (loc->name, ""))) {
-                        loc->name = strrchr (loc->path, '/');
-                        if (loc->name)
-                                loc->name++;
-                }
-        }
-
-        if (loc->parent)
-                parent = inode_ref (loc->parent);
-        else {
-                parent = inode_find (loc->inode->table, loc->pargfid);
-                if (!parent)
-                        parent = inode_parent (loc->inode, NULL, NULL);
-        }
-        if (parent)
-                parent_ctx = svs_inode_ctx_get (this, parent);
-
-        inode_ctx = svs_inode_ctx_get (this, loc->inode);
-
-        /* Initialize latest snapshot, which is used for nameless lookups */
-        dirent = svs_get_latest_snap_entry (this);
-
-        if (dirent && !dirent->fs) {
-                svs_initialise_snapshot_volume (this, dirent->name, NULL);
-        }
-
-        if (xdata && !inode_ctx) {
-                ret = dict_get_str_boolean (xdata, "entry-point", _gf_false);
-                if (ret == -1) {
-                        gf_log (this->name, GF_LOG_DEBUG, "failed to get the "
-                                "entry point info");
-                        entry_point_key = _gf_false;
-                } else {
-                        entry_point_key = ret;
-                }
-
-                if (loc->name && strlen (loc->name)) {
-                        /* lookup can come with the entry-point set in the dict
-                        * for the parent directory of the entry-point as well.
-                        * So consider entry_point only for named lookup
-                        */
-                        entry_point = entry_point_key;
-                }
-        }
-
-        if (inode_ctx && inode_ctx->type == SNAP_VIEW_ENTRY_POINT_INODE) {
-                /* entry-point may not be set in the dictonary.
-                 * This can happen if snap-view client is restarted where
-                 * inode-ctx not available and a nameless lookup has come
-                 */
-                entry_point = _gf_true;
-        }
-
-        /* lookup is on the entry point to the snapshot world */
-        if (entry_point) {
-                op_ret = svs_lookup_entry_point (this, loc, parent, &buf,
-                                                 &postparent, &op_errno);
-                goto out;
+    struct iatt buf = {
+        0,
+    };
+    int32_t op_ret = -1;
+    int32_t op_errno = EINVAL;
+    struct iatt postparent = {
+        0,
+    };
+    svs_inode_t *inode_ctx = NULL;
+    svs_inode_t *parent_ctx = NULL;
+    int32_t ret = -1;
+    inode_t *parent = NULL;
+    gf_boolean_t entry_point_key = _gf_false;
+    gf_boolean_t entry_point = _gf_false;
+    call_stack_t *root = NULL;
+
+    GF_VALIDATE_OR_GOTO("svs", this, out);
+    GF_VALIDATE_OR_GOTO(this->name, this->private, out);
+    GF_VALIDATE_OR_GOTO(this->name, frame, out);
+    GF_VALIDATE_OR_GOTO(this->name, loc, out);
+    GF_VALIDATE_OR_GOTO(this->name, loc->inode, out);
+
+    root = frame->root;
+    op_ret = gf_setcredentials(&root->uid, &root->gid, root->ngrps,
+                               root->groups);
+    if (op_ret != 0) {
+        goto out;
+    }
+
+    /* For lookups sent on inodes (i.e not parent inode + basename, but
+       direct inode itself which usually is a nameless lookup or revalidate
+       on the inode), loc->name will not be there. Get it from path if
+       it is there.
+       This is the difference between nameless lookup and revalidate lookup
+       on an inode:
+       nameless lookup: loc->path contains gfid and strrchr on it fails
+       revalidate lookup: loc->path contains the entry name of the inode
+                          and strrchr gives the name of the entry from path
+    */
+    if (loc->path) {
+        if (!loc->name || (loc->name && !strcmp(loc->name, ""))) {
+            loc->name = strrchr(loc->path, '/');
+            if (loc->name)
+                loc->name++;
+        }
+    }
+
+    if (loc->parent)
+        parent = inode_ref(loc->parent);
+    else {
+        parent = inode_find(loc->inode->table, loc->pargfid);
+        if (!parent)
+            parent = inode_parent(loc->inode, NULL, NULL);
+    }
+    if (parent)
+        parent_ctx = svs_inode_ctx_get(this, parent);
+
+    inode_ctx = svs_inode_ctx_get(this, loc->inode);
+
+    if (xdata && !inode_ctx) {
+        ret = dict_get_str_boolean(xdata, "entry-point", _gf_false);
+        if (ret == -1) {
+            gf_msg_debug(this->name, 0,
+                         "failed to get the "
+                         "entry point info");
+            entry_point_key = _gf_false;
+        } else {
+            entry_point_key = ret;
         }
 
-        /* revalidate */
-        if (inode_ctx) {
-                op_ret = svs_revalidate (this, loc, parent, inode_ctx,
-                                         parent_ctx, &buf, &postparent,
-                                         &op_errno);
-                goto out;
+        if (loc->name && strlen(loc->name)) {
+            /* lookup can come with the entry-point set in the dict
+             * for the parent directory of the entry-point as well.
+             * So consider entry_point only for named lookup
+             */
+            entry_point = entry_point_key;
         }
+    }
 
-        /* This can happen when entry point directory is entered from non-root
-           directory. (ex: if /mnt/glusterfs is the mount point, then entry
-           point (say .snaps) is entered from /mnt/glusterfs/dir/.snaps). Also
-           it can happen when client sends a nameless lookup on just a gfid and
-           the server does not have the inode in the inode table.
-        */
-        if (!inode_ctx && !parent_ctx) {
-                if (gf_uuid_is_null (loc->gfid) &&
-                    gf_uuid_is_null (loc->inode->gfid)) {
-                        gf_log (this->name, GF_LOG_ERROR, "gfid is NULL");
-                        op_ret = -1;
-                        op_errno = ESTALE;
-                        goto out;
-                }
-
-                if (!entry_point_key) {
-                        /* This can happen when there is no inode_ctx available.
-                        * snapview-server might have restarted or
-                        * graph change might have happened
-                        */
-                        op_ret = -1;
-                        op_errno = ESTALE;
-                        goto out;
-                }
-
-                /* lookup is on the parent directory of entry-point.
-                 * this would have already looked up by snap-view client
-                 * so return success
-                 */
-                if (!gf_uuid_is_null (loc->gfid))
-                        gf_uuid_copy (buf.ia_gfid, loc->gfid);
-                else
-                        gf_uuid_copy (buf.ia_gfid, loc->inode->gfid);
+    if (inode_ctx && inode_ctx->type == SNAP_VIEW_ENTRY_POINT_INODE) {
+        /* entry-point may not be set in the dictonary.
+         * This can happen if snap-view client is restarted where
+         * inode-ctx not available and a nameless lookup has come
+         */
+        entry_point = _gf_true;
+    }
+
+    /* lookup is on the entry point to the snapshot world */
+    if (entry_point) {
+        op_ret = svs_lookup_entry_point(this, loc, parent, &buf, &postparent,
+                                        &op_errno);
+        goto out;
+    }
+
+    /* revalidate */
+    if (inode_ctx) {
+        op_ret = svs_revalidate(this, loc, parent, inode_ctx, parent_ctx, &buf,
+                                &postparent, &op_errno);
+        goto out;
+    }
+
+    /* This can happen when entry point directory is entered from non-root
+       directory. (ex: if /mnt/glusterfs is the mount point, then entry
+       point (say .snaps) is entered from /mnt/glusterfs/dir/.snaps). Also
+       it can happen when client sends a nameless lookup on just a gfid and
+       the server does not have the inode in the inode table.
+    */
+    if (!inode_ctx && !parent_ctx) {
+        if (gf_uuid_is_null(loc->gfid) && gf_uuid_is_null(loc->inode->gfid)) {
+            op_ret = -1;
+            op_errno = ESTALE;
+            gf_msg_debug(this->name, 0,
+                         "gfid is NULL. Either the lookup "
+                         "came on missing entry or the "
+                         "entry is stale");
+            goto out;
+        }
+
+        if (!entry_point_key) {
+            /* This can happen when there is no inode_ctx available.
+             * snapview-server might have restarted or
+             * graph change might have happened
+             */
+            op_ret = -1;
+            op_errno = ESTALE;
+            goto out;
+        }
+
+        /* lookup is on the parent directory of entry-point.
+         * this would have already looked up by snap-view client
+         * so return success
+         */
+        if (!gf_uuid_is_null(loc->gfid))
+            gf_uuid_copy(buf.ia_gfid, loc->gfid);
+        else
+            gf_uuid_copy(buf.ia_gfid, loc->inode->gfid);
 
-                svs_iatt_fill (buf.ia_gfid, &buf);
-                svs_iatt_fill (buf.ia_gfid, &postparent);
+        svs_iatt_fill(buf.ia_gfid, &buf);
+        svs_iatt_fill(buf.ia_gfid, &postparent);
 
-                op_ret = 0;
-                goto out;
-        }
+        op_ret = 0;
+        goto out;
+    }
 
-        if (parent_ctx) {
-                if (parent_ctx->type == SNAP_VIEW_ENTRY_POINT_INODE)
-                        op_ret = svs_lookup_snapshot (this, loc, &buf,
-                                                      &postparent, parent,
-                                                      parent_ctx, &op_errno);
-                else
-                        op_ret = svs_lookup_entry (this, loc, &buf,
-                                                   &postparent, parent,
-                                                   parent_ctx, &op_errno);
-                goto out;
-        }
+    if (parent_ctx) {
+        if (parent_ctx->type == SNAP_VIEW_ENTRY_POINT_INODE)
+            op_ret = svs_lookup_snapshot(this, loc, &buf, &postparent, parent,
+                                         parent_ctx, &op_errno);
+        else
+            op_ret = svs_lookup_entry(this, loc, &buf, &postparent, parent,
+                                      parent_ctx, &op_errno);
+        goto out;
+    }
 
 out:
-        STACK_UNWIND_STRICT (lookup, frame, op_ret, op_errno,
-                             loc?loc->inode:NULL, &buf, xdata, &postparent);
+    STACK_UNWIND_STRICT(lookup, frame, op_ret, op_errno,
+                        loc ? loc->inode : NULL, &buf, xdata, &postparent);
 
-        if (parent)
-                inode_unref (parent);
+    if (parent)
+        inode_unref(parent);
 
-        return 0;
+    return 0;
 }
 
 int32_t
-svs_opendir (call_frame_t *frame, xlator_t *this, loc_t *loc, fd_t *fd,
-             dict_t *xdata)
+svs_opendir(call_frame_t *frame, xlator_t *this, loc_t *loc, fd_t *fd,
+            dict_t *xdata)
 {
-        svs_inode_t   *inode_ctx  = NULL;
-        int32_t        op_ret     = -1;
-        int32_t        op_errno   = EINVAL;
-        svs_fd_t      *svs_fd     = NULL;
-        glfs_fd_t     *glfd       = NULL;
-        glfs_t        *fs         = NULL;
-        glfs_object_t *object     = NULL;
-
-        GF_VALIDATE_OR_GOTO ("snap-view-daemon", this, out);
-        GF_VALIDATE_OR_GOTO (this->name, frame, out);
-        GF_VALIDATE_OR_GOTO (this->name, fd, out);
-        GF_VALIDATE_OR_GOTO (this->name, loc, out);
-        GF_VALIDATE_OR_GOTO (this->name, loc->inode, out);
-
-        inode_ctx = svs_inode_ctx_get (this, loc->inode);
-        if (!inode_ctx) {
-                gf_log (this->name, GF_LOG_ERROR, "inode context not found "
-                        "for the inode %s", uuid_utoa (loc->inode->gfid));
-                op_ret = -1;
-                op_errno = ESTALE;
-                goto out;
-        }
-
-        /* Fake success is sent if the opendir is on the entry point directory
-           or the inode is SNAP_VIEW_ENTRY_POINT_INODE
-        */
-        if  (inode_ctx->type == SNAP_VIEW_ENTRY_POINT_INODE) {
-                op_ret = 0;
-                op_errno = 0;
-                goto out;
-        }
-        else {
+    svs_inode_t *inode_ctx = NULL;
+    int32_t op_ret = -1;
+    int32_t op_errno = EINVAL;
+    svs_fd_t *svs_fd = NULL;
+    glfs_fd_t *glfd = NULL;
+    glfs_t *fs = NULL;
+    glfs_object_t *object = NULL;
+    call_stack_t *root = NULL;
+
+    GF_VALIDATE_OR_GOTO("snap-view-daemon", this, out);
+    GF_VALIDATE_OR_GOTO(this->name, frame, out);
+    GF_VALIDATE_OR_GOTO(this->name, fd, out);
+    GF_VALIDATE_OR_GOTO(this->name, loc, out);
+    GF_VALIDATE_OR_GOTO(this->name, loc->inode, out);
+
+    root = frame->root;
+    op_ret = gf_setcredentials(&root->uid, &root->gid, root->ngrps,
+                               root->groups);
+    if (op_ret != 0) {
+        goto out;
+    }
+
+    inode_ctx = svs_inode_ctx_get(this, loc->inode);
+    if (!inode_ctx) {
+        op_ret = -1;
+        op_errno = ESTALE;
+        gf_msg(this->name, GF_LOG_ERROR, op_errno,
+               SVS_MSG_GET_INODE_CONTEXT_FAILED,
+               "inode context not found "
+               "for the inode %s",
+               uuid_utoa(loc->inode->gfid));
+        goto out;
+    }
+
+    /* Fake success is sent if the opendir is on the entry point directory
+       or the inode is SNAP_VIEW_ENTRY_POINT_INODE
+    */
+    if (inode_ctx->type == SNAP_VIEW_ENTRY_POINT_INODE) {
+        op_ret = 0;
+        op_errno = 0;
+        goto out;
+    } else {
+        SVS_GET_INODE_CTX_INFO(inode_ctx, fs, object, this, loc, op_ret,
+                               op_errno, out);
 
-                SVS_GET_INODE_CTX_INFO(inode_ctx, fs, object, this, loc, op_ret,
-                                       op_errno, out);
-
-                glfd = glfs_h_opendir (fs, object);
-                if (!glfd) {
-                        op_ret = -1;
-                        op_errno = errno;
-                        gf_log (this->name, GF_LOG_ERROR, "opendir on %s "
-                                "failed (gfid: %s)", loc->name,
-                                uuid_utoa (loc->inode->gfid));
-                        goto out;
-                }
-                svs_fd = svs_fd_ctx_get_or_new (this, fd);
-                if (!svs_fd) {
-                        gf_log (this->name, GF_LOG_ERROR, "failed to allocate "
-                                "fd context %s (gfid: %s)", loc->name,
-                                uuid_utoa (fd->inode->gfid));
-                        op_ret = -1;
-                        op_errno = ENOMEM;
-                        glfs_closedir (glfd);
-                        goto out;
-                }
-                svs_fd->fd = glfd;
+        glfd = glfs_h_opendir(fs, object);
+        if (!glfd) {
+            op_ret = -1;
+            op_errno = errno;
+            gf_msg(this->name, GF_LOG_ERROR, op_errno, SVS_MSG_OPENDIR_FAILED,
+                   "opendir on %s failed "
+                   "(gfid: %s)",
+                   loc->name, uuid_utoa(loc->inode->gfid));
+            goto out;
+        }
+        svs_fd = svs_fd_ctx_get_or_new(this, fd);
+        if (!svs_fd) {
+            op_ret = -1;
+            op_errno = ENOMEM;
+            gf_msg(this->name, GF_LOG_ERROR, op_errno,
+                   SVS_MSG_NEW_FD_CTX_FAILED,
+                   "failed to allocate fd context "
+                   "for %s (gfid: %s)",
+                   loc->name, uuid_utoa(fd->inode->gfid));
+            glfs_closedir(glfd);
+            goto out;
+        }
+        svs_fd->fd = glfd;
 
-                op_ret = 0;
-                op_errno = 0;
-        }
+        op_ret = 0;
+        op_errno = 0;
+    }
 
 out:
-        STACK_UNWIND_STRICT (opendir, frame, op_ret, op_errno, fd, NULL);
+    STACK_UNWIND_STRICT(opendir, frame, op_ret, op_errno, fd, NULL);
 
-        return 0;
+    return 0;
 }
 
 /*
@@ -713,600 +828,726 @@ out:
  * back into the dict. But to get the values for those xattrs it has to do the
  * getxattr operation on each xattr which might turn out to be a costly
  * operation. So for each of the xattrs present in the list, a 0 byte value
- * ("") is set into the dict before unwinding. This can be treated as an
+ * ("") is set into the dict before unwinding. Since ("") is also a valid xattr
+ * value(in a file system) we use an extra key in the same dictionary as an
  * indicator to other xlators which want to cache the xattrs (as of now,
  * md-cache which caches acl and selinux related xattrs) to not to cache the
  * values of the xattrs present in the dict.
  */
 int32_t
-svs_add_xattrs_to_dict (xlator_t *this, dict_t *dict, char *list, ssize_t size)
+svs_add_xattrs_to_dict(xlator_t *this, dict_t *dict, char *list, ssize_t size)
 {
-        char           keybuffer[4096]  = {0,};
-        size_t         remaining_size   = 0;
-        int32_t        list_offset      = 0;
-        int32_t        ret              = -1;
-
-        GF_VALIDATE_OR_GOTO ("snapview-daemon", this, out);
-        GF_VALIDATE_OR_GOTO (this->name, dict, out);
-        GF_VALIDATE_OR_GOTO (this->name, list, out);
-
-        remaining_size = size;
-        list_offset = 0;
-        while (remaining_size > 0) {
-                strncpy (keybuffer, list + list_offset, sizeof (keybuffer) - 1);
+    char keybuffer[4096] = {
+        0,
+    };
+    size_t remaining_size = 0;
+    int32_t list_offset = 0;
+    int32_t ret = -1;
+
+    GF_VALIDATE_OR_GOTO("snapview-daemon", this, out);
+    GF_VALIDATE_OR_GOTO(this->name, dict, out);
+    GF_VALIDATE_OR_GOTO(this->name, list, out);
+
+    remaining_size = size;
+    list_offset = 0;
+    while (remaining_size > 0) {
+        strncpy(keybuffer, list + list_offset, sizeof(keybuffer) - 1);
 #ifdef GF_DARWIN_HOST_OS
-                /* The protocol expect namespace for now */
-                char *newkey = NULL;
-                gf_add_prefix (XATTR_USER_PREFIX, keybuffer, &newkey);
-                strcpy (keybuffer, newkey);
-                GF_FREE (newkey);
+        /* The protocol expect namespace for now */
+        char *newkey = NULL;
+        gf_add_prefix(XATTR_USER_PREFIX, keybuffer, &newkey);
+        strcpy(keybuffer, newkey);
+        GF_FREE(newkey);
 #endif
-                ret = dict_set_str (dict, keybuffer, "");
-                if (ret < 0) {
-                        gf_log (this->name, GF_LOG_ERROR, "dict set operation "
-                                "for the key %s failed.", keybuffer);
-                        goto out;
-                }
+        ret = dict_set_str(dict, keybuffer, "");
+        if (ret < 0) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, SVS_MSG_DICT_SET_FAILED,
+                   "dict set operation "
+                   "for the key %s failed.",
+                   keybuffer);
+            goto out;
+        }
 
-                remaining_size -= strlen (keybuffer) + 1;
-                list_offset += strlen (keybuffer) + 1;
-        } /* while (remaining_size > 0) */
+        remaining_size -= strlen(keybuffer) + 1;
+        list_offset += strlen(keybuffer) + 1;
+    } /* while (remaining_size > 0) */
 
-        ret = 0;
+    /* Add an additional key to indicate that we don't need to cache these
+     * xattrs(with value "") */
+    ret = dict_set_str(dict, "glusterfs.skip-cache", "");
+    if (ret < 0) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, SVS_MSG_DICT_SET_FAILED,
+               "dict set operation for the key glusterfs.skip-cache failed.");
+        goto out;
+    }
+
+    ret = 0;
 
 out:
-        return ret;
+    return ret;
 }
 
 int32_t
-svs_getxattr (call_frame_t *frame, xlator_t *this, loc_t *loc, const char *name,
-              dict_t *xdata)
+svs_getxattr(call_frame_t *frame, xlator_t *this, loc_t *loc, const char *name,
+             dict_t *xdata)
 {
-        svs_inode_t   *inode_ctx        = NULL;
-        int32_t        op_ret           = -1;
-        int32_t        op_errno         = EINVAL;
-        glfs_t        *fs               = NULL;
-        glfs_object_t *object           = NULL;
-        char          *value            = 0;
-        ssize_t        size             = 0;
-        dict_t        *dict             = NULL;
-
-        GF_VALIDATE_OR_GOTO ("snap-view-daemon", this, out);
-        GF_VALIDATE_OR_GOTO ("snap-view-daemon", frame, out);
-        GF_VALIDATE_OR_GOTO ("snap-view-daemon", loc, out);
-        GF_VALIDATE_OR_GOTO ("snap-view-daemon", loc->inode, out);
-
-        inode_ctx = svs_inode_ctx_get (this, loc->inode);
-        if (!inode_ctx) {
-                gf_log (this->name, GF_LOG_ERROR, "inode context not found "
-                        "for the inode %s", uuid_utoa (loc->inode->gfid));
-                op_ret = -1;
-                op_errno = ESTALE;
-                goto out;
-        }
+    svs_inode_t *inode_ctx = NULL;
+    int32_t op_ret = -1;
+    int32_t op_errno = EINVAL;
+    glfs_t *fs = NULL;
+    glfs_object_t *object = NULL;
+    char *value = 0;
+    ssize_t size = 0;
+    dict_t *dict = NULL;
+    call_stack_t *root = NULL;
+
+    GF_VALIDATE_OR_GOTO("snap-view-daemon", this, out);
+    GF_VALIDATE_OR_GOTO("snap-view-daemon", frame, out);
+    GF_VALIDATE_OR_GOTO("snap-view-daemon", loc, out);
+    GF_VALIDATE_OR_GOTO("snap-view-daemon", loc->inode, out);
+
+    root = frame->root;
+    op_ret = gf_setcredentials(&root->uid, &root->gid, root->ngrps,
+                               root->groups);
+    if (op_ret != 0) {
+        goto out;
+    }
+
+    inode_ctx = svs_inode_ctx_get(this, loc->inode);
+    if (!inode_ctx) {
+        op_ret = -1;
+        op_errno = ESTALE;
+        gf_msg(this->name, GF_LOG_ERROR, op_errno,
+               SVS_MSG_GET_INODE_CONTEXT_FAILED,
+               "inode context not found "
+               "for the inode %s",
+               uuid_utoa(loc->inode->gfid));
+        goto out;
+    }
+
+    /* ENODATA is sent if the getxattr is on entry point directory
+       or the inode is SNAP_VIEW_ENTRY_POINT_INODE. Entry point is
+       a virtual directory on which setxattr operations are not
+       allowed. If getxattr has to be faked as success, then a value
+       for the name of the xattr has to be sent which we don't have.
+    */
+    if (inode_ctx->type == SNAP_VIEW_ENTRY_POINT_INODE) {
+        op_ret = -1;
+        op_errno = ENODATA;
+        goto out;
+    } else {
+        SVS_GET_INODE_CTX_INFO(inode_ctx, fs, object, this, loc, op_ret,
+                               op_errno, out);
 
-        /* ENODATA is sent if the getxattr is on entry point directory
-           or the inode is SNAP_VIEW_ENTRY_POINT_INODE. Entry point is
-           a virtual directory on which setxattr operations are not
-           allowed. If getxattr has to be faked as success, then a value
-           for the name of the xattr has to be sent which we dont have.
-        */
-        if (inode_ctx->type == SNAP_VIEW_ENTRY_POINT_INODE) {
-                op_ret = -1;
-                op_errno = ENODATA;
+        dict = dict_new();
+        if (!dict) {
+            op_ret = -1;
+            op_errno = ENOMEM;
+            gf_msg(this->name, GF_LOG_ERROR, op_errno, SVS_MSG_NO_MEMORY,
+                   "failed to allocate dict");
+            goto out;
+        }
+
+        size = glfs_h_getxattrs(fs, object, name, NULL, 0);
+        if (size == -1) {
+            op_ret = -1;
+            op_errno = errno;
+            if (errno == ENODATA) {
+                gf_msg_debug(this->name, 0,
+                             "getxattr on "
+                             "%s failed (ket: %s) with %s",
+                             loc->path, name, strerror(errno));
+            } else {
+                gf_msg(this->name, GF_LOG_ERROR, op_errno,
+                       SVS_MSG_GETXATTR_FAILED,
+                       "getxattr on %s failed (key: %s) with %s", loc->path,
+                       name, strerror(errno));
+            }
+            goto out;
+        }
+        value = GF_CALLOC(size + 1, sizeof(char), gf_common_mt_char);
+        if (!value) {
+            op_ret = -1;
+            op_errno = ENOMEM;
+            gf_msg(this->name, GF_LOG_ERROR, op_errno, SVS_MSG_NO_MEMORY,
+                   "failed to allocate memory for getxattr "
+                   "on %s (key: %s)",
+                   loc->name, name);
+            goto out;
+        }
+
+        size = glfs_h_getxattrs(fs, object, name, value, size);
+        if (size == -1) {
+            op_ret = -1;
+            op_errno = errno;
+            gf_msg(this->name, GF_LOG_ERROR, op_errno, SVS_MSG_GETXATTR_FAILED,
+                   "failed to get the xattr %s for "
+                   "entry %s",
+                   name, loc->name);
+            goto out;
+        }
+        value[size] = '\0';
+
+        if (name) {
+            op_ret = dict_set_dynptr(dict, (char *)name, value, size);
+            if (op_ret < 0) {
+                op_errno = -op_ret;
+                gf_msg(this->name, GF_LOG_ERROR, op_errno,
+                       SVS_MSG_DICT_SET_FAILED,
+                       "dict set operation for %s for "
+                       "the key %s failed.",
+                       loc->path, name);
+                GF_FREE(value);
+                value = NULL;
                 goto out;
+            }
+        } else {
+            op_ret = svs_add_xattrs_to_dict(this, dict, value, size);
+            if (op_ret == -1) {
+                op_errno = ENOMEM;
+                gf_msg(this->name, GF_LOG_ERROR, op_errno, SVS_MSG_NO_MEMORY,
+                       "failed to add xattrs from the list to "
+                       "dict for %s (gfid: %s)",
+                       loc->path, uuid_utoa(loc->inode->gfid));
+                goto out;
+            }
+            GF_FREE(value);
+            value = NULL;
         }
-        else {
-
-                SVS_GET_INODE_CTX_INFO(inode_ctx, fs, object, this, loc, op_ret,
-                                       op_errno, out);
-
-                dict = dict_new ();
-                if (!dict) {
-                        gf_log (this->name, GF_LOG_ERROR, "failed to "
-                                "allocate dict");
-                        op_ret = -1;
-                        op_errno = ENOMEM;
-                        goto out;
-                }
-
-                size = glfs_h_getxattrs (fs, object, name, NULL, 0);
-                if (size == -1) {
-                        gf_log (this->name, GF_LOG_ERROR, "getxattr "
-                                "on %s failed (key: %s)", loc->name,
-                                name);
-                        op_ret = -1;
-                        op_errno = errno;
-                        goto out;
-                }
-                value = GF_CALLOC (size + 1, sizeof (char),
-                                   gf_common_mt_char);
-                if (!value) {
-                        gf_log (this->name, GF_LOG_ERROR, "failed to "
-                                "allocate memory for getxattr on %s "
-                                "(key: %s)", loc->name, name);
-                        op_ret = -1;
-                        op_errno = ENOMEM;
-                        goto out;
-                }
-
-                size = glfs_h_getxattrs (fs, object, name, value, size);
-                if (size == -1) {
-                        gf_log (this->name, GF_LOG_ERROR, "failed to "
-                                "get the xattr %s for entry %s", name,
-                                loc->name);
-                        op_ret = -1;
-                        op_errno = errno;
-                        goto out;
-                }
-                value[size] = '\0';
-
-                if (name) {
-                        op_ret = dict_set_dynptr (dict, (char *)name, value,
-                                                  size);
-                        if (op_ret < 0) {
-                                op_errno = -op_ret;
-                                gf_log (this->name, GF_LOG_ERROR, "dict set "
-                                        "operation for %s for the key %s "
-                                        "failed.", loc->path, name);
-                                GF_FREE (value);
-                                value = NULL;
-                                goto out;
-                        }
-                } else {
-                        op_ret = svs_add_xattrs_to_dict (this, dict, value,
-                                                         size);
-                        if (op_ret == -1) {
-                                gf_log (this->name, GF_LOG_ERROR, "failed to "
-                                        "add the xattrs from the list to dict");
-                                op_errno = ENOMEM;
-                                goto out;
-                        }
-                        GF_FREE (value);
-                }
-        }
+    }
 
 out:
-        if (op_ret)
-                GF_FREE (value);
+    if (op_ret && value)
+        GF_FREE(value);
 
-        STACK_UNWIND_STRICT (getxattr, frame, op_ret, op_errno, dict, NULL);
+    STACK_UNWIND_STRICT(getxattr, frame, op_ret, op_errno, dict, NULL);
 
-        if (dict)
-                dict_unref (dict);
+    if (dict)
+        dict_unref(dict);
 
-        return 0;
+    return 0;
 }
 
 int32_t
-svs_fgetxattr (call_frame_t *frame, xlator_t *this, fd_t *fd, const char *name,
-               dict_t *xdata)
+svs_fgetxattr(call_frame_t *frame, xlator_t *this, fd_t *fd, const char *name,
+              dict_t *xdata)
 {
-        svs_inode_t *inode_ctx  = NULL;
-        int32_t      op_ret     = -1;
-        int32_t      op_errno   = EINVAL;
-        char        *value      = 0;
-        ssize_t      size       = 0;
-        dict_t      *dict       = NULL;
-        svs_fd_t    *sfd        = NULL;
-        glfs_fd_t   *glfd       = NULL;
-
-        GF_VALIDATE_OR_GOTO ("snap-view-daemon", this, out);
-        GF_VALIDATE_OR_GOTO ("snap-view-daemon", frame, out);
-        GF_VALIDATE_OR_GOTO ("snap-view-daemon", fd, out);
-        GF_VALIDATE_OR_GOTO ("snap-view-daemon", fd->inode, out);
-
-        inode_ctx = svs_inode_ctx_get (this, fd->inode);
-        if (!inode_ctx) {
-                gf_log (this->name, GF_LOG_ERROR, "inode context not found "
-                        "for the inode %s", uuid_utoa (fd->inode->gfid));
+    svs_inode_t *inode_ctx = NULL;
+    int32_t op_ret = -1;
+    int32_t op_errno = EINVAL;
+    char *value = 0;
+    ssize_t size = 0;
+    dict_t *dict = NULL;
+    svs_fd_t *sfd = NULL;
+    glfs_fd_t *glfd = NULL;
+
+    GF_VALIDATE_OR_GOTO("snap-view-daemon", this, out);
+    GF_VALIDATE_OR_GOTO("snap-view-daemon", frame, out);
+    GF_VALIDATE_OR_GOTO("snap-view-daemon", fd, out);
+    GF_VALIDATE_OR_GOTO("snap-view-daemon", fd->inode, out);
+
+    inode_ctx = svs_inode_ctx_get(this, fd->inode);
+    if (!inode_ctx) {
+        op_ret = -1;
+        op_errno = ESTALE;
+        gf_msg(this->name, GF_LOG_ERROR, op_errno,
+               SVS_MSG_GET_INODE_CONTEXT_FAILED,
+               "inode context not found "
+               "for the inode %s",
+               uuid_utoa(fd->inode->gfid));
+        goto out;
+    }
+
+    if (!(svs_inode_ctx_glfs_mapping(this, inode_ctx))) {
+        op_ret = -1;
+        op_errno = EBADF;
+        gf_msg(this->name, GF_LOG_ERROR, op_errno, SVS_MSG_FS_INSTANCE_INVALID,
+               "glfs instance %p to which the inode %s "
+               "belongs to does not exist. The snapshot "
+               "corresponding to the instance might have"
+               "been deleted or deactivated",
+               inode_ctx->fs, uuid_utoa(fd->inode->gfid));
+        goto out;
+    }
+
+    sfd = svs_fd_ctx_get_or_new(this, fd);
+    if (!sfd) {
+        op_ret = -1;
+        op_errno = EBADFD;
+        gf_msg(this->name, GF_LOG_ERROR, op_errno,
+               SVS_MSG_GET_FD_CONTEXT_FAILED,
+               "failed to get the fd "
+               "context for %s",
+               uuid_utoa(fd->inode->gfid));
+        goto out;
+    }
+
+    glfd = sfd->fd;
+    /* EINVAL is sent if the getxattr is on entry point directory
+       or the inode is SNAP_VIEW_ENTRY_POINT_INODE. Entry point is
+       a virtual directory on which setxattr operations are not
+       allowed. If getxattr has to be faked as success, then a value
+       for the name of the xattr has to be sent which we don't have.
+    */
+    if (inode_ctx->type == SNAP_VIEW_ENTRY_POINT_INODE) {
+        op_ret = -1;
+        op_errno = EINVAL;
+        goto out;
+    } else {
+        dict = dict_new();
+        if (!dict) {
+            op_ret = -1;
+            op_errno = ENOMEM;
+            gf_msg(this->name, GF_LOG_ERROR, op_errno, SVS_MSG_NO_MEMORY,
+                   "failed to allocate dict "
+                   "(gfid: %s, key: %s)",
+                   uuid_utoa(fd->inode->gfid), name);
+            goto out;
+        }
+
+        if (name) {
+            size = glfs_fgetxattr(glfd, name, NULL, 0);
+            if (size == -1) {
                 op_ret = -1;
-                op_errno = ESTALE;
+                op_errno = errno;
+                gf_msg(this->name, GF_LOG_ERROR, op_errno,
+                       SVS_MSG_GETXATTR_FAILED,
+                       "getxattr on %s failed "
+                       "(key: %s)",
+                       uuid_utoa(fd->inode->gfid), name);
                 goto out;
-        }
+            }
+            value = GF_CALLOC(size + 1, sizeof(char), gf_common_mt_char);
+            if (!value) {
+                op_ret = -1;
+                op_errno = ENOMEM;
+                gf_msg(this->name, GF_LOG_ERROR, op_errno, SVS_MSG_NO_MEMORY,
+                       "failed to "
+                       "allocate memory for getxattr on %s "
+                       "(key: %s)",
+                       uuid_utoa(fd->inode->gfid), name);
+                goto out;
+            }
 
-        sfd = svs_fd_ctx_get_or_new (this, fd);
-        if (!sfd) {
-                gf_log (this->name, GF_LOG_ERROR, "failed to get the fd "
-                        "context for %s", uuid_utoa (fd->inode->gfid));
+            size = glfs_fgetxattr(glfd, name, value, size);
+            if (size == -1) {
                 op_ret = -1;
-                op_errno = EBADFD;
+                op_errno = errno;
+                gf_msg(this->name, GF_LOG_ERROR, op_errno,
+                       SVS_MSG_GETXATTR_FAILED,
+                       "failed to get the xattr %s "
+                       "for inode %s",
+                       name, uuid_utoa(fd->inode->gfid));
                 goto out;
-        }
+            }
+            value[size] = '\0';
+
+            op_ret = dict_set_dynptr(dict, (char *)name, value, size);
+            if (op_ret < 0) {
+                op_errno = -op_ret;
+                gf_msg(this->name, GF_LOG_ERROR, op_errno,
+                       SVS_MSG_DICT_SET_FAILED,
+                       "dict set operation for gfid %s "
+                       "for the key %s failed.",
+                       uuid_utoa(fd->inode->gfid), name);
+                goto out;
+            }
+        } else {
+            size = glfs_flistxattr(glfd, NULL, 0);
+            if (size == -1) {
+                op_errno = errno;
+                gf_msg(this->name, GF_LOG_ERROR, op_errno,
+                       SVS_MSG_LISTXATTR_FAILED, "listxattr on %s failed",
+                       uuid_utoa(fd->inode->gfid));
+                goto out;
+            }
 
-        glfd = sfd->fd;
-        /* EINVAL is sent if the getxattr is on entry point directory
-           or the inode is SNAP_VIEW_ENTRY_POINT_INODE. Entry point is
-           a virtual directory on which setxattr operations are not
-           allowed. If getxattr has to be faked as success, then a value
-           for the name of the xattr has to be sent which we dont have.
-        */
-        if (inode_ctx->type == SNAP_VIEW_ENTRY_POINT_INODE) {
+            value = GF_CALLOC(size + 1, sizeof(char), gf_common_mt_char);
+            if (!value) {
                 op_ret = -1;
-                op_errno = EINVAL;
+                op_errno = ENOMEM;
+                gf_msg(this->name, GF_LOG_ERROR, op_errno, SVS_MSG_NO_MEMORY,
+                       "failed to "
+                       "allocate buffer for xattr "
+                       "list (%s)",
+                       uuid_utoa(fd->inode->gfid));
                 goto out;
-        }
-        else {
-                dict = dict_new ();
-                if (!dict) {
-                        gf_log (this->name, GF_LOG_ERROR, "failed to "
-                                "allocate  dict");
-                        op_ret = -1;
-                        op_errno = ENOMEM;
-                        goto out;
-                }
+            }
 
-                if (name) {
-                        size = glfs_fgetxattr (glfd, name, NULL, 0);
-                        if (size == -1) {
-                                gf_log (this->name, GF_LOG_ERROR, "getxattr on "
-                                        "%s failed (key: %s)",
-                                        uuid_utoa (fd->inode->gfid), name);
-                                op_ret = -1;
-                                op_errno = errno;
-                                goto out;
-                        }
-                        value = GF_CALLOC (size + 1, sizeof (char),
-                                           gf_common_mt_char);
-                        if (!value) {
-                                gf_log (this->name, GF_LOG_ERROR, "failed to "
-                                        "allocate memory for getxattr on %s "
-                                        "(key: %s)",
-                                        uuid_utoa (fd->inode->gfid), name);
-                                op_ret = -1;
-                                op_errno = ENOMEM;
-                                goto out;
-                        }
-
-                        size = glfs_fgetxattr (glfd, name, value, size);
-                        if (size == -1) {
-                                gf_log (this->name, GF_LOG_ERROR, "failed to "
-                                        "get the xattr %s for inode %s", name,
-                                        uuid_utoa (fd->inode->gfid));
-                                op_ret = -1;
-                                op_errno = errno;
-                                goto out;
-                        }
-                        value[size] = '\0';
-
-                        op_ret = dict_set_dynptr (dict, (char *)name, value,
-                                                  size);
-                        if (op_ret < 0) {
-                                op_errno = -op_ret;
-                                gf_log (this->name, GF_LOG_ERROR, "dict set "
-                                        "operation for gfid %s for the key %s "
-                                        "failed.",
-                                        uuid_utoa (fd->inode->gfid), name);
-                                GF_FREE (value);
-                                goto out;
-                        }
-                } else {
-                        size = glfs_flistxattr (glfd, NULL, 0);
-                        if (size == -1) {
-                                gf_log (this->name, GF_LOG_ERROR, "listxattr "
-                                        "on %s failed",
-                                        uuid_utoa (fd->inode->gfid));
-                                goto out;
-                        }
-
-                        value = GF_CALLOC (size + 1, sizeof (char),
-                                           gf_common_mt_char);
-                        if (!value) {
-                                op_ret = -1;
-                                op_errno = ENOMEM;
-                                gf_log (this->name, GF_LOG_ERROR, "failed to "
-                                        "allocate buffer for xattr list (%s)",
-                                        uuid_utoa (fd->inode->gfid));
-                                goto out;
-                        }
-
-                        size = glfs_flistxattr (glfd, value, size);
-                        if (size == -1) {
-                                op_ret = -1;
-                                op_errno = errno;
-                                gf_log (this->name, GF_LOG_ERROR, "listxattr "
-                                        "on %s failed",
-                                        uuid_utoa (fd->inode->gfid));
-                                goto out;
-                        }
-
-                        op_ret = svs_add_xattrs_to_dict (this, dict, value,
-                                                         size);
-                        if (op_ret == -1) {
-                                gf_log (this->name, GF_LOG_ERROR, "failed to "
-                                        "add the xattrs from the list to dict");
-                                op_errno = ENOMEM;
-                                goto out;
-                        }
-                        GF_FREE (value);
-                }
+            size = glfs_flistxattr(glfd, value, size);
+            if (size == -1) {
+                op_ret = -1;
+                op_errno = errno;
+                gf_msg(this->name, GF_LOG_ERROR, op_errno,
+                       SVS_MSG_LISTXATTR_FAILED, "listxattr on %s failed",
+                       uuid_utoa(fd->inode->gfid));
+                goto out;
+            }
 
-                op_ret = 0;
-                op_errno = 0;
+            op_ret = svs_add_xattrs_to_dict(this, dict, value, size);
+            if (op_ret == -1) {
+                op_errno = ENOMEM;
+                gf_msg(this->name, GF_LOG_ERROR, op_errno, SVS_MSG_NO_MEMORY,
+                       "failed to add xattrs from the list "
+                       "to dict (gfid: %s)",
+                       uuid_utoa(fd->inode->gfid));
+                goto out;
+            }
+            GF_FREE(value);
         }
 
+        op_ret = 0;
+        op_errno = 0;
+    }
+
 out:
-        if (op_ret)
-                GF_FREE (value);
+    if (op_ret)
+        GF_FREE(value);
 
-        STACK_UNWIND_STRICT (fgetxattr, frame, op_ret, op_errno, dict, NULL);
+    STACK_UNWIND_STRICT(fgetxattr, frame, op_ret, op_errno, dict, NULL);
 
-        if (dict)
-                dict_unref (dict);
+    if (dict)
+        dict_unref(dict);
 
-        return 0;
+    return 0;
 }
 
 int32_t
-svs_releasedir (xlator_t *this, fd_t *fd)
+svs_releasedir(xlator_t *this, fd_t *fd)
 {
-        svs_fd_t *sfd      = NULL;
-        uint64_t          tmp_pfd  = 0;
-        int               ret      = 0;
-
-        GF_VALIDATE_OR_GOTO ("snapview-server", this, out);
-        GF_VALIDATE_OR_GOTO (this->name, fd, out);
-
-        ret = fd_ctx_del (fd, this, &tmp_pfd);
-        if (ret < 0) {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "pfd from fd=%p is NULL", fd);
-                goto out;
-        }
-
-        sfd = (svs_fd_t *)(long)tmp_pfd;
-        if (sfd->fd) {
-                ret = glfs_closedir (sfd->fd);
+    svs_fd_t *sfd = NULL;
+    uint64_t tmp_pfd = 0;
+    int ret = 0;
+    svs_inode_t *svs_inode = NULL;
+    glfs_t *fs = NULL;
+    inode_t *inode = NULL;
+
+    GF_VALIDATE_OR_GOTO("snapview-server", this, out);
+    GF_VALIDATE_OR_GOTO(this->name, fd, out);
+
+    ret = fd_ctx_del(fd, this, &tmp_pfd);
+    if (ret < 0) {
+        gf_msg_debug(this->name, 0, "pfd from fd=%p is NULL", fd);
+        goto out;
+    }
+
+    inode = fd->inode;
+
+    svs_inode = svs_inode_ctx_get(this, inode);
+    if (svs_inode) {
+        fs = svs_inode->fs; /* should inode->lock be held for this? */
+        SVS_CHECK_VALID_SNAPSHOT_HANDLE(fs, this);
+        if (fs) {
+            sfd = (svs_fd_t *)(long)tmp_pfd;
+            if (sfd->fd) {
+                ret = glfs_closedir(sfd->fd);
                 if (ret)
-                        gf_log (this->name, GF_LOG_WARNING, "failed to close "
-                                "the glfd for directory %s",
-                                uuid_utoa (fd->inode->gfid));
+                    gf_msg(this->name, GF_LOG_WARNING, errno,
+                           SVS_MSG_RELEASEDIR_FAILED,
+                           "failed to close the glfd for "
+                           "directory %s",
+                           uuid_utoa(fd->inode->gfid));
+            }
         }
+    }
 
-        GF_FREE (sfd);
+    GF_FREE(sfd);
 
 out:
-        return 0;
+    return 0;
 }
 
 int32_t
-svs_flush (call_frame_t *frame, xlator_t *this,
-           fd_t *fd, dict_t *xdata)
+svs_flush(call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *xdata)
 {
-        int32_t          op_ret         = -1;
-        int32_t          op_errno       = 0;
-        int              ret            = -1;
-        uint64_t         value          = 0;
-        svs_inode_t     *inode_ctx      = NULL;
-
-        GF_VALIDATE_OR_GOTO ("snapview-server", this, out);
-        GF_VALIDATE_OR_GOTO (this->name, frame, out);
-        GF_VALIDATE_OR_GOTO (this->name, fd, out);
-
-        inode_ctx = svs_inode_ctx_get (this, fd->inode);
-        if (!inode_ctx) {
-                gf_log (this->name, GF_LOG_ERROR, "inode context not found for"
-                        " the inode %s", uuid_utoa (fd->inode->gfid));
-                op_ret = -1;
-                op_errno = EINVAL;
-                goto out;
-        }
-
-        ret = fd_ctx_get (fd, this, &value);
-        if (ret < 0 && inode_ctx->type != SNAP_VIEW_ENTRY_POINT_INODE) {
-                op_errno = EINVAL;
-                gf_log (this->name, GF_LOG_WARNING,
-                        "pfd is NULL on fd=%p", fd);
-                goto out;
-        }
-
-        op_ret = 0;
+    int32_t op_ret = -1;
+    int32_t op_errno = 0;
+    int ret = -1;
+    uint64_t value = 0;
+    svs_inode_t *inode_ctx = NULL;
+    call_stack_t *root = NULL;
+
+    GF_VALIDATE_OR_GOTO("snapview-server", this, out);
+    GF_VALIDATE_OR_GOTO(this->name, frame, out);
+    GF_VALIDATE_OR_GOTO(this->name, fd, out);
+
+    root = frame->root;
+    op_ret = gf_setcredentials(&root->uid, &root->gid, root->ngrps,
+                               root->groups);
+    if (op_ret != 0) {
+        goto out;
+    }
+
+    inode_ctx = svs_inode_ctx_get(this, fd->inode);
+    if (!inode_ctx) {
+        op_ret = -1;
+        op_errno = EINVAL;
+        gf_msg(this->name, GF_LOG_ERROR, op_errno,
+               SVS_MSG_GET_INODE_CONTEXT_FAILED,
+               "inode context not found for"
+               " the inode %s",
+               uuid_utoa(fd->inode->gfid));
+        goto out;
+    }
+
+    ret = fd_ctx_get(fd, this, &value);
+    if (ret < 0 && inode_ctx->type != SNAP_VIEW_ENTRY_POINT_INODE) {
+        op_errno = EINVAL;
+        gf_msg(this->name, GF_LOG_WARNING, op_errno,
+               SVS_MSG_GET_FD_CONTEXT_FAILED, "pfd is NULL on fd=%p", fd);
+        goto out;
+    }
+
+    op_ret = 0;
 
 out:
-        STACK_UNWIND_STRICT (flush, frame, op_ret, op_errno, NULL);
+    STACK_UNWIND_STRICT(flush, frame, op_ret, op_errno, NULL);
 
-        return 0;
+    return 0;
 }
 
 int32_t
-svs_release (xlator_t *this, fd_t *fd)
+svs_release(xlator_t *this, fd_t *fd)
 {
-        svs_fd_t *sfd      = NULL;
-        uint64_t          tmp_pfd  = 0;
-        int               ret      = 0;
-
-        GF_VALIDATE_OR_GOTO ("snapview-server", this, out);
-        GF_VALIDATE_OR_GOTO (this->name, fd, out);
-
-        ret = fd_ctx_del (fd, this, &tmp_pfd);
-        if (ret < 0) {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "pfd from fd=%p is NULL", fd);
-                goto out;
-        }
-
-        sfd = (svs_fd_t *)(long)tmp_pfd;
-        if (sfd->fd) {
-                ret = glfs_close (sfd->fd);
-                if (ret) {
-                        gf_log (this->name, GF_LOG_ERROR, "failed to close "
-                                "the glfd for %s",
-                                uuid_utoa (fd->inode->gfid));
-                }
+    svs_fd_t *sfd = NULL;
+    uint64_t tmp_pfd = 0;
+    int ret = 0;
+    inode_t *inode = NULL;
+    svs_inode_t *svs_inode = NULL;
+    glfs_t *fs = NULL;
+
+    GF_VALIDATE_OR_GOTO("snapview-server", this, out);
+    GF_VALIDATE_OR_GOTO(this->name, fd, out);
+
+    ret = fd_ctx_del(fd, this, &tmp_pfd);
+    if (ret < 0) {
+        gf_msg_debug(this->name, 0, "pfd from fd=%p is NULL", fd);
+        goto out;
+    }
+
+    inode = fd->inode;
+
+    svs_inode = svs_inode_ctx_get(this, inode);
+    if (svs_inode) {
+        fs = svs_inode->fs; /* should inode->lock be held for this? */
+        SVS_CHECK_VALID_SNAPSHOT_HANDLE(fs, this);
+        if (fs) {
+            sfd = (svs_fd_t *)(long)tmp_pfd;
+            if (sfd->fd) {
+                ret = glfs_close(sfd->fd);
+                if (ret)
+                    gf_msg(this->name, GF_LOG_ERROR, errno,
+                           SVS_MSG_RELEASE_FAILED,
+                           "failed to close "
+                           "the glfd for %s",
+                           uuid_utoa(fd->inode->gfid));
+            }
         }
+    }
 
-        GF_FREE (sfd);
+    GF_FREE(sfd);
 out:
-        return 0;
+    return 0;
 }
 
 int32_t
-svs_forget  (xlator_t *this, inode_t *inode)
+svs_forget(xlator_t *this, inode_t *inode)
 {
-        int       ret = -1;
-        uint64_t  value = 0;
-        svs_inode_t  *inode_ctx = NULL;
-
-        GF_VALIDATE_OR_GOTO ("snapview-server", this, out);
-        GF_VALIDATE_OR_GOTO (this->name, inode, out);
-
-        ret = inode_ctx_del (inode, this, &value);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "failed to delte the inode "
-                        "context of %s", uuid_utoa (inode->gfid));
-                goto out;
-        }
-
-        inode_ctx = (svs_inode_t *)value;
-
-        if (inode_ctx->object)
-                glfs_h_close (inode_ctx->object);
-
-        GF_FREE (inode_ctx);
+    int ret = -1;
+    uint64_t value = 0;
+    svs_inode_t *inode_ctx = NULL;
+
+    GF_VALIDATE_OR_GOTO("snapview-server", this, out);
+    GF_VALIDATE_OR_GOTO(this->name, inode, out);
+
+    ret = inode_ctx_del(inode, this, &value);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, SVS_MSG_DELETE_INODE_CONTEXT_FAILED,
+               "failed to delete the inode "
+               "context of %s",
+               uuid_utoa(inode->gfid));
+        goto out;
+    }
+
+    inode_ctx = (svs_inode_t *)(uintptr_t)value;
+    if (!inode_ctx)
+        goto out;
+
+    if (inode_ctx->snapname)
+        GF_FREE(inode_ctx->snapname);
+
+    /*
+     * glfs_h_close leads to unref and forgetting of the
+     * underlying inode in the gfapi world. i.e. the inode
+     * which inode_ctx->object points to.
+     * As of now the only possibility is, this forget came as a
+     * result of snapdaemon's inode table reaching the lru
+     * limit and receiving forget as a result of purging of
+     * extra inodes that exceeded the limit. But, care must
+     * be taken to ensure that, the gfapi instance to which
+     * the glfs_h_object belongs to is not deleted. Otherwise
+     * this might result in access of a freed pointer.
+     * This will still be helpful in reducing the memory
+     * footprint of snapdaemon when the fs instance itself is
+     * valid (i.e. present and not destroyed due to either snap
+     * deactivate or snap delete), but the lru limit is reached.
+     * The forget due to lru limit will make the underlying inode
+     * being unrefed and forgotten.
+     */
+    if (svs_inode_ctx_glfs_mapping(this, inode_ctx)) {
+        glfs_h_close(inode_ctx->object);
+        inode_ctx->object = NULL;
+    }
+    GF_FREE(inode_ctx);
 
 out:
-        return 0;
+    return 0;
 }
 
 int
-svs_fill_readdir (xlator_t *this, gf_dirent_t *entries, size_t size, off_t off)
+svs_fill_readdir(xlator_t *this, gf_dirent_t *entries, size_t size, off_t off)
 {
-        gf_dirent_t             *entry          = NULL;
-        svs_private_t           *priv           = NULL;
-        int                     i               = 0;
-        snap_dirent_t           *dirents        = NULL;
-        int                     this_size       = 0;
-        int                     filled_size     = 0;
-        int                     count           = 0;
-
-        GF_VALIDATE_OR_GOTO ("snap-view-daemon", this, out);
-        GF_VALIDATE_OR_GOTO ("snap-view-daemon", entries, out);
-
-        priv = this->private;
-        GF_ASSERT (priv);
-
-        /* create the dir entries */
-        LOCK (&priv->snaplist_lock);
-        {
-                dirents = priv->dirents;
-
-                for (i = off; i < priv->num_snaps; ) {
-                        this_size = sizeof (gf_dirent_t) +
-                                strlen (dirents[i].name) + 1;
-                        if (this_size + filled_size > size )
-                                goto unlock;
-
-                        entry = gf_dirent_for_name (dirents[i].name);
-                        if (!entry) {
-                                gf_log (this->name, GF_LOG_ERROR,
-                                        "failed to allocate dentry for %s",
-                                        dirents[i].name);
-                                goto unlock;
-                        }
-
-                        entry->d_off = i + 1;
-                        /*
-                         * readdir on the entry-point directory to the snapshot
-                         * world, will return elements in the list of the
-                         * snapshots as the directory entries. Since the entries
-                         * returned are virtual entries which does not exist
-                         * physically on the disk, pseudo inode numbers are
-                         * generated.
-                         */
-                        entry->d_ino = i + 2*42;
-                        entry->d_type = DT_DIR;
-                        list_add_tail (&entry->list, &entries->list);
-                        ++i;
-                        count++;
-                        filled_size += this_size;
-                }
-        }
+    gf_dirent_t *entry = NULL;
+    svs_private_t *priv = NULL;
+    int i = 0;
+    snap_dirent_t *dirents = NULL;
+    int this_size = 0;
+    int filled_size = 0;
+    int count = 0;
+
+    GF_VALIDATE_OR_GOTO("snap-view-daemon", this, out);
+    GF_VALIDATE_OR_GOTO("snap-view-daemon", entries, out);
+
+    priv = this->private;
+    GF_ASSERT(priv);
+
+    /* create the dir entries */
+    LOCK(&priv->snaplist_lock);
+    {
+        dirents = priv->dirents;
+
+        for (i = off; i < priv->num_snaps;) {
+            this_size = sizeof(gf_dirent_t) + strlen(dirents[i].name) + 1;
+            if (this_size + filled_size > size)
+                goto unlock;
+
+            entry = gf_dirent_for_name(dirents[i].name);
+            if (!entry) {
+                gf_msg(this->name, GF_LOG_ERROR, ENOMEM, SVS_MSG_NO_MEMORY,
+                       "failed to allocate dentry for %s", dirents[i].name);
+                goto unlock;
+            }
+
+            entry->d_off = i + 1;
+            /*
+             * readdir on the entry-point directory to the snapshot
+             * world, will return elements in the list of the
+             * snapshots as the directory entries. Since the entries
+             * returned are virtual entries which does not exist
+             * physically on the disk, pseudo inode numbers are
+             * generated.
+             */
+            entry->d_ino = i + 2 * 42;
+            entry->d_type = DT_DIR;
+            list_add_tail(&entry->list, &entries->list);
+            ++i;
+            count++;
+            filled_size += this_size;
+        }
+    }
 unlock:
-        UNLOCK (&priv->snaplist_lock);
+    UNLOCK(&priv->snaplist_lock);
 
 out:
-        return count;
+    return count;
 }
 
 int32_t
-svs_glfs_readdir (xlator_t *this, glfs_fd_t *glfd, gf_dirent_t *entries,
-                  int32_t *op_errno, struct iatt *buf, gf_boolean_t readdirplus,
-                  size_t size)
+svs_glfs_readdir(xlator_t *this, glfs_fd_t *glfd, gf_dirent_t *entries,
+                 int32_t *op_errno, struct iatt *buf, gf_boolean_t readdirplus,
+                 size_t size)
 {
-        int              filled_size    = 0;
-        int              this_size      = 0;
-        int32_t          ret            = -1;
-        int32_t          count          = 0;
-        gf_dirent_t     *entry          = NULL;
-        struct   dirent *dirents        = NULL;
-        struct   dirent  de             = {0, };
-        struct stat      statbuf        = {0, };
-        off_t            in_case        = -1;
-
-        GF_VALIDATE_OR_GOTO ("svs", this, out);
-        GF_VALIDATE_OR_GOTO (this->name, glfd, out);
-        GF_VALIDATE_OR_GOTO (this->name, entries, out);
-        GF_VALIDATE_OR_GOTO (this->name, buf, out);
-
-        while (filled_size < size) {
-                in_case = glfs_telldir (glfd);
-                if (in_case == -1) {
-                        gf_log (this->name, GF_LOG_ERROR, "telldir failed");
-                        break;
-                }
-
-                if (readdirplus)
-                        ret = glfs_readdirplus_r (glfd, &statbuf, &de,
-                                                  &dirents);
-                else
-                        ret = glfs_readdir_r (glfd, &de, &dirents);
-
-                if (ret == 0 && dirents != NULL) {
-                        if (readdirplus)
-                                this_size = max (sizeof (gf_dirent_t),
-                                                 sizeof (gfs3_dirplist))
-                                        + strlen (de.d_name) + 1;
-                        else
-                                this_size = sizeof (gf_dirent_t)
-                                        + strlen (de.d_name) + 1;
-
-                        if (this_size + filled_size > size) {
-                                glfs_seekdir (glfd, in_case);
-                                break;
-                        }
-
-                        entry = gf_dirent_for_name (de.d_name);
-                        if (!entry) {
-                                gf_log (this->name, GF_LOG_ERROR,
-                                        "could not create gf_dirent "
-                                        "for entry %s: (%s)",
-                                        entry->d_name,
-                                        strerror (errno));
-                                break;
-                        }
-                        entry->d_off = glfs_telldir (glfd);
-                        entry->d_ino = de.d_ino;
-                        entry->d_type = de.d_type;
-                        iatt_from_stat (buf, &statbuf);
-                        if (readdirplus)
-                                entry->d_stat = *buf;
-                        list_add_tail (&entry->list, &entries->list);
-
-                        filled_size += this_size;
-                        count++;
-                } else if (ret == 0 && dirents == NULL) {
-                        *op_errno = ENOENT;
-                        break;
-                } else if (ret != 0) {
-                        *op_errno = errno;
-                        break;
-                }
-                dirents = NULL;
-                ret = -1;
-        }
+    int filled_size = 0;
+    int this_size = 0;
+    int32_t ret = -1;
+    int32_t count = 0;
+    gf_dirent_t *entry = NULL;
+    struct dirent *dirents = NULL;
+    struct dirent de = {
+        0,
+    };
+    struct stat statbuf = {
+        0,
+    };
+    off_t in_case = -1;
+
+    GF_VALIDATE_OR_GOTO("svs", this, out);
+    GF_VALIDATE_OR_GOTO(this->name, glfd, out);
+    GF_VALIDATE_OR_GOTO(this->name, entries, out);
+
+    while (filled_size < size) {
+        in_case = glfs_telldir(glfd);
+        if (in_case == -1) {
+            gf_msg(this->name, GF_LOG_ERROR, errno, SVS_MSG_TELLDIR_FAILED,
+                   "telldir failed");
+            break;
+        }
+
+        if (readdirplus)
+            ret = glfs_readdirplus_r(glfd, &statbuf, &de, &dirents);
+        else
+            ret = glfs_readdir_r(glfd, &de, &dirents);
+
+        if (ret == 0 && dirents != NULL) {
+            if (readdirplus)
+                this_size = max(sizeof(gf_dirent_t), sizeof(gfs3_dirplist)) +
+                            strlen(de.d_name) + 1;
+            else
+                this_size = sizeof(gf_dirent_t) + strlen(de.d_name) + 1;
+
+            if (this_size + filled_size > size) {
+                glfs_seekdir(glfd, in_case);
+                break;
+            }
+
+            entry = gf_dirent_for_name(de.d_name);
+            if (!entry) {
+                /*
+                 * Since gf_dirent_for_name can return
+                 * NULL only when it fails to allocate
+                 * memory for the directory entry,
+                 * SVS_MSG_NO_MEMORY is used as the
+                 * message-id.
+                 */
+                gf_msg(this->name, GF_LOG_ERROR, errno, SVS_MSG_NO_MEMORY,
+                       "could not create gf_dirent "
+                       "for entry %s: (%s)",
+                       entry->d_name, strerror(errno));
+                break;
+            }
+            entry->d_off = glfs_telldir(glfd);
+            entry->d_ino = de.d_ino;
+            entry->d_type = de.d_type;
+            if (readdirplus) {
+                iatt_from_stat(buf, &statbuf);
+                entry->d_stat = *buf;
+            }
+            list_add_tail(&entry->list, &entries->list);
+
+            filled_size += this_size;
+            count++;
+        } else if (ret == 0 && dirents == NULL) {
+            *op_errno = ENOENT;
+            break;
+        } else if (ret != 0) {
+            *op_errno = errno;
+            break;
+        }
+        dirents = NULL;
+    }
 
 out:
-        return count;
+    return count;
 }
 
 /* readdirp can be of 2 types.
@@ -1321,80 +1562,97 @@ out:
       numbers will be newly generated and filled in.
 */
 void
-svs_readdirp_fill (xlator_t *this, inode_t *parent, svs_inode_t *parent_ctx,
-                   gf_dirent_t *entry)
+svs_readdirp_fill(xlator_t *this, inode_t *parent, svs_inode_t *parent_ctx,
+                  gf_dirent_t *entry)
 {
-        inode_t                *inode          = NULL;
-        uuid_t                  random_gfid    = {0,};
-        struct  iatt            buf            = {0, };
-        svs_inode_t            *inode_ctx      = NULL;
-
-        GF_VALIDATE_OR_GOTO ("snapview-server", this, out);
-        GF_VALIDATE_OR_GOTO (this->name, parent, out);
-        GF_VALIDATE_OR_GOTO (this->name, parent_ctx, out);
-        GF_VALIDATE_OR_GOTO (this->name, entry, out);
-
-        if (!strcmp (entry->d_name, ".") || !strcmp (entry->d_name, ".."))
-                goto out;
-
-        inode = inode_grep (parent->table, parent, entry->d_name);
-        if (inode) {
-                entry->inode = inode;
-                inode_ctx = svs_inode_ctx_get (this, inode);
-                if (!inode_ctx) {
-                        gf_uuid_copy (buf.ia_gfid, inode->gfid);
-                        svs_iatt_fill (inode->gfid, &buf);
-                        buf.ia_type = inode->ia_type;
-                } else {
-                        buf = inode_ctx->buf;
-                }
-
-                entry->d_ino = buf.ia_ino;
-
-                if (parent_ctx->type == SNAP_VIEW_ENTRY_POINT_INODE)
-                        entry->d_stat = buf;
-                else {
-                        entry->d_stat.ia_ino = buf.ia_ino;
-                        gf_uuid_copy (entry->d_stat.ia_gfid, buf.ia_gfid);
-                }
+    inode_t *inode = NULL;
+    uuid_t random_gfid = {
+        0,
+    };
+    struct iatt buf = {
+        0,
+    };
+    svs_inode_t *inode_ctx = NULL;
+
+    GF_VALIDATE_OR_GOTO("snapview-server", this, out);
+    GF_VALIDATE_OR_GOTO(this->name, parent, out);
+    GF_VALIDATE_OR_GOTO(this->name, parent_ctx, out);
+    GF_VALIDATE_OR_GOTO(this->name, entry, out);
+
+    if (!strcmp(entry->d_name, ".") || !strcmp(entry->d_name, ".."))
+        goto out;
+
+    inode = inode_grep(parent->table, parent, entry->d_name);
+    if (inode) {
+        entry->inode = inode;
+        inode_ctx = svs_inode_ctx_get(this, inode);
+        if (!inode_ctx) {
+            gf_uuid_copy(buf.ia_gfid, inode->gfid);
+            svs_iatt_fill(inode->gfid, &buf);
+            buf.ia_type = inode->ia_type;
         } else {
-                inode = inode_new (parent->table);
-                entry->inode = inode;
-                gf_uuid_generate (random_gfid);
-                gf_uuid_copy (buf.ia_gfid, random_gfid);
-                svs_fill_ino_from_gfid (&buf);
-                entry->d_ino = buf.ia_ino;
-
-                /* If inode context allocation fails, then do not send the
-                   inode for that particular entry as part of readdirp
-                   response. Fuse and protocol/server will link the inodes
-                   in readdirp only if the entry contains inode in it.
-                */
-                inode_ctx = svs_inode_ctx_get_or_new (this, inode);
-                if (!inode_ctx) {
-                        gf_log (this->name, GF_LOG_ERROR, "failed to allocate "
-                                "inode context for %s", entry->d_name);
-                        inode_unref (entry->inode);
-                        entry->inode = NULL;
-                        goto out;
-                }
+            buf = inode_ctx->buf;
+        }
 
+        entry->d_ino = buf.ia_ino;
 
-                if (parent_ctx->type == SNAP_VIEW_ENTRY_POINT_INODE) {
-                        buf.ia_type = IA_IFDIR;
-                        inode_ctx->buf = buf;
-                        entry->d_stat = buf;
-                        inode_ctx->type = SNAP_VIEW_SNAPSHOT_INODE;
-                } else {
-                        gf_uuid_copy (entry->d_stat.ia_gfid, buf.ia_gfid);
-                        entry->d_stat.ia_ino = buf.ia_ino;
-                        inode_ctx->buf = entry->d_stat;
-                        inode_ctx->type = SNAP_VIEW_VIRTUAL_INODE;
-                }
+        if (parent_ctx->type == SNAP_VIEW_ENTRY_POINT_INODE)
+            entry->d_stat = buf;
+        else {
+            entry->d_stat.ia_ino = buf.ia_ino;
+            gf_uuid_copy(entry->d_stat.ia_gfid, buf.ia_gfid);
         }
+    } else {
+        if (parent_ctx->type == SNAP_VIEW_ENTRY_POINT_INODE) {
+            inode = inode_new(parent->table);
+            entry->inode = inode;
+
+            /* If inode context allocation fails, then do not send
+             * the inode for that particular entry as part of
+             * readdirp response. Fuse and protocol/server will link
+             * the inodes in readdirp only if the entry contains
+             * inode in it.
+             */
+            inode_ctx = svs_inode_ctx_get_or_new(this, inode);
+            if (!inode_ctx) {
+                gf_msg(this->name, GF_LOG_ERROR, ENOMEM, SVS_MSG_NO_MEMORY,
+                       "failed to allocate inode "
+                       "context for %s",
+                       entry->d_name);
+                inode_unref(entry->inode);
+                entry->inode = NULL;
+                goto out;
+            }
+
+            /* Generate virtual gfid for SNAPSHOT dir and
+             * update the statbuf
+             */
+            gf_uuid_generate(random_gfid);
+            gf_uuid_copy(buf.ia_gfid, random_gfid);
+            svs_fill_ino_from_gfid(&buf);
+            buf.ia_type = IA_IFDIR;
+            entry->d_ino = buf.ia_ino;
+            entry->d_stat = buf;
+            inode_ctx->buf = buf;
+            inode_ctx->type = SNAP_VIEW_SNAPSHOT_INODE;
+        } else {
+            /* For files under snapshot world do not set
+             * entry->inode and reset statbuf (except ia_ino),
+             * so that FUSE/Kernel will send an explicit lookup.
+             * entry->d_stat contains the statbuf information
+             * of original file, so for NFS not to cache this
+             * information and to send explicit lookup, it is
+             * required to reset the statbuf.
+             * Virtual gfid for these files will be generated in the
+             * first lookup.
+             */
+            buf.ia_ino = entry->d_ino;
+            entry->d_stat = buf;
+        }
+    }
 
 out:
-        return;
+    return;
 }
 
 /* In readdirp, though new inode is created along with the generation of
@@ -1406,151 +1664,176 @@ out:
    and is filled in when lookup comes on that object.
 */
 int32_t
-svs_readdirp (call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
-              off_t off, dict_t *dict)
+svs_readdirp(call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
+             off_t off, dict_t *dict)
 {
-        gf_dirent_t             entries;
-        gf_dirent_t            *entry                           = NULL;
-        struct  iatt            buf                             = {0, };
-        int                     count                           = 0;
-        int                     op_ret                          = -1;
-        int                     op_errno                        = EINVAL;
-        svs_inode_t            *parent_ctx                      = NULL;
-        svs_fd_t               *svs_fd                          = NULL;
-
-        GF_VALIDATE_OR_GOTO ("snap-view-daemon", this, unwind);
-        GF_VALIDATE_OR_GOTO (this->name, frame, unwind);
-        GF_VALIDATE_OR_GOTO (this->name, fd, unwind);
-        GF_VALIDATE_OR_GOTO (this->name, fd->inode, unwind);
-
-        INIT_LIST_HEAD (&entries.list);
-
-        parent_ctx = svs_inode_ctx_get (this, fd->inode);
-        if (!parent_ctx) {
-                gf_log (this->name, GF_LOG_ERROR, "failed to get the inode "
-                        "context for %s", uuid_utoa (fd->inode->gfid));
-                op_ret = -1;
-                op_errno = EINVAL;
-                goto unwind;
+    gf_dirent_t entries;
+    gf_dirent_t *entry = NULL;
+    struct iatt buf = {
+        0,
+    };
+    int count = 0;
+    int op_ret = -1;
+    int op_errno = EINVAL;
+    svs_inode_t *parent_ctx = NULL;
+    svs_fd_t *svs_fd = NULL;
+    call_stack_t *root = NULL;
+
+    GF_VALIDATE_OR_GOTO("snap-view-daemon", this, unwind);
+    GF_VALIDATE_OR_GOTO(this->name, frame, unwind);
+    GF_VALIDATE_OR_GOTO(this->name, fd, unwind);
+    GF_VALIDATE_OR_GOTO(this->name, fd->inode, unwind);
+
+    INIT_LIST_HEAD(&entries.list);
+
+    root = frame->root;
+    op_ret = gf_setcredentials(&root->uid, &root->gid, root->ngrps,
+                               root->groups);
+    if (op_ret != 0) {
+        goto unwind;
+    }
+
+    parent_ctx = svs_inode_ctx_get(this, fd->inode);
+    if (!parent_ctx) {
+        op_ret = -1;
+        op_errno = EINVAL;
+        gf_msg(this->name, GF_LOG_ERROR, op_errno,
+               SVS_MSG_GET_INODE_CONTEXT_FAILED,
+               "failed to get the inode "
+               "context for %s",
+               uuid_utoa(fd->inode->gfid));
+        goto unwind;
+    }
+
+    if (parent_ctx->type == SNAP_VIEW_ENTRY_POINT_INODE) {
+        LOCK(&fd->lock);
+        {
+            count = svs_fill_readdir(this, &entries, size, off);
         }
+        UNLOCK(&fd->lock);
 
-        if (parent_ctx->type == SNAP_VIEW_ENTRY_POINT_INODE) {
-                LOCK (&fd->lock);
-                {
-                        count = svs_fill_readdir (this, &entries, size, off);
-                }
-                UNLOCK (&fd->lock);
-
-                op_ret = count;
-
-                list_for_each_entry (entry, &entries.list, list) {
-                        svs_readdirp_fill (this, fd->inode, parent_ctx, entry);
-                }
+        op_ret = count;
 
-                goto unwind;
-        } else {
-                svs_fd = svs_fd_ctx_get_or_new (this, fd);
-                if (!svs_fd) {
-                        gf_log (this->name, GF_LOG_ERROR, "failed to get the "
-                                "fd context %s", uuid_utoa (fd->inode->gfid));
-                        op_ret = -1;
-                        op_errno = EBADFD;
-                        goto unwind;
-                }
+        list_for_each_entry(entry, &entries.list, list)
+        {
+            svs_readdirp_fill(this, fd->inode, parent_ctx, entry);
+        }
 
-                glfs_seekdir (svs_fd->fd, off);
+        goto unwind;
+    } else {
+        svs_fd = svs_fd_ctx_get_or_new(this, fd);
+        if (!svs_fd) {
+            op_ret = -1;
+            op_errno = EBADFD;
+            gf_msg(this->name, GF_LOG_ERROR, op_errno,
+                   SVS_MSG_GET_FD_CONTEXT_FAILED,
+                   "failed to get the fd context "
+                   "for the inode %s",
+                   uuid_utoa(fd->inode->gfid));
+            goto unwind;
+        }
 
-                LOCK (&fd->lock);
-                {
-                        count = svs_glfs_readdir (this, svs_fd->fd, &entries,
-                                                  &op_errno, &buf, _gf_true,
-                                                  size);
-                }
-                UNLOCK (&fd->lock);
+        glfs_seekdir(svs_fd->fd, off);
 
-                op_ret = count;
+        LOCK(&fd->lock);
+        {
+            count = svs_glfs_readdir(this, svs_fd->fd, &entries, &op_errno,
+                                     &buf, _gf_true, size);
+        }
+        UNLOCK(&fd->lock);
 
-                list_for_each_entry (entry, &entries.list, list) {
-                        svs_readdirp_fill (this, fd->inode, parent_ctx, entry);
-                }
+        op_ret = count;
 
-                goto unwind;
+        list_for_each_entry(entry, &entries.list, list)
+        {
+            svs_readdirp_fill(this, fd->inode, parent_ctx, entry);
         }
 
+        goto unwind;
+    }
+
 unwind:
-        STACK_UNWIND_STRICT (readdirp, frame, op_ret, op_errno, &entries, dict);
+    STACK_UNWIND_STRICT(readdirp, frame, op_ret, op_errno, &entries, dict);
 
-        gf_dirent_free (&entries);
+    gf_dirent_free(&entries);
 
-        return 0;
+    return 0;
 }
 
 int32_t
-svs_readdir (call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
-             off_t off, dict_t *xdata)
+svs_readdir(call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
+            off_t off, dict_t *xdata)
 {
-        svs_private_t *priv      = NULL;
-        gf_dirent_t    entries   = {{{0, }, }, };
-        int            count     = 0;
-        svs_inode_t   *inode_ctx = NULL;
-        int            op_errno  = EINVAL;
-        int            op_ret    = -1;
-        svs_fd_t      *svs_fd    = NULL;
-        glfs_fd_t     *glfd      = NULL;
-
-        INIT_LIST_HEAD (&entries.list);
-
-        GF_VALIDATE_OR_GOTO ("snap-view-server", this, unwind);
-        GF_VALIDATE_OR_GOTO (this->name, frame, unwind);
-        GF_VALIDATE_OR_GOTO (this->name, fd, unwind);
-        GF_VALIDATE_OR_GOTO (this->name, fd->inode, unwind);
-
-        priv = this->private;
-
-        inode_ctx = svs_inode_ctx_get (this, fd->inode);
-        if (!inode_ctx) {
-                gf_log (this->name, GF_LOG_ERROR, "inode context not found in "
-                        "the inode %s", uuid_utoa (fd->inode->gfid));
-                op_ret = -1;
-                op_errno = EINVAL;
-                goto unwind;
+    gf_dirent_t entries = {
+        {
+            {
+                0,
+            },
+        },
+    };
+    int count = 0;
+    svs_inode_t *inode_ctx = NULL;
+    int op_errno = EINVAL;
+    int op_ret = -1;
+    svs_fd_t *svs_fd = NULL;
+    glfs_fd_t *glfd = NULL;
+
+    INIT_LIST_HEAD(&entries.list);
+
+    GF_VALIDATE_OR_GOTO("snap-view-server", this, unwind);
+    GF_VALIDATE_OR_GOTO(this->name, frame, unwind);
+    GF_VALIDATE_OR_GOTO(this->name, fd, unwind);
+    GF_VALIDATE_OR_GOTO(this->name, fd->inode, unwind);
+
+    inode_ctx = svs_inode_ctx_get(this, fd->inode);
+    if (!inode_ctx) {
+        op_ret = -1;
+        op_errno = EINVAL;
+        gf_msg(this->name, GF_LOG_ERROR, op_errno,
+               SVS_MSG_GET_INODE_CONTEXT_FAILED,
+               "inode context not found in "
+               "the inode %s",
+               uuid_utoa(fd->inode->gfid));
+        goto unwind;
+    }
+
+    if (inode_ctx->type == SNAP_VIEW_ENTRY_POINT_INODE) {
+        LOCK(&fd->lock);
+        {
+            count = svs_fill_readdir(this, &entries, size, off);
+        }
+        UNLOCK(&fd->lock);
+    } else {
+        svs_fd = svs_fd_ctx_get_or_new(this, fd);
+        if (!svs_fd) {
+            op_ret = -1;
+            op_errno = EBADFD;
+            gf_msg(this->name, GF_LOG_ERROR, op_errno,
+                   SVS_MSG_GET_FD_CONTEXT_FAILED,
+                   "failed to get the fd "
+                   "context for %s",
+                   uuid_utoa(fd->inode->gfid));
+            goto unwind;
         }
 
-        if (inode_ctx->type == SNAP_VIEW_ENTRY_POINT_INODE) {
-                LOCK (&fd->lock);
-                {
-                        count = svs_fill_readdir (this, &entries, size, off);
-                }
-                UNLOCK (&fd->lock);
-        } else {
-                svs_fd = svs_fd_ctx_get_or_new (this, fd);
-                if (!svs_fd) {
-                        gf_log (this->name, GF_LOG_ERROR, "failed to get the "
-                                "fd context %s", uuid_utoa (fd->inode->gfid));
-                        op_ret = -1;
-                        op_errno = EBADFD;
-                        goto unwind;
-                }
-
-                glfd = svs_fd->fd;
+        glfd = svs_fd->fd;
 
-                LOCK (&fd->lock);
-                {
-                        count = svs_glfs_readdir (this, glfd, &entries,
-                                                  &op_errno, NULL, _gf_false,
-                                                  size);
-                }
-                UNLOCK (&fd->lock);
+        LOCK(&fd->lock);
+        {
+            count = svs_glfs_readdir(this, glfd, &entries, &op_errno, NULL,
+                                     _gf_false, size);
         }
+        UNLOCK(&fd->lock);
+    }
 
-        op_ret = count;
+    op_ret = count;
 
 unwind:
-        STACK_UNWIND_STRICT (readdir, frame, op_ret, op_errno, &entries, xdata);
+    STACK_UNWIND_STRICT(readdir, frame, op_ret, op_errno, &entries, xdata);
 
-        gf_dirent_free (&entries);
+    gf_dirent_free(&entries);
 
-        return 0;
+    return 0;
 }
 
 /*
@@ -1588,11 +1871,11 @@ unwind:
  * the gfid it got from NFS client, for which it was not able to find the right
  * inode. So snapview-server was able to get the fs instance (glfs_t) of the
  * snapshot volume to which the entry belongs to, and the handle for the entry
- * from the corresponding snapshot volume and fill those informations in the
+ * from the corresponding snapshot volume and fill those information in the
  * inode context.
  *
  * But now, since NFS server is able to find the inode from the inode table for
- * the gfid it got from the NFS client, it wont send lookup. Rather it directly
+ * the gfid it got from the NFS client, it won't send lookup. Rather it directly
  * sends the fop it received from the client. Now this causes problems for
  * snapview-server. Because for each fop snapview-server assumes that lookup has
  * been performed on that entry and the entry's inode context contains the
@@ -1606,640 +1889,832 @@ unwind:
  */
 
 int32_t
-svs_get_handle (xlator_t *this, loc_t *loc, svs_inode_t *inode_ctx,
-                int32_t *op_errno)
+svs_get_handle(xlator_t *this, loc_t *loc, svs_inode_t *inode_ctx,
+               int32_t *op_errno)
 {
-        svs_inode_t   *parent_ctx   = NULL;
-        int            ret          = -1;
-        inode_t       *parent       = NULL;
-        struct iatt    postparent   = {0, };
-        struct iatt    buf          = {0, };
-        char           uuid1[64];
-
-        GF_VALIDATE_OR_GOTO ("snap-view-daemon", this, out);
-        GF_VALIDATE_OR_GOTO (this->name, loc, out);
-        GF_VALIDATE_OR_GOTO (this->name, loc->inode, out);
-
-        if (loc->path) {
-                if (!loc->name || (loc->name && !strcmp (loc->name, ""))) {
-                        loc->name = strrchr (loc->path, '/');
-                        if (loc->name)
-                                loc->name++;
-                }
-        }
-
-        if (loc->parent)
-                parent = inode_ref (loc->parent);
-        else {
-                parent = inode_find (loc->inode->table, loc->pargfid);
-                if (!parent)
-                        parent = inode_parent (loc->inode, NULL, NULL);
-        }
-
-        if (parent)
-                parent_ctx = svs_inode_ctx_get (this, parent);
-
-        if (!parent_ctx) {
-                gf_log (this->name, GF_LOG_WARNING, "failed to get the parent "
-                        "context for %s (%s)", loc->path,
-                        uuid_utoa_r (loc->inode->gfid, uuid1));
-                *op_errno = EINVAL;
-                goto out;
-        }
-
-        if (parent_ctx) {
-                if (parent_ctx->type == SNAP_VIEW_ENTRY_POINT_INODE)
-                        ret = svs_lookup_snapshot (this, loc, &buf,
-                                                   &postparent, parent,
-                                                   parent_ctx, op_errno);
-                else
-                        ret = svs_lookup_entry (this, loc, &buf,
-                                                &postparent, parent,
-                                                parent_ctx, op_errno);
-        }
+    svs_inode_t *parent_ctx = NULL;
+    int ret = -1;
+    inode_t *parent = NULL;
+    struct iatt postparent = {
+        0,
+    };
+    struct iatt buf = {
+        0,
+    };
+    char uuid1[64];
+
+    GF_VALIDATE_OR_GOTO("snap-view-daemon", this, out);
+    GF_VALIDATE_OR_GOTO(this->name, loc, out);
+    GF_VALIDATE_OR_GOTO(this->name, loc->inode, out);
+
+    if (loc->path) {
+        if (!loc->name || (loc->name && !strcmp(loc->name, ""))) {
+            loc->name = strrchr(loc->path, '/');
+            if (loc->name)
+                loc->name++;
+        }
+    }
+
+    if (loc->parent)
+        parent = inode_ref(loc->parent);
+    else {
+        parent = inode_find(loc->inode->table, loc->pargfid);
+        if (!parent)
+            parent = inode_parent(loc->inode, NULL, NULL);
+    }
+
+    if (parent)
+        parent_ctx = svs_inode_ctx_get(this, parent);
+
+    if (!parent_ctx) {
+        *op_errno = EINVAL;
+        gf_msg(this->name, GF_LOG_WARNING, *op_errno,
+               SVS_MSG_GET_INODE_CONTEXT_FAILED,
+               "failed to get the parent "
+               "context for %s (%s)",
+               loc->path, uuid_utoa_r(loc->inode->gfid, uuid1));
+        goto out;
+    }
+
+    if (parent_ctx) {
+        if (parent_ctx->type == SNAP_VIEW_ENTRY_POINT_INODE)
+            ret = svs_lookup_snapshot(this, loc, &buf, &postparent, parent,
+                                      parent_ctx, op_errno);
+        else
+            ret = svs_lookup_entry(this, loc, &buf, &postparent, parent,
+                                   parent_ctx, op_errno);
+    }
 
 out:
-        if (parent)
-                inode_unref (parent);
+    if (parent)
+        inode_unref(parent);
 
-        return ret;
+    return ret;
 }
 
 int32_t
-svs_stat (call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xdata)
+svs_stat(call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xdata)
 {
-        svs_private_t *priv         = NULL;
-        struct iatt    buf          = {0, };
-        int32_t        op_errno     = EINVAL;
-        int32_t        op_ret       = -1;
-        svs_inode_t   *inode_ctx    = NULL;
-        glfs_t        *fs           = NULL;
-        glfs_object_t *object       = NULL;
-        struct stat    stat         = {0, };
-        int            ret          = -1;
-
-        GF_VALIDATE_OR_GOTO ("snap-view-daemon", this, out);
-        GF_VALIDATE_OR_GOTO (this->name, frame, out);
-        GF_VALIDATE_OR_GOTO (this->name, loc, out);
-        GF_VALIDATE_OR_GOTO (this->name, loc->inode, out);
-
-        priv = this->private;
-
-        /* Instead of doing the check of whether it is a entry point directory
-           or not by checking the name of the entry and then deciding what
-           to do, just check the inode context and decide what to be done.
-        */
-
-        inode_ctx = svs_inode_ctx_get (this, loc->inode);
-        if (!inode_ctx) {
-                gf_log (this->name, GF_LOG_ERROR, "inode context not found for"
-                        " %s", uuid_utoa (loc->inode->gfid));
-                op_ret = -1;
-                op_errno = EINVAL;
-                goto out;
-        }
-
-        if (inode_ctx->type == SNAP_VIEW_ENTRY_POINT_INODE) {
-                svs_iatt_fill (loc->inode->gfid, &buf);
-                op_ret = 0;
-        }
-        else {
-
-                SVS_GET_INODE_CTX_INFO(inode_ctx, fs, object, this, loc, op_ret,
-                                       op_errno, out);
-
-                ret = glfs_h_stat (fs, object, &stat);
-                if (ret) {
-                        gf_log (this->name, GF_LOG_ERROR, "glfs_h_stat on %s "
-                                "(gfid: %s) failed", loc->name,
-                                uuid_utoa (loc->inode->gfid));
-                        op_ret = -1;
-                        op_errno = errno;
-                        goto out;
-                }
+    struct iatt buf = {
+        0,
+    };
+    int32_t op_errno = EINVAL;
+    int32_t op_ret = -1;
+    svs_inode_t *inode_ctx = NULL;
+    glfs_t *fs = NULL;
+    glfs_object_t *object = NULL;
+    struct stat stat = {
+        0,
+    };
+    int ret = -1;
+    call_stack_t *root = NULL;
+
+    GF_VALIDATE_OR_GOTO("snap-view-daemon", this, out);
+    GF_VALIDATE_OR_GOTO(this->name, frame, out);
+    GF_VALIDATE_OR_GOTO(this->name, loc, out);
+    GF_VALIDATE_OR_GOTO(this->name, loc->inode, out);
+
+    root = frame->root;
+    op_ret = gf_setcredentials(&root->uid, &root->gid, root->ngrps,
+                               root->groups);
+    if (op_ret != 0) {
+        goto out;
+    }
+
+    /* Instead of doing the check of whether it is a entry point directory
+       or not by checking the name of the entry and then deciding what
+       to do, just check the inode context and decide what to be done.
+    */
+
+    inode_ctx = svs_inode_ctx_get(this, loc->inode);
+    if (!inode_ctx) {
+        op_ret = -1;
+        op_errno = EINVAL;
+        gf_msg(this->name, GF_LOG_ERROR, op_errno,
+               SVS_MSG_GET_INODE_CONTEXT_FAILED,
+               "inode context not found for %s", uuid_utoa(loc->inode->gfid));
+        goto out;
+    }
+
+    if (inode_ctx->type == SNAP_VIEW_ENTRY_POINT_INODE) {
+        svs_iatt_fill(loc->inode->gfid, &buf);
+        op_ret = 0;
+    } else {
+        SVS_GET_INODE_CTX_INFO(inode_ctx, fs, object, this, loc, op_ret,
+                               op_errno, out);
 
-                iatt_from_stat (&buf, &stat);
-                gf_uuid_copy (buf.ia_gfid, loc->inode->gfid);
-                svs_fill_ino_from_gfid (&buf);
-                op_ret = ret;
-        }
+        ret = glfs_h_stat(fs, object, &stat);
+        if (ret) {
+            op_ret = -1;
+            op_errno = errno;
+            gf_msg(this->name, GF_LOG_ERROR, op_errno, SVS_MSG_STAT_FAILED,
+                   "glfs_h_stat on %s (gfid: %s) "
+                   "failed",
+                   loc->name, uuid_utoa(loc->inode->gfid));
+            goto out;
+        } else
+            gf_msg_debug(this->name, 0, "stat on %s (%s) successful", loc->path,
+                         uuid_utoa(loc->inode->gfid));
+
+        iatt_from_stat(&buf, &stat);
+        gf_uuid_copy(buf.ia_gfid, loc->inode->gfid);
+        svs_fill_ino_from_gfid(&buf);
+        op_ret = ret;
+    }
 
 out:
-        STACK_UNWIND_STRICT (stat, frame, op_ret, op_errno, &buf, xdata);
-        return 0;
+    STACK_UNWIND_STRICT(stat, frame, op_ret, op_errno, &buf, xdata);
+    return 0;
 }
 
 int32_t
-svs_fstat (call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *xdata)
+svs_fstat(call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *xdata)
 {
-        svs_private_t *priv        = NULL;
-        struct iatt    buf         = {0, };
-        int32_t        op_errno    = EINVAL;
-        int32_t        op_ret      = -1;
-        svs_inode_t   *inode_ctx   = NULL;
-        struct stat    stat        = {0, };
-        int            ret         = -1;
-        glfs_fd_t     *glfd        = NULL;
-        svs_fd_t      *sfd         = NULL;
-
-        GF_VALIDATE_OR_GOTO ("snap-view-daemon", this, out);
-        GF_VALIDATE_OR_GOTO (this->name, frame, out);
-        GF_VALIDATE_OR_GOTO (this->name, fd, out);
-        GF_VALIDATE_OR_GOTO (this->name, fd->inode, out);
-
-        priv = this->private;
-
-        /* Instead of doing the check of whether it is a entry point directory
-           or not by checking the name of the entry and then deciding what
-           to do, just check the inode context and decide what to be done.
-        */
-
-        inode_ctx = svs_inode_ctx_get (this, fd->inode);
-        if (!inode_ctx) {
-                gf_log (this->name, GF_LOG_ERROR, "inode context not found for"
-                        " the inode %s", uuid_utoa (fd->inode->gfid));
-                op_ret = -1;
-                op_errno = EINVAL;
-                goto out;
+    struct iatt buf = {
+        0,
+    };
+    int32_t op_errno = EINVAL;
+    int32_t op_ret = -1;
+    svs_inode_t *inode_ctx = NULL;
+    struct stat stat = {
+        0,
+    };
+    int ret = -1;
+    glfs_fd_t *glfd = NULL;
+    svs_fd_t *sfd = NULL;
+    call_stack_t *root = NULL;
+
+    GF_VALIDATE_OR_GOTO("snap-view-daemon", this, out);
+    GF_VALIDATE_OR_GOTO(this->name, frame, out);
+    GF_VALIDATE_OR_GOTO(this->name, fd, out);
+    GF_VALIDATE_OR_GOTO(this->name, fd->inode, out);
+
+    /* Instead of doing the check of whether it is a entry point directory
+       or not by checking the name of the entry and then deciding what
+       to do, just check the inode context and decide what to be done.
+    */
+
+    root = frame->root;
+    op_ret = gf_setcredentials(&root->uid, &root->gid, root->ngrps,
+                               root->groups);
+    if (op_ret != 0) {
+        goto out;
+    }
+
+    inode_ctx = svs_inode_ctx_get(this, fd->inode);
+    if (!inode_ctx) {
+        op_ret = -1;
+        op_errno = EINVAL;
+        gf_msg(this->name, GF_LOG_ERROR, op_errno,
+               SVS_MSG_GET_INODE_CONTEXT_FAILED,
+               "inode context not found for"
+               " the inode %s",
+               uuid_utoa(fd->inode->gfid));
+        goto out;
+    }
+
+    if (inode_ctx->type == SNAP_VIEW_ENTRY_POINT_INODE) {
+        svs_iatt_fill(fd->inode->gfid, &buf);
+        op_ret = 0;
+    } else {
+        if (!(svs_inode_ctx_glfs_mapping(this, inode_ctx))) {
+            op_ret = -1;
+            op_errno = EBADF;
+            gf_msg(this->name, GF_LOG_ERROR, op_errno,
+                   SVS_MSG_FS_INSTANCE_INVALID,
+                   "glfs instance %p to which the inode %s "
+                   "belongs to does not exist. That snapshot "
+                   "corresponding to the fs instance "
+                   "might have been deleted or deactivated.",
+                   inode_ctx->fs, uuid_utoa(fd->inode->gfid));
+            goto out;
+        }
+
+        sfd = svs_fd_ctx_get_or_new(this, fd);
+        if (!sfd) {
+            op_ret = -1;
+            op_errno = EBADFD;
+            gf_msg(this->name, GF_LOG_ERROR, op_errno,
+                   SVS_MSG_GET_FD_CONTEXT_FAILED,
+                   "failed to get the fd context "
+                   "for %s",
+                   uuid_utoa(fd->inode->gfid));
+            goto out;
         }
 
-        if (inode_ctx->type == SNAP_VIEW_ENTRY_POINT_INODE) {
-                svs_iatt_fill (fd->inode->gfid, &buf);
-                op_ret = 0;
+        glfd = sfd->fd;
+        ret = glfs_fstat(glfd, &stat);
+        if (ret) {
+            op_ret = -1;
+            op_errno = errno;
+            gf_msg(this->name, GF_LOG_ERROR, op_errno, SVS_MSG_STAT_FAILED,
+                   "glfs_fstat on gfid: %s failed", uuid_utoa(fd->inode->gfid));
+            goto out;
         }
-        else {
-                sfd = svs_fd_ctx_get_or_new (this, fd);
-                if (!sfd) {
-                        gf_log (this->name, GF_LOG_ERROR, "failed to get the "
-                                "fd context for %s",
-                                uuid_utoa (fd->inode->gfid));
-                        op_ret = -1;
-                        op_errno = EBADFD;
-                        goto out;
-                }
 
-                glfd = sfd->fd;
-                ret = glfs_fstat (glfd, &stat);
-                if (ret) {
-                        gf_log (this->name, GF_LOG_ERROR, "glfs_fstat on "
-                                "gfid: %s failed", uuid_utoa (fd->inode->gfid));
-                        op_ret = -1;
-                        op_errno = errno;
-                        goto out;
-                }
-
-                iatt_from_stat (&buf, &stat);
-                gf_uuid_copy (buf.ia_gfid, fd->inode->gfid);
-                svs_fill_ino_from_gfid (&buf);
-                op_ret = ret;
-        }
+        iatt_from_stat(&buf, &stat);
+        gf_uuid_copy(buf.ia_gfid, fd->inode->gfid);
+        svs_fill_ino_from_gfid(&buf);
+        op_ret = ret;
+    }
 
 out:
-        STACK_UNWIND_STRICT (fstat, frame, op_ret, op_errno, &buf, xdata);
-        return 0;
+    STACK_UNWIND_STRICT(fstat, frame, op_ret, op_errno, &buf, xdata);
+    return 0;
 }
 
 int32_t
-svs_open (call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t flags,
-          fd_t *fd, dict_t *xdata)
+svs_statfs(call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xdata)
 {
-        svs_inode_t   *inode_ctx = NULL;
-        svs_fd_t      *sfd       = NULL;
-        int32_t        op_ret    = -1;
-        int32_t        op_errno  = EINVAL;
-        glfs_fd_t     *glfd      = NULL;
-        glfs_t        *fs        = NULL;
-        glfs_object_t *object    = NULL;
-
-
-        GF_VALIDATE_OR_GOTO ("snap-view-daemon", this, out);
-        GF_VALIDATE_OR_GOTO (this->name, frame, out);
-        GF_VALIDATE_OR_GOTO (this->name, fd, out);
-        GF_VALIDATE_OR_GOTO (this->name, loc, out);
-        GF_VALIDATE_OR_GOTO (this->name, loc->inode, out);
-
-        inode_ctx = svs_inode_ctx_get (this, loc->inode);
-        if (!inode_ctx) {
-                gf_log (this->name, GF_LOG_ERROR, "inode context for %s "
-                        "(gfid: %s) not found", loc->name,
-                        uuid_utoa (loc->inode->gfid));
-                goto out;
-        }
-
-        if (inode_ctx->type == SNAP_VIEW_ENTRY_POINT_INODE)
-                GF_ASSERT (0); // on entry point it should always be opendir
-
-        SVS_GET_INODE_CTX_INFO(inode_ctx, fs, object, this, loc, op_ret,
-                               op_errno, out);
-
-        glfd = glfs_h_open (fs, object, flags);
-        if (!glfd) {
-                gf_log (this->name, GF_LOG_ERROR, "glfs_h_open on %s failed "
-                        "(gfid: %s)", loc->name, uuid_utoa (loc->inode->gfid));
-                op_ret = -1;
-                op_errno = errno;
-                goto out;
-        }
-
-        sfd = svs_fd_ctx_get_or_new (this, fd);
-        if (!sfd) {
-                gf_log (this->name, GF_LOG_ERROR, "failed to allocate fd "
-                        "context for %s (gfid: %s)", loc->name,
-                        uuid_utoa (loc->inode->gfid));
-                op_ret = -1;
-                op_errno = ENOMEM;
-                glfs_close (glfd);
-                goto out;
-        }
-        sfd->fd = glfd;
-
-        op_ret = 0;
+    struct statvfs buf = {
+        0,
+    };
+    int32_t op_errno = EINVAL;
+    int32_t op_ret = -1;
+    svs_inode_t *inode_ctx = NULL;
+    glfs_t *fs = NULL;
+    glfs_object_t *object = NULL;
+    int ret = -1;
+    call_stack_t *root = NULL;
+
+    GF_VALIDATE_OR_GOTO("snap-view-daemon", this, out);
+    GF_VALIDATE_OR_GOTO(this->name, frame, out);
+    GF_VALIDATE_OR_GOTO(this->name, loc, out);
+    GF_VALIDATE_OR_GOTO(this->name, loc->inode, out);
+
+    root = frame->root;
+    op_ret = gf_setcredentials(&root->uid, &root->gid, root->ngrps,
+                               root->groups);
+    if (op_ret != 0) {
+        goto out;
+    }
+
+    /* Instead of doing the check of whether it is a entry point directory
+       or not by checking the name of the entry and then deciding what
+       to do, just check the inode context and decide what to be done.
+    */
+    inode_ctx = svs_inode_ctx_get(this, loc->inode);
+    if (!inode_ctx) {
+        op_ret = -1;
+        op_errno = EINVAL;
+        gf_msg(this->name, GF_LOG_ERROR, op_errno,
+               SVS_MSG_GET_INODE_CONTEXT_FAILED,
+               "inode context not found for %s", uuid_utoa(loc->inode->gfid));
+        goto out;
+    }
+
+    SVS_GET_INODE_CTX_INFO(inode_ctx, fs, object, this, loc, op_ret, op_errno,
+                           out);
+
+    ret = glfs_h_statfs(fs, object, &buf);
+    if (ret) {
+        op_ret = -1;
+        op_errno = errno;
+        gf_msg(this->name, GF_LOG_ERROR, op_errno, SVS_MSG_STATFS_FAILED,
+               "glfs_h_statvfs on %s (gfid: %s) "
+               "failed",
+               loc->name, uuid_utoa(loc->inode->gfid));
+        goto out;
+    }
+    op_ret = ret;
 
 out:
-        STACK_UNWIND_STRICT (open, frame, op_ret, op_errno, fd, NULL);
-        return 0;
+    STACK_UNWIND_STRICT(statfs, frame, op_ret, op_errno, &buf, xdata);
+    return 0;
 }
 
 int32_t
-svs_readv (call_frame_t *frame, xlator_t *this,
-           fd_t *fd, size_t size, off_t offset, uint32_t flags, dict_t *xdata)
+svs_open(call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t flags,
+         fd_t *fd, dict_t *xdata)
 {
-        int32_t                op_ret     = -1;
-        int32_t                op_errno   = 0;
-        svs_private_t         *priv       = NULL;
-        struct iobuf          *iobuf      = NULL;
-        struct iobref         *iobref     = NULL;
-        struct iovec           vec        = {0,};
-        svs_fd_t              *sfd        = NULL;
-        int                    ret        = -1;
-        struct stat            fstatbuf   = {0, };
-        glfs_fd_t             *glfd       = NULL;
-        struct iatt            stbuf      = {0, };
-
-        GF_VALIDATE_OR_GOTO ("snap-view-daemon", this, out);
-        GF_VALIDATE_OR_GOTO (this->name, frame, out);
-        GF_VALIDATE_OR_GOTO (this->name, fd, out);
-        GF_VALIDATE_OR_GOTO (this->name, fd->inode, out);
-
-        priv = this->private;
-        VALIDATE_OR_GOTO (priv, out);
-
-        sfd = svs_fd_ctx_get_or_new (this, fd);
-        if (!sfd) {
-                gf_log (this->name, GF_LOG_ERROR, "failed to get the fd "
-                        "context for %s", uuid_utoa (fd->inode->gfid));
-                op_ret = -1;
-                op_errno = EBADFD;
-                goto out;
-        }
-
-        glfd = sfd->fd;
-
-        iobuf = iobuf_get2 (this->ctx->iobuf_pool, size);
-        if (!iobuf) {
-                op_ret = -1;
-                op_errno = ENOMEM;
-                goto out;
-        }
-
-        ret = glfs_pread (glfd, iobuf->ptr, size, offset, 0);
-        if (ret < 0) {
-                op_ret = -1;
-                op_errno = errno;
-                gf_log (this->name, GF_LOG_ERROR, "glfs_read failed (%s)",
-                        strerror (op_errno));
-                goto out;
-        }
-
-        vec.iov_base = iobuf->ptr;
-        vec.iov_len  = ret;
-
-        iobref = iobref_new ();
-
-        iobref_add (iobref, iobuf);
-
-        ret = glfs_fstat (glfd, &fstatbuf);
-        if (ret) {
-                op_ret = -1;
-                op_errno = errno;
-                gf_log (this->name, GF_LOG_ERROR, "glfs_fstat failed after "
-                        "readv on %s", uuid_utoa (fd->inode->gfid));
-                goto out;
-        }
-
-        iatt_from_stat (&stbuf, &fstatbuf);
-        gf_uuid_copy (stbuf.ia_gfid, fd->inode->gfid);
-        svs_fill_ino_from_gfid (&stbuf);
-
-        /* Hack to notify higher layers of EOF. */
-        if (!stbuf.ia_size || (offset + vec.iov_len) >= stbuf.ia_size)
-                op_errno = ENOENT;
-
-        op_ret = vec.iov_len;
+    svs_inode_t *inode_ctx = NULL;
+    svs_fd_t *sfd = NULL;
+    int32_t op_ret = -1;
+    int32_t op_errno = EINVAL;
+    glfs_fd_t *glfd = NULL;
+    glfs_t *fs = NULL;
+    glfs_object_t *object = NULL;
+    call_stack_t *root = NULL;
+
+    GF_VALIDATE_OR_GOTO("snap-view-daemon", this, out);
+    GF_VALIDATE_OR_GOTO(this->name, frame, out);
+    GF_VALIDATE_OR_GOTO(this->name, fd, out);
+    GF_VALIDATE_OR_GOTO(this->name, loc, out);
+    GF_VALIDATE_OR_GOTO(this->name, loc->inode, out);
+
+    root = frame->root;
+
+    inode_ctx = svs_inode_ctx_get(this, loc->inode);
+    if (!inode_ctx) {
+        gf_msg(this->name, GF_LOG_ERROR, op_errno,
+               SVS_MSG_GET_INODE_CONTEXT_FAILED,
+               "inode context for %s (gfid: %s) "
+               "not found",
+               loc->name, uuid_utoa(loc->inode->gfid));
+        goto out;
+    }
+
+    if (inode_ctx->type == SNAP_VIEW_ENTRY_POINT_INODE)
+        GF_ASSERT(0);  // on entry point it should always be opendir
+
+    SVS_GET_INODE_CTX_INFO(inode_ctx, fs, object, this, loc, op_ret, op_errno,
+                           out);
+
+    op_ret = gf_setcredentials(&root->uid, &root->gid, root->ngrps,
+                               root->groups);
+    if (op_ret != 0) {
+        goto out;
+    }
+
+    glfd = glfs_h_open(fs, object, flags);
+    if (!glfd) {
+        op_ret = -1;
+        op_errno = errno;
+        gf_msg(this->name, GF_LOG_ERROR, op_errno, SVS_MSG_OPEN_FAILED,
+               "glfs_h_open on %s failed (gfid: %s)", loc->name,
+               uuid_utoa(loc->inode->gfid));
+        goto out;
+    }
+
+    sfd = svs_fd_ctx_get_or_new(this, fd);
+    if (!sfd) {
+        op_ret = -1;
+        op_errno = ENOMEM;
+        gf_msg(this->name, GF_LOG_ERROR, op_errno, SVS_MSG_NO_MEMORY,
+               "failed to allocate fd context "
+               "for %s (gfid: %s)",
+               loc->name, uuid_utoa(loc->inode->gfid));
+        glfs_close(glfd);
+        goto out;
+    }
+    sfd->fd = glfd;
+
+    op_ret = 0;
 
 out:
-
-        STACK_UNWIND_STRICT (readv, frame, op_ret, op_errno,
-                             &vec, 1, &stbuf, iobref, NULL);
-
-        if (iobref)
-                iobref_unref (iobref);
-        if (iobuf)
-                iobuf_unref (iobuf);
-
-        return 0;
+    STACK_UNWIND_STRICT(open, frame, op_ret, op_errno, fd, NULL);
+    return 0;
 }
 
 int32_t
-svs_readlink (call_frame_t *frame, xlator_t *this,
-              loc_t *loc, size_t size, dict_t *xdata)
+svs_readv(call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
+          off_t offset, uint32_t flags, dict_t *xdata)
 {
-        svs_inode_t     *inode_ctx = NULL;
-        glfs_t          *fs        = NULL;
-        glfs_object_t   *object    = NULL;
-        int              op_ret    = -1;
-        int              op_errno  = EINVAL;
-        char            *buf       = NULL;
-        struct iatt      stbuf     = {0, };
-        int              ret       = -1;
-        struct stat      stat      = {0, };
-
-        GF_VALIDATE_OR_GOTO ("snap-view-daemon", this, out);
-        GF_VALIDATE_OR_GOTO (this->name, frame, out);
-        GF_VALIDATE_OR_GOTO (this->name, loc, out);
-        GF_VALIDATE_OR_GOTO (this->name, loc->inode, out);
-
-        inode_ctx = svs_inode_ctx_get (this, loc->inode);
-        if (!inode_ctx) {
-                gf_log (this->name, GF_LOG_ERROR, "failed to get inode context "
-                        "for %s (gfid: %s)", loc->name,
-                        uuid_utoa (loc->inode->gfid));
-                op_ret = -1;
-                op_errno = EINVAL;
-                goto out;
-        }
+    int32_t op_ret = -1;
+    int32_t op_errno = 0;
+    svs_private_t *priv = NULL;
+    struct iobuf *iobuf = NULL;
+    struct iobref *iobref = NULL;
+    struct iovec vec = {
+        0,
+    };
+    svs_fd_t *sfd = NULL;
+    int ret = -1;
+    struct glfs_stat fstatbuf = {
+        0,
+    };
+    glfs_fd_t *glfd = NULL;
+    struct iatt stbuf = {
+        0,
+    };
+    call_stack_t *root = NULL;
+
+    GF_VALIDATE_OR_GOTO("snap-view-daemon", this, out);
+    GF_VALIDATE_OR_GOTO(this->name, frame, out);
+    GF_VALIDATE_OR_GOTO(this->name, fd, out);
+    GF_VALIDATE_OR_GOTO(this->name, fd->inode, out);
+
+    priv = this->private;
+    VALIDATE_OR_GOTO(priv, out);
+
+    root = frame->root;
+    op_ret = gf_setcredentials(&root->uid, &root->gid, root->ngrps,
+                               root->groups);
+    if (op_ret != 0) {
+        goto out;
+    }
+
+    if (!svs_inode_glfs_mapping(this, fd->inode)) {
+        op_ret = -1;
+        op_errno = EBADF; /* should this be some other error? */
+        gf_msg(this->name, GF_LOG_ERROR, op_errno, SVS_MSG_FS_INSTANCE_INVALID,
+               "glfs instance to which the inode "
+               "%s receiving read request belongs, "
+               "does not exist anymore",
+               uuid_utoa(fd->inode->gfid));
+        goto out;
+    }
+
+    sfd = svs_fd_ctx_get_or_new(this, fd);
+    if (!sfd) {
+        op_ret = -1;
+        op_errno = EBADFD;
+        gf_msg(this->name, GF_LOG_ERROR, op_errno,
+               SVS_MSG_GET_INODE_CONTEXT_FAILED,
+               "failed to get the fd "
+               "context for %s",
+               uuid_utoa(fd->inode->gfid));
+        goto out;
+    }
+
+    glfd = sfd->fd;
+
+    iobuf = iobuf_get2(this->ctx->iobuf_pool, size);
+    if (!iobuf) {
+        op_ret = -1;
+        op_errno = ENOMEM;
+        gf_msg(this->name, GF_LOG_ERROR, op_errno, SVS_MSG_NO_MEMORY,
+               "failed to "
+               "allocate iobuf while reading the "
+               "file with gfid %s",
+               uuid_utoa(fd->inode->gfid));
+        goto out;
+    }
+
+    ret = glfs_pread(glfd, iobuf->ptr, size, offset, 0, &fstatbuf);
+    if (ret < 0) {
+        op_ret = -1;
+        op_errno = errno;
+        gf_msg(this->name, GF_LOG_ERROR, op_errno, SVS_MSG_READ_FAILED,
+               "glfs_read failed on %s (%s)", uuid_utoa(fd->inode->gfid),
+               strerror(op_errno));
+        goto out;
+    }
+
+    vec.iov_base = iobuf->ptr;
+    vec.iov_len = ret;
+
+    iobref = iobref_new();
+
+    iobref_add(iobref, iobuf);
+    glfs_iatt_from_statx(&stbuf, &fstatbuf);
+    gf_uuid_copy(stbuf.ia_gfid, fd->inode->gfid);
+    svs_fill_ino_from_gfid(&stbuf);
+
+    /* Hack to notify higher layers of EOF. */
+    if (!stbuf.ia_size || (offset + vec.iov_len) >= stbuf.ia_size)
+        op_errno = ENOENT;
+
+    op_ret = vec.iov_len;
 
-        SVS_GET_INODE_CTX_INFO(inode_ctx, fs, object, this, loc, op_ret,
-                               op_errno, out);
+out:
 
-        ret = glfs_h_stat (fs, object, &stat);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "glfs_h_stat on %s "
-                        "(gfid: %s) failed", loc->name,
-                        uuid_utoa (loc->inode->gfid));
-                op_ret = -1;
-                op_errno = errno;
-                goto out;
-        }
+    STACK_UNWIND_STRICT(readv, frame, op_ret, op_errno, &vec, 1, &stbuf, iobref,
+                        NULL);
 
-        iatt_from_stat (&stbuf, &stat);
-        gf_uuid_copy (stbuf.ia_gfid, loc->inode->gfid);
-        svs_fill_ino_from_gfid (&stbuf);
+    if (iobref)
+        iobref_unref(iobref);
+    if (iobuf)
+        iobuf_unref(iobuf);
 
-        buf = alloca (size + 1);
-        op_ret = glfs_h_readlink (fs, object, buf, size);
-        if (op_ret == -1) {
-                gf_log (this->name, GF_LOG_ERROR, "readlink on %s failed "
-                        "(gfid: %s)", loc->name, uuid_utoa (loc->inode->gfid));
-                op_errno = errno;
-                goto out;
-        }
+    return 0;
+}
 
-        buf[op_ret] = 0;
+int32_t
+svs_readlink(call_frame_t *frame, xlator_t *this, loc_t *loc, size_t size,
+             dict_t *xdata)
+{
+    svs_inode_t *inode_ctx = NULL;
+    glfs_t *fs = NULL;
+    glfs_object_t *object = NULL;
+    int op_ret = -1;
+    int op_errno = EINVAL;
+    char *buf = NULL;
+    struct iatt stbuf = {
+        0,
+    };
+    int ret = -1;
+    struct stat stat = {
+        0,
+    };
+    call_stack_t *root = NULL;
+
+    GF_VALIDATE_OR_GOTO("snap-view-daemon", this, out);
+    GF_VALIDATE_OR_GOTO(this->name, frame, out);
+    GF_VALIDATE_OR_GOTO(this->name, loc, out);
+    GF_VALIDATE_OR_GOTO(this->name, loc->inode, out);
+
+    root = frame->root;
+    op_ret = gf_setcredentials(&root->uid, &root->gid, root->ngrps,
+                               root->groups);
+    if (op_ret != 0) {
+        goto out;
+    }
+
+    inode_ctx = svs_inode_ctx_get(this, loc->inode);
+    if (!inode_ctx) {
+        op_ret = -1;
+        op_errno = EINVAL;
+        gf_msg(this->name, GF_LOG_ERROR, op_errno,
+               SVS_MSG_GET_INODE_CONTEXT_FAILED,
+               "failed to get inode context "
+               "for %s (gfid: %s)",
+               loc->name, uuid_utoa(loc->inode->gfid));
+        goto out;
+    }
+
+    SVS_GET_INODE_CTX_INFO(inode_ctx, fs, object, this, loc, op_ret, op_errno,
+                           out);
+
+    ret = glfs_h_stat(fs, object, &stat);
+    if (ret) {
+        op_ret = -1;
+        op_errno = errno;
+        gf_msg(this->name, GF_LOG_ERROR, op_errno, SVS_MSG_STAT_FAILED,
+               "glfs_h_stat on %s (gfid: %s) "
+               "failed",
+               loc->name, uuid_utoa(loc->inode->gfid));
+        goto out;
+    }
+
+    iatt_from_stat(&stbuf, &stat);
+    gf_uuid_copy(stbuf.ia_gfid, loc->inode->gfid);
+    svs_fill_ino_from_gfid(&stbuf);
+
+    buf = alloca(size + 1);
+    op_ret = glfs_h_readlink(fs, object, buf, size);
+    if (op_ret == -1) {
+        op_errno = errno;
+        gf_msg(this->name, GF_LOG_ERROR, op_errno, SVS_MSG_READLINK_FAILED,
+               "readlink on %s failed (gfid: %s)", loc->name,
+               uuid_utoa(loc->inode->gfid));
+        goto out;
+    }
+
+    buf[op_ret] = 0;
 
 out:
-        STACK_UNWIND_STRICT (readlink, frame, op_ret, op_errno, buf, &stbuf,
-                             NULL);
+    STACK_UNWIND_STRICT(readlink, frame, op_ret, op_errno, buf, &stbuf, NULL);
 
-        return 0;
+    return 0;
 }
 
 int32_t
-svs_access (call_frame_t *frame, xlator_t *this, loc_t *loc, int mask,
-            dict_t *xdata)
+svs_access(call_frame_t *frame, xlator_t *this, loc_t *loc, int mask,
+           dict_t *xdata)
 {
-        int             ret          = -1;
-        int32_t         op_ret       = -1;
-        int32_t         op_errno     = EINVAL;
-        svs_private_t  *priv         = NULL;
-        glfs_t         *fs           = NULL;
-        glfs_object_t  *object       = NULL;
-        svs_inode_t    *inode_ctx    = NULL;
-        gf_boolean_t    is_fuse_call = 0;
-        int             mode         = 0;
-
-        GF_VALIDATE_OR_GOTO ("svs", this, out);
-        GF_VALIDATE_OR_GOTO (this->name, this->private, out);
-        GF_VALIDATE_OR_GOTO (this->name, loc, out);
-        GF_VALIDATE_OR_GOTO (this->name, loc->inode, out);
-
-        priv = this->private;
-
-        inode_ctx = svs_inode_ctx_get (this, loc->inode);
-        if (!inode_ctx) {
-                gf_log (this->name, GF_LOG_ERROR, "inode context not found for"
-                        " %s", uuid_utoa (loc->inode->gfid));
-                op_ret = -1;
-                op_errno = EINVAL;
-                goto out;
-        }
-
-        is_fuse_call = __is_fuse_call (frame);
-
-        /*
-         * For entry-point directory, set read and execute bits. But not write
-         * permissions.
-         */
-        if (inode_ctx->type == SNAP_VIEW_ENTRY_POINT_INODE) {
-                if (is_fuse_call) {
-                        op_ret = 0;
-                        op_errno = 0;
-                } else {
-                        op_ret = 0;
-                        mode |= POSIX_ACL_READ;
-                        mode |= POSIX_ACL_EXECUTE;
-                        op_errno = mode;
-                }
-                goto out;
-        }
-
-
-        SVS_GET_INODE_CTX_INFO(inode_ctx, fs, object, this, loc, op_ret,
-                               op_errno, out);
-
-        /* The actual posix_acl xlator does acl checks differently for
-           fuse and nfs. So set frame->root->pid as fspid of the syncop
-           if the call came from nfs
-        */
-        if (!is_fuse_call) {
-                syncopctx_setfspid (&frame->root->pid);
-                syncopctx_setfsuid (&frame->root->uid);
-                syncopctx_setfsgid (&frame->root->gid);
-                syncopctx_setfsgroups (frame->root->ngrps,
-                                       frame->root->groups);
-        }
-
-        ret = glfs_h_access (fs, object, mask);
-        if (ret < 0) {
-                gf_log (this->name, GF_LOG_ERROR, "failed to access %s "
-                        "(gfid: %s)", loc->path, uuid_utoa (loc->inode->gfid));
-                op_ret = -1;
-                op_errno = errno;
-                goto out;
-        }
-
-        op_ret = 0;
-        op_errno = ret;
+    int ret = -1;
+    int32_t op_ret = -1;
+    int32_t op_errno = EINVAL;
+    glfs_t *fs = NULL;
+    glfs_object_t *object = NULL;
+    svs_inode_t *inode_ctx = NULL;
+    gf_boolean_t is_fuse_call = 0;
+    int mode = 0;
+    call_stack_t *root = NULL;
+
+    GF_VALIDATE_OR_GOTO("svs", this, out);
+    GF_VALIDATE_OR_GOTO(this->name, this->private, out);
+    GF_VALIDATE_OR_GOTO(this->name, frame, out);
+    GF_VALIDATE_OR_GOTO(this->name, loc, out);
+    GF_VALIDATE_OR_GOTO(this->name, loc->inode, out);
+
+    root = frame->root;
+    op_ret = gf_setcredentials(&root->uid, &root->gid, root->ngrps,
+                               root->groups);
+    if (op_ret != 0) {
+        goto out;
+    }
+
+    inode_ctx = svs_inode_ctx_get(this, loc->inode);
+    if (!inode_ctx) {
+        op_ret = -1;
+        op_errno = EINVAL;
+        gf_msg(this->name, GF_LOG_ERROR, op_errno,
+               SVS_MSG_GET_INODE_CONTEXT_FAILED,
+               "inode context not found for %s", uuid_utoa(loc->inode->gfid));
+        goto out;
+    }
+
+    is_fuse_call = __is_fuse_call(frame);
+
+    /*
+     * For entry-point directory, set read and execute bits. But not write
+     * permissions.
+     */
+    if (inode_ctx->type == SNAP_VIEW_ENTRY_POINT_INODE) {
+        if (is_fuse_call) {
+            op_ret = 0;
+            op_errno = 0;
+        } else {
+            op_ret = 0;
+            mode |= POSIX_ACL_READ;
+            mode |= POSIX_ACL_EXECUTE;
+            op_errno = mode;
+        }
+        goto out;
+    }
+
+    SVS_GET_INODE_CTX_INFO(inode_ctx, fs, object, this, loc, op_ret, op_errno,
+                           out);
+
+    /* The actual posix_acl xlator does acl checks differently for
+       fuse and nfs. So set frame->root->pid as fspid of the syncop
+       if the call came from nfs
+    */
+    if (!is_fuse_call) {
+        syncopctx_setfspid(&frame->root->pid);
+        syncopctx_setfsuid(&frame->root->uid);
+        syncopctx_setfsgid(&frame->root->gid);
+        syncopctx_setfsgroups(frame->root->ngrps, frame->root->groups);
+    }
+
+    ret = glfs_h_access(fs, object, mask);
+    if (ret < 0) {
+        op_ret = -1;
+        op_errno = errno;
+        gf_msg(this->name, GF_LOG_ERROR, op_errno, SVS_MSG_ACCESS_FAILED,
+               "failed to access %s (gfid: %s)", loc->path,
+               uuid_utoa(loc->inode->gfid));
+        goto out;
+    }
+
+    op_ret = 0;
+    op_errno = ret;
 
 out:
 
-        STACK_UNWIND_STRICT (access, frame, op_ret, op_errno, NULL);
-        return 0;
+    STACK_UNWIND_STRICT(access, frame, op_ret, op_errno, NULL);
+    return 0;
 }
 
-
 int32_t
-mem_acct_init (xlator_t *this)
+notify(xlator_t *this, int32_t event, void *data, ...)
 {
-        int     ret = -1;
-
-        if (!this)
-                return ret;
-
-        ret = xlator_mem_acct_init (this, gf_svs_mt_end + 1);
-
-        if (ret != 0) {
-                gf_log (this->name, GF_LOG_WARNING, "Memory accounting"
-                        " init failed");
-                return ret;
-        }
-
-        return ret;
+    switch (event) {
+        case GF_EVENT_PARENT_UP: {
+            /* Tell the parent that snapview-server xlator is up */
+            default_notify(this, GF_EVENT_CHILD_UP, data);
+        } break;
+        default:
+            break;
+    }
+    return 0;
 }
 
 int32_t
-init (xlator_t *this)
+mem_acct_init(xlator_t *this)
 {
-        svs_private_t   *priv           = NULL;
-        int             ret             = -1;
-        pthread_t       snap_thread;
-
-        /* This can be the top of graph in certain cases */
-        if (!this->parents) {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "dangling volume. check volfile ");
-        }
-
-        priv = GF_CALLOC (1, sizeof (*priv), gf_svs_mt_priv_t);
-        if (!priv)
-                goto out;
+    int ret = -1;
 
-        this->private = priv;
-
-        GF_OPTION_INIT ("volname", priv->volname, str, out);
-        LOCK_INIT (&priv->snaplist_lock);
+    if (!this)
+        return ret;
 
-        LOCK (&priv->snaplist_lock);
-        {
-                priv->num_snaps = 0;
-        }
-        UNLOCK (&priv->snaplist_lock);
+    ret = xlator_mem_acct_init(this, gf_svs_mt_end + 1);
 
-        /* What to do here upon failure? should init be failed or succeed? */
-        /* If succeeded, then dynamic management of snapshots will not */
-        /* happen.*/
-        ret = svs_mgmt_init (this);
-        if (ret) {
-                gf_log (this->name, GF_LOG_WARNING, "failed to initiate the "
-                        "mgmt rpc callback for svs. Dymamic management of the"
-                        "snapshots will not happen");
-                goto out;
-        }
+    if (ret != 0) {
+        gf_msg(this->name, GF_LOG_WARNING, 0, SVS_MSG_MEM_ACNT_FAILED,
+               "Memory accounting"
+               " init failed");
+        return ret;
+    }
 
-        /* get the list of snaps first to return to client xlator */
-        ret = svs_get_snapshot_list (this);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "Error initializing snaplist infrastructure");
-                ret = -1;
-                goto out;
-        }
+    return ret;
+}
 
-        ret = 0;
+int32_t
+init(xlator_t *this)
+{
+    svs_private_t *priv = NULL;
+    int ret = -1;
+
+    /* This can be the top of graph in certain cases */
+    if (!this->parents) {
+        gf_msg_debug(this->name, 0, "dangling volume. check volfile ");
+    }
+
+    priv = GF_CALLOC(1, sizeof(*priv), gf_svs_mt_priv_t);
+    if (!priv) {
+        gf_msg(this->name, GF_LOG_ERROR, ENOMEM, SVS_MSG_NO_MEMORY,
+               "failed to "
+               "allocate memory for this->private ");
+        goto out;
+    }
+
+    this->private = priv;
+
+    GF_OPTION_INIT("volname", priv->volname, str, out);
+    LOCK_INIT(&priv->snaplist_lock);
+
+    LOCK(&priv->snaplist_lock);
+    {
+        priv->num_snaps = 0;
+    }
+    UNLOCK(&priv->snaplist_lock);
+
+    /* What to do here upon failure? should init be failed or succeed? */
+    /* If succeeded, then dynamic management of snapshots will not */
+    /* happen.*/
+    ret = svs_mgmt_init(this);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_WARNING, EINVAL, SVS_MSG_MGMT_INIT_FAILED,
+               "failed to initiate the "
+               "mgmt rpc callback for svs. Dymamic management of the"
+               "snapshots will not happen");
+        goto out;
+    }
+
+    /* get the list of snaps first to return to client xlator */
+    ret = svs_get_snapshot_list(this);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, EINVAL,
+               SVS_MSG_GET_SNAPSHOT_LIST_FAILED,
+               "Error initializing snaplist infrastructure");
+        ret = -1;
+        goto out;
+    }
+
+    ret = 0;
 
 out:
-        if (ret && priv) {
-                LOCK_DESTROY (&priv->snaplist_lock);
-                GF_FREE (priv->dirents);
-                GF_FREE (priv);
-        }
+    if (ret && priv) {
+        LOCK_DESTROY(&priv->snaplist_lock);
+        GF_FREE(priv->dirents);
+        GF_FREE(priv);
+    }
 
-        return ret;
+    return ret;
 }
 
 void
-fini (xlator_t *this)
+fini(xlator_t *this)
 {
-        svs_private_t   *priv   = NULL;
-        glusterfs_ctx_t *ctx    = NULL;
-        int             ret     = 0;
-
-        GF_ASSERT (this);
-        priv = this->private;
-        this->private = NULL;
-        ctx = this->ctx;
-        if (!ctx)
-                gf_log (this->name, GF_LOG_ERROR,
-                        "Invalid ctx found");
-
-        if (priv) {
-                ret = LOCK_DESTROY (&priv->snaplist_lock);
-                if (ret != 0) {
-                        gf_log (this->name, GF_LOG_WARNING,
-                                "Could not destroy mutex snaplist_lock");
-                }
-
-                if (priv->dirents) {
-                        GF_FREE (priv->dirents);
-                }
+    svs_private_t *priv = NULL;
+    glusterfs_ctx_t *ctx = NULL;
+    int ret = 0;
+
+    GF_ASSERT(this);
+    priv = this->private;
+    this->private = NULL;
+    ctx = this->ctx;
+    if (!ctx)
+        gf_msg(this->name, GF_LOG_ERROR, 0, SVS_MSG_INVALID_GLFS_CTX,
+               "Invalid ctx found");
+
+    if (priv) {
+        ret = LOCK_DESTROY(&priv->snaplist_lock);
+        if (ret != 0) {
+            gf_msg(this->name, GF_LOG_WARNING, errno,
+                   SVS_MSG_LOCK_DESTROY_FAILED,
+                   "Could not destroy mutex snaplist_lock");
+        }
 
-                if (priv->rpc) {
-                        /* cleanup the saved-frames before last unref */
-                        rpc_clnt_connection_cleanup (&priv->rpc->conn);
-                        rpc_clnt_unref (priv->rpc);
-                }
+        if (priv->dirents) {
+            GF_FREE(priv->dirents);
+        }
 
-                GF_FREE (priv);
+        if (priv->rpc) {
+            /* cleanup the saved-frames before last unref */
+            rpc_clnt_connection_cleanup(&priv->rpc->conn);
+            rpc_clnt_unref(priv->rpc);
         }
 
-        return;
+        GF_FREE(priv);
+    }
+
+    return;
 }
 
 struct xlator_fops fops = {
-        .lookup     = svs_lookup,
-        .stat       = svs_stat,
-        .opendir    = svs_opendir,
-        .readdirp   = svs_readdirp,
-        .readdir    = svs_readdir,
-        .open       = svs_open,
-        .readv      = svs_readv,
-        .flush      = svs_flush,
-        .fstat      = svs_fstat,
-        .getxattr   = svs_getxattr,
-        .access     = svs_access,
-        .readlink   = svs_readlink,
-        /* entry fops */
+    .lookup = svs_lookup,
+    .stat = svs_stat,
+    .statfs = svs_statfs,
+    .opendir = svs_opendir,
+    .readdirp = svs_readdirp,
+    .readdir = svs_readdir,
+    .open = svs_open,
+    .readv = svs_readv,
+    .flush = svs_flush,
+    .fstat = svs_fstat,
+    .getxattr = svs_getxattr,
+    .access = svs_access,
+    .readlink = svs_readlink,
+    /* entry fops */
 };
 
 struct xlator_cbks cbks = {
-        .release  = svs_release,
-        .releasedir = svs_releasedir,
-        .forget     = svs_forget,
+    .release = svs_release,
+    .releasedir = svs_releasedir,
+    .forget = svs_forget,
 };
 
 struct volume_options options[] = {
-        { .key  = {"volname"},
-          .type = GF_OPTION_TYPE_STR,
-        },
-        { .key  = {NULL} },
+    {
+        .key = {"volname"},
+        .type = GF_OPTION_TYPE_STR,
+    },
+    {.key = {NULL}},
+};
+
+xlator_api_t xlator_api = {
+    .init = init,
+    .fini = fini,
+    .notify = notify,
+    .mem_acct_init = mem_acct_init,
+    .op_version = {1},
+    .fops = &fops,
+    .cbks = &cbks,
+    .options = options,
+    .identifier = "snapview-server",
+    .category = GF_MAINTAINED,
 };
diff --git a/xlators/features/snapview-server/src/snapview-server.h b/xlators/features/snapview-server/src/snapview-server.h
index c80d3456c30..6472422e715 100644
--- a/xlators/features/snapview-server/src/snapview-server.h
+++ b/xlators/features/snapview-server/src/snapview-server.h
@@ -10,217 +10,246 @@
 #ifndef __SNAP_VIEW_H__
 #define __SNAP_VIEW_H__
 
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
-
-#include "dict.h"
-#include "defaults.h"
-#include "mem-types.h"
-#include "call-stub.h"
-#include "inode.h"
-#include "byte-order.h"
-#include "iatt.h"
+#include <glusterfs/dict.h>
+#include <glusterfs/defaults.h>
+#include <glusterfs/mem-types.h>
+#include <glusterfs/call-stub.h>
+#include <glusterfs/byte-order.h>
+#include <glusterfs/iatt.h>
 #include <ctype.h>
 #include <sys/uio.h>
-#include "glusterfs.h"
-#include "xlator.h"
-#include "logging.h"
+#include <glusterfs/glusterfs.h>
+#include <glusterfs/logging.h>
 #include "glfs.h"
-#include "common-utils.h"
 #include "glfs-handles.h"
 #include "glfs-internal.h"
 #include "glusterfs3-xdr.h"
-#include "glusterfs-acl.h"
-#include "syncop.h"
-#include "list.h"
-#include "timer.h"
+#include <glusterfs/glusterfs-acl.h>
+#include <glusterfs/syncop.h>
+#include <glusterfs/list.h>
+#include <glusterfs/timer.h>
 #include "rpc-clnt.h"
 #include "protocol-common.h"
 #include "xdr-generic.h"
-
+#include "snapview-server-messages.h"
 
 #define DEFAULT_SVD_LOG_FILE_DIRECTORY DATADIR "/log/glusterfs"
 
-#define SNAP_VIEW_MAX_GLFS_T            256
-#define SNAP_VIEW_MAX_GLFS_FDS          1024
-#define SNAP_VIEW_MAX_GLFS_OBJ_HANDLES  1024
-
-#define SVS_STACK_DESTROY(_frame)                                   \
-        do {                                                        \
-                ((call_frame_t *)_frame)->local = NULL;             \
-                STACK_DESTROY (((call_frame_t *)_frame)->root);     \
-        } while (0)
-
-#define SVS_CHECK_VALID_SNAPSHOT_HANDLE(fs, this)                       \
-        do {                                                            \
-                svs_private_t *_private = NULL;                         \
-                _private = this->private;                               \
-                int  i = 0;                                             \
-                gf_boolean_t found = _gf_false;                         \
-                LOCK (&_private->snaplist_lock);                        \
-                {                                                       \
-                        for (i = 0; i < _private->num_snaps; i++) {     \
-                                if (_private->dirents->fs && fs &&      \
-                                    _private->dirents->fs == fs) {      \
-                                        found = _gf_true;               \
-                                        break;                          \
-                                }                                       \
-                        }                                               \
-                }                                                       \
-                UNLOCK (&_private->snaplist_lock);                      \
-                                                                        \
-                if (!found)                                             \
-                        fs = NULL;                                      \
-        } while (0)
-
-#define SVS_GET_INODE_CTX_INFO(inode_ctx, fs, object, this, loc, ret,   \
-                               op_errno, label)                         \
-        do {                                                            \
-                fs = inode_ctx->fs;                                     \
-                object = inode_ctx->object;                             \
-                SVS_CHECK_VALID_SNAPSHOT_HANDLE (fs, this);             \
-                if (!fs)                                                \
-                        object = NULL;                                  \
-                                                                        \
-                if (!fs || !object) {                                   \
-                        int32_t tmp = -1;                               \
-                        char    tmp_uuid[64];                           \
-                                                                        \
-                        tmp = svs_get_handle (this, loc, inode_ctx,     \
-                                              &op_errno);               \
-                        if (tmp) {                                      \
-                                gf_log (this->name, GF_LOG_ERROR,       \
-                                        "failed to get the handle for %s " \
-                                        "(gfid: %s)", loc->path,        \
-                                        uuid_utoa_r (loc->inode->gfid,  \
-                                                     tmp_uuid));        \
-                                ret = -1;                               \
-                                goto label;                             \
-                        }                                               \
-                                                                        \
-                        fs = inode_ctx->fs;                             \
-                        object = inode_ctx->object;                     \
-                }                                                       \
-        } while(0);
+#define SNAP_VIEW_MAX_GLFS_T 256
+#define SNAP_VIEW_MAX_GLFS_FDS 1024
+#define SNAP_VIEW_MAX_GLFS_OBJ_HANDLES 1024
+
+#define SVS_STACK_DESTROY(_frame)                                              \
+    do {                                                                       \
+        ((call_frame_t *)_frame)->local = NULL;                                \
+        STACK_DESTROY(((call_frame_t *)_frame)->root);                         \
+    } while (0)
+
+#define SVS_CHECK_VALID_SNAPSHOT_HANDLE(fs, this)                              \
+    do {                                                                       \
+        svs_private_t *_private = NULL;                                        \
+        _private = this->private;                                              \
+        int i = 0;                                                             \
+        gf_boolean_t found = _gf_false;                                        \
+        glfs_t *tmp_fs = NULL;                                                 \
+        LOCK(&_private->snaplist_lock);                                        \
+        {                                                                      \
+            for (i = 0; i < _private->num_snaps; i++) {                        \
+                tmp_fs = _private->dirents[i].fs;                              \
+                gf_log(this->name, GF_LOG_DEBUG,                               \
+                       "snap name: %s, snap volume: %s,"                       \
+                       "dirent->fs: %p",                                       \
+                       _private->dirents[i].name,                              \
+                       _private->dirents[i].snap_volname, tmp_fs);             \
+                if (tmp_fs && fs && (tmp_fs == fs)) {                          \
+                    found = _gf_true;                                          \
+                    gf_msg_debug(this->name, 0,                                \
+                                 "found the fs "                               \
+                                 "instance");                                  \
+                    break;                                                     \
+                }                                                              \
+            }                                                                  \
+        }                                                                      \
+        UNLOCK(&_private->snaplist_lock);                                      \
+                                                                               \
+        if (!found) {                                                          \
+            gf_log(this->name, GF_LOG_WARNING,                                 \
+                   "failed to"                                                 \
+                   " find the fs instance %p",                                 \
+                   fs);                                                        \
+            fs = NULL;                                                         \
+        }                                                                      \
+    } while (0)
+
+#define SVS_GET_INODE_CTX_INFO(inode_ctx, fs, object, this, loc, ret,          \
+                               op_errno, label)                                \
+    do {                                                                       \
+        fs = inode_ctx->fs;                                                    \
+        object = inode_ctx->object;                                            \
+        SVS_CHECK_VALID_SNAPSHOT_HANDLE(fs, this);                             \
+        if (!fs)                                                               \
+            object = NULL;                                                     \
+                                                                               \
+        if (!fs || !object) {                                                  \
+            int32_t tmp = -1;                                                  \
+            char tmp_uuid[64];                                                 \
+                                                                               \
+            tmp = svs_get_handle(this, loc, inode_ctx, &op_errno);             \
+            if (tmp) {                                                         \
+                gf_log(this->name, GF_LOG_ERROR,                               \
+                       "failed to get the handle for %s "                      \
+                       "(gfid: %s)",                                           \
+                       loc->path, uuid_utoa_r(loc->inode->gfid, tmp_uuid));    \
+                ret = -1;                                                      \
+                goto label;                                                    \
+            }                                                                  \
+                                                                               \
+            fs = inode_ctx->fs;                                                \
+            object = inode_ctx->object;                                        \
+        }                                                                      \
+    } while (0);
+
+#define SVS_STRDUP(dst, src)                                                   \
+    do {                                                                       \
+        if (dst && strcmp(src, dst)) {                                         \
+            GF_FREE(dst);                                                      \
+            dst = NULL;                                                        \
+        }                                                                      \
+                                                                               \
+        if (!dst)                                                              \
+            dst = gf_strdup(src);                                              \
+    } while (0)
 
 int
-svs_mgmt_submit_request (void *req, call_frame_t *frame,
-                         glusterfs_ctx_t *ctx,
-                         rpc_clnt_prog_t *prog, int procnum,
-                         fop_cbk_fn_t cbkfn, xdrproc_t xdrproc);
+svs_mgmt_submit_request(void *req, call_frame_t *frame, glusterfs_ctx_t *ctx,
+                        rpc_clnt_prog_t *prog, int procnum, fop_cbk_fn_t cbkfn,
+                        xdrproc_t xdrproc);
 
 int
-svs_get_snapshot_list (xlator_t *this);
+svs_get_snapshot_list(xlator_t *this);
 
 int
-mgmt_get_snapinfo_cbk (struct rpc_req *req, struct iovec *iov,
-                       int count, void *myframe);
+mgmt_get_snapinfo_cbk(struct rpc_req *req, struct iovec *iov, int count,
+                      void *myframe);
 
 typedef enum {
-        SNAP_VIEW_ENTRY_POINT_INODE = 0,
-        SNAP_VIEW_SNAPSHOT_INODE,
-        SNAP_VIEW_VIRTUAL_INODE
+    SNAP_VIEW_ENTRY_POINT_INODE = 0,
+    SNAP_VIEW_SNAPSHOT_INODE,
+    SNAP_VIEW_VIRTUAL_INODE
 } inode_type_t;
 
 struct svs_inode {
-        glfs_t *fs;
-        glfs_object_t *object;
-        inode_type_t type;
-
-        /* used only for entry point directory where gfid of the directory
-           from where the entry point was entered is saved.
-        */
-        uuid_t pargfid;
-        struct iatt buf;
+    glfs_t *fs;
+    glfs_object_t *object;
+    inode_type_t type;
+
+    /* used only for entry point directory where gfid of the directory
+       from where the entry point was entered is saved.
+    */
+    uuid_t pargfid;
+
+    /* This is used to generate gfid for all sub files/dirs under this
+     * snapshot
+     */
+    char *snapname;
+    struct iatt buf;
 };
 typedef struct svs_inode svs_inode_t;
 
 struct svs_fd {
-        glfs_fd_t *fd;
+    glfs_fd_t *fd;
 };
 typedef struct svs_fd svs_fd_t;
 
 struct snap_dirent {
-        char name[NAME_MAX];
-        char uuid[UUID_CANONICAL_FORM_LEN + 1];
-        char snap_volname[NAME_MAX];
-        glfs_t *fs;
+    char name[NAME_MAX];
+    char uuid[UUID_CANONICAL_FORM_LEN + 1];
+    char snap_volname[NAME_MAX];
+    glfs_t *fs;
 };
 typedef struct snap_dirent snap_dirent_t;
 
 struct svs_private {
-        snap_dirent_t           *dirents;
-        int                     num_snaps;
-        char                    *volname;
-        struct list_head        snaplist;
-        gf_lock_t               snaplist_lock;
-        struct rpc_clnt         *rpc;
+    snap_dirent_t *dirents;
+    int num_snaps;
+    char *volname;
+    struct list_head snaplist;
+    gf_lock_t snaplist_lock;
+    struct rpc_clnt *rpc;
 };
 typedef struct svs_private svs_private_t;
 
 int
-__svs_inode_ctx_set (xlator_t *this, inode_t *inode, svs_inode_t *svs_inode);
+__svs_inode_ctx_set(xlator_t *this, inode_t *inode, svs_inode_t *svs_inode);
 
 svs_inode_t *
-__svs_inode_ctx_get (xlator_t *this, inode_t *inode);
+__svs_inode_ctx_get(xlator_t *this, inode_t *inode);
 
 svs_inode_t *
-svs_inode_ctx_get (xlator_t *this, inode_t *inode);
+svs_inode_ctx_get(xlator_t *this, inode_t *inode);
 
 int32_t
-svs_inode_ctx_set (xlator_t *this, inode_t *inode, svs_inode_t *svs_inode);
+svs_inode_ctx_set(xlator_t *this, inode_t *inode, svs_inode_t *svs_inode);
 
 svs_inode_t *
-svs_inode_ctx_get_or_new (xlator_t *this, inode_t *inode);
+svs_inode_ctx_get_or_new(xlator_t *this, inode_t *inode);
 
 int
-__svs_fd_ctx_set (xlator_t *this, fd_t *fd, svs_fd_t *svs_fd);
+__svs_fd_ctx_set(xlator_t *this, fd_t *fd, svs_fd_t *svs_fd);
 
 svs_fd_t *
-__svs_fd_ctx_get (xlator_t *this, fd_t *fd);
+__svs_fd_ctx_get(xlator_t *this, fd_t *fd);
 
 svs_fd_t *
-svs_fd_ctx_get (xlator_t *this, fd_t *fd);
+svs_fd_ctx_get(xlator_t *this, fd_t *fd);
 
 int32_t
-svs_fd_ctx_set (xlator_t *this, fd_t *fd, svs_fd_t *svs_fd);
+svs_fd_ctx_set(xlator_t *this, fd_t *fd, svs_fd_t *svs_fd);
 
 svs_fd_t *
-__svs_fd_ctx_get_or_new (xlator_t *this, fd_t *fd);
+__svs_fd_ctx_get_or_new(xlator_t *this, fd_t *fd);
 
 svs_fd_t *
-svs_fd_ctx_get_or_new (xlator_t *this, fd_t *fd);
+svs_fd_ctx_get_or_new(xlator_t *this, fd_t *fd);
+
+int
+svs_uuid_generate(xlator_t *this, uuid_t gfid, char *snapname,
+                  uuid_t origin_gfid);
 
 void
-svs_fill_ino_from_gfid (struct iatt *buf);
+svs_fill_ino_from_gfid(struct iatt *buf);
 
 void
-svs_iatt_fill (uuid_t gfid, struct iatt *buf);
+svs_iatt_fill(uuid_t gfid, struct iatt *buf);
 
 snap_dirent_t *
-svs_get_latest_snap_entry (xlator_t *this);
+svs_get_latest_snap_entry(xlator_t *this);
 
 glfs_t *
-svs_get_latest_snapshot (xlator_t *this);
+svs_get_latest_snapshot(xlator_t *this);
 
 glfs_t *
-svs_initialise_snapshot_volume (xlator_t *this, const char *name,
-                                int32_t *op_errno);
+svs_initialise_snapshot_volume(xlator_t *this, const char *name,
+                               int32_t *op_errno);
 
 glfs_t *
-__svs_initialise_snapshot_volume (xlator_t *this, const char *name,
-                                  int32_t *op_errno);
+__svs_initialise_snapshot_volume(xlator_t *this, const char *name,
+                                 int32_t *op_errno);
 
 snap_dirent_t *
-__svs_get_snap_dirent (xlator_t *this, const char *name);
+__svs_get_snap_dirent(xlator_t *this, const char *name);
 
 int
-svs_mgmt_init (xlator_t *this);
+svs_mgmt_init(xlator_t *this);
 
 int32_t
-svs_get_handle (xlator_t *this, loc_t *loc, svs_inode_t *inode_ctx,
-                int32_t *op_errno);
+svs_get_handle(xlator_t *this, loc_t *loc, svs_inode_t *inode_ctx,
+               int32_t *op_errno);
+
+glfs_t *
+svs_inode_glfs_mapping(xlator_t *this, inode_t *inode);
+
+glfs_t *
+svs_inode_ctx_glfs_mapping(xlator_t *this, svs_inode_t *inode_ctx);
 
 #endif /* __SNAP_VIEW_H__ */
diff --git a/xlators/features/thin-arbiter/Makefile.am b/xlators/features/thin-arbiter/Makefile.am
new file mode 100644
index 00000000000..a985f42a877
--- /dev/null
+++ b/xlators/features/thin-arbiter/Makefile.am
@@ -0,0 +1,3 @@
+SUBDIRS = src
+
+CLEANFILES =
diff --git a/xlators/features/thin-arbiter/src/Makefile.am b/xlators/features/thin-arbiter/src/Makefile.am
new file mode 100644
index 00000000000..a3c133e7798
--- /dev/null
+++ b/xlators/features/thin-arbiter/src/Makefile.am
@@ -0,0 +1,22 @@
+xlator_LTLIBRARIES = thin-arbiter.la
+
+xlatordir = $(libdir)/glusterfs/$(PACKAGE_VERSION)/xlator/features
+
+thin_arbiter_la_LDFLAGS = -module $(GF_XLATOR_DEFAULT_LDFLAGS)
+
+thin_arbiter_la_SOURCES = thin-arbiter.c \
+    $(top_builddir)/xlators/lib/src/libxlator.c
+
+thin_arbiter_la_LIBADD = $(top_builddir)/libglusterfs/src/libglusterfs.la
+
+noinst_HEADERS = thin-arbiter.h thin-arbiter-mem-types.h thin-arbiter-messages.h \
+    $(top_builddir)/xlators/lib/src/libxlator.h
+
+AM_CPPFLAGS = $(GF_CPPFLAGS) \
+    -I$(top_srcdir)/libglusterfs/src -I$(top_srcdir)/xlators/lib/src \
+    -I$(top_srcdir)/rpc/rpc-lib/src \
+    -I$(top_srcdir)/rpc/xdr/src -I$(top_builddir)/rpc/xdr/src
+
+AM_CFLAGS = -Wall $(GF_CFLAGS)
+
+CLEANFILES =
diff --git a/xlators/features/thin-arbiter/src/thin-arbiter-mem-types.h b/xlators/features/thin-arbiter/src/thin-arbiter-mem-types.h
new file mode 100644
index 00000000000..69562d2febc
--- /dev/null
+++ b/xlators/features/thin-arbiter/src/thin-arbiter-mem-types.h
@@ -0,0 +1,19 @@
+/*
+  Copyright (c) 2015 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
+*/
+
+#ifndef __THIN_ARBITER_MEM_TYPES_H__
+#define __THIN_ARBITER_MEM_TYPES_H__
+#include <glusterfs/mem-types.h>
+
+typedef enum gf_ta_mem_types_ {
+    gf_ta_mt_local_t = gf_common_mt_end + 1,
+    gf_ta_mt_char,
+    gf_ta_mt_end
+} gf_ta_mem_types_t;
+#endif
diff --git a/xlators/features/thin-arbiter/src/thin-arbiter-messages.h b/xlators/features/thin-arbiter/src/thin-arbiter-messages.h
new file mode 100644
index 00000000000..81d7491577a
--- /dev/null
+++ b/xlators/features/thin-arbiter/src/thin-arbiter-messages.h
@@ -0,0 +1,28 @@
+/*
+  Copyright (c) 2015 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
+*/
+
+#ifndef _TA_MESSAGES_H_
+#define _TA_MESSAGES_H_
+
+#include <glusterfs/glfs-message-id.h>
+
+/* To add new message IDs, append new identifiers at the end of the list.
+ *
+ * Never remove a message ID. If it's not used anymore, you can rename it or
+ * leave it as it is, but not delete it. This is to prevent reutilization of
+ * IDs by other messages.
+ *
+ * The component name must match one of the entries defined in
+ * glfs-message-id.h.
+ */
+
+GLFS_MSGID(TA, TA_MSG_INVALID_FOP);
+
+#endif /* !_TA_MESSAGES_H_ */
diff --git a/xlators/features/thin-arbiter/src/thin-arbiter.c b/xlators/features/thin-arbiter/src/thin-arbiter.c
new file mode 100644
index 00000000000..ce3008636f1
--- /dev/null
+++ b/xlators/features/thin-arbiter/src/thin-arbiter.c
@@ -0,0 +1,661 @@
+/*
+  Copyright (c) 2015 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
+*/
+
+#include "thin-arbiter.h"
+#include "thin-arbiter-messages.h"
+#include "thin-arbiter-mem-types.h"
+#include <glusterfs/glusterfs.h>
+#include <glusterfs/xlator.h>
+#include <glusterfs/logging.h>
+#include <glusterfs/byte-order.h>
+#include <glusterfs/common-utils.h>
+
+int
+ta_set_incoming_values(dict_t *dict, char *key, data_t *value, void *data)
+{
+    int32_t ret = 0;
+    ta_fop_t *fop = (ta_fop_t *)data;
+    int32_t *pending = NULL;
+
+    pending = GF_CALLOC(1, value->len, gf_ta_mt_char);
+    if (!pending) {
+        ret = -ENOMEM;
+        goto out;
+    }
+    ret = dict_set_bin(fop->brick_xattr, key, pending, value->len);
+out:
+    return ret;
+}
+
+int
+ta_get_incoming_and_brick_values(dict_t *dict, char *key, data_t *value,
+                                 void *data)
+{
+    ta_fop_t *fop = data;
+    char *source = NULL;
+    char *in_coming = NULL;
+    int32_t len = 0, ret = 0;
+
+    source = GF_CALLOC(1, value->len, gf_ta_mt_char);
+    if (!source) {
+        ret = -ENOMEM;
+        goto out;
+    }
+
+    ret = dict_get_ptr_and_len(fop->dict, key, (void **)&in_coming, &len);
+
+    if (!in_coming || value->len != len) {
+        ret = -EINVAL;
+        goto out;
+    }
+
+    if (!memcmp(value->data, source, value->len) &&
+        (!memcmp(in_coming, source, len))) {
+        fop->on_disk[fop->idx] = 0;
+    } else {
+        fop->on_disk[fop->idx] = 1;
+    }
+
+    fop->idx++;
+out:
+    GF_FREE(source);
+    return ret;
+}
+
+void
+ta_release_fop(ta_fop_t *fop)
+{
+    if (!fop) {
+        return;
+    }
+    if (fop->fd) {
+        fd_unref(fop->fd);
+    }
+    loc_wipe(&fop->loc);
+    if (fop->dict) {
+        dict_unref(fop->dict);
+    }
+    if (fop->brick_xattr) {
+        dict_unref(fop->brick_xattr);
+    }
+
+    GF_FREE(fop);
+    return;
+}
+
+int32_t
+ta_set_xattrop_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                   int32_t op_ret, int32_t op_errno, dict_t *dict,
+                   dict_t *xdata)
+{
+    TA_STACK_UNWIND(xattrop, frame, op_ret, op_errno, dict, xdata);
+    return 0;
+}
+
+/*
+case 1 - If brick value is 0 and incoming value is also 0, fine
+case 2 - If brick value is 0 and incoming value is non 0, fine
+case 3 - If brick value is non 0 and incoming value is also 0, fine
+case 4 - If brick value is non 0 and incoming value is non 0, fine
+case 5 - If incoming value is non zero on both brick, it is wrong
+case 6 - If incoming value is non zero but brick value for other
+brick is also non zero, wrong
+*/
+
+int32_t
+ta_verify_on_disk_source(ta_fop_t *fop, dict_t *dict)
+{
+    int ret = 0;
+
+    if (!fop) {
+        return -EINVAL;
+    }
+
+    ret = dict_foreach(dict, ta_get_incoming_and_brick_values, (void *)fop);
+    if (ret < 0) {
+        return ret;
+    }
+    if (fop->on_disk[0] && fop->on_disk[1]) {
+        return -EINVAL;
+    }
+    return 0;
+}
+
+int32_t
+ta_get_xattrop_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                   int32_t op_ret, int32_t op_errno, dict_t *dict,
+                   dict_t *xdata)
+{
+    ta_fop_t *fop = NULL;
+    int ret = 0;
+
+    fop = frame->local;
+    if (op_ret) {
+        goto unwind;
+    }
+
+    ret = ta_verify_on_disk_source(fop, dict);
+    if (ret < 0) {
+        op_errno = -ret;
+        goto unwind;
+    }
+
+    if (fop->fd) {
+        STACK_WIND(frame, ta_set_xattrop_cbk, FIRST_CHILD(this),
+                   FIRST_CHILD(this)->fops->fxattrop, fop->fd,
+                   fop->xattrop_flags, fop->dict, NULL);
+    } else {
+        STACK_WIND(frame, ta_set_xattrop_cbk, FIRST_CHILD(this),
+                   FIRST_CHILD(this)->fops->xattrop, &fop->loc,
+                   fop->xattrop_flags, fop->dict, NULL);
+    }
+    return 0;
+
+unwind:
+
+    TA_STACK_UNWIND(xattrop, frame, -1, op_errno, NULL, NULL);
+    return -1;
+}
+
+ta_fop_t *
+ta_prepare_fop(call_frame_t *frame, xlator_t *this, loc_t *loc, fd_t *fd,
+               gf_xattrop_flags_t flags, dict_t *dict, dict_t *xdata)
+{
+    ta_fop_t *fop = NULL;
+    int ret = 0;
+
+    fop = GF_CALLOC(1, sizeof(*fop), gf_ta_mt_local_t);
+    if (!fop) {
+        goto out;
+    }
+
+    if (loc) {
+        loc_copy(&fop->loc, loc);
+    }
+
+    if (fd) {
+        fop->fd = fd_ref(fd);
+    }
+
+    fop->xattrop_flags = flags;
+    fop->idx = 0;
+
+    if (dict != NULL) {
+        fop->dict = dict_ref(dict);
+    }
+    fop->brick_xattr = dict_new();
+    if (fop->brick_xattr == NULL) {
+        goto out;
+    }
+    ret = dict_foreach(dict, ta_set_incoming_values, (void *)fop);
+    if (ret < 0) {
+        goto out;
+    }
+    frame->local = fop;
+    return fop;
+
+out:
+    ta_release_fop(fop);
+    return NULL;
+}
+
+int32_t
+ta_fxattrop(call_frame_t *frame, xlator_t *this, fd_t *fd,
+            gf_xattrop_flags_t flags, dict_t *dict, dict_t *xdata)
+{
+    int ret = 0;
+    ta_fop_t *fop = NULL;
+
+    fop = ta_prepare_fop(frame, this, NULL, fd, flags, dict, xdata);
+    if (!fop) {
+        ret = -ENOMEM;
+        goto unwind;
+    }
+
+    STACK_WIND(frame, ta_get_xattrop_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->fxattrop, fd, flags, fop->brick_xattr,
+               xdata);
+    return 0;
+
+unwind:
+
+    TA_STACK_UNWIND(xattrop, frame, -1, -ret, NULL, NULL);
+    return 0;
+}
+
+int32_t
+ta_xattrop(call_frame_t *frame, xlator_t *this, loc_t *loc,
+           gf_xattrop_flags_t flags, dict_t *dict, dict_t *xdata)
+{
+    int ret = 0;
+    ta_fop_t *fop = NULL;
+
+    fop = ta_prepare_fop(frame, this, loc, NULL, flags, dict, xdata);
+    if (!fop) {
+        ret = -ENOMEM;
+        goto unwind;
+    }
+
+    STACK_WIND(frame, ta_get_xattrop_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->xattrop, loc, flags, fop->brick_xattr,
+               xdata);
+    return 0;
+
+unwind:
+
+    TA_STACK_UNWIND(xattrop, frame, -1, -ret, NULL, NULL);
+    return 0;
+}
+
+int32_t
+ta_writev(call_frame_t *frame, xlator_t *this, fd_t *fd, struct iovec *vector,
+          int32_t count, off_t off, uint32_t flags, struct iobref *iobref,
+          dict_t *xdata)
+{
+    TA_FAILED_FOP(writev, frame, EINVAL);
+    return 0;
+}
+
+int32_t
+ta_fsetxattr(call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *dict,
+             int32_t flags, dict_t *xdata)
+{
+    TA_FAILED_FOP(fsetxattr, frame, EINVAL);
+    return 0;
+}
+
+int32_t
+ta_setxattr(call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *dict,
+            int32_t flags, dict_t *xdata)
+{
+    TA_FAILED_FOP(setxattr, frame, EINVAL);
+    return 0;
+}
+
+int32_t
+ta_fallocate(call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t keep_size,
+             off_t offset, size_t len, dict_t *xdata)
+{
+    TA_FAILED_FOP(fallocate, frame, EINVAL);
+    return 0;
+}
+
+int32_t
+ta_access(call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t mask,
+          dict_t *xdata)
+{
+    TA_FAILED_FOP(access, frame, EINVAL);
+    return 0;
+}
+
+int32_t
+ta_discard(call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
+           size_t len, dict_t *xdata)
+{
+    TA_FAILED_FOP(discard, frame, EINVAL);
+    return 0;
+}
+
+int32_t
+ta_entrylk(call_frame_t *frame, xlator_t *this, const char *volume, loc_t *loc,
+           const char *basename, entrylk_cmd cmd, entrylk_type type,
+           dict_t *xdata)
+{
+    TA_FAILED_FOP(entrylk, frame, EINVAL);
+    return 0;
+}
+
+int32_t
+ta_fentrylk(call_frame_t *frame, xlator_t *this, const char *volume, fd_t *fd,
+            const char *basename, entrylk_cmd cmd, entrylk_type type,
+            dict_t *xdata)
+{
+    TA_FAILED_FOP(fentrylk, frame, EINVAL);
+    return 0;
+}
+
+int32_t
+ta_flush(call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *xdata)
+{
+    TA_FAILED_FOP(flush, frame, EINVAL);
+    return 0;
+}
+
+int32_t
+ta_fsync(call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t datasync,
+         dict_t *xdata)
+{
+    TA_FAILED_FOP(fsync, frame, EINVAL);
+    return 0;
+}
+int32_t
+ta_fsyncdir(call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t datasync,
+            dict_t *xdata)
+{
+    TA_FAILED_FOP(fsyncdir, frame, EINVAL);
+    return 0;
+}
+
+int32_t
+ta_getxattr(call_frame_t *frame, xlator_t *this, loc_t *loc, const char *name,
+            dict_t *xdata)
+{
+    TA_FAILED_FOP(getxattr, frame, EINVAL);
+    return 0;
+}
+
+int32_t
+ta_fgetxattr(call_frame_t *frame, xlator_t *this, fd_t *fd, const char *name,
+             dict_t *xdata)
+{
+    TA_FAILED_FOP(fgetxattr, frame, EINVAL);
+    return 0;
+}
+
+int32_t
+ta_link(call_frame_t *frame, xlator_t *this, loc_t *oldloc, loc_t *newloc,
+        dict_t *xdata)
+{
+    TA_FAILED_FOP(link, frame, EINVAL);
+    return 0;
+}
+
+int32_t
+ta_lk(call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t cmd,
+      struct gf_flock *flock, dict_t *xdata)
+{
+    TA_FAILED_FOP(lk, frame, EINVAL);
+    return 0;
+}
+
+int32_t
+ta_mkdir(call_frame_t *frame, xlator_t *this, loc_t *loc, mode_t mode,
+         mode_t umask, dict_t *xdata)
+{
+    TA_FAILED_FOP(mkdir, frame, EINVAL);
+    return 0;
+}
+
+int32_t
+ta_mknod(call_frame_t *frame, xlator_t *this, loc_t *loc, mode_t mode,
+         dev_t rdev, mode_t umask, dict_t *xdata)
+{
+    TA_FAILED_FOP(mknod, frame, EINVAL);
+    return 0;
+}
+
+int32_t
+ta_open(call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t flags,
+        fd_t *fd, dict_t *xdata)
+{
+    TA_FAILED_FOP(open, frame, EINVAL);
+    return 0;
+}
+
+int32_t
+ta_opendir(call_frame_t *frame, xlator_t *this, loc_t *loc, fd_t *fd,
+           dict_t *xdata)
+{
+    TA_FAILED_FOP(opendir, frame, EINVAL);
+    return 0;
+}
+
+int32_t
+ta_readdir(call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
+           off_t offset, dict_t *xdata)
+{
+    TA_FAILED_FOP(readdir, frame, EINVAL);
+    return 0;
+}
+
+int32_t
+ta_readdirp(call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
+            off_t offset, dict_t *xdata)
+{
+    TA_FAILED_FOP(readdirp, frame, EINVAL);
+    return 0;
+}
+
+int32_t
+ta_readlink(call_frame_t *frame, xlator_t *this, loc_t *loc, size_t size,
+            dict_t *xdata)
+{
+    TA_FAILED_FOP(readlink, frame, EINVAL);
+    return 0;
+}
+
+int32_t
+ta_readv(call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
+         off_t offset, uint32_t flags, dict_t *xdata)
+{
+    TA_FAILED_FOP(readv, frame, EINVAL);
+    return 0;
+}
+
+int32_t
+ta_removexattr(call_frame_t *frame, xlator_t *this, loc_t *loc,
+               const char *name, dict_t *xdata)
+{
+    TA_FAILED_FOP(removexattr, frame, EINVAL);
+    return 0;
+}
+
+int32_t
+ta_fremovexattr(call_frame_t *frame, xlator_t *this, fd_t *fd, const char *name,
+                dict_t *xdata)
+{
+    TA_FAILED_FOP(fremovexattr, frame, EINVAL);
+    return 0;
+}
+
+int32_t
+ta_rename(call_frame_t *frame, xlator_t *this, loc_t *oldloc, loc_t *newloc,
+          dict_t *xdata)
+{
+    TA_FAILED_FOP(rename, frame, EINVAL);
+    return 0;
+}
+
+int32_t
+ta_rmdir(call_frame_t *frame, xlator_t *this, loc_t *loc, int xflags,
+         dict_t *xdata)
+{
+    TA_FAILED_FOP(rmdir, frame, EINVAL);
+    return 0;
+}
+
+int32_t
+ta_setattr(call_frame_t *frame, xlator_t *this, loc_t *loc, struct iatt *stbuf,
+           int32_t valid, dict_t *xdata)
+{
+    TA_FAILED_FOP(setattr, frame, EINVAL);
+    return 0;
+}
+
+int32_t
+ta_fsetattr(call_frame_t *frame, xlator_t *this, fd_t *fd, struct iatt *stbuf,
+            int32_t valid, dict_t *xdata)
+{
+    TA_FAILED_FOP(fsetattr, frame, EINVAL);
+    return 0;
+}
+
+int32_t
+ta_stat(call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xdata)
+{
+    TA_FAILED_FOP(stat, frame, EINVAL);
+    return 0;
+}
+
+int32_t
+ta_fstat(call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *xdata)
+{
+    TA_FAILED_FOP(fstat, frame, EINVAL);
+    return 0;
+}
+
+int32_t
+ta_statfs(call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xdata)
+{
+    TA_FAILED_FOP(statfs, frame, EINVAL);
+    return 0;
+}
+
+int32_t
+ta_symlink(call_frame_t *frame, xlator_t *this, const char *linkname,
+           loc_t *loc, mode_t umask, dict_t *xdata)
+{
+    TA_FAILED_FOP(symlink, frame, EINVAL);
+    return 0;
+}
+
+int32_t
+ta_truncate(call_frame_t *frame, xlator_t *this, loc_t *loc, off_t offset,
+            dict_t *xdata)
+{
+    TA_FAILED_FOP(truncate, frame, EINVAL);
+    return 0;
+}
+
+int32_t
+ta_ftruncate(call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
+             dict_t *xdata)
+{
+    TA_FAILED_FOP(ftruncate, frame, EINVAL);
+    return 0;
+}
+
+int32_t
+ta_unlink(call_frame_t *frame, xlator_t *this, loc_t *loc, int xflags,
+          dict_t *xdata)
+{
+    TA_FAILED_FOP(unlink, frame, EINVAL);
+    return 0;
+}
+
+int32_t
+ta_zerofill(call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
+            off_t len, dict_t *xdata)
+{
+    TA_FAILED_FOP(zerofill, frame, EINVAL);
+    return 0;
+}
+
+int32_t
+ta_seek(call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
+        gf_seek_what_t what, dict_t *xdata)
+{
+    TA_FAILED_FOP(seek, frame, EINVAL);
+    return 0;
+}
+
+int32_t
+mem_acct_init(xlator_t *this)
+{
+    int ret = -1;
+
+    ret = xlator_mem_acct_init(this, gf_ta_mt_end + 1);
+    if (ret)
+        gf_log(this->name, GF_LOG_ERROR,
+               "Memory accounting "
+               "initialization failed.");
+    return ret;
+}
+
+int
+reconfigure(xlator_t *this, dict_t *options)
+{
+    return 0;
+}
+
+int32_t
+init(xlator_t *this)
+{
+    if (!this->children || this->children->next) {
+        gf_log(this->name, GF_LOG_ERROR,
+               "'thin_arbiter' not configured with exactly one child");
+        return -1;
+    }
+
+    if (!this->parents) {
+        gf_log(this->name, GF_LOG_ERROR, "dangling volume. check volfile ");
+    }
+    return 0;
+}
+
+void
+fini(xlator_t *this)
+{
+    return;
+}
+
+struct xlator_fops fops = {
+    /*Passed fop*/
+    .xattrop = ta_xattrop,
+    .fxattrop = ta_fxattrop,
+    /*Failed fop*/
+    .writev = ta_writev,
+    .stat = ta_stat,
+    .fstat = ta_fstat,
+    .truncate = ta_truncate,
+    .ftruncate = ta_ftruncate,
+    .access = ta_access,
+    .readlink = ta_readlink,
+    .mknod = ta_mknod,
+    .mkdir = ta_mkdir,
+    .unlink = ta_unlink,
+    .rmdir = ta_rmdir,
+    .symlink = ta_symlink,
+    .rename = ta_rename,
+    .link = ta_link,
+    .open = ta_open,
+    .readv = ta_readv,
+    .flush = ta_flush,
+    .fsync = ta_fsync,
+    .opendir = ta_opendir,
+    .readdir = ta_readdir,
+    .readdirp = ta_readdirp,
+    .fsyncdir = ta_fsyncdir,
+    .statfs = ta_statfs,
+    .setxattr = ta_setxattr,
+    .getxattr = ta_getxattr,
+    .fsetxattr = ta_fsetxattr,
+    .fgetxattr = ta_fgetxattr,
+    .removexattr = ta_removexattr,
+    .fremovexattr = ta_fremovexattr,
+    .lk = ta_lk,
+    .entrylk = ta_entrylk,
+    .fentrylk = ta_fentrylk,
+    .setattr = ta_setattr,
+    .fsetattr = ta_fsetattr,
+    .fallocate = ta_fallocate,
+    .discard = ta_discard,
+    .zerofill = ta_zerofill,
+    .seek = ta_seek,
+};
+
+struct xlator_cbks cbks = {};
+
+struct volume_options options[] = {
+    {.key = {NULL}},
+};
+
+xlator_api_t xlator_api = {
+    .init = init,
+    .fini = fini,
+    .reconfigure = reconfigure,
+    .mem_acct_init = mem_acct_init,
+    .op_version = {GD_OP_VERSION_6_0},
+    .fops = &fops,
+    .cbks = &cbks,
+    .options = options,
+    .identifier = "thin-arbiter",
+    .category = GF_MAINTAINED,
+};
diff --git a/xlators/features/thin-arbiter/src/thin-arbiter.h b/xlators/features/thin-arbiter/src/thin-arbiter.h
new file mode 100644
index 00000000000..e5f914b84bf
--- /dev/null
+++ b/xlators/features/thin-arbiter/src/thin-arbiter.h
@@ -0,0 +1,59 @@
+/*
+  Copyright (c) 2015 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
+*/
+
+#ifndef _THIN_ARBITER_H
+#define _THIN_ARBITER_H
+
+#include <glusterfs/locking.h>
+#include <glusterfs/common-utils.h>
+#include <glusterfs/glusterfs.h>
+#include <glusterfs/xlator.h>
+#include <glusterfs/defaults.h>
+#include <glusterfs/list.h>
+
+#define THIN_ARBITER_SOURCE_XATTR "trusted.ta.source"
+#define THIN_ARBITER_SOURCE_SIZE 2
+
+#define TA_FAILED_FOP(fop, frame, op_errno)                                    \
+    do {                                                                       \
+        default_##fop##_failure_cbk(frame, op_errno);                          \
+    } while (0)
+
+#define TA_STACK_UNWIND(fop, frame, op_ret, op_errno, params...)               \
+    do {                                                                       \
+        ta_fop_t *__local = NULL;                                              \
+        int32_t __op_ret = 0;                                                  \
+        int32_t __op_errno = 0;                                                \
+                                                                               \
+        __local = frame->local;                                                \
+        __op_ret = op_ret;                                                     \
+        __op_errno = op_errno;                                                 \
+        if (__local) {                                                         \
+            ta_release_fop(__local);                                           \
+            frame->local = NULL;                                               \
+        }                                                                      \
+        STACK_UNWIND_STRICT(fop, frame, __op_ret, __op_errno, params);         \
+                                                                               \
+    } while (0)
+
+struct _ta_fop;
+typedef struct _ta_fop ta_fop_t;
+
+struct _ta_fop {
+    gf_xattrop_flags_t xattrop_flags;
+    loc_t loc;
+    fd_t *fd;
+    dict_t *dict;
+    dict_t *brick_xattr;
+    int32_t on_disk[2];
+    int32_t idx;
+};
+
+#endif /* _THIN_ARBITER_H */
diff --git a/xlators/features/trash/src/Makefile.am b/xlators/features/trash/src/Makefile.am
index 2000359a6a9..8557e7171af 100644
--- a/xlators/features/trash/src/Makefile.am
+++ b/xlators/features/trash/src/Makefile.am
@@ -1,14 +1,17 @@
+if WITH_SERVER
 xlator_LTLIBRARIES = trash.la
+endif
 xlatordir = $(libdir)/glusterfs/$(PACKAGE_VERSION)/xlator/features
 
-trash_la_LDFLAGS = -module -avoid-version 
+trash_la_LDFLAGS = -module $(GF_XLATOR_DEFAULT_LDFLAGS)
 
 trash_la_SOURCES = trash.c
 trash_la_LIBADD = $(top_builddir)/libglusterfs/src/libglusterfs.la 
 
 noinst_HEADERS = trash.h trash-mem-types.h
 
-AM_CPPFLAGS = $(GF_CPPFLAGS) -I$(top_srcdir)/libglusterfs/src
+AM_CPPFLAGS = $(GF_CPPFLAGS) -I$(top_srcdir)/libglusterfs/src \
+	-I$(top_srcdir)/rpc/xdr/src -I$(top_builddir)/rpc/xdr/src
 
 AM_CFLAGS = -Wall $(GF_CFLAGS)
 
diff --git a/xlators/features/trash/src/trash-mem-types.h b/xlators/features/trash/src/trash-mem-types.h
index b7cad3ce3a9..43353c8f095 100644
--- a/xlators/features/trash/src/trash-mem-types.h
+++ b/xlators/features/trash/src/trash-mem-types.h
@@ -10,14 +10,13 @@
 #ifndef __TRASH_MEM_TYPES_H__
 #define __TRASH_MEM_TYPES_H__
 
-#include "mem-types.h"
+#include <glusterfs/mem-types.h>
 
 enum gf_trash_mem_types_ {
-        gf_trash_mt_trash_private_t = gf_common_mt_end + 1,
-        gf_trash_mt_char,
-        gf_trash_mt_uuid,
-        gf_trash_mt_trash_elim_path,
-        gf_trash_mt_end
+    gf_trash_mt_trash_private_t = gf_common_mt_end + 1,
+    gf_trash_mt_char,
+    gf_trash_mt_uuid,
+    gf_trash_mt_trash_elim_path,
+    gf_trash_mt_end
 };
 #endif
-
diff --git a/xlators/features/trash/src/trash.c b/xlators/features/trash/src/trash.c
index 07492e3addf..7d09cba3e9c 100644
--- a/xlators/features/trash/src/trash.c
+++ b/xlators/features/trash/src/trash.c
@@ -7,37 +7,34 @@
    later), or the GNU General Public License, version 2 (GPLv2), in all
    cases as published by the Free Software Foundation.
 */
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
-
 #include "trash.h"
 #include "trash-mem-types.h"
+#include <glusterfs/syscall.h>
 
-#define root_gfid        (uuid_t){0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1}
-#define trash_gfid       (uuid_t){0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5}
-#define internal_op_gfid (uuid_t){0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6}
+#define root_gfid                                                              \
+    (uuid_t) { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1 }
+#define trash_gfid                                                             \
+    (uuid_t) { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5 }
+#define internal_op_gfid                                                       \
+    (uuid_t) { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6 }
 
 int32_t
-trash_truncate_writev_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                           int32_t op_ret, int32_t op_errno,
-                           struct iatt *prebuf, struct iatt *postbuf,
-                           dict_t *xdata);
+trash_truncate_writev_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                          int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
+                          struct iatt *postbuf, dict_t *xdata);
 
 int32_t
-trash_truncate_mkdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                          int32_t op_ret, int32_t op_errno, inode_t *inode,
-                          struct iatt *stbuf, struct iatt *preparent,
-                          struct iatt *postparent, dict_t *xdata);
+trash_truncate_mkdir_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                         int32_t op_ret, int32_t op_errno, inode_t *inode,
+                         struct iatt *stbuf, struct iatt *preparent,
+                         struct iatt *postparent, dict_t *xdata);
 
 int32_t
-trash_unlink_rename_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                         int32_t op_ret, int32_t op_errno, struct iatt *buf,
-                         struct iatt *preoldparent, struct iatt *postoldparent,
-                         struct iatt *prenewparent, struct iatt *postnewparent,
-                         dict_t *xdata);
-
+trash_unlink_rename_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                        int32_t op_ret, int32_t op_errno, struct iatt *buf,
+                        struct iatt *preoldparent, struct iatt *postoldparent,
+                        struct iatt *prenewparent, struct iatt *postnewparent,
+                        dict_t *xdata);
 /* Common routines used in this translator */
 
 /**
@@ -46,21 +43,27 @@ trash_unlink_rename_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
  * the existing directory and returns the same
  */
 mode_t
-get_permission (char *path)
+get_permission(char *path)
 {
-        mode_t                  mode                    = 0755;
-        struct stat             sbuf                    = {0,};
-        struct iatt             ibuf                    = {0,};
-        int                     ret                     = 0;
-
-        ret = stat (path, &sbuf);
-        if (!ret) {
-                iatt_from_stat (&ibuf, &sbuf);
-                mode = st_mode_from_ia (ibuf.ia_prot, ibuf.ia_type);
-        } else
-                gf_log ("trash", GF_LOG_DEBUG, "stat on %s failed"
-                                " using default", path);
-        return mode;
+    mode_t mode = 0755;
+    struct stat sbuf = {
+        0,
+    };
+    struct iatt ibuf = {
+        0,
+    };
+    int ret = 0;
+
+    ret = sys_stat(path, &sbuf);
+    if (!ret) {
+        iatt_from_stat(&ibuf, &sbuf);
+        mode = st_mode_from_ia(ibuf.ia_prot, ibuf.ia_type);
+    } else
+        gf_log("trash", GF_LOG_DEBUG,
+               "stat on %s failed"
+               " using default",
+               path);
+    return mode;
 }
 
 /**
@@ -69,197 +72,198 @@ get_permission (char *path)
  * striped out for additional usage.
  */
 int
-extract_trash_directory (char *priv_value, const char **trash_directory)
+extract_trash_directory(char *priv_value, const char **trash_directory)
 {
-        char                    *tmp                    = NULL;
-        int                     ret                     = 0;
-
-        GF_VALIDATE_OR_GOTO("trash", priv_value, out);
-
-        tmp = gf_strdup (priv_value + 1);
-        if (!tmp) {
-                ret = ENOMEM;
-                goto out;
-        }
-        if (tmp[strlen(tmp)-1] == '/')
-                tmp[strlen(tmp)-1] = '\0';
-        *trash_directory = gf_strdup (tmp);
-        if (!(*trash_directory)) {
-                ret = ENOMEM;
-                goto out;
-        }
+    char *tmp = NULL;
+    int ret = 0;
+
+    GF_VALIDATE_OR_GOTO("trash", priv_value, out);
+
+    tmp = gf_strdup(priv_value + 1);
+    if (!tmp) {
+        ret = ENOMEM;
+        goto out;
+    }
+    if (tmp[strlen(tmp) - 1] == '/')
+        tmp[strlen(tmp) - 1] = '\0';
+    *trash_directory = gf_strdup(tmp);
+    if (!(*trash_directory)) {
+        ret = ENOMEM;
+        goto out;
+    }
 out:
-        if (tmp)
-                GF_FREE (tmp);
-        return ret;
+    if (tmp)
+        GF_FREE(tmp);
+    return ret;
 }
 
 /**
- * The trash directory path should be append at begining of file path for
+ * The trash directory path should be append at beginning of file path for
  * delete or truncate operations. Normal trashing moves the contents to
  * trash directory and trashing done by internal operations are moved to
  * internal_op directory inside trash.
  */
 void
-copy_trash_path (const char *priv_value, gf_boolean_t internal, char *path)
+copy_trash_path(const char *priv_value, gf_boolean_t internal, char *path,
+                size_t path_size)
 {
-        char                    trash_path[PATH_MAX]    = {0,};
-
-        strcpy (trash_path, priv_value);
-        if (internal)
-                strcat (trash_path, "internal_op/");
-
-        strcpy (path, trash_path);
+    char trash_path[PATH_MAX] = {
+        0,
+    };
+
+    strncpy(trash_path, priv_value, sizeof(trash_path));
+    trash_path[sizeof(trash_path) - 1] = 0;
+    if (internal)
+        strncat(trash_path, "internal_op/",
+                sizeof(trash_path) - strlen(trash_path) - 1);
+
+    strncpy(path, trash_path, path_size);
+    path[path_size - 1] = 0;
 }
 
 /**
  * This function performs the reverse operation of copy_trash_path(). It gives
  * out a pointer, whose starting value will be the path inside trash directory,
- * similar to orginal path.
+ * similar to original path.
  */
 void
-remove_trash_path (const char *path, gf_boolean_t internal, char **rem_path)
-{
-        if (rem_path == NULL) {
-                return;
-        }
-
-        *rem_path =  strchr (path + 1, '/');
-        if (internal)
-                *rem_path =  strchr (*rem_path + 1, '/');
-}
-
-/**
- * Check whether the path includes trash directory or internal op directory
- * inside trash. This check is used to make sure that we avoid deletion,
- * rename and creation operations from trash directory.
- */
-int
-check_whether_trash_directory (const char *path,
-                               const char *trash_directory_path)
+remove_trash_path(const char *path, gf_boolean_t internal, char **rem_path)
 {
-        char                    tmp_path[PATH_MAX]              = {0,};
-        char                    internal_op_path[PATH_MAX]      = {0,};
-        int                     ret                             = 0;
-
-        if (path[strlen(path)-1] == '/')
-                sprintf (tmp_path, "%s", path);
-        else
-                sprintf (tmp_path, "%s/", path);
-
-        copy_trash_path (trash_directory_path, _gf_true, internal_op_path);
-        ret = strcmp (tmp_path, trash_directory_path) &&
-              strcmp (tmp_path, internal_op_path);
+    if (rem_path == NULL) {
+        return;
+    }
 
-        return ret;
+    *rem_path = strchr(path + 1, '/');
+    if (internal)
+        *rem_path = strchr(*rem_path + 1, '/');
 }
 
 /**
  * Checks whether the given path reside under the specified eliminate path
  */
 int
-check_whether_eliminate_path (trash_elim_path *trav, const char *path)
+check_whether_eliminate_path(trash_elim_path *trav, const char *path)
 {
-        int                     match                           = 0;
-
-        while (trav) {
-                if (strncmp (path, trav->path, strlen(trav->path)) == 0) {
-                        match++;
-                        break;
-                }
-                trav = trav->next;
+    int match = 0;
+
+    while (trav) {
+        if (strncmp(path, trav->path, strlen(trav->path)) == 0) {
+            match++;
+            break;
         }
-        return match;
+        trav = trav->next;
+    }
+    return match;
 }
 
 /**
  * Stores the eliminate path into internal eliminate path structure
  */
 int
-store_eliminate_path (char *str, trash_elim_path **eliminate)
+store_eliminate_path(char *str, trash_elim_path **eliminate)
 {
-        trash_elim_path         *trav                   = NULL;
-        char                    *component              = NULL;
-        char                    elm_path[PATH_MAX]      = {0,};
-        int                     ret                     = 0;
-        char                    *strtokptr              = NULL;
-
-        if (eliminate == NULL) {
-                ret = EINVAL;
-                goto out;
-        }
-
-        component = strtok_r (str, ",", &strtokptr);
-        while (component) {
-                trav = GF_CALLOC (1, sizeof (*trav),
-                                 gf_trash_mt_trash_elim_path);
-                if (!trav) {
-                        ret = ENOMEM;
-                        goto out;
-                }
-                if (component[0] == '/')
-                        sprintf(elm_path, "%s", component);
-                else
-                        sprintf(elm_path, "/%s", component);
-
-                if (component[strlen(component)-1] != '/')
-                        strcat (elm_path, "/");
-
-                trav->path = gf_strdup(elm_path);
-                if (!trav->path) {
-                                ret = ENOMEM;
-                                gf_log ("trash", GF_LOG_DEBUG, "out of memory");
-                                goto out;
-                }
-                trav->next = *eliminate;
-                *eliminate = trav;
-                component = strtok_r (NULL, ",", &strtokptr);
-        }
+    trash_elim_path *trav = NULL;
+    char *component = NULL;
+    char elm_path[PATH_MAX] = {
+        0,
+    };
+    int ret = 0;
+    char *strtokptr = NULL;
+
+    if ((str == NULL) || (eliminate == NULL)) {
+        ret = EINVAL;
+        goto out;
+    }
+
+    component = strtok_r(str, ",", &strtokptr);
+    while (component) {
+        trav = GF_CALLOC(1, sizeof(*trav), gf_trash_mt_trash_elim_path);
+        if (!trav) {
+            ret = ENOMEM;
+            goto out;
+        }
+        if (component[0] == '/')
+            sprintf(elm_path, "%s", component);
+        else
+            sprintf(elm_path, "/%s", component);
+
+        if (component[strlen(component) - 1] != '/')
+            strncat(elm_path, "/", sizeof(elm_path) - strlen(elm_path) - 1);
+
+        trav->path = gf_strdup(elm_path);
+        if (!trav->path) {
+            ret = ENOMEM;
+            gf_log("trash", GF_LOG_DEBUG, "out of memory");
+            GF_FREE(trav);
+            goto out;
+        }
+        trav->next = *eliminate;
+        *eliminate = trav;
+        component = strtok_r(NULL, ",", &strtokptr);
+    }
 out:
-        return ret;
+    return ret;
 }
 
 /**
  * Appends time stamp to given string
  */
 void
-append_time_stamp (char *name)
+append_time_stamp(char *name, size_t name_size)
 {
-        int                     i;
-        char                    timestr[64]            = {0,};
+    int i;
+    char timestr[GF_TIMESTR_SIZE] = {
+        0,
+    };
+
+    gf_time_fmt(timestr, sizeof(timestr), gf_time(), gf_timefmt_F_HMS);
+
+    /* removing white spaces in timestamp */
+    for (i = 0; i < strlen(timestr); i++) {
+        if (timestr[i] == ' ')
+            timestr[i] = '_';
+    }
+    strncat(name, "_", name_size - strlen(name) - 1);
+    strncat(name, timestr, name_size - strlen(name) - 1);
+}
 
-        gf_time_fmt (timestr, sizeof(timestr), time (NULL),
-                     gf_timefmt_F_HMS);
+/* *
+ * Check whether delete/rename operation is permitted on
+ * trash directory
+ */
 
-        /* removing white spaces in timestamp */
-        for (i = 0; i < strlen (timestr); i++) {
-                if (timestr[i] == ' ')
-                        timestr[i] = '_';
-        }
-        strcat (name, "_");
-        strcat (name, timestr);
+gf_boolean_t
+check_whether_op_permitted(trash_private_t *priv, loc_t *loc)
+{
+    if ((priv->state && (gf_uuid_compare(loc->inode->gfid, trash_gfid) == 0)))
+        return _gf_false;
+    if (priv->internal &&
+        (gf_uuid_compare(loc->inode->gfid, internal_op_gfid) == 0))
+        return _gf_false;
+
+    return _gf_true;
 }
 
 /**
  * Wipe the memory used by trash location variable
  */
 void
-trash_local_wipe (trash_local_t *local)
+trash_local_wipe(trash_local_t *local)
 {
-        if (!local)
-                goto out;
+    if (!local)
+        goto out;
 
-        loc_wipe (&local->loc);
-        loc_wipe (&local->newloc);
+    loc_wipe(&local->loc);
+    loc_wipe(&local->newloc);
 
-        if (local->fd)
-                fd_unref (local->fd);
-        if (local->newfd)
-                fd_unref (local->newfd);
+    if (local->fd)
+        fd_unref(local->fd);
+    if (local->newfd)
+        fd_unref(local->newfd);
 
-        mem_put (local);
+    mem_put(local);
 out:
-        return;
+    return;
 }
 
 /**
@@ -267,308 +271,579 @@ out:
  * recursive call
  */
 void
-wipe_eliminate_path (trash_elim_path **trav)
+wipe_eliminate_path(trash_elim_path **trav)
 {
-        if (trav == NULL) {
-                return;
-        }
+    if (trav == NULL) {
+        return;
+    }
 
-        if (*trav == NULL) {
-                return;
-        }
+    if (*trav == NULL) {
+        return;
+    }
 
-        wipe_eliminate_path (&(*trav)->next);
-        GF_FREE ((*trav)->path);
-        GF_FREE (*trav);
-        *trav = NULL;
+    wipe_eliminate_path(&(*trav)->next);
+    GF_FREE((*trav)->path);
+    GF_FREE(*trav);
+    *trav = NULL;
 }
 
 /**
- * This getxattr calls returns existing trash directory path in
- * the dictionary
+ * This is the call back of rename fop initated using STACK_WIND in
+ * reconfigure/notify function which is used to rename trash directory
+ * in the brick when it is required either in volume start or set.
+ * This frame  must destroyed from this function itself since it was
+ * created by trash xlator
  */
 int32_t
-trash_notify_getxattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                      int32_t op_ret, int32_t op_errno, dict_t *dict,
-                      dict_t *xdata)
+trash_dir_rename_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                     int32_t op_ret, int32_t op_errno, struct iatt *buf,
+                     struct iatt *preoldparent, struct iatt *postoldparent,
+                     struct iatt *prenewparent, struct iatt *postnewparent,
+                     dict_t *xdata)
 {
-        data_t                 *data                  = NULL;
-        trash_private_t        *priv                  = NULL;
-        int ret                                       = 0;
-
-        priv = this->private;
-        GF_VALIDATE_OR_GOTO ("trash", priv, out);
-
-        data = dict_get (dict, GET_ANCESTRY_PATH_KEY);
-        if (!data) {
-                gf_log (this->name, GF_LOG_DEBUG,
-                                "oldtrash-directory doesnot exists");
-                priv->oldtrash_dir = gf_strdup (priv->newtrash_dir);
-                if (!priv->oldtrash_dir) {
-                        gf_log (this->name, GF_LOG_ERROR, "out of memory");
-                        ret = ENOMEM;
-                        goto out;
-                }
-        } else {
-                priv->oldtrash_dir = gf_strdup (data->data);
-                if (!priv->oldtrash_dir) {
-                        gf_log (this->name, GF_LOG_ERROR, "out of memory");
-                         ret = ENOMEM;
-                         goto out;
-                }
-                gf_log (this->name, GF_LOG_DEBUG, "old trash directory"
-                                                  " path is %s", data->data);
-        }
+    trash_private_t *priv = NULL;
+    trash_local_t *local = NULL;
 
-out:
-          return ret;
-}
-
-/**
- * This is a nameless look up for old trash directory
- * The lookup is based on gfid, because trash directory
- * has fixed gfid.
- */
-int32_t
-trash_notify_lookup_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                         int32_t op_ret, int32_t op_errno, inode_t *inode,
-                         struct iatt *buf, dict_t *xdata,
-                         struct iatt *postparent)
-{
-        trash_private_t         *priv                  = NULL;
-        loc_t                   loc                    = {0,};
-        int                     ret                    = 0;
+    priv = this->private;
 
-        priv = this->private;
-        GF_VALIDATE_OR_GOTO ("trash", priv, out);
+    local = frame->local;
 
-        if (op_ret == 0) {
+    if (op_ret == -1) {
+        gf_log(this->name, GF_LOG_ERROR,
+               "rename trash directory "
+               "failed: %s",
+               strerror(op_errno));
+        goto out;
+    }
 
-                gf_log (this->name, GF_LOG_DEBUG, "inode found with gfid %s",
-                                   uuid_utoa(buf->ia_gfid));
+    GF_FREE(priv->oldtrash_dir);
 
-                gf_uuid_copy (loc.gfid,  trash_gfid);
+    priv->oldtrash_dir = gf_strdup(priv->newtrash_dir);
+    if (!priv->oldtrash_dir) {
+        op_ret = ENOMEM;
+        gf_log(this->name, GF_LOG_DEBUG, "out of memory");
+    }
 
-                /* Find trash inode using available information */
-                priv->trash_inode = inode_link (inode, NULL, NULL, buf);
+out:
+    frame->local = NULL;
+    STACK_DESTROY(frame->root);
+    trash_local_wipe(local);
+    return op_ret;
+}
 
-                loc.inode = inode_ref (priv->trash_inode);
+int
+rename_trash_directory(xlator_t *this)
+{
+    trash_private_t *priv = NULL;
+    int ret = 0;
+    loc_t loc = {
+        0,
+    };
+    loc_t old_loc = {
+        0,
+    };
+    call_frame_t *frame = NULL;
+    trash_local_t *local = NULL;
+
+    priv = this->private;
+
+    frame = create_frame(this, this->ctx->pool);
+    if (frame == NULL) {
+        gf_log(this->name, GF_LOG_ERROR, "failed to create frame");
+        ret = ENOMEM;
+        goto out;
+    }
+
+    local = mem_get0(this->local_pool);
+    if (!local) {
+        gf_log(this->name, GF_LOG_DEBUG, "out of memory");
+        ret = ENOMEM;
+        goto out;
+    }
+    frame->local = local;
+
+    /* assign new location values to new_loc members */
+    gf_uuid_copy(loc.gfid, trash_gfid);
+    gf_uuid_copy(loc.pargfid, root_gfid);
+    ret = extract_trash_directory(priv->newtrash_dir, &loc.name);
+    if (ret) {
+        gf_log(this->name, GF_LOG_DEBUG, "out of memory");
+        goto out;
+    }
+    loc.path = gf_strdup(priv->newtrash_dir);
+    if (!loc.path) {
+        ret = ENOMEM;
+        gf_log(this->name, GF_LOG_DEBUG, "out of memory");
+        goto out;
+    }
+
+    /* assign old location values to old_loc members */
+    gf_uuid_copy(old_loc.gfid, trash_gfid);
+    gf_uuid_copy(old_loc.pargfid, root_gfid);
+    ret = extract_trash_directory(priv->oldtrash_dir, &old_loc.name);
+    if (ret) {
+        gf_log(this->name, GF_LOG_DEBUG, "out of memory");
+        goto out;
+    }
+    old_loc.path = gf_strdup(priv->oldtrash_dir);
+    if (!old_loc.path) {
+        ret = ENOMEM;
+        gf_log(this->name, GF_LOG_DEBUG, "out of memory");
+        goto out;
+    }
+
+    old_loc.inode = inode_ref(priv->trash_inode);
+    gf_uuid_copy(old_loc.inode->gfid, old_loc.gfid);
+
+    loc_copy(&local->loc, &old_loc);
+    loc_copy(&local->newloc, &loc);
+
+    STACK_WIND(frame, trash_dir_rename_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->rename, &old_loc, &loc, NULL);
+    return 0;
 
-                /*Used to find path of old trash directory*/
-                STACK_WIND (frame, trash_notify_getxattr_cbk, FIRST_CHILD(this),
-                            FIRST_CHILD(this)->fops->getxattr, &loc,
-                            GET_ANCESTRY_PATH_KEY, xdata);
-        }
+out:
+    if (frame) {
+        frame->local = NULL;
+        STACK_DESTROY(frame->root);
+    }
 
-        /* If there is no old trash directory we set its value to new one,
-         * which is the valid condition for trash directory creation
-         */
-        else {
-                priv->oldtrash_dir = gf_strdup (priv->newtrash_dir);
-                if (!priv->oldtrash_dir) {
-                        gf_log (this->name, GF_LOG_ERROR, "out of memory");
-                        goto out;
-                }
-        }
+    trash_local_wipe(local);
 
-out:
-        loc_wipe (&loc);
-        return ret;
+    return ret;
 }
 
 int32_t
-trash_internal_op_mkdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                        int32_t op_ret, int32_t op_errno, inode_t *inode,
-                        struct iatt *buf, struct iatt *preparent,
-                        struct iatt *postparent, dict_t *xdata)
+trash_internal_op_mkdir_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                            int32_t op_ret, int32_t op_errno, inode_t *inode,
+                            struct iatt *buf, struct iatt *preparent,
+                            struct iatt *postparent, dict_t *xdata)
 {
-        if (op_ret != 0)
-                gf_log (this->name, GF_LOG_ERROR, "mkdir failed for "
-                        "internal op directory : %s", strerror (op_errno));
-        return op_ret;
+    trash_local_t *local = NULL;
+    local = frame->local;
+
+    if (op_ret != 0 && !(op_errno == EEXIST))
+        gf_log(this->name, GF_LOG_ERROR,
+               "mkdir failed for "
+               "internal op directory : %s",
+               strerror(op_errno));
+
+    frame->local = NULL;
+    STACK_DESTROY(frame->root);
+    trash_local_wipe(local);
+    return op_ret;
 }
 
 /**
  * This is the call back of mkdir fop initated using STACK_WIND in
- * notify function which is used to create trash directory in the brick
- * when a volume starts.The frame of the mkdir must destroyed from
- * this function itself since it was created by trash xlator
+ * notify/reconfigure function which is used to create trash directory
+ * in the brick when "trash" is on. The frame of the mkdir must
+ * destroyed from this function itself since it was created by trash xlator
  */
+
 int32_t
-trash_notify_mkdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                        int32_t op_ret, int32_t op_errno, inode_t *inode,
-                        struct iatt *buf, struct iatt *preparent,
-                        struct iatt *postparent, dict_t *xdata)
+trash_dir_mkdir_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                    int32_t op_ret, int32_t op_errno, inode_t *inode,
+                    struct iatt *buf, struct iatt *preparent,
+                    struct iatt *postparent, dict_t *xdata)
 {
-        uuid_t                *gfid_ptr                         = NULL;
-        loc_t                 loc                               = {0, };
-        int                   ret                               = 0;
-        dict_t                *dict                             = NULL;
-        char                  internal_op_path[PATH_MAX]        = {0,};
-        trash_private_t       *priv       = NULL;
+    trash_private_t *priv = NULL;
+    trash_local_t *local = NULL;
 
-        priv = this->private;
-        GF_VALIDATE_OR_GOTO ("trash", priv, out);
+    priv = this->private;
 
-        dict = dict_new ();
-        if (!dict) {
-                ret = -1;
-                goto out;
-        }
-        if ((op_ret == 0) || (op_ret == -1 && op_errno == EEXIST)) {
-                gfid_ptr = GF_CALLOC (1, sizeof(uuid_t),
-                                           gf_common_mt_uuid_t);
-                if (!gfid_ptr) {
-                        ret = ENOMEM;
-                        goto out;
-                }
-                gf_uuid_copy (*gfid_ptr, internal_op_gfid);
-
-                gf_uuid_copy (loc.gfid, internal_op_gfid);
-                gf_uuid_copy (loc.pargfid, trash_gfid);
-                loc.name = gf_strdup ("internal_op");
-
-                if (!loc.name) {
-                        gf_log (this->name, GF_LOG_DEBUG,
-                                         "out of memory");
-                        ret = ENOMEM;
-                        goto out;
-                }
-                sprintf (internal_op_path, "%s%s",
-                                priv->newtrash_dir, loc.name);
-
-                loc.path = gf_strdup (internal_op_path);
-
-                if (!loc.path) {
-                        gf_log (this->name, GF_LOG_DEBUG,
-                                         "out of memory");
-                        ret = ENOMEM;
-                        goto out;
-                }
-
-                loc.inode = inode_new (priv->trash_itable);
-                loc.inode->ia_type = IA_IFDIR;
-                /* Fixed gfid is set for trash directory with
-                 * this function
-                 */
-                ret = dict_set_dynptr (dict, "gfid-req", gfid_ptr,
-                                       sizeof (uuid_t));
-                if (ret) {
-                        gf_log (this->name, GF_LOG_ERROR,
-                                 "setting key gfid-req failed");
-                        goto out;
-                }
-
-                /* The mkdir call for creating trash directory */
-                STACK_WIND (frame, trash_internal_op_mkdir_cbk,
-                            FIRST_CHILD(this),
-                            FIRST_CHILD(this)->fops->mkdir, &loc, 0755,
-                            0022, dict);
-                /* After creating we must call other notify functions */
-                default_notify (this, GF_EVENT_CHILD_UP, NULL);
-        } else {
-                gf_log (this->name, GF_LOG_ERROR, "mkdir failed for trash"
-                                " directory : %s", strerror (op_errno));
+    local = frame->local;
+
+    if (op_ret == 0) {
+        priv->oldtrash_dir = gf_strdup(priv->newtrash_dir);
+        if (!priv->oldtrash_dir) {
+            gf_log(this->name, GF_LOG_ERROR, "out of memory");
+            op_ret = ENOMEM;
         }
+    } else if (op_ret != 0 && errno != EEXIST)
+        gf_log(this->name, GF_LOG_ERROR,
+               "mkdir failed for trash"
+               " directory : %s",
+               strerror(op_errno));
 
-        STACK_DESTROY (frame->root);
-out:
-        if (ret && gfid_ptr)
-                GF_FREE (gfid_ptr);
-        if (dict)
-                dict_unref (dict);
-        return 0;
+    frame->local = NULL;
+    STACK_DESTROY(frame->root);
+    trash_local_wipe(local);
+    return op_ret;
 }
 
+/**
+ * This getxattr calls returns existing trash directory path in
+ * the dictionary
+ */
+int32_t
+trash_dir_getxattr_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                       int32_t op_ret, int32_t op_errno, dict_t *dict,
+                       dict_t *xdata)
+{
+    data_t *data = NULL;
+    trash_private_t *priv = NULL;
+    int ret = 0;
+    trash_local_t *local = NULL;
+
+    priv = this->private;
+    GF_VALIDATE_OR_GOTO("trash", priv, out);
+
+    local = frame->local;
+
+    data = dict_get(dict, GET_ANCESTRY_PATH_KEY);
+    if (!data) {
+        goto out;
+    }
+    priv->oldtrash_dir = GF_MALLOC(PATH_MAX, gf_common_mt_char);
+    if (!priv->oldtrash_dir) {
+        gf_log(this->name, GF_LOG_ERROR, "out of memory");
+        ret = ENOMEM;
+        goto out;
+    }
+    /* appending '/' if it is not present */
+    sprintf(priv->oldtrash_dir, "%s%c", data->data,
+            data->data[strlen(data->data) - 1] != '/' ? '/' : '\0');
+    gf_log(this->name, GF_LOG_DEBUG,
+           "old trash directory path "
+           "is %s",
+           priv->oldtrash_dir);
+    if (strcmp(priv->newtrash_dir, priv->oldtrash_dir) != 0) {
+        /* When user set a new name for trash directory, trash
+         * xlator will perform a rename operation on old trash
+         * directory to the new one using a STACK_WIND from here.
+         * This option can be configured only when volume is in
+         * started state
+         */
+        ret = rename_trash_directory(this);
+    }
+
+out:
+    frame->local = NULL;
+    STACK_DESTROY(frame->root);
+    trash_local_wipe(local);
+    return ret;
+}
+/**
+ * This is a nameless look up for internal op directory
+ * The lookup is based on gfid, because internal op directory
+ * has fixed gfid.
+ */
 int32_t
-trash_notify_rename_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                    int32_t op_ret, int32_t op_errno, struct iatt *buf,
-                    struct iatt *preoldparent, struct iatt *postoldparent,
-                    struct iatt *prenewparent, struct iatt *postnewparent,
-                    dict_t *xdata)
+trash_internalop_dir_lookup_cbk(call_frame_t *frame, void *cookie,
+                                xlator_t *this, int32_t op_ret,
+                                int32_t op_errno, inode_t *inode,
+                                struct iatt *buf, dict_t *xdata,
+                                struct iatt *postparent)
 {
-        if ((op_ret == 0) || (op_ret == -1 && op_errno == EEXIST)) {
-                /* After creating we must call other notify functions */
-                default_notify (this, GF_EVENT_CHILD_UP, NULL);
-        } else {
-                gf_log (this->name, GF_LOG_ERROR, "rename failed: %s",
-                        strerror (op_errno));
+    trash_private_t *priv = NULL;
+    int ret = 0;
+    uuid_t *gfid_ptr = NULL;
+    loc_t loc = {
+        0,
+    };
+    char internal_op_path[PATH_MAX] = {
+        0,
+    };
+    dict_t *dict = NULL;
+    trash_local_t *local = NULL;
+
+    priv = this->private;
+    GF_VALIDATE_OR_GOTO("trash", priv, out);
+
+    local = frame->local;
+    if (op_ret != 0 && op_errno == ENOENT) {
+        loc_wipe(&local->loc);
+        gfid_ptr = GF_MALLOC(sizeof(uuid_t), gf_common_mt_uuid_t);
+        if (!gfid_ptr) {
+            ret = ENOMEM;
+            goto out;
+        }
+
+        gf_uuid_copy(*gfid_ptr, internal_op_gfid);
+
+        dict = dict_new();
+        if (!dict) {
+            ret = ENOMEM;
+            goto out;
+        }
+        ret = dict_set_gfuuid(dict, "gfid-req", *gfid_ptr, false);
+        if (ret) {
+            gf_log(this->name, GF_LOG_ERROR, "setting key gfid-req failed");
+            goto out;
+        }
+        gf_uuid_copy(loc.gfid, internal_op_gfid);
+        gf_uuid_copy(loc.pargfid, trash_gfid);
+
+        loc.inode = inode_new(priv->trash_itable);
+
+        /* The mkdir call for creating internal op directory */
+        loc.name = gf_strdup("internal_op");
+        if (!loc.name) {
+            gf_log(this->name, GF_LOG_DEBUG, "out of memory");
+            ret = ENOMEM;
+            goto out;
         }
+        sprintf(internal_op_path, "%s%s/", priv->newtrash_dir, loc.name);
+
+        loc.path = gf_strdup(internal_op_path);
+        if (!loc.path) {
+            gf_log(this->name, GF_LOG_DEBUG, "out of memory");
+            ret = ENOMEM;
+            goto out;
+        }
+
+        loc_copy(&local->loc, &loc);
+        STACK_WIND(frame, trash_internal_op_mkdir_cbk, FIRST_CHILD(this),
+                   FIRST_CHILD(this)->fops->mkdir, &loc, 0755, 0022, dict);
+        return 0;
+    }
 
-        STACK_DESTROY (frame->root);
-        return op_ret;
+out:
+    if (ret && gfid_ptr)
+        GF_FREE(gfid_ptr);
+    if (dict)
+        dict_unref(dict);
+    frame->local = NULL;
+    STACK_DESTROY(frame->root);
+    trash_local_wipe(local);
+    return op_ret;
 }
 
 /**
- * This is the call back of rename fop initated using STACK_WIND in
- * reconfigure function which is used to rename trash directory in
- * the brick when we perform volume set.This frame  must destroyed
- * from this function itself since it was created by trash xlator
+ * This is a nameless look up for old trash directory
+ * The lookup is based on gfid, because trash directory
+ * has fixed gfid.
  */
 int32_t
-trash_reconf_rename_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                         int32_t op_ret, int32_t op_errno, struct iatt *buf,
-                         struct iatt *preoldparent, struct iatt *postoldparent,
-                         struct iatt *prenewparent, struct iatt *postnewparent,
-                         dict_t *xdata)
+trash_dir_lookup_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                     int32_t op_ret, int32_t op_errno, inode_t *inode,
+                     struct iatt *buf, dict_t *xdata, struct iatt *postparent)
 {
-        if (op_ret == -1 && op_errno == EEXIST) {
+    trash_private_t *priv = NULL;
+    loc_t loc = {
+        0,
+    };
+    int ret = 0;
+    uuid_t *gfid_ptr = NULL;
+    dict_t *dict = NULL;
+    trash_local_t *local = NULL;
+
+    priv = this->private;
+    GF_VALIDATE_OR_GOTO("trash", priv, out);
 
-                gf_log (this->name, GF_LOG_ERROR, "rename failed: %s",
-                        strerror (op_errno));
+    local = frame->local;
+
+    loc_wipe(&local->loc);
+    if (op_ret == 0) {
+        gf_log(this->name, GF_LOG_DEBUG, "inode found with gfid %s",
+               uuid_utoa(buf->ia_gfid));
+
+        gf_uuid_copy(loc.gfid, trash_gfid);
+
+        /* Find trash inode using available information */
+        priv->trash_inode = inode_link(inode, NULL, NULL, buf);
+
+        loc.inode = inode_ref(priv->trash_inode);
+        loc_copy(&local->loc, &loc);
+
+        /*Used to find path of old trash directory*/
+        STACK_WIND(frame, trash_dir_getxattr_cbk, FIRST_CHILD(this),
+                   FIRST_CHILD(this)->fops->getxattr, &loc,
+                   GET_ANCESTRY_PATH_KEY, xdata);
+        return 0;
+    }
+
+    /* If there is no old trash directory we set its value to new one,
+     * which is the valid condition for trash directory creation
+     */
+    else {
+        gf_log(this->name, GF_LOG_DEBUG,
+               "Creating trash "
+               "directory %s ",
+               priv->newtrash_dir);
+
+        gfid_ptr = GF_MALLOC(sizeof(uuid_t), gf_common_mt_uuid_t);
+        if (!gfid_ptr) {
+            ret = ENOMEM;
+            goto out;
+        }
+        gf_uuid_copy(*gfid_ptr, trash_gfid);
+
+        gf_uuid_copy(loc.gfid, trash_gfid);
+        gf_uuid_copy(loc.pargfid, root_gfid);
+        ret = extract_trash_directory(priv->newtrash_dir, &loc.name);
+        if (ret) {
+            gf_log(this->name, GF_LOG_DEBUG, "out of memory");
+            goto out;
+        }
+        loc.path = gf_strdup(priv->newtrash_dir);
+        if (!loc.path) {
+            gf_log(this->name, GF_LOG_DEBUG, "out of memory");
+            ret = ENOMEM;
+            goto out;
+        }
+
+        priv->trash_inode = inode_new(priv->trash_itable);
+        priv->trash_inode->ia_type = IA_IFDIR;
+        loc.inode = inode_ref(priv->trash_inode);
+        dict = dict_new();
+        if (!dict) {
+            ret = ENOMEM;
+            goto out;
+        }
+        /* Fixed gfid is set for trash directory with
+         * this function
+         */
+        ret = dict_set_gfuuid(dict, "gfid-req", *gfid_ptr, false);
+        if (ret) {
+            gf_log(this->name, GF_LOG_ERROR, "setting key gfid-req failed");
+            goto out;
         }
+        loc_copy(&local->loc, &loc);
 
-        STACK_DESTROY (frame->root);
+        /* The mkdir call for creating trash directory */
+        STACK_WIND(frame, trash_dir_mkdir_cbk, FIRST_CHILD(this),
+                   FIRST_CHILD(this)->fops->mkdir, &loc, 0755, 0022, dict);
+        return 0;
+    }
+out:
+    if (ret && gfid_ptr)
+        GF_FREE(gfid_ptr);
+    if (dict)
+        dict_unref(dict);
+    frame->local = NULL;
+    STACK_DESTROY(frame->root);
+    trash_local_wipe(local);
+    return ret;
+}
+
+int
+create_or_rename_trash_directory(xlator_t *this)
+{
+    trash_private_t *priv = NULL;
+    int ret = 0;
+    loc_t loc = {
+        0,
+    };
+    call_frame_t *frame = NULL;
+    trash_local_t *local = NULL;
+
+    priv = this->private;
+
+    frame = create_frame(this, this->ctx->pool);
+    if (frame == NULL) {
+        gf_log(this->name, GF_LOG_ERROR, "failed to create frame");
+        ret = ENOMEM;
+        goto out;
+    }
+
+    local = mem_get0(this->local_pool);
+    if (!local) {
+        gf_log(this->name, GF_LOG_DEBUG, "out of memory");
+        ret = ENOMEM;
+        goto out;
+    }
+    frame->local = local;
+
+    loc.inode = inode_new(priv->trash_itable);
+    gf_uuid_copy(loc.gfid, trash_gfid);
+    loc_copy(&local->loc, &loc);
+    gf_log(this->name, GF_LOG_DEBUG,
+           "nameless lookup for"
+           "old trash directory");
+    STACK_WIND(frame, trash_dir_lookup_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->lookup, &loc, NULL);
+out:
+    return ret;
+}
 
-        return op_ret;
+int
+create_internalop_directory(xlator_t *this)
+{
+    trash_private_t *priv = NULL;
+    int ret = 0;
+    loc_t loc = {
+        0,
+    };
+    call_frame_t *frame = NULL;
+    trash_local_t *local = NULL;
+
+    priv = this->private;
+
+    frame = create_frame(this, this->ctx->pool);
+    if (frame == NULL) {
+        gf_log(this->name, GF_LOG_ERROR, "failed to create frame");
+        ret = ENOMEM;
+        goto out;
+    }
+
+    local = mem_get0(this->local_pool);
+    if (!local) {
+        gf_log(this->name, GF_LOG_DEBUG, "out of memory");
+        ret = ENOMEM;
+        goto out;
+    }
+    frame->local = local;
+
+    gf_uuid_copy(loc.gfid, internal_op_gfid);
+    gf_uuid_copy(loc.pargfid, trash_gfid);
+    loc.inode = inode_new(priv->trash_itable);
+    loc.inode->ia_type = IA_IFDIR;
+
+    loc_copy(&local->loc, &loc);
+    STACK_WIND(frame, trash_internalop_dir_lookup_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->lookup, &loc, NULL);
+out:
+
+    return ret;
 }
 
 int32_t
-trash_common_mkdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                   int32_t op_ret, int32_t op_errno, inode_t *inode,
-                   struct iatt *buf, struct iatt *preparent,
-                   struct iatt *postparent, dict_t *xdata)
+trash_common_mkdir_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                       int32_t op_ret, int32_t op_errno, inode_t *inode,
+                       struct iatt *buf, struct iatt *preparent,
+                       struct iatt *postparent, dict_t *xdata)
 {
-        STACK_UNWIND_STRICT (mkdir, frame, op_ret, op_errno, inode,
-                             buf, preparent, postparent, xdata);
-        return 0;
+    STACK_UNWIND_STRICT(mkdir, frame, op_ret, op_errno, inode, buf, preparent,
+                        postparent, xdata);
+    return 0;
 }
 
 int32_t
-trash_common_rename_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                    int32_t op_ret, int32_t op_errno, struct iatt *buf,
-                    struct iatt *preoldparent, struct iatt *postoldparent,
-                    struct iatt *prenewparent, struct iatt *postnewparent,
-                    dict_t *xdata)
+trash_common_rename_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                        int32_t op_ret, int32_t op_errno, struct iatt *buf,
+                        struct iatt *preoldparent, struct iatt *postoldparent,
+                        struct iatt *prenewparent, struct iatt *postnewparent,
+                        dict_t *xdata)
 {
-        STACK_UNWIND_STRICT (rename, frame, op_ret, op_errno, buf, preoldparent,
-                             postoldparent, prenewparent, postnewparent, xdata);
-        return 0;
+    STACK_UNWIND_STRICT(rename, frame, op_ret, op_errno, buf, preoldparent,
+                        postoldparent, prenewparent, postnewparent, xdata);
+    return 0;
 }
 
 int32_t
-trash_common_rmdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                   int32_t op_ret, int32_t op_errno, struct iatt *preparent,
-                   struct iatt *postparent,
-                   dict_t *xdata)
+trash_common_rmdir_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                       int32_t op_ret, int32_t op_errno, struct iatt *preparent,
+                       struct iatt *postparent, dict_t *xdata)
 {
-        STACK_UNWIND_STRICT (rmdir, frame, op_ret, op_errno, preparent,
-                             postparent, xdata);
-        return 0;
+    STACK_UNWIND_STRICT(rmdir, frame, op_ret, op_errno, preparent, postparent,
+                        xdata);
+    return 0;
 }
 
 /**
  * move backs from trash translator to unlink call
  */
 int32_t
-trash_common_unwind_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                         int32_t op_ret, int32_t op_errno,
-                         struct iatt *preparent, struct iatt *postparent,
-                         dict_t *xdata)
+trash_common_unwind_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                        int32_t op_ret, int32_t op_errno,
+                        struct iatt *preparent, struct iatt *postparent,
+                        dict_t *xdata)
 {
-        TRASH_STACK_UNWIND (unlink, frame, op_ret, op_errno, preparent,
-                            postparent, xdata);
-        return 0;
+    TRASH_STACK_UNWIND(unlink, frame, op_ret, op_errno, preparent, postparent,
+                       xdata);
+    return 0;
 }
 
 /**
@@ -577,160 +852,183 @@ trash_common_unwind_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
  * the starting
  */
 int32_t
-trash_unlink_mkdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                        int32_t op_ret, int32_t op_errno, inode_t *inode,
-                        struct iatt *stbuf, struct iatt *preparent,
-                        struct iatt *postparent, dict_t *xdata)
+trash_unlink_mkdir_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                       int32_t op_ret, int32_t op_errno, inode_t *inode,
+                       struct iatt *stbuf, struct iatt *preparent,
+                       struct iatt *postparent, dict_t *xdata)
 {
-        trash_local_t       *local              = NULL;
-        char                *tmp_str            = NULL;
-        char                *tmp_path           = NULL;
-        char                *tmp_dirname        = NULL;
-        char                *tmp_stat           = NULL;
-        char                real_path[PATH_MAX] = {0,};
-        char                *dir_name           = NULL;
-        size_t              count               = 0;
-        int32_t             loop_count          = 0;
-        int                 i                   = 0;
-        loc_t               tmp_loc             = {0,};
-        trash_private_t     *priv               = NULL;
-        int                 ret                 = 0;
-
-        priv = this->private;
-        GF_VALIDATE_OR_GOTO ("trash", priv, out);
-
-        local   = frame->local;
-        GF_VALIDATE_OR_GOTO ("trash", local, out);
-
-        tmp_str = gf_strdup (local->newpath);
-        if (!tmp_str) {
-                gf_log (this->name, GF_LOG_ERROR, "out of memory");
-                ret = -1;
-                goto out;
-        }
-        loop_count = local->loop_count;
-
-        /* The directory is not present , need to create it */
-        if ((op_ret == -1) &&  (op_errno == ENOENT)) {
-                tmp_dirname = strchr (tmp_str, '/');
-                while (tmp_dirname) {
-                        count = tmp_dirname - tmp_str;
-                        if (count == 0)
-                                count = 1;
-                        i++;
-                        if (i > loop_count)
-                                break;
-                        tmp_dirname = strchr (tmp_str + count + 1, '/');
-                }
-                tmp_path = gf_memdup (local->newpath, count + 1);
-                if (!tmp_path) {
-                        gf_log (this->name, GF_LOG_ERROR, "out of memory");
-                        ret = ENOMEM;
-                        goto out;
-                }
-                tmp_path[count] = '\0';
-
-                loc_copy (&tmp_loc, &local->loc);
-                tmp_loc.path = gf_strdup (tmp_path);
-                if (!tmp_loc.path) {
-                        gf_log (this->name, GF_LOG_ERROR, "out of memory");
-                        ret = ENOMEM;
-                        goto out;
-                }
-
-                /* Stores the the name of directory to be created */
-                tmp_loc.name = gf_strdup (strrchr(tmp_path, '/') + 1);
-                if (!tmp_loc.name) {
-                        gf_log (this->name, GF_LOG_ERROR, "out of memory");
-                        ret = ENOMEM;
-                        goto out;
-                }
-                strcpy (real_path, priv->brick_path);
-                remove_trash_path (tmp_path, (frame->root->pid < 0), &tmp_stat);
-                if (tmp_stat)
-                        strcat (real_path, tmp_stat);
-                STACK_WIND_COOKIE (frame, trash_unlink_mkdir_cbk, tmp_path,
-                                   FIRST_CHILD(this),
-                                   FIRST_CHILD(this)->fops->mkdir,
-                                   &tmp_loc, get_permission(real_path),
-                                   0022, xdata);
-                loc_wipe (&tmp_loc);
-                goto out;
-        }
-
-        /* Given path is created , comparing to the required path */
-        if (op_ret == 0) {
-                dir_name = dirname (tmp_str);
-                if (strcmp((char *)cookie, dir_name) == 0) {
-                        /* File path exists we can rename it*/
-                        loc_copy (&tmp_loc, &local->loc);
-                        tmp_loc.path = local->newpath;
-                        STACK_WIND (frame, trash_unlink_rename_cbk,
-                                    FIRST_CHILD(this),
-                                    FIRST_CHILD(this)->fops->rename,
-                                    &local->loc, &tmp_loc, xdata);
-                        goto out;
-                }
-        }
-
-        LOCK (&frame->lock);
-        {
-                loop_count = ++local->loop_count;
-        }
-        UNLOCK (&frame->lock);
-
-        tmp_dirname = strchr (tmp_str, '/');
-
-        /* Path is not completed , need to create remaining path */
+    trash_local_t *local = NULL;
+    char *tmp_str = NULL;
+    char *tmp_path = NULL;
+    char *tmp_dirname = NULL;
+    char *tmp_stat = NULL;
+    char real_path[PATH_MAX] = {
+        0,
+    };
+    char *dir_name = NULL;
+    size_t count = 0;
+    int32_t loop_count = 0;
+    int i = 0;
+    loc_t tmp_loc = {
+        0,
+    };
+    trash_private_t *priv = NULL;
+    int ret = 0;
+
+    priv = this->private;
+    GF_VALIDATE_OR_GOTO("trash", priv, out);
+
+    local = frame->local;
+    GF_VALIDATE_OR_GOTO("trash", local, out);
+
+    TRASH_UNSET_PID(frame, local);
+
+    tmp_str = gf_strdup(local->newpath);
+    if (!tmp_str) {
+        gf_log(this->name, GF_LOG_ERROR, "out of memory");
+        ret = -1;
+        goto out;
+    }
+    loop_count = local->loop_count;
+
+    /* The directory is not present , need to create it */
+    if ((op_ret == -1) && (op_errno == ENOENT)) {
+        tmp_dirname = strchr(tmp_str, '/');
         while (tmp_dirname) {
-                count = tmp_dirname - tmp_str;
-                if (count == 0)
-                        count = 1;
-                i++;
-                if (i > loop_count)
-                        break;
-                tmp_dirname = strchr (tmp_str + count + 1, '/');
-        }
-        tmp_path = gf_memdup (local->newpath, count + 1);
+            count = tmp_dirname - tmp_str;
+            if (count == 0)
+                count = 1;
+            i++;
+            if (i > loop_count)
+                break;
+            tmp_dirname = strchr(tmp_str + count + 1, '/');
+        }
+        tmp_path = gf_memdup(local->newpath, count + 1);
         if (!tmp_path) {
-                gf_log (this->name, GF_LOG_ERROR, "out of memory");
-                ret = -1;
-                goto out;
+            gf_log(this->name, GF_LOG_ERROR, "out of memory");
+            ret = ENOMEM;
+            goto out;
         }
         tmp_path[count] = '\0';
 
-        loc_copy (&tmp_loc, &local->loc);
-        tmp_loc.path = gf_strdup (tmp_path);
+        loc_copy(&tmp_loc, &local->loc);
+        tmp_loc.path = gf_strdup(tmp_path);
         if (!tmp_loc.path) {
-                gf_log (this->name, GF_LOG_ERROR, "out of memory");
-                ret = -1;
-                goto out;
+            gf_log(this->name, GF_LOG_ERROR, "out of memory");
+            ret = ENOMEM;
+            goto out;
         }
 
         /* Stores the the name of directory to be created */
-        tmp_loc.name = gf_strdup (strrchr(tmp_path, '/') + 1);
+        tmp_loc.name = gf_strdup(strrchr(tmp_path, '/') + 1);
         if (!tmp_loc.name) {
-                gf_log (this->name, GF_LOG_ERROR, "out of memory");
-                ret = -1;
-                goto out;
+            gf_log(this->name, GF_LOG_ERROR, "out of memory");
+            ret = ENOMEM;
+            goto out;
         }
+        strncpy(real_path, priv->brick_path, sizeof(real_path));
+        real_path[sizeof(real_path) - 1] = 0;
 
-        strcpy (real_path, priv->brick_path);
-        remove_trash_path (tmp_path, (frame->root->pid < 0), &tmp_stat);
+        remove_trash_path(tmp_path, (frame->root->pid < 0), &tmp_stat);
         if (tmp_stat)
-                strcat (real_path, tmp_stat);
-
-        STACK_WIND_COOKIE (frame, trash_unlink_mkdir_cbk, tmp_path,
-                           FIRST_CHILD(this),
-                           FIRST_CHILD(this)->fops->mkdir, &tmp_loc,
-                           get_permission(real_path), 0022, xdata);
+            strncat(real_path, tmp_stat,
+                    sizeof(real_path) - strlen(real_path) - 1);
+
+        TRASH_SET_PID(frame, local);
+
+        STACK_WIND_COOKIE(frame, trash_unlink_mkdir_cbk, tmp_path,
+                          FIRST_CHILD(this), FIRST_CHILD(this)->fops->mkdir,
+                          &tmp_loc, get_permission(real_path), 0022, xdata);
+        loc_wipe(&tmp_loc);
+        goto out;
+    }
+
+    /* Given path is created , comparing to the required path */
+    if (op_ret == 0) {
+        dir_name = dirname(tmp_str);
+        if (strcmp((char *)cookie, dir_name) == 0) {
+            /* File path exists we can rename it*/
+            loc_copy(&tmp_loc, &local->loc);
+            tmp_loc.path = local->newpath;
+            STACK_WIND(frame, trash_unlink_rename_cbk, FIRST_CHILD(this),
+                       FIRST_CHILD(this)->fops->rename, &local->loc, &tmp_loc,
+                       xdata);
+            goto out;
+        }
+    }
+
+    if ((op_ret == -1) && (op_errno != EEXIST)) {
+        gf_log(this->name, GF_LOG_ERROR,
+               "Directory creation failed [%s]. "
+               "Therefore unlinking %s without moving to trash "
+               "directory",
+               strerror(op_errno), local->loc.name);
+        STACK_WIND(frame, trash_common_unwind_cbk, FIRST_CHILD(this),
+                   FIRST_CHILD(this)->fops->unlink, &local->loc, 0, xdata);
+        goto out;
+    }
+
+    LOCK(&frame->lock);
+    {
+        loop_count = ++local->loop_count;
+    }
+    UNLOCK(&frame->lock);
+
+    tmp_dirname = strchr(tmp_str, '/');
+
+    /* Path is not completed , need to create remaining path */
+    while (tmp_dirname) {
+        count = tmp_dirname - tmp_str;
+        if (count == 0)
+            count = 1;
+        i++;
+        if (i > loop_count)
+            break;
+        tmp_dirname = strchr(tmp_str + count + 1, '/');
+    }
+    tmp_path = gf_memdup(local->newpath, count + 1);
+    if (!tmp_path) {
+        gf_log(this->name, GF_LOG_ERROR, "out of memory");
+        ret = -1;
+        goto out;
+    }
+    tmp_path[count] = '\0';
+
+    loc_copy(&tmp_loc, &local->loc);
+    tmp_loc.path = gf_strdup(tmp_path);
+    if (!tmp_loc.path) {
+        gf_log(this->name, GF_LOG_ERROR, "out of memory");
+        ret = -1;
+        goto out;
+    }
+
+    /* Stores the the name of directory to be created */
+    tmp_loc.name = gf_strdup(strrchr(tmp_path, '/') + 1);
+    if (!tmp_loc.name) {
+        gf_log(this->name, GF_LOG_ERROR, "out of memory");
+        ret = -1;
+        goto out;
+    }
+
+    strncpy(real_path, priv->brick_path, sizeof(real_path));
+    real_path[sizeof(real_path) - 1] = 0;
+
+    remove_trash_path(tmp_path, (frame->root->pid < 0), &tmp_stat);
+    if (tmp_stat)
+        strncat(real_path, tmp_stat, sizeof(real_path) - strlen(real_path) - 1);
+
+    TRASH_SET_PID(frame, local);
+
+    STACK_WIND_COOKIE(frame, trash_unlink_mkdir_cbk, tmp_path,
+                      FIRST_CHILD(this), FIRST_CHILD(this)->fops->mkdir,
+                      &tmp_loc, get_permission(real_path), 0022, xdata);
 
 out:
-        if (tmp_path)
-                GF_FREE (tmp_path);
-        if (tmp_str)
-                GF_FREE (tmp_str);
-        return ret;
+    if (tmp_path)
+        GF_FREE(tmp_path);
+    if (tmp_str)
+        GF_FREE(tmp_str);
+    return ret;
 }
 
 /**
@@ -738,193 +1036,238 @@ out:
  * from trash directory as mentioned in the mount point
  */
 int32_t
-trash_unlink_rename_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                         int32_t op_ret, int32_t op_errno, struct iatt *buf,
-                         struct iatt *preoldparent, struct iatt *postoldparent,
-                         struct iatt *prenewparent, struct iatt *postnewparent,
-                         dict_t *xdata)
+trash_unlink_rename_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                        int32_t op_ret, int32_t op_errno, struct iatt *buf,
+                        struct iatt *preoldparent, struct iatt *postoldparent,
+                        struct iatt *prenewparent, struct iatt *postnewparent,
+                        dict_t *xdata)
 {
-        trash_local_t      *local              = NULL;
-        trash_private_t     *priv              = NULL;
-        char               *tmp_str            = NULL;
-        char               *dir_name           = NULL;
-        char               *tmp_cookie         = NULL;
-        loc_t              tmp_loc             = {0,};
-        char               *tmp_stat           = NULL;
-        char               real_path[PATH_MAX] = {0,};
-        int                ret                 = 0;
-
-        priv = this->private;
-        GF_VALIDATE_OR_GOTO ("trash", priv, out);
-
-        local = frame->local;
-        GF_VALIDATE_OR_GOTO ("trash", local, out);
-
-        if ((op_ret == -1) && (op_errno == ENOENT)) {
-                /* the file path doesnot exists we want to create path
-                 * for the file
-                 */
-                tmp_str = gf_strdup (local->newpath);
-                if (!tmp_str) {
-                        gf_log (this->name, GF_LOG_DEBUG, "out of memory");
-                        ret = ENOMEM;
-                        goto out;
-                }
-                dir_name = dirname (tmp_str); /* stores directory name */
-
-                loc_copy (&tmp_loc, &local->loc);
-                tmp_loc.path = gf_strdup (dir_name);
-                if (!tmp_loc.path) {
-                        gf_log (this->name, GF_LOG_ERROR, "out of memory");
-                        ret = ENOMEM;
-                        goto out;
-                }
-
-                tmp_cookie = gf_strdup (dir_name);
-                if (!tmp_cookie) {
-                        gf_log (this->name, GF_LOG_DEBUG, "out of memory");
-                        ret = ENOMEM;
-                        goto out;
-                }
-                strcpy (real_path, priv->brick_path);
-                remove_trash_path (tmp_str, (frame->root->pid < 0), &tmp_stat);
-                if (tmp_stat)
-                        strcat (real_path, tmp_stat);
-                /* create the directory with proper permissions */
-                STACK_WIND_COOKIE (frame, trash_unlink_mkdir_cbk, tmp_cookie,
-                                   FIRST_CHILD(this),
-                                   FIRST_CHILD(this)->fops->mkdir,
-                                   &tmp_loc, get_permission(real_path),
-                                   0022, xdata);
-                loc_wipe (&tmp_loc);
-                goto out;
+    trash_local_t *local = NULL;
+    trash_private_t *priv = NULL;
+    char *tmp_str = NULL;
+    char *dir_name = NULL;
+    char *tmp_cookie = NULL;
+    loc_t tmp_loc = {
+        0,
+    };
+    dict_t *new_xdata = NULL;
+    char *tmp_stat = NULL;
+    char real_path[PATH_MAX] = {
+        0,
+    };
+    int ret = 0;
+
+    priv = this->private;
+    GF_VALIDATE_OR_GOTO("trash", priv, out);
+
+    local = frame->local;
+    GF_VALIDATE_OR_GOTO("trash", local, out);
+
+    if ((op_ret == -1) && (op_errno == ENOENT)) {
+        /* the file path does not exist we want to create path
+         * for the file
+         */
+        tmp_str = gf_strdup(local->newpath);
+        if (!tmp_str) {
+            gf_log(this->name, GF_LOG_DEBUG, "out of memory");
+            ret = ENOMEM;
+            goto out;
         }
+        dir_name = dirname(tmp_str); /* stores directory name */
 
-        if ((op_ret == -1) && (op_errno == ENOTDIR)) {
-                /* if entry is already present in trash directory,
-                 * new one is not copied*/
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "target(%s) exists, cannot keep the copy, deleting",
-                        local->newpath);
-
-                STACK_WIND (frame, trash_common_unwind_cbk,
-                            FIRST_CHILD(this),
-                            FIRST_CHILD(this)->fops->unlink,
-                            &local->loc, 0, xdata);
-
-                goto out;
+        loc_copy(&tmp_loc, &local->loc);
+        tmp_loc.path = gf_strdup(dir_name);
+        if (!tmp_loc.path) {
+            gf_log(this->name, GF_LOG_ERROR, "out of memory");
+            ret = ENOMEM;
+            goto out;
         }
 
-        if ((op_ret == -1) && (op_errno == EISDIR)) {
-
-                /* if entry is directory,we remove directly */
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "target(%s) exists as directory, cannot keep copy, "
-                        "deleting", local->newpath);
-
-                STACK_WIND (frame, trash_common_unwind_cbk,
-                            FIRST_CHILD(this),
-                            FIRST_CHILD(this)->fops->unlink,
-                            &local->loc, 0, xdata);
-                goto out;
+        tmp_cookie = gf_strdup(dir_name);
+        if (!tmp_cookie) {
+            gf_log(this->name, GF_LOG_DEBUG, "out of memory");
+            ret = ENOMEM;
+            goto out;
         }
-
-        /* All other cases, unlink should return success */
-        TRASH_STACK_UNWIND (unlink, frame, 0, op_errno, &local->preparent,
-                            &local->postparent, xdata);
+        strncpy(real_path, priv->brick_path, sizeof(real_path));
+        real_path[sizeof(real_path) - 1] = 0;
+        remove_trash_path(tmp_str, (frame->root->pid < 0), &tmp_stat);
+        if (tmp_stat)
+            strncat(real_path, tmp_stat,
+                    sizeof(real_path) - strlen(real_path) - 1);
+
+        TRASH_SET_PID(frame, local);
+
+        /* create the directory with proper permissions */
+        STACK_WIND_COOKIE(frame, trash_unlink_mkdir_cbk, tmp_cookie,
+                          FIRST_CHILD(this), FIRST_CHILD(this)->fops->mkdir,
+                          &tmp_loc, get_permission(real_path), 0022, xdata);
+        loc_wipe(&tmp_loc);
+        goto out;
+    }
+
+    if ((op_ret == -1) && (op_errno == ENOTDIR)) {
+        /* if entry is already present in trash directory,
+         * new one is not copied*/
+        gf_log(this->name, GF_LOG_DEBUG,
+               "target(%s) exists, cannot keep the copy, deleting",
+               local->newpath);
+
+        STACK_WIND(frame, trash_common_unwind_cbk, FIRST_CHILD(this),
+                   FIRST_CHILD(this)->fops->unlink, &local->loc, 0, xdata);
+
+        goto out;
+    }
+
+    if ((op_ret == -1) && (op_errno == EISDIR)) {
+        /* if entry is directory,we remove directly */
+        gf_log(this->name, GF_LOG_DEBUG,
+               "target(%s) exists as directory, cannot keep copy, "
+               "deleting",
+               local->newpath);
+
+        STACK_WIND(frame, trash_common_unwind_cbk, FIRST_CHILD(this),
+                   FIRST_CHILD(this)->fops->unlink, &local->loc, 0, xdata);
+        goto out;
+    }
+
+    /**********************************************************************
+     *
+     * CTR Xlator message handling done here!
+     *
+     **********************************************************************/
+    /**
+     * If unlink is handled by trash translator, it should inform the
+     * CTR Xlator. And trash translator only handles the unlink for
+     * the last hardlink.
+     *
+     * Check if there is a GF_REQUEST_LINK_COUNT_XDATA from CTR Xlator
+     *
+     */
+
+    if (local->ctr_link_count_req) {
+        /* Sending back inode link count to ctr_unlink
+         * (changetimerecoder xlator) via
+         * "GF_RESPONSE_LINK_COUNT_XDATA" key using xdata.
+         * */
+        if (xdata) {
+            ret = dict_set_uint32(xdata, GF_RESPONSE_LINK_COUNT_XDATA, 1);
+            if (ret == -1) {
+                gf_log(this->name, GF_LOG_WARNING,
+                       "Failed to set"
+                       " GF_RESPONSE_LINK_COUNT_XDATA");
+            }
+        } else {
+            new_xdata = dict_new();
+            if (!new_xdata) {
+                gf_log(this->name, GF_LOG_WARNING,
+                       "Memory allocation failure while "
+                       "creating new_xdata");
+                goto ctr_out;
+            }
+            ret = dict_set_uint32(new_xdata, GF_RESPONSE_LINK_COUNT_XDATA, 1);
+            if (ret == -1) {
+                gf_log(this->name, GF_LOG_WARNING,
+                       "Failed to set"
+                       " GF_RESPONSE_LINK_COUNT_XDATA");
+            }
+        ctr_out:
+            TRASH_STACK_UNWIND(unlink, frame, 0, op_errno, preoldparent,
+                               postoldparent, new_xdata);
+            goto out;
+        }
+    }
+    /* All other cases, unlink should return success */
+    TRASH_STACK_UNWIND(unlink, frame, 0, op_errno, preoldparent, postoldparent,
+                       xdata);
 out:
-        if (tmp_str)
-                GF_FREE (tmp_str);
-        if (tmp_cookie)
-                GF_FREE (tmp_cookie);
 
-        return ret;
+    if (tmp_str)
+        GF_FREE(tmp_str);
+    if (tmp_cookie)
+        GF_FREE(tmp_cookie);
+    if (new_xdata)
+        dict_unref(new_xdata);
+
+    return ret;
 }
 
 /**
  * move backs from trash translator to truncate call
  */
 int32_t
-trash_common_unwind_buf_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                             int32_t op_ret, int32_t op_errno,
-                             struct iatt *prebuf, struct iatt *postbuf,
-                             dict_t *xdata)
+trash_common_unwind_buf_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                            int32_t op_ret, int32_t op_errno,
+                            struct iatt *prebuf, struct iatt *postbuf,
+                            dict_t *xdata)
 {
-        TRASH_STACK_UNWIND (truncate, frame, op_ret, op_errno, prebuf,
-                            postbuf, xdata);
-        return 0;
+    TRASH_STACK_UNWIND(truncate, frame, op_ret, op_errno, prebuf, postbuf,
+                       xdata);
+    return 0;
 }
 
-
-
 int32_t
-trash_unlink_stat_cbk (call_frame_t *frame,  void *cookie, xlator_t *this,
-                       int32_t op_ret, int32_t op_errno, struct iatt *buf,
-                       dict_t *xdata)
+trash_unlink_stat_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                      int32_t op_ret, int32_t op_errno, struct iatt *buf,
+                      dict_t *xdata)
 {
-        trash_private_t     *priv        = NULL;
-        trash_local_t       *local       = NULL;
-        loc_t               new_loc      = {0,};
-        int                 ret          = 0;
-
-        priv  = this->private;
-        GF_VALIDATE_OR_GOTO ("trash", priv, out);
-
-        local = frame->local;
-        GF_VALIDATE_OR_GOTO ("trash", local, out);
-
-        if (op_ret == -1) {
-                gf_log (this->name, GF_LOG_DEBUG, "%s: %s",
-                        local->loc.path, strerror (op_errno));
-                TRASH_STACK_UNWIND (unlink, frame, op_ret, op_errno, buf,
-                            NULL, xdata);
-                ret = -1;
-                goto out;
-        }
-
-        /* Only last hardlink will be moved to trash directory */
-        if (buf->ia_nlink > 1) {
-                STACK_WIND (frame, trash_common_unwind_cbk,
-                            FIRST_CHILD(this),
-                            FIRST_CHILD(this)->fops->unlink, &local->loc,
-                            0, xdata);
-                goto out;
-        }
-
-        /* if the file is too big  just unlink it */
-        if (buf->ia_size > (priv->max_trash_file_size)) {
-                gf_log (this->name, GF_LOG_DEBUG,
-                                "%s: file size too big (%"PRId64") to "
-                                "move into trash directory",
-                                local->loc.path, buf->ia_size);
-
-                STACK_WIND (frame, trash_common_unwind_cbk,
-                            FIRST_CHILD(this),
-                            FIRST_CHILD(this)->fops->unlink, &local->loc,
-                            0, xdata);
-                goto out;
-        }
-
-        /* Copies new path for renaming */
-        loc_copy (&new_loc, &local->loc);
-        new_loc.path = gf_strdup (local->newpath);
-        if (!new_loc.path) {
-                gf_log (this->name, GF_LOG_DEBUG, "out of memory");
-                ret = ENOMEM;
-                goto out;
-        }
-
-
-        STACK_WIND (frame, trash_unlink_rename_cbk,
-                    FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->rename,
-                    &local->loc, &new_loc, xdata);
+    trash_private_t *priv = NULL;
+    trash_local_t *local = NULL;
+    loc_t new_loc = {
+        0,
+    };
+    int ret = 0;
+
+    priv = this->private;
+    GF_VALIDATE_OR_GOTO("trash", priv, out);
+
+    local = frame->local;
+    GF_VALIDATE_OR_GOTO("trash", local, out);
+
+    if (op_ret == -1) {
+        gf_log(this->name, GF_LOG_DEBUG, "%s: %s", local->loc.path,
+               strerror(op_errno));
+        TRASH_STACK_UNWIND(unlink, frame, op_ret, op_errno, buf, NULL, xdata);
+        ret = -1;
+        goto out;
+    }
+
+    /* Only last hardlink will be moved to trash directory */
+    if (buf->ia_nlink > 1) {
+        STACK_WIND(frame, trash_common_unwind_cbk, FIRST_CHILD(this),
+                   FIRST_CHILD(this)->fops->unlink, &local->loc, 0, xdata);
+        goto out;
+    }
+
+    /* if the file is too big  just unlink it */
+    if (buf->ia_size > (priv->max_trash_file_size)) {
+        gf_log(this->name, GF_LOG_DEBUG,
+               "%s: file size too big (%" PRId64
+               ") to "
+               "move into trash directory",
+               local->loc.path, buf->ia_size);
+
+        STACK_WIND(frame, trash_common_unwind_cbk, FIRST_CHILD(this),
+                   FIRST_CHILD(this)->fops->unlink, &local->loc, 0, xdata);
+        goto out;
+    }
+
+    /* Copies new path for renaming */
+    loc_copy(&new_loc, &local->loc);
+    new_loc.path = gf_strdup(local->newpath);
+    if (!new_loc.path) {
+        gf_log(this->name, GF_LOG_DEBUG, "out of memory");
+        ret = ENOMEM;
+        goto out;
+    }
+
+    STACK_WIND(frame, trash_unlink_rename_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->rename, &local->loc, &new_loc, xdata);
 
 out:
-        loc_wipe (&new_loc);
-
-        return ret;
+    loc_wipe(&new_loc);
 
+    return ret;
 }
 
 /**
@@ -932,262 +1275,248 @@ out:
  * by internal operations of gluster such as self-heal
  */
 int32_t
-trash_unlink (call_frame_t *frame, xlator_t *this, loc_t *loc, int xflags,
-              dict_t *xdata)
+trash_unlink(call_frame_t *frame, xlator_t *this, loc_t *loc, int xflags,
+             dict_t *xdata)
 {
-        trash_private_t         *priv           = NULL;
-        trash_local_t           *local          = NULL;/* files inside trash */
-        int32_t                 match           = 0;
-        char                    *pathbuf        = NULL;
-        int                     ret             = 0;
-
-        priv = this->private;
-        GF_VALIDATE_OR_GOTO ("trash", priv, out);
-
-        /* If trash is not active or not enabled through cli, then
-         *  we bypass and wind back
-         */
-        if (!priv->state) {
-                STACK_WIND (frame, trash_common_unwind_cbk,
-                            FIRST_CHILD(this),
-                            FIRST_CHILD(this)->fops->unlink, loc, 0,
-                            xdata);
-                goto out;
-        }
-
-        /* The files removed by gluster internal operations such as self-heal,
-         * should moved to trash directory , but files by client should not
-         * moved
-         */
-        if ((frame->root->pid < 0) && !priv->internal) {
-                STACK_WIND (frame, trash_common_unwind_cbk,
-                            FIRST_CHILD(this),
-                            FIRST_CHILD(this)->fops->unlink, loc, 0,
-                            xdata);
-                goto out;
-        }
-        /* loc need some gfid which will be present in inode */
-        gf_uuid_copy (loc->gfid, loc->inode->gfid);
-
-        /* Checking for valid location */
-        if (gf_uuid_is_null (loc->gfid) && gf_uuid_is_null (loc->inode->gfid)) {
-                gf_log (this->name, GF_LOG_DEBUG, "Bad address");
-                STACK_WIND (frame, trash_common_unwind_cbk,
-                            FIRST_CHILD(this),
-                            FIRST_CHILD(this)->fops->unlink, loc, 0,
-                            xdata);
-                ret = EFAULT;
-                goto out;
-        }
-
-        /* This will be more accurate */
-        inode_path (loc->inode, NULL, &pathbuf);
-        /* Check whether the file is present under eliminate paths or
-         * inside trash directory. In both cases we don't need to move the
-         * file to trash directory. Instead delete it permanently
-         */
-        match = check_whether_eliminate_path (priv->eliminate, pathbuf);
-        if ((strncmp (pathbuf, priv->newtrash_dir,
-                      strlen (priv->newtrash_dir)) == 0) || (match)) {
-                if (match) {
-                        gf_log (this->name, GF_LOG_DEBUG,
-                                "%s is a file comes under an eliminate path, "
-                                "so it is not moved to trash", loc->name);
-                }
-
-                /* Trying to unlink from the trash-dir. So do the
-                 * actual unlink without moving to trash-dir.
-                 */
-                STACK_WIND (frame, trash_common_unwind_cbk,
-                            FIRST_CHILD(this),
-                            FIRST_CHILD(this)->fops->unlink, loc, 0,
-                            xdata);
-                goto out;
-        }
-
-        local = mem_get0 (this->local_pool);
-        if (!local) {
-                gf_log (this->name, GF_LOG_DEBUG, "out of memory");
-                TRASH_STACK_UNWIND (unlink, frame, -1, ENOMEM, NULL, NULL,
-                                    xdata);
-                ret = ENOMEM;
-                goto out;
-        }
-        frame->local = local;
-        loc_copy (&local->loc, loc);
-
-        /* rename new location of file as starting from trash directory */
-        strcpy (local->origpath, pathbuf);
-        copy_trash_path (priv->newtrash_dir, (frame->root->pid < 0),
-                                                        local->newpath);
-        strcat (local->newpath, pathbuf);
-
-        /* append timestamp to file name so that we can avoid
-         * name collisions inside trash
+    trash_private_t *priv = NULL;
+    trash_local_t *local = NULL; /* files inside trash */
+    int32_t match = 0;
+    int32_t ctr_link_req = 0;
+    char *pathbuf = NULL;
+    int ret = 0;
+
+    priv = this->private;
+    GF_VALIDATE_OR_GOTO("trash", priv, out);
+
+    /* If trash is not active or not enabled through cli, then
+     *  we bypass and wind back
+     */
+    if (!priv->state) {
+        STACK_WIND(frame, trash_common_unwind_cbk, FIRST_CHILD(this),
+                   FIRST_CHILD(this)->fops->unlink, loc, 0, xdata);
+        goto out;
+    }
+
+    /* The files removed by gluster internal operations such as self-heal,
+     * should moved to trash directory , but files by client should not
+     * moved
+     */
+    if ((frame->root->pid < 0) && !priv->internal) {
+        STACK_WIND(frame, trash_common_unwind_cbk, FIRST_CHILD(this),
+                   FIRST_CHILD(this)->fops->unlink, loc, 0, xdata);
+        goto out;
+    }
+    /* loc need some gfid which will be present in inode */
+    gf_uuid_copy(loc->gfid, loc->inode->gfid);
+
+    /* Checking for valid location */
+    if (gf_uuid_is_null(loc->gfid) && gf_uuid_is_null(loc->inode->gfid)) {
+        gf_log(this->name, GF_LOG_DEBUG, "Bad address");
+        STACK_WIND(frame, trash_common_unwind_cbk, FIRST_CHILD(this),
+                   FIRST_CHILD(this)->fops->unlink, loc, 0, xdata);
+        ret = EFAULT;
+        goto out;
+    }
+
+    /* This will be more accurate */
+    inode_path(loc->inode, NULL, &pathbuf);
+    /* Check whether the file is present under eliminate paths or
+     * inside trash directory. In both cases we don't need to move the
+     * file to trash directory. Instead delete it permanently
+     */
+    match = check_whether_eliminate_path(priv->eliminate, pathbuf);
+    if ((strncmp(pathbuf, priv->newtrash_dir, strlen(priv->newtrash_dir)) ==
+         0) ||
+        (match)) {
+        if (match) {
+            gf_log(this->name, GF_LOG_DEBUG,
+                   "%s is a file comes under an eliminate path, "
+                   "so it is not moved to trash",
+                   loc->name);
+        }
+
+        /* Trying to unlink from the trash-dir. So do the
+         * actual unlink without moving to trash-dir.
          */
-        append_time_stamp (local->newpath);
-        if (strlen (local->newpath) > PATH_MAX) {
-                STACK_WIND (frame, trash_common_unwind_cbk,
-                            FIRST_CHILD(this),
-                            FIRST_CHILD(this)->fops->unlink, loc, 0,
-                            xdata);
-                goto out;
-        }
+        STACK_WIND(frame, trash_common_unwind_cbk, FIRST_CHILD(this),
+                   FIRST_CHILD(this)->fops->unlink, loc, 0, xdata);
+        goto out;
+    }
+
+    local = mem_get0(this->local_pool);
+    if (!local) {
+        gf_log(this->name, GF_LOG_DEBUG, "out of memory");
+        TRASH_STACK_UNWIND(unlink, frame, -1, ENOMEM, NULL, NULL, xdata);
+        ret = ENOMEM;
+        goto out;
+    }
+    frame->local = local;
+    loc_copy(&local->loc, loc);
+
+    /* rename new location of file as starting from trash directory */
+    copy_trash_path(priv->newtrash_dir, (frame->root->pid < 0), local->newpath,
+                    sizeof(local->newpath));
+    strncat(local->newpath, pathbuf,
+            sizeof(local->newpath) - strlen(local->newpath) - 1);
+
+    /* append timestamp to file name so that we can avoid
+     * name collisions inside trash
+     */
+    append_time_stamp(local->newpath, sizeof(local->newpath));
+    if (strlen(local->newpath) > PATH_MAX) {
+        STACK_WIND(frame, trash_common_unwind_cbk, FIRST_CHILD(this),
+                   FIRST_CHILD(this)->fops->unlink, loc, 0, xdata);
+        goto out;
+    }
+
+    /* To know whether CTR xlator requested for the link count */
+    ret = dict_get_int32(xdata, GF_REQUEST_LINK_COUNT_XDATA, &ctr_link_req);
+    if (ret) {
+        local->ctr_link_count_req = _gf_false;
+        ret = 0;
+    } else
+        local->ctr_link_count_req = _gf_true;
 
-        LOCK_INIT (&frame->lock);
+    LOCK_INIT(&frame->lock);
 
-        STACK_WIND (frame, trash_unlink_stat_cbk,
-                    FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->stat, loc, xdata);
+    STACK_WIND(frame, trash_unlink_stat_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->stat, loc, xdata);
 out:
-        return ret;
+    return ret;
 }
 
 /**
  * Use this when a failure occurs, and delete the newly created file
  */
 int32_t
-trash_truncate_unlink_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                           int32_t op_ret, int32_t op_errno,
-                           struct iatt *preparent, struct iatt *postparent,
-                           dict_t *xdata)
+trash_truncate_unlink_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                          int32_t op_ret, int32_t op_errno,
+                          struct iatt *preparent, struct iatt *postparent,
+                          dict_t *xdata)
 {
-        trash_local_t     *local  = NULL;
+    trash_local_t *local = NULL;
 
-        local = frame->local;
-        GF_VALIDATE_OR_GOTO ("trash", local, out);
+    local = frame->local;
+    GF_VALIDATE_OR_GOTO("trash", local, out);
 
-        if (op_ret == -1) {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "deleting the newly created file: %s",
-                        strerror (op_errno));
-        }
+    if (op_ret == -1) {
+        gf_log(this->name, GF_LOG_DEBUG, "deleting the newly created file: %s",
+               strerror(op_errno));
+    }
 
-        STACK_WIND (frame, trash_common_unwind_buf_cbk,
-                    FIRST_CHILD(this), FIRST_CHILD(this)->fops->truncate,
-                    &local->loc, local->fop_offset, xdata);
+    STACK_WIND(frame, trash_common_unwind_buf_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->truncate, &local->loc,
+               local->fop_offset, xdata);
 out:
-        return 0;
+    return 0;
 }
 
 /**
  * Read from source file
  */
 int32_t
-trash_truncate_readv_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                          int32_t op_ret, int32_t op_errno,
-                          struct iovec *vector, int32_t count,
-                          struct iatt *stbuf, struct iobref *iobuf,
-                          dict_t *xdata)
+trash_truncate_readv_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                         int32_t op_ret, int32_t op_errno, struct iovec *vector,
+                         int32_t count, struct iatt *stbuf,
+                         struct iobref *iobuf, dict_t *xdata)
 {
+    trash_local_t *local = NULL;
 
-        trash_local_t    *local  = NULL;
+    local = frame->local;
+    GF_VALIDATE_OR_GOTO("trash", local, out);
 
-        local = frame->local;
-        GF_VALIDATE_OR_GOTO ("trash", local, out);
+    if (op_ret == -1) {
+        gf_log(this->name, GF_LOG_DEBUG,
+               "readv on the existing file failed: %s", strerror(op_errno));
 
-        if (op_ret == -1) {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "readv on the existing file failed: %s",
-                        strerror (op_errno));
+        STACK_WIND(frame, trash_truncate_unlink_cbk, FIRST_CHILD(this),
+                   FIRST_CHILD(this)->fops->unlink, &local->newloc, 0, xdata);
+        goto out;
+    }
 
-                STACK_WIND (frame, trash_truncate_unlink_cbk,
-                            FIRST_CHILD(this), FIRST_CHILD(this)->fops->unlink,
-                            &local->newloc, 0, xdata);
-                goto out;
-        }
-
-        local->fsize = stbuf->ia_size;
-        STACK_WIND (frame, trash_truncate_writev_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->writev,
-                    local->newfd, vector, count, local->cur_offset, 0, iobuf,
-                    xdata);
+    local->fsize = stbuf->ia_size;
+    STACK_WIND(frame, trash_truncate_writev_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->writev, local->newfd, vector, count,
+               local->cur_offset, 0, iobuf, xdata);
 
 out:
-        return 0;
-
+    return 0;
 }
 
 /**
  * Write to file created in trash directory
  */
 int32_t
-trash_truncate_writev_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                           int32_t op_ret, int32_t op_errno,
-                           struct iatt *prebuf, struct iatt *postbuf,
-                           dict_t *xdata)
+trash_truncate_writev_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                          int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
+                          struct iatt *postbuf, dict_t *xdata)
 {
-        trash_local_t    *local  = NULL;
-
-        local = frame->local;
-        GF_VALIDATE_OR_GOTO ("trash", local, out);
-
-        if (op_ret == -1) {
-                /* Let truncate work, but previous copy is not preserved. */
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "writev on the existing file failed: %s",
-                        strerror (op_errno));
-
-                STACK_WIND (frame, trash_truncate_unlink_cbk, FIRST_CHILD(this),
-                            FIRST_CHILD(this)->fops->unlink, &local->newloc, 0,
-                            xdata);
-                goto out;
-        }
-
-        if (local->cur_offset < local->fsize) {
-                local->cur_offset += GF_BLOCK_READV_SIZE;
-                /* Loop back and Read the contents again. */
-                STACK_WIND (frame, trash_truncate_readv_cbk,
-                            FIRST_CHILD(this), FIRST_CHILD(this)->fops->readv,
-                            local->fd, (size_t)GF_BLOCK_READV_SIZE,
-                            local->cur_offset, 0, xdata);
-                goto out;
-        }
-
-
-        /* OOFH.....Finally calling Truncate. */
-        STACK_WIND (frame, trash_common_unwind_buf_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->truncate, &local->loc,
-                    local->fop_offset, xdata);
+    trash_local_t *local = NULL;
+
+    local = frame->local;
+    GF_VALIDATE_OR_GOTO("trash", local, out);
+
+    if (op_ret == -1) {
+        /* Let truncate work, but previous copy is not preserved. */
+        gf_log(this->name, GF_LOG_DEBUG,
+               "writev on the existing file failed: %s", strerror(op_errno));
+
+        STACK_WIND(frame, trash_truncate_unlink_cbk, FIRST_CHILD(this),
+                   FIRST_CHILD(this)->fops->unlink, &local->newloc, 0, xdata);
+        goto out;
+    }
+
+    if (local->cur_offset < local->fsize) {
+        local->cur_offset += GF_BLOCK_READV_SIZE;
+        /* Loop back and Read the contents again. */
+        STACK_WIND(frame, trash_truncate_readv_cbk, FIRST_CHILD(this),
+                   FIRST_CHILD(this)->fops->readv, local->fd,
+                   (size_t)GF_BLOCK_READV_SIZE, local->cur_offset, 0, xdata);
+        goto out;
+    }
+
+    /* OOFH.....Finally calling Truncate. */
+    STACK_WIND(frame, trash_common_unwind_buf_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->truncate, &local->loc,
+               local->fop_offset, xdata);
 
 out:
-        return 0;
+    return 0;
 }
 
 /**
  * The source file is opened for reading and writing
  */
 int32_t
-trash_truncate_open_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                         int32_t op_ret, int32_t op_errno, fd_t *fd,
-                         dict_t *xdata)
+trash_truncate_open_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                        int32_t op_ret, int32_t op_errno, fd_t *fd,
+                        dict_t *xdata)
 {
-        trash_local_t    *local  = NULL;
+    trash_local_t *local = NULL;
 
-        local = frame->local;
-        GF_VALIDATE_OR_GOTO ("trash", local, out);
+    local = frame->local;
+    GF_VALIDATE_OR_GOTO("trash", local, out);
 
-        if (op_ret == -1) {
-                /* Let truncate work, but previous copy is not preserved. */
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "open on the existing file failed: %s",
-                        strerror (op_errno));
+    if (op_ret == -1) {
+        /* Let truncate work, but previous copy is not preserved. */
+        gf_log(this->name, GF_LOG_DEBUG, "open on the existing file failed: %s",
+               strerror(op_errno));
 
-                STACK_WIND (frame, trash_truncate_unlink_cbk,
-                            FIRST_CHILD(this), FIRST_CHILD(this)->fops->unlink,
-                            &local->newloc, 0, xdata);
-                goto out;
-        }
+        STACK_WIND(frame, trash_truncate_unlink_cbk, FIRST_CHILD(this),
+                   FIRST_CHILD(this)->fops->unlink, &local->newloc, 0, xdata);
+        goto out;
+    }
 
-        local->cur_offset = 0;
+    fd_bind(fd);
 
-        STACK_WIND (frame, trash_truncate_readv_cbk,
-                    FIRST_CHILD (this), FIRST_CHILD (this)->fops->readv,
-                    local->fd, (size_t)GF_BLOCK_READV_SIZE, local->cur_offset,
-                    0, xdata);
+    local->cur_offset = 0;
+
+    STACK_WIND(frame, trash_truncate_readv_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->readv, local->fd,
+               (size_t)GF_BLOCK_READV_SIZE, local->cur_offset, 0, xdata);
 
 out:
-        return 0;
+    return 0;
 }
 
 /**
@@ -1195,95 +1524,104 @@ out:
  * if the path is present in trash directory
  */
 int32_t
-trash_truncate_create_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                           int32_t op_ret, int32_t op_errno, fd_t *fd,
-                           inode_t *inode, struct iatt *buf,
-                           struct iatt *preparent, struct iatt *postparent,
-                           dict_t *xdata)
+trash_truncate_create_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                          int32_t op_ret, int32_t op_errno, fd_t *fd,
+                          inode_t *inode, struct iatt *buf,
+                          struct iatt *preparent, struct iatt *postparent,
+                          dict_t *xdata)
 {
-        trash_local_t        *local                  = NULL;
-        char                 *tmp_str                = NULL;
-        char                 *dir_name               = NULL;
-        char                 *tmp_path               = NULL;
-        int32_t              flags                   = 0;
-        loc_t                tmp_loc                 = {0,};
-        char                 *tmp_stat               = NULL;
-        char                 real_path[PATH_MAX]     = {0,};
-        trash_private_t     *priv               = NULL;
-
-        priv = this->private;
-        GF_VALIDATE_OR_GOTO ("trash", priv, out);
-
-        local = frame->local;
-        GF_VALIDATE_OR_GOTO ("trash", local, out);
-
-        /* Checks whether path is present in trash directory or not */
-
-        if ((op_ret == -1) && (op_errno == ENOENT)) {
-                /* Creating the directory structure here. */
-                tmp_str = gf_strdup (local->newpath);
-                if (!tmp_str) {
-                        gf_log (this->name, GF_LOG_DEBUG, "out of memory");
-                        goto out;
-                }
-                dir_name = dirname (tmp_str);
-
-                tmp_path = gf_strdup (dir_name);
-                if (!tmp_path) {
-                        gf_log (this->name, GF_LOG_DEBUG, "out of memory");
-                        goto out;
-                }
-                loc_copy (&tmp_loc, &local->newloc);
-                tmp_loc.path = gf_strdup (tmp_path);
-                if (!tmp_loc.path) {
-                        gf_log (this->name, GF_LOG_DEBUG, "out of memory");
-                        goto out;
-                }
-                strcpy (real_path, priv->brick_path);
-                remove_trash_path (tmp_path, (frame->root->pid < 0), &tmp_stat);
-                if (tmp_stat)
-                        strcat (real_path, tmp_stat);
-                /* create the directory with proper permissions */
-                STACK_WIND_COOKIE (frame, trash_truncate_mkdir_cbk,
-                                   tmp_path, FIRST_CHILD(this),
-                                   FIRST_CHILD(this)->fops->mkdir,
-                                   &tmp_loc, get_permission(real_path),
-                                   0022, xdata);
-                loc_wipe (&tmp_loc);
-                goto out;
+    trash_local_t *local = NULL;
+    char *tmp_str = NULL;
+    char *dir_name = NULL;
+    char *tmp_path = NULL;
+    int32_t flags = 0;
+    loc_t tmp_loc = {
+        0,
+    };
+    char *tmp_stat = NULL;
+    char real_path[PATH_MAX] = {
+        0,
+    };
+    trash_private_t *priv = NULL;
+
+    priv = this->private;
+    GF_VALIDATE_OR_GOTO("trash", priv, out);
+
+    local = frame->local;
+    GF_VALIDATE_OR_GOTO("trash", local, out);
+
+    TRASH_UNSET_PID(frame, local);
+
+    /* Checks whether path is present in trash directory or not */
+
+    if ((op_ret == -1) && (op_errno == ENOENT)) {
+        /* Creating the directory structure here. */
+        tmp_str = gf_strdup(local->newpath);
+        if (!tmp_str) {
+            gf_log(this->name, GF_LOG_DEBUG, "out of memory");
+            goto out;
         }
+        dir_name = dirname(tmp_str);
 
-        if (op_ret == -1) {
-                /* Let truncate work, but previous copy is not preserved.
-                 * Deleting the newly created copy.
-                 */
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "creation of new file in trash-dir failed, "
-                        "when truncate was called: %s", strerror (op_errno));
-
-                STACK_WIND (frame, trash_common_unwind_buf_cbk,
-                            FIRST_CHILD(this),
-                            FIRST_CHILD(this)->fops->truncate, &local->loc,
-                            local->fop_offset, xdata);
-                goto out;
+        tmp_path = gf_strdup(dir_name);
+        if (!tmp_path) {
+            gf_log(this->name, GF_LOG_DEBUG, "out of memory");
+            goto out;
         }
+        loc_copy(&tmp_loc, &local->newloc);
+        tmp_loc.path = gf_strdup(tmp_path);
+        if (!tmp_loc.path) {
+            gf_log(this->name, GF_LOG_DEBUG, "out of memory");
+            goto out;
+        }
+        strncpy(real_path, priv->brick_path, sizeof(real_path));
+        real_path[sizeof(real_path) - 1] = 0;
+        remove_trash_path(tmp_path, (frame->root->pid < 0), &tmp_stat);
+        if (tmp_stat)
+            strncat(real_path, tmp_stat,
+                    sizeof(real_path) - strlen(real_path) - 1);
+
+        TRASH_SET_PID(frame, local);
+
+        /* create the directory with proper permissions */
+        STACK_WIND_COOKIE(frame, trash_truncate_mkdir_cbk, tmp_path,
+                          FIRST_CHILD(this), FIRST_CHILD(this)->fops->mkdir,
+                          &tmp_loc, get_permission(real_path), 0022, xdata);
+        loc_wipe(&tmp_loc);
+        goto out;
+    }
+
+    if (op_ret == -1) {
+        /* Let truncate work, but previous copy is not preserved.
+         * Deleting the newly created copy.
+         */
+        gf_log(this->name, GF_LOG_DEBUG,
+               "creation of new file in trash-dir failed, "
+               "when truncate was called: %s",
+               strerror(op_errno));
+
+        STACK_WIND(frame, trash_common_unwind_buf_cbk, FIRST_CHILD(this),
+                   FIRST_CHILD(this)->fops->truncate, &local->loc,
+                   local->fop_offset, xdata);
+        goto out;
+    }
 
-        flags = O_RDONLY;
+    fd_bind(fd);
+    flags = O_RDONLY;
 
-        /* fd which represents source file for reading and writing from it */
+    /* fd which represents source file for reading and writing from it */
 
-        local->fd = fd_create (local->loc.inode, frame->root->pid);
+    local->fd = fd_create(local->loc.inode, frame->root->pid);
 
-        STACK_WIND (frame, trash_truncate_open_cbk,  FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->open, &local->loc, flags,
-                    local->fd, 0);
+    STACK_WIND(frame, trash_truncate_open_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->open, &local->loc, flags, local->fd, 0);
 out:
-        if (tmp_str)
-                GF_FREE (tmp_str);
-        if (tmp_path)
-                GF_FREE (tmp_path);
+    if (tmp_str)
+        GF_FREE(tmp_str);
+    if (tmp_path)
+        GF_FREE(tmp_path);
 
-        return 0;
+    return 0;
 }
 
 /**
@@ -1292,290 +1630,316 @@ out:
  * beginning
  */
 int32_t
-trash_truncate_mkdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                          int32_t op_ret, int32_t op_errno, inode_t *inode,
-                          struct iatt *stbuf, struct iatt *preparent,
-                          struct iatt *postparent, dict_t *xdata)
+trash_truncate_mkdir_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                         int32_t op_ret, int32_t op_errno, inode_t *inode,
+                         struct iatt *stbuf, struct iatt *preparent,
+                         struct iatt *postparent, dict_t *xdata)
 {
-        trash_local_t        *local              = NULL;
-        trash_private_t      *priv               = NULL;
-        char                 *tmp_str            = NULL;
-        char                 *tmp_path           = NULL;
-        char                 *tmp_dirname        = NULL;
-        char                 *dir_name           = NULL;
-        char                 *tmp_stat           = NULL;
-        char                 real_path[PATH_MAX] = {0,};
-        size_t               count               = 0;
-        int32_t              flags               = 0;
-        int32_t              loop_count          = 0;
-        int                  i                   = 0;
-        loc_t                tmp_loc             = {0,};
-        int                  ret                 = 0;
-
-        priv = this->private;
-        GF_VALIDATE_OR_GOTO ("trash", priv, out);
-
-        local = frame->local;
-        GF_VALIDATE_OR_GOTO ("trash", local, out);
-
-        loop_count = local->loop_count;
-
-        tmp_str = gf_strdup (local->newpath);
-        if (!tmp_str) {
-                gf_log (this->name, GF_LOG_DEBUG, "out of memory");
-                ret = ENOMEM;
-                goto out;
-        }
-
-        if ((op_ret == -1) && (op_errno == ENOENT)) {
-                tmp_dirname = strchr (tmp_str, '/');
-                while (tmp_dirname) {
-                        count = tmp_dirname - tmp_str;
-                        if (count == 0)
-                                count = 1;
-                        i++;
-                        if (i > loop_count)
-                                break;
-                        tmp_dirname = strchr (tmp_str + count + 1, '/');
-                }
-                tmp_path = gf_memdup (local->newpath, count + 1);
-                if (!tmp_path) {
-                        gf_log (this->name, GF_LOG_DEBUG, "out of memory");
-                        ret = ENOMEM;
-                        goto out;
-                }
-                tmp_path[count] = '\0';
-
-                loc_copy (&tmp_loc, &local->newloc);
-                tmp_loc.path = gf_strdup (tmp_path);
-                if (!tmp_loc.path) {
-                        gf_log (this->name, GF_LOG_DEBUG, "out of memory");
-                        ret = ENOMEM;
-                        goto out;
-                }
-
-                /* Stores the the name of directory to be created */
-                tmp_loc.name = gf_strdup (strrchr(tmp_path, '/') + 1);
-                if (!tmp_loc.name) {
-                        gf_log (this->name, GF_LOG_DEBUG, "out of memory");
-                        ret = ENOMEM;
-                        goto out;
-                }
-                strcpy (real_path, priv->brick_path);
-                remove_trash_path (tmp_path, (frame->root->pid < 0), &tmp_stat);
-                if (tmp_stat)
-                        strcat (real_path, tmp_stat);
-                STACK_WIND_COOKIE (frame, trash_truncate_mkdir_cbk,
-                                   tmp_path, FIRST_CHILD(this),
-                                   FIRST_CHILD(this)->fops->mkdir,
-                                   &tmp_loc, get_permission(real_path),
-                                   0022, xdata);
-                loc_wipe (&tmp_loc);
-                goto out;
-        }
-
-        if (op_ret == 0) {
-                dir_name = dirname (tmp_str);
-                if (strcmp ((char*)cookie, dir_name) == 0) {
-                        flags = O_CREAT|O_EXCL|O_WRONLY;
-                        strcpy (real_path, priv->brick_path);
-                        strcat (real_path, local->origpath);
-                        /* Call create again once directory structure
-                           is created. */
-                        STACK_WIND (frame, trash_truncate_create_cbk,
-                                    FIRST_CHILD(this),
-                                    FIRST_CHILD(this)->fops->create,
-                                    &local->newloc, flags,
-                                    get_permission (real_path),
-                                    0022, local->newfd, xdata);
-                        goto out;
-                }
-        }
-
-        LOCK (&frame->lock);
-        {
-                loop_count = ++local->loop_count;
-        }
-        UNLOCK (&frame->lock);
-
-        tmp_dirname = strchr (tmp_str, '/');
+    trash_local_t *local = NULL;
+    trash_private_t *priv = NULL;
+    char *tmp_str = NULL;
+    char *tmp_path = NULL;
+    char *tmp_dirname = NULL;
+    char *dir_name = NULL;
+    char *tmp_stat = NULL;
+    char real_path[PATH_MAX] = {
+        0,
+    };
+    size_t count = 0;
+    int32_t flags = 0;
+    int32_t loop_count = 0;
+    int i = 0;
+    loc_t tmp_loc = {
+        0,
+    };
+    int ret = 0;
+
+    priv = this->private;
+    GF_VALIDATE_OR_GOTO("trash", priv, out);
+
+    local = frame->local;
+    GF_VALIDATE_OR_GOTO("trash", local, out);
+
+    loop_count = local->loop_count;
+
+    TRASH_UNSET_PID(frame, local);
+
+    tmp_str = gf_strdup(local->newpath);
+    if (!tmp_str) {
+        gf_log(this->name, GF_LOG_DEBUG, "out of memory");
+        ret = ENOMEM;
+        goto out;
+    }
+
+    if ((op_ret == -1) && (op_errno == ENOENT)) {
+        tmp_dirname = strchr(tmp_str, '/');
         while (tmp_dirname) {
-                count = tmp_dirname - tmp_str;
-                if (count == 0)
-                        count = 1;
-                i++;
-                if (i > loop_count)
-                        break;
-                tmp_dirname = strchr (tmp_str + count + 1, '/');
-        }
-        tmp_path = gf_memdup (local->newpath, count + 1);
+            count = tmp_dirname - tmp_str;
+            if (count == 0)
+                count = 1;
+            i++;
+            if (i > loop_count)
+                break;
+            tmp_dirname = strchr(tmp_str + count + 1, '/');
+        }
+        tmp_path = gf_memdup(local->newpath, count + 1);
         if (!tmp_path) {
-                gf_log (this->name, GF_LOG_DEBUG, "out of memory");
-                ret = ENOMEM;
-                goto out;
+            gf_log(this->name, GF_LOG_DEBUG, "out of memory");
+            ret = ENOMEM;
+            goto out;
         }
         tmp_path[count] = '\0';
 
-        loc_copy (&tmp_loc, &local->newloc);
-        tmp_loc.path = gf_strdup (tmp_path);
+        loc_copy(&tmp_loc, &local->newloc);
+        tmp_loc.path = gf_strdup(tmp_path);
         if (!tmp_loc.path) {
-                gf_log (this->name, GF_LOG_DEBUG, "out of memory");
-                ret = ENOMEM;
-                goto out;
+            gf_log(this->name, GF_LOG_DEBUG, "out of memory");
+            ret = ENOMEM;
+            goto out;
         }
 
         /* Stores the the name of directory to be created */
-        tmp_loc.name = gf_strdup (strrchr(tmp_path, '/') + 1);
+        tmp_loc.name = gf_strdup(strrchr(tmp_path, '/') + 1);
         if (!tmp_loc.name) {
-                gf_log (this->name, GF_LOG_DEBUG, "out of memory");
-                goto out;
+            gf_log(this->name, GF_LOG_DEBUG, "out of memory");
+            ret = ENOMEM;
+            goto out;
         }
-
-        strcpy (real_path, priv->brick_path);
-        remove_trash_path (tmp_path, (frame->root->pid < 0), &tmp_stat);
+        strncpy(real_path, priv->brick_path, sizeof(real_path));
+        real_path[sizeof(real_path) - 1] = 0;
+        remove_trash_path(tmp_path, (frame->root->pid < 0), &tmp_stat);
         if (tmp_stat)
-                strcat (real_path, tmp_stat);
-
-        STACK_WIND_COOKIE (frame, trash_truncate_mkdir_cbk, tmp_path,
-                           FIRST_CHILD(this),
-                           FIRST_CHILD(this)->fops->mkdir, &tmp_loc,
-                           get_permission(real_path),
-                           0022, xdata);
+            strncat(real_path, tmp_stat,
+                    sizeof(real_path) - strlen(real_path) - 1);
+
+        TRASH_SET_PID(frame, local);
+
+        STACK_WIND_COOKIE(frame, trash_truncate_mkdir_cbk, tmp_path,
+                          FIRST_CHILD(this), FIRST_CHILD(this)->fops->mkdir,
+                          &tmp_loc, get_permission(real_path), 0022, xdata);
+        loc_wipe(&tmp_loc);
+        goto out;
+    }
+
+    if (op_ret == 0) {
+        dir_name = dirname(tmp_str);
+        if (strcmp((char *)cookie, dir_name) == 0) {
+            flags = O_CREAT | O_EXCL | O_WRONLY;
+            strncpy(real_path, priv->brick_path, sizeof(real_path));
+            real_path[sizeof(real_path) - 1] = 0;
+            strncat(real_path, local->origpath,
+                    sizeof(real_path) - strlen(real_path) - 1);
+            /* Call create again once directory structure
+               is created. */
+
+            TRASH_SET_PID(frame, local);
+
+            STACK_WIND(frame, trash_truncate_create_cbk, FIRST_CHILD(this),
+                       FIRST_CHILD(this)->fops->create, &local->newloc, flags,
+                       get_permission(real_path), 0022, local->newfd, xdata);
+            goto out;
+        }
+    }
+
+    if ((op_ret == -1) && (op_errno != EEXIST)) {
+        gf_log(this->name, GF_LOG_ERROR,
+               "Directory creation failed [%s]. "
+               "Therefore truncating %s without moving the "
+               "original copy to trash directory",
+               strerror(op_errno), local->loc.name);
+        STACK_WIND(frame, trash_common_unwind_buf_cbk, FIRST_CHILD(this),
+                   FIRST_CHILD(this)->fops->truncate, &local->loc,
+                   local->fop_offset, xdata);
+        goto out;
+    }
+
+    LOCK(&frame->lock);
+    {
+        loop_count = ++local->loop_count;
+    }
+    UNLOCK(&frame->lock);
+
+    tmp_dirname = strchr(tmp_str, '/');
+    while (tmp_dirname) {
+        count = tmp_dirname - tmp_str;
+        if (count == 0)
+            count = 1;
+        i++;
+        if (i > loop_count)
+            break;
+        tmp_dirname = strchr(tmp_str + count + 1, '/');
+    }
+    tmp_path = gf_memdup(local->newpath, count + 1);
+    if (!tmp_path) {
+        gf_log(this->name, GF_LOG_DEBUG, "out of memory");
+        ret = ENOMEM;
+        goto out;
+    }
+    tmp_path[count] = '\0';
+
+    loc_copy(&tmp_loc, &local->newloc);
+    tmp_loc.path = gf_strdup(tmp_path);
+    if (!tmp_loc.path) {
+        gf_log(this->name, GF_LOG_DEBUG, "out of memory");
+        ret = ENOMEM;
+        goto out;
+    }
+
+    /* Stores the the name of directory to be created */
+    tmp_loc.name = gf_strdup(strrchr(tmp_path, '/') + 1);
+    if (!tmp_loc.name) {
+        gf_log(this->name, GF_LOG_DEBUG, "out of memory");
+        goto out;
+    }
+
+    strncpy(real_path, priv->brick_path, sizeof(real_path));
+    real_path[sizeof(real_path) - 1] = 0;
+    remove_trash_path(tmp_path, (frame->root->pid < 0), &tmp_stat);
+    if (tmp_stat)
+        strncat(real_path, tmp_stat, sizeof(real_path) - strlen(real_path) - 1);
+
+    TRASH_SET_PID(frame, local);
+
+    STACK_WIND_COOKIE(frame, trash_truncate_mkdir_cbk, tmp_path,
+                      FIRST_CHILD(this), FIRST_CHILD(this)->fops->mkdir,
+                      &tmp_loc, get_permission(real_path), 0022, xdata);
 
 out:
-        if (tmp_str)
-                GF_FREE (tmp_str);
-        if (tmp_path)
-                GF_FREE (tmp_path);
+    if (tmp_str)
+        GF_FREE(tmp_str);
+    if (tmp_path)
+        GF_FREE(tmp_path);
 
-        return ret;
+    return ret;
 }
 
-
 int32_t
-trash_truncate_stat_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                         int32_t op_ret, int32_t op_errno, struct iatt *buf,
-                         dict_t *xdata)
+trash_truncate_stat_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                        int32_t op_ret, int32_t op_errno, struct iatt *buf,
+                        dict_t *xdata)
 {
-        trash_private_t       *priv                   = NULL;
-        trash_local_t         *local                  = NULL;
-        char                  loc_newname[PATH_MAX]   = {0,};
-        int32_t               flags                   = 0;
-        dentry_t              *dir_entry              = NULL;
-        inode_table_t         *table                  = NULL;
-        int                   ret                     = 0;
-
-        priv = this->private;
-        GF_VALIDATE_OR_GOTO ("trash", priv, out);
-
-        local = frame->local;
-        GF_VALIDATE_OR_GOTO ("trash", local, out);
-
-        table = local->loc.inode->table;
-
-        pthread_mutex_lock (&table->lock);
-        {
-                dir_entry = __dentry_search_arbit (local->loc.inode);
-        }
-        pthread_mutex_unlock (&table->lock);
-
-        if (op_ret == -1) {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "fstat on the file failed: %s",
-                        strerror (op_errno));
-
-                TRASH_STACK_UNWIND (truncate, frame, op_ret, op_errno, buf,
-                                NULL, xdata);
-                goto out;
-        }
-
-        /* Only last hardlink will be moved to trash directory */
-        if (buf->ia_nlink > 1) {
-                STACK_WIND (frame, trash_common_unwind_buf_cbk,
-                            FIRST_CHILD(this),
-                            FIRST_CHILD(this)->fops->truncate,
-                            &local->loc, local->fop_offset, xdata);
-                goto out;
-        }
-
-        /**
-         * If the file is too big or if it is extended truncate,
-         * just don't move it to trash directory.
-         */
-        if (buf->ia_size > (priv->max_trash_file_size) ||
-                                buf->ia_size <= local->fop_offset) {
-                gf_log (this->name, GF_LOG_DEBUG, "%s: not moving to trash , "
-                           "having inappropiate file size", local->loc.path);
-
-                STACK_WIND (frame,  trash_common_unwind_buf_cbk,
-                            FIRST_CHILD(this),
-                            FIRST_CHILD(this)->fops->truncate,
-                            &local->loc, local->fop_offset, xdata);
-                goto out;
-        }
-
-        /* Retrives the name of file from path */
-        local->loc.name = gf_strdup (strrchr (local->loc.path, '/'));
-        if (!local->loc.name) {
-                gf_log (this->name, GF_LOG_DEBUG, "out of memory");
-                goto out;
-        }
-
-        /* Stores new path for source file */
-        copy_trash_path (priv->newtrash_dir, (frame->root->pid < 0),
-                                                        local->newpath);
-        strcat (local->newpath, local->loc.path);
-
-        /* append timestamp to file name so that we can avoid
-           name collisions inside trash */
-        append_time_stamp (local->newpath);
-        if (strlen (local->newpath) > PATH_MAX) {
-                STACK_WIND (frame,  trash_common_unwind_buf_cbk,
-                            FIRST_CHILD(this),
-                            FIRST_CHILD(this)->fops->truncate,
-                            &local->loc, local->fop_offset, xdata);
-                goto out;
-        }
-
-        strcpy (loc_newname, local->loc.name);
-        append_time_stamp (loc_newname);
-        /* local->newloc represents old file(file inside trash),
-           where as local->loc represents truncated file. We need
-           to create new inode and fd for new file*/
-        local->newloc.name = gf_strdup (loc_newname);
-        if (!local->newloc.name) {
-                gf_log (this->name, GF_LOG_DEBUG, "out of memory");
-                ret = ENOMEM;
-                goto out;
-        }
-        local->newloc.path = gf_strdup (local->newpath);
-        if (!local->newloc.path) {
-                gf_log (this->name, GF_LOG_DEBUG, "out of memory");
-                ret = ENOMEM;
-                goto out;
-        }
-        local->newloc.inode = inode_new (local->loc.inode->table);
-        local->newfd = fd_create (local->newloc.inode, frame->root->pid);
-
-        /* Creating vaild parent and pargfids for both files */
-
-        local->loc.parent = inode_ref (dir_entry->parent);
-        gf_uuid_copy (local->loc.pargfid, dir_entry->parent->gfid);
-
-        local->newloc.parent = inode_ref (dir_entry->parent);
-        gf_uuid_copy (local->newloc.pargfid, dir_entry->parent->gfid);
-
-        flags = O_CREAT|O_EXCL|O_WRONLY;
-
-        STACK_WIND (frame, trash_truncate_create_cbk,
-                    FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->create,
-                    &local->newloc, flags,
-                    st_mode_from_ia (buf->ia_prot, local->loc.inode->ia_type),
-                    0022, local->newfd, xdata);
+    trash_private_t *priv = NULL;
+    trash_local_t *local = NULL;
+    char loc_newname[PATH_MAX] = {
+        0,
+    };
+    int32_t flags = 0;
+    dentry_t *dir_entry = NULL;
+    inode_table_t *table = NULL;
+    int ret = 0;
+
+    priv = this->private;
+    GF_VALIDATE_OR_GOTO("trash", priv, out);
+
+    local = frame->local;
+    GF_VALIDATE_OR_GOTO("trash", local, out);
+
+    table = local->loc.inode->table;
+
+    pthread_mutex_lock(&table->lock);
+    {
+        dir_entry = __dentry_search_arbit(local->loc.inode);
+    }
+    pthread_mutex_unlock(&table->lock);
+
+    if (op_ret == -1) {
+        gf_log(this->name, GF_LOG_DEBUG, "fstat on the file failed: %s",
+               strerror(op_errno));
+
+        TRASH_STACK_UNWIND(truncate, frame, op_ret, op_errno, buf, NULL, xdata);
+        goto out;
+    }
+
+    /* Only last hardlink will be moved to trash directory */
+    if (buf->ia_nlink > 1) {
+        STACK_WIND(frame, trash_common_unwind_buf_cbk, FIRST_CHILD(this),
+                   FIRST_CHILD(this)->fops->truncate, &local->loc,
+                   local->fop_offset, xdata);
+        goto out;
+    }
+
+    /**
+     * If the file is too big or if it is extended truncate,
+     * just don't move it to trash directory.
+     */
+    if (buf->ia_size > (priv->max_trash_file_size) ||
+        buf->ia_size <= local->fop_offset) {
+        gf_log(this->name, GF_LOG_DEBUG,
+               "%s: file is too large to move to trash", local->loc.path);
+
+        STACK_WIND(frame, trash_common_unwind_buf_cbk, FIRST_CHILD(this),
+                   FIRST_CHILD(this)->fops->truncate, &local->loc,
+                   local->fop_offset, xdata);
+        goto out;
+    }
+
+    /* Retrieves the name of file from path */
+    local->loc.name = gf_strdup(strrchr(local->loc.path, '/'));
+    if (!local->loc.name) {
+        gf_log(this->name, GF_LOG_DEBUG, "out of memory");
+        goto out;
+    }
+
+    /* Stores new path for source file */
+    copy_trash_path(priv->newtrash_dir, (frame->root->pid < 0), local->newpath,
+                    sizeof(local->newpath));
+    strncat(local->newpath, local->loc.path,
+            sizeof(local->newpath) - strlen(local->newpath) - 1);
+
+    /* append timestamp to file name so that we can avoid
+       name collisions inside trash */
+    append_time_stamp(local->newpath, sizeof(local->newpath));
+    if (strlen(local->newpath) > PATH_MAX) {
+        STACK_WIND(frame, trash_common_unwind_buf_cbk, FIRST_CHILD(this),
+                   FIRST_CHILD(this)->fops->truncate, &local->loc,
+                   local->fop_offset, xdata);
+        goto out;
+    }
+
+    strncpy(loc_newname, local->loc.name, sizeof(loc_newname));
+    loc_newname[sizeof(loc_newname) - 1] = 0;
+    append_time_stamp(loc_newname, sizeof(loc_newname));
+    /* local->newloc represents old file(file inside trash),
+       where as local->loc represents truncated file. We need
+       to create new inode and fd for new file*/
+    local->newloc.name = gf_strdup(loc_newname);
+    if (!local->newloc.name) {
+        gf_log(this->name, GF_LOG_DEBUG, "out of memory");
+        ret = ENOMEM;
+        goto out;
+    }
+    local->newloc.path = gf_strdup(local->newpath);
+    if (!local->newloc.path) {
+        gf_log(this->name, GF_LOG_DEBUG, "out of memory");
+        ret = ENOMEM;
+        goto out;
+    }
+    local->newloc.inode = inode_new(local->loc.inode->table);
+    local->newfd = fd_create(local->newloc.inode, frame->root->pid);
+
+    /* Creating valid parent and pargfids for both files */
+
+    if (dir_entry == NULL) {
+        ret = EINVAL;
+        goto out;
+    }
+    local->loc.parent = inode_ref(dir_entry->parent);
+    gf_uuid_copy(local->loc.pargfid, dir_entry->parent->gfid);
+
+    local->newloc.parent = inode_ref(dir_entry->parent);
+    gf_uuid_copy(local->newloc.pargfid, dir_entry->parent->gfid);
+
+    flags = O_CREAT | O_EXCL | O_WRONLY;
+
+    TRASH_SET_PID(frame, local);
+
+    STACK_WIND(frame, trash_truncate_create_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->create, &local->newloc, flags,
+               st_mode_from_ia(buf->ia_prot, local->loc.inode->ia_type), 0022,
+               local->newfd, xdata);
 
 out:
-        return ret;
+    return ret;
 }
 
 /**
@@ -1583,89 +1947,85 @@ out:
  * like text editors etc..
  */
 int32_t
-trash_truncate (call_frame_t *frame, xlator_t *this, loc_t *loc,
-                off_t offset, dict_t *xdata)
+trash_truncate(call_frame_t *frame, xlator_t *this, loc_t *loc, off_t offset,
+               dict_t *xdata)
 {
-        trash_private_t        *priv           = NULL;
-        trash_local_t          *local          = NULL;
-        int32_t                match           = 0;
-        char                   *pathbuf        = NULL;
-        int                    ret             = 0;
-
-        priv  = this->private;
-        GF_VALIDATE_OR_GOTO ("trash", priv, out);
-        /* If trash is not active or not enabled through cli, then
-         * we bypass and wind back
+    trash_private_t *priv = NULL;
+    trash_local_t *local = NULL;
+    int32_t match = 0;
+    char *pathbuf = NULL;
+    int ret = 0;
+
+    priv = this->private;
+    GF_VALIDATE_OR_GOTO("trash", priv, out);
+    /* If trash is not active or not enabled through cli, then
+     * we bypass and wind back
+     */
+    if (!priv->state) {
+        STACK_WIND(frame, trash_common_unwind_buf_cbk, FIRST_CHILD(this),
+                   FIRST_CHILD(this)->fops->truncate, loc, offset, xdata);
+        goto out;
+    }
+
+    /* The files removed by gluster operations such as self-heal,
+       should moved to trash directory, but files by client should
+       not moved */
+    if ((frame->root->pid < 0) && !priv->internal) {
+        STACK_WIND(frame, trash_common_unwind_buf_cbk, FIRST_CHILD(this),
+                   FIRST_CHILD(this)->fops->truncate, loc, offset, xdata);
+        goto out;
+    }
+    /* This will be more accurate */
+    inode_path(loc->inode, NULL, &pathbuf);
+
+    /* Checks whether file is in trash directory or eliminate path.
+     * In all such cases it does not move to trash directory,
+     * truncate will be performed
+     */
+    match = check_whether_eliminate_path(priv->eliminate, pathbuf);
+
+    if ((strncmp(pathbuf, priv->newtrash_dir, strlen(priv->newtrash_dir)) ==
+         0) ||
+        (match)) {
+        if (match) {
+            gf_log(this->name, GF_LOG_DEBUG,
+                   "%s: file not moved to trash as per option "
+                   "'eliminate path'",
+                   loc->path);
+        }
+
+        /* Trying to truncate from the trash-dir. So do the
+         * actual truncate without moving to trash-dir.
          */
-        if (!priv->state) {
-                STACK_WIND (frame, trash_common_unwind_buf_cbk,
-                            FIRST_CHILD(this),
-                            FIRST_CHILD(this)->fops->truncate, loc,
-                            offset, xdata);
-                goto out;
-        }
-
-        /* The files removed by gluster operations such as self-heal,
-           should moved to trash directory, but files by client should
-           not moved */
-        if ((frame->root->pid < 0) && !priv->internal) {
-                STACK_WIND (frame, trash_common_unwind_buf_cbk,
-                            FIRST_CHILD(this),
-                            FIRST_CHILD(this)->fops->truncate, loc,
-                            offset, xdata);
-                goto out;
-        }
-        /* This will be more accurate */
-        inode_path(loc->inode, NULL, &pathbuf);
+        STACK_WIND(frame, trash_common_unwind_buf_cbk, FIRST_CHILD(this),
+                   FIRST_CHILD(this)->fops->truncate, loc, offset, xdata);
+        goto out;
+    }
 
-        /* Checks whether file is in trash directory or eliminate path.
-         * In all such cases it does not move to trash directory,
-         * truncate will be performed
-         */
-        match = check_whether_eliminate_path (priv->eliminate, pathbuf);
-
-        if ((strncmp (pathbuf, priv->newtrash_dir,
-                      strlen (priv->newtrash_dir)) == 0) || (match)) {
-                if (match) {
-                        gf_log (this->name, GF_LOG_DEBUG,
-                                "%s: file not moved to trash as per option "
-                                "'eliminate path'", loc->path);
-                }
-
-                /* Trying to truncate from the trash-dir. So do the
-                 * actual truncate without moving to trash-dir.
-                 */
-                STACK_WIND (frame, trash_common_unwind_buf_cbk,
-                            FIRST_CHILD(this),
-                            FIRST_CHILD(this)->fops->truncate, loc, offset,
-                            xdata);
-                goto out;
-        }
+    LOCK_INIT(&frame->lock);
 
-        LOCK_INIT (&frame->lock);
+    local = mem_get0(this->local_pool);
+    if (!local) {
+        gf_log(this->name, GF_LOG_DEBUG, "out of memory");
+        TRASH_STACK_UNWIND(truncate, frame, -1, ENOMEM, NULL, NULL, xdata);
+        ret = ENOMEM;
+        goto out;
+    }
 
-        local = mem_get0 (this->local_pool);
-        if (!local) {
-                gf_log (this->name, GF_LOG_DEBUG, "out of memory");
-                TRASH_STACK_UNWIND (truncate, frame, -1, ENOMEM, NULL, NULL,
-                                    xdata);
-                ret = ENOMEM;
-                goto out;
-        }
+    strncpy(local->origpath, pathbuf, sizeof(local->origpath));
+    local->origpath[sizeof(local->origpath) - 1] = 0;
 
-        loc_copy (&local->loc, loc);
-        local->loc.path = pathbuf;
-        local->fop_offset = offset;
+    loc_copy(&local->loc, loc);
+    local->loc.path = pathbuf;
+    local->fop_offset = offset;
 
-        frame->local = local;
+    frame->local = local;
 
-        STACK_WIND (frame, trash_truncate_stat_cbk,
-                    FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->stat, loc,
-                    xdata);
+    STACK_WIND(frame, trash_truncate_stat_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->stat, loc, xdata);
 
 out:
-        return ret;
+    return ret;
 }
 
 /**
@@ -1674,95 +2034,91 @@ out:
  * other than that it also called by Rebalance operation
  */
 int32_t
-trash_ftruncate (call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
-                 dict_t *xdata)
+trash_ftruncate(call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
+                dict_t *xdata)
 {
-        trash_private_t      *priv           = NULL;
-        trash_local_t        *local          = NULL;/* file inside trash */
-        char                 *pathbuf        = NULL;/* path of file from fd */
-        int32_t              retval          = 0;
-        int32_t              match           = 0;
-        int                  ret             = 0;
-
-        priv = this->private;
-        GF_VALIDATE_OR_GOTO ("trash", priv, out);
-        /* If trash is not active or not enabled through cli, then
-         * we bypass and wind back
-         */
-        if (!priv->state) {
-                STACK_WIND (frame, trash_common_unwind_buf_cbk,
-                            FIRST_CHILD(this),
-                            FIRST_CHILD(this)->fops->ftruncate, fd,
-                            offset, xdata);
-                goto out;
-        }
-
-        /* The files removed by gluster operations such as self-heal,
-         * should moved to trash directory, but files by client
-         * should not moved
-         */
-        if ((frame->root->pid < 0) && !priv->internal) {
-                STACK_WIND (frame, trash_common_unwind_buf_cbk,
-                            FIRST_CHILD(this),
-                            FIRST_CHILD(this)->fops->ftruncate, fd,
-                            offset, xdata);
-                goto out;
-        }
-        /* This will be more accurate */
-        retval = inode_path (fd->inode, NULL, &pathbuf);
-
-        /* Checking  the eliminate path */
-
-        /* Checks whether file is trash directory or eliminate path or
-         * invalid fd. In all such cases it does not move to trash directory,
-         * ftruncate will be performed
-         */
-        match = check_whether_eliminate_path (priv->eliminate, pathbuf);
-        if ((strncmp (pathbuf, priv->newtrash_dir,
-                      strlen (priv->newtrash_dir)) == 0) || match ||
-                      !retval) {
-
-                if (match) {
-                        gf_log (this->name, GF_LOG_DEBUG,
-                                "%s: file matches eliminate path, "
-                                "not moved to trash", pathbuf);
-                }
-
-                /* Trying to ftruncate from the trash-dir. So do the
-                 * actual ftruncate without moving to trash-dir
-                 */
-                STACK_WIND (frame, trash_common_unwind_buf_cbk,
-                            FIRST_CHILD(this),
-                            FIRST_CHILD(this)->fops->ftruncate,
-                            fd, offset, xdata);
-                goto out;
-        }
-
-        local = mem_get0 (this->local_pool);
-        if (!local) {
-                gf_log (this->name, GF_LOG_DEBUG, "out of memory");
-                TRASH_STACK_UNWIND (ftruncate, frame, -1, ENOMEM, NULL,
-                                    NULL, xdata);
-                ret     = -1;
-                goto out;
-        }
-
-        /* To convert fd to location */
-        frame->local=local;
-
-        local->loc.path  = pathbuf;
-        local->loc.inode = inode_ref (fd->inode);
-        gf_uuid_copy (local->loc.gfid, local->loc.inode->gfid);
-
-        local->fop_offset = offset;
-
-        /* Else remains same to truncate code, so from here flow goes
-         * to truncate_stat
+    trash_private_t *priv = NULL;
+    trash_local_t *local = NULL; /* file inside trash */
+    char *pathbuf = NULL;        /* path of file from fd */
+    int32_t retval = 0;
+    int32_t match = 0;
+    int ret = 0;
+
+    priv = this->private;
+    GF_VALIDATE_OR_GOTO("trash", priv, out);
+    /* If trash is not active or not enabled through cli, then
+     * we bypass and wind back
+     */
+    if (!priv->state) {
+        STACK_WIND(frame, trash_common_unwind_buf_cbk, FIRST_CHILD(this),
+                   FIRST_CHILD(this)->fops->ftruncate, fd, offset, xdata);
+        goto out;
+    }
+
+    /* The files removed by gluster operations such as self-heal,
+     * should moved to trash directory, but files by client
+     * should not moved
+     */
+    if ((frame->root->pid < 0) && !priv->internal) {
+        STACK_WIND(frame, trash_common_unwind_buf_cbk, FIRST_CHILD(this),
+                   FIRST_CHILD(this)->fops->ftruncate, fd, offset, xdata);
+        goto out;
+    }
+    /* This will be more accurate */
+    retval = inode_path(fd->inode, NULL, &pathbuf);
+
+    /* Checking  the eliminate path */
+
+    /* Checks whether file is trash directory or eliminate path or
+     * invalid fd. In all such cases it does not move to trash directory,
+     * ftruncate will be performed
+     */
+    match = check_whether_eliminate_path(priv->eliminate, pathbuf);
+    if ((strncmp(pathbuf, priv->newtrash_dir, strlen(priv->newtrash_dir)) ==
+         0) ||
+        match || !retval) {
+        if (match) {
+            gf_log(this->name, GF_LOG_DEBUG,
+                   "%s: file matches eliminate path, "
+                   "not moved to trash",
+                   pathbuf);
+        }
+
+        /* Trying to ftruncate from the trash-dir. So do the
+         * actual ftruncate without moving to trash-dir
          */
-        STACK_WIND (frame, trash_truncate_stat_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->fstat, fd, xdata);
+        STACK_WIND(frame, trash_common_unwind_buf_cbk, FIRST_CHILD(this),
+                   FIRST_CHILD(this)->fops->ftruncate, fd, offset, xdata);
+        goto out;
+    }
+
+    local = mem_get0(this->local_pool);
+    if (!local) {
+        gf_log(this->name, GF_LOG_DEBUG, "out of memory");
+        TRASH_STACK_UNWIND(ftruncate, frame, -1, ENOMEM, NULL, NULL, xdata);
+        ret = -1;
+        goto out;
+    }
+
+    strncpy(local->origpath, pathbuf, sizeof(local->origpath));
+    local->origpath[sizeof(local->origpath) - 1] = 0;
+
+    /* To convert fd to location */
+    frame->local = local;
+
+    local->loc.path = pathbuf;
+    local->loc.inode = inode_ref(fd->inode);
+    gf_uuid_copy(local->loc.gfid, local->loc.inode->gfid);
+
+    local->fop_offset = offset;
+
+    /* Else remains same to truncate code, so from here flow goes
+     * to truncate_stat
+     */
+    STACK_WIND(frame, trash_truncate_stat_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->fstat, fd, xdata);
 out:
-        return ret;
+    return ret;
 }
 
 /**
@@ -1770,31 +2126,32 @@ out:
  * trash directory in the mount by the user
  */
 int32_t
-trash_mkdir (call_frame_t *frame, xlator_t *this,
-             loc_t *loc, mode_t mode, mode_t umask, dict_t *xdata)
+trash_mkdir(call_frame_t *frame, xlator_t *this, loc_t *loc, mode_t mode,
+            mode_t umask, dict_t *xdata)
 {
-        int32_t               op_ret      = 0;
-        int32_t               op_errno    = 0;
-        trash_private_t       *priv       = NULL;
-
-        priv = this->private;
-        GF_VALIDATE_OR_GOTO ("trash", priv, out);
-
-        if (!check_whether_trash_directory (loc->path, priv->newtrash_dir)) {
-                gf_log (this->name, GF_LOG_WARNING,
-                        "mkdir issued on %s, which is not permitted",
-                        priv->newtrash_dir);
-                op_errno = EPERM;
-                op_ret = -1;
-
-                STACK_UNWIND_STRICT (mkdir, frame, op_ret, op_errno,
-                             NULL, NULL, NULL, NULL, xdata);
-        } else {
-                STACK_WIND (frame, trash_common_mkdir_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->mkdir, loc, mode, umask, xdata);
-        }
+    int32_t op_ret = 0;
+    int32_t op_errno = 0;
+    trash_private_t *priv = NULL;
+
+    priv = this->private;
+    GF_VALIDATE_OR_GOTO("trash", priv, out);
+
+    if (!check_whether_op_permitted(priv, loc)) {
+        gf_log(this->name, GF_LOG_WARNING,
+               "mkdir issued on %s, which is not permitted",
+               priv->newtrash_dir);
+        op_errno = EPERM;
+        op_ret = -1;
+
+        STACK_UNWIND_STRICT(mkdir, frame, op_ret, op_errno, NULL, NULL, NULL,
+                            NULL, xdata);
+    } else {
+        STACK_WIND(frame, trash_common_mkdir_cbk, FIRST_CHILD(this),
+                   FIRST_CHILD(this)->fops->mkdir, loc, mode, umask, xdata);
+    }
+
 out:
-        return 0;
+    return 0;
 }
 
 /**
@@ -1802,31 +2159,32 @@ out:
  * of trash directory in the mount by the user
  */
 int
-trash_rename (call_frame_t *frame, xlator_t *this,
-              loc_t *oldloc, loc_t *newloc, dict_t *xdata)
+trash_rename(call_frame_t *frame, xlator_t *this, loc_t *oldloc, loc_t *newloc,
+             dict_t *xdata)
 {
-        int32_t               op_ret      = 0;
-        int32_t               op_errno    = 0;
-        trash_private_t       *priv       = NULL;
-
-        priv = this->private;
-        GF_VALIDATE_OR_GOTO ("trash", priv, out);
-
-        if (!check_whether_trash_directory (oldloc->path, priv->newtrash_dir)) {
-                gf_log (this->name, GF_LOG_WARNING,
-                        "rename issued on %s, which is not permitted",
-                        priv->newtrash_dir);
-                op_errno = EPERM;
-                op_ret = -1;
-
-                STACK_UNWIND_STRICT (rename, frame, op_ret, op_errno, NULL,
-                                     NULL, NULL, NULL, NULL, xdata);
-        } else {
-                STACK_WIND (frame, trash_common_rename_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->rename, oldloc, newloc, xdata);
-        }
+    int32_t op_ret = 0;
+    int32_t op_errno = 0;
+    trash_private_t *priv = NULL;
+
+    priv = this->private;
+    GF_VALIDATE_OR_GOTO("trash", priv, out);
+
+    if (!check_whether_op_permitted(priv, oldloc)) {
+        gf_log(this->name, GF_LOG_WARNING,
+               "rename issued on %s, which is not permitted",
+               priv->newtrash_dir);
+        op_errno = EPERM;
+        op_ret = -1;
+
+        STACK_UNWIND_STRICT(rename, frame, op_ret, op_errno, NULL, NULL, NULL,
+                            NULL, NULL, xdata);
+    } else {
+        STACK_WIND(frame, trash_common_rename_cbk, FIRST_CHILD(this),
+                   FIRST_CHILD(this)->fops->rename, oldloc, newloc, xdata);
+    }
+
 out:
-        return 0;
+    return 0;
 }
 
 /**
@@ -1834,187 +2192,128 @@ out:
  * trash directory in the mount by the user
  */
 int32_t
-trash_rmdir (call_frame_t *frame, xlator_t *this,
-             loc_t *loc, int flags, dict_t *xdata)
+trash_rmdir(call_frame_t *frame, xlator_t *this, loc_t *loc, int flags,
+            dict_t *xdata)
 {
-        int32_t               op_ret      = 0;
-        int32_t               op_errno    = 0;
-        trash_private_t       *priv       = NULL;
-
-        priv = this->private;
-        GF_VALIDATE_OR_GOTO ("trash", priv, out);
-
-        if (!check_whether_trash_directory (loc->path, priv->newtrash_dir)) {
-                gf_log (this->name, GF_LOG_WARNING,
-                        "rmdir issued on %s, which is not permitted",
-                        priv->newtrash_dir);
-                op_errno = EPERM;
-                op_ret = -1;
-
-                STACK_UNWIND_STRICT (rmdir, frame, op_ret, op_errno,
-                             NULL, NULL, xdata);
-        } else {
-                STACK_WIND (frame, trash_common_rmdir_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->rmdir, loc, flags, xdata);
-        }
+    int32_t op_ret = 0;
+    int32_t op_errno = 0;
+    trash_private_t *priv = NULL;
+
+    priv = this->private;
+    GF_VALIDATE_OR_GOTO("trash", priv, out);
+
+    if (!check_whether_op_permitted(priv, loc)) {
+        gf_log(this->name, GF_LOG_WARNING,
+               "rmdir issued on %s, which is not permitted",
+               priv->newtrash_dir);
+        op_errno = EPERM;
+        op_ret = -1;
+
+        STACK_UNWIND_STRICT(rmdir, frame, op_ret, op_errno, NULL, NULL, xdata);
+    } else {
+        STACK_WIND(frame, trash_common_rmdir_cbk, FIRST_CHILD(this),
+                   FIRST_CHILD(this)->fops->rmdir, loc, flags, xdata);
+    }
+
 out:
-        return 0;
+    return 0;
 }
 
 /**
- * Volume set option is handled by the reconfigure funtion.
+ * Volume set option is handled by the reconfigure function.
  * Here we checks whether each option is set or not ,if it
  * sets then corresponding modifciations will be made
  */
 int
-reconfigure (xlator_t *this, dict_t *options)
+reconfigure(xlator_t *this, dict_t *options)
 {
-        uint64_t              max_fsize           = 0;
-        int                   ret                 = 0;
-        char                  *tmp                = NULL;
-        char                  *tmp_str            = NULL;
-        trash_private_t       *priv               = NULL;
-        loc_t                 old_loc             = {0, };
-        loc_t                 new_loc             = {0, };
-        call_frame_t          *frame              = NULL;
-        char                  trash_dir[PATH_MAX] = {0,};
-
-        priv = this->private;
-        GF_VALIDATE_OR_GOTO ("trash", priv, out);
-
-        GF_OPTION_RECONF ("trash", priv->state, options, bool, out);
-
-        GF_OPTION_RECONF ("trash-dir", tmp, options, str, out);
-        if (tmp) {
-                sprintf(trash_dir, "/%s/", tmp);
-                if (strcmp(priv->newtrash_dir, trash_dir) != 0) {
-
-                        /* When user set a new name for trash directory, trash
-                        * xlator will perform a rename operation on old trash
-                        * directory to the new one using a STACK_WIND from here.
-                        * This option can be configured only when volume is in
-                        * started state
-                        */
-
-                        GF_FREE (priv->newtrash_dir);
-
-                        priv->newtrash_dir = gf_strdup (trash_dir);
-                        if (!priv->newtrash_dir) {
-                                ret = ENOMEM;
-                                gf_log (this->name, GF_LOG_DEBUG,
-                                                "out of memory");
-                                goto out;
-                        }
-                        gf_log (this->name, GF_LOG_DEBUG,
-                                "Renaming %s -> %s from reconfigure",
-                                priv->oldtrash_dir, priv->newtrash_dir);
-
-                        if (!priv->newtrash_dir) {
-                                gf_log (this->name, GF_LOG_DEBUG,
-                                                "out of memory");
-                                ret = ENOMEM;
-                                goto out;
-                        }
-                        frame = create_frame (this, this->ctx->pool);
-
-                        /* assign new location values to new_loc members */
-                        gf_uuid_copy (new_loc.gfid, trash_gfid);
-                        gf_uuid_copy (new_loc.pargfid, root_gfid);
-                        ret = extract_trash_directory (priv->newtrash_dir,
-                                                                &new_loc.name);
-                        if (ret) {
-                                gf_log (this->name, GF_LOG_DEBUG,
-                                                "out of memory");
-                                goto out;
-                        }
-                        new_loc.path = gf_strdup (priv->newtrash_dir);
-                        if (!new_loc.path) {
-                                ret = ENOMEM;
-                                gf_log (this->name, GF_LOG_DEBUG,
-                                                "out of memory");
-                                goto out;
-                        }
-
-                        /* assign old location values to old_loc members */
-                        gf_uuid_copy (old_loc.gfid, trash_gfid);
-                        gf_uuid_copy (old_loc.pargfid, root_gfid);
-                        ret = extract_trash_directory (priv->oldtrash_dir,
-                                                                &old_loc.name);
-                        if (ret) {
-                                gf_log (this->name, GF_LOG_DEBUG,
-                                                "out of memory");
-                                goto out;
-                        }
-                        old_loc.path = gf_strdup (priv->oldtrash_dir);
-                        if (!old_loc.path) {
-                                ret = ENOMEM;
-                                gf_log (this->name, GF_LOG_DEBUG,
-                                                "out of memory");
-                                goto out;
-                        }
-
-                        old_loc.inode = inode_ref (priv->trash_inode);
-                        gf_uuid_copy(old_loc.inode->gfid, old_loc.gfid);
-
-                        STACK_WIND (frame, trash_reconf_rename_cbk,
-                                    FIRST_CHILD(this),
-                                    FIRST_CHILD(this)->fops->rename,
-                                    &old_loc, &new_loc, options);
-                        if (priv->oldtrash_dir)
-                                GF_FREE (priv->oldtrash_dir);
-
-                        priv->oldtrash_dir = gf_strdup(priv->newtrash_dir);
-                        if (!priv->oldtrash_dir) {
-                                ret = ENOMEM;
-                                gf_log (this->name, GF_LOG_DEBUG,
-                                                "out of memory");
-                                goto out;
-                        }
-                }
-        }
-        tmp = NULL;
-
-        GF_OPTION_RECONF ("trash-internal-op", priv->internal, options,
-                                               bool, out);
-
-        GF_OPTION_RECONF ("trash-max-filesize", max_fsize, options,
-                                                size_uint64, out);
-        if (max_fsize) {
-                if (max_fsize > GF_ALLOWED_MAX_FILE_SIZE) {
-                        gf_log (this->name, GF_LOG_DEBUG,
-                                "Size specified for max-size(in MB) is too "
-                                "large so using 1GB as max-size (NOT IDEAL)");
-                        priv->max_trash_file_size = GF_ALLOWED_MAX_FILE_SIZE;
-                } else
-                        priv->max_trash_file_size = max_fsize;
-                gf_log (this->name, GF_LOG_DEBUG, "%"GF_PRI_SIZET" max-size",
-                        priv->max_trash_file_size);
-        }
-        GF_OPTION_RECONF ("trash-eliminate-path", tmp, options, str, out);
-        if (!tmp) {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "no option specified for 'eliminate', using NULL");
-        } else {
-                if (priv->eliminate)
-                        wipe_eliminate_path (&priv->eliminate);
+    uint64_t max_fsize = 0;
+    int ret = 0;
+    char *tmp = NULL;
+    char *tmp_str = NULL;
+    trash_private_t *priv = NULL;
+    char trash_dir[PATH_MAX] = {
+        0,
+    };
+
+    priv = this->private;
+
+    GF_VALIDATE_OR_GOTO("trash", priv, out);
+
+    GF_OPTION_RECONF("trash-internal-op", priv->internal, options, bool, out);
+    GF_OPTION_RECONF("trash-dir", tmp, options, str, out);
+
+    GF_OPTION_RECONF("trash", priv->state, options, bool, out);
+
+    if (priv->state) {
+        ret = create_or_rename_trash_directory(this);
 
-                tmp_str = gf_strdup (tmp);
-                if (!tmp_str) {
-                        gf_log (this->name, GF_LOG_DEBUG, "out of memory");
-                        ret = ENOMEM;
-                        goto out;
-                }
-                ret = store_eliminate_path (tmp_str, &priv->eliminate);
+        if (tmp)
+            sprintf(trash_dir, "/%s/", tmp);
+        else
+            sprintf(trash_dir, "%s", priv->oldtrash_dir);
+
+        if (strcmp(priv->newtrash_dir, trash_dir) != 0) {
+            /* When user set a new name for trash directory, trash
+             * xlator will perform a rename operation on old trash
+             * directory to the new one using a STACK_WIND from here.
+             * This option can be configured only when volume is in
+             * started state
+             */
+
+            GF_FREE(priv->newtrash_dir);
+
+            priv->newtrash_dir = gf_strdup(trash_dir);
+            if (!priv->newtrash_dir) {
+                ret = ENOMEM;
+                gf_log(this->name, GF_LOG_DEBUG, "out of memory");
+                goto out;
+            }
+            gf_log(this->name, GF_LOG_DEBUG,
+                   "Renaming %s -> %s from reconfigure", priv->oldtrash_dir,
+                   priv->newtrash_dir);
 
+            if (!priv->newtrash_dir) {
+                gf_log(this->name, GF_LOG_DEBUG, "out of memory");
+                ret = ENOMEM;
+                goto out;
+            }
+            ret = rename_trash_directory(this);
+        }
+
+        if (priv->internal) {
+            ret = create_internalop_directory(this);
+        }
+    }
+    tmp = NULL;
+
+    GF_OPTION_RECONF("trash-max-filesize", max_fsize, options, size_uint64,
+                     out);
+    if (max_fsize) {
+        priv->max_trash_file_size = max_fsize;
+        gf_log(this->name, GF_LOG_DEBUG, "%" GF_PRI_SIZET " max-size",
+               priv->max_trash_file_size);
+    }
+    GF_OPTION_RECONF("trash-eliminate-path", tmp, options, str, out);
+    if (!tmp) {
+        gf_log(this->name, GF_LOG_DEBUG,
+               "no option specified for 'eliminate', using NULL");
+    } else {
+        if (priv->eliminate)
+            wipe_eliminate_path(&priv->eliminate);
+
+        tmp_str = gf_strdup(tmp);
+        if (!tmp_str) {
+            gf_log(this->name, GF_LOG_DEBUG, "out of memory");
+            ret = ENOMEM;
+            goto out;
         }
+        ret = store_eliminate_path(tmp_str, &priv->eliminate);
+    }
 
 out:
-        if (tmp_str)
-                GF_FREE (tmp_str);
-        loc_wipe (&new_loc);
-        loc_wipe (&old_loc);
 
-        return ret;
+    return ret;
 }
 
 /**
@@ -2022,414 +2321,333 @@ out:
  * using STACK_WIND only when posix xlator is up
  */
 int
-notify (xlator_t *this, int event, void *data, ...)
+notify(xlator_t *this, int event, void *data, ...)
 {
-        trash_private_t       *priv      = NULL;
-        dict_t                *dict      = NULL;
-        int                   ret        = 0;
-        uuid_t                *tgfid_ptr = NULL;
-        loc_t                 loc        = {0, };
-        loc_t                 old_loc    = {0, };
-        call_frame_t          *frame     = NULL;
-
-        priv = this->private;
-        GF_VALIDATE_OR_GOTO ("trash", priv, out);
-
-        /* Check whether posix is up not */
-        if (event == GF_EVENT_CHILD_UP) {
-                frame = create_frame(this, this->ctx->pool);
-                dict = dict_new ();
-                if (!dict) {
-                        ret = ENOMEM;
-                        goto out;
-                }
-                priv->trash_itable = inode_table_new (0, this);
-
-                /* Here there is two possiblities ,if trash directory already
-                 * exist ,then we need to perform a rename operation on the
-                 * old one. Otherwise, we need to create the trash directory
-                 * For both, we need to pass location variable, gfid of parent
-                 * and a frame for calling STACK_WIND.The location variable
-                 * requires name,path,gfid and inode
-                 */
-                if (!priv->oldtrash_dir) {
-                        loc.inode = inode_new (priv->trash_itable);
-                        gf_uuid_copy (loc.gfid, trash_gfid);
-
-                        gf_log (this->name, GF_LOG_DEBUG, "nameless lookup for"
-                                           "old trash directory");
-                        STACK_WIND (frame, trash_notify_lookup_cbk,
-                                    FIRST_CHILD(this),
-                                    FIRST_CHILD(this)->fops->lookup,
-                                    &loc, dict);
-                        gf_log (this->name, GF_LOG_DEBUG, "old_trash_dir %s",
-                                        priv->oldtrash_dir);
-                        loc_wipe (&loc);
-                }
-
-                if (strcmp (priv->oldtrash_dir, priv->newtrash_dir) == 0) {
-                        gf_log (this->name, GF_LOG_DEBUG, "Creating trash "
-                                           "directory %s from notify",
-                                           priv->newtrash_dir);
-
-                        tgfid_ptr = GF_CALLOC (1, sizeof(uuid_t),
-                                                  gf_common_mt_uuid_t);
-                        if (!tgfid_ptr) {
-                                ret = ENOMEM;
-                                goto out;
-                        }
-                        gf_uuid_copy (*tgfid_ptr, trash_gfid);
-
-                        gf_uuid_copy (loc.gfid, trash_gfid);
-                        gf_uuid_copy (loc.pargfid, root_gfid);
-                        ret = extract_trash_directory (priv->newtrash_dir,
-                                                                &loc.name);
-                        if (ret) {
-                                gf_log (this->name, GF_LOG_DEBUG,
-                                                "out of memory");
-                                goto out;
-                        }
-                        loc.path = gf_strdup (priv->newtrash_dir);
-                        if (!loc.path) {
-                                gf_log (this->name, GF_LOG_DEBUG,
-                                                "out of memory");
-                                ret = ENOMEM;
-                                goto out;
-                        }
-
-                        priv->trash_inode = inode_new (priv->trash_itable);
-                        priv->trash_inode->ia_type = IA_IFDIR;
-                        loc.inode = inode_ref (priv->trash_inode);
-
-                        /* Fixed gfid is set for trash directory with
-                         * this function
-                         */
-                        ret = dict_set_dynptr (dict, "gfid-req", tgfid_ptr,
-                                              sizeof (uuid_t));
-                        if (ret) {
-                                gf_log (this->name, GF_LOG_ERROR,
-                                        "setting key gfid-req failed");
-                                goto out;
-                        }
-
-                        /* The mkdir call for creating trash directory */
-                        STACK_WIND (frame, trash_notify_mkdir_cbk,
-                                    FIRST_CHILD(this),
-                                    FIRST_CHILD(this)->fops->mkdir, &loc, 0755,
-                                    0022, dict);
-                } else {
-                        /* assign new location values to new_loc members */
-                        gf_log (this->name, GF_LOG_DEBUG, "Renaming %s -> %s"
-                                        " from notify", priv->oldtrash_dir,
-                                        priv->newtrash_dir);
-                        gf_uuid_copy (loc.gfid, trash_gfid);
-                        gf_uuid_copy (loc.pargfid, root_gfid);
-                        ret = extract_trash_directory (priv->newtrash_dir,
-                                                                &loc.name);
-                        if (ret) {
-                                gf_log (this->name, GF_LOG_DEBUG,
-                                                "out of memory");
-                                goto out;
-                        }
-                        loc.path = gf_strdup (priv->newtrash_dir);
-                        if (!loc.path) {
-                                gf_log (this->name, GF_LOG_DEBUG,
-                                                "out of memory");
-                                ret = ENOMEM;
-                                goto out;
-                        }
-                        /* assign old location values to old_loc members */
-                        gf_uuid_copy (old_loc.gfid, trash_gfid);
-                        gf_uuid_copy (old_loc.pargfid, root_gfid);
-                        ret = extract_trash_directory (priv->oldtrash_dir,
-                                                                &old_loc.name);
-                        if (ret) {
-                                gf_log (this->name, GF_LOG_DEBUG,
-                                                "out of memory");
-                                goto out;
-                        }
-                        old_loc.path = gf_strdup (priv->oldtrash_dir);
-                        if (!old_loc.path) {
-                                gf_log (this->name, GF_LOG_DEBUG,
-                                                "out of memory");
-                                ret = ENOMEM;
-                                goto out;
-                        }
-
-                        old_loc.inode = inode_ref (priv->trash_inode);
-                        gf_uuid_copy(old_loc.inode->gfid, old_loc.gfid);
-
-                        STACK_WIND (frame, trash_notify_rename_cbk,
-                                    FIRST_CHILD(this),
-                                    FIRST_CHILD(this)->fops->rename,
-                                    &old_loc, &loc, dict);
-                        if (priv->oldtrash_dir)
-                                GF_FREE (priv->oldtrash_dir);
-
-                        priv->oldtrash_dir = gf_strdup(priv->newtrash_dir);
-                        if (!priv->oldtrash_dir) {
-                                gf_log (this->name, GF_LOG_DEBUG,
-                                                "out of memory");
-                                ret = ENOMEM;
-                                goto out;
-                        }
-                }
-        } else {
-                ret = default_notify (this, event, data);
-                if (ret)
-                        gf_log (this->name, GF_LOG_INFO,
-                                "default notify event failed");
+    trash_private_t *priv = NULL;
+    int ret = 0;
+
+    priv = this->private;
+    GF_VALIDATE_OR_GOTO("trash", priv, out);
+
+    /* Check whether posix is up not */
+    if (event == GF_EVENT_CHILD_UP) {
+        if (!priv->state) {
+            gf_log(this->name, GF_LOG_DEBUG, "trash xlator is off");
+            goto out;
         }
 
-out:
-        if (ret && tgfid_ptr)
-                GF_FREE (tgfid_ptr);
-        if (dict)
-                dict_unref (dict);
-        loc_wipe (&loc);
-        loc_wipe (&old_loc);
+        /* Here there is two possibilities ,if trash directory already
+         * exist ,then we need to perform a rename operation on the
+         * old one. Otherwise, we need to create the trash directory
+         * For both, we need to pass location variable, gfid of parent
+         * and a frame for calling STACK_WIND.The location variable
+         * requires name,path,gfid and inode
+         */
+        if (!priv->oldtrash_dir)
+            ret = create_or_rename_trash_directory(this);
+        else if (strcmp(priv->newtrash_dir, priv->oldtrash_dir) != 0)
+            ret = rename_trash_directory(this);
+        if (ret)
+            goto out;
 
-        return ret;
+        if (priv->internal)
+            (void)create_internalop_directory(this);
+    }
+
+out:
+    ret = default_notify(this, event, data);
+    if (ret)
+        gf_log(this->name, GF_LOG_INFO, "default notify event failed");
+    return ret;
 }
 
 int32_t
-mem_acct_init (xlator_t *this)
+mem_acct_init(xlator_t *this)
 {
-        int         ret          = -1;
+    int ret = -1;
 
-        GF_VALIDATE_OR_GOTO ("trash", this, out);
+    GF_VALIDATE_OR_GOTO("trash", this, out);
 
-        ret = xlator_mem_acct_init (this, gf_trash_mt_end + 1);
-        if (ret != 0) {
-                gf_log(this->name, GF_LOG_ERROR, "Memory accounting init"
-                       "failed");
-                return ret;
-        }
-out:
+    ret = xlator_mem_acct_init(this, gf_trash_mt_end + 1);
+    if (ret != 0) {
+        gf_log(this->name, GF_LOG_ERROR,
+               "Memory accounting init"
+               "failed");
         return ret;
+    }
+out:
+    return ret;
 }
 
 /**
  * trash_init
  */
 int32_t
-init (xlator_t *this)
+init(xlator_t *this)
 {
-        trash_private_t        *priv                   = NULL;
-        int                    ret                     = -1;
-        char                   *tmp                    = NULL;
-        char                   *tmp_str                = NULL;
-        char                   trash_dir[PATH_MAX]     = {0,};
-        uint64_t               max_trash_file_size64   = 0;
-        data_t                *data                    = NULL;
-
-        GF_VALIDATE_OR_GOTO ("trash", this, out);
-
-        if (!this->children || this->children->next) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "not configured with exactly one child. exiting");
-                ret = -1;
-                goto out;
-        }
-
-        if (!this->parents) {
-                gf_log (this->name, GF_LOG_WARNING,
-                        "dangling volume. check volfile");
-        }
-
-        priv = GF_CALLOC (1, sizeof (*priv), gf_trash_mt_trash_private_t);
-        if (!priv) {
-                gf_log (this->name, GF_LOG_ERROR, "out of memory");
-                ret = ENOMEM;
-                goto out;
-        }
-
-        /* Trash priv data members are initialized through the following
-         * set of statements
-         */
-        GF_OPTION_INIT ("trash", priv->state, bool, out);
-
-        GF_OPTION_INIT ("trash-dir", tmp, str, out);
-
-        /* We store trash dir value as path for easier manipulation*/
-        if (!tmp) {
-                gf_log (this->name, GF_LOG_INFO,
-                        "no option specified for 'trash-dir', "
-                        "using \"/.trashcan/\"");
-                priv->newtrash_dir = gf_strdup ("/.trashcan/");
-                if (!priv->newtrash_dir) {
-                        ret = ENOMEM;
-                        gf_log (this->name, GF_LOG_DEBUG, "out of memory");
-                        goto out;
-                }
-        } else {
-                sprintf(trash_dir, "/%s/", tmp);
-                priv->newtrash_dir = gf_strdup (trash_dir);
-                if (!priv->newtrash_dir) {
-                        ret = ENOMEM;
-                        gf_log (this->name, GF_LOG_DEBUG, "out of memory");
-                        goto out;
-                }
-        }
-        tmp = NULL;
-
-        GF_OPTION_INIT ("trash-eliminate-path", tmp, str, out);
-        if (!tmp) {
-                gf_log (this->name, GF_LOG_INFO,
-                        "no option specified for 'eliminate', using NULL");
-        } else {
-                tmp_str = gf_strdup (tmp);
-                if (!tmp_str) {
-                        gf_log (this->name, GF_LOG_ERROR,
-                                        "out of memory");
-                        ret = ENOMEM;
-                        goto out;
-                }
-                ret = store_eliminate_path (tmp_str, &priv->eliminate);
-
-        }
-        tmp = NULL;
-
-        GF_OPTION_INIT ("trash-max-filesize", max_trash_file_size64,
-                        size_uint64, out);
-        if (!max_trash_file_size64) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "no option specified for 'max-trashable-file-size', "
-                        "using default = %lld MB",
-                        GF_DEFAULT_MAX_FILE_SIZE / GF_UNIT_MB);
-                priv->max_trash_file_size = GF_DEFAULT_MAX_FILE_SIZE;
-        } else {
-                if( max_trash_file_size64 > GF_ALLOWED_MAX_FILE_SIZE ) {
-                        gf_log (this->name, GF_LOG_DEBUG,
-                                "Size specified for max-size(in MB) is too "
-                                "large so using 1GB as max-size (NOT IDEAL)");
-                        priv->max_trash_file_size = GF_ALLOWED_MAX_FILE_SIZE;
-                } else
-                        priv->max_trash_file_size = max_trash_file_size64;
-                gf_log (this->name, GF_LOG_DEBUG, "%"GF_PRI_SIZET" max-size",
-                        priv->max_trash_file_size);
-        }
-
-        GF_OPTION_INIT ("trash-internal-op", priv->internal, bool, out);
-
-        this->local_pool = mem_pool_new (trash_local_t, 64);
-        if (!this->local_pool) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "failed to create local_t's memory pool");
-                ret = ENOMEM;
-                goto out;
-        }
-
-        /* For creating directories inside trash with proper permissions,
-         * we need to perform stat on that directories, for this we use
-         * brick path
-         */
-        data = dict_get (this->options, "brick-path");
-        if (!data) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "no option specified for 'brick-path'");
-                ret = ENOMEM;
-                goto out;
-        }
-        priv->brick_path = gf_strdup (data->data);
-        if (!priv->brick_path) {
-                ret = ENOMEM;
-                gf_log (this->name, GF_LOG_DEBUG, "out of memory");
-                goto out;
-        }
-
-        gf_log (this->name, GF_LOG_DEBUG, "brick path is%s", priv->brick_path);
-
-        this->private = (void *)priv;
-        ret = 0;
+    trash_private_t *priv = NULL;
+    int ret = -1;
+    char *tmp = NULL;
+    char *tmp_str = NULL;
+    char trash_dir[PATH_MAX] = {
+        0,
+    };
+    uint64_t max_trash_file_size64 = 0;
+    data_t *data = NULL;
+
+    GF_VALIDATE_OR_GOTO("trash", this, out);
+
+    if (!this->children || this->children->next) {
+        gf_log(this->name, GF_LOG_ERROR,
+               "not configured with exactly one child. exiting");
+        ret = -1;
+        goto out;
+    }
+
+    if (!this->parents) {
+        gf_log(this->name, GF_LOG_WARNING, "dangling volume. check volfile");
+    }
+
+    priv = GF_CALLOC(1, sizeof(*priv), gf_trash_mt_trash_private_t);
+    if (!priv) {
+        gf_log(this->name, GF_LOG_ERROR, "out of memory");
+        ret = ENOMEM;
+        goto out;
+    }
+
+    /* Trash priv data members are initialized through the following
+     * set of statements
+     */
+    GF_OPTION_INIT("trash", priv->state, bool, out);
+
+    GF_OPTION_INIT("trash-dir", tmp, str, out);
+
+    /* We store trash dir value as path for easier manipulation*/
+    if (!tmp) {
+        gf_log(this->name, GF_LOG_INFO,
+               "no option specified for 'trash-dir', "
+               "using \"/.trashcan/\"");
+        priv->newtrash_dir = gf_strdup("/.trashcan/");
+        if (!priv->newtrash_dir) {
+            ret = ENOMEM;
+            gf_log(this->name, GF_LOG_DEBUG, "out of memory");
+            goto out;
+        }
+    } else {
+        sprintf(trash_dir, "/%s/", tmp);
+        priv->newtrash_dir = gf_strdup(trash_dir);
+        if (!priv->newtrash_dir) {
+            ret = ENOMEM;
+            gf_log(this->name, GF_LOG_DEBUG, "out of memory");
+            goto out;
+        }
+    }
+    tmp = NULL;
+
+    GF_OPTION_INIT("trash-eliminate-path", tmp, str, out);
+    if (!tmp) {
+        gf_log(this->name, GF_LOG_INFO,
+               "no option specified for 'eliminate', using NULL");
+    } else {
+        tmp_str = gf_strdup(tmp);
+        if (!tmp_str) {
+            gf_log(this->name, GF_LOG_ERROR, "out of memory");
+            ret = ENOMEM;
+            goto out;
+        }
+        ret = store_eliminate_path(tmp_str, &priv->eliminate);
+    }
+    tmp = NULL;
+
+    GF_OPTION_INIT("trash-max-filesize", max_trash_file_size64, size_uint64,
+                   out);
+    if (!max_trash_file_size64) {
+        gf_log(this->name, GF_LOG_ERROR,
+               "no option specified for 'max-trashable-file-size', "
+               "using default = %lld MB",
+               GF_DEFAULT_MAX_FILE_SIZE / GF_UNIT_MB);
+        priv->max_trash_file_size = GF_DEFAULT_MAX_FILE_SIZE;
+    } else {
+        priv->max_trash_file_size = max_trash_file_size64;
+        gf_log(this->name, GF_LOG_DEBUG, "%" GF_PRI_SIZET " max-size",
+               priv->max_trash_file_size);
+    }
+
+    GF_OPTION_INIT("trash-internal-op", priv->internal, bool, out);
+
+    this->local_pool = mem_pool_new(trash_local_t, 64);
+    if (!this->local_pool) {
+        gf_log(this->name, GF_LOG_ERROR,
+               "failed to create local_t's memory pool");
+        ret = ENOMEM;
+        goto out;
+    }
+
+    /* For creating directories inside trash with proper permissions,
+     * we need to perform stat on that directories, for this we use
+     * brick path
+     */
+    data = dict_get(this->options, "brick-path");
+    if (!data) {
+        gf_log(this->name, GF_LOG_ERROR,
+               "no option specified for 'brick-path'");
+        ret = ENOMEM;
+        goto out;
+    }
+    priv->brick_path = gf_strdup(data->data);
+    if (!priv->brick_path) {
+        ret = ENOMEM;
+        gf_log(this->name, GF_LOG_DEBUG, "out of memory");
+        goto out;
+    }
+
+    priv->trash_itable = inode_table_new(0, this);
+    gf_log(this->name, GF_LOG_DEBUG, "brick path is%s", priv->brick_path);
+
+    this->private = (void *)priv;
+    ret = 0;
 
 out:
-        if (tmp_str)
-                GF_FREE (tmp_str);
-        if (ret) {
-                if (priv) {
-                        if (priv->newtrash_dir)
-                                GF_FREE (priv->newtrash_dir);
-                        if (priv->oldtrash_dir)
-                                GF_FREE (priv->oldtrash_dir);
-                        if (priv->brick_path)
-                                GF_FREE (priv->brick_path);
-                        if (priv->eliminate)
-                                wipe_eliminate_path (&priv->eliminate);
-                        GF_FREE (priv);
-                }
-                mem_pool_destroy (this->local_pool);
-        }
-        return ret;
+    if (tmp_str)
+        GF_FREE(tmp_str);
+    if (ret) {
+        if (priv) {
+            if (priv->newtrash_dir)
+                GF_FREE(priv->newtrash_dir);
+            if (priv->oldtrash_dir)
+                GF_FREE(priv->oldtrash_dir);
+            if (priv->brick_path)
+                GF_FREE(priv->brick_path);
+            if (priv->eliminate)
+                wipe_eliminate_path(&priv->eliminate);
+            GF_FREE(priv);
+        }
+        mem_pool_destroy(this->local_pool);
+        this->local_pool = NULL;
+    }
+    return ret;
 }
 
 /**
  * trash_fini
  */
 void
-fini (xlator_t *this)
+fini(xlator_t *this)
 {
-        trash_private_t *priv = NULL;
-
-        GF_VALIDATE_OR_GOTO ("trash", this, out);
-        priv = this->private;
-
-        if (priv) {
-                if (priv->newtrash_dir)
-                        GF_FREE (priv->newtrash_dir);
-                if (priv->oldtrash_dir)
-                        GF_FREE (priv->oldtrash_dir);
-                if (priv->brick_path)
-                        GF_FREE (priv->brick_path);
-                if (priv->eliminate)
-                        wipe_eliminate_path (&priv->eliminate);
-                GF_FREE (priv);
-        }
-        mem_pool_destroy (this->local_pool);
-        this->private = NULL;
+    trash_private_t *priv = NULL;
+    inode_table_t *inode_table = NULL;
+
+    GF_VALIDATE_OR_GOTO("trash", this, out);
+    priv = this->private;
+    if (priv) {
+        inode_table = priv->trash_itable;
+        if (priv->newtrash_dir) {
+            GF_FREE(priv->newtrash_dir);
+            priv->newtrash_dir = NULL;
+        }
+        if (priv->oldtrash_dir) {
+            GF_FREE(priv->oldtrash_dir);
+            priv->oldtrash_dir = NULL;
+        }
+        if (priv->brick_path) {
+            GF_FREE(priv->brick_path);
+            priv->brick_path = NULL;
+        }
+        if (priv->eliminate) {
+            wipe_eliminate_path(&priv->eliminate);
+            priv->eliminate = NULL;
+        }
+        if (inode_table) {
+            inode_table_destroy(inode_table);
+            priv->trash_itable = NULL;
+        }
+        GF_FREE(priv);
+    }
+
+    if (this->local_pool) {
+        mem_pool_destroy(this->local_pool);
+        this->local_pool = NULL;
+    }
+    this->private = NULL;
 out:
-        return;
+    return;
 }
 
 struct xlator_fops fops = {
-        .unlink          = trash_unlink,
-        .truncate        = trash_truncate,
-        .ftruncate       = trash_ftruncate,
-        .rmdir           = trash_rmdir,
-        .mkdir           = trash_mkdir,
-        .rename          = trash_rename,
+    .unlink = trash_unlink,
+    .truncate = trash_truncate,
+    .ftruncate = trash_ftruncate,
+    .rmdir = trash_rmdir,
+    .mkdir = trash_mkdir,
+    .rename = trash_rename,
 };
 
-struct xlator_cbks cbks = {
-};
+struct xlator_cbks cbks = {};
 
 struct volume_options options[] = {
-        { .key           = { "trash" },
-          .type          = GF_OPTION_TYPE_BOOL,
-          .default_value = "off",
-          .description   = "Enable/disable trash translator",
-        },
-        { .key           = { "trash-dir" },
-          .type          = GF_OPTION_TYPE_STR,
-          .default_value = ".trashcan",
-          .description   = "Directory for trash files",
-        },
-        { .key           = { "trash-eliminate-path" },
-          .type          = GF_OPTION_TYPE_STR,
-          .description   = "Eliminate paths to be excluded "
-                           "from trashing",
-        },
-        { .key           = { "trash-max-filesize" },
-          .type          = GF_OPTION_TYPE_SIZET,
-          .default_value = "5MB",
-          .description   = "Maximum size of file that can be "
-                           "moved to trash",
-        },
-        { .key           = { "trash-internal-op" },
-          .type          = GF_OPTION_TYPE_BOOL,
-          .default_value = "off",
-          .description   = "Enable/disable trash translator for "
-                           "internal operations",
-        },
-        { .key           = {NULL} },
+    {
+        .key = {"trash"},
+        .type = GF_OPTION_TYPE_BOOL,
+        .default_value = "off",
+        .description = "Enable/disable trash translator",
+        .op_version = {GD_OP_VERSION_3_7_0},
+        .flags = OPT_FLAG_SETTABLE | OPT_FLAG_DOC,
+        .tags = {"backup"},
+    },
+    {
+        .key = {"trash-dir"},
+        .type = GF_OPTION_TYPE_STR,
+        .default_value = ".trashcan",
+        .description = "Directory for trash files",
+        .op_version = {GD_OP_VERSION_3_7_0},
+        .flags = OPT_FLAG_SETTABLE | OPT_FLAG_DOC,
+        .tags = {"backup"},
+    },
+    {
+        .key = {"trash-eliminate-path"},
+        .type = GF_OPTION_TYPE_STR,
+        .description = "Eliminate paths to be excluded "
+                       "from trashing",
+        .op_version = {GD_OP_VERSION_3_7_0},
+        .flags = OPT_FLAG_SETTABLE | OPT_FLAG_DOC,
+        .tags = {"backup"},
+    },
+    {
+        .key = {"trash-max-filesize"},
+        .type = GF_OPTION_TYPE_SIZET,
+        .default_value = "5MB",
+        .description = "Maximum size of file that can be "
+                       "moved to trash",
+        .op_version = {GD_OP_VERSION_3_7_0},
+        .flags = OPT_FLAG_SETTABLE | OPT_FLAG_DOC,
+        .tags = {"backup"},
+    },
+    {
+        .key = {"trash-internal-op"},
+        .type = GF_OPTION_TYPE_BOOL,
+        .default_value = "off",
+        .description = "Enable/disable trash translator for "
+                       "internal operations",
+        .op_version = {GD_OP_VERSION_3_7_0},
+        .flags = OPT_FLAG_SETTABLE | OPT_FLAG_DOC,
+        .tags = {"backup"},
+    },
+    {.key = {"brick-path"},
+     .type = GF_OPTION_TYPE_PATH,
+     .default_value = "{{ brick.path }}"},
+    {.key = {NULL}},
+};
+
+xlator_api_t xlator_api = {
+    .init = init,
+    .fini = fini,
+    .notify = notify,
+    .reconfigure = reconfigure,
+    .mem_acct_init = mem_acct_init,
+    .op_version = {1}, /* Present from the initial version */
+    .fops = &fops,
+    .cbks = &cbks,
+    .options = options,
+    .identifier = "trash",
+    .category = GF_TECH_PREVIEW,
 };
diff --git a/xlators/features/trash/src/trash.h b/xlators/features/trash/src/trash.h
index 3e03edf5474..6671617c2c6 100644
--- a/xlators/features/trash/src/trash.h
+++ b/xlators/features/trash/src/trash.h
@@ -10,74 +10,88 @@
 #ifndef __TRASH_H__
 #define __TRASH_H__
 
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
-
-#include "glusterfs.h"
-#include "logging.h"
-#include "dict.h"
-#include "xlator.h"
-#include "defaults.h"
+#include <glusterfs/glusterfs.h>
+#include <glusterfs/logging.h>
+#include <glusterfs/dict.h>
+#include <glusterfs/xlator.h>
+#include <glusterfs/defaults.h>
 #include "inode.c"
 #include "fnmatch.h"
 
 #include <libgen.h>
 
 #ifndef GF_BLOCK_READV_SIZE
-#define GF_BLOCK_READV_SIZE      (128 * GF_UNIT_KB)
+#define GF_BLOCK_READV_SIZE (128 * GF_UNIT_KB)
 #endif
 
 #ifndef GF_DEFAULT_MAX_FILE_SIZE
 #define GF_DEFAULT_MAX_FILE_SIZE (200 * GF_UNIT_MB)
 #endif
 
-#ifndef GF_ALLOWED_MAX_FILE_SIZE
-#define GF_ALLOWED_MAX_FILE_SIZE (1 * GF_UNIT_GB)
-#endif
-
 struct trash_struct {
-        fd_t    *fd;         /* for the fd of existing file */
-        fd_t    *newfd;      /* for the newly created file */
-        loc_t    loc;        /* to store the location of the existing file */
-        loc_t    newloc;     /* to store the location for the new file */
-        size_t   fsize;      /* for keeping the size of existing file */
-        off_t    cur_offset; /* current offset for read and write ops */
-        off_t    fop_offset;
-        char     origpath[PATH_MAX];
-        char     newpath[PATH_MAX];
-        int32_t  loop_count;
-        struct iatt preparent;
-        struct iatt postparent;
+    fd_t *fd;         /* for the fd of existing file */
+    fd_t *newfd;      /* for the newly created file */
+    loc_t loc;        /* to store the location of the existing file */
+    loc_t newloc;     /* to store the location for the new file */
+    size_t fsize;     /* for keeping the size of existing file */
+    off_t cur_offset; /* current offset for read and write ops */
+    off_t fop_offset; /* original offset received with the fop */
+    pid_t pid;
+    char origpath[PATH_MAX];
+    char newpath[PATH_MAX];
+    int32_t loop_count;
+    gf_boolean_t is_set_pid;
+    struct iatt preparent;
+    struct iatt postparent;
+    gf_boolean_t ctr_link_count_req;
 };
 typedef struct trash_struct trash_local_t;
 
 struct _trash_elim_path {
-        struct _trash_elim_path    *next;
-        char                       *path;
+    struct _trash_elim_path *next;
+    char *path;
 };
 typedef struct _trash_elim_path trash_elim_path;
 
 struct trash_priv {
-        char                 *oldtrash_dir;
-        char                 *newtrash_dir;
-        char                 *brick_path;
-        trash_elim_path      *eliminate;
-        size_t                max_trash_file_size;
-        gf_boolean_t          state;
-        gf_boolean_t          internal;
-        inode_t              *trash_inode;
-        inode_table_t        *trash_itable;
+    char *oldtrash_dir;
+    char *newtrash_dir;
+    char *brick_path;
+    trash_elim_path *eliminate;
+    size_t max_trash_file_size;
+    gf_boolean_t state;
+    gf_boolean_t internal;
+    inode_t *trash_inode;
+    inode_table_t *trash_itable;
 };
 typedef struct trash_priv trash_private_t;
 
-#define TRASH_STACK_UNWIND(op, frame, params ...) do {    \
-                trash_local_t *__local = NULL;            \
-                __local = frame->local;                   \
-                frame->local = NULL;                      \
-                STACK_UNWIND_STRICT (op, frame, params);  \
-                trash_local_wipe (__local);               \
-        } while (0)
+#define TRASH_SET_PID(frame, local)                                            \
+    do {                                                                       \
+        GF_ASSERT(!local->is_set_pid);                                         \
+        if (!local->is_set_pid) {                                              \
+            local->pid = frame->root->pid;                                     \
+            frame->root->pid = GF_SERVER_PID_TRASH;                            \
+            local->is_set_pid = _gf_true;                                      \
+        }                                                                      \
+    } while (0)
+
+#define TRASH_UNSET_PID(frame, local)                                          \
+    do {                                                                       \
+        GF_ASSERT(local->is_set_pid);                                          \
+        if (local->is_set_pid) {                                               \
+            frame->root->pid = local->pid;                                     \
+            local->is_set_pid = _gf_false;                                     \
+        }                                                                      \
+    } while (0)
+
+#define TRASH_STACK_UNWIND(op, frame, params...)                               \
+    do {                                                                       \
+        trash_local_t *__local = NULL;                                         \
+        __local = frame->local;                                                \
+        frame->local = NULL;                                                   \
+        STACK_UNWIND_STRICT(op, frame, params);                                \
+        trash_local_wipe(__local);                                             \
+    } while (0)
 
 #endif /* __TRASH_H__ */
diff --git a/xlators/features/upcall/src/Makefile.am b/xlators/features/upcall/src/Makefile.am
index 2ac09551476..72b7f55ae0a 100644
--- a/xlators/features/upcall/src/Makefile.am
+++ b/xlators/features/upcall/src/Makefile.am
@@ -1,20 +1,22 @@
+if WITH_SERVER
 xlator_LTLIBRARIES = upcall.la
+endif
 xlatordir = $(libdir)/glusterfs/$(PACKAGE_VERSION)/xlator/features
 
-upcall_la_LDFLAGS = -module -avoid-version
+upcall_la_LDFLAGS = -module $(GF_XLATOR_DEFAULT_LDFLAGS)
 
 upcall_la_SOURCES = upcall.c upcall-internal.c
 
 upcall_la_LIBADD = $(top_builddir)/libglusterfs/src/libglusterfs.la \
-        $(top_builddir)/rpc/rpc-lib/src/libgfrpc.la \
-        $(top_builddir)/rpc/xdr/src/libgfxdr.la
+	$(top_builddir)/rpc/rpc-lib/src/libgfrpc.la \
+	$(top_builddir)/rpc/xdr/src/libgfxdr.la
 
 noinst_HEADERS = upcall.h upcall-mem-types.h upcall-messages.h \
 	upcall-cache-invalidation.h
 
 AM_CPPFLAGS = $(GF_CPPFLAGS) -I$(top_srcdir)/libglusterfs/src \
-        -I$(top_srcdir)/rpc/rpc-lib/src \
-        -I$(top_srcdir)/rpc/xdr/src
+	-I$(top_srcdir)/rpc/rpc-lib/src \
+	-I$(top_srcdir)/rpc/xdr/src -I$(top_builddir)/rpc/xdr/src
 
 AM_CFLAGS = -Wall -fno-strict-aliasing $(GF_CFLAGS)
 
diff --git a/xlators/features/upcall/src/upcall-cache-invalidation.h b/xlators/features/upcall/src/upcall-cache-invalidation.h
index 138595cecb6..db649b2c9a6 100644
--- a/xlators/features/upcall/src/upcall-cache-invalidation.h
+++ b/xlators/features/upcall/src/upcall-cache-invalidation.h
@@ -11,74 +11,8 @@
 #ifndef __UPCALL_CACHE_INVALIDATION_H__
 #define __UPCALL_CACHE_INVALIDATION_H__
 
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
-
 /* The time period for which a client will be notified of cache_invalidation
  * events post its last access */
 #define CACHE_INVALIDATION_TIMEOUT "60"
 
-/* Flags sent for cache_invalidation */
-#define UP_NLINK   0x00000001   /* update nlink */
-#define UP_MODE    0x00000002   /* update mode and ctime */
-#define UP_OWN     0x00000004   /* update mode,uid,gid and ctime */
-#define UP_SIZE    0x00000008   /* update fsize */
-#define UP_TIMES   0x00000010   /* update all times */
-#define UP_ATIME   0x00000020   /* update atime only */
-#define UP_PERM    0x00000040   /* update fields needed for
-                                   permission checking */
-#define UP_RENAME  0x00000080   /* this is a rename op -
-                                   delete the cache entry */
-#define UP_FORGET  0x00000100   /* inode_forget on server side -
-                                   invalidate the cache entry */
-
-/* for fops - open, read, lk, */
-#define UP_IDEMPOTENT_FLAGS     (UP_ATIME)
-
-/* for fop - write, truncate */
-#define UP_WRITE_FLAGS          (UP_SIZE | UP_TIMES)
-
-/* for fop - setattr */
-#define UP_ATTR_FLAGS           (UP_SIZE | UP_TIMES | UP_OWN |        \
-                                 UP_MODE | UP_PERM)
-/* for fop - rename */
-#define UP_RENAME_FLAGS         (UP_RENAME)
-
-/* to invalidate parent directory entries for fops -rename, unlink,
- * rmdir, mkdir, create */
-#define UP_PARENT_DENTRY_FLAGS  (UP_TIMES)
-
-/* for fop - unlink, link, rmdir, mkdir */
-#define UP_NLINK_FLAGS          (UP_NLINK | UP_TIMES)
-
-#define CACHE_INVALIDATE(frame, this, client, inode, p_flags) do {      \
-                                                                        \
-        if (!is_cache_invalidation_enabled(this))                       \
-                break;                                                  \
-                                                                        \
-        (void)upcall_cache_invalidate (frame, this, client,             \
-                                       inode, p_flags);                 \
-} while (0)
-
-#define CACHE_INVALIDATE_DIR(frame, this, client, inode_p, p_flags) do {\
-                                                                        \
-        if (!is_cache_invalidation_enabled(this))                       \
-                break;                                                  \
-                                                                        \
-        dentry_t *dentry;                                               \
-        dentry_t *dentry_tmp;                                           \
-        list_for_each_entry_safe (dentry, dentry_tmp,                   \
-                                  &inode_p->dentry_list,                \
-                                  inode_list) {                         \
-                (void)upcall_cache_invalidate (frame, this, client,     \
-                                               dentry->inode, p_flags); \
-        }                                                               \
-} while (0)
-
-/* xlator options */
-gf_boolean_t is_cache_invalidation_enabled(xlator_t *this);
-int32_t get_cache_invalidation_timeout(xlator_t *this);
-
 #endif /* __UPCALL_CACHE_INVALIDATION_H__ */
diff --git a/xlators/features/upcall/src/upcall-internal.c b/xlators/features/upcall/src/upcall-internal.c
index 29c387601f9..c641bd6f432 100644
--- a/xlators/features/upcall/src/upcall-internal.c
+++ b/xlators/features/upcall/src/upcall-internal.c
@@ -12,323 +12,454 @@
 #include <fcntl.h>
 #include <limits.h>
 
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
+#include <glusterfs/glusterfs.h>
+#include <glusterfs/compat.h>
+#include <glusterfs/xlator.h>
+#include <glusterfs/logging.h>
+#include <glusterfs/common-utils.h>
 
-#include "glusterfs.h"
-#include "compat.h"
-#include "xlator.h"
-#include "inode.h"
-#include "logging.h"
-#include "common-utils.h"
-
-#include "statedump.h"
-#include "syncop.h"
+#include <glusterfs/statedump.h>
+#include <glusterfs/syncop.h>
 
 #include "upcall.h"
 #include "upcall-mem-types.h"
 #include "glusterfs3-xdr.h"
 #include "protocol-common.h"
-#include "defaults.h"
+#include <glusterfs/defaults.h>
 
 /*
  * Check if any of the upcall options are enabled:
  *     - cache_invalidation
- *     - XXX: lease_lk
  */
 gf_boolean_t
-is_upcall_enabled(xlator_t *this) {
-        upcall_private_t *priv      = NULL;
-        gf_boolean_t     is_enabled = _gf_false;
-
-        if (this->private) {
-                priv = (upcall_private_t *)this->private;
+is_upcall_enabled(xlator_t *this)
+{
+    upcall_private_t *priv = NULL;
 
-                if (priv->cache_invalidation_enabled) {
-                        is_enabled = _gf_true;
-                }
-        }
+    if (this->private) {
+        priv = (upcall_private_t *)this->private;
+        return priv->cache_invalidation_enabled;
+    }
 
-        return is_enabled;
+    return _gf_false;
 }
 
 /*
- * Check if any of cache_invalidation is enabled
+ * Get the cache_invalidation_timeout
  */
-gf_boolean_t
-is_cache_invalidation_enabled(xlator_t *this) {
-        upcall_private_t *priv      = NULL;
-        gf_boolean_t     is_enabled = _gf_false;
-
-        if (this->private) {
-                priv = (upcall_private_t *)this->private;
+static int32_t
+get_cache_invalidation_timeout(xlator_t *this)
+{
+    upcall_private_t *priv = NULL;
 
-                if (priv->cache_invalidation_enabled) {
-                        is_enabled = _gf_true;
-                }
-        }
+    if (this->private) {
+        priv = (upcall_private_t *)this->private;
+        return priv->cache_invalidation_timeout;
+    }
 
-        return is_enabled;
+    return 0;
 }
 
-/*
- * Get the cache_invalidation_timeout
- */
-int32_t
-get_cache_invalidation_timeout(xlator_t *this) {
-        upcall_private_t *priv      = NULL;
-        int32_t          timeout    = 0;
-
-        if (this->private) {
-                priv = (upcall_private_t *)this->private;
-                timeout = priv->cache_invalidation_timeout;
-        }
+static upcall_client_t *
+__add_upcall_client(call_frame_t *frame, client_t *client,
+                    upcall_inode_ctx_t *up_inode_ctx, time_t now)
+{
+    upcall_client_t *up_client_entry = GF_MALLOC(
+        sizeof(*up_client_entry), gf_upcall_mt_upcall_client_entry_t);
+    if (!up_client_entry) {
+        gf_msg("upcall", GF_LOG_WARNING, 0, UPCALL_MSG_NO_MEMORY,
+               "Memory allocation failed");
+        return NULL;
+    }
+    INIT_LIST_HEAD(&up_client_entry->client_list);
+    up_client_entry->client_uid = gf_strdup(client->client_uid);
+    up_client_entry->access_time = now;
+    up_client_entry->expire_time_attr = get_cache_invalidation_timeout(
+        frame->this);
+
+    list_add_tail(&up_client_entry->client_list, &up_inode_ctx->client_list);
+
+    gf_log(THIS->name, GF_LOG_DEBUG, "upcall_entry_t client added - %s",
+           up_client_entry->client_uid);
+
+    return up_client_entry;
+}
 
-        return timeout;
+static int
+__upcall_inode_ctx_set(inode_t *inode, xlator_t *this)
+{
+    upcall_inode_ctx_t *inode_ctx = NULL;
+    upcall_private_t *priv = NULL;
+    int ret = -1;
+    uint64_t ctx = 0;
+
+    priv = this->private;
+    GF_ASSERT(priv);
+
+    ret = __inode_ctx_get(inode, this, &ctx);
+
+    if (!ret)
+        goto out;
+
+    inode_ctx = GF_MALLOC(sizeof(upcall_inode_ctx_t),
+                          gf_upcall_mt_upcall_inode_ctx_t);
+
+    if (!inode_ctx) {
+        ret = -ENOMEM;
+        goto out;
+    }
+
+    pthread_mutex_init(&inode_ctx->client_list_lock, NULL);
+    INIT_LIST_HEAD(&inode_ctx->inode_ctx_list);
+    INIT_LIST_HEAD(&inode_ctx->client_list);
+    inode_ctx->destroy = 0;
+    gf_uuid_copy(inode_ctx->gfid, inode->gfid);
+
+    ctx = (long)inode_ctx;
+    ret = __inode_ctx_set(inode, this, &ctx);
+    if (ret) {
+        gf_log(this->name, GF_LOG_DEBUG, "failed to set inode ctx (%p)", inode);
+        GF_FREE(inode_ctx);
+        goto out;
+    }
+
+    /* add this inode_ctx to the global list */
+    LOCK(&priv->inode_ctx_lk);
+    {
+        list_add_tail(&inode_ctx->inode_ctx_list, &priv->inode_ctx_list);
+    }
+    UNLOCK(&priv->inode_ctx_lk);
+out:
+    return ret;
 }
 
-/*
- * Allocate and add a new client entry to the given upcall entry
- */
-upcall_client_t*
-add_upcall_client (call_frame_t *frame, uuid_t gfid,
-                   client_t *client,
-                   upcall_inode_ctx_t *up_inode_ctx)
+static upcall_inode_ctx_t *
+__upcall_inode_ctx_get(inode_t *inode, xlator_t *this)
 {
-        upcall_client_t *up_client_entry = NULL;
+    upcall_inode_ctx_t *inode_ctx = NULL;
+    uint64_t ctx = 0;
+    int ret = 0;
 
-        pthread_mutex_lock (&up_inode_ctx->client_list_lock);
-        {
-                up_client_entry = __add_upcall_client (frame,
-                                                       gfid,
-                                                       client,
-                                                       up_inode_ctx);
-        }
-        pthread_mutex_unlock (&up_inode_ctx->client_list_lock);
+    ret = __inode_ctx_get(inode, this, &ctx);
+
+    if (ret < 0) {
+        ret = __upcall_inode_ctx_set(inode, this);
+        if (ret < 0)
+            goto out;
+
+        ret = __inode_ctx_get(inode, this, &ctx);
+        if (ret < 0)
+            goto out;
+    }
+
+    inode_ctx = (upcall_inode_ctx_t *)(long)(ctx);
 
-        return up_client_entry;
+out:
+    return inode_ctx;
 }
 
-upcall_client_t*
-__add_upcall_client (call_frame_t *frame, uuid_t gfid,
-                     client_t *client,
-                     upcall_inode_ctx_t *up_inode_ctx)
+upcall_inode_ctx_t *
+upcall_inode_ctx_get(inode_t *inode, xlator_t *this)
 {
-        upcall_client_t *up_client_entry = NULL;
-
-        up_client_entry = GF_CALLOC (1, sizeof(*up_client_entry),
-                                     gf_upcall_mt_upcall_client_entry_t);
-        if (!up_client_entry) {
-                gf_msg ("upcall", GF_LOG_WARNING, 0,
-                        UPCALL_MSG_NO_MEMORY,
-                        "Memory allocation failed");
-                return NULL;
-        }
-        INIT_LIST_HEAD (&up_client_entry->client_list);
-        up_client_entry->client_uid = gf_strdup(client->client_uid);
-        up_client_entry->access_time = time(NULL);
-        up_client_entry->expire_time_attr =
-                        get_cache_invalidation_timeout(frame->this);
+    upcall_inode_ctx_t *inode_ctx = NULL;
 
-        list_add_tail (&up_client_entry->client_list,
-                       &up_inode_ctx->client_list);
+    LOCK(&inode->lock);
+    {
+        inode_ctx = __upcall_inode_ctx_get(inode, this);
+    }
+    UNLOCK(&inode->lock);
 
-        gf_log (THIS->name, GF_LOG_DEBUG, "upcall_entry_t client added - %s",
-                up_client_entry->client_uid);
+    return inode_ctx;
+}
+
+static int
+__upcall_cleanup_client_entry(upcall_client_t *up_client)
+{
+    list_del_init(&up_client->client_list);
+
+    GF_FREE(up_client->client_uid);
+    GF_FREE(up_client);
 
-        return up_client_entry;
+    return 0;
 }
 
-/*
- * Given gfid and client->uid, retrieve the corresponding upcall client entry.
- * If none found, create a new entry.
- */
-upcall_client_t*
-__get_upcall_client (call_frame_t *frame, uuid_t gfid, client_t *client,
-                     upcall_inode_ctx_t *up_inode_ctx)
+static int
+upcall_cleanup_expired_clients(xlator_t *this, upcall_inode_ctx_t *up_inode_ctx,
+                               time_t now)
 {
-        upcall_client_t *up_client_entry = NULL;
-        upcall_client_t *up_client       = NULL;
-        upcall_client_t *tmp             = NULL;
-        gf_boolean_t    found_client     = _gf_false;
-
-        list_for_each_entry_safe (up_client_entry, tmp,
-                                  &up_inode_ctx->client_list,
-                                  client_list) {
-                if (strcmp(client->client_uid,
-                           up_client_entry->client_uid) == 0) {
-                        /* found client entry. Update the access_time */
-                        up_client_entry->access_time = time(NULL);
-                        found_client = _gf_true;
-                        gf_log (THIS->name, GF_LOG_DEBUG,
-                                "upcall_entry_t client found - %s",
-                                up_client_entry->client_uid);
-                        break;
-                }
-        }
+    upcall_client_t *up_client = NULL;
+    upcall_client_t *tmp = NULL;
+    int ret = -1;
+    time_t timeout = 0;
+    time_t t_expired = 0;
+
+    timeout = get_cache_invalidation_timeout(this);
+
+    pthread_mutex_lock(&up_inode_ctx->client_list_lock);
+    {
+        list_for_each_entry_safe(up_client, tmp, &up_inode_ctx->client_list,
+                                 client_list)
+        {
+            t_expired = now - up_client->access_time;
+
+            if (t_expired > (2 * timeout)) {
+                gf_log(THIS->name, GF_LOG_TRACE, "Cleaning up client_entry(%s)",
+                       up_client->client_uid);
+
+                ret = __upcall_cleanup_client_entry(up_client);
 
-        if (!found_client) { /* create one */
-                up_client_entry = __add_upcall_client (frame, gfid, client,
-                                                       up_inode_ctx);
+                if (ret) {
+                    gf_msg("upcall", GF_LOG_WARNING, 0,
+                           UPCALL_MSG_INTERNAL_ERROR,
+                           "Client entry cleanup failed (%p)", up_client);
+                    goto out;
+                }
+            }
         }
+    }
+    pthread_mutex_unlock(&up_inode_ctx->client_list_lock);
 
-        return up_client_entry;
+    ret = 0;
+out:
+    return ret;
 }
 
+/*
+ * Free Upcall inode_ctx client list
+ */
 int
-__upcall_inode_ctx_set (inode_t *inode, xlator_t *this)
+__upcall_cleanup_inode_ctx_client_list(upcall_inode_ctx_t *inode_ctx)
 {
-        upcall_inode_ctx_t *inode_ctx   = NULL;
-        int                ret          = -1;
-        uint64_t           ctx          = 0;
+    upcall_client_t *up_client = NULL;
+    upcall_client_t *tmp = NULL;
 
-        ret = __inode_ctx_get (inode, this, &ctx);
+    list_for_each_entry_safe(up_client, tmp, &inode_ctx->client_list,
+                             client_list)
+    {
+        __upcall_cleanup_client_entry(up_client);
+    }
 
-        if (!ret)
-                goto out;
+    return 0;
+}
 
-        inode_ctx = GF_CALLOC (1, sizeof (upcall_inode_ctx_t),
-                               gf_upcall_mt_upcall_inode_ctx_t);
+static void
+upcall_cache_forget(xlator_t *this, inode_t *inode,
+                    upcall_inode_ctx_t *up_inode_ctx);
 
-        if (!inode_ctx) {
-                ret = -ENOMEM;
-                goto out;
-        }
+/*
+ * Free upcall_inode_ctx
+ */
+int
+upcall_cleanup_inode_ctx(xlator_t *this, inode_t *inode)
+{
+    uint64_t ctx = 0;
+    upcall_inode_ctx_t *inode_ctx = NULL;
+    int ret = 0;
+    upcall_private_t *priv = NULL;
 
-        pthread_mutex_init (&inode_ctx->client_list_lock, NULL);
-        INIT_LIST_HEAD (&inode_ctx->client_list);
+    priv = this->private;
+    GF_ASSERT(priv);
 
-        ret = __inode_ctx_set (inode, this, (uint64_t *) inode_ctx);
-        if (ret)
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "failed to set inode ctx (%p)", inode);
-out:
-        return ret;
-}
+    ret = inode_ctx_del(inode, this, &ctx);
 
-upcall_inode_ctx_t *
-__upcall_inode_ctx_get (inode_t *inode, xlator_t *this)
-{
-        upcall_inode_ctx_t *inode_ctx   = NULL;
-        uint64_t           ctx          = 0;
-        int                ret          = 0;
+    if (ret < 0) {
+        gf_msg("upcall", GF_LOG_WARNING, 0, UPCALL_MSG_INTERNAL_ERROR,
+               "Failed to del upcall_inode_ctx (%p)", inode);
+        goto out;
+    }
 
-        ret = __inode_ctx_get (inode, this, &ctx);
+    inode_ctx = (upcall_inode_ctx_t *)(long)ctx;
 
-        if (ret < 0) {
-                ret = __upcall_inode_ctx_set (inode, this);
-                if (ret < 0)
-                        goto out;
+    if (inode_ctx) {
+        /* Invalidate all the upcall cache entries */
+        upcall_cache_forget(this, inode, inode_ctx);
 
-                ret = __inode_ctx_get (inode, this, &ctx);
-                if (ret < 0)
-                        goto out;
+        /* do we really need lock? yes now reaper thread
+         * may also be trying to cleanup the client entries.
+         */
+        pthread_mutex_lock(&inode_ctx->client_list_lock);
+        {
+            if (!list_empty(&inode_ctx->client_list)) {
+                __upcall_cleanup_inode_ctx_client_list(inode_ctx);
+            }
         }
+        pthread_mutex_unlock(&inode_ctx->client_list_lock);
 
-        inode_ctx = (upcall_inode_ctx_t *)(long) ctx;
+        /* Mark the inode_ctx to be destroyed */
+        inode_ctx->destroy = 1;
+        gf_msg_debug("upcall", 0, "set upcall_inode_ctx (%p) to destroy mode",
+                     inode_ctx);
+    }
 
 out:
-        return inode_ctx;
+    return ret;
 }
 
-upcall_inode_ctx_t *
-upcall_inode_ctx_get (inode_t *inode, xlator_t *this)
+/*
+ * Traverse through the list of upcall_inode_ctx(s),
+ * cleanup the expired client entries and destroy the ctx
+ * which is no longer valid and has destroy bit set.
+ */
+void *
+upcall_reaper_thread(void *data)
 {
-        upcall_inode_ctx_t *inode_ctx = NULL;
-
-        LOCK (&inode->lock);
+    upcall_private_t *priv = NULL;
+    upcall_inode_ctx_t *inode_ctx = NULL;
+    upcall_inode_ctx_t *tmp = NULL;
+    xlator_t *this = NULL;
+    time_t timeout = 0;
+    time_t time_now;
+
+    this = (xlator_t *)data;
+    GF_ASSERT(this);
+
+    priv = this->private;
+    GF_ASSERT(priv);
+
+    time_now = gf_time();
+    while (!priv->fini) {
+        list_for_each_entry_safe(inode_ctx, tmp, &priv->inode_ctx_list,
+                                 inode_ctx_list)
         {
-                inode_ctx = __upcall_inode_ctx_get (inode, this);
+            /* cleanup expired clients */
+            upcall_cleanup_expired_clients(this, inode_ctx, time_now);
+
+            if (!inode_ctx->destroy) {
+                continue;
+            }
+
+            /* client list would have been cleaned up*/
+            gf_msg_debug("upcall", 0, "Freeing upcall_inode_ctx (%p)",
+                         inode_ctx);
+            LOCK(&priv->inode_ctx_lk);
+            {
+                list_del_init(&inode_ctx->inode_ctx_list);
+                pthread_mutex_destroy(&inode_ctx->client_list_lock);
+            }
+            UNLOCK(&priv->inode_ctx_lk);
+            GF_FREE(inode_ctx);
+            inode_ctx = NULL;
         }
-        UNLOCK (&inode->lock);
 
-        return inode_ctx;
+        /* don't do a very busy loop */
+        timeout = get_cache_invalidation_timeout(this);
+        sleep(timeout / 2);
+        time_now = gf_time();
+    }
+
+    return NULL;
 }
 
+/*
+ * Initialize upcall reaper thread.
+ */
 int
-__upcall_cleanup_client_entry (upcall_client_t *up_client)
+upcall_reaper_thread_init(xlator_t *this)
 {
-        list_del_init (&up_client->client_list);
+    upcall_private_t *priv = NULL;
+    int ret = -1;
+
+    priv = this->private;
+    GF_ASSERT(priv);
 
-        GF_FREE (up_client->client_uid);
-        GF_FREE (up_client);
+    ret = gf_thread_create(&priv->reaper_thr, NULL, upcall_reaper_thread, this,
+                           "upreaper");
 
-        return 0;
+    return ret;
 }
 
-/*
- * Free Upcall inode_ctx client list
- */
 int
-__upcall_cleanup_inode_ctx_client_list (upcall_inode_ctx_t *inode_ctx)
+up_compare_afr_xattr(dict_t *d, char *k, data_t *v, void *tmp)
 {
-        upcall_client_t *up_client        = NULL;
-        upcall_client_t *tmp              = NULL;
+    dict_t *dict = tmp;
 
-        list_for_each_entry_safe (up_client, tmp,
-                                  &inode_ctx->client_list,
-                                  client_list) {
-                __upcall_cleanup_client_entry (up_client);
-        }
+    if (!strncmp(k, AFR_XATTR_PREFIX, SLEN(AFR_XATTR_PREFIX)) &&
+        (!is_data_equal(v, dict_get(dict, k))))
+        return -1;
 
-        return 0;
+    return 0;
 }
 
-/*
- * Free upcall_inode_ctx
- */
-int
-upcall_cleanup_inode_ctx (xlator_t *this, inode_t *inode)
+static void
+up_filter_afr_xattr(dict_t *xattrs, char *xattr, data_t *v)
 {
-        uint64_t           ctx          = 0;
-        upcall_inode_ctx_t *inode_ctx   = NULL;
-        int                ret          = 0;
+    /* Filter the afr pending xattrs, with value 0. Ideally this should
+     * be executed only in case of xattrop and not in set and removexattr,
+     * butset and remove xattr fops do not come with keys AFR_XATTR_PREFIX
+     */
+    if (!strncmp(xattr, AFR_XATTR_PREFIX, SLEN(AFR_XATTR_PREFIX)) &&
+        (mem_0filled(v->data, v->len) == 0)) {
+        dict_del(xattrs, xattr);
+    }
+    return;
+}
 
-        ret = inode_ctx_get (inode, this, &ctx);
+static gf_boolean_t
+up_key_is_regd_xattr(dict_t *regd_xattrs, char *regd_xattr, data_t *v,
+                     void *xattr)
+{
+    int ret = _gf_false;
+    char *key = xattr;
 
-        if (ret < 0) {
-                gf_log (THIS->name, GF_LOG_TRACE,
-                        "Failed to get upcall_inode_ctx (%p)",
-                        inode);
-                goto out;
-        }
+    if (fnmatch(regd_xattr, key, 0) == 0)
+        ret = _gf_true;
 
-        /* Invalidate all the upcall cache entries */
-        upcall_cache_forget (this, inode, inode_ctx);
+    return ret;
+}
 
-        /* Set inode context to NULL */
-        ret = __inode_ctx_set (inode, this, NULL);
+int
+up_filter_unregd_xattr(dict_t *xattrs, char *xattr, data_t *v,
+                       void *regd_xattrs)
+{
+    int ret = 0;
+
+    ret = dict_foreach_match(regd_xattrs, up_key_is_regd_xattr, xattr,
+                             dict_null_foreach_fn, NULL);
+    if (ret == 0) {
+        /* xattr was not found in the registered xattr, hence do not
+         * send notification for its change
+         */
+        dict_del(xattrs, xattr);
+        goto out;
+    }
+    up_filter_afr_xattr(xattrs, xattr, v);
+out:
+    return 0;
+}
 
-        if (!ret) {
-                gf_log (this->name, GF_LOG_WARNING,
-                        "_inode_ctx_set to NULL failed (%p)",
-                        inode);
-        }
-        inode_ctx = (upcall_inode_ctx_t *)(long) ctx;
-
-        if (inode_ctx) {
-                /* do we really need lock? */
-                pthread_mutex_lock (&inode_ctx->client_list_lock);
-                {
-                        if (!list_empty (&inode_ctx->client_list)) {
-                                __upcall_cleanup_inode_ctx_client_list (inode_ctx);
-                        }
-                }
-                pthread_mutex_unlock (&inode_ctx->client_list_lock);
+int
+up_filter_xattr(dict_t *xattr, dict_t *regd_xattrs)
+{
+    int ret = 0;
 
-                pthread_mutex_destroy (&inode_ctx->client_list_lock);
+    ret = dict_foreach(xattr, up_filter_unregd_xattr, regd_xattrs);
 
-                GF_FREE (inode_ctx);
-        }
+    return ret;
+}
 
-out:
-        return ret;
+static void
+upcall_client_cache_invalidate(xlator_t *this, uuid_t gfid,
+                               upcall_client_t *up_client_entry, uint32_t flags,
+                               struct iatt *stbuf, struct iatt *p_stbuf,
+                               struct iatt *oldp_stbuf, dict_t *xattr,
+                               time_t now);
+
+gf_boolean_t
+up_invalidate_needed(dict_t *xattrs)
+{
+    if (dict_key_count(xattrs) == 0) {
+        gf_msg_trace("upcall", 0,
+                     "None of xattrs requested for"
+                     " invalidation, were changed. Nothing to "
+                     "invalidate");
+        return _gf_false;
+    }
+
+    return _gf_true;
 }
 
 /*
- * Given a gfid, client, first fetch upcall_entry_t based on gfid.
+ * Given a client, first fetch upcall_entry_t from the inode_ctx client list.
  * Later traverse through the client list of that upcall entry. If this client
  * is not present in the list, create one client entry with this client info.
  * Also check if there are other clients which need to be notified of this
@@ -338,147 +469,221 @@ out:
  * any errors during the process are logged and ignored.
  */
 void
-upcall_cache_invalidate (call_frame_t *frame, xlator_t *this, client_t *client,
-                         inode_t *inode, uint32_t flags)
+upcall_cache_invalidate(call_frame_t *frame, xlator_t *this, client_t *client,
+                        inode_t *inode, uint32_t flags, struct iatt *stbuf,
+                        struct iatt *p_stbuf, struct iatt *oldp_stbuf,
+                        dict_t *xattr)
 {
-        upcall_client_t *up_client       = NULL;
-        upcall_client_t *up_client_entry = NULL;
-        upcall_client_t *tmp             = NULL;
-        upcall_inode_ctx_t *up_inode_ctx = NULL;
-        gf_boolean_t     found           = _gf_false;
-
-        up_inode_ctx = ((upcall_local_t *)frame->local)->upcall_inode_ctx;
-
-        if (!up_inode_ctx)
-                up_inode_ctx = upcall_inode_ctx_get (inode, this);
-
-        if (!up_inode_ctx) {
-                gf_log (this->name, GF_LOG_WARNING,
-                        "upcall_inode_ctx_get failed (%p)",
-                        inode);
-                return;
+    upcall_client_t *up_client_entry = NULL;
+    upcall_client_t *tmp = NULL;
+    upcall_inode_ctx_t *up_inode_ctx = NULL;
+    gf_boolean_t found = _gf_false;
+    time_t time_now;
+    inode_t *linked_inode = NULL;
+
+    if (!is_upcall_enabled(this))
+        return;
+
+    /* server-side generated fops like quota/marker will not have any
+     * client associated with them. Ignore such fops.
+     */
+    if (!client) {
+        gf_msg_debug("upcall", 0, "Internal fop - client NULL");
+        return;
+    }
+
+    /* For nameless LOOKUPs, inode created shall always be
+     * invalid. Hence check if there is any already linked inode.
+     * If yes, update the inode_ctx of that valid inode
+     */
+    if (inode && (inode->ia_type == IA_INVAL) && stbuf) {
+        linked_inode = inode_find(inode->table, stbuf->ia_gfid);
+        if (linked_inode) {
+            gf_log("upcall", GF_LOG_DEBUG,
+                   "upcall_inode_ctx_get of linked inode (%p)", inode);
+            up_inode_ctx = upcall_inode_ctx_get(linked_inode, this);
         }
-
-        pthread_mutex_lock (&up_inode_ctx->client_list_lock);
+    }
+
+    if (inode && !up_inode_ctx)
+        up_inode_ctx = upcall_inode_ctx_get(inode, this);
+
+    if (!up_inode_ctx) {
+        gf_msg("upcall", GF_LOG_WARNING, 0, UPCALL_MSG_INTERNAL_ERROR,
+               "upcall_inode_ctx_get failed (%p)", inode);
+        return;
+    }
+
+    /* In case of LOOKUP, if first time, inode created shall be
+     * invalid till it gets linked to inode table. Read gfid from
+     * the stat returned in such cases.
+     */
+    if (gf_uuid_is_null(up_inode_ctx->gfid) && stbuf) {
+        /* That means inode must have been invalid when this inode_ctx
+         * is created. Copy the gfid value from stbuf instead.
+         */
+        gf_uuid_copy(up_inode_ctx->gfid, stbuf->ia_gfid);
+    }
+
+    if (gf_uuid_is_null(up_inode_ctx->gfid)) {
+        gf_msg_debug(this->name, 0,
+                     "up_inode_ctx->gfid and "
+                     "stbuf->ia_gfid is NULL, fop:%s",
+                     gf_fop_list[frame->root->op]);
+        goto out;
+    }
+
+    time_now = gf_time();
+    pthread_mutex_lock(&up_inode_ctx->client_list_lock);
+    {
+        list_for_each_entry_safe(up_client_entry, tmp,
+                                 &up_inode_ctx->client_list, client_list)
         {
-                list_for_each_entry_safe (up_client_entry, tmp,
-                                          &up_inode_ctx->client_list,
-                                          client_list) {
-
-                        if (!strcmp(client->client_uid,
-                                   up_client_entry->client_uid)) {
-                                up_client_entry->access_time = time(NULL);
-                                found = _gf_true;
-                        }
-
-                        /*
-                         * Ignore sending notifications in case of only UP_ATIME
-                         */
-                        if (!(flags & ~(UP_ATIME))) {
-                                if (found)
-                                        break;
-                                else /* we still need to find current client entry*/
-                                        continue;
-                        }
-
-                        /* any other client */
-
-                        /* XXX: Send notifications asynchrounously
-                         * instead of in the I/O path - BZ 1200264
-                         *  Also if the file is frequently accessed, set
-                         *  expire_time_attr to 0.
-                         */
-                        upcall_client_cache_invalidate(this,
-                                                       inode->gfid,
-                                                       up_client_entry,
-                                                       flags);
-                }
+            /* Do not send UPCALL event if same client. */
+            if (!strcmp(client->client_uid, up_client_entry->client_uid)) {
+                up_client_entry->access_time = time_now;
+                found = _gf_true;
+                continue;
+            }
+
+            /*
+             * Ignore sending notifications in case of only UP_ATIME
+             */
+            if (!(flags & ~(UP_ATIME))) {
+                if (found)
+                    break;
+                else /* we still need to find current client entry*/
+                    continue;
+            }
+
+            /* any other client */
+
+            /* XXX: Send notifications asynchrounously
+             * instead of in the I/O path - BZ 1200264
+             *  Also if the file is frequently accessed, set
+             *  expire_time_attr to 0.
+             */
+            upcall_client_cache_invalidate(
+                this, up_inode_ctx->gfid, up_client_entry, flags, stbuf,
+                p_stbuf, oldp_stbuf, xattr, time_now);
+        }
 
-                if (!found) {
-                        up_client_entry = __add_upcall_client (frame,
-                                                               inode->gfid,
-                                                               client,
-                                                               up_inode_ctx);
-                }
+        if (!found) {
+            up_client_entry = __add_upcall_client(frame, client, up_inode_ctx,
+                                                  time_now);
         }
-        pthread_mutex_unlock (&up_inode_ctx->client_list_lock);
+    }
+    pthread_mutex_unlock(&up_inode_ctx->client_list_lock);
+out:
+    /* release the ref from inode_find */
+    if (linked_inode)
+        inode_unref(linked_inode);
+    return;
 }
 
 /*
  * If the upcall_client_t has recently accessed the file (i.e, within
  * priv->cache_invalidation_timeout), send a upcall notification.
  */
-void
-upcall_client_cache_invalidate (xlator_t *this, uuid_t gfid,
-                                upcall_client_t *up_client_entry,
-                                uint32_t flags)
+static void
+upcall_client_cache_invalidate(xlator_t *this, uuid_t gfid,
+                               upcall_client_t *up_client_entry, uint32_t flags,
+                               struct iatt *stbuf, struct iatt *p_stbuf,
+                               struct iatt *oldp_stbuf, dict_t *xattr,
+                               time_t now)
 {
-        notify_event_data_t n_event_data;
-        time_t timeout   = 0;
-        time_t t_expired = time(NULL) - up_client_entry->access_time;
-
-        timeout = get_cache_invalidation_timeout(this);
-
-        if (t_expired < timeout) {
-                /* Send notify call */
-                gf_uuid_copy(n_event_data.gfid, gfid);
-                n_event_data.client_entry = up_client_entry;
-                n_event_data.event_type = CACHE_INVALIDATION;
-                n_event_data.invalidate_flags = flags;
-
-                /* Need to send inode flags */
-                this->notify (this, GF_EVENT_UPCALL, &n_event_data);
-
-                gf_log (THIS->name, GF_LOG_TRACE,
-                        "Cache invalidation notification sent to %s",
-                        up_client_entry->client_uid);
-
-        } else {
-                if (t_expired > (2*timeout)) {
-                        /* Cleanup the entry */
-                        __upcall_cleanup_client_entry (up_client_entry);
-                }
-
-                gf_log (THIS->name, GF_LOG_TRACE,
-                        "Cache invalidation notification NOT sent to %s",
-                        up_client_entry->client_uid);
+    struct gf_upcall up_req = {
+        0,
+    };
+    struct gf_upcall_cache_invalidation ca_req = {
+        0,
+    };
+    time_t timeout = 0;
+    int ret = -1;
+    time_t t_expired = now - up_client_entry->access_time;
+
+    GF_VALIDATE_OR_GOTO("upcall_client_cache_invalidate",
+                        !(gf_uuid_is_null(gfid)), out);
+    timeout = get_cache_invalidation_timeout(this);
+
+    if (t_expired < timeout) {
+        /* Send notify call */
+        up_req.client_uid = up_client_entry->client_uid;
+        gf_uuid_copy(up_req.gfid, gfid);
+
+        ca_req.flags = flags;
+        ca_req.expire_time_attr = up_client_entry->expire_time_attr;
+        if (stbuf)
+            ca_req.stat = *stbuf;
+        if (p_stbuf)
+            ca_req.p_stat = *p_stbuf;
+        if (oldp_stbuf)
+            ca_req.oldp_stat = *oldp_stbuf;
+        ca_req.dict = xattr;
+
+        up_req.data = &ca_req;
+        up_req.event_type = GF_UPCALL_CACHE_INVALIDATION;
+
+        gf_log(THIS->name, GF_LOG_TRACE,
+               "Cache invalidation notification sent to %s",
+               up_client_entry->client_uid);
+
+        /* Need to send inode flags */
+        ret = this->notify(this, GF_EVENT_UPCALL, &up_req);
+
+        /*
+         * notify may fail as the client could have been
+         * dis(re)connected. Cleanup the client entry.
+         */
+        if (ret < 0)
+            __upcall_cleanup_client_entry(up_client_entry);
+
+    } else {
+        gf_log(THIS->name, GF_LOG_TRACE,
+               "Cache invalidation notification NOT sent to %s",
+               up_client_entry->client_uid);
+
+        if (t_expired > (2 * timeout)) {
+            /* Cleanup the entry */
+            __upcall_cleanup_client_entry(up_client_entry);
         }
+    }
+out:
+    return;
 }
 
 /*
- * This is called during upcall_inode_ctx cleanup incase of 'inode_forget'.
+ * This is called during upcall_inode_ctx cleanup in case of 'inode_forget'.
  * Send "UP_FORGET" to all the clients so that they invalidate their cache
  * entry and do a fresh lookup next time when any I/O comes in.
  */
-void
-upcall_cache_forget (xlator_t *this, inode_t *inode, upcall_inode_ctx_t *up_inode_ctx)
+static void
+upcall_cache_forget(xlator_t *this, inode_t *inode,
+                    upcall_inode_ctx_t *up_inode_ctx)
 {
-        upcall_client_t *up_client       = NULL;
-        upcall_client_t *up_client_entry = NULL;
-        upcall_client_t *tmp             = NULL;
-        uint32_t        flags            = 0;
-
-        if (!up_inode_ctx) {
-                return;
-        }
-
-        pthread_mutex_lock (&up_inode_ctx->client_list_lock);
+    upcall_client_t *up_client_entry = NULL;
+    upcall_client_t *tmp = NULL;
+    uint32_t flags = UP_FORGET;
+    time_t time_now;
+
+    if (!up_inode_ctx) {
+        return;
+    }
+
+    time_now = gf_time();
+    pthread_mutex_lock(&up_inode_ctx->client_list_lock);
+    {
+        list_for_each_entry_safe(up_client_entry, tmp,
+                                 &up_inode_ctx->client_list, client_list)
         {
-                list_for_each_entry_safe (up_client_entry, tmp,
-                                          &up_inode_ctx->client_list,
-                                          client_list) {
-                        flags = UP_FORGET;
-
-                        /* Set the access time to time(NULL)
-                         * to send notify */
-                        up_client_entry->access_time = time(NULL);
-
-                        upcall_client_cache_invalidate(this,
-                                                       inode->gfid,
-                                                       up_client_entry,
-                                                       flags);
-                }
+            /* Set the access time to gf_time()
+             * to send notify */
+            up_client_entry->access_time = time_now;
 
+            upcall_client_cache_invalidate(this, up_inode_ctx->gfid,
+                                           up_client_entry, flags, NULL, NULL,
+                                           NULL, NULL, time_now);
         }
-        pthread_mutex_unlock (&up_inode_ctx->client_list_lock);
+    }
+    pthread_mutex_unlock(&up_inode_ctx->client_list_lock);
 }
diff --git a/xlators/features/upcall/src/upcall-mem-types.h b/xlators/features/upcall/src/upcall-mem-types.h
index 55793ec65ca..f9883d9d72c 100644
--- a/xlators/features/upcall/src/upcall-mem-types.h
+++ b/xlators/features/upcall/src/upcall-mem-types.h
@@ -11,14 +11,13 @@
 #ifndef __UPCALL_MEM_TYPES_H__
 #define __UPCALL_MEM_TYPES_H__
 
-#include "mem-types.h"
+#include <glusterfs/mem-types.h>
 
 enum gf_upcall_mem_types_ {
-        gf_upcall_mt_conf_t = gf_common_mt_end + 1,
-        gf_upcall_mt_private_t,
-        gf_upcall_mt_upcall_inode_ctx_t,
-        gf_upcall_mt_upcall_client_entry_t,
-        gf_upcall_mt_end
+    gf_upcall_mt_conf_t = gf_common_mt_end + 1,
+    gf_upcall_mt_private_t,
+    gf_upcall_mt_upcall_inode_ctx_t,
+    gf_upcall_mt_upcall_client_entry_t,
+    gf_upcall_mt_end
 };
 #endif
-
diff --git a/xlators/features/upcall/src/upcall-messages.h b/xlators/features/upcall/src/upcall-messages.h
index c8483f16528..4095a34c200 100644
--- a/xlators/features/upcall/src/upcall-messages.h
+++ b/xlators/features/upcall/src/upcall-messages.h
@@ -11,52 +11,19 @@
 #ifndef _UPCALL_MESSAGES_H_
 #define _UPCALL_MESSAGES_H_
 
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
-
-#include "glfs-message-id.h"
-
-/*! \file upcall-messages.h
- *  \brief UPCALL log-message IDs and their descriptions.
- */
-
-/* NOTE: Rules for message additions
- * 1) Each instance of a message is _better_ left with a unique message ID, even
- * if the message format is the same. Reasoning is that, if the message
- * format needs to change in one instance, the other instances are not
- * impacted or the new change does not change the ID of the instance being
- * modified.
- * 2) Addition of a message,
- * - Should increment the GLFS_NUM_MESSAGES
- * - Append to the list of messages defined, towards the end
- * - Retain macro naming as glfs_msg_X (for redability across developers)
- * NOTE: Rules for message format modifications
- * 3) Check across the code if the message ID macro in question is reused
- * anywhere. If reused then then the modifications should ensure correctness
- * everywhere, or needs a new message ID as (1) above was not adhered to. If
- * not used anywhere, proceed with the required modification.
- * NOTE: Rules for message deletion
- * 4) Check (3) and if used anywhere else, then cannot be deleted. If not used
- * anywhere, then can be deleted, but will leave a hole by design, as
- * addition rules specify modification to the end of the list and not filling
- * holes.
- */
-
-#define GLFS_COMP_BASE_UPCALL GLFS_MSGID_COMP_UPCALL
-#define GLFS_NUM_MESSAGES 1
-#define GLFS_MSGID_END (GLFS_COMP_BASE_UPCALL + GLFS_NUM_MESSAGES + 1)
-
-#define glfs_msg_start_x GLFS_COMP_BASE_UPCALL, "Invalid: Start of messages"
-
-/*!
- * @messageid 110001
- * @diagnosis Out of Memory
- * @recommendedaction None
+#include <glusterfs/glfs-message-id.h>
+
+/* To add new message IDs, append new identifiers at the end of the list.
+ *
+ * Never remove a message ID. If it's not used anymore, you can rename it or
+ * leave it as it is, but not delete it. This is to prevent reutilization of
+ * IDs by other messages.
+ *
+ * The component name must match one of the entries defined in
+ * glfs-message-id.h.
  */
-#define UPCALL_MSG_NO_MEMORY             (GLFS_COMP_BASE_UPCALL + 1)
 
-#define glfs_msg_end_x GLFS_MSGID_END, "Invalid: End of messages"
+GLFS_MSGID(UPCALL, UPCALL_MSG_NO_MEMORY, UPCALL_MSG_INTERNAL_ERROR,
+           UPCALL_MSG_NOTIFY_FAILED);
 
 #endif /* !_UPCALL_MESSAGES_H_ */
diff --git a/xlators/features/upcall/src/upcall.c b/xlators/features/upcall/src/upcall.c
index 93861e7955f..0795f58059d 100644
--- a/xlators/features/upcall/src/upcall.c
+++ b/xlators/features/upcall/src/upcall.c
@@ -13,982 +13,2493 @@
 #include <limits.h>
 #include <pthread.h>
 
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
-
-#include "glusterfs.h"
-#include "compat.h"
-#include "xlator.h"
-#include "inode.h"
-#include "logging.h"
-#include "common-utils.h"
+#include <glusterfs/glusterfs.h>
+#include <glusterfs/compat.h>
+#include <glusterfs/xlator.h>
+#include <glusterfs/logging.h>
+#include <glusterfs/common-utils.h>
 
-#include "statedump.h"
-#include "syncop.h"
+#include <glusterfs/statedump.h>
 
 #include "upcall.h"
 #include "upcall-mem-types.h"
 #include "glusterfs3-xdr.h"
 #include "protocol-common.h"
-#include "defaults.h"
+#include <glusterfs/defaults.h>
 
-int
-up_open_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-             int32_t op_ret, int32_t op_errno, fd_t *fd, dict_t *xdata)
+static int32_t
+up_open_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int32_t op_ret,
+            int32_t op_errno, fd_t *fd, dict_t *xdata)
 {
-        client_t         *client        = NULL;
-        uint32_t         flags          = 0;
-        upcall_local_t   *local         = NULL;
+    client_t *client = NULL;
+    uint32_t flags = 0;
+    upcall_local_t *local = NULL;
 
-        EXIT_IF_UPCALL_OFF (this, out);
+    EXIT_IF_UPCALL_OFF(this, out);
 
-        client = frame->root->client;
-        local = frame->local;
+    client = frame->root->client;
+    local = frame->local;
 
-        if ((op_ret < 0) || !local) {
-                goto out;
-        }
-        flags = UP_IDEMPOTENT_FLAGS;
-        CACHE_INVALIDATE (frame, this, client, local->inode, flags);
+    if ((op_ret < 0) || !local) {
+        goto out;
+    }
+    flags = UP_UPDATE_CLIENT;
+    upcall_cache_invalidate(frame, this, client, local->inode, flags, NULL,
+                            NULL, NULL, NULL);
 
 out:
-        UPCALL_STACK_UNWIND (open, frame, op_ret, op_errno, fd, xdata);
+    UPCALL_STACK_UNWIND(open, frame, op_ret, op_errno, fd, xdata);
 
-        return 0;
+    return 0;
 }
 
+static int32_t
+up_open(call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t flags,
+        fd_t *fd, dict_t *xdata)
+{
+    int32_t op_errno = ENOMEM;
+    upcall_local_t *local = NULL;
 
-int
-up_open (call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t flags,
-         fd_t *fd, dict_t *xdata)
+    EXIT_IF_UPCALL_OFF(this, out);
+
+    local = upcall_local_init(frame, this, NULL, NULL, fd->inode, NULL);
+    if (!local) {
+        goto err;
+    }
+
+out:
+    STACK_WIND(frame, up_open_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->open, loc, flags, fd, xdata);
+
+    return 0;
+
+err:
+    UPCALL_STACK_UNWIND(open, frame, -1, op_errno, NULL, NULL);
+
+    return 0;
+}
+
+static int32_t
+up_writev_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int op_ret,
+              int op_errno, struct iatt *prebuf, struct iatt *postbuf,
+              dict_t *xdata)
 {
-        int32_t          op_errno        = -1;
-        upcall_local_t   *local          = NULL;
+    client_t *client = NULL;
+    uint32_t flags = 0;
+    upcall_local_t *local = NULL;
 
-        EXIT_IF_UPCALL_OFF (this, out);
+    client = frame->root->client;
+    local = frame->local;
 
-        local = upcall_local_init (frame, this, fd->inode);
-        if (!local) {
-                op_errno = ENOMEM;
-                goto err;
-        }
+    if ((op_ret < 0) || !local) {
+        goto out;
+    }
+    flags = UP_WRITE_FLAGS;
+    upcall_cache_invalidate(frame, this, client, local->inode, flags, postbuf,
+                            NULL, NULL, NULL);
 
 out:
-        STACK_WIND (frame, up_open_cbk,
-                    FIRST_CHILD(this), FIRST_CHILD(this)->fops->open,
-                    loc, flags, fd, xdata);
+    UPCALL_STACK_UNWIND(writev, frame, op_ret, op_errno, prebuf, postbuf,
+                        xdata);
 
-        return 0;
+    return 0;
+}
+
+static int32_t
+up_writev(call_frame_t *frame, xlator_t *this, fd_t *fd, struct iovec *vector,
+          int count, off_t off, uint32_t flags, struct iobref *iobref,
+          dict_t *xdata)
+{
+    int32_t op_errno = ENOMEM;
+    upcall_local_t *local = NULL;
+
+    EXIT_IF_UPCALL_OFF(this, out);
+
+    local = upcall_local_init(frame, this, NULL, NULL, fd->inode, NULL);
+    if (!local) {
+        goto err;
+    }
+
+out:
+    STACK_WIND(frame, up_writev_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->writev, fd, vector, count, off, flags,
+               iobref, xdata);
+
+    return 0;
 
 err:
-        op_errno = (op_errno == -1) ? errno : op_errno;
-        UPCALL_STACK_UNWIND (open, frame, -1, op_errno, NULL, NULL);
+    UPCALL_STACK_UNWIND(writev, frame, -1, op_errno, NULL, NULL, NULL);
 
-        return 0;
+    return 0;
 }
 
-int
-up_writev_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                int op_ret, int op_errno, struct iatt *prebuf,
-                struct iatt *postbuf, dict_t *xdata)
+static int32_t
+up_readv_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int op_ret,
+             int op_errno, struct iovec *vector, int count, struct iatt *stbuf,
+             struct iobref *iobref, dict_t *xdata)
 {
-        client_t         *client        = NULL;
-        uint32_t         flags          = 0;
-        upcall_local_t   *local         = NULL;
+    client_t *client = NULL;
+    uint32_t flags = 0;
+    upcall_local_t *local = NULL;
 
-        client = frame->root->client;
-        local = frame->local;
+    EXIT_IF_UPCALL_OFF(this, out);
 
-        if ((op_ret < 0) || !local) {
-                goto out;
-        }
-        flags = UP_WRITE_FLAGS;
-        CACHE_INVALIDATE (frame, this, client, local->inode, flags);
+    client = frame->root->client;
+    local = frame->local;
+
+    if ((op_ret < 0) || !local) {
+        goto out;
+    }
+    flags = UP_UPDATE_CLIENT;
+    upcall_cache_invalidate(frame, this, client, local->inode, flags, stbuf,
+                            NULL, NULL, NULL);
 
 out:
-        UPCALL_STACK_UNWIND (writev, frame, op_ret, op_errno,
-                             prebuf, postbuf, xdata);
+    UPCALL_STACK_UNWIND(readv, frame, op_ret, op_errno, vector, count, stbuf,
+                        iobref, xdata);
 
-        return 0;
+    return 0;
 }
 
+static int32_t
+up_readv(call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
+         off_t offset, uint32_t flags, dict_t *xdata)
+{
+    int32_t op_errno = ENOMEM;
+    upcall_local_t *local = NULL;
 
-int
-up_writev (call_frame_t *frame, xlator_t *this, fd_t *fd,
-            struct iovec *vector, int count, off_t off, uint32_t flags,
-            struct iobref *iobref, dict_t *xdata)
+    EXIT_IF_UPCALL_OFF(this, out);
+
+    local = upcall_local_init(frame, this, NULL, NULL, fd->inode, NULL);
+    if (!local) {
+        goto err;
+    }
+
+out:
+    STACK_WIND(frame, up_readv_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->readv, fd, size, offset, flags, xdata);
+
+    return 0;
+
+err:
+    UPCALL_STACK_UNWIND(readv, frame, -1, op_errno, NULL, 0, NULL, NULL, NULL);
+
+    return 0;
+}
+
+static int32_t
+up_lk_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int32_t op_ret,
+          int32_t op_errno, struct gf_flock *lock, dict_t *xdata)
 {
-        int32_t          op_errno        = -1;
-        upcall_local_t   *local          = NULL;
+    client_t *client = NULL;
+    uint32_t flags = 0;
+    upcall_local_t *local = NULL;
 
-        EXIT_IF_UPCALL_OFF (this, out);
+    EXIT_IF_UPCALL_OFF(this, out);
 
-        local = upcall_local_init (frame, this, fd->inode);
-        if (!local) {
-                op_errno = ENOMEM;
-                goto err;
-        }
+    client = frame->root->client;
+    local = frame->local;
+
+    if ((op_ret < 0) || !local) {
+        goto out;
+    }
+    flags = UP_UPDATE_CLIENT;
+    upcall_cache_invalidate(frame, this, client, local->inode, flags, NULL,
+                            NULL, NULL, NULL);
 
 out:
-        STACK_WIND (frame, up_writev_cbk,
-                    FIRST_CHILD(this), FIRST_CHILD(this)->fops->writev,
-                    fd, vector, count, off, flags, iobref, xdata);
+    UPCALL_STACK_UNWIND(lk, frame, op_ret, op_errno, lock, xdata);
+
+    return 0;
+}
+
+static int32_t
+up_lk(call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t cmd,
+      struct gf_flock *flock, dict_t *xdata)
+{
+    int32_t op_errno = ENOMEM;
+    upcall_local_t *local = NULL;
 
-        return 0;
+    EXIT_IF_UPCALL_OFF(this, out);
+
+    local = upcall_local_init(frame, this, NULL, NULL, fd->inode, NULL);
+    if (!local) {
+        goto err;
+    }
+
+out:
+    STACK_WIND(frame, up_lk_cbk, FIRST_CHILD(this), FIRST_CHILD(this)->fops->lk,
+               fd, cmd, flock, xdata);
+    return 0;
 
 err:
-        op_errno = (op_errno == -1) ? errno : op_errno;
-        op_errno = (op_errno == -1) ? errno : op_errno;
-        UPCALL_STACK_UNWIND (writev, frame, -1, op_errno, NULL, NULL, NULL);
+    UPCALL_STACK_UNWIND(lk, frame, -1, op_errno, NULL, NULL);
 
-        return 0;
+    return 0;
 }
 
+static int32_t
+up_truncate_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int op_ret,
+                int op_errno, struct iatt *prebuf, struct iatt *postbuf,
+                dict_t *xdata)
+{
+    client_t *client = NULL;
+    uint32_t flags = 0;
+    upcall_local_t *local = NULL;
 
-int
-up_readv_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-               int op_ret, int op_errno,
-               struct iovec *vector, int count, struct iatt *stbuf,
-               struct iobref *iobref, dict_t *xdata)
+    EXIT_IF_UPCALL_OFF(this, out);
+
+    client = frame->root->client;
+    local = frame->local;
+
+    if ((op_ret < 0) || !local) {
+        goto out;
+    }
+    flags = UP_WRITE_FLAGS;
+    upcall_cache_invalidate(frame, this, client, local->inode, flags, postbuf,
+                            NULL, NULL, NULL);
+
+out:
+    UPCALL_STACK_UNWIND(truncate, frame, op_ret, op_errno, prebuf, postbuf,
+                        xdata);
+
+    return 0;
+}
+
+static int32_t
+up_truncate(call_frame_t *frame, xlator_t *this, loc_t *loc, off_t offset,
+            dict_t *xdata)
 {
-        client_t         *client        = NULL;
-        uint32_t         flags          = 0;
-        upcall_local_t   *local         = NULL;
+    int32_t op_errno = ENOMEM;
+    upcall_local_t *local = NULL;
 
-        EXIT_IF_UPCALL_OFF (this, out);
+    EXIT_IF_UPCALL_OFF(this, out);
 
-        client = frame->root->client;
-        local = frame->local;
+    local = upcall_local_init(frame, this, NULL, NULL, loc->inode, NULL);
+    if (!local) {
+        goto err;
+    }
 
-        if ((op_ret < 0) || !local) {
-                goto out;
-        }
-        flags = UP_IDEMPOTENT_FLAGS;
-        CACHE_INVALIDATE (frame, this, client, local->inode, flags);
+out:
+    STACK_WIND(frame, up_truncate_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->truncate, loc, offset, xdata);
+
+    return 0;
+
+err:
+    UPCALL_STACK_UNWIND(truncate, frame, -1, op_errno, NULL, NULL, NULL);
+
+    return 0;
+}
+
+static int32_t
+up_setattr_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int op_ret,
+               int op_errno, struct iatt *statpre, struct iatt *statpost,
+               dict_t *xdata)
+{
+    client_t *client = NULL;
+    uint32_t flags = 0;
+    upcall_local_t *local = NULL;
+
+    EXIT_IF_UPCALL_OFF(this, out);
+
+    client = frame->root->client;
+    local = frame->local;
+
+    if ((op_ret < 0) || !local) {
+        goto out;
+    }
+    /* XXX: setattr -> UP_SIZE or UP_OWN or UP_MODE or UP_TIMES
+     * or INODE_UPDATE (or UP_PERM esp in case of ACLs -> INODE_INVALIDATE)
+     * Need to check what attr is changed and accordingly pass UP_FLAGS.
+     * Bug1200271.
+     */
+    flags = UP_ATTR_FLAGS;
+    /* If mode bits have changed invalidate the xattrs, as posix-acl and
+     * others store permission related information in xattrs. With changing
+     * of permissions/mode, we need to make clients to forget all the
+     * xattrs related to permissions.
+     * TODO: Invalidate the xattr system.posix_acl_access alone.
+     */
+    if (is_same_mode(statpre->ia_prot, statpost->ia_prot) != 0)
+        flags |= UP_XATTR;
+
+    upcall_cache_invalidate(frame, this, client, local->inode, flags, statpost,
+                            NULL, NULL, NULL);
 
 out:
-        UPCALL_STACK_UNWIND (readv, frame, op_ret, op_errno, vector,
-                             count, stbuf, iobref, xdata);
+    UPCALL_STACK_UNWIND(setattr, frame, op_ret, op_errno, statpre, statpost,
+                        xdata);
 
-        return 0;
+    return 0;
 }
 
-int
-up_readv (call_frame_t *frame, xlator_t *this,
-          fd_t *fd, size_t size, off_t offset,
-          uint32_t flags, dict_t *xdata)
+static int32_t
+up_setattr(call_frame_t *frame, xlator_t *this, loc_t *loc, struct iatt *stbuf,
+           int32_t valid, dict_t *xdata)
 {
-        int32_t          op_errno        = -1;
-        upcall_local_t   *local          = NULL;
+    int32_t op_errno = ENOMEM;
+    upcall_local_t *local = NULL;
 
-        EXIT_IF_UPCALL_OFF (this, out);
+    EXIT_IF_UPCALL_OFF(this, out);
 
-        local = upcall_local_init (frame, this, fd->inode);
-        if (!local) {
-                op_errno = ENOMEM;
-                goto err;
-        }
+    local = upcall_local_init(frame, this, NULL, NULL, loc->inode, NULL);
+    if (!local) {
+        goto err;
+    }
 
 out:
-        STACK_WIND (frame, up_readv_cbk,
-                    FIRST_CHILD(this), FIRST_CHILD(this)->fops->readv,
-                    fd, size, offset, flags, xdata);
+    STACK_WIND(frame, up_setattr_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->setattr, loc, stbuf, valid, xdata);
 
-        return 0;
+    return 0;
 
 err:
-        op_errno = (op_errno == -1) ? errno : op_errno;
-        UPCALL_STACK_UNWIND (readv, frame, -1, op_errno, NULL, 0,
-                             NULL, NULL, NULL);
+    UPCALL_STACK_UNWIND(setattr, frame, -1, op_errno, NULL, NULL, NULL);
 
-        return 0;
+    return 0;
 }
 
-int32_t
-up_lk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                int32_t op_ret, int32_t op_errno, struct gf_flock *lock,
+static int32_t
+up_rename_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int32_t op_ret,
+              int32_t op_errno, struct iatt *stbuf, struct iatt *preoldparent,
+              struct iatt *postoldparent, struct iatt *prenewparent,
+              struct iatt *postnewparent, dict_t *xdata)
+{
+    client_t *client = NULL;
+    uint32_t flags = 0;
+    upcall_local_t *local = NULL;
+
+    EXIT_IF_UPCALL_OFF(this, out);
+
+    client = frame->root->client;
+    local = frame->local;
+
+    if ((op_ret < 0) || !local) {
+        goto out;
+    }
+    flags = (UP_RENAME_FLAGS | UP_PARENT_DENTRY_FLAGS);
+    upcall_cache_invalidate(frame, this, client, local->inode, flags, stbuf,
+                            postnewparent, postoldparent, NULL);
+
+    flags = UP_UPDATE_CLIENT;
+    upcall_cache_invalidate(frame, this, client, local->rename_oldloc.parent,
+                            flags, postoldparent, NULL, NULL, NULL);
+
+    if (local->rename_oldloc.parent == local->loc.parent)
+        goto out;
+
+    flags = UP_UPDATE_CLIENT;
+    upcall_cache_invalidate(frame, this, client, local->loc.parent, flags,
+                            postnewparent, NULL, NULL, NULL);
+
+out:
+    UPCALL_STACK_UNWIND(rename, frame, op_ret, op_errno, stbuf, preoldparent,
+                        postoldparent, prenewparent, postnewparent, xdata);
+
+    return 0;
+}
+
+static int32_t
+up_rename(call_frame_t *frame, xlator_t *this, loc_t *oldloc, loc_t *newloc,
+          dict_t *xdata)
+{
+    int32_t op_errno = ENOMEM;
+    upcall_local_t *local = NULL;
+
+    EXIT_IF_UPCALL_OFF(this, out);
+
+    local = upcall_local_init(frame, this, newloc, NULL, oldloc->inode, NULL);
+    if (!local) {
+        goto err;
+    }
+
+    /* copy oldloc */
+    loc_copy(&local->rename_oldloc, oldloc);
+out:
+    STACK_WIND(frame, up_rename_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->rename, oldloc, newloc, xdata);
+
+    return 0;
+
+err:
+    UPCALL_STACK_UNWIND(rename, frame, -1, op_errno, NULL, NULL, NULL, NULL,
+                        NULL, NULL);
+
+    return 0;
+}
+
+static int32_t
+up_unlink_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int op_ret,
+              int op_errno, struct iatt *preparent, struct iatt *postparent,
+              dict_t *xdata)
+{
+    client_t *client = NULL;
+    uint32_t flags = 0;
+    upcall_local_t *local = NULL;
+
+    EXIT_IF_UPCALL_OFF(this, out);
+
+    client = frame->root->client;
+    local = frame->local;
+
+    if ((op_ret < 0) || !local) {
+        goto out;
+    }
+    flags = (UP_NLINK_FLAGS | UP_PARENT_DENTRY_FLAGS);
+    upcall_cache_invalidate(frame, this, client, local->inode, flags, NULL,
+                            postparent, NULL, NULL);
+
+    flags = UP_UPDATE_CLIENT;
+    upcall_cache_invalidate(frame, this, client, local->loc.parent, flags,
+                            postparent, NULL, NULL, NULL);
+
+out:
+    UPCALL_STACK_UNWIND(unlink, frame, op_ret, op_errno, preparent, postparent,
+                        xdata);
+
+    return 0;
+}
+
+static int32_t
+up_unlink(call_frame_t *frame, xlator_t *this, loc_t *loc, int xflag,
+          dict_t *xdata)
+{
+    int32_t op_errno = ENOMEM;
+    upcall_local_t *local = NULL;
+
+    EXIT_IF_UPCALL_OFF(this, out);
+
+    local = upcall_local_init(frame, this, loc, NULL, loc->inode, NULL);
+    if (!local) {
+        goto err;
+    }
+
+out:
+    STACK_WIND(frame, up_unlink_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->unlink, loc, xflag, xdata);
+
+    return 0;
+
+err:
+    UPCALL_STACK_UNWIND(unlink, frame, -1, op_errno, NULL, NULL, NULL);
+
+    return 0;
+}
+
+static int32_t
+up_link_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int op_ret,
+            int op_errno, inode_t *inode, struct iatt *stbuf,
+            struct iatt *preparent, struct iatt *postparent, dict_t *xdata)
+{
+    client_t *client = NULL;
+    uint32_t flags = 0;
+    upcall_local_t *local = NULL;
+
+    EXIT_IF_UPCALL_OFF(this, out);
+
+    client = frame->root->client;
+    local = frame->local;
+
+    if ((op_ret < 0) || !local) {
+        goto out;
+    }
+    flags = (UP_NLINK_FLAGS | UP_PARENT_DENTRY_FLAGS);
+    upcall_cache_invalidate(frame, this, client, local->inode, flags, stbuf,
+                            postparent, NULL, NULL);
+
+    flags = UP_UPDATE_CLIENT;
+    upcall_cache_invalidate(frame, this, client, local->loc.parent, flags,
+                            postparent, NULL, NULL, NULL);
+
+out:
+    UPCALL_STACK_UNWIND(link, frame, op_ret, op_errno, inode, stbuf, preparent,
+                        postparent, xdata);
+
+    return 0;
+}
+
+static int32_t
+up_link(call_frame_t *frame, xlator_t *this, loc_t *oldloc, loc_t *newloc,
+        dict_t *xdata)
+{
+    int32_t op_errno = ENOMEM;
+    upcall_local_t *local = NULL;
+
+    EXIT_IF_UPCALL_OFF(this, out);
+
+    local = upcall_local_init(frame, this, newloc, NULL, oldloc->inode, NULL);
+    if (!local) {
+        goto err;
+    }
+
+out:
+    STACK_WIND(frame, up_link_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->link, oldloc, newloc, xdata);
+
+    return 0;
+
+err:
+    UPCALL_STACK_UNWIND(link, frame, -1, op_errno, NULL, NULL, NULL, NULL,
+                        NULL);
+
+    return 0;
+}
+
+static int32_t
+up_rmdir_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int op_ret,
+             int op_errno, struct iatt *preparent, struct iatt *postparent,
+             dict_t *xdata)
+{
+    client_t *client = NULL;
+    uint32_t flags = 0;
+    upcall_local_t *local = NULL;
+
+    EXIT_IF_UPCALL_OFF(this, out);
+
+    client = frame->root->client;
+    local = frame->local;
+
+    if ((op_ret < 0) || !local) {
+        goto out;
+    }
+
+    flags = (UP_NLINK_FLAGS | UP_PARENT_DENTRY_FLAGS);
+    upcall_cache_invalidate(frame, this, client, local->inode, flags, NULL,
+                            postparent, NULL, NULL);
+
+    flags = UP_UPDATE_CLIENT;
+    upcall_cache_invalidate(frame, this, client, local->loc.parent, flags,
+                            postparent, NULL, NULL, NULL);
+
+out:
+    UPCALL_STACK_UNWIND(rmdir, frame, op_ret, op_errno, preparent, postparent,
+                        xdata);
+
+    return 0;
+}
+
+static int32_t
+up_rmdir(call_frame_t *frame, xlator_t *this, loc_t *loc, int flags,
+         dict_t *xdata)
+{
+    int32_t op_errno = ENOMEM;
+    upcall_local_t *local = NULL;
+
+    EXIT_IF_UPCALL_OFF(this, out);
+
+    local = upcall_local_init(frame, this, loc, NULL, loc->inode, NULL);
+    if (!local) {
+        goto err;
+    }
+
+out:
+    STACK_WIND(frame, up_rmdir_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->rmdir, loc, flags, xdata);
+
+    return 0;
+
+err:
+    UPCALL_STACK_UNWIND(rmdir, frame, -1, op_errno, NULL, NULL, NULL);
+
+    return 0;
+}
+
+static int32_t
+up_mkdir_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int op_ret,
+             int op_errno, inode_t *inode, struct iatt *stbuf,
+             struct iatt *preparent, struct iatt *postparent, dict_t *xdata)
+{
+    client_t *client = NULL;
+    uint32_t flags = 0;
+    upcall_local_t *local = NULL;
+
+    EXIT_IF_UPCALL_OFF(this, out);
+
+    client = frame->root->client;
+    local = frame->local;
+
+    if ((op_ret < 0) || !local) {
+        goto out;
+    }
+
+    /* invalidate parent's entry too */
+    flags = UP_TIMES;
+    upcall_cache_invalidate(frame, this, client, local->inode, flags,
+                            postparent, NULL, NULL, NULL);
+
+    flags = UP_UPDATE_CLIENT;
+    upcall_cache_invalidate(frame, this, client, local->loc.inode, flags, stbuf,
+                            NULL, NULL, NULL);
+
+out:
+    UPCALL_STACK_UNWIND(mkdir, frame, op_ret, op_errno, inode, stbuf, preparent,
+                        postparent, xdata);
+
+    return 0;
+}
+
+static int32_t
+up_mkdir(call_frame_t *frame, xlator_t *this, loc_t *loc, mode_t mode,
+         mode_t umask, dict_t *params)
+{
+    int32_t op_errno = ENOMEM;
+    upcall_local_t *local = NULL;
+
+    EXIT_IF_UPCALL_OFF(this, out);
+
+    local = upcall_local_init(frame, this, loc, NULL, loc->parent, NULL);
+    if (!local) {
+        goto err;
+    }
+
+out:
+    STACK_WIND(frame, up_mkdir_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->mkdir, loc, mode, umask, params);
+
+    return 0;
+
+err:
+    UPCALL_STACK_UNWIND(mkdir, frame, -1, op_errno, NULL, NULL, NULL, NULL,
+                        NULL);
+
+    return 0;
+}
+
+static int32_t
+up_create_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int op_ret,
+              int op_errno, fd_t *fd, inode_t *inode, struct iatt *stbuf,
+              struct iatt *preparent, struct iatt *postparent, dict_t *xdata)
+{
+    client_t *client = NULL;
+    uint32_t flags = 0;
+    upcall_local_t *local = NULL;
+
+    EXIT_IF_UPCALL_OFF(this, out);
+
+    client = frame->root->client;
+    local = frame->local;
+
+    if ((op_ret < 0) || !local) {
+        goto out;
+    }
+
+    /* As its a new file create, no need of sending notification
+     * However invalidate parent's entry and update that fact that the
+     * client has accessed the newly created entry */
+    flags = UP_TIMES;
+    upcall_cache_invalidate(frame, this, client, local->inode, flags,
+                            postparent, NULL, NULL, NULL);
+
+    flags = UP_UPDATE_CLIENT;
+    upcall_cache_invalidate(frame, this, client, local->loc.inode, flags, stbuf,
+                            NULL, NULL, NULL);
+
+out:
+    UPCALL_STACK_UNWIND(create, frame, op_ret, op_errno, fd, inode, stbuf,
+                        preparent, postparent, xdata);
+
+    return 0;
+}
+
+static int32_t
+up_create(call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t flags,
+          mode_t mode, mode_t umask, fd_t *fd, dict_t *params)
+{
+    int32_t op_errno = ENOMEM;
+    upcall_local_t *local = NULL;
+
+    EXIT_IF_UPCALL_OFF(this, out);
+
+    local = upcall_local_init(frame, this, loc, NULL, loc->parent, NULL);
+    if (!local) {
+        goto err;
+    }
+
+out:
+    STACK_WIND(frame, up_create_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->create, loc, flags, mode, umask, fd,
+               params);
+
+    return 0;
+
+err:
+    UPCALL_STACK_UNWIND(create, frame, -1, op_errno, NULL, NULL, NULL, NULL,
+                        NULL, NULL);
+
+    return 0;
+}
+
+static int32_t
+up_lookup_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int op_ret,
+              int op_errno, inode_t *inode, struct iatt *stbuf, dict_t *xattr,
+              struct iatt *postparent)
+{
+    client_t *client = NULL;
+    uint32_t flags = 0;
+    upcall_local_t *local = NULL;
+
+    EXIT_IF_UPCALL_OFF(this, out);
+
+    client = frame->root->client;
+    local = frame->local;
+
+    if ((op_ret < 0) || !local) {
+        goto out;
+    }
+    flags = UP_UPDATE_CLIENT;
+    upcall_cache_invalidate(frame, this, client, local->inode, flags, stbuf,
+                            NULL, NULL, NULL);
+
+out:
+    UPCALL_STACK_UNWIND(lookup, frame, op_ret, op_errno, inode, stbuf, xattr,
+                        postparent);
+
+    return 0;
+}
+
+static int32_t
+up_lookup(call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xattr_req)
+{
+    int32_t op_errno = ENOMEM;
+    upcall_local_t *local = NULL;
+
+    EXIT_IF_UPCALL_OFF(this, out);
+
+    local = upcall_local_init(frame, this, NULL, NULL, loc->inode, NULL);
+    if (!local) {
+        goto err;
+    }
+
+out:
+    STACK_WIND(frame, up_lookup_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->lookup, loc, xattr_req);
+
+    return 0;
+
+err:
+    UPCALL_STACK_UNWIND(lookup, frame, -1, op_errno, NULL, NULL, NULL, NULL);
+
+    return 0;
+}
+
+static int32_t
+up_stat_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int32_t op_ret,
+            int32_t op_errno, struct iatt *buf, dict_t *xdata)
+{
+    client_t *client = NULL;
+    uint32_t flags = 0;
+    upcall_local_t *local = NULL;
+
+    EXIT_IF_UPCALL_OFF(this, out);
+
+    client = frame->root->client;
+    local = frame->local;
+
+    if ((op_ret < 0) || !local) {
+        goto out;
+    }
+    flags = UP_UPDATE_CLIENT;
+    upcall_cache_invalidate(frame, this, client, local->inode, flags, buf, NULL,
+                            NULL, NULL);
+
+out:
+    UPCALL_STACK_UNWIND(stat, frame, op_ret, op_errno, buf, xdata);
+
+    return 0;
+}
+
+static int32_t
+up_stat(call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xdata)
+{
+    int32_t op_errno = ENOMEM;
+    upcall_local_t *local = NULL;
+
+    EXIT_IF_UPCALL_OFF(this, out);
+
+    local = upcall_local_init(frame, this, NULL, NULL, loc->inode, NULL);
+    if (!local) {
+        goto err;
+    }
+
+out:
+    STACK_WIND(frame, up_stat_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->stat, loc, xdata);
+
+    return 0;
+
+err:
+    UPCALL_STACK_UNWIND(stat, frame, -1, op_errno, NULL, NULL);
+
+    return 0;
+}
+
+static int32_t
+up_fstat(call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *xdata)
+{
+    int32_t op_errno = ENOMEM;
+    upcall_local_t *local = NULL;
+
+    EXIT_IF_UPCALL_OFF(this, out);
+
+    local = upcall_local_init(frame, this, NULL, NULL, fd->inode, NULL);
+    if (!local) {
+        goto err;
+    }
+
+out:
+    STACK_WIND(frame, up_stat_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->fstat, fd, xdata);
+
+    return 0;
+
+err:
+    UPCALL_STACK_UNWIND(fstat, frame, -1, op_errno, NULL, NULL);
+
+    return 0;
+}
+
+static int32_t
+up_ftruncate(call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
+             dict_t *xdata)
+{
+    int32_t op_errno = ENOMEM;
+    upcall_local_t *local = NULL;
+
+    EXIT_IF_UPCALL_OFF(this, out);
+
+    local = upcall_local_init(frame, this, NULL, NULL, fd->inode, NULL);
+    if (!local) {
+        goto err;
+    }
+
+out:
+    STACK_WIND(frame, up_truncate_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->ftruncate, fd, offset, xdata);
+
+    return 0;
+
+err:
+    UPCALL_STACK_UNWIND(ftruncate, frame, -1, op_errno, NULL, NULL, NULL);
+
+    return 0;
+}
+
+static int32_t
+up_access_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int op_ret,
+              int op_errno, dict_t *xdata)
+{
+    client_t *client = NULL;
+    uint32_t flags = 0;
+    upcall_local_t *local = NULL;
+
+    EXIT_IF_UPCALL_OFF(this, out);
+
+    client = frame->root->client;
+    local = frame->local;
+
+    if ((op_ret < 0) || !local) {
+        goto out;
+    }
+    flags = UP_UPDATE_CLIENT;
+    upcall_cache_invalidate(frame, this, client, local->inode, flags, NULL,
+                            NULL, NULL, NULL);
+
+out:
+    UPCALL_STACK_UNWIND(access, frame, op_ret, op_errno, xdata);
+
+    return 0;
+}
+
+static int32_t
+up_access(call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t mask,
+          dict_t *xdata)
+{
+    int32_t op_errno = ENOMEM;
+    upcall_local_t *local = NULL;
+
+    EXIT_IF_UPCALL_OFF(this, out);
+
+    local = upcall_local_init(frame, this, NULL, NULL, loc->inode, NULL);
+    if (!local) {
+        goto err;
+    }
+
+out:
+    STACK_WIND(frame, up_access_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->access, loc, mask, xdata);
+
+    return 0;
+
+err:
+    UPCALL_STACK_UNWIND(access, frame, -1, op_errno, NULL);
+
+    return 0;
+}
+
+static int32_t
+up_readlink_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int op_ret,
+                int op_errno, const char *path, struct iatt *stbuf,
                 dict_t *xdata)
 {
-        client_t         *client        = NULL;
-        uint32_t         flags          = 0;
-        upcall_local_t   *local         = NULL;
+    client_t *client = NULL;
+    uint32_t flags = 0;
+    upcall_local_t *local = NULL;
 
-        EXIT_IF_UPCALL_OFF (this, out);
+    EXIT_IF_UPCALL_OFF(this, out);
 
-        client = frame->root->client;
-        local = frame->local;
+    client = frame->root->client;
+    local = frame->local;
 
-        if ((op_ret < 0) || !local) {
-                goto out;
-        }
-        flags = UP_IDEMPOTENT_FLAGS;
-        CACHE_INVALIDATE (frame, this, client, local->inode, flags);
+    if ((op_ret < 0) || !local) {
+        goto out;
+    }
+    flags = UP_UPDATE_CLIENT;
+    upcall_cache_invalidate(frame, this, client, local->inode, flags, stbuf,
+                            NULL, NULL, NULL);
 
 out:
-        UPCALL_STACK_UNWIND (lk, frame, op_ret, op_errno, lock, xdata);
+    UPCALL_STACK_UNWIND(readlink, frame, op_ret, op_errno, path, stbuf, xdata);
 
-        return 0;
+    return 0;
 }
 
-int
-up_lk (call_frame_t *frame, xlator_t *this,
-       fd_t *fd, int32_t cmd, struct gf_flock *flock, dict_t *xdata)
+static int32_t
+up_readlink(call_frame_t *frame, xlator_t *this, loc_t *loc, size_t size,
+            dict_t *xdata)
 {
-        int32_t          op_errno        = -1;
-        upcall_local_t   *local          = NULL;
+    int32_t op_errno = ENOMEM;
+    upcall_local_t *local = NULL;
 
-        EXIT_IF_UPCALL_OFF (this, out);
+    EXIT_IF_UPCALL_OFF(this, out);
 
-        local = upcall_local_init (frame, this, fd->inode);
-        if (!local) {
-                op_errno = ENOMEM;
-                goto err;
-        }
+    local = upcall_local_init(frame, this, NULL, NULL, loc->inode, NULL);
+    if (!local) {
+        goto err;
+    }
 
 out:
-        STACK_WIND (frame, up_lk_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->lk,
-                    fd, cmd, flock, xdata);
-        return 0;
+    STACK_WIND(frame, up_readlink_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->readlink, loc, size, xdata);
+
+    return 0;
 
 err:
-        op_errno = (op_errno == -1) ? errno : op_errno;
-        UPCALL_STACK_UNWIND (lk, frame, -1, op_errno, NULL, NULL);
+    UPCALL_STACK_UNWIND(readlink, frame, -1, op_errno, NULL, NULL, NULL);
 
-        return 0;
+    return 0;
 }
 
-int
-up_truncate_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                  int op_ret, int op_errno, struct iatt *prebuf,
-                  struct iatt *postbuf, dict_t *xdata)
+static int32_t
+up_mknod_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int32_t op_ret,
+             int32_t op_errno, inode_t *inode, struct iatt *buf,
+             struct iatt *preparent, struct iatt *postparent, dict_t *xdata)
 {
-        client_t         *client        = NULL;
-        uint32_t         flags          = 0;
-        upcall_local_t   *local         = NULL;
+    client_t *client = NULL;
+    uint32_t flags = 0;
+    upcall_local_t *local = NULL;
 
-        EXIT_IF_UPCALL_OFF (this, out);
+    EXIT_IF_UPCALL_OFF(this, out);
 
-        client = frame->root->client;
-        local = frame->local;
+    client = frame->root->client;
+    local = frame->local;
 
-        if ((op_ret < 0) || !local) {
-                goto out;
-        }
-        flags = UP_WRITE_FLAGS;
-        CACHE_INVALIDATE (frame, this, client, local->inode, flags);
+    if ((op_ret < 0) || !local) {
+        goto out;
+    }
+
+    /* invalidate parent's entry too */
+    flags = UP_TIMES;
+    upcall_cache_invalidate(frame, this, client, local->inode, flags,
+                            postparent, NULL, NULL, NULL);
+
+    flags = UP_UPDATE_CLIENT;
+    upcall_cache_invalidate(frame, this, client, local->loc.inode, flags, buf,
+                            NULL, NULL, NULL);
 
 out:
-        UPCALL_STACK_UNWIND (truncate, frame, op_ret, op_errno,
-                             prebuf, postbuf, xdata);
+    UPCALL_STACK_UNWIND(mknod, frame, op_ret, op_errno, inode, buf, preparent,
+                        postparent, xdata);
 
-        return 0;
+    return 0;
 }
 
-int
-up_truncate (call_frame_t *frame, xlator_t *this, loc_t *loc, off_t offset,
-              dict_t *xdata)
+static int32_t
+up_mknod(call_frame_t *frame, xlator_t *this, loc_t *loc, mode_t mode,
+         dev_t rdev, mode_t umask, dict_t *xdata)
 {
-        int32_t          op_errno        = -1;
-        upcall_local_t   *local          = NULL;
+    int32_t op_errno = ENOMEM;
+    upcall_local_t *local = NULL;
 
-        EXIT_IF_UPCALL_OFF (this, out);
+    EXIT_IF_UPCALL_OFF(this, out);
 
-        local = upcall_local_init (frame, this, loc->inode);
-        if (!local) {
-                op_errno = ENOMEM;
-                goto err;
-        }
+    local = upcall_local_init(frame, this, loc, NULL, loc->parent, NULL);
+    if (!local) {
+        goto err;
+    }
 
 out:
-        STACK_WIND (frame, up_truncate_cbk,
-                    FIRST_CHILD(this), FIRST_CHILD(this)->fops->truncate,
-                    loc, offset, xdata);
+    STACK_WIND(frame, up_mknod_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->mknod, loc, mode, rdev, umask, xdata);
 
-        return 0;
+    return 0;
 
 err:
-        op_errno = (op_errno == -1) ? errno : op_errno;
-        UPCALL_STACK_UNWIND (truncate, frame, -1, op_errno, NULL, NULL, NULL);
+    UPCALL_STACK_UNWIND(mknod, frame, -1, op_errno, NULL, NULL, NULL, NULL,
+                        NULL);
 
-        return 0;
+    return 0;
 }
 
-int
-up_setattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                 int op_ret, int op_errno, struct iatt *statpre,
-                 struct iatt *statpost, dict_t *xdata)
+static int32_t
+up_symlink_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+               int32_t op_ret, int32_t op_errno, inode_t *inode,
+               struct iatt *buf, struct iatt *preparent,
+               struct iatt *postparent, dict_t *xdata)
 {
-        client_t         *client        = NULL;
-        uint32_t         flags          = 0;
-        upcall_local_t   *local         = NULL;
+    client_t *client = NULL;
+    uint32_t flags = 0;
+    upcall_local_t *local = NULL;
 
-        EXIT_IF_UPCALL_OFF (this, out);
+    EXIT_IF_UPCALL_OFF(this, out);
 
-        client = frame->root->client;
-        local = frame->local;
+    client = frame->root->client;
+    local = frame->local;
 
-        if ((op_ret < 0) || !local) {
-                goto out;
-        }
-        /* XXX: setattr -> UP_SIZE or UP_OWN or UP_MODE or UP_TIMES
-         * or INODE_UPDATE (or UP_PERM esp incase of ACLs -> INODE_INVALIDATE)
-         * Need to check what attr is changed and accordingly pass UP_FLAGS.
-         * Bug1200271.
-         */
-        flags = UP_ATTR_FLAGS;
-        CACHE_INVALIDATE (frame, this, client, local->inode, flags);
+    if ((op_ret < 0) || !local) {
+        goto out;
+    }
+
+    /* invalidate parent's entry too */
+    flags = UP_TIMES;
+    upcall_cache_invalidate(frame, this, client, local->inode, flags,
+                            postparent, NULL, NULL, NULL);
+
+    flags = UP_UPDATE_CLIENT;
+    upcall_cache_invalidate(frame, this, client, local->loc.inode, flags, buf,
+                            NULL, NULL, NULL);
 
 out:
-        UPCALL_STACK_UNWIND (setattr, frame, op_ret, op_errno,
-                             statpre, statpost, xdata);
+    UPCALL_STACK_UNWIND(symlink, frame, op_ret, op_errno, inode, buf, preparent,
+                        postparent, xdata);
 
-        return 0;
+    return 0;
 }
 
-int
-up_setattr (call_frame_t *frame, xlator_t *this, loc_t *loc,
-             struct iatt *stbuf, int32_t valid, dict_t *xdata)
+static int32_t
+up_symlink(call_frame_t *frame, xlator_t *this, const char *linkpath,
+           loc_t *loc, mode_t umask, dict_t *xdata)
 {
-        int32_t          op_errno        = -1;
-        upcall_local_t   *local          = NULL;
+    int32_t op_errno = ENOMEM;
+    upcall_local_t *local = NULL;
 
-        EXIT_IF_UPCALL_OFF (this, out);
+    EXIT_IF_UPCALL_OFF(this, out);
 
-        local = upcall_local_init (frame, this, loc->inode);
-        if (!local) {
-                op_errno = ENOMEM;
-                goto err;
-        }
+    local = upcall_local_init(frame, this, loc, NULL, loc->parent, NULL);
+    if (!local) {
+        goto err;
+    }
 
 out:
-        STACK_WIND (frame, up_setattr_cbk,
-                           FIRST_CHILD(this),
-                           FIRST_CHILD(this)->fops->setattr,
-                            loc, stbuf, valid, xdata);
+    STACK_WIND(frame, up_symlink_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->symlink, linkpath, loc, umask, xdata);
 
-        return 0;
+    return 0;
 
 err:
-        op_errno = (op_errno == -1) ? errno : op_errno;
-        UPCALL_STACK_UNWIND (setattr, frame, -1, op_errno, NULL, NULL, NULL);
+    UPCALL_STACK_UNWIND(symlink, frame, -1, op_errno, NULL, NULL, NULL, NULL,
+                        NULL);
 
-        return 0;
+    return 0;
 }
 
-int
-up_rename_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                   int32_t op_ret, int32_t op_errno, struct iatt *stbuf,
-                   struct iatt *preoldparent, struct iatt *postoldparent,
-                   struct iatt *prenewparent, struct iatt *postnewparent,
-                   dict_t *xdata)
+static int32_t
+up_opendir_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+               int32_t op_ret, int32_t op_errno, fd_t *fd, dict_t *xdata)
 {
-        client_t         *client        = NULL;
-        uint32_t         flags          = 0;
-        upcall_local_t   *local         = NULL;
+    client_t *client = NULL;
+    uint32_t flags = 0;
+    upcall_local_t *local = NULL;
 
-        EXIT_IF_UPCALL_OFF (this, out);
+    EXIT_IF_UPCALL_OFF(this, out);
 
-        client = frame->root->client;
-        local = frame->local;
+    client = frame->root->client;
+    local = frame->local;
 
-        if ((op_ret < 0) || !local) {
-                goto out;
-        }
-        flags = UP_RENAME_FLAGS;
-        CACHE_INVALIDATE (frame, this, client, local->inode, flags);
+    if ((op_ret < 0) || !local) {
+        goto out;
+    }
+    flags = UP_UPDATE_CLIENT;
+    upcall_cache_invalidate(frame, this, client, local->inode, flags, NULL,
+                            NULL, NULL, NULL);
 
-        /* Need to invalidate old and new parent entries as well */
-        flags = UP_PARENT_DENTRY_FLAGS;
-        CACHE_INVALIDATE_DIR (frame, this, client, local->inode, flags);
+out:
+    UPCALL_STACK_UNWIND(opendir, frame, op_ret, op_errno, fd, xdata);
+
+    return 0;
+}
 
-        /* XXX: notify oldparent as well */
-/*        if (gf_uuid_compare (preoldparent->ia_gfid, prenewparent->ia_gfid))
-                CACHE_INVALIDATE (frame, this, client, prenewparent->ia_gfid, flags);*/
+static int32_t
+up_opendir(call_frame_t *frame, xlator_t *this, loc_t *loc, fd_t *fd,
+           dict_t *xdata)
+{
+    int32_t op_errno = ENOMEM;
+    upcall_local_t *local = NULL;
+
+    EXIT_IF_UPCALL_OFF(this, out);
+
+    local = upcall_local_init(frame, this, NULL, NULL, loc->inode, NULL);
+    if (!local) {
+        goto err;
+    }
 
 out:
-        UPCALL_STACK_UNWIND (rename, frame, op_ret, op_errno,
-                             stbuf, preoldparent, postoldparent,
-                             prenewparent, postnewparent, xdata);
+    STACK_WIND(frame, up_opendir_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->opendir, loc, fd, xdata);
+
+    return 0;
+
+err:
+    UPCALL_STACK_UNWIND(opendir, frame, -1, op_errno, NULL, NULL);
 
-        return 0;
+    return 0;
 }
 
-int
-up_rename (call_frame_t *frame, xlator_t *this,
-          loc_t *oldloc, loc_t *newloc, dict_t *xdata)
+static int32_t
+up_statfs_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int32_t op_ret,
+              int32_t op_errno, struct statvfs *buf, dict_t *xdata)
 {
-        int32_t          op_errno        = -1;
-        upcall_local_t   *local          = NULL;
+    client_t *client = NULL;
+    uint32_t flags = 0;
+    upcall_local_t *local = NULL;
 
-        EXIT_IF_UPCALL_OFF (this, out);
+    EXIT_IF_UPCALL_OFF(this, out);
 
-        local = upcall_local_init (frame, this, oldloc->inode);
-        if (!local) {
-                op_errno = ENOMEM;
-                goto err;
-        }
+    client = frame->root->client;
+    local = frame->local;
+
+    if ((op_ret < 0) || !local) {
+        goto out;
+    }
+    flags = UP_UPDATE_CLIENT;
+    upcall_cache_invalidate(frame, this, client, local->inode, flags, NULL,
+                            NULL, NULL, NULL);
+
+out:
+    UPCALL_STACK_UNWIND(statfs, frame, op_ret, op_errno, buf, xdata);
+
+    return 0;
+}
+
+static int32_t
+up_statfs(call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xdata)
+{
+    int32_t op_errno = ENOMEM;
+    upcall_local_t *local = NULL;
+
+    EXIT_IF_UPCALL_OFF(this, out);
+
+    local = upcall_local_init(frame, this, NULL, NULL, loc->inode, NULL);
+    if (!local) {
+        goto err;
+    }
 
 out:
-        STACK_WIND (frame, up_rename_cbk,
-                    FIRST_CHILD(this), FIRST_CHILD(this)->fops->rename,
-                    oldloc, newloc, xdata);
+    STACK_WIND(frame, up_statfs_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->statfs, loc, xdata);
 
-        return 0;
+    return 0;
 
 err:
-        op_errno = (op_errno == -1) ? errno : op_errno;
-        UPCALL_STACK_UNWIND (rename, frame, -1, op_errno, NULL,
-                             NULL, NULL, NULL, NULL, NULL);
+    UPCALL_STACK_UNWIND(statfs, frame, -1, op_errno, NULL, NULL);
 
-        return 0;
+    return 0;
 }
 
-int
-up_unlink_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                int op_ret, int op_errno, struct iatt *preparent,
-                struct iatt *postparent, dict_t *xdata)
+static int32_t
+up_readdir_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+               int32_t op_ret, int32_t op_errno, gf_dirent_t *entries,
+               dict_t *xdata)
 {
-        client_t         *client        = NULL;
-        uint32_t         flags          = 0;
-        upcall_local_t   *local         = NULL;
+    client_t *client = NULL;
+    uint32_t flags = 0;
+    upcall_local_t *local = NULL;
 
-        EXIT_IF_UPCALL_OFF (this, out);
+    EXIT_IF_UPCALL_OFF(this, out);
 
-        client = frame->root->client;
-        local = frame->local;
+    client = frame->root->client;
+    local = frame->local;
 
-        if ((op_ret < 0) || !local) {
-                goto out;
+    if ((op_ret < 0) || !local) {
+        goto out;
+    }
+    flags = UP_UPDATE_CLIENT;
+    upcall_cache_invalidate(frame, this, client, local->inode, flags, NULL,
+                            NULL, NULL, NULL);
+
+out:
+    UPCALL_STACK_UNWIND(readdir, frame, op_ret, op_errno, entries, xdata);
+
+    return 0;
+}
+
+static int32_t
+up_readdir(call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
+           off_t off, dict_t *xdata)
+{
+    int32_t op_errno = ENOMEM;
+    upcall_local_t *local = NULL;
+
+    EXIT_IF_UPCALL_OFF(this, out);
+
+    local = upcall_local_init(frame, this, NULL, NULL, fd->inode, NULL);
+    if (!local) {
+        goto err;
+    }
+
+out:
+    STACK_WIND(frame, up_readdir_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->readdir, fd, size, off, xdata);
+
+    return 0;
+
+err:
+    UPCALL_STACK_UNWIND(readdir, frame, -1, op_errno, NULL, NULL);
+
+    return 0;
+}
+
+static int32_t
+up_readdirp_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                int32_t op_ret, int32_t op_errno, gf_dirent_t *entries,
+                dict_t *xdata)
+{
+    client_t *client = NULL;
+    uint32_t flags = 0;
+    upcall_local_t *local = NULL;
+    gf_dirent_t *entry = NULL;
+
+    EXIT_IF_UPCALL_OFF(this, out);
+
+    client = frame->root->client;
+    local = frame->local;
+
+    if ((op_ret < 0) || !local) {
+        goto out;
+    }
+    flags = UP_UPDATE_CLIENT;
+    upcall_cache_invalidate(frame, this, client, local->inode, flags, NULL,
+                            NULL, NULL, NULL);
+
+    list_for_each_entry(entry, &entries->list, list)
+    {
+        if (entry->inode == NULL) {
+            continue;
         }
-        flags = UP_NLINK_FLAGS;
-        CACHE_INVALIDATE (frame, this, client, local->inode, flags);
+        upcall_cache_invalidate(frame, this, client, entry->inode, flags,
+                                &entry->d_stat, NULL, NULL, NULL);
+    }
 
-        flags = UP_PARENT_DENTRY_FLAGS;
-        /* invalidate parent's entry too */
-        CACHE_INVALIDATE_DIR (frame, this, client, local->inode, flags);
+out:
+    UPCALL_STACK_UNWIND(readdirp, frame, op_ret, op_errno, entries, xdata);
+
+    return 0;
+}
+
+static int32_t
+up_readdirp(call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
+            off_t off, dict_t *dict)
+{
+    int32_t op_errno = ENOMEM;
+    upcall_local_t *local = NULL;
+
+    EXIT_IF_UPCALL_OFF(this, out);
+
+    local = upcall_local_init(frame, this, NULL, NULL, fd->inode, NULL);
+    if (!local) {
+        goto err;
+    }
 
 out:
-        UPCALL_STACK_UNWIND (unlink, frame, op_ret, op_errno,
-                             preparent, postparent, xdata);
+    STACK_WIND(frame, up_readdirp_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->readdirp, fd, size, off, dict);
+
+    return 0;
+
+err:
+    UPCALL_STACK_UNWIND(readdirp, frame, -1, op_errno, NULL, NULL);
 
-        return 0;
+    return 0;
 }
 
-int
-up_unlink (call_frame_t *frame, xlator_t *this, loc_t *loc, int xflag,
-              dict_t *xdata)
+static int32_t
+up_fsetattr(call_frame_t *frame, xlator_t *this, fd_t *fd, struct iatt *stbuf,
+            int32_t valid, dict_t *xdata)
 {
-        int32_t          op_errno        = -1;
-        upcall_local_t   *local          = NULL;
+    int32_t op_errno = ENOMEM;
+    upcall_local_t *local = NULL;
 
-        EXIT_IF_UPCALL_OFF (this, out);
+    EXIT_IF_UPCALL_OFF(this, out);
 
-        local = upcall_local_init (frame, this, loc->inode);
-        if (!local) {
-                op_errno = ENOMEM;
-                goto err;
-        }
+    local = upcall_local_init(frame, this, NULL, NULL, fd->inode, NULL);
+    if (!local) {
+        goto err;
+    }
 
 out:
-        STACK_WIND (frame, up_unlink_cbk,
-                    FIRST_CHILD(this), FIRST_CHILD(this)->fops->unlink,
-                    loc, xflag, xdata);
+    STACK_WIND(frame, up_setattr_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->fsetattr, fd, stbuf, valid, xdata);
 
-        return 0;
+    return 0;
 
 err:
-        op_errno = (op_errno == -1) ? errno : op_errno;
-        UPCALL_STACK_UNWIND (unlink, frame, -1, op_errno, NULL, NULL, NULL);
+    UPCALL_STACK_UNWIND(fsetattr, frame, -1, op_errno, NULL, NULL, NULL);
 
-        return 0;
+    return 0;
 }
 
-int
-up_link_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                int op_ret, int op_errno, inode_t *inode, struct iatt *stbuf,
-                struct iatt *preparent, struct iatt *postparent, dict_t *xdata)
+static int32_t
+up_fallocate_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                 int32_t op_ret, int32_t op_errno, struct iatt *pre,
+                 struct iatt *post, dict_t *xdata)
 {
-        client_t         *client        = NULL;
-        uint32_t         flags          = 0;
-        upcall_local_t   *local         = NULL;
+    client_t *client = NULL;
+    uint32_t flags = 0;
+    upcall_local_t *local = NULL;
 
-        EXIT_IF_UPCALL_OFF (this, out);
+    EXIT_IF_UPCALL_OFF(this, out);
 
-        client = frame->root->client;
-        local = frame->local;
+    client = frame->root->client;
+    local = frame->local;
 
-        if ((op_ret < 0) || !local) {
-                goto out;
-        }
-        flags = UP_NLINK_FLAGS;
-        CACHE_INVALIDATE (frame, this, client, local->inode, flags);
+    if ((op_ret < 0) || !local) {
+        goto out;
+    }
+    flags = UP_WRITE_FLAGS;
+    upcall_cache_invalidate(frame, this, client, local->inode, flags, post,
+                            NULL, NULL, NULL);
 
-        /* do we need to update parent as well?? */
 out:
-        UPCALL_STACK_UNWIND (link, frame, op_ret, op_errno,
-                             inode, stbuf, preparent, postparent, xdata);
+    UPCALL_STACK_UNWIND(fallocate, frame, op_ret, op_errno, pre, post, xdata);
 
-        return 0;
+    return 0;
 }
 
-int
-up_link (call_frame_t *frame, xlator_t *this, loc_t *oldloc,
-         loc_t *newloc, dict_t *xdata)
+static int32_t
+up_fallocate(call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t mode,
+             off_t offset, size_t len, dict_t *xdata)
 {
-        int32_t          op_errno        = -1;
-        upcall_local_t   *local          = NULL;
+    int32_t op_errno = ENOMEM;
+    upcall_local_t *local = NULL;
 
-        EXIT_IF_UPCALL_OFF (this, out);
+    EXIT_IF_UPCALL_OFF(this, out);
 
-        local = upcall_local_init (frame, this, oldloc->inode);
-        if (!local) {
-                op_errno = ENOMEM;
-                goto err;
-        }
+    local = upcall_local_init(frame, this, NULL, NULL, fd->inode, NULL);
+    if (!local) {
+        goto err;
+    }
 
 out:
-        STACK_WIND (frame, up_link_cbk,
-                    FIRST_CHILD(this), FIRST_CHILD(this)->fops->link,
-                    oldloc, newloc, xdata);
+    STACK_WIND(frame, up_fallocate_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->fallocate, fd, mode, offset, len,
+               xdata);
 
-        return 0;
+    return 0;
 
 err:
-        op_errno = (op_errno == -1) ? errno : op_errno;
-        UPCALL_STACK_UNWIND (link, frame, -1, op_errno, NULL,
-                             NULL, NULL, NULL, NULL);
+    UPCALL_STACK_UNWIND(fallocate, frame, -1, op_errno, NULL, NULL, NULL);
 
-        return 0;
+    return 0;
 }
 
-int
-up_rmdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                int op_ret, int op_errno, struct iatt *preparent,
-                struct iatt *postparent, dict_t *xdata)
+static int32_t
+up_discard_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+               int32_t op_ret, int32_t op_errno, struct iatt *pre,
+               struct iatt *post, dict_t *xdata)
 {
-        client_t         *client        = NULL;
-        uint32_t         flags          = 0;
-        upcall_local_t   *local         = NULL;
+    client_t *client = NULL;
+    uint32_t flags = 0;
+    upcall_local_t *local = NULL;
 
-        EXIT_IF_UPCALL_OFF (this, out);
+    EXIT_IF_UPCALL_OFF(this, out);
 
-        client = frame->root->client;
-        local = frame->local;
+    client = frame->root->client;
+    local = frame->local;
 
-        if ((op_ret < 0) || !local) {
-                goto out;
-        }
-        flags = UP_NLINK_FLAGS;
-        CACHE_INVALIDATE (frame, this, client, local->inode, flags);
+    if ((op_ret < 0) || !local) {
+        goto out;
+    }
+    flags = UP_WRITE_FLAGS;
+    upcall_cache_invalidate(frame, this, client, local->inode, flags, post,
+                            NULL, NULL, NULL);
+
+out:
+    UPCALL_STACK_UNWIND(discard, frame, op_ret, op_errno, pre, post, xdata);
+
+    return 0;
+}
+
+static int32_t
+up_discard(call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
+           size_t len, dict_t *xdata)
+{
+    int32_t op_errno = ENOMEM;
+    upcall_local_t *local = NULL;
 
-        /* invalidate parent's entry too */
-        flags = UP_PARENT_DENTRY_FLAGS;
-        CACHE_INVALIDATE_DIR (frame, this, client, local->inode, flags);
+    EXIT_IF_UPCALL_OFF(this, out);
+
+    local = upcall_local_init(frame, this, NULL, NULL, fd->inode, NULL);
+    if (!local) {
+        goto err;
+    }
 
 out:
-        UPCALL_STACK_UNWIND (rmdir, frame, op_ret, op_errno,
-                             preparent, postparent, xdata);
+    STACK_WIND(frame, up_discard_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->discard, fd, offset, len, xdata);
+
+    return 0;
 
-        return 0;
+err:
+    UPCALL_STACK_UNWIND(discard, frame, -1, op_errno, NULL, NULL, NULL);
+
+    return 0;
 }
 
-int
-up_rmdir (call_frame_t *frame, xlator_t *this, loc_t *loc, int flags,
-              dict_t *xdata)
+static int32_t
+up_zerofill_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                int32_t op_ret, int32_t op_errno, struct iatt *pre,
+                struct iatt *post, dict_t *xdata)
 {
-        int32_t          op_errno        = -1;
-        upcall_local_t   *local          = NULL;
+    client_t *client = NULL;
+    uint32_t flags = 0;
+    upcall_local_t *local = NULL;
 
-        EXIT_IF_UPCALL_OFF (this, out);
+    EXIT_IF_UPCALL_OFF(this, out);
 
-        local = upcall_local_init (frame, this, loc->inode);
-        if (!local) {
-                op_errno = ENOMEM;
-                goto err;
-        }
+    client = frame->root->client;
+    local = frame->local;
+
+    if ((op_ret < 0) || !local) {
+        goto out;
+    }
+    flags = UP_WRITE_FLAGS;
+    upcall_cache_invalidate(frame, this, client, local->inode, flags, post,
+                            NULL, NULL, NULL);
 
 out:
-        STACK_WIND (frame, up_rmdir_cbk,
-                    FIRST_CHILD(this), FIRST_CHILD(this)->fops->rmdir,
-                    loc, flags, xdata);
+    UPCALL_STACK_UNWIND(zerofill, frame, op_ret, op_errno, pre, post, xdata);
+
+    return 0;
+}
 
-        return 0;
+static int
+up_zerofill(call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
+            off_t len, dict_t *xdata)
+{
+    int32_t op_errno = ENOMEM;
+    upcall_local_t *local = NULL;
+
+    EXIT_IF_UPCALL_OFF(this, out);
+
+    local = upcall_local_init(frame, this, NULL, NULL, fd->inode, NULL);
+    if (!local) {
+        goto err;
+    }
+
+out:
+    STACK_WIND(frame, up_zerofill_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->zerofill, fd, offset, len, xdata);
+
+    return 0;
 
 err:
-        op_errno = (op_errno == -1) ? errno : op_errno;
-        UPCALL_STACK_UNWIND (rmdir, frame, -1, op_errno, NULL, NULL, NULL);
+    UPCALL_STACK_UNWIND(zerofill, frame, -1, op_errno, NULL, NULL, NULL);
 
-        return 0;
+    return 0;
 }
 
-int
-up_mkdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                int op_ret, int op_errno, inode_t *inode,
-                struct iatt *stbuf, struct iatt *preparent,
-                struct iatt *postparent, dict_t *xdata)
+static int32_t
+up_seek_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int op_ret,
+            int op_errno, off_t offset, dict_t *xdata)
 {
-        client_t         *client        = NULL;
-        uint32_t         flags          = 0;
-        upcall_local_t   *local         = NULL;
+    client_t *client = NULL;
+    uint32_t flags = 0;
+    upcall_local_t *local = NULL;
 
-        EXIT_IF_UPCALL_OFF (this, out);
+    EXIT_IF_UPCALL_OFF(this, out);
 
-        client = frame->root->client;
-        local = frame->local;
+    client = frame->root->client;
+    local = frame->local;
 
-        if ((op_ret < 0) || !local) {
-                goto out;
-        }
-        flags = UP_NLINK_FLAGS;
-        CACHE_INVALIDATE (frame, this, client, local->inode, flags);
+    if ((op_ret < 0) || !local) {
+        goto out;
+    }
+    flags = UP_UPDATE_CLIENT;
+    upcall_cache_invalidate(frame, this, client, local->inode, flags, NULL,
+                            NULL, NULL, NULL);
+
+out:
+    UPCALL_STACK_UNWIND(seek, frame, op_ret, op_errno, offset, xdata);
+
+    return 0;
+}
+
+static int32_t
+up_seek(call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
+        gf_seek_what_t what, dict_t *xdata)
+{
+    int32_t op_errno = ENOMEM;
+    upcall_local_t *local = NULL;
 
-        /* invalidate parent's entry too */
-        flags = UP_PARENT_DENTRY_FLAGS;
-        CACHE_INVALIDATE_DIR (frame, this, client, local->inode, flags);
+    EXIT_IF_UPCALL_OFF(this, out);
+
+    local = upcall_local_init(frame, this, NULL, NULL, fd->inode, NULL);
+    if (!local) {
+        goto err;
+    }
 
 out:
-        UPCALL_STACK_UNWIND (mkdir, frame, op_ret, op_errno,
-                             inode, stbuf, preparent, postparent, xdata);
+    STACK_WIND(frame, up_seek_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->seek, fd, offset, what, xdata);
+
+    return 0;
+
+err:
+    UPCALL_STACK_UNWIND(seek, frame, -1, op_errno, 0, NULL);
 
-        return 0;
+    return 0;
 }
 
-int
-up_mkdir (call_frame_t *frame, xlator_t *this,
-          loc_t *loc, mode_t mode, mode_t umask, dict_t *params)
+static int32_t
+up_setxattr_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
-        int32_t          op_errno        = -1;
-        upcall_local_t   *local          = NULL;
+    client_t *client = NULL;
+    uint32_t flags = 0;
+    upcall_local_t *local = NULL;
+    int ret = 0;
+    struct iatt stbuf = {
+        0,
+    };
+    upcall_private_t *priv = NULL;
 
-        EXIT_IF_UPCALL_OFF (this, out);
+    EXIT_IF_UPCALL_OFF(this, out);
 
-        local = upcall_local_init (frame, this, loc->inode);
-        if (!local) {
-                op_errno = ENOMEM;
-                goto err;
-        }
+    priv = this->private;
+    GF_VALIDATE_OR_GOTO(this->name, priv, out);
+
+    client = frame->root->client;
+    local = frame->local;
+
+    if ((op_ret < 0) || !local) {
+        goto out;
+    }
+
+    flags = UP_XATTR;
+
+    ret = up_filter_xattr(local->xattr, priv->xattrs);
+    if (ret < 0) {
+        op_ret = ret;
+        goto out;
+    }
+    if (!up_invalidate_needed(local->xattr))
+        goto out;
+
+    ret = dict_get_iatt(xdata, GF_POSTSTAT, &stbuf);
+    if (ret == 0)
+        flags |= UP_TIMES;
+
+    upcall_cache_invalidate(frame, this, client, local->inode, flags, &stbuf,
+                            NULL, NULL, local->xattr);
+
+out:
+    UPCALL_STACK_UNWIND(setxattr, frame, op_ret, op_errno, xdata);
+
+    return 0;
+}
+
+static int32_t
+up_setxattr(call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *dict,
+            int32_t flags, dict_t *xdata)
+{
+    int32_t op_errno = ENOMEM;
+    upcall_local_t *local = NULL;
+
+    EXIT_IF_UPCALL_OFF(this, out);
+
+    local = upcall_local_init(frame, this, loc, NULL, loc->inode, dict);
+    if (!local) {
+        goto err;
+    }
 
 out:
-        STACK_WIND (frame, up_mkdir_cbk,
-                    FIRST_CHILD(this), FIRST_CHILD(this)->fops->mkdir,
-                    loc, mode, umask, params);
+    STACK_WIND(frame, up_setxattr_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->setxattr, loc, dict, flags, xdata);
 
-        return 0;
+    return 0;
 
 err:
-        op_errno = (op_errno == -1) ? errno : op_errno;
-        UPCALL_STACK_UNWIND (mkdir, frame, -1, op_errno, NULL,
-                             NULL, NULL, NULL, NULL);
+    UPCALL_STACK_UNWIND(setxattr, frame, -1, op_errno, NULL);
 
-        return 0;
+    return 0;
 }
 
-int
-up_create_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                int op_ret, int op_errno, fd_t *fd, inode_t *inode,
-                struct iatt *stbuf, struct iatt *preparent,
-                struct iatt *postparent, dict_t *xdata)
+static int32_t
+up_fsetxattr_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                 int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
-        client_t         *client        = NULL;
-        uint32_t         flags          = 0;
-        upcall_local_t   *local         = NULL;
+    client_t *client = NULL;
+    uint32_t flags = 0;
+    upcall_local_t *local = NULL;
+    int ret = 0;
+    struct iatt stbuf = {
+        0,
+    };
+    upcall_private_t *priv = NULL;
 
-        EXIT_IF_UPCALL_OFF (this, out);
+    EXIT_IF_UPCALL_OFF(this, out);
 
-        client = frame->root->client;
-        local = frame->local;
+    priv = this->private;
+    GF_VALIDATE_OR_GOTO(this->name, priv, out);
 
-        if ((op_ret < 0) || !local) {
-                goto out;
-        }
+    client = frame->root->client;
+    local = frame->local;
+
+    if ((op_ret < 0) || !local) {
+        goto out;
+    }
+
+    flags = UP_XATTR;
+
+    ret = up_filter_xattr(local->xattr, priv->xattrs);
+    if (ret < 0) {
+        op_ret = ret;
+        goto out;
+    }
+    if (!up_invalidate_needed(local->xattr))
+        goto out;
+
+    ret = dict_get_iatt(xdata, GF_POSTSTAT, &stbuf);
+    if (ret == 0)
+        flags |= UP_TIMES;
 
-        /* As its a new file create, no need of sending notification */
-        /* However invalidate parent's entry */
-        flags = UP_PARENT_DENTRY_FLAGS;
-        CACHE_INVALIDATE_DIR (frame, this, client, local->inode, flags);
+    upcall_cache_invalidate(frame, this, client, local->inode, flags, &stbuf,
+                            NULL, NULL, local->xattr);
 
 out:
-        UPCALL_STACK_UNWIND (create, frame, op_ret, op_errno, fd,
-                             inode, stbuf, preparent, postparent, xdata);
+    UPCALL_STACK_UNWIND(fsetxattr, frame, op_ret, op_errno, xdata);
 
-        return 0;
+    return 0;
 }
 
-int
-up_create (call_frame_t *frame, xlator_t *this,
-          loc_t *loc, int32_t flags, mode_t mode,
-          mode_t umask, fd_t *fd, dict_t *params)
+static int32_t
+up_fsetxattr(call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *dict,
+             int32_t flags, dict_t *xdata)
 {
-        int32_t          op_errno        = -1;
-        upcall_local_t   *local          = NULL;
+    int32_t op_errno = ENOMEM;
+    upcall_local_t *local = NULL;
 
-        EXIT_IF_UPCALL_OFF (this, out);
+    EXIT_IF_UPCALL_OFF(this, out);
 
-        local = upcall_local_init (frame, this, loc->inode);
+    local = upcall_local_init(frame, this, NULL, fd, fd->inode, dict);
+    if (!local) {
+        goto err;
+    }
 
-        if (!local) {
-                op_errno = ENOMEM;
-                goto err;
-        }
+out:
+    STACK_WIND(frame, up_fsetxattr_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->fsetxattr, fd, dict, flags, xdata);
+
+    return 0;
+
+err:
+    UPCALL_STACK_UNWIND(fsetxattr, frame, -1, op_errno, NULL);
+
+    return 0;
+}
+
+static int32_t
+up_fremovexattr_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                    int32_t op_ret, int32_t op_errno, dict_t *xdata)
+{
+    client_t *client = NULL;
+    uint32_t flags = 0;
+    upcall_local_t *local = NULL;
+    struct iatt stbuf = {
+        0,
+    };
+    int ret = 0;
+    upcall_private_t *priv = NULL;
+
+    EXIT_IF_UPCALL_OFF(this, out);
+
+    priv = this->private;
+    GF_VALIDATE_OR_GOTO(this->name, priv, out);
+
+    client = frame->root->client;
+    local = frame->local;
+
+    if ((op_ret < 0) || !local) {
+        goto out;
+    }
+    flags = UP_XATTR_RM;
+
+    ret = up_filter_xattr(local->xattr, priv->xattrs);
+    if (ret < 0) {
+        op_ret = ret;
+        goto out;
+    }
+    if (!up_invalidate_needed(local->xattr))
+        goto out;
+
+    ret = dict_get_iatt(xdata, GF_POSTSTAT, &stbuf);
+    if (ret == 0)
+        flags |= UP_TIMES;
+
+    upcall_cache_invalidate(frame, this, client, local->inode, flags, &stbuf,
+                            NULL, NULL, local->xattr);
+
+out:
+    UPCALL_STACK_UNWIND(fremovexattr, frame, op_ret, op_errno, xdata);
+    return 0;
+}
+
+static int32_t
+up_fremovexattr(call_frame_t *frame, xlator_t *this, fd_t *fd, const char *name,
+                dict_t *xdata)
+{
+    int32_t op_errno = ENOMEM;
+    upcall_local_t *local = NULL;
+    dict_t *xattr = NULL;
+
+    EXIT_IF_UPCALL_OFF(this, out);
+
+    xattr = dict_for_key_value(name, "", 1, _gf_true);
+    if (!xattr) {
+        goto err;
+    }
+
+    local = upcall_local_init(frame, this, NULL, fd, fd->inode, xattr);
+    if (!local) {
+        goto err;
+    }
 
 out:
-        STACK_WIND (frame, up_create_cbk,
-                    FIRST_CHILD(this), FIRST_CHILD(this)->fops->create,
-                    loc, flags, mode, umask, fd, params);
+    if (xattr)
+        dict_unref(xattr);
 
-        return 0;
+    STACK_WIND(frame, up_fremovexattr_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->fremovexattr, fd, name, xdata);
+    return 0;
 
 err:
-        op_errno = (op_errno == -1) ? errno : op_errno;
-        UPCALL_STACK_UNWIND (create, frame, -1, op_errno, NULL,
-                             NULL, NULL, NULL, NULL, NULL);
+    if (xattr)
+        dict_unref(xattr);
+
+    UPCALL_STACK_UNWIND(fremovexattr, frame, -1, op_errno, NULL);
 
-        return 0;
+    return 0;
 }
 
-int32_t
-mem_acct_init (xlator_t *this)
+static int32_t
+up_removexattr_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                   int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
-        int     ret = -1;
+    client_t *client = NULL;
+    uint32_t flags = 0;
+    upcall_local_t *local = NULL;
+    struct iatt stbuf = {
+        0,
+    };
+    int ret = 0;
+    upcall_private_t *priv = NULL;
+
+    EXIT_IF_UPCALL_OFF(this, out);
+
+    priv = this->private;
+    GF_VALIDATE_OR_GOTO(this->name, priv, out);
+
+    client = frame->root->client;
+    local = frame->local;
+
+    if ((op_ret < 0) || !local) {
+        goto out;
+    }
+    flags = UP_XATTR_RM;
+
+    ret = up_filter_xattr(local->xattr, priv->xattrs);
+    if (ret < 0) {
+        op_ret = ret;
+        goto out;
+    }
+    if (!up_invalidate_needed(local->xattr))
+        goto out;
+
+    ret = dict_get_iatt(xdata, GF_POSTSTAT, &stbuf);
+    if (ret == 0)
+        flags |= UP_TIMES;
+
+    upcall_cache_invalidate(frame, this, client, local->inode, flags, &stbuf,
+                            NULL, NULL, local->xattr);
 
-        if (!this)
-                return ret;
+out:
+    UPCALL_STACK_UNWIND(removexattr, frame, op_ret, op_errno, xdata);
+    return 0;
+}
 
-        ret = xlator_mem_acct_init (this, gf_upcall_mt_end + 1);
+static int32_t
+up_removexattr(call_frame_t *frame, xlator_t *this, loc_t *loc,
+               const char *name, dict_t *xdata)
+{
+    int32_t op_errno = ENOMEM;
+    upcall_local_t *local = NULL;
+    dict_t *xattr = NULL;
 
-        if (ret != 0) {
-                gf_msg ("upcall", GF_LOG_WARNING, 0,
-                        UPCALL_MSG_NO_MEMORY,
-                        "Memory allocation failed");
-                return ret;
-        }
+    EXIT_IF_UPCALL_OFF(this, out);
+
+    xattr = dict_for_key_value(name, "", 1, _gf_true);
+    if (!xattr) {
+        goto err;
+    }
+
+    local = upcall_local_init(frame, this, loc, NULL, loc->inode, xattr);
+    if (!local) {
+        goto err;
+    }
+
+out:
+    if (xattr)
+        dict_unref(xattr);
+
+    STACK_WIND(frame, up_removexattr_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->removexattr, loc, name, xdata);
+    return 0;
+
+err:
+    if (xattr)
+        dict_unref(xattr);
+
+    UPCALL_STACK_UNWIND(removexattr, frame, -1, op_errno, NULL);
+
+    return 0;
+}
+
+static int32_t
+up_fgetxattr_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                 int32_t op_ret, int32_t op_errno, dict_t *dict, dict_t *xdata)
+{
+    client_t *client = NULL;
+    uint32_t flags = 0;
+    upcall_local_t *local = NULL;
+
+    EXIT_IF_UPCALL_OFF(this, out);
+
+    client = frame->root->client;
+    local = frame->local;
+
+    if ((op_ret < 0) || !local) {
+        goto out;
+    }
+
+    flags = UP_UPDATE_CLIENT;
+    upcall_cache_invalidate(frame, this, client, local->inode, flags, NULL,
+                            NULL, NULL, NULL);
+
+out:
+    UPCALL_STACK_UNWIND(fgetxattr, frame, op_ret, op_errno, dict, xdata);
+    return 0;
+}
+
+static int32_t
+up_fgetxattr(call_frame_t *frame, xlator_t *this, fd_t *fd, const char *name,
+             dict_t *xdata)
+{
+    int32_t op_errno = ENOMEM;
+    upcall_local_t *local = NULL;
+
+    EXIT_IF_UPCALL_OFF(this, out);
+
+    local = upcall_local_init(frame, this, NULL, NULL, fd->inode, NULL);
+    if (!local) {
+        goto err;
+    }
+
+out:
+    STACK_WIND(frame, up_fgetxattr_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->fgetxattr, fd, name, xdata);
+    return 0;
+err:
+    UPCALL_STACK_UNWIND(fgetxattr, frame, -1, op_errno, NULL, NULL);
+    return 0;
+}
+
+static int32_t
+up_getxattr_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                int32_t op_ret, int32_t op_errno, dict_t *dict, dict_t *xdata)
+{
+    client_t *client = NULL;
+    uint32_t flags = 0;
+    upcall_local_t *local = NULL;
+
+    EXIT_IF_UPCALL_OFF(this, out);
+
+    client = frame->root->client;
+    local = frame->local;
+
+    if ((op_ret < 0) || !local) {
+        goto out;
+    }
+
+    flags = UP_UPDATE_CLIENT;
+    upcall_cache_invalidate(frame, this, client, local->inode, flags, NULL,
+                            NULL, NULL, NULL);
+
+out:
+    UPCALL_STACK_UNWIND(getxattr, frame, op_ret, op_errno, dict, xdata);
+    return 0;
+}
+
+static int32_t
+up_getxattr(call_frame_t *frame, xlator_t *this, loc_t *loc, const char *name,
+            dict_t *xdata)
+{
+    int32_t op_errno = ENOMEM;
+    upcall_local_t *local = NULL;
+
+    EXIT_IF_UPCALL_OFF(this, out);
+
+    local = upcall_local_init(frame, this, NULL, NULL, loc->inode, NULL);
+    if (!local) {
+        goto err;
+    }
+
+out:
+    STACK_WIND(frame, up_getxattr_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->getxattr, loc, name, xdata);
+    return 0;
+err:
+    UPCALL_STACK_UNWIND(getxattr, frame, -1, op_errno, NULL, NULL);
+    return 0;
+}
 
+/* The xattrops here mainly tracks changes in afr pending xattr.
+ *    1. xattrop doesn't carry info saying post op/pre op.
+ *    2. Pre xattrop will have 0 value for all pending xattrs,
+ *       the cbk of pre xattrop carries the on-disk xattr value.
+ *       Non zero on-disk xattr indicates pending healing.
+ *    3. Post xattrop will either have 0 or 1 as value of pending xattrs,
+ *       0 on success, 1 on failure. But the post xattrop cbk will have
+ *       0 or 1 or any higher value.
+ *       0 - if no healing required*
+ *       1 - if this is the first time pending xattr is being set.
+ *       n - if there is already a pending xattr set, it will increment
+ *       the on-disk value and send that in cbk.
+ * Our aim is to send an invalidation, only the first time a pending
+ * xattr was set on a file. Below are some of the exceptions in handling
+ * xattrop:
+ * - Do not filter unregistered xattrs in the cbk, but in the call path.
+ *   Else, we will be invalidating on every preop, if the file already has
+ *   pending xattr set. Filtering unregistered xattrs on the fop path
+ *   ensures we invalidate only in postop, every time a postop comes with
+ *   pending xattr value 1.
+ * - Consider a brick is down, and the postop sets pending xattrs as long
+ *   as the other brick is down. But we do not want to invalidate every time
+ *   a pending xattr is set, but we want to invalidate only the first time
+ *   a pending xattr is set on any file. Hence, to identify if its the first
+ *   time a pending xattr is set, we compare the value of pending xattrs that
+ *   came in postop and postop cbk, if its same then its the first time.
+ */
+static int32_t
+up_xattrop_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+               int32_t op_ret, int32_t op_errno, dict_t *dict, dict_t *xdata)
+{
+    client_t *client = NULL;
+    upcall_local_t *local = NULL;
+
+    EXIT_IF_UPCALL_OFF(this, out);
+
+    client = frame->root->client;
+    local = frame->local;
+
+    if ((op_ret < 0) || !local) {
+        goto out;
+    }
+
+    if (up_invalidate_needed(local->xattr)) {
+        if (dict_foreach(local->xattr, up_compare_afr_xattr, dict) < 0)
+            goto out;
+
+        upcall_cache_invalidate(frame, this, client, local->inode, UP_XATTR,
+                                NULL, NULL, NULL, local->xattr);
+    }
+out:
+    if (frame->root->op == GF_FOP_FXATTROP) {
+        UPCALL_STACK_UNWIND(fxattrop, frame, op_ret, op_errno, dict, xdata);
+    } else {
+        UPCALL_STACK_UNWIND(xattrop, frame, op_ret, op_errno, dict, xdata);
+    }
+    return 0;
+}
+
+static int32_t
+up_xattrop(call_frame_t *frame, xlator_t *this, loc_t *loc,
+           gf_xattrop_flags_t optype, dict_t *xattr, dict_t *xdata)
+{
+    int32_t op_errno = EINVAL;
+    upcall_local_t *local = NULL;
+    int ret = 0;
+    upcall_private_t *priv = NULL;
+
+    EXIT_IF_UPCALL_OFF(this, out);
+
+    priv = this->private;
+    GF_VALIDATE_OR_GOTO(this->name, priv, out);
+
+    local = upcall_local_init(frame, this, loc, NULL, loc->inode, xattr);
+    if (!local) {
+        op_errno = ENOMEM;
+        goto err;
+    }
+
+    ret = up_filter_xattr(local->xattr, priv->xattrs);
+    if (ret < 0) {
+        goto err;
+    }
+
+out:
+    STACK_WIND(frame, up_xattrop_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->xattrop, loc, optype, xattr, xdata);
+    return 0;
+err:
+    UPCALL_STACK_UNWIND(xattrop, frame, -1, op_errno, NULL, NULL);
+    return 0;
+}
+
+static int32_t
+up_fxattrop(call_frame_t *frame, xlator_t *this, fd_t *fd,
+            gf_xattrop_flags_t optype, dict_t *xattr, dict_t *xdata)
+{
+    int32_t op_errno = EINVAL;
+    upcall_local_t *local = NULL;
+    int ret = 0;
+    upcall_private_t *priv = NULL;
+
+    EXIT_IF_UPCALL_OFF(this, out);
+
+    priv = this->private;
+    GF_VALIDATE_OR_GOTO(this->name, priv, out);
+
+    local = upcall_local_init(frame, this, NULL, fd, fd->inode, xattr);
+    if (!local) {
+        op_errno = ENOMEM;
+        goto err;
+    }
+
+    ret = up_filter_xattr(local->xattr, priv->xattrs);
+    if (ret < 0) {
+        goto err;
+    }
+
+out:
+    STACK_WIND(frame, up_xattrop_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->fxattrop, fd, optype, xattr, xdata);
+    return 0;
+err:
+    STACK_UNWIND_STRICT(fxattrop, frame, -1, op_errno, NULL, NULL);
+    return 0;
+}
+
+int32_t
+mem_acct_init(xlator_t *this)
+{
+    int ret = -1;
+
+    if (!this)
         return ret;
+
+    ret = xlator_mem_acct_init(this, gf_upcall_mt_end + 1);
+
+    if (ret != 0) {
+        gf_msg("upcall", GF_LOG_WARNING, 0, UPCALL_MSG_NO_MEMORY,
+               "Memory allocation failed");
+        return ret;
+    }
+
+    return ret;
 }
 
 void
-upcall_local_wipe (xlator_t *this, upcall_local_t *local)
+upcall_local_wipe(xlator_t *this, upcall_local_t *local)
 {
-        if (local) {
-                inode_unref (local->inode);
-                mem_put (local);
-        }
+    if (local) {
+        inode_unref(local->inode);
+        if (local->xattr)
+            dict_unref(local->xattr);
+        loc_wipe(&local->rename_oldloc);
+        loc_wipe(&local->loc);
+        if (local->fd)
+            fd_unref(local->fd);
+        mem_put(local);
+    }
 }
 
 upcall_local_t *
-upcall_local_init (call_frame_t *frame, xlator_t *this, inode_t *inode)
+upcall_local_init(call_frame_t *frame, xlator_t *this, loc_t *loc, fd_t *fd,
+                  inode_t *inode, dict_t *xattr)
 {
-        upcall_local_t *local = NULL;
+    upcall_local_t *local = NULL;
 
-        local = mem_get0 (THIS->local_pool);
+    GF_VALIDATE_OR_GOTO("upcall", this, out);
+    GF_VALIDATE_OR_GOTO(this->name, frame, out);
+    GF_VALIDATE_OR_GOTO(this->name, inode, out);
 
-        if (!local)
-                goto out;
+    local = mem_get0(THIS->local_pool);
+
+    if (!local)
+        goto out;
 
-        local->inode = inode_ref (inode);
+    local->inode = inode_ref(inode);
+    if (xattr)
+        local->xattr = dict_copy_with_ref(xattr, NULL);
 
-        /* Shall we get inode_ctx and store it here itself? */
-        local->upcall_inode_ctx = upcall_inode_ctx_get (inode, this);
+    if (loc)
+        loc_copy(&local->loc, loc);
+    if (fd)
+        local->fd = fd_ref(fd);
 
-        frame->local = local;
+    frame->local = local;
 
 out:
-        return local;
+    return local;
 }
 
-int
-reconfigure (xlator_t *this, dict_t *options)
+static int32_t
+update_xattrs(dict_t *dict, char *key, data_t *value, void *data)
+{
+    dict_t *xattrs = data;
+    int ret = 0;
+
+    ret = dict_set_int8(xattrs, key, 0);
+    return ret;
+}
+
+int32_t
+up_ipc(call_frame_t *frame, xlator_t *this, int32_t op, dict_t *xdata)
 {
-        upcall_private_t *priv                   = NULL;
-        int              ret                    = -1;
+    upcall_private_t *priv = NULL;
+    int ret = 0;
 
-        priv = this->private;
-        GF_ASSERT (priv);
+    priv = this->private;
+    GF_VALIDATE_OR_GOTO(this->name, priv, out);
 
-        GF_OPTION_RECONF ("cache-invalidation", priv->cache_invalidation_enabled,
-                          options, bool, out);
-        GF_OPTION_RECONF ("cache-invalidation-timeout", priv->cache_invalidation_timeout,
-                          options, int32, out);
+    if (op != GF_IPC_TARGET_UPCALL)
+        goto wind;
+
+    /* TODO: Bz-1371622 Along with the xattrs also store list of clients
+     * that are interested in notifications, so that the notification
+     * can be sent to the clients that have registered.
+     * Once this implemented there can be unregister of xattrs for
+     * notifications. Until then there is no unregister of xattrs*/
+    if (xdata && priv->xattrs) {
+        ret = dict_foreach(xdata, update_xattrs, priv->xattrs);
+    }
 
-        ret = 0;
 out:
-        return ret;
+    STACK_UNWIND_STRICT(ipc, frame, ret, 0, NULL);
+    return 0;
+
+wind:
+    STACK_WIND(frame, default_ipc_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->ipc, op, xdata);
+    return 0;
 }
 
 int
-init (xlator_t *this)
+reconfigure(xlator_t *this, dict_t *options)
 {
-        int                       ret        = -1;
-        upcall_private_t         *priv       = NULL;
+    upcall_private_t *priv = NULL;
+    int ret = -1;
 
-        priv = GF_CALLOC (1, sizeof (*priv),
-                          gf_upcall_mt_private_t);
-        if (!priv) {
-                gf_msg ("upcall", GF_LOG_WARNING, 0,
-                        UPCALL_MSG_NO_MEMORY,
-                        "Memory allocation failed");
-                goto out;
-        }
+    priv = this->private;
+    GF_VALIDATE_OR_GOTO(this->name, priv, out);
 
-        GF_OPTION_INIT ("cache-invalidation", priv->cache_invalidation_enabled,
-                        bool, out);
-        GF_OPTION_INIT ("cache-invalidation-timeout",
-                        priv->cache_invalidation_timeout, int32, out);
+    GF_OPTION_RECONF("cache-invalidation", priv->cache_invalidation_enabled,
+                     options, bool, out);
+    GF_OPTION_RECONF("cache-invalidation-timeout",
+                     priv->cache_invalidation_timeout, options, int32, out);
 
-        this->private = priv;
-        this->local_pool = mem_pool_new (upcall_local_t, 512);
-        ret = 0;
+    ret = 0;
+
+    if (priv->cache_invalidation_enabled && !priv->reaper_init_done) {
+        ret = upcall_reaper_thread_init(this);
 
-out:
         if (ret) {
-                GF_FREE (priv);
+            gf_msg("upcall", GF_LOG_WARNING, 0, UPCALL_MSG_INTERNAL_ERROR,
+                   "reaper_thread creation failed (%s)."
+                   " Disabling cache_invalidation",
+                   strerror(errno));
         }
+        priv->reaper_init_done = _gf_true;
+    }
 
-        return ret;
+out:
+    return ret;
 }
 
 int
-fini (xlator_t *this)
+init(xlator_t *this)
 {
-        upcall_private_t *priv = NULL;
+    int ret = -1;
+    upcall_private_t *priv = NULL;
 
-        priv = this->private;
-        if (!priv) {
-                return 0;
+    priv = GF_CALLOC(1, sizeof(*priv), gf_upcall_mt_private_t);
+    if (!priv)
+        goto out;
+
+    priv->xattrs = dict_new();
+    if (!priv->xattrs)
+        goto out;
+
+    GF_OPTION_INIT("cache-invalidation", priv->cache_invalidation_enabled, bool,
+                   out);
+    GF_OPTION_INIT("cache-invalidation-timeout",
+                   priv->cache_invalidation_timeout, int32, out);
+
+    LOCK_INIT(&priv->inode_ctx_lk);
+    INIT_LIST_HEAD(&priv->inode_ctx_list);
+
+    priv->fini = 0;
+    priv->reaper_init_done = _gf_false;
+
+    this->private = priv;
+    this->local_pool = mem_pool_new(upcall_local_t, 512);
+    ret = 0;
+
+    if (priv->cache_invalidation_enabled) {
+        ret = upcall_reaper_thread_init(this);
+
+        if (ret) {
+            gf_msg("upcall", GF_LOG_WARNING, 0, UPCALL_MSG_INTERNAL_ERROR,
+                   "reaper_thread creation failed (%s)."
+                   " Disabling cache_invalidation",
+                   strerror(errno));
         }
-        this->private = NULL;
+        priv->reaper_init_done = _gf_true;
+    }
+out:
+    if (ret && priv) {
+        if (priv->xattrs)
+            dict_unref(priv->xattrs);
+
+        GF_FREE(priv);
+    }
+
+    return ret;
+}
+
+void
+fini(xlator_t *this)
+{
+    upcall_private_t *priv = NULL;
+
+    priv = this->private;
+    if (!priv) {
+        return;
+    }
+    this->private = NULL;
+
+    priv->fini = 1;
+
+    if (priv->reaper_thr) {
+        gf_thread_cleanup_xint(priv->reaper_thr);
+        priv->reaper_thr = 0;
+        priv->reaper_init_done = _gf_false;
+    }
 
-        GF_FREE (priv);
+    dict_unref(priv->xattrs);
+    LOCK_DESTROY(&priv->inode_ctx_lk);
 
-        return 0;
+    /* Do we need to cleanup the inode_ctxs? IMO not required
+     * as inode_forget would have been done on all the inodes
+     * before calling xlator_fini */
+    GF_FREE(priv);
+
+    if (this->local_pool) {
+        mem_pool_destroy(this->local_pool);
+        this->local_pool = NULL;
+    }
+
+    return;
 }
 
 int
-upcall_forget (xlator_t *this, inode_t *inode)
+upcall_forget(xlator_t *this, inode_t *inode)
 {
-        upcall_cleanup_inode_ctx (this, inode);
-        return 0;
+    upcall_private_t *priv = this->private;
+
+    if (!priv)
+        goto out;
+
+    upcall_cleanup_inode_ctx(this, inode);
+out:
+    return 0;
 }
 
 int
-upcall_release (xlator_t *this, fd_t *fd)
+upcall_release(xlator_t *this, fd_t *fd)
 {
-        return 0;
+    return 0;
 }
 
 int
-notify (xlator_t *this, int32_t event, void *data, ...)
-{
-        int                 ret              = -1;
-        int32_t             val              = 0;
-        notify_event_data_t *notify_event    = NULL;
-        struct gf_upcall    up_req           = {0,};
-        upcall_client_t     *up_client_entry = NULL;
-
-        switch (event) {
-        case GF_EVENT_UPCALL:
-        {
-                gf_log (this->name, GF_LOG_DEBUG, "Upcall Notify event = %d",
-                        event);
-
-                notify_event = (notify_event_data_t *) data;
-                up_client_entry = notify_event->client_entry;
-
-                if (!up_client_entry) {
-                        goto out;
-                }
-
-                up_req.client_uid = up_client_entry->client_uid;
-
-                memcpy (up_req.gfid, notify_event->gfid, 16);
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "Sending notify to the client- %s, gfid - %s",
-                        up_client_entry->client_uid, up_req.gfid);
-
-                switch (notify_event->event_type) {
-                case CACHE_INVALIDATION:
-                        GF_ASSERT (notify_event->extra);
-                        up_req.flags = notify_event->invalidate_flags;
-                        up_req.expire_time_attr = up_client_entry->expire_time_attr;
-                        break;
-                default:
-                        goto out;
-                }
-
-                up_req.event_type = notify_event->event_type;
-
-                ret = default_notify (this, event, &up_req);
-
-                /*
-                 * notify may fail as the client could have been
-                 * dis(re)connected. Cleanup the client entry.
-                 */
-                if (ret < 0)
-                        __upcall_cleanup_client_entry (up_client_entry);
-        }
-        break;
+notify(xlator_t *this, int32_t event, void *data, ...)
+{
+    int ret = -1;
+    struct gf_upcall *up_req = NULL;
+
+    switch (event) {
+        case GF_EVENT_UPCALL: {
+            gf_log(this->name, GF_LOG_DEBUG, "Upcall Notify event = %d", event);
+
+            up_req = (struct gf_upcall *)data;
+
+            GF_VALIDATE_OR_GOTO(this->name, up_req, out);
+
+            ret = default_notify(this, event, up_req);
+
+            if (ret) {
+                gf_msg(this->name, GF_LOG_WARNING, 0, UPCALL_MSG_NOTIFY_FAILED,
+                       "Failed to notify cache invalidation"
+                       " to client(%s)",
+                       up_req->client_uid);
+                goto out;
+            }
+        } break;
         default:
-                default_notify (this, event, data);
-        break;
-        }
-        ret = 0;
+            default_notify(this, event, data);
+            break;
+    }
+    ret = 0;
 
 out:
-        return ret;
+    return ret;
 }
 
 struct xlator_fops fops = {
-        .open        = up_open,
-        .readv       = up_readv,
-        .writev      = up_writev,
-        .truncate    = up_truncate,
-        .lk          = up_lk,
-        .setattr     = up_setattr,
-        .rename      = up_rename,
-        .unlink      = up_unlink, /* invalidate both file and parent dir */
-        .rmdir       = up_rmdir, /* same as above */
-        .link        = up_link, /* invalidate both file and parent dir */
-        .create      = up_create, /* update only direntry */
-        .mkdir       = up_mkdir, /* update only dirent */
-#ifdef WIP
-        .ftruncate   = up_ftruncate, /* reqd? */
-        .getattr     = up_getattr, /* ?? */
-        .getxattr    = up_getxattr, /* ?? */
-        .access      = up_access,
-        .lookup      = up_lookup,
-        .symlink     = up_symlink, /* invalidate both file and parent dir maybe */
-        .readlink    = up_readlink, /* Needed? readlink same as read? */
-        .readdirp    = up_readdirp,
-        .readdir     = up_readdir,
-/*  other fops to be considered - Bug1200271
- *   lookup, stat, opendir, readdir, readdirp, readlink, mknod, statfs, flush,
- *   fsync, mknod, fsyncdir, setxattr, removexattr, rchecksum, fallocate, discard,
- *   zerofill, (also variants of above similar to fsetattr)
- */
+    .ipc = up_ipc,
+    /* fops which change only "ATIME" do not result
+     * in any cache invalidation. Hence upcall
+     * notifications are not sent in this case.
+     * But however, we need to store/update the
+     * client info in the upcall state to be able
+     * to notify them in case of any changes done
+     * to the data.
+     *
+     * Below such fops do not trigger upcall
+     * notifications but will add/update
+     * clients info in the upcall inode ctx.*/
+    .lookup = up_lookup,
+    .open = up_open,
+    .statfs = up_statfs,
+    .opendir = up_opendir,
+    .readdir = up_readdir,
+    .readdirp = up_readdirp,
+    .stat = up_stat,
+    .fstat = up_fstat,
+    .access = up_access,
+    .readlink = up_readlink,
+    .readv = up_readv,
+    .lk = up_lk,
+    .seek = up_seek,
+
+    /* fops doing  write */
+    .truncate = up_truncate,
+    .ftruncate = up_ftruncate,
+    .writev = up_writev,
+    .zerofill = up_zerofill,
+    .fallocate = up_fallocate,
+    .discard = up_discard,
+
+    /* fops changing attributes */
+    .fsetattr = up_fsetattr,
+    .setattr = up_setattr,
+
+    /* fops affecting parent dirent */
+    .mknod = up_mknod,
+    .create = up_create,
+    .symlink = up_symlink,
+    .mkdir = up_mkdir,
+
+    /* fops affecting both file and parent
+     * cache entries */
+    .unlink = up_unlink,
+    .link = up_link,
+    .rmdir = up_rmdir,
+    .rename = up_rename,
+
+    .setxattr = up_setxattr,
+    .fsetxattr = up_fsetxattr,
+    .getxattr = up_getxattr,
+    .fgetxattr = up_fgetxattr,
+    .fremovexattr = up_fremovexattr,
+    .removexattr = up_removexattr,
+    .xattrop = up_xattrop,
+    .fxattrop = up_fxattrop,
+
+#ifdef NOT_SUPPORTED
+    /* internal lk fops */
+    .inodelk = up_inodelk,
+    .finodelk = up_finodelk,
+    .entrylk = up_entrylk,
+    .fentrylk = up_fentrylk,
+
+    /* Below fops follow 'WRITE' which
+     * would have already sent upcall
+     * notifications */
+    .flush = up_flush,
+    .fsync = up_fsync,
+    .fsyncdir = up_fsyncdir,
 #endif
 };
 
 struct xlator_cbks cbks = {
-        .forget = upcall_forget,
-        .release = upcall_release,
+    .forget = upcall_forget,
+    .release = upcall_release,
 };
 
 struct volume_options options[] = {
-        { .key  = {"cache-invalidation"},
-          .type = GF_OPTION_TYPE_BOOL,
-          .default_value = "off",
-          .description = "When \"on\", sends cache-invalidation"
-                         " notifications."
-        },
-        { .key  = {"cache-invalidation-timeout"},
-          .type = GF_OPTION_TYPE_INT,
-          .default_value = CACHE_INVALIDATION_TIMEOUT,
-          .description = "After 'timeout' seconds since the time"
-                         " client accessed any file, cache-invalidation"
-                         " notifications are no longer sent to that client."
-        },
-        { .key = {NULL} },
+    {
+        .key = {"cache-invalidation"},
+        .type = GF_OPTION_TYPE_BOOL,
+        .default_value = "off",
+        .description = "When \"on\", sends cache-invalidation"
+                       " notifications.",
+        .op_version = {GD_OP_VERSION_3_7_0},
+        .flags = OPT_FLAG_SETTABLE | OPT_FLAG_DOC,
+        .tags = {"cache", "cacheconsistency", "upcall"},
+    },
+    {.key = {"cache-invalidation-timeout"},
+     .type = GF_OPTION_TYPE_INT,
+     .default_value = CACHE_INVALIDATION_TIMEOUT,
+     .description = "After 'timeout' seconds since the time"
+                    " client accessed any file, cache-invalidation"
+                    " notifications are no longer sent to that client.",
+     .op_version = {GD_OP_VERSION_3_7_0},
+     .flags = OPT_FLAG_SETTABLE | OPT_FLAG_DOC,
+     .tags = {"cache", "cachetimeout", "upcall"}},
+    {.key = {NULL}},
+};
+
+xlator_api_t xlator_api = {
+    .init = init,
+    .fini = fini,
+    .notify = notify,
+    .reconfigure = reconfigure,
+    .mem_acct_init = mem_acct_init,
+    .op_version = {1}, /* Present from the initial version */
+    .fops = &fops,
+    .cbks = &cbks,
+    .options = options,
+    .identifier = "upcall",
+    .category = GF_MAINTAINED,
 };
diff --git a/xlators/features/upcall/src/upcall.h b/xlators/features/upcall/src/upcall.h
index 7cbec22bbf2..aa535088ad7 100644
--- a/xlators/features/upcall/src/upcall.h
+++ b/xlators/features/upcall/src/upcall.h
@@ -10,130 +10,122 @@
 #ifndef __UPCALL_H__
 #define __UPCALL_H__
 
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
-
-#include "compat-errno.h"
-#include "stack.h"
-#include "call-stub.h"
+#include <glusterfs/compat-errno.h>
 #include "upcall-mem-types.h"
-#include "client_t.h"
-#include "rpcsvc.h"
-#include "lkowner.h"
-#include "locking.h"
+#include <glusterfs/client_t.h>
 #include "upcall-messages.h"
 #include "upcall-cache-invalidation.h"
-
-#define EXIT_IF_UPCALL_OFF(this, label) do {                       \
-        if (!is_upcall_enabled(this))                              \
-                goto label;                                        \
-} while (0)
-
-#define UPCALL_STACK_UNWIND(fop, frame, params ...) do {        \
-        upcall_local_t *__local = NULL;                         \
-        xlator_t *__xl          = NULL;                         \
-        if (frame) {                                            \
-                        __xl         = frame->this;             \
-                        __local      = frame->local;            \
-                        frame->local = NULL;                    \
-        }                                                       \
-        STACK_UNWIND_STRICT (fop, frame, params);               \
-        upcall_local_wipe (__xl, __local);                      \
-} while (0)
-
-#define UPCALL_STACK_DESTROY(frame) do {                   \
-                upcall_local_t *__local = NULL;            \
-                xlator_t    *__xl    = NULL;               \
-                __xl                 = frame->this;        \
-                __local              = frame->local;       \
-                frame->local         = NULL;               \
-                STACK_DESTROY (frame->root);               \
-                upcall_local_wipe (__xl, __local);         \
-} while (0)
-
-struct _upcall_private_t {
-        gf_boolean_t     cache_invalidation_enabled;
-        int32_t          cache_invalidation_timeout;
-};
-typedef struct _upcall_private_t upcall_private_t;
-
-enum _upcall_event_type_t {
-        EVENT_NULL,
-        CACHE_INVALIDATION,
+#include <glusterfs/upcall-utils.h>
+
+#define EXIT_IF_UPCALL_OFF(this, label)                                        \
+    do {                                                                       \
+        if (!is_upcall_enabled(this))                                          \
+            goto label;                                                        \
+    } while (0)
+
+#define UPCALL_STACK_UNWIND(fop, frame, params...)                             \
+    do {                                                                       \
+        upcall_local_t *__local = NULL;                                        \
+        xlator_t *__xl = NULL;                                                 \
+        if (frame) {                                                           \
+            __xl = frame->this;                                                \
+            __local = frame->local;                                            \
+            frame->local = NULL;                                               \
+        }                                                                      \
+        STACK_UNWIND_STRICT(fop, frame, params);                               \
+        upcall_local_wipe(__xl, __local);                                      \
+    } while (0)
+
+#define UPCALL_STACK_DESTROY(frame)                                            \
+    do {                                                                       \
+        upcall_local_t *__local = NULL;                                        \
+        xlator_t *__xl = NULL;                                                 \
+        __xl = frame->this;                                                    \
+        __local = frame->local;                                                \
+        frame->local = NULL;                                                   \
+        STACK_DESTROY(frame->root);                                            \
+        upcall_local_wipe(__xl, __local);                                      \
+    } while (0)
+
+struct _upcall_private {
+    gf_boolean_t cache_invalidation_enabled;
+    int32_t cache_invalidation_timeout;
+    struct list_head inode_ctx_list;
+    gf_lock_t inode_ctx_lk;
+    gf_boolean_t reaper_init_done;
+    pthread_t reaper_thr;
+    int32_t fini;
+    dict_t *xattrs; /* list of xattrs registered by clients
+                       for receiving invalidation */
 };
-typedef enum _upcall_event_type_t upcall_event_type_t;
-
-struct _upcall_client_t {
-        struct list_head client_list;
-        /* strdup to store client_uid, strdup. Free it explicitly */
-        char *client_uid;
-        time_t access_time; /* time last accessed */
-        /* the amount of time which client can cache this entry */
-        uint32_t expire_time_attr;
+typedef struct _upcall_private upcall_private_t;
+
+struct _upcall_client {
+    struct list_head client_list;
+    /* strdup to store client_uid, strdup. Free it explicitly */
+    char *client_uid;
+    time_t access_time; /* time last accessed */
+    /* the amount of time which client can cache this entry */
+    uint32_t expire_time_attr;
 };
-typedef struct _upcall_client_t upcall_client_t;
+typedef struct _upcall_client upcall_client_t;
 
 /* Upcall entries are maintained in inode_ctx */
-struct _upcall_inode_ctx_t {
-        struct list_head client_list;
-        pthread_mutex_t client_list_lock; /* mutex for clients list
-                                                of this upcall entry */
-};
-typedef struct _upcall_inode_ctx_t upcall_inode_ctx_t;
-
-struct _notify_event_data {
-        uuid_t gfid;
-        upcall_client_t *client_entry;
-        upcall_event_type_t event_type;
-        uint32_t invalidate_flags;
-        /* any extra data needed, like inode flags
-         * to be invalidated incase of cache invalidation,
-         * may be fd for lease recalls */
-        void *extra;
+struct _upcall_inode_ctx {
+    struct list_head inode_ctx_list;
+    struct list_head client_list;
+    pthread_mutex_t client_list_lock; /* mutex for clients list
+                                         of this upcall entry */
+    int destroy;
+    uuid_t gfid; /* gfid of the entry */
 };
-typedef struct _notify_event_data notify_event_data_t;
+typedef struct _upcall_inode_ctx upcall_inode_ctx_t;
 
 struct upcall_local {
-        /* XXX: need to check if we can store
-         * pointers in 'local' which may get freed
-         * in future by other thread
-         */
-        upcall_inode_ctx_t *upcall_inode_ctx;
-        inode_t   *inode;
+    /* XXX: need to check if we can store
+     * pointers in 'local' which may get freed
+     * in future by other thread
+     */
+    inode_t *inode;
+    loc_t rename_oldloc;
+    loc_t loc; /* required for stat in *xattr_cbk */
+    fd_t *fd;  /* required for fstat in *xattr_cbk */
+    dict_t *xattr;
 };
 typedef struct upcall_local upcall_local_t;
 
-void upcall_local_wipe (xlator_t *this, upcall_local_t *local);
-upcall_local_t *upcall_local_init (call_frame_t *frame, xlator_t *this, inode_t *inode);
-
-upcall_client_t *add_upcall_client (call_frame_t *frame, uuid_t gfid,
-                                    client_t *client,
-                                    upcall_inode_ctx_t *up_inode_ctx);
-upcall_client_t *__add_upcall_client (call_frame_t *frame, uuid_t gfid,
-                                      client_t *client,
-                                      upcall_inode_ctx_t *up_inode_ctx);
-upcall_client_t *__get_upcall_client (call_frame_t *frame, uuid_t gfid,
-                                      client_t *client,
-                                      upcall_inode_ctx_t *up_inode_ctx);
-int __upcall_cleanup_client_entry (upcall_client_t *up_client);
-
-int __upcall_inode_ctx_set (inode_t *inode, xlator_t *this);
-upcall_inode_ctx_t *__upcall_inode_ctx_get (inode_t *inode, xlator_t *this);
-upcall_inode_ctx_t *upcall_inode_ctx_get (inode_t *inode, xlator_t *this);
-int upcall_cleanup_inode_ctx (xlator_t *this, inode_t *inode);
-void upcall_cache_forget (xlator_t *this, inode_t *inode,
-                          upcall_inode_ctx_t *up_inode_ctx);
+void
+upcall_local_wipe(xlator_t *this, upcall_local_t *local);
+upcall_local_t *
+upcall_local_init(call_frame_t *frame, xlator_t *this, loc_t *loc, fd_t *fd,
+                  inode_t *inode, dict_t *xattr);
+
+upcall_inode_ctx_t *
+upcall_inode_ctx_get(inode_t *inode, xlator_t *this);
+int
+upcall_cleanup_inode_ctx(xlator_t *this, inode_t *inode);
+
+void *
+upcall_reaper_thread(void *data);
+int
+upcall_reaper_thread_init(xlator_t *this);
 
 /* Xlator options */
-gf_boolean_t is_upcall_enabled(xlator_t *this);
+gf_boolean_t
+is_upcall_enabled(xlator_t *this);
 
 /* Cache invalidation specific */
-void upcall_cache_invalidate (call_frame_t *frame, xlator_t *this, client_t *client,
-                              inode_t *inode, uint32_t flags);
-void upcall_client_cache_invalidate (xlator_t *xl, uuid_t gfid,
-                                     upcall_client_t *up_client_entry,
-                                     uint32_t flags);
-
+void
+upcall_cache_invalidate(call_frame_t *frame, xlator_t *this, client_t *client,
+                        inode_t *inode, uint32_t flags, struct iatt *stbuf,
+                        struct iatt *p_stbuf, struct iatt *oldp_stbuf,
+                        dict_t *xattr);
+int
+up_filter_xattr(dict_t *xattr, dict_t *regd_xattrs);
+
+int
+up_compare_afr_xattr(dict_t *d, char *k, data_t *v, void *tmp);
+
+gf_boolean_t
+up_invalidate_needed(dict_t *xattrs);
 #endif /* __UPCALL_H__ */
diff --git a/xlators/features/utime/Makefile.am b/xlators/features/utime/Makefile.am
new file mode 100644
index 00000000000..a985f42a877
--- /dev/null
+++ b/xlators/features/utime/Makefile.am
@@ -0,0 +1,3 @@
+SUBDIRS = src
+
+CLEANFILES =
diff --git a/xlators/features/utime/src/Makefile.am b/xlators/features/utime/src/Makefile.am
new file mode 100644
index 00000000000..7c3adbc2195
--- /dev/null
+++ b/xlators/features/utime/src/Makefile.am
@@ -0,0 +1,41 @@
+xlator_LTLIBRARIES = utime.la
+xlatordir = $(libdir)/glusterfs/$(PACKAGE_VERSION)/xlator/features
+
+UTIME_SRC = $(top_srcdir)/xlators/features/utime/src
+
+utime_sources  = $(UTIME_SRC)/utime-helpers.c
+utime_sources += $(UTIME_SRC)/utime.c
+
+utime_la_SOURCES = $(utime_sources)
+nodist_utime_la_SOURCES = utime-autogen-fops.c utime-autogen-fops.h
+BUILT_SOURCES = utime-autogen-fops.h
+
+utime_la_LDFLAGS = -module $(GF_XLATOR_DEFAULT_LDFLAGS)
+utime_la_LIBADD = $(top_builddir)/libglusterfs/src/libglusterfs.la
+
+noinst_HEADERS_utime  = $(UTIME_SRC)/utime-helpers.h
+noinst_HEADERS_utime += $(UTIME_SRC)/utime.h
+noinst_HEADERS_utime += $(UTIME_SRC)/utime-messages.h
+noinst_HEADERS_utime += $(UTIME_SRC)/utime-mem-types.h
+noinst_HEADERS = $(top_srcdir)/xlators/lib/src/libxlator.h
+noinst_HEADERS += $(noinst_HEADERS_utime)
+
+AM_CPPFLAGS = $(GF_CPPFLAGS) -I$(top_srcdir)/libglusterfs/src \
+	-I$(top_srcdir)/rpc/xdr/src -I$(top_builddir)/rpc/xdr/src \
+	-I$(top_srcdir)/xlators/lib/src
+
+AM_CFLAGS = -Wall $(GF_CFLAGS)
+
+noinst_PYTHON	= utime-gen-fops-c.py utime-gen-fops-h.py
+EXTRA_DIST	= utime-autogen-fops-tmpl.c utime-autogen-fops-tmpl.h
+
+utime-autogen-fops.c: utime-gen-fops-c.py utime-autogen-fops-tmpl.c
+	$(PYTHON) $(UTIME_SRC)/utime-gen-fops-c.py $(UTIME_SRC)/utime-autogen-fops-tmpl.c > $@
+
+utime-autogen-fops.h: utime-gen-fops-h.py utime-autogen-fops-tmpl.h
+	$(PYTHON) $(UTIME_SRC)/utime-gen-fops-h.py $(UTIME_SRC)/utime-autogen-fops-tmpl.h > $@
+
+CLEANFILES = $(nodist_utime_la_SOURCES)
+
+uninstall-local:
+	rm -f $(DESTDIR)$(xlatordir)/utime.so
diff --git a/xlators/features/utime/src/utime-autogen-fops-tmpl.c b/xlators/features/utime/src/utime-autogen-fops-tmpl.c
new file mode 100644
index 00000000000..f2f35322926
--- /dev/null
+++ b/xlators/features/utime/src/utime-autogen-fops-tmpl.c
@@ -0,0 +1,28 @@
+/*
+  Copyright (c) 2018 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
+*/
+
+/* File: utime-autogen-fops-tmpl.c
+ * This file contains the utime autogenerated FOPs. This is run through
+ * the code generator, generator.py to generate the required FOPs.
+ */
+
+#ifndef _CONFIG_H
+#define _CONFIG_H
+#include "config.h"
+#endif
+
+#include <glusterfs/glusterfs.h>
+#include <glusterfs/xlator.h>
+#include <glusterfs/logging.h>
+#include <glusterfs/statedump.h>
+#include "utime-helpers.h"
+#include <glusterfs/timespec.h>
+
+#pragma generate
diff --git a/xlators/features/utime/src/utime-autogen-fops-tmpl.h b/xlators/features/utime/src/utime-autogen-fops-tmpl.h
new file mode 100644
index 00000000000..4e102ffed6c
--- /dev/null
+++ b/xlators/features/utime/src/utime-autogen-fops-tmpl.h
@@ -0,0 +1,22 @@
+/*
+  Copyright (c) 2018 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
+*/
+
+/* File: utime-autogen-fops-tmpl.h
+ * This file contains the utime autogenerated FOPs declarations.
+ */
+
+#ifndef _UTIME_AUTOGEN_FOPS_H
+#define _UTIME_AUTOGEN_FOPS_H
+
+#include <glusterfs/xlator.h>
+
+#pragma generate
+
+#endif /* _UTIME_AUTOGEN_FOPS_H */
diff --git a/xlators/features/utime/src/utime-gen-fops-c.py b/xlators/features/utime/src/utime-gen-fops-c.py
new file mode 100755
index 00000000000..9fb3e1b8b1a
--- /dev/null
+++ b/xlators/features/utime/src/utime-gen-fops-c.py
@@ -0,0 +1,147 @@
+#!/usr/bin/python3
+
+from __future__ import print_function
+import os
+import sys
+
+curdir = os.path.dirname(sys.argv[0])
+gendir = os.path.join(curdir, '../../../../libglusterfs/src')
+sys.path.append(gendir)
+from generator import ops, fop_subs, cbk_subs, generate
+
+FOPS_COMMON_TEMPLATE = """
+int32_t
+gf_utime_@NAME@ (call_frame_t *frame, xlator_t *this,
+                @LONG_ARGS@)
+{
+        gl_timespec_get(&frame->root->ctime);
+
+        (void) utime_update_attribute_flags(frame, this, GF_FOP_@UPNAME@);
+        STACK_WIND (frame, gf_utime_@NAME@_cbk, FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->@NAME@, @SHORT_ARGS@);
+        return 0;
+}
+"""
+
+FOPS_CBK_COMMON_TEMPLATE = """
+int32_t
+gf_utime_@NAME@_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                     int32_t op_ret, int32_t op_errno,
+                    @LONG_ARGS@)
+{
+        STACK_UNWIND_STRICT (@NAME@, frame, op_ret, op_errno, @SHORT_ARGS@);
+        return 0;
+}
+"""
+
+FOPS_READ_TEMPLATE = """
+int32_t
+gf_utime_@NAME@ (call_frame_t *frame, xlator_t *this,
+                @LONG_ARGS@)
+{
+        gl_timespec_get(&frame->root->ctime);
+
+        (void) utime_update_attribute_flags(frame, this, GF_FOP_READ);
+        STACK_WIND (frame, gf_utime_@NAME@_cbk, FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->@NAME@, @SHORT_ARGS@);
+        return 0;
+}
+"""
+
+FOPS_WRITE_TEMPLATE = """
+int32_t
+gf_utime_@NAME@ (call_frame_t *frame, xlator_t *this,
+                @LONG_ARGS@)
+{
+        gl_timespec_get(&frame->root->ctime);
+
+        (void) utime_update_attribute_flags(frame, this, GF_FOP_WRITE);
+        STACK_WIND (frame, gf_utime_@NAME@_cbk, FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->@NAME@, @SHORT_ARGS@);
+        return 0;
+}
+"""
+
+FOPS_COPY_FILE_RANGE_TEMPLATE = """
+int32_t
+gf_utime_@NAME@ (call_frame_t *frame, xlator_t *this,
+                @LONG_ARGS@)
+{
+        gl_timespec_get(&frame->root->ctime);
+
+        (void) utime_update_attribute_flags(frame, this, GF_FOP_COPY_FILE_RANGE);
+        STACK_WIND (frame, gf_utime_@NAME@_cbk, FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->@NAME@, @SHORT_ARGS@);
+        return 0;
+}
+"""
+
+FOPS_SETATTR_TEMPLATE = """
+int32_t
+gf_utime_@NAME@ (call_frame_t *frame, xlator_t *this,
+             @LONG_ARGS@)
+{
+        gl_timespec_get(&frame->root->ctime);
+
+        if (!valid) {
+                frame->root->flags |= MDATA_CTIME;
+        }
+
+        if (valid & (GF_SET_ATTR_UID | GF_SET_ATTR_GID)) {
+                frame->root->flags |= MDATA_CTIME;
+        }
+
+        if (valid & GF_SET_ATTR_MODE) {
+                frame->root->flags |= MDATA_CTIME;
+        }
+
+        if (valid & (GF_SET_ATTR_ATIME | GF_SET_ATTR_MTIME)) {
+            if (valid & GF_ATTR_ATIME_NOW) {
+                frame->root->ctime.tv_sec = stbuf->ia_atime;
+                frame->root->ctime.tv_nsec = stbuf->ia_atime_nsec;
+            } else if (valid & GF_ATTR_MTIME_NOW) {
+                frame->root->ctime.tv_sec = stbuf->ia_mtime;
+                frame->root->ctime.tv_nsec = stbuf->ia_mtime_nsec;
+            }
+        }
+
+        STACK_WIND (frame, gf_utime_@NAME@_cbk, FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->@NAME@, @SHORT_ARGS@);
+        return 0;
+}
+"""
+
+utime_ops = ['fallocate', 'zerofill', 'opendir', 'mknod', 'mkdir',
+             'unlink', 'rmdir', 'symlink', 'rename', 'link', 'truncate',
+             'ftruncate', 'create', 'open', 'removexattr', 'fremovexattr']
+
+utime_read_op = ['readv']
+utime_write_op = ['writev']
+utime_setattr_ops = ['setattr', 'fsetattr']
+utime_copy_file_range_ops = ['copy_file_range']
+
+def gen_defaults():
+    for name in ops:
+        if name in utime_ops:
+            print(generate(FOPS_CBK_COMMON_TEMPLATE, name, cbk_subs))
+            print(generate(FOPS_COMMON_TEMPLATE, name, fop_subs))
+        if name in utime_read_op:
+            print(generate(FOPS_CBK_COMMON_TEMPLATE, name, cbk_subs))
+            print(generate(FOPS_READ_TEMPLATE, name, fop_subs))
+        if name in utime_write_op:
+            print(generate(FOPS_CBK_COMMON_TEMPLATE, name, cbk_subs))
+            print(generate(FOPS_WRITE_TEMPLATE, name, fop_subs))
+        if name in utime_setattr_ops:
+            print(generate(FOPS_CBK_COMMON_TEMPLATE, name, cbk_subs))
+            print(generate(FOPS_SETATTR_TEMPLATE, name, fop_subs))
+        if name in utime_copy_file_range_ops:
+            print(generate(FOPS_CBK_COMMON_TEMPLATE, name, cbk_subs))
+            print(generate(FOPS_COPY_FILE_RANGE_TEMPLATE, name, fop_subs))
+
+for l in open(sys.argv[1], 'r').readlines():
+    if l.find('#pragma generate') != -1:
+        print("/* BEGIN GENERATED CODE - DO NOT MODIFY */")
+        gen_defaults()
+        print("/* END GENERATED CODE */")
+    else:
+        print(l[:-1])
diff --git a/xlators/features/utime/src/utime-gen-fops-h.py b/xlators/features/utime/src/utime-gen-fops-h.py
new file mode 100755
index 00000000000..e96274c229a
--- /dev/null
+++ b/xlators/features/utime/src/utime-gen-fops-h.py
@@ -0,0 +1,35 @@
+#!/usr/bin/python3
+
+from __future__ import print_function
+import os
+import sys
+
+curdir = os.path.dirname(sys.argv[0])
+gendir = os.path.join(curdir, '../../../../libglusterfs/src')
+sys.path.append(gendir)
+from generator import ops, fop_subs, generate
+
+OP_FOP_TEMPLATE = """
+int32_t
+gf_utime_@NAME@ (call_frame_t *frame, xlator_t *this,
+                @LONG_ARGS@);
+"""
+
+utime_ops = ['fallocate', 'zerofill', 'opendir', 'mknod', 'mkdir',
+             'unlink', 'rmdir', 'symlink', 'rename', 'link', 'truncate',
+             'ftruncate', 'create', 'open', 'removexattr', 'fremovexattr',
+             'readv', 'writev', 'setattr', 'fsetattr', 'copy_file_range']
+
+def gen_defaults():
+    for name, value in ops.items():
+        if name in utime_ops:
+            print(generate(OP_FOP_TEMPLATE, name, fop_subs))
+
+
+for l in open(sys.argv[1], 'r').readlines():
+    if l.find('#pragma generate') != -1:
+        print("/* BEGIN GENERATED CODE - DO NOT MODIFY */")
+        gen_defaults()
+        print("/* END GENERATED CODE */")
+    else:
+        print(l[:-1])
diff --git a/xlators/features/utime/src/utime-helpers.c b/xlators/features/utime/src/utime-helpers.c
new file mode 100644
index 00000000000..29d9ad93561
--- /dev/null
+++ b/xlators/features/utime/src/utime-helpers.c
@@ -0,0 +1,110 @@
+/*
+  Copyright (c) 2018 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
+*/
+
+#include "utime-helpers.h"
+#include "utime.h"
+
+void
+gl_timespec_get(struct timespec *ts)
+{
+#ifdef TIME_UTC
+    timespec_get(ts, TIME_UTC);
+#else
+    timespec_now_realtime(ts);
+#endif
+}
+
+void
+utime_update_attribute_flags(call_frame_t *frame, xlator_t *this,
+                             glusterfs_fop_t fop)
+{
+    utime_priv_t *utime_priv = NULL;
+
+    if (!frame || !this) {
+        goto out;
+    }
+
+    utime_priv = this->private;
+
+    switch (fop) {
+        case GF_FOP_SETXATTR:
+        case GF_FOP_FSETXATTR:
+            frame->root->flags |= MDATA_CTIME;
+            break;
+
+        case GF_FOP_FALLOCATE:
+        case GF_FOP_ZEROFILL:
+            frame->root->flags |= MDATA_MTIME;
+            frame->root->flags |= MDATA_ATIME;
+            break;
+
+        case GF_FOP_OPENDIR:
+        case GF_FOP_OPEN:
+        case GF_FOP_READ:
+            if (!utime_priv->noatime) {
+                frame->root->flags |= MDATA_ATIME;
+            }
+            break;
+        case GF_FOP_MKNOD:
+        case GF_FOP_MKDIR:
+        case GF_FOP_SYMLINK:
+        case GF_FOP_CREATE:
+            frame->root->flags |= MDATA_ATIME;
+            frame->root->flags |= MDATA_CTIME;
+            frame->root->flags |= MDATA_MTIME;
+            frame->root->flags |= MDATA_PAR_CTIME;
+            frame->root->flags |= MDATA_PAR_MTIME;
+            break;
+
+        case GF_FOP_UNLINK:
+        case GF_FOP_RMDIR:
+            frame->root->flags |= MDATA_CTIME;
+            frame->root->flags |= MDATA_PAR_CTIME;
+            frame->root->flags |= MDATA_PAR_MTIME;
+            break;
+
+        case GF_FOP_WRITE:
+            frame->root->flags |= MDATA_MTIME;
+            frame->root->flags |= MDATA_CTIME;
+            break;
+
+        case GF_FOP_LINK:
+        case GF_FOP_RENAME:
+            frame->root->flags |= MDATA_CTIME;
+            frame->root->flags |= MDATA_PAR_CTIME;
+            frame->root->flags |= MDATA_PAR_MTIME;
+            break;
+
+        case GF_FOP_TRUNCATE:
+        case GF_FOP_FTRUNCATE:
+            frame->root->flags |= MDATA_CTIME;
+            frame->root->flags |= MDATA_MTIME;
+            break;
+
+        case GF_FOP_REMOVEXATTR:
+        case GF_FOP_FREMOVEXATTR:
+            frame->root->flags |= MDATA_CTIME;
+            break;
+
+        case GF_FOP_COPY_FILE_RANGE:
+            /* Below 2 are for destination fd */
+            frame->root->flags |= MDATA_CTIME;
+            frame->root->flags |= MDATA_MTIME;
+            /* Below flag is for the source fd */
+            if (!utime_priv->noatime) {
+                frame->root->flags |= MDATA_ATIME;
+            }
+            break;
+        default:
+            frame->root->flags = 0;
+    }
+out:
+    return;
+}
diff --git a/xlators/features/utime/src/utime-helpers.h b/xlators/features/utime/src/utime-helpers.h
new file mode 100644
index 00000000000..2e32d4bece6
--- /dev/null
+++ b/xlators/features/utime/src/utime-helpers.h
@@ -0,0 +1,25 @@
+/*
+  Copyright (c) 2018 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
+*/
+
+#ifndef _UTIME_HELPERS_H
+#define _UTIME_HELPERS_H
+
+#include <glusterfs/stack.h>
+#include <glusterfs/xlator.h>
+#include <glusterfs/timespec.h>
+#include <time.h>
+
+void
+gl_timespec_get(struct timespec *ts);
+void
+utime_update_attribute_flags(call_frame_t *frame, xlator_t *this,
+                             glusterfs_fop_t fop);
+
+#endif /* _UTIME_HELPERS_H */
diff --git a/xlators/features/utime/src/utime-mem-types.h b/xlators/features/utime/src/utime-mem-types.h
new file mode 100644
index 00000000000..ad1255f85f3
--- /dev/null
+++ b/xlators/features/utime/src/utime-mem-types.h
@@ -0,0 +1,21 @@
+/*
+  Copyright (c) 2018 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
+*/
+
+#ifndef __UTIME_MEM_TYPES_H__
+#define __UTIME_MEM_TYPES_H__
+
+#include <glusterfs/mem-types.h>
+
+enum gf_utime_mem_types_ {
+    utime_mt_utime_t = gf_common_mt_end + 1,
+    utime_mt_end
+};
+
+#endif /* __UTIME_MEM_TYPES_H__ */
diff --git a/xlators/features/utime/src/utime-messages.h b/xlators/features/utime/src/utime-messages.h
new file mode 100644
index 00000000000..bd40265abaf
--- /dev/null
+++ b/xlators/features/utime/src/utime-messages.h
@@ -0,0 +1,29 @@
+/*
+  Copyright (c) 2018 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
+*/
+
+#ifndef __UTIME_MESSAGES_H__
+#define __UTIME_MESSAGES_H__
+
+#include <glusterfs/glfs-message-id.h>
+
+/* To add new message IDs, append new identifiers at the end of the list.
+ *
+ * Never remove a message ID. If it's not used anymore, you can rename it or
+ * leave it as it is, but not delete it. This is to prevent reutilization of
+ * IDs by other messages.
+ *
+ * The component name must match one of the entries defined in
+ * glfs-message-id.h.
+ */
+
+GLFS_MSGID(UTIME, UTIME_MSG_NO_MEMORY, UTIME_MSG_SET_MDATA_FAILED,
+           UTIME_MSG_DICT_SET_FAILED);
+
+#endif /* __UTIME_MESSAGES_H__ */
diff --git a/xlators/features/utime/src/utime.c b/xlators/features/utime/src/utime.c
new file mode 100644
index 00000000000..2acc63e6a05
--- /dev/null
+++ b/xlators/features/utime/src/utime.c
@@ -0,0 +1,392 @@
+/*
+  Copyright (c) 2018 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
+*/
+
+#include "utime.h"
+#include "utime-helpers.h"
+#include "utime-messages.h"
+#include "utime-mem-types.h"
+#include <glusterfs/call-stub.h>
+
+int32_t
+gf_utime_invalidate(xlator_t *this, inode_t *inode)
+{
+    return 0;
+}
+
+int32_t
+gf_utime_forget(xlator_t *this, inode_t *inode)
+{
+    return 0;
+}
+
+int32_t
+gf_utime_client_destroy(xlator_t *this, client_t *client)
+{
+    return 0;
+}
+
+void
+gf_utime_ictxmerge(xlator_t *this, fd_t *fd, inode_t *inode,
+                   inode_t *linked_inode)
+{
+    return;
+}
+
+int32_t
+gf_utime_release(xlator_t *this, fd_t *fd)
+{
+    return 0;
+}
+
+int32_t
+gf_utime_releasedir(xlator_t *this, fd_t *fd)
+{
+    return 0;
+}
+
+int32_t
+gf_utime_client_disconnect(xlator_t *this, client_t *client)
+{
+    return 0;
+}
+
+int32_t
+gf_utime_fdctx_to_dict(xlator_t *this, fd_t *fd, dict_t *dict)
+{
+    return 0;
+}
+
+int32_t
+gf_utime_inode(xlator_t *this)
+{
+    return 0;
+}
+
+int32_t
+gf_utime_inode_to_dict(xlator_t *this, dict_t *dict)
+{
+    return 0;
+}
+
+int32_t
+gf_utime_history(xlator_t *this)
+{
+    return 0;
+}
+
+int32_t
+gf_utime_fd(xlator_t *this)
+{
+    return 0;
+}
+
+int32_t
+gf_utime_fd_to_dict(xlator_t *this, dict_t *dict)
+{
+    return 0;
+}
+
+int32_t
+gf_utime_fdctx(xlator_t *this, fd_t *fd)
+{
+    return 0;
+}
+
+int32_t
+gf_utime_inodectx(xlator_t *this, inode_t *ino)
+{
+    return 0;
+}
+
+int32_t
+gf_utime_inodectx_to_dict(xlator_t *this, inode_t *ino, dict_t *dict)
+{
+    return 0;
+}
+
+int32_t
+gf_utime_priv_to_dict(xlator_t *this, dict_t *dict, char *brickname)
+{
+    return 0;
+}
+
+int32_t
+gf_utime_priv(xlator_t *this)
+{
+    return 0;
+}
+
+int32_t
+mem_acct_init(xlator_t *this)
+{
+    if (xlator_mem_acct_init(this, utime_mt_end + 1) != 0) {
+        gf_msg(this->name, GF_LOG_ERROR, ENOMEM, UTIME_MSG_NO_MEMORY,
+               "Memory accounting initialization failed.");
+        return -1;
+    }
+    return 0;
+}
+
+int32_t
+gf_utime_set_mdata_setxattr_cbk(call_frame_t *frame, void *cookie,
+                                xlator_t *this, int op_ret, int op_errno,
+                                dict_t *xdata)
+{
+    call_stub_t *stub = frame->local;
+    /* Don't fail lookup if mdata setxattr fails */
+    if (op_ret) {
+        gf_msg(this->name, GF_LOG_ERROR, op_errno, UTIME_MSG_SET_MDATA_FAILED,
+               "dict set of key for set-ctime-mdata failed");
+    }
+    frame->local = NULL;
+    call_resume(stub);
+    STACK_DESTROY(frame->root);
+    return 0;
+}
+
+int32_t
+gf_utime_set_mdata_lookup_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                              int32_t op_ret, int32_t op_errno, inode_t *inode,
+                              struct iatt *stbuf, dict_t *xdata,
+                              struct iatt *postparent)
+{
+    dict_t *dict = NULL;
+    struct mdata_iatt *mdata = NULL;
+    int ret = 0;
+    loc_t loc = {
+        0,
+    };
+    call_frame_t *new_frame = NULL;
+
+    if (!op_ret && dict_get(xdata, GF_XATTR_MDATA_KEY) == NULL) {
+        dict = dict_new();
+        if (!dict) {
+            op_errno = ENOMEM;
+            goto err;
+        }
+        mdata = GF_MALLOC(sizeof(struct mdata_iatt), gf_common_mt_char);
+        if (mdata == NULL) {
+            op_errno = ENOMEM;
+            goto err;
+        }
+        iatt_to_mdata(mdata, stbuf);
+        ret = dict_set_mdata(dict, CTIME_MDATA_XDATA_KEY, mdata, _gf_false);
+        if (ret < 0) {
+            gf_msg(this->name, GF_LOG_WARNING, ENOMEM, UTIME_MSG_NO_MEMORY,
+                   "dict set of key for set-ctime-mdata failed");
+            goto err;
+        }
+        new_frame = copy_frame(frame);
+        if (!new_frame) {
+            op_errno = ENOMEM;
+            goto stub_err;
+        }
+
+        new_frame->local = fop_lookup_cbk_stub(frame, default_lookup_cbk,
+                                               op_ret, op_errno, inode, stbuf,
+                                               xdata, postparent);
+        if (!new_frame->local) {
+            gf_msg(this->name, GF_LOG_WARNING, ENOMEM, UTIME_MSG_NO_MEMORY,
+                   "lookup_cbk stub allocation failed");
+            op_errno = ENOMEM;
+            STACK_DESTROY(new_frame->root);
+            goto stub_err;
+        }
+
+        loc.inode = inode_ref(inode);
+        gf_uuid_copy(loc.gfid, stbuf->ia_gfid);
+
+        new_frame->root->uid = 0;
+        new_frame->root->gid = 0;
+        new_frame->root->pid = GF_CLIENT_PID_SET_UTIME;
+        STACK_WIND(new_frame, gf_utime_set_mdata_setxattr_cbk,
+                   FIRST_CHILD(this), FIRST_CHILD(this)->fops->setxattr, &loc,
+                   dict, 0, NULL);
+
+        dict_unref(dict);
+        inode_unref(loc.inode);
+        return 0;
+    }
+
+    STACK_UNWIND_STRICT(lookup, frame, op_ret, op_errno, inode, stbuf, xdata,
+                        postparent);
+    return 0;
+
+err:
+    if (mdata) {
+        GF_FREE(mdata);
+    }
+stub_err:
+    if (dict) {
+        dict_unref(dict);
+    }
+    STACK_UNWIND_STRICT(lookup, frame, -1, op_errno, NULL, NULL, NULL, NULL);
+    return 0;
+}
+
+int
+gf_utime_lookup(call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xdata)
+{
+    int op_errno = EINVAL;
+    int ret = -1;
+
+    VALIDATE_OR_GOTO(frame, err);
+    VALIDATE_OR_GOTO(this, err);
+    VALIDATE_OR_GOTO(loc, err);
+    VALIDATE_OR_GOTO(loc->inode, err);
+
+    xdata = xdata ? dict_ref(xdata) : dict_new();
+    if (!xdata) {
+        op_errno = ENOMEM;
+        goto err;
+    }
+
+    ret = dict_set_int8(xdata, GF_XATTR_MDATA_KEY, 1);
+    if (ret < 0) {
+        gf_msg(this->name, GF_LOG_WARNING, -ret, UTIME_MSG_DICT_SET_FAILED,
+               "%s: Unable to set dict value for %s", loc->path,
+               GF_XATTR_MDATA_KEY);
+        op_errno = -ret;
+        goto free_dict;
+    }
+
+    STACK_WIND(frame, gf_utime_set_mdata_lookup_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->lookup, loc, xdata);
+    dict_unref(xdata);
+    return 0;
+
+free_dict:
+    dict_unref(xdata);
+err:
+    STACK_UNWIND_STRICT(lookup, frame, ret, op_errno, NULL, NULL, NULL, NULL);
+    return 0;
+}
+
+int32_t
+init(xlator_t *this)
+{
+    utime_priv_t *utime = NULL;
+
+    utime = GF_MALLOC(sizeof(*utime), utime_mt_utime_t);
+    if (utime == NULL) {
+        gf_msg(this->name, GF_LOG_ERROR, ENOMEM, UTIME_MSG_NO_MEMORY,
+               "Failed to allocate private memory.");
+        return -1;
+    }
+    memset(utime, 0, sizeof(*utime));
+
+    this->private = utime;
+    GF_OPTION_INIT("noatime", utime->noatime, bool, err);
+
+    return 0;
+err:
+    return -1;
+}
+
+void
+fini(xlator_t *this)
+{
+    utime_priv_t *utime = NULL;
+
+    utime = this->private;
+    GF_FREE(utime);
+    return;
+}
+
+int32_t
+reconfigure(xlator_t *this, dict_t *options)
+{
+    utime_priv_t *utime = this->private;
+
+    GF_OPTION_RECONF("noatime", utime->noatime, options, bool, err);
+
+    return 0;
+err:
+    return -1;
+}
+
+int
+notify(xlator_t *this, int event, void *data, ...)
+{
+    return default_notify(this, event, data);
+}
+
+struct xlator_fops fops = {
+    .rename = gf_utime_rename,
+    .mknod = gf_utime_mknod,
+    .readv = gf_utime_readv,
+    .fremovexattr = gf_utime_fremovexattr,
+    .open = gf_utime_open,
+    .create = gf_utime_create,
+    .mkdir = gf_utime_mkdir,
+    .writev = gf_utime_writev,
+    .rmdir = gf_utime_rmdir,
+    .fallocate = gf_utime_fallocate,
+    .truncate = gf_utime_truncate,
+    .symlink = gf_utime_symlink,
+    .zerofill = gf_utime_zerofill,
+    .link = gf_utime_link,
+    .ftruncate = gf_utime_ftruncate,
+    .unlink = gf_utime_unlink,
+    .setattr = gf_utime_setattr,
+    .fsetattr = gf_utime_fsetattr,
+    .opendir = gf_utime_opendir,
+    .removexattr = gf_utime_removexattr,
+    .lookup = gf_utime_lookup,
+};
+struct xlator_cbks cbks = {
+    .invalidate = gf_utime_invalidate,
+    .forget = gf_utime_forget,
+    .client_destroy = gf_utime_client_destroy,
+    .ictxmerge = gf_utime_ictxmerge,
+    .release = gf_utime_release,
+    .releasedir = gf_utime_releasedir,
+    .client_disconnect = gf_utime_client_disconnect,
+};
+struct xlator_dumpops dumpops = {
+    .fdctx_to_dict = gf_utime_fdctx_to_dict,
+    .inode = gf_utime_inode,
+    .inode_to_dict = gf_utime_inode_to_dict,
+    .history = gf_utime_history,
+    .fd = gf_utime_fd,
+    .fd_to_dict = gf_utime_fd_to_dict,
+    .fdctx = gf_utime_fdctx,
+    .inodectx = gf_utime_inodectx,
+    .inodectx_to_dict = gf_utime_inodectx_to_dict,
+    .priv_to_dict = gf_utime_priv_to_dict,
+    .priv = gf_utime_priv,
+};
+
+struct volume_options options[] = {
+    {.key = {"noatime"},
+     .type = GF_OPTION_TYPE_BOOL,
+     .default_value = "on",
+     .op_version = {GD_OP_VERSION_5_0},
+     .flags = OPT_FLAG_SETTABLE | OPT_FLAG_CLIENT_OPT | OPT_FLAG_DOC,
+     .tags = {"ctime"},
+     .description = "Enable/Disable atime updation when ctime feature is "
+                    "enabled. When noatime is on, atime is not updated with "
+                    "ctime feature enabled and vice versa."},
+    {.key = {NULL}}};
+
+xlator_api_t xlator_api = {
+    .init = init,
+    .fini = fini,
+    .notify = notify,
+    .reconfigure = reconfigure,
+    .mem_acct_init = mem_acct_init,
+    .op_version = {GD_OP_VERSION_5_0},
+    .dumpops = &dumpops,
+    .fops = &fops,
+    .cbks = &cbks,
+    .options = options,
+    .identifier = "utime",
+    .category = GF_MAINTAINED,
+};
diff --git a/xlators/features/utime/src/utime.h b/xlators/features/utime/src/utime.h
new file mode 100644
index 00000000000..ba55eec00de
--- /dev/null
+++ b/xlators/features/utime/src/utime.h
@@ -0,0 +1,23 @@
+/*
+  Copyright (c) 2018 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
+*/
+
+#ifndef __UTIME_H__
+#define __UTIME_H__
+
+#include <glusterfs/glusterfs.h>
+#include <glusterfs/xlator.h>
+#include <glusterfs/defaults.h>
+#include "utime-autogen-fops.h"
+
+typedef struct utime_priv {
+    gf_boolean_t noatime;
+} utime_priv_t;
+
+#endif /* __UTIME_H__ */
diff --git a/xlators/lib/src/libxlator.c b/xlators/lib/src/libxlator.c
index 11d0ef1caf8..8075fa0c29f 100644
--- a/xlators/lib/src/libxlator.c
+++ b/xlators/lib/src/libxlator.c
@@ -7,506 +7,484 @@
    later), or the GNU General Public License, version 2 (GPLv2), in all
    cases as published by the Free Software Foundation.
 */
-#include "mem-types.h"
-#include "libxlator.h"
 
+#include "libxlator.h"
 
 int marker_xtime_default_gauge[] = {
-        [MCNT_FOUND]    =  1,
-        [MCNT_NOTFOUND] = -1,
-        [MCNT_ENODATA]  = -1,
-        [MCNT_ENOTCONN] = -1,
-        [MCNT_ENOENT]   = -1,
-        [MCNT_EOTHER]   = -1,
+    [MCNT_FOUND] = 1,     [MCNT_NOTFOUND] = -1, [MCNT_ENODATA] = -1,
+    [MCNT_ENOTCONN] = -1, [MCNT_ENOENT] = -1,   [MCNT_EOTHER] = -1,
 };
 
 int marker_uuid_default_gauge[] = {
-        [MCNT_FOUND]    =  1,
-        [MCNT_NOTFOUND] =  0,
-        [MCNT_ENODATA]  =  0,
-        [MCNT_ENOTCONN] =  0,
-        [MCNT_ENOENT]   =  0,
-        [MCNT_EOTHER]   =  0,
+    [MCNT_FOUND] = 1,    [MCNT_NOTFOUND] = 0, [MCNT_ENODATA] = 0,
+    [MCNT_ENOTCONN] = 0, [MCNT_ENOENT] = 0,   [MCNT_EOTHER] = 0,
 };
 
 static int marker_idx_errno_map[] = {
-        [MCNT_FOUND]    = EINVAL,
-        [MCNT_NOTFOUND] = EINVAL,
-        [MCNT_ENOENT]   = ENOENT,
-        [MCNT_ENOTCONN] = ENOTCONN,
-        [MCNT_ENODATA]  = ENODATA,
-        [MCNT_EOTHER]   = EINVAL,
-        [MCNT_MAX]      = 0,
+    [MCNT_FOUND] = EINVAL,    [MCNT_NOTFOUND] = EINVAL,
+    [MCNT_ENOENT] = ENOENT,   [MCNT_ENOTCONN] = ENOTCONN,
+    [MCNT_ENODATA] = ENODATA, [MCNT_EOTHER] = EINVAL,
+    [MCNT_MAX] = 0,
 };
 
 /*Copy the contents of oldtimebuf to newtimbuf*/
 static void
-update_timebuf (uint32_t *oldtimbuf, uint32_t *newtimebuf)
+update_timebuf(uint32_t *oldtimbuf, uint32_t *newtimebuf)
 {
-        newtimebuf[0] =  (oldtimbuf[0]);
-        newtimebuf[1] =  (oldtimbuf[1]);
+    newtimebuf[0] = (oldtimbuf[0]);
+    newtimebuf[1] = (oldtimbuf[1]);
 }
 
 /* Convert Timebuf in network order to host order */
 static void
-get_hosttime (uint32_t *oldtimbuf, uint32_t *newtimebuf)
+get_hosttime(uint32_t *oldtimbuf, uint32_t *newtimebuf)
 {
-        newtimebuf[0] = ntohl (oldtimbuf[0]);
-        newtimebuf[1] = ntohl (oldtimbuf[1]);
+    newtimebuf[0] = ntohl(oldtimbuf[0]);
+    newtimebuf[1] = ntohl(oldtimbuf[1]);
 }
 
-
-
 /* Match the Incoming trusted.glusterfs.<uuid>.xtime against volume uuid */
 int
-match_uuid_local (const char *name, char *uuid)
+match_uuid_local(const char *name, char *uuid)
 {
-        if (!uuid || !*uuid)
-                return -1;
+    if (!uuid || !*uuid)
+        return -1;
 
-        name = strtail ((char *)name, MARKER_XATTR_PREFIX);
-        if (!name || name++[0] != '.')
-                return -1;
+    name = strtail((char *)name, MARKER_XATTR_PREFIX);
+    if (!name || name++ [0] != '.')
+        return -1;
 
-        name = strtail ((char *)name, uuid);
-        if (!name || strcmp (name, ".xtime") != 0)
-                return -1;
+    name = strtail((char *)name, uuid);
+    if (!name || strcmp(name, ".xtime") != 0)
+        return -1;
 
-        return 0;
+    return 0;
 }
 
 static void
-marker_local_incr_errcount (xl_marker_local_t *local, int op_errno)
+marker_local_incr_errcount(xl_marker_local_t *local, int op_errno)
 {
-        marker_result_idx_t i = -1;
-
-        if (!local)
-                return;
-
-        switch (op_errno) {
-                case ENODATA:
-                        i = MCNT_ENODATA;
-                        break;
-                case ENOENT:
-                        i = MCNT_ENOENT;
-                        break;
-                case ENOTCONN:
-                        i = MCNT_ENOTCONN;
-                        break;
-                default:
-                        i = MCNT_EOTHER;
-                        break;
-        }
-
-        local->count[i]++;
+    marker_result_idx_t i = -1;
+
+    if (!local)
+        return;
+
+    switch (op_errno) {
+        case ENODATA:
+            i = MCNT_ENODATA;
+            break;
+        case ENOENT:
+            i = MCNT_ENOENT;
+            break;
+        case ENOTCONN:
+            i = MCNT_ENOTCONN;
+            break;
+        default:
+            i = MCNT_EOTHER;
+            break;
+    }
+
+    local->count[i]++;
 }
 
 static int
-evaluate_marker_results (int *gauge, int *count)
+evaluate_marker_results(int *gauge, int *count)
 {
-        int i = 0;
-        int op_errno = 0;
-        gf_boolean_t sane = _gf_true;
-
-        /* check if the policy of the gauge is violated;
-         * if yes, try to get the best errno, ie. look
-         * for the first position where there is a more
-         * specific kind of vioilation than the generic EINVAL
-         */
-        for (i = 0; i < MCNT_MAX; i++) {
-                if (sane) {
-                        if ((gauge[i] > 0 && count[i] < gauge[i]) ||
-                            (gauge[i] < 0 && count[i] >= -gauge[i])) {
-                                sane = _gf_false;
-                                /* generic action: adopt corresponding errno */
-                                op_errno = marker_idx_errno_map[i];
-                        }
-                } else {
-                        /* already insane; trying to get a more informative
-                         * errno by checking subsequent counters
-                         */
-                        if (count[i] > 0)
-                                op_errno = marker_idx_errno_map[i];
-                }
-                if (op_errno && op_errno != EINVAL)
-                        break;
+    int i = 0;
+    int op_errno = 0;
+    gf_boolean_t sane = _gf_true;
+
+    /* check if the policy of the gauge is violated;
+     * if yes, try to get the best errno, ie. look
+     * for the first position where there is a more
+     * specific kind of vioilation than the generic EINVAL
+     */
+    for (i = 0; i < MCNT_MAX; i++) {
+        if (sane) {
+            if ((gauge[i] > 0 && count[i] < gauge[i]) ||
+                (gauge[i] < 0 && count[i] >= -gauge[i])) {
+                sane = _gf_false;
+                /* generic action: adopt corresponding errno */
+                op_errno = marker_idx_errno_map[i];
+            }
+        } else {
+            /* already insane; trying to get a more informative
+             * errno by checking subsequent counters
+             */
+            if (count[i] > 0)
+                op_errno = marker_idx_errno_map[i];
         }
+        if (op_errno && op_errno != EINVAL)
+            break;
+    }
 
-        return op_errno;
+    return op_errno;
 }
 
 static void
-cluster_marker_unwind (call_frame_t *frame, char *key, void *value, size_t size,
-                       dict_t *dict)
+cluster_marker_unwind(call_frame_t *frame, char *key, void *value, size_t size,
+                      dict_t *dict)
 {
-        xl_marker_local_t *local   = frame->local;
-        int               ret      = 0;
-        int32_t           op_ret   = 0;
-        int32_t           op_errno = 0;
-        gf_boolean_t      unref    = _gf_false;
-
-        frame->local = local->xl_local;
-
-        if (local->count[MCNT_FOUND]) {
-                if (!dict) {
-                        dict = dict_new();
-                        if (dict) {
-                                unref = _gf_true;
-                        } else {
-                                op_ret = -1;
-                                op_errno = ENOMEM;
-                                goto out;
-                        }
-                }
-
-                ret = dict_set_static_bin (dict, key, value, size);
-                if (ret) {
-                        op_ret = -1;
-                        op_errno = ENOMEM;
-                        goto out;
-                }
-        }
-
-        op_errno = evaluate_marker_results (local->gauge, local->count);
-        if (op_errno)
+    xl_marker_local_t *local = frame->local;
+    int ret = 0;
+    int32_t op_ret = 0;
+    int32_t op_errno = 0;
+    gf_boolean_t unref = _gf_false;
+
+    frame->local = local->xl_local;
+
+    if (local->count[MCNT_FOUND]) {
+        if (!dict) {
+            dict = dict_new();
+            if (dict) {
+                unref = _gf_true;
+            } else {
                 op_ret = -1;
+                op_errno = ENOMEM;
+                goto out;
+            }
+        }
 
-out:
-        if (local->xl_specf_unwind) {
-                local->xl_specf_unwind (frame, op_ret,
-                                        op_errno, dict, NULL);
-        } else {
-                STACK_UNWIND_STRICT (getxattr, frame, op_ret, op_errno,
-                                     dict, NULL);
+        ret = dict_set_static_bin(dict, key, value, size);
+        if (ret) {
+            op_ret = -1;
+            op_errno = ENOMEM;
+            goto out;
         }
+    }
 
-        GF_FREE (local);
-        if (unref)
-                dict_unref (dict);
+    op_errno = evaluate_marker_results(local->gauge, local->count);
+    if (op_errno)
+        op_ret = -1;
 
+out:
+    if (local->xl_specf_unwind) {
+        local->xl_specf_unwind(frame, op_ret, op_errno, dict, NULL);
+    } else {
+        STACK_UNWIND_STRICT(getxattr, frame, op_ret, op_errno, dict, NULL);
+    }
+
+    GF_FREE(local);
+    if (unref)
+        dict_unref(dict);
 }
 
 /* Aggregate all the <volid>.xtime attrs of the cluster and send the max*/
 int32_t
-cluster_markerxtime_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                         int op_ret, int op_errno, dict_t *dict, dict_t *xdata)
+cluster_markerxtime_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                        int op_ret, int op_errno, dict_t *dict, dict_t *xdata)
 
 {
+    int32_t callcnt = 0;
+    uint32_t *net_timebuf = NULL;
+    uint32_t host_timebuf[2] = {
+        0,
+    };
+    char marker_xattr[128] = {0};
+    xl_marker_local_t *local = NULL;
+
+    local = frame->local;
+
+    snprintf(marker_xattr, sizeof(marker_xattr), "%s.%s.%s",
+             MARKER_XATTR_PREFIX, local->vol_uuid, XTIME);
+
+    LOCK(&frame->lock);
+    {
+        callcnt = --local->call_count;
+
+        if (op_ret) {
+            marker_local_incr_errcount(local, op_errno);
+            goto unlock;
+        }
 
-        int32_t           callcnt         = 0;
-        uint32_t          *net_timebuf    = NULL;
-        uint32_t          host_timebuf[2] = {0,};
-        char              marker_xattr[128]   = {0};
-        xl_marker_local_t *local          = NULL;
-
-        local = frame->local;
-
-        snprintf (marker_xattr, sizeof (marker_xattr), "%s.%s.%s",
-                  MARKER_XATTR_PREFIX, local->vol_uuid, XTIME);
-
-        LOCK (&frame->lock);
-        {
-                callcnt = --local->call_count;
-
-                if (op_ret) {
-                        marker_local_incr_errcount (local, op_errno);
-                        goto unlock;
-                }
-
-                if (dict_get_ptr (dict, marker_xattr, (void **)&net_timebuf)) {
-                        gf_log (this->name, GF_LOG_WARNING,
-                                "Unable to get <uuid>.xtime attr");
-                        local->count[MCNT_NOTFOUND]++;
-                        goto unlock;
-                }
-
-                if (local->count[MCNT_FOUND]) {
-                        get_hosttime (net_timebuf, host_timebuf);
-                        if ( (host_timebuf[0]>local->host_timebuf[0]) ||
-                                (host_timebuf[0] == local->host_timebuf[0] &&
-                                 host_timebuf[1] >= local->host_timebuf[1])) {
-                                update_timebuf (net_timebuf, local->net_timebuf);
-                                update_timebuf (host_timebuf, local->host_timebuf);
-                        }
-
-                } else {
-                        get_hosttime (net_timebuf, local->host_timebuf);
-                        update_timebuf (net_timebuf, local->net_timebuf);
-                        local->count[MCNT_FOUND]++;
-                }
-
+        if (dict_get_ptr(dict, marker_xattr, (void **)&net_timebuf)) {
+            local->count[MCNT_NOTFOUND]++;
+            UNLOCK(&frame->lock);
+            gf_log(this->name, GF_LOG_WARNING,
+                   "Unable to get <uuid>.xtime attr");
+            goto post_unlock;
         }
-unlock:
-        UNLOCK (&frame->lock);
 
-        if (callcnt == 0)
-                cluster_marker_unwind (frame, marker_xattr, local->net_timebuf,
-                                       8, dict);
+        if (local->count[MCNT_FOUND]) {
+            get_hosttime(net_timebuf, host_timebuf);
+            if ((host_timebuf[0] > local->host_timebuf[0]) ||
+                (host_timebuf[0] == local->host_timebuf[0] &&
+                 host_timebuf[1] >= local->host_timebuf[1])) {
+                update_timebuf(net_timebuf, local->net_timebuf);
+                update_timebuf(host_timebuf, local->host_timebuf);
+            }
 
-        return 0;
+        } else {
+            get_hosttime(net_timebuf, local->host_timebuf);
+            update_timebuf(net_timebuf, local->net_timebuf);
+            local->count[MCNT_FOUND]++;
+        }
+    }
+unlock:
+    UNLOCK(&frame->lock);
+post_unlock:
+    if (callcnt == 0)
+        cluster_marker_unwind(frame, marker_xattr, local->net_timebuf, 8, dict);
 
+    return 0;
 }
 
 int32_t
-cluster_markeruuid_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                        int op_ret, int op_errno, dict_t *dict, dict_t *xdata)
+cluster_markeruuid_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                       int op_ret, int op_errno, dict_t *dict, dict_t *xdata)
 {
-        int32_t             callcnt     = 0;
-        struct volume_mark  *volmark    = NULL;
-        xl_marker_local_t   *local      = NULL;
-        int32_t             ret         = -1;
-        char                *vol_uuid   = NULL;
-
-        local = frame->local;
-
-        LOCK (&frame->lock);
-        {
-                callcnt = --local->call_count;
-                vol_uuid = local->vol_uuid;
-
-                if (op_ret) {
-                        marker_local_incr_errcount (local, op_errno);
-                        goto unlock;
-                }
-
-                ret = dict_get_bin (dict, GF_XATTR_MARKER_KEY,
-                                    (void *)&volmark);
-                if (ret)
-                        goto unlock;
-
-                if (local->count[MCNT_FOUND]) {
-                        if ((local->volmark->major != volmark->major) ||
-                            (local->volmark->minor != volmark->minor)) {
-                                op_ret = -1;
-                                op_errno = EINVAL;
-                                goto unlock;
-                        }
-
-                        if (local->retval) {
-                                goto unlock;
-                        } else if (volmark->retval) {
-                                GF_FREE (local->volmark);
-                                local->volmark =
-                                        memdup (volmark, sizeof (*volmark));
-                                local->retval = volmark->retval;
-                        } else if ((volmark->sec > local->volmark->sec) ||
-                                   ((volmark->sec == local->volmark->sec) &&
-                                    (volmark->usec >= local->volmark->usec))) {
-                                GF_FREE (local->volmark);
-                                local->volmark =
-                                      memdup (volmark, sizeof (*volmark));
-                        }
-
-                } else {
-                        local->volmark = memdup (volmark, sizeof (*volmark));
-                        VALIDATE_OR_GOTO (local->volmark, unlock);
-                        gf_uuid_unparse (volmark->uuid, vol_uuid);
-                        if (volmark->retval)
-                                local->retval = volmark->retval;
-                        local->count[MCNT_FOUND]++;
-                }
+    int32_t callcnt = 0;
+    struct volume_mark *volmark = NULL;
+    xl_marker_local_t *local = NULL;
+    int32_t ret = -1;
+    char *vol_uuid = NULL;
+
+    local = frame->local;
+
+    LOCK(&frame->lock);
+    {
+        callcnt = --local->call_count;
+        vol_uuid = local->vol_uuid;
+
+        if (op_ret) {
+            marker_local_incr_errcount(local, op_errno);
+            goto unlock;
         }
+
+        ret = dict_get_bin(dict, GF_XATTR_MARKER_KEY, (void *)&volmark);
+        if (ret)
+            goto unlock;
+
+        if (local->count[MCNT_FOUND]) {
+            if ((local->volmark->major != volmark->major) ||
+                (local->volmark->minor != volmark->minor)) {
+                op_ret = -1;
+                op_errno = EINVAL;
+                goto unlock;
+            }
+
+            if (local->retval) {
+                goto unlock;
+            } else if (volmark->retval) {
+                GF_FREE(local->volmark);
+                local->volmark = gf_memdup(volmark, sizeof(*volmark));
+                local->retval = volmark->retval;
+            } else if ((volmark->sec > local->volmark->sec) ||
+                       ((volmark->sec == local->volmark->sec) &&
+                        (volmark->usec >= local->volmark->usec))) {
+                GF_FREE(local->volmark);
+                local->volmark = gf_memdup(volmark, sizeof(*volmark));
+            }
+
+        } else {
+            local->volmark = gf_memdup(volmark, sizeof(*volmark));
+            VALIDATE_OR_GOTO(local->volmark, unlock);
+            gf_uuid_unparse(volmark->uuid, vol_uuid);
+            if (volmark->retval)
+                local->retval = volmark->retval;
+            local->count[MCNT_FOUND]++;
+        }
+    }
 unlock:
-        UNLOCK (&frame->lock);
+    UNLOCK(&frame->lock);
 
-        if (callcnt == 0)
-                cluster_marker_unwind (frame, GF_XATTR_MARKER_KEY,
-                                       local->volmark, sizeof (*local->volmark),
-                                       dict);
+    if (callcnt == 0)
+        cluster_marker_unwind(frame, GF_XATTR_MARKER_KEY, local->volmark,
+                              sizeof(*local->volmark), dict);
 
-        return 0;
+    return 0;
 }
 
 int
-gf_get_min_stime (xlator_t *this, dict_t *dst, char *key, data_t *value)
+gf_get_min_stime(xlator_t *this, dict_t *dst, char *key, data_t *value)
 {
-        int ret = -1;
-        uint32_t *net_timebuf = NULL;
-        uint32_t *value_timebuf = NULL;
-        uint32_t host_timebuf[2] = {0,};
-        uint32_t host_value_timebuf[2] = {0,};
-
-        /* stime should be minimum of all the other nodes */
-        ret = dict_get_bin (dst, key, (void **)&net_timebuf);
+    int ret = -1;
+    uint32_t *net_timebuf = NULL;
+    uint32_t *value_timebuf = NULL;
+    uint32_t host_timebuf[2] = {
+        0,
+    };
+    uint32_t host_value_timebuf[2] = {
+        0,
+    };
+
+    /* stime should be minimum of all the other nodes */
+    ret = dict_get_bin(dst, key, (void **)&net_timebuf);
+    if (ret < 0) {
+        net_timebuf = GF_CALLOC(1, sizeof(int64_t), gf_common_mt_char);
+        if (!net_timebuf)
+            goto out;
+
+        ret = dict_set_bin(dst, key, net_timebuf, sizeof(int64_t));
         if (ret < 0) {
-                net_timebuf = GF_CALLOC (1, sizeof (int64_t),
-                                           gf_common_mt_char);
-                if (!net_timebuf)
-                        goto out;
-
-                ret = dict_set_bin (dst, key, net_timebuf, sizeof (int64_t));
-                if (ret < 0) {
-                        gf_log (this->name, GF_LOG_WARNING,
-                                "key=%s: dict set failed", key);
-                        goto error;
-                }
+            gf_log(this->name, GF_LOG_WARNING, "key=%s: dict set failed", key);
+            goto error;
         }
-
-        value_timebuf = data_to_bin (value);
-        if (!value_timebuf) {
-                gf_log (this->name, GF_LOG_WARNING,
-                        "key=%s: getting value of stime failed", key);
-                ret = -1;
-                goto out;
-        }
-
-        get_hosttime (value_timebuf, host_value_timebuf);
-        get_hosttime (net_timebuf, host_timebuf);
-
-        /* can't use 'min()' macro here as we need to compare two fields
-           in the array, selectively */
-        if ((host_value_timebuf[0] < host_timebuf[0]) ||
-            ((host_value_timebuf[0] == host_timebuf[0]) &&
-             (host_value_timebuf[1] < host_timebuf[1]))) {
-                update_timebuf (value_timebuf, net_timebuf);
-        }
-
-        ret = 0;
+    }
+
+    value_timebuf = data_to_bin(value);
+    if (!value_timebuf) {
+        gf_log(this->name, GF_LOG_WARNING,
+               "key=%s: getting value of stime failed", key);
+        ret = -1;
+        goto out;
+    }
+
+    get_hosttime(value_timebuf, host_value_timebuf);
+    get_hosttime(net_timebuf, host_timebuf);
+
+    /* can't use 'min()' macro here as we need to compare two fields
+       in the array, selectively */
+    if ((host_value_timebuf[0] < host_timebuf[0]) ||
+        ((host_value_timebuf[0] == host_timebuf[0]) &&
+         (host_value_timebuf[1] < host_timebuf[1]))) {
+        update_timebuf(value_timebuf, net_timebuf);
+    }
+
+    ret = 0;
 out:
-        return ret;
+    return ret;
 error:
-        /* To be used only when net_timebuf is not set in the dict */
-        if (net_timebuf)
-                GF_FREE (net_timebuf);
+    /* To be used only when net_timebuf is not set in the dict */
+    if (net_timebuf)
+        GF_FREE(net_timebuf);
 
-        return ret;
+    return ret;
 }
 
 int
-gf_get_max_stime (xlator_t *this, dict_t *dst, char *key, data_t *value)
+gf_get_max_stime(xlator_t *this, dict_t *dst, char *key, data_t *value)
 {
-        int ret = -1;
-        uint32_t *net_timebuf = NULL;
-        uint32_t *value_timebuf = NULL;
-        uint32_t host_timebuf[2] = {0,};
-        uint32_t host_value_timebuf[2] = {0,};
-
-        /* stime should be maximum of all the other nodes */
-        ret = dict_get_bin (dst, key, (void **)&net_timebuf);
+    int ret = -ENOMEM;
+    uint32_t *net_timebuf = NULL;
+    uint32_t *value_timebuf = NULL;
+    uint32_t host_timebuf[2] = {
+        0,
+    };
+    uint32_t host_value_timebuf[2] = {
+        0,
+    };
+
+    /* stime should be maximum of all the other nodes */
+    ret = dict_get_bin(dst, key, (void **)&net_timebuf);
+    if (ret < 0) {
+        net_timebuf = GF_CALLOC(1, sizeof(int64_t), gf_common_mt_char);
+        if (!net_timebuf)
+            goto out;
+
+        ret = dict_set_bin(dst, key, net_timebuf, sizeof(int64_t));
         if (ret < 0) {
-                net_timebuf = GF_CALLOC (1, sizeof (int64_t),
-                                           gf_common_mt_char);
-                if (!net_timebuf)
-                        goto out;
-
-                ret = dict_set_bin (dst, key, net_timebuf, sizeof (int64_t));
-                if (ret < 0) {
-                        gf_log (this->name, GF_LOG_WARNING,
-                                "key=%s: dict set failed", key);
-                        goto error;
-                }
+            gf_log(this->name, GF_LOG_WARNING, "key=%s: dict set failed", key);
+            goto error;
         }
-
-        value_timebuf = data_to_bin (value);
-        if (!value_timebuf) {
-                gf_log (this->name, GF_LOG_WARNING,
-                        "key=%s: getting value of stime failed", key);
-                ret = -1;
-                goto out;
-        }
-
-        get_hosttime (value_timebuf, host_value_timebuf);
-        get_hosttime (net_timebuf, host_timebuf);
-
-        /* can't use 'max()' macro here as we need to compare two fields
-           in the array, selectively */
-        if ((host_value_timebuf[0] > host_timebuf[0]) ||
-            ((host_value_timebuf[0] == host_timebuf[0]) &&
-             (host_value_timebuf[1] > host_timebuf[1]))) {
-                update_timebuf (value_timebuf, net_timebuf);
-        }
-
-        ret = 0;
+    }
+
+    value_timebuf = data_to_bin(value);
+    if (!value_timebuf) {
+        gf_log(this->name, GF_LOG_WARNING,
+               "key=%s: getting value of stime failed", key);
+        ret = -EINVAL;
+        goto out;
+    }
+
+    get_hosttime(value_timebuf, host_value_timebuf);
+    get_hosttime(net_timebuf, host_timebuf);
+
+    /* can't use 'max()' macro here as we need to compare two fields
+       in the array, selectively */
+    if ((host_value_timebuf[0] > host_timebuf[0]) ||
+        ((host_value_timebuf[0] == host_timebuf[0]) &&
+         (host_value_timebuf[1] > host_timebuf[1]))) {
+        update_timebuf(value_timebuf, net_timebuf);
+    }
+
+    ret = 0;
 out:
-        return ret;
+    return ret;
 error:
-        /* To be used only when net_timebuf is not set in the dict */
-        if (net_timebuf)
-                GF_FREE (net_timebuf);
+    /* To be used only when net_timebuf is not set in the dict */
+    if (net_timebuf)
+        GF_FREE(net_timebuf);
 
-        return ret;
+    return ret;
 }
 
 static int
-_get_children_count (xlator_t *xl)
+_get_children_count(xlator_t *xl)
 {
-        int i = 0;
-        xlator_list_t *trav = NULL;
-        for (i = 0, trav = xl->children; trav ; trav = trav->next, i++) {
-                /*'i' will have the value */
-        }
+    int i = 0;
+    xlator_list_t *trav = NULL;
+    for (i = 0, trav = xl->children; trav; trav = trav->next, i++) {
+        /*'i' will have the value */
+    }
 
-        return i;
+    return i;
 }
 
 int
-cluster_handle_marker_getxattr (call_frame_t *frame, loc_t *loc,
-                                const char *name, char *vol_uuid,
-                                xlator_specf_unwind_t unwind,
-                                int (*populate_args) (call_frame_t *frame,
-                                                      int type, int *gauge,
-                                                      xlator_t **subvols))
+cluster_handle_marker_getxattr(call_frame_t *frame, loc_t *loc,
+                               const char *name, char *vol_uuid,
+                               xlator_specf_unwind_t unwind,
+                               int (*populate_args)(call_frame_t *frame,
+                                                    int type, int *gauge,
+                                                    xlator_t **subvols))
 {
-        xlator_t                *this = frame->this;
-        xlator_t                **subvols = NULL;
-        int                     num_subvols = 0;
-        int                     type = 0;
-        int                i     = 0;
-        int                     gauge[MCNT_MAX] = {0};
-        xl_marker_local_t  *local = NULL;
-
-        if (GF_CLIENT_PID_GSYNCD != frame->root->pid)
-                return -EINVAL;
-
-        if (name == NULL)
-                return -EINVAL;
-
-        if (strcmp (GF_XATTR_MARKER_KEY, name) == 0) {
-                type = MARKER_UUID_TYPE;
-                memcpy (gauge, marker_uuid_default_gauge, sizeof (gauge));
-        } else if (match_uuid_local (name, vol_uuid) == 0) {
-                type = MARKER_XTIME_TYPE;
-                memcpy (gauge, marker_xtime_default_gauge, sizeof (gauge));
-        } else {
-                return -EINVAL;
-        }
-
-        num_subvols = _get_children_count (this);
-        subvols = alloca (num_subvols * sizeof (*subvols));
-        num_subvols = populate_args (frame, type, gauge, subvols);
-
-        local = GF_CALLOC (sizeof (struct marker_str), 1,
-                            gf_common_mt_libxl_marker_local);
-
-        if (!local)
-                goto fail;
-
-        local->xl_local = frame->local;
-        local->call_count = num_subvols;
-        local->xl_specf_unwind = unwind;
-        local->vol_uuid = vol_uuid;
-        memcpy (local->gauge, gauge, sizeof (local->gauge));
-
-        frame->local = local;
-
-        for (i = 0; i < num_subvols; i++) {
-                if (MARKER_UUID_TYPE == type)
-                        STACK_WIND (frame, cluster_markeruuid_cbk,
-                                    subvols[i],
-                                    subvols[i]->fops->getxattr,
-                                    loc, name, NULL);
-                else if (MARKER_XTIME_TYPE == type)
-                        STACK_WIND (frame, cluster_markerxtime_cbk,
-                                    subvols[i],
-                                    subvols[i]->fops->getxattr,
-                                    loc, name, NULL);
-        }
-
-        return 0;
+    xlator_t *this = frame->this;
+    xlator_t **subvols = NULL;
+    int num_subvols = 0;
+    int type = 0;
+    int i = 0;
+    int gauge[MCNT_MAX] = {0};
+    xl_marker_local_t *local = NULL;
+
+    if (GF_CLIENT_PID_GSYNCD != frame->root->pid)
+        return -EINVAL;
+
+    if (name == NULL)
+        return -EINVAL;
+
+    if (strcmp(GF_XATTR_MARKER_KEY, name) == 0) {
+        type = MARKER_UUID_TYPE;
+        memcpy(gauge, marker_uuid_default_gauge, sizeof(gauge));
+    } else if (match_uuid_local(name, vol_uuid) == 0) {
+        type = MARKER_XTIME_TYPE;
+        memcpy(gauge, marker_xtime_default_gauge, sizeof(gauge));
+    } else {
+        return -EINVAL;
+    }
+
+    num_subvols = _get_children_count(this);
+    subvols = alloca(num_subvols * sizeof(*subvols));
+    num_subvols = populate_args(frame, type, gauge, subvols);
+
+    local = GF_CALLOC(sizeof(struct marker_str), 1,
+                      gf_common_mt_libxl_marker_local);
+
+    if (!local)
+        goto fail;
+
+    local->xl_local = frame->local;
+    local->call_count = num_subvols;
+    local->xl_specf_unwind = unwind;
+    local->vol_uuid = vol_uuid;
+    memcpy(local->gauge, gauge, sizeof(local->gauge));
+
+    frame->local = local;
+
+    for (i = 0; i < num_subvols; i++) {
+        if (MARKER_UUID_TYPE == type)
+            STACK_WIND(frame, cluster_markeruuid_cbk, subvols[i],
+                       subvols[i]->fops->getxattr, loc, name, NULL);
+        else if (MARKER_XTIME_TYPE == type)
+            STACK_WIND(frame, cluster_markerxtime_cbk, subvols[i],
+                       subvols[i]->fops->getxattr, loc, name, NULL);
+    }
+
+    return 0;
 fail:
-        if (unwind)
-                unwind (frame, -1, ENOMEM, NULL, NULL);
-        else
-                default_getxattr_failure_cbk (frame, ENOMEM);
-        return 0;
+    if (unwind)
+        unwind(frame, -1, ENOMEM, NULL, NULL);
+    else
+        default_getxattr_failure_cbk(frame, ENOMEM);
+    return 0;
 }
diff --git a/xlators/lib/src/libxlator.h b/xlators/lib/src/libxlator.h
index 20d5a32f248..81da4060d55 100644
--- a/xlators/lib/src/libxlator.h
+++ b/xlators/lib/src/libxlator.h
@@ -10,46 +10,35 @@
 #ifndef _LIBXLATOR_H
 #define _LIBXLATOR_H
 
+#include <glusterfs/defaults.h>
 
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
-
-#include "xlator.h"
-#include "logging.h"
-#include "defaults.h"
-#include "common-utils.h"
-#include "compat.h"
-#include "compat-errno.h"
-
+#include <stdint.h>             // for int32_t
+#include "glusterfs/dict.h"     // for dict_t, data_t
+#include "glusterfs/globals.h"  // for xlator_t, loc_t
+#include "glusterfs/stack.h"    // for call_frame_t
+#include <glusterfs/compat.h>
+#include <glusterfs/compat-errno.h>
 
 #define MARKER_XATTR_PREFIX "trusted.glusterfs"
-#define XTIME               "xtime"
-#define VOLUME_MARK         "volume-mark"
+#define XTIME "xtime"
+#define VOLUME_MARK "volume-mark"
 #define GF_XATTR_MARKER_KEY MARKER_XATTR_PREFIX "." VOLUME_MARK
 #define UUID_SIZE 36
-#define MARKER_UUID_TYPE    1
-#define MARKER_XTIME_TYPE   2
-#define GF_XATTR_QUOTA_SIZE_KEY "trusted.glusterfs.quota.size"
-#define GF_XATTR_QUOTA_LIMIT_LIST "trusted.limit.list"
-#define GF_XATTR_QUOTA_LIMIT_LIST_OBJECT "trusted.limit.objects"
-
-
-typedef int32_t (*xlator_specf_unwind_t) (call_frame_t *frame,
-                                          int op_ret, int op_errno,
-                                          dict_t *dict, dict_t *xdata);
+#define MARKER_UUID_TYPE 1
+#define MARKER_XTIME_TYPE 2
 
+typedef int32_t (*xlator_specf_unwind_t)(call_frame_t *frame, int op_ret,
+                                         int op_errno, dict_t *dict,
+                                         dict_t *xdata);
 
 struct volume_mark {
-        uint8_t major;
-        uint8_t minor;
-        uint8_t uuid[16];
-        uint8_t retval;
-        uint32_t sec;
-        uint32_t usec;
-}__attribute__ ((__packed__));
-
+    uint8_t major;
+    uint8_t minor;
+    uint8_t uuid[16];
+    uint8_t retval;
+    uint32_t sec;
+    uint32_t usec;
+} __attribute__((__packed__));
 
 /*
  * The enumerated type here
@@ -93,7 +82,7 @@ struct volume_mark {
  * Cf. evaluate_marker_results() and marker_idx_errno_map[]
  * in libxlator.c
 
- * We provide two default gauges, one inteded for xtime
+ * We provide two default gauges, one intended for xtime
  * aggregation, other for volume mark aggregation. The
  * policies they represent agree with the hard-coded
  * one prior to gauges. Cf. marker_xtime_default_gauge
@@ -101,58 +90,58 @@ struct volume_mark {
  */
 
 typedef enum {
-        MCNT_FOUND,
-        MCNT_NOTFOUND,
-        MCNT_ENODATA,
-        MCNT_ENOTCONN,
-        MCNT_ENOENT,
-        MCNT_EOTHER,
-        MCNT_MAX
+    MCNT_FOUND,
+    MCNT_NOTFOUND,
+    MCNT_ENODATA,
+    MCNT_ENOTCONN,
+    MCNT_ENOENT,
+    MCNT_EOTHER,
+    MCNT_MAX
 } marker_result_idx_t;
 
 extern int marker_xtime_default_gauge[];
 extern int marker_uuid_default_gauge[];
 
 struct marker_str {
-        struct volume_mark    *volmark;
-        data_t                *data;
-
-        uint32_t               host_timebuf[2];
-        uint32_t               net_timebuf[2];
-        int32_t                call_count;
-        int                    gauge[MCNT_MAX];
-        int                    count[MCNT_MAX];
-
-        xlator_specf_unwind_t  xl_specf_unwind;
-        void                  *xl_local;
-        char                  *vol_uuid;
-        uint8_t                retval;
+    struct volume_mark *volmark;
+    data_t *data;
+
+    uint32_t host_timebuf[2];
+    uint32_t net_timebuf[2];
+    int32_t call_count;
+    int gauge[MCNT_MAX];
+    int count[MCNT_MAX];
+
+    xlator_specf_unwind_t xl_specf_unwind;
+    void *xl_local;
+    char *vol_uuid;
+    uint8_t retval;
 };
 
 typedef struct marker_str xl_marker_local_t;
 
 int32_t
-cluster_markerxtime_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                         int op_ret, int op_errno, dict_t *dict, dict_t *xdata);
+cluster_markerxtime_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                        int op_ret, int op_errno, dict_t *dict, dict_t *xdata);
 
 int32_t
-cluster_markeruuid_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                        int op_ret, int op_errno, dict_t *dict, dict_t *xdata);
+cluster_markeruuid_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                       int op_ret, int op_errno, dict_t *dict, dict_t *xdata);
 
 int
-cluster_handle_marker_getxattr (call_frame_t *frame, loc_t *loc,
-                                const char *name, char *vol_uuid,
-                                xlator_specf_unwind_t unwind,
-                                int (*populate_args) (call_frame_t *frame,
-                                                      int type, int *gauge,
-                                                      xlator_t **subvols));
+cluster_handle_marker_getxattr(call_frame_t *frame, loc_t *loc,
+                               const char *name, char *vol_uuid,
+                               xlator_specf_unwind_t unwind,
+                               int (*populate_args)(call_frame_t *frame,
+                                                    int type, int *gauge,
+                                                    xlator_t **subvols));
 int
-match_uuid_local (const char *name, char *uuid);
+match_uuid_local(const char *name, char *uuid);
 
 int
-gf_get_min_stime (xlator_t *this, dict_t *dst, char *key, data_t *value);
+gf_get_min_stime(xlator_t *this, dict_t *dst, char *key, data_t *value);
 
 int
-gf_get_max_stime (xlator_t *this, dict_t *dst, char *key, data_t *value);
+gf_get_max_stime(xlator_t *this, dict_t *dst, char *key, data_t *value);
 
 #endif /* !_LIBXLATOR_H */
diff --git a/xlators/meta/Makefile.am b/xlators/meta/Makefile.am
index e1c45f3051c..af437a64d6d 100644
--- a/xlators/meta/Makefile.am
+++ b/xlators/meta/Makefile.am
@@ -1 +1 @@
-SUBDIRS=src
-\ No newline at end of file
+SUBDIRS = src
diff --git a/xlators/meta/src/Makefile.am b/xlators/meta/src/Makefile.am
index b705e86b60a..29b871d7984 100644
--- a/xlators/meta/src/Makefile.am
+++ b/xlators/meta/src/Makefile.am
@@ -1,7 +1,7 @@
 xlator_LTLIBRARIES = meta.la
 xlatordir = $(libdir)/glusterfs/$(PACKAGE_VERSION)/xlator
 
-meta_la_LDFLAGS = -module -avoid-version
+meta_la_LDFLAGS = -module $(GF_XLATOR_DEFAULT_LDFLAGS)
 
 meta_la_SOURCES = meta.c meta-helpers.c meta-defaults.c \
 	root-dir.c \
@@ -36,7 +36,8 @@ meta_la_LIBADD = $(top_builddir)/libglusterfs/src/libglusterfs.la
 
 noinst_HEADERS = meta.h meta-hooks.h meta-mem-types.h
 
-AM_CPPFLAGS = $(GF_CPPFLAGS) -I$(top_srcdir)/libglusterfs/src
+AM_CPPFLAGS = $(GF_CPPFLAGS) -I$(top_srcdir)/libglusterfs/src \
+	-I$(top_srcdir)/rpc/xdr/src -I$(top_builddir)/rpc/xdr/src
 
 AM_CFLAGS = -Wall $(GF_CFLAGS)
 
diff --git a/xlators/meta/src/active-link.c b/xlators/meta/src/active-link.c
index 99d38597bdf..7ee780d89e9 100644
--- a/xlators/meta/src/active-link.c
+++ b/xlators/meta/src/active-link.c
@@ -8,37 +8,27 @@
    cases as published by the Free Software Foundation.
 */
 
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
-
-#include "xlator.h"
-#include "defaults.h"
+#include <glusterfs/xlator.h>
+#include <glusterfs/defaults.h>
 
 #include "meta-mem-types.h"
 #include "meta.h"
 
-
 static int
-active_link_fill (xlator_t *this, inode_t *inode, strfd_t *strfd)
+active_link_fill(xlator_t *this, inode_t *inode, strfd_t *strfd)
 {
-	strprintf (strfd, "%s", this->ctx->active->graph_uuid);
+    strprintf(strfd, "%s", this->ctx->active->graph_uuid);
 
-	return 0;
+    return 0;
 }
 
-
-struct meta_ops active_link_ops = {
-	.link_fill = active_link_fill
-};
-
+struct meta_ops active_link_ops = {.link_fill = active_link_fill};
 
 int
-meta_active_link_hook (call_frame_t *frame, xlator_t *this, loc_t *loc,
-		       dict_t *xdata)
+meta_active_link_hook(call_frame_t *frame, xlator_t *this, loc_t *loc,
+                      dict_t *xdata)
 {
-	meta_ops_set (loc->inode, this, &active_link_ops);
+    meta_ops_set(loc->inode, this, &active_link_ops);
 
-	return 0;
+    return 0;
 }
diff --git a/xlators/meta/src/cmdline-file.c b/xlators/meta/src/cmdline-file.c
index 88411f2629b..eb24e985af9 100644
--- a/xlators/meta/src/cmdline-file.c
+++ b/xlators/meta/src/cmdline-file.c
@@ -8,41 +8,32 @@
    cases as published by the Free Software Foundation.
 */
 
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
-
-#include "xlator.h"
-#include "defaults.h"
+#include <glusterfs/xlator.h>
+#include <glusterfs/defaults.h>
 
 #include "meta-mem-types.h"
 #include "meta.h"
-#include "strfd.h"
-#include "globals.h"
-#include "lkowner.h"
-
+#include <glusterfs/strfd.h>
+#include <glusterfs/lkowner.h>
 
 static int
-cmdline_file_fill (xlator_t *this, inode_t *file, strfd_t *strfd)
+cmdline_file_fill(xlator_t *this, inode_t *file, strfd_t *strfd)
 {
-        if (this->ctx->cmdlinestr)
-                strprintf (strfd, "{ \n  \"Cmdlinestr\": \"%s\"\n}",
-                           this->ctx->cmdlinestr);
-        return strfd->size;
+    if (this->ctx->cmdlinestr)
+        strprintf(strfd, "{ \n  \"Cmdlinestr\": \"%s\"\n}",
+                  this->ctx->cmdlinestr);
+    return strfd->size;
 }
 
-
 static struct meta_ops cmdline_file_ops = {
-        .file_fill = cmdline_file_fill,
+    .file_fill = cmdline_file_fill,
 };
 
-
 int
-meta_cmdline_file_hook (call_frame_t *frame, xlator_t *this, loc_t *loc,
-                        dict_t *xdata)
+meta_cmdline_file_hook(call_frame_t *frame, xlator_t *this, loc_t *loc,
+                       dict_t *xdata)
 {
-        meta_ops_set (loc->inode, this, &cmdline_file_ops);
+    meta_ops_set(loc->inode, this, &cmdline_file_ops);
 
-        return 0;
+    return 0;
 }
diff --git a/xlators/meta/src/frames-file.c b/xlators/meta/src/frames-file.c
index 0e9777c9da2..9a13db9a934 100644
--- a/xlators/meta/src/frames-file.c
+++ b/xlators/meta/src/frames-file.c
@@ -8,115 +8,100 @@
    cases as published by the Free Software Foundation.
 */
 
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
-
-#include "xlator.h"
-#include "defaults.h"
+#include <glusterfs/xlator.h>
+#include <glusterfs/defaults.h>
 
 #include "meta-mem-types.h"
 #include "meta.h"
-#include "strfd.h"
-#include "globals.h"
-#include "lkowner.h"
+#include <glusterfs/strfd.h>
+#include <glusterfs/lkowner.h>
 
 static int
-frames_file_fill (xlator_t *this, inode_t *file, strfd_t *strfd)
+frames_file_fill(xlator_t *this, inode_t *file, strfd_t *strfd)
 {
-        struct call_pool *pool = NULL;
-        call_stack_t *stack = NULL;
-        call_frame_t *frame = NULL;
-        int i = 0;
-        int j = 1;
+    struct call_pool *pool = NULL;
+    call_stack_t *stack = NULL;
+    call_frame_t *frame = NULL;
+    int i = 0;
+    int j = 1;
+
+    if (!this || !file || !strfd)
+        return -1;
 
-        if (!this || !file || !strfd)
-                return -1;
+    pool = this->ctx->pool;
 
-        pool = this->ctx->pool;
+    strprintf(strfd, "{ \n\t\"Stack\": [\n");
 
-        LOCK (&pool->lock);
+    LOCK(&pool->lock);
+    {
+        list_for_each_entry(stack, &pool->all_frames, all_frames)
         {
-                strprintf (strfd, "{ \n\t\"Stack\": [\n");
-                list_for_each_entry (stack, &pool->all_frames, all_frames) {
-                        strprintf (strfd, "\t   {\n");
-                        strprintf (strfd, "\t\t\"Number\": %d,\n", ++i);
-                        strprintf (strfd, "\t\t\"Frame\": [\n");
-                        j = 1;
-                        for (frame = &stack->frames; frame;
-                             frame = frame->next) {
-                                strprintf (strfd, "\t\t   {\n");
-                                strprintf (strfd, "\t\t\t\"Number\": %d,\n",
-                                                j++);
-                                strprintf (strfd,
-                                                "\t\t\t\"Xlator\": \"%s\",\n",
-                                                frame->this->name);
-                                if (frame->begin.tv_sec)
-                                        strprintf (strfd,
-                                                        "\t\t\t\"Creation_time\": %d.%d,\n",
-                                                        (int)frame->begin.tv_sec,
-                                                        (int)frame->begin.tv_usec);
-                                strprintf (strfd, " \t\t\t\"Refcount\": %d,\n",
-                                                frame->ref_count);
-                                if (frame->parent)
-                                        strprintf (strfd, "\t\t\t\"Parent\": \"%s\",\n",
-                                                        frame->parent->this->name);
-                                if (frame->wind_from)
-                                        strprintf (strfd, "\t\t\t\"Wind_from\": \"%s\",\n",
-                                                        frame->wind_from);
-                                if (frame->wind_to)
-                                        strprintf (strfd, "\t\t\t\"Wind_to\": \"%s\",\n",
-                                                        frame->wind_to);
-                                if (frame->unwind_from)
-                                        strprintf (strfd, "\t\t\t\"Unwind_from\": \"%s\",\n",
-                                                        frame->unwind_from);
-                                if (frame->unwind_to)
-                                        strprintf (strfd, "\t\t\t\"Unwind_to\": \"%s\",\n",
-                                                        frame->unwind_to);
-                                strprintf (strfd, "\t\t\t\"Complete\": %d\n",
-                                                frame->complete);
-                                if (frame->next == NULL)
-                                        strprintf (strfd, "\t\t   }\n");
-                                else
-                                        strprintf (strfd, "\t\t   },\n");
-                        }
-                        strprintf (strfd, "\t\t],\n");
-                        strprintf (strfd, "\t\t\"Unique\": %"PRId64",\n",
-                                        stack->unique);
-                        strprintf (strfd, "\t\t\"Type\": \"%s\",\n",
-                                        gf_fop_list[stack->op]);
-                        strprintf (strfd, "\t\t\"UID\": %d,\n",
-                                        stack->uid);
-                        strprintf (strfd, "\t\t\"GID\": %d,\n",
-                                        stack->gid);
-                        strprintf (strfd, "\t\t\"LK_owner\": \"%s\"\n",
-                                        lkowner_utoa (&stack->lk_owner));
-                        if (i == (int)pool->cnt)
-                                strprintf (strfd, "\t   }\n");
-                        else
-                                strprintf (strfd, "\t   },\n");
-                }
-                strprintf (strfd, "\t],\n");
-                strprintf (strfd, "\t\"Call_Count\": %d\n",
-                                (int)pool->cnt);
-                strprintf (strfd, "}");
+            strprintf(strfd, "\t   {\n");
+            strprintf(strfd, "\t\t\"Number\": %d,\n", ++i);
+            strprintf(strfd, "\t\t\"Frame\": [\n");
+            j = 1;
+            list_for_each_entry(frame, &stack->myframes, frames)
+            {
+                strprintf(strfd, "\t\t   {\n");
+                strprintf(strfd, "\t\t\t\"Number\": %d,\n", j++);
+                strprintf(strfd, "\t\t\t\"Xlator\": \"%s\",\n",
+                          frame->this->name);
+                if (frame->begin.tv_sec)
+                    strprintf(strfd, "\t\t\t\"Creation_time\": %d.%09d,\n",
+                              (int)frame->begin.tv_sec,
+                              (int)frame->begin.tv_nsec);
+                strprintf(strfd, " \t\t\t\"Refcount\": %d,\n",
+                          frame->ref_count);
+                if (frame->parent)
+                    strprintf(strfd, "\t\t\t\"Parent\": \"%s\",\n",
+                              frame->parent->this->name);
+                if (frame->wind_from)
+                    strprintf(strfd, "\t\t\t\"Wind_from\": \"%s\",\n",
+                              frame->wind_from);
+                if (frame->wind_to)
+                    strprintf(strfd, "\t\t\t\"Wind_to\": \"%s\",\n",
+                              frame->wind_to);
+                if (frame->unwind_from)
+                    strprintf(strfd, "\t\t\t\"Unwind_from\": \"%s\",\n",
+                              frame->unwind_from);
+                if (frame->unwind_to)
+                    strprintf(strfd, "\t\t\t\"Unwind_to\": \"%s\",\n",
+                              frame->unwind_to);
+                strprintf(strfd, "\t\t\t\"Complete\": %d\n", frame->complete);
+                if (list_is_last(&frame->frames, &stack->myframes))
+                    strprintf(strfd, "\t\t   }\n");
+                else
+                    strprintf(strfd, "\t\t   },\n");
+            }
+            strprintf(strfd, "\t\t],\n");
+            strprintf(strfd, "\t\t\"Unique\": %" PRId64 ",\n", stack->unique);
+            strprintf(strfd, "\t\t\"Type\": \"%s\",\n", gf_fop_list[stack->op]);
+            strprintf(strfd, "\t\t\"UID\": %d,\n", stack->uid);
+            strprintf(strfd, "\t\t\"GID\": %d,\n", stack->gid);
+            strprintf(strfd, "\t\t\"LK_owner\": \"%s\"\n",
+                      lkowner_utoa(&stack->lk_owner));
+            if (i == (int)pool->cnt)
+                strprintf(strfd, "\t   }\n");
+            else
+                strprintf(strfd, "\t   },\n");
         }
-        UNLOCK (&pool->lock);
+        strprintf(strfd, "\t],\n");
+        strprintf(strfd, "\t\"Call_Count\": %d\n", (int)pool->cnt);
+        strprintf(strfd, "}");
+    }
+    UNLOCK(&pool->lock);
 
-        return strfd->size;
+    return strfd->size;
 }
 
-
 static struct meta_ops frames_file_ops = {
-        .file_fill = frames_file_fill,
+    .file_fill = frames_file_fill,
 };
 
-
 int
-meta_frames_file_hook (call_frame_t *frame, xlator_t *this, loc_t *loc,
-                       dict_t *xdata)
+meta_frames_file_hook(call_frame_t *frame, xlator_t *this, loc_t *loc,
+                      dict_t *xdata)
 {
-        meta_ops_set (loc->inode, this, &frames_file_ops);
-        return 0;
+    meta_ops_set(loc->inode, this, &frames_file_ops);
+    return 0;
 }
diff --git a/xlators/meta/src/graph-dir.c b/xlators/meta/src/graph-dir.c
index 3cd6b482e74..a8f4787880d 100644
--- a/xlators/meta/src/graph-dir.c
+++ b/xlators/meta/src/graph-dir.c
@@ -8,99 +8,91 @@
    cases as published by the Free Software Foundation.
 */
 
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
-
-#include "xlator.h"
-#include "defaults.h"
+#include <glusterfs/xlator.h>
+#include <glusterfs/defaults.h>
 
 #include "meta-mem-types.h"
 #include "meta.h"
 #include "meta-hooks.h"
 
-
 static struct meta_dirent graph_dir_dirents[] = {
-	DOT_DOTDOT,
-
-	{ .name = "top",
-	  .type = IA_IFLNK,
-	  .hook = meta_top_link_hook,
-	},
-	{ .name = "volfile",
-	  .type = IA_IFREG,
-	  .hook = meta_volfile_file_hook,
-	},
-	{ .name = NULL }
-};
-
+    DOT_DOTDOT,
+
+    {
+        .name = "top",
+        .type = IA_IFLNK,
+        .hook = meta_top_link_hook,
+    },
+    {
+        .name = "volfile",
+        .type = IA_IFREG,
+        .hook = meta_volfile_file_hook,
+    },
+    {.name = NULL}};
 
 static int
-graph_dir_fill (xlator_t *this, inode_t *inode, struct meta_dirent **dp)
+graph_dir_fill(xlator_t *this, inode_t *inode, struct meta_dirent **dp)
 {
-	struct meta_dirent *dirents = NULL;
-	glusterfs_graph_t *graph = NULL;
-	int i = 0;
-	int count = 0;
-	xlator_t *xl = NULL;
-
-	graph = meta_ctx_get (inode, this);
-
-	for (xl = graph->first; xl; xl = xl->next)
-		count++;
-
-	dirents = GF_CALLOC (sizeof (*dirents), count, gf_meta_mt_dirents_t);
-	if (!dirents)
-		return -1;
-
-	i = 0;
-	for (xl = graph->first; xl; xl = xl->next) {
-		dirents[i].name = gf_strdup (xl->name);
-		dirents[i].type = IA_IFDIR;
-		dirents[i].hook = meta_xlator_dir_hook;
-		i++;
-	}
-
-	*dp = dirents;
-	return i;
+    struct meta_dirent *dirents = NULL;
+    glusterfs_graph_t *graph = NULL;
+    int i = 0;
+    int count = 0;
+    xlator_t *xl = NULL;
+
+    graph = meta_ctx_get(inode, this);
+
+    for (xl = graph->first; xl; xl = xl->next)
+        count++;
+
+    dirents = GF_MALLOC(sizeof(*dirents) * count, gf_meta_mt_dirents_t);
+    if (!dirents)
+        return -1;
+
+    i = 0;
+    for (xl = graph->first; xl; xl = xl->next) {
+        dirents[i].name = gf_strdup(xl->name);
+        dirents[i].type = IA_IFDIR;
+        dirents[i].hook = meta_xlator_dir_hook;
+        i++;
+    }
+
+    *dp = dirents;
+    return i;
 }
 
-
 struct meta_ops graph_dir_ops = {
-	.fixed_dirents = graph_dir_dirents,
-	.dir_fill = graph_dir_fill,
+    .fixed_dirents = graph_dir_dirents,
+    .dir_fill = graph_dir_fill,
 };
 
-
 static glusterfs_graph_t *
-glusterfs_graph_lookup (xlator_t *this, const char *graph_uuid)
+glusterfs_graph_lookup(xlator_t *this, const char *graph_uuid)
 {
-	glusterfs_graph_t *graph = NULL;
-	glusterfs_graph_t *tmp = NULL;
-
-	list_for_each_entry (tmp, &this->ctx->graphs, list) {
-		if (strcmp (graph_uuid, tmp->graph_uuid) == 0) {
-			graph = tmp;
-			break;
-		}
-	}
-
-	return graph;
+    glusterfs_graph_t *graph = NULL;
+    glusterfs_graph_t *tmp = NULL;
+
+    list_for_each_entry(tmp, &this->ctx->graphs, list)
+    {
+        if (strcmp(graph_uuid, tmp->graph_uuid) == 0) {
+            graph = tmp;
+            break;
+        }
+    }
+
+    return graph;
 }
 
-
 int
-meta_graph_dir_hook (call_frame_t *frame, xlator_t *this, loc_t *loc,
-		     dict_t *xdata)
+meta_graph_dir_hook(call_frame_t *frame, xlator_t *this, loc_t *loc,
+                    dict_t *xdata)
 {
-	glusterfs_graph_t *graph = NULL;
+    glusterfs_graph_t *graph = NULL;
 
-	graph = glusterfs_graph_lookup (this, loc->name);
+    graph = glusterfs_graph_lookup(this, loc->name);
 
-	meta_ops_set (loc->inode, this, &graph_dir_ops);
+    meta_ops_set(loc->inode, this, &graph_dir_ops);
 
-	meta_ctx_set (loc->inode, this, (void *) graph);
+    meta_ctx_set(loc->inode, this, (void *)graph);
 
-	return 0;
+    return 0;
 }
diff --git a/xlators/meta/src/graphs-dir.c b/xlators/meta/src/graphs-dir.c
index 4a538fb6176..a1ffbca7d5a 100644
--- a/xlators/meta/src/graphs-dir.c
+++ b/xlators/meta/src/graphs-dir.c
@@ -8,72 +8,60 @@
    cases as published by the Free Software Foundation.
 */
 
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
-
-#include "xlator.h"
-#include "defaults.h"
+#include <glusterfs/xlator.h>
+#include <glusterfs/defaults.h>
 
 #include "meta-mem-types.h"
 #include "meta.h"
 #include "meta-hooks.h"
 
-
 static struct meta_dirent graphs_dir_dirents[] = {
-	DOT_DOTDOT,
-
-	{ .name = "active",
-	  .type = IA_IFLNK,
-	  .hook = meta_active_link_hook,
-	},
-	{ .name = NULL }
-};
+    DOT_DOTDOT,
 
+    {
+        .name = "active",
+        .type = IA_IFLNK,
+        .hook = meta_active_link_hook,
+    },
+    {.name = NULL}};
 
 static int
-graphs_dir_fill (xlator_t *this, inode_t *dir, struct meta_dirent **dp)
+graphs_dir_fill(xlator_t *this, inode_t *dir, struct meta_dirent **dp)
 {
-	glusterfs_graph_t *graph = NULL;
-	int graphs_count = 0;
-	int i = 0;
-	struct meta_dirent *dirents = NULL;
-
-	list_for_each_entry (graph, &this->ctx->graphs, list) {
-		graphs_count++;
-	}
-
-	dirents = GF_CALLOC (sizeof (*dirents), graphs_count + 3,
-			     gf_meta_mt_dirents_t);
-	if (!dirents)
-		return -1;
-
-	i = 0;
-	list_for_each_entry (graph, &this->ctx->graphs, list) {
-		dirents[i].name = gf_strdup (graph->graph_uuid);
-		dirents[i].type = IA_IFDIR;
-		dirents[i].hook = meta_graph_dir_hook;
-		i++;
-	}
-
-	*dp = dirents;
-
-	return i;
+    glusterfs_graph_t *graph = NULL;
+    int graphs_count = 0;
+    int i = 0;
+    struct meta_dirent *dirents = NULL;
+
+    list_for_each_entry(graph, &this->ctx->graphs, list) { graphs_count++; }
+
+    dirents = GF_CALLOC(sizeof(*dirents), graphs_count + 3,
+                        gf_meta_mt_dirents_t);
+    if (!dirents)
+        return -1;
+
+    i = 0;
+    list_for_each_entry(graph, &this->ctx->graphs, list)
+    {
+        dirents[i].name = gf_strdup(graph->graph_uuid);
+        dirents[i].type = IA_IFDIR;
+        dirents[i].hook = meta_graph_dir_hook;
+        i++;
+    }
+
+    *dp = dirents;
+
+    return i;
 }
 
-
-struct meta_ops graphs_dir_ops = {
-	.fixed_dirents = graphs_dir_dirents,
-	.dir_fill = graphs_dir_fill
-};
-
+struct meta_ops graphs_dir_ops = {.fixed_dirents = graphs_dir_dirents,
+                                  .dir_fill = graphs_dir_fill};
 
 int
-meta_graphs_dir_hook (call_frame_t *frame, xlator_t *this, loc_t *loc,
-		      dict_t *xdata)
+meta_graphs_dir_hook(call_frame_t *frame, xlator_t *this, loc_t *loc,
+                     dict_t *xdata)
 {
-	meta_ops_set (loc->inode, this, &graphs_dir_ops);
+    meta_ops_set(loc->inode, this, &graphs_dir_ops);
 
-	return 0;
+    return 0;
 }
diff --git a/xlators/meta/src/history-file.c b/xlators/meta/src/history-file.c
index d88040fe58f..7742a635fed 100644
--- a/xlators/meta/src/history-file.c
+++ b/xlators/meta/src/history-file.c
@@ -8,45 +8,37 @@
    cases as published by the Free Software Foundation.
 */
 
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
-
-#include "xlator.h"
-#include "defaults.h"
+#include <glusterfs/xlator.h>
+#include <glusterfs/defaults.h>
 
 #include "meta-mem-types.h"
 #include "meta.h"
-#include "strfd.h"
-#include "statedump.h"
-
+#include <glusterfs/strfd.h>
+#include <glusterfs/statedump.h>
 
 static int
-history_file_fill (xlator_t *this, inode_t *file, strfd_t *strfd)
+history_file_fill(xlator_t *this, inode_t *file, strfd_t *strfd)
 {
-	xlator_t *xl = NULL;
+    xlator_t *xl = NULL;
 
-	xl = meta_ctx_get (file, this);
+    xl = meta_ctx_get(file, this);
 
-	gf_proc_dump_xlator_history (xl, strfd);
+    gf_proc_dump_xlator_history(xl, strfd);
 
-	return strfd->size;
+    return strfd->size;
 }
 
-
 static struct meta_ops history_file_ops = {
-	.file_fill = history_file_fill,
+    .file_fill = history_file_fill,
 };
 
-
 int
-meta_history_file_hook (call_frame_t *frame, xlator_t *this, loc_t *loc,
-			dict_t *xdata)
+meta_history_file_hook(call_frame_t *frame, xlator_t *this, loc_t *loc,
+                       dict_t *xdata)
 {
-	meta_ops_set (loc->inode, this, &history_file_ops);
+    meta_ops_set(loc->inode, this, &history_file_ops);
 
-	meta_ctx_set (loc->inode, this, meta_ctx_get (loc->parent, this));
+    meta_ctx_set(loc->inode, this, meta_ctx_get(loc->parent, this));
 
-	return 0;
+    return 0;
 }
diff --git a/xlators/meta/src/logfile-link.c b/xlators/meta/src/logfile-link.c
index 435cab80c33..616a54518c0 100644
--- a/xlators/meta/src/logfile-link.c
+++ b/xlators/meta/src/logfile-link.c
@@ -8,37 +8,27 @@
    cases as published by the Free Software Foundation.
 */
 
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
-
-#include "xlator.h"
-#include "defaults.h"
+#include <glusterfs/xlator.h>
+#include <glusterfs/defaults.h>
 
 #include "meta-mem-types.h"
 #include "meta.h"
 
-
 static int
-logfile_link_fill (xlator_t *this, inode_t *inode, strfd_t *strfd)
+logfile_link_fill(xlator_t *this, inode_t *inode, strfd_t *strfd)
 {
-	strprintf (strfd, "%s", this->ctx->log.filename);
+    strprintf(strfd, "%s", this->ctx->log.filename);
 
-	return 0;
+    return 0;
 }
 
-
-struct meta_ops logfile_link_ops = {
-	.link_fill = logfile_link_fill
-};
-
+struct meta_ops logfile_link_ops = {.link_fill = logfile_link_fill};
 
 int
-meta_logfile_link_hook (call_frame_t *frame, xlator_t *this, loc_t *loc,
-			dict_t *xdata)
+meta_logfile_link_hook(call_frame_t *frame, xlator_t *this, loc_t *loc,
+                       dict_t *xdata)
 {
-	meta_ops_set (loc->inode, this, &logfile_link_ops);
+    meta_ops_set(loc->inode, this, &logfile_link_ops);
 
-	return 0;
+    return 0;
 }
diff --git a/xlators/meta/src/logging-dir.c b/xlators/meta/src/logging-dir.c
index 783d38e8e12..46e6f9e95dd 100644
--- a/xlators/meta/src/logging-dir.c
+++ b/xlators/meta/src/logging-dir.c
@@ -8,44 +8,37 @@
    cases as published by the Free Software Foundation.
 */
 
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
-
-#include "xlator.h"
-#include "defaults.h"
+#include <glusterfs/xlator.h>
+#include <glusterfs/defaults.h>
 
 #include "meta-mem-types.h"
 #include "meta.h"
 #include "meta-hooks.h"
 
-
 static struct meta_dirent logging_dir_dirents[] = {
-	DOT_DOTDOT,
-
-	{ .name = "logfile",
-	  .type = IA_IFLNK,
-	  .hook = meta_logfile_link_hook,
-	},
-	{ .name = "loglevel",
-	  .type = IA_IFREG,
-	  .hook = meta_loglevel_file_hook,
-	},
-	{ .name = NULL }
-};
-
+    DOT_DOTDOT,
+
+    {
+        .name = "logfile",
+        .type = IA_IFLNK,
+        .hook = meta_logfile_link_hook,
+    },
+    {
+        .name = "loglevel",
+        .type = IA_IFREG,
+        .hook = meta_loglevel_file_hook,
+    },
+    {.name = NULL}};
 
 struct meta_ops logging_dir_ops = {
-	.fixed_dirents = logging_dir_dirents,
+    .fixed_dirents = logging_dir_dirents,
 };
 
-
 int
-meta_logging_dir_hook (call_frame_t *frame, xlator_t *this, loc_t *loc,
-		     dict_t *xdata)
+meta_logging_dir_hook(call_frame_t *frame, xlator_t *this, loc_t *loc,
+                      dict_t *xdata)
 {
-	meta_ops_set (loc->inode, this, &logging_dir_ops);
+    meta_ops_set(loc->inode, this, &logging_dir_ops);
 
-	return 0;
+    return 0;
 }
diff --git a/xlators/meta/src/loglevel-file.c b/xlators/meta/src/loglevel-file.c
index 6b00d123a2c..eeeeeaa5907 100644
--- a/xlators/meta/src/loglevel-file.c
+++ b/xlators/meta/src/loglevel-file.c
@@ -8,52 +8,43 @@
    cases as published by the Free Software Foundation.
 */
 
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
-
-#include "xlator.h"
-#include "defaults.h"
+#include <glusterfs/xlator.h>
+#include <glusterfs/defaults.h>
 
 #include "meta-mem-types.h"
 #include "meta.h"
-#include "strfd.h"
-
+#include <glusterfs/strfd.h>
 
 static int
-loglevel_file_fill (xlator_t *this, inode_t *file, strfd_t *strfd)
+loglevel_file_fill(xlator_t *this, inode_t *file, strfd_t *strfd)
 {
-	strprintf (strfd, "%d\n", this->ctx->log.loglevel);
+    strprintf(strfd, "%d\n", this->ctx->log.loglevel);
 
-	return strfd->size;
+    return strfd->size;
 }
 
-
 static int
-loglevel_file_write (xlator_t *this, fd_t *fd, struct iovec *iov, int count)
+loglevel_file_write(xlator_t *this, fd_t *fd, struct iovec *iov, int count)
 {
-	long int level = -1;
+    long int level = -1;
 
-	level = strtol (iov[0].iov_base, NULL, 0);
-	if (level >= GF_LOG_NONE && level <= GF_LOG_TRACE)
-		gf_log_set_loglevel (level);
+    level = strtol(iov[0].iov_base, NULL, 0);
+    if (level >= GF_LOG_NONE && level <= GF_LOG_TRACE)
+        gf_log_set_loglevel(this->ctx, level);
 
-	return iov_length (iov, count);
+    return iov_length(iov, count);
 }
 
-
 static struct meta_ops loglevel_file_ops = {
-	.file_fill = loglevel_file_fill,
-	.file_write = loglevel_file_write,
+    .file_fill = loglevel_file_fill,
+    .file_write = loglevel_file_write,
 };
 
-
 int
-meta_loglevel_file_hook (call_frame_t *frame, xlator_t *this, loc_t *loc,
-			 dict_t *xdata)
+meta_loglevel_file_hook(call_frame_t *frame, xlator_t *this, loc_t *loc,
+                        dict_t *xdata)
 {
-	meta_ops_set (loc->inode, this, &loglevel_file_ops);
+    meta_ops_set(loc->inode, this, &loglevel_file_ops);
 
-	return 0;
+    return 0;
 }
diff --git a/xlators/meta/src/mallinfo-file.c b/xlators/meta/src/mallinfo-file.c
index 38589f515ff..b4396d72189 100644
--- a/xlators/meta/src/mallinfo-file.c
+++ b/xlators/meta/src/mallinfo-file.c
@@ -8,37 +8,29 @@
    cases as published by the Free Software Foundation.
 */
 
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
-
-#include "xlator.h"
-#include "defaults.h"
+#include <glusterfs/xlator.h>
+#include <glusterfs/defaults.h>
 
 #include "meta-mem-types.h"
 #include "meta.h"
-#include "statedump.h"
-
+#include <glusterfs/statedump.h>
 
 static int
-mallinfo_file_fill (xlator_t *this, inode_t *file, strfd_t *strfd)
+mallinfo_file_fill(xlator_t *this, inode_t *file, strfd_t *strfd)
 {
-	gf_proc_dump_mallinfo (strfd);
-	return strfd->size;
+    gf_proc_dump_mallinfo(strfd);
+    return strfd->size;
 }
 
-
 static struct meta_ops mallinfo_file_ops = {
-	.file_fill = mallinfo_file_fill,
+    .file_fill = mallinfo_file_fill,
 };
 
-
 int
-meta_mallinfo_file_hook (call_frame_t *frame, xlator_t *this, loc_t *loc,
-			dict_t *xdata)
+meta_mallinfo_file_hook(call_frame_t *frame, xlator_t *this, loc_t *loc,
+                        dict_t *xdata)
 {
-	meta_ops_set (loc->inode, this, &mallinfo_file_ops);
+    meta_ops_set(loc->inode, this, &mallinfo_file_ops);
 
-	return 0;
+    return 0;
 }
diff --git a/xlators/meta/src/measure-file.c b/xlators/meta/src/measure-file.c
index e4498ab092e..52e92e48590 100644
--- a/xlators/meta/src/measure-file.c
+++ b/xlators/meta/src/measure-file.c
@@ -8,50 +8,42 @@
    cases as published by the Free Software Foundation.
 */
 
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
-
-#include "xlator.h"
-#include "defaults.h"
+#include <glusterfs/xlator.h>
+#include <glusterfs/defaults.h>
 
 #include "meta-mem-types.h"
 #include "meta.h"
-#include "strfd.h"
-
+#include <glusterfs/strfd.h>
 
 static int
-measure_file_fill (xlator_t *this, inode_t *file, strfd_t *strfd)
+measure_file_fill(xlator_t *this, inode_t *file, strfd_t *strfd)
 {
-	strprintf (strfd, "%d\n", this->ctx->measure_latency);
+    strprintf(strfd, "%d\n", this->ctx->measure_latency);
 
-	return strfd->size;
+    return strfd->size;
 }
 
-
 static int
-measure_file_write (xlator_t *this, fd_t *fd, struct iovec *iov, int count)
+measure_file_write(xlator_t *this, fd_t *fd, struct iovec *iov, int count)
 {
-	long int num = -1;
+    long int num = -1;
 
-	num = strtol (iov[0].iov_base, NULL, 0);
-	this->ctx->measure_latency = !!num;
+    num = strtol(iov[0].iov_base, NULL, 0);
+    this->ctx->measure_latency = !!num;
 
-	return iov_length (iov, count);
+    return iov_length(iov, count);
 }
 
 static struct meta_ops measure_file_ops = {
-	.file_fill = measure_file_fill,
-	.file_write = measure_file_write,
+    .file_fill = measure_file_fill,
+    .file_write = measure_file_write,
 };
 
-
 int
-meta_measure_file_hook (call_frame_t *frame, xlator_t *this, loc_t *loc,
-			dict_t *xdata)
+meta_measure_file_hook(call_frame_t *frame, xlator_t *this, loc_t *loc,
+                       dict_t *xdata)
 {
-	meta_ops_set (loc->inode, this, &measure_file_ops);
+    meta_ops_set(loc->inode, this, &measure_file_ops);
 
-	return 0;
+    return 0;
 }
diff --git a/xlators/meta/src/meminfo-file.c b/xlators/meta/src/meminfo-file.c
index 807446cfdc4..d889dfb2ae8 100644
--- a/xlators/meta/src/meminfo-file.c
+++ b/xlators/meta/src/meminfo-file.c
@@ -8,45 +8,37 @@
    cases as published by the Free Software Foundation.
 */
 
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
-
-#include "xlator.h"
-#include "defaults.h"
+#include <glusterfs/xlator.h>
+#include <glusterfs/defaults.h>
 
 #include "meta-mem-types.h"
 #include "meta.h"
-#include "strfd.h"
-#include "statedump.h"
-
+#include <glusterfs/strfd.h>
+#include <glusterfs/statedump.h>
 
 static int
-meminfo_file_fill (xlator_t *this, inode_t *file, strfd_t *strfd)
+meminfo_file_fill(xlator_t *this, inode_t *file, strfd_t *strfd)
 {
-	xlator_t *xl = NULL;
+    xlator_t *xl = NULL;
 
-	xl = meta_ctx_get (file, this);
+    xl = meta_ctx_get(file, this);
 
-	gf_proc_dump_xlator_meminfo (xl, strfd);
+    gf_proc_dump_xlator_meminfo(xl, strfd);
 
-	return strfd->size;
+    return strfd->size;
 }
 
-
 static struct meta_ops meminfo_file_ops = {
-	.file_fill = meminfo_file_fill,
+    .file_fill = meminfo_file_fill,
 };
 
-
 int
-meta_meminfo_file_hook (call_frame_t *frame, xlator_t *this, loc_t *loc,
-			dict_t *xdata)
+meta_meminfo_file_hook(call_frame_t *frame, xlator_t *this, loc_t *loc,
+                       dict_t *xdata)
 {
-	meta_ops_set (loc->inode, this, &meminfo_file_ops);
+    meta_ops_set(loc->inode, this, &meminfo_file_ops);
 
-	meta_ctx_set (loc->inode, this, meta_ctx_get (loc->parent, this));
+    meta_ctx_set(loc->inode, this, meta_ctx_get(loc->parent, this));
 
-	return 0;
+    return 0;
 }
diff --git a/xlators/meta/src/meta-defaults.c b/xlators/meta/src/meta-defaults.c
index c34055bb221..91c328473f8 100644
--- a/xlators/meta/src/meta-defaults.c
+++ b/xlators/meta/src/meta-defaults.c
@@ -8,634 +8,648 @@
    cases as published by the Free Software Foundation.
 */
 
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
-
-#include "xlator.h"
-#include "defaults.h"
+#include <glusterfs/xlator.h>
+#include <glusterfs/defaults.h>
 
 #include "meta-mem-types.h"
 #include "meta.h"
 
-#include "compat-errno.h"
+#include <glusterfs/compat-errno.h>
 
 int
-meta_default_fgetxattr (call_frame_t *frame, xlator_t *this, fd_t *fd,
-			const char *name, dict_t *xdata)
+meta_default_fgetxattr(call_frame_t *frame, xlator_t *this, fd_t *fd,
+                       const char *name, dict_t *xdata)
 {
-        return default_fgetxattr_failure_cbk (frame, EPERM);
+    return default_fgetxattr_failure_cbk(frame, EPERM);
 }
 
 int
-meta_default_fsetxattr (call_frame_t *frame, xlator_t *this, fd_t *fd,
-			dict_t *dict, int32_t flags, dict_t *xdata)
+meta_default_fsetxattr(call_frame_t *frame, xlator_t *this, fd_t *fd,
+                       dict_t *dict, int32_t flags, dict_t *xdata)
 {
-        return default_fsetxattr_failure_cbk (frame, EPERM);
+    return default_fsetxattr_failure_cbk(frame, EPERM);
 }
 
 int
-meta_default_setxattr (call_frame_t *frame, xlator_t *this, loc_t *loc,
-		       dict_t *dict, int32_t flags, dict_t *xdata)
+meta_default_setxattr(call_frame_t *frame, xlator_t *this, loc_t *loc,
+                      dict_t *dict, int32_t flags, dict_t *xdata)
 {
-        return default_setxattr_failure_cbk (frame, EPERM);
+    return default_setxattr_failure_cbk(frame, EPERM);
 }
 
 int
-meta_default_statfs (call_frame_t *frame, xlator_t *this, loc_t *loc,
-		     dict_t *xdata)
+meta_default_statfs(call_frame_t *frame, xlator_t *this, loc_t *loc,
+                    dict_t *xdata)
 {
-        return default_statfs_failure_cbk (frame, EPERM);
+    return default_statfs_failure_cbk(frame, EPERM);
 }
 
 int
-meta_default_fsyncdir (call_frame_t *frame, xlator_t *this, fd_t *fd,
-		       int32_t flags, dict_t *xdata)
+meta_default_fsyncdir(call_frame_t *frame, xlator_t *this, fd_t *fd,
+                      int32_t flags, dict_t *xdata)
 {
-        return default_fsyncdir_failure_cbk (frame, EPERM);
+    return default_fsyncdir_failure_cbk(frame, EPERM);
 }
 
 int
-meta_default_opendir (call_frame_t *frame, xlator_t *this, loc_t *loc,
-		      fd_t *fd, dict_t *xdata)
+meta_default_opendir(call_frame_t *frame, xlator_t *this, loc_t *loc, fd_t *fd,
+                     dict_t *xdata)
 {
-	META_STACK_UNWIND (opendir, frame, 0, 0, fd, xdata);
-	return 0;
+    META_STACK_UNWIND(opendir, frame, 0, 0, fd, xdata);
+    return 0;
 }
 
 int
-meta_default_fstat (call_frame_t *frame, xlator_t *this, fd_t *fd,
-		    dict_t *xdata)
+meta_default_fstat(call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *xdata)
 {
-	struct iatt iatt = { };
+    struct iatt iatt = {};
 
-	meta_iatt_fill (&iatt, fd->inode, fd->inode->ia_type);
+    meta_iatt_fill(&iatt, fd->inode, fd->inode->ia_type);
 
-	META_STACK_UNWIND (fstat, frame, 0, 0, &iatt, xdata);
+    META_STACK_UNWIND(fstat, frame, 0, 0, &iatt, xdata);
 
-	return 0;
+    return 0;
 }
 
 int
-meta_default_fsync (call_frame_t *frame, xlator_t *this, fd_t *fd,
-		    int32_t flags, dict_t *xdata)
+meta_default_fsync(call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t flags,
+                   dict_t *xdata)
 {
-        return default_fsync_failure_cbk (frame, EPERM);
+    return default_fsync_failure_cbk(frame, EPERM);
 }
 
 int
-meta_default_flush (call_frame_t *frame, xlator_t *this, fd_t *fd,
-		    dict_t *xdata)
+meta_default_flush(call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *xdata)
 {
-	META_STACK_UNWIND (flush, frame, 0, 0, xdata);
-	return 0;
+    META_STACK_UNWIND(flush, frame, 0, 0, xdata);
+    return 0;
 }
 
 int
-meta_default_writev (call_frame_t *frame, xlator_t *this, fd_t *fd,
-		     struct iovec *vector, int32_t count, off_t off,
-		     uint32_t flags, struct iobref *iobref, dict_t *xdata)
+meta_default_writev(call_frame_t *frame, xlator_t *this, fd_t *fd,
+                    struct iovec *vector, int32_t count, off_t off,
+                    uint32_t flags, struct iobref *iobref, dict_t *xdata)
 {
-	struct meta_ops *ops = NULL;
-	int ret = 0;
-	struct iatt dummy = { };
+    struct meta_ops *ops = NULL;
+    int ret = 0;
+    struct iatt dummy = {};
 
-	ops = meta_ops_get (fd->inode, this);
-	if (!ops)
-		goto err;
+    ops = meta_ops_get(fd->inode, this);
+    if (!ops)
+        goto err;
 
-	if (!ops->file_write)
-		goto err;
+    if (!ops->file_write)
+        goto err;
 
-	ret = ops->file_write (this, fd, vector, count);
+    ret = ops->file_write(this, fd, vector, count);
 
-	META_STACK_UNWIND (writev, frame, (ret >= 0 ? ret : -1), (ret < 0 ? -ret : 0),
-			   &dummy, &dummy, xdata);
-	return 0;
+    META_STACK_UNWIND(writev, frame, (ret >= 0 ? ret : -1),
+                      (ret < 0 ? -ret : 0), &dummy, &dummy, xdata);
+    return 0;
 err:
-        return default_writev_failure_cbk (frame, EPERM);
+    return default_writev_failure_cbk(frame, EPERM);
 }
 
 int
-meta_default_readv (call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
-		    off_t offset, uint32_t flags, dict_t *xdata)
+meta_default_readv(call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
+                   off_t offset, uint32_t flags, dict_t *xdata)
 {
-	meta_fd_t *meta_fd = NULL;
-	struct iovec iov = {};
-	struct iobuf *iobuf = NULL;
-	struct iobref *iobref = NULL;
-	off_t copy_offset = 0;
-	size_t copy_size = 0;
-	struct iatt iatt = {};
+    meta_fd_t *meta_fd = NULL;
+    struct iovec iov = {};
+    struct iobuf *iobuf = NULL;
+    struct iobref *iobref = NULL;
+    off_t copy_offset = 0;
+    int copy_size = 0;
+    struct iatt iatt = {};
 
+    meta_fd = meta_fd_get(fd, this);
+    if (!meta_fd)
+        return default_readv_failure_cbk(frame, ENODATA);
 
-	meta_fd = meta_fd_get (fd, this);
-	if (!meta_fd)
-		return default_readv_failure_cbk (frame, ENODATA);
+    if (!meta_fd->size)
+        meta_file_fill(this, fd);
 
-	if (!meta_fd->size)
-		meta_file_fill (this, fd);
+    iobuf = iobuf_get2(this->ctx->iobuf_pool, size);
+    if (!iobuf)
+        return default_readv_failure_cbk(frame, ENOMEM);
 
-	iobuf = iobuf_get2 (this->ctx->iobuf_pool, size);
-	if (!iobuf)
-		return default_readv_failure_cbk (frame, ENOMEM);
+    iobref = iobref_new();
+    if (!iobref) {
+        iobuf_unref(iobuf);
+        return default_readv_failure_cbk(frame, ENOMEM);
+    }
 
-	iobref = iobref_new ();
-	if (!iobref) {
-		iobuf_unref (iobuf);
-		return default_readv_failure_cbk (frame, ENOMEM);
-	}
+    if (iobref_add(iobref, iobuf) != 0) {
+        iobref_unref(iobref);
+        iobuf_unref(iobuf);
+        return default_readv_failure_cbk(frame, ENOMEM);
+    }
 
-	if (iobref_add (iobref, iobuf) != 0) {
-		iobref_unref (iobref);
-		iobuf_unref (iobuf);
-		return default_readv_failure_cbk (frame, ENOMEM);
-	}
+    iov.iov_base = iobuf_ptr(iobuf);
 
-	iov.iov_base = iobuf_ptr (iobuf);
+    /* iobref would have taken a ref */
+    iobuf_unref(iobuf);
 
-	copy_offset = min (meta_fd->size, offset);
-	copy_size = min (size, (meta_fd->size - copy_offset));
+    copy_offset = min(meta_fd->size, offset);
+    copy_size = min(size, (meta_fd->size - copy_offset));
 
-	if (copy_size)
-		memcpy (iov.iov_base, meta_fd->data + copy_offset, copy_size);
-	iov.iov_len = copy_size;
+    if (copy_size)
+        memcpy(iov.iov_base, meta_fd->data + copy_offset, copy_size);
+    iov.iov_len = copy_size;
 
-	META_STACK_UNWIND (readv, frame, copy_size, 0, &iov, 1, &iatt, iobref, 0);
+    META_STACK_UNWIND(readv, frame, copy_size, 0, &iov, 1, &iatt, iobref, 0);
 
-	return 0;
-}
+    iobref_unref(iobref);
 
+    return 0;
+}
 
 int
-meta_default_open (call_frame_t *frame, xlator_t *this, loc_t *loc,
-		   int32_t flags, fd_t *fd, dict_t *xdata)
+meta_default_open(call_frame_t *frame, xlator_t *this, loc_t *loc,
+                  int32_t flags, fd_t *fd, dict_t *xdata)
 {
-	dict_t *xdata_rsp = NULL;
+    dict_t *xdata_rsp = NULL;
 
-	xdata_rsp = meta_direct_io_mode (xdata, frame);
+    xdata_rsp = meta_direct_io_mode(xdata, frame);
 
-	META_STACK_UNWIND (open, frame, 0, 0, fd, xdata_rsp);
+    META_STACK_UNWIND(open, frame, 0, 0, fd, xdata_rsp);
 
-	return 0;
+    return 0;
 }
 
 int
-meta_default_create (call_frame_t *frame, xlator_t *this, loc_t *loc,
-		     int32_t flags, mode_t mode, mode_t umask, fd_t *fd,
-		     dict_t *xdata)
+meta_default_create(call_frame_t *frame, xlator_t *this, loc_t *loc,
+                    int32_t flags, mode_t mode, mode_t umask, fd_t *fd,
+                    dict_t *xdata)
 {
-        return default_create_failure_cbk (frame, EPERM);
+    return default_create_failure_cbk(frame, EPERM);
 }
 
 int
-meta_default_link (call_frame_t *frame, xlator_t *this, loc_t *oldloc,
-		   loc_t *newloc, dict_t *xdata)
+meta_default_link(call_frame_t *frame, xlator_t *this, loc_t *oldloc,
+                  loc_t *newloc, dict_t *xdata)
 {
-        return default_link_failure_cbk (frame, EPERM);
+    return default_link_failure_cbk(frame, EPERM);
 }
 
 int
-meta_default_rename (call_frame_t *frame, xlator_t *this, loc_t *oldloc,
-		     loc_t *newloc, dict_t *xdata)
+meta_default_rename(call_frame_t *frame, xlator_t *this, loc_t *oldloc,
+                    loc_t *newloc, dict_t *xdata)
 {
-        return default_rename_failure_cbk (frame, EPERM);
+    return default_rename_failure_cbk(frame, EPERM);
 }
 
 int
-meta_default_symlink (call_frame_t *frame, xlator_t *this, const char *linkpath,
-		      loc_t *loc, mode_t umask, dict_t *xdata)
+meta_default_symlink(call_frame_t *frame, xlator_t *this, const char *linkpath,
+                     loc_t *loc, mode_t umask, dict_t *xdata)
 {
-        return default_symlink_failure_cbk (frame, EPERM);
+    return default_symlink_failure_cbk(frame, EPERM);
 }
 
 int
-meta_default_rmdir (call_frame_t *frame, xlator_t *this, loc_t *loc, int flags,
-		    dict_t *xdata)
+meta_default_rmdir(call_frame_t *frame, xlator_t *this, loc_t *loc, int flags,
+                   dict_t *xdata)
 {
-        return default_rmdir_failure_cbk (frame, EPERM);
+    return default_rmdir_failure_cbk(frame, EPERM);
 }
 
 int
-meta_default_unlink (call_frame_t *frame, xlator_t *this, loc_t *loc, int xflag,
-		     dict_t *xdata)
+meta_default_unlink(call_frame_t *frame, xlator_t *this, loc_t *loc, int xflag,
+                    dict_t *xdata)
 {
-        return default_unlink_failure_cbk (frame, EPERM);
+    return default_unlink_failure_cbk(frame, EPERM);
 }
 
 int
-meta_default_mkdir (call_frame_t *frame, xlator_t *this, loc_t *loc,
-		    mode_t mode, mode_t umask, dict_t *xdata)
+meta_default_mkdir(call_frame_t *frame, xlator_t *this, loc_t *loc, mode_t mode,
+                   mode_t umask, dict_t *xdata)
 {
-        return default_mkdir_failure_cbk (frame, EPERM);
+    return default_mkdir_failure_cbk(frame, EPERM);
 }
 
 int
-meta_default_mknod (call_frame_t *frame, xlator_t *this, loc_t *loc,
-		    mode_t mode, dev_t rdev, mode_t umask, dict_t *xdata)
+meta_default_mknod(call_frame_t *frame, xlator_t *this, loc_t *loc, mode_t mode,
+                   dev_t rdev, mode_t umask, dict_t *xdata)
 {
-        return default_mknod_failure_cbk (frame, EPERM);
+    return default_mknod_failure_cbk(frame, EPERM);
 }
 
 int
-meta_default_readlink (call_frame_t *frame, xlator_t *this, loc_t *loc,
-		       size_t size, dict_t *xdata)
+meta_default_readlink(call_frame_t *frame, xlator_t *this, loc_t *loc,
+                      size_t size, dict_t *xdata)
 {
-	struct meta_ops *ops = NULL;
-	strfd_t *strfd = NULL;
-	struct iatt iatt = { };
+    struct meta_ops *ops = NULL;
+    strfd_t *strfd = NULL;
+    struct iatt iatt = {};
+    int len = -1;
 
-	ops = meta_ops_get (loc->inode, this);
-	if (!ops->link_fill) {
-		META_STACK_UNWIND (readlink, frame, -1, EPERM, 0, 0, 0);
-		return 0;
-	}
+    ops = meta_ops_get(loc->inode, this);
+    if (!ops || !ops->link_fill) {
+        META_STACK_UNWIND(readlink, frame, -1, EPERM, 0, 0, 0);
+        return 0;
+    }
 
-	strfd = strfd_open ();
-	if (!strfd) {
-		META_STACK_UNWIND (readlink, frame, -1, ENOMEM, 0, 0, 0);
-		return 0;
-	}
+    strfd = strfd_open();
+    if (!strfd) {
+        META_STACK_UNWIND(readlink, frame, -1, ENOMEM, 0, 0, 0);
+        return 0;
+    }
 
-	ops->link_fill (this, loc->inode, strfd);
+    ops->link_fill(this, loc->inode, strfd);
 
-	meta_iatt_fill (&iatt, loc->inode, IA_IFLNK);
+    meta_iatt_fill(&iatt, loc->inode, IA_IFLNK);
 
-	if (strfd->data)
-		META_STACK_UNWIND (readlink, frame, strlen (strfd->data), 0,
-				   strfd->data, &iatt, xdata);
-	else
-		META_STACK_UNWIND (readlink, frame, -1, ENODATA, 0, 0, 0);
+    if (strfd->data) {
+        len = strlen(strfd->data);
+        META_STACK_UNWIND(readlink, frame, len, 0, strfd->data, &iatt, xdata);
+    } else
+        META_STACK_UNWIND(readlink, frame, -1, ENODATA, 0, 0, 0);
 
-	strfd_close (strfd);
+    strfd_close(strfd);
 
-	return 0;
+    return 0;
 }
 
 int
-meta_default_access (call_frame_t *frame, xlator_t *this, loc_t *loc,
-		     int32_t mask, dict_t *xdata)
+meta_default_access(call_frame_t *frame, xlator_t *this, loc_t *loc,
+                    int32_t mask, dict_t *xdata)
 {
-        return default_access_failure_cbk (frame, EPERM);
+    return default_access_failure_cbk(frame, EPERM);
 }
 
 int
-meta_default_ftruncate (call_frame_t *frame, xlator_t *this, fd_t *fd,
-			off_t offset, dict_t *xdata)
+meta_default_ftruncate(call_frame_t *frame, xlator_t *this, fd_t *fd,
+                       off_t offset, dict_t *xdata)
 {
-	struct iatt iatt = { };
+    struct iatt iatt = {};
 
-	meta_iatt_fill (&iatt, fd->inode, IA_IFREG);
+    meta_iatt_fill(&iatt, fd->inode, IA_IFREG);
 
-	META_STACK_UNWIND (ftruncate, frame, 0, 0, &iatt, &iatt, xdata);
+    META_STACK_UNWIND(ftruncate, frame, 0, 0, &iatt, &iatt, xdata);
 
-        return 0;
+    return 0;
 }
 
 int
-meta_default_getxattr (call_frame_t *frame, xlator_t *this, loc_t *loc,
-		       const char *name, dict_t *xdata)
+meta_default_getxattr(call_frame_t *frame, xlator_t *this, loc_t *loc,
+                      const char *name, dict_t *xdata)
 {
-        return default_getxattr_failure_cbk (frame, EPERM);
+    return default_getxattr_failure_cbk(frame, EPERM);
 }
 
 int
-meta_default_xattrop (call_frame_t *frame, xlator_t *this, loc_t *loc,
-		      gf_xattrop_flags_t flags, dict_t *dict, dict_t *xdata)
+meta_default_xattrop(call_frame_t *frame, xlator_t *this, loc_t *loc,
+                     gf_xattrop_flags_t flags, dict_t *dict, dict_t *xdata)
 {
-        return default_xattrop_failure_cbk (frame, EPERM);
+    return default_xattrop_failure_cbk(frame, EPERM);
 }
 
 int
-meta_default_fxattrop (call_frame_t *frame, xlator_t *this, fd_t *fd,
-		       gf_xattrop_flags_t flags, dict_t *dict, dict_t *xdata)
+meta_default_fxattrop(call_frame_t *frame, xlator_t *this, fd_t *fd,
+                      gf_xattrop_flags_t flags, dict_t *dict, dict_t *xdata)
 {
-        return default_fxattrop_failure_cbk (frame, EPERM);
+    return default_fxattrop_failure_cbk(frame, EPERM);
 }
 
 int
-meta_default_removexattr (call_frame_t *frame, xlator_t *this, loc_t *loc,
-			  const char *name, dict_t *xdata)
+meta_default_removexattr(call_frame_t *frame, xlator_t *this, loc_t *loc,
+                         const char *name, dict_t *xdata)
 {
-        return default_removexattr_failure_cbk (frame, EPERM);
+    return default_removexattr_failure_cbk(frame, EPERM);
 }
 
 int
-meta_default_fremovexattr (call_frame_t *frame, xlator_t *this, fd_t *fd,
-			   const char *name, dict_t *xdata)
+meta_default_fremovexattr(call_frame_t *frame, xlator_t *this, fd_t *fd,
+                          const char *name, dict_t *xdata)
 {
-        return default_fremovexattr_failure_cbk (frame, EPERM);
+    return default_fremovexattr_failure_cbk(frame, EPERM);
 }
 
 int
-meta_default_lk (call_frame_t *frame, xlator_t *this, fd_t *fd,
-		 int32_t cmd, struct gf_flock *lock, dict_t *xdata)
+meta_default_lk(call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t cmd,
+                struct gf_flock *lock, dict_t *xdata)
 {
-        return default_lk_failure_cbk (frame, EPERM);
+    return default_lk_failure_cbk(frame, EPERM);
 }
 
-
 int
-meta_default_inodelk (call_frame_t *frame, xlator_t *this, const char *volume,
-		      loc_t *loc, int32_t cmd, struct gf_flock *lock,
-		      dict_t *xdata)
+meta_default_inodelk(call_frame_t *frame, xlator_t *this, const char *volume,
+                     loc_t *loc, int32_t cmd, struct gf_flock *lock,
+                     dict_t *xdata)
 {
-        return default_inodelk_failure_cbk (frame, EPERM);
+    return default_inodelk_failure_cbk(frame, EPERM);
 }
 
 int
-meta_default_finodelk (call_frame_t *frame, xlator_t *this, const char *volume,
-		       fd_t *fd, int32_t cmd, struct gf_flock *lock,
-		       dict_t *xdata)
+meta_default_finodelk(call_frame_t *frame, xlator_t *this, const char *volume,
+                      fd_t *fd, int32_t cmd, struct gf_flock *lock,
+                      dict_t *xdata)
 {
-        return default_finodelk_failure_cbk (frame, EPERM);
+    return default_finodelk_failure_cbk(frame, EPERM);
 }
 
 int
-meta_default_entrylk (call_frame_t *frame, xlator_t *this, const char *volume,
-		      loc_t *loc, const char *basename, entrylk_cmd cmd,
-		      entrylk_type type, dict_t *xdata)
+meta_default_entrylk(call_frame_t *frame, xlator_t *this, const char *volume,
+                     loc_t *loc, const char *basename, entrylk_cmd cmd,
+                     entrylk_type type, dict_t *xdata)
 {
-        return default_entrylk_failure_cbk (frame, EPERM);
+    return default_entrylk_failure_cbk(frame, EPERM);
 }
 
 int
-meta_default_fentrylk (call_frame_t *frame, xlator_t *this, const char *volume,
-		       fd_t *fd, const char *basename, entrylk_cmd cmd,
-		       entrylk_type type, dict_t *xdata)
+meta_default_fentrylk(call_frame_t *frame, xlator_t *this, const char *volume,
+                      fd_t *fd, const char *basename, entrylk_cmd cmd,
+                      entrylk_type type, dict_t *xdata)
 {
-        return default_fentrylk_failure_cbk (frame, EPERM);
+    return default_fentrylk_failure_cbk(frame, EPERM);
 }
 
 int
-meta_default_rchecksum (call_frame_t *frame, xlator_t *this, fd_t *fd,
-			off_t offset, int32_t len, dict_t *xdata)
+meta_default_rchecksum(call_frame_t *frame, xlator_t *this, fd_t *fd,
+                       off_t offset, int32_t len, dict_t *xdata)
 {
-        return default_rchecksum_failure_cbk (frame, EPERM);
+    return default_rchecksum_failure_cbk(frame, EPERM);
 }
 
-
 int
-meta_default_readdir (call_frame_t *frame, xlator_t *this, fd_t *fd,
-		      size_t size, off_t off, dict_t *xdata)
+meta_default_readdir(call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
+                     off_t off, dict_t *xdata)
 {
-	meta_fd_t *meta_fd = NULL;
-	int i = 0;
-	gf_dirent_t head;
-	gf_dirent_t *list = NULL;
-	int ret = 0;
-	int this_size = 0;
-	int filled_size = 0;
-	int fixed_size = 0;
-	int dyn_size = 0;
-	struct meta_dirent *fixed_dirents = NULL;
-	struct meta_dirent *dyn_dirents = NULL;
-	struct meta_dirent *dirents = NULL;
-	struct meta_dirent *end = NULL;
-	struct meta_ops *ops = NULL;
-
-	INIT_LIST_HEAD (&head.list);
+    meta_fd_t *meta_fd = NULL;
+    int i = 0;
+    gf_dirent_t head;
+    gf_dirent_t *list = NULL;
+    int ret = 0;
+    int this_size = 0;
+    int filled_size = 0;
+    int fixed_size = 0;
+    int dyn_size = 0;
+    struct meta_dirent *fixed_dirents = NULL;
+    struct meta_dirent *dyn_dirents = NULL;
+    struct meta_dirent *dirents = NULL;
+    struct meta_dirent *end = NULL;
+    struct meta_ops *ops = NULL;
 
-	ops = meta_ops_get (fd->inode, this);
-	if (!ops)
-		goto err;
+    INIT_LIST_HEAD(&head.list);
 
-	meta_fd = meta_fd_get (fd, this);
-	if (!meta_fd)
-		goto err;
+    ops = meta_ops_get(fd->inode, this);
+    if (!ops)
+        goto err;
 
-	meta_dir_fill (this, fd);
+    meta_fd = meta_fd_get(fd, this);
+    if (!meta_fd)
+        goto err;
 
-	fixed_dirents = ops->fixed_dirents;
-	fixed_size = fixed_dirents_len (fixed_dirents);
+    meta_dir_fill(this, fd);
 
-	dyn_dirents = meta_fd->dirents;
-	dyn_size = meta_fd->size;
+    fixed_dirents = ops->fixed_dirents;
+    fixed_size = fixed_dirents_len(fixed_dirents);
 
-	for (i = off; i < (fixed_size + dyn_size);) {
-		if (i >= fixed_size) {
-			dirents = dyn_dirents + (i - fixed_size);
-			end = dyn_dirents + dyn_size;
-		} else {
-			dirents = fixed_dirents + i;
-			end = fixed_dirents + fixed_size;
-		}
+    dyn_dirents = meta_fd->dirents;
+    dyn_size = meta_fd->size;
 
-		while (dirents < end) {
-			this_size = sizeof (gf_dirent_t) +
-				strlen (dirents->name) + 1;
-			if (this_size + filled_size > size)
-				goto unwind;
+    for (i = off; i < (fixed_size + dyn_size);) {
+        if (i >= fixed_size) {
+            dirents = dyn_dirents + (i - fixed_size);
+            end = dyn_dirents + dyn_size;
+        } else {
+            dirents = fixed_dirents + i;
+            end = fixed_dirents + fixed_size;
+        }
 
-			list = gf_dirent_for_name (dirents->name);
-			if (!list)
-				break;
+        while (dirents < end) {
+            this_size = sizeof(gf_dirent_t) + strlen(dirents->name) + 1;
+            if (this_size + filled_size > size)
+                goto unwind;
 
-			list->d_off = i + 1;
-			list->d_ino = i + 42;
-			switch (dirents->type) {
-			case IA_IFDIR: list->d_type = DT_DIR; break;
-			case IA_IFCHR: list->d_type = DT_CHR; break;
-			case IA_IFBLK: list->d_type = DT_BLK; break;
-			case IA_IFIFO: list->d_type = DT_FIFO; break;
-			case IA_IFLNK: list->d_type = DT_LNK; break;
-			case IA_IFREG: list->d_type = DT_REG; break;
-			case IA_IFSOCK: list->d_type = DT_SOCK; break;
-			case IA_INVAL: list->d_type = DT_UNKNOWN; break;
-			}
+            list = gf_dirent_for_name(dirents->name);
+            if (!list)
+                break;
 
-			list_add_tail (&list->list, &head.list);
-			ret++; i++; dirents++;
-			filled_size += this_size;
-		}
-	}
+            list->d_off = i + 1;
+            list->d_ino = i + 42;
+            switch (dirents->type) {
+                case IA_IFDIR:
+                    list->d_type = DT_DIR;
+                    break;
+                case IA_IFCHR:
+                    list->d_type = DT_CHR;
+                    break;
+                case IA_IFBLK:
+                    list->d_type = DT_BLK;
+                    break;
+                case IA_IFIFO:
+                    list->d_type = DT_FIFO;
+                    break;
+                case IA_IFLNK:
+                    list->d_type = DT_LNK;
+                    break;
+                case IA_IFREG:
+                    list->d_type = DT_REG;
+                    break;
+                case IA_IFSOCK:
+                    list->d_type = DT_SOCK;
+                    break;
+                case IA_INVAL:
+                    list->d_type = DT_UNKNOWN;
+                    break;
+            }
+
+            list_add_tail(&list->list, &head.list);
+            ret++;
+            i++;
+            dirents++;
+            filled_size += this_size;
+        }
+    }
 
 unwind:
-	META_STACK_UNWIND (readdir, frame, ret, 0, &head, xdata);
+    META_STACK_UNWIND(readdir, frame, ret, 0, &head, xdata);
 
-	gf_dirent_free (&head);
+    gf_dirent_free(&head);
 
-	return 0;
+    return 0;
 err:
-	META_STACK_UNWIND (readdir, frame, -1, ENOMEM, 0, 0);
-	return 0;
+    META_STACK_UNWIND(readdir, frame, -1, ENOMEM, 0, 0);
+    return 0;
 }
 
-
 int
-meta_default_readdirp (call_frame_t *frame, xlator_t *this, fd_t *fd,
-		       size_t size, off_t off, dict_t *xdata)
+meta_default_readdirp(call_frame_t *frame, xlator_t *this, fd_t *fd,
+                      size_t size, off_t off, dict_t *xdata)
 {
-        return meta_default_readdir (frame, this, fd, size, off, xdata);
+    return meta_default_readdir(frame, this, fd, size, off, xdata);
 }
 
 int
-meta_default_setattr (call_frame_t *frame, xlator_t *this, loc_t *loc,
-		      struct iatt *stbuf, int32_t valid,
-		      dict_t *xdata)
+meta_default_setattr(call_frame_t *frame, xlator_t *this, loc_t *loc,
+                     struct iatt *stbuf, int32_t valid, dict_t *xdata)
 {
-        return default_setattr_failure_cbk (frame, EPERM);
+    return default_setattr_failure_cbk(frame, EPERM);
 }
 
 int
-meta_default_truncate (call_frame_t *frame, xlator_t *this, loc_t *loc,
-		       off_t offset, dict_t *xdata)
+meta_default_truncate(call_frame_t *frame, xlator_t *this, loc_t *loc,
+                      off_t offset, dict_t *xdata)
 {
-	struct iatt iatt = { };
+    struct iatt iatt = {};
 
-	meta_iatt_fill (&iatt, loc->inode, IA_IFREG);
+    meta_iatt_fill(&iatt, loc->inode, IA_IFREG);
 
-	META_STACK_UNWIND (truncate, frame, 0, 0, &iatt, &iatt, xdata);
+    META_STACK_UNWIND(truncate, frame, 0, 0, &iatt, &iatt, xdata);
 
-        return 0;
+    return 0;
 }
 
 int
-meta_default_stat (call_frame_t *frame, xlator_t *this, loc_t *loc,
-		   dict_t *xdata)
+meta_default_stat(call_frame_t *frame, xlator_t *this, loc_t *loc,
+                  dict_t *xdata)
 {
-	struct iatt iatt = { };
+    struct iatt iatt = {};
 
-	meta_iatt_fill (&iatt, loc->inode, loc->inode->ia_type);
+    meta_iatt_fill(&iatt, loc->inode, loc->inode->ia_type);
 
-	META_STACK_UNWIND (stat, frame, 0, 0, &iatt, xdata);
+    META_STACK_UNWIND(stat, frame, 0, 0, &iatt, xdata);
 
-	return 0;
+    return 0;
 }
 
 int
-meta_default_lookup (call_frame_t *frame, xlator_t *this, loc_t *loc,
-		     dict_t *xdata)
+meta_default_lookup(call_frame_t *frame, xlator_t *this, loc_t *loc,
+                    dict_t *xdata)
 {
-	struct meta_ops *ops = NULL;
-	struct meta_dirent *dirent = NULL;
-	struct meta_dirent *dp = NULL;
-	int i = 0;
-	int ret = 0;
+    struct meta_ops *ops = NULL;
+    struct meta_dirent *dirent = NULL;
+    struct meta_dirent *dp = NULL;
+    int i = 0;
+    int ret = 0;
 
-	if (!loc->name)
-		return meta_inode_discover (frame, this, loc, xdata);
+    if (!loc->name)
+        return meta_inode_discover(frame, this, loc, xdata);
 
-	ops = meta_ops_get (loc->parent, this);
-	if (!ops)
-		return default_lookup_failure_cbk (frame, EPERM);
+    ops = meta_ops_get(loc->parent, this);
+    if (!ops)
+        return default_lookup_failure_cbk(frame, EPERM);
 
-	for (dirent = ops->fixed_dirents; dirent && dirent->name; dirent++) {
-		if (strcmp (dirent->name, loc->name) == 0)
-			goto hook;
-	}
+    for (dirent = ops->fixed_dirents; dirent && dirent->name; dirent++) {
+        if (strcmp(dirent->name, loc->name) == 0)
+            goto hook;
+    }
 
-	dirent = NULL;
-	if (ops->dir_fill)
-		ret = ops->dir_fill (this, loc->parent, &dp);
+    dirent = NULL;
+    if (ops->dir_fill)
+        ret = ops->dir_fill(this, loc->parent, &dp);
 
-	for (i = 0; i < ret; i++) {
-		if (strcmp (dp[i].name, loc->name) == 0) {
-			dirent = &dp[i];
-			goto hook;
-		}
-	}
+    for (i = 0; i < ret; i++) {
+        if (strcmp(dp[i].name, loc->name) == 0) {
+            dirent = &dp[i];
+            goto hook;
+        }
+    }
 hook:
-	if (dirent && dirent->hook) {
-		struct iatt parent = { };
-		struct iatt iatt = { };
+    if (dirent && dirent->hook) {
+        struct iatt parent = {};
+        struct iatt iatt = {};
 
-		dirent->hook (frame, this, loc, xdata);
+        dirent->hook(frame, this, loc, xdata);
 
-		meta_iatt_fill (&iatt, loc->inode, dirent->type);
+        meta_iatt_fill(&iatt, loc->inode, dirent->type);
 
-		META_STACK_UNWIND (lookup, frame, 0, 0, loc->inode, &iatt,
-				   xdata, &parent);
-	} else {
-		META_STACK_UNWIND (lookup, frame, -1, ENOENT, 0, 0, 0, 0);
-	}
+        META_STACK_UNWIND(lookup, frame, 0, 0, loc->inode, &iatt, xdata,
+                          &parent);
+    } else {
+        META_STACK_UNWIND(lookup, frame, -1, ENOENT, 0, 0, 0, 0);
+    }
 
-	for (i = 0; i < ret; i++)
-		GF_FREE ((void *)dp[i].name);
-	GF_FREE (dp);
+    for (i = 0; i < ret; i++)
+        GF_FREE((void *)dp[i].name);
+    GF_FREE(dp);
 
-	return 0;
+    return 0;
 }
 
 int
-meta_default_fsetattr (call_frame_t *frame, xlator_t *this, fd_t *fd,
-		       struct iatt *stbuf, int32_t valid, dict_t *xdata)
+meta_default_fsetattr(call_frame_t *frame, xlator_t *this, fd_t *fd,
+                      struct iatt *stbuf, int32_t valid, dict_t *xdata)
 {
-        return default_fsetattr_failure_cbk (frame, EPERM);
+    return default_fsetattr_failure_cbk(frame, EPERM);
 }
 
 int
-meta_default_fallocate (call_frame_t *frame, xlator_t *this, fd_t *fd,
-			int32_t keep_size, off_t offset, size_t len,
-			dict_t *xdata)
+meta_default_fallocate(call_frame_t *frame, xlator_t *this, fd_t *fd,
+                       int32_t keep_size, off_t offset, size_t len,
+                       dict_t *xdata)
 {
-        return default_fallocate_failure_cbk (frame, EPERM);
+    return default_fallocate_failure_cbk(frame, EPERM);
 }
 
 int
-meta_default_discard (call_frame_t *frame, xlator_t *this, fd_t *fd,
-		      off_t offset, size_t len, dict_t *xdata)
+meta_default_discard(call_frame_t *frame, xlator_t *this, fd_t *fd,
+                     off_t offset, size_t len, dict_t *xdata)
 {
-        return default_discard_failure_cbk (frame, EPERM);
+    return default_discard_failure_cbk(frame, EPERM);
 }
 
 int
-meta_default_zerofill (call_frame_t *frame, xlator_t *this, fd_t *fd,
-		       off_t offset, off_t len, dict_t *xdata)
+meta_default_zerofill(call_frame_t *frame, xlator_t *this, fd_t *fd,
+                      off_t offset, off_t len, dict_t *xdata)
 {
-        return default_zerofill_failure_cbk (frame, EPERM);
+    return default_zerofill_failure_cbk(frame, EPERM);
 }
 
-#define SET_META_DEFAULT_FOP(f,name) do { if (!f->name) f->name = meta_default_##name ; } while (0)
+#define SET_META_DEFAULT_FOP(f, name)                                          \
+    do {                                                                       \
+        if (!f->name)                                                          \
+            f->name = meta_default_##name;                                     \
+    } while (0)
 
 struct xlator_fops *
-meta_defaults_init (struct xlator_fops *fops)
-{
-        SET_META_DEFAULT_FOP (fops,create);
-        SET_META_DEFAULT_FOP (fops,open);
-        SET_META_DEFAULT_FOP (fops,stat);
-        SET_META_DEFAULT_FOP (fops,readlink);
-        SET_META_DEFAULT_FOP (fops,mknod);
-        SET_META_DEFAULT_FOP (fops,mkdir);
-        SET_META_DEFAULT_FOP (fops,unlink);
-        SET_META_DEFAULT_FOP (fops,rmdir);
-        SET_META_DEFAULT_FOP (fops,symlink);
-        SET_META_DEFAULT_FOP (fops,rename);
-        SET_META_DEFAULT_FOP (fops,link);
-        SET_META_DEFAULT_FOP (fops,truncate);
-        SET_META_DEFAULT_FOP (fops,readv);
-        SET_META_DEFAULT_FOP (fops,writev);
-        SET_META_DEFAULT_FOP (fops,statfs);
-        SET_META_DEFAULT_FOP (fops,flush);
-        SET_META_DEFAULT_FOP (fops,fsync);
-        SET_META_DEFAULT_FOP (fops,setxattr);
-        SET_META_DEFAULT_FOP (fops,getxattr);
-        SET_META_DEFAULT_FOP (fops,fsetxattr);
-        SET_META_DEFAULT_FOP (fops,fgetxattr);
-        SET_META_DEFAULT_FOP (fops,removexattr);
-        SET_META_DEFAULT_FOP (fops,fremovexattr);
-        SET_META_DEFAULT_FOP (fops,opendir);
-        SET_META_DEFAULT_FOP (fops,readdir);
-        SET_META_DEFAULT_FOP (fops,readdirp);
-        SET_META_DEFAULT_FOP (fops,fsyncdir);
-        SET_META_DEFAULT_FOP (fops,access);
-        SET_META_DEFAULT_FOP (fops,ftruncate);
-        SET_META_DEFAULT_FOP (fops,fstat);
-        SET_META_DEFAULT_FOP (fops,lk);
-        SET_META_DEFAULT_FOP (fops,inodelk);
-        SET_META_DEFAULT_FOP (fops,finodelk);
-        SET_META_DEFAULT_FOP (fops,entrylk);
-        SET_META_DEFAULT_FOP (fops,fentrylk);
-        SET_META_DEFAULT_FOP (fops,lookup);
-        SET_META_DEFAULT_FOP (fops,rchecksum);
-        SET_META_DEFAULT_FOP (fops,xattrop);
-        SET_META_DEFAULT_FOP (fops,fxattrop);
-        SET_META_DEFAULT_FOP (fops,setattr);
-        SET_META_DEFAULT_FOP (fops,fsetattr);
-	SET_META_DEFAULT_FOP (fops,fallocate);
-	SET_META_DEFAULT_FOP (fops,discard);
-        SET_META_DEFAULT_FOP (fops,zerofill);
-
-	return fops;
+meta_defaults_init(struct xlator_fops *fops)
+{
+    SET_META_DEFAULT_FOP(fops, create);
+    SET_META_DEFAULT_FOP(fops, open);
+    SET_META_DEFAULT_FOP(fops, stat);
+    SET_META_DEFAULT_FOP(fops, readlink);
+    SET_META_DEFAULT_FOP(fops, mknod);
+    SET_META_DEFAULT_FOP(fops, mkdir);
+    SET_META_DEFAULT_FOP(fops, unlink);
+    SET_META_DEFAULT_FOP(fops, rmdir);
+    SET_META_DEFAULT_FOP(fops, symlink);
+    SET_META_DEFAULT_FOP(fops, rename);
+    SET_META_DEFAULT_FOP(fops, link);
+    SET_META_DEFAULT_FOP(fops, truncate);
+    SET_META_DEFAULT_FOP(fops, readv);
+    SET_META_DEFAULT_FOP(fops, writev);
+    SET_META_DEFAULT_FOP(fops, statfs);
+    SET_META_DEFAULT_FOP(fops, flush);
+    SET_META_DEFAULT_FOP(fops, fsync);
+    SET_META_DEFAULT_FOP(fops, setxattr);
+    SET_META_DEFAULT_FOP(fops, getxattr);
+    SET_META_DEFAULT_FOP(fops, fsetxattr);
+    SET_META_DEFAULT_FOP(fops, fgetxattr);
+    SET_META_DEFAULT_FOP(fops, removexattr);
+    SET_META_DEFAULT_FOP(fops, fremovexattr);
+    SET_META_DEFAULT_FOP(fops, opendir);
+    SET_META_DEFAULT_FOP(fops, readdir);
+    SET_META_DEFAULT_FOP(fops, readdirp);
+    SET_META_DEFAULT_FOP(fops, fsyncdir);
+    SET_META_DEFAULT_FOP(fops, access);
+    SET_META_DEFAULT_FOP(fops, ftruncate);
+    SET_META_DEFAULT_FOP(fops, fstat);
+    SET_META_DEFAULT_FOP(fops, lk);
+    SET_META_DEFAULT_FOP(fops, inodelk);
+    SET_META_DEFAULT_FOP(fops, finodelk);
+    SET_META_DEFAULT_FOP(fops, entrylk);
+    SET_META_DEFAULT_FOP(fops, fentrylk);
+    SET_META_DEFAULT_FOP(fops, lookup);
+    SET_META_DEFAULT_FOP(fops, rchecksum);
+    SET_META_DEFAULT_FOP(fops, xattrop);
+    SET_META_DEFAULT_FOP(fops, fxattrop);
+    SET_META_DEFAULT_FOP(fops, setattr);
+    SET_META_DEFAULT_FOP(fops, fsetattr);
+    SET_META_DEFAULT_FOP(fops, fallocate);
+    SET_META_DEFAULT_FOP(fops, discard);
+    SET_META_DEFAULT_FOP(fops, zerofill);
+
+    return fops;
 }
diff --git a/xlators/meta/src/meta-helpers.c b/xlators/meta/src/meta-helpers.c
index cd47f0fb54a..cb54f547468 100644
--- a/xlators/meta/src/meta-helpers.c
+++ b/xlators/meta/src/meta-helpers.c
@@ -8,363 +8,325 @@
    cases as published by the Free Software Foundation.
 */
 
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
-
-#include "xlator.h"
-#include "defaults.h"
+#include <glusterfs/xlator.h>
+#include <glusterfs/defaults.h>
 
 #include "meta-mem-types.h"
 #include "meta.h"
 
-
 meta_fd_t *
-meta_fd_get (fd_t *fd, xlator_t *this)
+meta_fd_get(fd_t *fd, xlator_t *this)
 {
-	uint64_t value = 0;
-	meta_fd_t *meta_fd = NULL;
-
-	LOCK (&fd->lock);
-	{
-		__fd_ctx_get (fd, this, &value);
-		if (!value) {
-			meta_fd = GF_CALLOC (1, sizeof (*meta_fd),
-					     gf_meta_mt_fd_t);
-			if (!meta_fd)
-				goto unlock;
-
-			value = (long) meta_fd;
-			__fd_ctx_set (fd, this, value);
-		}
-
-		meta_fd = (void *) value;
-	}
+    uint64_t value = 0;
+    meta_fd_t *meta_fd = NULL;
+
+    LOCK(&fd->lock);
+    {
+        if (__fd_ctx_get(fd, this, &value) < 0) {
+            if (!value) {
+                meta_fd = GF_CALLOC(1, sizeof(*meta_fd), gf_meta_mt_fd_t);
+                if (!meta_fd)
+                    goto unlock;
+                value = (long)meta_fd;
+                __fd_ctx_set(fd, this, value);
+            }
+        } else {
+            meta_fd = (void *)(uintptr_t)value;
+        }
+    }
 unlock:
-	UNLOCK (&fd->lock);
+    UNLOCK(&fd->lock);
 
-	return meta_fd;
+    return meta_fd;
 }
 
-
 int
-meta_fd_release (fd_t *fd, xlator_t *this)
+meta_fd_release(fd_t *fd, xlator_t *this)
 {
-	uint64_t value = 0;
-	meta_fd_t *meta_fd = NULL;
-	int i = 0;
-
-	fd_ctx_get (fd, this, &value);
-	meta_fd = (void *) value;
-
-	if (meta_fd->dirents) {
-		for (i = 0; i < meta_fd->size; i++)
-			GF_FREE ((void *)meta_fd->dirents[i].name);
-		GF_FREE (meta_fd->dirents);
-	}
-
-	if (meta_fd) {
-		GF_FREE (meta_fd->data);
-		GF_FREE (meta_fd);
-	}
-	return 0;
+    uint64_t value = 0;
+    meta_fd_t *meta_fd = NULL;
+    int i = 0;
+
+    fd_ctx_get(fd, this, &value);
+    meta_fd = (void *)(uintptr_t)value;
+
+    if (meta_fd && meta_fd->dirents) {
+        for (i = 0; i < meta_fd->size; i++)
+            GF_FREE((void *)meta_fd->dirents[i].name);
+        GF_FREE(meta_fd->dirents);
+    }
+
+    if (meta_fd) {
+        GF_FREE(meta_fd->data);
+        GF_FREE(meta_fd);
+    }
+    return 0;
 }
 
-
 struct meta_ops *
-meta_ops_get (inode_t *inode, xlator_t *this)
+meta_ops_get(inode_t *inode, xlator_t *this)
 {
-	struct meta_ops *ops = NULL;
-	uint64_t value = 0;
+    struct meta_ops *ops = NULL;
+    uint64_t value = 0;
 
-	inode_ctx_get2 (inode, this, NULL, &value);
+    inode_ctx_get2(inode, this, NULL, &value);
 
-	ops = (void *) value;
+    ops = (void *)(uintptr_t)value;
 
-	return ops;
+    return ops;
 }
 
-
 struct xlator_fops *
-meta_fops_get (inode_t *inode, xlator_t *this)
+meta_fops_get(inode_t *inode, xlator_t *this)
 {
-	struct meta_ops *ops = NULL;
+    struct meta_ops *ops = NULL;
 
-	ops = meta_ops_get (inode, this);
-	if (!ops)
-		return default_fops;
+    ops = meta_ops_get(inode, this);
+    if (!ops)
+        return default_fops;
 
-	return &ops->fops;
+    return &ops->fops;
 }
 
-
 int
-meta_ops_set (inode_t *inode, xlator_t *this, struct meta_ops *ops)
+meta_ops_set(inode_t *inode, xlator_t *this, struct meta_ops *ops)
 {
-	uint64_t value = 0;
-	int ret = 0;
+    uint64_t value = 0;
+    int ret = 0;
 
-	meta_defaults_init (&ops->fops);
+    meta_defaults_init(&ops->fops);
 
-	value = (long) ops;
+    value = (long)ops;
 
-	ret = inode_ctx_set2 (inode, this, NULL, &value);
+    ret = inode_ctx_set2(inode, this, NULL, &value);
 
-	return ret;
+    return ret;
 }
 
 void *
-meta_ctx_get (inode_t *inode, xlator_t *this)
+meta_ctx_get(inode_t *inode, xlator_t *this)
 {
-	void *ctx = NULL;
-	uint64_t value = 0;
+    void *ctx = NULL;
+    uint64_t value = 0;
 
-	inode_ctx_get2 (inode, this, &value, 0);
+    inode_ctx_get2(inode, this, &value, 0);
 
-	ctx = (void *) value;
+    ctx = (void *)(uintptr_t)value;
 
-	return ctx;
+    return ctx;
 }
 
-
 int
-meta_ctx_set (inode_t *inode, xlator_t *this, void *ctx)
+meta_ctx_set(inode_t *inode, xlator_t *this, void *ctx)
 {
-	uint64_t value = 0;
-	int ret = 0;
+    uint64_t value = 0;
+    int ret = 0;
 
-	value = (long) ctx;
+    value = (long)ctx;
 
-	ret = inode_ctx_set2 (inode, this, &value, 0);
+    ret = inode_ctx_set2(inode, this, &value, 0);
 
-	return ret;
+    return ret;
 }
 
-
 void
-meta_local_cleanup (meta_local_t *local, xlator_t *this)
+meta_local_cleanup(meta_local_t *local, xlator_t *this)
 {
-	if (!local)
-		return;
+    if (!local)
+        return;
 
-	if (local->xdata)
-		dict_unref (local->xdata);
+    if (local->xdata)
+        dict_unref(local->xdata);
 
-	GF_FREE (local);
-	return;
+    GF_FREE(local);
+    return;
 }
 
-
 meta_local_t *
-meta_local (call_frame_t *frame)
+meta_local(call_frame_t *frame)
 {
-	meta_local_t *local = NULL;
+    meta_local_t *local = NULL;
 
-	local = frame->local;
-	if (!local)
-		local = frame->local = GF_CALLOC (1, sizeof(*local),
-						  gf_meta_mt_local_t);
-	return local;
+    local = frame->local;
+    if (!local)
+        local = frame->local = GF_CALLOC(1, sizeof(*local), gf_meta_mt_local_t);
+    return local;
 }
 
-
 dict_t *
-meta_direct_io_mode (dict_t *xdata, call_frame_t *frame)
+meta_direct_io_mode(dict_t *xdata, call_frame_t *frame)
 {
-	meta_local_t *local = NULL;
+    meta_local_t *local = NULL;
 
-	if (!xdata) {
-		local = meta_local (frame);
-		if (!local)
-			return NULL;
-		xdata = local->xdata = dict_new();
-		if (!xdata)
-			return NULL;
-	}
+    if (!xdata) {
+        local = meta_local(frame);
+        if (!local)
+            return NULL;
+        xdata = local->xdata = dict_new();
+        if (!xdata)
+            return NULL;
+    }
 
-	if (dict_set_int8 (xdata, "direct-io-mode", 1) != 0)
-		return NULL;
+    if (dict_set_int8(xdata, "direct-io-mode", 1) != 0)
+        return NULL;
 
-	return xdata;
+    return xdata;
 }
 
-
-static uint64_t
-gfid_to_ino (uuid_t gfid)
-{
-        uint64_t  ino = 0;
-        int       i = 0, j = 0;
-
-        for (i = 15; i > (15 - 8); i--) {
-                ino += (uint64_t)(gfid[i]) << j;
-                j += 8;
-        }
-
-        return ino;
-}
-
-
 static void
-meta_uuid_copy (uuid_t dst, uuid_t src)
+meta_uuid_copy(uuid_t dst, uuid_t src)
 {
-	gf_uuid_copy (dst, src);
-	if (gf_uuid_is_null (dst))
-		gf_uuid_generate (dst);
+    gf_uuid_copy(dst, src);
+    if (gf_uuid_is_null(dst))
+        gf_uuid_generate(dst);
 }
 
-
 static void
-default_meta_iatt_fill (struct iatt *iatt, inode_t *inode, ia_type_t type)
+default_meta_iatt_fill(struct iatt *iatt, inode_t *inode, ia_type_t type,
+                       gf_boolean_t is_tunable)
 {
-	struct timeval tv = { };
-
-	iatt->ia_type = type;
-	switch (type)
-	{
-	case IA_IFDIR:
-		iatt->ia_prot = ia_prot_from_st_mode (0755);
-		iatt->ia_nlink = 2;
-		break;
-	case IA_IFLNK:
-		iatt->ia_prot = ia_prot_from_st_mode (0777);
-		iatt->ia_nlink = 1;
-		break;
-	default:
-		iatt->ia_prot = ia_prot_from_st_mode (0644);
-		iatt->ia_nlink = 1;
-		break;
-	}
-	iatt->ia_uid = 0;
-	iatt->ia_gid = 0;
-	iatt->ia_size = 0;
-
-	meta_uuid_copy (iatt->ia_gfid, inode->gfid);
-	iatt->ia_ino = gfid_to_ino (iatt->ia_gfid);
-
-	gettimeofday (&tv, 0);
-	iatt->ia_mtime = iatt->ia_ctime = iatt->ia_atime = tv.tv_sec;
-	iatt->ia_mtime_nsec = iatt->ia_ctime_nsec = iatt->ia_atime_nsec =
-		(tv.tv_usec * 1000);
-	return;
+    struct timeval tv = {};
+
+    iatt->ia_type = type;
+    switch (type) {
+        case IA_IFDIR:
+            iatt->ia_prot = ia_prot_from_st_mode(0555);
+            iatt->ia_nlink = 2;
+            break;
+        case IA_IFLNK:
+            iatt->ia_prot = ia_prot_from_st_mode(0777);
+            iatt->ia_nlink = 1;
+            break;
+        default:
+            iatt->ia_prot = ia_prot_from_st_mode(is_tunable ? 0644 : 0444);
+            iatt->ia_nlink = 1;
+            break;
+    }
+    iatt->ia_uid = 0;
+    iatt->ia_gid = 0;
+    iatt->ia_size = 0;
+
+    meta_uuid_copy(iatt->ia_gfid, inode->gfid);
+    iatt->ia_ino = gfid_to_ino(iatt->ia_gfid);
+
+    gettimeofday(&tv, 0);
+    iatt->ia_mtime = iatt->ia_ctime = iatt->ia_atime = tv.tv_sec;
+    iatt->ia_mtime_nsec = iatt->ia_ctime_nsec = iatt->ia_atime_nsec =
+        (tv.tv_usec * 1000);
+    return;
 }
 
-
 void
-meta_iatt_fill (struct iatt *iatt, inode_t *inode, ia_type_t type)
+meta_iatt_fill(struct iatt *iatt, inode_t *inode, ia_type_t type)
 {
-	struct meta_ops *ops = NULL;
+    struct meta_ops *ops = NULL;
 
-	ops = meta_ops_get (inode, THIS);
-	if (!ops)
-		return;
+    ops = meta_ops_get(inode, THIS);
+    if (!ops)
+        return;
 
-	if (!ops->iatt_fill)
-		default_meta_iatt_fill (iatt, inode, type);
-	else
-		ops->iatt_fill (THIS, inode, iatt);
-	return;
+    if (!ops->iatt_fill)
+        default_meta_iatt_fill(iatt, inode, type, !!ops->file_write);
+    else
+        ops->iatt_fill(THIS, inode, iatt);
+    return;
 }
 
-
 int
-meta_inode_discover (call_frame_t *frame, xlator_t *this, loc_t *loc,
-		     dict_t *xdata)
+meta_inode_discover(call_frame_t *frame, xlator_t *this, loc_t *loc,
+                    dict_t *xdata)
 {
-	struct iatt iatt = { };
-	struct iatt postparent = { };
+    struct iatt iatt = {};
+    struct iatt postparent = {};
 
-	meta_iatt_fill (&iatt, loc->inode, loc->inode->ia_type);
+    meta_iatt_fill(&iatt, loc->inode, loc->inode->ia_type);
 
-	META_STACK_UNWIND (lookup, frame, 0, 0, loc->inode, &iatt, xdata,
-			   &postparent);
-	return 0;
+    META_STACK_UNWIND(lookup, frame, 0, 0, loc->inode, &iatt, xdata,
+                      &postparent);
+    return 0;
 }
 
-
 int
-meta_file_fill (xlator_t *this, fd_t *fd)
+meta_file_fill(xlator_t *this, fd_t *fd)
 {
-	meta_fd_t *meta_fd = NULL;
-	strfd_t *strfd = NULL;
-	struct meta_ops *ops = NULL;
-	int ret = 0;
+    meta_fd_t *meta_fd = NULL;
+    strfd_t *strfd = NULL;
+    struct meta_ops *ops = NULL;
+    int ret = 0;
 
-	meta_fd = meta_fd_get (fd, this);
-	if (!meta_fd)
-		return -1;
+    meta_fd = meta_fd_get(fd, this);
+    if (!meta_fd)
+        return -1;
 
-	if (meta_fd->data)
-		return meta_fd->size;
+    if (meta_fd->data)
+        return meta_fd->size;
 
-	strfd = strfd_open ();
-	if (!strfd)
-		return -1;
+    strfd = strfd_open();
+    if (!strfd)
+        return -1;
 
-	ops = meta_ops_get (fd->inode, this);
-	if (!ops) {
-		strfd_close (strfd);
-		return -1;
-	}
+    ops = meta_ops_get(fd->inode, this);
+    if (!ops) {
+        strfd_close(strfd);
+        return -1;
+    }
 
-	if (ops->file_fill)
-		ret = ops->file_fill (this, fd->inode, strfd);
+    if (ops->file_fill)
+        ret = ops->file_fill(this, fd->inode, strfd);
 
-	if (ret >= 0) {
-		meta_fd->data = strfd->data;
-		meta_fd->size = strfd->size;
+    if (ret >= 0) {
+        meta_fd->data = strfd->data;
+        meta_fd->size = strfd->size;
 
-		strfd->data = NULL;
-	}
+        strfd->data = NULL;
+    }
 
-	strfd_close (strfd);
+    strfd_close(strfd);
 
-	return meta_fd->size;
+    return meta_fd->size;
 }
 
-
 int
-meta_dir_fill (xlator_t *this, fd_t *fd)
+meta_dir_fill(xlator_t *this, fd_t *fd)
 {
-	meta_fd_t *meta_fd = NULL;
-	struct meta_ops *ops = NULL;
-	struct meta_dirent *dp = NULL;
-	int ret = 0;
+    meta_fd_t *meta_fd = NULL;
+    struct meta_ops *ops = NULL;
+    struct meta_dirent *dp = NULL;
+    int ret = 0;
 
-	meta_fd = meta_fd_get (fd, this);
-	if (!meta_fd)
-		return -1;
+    meta_fd = meta_fd_get(fd, this);
+    if (!meta_fd)
+        return -1;
 
-	if (meta_fd->dirents)
-		return meta_fd->size;
+    if (meta_fd->dirents)
+        return meta_fd->size;
 
-	ops = meta_ops_get (fd->inode, this);
-	if (!ops)
-		return -1;
+    ops = meta_ops_get(fd->inode, this);
+    if (!ops)
+        return -1;
 
-	if (ops->dir_fill)
-		ret = ops->dir_fill (this, fd->inode, &dp);
+    if (ops->dir_fill)
+        ret = ops->dir_fill(this, fd->inode, &dp);
 
-	if (dp) {
-		meta_fd->dirents = dp;
-		meta_fd->size = ret;
-	}
+    if (dp) {
+        meta_fd->dirents = dp;
+        meta_fd->size = ret;
+    }
 
-	return meta_fd->size;
+    return meta_fd->size;
 }
 
-
 int
-fixed_dirents_len (struct meta_dirent *dirents)
+fixed_dirents_len(struct meta_dirent *dirents)
 {
-	int i = 0;
-	struct meta_dirent *dirent = NULL;
+    int i = 0;
+    struct meta_dirent *dirent = NULL;
 
-	if (!dirents)
-		return 0;
+    if (!dirents)
+        return 0;
 
-	for (dirent = dirents; dirent->name; dirent++)
-		i++;
+    for (dirent = dirents; dirent->name; dirent++)
+        i++;
 
-	return i;
+    return i;
 }
diff --git a/xlators/meta/src/meta-hooks.h b/xlators/meta/src/meta-hooks.h
index bcf3643d223..7208641398a 100644
--- a/xlators/meta/src/meta-hooks.h
+++ b/xlators/meta/src/meta-hooks.h
@@ -10,9 +10,11 @@
 
 #ifndef __META_HOOKS_H
 #define __META_HOOKS_H
-#include "xlator.h"
+#include <glusterfs/xlator.h>
 
-#define DECLARE_HOOK(name) int meta_##name##_hook (call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xdata)
+#define DECLARE_HOOK(name)                                                     \
+    int meta_##name##_hook(call_frame_t *frame, xlator_t *this, loc_t *loc,    \
+                           dict_t *xdata)
 
 DECLARE_HOOK(root_dir);
 DECLARE_HOOK(graphs_dir);
diff --git a/xlators/meta/src/meta-mem-types.h b/xlators/meta/src/meta-mem-types.h
index e8a31856e71..033c306682f 100644
--- a/xlators/meta/src/meta-mem-types.h
+++ b/xlators/meta/src/meta-mem-types.h
@@ -11,16 +11,15 @@
 #ifndef __META_MEM_TYPES_H__
 #define __META_MEM_TYPES_H__
 
-#include "mem-types.h"
+#include <glusterfs/mem-types.h>
 
 enum gf_meta_mem_types_ {
-        gf_meta_mt_priv_t = gf_common_mt_end + 1,
-	gf_meta_mt_fd_t,
-	gf_meta_mt_fd_data_t,
-	gf_meta_mt_strfd_t,
-	gf_meta_mt_dirents_t,
-	gf_meta_mt_local_t,
-        gf_meta_mt_end
+    gf_meta_mt_priv_t = gf_common_mt_end + 1,
+    gf_meta_mt_fd_t,
+    gf_meta_mt_fd_data_t,
+    gf_meta_mt_strfd_t,
+    gf_meta_mt_dirents_t,
+    gf_meta_mt_local_t,
+    gf_meta_mt_end
 };
 #endif
-
diff --git a/xlators/meta/src/meta.c b/xlators/meta/src/meta.c
index 20c643f2f75..e1b9a2b6581 100644
--- a/xlators/meta/src/meta.c
+++ b/xlators/meta/src/meta.c
@@ -8,261 +8,269 @@
    cases as published by the Free Software Foundation.
 */
 
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
-
-#include "xlator.h"
-#include "defaults.h"
+#include <glusterfs/xlator.h>
+#include <glusterfs/defaults.h>
 
 #include "meta-mem-types.h"
 #include "meta.h"
 
 #include "meta-hooks.h"
 
-
 int
-meta_lookup (call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xdata)
+meta_lookup(call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xdata)
 {
-	inode_t *inode = NULL;
+    inode_t *inode = NULL;
 
-	if (META_HOOK (loc) || IS_META_ROOT_GFID (loc->gfid)) {
-		struct iatt iatt = { };
-		struct iatt parent = { };
+    if (META_HOOK(loc) || IS_META_ROOT_GFID(loc->gfid)) {
+        struct iatt iatt = {};
+        struct iatt parent = {};
 
-		meta_root_dir_hook (frame, this, loc, xdata);
+        meta_root_dir_hook(frame, this, loc, xdata);
 
-		meta_iatt_fill (&iatt, loc->inode, IA_IFDIR);
-		gf_uuid_parse (META_ROOT_GFID, iatt.ia_gfid);
+        meta_iatt_fill(&iatt, loc->inode, IA_IFDIR);
+        gf_uuid_parse(META_ROOT_GFID, iatt.ia_gfid);
 
-		META_STACK_UNWIND (lookup, frame, 0, 0, loc->inode, &iatt,
-				   xdata, &parent);
-		return 0;
-	}
+        META_STACK_UNWIND(lookup, frame, 0, 0, loc->inode, &iatt, xdata,
+                          &parent);
+        return 0;
+    }
 
-	if (loc->parent)
-		inode = loc->parent;
-	else
-		inode = loc->inode;
+    if (loc->parent)
+        inode = loc->parent;
+    else
+        inode = loc->inode;
 
-	META_FOP (inode, lookup, frame, this, loc, xdata);
+    META_FOP(inode, lookup, frame, this, loc, xdata);
 
-	return 0;
+    return 0;
 }
 
-
 int
-meta_opendir (call_frame_t *frame, xlator_t *this, loc_t *loc, fd_t *fd,
-	      dict_t *xdata)
+meta_opendir(call_frame_t *frame, xlator_t *this, loc_t *loc, fd_t *fd,
+             dict_t *xdata)
 {
-	META_FOP (fd->inode, opendir, frame, this, loc, fd, xdata);
+    META_FOP(fd->inode, opendir, frame, this, loc, fd, xdata);
 
-	return 0;
+    return 0;
 }
 
-
 int
-meta_open (call_frame_t *frame, xlator_t *this, loc_t *loc, int flags, fd_t *fd,
-	   dict_t *xdata)
+meta_open(call_frame_t *frame, xlator_t *this, loc_t *loc, int flags, fd_t *fd,
+          dict_t *xdata)
 {
-	META_FOP (fd->inode, open, frame, this, loc, flags, fd, xdata);
+    META_FOP(fd->inode, open, frame, this, loc, flags, fd, xdata);
 
-	return 0;
+    return 0;
 }
 
-
 int
-meta_readv (call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
-	    off_t offset, uint32_t flags, dict_t *xdata)
+meta_readv(call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
+           off_t offset, uint32_t flags, dict_t *xdata)
 {
-	META_FOP (fd->inode, readv, frame, this, fd, size, offset, flags, xdata);
+    META_FOP(fd->inode, readv, frame, this, fd, size, offset, flags, xdata);
 
-	return 0;
+    return 0;
 }
 
-
 int
-meta_flush (call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *xdata)
+meta_flush(call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *xdata)
 {
-	META_FOP (fd->inode, flush, frame, this, fd, xdata);
+    META_FOP(fd->inode, flush, frame, this, fd, xdata);
 
-	return 0;
+    return 0;
 }
 
-
 int
-meta_stat (call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xdata)
+meta_stat(call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xdata)
 {
-	META_FOP (loc->inode, stat, frame, this, loc, xdata);
+    META_FOP(loc->inode, stat, frame, this, loc, xdata);
 
-	return 0;
+    return 0;
 }
 
-
 int
-meta_fstat (call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *xdata)
+meta_fstat(call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *xdata)
 {
-	META_FOP (fd->inode, fstat, frame, this, fd, xdata);
+    META_FOP(fd->inode, fstat, frame, this, fd, xdata);
 
-	return 0;
+    return 0;
 }
 
-
 int
-meta_readdir (call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
-	      off_t offset, dict_t *xdata)
+meta_readdir(call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
+             off_t offset, dict_t *xdata)
 {
-	META_FOP (fd->inode, readdir, frame, this, fd, size, offset, xdata);
+    META_FOP(fd->inode, readdir, frame, this, fd, size, offset, xdata);
 
-	return 0;
+    return 0;
 }
 
-
 int
-meta_readdirp (call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
-	       off_t offset, dict_t *xdata)
+meta_readdirp(call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
+              off_t offset, dict_t *xdata)
 {
-	META_FOP (fd->inode, readdirp, frame, this, fd, size, offset, xdata);
+    META_FOP(fd->inode, readdirp, frame, this, fd, size, offset, xdata);
 
-	return 0;
+    return 0;
 }
 
-
 int
-meta_readlink (call_frame_t *frame, xlator_t *this, loc_t *loc, size_t size,
-	       dict_t *xdata)
+meta_readlink(call_frame_t *frame, xlator_t *this, loc_t *loc, size_t size,
+              dict_t *xdata)
 {
-	META_FOP (loc->inode, readlink, frame, this, loc, size, xdata);
+    META_FOP(loc->inode, readlink, frame, this, loc, size, xdata);
 
-	return 0;
+    return 0;
 }
 
-
 int
-meta_writev (call_frame_t *frame, xlator_t *this, fd_t *fd, struct iovec *iov,
-	     int count, off_t offset, uint32_t flags, struct iobref *iobref,
-	     dict_t *xdata)
+meta_writev(call_frame_t *frame, xlator_t *this, fd_t *fd, struct iovec *iov,
+            int count, off_t offset, uint32_t flags, struct iobref *iobref,
+            dict_t *xdata)
 {
-	META_FOP (fd->inode, writev, frame, this, fd, iov, count, offset, flags,
-		  iobref, xdata);
-	return 0;
+    META_FOP(fd->inode, writev, frame, this, fd, iov, count, offset, flags,
+             iobref, xdata);
+    return 0;
 }
 
-
 int
-meta_truncate (call_frame_t *frame, xlator_t *this, loc_t *loc, off_t offset,
-	       dict_t *xdata)
+meta_truncate(call_frame_t *frame, xlator_t *this, loc_t *loc, off_t offset,
+              dict_t *xdata)
 {
-	META_FOP (loc->inode, truncate, frame, this, loc, offset, xdata);
+    META_FOP(loc->inode, truncate, frame, this, loc, offset, xdata);
 
-	return 0;
+    return 0;
 }
 
-
 int
-meta_ftruncate (call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
-		dict_t *xdata)
+meta_ftruncate(call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
+               dict_t *xdata)
 {
-	META_FOP (fd->inode, ftruncate, frame, this, fd, offset, xdata);
+    META_FOP(fd->inode, ftruncate, frame, this, fd, offset, xdata);
 
-	return 0;
+    return 0;
 }
 
-int
-meta_forget (xlator_t *this, inode_t *inode)
+int32_t
+meta_fsync(call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t flags,
+           dict_t *xdata)
 {
-	return 0;
+    META_FOP(fd->inode, fsync, frame, this, fd, flags, xdata);
+
+    return 0;
 }
 
+int32_t
+meta_fsyncdir(call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t flags,
+              dict_t *xdata)
+{
+    META_FOP(fd->inode, fsyncdir, frame, this, fd, flags, xdata);
+
+    return 0;
+}
 
 int
-meta_release (xlator_t *this, fd_t *fd)
+meta_forget(xlator_t *this, inode_t *inode)
 {
-	return meta_fd_release (fd, this);
+    return 0;
 }
 
-
 int
-meta_releasedir (xlator_t *this, fd_t *fd)
+meta_release(xlator_t *this, fd_t *fd)
 {
-	return meta_fd_release (fd, this);
+    return meta_fd_release(fd, this);
 }
 
-
 int
-mem_acct_init (xlator_t *this)
+meta_releasedir(xlator_t *this, fd_t *fd)
 {
-        int ret = -1;
+    return meta_fd_release(fd, this);
+}
 
-        if (!this)
-                return ret;
+int
+mem_acct_init(xlator_t *this)
+{
+    int ret = -1;
 
-        ret = xlator_mem_acct_init (this, gf_meta_mt_end + 1);
+    if (!this)
+        return ret;
 
-        if (ret != 0) {
-                gf_log (this->name, GF_LOG_ERROR,
-			"Memory accounting init failed");
-                return ret;
-        }
+    ret = xlator_mem_acct_init(this, gf_meta_mt_end + 1);
 
+    if (ret != 0) {
+        gf_log(this->name, GF_LOG_ERROR, "Memory accounting init failed");
         return ret;
-}
+    }
 
+    return ret;
+}
 
 int
-init (xlator_t *this)
+init(xlator_t *this)
 {
-	meta_priv_t *priv = NULL;
+    meta_priv_t *priv = NULL;
+    int ret = -1;
 
-	priv = GF_CALLOC (sizeof(*priv), 1, gf_meta_mt_priv_t);
-	if (!priv)
-		return -1;
+    priv = GF_CALLOC(sizeof(*priv), 1, gf_meta_mt_priv_t);
+    if (!priv)
+        return ret;
 
-	GF_OPTION_INIT ("meta-dir-name", priv->meta_dir_name, str, out);
+    GF_OPTION_INIT("meta-dir-name", priv->meta_dir_name, str, out);
 
-	this->private = priv;
+    this->private = priv;
+    ret = 0;
 out:
-	return 0;
-}
+    if (ret)
+        GF_FREE(priv);
 
+    return ret;
+}
 
-int
-fini (xlator_t *this)
+void
+fini(xlator_t *this)
 {
-	return 0;
+    GF_FREE(this->private);
+    return;
 }
 
-
-struct xlator_fops fops = {
-	.lookup = meta_lookup,
-	.opendir = meta_opendir,
-	.open = meta_open,
-	.readv = meta_readv,
-	.flush = meta_flush,
-	.stat = meta_stat,
-	.fstat = meta_fstat,
-	.readdir = meta_readdir,
-	.readdirp = meta_readdirp,
-	.readlink = meta_readlink,
-	.writev = meta_writev,
-	.truncate = meta_truncate,
-	.ftruncate = meta_ftruncate
-};
-
+struct xlator_fops fops = {.lookup = meta_lookup,
+                           .opendir = meta_opendir,
+                           .open = meta_open,
+                           .readv = meta_readv,
+                           .flush = meta_flush,
+                           .stat = meta_stat,
+                           .fstat = meta_fstat,
+                           .readdir = meta_readdir,
+                           .readdirp = meta_readdirp,
+                           .readlink = meta_readlink,
+                           .writev = meta_writev,
+                           .truncate = meta_truncate,
+                           .ftruncate = meta_ftruncate,
+                           .fsync = meta_fsync,
+                           .fsyncdir = meta_fsyncdir};
 
 struct xlator_cbks cbks = {
-	.forget = meta_forget,
-	.release = meta_release,
-	.releasedir = meta_releasedir,
+    .forget = meta_forget,
+    .release = meta_release,
+    .releasedir = meta_releasedir,
 };
 
-
 struct volume_options options[] = {
-	{ .key = {"meta-dir-name"},
-	  .type = GF_OPTION_TYPE_STR,
-	  .default_value = DEFAULT_META_DIR_NAME,
-	  .description = "Name of default meta directory."
-	},
-	{ .key = {NULL} },
+    {.key = {"meta-dir-name"},
+     .type = GF_OPTION_TYPE_STR,
+     .default_value = DEFAULT_META_DIR_NAME,
+     .description = "Name of default meta directory."},
+    {.key = {NULL}},
+};
+
+xlator_api_t xlator_api = {
+    .init = init,
+    .fini = fini,
+    .mem_acct_init = mem_acct_init,
+    .op_version = {1}, /* Present from the initial version */
+    .fops = &fops,
+    .cbks = &cbks,
+    .options = options,
+    .identifier = "meta",
+    .category = GF_TECH_PREVIEW,
 };
diff --git a/xlators/meta/src/meta.h b/xlators/meta/src/meta.h
index 652ebd7321c..7f0cf28808a 100644
--- a/xlators/meta/src/meta.h
+++ b/xlators/meta/src/meta.h
@@ -10,117 +10,131 @@
 #ifndef __META_H__
 #define __META_H__
 
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
-
-#include "strfd.h"
+#include <glusterfs/strfd.h>
 
 #define DEFAULT_META_DIR_NAME ".meta"
 
 #define META_ROOT_GFID "ba926388-bb9c-4eec-ad60-79dba4cc083a"
 
-#define IS_META_ROOT_GFID(g) (strcmp (uuid_utoa(g), META_ROOT_GFID) == 0)
+#define IS_META_ROOT_GFID(g) (strcmp(uuid_utoa(g), META_ROOT_GFID) == 0)
 
-typedef int (*meta_hook_t) (call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xdata);
+typedef int (*meta_hook_t)(call_frame_t *frame, xlator_t *this, loc_t *loc,
+                           dict_t *xdata);
 
 typedef struct {
-	dict_t *xdata;
+    dict_t *xdata;
 } meta_local_t;
 
 typedef struct {
-	char *meta_dir_name;
+    char *meta_dir_name;
 } meta_priv_t;
 
 struct meta_dirent {
-	const char *name;
-	ia_type_t type;
-	meta_hook_t hook;
+    const char *name;
+    ia_type_t type;
+    meta_hook_t hook;
 };
 
-#define DOT_DOTDOT { .name = ".", .type = IA_IFDIR }, { .name = "..", .type = IA_IFDIR }
+#define DOT_DOTDOT                                                             \
+    {.name = ".", .type = IA_IFDIR}, { .name = "..", .type = IA_IFDIR }
 
 struct meta_ops {
-	struct meta_dirent *fixed_dirents;
-	int (*dir_fill) (xlator_t *this, inode_t *dir, struct meta_dirent **entries);
-	int (*file_fill) (xlator_t *this, inode_t *file, strfd_t *strfd);
-	int (*iatt_fill) (xlator_t *this, inode_t *inode, struct iatt *iatt);
-	int (*link_fill) (xlator_t *this, inode_t *inode, strfd_t *strfd);
-	int (*file_write) (xlator_t *this, fd_t *fd, struct iovec *iov, int count);
-	struct xlator_fops fops;
-	struct xlator_cbks cbks;
+    struct meta_dirent *fixed_dirents;
+    int (*dir_fill)(xlator_t *this, inode_t *dir, struct meta_dirent **entries);
+    int (*file_fill)(xlator_t *this, inode_t *file, strfd_t *strfd);
+    int (*iatt_fill)(xlator_t *this, inode_t *inode, struct iatt *iatt);
+    int (*link_fill)(xlator_t *this, inode_t *inode, strfd_t *strfd);
+    int (*file_write)(xlator_t *this, fd_t *fd, struct iovec *iov, int count);
+    struct xlator_fops fops;
+    struct xlator_cbks cbks;
 };
 
 typedef struct {
-	char *data;
-	struct meta_dirent *dirents;
-	size_t size;
+    char *data;
+    struct meta_dirent *dirents;
+    size_t size;
 } meta_fd_t;
 
+#define COUNT(arr) (sizeof(arr) / sizeof(arr[0]))
 
-#define COUNT(arr) (sizeof(arr)/sizeof(arr[0]))
-
-#define META_HOOK(loc) (__is_root_gfid (loc->pargfid) && !strcmp (loc->name, META_PRIV(THIS)->meta_dir_name))
+#define META_HOOK(loc)                                                         \
+    (__is_root_gfid(loc->pargfid) &&                                           \
+     !strcmp(loc->name, META_PRIV(THIS)->meta_dir_name))
 
 #define META_PRIV(t) ((meta_priv_t *)(t->private))
 
-#define META_STACK_UNWIND(fop, frame, params ...)		\
-        do {                                                    \
-                meta_local_t *__local = NULL;			\
-                xlator_t    *__this = NULL;                     \
-                if (frame) {                                    \
-                        __local = frame->local;                 \
-                        __this = frame->this;                   \
-                        frame->local = NULL;                    \
-                }                                               \
-                STACK_UNWIND_STRICT (fop, frame, params);       \
-                if (__local) {                                  \
-                        meta_local_cleanup (__local, __this);	\
-                }                                               \
-        } while (0)
-
-
-#define META_FOP(i, fop, fr, t, params ...) {	\
-	struct xlator_fops *_fops = NULL;	\
-						\
-	_fops = meta_fops_get (i, t);		\
-						\
-	_fops->fop (fr, t, params);		\
-	} while (0)
-
-
-void meta_iatt_fill (struct iatt *iatt, inode_t *inode, ia_type_t type);
-
-int meta_inode_discover (call_frame_t *frame, xlator_t *this, loc_t *loc,
-			 dict_t *xdata);
-
-int meta_ops_set (inode_t *inode, xlator_t *this, struct meta_ops *ops);
-
-struct xlator_fops *meta_fops_get (inode_t *inode, xlator_t *this);
-struct xlator_cbks *meta_cbks_get (inode_t *inode, xlator_t *this);
-struct meta_ops *meta_ops_get (inode_t *inode, xlator_t *this);
-
-int meta_ctx_set (inode_t *inode, xlator_t *this, void *ctx);
-
-void *meta_ctx_get (inode_t *inode, xlator_t *this);
-
-
-void meta_local_cleanup (meta_local_t *local, xlator_t *this);
-
-struct xlator_fops *meta_defaults_init (struct xlator_fops *fops);
-
-meta_fd_t *meta_fd_get (fd_t *fd, xlator_t *this);
-
-int meta_fd_release (fd_t *fd, xlator_t *this);
-
-dict_t *meta_direct_io_mode (dict_t *xdata, call_frame_t *frame);
-
-meta_local_t *meta_local (call_frame_t *frame);
-
-int meta_file_fill (xlator_t *this, fd_t *fd);
-
-int meta_dir_fill (xlator_t *this, fd_t *fd);
-
-int fixed_dirents_len (struct meta_dirent *dirents);
+#define META_STACK_UNWIND(fop, frame, params...)                               \
+    do {                                                                       \
+        meta_local_t *__local = NULL;                                          \
+        xlator_t *__this = NULL;                                               \
+        if (frame) {                                                           \
+            __local = frame->local;                                            \
+            __this = frame->this;                                              \
+            frame->local = NULL;                                               \
+        }                                                                      \
+        STACK_UNWIND_STRICT(fop, frame, params);                               \
+        if (__local) {                                                         \
+            meta_local_cleanup(__local, __this);                               \
+        }                                                                      \
+    } while (0)
+
+#define META_FOP(i, fop, fr, t, params...)                                     \
+    {                                                                          \
+        struct xlator_fops *_fops = NULL;                                      \
+                                                                               \
+        _fops = meta_fops_get(i, t);                                           \
+                                                                               \
+        _fops->fop(fr, t, params);                                             \
+    }                                                                          \
+    while (0)
+
+void
+meta_iatt_fill(struct iatt *iatt, inode_t *inode, ia_type_t type);
+
+int
+meta_inode_discover(call_frame_t *frame, xlator_t *this, loc_t *loc,
+                    dict_t *xdata);
+
+int
+meta_ops_set(inode_t *inode, xlator_t *this, struct meta_ops *ops);
+
+struct xlator_fops *
+meta_fops_get(inode_t *inode, xlator_t *this);
+struct xlator_cbks *
+meta_cbks_get(inode_t *inode, xlator_t *this);
+struct meta_ops *
+meta_ops_get(inode_t *inode, xlator_t *this);
+
+int
+meta_ctx_set(inode_t *inode, xlator_t *this, void *ctx);
+
+void *
+meta_ctx_get(inode_t *inode, xlator_t *this);
+
+void
+meta_local_cleanup(meta_local_t *local, xlator_t *this);
+
+struct xlator_fops *
+meta_defaults_init(struct xlator_fops *fops);
+
+meta_fd_t *
+meta_fd_get(fd_t *fd, xlator_t *this);
+
+int
+meta_fd_release(fd_t *fd, xlator_t *this);
+
+dict_t *
+meta_direct_io_mode(dict_t *xdata, call_frame_t *frame);
+
+meta_local_t *
+meta_local(call_frame_t *frame);
+
+int
+meta_file_fill(xlator_t *this, fd_t *fd);
+
+int
+meta_dir_fill(xlator_t *this, fd_t *fd);
+
+int
+fixed_dirents_len(struct meta_dirent *dirents);
 #endif /* __META_H__ */
diff --git a/xlators/meta/src/name-file.c b/xlators/meta/src/name-file.c
index 8ddd42945d7..5874a24d78a 100644
--- a/xlators/meta/src/name-file.c
+++ b/xlators/meta/src/name-file.c
@@ -8,46 +8,37 @@
    cases as published by the Free Software Foundation.
 */
 
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
-
-#include "xlator.h"
-#include "defaults.h"
+#include <glusterfs/xlator.h>
+#include <glusterfs/defaults.h>
 
 #include "meta-mem-types.h"
 #include "meta.h"
-#include "strfd.h"
-#include "globals.h"
-#include "lkowner.h"
-
+#include <glusterfs/strfd.h>
+#include <glusterfs/lkowner.h>
 
 static int
-name_file_fill (xlator_t *this, inode_t *file, strfd_t *strfd)
+name_file_fill(xlator_t *this, inode_t *file, strfd_t *strfd)
 {
-	xlator_t *xl = NULL;
+    xlator_t *xl = NULL;
 
-	xl = meta_ctx_get (file, this);
+    xl = meta_ctx_get(file, this);
 
-	strprintf (strfd, "%s\n", xl->name);
+    strprintf(strfd, "%s\n", xl->name);
 
-	return strfd->size;
+    return strfd->size;
 }
 
-
 static struct meta_ops name_file_ops = {
-	.file_fill = name_file_fill,
+    .file_fill = name_file_fill,
 };
 
-
 int
-meta_name_file_hook (call_frame_t *frame, xlator_t *this, loc_t *loc,
-		     dict_t *xdata)
+meta_name_file_hook(call_frame_t *frame, xlator_t *this, loc_t *loc,
+                    dict_t *xdata)
 {
-	meta_ops_set (loc->inode, this, &name_file_ops);
+    meta_ops_set(loc->inode, this, &name_file_ops);
 
-	meta_ctx_set (loc->inode, this, meta_ctx_get (loc->parent, this));
+    meta_ctx_set(loc->inode, this, meta_ctx_get(loc->parent, this));
 
-	return 0;
+    return 0;
 }
diff --git a/xlators/meta/src/option-file.c b/xlators/meta/src/option-file.c
index 4f9067e1490..ff55eca592f 100644
--- a/xlators/meta/src/option-file.c
+++ b/xlators/meta/src/option-file.c
@@ -8,49 +8,38 @@
    cases as published by the Free Software Foundation.
 */
 
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
-
-#include "xlator.h"
-#include "defaults.h"
+#include <glusterfs/xlator.h>
+#include <glusterfs/defaults.h>
 
 #include "meta-mem-types.h"
 #include "meta.h"
 #include "meta-hooks.h"
 
-
 static int
-option_file_fill (xlator_t *this, inode_t *inode, strfd_t *strfd)
+option_file_fill(xlator_t *this, inode_t *inode, strfd_t *strfd)
 {
-	data_t *data = NULL;
+    data_t *data = NULL;
 
-	data = meta_ctx_get (inode, this);
+    data = meta_ctx_get(inode, this);
 
-	strprintf (strfd, "%s\n", data_to_str (data));
+    strprintf(strfd, "%s\n", data_to_str(data));
 
-	return strfd->size;
+    return strfd->size;
 }
 
-
-static struct meta_ops option_file_ops = {
-	.file_fill = option_file_fill
-};
-
+static struct meta_ops option_file_ops = {.file_fill = option_file_fill};
 
 int
-meta_option_file_hook (call_frame_t *frame, xlator_t *this, loc_t *loc,
-		       dict_t *xdata)
+meta_option_file_hook(call_frame_t *frame, xlator_t *this, loc_t *loc,
+                      dict_t *xdata)
 {
-	xlator_t *xl = NULL;
+    xlator_t *xl = NULL;
 
-	xl = meta_ctx_get (loc->parent, this);
+    xl = meta_ctx_get(loc->parent, this);
 
-	meta_ctx_set (loc->inode, this,
-		      dict_get (xl->options, (char *) loc->name));
+    meta_ctx_set(loc->inode, this, dict_get(xl->options, (char *)loc->name));
 
-	meta_ops_set (loc->inode, this, &option_file_ops);
+    meta_ops_set(loc->inode, this, &option_file_ops);
 
-	return 0;
+    return 0;
 }
diff --git a/xlators/meta/src/options-dir.c b/xlators/meta/src/options-dir.c
index 229d365a9c7..d68a7eeaffc 100644
--- a/xlators/meta/src/options-dir.c
+++ b/xlators/meta/src/options-dir.c
@@ -8,69 +8,58 @@
    cases as published by the Free Software Foundation.
 */
 
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
-
-#include "xlator.h"
-#include "defaults.h"
+#include <glusterfs/xlator.h>
+#include <glusterfs/defaults.h>
 
 #include "meta-mem-types.h"
 #include "meta.h"
 #include "meta-hooks.h"
 
-
 static int
-dict_key_add (dict_t *dict, char *key, data_t *value, void *data)
+dict_key_add(dict_t *dict, char *key, data_t *value, void *data)
 {
-	struct meta_dirent **direntp = data;
+    struct meta_dirent **direntp = data;
 
-	(*direntp)->name = gf_strdup (key);
-	(*direntp)->type = IA_IFREG;
-	(*direntp)->hook = meta_option_file_hook;
+    (*direntp)->name = gf_strdup(key);
+    (*direntp)->type = IA_IFREG;
+    (*direntp)->hook = meta_option_file_hook;
 
-	(*direntp)++;
-	return 0;
+    (*direntp)++;
+    return 0;
 }
 
-
 static int
-options_dir_fill (xlator_t *this, inode_t *inode, struct meta_dirent **dp)
+options_dir_fill(xlator_t *this, inode_t *inode, struct meta_dirent **dp)
 {
-	struct meta_dirent *dirent = NULL;
-	struct meta_dirent *direntp = NULL;
-	xlator_t *xl = NULL;
+    struct meta_dirent *dirent = NULL;
+    struct meta_dirent *direntp = NULL;
+    xlator_t *xl = NULL;
 
-	xl = meta_ctx_get (inode, this);
+    xl = meta_ctx_get(inode, this);
 
-	dirent = GF_CALLOC (sizeof (*dirent), xl->options->count,
-			    gf_meta_mt_dirents_t);
-	if (!dirent)
-		return -1;
+    dirent = GF_CALLOC(sizeof(*dirent), xl->options->count,
+                       gf_meta_mt_dirents_t);
+    if (!dirent)
+        return -1;
 
-	direntp = dirent;
+    direntp = dirent;
 
-	dict_foreach (xl->options, dict_key_add, &direntp);
+    dict_foreach(xl->options, dict_key_add, &direntp);
 
-	*dp = dirent;
+    *dp = dirent;
 
-	return xl->options->count;
+    return xl->options->count;
 }
 
-
-static struct meta_ops options_dir_ops = {
-	.dir_fill = options_dir_fill
-};
-
+static struct meta_ops options_dir_ops = {.dir_fill = options_dir_fill};
 
 int
-meta_options_dir_hook (call_frame_t *frame, xlator_t *this, loc_t *loc,
-		       dict_t *xdata)
+meta_options_dir_hook(call_frame_t *frame, xlator_t *this, loc_t *loc,
+                      dict_t *xdata)
 {
-	meta_ctx_set (loc->inode, this, meta_ctx_get (loc->parent, this));
+    meta_ctx_set(loc->inode, this, meta_ctx_get(loc->parent, this));
 
-	meta_ops_set (loc->inode, this, &options_dir_ops);
+    meta_ops_set(loc->inode, this, &options_dir_ops);
 
-	return 0;
+    return 0;
 }
diff --git a/xlators/meta/src/private-file.c b/xlators/meta/src/private-file.c
index d2402c08bb4..23ec319456b 100644
--- a/xlators/meta/src/private-file.c
+++ b/xlators/meta/src/private-file.c
@@ -8,45 +8,37 @@
    cases as published by the Free Software Foundation.
 */
 
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
-
-#include "xlator.h"
-#include "defaults.h"
+#include <glusterfs/xlator.h>
+#include <glusterfs/defaults.h>
 
 #include "meta-mem-types.h"
 #include "meta.h"
-#include "strfd.h"
-#include "statedump.h"
-
+#include <glusterfs/strfd.h>
+#include <glusterfs/statedump.h>
 
 static int
-private_file_fill (xlator_t *this, inode_t *file, strfd_t *strfd)
+private_file_fill(xlator_t *this, inode_t *file, strfd_t *strfd)
 {
-	xlator_t *xl = NULL;
+    xlator_t *xl = NULL;
 
-	xl = meta_ctx_get (file, this);
+    xl = meta_ctx_get(file, this);
 
-	gf_proc_dump_xlator_private (xl, strfd);
+    gf_proc_dump_xlator_private(xl, strfd);
 
-	return strfd->size;
+    return strfd->size;
 }
 
-
 static struct meta_ops private_file_ops = {
-	.file_fill = private_file_fill,
+    .file_fill = private_file_fill,
 };
 
-
 int
-meta_private_file_hook (call_frame_t *frame, xlator_t *this, loc_t *loc,
-			dict_t *xdata)
+meta_private_file_hook(call_frame_t *frame, xlator_t *this, loc_t *loc,
+                       dict_t *xdata)
 {
-	meta_ops_set (loc->inode, this, &private_file_ops);
+    meta_ops_set(loc->inode, this, &private_file_ops);
 
-	meta_ctx_set (loc->inode, this, meta_ctx_get (loc->parent, this));
+    meta_ctx_set(loc->inode, this, meta_ctx_get(loc->parent, this));
 
-	return 0;
+    return 0;
 }
diff --git a/xlators/meta/src/process_uuid-file.c b/xlators/meta/src/process_uuid-file.c
index c16ceaac150..a24c1b57ab3 100644
--- a/xlators/meta/src/process_uuid-file.c
+++ b/xlators/meta/src/process_uuid-file.c
@@ -8,39 +8,30 @@
    cases as published by the Free Software Foundation.
 */
 
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
-
-#include "xlator.h"
-#include "defaults.h"
+#include <glusterfs/xlator.h>
+#include <glusterfs/defaults.h>
 
 #include "meta-mem-types.h"
 #include "meta.h"
-#include "strfd.h"
-#include "globals.h"
-#include "lkowner.h"
-
+#include <glusterfs/strfd.h>
+#include <glusterfs/lkowner.h>
 
 static int
-process_uuid_file_fill (xlator_t *this, inode_t *file, strfd_t *strfd)
+process_uuid_file_fill(xlator_t *this, inode_t *file, strfd_t *strfd)
 {
-	strprintf (strfd, "%s\n", this->ctx->process_uuid);
-	return strfd->size;
+    strprintf(strfd, "%s\n", this->ctx->process_uuid);
+    return strfd->size;
 }
 
-
 static struct meta_ops process_uuid_file_ops = {
-	.file_fill = process_uuid_file_fill,
+    .file_fill = process_uuid_file_fill,
 };
 
-
 int
-meta_process_uuid_file_hook (call_frame_t *frame, xlator_t *this, loc_t *loc,
-			     dict_t *xdata)
+meta_process_uuid_file_hook(call_frame_t *frame, xlator_t *this, loc_t *loc,
+                            dict_t *xdata)
 {
-	meta_ops_set (loc->inode, this, &process_uuid_file_ops);
+    meta_ops_set(loc->inode, this, &process_uuid_file_ops);
 
-	return 0;
+    return 0;
 }
diff --git a/xlators/meta/src/profile-file.c b/xlators/meta/src/profile-file.c
index 2a25c154699..829dcb77451 100644
--- a/xlators/meta/src/profile-file.c
+++ b/xlators/meta/src/profile-file.c
@@ -8,45 +8,37 @@
    cases as published by the Free Software Foundation.
 */
 
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
-
-#include "xlator.h"
-#include "defaults.h"
+#include <glusterfs/xlator.h>
+#include <glusterfs/defaults.h>
 
 #include "meta-mem-types.h"
 #include "meta.h"
-#include "strfd.h"
-#include "statedump.h"
-
+#include <glusterfs/strfd.h>
+#include <glusterfs/statedump.h>
 
 static int
-profile_file_fill (xlator_t *this, inode_t *file, strfd_t *strfd)
+profile_file_fill(xlator_t *this, inode_t *file, strfd_t *strfd)
 {
-	xlator_t *xl = NULL;
+    xlator_t *xl = NULL;
 
-	xl = meta_ctx_get (file, this);
+    xl = meta_ctx_get(file, this);
 
-	gf_proc_dump_xlator_profile (xl, strfd);
+    gf_proc_dump_xlator_profile(xl, strfd);
 
-	return strfd->size;
+    return strfd->size;
 }
 
-
 static struct meta_ops profile_file_ops = {
-	.file_fill = profile_file_fill,
+    .file_fill = profile_file_fill,
 };
 
-
 int
-meta_profile_file_hook (call_frame_t *frame, xlator_t *this, loc_t *loc,
-			dict_t *xdata)
+meta_profile_file_hook(call_frame_t *frame, xlator_t *this, loc_t *loc,
+                       dict_t *xdata)
 {
-	meta_ops_set (loc->inode, this, &profile_file_ops);
+    meta_ops_set(loc->inode, this, &profile_file_ops);
 
-	meta_ctx_set (loc->inode, this, meta_ctx_get (loc->parent, this));
+    meta_ctx_set(loc->inode, this, meta_ctx_get(loc->parent, this));
 
-	return 0;
+    return 0;
 }
diff --git a/xlators/meta/src/root-dir.c b/xlators/meta/src/root-dir.c
index d104d842cf9..80292bd3dda 100644
--- a/xlators/meta/src/root-dir.c
+++ b/xlators/meta/src/root-dir.c
@@ -8,72 +8,70 @@
    cases as published by the Free Software Foundation.
 */
 
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
-
-#include "xlator.h"
-#include "defaults.h"
+#include <glusterfs/xlator.h>
+#include <glusterfs/defaults.h>
 
 #include "meta-mem-types.h"
 #include "meta.h"
 #include "meta-hooks.h"
 
-
 static struct meta_dirent root_dir_dirents[] = {
-	DOT_DOTDOT,
-
-	{ .name = "graphs",
-	  .type = IA_IFDIR,
-	  .hook = meta_graphs_dir_hook,
-	},
-	{ .name = "frames",
-	  .type = IA_IFREG,
-	  .hook = meta_frames_file_hook,
-	},
-	{ .name = "logging",
-	  .type = IA_IFDIR,
-	  .hook = meta_logging_dir_hook,
-	},
-	{ .name = "process_uuid",
-	  .type = IA_IFREG,
-	  .hook = meta_process_uuid_file_hook,
-	},
-	{ .name = "version",
-	  .type = IA_IFREG,
-	  .hook = meta_version_file_hook,
-	},
-	{ .name = "cmdline",
-	  .type = IA_IFREG,
-	  .hook = meta_cmdline_file_hook,
-	},
-	{ .name = "mallinfo",
-	  .type = IA_IFREG,
-	  .hook = meta_mallinfo_file_hook,
-	},
-	{ .name = "master",
-	  .type = IA_IFDIR,
-	  .hook = meta_master_dir_hook,
-	},
-	{ .name = "measure_latency",
-	  .type = IA_IFREG,
-	  .hook = meta_measure_file_hook,
-	},
-	{ .name = NULL }
-};
-
+    DOT_DOTDOT,
 
-static struct meta_ops meta_root_dir_ops = {
-	.fixed_dirents = root_dir_dirents
-};
+    {
+        .name = "graphs",
+        .type = IA_IFDIR,
+        .hook = meta_graphs_dir_hook,
+    },
+    {
+        .name = "frames",
+        .type = IA_IFREG,
+        .hook = meta_frames_file_hook,
+    },
+    {
+        .name = "logging",
+        .type = IA_IFDIR,
+        .hook = meta_logging_dir_hook,
+    },
+    {
+        .name = "process_uuid",
+        .type = IA_IFREG,
+        .hook = meta_process_uuid_file_hook,
+    },
+    {
+        .name = "version",
+        .type = IA_IFREG,
+        .hook = meta_version_file_hook,
+    },
+    {
+        .name = "cmdline",
+        .type = IA_IFREG,
+        .hook = meta_cmdline_file_hook,
+    },
+    {
+        .name = "mallinfo",
+        .type = IA_IFREG,
+        .hook = meta_mallinfo_file_hook,
+    },
+    {
+        .name = "master",
+        .type = IA_IFDIR,
+        .hook = meta_master_dir_hook,
+    },
+    {
+        .name = "measure_latency",
+        .type = IA_IFREG,
+        .hook = meta_measure_file_hook,
+    },
+    {.name = NULL}};
 
+static struct meta_ops meta_root_dir_ops = {.fixed_dirents = root_dir_dirents};
 
 int
-meta_root_dir_hook (call_frame_t *frame, xlator_t *this, loc_t *loc,
-		    dict_t *xdata)
+meta_root_dir_hook(call_frame_t *frame, xlator_t *this, loc_t *loc,
+                   dict_t *xdata)
 {
-	meta_ops_set (loc->inode, this, &meta_root_dir_ops);
+    meta_ops_set(loc->inode, this, &meta_root_dir_ops);
 
-	return 0;
+    return 0;
 }
diff --git a/xlators/meta/src/subvolume-link.c b/xlators/meta/src/subvolume-link.c
index ca71a751541..5b1f752efd0 100644
--- a/xlators/meta/src/subvolume-link.c
+++ b/xlators/meta/src/subvolume-link.c
@@ -8,59 +8,49 @@
    cases as published by the Free Software Foundation.
 */
 
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
-
-#include "xlator.h"
-#include "defaults.h"
+#include <glusterfs/xlator.h>
+#include <glusterfs/defaults.h>
 
 #include "meta-mem-types.h"
 #include "meta.h"
 
-
 static int
-subvolume_link_fill (xlator_t *this, inode_t *inode, strfd_t *strfd)
+subvolume_link_fill(xlator_t *this, inode_t *inode, strfd_t *strfd)
 {
-	xlator_t *xl = NULL;
+    xlator_t *xl = NULL;
 
-	xl = meta_ctx_get (inode, this);
+    xl = meta_ctx_get(inode, this);
 
-	strprintf (strfd, "../../%s", xl->name);
+    strprintf(strfd, "../../%s", xl->name);
 
-	return 0;
+    return 0;
 }
 
-
-struct meta_ops subvolume_link_ops = {
-	.link_fill = subvolume_link_fill
-};
-
+struct meta_ops subvolume_link_ops = {.link_fill = subvolume_link_fill};
 
 int
-meta_subvolume_link_hook (call_frame_t *frame, xlator_t *this, loc_t *loc,
-			  dict_t *xdata)
+meta_subvolume_link_hook(call_frame_t *frame, xlator_t *this, loc_t *loc,
+                         dict_t *xdata)
 {
-	int count = 0;
-	int i = 0;
-	xlator_t *xl = NULL;
-	xlator_list_t *subv = NULL;
-	xlator_t *subvol = NULL;
-
-	count = strtol (loc->name, 0, 0);
-	xl = meta_ctx_get (loc->parent, this);
-
-	for (subv = xl->children; subv; subv = subv->next) {
-		if (i == count) {
-			subvol = subv->xlator;
-			break;
-		}
-		i++;
-	}
-
-	meta_ctx_set (loc->inode, this, subvol);
-
-	meta_ops_set (loc->inode, this, &subvolume_link_ops);
-	return 0;
+    int count = 0;
+    int i = 0;
+    xlator_t *xl = NULL;
+    xlator_list_t *subv = NULL;
+    xlator_t *subvol = NULL;
+
+    count = strtol(loc->name, 0, 0);
+    xl = meta_ctx_get(loc->parent, this);
+
+    for (subv = xl->children; subv; subv = subv->next) {
+        if (i == count) {
+            subvol = subv->xlator;
+            break;
+        }
+        i++;
+    }
+
+    meta_ctx_set(loc->inode, this, subvol);
+
+    meta_ops_set(loc->inode, this, &subvolume_link_ops);
+    return 0;
 }
diff --git a/xlators/meta/src/subvolumes-dir.c b/xlators/meta/src/subvolumes-dir.c
index f8d0e8039a1..3cb170ea1f4 100644
--- a/xlators/meta/src/subvolumes-dir.c
+++ b/xlators/meta/src/subvolumes-dir.c
@@ -8,65 +8,55 @@
    cases as published by the Free Software Foundation.
 */
 
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
-
-#include "xlator.h"
-#include "defaults.h"
+#include <glusterfs/xlator.h>
+#include <glusterfs/defaults.h>
 
 #include "meta-mem-types.h"
 #include "meta.h"
 #include "meta-hooks.h"
 
-
 static int
-subvolumes_dir_fill (xlator_t *this, inode_t *dir, struct meta_dirent **dp)
+subvolumes_dir_fill(xlator_t *this, inode_t *dir, struct meta_dirent **dp)
 {
-	struct meta_dirent *dirents = NULL;
-	xlator_t *xl = NULL;
-	xlator_list_t *subv = NULL;
-	int i = 0;
-	int count = 0;
+    struct meta_dirent *dirents = NULL;
+    xlator_t *xl = NULL;
+    xlator_list_t *subv = NULL;
+    int i = 0;
+    int count = 0;
 
-	xl = meta_ctx_get (dir, this);
+    xl = meta_ctx_get(dir, this);
 
-	for (subv = xl->children; subv; subv = subv->next)
-		count++;
+    for (subv = xl->children; subv; subv = subv->next)
+        count++;
 
-	dirents = GF_CALLOC (sizeof (*dirents), count, gf_meta_mt_dirents_t);
-	if (!dirents)
-		return -1;
+    dirents = GF_MALLOC(sizeof(*dirents) * count, gf_meta_mt_dirents_t);
+    if (!dirents)
+        return -1;
 
-	for (subv = xl->children; subv; subv = subv->next) {
-		char num[16] = { };
-		snprintf (num, 16, "%d", i);
+    for (subv = xl->children; subv; subv = subv->next) {
+        char num[16] = {};
+        snprintf(num, 16, "%d", i);
 
-		dirents[i].name = gf_strdup (num);
-		dirents[i].type = IA_IFLNK;
-		dirents[i].hook = meta_subvolume_link_hook;
-		i++;
-	}
+        dirents[i].name = gf_strdup(num);
+        dirents[i].type = IA_IFLNK;
+        dirents[i].hook = meta_subvolume_link_hook;
+        i++;
+    }
 
-	*dp = dirents;
+    *dp = dirents;
 
-	return count;
+    return count;
 }
 
-
-static struct meta_ops subvolumes_dir_ops = {
-	.dir_fill = subvolumes_dir_fill
-};
-
+static struct meta_ops subvolumes_dir_ops = {.dir_fill = subvolumes_dir_fill};
 
 int
-meta_subvolumes_dir_hook (call_frame_t *frame, xlator_t *this, loc_t *loc,
-			  dict_t *xdata)
+meta_subvolumes_dir_hook(call_frame_t *frame, xlator_t *this, loc_t *loc,
+                         dict_t *xdata)
 {
-	meta_ctx_set (loc->inode, this, meta_ctx_get (loc->parent, this));
+    meta_ctx_set(loc->inode, this, meta_ctx_get(loc->parent, this));
 
-	meta_ops_set (loc->inode, this, &subvolumes_dir_ops);
+    meta_ops_set(loc->inode, this, &subvolumes_dir_ops);
 
-	return 0;
+    return 0;
 }
diff --git a/xlators/meta/src/top-link.c b/xlators/meta/src/top-link.c
index 94b7f7a9f56..33f0d407411 100644
--- a/xlators/meta/src/top-link.c
+++ b/xlators/meta/src/top-link.c
@@ -8,43 +8,33 @@
    cases as published by the Free Software Foundation.
 */
 
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
-
-#include "xlator.h"
-#include "defaults.h"
+#include <glusterfs/xlator.h>
+#include <glusterfs/defaults.h>
 
 #include "meta-mem-types.h"
 #include "meta.h"
 
-
 static int
-top_link_fill (xlator_t *this, inode_t *inode, strfd_t *strfd)
+top_link_fill(xlator_t *this, inode_t *inode, strfd_t *strfd)
 {
-	glusterfs_graph_t *graph = NULL;
+    glusterfs_graph_t *graph = NULL;
 
-	graph = meta_ctx_get (inode, this);
+    graph = meta_ctx_get(inode, this);
 
-	strprintf (strfd, "%s", ((xlator_t *)graph->top)->name);
+    strprintf(strfd, "%s", ((xlator_t *)graph->top)->name);
 
-	return 0;
+    return 0;
 }
 
-
-struct meta_ops top_link_ops = {
-	.link_fill = top_link_fill
-};
-
+struct meta_ops top_link_ops = {.link_fill = top_link_fill};
 
 int
-meta_top_link_hook (call_frame_t *frame, xlator_t *this, loc_t *loc,
-		    dict_t *xdata)
+meta_top_link_hook(call_frame_t *frame, xlator_t *this, loc_t *loc,
+                   dict_t *xdata)
 {
-	meta_ops_set (loc->inode, this, &top_link_ops);
+    meta_ops_set(loc->inode, this, &top_link_ops);
 
-	meta_ctx_set (loc->inode, this, meta_ctx_get (loc->parent, this));
+    meta_ctx_set(loc->inode, this, meta_ctx_get(loc->parent, this));
 
-	return 0;
+    return 0;
 }
diff --git a/xlators/meta/src/type-file.c b/xlators/meta/src/type-file.c
index 1970f4e33e8..ece342a0b2a 100644
--- a/xlators/meta/src/type-file.c
+++ b/xlators/meta/src/type-file.c
@@ -8,46 +8,37 @@
    cases as published by the Free Software Foundation.
 */
 
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
-
-#include "xlator.h"
-#include "defaults.h"
+#include <glusterfs/xlator.h>
+#include <glusterfs/defaults.h>
 
 #include "meta-mem-types.h"
 #include "meta.h"
-#include "strfd.h"
-#include "globals.h"
-#include "lkowner.h"
-
+#include <glusterfs/strfd.h>
+#include <glusterfs/lkowner.h>
 
 static int
-type_file_fill (xlator_t *this, inode_t *file, strfd_t *strfd)
+type_file_fill(xlator_t *this, inode_t *file, strfd_t *strfd)
 {
-	xlator_t *xl = NULL;
+    xlator_t *xl = NULL;
 
-	xl = meta_ctx_get (file, this);
+    xl = meta_ctx_get(file, this);
 
-	strprintf (strfd, "%s\n", xl->type);
+    strprintf(strfd, "%s\n", xl->type);
 
-	return strfd->size;
+    return strfd->size;
 }
 
-
 static struct meta_ops type_file_ops = {
-	.file_fill = type_file_fill,
+    .file_fill = type_file_fill,
 };
 
-
 int
-meta_type_file_hook (call_frame_t *frame, xlator_t *this, loc_t *loc,
-		     dict_t *xdata)
+meta_type_file_hook(call_frame_t *frame, xlator_t *this, loc_t *loc,
+                    dict_t *xdata)
 {
-	meta_ops_set (loc->inode, this, &type_file_ops);
+    meta_ops_set(loc->inode, this, &type_file_ops);
 
-	meta_ctx_set (loc->inode, this, meta_ctx_get (loc->parent, this));
+    meta_ctx_set(loc->inode, this, meta_ctx_get(loc->parent, this));
 
-	return 0;
+    return 0;
 }
diff --git a/xlators/meta/src/version-file.c b/xlators/meta/src/version-file.c
index c402453ee96..36276fb810a 100644
--- a/xlators/meta/src/version-file.c
+++ b/xlators/meta/src/version-file.c
@@ -8,40 +8,30 @@
    cases as published by the Free Software Foundation.
 */
 
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
-
-#include "xlator.h"
-#include "defaults.h"
+#include <glusterfs/xlator.h>
+#include <glusterfs/defaults.h>
 
 #include "meta-mem-types.h"
 #include "meta.h"
-#include "strfd.h"
-#include "globals.h"
-#include "lkowner.h"
-
+#include <glusterfs/strfd.h>
+#include <glusterfs/lkowner.h>
 
 static int
-version_file_fill (xlator_t *this, inode_t *file, strfd_t *strfd)
+version_file_fill(xlator_t *this, inode_t *file, strfd_t *strfd)
 {
-        strprintf (strfd, "{ \n  \"Package Version\": \"%s\"\n}",
-                   PACKAGE_VERSION);
-        return strfd->size;
+    strprintf(strfd, "{ \n  \"Package Version\": \"%s\"\n}", PACKAGE_VERSION);
+    return strfd->size;
 }
 
-
 static struct meta_ops version_file_ops = {
-        .file_fill = version_file_fill,
+    .file_fill = version_file_fill,
 };
 
-
 int
-meta_version_file_hook (call_frame_t *frame, xlator_t *this, loc_t *loc,
-                        dict_t *xdata)
+meta_version_file_hook(call_frame_t *frame, xlator_t *this, loc_t *loc,
+                       dict_t *xdata)
 {
-        meta_ops_set (loc->inode, this, &version_file_ops);
+    meta_ops_set(loc->inode, this, &version_file_ops);
 
-        return 0;
+    return 0;
 }
diff --git a/xlators/meta/src/view-dir.c b/xlators/meta/src/view-dir.c
index 6edd455b072..30931061567 100644
--- a/xlators/meta/src/view-dir.c
+++ b/xlators/meta/src/view-dir.c
@@ -8,38 +8,26 @@
    cases as published by the Free Software Foundation.
 */
 
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
-
-#include "xlator.h"
-#include "defaults.h"
+#include <glusterfs/xlator.h>
+#include <glusterfs/defaults.h>
 
 #include "meta-mem-types.h"
 #include "meta.h"
 #include "meta-hooks.h"
 
+static struct meta_dirent view_dir_dirents[] = {DOT_DOTDOT,
 
-static struct meta_dirent view_dir_dirents[] = {
-	DOT_DOTDOT,
-
-	{ .name = NULL }
-};
-
-
-static struct meta_ops view_dir_ops = {
-	.fixed_dirents = view_dir_dirents
-};
+                                                {.name = NULL}};
 
+static struct meta_ops view_dir_ops = {.fixed_dirents = view_dir_dirents};
 
 int
-meta_view_dir_hook (call_frame_t *frame, xlator_t *this, loc_t *loc,
-		    dict_t *xdata)
+meta_view_dir_hook(call_frame_t *frame, xlator_t *this, loc_t *loc,
+                   dict_t *xdata)
 {
-	meta_ctx_set (loc->inode, this, meta_ctx_get (loc->parent, this));
+    meta_ctx_set(loc->inode, this, meta_ctx_get(loc->parent, this));
 
-	meta_ops_set (loc->inode, this, &view_dir_ops);
+    meta_ops_set(loc->inode, this, &view_dir_ops);
 
-	return 0;
+    return 0;
 }
diff --git a/xlators/meta/src/volfile-file.c b/xlators/meta/src/volfile-file.c
index 899285763e0..b2e2562ab8b 100644
--- a/xlators/meta/src/volfile-file.c
+++ b/xlators/meta/src/volfile-file.c
@@ -8,84 +8,72 @@
    cases as published by the Free Software Foundation.
 */
 
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
-
-#include "xlator.h"
-#include "defaults.h"
+#include <glusterfs/xlator.h>
+#include <glusterfs/defaults.h>
 
 #include "meta-mem-types.h"
 #include "meta.h"
-#include "strfd.h"
-
-
+#include <glusterfs/strfd.h>
 
 static int
-xldump_options (dict_t *this, char *key, data_t *value,	void *strfd)
+xldump_options(dict_t *this, char *key, data_t *value, void *strfd)
 {
-	strprintf (strfd, "    option %s %s\n", key, value->data);
-	return 0;
+    strprintf(strfd, "    option %s %s\n", key, value->data);
+    return 0;
 }
 
-
 static void
-xldump_subvolumes (xlator_t *this, void *strfd)
+xldump_subvolumes(xlator_t *this, void *strfd)
 {
-	xlator_list_t *subv = NULL;
+    xlator_list_t *subv = NULL;
 
-	if (!this->children)
-		return;
+    if (!this->children)
+        return;
 
-	strprintf (strfd, "    subvolumes");
+    strprintf(strfd, "    subvolumes");
 
-	for (subv = this->children; subv; subv= subv->next)
-		strprintf (strfd, " %s", subv->xlator->name);
+    for (subv = this->children; subv; subv = subv->next)
+        strprintf(strfd, " %s", subv->xlator->name);
 
-	strprintf (strfd, "\n");
+    strprintf(strfd, "\n");
 }
 
-
 static void
-xldump (xlator_t *each, void *strfd)
+xldump(xlator_t *each, void *strfd)
 {
-	strprintf (strfd, "volume %s\n", each->name);
-	strprintf (strfd, "    type %s\n", each->type);
-	dict_foreach (each->options, xldump_options, strfd);
+    strprintf(strfd, "volume %s\n", each->name);
+    strprintf(strfd, "    type %s\n", each->type);
+    dict_foreach(each->options, xldump_options, strfd);
 
-	xldump_subvolumes (each, strfd);
+    xldump_subvolumes(each, strfd);
 
-	strprintf (strfd, "end-volume\n");
-	strprintf (strfd, "\n");
+    strprintf(strfd, "end-volume\n");
+    strprintf(strfd, "\n");
 }
 
-
 static int
-volfile_file_fill (xlator_t *this, inode_t *file, strfd_t *strfd)
+volfile_file_fill(xlator_t *this, inode_t *file, strfd_t *strfd)
 {
-	glusterfs_graph_t *graph = NULL;
+    glusterfs_graph_t *graph = NULL;
 
-	graph = meta_ctx_get (file, this);
+    graph = meta_ctx_get(file, this);
 
-	xlator_foreach_depth_first (graph->top, xldump, strfd);
+    xlator_foreach_depth_first(graph->top, xldump, strfd);
 
-	return strfd->size;
+    return strfd->size;
 }
 
-
 static struct meta_ops volfile_file_ops = {
-	.file_fill = volfile_file_fill,
+    .file_fill = volfile_file_fill,
 };
 
-
 int
-meta_volfile_file_hook (call_frame_t *frame, xlator_t *this, loc_t *loc,
-			dict_t *xdata)
+meta_volfile_file_hook(call_frame_t *frame, xlator_t *this, loc_t *loc,
+                       dict_t *xdata)
 {
-	meta_ops_set (loc->inode, this, &volfile_file_ops);
+    meta_ops_set(loc->inode, this, &volfile_file_ops);
 
-	meta_ctx_set (loc->inode, this, meta_ctx_get (loc->parent, this));
+    meta_ctx_set(loc->inode, this, meta_ctx_get(loc->parent, this));
 
-	return 0;
+    return 0;
 }
diff --git a/xlators/meta/src/xlator-dir.c b/xlators/meta/src/xlator-dir.c
index 09b7bde62de..86189715790 100644
--- a/xlators/meta/src/xlator-dir.c
+++ b/xlators/meta/src/xlator-dir.c
@@ -8,93 +8,90 @@
    cases as published by the Free Software Foundation.
 */
 
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
-
-#include "xlator.h"
-#include "defaults.h"
+#include <glusterfs/xlator.h>
+#include <glusterfs/defaults.h>
 
 #include "meta-mem-types.h"
 #include "meta.h"
 #include "meta-hooks.h"
 
-
 static struct meta_dirent xlator_dir_dirents[] = {
-	DOT_DOTDOT,
-
-	{ .name = "view",
-	  .type = IA_IFDIR,
-	  .hook = meta_view_dir_hook,
-	},
-	{ .name = "type",
-	  .type = IA_IFREG,
-	  .hook = meta_type_file_hook,
-	},
-	{ .name = "name",
-	  .type = IA_IFREG,
-	  .hook = meta_name_file_hook,
-	},
-	{ .name = "subvolumes",
-	  .type = IA_IFDIR,
-	  .hook = meta_subvolumes_dir_hook,
-	},
-	{ .name = "options",
-	  .type = IA_IFDIR,
-	  .hook = meta_options_dir_hook,
-	},
-	{ .name = "private",
-	  .type = IA_IFREG,
-	  .hook = meta_private_file_hook,
-	},
-	{ .name = "history",
-	  .type = IA_IFREG,
-	  .hook = meta_history_file_hook,
-	},
-	{ .name = "meminfo",
-	  .type = IA_IFREG,
-	  .hook = meta_meminfo_file_hook,
-	},
-	{ .name = "profile",
-	  .type = IA_IFREG,
-	  .hook = meta_profile_file_hook,
-	},
-	{ .name = NULL }
-};
-
-
-static struct meta_ops xlator_dir_ops = {
-	.fixed_dirents = xlator_dir_dirents
-};
-
+    DOT_DOTDOT,
+
+    {
+        .name = "view",
+        .type = IA_IFDIR,
+        .hook = meta_view_dir_hook,
+    },
+    {
+        .name = "type",
+        .type = IA_IFREG,
+        .hook = meta_type_file_hook,
+    },
+    {
+        .name = "name",
+        .type = IA_IFREG,
+        .hook = meta_name_file_hook,
+    },
+    {
+        .name = "subvolumes",
+        .type = IA_IFDIR,
+        .hook = meta_subvolumes_dir_hook,
+    },
+    {
+        .name = "options",
+        .type = IA_IFDIR,
+        .hook = meta_options_dir_hook,
+    },
+    {
+        .name = "private",
+        .type = IA_IFREG,
+        .hook = meta_private_file_hook,
+    },
+    {
+        .name = "history",
+        .type = IA_IFREG,
+        .hook = meta_history_file_hook,
+    },
+    {
+        .name = "meminfo",
+        .type = IA_IFREG,
+        .hook = meta_meminfo_file_hook,
+    },
+    {
+        .name = "profile",
+        .type = IA_IFREG,
+        .hook = meta_profile_file_hook,
+    },
+    {.name = NULL}};
+
+static struct meta_ops xlator_dir_ops = {.fixed_dirents = xlator_dir_dirents};
 
 int
-meta_xlator_dir_hook (call_frame_t *frame, xlator_t *this, loc_t *loc,
-		      dict_t *xdata)
+meta_xlator_dir_hook(call_frame_t *frame, xlator_t *this, loc_t *loc,
+                     dict_t *xdata)
 {
-	glusterfs_graph_t *graph = NULL;
-	xlator_t *xl = NULL;
+    glusterfs_graph_t *graph = NULL;
+    xlator_t *xl = NULL;
 
-	graph = meta_ctx_get (loc->parent, this);
+    graph = meta_ctx_get(loc->parent, this);
 
-	xl = xlator_search_by_name (graph->first, loc->name);
+    xl = xlator_search_by_name(graph->first, loc->name);
 
-	meta_ctx_set (loc->inode, this, xl);
+    meta_ctx_set(loc->inode, this, xl);
 
-	meta_ops_set (loc->inode, this, &xlator_dir_ops);
+    meta_ops_set(loc->inode, this, &xlator_dir_ops);
 
-	return 0;
+    return 0;
 }
 
-
 int
-meta_master_dir_hook (call_frame_t *frame, xlator_t *this, loc_t *loc,
-		      dict_t *xdata)
+meta_master_dir_hook(call_frame_t *frame, xlator_t *this, loc_t *loc,
+                     dict_t *xdata)
 {
-	meta_ctx_set (loc->inode, this, this->ctx->master);
+    meta_ctx_set(loc->inode, this, this->ctx->master);
 
-	meta_ops_set (loc->inode, this, &xlator_dir_ops);
+    meta_ops_set(loc->inode, this, &xlator_dir_ops);
 
-	return 0;
+    return 0;
 }
diff --git a/xlators/mgmt/glusterd/src/Makefile.am b/xlators/mgmt/glusterd/src/Makefile.am
index 4793f31a034..685beb42d27 100644
--- a/xlators/mgmt/glusterd/src/Makefile.am
+++ b/xlators/mgmt/glusterd/src/Makefile.am
@@ -1,67 +1,79 @@
+if WITH_SERVER
 xlator_LTLIBRARIES = glusterd.la
+endif
+
 xlatordir = $(libdir)/glusterfs/$(PACKAGE_VERSION)/xlator/mgmt
-glusterd_la_CPPFLAGS = $(AM_CPPFLAGS) "-DFILTERDIR=\"$(libdir)/glusterfs/$(PACKAGE_VERSION)/filter\""
-glusterd_la_LDFLAGS = -module -avoid-version
+glusterd_la_CPPFLAGS = $(AM_CPPFLAGS) \
+	-DFILTERDIR=\"$(libdir)/glusterfs/$(PACKAGE_VERSION)/filter\" \
+	-DXLATORDIR=\"$(libdir)/glusterfs/$(PACKAGE_VERSION)/xlator\" \
+	-I$(top_srcdir)/libglusterd/src/
+
+glusterd_la_LDFLAGS = -module $(GF_XLATOR_DEFAULT_LDFLAGS)
 glusterd_la_SOURCES = glusterd.c glusterd-handler.c glusterd-sm.c \
 	glusterd-op-sm.c glusterd-utils.c glusterd-rpc-ops.c \
 	glusterd-store.c glusterd-handshake.c glusterd-pmap.c \
-	glusterd-volgen.c glusterd-rebalance.c glusterd-ganesha.c \
+	glusterd-volgen.c glusterd-rebalance.c \
 	glusterd-quota.c glusterd-bitrot.c glusterd-geo-rep.c \
 	glusterd-replace-brick.c glusterd-log-ops.c \
 	glusterd-volume-ops.c glusterd-brick-ops.c glusterd-mountbroker.c \
 	glusterd-syncop.c glusterd-hooks.c glusterd-volume-set.c \
 	glusterd-locks.c glusterd-snapshot.c glusterd-mgmt-handler.c \
 	glusterd-mgmt.c glusterd-peer-utils.c glusterd-statedump.c \
-        glusterd-snapshot-utils.c glusterd-conn-mgmt.c \
-	glusterd-proc-mgmt.c glusterd-svc-mgmt.c glusterd-shd-svc.c \
+	glusterd-snapshot-utils.c glusterd-conn-mgmt.c \
+	glusterd-proc-mgmt.c glusterd-svc-mgmt.c \
 	glusterd-nfs-svc.c glusterd-quotad-svc.c glusterd-svc-helper.c \
 	glusterd-conn-helper.c glusterd-snapd-svc.c glusterd-snapd-svc-helper.c \
-	glusterd-bitd-svc.c glusterd-scrub-svc.c glusterd-server-quorum.c
-
+	glusterd-bitd-svc.c glusterd-scrub-svc.c glusterd-server-quorum.c \
+	glusterd-reset-brick.c glusterd-shd-svc.c glusterd-shd-svc-helper.c \
+        glusterd-gfproxyd-svc.c glusterd-gfproxyd-svc-helper.c glusterd-ganesha.c \
+	$(CONTRIBDIR)/mount/mntent.c
 
 glusterd_la_LIBADD = $(top_builddir)/libglusterfs/src/libglusterfs.la \
-		     $(top_builddir)/rpc/xdr/src/libgfxdr.la \
-		     $(top_builddir)/rpc/rpc-lib/src/libgfrpc.la \
-		     $(XML_LIBS) -lcrypto $(URCU_LIBS) $(URCU_CDS_LIBS)
-if ENABLE_BD_XLATOR
-glusterd_la_LIBADD += -llvm2app
-endif
+	$(top_builddir)/libglusterd/src/libglusterd.la \
+	$(top_builddir)/rpc/xdr/src/libgfxdr.la \
+	$(top_builddir)/rpc/rpc-lib/src/libgfrpc.la \
+	$(XML_LIBS) -lcrypto $(URCU_LIBS) $(URCU_CDS_LIBS) $(LIB_DL) $(GF_XLATOR_MGNT_LIBADD)
 
 noinst_HEADERS = glusterd.h glusterd-utils.h glusterd-op-sm.h \
 	glusterd-sm.h glusterd-store.h glusterd-mem-types.h \
 	glusterd-pmap.h glusterd-volgen.h glusterd-mountbroker.h \
-	glusterd-syncop.h glusterd-hooks.h glusterd-locks.h \
+	glusterd-syncop.h glusterd-hooks.h glusterd-locks.h glusterd-quota.h \
 	glusterd-mgmt.h glusterd-messages.h glusterd-peer-utils.h \
 	glusterd-statedump.h glusterd-snapshot-utils.h glusterd-geo-rep.h \
 	glusterd-conn-mgmt.h glusterd-conn-helper.h glusterd-proc-mgmt.h \
-	glusterd-svc-mgmt.h glusterd-shd-svc.h glusterd-nfs-svc.h \
+	glusterd-svc-mgmt.h glusterd-nfs-svc.h \
 	glusterd-quotad-svc.h glusterd-svc-helper.h glusterd-snapd-svc.h \
 	glusterd-snapd-svc-helper.h glusterd-rcu.h glusterd-bitd-svc.h \
-	glusterd-scrub-svc.h $(CONTRIBDIR)/userspace-rcu/rculist-extra.h \
-	glusterd-server-quorum.h
+	glusterd-scrub-svc.h glusterd-server-quorum.h glusterd-errno.h \
+        glusterd-shd-svc.h glusterd-shd-svc-helper.h \
+        glusterd-gfproxyd-svc.h glusterd-gfproxyd-svc-helper.h \
+	$(CONTRIBDIR)/userspace-rcu/rculist-extra.h \
+	$(CONTRIBDIR)/mount/mntent_compat.h
 
 AM_CPPFLAGS = $(GF_CPPFLAGS) -I$(top_srcdir)/libglusterfs/src \
-	-I$(rpclibdir) -I$(CONTRIBDIR)/rbtree \
-	-I$(top_srcdir)/rpc/xdr/src -I$(top_srcdir)/rpc/rpc-lib/src \
-	-I$(CONTRIBDIR)/mount \
-	-I$(CONTRIBDIR)/userspace-rcu \
+	-I$(top_srcdir)/rpc/xdr/src -I$(top_builddir)/rpc/xdr/src \
+	-I$(CONTRIBDIR)/rbtree -I$(top_srcdir)/rpc/rpc-lib/src \
+	-I$(CONTRIBDIR)/mount -I$(CONTRIBDIR)/userspace-rcu \
 	-DSBIN_DIR=\"$(sbindir)\" -DDATADIR=\"$(localstatedir)\" \
-	-DGSYNCD_PREFIX=\"$(libexecdir)/glusterfs\" \
-	-DCONFDIR=\"$(sysconfdir)/ganesha\" \
+	-DGSYNCD_PREFIX=\"$(GLUSTERFS_LIBEXECDIR)\" \
+	-DCONFDIR=\"$(localstatedir)/run/gluster/shared_storage/nfs-ganesha\" \
 	-DGANESHA_PREFIX=\"$(libexecdir)/ganesha\" \
-	-DSYNCDAEMON_COMPILE=$(SYNCDAEMON_COMPILE) $(XML_CPPFLAGS)
+	-DSYNCDAEMON_COMPILE=$(SYNCDAEMON_COMPILE) \
+	-I$(top_srcdir)/libglusterd/src/
 
 
-AM_CFLAGS = -Wall $(GF_CFLAGS) $(URCU_CFLAGS) $(URCU_CDS_CFLAGS)
+AM_CFLAGS = -Wall $(GF_CFLAGS) $(URCU_CFLAGS) $(URCU_CDS_CFLAGS) $(XML_CFLAGS)
 
 AM_LDFLAGS = -L$(xlatordir) $(URCU_LIBS) $(URCU_CDS_LIBS)
 
 CLEANFILES =
 
 install-data-hook:
+if WITH_SERVER
 if GF_INSTALL_GLUSTERD_WORKDIR
 	$(mkdir_p) $(DESTDIR)$(GLUSTERD_WORKDIR)
 	(stat $(DESTDIR)$(sysconfdir)/glusterd && \
 	    mv $(DESTDIR)$(sysconfdir)/glusterd $(DESTDIR)$(GLUSTERD_WORKDIR)) || true;
 	(ln -sf $(DESTDIR)$(GLUSTERD_WORKDIR) $(sysconfdir)/glusterd) || true;
 endif
+endif
diff --git a/xlators/mgmt/glusterd/src/glusterd-bitd-svc.c b/xlators/mgmt/glusterd/src/glusterd-bitd-svc.c
index 14c60fd5fba..6adb799b18f 100644
--- a/xlators/mgmt/glusterd/src/glusterd-bitd-svc.c
+++ b/xlators/mgmt/glusterd/src/glusterd-bitd-svc.c
@@ -8,124 +8,199 @@
   cases as published by the Free Software Foundation.
 */
 
-#include "globals.h"
-#include "run.h"
+#include <glusterfs/globals.h>
+#include <glusterfs/run.h>
 #include "glusterd.h"
 #include "glusterd-utils.h"
 #include "glusterd-volgen.h"
 #include "glusterd-bitd-svc.h"
+#include "glusterd-svc-helper.h"
+
+void
+glusterd_bitdsvc_build(glusterd_svc_t *svc)
+{
+    svc->manager = glusterd_bitdsvc_manager;
+    svc->start = glusterd_bitdsvc_start;
+    svc->stop = glusterd_bitdsvc_stop;
+}
 
 int
-glusterd_bitdsvc_init (glusterd_svc_t *svc)
+glusterd_bitdsvc_init(glusterd_svc_t *svc)
 {
-        return glusterd_svc_init (svc, bitd_svc_name,
-                                  glusterd_bitdsvc_manager,
-                                  glusterd_bitdsvc_start,
-                                  glusterd_bitdsvc_stop);
+    return glusterd_svc_init(svc, bitd_svc_name);
 }
 
 static int
-glusterd_bitdsvc_create_volfile ()
+glusterd_bitdsvc_create_volfile()
 {
-        char              filepath[PATH_MAX] = {0,};
-        int               ret                = -1;
-        glusterd_conf_t   *conf              = NULL;
-        dict_t            *mod_dict          = NULL;
-        xlator_t          *this              = NULL;
-
-        this = THIS;
-        conf = this->private;
-        GF_ASSERT (conf);
-
-
-        mod_dict = dict_new ();
-        if (!mod_dict) {
-                gf_log (this->name, GF_LOG_ERROR, "failed to allocate new "
-                        "dict");
-                goto out;
-        }
-
-        ret = dict_set_uint32 (mod_dict, "trusted-client", GF_CLIENT_TRUSTED);
-        if (ret)
-                goto free_dict;
-
-        glusterd_svc_build_volfile_path (bitd_svc_name, conf->workdir,
-                                         filepath, sizeof (filepath));
+    char filepath[PATH_MAX] = {
+        0,
+    };
+    int ret = -1;
+    glusterd_conf_t *conf = NULL;
+    xlator_t *this = NULL;
+
+    this = THIS;
+    conf = this->private;
+    GF_ASSERT(conf);
+
+    glusterd_svc_build_volfile_path(bitd_svc_name, conf->workdir, filepath,
+                                    sizeof(filepath));
+
+    ret = glusterd_create_global_volfile(build_bitd_graph, filepath, NULL);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_VOLFILE_CREATE_FAIL,
+               "Failed to create volfile");
+        goto out;
+    }
 
-        ret = glusterd_create_global_volfile (build_bitd_graph,
-                                              filepath, mod_dict);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "Failed to create volfile");
-                goto free_dict;
-        }
-
-free_dict:
-        dict_unref (mod_dict);
 out:
-        gf_log (this->name, GF_LOG_DEBUG, "Returning %d", ret);
+    gf_msg_debug(this->name, 0, "Returning %d", ret);
 
-        return ret;
+    return ret;
 }
 
 int
-glusterd_bitdsvc_manager (glusterd_svc_t *svc, void *data, int flags)
+glusterd_bitdsvc_manager(glusterd_svc_t *svc, void *data, int flags)
 {
-        int                      ret           = 0;
-        xlator_t                *this          = NULL;
-        glusterd_volinfo_t      *volinfo       = data;
-        glusterd_brickinfo_t    *brickinfo     = NULL;
+    int ret = 0;
+    xlator_t *this = NULL;
 
-        this = THIS;
-        GF_ASSERT (this);
-        GF_ASSERT (volinfo);
+    this = THIS;
+    GF_ASSERT(this);
 
-        if (glusterd_all_volumes_with_bitrot_stopped ()) {
-                ret = svc->stop (svc, SIGTERM);
+    if (!svc->inited) {
+        ret = glusterd_bitdsvc_init(svc);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_BITD_INIT_FAIL,
+                   "Failed to init "
+                   "bitd service");
+            goto out;
         } else {
-                cds_list_for_each_entry (brickinfo, &volinfo->bricks,
-                                         brick_list) {
-                        if (!glusterd_is_local_brick (this, volinfo, brickinfo))
-                                continue;
-
-                        ret = glusterd_bitdsvc_create_volfile ();
-                        if (ret)
-                                goto out;
-
-                        ret = svc->stop (svc, SIGKILL);
-                        if (ret)
-                                goto out;
-
-                        ret = svc->start (svc, flags);
-                        if (ret)
-                                goto out;
-
-                        ret = glusterd_conn_connect (&(svc->conn));
-                        if (ret)
-                                goto out;
-                        break;
-                }
+            svc->inited = _gf_true;
+            gf_msg_debug(this->name, 0,
+                         "BitD service "
+                         "initialized");
         }
+    }
+
+    if (glusterd_should_i_stop_bitd()) {
+        ret = svc->stop(svc, SIGTERM);
+    } else {
+        ret = glusterd_bitdsvc_create_volfile();
+        if (ret)
+            goto out;
+
+        ret = svc->stop(svc, SIGKILL);
+        if (ret)
+            goto out;
+
+        ret = svc->start(svc, flags);
+        if (ret)
+            goto out;
+
+        ret = glusterd_conn_connect(&(svc->conn));
+        if (ret)
+            goto out;
+    }
 
 out:
-        gf_log (THIS->name, GF_LOG_DEBUG, "Returning %d", ret);
+    if (ret)
+        gf_event(EVENT_SVC_MANAGER_FAILED, "svc_name=%s", svc->name);
+
+    gf_msg_debug(THIS->name, 0, "Returning %d", ret);
 
-        return ret;
+    return ret;
 }
 
 int
-glusterd_bitdsvc_start (glusterd_svc_t *svc, int flags)
+glusterd_bitdsvc_start(glusterd_svc_t *svc, int flags)
 {
-        return glusterd_svc_start (svc, flags, NULL);
+    int ret = -1;
+    dict_t *cmdict = NULL;
+
+    cmdict = dict_new();
+    if (!cmdict)
+        goto error_return;
+
+    ret = dict_set_str(cmdict, "cmdarg0", "--global-timer-wheel");
+    if (ret)
+        goto dealloc_dict;
+
+    ret = glusterd_svc_start(svc, flags, cmdict);
+
+dealloc_dict:
+    dict_unref(cmdict);
+error_return:
+    return ret;
 }
 
 int
-glusterd_bitdsvc_stop (glusterd_svc_t *svc, int sig)
+glusterd_bitdsvc_stop(glusterd_svc_t *svc, int sig)
 {
-        return glusterd_svc_stop (svc, sig);
+    return glusterd_svc_stop(svc, sig);
 }
 
 int
-glusterd_bitdsvc_reconfigure ()
+glusterd_bitdsvc_reconfigure()
 {
-        return glusterd_svc_reconfigure (glusterd_bitdsvc_create_volfile);
+    int ret = -1;
+    xlator_t *this = NULL;
+    glusterd_conf_t *priv = NULL;
+    gf_boolean_t identical = _gf_false;
+
+    this = THIS;
+    GF_VALIDATE_OR_GOTO("glusterd", this, out);
+
+    priv = this->private;
+    GF_VALIDATE_OR_GOTO(this->name, priv, out);
+
+    if (glusterd_should_i_stop_bitd())
+        goto manager;
+    /*
+     * Check both OLD and NEW volfiles, if they are SAME by size
+     * and cksum i.e. "character-by-character". If YES, then
+     * NOTHING has been changed, just return.
+     */
+    ret = glusterd_svc_check_volfile_identical(priv->bitd_svc.name,
+                                               build_bitd_graph, &identical);
+    if (ret)
+        goto out;
+    if (identical) {
+        ret = 0;
+        goto out;
+    }
+
+    /*
+     * They are not identical. Find out if the topology is changed
+     * OR just the volume options. If just the options which got
+     * changed, then inform the xlator to reconfigure the options.
+     */
+    identical = _gf_false; /* RESET the FLAG */
+    ret = glusterd_svc_check_topology_identical(priv->bitd_svc.name,
+                                                build_bitd_graph, &identical);
+    if (ret)
+        goto out; /*not able to compare due to some corruption */
+
+    /* Topology is not changed, but just the options. But write the
+     * options to bitd volfile, so that bitd will be reconfigured.
+     */
+    if (identical) {
+        ret = glusterd_bitdsvc_create_volfile();
+        if (ret == 0) { /* Only if above PASSES */
+            ret = glusterd_fetchspec_notify(THIS);
+        }
+        goto out;
+    }
+
+manager:
+    /*
+     * bitd volfile's topology has been changed. bitd server needs
+     * to be RESTARTED to ACT on the changed volfile.
+     */
+    ret = priv->bitd_svc.manager(&(priv->bitd_svc), NULL, PROC_START_NO_WAIT);
+
+out:
+    gf_msg_debug(this ? this->name : "glusterd", 0, "Returning %d", ret);
+    return ret;
 }
diff --git a/xlators/mgmt/glusterd/src/glusterd-bitd-svc.h b/xlators/mgmt/glusterd/src/glusterd-bitd-svc.h
index e6f5d51c7cc..1bff084a9a8 100644
--- a/xlators/mgmt/glusterd/src/glusterd-bitd-svc.h
+++ b/xlators/mgmt/glusterd/src/glusterd-bitd-svc.h
@@ -11,32 +11,30 @@
 #ifndef _GLUSTERD_BITD_SVC_H_
 #define _GLUSTERD_BITD_SVC_H_
 
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
-
 #include "glusterd-svc-mgmt.h"
 
 #define bitd_svc_name "bitd"
 
+void
+glusterd_bitdsvc_build(glusterd_svc_t *svc);
+
 int
-glusterd_bitdsvc_init (glusterd_svc_t *svc);
+glusterd_bitdsvc_init(glusterd_svc_t *svc);
 
 int
-glusterd_bitdsvc_manager (glusterd_svc_t *svc, void *data, int flags);
+glusterd_bitdsvc_manager(glusterd_svc_t *svc, void *data, int flags);
 
 int
-glusterd_bitdsvc_start (glusterd_svc_t *svc, int flags);
+glusterd_bitdsvc_start(glusterd_svc_t *svc, int flags);
 
 int
-glusterd_bitdsvc_stop (glusterd_svc_t *svc, int sig);
+glusterd_bitdsvc_stop(glusterd_svc_t *svc, int sig);
 
 int
-glusterd_bitdsvc_reconfigure ();
+glusterd_bitdsvc_reconfigure();
 
 void
-glusterd_bitdsvc_build_volfile_path (char *server, char *workdir,
-                                     char *volfile, size_t len);
+glusterd_bitdsvc_build_volfile_path(char *server, char *workdir, char *volfile,
+                                    size_t len);
 
 #endif
diff --git a/xlators/mgmt/glusterd/src/glusterd-bitrot.c b/xlators/mgmt/glusterd/src/glusterd-bitrot.c
index 083a714f596..37429fe9214 100644
--- a/xlators/mgmt/glusterd/src/glusterd-bitrot.c
+++ b/xlators/mgmt/glusterd/src/glusterd-bitrot.c
@@ -8,12 +8,7 @@
    cases as published by the Free Software Foundation.
  */
 
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
-
-#include "common-utils.h"
+#include <glusterfs/common-utils.h>
 #include "cli1-xdr.h"
 #include "xdr-generic.h"
 #include "glusterd.h"
@@ -21,485 +16,807 @@
 #include "glusterd-store.h"
 #include "glusterd-utils.h"
 #include "glusterd-volgen.h"
-#include "run.h"
-#include "syscall.h"
-#include "byte-order.h"
-#include "compat-errno.h"
+#include <glusterfs/run.h>
+#include <glusterfs/syscall.h>
+#include <glusterfs/byte-order.h>
+#include <glusterfs/compat-errno.h>
+#include "glusterd-scrub-svc.h"
+#include "glusterd-messages.h"
 
 #include <sys/wait.h>
 #include <dlfcn.h>
 
 const char *gd_bitrot_op_list[GF_BITROT_OPTION_TYPE_MAX] = {
-        [GF_BITROT_OPTION_TYPE_NONE]            = "none",
-        [GF_BITROT_OPTION_TYPE_ENABLE]          = "enable",
-        [GF_BITROT_OPTION_TYPE_DISABLE]         = "disable",
-        [GF_BITROT_OPTION_TYPE_SCRUB_THROTTLE]  = "scrub-throttle",
-        [GF_BITROT_OPTION_TYPE_SCRUB_FREQ]      = "scrub-frequency",
-        [GF_BITROT_OPTION_TYPE_SCRUB]           = "scrub",
+    [GF_BITROT_OPTION_TYPE_NONE] = "none",
+    [GF_BITROT_OPTION_TYPE_ENABLE] = "enable",
+    [GF_BITROT_OPTION_TYPE_DISABLE] = "disable",
+    [GF_BITROT_OPTION_TYPE_SCRUB_THROTTLE] = "scrub-throttle",
+    [GF_BITROT_OPTION_TYPE_SCRUB_FREQ] = "scrub-frequency",
+    [GF_BITROT_OPTION_TYPE_SCRUB] = "scrub",
+    [GF_BITROT_OPTION_TYPE_EXPIRY_TIME] = "expiry-time",
+    [GF_BITROT_OPTION_TYPE_SIGNER_THREADS] = "signer-threads",
 };
 
 int
-__glusterd_handle_bitrot (rpcsvc_request_t *req)
+__glusterd_handle_bitrot(rpcsvc_request_t *req)
 {
-        int32_t                         ret = -1;
-        gf_cli_req                      cli_req = { {0,} };
-        dict_t                         *dict = NULL;
-        glusterd_op_t                   cli_op = GD_OP_BITROT;
-        char                           *volname = NULL;
-        int32_t                         type = 0;
-        char                            msg[2048] = {0,};
-        xlator_t                       *this = NULL;
-        glusterd_conf_t                *conf = NULL;
-
-        GF_ASSERT (req);
-
-        this = THIS;
-        GF_ASSERT (this);
-
-        conf = this->private;
-        GF_ASSERT (conf);
-
-        ret = xdr_to_generic (req->msg[0], &cli_req, (xdrproc_t)xdr_gf_cli_req);
+    int32_t ret = -1;
+    gf_cli_req cli_req = {{
+        0,
+    }};
+    dict_t *dict = NULL;
+    glusterd_op_t cli_op = GD_OP_BITROT;
+    char *volname = NULL;
+    char *scrub = NULL;
+    int32_t type = 0;
+    char msg[256] = {
+        0,
+    };
+    xlator_t *this = NULL;
+    glusterd_conf_t *conf = NULL;
+
+    GF_ASSERT(req);
+
+    this = THIS;
+    GF_ASSERT(this);
+
+    conf = this->private;
+    GF_ASSERT(conf);
+
+    ret = xdr_to_generic(req->msg[0], &cli_req, (xdrproc_t)xdr_gf_cli_req);
+    if (ret < 0) {
+        req->rpc_err = GARBAGE_ARGS;
+        goto out;
+    }
+
+    if (cli_req.dict.dict_len) {
+        /* Unserialize the dictionary */
+        dict = dict_new();
+
+        ret = dict_unserialize(cli_req.dict.dict_val, cli_req.dict.dict_len,
+                               &dict);
         if (ret < 0) {
-                req->rpc_err = GARBAGE_ARGS;
-                goto out;
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_UNSERIALIZE_FAIL,
+                   "failed to "
+                   "unserialize req-buffer to dictionary");
+            snprintf(msg, sizeof(msg),
+                     "Unable to decode the "
+                     "command");
+            goto out;
+        } else {
+            dict->extra_stdfree = cli_req.dict.dict_val;
         }
-
-        if (cli_req.dict.dict_len) {
-                /* Unserialize the dictionary */
-                dict  = dict_new ();
-
-                ret = dict_unserialize (cli_req.dict.dict_val,
-                                        cli_req.dict.dict_len,
-                                        &dict);
-                if (ret < 0) {
-                        gf_log (this->name, GF_LOG_ERROR, "failed to "
-                                "unserialize req-buffer to dictionary");
-                        snprintf (msg, sizeof (msg), "Unable to decode the "
-                                  "command");
-                        goto out;
-                } else {
-                        dict->extra_stdfree = cli_req.dict.dict_val;
-                }
+    }
+
+    ret = dict_get_str(dict, "volname", &volname);
+    if (ret) {
+        snprintf(msg, sizeof(msg), "Unable to get volume name");
+        gf_msg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_GET_FAILED,
+               "Unable to get volume name, "
+               "while handling bitrot command");
+        goto out;
+    }
+
+    ret = dict_get_int32(dict, "type", &type);
+    if (ret) {
+        snprintf(msg, sizeof(msg), "Unable to get type of command");
+        gf_msg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_GET_FAILED,
+               "Unable to get type of cmd, "
+               "while handling bitrot command");
+        goto out;
+    }
+
+    if (conf->op_version < GD_OP_VERSION_3_7_0) {
+        snprintf(msg, sizeof(msg),
+                 "Cannot execute command. The "
+                 "cluster is operating at version %d. Bitrot command "
+                 "%s is unavailable in this version",
+                 conf->op_version, gd_bitrot_op_list[type]);
+        ret = -1;
+        goto out;
+    }
+
+    if (type == GF_BITROT_CMD_SCRUB_STATUS) {
+        /* Backward compatibility handling for scrub status command*/
+        if (conf->op_version < GD_OP_VERSION_3_7_7) {
+            snprintf(msg, sizeof(msg),
+                     "Cannot execute command. "
+                     "The cluster is operating at version %d. "
+                     "Bitrot scrub status command unavailable in "
+                     "this version",
+                     conf->op_version);
+            ret = -1;
+            goto out;
         }
 
-        ret = dict_get_str (dict, "volname", &volname);
+        ret = dict_get_str(dict, "scrub-value", &scrub);
         if (ret) {
-                snprintf (msg, sizeof (msg), "Unable to get volume name");
-                gf_log (this->name, GF_LOG_ERROR, "Unable to get volume name, "
-                        "while handling bitrot command");
-                goto out;
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+                   "Failed to get scrub value.");
+            ret = -1;
+            goto out;
         }
 
-        ret = dict_get_int32 (dict, "type", &type);
+        if (!strncmp(scrub, "status", SLEN("status"))) {
+            ret = glusterd_op_begin_synctask(req, GD_OP_SCRUB_STATUS, dict);
+            goto out;
+        }
+    }
+
+    if (type == GF_BITROT_CMD_SCRUB_ONDEMAND) {
+        /* Backward compatibility handling for scrub status command*/
+        if (conf->op_version < GD_OP_VERSION_3_9_0) {
+            snprintf(msg, sizeof(msg),
+                     "Cannot execute command. "
+                     "The cluster is operating at version %d. "
+                     "Bitrot scrub ondemand command unavailable in "
+                     "this version",
+                     conf->op_version);
+            ret = -1;
+            goto out;
+        }
+
+        ret = dict_get_str(dict, "scrub-value", &scrub);
         if (ret) {
-                snprintf (msg, sizeof (msg), "Unable to get type of command");
-                gf_log (this->name, GF_LOG_ERROR, "Unable to get type of cmd, "
-                        "while handling bitrot command");
-                goto out;
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+                   "Failed to get scrub value.");
+            ret = -1;
+            goto out;
         }
 
-        if (conf->op_version < GD_OP_VERSION_3_7_0) {
-                snprintf (msg, sizeof (msg), "Cannot execute command. The "
-                          "cluster is operating at version %d. Bitrot command "
-                          "%s is unavailable in this version", conf->op_version,
-                          gd_bitrot_op_list[type]);
-                ret = -1;
-                goto out;
+        if (!strncmp(scrub, "ondemand", SLEN("ondemand"))) {
+            ret = glusterd_op_begin_synctask(req, GD_OP_SCRUB_ONDEMAND, dict);
+            goto out;
         }
+    }
 
-        ret = glusterd_op_begin_synctask (req, GD_OP_BITROT, dict);
+    ret = glusterd_op_begin_synctask(req, GD_OP_BITROT, dict);
 
 out:
-        if (ret) {
-                if (msg[0] == '\0')
-                        snprintf (msg, sizeof (msg), "Bitrot operation failed");
-                ret = glusterd_op_send_cli_response (cli_op, ret, 0, req,
-                                                     dict, msg);
-        }
+    if (ret) {
+        if (msg[0] == '\0')
+            snprintf(msg, sizeof(msg), "Bitrot operation failed");
+        ret = glusterd_op_send_cli_response(cli_op, ret, 0, req, dict, msg);
+    }
 
-        return ret;
+    return ret;
 }
 
 int
-glusterd_handle_bitrot (rpcsvc_request_t *req)
+glusterd_handle_bitrot(rpcsvc_request_t *req)
 {
-        return glusterd_big_locked_handler (req, __glusterd_handle_bitrot);
+    return glusterd_big_locked_handler(req, __glusterd_handle_bitrot);
 }
 
 static int
-glusterd_bitrot_scrub_throttle (glusterd_volinfo_t *volinfo, dict_t *dict,
-                                char *key, char **op_errstr)
+glusterd_bitrot_scrub_throttle(glusterd_volinfo_t *volinfo, dict_t *dict,
+                               char *key, char **op_errstr)
 {
-        int32_t        ret                  = -1;
-        char           *scrub_throttle      = NULL;
-        char           *option              = NULL;
-        xlator_t       *this                = NULL;
-
-        this = THIS;
-        GF_ASSERT (this);
-
-        ret = dict_get_str (dict, "scrub-throttle-value", &scrub_throttle);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "Unable to fetch scrub-"
-                        "throttle value");
-                goto out;
-        }
-
-        option = gf_strdup (scrub_throttle);
-        ret = dict_set_dynstr (volinfo->dict, key, option);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "Failed to set option %s",
-                        key);
-                goto out;
-        }
+    int32_t ret = -1;
+    char *scrub_throttle = NULL;
+    char *option = NULL;
+    xlator_t *this = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+
+    ret = dict_get_str(dict, "scrub-throttle-value", &scrub_throttle);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_GET_FAILED,
+               "Unable to fetch scrub-"
+               "throttle value");
+        goto out;
+    }
+
+    option = gf_strdup(scrub_throttle);
+    ret = dict_set_dynstr(volinfo->dict, key, option);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_SET_FAILED,
+               "Failed to set option %s", key);
+        goto out;
+    }
+
+    ret = glusterd_scrubsvc_reconfigure();
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_SCRUBSVC_RECONF_FAIL,
+               "Failed to reconfigure scrub "
+               "services");
+        goto out;
+    }
 
 out:
-        return ret;
+    return ret;
 }
 
 static int
-glusterd_bitrot_scrub_freq (glusterd_volinfo_t *volinfo, dict_t *dict,
-                            char *key, char **op_errstr)
+glusterd_bitrot_scrub_freq(glusterd_volinfo_t *volinfo, dict_t *dict, char *key,
+                           char **op_errstr)
 {
-        int32_t        ret                  = -1;
-        char           *scrub_freq          = NULL;
-        xlator_t       *this                = NULL;
-        char           *option              = NULL;
-
-        this = THIS;
-        GF_ASSERT (this);
-
-        ret = dict_get_str (dict, "scrub-frequency-value", &scrub_freq);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "Unable to fetch scrub-"
-                        "freq value");
-                goto out;
-        }
-
-        option = gf_strdup (scrub_freq);
-        ret = dict_set_dynstr (volinfo->dict, key, option);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "Failed to set option %s",
-                        key);
-                goto out;
-        }
+    int32_t ret = -1;
+    char *scrub_freq = NULL;
+    xlator_t *this = NULL;
+    char *option = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+
+    ret = dict_get_str(dict, "scrub-frequency-value", &scrub_freq);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_GET_FAILED,
+               "Unable to fetch scrub-"
+               "freq value");
+        goto out;
+    }
+
+    option = gf_strdup(scrub_freq);
+    ret = dict_set_dynstr(volinfo->dict, key, option);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_SET_FAILED,
+               "Failed to set option %s", key);
+        goto out;
+    }
+
+    ret = glusterd_scrubsvc_reconfigure();
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_SCRUBSVC_RECONF_FAIL,
+               "Failed to reconfigure scrub "
+               "services");
+        goto out;
+    }
 
 out:
-        return ret;
+    return ret;
 }
 
 static int
-glusterd_bitrot_scrub (glusterd_volinfo_t *volinfo, dict_t *dict,
-                       char *key, char **op_errstr)
+glusterd_bitrot_scrub(glusterd_volinfo_t *volinfo, dict_t *dict, char *key,
+                      char **op_errstr)
 {
-        int32_t        ret                  = -1;
-        char           *scrub_value         = NULL;
-        xlator_t       *this                = NULL;
-        char           *option              = NULL;
-
-        this = THIS;
-        GF_ASSERT (this);
-
-        ret = dict_get_str (dict, "scrub-value", &scrub_value);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "Unable to fetch scrub"
-                        "value");
-                goto out;
-        }
-
-        option = gf_strdup (scrub_value);
-        ret = dict_set_dynstr (volinfo->dict, key, option);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "Failed to set option %s",
-                        key);
-                goto out;
-        }
+    int32_t ret = -1;
+    char *scrub_value = NULL;
+    xlator_t *this = NULL;
+    char *option = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+
+    ret = dict_get_str(dict, "scrub-value", &scrub_value);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+               "Unable to fetch scrub"
+               "value");
+        goto out;
+    }
+
+    if (!strcmp(scrub_value, "resume")) {
+        option = gf_strdup("Active");
+    } else {
+        option = gf_strdup(scrub_value);
+    }
+
+    ret = dict_set_dynstr(volinfo->dict, key, option);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_SET_FAILED,
+               "Failed to set option %s", key);
+        goto out;
+    }
+
+    ret = glusterd_scrubsvc_reconfigure();
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_SCRUBSVC_RECONF_FAIL,
+               "Failed to reconfigure scrub "
+               "services");
+        goto out;
+    }
 
 out:
-        return ret;
+    return ret;
 }
 
 static int
-glusterd_bitrot_enable (glusterd_volinfo_t *volinfo, char **op_errstr)
+glusterd_bitrot_expiry_time(glusterd_volinfo_t *volinfo, dict_t *dict,
+                            char *key, char **op_errstr)
 {
-        int32_t         ret             = -1;
-        xlator_t        *this           = NULL;
-
-        this = THIS;
-        GF_ASSERT (this);
-
-        GF_VALIDATE_OR_GOTO (this->name, volinfo, out);
-        GF_VALIDATE_OR_GOTO (this->name, op_errstr, out);
-
-        if (glusterd_is_volume_started (volinfo) == 0) {
-                *op_errstr = gf_strdup ("Volume is stopped, start volume "
-                                        "to enable bitrot.");
-                ret = -1;
-                goto out;
-        }
+    int32_t ret = -1;
+    uint32_t expiry_time = 0;
+    xlator_t *this = NULL;
+    char dkey[32] = {
+        0,
+    };
+
+    this = THIS;
+    GF_ASSERT(this);
+
+    ret = dict_get_uint32(dict, "expiry-time", &expiry_time);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_GET_FAILED,
+               "Unable to get bitrot expiry"
+               " timer value.");
+        goto out;
+    }
+
+    snprintf(dkey, sizeof(dkey), "%d", expiry_time);
+
+    ret = dict_set_dynstr_with_alloc(volinfo->dict, key, dkey);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_SET_FAILED,
+               "Failed to set option %s", key);
+        goto out;
+    }
+
+    ret = glusterd_bitdsvc_reconfigure();
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_BITDSVC_RECONF_FAIL,
+               "Failed to reconfigure bitrot"
+               "services");
+        goto out;
+    }
+out:
+    return ret;
+}
 
-        ret = glusterd_is_bitrot_enabled (volinfo);
-        if (ret) {
-                *op_errstr = gf_strdup ("Bitrot is already enabled");
-                ret = -1;
-                goto out;
+static gf_boolean_t
+is_bitd_configure_noop(xlator_t *this, glusterd_volinfo_t *volinfo)
+{
+    gf_boolean_t noop = _gf_true;
+    glusterd_brickinfo_t *brickinfo = NULL;
+
+    if (!glusterd_is_bitrot_enabled(volinfo))
+        goto out;
+    else if (volinfo->status != GLUSTERD_STATUS_STARTED)
+        goto out;
+    else {
+        cds_list_for_each_entry(brickinfo, &volinfo->bricks, brick_list)
+        {
+            if (!glusterd_is_local_brick(this, volinfo, brickinfo))
+                continue;
+            noop = _gf_false;
+            return noop;
         }
+    }
+out:
+    return noop;
+}
 
-        ret = dict_set_dynstr_with_alloc (volinfo->dict, VKEY_FEATURES_BITROT,
-                                          "on");
+static int
+glusterd_bitrot_signer_threads(glusterd_volinfo_t *volinfo, dict_t *dict,
+                               char *key, char **op_errstr)
+{
+    int32_t ret = -1;
+    uint32_t signer_th_count = 0;
+    uint32_t existing_th_count = 0;
+    xlator_t *this = NULL;
+    glusterd_conf_t *priv = NULL;
+    char dkey[32] = {
+        0,
+    };
+
+    this = THIS;
+    GF_ASSERT(this);
+
+    priv = this->private;
+    GF_VALIDATE_OR_GOTO(this->name, priv, out);
+
+    ret = dict_get_uint32(dict, "signer-threads", &signer_th_count);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_GET_FAILED,
+               "Unable to get bitrot signer thread count.");
+        goto out;
+    }
+
+    ret = dict_get_uint32(volinfo->dict, key, &existing_th_count);
+    if (ret == 0 && signer_th_count == existing_th_count) {
+        goto out;
+    }
+
+    snprintf(dkey, sizeof(dkey), "%d", signer_th_count);
+    ret = dict_set_dynstr_with_alloc(volinfo->dict, key, dkey);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_SET_FAILED,
+               "Failed to set option %s", key);
+        goto out;
+    }
+
+    if (!is_bitd_configure_noop(this, volinfo)) {
+        ret = priv->bitd_svc.manager(&(priv->bitd_svc), NULL,
+                                     PROC_START_NO_WAIT);
         if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "dict set failed");
-                goto out;
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_BITDSVC_RECONF_FAIL,
+                   "Failed to reconfigure bitrot services");
+            goto out;
         }
-
-        ret = 0;
+    }
 out:
-        if (ret && op_errstr && !*op_errstr)
-                gf_asprintf (op_errstr, "Enabling bitrot on volume %s has been "
-                             "unsuccessful", volinfo->volname);
-        return ret;
+    return ret;
 }
 
 static int
-glusterd_bitrot_disable (glusterd_volinfo_t *volinfo, char **op_errstr)
+glusterd_bitrot_enable(glusterd_volinfo_t *volinfo, char **op_errstr)
 {
-        int32_t           ret            = -1;
-        xlator_t          *this          = NULL;
-
-        this = THIS;
-        GF_VALIDATE_OR_GOTO ("glusterd", this, out);
-
-        GF_VALIDATE_OR_GOTO (this->name, volinfo, out);
-        GF_VALIDATE_OR_GOTO (this->name, op_errstr, out);
-
-        ret = dict_set_dynstr_with_alloc (volinfo->dict, VKEY_FEATURES_BITROT,
-                                          "off");
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "dict set failed");
-                goto out;
-        }
+    int32_t ret = -1;
+    xlator_t *this = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+
+    GF_VALIDATE_OR_GOTO(this->name, volinfo, out);
+    GF_VALIDATE_OR_GOTO(this->name, op_errstr, out);
+
+    if (glusterd_is_volume_started(volinfo) == 0) {
+        *op_errstr = gf_strdup(
+            "Volume is stopped, start volume "
+            "to enable bitrot.");
+        ret = -1;
+        goto out;
+    }
+
+    ret = glusterd_is_bitrot_enabled(volinfo);
+    if (ret) {
+        *op_errstr = gf_strdup("Bitrot is already enabled");
+        ret = -1;
+        goto out;
+    }
+
+    ret = dict_set_dynstr_with_alloc(volinfo->dict, VKEY_FEATURES_BITROT, "on");
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_SET_FAILED,
+               "dict set failed");
+        goto out;
+    }
+
+    /*Once bitrot is enable scrubber should be in Active state*/
+    ret = dict_set_dynstr_with_alloc(volinfo->dict, "features.scrub", "Active");
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_SET_FAILED,
+               "Failed to set option "
+               "features.scrub value");
+        goto out;
+    }
+
+    ret = 0;
+out:
+    if (ret && op_errstr && !*op_errstr)
+        gf_asprintf(op_errstr,
+                    "Enabling bitrot on volume %s has been "
+                    "unsuccessful",
+                    volinfo->volname);
+    return ret;
+}
 
-        ret = 0;
+static int
+glusterd_bitrot_disable(glusterd_volinfo_t *volinfo, char **op_errstr)
+{
+    int32_t ret = -1;
+    xlator_t *this = NULL;
+
+    this = THIS;
+    GF_VALIDATE_OR_GOTO("glusterd", this, out);
+
+    GF_VALIDATE_OR_GOTO(this->name, volinfo, out);
+    GF_VALIDATE_OR_GOTO(this->name, op_errstr, out);
+
+    ret = dict_set_dynstr_with_alloc(volinfo->dict, VKEY_FEATURES_BITROT,
+                                     "off");
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_SET_FAILED,
+               "dict set failed");
+        goto out;
+    }
+
+    /*Once bitrot disabled scrubber should be Inactive state*/
+    ret = dict_set_dynstr_with_alloc(volinfo->dict, "features.scrub",
+                                     "Inactive");
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_SET_FAILED,
+               "Failed to set "
+               "features.scrub value");
+        goto out;
+    }
+
+    ret = 0;
 out:
-        if (ret && op_errstr && !*op_errstr)
-                gf_asprintf (op_errstr, "Disabling bitrot on volume %s has "
-                             "been unsuccessful", volinfo->volname);
-        return ret;
+    if (ret && op_errstr && !*op_errstr)
+        gf_asprintf(op_errstr,
+                    "Disabling bitrot on volume %s has "
+                    "been unsuccessful",
+                    volinfo->volname);
+    return ret;
 }
 
 gf_boolean_t
-glusterd_all_volumes_with_bitrot_stopped ()
+glusterd_should_i_stop_bitd()
 {
-        glusterd_conf_t *conf = THIS->private;
-        glusterd_volinfo_t *volinfo = NULL;
-        gf_boolean_t        stopped = _gf_true;
-
-        cds_list_for_each_entry (volinfo, &conf->volumes, vol_list) {
-                if (!glusterd_is_bitrot_enabled (volinfo))
-                        continue;
-                else if (volinfo->status != GLUSTERD_STATUS_STARTED)
-                        continue;
-                else {
-                        stopped = _gf_false;
-                        break;
-                }
+    glusterd_conf_t *conf = THIS->private;
+    glusterd_volinfo_t *volinfo = NULL;
+    gf_boolean_t stopped = _gf_true;
+    glusterd_brickinfo_t *brickinfo = NULL;
+    xlator_t *this = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+
+    cds_list_for_each_entry(volinfo, &conf->volumes, vol_list)
+    {
+        if (!glusterd_is_bitrot_enabled(volinfo))
+            continue;
+        else if (volinfo->status != GLUSTERD_STATUS_STARTED)
+            continue;
+        else {
+            cds_list_for_each_entry(brickinfo, &volinfo->bricks, brick_list)
+            {
+                if (!glusterd_is_local_brick(this, volinfo, brickinfo))
+                    continue;
+                stopped = _gf_false;
+                return stopped;
+            }
+
+            /* Before stopping bitrot/scrubber daemon check
+             * other volume also whether respective volume
+             * host a brick from this node or not.*/
+            continue;
         }
+    }
 
-        return stopped;
+    return stopped;
 }
 
 static int
-glusterd_manage_bitrot (int opcode, glusterd_volinfo_t *volinfo)
+glusterd_manage_bitrot(int opcode)
 {
-        int              ret   = -1;
-        xlator_t         *this = NULL;
-        glusterd_conf_t  *priv = NULL;
+    int ret = -1;
+    xlator_t *this = NULL;
+    glusterd_conf_t *priv = NULL;
 
-        this = THIS;
-        GF_ASSERT (this);
+    this = THIS;
+    GF_ASSERT(this);
 
-        priv = this->private;
-        GF_ASSERT (priv);
+    priv = this->private;
+    GF_ASSERT(priv);
 
-        switch (opcode) {
+    switch (opcode) {
         case GF_BITROT_OPTION_TYPE_ENABLE:
         case GF_BITROT_OPTION_TYPE_DISABLE:
-                ret = priv->bitd_svc.manager (&(priv->bitd_svc),
-                                                volinfo, PROC_START_NO_WAIT);
-                if (ret)
-                        break;
-                ret = priv->scrub_svc.manager (&(priv->scrub_svc), NULL,
-                                               PROC_START_NO_WAIT);
+            ret = priv->bitd_svc.manager(&(priv->bitd_svc), NULL,
+                                         PROC_START_NO_WAIT);
+            if (ret)
                 break;
+            ret = priv->scrub_svc.manager(&(priv->scrub_svc), NULL,
+                                          PROC_START_NO_WAIT);
+            break;
         default:
-                ret = 0;
-                break;
-        }
-
-        return ret;
+            ret = 0;
+            break;
+    }
 
+    return ret;
 }
 
 int
-glusterd_op_bitrot (dict_t *dict, char **op_errstr, dict_t *rsp_dict)
+glusterd_op_bitrot(dict_t *dict, char **op_errstr, dict_t *rsp_dict)
 {
-        glusterd_volinfo_t     *volinfo      = NULL;
-        int32_t                 ret          = -1;
-        char                   *volname      = NULL;
-        int                     type         = -1;
-        glusterd_conf_t        *priv         = NULL;
-        xlator_t               *this         = NULL;
-
-        GF_ASSERT (dict);
-        GF_ASSERT (op_errstr);
-
-        this = THIS;
-        GF_ASSERT (this);
-        priv = this->private;
-        GF_ASSERT (priv);
-
-        ret = dict_get_str (dict, "volname", &volname);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "Unable to get volume name");
-                goto out;
-        }
-
-        ret = glusterd_volinfo_find (volname, &volinfo);
-        if (ret) {
-                gf_asprintf (op_errstr, FMTSTR_CHECK_VOL_EXISTS, volname);
-                goto out;
-        }
-
-        ret = dict_get_int32 (dict, "type", &type);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "Unable to get type from "
-                        "dict");
-                goto out;
-        }
-
-        switch (type) {
+    glusterd_volinfo_t *volinfo = NULL;
+    int32_t ret = -1;
+    char *volname = NULL;
+    int type = -1;
+    glusterd_conf_t *priv = NULL;
+    xlator_t *this = NULL;
+
+    GF_ASSERT(dict);
+    GF_ASSERT(op_errstr);
+
+    this = THIS;
+    GF_ASSERT(this);
+    priv = this->private;
+    GF_ASSERT(priv);
+
+    ret = dict_get_str(dict, "volname", &volname);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_GET_FAILED,
+               "Unable to get volume name");
+        goto out;
+    }
+
+    ret = glusterd_volinfo_find(volname, &volinfo);
+    if (ret) {
+        gf_asprintf(op_errstr, FMTSTR_CHECK_VOL_EXISTS, volname);
+        goto out;
+    }
+
+    ret = dict_get_int32(dict, "type", &type);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_GET_FAILED,
+               "Unable to get type from "
+               "dict");
+        goto out;
+    }
+
+    switch (type) {
         case GF_BITROT_OPTION_TYPE_ENABLE:
-                ret = glusterd_bitrot_enable (volinfo, op_errstr);
-                if (ret < 0)
-                        goto out;
-                break;
+            ret = glusterd_bitrot_enable(volinfo, op_errstr);
+            if (ret < 0)
+                goto out;
+            break;
 
         case GF_BITROT_OPTION_TYPE_DISABLE:
-                ret = glusterd_bitrot_disable (volinfo, op_errstr);
-                if (ret < 0)
-                        goto out;
+            ret = glusterd_bitrot_disable(volinfo, op_errstr);
+            if (ret < 0)
+                goto out;
 
-                break;
+            break;
 
         case GF_BITROT_OPTION_TYPE_SCRUB_THROTTLE:
-                ret = glusterd_bitrot_scrub_throttle (volinfo, dict,
-                                                      "features.scrub-throttle",
-                                                      op_errstr);
+            ret = glusterd_bitrot_scrub_throttle(
+                volinfo, dict, "features.scrub-throttle", op_errstr);
+            if (ret)
                 goto out;
+            break;
 
         case GF_BITROT_OPTION_TYPE_SCRUB_FREQ:
-                ret = glusterd_bitrot_scrub_freq (volinfo, dict,
-                                                  "features.scrub-freq",
-                                                  op_errstr);
+            ret = glusterd_bitrot_scrub_freq(volinfo, dict,
+                                             "features.scrub-freq", op_errstr);
+            if (ret)
                 goto out;
+            break;
 
         case GF_BITROT_OPTION_TYPE_SCRUB:
-                ret = glusterd_bitrot_scrub (volinfo, dict, "features.scrub",
-                                             op_errstr);
+            ret = glusterd_bitrot_scrub(volinfo, dict, "features.scrub",
+                                        op_errstr);
+            if (ret)
                 goto out;
+            break;
 
-        default:
-                gf_asprintf (op_errstr, "Bitrot command failed. Invalid "
-                             "opcode");
-                ret = -1;
+        case GF_BITROT_OPTION_TYPE_EXPIRY_TIME:
+            ret = glusterd_bitrot_expiry_time(
+                volinfo, dict, "features.expiry-time", op_errstr);
+            if (ret)
                 goto out;
-        }
+            break;
 
-        ret = glusterd_manage_bitrot (type, volinfo);
-        if (ret)
+        case GF_BITROT_OPTION_TYPE_SIGNER_THREADS:
+            ret = glusterd_bitrot_signer_threads(
+                volinfo, dict, "features.signer-threads", op_errstr);
+            if (ret)
                 goto out;
+            break;
 
-        ret = glusterd_create_volfiles_and_notify_services (volinfo);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "Unable to re-create "
-                        "volfiles");
-                ret = -1;
-                goto out;
-        }
+        case GF_BITROT_CMD_SCRUB_STATUS:
+        case GF_BITROT_CMD_SCRUB_ONDEMAND:
+            break;
 
-        ret = glusterd_store_volinfo (volinfo,
-                                      GLUSTERD_VOLINFO_VER_AC_INCREMENT);
-        if (ret) {
-                gf_log (this->name, GF_LOG_DEBUG, "Failed to store volinfo for "
-                        "bitrot");
-                goto out;
-        }
+        default:
+            gf_asprintf(op_errstr,
+                        "Bitrot command failed. Invalid "
+                        "opcode");
+            ret = -1;
+            goto out;
+    }
+
+    ret = glusterd_manage_bitrot(type);
+    if (ret)
+        goto out;
+
+    ret = glusterd_create_volfiles_and_notify_services(volinfo);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_VOLFILE_CREATE_FAIL,
+               "Unable to re-create "
+               "volfiles");
+        ret = -1;
+        goto out;
+    }
+
+    ret = glusterd_store_volinfo(volinfo, GLUSTERD_VOLINFO_VER_AC_INCREMENT);
+    if (ret) {
+        gf_msg_debug(this->name, 0,
+                     "Failed to store volinfo for "
+                     "bitrot");
+        goto out;
+    }
 
 out:
-        return ret;
+    return ret;
 }
 
 int
-glusterd_op_stage_bitrot (dict_t *dict, char **op_errstr, dict_t *rsp_dict)
+glusterd_op_stage_bitrot(dict_t *dict, char **op_errstr, dict_t *rsp_dict)
 {
-        int                  ret             = 0;
-        char                *volname         = NULL;
-        int                  type            = 0;
-        xlator_t            *this            = NULL;
-        glusterd_conf_t     *priv            = NULL;
-        glusterd_volinfo_t  *volinfo         = NULL;
-
-        this = THIS;
-        GF_ASSERT (this);
-        priv = this->private;
-        GF_ASSERT (priv);
-
-        GF_ASSERT (dict);
-        GF_ASSERT (op_errstr);
-
-        ret = dict_get_str (dict, "volname", &volname);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "Unable to get volume name");
+    int ret = 0;
+    char *volname = NULL;
+    char *scrub_cmd = NULL;
+    char *scrub_cmd_from_dict = NULL;
+    char msg[2048] = {
+        0,
+    };
+    int type = 0;
+    xlator_t *this = NULL;
+    glusterd_conf_t *priv = NULL;
+    glusterd_volinfo_t *volinfo = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+    priv = this->private;
+    GF_ASSERT(priv);
+
+    GF_ASSERT(dict);
+    GF_ASSERT(op_errstr);
+
+    ret = dict_get_str(dict, "volname", &volname);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_GET_FAILED,
+               "Unable to get volume name");
+        goto out;
+    }
+
+    ret = glusterd_volinfo_find(volname, &volinfo);
+    if (ret) {
+        gf_asprintf(op_errstr, FMTSTR_CHECK_VOL_EXISTS, volname);
+        goto out;
+    }
+
+    if (!glusterd_is_volume_started(volinfo)) {
+        *op_errstr = gf_strdup(
+            "Volume is stopped, start volume "
+            "before executing bit rot command.");
+        ret = -1;
+        goto out;
+    }
+
+    ret = dict_get_int32(dict, "type", &type);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_GET_FAILED,
+               "Unable to get type for "
+               "operation");
+
+        *op_errstr = gf_strdup(
+            "Staging stage failed for bitrot "
+            "operation.");
+        goto out;
+    }
+
+    if ((GF_BITROT_OPTION_TYPE_ENABLE != type) &&
+        (glusterd_is_bitrot_enabled(volinfo) == 0)) {
+        ret = -1;
+        gf_asprintf(op_errstr, "Bitrot is not enabled on volume %s", volname);
+        goto out;
+    }
+
+    if ((GF_BITROT_OPTION_TYPE_SCRUB == type)) {
+        ret = dict_get_str(volinfo->dict, "features.scrub",
+                           &scrub_cmd_from_dict);
+        if (!ret) {
+            ret = dict_get_str(dict, "scrub-value", &scrub_cmd);
+            if (ret) {
+                gf_msg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_GET_FAILED,
+                       "Unable to "
+                       "get scrub-value");
+                *op_errstr = gf_strdup(
+                    "Staging failed for "
+                    "bitrot operation. "
+                    "Please check log file"
+                    " for more details.");
                 goto out;
-        }
-
-        ret = glusterd_volinfo_find (volname, &volinfo);
-        if (ret) {
-                gf_asprintf (op_errstr, FMTSTR_CHECK_VOL_EXISTS, volname);
-                goto out;
-        }
-
-        if (!glusterd_is_volume_started (volinfo)) {
-                *op_errstr = gf_strdup ("Volume is stopped, start volume "
-                                        "before executing bit rot command.");
+            }
+            /* If scrubber is resume then value of scrubber will be
+             * "Active" in the dictionary. */
+            if (!strcmp(scrub_cmd_from_dict, scrub_cmd) ||
+                (!strncmp("Active", scrub_cmd_from_dict, SLEN("Active")) &&
+                 !strncmp("resume", scrub_cmd, SLEN("resume")))) {
+                snprintf(msg, sizeof(msg),
+                         "Scrub is already"
+                         " %sd for volume %s",
+                         scrub_cmd, volinfo->volname);
+                *op_errstr = gf_strdup(msg);
                 ret = -1;
                 goto out;
+            }
         }
+        ret = 0;
+    }
 
-        ret = dict_get_int32 (dict, "type", &type);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "Unable to get type for "
-                        "operation");
-
-                *op_errstr = gf_strdup ("Staging stage failed for bitrot "
-                                        "operation.");
-                goto out;
-        }
-
-
-        if ((GF_BITROT_OPTION_TYPE_ENABLE != type) &&
-            (glusterd_is_bitrot_enabled (volinfo) == 0)) {
-                ret = -1;
-                gf_asprintf (op_errstr, "Bitrot is not enabled on volume %s",
-                             volname);
-                goto out;
-        }
-
- out:
-        if (ret && op_errstr && *op_errstr)
-                gf_log (this->name, GF_LOG_ERROR, "%s", *op_errstr);
-        gf_log (this->name, GF_LOG_DEBUG, "Returning %d", ret);
+out:
+    if (ret && op_errstr && *op_errstr)
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_OP_STAGE_BITROT_FAIL, "%s",
+               *op_errstr);
+    gf_msg_debug(this->name, 0, "Returning %d", ret);
 
-        return ret;
+    return ret;
 }
diff --git a/xlators/mgmt/glusterd/src/glusterd-brick-ops.c b/xlators/mgmt/glusterd/src/glusterd-brick-ops.c
index 165622f87ef..e56cd0e6c74 100644
--- a/xlators/mgmt/glusterd/src/glusterd-brick-ops.c
+++ b/xlators/mgmt/glusterd/src/glusterd-brick-ops.c
@@ -7,22 +7,21 @@
    later), or the GNU General Public License, version 2 (GPLv2), in all
    cases as published by the Free Software Foundation.
 */
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
-
-#include "common-utils.h"
+#include <glusterfs/common-utils.h>
 #include "cli1-xdr.h"
 #include "xdr-generic.h"
 #include "glusterd.h"
 #include "glusterd-op-sm.h"
 #include "glusterd-geo-rep.h"
 #include "glusterd-store.h"
+#include "glusterd-mgmt.h"
 #include "glusterd-utils.h"
 #include "glusterd-volgen.h"
 #include "glusterd-svc-helper.h"
-#include "run.h"
+#include "glusterd-messages.h"
+#include "glusterd-server-quorum.h"
+#include <glusterfs/run.h>
+#include <glusterfs/syscall.h>
 #include <sys/signal.h>
 
 /* misc */
@@ -32,1734 +31,2064 @@
    how many of the given bricks are added. other argument are self-
    descriptive. */
 int
-add_brick_at_right_order (glusterd_brickinfo_t *brickinfo,
-                          glusterd_volinfo_t *volinfo, int count,
-                          int32_t stripe_cnt, int32_t replica_cnt)
+add_brick_at_right_order(glusterd_brickinfo_t *brickinfo,
+                         glusterd_volinfo_t *volinfo, int count,
+                         int32_t stripe_cnt, int32_t replica_cnt)
 {
-        int                   idx     = 0;
-        int                   i       = 0;
-        int                   sub_cnt = 0;
-        glusterd_brickinfo_t *brick   = NULL;
-
-        /* The complexity of the function is in deciding at which index
-           to add new brick. Even though it can be defined with a complex
-           single formula for all volume, it is separated out to make it
-           more readable */
-        if (stripe_cnt) {
-                /* common formula when 'stripe_count' is set */
-                /* idx = ((count / ((stripe_cnt * volinfo->replica_count) -
-                   volinfo->dist_leaf_count)) * volinfo->dist_leaf_count) +
-                   (count + volinfo->dist_leaf_count);
-                */
-
-                sub_cnt = volinfo->dist_leaf_count;
-
-                idx = ((count / ((stripe_cnt * volinfo->replica_count) -
-                                 sub_cnt)) * sub_cnt) +
-                        (count + sub_cnt);
-
-                goto insert_brick;
-        }
-
-        /* replica count is set */
-        /* common formula when 'replica_count' is set */
-        /* idx = ((count / (replica_cnt - existing_replica_count)) *
-           existing_replica_count) +
-           (count + existing_replica_count);
+    int idx = 0;
+    int i = 0;
+    int sub_cnt = 0;
+    glusterd_brickinfo_t *brick = NULL;
+
+    /* The complexity of the function is in deciding at which index
+       to add new brick. Even though it can be defined with a complex
+       single formula for all volume, it is separated out to make it
+       more readable */
+    if (stripe_cnt) {
+        /* common formula when 'stripe_count' is set */
+        /* idx = ((count / ((stripe_cnt * volinfo->replica_count) -
+           volinfo->dist_leaf_count)) * volinfo->dist_leaf_count) +
+           (count + volinfo->dist_leaf_count);
         */
 
-        sub_cnt = volinfo->replica_count;
-        idx = (count / (replica_cnt - sub_cnt) * sub_cnt) +
-                (count + sub_cnt);
-
-insert_brick:
-        i = 0;
-        cds_list_for_each_entry (brick, &volinfo->bricks, brick_list) {
-                i++;
-                if (i < idx)
-                        continue;
-                gf_log (THIS->name, GF_LOG_DEBUG, "brick:%s index=%d, count=%d",
-                        brick->path, idx, count);
-
-                cds_list_add (&brickinfo->brick_list, &brick->brick_list);
-                break;
-        }
+        sub_cnt = volinfo->dist_leaf_count;
 
-        return 0;
-}
+        idx = ((count / ((stripe_cnt * volinfo->replica_count) - sub_cnt)) *
+               sub_cnt) +
+              (count + sub_cnt);
 
+        goto insert_brick;
+    }
 
-static int
-gd_addbr_validate_stripe_count (glusterd_volinfo_t *volinfo, int stripe_count,
-                                int total_bricks, int *type, char *err_str,
-                                size_t err_len)
-{
-        int ret = -1;
+    /* replica count is set */
+    /* common formula when 'replica_count' is set */
+    /* idx = ((count / (replica_cnt - existing_replica_count)) *
+       existing_replica_count) +
+       (count + existing_replica_count);
+    */
 
-        switch (volinfo->type) {
-        case GF_CLUSTER_TYPE_NONE:
-                if ((volinfo->brick_count * stripe_count) == total_bricks) {
-                        /* Change the volume type */
-                        *type = GF_CLUSTER_TYPE_STRIPE;
-                        gf_log (THIS->name, GF_LOG_INFO,
-                                "Changing the type of volume %s from "
-                                "'distribute' to 'stripe'", volinfo->volname);
-                        ret = 0;
-                        goto out;
-                } else {
-                        snprintf (err_str, err_len, "Incorrect number of "
-                                  "bricks (%d) supplied for stripe count (%d).",
-                                  (total_bricks - volinfo->brick_count),
-                                  stripe_count);
-                        gf_log (THIS->name, GF_LOG_ERROR, "%s", err_str);
-                        goto out;
-                }
-                break;
-        case GF_CLUSTER_TYPE_REPLICATE:
-                if (!(total_bricks % (volinfo->replica_count * stripe_count))) {
-                        /* Change the volume type */
-                        *type = GF_CLUSTER_TYPE_STRIPE_REPLICATE;
-                        gf_log (THIS->name, GF_LOG_INFO,
-                                "Changing the type of volume %s from "
-                                "'replicate' to 'replicate-stripe'",
-                                volinfo->volname);
-                        ret = 0;
-                        goto out;
-                } else {
-                        snprintf (err_str, err_len, "Incorrect number of "
-                                  "bricks (%d) supplied for changing volume's "
-                                  "stripe count to %d, need at least %d bricks",
-                                  (total_bricks - volinfo->brick_count),
-                                  stripe_count,
-                                  (volinfo->replica_count * stripe_count));
-                        gf_log (THIS->name, GF_LOG_ERROR, "%s", err_str);
-                        goto out;
-                }
-                break;
-        case GF_CLUSTER_TYPE_STRIPE:
-        case GF_CLUSTER_TYPE_STRIPE_REPLICATE:
-                if (stripe_count < volinfo->stripe_count) {
-                        snprintf (err_str, err_len,
-                                  "Incorrect stripe count (%d) supplied. "
-                                  "Volume already has stripe count (%d)",
-                                  stripe_count, volinfo->stripe_count);
-                        gf_log (THIS->name, GF_LOG_ERROR, "%s", err_str);
-                        goto out;
-                }
-                if (stripe_count == volinfo->stripe_count) {
-                        if (!(total_bricks % volinfo->dist_leaf_count)) {
-                                /* its same as the one which exists */
-                                ret = 1;
-                                goto out;
-                        }
-                }
-                if (stripe_count > volinfo->stripe_count) {
-                        /* We have to make sure before and after 'add-brick',
-                           the number or subvolumes for distribute will remain
-                           same, when stripe count is given */
-                        if ((volinfo->brick_count * (stripe_count *
-                                                     volinfo->replica_count)) ==
-                            (total_bricks * volinfo->dist_leaf_count)) {
-                                /* Change the dist_leaf_count */
-                                gf_log (THIS->name, GF_LOG_INFO,
-                                        "Changing the stripe count of "
-                                        "volume %s from %d to %d",
-                                        volinfo->volname,
-                                        volinfo->stripe_count, stripe_count);
-                                ret = 0;
-                                goto out;
-                        }
-                }
-                break;
-        case GF_CLUSTER_TYPE_DISPERSE:
-                snprintf (err_str, err_len, "Volume %s cannot be converted "
-                                            "from dispersed to striped-"
-                                            "dispersed", volinfo->volname);
-                gf_log(THIS->name, GF_LOG_ERROR, "%s", err_str);
-                goto out;
-        }
+    sub_cnt = volinfo->replica_count;
+    idx = (count / (replica_cnt - sub_cnt) * sub_cnt) + (count + sub_cnt);
 
-out:
-        return ret;
+insert_brick:
+    i = 0;
+    cds_list_for_each_entry(brick, &volinfo->bricks, brick_list)
+    {
+        i++;
+        if (i < idx)
+            continue;
+        gf_msg_debug(THIS->name, 0, "brick:%s index=%d, count=%d", brick->path,
+                     idx, count);
+
+        cds_list_add(&brickinfo->brick_list, &brick->brick_list);
+        break;
+    }
+
+    return 0;
 }
 
 static int
-gd_addbr_validate_replica_count (glusterd_volinfo_t *volinfo, int replica_count,
-                                 int total_bricks, int *type, char *err_str,
-                                 int err_len)
+gd_addbr_validate_replica_count(glusterd_volinfo_t *volinfo, int replica_count,
+                                int arbiter_count, int total_bricks, int *type,
+                                char *err_str, int err_len)
 {
-        int ret = -1;
+    int ret = -1;
 
-        /* replica count is set */
-        switch (volinfo->type) {
+    /* replica count is set */
+    switch (volinfo->type) {
         case GF_CLUSTER_TYPE_NONE:
-                if ((volinfo->brick_count * replica_count) == total_bricks) {
-                        /* Change the volume type */
-                        *type = GF_CLUSTER_TYPE_REPLICATE;
-                        gf_log (THIS->name, GF_LOG_INFO,
-                                "Changing the type of volume %s from "
-                                "'distribute' to 'replica'", volinfo->volname);
-                        ret = 0;
-                        goto out;
-
-                } else {
-                        snprintf (err_str, err_len, "Incorrect number of "
-                                  "bricks (%d) supplied for replica count (%d).",
-                                  (total_bricks - volinfo->brick_count),
-                                  replica_count);
-                        gf_log (THIS->name, GF_LOG_ERROR, "%s", err_str);
-                        goto out;
-                }
-                break;
-        case GF_CLUSTER_TYPE_STRIPE:
-                if (!(total_bricks % (volinfo->dist_leaf_count * replica_count))) {
-                        /* Change the volume type */
-                        *type = GF_CLUSTER_TYPE_STRIPE_REPLICATE;
-                        gf_log (THIS->name, GF_LOG_INFO,
-                                "Changing the type of volume %s from "
-                                "'stripe' to 'replicate-stripe'",
-                                volinfo->volname);
-                        ret = 0;
-                        goto out;
-                } else {
-                        snprintf (err_str, err_len, "Incorrect number of "
-                                  "bricks (%d) supplied for changing volume's "
-                                  "replica count to %d, need at least %d "
-                                  "bricks",
-                                  (total_bricks - volinfo->brick_count),
-                                  replica_count, (volinfo->dist_leaf_count *
-                                                  replica_count));
-                        gf_log (THIS->name, GF_LOG_ERROR, "%s", err_str);
-                        goto out;
-                }
-                break;
+            if ((volinfo->brick_count * replica_count) == total_bricks) {
+                /* Change the volume type */
+                *type = GF_CLUSTER_TYPE_REPLICATE;
+                gf_msg(THIS->name, GF_LOG_INFO, 0,
+                       GD_MSG_VOL_TYPE_CHANGING_INFO,
+                       "Changing the type of volume %s from "
+                       "'distribute' to 'replica'",
+                       volinfo->volname);
+                ret = 0;
+                goto out;
+
+            } else {
+                snprintf(err_str, err_len,
+                         "Incorrect number of "
+                         "bricks (%d) supplied for replica count (%d).",
+                         (total_bricks - volinfo->brick_count), replica_count);
+                gf_msg(THIS->name, GF_LOG_ERROR, EINVAL, GD_MSG_INVALID_ENTRY,
+                       "%s", err_str);
+                goto out;
+            }
+            break;
         case GF_CLUSTER_TYPE_REPLICATE:
-        case GF_CLUSTER_TYPE_STRIPE_REPLICATE:
-                if (replica_count < volinfo->replica_count) {
-                        snprintf (err_str, err_len,
-                                  "Incorrect replica count (%d) supplied. "
-                                  "Volume already has (%d)",
-                                  replica_count, volinfo->replica_count);
-                        gf_log (THIS->name, GF_LOG_ERROR, "%s", err_str);
-                        goto out;
+            if (replica_count < volinfo->replica_count) {
+                snprintf(err_str, err_len,
+                         "Incorrect replica count (%d) supplied. "
+                         "Volume already has (%d)",
+                         replica_count, volinfo->replica_count);
+                gf_msg(THIS->name, GF_LOG_ERROR, EINVAL, GD_MSG_INVALID_ENTRY,
+                       "%s", err_str);
+                goto out;
+            }
+            if (replica_count == volinfo->replica_count) {
+                if (arbiter_count && !volinfo->arbiter_count) {
+                    snprintf(err_str, err_len,
+                             "Cannot convert replica 3 volume "
+                             "to arbiter volume.");
+                    gf_msg(THIS->name, GF_LOG_ERROR, EINVAL,
+                           GD_MSG_INVALID_ENTRY, "%s", err_str);
+                    goto out;
                 }
-                if (replica_count == volinfo->replica_count) {
-                        if (!(total_bricks % volinfo->dist_leaf_count)) {
-                                ret = 1;
-                                goto out;
-                        }
+                if (!(total_bricks % volinfo->dist_leaf_count)) {
+                    ret = 1;
+                    goto out;
                 }
-                if (replica_count > volinfo->replica_count) {
-                        /* We have to make sure before and after 'add-brick',
-                           the number or subvolumes for distribute will remain
-                           same, when replica count is given */
-                        if ((total_bricks * volinfo->dist_leaf_count) ==
-                            (volinfo->brick_count * (replica_count *
-                                                     volinfo->stripe_count))) {
-                                /* Change the dist_leaf_count */
-                                gf_log (THIS->name, GF_LOG_INFO,
-                                        "Changing the replica count of "
-                                        "volume %s from %d to %d",
-                                        volinfo->volname, volinfo->replica_count,
-                                        replica_count);
-                                ret = 0;
-                                goto out;
-                        }
+            }
+            if (replica_count > volinfo->replica_count) {
+                /* We have to make sure before and after 'add-brick',
+                   the number or subvolumes for distribute will remain
+                   same, when replica count is given */
+                if ((total_bricks * volinfo->dist_leaf_count) ==
+                    (volinfo->brick_count *
+                     (replica_count * volinfo->stripe_count))) {
+                    /* Change the dist_leaf_count */
+                    gf_msg(THIS->name, GF_LOG_INFO, 0,
+                           GD_MSG_REPLICA_COUNT_CHANGE_INFO,
+                           "Changing the replica count of "
+                           "volume %s from %d to %d",
+                           volinfo->volname, volinfo->replica_count,
+                           replica_count);
+                    ret = 0;
+                    goto out;
                 }
-                break;
+            }
+            break;
         case GF_CLUSTER_TYPE_DISPERSE:
-                snprintf (err_str, err_len, "Volume %s cannot be converted "
-                                            "from dispersed to replicated-"
-                                            "dispersed", volinfo->volname);
-                gf_log(THIS->name, GF_LOG_ERROR, "%s", err_str);
-                goto out;
-        }
+            snprintf(err_str, err_len,
+                     "Volume %s cannot be converted "
+                     "from dispersed to replicated-"
+                     "dispersed",
+                     volinfo->volname);
+            gf_msg(THIS->name, GF_LOG_ERROR, EPERM, GD_MSG_OP_NOT_PERMITTED,
+                   "%s", err_str);
+            goto out;
+    }
 out:
-        return ret;
+    return ret;
 }
 
 static int
-gd_rmbr_validate_replica_count (glusterd_volinfo_t *volinfo,
-                                int32_t replica_count,
-                                int32_t brick_count, char *err_str,
-                                size_t err_len)
+gd_rmbr_validate_replica_count(glusterd_volinfo_t *volinfo,
+                               int32_t replica_count, int32_t brick_count,
+                               char *err_str, size_t err_len)
 {
-        int ret = -1;
-        int replica_nodes = 0;
-
-        switch (volinfo->type) {
-        case GF_CLUSTER_TYPE_TIER:
-                ret = 1;
-                goto out;
+    int ret = -1;
+    int replica_nodes = 0;
+    xlator_t *this = NULL;
+    this = THIS;
+    GF_ASSERT(this);
 
+    switch (volinfo->type) {
         case GF_CLUSTER_TYPE_NONE:
-        case GF_CLUSTER_TYPE_STRIPE:
         case GF_CLUSTER_TYPE_DISPERSE:
-                snprintf (err_str, err_len,
-                          "replica count (%d) option given for non replicate "
-                          "volume %s", replica_count, volinfo->volname);
-                gf_log (THIS->name, GF_LOG_WARNING, "%s", err_str);
-                goto out;
+            snprintf(err_str, err_len,
+                     "replica count (%d) option given for non replicate "
+                     "volume %s",
+                     replica_count, volinfo->volname);
+            gf_smsg(this->name, GF_LOG_WARNING, EINVAL, GD_MSG_INVALID_ARGUMENT,
+                    err_str, NULL);
+            goto out;
 
         case GF_CLUSTER_TYPE_REPLICATE:
-        case GF_CLUSTER_TYPE_STRIPE_REPLICATE:
-                /* in remove brick, you can only reduce the replica count */
-                if (replica_count > volinfo->replica_count) {
-                        snprintf (err_str, err_len,
-                                  "given replica count (%d) option is more "
-                                  "than volume %s's replica count (%d)",
-                                  replica_count, volinfo->volname,
-                                  volinfo->replica_count);
-                        gf_log (THIS->name, GF_LOG_WARNING, "%s", err_str);
-                        goto out;
-                }
-                if (replica_count == volinfo->replica_count) {
-                        /* This means the 'replica N' option on CLI was
-                           redundant. Check if the total number of bricks given
-                           for removal is same as 'dist_leaf_count' */
-                        if (brick_count % volinfo->dist_leaf_count) {
-                                snprintf (err_str, err_len,
-                                          "number of bricks provided (%d) is "
-                                          "not valid. need at least %d "
-                                          "(or %dxN)", brick_count,
-                                          volinfo->dist_leaf_count,
-                                          volinfo->dist_leaf_count);
-                                gf_log (THIS->name, GF_LOG_WARNING, "%s",
-                                        err_str);
-                                goto out;
-                        }
-                        ret = 1;
-                        goto out;
+            /* in remove brick, you can only reduce the replica count */
+            if (replica_count > volinfo->replica_count) {
+                snprintf(err_str, err_len,
+                         "given replica count (%d) option is more "
+                         "than volume %s's replica count (%d)",
+                         replica_count, volinfo->volname,
+                         volinfo->replica_count);
+                gf_smsg(this->name, GF_LOG_WARNING, EINVAL,
+                        GD_MSG_INVALID_ARGUMENT, err_str, NULL);
+                goto out;
+            }
+            if (replica_count == volinfo->replica_count) {
+                /* This means the 'replica N' option on CLI was
+                   redundant. Check if the total number of bricks given
+                   for removal is same as 'dist_leaf_count' */
+                if (brick_count % volinfo->dist_leaf_count) {
+                    snprintf(err_str, err_len,
+                             "number of bricks provided (%d) is "
+                             "not valid. need at least %d "
+                             "(or %dxN)",
+                             brick_count, volinfo->dist_leaf_count,
+                             volinfo->dist_leaf_count);
+                    gf_smsg(this->name, GF_LOG_WARNING, EINVAL,
+                            GD_MSG_INVALID_ARGUMENT, err_str, NULL);
+                    goto out;
                 }
+                ret = 1;
+                goto out;
+            }
 
-                replica_nodes = ((volinfo->brick_count /
-                                  volinfo->replica_count) *
-                                 (volinfo->replica_count - replica_count));
+            replica_nodes = ((volinfo->brick_count / volinfo->replica_count) *
+                             (volinfo->replica_count - replica_count));
 
-                if (brick_count % replica_nodes) {
-                        snprintf (err_str, err_len,
-                                  "need %d(xN) bricks for reducing replica "
-                                  "count of the volume from %d to %d",
-                                  replica_nodes, volinfo->replica_count,
-                                  replica_count);
-                        goto out;
-                }
-                break;
-        }
+            if (brick_count % replica_nodes) {
+                snprintf(err_str, err_len,
+                         "need %d(xN) bricks for reducing replica "
+                         "count of the volume from %d to %d",
+                         replica_nodes, volinfo->replica_count, replica_count);
+                gf_smsg(this->name, GF_LOG_WARNING, EINVAL,
+                        GD_MSG_INVALID_ARGUMENT, err_str, NULL);
+                goto out;
+            }
+            break;
+    }
 
-        ret = 0;
+    ret = 0;
 out:
-        return ret;
+    return ret;
 }
 
 /* Handler functions */
 int
-__glusterd_handle_add_brick (rpcsvc_request_t *req)
+__glusterd_handle_add_brick(rpcsvc_request_t *req)
 {
-        int32_t                         ret = -1;
-        gf_cli_req                      cli_req = {{0,}};
-        dict_t                          *dict = NULL;
-        char                            *bricks = NULL;
-        char                            *volname = NULL;
-        int                             brick_count = 0;
-        void                            *cli_rsp = NULL;
-        char                            err_str[2048] = {0,};
-        gf_cli_rsp                      rsp = {0,};
-        glusterd_volinfo_t              *volinfo = NULL;
-        xlator_t                        *this = NULL;
-        int                             total_bricks = 0;
-        int32_t                         replica_count = 0;
-        int32_t                         stripe_count = 0;
-        int                             type = 0;
-        this = THIS;
-        GF_ASSERT(this);
-
-        GF_ASSERT (req);
-
-        ret = xdr_to_generic (req->msg[0], &cli_req,
-                              (xdrproc_t)xdr_gf_cli_req);
+    int32_t ret = -1;
+    gf_cli_req cli_req = {{
+        0,
+    }};
+    dict_t *dict = NULL;
+    char *bricks = NULL;
+    char *volname = NULL;
+    int brick_count = 0;
+    void *cli_rsp = NULL;
+    char err_str[2048] = "";
+    gf_cli_rsp rsp = {
+        0,
+    };
+    glusterd_volinfo_t *volinfo = NULL;
+    xlator_t *this = NULL;
+    int total_bricks = 0;
+    int32_t replica_count = 0;
+    int32_t arbiter_count = 0;
+    int32_t stripe_count = 0;
+    int type = 0;
+    glusterd_conf_t *conf = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+
+    GF_ASSERT(req);
+
+    conf = this->private;
+    GF_ASSERT(conf);
+
+    ret = xdr_to_generic(req->msg[0], &cli_req, (xdrproc_t)xdr_gf_cli_req);
+    if (ret < 0) {
+        // failed to decode msg;
+        req->rpc_err = GARBAGE_ARGS;
+        snprintf(err_str, sizeof(err_str), "Garbage args received");
+        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_GARBAGE_ARGS, NULL);
+        goto out;
+    }
+
+    gf_msg(this->name, GF_LOG_INFO, 0, GD_MSG_ADD_BRICK_REQ_RECVD,
+           "Received add brick req");
+
+    if (cli_req.dict.dict_len) {
+        /* Unserialize the dictionary */
+        dict = dict_new();
+
+        ret = dict_unserialize(cli_req.dict.dict_val, cli_req.dict.dict_len,
+                               &dict);
         if (ret < 0) {
-                //failed to decode msg;
-                req->rpc_err = GARBAGE_ARGS;
-                snprintf (err_str, sizeof (err_str), "Garbage args received");
-                goto out;
-        }
-
-        gf_log (this->name, GF_LOG_INFO, "Received add brick req");
-
-        if (cli_req.dict.dict_len) {
-                /* Unserialize the dictionary */
-                dict  = dict_new ();
-
-                ret = dict_unserialize (cli_req.dict.dict_val,
-                                        cli_req.dict.dict_len,
-                                        &dict);
-                if (ret < 0) {
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "failed to "
-                                "unserialize req-buffer to dictionary");
-                        snprintf (err_str, sizeof (err_str), "Unable to decode "
-                                  "the command");
-                        goto out;
-                }
-        }
-
-        ret = dict_get_str (dict, "volname", &volname);
-
-        if (ret) {
-                snprintf (err_str, sizeof (err_str), "Unable to get volume "
-                          "name");
-                gf_log (this->name, GF_LOG_ERROR, "%s", err_str);
-                goto out;
-        }
-
-        if (!(ret = glusterd_check_volume_exists (volname))) {
-                ret = -1;
-                snprintf (err_str, sizeof (err_str), "Volume %s does not exist",
-                          volname);
-                gf_log (this->name, GF_LOG_ERROR, "%s", err_str);
-                goto out;
-        }
-
-        ret = dict_get_int32 (dict, "count", &brick_count);
-        if (ret) {
-                snprintf (err_str, sizeof (err_str), "Unable to get volume "
-                          "brick count");
-                gf_log (this->name, GF_LOG_ERROR, "%s", err_str);
-                goto out;
-        }
-
-        ret = dict_get_int32 (dict, "replica-count", &replica_count);
-        if (!ret) {
-                gf_log (this->name, GF_LOG_INFO, "replica-count is %d",
-                        replica_count);
-        }
-
-        ret = dict_get_int32 (dict, "stripe-count", &stripe_count);
-        if (!ret) {
-                gf_log (this->name, GF_LOG_INFO, "stripe-count is %d",
-                        stripe_count);
-        }
-
-        if (!dict_get (dict, "force")) {
-                gf_log (this->name, GF_LOG_ERROR, "Failed to get flag");
-                goto out;
-        }
-
-        ret = glusterd_volinfo_find (volname, &volinfo);
-        if (ret) {
-                snprintf (err_str, sizeof (err_str), "Unable to get volinfo "
-                          "for volume name %s", volname);
-                gf_log (this->name, GF_LOG_ERROR, "%s", err_str);
-                goto out;
-
-        }
-
-        total_bricks = volinfo->brick_count + brick_count;
-
-        if (dict_get (dict, "attach-tier")) {
-                if (volinfo->type == GF_CLUSTER_TYPE_TIER) {
-                        snprintf (err_str, sizeof (err_str),
-                                  "Volume %s is already a tier.", volname);
-                        gf_log (this->name, GF_LOG_ERROR, "%s", err_str);
-                        ret = -1;
-                        goto out;
-                }
-
-                ret = dict_get_int32 (dict, "type", &type);
-
-                goto brick_val;
-        }
-
-        if (!stripe_count && !replica_count) {
-                if (volinfo->type == GF_CLUSTER_TYPE_NONE)
-                        goto brick_val;
-
-                if ((volinfo->brick_count < volinfo->dist_leaf_count) &&
-                    (total_bricks <= volinfo->dist_leaf_count))
-                        goto brick_val;
-
-                if ((brick_count % volinfo->dist_leaf_count) != 0) {
-                        snprintf (err_str, sizeof (err_str), "Incorrect number "
-                                  "of bricks supplied %d with count %d",
-                                 brick_count, volinfo->dist_leaf_count);
-                        gf_log (this->name, GF_LOG_ERROR, "%s", err_str);
-                        ret = -1;
-                        goto out;
-                }
-                goto brick_val;
-                /* done with validation.. below section is if stripe|replica
-                   count is given */
-        }
-
-        /* These bricks needs to be added one per a replica or stripe volume */
-        if (stripe_count) {
-                ret = gd_addbr_validate_stripe_count (volinfo, stripe_count,
-                                                      total_bricks, &type,
-                                                      err_str,
-                                                      sizeof (err_str));
-                if (ret == -1) {
-                        gf_log (this->name, GF_LOG_ERROR, "%s", err_str);
-                        goto out;
-                }
-
-                /* if stripe count is same as earlier, set it back to 0 */
-                if (ret == 1)
-                        stripe_count = 0;
-
-                ret = dict_set_int32 (dict, "stripe-count", stripe_count);
-                if (ret) {
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "failed to set the stripe-count in dict");
-                        goto out;
-                }
-                goto brick_val;
-        }
-
-        ret = gd_addbr_validate_replica_count (volinfo, replica_count,
-                                               total_bricks,
-                                               &type, err_str,
-                                               sizeof (err_str));
-        if (ret == -1) {
-                gf_log (this->name, GF_LOG_ERROR, "%s", err_str);
-                goto out;
-        }
-
-        /* if replica count is same as earlier, set it back to 0 */
-        if (ret == 1)
-                replica_count = 0;
-
-        ret = dict_set_int32 (dict, "replica-count", replica_count);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "failed to set the replica-count in dict");
-                goto out;
-        }
+            gf_msg(this->name, GF_LOG_ERROR, errno,
+                   GD_MSG_DICT_UNSERIALIZE_FAIL,
+                   "failed to "
+                   "unserialize req-buffer to dictionary");
+            snprintf(err_str, sizeof(err_str),
+                     "Unable to decode "
+                     "the command");
+            goto out;
+        }
+    }
+
+    ret = dict_get_strn(dict, "volname", SLEN("volname"), &volname);
+
+    if (ret) {
+        snprintf(err_str, sizeof(err_str),
+                 "Unable to get volume "
+                 "name");
+        gf_msg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_GET_FAILED, "%s",
+               err_str);
+        goto out;
+    }
+
+    ret = glusterd_volinfo_find(volname, &volinfo);
+    if (ret) {
+        snprintf(err_str, sizeof(err_str),
+                 "Unable to get volinfo "
+                 "for volume name %s",
+                 volname);
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_VOLINFO_GET_FAIL, "%s",
+               err_str);
+        goto out;
+    }
+
+    ret = dict_get_int32n(dict, "count", SLEN("count"), &brick_count);
+    if (ret) {
+        snprintf(err_str, sizeof(err_str),
+                 "Unable to get volume "
+                 "brick count");
+        gf_msg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_GET_FAILED, "%s",
+               err_str);
+        goto out;
+    }
+
+    ret = dict_get_int32n(dict, "replica-count", SLEN("replica-count"),
+                          &replica_count);
+    if (!ret) {
+        gf_msg(this->name, GF_LOG_INFO, errno, GD_MSG_DICT_GET_SUCCESS,
+               "replica-count is %d", replica_count);
+    }
+
+    ret = dict_get_int32n(dict, "arbiter-count", SLEN("arbiter-count"),
+                          &arbiter_count);
+    if (!ret) {
+        gf_msg(this->name, GF_LOG_INFO, errno, GD_MSG_DICT_GET_SUCCESS,
+               "arbiter-count is %d", arbiter_count);
+    }
+
+    ret = dict_get_int32n(dict, "stripe-count", SLEN("stripe-count"),
+                          &stripe_count);
+    if (!ret) {
+        gf_msg(this->name, GF_LOG_INFO, errno, GD_MSG_DICT_GET_SUCCESS,
+               "stripe-count is %d", stripe_count);
+    }
+
+    if (!dict_getn(dict, "force", SLEN("force"))) {
+        gf_msg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_GET_FAILED,
+               "Failed to get flag");
+        goto out;
+    }
+
+    total_bricks = volinfo->brick_count + brick_count;
+
+    if (!stripe_count && !replica_count) {
+        if (volinfo->type == GF_CLUSTER_TYPE_NONE)
+            goto brick_val;
+
+        if ((volinfo->brick_count < volinfo->dist_leaf_count) &&
+            (total_bricks <= volinfo->dist_leaf_count))
+            goto brick_val;
+
+        if ((brick_count % volinfo->dist_leaf_count) != 0) {
+            snprintf(err_str, sizeof(err_str),
+                     "Incorrect number "
+                     "of bricks supplied %d with count %d",
+                     brick_count, volinfo->dist_leaf_count);
+            gf_msg(this->name, GF_LOG_ERROR, EINVAL, GD_MSG_VOL_NOT_REPLICA,
+                   "%s", err_str);
+            ret = -1;
+            goto out;
+        }
+        goto brick_val;
+        /* done with validation.. below section is if stripe|replica
+           count is given */
+    }
+
+    ret = gd_addbr_validate_replica_count(volinfo, replica_count, arbiter_count,
+                                          total_bricks, &type, err_str,
+                                          sizeof(err_str));
+    if (ret == -1) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_COUNT_VALIDATE_FAILED, "%s",
+               err_str);
+        goto out;
+    }
+
+    /* if replica count is same as earlier, set it back to 0 */
+    if (ret == 1)
+        replica_count = 0;
+
+    ret = dict_set_int32n(dict, "replica-count", SLEN("replica-count"),
+                          replica_count);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_SET_FAILED,
+               "failed to set the replica-count in dict");
+        goto out;
+    }
 
 brick_val:
-        ret = dict_get_str (dict, "bricks", &bricks);
+    ret = dict_get_strn(dict, "bricks", SLEN("bricks"), &bricks);
+    if (ret) {
+        snprintf(err_str, sizeof(err_str),
+                 "Unable to get volume "
+                 "bricks");
+        gf_msg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_GET_FAILED, "%s",
+               err_str);
+        goto out;
+    }
+
+    if (type != volinfo->type) {
+        ret = dict_set_int32n(dict, "type", SLEN("type"), type);
         if (ret) {
-                snprintf (err_str, sizeof (err_str), "Unable to get volume "
-                          "bricks");
-                gf_log (this->name, GF_LOG_ERROR, "%s", err_str);
-                goto out;
-        }
-
-        if (type != volinfo->type) {
-                ret = dict_set_int32 (dict, "type", type);
-                if (ret)
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "failed to set the new type in dict");
-        }
-
-        ret = glusterd_op_begin_synctask (req, GD_OP_ADD_BRICK, dict);
+            gf_msg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_SET_FAILED,
+                   "failed to set the new type in dict");
+            goto out;
+        }
+    }
+
+    if (conf->op_version <= GD_OP_VERSION_3_7_5) {
+        gf_msg_debug(this->name, 0,
+                     "The cluster is operating at "
+                     "version less than or equal to %d. Falling back "
+                     "to syncop framework.",
+                     GD_OP_VERSION_3_7_5);
+        ret = glusterd_op_begin_synctask(req, GD_OP_ADD_BRICK, dict);
+    } else {
+        ret = glusterd_mgmt_v3_initiate_all_phases(req, GD_OP_ADD_BRICK, dict);
+    }
 
 out:
-        if (ret) {
-                rsp.op_ret = -1;
-                rsp.op_errno = 0;
-                if (err_str[0] == '\0')
-                        snprintf (err_str, sizeof (err_str), "Operation failed");
-                rsp.op_errstr = err_str;
-                cli_rsp = &rsp;
-                glusterd_to_cli (req, cli_rsp, NULL, 0, NULL,
-                                 (xdrproc_t)xdr_gf_cli_rsp, dict);
-                ret = 0; //sent error to cli, prevent second reply
-        }
-
-        free (cli_req.dict.dict_val); //its malloced by xdr
-
-        return ret;
+    if (ret) {
+        rsp.op_ret = -1;
+        rsp.op_errno = 0;
+        if (err_str[0] == '\0')
+            snprintf(err_str, sizeof(err_str), "Operation failed");
+        rsp.op_errstr = err_str;
+        cli_rsp = &rsp;
+        glusterd_to_cli(req, cli_rsp, NULL, 0, NULL, (xdrproc_t)xdr_gf_cli_rsp,
+                        dict);
+        ret = 0;  // sent error to cli, prevent second reply
+    }
+
+    free(cli_req.dict.dict_val);  // its malloced by xdr
+
+    return ret;
 }
 
 int
-glusterd_handle_add_brick (rpcsvc_request_t *req)
+glusterd_handle_add_brick(rpcsvc_request_t *req)
 {
-        return glusterd_big_locked_handler (req, __glusterd_handle_add_brick);
+    return glusterd_big_locked_handler(req, __glusterd_handle_add_brick);
 }
 
 static int
-subvol_matcher_init (int **subvols, int count)
+subvol_matcher_init(int **subvols, int count)
 {
-        int ret = -1;
+    int ret = -1;
 
-        *subvols = GF_CALLOC (count, sizeof(int), gf_gld_mt_int);
-        if (*subvols)
-                ret = 0;
+    *subvols = GF_CALLOC(count, sizeof(int), gf_gld_mt_int);
+    if (*subvols)
+        ret = 0;
 
-        return ret;
+    return ret;
 }
 
 static void
-subvol_matcher_update (int *subvols, glusterd_volinfo_t *volinfo,
-                       glusterd_brickinfo_t *brickinfo)
+subvol_matcher_update(int *subvols, glusterd_volinfo_t *volinfo,
+                      glusterd_brickinfo_t *brickinfo)
 {
-        glusterd_brickinfo_t *tmp        = NULL;
-        int32_t               sub_volume = 0;
-        int                   pos        = 0;
-
-        cds_list_for_each_entry (tmp, &volinfo->bricks, brick_list) {
-
-                if (strcmp (tmp->hostname, brickinfo->hostname) ||
-                    strcmp (tmp->path, brickinfo->path)) {
-                        pos++;
-                        continue;
-                }
-                gf_log (THIS->name, GF_LOG_DEBUG, LOGSTR_FOUND_BRICK,
-                                        brickinfo->hostname, brickinfo->path,
-                                        volinfo->volname);
-                sub_volume = (pos / volinfo->dist_leaf_count);
-                subvols[sub_volume]++;
-                break;
-        }
-
+    glusterd_brickinfo_t *tmp = NULL;
+    int32_t sub_volume = 0;
+    int pos = 0;
+    if (subvols) {
+        cds_list_for_each_entry(tmp, &volinfo->bricks, brick_list)
+        {
+            if (strcmp(tmp->hostname, brickinfo->hostname) ||
+                strcmp(tmp->path, brickinfo->path)) {
+                pos++;
+                continue;
+            }
+            gf_msg_debug(THIS->name, 0, LOGSTR_FOUND_BRICK, brickinfo->hostname,
+                         brickinfo->path, volinfo->volname);
+            sub_volume = (pos / volinfo->dist_leaf_count);
+            subvols[sub_volume]++;
+            break;
+        }
+    }
 }
 
 static int
-subvol_matcher_verify (int *subvols, glusterd_volinfo_t *volinfo, char *err_str,
-                       size_t err_len, char *vol_type, int replica_count)
+subvol_matcher_verify(int *subvols, glusterd_volinfo_t *volinfo, char *err_str,
+                      size_t err_len, char *vol_type, int replica_count)
 {
-        int i = 0;
-        int ret = 0;
-        int count = volinfo->replica_count-replica_count;
-
-        if (replica_count) {
-                for (i = 0; i < volinfo->subvol_count; i++) {
-                        if (subvols[i] != count) {
-                                ret = -1;
-                                snprintf (err_str, err_len, "Remove exactly %d"
-                                " brick(s) from each subvolume.", count);
-                                break;
-                        }
-                }
-                return ret;
+    int i = 0;
+    int ret = 0;
+    int count = volinfo->replica_count - replica_count;
+    xlator_t *this = THIS;
+    GF_ASSERT(this);
+
+    if (replica_count && subvols) {
+        for (i = 0; i < volinfo->subvol_count; i++) {
+            if (subvols[i] != count) {
+                ret = -1;
+                snprintf(err_str, err_len,
+                         "Remove exactly %d"
+                         " brick(s) from each subvolume.",
+                         count);
+                gf_smsg(this->name, GF_LOG_ERROR, errno,
+                        GD_MSG_BRICK_SUBVOL_VERIFY_FAIL, err_str, NULL);
+                break;
+            }
         }
+        return ret;
+    }
 
-       do {
-
-                if (subvols[i] % volinfo->dist_leaf_count == 0) {
-                        continue;
-                } else {
-                        ret = -1;
-                        snprintf (err_str, err_len,
-                                "Bricks not from same subvol for %s", vol_type);
-                        break;
-                }
-        } while (++i < volinfo->subvol_count);
+    do {
+        if (subvols && (subvols[i] % volinfo->dist_leaf_count == 0)) {
+            continue;
+        } else {
+            ret = -1;
+            snprintf(err_str, err_len, "Bricks not from same subvol for %s",
+                     vol_type);
+            gf_smsg(this->name, GF_LOG_ERROR, errno,
+                    GD_MSG_BRICK_SUBVOL_VERIFY_FAIL, err_str, NULL);
+            break;
+        }
+    } while (++i < volinfo->subvol_count);
 
-        return ret;
+    return ret;
 }
 
 static void
-subvol_matcher_destroy (int *subvols)
+subvol_matcher_destroy(int *subvols)
 {
-        GF_FREE (subvols);
+    GF_FREE(subvols);
 }
 
 static int
-glusterd_set_detach_bricks(dict_t *dict, glusterd_volinfo_t *volinfo)
+glusterd_remove_brick_validate_arbiters(glusterd_volinfo_t *volinfo,
+                                        int32_t count, int32_t replica_count,
+                                        glusterd_brickinfo_t **brickinfo_list,
+                                        char *err_str, size_t err_len)
 {
-        char key[256] = {0,};
-        char value[256] = {0,};
-        int brick_num = 0;
-        int hot_brick_num = 0;
-        glusterd_brickinfo_t *brickinfo;
-        int ret = 0;
-
-        /* cold tier bricks at tail of list so use reverse iteration */
-        cds_list_for_each_entry_reverse (brickinfo, &volinfo->bricks,
-                                         brick_list) {
-                brick_num++;
-                if (brick_num > volinfo->tier_info.cold_brick_count) {
-                        hot_brick_num++;
-                        sprintf (key, "brick%d", hot_brick_num);
-                        snprintf (value, 256, "%s:%s",
-                                  brickinfo->hostname,
-                                  brickinfo->path);
-
-                        ret = dict_set_str (dict, key, strdup(value));
-                        if (ret)
-                                break;
-                }
-        }
-
-        ret = dict_set_int32(dict, "count", hot_brick_num);
-        if (ret)
-                return -1;
+    int i = 0;
+    int ret = 0;
+    glusterd_brickinfo_t *brickinfo = NULL;
+    glusterd_brickinfo_t *last = NULL;
+    char *arbiter_array = NULL;
+    xlator_t *this = NULL;
+    this = THIS;
+    GF_ASSERT(this);
+
+    if (volinfo->type != GF_CLUSTER_TYPE_REPLICATE)
+        goto out;
+
+    if (!replica_count || !volinfo->arbiter_count)
+        goto out;
+
+    if (replica_count == 2) {
+        /* If it is an arbiter to replica 2 conversion, only permit
+         *  removal of the arbiter brick.*/
+        for (i = 0; i < count; i++) {
+            brickinfo = brickinfo_list[i];
+            last = get_last_brick_of_brick_group(volinfo, brickinfo);
+            if (last != brickinfo) {
+                snprintf(err_str, err_len,
+                         "Remove arbiter "
+                         "brick(s) only when converting from "
+                         "arbiter to replica 2 subvolume.");
+                gf_smsg(this->name, GF_LOG_ERROR, errno,
+                        GD_MSG_REMOVE_ARBITER_BRICK, err_str, NULL);
+                ret = -1;
+                goto out;
+            }
+        }
+    } else if (replica_count == 1) {
+        /* If it is an arbiter to plain distribute conversion, in every
+         * replica subvol, the arbiter has to be one of the bricks that
+         * are removed. */
+        arbiter_array = GF_CALLOC(volinfo->subvol_count, sizeof(*arbiter_array),
+                                  gf_common_mt_char);
+        if (!arbiter_array)
+            return -1;
+        for (i = 0; i < count; i++) {
+            brickinfo = brickinfo_list[i];
+            last = get_last_brick_of_brick_group(volinfo, brickinfo);
+            if (last == brickinfo)
+                arbiter_array[brickinfo->group] = 1;
+        }
+        for (i = 0; i < volinfo->subvol_count; i++)
+            if (!arbiter_array[i]) {
+                snprintf(err_str, err_len,
+                         "Removed bricks "
+                         "must contain arbiter when converting"
+                         " to plain distribute.");
+                gf_smsg(this->name, GF_LOG_ERROR, errno,
+                        GD_MSG_REMOVE_ARBITER_BRICK, err_str, NULL);
+                ret = -1;
+                break;
+            }
+        GF_FREE(arbiter_array);
+    }
 
-        return hot_brick_num;
+out:
+    return ret;
 }
 
 int
-__glusterd_handle_remove_brick (rpcsvc_request_t *req)
+__glusterd_handle_remove_brick(rpcsvc_request_t *req)
 {
-        int32_t                   ret              = -1;
-        gf_cli_req                cli_req          = {{0,}};
-        dict_t                   *dict             = NULL;
-        int32_t                   count            = 0;
-        char                     *brick            = NULL;
-        char                      key[256]         = {0,};
-        char                     *brick_list       = NULL;
-        int                       i                = 1;
-        glusterd_volinfo_t       *volinfo          = NULL;
-        glusterd_brickinfo_t     *brickinfo        = NULL;
-        int                      *subvols          = NULL;
-        char                      err_str[2048]    = {0};
-        gf_cli_rsp                rsp              = {0,};
-        void                     *cli_rsp          = NULL;
-        char                      vol_type[256]    = {0,};
-        int32_t                   replica_count    = 0;
-        char                     *volname          = 0;
-        xlator_t                 *this             = NULL;
-
-        GF_ASSERT (req);
-        this = THIS;
-        GF_ASSERT (this);
-
-        ret = xdr_to_generic (req->msg[0], &cli_req,
-                              (xdrproc_t)xdr_gf_cli_req);
+    int32_t ret = -1;
+    gf_cli_req cli_req = {{
+        0,
+    }};
+    dict_t *dict = NULL;
+    int32_t count = 0;
+    char *brick = NULL;
+    char key[64] = "";
+    int keylen;
+    int i = 1;
+    glusterd_conf_t *conf = NULL;
+    glusterd_volinfo_t *volinfo = NULL;
+    glusterd_brickinfo_t *brickinfo = NULL;
+    glusterd_brickinfo_t **brickinfo_list = NULL;
+    int *subvols = NULL;
+    char err_str[2048] = "";
+    gf_cli_rsp rsp = {
+        0,
+    };
+    void *cli_rsp = NULL;
+    char vol_type[256] = "";
+    int32_t replica_count = 0;
+    char *volname = 0;
+    xlator_t *this = NULL;
+    int cmd = -1;
+
+    GF_ASSERT(req);
+    this = THIS;
+    GF_ASSERT(this);
+    conf = this->private;
+    GF_ASSERT(conf);
+
+    ret = xdr_to_generic(req->msg[0], &cli_req, (xdrproc_t)xdr_gf_cli_req);
+    if (ret < 0) {
+        // failed to decode msg;
+        req->rpc_err = GARBAGE_ARGS;
+        snprintf(err_str, sizeof(err_str), "Received garbage args");
+        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_GARBAGE_ARGS, NULL);
+        goto out;
+    }
+
+    gf_msg(this->name, GF_LOG_INFO, 0, GD_MSG_REM_BRICK_REQ_RECVD,
+           "Received rem brick req");
+
+    if (cli_req.dict.dict_len) {
+        /* Unserialize the dictionary */
+        dict = dict_new();
+
+        ret = dict_unserialize(cli_req.dict.dict_val, cli_req.dict.dict_len,
+                               &dict);
         if (ret < 0) {
-                //failed to decode msg;
-                req->rpc_err = GARBAGE_ARGS;
-                snprintf (err_str, sizeof (err_str), "Received garbage args");
-                goto out;
-        }
-
-
-        gf_log (this->name, GF_LOG_INFO, "Received rem brick req");
-
-        if (cli_req.dict.dict_len) {
-                /* Unserialize the dictionary */
-                dict  = dict_new ();
-
-                ret = dict_unserialize (cli_req.dict.dict_val,
-                                        cli_req.dict.dict_len,
-                                        &dict);
-                if (ret < 0) {
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "failed to "
-                                "unserialize req-buffer to dictionary");
-                        snprintf (err_str, sizeof (err_str), "Unable to decode "
-                                  "the command");
-                        goto out;
-                }
-        }
-
-        ret = dict_get_str (dict, "volname", &volname);
-        if (ret) {
-                snprintf (err_str, sizeof (err_str), "Unable to get volume "
-                          "name");
-                gf_log (this->name, GF_LOG_ERROR, "%s", err_str);
-                goto out;
-        }
-
-        ret = dict_get_int32 (dict, "count", &count);
-        if (ret) {
-                snprintf (err_str, sizeof (err_str), "Unable to get brick "
-                          "count");
-                gf_log (this->name, GF_LOG_ERROR, "%s", err_str);
-                goto out;
+            gf_msg(this->name, GF_LOG_ERROR, errno,
+                   GD_MSG_DICT_UNSERIALIZE_FAIL,
+                   "failed to "
+                   "unserialize req-buffer to dictionary");
+            snprintf(err_str, sizeof(err_str),
+                     "Unable to decode "
+                     "the command");
+            goto out;
+        }
+    }
+
+    ret = dict_get_strn(dict, "volname", SLEN("volname"), &volname);
+    if (ret) {
+        snprintf(err_str, sizeof(err_str),
+                 "Unable to get volume "
+                 "name");
+        gf_msg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_GET_FAILED, "%s",
+               err_str);
+        goto out;
+    }
+
+    ret = dict_get_int32n(dict, "count", SLEN("count"), &count);
+    if (ret) {
+        snprintf(err_str, sizeof(err_str),
+                 "Unable to get brick "
+                 "count");
+        gf_msg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_GET_FAILED, "%s",
+               err_str);
+        goto out;
+    }
+
+    ret = glusterd_volinfo_find(volname, &volinfo);
+    if (ret) {
+        snprintf(err_str, sizeof(err_str), "Volume %s does not exist", volname);
+        gf_msg(this->name, GF_LOG_ERROR, EINVAL, GD_MSG_VOL_NOT_FOUND, "%s",
+               err_str);
+        goto out;
+    }
+
+    ret = dict_get_int32n(dict, "command", SLEN("command"), &cmd);
+    if (ret) {
+        snprintf(err_str, sizeof(err_str),
+                 "Unable to get cmd "
+                 "ccommand");
+        gf_msg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_GET_FAILED, "%s",
+               err_str);
+        goto out;
+    }
+
+    ret = dict_get_int32n(dict, "replica-count", SLEN("replica-count"),
+                          &replica_count);
+    if (!ret) {
+        gf_msg(this->name, GF_LOG_INFO, errno, GD_MSG_DICT_GET_FAILED,
+               "request to change replica-count to %d", replica_count);
+        ret = gd_rmbr_validate_replica_count(volinfo, replica_count, count,
+                                             err_str, sizeof(err_str));
+        if (ret < 0) {
+            /* logging and error msg are done in above function
+               itself */
+            goto out;
         }
-
-        ret = glusterd_volinfo_find (volname, &volinfo);
+        dict_deln(dict, "replica-count", SLEN("replica-count"));
         if (ret) {
-                 snprintf (err_str, sizeof (err_str),"Volume %s does not exist",
-                           volname);
-                 gf_log (this->name, GF_LOG_ERROR, "%s", err_str);
-                 goto out;
-        }
-
-        ret = dict_get_int32 (dict, "replica-count", &replica_count);
-        if (!ret) {
-                gf_log (this->name, GF_LOG_INFO,
-                        "request to change replica-count to %d", replica_count);
-                ret = gd_rmbr_validate_replica_count (volinfo, replica_count,
-                                                      count, err_str,
-                                                      sizeof (err_str));
-                if (ret < 0) {
-                        /* logging and error msg are done in above function
-                           itself */
-                        goto out;
-                }
-                dict_del (dict, "replica-count");
-                if (ret) {
-                        replica_count = 0;
-                } else {
-                        ret = dict_set_int32 (dict, "replica-count",
-                                              replica_count);
-                        if (ret) {
-                                gf_log (this->name, GF_LOG_WARNING,
-                                        "failed to set the replica_count "
-                                        "in dict");
-                                goto out;
-                        }
-                }
-        }
-
-        /* 'vol_type' is used for giving the meaning full error msg for user */
-        if (volinfo->type == GF_CLUSTER_TYPE_REPLICATE) {
-                strcpy (vol_type, "replica");
-        } else if (volinfo->type == GF_CLUSTER_TYPE_STRIPE) {
-                strcpy (vol_type, "stripe");
-        } else if (volinfo->type == GF_CLUSTER_TYPE_STRIPE_REPLICATE) {
-                strcpy (vol_type, "stripe-replicate");
-        } else if (volinfo->type == GF_CLUSTER_TYPE_DISPERSE) {
-                strcpy (vol_type, "disperse");
+            replica_count = 0;
         } else {
-                strcpy (vol_type, "distribute");
-        }
-
-	/* Do not allow remove-brick if the volume is a stripe volume*/
-	if ((volinfo->type == GF_CLUSTER_TYPE_STRIPE) &&
-            (volinfo->brick_count == volinfo->stripe_count)) {
-                snprintf (err_str, sizeof (err_str),
-                          "Removing brick from a stripe volume is not allowed");
-                gf_log (this->name, GF_LOG_ERROR, "%s", err_str);
-                ret = -1;
-                goto out;
-	}
-
-	if (!replica_count &&
-            (volinfo->type == GF_CLUSTER_TYPE_STRIPE_REPLICATE) &&
-            (volinfo->brick_count == volinfo->dist_leaf_count)) {
-                snprintf (err_str, sizeof(err_str),
-                          "Removing bricks from stripe-replicate"
-                          " configuration is not allowed without reducing "
-                          "replica or stripe count explicitly.");
-                gf_log (this->name, GF_LOG_ERROR, "%s", err_str);
-                ret = -1;
-                goto out;
-        }
-
-	if (!replica_count &&
-            (volinfo->type == GF_CLUSTER_TYPE_REPLICATE) &&
-            (volinfo->brick_count == volinfo->dist_leaf_count)) {
-                snprintf (err_str, sizeof (err_str),
-                          "Removing bricks from replicate configuration "
-                          "is not allowed without reducing replica count "
-                          "explicitly.");
-                gf_log (this->name, GF_LOG_ERROR, "%s", err_str);
-                ret = -1;
-                goto out;
-        }
-
-	/* Do not allow remove-brick if the bricks given is less than
-           the replica count or stripe count */
-        if (!replica_count && (volinfo->type != GF_CLUSTER_TYPE_NONE) &&
-            (volinfo->type != GF_CLUSTER_TYPE_TIER))  {
-                if (volinfo->dist_leaf_count &&
-                    (count % volinfo->dist_leaf_count)) {
-                        snprintf (err_str, sizeof (err_str), "Remove brick "
-                                  "incorrect brick count of %d for %s %d",
-                                  count, vol_type, volinfo->dist_leaf_count);
-                        gf_log (this->name, GF_LOG_ERROR, "%s", err_str);
-                        ret = -1;
-                        goto out;
-                }
-        }
-
-        brick_list = GF_MALLOC (120000 * sizeof(*brick_list),gf_common_mt_char);
-
-        if (!brick_list) {
-                ret = -1;
-                goto out;
-        }
-
-
-        strcpy (brick_list, " ");
-
-        if ((volinfo->type != GF_CLUSTER_TYPE_NONE) &&
-            (volinfo->subvol_count > 1)) {
-                ret = subvol_matcher_init (&subvols, volinfo->subvol_count);
-                if (ret)
-                        goto out;
-        }
-
-        if (volinfo->type == GF_CLUSTER_TYPE_TIER)
-                count = glusterd_set_detach_bricks(dict, volinfo);
-
-        while ( i <= count) {
-                snprintf (key, sizeof (key), "brick%d", i);
-                ret = dict_get_str (dict, key, &brick);
-                if (ret) {
-                        snprintf (err_str, sizeof (err_str), "Unable to get %s",
-                                  key);
-                        gf_log (this->name, GF_LOG_ERROR, "%s", err_str);
-                        goto out;
-                }
-                gf_log (this->name, GF_LOG_DEBUG, "Remove brick count %d brick:"
-                        " %s", i, brick);
-
-                ret = glusterd_volume_brickinfo_get_by_brick(brick, volinfo,
-                                                             &brickinfo);
-
-                if (ret) {
-                        snprintf (err_str, sizeof (err_str), "Incorrect brick "
-                                  "%s for volume %s", brick, volname);
-                        gf_log (this->name, GF_LOG_ERROR, "%s", err_str);
-                        goto out;
-                }
-                strcat(brick_list, brick);
-                strcat(brick_list, " ");
-
-                i++;
-                if ((volinfo->type == GF_CLUSTER_TYPE_NONE) ||
-                    (volinfo->brick_count <= volinfo->dist_leaf_count))
-                        continue;
+            ret = dict_set_int32n(dict, "replica-count", SLEN("replica-count"),
+                                  replica_count);
+            if (ret) {
+                gf_msg(this->name, GF_LOG_WARNING, errno,
+                       GD_MSG_DICT_SET_FAILED,
+                       "failed to set the replica_count "
+                       "in dict");
+                goto out;
+            }
+        }
+    }
+
+    /* 'vol_type' is used for giving the meaning full error msg for user */
+    if (volinfo->type == GF_CLUSTER_TYPE_REPLICATE) {
+        strcpy(vol_type, "replica");
+    } else if (volinfo->type == GF_CLUSTER_TYPE_DISPERSE) {
+        strcpy(vol_type, "disperse");
+    } else {
+        strcpy(vol_type, "distribute");
+    }
+
+    if (!replica_count && (volinfo->type == GF_CLUSTER_TYPE_REPLICATE) &&
+        (volinfo->brick_count == volinfo->dist_leaf_count)) {
+        snprintf(err_str, sizeof(err_str),
+                 "Removing bricks from replicate configuration "
+                 "is not allowed without reducing replica count "
+                 "explicitly.");
+        gf_msg(this->name, GF_LOG_ERROR, EPERM, GD_MSG_OP_NOT_PERMITTED_AC_REQD,
+               "%s", err_str);
+        ret = -1;
+        goto out;
+    }
+
+    /* Do not allow remove-brick if the bricks given is less than
+       the replica count or stripe count */
+    if (!replica_count && (volinfo->type != GF_CLUSTER_TYPE_NONE)) {
+        if (volinfo->dist_leaf_count && (count % volinfo->dist_leaf_count)) {
+            snprintf(err_str, sizeof(err_str),
+                     "Remove brick "
+                     "incorrect brick count of %d for %s %d",
+                     count, vol_type, volinfo->dist_leaf_count);
+            gf_msg(this->name, GF_LOG_ERROR, EINVAL, GD_MSG_INVALID_ENTRY, "%s",
+                   err_str);
+            ret = -1;
+            goto out;
+        }
+    }
+
+    if ((volinfo->type != GF_CLUSTER_TYPE_NONE) &&
+        (volinfo->subvol_count > 1)) {
+        ret = subvol_matcher_init(&subvols, volinfo->subvol_count);
+        if (ret)
+            goto out;
+    }
 
-                /* Find which subvolume the brick belongs to */
-                subvol_matcher_update (subvols, volinfo, brickinfo);
-        }
+    brickinfo_list = GF_CALLOC(count, sizeof(*brickinfo_list),
+                               gf_common_mt_pointer);
+    if (!brickinfo_list) {
+        ret = -1;
+        goto out;
+    }
 
-        /* Check if the bricks belong to the same subvolumes.*/
-        if ((volinfo->type != GF_CLUSTER_TYPE_NONE) &&
-            (volinfo->subvol_count > 1)) {
-                ret = subvol_matcher_verify (subvols, volinfo,
-                                             err_str, sizeof(err_str),
-                                             vol_type, replica_count);
-                if (ret)
-                        goto out;
+    while (i <= count) {
+        keylen = snprintf(key, sizeof(key), "brick%d", i);
+        ret = dict_get_strn(dict, key, keylen, &brick);
+        if (ret) {
+            snprintf(err_str, sizeof(err_str), "Unable to get %s", key);
+            gf_msg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_GET_FAILED,
+                   "%s", err_str);
+            goto out;
         }
+        gf_msg_debug(this->name, 0,
+                     "Remove brick count %d brick:"
+                     " %s",
+                     i, brick);
 
-        ret = glusterd_op_begin_synctask (req, GD_OP_REMOVE_BRICK, dict);
+        ret = glusterd_volume_brickinfo_get_by_brick(brick, volinfo, &brickinfo,
+                                                     _gf_false);
 
-out:
         if (ret) {
-                rsp.op_ret = -1;
-                rsp.op_errno = 0;
-                if (err_str[0] == '\0')
-                        snprintf (err_str, sizeof (err_str),
-                                  "Operation failed");
-                gf_log (this->name, GF_LOG_ERROR, "%s", err_str);
-                rsp.op_errstr = err_str;
-                cli_rsp = &rsp;
-                glusterd_to_cli (req, cli_rsp, NULL, 0, NULL,
-                                 (xdrproc_t)xdr_gf_cli_rsp, dict);
-
-                ret = 0; //sent error to cli, prevent second reply
-
-        }
-
-        if (brick_list)
-                GF_FREE (brick_list);
-        subvol_matcher_destroy (subvols);
-        free (cli_req.dict.dict_val); //its malloced by xdr
+            snprintf(err_str, sizeof(err_str),
+                     "Incorrect brick "
+                     "%s for volume %s",
+                     brick, volname);
+            gf_msg(this->name, GF_LOG_ERROR, EINVAL, GD_MSG_BRICK_NOT_FOUND,
+                   "%s", err_str);
+            goto out;
+        }
+        brickinfo_list[i - 1] = brickinfo;
+
+        i++;
+        if ((volinfo->type == GF_CLUSTER_TYPE_NONE) ||
+            (volinfo->brick_count <= volinfo->dist_leaf_count))
+            continue;
+
+        subvol_matcher_update(subvols, volinfo, brickinfo);
+    }
+
+    if ((volinfo->type != GF_CLUSTER_TYPE_NONE) &&
+        (volinfo->subvol_count > 1)) {
+        ret = subvol_matcher_verify(subvols, volinfo, err_str, sizeof(err_str),
+                                    vol_type, replica_count);
+        if (ret)
+            goto out;
+    }
+
+    ret = glusterd_remove_brick_validate_arbiters(volinfo, count, replica_count,
+                                                  brickinfo_list, err_str,
+                                                  sizeof(err_str));
+    if (ret)
+        goto out;
+
+    if (conf->op_version < GD_OP_VERSION_8_0) {
+        gf_msg_debug(this->name, 0,
+                     "The cluster is operating at "
+                     "version less than %d. remove-brick operation"
+                     "falling back to syncop framework.",
+                     GD_OP_VERSION_8_0);
+        ret = glusterd_op_begin_synctask(req, GD_OP_REMOVE_BRICK, dict);
+    } else {
+        ret = glusterd_mgmt_v3_initiate_all_phases(req, GD_OP_REMOVE_BRICK,
+                                                   dict);
+    }
 
-        return ret;
+out:
+    if (ret) {
+        rsp.op_ret = -1;
+        rsp.op_errno = 0;
+        if (err_str[0] == '\0')
+            snprintf(err_str, sizeof(err_str), "Operation failed");
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_GLUSTERD_OP_FAILED, "%s",
+               err_str);
+        rsp.op_errstr = err_str;
+        cli_rsp = &rsp;
+        glusterd_to_cli(req, cli_rsp, NULL, 0, NULL, (xdrproc_t)xdr_gf_cli_rsp,
+                        dict);
+
+        ret = 0;  // sent error to cli, prevent second reply
+    }
+
+    if (brickinfo_list)
+        GF_FREE(brickinfo_list);
+    subvol_matcher_destroy(subvols);
+    free(cli_req.dict.dict_val);  // its malloced by xdr
+
+    return ret;
 }
 
 int
-glusterd_handle_remove_brick (rpcsvc_request_t *req)
+glusterd_handle_remove_brick(rpcsvc_request_t *req)
 {
-        return glusterd_big_locked_handler (req,
-                                            __glusterd_handle_remove_brick);
+    return glusterd_big_locked_handler(req, __glusterd_handle_remove_brick);
 }
 
 static int
-_glusterd_restart_gsync_session (dict_t *this, char *key,
-                                 data_t *value, void *data)
+_glusterd_restart_gsync_session(dict_t *this, char *key, data_t *value,
+                                void *data)
 {
-        char                          *slave      = NULL;
-        char                          *slave_buf  = NULL;
-        char                          *path_list  = NULL;
-        char                          *slave_vol  = NULL;
-        char                          *slave_host = NULL;
-        char                          *slave_url  = NULL;
-        char                          *conf_path  = NULL;
-        char                         **errmsg     = NULL;
-        int                            ret        = -1;
-        glusterd_gsync_status_temp_t  *param      = NULL;
-        gf_boolean_t                   is_running = _gf_false;
-
-        param = (glusterd_gsync_status_temp_t *)data;
-
-        GF_ASSERT (param);
-        GF_ASSERT (param->volinfo);
-
-        slave = strchr(value->data, ':');
-        if (slave) {
-                slave++;
-                slave_buf = gf_strdup (slave);
-                if (!slave_buf) {
-                        gf_log ("", GF_LOG_ERROR,
-                                "Failed to gf_strdup");
-                        ret = -1;
-                        goto out;
-                }
-        }
-        else
-                return 0;
-
-        ret = dict_set_dynstr (param->rsp_dict, "slave", slave_buf);
-        if (ret) {
-                gf_log ("", GF_LOG_ERROR,
-                        "Unable to store slave");
-                if (slave_buf)
-                        GF_FREE(slave_buf);
-                goto out;
-        }
-
-        ret = glusterd_get_slave_details_confpath (param->volinfo,
-                                                   param->rsp_dict, &slave_url,
-                                                   &slave_host, &slave_vol,
-                                                   &conf_path, errmsg);
-        if (ret) {
-                if (*errmsg)
-                        gf_log ("", GF_LOG_ERROR, "%s", *errmsg);
-                else
-                        gf_log ("", GF_LOG_ERROR,
-                                "Unable to fetch slave or confpath details.");
-                goto out;
-        }
-
-        /* In cases that gsyncd is not running, we will not invoke it
-         * because of add-brick. */
-        ret = glusterd_check_gsync_running_local (param->volinfo->volname,
-                                                  slave, conf_path,
-                                                  &is_running);
-        if (ret) {
-                gf_log ("", GF_LOG_ERROR, "gsync running validation failed.");
-                goto out;
-        }
-        if (_gf_false == is_running) {
-                gf_log ("", GF_LOG_DEBUG, "gsync session for %s and %s is"
-                        " not running on this node. Hence not restarting.",
-                        param->volinfo->volname, slave);
-                ret = 0;
-                goto out;
-        }
+    char *slave = NULL;
+    char *slave_buf = NULL;
+    char *path_list = NULL;
+    char *slave_vol = NULL;
+    char *slave_host = NULL;
+    char *slave_url = NULL;
+    char *conf_path = NULL;
+    char **errmsg = NULL;
+    int ret = -1;
+    glusterd_gsync_status_temp_t *param = NULL;
+    gf_boolean_t is_running = _gf_false;
+
+    param = (glusterd_gsync_status_temp_t *)data;
+
+    GF_ASSERT(param);
+    GF_ASSERT(param->volinfo);
+
+    slave = strchr(value->data, ':');
+    if (slave) {
+        slave++;
+        slave_buf = gf_strdup(slave);
+        if (!slave_buf) {
+            gf_msg("glusterd", GF_LOG_ERROR, ENOMEM, GD_MSG_NO_MEMORY,
+                   "Failed to gf_strdup");
+            ret = -1;
+            goto out;
+        }
+    } else
+        return 0;
 
-        ret = glusterd_get_local_brickpaths (param->volinfo, &path_list);
-        if (!path_list) {
-                gf_log ("", GF_LOG_DEBUG, "This node not being part of"
-                        " volume should not be running gsyncd. Hence"
-                        " no gsyncd process to restart.");
-                ret = 0;
-                goto out;
-        }
+    ret = dict_set_dynstrn(param->rsp_dict, "slave", SLEN("slave"), slave_buf);
+    if (ret) {
+        gf_msg("glusterd", GF_LOG_ERROR, errno, GD_MSG_DICT_SET_FAILED,
+               "Unable to store slave");
+        if (slave_buf)
+            GF_FREE(slave_buf);
+        goto out;
+    }
+
+    ret = glusterd_get_slave_details_confpath(param->volinfo, param->rsp_dict,
+                                              &slave_url, &slave_host,
+                                              &slave_vol, &conf_path, errmsg);
+    if (ret) {
+        if (errmsg && *errmsg)
+            gf_msg("glusterd", GF_LOG_ERROR, 0,
+                   GD_MSG_SLAVE_CONFPATH_DETAILS_FETCH_FAIL, "%s", *errmsg);
+        else
+            gf_msg("glusterd", GF_LOG_ERROR, 0,
+                   GD_MSG_SLAVE_CONFPATH_DETAILS_FETCH_FAIL,
+                   "Unable to fetch slave or confpath details.");
+        goto out;
+    }
+
+    /* In cases that gsyncd is not running, we will not invoke it
+     * because of add-brick. */
+    ret = glusterd_check_gsync_running_local(param->volinfo->volname, slave,
+                                             conf_path, &is_running);
+    if (ret) {
+        gf_msg("glusterd", GF_LOG_ERROR, 0, GD_MSG_GSYNC_VALIDATION_FAIL,
+               "gsync running validation failed.");
+        goto out;
+    }
+    if (_gf_false == is_running) {
+        gf_msg_debug("glusterd", 0,
+                     "gsync session for %s and %s is"
+                     " not running on this node. Hence not restarting.",
+                     param->volinfo->volname, slave);
+        ret = 0;
+        goto out;
+    }
+
+    ret = glusterd_get_local_brickpaths(param->volinfo, &path_list);
+    if (!path_list) {
+        gf_msg_debug("glusterd", 0,
+                     "This node not being part of"
+                     " volume should not be running gsyncd. Hence"
+                     " no gsyncd process to restart.");
+        ret = 0;
+        goto out;
+    }
 
-        ret = glusterd_check_restart_gsync_session (param->volinfo, slave,
-                                                    param->rsp_dict, path_list,
-                                                    conf_path, 0);
-        if (ret)
-                gf_log ("", GF_LOG_ERROR,
-                        "Unable to restart gsync session.");
+    ret = glusterd_check_restart_gsync_session(
+        param->volinfo, slave, param->rsp_dict, path_list, conf_path, 0);
+    if (ret)
+        gf_msg("glusterd", GF_LOG_ERROR, 0, GD_MSG_GSYNC_RESTART_FAIL,
+               "Unable to restart gsync session.");
 
 out:
-        gf_log ("", GF_LOG_DEBUG, "Returning %d.", ret);
-        return ret;
+    gf_msg_debug("glusterd", 0, "Returning %d.", ret);
+    return ret;
 }
 
 /* op-sm */
 
 int
-glusterd_op_perform_add_bricks (glusterd_volinfo_t *volinfo, int32_t count,
-                                char  *bricks, dict_t *dict)
+glusterd_op_perform_add_bricks(glusterd_volinfo_t *volinfo, int32_t count,
+                               char *bricks, dict_t *dict)
 {
-        char                         *brick          = NULL;
-        int32_t                       i              = 1;
-        char                         *brick_list     = NULL;
-        char                         *free_ptr1      = NULL;
-        char                         *free_ptr2      = NULL;
-        char                         *saveptr        = NULL;
-        int32_t                       ret            = -1;
-        int32_t                       stripe_count   = 0;
-        int32_t                       replica_count  = 0;
-        int32_t                       type           = 0;
-        glusterd_brickinfo_t         *brickinfo      = NULL;
-        glusterd_gsync_status_temp_t  param          = {0, };
-        gf_boolean_t                  restart_needed = 0;
-        char                          msg[1024] __attribute__((unused)) = {0, };
-        int                           caps           = 0;
-        int                           brickid        = 0;
-        char                          key[PATH_MAX]  = "";
-        char                         *brick_mount_dir  = NULL;
-        xlator_t                     *this           = NULL;
-        glusterd_conf_t              *conf           = NULL;
-
-        this = THIS;
-        GF_ASSERT (this);
-        GF_ASSERT (volinfo);
-
-        conf = this->private;
-        GF_ASSERT (conf);
-
-        if (bricks) {
-                brick_list = gf_strdup (bricks);
-                free_ptr1 = brick_list;
-        }
-
-        if (count)
-                brick = strtok_r (brick_list+1, " \n", &saveptr);
-
-        if (dict) {
-                ret = dict_get_int32 (dict, "stripe-count", &stripe_count);
-                if (!ret)
-                        gf_log (THIS->name, GF_LOG_INFO,
-                                "stripe-count is set %d", stripe_count);
-
-                ret = dict_get_int32 (dict, "replica-count", &replica_count);
-                if (!ret)
-                        gf_log (THIS->name, GF_LOG_INFO,
-                                "replica-count is set %d", replica_count);
-                ret = dict_get_int32 (dict, "type", &type);
-                if (!ret)
-                        gf_log (THIS->name, GF_LOG_INFO,
-                                "type is set %d, need to change it", type);
-        }
+    char *brick = NULL;
+    int32_t i = 1;
+    char *brick_list = NULL;
+    char *free_ptr1 = NULL;
+    char *free_ptr2 = NULL;
+    char *saveptr = NULL;
+    int32_t ret = -1;
+    int32_t stripe_count = 0;
+    int32_t replica_count = 0;
+    int32_t arbiter_count = 0;
+    int32_t type = 0;
+    glusterd_brickinfo_t *brickinfo = NULL;
+    glusterd_gsync_status_temp_t param = {
+        0,
+    };
+    gf_boolean_t restart_needed = 0;
+    int brickid = 0;
+    char key[64] = "";
+    char *brick_mount_dir = NULL;
+    xlator_t *this = NULL;
+    glusterd_conf_t *conf = NULL;
+    gf_boolean_t is_valid_add_brick = _gf_false;
+    gf_boolean_t restart_shd = _gf_false;
+    struct statvfs brickstat = {
+        0,
+    };
+
+    this = THIS;
+    GF_ASSERT(this);
+    GF_ASSERT(volinfo);
+
+    conf = this->private;
+    GF_ASSERT(conf);
+
+    if (bricks) {
+        brick_list = gf_strdup(bricks);
+        free_ptr1 = brick_list;
+    }
+
+    if (count)
+        brick = strtok_r(brick_list + 1, " \n", &saveptr);
+
+    if (dict) {
+        ret = dict_get_int32n(dict, "stripe-count", SLEN("stripe-count"),
+                              &stripe_count);
+        if (!ret)
+            gf_msg(THIS->name, GF_LOG_INFO, errno, GD_MSG_DICT_GET_SUCCESS,
+                   "stripe-count is set %d", stripe_count);
 
-        brickid = glusterd_get_next_available_brickid (volinfo);
-        if (brickid < 0)
-                goto out;
-        while ( i <= count) {
-                ret = glusterd_brickinfo_new_from_brick (brick, &brickinfo);
-                if (ret)
-                        goto out;
-
-                GLUSTERD_ASSIGN_BRICKID_TO_BRICKINFO (brickinfo, volinfo,
-                                                      brickid++);
-
-                /* A bricks mount dir is required only by snapshots which were
-                 * introduced in gluster-3.6.0
-                 */
-                if (conf->op_version >= GD_OP_VERSION_3_6_0) {
-                        brick_mount_dir = NULL;
-
-                        snprintf (key, sizeof(key), "brick%d.mount_dir", i);
-                        ret = dict_get_str (dict, key, &brick_mount_dir);
-                        if (ret) {
-                                gf_log (this->name, GF_LOG_ERROR,
-                                                "%s not present", key);
-                                goto out;
-                        }
-                        strncpy (brickinfo->mount_dir, brick_mount_dir,
-                                 sizeof(brickinfo->mount_dir));
-                }
+        ret = dict_get_int32n(dict, "replica-count", SLEN("replica-count"),
+                              &replica_count);
+        if (!ret)
+            gf_msg(THIS->name, GF_LOG_INFO, errno, GD_MSG_DICT_GET_SUCCESS,
+                   "replica-count is set %d", replica_count);
+        ret = dict_get_int32n(dict, "arbiter-count", SLEN("arbiter-count"),
+                              &arbiter_count);
+        if (!ret)
+            gf_msg(THIS->name, GF_LOG_INFO, errno, GD_MSG_DICT_GET_SUCCESS,
+                   "arbiter-count is set %d", arbiter_count);
+        ret = dict_get_int32n(dict, "type", SLEN("type"), &type);
+        if (!ret)
+            gf_msg(THIS->name, GF_LOG_INFO, errno, GD_MSG_DICT_GET_SUCCESS,
+                   "type is set %d, need to change it", type);
+    }
+
+    brickid = glusterd_get_next_available_brickid(volinfo);
+    if (brickid < 0)
+        goto out;
+    while (i <= count) {
+        ret = glusterd_brickinfo_new_from_brick(brick, &brickinfo, _gf_true,
+                                                NULL);
+        if (ret)
+            goto out;
 
-                ret = glusterd_resolve_brick (brickinfo);
-                if (ret)
-                        goto out;
-
-                /* hot tier bricks are added to head of brick list */
-                if (dict_get (dict, "attach-tier")) {
-                        cds_list_add (&brickinfo->brick_list, &volinfo->bricks);
-                } else if (stripe_count || replica_count) {
-                        add_brick_at_right_order (brickinfo, volinfo, (i - 1),
-                                                  stripe_count, replica_count);
-                } else {
-                        cds_list_add_tail (&brickinfo->brick_list,
-                                           &volinfo->bricks);
-                }
-                brick = strtok_r (NULL, " \n", &saveptr);
-                i++;
-                volinfo->brick_count++;
+        GLUSTERD_ASSIGN_BRICKID_TO_BRICKINFO(brickinfo, volinfo, brickid++);
 
-        }
+        /* A bricks mount dir is required only by snapshots which were
+         * introduced in gluster-3.6.0
+         */
+        if (conf->op_version >= GD_OP_VERSION_3_6_0) {
+            brick_mount_dir = NULL;
 
-        /* Gets changed only if the options are given in add-brick cli */
-        if (type)
-                volinfo->type = type;
-        if (replica_count) {
-                volinfo->replica_count = replica_count;
-        }
-        if (stripe_count) {
-                volinfo->stripe_count = stripe_count;
+            snprintf(key, sizeof(key), "brick%d.mount_dir", i);
+            ret = dict_get_str(dict, key, &brick_mount_dir);
+            if (ret) {
+                gf_msg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_GET_FAILED,
+                       "%s not present", key);
+                goto out;
+            }
+            strncpy(brickinfo->mount_dir, brick_mount_dir,
+                    SLEN(brickinfo->mount_dir));
         }
-        volinfo->dist_leaf_count = glusterd_get_dist_leaf_count (volinfo);
 
-        /* backward compatibility */
-        volinfo->sub_count = ((volinfo->dist_leaf_count == 1) ? 0:
-                              volinfo->dist_leaf_count);
+        ret = glusterd_resolve_brick(brickinfo);
+        if (ret)
+            goto out;
 
-        volinfo->subvol_count = (volinfo->brick_count /
-                                 volinfo->dist_leaf_count);
+        if (!gf_uuid_compare(brickinfo->uuid, MY_UUID)) {
+            ret = sys_statvfs(brickinfo->path, &brickstat);
+            if (ret) {
+                gf_msg(this->name, GF_LOG_ERROR, errno, GD_MSG_STATVFS_FAILED,
+                       "Failed to fetch disk utilization "
+                       "from the brick (%s:%s). Please check the health of "
+                       "the brick. Error code was %s",
+                       brickinfo->hostname, brickinfo->path, strerror(errno));
 
-        ret = glusterd_create_volfiles_and_notify_services (volinfo);
-        if (ret)
                 goto out;
-
-        ret = 0;
-        if (GLUSTERD_STATUS_STARTED != volinfo->status)
+            }
+            brickinfo->statfs_fsid = brickstat.f_fsid;
+        }
+        if (stripe_count || replica_count) {
+            add_brick_at_right_order(brickinfo, volinfo, (i - 1), stripe_count,
+                                     replica_count);
+        } else {
+            cds_list_add_tail(&brickinfo->brick_list, &volinfo->bricks);
+        }
+        brick = strtok_r(NULL, " \n", &saveptr);
+        i++;
+        volinfo->brick_count++;
+    }
+
+    /* Gets changed only if the options are given in add-brick cli */
+    if (type)
+        volinfo->type = type;
+    /* performance.client-io-threads is turned on by default,
+     * however this has adverse effects on replicate volumes due to
+     * replication design issues, till that get addressed
+     * performance.client-io-threads option is turned off for all
+     * replicate volumes if not already explicitly enabled.
+     */
+    if (type && glusterd_is_volume_replicate(volinfo) &&
+        conf->op_version >= GD_OP_VERSION_3_12_2) {
+        ret = dict_set_nstrn(volinfo->dict, "performance.client-io-threads",
+                             SLEN("performance.client-io-threads"), "off",
+                             SLEN("off"));
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+                   "Failed to set "
+                   "performance.client-io-threads to off");
+            goto out;
+        }
+    }
+
+    if (replica_count) {
+        volinfo->replica_count = replica_count;
+    }
+    if (arbiter_count) {
+        volinfo->arbiter_count = arbiter_count;
+    }
+    if (stripe_count) {
+        volinfo->stripe_count = stripe_count;
+    }
+    volinfo->dist_leaf_count = glusterd_get_dist_leaf_count(volinfo);
+
+    /* backward compatibility */
+    volinfo->sub_count = ((volinfo->dist_leaf_count == 1)
+                              ? 0
+                              : volinfo->dist_leaf_count);
+
+    volinfo->subvol_count = (volinfo->brick_count / volinfo->dist_leaf_count);
+
+    ret = 0;
+    if (GLUSTERD_STATUS_STARTED != volinfo->status)
+        goto generate_volfiles;
+
+    ret = generate_brick_volfiles(volinfo);
+    if (ret)
+        goto out;
+
+    brick_list = gf_strdup(bricks);
+    free_ptr2 = brick_list;
+    i = 1;
+
+    if (count)
+        brick = strtok_r(brick_list + 1, " \n", &saveptr);
+
+    if (glusterd_is_volume_replicate(volinfo)) {
+        if (replica_count && conf->op_version >= GD_OP_VERSION_3_7_10) {
+            is_valid_add_brick = _gf_true;
+            if (volinfo->status == GLUSTERD_STATUS_STARTED) {
+                ret = volinfo->shd.svc.stop(&(volinfo->shd.svc), SIGTERM);
+                if (ret) {
+                    gf_msg("glusterd", GF_LOG_ERROR, 0,
+                           GD_MSG_GLUSTER_SERVICES_STOP_FAIL,
+                           "Failed to stop shd for %s.", volinfo->volname);
+                }
+                restart_shd = _gf_true;
+            }
+            ret = generate_dummy_client_volfiles(volinfo);
+            if (ret) {
+                gf_msg(THIS->name, GF_LOG_ERROR, 0, GD_MSG_VOLFILE_CREATE_FAIL,
+                       "Failed to create volfile.");
                 goto out;
+            }
+        }
+    }
 
-        brick_list = gf_strdup (bricks);
-        free_ptr2 = brick_list;
-        i = 1;
-
-        if (count)
-                brick = strtok_r (brick_list+1, " \n", &saveptr);
-#ifdef HAVE_BD_XLATOR
-        if (brickinfo->vg[0])
-                caps = CAPS_BD | CAPS_THIN |
-                        CAPS_OFFLOAD_COPY | CAPS_OFFLOAD_SNAPSHOT;
-#endif
-
-        while (i <= count) {
-                ret = glusterd_volume_brickinfo_get_by_brick (brick, volinfo,
-                                                              &brickinfo);
-                if (ret)
-                        goto out;
-#ifdef HAVE_BD_XLATOR
-                /* Check for VG/thin pool if its BD volume */
-                if (brickinfo->vg[0]) {
-                        ret = glusterd_is_valid_vg (brickinfo, 0, msg);
-                        if (ret) {
-                                gf_log (THIS->name, GF_LOG_CRITICAL, "%s", msg);
-                                goto out;
-                        }
-                        /* if anyone of the brick does not have thin support,
-                           disable it for entire volume */
-                        caps &= brickinfo->caps;
-                } else
-                        caps = 0;
-#endif
-
-                if (gf_uuid_is_null (brickinfo->uuid)) {
-                        ret = glusterd_resolve_brick (brickinfo);
-                        if (ret) {
-                                gf_log ("", GF_LOG_ERROR, FMTSTR_RESOLVE_BRICK,
-                                        brickinfo->hostname, brickinfo->path);
-                                goto out;
-                        }
-                }
+    while (i <= count) {
+        ret = glusterd_volume_brickinfo_get_by_brick(brick, volinfo, &brickinfo,
+                                                     _gf_true);
+        if (ret)
+            goto out;
 
-                ret = glusterd_brick_start (volinfo, brickinfo,
-                                            _gf_true);
-                if (ret)
-                        goto out;
-                i++;
-                brick = strtok_r (NULL, " \n", &saveptr);
-
-                /* Check if the brick is added in this node, and set
-                 * the restart_needed flag. */
-                if ((!gf_uuid_compare (brickinfo->uuid, MY_UUID)) &&
-                    !restart_needed) {
-                        restart_needed = 1;
-                        gf_log ("", GF_LOG_DEBUG,
-                                "Restart gsyncd session, if it's already "
-                                "running.");
-                }
+        if (gf_uuid_is_null(brickinfo->uuid)) {
+            ret = glusterd_resolve_brick(brickinfo);
+            if (ret) {
+                gf_msg("glusterd", GF_LOG_ERROR, 0, GD_MSG_RESOLVE_BRICK_FAIL,
+                       FMTSTR_RESOLVE_BRICK, brickinfo->hostname,
+                       brickinfo->path);
+                goto out;
+            }
         }
 
-        /* If the restart_needed flag is set, restart gsyncd sessions for that
-         * particular master with all the slaves. */
-        if (restart_needed) {
-                param.rsp_dict = dict;
-                param.volinfo = volinfo;
-                dict_foreach (volinfo->gsync_slaves,
-                              _glusterd_restart_gsync_session, &param);
+        /* if the volume is a replicate volume, do: */
+        if (is_valid_add_brick) {
+            if (!gf_uuid_compare(brickinfo->uuid, MY_UUID)) {
+                ret = glusterd_handle_replicate_brick_ops(volinfo, brickinfo,
+                                                          GD_OP_ADD_BRICK);
+                if (ret < 0)
+                    goto out;
+            }
         }
-        volinfo->caps = caps;
-out:
-        GF_FREE (free_ptr1);
-        GF_FREE (free_ptr2);
+        ret = glusterd_brick_start(volinfo, brickinfo, _gf_true, _gf_false);
+        if (ret)
+            goto out;
+        i++;
+        brick = strtok_r(NULL, " \n", &saveptr);
+
+        /* Check if the brick is added in this node, and set
+         * the restart_needed flag. */
+        if ((!gf_uuid_compare(brickinfo->uuid, MY_UUID)) && !restart_needed) {
+            restart_needed = 1;
+            gf_msg_debug("glusterd", 0,
+                         "Restart gsyncd session, if it's already "
+                         "running.");
+        }
+    }
+
+    /* If the restart_needed flag is set, restart gsyncd sessions for that
+     * particular master with all the slaves. */
+    if (restart_needed) {
+        param.rsp_dict = dict;
+        param.volinfo = volinfo;
+        dict_foreach(volinfo->gsync_slaves, _glusterd_restart_gsync_session,
+                     &param);
+    }
+
+generate_volfiles:
+    if (conf->op_version <= GD_OP_VERSION_3_7_5) {
+        ret = glusterd_create_volfiles_and_notify_services(volinfo);
+    } else {
+        /*
+         * The cluster is operating at version greater than
+         * gluster-3.7.5. So no need to sent volfile fetch
+         * request in commit phase, the same will be done
+         * in post validate phase with v3 framework.
+         */
+    }
 
-        gf_log ("", GF_LOG_DEBUG, "Returning %d", ret);
-        return ret;
+out:
+    GF_FREE(free_ptr1);
+    GF_FREE(free_ptr2);
+    if (restart_shd) {
+        if (volinfo->shd.svc.manager(&(volinfo->shd.svc), volinfo,
+                                     PROC_START_NO_WAIT)) {
+            gf_msg("glusterd", GF_LOG_CRITICAL, 0,
+                   GD_MSG_GLUSTER_SERVICE_START_FAIL,
+                   "Failed to start shd for %s.", volinfo->volname);
+        }
+    }
+
+    gf_msg_debug("glusterd", 0, "Returning %d", ret);
+    return ret;
 }
 
-
 int
-glusterd_op_perform_remove_brick (glusterd_volinfo_t  *volinfo, char *brick,
-                                  int force, int *need_migrate)
+glusterd_op_perform_remove_brick(glusterd_volinfo_t *volinfo, char *brick,
+                                 int force, int *need_migrate)
 {
-        glusterd_brickinfo_t    *brickinfo = NULL;
-        int32_t                  ret = -1;
-        glusterd_conf_t         *priv = NULL;
+    glusterd_brickinfo_t *brickinfo = NULL;
+    int32_t ret = -1;
+    glusterd_conf_t *priv = NULL;
 
-        GF_ASSERT (volinfo);
-        GF_ASSERT (brick);
+    GF_ASSERT(volinfo);
+    GF_ASSERT(brick);
 
-        priv = THIS->private;
-        GF_ASSERT (priv);
+    priv = THIS->private;
+    GF_ASSERT(priv);
 
-        ret = glusterd_volume_brickinfo_get_by_brick (brick, volinfo,
-                                                      &brickinfo);
-        if (ret)
-                goto out;
+    ret = glusterd_volume_brickinfo_get_by_brick(brick, volinfo, &brickinfo,
+                                                 _gf_false);
+    if (ret)
+        goto out;
 
-        ret = glusterd_resolve_brick (brickinfo);
-        if (ret)
-                goto out;
+    ret = glusterd_resolve_brick(brickinfo);
+    if (ret)
+        goto out;
 
-        glusterd_volinfo_reset_defrag_stats (volinfo);
+    glusterd_volinfo_reset_defrag_stats(volinfo);
 
-        if (!gf_uuid_compare (brickinfo->uuid, MY_UUID)) {
-                /* Only if the brick is in this glusterd, do the rebalance */
-                if (need_migrate)
-                        *need_migrate = 1;
-        }
+    if (!gf_uuid_compare(brickinfo->uuid, MY_UUID)) {
+        /* Only if the brick is in this glusterd, do the rebalance */
+        if (need_migrate)
+            *need_migrate = 1;
+    }
 
-        if (force) {
-                ret = glusterd_brick_stop (volinfo, brickinfo,
-                                           _gf_true);
-                if (ret) {
-                        gf_log (THIS->name, GF_LOG_ERROR, "Unable to stop "
-                                "glusterfs, ret: %d", ret);
-                }
-                goto out;
+    if (force) {
+        ret = glusterd_brick_stop(volinfo, brickinfo, _gf_true);
+        if (ret) {
+            gf_msg(THIS->name, GF_LOG_ERROR, 0, GD_MSG_BRICK_STOP_FAIL,
+                   "Unable to stop "
+                   "glusterfs, ret: %d",
+                   ret);
         }
+        goto out;
+    }
 
-        brickinfo->decommissioned = 1;
-        ret = 0;
+    brickinfo->decommissioned = 1;
+    ret = 0;
 out:
-        gf_log ("", GF_LOG_DEBUG, "Returning %d", ret);
-        return ret;
+    gf_msg_debug("glusterd", 0, "Returning %d", ret);
+    return ret;
 }
 
 int
-glusterd_op_stage_add_brick (dict_t *dict, char **op_errstr, dict_t *rsp_dict)
+glusterd_op_stage_add_brick(dict_t *dict, char **op_errstr, dict_t *rsp_dict)
 {
-        int                                     ret = 0;
-        char                                    *volname = NULL;
-        int                                     count = 0;
-        int                                     replica_count = 0;
-        int                                     i = 0;
-        int32_t                                 local_brick_count = 0;
-        char                                    *bricks    = NULL;
-        char                                    *brick_list = NULL;
-        char                                    *saveptr = NULL;
-        char                                    *free_ptr = NULL;
-        char                                    *brick = NULL;
-        glusterd_brickinfo_t                    *brickinfo = NULL;
-        glusterd_volinfo_t                      *volinfo = NULL;
-        xlator_t                                *this = NULL;
-        char                                    msg[2048] = {0,};
-        char                                    key[PATH_MAX] = "";
-        gf_boolean_t                            brick_alloc = _gf_false;
-        char                                    *all_bricks = NULL;
-        char                                    *str_ret = NULL;
-        gf_boolean_t                            is_force = _gf_false;
-        glusterd_conf_t                         *conf = NULL;
-
-        this = THIS;
-        GF_ASSERT (this);
-        conf = this->private;
-        GF_ASSERT (conf);
-
-        ret = dict_get_int32 (dict, "replica-count", &replica_count);
+    int ret = 0;
+    char *volname = NULL;
+    int count = 0;
+    int replica_count = 0;
+    int arbiter_count = 0;
+    int i = 0;
+    int32_t local_brick_count = 0;
+    char *bricks = NULL;
+    char *brick_list = NULL;
+    char *saveptr = NULL;
+    char *free_ptr = NULL;
+    char *brick = NULL;
+    glusterd_brickinfo_t *brickinfo = NULL;
+    glusterd_volinfo_t *volinfo = NULL;
+    xlator_t *this = NULL;
+    char msg[4096] = "";
+    char key[64] = "";
+    gf_boolean_t brick_alloc = _gf_false;
+    char *all_bricks = NULL;
+    char *str_ret = NULL;
+    gf_boolean_t is_force = _gf_false;
+    glusterd_conf_t *conf = NULL;
+    int32_t len = 0;
+
+    this = THIS;
+    GF_ASSERT(this);
+    conf = this->private;
+    GF_ASSERT(conf);
+
+    ret = dict_get_strn(dict, "volname", SLEN("volname"), &volname);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_GET_FAILED,
+               "Unable to get volume name");
+        goto out;
+    }
+
+    ret = glusterd_volinfo_find(volname, &volinfo);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_VOL_NOT_FOUND,
+               "Unable to find volume: %s", volname);
+        goto out;
+    }
+
+    ret = glusterd_validate_volume_id(dict, volinfo);
+    if (ret)
+        goto out;
+
+    ret = dict_get_int32n(dict, "replica-count", SLEN("replica-count"),
+                          &replica_count);
+    if (ret) {
+        gf_msg_debug(this->name, 0, "Unable to get replica count");
+    }
+
+    if (replica_count > 0) {
+        ret = op_version_check(this, GD_OP_VER_PERSISTENT_AFR_XATTRS, msg,
+                               sizeof(msg));
         if (ret) {
-                gf_log (THIS->name, GF_LOG_ERROR,
-                        "Unable to get replica count");
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_OP_VERSION_MISMATCH,
+                   "%s", msg);
+            *op_errstr = gf_strdup(msg);
+            goto out;
         }
+    }
 
-        if (replica_count > 0) {
-                ret = op_version_check (this, GD_OP_VER_PERSISTENT_AFR_XATTRS,
-                                        msg, sizeof(msg));
-                if (ret) {
-                        gf_log (this->name, GF_LOG_ERROR, "%s", msg);
-                        *op_errstr = gf_strdup (msg);
-                        goto out;
-                }
-        }
-        ret = dict_get_str (dict, "volname", &volname);
-        if (ret) {
-                gf_log (THIS->name, GF_LOG_ERROR,
-                        "Unable to get volume name");
-                goto out;
-        }
+    glusterd_add_peers_to_auth_list(volname);
 
-        ret = glusterd_volinfo_find (volname, &volinfo);
+    if (replica_count && glusterd_is_volume_replicate(volinfo)) {
+        /* Do not allow add-brick for stopped volumes when replica-count
+         * is being increased.
+         */
+        if (GLUSTERD_STATUS_STOPPED == volinfo->status &&
+            conf->op_version >= GD_OP_VERSION_3_7_10) {
+            ret = -1;
+            snprintf(msg, sizeof(msg),
+                     " Volume must not be in"
+                     " stopped state when replica-count needs to "
+                     " be increased.");
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_BRICK_ADD_FAIL, "%s",
+                   msg);
+            *op_errstr = gf_strdup(msg);
+            goto out;
+        }
+        /* op-version check for replica 2 to arbiter conversion. If we
+         * don't have this check, an older peer added as arbiter brick
+         * will not have the  arbiter xlator in its volfile. */
+        if ((replica_count == 3) && (conf->op_version < GD_OP_VERSION_3_8_0)) {
+            ret = dict_get_int32n(dict, "arbiter-count", SLEN("arbiter-count"),
+                                  &arbiter_count);
+            if (ret) {
+                gf_msg_debug(this->name, 0,
+                             "No arbiter count present in the dict");
+            } else if (arbiter_count == 1) {
+                ret = -1;
+                snprintf(msg, sizeof(msg),
+                         "Cluster op-version must "
+                         "be >= 30800 to add arbiter brick to a "
+                         "replica 2 volume.");
+                gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_BRICK_ADD_FAIL, "%s",
+                       msg);
+                *op_errstr = gf_strdup(msg);
+                goto out;
+            }
+        }
+        /* Do not allow increasing replica count for arbiter volumes. */
+        if (volinfo->arbiter_count) {
+            ret = -1;
+            snprintf(msg, sizeof(msg),
+                     "Increasing replica count "
+                     "for arbiter volumes is not supported.");
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_BRICK_ADD_FAIL, "%s",
+                   msg);
+            *op_errstr = gf_strdup(msg);
+            goto out;
+        }
+    }
+
+    is_force = dict_get_str_boolean(dict, "force", _gf_false);
+
+    /* Check brick order if the volume type is replicate or disperse. If
+     * force at the end of command not given then check brick order.
+     * doing this check at the originator node is sufficient.
+     */
+
+    if (!is_force && is_origin_glusterd(dict)) {
+        ret = 0;
+        if (volinfo->type == GF_CLUSTER_TYPE_REPLICATE) {
+            gf_msg_debug(this->name, 0,
+                         "Replicate cluster type "
+                         "found. Checking brick order.");
+            if (replica_count)
+                ret = glusterd_check_brick_order(dict, msg, volinfo->type,
+                                                 &volname, &bricks, &count,
+                                                 replica_count);
+            else
+                ret = glusterd_check_brick_order(dict, msg, volinfo->type,
+                                                 &volname, &bricks, &count,
+                                                 volinfo->replica_count);
+        } else if (volinfo->type == GF_CLUSTER_TYPE_DISPERSE) {
+            gf_msg_debug(this->name, 0,
+                         "Disperse cluster type"
+                         " found. Checking brick order.");
+            ret = glusterd_check_brick_order(dict, msg, volinfo->type, &volname,
+                                             &bricks, &count,
+                                             volinfo->disperse_count);
+        }
         if (ret) {
-                gf_log (THIS->name, GF_LOG_ERROR,
-                        "Unable to find volume: %s", volname);
-                goto out;
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_BAD_BRKORDER,
+                   "Not adding brick because of "
+                   "bad brick order. %s",
+                   msg);
+            *op_errstr = gf_strdup(msg);
+            goto out;
         }
+    }
 
-        ret = glusterd_validate_volume_id (dict, volinfo);
-        if (ret)
-                goto out;
-
-        if (glusterd_is_rb_ongoing (volinfo)) {
-                snprintf (msg, sizeof (msg), "Replace brick is in progress on "
-                          "volume %s. Please retry after replace-brick "
-                          "operation is committed or aborted", volname);
-                gf_log (THIS->name, GF_LOG_ERROR, "%s", msg);
-                *op_errstr = gf_strdup (msg);
+    if (volinfo->replica_count < replica_count && !is_force) {
+        cds_list_for_each_entry(brickinfo, &volinfo->bricks, brick_list)
+        {
+            if (gf_uuid_compare(brickinfo->uuid, MY_UUID))
+                continue;
+            if (brickinfo->status == GF_BRICK_STOPPED) {
                 ret = -1;
+                len = snprintf(msg, sizeof(msg),
+                               "Brick %s "
+                               "is down, changing replica "
+                               "count needs all the bricks "
+                               "to be up to avoid data loss",
+                               brickinfo->path);
+                if (len < 0) {
+                    strcpy(msg, "<error>");
+                }
+                gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_BRICK_ADD_FAIL, "%s",
+                       msg);
+                *op_errstr = gf_strdup(msg);
                 goto out;
+            }
         }
+    }
 
-        if (glusterd_is_defrag_on(volinfo)) {
-                snprintf (msg, sizeof(msg), "Volume name %s rebalance is in "
-                          "progress. Please retry after completion", volname);
-                gf_log (THIS->name, GF_LOG_ERROR, "%s", msg);
-                *op_errstr = gf_strdup (msg);
-                ret = -1;
-                goto out;
-        }
-        ret = dict_get_int32 (dict, "count", &count);
+    if (conf->op_version > GD_OP_VERSION_3_7_5 && is_origin_glusterd(dict)) {
+        ret = glusterd_validate_quorum(this, GD_OP_ADD_BRICK, dict, op_errstr);
         if (ret) {
-                gf_log ("", GF_LOG_ERROR, "Unable to get count");
-                goto out;
+            gf_msg(this->name, GF_LOG_CRITICAL, 0, GD_MSG_SERVER_QUORUM_NOT_MET,
+                   "Server quorum not met. Rejecting operation.");
+            goto out;
+        }
+    } else {
+        /* Case 1: conf->op_version <= GD_OP_VERSION_3_7_5
+         *         in this case the add-brick is running
+         *         syncop framework that will do a quorum
+         *         check by default
+         * Case 2: We don't need to do quorum check on every
+         *         node, only originator glusterd need to
+         *         check for quorum
+         * So nothing need to be done in else
+         */
+    }
+
+    if (glusterd_is_defrag_on(volinfo)) {
+        snprintf(msg, sizeof(msg),
+                 "Volume name %s rebalance is in "
+                 "progress. Please retry after completion",
+                 volname);
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_OIP_RETRY_LATER, "%s", msg);
+        *op_errstr = gf_strdup(msg);
+        ret = -1;
+        goto out;
+    }
+
+    if (volinfo->snap_count > 0 || !cds_list_empty(&volinfo->snap_volumes)) {
+        snprintf(msg, sizeof(msg),
+                 "Volume %s  has %" PRIu64
+                 " snapshots. "
+                 "Changing the volume configuration will not effect snapshots."
+                 "But the snapshot brick mount should be intact to "
+                 "make them function.",
+                 volname, volinfo->snap_count);
+        gf_msg("glusterd", GF_LOG_WARNING, 0, GD_MSG_SNAP_WARN, "%s", msg);
+        msg[0] = '\0';
+    }
+
+    if (!count) {
+        ret = dict_get_int32n(dict, "count", SLEN("count"), &count);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_GET_FAILED,
+                   "Unable to get count");
+            goto out;
         }
+    }
 
-        ret = dict_get_str (dict, "bricks", &bricks);
+    if (!bricks) {
+        ret = dict_get_strn(dict, "bricks", SLEN("bricks"), &bricks);
         if (ret) {
-                gf_log (THIS->name, GF_LOG_ERROR, "Unable to get bricks");
-                goto out;
+            gf_msg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_GET_FAILED,
+                   "Unable to get bricks");
+            goto out;
+        }
+    }
+
+    if (bricks) {
+        brick_list = gf_strdup(bricks);
+        all_bricks = gf_strdup(bricks);
+        free_ptr = brick_list;
+    }
+
+    if (count)
+        brick = strtok_r(brick_list + 1, " \n", &saveptr);
+
+    while (i < count) {
+        if (!glusterd_store_is_valid_brickpath(volname, brick) ||
+            !glusterd_is_valid_volfpath(volname, brick)) {
+            snprintf(msg, sizeof(msg),
+                     "brick path %s is "
+                     "too long",
+                     brick);
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_BRKPATH_TOO_LONG, "%s",
+                   msg);
+            *op_errstr = gf_strdup(msg);
+
+            ret = -1;
+            goto out;
+        }
+
+        ret = glusterd_brickinfo_new_from_brick(brick, &brickinfo, _gf_true,
+                                                NULL);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_BRICK_NOT_FOUND,
+                   "Add-brick: Unable"
+                   " to get brickinfo");
+            goto out;
         }
+        brick_alloc = _gf_true;
 
-        is_force = dict_get_str_boolean (dict, "force", _gf_false);
-
-        if (bricks) {
-                brick_list = gf_strdup (bricks);
-                all_bricks = gf_strdup (bricks);
-                free_ptr = brick_list;
+        ret = glusterd_new_brick_validate(brick, brickinfo, msg, sizeof(msg),
+                                          NULL);
+        if (ret) {
+            *op_errstr = gf_strdup(msg);
+            ret = -1;
+            goto out;
         }
 
-        if (count)
-                brick = strtok_r (brick_list+1, " \n", &saveptr);
-
-
-        while ( i < count) {
-                if (!glusterd_store_is_valid_brickpath (volname, brick) ||
-                    !glusterd_is_valid_volfpath (volname, brick)) {
-                        snprintf (msg, sizeof (msg), "brick path %s is "
-                                  "too long", brick);
-                        gf_log (THIS->name, GF_LOG_ERROR, "%s", msg);
-                        *op_errstr = gf_strdup (msg);
-
-                        ret = -1;
-                        goto out;
-
-                }
+        if (!gf_uuid_compare(brickinfo->uuid, MY_UUID)) {
+            ret = glusterd_validate_and_create_brickpath(
+                brickinfo, volinfo->volume_id, volinfo->volname, op_errstr,
+                is_force, _gf_false);
+            if (ret)
+                goto out;
 
-                ret = glusterd_brickinfo_new_from_brick (brick, &brickinfo);
+            /* A bricks mount dir is required only by snapshots which were
+             * introduced in gluster-3.6.0
+             */
+            if (conf->op_version >= GD_OP_VERSION_3_6_0) {
+                ret = glusterd_get_brick_mount_dir(
+                    brickinfo->path, brickinfo->hostname, brickinfo->mount_dir);
                 if (ret) {
-                        gf_log (THIS->name, GF_LOG_ERROR,
-                                "Add-brick: Unable"
-                                " to get brickinfo");
-                        goto out;
+                    gf_msg(this->name, GF_LOG_ERROR, 0,
+                           GD_MSG_BRICK_MOUNTDIR_GET_FAIL,
+                           "Failed to get brick mount_dir");
+                    goto out;
                 }
-                brick_alloc = _gf_true;
 
-                ret = glusterd_new_brick_validate (brick, brickinfo, msg,
-                                                   sizeof (msg));
+                snprintf(key, sizeof(key), "brick%d.mount_dir", i + 1);
+                ret = dict_set_dynstr_with_alloc(rsp_dict, key,
+                                                 brickinfo->mount_dir);
                 if (ret) {
-                        *op_errstr = gf_strdup (msg);
-                        ret = -1;
-                        goto out;
-                }
-
-                if (!gf_uuid_compare (brickinfo->uuid, MY_UUID)) {
-#ifdef HAVE_BD_XLATOR
-                        if (brickinfo->vg[0]) {
-                                ret = glusterd_is_valid_vg (brickinfo, 1, msg);
-                                if (ret) {
-                                        gf_log (THIS->name, GF_LOG_ERROR, "%s",
-                                                msg);
-                                        *op_errstr = gf_strdup (msg);
-                                        goto out;
-                                }
-                        }
-#endif
-
-                        ret = glusterd_validate_and_create_brickpath (brickinfo,
-                                                          volinfo->volume_id,
-                                                          op_errstr, is_force);
-                        if (ret)
-                                goto out;
-
-                        /* A bricks mount dir is required only by snapshots which were
-                         * introduced in gluster-3.6.0
-                         */
-                        if (conf->op_version >= GD_OP_VERSION_3_6_0) {
-                                ret = glusterd_get_brick_mount_dir
-                                        (brickinfo->path, brickinfo->hostname,
-                                         brickinfo->mount_dir);
-                                if (ret) {
-                                        gf_log (this->name, GF_LOG_ERROR,
-                                                "Failed to get brick mount_dir");
-                                        goto out;
-                                }
-
-                                snprintf (key, sizeof(key), "brick%d.mount_dir",
-                                          i + 1);
-                                ret = dict_set_dynstr_with_alloc
-                                        (rsp_dict, key, brickinfo->mount_dir);
-                                if (ret) {
-                                        gf_log (this->name, GF_LOG_ERROR,
-                                                "Failed to set %s", key);
-                                        goto out;
-                                }
-                        }
-
-                        local_brick_count = i + 1;
+                    gf_msg(this->name, GF_LOG_ERROR, errno,
+                           GD_MSG_DICT_SET_FAILED, "Failed to set %s", key);
+                    goto out;
                 }
+            }
 
-                glusterd_brickinfo_delete (brickinfo);
-                brick_alloc = _gf_false;
-                brickinfo = NULL;
-                brick = strtok_r (NULL, " \n", &saveptr);
-                i++;
+            local_brick_count = i + 1;
         }
 
-        ret = dict_set_int32 (rsp_dict, "brick_count",
-                              local_brick_count);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "Failed to set local_brick_count");
-                goto out;
-        }
+        glusterd_brickinfo_delete(brickinfo);
+        brick_alloc = _gf_false;
+        brickinfo = NULL;
+        brick = strtok_r(NULL, " \n", &saveptr);
+        i++;
+    }
+
+    ret = dict_set_int32n(rsp_dict, "brick_count", SLEN("brick_count"),
+                          local_brick_count);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_SET_FAILED,
+               "Failed to set local_brick_count");
+        goto out;
+    }
 
 out:
-        GF_FREE (free_ptr);
-        if (brick_alloc && brickinfo)
-                glusterd_brickinfo_delete (brickinfo);
-        GF_FREE (str_ret);
-        GF_FREE (all_bricks);
+    GF_FREE(free_ptr);
+    if (brick_alloc && brickinfo)
+        glusterd_brickinfo_delete(brickinfo);
+    GF_FREE(str_ret);
+    GF_FREE(all_bricks);
 
-        gf_log (THIS->name, GF_LOG_DEBUG, "Returning %d", ret);
+    gf_msg_debug(this->name, 0, "Returning %d", ret);
 
-        return ret;
+    return ret;
 }
 
 int
-glusterd_op_stage_remove_brick (dict_t *dict, char **op_errstr)
+glusterd_remove_brick_validate_bricks(gf1_op_commands cmd, int32_t brick_count,
+                                      dict_t *dict, glusterd_volinfo_t *volinfo,
+                                      char **errstr,
+                                      gf_cli_defrag_type cmd_defrag)
 {
-        int                     ret         = -1;
-        char                   *volname     = NULL;
-        glusterd_volinfo_t     *volinfo     = NULL;
-        char                   *errstr      = NULL;
-        int32_t                 brick_count = 0;
-        char                    msg[2048]   = {0,};
-        int32_t                 flag        = 0;
-        gf1_op_commands         cmd         = GF_OP_CMD_NONE;
-        char                   *task_id_str = NULL;
-        xlator_t               *this        = NULL;
-        int                     i           = 1;
-        char                    key[256]    = {0,};
-        char                   *brick       = NULL;
-        glusterd_brickinfo_t   *brickinfo   = NULL;
-        gsync_status_param_t    param       = {0,};
-
-        this = THIS;
-        GF_ASSERT (this);
-
-        ret = op_version_check (this, GD_OP_VER_PERSISTENT_AFR_XATTRS,
-                                msg, sizeof(msg));
+    char *brick = NULL;
+    char msg[2048] = "";
+    char key[64] = "";
+    int keylen;
+    glusterd_brickinfo_t *brickinfo = NULL;
+    glusterd_peerinfo_t *peerinfo = NULL;
+    int i = 0;
+    int ret = -1;
+    char pidfile[PATH_MAX + 1] = {
+        0,
+    };
+    glusterd_conf_t *priv = THIS->private;
+    int pid = -1;
+    xlator_t *this = THIS;
+    GF_ASSERT(this);
+
+    /* Check whether all the nodes of the bricks to be removed are
+     * up, if not fail the operation */
+    for (i = 1; i <= brick_count; i++) {
+        keylen = snprintf(key, sizeof(key), "brick%d", i);
+        ret = dict_get_strn(dict, key, keylen, &brick);
         if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "%s", msg);
-                *op_errstr = gf_strdup (msg);
-                goto out;
-        }
-
-        ret = dict_get_str (dict, "volname", &volname);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "Unable to get volume name");
-                goto out;
+            snprintf(msg, sizeof(msg), "Unable to get %s", key);
+            gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_GET_FAILED,
+                    "key=%s", key, NULL);
+            *errstr = gf_strdup(msg);
+            goto out;
         }
 
-        ret = glusterd_volinfo_find (volname, &volinfo);
-
+        ret = glusterd_volume_brickinfo_get_by_brick(brick, volinfo, &brickinfo,
+                                                     _gf_false);
         if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "Volume %s does not exist", volname);
-                goto out;
-        }
-
-        ret = glusterd_validate_volume_id (dict, volinfo);
-        if (ret)
-                goto out;
-
-        if (glusterd_is_rb_ongoing (volinfo)) {
-                snprintf (msg, sizeof (msg), "Replace brick is in progress on "
-                          "volume %s. Please retry after replace-brick "
-                          "operation is committed or aborted", volname);
-                gf_log (this->name, GF_LOG_ERROR, "%s", msg);
-                *op_errstr = gf_strdup (msg);
+            snprintf(msg, sizeof(msg),
+                     "Incorrect brick "
+                     "%s for volume %s",
+                     brick, volinfo->volname);
+            gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_INCORRECT_BRICK,
+                    "Brick=%s, Volume=%s", brick, volinfo->volname, NULL);
+            *errstr = gf_strdup(msg);
+            goto out;
+        }
+        /* Do not allow commit if the bricks are not decommissioned
+         * if its a remove brick commit
+         */
+        if (!brickinfo->decommissioned && cmd == GF_OP_CMD_COMMIT) {
+            snprintf(msg, sizeof(msg),
+                     "Brick %s "
+                     "is not decommissioned. "
+                     "Use start or force option",
+                     brick);
+            gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_BRICK_NOT_DECOM,
+                    "Use 'start' or 'force' option, Brick=%s", brick, NULL);
+            *errstr = gf_strdup(msg);
+            ret = -1;
+            goto out;
+        }
+
+        if (glusterd_is_local_brick(THIS, volinfo, brickinfo)) {
+            switch (cmd) {
+                case GF_OP_CMD_START:
+                    goto check;
+                case GF_OP_CMD_NONE:
+                default:
+                    break;
+            }
+
+            switch (cmd_defrag) {
+                case GF_DEFRAG_CMD_NONE:
+                default:
+                    continue;
+            }
+        check:
+            if (brickinfo->status != GF_BRICK_STARTED) {
+                snprintf(msg, sizeof(msg),
+                         "Found stopped "
+                         "brick %s. Use force option to "
+                         "remove the offline brick",
+                         brick);
+                gf_smsg(
+                    this->name, GF_LOG_ERROR, errno, GD_MSG_BRICK_STOPPED,
+                    "Use 'force' option to remove the offline brick, Brick=%s",
+                    brick, NULL);
+                *errstr = gf_strdup(msg);
                 ret = -1;
                 goto out;
-        }
-
-        ret = dict_get_int32 (dict, "command", &flag);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "Unable to get brick command");
-                goto out;
-        }
-        cmd = flag;
-
-        ret = dict_get_int32 (dict, "count", &brick_count);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "Unable to get brick count");
-                goto out;
-        }
-
-        ret = 0;
-        if (volinfo->brick_count == brick_count) {
-                errstr = gf_strdup ("Deleting all the bricks of the "
-                                    "volume is not allowed");
+            }
+            GLUSTERD_GET_BRICK_PIDFILE(pidfile, volinfo, brickinfo, priv);
+            if (!gf_is_service_running(pidfile, &pid)) {
+                snprintf(msg, sizeof(msg),
+                         "Found dead "
+                         "brick %s",
+                         brick);
+                gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_BRICK_DEAD,
+                        "Brick=%s", brick, NULL);
+                *errstr = gf_strdup(msg);
                 ret = -1;
                 goto out;
-        }
+            } else {
+                ret = 0;
+            }
+            continue;
+        }
+
+        RCU_READ_LOCK;
+        peerinfo = glusterd_peerinfo_find_by_uuid(brickinfo->uuid);
+        if (!peerinfo) {
+            RCU_READ_UNLOCK;
+            snprintf(msg, sizeof(msg),
+                     "Host node of the "
+                     "brick %s is not in cluster",
+                     brick);
+            gf_smsg(this->name, GF_LOG_ERROR, errno,
+                    GD_MSG_BRICK_HOST_NOT_FOUND, "Brick=%s", brick, NULL);
+            *errstr = gf_strdup(msg);
+            ret = -1;
+            goto out;
+        }
+        if (!peerinfo->connected) {
+            RCU_READ_UNLOCK;
+            snprintf(msg, sizeof(msg),
+                     "Host node of the "
+                     "brick %s is down",
+                     brick);
+            gf_smsg(this->name, GF_LOG_ERROR, 0, GD_MSG_BRICK_HOST_DOWN,
+                    "Brick=%s", brick, NULL);
+            *errstr = gf_strdup(msg);
+            ret = -1;
+            goto out;
+        }
+        RCU_READ_UNLOCK;
+    }
 
+out:
+    return ret;
+}
+
+int
+glusterd_op_stage_remove_brick(dict_t *dict, char **op_errstr)
+{
+    int ret = -1;
+    char *volname = NULL;
+    glusterd_volinfo_t *volinfo = NULL;
+    char *errstr = NULL;
+    int32_t brick_count = 0;
+    char msg[2048] = "";
+    int32_t flag = 0;
+    gf1_op_commands cmd = GF_OP_CMD_NONE;
+    char *task_id_str = NULL;
+    xlator_t *this = NULL;
+    gsync_status_param_t param = {
+        0,
+    };
+
+    this = THIS;
+    GF_ASSERT(this);
+
+    ret = op_version_check(this, GD_OP_VER_PERSISTENT_AFR_XATTRS, msg,
+                           sizeof(msg));
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_OP_VERSION_MISMATCH, "%s",
+               msg);
+        *op_errstr = gf_strdup(msg);
+        goto out;
+    }
+
+    ret = dict_get_strn(dict, "volname", SLEN("volname"), &volname);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_SET_FAILED,
+               "Unable to get volume name");
+        goto out;
+    }
+
+    ret = glusterd_volinfo_find(volname, &volinfo);
+
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_VOL_NOT_FOUND,
+               "Volume %s does not exist", volname);
+        goto out;
+    }
+
+    ret = glusterd_validate_volume_id(dict, volinfo);
+    if (ret)
+        goto out;
+
+    ret = dict_get_int32n(dict, "command", SLEN("command"), &flag);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_GET_FAILED,
+               "Unable to get brick command");
+        goto out;
+    }
+    cmd = flag;
+
+    ret = dict_get_int32n(dict, "count", SLEN("count"), &brick_count);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_GET_FAILED,
+               "Unable to get brick count");
+        goto out;
+    }
+
+    ret = 0;
+    if (volinfo->brick_count == brick_count) {
+        errstr = gf_strdup(
+            "Deleting all the bricks of the "
+            "volume is not allowed");
+        gf_smsg(this->name, GF_LOG_ERROR, 0, GD_MSG_BRICK_DELETE, NULL);
         ret = -1;
-        switch (cmd) {
+        goto out;
+    }
+
+    ret = -1;
+    switch (cmd) {
         case GF_OP_CMD_NONE:
-                errstr = gf_strdup ("no remove-brick command issued");
-                goto out;
+            errstr = gf_strdup("no remove-brick command issued");
+            gf_smsg(this->name, GF_LOG_ERROR, 0, GD_MSG_BRICK_NO_REMOVE_CMD,
+                    NULL);
+            goto out;
 
         case GF_OP_CMD_STATUS:
-                ret = 0;
-                goto out;
-
-        case GF_OP_CMD_START:
-        {
-                if ((volinfo->type == GF_CLUSTER_TYPE_REPLICATE) &&
-                    dict_get (dict, "replica-count")) {
-                        snprintf (msg, sizeof(msg), "Migration of data is not "
-                                  "needed when reducing replica count. Use the"
-                                  " 'force' option");
-                        errstr = gf_strdup (msg);
-                        gf_log (this->name, GF_LOG_ERROR, "%s", errstr);
-                        goto out;
-                }
-
-                if (GLUSTERD_STATUS_STARTED != volinfo->status) {
-                        snprintf (msg, sizeof (msg), "Volume %s needs to be "
-                                  "started before remove-brick (you can use "
-                                  "'force' or 'commit' to override this "
-                                  "behavior)", volinfo->volname);
-                        errstr = gf_strdup (msg);
-                        gf_log (this->name, GF_LOG_ERROR, "%s", errstr);
-                        goto out;
-                }
-                if (!gd_is_remove_brick_committed (volinfo)) {
-                        snprintf (msg, sizeof (msg), "An earlier remove-brick "
-                                  "task exists for volume %s. Either commit it"
-                                  " or stop it before starting a new task.",
-                                  volinfo->volname);
-                        errstr = gf_strdup (msg);
-                        gf_log (this->name, GF_LOG_ERROR, "Earlier remove-brick"
-                                " task exists for volume %s.",
-                                volinfo->volname);
-                        goto out;
-                }
-                if (glusterd_is_defrag_on(volinfo)) {
-                        errstr = gf_strdup("Rebalance is in progress. Please "
-                                           "retry after completion");
-                        gf_log (this->name, GF_LOG_ERROR, "%s", errstr);
-                        goto out;
-                }
-
-                /* Check if the connected clients are all of version
-                 * glusterfs-3.6 and higher. This is needed to prevent some data
-                 * loss issues that could occur when older clients are connected
-                 * when rebalance is run.
-                 */
-                ret = glusterd_check_client_op_version_support
-                        (volname, GD_OP_VERSION_3_6_0, NULL);
+            ret = 0;
+            goto out;
+        case GF_OP_CMD_START: {
+            if ((volinfo->type == GF_CLUSTER_TYPE_REPLICATE) &&
+                dict_getn(dict, "replica-count", SLEN("replica-count"))) {
+                snprintf(msg, sizeof(msg),
+                         "Migration of data is not "
+                         "needed when reducing replica count. Use the"
+                         " 'force' option");
+                errstr = gf_strdup(msg);
+                gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_USE_THE_FORCE, "%s",
+                       errstr);
+                goto out;
+            }
+
+            if (GLUSTERD_STATUS_STARTED != volinfo->status) {
+                snprintf(msg, sizeof(msg),
+                         "Volume %s needs "
+                         "to be started before remove-brick "
+                         "(you can use 'force' or 'commit' "
+                         "to override this behavior)",
+                         volinfo->volname);
+                errstr = gf_strdup(msg);
+                gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_VOL_NOT_STARTED,
+                       "%s", errstr);
+                goto out;
+            }
+            if (!gd_is_remove_brick_committed(volinfo)) {
+                snprintf(msg, sizeof(msg),
+                         "An earlier remove-brick "
+                         "task exists for volume %s. Either commit it"
+                         " or stop it before starting a new task.",
+                         volinfo->volname);
+                errstr = gf_strdup(msg);
+                gf_msg(this->name, GF_LOG_ERROR, 0,
+                       GD_MSG_OLD_REMOVE_BRICK_EXISTS,
+                       "Earlier remove-brick"
+                       " task exists for volume %s.",
+                       volinfo->volname);
+                goto out;
+            }
+            if (glusterd_is_defrag_on(volinfo)) {
+                errstr = gf_strdup(
+                    "Rebalance is in progress. Please "
+                    "retry after completion");
+                gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_OIP_RETRY_LATER,
+                       "%s", errstr);
+                goto out;
+            }
+
+            /* Check if the connected clients are all of version
+             * glusterfs-3.6 and higher. This is needed to prevent some data
+             * loss issues that could occur when older clients are connected
+             * when rebalance is run.
+             */
+            ret = glusterd_check_client_op_version_support(
+                volname, GD_OP_VERSION_3_6_0, NULL);
+            if (ret) {
+                ret = gf_asprintf(op_errstr,
+                                  "Volume %s has one or "
+                                  "more connected clients of a version"
+                                  " lower than GlusterFS-v3.6.0. "
+                                  "Starting remove-brick in this state "
+                                  "could lead to data loss.\nPlease "
+                                  "disconnect those clients before "
+                                  "attempting this command again.",
+                                  volname);
+                goto out;
+            }
+
+            if (volinfo->snap_count > 0 ||
+                !cds_list_empty(&volinfo->snap_volumes)) {
+                snprintf(msg, sizeof(msg),
+                         "Volume %s  has %" PRIu64
+                         " snapshots. "
+                         "Changing the volume configuration will not effect "
+                         "snapshots."
+                         "But the snapshot brick mount should be intact to "
+                         "make them function.",
+                         volname, volinfo->snap_count);
+                gf_msg("glusterd", GF_LOG_WARNING, 0, GD_MSG_SNAP_WARN, "%s",
+                       msg);
+                msg[0] = '\0';
+            }
+
+            ret = glusterd_remove_brick_validate_bricks(
+                cmd, brick_count, dict, volinfo, &errstr, GF_DEFRAG_CMD_NONE);
+            if (ret)
+                goto out;
+
+            if (is_origin_glusterd(dict)) {
+                ret = glusterd_generate_and_set_task_id(
+                    dict, GF_REMOVE_BRICK_TID_KEY,
+                    SLEN(GF_REMOVE_BRICK_TID_KEY));
                 if (ret) {
-                        ret = gf_asprintf (op_errstr, "Volume %s has one or "
-                                           "more connected clients of a version"
-                                           " lower than GlusterFS-v3.6.0. "
-                                           "Starting remove-brick in this state "
-                                           "could lead to data loss.\nPlease "
-                                           "disconnect those clients before "
-                                           "attempting this command again.",
-                                           volname);
-                        goto out;
+                    gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_TASKID_GEN_FAIL,
+                           "Failed to generate task-id");
+                    goto out;
                 }
-
-                if (is_origin_glusterd (dict)) {
-                        ret = glusterd_generate_and_set_task_id
-                                (dict, GF_REMOVE_BRICK_TID_KEY);
-                        if (ret) {
-                                gf_log (this->name, GF_LOG_ERROR,
-                                        "Failed to generate task-id");
-                                goto out;
-                        }
-                } else {
-                        ret = dict_get_str (dict, GF_REMOVE_BRICK_TID_KEY,
-                                            &task_id_str);
-                        if (ret) {
-                                gf_log (this->name, GF_LOG_WARNING,
-                                        "Missing remove-brick-id");
-                                ret = 0;
-                        }
+            } else {
+                ret = dict_get_strn(dict, GF_REMOVE_BRICK_TID_KEY,
+                                    SLEN(GF_REMOVE_BRICK_TID_KEY),
+                                    &task_id_str);
+                if (ret) {
+                    gf_msg(this->name, GF_LOG_WARNING, errno,
+                           GD_MSG_DICT_GET_FAILED, "Missing remove-brick-id");
+                    ret = 0;
                 }
-                break;
+            }
+            break;
         }
 
         case GF_OP_CMD_STOP:
-                ret = 0;
-                break;
+            ret = 0;
+            break;
 
         case GF_OP_CMD_COMMIT:
-                if (volinfo->decommission_in_progress) {
-                        errstr = gf_strdup ("use 'force' option as migration "
-                                            "is in progress");
-                        goto out;
+            if (volinfo->decommission_in_progress) {
+                errstr = gf_strdup(
+                    "use 'force' option as migration "
+                    "is in progress");
+                gf_smsg(this->name, GF_LOG_WARNING, 0, GD_MSG_MIGRATION_PROG,
+                        "Use 'force' option", NULL);
+                goto out;
+            }
+
+            if (volinfo->rebal.defrag_status == GF_DEFRAG_STATUS_FAILED) {
+                errstr = gf_strdup(
+                    "use 'force' option as migration "
+                    "has failed");
+                gf_smsg(this->name, GF_LOG_WARNING, 0, GD_MSG_MIGRATION_FAIL,
+                        "Use 'force' option", NULL);
+                goto out;
+            }
+
+            if (volinfo->rebal.defrag_status == GF_DEFRAG_STATUS_COMPLETE) {
+                if (volinfo->rebal.rebalance_failures > 0 ||
+                    volinfo->rebal.skipped_files > 0) {
+                    errstr = gf_strdup(
+                        "use 'force' option as migration "
+                        "of some files might have been skipped or "
+                        "has failed");
+                    gf_smsg(this->name, GF_LOG_WARNING, 0,
+                            GD_MSG_MIGRATION_FAIL,
+                            "Use 'force' option, some files might have been "
+                            "skipped",
+                            NULL);
+                    goto out;
                 }
+            }
 
-                /* Do not allow commit if the bricks are not decommissioned */
-                for ( i = 1; i <= brick_count; i++ ) {
-                        snprintf (key, sizeof (key), "brick%d", i);
-                        ret = dict_get_str (dict, key, &brick);
-                        if (ret) {
-                                snprintf (msg, sizeof (msg),
-                                          "Unable to get %s", key);
-                                errstr = gf_strdup (msg);
-                                goto out;
-                        }
-
-                        ret =
-                        glusterd_volume_brickinfo_get_by_brick(brick, volinfo,
-                                                               &brickinfo);
-                        if (ret) {
-                                snprintf (msg, sizeof (msg), "Incorrect brick "
-                                          "%s for volume %s", brick, volname);
-                                errstr = gf_strdup (msg);
-                                goto out;
-                        }
-                        if ( !brickinfo->decommissioned ) {
-                                snprintf (msg, sizeof (msg), "Brick %s "
-                                          "is not decommissioned. "
-                                          "Use start or force option",
-                                          brick);
-                                errstr = gf_strdup (msg);
-                                ret = -1;
-                                goto out;
-                        }
-                }
+            ret = glusterd_remove_brick_validate_bricks(
+                cmd, brick_count, dict, volinfo, &errstr, GF_DEFRAG_CMD_NONE);
+            if (ret)
+                goto out;
 
-                /* If geo-rep is configured, for this volume, it should be
-                 * stopped.
-                 */
-                param.volinfo = volinfo;
-                ret = glusterd_check_geo_rep_running (&param, op_errstr);
-                if (ret || param.is_active) {
-                        ret = -1;
-                        goto out;
-                }
+            /* If geo-rep is configured, for this volume, it should be
+             * stopped.
+             */
+            param.volinfo = volinfo;
+            ret = glusterd_check_geo_rep_running(&param, op_errstr);
+            if (ret || param.is_active) {
+                ret = -1;
+                goto out;
+            }
 
-                break;
+            break;
 
-        case GF_OP_CMD_DETACH:
         case GF_OP_CMD_COMMIT_FORCE:
-                break;
-        }
-        ret = 0;
+        case GF_OP_CMD_DETACH_START:
+        case GF_OP_CMD_DETACH_COMMIT:
+        case GF_OP_CMD_DETACH_COMMIT_FORCE:
+        case GF_OP_CMD_STOP_DETACH_TIER:
+            break;
+    }
+    ret = 0;
 
 out:
-        gf_log (this->name, GF_LOG_DEBUG, "Returning %d", ret);
-        if (ret && errstr) {
-                if (op_errstr)
-                        *op_errstr = errstr;
-        }
-
-        return ret;
+    gf_msg_debug(this->name, 0, "Returning %d", ret);
+    if (ret && errstr) {
+        if (op_errstr)
+            *op_errstr = errstr;
+    }
+    if (!op_errstr && errstr)
+        GF_FREE(errstr);
+    return ret;
 }
 
 int
-glusterd_remove_brick_migrate_cbk (glusterd_volinfo_t *volinfo,
-                                   gf_defrag_status_t status)
+glusterd_remove_brick_migrate_cbk(glusterd_volinfo_t *volinfo,
+                                  gf_defrag_status_t status)
 {
-        int                   ret = 0;
+    int ret = 0;
 
-#if 0  /* TODO: enable this behavior once cluster-wide awareness comes for
-          defrag cbk function */
+#if 0 /* TODO: enable this behavior once cluster-wide awareness comes for      \
+         defrag cbk function */
         glusterd_brickinfo_t *brickinfo = NULL;
         glusterd_brickinfo_t *tmp = NULL;
 
@@ -1826,533 +2155,642 @@ glusterd_remove_brick_migrate_cbk (glusterd_volinfo_t *volinfo,
 
 #endif
 
-        volinfo->decommission_in_progress = 0;
-        return ret;
+    volinfo->decommission_in_progress = 0;
+    return ret;
 }
 
-static int
-glusterd_op_perform_attach_tier (dict_t *dict,
-                                 glusterd_volinfo_t *volinfo,
-                                 int count,
-                                 char *bricks)
+int
+glusterd_op_add_brick(dict_t *dict, char **op_errstr)
 {
-        int                                     ret = 0;
-        int                                     replica_count = 0;
-        int                                     type = 0;
-
+    int ret = 0;
+    char *volname = NULL;
+    glusterd_conf_t *priv = NULL;
+    glusterd_volinfo_t *volinfo = NULL;
+    xlator_t *this = NULL;
+    char *bricks = NULL;
+    int32_t count = 0;
+
+    this = THIS;
+    GF_ASSERT(this);
+
+    priv = this->private;
+    GF_ASSERT(priv);
+
+    ret = dict_get_strn(dict, "volname", SLEN("volname"), &volname);
+
+    if (ret) {
+        gf_msg("glusterd", GF_LOG_ERROR, errno, GD_MSG_DICT_GET_FAILED,
+               "Unable to get volume name");
+        goto out;
+    }
+
+    ret = glusterd_volinfo_find(volname, &volinfo);
+
+    if (ret) {
+        gf_msg("glusterd", GF_LOG_ERROR, EINVAL, GD_MSG_VOL_NOT_FOUND,
+               "Unable to allocate memory");
+        goto out;
+    }
+
+    ret = dict_get_int32n(dict, "count", SLEN("count"), &count);
+    if (ret) {
+        gf_msg("glusterd", GF_LOG_ERROR, errno, GD_MSG_DICT_GET_FAILED,
+               "Unable to get count");
+        goto out;
+    }
+
+    ret = dict_get_strn(dict, "bricks", SLEN("bricks"), &bricks);
+    if (ret) {
+        gf_msg("glusterd", GF_LOG_ERROR, errno, GD_MSG_DICT_GET_FAILED,
+               "Unable to get bricks");
+        goto out;
+    }
+
+    ret = glusterd_op_perform_add_bricks(volinfo, count, bricks, dict);
+    if (ret) {
+        gf_msg("glusterd", GF_LOG_ERROR, 0, GD_MSG_BRICK_ADD_FAIL,
+               "Unable to add bricks");
+        goto out;
+    }
+    if (priv->op_version <= GD_OP_VERSION_3_7_5) {
+        ret = glusterd_store_volinfo(volinfo,
+                                     GLUSTERD_VOLINFO_VER_AC_INCREMENT);
+        if (ret)
+            goto out;
+    } else {
         /*
-         * Store the new (cold) tier's structure until the graph is generated.
-         * If there is a failure before the graph is generated the
-         * structure will revert to its original state.
+         * The cluster is operating at version greater than
+         * gluster-3.7.5. So no need to store volfiles
+         * in commit phase, the same will be done
+         * in post validate phase with v3 framework.
          */
-        volinfo->tier_info.cold_dist_leaf_count = volinfo->dist_leaf_count;
-        volinfo->tier_info.cold_type           = volinfo->type;
-        volinfo->tier_info.cold_brick_count    = volinfo->brick_count;
-        volinfo->tier_info.cold_replica_count  = volinfo->replica_count;
-        volinfo->tier_info.cold_disperse_count = volinfo->disperse_count;
+    }
 
-        ret = dict_get_int32 (dict, "replica-count", &replica_count);
-        if (!ret)
-                volinfo->tier_info.hot_replica_count  = replica_count;
-        else
-                volinfo->tier_info.hot_replica_count  = 1;
-        volinfo->tier_info.hot_brick_count     = count;
-        ret = dict_get_int32 (dict, "type", &type);
-        volinfo->tier_info.hot_type      = type;
-        ret = dict_set_int32 (dict, "type", GF_CLUSTER_TYPE_TIER);
+    if (GLUSTERD_STATUS_STARTED == volinfo->status)
+        ret = glusterd_svcs_manager(volinfo);
 
-        return ret;
+out:
+    return ret;
 }
 
 int
-glusterd_op_add_brick (dict_t *dict, char **op_errstr)
+glusterd_post_commit_add_brick(dict_t *dict, char **op_errstr)
 {
-        int                                     ret = 0;
-        char                                    *volname = NULL;
-        glusterd_conf_t                         *priv = NULL;
-        glusterd_volinfo_t                      *volinfo = NULL;
-        xlator_t                                *this = NULL;
-        char                                    *bricks = NULL;
-        int32_t                                 count = 0;
-        int32_t                                 replica_count = 0;
-
-        this = THIS;
-        GF_ASSERT (this);
-
-        priv = this->private;
-        GF_ASSERT (priv);
-
-        ret = dict_get_str (dict, "volname", &volname);
-
-        if (ret) {
-                gf_log ("", GF_LOG_ERROR, "Unable to get volume name");
-                goto out;
-        }
+    int ret = 0;
+    char *volname = NULL;
 
-        ret = glusterd_volinfo_find (volname, &volinfo);
-
-        if (ret) {
-                gf_log ("", GF_LOG_ERROR, "Unable to allocate memory");
-                goto out;
-        }
-
-        ret = dict_get_int32 (dict, "count", &count);
-        if (ret) {
-                gf_log ("", GF_LOG_ERROR, "Unable to get count");
-                goto out;
-        }
-
-
-        ret = dict_get_str (dict, "bricks", &bricks);
-        if (ret) {
-                gf_log ("", GF_LOG_ERROR, "Unable to get bricks");
-                goto out;
-        }
-
-        if (dict_get(dict, "attach-tier")) {
-                gf_log (THIS->name, GF_LOG_DEBUG, "Adding tier");
-                glusterd_op_perform_attach_tier (dict, volinfo, count, bricks);
-        }
-
-        ret = glusterd_op_perform_add_bricks (volinfo, count, bricks, dict);
-        if (ret) {
-                gf_log ("", GF_LOG_ERROR, "Unable to add bricks");
-                goto out;
-        }
-
-        ret = glusterd_store_volinfo (volinfo, GLUSTERD_VOLINFO_VER_AC_INCREMENT);
-        if (ret)
-                goto out;
-
-        if (GLUSTERD_STATUS_STARTED == volinfo->status)
-                ret = glusterd_svcs_manager (volinfo);
+    ret = dict_get_strn(dict, "volname", SLEN("volname"), &volname);
 
+    if (ret) {
+        gf_msg(THIS->name, GF_LOG_ERROR, errno, GD_MSG_DICT_GET_FAILED,
+               "Unable to get volume name");
+        goto out;
+    }
+    ret = glusterd_replace_old_auth_allow_list(volname);
 out:
-        return ret;
+    return ret;
 }
 
-static void
-glusterd_op_perform_detach_tier (glusterd_volinfo_t *volinfo)
+int
+glusterd_post_commit_replace_brick(dict_t *dict, char **op_errstr)
 {
-        volinfo->type           = volinfo->tier_info.cold_type;
-        volinfo->replica_count  = volinfo->tier_info.cold_replica_count;
-        volinfo->disperse_count = volinfo->tier_info.cold_disperse_count;
+    int ret = 0;
+    char *volname = NULL;
+
+    ret = dict_get_strn(dict, "volname", SLEN("volname"), &volname);
+
+    if (ret) {
+        gf_msg(THIS->name, GF_LOG_ERROR, errno, GD_MSG_DICT_GET_FAILED,
+               "Unable to get volume name");
+        goto out;
+    }
+    ret = glusterd_replace_old_auth_allow_list(volname);
+out:
+    return ret;
 }
 
 int
-glusterd_op_remove_brick (dict_t *dict, char **op_errstr)
+glusterd_set_rebalance_id_for_remove_brick(dict_t *req_dict, dict_t *rsp_dict)
 {
-        int                     ret            = -1;
-        char                    *volname       = NULL;
-        glusterd_volinfo_t      *volinfo       = NULL;
-        char                    *brick         = NULL;
-        int32_t                 count          = 0;
-        int32_t                 i              = 1;
-        char                    key[256]       = {0,};
-        int32_t                 flag           = 0;
-        char                    err_str[4096]  = {0,};
-        int                     need_rebalance = 0;
-        int                     force          = 0;
-        gf1_op_commands         cmd            = 0;
-        int32_t                 replica_count  = 0;
-        glusterd_brickinfo_t    *brickinfo     = NULL;
-        glusterd_brickinfo_t    *tmp           = NULL;
-        char                    *task_id_str   = NULL;
-        xlator_t                *this          = NULL;
-        dict_t                  *bricks_dict   = NULL;
-        char                    *brick_tmpstr  = NULL;
-
-        this = THIS;
-        GF_ASSERT (this);
-
-        ret = dict_get_str (dict, "volname", &volname);
-
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "Unable to get volume name");
-                goto out;
-        }
-
-        ret = glusterd_volinfo_find (volname, &volinfo);
+    int ret = -1;
+    char *volname = NULL;
+    glusterd_volinfo_t *volinfo = NULL;
+    char msg[2048] = {0};
+    char *task_id_str = NULL;
+    xlator_t *this = NULL;
+    int32_t cmd = 0;
+
+    this = THIS;
+    GF_ASSERT(this);
+
+    GF_ASSERT(rsp_dict);
+    GF_ASSERT(req_dict);
+
+    ret = dict_get_strn(rsp_dict, "volname", SLEN("volname"), &volname);
+    if (ret) {
+        gf_msg_debug(this->name, 0, "volname not found");
+        goto out;
+    }
+
+    ret = glusterd_volinfo_find(volname, &volinfo);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, EINVAL, GD_MSG_VOL_NOT_FOUND,
+               "Unable to allocate memory");
+        goto out;
+    }
+
+    ret = dict_get_int32n(rsp_dict, "command", SLEN("command"), &cmd);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_GET_FAILED,
+               "Unable to get command");
+        goto out;
+    }
+
+    /* remove brick task id is generted in glusterd_op_stage_remove_brick(),
+     * but rsp_dict is unavailable there. So copying it to rsp_dict from
+     * req_dict here. */
+
+    if (is_origin_glusterd(rsp_dict)) {
+        ret = dict_get_strn(req_dict, GF_REMOVE_BRICK_TID_KEY,
+                            SLEN(GF_REMOVE_BRICK_TID_KEY), &task_id_str);
         if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "Unable to allocate memory");
-                goto out;
-        }
-
-        ret = dict_get_int32 (dict, "command", &flag);
+            snprintf(msg, sizeof(msg), "Missing rebalance id for remove-brick");
+            gf_msg(this->name, GF_LOG_WARNING, 0, GD_MSG_REBALANCE_ID_MISSING,
+                   "%s", msg);
+            ret = 0;
+        } else {
+            gf_uuid_parse(task_id_str, volinfo->rebal.rebalance_id);
+
+            ret = glusterd_copy_uuid_to_dict(volinfo->rebal.rebalance_id,
+                                             rsp_dict, GF_REMOVE_BRICK_TID_KEY,
+                                             SLEN(GF_REMOVE_BRICK_TID_KEY));
+            if (ret) {
+                gf_msg(this->name, GF_LOG_ERROR, 0,
+                       GD_MSG_REMOVE_BRICK_ID_SET_FAIL,
+                       "Failed to set remove-brick-id");
+                goto out;
+            }
+        }
+    }
+    if (!gf_uuid_is_null(volinfo->rebal.rebalance_id) &&
+        GD_OP_REMOVE_BRICK == volinfo->rebal.op) {
+        ret = glusterd_copy_uuid_to_dict(volinfo->rebal.rebalance_id, rsp_dict,
+                                         GF_REMOVE_BRICK_TID_KEY,
+                                         SLEN(GF_REMOVE_BRICK_TID_KEY));
         if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "Unable to get command");
-                goto out;
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+                   "Failed to set task-id for volume %s", volname);
+            goto out;
         }
-        cmd = flag;
-
-        /* Set task-id, if available, in ctx dict for operations other than
-         * start
-         */
-        if (is_origin_glusterd (dict) && (cmd != GF_OP_CMD_START)) {
-                if (!gf_uuid_is_null (volinfo->rebal.rebalance_id)) {
-                        ret = glusterd_copy_uuid_to_dict
-                                (volinfo->rebal.rebalance_id, dict,
-                                 GF_REMOVE_BRICK_TID_KEY);
-                        if (ret) {
-                                gf_log (this->name, GF_LOG_ERROR,
-                                        "Failed to set remove-brick-id");
-                                goto out;
-                        }
-                }
-        }
-
-        /* Clear task-id, rebal.op and stored bricks on commmitting/stopping
-         * remove-brick */
-        if ((cmd != GF_OP_CMD_START) || (cmd != GF_OP_CMD_STATUS)) {
-                gf_uuid_clear (volinfo->rebal.rebalance_id);
-                volinfo->rebal.op = GD_OP_NONE;
-                dict_unref (volinfo->rebal.dict);
-                volinfo->rebal.dict = NULL;
-        }
-
-        ret = -1;
-        switch (cmd) {
+    }
+out:
+    return ret;
+}
+int
+glusterd_op_remove_brick(dict_t *dict, char **op_errstr)
+{
+    int ret = -1;
+    char *volname = NULL;
+    glusterd_volinfo_t *volinfo = NULL;
+    char *brick = NULL;
+    int32_t count = 0;
+    int32_t i = 1;
+    char key[64] = "";
+    int keylen;
+    int32_t flag = 0;
+    int need_rebalance = 0;
+    int force = 0;
+    gf1_op_commands cmd = 0;
+    int32_t replica_count = 0;
+    char *task_id_str = NULL;
+    xlator_t *this = NULL;
+    dict_t *bricks_dict = NULL;
+    char *brick_tmpstr = NULL;
+    int start_remove = 0;
+    uint32_t commit_hash = 0;
+    int defrag_cmd = 0;
+    glusterd_conf_t *conf = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+    conf = this->private;
+    GF_VALIDATE_OR_GOTO(this->name, conf, out);
+
+    ret = dict_get_strn(dict, "volname", SLEN("volname"), &volname);
+
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_BRICK_ADD_FAIL,
+               "Unable to get volume name");
+        goto out;
+    }
+
+    ret = glusterd_volinfo_find(volname, &volinfo);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, EINVAL, GD_MSG_VOL_NOT_FOUND,
+               "Unable to allocate memory");
+        goto out;
+    }
+
+    ret = dict_get_int32n(dict, "command", SLEN("command"), &flag);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_GET_FAILED,
+               "Unable to get command");
+        goto out;
+    }
+    cmd = flag;
+
+    if (GF_OP_CMD_START == cmd)
+        start_remove = 1;
+
+    /* Set task-id, if available, in ctx dict for operations other than
+     * start
+     */
+
+    if (is_origin_glusterd(dict) && (!start_remove)) {
+        if (!gf_uuid_is_null(volinfo->rebal.rebalance_id)) {
+            ret = glusterd_copy_uuid_to_dict(volinfo->rebal.rebalance_id, dict,
+                                             GF_REMOVE_BRICK_TID_KEY,
+                                             SLEN(GF_REMOVE_BRICK_TID_KEY));
+            if (ret) {
+                gf_msg(this->name, GF_LOG_ERROR, 0,
+                       GD_MSG_REMOVE_BRICK_ID_SET_FAIL,
+                       "Failed to set remove-brick-id");
+                goto out;
+            }
+        }
+    }
+
+    /* Clear task-id, rebal.op and stored bricks on commmitting/stopping
+     * remove-brick */
+    if ((!start_remove) && (cmd != GF_OP_CMD_STATUS)) {
+        gf_uuid_clear(volinfo->rebal.rebalance_id);
+        volinfo->rebal.op = GD_OP_NONE;
+        dict_unref(volinfo->rebal.dict);
+        volinfo->rebal.dict = NULL;
+    }
+
+    ret = -1;
+    switch (cmd) {
         case GF_OP_CMD_NONE:
-                goto out;
+            goto out;
 
         case GF_OP_CMD_STATUS:
-                ret = 0;
-                goto out;
+            ret = 0;
+            goto out;
 
         case GF_OP_CMD_STOP:
-        {
-                /* Fall back to the old volume file */
-                cds_list_for_each_entry_safe (brickinfo, tmp, &volinfo->bricks,
-                                              brick_list) {
-                        if (!brickinfo->decommissioned)
-                                continue;
-                        brickinfo->decommissioned = 0;
-                }
-                ret = glusterd_create_volfiles_and_notify_services (volinfo);
-                if (ret) {
-                        gf_log (this->name, GF_LOG_WARNING,
-                                "failed to create volfiles");
-                        goto out;
-                }
-
-                ret = glusterd_store_volinfo (volinfo,
-                                             GLUSTERD_VOLINFO_VER_AC_INCREMENT);
-                if (ret) {
-                        gf_log (this->name, GF_LOG_WARNING,
-                                "failed to store volinfo");
-                        goto out;
-                }
-
-                ret = 0;
-                goto out;
-        }
-
         case GF_OP_CMD_START:
-                /* Reset defrag status to 'NOT STARTED' whenever a
-                 * remove-brick/rebalance command is issued to remove
-                 * stale information from previous run.
-                 */
-                volinfo->rebal.defrag_status = GF_DEFRAG_STATUS_NOT_STARTED;
-                ret = dict_get_str (dict, GF_REMOVE_BRICK_TID_KEY, &task_id_str);
-                if (ret) {
-                        gf_log (this->name, GF_LOG_DEBUG,
-                                "Missing remove-brick-id");
-                        ret = 0;
-                } else {
-                        gf_uuid_parse (task_id_str, volinfo->rebal.rebalance_id) ;
-                        volinfo->rebal.op = GD_OP_REMOVE_BRICK;
-                }
-                force = 0;
-                break;
+            /* Reset defrag status to 'NOT STARTED' whenever a
+             * remove-brick/rebalance command is issued to remove
+             * stale information from previous run.
+             * Update defrag_cmd as well or it will only be done
+             * for nodes on which the brick to be removed exists.
+             */
+            /* coverity[MIXED_ENUMS] */
+            volinfo->rebal.defrag_cmd = cmd;
+            volinfo->rebal.defrag_status = GF_DEFRAG_STATUS_NOT_STARTED;
+            ret = dict_get_strn(dict, GF_REMOVE_BRICK_TID_KEY,
+                                SLEN(GF_REMOVE_BRICK_TID_KEY), &task_id_str);
+            if (ret) {
+                gf_msg_debug(this->name, errno, "Missing remove-brick-id");
+                ret = 0;
+            } else {
+                gf_uuid_parse(task_id_str, volinfo->rebal.rebalance_id);
+                volinfo->rebal.op = GD_OP_REMOVE_BRICK;
+            }
+            force = 0;
+            break;
 
         case GF_OP_CMD_COMMIT:
-                force = 1;
-                break;
-
-        case GF_OP_CMD_DETACH:
-                glusterd_op_perform_detach_tier (volinfo);
-                /* fall through */
+            force = 1;
+            break;
 
         case GF_OP_CMD_COMMIT_FORCE:
 
-                if (volinfo->decommission_in_progress) {
-                        if (volinfo->rebal.defrag) {
-                                LOCK (&volinfo->rebal.defrag->lock);
-                                /* Fake 'rebalance-complete' so the graph change
-                                   happens right away */
-                                volinfo->rebal.defrag_status =
-                                                GF_DEFRAG_STATUS_COMPLETE;
+            if (volinfo->decommission_in_progress) {
+                if (volinfo->rebal.defrag) {
+                    LOCK(&volinfo->rebal.defrag->lock);
+                    /* Fake 'rebalance-complete' so the graph change
+                       happens right away */
+                    volinfo->rebal.defrag_status = GF_DEFRAG_STATUS_COMPLETE;
 
-                                UNLOCK (&volinfo->rebal.defrag->lock);
-                        }
-                        /* Graph change happens in rebalance _cbk function,
-                           no need to do anything here */
-                        /* TODO: '_cbk' function is not doing anything for now */
+                    UNLOCK(&volinfo->rebal.defrag->lock);
                 }
-
-                ret = 0;
-                force = 1;
-                break;
+                /* Graph change happens in rebalance _cbk function,
+                   no need to do anything here */
+                /* TODO: '_cbk' function is not doing anything for now */
+            }
+
+            ret = 0;
+            force = 1;
+            break;
+        case GF_OP_CMD_DETACH_START:
+        case GF_OP_CMD_DETACH_COMMIT_FORCE:
+        case GF_OP_CMD_DETACH_COMMIT:
+        case GF_OP_CMD_STOP_DETACH_TIER:
+            break;
+    }
+
+    ret = dict_get_int32n(dict, "count", SLEN("count"), &count);
+    if (ret) {
+        gf_msg("glusterd", GF_LOG_ERROR, errno, GD_MSG_DICT_GET_FAILED,
+               "Unable to get count");
+        goto out;
+    }
+    /* Save the list of bricks for later usage only on starting a
+     * remove-brick. Right now this is required for displaying the task
+     * parameters with task status in volume status.
+     */
+
+    if (start_remove) {
+        bricks_dict = dict_new();
+        if (!bricks_dict) {
+            ret = -1;
+            goto out;
+        }
+        ret = dict_set_int32n(bricks_dict, "count", SLEN("count"), count);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_SET_FAILED,
+                   "Failed to save remove-brick count");
+            goto out;
         }
+    }
 
-        ret = dict_get_int32 (dict, "count", &count);
+    while (i <= count) {
+        keylen = snprintf(key, sizeof(key), "brick%d", i);
+        ret = dict_get_strn(dict, key, keylen, &brick);
         if (ret) {
-                gf_log ("", GF_LOG_ERROR, "Unable to get count");
-                goto out;
+            gf_msg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_GET_FAILED,
+                   "Unable to get %s", key);
+            goto out;
         }
 
-        /* Save the list of bricks for later usage only on starting a
-         * remove-brick. Right now this is required for displaying the task
-         * parameters with task status in volume status.
-         */
-        if (GF_OP_CMD_START == cmd) {
-                bricks_dict = dict_new ();
-                if (!bricks_dict) {
-                        ret = -1;
-                        goto out;
-                }
-                ret = dict_set_int32 (bricks_dict, "count", count);
-                if (ret) {
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "Failed to save remove-brick count");
-                        goto out;
-                }
+        if (start_remove) {
+            brick_tmpstr = gf_strdup(brick);
+            if (!brick_tmpstr) {
+                ret = -1;
+                gf_msg(this->name, GF_LOG_ERROR, ENOMEM, GD_MSG_NO_MEMORY,
+                       "Failed to duplicate brick name");
+                goto out;
+            }
+            ret = dict_set_dynstrn(bricks_dict, key, keylen, brick_tmpstr);
+            if (ret) {
+                gf_msg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_SET_FAILED,
+                       "Failed to add brick to dict");
+                goto out;
+            }
+            brick_tmpstr = NULL;
         }
-        while ( i <= count) {
-                snprintf (key, 256, "brick%d", i);
-                ret = dict_get_str (dict, key, &brick);
-                if (ret) {
-                        gf_log (this->name, GF_LOG_ERROR, "Unable to get %s",
-                                key);
-                        goto out;
-                }
-
-                if (GF_OP_CMD_START == cmd) {
-                        brick_tmpstr = gf_strdup (brick);
-                        if (!brick_tmpstr) {
-                                ret = -1;
-                                gf_log (this->name, GF_LOG_ERROR,
-                                        "Failed to duplicate brick name");
-                                goto out;
-                        }
-                        ret = dict_set_dynstr (bricks_dict, key, brick_tmpstr);
-                        if (ret) {
-                                gf_log (this->name, GF_LOG_ERROR,
-                                        "Failed to add brick to dict");
-                                goto out;
-                        }
-                        brick_tmpstr = NULL;
-                }
 
-                ret = glusterd_op_perform_remove_brick (volinfo, brick, force,
-                                                        &need_rebalance);
-                if (ret)
-                        goto out;
-                i++;
-        }
-        if (GF_OP_CMD_START == cmd)
-                volinfo->rebal.dict = dict_ref (bricks_dict);
-
-        volinfo->subvol_count = (volinfo->brick_count /
-                                 volinfo->dist_leaf_count);
-        ret = dict_get_int32 (dict, "replica-count", &replica_count);
-        if (!ret) {
-                gf_log (this->name, GF_LOG_INFO,
-                        "changing replica count %d to %d on volume %s",
-                        volinfo->replica_count, replica_count,
-                        volinfo->volname);
-                volinfo->replica_count = replica_count;
-                volinfo->sub_count = replica_count;
-                volinfo->dist_leaf_count = glusterd_get_dist_leaf_count (volinfo);
-
-                /*
-                 * volinfo->type and sub_count have already been set for
-                 * volumes undergoing a detach operation, they should not
-                 * be modified here.
-                 */
-                if ((replica_count == 1) && (cmd != GF_OP_CMD_DETACH)) {
-                        if (volinfo->type == GF_CLUSTER_TYPE_REPLICATE) {
-                                volinfo->type = GF_CLUSTER_TYPE_NONE;
-                                /* backward compatibility */
-                                volinfo->sub_count = 0;
-                        } else {
-                                volinfo->type = GF_CLUSTER_TYPE_STRIPE;
-                                /* backward compatibility */
-                                volinfo->sub_count = volinfo->dist_leaf_count;
-                        }
-                }
-        }
+        ret = glusterd_op_perform_remove_brick(volinfo, brick, force,
+                                               &need_rebalance);
+        if (ret)
+            goto out;
+        i++;
+    }
+
+    if (start_remove)
+        volinfo->rebal.dict = dict_ref(bricks_dict);
+
+    ret = dict_get_int32n(dict, "replica-count", SLEN("replica-count"),
+                          &replica_count);
+    if (!ret) {
+        gf_msg(this->name, GF_LOG_INFO, errno, GD_MSG_DICT_GET_FAILED,
+               "changing replica count %d to %d on volume %s",
+               volinfo->replica_count, replica_count, volinfo->volname);
+        volinfo->replica_count = replica_count;
+        /* A reduction in replica count implies an arbiter volume
+         * earlier is now no longer one. */
+        if (volinfo->arbiter_count)
+            volinfo->arbiter_count = 0;
+        volinfo->sub_count = replica_count;
+        volinfo->dist_leaf_count = glusterd_get_dist_leaf_count(volinfo);
 
-        ret = glusterd_create_volfiles_and_notify_services (volinfo);
+        /*
+         * volinfo->type and sub_count have already been set for
+         * volumes undergoing a detach operation, they should not
+         * be modified here.
+         */
+        if (replica_count == 1) {
+            if (volinfo->type == GF_CLUSTER_TYPE_REPLICATE) {
+                volinfo->type = GF_CLUSTER_TYPE_NONE;
+                /* backward compatibility */
+                volinfo->sub_count = 0;
+            }
+        }
+    }
+    volinfo->subvol_count = (volinfo->brick_count / volinfo->dist_leaf_count);
+
+    if (!glusterd_is_volume_replicate(volinfo) &&
+        conf->op_version >= GD_OP_VERSION_3_12_2) {
+        ret = dict_set_nstrn(volinfo->dict, "performance.client-io-threads",
+                             SLEN("performance.client-io-threads"), "on",
+                             SLEN("on"));
         if (ret) {
-                gf_log (this->name, GF_LOG_WARNING, "failed to create volfiles");
-                goto out;
-        }
-
-        ret = glusterd_store_volinfo (volinfo, GLUSTERD_VOLINFO_VER_AC_INCREMENT);
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+                   "Failed to set "
+                   "performance.client-io-threads to on");
+            goto out;
+        }
+    }
+
+    ret = glusterd_create_volfiles_and_notify_services(volinfo);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_WARNING, 0, GD_MSG_VOLFILE_CREATE_FAIL,
+               "failed to create volfiles");
+        goto out;
+    }
+
+    ret = glusterd_store_volinfo(volinfo, GLUSTERD_VOLINFO_VER_AC_INCREMENT);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_WARNING, 0, GD_MSG_VOLINFO_STORE_FAIL,
+               "failed to store volinfo");
+        goto out;
+    }
+
+    if (start_remove && volinfo->status == GLUSTERD_STATUS_STARTED) {
+        ret = glusterd_svcs_reconfigure(volinfo);
         if (ret) {
-                gf_log (this->name, GF_LOG_WARNING, "failed to store volinfo");
-                goto out;
-        }
-
-        if (GF_OP_CMD_START == cmd &&
-                        volinfo->status == GLUSTERD_STATUS_STARTED) {
-                ret = glusterd_svcs_reconfigure (volinfo);
-                if (ret) {
-                        gf_log (this->name, GF_LOG_WARNING,
-                               "Unable to reconfigure NFS-Server");
-                goto out;
-                }
+            gf_msg(this->name, GF_LOG_WARNING, 0, GD_MSG_NFS_RECONF_FAIL,
+                   "Unable to reconfigure NFS-Server");
+            goto out;
         }
+    }
 
-        /* Need to reset the defrag/rebalance status accordingly */
-        switch (volinfo->rebal.defrag_status) {
+    /* Need to reset the defrag/rebalance status accordingly */
+    switch (volinfo->rebal.defrag_status) {
         case GF_DEFRAG_STATUS_FAILED:
         case GF_DEFRAG_STATUS_COMPLETE:
-                volinfo->rebal.defrag_status = 0;
+            volinfo->rebal.defrag_status = 0;
+        /* FALLTHROUGH */
         default:
-                break;
-        }
-        if (!force && need_rebalance) {
-                /* perform the rebalance operations */
-                ret = glusterd_handle_defrag_start
-                        (volinfo, err_str, sizeof (err_str),
-                         GF_DEFRAG_CMD_START_FORCE,
-                         glusterd_remove_brick_migrate_cbk, GD_OP_REMOVE_BRICK);
-
-                if (!ret)
-                        volinfo->decommission_in_progress = 1;
-
-                if (ret) {
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "failed to start the rebalance");
-                }
-        } else {
-                if (GLUSTERD_STATUS_STARTED == volinfo->status)
-                        ret = glusterd_svcs_manager (volinfo);
-        }
+            break;
+    }
+    if (!force && need_rebalance) {
+        if (dict_get_uint32(dict, "commit-hash", &commit_hash) == 0) {
+            volinfo->rebal.commit_hash = commit_hash;
+        }
+        /* perform the rebalance operations */
+        defrag_cmd = GF_DEFRAG_CMD_START_FORCE;
+        /*
+         * We need to set this *before* we issue commands to the
+         * bricks, or else we might end up setting it after the bricks
+         * have responded.  If we fail to send the request(s) we'll
+         * clear it ourselves because nobody else will.
+         */
+        volinfo->decommission_in_progress = 1;
+        char err_str[4096] = "";
+        ret = glusterd_handle_defrag_start(
+            volinfo, err_str, sizeof(err_str), defrag_cmd,
+            glusterd_remove_brick_migrate_cbk, GD_OP_REMOVE_BRICK);
 
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_REBALANCE_START_FAIL,
+                   "failed to start the rebalance");
+            /* TBD: shouldn't we do more than print a message? */
+            volinfo->decommission_in_progress = 0;
+            if (op_errstr)
+                *op_errstr = gf_strdup(err_str);
+        }
+    } else {
+        if (GLUSTERD_STATUS_STARTED == volinfo->status)
+            ret = glusterd_svcs_manager(volinfo);
+    }
 out:
-        if (ret && err_str[0] && op_errstr)
-                *op_errstr = gf_strdup (err_str);
-
-        GF_FREE (brick_tmpstr);
-        if (bricks_dict)
-                dict_unref (bricks_dict);
-
-        return ret;
+    GF_FREE(brick_tmpstr);
+    if (bricks_dict)
+        dict_unref(bricks_dict);
+    gf_msg_debug(this->name, 0, "returning %d ", ret);
+    return ret;
 }
 
 int
-glusterd_op_stage_barrier (dict_t *dict, char **op_errstr)
+glusterd_op_stage_barrier(dict_t *dict, char **op_errstr)
 {
-        int                  ret         = -1;
-        xlator_t             *this       = NULL;
-        char                 *volname    = NULL;
-        glusterd_volinfo_t   *vol        = NULL;
-
-        GF_ASSERT (dict);
-        this = THIS;
-        GF_ASSERT (this);
-
-        ret = dict_get_str (dict, "volname", &volname);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "Volname not present in "
-                        "dict");
-                goto out;
-        }
-
-        ret = glusterd_volinfo_find (volname, &vol);
-        if (ret) {
-                gf_asprintf (op_errstr, "Volume %s does not exist", volname);
-                gf_log (this->name, GF_LOG_ERROR, "%s", *op_errstr);
-                goto out;
-        }
-
-        if (!glusterd_is_volume_started (vol)) {
-                gf_asprintf (op_errstr, "Volume %s is not started", volname);
-                ret = -1;
-                goto out;
-        }
-
-        ret = dict_get_str_boolean (dict, "barrier", -1);
-        if (ret == -1) {
-                gf_asprintf (op_errstr, "Barrier op for volume %s not present "
-                             "in dict", volname);
-                gf_log (this->name, GF_LOG_ERROR, "%s", *op_errstr);
-                goto out;
-        }
-        ret = 0;
+    int ret = -1;
+    xlator_t *this = NULL;
+    char *volname = NULL;
+    glusterd_volinfo_t *vol = NULL;
+    char *barrier_op = NULL;
+
+    GF_ASSERT(dict);
+    this = THIS;
+    GF_ASSERT(this);
+
+    ret = dict_get_strn(dict, "volname", SLEN("volname"), &volname);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_GET_FAILED,
+               "Volname not present in "
+               "dict");
+        goto out;
+    }
+
+    ret = glusterd_volinfo_find(volname, &vol);
+    if (ret) {
+        gf_asprintf(op_errstr, "Volume %s does not exist", volname);
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_VOL_NOT_FOUND, "%s",
+               *op_errstr);
+        goto out;
+    }
+
+    if (!glusterd_is_volume_started(vol)) {
+        gf_asprintf(op_errstr, "Volume %s is not started", volname);
+        ret = -1;
+        goto out;
+    }
+
+    ret = dict_get_strn(dict, "barrier", SLEN("barrier"), &barrier_op);
+    if (ret == -1) {
+        gf_asprintf(op_errstr,
+                    "Barrier op for volume %s not present "
+                    "in dict",
+                    volname);
+        gf_msg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_GET_FAILED, "%s",
+               *op_errstr);
+        goto out;
+    }
+    ret = 0;
 out:
-        gf_log (this->name, GF_LOG_DEBUG, "Returning %d", ret);
-        return ret;
+    gf_msg_debug(this->name, 0, "Returning %d", ret);
+    return ret;
 }
 
 int
-glusterd_op_barrier (dict_t *dict, char **op_errstr)
+glusterd_op_barrier(dict_t *dict, char **op_errstr)
 {
-        int                  ret         = -1;
-        xlator_t             *this       = NULL;
-        char                 *volname    = NULL;
-        glusterd_volinfo_t   *vol        = NULL;
-        char                 *barrier_op = NULL;
-
-        GF_ASSERT (dict);
-        this = THIS;
-        GF_ASSERT (this);
-
-        ret = dict_get_str (dict, "volname", &volname);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "Volname not present in "
-                        "dict");
-                goto out;
-        }
-
-        ret = glusterd_volinfo_find (volname, &vol);
-        if (ret) {
-                gf_asprintf (op_errstr, "Volume %s does not exist", volname);
-                gf_log (this->name, GF_LOG_ERROR, "%s", *op_errstr);
-                goto out;
-        }
-
-        ret = dict_get_str (dict, "barrier", &barrier_op);
-        if (ret) {
-                gf_asprintf (op_errstr, "Barrier op for volume %s not present "
-                             "in dict", volname);
-                gf_log (this->name, GF_LOG_ERROR, "%s", *op_errstr);
-                goto out;
-        }
-
-        ret = dict_set_dynstr_with_alloc (vol->dict, "features.barrier",
-                                          barrier_op);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "Failed to set barrier op in"
-                        " volume option dict");
-                goto out;
-        }
-
-        gd_update_volume_op_versions (vol);
-        ret = glusterd_create_volfiles (vol);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "Failed to create volfiles");
-                goto out;
-        }
-        ret = glusterd_store_volinfo (vol, GLUSTERD_VOLINFO_VER_AC_INCREMENT);
+    int ret = -1;
+    xlator_t *this = NULL;
+    char *volname = NULL;
+    glusterd_volinfo_t *vol = NULL;
+    char *barrier_op = NULL;
+
+    GF_ASSERT(dict);
+    this = THIS;
+    GF_ASSERT(this);
+
+    ret = dict_get_strn(dict, "volname", SLEN("volname"), &volname);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_GET_FAILED,
+               "Volname not present in "
+               "dict");
+        goto out;
+    }
+
+    ret = glusterd_volinfo_find(volname, &vol);
+    if (ret) {
+        gf_asprintf(op_errstr, "Volume %s does not exist", volname);
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_VOL_NOT_FOUND, "%s",
+               *op_errstr);
+        goto out;
+    }
+
+    ret = dict_get_strn(dict, "barrier", SLEN("barrier"), &barrier_op);
+    if (ret) {
+        gf_asprintf(op_errstr,
+                    "Barrier op for volume %s not present "
+                    "in dict",
+                    volname);
+        gf_msg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_GET_FAILED, "%s",
+               *op_errstr);
+        goto out;
+    }
+
+    ret = dict_set_dynstr_with_alloc(vol->dict, "features.barrier", barrier_op);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_SET_FAILED,
+               "Failed to set barrier op in"
+               " volume option dict");
+        goto out;
+    }
+
+    gd_update_volume_op_versions(vol);
+    ret = glusterd_create_volfiles(vol);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_VOLFILE_CREATE_FAIL,
+               "Failed to create volfiles");
+        goto out;
+    }
+    ret = glusterd_store_volinfo(vol, GLUSTERD_VOLINFO_VER_AC_INCREMENT);
 
 out:
-        gf_log (this->name, GF_LOG_DEBUG, "Returning %d", ret);
-        return ret;
+    gf_msg_debug(this->name, 0, "Returning %d", ret);
+    return ret;
+}
+
+int
+glusterd_handle_add_tier_brick(rpcsvc_request_t *req)
+{
+    return 0;
 }
 
 int
-glusterd_handle_attach_tier (rpcsvc_request_t *req)
+glusterd_handle_attach_tier(rpcsvc_request_t *req)
 {
-        return glusterd_big_locked_handler (req, __glusterd_handle_add_brick);
+    return 0;
 }
 
 int
-glusterd_handle_detach_tier (rpcsvc_request_t *req)
+glusterd_handle_detach_tier(rpcsvc_request_t *req)
 {
-        return glusterd_big_locked_handler (req,
-                                            __glusterd_handle_remove_brick);
+    return 0;
 }
diff --git a/xlators/mgmt/glusterd/src/glusterd-conn-helper.c b/xlators/mgmt/glusterd/src/glusterd-conn-helper.c
index bfa9d02aa1b..a7f54ec24b7 100644
--- a/xlators/mgmt/glusterd/src/glusterd-conn-helper.c
+++ b/xlators/mgmt/glusterd/src/glusterd-conn-helper.c
@@ -15,7 +15,7 @@
 #include <urcu/rculist.h>
 
 glusterd_svc_t *
-glusterd_conn_get_svc_object (glusterd_conn_t *conn)
+glusterd_conn_get_svc_object(glusterd_conn_t *conn)
 {
-        return cds_list_entry (conn, glusterd_svc_t, conn);
+    return cds_list_entry(conn, glusterd_svc_t, conn);
 }
diff --git a/xlators/mgmt/glusterd/src/glusterd-conn-helper.h b/xlators/mgmt/glusterd/src/glusterd-conn-helper.h
index 2431c316fd4..6f500309175 100644
--- a/xlators/mgmt/glusterd/src/glusterd-conn-helper.h
+++ b/xlators/mgmt/glusterd/src/glusterd-conn-helper.h
@@ -13,14 +13,9 @@
 
 #include "rpc-clnt.h"
 
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
-
 #include "glusterd-conn-mgmt.h"
 
 glusterd_svc_t *
-glusterd_conn_get_svc_object (glusterd_conn_t *conn);
+glusterd_conn_get_svc_object(glusterd_conn_t *conn);
 
 #endif
diff --git a/xlators/mgmt/glusterd/src/glusterd-conn-mgmt.c b/xlators/mgmt/glusterd/src/glusterd-conn-mgmt.c
index 662aba6e724..5c01f0c70b6 100644
--- a/xlators/mgmt/glusterd/src/glusterd-conn-mgmt.c
+++ b/xlators/mgmt/glusterd/src/glusterd-conn-mgmt.c
@@ -8,128 +8,184 @@
    cases as published by the Free Software Foundation.
 */
 
-#include "xlator.h"
+#include <glusterfs/xlator.h>
 #include "rpc-clnt.h"
 #include "glusterd.h"
 #include "glusterd-conn-mgmt.h"
 #include "glusterd-conn-helper.h"
 #include "glusterd-utils.h"
+#include "glusterd-messages.h"
 
 int
-glusterd_conn_init (glusterd_conn_t *conn, char *sockpath,
-                    int frame_timeout, glusterd_conn_notify_t notify)
+glusterd_conn_init(glusterd_conn_t *conn, char *sockpath, int frame_timeout,
+                   glusterd_conn_notify_t notify)
 {
-        int                  ret      = -1;
-        dict_t              *options  = NULL;
-        struct rpc_clnt     *rpc      = NULL;
-        xlator_t            *this     = THIS;
-        glusterd_svc_t      *svc      = NULL;
-
-        if (!this)
-                goto out;
-
-        svc = glusterd_conn_get_svc_object (conn);
-        if (!svc) {
-                gf_log (this->name, GF_LOG_ERROR, "Failed to get the service");
-                goto out;
-        }
-
-        ret = rpc_transport_unix_options_build (&options, sockpath,
-                                                frame_timeout);
-        if (ret)
-                goto out;
-
-        ret = dict_set_str (options, "transport.socket.ignore-enoent", "on");
-        if (ret)
-                goto out;
-
-        /* @options is free'd by rpc_transport when destroyed */
-        rpc = rpc_clnt_new (options, this->ctx, (char *)svc->name, 16);
-        if (!rpc) {
-                ret = -1;
-                goto out;
-        }
-
-        ret = rpc_clnt_register_notify (rpc, glusterd_conn_common_notify,
-                                        conn);
-        if (ret)
-                goto out;
-
-        ret = snprintf (conn->sockpath, sizeof (conn->sockpath), "%s",
-                        sockpath);
-        if (ret < 0)
-                goto out;
-        else
-                ret = 0;
-
-        conn->frame_timeout = frame_timeout;
-        conn->rpc = rpc;
-        conn->notify = notify;
+    int ret = -1;
+    dict_t *options = NULL;
+    struct rpc_clnt *rpc = NULL;
+    xlator_t *this = THIS;
+    glusterd_svc_t *svc = NULL;
+
+    if (!this) {
+        gf_smsg(THIS->name, GF_LOG_ERROR, errno, GD_MSG_XLATOR_NOT_DEFINED,
+                NULL);
+        goto out;
+    }
+
+    options = dict_new();
+    if (!options) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_CREATE_FAIL, NULL);
+        goto out;
+    }
+
+    svc = glusterd_conn_get_svc_object(conn);
+    if (!svc) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_SVC_GET_FAIL,
+               "Failed to get the service");
+        goto out;
+    }
+
+    ret = rpc_transport_unix_options_build(options, sockpath, frame_timeout);
+    if (ret)
+        goto out;
+
+    ret = dict_set_int32n(options, "transport.socket.ignore-enoent",
+                          SLEN("transport.socket.ignore-enoent"), 1);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_SET_FAILED,
+                "Key=transport.socket.ignore-enoent", NULL);
+        goto out;
+    }
+
+    /* @options is free'd by rpc_transport when destroyed */
+    rpc = rpc_clnt_new(options, this, (char *)svc->name, 16);
+    if (!rpc) {
+        ret = -1;
+        goto out;
+    }
+
+    ret = rpc_clnt_register_notify(rpc, glusterd_conn_common_notify, conn);
+    if (ret)
+        goto out;
+
+    ret = snprintf(conn->sockpath, sizeof(conn->sockpath), "%s", sockpath);
+    if (ret < 0) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_COPY_FAIL, NULL);
+        goto out;
+    } else
+        ret = 0;
+
+    conn->frame_timeout = frame_timeout;
+    conn->rpc = rpc;
+    conn->notify = notify;
 out:
-        if (ret) {
-                if (rpc) {
-                        rpc_clnt_unref (rpc);
-                        rpc = NULL;
-                }
+    if (options)
+        dict_unref(options);
+    if (ret) {
+        if (rpc) {
+            rpc_clnt_unref(rpc);
+            rpc = NULL;
         }
-        return ret;
+    }
+    return ret;
 }
 
 int
-glusterd_conn_term (glusterd_conn_t *conn)
+glusterd_conn_term(glusterd_conn_t *conn)
 {
-        rpc_clnt_disable (conn->rpc);
-        rpc_clnt_unref (conn->rpc);
-        return 0;
+    rpc_clnt_unref(conn->rpc);
+    return 0;
 }
 
 int
-glusterd_conn_connect (glusterd_conn_t *conn)
+glusterd_conn_connect(glusterd_conn_t *conn)
 {
-        return rpc_clnt_start (conn->rpc);
+    return rpc_clnt_start(conn->rpc);
 }
 
 int
-glusterd_conn_disconnect (glusterd_conn_t *conn)
+glusterd_conn_disconnect(glusterd_conn_t *conn)
 {
-        rpc_clnt_disconnect (conn->rpc);
+    rpc_clnt_disable(conn->rpc);
 
-        return 0;
+    return 0;
 }
 
-
 int
-__glusterd_conn_common_notify (struct rpc_clnt *rpc, void *mydata,
-                               rpc_clnt_event_t event, void *data)
+__glusterd_conn_common_notify(struct rpc_clnt *rpc, void *mydata,
+                              rpc_clnt_event_t event, void *data)
 {
-        glusterd_conn_t *conn = mydata;
+    glusterd_conn_t *conn = mydata;
 
-        /* Silently ignoring this error, exactly like the current
-         * implementation */
-        if (!conn)
-                return 0;
+    /* Silently ignoring this error, exactly like the current
+     * implementation */
+    if (!conn)
+        return 0;
 
-        return conn->notify (conn, event);
+    return conn->notify(conn, event);
 }
 
 int
-glusterd_conn_common_notify (struct rpc_clnt *rpc, void *mydata,
-                             rpc_clnt_event_t event, void *data)
+glusterd_conn_common_notify(struct rpc_clnt *rpc, void *mydata,
+                            rpc_clnt_event_t event, void *data)
 {
-        return glusterd_big_locked_notify
-                                (rpc, mydata, event, data,
-                                 __glusterd_conn_common_notify);
+    return glusterd_big_locked_notify(rpc, mydata, event, data,
+                                      __glusterd_conn_common_notify);
 }
 
 int32_t
-glusterd_conn_build_socket_filepath (char *rundir, uuid_t uuid,
-                                     char *socketpath, int len)
+glusterd_conn_build_socket_filepath(char *rundir, uuid_t uuid, char *socketpath,
+                                    int len)
 {
-        char sockfilepath[PATH_MAX] = {0,};
+    char sockfilepath[PATH_MAX] = {
+        0,
+    };
+
+    snprintf(sockfilepath, sizeof(sockfilepath), "%s/run-%s", rundir,
+             uuid_utoa(uuid));
 
-        snprintf (sockfilepath, sizeof (sockfilepath), "%s/run-%s",
-                  rundir, uuid_utoa (uuid));
+    glusterd_set_socket_filepath(sockfilepath, socketpath, len);
+    return 0;
+}
+
+int
+__glusterd_muxsvc_conn_common_notify(struct rpc_clnt *rpc, void *mydata,
+                                     rpc_clnt_event_t event, void *data)
+{
+    glusterd_conf_t *conf = THIS->private;
+    glusterd_svc_proc_t *mux_proc = mydata;
+    int ret = -1;
 
-        glusterd_set_socket_filepath (sockfilepath, socketpath, len);
+    /* Silently ignoring this error, exactly like the current
+     * implementation */
+    if (!mux_proc)
         return 0;
+
+    if (event == RPC_CLNT_DESTROY) {
+        /*RPC_CLNT_DESTROY will only called after mux_proc detached from the
+         * list. So it is safe to call without lock. Processing
+         * RPC_CLNT_DESTROY under a lock will lead to deadlock.
+         */
+        if (mux_proc->data) {
+            glusterd_volinfo_unref(mux_proc->data);
+            mux_proc->data = NULL;
+        }
+        GF_FREE(mux_proc);
+        ret = 0;
+    } else {
+        pthread_mutex_lock(&conf->attach_lock);
+        {
+            ret = mux_proc->notify(mux_proc, event);
+        }
+        pthread_mutex_unlock(&conf->attach_lock);
+    }
+    return ret;
+}
+
+int
+glusterd_muxsvc_conn_common_notify(struct rpc_clnt *rpc, void *mydata,
+                                   rpc_clnt_event_t event, void *data)
+{
+    return glusterd_big_locked_notify(rpc, mydata, event, data,
+                                      __glusterd_muxsvc_conn_common_notify);
 }
diff --git a/xlators/mgmt/glusterd/src/glusterd-conn-mgmt.h b/xlators/mgmt/glusterd/src/glusterd-conn-mgmt.h
index 6a058e36c94..1b225621ab1 100644
--- a/xlators/mgmt/glusterd/src/glusterd-conn-mgmt.h
+++ b/xlators/mgmt/glusterd/src/glusterd-conn-mgmt.h
@@ -13,44 +13,41 @@
 
 #include "rpc-clnt.h"
 
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
-
 typedef struct glusterd_conn_ glusterd_conn_t;
 
-typedef int (*glusterd_conn_notify_t)
-                (glusterd_conn_t *conn, rpc_clnt_event_t event);
+typedef int (*glusterd_conn_notify_t)(glusterd_conn_t *conn,
+                                      rpc_clnt_event_t event);
 
 struct glusterd_conn_ {
-        struct rpc_clnt *rpc;
-        char sockpath[PATH_MAX];
-        int frame_timeout;
-        /* Existing daemons tend to specialize their respective
-         * notify implementations, so ... */
-        glusterd_conn_notify_t notify;
+    struct rpc_clnt *rpc;
+    /* Existing daemons tend to specialize their respective
+     * notify implementations, so ... */
+    glusterd_conn_notify_t notify;
+    int frame_timeout;
+    char sockpath[PATH_MAX];
 };
 
 int
-glusterd_conn_init (glusterd_conn_t *conn, char *sockpath,
-                    int frame_timeout, glusterd_conn_notify_t notify);
+glusterd_conn_init(glusterd_conn_t *conn, char *sockpath, int frame_timeout,
+                   glusterd_conn_notify_t notify);
 
 int
-glusterd_conn_term (glusterd_conn_t *conn);
+glusterd_conn_term(glusterd_conn_t *conn);
 
 int
-glusterd_conn_connect (glusterd_conn_t *conn);
+glusterd_conn_connect(glusterd_conn_t *conn);
 
 int
-glusterd_conn_disconnect (glusterd_conn_t *conn);
+glusterd_conn_disconnect(glusterd_conn_t *conn);
 
 int
-glusterd_conn_common_notify (struct rpc_clnt *rpc, void *mydata,
-                             rpc_clnt_event_t event, void *data);
+glusterd_conn_common_notify(struct rpc_clnt *rpc, void *mydata,
+                            rpc_clnt_event_t event, void *data);
+int
+glusterd_muxsvc_conn_common_notify(struct rpc_clnt *rpc, void *mydata,
+                                   rpc_clnt_event_t event, void *data);
 
 int32_t
-glusterd_conn_build_socket_filepath (char *rundir, uuid_t uuid,
-                                     char *socketpath, int len);
-
+glusterd_conn_build_socket_filepath(char *rundir, uuid_t uuid, char *socketpath,
+                                    int len);
 #endif
diff --git a/xlators/mgmt/glusterd/src/glusterd-errno.h b/xlators/mgmt/glusterd/src/glusterd-errno.h
new file mode 100644
index 00000000000..c74070e0e8d
--- /dev/null
+++ b/xlators/mgmt/glusterd/src/glusterd-errno.h
@@ -0,0 +1,33 @@
+/*
+   Copyright (c) 2015 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
+
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
+#ifndef _GLUSTERD_ERRNO_H
+#define _GLUSTERD_ERRNO_H
+
+enum glusterd_op_errno {
+    EG_INTRNL = 30800,    /* Internal Error                    */
+    EG_OPNOTSUP = 30801,  /* Gluster Op Not Supported          */
+    EG_ANOTRANS = 30802,  /* Another Transaction in Progress   */
+    EG_BRCKDWN = 30803,   /* One or more brick is down         */
+    EG_NODEDWN = 30804,   /* One or more node is down          */
+    EG_HRDLMT = 30805,    /* Hard Limit is reached             */
+    EG_NOVOL = 30806,     /* Volume does not exist             */
+    EG_NOSNAP = 30807,    /* Snap does not exist               */
+    EG_RBALRUN = 30808,   /* Rebalance is running              */
+    EG_VOLRUN = 30809,    /* Volume is running                 */
+    EG_VOLSTP = 30810,    /* Volume is not running             */
+    EG_VOLEXST = 30811,   /* Volume exists                     */
+    EG_SNAPEXST = 30812,  /* Snapshot exists                   */
+    EG_ISSNAP = 30813,    /* Volume is a snap volume           */
+    EG_GEOREPRUN = 30814, /* Geo-Replication is running        */
+    EG_NOTTHINP = 30815,  /* Bricks are not thinly provisioned */
+    EG_NOGANESHA = 30816, /* Global ganesha is not enabled   */
+};
+
+#endif
diff --git a/xlators/mgmt/glusterd/src/glusterd-ganesha.c b/xlators/mgmt/glusterd/src/glusterd-ganesha.c
index 534e332cdee..f08bd6cebee 100644
--- a/xlators/mgmt/glusterd/src/glusterd-ganesha.c
+++ b/xlators/mgmt/glusterd/src/glusterd-ganesha.c
@@ -8,576 +8,920 @@
    cases as published by the Free Software Foundation.
 */
 
-
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
-
-#include "common-utils.h"
+#include <glusterfs/common-utils.h>
 #include "glusterd.h"
 #include "glusterd-op-sm.h"
 #include "glusterd-store.h"
 #include "glusterd-utils.h"
-#include "glusterd-nfs-svc.h"
-#define MAXBUF 1024
-#define DELIM "=\""
+#include "glusterd-volgen.h"
+#include "glusterd-messages.h"
+#include <glusterfs/syscall.h>
+
+#include <ctype.h>
+
+int
+start_ganesha(char **op_errstr);
 
 typedef struct service_command {
-        char *binary;
-        char *service;
-        int (*action) (struct service_command *, char *);
+    char *binary;
+    char *service;
+    int (*action)(struct service_command *, char *);
 } service_command;
 
-static int
-sc_systemctl_action (struct service_command *sc, char *command)
+/* parsing_ganesha_ha_conf will allocate the returned string
+ * to be freed (GF_FREE) by the caller
+ * return NULL if error or not found */
+static char *
+parsing_ganesha_ha_conf(const char *key)
 {
-        runner_t        runner   = {0,};
+#define MAX_LINE 1024
+    char scratch[MAX_LINE * 2] = {
+        0,
+    };
+    char *value = NULL, *pointer = NULL, *end_pointer = NULL;
+    FILE *fp;
+
+    fp = fopen(GANESHA_HA_CONF, "r");
+    if (fp == NULL) {
+        gf_msg(THIS->name, GF_LOG_ERROR, errno, GD_MSG_FILE_OP_FAILED,
+               "couldn't open the file %s", GANESHA_HA_CONF);
+        goto end_ret;
+    }
+    while ((pointer = fgets(scratch, MAX_LINE, fp)) != NULL) {
+        /* Read config file until we get matching "^[[:space:]]*key" */
+        if (*pointer == '#') {
+            continue;
+        }
+        while (isblank(*pointer)) {
+            pointer++;
+        }
+        if (strncmp(pointer, key, strlen(key))) {
+            continue;
+        }
+        pointer += strlen(key);
+        /* key found : if we fail to parse, we'll return an error
+         * rather than trying next one
+         * - supposition : conf file is bash compatible : no space
+         *   around the '=' */
+        if (*pointer != '=') {
+            gf_msg(THIS->name, GF_LOG_ERROR, errno,
+                   GD_MSG_GET_CONFIG_INFO_FAILED, "Parsing %s failed at key %s",
+                   GANESHA_HA_CONF, key);
+            goto end_close;
+        }
+        pointer++; /* jump the '=' */
 
-        runinit (&runner);
-        runner_add_args (&runner, sc->binary, command, sc->service, NULL);
-        return runner_run (&runner);
+        if (*pointer == '"' || *pointer == '\'') {
+            /* dont get the quote */
+            pointer++;
+        }
+        end_pointer = pointer;
+        /* stop at the next closing quote or  blank/newline */
+        do {
+            end_pointer++;
+        } while (!(*end_pointer == '\'' || *end_pointer == '"' ||
+                   isspace(*end_pointer) || *end_pointer == '\0'));
+        *end_pointer = '\0';
+
+        /* got it. copy it and return */
+        value = gf_strdup(pointer);
+        break;
+    }
+
+end_close:
+    fclose(fp);
+end_ret:
+    return value;
 }
 
 static int
-sc_service_action (struct service_command *sc, char *command)
+sc_systemctl_action(struct service_command *sc, char *command)
 {
-        runner_t      runner = {0,};
+    runner_t runner = {
+        0,
+    };
 
-        runinit (&runner);
-        runner_add_args (&runner, sc->binary, sc->service, command, NULL);
-        return runner_run (&runner);
+    runinit(&runner);
+    runner_add_args(&runner, sc->binary, command, sc->service, NULL);
+    return runner_run(&runner);
 }
 
 static int
-manage_service (char *action)
+sc_service_action(struct service_command *sc, char *command)
 {
-        struct stat stbuf       = {0,};
-        int     i               = 0;
-        int     ret             = 0;
-        struct service_command sc_list[] = {
-                { .binary  = "/usr/bin/systemctl",
-                  .service = "nfs-ganesha",
-                  .action  = sc_systemctl_action
-                },
-                { .binary  = "/sbin/invoke-rc.d",
-                  .service = "nfs-ganesha",
-                  .action  = sc_service_action
-                },
-                { .binary  = "/sbin/service",
-                  .service = "nfs-ganesha",
-                  .action  = sc_service_action
-                },
-                { .binary  = NULL
-                }
-        };
-
-        while (sc_list[i].binary != NULL) {
-                ret = stat (sc_list[i].binary, &stbuf);
-                if (ret == 0) {
-                        gf_log (THIS->name, GF_LOG_DEBUG,
-                                "%s found.", sc_list[i].binary);
-                        if (strcmp (sc_list[i].binary, "/usr/bin/systemctl") == 0)
-                                ret = sc_systemctl_action (&sc_list[i], action);
-                        else
-                                ret = sc_service_action (&sc_list[i], action);
-
-                        return ret;
-                }
-                i++;
-        }
-        gf_log (THIS->name, GF_LOG_ERROR,
-                "Could not %s NFS-Ganesha.Service manager for distro"
-                " not recognized.", action);
-        return ret;
+    runner_t runner = {
+        0,
+    };
+
+    runinit(&runner);
+    runner_add_args(&runner, sc->binary, sc->service, command, NULL);
+    return runner_run(&runner);
 }
 
-int
-glusterd_check_ganesha_cmd (char *key, char *value, char **errstr, dict_t *dict)
+static int
+manage_service(char *action)
 {
-        int                ret = 0;
-        gf_boolean_t       b   = _gf_false;
-        xlator_t                *this = NULL;
-
-        this = THIS;
-        GF_ASSERT (this);
-        GF_ASSERT (key);
-        GF_ASSERT (value);
-
-        if ((strcmp (key, "ganesha.enable") == 0) ||
-           (strcmp (key, "features.ganesha") == 0)) {
-                ret = gf_string2boolean (value, &b);
-                if (ret < 0) {
-                        gf_log (this->name, GF_LOG_ERROR, "Failed to parse bool"
-                                "string");
-                        goto out;
-                }
-                if ((strcmp (value, "on")) && (strcmp (value, "off"))) {
-                        gf_log (this->name, GF_LOG_ERROR, "Invalid value"
-                                "for volume set command. Use on/off only");
-                        ret = -1;
-                        goto out;
-                }
-                ret = glusterd_handle_ganesha_op (dict, errstr, key, value);
-                if (ret) {
-                        gf_log (this->name, GF_LOG_ERROR, "Handling NFS-Ganesha op"
-                                "failed.");
-                }
+    int i = 0;
+    int ret = 0;
+    struct service_command sc_list[] = {{.binary = "/bin/systemctl",
+                                         .service = "nfs-ganesha",
+                                         .action = sc_systemctl_action},
+                                        {.binary = "/sbin/invoke-rc.d",
+                                         .service = "nfs-ganesha",
+                                         .action = sc_service_action},
+                                        {.binary = "/sbin/service",
+                                         .service = "nfs-ganesha",
+                                         .action = sc_service_action},
+                                        {.binary = NULL}};
+
+    while (sc_list[i].binary != NULL) {
+        ret = sys_access(sc_list[i].binary, X_OK);
+        if (ret == 0) {
+            gf_msg_debug(THIS->name, 0, "%s found.", sc_list[i].binary);
+            return sc_list[i].action(&sc_list[i], action);
         }
-out:
-        return ret;
+        i++;
+    }
+    gf_msg(THIS->name, GF_LOG_ERROR, 0, GD_MSG_UNRECOGNIZED_SVC_MNGR,
+           "Could not %s NFS-Ganesha.Service manager for distro"
+           " not recognized.",
+           action);
+    return ret;
 }
 
-int
-glusterd_op_stage_set_ganesha (dict_t *dict, char **op_errstr)
+/*
+ * Check if the cluster is a ganesha cluster or not *
+ */
+gf_boolean_t
+glusterd_is_ganesha_cluster()
 {
-        int                             ret                     = -1;
-        char                            *volname                = NULL;
-        int                             exists                  = 0;
-        char                            *key                    = NULL;
-        char                            *value                  = NULL;
-        char                            str[100]                = {0, } ;
-        int                             dict_count              = 0;
-        int                             flags                   = 0;
-        char                            errstr[2048]            = {0, } ;
-        glusterd_volinfo_t              *volinfo                = NULL;
-        glusterd_conf_t                 *priv                   = NULL;
-        xlator_t                        *this                   = NULL;
-
-        GF_ASSERT (dict);
-        this = THIS;
-        GF_ASSERT (this);
-        priv = this->private;
-        GF_ASSERT (priv);
-
-        ret = dict_get_str (dict, "key", &key);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR,
-                       "invalid key");
-                goto out;
-        }
+    int ret = -1;
+    glusterd_conf_t *priv = NULL;
+    xlator_t *this = NULL;
+    gf_boolean_t ret_bool = _gf_false;
+
+    this = THIS;
+    GF_VALIDATE_OR_GOTO("ganesha", this, out);
+    priv = this->private;
+    GF_VALIDATE_OR_GOTO(this->name, priv, out);
+
+    ret = dict_get_str_boolean(priv->opts, GLUSTERD_STORE_KEY_GANESHA_GLOBAL,
+                               _gf_false);
+    if (ret == _gf_true) {
+        ret_bool = _gf_true;
+        gf_msg_debug(this->name, 0, "nfs-ganesha is enabled for the cluster");
+    } else
+        gf_msg_debug(this->name, 0, "nfs-ganesha is disabled for the cluster");
 
-        ret = dict_get_str (dict, "value", &value);
-        if (ret) {
-               gf_log (this->name, GF_LOG_ERROR,
-                        "invalid key,value pair in 'global vol set'");
-                goto out;
-        }
 out:
+    return ret_bool;
+}
 
-        if (ret) {
-                if (!(*op_errstr)) {
-                        *op_errstr = gf_strdup ("Error, Validation Failed");
-                        gf_log (this->name, GF_LOG_DEBUG,
-                                "Error, Cannot Validate option :%s %s",
-                                key, value);
-                } else {
-                        gf_log (this->name, GF_LOG_DEBUG,
-                                "Error, Cannot Validate option");
-                }
+/* Check if ganesha.enable is set to 'on', that checks if
+ * a  particular volume is exported via NFS-Ganesha */
+gf_boolean_t
+glusterd_check_ganesha_export(glusterd_volinfo_t *volinfo)
+{
+    char *value = NULL;
+    gf_boolean_t is_exported = _gf_false;
+    int ret = 0;
+
+    ret = glusterd_volinfo_get(volinfo, "ganesha.enable", &value);
+    if ((ret == 0) && value) {
+        if (strcmp(value, "on") == 0) {
+            gf_msg_debug(THIS->name, 0,
+                         "ganesha.enable set"
+                         " to %s",
+                         value);
+            is_exported = _gf_true;
         }
-        return ret;
+    }
+    return is_exported;
 }
 
+/* *
+ * The below function is called as part of commit phase for volume set option
+ * "ganesha.enable". If the value is "on", it creates export configuration file
+ * and then export the volume via dbus command. Incase of "off", the volume
+ * will be already unexported during stage phase, so it will remove the conf
+ * file from shared storage
+ */
 int
-glusterd_op_set_ganesha (dict_t *dict, char **errstr)
+glusterd_check_ganesha_cmd(char *key, char *value, char **errstr, dict_t *dict)
 {
-        int                                      ret = 0;
-        int                                      flags = 0;
-        glusterd_volinfo_t                      *volinfo = NULL;
-        char                                    *volname = NULL;
-        xlator_t                                *this = NULL;
-        glusterd_conf_t                         *priv = NULL;
-        char                                    *key = NULL;
-        char                                    *value = NULL;
-        char                                     str[50] = {0, };
-        int32_t                                  dict_count = 0;
-        dict_t                                  *vol_opts = NULL;
-        int count                                = 0;
-
-        this = THIS;
-        GF_ASSERT (this);
-        GF_ASSERT (dict);
-
-        priv = this->private;
-        GF_ASSERT (priv);
-
-
-        ret = dict_get_str (dict, "key", &key);
-        if (ret) {
-               gf_log (this->name, GF_LOG_ERROR,
-                       "Couldn't get key in global option set");
-                goto out;
-       }
-
-        ret = dict_get_str (dict, "value", &value);
-        if (ret) {
-               gf_log (this->name, GF_LOG_ERROR,
-                        "Couldn't get value in global option set");
-                goto out;
+    int ret = 0;
+    char *volname = NULL;
+
+    GF_ASSERT(key);
+    GF_ASSERT(value);
+    GF_ASSERT(dict);
+
+    if ((strcmp(key, "ganesha.enable") == 0)) {
+        if ((strcmp(value, "on")) && (strcmp(value, "off"))) {
+            gf_asprintf(errstr,
+                        "Invalid value"
+                        " for volume set command. Use on/off only.");
+            ret = -1;
+            goto out;
         }
-
-        ret = glusterd_handle_ganesha_op (dict, errstr, key, value);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR,
-                "Initial NFS-Ganesha set up failed");
-                ret = -1;
+        if (strcmp(value, "on") == 0) {
+            ret = glusterd_handle_ganesha_op(dict, errstr, key, value);
+
+        } else if (is_origin_glusterd(dict)) {
+            ret = dict_get_str(dict, "volname", &volname);
+            if (ret) {
+                gf_msg("glusterd-ganesha", GF_LOG_ERROR, errno,
+                       GD_MSG_DICT_GET_FAILED, "Unable to get volume name");
                 goto out;
+            }
+            ret = manage_export_config(volname, "off", errstr);
         }
-        ret = dict_set_str(priv->opts, "features.ganesha", value);
+    }
+out:
+    if (ret) {
+        gf_msg("glusterd-ganesha", GF_LOG_ERROR, 0,
+               GD_MSG_NFS_GNS_OP_HANDLE_FAIL,
+               "Handling NFS-Ganesha"
+               " op failed.");
+    }
+    return ret;
+}
+
+int
+glusterd_op_stage_set_ganesha(dict_t *dict, char **op_errstr)
+{
+    int ret = -1;
+    char *value = NULL;
+    char *str = NULL;
+    glusterd_conf_t *priv = NULL;
+    xlator_t *this = NULL;
+
+    GF_ASSERT(dict);
+    this = THIS;
+    GF_ASSERT(this);
+    priv = this->private;
+    GF_ASSERT(priv);
+
+    ret = dict_get_str(dict, "value", &value);
+    if (value == NULL) {
+        gf_msg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_GET_FAILED,
+               "value not present.");
+        goto out;
+    }
+    /* This dict_get will fail if the user had never set the key before */
+    /*Ignoring the ret value and proceeding */
+    ret = dict_get_str(priv->opts, GLUSTERD_STORE_KEY_GANESHA_GLOBAL, &str);
+    if (str ? strcmp(value, str) == 0 : strcmp(value, "disable") == 0) {
+        gf_asprintf(op_errstr, "nfs-ganesha is already %sd.", value);
+        ret = -1;
+        goto out;
+    }
+
+    if (strcmp(value, "enable") == 0) {
+        ret = start_ganesha(op_errstr);
         if (ret) {
-                gf_log (this->name, GF_LOG_WARNING, "Failed to set"
-                        " features.ganesha in dict.");
-                goto out;
+            gf_msg(THIS->name, GF_LOG_ERROR, 0, GD_MSG_NFS_GNS_START_FAIL,
+                   "Could not start NFS-Ganesha");
         }
+    } else {
+        ret = stop_ganesha(op_errstr);
+        if (ret)
+            gf_msg_debug(THIS->name, 0,
+                         "Could not stop "
+                         "NFS-Ganesha.");
+    }
 
-        /* To do : Lock the global options file before writing */
-        /* into this file. Bug ID : 1200254    */
+out:
 
-        ret = glusterd_store_options (this, priv->opts);
-        if (ret) {
-             gf_log (this->name, GF_LOG_ERROR,
-                                "Failed to store options");
-                        goto out;
+    if (ret) {
+        if (!(*op_errstr)) {
+            *op_errstr = gf_strdup("Error, Validation Failed");
+            gf_msg_debug(this->name, 0, "Error, Cannot Validate option :%s",
+                         GLUSTERD_STORE_KEY_GANESHA_GLOBAL);
+        } else {
+            gf_msg_debug(this->name, 0, "Error, Cannot Validate option");
         }
+    }
+    return ret;
+}
+
+int
+glusterd_op_set_ganesha(dict_t *dict, char **errstr)
+{
+    int ret = 0;
+    xlator_t *this = NULL;
+    glusterd_conf_t *priv = NULL;
+    char *key = NULL;
+    char *value = NULL;
+    char *next_version = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+    GF_ASSERT(dict);
+
+    priv = this->private;
+    GF_ASSERT(priv);
+
+    ret = dict_get_str(dict, "key", &key);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_GET_FAILED,
+               "Couldn't get key in global option set");
+        goto out;
+    }
+
+    ret = dict_get_str(dict, "value", &value);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_GET_FAILED,
+               "Couldn't get value in global option set");
+        goto out;
+    }
+
+    ret = glusterd_handle_ganesha_op(dict, errstr, key, value);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_NFS_GNS_SETUP_FAIL,
+               "Initial NFS-Ganesha set up failed");
+        ret = -1;
+        goto out;
+    }
+    ret = dict_set_dynstr_with_alloc(priv->opts,
+                                     GLUSTERD_STORE_KEY_GANESHA_GLOBAL, value);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_WARNING, errno, GD_MSG_DICT_SET_FAILED,
+               "Failed to set"
+               " nfs-ganesha in dict.");
+        goto out;
+    }
+    ret = glusterd_get_next_global_opt_version_str(priv->opts, &next_version);
+    if (ret) {
+        gf_msg_debug(THIS->name, 0,
+                     "Could not fetch "
+                     " global op version");
+        goto out;
+    }
+    ret = dict_set_str(priv->opts, GLUSTERD_GLOBAL_OPT_VERSION, next_version);
+    if (ret)
+        goto out;
+
+    ret = glusterd_store_options(this, priv->opts);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_STORE_FAIL,
+               "Failed to store options");
+        goto out;
+    }
 
 out:
-       gf_log (this->name, GF_LOG_DEBUG, "returning %d", ret);
-       return ret;
+    gf_msg_debug(this->name, 0, "returning %d", ret);
+    return ret;
 }
 
-/* Following 2 functions parse GANESHA_HA_CONF
+/* Following function parse GANESHA_HA_CONF
  * The sample file looks like below,
  * HA_NAME="ganesha-ha-360"
  * HA_VOL_NAME="ha-state"
- * HA_VOL_MNT="/mount-point"
- * HA_VOL_SERVER="server1"
  * HA_CLUSTER_NODES="server1,server2"
  * VIP_rhs_1="10.x.x.x"
  * VIP_rhs_2="10.x.x.x." */
 
+/* Check if the localhost is listed as one of nfs-ganesha nodes */
 gf_boolean_t
-is_ganesha_host (void)
+check_host_list(void)
 {
-        char    *host_from_file          = NULL;
-        FILE    *fp;
-        char    line[MAXBUF];
-        gf_boolean_t    ret              = _gf_false;
-        int     i                        = 1;
-        xlator_t        *this            = NULL;
-
-        this = THIS;
-
-        fp = fopen (GANESHA_HA_CONF, "r");
-
-        if (fp == NULL) {
-                gf_log (this->name, GF_LOG_INFO, "couldn't open the file %s",
-                        GANESHA_HA_CONF);
-                return _gf_false;
-        }
-
-         while (fgets (line, sizeof(line), fp) != NULL) {
-                /* Read GANESHA_HA_CONFIG till we find the HA VOL server */
-                host_from_file = strstr ((char *)line, "HA_VOL_SERVER");
-                if (host_from_file != NULL) {
-                        host_from_file = strstr (host_from_file, DELIM);
-                        host_from_file = host_from_file + strlen(DELIM);
-                        i = strlen(host_from_file);
-                        host_from_file[i - 2] = '\0';
-                        break;
-                }
-        }
-
-        ret = gf_is_local_addr (host_from_file);
+    glusterd_conf_t *priv = NULL;
+    char *hostname, *hostlist;
+    gf_boolean_t ret = _gf_false;
+    xlator_t *this = NULL;
+
+    this = THIS;
+    priv = THIS->private;
+    GF_ASSERT(priv);
+
+    hostlist = parsing_ganesha_ha_conf("HA_CLUSTER_NODES");
+    if (hostlist == NULL) {
+        gf_msg(this->name, GF_LOG_INFO, errno, GD_MSG_GET_CONFIG_INFO_FAILED,
+               "couldn't get HA_CLUSTER_NODES from file %s", GANESHA_HA_CONF);
+        return _gf_false;
+    }
+
+    /* Hostlist is a comma separated list now */
+    hostname = strtok(hostlist, ",");
+    while (hostname != NULL) {
+        ret = gf_is_local_addr(hostname);
         if (ret) {
-                gf_log (this->name, GF_LOG_INFO, "ganesha host found "
-                        "Hostname is %s", host_from_file);
+            gf_msg(this->name, GF_LOG_INFO, 0, GD_MSG_NFS_GNS_HOST_FOUND,
+                   "ganesha host found "
+                   "Hostname is %s",
+                   hostname);
+            break;
         }
+        hostname = strtok(NULL, ",");
+    }
 
-        fclose (fp);
-        return ret;
+    GF_FREE(hostlist);
+    return ret;
 }
 
-/* Check if the localhost is listed as one of nfs-ganesha nodes */
-gf_boolean_t
-check_host_list (void)
+int
+gd_ganesha_send_dbus(char *volname, char *value)
 {
-
-        char    *host_from_file          = NULL;
-        glusterd_conf_t     *priv        = NULL;
-        char    *hostname                = NULL;
-        FILE    *fp;
-        char    line[MAXBUF];
-        int     ret                      = _gf_false;
-        int     i                        = 1;
-        xlator_t        *this            = NULL;
-
-        this = THIS;
-        priv =  THIS->private;
-        GF_ASSERT (priv);
-
-        fp = fopen (GANESHA_HA_CONF, "r");
-
-        if (fp == NULL) {
-                gf_log (this->name, GF_LOG_INFO, "couldn't open the file %s",
-                        GANESHA_HA_CONF);
-                return 0;
-        }
-
-       while (fgets (line, sizeof(line), fp) != NULL) {
-        /* Read GANESHA_HA_CONFIG till we find the list of HA_CLUSTER_NODES */
-                hostname = strstr ((char *)line, "HA_CLUSTER_NODES");
-                if (hostname != NULL) {
-                        hostname = strstr (hostname, DELIM);
-                        hostname = hostname + strlen(DELIM);
-                        i = strlen (hostname);
-                        hostname[i - 2] = '\0';
-                        break;
-                }
+    runner_t runner = {
+        0,
+    };
+    int ret = -1;
+    runinit(&runner);
+
+    GF_VALIDATE_OR_GOTO("glusterd-ganesha", volname, out);
+    GF_VALIDATE_OR_GOTO("glusterd-ganesha", value, out);
+
+    ret = 0;
+    if (check_host_list()) {
+        /* Check whether ganesha is running on this node */
+        if (manage_service("status")) {
+            gf_msg("glusterd-ganesha", GF_LOG_WARNING, 0,
+                   GD_MSG_GANESHA_NOT_RUNNING,
+                   "Export failed, NFS-Ganesha is not running");
+        } else {
+            runner_add_args(&runner, GANESHA_PREFIX "/dbus-send.sh", CONFDIR,
+                            value, volname, NULL);
+            ret = runner_run(&runner);
         }
-
-        /* Hostname is a comma separated list now */
-        hostname = strtok (hostname, ",");
-        while (hostname != NULL) {
-                ret = gf_is_local_addr (hostname);
-                if (ret) {
-                        gf_log (this->name, GF_LOG_INFO, "ganesha host found "
-                                "Hostname is %s", hostname);
-                        break;
-                }
-                hostname = strtok (NULL, ",");
-        }
-
-        fclose (fp);
-        return ret;
-
+    }
+out:
+    return ret;
 }
 
 int
-create_export_config (char *volname, char **op_errstr)
+manage_export_config(char *volname, char *value, char **op_errstr)
 {
-        runner_t                runner                     = {0,};
-        int                     ret                        = -1;
-
-        GF_ASSERT(volname);
-        runinit (&runner);
-        runner_add_args (&runner, "sh",
-                        GANESHA_PREFIX"/create-export-ganesha.sh",
-                        CONFDIR, volname, NULL);
-        ret = runner_run(&runner);
+    runner_t runner = {
+        0,
+    };
+    int ret = -1;
+
+    GF_ASSERT(volname);
+    runinit(&runner);
+    runner_add_args(&runner, GANESHA_PREFIX "/create-export-ganesha.sh",
+                    CONFDIR, value, volname, NULL);
+    ret = runner_run(&runner);
+
+    if (ret && op_errstr)
+        gf_asprintf(op_errstr,
+                    "Failed to create"
+                    " NFS-Ganesha export config file.");
+
+    return ret;
+}
 
+/* Exports and unexports a particular volume via NFS-Ganesha */
+int
+ganesha_manage_export(dict_t *dict, char *value,
+                      gf_boolean_t update_cache_invalidation, char **op_errstr)
+{
+    int ret = -1;
+    glusterd_volinfo_t *volinfo = NULL;
+    dict_t *vol_opts = NULL;
+    char *volname = NULL;
+    xlator_t *this = NULL;
+    glusterd_conf_t *priv = NULL;
+    gf_boolean_t option = _gf_false;
+
+    this = THIS;
+    GF_ASSERT(this);
+    priv = this->private;
+
+    GF_ASSERT(value);
+    GF_ASSERT(dict);
+    GF_ASSERT(priv);
+
+    ret = dict_get_str(dict, "volname", &volname);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_GET_FAILED,
+               "Unable to get volume name");
+        goto out;
+    }
+    ret = gf_string2boolean(value, &option);
+    if (ret == -1) {
+        gf_msg(this->name, GF_LOG_ERROR, EINVAL, GD_MSG_INVALID_ENTRY,
+               "invalid value.");
+        goto out;
+    }
+
+    ret = glusterd_volinfo_find(volname, &volinfo);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, EINVAL, GD_MSG_VOL_NOT_FOUND,
+               FMTSTR_CHECK_VOL_EXISTS, volname);
+        goto out;
+    }
+
+    ret = glusterd_check_ganesha_export(volinfo);
+    if (ret && option) {
+        gf_asprintf(op_errstr,
+                    "ganesha.enable "
+                    "is already 'on'.");
+        ret = -1;
+        goto out;
+
+    } else if (!option && !ret) {
+        gf_asprintf(op_errstr,
+                    "ganesha.enable "
+                    "is already 'off'.");
+        ret = -1;
+        goto out;
+    }
+
+    /* Check if global option is enabled, proceed only then */
+    ret = dict_get_str_boolean(priv->opts, GLUSTERD_STORE_KEY_GANESHA_GLOBAL,
+                               _gf_false);
+    if (ret == -1) {
+        gf_msg_debug(this->name, 0,
+                     "Failed to get "
+                     "global option dict.");
+        gf_asprintf(op_errstr,
+                    "The option "
+                    "nfs-ganesha should be "
+                    "enabled before setting ganesha.enable.");
+        goto out;
+    }
+    if (!ret) {
+        gf_asprintf(op_errstr,
+                    "The option "
+                    "nfs-ganesha should be "
+                    "enabled before setting ganesha.enable.");
+        ret = -1;
+        goto out;
+    }
+
+    /* *
+     * Create the export file from the node where ganesha.enable "on"
+     * is executed
+     * */
+    if (option && is_origin_glusterd(dict)) {
+        ret = manage_export_config(volname, "on", op_errstr);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_EXPORT_FILE_CREATE_FAIL,
+                   "Failed to create"
+                   "export file for NFS-Ganesha\n");
+            goto out;
+        }
+    }
+    ret = gd_ganesha_send_dbus(volname, value);
+    if (ret) {
+        gf_asprintf(op_errstr,
+                    "Dynamic export addition/deletion failed."
+                    " Please see log file for details");
+        goto out;
+    }
+    if (update_cache_invalidation) {
+        vol_opts = volinfo->dict;
+        ret = dict_set_dynstr_with_alloc(vol_opts,
+                                         "features.cache-invalidation", value);
         if (ret)
-                gf_asprintf (op_errstr, "Failed to create"
-                            " NFS-Ganesha export config file.");
-
-        return ret;
+            gf_asprintf(op_errstr,
+                        "Cache-invalidation could not"
+                        " be set to %s.",
+                        value);
+        ret = glusterd_store_volinfo(volinfo,
+                                     GLUSTERD_VOLINFO_VER_AC_INCREMENT);
+        if (ret)
+            gf_asprintf(op_errstr, "failed to store volinfo for %s",
+                        volinfo->volname);
+    }
+out:
+    return ret;
 }
 
 int
-ganesha_manage_export (dict_t *dict, char *value, char **op_errstr)
+tear_down_cluster(gf_boolean_t run_teardown)
 {
-        runner_t                 runner = {0,};
-        int                      ret = -1;
-        char                     str[1024];
-        glusterd_volinfo_t      *volinfo = NULL;
-        char                    *volname = NULL;
-        xlator_t                *this    = NULL;
-        int                     i = 1;
-
-        runinit (&runner);
-        this =  THIS;
-
-        GF_ASSERT (value);
-        GF_ASSERT (dict);
+    int ret = 0;
+    runner_t runner = {
+        0,
+    };
+    struct stat st = {
+        0,
+    };
+    DIR *dir = NULL;
+    struct dirent *entry = NULL;
+    struct dirent scratch[2] = {
+        {
+            0,
+        },
+    };
+    char path[PATH_MAX] = {
+        0,
+    };
+
+    if (run_teardown) {
+        runinit(&runner);
+        runner_add_args(&runner, GANESHA_PREFIX "/ganesha-ha.sh", "teardown",
+                        CONFDIR, NULL);
+        ret = runner_run(&runner);
+        /* *
+         * Remove all the entries in CONFDIR expect ganesha.conf and
+         * ganesha-ha.conf
+         */
+        dir = sys_opendir(CONFDIR);
+        if (!dir) {
+            gf_msg_debug(THIS->name, 0,
+                         "Failed to open directory %s. "
+                         "Reason : %s",
+                         CONFDIR, strerror(errno));
+            ret = 0;
+            goto out;
+        }
 
-        ret = dict_get_str (dict, "volname", &volname);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "Unable to get volume name");
+        while ((entry = sys_readdir(dir, scratch))) {
+            if (gf_irrelevant_entry(entry))
+                continue;
+            snprintf(path, PATH_MAX, "%s/%s", CONFDIR, entry->d_name);
+            ret = sys_lstat(path, &st);
+            if (ret == -1) {
+                gf_msg_debug(THIS->name, 0,
+                             "Failed to stat entry %s :"
+                             " %s",
+                             path, strerror(errno));
                 goto out;
+            }
+
+            if (strcmp(entry->d_name, "ganesha.conf") == 0 ||
+                strcmp(entry->d_name, "ganesha-ha.conf") == 0)
+                gf_msg_debug(THIS->name, 0,
+                             " %s is not required"
+                             " to remove",
+                             path);
+            else if (S_ISDIR(st.st_mode))
+                ret = recursive_rmdir(path);
+            else
+                ret = sys_unlink(path);
+
+            if (ret) {
+                gf_msg_debug(THIS->name, 0,
+                             " Failed to remove %s. "
+                             "Reason : %s",
+                             path, strerror(errno));
+            }
+
+            gf_msg_debug(THIS->name, 0, "%s %s",
+                         ret ? "Failed to remove" : "Removed", entry->d_name);
         }
 
-        ret = glusterd_volinfo_find (volname, &volinfo);
+        ret = sys_closedir(dir);
         if (ret) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        FMTSTR_CHECK_VOL_EXISTS, volname);
-                goto out;
-        }
-        /* Todo : check if global option is enabled, proceed only then */
-
-        /* Create the export file only when ganesha.enable "on" is executed */
-         if (strcmp (value, "on") == 0) {
-                ret  =  create_export_config (volname, op_errstr);
-                if (ret) {
-                        gf_log (this->name, GF_LOG_ERROR, "Failed to create"
-                                "export file for NFS-Ganesha\n");
-                        goto out;
-                }
+            gf_msg_debug(THIS->name, 0,
+                         "Failed to close dir %s. Reason :"
+                         " %s",
+                         CONFDIR, strerror(errno));
         }
+        goto exit;
+    }
 
-        if (check_host_list()) {
-                runner_add_args (&runner, "sh", GANESHA_PREFIX"/dbus-send.sh",
-                         CONFDIR, value, volname, NULL);
-                ret = runner_run (&runner);
-                if (ret)
-                        gf_asprintf(op_errstr, "Dynamic export"
-                                    " addition/deletion failed."
-                                    "Please see log file for details");
-        }
 out:
-        return ret;
+    if (dir && sys_closedir(dir)) {
+        gf_msg_debug(THIS->name, 0,
+                     "Failed to close dir %s. Reason :"
+                     " %s",
+                     CONFDIR, strerror(errno));
+    }
+exit:
+    return ret;
 }
 
 int
-tear_down_cluster(void)
+setup_cluster(gf_boolean_t run_setup)
 {
-        int     ret     =       0;
-        runner_t runner =       {0,};
-
-        if (is_ganesha_host()) {
-                runinit (&runner);
-                runner_add_args (&runner, "sh", CONFDIR,
-                                GANESHA_PREFIX"/ganesha-ha.sh", "teardown",
-                                CONFDIR, NULL);
-                ret = runner_run(&runner);
-        }
-        return ret;
+    int ret = 0;
+    runner_t runner = {
+        0,
+    };
+
+    if (run_setup) {
+        runinit(&runner);
+        runner_add_args(&runner, GANESHA_PREFIX "/ganesha-ha.sh", "setup",
+                        CONFDIR, NULL);
+        ret = runner_run(&runner);
+    }
+    return ret;
 }
 
-
-int
-setup_cluster(void)
+static int
+teardown(gf_boolean_t run_teardown, char **op_errstr)
 {
-        int ret         = 0;
-        runner_t runner = {0,};
-
-        if (is_ganesha_host()) {
-                runinit (&runner);
-                runner_add_args (&runner, "sh", GANESHA_PREFIX"/ganesha-ha.sh",
-                                 "setup", CONFDIR,  NULL);
-                ret =  runner_run (&runner);
-        }
-        return ret;
-}
+    runner_t runner = {
+        0,
+    };
+    int ret = 1;
+    glusterd_volinfo_t *volinfo = NULL;
+    glusterd_conf_t *priv = NULL;
+    dict_t *vol_opts = NULL;
+
+    priv = THIS->private;
+
+    ret = tear_down_cluster(run_teardown);
+    if (ret == -1) {
+        gf_asprintf(op_errstr,
+                    "Cleanup of NFS-Ganesha"
+                    " HA config failed.");
+        goto out;
+    }
+
+    runinit(&runner);
+    runner_add_args(&runner, GANESHA_PREFIX "/ganesha-ha.sh", "cleanup",
+                    CONFDIR, NULL);
+    ret = runner_run(&runner);
+    if (ret)
+        gf_msg_debug(THIS->name, 0,
+                     "Could not clean up"
+                     " NFS-Ganesha related config");
+
+    cds_list_for_each_entry(volinfo, &priv->volumes, vol_list)
+    {
+        vol_opts = volinfo->dict;
+        /* All the volumes exported via NFS-Ganesha will be
+        unexported, hence setting the appropriate keys */
+        ret = dict_set_str(vol_opts, "features.cache-invalidation", "off");
+        if (ret)
+            gf_msg(THIS->name, GF_LOG_WARNING, errno, GD_MSG_DICT_SET_FAILED,
+                   "Could not set features.cache-invalidation "
+                   "to off for %s",
+                   volinfo->volname);
 
+        ret = dict_set_str(vol_opts, "ganesha.enable", "off");
+        if (ret)
+            gf_msg(THIS->name, GF_LOG_WARNING, errno, GD_MSG_DICT_SET_FAILED,
+                   "Could not set ganesha.enable to off for %s",
+                   volinfo->volname);
+
+        ret = glusterd_store_volinfo(volinfo,
+                                     GLUSTERD_VOLINFO_VER_AC_INCREMENT);
+        if (ret)
+            gf_msg(THIS->name, GF_LOG_WARNING, 0, GD_MSG_VOLINFO_SET_FAIL,
+                   "failed to store volinfo for %s", volinfo->volname);
+    }
+out:
+    return ret;
+}
 
 int
-stop_ganesha (char **op_errstr)
+stop_ganesha(char **op_errstr)
 {
-        runner_t                runner                     = {0,};
-        int                     ret                        = 1;
-
-        ret = tear_down_cluster();
-        if (ret == -1) {
-                gf_asprintf (op_errstr, "Cleanup of NFS-Ganesha"
-                             "HA config failed.");
-                goto out;
-        }
-
-        if (check_host_list ()) {
-                ret = manage_service ("stop");
-                if (ret)
-                        gf_asprintf (op_errstr, "NFS-Ganesha service could not"
-                                     "be stopped.");
+    int ret = 0;
+    runner_t runner = {
+        0,
+    };
+
+    if (check_host_list()) {
+        runinit(&runner);
+        runner_add_args(&runner, GANESHA_PREFIX "/ganesha-ha.sh",
+                        "--setup-ganesha-conf-files", CONFDIR, "no", NULL);
+        ret = runner_run(&runner);
+        if (ret) {
+            gf_asprintf(op_errstr,
+                        "removal of symlink ganesha.conf "
+                        "in /etc/ganesha failed");
         }
-out:
-        return ret;
+        ret = manage_service("stop");
+        if (ret)
+            gf_asprintf(op_errstr,
+                        "NFS-Ganesha service could not"
+                        "be stopped.");
+    }
+    return ret;
 }
 
 int
-start_ganesha (char **op_errstr)
+start_ganesha(char **op_errstr)
 {
-        int                     ret                        = -1;
-        char                    key[1024]                  = {0,};
-        char                    *hostname                  = NULL;
-        dict_t *vol_opts                                   = NULL;
-        glusterd_volinfo_t *volinfo                        = NULL;
-        int count                                          = 0;
-        char *volname                                      = NULL;
-        glusterd_conf_t *priv                              = NULL;
-
-        priv =  THIS->private;
-        GF_ASSERT (priv);
-
-        cds_list_for_each_entry (volinfo, &priv->volumes, vol_list) {
-                vol_opts = volinfo->dict;
-                /* Gluster-nfs has to be disabled across the trusted pool */
-                /* before attempting to start nfs-ganesha */
-                ret = dict_set_str (vol_opts, "nfs.disable", "on");
-                if (ret)
-                        goto out;
+    int ret = -1;
+    glusterd_volinfo_t *volinfo = NULL;
+    glusterd_conf_t *priv = NULL;
+    runner_t runner = {
+        0,
+    };
+
+    priv = THIS->private;
+    GF_ASSERT(priv);
+
+    cds_list_for_each_entry(volinfo, &priv->volumes, vol_list)
+    {
+#ifdef BUILD_GNFS
+        /* Gluster-nfs has to be disabled across the trusted pool */
+        /* before attempting to start nfs-ganesha */
+        ret = dict_set_str_sizen(volinfo->dict, NFS_DISABLE_MAP_KEY, "on");
+        if (ret)
+            goto out;
+#endif
+        ret = glusterd_store_volinfo(volinfo,
+                                     GLUSTERD_VOLINFO_VER_AC_INCREMENT);
+        if (ret) {
+            *op_errstr = gf_strdup(
+                "Failed to store the "
+                "Volume information");
+            goto out;
         }
-
-        ret = priv->nfs_svc.stop (&(priv->nfs_svc), SIGKILL);
+    }
+
+    /* If the nfs svc is not initialized it means that the service is not
+     * running, hence we can skip the process of stopping gluster-nfs
+     * service
+     */
+#ifdef BUILD_GNFS
+    if (priv->nfs_svc.inited) {
+        ret = priv->nfs_svc.stop(&(priv->nfs_svc), SIGKILL);
         if (ret) {
-                ret = -1;
-                gf_asprintf (op_errstr, "Gluster-NFS service could"
-                             "not be stopped, exiting.");
-                goto out;
+            ret = -1;
+            gf_asprintf(op_errstr,
+                        "Gluster-NFS service could"
+                        "not be stopped, exiting.");
+            goto out;
         }
+    }
+#endif
 
-        if (check_host_list()) {
-                ret = manage_service ("start");
-                if (ret) {
-                        gf_asprintf (op_errstr, "NFS-Ganesha failed to start."
-                        "Please see log file for details");
-                        goto out;
-                }
-
-                ret = setup_cluster();
-                if (ret == -1) {
-                        gf_asprintf (op_errstr, "Failed to set up HA "
-                                     "config for NFS-Ganesha."
-                                     "Please check the log file for details");
-                        goto out;
-                }
+    if (check_host_list()) {
+        runinit(&runner);
+        runner_add_args(&runner, GANESHA_PREFIX "/ganesha-ha.sh",
+                        "--setup-ganesha-conf-files", CONFDIR, "yes", NULL);
+        ret = runner_run(&runner);
+        if (ret) {
+            gf_asprintf(op_errstr,
+                        "creation of symlink ganesha.conf "
+                        "in /etc/ganesha failed");
+            goto out;
         }
+        ret = manage_service("start");
+        if (ret)
+            gf_asprintf(op_errstr,
+                        "NFS-Ganesha failed to start."
+                        "Please see log file for details");
+    }
 
 out:
-        return ret;
+    return ret;
 }
 
-int
-glusterd_handle_ganesha_op (dict_t *dict, char **op_errstr,
-                            char *key, char *value)
+static int
+pre_setup(gf_boolean_t run_setup, char **op_errstr)
 {
-
-        int32_t                 ret          = -1;
-        char                   *volname      = NULL;
-        xlator_t               *this         = NULL;
-        static int             export_id     = 1;
-        glusterd_volinfo_t     *volinfo      = NULL;
-        char *option                         = NULL;
-
-        GF_ASSERT (dict);
-        GF_ASSERT (op_errstr);
-        GF_ASSERT (key);
-        GF_ASSERT (value);
-
-        /* TODO: enable only if global option is set */
-        /* BUG ID : 1200265 */
-
-        if (strcmp (key, "ganesha.enable") == 0) {
-                ret =  ganesha_manage_export(dict, value, op_errstr);
-                if (ret < 0)
-                        goto out;
+    int ret = 0;
+    if (run_setup) {
+        if (!check_host_list()) {
+            gf_asprintf(op_errstr,
+                        "Running nfs-ganesha setup command "
+                        "from node which is not part of ganesha cluster");
+            return -1;
         }
+    }
+    ret = setup_cluster(run_setup);
+    if (ret == -1)
+        gf_asprintf(op_errstr,
+                    "Failed to set up HA "
+                    "config for NFS-Ganesha. "
+                    "Please check the log file for details");
+    return ret;
+}
 
-        if (strcmp (key, "features.ganesha") == 0) {
-                if (strcmp (value, "enable") == 0) {
-                        ret = start_ganesha(op_errstr);
-                        if (ret < 0)
-                                goto out;
-                }
-
-        else if (strcmp (value, "disable") == 0) {
-                ret = stop_ganesha (op_errstr);
-                if (ret < 0)
-                        goto out;
-                }
+int
+glusterd_handle_ganesha_op(dict_t *dict, char **op_errstr, char *key,
+                           char *value)
+{
+    int32_t ret = -1;
+    gf_boolean_t option = _gf_false;
+
+    GF_ASSERT(dict);
+    GF_ASSERT(op_errstr);
+    GF_ASSERT(key);
+    GF_ASSERT(value);
+
+    if (strcmp(key, "ganesha.enable") == 0) {
+        ret = ganesha_manage_export(dict, value, _gf_true, op_errstr);
+        if (ret < 0)
+            goto out;
+    }
+
+    /* It is possible that the key might not be set */
+    ret = gf_string2boolean(value, &option);
+    if (ret == -1) {
+        gf_asprintf(op_errstr, "Invalid value in key-value pair.");
+        goto out;
+    }
+
+    if (strcmp(key, GLUSTERD_STORE_KEY_GANESHA_GLOBAL) == 0) {
+        /* *
+         * The set up/teardown of pcs cluster should be performed only
+         * once. This will done on the node in which the cli command
+         * 'gluster nfs-ganesha <enable/disable>' got executed. So that
+         * node should part of ganesha HA cluster
+         */
+        if (option) {
+            ret = pre_setup(is_origin_glusterd(dict), op_errstr);
+            if (ret < 0)
+                goto out;
+        } else {
+            ret = teardown(is_origin_glusterd(dict), op_errstr);
+            if (ret < 0)
+                goto out;
         }
+    }
 
 out:
-        return ret;
+    return ret;
 }
-
diff --git a/xlators/mgmt/glusterd/src/glusterd-geo-rep.c b/xlators/mgmt/glusterd/src/glusterd-geo-rep.c
index a190e1c8aea..bf062c87060 100644
--- a/xlators/mgmt/glusterd/src/glusterd-geo-rep.c
+++ b/xlators/mgmt/glusterd/src/glusterd-geo-rep.c
@@ -7,12 +7,7 @@
    later), or the GNU General Public License, version 2 (GPLv2), in all
    cases as published by the Free Software Foundation.
 */
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
-
-#include "common-utils.h"
+#include <glusterfs/common-utils.h>
 #include "cli1-xdr.h"
 #include "xdr-generic.h"
 #include "glusterd.h"
@@ -22,344 +17,397 @@
 #include "glusterd-utils.h"
 #include "glusterd-volgen.h"
 #include "glusterd-svc-helper.h"
-#include "run.h"
-#include "syscall.h"
+#include <glusterfs/run.h>
+#include <glusterfs/syscall.h>
+#include "glusterd-messages.h"
 
 #include <signal.h>
 
 static int
-dict_get_param (dict_t *dict, char *key, char **param);
+dict_get_param(dict_t *dict, char *key, char **param);
 
 struct gsync_config_opt_vals_ gsync_confopt_vals[] = {
-        {.op_name        = "change_detector",
-         .no_of_pos_vals = 2,
-         .case_sensitive = _gf_true,
-         .values         = {"xsync", "changelog"},
-        },
-        {.op_name        = "special_sync_mode",
-         .no_of_pos_vals = 2,
-         .case_sensitive = _gf_true,
-         .values         = {"partial", "recover"}
-        },
-        {.op_name        = "log-level",
-         .no_of_pos_vals = 5,
-         .case_sensitive = _gf_false,
-         .values         = {"critical", "error", "warning", "info", "debug"}
-        },
-        {.op_name        = "use-tarssh",
-         .no_of_pos_vals = 6,
-         .case_sensitive = _gf_false,
-         .values         = {"true", "false", "0", "1", "yes", "no"}
-        },
-        {.op_name = NULL,
-        },
+    {
+        .op_name = "change_detector",
+        .no_of_pos_vals = 2,
+        .case_sensitive = _gf_true,
+        .values = {"xsync", "changelog"},
+    },
+    {.op_name = "special_sync_mode",
+     .no_of_pos_vals = 2,
+     .case_sensitive = _gf_true,
+     .values = {"partial", "recover"}},
+    {.op_name = "log-level",
+     .no_of_pos_vals = 5,
+     .case_sensitive = _gf_false,
+     .values = {"critical", "error", "warning", "info", "debug"}},
+    {.op_name = "use-tarssh",
+     .no_of_pos_vals = 6,
+     .case_sensitive = _gf_false,
+     .values = {"true", "false", "0", "1", "yes", "no"}},
+    {.op_name = "ignore_deletes",
+     .no_of_pos_vals = 6,
+     .case_sensitive = _gf_false,
+     .values = {"true", "false", "0", "1", "yes", "no"}},
+    {.op_name = "use_meta_volume",
+     .no_of_pos_vals = 6,
+     .case_sensitive = _gf_false,
+     .values = {"true", "false", "0", "1", "yes", "no"}},
+    {.op_name = "use-meta-volume",
+     .no_of_pos_vals = 6,
+     .case_sensitive = _gf_false,
+     .values = {"true", "false", "0", "1", "yes", "no"}},
+    {
+        .op_name = NULL,
+    },
 };
 
-static char *gsync_reserved_opts[] = {
-        "gluster-command-dir",
-        "pid-file",
-        "remote-gsyncd"
-        "state-file",
-        "session-owner",
-        "state-socket-unencoded",
-        "socketdir",
-        "ignore-deletes",
-        "local-id",
-        "local-path",
-        "slave-id",
-        NULL
-};
+static char *gsync_reserved_opts[] = {"gluster-command",
+                                      "pid-file",
+                                      "state-file",
+                                      "session-owner",
+                                      "state-socket-unencoded",
+                                      "socketdir",
+                                      "local-id",
+                                      "local-path",
+                                      "slave-id",
+                                      NULL};
 
-static char *gsync_no_restart_opts[] = {
-        "checkpoint",
-        NULL
-};
+static char *gsync_no_restart_opts[] = {"checkpoint", "log_rsync_performance",
+                                        "log-rsync-performance", NULL};
 
-int
-__glusterd_handle_sys_exec (rpcsvc_request_t *req)
+void
+set_gsyncd_inet6_arg(runner_t *runner)
 {
-        int32_t                 ret     = 0;
-        dict_t                  *dict   = NULL;
-        gf_cli_req              cli_req = {{0},};
-        glusterd_op_t           cli_op = GD_OP_SYS_EXEC;
-        glusterd_conf_t         *priv   = NULL;
-        char                    *host_uuid = NULL;
-        char                    err_str[2048] = {0,};
-        xlator_t                *this = NULL;
-
-        GF_ASSERT (req);
-
-        this = THIS;
-        GF_ASSERT (this);
-        priv = this->private;
-        GF_ASSERT (priv);
+    xlator_t *this = NULL;
+    char *af;
+    int ret;
+
+    this = THIS;
+    ret = dict_get_str(this->options, "transport.address-family", &af);
+    if (ret == 0)
+        runner_argprintf(runner, "--%s", af);
+}
 
-        ret = xdr_to_generic (req->msg[0], &cli_req,
-                              (xdrproc_t)xdr_gf_cli_req);
+int
+__glusterd_handle_sys_exec(rpcsvc_request_t *req)
+{
+    int32_t ret = 0;
+    dict_t *dict = NULL;
+    gf_cli_req cli_req = {
+        {0},
+    };
+    glusterd_op_t cli_op = GD_OP_SYS_EXEC;
+    glusterd_conf_t *priv = NULL;
+    char *host_uuid = NULL;
+    char err_str[64] = {
+        0,
+    };
+    xlator_t *this = NULL;
+
+    GF_ASSERT(req);
+
+    this = THIS;
+    GF_ASSERT(this);
+    priv = this->private;
+    GF_ASSERT(priv);
+
+    ret = xdr_to_generic(req->msg[0], &cli_req, (xdrproc_t)xdr_gf_cli_req);
+    if (ret < 0) {
+        req->rpc_err = GARBAGE_ARGS;
+        snprintf(err_str, sizeof(err_str), "Garbage args received");
+        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_GARBAGE_ARGS, NULL);
+        goto out;
+    }
+
+    if (cli_req.dict.dict_len) {
+        dict = dict_new();
+        if (!dict) {
+            gf_smsg(THIS->name, GF_LOG_ERROR, errno, GD_MSG_DICT_CREATE_FAIL,
+                    NULL);
+            goto out;
+        }
+
+        ret = dict_unserialize(cli_req.dict.dict_val, cli_req.dict.dict_len,
+                               &dict);
         if (ret < 0) {
-                req->rpc_err = GARBAGE_ARGS;
-                goto out;
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_UNSERIALIZE_FAIL,
+                   "failed to "
+                   "unserialize req-buffer to dictionary");
+            snprintf(err_str, sizeof(err_str),
+                     "Unable to decode "
+                     "the command");
+            goto out;
+        } else {
+            dict->extra_stdfree = cli_req.dict.dict_val;
         }
 
-        if (cli_req.dict.dict_len) {
-                dict = dict_new ();
-                if (!dict)
-                        goto out;
-
-
-                ret = dict_unserialize (cli_req.dict.dict_val,
-                                        cli_req.dict.dict_len,
-                                        &dict);
-                if (ret < 0) {
-                        gf_log (this->name, GF_LOG_ERROR, "failed to "
-                                "unserialize req-buffer to dictionary");
-                        snprintf (err_str, sizeof (err_str), "Unable to decode "
-                                  "the command");
-                        goto out;
-               } else {
-                        dict->extra_stdfree = cli_req.dict.dict_val;
-                }
-
-                host_uuid = gf_strdup (uuid_utoa(MY_UUID));
-                if (host_uuid == NULL) {
-                        snprintf (err_str, sizeof (err_str), "Failed to get "
-                                  "the uuid of local glusterd");
-                        ret = -1;
-                        goto out;
-                }
+        host_uuid = gf_strdup(uuid_utoa(MY_UUID));
+        if (host_uuid == NULL) {
+            snprintf(err_str, sizeof(err_str),
+                     "Failed to get "
+                     "the uuid of local glusterd");
+            gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_UUID_GET_FAIL,
+                    NULL);
+            ret = -1;
+            goto out;
+        }
 
-                ret = dict_set_dynstr (dict, "host-uuid", host_uuid);
-                if (ret)
-                        goto out;
+        ret = dict_set_dynstr(dict, "host-uuid", host_uuid);
+        if (ret) {
+            gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_SET_FAILED,
+                    "Key=host-uuid", NULL);
+            goto out;
         }
+    }
 
-        ret = glusterd_op_begin_synctask (req, cli_op, dict);
+    ret = glusterd_op_begin_synctask(req, cli_op, dict);
 
 out:
-        if (ret) {
-                if (err_str[0] == '\0')
-                        snprintf (err_str, sizeof (err_str),
-                                  "Operation failed");
-                ret = glusterd_op_send_cli_response (cli_op, ret, 0, req,
-                                                     dict, err_str);
-        }
-        return ret;
+    if (ret) {
+        if (err_str[0] == '\0')
+            snprintf(err_str, sizeof(err_str), "Operation failed");
+        ret = glusterd_op_send_cli_response(cli_op, ret, 0, req, dict, err_str);
+    }
+    return ret;
 }
 
 int
-__glusterd_handle_copy_file (rpcsvc_request_t *req)
+__glusterd_handle_copy_file(rpcsvc_request_t *req)
 {
-        int32_t                 ret     = 0;
-        dict_t                  *dict   = NULL;
-        gf_cli_req              cli_req = {{0},};
-        glusterd_op_t           cli_op = GD_OP_COPY_FILE;
-        glusterd_conf_t         *priv   = NULL;
-        char                    *host_uuid = NULL;
-        char                    err_str[2048] = {0,};
-        xlator_t                *this = NULL;
-
-        GF_ASSERT (req);
-
-        this = THIS;
-        GF_ASSERT (this);
-        priv = this->private;
-        GF_ASSERT (priv);
-
-        ret = xdr_to_generic (req->msg[0], &cli_req,
-                              (xdrproc_t)xdr_gf_cli_req);
+    int32_t ret = 0;
+    dict_t *dict = NULL;
+    gf_cli_req cli_req = {
+        {0},
+    };
+    glusterd_op_t cli_op = GD_OP_COPY_FILE;
+    glusterd_conf_t *priv = NULL;
+    char *host_uuid = NULL;
+    char err_str[64] = {
+        0,
+    };
+    xlator_t *this = NULL;
+
+    GF_ASSERT(req);
+
+    this = THIS;
+    GF_ASSERT(this);
+    priv = this->private;
+    GF_ASSERT(priv);
+
+    ret = xdr_to_generic(req->msg[0], &cli_req, (xdrproc_t)xdr_gf_cli_req);
+    if (ret < 0) {
+        req->rpc_err = GARBAGE_ARGS;
+        snprintf(err_str, sizeof(err_str), "Garbage args received");
+        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_GARBAGE_ARGS, NULL);
+        goto out;
+    }
+
+    if (cli_req.dict.dict_len) {
+        dict = dict_new();
+        if (!dict) {
+            gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_CREATE_FAIL,
+                    NULL);
+            goto out;
+        }
+
+        ret = dict_unserialize(cli_req.dict.dict_val, cli_req.dict.dict_len,
+                               &dict);
         if (ret < 0) {
-                req->rpc_err = GARBAGE_ARGS;
-                goto out;
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_UNSERIALIZE_FAIL,
+                   "failed to"
+                   "unserialize req-buffer to dictionary");
+            snprintf(err_str, sizeof(err_str),
+                     "Unable to decode "
+                     "the command");
+            goto out;
+        } else {
+            dict->extra_stdfree = cli_req.dict.dict_val;
         }
 
-        if (cli_req.dict.dict_len) {
-                dict = dict_new ();
-                if (!dict)
-                        goto out;
-
-
-                ret = dict_unserialize (cli_req.dict.dict_val,
-                                        cli_req.dict.dict_len,
-                                        &dict);
-                if (ret < 0) {
-                        gf_log (this->name, GF_LOG_ERROR, "failed to "
-                                "unserialize req-buffer to dictionary");
-                        snprintf (err_str, sizeof (err_str), "Unable to decode "
-                                  "the command");
-                        goto out;
-               } else {
-                        dict->extra_stdfree = cli_req.dict.dict_val;
-                }
-
-                host_uuid = gf_strdup (uuid_utoa(MY_UUID));
-                if (host_uuid == NULL) {
-                        snprintf (err_str, sizeof (err_str), "Failed to get "
-                                  "the uuid of local glusterd");
-                        ret = -1;
-                        goto out;
-                }
-
-                ret = dict_set_dynstr (dict, "host-uuid", host_uuid);
-                if (ret)
-                        goto out;
+        host_uuid = gf_strdup(uuid_utoa(MY_UUID));
+        if (host_uuid == NULL) {
+            snprintf(err_str, sizeof(err_str),
+                     "Failed to get "
+                     "the uuid of local glusterd");
+            gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_UUID_GET_FAIL,
+                    NULL);
+            ret = -1;
+            goto out;
         }
 
-        ret = glusterd_op_begin_synctask (req, cli_op, dict);
+        ret = dict_set_dynstr(dict, "host-uuid", host_uuid);
+        if (ret)
+            goto out;
+    }
+
+    ret = glusterd_op_begin_synctask(req, cli_op, dict);
 
 out:
-        if (ret) {
-                if (err_str[0] == '\0')
-                        snprintf (err_str, sizeof (err_str),
-                                  "Operation failed");
-                ret = glusterd_op_send_cli_response (cli_op, ret, 0, req,
-                                                     dict, err_str);
-        }
-        return ret;
+    if (ret) {
+        if (err_str[0] == '\0')
+            snprintf(err_str, sizeof(err_str), "Operation failed");
+        ret = glusterd_op_send_cli_response(cli_op, ret, 0, req, dict, err_str);
+    }
+    return ret;
 }
 
 int
-__glusterd_handle_gsync_set (rpcsvc_request_t *req)
+__glusterd_handle_gsync_set(rpcsvc_request_t *req)
 {
-        int32_t                 ret     = 0;
-        dict_t                  *dict   = NULL;
-        gf_cli_req              cli_req = {{0},};
-        glusterd_op_t           cli_op = GD_OP_GSYNC_SET;
-        char                    *master = NULL;
-        char                    *slave = NULL;
-        char                    operation[256] = {0,};
-        int                     type = 0;
-        glusterd_conf_t         *priv   = NULL;
-        char                    *host_uuid = NULL;
-        char                    err_str[2048] = {0,};
-        xlator_t                *this = NULL;
-
-        GF_ASSERT (req);
-
-       this = THIS;
-       GF_ASSERT (this);
-       priv = this->private;
-       GF_ASSERT (priv);
-
-        ret = xdr_to_generic (req->msg[0], &cli_req,
-                              (xdrproc_t)xdr_gf_cli_req);
+    int32_t ret = 0;
+    dict_t *dict = NULL;
+    gf_cli_req cli_req = {
+        {0},
+    };
+    glusterd_op_t cli_op = GD_OP_GSYNC_SET;
+    char *master = NULL;
+    char *slave = NULL;
+    char operation[64] = {
+        0,
+    };
+    int type = 0;
+    glusterd_conf_t *priv = NULL;
+    char *host_uuid = NULL;
+    char err_str[64] = {
+        0,
+    };
+    xlator_t *this = NULL;
+
+    GF_ASSERT(req);
+
+    this = THIS;
+    GF_ASSERT(this);
+    priv = this->private;
+    GF_ASSERT(priv);
+
+    ret = xdr_to_generic(req->msg[0], &cli_req, (xdrproc_t)xdr_gf_cli_req);
+    if (ret < 0) {
+        req->rpc_err = GARBAGE_ARGS;
+        snprintf(err_str, sizeof(err_str), "Garbage args received");
+        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_GARBAGE_ARGS, NULL);
+        goto out;
+    }
+
+    if (cli_req.dict.dict_len) {
+        dict = dict_new();
+        if (!dict) {
+            gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_CREATE_FAIL,
+                    NULL);
+            goto out;
+        }
+
+        ret = dict_unserialize(cli_req.dict.dict_val, cli_req.dict.dict_len,
+                               &dict);
         if (ret < 0) {
-                req->rpc_err = GARBAGE_ARGS;
-                goto out;
-        }
-
-        if (cli_req.dict.dict_len) {
-                dict = dict_new ();
-                if (!dict)
-                        goto out;
-
-                ret = dict_unserialize (cli_req.dict.dict_val,
-                                        cli_req.dict.dict_len,
-                                        &dict);
-                if (ret < 0) {
-                        gf_log (this->name, GF_LOG_ERROR, "failed to "
-                                "unserialize req-buffer to dictionary");
-                        snprintf (err_str, sizeof (err_str), "Unable to decode "
-                                  "the command");
-                        goto out;
-                } else {
-                        dict->extra_stdfree = cli_req.dict.dict_val;
-                }
-
-                host_uuid = gf_strdup (uuid_utoa(MY_UUID));
-                if (host_uuid == NULL) {
-                        snprintf (err_str, sizeof (err_str), "Failed to get "
-                                  "the uuid of local glusterd");
-                        ret = -1;
-                        goto out;
-                }
-                ret = dict_set_dynstr (dict, "host-uuid", host_uuid);
-                if (ret)
-                        goto out;
-
-        }
-
-        ret = dict_get_str (dict, "master", &master);
-        if (ret < 0) {
-                gf_log (this->name, GF_LOG_INFO, "master not found, while "
-                        "handling "GEOREP" options");
-                master = "(No Master)";
-        }
-
-        ret = dict_get_str (dict, "slave", &slave);
-        if (ret < 0) {
-                gf_log (this->name, GF_LOG_INFO, "slave not found, while "
-                        "handling "GEOREP" options");
-                slave = "(No Slave)";
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_UNSERIALIZE_FAIL,
+                   "failed to "
+                   "unserialize req-buffer to dictionary");
+            snprintf(err_str, sizeof(err_str),
+                     "Unable to decode "
+                     "the command");
+            goto out;
+        } else {
+            dict->extra_stdfree = cli_req.dict.dict_val;
         }
 
-        ret = dict_get_int32 (dict, "type", &type);
-        if (ret < 0) {
-                snprintf (err_str, sizeof (err_str), "Command type not found "
-                          "while handling "GEOREP" options");
-                gf_log (this->name, GF_LOG_ERROR, "%s", err_str);
-                goto out;
+        host_uuid = gf_strdup(uuid_utoa(MY_UUID));
+        if (host_uuid == NULL) {
+            snprintf(err_str, sizeof(err_str),
+                     "Failed to get "
+                     "the uuid of local glusterd");
+            gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_UUID_GET_FAIL,
+                    NULL);
+            ret = -1;
+            goto out;
         }
-
-        switch (type) {
+        ret = dict_set_dynstr(dict, "host-uuid", host_uuid);
+        if (ret)
+            goto out;
+    }
+
+    ret = dict_get_str(dict, "master", &master);
+    if (ret < 0) {
+        gf_msg(this->name, GF_LOG_INFO, 0, GD_MSG_DICT_GET_FAILED,
+               "master not found, while handling " GEOREP " options");
+        master = "(No Master)";
+    }
+
+    ret = dict_get_str(dict, "slave", &slave);
+    if (ret < 0) {
+        gf_msg(this->name, GF_LOG_INFO, 0, GD_MSG_DICT_GET_FAILED,
+               "slave not found, while handling " GEOREP " options");
+        slave = "(No Slave)";
+    }
+
+    ret = dict_get_int32(dict, "type", &type);
+    if (ret < 0) {
+        snprintf(err_str, sizeof(err_str),
+                 "Command type not found "
+                 "while handling " GEOREP " options");
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED, "%s",
+               err_str);
+        goto out;
+    }
+
+    switch (type) {
         case GF_GSYNC_OPTION_TYPE_CREATE:
-                strncpy (operation, "create", sizeof (operation));
-                cli_op = GD_OP_GSYNC_CREATE;
-                break;
+            snprintf(operation, sizeof(operation), "create");
+            cli_op = GD_OP_GSYNC_CREATE;
+            break;
 
         case GF_GSYNC_OPTION_TYPE_START:
-                strncpy (operation, "start", sizeof (operation));
-                break;
+            snprintf(operation, sizeof(operation), "start");
+            break;
 
         case GF_GSYNC_OPTION_TYPE_STOP:
-                strncpy (operation, "stop", sizeof (operation));
-                break;
+            snprintf(operation, sizeof(operation), "stop");
+            break;
 
         case GF_GSYNC_OPTION_TYPE_PAUSE:
-                strncpy (operation, "pause", sizeof (operation));
-                break;
+            snprintf(operation, sizeof(operation), "pause");
+            break;
 
         case GF_GSYNC_OPTION_TYPE_RESUME:
-                strncpy (operation, "resume", sizeof (operation));
-                break;
+            snprintf(operation, sizeof(operation), "resume");
+            break;
 
         case GF_GSYNC_OPTION_TYPE_CONFIG:
-                strncpy (operation, "config", sizeof (operation));
-                break;
+            snprintf(operation, sizeof(operation), "config");
+            break;
 
         case GF_GSYNC_OPTION_TYPE_STATUS:
-                strncpy (operation, "status", sizeof (operation));
-                break;
-        }
+            snprintf(operation, sizeof(operation), "status");
+            break;
+    }
 
-        ret = glusterd_op_begin_synctask (req, cli_op, dict);
+    ret = glusterd_op_begin_synctask(req, cli_op, dict);
 
 out:
-        if (ret) {
-                if (err_str[0] == '\0')
-                        snprintf (err_str, sizeof (err_str),
-                                  "Operation failed");
-                ret = glusterd_op_send_cli_response (cli_op, ret, 0, req,
-                                                     dict, err_str);
-        }
-        return ret;
+    if (ret) {
+        if (err_str[0] == '\0')
+            snprintf(err_str, sizeof(err_str), "Operation failed");
+        ret = glusterd_op_send_cli_response(cli_op, ret, 0, req, dict, err_str);
+    }
+    return ret;
 }
 
 int
-glusterd_handle_sys_exec (rpcsvc_request_t *req)
+glusterd_handle_sys_exec(rpcsvc_request_t *req)
 {
-        return glusterd_big_locked_handler (req, __glusterd_handle_sys_exec);
+    return glusterd_big_locked_handler(req, __glusterd_handle_sys_exec);
 }
 
 int
-glusterd_handle_copy_file (rpcsvc_request_t *req)
+glusterd_handle_copy_file(rpcsvc_request_t *req)
 {
-        return glusterd_big_locked_handler (req, __glusterd_handle_copy_file);
+    return glusterd_big_locked_handler(req, __glusterd_handle_copy_file);
 }
 
 int
-glusterd_handle_gsync_set (rpcsvc_request_t *req)
+glusterd_handle_gsync_set(rpcsvc_request_t *req)
 {
-        return glusterd_big_locked_handler (req, __glusterd_handle_gsync_set);
+    return glusterd_big_locked_handler(req, __glusterd_handle_gsync_set);
 }
 
 /*****
@@ -369,986 +417,1603 @@ glusterd_handle_gsync_set (rpcsvc_request_t *req)
  *****/
 
 static void
-glusterd_urltransform_init (runner_t *runner, const char *transname)
+glusterd_urltransform_init(runner_t *runner, const char *transname)
 {
-        runinit (runner);
-        runner_add_arg (runner, GSYNCD_PREFIX"/gsyncd");
-        runner_argprintf (runner, "--%s-url", transname);
+    runinit(runner);
+    runner_add_arg(runner, GSYNCD_PREFIX "/gsyncd");
+    set_gsyncd_inet6_arg(runner);
+    runner_argprintf(runner, "--%s-url", transname);
 }
 
 static void
-glusterd_urltransform_add (runner_t *runner, const char *url)
+glusterd_urltransform_add(runner_t *runner, const char *url)
 {
-        runner_add_arg (runner, url);
+    runner_add_arg(runner, url);
 }
 
-static int
-_glusterd_urltransform_add_iter (dict_t *dict, char *key, data_t *value, void *data)
+/* Helper routine to terminate just before slave_voluuid */
+static int32_t
+parse_slave_url(char *slv_url, char **slave)
 {
-        runner_t *runner = (runner_t *)data;
-        char *slave = NULL;
-
-        slave = strchr (value->data, ':');
-        GF_ASSERT (slave);
-        slave++;
-        runner_add_arg (runner, slave);
+    char *tmp = NULL;
+    xlator_t *this = NULL;
+    int32_t ret = -1;
+
+    this = THIS;
+
+    /* slave format:
+     * master_node_uuid:ssh://slave_host::slave_vol:slave_voluuid */
+    *slave = strchr(slv_url, ':');
+    if (!(*slave)) {
+        goto out;
+    }
+    (*slave)++;
+
+    /* To terminate at : before slave volume uuid */
+    tmp = strstr(*slave, "::");
+    if (!tmp) {
+        goto out;
+    }
+    tmp += 2;
+    tmp = strchr(tmp, ':');
+    if (!tmp)
+        gf_msg_debug(this->name, 0, "old slave: %s!", *slave);
+    else
+        *tmp = '\0';
+
+    ret = 0;
+    gf_msg_debug(this->name, 0, "parsed slave: %s!", *slave);
+out:
+    return ret;
+}
 
-        return 0;
+static int
+_glusterd_urltransform_add_iter(dict_t *dict, char *key, data_t *value,
+                                void *data)
+{
+    runner_t *runner = (runner_t *)data;
+    char slv_url[VOLINFO_SLAVE_URL_MAX] = {0};
+    char *slave = NULL;
+    xlator_t *this = NULL;
+    int32_t ret = -1;
+
+    this = THIS;
+    GF_VALIDATE_OR_GOTO("glusterd", this, out);
+
+    gf_msg_debug(this->name, 0, "value->data %s", value->data);
+
+    if (snprintf(slv_url, sizeof(slv_url), "%s", value->data) >=
+        sizeof(slv_url)) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_SLAVE_VOL_PARSE_FAIL,
+               "Error in copying slave: %s!", value->data);
+        goto out;
+    }
+
+    ret = parse_slave_url(slv_url, &slave);
+    if (ret == -1) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_SLAVE_VOL_PARSE_FAIL,
+               "Error in parsing slave: %s!", value->data);
+        goto out;
+    }
+
+    runner_add_arg(runner, slave);
+    ret = 0;
+out:
+    return ret;
 }
 
 static void
-glusterd_urltransform_free (char **linearr, unsigned n)
+glusterd_urltransform_free(char **linearr, unsigned n)
 {
-        int i = 0;
+    int i = 0;
 
-        for (; i < n; i++)
-                GF_FREE (linearr[i]);
+    for (; i < n; i++)
+        GF_FREE(linearr[i]);
 
-        GF_FREE (linearr);
+    GF_FREE(linearr);
 }
 
 static int
-glusterd_urltransform (runner_t *runner, char ***linearrp)
+glusterd_urltransform(runner_t *runner, char ***linearrp)
 {
-        char **linearr = NULL;
-        char *line = NULL;
-        unsigned arr_len = 32;
-        unsigned arr_idx = 0;
-        gf_boolean_t error = _gf_false;
-
-        linearr = GF_CALLOC (arr_len, sizeof (char *), gf_gld_mt_linearr);
-        if (!linearr) {
+    char **linearr = NULL;
+    char *line = NULL;
+    unsigned arr_len = 32;
+    unsigned arr_idx = 0;
+    gf_boolean_t error = _gf_false;
+    xlator_t *this = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+
+    linearr = GF_CALLOC(arr_len, sizeof(char *), gf_gld_mt_linearr);
+    if (!linearr) {
+        error = _gf_true;
+        goto out;
+    }
+
+    runner_redir(runner, STDOUT_FILENO, RUN_PIPE);
+    if (runner_start(runner) != 0) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_SPAWNING_CHILD_FAILED,
+               "spawning child failed");
+
+        error = _gf_true;
+        goto out;
+    }
+
+    arr_idx = 0;
+    for (;;) {
+        size_t len;
+        line = GF_MALLOC(1024, gf_gld_mt_linebuf);
+        if (!line) {
+            error = _gf_true;
+            goto out;
+        }
+
+        if (fgets(line, 1024, runner_chio(runner, STDOUT_FILENO)) == NULL) {
+            GF_FREE(line);
+            break;
+        }
+
+        len = strlen(line);
+        if (len == 0 || line[len - 1] != '\n') {
+            GF_FREE(line);
+            error = _gf_true;
+            goto out;
+        }
+        line[len - 1] = '\0';
+
+        if (arr_idx == arr_len) {
+            void *p = linearr;
+            arr_len <<= 1;
+            p = GF_REALLOC(linearr, arr_len);
+            if (!p) {
+                GF_FREE(line);
                 error = _gf_true;
                 goto out;
+            }
+            linearr = p;
         }
+        linearr[arr_idx] = line;
 
-        runner_redir (runner, STDOUT_FILENO, RUN_PIPE);
-        if (runner_start (runner) != 0) {
-                gf_log ("", GF_LOG_ERROR, "spawning child failed");
-
-                error = _gf_true;
-                goto out;
-        }
-
-        arr_idx = 0;
-        for (;;) {
-                size_t len;
-                line = GF_MALLOC (1024, gf_gld_mt_linebuf);
-                if (!line) {
-                        error = _gf_true;
-                        goto out;
-                }
-
-                if (fgets (line, 1024, runner_chio (runner, STDOUT_FILENO)) ==
-                    NULL)
-                        break;
-
-                len = strlen (line);
-                if (len == 0 || line[len - 1] != '\n') {
-                        GF_FREE (line);
-                        error = _gf_true;
-                        goto out;
-                }
-                line[len - 1] = '\0';
-
-                if (arr_idx == arr_len) {
-                        void *p = linearr;
-                        arr_len <<= 1;
-                        p = GF_REALLOC (linearr, arr_len);
-                        if (!p) {
-                                GF_FREE (line);
-                                error = _gf_true;
-                                goto out;
-                        }
-                        linearr = p;
-                }
-                linearr[arr_idx] = line;
-
-                arr_idx++;
-        }
-
- out:
+        arr_idx++;
+    }
 
-        /* XXX chpid field is not exported by run API
-         * but runner_end() does not abort the invoked
-         * process (ie. it might block in waitpid(2))
-         * so we resort to a manual kill a the private field
-         */
-        if (error && runner->chpid > 0)
-                kill (runner->chpid, SIGKILL);
-
-        if (runner_end (runner) != 0)
-                error = _gf_true;
+out:
 
-        if (error) {
-                gf_log ("", GF_LOG_ERROR, "reading data from child failed");
-                glusterd_urltransform_free (linearr, arr_idx);
-                return -1;
-        }
+    /* XXX chpid field is not exported by run API
+     * but runner_end() does not abort the invoked
+     * process (ie. it might block in waitpid(2))
+     * so we resort to a manual kill a the private field
+     */
+    if (error && runner->chpid > 0)
+        kill(runner->chpid, SIGKILL);
+
+    if (runner_end(runner) != 0)
+        error = _gf_true;
+
+    if (error) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_READ_CHILD_DATA_FAILED,
+               "reading data from child failed");
+        glusterd_urltransform_free(linearr, arr_idx);
+        return -1;
+    }
 
-        *linearrp = linearr;
-        return arr_idx;
+    *linearrp = linearr;
+    return arr_idx;
 }
 
 static int
-glusterd_urltransform_single (const char *url, const char *transname,
-                              char ***linearrp)
+glusterd_urltransform_single(const char *url, const char *transname,
+                             char ***linearrp)
 {
-        runner_t runner = {0,};
+    runner_t runner = {
+        0,
+    };
 
-        glusterd_urltransform_init (&runner, transname);
-        glusterd_urltransform_add (&runner, url);
-        return glusterd_urltransform (&runner, linearrp);
+    glusterd_urltransform_init(&runner, transname);
+    glusterd_urltransform_add(&runner, url);
+    return glusterd_urltransform(&runner, linearrp);
 }
 
-
 struct dictidxmark {
-        unsigned isrch;
-        unsigned ithis;
-        char *ikey;
+    unsigned isrch;
+    unsigned ithis;
+    char *ikey;
+};
+
+struct slave_vol_config {
+    char old_slvhost[_POSIX_HOST_NAME_MAX + 1];
+    char old_slvuser[LOGIN_NAME_MAX];
+    unsigned old_slvidx;
+    char slave_voluuid[UUID_CANONICAL_FORM_LEN + 1];
 };
 
 static int
-_dict_mark_atindex (dict_t *dict, char *key, data_t *value, void *data)
+_dict_mark_atindex(dict_t *dict, char *key, data_t *value, void *data)
 {
-        struct dictidxmark *dim = data;
+    struct dictidxmark *dim = data;
 
-        if (dim->isrch == dim->ithis)
-                dim->ikey = key;
+    if (dim->isrch == dim->ithis)
+        dim->ikey = key;
 
-        dim->ithis++;
-        return 0;
+    dim->ithis++;
+    return 0;
 }
 
 static char *
-dict_get_by_index (dict_t *dict, unsigned i)
+dict_get_by_index(dict_t *dict, unsigned i)
 {
-        struct dictidxmark dim = {0,};
+    struct dictidxmark dim = {
+        0,
+    };
 
-        dim.isrch = i;
-        dict_foreach (dict, _dict_mark_atindex, &dim);
+    dim.isrch = i;
+    dict_foreach(dict, _dict_mark_atindex, &dim);
 
-        return dim.ikey;
+    return dim.ikey;
 }
 
 static int
-glusterd_get_slave (glusterd_volinfo_t *vol, const char *slaveurl, char **slavekey)
+glusterd_get_slave(glusterd_volinfo_t *vol, const char *slaveurl,
+                   char **slavekey)
 {
-        runner_t runner = {0,};
-        int n = 0;
-        int i = 0;
-        char **linearr = NULL;
-
-        glusterd_urltransform_init (&runner, "canonicalize");
-        dict_foreach (vol->gsync_slaves, _glusterd_urltransform_add_iter, &runner);
-        glusterd_urltransform_add (&runner, slaveurl);
-
-        n = glusterd_urltransform (&runner, &linearr);
-        if (n == -1)
-                return -2;
-
-        for (i = 0; i < n - 1; i++) {
-                if (strcmp (linearr[i], linearr[n - 1]) == 0)
-                        break;
-        }
-        glusterd_urltransform_free (linearr, i);
-
-        if (i < n - 1)
-                *slavekey = dict_get_by_index (vol->gsync_slaves, i);
-        else
-                i = -1;
-
-        return i;
+    runner_t runner = {
+        0,
+    };
+    int n = 0;
+    int i = 0;
+    char **linearr = NULL;
+    int32_t ret = 0;
+
+    glusterd_urltransform_init(&runner, "canonicalize");
+    ret = dict_foreach(vol->gsync_slaves, _glusterd_urltransform_add_iter,
+                       &runner);
+    if (ret < 0)
+        return -2;
+
+    glusterd_urltransform_add(&runner, slaveurl);
+
+    n = glusterd_urltransform(&runner, &linearr);
+    if (n == -1)
+        return -2;
+
+    for (i = 0; i < n - 1; i++) {
+        if (strcmp(linearr[i], linearr[n - 1]) == 0)
+            break;
+    }
+    glusterd_urltransform_free(linearr, n);
+
+    if (i < n - 1)
+        *slavekey = dict_get_by_index(vol->gsync_slaves, i);
+    else
+        i = -1;
+
+    return i;
 }
 
-
 static int
-glusterd_query_extutil_generic (char *resbuf, size_t blen, runner_t *runner, void *data,
-                                int (*fcbk)(char *resbuf, size_t blen, FILE *fp, void *data))
+glusterd_query_extutil_generic(char *resbuf, size_t blen, runner_t *runner,
+                               void *data,
+                               int (*fcbk)(char *resbuf, size_t blen, FILE *fp,
+                                           void *data))
 {
-        int                 ret = 0;
+    int ret = 0;
+    xlator_t *this = NULL;
 
-        runner_redir (runner, STDOUT_FILENO, RUN_PIPE);
-        if (runner_start (runner) != 0) {
-                gf_log ("", GF_LOG_ERROR, "spawning child failed");
+    this = THIS;
+    GF_ASSERT(this);
 
-                return -1;
-        }
+    runner_redir(runner, STDOUT_FILENO, RUN_PIPE);
+    if (runner_start(runner) != 0) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_SPAWNING_CHILD_FAILED,
+               "spawning child failed");
 
-        ret = fcbk (resbuf, blen, runner_chio (runner, STDOUT_FILENO), data);
+        return -1;
+    }
 
-        ret |= runner_end (runner);
-        if (ret)
-                gf_log ("", GF_LOG_ERROR, "reading data from child failed");
+    ret = fcbk(resbuf, blen, runner_chio(runner, STDOUT_FILENO), data);
+
+    ret |= runner_end(runner);
+    if (ret)
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_READ_CHILD_DATA_FAILED,
+               "reading data from child failed");
 
-        return ret ? -1 : 0;
+    return ret ? -1 : 0;
 }
 
 static int
 _fcbk_singleline(char *resbuf, size_t blen, FILE *fp, void *data)
 {
-        char *ptr = NULL;
+    char *ptr = NULL;
 
-        errno = 0;
-        ptr = fgets (resbuf, blen, fp);
-        if (ptr) {
-                size_t len = strlen(resbuf);
-                if (len && resbuf[len-1] == '\n')
-                        resbuf[len-1] = '\0'; //strip off \n
-        }
+    errno = 0;
+    ptr = fgets(resbuf, blen, fp);
+    if (ptr) {
+        size_t len = strlen(resbuf);
+        if (len && resbuf[len - 1] == '\n')
+            resbuf[len - 1] = '\0';  // strip off \n
+    }
 
-        return errno ? -1 : 0;
+    return errno ? -1 : 0;
 }
 
 static int
-glusterd_query_extutil (char *resbuf, runner_t *runner)
+glusterd_query_extutil(char *resbuf, runner_t *runner)
 {
-        return glusterd_query_extutil_generic (resbuf, PATH_MAX, runner, NULL,
-                                               _fcbk_singleline);
+    return glusterd_query_extutil_generic(resbuf, PATH_MAX, runner, NULL,
+                                          _fcbk_singleline);
 }
 
 static int
-_fcbk_conftodict (char *resbuf, size_t blen, FILE *fp, void *data)
+glusterd_get_slave_voluuid(char *slave_host, char *slave_vol, char *vol_uuid)
 {
-        char   *ptr = NULL;
-        dict_t *dict = data;
-        char   *v = NULL;
-
-        for (;;) {
-                errno = 0;
-                ptr = fgets (resbuf, blen, fp);
-                if (!ptr)
-                        break;
-                v = resbuf + strlen(resbuf) - 1;
-                while (isspace (*v))
-                        /* strip trailing space */
-                        *v-- = '\0';
-                if (v == resbuf)
-                        /* skip empty line */
-                        continue;
-                v = strchr (resbuf, ':');
-                if (!v)
-                        return -1;
-                *v++ = '\0';
-                while (isspace (*v))
-                        v++;
-                v = gf_strdup (v);
-                if (!v)
-                        return -1;
-                if (dict_set_dynstr (dict, resbuf, v) != 0) {
-                        GF_FREE (v);
-                        return -1;
-                }
-        }
+    runner_t runner = {
+        0,
+    };
+    glusterd_conf_t *priv = NULL;
+    xlator_t *this = NULL;
+    int ret = -1;
+
+    this = THIS;
+    GF_VALIDATE_OR_GOTO("glusterd", this, out);
+
+    priv = this->private;
+    GF_VALIDATE_OR_GOTO(this->name, priv, out);
+
+    runinit(&runner);
+    runner_add_arg(&runner, GSYNCD_PREFIX "/gsyncd");
+    set_gsyncd_inet6_arg(&runner);
+    runner_add_arg(&runner, "--slavevoluuid-get");
+    runner_argprintf(&runner, "%s::%s", slave_host, slave_vol);
 
-        return errno ? -1 : 0;
+    synclock_unlock(&priv->big_lock);
+    ret = glusterd_query_extutil(vol_uuid, &runner);
+    synclock_lock(&priv->big_lock);
+
+out:
+    return ret;
 }
 
 static int
-glusterd_gsync_get_config (char *master, char *slave, char *conf_path, dict_t *dict)
+_fcbk_conftodict(char *resbuf, size_t blen, FILE *fp, void *data)
 {
-        /* key + value, where value must be able to accommodate a path */
-        char resbuf[256 + PATH_MAX] = {0,};
-        runner_t             runner = {0,};
-
-        runinit (&runner);
-        runner_add_args  (&runner, GSYNCD_PREFIX"/gsyncd", "-c", NULL);
-        runner_argprintf (&runner, "%s", conf_path);
-        runner_argprintf (&runner, "--iprefix=%s", DATADIR);
-        runner_argprintf (&runner, ":%s", master);
-        runner_add_args  (&runner, slave, "--config-get-all", NULL);
-
-        return glusterd_query_extutil_generic (resbuf, sizeof (resbuf),
-                                               &runner, dict, _fcbk_conftodict);
+    char *ptr = NULL;
+    dict_t *dict = data;
+    char *v = NULL;
+
+    for (;;) {
+        errno = 0;
+        ptr = fgets(resbuf, blen - 2, fp);
+        if (!ptr)
+            break;
+        v = resbuf + strlen(resbuf) - 1;
+        while (isspace(*v))
+            /* strip trailing space */
+            *v-- = '\0';
+        if (v == resbuf)
+            /* skip empty line */
+            continue;
+        v = strchr(resbuf, ':');
+        if (!v)
+            return -1;
+        *v++ = '\0';
+        while (isspace(*v))
+            v++;
+        v = gf_strdup(v);
+        if (!v)
+            return -1;
+        if (dict_set_dynstr(dict, resbuf, v) != 0) {
+            GF_FREE(v);
+            return -1;
+        }
+    }
+
+    return errno ? -1 : 0;
 }
 
 static int
-glusterd_gsync_get_param_file (char *prmfile, const char *param, char *master,
-                               char *slave, char *conf_path)
+glusterd_gsync_get_config(char *master, char *slave, char *conf_path,
+                          dict_t *dict)
 {
-        runner_t            runner = {0,};
-
-        runinit (&runner);
-        runner_add_args  (&runner, GSYNCD_PREFIX"/gsyncd", "-c", NULL);
-        runner_argprintf (&runner, "%s", conf_path);
-        runner_argprintf (&runner, "--iprefix=%s", DATADIR);
-        runner_argprintf (&runner, ":%s", master);
-        runner_add_args  (&runner, slave, "--config-get", NULL);
-        runner_argprintf (&runner, "%s-file", param);
-
-        return glusterd_query_extutil (prmfile, &runner);
+    /* key + value, where value must be able to accommodate a path */
+    char resbuf[256 + PATH_MAX] = {
+        0,
+    };
+    runner_t runner = {
+        0,
+    };
+
+    runinit(&runner);
+    runner_add_args(&runner, GSYNCD_PREFIX "/gsyncd", "-c", NULL);
+    runner_argprintf(&runner, "%s", conf_path);
+    set_gsyncd_inet6_arg(&runner);
+    runner_argprintf(&runner, "--iprefix=%s", DATADIR);
+    runner_argprintf(&runner, ":%s", master);
+    runner_add_args(&runner, slave, "--config-get-all", NULL);
+
+    return glusterd_query_extutil_generic(resbuf, sizeof(resbuf), &runner, dict,
+                                          _fcbk_conftodict);
 }
 
 static int
-gsyncd_getpidfile (char *master, char *slave, char *pidfile,
-                   char *conf_path, gf_boolean_t *is_template_in_use)
+_fcbk_statustostruct(char *resbuf, size_t blen, FILE *fp, void *data)
 {
-        char                temp_conf_path[PATH_MAX] = "";
-        char               *working_conf_path        = NULL;
-        glusterd_conf_t    *priv                     = NULL;
-        int                 ret                      = -1;
-        struct stat         stbuf                    = {0,};
+    char *ptr = NULL;
+    char *v = NULL;
+    char *k = NULL;
+    gf_gsync_status_t *sts_val = NULL;
+    size_t len = 0;
 
-        GF_ASSERT (THIS);
-        GF_ASSERT (THIS->private);
-        GF_ASSERT (conf_path);
+    sts_val = (gf_gsync_status_t *)data;
 
-        priv = THIS->private;
+    for (;;) {
+        errno = 0;
+        ptr = fgets(resbuf, blen - 2, fp);
+        if (!ptr)
+            break;
+
+        v = resbuf + strlen(resbuf) - 1;
+        while (isspace(*v))
+            /* strip trailing space */
+            *v-- = '\0';
+        if (v == resbuf)
+            /* skip empty line */
+            continue;
+        v = strchr(resbuf, ':');
+        if (!v)
+            return -1;
+        *v++ = '\0';
+        while (isspace(*v))
+            v++;
+        v = gf_strdup(v);
+        if (!v)
+            return -1;
+
+        k = gf_strdup(resbuf);
+        if (!k) {
+            GF_FREE(v);
+            return -1;
+        }
+
+        if (strcmp(k, "worker_status") == 0) {
+            len = min(strlen(v), (sizeof(sts_val->worker_status) - 1));
+            memcpy(sts_val->worker_status, v, len);
+            sts_val->worker_status[len] = '\0';
+        } else if (strcmp(k, "slave_node") == 0) {
+            len = min(strlen(v), (sizeof(sts_val->slave_node) - 1));
+            memcpy(sts_val->slave_node, v, len);
+            sts_val->slave_node[len] = '\0';
+        } else if (strcmp(k, "crawl_status") == 0) {
+            len = min(strlen(v), (sizeof(sts_val->crawl_status) - 1));
+            memcpy(sts_val->crawl_status, v, len);
+            sts_val->crawl_status[len] = '\0';
+        } else if (strcmp(k, "last_synced") == 0) {
+            len = min(strlen(v), (sizeof(sts_val->last_synced) - 1));
+            memcpy(sts_val->last_synced, v, len);
+            sts_val->last_synced[len] = '\0';
+        } else if (strcmp(k, "last_synced_utc") == 0) {
+            len = min(strlen(v), (sizeof(sts_val->last_synced_utc) - 1));
+            memcpy(sts_val->last_synced_utc, v, len);
+            sts_val->last_synced_utc[len] = '\0';
+        } else if (strcmp(k, "entry") == 0) {
+            len = min(strlen(v), (sizeof(sts_val->entry) - 1));
+            memcpy(sts_val->entry, v, len);
+            sts_val->entry[len] = '\0';
+        } else if (strcmp(k, "data") == 0) {
+            len = min(strlen(v), (sizeof(sts_val->data) - 1));
+            memcpy(sts_val->data, v, len);
+            sts_val->data[len] = '\0';
+        } else if (strcmp(k, "meta") == 0) {
+            len = min(strlen(v), (sizeof(sts_val->meta) - 1));
+            memcpy(sts_val->meta, v, len);
+            sts_val->meta[len] = '\0';
+        } else if (strcmp(k, "failures") == 0) {
+            len = min(strlen(v), (sizeof(sts_val->failures) - 1));
+            memcpy(sts_val->failures, v, len);
+            sts_val->failures[len] = '\0';
+        } else if (strcmp(k, "checkpoint_time") == 0) {
+            len = min(strlen(v), (sizeof(sts_val->checkpoint_time) - 1));
+            memcpy(sts_val->checkpoint_time, v, len);
+            sts_val->checkpoint_time[len] = '\0';
+        } else if (strcmp(k, "checkpoint_time_utc") == 0) {
+            len = min(strlen(v), (sizeof(sts_val->checkpoint_time_utc) - 1));
+            memcpy(sts_val->checkpoint_time_utc, v, len);
+            sts_val->checkpoint_time_utc[len] = '\0';
+        } else if (strcmp(k, "checkpoint_completed") == 0) {
+            len = min(strlen(v), (sizeof(sts_val->checkpoint_completed) - 1));
+            memcpy(sts_val->checkpoint_completed, v, len);
+            sts_val->checkpoint_completed[len] = '\0';
+        } else if (strcmp(k, "checkpoint_completion_time") == 0) {
+            len = min(strlen(v),
+                      (sizeof(sts_val->checkpoint_completion_time) - 1));
+            memcpy(sts_val->checkpoint_completion_time, v, len);
+            sts_val->checkpoint_completion_time[len] = '\0';
+        } else if (strcmp(k, "checkpoint_completion_time_utc") == 0) {
+            len = min(strlen(v),
+                      (sizeof(sts_val->checkpoint_completion_time_utc) - 1));
+            memcpy(sts_val->checkpoint_completion_time_utc, v, len);
+            sts_val->checkpoint_completion_time_utc[len] = '\0';
+        }
+        GF_FREE(v);
+        GF_FREE(k);
+    }
+
+    return errno ? -1 : 0;
+}
 
-        GF_VALIDATE_OR_GOTO ("gsync", master, out);
-        GF_VALIDATE_OR_GOTO ("gsync", slave, out);
+static int
+glusterd_gsync_get_status(char *master, char *slave, char *conf_path,
+                          char *brick_path, gf_gsync_status_t *sts_val)
+{
+    /* key + value, where value must be able to accommodate a path */
+    char resbuf[256 + PATH_MAX] = {
+        0,
+    };
+    runner_t runner = {
+        0,
+    };
+
+    runinit(&runner);
+    runner_add_args(&runner, GSYNCD_PREFIX "/gsyncd", "-c", NULL);
+    runner_argprintf(&runner, "%s", conf_path);
+    set_gsyncd_inet6_arg(&runner);
+    runner_argprintf(&runner, "--iprefix=%s", DATADIR);
+    runner_argprintf(&runner, ":%s", master);
+    runner_add_args(&runner, slave, "--status-get", NULL);
+    runner_add_args(&runner, "--path", brick_path, NULL);
+
+    return glusterd_query_extutil_generic(resbuf, sizeof(resbuf), &runner,
+                                          sts_val, _fcbk_statustostruct);
+}
 
-        snprintf (temp_conf_path, sizeof(temp_conf_path) - 1,
-                  "%s/"GSYNC_CONF_TEMPLATE, priv->workdir);
+static int
+glusterd_gsync_get_param_file(char *prmfile, const char *param, char *master,
+                              char *slave, char *conf_path)
+{
+    runner_t runner = {
+        0,
+    };
+
+    runinit(&runner);
+    runner_add_args(&runner, GSYNCD_PREFIX "/gsyncd", "-c", NULL);
+    runner_argprintf(&runner, "%s", conf_path);
+    set_gsyncd_inet6_arg(&runner);
+    runner_argprintf(&runner, "--iprefix=%s", DATADIR);
+    runner_argprintf(&runner, ":%s", master);
+    runner_add_args(&runner, slave, "--config-get", NULL);
+    runner_argprintf(&runner, "%s-file", param);
+
+    return glusterd_query_extutil(prmfile, &runner);
+}
 
-        ret = lstat (conf_path, &stbuf);
-        if (!ret) {
-                gf_log ("", GF_LOG_DEBUG, "Using passed config template(%s).",
-                        conf_path);
-                working_conf_path = conf_path;
-        } else {
-                gf_log ("", GF_LOG_WARNING, "Config file (%s) missing. "
-                        "Looking for template config file (%s)",
-                        conf_path, temp_conf_path);
-                ret = lstat (temp_conf_path, &stbuf);
-                if (ret) {
-                        gf_log ("", GF_LOG_ERROR,
-                                "Template config file (%s) missing.",
-                                temp_conf_path);
-                        goto out;
-                }
-                gf_log ("", GF_LOG_INFO, "Using default config template(%s).",
-                        temp_conf_path);
-                working_conf_path = temp_conf_path;
-                *is_template_in_use = _gf_true;
+static int
+gsyncd_getpidfile(char *master, char *slave, char *pidfile, char *conf_path,
+                  gf_boolean_t *is_template_in_use)
+{
+    char temp_conf_path[PATH_MAX] = "";
+    char *working_conf_path = NULL;
+    glusterd_conf_t *priv = NULL;
+    int ret = -1;
+    struct stat stbuf = {
+        0,
+    };
+    xlator_t *this = NULL;
+    int32_t len = 0;
+
+    this = THIS;
+    GF_ASSERT(this);
+
+    GF_ASSERT(this->private);
+    GF_ASSERT(conf_path);
+
+    priv = this->private;
+
+    GF_VALIDATE_OR_GOTO("gsync", master, out);
+    GF_VALIDATE_OR_GOTO("gsync", slave, out);
+
+    len = snprintf(temp_conf_path, sizeof(temp_conf_path),
+                   "%s/" GSYNC_CONF_TEMPLATE, priv->workdir);
+    if ((len < 0) || (len >= sizeof(temp_conf_path))) {
+        goto out;
+    }
+
+    ret = sys_lstat(conf_path, &stbuf);
+    if (!ret) {
+        gf_msg_debug(this->name, 0, "Using passed config template(%s).",
+                     conf_path);
+        working_conf_path = conf_path;
+    } else {
+        gf_msg(this->name, GF_LOG_WARNING, ENOENT, GD_MSG_FILE_OP_FAILED,
+               "Config file (%s) missing. Looking for template "
+               "config file (%s)",
+               conf_path, temp_conf_path);
+        ret = sys_lstat(temp_conf_path, &stbuf);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, ENOENT, GD_MSG_FILE_OP_FAILED,
+                   "Template config file (%s) missing.", temp_conf_path);
+            goto out;
         }
+        gf_msg(this->name, GF_LOG_INFO, 0, GD_MSG_DEFAULT_TEMP_CONFIG,
+               "Using default config template(%s).", temp_conf_path);
+        working_conf_path = temp_conf_path;
+        *is_template_in_use = _gf_true;
+    }
 
 fetch_data:
 
-        ret = glusterd_gsync_get_param_file (pidfile, "pid", master,
-                                             slave, working_conf_path);
-        if ((ret == -1) || strlen(pidfile) == 0) {
-                if (*is_template_in_use == _gf_false) {
-                        gf_log ("", GF_LOG_WARNING,
-                                "failed to create the pidfile string. "
-                                "Trying default config template");
-                        working_conf_path = temp_conf_path;
-                        *is_template_in_use = _gf_true;
-                        goto fetch_data;
-                } else {
-                        ret = -2;
-                        gf_log ("", GF_LOG_WARNING,
-                                "failed to create the pidfile string from template config");
-                        goto out;
-                }
+    ret = glusterd_gsync_get_param_file(pidfile, "pid", master, slave,
+                                        working_conf_path);
+    if ((ret == -1) || strlen(pidfile) == 0) {
+        if (*is_template_in_use == _gf_false) {
+            gf_msg(this->name, GF_LOG_WARNING, 0, GD_MSG_PIDFILE_CREATE_FAILED,
+                   "failed to create the pidfile string. "
+                   "Trying default config template");
+            working_conf_path = temp_conf_path;
+            *is_template_in_use = _gf_true;
+            goto fetch_data;
+        } else {
+            ret = -2;
+            gf_msg(this->name, GF_LOG_WARNING, 0, GD_MSG_PIDFILE_CREATE_FAILED,
+                   "failed to "
+                   "create the pidfile string from template "
+                   "config");
+            goto out;
         }
+    }
 
-        gf_log ("", GF_LOG_DEBUG, "pidfile = %s", pidfile);
+    gf_msg_debug(this->name, 0, "pidfile = %s", pidfile);
 
-        ret = open (pidfile, O_RDWR);
- out:
-        return ret;
+    ret = open(pidfile, O_RDWR);
+out:
+    return ret;
 }
 
 static int
-gsync_status_byfd (int fd)
+gsync_status_byfd(int fd)
 {
-        GF_ASSERT (fd >= -1);
+    GF_ASSERT(fd >= -1);
 
-        if (lockf (fd, F_TEST, 0) == -1 &&
-            (errno == EAGAIN || errno == EACCES))
-                /* gsyncd keeps the pidfile locked */
-                return 0;
+    if (lockf(fd, F_TEST, 0) == -1 && (errno == EAGAIN || errno == EACCES))
+        /* gsyncd keeps the pidfile locked */
+        return 0;
 
-        return -1;
+    return -1;
 }
 
 /* status: return 0 when gsync is running
  * return -1 when not running
  */
 int
-gsync_status (char *master, char *slave, char *conf_path,
-              int *status, gf_boolean_t *is_template_in_use)
+gsync_status(char *master, char *slave, char *conf_path, int *status,
+             gf_boolean_t *is_template_in_use)
 {
-        char pidfile[PATH_MAX] = {0,};
-        int  fd                = -1;
-
-        fd = gsyncd_getpidfile (master, slave, pidfile,
-                                conf_path, is_template_in_use);
-        if (fd == -2)
-                return -1;
+    char pidfile[PATH_MAX] = {
+        0,
+    };
+    int fd = -1;
+
+    fd = gsyncd_getpidfile(master, slave, pidfile, conf_path,
+                           is_template_in_use);
+    if (fd == -2)
+        return -1;
 
-        *status = gsync_status_byfd (fd);
+    *status = gsync_status_byfd(fd);
 
-        sys_close (fd);
+    sys_close(fd);
 
-        return 0;
+    return 0;
 }
 
-
 static int32_t
-glusterd_gsync_volinfo_dict_set (glusterd_volinfo_t *volinfo,
-                                 char *key, char *value)
+glusterd_gsync_volinfo_dict_set(glusterd_volinfo_t *volinfo, char *key,
+                                char *value)
 {
-        int32_t  ret            = -1;
-        char    *gsync_status   = NULL;
-
-        gsync_status = gf_strdup (value);
-        if (!gsync_status) {
-                gf_log ("", GF_LOG_ERROR, "Unable to allocate memory");
-                goto out;
-        }
-
-        ret = dict_set_dynstr (volinfo->dict, key, gsync_status);
-        if (ret) {
-                gf_log ("", GF_LOG_ERROR, "Unable to set dict");
-                goto out;
-        }
-
-        ret = 0;
+    int32_t ret = -1;
+    char *gsync_status = NULL;
+    xlator_t *this = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+
+    gsync_status = gf_strdup(value);
+    if (!gsync_status) {
+        gf_msg(this->name, GF_LOG_ERROR, ENOMEM, GD_MSG_NO_MEMORY,
+               "Unable to allocate memory");
+        goto out;
+    }
+
+    ret = dict_set_dynstr(volinfo->dict, key, gsync_status);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+               "Unable to set dict");
+        goto out;
+    }
+
+    ret = 0;
 out:
-        return 0;
+    return ret;
 }
 
 static int
-glusterd_verify_gsyncd_spawn (char *master, char *slave)
+glusterd_verify_gsyncd_spawn(char *master, char *slave)
 {
-        int                 ret = 0;
-        runner_t            runner = {0,};
-
-        runinit (&runner);
-        runner_add_args (&runner, GSYNCD_PREFIX"/gsyncd",
-                         "--verify", "spawning", NULL);
-        runner_argprintf (&runner, ":%s", master);
-        runner_add_args  (&runner, slave, NULL);
-        runner_redir (&runner, STDOUT_FILENO, RUN_PIPE);
-        ret = runner_start (&runner);
-        if (ret) {
-                gf_log ("", GF_LOG_ERROR, "spawning child failed");
-                ret = -1;
-                goto out;
-        }
-
-        if (runner_end (&runner) != 0)
-                ret = -1;
+    int ret = 0;
+    runner_t runner = {
+        0,
+    };
+    xlator_t *this = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+
+    runinit(&runner);
+    runner_add_args(&runner, GSYNCD_PREFIX "/gsyncd", "--verify", "spawning",
+                    NULL);
+    runner_argprintf(&runner, ":%s", master);
+    runner_add_args(&runner, slave, NULL);
+    runner_redir(&runner, STDOUT_FILENO, RUN_PIPE);
+    ret = runner_start(&runner);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_SPAWNING_CHILD_FAILED,
+               "spawning child failed");
+        ret = -1;
+        goto out;
+    }
+
+    if (runner_end(&runner) != 0)
+        ret = -1;
 
 out:
-        gf_log ("", GF_LOG_DEBUG, "returning %d", ret);
-        return ret;
+    gf_msg_debug(this->name, 0, "returning %d", ret);
+    return ret;
 }
 
 static int
-gsync_verify_config_options (dict_t *dict, char **op_errstr, char *volname)
+gsync_verify_config_options(dict_t *dict, char **op_errstr, char *volname)
 {
-        char                         **resopt           = NULL;
-        int                            i                = 0;
-        int                            ret              = -1;
-        char                          *subop            = NULL;
-        char                          *slave            = NULL;
-        char                          *op_name          = NULL;
-        char                          *op_value         = NULL;
-        char                          *t                = NULL;
-        char                           errmsg[PATH_MAX] = "";
-        gf_boolean_t                   banned           = _gf_true;
-        gf_boolean_t                   op_match         = _gf_true;
-        gf_boolean_t                   val_match        = _gf_true;
-        struct gsync_config_opt_vals_ *conf_vals        = NULL;
-
-        if (dict_get_str (dict, "subop", &subop) != 0) {
-                gf_log ("", GF_LOG_WARNING, "missing subop");
-                *op_errstr = gf_strdup ("Invalid config request");
-                return -1;
-        }
-
-        if (dict_get_str (dict, "slave", &slave) != 0) {
-                gf_log ("", GF_LOG_WARNING, GEOREP" CONFIG: no slave given");
-                *op_errstr = gf_strdup ("Slave required");
-                return -1;
-        }
-
-        if (strcmp (subop, "get-all") == 0)
-                return 0;
+    char **resopt = NULL;
+    int i = 0;
+    int ret = -1;
+    char *subop = NULL;
+    char *slave = NULL;
+    char *op_name = NULL;
+    char *op_value = NULL;
+    char *t = NULL;
+    char errmsg[PATH_MAX] = "";
+    gf_boolean_t banned = _gf_true;
+    gf_boolean_t op_match = _gf_true;
+    gf_boolean_t val_match = _gf_true;
+    struct gsync_config_opt_vals_ *conf_vals = NULL;
+    xlator_t *this = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+
+    if (dict_get_str(dict, "subop", &subop) != 0) {
+        gf_msg(this->name, GF_LOG_WARNING, 0, GD_MSG_DICT_GET_FAILED,
+               "missing subop");
+        *op_errstr = gf_strdup("Invalid config request");
+        return -1;
+    }
 
-        if (dict_get_str (dict, "op_name", &op_name) != 0) {
-                gf_log ("", GF_LOG_WARNING, "option name missing");
-                *op_errstr = gf_strdup ("Option name missing");
-                return -1;
-        }
+    if (dict_get_str(dict, "slave", &slave) != 0) {
+        gf_msg(this->name, GF_LOG_WARNING, 0, GD_MSG_DICT_GET_FAILED,
+               GEOREP " CONFIG: no slave given");
+        *op_errstr = gf_strdup("Slave required");
+        return -1;
+    }
 
-        if (runcmd (GSYNCD_PREFIX"/gsyncd", "--config-check", op_name, NULL)) {
-                ret = glusterd_verify_gsyncd_spawn (volname, slave);
-                if (ret) {
-                        gf_log ("", GF_LOG_ERROR, "Unable to spawn gsyncd");
-                        return 0;
-                }
+    if (strcmp(subop, "get-all") == 0)
+        return 0;
 
-                gf_log ("", GF_LOG_WARNING, "Invalid option %s", op_name);
-                *op_errstr = gf_strdup ("Invalid option");
+    if (dict_get_str(dict, "op_name", &op_name) != 0) {
+        gf_msg(this->name, GF_LOG_WARNING, 0, GD_MSG_DICT_GET_FAILED,
+               "option name missing");
+        *op_errstr = gf_strdup("Option name missing");
+        return -1;
+    }
 
-                return -1;
+    if (runcmd(GSYNCD_PREFIX "/gsyncd", "--config-check", op_name, NULL)) {
+        ret = glusterd_verify_gsyncd_spawn(volname, slave);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_GSYNCD_SPAWN_FAILED,
+                   "Unable to spawn "
+                   "gsyncd");
+            return 0;
         }
 
-        if (strcmp (subop, "get") == 0)
-                return 0;
+        gf_msg(this->name, GF_LOG_WARNING, EINVAL, GD_MSG_INVALID_ENTRY,
+               "Invalid option %s", op_name);
+        *op_errstr = gf_strdup("Invalid option");
 
-        t = strtail (subop, "set");
-        if (!t)
-                t = strtail (subop, "del");
-        if (!t || (t[0] && strcmp (t, "-glob") != 0)) {
-                gf_log ("", GF_LOG_WARNING, "unknown subop %s", subop);
-                *op_errstr = gf_strdup ("Invalid config request");
-                return -1;
-        }
-
-        if (strtail (subop, "set") &&
-            dict_get_str (dict, "op_value", &op_value) != 0) {
-                gf_log ("", GF_LOG_WARNING, "missing value for set");
-                *op_errstr = gf_strdup ("missing value");
-        }
+        return -1;
+    }
 
-        /* match option name against reserved options, modulo -/_
-         * difference
-         */
-        for (resopt = gsync_reserved_opts; *resopt; resopt++) {
-                banned = _gf_true;
-                for (i = 0; (*resopt)[i] && op_name[i]; i++) {
-                        if ((*resopt)[i] == op_name[i] ||
-                            ((*resopt)[i] == '-' && op_name[i] == '_'))
-                                continue;
-                        banned = _gf_false;
-                }
-                if (banned) {
-                        gf_log ("", GF_LOG_WARNING, "Reserved option %s", op_name);
-                        *op_errstr = gf_strdup ("Reserved option");
+    if (strcmp(subop, "get") == 0)
+        return 0;
 
-                        return -1;
-                        break;
+    t = strtail(subop, "set");
+    if (!t)
+        t = strtail(subop, "del");
+    if (!t || (t[0] && strcmp(t, "-glob") != 0)) {
+        gf_msg(this->name, GF_LOG_WARNING, 0, GD_MSG_SUBOP_NOT_FOUND,
+               "unknown subop %s", subop);
+        *op_errstr = gf_strdup("Invalid config request");
+        return -1;
+    }
+
+    if (strtail(subop, "set") &&
+        dict_get_str(dict, "op_value", &op_value) != 0) {
+        gf_msg(this->name, GF_LOG_WARNING, 0, GD_MSG_DICT_GET_FAILED,
+               "missing value for set");
+        *op_errstr = gf_strdup("missing value");
+    }
+
+    /* match option name against reserved options, modulo -/_
+     * difference
+     */
+    for (resopt = gsync_reserved_opts; *resopt; resopt++) {
+        banned = _gf_true;
+        for (i = 0; (*resopt)[i] && op_name[i]; i++) {
+            if ((*resopt)[i] == op_name[i] ||
+                ((*resopt)[i] == '-' && op_name[i] == '_'))
+                continue;
+            banned = _gf_false;
+        }
+
+        if (op_name[i] != '\0')
+            banned = _gf_false;
+
+        if (banned) {
+            gf_msg(this->name, GF_LOG_WARNING, 0, GD_MSG_RESERVED_OPTION,
+                   "Reserved option %s", op_name);
+            *op_errstr = gf_strdup("Reserved option");
+
+            return -1;
+            break;
+        }
+    }
+
+    /* Check options in gsync_confopt_vals for invalid values */
+    for (conf_vals = gsync_confopt_vals; conf_vals->op_name; conf_vals++) {
+        op_match = _gf_true;
+        for (i = 0; conf_vals->op_name[i] && op_name[i]; i++) {
+            if (conf_vals->op_name[i] == op_name[i] ||
+                (conf_vals->op_name[i] == '_' && op_name[i] == '-'))
+                continue;
+            op_match = _gf_false;
+        }
+
+        if (op_match) {
+            if (!op_value)
+                goto out;
+            val_match = _gf_false;
+            for (i = 0; i < conf_vals->no_of_pos_vals; i++) {
+                if (conf_vals->case_sensitive) {
+                    if (!strcmp(conf_vals->values[i], op_value))
+                        val_match = _gf_true;
+                } else {
+                    if (!strcasecmp(conf_vals->values[i], op_value))
+                        val_match = _gf_true;
                 }
-        }
+            }
 
-        /* Check options in gsync_confopt_vals for invalid values */
-        for (conf_vals = gsync_confopt_vals; conf_vals->op_name; conf_vals++) {
-                op_match = _gf_true;
-                for (i = 0; conf_vals->op_name[i] && op_name[i]; i++) {
-                        if (conf_vals->op_name[i] == op_name[i] ||
-                            (conf_vals->op_name[i] == '_' && op_name[i] == '-'))
-                                continue;
-                        op_match = _gf_false;
-                }
+            if (!val_match) {
+                ret = snprintf(errmsg, sizeof(errmsg) - 1,
+                               "Invalid value(%s) for"
+                               " option %s",
+                               op_value, op_name);
+                errmsg[ret] = '\0';
 
-                if (op_match) {
-                        if (!op_value)
-                                goto out;
-                        val_match = _gf_false;
-                        for (i = 0; i < conf_vals->no_of_pos_vals; i++) {
-                                if(conf_vals->case_sensitive){
-                                        if (!strcmp (conf_vals->values[i], op_value))
-                                                val_match = _gf_true;
-                                } else {
-                                        if (!strcasecmp (conf_vals->values[i], op_value))
-                                                val_match = _gf_true;
-                                }
-                        }
-
-                        if (!val_match) {
-                                ret = snprintf (errmsg, sizeof(errmsg) - 1,
-                                                "Invalid value(%s) for"
-                                                " option %s", op_value,
-                                                op_name);
-                                errmsg[ret] = '\0';
-
-                                gf_log ("", GF_LOG_ERROR, "%s", errmsg);
-                                *op_errstr = gf_strdup (errmsg);
-                                return -1;
-                        }
-                }
+                gf_msg(this->name, GF_LOG_ERROR, EINVAL, GD_MSG_INVALID_ENTRY,
+                       "%s", errmsg);
+                *op_errstr = gf_strdup(errmsg);
+                return -1;
+            }
         }
+    }
 out:
-        return 0;
+    return 0;
 }
 
 static int
-glusterd_get_gsync_status_mst_slv (glusterd_volinfo_t *volinfo,
-                                   char *slave, char *conf_path,
-                                   dict_t *rsp_dict, char *node);
+glusterd_get_gsync_status_mst_slv(glusterd_volinfo_t *volinfo, char *slave,
+                                  char *conf_path, dict_t *rsp_dict,
+                                  char *node);
 
 static int
-_get_status_mst_slv (dict_t *this, char *key, data_t *value, void *data)
+_get_status_mst_slv(dict_t *dict, char *key, data_t *value, void *data)
 {
-        glusterd_gsync_status_temp_t *param                = NULL;
-        char                         *slave                = NULL;
-        char                         *slave_buf            = NULL;
-        char                         *slave_url            = NULL;
-        char                         *slave_vol            = NULL;
-        char                         *slave_host           = NULL;
-        char                         *errmsg               = NULL;
-        char                           conf_path[PATH_MAX] = "";
-        int                           ret                  = -1;
-        glusterd_conf_t              *priv                 = NULL;
-
-        param = (glusterd_gsync_status_temp_t *)data;
-
-        GF_ASSERT (param);
-        GF_ASSERT (param->volinfo);
-
-        if (THIS)
-                priv = THIS->private;
-        if (priv == NULL) {
-                gf_log ("", GF_LOG_ERROR, "priv of glusterd not present");
-                goto out;
-        }
-
-        slave = strchr(value->data, ':');
-        if (!slave)
-                return 0;
-        slave++;
+    glusterd_gsync_status_temp_t *param = NULL;
+    char *slave = NULL;
+    char *slave_buf = NULL;
+    char *slave_url = NULL;
+    char *slave_vol = NULL;
+    char *slave_host = NULL;
+    char *errmsg = NULL;
+    char conf_path[PATH_MAX] = "";
+    int ret = -1;
+    glusterd_conf_t *priv = NULL;
+    xlator_t *this = NULL;
+    char slv_url[VOLINFO_SLAVE_URL_MAX] = {0};
+
+    this = THIS;
+    GF_VALIDATE_OR_GOTO("glusterd", this, out);
+
+    param = (glusterd_gsync_status_temp_t *)data;
+
+    GF_VALIDATE_OR_GOTO(this->name, param, out);
+    GF_VALIDATE_OR_GOTO(this->name, param->volinfo, out);
+
+    if (this)
+        priv = this->private;
+    GF_VALIDATE_OR_GOTO(this->name, priv, out);
+
+    if (snprintf(slv_url, sizeof(slv_url), "%s", value->data) >=
+        sizeof(slv_url)) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_SLAVE_VOL_PARSE_FAIL,
+               "Error in copying slave: %s!", value->data);
+        goto out;
+    }
+
+    ret = parse_slave_url(slv_url, &slave);
+    if (ret == -1) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_SLAVE_VOL_PARSE_FAIL,
+               "Error in parsing slave: %s!", value->data);
+        goto out;
+    }
+
+    ret = glusterd_get_slave_info(slave, &slave_url, &slave_host, &slave_vol,
+                                  &errmsg);
+    if (ret) {
+        if (errmsg)
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_SLAVEINFO_FETCH_ERROR,
+                   "Unable to fetch slave details. Error: %s", errmsg);
+        else
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_SLAVEINFO_FETCH_ERROR,
+                   "Unable to fetch slave details.");
+        ret = -1;
+        goto out;
+    }
+
+    ret = snprintf(conf_path, sizeof(conf_path) - 1,
+                   "%s/" GEOREP "/%s_%s_%s/gsyncd.conf", priv->workdir,
+                   param->volinfo->volname, slave_host, slave_vol);
+    conf_path[ret] = '\0';
+
+    ret = glusterd_get_gsync_status_mst_slv(param->volinfo, slave, conf_path,
+                                            param->rsp_dict, param->node);
+out:
 
-        ret = glusterd_get_slave_info (slave, &slave_url,
-                                       &slave_host, &slave_vol, &errmsg);
-        if (ret) {
-                if (errmsg)
-                        gf_log ("", GF_LOG_ERROR, "Unable to fetch "
-                                "slave details. Error: %s", errmsg);
-                else
-                        gf_log ("", GF_LOG_ERROR,
-                                "Unable to fetch slave details.");
-                ret = -1;
-                goto out;
-        }
+    if (errmsg)
+        GF_FREE(errmsg);
 
-        ret = snprintf (conf_path, sizeof(conf_path) - 1,
-                        "%s/"GEOREP"/%s_%s_%s/gsyncd.conf",
-                        priv->workdir, param->volinfo->volname,
-                        slave_host, slave_vol);
-        conf_path[ret] = '\0';
+    if (slave_buf)
+        GF_FREE(slave_buf);
 
-        ret = glusterd_get_gsync_status_mst_slv(param->volinfo,
-                                                slave, conf_path,
-                                                param->rsp_dict,
-                                                param->node);
-out:
+    if (slave_vol)
+        GF_FREE(slave_vol);
 
-        GF_FREE (errmsg);
+    if (slave_url)
+        GF_FREE(slave_url);
 
-        if (slave_buf)
-                GF_FREE(slave_buf);
+    if (slave_host)
+        GF_FREE(slave_host);
 
-        gf_log ("", GF_LOG_DEBUG, "Returning %d.", ret);
-        return ret;
+    gf_msg_debug(this ? this->name : "glusterd", 0, "Returning %d.", ret);
+    return ret;
 }
 
-
 static int
-_get_max_gsync_slave_num (dict_t *this, char *key, data_t *value, void *data)
+_get_max_gsync_slave_num(dict_t *dict, char *key, data_t *value, void *data)
 {
-        int  tmp_slvnum = 0;
-        int *slvnum = (int *)data;
+    int tmp_slvnum = 0;
+    int *slvnum = (int *)data;
 
-        sscanf (key, "slave%d", &tmp_slvnum);
-        if (tmp_slvnum > *slvnum)
-                *slvnum = tmp_slvnum;
+    sscanf(key, "slave%d", &tmp_slvnum);
+    if (tmp_slvnum > *slvnum)
+        *slvnum = tmp_slvnum;
 
-        return 0;
+    return 0;
 }
 
 static int
-glusterd_remove_slave_in_info (glusterd_volinfo_t *volinfo, char *slave,
-                               char **op_errstr)
+_get_slave_idx_slave_voluuid(dict_t *dict, char *key, data_t *value, void *data)
 {
-        int   zero_slave_entries = _gf_true;
-        int   ret = 0;
-        char *slavekey = NULL;
+    char *slave_info = NULL;
+    xlator_t *this = NULL;
+    struct slave_vol_config *slave_cfg = NULL;
+    int i = 0;
+    int ret = -1;
+    unsigned tmp_slvnum = 0;
+
+    this = THIS;
+    GF_VALIDATE_OR_GOTO("glusterd", this, out);
+
+    slave_cfg = data;
+
+    if (value)
+        slave_info = value->data;
+
+    if (!(slave_info) || strlen(slave_info) == 0) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_INVALID_SLAVE,
+               "Invalid slave in dict");
+        ret = -2;
+        goto out;
+    }
+
+    /* slave format:
+     * master_node_uuid:ssh://slave_host::slave_vol:slave_voluuid */
+    while (i++ < 5) {
+        slave_info = strchr(slave_info, ':');
+        if (slave_info)
+            slave_info++;
+        else {
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_SLAVE_VOL_PARSE_FAIL,
+                   "slave_info becomes NULL!");
+            ret = -2;
+            goto out;
+        }
+    }
+    if (strcmp(slave_info, slave_cfg->slave_voluuid) == 0) {
+        gf_msg_debug(this->name, 0,
+                     "Same slave volume "
+                     "already present %s",
+                     slave_cfg->slave_voluuid);
+        ret = -1;
+
+        sscanf(key, "slave%d", &tmp_slvnum);
+        slave_cfg->old_slvidx = tmp_slvnum;
+
+        gf_msg_debug(this->name, 0,
+                     "and "
+                     "its index is: %d",
+                     tmp_slvnum);
+        goto out;
+    }
+
+    ret = 0;
+out:
+    return ret;
+}
 
-        GF_ASSERT (volinfo);
-        GF_ASSERT (slave);
+static int
+glusterd_remove_slave_in_info(glusterd_volinfo_t *volinfo, char *slave,
+                              char **op_errstr)
+{
+    int zero_slave_entries = _gf_true;
+    int ret = 0;
+    char *slavekey = NULL;
+    xlator_t *this = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+
+    GF_ASSERT(volinfo);
+    GF_ASSERT(slave);
+
+    do {
+        ret = glusterd_get_slave(volinfo, slave, &slavekey);
+        if (ret < 0 && zero_slave_entries) {
+            ret++;
+            goto out;
+        }
+        zero_slave_entries = _gf_false;
+        dict_del(volinfo->gsync_slaves, slavekey);
+    } while (ret >= 0);
+
+    ret = glusterd_store_volinfo(volinfo, GLUSTERD_VOLINFO_VER_AC_INCREMENT);
+    if (ret) {
+        *op_errstr = gf_strdup(
+            "Failed to store the Volume"
+            "information");
+        goto out;
+    }
+out:
+    gf_msg_debug(this->name, 0, "returning %d", ret);
+    return ret;
+}
 
-        do {
-                ret = glusterd_get_slave (volinfo, slave, &slavekey);
-                if (ret < 0 && zero_slave_entries) {
-                        ret++;
-                        goto out;
-                }
-                zero_slave_entries = _gf_false;
-                dict_del (volinfo->gsync_slaves, slavekey);
-        } while (ret >= 0);
+static int
+glusterd_gsync_get_uuid(char *slave, glusterd_volinfo_t *vol, uuid_t uuid)
+{
+    int ret = 0;
+    char *slavekey = NULL;
+    char *slaveentry = NULL;
+    char *t = NULL;
+    xlator_t *this = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+
+    GF_ASSERT(vol);
+    GF_ASSERT(slave);
+
+    ret = glusterd_get_slave(vol, slave, &slavekey);
+    if (ret < 0) {
+        /* XXX colliding cases of failure and non-extant
+         * slave... now just doing this as callers of this
+         * function can make sense only of -1 and 0 as retvals;
+         * getting at the proper semanticals will involve
+         * fixing callers as well.
+         */
+        ret = -1;
+        goto out;
+    }
 
-        ret = glusterd_store_volinfo (volinfo,
-                                      GLUSTERD_VOLINFO_VER_AC_INCREMENT);
-        if (ret) {
-                 *op_errstr = gf_strdup ("Failed to store the Volume"
-                                         "information");
-                goto out;
-        }
- out:
-        gf_log ("", GF_LOG_DEBUG, "returning %d", ret);
-        return ret;
+    ret = dict_get_str(vol->gsync_slaves, slavekey, &slaveentry);
+    GF_ASSERT(ret == 0);
+
+    t = strchr(slaveentry, ':');
+    GF_ASSERT(t);
+    *t = '\0';
+    ret = gf_uuid_parse(slaveentry, uuid);
+    *t = ':';
 
+out:
+    gf_msg_debug(this->name, 0, "Returning %d", ret);
+    return ret;
 }
 
 static int
-glusterd_gsync_get_uuid (char *slave, glusterd_volinfo_t *vol,
-                         uuid_t uuid)
+update_slave_voluuid(dict_t *dict, char *key, data_t *value, void *data)
 {
-        int   ret = 0;
-        char *slavekey = NULL;
-        char *slaveentry = NULL;
-        char *t = NULL;
+    char *slave = NULL;
+    char *slave_url = NULL;
+    char *slave_vol = NULL;
+    char *slave_host = NULL;
+    char *errmsg = NULL;
+    xlator_t *this = NULL;
+    int ret = -1;
+    char slv_url[VOLINFO_SLAVE_URL_MAX] = {0};
+    char slave_voluuid[GF_UUID_BUF_SIZE] = {0};
+    char *slave_info = NULL;
+    char *new_value = NULL;
+    char *same_key = NULL;
+    int cnt = 0;
+    gf_boolean_t *voluuid_updated = NULL;
+
+    this = THIS;
+
+    voluuid_updated = data;
+    slave_info = value->data;
+    gf_msg_debug(this->name, 0, "slave_info: %s!", slave_info);
+
+    /* old slave format:
+     * master_node_uuid:ssh://slave_host::slave_vol
+     * New slave format:
+     * master_node_uuid:ssh://slave_host::slave_vol:slave_voluuid */
+    while (slave_info) {
+        slave_info = strchr(slave_info, ':');
+        if (slave_info)
+            cnt++;
+        else
+            break;
 
-        GF_ASSERT (vol);
-        GF_ASSERT (slave);
+        slave_info++;
+    }
 
-        ret = glusterd_get_slave (vol, slave, &slavekey);
-        if (ret < 0) {
-                /* XXX colliding cases of failure and non-extant
-                 * slave... now just doing this as callers of this
-                 * function can make sense only of -1 and 0 as retvals;
-                 * getting at the proper semanticals will involve
-                 * fixing callers as well.
-                 */
-                ret = -1;
-                goto out;
+    gf_msg_debug(this->name, 0, "cnt: %d", cnt);
+    /* check whether old slave format and update vol uuid if old format.
+     * With volume uuid, number of ':' is 5 and is 4 without.
+     */
+    if (cnt == 4) {
+        if (snprintf(slv_url, sizeof(slv_url), "%s", value->data) >=
+            sizeof(slv_url)) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_SLAVE_VOL_PARSE_FAIL,
+                   "Error in copying slave: %s!", value->data);
+            goto out;
+        }
+
+        ret = parse_slave_url(slv_url, &slave);
+        if (ret == -1) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_SLAVE_VOL_PARSE_FAIL,
+                   "Error in parsing slave: %s!", value->data);
+            goto out;
         }
 
-        ret = dict_get_str (vol->gsync_slaves, slavekey, &slaveentry);
-        GF_ASSERT (ret == 0);
+        ret = glusterd_get_slave_info(slave, &slave_url, &slave_host,
+                                      &slave_vol, &errmsg);
+        if (ret) {
+            if (errmsg)
+                gf_msg(this->name, GF_LOG_ERROR, 0,
+                       GD_MSG_SLAVEINFO_FETCH_ERROR,
+                       "Unable to fetch slave details. Error: %s", errmsg);
+            else
+                gf_msg(this->name, GF_LOG_ERROR, 0,
+                       GD_MSG_SLAVEINFO_FETCH_ERROR,
+                       "Unable to fetch slave details.");
+            ret = -1;
+            goto out;
+        }
+
+        ret = glusterd_get_slave_voluuid(slave_host, slave_vol, slave_voluuid);
+        if ((ret) || (strlen(slave_voluuid) == 0)) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_REMOTE_VOL_UUID_FAIL,
+                   "Unable to get remote volume uuid"
+                   "slavehost:%s slavevol:%s",
+                   slave_host, slave_vol);
+            /* Avoiding failure due to remote vol uuid fetch */
+            ret = 0;
+            goto out;
+        }
+        ret = gf_asprintf(&new_value, "%s:%s", value->data, slave_voluuid);
+        ret = gf_asprintf(&same_key, "%s", key);
+
+        /* delete old key and add new value */
+        dict_del(dict, key);
+
+        /* set new value for the same key*/
+        ret = dict_set_dynstr(dict, same_key, new_value);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_REMOTE_VOL_UUID_FAIL,
+                   "Error in setting dict value"
+                   "new_value :%s",
+                   new_value);
+            goto out;
+        }
+        *voluuid_updated = _gf_true;
+    }
+
+    ret = 0;
+out:
+    if (errmsg)
+        GF_FREE(errmsg);
+
+    if (slave_url)
+        GF_FREE(slave_url);
 
-        t = strchr (slaveentry, ':');
-        GF_ASSERT (t);
-        *t = '\0';
-        ret = gf_uuid_parse (slaveentry, uuid);
-        *t = ':';
+    if (slave_vol)
+        GF_FREE(slave_vol);
 
- out:
-        gf_log ("", GF_LOG_DEBUG, "Returning %d", ret);
-        return ret;
+    if (slave_host)
+        GF_FREE(slave_host);
+
+    gf_msg_debug(this->name, 0, "Returning %d.", ret);
+    return ret;
 }
 
-int
-glusterd_check_gsync_running_local (char *master, char *slave,
-                                    char *conf_path,
-                                    gf_boolean_t *is_run)
+static int
+glusterd_update_slave_voluuid_slaveinfo(glusterd_volinfo_t *volinfo)
 {
-        int                 ret    = -1;
-        int                 ret_status = 0;
-        gf_boolean_t        is_template_in_use = _gf_false;
-
-        GF_ASSERT (master);
-        GF_ASSERT (slave);
-        GF_ASSERT (is_run);
-
-        *is_run = _gf_false;
-        ret = gsync_status (master, slave, conf_path,
-                            &ret_status, &is_template_in_use);
-        if (ret == 0 && ret_status == 0)
-                *is_run = _gf_true;
-        else if (ret == -1) {
-                gf_log ("", GF_LOG_WARNING, GEOREP" validation "
-                        " failed");
-                goto out;
+    int ret = -1;
+    xlator_t *this = NULL;
+    gf_boolean_t voluuid_updated = _gf_false;
+
+    this = THIS;
+    GF_VALIDATE_OR_GOTO("glusterd", this, out);
+    GF_VALIDATE_OR_GOTO(this->name, volinfo, out);
+
+    ret = dict_foreach(volinfo->gsync_slaves, update_slave_voluuid,
+                       &voluuid_updated);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_REMOTE_VOL_UUID_FAIL,
+               "Error in updating"
+               "volinfo");
+        goto out;
+    }
+
+    if (_gf_true == voluuid_updated) {
+        ret = glusterd_store_volinfo(volinfo,
+                                     GLUSTERD_VOLINFO_VER_AC_INCREMENT);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_VOLINFO_STORE_FAIL,
+                   "Error in storing"
+                   "volinfo");
+            goto out;
         }
-        ret = 0;
- out:
-        gf_log ("", GF_LOG_DEBUG, "Returning %d", ret);
-        return ret;
+    }
 
+    ret = 0;
+out:
+    gf_msg_debug((this ? this->name : "glusterd"), 0, "Returning %d", ret);
+    return ret;
+}
+
+int
+glusterd_check_gsync_running_local(char *master, char *slave, char *conf_path,
+                                   gf_boolean_t *is_run)
+{
+    int ret = -1;
+    int ret_status = 0;
+    gf_boolean_t is_template_in_use = _gf_false;
+    xlator_t *this = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+
+    GF_ASSERT(master);
+    GF_ASSERT(slave);
+    GF_ASSERT(is_run);
+
+    *is_run = _gf_false;
+    ret = gsync_status(master, slave, conf_path, &ret_status,
+                       &is_template_in_use);
+    if (ret == 0 && ret_status == 0)
+        *is_run = _gf_true;
+    else if (ret == -1) {
+        gf_msg(this->name, GF_LOG_WARNING, 0, GD_MSG_VALIDATE_FAILED,
+               GEOREP " validation failed");
+        goto out;
+    }
+    ret = 0;
+out:
+    gf_msg_debug(this->name, 0, "Returning %d", ret);
+    return ret;
 }
 
 static int
-glusterd_store_slave_in_info (glusterd_volinfo_t *volinfo, char *slave,
-                              char *host_uuid, char **op_errstr,
-                              gf_boolean_t is_force)
+glusterd_store_slave_in_info(glusterd_volinfo_t *volinfo, char *slave,
+                             char *host_uuid, char *slave_voluuid,
+                             char **op_errstr, gf_boolean_t is_force)
 {
-        int    ret = 0;
-        int    maxslv = 0;
-        char **linearr = NULL;
-        char  *value = NULL;
-        char  *slavekey = NULL;
-        char  *slaveentry = NULL;
-        char   key[512] = {0, };
-        char  *t = NULL;
-
-        GF_ASSERT (volinfo);
-        GF_ASSERT (slave);
-        GF_ASSERT (host_uuid);
-
-        ret = glusterd_get_slave (volinfo, slave, &slavekey);
-        switch (ret) {
+    int ret = 0;
+    int maxslv = 0;
+    char **linearr = NULL;
+    char *value = NULL;
+    char *slavekey = NULL;
+    char *slaveentry = NULL;
+    char key[32] = {
+        0,
+    };
+    int keylen;
+    char *t = NULL;
+    xlator_t *this = NULL;
+    struct slave_vol_config slave1 = {
+        {0},
+    };
+
+    this = THIS;
+    GF_ASSERT(this);
+
+    GF_ASSERT(volinfo);
+    GF_ASSERT(slave);
+    GF_ASSERT(host_uuid);
+    GF_VALIDATE_OR_GOTO(this->name, slave_voluuid, out);
+
+    ret = glusterd_get_slave(volinfo, slave, &slavekey);
+    switch (ret) {
         case -2:
-                ret = -1;
-                goto out;
+            ret = -1;
+            goto out;
         case -1:
-                                break;
+            break;
         default:
-                if (!is_force)
-                        GF_ASSERT (ret > 0);
-                ret = dict_get_str (volinfo->gsync_slaves, slavekey, &slaveentry);
-                GF_ASSERT (ret == 0);
-
-                /* same-name + same-uuid slave entries should have been filtered
-                 * out in glusterd_op_verify_gsync_start_options(), so we can
-                 * assert an uuid mismatch
-                 */
-                t = strtail (slaveentry, host_uuid);
-                if (!is_force)
-                        GF_ASSERT (!t || *t != ':');
-
-                if (is_force) {
-                        gf_log ("", GF_LOG_DEBUG, GEOREP" has already been "
-                                "invoked for the %s (master) and %s (slave)."
-                                " Allowing without saving info again due to"
-                                " force command.", volinfo->volname, slave);
-                        ret = 0;
-                        goto out;
-                }
-
-                gf_log ("", GF_LOG_ERROR, GEOREP" has already been invoked for "
-                                          "the %s (master) and %s (slave) "
-                                          "from a different machine",
-                                           volinfo->volname, slave);
-                *op_errstr = gf_strdup (GEOREP" already running in "
-                                        "another machine");
-                ret = -1;
-                goto out;
-        }
-
-        ret = glusterd_urltransform_single (slave, "normalize", &linearr);
-        if (ret == -1)
+            if (!is_force)
+                GF_ASSERT(ret > 0);
+            ret = dict_get_str(volinfo->gsync_slaves, slavekey, &slaveentry);
+            GF_ASSERT(ret == 0);
+
+            /* same-name + same-uuid slave entries should have been filtered
+             * out in glusterd_op_verify_gsync_start_options(), so we can
+             * assert an uuid mismatch
+             */
+            t = strtail(slaveentry, host_uuid);
+            if (!is_force)
+                GF_ASSERT(!t || *t != ':');
+
+            if (is_force) {
+                gf_msg_debug(this->name, 0,
+                             GEOREP
+                             " has already "
+                             "been invoked for the %s (master) and "
+                             "%s (slave). Allowing without saving "
+                             "info again due to force command.",
+                             volinfo->volname, slave);
+                ret = 0;
                 goto out;
+            }
 
-        ret = gf_asprintf (&value,  "%s:%s", host_uuid, linearr[0]);
-        glusterd_urltransform_free (linearr, 1);
-        if (ret  == -1)
-                goto out;
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_INVOKE_ERROR,
+                   GEOREP
+                   " has already been invoked for "
+                   "the %s (master) and %s (slave) from a different "
+                   "machine",
+                   volinfo->volname, slave);
+            *op_errstr = gf_strdup(GEOREP
+                                   " already running in "
+                                   "another machine");
+            ret = -1;
+            goto out;
+    }
+
+    ret = glusterd_urltransform_single(slave, "normalize", &linearr);
+    if (ret == -1)
+        goto out;
+
+    ret = gf_asprintf(&value, "%s:%s:%s", host_uuid, linearr[0], slave_voluuid);
+
+    glusterd_urltransform_free(linearr, 1);
+    if (ret == -1)
+        goto out;
+
+    /* Given the slave volume uuid, check and get any existing slave */
+    memcpy(slave1.slave_voluuid, slave_voluuid, UUID_CANONICAL_FORM_LEN);
+    ret = dict_foreach(volinfo->gsync_slaves, _get_slave_idx_slave_voluuid,
+                       &slave1);
+
+    if (ret == 0) { /* New slave */
+        dict_foreach(volinfo->gsync_slaves, _get_max_gsync_slave_num, &maxslv);
+        keylen = snprintf(key, sizeof(key), "slave%d", maxslv + 1);
+
+        ret = dict_set_dynstrn(volinfo->gsync_slaves, key, keylen, value);
+        if (ret) {
+            GF_FREE(value);
+            goto out;
+        }
+    } else if (ret == -1) { /* Existing slave */
+        keylen = snprintf(key, sizeof(key), "slave%d", slave1.old_slvidx);
 
-        dict_foreach (volinfo->gsync_slaves, _get_max_gsync_slave_num, &maxslv);
-        snprintf (key, 512, "slave%d", maxslv + 1);
-        ret = dict_set_dynstr (volinfo->gsync_slaves, key, value);
-        if (ret)
-                goto out;
+        gf_msg_debug(this->name, 0,
+                     "Replacing key:%s with new value"
+                     ":%s",
+                     key, value);
 
-        ret = glusterd_store_volinfo (volinfo,
-                                      GLUSTERD_VOLINFO_VER_AC_INCREMENT);
+        /* Add new slave's value, with the same slave index */
+        ret = dict_set_dynstrn(volinfo->gsync_slaves, key, keylen, value);
         if (ret) {
-                *op_errstr = gf_strdup ("Failed to store the Volume "
-                                        "information");
-                goto out;
-        }
-        ret = 0;
- out:
-        return ret;
+            GF_FREE(value);
+            goto out;
+        }
+    } else {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_REMOTE_VOL_UUID_FAIL,
+               "_get_slave_idx_slave_voluuid failed!");
+        GF_FREE(value);
+        ret = -1;
+        goto out;
+    }
+
+    ret = glusterd_store_volinfo(volinfo, GLUSTERD_VOLINFO_VER_AC_INCREMENT);
+    if (ret) {
+        *op_errstr = gf_strdup(
+            "Failed to store the Volume "
+            "information");
+        goto out;
+    }
+    ret = 0;
+out:
+    return ret;
 }
 
 static int
-glusterd_op_verify_gsync_start_options (glusterd_volinfo_t *volinfo,
-                                        char *slave, char *conf_path,
-                                        char *statefile, char **op_errstr,
-                                        gf_boolean_t is_force)
+glusterd_op_verify_gsync_start_options(glusterd_volinfo_t *volinfo, char *slave,
+                                       char *conf_path, char *statefile,
+                                       char **op_errstr, gf_boolean_t is_force)
 {
-        int                     ret = -1;
-        int                     ret_status = 0;
-        gf_boolean_t            is_template_in_use = _gf_false;
-        char                    msg[2048] = {0};
-        uuid_t                  uuid = {0};
-        glusterd_conf_t        *priv = NULL;
-        xlator_t               *this = NULL;
-        struct stat             stbuf = {0,};
-
-        this = THIS;
-
-        GF_ASSERT (volinfo);
-        GF_ASSERT (slave);
-        GF_ASSERT (op_errstr);
-        GF_ASSERT (conf_path);
-        GF_ASSERT (this && this->private);
-
-        priv  = this->private;
-
-        if (GLUSTERD_STATUS_STARTED != volinfo->status) {
-                snprintf (msg, sizeof (msg), "Volume %s needs to be started "
-                          "before "GEOREP" start", volinfo->volname);
-                goto out;
-        }
-
-        ret = lstat (statefile, &stbuf);
-        if (ret) {
-                snprintf (msg, sizeof (msg), "Session between %s and %s has"
-                          " not been created. Please create session and retry.",
-                          volinfo->volname, slave);
-                gf_log ("", GF_LOG_ERROR, "%s", msg);
-                *op_errstr = gf_strdup (msg);
-                goto out;
-        }
-
-        /* Check if the gsync slave info is stored. If not
-         * session has not been created */
-        ret = glusterd_gsync_get_uuid (slave, volinfo, uuid);
-        if (ret) {
-                snprintf (msg, sizeof (msg), "Session between %s and %s has"
-                          " not been created. Please create session and retry.",
-                          volinfo->volname, slave);
-                gf_log ("", GF_LOG_ERROR, "%s", msg);
-                goto out;
-        }
-
-        /*Check if the gsync is already started in cmd. inited host
-         * If so initiate add it into the glusterd's priv*/
-        ret = gsync_status (volinfo->volname, slave, conf_path,
-                            &ret_status, &is_template_in_use);
-        if (ret == 0) {
-                if ((ret_status == 0) && !is_force) {
-                        snprintf (msg, sizeof (msg), GEOREP " session between"
-                                  " %s & %s already started", volinfo->volname,
-                                  slave);
-                        ret = -1;
-                        goto out;
-                }
-        } else if (ret == -1) {
-                snprintf (msg, sizeof (msg), GEOREP" start option "
-                          "validation failed ");
-                goto out;
-        }
-
-        if (is_template_in_use == _gf_true) {
-                snprintf (msg, sizeof (msg), GEOREP" start "
-                          "failed : pid-file entry missing "
-                          "in config file.");
-                ret = -1;
-                goto out;
-        }
-
-        ret = glusterd_verify_gsyncd_spawn (volinfo->volname, slave);
-        if (ret && !is_force) {
-                snprintf (msg, sizeof (msg), "Unable to spawn gsyncd");
-                gf_log ("", GF_LOG_ERROR, "%s", msg);
-        }
+    int ret = -1;
+    int ret_status = 0;
+    gf_boolean_t is_template_in_use = _gf_false;
+    char msg[2048] = {0};
+    uuid_t uuid = {0};
+    xlator_t *this = NULL;
+    struct stat stbuf = {
+        0,
+    };
+    char statefiledir[PATH_MAX] = {
+        0,
+    };
+    char *statedir = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+
+    GF_ASSERT(volinfo);
+    GF_ASSERT(slave);
+    GF_ASSERT(op_errstr);
+    GF_ASSERT(conf_path);
+    GF_ASSERT(this && this->private);
+
+    if (GLUSTERD_STATUS_STARTED != volinfo->status) {
+        snprintf(msg, sizeof(msg),
+                 "Volume %s needs to be started "
+                 "before " GEOREP " start",
+                 volinfo->volname);
+        goto out;
+    }
+
+    /* check session directory as statefile may not present
+     * during upgrade */
+    if (snprintf(statefiledir, sizeof(statefiledir), "%s", statefile) >=
+        sizeof(statefiledir)) {
+        snprintf(msg, sizeof(msg), "statefiledir truncated");
+        gf_msg(this->name, GF_LOG_ERROR, errno, GD_MSG_FILE_OP_FAILED, "%s",
+               msg);
+        *op_errstr = gf_strdup(msg);
+        goto out;
+    }
+    statedir = dirname(statefiledir);
+
+    ret = sys_lstat(statedir, &stbuf);
+    if (ret) {
+        snprintf(msg, sizeof(msg),
+                 "Session between %s and %s has"
+                 " not been created. Please create session and retry.",
+                 volinfo->volname, slave);
+        gf_msg(this->name, GF_LOG_ERROR, errno, GD_MSG_FILE_OP_FAILED,
+               "%s statefile: %s", msg, statefile);
+        *op_errstr = gf_strdup(msg);
+        goto out;
+    }
+
+    /* Check if the gsync slave info is stored. If not
+     * session has not been created */
+    ret = glusterd_gsync_get_uuid(slave, volinfo, uuid);
+    if (ret) {
+        snprintf(msg, sizeof(msg),
+                 "Session between %s and %s has"
+                 " not been created. Please create session and retry.",
+                 volinfo->volname, slave);
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_SESSION_CREATE_ERROR, "%s",
+               msg);
+        goto out;
+    }
+
+    /*Check if the gsync is already started in cmd. inited host
+     * If so initiate add it into the glusterd's priv*/
+    ret = gsync_status(volinfo->volname, slave, conf_path, &ret_status,
+                       &is_template_in_use);
+    if (ret == 0) {
+        if ((ret_status == 0) && !is_force) {
+            snprintf(msg, sizeof(msg),
+                     GEOREP
+                     " session between"
+                     " %s & %s already started",
+                     volinfo->volname, slave);
+            ret = -1;
+            goto out;
+        }
+    } else if (ret == -1) {
+        snprintf(msg, sizeof(msg),
+                 GEOREP
+                 " start option "
+                 "validation failed ");
+        goto out;
+    }
+
+    if (is_template_in_use == _gf_true) {
+        snprintf(msg, sizeof(msg),
+                 GEOREP
+                 " start "
+                 "failed : pid-file entry missing "
+                 "in config file.");
+        ret = -1;
+        goto out;
+    }
+
+    ret = glusterd_verify_gsyncd_spawn(volinfo->volname, slave);
+    if (ret && !is_force) {
+        snprintf(msg, sizeof(msg), "Unable to spawn gsyncd");
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_GSYNCD_SPAWN_FAILED, "%s",
+               msg);
+    }
 out:
-        if (ret && (msg[0] != '\0')) {
-                *op_errstr = gf_strdup (msg);
-        }
-        gf_log ("", GF_LOG_DEBUG, "Returning %d", ret);
-        return ret;
+    if (ret && (msg[0] != '\0')) {
+        *op_errstr = gf_strdup(msg);
+    }
+    gf_msg_debug(this->name, 0, "Returning %d", ret);
+    return ret;
 }
 
 void
-glusterd_check_geo_rep_configured (glusterd_volinfo_t *volinfo,
-                                   gf_boolean_t *flag)
+glusterd_check_geo_rep_configured(glusterd_volinfo_t *volinfo,
+                                  gf_boolean_t *flag)
 {
+    GF_ASSERT(volinfo);
+    GF_ASSERT(flag);
 
-        GF_ASSERT (volinfo);
-        GF_ASSERT (flag);
+    if (volinfo->gsync_slaves->count)
+        *flag = _gf_true;
+    else
+        *flag = _gf_false;
 
-        if (volinfo->gsync_slaves->count)
-                *flag = _gf_true;
-        else
-                *flag = _gf_false;
-
-        return;
+    return;
 }
 
 /*
  * is_geo_rep_active:
  *      This function reads the state_file and sets is_active to 1 if the
- *      monitor status is neither "Stopped" or "Not Started"
+ *      monitor status is neither "Stopped" or "Created"
  *
  * RETURN VALUE:
  *       0: On successful read of state_file.
@@ -1356,64 +2021,70 @@ glusterd_check_geo_rep_configured (glusterd_volinfo_t *volinfo,
  */
 
 static int
-is_geo_rep_active (glusterd_volinfo_t *volinfo, char *slave,
-                   char *conf_path, int *is_active)
+is_geo_rep_active(glusterd_volinfo_t *volinfo, char *slave, char *conf_path,
+                  int *is_active)
 {
-        dict_t                 *confd                      = NULL;
-        char                   *statefile                  = NULL;
-        char                   *master                     = NULL;
-        char                    monitor_status[PATH_MAX]   = "";
-        int                     ret                        = -1;
-        xlator_t               *this                       = NULL;
-
-        this = THIS;
-        GF_ASSERT (this);
-
-        master = volinfo->volname;
-
-        confd = dict_new ();
-        if (!confd) {
-                gf_log ("", GF_LOG_ERROR, "Not able to create dict.");
-                goto out;
-        }
-
-        ret = glusterd_gsync_get_config (master, slave, conf_path,
-                                         confd);
-        if (ret) {
-                gf_log ("", GF_LOG_ERROR, "Unable to get configuration data "
-                        "for %s(master), %s(slave)", master, slave);
-                ret = -1;
-                goto out;
-        }
-
-        ret = dict_get_param (confd, "state_file", &statefile);
-        if (ret) {
-                gf_log ("", GF_LOG_ERROR, "Unable to get state_file's name "
-                        "for %s(master), %s(slave). Please check gsync "
-                        "config file.", master, slave);
-                ret = -1;
-                goto out;
-        }
-
-        ret = glusterd_gsync_read_frm_status (statefile, monitor_status,
-                                              sizeof (monitor_status));
-        if (ret <= 0) {
-                gf_log ("", GF_LOG_ERROR, "Unable to read the status "
-                        "file for %s(master), %s(slave)", master, slave);
-                strncpy (monitor_status, "defunct", sizeof (monitor_status));
-        }
-
-        if ((!strcmp(monitor_status, "Stopped")) ||
-            (!strcmp(monitor_status, "Not Started"))) {
-                *is_active = 0;
-        } else {
-                *is_active = 1;
-        }
-        ret = 0;
+    dict_t *confd = NULL;
+    char *statefile = NULL;
+    char *master = NULL;
+    char monitor_status[PATH_MAX] = "";
+    int ret = -1;
+    xlator_t *this = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+
+    master = volinfo->volname;
+
+    confd = dict_new();
+    if (!confd) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_CREATE_FAIL,
+               "Not able to create dict.");
+        goto out;
+    }
+
+    ret = glusterd_gsync_get_config(master, slave, conf_path, confd);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_GET_CONFIG_INFO_FAILED,
+               "Unable to get configuration data "
+               "for %s(master), %s(slave)",
+               master, slave);
+        ret = -1;
+        goto out;
+    }
+
+    ret = dict_get_param(confd, "state_file", &statefile);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+               "Unable to get state_file's name "
+               "for %s(master), %s(slave). Please check gsync "
+               "config file.",
+               master, slave);
+        ret = -1;
+        goto out;
+    }
+
+    ret = glusterd_gsync_read_frm_status(statefile, monitor_status,
+                                         sizeof(monitor_status));
+    if (ret <= 0) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_STAT_FILE_READ_FAILED,
+               "Unable to read the status file for %s(master), "
+               "%s(slave)",
+               master, slave);
+        snprintf(monitor_status, sizeof(monitor_status), "defunct");
+    }
+
+    if ((!strcmp(monitor_status, "Stopped")) ||
+        (!strcmp(monitor_status, "Created"))) {
+        *is_active = 0;
+    } else {
+        *is_active = 1;
+    }
+    ret = 0;
 out:
-        if (confd)
-                dict_destroy (confd);
-        return ret;
+    if (confd)
+        dict_unref(confd);
+    return ret;
 }
 
 /*
@@ -1429,75 +2100,86 @@ out:
  */
 
 int
-_get_slave_status (dict_t *dict, char *key, data_t *value, void *data)
+_get_slave_status(dict_t *dict, char *key, data_t *value, void *data)
 {
-        gsync_status_param_t *param                        = NULL;
-        char                 *slave                        = NULL;
-        char                 *slave_url                    = NULL;
-        char                 *slave_vol                    = NULL;
-        char                 *slave_host                   = NULL;
-        char                 *errmsg                       = NULL;
-        char                           conf_path[PATH_MAX] = "";
-        int                   ret                          = -1;
-        glusterd_conf_t      *priv                         = NULL;
-        xlator_t             *this                         = NULL;
-
-        param = (gsync_status_param_t *)data;
-
-        GF_ASSERT (param);
-        GF_ASSERT (param->volinfo);
-
-        if (param->is_active) {
-                ret = 0;
-                goto out;
-        }
+    gsync_status_param_t *param = NULL;
+    char *slave = NULL;
+    char *slave_url = NULL;
+    char *slave_vol = NULL;
+    char *slave_host = NULL;
+    char *errmsg = NULL;
+    char conf_path[PATH_MAX] = "";
+    int ret = -1;
+    glusterd_conf_t *priv = NULL;
+    xlator_t *this = NULL;
+
+    param = (gsync_status_param_t *)data;
+
+    GF_ASSERT(param);
+    GF_ASSERT(param->volinfo);
+    if (param->is_active) {
+        ret = 0;
+        goto out;
+    }
 
-        this = THIS;
-        GF_ASSERT (this);
+    this = THIS;
+    GF_ASSERT(this);
 
-        if (this)
-                priv = this->private;
-        if (priv == NULL) {
-                gf_log ("", GF_LOG_ERROR, "priv of glusterd not present");
-                goto out;
-        }
+    priv = this->private;
+    if (priv == NULL) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_GLUSTERD_PRIV_NOT_FOUND,
+               "priv of glusterd not present");
+        goto out;
+    }
 
-        slave = strchr(value->data, ':');
-        if (!slave) {
-                ret = 0;
-                goto out;
-        }
-        slave++;
+    slave = strchr(value->data, ':');
+    if (!slave) {
+        ret = 0;
+        goto out;
+    }
+    slave++;
+
+    ret = glusterd_get_slave_info(slave, &slave_url, &slave_host, &slave_vol,
+                                  &errmsg);
+    if (ret) {
+        if (errmsg)
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_SLAVEINFO_FETCH_ERROR,
+                   "Unable to fetch"
+                   " slave details. Error: %s",
+                   errmsg);
+        else
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_SLAVEINFO_FETCH_ERROR,
+                   "Unable to fetch slave details.");
+        ret = -1;
+        goto out;
+    }
+
+    ret = snprintf(conf_path, sizeof(conf_path) - 1,
+                   "%s/" GEOREP "/%s_%s_%s/gsyncd.conf", priv->workdir,
+                   param->volinfo->volname, slave_host, slave_vol);
+    if (ret < 0) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_CONF_PATH_ASSIGN_FAILED,
+               "Unable to assign conf_path.");
+        ret = -1;
+        goto out;
+    }
+    conf_path[ret] = '\0';
+
+    ret = is_geo_rep_active(param->volinfo, slave, conf_path,
+                            &param->is_active);
+out:
+    if (errmsg)
+        GF_FREE(errmsg);
 
-        ret = glusterd_get_slave_info (slave, &slave_url,
-                                       &slave_host, &slave_vol, &errmsg);
-        if (ret) {
-                if (errmsg)
-                        gf_log ("", GF_LOG_ERROR, "Unable to fetch "
-                                "slave details. Error: %s", errmsg);
-                else
-                        gf_log ("", GF_LOG_ERROR,
-                                "Unable to fetch slave details.");
-                ret = -1;
-                goto out;
-        }
+    if (slave_vol)
+        GF_FREE(slave_vol);
 
-        ret = snprintf (conf_path, sizeof(conf_path) - 1,
-                        "%s/"GEOREP"/%s_%s_%s/gsyncd.conf",
-                        priv->workdir, param->volinfo->volname,
-                        slave_host, slave_vol);
-        if (ret < 0) {
-                gf_log ("", GF_LOG_ERROR, "Unable to assign conf_path.");
-                ret = -1;
-                goto out;
-        }
-        conf_path[ret] = '\0';
+    if (slave_url)
+        GF_FREE(slave_url);
+    if (slave_host)
+        GF_FREE(slave_host);
 
-        ret = is_geo_rep_active (param->volinfo,slave, conf_path,
-                                 &param->is_active);
-out:
-        GF_FREE(errmsg);
-        return ret;
+    return ret;
 }
 
 /* glusterd_check_geo_rep_running:
@@ -1511,912 +2193,1391 @@ out:
  */
 
 int
-glusterd_check_geo_rep_running (gsync_status_param_t *param, char **op_errstr)
+glusterd_check_geo_rep_running(gsync_status_param_t *param, char **op_errstr)
 {
-        char                    msg[2048]   = {0,};
-        gf_boolean_t            enabled     = _gf_false;
-        int                     ret         = 0;
-        xlator_t               *this        = NULL;
-
-        this = THIS;
-        GF_ASSERT (this);
-        GF_ASSERT (param);
-        GF_ASSERT (param->volinfo);
-        GF_ASSERT (op_errstr);
-
-        glusterd_check_geo_rep_configured (param->volinfo, &enabled);
-
-        if (enabled) {
-                ret = dict_foreach (param->volinfo->gsync_slaves,
-                                    _get_slave_status, param);
-                if (ret) {
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "_get_slave_satus failed");
-                        snprintf (msg, sizeof(msg), GEOREP" Unable to"
-                                  " get the status of active "GEOREP""
-                                  " session for the volume '%s'.\n"
-                                  " Please check the log file for"
-                                  " more info.", param->volinfo->volname);
-                        *op_errstr = gf_strdup (msg);
-                        ret = -1;
-                        goto out;
-                }
+    char msg[2048] = {
+        0,
+    };
+    gf_boolean_t enabled = _gf_false;
+    int ret = 0;
+    xlator_t *this = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+    GF_ASSERT(param);
+    GF_ASSERT(param->volinfo);
+    GF_ASSERT(op_errstr);
+
+    glusterd_check_geo_rep_configured(param->volinfo, &enabled);
+
+    if (enabled) {
+        ret = dict_foreach(param->volinfo->gsync_slaves, _get_slave_status,
+                           param);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_SLAVEINFO_FETCH_ERROR,
+                   "_get_slave_satus failed");
+            snprintf(msg, sizeof(msg),
+                     GEOREP
+                     " Unable to"
+                     " get the status of active " GEOREP
+                     ""
+                     " session for the volume '%s'.\n"
+                     " Please check the log file for"
+                     " more info.",
+                     param->volinfo->volname);
+            *op_errstr = gf_strdup(msg);
+            ret = -1;
+            goto out;
+        }
 
-                if (param->is_active) {
-                        snprintf (msg, sizeof(msg), GEOREP" sessions"
-                                  " are active for the volume %s.\nStop"
-                                  " "GEOREP " sessions involved in this"
-                                  " volume. Use 'volume "GEOREP
-                                  " status' command for more info.",
-                                  param->volinfo->volname);
-                        *op_errstr = gf_strdup (msg);
-                        goto out;
-                 }
-         }
- out:
-        return ret;
+        if (param->is_active) {
+            snprintf(msg, sizeof(msg),
+                     GEOREP
+                     " sessions"
+                     " are active for the volume %s.\nStop"
+                     " " GEOREP
+                     " sessions involved in this"
+                     " volume. Use 'volume " GEOREP
+                     " status' command for more info.",
+                     param->volinfo->volname);
+            *op_errstr = gf_strdup(msg);
+            goto out;
+        }
+    }
+out:
+    return ret;
 }
 
 static int
-glusterd_op_verify_gsync_running (glusterd_volinfo_t *volinfo,
-                                  char *slave, char *conf_path,
-                                  char **op_errstr)
+glusterd_op_verify_gsync_running(glusterd_volinfo_t *volinfo, char *slave,
+                                 char *conf_path, char **op_errstr)
 {
-        int        pfd               = -1;
-        int        ret               = -1;
-        char       msg[2048]         = {0};
-        char       pidfile[PATH_MAX] = {0,};
-        gf_boolean_t is_template_in_use = _gf_false;
-
-        GF_ASSERT (THIS && THIS->private);
-        GF_ASSERT (volinfo);
-        GF_ASSERT (slave);
-        GF_ASSERT (conf_path);
-        GF_ASSERT (op_errstr);
-
-        if (GLUSTERD_STATUS_STARTED != volinfo->status) {
-                snprintf (msg, sizeof (msg), "Volume %s needs to be started "
-                          "before "GEOREP" start", volinfo->volname);
-
-                goto out;
-        }
-
-        pfd = gsyncd_getpidfile (volinfo->volname, slave, pidfile,
-                                 conf_path, &is_template_in_use);
-        if (pfd == -2) {
-                gf_log ("", GF_LOG_ERROR, GEOREP" stop validation "
-                        "failed for %s & %s", volinfo->volname, slave);
-                ret = -1;
-                goto out;
-        }
-        if (gsync_status_byfd (pfd) == -1) {
-                snprintf (msg, sizeof (msg), GEOREP" session b/w %s & %s is not"
-                          " running on this node.", volinfo->volname, slave);
-                gf_log ("", GF_LOG_ERROR, "%s", msg);
-                ret = -1;
-                /* monitor gsyncd already dead */
-                goto out;
-        }
-
-        if (is_template_in_use) {
-                snprintf (msg, sizeof (msg), "pid-file entry missing in "
-                          "the config file(%s).", conf_path);
-                gf_log ("", GF_LOG_ERROR, "%s", msg);
-                ret = -1;
-                goto out;
-        }
-
-        if (pfd < 0)
-                goto out;
+    int pfd = -1;
+    int ret = -1;
+    char msg[2048] = {0};
+    char pidfile[PATH_MAX] = {
+        0,
+    };
+    gf_boolean_t is_template_in_use = _gf_false;
+    xlator_t *this = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+
+    GF_ASSERT(THIS && THIS->private);
+    GF_ASSERT(volinfo);
+    GF_ASSERT(slave);
+    GF_ASSERT(conf_path);
+    GF_ASSERT(op_errstr);
+
+    if (GLUSTERD_STATUS_STARTED != volinfo->status) {
+        snprintf(msg, sizeof(msg),
+                 "Volume %s needs to be started "
+                 "before " GEOREP " start",
+                 volinfo->volname);
+        gf_smsg(this->name, GF_LOG_ERROR, 0, GD_MSG_GEO_REP_START_FAILED,
+                "Volume is not in a started state, Volname=%s",
+                volinfo->volname, NULL);
+
+        goto out;
+    }
+
+    pfd = gsyncd_getpidfile(volinfo->volname, slave, pidfile, conf_path,
+                            &is_template_in_use);
+    if (pfd == -2) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_VALIDATE_FAILED,
+               GEOREP " stop validation failed for %s & %s", volinfo->volname,
+               slave);
+        ret = -1;
+        goto out;
+    }
+    if (gsync_status_byfd(pfd) == -1) {
+        snprintf(msg, sizeof(msg),
+                 GEOREP
+                 " session b/w %s & %s is "
+                 "not running on this node.",
+                 volinfo->volname, slave);
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_SESSION_INACTIVE, "%s", msg);
+        ret = -1;
+        /* monitor gsyncd already dead */
+        goto out;
+    }
+
+    if (is_template_in_use) {
+        snprintf(msg, sizeof(msg),
+                 "pid-file entry missing in "
+                 "the config file(%s).",
+                 conf_path);
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_PIDFILE_NOT_FOUND, "%s",
+               msg);
+        ret = -1;
+        goto out;
+    }
+
+    if (pfd < 0)
+        goto out;
+
+    ret = 0;
+out:
+    if (ret && (msg[0] != '\0')) {
+        *op_errstr = gf_strdup(msg);
+    }
+    gf_msg_debug(this->name, 0, "Returning %d", ret);
+    return ret;
+}
 
+static int
+glusterd_verify_gsync_status_opts(dict_t *dict, char **op_errstr)
+{
+    char *slave = NULL;
+    char *volname = NULL;
+    char errmsg[PATH_MAX] = {
+        0,
+    };
+    glusterd_volinfo_t *volinfo = NULL;
+    int ret = 0;
+    char *conf_path = NULL;
+    char *slave_url = NULL;
+    char *slave_host = NULL;
+    char *slave_vol = NULL;
+    glusterd_conf_t *priv = NULL;
+    xlator_t *this = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+
+    if (THIS)
+        priv = THIS->private;
+    if (priv == NULL) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_GLUSTERD_PRIV_NOT_FOUND,
+               "priv of glusterd not present");
+        *op_errstr = gf_strdup("glusterd defunct");
+        goto out;
+    }
+
+    ret = dict_get_str(dict, "master", &volname);
+    if (ret < 0) {
+        ret = 0;
+        goto out;
+    }
+
+    ret = glusterd_volinfo_find(volname, &volinfo);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_WARNING, 0, GD_MSG_VOL_NOT_FOUND,
+               "volume name does not exist");
+        snprintf(errmsg, sizeof(errmsg),
+                 "Volume name %s does not"
+                 " exist",
+                 volname);
+        *op_errstr = gf_strdup(errmsg);
+        goto out;
+    }
+
+    ret = dict_get_str(dict, "slave", &slave);
+    if (ret < 0) {
         ret = 0;
+        goto out;
+    }
+
+    ret = glusterd_get_slave_details_confpath(volinfo, dict, &slave_url,
+                                              &slave_host, &slave_vol,
+                                              &conf_path, op_errstr);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_SLAVEINFO_FETCH_ERROR,
+               "Unable to fetch slave  or confpath details.");
+        ret = -1;
+        goto out;
+    }
+
 out:
-        if (ret && (msg[0] != '\0')) {
-                *op_errstr = gf_strdup (msg);
-        }
-        gf_log ("", GF_LOG_DEBUG, "Returning %d", ret);
-        return ret;
+    gf_msg_debug(this->name, 0, "Returning %d", ret);
+    return ret;
 }
 
-static int
-glusterd_verify_gsync_status_opts (dict_t *dict, char **op_errstr)
+int
+glusterd_op_gsync_args_get(dict_t *dict, char **op_errstr, char **master,
+                           char **slave, char **host_uuid)
 {
-        char               *slave           = NULL;
-        char               *volname         = NULL;
-        char               errmsg[PATH_MAX] = {0, };
-        gf_boolean_t        exists          = _gf_false;
-        glusterd_volinfo_t *volinfo         = NULL;
-        int                 ret             = 0;
-        char               *conf_path       = NULL;
-        char               *slave_url       = NULL;
-        char               *slave_host      = NULL;
-        char               *slave_vol       = NULL;
-        glusterd_conf_t    *priv            = NULL;
-
-        if (THIS)
-                priv = THIS->private;
-        if (priv == NULL) {
-                gf_log ("", GF_LOG_ERROR, "priv of glusterd not present");
-                *op_errstr = gf_strdup ("glusterd defunct");
-                goto out;
-        }
+    int ret = -1;
+    xlator_t *this = NULL;
 
-        ret = dict_get_str (dict, "master", &volname);
-        if (ret < 0) {
-                ret = 0;
-                goto out;
-        }
+    this = THIS;
+    GF_ASSERT(this);
+    GF_ASSERT(dict);
+    GF_ASSERT(op_errstr);
 
-        exists = glusterd_check_volume_exists (volname);
-        ret = glusterd_volinfo_find (volname, &volinfo);
-        if ((ret) || (!exists)) {
-                gf_log ("", GF_LOG_WARNING, "volume name does not exist");
-                snprintf (errmsg, sizeof(errmsg), "Volume name %s does not"
-                          " exist", volname);
-                *op_errstr = gf_strdup (errmsg);
-                ret = -1;
-                goto out;
+    if (master) {
+        ret = dict_get_str(dict, "master", master);
+        if (ret < 0) {
+            gf_msg(this->name, GF_LOG_WARNING, 0, GD_MSG_DICT_GET_FAILED,
+                   "master not found");
+            *op_errstr = gf_strdup("master not found");
+            goto out;
         }
+    }
 
-        ret = dict_get_str (dict, "slave", &slave);
+    if (slave) {
+        ret = dict_get_str(dict, "slave", slave);
         if (ret < 0) {
-                ret = 0;
-                goto out;
+            gf_msg(this->name, GF_LOG_WARNING, 0, GD_MSG_DICT_GET_FAILED,
+                   "slave not found");
+            *op_errstr = gf_strdup("slave not found");
+            goto out;
         }
+    }
 
-        ret = glusterd_get_slave_details_confpath (volinfo, dict, &slave_url,
-                                                   &slave_host, &slave_vol,
-                                                   &conf_path, op_errstr);
-        if (ret) {
-                gf_log ("", GF_LOG_ERROR,
-                        "Unable to fetch slave  or confpath details.");
-                ret = -1;
-                goto out;
+    if (host_uuid) {
+        ret = dict_get_str(dict, "host-uuid", host_uuid);
+        if (ret < 0) {
+            gf_msg(this->name, GF_LOG_WARNING, 0, GD_MSG_DICT_GET_FAILED,
+                   "host_uuid not found");
+            *op_errstr = gf_strdup("host_uuid not found");
+            goto out;
         }
+    }
 
+    ret = 0;
 out:
-        gf_log ("", GF_LOG_DEBUG, "Returning %d", ret);
-        return ret;
+    gf_msg_debug(this->name, 0, "Returning %d", ret);
+    return ret;
 }
 
-
 int
-glusterd_op_gsync_args_get (dict_t *dict, char **op_errstr,
-                            char **master, char **slave, char **host_uuid)
+glusterd_op_stage_sys_exec(dict_t *dict, char **op_errstr)
 {
+    char errmsg[PATH_MAX] = "";
+    char *command = NULL;
+    char command_path[PATH_MAX] = "";
+    struct stat st = {
+        0,
+    };
+    int ret = -1;
+    glusterd_conf_t *conf = NULL;
+    xlator_t *this = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+    conf = this->private;
+    GF_ASSERT(conf);
+
+    if (conf->op_version < 2) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_UNSUPPORTED_VERSION,
+               "Op Version not supported.");
+        snprintf(errmsg, sizeof(errmsg),
+                 "One or more nodes do not"
+                 " support the required op version.");
+        *op_errstr = gf_strdup(errmsg);
+        ret = -1;
+        goto out;
+    }
+
+    ret = dict_get_str(dict, "command", &command);
+    if (ret) {
+        strcpy(errmsg, "internal error");
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+               "Unable to get command from dict");
+        goto out;
+    }
+
+    /* enforce local occurrence of the command */
+    if (strchr(command, '/')) {
+        strcpy(errmsg, "invalid command name");
+        ret = -1;
+        goto out;
+    }
+
+    sprintf(command_path, GSYNCD_PREFIX "/peer_%s", command);
+    /* check if it's executable */
+    ret = sys_access(command_path, X_OK);
+    if (!ret)
+        /* check if it's a regular file */
+        ret = sys_stat(command_path, &st);
+    if (!ret && !S_ISREG(st.st_mode))
+        ret = -1;
 
-        int             ret = -1;
-        GF_ASSERT (dict);
-        GF_ASSERT (op_errstr);
-
-        if (master) {
-                ret = dict_get_str (dict, "master", master);
-                if (ret < 0) {
-                        gf_log ("", GF_LOG_WARNING, "master not found");
-                        *op_errstr = gf_strdup ("master not found");
-                        goto out;
-                }
-        }
-
-        if (slave) {
-                ret = dict_get_str (dict, "slave", slave);
-                if (ret < 0) {
-                        gf_log ("", GF_LOG_WARNING, "slave not found");
-                        *op_errstr = gf_strdup ("slave not found");
-                        goto out;
-                }
-        }
-
-        if (host_uuid) {
-                ret = dict_get_str (dict, "host-uuid", host_uuid);
-                if (ret < 0) {
-                        gf_log ("", GF_LOG_WARNING, "host_uuid not found");
-                        *op_errstr = gf_strdup ("host_uuid not found");
-                        goto out;
-                }
-        }
-
-        ret = 0;
 out:
-        gf_log ("", GF_LOG_DEBUG, "Returning %d", ret);
-        return ret;
+    if (ret) {
+        if (errmsg[0] == '\0') {
+            if (command)
+                snprintf(errmsg, sizeof(errmsg),
+                         "gsync peer_%s command not found.", command);
+            else
+                snprintf(errmsg, sizeof(errmsg), "%s",
+                         "gsync peer command was not "
+                         "specified");
+        }
+        *op_errstr = gf_strdup(errmsg);
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_PEER_CMD_ERROR, "%s",
+               errmsg);
+    }
+
+    gf_msg_debug(this->name, 0, "Returning %d", ret);
+    return ret;
 }
 
 int
-glusterd_op_stage_sys_exec (dict_t *dict, char **op_errstr)
+glusterd_op_stage_copy_file(dict_t *dict, char **op_errstr)
 {
-        char             errmsg[PATH_MAX]          = "";
-        char            *command                   = NULL;
-        char             command_path[PATH_MAX]    = "";
-        struct stat      st                        = {0,};
-        int              ret                       = -1;
-        glusterd_conf_t *conf                      = NULL;
-        xlator_t        *this                      = NULL;
-
-        this = THIS;
-        GF_ASSERT (this);
-        conf = this->private;
-        GF_ASSERT (conf);
-
-        if (conf->op_version < 2) {
-                gf_log ("", GF_LOG_ERROR, "Op Version not supported.");
-                snprintf (errmsg, sizeof(errmsg), "One or more nodes do not"
-                          " support the required op version.");
-                *op_errstr = gf_strdup (errmsg);
-                ret = -1;
-                goto out;
+    char abs_filename[PATH_MAX] = "";
+    char errmsg[PATH_MAX] = "";
+    char *filename = NULL;
+    char *host_uuid = NULL;
+    char uuid_str[64] = {0};
+    int ret = -1;
+    glusterd_conf_t *priv = NULL;
+    struct stat stbuf = {
+        0,
+    };
+    xlator_t *this = NULL;
+    char workdir[PATH_MAX] = {
+        0,
+    };
+    char realpath_filename[PATH_MAX] = {
+        0,
+    };
+    char realpath_workdir[PATH_MAX] = {
+        0,
+    };
+    int32_t len = 0;
+
+    this = THIS;
+    GF_ASSERT(this);
+
+    if (THIS)
+        priv = THIS->private;
+    if (priv == NULL) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_GLUSTERD_PRIV_NOT_FOUND,
+               "priv of glusterd not present");
+        *op_errstr = gf_strdup("glusterd defunct");
+        goto out;
+    }
+
+    if (priv->op_version < 2) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_UNSUPPORTED_VERSION,
+               "Op Version not supported.");
+        snprintf(errmsg, sizeof(errmsg),
+                 "One or more nodes do not"
+                 " support the required op version.");
+        *op_errstr = gf_strdup(errmsg);
+        ret = -1;
+        goto out;
+    }
+
+    ret = dict_get_str(dict, "host-uuid", &host_uuid);
+    if (ret < 0) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+               "Unable to fetch host-uuid from dict.");
+        goto out;
+    }
+
+    uuid_utoa_r(MY_UUID, uuid_str);
+    if (!strcmp(uuid_str, host_uuid)) {
+        ret = dict_get_str(dict, "source", &filename);
+        if (ret < 0) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+                   "Unable to fetch filename from dict.");
+            *op_errstr = gf_strdup("command unsuccessful");
+            goto out;
+        }
+        len = snprintf(abs_filename, sizeof(abs_filename), "%s/%s",
+                       priv->workdir, filename);
+        if ((len < 0) || (len >= sizeof(abs_filename))) {
+            gf_smsg(this->name, GF_LOG_ERROR, 0, GD_MSG_COPY_FAIL, NULL);
+            ret = -1;
+            goto out;
+        }
+
+        if (!realpath(priv->workdir, realpath_workdir)) {
+            len = snprintf(errmsg, sizeof(errmsg),
+                           "Failed to "
+                           "get realpath of %s: %s",
+                           priv->workdir, strerror(errno));
+            if (len < 0) {
+                strcpy(errmsg, "<error>");
+            }
+            gf_smsg(this->name, GF_LOG_ERROR, 0, GD_MSG_REALPATH_GET_FAIL,
+                    "Realpath=%s, Reason=%s", priv->workdir, strerror(errno),
+                    NULL);
+            *op_errstr = gf_strdup(errmsg);
+            ret = -1;
+            goto out;
+        }
+
+        if (!realpath(abs_filename, realpath_filename)) {
+            snprintf(errmsg, sizeof(errmsg),
+                     "Failed to get "
+                     "realpath of %s: %s",
+                     filename, strerror(errno));
+            gf_smsg(this->name, GF_LOG_ERROR, 0, GD_MSG_REALPATH_GET_FAIL,
+                    "Filename=%s, Reason=%s", filename, strerror(errno), NULL);
+            *op_errstr = gf_strdup(errmsg);
+            ret = -1;
+            goto out;
+        }
+
+        /* Add Trailing slash to workdir, without slash strncmp
+           will succeed for /var/lib/glusterd_bad */
+        len = snprintf(workdir, sizeof(workdir), "%s/", realpath_workdir);
+        if ((len < 0) || (len >= sizeof(workdir))) {
+            gf_smsg(this->name, GF_LOG_ERROR, 0, GD_MSG_COPY_FAIL, NULL);
+            ret = -1;
+            goto out;
+        }
+
+        /* Protect against file copy outside $workdir */
+        if (strncmp(workdir, realpath_filename, strlen(workdir))) {
+            len = snprintf(errmsg, sizeof(errmsg),
+                           "Source file"
+                           " is outside of %s directory",
+                           priv->workdir);
+            if (len < 0) {
+                strcpy(errmsg, "<error>");
+            }
+            gf_smsg(this->name, GF_LOG_ERROR, 0, GD_MSG_SRC_FILE_ERROR, errmsg,
+                    NULL);
+            *op_errstr = gf_strdup(errmsg);
+            ret = -1;
+            goto out;
         }
 
-        ret = dict_get_str (dict, "command", &command);
+        ret = sys_lstat(abs_filename, &stbuf);
         if (ret) {
-                strcpy (errmsg, "internal error");
-                gf_log ("", GF_LOG_ERROR,
-                        "Unable to get command from dict");
-                goto out;
+            len = snprintf(errmsg, sizeof(errmsg),
+                           "Source file"
+                           " does not exist in %s",
+                           priv->workdir);
+            if (len < 0) {
+                strcpy(errmsg, "<error>");
+            }
+            gf_smsg(this->name, GF_LOG_ERROR, 0, GD_MSG_SRC_FILE_ERROR, errmsg,
+                    NULL);
+            *op_errstr = gf_strdup(errmsg);
+            goto out;
         }
 
-        /* enforce local occurrence of the command */
-        if (strchr (command, '/')) {
-                strcpy (errmsg, "invalid command name");
-                ret = -1;
-                goto out;
+        if (!S_ISREG(stbuf.st_mode)) {
+            snprintf(errmsg, sizeof(errmsg),
+                     "Source file"
+                     " is not a regular file.");
+            gf_smsg(this->name, GF_LOG_ERROR, 0, GD_MSG_SRC_FILE_ERROR, errmsg,
+                    NULL);
+            *op_errstr = gf_strdup(errmsg);
+            ret = -1;
+            goto out;
         }
+    }
 
-        sprintf (command_path, GSYNCD_PREFIX"/peer_%s", command);
-        /* check if it's executable */
-        ret = access (command_path, X_OK);
-        if (!ret)
-                /* check if it's a regular file */
-                ret = stat (command_path, &st);
-        if (!ret && !S_ISREG (st.st_mode))
-                ret = -1;
-
+    ret = 0;
 out:
-        if (ret) {
-                if (errmsg[0] == '\0') {
-                        if (command)
-                                snprintf (errmsg, sizeof (errmsg),
-                                          "gsync peer_%s command not found.",
-                                          command);
-                        else
-                                snprintf (errmsg, sizeof (errmsg), "%s",
-                                          "gsync peer command was not specified");
-                }
-                *op_errstr = gf_strdup (errmsg);
-                gf_log ("", GF_LOG_ERROR, "%s", errmsg);
-        }
-
-        gf_log ("", GF_LOG_DEBUG, "Returning %d", ret);
-        return ret;
+    gf_msg_debug(this->name, 0, "Returning %d", ret);
+    return ret;
 }
 
 int
-glusterd_op_stage_copy_file (dict_t *dict, char **op_errstr)
+glusterd_get_statefile_name(glusterd_volinfo_t *volinfo, char *slave,
+                            char *conf_path, char **statefile,
+                            gf_boolean_t *is_template_in_use)
 {
-        char             abs_filename[PATH_MAX] = "";
-        char             errmsg[PATH_MAX]       = "";
-        char            *filename               = NULL;
-        char            *host_uuid              = NULL;
-        char             uuid_str [64]          = {0};
-        int              ret                    = -1;
-        glusterd_conf_t *priv                   = NULL;
-        struct stat      stbuf                  = {0,};
-
-        if (THIS)
-                priv = THIS->private;
-        if (priv == NULL) {
-                gf_log ("", GF_LOG_ERROR, "priv of glusterd not present");
-                *op_errstr = gf_strdup ("glusterd defunct");
-                goto out;
-        }
-
-        if (priv->op_version < 2) {
-                gf_log ("", GF_LOG_ERROR, "Op Version not supported.");
-                snprintf (errmsg, sizeof(errmsg), "One or more nodes do not"
-                          " support the required op version.");
-                *op_errstr = gf_strdup (errmsg);
-                ret = -1;
-                goto out;
-        }
+    char *master = NULL;
+    char *buf = NULL;
+    char *working_conf_path = NULL;
+    char temp_conf_path[PATH_MAX] = "";
+    dict_t *confd = NULL;
+    glusterd_conf_t *priv = NULL;
+    int ret = -1;
+    struct stat stbuf = {
+        0,
+    };
+    xlator_t *this = NULL;
+    int32_t len = 0;
+
+    this = THIS;
+    GF_ASSERT(this);
+
+    GF_ASSERT(this->private);
+    GF_ASSERT(volinfo);
+    GF_ASSERT(conf_path);
+    GF_ASSERT(is_template_in_use);
+
+    master = volinfo->volname;
+
+    confd = dict_new();
+    if (!confd) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_CREATE_FAIL,
+               "Unable to create new dict");
+        goto out;
+    }
+
+    priv = THIS->private;
+
+    len = snprintf(temp_conf_path, sizeof(temp_conf_path),
+                   "%s/" GSYNC_CONF_TEMPLATE, priv->workdir);
+    if ((len < 0) || (len >= sizeof(temp_conf_path))) {
+        goto out;
+    }
+
+    ret = sys_lstat(conf_path, &stbuf);
+    if (!ret) {
+        gf_msg(this->name, GF_LOG_INFO, 0, GD_MSG_CONFIG_INFO,
+               "Using passed config template(%s).", conf_path);
+        working_conf_path = conf_path;
+    } else {
+        gf_msg(this->name, GF_LOG_WARNING, ENOENT, GD_MSG_FILE_OP_FAILED,
+               "Config file (%s) missing. Looking for template config"
+               " file (%s)",
+               conf_path, temp_conf_path);
+        ret = sys_lstat(temp_conf_path, &stbuf);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, ENOENT, GD_MSG_FILE_OP_FAILED,
+                   "Template "
+                   "config file (%s) missing.",
+                   temp_conf_path);
+            goto out;
+        }
+        gf_msg(this->name, GF_LOG_INFO, 0, GD_MSG_DEFAULT_TEMP_CONFIG,
+               "Using default config template(%s).", temp_conf_path);
+        working_conf_path = temp_conf_path;
+        *is_template_in_use = _gf_true;
+    }
 
-        ret = dict_get_str (dict, "host-uuid", &host_uuid);
-        if (ret < 0) {
-                gf_log ("", GF_LOG_ERROR, "Unable to fetch"
-                        " host-uuid from dict.");
-                goto out;
+fetch_data:
+    ret = glusterd_gsync_get_config(master, slave, working_conf_path, confd);
+    if (ret) {
+        if (*is_template_in_use == _gf_false) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_GET_CONFIG_INFO_FAILED,
+                   "Unable to get configuration data "
+                   "for %s(master), %s(slave). "
+                   "Trying template config.",
+                   master, slave);
+            working_conf_path = temp_conf_path;
+            *is_template_in_use = _gf_true;
+            goto fetch_data;
+        } else {
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_GET_CONFIG_INFO_FAILED,
+                   "Unable to get configuration data "
+                   "for %s(master), %s(slave) from "
+                   "template config",
+                   master, slave);
+            goto out;
+        }
+    }
+
+    ret = dict_get_param(confd, "state_file", &buf);
+    if (ret) {
+        if (*is_template_in_use == _gf_false) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+                   "Unable to get state_file's name. "
+                   "Trying template config.");
+            working_conf_path = temp_conf_path;
+            *is_template_in_use = _gf_true;
+            goto fetch_data;
+        } else {
+            gf_msg(this->name, GF_LOG_ERROR, 0,
+                   GD_MSG_GET_STATEFILE_NAME_FAILED,
+                   "Unable to get state_file's "
+                   "name from template.");
+            goto out;
         }
+    }
 
-        uuid_utoa_r (MY_UUID, uuid_str);
-        if (!strcmp (uuid_str, host_uuid)) {
-                ret = dict_get_str (dict, "source", &filename);
-                if (ret < 0) {
-                        gf_log ("", GF_LOG_ERROR, "Unable to fetch"
-                                " filename from dict.");
-                        *op_errstr = gf_strdup ("command unsuccessful");
-                        goto out;
-                }
-                snprintf (abs_filename, sizeof(abs_filename),
-                          "%s/%s", priv->workdir, filename);
-
-                ret = lstat (abs_filename, &stbuf);
-                if (ret) {
-                        snprintf (errmsg, sizeof (errmsg), "Source file"
-                                  " does not exist in %s", priv->workdir);
-                        *op_errstr = gf_strdup (errmsg);
-                        goto out;
-                }
+    ret = 0;
+out:
+    if (buf) {
+        *statefile = gf_strdup(buf);
+        if (!*statefile)
+            ret = -1;
+    }
 
-                if (!S_ISREG(stbuf.st_mode)) {
-                        snprintf (errmsg, sizeof (errmsg), "Source file"
-                                 " is not a regular file.");
-                        *op_errstr = gf_strdup (errmsg);
-                        gf_log ("", GF_LOG_ERROR, "%s", errmsg);
-                        ret = -1;
-                        goto out;
-                }
-        }
+    if (confd)
+        dict_unref(confd);
 
-        ret = 0;
-out:
-        gf_log ("", GF_LOG_DEBUG, "Returning %d", ret);
-        return ret;
+    gf_msg_debug(this->name, 0, "Returning %d ", ret);
+    return ret;
 }
 
 int
-glusterd_get_statefile_name (glusterd_volinfo_t *volinfo, char *slave,
-                             char *conf_path, char **statefile,
-                             gf_boolean_t *is_template_in_use)
+glusterd_create_status_file(char *master, char *slave, char *slave_host,
+                            char *slave_vol, char *status)
 {
-        char            *master                   = NULL;
-        char            *buf                      = NULL;
-        char            *working_conf_path        = NULL;
-        char             temp_conf_path[PATH_MAX] = "";
-        dict_t          *confd                    = NULL;
-        glusterd_conf_t *priv                     = NULL;
-        int              ret                      = -1;
-        struct stat      stbuf                    = {0,};
-
-        GF_ASSERT (THIS);
-        GF_ASSERT (THIS->private);
-        GF_ASSERT (volinfo);
-        GF_ASSERT (conf_path);
-        GF_ASSERT (is_template_in_use);
-
-        master = volinfo->volname;
+    int ret = -1;
+    runner_t runner = {
+        0,
+    };
+    glusterd_conf_t *priv = NULL;
+    xlator_t *this = NULL;
 
-        confd = dict_new ();
-        if (!confd) {
-                gf_log ("", GF_LOG_ERROR, "Unable to create new dict");
-                goto out;
-        }
+    this = THIS;
+    GF_ASSERT(this);
 
+    if (THIS)
         priv = THIS->private;
+    if (priv == NULL) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_GLUSTERD_PRIV_NOT_FOUND,
+               "priv of glusterd not present");
+        goto out;
+    }
+
+    if (!status) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_STATUS_NULL, "Status Empty");
+        goto out;
+    }
+    gf_msg_debug(this->name, 0, "slave = %s", slave);
+
+    runinit(&runner);
+    runner_add_args(&runner, GSYNCD_PREFIX "/gsyncd", "--create", status, "-c",
+                    NULL);
+    runner_argprintf(&runner, "%s/" GEOREP "/%s_%s_%s/gsyncd.conf",
+                     priv->workdir, master, slave_host, slave_vol);
+    runner_argprintf(&runner, "--iprefix=%s", DATADIR);
+    runner_argprintf(&runner, ":%s", master);
+    runner_add_args(&runner, slave, NULL);
+    synclock_unlock(&priv->big_lock);
+    ret = runner_run(&runner);
+    synclock_lock(&priv->big_lock);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_STATUSFILE_CREATE_FAILED,
+               "Creating status file failed.");
+        ret = -1;
+        goto out;
+    }
+
+    ret = 0;
+out:
+    gf_msg_debug(this->name, 0, "returning %d", ret);
+    return ret;
+}
 
-        snprintf (temp_conf_path, sizeof(temp_conf_path) - 1,
-                  "%s/"GSYNC_CONF_TEMPLATE, priv->workdir);
-
-        ret = lstat (conf_path, &stbuf);
-        if (!ret) {
-                gf_log ("", GF_LOG_INFO, "Using passed config template(%s).",
-                        conf_path);
-                working_conf_path = conf_path;
-        } else {
-                gf_log ("", GF_LOG_WARNING, "Config file (%s) missing. "
-                        "Looking for template config file (%s)",
-                        conf_path, temp_conf_path);
-                ret = lstat (temp_conf_path, &stbuf);
-                if (ret) {
-                        gf_log ("", GF_LOG_ERROR,
-                                "Template config file (%s) missing.",
-                                temp_conf_path);
-                        goto out;
-                }
-                gf_log ("", GF_LOG_INFO, "Using default config template(%s).",
-                        temp_conf_path);
-                working_conf_path = temp_conf_path;
-                *is_template_in_use = _gf_true;
-        }
-
-fetch_data:
-        ret = glusterd_gsync_get_config (master, slave, working_conf_path,
-                                         confd);
-        if (ret) {
-                if (*is_template_in_use == _gf_false) {
-                        gf_log ("", GF_LOG_ERROR,
-                                "Unable to get configuration data "
-                                "for %s(master), %s(slave). "
-                                "Trying template config.",
-                                master, slave);
-                        working_conf_path = temp_conf_path;
-                        *is_template_in_use = _gf_true;
-                        goto fetch_data;
-                } else {
-                        gf_log ("", GF_LOG_ERROR,
-                                "Unable to get configuration data "
-                                "for %s(master), %s(slave) from "
-                                "template config",
-                                master, slave);
-                        goto out;
-                }
-        }
+static int
+glusterd_verify_slave(char *volname, char *slave_url, char *slave_vol,
+                      int ssh_port, char **op_errstr,
+                      gf_boolean_t *is_force_blocker)
+{
+    int32_t ret = -1;
+    runner_t runner = {
+        0,
+    };
+    char log_file_path[PATH_MAX] = "";
+    char buf[PATH_MAX] = "";
+    char *tmp = NULL;
+    char *slave_url_buf = NULL;
+    char *save_ptr = NULL;
+    char *slave_user = NULL;
+    char *slave_ip = NULL;
+    glusterd_conf_t *priv = NULL;
+    xlator_t *this = NULL;
+    char *af = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+    priv = this->private;
+    GF_ASSERT(priv);
+    GF_ASSERT(volname);
+    GF_ASSERT(slave_url);
+    GF_ASSERT(slave_vol);
+
+    /* Fetch the slave_user and slave_ip from the slave_url.
+     * If the slave_user is not present. Use "root"
+     */
+    if (strstr(slave_url, "@")) {
+        slave_url_buf = gf_strdup(slave_url);
+        if (!slave_url_buf) {
+            gf_smsg(this->name, GF_LOG_ERROR, 0, GD_MSG_STRDUP_FAILED,
+                    "Slave_url=%s", slave_url, NULL);
+            goto out;
+        }
+
+        slave_user = strtok_r(slave_url_buf, "@", &save_ptr);
+        slave_ip = strtok_r(NULL, "@", &save_ptr);
+    } else {
+        slave_user = "root";
+        slave_ip = slave_url;
+    }
+
+    if (!slave_user || !slave_ip) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_SLAVE_URL_INVALID,
+               "Invalid slave url.");
+        goto out;
+    }
+
+    snprintf(log_file_path, sizeof(log_file_path), "%s/create_verify_log",
+             priv->logdir);
+
+    runinit(&runner);
+    runner_add_args(&runner, GSYNCD_PREFIX "/gverify.sh", NULL);
+    runner_argprintf(&runner, "%s", volname);
+    runner_argprintf(&runner, "%s", slave_user);
+    runner_argprintf(&runner, "%s", slave_ip);
+    runner_argprintf(&runner, "%s", slave_vol);
+    runner_argprintf(&runner, "%d", ssh_port);
+    runner_argprintf(&runner, "%s", log_file_path);
+    ret = dict_get_str(this->options, "transport.address-family", &af);
+    if (ret)
+        af = "-";
+
+    runner_argprintf(&runner, "%s", af);
+
+    gf_msg_debug(this->name, 0, "gverify Args = %s %s %s %s %s %s %s %s",
+                 runner.argv[0], runner.argv[1], runner.argv[2], runner.argv[3],
+                 runner.argv[4], runner.argv[5], runner.argv[6],
+                 runner.argv[7]);
+    runner_redir(&runner, STDOUT_FILENO, RUN_PIPE);
+    synclock_unlock(&priv->big_lock);
+    ret = runner_run(&runner);
+    synclock_lock(&priv->big_lock);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_INVALID_SLAVE,
+               "Not a valid slave");
+        ret = glusterd_gsync_read_frm_status(log_file_path, buf, sizeof(buf));
+        if (ret <= 0) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_READ_ERROR,
+                   "Unable to read from %s", log_file_path);
+            goto out;
+        }
+
+        /* Tokenize the error message from gverify.sh to figure out
+         * if the error is a force blocker or not. */
+        tmp = strtok_r(buf, "|", &save_ptr);
+        if (!tmp) {
+            ret = -1;
+            goto out;
+        }
+        if (!strcmp(tmp, "FORCE_BLOCKER"))
+            *is_force_blocker = 1;
+        else {
+            /* No FORCE_BLOCKER flag present so all that is
+             * present is the error message. */
+            *is_force_blocker = 0;
+            *op_errstr = gf_strdup(tmp);
+            ret = -1;
+            goto out;
+        }
+
+        /* Copy rest of the error message to op_errstr */
+        tmp = strtok_r(NULL, "|", &save_ptr);
+        if (tmp)
+            *op_errstr = gf_strdup(tmp);
+        ret = -1;
+        goto out;
+    }
+    ret = 0;
+out:
+    GF_FREE(slave_url_buf);
+    sys_unlink(log_file_path);
+    gf_msg_debug(this->name, 0, "Returning %d", ret);
+    return ret;
+}
 
-        ret = dict_get_param (confd, "state_file", &buf);
-        if (ret) {
-                if (*is_template_in_use == _gf_false) {
-                        gf_log ("", GF_LOG_ERROR,
-                                "Unable to get state_file's name. "
-                                "Trying template config.");
-                        working_conf_path = temp_conf_path;
-                        *is_template_in_use = _gf_true;
-                        goto fetch_data;
-                } else {
-                        gf_log ("", GF_LOG_ERROR,
-                                "Unable to get state_file's "
-                                "name from template.");
-                        goto out;
-                }
-        }
+/** @slave_ip remains unmodified */
+int
+glusterd_geo_rep_parse_slave(char *slave_url, char **hostname, char **op_errstr)
+{
+    int ret = -1;
+    char *tmp = NULL;
+    char *save_ptr = NULL;
+    char *host = NULL;
+    char errmsg[PATH_MAX] = "";
+    char *saved_url = NULL;
+    xlator_t *this = NULL;
 
-        ret = 0;
- out:
-        if (buf) {
-                *statefile = gf_strdup(buf);
-                if (!*statefile)
-                        ret = -1;
-        }
+    this = THIS;
+    GF_ASSERT(this);
 
-        if (confd)
-                dict_destroy (confd);
+    GF_ASSERT(slave_url);
+    GF_ASSERT(*slave_url);
 
-        gf_log ("", GF_LOG_DEBUG, "Returning %d ", ret);
-        return ret;
-}
+    saved_url = gf_strdup(slave_url);
+    if (!saved_url)
+        goto out;
 
-int
-glusterd_create_status_file (char *master, char *slave, char *slave_host,
-                             char *slave_vol, char *status)
-{
-        int                ret    = -1;
-        runner_t           runner = {0,};
-        glusterd_conf_t   *priv   = NULL;
-
-        if (THIS)
-                priv = THIS->private;
-        if (priv == NULL) {
-                gf_log ("", GF_LOG_ERROR, "priv of glusterd not present");
+    /* Checking if hostname has user specified */
+    host = strstr(saved_url, "@");
+    if (!host) { /* no user specified */
+        if (hostname) {
+            *hostname = gf_strdup(saved_url);
+            if (!*hostname)
                 goto out;
         }
 
-        if (!status) {
-                gf_log ("", GF_LOG_ERROR, "Status Empty");
-                goto out;
-        }
-        gf_log ("", GF_LOG_DEBUG, "slave = %s", slave);
-
-        runinit (&runner);
-        runner_add_args (&runner, GSYNCD_PREFIX"/gsyncd", "--create",
-                         status, "-c", NULL);
-        runner_argprintf (&runner, "%s/"GEOREP"/%s_%s_%s/gsyncd.conf",
-                          priv->workdir, master, slave_host, slave_vol);
-        runner_argprintf (&runner, "--iprefix=%s", DATADIR);
-        runner_argprintf (&runner, ":%s", master);
-        runner_add_args  (&runner, slave, NULL);
-        synclock_unlock (&priv->big_lock);
-        ret = runner_run (&runner);
-        synclock_lock (&priv->big_lock);
-        if (ret) {
-                gf_log ("", GF_LOG_ERROR, "Creating status file failed.");
-                ret = -1;
+        ret = 0;
+        goto out;
+    } else {
+        /* Moving the host past the '@' and checking if the
+         * actual hostname also has '@' */
+        host++;
+        if (strstr(host, "@")) {
+            gf_msg_debug(this->name, 0, "host = %s", host);
+            ret = snprintf(errmsg, sizeof(errmsg) - 1, "Invalid Hostname (%s).",
+                           host);
+            errmsg[ret] = '\0';
+            gf_msg(this->name, GF_LOG_ERROR, EINVAL, GD_MSG_INVALID_ENTRY, "%s",
+                   errmsg);
+            ret = -1;
+            if (op_errstr)
+                *op_errstr = gf_strdup(errmsg);
+            goto out;
+        }
+
+        ret = -1;
+
+        /**
+         * preliminary check for valid slave format.
+         */
+        tmp = strtok_r(saved_url, "@", &save_ptr);
+        tmp = strtok_r(NULL, "@", &save_ptr);
+        if (!tmp)
+            goto out;
+        if (hostname) {
+            *hostname = gf_strdup(tmp);
+            if (!*hostname)
                 goto out;
         }
+    }
 
-        ret = 0;
+    ret = 0;
 out:
-        gf_log ("", GF_LOG_DEBUG, "returning %d", ret);
-        return ret;
+    GF_FREE(saved_url);
+    if (ret)
+        if (hostname)
+            GF_FREE(*hostname);
+    gf_msg_debug(this->name, 0, "Returning %d", ret);
+    return ret;
 }
 
+/* Return -1 only if there is a match in volume uuid */
 static int
-glusterd_verify_slave (char *volname, char *slave_url, char *slave_vol,
-                       char **op_errstr, gf_boolean_t *is_force_blocker)
+get_slavehost_from_voluuid(dict_t *dict, char *key, data_t *value, void *data)
 {
-        int32_t          ret                     = -1;
-        runner_t         runner                  = {0,};
-        char             log_file_path[PATH_MAX] = "";
-        char             buf[PATH_MAX]           = "";
-        char            *tmp                     = NULL;
-        char            *slave_url_buf           = NULL;
-        char            *save_ptr                = NULL;
-        char            *slave_user              = NULL;
-        char            *slave_ip                = NULL;
-        glusterd_conf_t *priv                    = NULL;
-        xlator_t        *this                    = NULL;
-
-        this = THIS;
-        GF_ASSERT (this);
-        priv = this->private;
-        GF_ASSERT (priv);
-        GF_ASSERT (volname);
-        GF_ASSERT (slave_url);
-        GF_ASSERT (slave_vol);
+    char *slave_info = NULL;
+    char *tmp = NULL;
+    char *slave_host = NULL;
+    xlator_t *this = NULL;
+    struct slave_vol_config *slave_vol = NULL;
+    int i = 0;
+    int ret = -1;
 
-        /* Fetch the slave_user and slave_ip from the slave_url.
-         * If the slave_user is not present. Use "root"
-         */
-        if (strstr(slave_url, "@")) {
-                slave_url_buf = gf_strdup (slave_url);
-                if (!slave_url_buf)
-                        goto out;
+    this = THIS;
+    GF_VALIDATE_OR_GOTO("glusterd", this, out);
 
-                slave_user = strtok_r (slave_url_buf, "@", &save_ptr);
-                slave_ip = strtok_r (NULL, "@", &save_ptr);
-        } else {
-                slave_user = "root";
-                slave_ip = slave_url;
-        }
+    slave_vol = data;
+    slave_info = value->data;
 
-        if (!slave_user || !slave_ip) {
-                gf_log (this->name, GF_LOG_ERROR, "Invalid slave url.");
-                goto out;
-        }
+    gf_msg_debug(this->name, 0, "slave_info:%s !", slave_info);
 
-        snprintf (log_file_path, sizeof(log_file_path),
-                  DEFAULT_LOG_FILE_DIRECTORY"/create_verify_log");
-
-        runinit (&runner);
-        runner_add_args  (&runner, GSYNCD_PREFIX"/gverify.sh", NULL);
-        runner_argprintf (&runner, "%s", volname);
-        runner_argprintf (&runner, "%s", slave_user);
-        runner_argprintf (&runner, "%s", slave_ip);
-        runner_argprintf (&runner, "%s", slave_vol);
-        runner_argprintf (&runner, "%s", log_file_path);
-        runner_redir (&runner, STDOUT_FILENO, RUN_PIPE);
-        synclock_unlock (&priv->big_lock);
-        ret = runner_run (&runner);
-        synclock_lock (&priv->big_lock);
-        if (ret) {
-                gf_log ("", GF_LOG_ERROR, "Not a valid slave");
-                ret = glusterd_gsync_read_frm_status (log_file_path,
-                                                      buf, sizeof(buf));
-                if (ret <= 0) {
-                        gf_log ("", GF_LOG_ERROR, "Unable to read from %s",
-                                log_file_path);
-                        goto out;
+    if (!(slave_info) || strlen(slave_info) == 0) {
+        /* no slaves present, peace  */
+        ret = 0;
+        goto out;
+    }
+
+    /* slave format:
+     * master_node_uuid:ssh://slave_host::slave_vol:slave_voluuid */
+    while (i++ < 5) {
+        slave_info = strchr(slave_info, ':');
+        if (slave_info)
+            slave_info++;
+        else
+            break;
+    }
+
+    if (!(slave_info) || strlen(slave_info) == 0) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_SLAVE_VOL_PARSE_FAIL,
+               "slave_info format is wrong!");
+        ret = -2;
+        goto out;
+    } else {
+        if (strcmp(slave_info, slave_vol->slave_voluuid) == 0) {
+            ret = -1;
+
+            /* get corresponding slave host for reference*/
+            slave_host = value->data;
+            slave_host = strstr(slave_host, "://");
+            if (slave_host) {
+                slave_host += 3;
+            } else {
+                gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_SLAVE_VOL_PARSE_FAIL,
+                       "Invalid slave_host format!");
+                ret = -2;
+                goto out;
+            }
+            /* To go past username in non-root geo-rep session */
+            tmp = strchr(slave_host, '@');
+            if (tmp) {
+                if ((tmp - slave_host) >= LOGIN_NAME_MAX) {
+                    gf_msg(this->name, GF_LOG_ERROR, 0,
+                           GD_MSG_SLAVE_VOL_PARSE_FAIL,
+                           "Invalid slave user length in %s", slave_host);
+                    ret = -2;
+                    goto out;
                 }
+                strncpy(slave_vol->old_slvuser, slave_host, (tmp - slave_host));
+                slave_vol->old_slvuser[(tmp - slave_host) + 1] = '\0';
+                slave_host = tmp + 1;
+            } else
+                strcpy(slave_vol->old_slvuser, "root");
+
+            tmp = strchr(slave_host, ':');
+            if (!tmp) {
+                gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_SLAVE_VOL_PARSE_FAIL,
+                       "Invalid slave_host!");
+                ret = -2;
+                goto out;
+            }
 
-                /* Tokenize the error message from gverify.sh to figure out
-                 * if the error is a force blocker or not. */
-                tmp = strtok_r (buf, "|", &save_ptr);
-                if (!strcmp (tmp, "FORCE_BLOCKER"))
-                        *is_force_blocker = 1;
-                else {
-                        /* No FORCE_BLOCKER flag present so all that is
-                         * present is the error message. */
-                        *is_force_blocker = 0;
-                        if (tmp)
-                                *op_errstr = gf_strdup (tmp);
-                        ret = -1;
-                        goto out;
-                }
+            strncpy(slave_vol->old_slvhost, slave_host, (tmp - slave_host));
+            slave_vol->old_slvhost[(tmp - slave_host) + 1] = '\0';
 
-                /* Copy rest of the error message to op_errstr */
-                tmp = strtok_r (NULL, "|", &save_ptr);
-                if (tmp)
-                        *op_errstr = gf_strdup (tmp);
-                ret = -1;
-                goto out;
+            goto out;
         }
-        ret = 0;
+    }
+
+    ret = 0;
 out:
-        GF_FREE (slave_url_buf);
-        unlink (log_file_path);
-        gf_log ("", GF_LOG_DEBUG, "Returning %d", ret);
-        return ret;
+    return ret;
 }
 
-/** @slave_ip remains unmodified */
-int
-glusterd_geo_rep_parse_slave (char *slave_url,
-                              char **hostname, char **op_errstr)
+/* Given slave host and slave volume, check whether slave volume uuid
+ * already present.
+ * If slave volume uuid is present, get corresponding slave host
+ * for reference */
+static int
+glusterd_get_slavehost_from_voluuid(glusterd_volinfo_t *volinfo,
+                                    char *slave_host, char *slave_vol,
+                                    struct slave_vol_config *slave1)
 {
-        int   ret             = -1;
-        char *tmp             = NULL;
-        char *save_ptr        = NULL;
-        char *host            = NULL;
-        char errmsg[PATH_MAX] = "";
-        char *saved_url       = NULL;
-
-        GF_ASSERT (slave_url);
-        GF_ASSERT (*slave_url);
-
-        saved_url = gf_strdup (slave_url);
-        if (!saved_url)
-                goto out;
+    int ret = -1;
+    xlator_t *this = NULL;
 
-        /* Checking if hostname has user specified */
-        host = strstr (saved_url, "@");
-        if (!host) { /* no user specified */
-                if (hostname) {
-                        *hostname = gf_strdup (saved_url);
-                        if (!*hostname)
-                                goto out;
-                }
+    this = THIS;
 
-                ret = 0;
-                goto out;
-        } else {
-                /* Moving the host past the '@' and checking if the
-                 * actual hostname also has '@' */
-                host++;
-                if (strstr (host, "@")) {
-                        gf_log ("", GF_LOG_DEBUG, "host = %s", host);
-                        ret = snprintf (errmsg, sizeof(errmsg) - 1,
-                                        "Invalid Hostname (%s).", host);
-                        errmsg[ret] = '\0';
-                        gf_log ("", GF_LOG_ERROR, "%s", errmsg);
-                        ret = -1;
-                        if (op_errstr)
-                                *op_errstr = gf_strdup (errmsg);
-                        goto out;
-                }
+    GF_VALIDATE_OR_GOTO(this->name, volinfo, out);
 
-                ret = -1;
-
-                /**
-                 * preliminary check for valid slave format.
-                 */
-                tmp = strtok_r (saved_url, "@", &save_ptr);
-                tmp = strtok_r (NULL, "@", &save_ptr);
-                if (!tmp)
-                        goto out;
-                if (hostname) {
-                        *hostname = gf_strdup (tmp);
-                        if (!*hostname)
-                                goto out;
-                }
-        }
-
-        ret = 0;
+    ret = dict_foreach(volinfo->gsync_slaves, get_slavehost_from_voluuid,
+                       slave1);
 out:
-        GF_FREE (saved_url);
-        if (ret)
-                if (hostname)
-                        GF_FREE (*hostname);
-        gf_log ("", GF_LOG_DEBUG, "Returning %d", ret);
-        return ret;
+    return ret;
 }
 
 int
-glusterd_op_stage_gsync_create (dict_t *dict, char **op_errstr)
+glusterd_op_stage_gsync_create(dict_t *dict, char **op_errstr)
 {
-        char               *down_peerstr              = NULL;
-        char               *slave                     = NULL;
-        char               *volname                   = NULL;
-        char               *host_uuid                 = NULL;
-        char               *statefile                 = NULL;
-        char               *slave_url                 = NULL;
-        char               *slave_host                = NULL;
-        char               *slave_vol                 = NULL;
-        char               *conf_path                 = NULL;
-        char                errmsg[PATH_MAX]          = "";
-        char                common_pem_file[PATH_MAX] = "";
-        char                hook_script[PATH_MAX]     = "";
-        char                uuid_str [64]             = "";
-        int                 ret                       = -1;
-        int                 is_pem_push               = -1;
-        gf_boolean_t        is_force                  = -1;
-        gf_boolean_t        is_no_verify              = -1;
-        gf_boolean_t        is_force_blocker          = -1;
-        gf_boolean_t        exists                    = _gf_false;
-        gf_boolean_t        is_template_in_use        = _gf_false;
-        glusterd_conf_t    *conf                      = NULL;
-        glusterd_volinfo_t *volinfo                   = NULL;
-        struct stat         stbuf                     = {0,};
-        xlator_t           *this                      = NULL;
-
-        this = THIS;
-        GF_ASSERT (this);
-        conf = this->private;
-        GF_ASSERT (conf);
-
-        ret = glusterd_op_gsync_args_get (dict, op_errstr, &volname,
-                                          &slave, &host_uuid);
-        if (ret) {
-                gf_log ("", GF_LOG_ERROR, "Unable to fetch arguments");
-                gf_log ("", GF_LOG_DEBUG, "Returning %d", ret);
-                return -1;
+    char *down_peerstr = NULL;
+    char *slave = NULL;
+    char *volname = NULL;
+    char *host_uuid = NULL;
+    char *statefile = NULL;
+    char *slave_url = NULL;
+    char *slave_host = NULL;
+    char *slave_vol = NULL;
+    char *conf_path = NULL;
+    char errmsg[PATH_MAX] = "";
+    char common_pem_file[PATH_MAX] = "";
+    char hook_script[PATH_MAX] = "";
+    char uuid_str[64] = "";
+    int ret = -1;
+    int is_pem_push = -1;
+    int ssh_port = 22;
+    gf_boolean_t is_force = -1;
+    gf_boolean_t is_no_verify = -1;
+    gf_boolean_t is_force_blocker = -1;
+    gf_boolean_t is_template_in_use = _gf_false;
+    glusterd_conf_t *conf = NULL;
+    glusterd_volinfo_t *volinfo = NULL;
+    struct stat stbuf = {
+        0,
+    };
+    xlator_t *this = NULL;
+    struct slave_vol_config slave1 = {
+        {0},
+    };
+    char old_slave_url[SLAVE_URL_INFO_MAX] = {0};
+    char old_confpath[PATH_MAX] = {0};
+    gf_boolean_t is_running = _gf_false;
+    char *statedir = NULL;
+    char statefiledir[PATH_MAX] = {
+        0,
+    };
+    gf_boolean_t is_different_slavehost = _gf_false;
+    gf_boolean_t is_different_username = _gf_false;
+    char *slave_user = NULL;
+    char *save_ptr = NULL;
+    char *slave_url_buf = NULL;
+    int32_t len = 0;
+
+    this = THIS;
+    GF_ASSERT(this);
+    conf = this->private;
+    GF_ASSERT(conf);
+
+    ret = glusterd_op_gsync_args_get(dict, op_errstr, &volname, &slave,
+                                     &host_uuid);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_ARG_FETCH_ERROR,
+               "Unable to fetch arguments");
+        gf_msg_debug(this->name, 0, "Returning %d", ret);
+        return -1;
+    }
+
+    if (conf->op_version < 2) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_UNSUPPORTED_VERSION,
+               "Op Version not supported.");
+        snprintf(errmsg, sizeof(errmsg),
+                 "One or more nodes do not"
+                 " support the required op version.");
+        *op_errstr = gf_strdup(errmsg);
+        ret = -1;
+        goto out;
+    }
+
+    ret = glusterd_volinfo_find(volname, &volinfo);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_WARNING, 0, GD_MSG_VOL_NOT_FOUND,
+               "volume name does not exist");
+        snprintf(errmsg, sizeof(errmsg),
+                 "Volume name %s does not"
+                 " exist",
+                 volname);
+        goto out;
+    }
+
+    ret = glusterd_get_slave_details_confpath(volinfo, dict, &slave_url,
+                                              &slave_host, &slave_vol,
+                                              &conf_path, op_errstr);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_SLAVEINFO_FETCH_ERROR,
+               "Unable to fetch slave or confpath details.");
+        ret = -1;
+        goto out;
+    }
+
+    is_force = dict_get_str_boolean(dict, "force", _gf_false);
+
+    uuid_utoa_r(MY_UUID, uuid_str);
+    if (!strcmp(uuid_str, host_uuid)) {
+        ret = glusterd_are_vol_all_peers_up(volinfo, &conf->peers,
+                                            &down_peerstr);
+        if ((ret == _gf_false) && !is_force) {
+            snprintf(errmsg, sizeof(errmsg),
+                     "Peer %s,"
+                     " which is a part of %s volume, is"
+                     " down. Please bring up the peer and"
+                     " retry.",
+                     down_peerstr, volinfo->volname);
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_PEER_DISCONNECTED, "%s",
+                   errmsg);
+            *op_errstr = gf_strdup(errmsg);
+            GF_FREE(down_peerstr);
+            down_peerstr = NULL;
+            gf_msg_debug(this->name, 0, "Returning %d", ret);
+            return -1;
+        } else if (ret == _gf_false) {
+            gf_msg(this->name, GF_LOG_INFO, 0, GD_MSG_PEER_DISCONNECTED,
+                   "Peer %s, which is a part of %s volume, is"
+                   " down. Force creating geo-rep session."
+                   " On bringing up the peer, re-run"
+                   " \"gluster system:: execute"
+                   " gsec_create\" and \"gluster volume"
+                   " geo-replication %s %s create push-pem"
+                   " force\"",
+                   down_peerstr, volinfo->volname, volinfo->volname, slave);
+            GF_FREE(down_peerstr);
+            down_peerstr = NULL;
+        }
+
+        ret = dict_get_int32(dict, "ssh_port", &ssh_port);
+        if (ret < 0 && ret != -ENOENT) {
+            snprintf(errmsg, sizeof(errmsg),
+                     "Fetching ssh_port failed while "
+                     "handling " GEOREP " options");
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED, "%s",
+                   errmsg);
+            goto out;
+        }
+
+        is_no_verify = dict_get_str_boolean(dict, "no_verify", _gf_false);
+
+        if (!is_no_verify) {
+            /* Checking if slave host is pingable, has proper passwordless
+             * ssh login setup, slave volume is created, slave vol is empty,
+             * and if it has enough memory and bypass in case of force if
+             * the error is not a force blocker */
+            ret = glusterd_verify_slave(volname, slave_url, slave_vol, ssh_port,
+                                        op_errstr, &is_force_blocker);
+            if (ret) {
+                if (is_force && !is_force_blocker) {
+                    gf_msg(this->name, GF_LOG_INFO, 0, GD_MSG_INVALID_SLAVE,
+                           "%s is not a valid slave "
+                           "volume. Error: %s. Force "
+                           "creating geo-rep"
+                           " session.",
+                           slave, *op_errstr);
+                } else {
+                    gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_INVALID_SLAVE,
+                           "%s is not a valid slave "
+                           "volume. Error: %s",
+                           slave, *op_errstr);
+                    ret = -1;
+
+                    goto out;
+                }
+            }
         }
 
-        if (conf->op_version < 2) {
-                gf_log ("", GF_LOG_ERROR, "Op Version not supported.");
-                snprintf (errmsg, sizeof(errmsg), "One or more nodes do not"
-                          " support the required op version.");
-                *op_errstr = gf_strdup (errmsg);
+        ret = dict_get_int32(dict, "push_pem", &is_pem_push);
+        if (!ret && is_pem_push) {
+            ret = snprintf(common_pem_file, sizeof(common_pem_file),
+                           "%s" GLUSTERD_COMMON_PEM_PUB_FILE, conf->workdir);
+            if ((ret < 0) || (ret >= sizeof(common_pem_file))) {
                 ret = -1;
                 goto out;
-        }
-
-        exists = glusterd_check_volume_exists (volname);
-        ret = glusterd_volinfo_find (volname, &volinfo);
-        if ((ret) || (!exists)) {
-                gf_log ("", GF_LOG_WARNING, "volume name does not exist");
-                snprintf (errmsg, sizeof(errmsg), "Volume name %s does not"
-                          " exist", volname);
-                *op_errstr = gf_strdup (errmsg);
-                gf_log ("", GF_LOG_DEBUG, "Returning %d", ret);
-                return -1;
-        }
+            }
 
-        ret = glusterd_get_slave_details_confpath (volinfo, dict, &slave_url,
-                                                   &slave_host, &slave_vol,
-                                                   &conf_path, op_errstr);
-        if (ret) {
-                gf_log ("", GF_LOG_ERROR,
-                        "Unable to fetch slave or confpath details.");
+            ret = snprintf(hook_script, sizeof(hook_script),
+                           "%s" GLUSTERD_CREATE_HOOK_SCRIPT, conf->workdir);
+            if ((ret < 0) || (ret >= sizeof(hook_script))) {
                 ret = -1;
                 goto out;
-        }
+            }
 
-        is_force = dict_get_str_boolean (dict, "force", _gf_false);
-
-        uuid_utoa_r (MY_UUID, uuid_str);
-        if (!strcmp (uuid_str, host_uuid)) {
-                ret = glusterd_are_vol_all_peers_up (volinfo,
-                                                     &conf->peers,
-                                                     &down_peerstr);
-                if ((ret == _gf_false) && !is_force) {
-                        snprintf (errmsg, sizeof (errmsg), "Peer %s,"
-                                  " which is a part of %s volume, is"
-                                  " down. Please bring up the peer and"
-                                  " retry.", down_peerstr,
-                                  volinfo->volname);
-                        gf_log ("", GF_LOG_ERROR, "%s", errmsg);
-                        *op_errstr = gf_strdup (errmsg);
-                        GF_FREE (down_peerstr);
-                        down_peerstr = NULL;
-                        gf_log ("", GF_LOG_DEBUG, "Returning %d", ret);
-                        return -1;
-                } else if (ret == _gf_false) {
-                        gf_log ("", GF_LOG_INFO, "Peer %s,"
-                                " which is a part of %s volume, is"
-                                " down. Force creating geo-rep session."
-                                " On bringing up the peer, re-run"
-                                " \"gluster system:: execute"
-                                " gsec_create\" and \"gluster volume"
-                                " geo-replication %s %s create push-pem"
-                                " force\"", down_peerstr, volinfo->volname,
-                                volinfo->volname, slave);
-                         GF_FREE (down_peerstr);
-                         down_peerstr = NULL;
+            ret = sys_lstat(common_pem_file, &stbuf);
+            if (ret) {
+                len = snprintf(errmsg, sizeof(errmsg),
+                               "%s"
+                               " required for push-pem is"
+                               " not present. Please run"
+                               " \"gluster system:: execute"
+                               " gsec_create\"",
+                               common_pem_file);
+                if (len < 0) {
+                    strcpy(errmsg, "<error>");
                 }
+                gf_msg(this->name, GF_LOG_ERROR, ENOENT, GD_MSG_FILE_OP_FAILED,
+                       "%s", errmsg);
+                *op_errstr = gf_strdup(errmsg);
+                ret = -1;
+                goto out;
+            }
 
-                is_no_verify = dict_get_str_boolean (dict, "no_verify", _gf_false);
-
-                if (!is_no_verify) {
-                        /* Checking if slave host is pingable, has proper passwordless
-                        * ssh login setup, slave volume is created, slave vol is empty,
-                        * and if it has enough memory and bypass in case of force if
-                        * the error is not a force blocker */
-                        ret = glusterd_verify_slave (volname, slave_url, slave_vol,
-                                                     op_errstr, &is_force_blocker);
-                        if (ret) {
-                                if (is_force && !is_force_blocker) {
-                                        gf_log ("", GF_LOG_INFO, "%s is not a valid slave"
-                                                " volume. Error: %s. Force creating geo-rep"
-                                                " session.", slave, *op_errstr);
-                                } else {
-                                        gf_log ("", GF_LOG_ERROR,
-                                                "%s is not a valid slave volume. Error: %s",
-                                                slave, *op_errstr);
-                                        ret = -1;
-
-                                        goto out;
-                                }
-                        }
+            ret = sys_lstat(hook_script, &stbuf);
+            if (ret) {
+                len = snprintf(errmsg, sizeof(errmsg),
+                               "The hook-script (%s) "
+                               "required for push-pem is not "
+                               "present. Please install the "
+                               "hook-script and retry",
+                               hook_script);
+                if (len < 0) {
+                    strcpy(errmsg, "<error>");
                 }
+                gf_msg(this->name, GF_LOG_ERROR, ENOENT, GD_MSG_FILE_OP_FAILED,
+                       "%s", errmsg);
+                *op_errstr = gf_strdup(errmsg);
+                ret = -1;
+                goto out;
+            }
 
-                ret = dict_get_int32 (dict, "push_pem", &is_pem_push);
-                if (!ret && is_pem_push) {
-                        ret = snprintf (common_pem_file,
-                                        sizeof(common_pem_file) - 1,
-                                        "%s"GLUSTERD_COMMON_PEM_PUB_FILE,
-                                        conf->workdir);
-                        common_pem_file[ret] = '\0';
-
-                        ret = snprintf (hook_script, sizeof(hook_script) - 1,
-                                        "%s"GLUSTERD_CREATE_HOOK_SCRIPT,
-                                        conf->workdir);
-                        hook_script[ret] = '\0';
-
-                        ret = lstat (common_pem_file, &stbuf);
-                        if (ret) {
-                                snprintf (errmsg, sizeof (errmsg), "%s"
-                                          " required for push-pem is"
-                                          " not present. Please run"
-                                          " \"gluster system:: execute"
-                                          " gsec_create\"", common_pem_file);
-                                gf_log ("", GF_LOG_ERROR, "%s", errmsg);
-                                *op_errstr = gf_strdup (errmsg);
-                                ret = -1;
-                                goto out;
-                        }
-
-                        ret = lstat (hook_script, &stbuf);
-                        if (ret) {
-                                snprintf (errmsg, sizeof (errmsg),
-                                          "The hook-script (%s) required "
-                                          "for push-pem is not present. "
-                                          "Please install the hook-script "
-                                          "and retry", hook_script);
-                                gf_log ("", GF_LOG_ERROR, "%s", errmsg);
-                                *op_errstr = gf_strdup (errmsg);
-                                ret = -1;
-                                goto out;
-                        }
-
-                        if (!S_ISREG(stbuf.st_mode)) {
-                                snprintf (errmsg, sizeof (errmsg), "%s"
-                                          " required for push-pem is"
-                                          " not a regular file. Please run"
-                                          " \"gluster system:: execute"
-                                          " gsec_create\"", common_pem_file);
-                                gf_log ("", GF_LOG_ERROR, "%s", errmsg);
-                                ret = -1;
-                                goto out;
-                        }
+            if (!S_ISREG(stbuf.st_mode)) {
+                len = snprintf(errmsg, sizeof(errmsg),
+                               "%s"
+                               " required for push-pem is"
+                               " not a regular file. Please"
+                               " run \"gluster system:: "
+                               "execute gsec_create\"",
+                               common_pem_file);
+                if (len < 0) {
+                    strcpy(errmsg, "<error>");
                 }
+                gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_REG_FILE_MISSING,
+                       "%s", errmsg);
+                ret = -1;
+                goto out;
+            }
         }
+    }
 
-        ret = glusterd_get_statefile_name (volinfo, slave,
-                                           conf_path, &statefile,
-                                           &is_template_in_use);
-        if (ret) {
-                if (!strstr(slave, "::"))
-                        snprintf (errmsg, sizeof (errmsg),
-                                  "%s is not a valid slave url.", slave);
-                else
-                        snprintf (errmsg, sizeof (errmsg), "Please check gsync "
-                                  "config file. Unable to get statefile's name");
-                gf_log ("", GF_LOG_ERROR, "%s", errmsg);
+    ret = glusterd_get_statefile_name(volinfo, slave, conf_path, &statefile,
+                                      &is_template_in_use);
+    if (ret) {
+        if (!strstr(slave, "::"))
+            snprintf(errmsg, sizeof(errmsg), "%s is not a valid slave url.",
+                     slave);
+        else
+            snprintf(errmsg, sizeof(errmsg),
+                     "Please check gsync "
+                     "config file. Unable to get statefile's name");
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_STATEFILE_NAME_NOT_FOUND,
+               "%s", errmsg);
+        ret = -1;
+        goto out;
+    }
+
+    ret = dict_set_str(dict, "statefile", statefile);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+               "Unable to store statefile path");
+        goto out;
+    }
+
+    if (snprintf(statefiledir, sizeof(statefiledir), "%s", statefile) >=
+        sizeof(statefiledir)) {
+        snprintf(errmsg, sizeof(errmsg), "Failed copying statefiledir");
+        goto out;
+    }
+    statedir = dirname(statefiledir);
+
+    ret = sys_lstat(statedir, &stbuf);
+    if (!ret && !is_force) {
+        snprintf(errmsg, sizeof(errmsg),
+                 "Session between %s"
+                 " and %s is already created.",
+                 volinfo->volname, slave);
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_SESSION_ALREADY_EXIST, "%s",
+               errmsg);
+        ret = -1;
+        goto out;
+    } else if (!ret)
+        gf_msg(this->name, GF_LOG_INFO, 0, GD_MSG_FORCE_CREATE_SESSION,
+               "Session between %s and %s is already created. Force"
+               " creating again.",
+               volinfo->volname, slave);
+
+    ret = glusterd_get_slave_voluuid(slave_host, slave_vol,
+                                     slave1.slave_voluuid);
+    if ((ret) || (strlen(slave1.slave_voluuid) == 0)) {
+        snprintf(errmsg, sizeof(errmsg), "Unable to get remote volume uuid.");
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_REMOTE_VOL_UUID_FAIL, "%s",
+               errmsg);
+        ret = -1;
+        goto out;
+    }
+
+    ret = dict_set_dynstr_with_alloc(dict, "slave_voluuid",
+                                     slave1.slave_voluuid);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+               "Unable to set slave volume uuid in the dict");
+        goto out;
+    }
+
+    /* Check whether session is already created using slave volume uuid */
+    ret = glusterd_get_slavehost_from_voluuid(volinfo, slave_host, slave_vol,
+                                              &slave1);
+    if (ret == -1) {
+        if (!is_force) {
+            snprintf(errmsg, sizeof(errmsg),
+                     "Session between %s"
+                     " and %s:%s is already created! Cannot create "
+                     "with new slave:%s again!",
+                     volinfo->volname, slave1.old_slvhost, slave_vol,
+                     slave_host);
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_FORCE_CREATE_SESSION,
+                   "Session between"
+                   " %s and %s:%s is already created! "
+                   "Cannot create with new slave:%s again!",
+                   volinfo->volname, slave1.old_slvhost, slave_vol, slave_host);
+            goto out;
+        }
+
+        /* There is a remote possibility that slave_host can be NULL when
+           control reaches here. Add a check so we wouldn't crash in next
+           line */
+        if (!slave_host)
+            goto out;
+
+        /* Now, check whether session is already started.If so, warn!*/
+        is_different_slavehost = (strcmp(slave_host, slave1.old_slvhost) != 0)
+                                     ? _gf_true
+                                     : _gf_false;
+
+        if (strstr(slave_url, "@")) {
+            slave_url_buf = gf_strdup(slave_url);
+            if (!slave_url_buf) {
+                gf_msg(this->name, GF_LOG_ERROR, ENOMEM, GD_MSG_NO_MEMORY,
+                       "Unable to allocate memory");
                 ret = -1;
                 goto out;
-        }
+            }
+            slave_user = strtok_r(slave_url_buf, "@", &save_ptr);
+        } else
+            slave_user = "root";
+        is_different_username = (strcmp(slave_user, slave1.old_slvuser) != 0)
+                                    ? _gf_true
+                                    : _gf_false;
+
+        /* Do the check, only if different slave host/slave user */
+        if (is_different_slavehost || is_different_username) {
+            len = snprintf(old_confpath, sizeof(old_confpath),
+                           "%s/" GEOREP "/%s_%s_%s/gsyncd.conf", conf->workdir,
+                           volinfo->volname, slave1.old_slvhost, slave_vol);
+            if ((len < 0) || (len >= sizeof(old_confpath))) {
+                ret = -1;
+                goto out;
+            }
 
-        ret = dict_set_str (dict, "statefile", statefile);
-        if (ret) {
-                gf_log ("", GF_LOG_ERROR,
-                        "Unable to store statefile path");
+            /* construct old slave url with (old) slave host */
+            len = snprintf(old_slave_url, sizeof(old_slave_url), "%s::%s",
+                           slave1.old_slvhost, slave_vol);
+            if ((len < 0) || (len >= sizeof(old_slave_url))) {
+                ret = -1;
                 goto out;
-        }
+            }
 
-        ret = lstat (statefile, &stbuf);
-        if (!ret && !is_force) {
-                snprintf (errmsg, sizeof (errmsg), "Session between %s"
-                          " and %s is already created.",
-                          volinfo->volname, slave);
-                gf_log ("", GF_LOG_ERROR, "%s", errmsg);
+            ret = glusterd_check_gsync_running_local(
+                volinfo->volname, old_slave_url, old_confpath, &is_running);
+            if (_gf_true == is_running) {
+                (void)snprintf(errmsg, sizeof(errmsg),
+                               "Geo"
+                               "-replication session between %s and %s"
+                               " is still active. Please stop the "
+                               "session and retry.",
+                               volinfo->volname, old_slave_url);
                 ret = -1;
                 goto out;
-        } else if (!ret)
-                gf_log ("", GF_LOG_INFO, "Session between %s"
-                        " and %s is already created. Force"
-                        " creating again.", volinfo->volname, slave);
+            }
+        }
 
-        ret = glusterd_verify_gsyncd_spawn (volinfo->volname, slave);
+        ret = dict_set_dynstr_with_alloc(dict, "old_slavehost",
+                                         slave1.old_slvhost);
         if (ret) {
-                snprintf (errmsg, sizeof (errmsg), "Unable to spawn gsyncd.");
-                gf_log ("", GF_LOG_ERROR, "%s", errmsg);
-                goto out;
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+                   "Unable to set old_slavehost in the dict");
+            goto out;
         }
 
-        ret = 0;
+        ret = dict_set_int32(dict, "existing_session", _gf_true);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+                   "Unable to set existing_session in the dict");
+            goto out;
+        }
+    } else if (ret == -2) {
+        snprintf(errmsg, sizeof(errmsg),
+                 "get_slavehost_from_voluuid"
+                 " failed for %s::%s. Please check the glusterd logs.",
+                 slave_host, slave_vol);
+        gf_msg(this->name, GF_LOG_INFO, 0, GD_MSG_FORCE_CREATE_SESSION,
+               "get_slavehost_from_voluuid failed %s %s!!", slave_host,
+               slave_vol);
+        goto out;
+    }
+
+    ret = glusterd_verify_gsyncd_spawn(volinfo->volname, slave);
+    if (ret) {
+        snprintf(errmsg, sizeof(errmsg), "Unable to spawn gsyncd.");
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_GSYNCD_SPAWN_FAILED, "%s",
+               errmsg);
+        goto out;
+    }
+
+    ret = 0;
 out:
 
-        if (ret && errmsg[0] != '\0')
-                *op_errstr = gf_strdup (errmsg);
+    if (ret && errmsg[0] != '\0')
+        *op_errstr = gf_strdup(errmsg);
+
+    if (slave_url_buf)
+        GF_FREE(slave_url_buf);
 
-        gf_log ("", GF_LOG_DEBUG, "Returning %d", ret);
-        return ret;
+    return ret;
 }
 
 /* pre-condition check for geo-rep pause/resume.
@@ -2424,590 +3585,649 @@ out:
  *        -1 on any check failed.
  */
 static int
-gd_pause_resume_validation (int type, glusterd_volinfo_t *volinfo,
-                            char *slave, char *statefile, char **op_errstr)
+gd_pause_resume_validation(int type, glusterd_volinfo_t *volinfo, char *slave,
+                           char *statefile, char **op_errstr)
 {
-        int        ret                        = 0;
-        char       errmsg[PATH_MAX]           = {0,};
-        char       monitor_status[NAME_MAX]   = {0,};
-
-        GF_ASSERT (volinfo);
-        GF_ASSERT (slave);
-        GF_ASSERT (statefile);
-        GF_ASSERT (op_errstr);
-
-        ret = glusterd_gsync_read_frm_status (statefile, monitor_status,
-                                               sizeof (monitor_status));
-        if (ret <= 0) {
-                snprintf (errmsg, sizeof(errmsg), "Pause check Failed:"
-                          " Geo-rep session is not setup");
-                ret = -1;
-                goto out;
-        }
-
-        if ( type == GF_GSYNC_OPTION_TYPE_PAUSE &&
-             strstr (monitor_status, "Paused")) {
-                snprintf (errmsg, sizeof(errmsg), "Geo-replication"
-                          " session between %s and %s already Paused.",
-                          volinfo->volname, slave);
-                ret = -1;
-                goto out;
-        }
-        if ( type == GF_GSYNC_OPTION_TYPE_RESUME &&
-             !strstr (monitor_status, "Paused")) {
-                snprintf (errmsg, sizeof(errmsg), "Geo-replication"
-                          " session between %s and %s is not Paused.",
-                          volinfo->volname, slave);
-                ret = -1;
-                goto out;
-        }
-        ret = 0;
+    int ret = 0;
+    char errmsg[PATH_MAX] = {
+        0,
+    };
+    char monitor_status[NAME_MAX] = {
+        0,
+    };
+
+    GF_ASSERT(volinfo);
+    GF_ASSERT(slave);
+    GF_ASSERT(statefile);
+    GF_ASSERT(op_errstr);
+
+    ret = glusterd_gsync_read_frm_status(statefile, monitor_status,
+                                         sizeof(monitor_status));
+    if (ret <= 0) {
+        snprintf(errmsg, sizeof(errmsg),
+                 "Pause check Failed:"
+                 " Geo-rep session is not setup");
+        ret = -1;
+        goto out;
+    }
+
+    if (type == GF_GSYNC_OPTION_TYPE_PAUSE &&
+        strstr(monitor_status, "Paused")) {
+        snprintf(errmsg, sizeof(errmsg),
+                 "Geo-replication"
+                 " session between %s and %s already Paused.",
+                 volinfo->volname, slave);
+        ret = -1;
+        goto out;
+    }
+    if (type == GF_GSYNC_OPTION_TYPE_RESUME &&
+        !strstr(monitor_status, "Paused")) {
+        snprintf(errmsg, sizeof(errmsg),
+                 "Geo-replication"
+                 " session between %s and %s is not Paused.",
+                 volinfo->volname, slave);
+        ret = -1;
+        goto out;
+    }
+    ret = 0;
 out:
-        if (ret && (errmsg[0] != '\0')) {
-                *op_errstr = gf_strdup (errmsg);
-        }
-        return ret;
+    if (ret && (errmsg[0] != '\0')) {
+        *op_errstr = gf_strdup(errmsg);
+    }
+    return ret;
 }
 
 int
-glusterd_op_stage_gsync_set (dict_t *dict, char **op_errstr)
+glusterd_op_stage_gsync_set(dict_t *dict, char **op_errstr)
 {
-        int                 ret                   = 0;
-        int                 type                  = 0;
-        int                 pfd                   = -1;
-        char               *volname               = NULL;
-        char               *slave                 = NULL;
-        char               *slave_url             = NULL;
-        char               *slave_host            = NULL;
-        char               *slave_vol             = NULL;
-        char               *down_peerstr          = NULL;
-        char               *statefile             = NULL;
-        char               *path_list             = NULL;
-        char               *conf_path             = NULL;
-        gf_boolean_t        exists                = _gf_false;
-        glusterd_volinfo_t *volinfo               = NULL;
-        char                    errmsg[PATH_MAX]  = {0,};
-        char                    pidfile[PATH_MAX] = {0,};
-        dict_t             *ctx                   = NULL;
-        gf_boolean_t        is_force              = 0;
-        gf_boolean_t        is_running            = _gf_false;
-        gf_boolean_t        is_template_in_use    = _gf_false;
-        uuid_t              uuid                  = {0};
-        char                    uuid_str [64]     = {0};
-        char               *host_uuid             = NULL;
-        xlator_t           *this                  = NULL;
-        glusterd_conf_t    *conf                  = NULL;
-        struct stat         stbuf                 = {0,};
-
-        this = THIS;
-        GF_ASSERT (this);
-        conf = this->private;
-        GF_ASSERT (conf);
-
-        ret = dict_get_int32 (dict, "type", &type);
-        if (ret < 0) {
-                gf_log ("", GF_LOG_WARNING, "command type not found");
-                *op_errstr = gf_strdup ("command unsuccessful");
-                goto out;
-        }
-
-        if (type == GF_GSYNC_OPTION_TYPE_STATUS) {
-                ret = glusterd_verify_gsync_status_opts (dict, op_errstr);
-                goto out;
-        }
-
-        ret = glusterd_op_gsync_args_get (dict, op_errstr,
-                                          &volname, &slave, &host_uuid);
-        if (ret)
-                goto out;
-
-        uuid_utoa_r (MY_UUID, uuid_str);
-
-        if (conf->op_version < 2) {
-                snprintf (errmsg, sizeof(errmsg), "One or more nodes do not"
-                          " support the required op version.");
-                ret = -1;
+    int ret = 0;
+    int type = 0;
+    char *volname = NULL;
+    char *slave = NULL;
+    char *slave_url = NULL;
+    char *slave_host = NULL;
+    char *slave_vol = NULL;
+    char *down_peerstr = NULL;
+    char *statefile = NULL;
+    char statefiledir[PATH_MAX] = {
+        0,
+    };
+    char *statedir = NULL;
+    char *path_list = NULL;
+    char *conf_path = NULL;
+    glusterd_volinfo_t *volinfo = NULL;
+    char errmsg[PATH_MAX] = {
+        0,
+    };
+    dict_t *ctx = NULL;
+    gf_boolean_t is_force = 0;
+    gf_boolean_t is_running = _gf_false;
+    gf_boolean_t is_template_in_use = _gf_false;
+    uuid_t uuid = {0};
+    char uuid_str[64] = {0};
+    char *host_uuid = NULL;
+    xlator_t *this = NULL;
+    glusterd_conf_t *conf = NULL;
+    struct stat stbuf = {
+        0,
+    };
+
+    this = THIS;
+    GF_ASSERT(this);
+    conf = this->private;
+    GF_ASSERT(conf);
+
+    ret = dict_get_int32(dict, "type", &type);
+    if (ret < 0) {
+        gf_msg(this->name, GF_LOG_WARNING, 0, GD_MSG_DICT_GET_FAILED,
+               "command type not found");
+        *op_errstr = gf_strdup("command unsuccessful");
+        goto out;
+    }
+
+    if (type == GF_GSYNC_OPTION_TYPE_STATUS) {
+        ret = glusterd_verify_gsync_status_opts(dict, op_errstr);
+        goto out;
+    }
+
+    ret = glusterd_op_gsync_args_get(dict, op_errstr, &volname, &slave,
+                                     &host_uuid);
+    if (ret)
+        goto out;
+
+    uuid_utoa_r(MY_UUID, uuid_str);
+
+    if (conf->op_version < 2) {
+        snprintf(errmsg, sizeof(errmsg),
+                 "One or more nodes do not"
+                 " support the required op version.");
+        ret = -1;
+        goto out;
+    }
+
+    ret = glusterd_volinfo_find(volname, &volinfo);
+    if (ret) {
+        snprintf(errmsg, sizeof(errmsg),
+                 "Volume name %s does not"
+                 " exist",
+                 volname);
+        goto out;
+    }
+
+    ret = glusterd_get_slave_details_confpath(volinfo, dict, &slave_url,
+                                              &slave_host, &slave_vol,
+                                              &conf_path, op_errstr);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_SLAVEINFO_FETCH_ERROR,
+               "Unable to fetch slave or confpath details.");
+        ret = -1;
+        goto out;
+    }
+
+    is_force = dict_get_str_boolean(dict, "force", _gf_false);
+
+    ret = glusterd_get_statefile_name(volinfo, slave, conf_path, &statefile,
+                                      &is_template_in_use);
+    if (ret) {
+        if (!strstr(slave, "::")) {
+            snprintf(errmsg, sizeof(errmsg), "%s is not a valid slave url.",
+                     slave);
+            ret = -1;
+            goto out;
+        } else {
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_SLAVE_URL_INVALID,
+                   "state_file entry missing in config file (%s)", conf_path);
+
+            if ((type == GF_GSYNC_OPTION_TYPE_STOP) && is_force) {
+                gf_msg(this->name, GF_LOG_WARNING, 0, GD_MSG_STOP_FORCE,
+                       "Allowing stop "
+                       "force to bypass missing statefile "
+                       "entry in config file (%s), and "
+                       "template file",
+                       conf_path);
+                ret = 0;
+            } else
                 goto out;
         }
-
-        exists = glusterd_check_volume_exists (volname);
-        ret = glusterd_volinfo_find (volname, &volinfo);
-        if ((ret) || (!exists)) {
-                snprintf (errmsg, sizeof(errmsg), "Volume name %s does not"
-                          " exist", volname);
+    } else {
+        ret = dict_set_str(dict, "statefile", statefile);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+                   "Unable to store statefile path");
+            goto out;
+        }
+    }
+
+    /* Allowing stop force to bypass the statefile check
+     * as this command acts as a fail safe method to stop geo-rep
+     * session. */
+    if (!((type == GF_GSYNC_OPTION_TYPE_STOP) && is_force)) {
+        /* check session directory as statefile may not present
+         * during upgrade */
+        if (snprintf(statefiledir, sizeof(statefiledir), "%s", statefile) >=
+            sizeof(statefiledir)) {
+            snprintf(errmsg, sizeof(errmsg), "Failed copying statefiledir");
+            ret = -1;
+            goto out;
+        }
+        statedir = dirname(statefiledir);
+
+        ret = sys_lstat(statedir, &stbuf);
+        if (ret) {
+            snprintf(errmsg, sizeof(errmsg),
+                     "Geo-replication"
+                     " session between %s and %s does not exist.",
+                     volinfo->volname, slave);
+            gf_msg(this->name, GF_LOG_ERROR, ENOENT, GD_MSG_FILE_OP_FAILED,
+                   "%s. statefile = %s", errmsg, statefile);
+            ret = -1;
+            goto out;
+        }
+    }
+
+    /* Check if all peers that are a part of the volume are up or not */
+    if ((type == GF_GSYNC_OPTION_TYPE_DELETE) ||
+        ((type == GF_GSYNC_OPTION_TYPE_STOP) && !is_force) ||
+        (type == GF_GSYNC_OPTION_TYPE_PAUSE) ||
+        (type == GF_GSYNC_OPTION_TYPE_RESUME)) {
+        if (!strcmp(uuid_str, host_uuid)) {
+            ret = glusterd_are_vol_all_peers_up(volinfo, &conf->peers,
+                                                &down_peerstr);
+            if (ret == _gf_false) {
+                snprintf(errmsg, sizeof(errmsg),
+                         "Peer %s,"
+                         " which is a part of %s volume, is"
+                         " down. Please bring up the peer and"
+                         " retry.",
+                         down_peerstr, volinfo->volname);
                 ret = -1;
+                GF_FREE(down_peerstr);
+                down_peerstr = NULL;
                 goto out;
+            }
         }
+    }
 
-        ret = glusterd_get_slave_details_confpath (volinfo, dict, &slave_url,
-                                                   &slave_host, &slave_vol,
-                                                   &conf_path, op_errstr);
-        if (ret) {
-                gf_log ("", GF_LOG_ERROR,
-                        "Unable to fetch slave or confpath details.");
+    switch (type) {
+        case GF_GSYNC_OPTION_TYPE_START:
+            if (is_template_in_use) {
+                snprintf(errmsg, sizeof(errmsg),
+                         "state-file entry "
+                         "missing in the config file(%s).",
+                         conf_path);
                 ret = -1;
                 goto out;
-        }
-
-        is_force = dict_get_str_boolean (dict, "force", _gf_false);
-
-        ret = glusterd_get_statefile_name (volinfo, slave,
-                                           conf_path, &statefile,
-                                           &is_template_in_use);
-        if (ret) {
-                if (!strstr(slave, "::")) {
-                        snprintf (errmsg, sizeof(errmsg),
-                                  "%s is not a valid slave url.", slave);
-                        ret = -1;
-                        goto out;
-                } else {
-                        gf_log ("", GF_LOG_ERROR,
-                                "state_file entry missing in config file (%s)",
-                                conf_path);
-
-                        if ((type == GF_GSYNC_OPTION_TYPE_STOP) && is_force) {
-                                gf_log ("", GF_LOG_WARNING, "Allowing stop "
-                                        "force to bypass missing statefile "
-                                        "entry in config file (%s), and "
-                                        "template file", conf_path);
-                                ret = 0;
-                        } else
-                                goto out;
-                }
-        } else {
-                ret = dict_set_str (dict, "statefile", statefile);
-                if (ret) {
-                        gf_log ("", GF_LOG_ERROR,
-                                "Unable to store statefile path");
-                        goto out;
-                }
-        }
-
-        /* Allowing stop force to bypass the statefile check
-         * as this command acts as a fail safe method to stop geo-rep
-         * session. */
-        if (!((type == GF_GSYNC_OPTION_TYPE_STOP) && is_force)) {
-                ret = lstat (statefile, &stbuf);
-                if (ret) {
-                        snprintf (errmsg, sizeof(errmsg), "Geo-replication"
-                                  " session between %s and %s does not exist.",
-                                  volinfo->volname, slave);
-                        gf_log ("", GF_LOG_ERROR, "%s. statefile = %s",
-                                errmsg, statefile);
-                        ret = -1;
-                        goto out;
-                }
-        }
+            }
 
-        /* Check if all peers that are a part of the volume are up or not */
-        if ((type == GF_GSYNC_OPTION_TYPE_DELETE) ||
-            ((type == GF_GSYNC_OPTION_TYPE_STOP) && !is_force) ||
-            (type == GF_GSYNC_OPTION_TYPE_PAUSE) ||
-            (type == GF_GSYNC_OPTION_TYPE_RESUME)) {
-                if (!strcmp (uuid_str, host_uuid)) {
-                        ret = glusterd_are_vol_all_peers_up (volinfo,
-                                                             &conf->peers,
-                                                             &down_peerstr);
-                        if (ret == _gf_false) {
-                                snprintf (errmsg, sizeof (errmsg), "Peer %s,"
-                                          " which is a part of %s volume, is"
-                                          " down. Please bring up the peer and"
-                                          " retry.", down_peerstr,
-                                          volinfo->volname);
-                                ret = -1;
-                                GF_FREE (down_peerstr);
-                                down_peerstr = NULL;
-                                goto out;
-                        }
+            ret = glusterd_op_verify_gsync_start_options(
+                volinfo, slave, conf_path, statefile, op_errstr, is_force);
+            if (ret)
+                goto out;
+            ctx = glusterd_op_get_ctx();
+            if (ctx) {
+                /* gsyncd does a fuse mount to start
+                 * the geo-rep session */
+                if (!glusterd_is_fuse_available()) {
+                    gf_msg("glusterd", GF_LOG_ERROR, errno,
+                           GD_MSG_GEO_REP_START_FAILED,
+                           "Unable "
+                           "to open /dev/fuse (%s), "
+                           "geo-replication start failed",
+                           strerror(errno));
+                    snprintf(errmsg, sizeof(errmsg), "fuse unavailable");
+                    ret = -1;
+                    goto out;
                 }
-        }
+            }
+            break;
 
-        switch (type) {
-        case GF_GSYNC_OPTION_TYPE_START:
+        case GF_GSYNC_OPTION_TYPE_STOP:
+            if (!is_force) {
                 if (is_template_in_use) {
-                        snprintf (errmsg, sizeof(errmsg), "state-file entry "
-                                  "missing in the config file(%s).",
-                                  conf_path);
-                        ret = -1;
-                        goto out;
+                    snprintf(errmsg, sizeof(errmsg),
+                             "state-file entry missing in "
+                             "the config file(%s).",
+                             conf_path);
+                    ret = -1;
+                    goto out;
                 }
 
-                /* don't attempt to start gsync if replace-brick is
-                 * in progress */
-                if (glusterd_is_rb_ongoing (volinfo)) {
-                        snprintf (errmsg, sizeof(errmsg), "replace-brick is in"
-                                   " progress, not starting geo-replication");
-                        ret = -1;
-                        goto out;
-                }
-
-                ret = glusterd_op_verify_gsync_start_options (volinfo, slave,
-                                                              conf_path,
-                                                              statefile,
-                                                              op_errstr, is_force);
-                if (ret)
+                ret = glusterd_op_verify_gsync_running(volinfo, slave,
+                                                       conf_path, op_errstr);
+                if (ret) {
+                    ret = glusterd_get_local_brickpaths(volinfo, &path_list);
+                    if (!path_list && ret == -1)
                         goto out;
-                ctx = glusterd_op_get_ctx();
-                if (ctx) {
-                        /* gsyncd does a fuse mount to start
-                         * the geo-rep session */
-                        if (!glusterd_is_fuse_available ()) {
-                                gf_log ("glusterd", GF_LOG_ERROR, "Unable to "
-                                        "open /dev/fuse (%s), geo-replication "
-                                        "start failed", strerror (errno));
-                                snprintf (errmsg, sizeof(errmsg),
-                                          "fuse unvailable");
-                                ret = -1;
-                                goto out;
-                        }
                 }
-                break;
 
-        case GF_GSYNC_OPTION_TYPE_STOP:
-                if (!is_force) {
-                        if (is_template_in_use) {
-                                snprintf (errmsg, sizeof(errmsg),
-                                          "state-file entry missing in "
-                                          "the config file(%s).", conf_path);
-                                ret = -1;
-                                goto out;
-                        }
-
-                        ret = glusterd_op_verify_gsync_running (volinfo, slave,
-                                                                conf_path,
-                                                                op_errstr);
-                        if (ret) {
-                                ret = glusterd_get_local_brickpaths (volinfo,
-                                                                     &path_list);
-                                if (path_list)
-                                        ret = -1;
-                        }
+                /* Check for geo-rep session is active or not for
+                 * configured user.*/
+                ret = glusterd_gsync_get_uuid(slave, volinfo, uuid);
+                if (ret) {
+                    snprintf(errmsg, sizeof(errmsg),
+                             "Geo-replication session between %s "
+                             "and %s does not exist.",
+                             volinfo->volname, slave);
+                    ret = -1;
+                    goto out;
                 }
-                break;
+            }
+            break;
 
         case GF_GSYNC_OPTION_TYPE_PAUSE:
         case GF_GSYNC_OPTION_TYPE_RESUME:
-                if (is_template_in_use) {
-                        snprintf (errmsg, sizeof(errmsg),
-                                  "state-file entry missing in "
-                                  "the config file(%s).", conf_path);
-                        ret = -1;
-                        goto out;
-                }
+            if (is_template_in_use) {
+                snprintf(errmsg, sizeof(errmsg),
+                         "state-file entry missing in "
+                         "the config file(%s).",
+                         conf_path);
+                ret = -1;
+                goto out;
+            }
 
-                ret = glusterd_op_verify_gsync_running (volinfo, slave,
-                                                        conf_path, op_errstr);
-                if (ret)
-                        goto out;
-                if (!is_force) {
-                        ret = gd_pause_resume_validation (type, volinfo, slave,
-                                                          statefile, op_errstr);
-                        if (ret)
-                                goto out;
-                }
-                break;
+            ret = glusterd_op_verify_gsync_running(volinfo, slave, conf_path,
+                                                   op_errstr);
+            if (ret) {
+                ret = glusterd_get_local_brickpaths(volinfo, &path_list);
+                if (!path_list && ret == -1)
+                    goto out;
+            }
 
-        case GF_GSYNC_OPTION_TYPE_CONFIG:
-                if (is_template_in_use) {
-                        snprintf (errmsg, sizeof(errmsg), "state-file entry "
-                                  "missing in the config file(%s).",
-                                  conf_path);
-                        ret = -1;
-                        goto out;
-                }
+            /* Check for geo-rep session is active or not
+             * for configured user.*/
+            ret = glusterd_gsync_get_uuid(slave, volinfo, uuid);
+            if (ret) {
+                snprintf(errmsg, sizeof(errmsg),
+                         "Geo-replication"
+                         " session between %s and %s does not exist.",
+                         volinfo->volname, slave);
+                ret = -1;
+                goto out;
+            }
 
-                pfd = gsyncd_getpidfile (volname, slave, pidfile,
-                                         conf_path, &is_template_in_use);
-                if (is_template_in_use) {
-                        snprintf (errmsg, sizeof(errmsg), "pid-file entry "
-                                  "missing in the config file(%s).",
-                                  conf_path);
-                        ret = -1;
+            if (!is_force) {
+                ret = gd_pause_resume_validation(type, volinfo, slave,
+                                                 statefile, op_errstr);
+                if (ret) {
+                    ret = glusterd_get_local_brickpaths(volinfo, &path_list);
+                    if (!path_list && ret == -1)
                         goto out;
                 }
+            }
+            break;
 
-                ret = gsync_verify_config_options (dict, op_errstr, volname);
+        case GF_GSYNC_OPTION_TYPE_CONFIG:
+            if (is_template_in_use) {
+                snprintf(errmsg, sizeof(errmsg),
+                         "state-file entry "
+                         "missing in the config file(%s).",
+                         conf_path);
+                ret = -1;
                 goto out;
-                break;
+            }
+
+            ret = gsync_verify_config_options(dict, op_errstr, volname);
+            goto out;
+            break;
 
         case GF_GSYNC_OPTION_TYPE_DELETE:
-                /* Check if the gsync session is still running
-                 * If so ask the user to stop geo-replication first.*/
-                if (is_template_in_use) {
-                        snprintf (errmsg, sizeof(errmsg), "state-file entry "
-                                  "missing in the config file(%s).",
-                                  conf_path);
-                        ret = -1;
-                        goto out;
-                }
+            /* Check if the gsync session is still running
+             * If so ask the user to stop geo-replication first.*/
+            if (is_template_in_use) {
+                snprintf(errmsg, sizeof(errmsg),
+                         "state-file entry "
+                         "missing in the config file(%s).",
+                         conf_path);
+                ret = -1;
+                goto out;
+            }
 
-                ret = glusterd_gsync_get_uuid (slave, volinfo, uuid);
-                if (ret) {
-                        snprintf (errmsg, sizeof(errmsg), "Geo-replication"
-                                  " session between %s and %s does not exist.",
-                                  volinfo->volname, slave);
-                        ret = -1;
-                        goto out;
-                } else {
-                        ret = glusterd_check_gsync_running_local (volinfo->volname,
-                                                                  slave, conf_path,
-                                                                  &is_running);
-                        if (_gf_true == is_running) {
-                                snprintf (errmsg, sizeof (errmsg), GEOREP
-                                          " session between %s & %s is "
-                                          "still active. Please stop the "
-                                          "session and retry.",
-                                          volinfo->volname, slave);
-                                ret = -1;
-                                goto out;
-                        }
+            ret = glusterd_gsync_get_uuid(slave, volinfo, uuid);
+            if (ret) {
+                snprintf(errmsg, sizeof(errmsg),
+                         "Geo-replication"
+                         " session between %s and %s does not exist.",
+                         volinfo->volname, slave);
+                ret = -1;
+                goto out;
+            } else {
+                ret = glusterd_check_gsync_running_local(
+                    volinfo->volname, slave, conf_path, &is_running);
+                if (_gf_true == is_running) {
+                    snprintf(errmsg, sizeof(errmsg),
+                             GEOREP
+                             " session between %s & %s is "
+                             "still active. Please stop the "
+                             "session and retry.",
+                             volinfo->volname, slave);
+                    ret = -1;
+                    goto out;
                 }
+            }
 
-                ret = glusterd_verify_gsyncd_spawn (volinfo->volname, slave);
-                if (ret) {
-                        snprintf (errmsg, sizeof (errmsg),
-                                  "Unable to spawn gsyncd");
-                }
+            ret = glusterd_verify_gsyncd_spawn(volinfo->volname, slave);
+            if (ret) {
+                snprintf(errmsg, sizeof(errmsg), "Unable to spawn gsyncd");
+            }
 
-                break;
-        }
+            break;
+    }
 
 out:
-        if (path_list)
-                GF_FREE (path_list);
 
-        if (ret && errmsg[0] != '\0') {
-                gf_log (this->name, GF_LOG_ERROR, "%s", errmsg);
-                *op_errstr = gf_strdup (errmsg);
-        }
+    if (path_list)
+        GF_FREE(path_list);
+
+    if (ret && errmsg[0] != '\0') {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_GSYNCD_ERROR, "%s", errmsg);
+        *op_errstr = gf_strdup(errmsg);
+    }
 
-        gf_log ("", GF_LOG_DEBUG, "Returning %d", ret);
-        return ret;
+    gf_msg_debug(this->name, 0, "Returning %d", ret);
+    return ret;
 }
 
 static int
-gd_pause_or_resume_gsync (dict_t *dict, char *master, char *slave,
-                          char *slave_host, char *slave_vol, char *conf_path,
-                          char **op_errstr, gf_boolean_t is_pause)
+gd_pause_or_resume_gsync(dict_t *dict, char *master, char *slave,
+                         char *slave_host, char *slave_vol, char *conf_path,
+                         char **op_errstr, gf_boolean_t is_pause)
 {
-        int32_t         ret                      = 0;
-        int             pfd                      = -1;
-        pid_t           pid                      = 0;
-        char            pidfile[PATH_MAX]        = {0,};
-        char            errmsg[PATH_MAX]         = "";
-        char            buf [1024]               = {0,};
-        int             i                        = 0;
-        gf_boolean_t    is_template_in_use       = _gf_false;
-        char            monitor_status[NAME_MAX] = {0,};
-        char            *statefile               = NULL;
-        char            *token                   = NULL;
-        xlator_t        *this                    = NULL;
-
-        this = THIS;
-        GF_ASSERT (this);
-        GF_ASSERT (dict);
-        GF_ASSERT (master);
-        GF_ASSERT (slave);
-        GF_ASSERT (slave_host);
-        GF_ASSERT (slave_vol);
-        GF_ASSERT (conf_path);
-
-        pfd = gsyncd_getpidfile (master, slave, pidfile,
-                                 conf_path, &is_template_in_use);
-        if (pfd == -2) {
-                snprintf (errmsg, sizeof(errmsg),
-                          "pid-file entry mising in config file and "
-                          "template config file.");
-                gf_log (this->name, GF_LOG_ERROR, "%s", errmsg);
-                *op_errstr = gf_strdup (errmsg);
-                ret = -1;
-                goto out;
-        }
-
-        if (gsync_status_byfd (pfd) == -1) {
-                gf_log (this->name, GF_LOG_ERROR, "gsyncd b/w %s & %s is not"
-                        " running", master, slave);
-                /* monitor gsyncd already dead */
-                goto out;
-        }
-
-        if (pfd < 0)
+    int32_t ret = 0;
+    int pfd = -1;
+    long pid = 0;
+    char pidfile[PATH_MAX] = {
+        0,
+    };
+    char errmsg[PATH_MAX] = "";
+    char buf[4096] = {
+        0,
+    };
+    gf_boolean_t is_template_in_use = _gf_false;
+    char monitor_status[NAME_MAX] = {
+        0,
+    };
+    char *statefile = NULL;
+    xlator_t *this = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+    GF_ASSERT(dict);
+    GF_ASSERT(master);
+    GF_ASSERT(slave);
+    GF_ASSERT(slave_host);
+    GF_ASSERT(slave_vol);
+    GF_ASSERT(conf_path);
+
+    pfd = gsyncd_getpidfile(master, slave, pidfile, conf_path,
+                            &is_template_in_use);
+    if (pfd == -2) {
+        snprintf(errmsg, sizeof(errmsg),
+                 "pid-file entry mising in config file and "
+                 "template config file.");
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_PIDFILE_NOT_FOUND, "%s",
+               errmsg);
+        *op_errstr = gf_strdup(errmsg);
+        ret = -1;
+        goto out;
+    }
+
+    if (gsync_status_byfd(pfd) == -1) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_GSYNCD_ERROR,
+               "gsyncd b/w %s & %s is not running", master, slave);
+        /* monitor gsyncd already dead */
+        goto out;
+    }
+
+    if (pfd < 0)
+        goto out;
+
+    /* Prepare to update status file*/
+    ret = dict_get_str(dict, "statefile", &statefile);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+               "Pause/Resume Failed: Unable to fetch statefile path");
+        goto out;
+    }
+    ret = glusterd_gsync_read_frm_status(statefile, monitor_status,
+                                         sizeof(monitor_status));
+    if (ret <= 0) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_STAT_FILE_READ_FAILED,
+               "Pause/Resume Failed: "
+               "Unable to read status file for %s(master)"
+               " %s(slave)",
+               master, slave);
+        goto out;
+    }
+
+    ret = sys_read(pfd, buf, sizeof(buf) - 1);
+    if (ret > 0) {
+        buf[ret] = '\0';
+        pid = strtol(buf, NULL, 10);
+        if (is_pause) {
+            ret = kill(-pid, SIGSTOP);
+            if (ret) {
+                gf_msg(this->name, GF_LOG_ERROR, errno, GD_MSG_PID_KILL_FAIL,
+                       "Failed"
+                       " to pause gsyncd. Error: %s",
+                       strerror(errno));
                 goto out;
-
-        /* Prepare to update status file*/
-        ret = dict_get_str (dict, "statefile", &statefile);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "Pause/Resume Failed:"
-                        " Unable to fetch statefile path");
+            }
+            /*On pause force, if status is already paused
+              do not update status again*/
+            if (strstr(monitor_status, "Paused"))
+                goto out;
+
+            ret = glusterd_create_status_file(master, slave, slave_host,
+                                              slave_vol, "Paused");
+            if (ret) {
+                gf_msg(this->name, GF_LOG_ERROR, 0,
+                       GD_MSG_UPDATE_STATEFILE_FAILED,
+                       "Unable  to update state_file."
+                       " Error : %s",
+                       strerror(errno));
+                /* If status cannot be updated resume back */
+                if (kill(-pid, SIGCONT)) {
+                    snprintf(errmsg, sizeof(errmsg),
+                             "Pause successful but could "
+                             "not update status file. "
+                             "Please use 'resume force' to"
+                             " resume back and retry pause"
+                             " to reflect in status");
+                    gf_msg(this->name, GF_LOG_ERROR, errno,
+                           GD_MSG_PID_KILL_FAIL,
+                           "Resume back Failed. Error:"
+                           "%s",
+                           strerror(errno));
+                    *op_errstr = gf_strdup(errmsg);
+                }
                 goto out;
-        }
-        ret = glusterd_gsync_read_frm_status (statefile, monitor_status,
-                                              sizeof (monitor_status));
-        if (ret <= 0) {
-                gf_log (this->name, GF_LOG_ERROR, "Pause/Resume Failed: "
-                        "Unable to read status file for %s(master)"
-                        " %s(slave)", master, slave);
+            }
+        } else {
+            ret = glusterd_create_status_file(master, slave, slave_host,
+                                              slave_vol, "Started");
+            if (ret) {
+                gf_msg(this->name, GF_LOG_ERROR, 0,
+                       GD_MSG_UPDATE_STATEFILE_FAILED,
+                       "Resume Failed: Unable to update "
+                       "state_file. Error : %s",
+                       strerror(errno));
                 goto out;
-        }
-
-        ret = read (pfd, buf, 1024);
-        if (ret > 0) {
-                pid = strtol (buf, NULL, 10);
-                if (is_pause) {
-                        ret = kill (-pid, SIGSTOP);
-                        if (ret) {
-                                gf_log (this->name, GF_LOG_ERROR, "Failed"
-                                        " to pause gsyncd. Error: %s",
-                                        strerror (errno));
-                                goto out;
-                        }
-                        /*On pause force, if status is already paused
-                          do not update status again*/
-                        if (strstr (monitor_status, "Paused"))
-                                goto out;
-                        (void) strcat (monitor_status, "(Paused)");
-                        ret = glusterd_create_status_file ( master, slave,
-                                                     slave_host, slave_vol,
-                                                     monitor_status);
-                        if (ret) {
-                                gf_log (this->name, GF_LOG_ERROR,
-                                        "Unable  to update state_file."
-                                        " Error : %s", strerror (errno));
-                                /* If status cannot be updated resume back */
-                                if (kill (-pid, SIGCONT)) {
-                                        snprintf (errmsg, sizeof(errmsg),
-                                                  "Pause successful but could "
-                                                  "not update status file. "
-                                                  "Please use 'resume force' to"
-                                                  " resume back and retry pause"
-                                                  " to reflect in status");
-                                        gf_log (this->name, GF_LOG_ERROR,
-                                                "Resume back Failed. Error: %s",
-                                                 strerror (errno));
-                                        *op_errstr = gf_strdup (errmsg);
-                                }
-                                goto out;
-                        }
-                } else {
-                        token = strtok (monitor_status, "(");
-                        ret = glusterd_create_status_file (master, slave,
-                                                           slave_host,
-                                                           slave_vol, token);
-                        if (ret) {
-                                gf_log (this->name, GF_LOG_ERROR,
-                                        "Resume Failed: Unable to update "
-                                        "state_file. Error : %s",
-                                        strerror (errno));
-                                goto out;
-                        }
-                        ret = kill (-pid, SIGCONT);
-                        if (ret) {
-                                gf_log (this->name, GF_LOG_ERROR,
-                                        "Resumed Failed: Unable to send"
-                                        " SIGCONT. Error: %s",
-                                         strerror (errno));
-                                /* Process can't be resumed, update status
-                                 * back to paused. */
-                                ret = glusterd_create_status_file (master,
-                                                                   slave,
-                                                                   slave_host,
-                                                                   slave_vol,
-                                                                monitor_status);
-                                if (ret) {
-                                        snprintf (errmsg, sizeof(errmsg),
-                                                  "Resume failed!!! Status "
-                                                  "inconsistent. Please use "
-                                                  "'resume force' to resume and"
-                                                  " reach consistent state");
-                                        gf_log (this->name, GF_LOG_ERROR,
-                                                "Updating status back to paused"
-                                                " Failed. Error: %s",
-                                                strerror (errno));
-                                        *op_errstr = gf_strdup (errmsg);
-                                }
-                                goto out;
-                        }
+            }
+            ret = kill(-pid, SIGCONT);
+            if (ret) {
+                gf_msg(this->name, GF_LOG_ERROR, errno, GD_MSG_PID_KILL_FAIL,
+                       "Resumed Failed: Unable to send"
+                       " SIGCONT. Error: %s",
+                       strerror(errno));
+                /* Process can't be resumed, update status
+                 * back to paused. */
+                ret = glusterd_create_status_file(master, slave, slave_host,
+                                                  slave_vol, monitor_status);
+                if (ret) {
+                    snprintf(errmsg, sizeof(errmsg),
+                             "Resume failed!!! Status "
+                             "inconsistent. Please use "
+                             "'resume force' to resume and"
+                             " reach consistent state");
+                    gf_msg(this->name, GF_LOG_ERROR, 0,
+                           GD_MSG_STATUS_UPDATE_FAILED,
+                           "Updating status back to paused"
+                           " Failed. Error: %s",
+                           strerror(errno));
+                    *op_errstr = gf_strdup(errmsg);
                 }
+                goto out;
+            }
         }
-        ret = 0;
+    }
+    ret = 0;
 
 out:
-        sys_close (pfd);
-        return ret;
+    sys_close(pfd);
+    /* coverity[INTEGER_OVERFLOW] */
+    return ret;
 }
 
 static int
-stop_gsync (char *master, char *slave, char **msg,
-            char *conf_path, char **op_errstr,
-            gf_boolean_t is_force)
+stop_gsync(char *master, char *slave, char **msg, char *conf_path,
+           char **op_errstr, gf_boolean_t is_force)
 {
-        int32_t         ret     = 0;
-        int             pfd     = -1;
-        pid_t           pid     = 0;
-        char            pidfile[PATH_MAX] = {0,};
-        char            errmsg[PATH_MAX] = "";
-        char            buf [1024] = {0,};
-        int             i       = 0;
-        gf_boolean_t    is_template_in_use = _gf_false;
-
-        GF_ASSERT (THIS);
-        GF_ASSERT (THIS->private);
-
-        pfd = gsyncd_getpidfile (master, slave, pidfile,
-                                 conf_path, &is_template_in_use);
-        if (pfd == -2) {
-                snprintf (errmsg, sizeof(errmsg) - 1,
-                          "pid-file entry mising in config file and "
-                          "template config file.");
-                gf_log ("", GF_LOG_ERROR, "%s", errmsg);
-                *op_errstr = gf_strdup (errmsg);
-                ret = -1;
-                goto out;
-        }
-        if (gsync_status_byfd (pfd) == -1 && !is_force) {
-                gf_log ("", GF_LOG_ERROR, "gsyncd b/w %s & %s is not"
-                        " running", master, slave);
-                /* monitor gsyncd already dead */
-                goto out;
-        }
-
-        if (pfd < 0)
-                goto out;
-
-        ret = read (pfd, buf, 1024);
-        if (ret > 0) {
-                pid = strtol (buf, NULL, 10);
-                ret = kill (-pid, SIGTERM);
-                if (ret && !is_force) {
-                        gf_log ("", GF_LOG_WARNING,
-                                "failed to kill gsyncd");
-                        goto out;
-                }
-                for (i = 0; i < 20; i++) {
-                        if (gsync_status_byfd (pfd) == -1) {
-                                /* monitor gsyncd is dead but worker may
-                                 * still be alive, give some more time
-                                 * before SIGKILL (hack)
-                                 */
-                                usleep (50000);
-                                break;
-                        }
-                        usleep (50000);
-                }
-                kill (-pid, SIGKILL);
-                unlink (pidfile);
+    int32_t ret = 0;
+    int pfd = -1;
+    long pid = 0;
+    char pidfile[PATH_MAX] = {
+        0,
+    };
+    char errmsg[PATH_MAX] = "";
+    char buf[4096] = {
+        0,
+    };
+    int i = 0;
+    gf_boolean_t is_template_in_use = _gf_false;
+    xlator_t *this = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+
+    GF_ASSERT(this->private);
+
+    pfd = gsyncd_getpidfile(master, slave, pidfile, conf_path,
+                            &is_template_in_use);
+    if (pfd == -2) {
+        snprintf(errmsg, sizeof(errmsg) - 1,
+                 "pid-file entry mising in config file and "
+                 "template config file.");
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_PIDFILE_NOT_FOUND, "%s",
+               errmsg);
+        *op_errstr = gf_strdup(errmsg);
+        ret = -1;
+        goto out;
+    }
+    if (gsync_status_byfd(pfd) == -1 && !is_force) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_GSYNCD_ERROR,
+               "gsyncd b/w %s & %s is not running", master, slave);
+        /* monitor gsyncd already dead */
+        goto out;
+    }
+
+    if (pfd < 0)
+        goto out;
+
+    ret = sys_read(pfd, buf, sizeof(buf) - 1);
+    if (ret > 0) {
+        buf[ret] = '\0';
+        pid = strtol(buf, NULL, 10);
+        ret = kill(-pid, SIGTERM);
+        if (ret && !is_force) {
+            gf_msg(this->name, GF_LOG_WARNING, errno, GD_MSG_PID_KILL_FAIL,
+                   "failed to kill gsyncd");
+            goto out;
+        }
+        for (i = 0; i < 20; i++) {
+            if (gsync_status_byfd(pfd) == -1) {
+                /* monitor gsyncd is dead but worker may
+                 * still be alive, give some more time
+                 * before SIGKILL (hack)
+                 */
+                gf_nanosleep(50000 * GF_US_IN_NS);
+                break;
+            }
+            gf_nanosleep(50000 * GF_US_IN_NS);
         }
-        ret = 0;
+        kill(-pid, SIGKILL);
+        sys_unlink(pidfile);
+    }
+    ret = 0;
 
 out:
-        sys_close (pfd);
-
-        return ret;
+    sys_close(pfd);
+    /* coverity[INTEGER_OVERFLOW] */
+    return ret;
 }
 
 /*
  * glusterd_gsync_op_already_set:
- *      This funcion checks whether the op_value is same as in the
+ *      This function checks whether the op_value is same as in the
  *      gsyncd.conf file.
  *
  * RETURN VALUE:
@@ -3018,2426 +4238,2545 @@ out:
  */
 
 int
-glusterd_gsync_op_already_set (char* master, char* slave, char* conf_path,
-                                             char* op_name, char* op_value)
+glusterd_gsync_op_already_set(char *master, char *slave, char *conf_path,
+                              char *op_name, char *op_value)
 {
-        dict_t                 *confd                       = NULL;
-        char                   *op_val_buf                  = NULL;
-        int32_t                 op_val_conf                 = 0;
-        int32_t                 op_val_cli                  = 0;
-        int32_t                 ret                         = -1;
-        gf_boolean_t            is_bool                     = _gf_true;
-
-        confd = dict_new ();
-        if (!confd) {
-                gf_log ("", GF_LOG_ERROR, "Not able to create dict.");
-                return -1;
-        }
-
-        ret = glusterd_gsync_get_config (master, slave, conf_path,
-                                         confd);
-        if (ret) {
-                gf_log ("", GF_LOG_ERROR, "Unable to get configuration data"
-                            "for %s(master), %s(slave)", master, slave);
-                goto out;
-        }
-
-        ret = dict_get_param (confd, op_name, &op_val_buf);
-        if (ret) {
-                gf_log ("", GF_LOG_ERROR, "Unable to get op_value "
-                            "for %s(master), %s(slave). Please check gsync "
-                            "config file.", master, slave);
-                ret = 1;
-                goto out;
-        }
-
-        gf_log("",GF_LOG_DEBUG, "val_cli:%s  val_conf:%s",op_value,op_val_buf);
-
-        if (!strcmp(op_val_buf,"true") || !strcmp(op_val_buf,"1")
-            || !strcmp(op_val_buf,"yes")) {
-                op_val_conf = 1;
-        } else if(!strcmp(op_val_buf,"false") || !strcmp(op_val_buf,"0")
-                || !strcmp(op_val_buf,"no")) {
-                op_val_conf = 0;
+    dict_t *confd = NULL;
+    char *op_val_buf = NULL;
+    int32_t op_val_conf = 0;
+    int32_t op_val_cli = 0;
+    int32_t ret = -1;
+    gf_boolean_t is_bool = _gf_true;
+    xlator_t *this = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+
+    confd = dict_new();
+    if (!confd) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_CREATE_FAIL,
+               "Not able to create dict.");
+        return -1;
+    }
+
+    ret = glusterd_gsync_get_config(master, slave, conf_path, confd);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_GET_CONFIG_INFO_FAILED,
+               "Unable to get configuration data for %s(master), "
+               "%s(slave)",
+               master, slave);
+        goto out;
+    }
+
+    ret = dict_get_param(confd, op_name, &op_val_buf);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+               "Unable to get op_value for %s(master), %s(slave). "
+               "Please check gsync config file.",
+               master, slave);
+        ret = 1;
+        goto out;
+    }
+
+    gf_msg_debug(this->name, 0, "val_cli:%s  val_conf:%s", op_value,
+                 op_val_buf);
+
+    if (!strcmp(op_val_buf, "true") || !strcmp(op_val_buf, "1") ||
+        !strcmp(op_val_buf, "yes")) {
+        op_val_conf = 1;
+    } else if (!strcmp(op_val_buf, "false") || !strcmp(op_val_buf, "0") ||
+               !strcmp(op_val_buf, "no")) {
+        op_val_conf = 0;
+    } else {
+        is_bool = _gf_false;
+    }
+
+    if (is_bool) {
+        if (op_value && (!strcmp(op_value, "true") || !strcmp(op_value, "1") ||
+                         !strcmp(op_value, "yes"))) {
+            op_val_cli = 1;
         } else {
-                is_bool = _gf_false;
+            op_val_cli = 0;
         }
 
-        if (is_bool) {
-                if (!strcmp(op_value,"true") || !strcmp(op_value,"1")
-                    || !strcmp(op_value,"yes")) {
-                        op_val_cli = 1;
-                } else {
-                        op_val_cli = 0;
-                }
-
-                if ( op_val_cli == op_val_conf ) {
-                        ret = 0;
-                        goto out;
-                }
-        } else {
-                if (!strcmp(op_val_buf,op_value)) {
-                        ret = 0;
-                        goto out;
-                }
+        if (op_val_cli == op_val_conf) {
+            ret = 0;
+            goto out;
+        }
+    } else {
+        if (op_value && !strcmp(op_val_buf, op_value)) {
+            ret = 0;
+            goto out;
         }
+    }
 
-        ret = 1;
+    ret = 1;
 
 out:
-        dict_unref(confd);
-        return ret;
+    dict_unref(confd);
+    return ret;
 }
 
 static int
-glusterd_gsync_configure (glusterd_volinfo_t *volinfo, char *slave,
-                          char *path_list, dict_t *dict,
-                          dict_t *resp_dict, char **op_errstr)
+glusterd_gsync_configure(glusterd_volinfo_t *volinfo, char *slave,
+                         char *path_list, dict_t *dict, dict_t *resp_dict,
+                         char **op_errstr)
 {
-        int32_t         ret     = -1;
-        char            *op_name = NULL;
-        char            *op_value = NULL;
-        runner_t        runner    = {0,};
-        glusterd_conf_t *priv   = NULL;
-        char            *subop  = NULL;
-        char            *master = NULL;
-        char            *conf_path = NULL;
-        char            *slave_host = NULL;
-        char            *slave_vol = NULL;
-        struct stat      stbuf     = {0, };
-        gf_boolean_t     restart_required = _gf_true;
-        char           **resopt    = NULL;
-        gf_boolean_t     op_already_set = _gf_false;
-
-        GF_ASSERT (slave);
-        GF_ASSERT (op_errstr);
-        GF_ASSERT (dict);
-        GF_ASSERT (resp_dict);
-
-        ret = dict_get_str (dict, "subop", &subop);
-        if (ret != 0)
-                goto out;
+    int32_t ret = -1;
+    char *op_name = NULL;
+    char *op_value = NULL;
+    runner_t runner = {
+        0,
+    };
+    glusterd_conf_t *priv = NULL;
+    char *subop = NULL;
+    char *master = NULL;
+    char *conf_path = NULL;
+    char *slave_host = NULL;
+    char *slave_vol = NULL;
+    struct stat stbuf = {
+        0,
+    };
+    gf_boolean_t restart_required = _gf_true;
+    char **resopt = NULL;
+    gf_boolean_t op_already_set = _gf_false;
+    xlator_t *this = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+
+    GF_ASSERT(slave);
+    GF_ASSERT(op_errstr);
+    GF_ASSERT(dict);
+    GF_ASSERT(resp_dict);
+
+    ret = dict_get_str(dict, "subop", &subop);
+    if (ret != 0)
+        goto out;
+
+    if (strcmp(subop, "get") == 0 || strcmp(subop, "get-all") == 0) {
+        /* deferred to cli */
+        gf_msg_debug(this->name, 0, "Returning 0");
+        return 0;
+    }
 
-        if (strcmp (subop, "get") == 0 || strcmp (subop, "get-all") == 0) {
-                /* deferred to cli */
-                gf_log ("", GF_LOG_DEBUG, "Returning 0");
-                return 0;
-        }
+    ret = dict_get_str(dict, "op_name", &op_name);
+    if (ret != 0)
+        goto out;
 
-        ret = dict_get_str (dict, "op_name", &op_name);
+    if (strtail(subop, "set")) {
+        ret = dict_get_str(dict, "op_value", &op_value);
         if (ret != 0)
-                goto out;
-
-        if (strtail (subop, "set")) {
-                ret = dict_get_str (dict, "op_value", &op_value);
-                if (ret != 0)
-                        goto out;
-        }
-
-        if (THIS)
-                priv = THIS->private;
-        if (priv == NULL) {
-                gf_log ("", GF_LOG_ERROR, "priv of glusterd not present");
-                *op_errstr = gf_strdup ("glusterd defunct");
-                goto out;
+            goto out;
+    }
+
+    priv = THIS->private;
+    if (priv == NULL) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_GLUSTERD_PRIV_NOT_FOUND,
+               "priv of glusterd not present");
+        *op_errstr = gf_strdup("glusterd defunct");
+        goto out;
+    }
+
+    ret = dict_get_str(dict, "conf_path", &conf_path);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+               "Unable to fetch conf file path.");
+        goto out;
+    }
+
+    master = "";
+    runinit(&runner);
+    runner_add_args(&runner, GSYNCD_PREFIX "/gsyncd", "-c", NULL);
+    runner_argprintf(&runner, "%s", conf_path);
+    runner_argprintf(&runner, "--iprefix=%s", DATADIR);
+    if (volinfo) {
+        master = volinfo->volname;
+        runner_argprintf(&runner, ":%s", master);
+    }
+    runner_add_arg(&runner, slave);
+    runner_argprintf(&runner, "--config-%s", subop);
+    runner_add_arg(&runner, op_name);
+    if (op_value) {
+        runner_argprintf(&runner, "--value=%s", op_value);
+    }
+
+    if (strcmp(op_name, "checkpoint") != 0 && strtail(subop, "set")) {
+        ret = glusterd_gsync_op_already_set(master, slave, conf_path, op_name,
+                                            op_value);
+        if (ret == -1) {
+            gf_msg(this->name, GF_LOG_WARNING, 0, GD_MSG_GSYNCD_OP_SET_FAILED,
+                   "glusterd_gsync_op_already_set failed.");
+            gf_asprintf(op_errstr,
+                        GEOREP
+                        " config-%s failed for "
+                        "%s %s",
+                        subop, master, slave);
+            goto out;
         }
-
-        ret = dict_get_str (dict, "conf_path", &conf_path);
-        if (ret) {
-                gf_log ("", GF_LOG_ERROR,
-                        "Unable to fetch conf file path.");
-                goto out;
+        if (ret == 0) {
+            gf_msg_debug(this->name, 0, "op_value is already set");
+            op_already_set = _gf_true;
+            goto out;
         }
+    }
 
-        master = "";
-        runinit (&runner);
-        runner_add_args (&runner, GSYNCD_PREFIX"/gsyncd", "-c", NULL);
-        runner_argprintf (&runner, "%s", conf_path);
-        runner_argprintf (&runner, "--iprefix=%s", DATADIR);
-        if (volinfo) {
-                master = volinfo->volname;
-                runner_argprintf (&runner, ":%s", master);
-        }
-        runner_add_arg (&runner, slave);
-        runner_argprintf (&runner, "--config-%s", subop);
-        runner_add_arg (&runner, op_name);
-        if (op_value)
-                runner_add_arg (&runner, op_value);
-
-        if ( strcmp(op_name,"checkpoint") != 0 && strtail (subop, "set")) {
-                ret = glusterd_gsync_op_already_set(master,slave,conf_path,
-                                                           op_name,op_value);
-                if (ret == -1) {
-                        gf_log ("", GF_LOG_WARNING,
-                                "glusterd_gsync_op_already_set failed.");
-                        gf_asprintf (op_errstr, GEOREP" config-%s failed for "
-                                     "%s %s", subop, master, slave);
-                        goto out;
-                }
-                if (ret == 0) {
-                        gf_log("", GF_LOG_DEBUG, "op_value is already set");
-                        op_already_set = _gf_true;
-                        goto out;
-                }
-        }
+    synclock_unlock(&priv->big_lock);
+    ret = runner_run(&runner);
+    synclock_lock(&priv->big_lock);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_WARNING, 0, GD_MSG_GSYNCD_ERROR,
+               "gsyncd failed to %s %s option for "
+               "%s %s peers",
+               subop, op_name, master, slave);
 
-        synclock_unlock (&priv->big_lock);
-        ret = runner_run (&runner);
-        synclock_lock (&priv->big_lock);
-        if (ret) {
-                gf_log ("", GF_LOG_WARNING, "gsyncd failed to "
-                        "%s %s option for %s %s peers",
-                        subop, op_name, master, slave);
+        gf_asprintf(op_errstr, GEOREP " config-%s failed for %s %s", subop,
+                    master, slave);
 
-                gf_asprintf (op_errstr, GEOREP" config-%s failed for %s %s",
-                             subop, master, slave);
+        goto out;
+    }
 
+    if ((!strcmp(op_name, "state_file")) && (op_value)) {
+        ret = sys_lstat(op_value, &stbuf);
+        if (ret) {
+            ret = dict_get_str(dict, "slave_host", &slave_host);
+            if (ret) {
+                gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+                       "Unable to fetch slave host.");
                 goto out;
-        }
+            }
 
-        if ((!strcmp (op_name, "state_file")) && (op_value)) {
+            ret = dict_get_str(dict, "slave_vol", &slave_vol);
+            if (ret) {
+                gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+                       "Unable to fetch slave volume name.");
+                goto out;
+            }
 
-                ret = lstat (op_value, &stbuf);
-                if (ret) {
-                        ret = dict_get_str (dict, "slave_host", &slave_host);
-                        if (ret) {
-                                gf_log ("", GF_LOG_ERROR,
-                                        "Unable to fetch slave host.");
-                                goto out;
-                        }
-
-                        ret = dict_get_str (dict, "slave_vol", &slave_vol);
-                        if (ret) {
-                                gf_log ("", GF_LOG_ERROR,
-                                        "Unable to fetch slave volume name.");
-                                goto out;
-                        }
-
-                        ret = glusterd_create_status_file (volinfo->volname,
-                                                           slave, slave_host,
-                                                           slave_vol,
-                                                           "Switching Status "
-                                                           "File");
-                        if (ret || lstat (op_value, &stbuf)) {
-                                gf_log ("", GF_LOG_ERROR, "Unable to create %s"
-                                        ". Error : %s", op_value,
-                                        strerror (errno));
-                                ret = -1;
-                                goto out;
-                        }
-                }
+            ret = glusterd_create_status_file(volinfo->volname, slave,
+                                              slave_host, slave_vol,
+                                              "Switching Status "
+                                              "File");
+            if (ret || sys_lstat(op_value, &stbuf)) {
+                gf_msg(this->name, GF_LOG_ERROR, errno, GD_MSG_FILE_OP_FAILED,
+                       "Unable to "
+                       "create %s. Error : %s",
+                       op_value, strerror(errno));
+                ret = -1;
+                goto out;
+            }
         }
+    }
 
-        ret = 0;
-        gf_asprintf (op_errstr, "config-%s successful", subop);
+    ret = 0;
+    gf_asprintf(op_errstr, "config-%s successful", subop);
 
 out:
-        if (!ret && volinfo && !op_already_set) {
-            for (resopt = gsync_no_restart_opts; *resopt; resopt++) {
-                restart_required = _gf_true;
-                if (!strcmp ((*resopt), op_name)){
-                    restart_required = _gf_false;
-                    break;
-                }
+    if (!ret && volinfo && !op_already_set) {
+        for (resopt = gsync_no_restart_opts; *resopt; resopt++) {
+            restart_required = _gf_true;
+            if (!strcmp((*resopt), op_name)) {
+                restart_required = _gf_false;
+                break;
             }
+        }
 
-            if (restart_required) {
-                ret = glusterd_check_restart_gsync_session (volinfo, slave,
-                                                            resp_dict, path_list,
-                                                            conf_path, 0);
-                if (ret)
-                    *op_errstr = gf_strdup ("internal error");
-            }
+        if (restart_required) {
+            ret = glusterd_check_restart_gsync_session(
+                volinfo, slave, resp_dict, path_list, conf_path, 0);
+            if (ret)
+                *op_errstr = gf_strdup("internal error");
         }
+    }
 
-        gf_log ("", GF_LOG_DEBUG, "Returning %d", ret);
-        return ret;
+    gf_msg_debug(this->name, 0, "Returning %d", ret);
+    return ret;
 }
 
 int
-glusterd_gsync_read_frm_status (char *path, char *buf, size_t blen)
+glusterd_gsync_read_frm_status(char *path, char *buf, size_t blen)
 {
-        int                 ret = 0;
-        int           status_fd = -1;
-
-        GF_ASSERT (path);
-        GF_ASSERT (buf);
-        status_fd = open (path, O_RDONLY);
-        if (status_fd == -1) {
-                gf_log ("", GF_LOG_ERROR, "Unable to read gsyncd status"
-                        " file");
-                return -1;
-        }
-        ret = read (status_fd, buf, blen - 1);
-        if (ret > 0) {
-                size_t len = strnlen (buf, ret);
-                /* Ensure there is a NUL byte and that it's not the first.  */
-                if (len == 0 || len == blen - 1) {
-                        ret = -1;
-                } else {
-                        char *p = buf + len - 1;
-                        while (isspace (*p))
-                                *p-- = '\0';
-                }
-        } else if (ret < 0)
-                gf_log ("", GF_LOG_ERROR, "Status file of gsyncd is corrupt");
-
-        close (status_fd);
-        return ret;
+    int ret = 0;
+    int status_fd = -1;
+    xlator_t *this = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+
+    GF_ASSERT(path);
+    GF_ASSERT(buf);
+    status_fd = open(path, O_RDONLY);
+    if (status_fd == -1) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_FILE_OP_FAILED,
+               "Unable to read gsyncd status file %s", path);
+        return -1;
+    }
+    ret = sys_read(status_fd, buf, blen - 1);
+    if (ret > 0) {
+        size_t len = strnlen(buf, ret);
+        /* Ensure there is a NUL byte and that it's not the first.  */
+        if (len == 0 || len == blen - 1) {
+            ret = -1;
+        } else {
+            char *p = buf + len - 1;
+            while (isspace(*p))
+                *p-- = '\0';
+        }
+    } else if (ret == 0)
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_GSYNCD_ERROR,
+               "Status file of gsyncd is empty");
+    else /* ret < 0 */
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_GSYNCD_ERROR,
+               "Status file of gsyncd is corrupt");
+
+    sys_close(status_fd);
+    return ret;
 }
 
 static int
-dict_get_param (dict_t *dict, char *key, char **param)
+dict_get_param(dict_t *dict, char *key, char **param)
 {
-        char  *dk = NULL;
-        char   *s = NULL;
-        char    x = '\0';
-        int   ret = 0;
+    char *dk = NULL;
+    char *s = NULL;
+    char x = '\0';
+    int ret = 0;
 
-        if (dict_get_str (dict, key, param) == 0)
-                return 0;
+    if (dict_get_str(dict, key, param) == 0)
+        return 0;
 
-        dk = gf_strdup (key);
-        if (!key)
-                return -1;
+    dk = gf_strdup(key);
+    if (!dk)
+        return -1;
 
-        s = strpbrk (dk, "-_");
-        if (!s) {
-                ret = -1;
-                goto out;
-        }
-        x = (*s == '-') ? '_' : '-';
+    s = strpbrk(dk, "-_");
+    if (!s) {
+        ret = -1;
+        goto out;
+    }
+    x = (*s == '-') ? '_' : '-';
+    *s++ = x;
+    while ((s = strpbrk(s, "-_")))
         *s++ = x;
-        while ((s = strpbrk (s, "-_")))
-                *s++ = x;
 
-        ret = dict_get_str (dict, dk, param);
+    ret = dict_get_str(dict, dk, param);
 out:
-        GF_FREE (dk);
-        return ret;
+    GF_FREE(dk);
+    return ret;
 }
 
-static int
-glusterd_parse_gsync_status (char *buf, gf_gsync_status_t *sts_val)
+int
+glusterd_fetch_values_from_config(char *master, char *slave, char *confpath,
+                                  dict_t *confd, char **statefile,
+                                  char **georep_session_wrkng_dir,
+                                  char **socketfile)
 {
-        int              ret      = -1;
-        int              i      = -1;
-        int              num_of_fields = 8;
-        char            *token    = NULL;
-        char           **tokens   = NULL;
-        char           **ptr   = NULL;
-        char            *save_ptr = NULL;
-        char             na_buf[] = "N/A";
-
-        if (!buf) {
-                gf_log ("", GF_LOG_ERROR, "Empty buf");
-                goto out;
-        }
-
-        tokens = calloc (num_of_fields, sizeof (char *));
-        if (!tokens) {
-                gf_log ("", GF_LOG_ERROR, "Out of memory");
-                goto out;
-        }
-
-        ptr = tokens;
-
-        for (token = strtok_r (buf, ",", &save_ptr); token;
-             token = strtok_r (NULL, ",", &save_ptr)) {
-                *ptr = gf_strdup(token);
-                if (!*ptr) {
-                        gf_log ("", GF_LOG_ERROR, "Out of memory");
-                        goto out;
-                }
-                ptr++;
-        }
-
-        for (i = 0; i < num_of_fields; i++) {
-                token = strtok_r (tokens[i], ":", &save_ptr);
-                token = strtok_r (NULL, "\0", &save_ptr);
-                token++;
-
-                /* token NULL check */
-                if (!token && (i != 0) &&
-                    (i != 5) && (i != 7))
-                    token = na_buf;
-
-                if (i == 0) {
-                        if (!token)
-                            token = na_buf;
-                        else {
-                            token++;
-                            if (!token)
-                                token = na_buf;
-                            else
-                                token[strlen(token) - 1] = '\0';
-                        }
-                        memcpy (sts_val->slave_node, token, strlen(token));
-                }
-                if (i == 1)
-                        memcpy (sts_val->files_syncd, token, strlen(token));
-                if (i == 2)
-                        memcpy (sts_val->purges_remaining, token, strlen(token));
-                if (i == 3)
-                        memcpy (sts_val->total_files_skipped, token, strlen(token));
-                if (i == 4)
-                        memcpy (sts_val->files_remaining, token, strlen(token));
-                if (i == 5) {
-                        if (!token)
-                            token = na_buf;
-                        else {
-                            token++;
-                            if (!token)
-                                token = na_buf;
-                            else
-                                token[strlen(token) - 1] = '\0';
-                        }
-                        memcpy (sts_val->worker_status, token, strlen(token));
-                }
-                if (i == 6)
-                        memcpy (sts_val->bytes_remaining, token, strlen(token));
-                if (i == 7) {
-                        if (!token)
-                            token = na_buf;
-                        else {
-                            token++;
-                            if (!token)
-                                token = na_buf;
-                            else
-                                token[strlen(token) - 2] = '\0';
-                        }
-                        memcpy (sts_val->crawl_status, token, strlen(token));
-                }
+    int ret = 0;
+    xlator_t *this = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+
+    ret = glusterd_gsync_get_config(master, slave, confpath, confd);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_GET_CONFIG_INFO_FAILED,
+               "Unable to get configuration data for %s(master), "
+               "%s(slave)",
+               master, slave);
+        goto out;
+    }
+
+    if (statefile) {
+        ret = dict_get_param(confd, "state_file", statefile);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+                   "Unable to get state_file's name "
+                   "for %s(master), %s(slave). "
+                   "Please check gsync config file.",
+                   master, slave);
+            goto out;
+        }
+    }
+
+    if (georep_session_wrkng_dir) {
+        ret = dict_get_param(confd, "georep_session_working_dir",
+                             georep_session_wrkng_dir);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+                   "Unable to get geo-rep session's "
+                   "working directory name for %s(master), "
+                   "%s(slave). Please check gsync config file.",
+                   master, slave);
+            goto out;
         }
+    }
 
-        ret = 0;
-out:
-        for (i = 0; i< num_of_fields; i++)
-               if (tokens[i])
-                       GF_FREE(tokens[i]);
-
-        gf_log ("", GF_LOG_DEBUG, "Returning %d", ret);
-        return ret;
-}
-
-static int
-glusterd_gsync_fetch_status_extra (char *path, gf_gsync_status_t *sts_val)
-{
-        char sockpath[PATH_MAX] = {0,};
-        struct sockaddr_un   sa = {0,};
-        int                   s = -1;
-        struct pollfd       pfd = {0,};
-        int                 ret = 0;
-
-        glusterd_set_socket_filepath (path, sockpath, sizeof (sockpath));
-
-        strncpy(sa.sun_path, sockpath, sizeof(sa.sun_path));
-        if (sa.sun_path[sizeof (sa.sun_path) - 1])
-                return -1;
-        sa.sun_family = AF_UNIX;
-
-        s = socket(AF_UNIX, SOCK_STREAM, 0);
-        if (s == -1)
-                return -1;
-        ret = fcntl (s, F_GETFL);
-        if (ret != -1)
-                ret = fcntl (s, F_SETFL, ret | O_NONBLOCK);
-        if (ret == -1)
-                goto out;
-
-        ret = connect (s, (struct sockaddr *)&sa, sizeof (sa));
-        if (ret == -1)
-                goto out;
-        pfd.fd = s;
-        pfd.events = POLLIN;
-        /* we don't want to hang on gsyncd */
-        if (poll (&pfd, 1, 5000) < 1 ||
-            !(pfd.revents & POLLIN)) {
-                ret = -1;
-                goto out;
-        }
-        ret = read(s, sts_val->checkpoint_status,
-                   sizeof(sts_val->checkpoint_status));
-        /* we expect a terminating 0 byte */
-        if (ret == 0 || (ret > 0 && sts_val->checkpoint_status[ret - 1]))
-                ret = -1;
-        if (ret > 0) {
-                ret = 0;
+    if (socketfile) {
+        ret = dict_get_param(confd, "state_socket_unencoded", socketfile);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+                   "Unable to get socket file's name "
+                   "for %s(master), %s(slave). "
+                   "Please check gsync config file.",
+                   master, slave);
+            goto out;
         }
+    }
 
+    ret = 0;
 out:
-        close (s);
-        return ret;
+    gf_msg_debug(this->name, 0, "Returning %d", ret);
+    return ret;
 }
 
 int
-glusterd_fetch_values_from_config (char *master, char *slave,
-                                   char *confpath, dict_t *confd,
-                                   char **statefile,
-                                   char **georep_session_wrkng_dir,
-                                   char **socketfile)
+glusterd_read_status_file(glusterd_volinfo_t *volinfo, char *slave,
+                          char *conf_path, dict_t *dict, char *node)
 {
-        int                     ret                        = 0;
-
-        ret = glusterd_gsync_get_config (master, slave, confpath,
-                                         confd);
-        if (ret) {
-                gf_log ("", GF_LOG_ERROR, "Unable to get configuration data"
-                        "for %s(master), %s(slave)", master, slave);
-                goto out;
-        }
+    char temp_conf_path[PATH_MAX] = "";
+    char *working_conf_path = NULL;
+    char *georep_session_wrkng_dir = NULL;
+    char *master = NULL;
+    char sts_val_name[1024] = "";
+    char monitor_status[NAME_MAX] = "";
+    char *statefile = NULL;
+    char *socketfile = NULL;
+    dict_t *confd = NULL;
+    char *slavekey = NULL;
+    char *slaveentry = NULL;
+    char *slaveuser = NULL;
+    char *saveptr = NULL;
+    char *temp = NULL;
+    char *temp_inp = NULL;
+    char *brick_host_uuid = NULL;
+    int brick_host_uuid_length = 0;
+    int gsync_count = 0;
+    int ret = 0;
+    glusterd_brickinfo_t *brickinfo = NULL;
+    gf_gsync_status_t *sts_val = NULL;
+    gf_boolean_t is_template_in_use = _gf_false;
+    glusterd_conf_t *priv = NULL;
+    struct stat stbuf = {
+        0,
+    };
+    xlator_t *this = NULL;
+    int32_t len = 0;
+
+    this = THIS;
+    GF_ASSERT(this);
+
+    GF_ASSERT(this->private);
+    GF_ASSERT(volinfo);
+    GF_ASSERT(conf_path);
+
+    master = volinfo->volname;
+
+    confd = dict_new();
+    if (!confd) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_CREATE_FAIL,
+               "Not able to create dict.");
+        return -1;
+    }
 
-        if (statefile) {
-                ret = dict_get_param (confd, "state_file", statefile);
-                if (ret) {
-                        gf_log ("", GF_LOG_ERROR,
-                                "Unable to get state_file's name "
-                                "for %s(master), %s(slave). "
-                                "Please check gsync config file.",
-                                master, slave);
-                        goto out;
-                }
-        }
+    priv = THIS->private;
 
-        if (georep_session_wrkng_dir) {
-                ret = dict_get_param (confd, "georep_session_working_dir",
-                                      georep_session_wrkng_dir);
-                if (ret) {
-                        gf_log ("", GF_LOG_ERROR,
-                                "Unable to get geo-rep session's "
-                                "working directory name for %s(master), "
-                                "%s(slave). Please check gsync config file.",
-                                master, slave);
-                        goto out;
-                }
-        }
+    len = snprintf(temp_conf_path, sizeof(temp_conf_path),
+                   "%s/" GSYNC_CONF_TEMPLATE, priv->workdir);
+    if ((len < 0) || (len >= sizeof(temp_conf_path))) {
+        return -1;
+    }
+
+    ret = sys_lstat(conf_path, &stbuf);
+    if (!ret) {
+        gf_msg(this->name, GF_LOG_INFO, 0, GD_MSG_CONFIG_INFO,
+               "Using passed config template(%s).", conf_path);
+        working_conf_path = conf_path;
+    } else {
+        gf_msg(this->name, GF_LOG_WARNING, ENOENT, GD_MSG_FILE_OP_FAILED,
+               "Config file (%s) missing. Looking for template "
+               "config file (%s)",
+               conf_path, temp_conf_path);
+        ret = sys_lstat(temp_conf_path, &stbuf);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, ENOENT, GD_MSG_FILE_OP_FAILED,
+                   "Template "
+                   "config file (%s) missing.",
+                   temp_conf_path);
+            goto out;
+        }
+        gf_msg(this->name, GF_LOG_INFO, 0, GD_MSG_DEFAULT_TEMP_CONFIG,
+               "Using default config template(%s).", temp_conf_path);
+        working_conf_path = temp_conf_path;
+        is_template_in_use = _gf_true;
+    }
 
-        if (socketfile) {
-                ret = dict_get_param (confd, "state_socket_unencoded",
-                                      socketfile);
-                if (ret) {
-                        gf_log ("", GF_LOG_ERROR,
-                                "Unable to get socket file's name "
-                                "for %s(master), %s(slave). "
-                                "Please check gsync config file.",
-                                master, slave);
-                        goto out;
-                }
+fetch_data:
+    ret = glusterd_fetch_values_from_config(
+        master, slave, working_conf_path, confd, &statefile,
+        &georep_session_wrkng_dir, &socketfile);
+    if (ret) {
+        if (is_template_in_use == _gf_false) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_FETCH_CONFIG_VAL_FAILED,
+                   "Unable to fetch config values "
+                   "for %s(master), %s(slave). "
+                   "Trying default config template",
+                   master, slave);
+            working_conf_path = temp_conf_path;
+            is_template_in_use = _gf_true;
+            goto fetch_data;
+        } else {
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_FETCH_CONFIG_VAL_FAILED,
+                   "Unable to "
+                   "fetch config values for %s(master), "
+                   "%s(slave)",
+                   master, slave);
+            goto out;
+        }
+    }
+
+    ret = glusterd_gsync_read_frm_status(statefile, monitor_status,
+                                         sizeof(monitor_status));
+    if (ret <= 0) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_STAT_FILE_READ_FAILED,
+               "Unable to read the status file for %s(master), "
+               "%s(slave) statefile: %s",
+               master, slave, statefile);
+        snprintf(monitor_status, sizeof(monitor_status), "defunct");
+    }
+
+    ret = dict_get_int32(dict, "gsync-count", &gsync_count);
+    if (ret)
+        gsync_count = 0;
+
+    cds_list_for_each_entry(brickinfo, &volinfo->bricks, brick_list)
+    {
+        if (gf_uuid_compare(brickinfo->uuid, MY_UUID))
+            continue;
+
+        sts_val = GF_CALLOC(1, sizeof(gf_gsync_status_t),
+                            gf_common_mt_gsync_status_t);
+        if (!sts_val) {
+            gf_msg(this->name, GF_LOG_ERROR, ENOMEM, GD_MSG_NO_MEMORY,
+                   "Out Of Memory");
+            goto out;
+        }
+
+        /* Slave Key */
+        ret = glusterd_get_slave(volinfo, slave, &slavekey);
+        if (ret < 0) {
+            GF_FREE(sts_val);
+            goto out;
         }
+        memcpy(sts_val->slavekey, slavekey, strlen(slavekey));
+        sts_val->slavekey[strlen(slavekey)] = '\0';
 
-        ret = 0;
-out:
-        gf_log ("", GF_LOG_DEBUG, "Returning %d", ret);
-        return ret;
-}
+        /* Master Volume */
+        memcpy(sts_val->master, master, strlen(master));
+        sts_val->master[strlen(master)] = '\0';
 
-int
-glusterd_read_status_file (glusterd_volinfo_t *volinfo, char *slave,
-                           char *conf_path, dict_t *dict, char *node)
-{
-        char                    brick_state_file[PATH_MAX] = "";
-        char                    brick_path[PATH_MAX]       = "";
-        char                    temp_conf_path[PATH_MAX]   = "";
-        char                   *working_conf_path          = NULL;
-        char                   *georep_session_wrkng_dir   = NULL;
-        char                   *master                     = NULL;
-        char                    tmp[1024]                  = "";
-        char                    sts_val_name[1024]         = "";
-        char                    monitor_status[NAME_MAX]   = "";
-        char                   *statefile                  = NULL;
-        char                   *socketfile                 = NULL;
-        dict_t                 *confd                      = NULL;
-        char                   *slavekey                   = NULL;
-        char                   *slaveentry                 = NULL;
-        char                   *slaveuser                  = NULL;
-        char                   *saveptr                    = NULL;
-        char                   *temp                       = NULL;
-        char                   *temp_inp                   = NULL;
-        char                   *brick_host_uuid            = NULL;
-        int                     brick_host_uuid_length     = 0;
-        int                     gsync_count                = 0;
-        int                     i                          = 0;
-        int                     ret                        = 0;
-        glusterd_brickinfo_t   *brickinfo                  = NULL;
-        gf_gsync_status_t      *sts_val                    = NULL;
-        gf_boolean_t            is_template_in_use         = _gf_false;
-        glusterd_conf_t        *priv                       = NULL;
-        struct stat             stbuf                      = {0,};
-
-        GF_ASSERT (THIS);
-        GF_ASSERT (THIS->private);
-        GF_ASSERT (volinfo);
-        GF_ASSERT (conf_path);
+        /* Master Brick Node */
+        memcpy(sts_val->node, brickinfo->hostname, strlen(brickinfo->hostname));
+        sts_val->node[strlen(brickinfo->hostname)] = '\0';
 
-        master = volinfo->volname;
+        /* Master Brick Path */
+        memcpy(sts_val->brick, brickinfo->path, strlen(brickinfo->path));
+        sts_val->brick[strlen(brickinfo->path)] = '\0';
 
-        confd = dict_new ();
-        if (!confd) {
-                gf_log ("", GF_LOG_ERROR, "Not able to create dict.");
-                return -1;
-        }
+        /* Brick Host UUID */
+        brick_host_uuid = uuid_utoa(brickinfo->uuid);
+        brick_host_uuid_length = strlen(brick_host_uuid);
+        memcpy(sts_val->brick_host_uuid, brick_host_uuid,
+               brick_host_uuid_length);
+        sts_val->brick_host_uuid[brick_host_uuid_length] = '\0';
 
-        priv = THIS->private;
+        /* Slave */
+        memcpy(sts_val->slave, slave, strlen(slave));
+        sts_val->slave[strlen(slave)] = '\0';
 
-        snprintf (temp_conf_path, sizeof(temp_conf_path) - 1,
-                  "%s/"GSYNC_CONF_TEMPLATE, priv->workdir);
+        snprintf(sts_val->slave_node, sizeof(sts_val->slave_node), "N/A");
 
-        ret = lstat (conf_path, &stbuf);
-        if (!ret) {
-                gf_log ("", GF_LOG_INFO, "Using passed config template(%s).",
-                        conf_path);
-                working_conf_path = conf_path;
-        } else {
-                gf_log ("", GF_LOG_WARNING, "Config file (%s) missing. "
-                        "Looking for template config file (%s)",
-                        conf_path, temp_conf_path);
-                ret = lstat (temp_conf_path, &stbuf);
-                if (ret) {
-                        gf_log ("", GF_LOG_ERROR,
-                                "Template config file (%s) missing.",
-                                temp_conf_path);
-                        goto out;
-                }
-                gf_log ("", GF_LOG_INFO, "Using default config template(%s).",
-                        temp_conf_path);
-                working_conf_path = temp_conf_path;
-                is_template_in_use = _gf_true;
-        }
+        snprintf(sts_val->worker_status, sizeof(sts_val->worker_status), "N/A");
 
-fetch_data:
-        ret = glusterd_fetch_values_from_config (master, slave,
-                                                 working_conf_path,
-                                                 confd,
-                                                 &statefile,
-                                                 &georep_session_wrkng_dir,
-                                                 &socketfile);
-        if (ret) {
-                if (is_template_in_use == _gf_false) {
-                        gf_log ("", GF_LOG_ERROR,
-                                "Unable to fetch config values "
-                                "for %s(master), %s(slave). "
-                                "Trying default config template",
-                                master, slave);
-                        working_conf_path = temp_conf_path;
-                        is_template_in_use = _gf_true;
-                        goto fetch_data;
-                } else {
-                        gf_log ("", GF_LOG_ERROR,
-                                "Unable to fetch config values "
-                                "for %s(master), %s(slave)",
-                                master, slave);
-                        goto out;
-                }
-        }
+        snprintf(sts_val->crawl_status, sizeof(sts_val->crawl_status), "N/A");
 
-        ret = glusterd_gsync_read_frm_status (statefile, monitor_status,
-                                              sizeof (monitor_status));
-        if (ret <= 0) {
-                gf_log ("", GF_LOG_ERROR, "Unable to read the status"
-                        "file for %s(master), %s(slave)", master, slave);
-                strncpy (monitor_status, "defunct", sizeof (monitor_status));
-        }
+        snprintf(sts_val->last_synced, sizeof(sts_val->last_synced), "N/A");
 
-        ret = dict_get_int32 (dict, "gsync-count", &gsync_count);
-        if (ret)
-                gsync_count = 0;
+        snprintf(sts_val->last_synced_utc, sizeof(sts_val->last_synced_utc),
+                 "N/A");
 
-        cds_list_for_each_entry (brickinfo, &volinfo->bricks, brick_list) {
-                if (gf_uuid_compare (brickinfo->uuid, MY_UUID))
-                        continue;
+        snprintf(sts_val->entry, sizeof(sts_val->entry), "N/A");
 
-                sts_val = GF_CALLOC (1, sizeof(gf_gsync_status_t),
-                                     gf_common_mt_gsync_status_t);
-                if (!sts_val) {
-                        gf_log ("", GF_LOG_ERROR, "Out Of Memory");
-                        goto out;
-                }
+        snprintf(sts_val->data, sizeof(sts_val->data), "N/A");
 
-                /* Creating the brick state file's path */
-                memset(brick_state_file, '\0', PATH_MAX);
-                memcpy (brick_path, brickinfo->path, PATH_MAX - 1);
-                for (i = 0; i < strlen(brick_path) - 1; i++)
-                        if (brick_path[i] == '/')
-                                brick_path[i] = '_';
-                ret = snprintf(brick_state_file, PATH_MAX - 1, "%s%s.status",
-                               georep_session_wrkng_dir, brick_path);
-                brick_state_file[ret] = '\0';
-
-                gf_log ("", GF_LOG_DEBUG, "brick_state_file = %s", brick_state_file);
-
-                memset (tmp, '\0', sizeof(tmp));
-
-                ret = glusterd_gsync_read_frm_status (brick_state_file,
-                                                      tmp, sizeof (tmp));
-                if (ret <= 0) {
-                        gf_log ("", GF_LOG_ERROR, "Unable to read the status"
-                                "file for %s brick for  %s(master), %s(slave) "
-                                "session", brickinfo->path, master, slave);
-                        memcpy (sts_val->slave_node, slave, strlen(slave));
-                        sts_val->slave_node[strlen(slave)] = '\0';
-                        ret = snprintf (sts_val->worker_status, sizeof(sts_val->worker_status), "N/A");
-                        sts_val->worker_status[ret] = '\0';
-                        ret = snprintf (sts_val->checkpoint_status, sizeof(sts_val->checkpoint_status), "N/A");
-                        sts_val->checkpoint_status[ret] = '\0';
-                        ret = snprintf (sts_val->crawl_status, sizeof(sts_val->crawl_status), "N/A");
-                        sts_val->crawl_status[ret] = '\0';
-                        ret = snprintf (sts_val->files_syncd, sizeof(sts_val->files_syncd), "N/A");
-                        sts_val->files_syncd[ret] = '\0';
-                        ret = snprintf (sts_val->purges_remaining, sizeof(sts_val->purges_remaining), "N/A");
-                        sts_val->purges_remaining[ret] = '\0';
-                        ret = snprintf (sts_val->total_files_skipped, sizeof(sts_val->total_files_skipped), "N/A");
-                        sts_val->total_files_skipped[ret] = '\0';
-                        ret = snprintf (sts_val->files_remaining, sizeof(sts_val->files_remaining), "N/A");
-                        sts_val->files_remaining[ret] = '\0';
-                        ret = snprintf (sts_val->bytes_remaining, sizeof(sts_val->bytes_remaining), "N/A");
-                        sts_val->bytes_remaining[ret] = '\0';
-                        goto store_status;
-                }
+        snprintf(sts_val->meta, sizeof(sts_val->meta), "N/A");
 
-                ret = glusterd_gsync_fetch_status_extra (socketfile, sts_val);
-                if (ret || strlen(sts_val->checkpoint_status) == 0) {
-                        gf_log ("", GF_LOG_DEBUG, "No checkpoint status"
-                                "for %s(master), %s(slave)", master, slave);
-                        ret = snprintf (sts_val->checkpoint_status, sizeof(sts_val->checkpoint_status), "N/A");
-                        sts_val->checkpoint_status[ret] = '\0';
-                }
+        snprintf(sts_val->failures, sizeof(sts_val->failures), "N/A");
 
-                ret = glusterd_parse_gsync_status (tmp, sts_val);
-                if (ret) {
-                        gf_log ("", GF_LOG_ERROR,
-                                "Unable to parse the gsync status for %s",
-                                brickinfo->path);
-                        memcpy (sts_val->slave_node, slave, strlen(slave));
-                        sts_val->slave_node[strlen(slave)] = '\0';
-                        ret = snprintf (sts_val->worker_status, sizeof(sts_val->worker_status), "N/A");
-                        sts_val->worker_status[ret] = '\0';
-                        ret = snprintf (sts_val->checkpoint_status, sizeof(sts_val->checkpoint_status), "N/A");
-                        sts_val->checkpoint_status[ret] = '\0';
-                        ret = snprintf (sts_val->crawl_status, sizeof(sts_val->crawl_status), "N/A");
-                        sts_val->crawl_status[ret] = '\0';
-                        ret = snprintf (sts_val->files_syncd, sizeof(sts_val->files_syncd), "N/A");
-                        sts_val->files_syncd[ret] = '\0';
-                        ret = snprintf (sts_val->purges_remaining, sizeof(sts_val->purges_remaining), "N/A");
-                        sts_val->purges_remaining[ret] = '\0';
-                        ret = snprintf (sts_val->total_files_skipped, sizeof(sts_val->total_files_skipped), "N/A");
-                        sts_val->total_files_skipped[ret] = '\0';
-                        ret = snprintf (sts_val->files_remaining, sizeof(sts_val->files_remaining), "N/A");
-                        sts_val->files_remaining[ret] = '\0';
-                        ret = snprintf (sts_val->bytes_remaining, sizeof(sts_val->bytes_remaining), "N/A");
-                        sts_val->bytes_remaining[ret] = '\0';
-                }
+        snprintf(sts_val->checkpoint_time, sizeof(sts_val->checkpoint_time),
+                 "N/A");
 
-store_status:
-                if ((strcmp (monitor_status, "Stable"))) {
-                        memcpy (sts_val->worker_status, monitor_status, strlen(monitor_status));
-                        sts_val->worker_status[strlen(monitor_status)] = '\0';
-                        ret = snprintf (sts_val->crawl_status, sizeof(sts_val->crawl_status), "N/A");
-                        sts_val->crawl_status[ret] = '\0';
-                        ret = snprintf (sts_val->checkpoint_status, sizeof(sts_val->checkpoint_status), "N/A");
-                        sts_val->checkpoint_status[ret] = '\0';
-                }
+        snprintf(sts_val->checkpoint_time_utc,
+                 sizeof(sts_val->checkpoint_time_utc), "N/A");
 
-                if (is_template_in_use) {
-                        ret = snprintf (sts_val->worker_status,
-                                        sizeof(sts_val->worker_status),
-                                        "Config Corrupted");
-                        sts_val->worker_status[ret] = '\0';
-                }
+        snprintf(sts_val->checkpoint_completed,
+                 sizeof(sts_val->checkpoint_completed), "N/A");
 
-                if (strcmp (sts_val->worker_status, "Active")) {
-                        ret = snprintf (sts_val->checkpoint_status, sizeof(sts_val->checkpoint_status), "N/A");
-                        sts_val->checkpoint_status[ret] = '\0';
-                        ret = snprintf (sts_val->crawl_status, sizeof(sts_val->crawl_status), "N/A");
-                        sts_val->crawl_status[ret] = '\0';
-                }
+        snprintf(sts_val->checkpoint_completion_time,
+                 sizeof(sts_val->checkpoint_completion_time), "N/A");
 
-                if (!strcmp (sts_val->slave_node, "N/A")) {
-                        memcpy (sts_val->slave_node, slave, strlen(slave));
-                        sts_val->slave_node[strlen(slave)] = '\0';
-                }
+        snprintf(sts_val->checkpoint_completion_time_utc,
+                 sizeof(sts_val->checkpoint_completion_time_utc), "N/A");
 
-                memcpy (sts_val->node, node, strlen(node));
-                sts_val->node[strlen(node)] = '\0';
-                memcpy (sts_val->brick, brickinfo->path, strlen(brickinfo->path));
-                sts_val->brick[strlen(brickinfo->path)] = '\0';
+        /* Get all the other values from Gsyncd */
+        ret = glusterd_gsync_get_status(master, slave, conf_path,
+                                        brickinfo->path, sts_val);
 
-                ret = glusterd_get_slave (volinfo, slave, &slavekey);
-                if (ret < 0) {
-                        GF_FREE (sts_val);
-                        goto out;
-                }
-                memcpy (sts_val->slavekey, slavekey, strlen(slavekey));
-                sts_val->slavekey[strlen(slavekey)] = '\0';
-
-                brick_host_uuid = uuid_utoa(brickinfo->uuid);
-                brick_host_uuid_length = strlen (brick_host_uuid);
-                memcpy (sts_val->brick_host_uuid, brick_host_uuid,
-                        brick_host_uuid_length);
-                sts_val->brick_host_uuid[brick_host_uuid_length] = '\0';
-
-                memcpy (sts_val->master, master, strlen(master));
-                sts_val->master[strlen(master)] = '\0';
-
-                ret = dict_get_str (volinfo->gsync_slaves, slavekey,
-                                    &slaveentry);
-                if (ret < 0) {
-                        GF_FREE (sts_val);
-                        goto out;
-                }
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_GET_STATUS_DATA_FAIL,
+                   "Unable to get status data "
+                   "for %s(master), %s(slave), %s(brick)",
+                   master, slave, brickinfo->path);
+            ret = -1;
+            goto out;
+        }
 
+        if (is_template_in_use) {
+            snprintf(sts_val->worker_status, sizeof(sts_val->worker_status),
+                     "Config Corrupted");
+        }
 
-                memcpy (sts_val->session_slave, slaveentry,
-                        strlen(slaveentry));
-                sts_val->session_slave[strlen(slaveentry)] = '\0';
+        ret = dict_get_str(volinfo->gsync_slaves, slavekey, &slaveentry);
+        if (ret < 0) {
+            GF_FREE(sts_val);
+            goto out;
+        }
 
-                temp_inp = gf_strdup(slaveentry);
-                if (!temp_inp)
-                        goto out;
+        memcpy(sts_val->session_slave, slaveentry, strlen(slaveentry));
+        sts_val->session_slave[strlen(slaveentry)] = '\0';
 
-                if (strstr(temp_inp, "@") == NULL) {
-                        slaveuser = "root";
-                } else {
-                        temp = strtok_r(temp_inp, "//", &saveptr);
-                        temp = strtok_r(NULL, "/", &saveptr);
-                        slaveuser = strtok_r(temp, "@", &saveptr);
-                }
-                memcpy (sts_val->slave_user, slaveuser,
-                        strlen(slaveuser));
-                sts_val->slave_user[strlen(slaveuser)] = '\0';
+        temp_inp = gf_strdup(slaveentry);
+        if (!temp_inp)
+            goto out;
 
-                snprintf (sts_val_name, sizeof (sts_val_name), "status_value%d", gsync_count);
-                ret = dict_set_bin (dict, sts_val_name, sts_val, sizeof(gf_gsync_status_t));
-                if (ret) {
-                        GF_FREE (sts_val);
-                        goto out;
-                }
+        if (strstr(temp_inp, "@") == NULL) {
+            slaveuser = "root";
+        } else {
+            temp = strtok_r(temp_inp, "//", &saveptr);
+            temp = strtok_r(NULL, "/", &saveptr);
+            slaveuser = strtok_r(temp, "@", &saveptr);
+        }
+        memcpy(sts_val->slave_user, slaveuser, strlen(slaveuser));
+        sts_val->slave_user[strlen(slaveuser)] = '\0';
 
-                gsync_count++;
-                sts_val = NULL;
+        snprintf(sts_val_name, sizeof(sts_val_name), "status_value%d",
+                 gsync_count);
+        ret = dict_set_bin(dict, sts_val_name, sts_val,
+                           sizeof(gf_gsync_status_t));
+        if (ret) {
+            GF_FREE(sts_val);
+            goto out;
         }
 
-        ret = dict_set_int32 (dict, "gsync-count", gsync_count);
-        if (ret)
-                goto out;
+        gsync_count++;
+        sts_val = NULL;
+    }
+
+    ret = dict_set_int32(dict, "gsync-count", gsync_count);
+    if (ret)
+        goto out;
 
 out:
-        GF_FREE (temp_inp);
-        dict_unref (confd);
+    GF_FREE(temp_inp);
+    dict_unref(confd);
 
-        return 0;
+    return 0;
 }
 
 int
-glusterd_check_restart_gsync_session (glusterd_volinfo_t *volinfo, char *slave,
-                                      dict_t *resp_dict, char *path_list,
-                                      char *conf_path, gf_boolean_t is_force)
+glusterd_check_restart_gsync_session(glusterd_volinfo_t *volinfo, char *slave,
+                                     dict_t *resp_dict, char *path_list,
+                                     char *conf_path, gf_boolean_t is_force)
 {
-
-        int                    ret              = 0;
-        glusterd_conf_t        *priv            = NULL;
-        char                   *status_msg      = NULL;
-        gf_boolean_t           is_running       = _gf_false;
-        char                   *op_errstr       = NULL;
-        char                   *key             = NULL;
-        xlator_t               *this            = NULL;
-
-        GF_ASSERT (volinfo);
-        GF_ASSERT (slave);
-        this = THIS;
-        GF_ASSERT (this);
-        priv = this->private;
-        GF_ASSERT (priv);
-
-        key  = slave;
-
-        ret = glusterd_check_gsync_running_local (volinfo->volname,
-                                                  slave, conf_path,
-                                                  &is_running);
-        if (!ret && (_gf_true != is_running))
-                /* gsynd not running, nothing to do */
+    int ret = 0;
+    glusterd_conf_t *priv = NULL;
+    char *status_msg = NULL;
+    gf_boolean_t is_running = _gf_false;
+    char *op_errstr = NULL;
+    char *key = NULL;
+    xlator_t *this = NULL;
+
+    GF_ASSERT(volinfo);
+    GF_ASSERT(slave);
+    this = THIS;
+    GF_ASSERT(this);
+    priv = this->private;
+    GF_ASSERT(priv);
+
+    key = slave;
+
+    ret = glusterd_check_gsync_running_local(volinfo->volname, slave, conf_path,
+                                             &is_running);
+    if (!ret && (_gf_true != is_running))
+        /* gsynd not running, nothing to do */
+        goto out;
+
+    ret = stop_gsync(volinfo->volname, slave, &status_msg, conf_path,
+                     &op_errstr, is_force);
+    if (ret == 0 && status_msg)
+        ret = dict_set_str(resp_dict, "gsync-status", status_msg);
+    if (ret == 0) {
+        dict_del(volinfo->gsync_active_slaves, key);
+        ret = glusterd_start_gsync(volinfo, slave, path_list, conf_path,
+                                   uuid_utoa(MY_UUID), NULL, _gf_false);
+        if (!ret) {
+            /* Add slave to the dict indicating geo-rep session is
+             * running.*/
+            ret = dict_set_dynstr_with_alloc(volinfo->gsync_active_slaves, key,
+                                             "running");
+            if (ret) {
+                gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+                       "Unable to set"
+                       " key:%s value:running in dict. But "
+                       "the config succeeded.",
+                       key);
                 goto out;
-
-        ret = stop_gsync (volinfo->volname, slave, &status_msg,
-                          conf_path, &op_errstr,
-                          is_force);
-        if (ret == 0 && status_msg)
-                ret = dict_set_str (resp_dict, "gsync-status",
-                                    status_msg);
-        if (ret == 0) {
-                dict_del (volinfo->gsync_active_slaves, key);
-                ret = glusterd_start_gsync (volinfo, slave, path_list,
-                                            conf_path, uuid_utoa(MY_UUID),
-                                            NULL, _gf_false);
-                if (!ret) {
-                        /* Add slave to the dict indicating geo-rep session is
-                         * running.*/
-                        ret = dict_set_dynstr_with_alloc (
-                                              volinfo->gsync_active_slaves,
-                                              key, "running");
-                        if (ret) {
-                              gf_log (this->name, GF_LOG_ERROR, "Unable to set "
-                                      "key:%s value:running in dict. But the "
-                                      "config succeeded.", key);
-                              goto out;
-                        }
-                }
+            }
         }
+    }
 
- out:
-        gf_log (this->name, GF_LOG_DEBUG, "Returning %d", ret);
-        return ret;
+out:
+    gf_msg_debug(this->name, 0, "Returning %d", ret);
+    if (op_errstr)
+        GF_FREE(op_errstr);
+    return ret;
 }
 
 static int32_t
-glusterd_marker_changelog_create_volfile (glusterd_volinfo_t *volinfo)
+glusterd_marker_changelog_create_volfile(glusterd_volinfo_t *volinfo)
 {
-        int32_t          ret     = 0;
-
-        ret = glusterd_create_volfiles_and_notify_services (volinfo);
-        if (ret) {
-                gf_log ("", GF_LOG_ERROR, "Unable to create volfile"
-                        " for setting of marker while '"GEOREP" start'");
-                ret = -1;
-                goto out;
-        }
-
-        ret = glusterd_store_volinfo (volinfo, GLUSTERD_VOLINFO_VER_AC_INCREMENT);
-        if (ret)
-                goto out;
-
-        if (GLUSTERD_STATUS_STARTED == volinfo->status)
-                ret = glusterd_svcs_manager (volinfo);
-        ret = 0;
+    int32_t ret = 0;
+    xlator_t *this = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+
+    ret = glusterd_create_volfiles_and_notify_services(volinfo);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_VOLFILE_CREATE_FAIL,
+               "Unable to create volfile for setting of marker "
+               "while '" GEOREP " start'");
+        ret = -1;
+        goto out;
+    }
+
+    ret = glusterd_store_volinfo(volinfo, GLUSTERD_VOLINFO_VER_AC_INCREMENT);
+    if (ret)
+        goto out;
+
+    if (GLUSTERD_STATUS_STARTED == volinfo->status) {
+        ret = glusterd_svcs_manager(volinfo);
+        goto out;
+    }
+    ret = 0;
 out:
-        return ret;
+    return ret;
 }
 
 static int
-glusterd_set_gsync_knob (glusterd_volinfo_t *volinfo, char *key, int *vc)
+glusterd_set_gsync_knob(glusterd_volinfo_t *volinfo, char *key, int *vc)
 {
-        int   ret          = -1;
-        int   conf_enabled = _gf_false;
+    int ret = -1;
+    int conf_enabled = _gf_false;
+    xlator_t *this = NULL;
 
-        GF_ASSERT (THIS);
-        GF_ASSERT (THIS->private);
+    this = THIS;
+    GF_ASSERT(this);
 
-        conf_enabled = glusterd_volinfo_get_boolean (volinfo, key);
-        if (conf_enabled == -1) {
-                gf_log ("", GF_LOG_ERROR,
-                        "failed to get key %s from volinfo", key);
-                goto out;
-        }
+    GF_ASSERT(this->private);
 
-        ret = 0;
-        if (conf_enabled == _gf_false) {
-                *vc = 1;
-                ret = glusterd_gsync_volinfo_dict_set (volinfo,
-                                                       key, "on");
-        }
+    conf_enabled = glusterd_volinfo_get_boolean(volinfo, key);
+    if (conf_enabled == -1) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_GET_KEY_FAILED,
+               "failed to get key %s from volinfo", key);
+        goto out;
+    }
 
- out:
-        gf_log ("", GF_LOG_DEBUG, "Returning %d", ret);
-        return ret;
+    ret = 0;
+    if (conf_enabled == _gf_false) {
+        *vc = 1;
+        ret = glusterd_gsync_volinfo_dict_set(volinfo, key, "on");
+    }
+
+out:
+    gf_msg_debug(this->name, 0, "Returning %d", ret);
+    return ret;
 }
 
 static int
-glusterd_set_gsync_confs (glusterd_volinfo_t *volinfo)
+glusterd_set_gsync_confs(glusterd_volinfo_t *volinfo)
 {
-        int ret             = -1;
-        int volfile_changed = 0;
+    int ret = -1;
+    int volfile_changed = 0;
 
-        ret = glusterd_set_gsync_knob (volinfo,
-                                       VKEY_MARKER_XTIME, &volfile_changed);
-        if (ret)
-                goto out;
+    ret = glusterd_set_gsync_knob(volinfo, VKEY_MARKER_XTIME, &volfile_changed);
+    if (ret)
+        goto out;
 
-        /**
-         * enable ignore-pid-check blindly as it could be needed for
-         * cascading setups.
-         */
-        ret = glusterd_set_gsync_knob (volinfo, VKEY_MARKER_XTIME_FORCE,
-                                       &volfile_changed);
-        if (ret)
-                goto out;
+    /**
+     * enable ignore-pid-check blindly as it could be needed for
+     * cascading setups.
+     */
+    ret = glusterd_set_gsync_knob(volinfo, VKEY_MARKER_XTIME_FORCE,
+                                  &volfile_changed);
+    if (ret)
+        goto out;
 
-        ret = glusterd_set_gsync_knob (volinfo,
-                                       VKEY_CHANGELOG, &volfile_changed);
-        if (ret)
-                goto out;
+    ret = glusterd_set_gsync_knob(volinfo, VKEY_CHANGELOG, &volfile_changed);
+    if (ret)
+        goto out;
 
-        if (volfile_changed)
-                ret = glusterd_marker_changelog_create_volfile (volinfo);
+    if (volfile_changed)
+        ret = glusterd_marker_changelog_create_volfile(volinfo);
 
- out:
-        return ret;
+out:
+    return ret;
 }
 
 static int
-glusterd_get_gsync_status_mst_slv (glusterd_volinfo_t *volinfo,
-                                   char *slave, char *conf_path,
-                                   dict_t *rsp_dict, char *node)
+glusterd_get_gsync_status_mst_slv(glusterd_volinfo_t *volinfo, char *slave,
+                                  char *conf_path, dict_t *rsp_dict, char *node)
 {
-        char              *statefile = NULL;
-        uuid_t             uuid = {0, };
-        glusterd_conf_t    *priv = NULL;
-        int                ret = 0;
-        gf_boolean_t       is_template_in_use = _gf_false;
-        struct stat        stbuf = {0, };
-
-        GF_ASSERT (volinfo);
-        GF_ASSERT (slave);
-        GF_ASSERT (THIS);
-        GF_ASSERT (THIS->private);
-
-        priv = THIS->private;
-
-        ret = glusterd_gsync_get_uuid (slave, volinfo, uuid);
+    char *statefile = NULL;
+    uuid_t uuid = {
+        0,
+    };
+    int ret = 0;
+    gf_boolean_t is_template_in_use = _gf_false;
+    struct stat stbuf = {
+        0,
+    };
+    xlator_t *this = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+
+    GF_ASSERT(volinfo);
+    GF_ASSERT(slave);
+    GF_ASSERT(this->private);
+
+    ret = glusterd_gsync_get_uuid(slave, volinfo, uuid);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_INFO, 0, GD_MSG_SESSION_INACTIVE,
+               "geo-replication status %s %s : session is not "
+               "active",
+               volinfo->volname, slave);
+
+        ret = glusterd_get_statefile_name(volinfo, slave, conf_path, &statefile,
+                                          &is_template_in_use);
         if (ret) {
-                gf_log ("", GF_LOG_INFO, "geo-replication status %s %s :"
-                        "session is not active", volinfo->volname, slave);
-
-                ret = glusterd_get_statefile_name (volinfo, slave,
-                                                   conf_path, &statefile,
-                                                   &is_template_in_use);
-                if (ret) {
-                        if (!strstr(slave, "::"))
-                                gf_log ("", GF_LOG_INFO,
-                                        "%s is not a valid slave url.", slave);
-                        else
-                                gf_log ("", GF_LOG_INFO, "Unable to get"
-                                        " statefile's name");
-                        ret = 0;
-                        goto out;
-                }
-
-                ret = lstat (statefile, &stbuf);
-                if (ret) {
-                        gf_log ("", GF_LOG_INFO, "%s statefile not present.",
-                                statefile);
-                        ret = 0;
-                        goto out;
-                }
+            if (!strstr(slave, "::"))
+                gf_msg(this->name, GF_LOG_INFO, 0, GD_MSG_SLAVE_URL_INVALID,
+                       "%s is not a valid slave url.", slave);
+            else
+                gf_msg(this->name, GF_LOG_INFO, 0,
+                       GD_MSG_GET_STATEFILE_NAME_FAILED,
+                       "Unable to get statefile's name");
+            ret = 0;
+            goto out;
+        }
+
+        ret = sys_lstat(statefile, &stbuf);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_INFO, ENOENT, GD_MSG_FILE_OP_FAILED,
+                   "%s statefile not present.", statefile);
+            ret = 0;
+            goto out;
         }
+    }
 
-        ret = glusterd_read_status_file (volinfo, slave, conf_path,
-                                         rsp_dict, node);
+    ret = glusterd_read_status_file(volinfo, slave, conf_path, rsp_dict, node);
 out:
-        if (statefile)
-                GF_FREE (statefile);
+    if (statefile)
+        GF_FREE(statefile);
 
-        gf_log ("", GF_LOG_DEBUG, "Returning with %d", ret);
-        return ret;
+    gf_msg_debug(this->name, 0, "Returning with %d", ret);
+    return ret;
 }
 
-static int
-glusterd_get_gsync_status_mst (glusterd_volinfo_t *volinfo, dict_t *rsp_dict,
-                               char *node)
+int
+glusterd_get_gsync_status_mst(glusterd_volinfo_t *volinfo, dict_t *rsp_dict,
+                              char *node)
 {
-        glusterd_gsync_status_temp_t  param = {0, };
+    glusterd_gsync_status_temp_t param = {
+        0,
+    };
 
-        GF_ASSERT (volinfo);
+    GF_ASSERT(volinfo);
 
-        param.rsp_dict = rsp_dict;
-        param.volinfo = volinfo;
-        param.node = node;
-        dict_foreach (volinfo->gsync_slaves, _get_status_mst_slv, &param);
+    param.rsp_dict = rsp_dict;
+    param.volinfo = volinfo;
+    param.node = node;
+    dict_foreach(volinfo->gsync_slaves, _get_status_mst_slv, &param);
 
-        return 0;
+    return 0;
 }
 
 static int
-glusterd_get_gsync_status_all (dict_t *rsp_dict, char *node)
+glusterd_get_gsync_status_all(dict_t *rsp_dict, char *node)
 {
+    int32_t ret = 0;
+    glusterd_conf_t *priv = NULL;
+    glusterd_volinfo_t *volinfo = NULL;
+    xlator_t *this = NULL;
 
-        int32_t                 ret = 0;
-        glusterd_conf_t         *priv = NULL;
-        glusterd_volinfo_t      *volinfo = NULL;
+    this = THIS;
+    GF_ASSERT(this);
 
-        GF_ASSERT (THIS);
-        priv = THIS->private;
+    priv = this->private;
 
-        GF_ASSERT (priv);
+    GF_ASSERT(priv);
 
-        cds_list_for_each_entry (volinfo, &priv->volumes, vol_list) {
-                ret = glusterd_get_gsync_status_mst (volinfo, rsp_dict, node);
-                if (ret)
-                        goto out;
-        }
+    cds_list_for_each_entry(volinfo, &priv->volumes, vol_list)
+    {
+        ret = glusterd_get_gsync_status_mst(volinfo, rsp_dict, node);
+        if (ret)
+            goto out;
+    }
 
 out:
-        gf_log ("", GF_LOG_DEBUG, "Returning with %d", ret);
-        return ret;
-
+    gf_msg_debug(this->name, 0, "Returning with %d", ret);
+    return ret;
 }
 
 static int
-glusterd_get_gsync_status (dict_t *dict, char **op_errstr, dict_t *rsp_dict)
+glusterd_get_gsync_status(dict_t *dict, char **op_errstr, dict_t *rsp_dict)
 {
-        char               *slave  = NULL;
-        char               *volname = NULL;
-        char               *conf_path = NULL;
-        char               errmsg[PATH_MAX] = {0, };
-        gf_boolean_t       exists = _gf_false;
-        glusterd_volinfo_t *volinfo = NULL;
-        int                ret = 0;
-        char my_hostname[256] = {0,};
-
-        ret = gethostname(my_hostname, 256);
-        if (ret) {
-                /* stick to N/A */
-                (void) strcpy (my_hostname, "N/A");
-        }
-
-        ret = dict_get_str (dict, "master", &volname);
-        if (ret < 0){
-                ret = glusterd_get_gsync_status_all (rsp_dict, my_hostname);
-                goto out;
-        }
+    char *slave = NULL;
+    char *volname = NULL;
+    char *conf_path = NULL;
+    char errmsg[PATH_MAX] = {
+        0,
+    };
+    glusterd_volinfo_t *volinfo = NULL;
+    int ret = 0;
+    char my_hostname[256] = {
+        0,
+    };
+    xlator_t *this = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+
+    ret = gethostname(my_hostname, 256);
+    if (ret) {
+        /* stick to N/A */
+        (void)strcpy(my_hostname, "N/A");
+    }
+
+    ret = dict_get_str(dict, "master", &volname);
+    if (ret < 0) {
+        ret = glusterd_get_gsync_status_all(rsp_dict, my_hostname);
+        goto out;
+    }
+
+    ret = glusterd_volinfo_find(volname, &volinfo);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_WARNING, 0, GD_MSG_VOL_NOT_FOUND,
+               "volume name does not exist");
+        snprintf(errmsg, sizeof(errmsg),
+                 "Volume name %s does not"
+                 " exist",
+                 volname);
+        *op_errstr = gf_strdup(errmsg);
+        goto out;
+    }
+
+    ret = dict_get_str(dict, "slave", &slave);
+    if (ret < 0) {
+        ret = glusterd_get_gsync_status_mst(volinfo, rsp_dict, my_hostname);
+        goto out;
+    }
+
+    ret = dict_get_str(dict, "conf_path", &conf_path);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+               "Unable to fetch conf file path.");
+        goto out;
+    }
+
+    ret = glusterd_get_gsync_status_mst_slv(volinfo, slave, conf_path, rsp_dict,
+                                            my_hostname);
 
-        exists = glusterd_check_volume_exists (volname);
-        ret = glusterd_volinfo_find (volname, &volinfo);
-        if ((ret) || (!exists)) {
-                gf_log ("", GF_LOG_WARNING, "volume name does not exist");
-                snprintf (errmsg, sizeof(errmsg), "Volume name %s does not"
-                          " exist", volname);
-                *op_errstr = gf_strdup (errmsg);
-                ret = -1;
-                goto out;
-        }
-
-
-        ret = dict_get_str (dict, "slave", &slave);
-        if (ret < 0) {
-                ret = glusterd_get_gsync_status_mst (volinfo,
-                                                     rsp_dict, my_hostname);
-                goto out;
-        }
-
-        ret = dict_get_str (dict, "conf_path", &conf_path);
-        if (ret) {
-                gf_log ("", GF_LOG_ERROR,
-                        "Unable to fetch conf file path.");
-                goto out;
-        }
-
-        ret = glusterd_get_gsync_status_mst_slv (volinfo, slave, conf_path,
-                                                 rsp_dict, my_hostname);
-
- out:
-        gf_log ("", GF_LOG_DEBUG, "Returning %d", ret);
-        return ret;
+out:
+    gf_msg_debug(this->name, 0, "Returning %d", ret);
+    return ret;
 }
 
 static int
-glusterd_gsync_delete (glusterd_volinfo_t *volinfo, char *slave,
-                       char *slave_host, char *slave_vol, char *path_list,
-                       dict_t *dict, dict_t *resp_dict, char **op_errstr)
+glusterd_gsync_delete(glusterd_volinfo_t *volinfo, char *slave,
+                      char *slave_host, char *slave_vol, char *path_list,
+                      dict_t *dict, dict_t *resp_dict, char **op_errstr)
 {
-        int32_t         ret     = -1;
-        runner_t        runner    = {0,};
-        glusterd_conf_t *priv   = NULL;
-        char            *master = NULL;
-        char            *gl_workdir = NULL;
-        char             geo_rep_dir[PATH_MAX] = "";
-        char            *conf_path = NULL;
-
-        GF_ASSERT (slave);
-        GF_ASSERT (slave_host);
-        GF_ASSERT (slave_vol);
-        GF_ASSERT (op_errstr);
-        GF_ASSERT (dict);
-        GF_ASSERT (resp_dict);
-
-        if (THIS)
-                priv = THIS->private;
-        if (priv == NULL) {
-                gf_log ("", GF_LOG_ERROR, "priv of glusterd not present");
-                *op_errstr = gf_strdup ("glusterd defunct");
-                goto out;
-        }
-
-        ret = dict_get_str (dict, "conf_path", &conf_path);
-        if (ret) {
-                gf_log ("", GF_LOG_ERROR,
-                        "Unable to fetch conf file path.");
-                goto out;
-        }
-
-        gl_workdir = priv->workdir;
-        master = "";
-        runinit (&runner);
-        runner_add_args  (&runner, GSYNCD_PREFIX"/gsyncd",
-                          "--delete", "-c", NULL);
-        runner_argprintf (&runner, "%s", conf_path);
-        runner_argprintf (&runner, "--iprefix=%s", DATADIR);
-
-        if (volinfo) {
-                master = volinfo->volname;
-                runner_argprintf (&runner, ":%s", master);
-        }
-        runner_add_arg (&runner, slave);
-        runner_redir (&runner, STDOUT_FILENO, RUN_PIPE);
-        synclock_unlock (&priv->big_lock);
-        ret = runner_run (&runner);
-        synclock_lock (&priv->big_lock);
-        if (ret) {
-                gf_log ("", GF_LOG_ERROR, "gsyncd failed to "
-                        "delete session info for %s and %s peers",
-                        master, slave);
-
-                gf_asprintf (op_errstr, "gsyncd failed to "
-                             "delete session info for %s and %s peers",
-                             master, slave);
-
-                goto out;
-        }
-
-        ret = snprintf (geo_rep_dir, sizeof(geo_rep_dir) - 1,
-                        "%s/"GEOREP"/%s_%s_%s", gl_workdir,
-                        volinfo->volname, slave_host, slave_vol);
-        geo_rep_dir[ret] = '\0';
-
-        ret = rmdir (geo_rep_dir);
-        if (ret) {
-                if (errno == ENOENT)
-                        gf_log ("", GF_LOG_DEBUG, "Geo Rep Dir(%s) Not Present.",
-                                geo_rep_dir);
-                else {
-                        gf_log ("", GF_LOG_ERROR, "Unable to delete "
-                                "Geo Rep Dir(%s). Error: %s", geo_rep_dir,
-                                strerror (errno));
-                        goto out;
-                }
-        }
-
-        ret = 0;
-
-        gf_asprintf (op_errstr, "delete successful");
+    int32_t ret = -1;
+    runner_t runner = {
+        0,
+    };
+    glusterd_conf_t *priv = NULL;
+    char *master = NULL;
+    char *gl_workdir = NULL;
+    char geo_rep_dir[PATH_MAX] = "";
+    char *conf_path = NULL;
+    xlator_t *this = NULL;
+    uint32_t reset_sync_time = _gf_false;
+
+    this = THIS;
+    GF_ASSERT(this);
+
+    GF_ASSERT(slave);
+    GF_ASSERT(slave_host);
+    GF_ASSERT(slave_vol);
+    GF_ASSERT(op_errstr);
+    GF_ASSERT(dict);
+    GF_ASSERT(resp_dict);
+
+    if (THIS)
+        priv = THIS->private;
+    if (priv == NULL) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_GLUSTERD_PRIV_NOT_FOUND,
+               "priv of glusterd not present");
+        *op_errstr = gf_strdup("glusterd defunct");
+        goto out;
+    }
+
+    ret = dict_get_str(dict, "conf_path", &conf_path);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+               "Unable to fetch conf file path.");
+        goto out;
+    }
+
+    gl_workdir = priv->workdir;
+    master = "";
+    runinit(&runner);
+    runner_add_args(&runner, GSYNCD_PREFIX "/gsyncd", "--delete", "-c", NULL);
+    runner_argprintf(&runner, "%s", conf_path);
+    runner_argprintf(&runner, "--iprefix=%s", DATADIR);
+
+    runner_argprintf(&runner, "--path-list=%s", path_list);
+
+    ret = dict_get_uint32(dict, "reset-sync-time", &reset_sync_time);
+    if (!ret && reset_sync_time) {
+        runner_add_args(&runner, "--reset-sync-time", NULL);
+    }
+
+    if (volinfo) {
+        master = volinfo->volname;
+        runner_argprintf(&runner, ":%s", master);
+    }
+    runner_add_arg(&runner, slave);
+    runner_redir(&runner, STDOUT_FILENO, RUN_PIPE);
+    synclock_unlock(&priv->big_lock);
+    ret = runner_run(&runner);
+    synclock_lock(&priv->big_lock);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_SESSION_DEL_FAILED,
+               "gsyncd failed to delete session info for %s and "
+               "%s peers",
+               master, slave);
+
+        gf_asprintf(op_errstr,
+                    "gsyncd failed to "
+                    "delete session info for %s and %s peers",
+                    master, slave);
+
+        goto out;
+    }
+
+    ret = snprintf(geo_rep_dir, sizeof(geo_rep_dir) - 1,
+                   "%s/" GEOREP "/%s_%s_%s", gl_workdir, volinfo->volname,
+                   slave_host, slave_vol);
+    geo_rep_dir[ret] = '\0';
+
+    ret = sys_rmdir(geo_rep_dir);
+    if (ret) {
+        if (errno == ENOENT)
+            gf_msg_debug(this->name, 0, "Geo Rep Dir(%s) Not Present.",
+                         geo_rep_dir);
+        else {
+            gf_msg(this->name, GF_LOG_ERROR, errno, GD_MSG_DIR_OP_FAILED,
+                   "Unable to delete Geo Rep Dir(%s). Error: %s", geo_rep_dir,
+                   strerror(errno));
+            goto out;
+        }
+    }
+
+    ret = 0;
+
+    gf_asprintf(op_errstr, "delete successful");
 
 out:
-        gf_log ("", GF_LOG_DEBUG, "Returning %d", ret);
-        return ret;
+    gf_msg_debug(this->name, 0, "Returning %d", ret);
+    return ret;
 }
 
 int
-glusterd_op_sys_exec (dict_t *dict, char **op_errstr, dict_t *rsp_dict)
+glusterd_op_sys_exec(dict_t *dict, char **op_errstr, dict_t *rsp_dict)
 {
-        char                   buf[PATH_MAX]          = "";
-        char                   cmd_arg_name[PATH_MAX] = "";
-        char                   output_name[PATH_MAX]  = "";
-        char                   errmsg[PATH_MAX]       = "";
-        char                  *ptr                    = NULL;
-        char                  *bufp                   = NULL;
-        char                  *command                = NULL;
-        char                 **cmd_args               = NULL;
-        int                    ret                    = -1;
-        int                    i                      = -1;
-        int                    cmd_args_count         = 0;
-        int                    output_count           = 0;
-        glusterd_conf_t       *priv                   = NULL;
-        runner_t               runner                 = {0,};
-
-        GF_ASSERT (dict);
-        GF_ASSERT (op_errstr);
-        GF_ASSERT (rsp_dict);
-
-        if (THIS)
-                priv = THIS->private;
-        if (priv == NULL) {
-                gf_log ("", GF_LOG_ERROR, "priv of glusterd not present");
-                *op_errstr = gf_strdup ("glusterd defunct");
+    char buf[PATH_MAX] = "";
+    char cmd_arg_name[PATH_MAX] = "";
+    char output_name[PATH_MAX] = "";
+    char errmsg[PATH_MAX] = "";
+    char *ptr = NULL;
+    char *bufp = NULL;
+    char *command = NULL;
+    char **cmd_args = NULL;
+    int ret = -1;
+    int i = -1;
+    int cmd_args_count = 0;
+    int output_count = 0;
+    glusterd_conf_t *priv = NULL;
+    runner_t runner = {
+        0,
+    };
+    xlator_t *this = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+
+    GF_ASSERT(dict);
+    GF_ASSERT(op_errstr);
+    GF_ASSERT(rsp_dict);
+
+    if (THIS)
+        priv = THIS->private;
+    if (priv == NULL) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_GLUSTERD_PRIV_NOT_FOUND,
+               "priv of glusterd not present");
+        *op_errstr = gf_strdup("glusterd defunct");
+        goto out;
+    }
+
+    ret = dict_get_str(dict, "command", &command);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+               "Unable to get command from dict");
+        goto out;
+    }
+
+    ret = dict_get_int32(dict, "cmd_args_count", &cmd_args_count);
+    if (ret)
+        gf_msg(this->name, GF_LOG_INFO, 0, GD_MSG_DICT_GET_FAILED,
+               "No cmd_args_count");
+
+    if (cmd_args_count) {
+        cmd_args = GF_CALLOC(cmd_args_count, sizeof(char *), gf_common_mt_char);
+        if (!cmd_args) {
+            gf_msg(this->name, GF_LOG_ERROR, ENOMEM, GD_MSG_NO_MEMORY,
+                   "Unable to calloc. Errno = %s", strerror(errno));
+            goto out;
+        }
+
+        for (i = 1; i <= cmd_args_count; i++) {
+            snprintf(cmd_arg_name, sizeof(cmd_arg_name), "cmd_arg_%d", i);
+            ret = dict_get_str(dict, cmd_arg_name, &cmd_args[i - 1]);
+            if (ret) {
+                gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+                       "Unable to get"
+                       " %s in dict",
+                       cmd_arg_name);
                 goto out;
+            }
         }
+    }
+
+    runinit(&runner);
+    runner_argprintf(&runner, GSYNCD_PREFIX "/peer_%s", command);
+    for (i = 0; i < cmd_args_count; i++)
+        runner_add_arg(&runner, cmd_args[i]);
+    runner_redir(&runner, STDOUT_FILENO, RUN_PIPE);
+    synclock_unlock(&priv->big_lock);
+    ret = runner_start(&runner);
+    if (ret == -1) {
+        snprintf(errmsg, sizeof(errmsg),
+                 "Unable to "
+                 "execute command. Error : %s",
+                 strerror(errno));
+        *op_errstr = gf_strdup(errmsg);
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_CMD_EXEC_FAIL, "%s", errmsg);
+        ret = -1;
+        synclock_lock(&priv->big_lock);
+        goto out;
+    }
+
+    do {
+        ptr = fgets(buf, sizeof(buf), runner_chio(&runner, STDOUT_FILENO));
+        if (ptr) {
+            ret = dict_get_int32(rsp_dict, "output_count", &output_count);
+            if (ret)
+                output_count = 1;
+            else
+                output_count++;
+            snprintf(output_name, sizeof(output_name), "output_%d",
+                     output_count);
+            if (buf[strlen(buf) - 1] == '\n')
+                buf[strlen(buf) - 1] = '\0';
+            bufp = gf_strdup(buf);
+            if (!bufp)
+                gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_STRDUP_FAILED,
+                       "gf_strdup failed.");
+            ret = dict_set_dynstr(rsp_dict, output_name, bufp);
+            if (ret) {
+                GF_FREE(bufp);
+                gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+                       "output set "
+                       "failed.");
+            }
+            ret = dict_set_int32(rsp_dict, "output_count", output_count);
+            if (ret)
+                gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+                       "output_count "
+                       "set failed.");
+        }
+    } while (ptr);
+
+    ret = runner_end(&runner);
+    if (ret) {
+        snprintf(errmsg, sizeof(errmsg),
+                 "Unable to "
+                 "end. Error : %s",
+                 strerror(errno));
+        *op_errstr = gf_strdup(errmsg);
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_UNABLE_TO_END, "%s", errmsg);
+        ret = -1;
+        synclock_lock(&priv->big_lock);
+        goto out;
+    }
+    synclock_lock(&priv->big_lock);
+
+    ret = 0;
+out:
+    if (cmd_args) {
+        GF_FREE(cmd_args);
+        cmd_args = NULL;
+    }
 
-        ret = dict_get_str (dict, "command", &command);
+    gf_msg_debug(this->name, 0, "Returning %d", ret);
+    return ret;
+}
+
+int
+glusterd_op_copy_file(dict_t *dict, char **op_errstr)
+{
+    char abs_filename[PATH_MAX] = "";
+    char errmsg[PATH_MAX] = "";
+    char *filename = NULL;
+    char *host_uuid = NULL;
+    char uuid_str[64] = {0};
+    char *contents = NULL;
+    char buf[4096] = "";
+    int ret = -1;
+    int fd = -1;
+    int bytes_writen = 0;
+    int bytes_read = 0;
+    int contents_size = -1;
+    int file_mode = -1;
+    glusterd_conf_t *priv = NULL;
+    struct stat stbuf = {
+        0,
+    };
+    gf_boolean_t free_contents = _gf_true;
+    xlator_t *this = NULL;
+    int32_t len = 0;
+
+    this = THIS;
+    GF_ASSERT(this);
+
+    if (THIS)
+        priv = THIS->private;
+    if (priv == NULL) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_GLUSTERD_PRIV_NOT_FOUND,
+               "priv of glusterd not present");
+        *op_errstr = gf_strdup("glusterd defunct");
+        goto out;
+    }
+
+    ret = dict_get_str(dict, "host-uuid", &host_uuid);
+    if (ret < 0)
+        goto out;
+
+    ret = dict_get_str(dict, "source", &filename);
+    if (ret < 0) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+               "Unable to fetch filename from dict.");
+        *op_errstr = gf_strdup("command unsuccessful");
+        goto out;
+    }
+    len = snprintf(abs_filename, sizeof(abs_filename), "%s/%s", priv->workdir,
+                   filename);
+    if ((len < 0) || (len >= sizeof(abs_filename))) {
+        ret = -1;
+        goto out;
+    }
+
+    uuid_utoa_r(MY_UUID, uuid_str);
+    if (!strcmp(uuid_str, host_uuid)) {
+        ret = sys_lstat(abs_filename, &stbuf);
         if (ret) {
-                gf_log ("", GF_LOG_ERROR,
-                        "Unable to get command from dict");
-                goto out;
+            len = snprintf(errmsg, sizeof(errmsg),
+                           "Source file "
+                           "does not exist in %s",
+                           priv->workdir);
+            if (len < 0) {
+                strcpy(errmsg, "<error>");
+            }
+            *op_errstr = gf_strdup(errmsg);
+            gf_msg(this->name, GF_LOG_ERROR, ENOENT, GD_MSG_FILE_OP_FAILED,
+                   "%s", errmsg);
+            goto out;
+        }
+
+        contents = GF_CALLOC(1, stbuf.st_size + 1, gf_common_mt_char);
+        if (!contents) {
+            snprintf(errmsg, sizeof(errmsg), "Unable to allocate memory");
+            *op_errstr = gf_strdup(errmsg);
+            gf_msg(this->name, GF_LOG_ERROR, ENOMEM, GD_MSG_NO_MEMORY, "%s",
+                   errmsg);
+            ret = -1;
+            goto out;
+        }
+
+        fd = open(abs_filename, O_RDONLY);
+        if (fd < 0) {
+            len = snprintf(errmsg, sizeof(errmsg), "Unable to open %s",
+                           abs_filename);
+            if (len < 0) {
+                strcpy(errmsg, "<error>");
+            }
+            *op_errstr = gf_strdup(errmsg);
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_FILE_OP_FAILED, "%s",
+                   errmsg);
+            ret = -1;
+            goto out;
         }
 
-        ret = dict_get_int32 (dict, "cmd_args_count", &cmd_args_count);
-        if (ret)
-                gf_log ("", GF_LOG_INFO, "No cmd_args_count");
-
-        if (cmd_args_count) {
-                cmd_args = GF_CALLOC (cmd_args_count, sizeof (char*),
-                                      gf_common_mt_char);
-                if (!cmd_args) {
-                        gf_log ("", GF_LOG_ERROR, "Unable to calloc. "
-                                "Errno = %s", strerror(errno));
-                        goto out;
-                }
+        do {
+            ret = sys_read(fd, buf, sizeof(buf) - 1);
+            if (ret > 0) {
+                buf[ret] = '\0';
+                memcpy(contents + bytes_read, buf, ret);
+                bytes_read += ret;
+            }
+        } while (ret > 0);
 
-                for (i=1; i <= cmd_args_count; i++) {
-                        memset (cmd_arg_name, '\0', sizeof(cmd_arg_name));
-                        snprintf (cmd_arg_name, sizeof(cmd_arg_name),
-                                  "cmd_arg_%d", i);
-                        ret = dict_get_str (dict, cmd_arg_name, &cmd_args[i-1]);
-                        if (ret) {
-                                gf_log ("", GF_LOG_ERROR,
-                                        "Unable to get %s in dict",
-                                        cmd_arg_name);
-                                goto out;
-                        }
-                }
+        if (bytes_read != stbuf.st_size) {
+            len = snprintf(errmsg, sizeof(errmsg),
+                           "Unable to read all the data from %s", abs_filename);
+            if (len < 0) {
+                strcpy(errmsg, "<error>");
+            }
+            *op_errstr = gf_strdup(errmsg);
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_READ_ERROR, "%s",
+                   errmsg);
+            ret = -1;
+            goto out;
         }
 
-        runinit (&runner);
-        runner_argprintf (&runner, GSYNCD_PREFIX"/peer_%s", command);
-        for (i=0; i < cmd_args_count; i++)
-                runner_add_arg (&runner, cmd_args[i]);
-        runner_redir (&runner, STDOUT_FILENO, RUN_PIPE);
-        synclock_unlock (&priv->big_lock);
-        ret = runner_start (&runner);
-        if (ret == -1) {
-                snprintf (errmsg, sizeof (errmsg), "Unable to "
-                          "execute command. Error : %s",
-                          strerror (errno));
-                *op_errstr = gf_strdup (errmsg);
-                gf_log ("", GF_LOG_ERROR, "%s", errmsg);
-                ret = -1;
-                synclock_lock (&priv->big_lock);
-                goto out;
+        ret = dict_set_int32(dict, "contents_size", stbuf.st_size);
+        if (ret) {
+            snprintf(errmsg, sizeof(errmsg),
+                     "Unable to set"
+                     " contents size in dict.");
+            *op_errstr = gf_strdup(errmsg);
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED, "%s",
+                   errmsg);
+            goto out;
         }
 
-        do {
-                ptr = fgets(buf, sizeof(buf), runner_chio (&runner, STDOUT_FILENO));
-                if (ptr) {
-                        ret = dict_get_int32 (rsp_dict, "output_count", &output_count);
-                        if (ret)
-                                output_count = 1;
-                        else
-                                output_count++;
-                        memset (output_name, '\0', sizeof (output_name));
-                        snprintf (output_name, sizeof (output_name),
-                                  "output_%d", output_count);
-                        if (buf[strlen(buf) - 1] == '\n')
-                                buf[strlen(buf) - 1] = '\0';
-                        bufp = gf_strdup (buf);
-                        if (!bufp)
-                                gf_log ("", GF_LOG_ERROR, "gf_strdup failed.");
-                        ret = dict_set_dynstr (rsp_dict, output_name, bufp);
-                        if (ret) {
-                                GF_FREE (bufp);
-                                gf_log ("", GF_LOG_ERROR, "output set failed.");
-                        }
-                        ret = dict_set_int32 (rsp_dict, "output_count", output_count);
-                        if (ret)
-                                gf_log ("", GF_LOG_ERROR, "output_count set failed.");
-                }
-        } while (ptr);
-
-        ret = runner_end (&runner);
+        ret = dict_set_int32(dict, "file_mode", (int32_t)stbuf.st_mode);
         if (ret) {
-                snprintf (errmsg, sizeof (errmsg), "Unable to "
-                          "end. Error : %s",
-                          strerror (errno));
-                *op_errstr = gf_strdup (errmsg);
-                gf_log ("", GF_LOG_ERROR, "%s", errmsg);
-                ret = -1;
-                synclock_lock (&priv->big_lock);
-                goto out;
+            snprintf(errmsg, sizeof(errmsg),
+                     "Unable to set"
+                     " file mode in dict.");
+            *op_errstr = gf_strdup(errmsg);
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED, "%s",
+                   errmsg);
+            goto out;
         }
-        synclock_lock (&priv->big_lock);
 
-        ret = 0;
-out:
-        if (cmd_args) {
-                GF_FREE (cmd_args);
-                cmd_args = NULL;
+        ret = dict_set_bin(dict, "common_pem_contents", contents,
+                           stbuf.st_size);
+        if (ret) {
+            snprintf(errmsg, sizeof(errmsg),
+                     "Unable to set"
+                     " pem contents in dict.");
+            *op_errstr = gf_strdup(errmsg);
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED, "%s",
+                   errmsg);
+            goto out;
+        }
+        free_contents = _gf_false;
+    } else {
+        free_contents = _gf_false;
+        ret = dict_get_bin(dict, "common_pem_contents", (void **)&contents);
+        if (ret) {
+            snprintf(errmsg, sizeof(errmsg),
+                     "Unable to get"
+                     " pem contents in dict.");
+            *op_errstr = gf_strdup(errmsg);
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED, "%s",
+                   errmsg);
+            goto out;
+        }
+        ret = dict_get_int32(dict, "contents_size", &contents_size);
+        if (ret) {
+            snprintf(errmsg, sizeof(errmsg),
+                     "Unable to set"
+                     " contents size in dict.");
+            *op_errstr = gf_strdup(errmsg);
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED, "%s",
+                   errmsg);
+            goto out;
         }
 
-        gf_log ("", GF_LOG_DEBUG, "Returning %d", ret);
-        return ret;
-}
-
-int
-glusterd_op_copy_file (dict_t *dict, char **op_errstr)
-{
-        char             abs_filename[PATH_MAX] = "";
-        char             errmsg[PATH_MAX]       = "";
-        char            *filename               = NULL;
-        char            *host_uuid              = NULL;
-        char             uuid_str [64]          = {0};
-        char            *contents               = NULL;
-        char             buf[1024]              = "";
-        int              ret                    = -1;
-        int              fd                     = -1;
-        int              bytes_writen           = 0;
-        int              bytes_read             = 0;
-        int              contents_size          = -1;
-        int              file_mode              = -1;
-        glusterd_conf_t *priv                   = NULL;
-        struct stat      stbuf                  = {0,};
-        gf_boolean_t     free_contents          = _gf_true;
-
-        if (THIS)
-                priv = THIS->private;
-        if (priv == NULL) {
-                gf_log ("", GF_LOG_ERROR, "priv of glusterd not present");
-                *op_errstr = gf_strdup ("glusterd defunct");
-                goto out;
+        ret = dict_get_int32(dict, "file_mode", &file_mode);
+        if (ret) {
+            snprintf(errmsg, sizeof(errmsg),
+                     "Unable to get"
+                     " file mode in dict.");
+            *op_errstr = gf_strdup(errmsg);
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED, "%s",
+                   errmsg);
+            goto out;
+        }
+
+        fd = open(abs_filename, O_WRONLY | O_TRUNC | O_CREAT, 0600);
+        if (fd < 0) {
+            len = snprintf(errmsg, sizeof(errmsg), "Unable to open %s",
+                           abs_filename);
+            if (len < 0) {
+                strcpy(errmsg, "<error>");
+            }
+            *op_errstr = gf_strdup(errmsg);
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_FILE_OP_FAILED, "%s",
+                   errmsg);
+            ret = -1;
+            goto out;
         }
 
-        ret = dict_get_str (dict, "host-uuid", &host_uuid);
-        if (ret < 0)
-                goto out;
+        bytes_writen = sys_write(fd, contents, contents_size);
 
-        ret = dict_get_str (dict, "source", &filename);
-        if (ret < 0) {
-               gf_log ("", GF_LOG_ERROR, "Unable to fetch"
-                       " filename from dict.");
-               *op_errstr = gf_strdup ("command unsuccessful");
-               goto out;
+        if (bytes_writen != contents_size) {
+            len = snprintf(errmsg, sizeof(errmsg), "Failed to write to %s",
+                           abs_filename);
+            if (len < 0) {
+                strcpy(errmsg, "<error>");
+            }
+            *op_errstr = gf_strdup(errmsg);
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_FILE_OP_FAILED, "%s",
+                   errmsg);
+            ret = -1;
+            goto out;
         }
-        snprintf (abs_filename, sizeof(abs_filename),
-                  "%s/%s", priv->workdir, filename);
-
-        uuid_utoa_r (MY_UUID, uuid_str);
-        if (!strcmp (uuid_str, host_uuid)) {
-                ret = lstat (abs_filename, &stbuf);
-                if (ret) {
-                        snprintf (errmsg, sizeof (errmsg), "Source file"
-                                 " does not exist in %s", priv->workdir);
-                        *op_errstr = gf_strdup (errmsg);
-                        gf_log ("", GF_LOG_ERROR, "%s", errmsg);
-                        goto out;
-                }
-
-                contents = GF_CALLOC(1, stbuf.st_size+1, gf_common_mt_char);
-                if (!contents) {
-                        snprintf (errmsg, sizeof (errmsg),
-                                  "Unable to allocate memory");
-                        *op_errstr = gf_strdup (errmsg);
-                        gf_log ("", GF_LOG_ERROR, "%s", errmsg);
-                        ret = -1;
-                        goto out;
-                }
-
-                fd = open (abs_filename, O_RDONLY);
-                if (fd < 0) {
-                        snprintf (errmsg, sizeof (errmsg), "Unable to open %s",
-                                  abs_filename);
-                        *op_errstr = gf_strdup (errmsg);
-                        gf_log ("", GF_LOG_ERROR, "%s", errmsg);
-                        ret = -1;
-                        goto out;
-                }
-
-                do {
-                        ret = read (fd, buf, sizeof(buf));
-                        if (ret > 0) {
-                                memcpy (contents+bytes_read, buf, ret);
-                                bytes_read += ret;
-                                memset (buf, '\0', sizeof(buf));
-                        }
-                } while (ret > 0);
-
-                if (bytes_read != stbuf.st_size) {
-                        snprintf (errmsg, sizeof (errmsg), "Unable to read all "
-                                  "the data from %s", abs_filename);
-                        *op_errstr = gf_strdup (errmsg);
-                        gf_log ("", GF_LOG_ERROR, "%s", errmsg);
-                        ret = -1;
-                        goto out;
-                }
-
-                ret = dict_set_int32 (dict, "contents_size", stbuf.st_size);
-                if (ret) {
-                        snprintf (errmsg, sizeof (errmsg), "Unable to set"
-                                  " contents size in dict.");
-                        *op_errstr = gf_strdup (errmsg);
-                        gf_log ("", GF_LOG_ERROR, "%s", errmsg);
-                        goto out;
-                }
-
-                ret = dict_set_int32 (dict, "file_mode",
-                                      (int32_t)stbuf.st_mode);
-                if (ret) {
-                        snprintf (errmsg, sizeof (errmsg), "Unable to set"
-                                  " file mode in dict.");
-                        *op_errstr = gf_strdup (errmsg);
-                        gf_log ("", GF_LOG_ERROR, "%s", errmsg);
-                        goto out;
-                }
-
-                ret = dict_set_bin (dict, "common_pem_contents",
-                                    contents, stbuf.st_size);
-                if (ret) {
-                        snprintf (errmsg, sizeof (errmsg), "Unable to set"
-                                  " pem contents in dict.");
-                        *op_errstr = gf_strdup (errmsg);
-                        gf_log ("", GF_LOG_ERROR, "%s", errmsg);
-                        goto out;
-                }
-                free_contents = _gf_false;
-        } else {
-                free_contents = _gf_false;
-                ret = dict_get_bin (dict, "common_pem_contents",
-                                    (void **) &contents);
-                if (ret) {
-                        snprintf (errmsg, sizeof (errmsg), "Unable to get"
-                                  " pem contents in dict.");
-                        *op_errstr = gf_strdup (errmsg);
-                        gf_log ("", GF_LOG_ERROR, "%s", errmsg);
-                        goto out;
-                }
-                ret = dict_get_int32 (dict, "contents_size", &contents_size);
-                if (ret) {
-                        snprintf (errmsg, sizeof (errmsg), "Unable to set"
-                                  " contents size in dict.");
-                        *op_errstr = gf_strdup (errmsg);
-                        gf_log ("", GF_LOG_ERROR, "%s", errmsg);
-                        goto out;
-                }
-
-                ret = dict_get_int32 (dict, "file_mode", &file_mode);
-                if (ret) {
-                        snprintf (errmsg, sizeof (errmsg), "Unable to get"
-                                  " file mode in dict.");
-                        *op_errstr = gf_strdup (errmsg);
-                        gf_log ("", GF_LOG_ERROR, "%s", errmsg);
-                        goto out;
-                }
-
-                fd = open (abs_filename, O_WRONLY | O_TRUNC | O_CREAT, 0600);
-                if (fd < 0) {
-                        snprintf (errmsg, sizeof (errmsg), "Unable to open %s",
-                                  abs_filename);
-                        *op_errstr = gf_strdup (errmsg);
-                        gf_log ("", GF_LOG_ERROR, "%s", errmsg);
-                        ret = -1;
-                        goto out;
-                }
-
-                bytes_writen = write (fd, contents, contents_size);
 
-                if (bytes_writen != contents_size) {
-                        snprintf (errmsg, sizeof (errmsg), "Failed to write"
-                                  " to %s", abs_filename);
-                        *op_errstr = gf_strdup (errmsg);
-                        gf_log ("", GF_LOG_ERROR, "%s", errmsg);
-                        ret = -1;
-                        goto out;
-                }
+        sys_fchmod(fd, file_mode);
+    }
 
-                fchmod (fd, file_mode);
-        }
-
-        ret = 0;
+    ret = 0;
 out:
-        if (fd != -1)
-                close (fd);
+    if (fd != -1)
+        sys_close(fd);
 
-        if (free_contents)
-                GF_FREE(contents);
+    if (free_contents)
+        GF_FREE(contents);
 
-        gf_log ("", GF_LOG_DEBUG, "Returning %d", ret);
-        return ret;
+    gf_msg_debug(this->name, 0, "Returning %d", ret);
+    return ret;
 }
 
 int
-glusterd_op_gsync_set (dict_t *dict, char **op_errstr, dict_t *rsp_dict)
+glusterd_op_gsync_set(dict_t *dict, char **op_errstr, dict_t *rsp_dict)
 {
-        int32_t             ret                 = -1;
-        int32_t             type                = -1;
-        char               *host_uuid           = NULL;
-        char               *slave               = NULL;
-        char               *slave_url           = NULL;
-        char               *slave_vol           = NULL;
-        char               *slave_host          = NULL;
-        char               *volname             = NULL;
-        char               *path_list           = NULL;
-        glusterd_volinfo_t *volinfo             = NULL;
-        glusterd_conf_t    *priv                = NULL;
-        gf_boolean_t        is_force            = _gf_false;
-        char               *status_msg          = NULL;
-        gf_boolean_t        is_running          = _gf_false;
-        char               *conf_path           = NULL;
-        char                errmsg[PATH_MAX]    = "";
-        char               *key                 = NULL;
-        xlator_t           *this                = NULL;
-
-        this = THIS;
-        GF_ASSERT (this);
-        priv = this->private;
-        GF_ASSERT (priv);
-        GF_ASSERT (dict);
-        GF_ASSERT (op_errstr);
-        GF_ASSERT (rsp_dict);
-
-        ret = dict_get_int32 (dict, "type", &type);
-        if (ret < 0)
-                goto out;
+    int32_t ret = -1;
+    int32_t type = -1;
+    char *host_uuid = NULL;
+    char *slave = NULL;
+    char *slave_url = NULL;
+    char *slave_vol = NULL;
+    char *slave_host = NULL;
+    char *volname = NULL;
+    char *path_list = NULL;
+    glusterd_volinfo_t *volinfo = NULL;
+    glusterd_conf_t *priv = NULL;
+    gf_boolean_t is_force = _gf_false;
+    char *status_msg = NULL;
+    gf_boolean_t is_running = _gf_false;
+    char *conf_path = NULL;
+    char *key = NULL;
+    xlator_t *this = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+    priv = this->private;
+    GF_ASSERT(priv);
+    GF_ASSERT(dict);
+    GF_ASSERT(op_errstr);
+    GF_ASSERT(rsp_dict);
+
+    ret = dict_get_int32(dict, "type", &type);
+    if (ret < 0)
+        goto out;
+
+    ret = dict_get_str(dict, "host-uuid", &host_uuid);
+    if (ret < 0)
+        goto out;
+
+    if (type == GF_GSYNC_OPTION_TYPE_STATUS) {
+        ret = glusterd_get_gsync_status(dict, op_errstr, rsp_dict);
+        goto out;
+    }
+
+    ret = dict_get_str(dict, "slave", &slave);
+    if (ret < 0)
+        goto out;
+
+    key = slave;
+
+    ret = dict_get_str(dict, "slave_url", &slave_url);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+               "Unable to fetch slave url.");
+        goto out;
+    }
+
+    ret = dict_get_str(dict, "slave_host", &slave_host);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+               "Unable to fetch slave hostname.");
+        goto out;
+    }
+
+    ret = dict_get_str(dict, "slave_vol", &slave_vol);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+               "Unable to fetch slave volume name.");
+        goto out;
+    }
+
+    ret = dict_get_str(dict, "conf_path", &conf_path);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+               "Unable to fetch conf file path.");
+        goto out;
+    }
+
+    if (dict_get_str(dict, "master", &volname) == 0) {
+        ret = glusterd_volinfo_find(volname, &volinfo);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_WARNING, 0, GD_MSG_DICT_GET_FAILED,
+                   "Volinfo for"
+                   " %s (master) not found",
+                   volname);
+            goto out;
+        }
 
-        ret = dict_get_str (dict, "host-uuid", &host_uuid);
-        if (ret < 0)
-                goto out;
+        ret = glusterd_get_local_brickpaths(volinfo, &path_list);
+        if (!path_list && ret == -1)
+            goto out;
+    }
 
-        if (type == GF_GSYNC_OPTION_TYPE_STATUS) {
-                ret = glusterd_get_gsync_status (dict, op_errstr, rsp_dict);
+    if (type == GF_GSYNC_OPTION_TYPE_CONFIG) {
+        ret = glusterd_gsync_configure(volinfo, slave, path_list, dict,
+                                       rsp_dict, op_errstr);
+        if (!ret) {
+            ret = dict_set_str(rsp_dict, "conf_path", conf_path);
+            if (ret) {
+                gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+                       "Unable to store conf_file_path.");
                 goto out;
+            }
         }
+        goto out;
+    }
 
-        ret = dict_get_str (dict, "slave", &slave);
-        if (ret < 0)
-                goto out;
+    if (type == GF_GSYNC_OPTION_TYPE_DELETE) {
+        ret = glusterd_remove_slave_in_info(volinfo, slave, op_errstr);
+        if (ret && !is_force && path_list)
+            goto out;
 
-        key = slave;
+        ret = glusterd_gsync_delete(volinfo, slave, slave_host, slave_vol,
+                                    path_list, dict, rsp_dict, op_errstr);
+        goto out;
+    }
 
-        ret = dict_get_str (dict, "slave_url", &slave_url);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "Unable to fetch slave url.");
-                goto out;
-        }
+    if (!volinfo) {
+        ret = -1;
+        goto out;
+    }
 
-        ret = dict_get_str (dict, "slave_host", &slave_host);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "Unable to fetch slave hostname.");
-                goto out;
-        }
+    is_force = dict_get_str_boolean(dict, "force", _gf_false);
 
-        ret = dict_get_str (dict, "slave_vol", &slave_vol);
+    if (type == GF_GSYNC_OPTION_TYPE_START) {
+        /* Add slave to the dict indicating geo-rep session is running*/
+        ret = dict_set_dynstr_with_alloc(volinfo->gsync_active_slaves, key,
+                                         "running");
         if (ret) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "Unable to fetch slave volume name.");
-                goto out;
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+                   "Unable to set key:%s"
+                   " value:running in the dict",
+                   key);
+            goto out;
         }
 
-        ret = dict_get_str (dict, "conf_path", &conf_path);
+        /* If slave volume uuid is not present in gsync_slaves
+         * update it*/
+        ret = glusterd_update_slave_voluuid_slaveinfo(volinfo);
         if (ret) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "Unable to fetch conf file path.");
-                goto out;
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_REMOTE_VOL_UUID_FAIL,
+                   "Error in updating"
+                   " slave volume uuid for old slave info");
+            goto out;
         }
 
-        if (dict_get_str (dict, "master", &volname) == 0) {
-                ret = glusterd_volinfo_find (volname, &volinfo);
-                if (ret) {
-                        gf_log (this->name, GF_LOG_WARNING, "Volinfo for"
-                                " %s (master) not found", volname);
-                        goto out;
-                }
-
-                ret = glusterd_get_local_brickpaths (volinfo, &path_list);
-        }
+        ret = glusterd_start_gsync(volinfo, slave, path_list, conf_path,
+                                   host_uuid, op_errstr, _gf_false);
 
-        if (type == GF_GSYNC_OPTION_TYPE_CONFIG) {
-                ret = glusterd_gsync_configure (volinfo, slave, path_list,
-                                                dict, rsp_dict, op_errstr);
-                if (!ret) {
-                        ret = dict_set_str (rsp_dict, "conf_path", conf_path);
-                        if (ret) {
-                                gf_log (this->name, GF_LOG_ERROR,
-                                        "Unable to store conf_file_path.");
-                                goto out;
-                        }
-                }
-                goto out;
-        }
-
-        if (type == GF_GSYNC_OPTION_TYPE_DELETE) {
-                ret = glusterd_remove_slave_in_info(volinfo, slave, op_errstr);
-                if (ret && !is_force && path_list)
-                        goto out;
-
-                ret = glusterd_gsync_delete (volinfo, slave, slave_host,
-                                             slave_vol, path_list, dict,
-                                             rsp_dict, op_errstr);
-                goto out;
-        }
-
-        if (!volinfo) {
-                ret = -1;
+        /* Delete added slave in the dict if start fails*/
+        if (ret)
+            dict_del(volinfo->gsync_active_slaves, key);
+    }
+
+    if (type == GF_GSYNC_OPTION_TYPE_STOP ||
+        type == GF_GSYNC_OPTION_TYPE_PAUSE ||
+        type == GF_GSYNC_OPTION_TYPE_RESUME) {
+        ret = glusterd_check_gsync_running_local(volinfo->volname, slave,
+                                                 conf_path, &is_running);
+        if (!ret && !is_force && path_list && (_gf_true != is_running)) {
+            gf_msg(this->name, GF_LOG_WARNING, 0, GD_MSG_GSYNCD_OP_SET_FAILED,
+                   GEOREP
+                   " is not "
+                   "set up for %s(master) and %s(slave)",
+                   volname, slave);
+            *op_errstr = gf_strdup(GEOREP " is not set up");
+            goto out;
+        }
+
+        if (type == GF_GSYNC_OPTION_TYPE_PAUSE) {
+            ret = gd_pause_or_resume_gsync(dict, volname, slave, slave_host,
+                                           slave_vol, conf_path, op_errstr,
+                                           _gf_true);
+            if (ret)
+                gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_PAUSE_FAILED,
+                       GEOREP " Pause Failed");
+            else
+                dict_del(volinfo->gsync_active_slaves, key);
+
+        } else if (type == GF_GSYNC_OPTION_TYPE_RESUME) {
+            /* Add slave to the dict indicating geo-rep session is
+             * running*/
+            ret = dict_set_dynstr_with_alloc(volinfo->gsync_active_slaves, key,
+                                             "running");
+            if (ret) {
+                gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+                       "Unable to set "
+                       "key:%s value:running in dict",
+                       key);
                 goto out;
-        }
-
-        is_force = dict_get_str_boolean (dict, "force", _gf_false);
+            }
 
-        if (type == GF_GSYNC_OPTION_TYPE_START) {
-                /* Add slave to the dict indicating geo-rep session is running*/
-                ret = dict_set_dynstr_with_alloc (volinfo->gsync_active_slaves,
-                                                  key, "running");
+            ret = gd_pause_or_resume_gsync(dict, volname, slave, slave_host,
+                                           slave_vol, conf_path, op_errstr,
+                                           _gf_false);
+            if (ret) {
+                gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_RESUME_FAILED,
+                       GEOREP " Resume Failed");
+                dict_del(volinfo->gsync_active_slaves, key);
+            }
+        } else {
+            ret = stop_gsync(volname, slave, &status_msg, conf_path, op_errstr,
+                             is_force);
+
+            if (ret == 0 && status_msg)
+                ret = dict_set_str(rsp_dict, "gsync-status", status_msg);
+            if (!ret) {
+                ret = glusterd_create_status_file(
+                    volinfo->volname, slave, slave_host, slave_vol, "Stopped");
                 if (ret) {
-                        gf_log (this->name, GF_LOG_ERROR, "Unable to set key:%s"
-                                " value:running in the dict", key);
-                        goto out;
-                }
-
-                ret = glusterd_start_gsync (volinfo, slave, path_list,
-                                            conf_path, host_uuid, op_errstr,
-                                            _gf_false);
-
-                /* Delete added slave in the dict if start fails*/
-                if (ret)
-                        dict_del (volinfo->gsync_active_slaves, key);
-       }
-
-        if (type == GF_GSYNC_OPTION_TYPE_STOP ||
-            type == GF_GSYNC_OPTION_TYPE_PAUSE ||
-            type == GF_GSYNC_OPTION_TYPE_RESUME) {
-                ret = glusterd_check_gsync_running_local (volinfo->volname,
-                                                          slave, conf_path,
-                                                          &is_running);
-                if (!ret && !is_force && path_list &&
-                    (_gf_true != is_running)) {
-                        gf_log (this->name, GF_LOG_WARNING, GEOREP" is not set "
-                                "up for %s(master) and %s(slave)",
-                                volname, slave);
-                        *op_errstr = strdup (GEOREP" is not set up");
-                        goto out;
-                }
-
-                if (type == GF_GSYNC_OPTION_TYPE_PAUSE) {
-                        ret = gd_pause_or_resume_gsync (dict, volname, slave,
-                                                        slave_host, slave_vol,
-                                                        conf_path, op_errstr,
-                                                        _gf_true);
-                        if (ret)
-                                gf_log(this->name, GF_LOG_ERROR, GEOREP
-                                       " Pause Failed");
-                        else
-                                dict_del (volinfo->gsync_active_slaves, key);
-
-                } else if (type == GF_GSYNC_OPTION_TYPE_RESUME) {
-
-                        /* Add slave to the dict indicating geo-rep session is
-                         * running*/
-                        ret = dict_set_dynstr_with_alloc (
-                                              volinfo->gsync_active_slaves,
-                                              key, "running");
-                        if (ret) {
-                              gf_log (this->name, GF_LOG_ERROR, "Unable to set "
-                                      "key:%s value:running in dict", key);
-                              goto out;
-                        }
-
-                        ret = gd_pause_or_resume_gsync (dict, volname, slave,
-                                                        slave_host, slave_vol,
-                                                        conf_path, op_errstr,
-                                                        _gf_false);
-                        if (ret) {
-                                gf_log(this->name, GF_LOG_ERROR, GEOREP
-                                       " Resume Failed");
-                                dict_del (volinfo->gsync_active_slaves, key);
-                        }
-                } else {
-
-                        ret = stop_gsync (volname, slave, &status_msg,
-                                          conf_path, op_errstr, is_force);
-
-                        if (ret == 0 && status_msg)
-                                ret = dict_set_str (rsp_dict, "gsync-status",
-                                                    status_msg);
-                        if (!ret) {
-                                ret = glusterd_create_status_file (
-                                                           volinfo->volname,
-                                                           slave, slave_host,
-                                                           slave_vol,"Stopped");
-                                if (ret) {
-                                        gf_log (this->name, GF_LOG_ERROR,
-                                                "Unable to update state_file. "
-                                                "Error : %s", strerror (errno));
-                                }
-                                dict_del (volinfo->gsync_active_slaves, key);
-                        }
+                    gf_msg(this->name, GF_LOG_ERROR, 0,
+                           GD_MSG_UPDATE_STATEFILE_FAILED,
+                           "Unable to update state_file. "
+                           "Error : %s",
+                           strerror(errno));
                 }
+                dict_del(volinfo->gsync_active_slaves, key);
+            }
         }
+    }
 
 out:
-        if (path_list) {
-                GF_FREE (path_list);
-                path_list = NULL;
-        }
+    if (path_list) {
+        GF_FREE(path_list);
+        path_list = NULL;
+    }
 
-        gf_log (this->name, GF_LOG_DEBUG, "Returning %d", ret);
-        return ret;
+    gf_msg_debug(this->name, 0, "Returning %d", ret);
+    return ret;
 }
 
 int
-glusterd_get_slave_details_confpath (glusterd_volinfo_t *volinfo,
-                                     dict_t *dict, char **slave_url,
-                                     char **slave_host, char **slave_vol,
-                                     char **conf_path, char **op_errstr)
+glusterd_get_slave_details_confpath(glusterd_volinfo_t *volinfo, dict_t *dict,
+                                    char **slave_url, char **slave_host,
+                                    char **slave_vol, char **conf_path,
+                                    char **op_errstr)
 {
-        int              ret                  = -1;
-        char               confpath[PATH_MAX] = "";
-        glusterd_conf_t *priv                 = NULL;
-        char            *slave                = NULL;
-
-        GF_ASSERT (THIS);
-        priv = THIS->private;
-        GF_ASSERT (priv);
-
-        ret = dict_get_str (dict, "slave", &slave);
-        if (ret || !slave) {
-                gf_log ("", GF_LOG_ERROR, "Unable to fetch slave from dict");
-                ret = -1;
-                goto out;
-        }
-
-        ret = glusterd_get_slave_info (slave, slave_url,
-                                       slave_host, slave_vol, op_errstr);
-        if (ret) {
-                gf_log ("", GF_LOG_ERROR,
-                        "Unable to fetch slave details.");
-                ret = -1;
-                goto out;
-        }
-
-        ret = dict_set_str (dict, "slave_url", *slave_url);
-        if (ret) {
-                gf_log ("", GF_LOG_ERROR,
-                        "Unable to store slave IP.");
-                goto out;
-        }
-
-        ret = dict_set_str (dict, "slave_host", *slave_host);
-        if (ret) {
-                gf_log ("", GF_LOG_ERROR,
-                        "Unable to store slave hostname");
-                goto out;
-        }
-
-        ret = dict_set_str (dict, "slave_vol", *slave_vol);
-        if (ret) {
-                gf_log ("", GF_LOG_ERROR,
-                        "Unable to store slave volume name.");
-                goto out;
-        }
-
-        ret = snprintf (confpath, sizeof(confpath) - 1,
-                        "%s/"GEOREP"/%s_%s_%s/gsyncd.conf",
-                        priv->workdir, volinfo->volname,
-                        *slave_host, *slave_vol);
-        confpath[ret] = '\0';
-        *conf_path = gf_strdup (confpath);
-        if (!(*conf_path)) {
-                gf_log ("", GF_LOG_ERROR,
-                        "Unable to gf_strdup. Error: %s", strerror (errno));
-                ret = -1;
-                goto out;
-        }
-
-        ret = dict_set_str (dict, "conf_path", *conf_path);
-        if (ret) {
-                gf_log ("", GF_LOG_ERROR,
-                        "Unable to store conf_path");
-                goto out;
-        }
+    int ret = -1;
+    char confpath[PATH_MAX] = "";
+    glusterd_conf_t *priv = NULL;
+    char *slave = NULL;
+    xlator_t *this = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+
+    priv = this->private;
+    GF_ASSERT(priv);
+
+    ret = dict_get_str(dict, "slave", &slave);
+    if (ret || !slave) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+               "Unable to fetch slave from dict");
+        ret = -1;
+        goto out;
+    }
+
+    ret = glusterd_get_slave_info(slave, slave_url, slave_host, slave_vol,
+                                  op_errstr);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_SLAVEINFO_FETCH_ERROR,
+               "Unable to fetch slave details.");
+        ret = -1;
+        goto out;
+    }
+
+    ret = dict_set_str(dict, "slave_url", *slave_url);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+               "Unable to store slave IP.");
+        goto out;
+    }
+
+    ret = dict_set_str(dict, "slave_host", *slave_host);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+               "Unable to store slave hostname");
+        goto out;
+    }
+
+    ret = dict_set_str(dict, "slave_vol", *slave_vol);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+               "Unable to store slave volume name.");
+        goto out;
+    }
+
+    ret = snprintf(confpath, sizeof(confpath) - 1,
+                   "%s/" GEOREP "/%s_%s_%s/gsyncd.conf", priv->workdir,
+                   volinfo->volname, *slave_host, *slave_vol);
+    confpath[ret] = '\0';
+    *conf_path = gf_strdup(confpath);
+    if (!(*conf_path)) {
+        gf_msg(this->name, GF_LOG_ERROR, errno, GD_MSG_STRDUP_FAILED,
+               "Unable to gf_strdup. Error: %s", strerror(errno));
+        ret = -1;
+        goto out;
+    }
+
+    ret = dict_set_str(dict, "conf_path", *conf_path);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+               "Unable to store conf_path");
+        goto out;
+    }
 
 out:
-        gf_log ("", GF_LOG_DEBUG,"Returning %d", ret);
-        return ret;
-
+    gf_msg_debug(this->name, 0, "Returning %d", ret);
+    return ret;
 }
 
 int
-glusterd_get_slave_info (char *slave,
-                         char **slave_url, char **hostname,
-                         char **slave_vol, char **op_errstr)
+glusterd_get_slave_info(char *slave, char **slave_url, char **hostname,
+                        char **slave_vol, char **op_errstr)
 {
-        char     *tmp              = NULL;
-        char     *save_ptr         = NULL;
-        char    **linearr          = NULL;
-        int32_t   ret              = -1;
-        char      errmsg[PATH_MAX] = "";
-        xlator_t *this             = NULL;
-
-        this = THIS;
-        GF_ASSERT (this);
-
-        ret = glusterd_urltransform_single (slave, "normalize",
-                                            &linearr);
-        if (ret == -1) {
-                ret = snprintf (errmsg, sizeof(errmsg) - 1,
-                                "Invalid Url: %s", slave);
-                errmsg[ret] = '\0';
-                *op_errstr = gf_strdup (errmsg);
-                gf_log (this->name, GF_LOG_ERROR, "Failed to normalize url");
-                goto out;
-        }
-
-        tmp = strtok_r (linearr[0], "/", &save_ptr);
-        tmp = strtok_r (NULL, "/", &save_ptr);
-        slave = strtok_r (tmp, ":", &save_ptr);
-        if (slave) {
-                ret = glusterd_geo_rep_parse_slave (slave, hostname, op_errstr);
-                if (ret) {
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "Invalid slave url: %s", *op_errstr);
-                        goto out;
-                }
-                gf_log (this->name, GF_LOG_DEBUG, "Hostname : %s", *hostname);
-
-                *slave_url = gf_strdup (slave);
-                if (!*slave_url) {
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "Failed to gf_strdup");
-                        ret = -1;
-                        goto out;
-                }
-                gf_log (this->name, GF_LOG_DEBUG, "Slave URL : %s", *slave_url);
-                ret = 0;
-        } else {
-                gf_log (this->name, GF_LOG_ERROR, "Invalid slave name");
-                goto out;
+    char *tmp = NULL;
+    char *save_ptr = NULL;
+    char **linearr = NULL;
+    int32_t ret = -1;
+    char errmsg[PATH_MAX] = "";
+    xlator_t *this = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+
+    ret = glusterd_urltransform_single(slave, "normalize", &linearr);
+    if ((ret == -1) || (linearr[0] == NULL)) {
+        ret = snprintf(errmsg, sizeof(errmsg) - 1, "Invalid Url: %s", slave);
+        errmsg[ret] = '\0';
+        *op_errstr = gf_strdup(errmsg);
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_NORMALIZE_URL_FAIL,
+               "Failed to normalize url");
+        goto out;
+    }
+
+    tmp = strtok_r(linearr[0], "/", &save_ptr);
+    tmp = strtok_r(NULL, "/", &save_ptr);
+    slave = NULL;
+    if (tmp != NULL) {
+        slave = strtok_r(tmp, ":", &save_ptr);
+    }
+    if (slave) {
+        ret = glusterd_geo_rep_parse_slave(slave, hostname, op_errstr);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_SLAVE_URL_INVALID,
+                   "Invalid slave url: %s", *op_errstr);
+            goto out;
         }
+        gf_msg_debug(this->name, 0, "Hostname : %s", *hostname);
 
-        slave = strtok_r (NULL, ":", &save_ptr);
-        if (slave) {
-                *slave_vol = gf_strdup (slave);
-                if (!*slave_vol) {
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "Failed to gf_strdup");
-                        ret = -1;
-                        GF_FREE (*slave_url);
-                        goto out;
-                }
-                gf_log (this->name, GF_LOG_DEBUG, "Slave Vol : %s", *slave_vol);
-                ret = 0;
-        } else {
-                gf_log (this->name, GF_LOG_ERROR, "Invalid slave name");
-                goto out;
+        *slave_url = gf_strdup(slave);
+        if (!*slave_url) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_STRDUP_FAILED,
+                   "Failed to gf_strdup");
+            ret = -1;
+            goto out;
         }
+        gf_msg_debug(this->name, 0, "Slave URL : %s", *slave_url);
+        ret = 0;
+    } else {
+        gf_msg(this->name, GF_LOG_ERROR, EINVAL, GD_MSG_INVALID_ENTRY,
+               "Invalid slave name");
+        goto out;
+    }
+
+    slave = strtok_r(NULL, ":", &save_ptr);
+    if (slave) {
+        *slave_vol = gf_strdup(slave);
+        if (!*slave_vol) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_STRDUP_FAILED,
+                   "Failed to gf_strdup");
+            ret = -1;
+            GF_FREE(*slave_url);
+            goto out;
+        }
+        gf_msg_debug(this->name, 0, "Slave Vol : %s", *slave_vol);
+        ret = 0;
+    } else {
+        gf_msg(this->name, GF_LOG_ERROR, EINVAL, GD_MSG_INVALID_ENTRY,
+               "Invalid slave name");
+        goto out;
+    }
 
 out:
-        gf_log (this->name, GF_LOG_DEBUG, "Returning %d", ret);
-        return ret;
+    if (linearr)
+        glusterd_urltransform_free(linearr, 1);
+    gf_msg_debug(this->name, 0, "Returning %d", ret);
+    return ret;
 }
 
 static void
-runinit_gsyncd_setrx (runner_t *runner, char *conf_path)
+runinit_gsyncd_setrx(runner_t *runner, char *conf_path)
 {
-        runinit (runner);
-        runner_add_args (runner, GSYNCD_PREFIX"/gsyncd", "-c", NULL);
-        runner_argprintf (runner, "%s", conf_path);
-        runner_add_arg (runner, "--config-set-rx");
+    runinit(runner);
+    runner_add_args(runner, GSYNCD_PREFIX "/gsyncd", "-c", NULL);
+    runner_argprintf(runner, "%s", conf_path);
+    runner_add_arg(runner, "--config-set-rx");
 }
 
 static int
-glusterd_check_gsync_present (int *valid_state)
+glusterd_check_gsync_present(int *valid_state)
 {
-        char                buff[PATH_MAX] = {0, };
-        runner_t            runner = {0,};
-        char               *ptr = NULL;
-        int                 ret = 0;
-
-        runinit (&runner);
-        runner_add_args (&runner, GSYNCD_PREFIX"/gsyncd", "--version", NULL);
-        runner_redir (&runner, STDOUT_FILENO, RUN_PIPE);
-        ret = runner_start (&runner);
-        if (ret == -1) {
-                if (errno == ENOENT) {
-                        gf_log ("glusterd", GF_LOG_INFO, GEOREP
-                                 " module not installed in the system");
-                        *valid_state = 0;
-                }
-                else {
-                        gf_log ("glusterd", GF_LOG_ERROR, GEOREP
-                                  " module not working as desired");
-                        *valid_state = -1;
-                }
-                goto out;
-        }
-
-        ptr = fgets(buff, sizeof(buff), runner_chio (&runner, STDOUT_FILENO));
-        if (ptr) {
-                if (!strstr (buff, "gsyncd")) {
-                        ret = -1;
-                        gf_log ("glusterd", GF_LOG_ERROR, GEOREP" module not "
-                                 "working as desired");
-                        *valid_state = -1;
-                        goto out;
-                }
+    char buff[PATH_MAX] = {
+        0,
+    };
+    runner_t runner = {
+        0,
+    };
+    char *ptr = NULL;
+    int ret = 0;
+    xlator_t *this = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+
+    runinit(&runner);
+    runner_add_args(&runner, GSYNCD_PREFIX "/gsyncd", "--version", NULL);
+    runner_redir(&runner, STDOUT_FILENO, RUN_PIPE);
+    ret = runner_start(&runner);
+    if (ret == -1) {
+        if (errno == ENOENT) {
+            gf_msg("glusterd", GF_LOG_INFO, ENOENT, GD_MSG_MODULE_NOT_INSTALLED,
+                   GEOREP
+                   " module "
+                   "not installed in the system");
+            *valid_state = 0;
         } else {
-                ret = -1;
-                gf_log ("glusterd", GF_LOG_ERROR, GEOREP" module not "
-                         "working as desired");
-                *valid_state = -1;
-                goto out;
-        }
-
-        ret = 0;
- out:
-
-        runner_end (&runner);
+            gf_msg("glusterd", GF_LOG_ERROR, 0, GD_MSG_MODULE_ERROR,
+                   GEOREP " module not working as desired");
+            *valid_state = -1;
+        }
+        goto out;
+    }
+
+    ptr = fgets(buff, sizeof(buff), runner_chio(&runner, STDOUT_FILENO));
+    if (ptr) {
+        if (!strstr(buff, "gsyncd")) {
+            ret = -1;
+            gf_msg("glusterd", GF_LOG_ERROR, 0, GD_MSG_MODULE_ERROR,
+                   GEOREP " module not working as desired");
+            *valid_state = -1;
+            goto out;
+        }
+    } else {
+        ret = -1;
+        gf_msg("glusterd", GF_LOG_ERROR, 0, GD_MSG_MODULE_ERROR,
+               GEOREP " module not working as desired");
+        *valid_state = -1;
+        goto out;
+    }
+
+    ret = 0;
+out:
 
-        gf_log ("glusterd", GF_LOG_DEBUG, "Returning %d", ret);
-        return ret;
+    runner_end(&runner);
 
+    gf_msg_debug("glusterd", 0, "Returning %d", ret);
+    return ret;
 }
 
 static int
-create_conf_file (glusterd_conf_t *conf, char *conf_path)
-#define RUN_GSYNCD_CMD do {                                                          \
-        ret = runner_run_reuse (&runner);                                            \
-        if (ret == -1) {                                                             \
-                runner_log (&runner, "glusterd", GF_LOG_ERROR, "command failed");    \
-                runner_end (&runner);                                                \
-                goto out;                                                            \
-        }                                                                            \
-        runner_end (&runner);                                                        \
-} while (0)
+create_conf_file(glusterd_conf_t *conf, char *conf_path)
+#define RUN_GSYNCD_CMD                                                         \
+    do {                                                                       \
+        ret = runner_run_reuse(&runner);                                       \
+        if (ret == -1) {                                                       \
+            runner_log(&runner, "glusterd", GF_LOG_ERROR, "command failed");   \
+            runner_end(&runner);                                               \
+            goto out;                                                          \
+        }                                                                      \
+        runner_end(&runner);                                                   \
+    } while (0)
 {
-        int ret = 0;
-        runner_t runner = {0,};
-        char georepdir[PATH_MAX] = {0,};
-        int valid_state = 0;
-
-        valid_state = -1;
-        ret = glusterd_check_gsync_present (&valid_state);
-        if (-1 == ret) {
-                ret = valid_state;
-                goto out;
-        }
+    int ret = 0;
+    runner_t runner = {
+        0,
+    };
+    char georepdir[PATH_MAX] = {
+        0,
+    };
+    int valid_state = 0;
+
+    valid_state = -1;
+    ret = glusterd_check_gsync_present(&valid_state);
+    if (-1 == ret) {
+        ret = valid_state;
+        goto out;
+    }
+
+    ret = snprintf(georepdir, sizeof(georepdir) - 1, "%s/" GEOREP,
+                   conf->workdir);
+    georepdir[ret] = '\0';
+
+    /************
+     * master pre-configuration
+     ************/
+
+    /* remote-gsyncd */
+    runinit_gsyncd_setrx(&runner, conf_path);
+    runner_add_args(&runner, "remote-gsyncd", GSYNCD_PREFIX "/gsyncd", ".", ".",
+                    NULL);
+    RUN_GSYNCD_CMD;
+
+    runinit_gsyncd_setrx(&runner, conf_path);
+    runner_add_args(&runner, "remote-gsyncd", "/nonexistent/gsyncd", ".",
+                    "^ssh:", NULL);
+    RUN_GSYNCD_CMD;
+
+    /* gluster-command-dir */
+    runinit_gsyncd_setrx(&runner, conf_path);
+    runner_add_args(&runner, "gluster-command-dir", SBIN_DIR "/", ".", ".",
+                    NULL);
+    RUN_GSYNCD_CMD;
+
+    /* gluster-params */
+    runinit_gsyncd_setrx(&runner, conf_path);
+    runner_add_args(&runner, "gluster-params", "aux-gfid-mount acl", ".", ".",
+                    NULL);
+    RUN_GSYNCD_CMD;
+
+    /* ssh-command */
+    runinit_gsyncd_setrx(&runner, conf_path);
+    runner_add_arg(&runner, "ssh-command");
+    runner_argprintf(&runner,
+                     "ssh -oPasswordAuthentication=no "
+                     "-oStrictHostKeyChecking=no "
+                     "-i %s/secret.pem",
+                     georepdir);
+    runner_add_args(&runner, ".", ".", NULL);
+    RUN_GSYNCD_CMD;
+
+    /* ssh-command tar */
+    runinit_gsyncd_setrx(&runner, conf_path);
+    runner_add_arg(&runner, "ssh-command-tar");
+    runner_argprintf(&runner,
+                     "ssh -oPasswordAuthentication=no "
+                     "-oStrictHostKeyChecking=no "
+                     "-i %s/tar_ssh.pem",
+                     georepdir);
+    runner_add_args(&runner, ".", ".", NULL);
+    RUN_GSYNCD_CMD;
+
+    /* pid-file */
+    runinit_gsyncd_setrx(&runner, conf_path);
+    runner_add_arg(&runner, "pid-file");
+    runner_argprintf(&runner,
+                     "%s/${mastervol}_${remotehost}_${slavevol}/monitor.pid",
+                     georepdir);
+    runner_add_args(&runner, ".", ".", NULL);
+    RUN_GSYNCD_CMD;
+
+    /* geo-rep-working-dir */
+    runinit_gsyncd_setrx(&runner, conf_path);
+    runner_add_arg(&runner, "georep-session-working-dir");
+    runner_argprintf(&runner, "%s/${mastervol}_${remotehost}_${slavevol}/",
+                     georepdir);
+    runner_add_args(&runner, ".", ".", NULL);
+    RUN_GSYNCD_CMD;
+
+    /* state-file */
+    runinit_gsyncd_setrx(&runner, conf_path);
+    runner_add_arg(&runner, "state-file");
+    runner_argprintf(&runner,
+                     "%s/${mastervol}_${remotehost}_${slavevol}/monitor.status",
+                     georepdir);
+    runner_add_args(&runner, ".", ".", NULL);
+    RUN_GSYNCD_CMD;
+
+    /* state-detail-file */
+    runinit_gsyncd_setrx(&runner, conf_path);
+    runner_add_arg(&runner, "state-detail-file");
+    runner_argprintf(
+        &runner,
+        "%s/${mastervol}_${remotehost}_${slavevol}/${eSlave}-detail.status",
+        georepdir);
+    runner_add_args(&runner, ".", ".", NULL);
+    RUN_GSYNCD_CMD;
+
+    /* state-socket */
+    runinit_gsyncd_setrx(&runner, conf_path);
+    runner_add_arg(&runner, "state-socket-unencoded");
+    runner_argprintf(
+        &runner, "%s/${mastervol}_${remotehost}_${slavevol}/${eSlave}.socket",
+        georepdir);
+    runner_add_args(&runner, ".", ".", NULL);
+    RUN_GSYNCD_CMD;
+
+    /* socketdir */
+    runinit_gsyncd_setrx(&runner, conf_path);
+    runner_add_args(&runner, "socketdir", GLUSTERD_SOCK_DIR, ".", ".", NULL);
+    RUN_GSYNCD_CMD;
+
+    /* log-file */
+    runinit_gsyncd_setrx(&runner, conf_path);
+    runner_add_arg(&runner, "log-file");
+    runner_argprintf(&runner, "%s/%s/${mastervol}/${eSlave}.log", conf->logdir,
+                     GEOREP);
+    runner_add_args(&runner, ".", ".", NULL);
+    RUN_GSYNCD_CMD;
+
+    /* changelog-log-file */
+    runinit_gsyncd_setrx(&runner, conf_path);
+    runner_add_arg(&runner, "changelog-log-file");
+    runner_argprintf(&runner,
+                     "%s/%s/${mastervol}/${eSlave}${local_id}-changes.log",
+                     conf->logdir, GEOREP);
+    runner_add_args(&runner, ".", ".", NULL);
+    RUN_GSYNCD_CMD;
+
+    /* gluster-log-file */
+    runinit_gsyncd_setrx(&runner, conf_path);
+    runner_add_arg(&runner, "gluster-log-file");
+    runner_argprintf(&runner,
+                     "%s/%s/${mastervol}/${eSlave}${local_id}.gluster.log",
+                     conf->logdir, GEOREP);
+    runner_add_args(&runner, ".", ".", NULL);
+    RUN_GSYNCD_CMD;
+
+    /* ignore-deletes */
+    runinit_gsyncd_setrx(&runner, conf_path);
+    runner_add_args(&runner, "ignore-deletes", "false", ".", ".", NULL);
+    RUN_GSYNCD_CMD;
+
+    /* special-sync-mode */
+    runinit_gsyncd_setrx(&runner, conf_path);
+    runner_add_args(&runner, "special-sync-mode", "partial", ".", ".", NULL);
+    RUN_GSYNCD_CMD;
+
+    /* change-detector == changelog */
+    runinit_gsyncd_setrx(&runner, conf_path);
+    runner_add_args(&runner, "change-detector", "changelog", ".", ".", NULL);
+    RUN_GSYNCD_CMD;
+
+    runinit_gsyncd_setrx(&runner, conf_path);
+    runner_add_arg(&runner, "working-dir");
+    runner_argprintf(&runner, "%s/${mastervol}/${eSlave}",
+                     DEFAULT_GLUSTERFSD_MISC_DIRETORY);
+    runner_add_args(&runner, ".", ".", NULL);
+    RUN_GSYNCD_CMD;
+
+    /************
+     * slave pre-configuration
+     ************/
+
+    /* slave-gluster-command-dir */
+    runinit_gsyncd_setrx(&runner, conf_path);
+    runner_add_args(&runner, "slave-gluster-command-dir", SBIN_DIR "/", ".",
+                    NULL);
+    RUN_GSYNCD_CMD;
+
+    /* gluster-params */
+    runinit_gsyncd_setrx(&runner, conf_path);
+    runner_add_args(&runner, "gluster-params", "aux-gfid-mount acl", ".", NULL);
+    RUN_GSYNCD_CMD;
+
+    /* log-file */
+    runinit_gsyncd_setrx(&runner, conf_path);
+    runner_add_arg(&runner, "log-file");
+    runner_argprintf(&runner,
+                     "%s/%s-slaves/"
+                     "${session_owner}:${local_node}${local_id}.${slavevol}."
+                     "log",
+                     conf->logdir, GEOREP);
+    runner_add_args(&runner, ".", ".", NULL);
+    RUN_GSYNCD_CMD;
+
+    /* MountBroker log-file */
+    runinit_gsyncd_setrx(&runner, conf_path);
+    runner_add_arg(&runner, "log-file-mbr");
+    runner_argprintf(&runner,
+                     "%s/%s-slaves/mbr/"
+                     "${session_owner}:${local_node}${local_id}.${slavevol}."
+                     "log",
+                     conf->logdir, GEOREP);
+    runner_add_args(&runner, ".", ".", NULL);
+    RUN_GSYNCD_CMD;
+
+    /* gluster-log-file */
+    runinit_gsyncd_setrx(&runner, conf_path);
+    runner_add_arg(&runner, "gluster-log-file");
+    runner_argprintf(&runner,
+                     "%s/%s-slaves/"
+                     "${session_owner}:${local_node}${local_id}.${slavevol}."
+                     "gluster.log",
+                     conf->logdir, GEOREP);
+    runner_add_args(&runner, ".", ".", NULL);
+    RUN_GSYNCD_CMD;
 
-        ret = snprintf (georepdir, sizeof(georepdir) - 1, "%s/"GEOREP,
-                        conf->workdir);
-        georepdir[ret] = '\0';
-
-        /************
-         * master pre-configuration
-         ************/
-
-        /* remote-gsyncd */
-        runinit_gsyncd_setrx (&runner, conf_path);
-        runner_add_args (&runner, "remote-gsyncd", GSYNCD_PREFIX"/gsyncd", ".", ".", NULL);
-        RUN_GSYNCD_CMD;
-
-        runinit_gsyncd_setrx (&runner, conf_path);
-        runner_add_args (&runner, "remote-gsyncd", "/nonexistent/gsyncd",
-                         ".", "^ssh:", NULL);
-        RUN_GSYNCD_CMD;
-
-        /* gluster-command-dir */
-        runinit_gsyncd_setrx (&runner, conf_path);
-        runner_add_args (&runner, "gluster-command-dir", SBIN_DIR"/",
-                         ".", ".", NULL);
-        RUN_GSYNCD_CMD;
-
-        /* gluster-params */
-        runinit_gsyncd_setrx (&runner, conf_path);
-        runner_add_args (&runner, "gluster-params",
-                         "aux-gfid-mount acl",
-                         ".", ".", NULL);
-        RUN_GSYNCD_CMD;
-
-        /* ssh-command */
-        runinit_gsyncd_setrx (&runner, conf_path);
-        runner_add_arg (&runner, "ssh-command");
-        runner_argprintf (&runner,
-                          "ssh -oPasswordAuthentication=no "
-                           "-oStrictHostKeyChecking=no "
-                           "-i %s/secret.pem", georepdir);
-        runner_add_args (&runner, ".", ".", NULL);
-        RUN_GSYNCD_CMD;
-
-        /* ssh-command tar */
-        runinit_gsyncd_setrx (&runner, conf_path);
-        runner_add_arg (&runner, "ssh-command-tar");
-        runner_argprintf (&runner,
-                          "ssh -oPasswordAuthentication=no "
-                           "-oStrictHostKeyChecking=no "
-                           "-i %s/tar_ssh.pem", georepdir);
-        runner_add_args (&runner, ".", ".", NULL);
-        RUN_GSYNCD_CMD;
-
-        /* pid-file */
-        runinit_gsyncd_setrx (&runner, conf_path);
-        runner_add_arg (&runner, "pid-file");
-        runner_argprintf (&runner, "%s/${mastervol}_${remotehost}_${slavevol}/${eSlave}.pid", georepdir);
-        runner_add_args (&runner, ".", ".", NULL);
-        RUN_GSYNCD_CMD;
-
-        /* geo-rep-working-dir */
-        runinit_gsyncd_setrx (&runner, conf_path);
-        runner_add_arg (&runner, "georep-session-working-dir");
-        runner_argprintf (&runner, "%s/${mastervol}_${remotehost}_${slavevol}/", georepdir);
-        runner_add_args (&runner, ".", ".", NULL);
-        RUN_GSYNCD_CMD;
-
-        /* state-file */
-        runinit_gsyncd_setrx (&runner, conf_path);
-        runner_add_arg (&runner, "state-file");
-        runner_argprintf (&runner, "%s/${mastervol}_${remotehost}_${slavevol}/${eSlave}.status", georepdir);
-        runner_add_args (&runner, ".", ".", NULL);
-        RUN_GSYNCD_CMD;
-
-        /* state-detail-file */
-        runinit_gsyncd_setrx (&runner, conf_path);
-        runner_add_arg (&runner, "state-detail-file");
-        runner_argprintf (&runner, "%s/${mastervol}_${remotehost}_${slavevol}/${eSlave}-detail.status", georepdir);
-        runner_add_args (&runner, ".", ".", NULL);
-        RUN_GSYNCD_CMD;
-
-        /* state-socket */
-        runinit_gsyncd_setrx (&runner, conf_path);
-        runner_add_arg (&runner, "state-socket-unencoded");
-        runner_argprintf (&runner, "%s/${mastervol}_${remotehost}_${slavevol}/${eSlave}.socket", georepdir);
-        runner_add_args (&runner, ".", ".", NULL);
-        RUN_GSYNCD_CMD;
-
-        /* socketdir */
-        runinit_gsyncd_setrx (&runner, conf_path);
-        runner_add_args (&runner, "socketdir", GLUSTERD_SOCK_DIR, ".", ".", NULL);
-        RUN_GSYNCD_CMD;
-
-        /* log-file */
-        runinit_gsyncd_setrx (&runner, conf_path);
-        runner_add_args (&runner,
-                         "log-file",
-                         DEFAULT_LOG_FILE_DIRECTORY"/"GEOREP"/${mastervol}/${eSlave}.log",
-                         ".", ".", NULL);
-        RUN_GSYNCD_CMD;
-
-        /* changelog-log-file */
-        runinit_gsyncd_setrx (&runner, conf_path);
-        runner_add_args (&runner,
-                         "changelog-log-file",
-                         DEFAULT_LOG_FILE_DIRECTORY"/"GEOREP"/${mastervol}/${eSlave}${local_id}-changes.log",
-                         ".", ".", NULL);
-        RUN_GSYNCD_CMD;
-
-        /* gluster-log-file */
-        runinit_gsyncd_setrx (&runner, conf_path);
-        runner_add_args (&runner,
-                         "gluster-log-file",
-                         DEFAULT_LOG_FILE_DIRECTORY"/"GEOREP"/${mastervol}/${eSlave}${local_id}.gluster.log",
-                         ".", ".", NULL);
-        RUN_GSYNCD_CMD;
-
-        /* ignore-deletes */
-        runinit_gsyncd_setrx (&runner, conf_path);
-        runner_add_args (&runner, "ignore-deletes", "true", ".", ".", NULL);
-        RUN_GSYNCD_CMD;
-
-        /* special-sync-mode */
-        runinit_gsyncd_setrx (&runner, conf_path);
-        runner_add_args (&runner, "special-sync-mode", "partial", ".", ".", NULL);
-        RUN_GSYNCD_CMD;
-
-        /* change-detector == changelog */
-        runinit_gsyncd_setrx (&runner, conf_path);
-        runner_add_args(&runner, "change-detector", "changelog", ".", ".", NULL);
-        RUN_GSYNCD_CMD;
-
-        runinit_gsyncd_setrx (&runner, conf_path);
-        runner_add_arg(&runner, "working-dir");
-        runner_argprintf(&runner, "%s/${mastervol}/${eSlave}",
-                         DEFAULT_GLUSTERFSD_MISC_DIRETORY);
-        runner_add_args (&runner, ".", ".", NULL);
-        RUN_GSYNCD_CMD;
-
-        /************
-         * slave pre-configuration
-         ************/
-
-        /* gluster-command-dir */
-        runinit_gsyncd_setrx (&runner, conf_path);
-        runner_add_args (&runner, "gluster-command-dir", SBIN_DIR"/",
-                         ".", NULL);
-        RUN_GSYNCD_CMD;
-
-        /* gluster-params */
-        runinit_gsyncd_setrx (&runner, conf_path);
-        runner_add_args (&runner, "gluster-params",
-                         "aux-gfid-mount acl",
-                         ".", NULL);
-        RUN_GSYNCD_CMD;
-
-        /* log-file */
-        runinit_gsyncd_setrx (&runner, conf_path);
-        runner_add_args (&runner,
-                         "log-file",
-                         DEFAULT_LOG_FILE_DIRECTORY"/"GEOREP"-slaves/${session_owner}:${eSlave}.log",
-                         ".", NULL);
-        RUN_GSYNCD_CMD;
-
-        /* MountBroker log-file */
-        runinit_gsyncd_setrx (&runner, conf_path);
-        runner_add_args (&runner,
-                         "log-file-mbr",
-                         DEFAULT_LOG_FILE_DIRECTORY"/"GEOREP"-slaves/mbr/${session_owner}:${eSlave}.log",
-                         ".", NULL);
-        RUN_GSYNCD_CMD;
-
-        /* gluster-log-file */
-        runinit_gsyncd_setrx (&runner, conf_path);
-        runner_add_args (&runner,
-                         "gluster-log-file",
-                         DEFAULT_LOG_FILE_DIRECTORY"/"GEOREP"-slaves/${session_owner}:${eSlave}.gluster.log",
-                         ".", NULL);
-        RUN_GSYNCD_CMD;
-
- out:
-        return ret ? -1 : 0;
+out:
+    return ret ? -1 : 0;
 }
 
 static int
-glusterd_create_essential_dir_files (glusterd_volinfo_t *volinfo, dict_t *dict,
-                                     char *slave, char *slave_host,
-                                     char *slave_vol, char **op_errstr)
+glusterd_create_essential_dir_files(glusterd_volinfo_t *volinfo, dict_t *dict,
+                                    char *slave, char *slave_host,
+                                    char *slave_vol, char **op_errstr)
 {
-        int                ret              = -1;
-        char              *conf_path        = NULL;
-        char              *statefile        = NULL;
-        char               buf[PATH_MAX]    = "";
-        char               errmsg[PATH_MAX] = "";
-        glusterd_conf_t   *conf             = NULL;
-        struct stat        stbuf            = {0,};
-
-        GF_ASSERT (THIS);
-        conf = THIS->private;
-
-        ret = dict_get_str (dict, "conf_path", &conf_path);
-        if (ret) {
-                snprintf (errmsg, sizeof (errmsg),
-                          "Unable to fetch conf file path.");
-                gf_log ("", GF_LOG_ERROR, "%s", errmsg);
-                goto out;
-        }
-
-        ret = dict_get_str (dict, "statefile", &statefile);
-        if (ret) {
-                snprintf (errmsg, sizeof (errmsg),
-                          "Unable to fetch statefile path.");
-                gf_log ("", GF_LOG_ERROR, "%s", errmsg);
-                goto out;
-        }
-
-        ret = snprintf (buf, sizeof(buf) - 1, "%s/"GEOREP"/%s_%s_%s",
-                        conf->workdir, volinfo->volname, slave_host, slave_vol);
-        buf[ret] = '\0';
-        ret = mkdir_p (buf, 0777, _gf_true);
-        if (ret) {
-                snprintf (errmsg, sizeof (errmsg), "Unable to create %s"
-                          ". Error : %s", buf, strerror (errno));
-                *op_errstr = gf_strdup (errmsg);
-                gf_log ("", GF_LOG_ERROR, "%s", errmsg);
-                goto out;
-        }
-
-        ret = snprintf (buf, PATH_MAX, DEFAULT_LOG_FILE_DIRECTORY"/"GEOREP"/%s",
-                        volinfo->volname);
-        buf[ret] = '\0';
-        ret = mkdir_p (buf, 0777, _gf_true);
-        if (ret) {
-                snprintf (errmsg, sizeof (errmsg), "Unable to create %s"
-                          ". Error : %s", buf, strerror (errno));
-                *op_errstr = gf_strdup (errmsg);
-                gf_log ("", GF_LOG_ERROR, "%s", errmsg);
-                goto out;
-        }
-
-        ret = lstat (conf_path, &stbuf);
-        if (!ret) {
-                gf_log ("", GF_LOG_DEBUG, "Session already running."
-                        " Not creating config file again.");
-        } else {
-                ret = create_conf_file (conf, conf_path);
-                if (ret || lstat (conf_path, &stbuf)) {
-                        snprintf (errmsg, sizeof (errmsg), "Failed to create"
-                                  " config file(%s).", conf_path);
-                        gf_log ("", GF_LOG_ERROR, "%s", errmsg);
-                        goto out;
-                }
-        }
-
-        ret = lstat (statefile, &stbuf);
-        if (!ret) {
-                gf_log ("", GF_LOG_DEBUG, "Session already running."
-                        " Not creating status file again.");
-                goto out;
-        } else {
-                ret = glusterd_create_status_file (volinfo->volname, slave,
-                                                   slave_host, slave_vol,
-                                                   "Not Started");
-                if (ret || lstat (statefile, &stbuf)) {
-                        snprintf (errmsg, sizeof (errmsg), "Unable to create %s"
-                                  ". Error : %s", statefile, strerror (errno));
-                        *op_errstr = gf_strdup (errmsg);
-                        gf_log ("", GF_LOG_ERROR, "%s", errmsg);
-                        ret = -1;
-                        goto out;
-                }
-        }
+    int ret = -1;
+    char *conf_path = NULL;
+    char *statefile = NULL;
+    char buf[PATH_MAX] = "";
+    char errmsg[PATH_MAX] = "";
+    glusterd_conf_t *conf = NULL;
+    struct stat stbuf = {
+        0,
+    };
+    xlator_t *this = NULL;
+    int32_t len = 0;
+
+    this = THIS;
+    GF_ASSERT(this);
+
+    conf = this->private;
+
+    ret = dict_get_str(dict, "conf_path", &conf_path);
+    if (ret) {
+        snprintf(errmsg, sizeof(errmsg), "Unable to fetch conf file path.");
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED, "%s",
+               errmsg);
+        goto out;
+    }
+
+    ret = dict_get_str(dict, "statefile", &statefile);
+    if (ret) {
+        snprintf(errmsg, sizeof(errmsg), "Unable to fetch statefile path.");
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED, "%s",
+               errmsg);
+        goto out;
+    }
+
+    ret = snprintf(buf, sizeof(buf), "%s/" GEOREP "/%s_%s_%s", conf->workdir,
+                   volinfo->volname, slave_host, slave_vol);
+    if ((ret < 0) || (ret >= sizeof(buf))) {
+        ret = -1;
+        goto out;
+    }
+    ret = mkdir_p(buf, 0755, _gf_true);
+    if (ret) {
+        len = snprintf(errmsg, sizeof(errmsg),
+                       "Unable to create %s"
+                       ". Error : %s",
+                       buf, strerror(errno));
+        if (len < 0) {
+            strcpy(errmsg, "<error>");
+        }
+        *op_errstr = gf_strdup(errmsg);
+        gf_msg(this->name, GF_LOG_ERROR, errno, GD_MSG_DIR_OP_FAILED, "%s",
+               errmsg);
+        goto out;
+    }
+
+    ret = snprintf(buf, PATH_MAX, "%s/" GEOREP "/%s", conf->logdir,
+                   volinfo->volname);
+    if ((ret < 0) || (ret >= PATH_MAX)) {
+        ret = -1;
+        goto out;
+    }
+    ret = mkdir_p(buf, 0755, _gf_true);
+    if (ret) {
+        len = snprintf(errmsg, sizeof(errmsg),
+                       "Unable to create %s"
+                       ". Error : %s",
+                       buf, strerror(errno));
+        if (len < 0) {
+            strcpy(errmsg, "<error>");
+        }
+        *op_errstr = gf_strdup(errmsg);
+        gf_msg(this->name, GF_LOG_ERROR, errno, GD_MSG_DIR_OP_FAILED, "%s",
+               errmsg);
+        goto out;
+    }
+
+    ret = sys_lstat(conf_path, &stbuf);
+    if (!ret) {
+        gf_msg_debug(this->name, 0,
+                     "Session already running."
+                     " Not creating config file again.");
+    } else {
+        ret = create_conf_file(conf, conf_path);
+        if (ret || sys_lstat(conf_path, &stbuf)) {
+            snprintf(errmsg, sizeof(errmsg),
+                     "Failed to create"
+                     " config file(%s).",
+                     conf_path);
+            gf_msg(this->name, GF_LOG_ERROR, errno, GD_MSG_FILE_OP_FAILED, "%s",
+                   errmsg);
+            goto out;
+        }
+    }
+
+    ret = sys_lstat(statefile, &stbuf);
+    if (!ret) {
+        gf_msg_debug(this->name, 0,
+                     "Session already running."
+                     " Not creating status file again.");
+        goto out;
+    } else {
+        ret = glusterd_create_status_file(volinfo->volname, slave, slave_host,
+                                          slave_vol, "Created");
+        if (ret || sys_lstat(statefile, &stbuf)) {
+            snprintf(errmsg, sizeof(errmsg),
+                     "Unable to create %s"
+                     ". Error : %s",
+                     statefile, strerror(errno));
+            *op_errstr = gf_strdup(errmsg);
+            gf_msg(this->name, GF_LOG_ERROR, errno, GD_MSG_FILE_OP_FAILED, "%s",
+                   errmsg);
+            ret = -1;
+            goto out;
+        }
+    }
 
 out:
-        gf_log ("", GF_LOG_DEBUG,"Returning %d", ret);
-        return ret;
+    gf_msg_debug(this->name, 0, "Returning %d", ret);
+    return ret;
 }
 
 int
-glusterd_op_gsync_create (dict_t *dict, char **op_errstr, dict_t *rsp_dict)
+glusterd_op_gsync_create(dict_t *dict, char **op_errstr, dict_t *rsp_dict)
 {
-        char                common_pem_file[PATH_MAX] = "";
-        char                errmsg[PATH_MAX]          = "";
-        char                hooks_args[PATH_MAX]      = "";
-        char                uuid_str [64]             = "";
-        char               *host_uuid                 = NULL;
-        char               *slave_url                 = NULL;
-        char               *slave_url_buf             = NULL;
-        char               *slave_user                = NULL;
-        char               *slave_ip                  = NULL;
-        char               *save_ptr                  = NULL;
-        char               *slave_host                = NULL;
-        char               *slave_vol                 = NULL;
-        char               *arg_buf                   = NULL;
-        char               *volname                   = NULL;
-        char               *slave                     = NULL;
-        int32_t             ret                       = -1;
-        int32_t             is_pem_push               = -1;
-        gf_boolean_t        is_force                  = -1;
-        glusterd_conf_t    *conf                      = NULL;
-        glusterd_volinfo_t *volinfo                   = NULL;
-        xlator_t           *this                      = NULL;
-
-        this = THIS;
-        GF_ASSERT (this);
-        conf = this->private;
-        GF_ASSERT (conf);
-        GF_ASSERT (dict);
-        GF_ASSERT (op_errstr);
-
-        ret = glusterd_op_gsync_args_get (dict, op_errstr,
-                                          &volname, &slave, &host_uuid);
-        if (ret)
-                goto out;
-
-        snprintf (common_pem_file, sizeof(common_pem_file),
-                  "%s"GLUSTERD_COMMON_PEM_PUB_FILE, conf->workdir);
+    char common_pem_file[PATH_MAX] = "";
+    char errmsg[PATH_MAX] = {
+        0,
+    };
+    char hooks_args[PATH_MAX] = "";
+    char uuid_str[64] = "";
+    char *host_uuid = NULL;
+    char *slave_url = NULL;
+    char *slave_url_buf = NULL;
+    char *slave_user = NULL;
+    char *slave_ip = NULL;
+    char *save_ptr = NULL;
+    char *slave_host = NULL;
+    char *slave_vol = NULL;
+    char *arg_buf = NULL;
+    char *volname = NULL;
+    char *slave = NULL;
+    int32_t ret = -1;
+    int32_t is_pem_push = -1;
+    int32_t ssh_port = 22;
+    gf_boolean_t is_force = -1;
+    glusterd_conf_t *conf = NULL;
+    glusterd_volinfo_t *volinfo = NULL;
+    xlator_t *this = NULL;
+    char old_working_dir[PATH_MAX] = {0};
+    char new_working_dir[PATH_MAX] = {0};
+    char *slave_voluuid = NULL;
+    char *old_slavehost = NULL;
+    gf_boolean_t is_existing_session = _gf_false;
+    int32_t len = 0;
+
+    this = THIS;
+    GF_ASSERT(this);
+    conf = this->private;
+    GF_ASSERT(conf);
+    GF_ASSERT(dict);
+    GF_ASSERT(op_errstr);
+
+    ret = glusterd_op_gsync_args_get(dict, op_errstr, &volname, &slave,
+                                     &host_uuid);
+    if (ret)
+        goto out;
+
+    len = snprintf(common_pem_file, sizeof(common_pem_file),
+                   "%s" GLUSTERD_COMMON_PEM_PUB_FILE, conf->workdir);
+    if ((len < 0) || (len >= sizeof(common_pem_file))) {
+        ret = -1;
+        goto out;
+    }
+
+    ret = glusterd_volinfo_find(volname, &volinfo);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_VOL_NOT_FOUND,
+               "Volinfo for %s (master) not found", volname);
+        goto out;
+    }
+
+    ret = dict_get_str(dict, "slave_vol", &slave_vol);
+    if (ret) {
+        snprintf(errmsg, sizeof(errmsg), "Unable to fetch slave volume name.");
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED, "%s",
+               errmsg);
+        goto out;
+    }
+
+    ret = dict_get_str(dict, "slave_url", &slave_url);
+    if (ret) {
+        snprintf(errmsg, sizeof(errmsg), "Unable to fetch slave IP.");
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED, "%s",
+               errmsg);
+        ret = -1;
+        goto out;
+    }
+
+    /* Fetch the slave_user and slave_ip from the slave_url.
+     * If the slave_user is not present. Use "root"
+     */
+    if (strstr(slave_url, "@")) {
+        slave_url_buf = gf_strdup(slave_url);
+        if (!slave_url_buf) {
+            ret = -1;
+            goto out;
+        }
+        slave_user = strtok_r(slave_url, "@", &save_ptr);
+        slave_ip = strtok_r(NULL, "@", &save_ptr);
+    } else {
+        slave_user = "root";
+        slave_ip = slave_url;
+    }
+
+    if (!slave_user || !slave_ip) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_SLAVE_URL_INVALID,
+               "Invalid slave url.");
+        ret = -1;
+        goto out;
+    }
+
+    ret = dict_get_str(dict, "slave_host", &slave_host);
+    if (ret) {
+        snprintf(errmsg, sizeof(errmsg), "Unable to fetch slave host");
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED, "%s",
+               errmsg);
+        ret = -1;
+        goto out;
+    }
+
+    ret = dict_get_int32(dict, "ssh_port", &ssh_port);
+    if (ret < 0 && ret != -ENOENT) {
+        snprintf(errmsg, sizeof(errmsg), "Fetching ssh_port failed");
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED, "%s",
+               errmsg);
+        ret = -1;
+        goto out;
+    }
+
+    is_force = dict_get_str_boolean(dict, "force", _gf_false);
+
+    uuid_utoa_r(MY_UUID, uuid_str);
+    if (!strcmp(uuid_str, host_uuid)) {
+        ret = dict_get_int32(dict, "push_pem", &is_pem_push);
+        if (!ret && is_pem_push) {
+            gf_msg_debug(this->name, 0,
+                         "Trying to setup"
+                         " pem files in slave");
+            is_pem_push = 1;
+        } else
+            is_pem_push = 0;
+
+        len = snprintf(hooks_args, sizeof(hooks_args),
+                       "is_push_pem=%d,pub_file=%s,slave_user=%s,"
+                       "slave_ip=%s,slave_vol=%s,ssh_port=%d",
+                       is_pem_push, common_pem_file, slave_user, slave_ip,
+                       slave_vol, ssh_port);
+        if ((len < 0) || (len >= sizeof(hooks_args))) {
+            ret = -1;
+            goto out;
+        }
+    } else
+        snprintf(hooks_args, sizeof(hooks_args),
+                 "This argument will stop the hooks script");
+
+    arg_buf = gf_strdup(hooks_args);
+    if (!arg_buf) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_STRDUP_FAILED,
+               "Failed to gf_strdup");
+        if (is_force) {
+            ret = 0;
+            goto create_essentials;
+        }
+        ret = -1;
+        goto out;
+    }
+
+    ret = dict_set_str(dict, "hooks_args", arg_buf);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+               "Failed to set hooks_args in dict.");
+        if (is_force) {
+            ret = 0;
+            goto create_essentials;
+        }
+        goto out;
+    }
 
-        ret = glusterd_volinfo_find (volname, &volinfo);
+create_essentials:
+    /* Fetch slave volume uuid, to get stored in volume info. */
+    ret = dict_get_str(dict, "slave_voluuid", &slave_voluuid);
+    if (ret) {
+        snprintf(errmsg, sizeof(errmsg),
+                 "Unable to fetch slave volume uuid from dict");
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED, "%s",
+               errmsg);
+        ret = -1;
+        goto out;
+    }
+
+    is_existing_session = dict_get_str_boolean(dict, "existing_session",
+                                               _gf_false);
+    if (is_existing_session) {
+        ret = dict_get_str(dict, "old_slavehost", &old_slavehost);
         if (ret) {
-                gf_log ("", GF_LOG_ERROR, "Volinfo for %s"
-                        " (master) not found", volname);
-                goto out;
+            snprintf(errmsg, sizeof(errmsg), "Unable to fetch old_slavehost");
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED, "%s",
+                   errmsg);
+            ret = -1;
+            goto out;
         }
 
-        ret = dict_get_str (dict, "slave_vol", &slave_vol);
-        if (ret) {
-                snprintf (errmsg, sizeof (errmsg),
-                          "Unable to fetch slave volume name.");
-                gf_log ("", GF_LOG_ERROR, "%s", errmsg);
-                goto out;
-        }
+        /* Rename existing geo-rep session with new Slave Host */
+        ret = snprintf(old_working_dir, sizeof(old_working_dir) - 1,
+                       "%s/" GEOREP "/%s_%s_%s", conf->workdir,
+                       volinfo->volname, old_slavehost, slave_vol);
 
-        ret = dict_get_str (dict, "slave_url", &slave_url);
-        if (ret) {
-                snprintf (errmsg, sizeof (errmsg),
-                          "Unable to fetch slave IP.");
-                gf_log ("", GF_LOG_ERROR, "%s", errmsg);
-                ret = -1;
-                goto out;
-        }
+        ret = snprintf(new_working_dir, sizeof(new_working_dir) - 1,
+                       "%s/" GEOREP "/%s_%s_%s", conf->workdir,
+                       volinfo->volname, slave_host, slave_vol);
 
-        /* Fetch the slave_user and slave_ip from the slave_url.
-         * If the slave_user is not present. Use "root"
-         */
-        if (strstr(slave_url, "@")) {
-                slave_url_buf = gf_strdup (slave_url);
-                if (!slave_url_buf) {
-                        ret = -1;
-                        goto out;
-                }
-                slave_user = strtok_r (slave_url, "@", &save_ptr);
-                slave_ip = strtok_r (NULL, "@", &save_ptr);
+        ret = sys_rename(old_working_dir, new_working_dir);
+        if (!ret) {
+            gf_msg(this->name, GF_LOG_INFO, 0, GD_MSG_FORCE_CREATE_SESSION,
+                   "rename of old working dir %s to "
+                   "new working dir %s is done! ",
+                   old_working_dir, new_working_dir);
         } else {
-                slave_user = "root";
-                slave_ip = slave_url;
-        }
-
-        if (!slave_user || !slave_ip) {
-                gf_log (this->name, GF_LOG_ERROR, "Invalid slave url.");
-                ret = -1;
-                goto out;
-        }
-
-        ret = dict_get_str (dict, "slave_host", &slave_host);
-        if (ret) {
-                snprintf (errmsg, sizeof (errmsg),
-                          "Unable to fetch slave host");
-                gf_log ("", GF_LOG_ERROR, "%s", errmsg);
-                ret = -1;
-                goto out;
-        }
-
-        is_force = dict_get_str_boolean (dict, "force", _gf_false);
-
-        uuid_utoa_r (MY_UUID, uuid_str);
-        if (!strcmp (uuid_str, host_uuid)) {
-                ret = dict_get_int32 (dict, "push_pem", &is_pem_push);
-                if (!ret && is_pem_push) {
-                        gf_log ("", GF_LOG_DEBUG, "Trying to setup"
-                                " pem files in slave");
-                        is_pem_push = 1;
-                } else
-                        is_pem_push = 0;
-
-                snprintf(hooks_args, sizeof(hooks_args),
-                         "is_push_pem=%d,pub_file=%s,slave_user=%s,slave_ip=%s,"
-                         "slave_vol=%s", is_pem_push, common_pem_file,
-                         slave_user, slave_ip, slave_vol);
-        } else
-                snprintf(hooks_args, sizeof(hooks_args),
-                         "This argument will stop the hooks script");
-
-        arg_buf = gf_strdup (hooks_args);
-        if (!arg_buf) {
-                gf_log ("", GF_LOG_ERROR, "Failed to"
-                        " gf_strdup");
-                if (is_force) {
-                        ret = 0;
-                        goto create_essentials;
+            if (errno == ENOENT) {
+                /* log error, but proceed with directory
+                 * creation below */
+                gf_msg_debug(this->name, 0,
+                             "old_working_dir(%s) "
+                             "not present.",
+                             old_working_dir);
+            } else {
+                len = snprintf(errmsg, sizeof(errmsg),
+                               "rename of old working dir %s "
+                               "to new working dir %s "
+                               "failed! Error: %s",
+                               old_working_dir, new_working_dir,
+                               strerror(errno));
+                if (len < 0) {
+                    strcpy(errmsg, "<error>");
                 }
-                ret = -1;
-                goto out;
-        }
+                gf_msg(this->name, GF_LOG_INFO, 0, GD_MSG_FORCE_CREATE_SESSION,
+                       "rename of old working dir %s to "
+                       "new working dir %s failed! Error: %s!",
+                       old_working_dir, new_working_dir, strerror(errno));
 
-        ret = dict_set_str (dict, "hooks_args", arg_buf);
-        if (ret) {
-                gf_log ("", GF_LOG_ERROR, "Failed to set"
-                        " hooks_args in dict.");
-                if (is_force) {
-                        ret = 0;
-                        goto create_essentials;
-                }
-                goto out;
-        }
-
-create_essentials:
-
-        ret = glusterd_create_essential_dir_files (volinfo, dict, slave,
-                                                   slave_host, slave_vol,
-                                                   op_errstr);
-        if (ret)
-                goto out;
-
-        ret = glusterd_store_slave_in_info (volinfo, slave,
-                                            host_uuid, op_errstr,
-                                            is_force);
-        if (ret) {
-                snprintf (errmsg, sizeof (errmsg), "Unable to store"
-                          " slave info.");
-                gf_log ("", GF_LOG_ERROR, "%s", errmsg);
-                goto out;
-        }
-
-        /* Enable marker and changelog */
-        ret = glusterd_set_gsync_confs (volinfo);
-        if (ret != 0) {
-                gf_log (this->name, GF_LOG_WARNING, "marker/changelog"
-                        " start failed");
-                *op_errstr = gf_strdup ("Index initialization failed");
                 ret = -1;
                 goto out;
+            }
         }
+    }
+
+    ret = glusterd_create_essential_dir_files(volinfo, dict, slave, slave_host,
+                                              slave_vol, op_errstr);
+    if (ret)
+        goto out;
+
+    ret = glusterd_store_slave_in_info(volinfo, slave, host_uuid, slave_voluuid,
+                                       op_errstr, is_force);
+    if (ret) {
+        snprintf(errmsg, sizeof(errmsg),
+                 "Unable to store"
+                 " slave info.");
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_SLAVEINFO_STORE_ERROR, "%s",
+               errmsg);
+        goto out;
+    }
+
+    /* Enable marker and changelog */
+    ret = glusterd_set_gsync_confs(volinfo);
+    if (ret != 0) {
+        gf_msg(this->name, GF_LOG_WARNING, 0, GD_MSG_MARKER_START_FAIL,
+               "marker/changelog"
+               " start failed");
+        snprintf(errmsg, sizeof(errmsg), "Index initialization failed");
+
+        ret = -1;
+        goto out;
+    }
 
 out:
-        GF_FREE (slave_url_buf);
-        gf_log ("", GF_LOG_DEBUG,"Returning %d", ret);
-        return ret;
+    if (ret && errmsg[0] != '\0') {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_GSYNCD_ERROR, "%s", errmsg);
+        *op_errstr = gf_strdup(errmsg);
+    }
+
+    GF_FREE(slave_url_buf);
+    gf_msg_debug(this->name, 0, "Returning %d", ret);
+    return ret;
 }
diff --git a/xlators/mgmt/glusterd/src/glusterd-geo-rep.h b/xlators/mgmt/glusterd/src/glusterd-geo-rep.h
index 2543b9482c1..7d1318f522c 100644
--- a/xlators/mgmt/glusterd/src/glusterd-geo-rep.h
+++ b/xlators/mgmt/glusterd/src/glusterd-geo-rep.h
@@ -10,36 +10,43 @@
 #ifndef _GLUSTERD_GEO_REP_H_
 #define _GLUSTERD_GEO_REP_H_
 
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
-
 #ifndef GSYNC_CONF_TEMPLATE
-#define GSYNC_CONF_TEMPLATE GEOREP"/gsyncd_template.conf"
+#define GSYNC_CONF_TEMPLATE GEOREP "/gsyncd_template.conf"
 #endif
 
+/* <slave host>::<slave volume> */
+#define SLAVE_URL_INFO_MAX (_POSIX_HOST_NAME_MAX + GD_VOLUME_NAME_MAX + 3)
+
+/* slave info format:
+ * <master host uuid>:ssh://{<slave_user>@}<slave host>::<slave volume> \
+ * :<slave volume uuid> */
+#define VOLINFO_SLAVE_URL_MAX                                                  \
+    (LOGIN_NAME_MAX + (2 * GF_UUID_BUF_SIZE) + SLAVE_URL_INFO_MAX + 10)
+
 typedef struct glusterd_gsync_status_temp {
-        dict_t *rsp_dict;
-        glusterd_volinfo_t *volinfo;
-        char *node;
+    dict_t *rsp_dict;
+    glusterd_volinfo_t *volinfo;
+    char *node;
 } glusterd_gsync_status_temp_t;
 
 typedef struct gsync_status_param {
-        int is_active;
-        glusterd_volinfo_t *volinfo;
+    glusterd_volinfo_t *volinfo;
+    int is_active;
 } gsync_status_param_t;
 
 int
-gsync_status (char *master, char *slave, char *conf_path,
-              int *status, gf_boolean_t *is_template_in_use);
+gsync_status(char *master, char *slave, char *conf_path, int *status,
+             gf_boolean_t *is_template_in_use);
 
 void
-glusterd_check_geo_rep_configured (glusterd_volinfo_t *volinfo,
-                                   gf_boolean_t *flag);
+glusterd_check_geo_rep_configured(glusterd_volinfo_t *volinfo,
+                                  gf_boolean_t *flag);
 int
-_get_slave_status (dict_t *dict, char *key, data_t *value, void *data);
+_get_slave_status(dict_t *dict, char *key, data_t *value, void *data);
 int
-glusterd_check_geo_rep_running (gsync_status_param_t *param, char **op_errstr);
-#endif
+glusterd_check_geo_rep_running(gsync_status_param_t *param, char **op_errstr);
 
+int
+glusterd_get_gsync_status_mst(glusterd_volinfo_t *volinfo, dict_t *rsp_dict,
+                              char *node);
+#endif
diff --git a/xlators/mgmt/glusterd/src/glusterd-gfproxyd-svc-helper.c b/xlators/mgmt/glusterd/src/glusterd-gfproxyd-svc-helper.c
new file mode 100644
index 00000000000..319bfa140f3
--- /dev/null
+++ b/xlators/mgmt/glusterd/src/glusterd-gfproxyd-svc-helper.c
@@ -0,0 +1,235 @@
+/*
+   Copyright (c) 2016 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
+
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
+
+#include "glusterd.h"
+#include "glusterd-utils.h"
+#include "glusterd-gfproxyd-svc-helper.h"
+#include "glusterd-messages.h"
+#include <glusterfs/syscall.h>
+#include "glusterd-volgen.h"
+
+void
+glusterd_svc_build_gfproxyd_rundir(glusterd_volinfo_t *volinfo, char *path,
+                                   int path_len)
+{
+    char workdir[PATH_MAX] = {
+        0,
+    };
+    glusterd_conf_t *priv = THIS->private;
+
+    GLUSTERD_GET_VOLUME_PID_DIR(workdir, volinfo, priv);
+
+    snprintf(path, path_len, "%s", workdir);
+}
+
+void
+glusterd_svc_build_gfproxyd_socket_filepath(glusterd_volinfo_t *volinfo,
+                                            char *path, int path_len)
+{
+    char sockfilepath[PATH_MAX] = {
+        0,
+    };
+    char rundir[PATH_MAX] = {
+        0,
+    };
+    int32_t len = 0;
+
+    glusterd_svc_build_gfproxyd_rundir(volinfo, rundir, sizeof(rundir));
+    len = snprintf(sockfilepath, sizeof(sockfilepath), "%s/run-%s", rundir,
+                   uuid_utoa(MY_UUID));
+    if ((len < 0) || (len >= sizeof(sockfilepath))) {
+        sockfilepath[0] = 0;
+    }
+
+    glusterd_set_socket_filepath(sockfilepath, path, path_len);
+}
+
+void
+glusterd_svc_build_gfproxyd_pidfile(glusterd_volinfo_t *volinfo, char *path,
+                                    int path_len)
+{
+    char rundir[PATH_MAX] = {
+        0,
+    };
+
+    glusterd_svc_build_gfproxyd_rundir(volinfo, rundir, sizeof(rundir));
+
+    snprintf(path, path_len, "%s/%s.gfproxyd.pid", rundir, volinfo->volname);
+}
+
+void
+glusterd_svc_build_gfproxyd_volfile_path(glusterd_volinfo_t *volinfo,
+                                         char *path, int path_len)
+{
+    char workdir[PATH_MAX] = {
+        0,
+    };
+    glusterd_conf_t *priv = THIS->private;
+
+    GLUSTERD_GET_VOLUME_DIR(workdir, volinfo, priv);
+
+    snprintf(path, path_len, "%s/%s.gfproxyd.vol", workdir, volinfo->volname);
+}
+
+void
+glusterd_svc_build_gfproxyd_logdir(char *logdir, char *volname, size_t len)
+{
+    glusterd_conf_t *conf = THIS->private;
+    snprintf(logdir, len, "%s/gfproxy/%s", conf->logdir, volname);
+}
+
+void
+glusterd_svc_build_gfproxyd_logfile(char *logfile, char *logdir, size_t len)
+{
+    snprintf(logfile, len, "%s/gfproxyd.log", logdir);
+}
+
+int
+glusterd_is_gfproxyd_enabled(glusterd_volinfo_t *volinfo)
+{
+    return glusterd_volinfo_get_boolean(volinfo, VKEY_CONFIG_GFPROXY);
+}
+
+static int
+glusterd_svc_get_gfproxyd_volfile(glusterd_volinfo_t *volinfo, char *svc_name,
+                                  char *orgvol, char **tmpvol, int path_len)
+{
+    int tmp_fd = -1;
+    int ret = -1;
+    int need_unlink = 0;
+
+    glusterd_svc_build_gfproxyd_volfile_path(volinfo, orgvol, path_len);
+
+    ret = gf_asprintf(tmpvol, "/tmp/g%s-XXXXXX", svc_name);
+    if (ret < 0) {
+        goto out;
+    }
+
+    /* coverity[SECURE_TEMP] mkstemp uses 0600 as the mode and is safe */
+    tmp_fd = mkstemp(*tmpvol);
+    if (tmp_fd < 0) {
+        gf_msg("glusterd", GF_LOG_WARNING, errno, GD_MSG_FILE_OP_FAILED,
+               "Unable to create temp file"
+               " %s:(%s)",
+               *tmpvol, strerror(errno));
+        ret = -1;
+        goto out;
+    }
+
+    need_unlink = 1;
+    ret = glusterd_build_gfproxyd_volfile(volinfo, *tmpvol);
+out:
+    if (need_unlink && ret < 0)
+        sys_unlink(*tmpvol);
+
+    if ((ret < 0) && (*tmpvol != NULL)) {
+        GF_FREE(*tmpvol);
+        *tmpvol = NULL;
+    }
+
+    if (tmp_fd >= 0)
+        sys_close(tmp_fd);
+
+    return ret;
+}
+
+int
+glusterd_svc_check_gfproxyd_volfile_identical(char *svc_name,
+                                              glusterd_volinfo_t *volinfo,
+                                              gf_boolean_t *identical)
+{
+    char orgvol[PATH_MAX] = {
+        0,
+    };
+    char *tmpvol = NULL;
+    int ret = -1;
+    int need_unlink = 0;
+
+    GF_VALIDATE_OR_GOTO("glusterd", identical, out);
+
+    ret = glusterd_svc_get_gfproxyd_volfile(volinfo, svc_name, orgvol, &tmpvol,
+                                            PATH_MAX);
+    if (ret)
+        goto out;
+
+    need_unlink = 1;
+    ret = glusterd_check_files_identical(orgvol, tmpvol, identical);
+    if (ret)
+        goto out;
+
+out:
+    if (need_unlink)
+        sys_unlink(tmpvol);
+
+    if (tmpvol != NULL)
+        GF_FREE(tmpvol);
+
+    return ret;
+}
+
+int
+glusterd_svc_check_gfproxyd_topology_identical(char *svc_name,
+                                               glusterd_volinfo_t *volinfo,
+                                               gf_boolean_t *identical)
+{
+    char orgvol[PATH_MAX] = {
+        0,
+    };
+    char *tmpvol = NULL;
+    int ret = -1;
+    int tmpclean = 0;
+
+    GF_VALIDATE_OR_GOTO("glusterd", identical, out);
+
+    ret = glusterd_svc_get_gfproxyd_volfile(volinfo, svc_name, orgvol, &tmpvol,
+                                            PATH_MAX);
+    if (ret)
+        goto out;
+
+    tmpclean = 1; /* SET the flag to unlink() tmpfile */
+
+    /* Compare the topology of volfiles */
+    ret = glusterd_check_topology_identical(orgvol, tmpvol, identical);
+out:
+    if (tmpclean)
+        sys_unlink(tmpvol);
+
+    if (tmpvol != NULL)
+        GF_FREE(tmpvol);
+
+    return ret;
+}
+
+glusterd_volinfo_t *
+glusterd_gfproxyd_volinfo_from_svc(glusterd_svc_t *svc)
+{
+    glusterd_volinfo_t *volinfo = NULL;
+    glusterd_gfproxydsvc_t *gfproxyd = NULL;
+
+    /* Get volinfo->gfproxyd from svc object */
+    gfproxyd = cds_list_entry(svc, glusterd_gfproxydsvc_t, svc);
+    if (!gfproxyd) {
+        gf_msg("glusterd", GF_LOG_ERROR, 0, GD_MSG_SNAPD_OBJ_GET_FAIL,
+               "Failed to get gfproxyd "
+               "object from gfproxyd service");
+        goto out;
+    }
+
+    /* Get volinfo from gfproxyd */
+    volinfo = cds_list_entry(gfproxyd, glusterd_volinfo_t, gfproxyd);
+    if (!volinfo) {
+        gf_msg("glusterd", GF_LOG_ERROR, 0, GD_MSG_VOLINFO_GET_FAIL,
+               "Failed to get volinfo from "
+               "from gfproxyd");
+        goto out;
+    }
+out:
+    return volinfo;
+}
diff --git a/xlators/mgmt/glusterd/src/glusterd-gfproxyd-svc-helper.h b/xlators/mgmt/glusterd/src/glusterd-gfproxyd-svc-helper.h
new file mode 100644
index 00000000000..3aca218a65d
--- /dev/null
+++ b/xlators/mgmt/glusterd/src/glusterd-gfproxyd-svc-helper.h
@@ -0,0 +1,51 @@
+/*
+   Copyright (c) 2016 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
+
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
+
+#ifndef _GLUSTERD_GFPROXYD_SVC_HELPER_H_
+#define _GLUSTERD_GFPROXYD_SVC_HELPER_H_
+
+#include "glusterd.h"
+
+void
+glusterd_svc_build_gfproxyd_rundir(glusterd_volinfo_t *volinfo, char *path,
+                                   int path_len);
+
+void
+glusterd_svc_build_gfproxyd_socket_filepath(glusterd_volinfo_t *volinfo,
+                                            char *path, int path_len);
+
+void
+glusterd_svc_build_gfproxyd_pidfile(glusterd_volinfo_t *volinfo, char *path,
+                                    int path_len);
+
+void
+glusterd_svc_build_gfproxyd_volfile_path(glusterd_volinfo_t *volinfo,
+                                         char *path, int path_len);
+
+void
+glusterd_svc_build_gfproxyd_logdir(char *logdir, char *volname, size_t len);
+
+void
+glusterd_svc_build_gfproxyd_logfile(char *logfile, char *logdir, size_t len);
+
+int
+glusterd_svc_check_gfproxyd_volfile_identical(char *svc_name,
+                                              glusterd_volinfo_t *volinfo,
+                                              gf_boolean_t *identical);
+int
+glusterd_svc_check_gfproxyd_topology_identical(char *svc_name,
+                                               glusterd_volinfo_t *volinfo,
+                                               gf_boolean_t *identical);
+int
+glusterd_is_gfproxyd_enabled(glusterd_volinfo_t *volinfo);
+
+glusterd_volinfo_t *
+glusterd_gfproxyd_volinfo_from_svc(glusterd_svc_t *svc);
+#endif
diff --git a/xlators/mgmt/glusterd/src/glusterd-gfproxyd-svc.c b/xlators/mgmt/glusterd/src/glusterd-gfproxyd-svc.c
new file mode 100644
index 00000000000..a0bfea41f0f
--- /dev/null
+++ b/xlators/mgmt/glusterd/src/glusterd-gfproxyd-svc.c
@@ -0,0 +1,478 @@
+/*
+   Copyright (c) 2014 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
+
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
+
+#include <glusterfs/globals.h>
+#include <glusterfs/run.h>
+#include "glusterd.h"
+#include "glusterd-utils.h"
+#include "glusterd-volgen.h"
+#include "glusterd-gfproxyd-svc.h"
+#include "glusterd-messages.h"
+#include "glusterd-svc-helper.h"
+#include "glusterd-svc-mgmt.h"
+#include "glusterd-gfproxyd-svc-helper.h"
+#include <glusterfs/syscall.h>
+
+void
+glusterd_gfproxydsvc_build(glusterd_svc_t *svc)
+{
+    svc->manager = glusterd_gfproxydsvc_manager;
+    svc->start = glusterd_gfproxydsvc_start;
+    svc->stop = glusterd_gfproxydsvc_stop;
+    svc->reconfigure = glusterd_gfproxydsvc_reconfigure;
+}
+
+int
+glusterd_gfproxydsvc_stop(glusterd_svc_t *svc, int sig)
+{
+    glusterd_volinfo_t *volinfo = NULL;
+    int ret = 0;
+
+    ret = glusterd_svc_stop(svc, sig);
+    if (ret)
+        goto out;
+
+    volinfo = glusterd_gfproxyd_volinfo_from_svc(svc);
+    volinfo->gfproxyd.port = 0;
+
+out:
+    return ret;
+}
+
+int
+glusterd_gfproxydsvc_init(glusterd_volinfo_t *volinfo)
+{
+    int ret = -1;
+    char rundir[PATH_MAX] = {
+        0,
+    };
+    char sockpath[PATH_MAX] = {
+        0,
+    };
+    char pidfile[PATH_MAX] = {
+        0,
+    };
+    char volfile[PATH_MAX] = {
+        0,
+    };
+    char logdir[PATH_MAX] = {
+        0,
+    };
+    char logfile[PATH_MAX] = {
+        0,
+    };
+    char volfileid[256] = {0};
+    glusterd_svc_t *svc = NULL;
+    glusterd_conf_t *priv = NULL;
+    glusterd_conn_notify_t notify = NULL;
+    xlator_t *this = NULL;
+    char *volfileserver = NULL;
+    int32_t len = 0;
+
+    this = THIS;
+    GF_VALIDATE_OR_GOTO("glusterd", this, out);
+
+    priv = this->private;
+    GF_VALIDATE_OR_GOTO(this->name, priv, out);
+
+    svc = &(volinfo->gfproxyd.svc);
+
+    ret = snprintf(svc->name, sizeof(svc->name), "%s", gfproxyd_svc_name);
+    if (ret < 0)
+        goto out;
+
+    notify = glusterd_svc_common_rpc_notify;
+
+    glusterd_svc_build_gfproxyd_rundir(volinfo, rundir, sizeof(rundir));
+    glusterd_svc_create_rundir(rundir);
+
+    /* Initialize the connection mgmt */
+    glusterd_svc_build_gfproxyd_socket_filepath(volinfo, sockpath,
+                                                sizeof(sockpath));
+    ret = glusterd_conn_init(&(svc->conn), sockpath, 600, notify);
+    if (ret)
+        goto out;
+
+    /* Initialize the process mgmt */
+    glusterd_svc_build_gfproxyd_pidfile(volinfo, pidfile, sizeof(pidfile));
+    glusterd_svc_build_gfproxyd_volfile_path(volinfo, volfile, sizeof(volfile));
+    glusterd_svc_build_gfproxyd_logdir(logdir, volinfo->volname,
+                                       sizeof(logdir));
+    ret = mkdir_p(logdir, 0755, _gf_true);
+    if ((ret == -1) && (EEXIST != errno)) {
+        gf_msg(this->name, GF_LOG_ERROR, errno, GD_MSG_CREATE_DIR_FAILED,
+               "Unable to create logdir %s", logdir);
+        goto out;
+    }
+    glusterd_svc_build_gfproxyd_logfile(logfile, logdir, sizeof(logfile));
+    len = snprintf(volfileid, sizeof(volfileid), "gfproxyd/%s",
+                   volinfo->volname);
+    if ((len < 0) || (len >= sizeof(volfileid))) {
+        ret = -1;
+        goto out;
+    }
+
+    if (dict_get_strn(this->options, "transport.socket.bind-address",
+                      SLEN("transport.socket.bind-address"),
+                      &volfileserver) != 0) {
+        volfileserver = "localhost";
+    }
+    ret = glusterd_proc_init(&(svc->proc), gfproxyd_svc_name, pidfile, logdir,
+                             logfile, volfile, volfileid, volfileserver);
+    if (ret)
+        goto out;
+
+out:
+    gf_msg_debug(this ? this->name : "glusterd", 0, "Returning %d", ret);
+    return ret;
+}
+
+static int
+glusterd_gfproxydsvc_create_volfile(glusterd_volinfo_t *volinfo)
+{
+    int ret = -1;
+    xlator_t *this = NULL;
+
+    this = THIS;
+    GF_VALIDATE_OR_GOTO("glusterd", this, out);
+
+    ret = glusterd_generate_gfproxyd_volfile(volinfo);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_VOLFILE_CREATE_FAIL,
+               "Failed to create volfile");
+        goto out;
+    }
+
+out:
+    gf_msg_debug(this ? this->name : "glusterd", 0, "Returning %d", ret);
+
+    return ret;
+}
+
+int
+glusterd_gfproxydsvc_manager(glusterd_svc_t *svc, void *data, int flags)
+{
+    int ret = -1;
+    glusterd_volinfo_t *volinfo = NULL;
+    xlator_t *this = NULL;
+
+    this = THIS;
+    GF_VALIDATE_OR_GOTO("glusterd", this, out);
+
+    volinfo = data;
+    GF_VALIDATE_OR_GOTO(this->name, data, out);
+
+    if (!svc->inited) {
+        ret = glusterd_gfproxydsvc_init(volinfo);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_FAILED_INIT_QUOTASVC,
+                   "Failed to init "
+                   "gfproxyd service");
+            goto out;
+        } else {
+            svc->inited = _gf_true;
+            gf_msg_debug(this->name, 0,
+                         "gfproxyd service "
+                         "initialized");
+        }
+    }
+
+    ret = glusterd_is_gfproxyd_enabled(volinfo);
+    if (ret == -1) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_VOLINFO_GET_FAIL,
+               "Failed to read volume "
+               "options");
+        goto out;
+    }
+
+    if (ret) {
+        if (!glusterd_is_volume_started(volinfo)) {
+            if (glusterd_proc_is_running(&svc->proc)) {
+                ret = svc->stop(svc, SIGTERM);
+                if (ret)
+                    gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_SNAPD_STOP_FAIL,
+                           "Couldn't stop gfproxyd for "
+                           "volume: %s",
+                           volinfo->volname);
+            } else {
+                /* Since gfproxyd is not running set ret to 0 */
+                ret = 0;
+            }
+            goto out;
+        }
+
+        ret = glusterd_gfproxydsvc_create_volfile(volinfo);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_SNAPD_CREATE_FAIL,
+                   "Couldn't create "
+                   "gfroxyd volfile for volume: %s",
+                   volinfo->volname);
+            goto out;
+        }
+        ret = svc->stop(svc, SIGTERM);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_SNAPD_START_FAIL,
+                   "Couldn't stop "
+                   "gfproxyd for volume: %s",
+                   volinfo->volname);
+            goto out;
+        }
+
+        ret = svc->start(svc, flags);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_SNAPD_START_FAIL,
+                   "Couldn't start "
+                   "gfproxyd for volume: %s",
+                   volinfo->volname);
+            goto out;
+        }
+
+        glusterd_volinfo_ref(volinfo);
+        ret = glusterd_conn_connect(&(svc->conn));
+        if (ret) {
+            glusterd_volinfo_unref(volinfo);
+            volinfo = NULL;
+            goto out;
+        }
+
+    } else if (glusterd_proc_is_running(&svc->proc)) {
+        ret = svc->stop(svc, SIGTERM);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_SNAPD_STOP_FAIL,
+                   "Couldn't stop gfproxyd for volume: %s", volinfo->volname);
+            goto out;
+        }
+    }
+
+out:
+    if (ret) {
+        if (volinfo) {
+            gf_event(EVENT_SVC_MANAGER_FAILED, "volume=%s;svc_name=%s",
+                     volinfo->volname, svc->name);
+        }
+    }
+
+    gf_msg_debug("glusterd", 0, "Returning %d", ret);
+
+    return ret;
+}
+
+int
+glusterd_gfproxydsvc_start(glusterd_svc_t *svc, int flags)
+{
+    int ret = -1;
+    runner_t runner = {
+        0,
+    };
+    glusterd_conf_t *priv = NULL;
+    xlator_t *this = NULL;
+    char valgrind_logfile[PATH_MAX] = {0};
+    int gfproxyd_port = 0;
+    char msg[1024] = {
+        0,
+    };
+    char gfproxyd_id[PATH_MAX] = {
+        0,
+    };
+    glusterd_volinfo_t *volinfo = NULL;
+    char *localtime_logging = NULL;
+    int32_t len = 0;
+
+    this = THIS;
+    GF_VALIDATE_OR_GOTO("glusterd", this, out);
+
+    priv = this->private;
+    GF_VALIDATE_OR_GOTO(this->name, priv, out);
+
+    volinfo = glusterd_gfproxyd_volinfo_from_svc(svc);
+    if (!volinfo)
+        goto out;
+
+    ret = sys_access(svc->proc.volfile, F_OK);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_DEBUG, 0, GD_MSG_VOLINFO_GET_FAIL,
+               "gfproxyd Volfile %s is not present", svc->proc.volfile);
+        ret = glusterd_gfproxydsvc_create_volfile(volinfo);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_VOLFILE_CREATE_FAIL,
+                   "Couldn't create "
+                   "gfproxyd volfile for volume: %s",
+                   volinfo->volname);
+            goto out;
+        }
+    }
+    runinit(&runner);
+
+    if (this->ctx->cmd_args.vgtool != _gf_none) {
+        len = snprintf(valgrind_logfile, PATH_MAX, "%s/valgrind-%s",
+                       svc->proc.logdir, svc->proc.logfile);
+        if ((len < 0) || (len >= PATH_MAX)) {
+            ret = -1;
+            goto out;
+        }
+
+        if (this->ctx->cmd_args.vgtool == _gf_memcheck)
+            runner_add_args(&runner, "valgrind", "--leak-check=full",
+                            "--trace-children=yes", "--track-origins=yes",
+                            NULL);
+        else
+            runner_add_args(&runner, "valgrind", "--tool=drd", NULL);
+
+        runner_argprintf(&runner, "--log-file=%s", valgrind_logfile);
+    }
+
+    snprintf(gfproxyd_id, sizeof(gfproxyd_id), "gfproxyd-%s", volinfo->volname);
+    runner_add_args(&runner, SBIN_DIR "/glusterfsd", "-s",
+                    svc->proc.volfileserver, "--volfile-id",
+                    svc->proc.volfileid, "-p", svc->proc.pidfile, "-l",
+                    svc->proc.logfile, "--brick-name", gfproxyd_id, "-S",
+                    svc->conn.sockpath, NULL);
+
+    if (volinfo->memory_accounting)
+        runner_add_arg(&runner, "--mem-accounting");
+    if (dict_get_strn(priv->opts, GLUSTERD_LOCALTIME_LOGGING_KEY,
+                      SLEN(GLUSTERD_LOCALTIME_LOGGING_KEY),
+                      &localtime_logging) == 0) {
+        if (strcmp(localtime_logging, "enable") == 0)
+            runner_add_arg(&runner, "--localtime-logging");
+    }
+
+    gfproxyd_port = pmap_assign_port(this, volinfo->gfproxyd.port, gfproxyd_id);
+    volinfo->gfproxyd.port = gfproxyd_port;
+
+    runner_add_arg(&runner, "--brick-port");
+    runner_argprintf(&runner, "%d", gfproxyd_port);
+    runner_add_arg(&runner, "--xlator-option");
+    runner_argprintf(&runner, "%s-server.listen-port=%d", volinfo->volname,
+                     gfproxyd_port);
+
+    snprintf(msg, sizeof(msg), "Starting the gfproxyd service for volume %s",
+             volinfo->volname);
+    runner_log(&runner, this->name, GF_LOG_DEBUG, msg);
+
+    if (flags == PROC_START_NO_WAIT) {
+        ret = runner_run_nowait(&runner);
+    } else {
+        synclock_unlock(&priv->big_lock);
+        {
+            ret = runner_run(&runner);
+        }
+        synclock_lock(&priv->big_lock);
+    }
+
+out:
+    return ret;
+}
+
+int
+glusterd_gfproxydsvc_restart()
+{
+    glusterd_volinfo_t *volinfo = NULL;
+    glusterd_volinfo_t *tmp = NULL;
+    int ret = -1;
+    xlator_t *this = THIS;
+    glusterd_conf_t *conf = NULL;
+    glusterd_svc_t *svc = NULL;
+
+    GF_VALIDATE_OR_GOTO("glusterd", this, out);
+
+    conf = this->private;
+    GF_VALIDATE_OR_GOTO(this->name, conf, out);
+
+    cds_list_for_each_entry_safe(volinfo, tmp, &conf->volumes, vol_list)
+    {
+        /* Start per volume gfproxyd svc */
+        if (volinfo->status == GLUSTERD_STATUS_STARTED) {
+            svc = &(volinfo->gfproxyd.svc);
+            ret = svc->manager(svc, volinfo, PROC_START_NO_WAIT);
+            if (ret) {
+                gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_SNAPD_START_FAIL,
+                       "Couldn't resolve gfproxyd for "
+                       "vol: %s on restart",
+                       volinfo->volname);
+                gf_event(EVENT_SVC_MANAGER_FAILED, "volume=%s;svc_name=%s",
+                         volinfo->volname, svc->name);
+                goto out;
+            }
+        }
+    }
+out:
+    return ret;
+}
+
+int
+glusterd_gfproxydsvc_reconfigure(void *data)
+{
+    int ret = -1;
+    xlator_t *this = NULL;
+    gf_boolean_t identical = _gf_false;
+    glusterd_volinfo_t *volinfo = NULL;
+
+    volinfo = data;
+
+    this = THIS;
+    GF_VALIDATE_OR_GOTO("glusterd", this, out);
+
+    if (!volinfo->gfproxyd.svc.inited)
+        goto manager;
+
+    if (!glusterd_is_gfproxyd_enabled(volinfo))
+        goto manager;
+    else if (!glusterd_proc_is_running(&volinfo->gfproxyd.svc.proc))
+        goto manager;
+
+    /*
+     * Check both OLD and NEW volfiles, if they are SAME by size
+     * and cksum i.e. "character-by-character". If YES, then
+     * NOTHING has been changed, just return.
+     */
+    ret = glusterd_svc_check_gfproxyd_volfile_identical(
+        volinfo->gfproxyd.svc.name, volinfo, &identical);
+    if (ret)
+        goto out;
+
+    if (identical) {
+        ret = 0;
+        goto out;
+    }
+
+    /*
+     * They are not identical. Find out if the topology is changed
+     * OR just the volume options. If just the options which got
+     * changed, then inform the xlator to reconfigure the options.
+     */
+    identical = _gf_false; /* RESET the FLAG */
+    ret = glusterd_svc_check_gfproxyd_topology_identical(
+        volinfo->gfproxyd.svc.name, volinfo, &identical);
+    if (ret)
+        goto out;
+
+    /* Topology is not changed, but just the options. But write the
+     * options to gfproxyd volfile, so that gfproxyd will be reconfigured.
+     */
+    if (identical) {
+        ret = glusterd_gfproxydsvc_create_volfile(volinfo);
+        if (ret == 0) { /* Only if above PASSES */
+            ret = glusterd_fetchspec_notify(this);
+        }
+        goto out;
+    }
+manager:
+    /*
+     * gfproxyd volfile's topology has been changed. gfproxyd server needs
+     * to be RESTARTED to ACT on the changed volfile.
+     */
+    ret = volinfo->gfproxyd.svc.manager(&(volinfo->gfproxyd.svc), volinfo,
+                                        PROC_START_NO_WAIT);
+
+out:
+    gf_msg_debug("glusterd", 0, "Returning %d", ret);
+    return ret;
+}
diff --git a/xlators/mgmt/glusterd/src/glusterd-gfproxyd-svc.h b/xlators/mgmt/glusterd/src/glusterd-gfproxyd-svc.h
new file mode 100644
index 00000000000..d396b4015f3
--- /dev/null
+++ b/xlators/mgmt/glusterd/src/glusterd-gfproxyd-svc.h
@@ -0,0 +1,47 @@
+/*
+  Copyright (c) 2006-2012 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
+*/
+
+#ifndef _GLUSTERD_GFPROXYD_SVC_H_
+#define _GLUSTERD_GFPROXYD_SVC_H_
+
+#include "glusterd-svc-mgmt.h"
+
+#define gfproxyd_svc_name "gfproxyd"
+
+struct glusterd_gfproxydsvc_ {
+    glusterd_svc_t svc;
+    gf_store_handle_t *handle;
+    int port;
+};
+
+typedef struct glusterd_gfproxydsvc_ glusterd_gfproxydsvc_t;
+
+void
+glusterd_gfproxydsvc_build(glusterd_svc_t *svc);
+
+int
+glusterd_gfproxydsvc_manager(glusterd_svc_t *svc, void *data, int flags);
+
+int
+glusterd_gfproxydsvc_start(glusterd_svc_t *svc, int flags);
+
+int
+glusterd_gfproxydsvc_stop(glusterd_svc_t *svc, int sig);
+
+int
+glusterd_gfproxydsvc_reconfigure();
+
+void
+glusterd_gfproxydsvc_build_volfile_path(char *server, char *workdir,
+                                        char *volfile, size_t len);
+
+int
+glusterd_gfproxydsvc_restart();
+#endif
diff --git a/xlators/mgmt/glusterd/src/glusterd-handler.c b/xlators/mgmt/glusterd/src/glusterd-handler.c
index 469c95c9890..1b21c40596d 100644
--- a/xlators/mgmt/glusterd/src/glusterd-handler.c
+++ b/xlators/mgmt/glusterd/src/glusterd-handler.c
@@ -7,34 +7,32 @@
    later), or the GNU General Public License, version 2 (GPLv2), in all
    cases as published by the Free Software Foundation.
 */
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
 #include <inttypes.h>
 
-#include "globals.h"
-#include "glusterfs.h"
-#include "compat.h"
-#include "dict.h"
+#include <glusterfs/glusterfs.h>
+#include <glusterfs/compat.h>
+#include <glusterfs/dict.h>
 #include "protocol-common.h"
-#include "xlator.h"
-#include "logging.h"
-#include "timer.h"
-#include "defaults.h"
-#include "compat.h"
-#include "compat-errno.h"
-#include "statedump.h"
-#include "run.h"
+#include <glusterfs/xlator.h>
+#include <glusterfs/logging.h>
+#include <glusterfs/syscall.h>
+#include <glusterfs/timer.h>
+#include <glusterfs/defaults.h>
+#include <glusterfs/compat.h>
+#include <glusterfs/compat-errno.h>
+#include <glusterfs/statedump.h>
+#include <glusterfs/run.h>
 #include "glusterd-mem-types.h"
 #include "glusterd.h"
 #include "glusterd-sm.h"
 #include "glusterd-op-sm.h"
 #include "glusterd-utils.h"
+#include "glusterd-mgmt.h"
 #include "glusterd-server-quorum.h"
 #include "glusterd-store.h"
 #include "glusterd-locks.h"
 #include "glusterd-snapshot-utils.h"
+#include "glusterd-geo-rep.h"
 
 #include "glusterd1-xdr.h"
 #include "cli1-xdr.h"
@@ -43,233 +41,241 @@
 #include "glusterd-volgen.h"
 #include "glusterd-mountbroker.h"
 #include "glusterd-messages.h"
+#include "glusterd-errno.h"
 
 #include <sys/resource.h>
 #include <inttypes.h>
 
-#include "defaults.c"
-#include "common-utils.h"
+#include <glusterfs/common-utils.h>
 
-#include "globals.h"
 #include "glusterd-syncop.h"
 #include "glusterd-messages.h"
 
-#ifdef HAVE_BD_XLATOR
-#include <lvm2app.h>
-#endif
-
 extern glusterd_op_info_t opinfo;
+static int volcount;
 
-int glusterd_big_locked_notify (struct rpc_clnt *rpc, void *mydata,
-                                rpc_clnt_event_t event,
-                                void *data, rpc_clnt_notify_t notify_fn)
+int
+glusterd_big_locked_notify(struct rpc_clnt *rpc, void *mydata,
+                           rpc_clnt_event_t event, void *data,
+                           rpc_clnt_notify_t notify_fn)
 {
-        glusterd_conf_t *priv = THIS->private;
-        int              ret   = -1;
+    glusterd_conf_t *priv = THIS->private;
+    int ret = -1;
 
-        synclock_lock (&priv->big_lock);
-        ret = notify_fn (rpc, mydata, event, data);
-        synclock_unlock (&priv->big_lock);
+    synclock_lock(&priv->big_lock);
+    ret = notify_fn(rpc, mydata, event, data);
+    synclock_unlock(&priv->big_lock);
 
-        return ret;
+    return ret;
 }
 
-int glusterd_big_locked_handler (rpcsvc_request_t *req, rpcsvc_actor actor_fn)
+int
+glusterd_big_locked_handler(rpcsvc_request_t *req, rpcsvc_actor actor_fn)
 {
-        glusterd_conf_t *priv = THIS->private;
-        int             ret   = -1;
+    glusterd_conf_t *priv = THIS->private;
+    int ret = -1;
 
-        synclock_lock (&priv->big_lock);
-        ret = actor_fn (req);
-        synclock_unlock (&priv->big_lock);
+    synclock_lock(&priv->big_lock);
+    ret = actor_fn(req);
+    synclock_unlock(&priv->big_lock);
 
-        return ret;
+    return ret;
 }
 
 static int
-glusterd_handle_friend_req (rpcsvc_request_t *req, uuid_t  uuid,
-                            char *hostname, int port,
-                            gd1_mgmt_friend_req *friend_req)
+glusterd_handle_friend_req(rpcsvc_request_t *req, uuid_t uuid, char *hostname,
+                           int port, gd1_mgmt_friend_req *friend_req)
 {
-        int                             ret = -1;
-        glusterd_peerinfo_t             *peerinfo = NULL;
-        glusterd_friend_sm_event_t      *event = NULL;
-        glusterd_friend_req_ctx_t       *ctx = NULL;
-        char                            rhost[UNIX_PATH_MAX + 1] = {0};
-        uuid_t                          friend_uuid = {0};
-        dict_t                          *dict = NULL;
-
-        gf_uuid_parse (uuid_utoa (uuid), friend_uuid);
-        if (!port)
-                port = GF_DEFAULT_BASE_PORT;
-
-        ret = glusterd_remote_hostname_get (req, rhost, sizeof (rhost));
-
-        rcu_read_lock ();
-
-        peerinfo = glusterd_peerinfo_find (uuid, rhost);
+    int ret = -1;
+    glusterd_peerinfo_t *peerinfo = NULL;
+    glusterd_friend_sm_event_t *event = NULL;
+    glusterd_friend_req_ctx_t *ctx = NULL;
+    char rhost[UNIX_PATH_MAX + 1] = {0};
+    dict_t *dict = NULL;
 
-        if (peerinfo == NULL) {
-                ret = glusterd_xfer_friend_add_resp (req, hostname, rhost, port,
-                                                     -1, GF_PROBE_UNKNOWN_PEER);
-                if (friend_req->vols.vols_val) {
-                        free (friend_req->vols.vols_val);
-                        friend_req->vols.vols_val = NULL;
-                }
-                goto out;
-        }
+    if (!port)
+        port = GF_DEFAULT_BASE_PORT;
 
-        ret = glusterd_friend_sm_new_event
-                        (GD_FRIEND_EVENT_RCVD_FRIEND_REQ, &event);
+    ret = glusterd_remote_hostname_get(req, rhost, sizeof(rhost));
 
-        if (ret) {
-                gf_log ("", GF_LOG_ERROR, "event generation failed: %d", ret);
-                goto out;
-        }
+    ctx = GF_CALLOC(1, sizeof(*ctx), gf_gld_mt_friend_req_ctx_t);
+    dict = dict_new();
 
-        event->peername = gf_strdup (peerinfo->hostname);
-        gf_uuid_copy (event->peerid, peerinfo->uuid);
+    RCU_READ_LOCK;
 
-        ctx = GF_CALLOC (1, sizeof (*ctx), gf_gld_mt_friend_req_ctx_t);
+    peerinfo = glusterd_peerinfo_find(uuid, rhost);
 
-        if (!ctx) {
-                gf_log ("", GF_LOG_ERROR, "Unable to allocate memory");
-                ret = -1;
-                goto out;
+    if (peerinfo == NULL) {
+        gf_event(EVENT_PEER_REJECT, "peer=%s", hostname);
+        ret = glusterd_xfer_friend_add_resp(req, hostname, rhost, port, -1,
+                                            GF_PROBE_UNKNOWN_PEER);
+        if (friend_req->vols.vols_val) {
+            free(friend_req->vols.vols_val);
+            friend_req->vols.vols_val = NULL;
         }
+        goto out;
+    }
 
-        gf_uuid_copy (ctx->uuid, uuid);
-        if (hostname)
-                ctx->hostname = gf_strdup (hostname);
-        ctx->req = req;
-
-        dict = dict_new ();
-        if (!dict) {
-                ret = -1;
-                goto out;
-        }
+    ret = glusterd_friend_sm_new_event(GD_FRIEND_EVENT_RCVD_FRIEND_REQ, &event);
 
-        ret = dict_unserialize (friend_req->vols.vols_val,
-                                friend_req->vols.vols_len,
-                                &dict);
+    if (ret) {
+        gf_msg("glusterd", GF_LOG_ERROR, 0, GD_MSG_EVENT_NEW_GET_FAIL,
+               "event generation failed: %d", ret);
+        goto out;
+    }
 
-        if (ret)
-                goto out;
-        else
-                dict->extra_stdfree = friend_req->vols.vols_val;
+    event->peername = gf_strdup(peerinfo->hostname);
+    gf_uuid_copy(event->peerid, peerinfo->uuid);
 
-        ctx->vols = dict;
-        event->ctx = ctx;
+    if (!ctx) {
+        gf_msg("glusterd", GF_LOG_ERROR, ENOMEM, GD_MSG_NO_MEMORY,
+               "Unable to allocate memory");
+        ret = -1;
+        goto out;
+    }
 
-        ret = glusterd_friend_sm_inject_event (event);
-        if (ret) {
-                gf_log ("glusterd", GF_LOG_ERROR, "Unable to inject event %d, "
-                        "ret = %d", event->event, ret);
-                goto out;
-        }
+    gf_uuid_copy(ctx->uuid, uuid);
+    if (hostname)
+        ctx->hostname = gf_strdup(hostname);
+    ctx->req = req;
 
-        ret = 0;
-        if (peerinfo && (0 == peerinfo->connected))
-                ret = GLUSTERD_CONNECTION_AWAITED;
+    if (!dict) {
+        gf_smsg("glusterd", GF_LOG_ERROR, errno, GD_MSG_DICT_CREATE_FAIL, NULL);
+        ret = -1;
+        goto out;
+    }
+
+    ret = dict_unserialize(friend_req->vols.vols_val, friend_req->vols.vols_len,
+                           &dict);
+
+    if (ret) {
+        gf_smsg("glusterd", GF_LOG_ERROR, 0, GD_MSG_DICT_UNSERIALIZE_FAIL,
+                NULL);
+        goto out;
+    } else
+        dict->extra_stdfree = friend_req->vols.vols_val;
+
+    ctx->vols = dict;
+    event->ctx = ctx;
+
+    ret = glusterd_friend_sm_inject_event(event);
+    if (ret) {
+        gf_msg("glusterd", GF_LOG_ERROR, 0, GD_MSG_EVENT_INJECT_FAIL,
+               "Unable to inject event %d, "
+               "ret = %d",
+               event->event, ret);
+        goto out;
+    }
+
+    ret = 0;
+    if (peerinfo && (0 == peerinfo->connected))
+        ret = GLUSTERD_CONNECTION_AWAITED;
 
 out:
-        rcu_read_unlock ();
-
-        if (ret && (ret != GLUSTERD_CONNECTION_AWAITED)) {
-                if (ctx && ctx->hostname)
-                        GF_FREE (ctx->hostname);
-                GF_FREE (ctx);
-                if (dict) {
-                        if ((!dict->extra_stdfree) &&
-                            friend_req->vols.vols_val)
-                                free (friend_req->vols.vols_val);
-                        dict_unref (dict);
-                } else {
-                    free (friend_req->vols.vols_val);
-                }
-                if (event)
-                        GF_FREE (event->peername);
-                GF_FREE (event);
-        }
+    RCU_READ_UNLOCK;
 
+    if (ret && (ret != GLUSTERD_CONNECTION_AWAITED)) {
+        if (ctx && ctx->hostname)
+            GF_FREE(ctx->hostname);
+        GF_FREE(ctx);
+        if (dict) {
+            if ((!dict->extra_stdfree) && friend_req->vols.vols_val)
+                free(friend_req->vols.vols_val);
+            dict_unref(dict);
+        } else {
+            free(friend_req->vols.vols_val);
+        }
+        if (event)
+            GF_FREE(event->peername);
+        GF_FREE(event);
+    }
 
-        return ret;
+    return ret;
 }
 
 static int
-glusterd_handle_unfriend_req (rpcsvc_request_t *req, uuid_t  uuid,
-                              char *hostname, int port)
+glusterd_handle_unfriend_req(rpcsvc_request_t *req, uuid_t uuid, char *hostname,
+                             int port)
 {
-        int                             ret = -1;
-        glusterd_peerinfo_t             *peerinfo = NULL;
-        glusterd_friend_sm_event_t      *event = NULL;
-        glusterd_friend_req_ctx_t       *ctx = NULL;
+    int ret = -1;
+    glusterd_peerinfo_t *peerinfo = NULL;
+    glusterd_friend_sm_event_t *event = NULL;
+    glusterd_friend_req_ctx_t *ctx = NULL;
 
-        if (!port)
-                port = GF_DEFAULT_BASE_PORT;
+    if (!port)
+        port = GF_DEFAULT_BASE_PORT;
 
-        rcu_read_lock ();
+    ctx = GF_CALLOC(1, sizeof(*ctx), gf_gld_mt_friend_req_ctx_t);
 
-        peerinfo = glusterd_peerinfo_find (uuid, hostname);
+    RCU_READ_LOCK;
 
-        if (peerinfo == NULL) {
-                gf_log ("glusterd", GF_LOG_CRITICAL,
-                        "Received remove-friend from unknown peer %s",
-                        hostname);
-                ret = glusterd_xfer_friend_remove_resp (req, hostname,
-                                                        port);
-                goto out;
-        }
+    peerinfo = glusterd_peerinfo_find(uuid, hostname);
 
-        ret = glusterd_friend_sm_new_event
-                        (GD_FRIEND_EVENT_RCVD_REMOVE_FRIEND, &event);
+    if (peerinfo == NULL) {
+        RCU_READ_UNLOCK;
+        gf_msg("glusterd", GF_LOG_CRITICAL, 0, GD_MSG_REQ_FROM_UNKNOWN_PEER,
+               "Received remove-friend from unknown peer %s", hostname);
+        ret = glusterd_xfer_friend_remove_resp(req, hostname, port);
+        goto out;
+    }
 
-        if (ret) {
-                gf_log ("", GF_LOG_ERROR, "event generation failed: %d", ret);
-                goto out;
-        }
+    ret = glusterd_friend_sm_new_event(GD_FRIEND_EVENT_RCVD_REMOVE_FRIEND,
+                                       &event);
 
-        event->peername = gf_strdup (hostname);
-        gf_uuid_copy (event->peerid, uuid);
+    if (ret) {
+        RCU_READ_UNLOCK;
+        gf_msg("glusterd", GF_LOG_ERROR, 0, GD_MSG_EVENT_NEW_GET_FAIL,
+               "event generation failed: %d", ret);
+        goto out;
+    }
 
-        ctx = GF_CALLOC (1, sizeof (*ctx), gf_gld_mt_friend_req_ctx_t);
+    if (hostname)
+        event->peername = gf_strdup(hostname);
 
-        if (!ctx) {
-                gf_log ("", GF_LOG_ERROR, "Unable to allocate memory");
-                ret = -1;
-                goto out;
-        }
+    gf_uuid_copy(event->peerid, uuid);
 
-        gf_uuid_copy (ctx->uuid, uuid);
-        if (hostname)
-                ctx->hostname = gf_strdup (hostname);
-        ctx->req = req;
+    if (!ctx) {
+        RCU_READ_UNLOCK;
+        ret = -1;
+        gf_msg("glusterd", GF_LOG_ERROR, ENOMEM, GD_MSG_NO_MEMORY,
+               "Unable to allocate memory");
+        goto out;
+    }
 
-        event->ctx = ctx;
+    gf_uuid_copy(ctx->uuid, uuid);
+    if (hostname)
+        ctx->hostname = gf_strdup(hostname);
+    ctx->req = req;
 
-        ret = glusterd_friend_sm_inject_event (event);
+    event->ctx = ctx;
 
-        if (ret) {
-                gf_log ("glusterd", GF_LOG_ERROR, "Unable to inject event %d, "
-                        "ret = %d", event->event, ret);
-                goto out;
-        }
+    ret = glusterd_friend_sm_inject_event(event);
 
-        ret = 0;
+    if (ret) {
+        RCU_READ_UNLOCK;
+        gf_msg("glusterd", GF_LOG_ERROR, 0, GD_MSG_EVENT_INJECT_FAIL,
+               "Unable to inject event %d, "
+               "ret = %d",
+               event->event, ret);
+        goto out;
+    }
+
+    RCU_READ_UNLOCK;
+
+    return 0;
 
 out:
-        rcu_read_unlock ();
-
-        if (0 != ret) {
-                if (ctx && ctx->hostname)
-                        GF_FREE (ctx->hostname);
-                GF_FREE (ctx);
-                if (event)
-                        GF_FREE (event->peername);
-                GF_FREE (event);
-        }
 
-        return ret;
+    if (0 != ret) {
+        if (ctx && ctx->hostname)
+            GF_FREE(ctx->hostname);
+        GF_FREE(ctx);
+        if (event)
+            GF_FREE(event->peername);
+        GF_FREE(event);
+    }
+
+    return ret;
 }
 
 struct args_pack {
@@ -279,4627 +285,6429 @@ struct args_pack {
 };
 
 static int
-_build_option_key (dict_t *d, char *k, data_t *v, void *tmp)
+_build_option_key(dict_t *d, char *k, data_t *v, void *tmp)
 {
-        char                    reconfig_key[256] = {0, };
-        struct args_pack        *pack             = NULL;
-        int                     ret               = -1;
-        xlator_t                *this             = NULL;
-        glusterd_conf_t         *priv             = NULL;
-
-        this = THIS;
-        GF_ASSERT (this);
-        priv = this->private;
-        GF_ASSERT (priv);
-
-        pack = tmp;
-        if (strcmp (k, GLUSTERD_GLOBAL_OPT_VERSION) == 0)
-                return 0;
-
-        if (priv->op_version > GD_OP_VERSION_MIN) {
-                if ((strcmp (k, "features.limit-usage") == 0) ||
-                    (strcmp (k, "features.soft-limit") == 0))
-                        return 0;
-        }
-        snprintf (reconfig_key, 256, "volume%d.option.%s",
-                  pack->vol_count, k);
-        ret = dict_set_str (pack->dict, reconfig_key, v->data);
-        if (0 == ret)
-                pack->opt_count++;
+    char reconfig_key[256] = {
+        0,
+    };
+    int keylen;
+    struct args_pack *pack = NULL;
+    int ret = -1;
+    xlator_t *this = NULL;
+    glusterd_conf_t *priv = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+    priv = this->private;
+    GF_ASSERT(priv);
+
+    pack = tmp;
+    if (strcmp(k, GLUSTERD_GLOBAL_OPT_VERSION) == 0)
+        return 0;
 
+    if (priv->op_version > GD_OP_VERSION_MIN) {
+        if ((strcmp(k, "features.limit-usage") == 0) ||
+            (strcmp(k, "features.soft-limit") == 0))
+            return 0;
+    }
+
+    /* snap-max-hard-limit and snap-max-soft-limit are system   *
+     * options set and managed by snapshot config option. Hence *
+     * they should not be displayed in gluster volume info.     *
+     */
+    if ((strcmp(k, "snap-max-hard-limit") == 0) ||
+        (strcmp(k, "snap-max-soft-limit") == 0))
         return 0;
+
+    keylen = snprintf(reconfig_key, sizeof(reconfig_key), "volume%d.option.%s",
+                      pack->vol_count, k);
+    ret = dict_set_strn(pack->dict, reconfig_key, keylen, v->data);
+    if (0 == ret)
+        pack->opt_count++;
+
+    return 0;
 }
 
 int
-glusterd_add_volume_detail_to_dict (glusterd_volinfo_t *volinfo,
-                                    dict_t  *volumes, int count)
+glusterd_add_arbiter_info_to_bricks(glusterd_volinfo_t *volinfo,
+                                    dict_t *volumes, int count)
 {
-
-        int                     ret = -1;
-        char                    key[256] = {0, };
-        glusterd_brickinfo_t    *brickinfo = NULL;
-        char                    *buf = NULL;
-        int                     i = 1;
-        dict_t                  *dict = NULL;
-        glusterd_conf_t         *priv = NULL;
-        char                    *volume_id_str  = NULL;
-        struct args_pack        pack = {0,};
-        xlator_t                *this = NULL;
-        GF_UNUSED int           caps = 0;
-
-        GF_ASSERT (volinfo);
-        GF_ASSERT (volumes);
-
-        this = THIS;
-        priv = this->private;
-
-        GF_ASSERT (priv);
-
-        snprintf (key, 256, "volume%d.name", count);
-        ret = dict_set_str (volumes, key, volinfo->volname);
-        if (ret)
-                goto out;
-
-        snprintf (key, 256, "volume%d.type", count);
-        ret = dict_set_int32 (volumes, key, volinfo->type);
-        if (ret)
-                goto out;
-
-        snprintf (key, 256, "volume%d.status", count);
-        ret = dict_set_int32 (volumes, key, volinfo->status);
-        if (ret)
-                goto out;
-
-        snprintf (key, 256, "volume%d.brick_count", count);
-        ret = dict_set_int32 (volumes, key, volinfo->brick_count);
-        if (ret)
-                goto out;
-
-        snprintf (key, 256, "volume%d.dist_count", count);
-        ret = dict_set_int32 (volumes, key, volinfo->dist_leaf_count);
-        if (ret)
-                goto out;
-
-        snprintf (key, 256, "volume%d.stripe_count", count);
-        ret = dict_set_int32 (volumes, key, volinfo->stripe_count);
-        if (ret)
-                goto out;
-
-        snprintf (key, 256, "volume%d.replica_count", count);
-        ret = dict_set_int32 (volumes, key, volinfo->replica_count);
-        if (ret)
-                goto out;
-
-        snprintf (key, 256, "volume%d.disperse_count", count);
-        ret = dict_set_int32 (volumes, key, volinfo->disperse_count);
-        if (ret)
-                goto out;
-
-        snprintf (key, 256, "volume%d.redundancy_count", count);
-        ret = dict_set_int32 (volumes, key, volinfo->redundancy_count);
-        if (ret)
-                goto out;
-
-        snprintf (key, 256, "volume%d.transport", count);
-        ret = dict_set_int32 (volumes, key, volinfo->transport_type);
-        if (ret)
-                goto out;
-
-        volume_id_str = gf_strdup (uuid_utoa (volinfo->volume_id));
-        if (!volume_id_str)
-                goto out;
-
-        snprintf (key, sizeof (key), "volume%d.volume_id", count);
-        ret = dict_set_dynstr (volumes, key, volume_id_str);
-        if (ret)
-                goto out;
-
-        snprintf (key, 256, "volume%d.rebalance", count);
-        ret = dict_set_int32 (volumes, key, volinfo->rebal.defrag_cmd);
+    char key[64] = {
+        0,
+    };
+    int keylen;
+    int i = 0;
+    int ret = 0;
+
+    if (volinfo->replica_count == 1 || volinfo->arbiter_count != 1)
+        return 0;
+    for (i = 1; i <= volinfo->brick_count; i++) {
+        if (i % volinfo->replica_count != 0)
+            continue;
+        keylen = snprintf(key, sizeof(key), "volume%d.brick%d.isArbiter", count,
+                          i);
+        ret = dict_set_int32n(volumes, key, keylen, 1);
         if (ret)
-                goto out;
-
-#ifdef HAVE_BD_XLATOR
-        if (volinfo->caps) {
-                caps = 0;
-                snprintf (key, 256, "volume%d.xlator0", count);
-                buf = GF_MALLOC (256, gf_common_mt_char);
-                if (!buf) {
-                        ret = ENOMEM;
-                        goto out;
-                }
-                if (volinfo->caps & CAPS_BD)
-                        snprintf (buf, 256, "BD");
-                ret = dict_set_dynstr (volumes, key, buf);
-                if (ret) {
-                        GF_FREE (buf);
-                        goto out;
-                }
-
-                if (volinfo->caps & CAPS_THIN) {
-                        snprintf (key, 256, "volume%d.xlator0.caps%d", count,
-                                  caps++);
-                        buf = GF_MALLOC (256, gf_common_mt_char);
-                        if (!buf) {
-                                ret = ENOMEM;
-                                goto out;
-                        }
-                        snprintf (buf, 256, "thin");
-                        ret = dict_set_dynstr (volumes, key, buf);
-                        if (ret) {
-                                GF_FREE (buf);
-                                goto out;
-                        }
-                }
-
-                if (volinfo->caps & CAPS_OFFLOAD_COPY) {
-                        snprintf (key, 256, "volume%d.xlator0.caps%d", count,
-                                  caps++);
-                        buf = GF_MALLOC (256, gf_common_mt_char);
-                        if (!buf) {
-                                ret = ENOMEM;
-                                goto out;
-                        }
-                        snprintf (buf, 256, "offload_copy");
-                        ret = dict_set_dynstr (volumes, key, buf);
-                        if (ret) {
-                                GF_FREE (buf);
-                                goto out;
-                        }
-                }
-
-                if (volinfo->caps & CAPS_OFFLOAD_SNAPSHOT) {
-                        snprintf (key, 256, "volume%d.xlator0.caps%d", count,
-                                  caps++);
-                        buf = GF_MALLOC (256, gf_common_mt_char);
-                        if (!buf) {
-                                ret = ENOMEM;
-                                goto out;
-                        }
-                        snprintf (buf, 256, "offload_snapshot");
-                        ret = dict_set_dynstr (volumes, key, buf);
-                        if (ret)  {
-                                GF_FREE (buf);
-                                goto out;
-                        }
-                }
-
-                if (volinfo->caps & CAPS_OFFLOAD_ZERO) {
-                        snprintf (key, 256, "volume%d.xlator0.caps%d", count,
-                                  caps++);
-                        buf = GF_MALLOC (256, gf_common_mt_char);
-                        if (!buf) {
-                                ret = ENOMEM;
-                                goto out;
-                        }
-                        snprintf (buf, 256, "offload_zerofill");
-                        ret = dict_set_dynstr (volumes, key, buf);
-                        if (ret)  {
-                                GF_FREE (buf);
-                                goto out;
-                        }
-                }
-
-        }
-#endif
-
-        cds_list_for_each_entry (brickinfo, &volinfo->bricks, brick_list) {
-                char    brick[1024] = {0,};
-                char    brick_uuid[64] = {0,};
-                snprintf (key, 256, "volume%d.brick%d", count, i);
-                snprintf (brick, 1024, "%s:%s", brickinfo->hostname,
-                          brickinfo->path);
-                buf = gf_strdup (brick);
-                ret = dict_set_dynstr (volumes, key, buf);
-                if (ret)
-                        goto out;
-                snprintf (key, 256, "volume%d.brick%d.uuid", count, i);
-                snprintf (brick_uuid, 64, "%s", uuid_utoa (brickinfo->uuid));
-                buf = gf_strdup (brick_uuid);
-                if (!buf)
-                        goto out;
-                ret = dict_set_dynstr (volumes, key, buf);
-                if (ret)
-                        goto out;
-
-#ifdef HAVE_BD_XLATOR
-                if (volinfo->caps & CAPS_BD) {
-                        snprintf (key, 256, "volume%d.vg%d", count, i);
-                        snprintf (brick, 1024, "%s", brickinfo->vg);
-                        buf = gf_strdup (brick);
-                        ret = dict_set_dynstr (volumes, key, buf);
-                        if (ret)
-                                goto out;
-                }
-#endif
-                i++;
-        }
+            return ret;
+    }
+    return 0;
+}
 
-        dict = volinfo->dict;
-        if (!dict) {
-                ret = 0;
-                goto out;
-        }
+int
+glusterd_add_volume_detail_to_dict(glusterd_volinfo_t *volinfo, dict_t *volumes,
+                                   int count)
+{
+    int ret = -1;
+    char key[64] = {
+        0,
+    };
+    int keylen;
+    glusterd_brickinfo_t *brickinfo = NULL;
+    glusterd_brickinfo_t *ta_brickinfo = NULL;
+    char *buf = NULL;
+    int i = 1;
+    dict_t *dict = NULL;
+    glusterd_conf_t *priv = NULL;
+    char *volume_id_str = NULL;
+    struct args_pack pack = {
+        0,
+    };
+    xlator_t *this = NULL;
+    int32_t len = 0;
+
+    char ta_brick[4096] = {
+        0,
+    };
+
+    GF_ASSERT(volinfo);
+    GF_ASSERT(volumes);
+
+    this = THIS;
+    priv = this->private;
+
+    GF_ASSERT(priv);
+
+    keylen = snprintf(key, sizeof(key), "volume%d.name", count);
+    ret = dict_set_strn(volumes, key, keylen, volinfo->volname);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_SET_FAILED,
+                "Key=%s", key, NULL);
+        goto out;
+    }
+
+    keylen = snprintf(key, sizeof(key), "volume%d.type", count);
+    ret = dict_set_int32n(volumes, key, keylen, volinfo->type);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_SET_FAILED,
+                "Key=%s", key, NULL);
+        goto out;
+    }
+
+    keylen = snprintf(key, sizeof(key), "volume%d.status", count);
+    ret = dict_set_int32n(volumes, key, keylen, volinfo->status);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_SET_FAILED,
+                "Key=%s", key, NULL);
+        goto out;
+    }
+
+    keylen = snprintf(key, sizeof(key), "volume%d.brick_count", count);
+    ret = dict_set_int32n(volumes, key, keylen, volinfo->brick_count);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_SET_FAILED,
+                "Key=%s", key, NULL);
+        goto out;
+    }
+
+    keylen = snprintf(key, sizeof(key), "volume%d.dist_count", count);
+    ret = dict_set_int32n(volumes, key, keylen, volinfo->dist_leaf_count);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_SET_FAILED,
+                "Key=%s", key, NULL);
+        goto out;
+    }
+
+    keylen = snprintf(key, sizeof(key), "volume%d.stripe_count", count);
+    ret = dict_set_int32n(volumes, key, keylen, volinfo->stripe_count);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_SET_FAILED,
+                "Key=%s", key, NULL);
+        goto out;
+    }
+
+    keylen = snprintf(key, sizeof(key), "volume%d.replica_count", count);
+    ret = dict_set_int32n(volumes, key, keylen, volinfo->replica_count);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_SET_FAILED,
+                "Key=%s", key, NULL);
+        goto out;
+    }
+
+    keylen = snprintf(key, sizeof(key), "volume%d.disperse_count", count);
+    ret = dict_set_int32n(volumes, key, keylen, volinfo->disperse_count);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_SET_FAILED,
+                "Key=%s", key, NULL);
+        goto out;
+    }
+
+    keylen = snprintf(key, sizeof(key), "volume%d.redundancy_count", count);
+    ret = dict_set_int32n(volumes, key, keylen, volinfo->redundancy_count);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_SET_FAILED,
+                "Key=%s", key, NULL);
+        goto out;
+    }
+
+    keylen = snprintf(key, sizeof(key), "volume%d.arbiter_count", count);
+    ret = dict_set_int32n(volumes, key, keylen, volinfo->arbiter_count);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_SET_FAILED,
+                "Key=%s", key, NULL);
+        goto out;
+    }
+
+    keylen = snprintf(key, sizeof(key), "volume%d.transport", count);
+    ret = dict_set_int32n(volumes, key, keylen, volinfo->transport_type);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_SET_FAILED,
+                "Key=%s", key, NULL);
+        goto out;
+    }
+
+    keylen = snprintf(key, sizeof(key), "volume%d.thin_arbiter_count", count);
+    ret = dict_set_int32n(volumes, key, keylen, volinfo->thin_arbiter_count);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_SET_FAILED,
+                "Key=%s", key, NULL);
+        goto out;
+    }
+
+    volume_id_str = gf_strdup(uuid_utoa(volinfo->volume_id));
+    if (!volume_id_str) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_STRDUP_FAILED, NULL);
+        goto out;
+    }
+
+    keylen = snprintf(key, sizeof(key), "volume%d.volume_id", count);
+    ret = dict_set_dynstrn(volumes, key, keylen, volume_id_str);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_SET_FAILED,
+                "Key=%s", key, NULL);
+        goto out;
+    }
+
+    keylen = snprintf(key, sizeof(key), "volume%d.rebalance", count);
+    ret = dict_set_int32n(volumes, key, keylen, volinfo->rebal.defrag_cmd);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_SET_FAILED,
+                "Key=%s", key, NULL);
+        goto out;
+    }
+
+    keylen = snprintf(key, sizeof(key), "volume%d.snap_count", count);
+    ret = dict_set_int32n(volumes, key, keylen, volinfo->snap_count);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_SET_FAILED,
+                "Key=%s", key, NULL);
+        goto out;
+    }
+
+    cds_list_for_each_entry(brickinfo, &volinfo->bricks, brick_list)
+    {
+        char brick[1024] = {
+            0,
+        };
+        char brick_uuid[64] = {
+            0,
+        };
+        len = snprintf(brick, sizeof(brick), "%s:%s", brickinfo->hostname,
+                       brickinfo->path);
+        if ((len < 0) || (len >= sizeof(brick))) {
+            gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_COPY_FAIL, NULL);
+            ret = -1;
+            goto out;
+        }
+        buf = gf_strdup(brick);
+        keylen = snprintf(key, sizeof(key), "volume%d.brick%d", count, i);
+        ret = dict_set_dynstrn(volumes, key, keylen, buf);
+        if (ret) {
+            gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_SET_FAILED,
+                    "Key=%s", key, NULL);
+            goto out;
+        }
+        keylen = snprintf(key, sizeof(key), "volume%d.brick%d.uuid", count, i);
+        snprintf(brick_uuid, sizeof(brick_uuid), "%s",
+                 uuid_utoa(brickinfo->uuid));
+        buf = gf_strdup(brick_uuid);
+        if (!buf) {
+            gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_STRDUP_FAILED,
+                    "brick_uuid=%s", brick_uuid, NULL);
+            goto out;
+        }
+        ret = dict_set_dynstrn(volumes, key, keylen, buf);
+        if (ret) {
+            gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_SET_FAILED,
+                    "Key=%s", key, NULL);
+            goto out;
+        }
+
+        i++;
+    }
+    if (volinfo->thin_arbiter_count == 1) {
+        ta_brickinfo = list_first_entry(&volinfo->ta_bricks,
+                                        glusterd_brickinfo_t, brick_list);
+        len = snprintf(ta_brick, sizeof(ta_brick), "%s:%s",
+                       ta_brickinfo->hostname, ta_brickinfo->path);
+        if ((len < 0) || (len >= sizeof(ta_brick))) {
+            gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_COPY_FAIL, NULL);
+            ret = -1;
+            goto out;
+        }
+        buf = gf_strdup(ta_brick);
+        keylen = snprintf(key, sizeof(key), "volume%d.thin_arbiter_brick",
+                          count);
+        ret = dict_set_dynstrn(volumes, key, keylen, buf);
+        if (ret) {
+            gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_SET_FAILED,
+                    "Key=%s", key, NULL);
+            goto out;
+        }
+    }
+
+    ret = glusterd_add_arbiter_info_to_bricks(volinfo, volumes, count);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno,
+                GD_MSG_ARBITER_BRICK_SET_INFO_FAIL, NULL);
+        goto out;
+    }
+
+    dict = volinfo->dict;
+    if (!dict) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_CREATE_FAIL, NULL);
+        ret = 0;
+        goto out;
+    }
 
-        pack.dict = volumes;
-        pack.vol_count = count;
-        pack.opt_count = 0;
-        dict_foreach (dict, _build_option_key, (void *) &pack);
-        dict_foreach (priv->opts, _build_option_key, &pack);
+    pack.dict = volumes;
+    pack.vol_count = count;
+    pack.opt_count = 0;
+    dict_foreach(dict, _build_option_key, (void *)&pack);
+    dict_foreach(priv->opts, _build_option_key, &pack);
 
-        snprintf (key, 256, "volume%d.opt_count", pack.vol_count);
-        ret = dict_set_int32 (volumes, key, pack.opt_count);
+    keylen = snprintf(key, sizeof(key), "volume%d.opt_count", pack.vol_count);
+    ret = dict_set_int32n(volumes, key, keylen, pack.opt_count);
 out:
-        return ret;
+    return ret;
 }
 
 int32_t
-glusterd_op_txn_begin (rpcsvc_request_t *req, glusterd_op_t op, void *ctx,
-                       char *err_str, size_t err_len)
+glusterd_op_txn_begin(rpcsvc_request_t *req, glusterd_op_t op, void *ctx,
+                      char *err_str, size_t err_len)
 {
-        int32_t                     ret             = -1;
-        int                         npeers          = 0;
-        dict_t                     *dict            = NULL;
-        xlator_t                   *this            = NULL;
-        glusterd_conf_t            *priv            = NULL;
-        int32_t                     locked          = 0;
-        char                       *tmp             = NULL;
-        char                       *volname         = NULL;
-        uuid_t                     *txn_id          = NULL;
-        glusterd_op_info_t          txn_op_info     = {{0},};
-        glusterd_op_sm_event_type_t event_type      = GD_OP_EVENT_NONE;
-
-        GF_ASSERT (req);
-        GF_ASSERT ((op > GD_OP_NONE) && (op < GD_OP_MAX));
-        GF_ASSERT (NULL != ctx);
-
-        this = THIS;
-        GF_ASSERT (this);
-        priv = this->private;
-        GF_ASSERT (priv);
-
-        dict = ctx;
-
-        /* Generate a transaction-id for this operation and
-         * save it in the dict. This transaction id distinguishes
-         * each transaction, and helps separate opinfos in the
-         * op state machine. */
-        ret = glusterd_generate_txn_id (dict, &txn_id);
+    int32_t ret = -1;
+    dict_t *dict = NULL;
+    xlator_t *this = NULL;
+    glusterd_conf_t *priv = NULL;
+    int32_t locked = 0;
+    char *tmp = NULL;
+    char *volname = NULL;
+    uuid_t *txn_id = NULL;
+    glusterd_op_info_t txn_op_info = {
+        {0},
+    };
+    glusterd_op_sm_event_type_t event_type = GD_OP_EVENT_NONE;
+    uint32_t op_errno = 0;
+    uint32_t timeout = 0;
+
+    GF_ASSERT(req);
+    GF_ASSERT((op > GD_OP_NONE) && (op < GD_OP_MAX));
+    GF_ASSERT(NULL != ctx);
+
+    this = THIS;
+    GF_ASSERT(this);
+    priv = this->private;
+    GF_ASSERT(priv);
+
+    dict = ctx;
+
+    /* Generate a transaction-id for this operation and
+     * save it in the dict. This transaction id distinguishes
+     * each transaction, and helps separate opinfos in the
+     * op state machine. */
+    ret = glusterd_generate_txn_id(dict, &txn_id);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_TRANS_IDGEN_FAIL,
+               "Failed to generate transaction id");
+        goto out;
+    }
+
+    /* Save the MY_UUID as the originator_uuid. This originator_uuid
+     * will be used by is_origin_glusterd() to determine if a node
+     * is the originator node for a command. */
+    ret = glusterd_set_originator_uuid(dict);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_UUID_SET_FAIL,
+               "Failed to set originator_uuid.");
+        goto out;
+    }
+
+    /* Based on the op_version, acquire a cluster or mgmt_v3 lock */
+    if (priv->op_version < GD_OP_VERSION_3_6_0) {
+        ret = glusterd_lock(MY_UUID);
         if (ret) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "Failed to generate transaction id");
-                goto out;
-        }
-
-        /* Save the MY_UUID as the originator_uuid. This originator_uuid
-         * will be used by is_origin_glusterd() to determine if a node
-         * is the originator node for a command. */
-        ret = glusterd_set_originator_uuid (dict);
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_GLUSTERD_LOCK_FAIL,
+                   "Unable to acquire lock on localhost, ret: %d", ret);
+            snprintf(err_str, err_len,
+                     "Another transaction is in progress. "
+                     "Please try again after some time.");
+            goto out;
+        }
+    } else {
+        /* If no volname is given as a part of the command, locks will
+         * not be held */
+        ret = dict_get_strn(dict, "volname", SLEN("volname"), &tmp);
         if (ret) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "Failed to set originator_uuid.");
-                goto out;
-        }
-
-        /* Based on the op_version, acquire a cluster or mgmt_v3 lock */
-        if (priv->op_version < GD_OP_VERSION_3_6_0) {
-                ret = glusterd_lock (MY_UUID);
-                if (ret) {
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "Unable to acquire lock on localhost, ret: %d",
-                                ret);
-                        snprintf (err_str, err_len,
-                                  "Another transaction is in progress. "
-                                  "Please try again after sometime.");
-                        goto out;
-                }
+            gf_msg(this->name, GF_LOG_INFO, errno, GD_MSG_DICT_GET_FAILED,
+                   "No Volume name present. "
+                   "Locks not being held.");
+            goto local_locking_done;
         } else {
-                /* If no volname is given as a part of the command, locks will
-                 * not be held */
-                ret = dict_get_str (dict, "volname", &tmp);
-                if (ret) {
-                        gf_log ("", GF_LOG_INFO,
-                                "No Volume name present. "
-                                "Locks not being held.");
-                        goto local_locking_done;
-                } else {
-                        /* Use a copy of volname, as cli response will be
-                         * sent before the unlock, and the volname in the
-                         * dict, might be removed */
-                        volname = gf_strdup (tmp);
-                        if (!volname)
-                                goto out;
-                }
-
-                ret = glusterd_mgmt_v3_lock (volname, MY_UUID, "vol");
-                if (ret) {
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "Unable to acquire lock for %s", volname);
-                        snprintf (err_str, err_len,
-                                  "Another transaction is in progress for %s. "
-                                  "Please try again after sometime.", volname);
-                        goto out;
-                }
-        }
-
-        locked = 1;
-        gf_log (this->name, GF_LOG_DEBUG, "Acquired lock on localhost");
-
-local_locking_done:
-        txn_op_info.local_xaction_peers =
-                GF_CALLOC (1, sizeof (struct cds_list_head),
-                           gf_common_mt_list_head_t);
-        if (!txn_op_info.local_xaction_peers) {
-                ret = -1;
-                gf_log (this->name, GF_LOG_ERROR, "Out of memory");
+            /* Use a copy of volname, as cli response will be
+             * sent before the unlock, and the volname in the
+             * dict, might be removed */
+            volname = gf_strdup(tmp);
+            if (!volname)
                 goto out;
         }
-        CDS_INIT_LIST_HEAD (txn_op_info.local_xaction_peers);
-
-        /* Maintain xaction_peers on per transaction basis */
-        npeers = gd_build_local_xaction_peers_list
-                                        (&priv->peers,
-                                         txn_op_info.local_xaction_peers,
-                                         op);
 
-        /* If no volname is given as a part of the command, locks will
-         * not be held, hence sending stage event. */
-        if (volname || (priv->op_version < GD_OP_VERSION_3_6_0))
-                event_type = GD_OP_EVENT_START_LOCK;
-        else {
-                txn_op_info.state.state = GD_OP_STATE_LOCK_SENT;
-                event_type = GD_OP_EVENT_ALL_ACC;
-        }
+        /* Cli will add timeout key to dict if the default timeout is
+         * other than 2 minutes. Here we use this value to check whether
+         * mgmt_v3_lock_timeout should be set to default value or we
+         * need to change the value according to timeout value
+         * i.e, timeout + 120 seconds. */
+        ret = dict_get_uint32(dict, "timeout", &timeout);
+        if (!ret)
+            priv->mgmt_v3_lock_timeout = timeout + 120;
 
-        /* Save opinfo for this transaction with the transaction id */
-        glusterd_txn_opinfo_init (&txn_op_info, NULL, &op, ctx, req);
-
-        ret = glusterd_set_txn_opinfo (txn_id, &txn_op_info);
+        ret = glusterd_mgmt_v3_lock(volname, MY_UUID, &op_errno, "vol");
         if (ret) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "Unable to set transaction's opinfo");
-                if (ctx)
-                        dict_unref (ctx);
-                goto out;
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_MGMTV3_LOCK_GET_FAIL,
+                   "Unable to acquire lock for %s", volname);
+            snprintf(err_str, err_len,
+                     "Another transaction is in progress for %s. "
+                     "Please try again after some time.",
+                     volname);
+            goto out;
         }
+    }
 
-        ret = glusterd_op_sm_inject_event (event_type, txn_id, ctx);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "Failed to acquire cluster"
-                        " lock.");
-                goto out;
-        }
+    locked = 1;
+    gf_msg_debug(this->name, 0, "Acquired lock on localhost");
+
+local_locking_done:
+    /* If no volname is given as a part of the command, locks will
+     * not be held, hence sending stage event. */
+    if (volname || (priv->op_version < GD_OP_VERSION_3_6_0))
+        event_type = GD_OP_EVENT_START_LOCK;
+    else {
+        txn_op_info.state.state = GD_OP_STATE_LOCK_SENT;
+        event_type = GD_OP_EVENT_ALL_ACC;
+    }
+
+    /* Save opinfo for this transaction with the transaction id */
+    glusterd_txn_opinfo_init(&txn_op_info, NULL, &op, ctx, req);
+
+    ret = glusterd_set_txn_opinfo(txn_id, &txn_op_info);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_TRANS_OPINFO_SET_FAIL,
+               "Unable to set transaction's opinfo");
+        if (ctx)
+            dict_unref(ctx);
+        goto out;
+    }
+
+    ret = glusterd_op_sm_inject_event(event_type, txn_id, ctx);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_EVENT_INJECT_FAIL,
+               "Failed to acquire cluster"
+               " lock.");
+        goto out;
+    }
 
 out:
-        if (locked && ret) {
-                /* Based on the op-version, we release the
-                 * cluster or mgmt_v3 lock */
-                if (priv->op_version < GD_OP_VERSION_3_6_0)
-                        glusterd_unlock (MY_UUID);
-                else {
-                        ret = glusterd_mgmt_v3_unlock (volname, MY_UUID,
-                                                       "vol");
-                        if (ret)
-                                gf_log (this->name, GF_LOG_ERROR,
-                                        "Unable to release lock for %s",
-                                        volname);
-                        ret = -1;
-                }
+    if (locked && ret) {
+        /* Based on the op-version, we release the
+         * cluster or mgmt_v3 lock */
+        if (priv->op_version < GD_OP_VERSION_3_6_0)
+            glusterd_unlock(MY_UUID);
+        else {
+            ret = glusterd_mgmt_v3_unlock(volname, MY_UUID, "vol");
+            if (ret)
+                gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_MGMTV3_UNLOCK_FAIL,
+                       "Unable to release lock for %s", volname);
+            ret = -1;
         }
+    }
 
-        if (volname)
-                GF_FREE (volname);
+    if (volname)
+        GF_FREE(volname);
 
-        gf_log (this->name, GF_LOG_DEBUG, "Returning %d", ret);
-        return ret;
+    gf_msg_debug(this->name, 0, "Returning %d", ret);
+    return ret;
 }
 
 int
-__glusterd_handle_cluster_lock (rpcsvc_request_t *req)
+__glusterd_handle_cluster_lock(rpcsvc_request_t *req)
 {
-        dict_t                         *op_ctx      = NULL;
-        int32_t                         ret         = -1;
-        gd1_mgmt_cluster_lock_req       lock_req    = {{0},};
-        glusterd_op_lock_ctx_t         *ctx         = NULL;
-        glusterd_op_t                   op          = GD_OP_EVENT_LOCK;
-        glusterd_op_info_t              txn_op_info = {{0},};
-        glusterd_conf_t                *priv        = NULL;
-        uuid_t                         *txn_id      = NULL;
-        xlator_t                       *this        = NULL;
-
-        this = THIS;
-        GF_ASSERT (this);
-        priv = this->private;
-        GF_ASSERT (priv);
-        GF_ASSERT (req);
-
-        txn_id = &priv->global_txn_id;
-
-        ret = xdr_to_generic (req->msg[0], &lock_req,
-                              (xdrproc_t)xdr_gd1_mgmt_cluster_lock_req);
-        if (ret < 0) {
-                gf_log (this->name, GF_LOG_ERROR, "Failed to decode lock "
-                        "request received from peer");
-                req->rpc_err = GARBAGE_ARGS;
-                goto out;
-        }
-
-        gf_log (this->name, GF_LOG_DEBUG, "Received LOCK from uuid: %s",
-                uuid_utoa (lock_req.uuid));
-
-        rcu_read_lock ();
-        ret = (glusterd_peerinfo_find_by_uuid (lock_req.uuid) == NULL);
-        rcu_read_unlock ();
-        if (ret) {
-                gf_log (this->name, GF_LOG_WARNING, "%s doesn't "
-                        "belong to the cluster. Ignoring request.",
-                        uuid_utoa (lock_req.uuid));
-                ret = -1;
-                goto out;
-        }
-
-        ctx = GF_CALLOC (1, sizeof (*ctx), gf_gld_mt_op_lock_ctx_t);
-
-        if (!ctx) {
-                //respond here
-                return -1;
-        }
-
-        gf_uuid_copy (ctx->uuid, lock_req.uuid);
-        ctx->req = req;
-        ctx->dict = NULL;
+    dict_t *op_ctx = NULL;
+    int32_t ret = -1;
+    gd1_mgmt_cluster_lock_req lock_req = {
+        {0},
+    };
+    glusterd_op_lock_ctx_t *ctx = NULL;
+    glusterd_op_sm_event_type_t op = GD_OP_EVENT_LOCK;
+    glusterd_op_info_t txn_op_info = {
+        {0},
+    };
+    glusterd_conf_t *priv = NULL;
+    uuid_t *txn_id = NULL;
+    xlator_t *this = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+    priv = this->private;
+    GF_ASSERT(priv);
+    GF_ASSERT(req);
+
+    txn_id = &priv->global_txn_id;
+
+    ret = xdr_to_generic(req->msg[0], &lock_req,
+                         (xdrproc_t)xdr_gd1_mgmt_cluster_lock_req);
+    if (ret < 0) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_REQ_DECODE_FAIL,
+               "Failed to decode lock "
+               "request received from peer");
+        req->rpc_err = GARBAGE_ARGS;
+        goto out;
+    }
+
+    gf_msg_debug(this->name, 0, "Received LOCK from uuid: %s",
+                 uuid_utoa(lock_req.uuid));
+
+    RCU_READ_LOCK;
+    ret = (glusterd_peerinfo_find_by_uuid(lock_req.uuid) == NULL);
+    RCU_READ_UNLOCK;
+    if (ret) {
+        gf_msg(this->name, GF_LOG_WARNING, 0, GD_MSG_PEER_NOT_FOUND,
+               "%s doesn't "
+               "belong to the cluster. Ignoring request.",
+               uuid_utoa(lock_req.uuid));
+        ret = -1;
+        goto out;
+    }
 
-        op_ctx =  dict_new ();
-        if (!op_ctx) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "Unable to set new dict");
-                goto out;
-        }
+    ctx = GF_CALLOC(1, sizeof(*ctx), gf_gld_mt_op_lock_ctx_t);
 
-        glusterd_txn_opinfo_init (&txn_op_info, NULL, &op, op_ctx, req);
+    if (!ctx) {
+        // respond here
+        return -1;
+    }
 
-        ret = glusterd_set_txn_opinfo (txn_id, &txn_op_info);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "Unable to set transaction's opinfo");
-                dict_unref (txn_op_info.op_ctx);
-                goto out;
-        }
+    gf_uuid_copy(ctx->uuid, lock_req.uuid);
+    ctx->req = req;
+    ctx->dict = NULL;
 
-        ret = glusterd_op_sm_inject_event (GD_OP_EVENT_LOCK, txn_id, ctx);
-        if (ret)
-                gf_log (this->name, GF_LOG_ERROR,
-                        "Failed to inject event GD_OP_EVENT_LOCK");
+    op_ctx = dict_new();
+    if (!op_ctx) {
+        ret = -1;
+        gf_msg(this->name, GF_LOG_ERROR, ENOMEM, GD_MSG_DICT_CREATE_FAIL,
+               "Unable to set new dict");
+        goto out;
+    }
+
+    glusterd_txn_opinfo_init(&txn_op_info, NULL, &op, op_ctx, req);
+
+    ret = glusterd_set_txn_opinfo(txn_id, &txn_op_info);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_TRANS_OPINFO_SET_FAIL,
+               "Unable to set transaction's opinfo");
+        dict_unref(txn_op_info.op_ctx);
+        goto out;
+    }
+
+    ret = glusterd_op_sm_inject_event(GD_OP_EVENT_LOCK, txn_id, ctx);
+    if (ret)
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_EVENT_INJECT_FAIL,
+               "Failed to inject event GD_OP_EVENT_LOCK");
 
 out:
-        gf_log (this->name, GF_LOG_DEBUG, "Returning %d", ret);
+    gf_msg_debug(this->name, 0, "Returning %d", ret);
 
-        glusterd_friend_sm ();
-        glusterd_op_sm ();
+    glusterd_friend_sm();
+    glusterd_op_sm();
 
-        return ret;
+    if (ret)
+        GF_FREE(ctx);
+
+    return ret;
 }
 
 int
-glusterd_handle_cluster_lock (rpcsvc_request_t *req)
+glusterd_handle_cluster_lock(rpcsvc_request_t *req)
 {
-        return glusterd_big_locked_handler (req,
-                                            __glusterd_handle_cluster_lock);
+    return glusterd_big_locked_handler(req, __glusterd_handle_cluster_lock);
 }
 
-int
-glusterd_req_ctx_create (rpcsvc_request_t *rpc_req,
-                         glusterd_op_t op, uuid_t uuid,
-                         char *buf_val, size_t buf_len,
-                         gf_gld_mem_types_t mem_type,
-                         glusterd_req_ctx_t **req_ctx_out)
+static int
+glusterd_req_ctx_create(rpcsvc_request_t *rpc_req, int op, uuid_t uuid,
+                        char *buf_val, size_t buf_len,
+                        gf_gld_mem_types_t mem_type,
+                        glusterd_req_ctx_t **req_ctx_out)
 {
-        int                 ret     = -1;
-        char                str[50] = {0,};
-        glusterd_req_ctx_t *req_ctx = NULL;
-        dict_t             *dict    = NULL;
-        xlator_t           *this    = NULL;
-
-        this = THIS;
-        GF_ASSERT (this);
-
-        gf_uuid_unparse (uuid, str);
-        gf_log (this->name, GF_LOG_DEBUG, "Received op from uuid %s", str);
-
-        dict = dict_new ();
-        if (!dict)
-                goto out;
-
-        req_ctx = GF_CALLOC (1, sizeof (*req_ctx), mem_type);
-        if (!req_ctx) {
-                goto out;
-        }
-
-        gf_uuid_copy (req_ctx->uuid, uuid);
-        req_ctx->op = op;
-        ret = dict_unserialize (buf_val, buf_len, &dict);
-        if (ret) {
-                gf_log (this->name, GF_LOG_WARNING,
-                        "failed to unserialize the dictionary");
-                goto out;
-        }
-
-        req_ctx->dict = dict;
-        req_ctx->req = rpc_req;
-        *req_ctx_out = req_ctx;
-        ret = 0;
+    int ret = -1;
+    char str[50] = {
+        0,
+    };
+    glusterd_req_ctx_t *req_ctx = NULL;
+    dict_t *dict = NULL;
+    xlator_t *this = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+
+    gf_uuid_unparse(uuid, str);
+    gf_msg_debug(this->name, 0, "Received op from uuid %s", str);
+
+    dict = dict_new();
+    if (!dict) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_CREATE_FAIL, NULL);
+        goto out;
+    }
+
+    req_ctx = GF_CALLOC(1, sizeof(*req_ctx), mem_type);
+    if (!req_ctx) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_NO_MEMORY, NULL);
+        goto out;
+    }
+
+    gf_uuid_copy(req_ctx->uuid, uuid);
+    req_ctx->op = op;
+    ret = dict_unserialize(buf_val, buf_len, &dict);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_WARNING, 0, GD_MSG_DICT_UNSERIALIZE_FAIL,
+                NULL);
+        goto out;
+    }
+
+    req_ctx->dict = dict;
+    req_ctx->req = rpc_req;
+    *req_ctx_out = req_ctx;
+    ret = 0;
 out:
-        if (ret) {
-                if (dict)
-                        dict_unref (dict);
-                GF_FREE (req_ctx);
-        }
-        return ret;
+    if (ret) {
+        if (dict)
+            dict_unref(dict);
+        GF_FREE(req_ctx);
+    }
+    return ret;
 }
 
 int
-__glusterd_handle_stage_op (rpcsvc_request_t *req)
+__glusterd_handle_stage_op(rpcsvc_request_t *req)
 {
-        int32_t                         ret = -1;
-        glusterd_req_ctx_t              *req_ctx = NULL;
-        gd1_mgmt_stage_op_req           op_req = {{0},};
-        xlator_t                        *this = NULL;
-        uuid_t                          *txn_id = NULL;
-        glusterd_op_info_t              txn_op_info = {{0},};
-        glusterd_op_sm_state_info_t     state = {0,};
-        glusterd_conf_t                 *priv = NULL;
-
-        this = THIS;
-        GF_ASSERT (this);
-        priv = this->private;
-        GF_ASSERT (priv);
-        GF_ASSERT (req);
-
-        txn_id = &priv->global_txn_id;
-
-        ret = xdr_to_generic (req->msg[0], &op_req,
-                              (xdrproc_t)xdr_gd1_mgmt_stage_op_req);
-        if (ret < 0) {
-                gf_log (this->name, GF_LOG_ERROR, "Failed to decode stage "
-                        "request received from peer");
-                req->rpc_err = GARBAGE_ARGS;
-                goto out;
-        }
-
-        ret = glusterd_req_ctx_create (req, op_req.op, op_req.uuid,
-                                       op_req.buf.buf_val, op_req.buf.buf_len,
-                                       gf_gld_mt_op_stage_ctx_t, &req_ctx);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "Failed to create req_ctx");
-                goto out;
-        }
-
-        ret = dict_get_bin (req_ctx->dict, "transaction_id", (void **)&txn_id);
-        gf_log (this->name, GF_LOG_DEBUG, "transaction ID = %s",
-                uuid_utoa (*txn_id));
-
-        rcu_read_lock ();
-        ret = (glusterd_peerinfo_find_by_uuid (op_req.uuid) == NULL);
-        rcu_read_unlock ();
-        if (ret) {
-                gf_log (this->name, GF_LOG_WARNING, "%s doesn't "
-                        "belong to the cluster. Ignoring request.",
-                        uuid_utoa (op_req.uuid));
-                ret = -1;
-                goto out;
-        }
-
-        /* In cases where there is no volname, the receivers won't have a
-         * transaction opinfo created, as for those operations, the locking
-         * phase where the transaction opinfos are created, won't be called. */
-        ret = glusterd_get_txn_opinfo (txn_id, &txn_op_info);
+    int32_t ret = -1;
+    glusterd_req_ctx_t *req_ctx = NULL;
+    gd1_mgmt_stage_op_req op_req = {
+        {0},
+    };
+    xlator_t *this = NULL;
+    uuid_t *txn_id = NULL;
+    glusterd_op_info_t txn_op_info = {
+        {0},
+    };
+    glusterd_op_sm_state_info_t state = {
+        0,
+    };
+    glusterd_conf_t *priv = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+    priv = this->private;
+    GF_ASSERT(priv);
+    GF_ASSERT(req);
+
+    txn_id = &priv->global_txn_id;
+
+    ret = xdr_to_generic(req->msg[0], &op_req,
+                         (xdrproc_t)xdr_gd1_mgmt_stage_op_req);
+    if (ret < 0) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_REQ_DECODE_FAIL,
+               "Failed to decode stage "
+               "request received from peer");
+        req->rpc_err = GARBAGE_ARGS;
+        goto out;
+    }
+
+    ret = glusterd_req_ctx_create(req, op_req.op, op_req.uuid,
+                                  op_req.buf.buf_val, op_req.buf.buf_len,
+                                  gf_gld_mt_op_stage_ctx_t, &req_ctx);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_REQ_CTX_CREATE_FAIL,
+               "Failed to create req_ctx");
+        goto out;
+    }
+
+    ret = dict_get_bin(req_ctx->dict, "transaction_id", (void **)&txn_id);
+    gf_msg_debug(this->name, 0, "transaction ID = %s", uuid_utoa(*txn_id));
+
+    RCU_READ_LOCK;
+    ret = (glusterd_peerinfo_find_by_uuid(op_req.uuid) == NULL);
+    RCU_READ_UNLOCK;
+    if (ret) {
+        gf_msg(this->name, GF_LOG_WARNING, 0, GD_MSG_PEER_NOT_FOUND,
+               "%s doesn't "
+               "belong to the cluster. Ignoring request.",
+               uuid_utoa(op_req.uuid));
+        ret = -1;
+        goto out;
+    }
+
+    /* In cases where there is no volname, the receivers won't have a
+     * transaction opinfo created, as for those operations, the locking
+     * phase where the transaction opinfos are created, won't be called.
+     * skip_locking will be true for all such transaction and we clear
+     * the txn_opinfo after the staging phase, except for geo-replication
+     * operations where we need to access txn_opinfo in the later phases also.
+     */
+    ret = glusterd_get_txn_opinfo(txn_id, &txn_op_info);
+    if (ret) {
+        gf_msg_debug(this->name, 0, "No transaction's opinfo set");
+
+        state.state = GD_OP_STATE_LOCKED;
+        glusterd_txn_opinfo_init(&txn_op_info, &state, &op_req.op,
+                                 req_ctx->dict, req);
+
+        if (req_ctx->op != GD_OP_GSYNC_SET)
+            txn_op_info.skip_locking = _gf_true;
+        ret = glusterd_set_txn_opinfo(txn_id, &txn_op_info);
         if (ret) {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "No transaction's opinfo set");
-
-                state.state = GD_OP_STATE_LOCKED;
-                glusterd_txn_opinfo_init (&txn_op_info, &state,
-                                          &op_req.op, req_ctx->dict, req);
-
-                ret = glusterd_set_txn_opinfo (txn_id, &txn_op_info);
-                if (ret) {
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "Unable to set transaction's opinfo");
-                        dict_unref (req_ctx->dict);
-                        goto out;
-                }
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_TRANS_OPINFO_SET_FAIL,
+                   "Unable to set transaction's opinfo");
+            dict_unref(req_ctx->dict);
+            goto out;
         }
+    }
 
-        ret = glusterd_op_sm_inject_event (GD_OP_EVENT_STAGE_OP,
-                                           txn_id, req_ctx);
-        if (ret)
-                gf_log (this->name, GF_LOG_ERROR,
-                        "Failed to inject event GD_OP_EVENT_STAGE_OP");
+    ret = glusterd_op_sm_inject_event(GD_OP_EVENT_STAGE_OP, txn_id, req_ctx);
+    if (ret)
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_EVENT_INJECT_FAIL,
+               "Failed to inject event GD_OP_EVENT_STAGE_OP");
 
- out:
-        free (op_req.buf.buf_val);//malloced by xdr
-        glusterd_friend_sm ();
-        glusterd_op_sm ();
-        return ret;
+out:
+    free(op_req.buf.buf_val);  // malloced by xdr
+    glusterd_friend_sm();
+    glusterd_op_sm();
+    return ret;
 }
 
 int
-glusterd_handle_stage_op (rpcsvc_request_t *req)
+glusterd_handle_stage_op(rpcsvc_request_t *req)
 {
-        return glusterd_big_locked_handler (req, __glusterd_handle_stage_op);
+    return glusterd_big_locked_handler(req, __glusterd_handle_stage_op);
 }
 
-
 int
-__glusterd_handle_commit_op (rpcsvc_request_t *req)
+__glusterd_handle_commit_op(rpcsvc_request_t *req)
 {
-        int32_t                         ret = -1;
-        glusterd_req_ctx_t              *req_ctx = NULL;
-        gd1_mgmt_commit_op_req          op_req = {{0},};
-        xlator_t                        *this = NULL;
-        uuid_t                          *txn_id = NULL;
-        glusterd_conf_t                 *priv = NULL;
-
-        this = THIS;
-        GF_ASSERT (this);
-        priv = this->private;
-        GF_ASSERT (priv);
-        GF_ASSERT (req);
-
-        txn_id = &priv->global_txn_id;
-
-        ret = xdr_to_generic (req->msg[0], &op_req,
-                              (xdrproc_t)xdr_gd1_mgmt_commit_op_req);
-        if (ret < 0) {
-                gf_log (this->name, GF_LOG_ERROR, "Failed to decode commit "
-                        "request received from peer");
-                req->rpc_err = GARBAGE_ARGS;
-                goto out;
-        }
-
-        rcu_read_lock ();
-        ret = (glusterd_peerinfo_find_by_uuid (op_req.uuid) == NULL);
-        rcu_read_unlock ();
-        if (ret) {
-                gf_log (this->name, GF_LOG_WARNING, "%s doesn't "
-                        "belong to the cluster. Ignoring request.",
-                        uuid_utoa (op_req.uuid));
-                ret = -1;
-                goto out;
-        }
+    int32_t ret = -1;
+    glusterd_req_ctx_t *req_ctx = NULL;
+    gd1_mgmt_commit_op_req op_req = {
+        {0},
+    };
+    xlator_t *this = NULL;
+    uuid_t *txn_id = NULL;
+    glusterd_conf_t *priv = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+    priv = this->private;
+    GF_ASSERT(priv);
+    GF_ASSERT(req);
+
+    txn_id = &priv->global_txn_id;
+
+    ret = xdr_to_generic(req->msg[0], &op_req,
+                         (xdrproc_t)xdr_gd1_mgmt_commit_op_req);
+    if (ret < 0) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_REQ_DECODE_FAIL,
+               "Failed to decode commit "
+               "request received from peer");
+        req->rpc_err = GARBAGE_ARGS;
+        goto out;
+    }
+
+    RCU_READ_LOCK;
+    ret = (glusterd_peerinfo_find_by_uuid(op_req.uuid) == NULL);
+    RCU_READ_UNLOCK;
+    if (ret) {
+        gf_msg(this->name, GF_LOG_WARNING, 0, GD_MSG_PEER_NOT_FOUND,
+               "%s doesn't "
+               "belong to the cluster. Ignoring request.",
+               uuid_utoa(op_req.uuid));
+        ret = -1;
+        goto out;
+    }
 
-        //the structures should always be equal
-        GF_ASSERT (sizeof (gd1_mgmt_commit_op_req) == sizeof (gd1_mgmt_stage_op_req));
-        ret = glusterd_req_ctx_create (req, op_req.op, op_req.uuid,
-                                       op_req.buf.buf_val, op_req.buf.buf_len,
-                                       gf_gld_mt_op_commit_ctx_t, &req_ctx);
-        if (ret)
-                goto out;
+    // the structures should always be equal
+    GF_ASSERT(sizeof(gd1_mgmt_commit_op_req) == sizeof(gd1_mgmt_stage_op_req));
+    ret = glusterd_req_ctx_create(req, op_req.op, op_req.uuid,
+                                  op_req.buf.buf_val, op_req.buf.buf_len,
+                                  gf_gld_mt_op_commit_ctx_t, &req_ctx);
+    if (ret)
+        goto out;
 
-        ret = dict_get_bin (req_ctx->dict, "transaction_id", (void **)&txn_id);
-        gf_log (this->name, GF_LOG_DEBUG, "transaction ID = %s",
-                uuid_utoa (*txn_id));
+    ret = dict_get_bin(req_ctx->dict, "transaction_id", (void **)&txn_id);
+    gf_msg_debug(this->name, 0, "transaction ID = %s", uuid_utoa(*txn_id));
 
-        ret = glusterd_op_sm_inject_event (GD_OP_EVENT_COMMIT_OP,
-                                           txn_id, req_ctx);
+    ret = glusterd_op_sm_inject_event(GD_OP_EVENT_COMMIT_OP, txn_id, req_ctx);
 
 out:
-        free (op_req.buf.buf_val);//malloced by xdr
-        glusterd_friend_sm ();
-        glusterd_op_sm ();
-        return ret;
+    free(op_req.buf.buf_val);  // malloced by xdr
+    glusterd_friend_sm();
+    glusterd_op_sm();
+    return ret;
 }
 
 int
-glusterd_handle_commit_op (rpcsvc_request_t *req)
+glusterd_handle_commit_op(rpcsvc_request_t *req)
 {
-        return glusterd_big_locked_handler (req, __glusterd_handle_commit_op);
+    return glusterd_big_locked_handler(req, __glusterd_handle_commit_op);
 }
 
 int
-__glusterd_handle_cli_probe (rpcsvc_request_t *req)
+__glusterd_handle_cli_probe(rpcsvc_request_t *req)
 {
-        int32_t                         ret = -1;
-        gf_cli_req                  cli_req = {{0,},};
-        glusterd_peerinfo_t       *peerinfo = NULL;
-        gf_boolean_t                run_fsm = _gf_true;
-        xlator_t                      *this = NULL;
-        char                     *bind_name = NULL;
-        dict_t                        *dict = NULL;
-        char                      *hostname = NULL;
-        int                            port = 0;
-        int                        op_errno = 0;
-
-        GF_ASSERT (req);
-        this = THIS;
-
-        ret = xdr_to_generic (req->msg[0], &cli_req, (xdrproc_t)xdr_gf_cli_req);
-        if (ret < 0)  {
-                //failed to decode msg;
-                gf_log ("", GF_LOG_ERROR, "xdr decoding error");
-                req->rpc_err = GARBAGE_ARGS;
-                goto out;
-        }
-
-        if (cli_req.dict.dict_len) {
-                dict = dict_new ();
-
-                ret = dict_unserialize (cli_req.dict.dict_val,
-                                        cli_req.dict.dict_len, &dict);
-                if (ret < 0) {
-                        gf_log (this->name, GF_LOG_ERROR, "Failed to "
-                                "unserialize req-buffer to dictionary");
-                        goto out;
-                }
-        }
-
-        ret = dict_get_str (dict, "hostname", &hostname);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "Failed to get hostname");
-                goto out;
-        }
-
-        ret = dict_get_int32 (dict, "port", &port);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "Failed to get port");
-                goto out;
-        }
-
-        if (glusterd_is_any_volume_in_server_quorum (this) &&
-            !does_gd_meet_server_quorum (this)) {
-                glusterd_xfer_cli_probe_resp (req, -1, GF_PROBE_QUORUM_NOT_MET,
-                                              NULL, hostname, port, dict);
-                gf_msg (this->name, GF_LOG_CRITICAL, 0,
-                        GD_MSG_SERVER_QUORUM_NOT_MET,
-                        "Server quorum not met. Rejecting operation.");
-                ret = 0;
-                goto out;
-        }
-
-        gf_log ("glusterd", GF_LOG_INFO, "Received CLI probe req %s %d",
-                hostname, port);
+    int32_t ret = -1;
+    gf_cli_req cli_req = {
+        {
+            0,
+        },
+    };
+    glusterd_peerinfo_t *peerinfo = NULL;
+    gf_boolean_t run_fsm = _gf_true;
+    xlator_t *this = NULL;
+    char *bind_name = NULL;
+    dict_t *dict = NULL;
+    char *hostname = NULL;
+    int port = 0;
+    int op_errno = 0;
+
+    GF_ASSERT(req);
+    this = THIS;
+
+    ret = xdr_to_generic(req->msg[0], &cli_req, (xdrproc_t)xdr_gf_cli_req);
+    if (ret < 0) {
+        // failed to decode msg;
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_REQ_DECODE_FAIL,
+               "xdr decoding error");
+        req->rpc_err = GARBAGE_ARGS;
+        goto out;
+    }
+
+    if (cli_req.dict.dict_len) {
+        dict = dict_new();
 
-        if (dict_get_str(this->options,"transport.socket.bind-address",
-                         &bind_name) == 0) {
-                gf_log ("glusterd", GF_LOG_DEBUG,
-                        "only checking probe address vs. bind address");
-                ret = gf_is_same_address (bind_name, hostname);
-        }
-        else {
-                ret = gf_is_local_addr (hostname);
-        }
-        if (ret) {
-                glusterd_xfer_cli_probe_resp (req, 0, GF_PROBE_LOCALHOST,
-                                              NULL, hostname, port, dict);
-                ret = 0;
-                goto out;
-        }
+        ret = dict_unserialize(cli_req.dict.dict_val, cli_req.dict.dict_len,
+                               &dict);
+        if (ret < 0) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_UNSERIALIZE_FAIL,
+                   "Failed to "
+                   "unserialize req-buffer to dictionary");
+            goto out;
+        }
+    }
+
+    ret = dict_get_strn(dict, "hostname", SLEN("hostname"), &hostname);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_HOSTNAME_NOTFOUND_IN_DICT,
+               "Failed to get hostname");
+        goto out;
+    }
+
+    ret = dict_get_int32n(dict, "port", SLEN("port"), &port);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_PORT_NOTFOUND_IN_DICT,
+               "Failed to get port");
+        goto out;
+    }
+
+    if (glusterd_is_any_volume_in_server_quorum(this) &&
+        !does_gd_meet_server_quorum(this)) {
+        glusterd_xfer_cli_probe_resp(req, -1, GF_PROBE_QUORUM_NOT_MET, NULL,
+                                     hostname, port, dict);
+        gf_msg(this->name, GF_LOG_CRITICAL, 0, GD_MSG_SERVER_QUORUM_NOT_MET,
+               "Server quorum not met. Rejecting operation.");
+        ret = 0;
+        goto out;
+    }
+
+    gf_msg("glusterd", GF_LOG_INFO, 0, GD_MSG_CLI_REQ_RECVD,
+           "Received CLI probe req %s %d", hostname, port);
+
+    if (dict_get_strn(this->options, "transport.socket.bind-address",
+                      SLEN("transport.socket.bind-address"), &bind_name) == 0) {
+        gf_msg_debug("glusterd", 0,
+                     "only checking probe address vs. bind address");
+        ret = gf_is_same_address(bind_name, hostname);
+    } else {
+        ret = gf_is_local_addr(hostname);
+    }
+    if (ret) {
+        glusterd_xfer_cli_probe_resp(req, 0, GF_PROBE_LOCALHOST, NULL, hostname,
+                                     port, dict);
+        ret = 0;
+        goto out;
+    }
 
-        rcu_read_lock ();
+    RCU_READ_LOCK;
 
-        peerinfo = glusterd_peerinfo_find_by_hostname (hostname);
-        ret = (peerinfo && gd_peer_has_address (peerinfo, hostname));
+    peerinfo = glusterd_peerinfo_find_by_hostname(hostname);
+    ret = (peerinfo && gd_peer_has_address(peerinfo, hostname));
 
-        rcu_read_unlock ();
+    RCU_READ_UNLOCK;
 
-        if (ret) {
-                gf_log ("glusterd", GF_LOG_DEBUG, "Probe host %s port %d "
-                        "already a peer", hostname, port);
-                glusterd_xfer_cli_probe_resp (req, 0, GF_PROBE_FRIEND, NULL,
-                                              hostname, port, dict);
-                ret = 0;
-                goto out;
-        }
+    if (ret) {
+        gf_msg_debug("glusterd", 0,
+                     "Probe host %s port %d "
+                     "already a peer",
+                     hostname, port);
+        glusterd_xfer_cli_probe_resp(req, 0, GF_PROBE_FRIEND, NULL, hostname,
+                                     port, dict);
+        ret = 0;
+        goto out;
+    }
 
-        ret = glusterd_probe_begin (req, hostname, port, dict, &op_errno);
+    ret = glusterd_probe_begin(req, hostname, port, dict, &op_errno);
 
-        if (ret == GLUSTERD_CONNECTION_AWAITED) {
-                //fsm should be run after connection establishes
-                run_fsm = _gf_false;
-                ret = 0;
+    if (ret == GLUSTERD_CONNECTION_AWAITED) {
+        // fsm should be run after connection establishes
+        run_fsm = _gf_false;
+        ret = 0;
 
-        } else if (ret == -1) {
-                glusterd_xfer_cli_probe_resp (req, -1, op_errno,
-                                              NULL, hostname, port, dict);
-                goto out;
-        }
+    } else if (ret == -1) {
+        glusterd_xfer_cli_probe_resp(req, -1, op_errno, NULL, hostname, port,
+                                     dict);
+        goto out;
+    }
 
 out:
-        free (cli_req.dict.dict_val);
+    free(cli_req.dict.dict_val);
 
-        if (run_fsm) {
-                glusterd_friend_sm ();
-                glusterd_op_sm ();
-        }
+    if (run_fsm) {
+        glusterd_friend_sm();
+        glusterd_op_sm();
+    }
 
-        return ret;
+    return ret;
 }
 
 int
-glusterd_handle_cli_probe (rpcsvc_request_t *req)
+glusterd_handle_cli_probe(rpcsvc_request_t *req)
 {
-        return glusterd_big_locked_handler (req, __glusterd_handle_cli_probe);
+    return glusterd_big_locked_handler(req, __glusterd_handle_cli_probe);
 }
 
 int
-__glusterd_handle_cli_deprobe (rpcsvc_request_t *req)
+__glusterd_handle_cli_deprobe(rpcsvc_request_t *req)
 {
-        int32_t                         ret = -1;
-        gf_cli_req                  cli_req = {{0,},};
-        uuid_t                         uuid = {0};
-        int                        op_errno = 0;
-        xlator_t                      *this = NULL;
-        glusterd_conf_t               *priv = NULL;
-        dict_t                        *dict = NULL;
-        char                      *hostname = NULL;
-        int                            port = 0;
-        int                           flags = 0;
-        glusterd_volinfo_t         *volinfo = NULL;
-        glusterd_volinfo_t             *tmp = NULL;
-
-        this = THIS;
-        GF_ASSERT (this);
-        priv = this->private;
-        GF_ASSERT (priv);
-        GF_ASSERT (req);
-
-        ret = xdr_to_generic (req->msg[0], &cli_req,
-                              (xdrproc_t)xdr_gf_cli_req);
-        if (ret < 0) {
-                //failed to decode msg;
-                req->rpc_err = GARBAGE_ARGS;
-                goto out;
-        }
-
-        if (cli_req.dict.dict_len) {
-                dict = dict_new ();
-
-                ret = dict_unserialize (cli_req.dict.dict_val,
-                                        cli_req.dict.dict_len, &dict);
-                if (ret < 0) {
-                        gf_log (this->name, GF_LOG_ERROR, "Failed to "
-                                "unserialize req-buffer to dictionary");
-                        goto out;
-                }
-        }
-
-        gf_log ("glusterd", GF_LOG_INFO, "Received CLI deprobe req");
-
-        ret = dict_get_str (dict, "hostname", &hostname);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "Failed to get hostname");
-                goto out;
-        }
-
-        ret = dict_get_int32 (dict, "port", &port);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "Failed to get port");
-                goto out;
-        }
-        ret = dict_get_int32 (dict, "flags", &flags);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "Failed to get flags");
-                goto out;
-        }
-
-        ret = glusterd_hostname_to_uuid (hostname, uuid);
-        if (ret) {
-                op_errno = GF_DEPROBE_NOT_FRIEND;
-                goto out;
-        }
-
-        if (!gf_uuid_compare (uuid, MY_UUID)) {
-                op_errno = GF_DEPROBE_LOCALHOST;
-                ret = -1;
-                goto out;
-        }
-
-        if (!(flags & GF_CLI_FLAG_OP_FORCE)) {
-                /* Check if peers are connected, except peer being
-                * detached*/
-                if (!glusterd_chk_peers_connected_befriended (uuid)) {
-                        ret = -1;
-                        op_errno = GF_DEPROBE_FRIEND_DOWN;
-                        goto out;
-                }
-        }
-
-        /* Check for if volumes exist with some bricks on the peer being
-        * detached. It's not a problem if a volume contains none or all
-        * of its bricks on the peer being detached
-        */
-        cds_list_for_each_entry_safe (volinfo, tmp, &priv->volumes,
-                                      vol_list) {
-                ret = glusterd_friend_contains_vol_bricks (volinfo,
-                                                           uuid);
-                if (ret == 1) {
-                        op_errno = GF_DEPROBE_BRICK_EXIST;
-                        goto out;
-                }
-        }
-
-        if (!(flags & GF_CLI_FLAG_OP_FORCE)) {
-                if (glusterd_is_any_volume_in_server_quorum (this) &&
-                    !does_gd_meet_server_quorum (this)) {
-                        gf_msg (this->name, GF_LOG_CRITICAL, 0,
-                                GD_MSG_SERVER_QUORUM_NOT_MET,
-                                "Server quorum not met. Rejecting operation.");
-                        ret = -1;
-                        op_errno = GF_DEPROBE_QUORUM_NOT_MET;
-                        goto out;
-                }
-        }
+    int32_t ret = -1;
+    gf_cli_req cli_req = {
+        {
+            0,
+        },
+    };
+    uuid_t uuid = {0};
+    int op_errno = 0;
+    xlator_t *this = NULL;
+    glusterd_conf_t *priv = NULL;
+    dict_t *dict = NULL;
+    char *hostname = NULL;
+    int port = 0;
+    int flags = 0;
+    glusterd_volinfo_t *volinfo = NULL;
+    glusterd_volinfo_t *tmp = NULL;
+    glusterd_snap_t *snapinfo = NULL;
+    glusterd_snap_t *tmpsnap = NULL;
+    gf_boolean_t need_free = _gf_false;
+
+    this = THIS;
+    GF_ASSERT(this);
+    priv = this->private;
+    GF_ASSERT(priv);
+    GF_ASSERT(req);
+
+    ret = xdr_to_generic(req->msg[0], &cli_req, (xdrproc_t)xdr_gf_cli_req);
+    if (ret < 0) {
+        // failed to decode msg;
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_REQ_DECODE_FAIL,
+               "Failed to decode "
+               "request received from cli");
+        req->rpc_err = GARBAGE_ARGS;
+        goto out;
+    }
+
+    if (cli_req.dict.dict_len) {
+        dict = dict_new();
 
-        if (!gf_uuid_is_null (uuid)) {
-                ret = glusterd_deprobe_begin (req, hostname, port, uuid, dict,
-                                              &op_errno);
+        if (dict) {
+            need_free = _gf_true;
         } else {
-                ret = glusterd_deprobe_begin (req, hostname, port, NULL, dict,
-                                              &op_errno);
+            ret = -1;
+            goto out;
         }
 
+        ret = dict_unserialize(cli_req.dict.dict_val, cli_req.dict.dict_len,
+                               &dict);
+        if (ret < 0) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_UNSERIALIZE_FAIL,
+                   "Failed to "
+                   "unserialize req-buffer to dictionary");
+            goto out;
+        }
+    }
+
+    gf_msg("glusterd", GF_LOG_INFO, 0, GD_MSG_CLI_REQ_RECVD,
+           "Received CLI deprobe req");
+
+    ret = dict_get_strn(dict, "hostname", SLEN("hostname"), &hostname);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_HOSTNAME_NOTFOUND_IN_DICT,
+               "Failed to get hostname");
+        goto out;
+    }
+
+    ret = dict_get_int32n(dict, "port", SLEN("port"), &port);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_PORT_NOTFOUND_IN_DICT,
+               "Failed to get port");
+        goto out;
+    }
+    ret = dict_get_int32n(dict, "flags", SLEN("flags"), &flags);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_FLAGS_NOTFOUND_IN_DICT,
+               "Failed to get flags");
+        goto out;
+    }
+
+    ret = glusterd_hostname_to_uuid(hostname, uuid);
+    if (ret) {
+        op_errno = GF_DEPROBE_NOT_FRIEND;
+        goto out;
+    }
+
+    if (!gf_uuid_compare(uuid, MY_UUID)) {
+        op_errno = GF_DEPROBE_LOCALHOST;
+        ret = -1;
+        goto out;
+    }
+
+    if (!(flags & GF_CLI_FLAG_OP_FORCE)) {
+        /* Check if peers are connected, except peer being
+         * detached*/
+        if (!glusterd_chk_peers_connected_befriended(uuid)) {
+            ret = -1;
+            op_errno = GF_DEPROBE_FRIEND_DOWN;
+            goto out;
+        }
+    }
+
+    /* Check for if volumes exist with some bricks on the peer being
+     * detached. It's not a problem if a volume contains none or all
+     * of its bricks on the peer being detached
+     */
+    cds_list_for_each_entry_safe(volinfo, tmp, &priv->volumes, vol_list)
+    {
+        ret = glusterd_friend_contains_vol_bricks(volinfo, uuid);
+        if (ret == 1) {
+            op_errno = GF_DEPROBE_BRICK_EXIST;
+            goto out;
+        }
+    }
+
+    cds_list_for_each_entry_safe(snapinfo, tmpsnap, &priv->snapshots, snap_list)
+    {
+        ret = glusterd_friend_contains_snap_bricks(snapinfo, uuid);
+        if (ret == 1) {
+            op_errno = GF_DEPROBE_SNAP_BRICK_EXIST;
+            goto out;
+        }
+    }
+    if (!(flags & GF_CLI_FLAG_OP_FORCE)) {
+        if (glusterd_is_any_volume_in_server_quorum(this) &&
+            !does_gd_meet_server_quorum(this)) {
+            gf_msg(this->name, GF_LOG_CRITICAL, 0, GD_MSG_SERVER_QUORUM_NOT_MET,
+                   "Server quorum not met. Rejecting operation.");
+            ret = -1;
+            op_errno = GF_DEPROBE_QUORUM_NOT_MET;
+            goto out;
+        }
+    }
+
+    if (!gf_uuid_is_null(uuid)) {
+        ret = glusterd_deprobe_begin(req, hostname, port, uuid, dict,
+                                     &op_errno);
+    } else {
+        ret = glusterd_deprobe_begin(req, hostname, port, NULL, dict,
+                                     &op_errno);
+    }
+
+    need_free = _gf_false;
+
 out:
-        free (cli_req.dict.dict_val);
+    free(cli_req.dict.dict_val);
 
-        if (ret) {
-                ret = glusterd_xfer_cli_deprobe_resp (req, ret, op_errno, NULL,
-                                                      hostname, dict);
+    if (ret) {
+        ret = glusterd_xfer_cli_deprobe_resp(req, ret, op_errno, NULL, hostname,
+                                             dict);
+        if (need_free) {
+            dict_unref(dict);
         }
+    }
 
-        glusterd_friend_sm ();
-        glusterd_op_sm ();
+    glusterd_friend_sm();
+    glusterd_op_sm();
 
-        return ret;
+    return ret;
 }
 
 int
-glusterd_handle_cli_deprobe (rpcsvc_request_t *req)
+glusterd_handle_cli_deprobe(rpcsvc_request_t *req)
 {
-        return glusterd_big_locked_handler (req, __glusterd_handle_cli_deprobe);
+    return glusterd_big_locked_handler(req, __glusterd_handle_cli_deprobe);
 }
 
 int
-__glusterd_handle_cli_list_friends (rpcsvc_request_t *req)
+__glusterd_handle_cli_list_friends(rpcsvc_request_t *req)
 {
-        int32_t                         ret = -1;
-        gf1_cli_peer_list_req           cli_req = {0,};
-        dict_t                          *dict = NULL;
-
-        GF_ASSERT (req);
+    int32_t ret = -1;
+    gf1_cli_peer_list_req cli_req = {
+        0,
+    };
+    dict_t *dict = NULL;
+
+    GF_ASSERT(req);
+
+    ret = xdr_to_generic(req->msg[0], &cli_req,
+                         (xdrproc_t)xdr_gf1_cli_peer_list_req);
+    if (ret < 0) {
+        // failed to decode msg;
+        gf_msg("glusterd", GF_LOG_ERROR, 0, GD_MSG_REQ_DECODE_FAIL,
+               "Failed to decode "
+               "request received from cli");
+        req->rpc_err = GARBAGE_ARGS;
+        goto out;
+    }
+
+    gf_msg("glusterd", GF_LOG_INFO, 0, GD_MSG_CLI_REQ_RECVD,
+           "Received cli list req");
+
+    if (cli_req.dict.dict_len) {
+        /* Unserialize the dictionary */
+        dict = dict_new();
 
-        ret = xdr_to_generic (req->msg[0], &cli_req,
-                              (xdrproc_t)xdr_gf1_cli_peer_list_req);
+        ret = dict_unserialize(cli_req.dict.dict_val, cli_req.dict.dict_len,
+                               &dict);
         if (ret < 0) {
-                //failed to decode msg;
-                req->rpc_err = GARBAGE_ARGS;
-                goto out;
-        }
-
-        gf_log ("glusterd", GF_LOG_INFO, "Received cli list req");
-
-        if (cli_req.dict.dict_len) {
-                /* Unserialize the dictionary */
-                dict  = dict_new ();
-
-                ret = dict_unserialize (cli_req.dict.dict_val,
-                                        cli_req.dict.dict_len,
-                                        &dict);
-                if (ret < 0) {
-                        gf_log ("glusterd", GF_LOG_ERROR,
-                                "failed to "
-                                "unserialize req-buffer to dictionary");
-                        goto out;
-                } else {
-                        dict->extra_stdfree = cli_req.dict.dict_val;
-                }
+            gf_msg("glusterd", GF_LOG_ERROR, 0, GD_MSG_DICT_UNSERIALIZE_FAIL,
+                   "failed to "
+                   "unserialize req-buffer to dictionary");
+            goto out;
+        } else {
+            dict->extra_stdfree = cli_req.dict.dict_val;
         }
+    }
 
-        ret = glusterd_list_friends (req, dict, cli_req.flags);
+    ret = glusterd_list_friends(req, dict, cli_req.flags);
 
 out:
-        if (dict)
-                dict_unref (dict);
+    if (dict)
+        dict_unref(dict);
 
-        glusterd_friend_sm ();
-        glusterd_op_sm ();
+    glusterd_friend_sm();
+    glusterd_op_sm();
 
-        return ret;
+    return ret;
 }
 
 int
-glusterd_handle_cli_list_friends (rpcsvc_request_t *req)
+glusterd_handle_cli_list_friends(rpcsvc_request_t *req)
 {
-        return glusterd_big_locked_handler (req,
-                                            __glusterd_handle_cli_list_friends);
+    return glusterd_big_locked_handler(req, __glusterd_handle_cli_list_friends);
 }
 
-int
-__glusterd_handle_cli_get_volume (rpcsvc_request_t *req)
+static int
+__glusterd_handle_cli_get_volume(rpcsvc_request_t *req)
 {
-        int32_t                         ret = -1;
-        gf_cli_req                      cli_req = {{0,}};
-        dict_t                          *dict = NULL;
-        int32_t                         flags = 0;
-
-        GF_ASSERT (req);
+    int32_t ret = -1;
+    gf_cli_req cli_req = {{
+        0,
+    }};
+    int32_t flags = 0;
+    dict_t *dict = NULL;
+    xlator_t *this = NULL;
+
+    GF_ASSERT(req);
+    this = THIS;
+
+    ret = xdr_to_generic(req->msg[0], &cli_req, (xdrproc_t)xdr_gf_cli_req);
+    if (ret < 0) {
+        // failed to decode msg;
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_REQ_DECODE_FAIL,
+               "Failed to decode "
+               "request received from cli");
+        req->rpc_err = GARBAGE_ARGS;
+        goto out;
+    }
+
+    gf_msg(this->name, GF_LOG_DEBUG, 0, GD_MSG_GET_VOL_REQ_RCVD,
+           "Received get vol req");
+
+    if (cli_req.dict.dict_len) {
+        /* Unserialize the dictionary */
+        dict = dict_new();
 
-        ret = xdr_to_generic (req->msg[0], &cli_req, (xdrproc_t)xdr_gf_cli_req);
+        ret = dict_unserialize(cli_req.dict.dict_val, cli_req.dict.dict_len,
+                               &dict);
         if (ret < 0) {
-                //failed to decode msg;
-                req->rpc_err = GARBAGE_ARGS;
-                goto out;
-        }
-
-        gf_log ("glusterd", GF_LOG_INFO, "Received get vol req");
-
-        if (cli_req.dict.dict_len) {
-                /* Unserialize the dictionary */
-                dict  = dict_new ();
-
-                ret = dict_unserialize (cli_req.dict.dict_val,
-                                        cli_req.dict.dict_len,
-                                        &dict);
-                if (ret < 0) {
-                        gf_log ("glusterd", GF_LOG_ERROR,
-                                "failed to "
-                                "unserialize req-buffer to dictionary");
-                        goto out;
-                } else {
-                        dict->extra_stdfree = cli_req.dict.dict_val;
-                }
-        }
-
-        ret = dict_get_int32 (dict, "flags", &flags);
-        if (ret) {
-                gf_log (THIS->name, GF_LOG_ERROR, "failed to get flags");
-                goto out;
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_UNSERIALIZE_FAIL,
+                   "failed to "
+                   "unserialize req-buffer to dictionary");
+            goto out;
+        } else {
+            dict->extra_stdfree = cli_req.dict.dict_val;
         }
+    }
 
-        ret = glusterd_get_volumes (req, dict, flags);
+    ret = dict_get_int32n(dict, "flags", SLEN("flags"), &flags);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_FLAGS_NOTFOUND_IN_DICT,
+               "failed to get flags");
+        goto out;
+    }
+    ret = glusterd_get_volumes(req, dict, flags);
 
 out:
-        if (dict)
-                dict_unref (dict);
+    if (dict)
+        dict_unref(dict);
 
-        glusterd_friend_sm ();
-        glusterd_op_sm ();
+    glusterd_friend_sm();
+    glusterd_op_sm();
 
-        return ret;
+    return ret;
 }
 
 int
-glusterd_handle_cli_get_volume (rpcsvc_request_t *req)
+glusterd_handle_cli_get_volume(rpcsvc_request_t *req)
 {
-        return glusterd_big_locked_handler (req,
-                                            __glusterd_handle_cli_get_volume);
+    return glusterd_big_locked_handler(req, __glusterd_handle_cli_get_volume);
 }
 
 int
-__glusterd_handle_cli_uuid_reset (rpcsvc_request_t *req)
+__glusterd_handle_cli_uuid_reset(rpcsvc_request_t *req)
 {
-        int                     ret     = -1;
-        dict_t                  *dict   = NULL;
-        xlator_t                *this   = NULL;
-        glusterd_conf_t         *priv   = NULL;
-        uuid_t                  uuid    = {0};
-        gf_cli_rsp              rsp     = {0,};
-        gf_cli_req              cli_req = {{0,}};
-        char                    msg_str[2048] = {0,};
-
-        GF_ASSERT (req);
-
-        this = THIS;
-        priv = this->private;
-        GF_ASSERT (priv);
-
-        ret = xdr_to_generic (req->msg[0], &cli_req, (xdrproc_t)xdr_gf_cli_req);
-        if (ret < 0) {
-                //failed to decode msg;
-                req->rpc_err = GARBAGE_ARGS;
-                goto out;
-        }
-
-        gf_log ("glusterd", GF_LOG_DEBUG, "Received uuid reset req");
-
-        if (cli_req.dict.dict_len) {
-                /* Unserialize the dictionary */
-                dict  = dict_new ();
-
-                ret = dict_unserialize (cli_req.dict.dict_val,
-                                        cli_req.dict.dict_len,
-                                        &dict);
-                if (ret < 0) {
-                        gf_log ("glusterd", GF_LOG_ERROR,
-                                "failed to "
-                                "unserialize req-buffer to dictionary");
-                        snprintf (msg_str, sizeof (msg_str), "Unable to decode "
-                                  "the buffer");
-                        goto out;
-                } else {
-                        dict->extra_stdfree = cli_req.dict.dict_val;
-                }
-        }
+    int ret = -1;
+    dict_t *dict = NULL;
+    xlator_t *this = NULL;
+    glusterd_conf_t *priv = NULL;
+    uuid_t uuid = {0};
+    gf_cli_rsp rsp = {
+        0,
+    };
+    gf_cli_req cli_req = {{
+        0,
+    }};
+    char msg_str[128] = {
+        0,
+    };
+
+    GF_ASSERT(req);
+
+    this = THIS;
+    priv = this->private;
+    GF_ASSERT(priv);
+
+    ret = xdr_to_generic(req->msg[0], &cli_req, (xdrproc_t)xdr_gf_cli_req);
+    if (ret < 0) {
+        // failed to decode msg;
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_REQ_DECODE_FAIL,
+               "Failed to decode "
+               "request received from cli");
+        req->rpc_err = GARBAGE_ARGS;
+        goto out;
+    }
+
+    gf_msg_debug("glusterd", 0, "Received uuid reset req");
+
+    if (cli_req.dict.dict_len) {
+        /* Unserialize the dictionary */
+        dict = dict_new();
 
-        /* In the above section if dict_unserialize is successful, ret is set
-         * to zero.
-         */
+        ret = dict_unserialize(cli_req.dict.dict_val, cli_req.dict.dict_len,
+                               &dict);
+        if (ret < 0) {
+            gf_msg("glusterd", GF_LOG_ERROR, 0, GD_MSG_DICT_UNSERIALIZE_FAIL,
+                   "failed to "
+                   "unserialize req-buffer to dictionary");
+            snprintf(msg_str, sizeof(msg_str),
+                     "Unable to decode "
+                     "the buffer");
+            goto out;
+        } else {
+            dict->extra_stdfree = cli_req.dict.dict_val;
+        }
+    }
+
+    /* In the above section if dict_unserialize is successful, ret is set
+     * to zero.
+     */
+    ret = -1;
+    // Do not allow peer reset if there are any volumes in the cluster
+    if (!cds_list_empty(&priv->volumes)) {
+        snprintf(msg_str, sizeof(msg_str),
+                 "volumes are already "
+                 "present in the cluster. Resetting uuid is not "
+                 "allowed");
+        gf_msg(this->name, GF_LOG_WARNING, 0, GD_MSG_VOLS_ALREADY_PRESENT, "%s",
+               msg_str);
+        goto out;
+    }
+
+    // Do not allow peer reset if trusted storage pool is already formed
+    if (!cds_list_empty(&priv->peers)) {
+        snprintf(msg_str, sizeof(msg_str),
+                 "trusted storage pool "
+                 "has been already formed. Please detach this peer "
+                 "from the pool and reset its uuid.");
+        gf_msg(this->name, GF_LOG_WARNING, 0, GD_MSG_TSP_ALREADY_FORMED, "%s",
+               msg_str);
+        goto out;
+    }
+
+    gf_uuid_copy(uuid, priv->uuid);
+    ret = glusterd_uuid_generate_save();
+
+    if (!gf_uuid_compare(uuid, MY_UUID)) {
+        snprintf(msg_str, sizeof(msg_str),
+                 "old uuid and the new uuid"
+                 " are same. Try gluster peer reset again");
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_UUIDS_SAME_RETRY, "%s",
+               msg_str);
         ret = -1;
-        // Do not allow peer reset if there are any volumes in the cluster
-        if (!cds_list_empty (&priv->volumes)) {
-                snprintf (msg_str, sizeof (msg_str), "volumes are already "
-                          "present in the cluster. Resetting uuid is not "
-                          "allowed");
-                gf_log (this->name, GF_LOG_WARNING, "%s", msg_str);
-                goto out;
-        }
-
-        // Do not allow peer reset if trusted storage pool is already formed
-        if (!cds_list_empty (&priv->peers)) {
-                snprintf (msg_str, sizeof (msg_str),"trusted storage pool "
-                          "has been already formed. Please detach this peer "
-                          "from the pool and reset its uuid.");
-                gf_log (this->name, GF_LOG_WARNING, "%s", msg_str);
-                goto out;
-        }
-
-        gf_uuid_copy (uuid, priv->uuid);
-        ret = glusterd_uuid_generate_save ();
-
-        if (!gf_uuid_compare (uuid, MY_UUID)) {
-                snprintf (msg_str, sizeof (msg_str), "old uuid and the new uuid"
-                          " are same. Try gluster peer reset again");
-                gf_log (this->name, GF_LOG_ERROR, "%s", msg_str);
-                ret = -1;
-                goto out;
-        }
+        goto out;
+    }
 
 out:
-        if (ret) {
-                rsp.op_ret = -1;
-                if (msg_str[0] == '\0')
-                        snprintf (msg_str, sizeof (msg_str), "Operation "
-                                  "failed");
-                rsp.op_errstr = msg_str;
-                ret = 0;
-        } else {
-                rsp.op_errstr = "";
-        }
+    if (ret) {
+        rsp.op_ret = -1;
+        if (msg_str[0] == '\0')
+            snprintf(msg_str, sizeof(msg_str),
+                     "Operation "
+                     "failed");
+        rsp.op_errstr = msg_str;
+        ret = 0;
+    } else {
+        rsp.op_errstr = "";
+    }
 
-        glusterd_to_cli (req, &rsp, NULL, 0, NULL,
-                         (xdrproc_t)xdr_gf_cli_rsp, dict);
+    glusterd_to_cli(req, &rsp, NULL, 0, NULL, (xdrproc_t)xdr_gf_cli_rsp, dict);
 
-        return ret;
+    return ret;
 }
 
 int
-glusterd_handle_cli_uuid_reset (rpcsvc_request_t *req)
+glusterd_handle_cli_uuid_reset(rpcsvc_request_t *req)
 {
-        return glusterd_big_locked_handler (req,
-                                            __glusterd_handle_cli_uuid_reset);
+    return glusterd_big_locked_handler(req, __glusterd_handle_cli_uuid_reset);
 }
 
 int
-__glusterd_handle_cli_uuid_get (rpcsvc_request_t *req)
+__glusterd_handle_cli_uuid_get(rpcsvc_request_t *req)
 {
-        int                     ret         = -1;
-        dict_t                  *dict       = NULL;
-        dict_t                  *rsp_dict   = NULL;
-        xlator_t                *this       = NULL;
-        glusterd_conf_t         *priv       = NULL;
-        gf_cli_rsp              rsp         = {0,};
-        gf_cli_req              cli_req     = {{0,}};
-        char                    msg_str[2048] = {0,};
-        char                    uuid_str[64] = {0,};
-
-        GF_ASSERT (req);
-
-        this = THIS;
-        priv = this->private;
-        GF_ASSERT (priv);
-
-        ret = xdr_to_generic (req->msg[0], &cli_req, (xdrproc_t)xdr_gf_cli_req);
-        if (ret < 0) {
-                req->rpc_err = GARBAGE_ARGS;
-                goto out;
-        }
-
-        gf_log ("glusterd", GF_LOG_DEBUG, "Received uuid get req");
-
-        if (cli_req.dict.dict_len) {
-                dict  = dict_new ();
-                if (!dict) {
-                        ret = -1;
-                        goto out;
-                }
-
-                ret = dict_unserialize (cli_req.dict.dict_val,
-                                        cli_req.dict.dict_len,
-                                        &dict);
-                if (ret < 0) {
-                        gf_log ("glusterd", GF_LOG_ERROR,
-                                "failed to "
-                                "unserialize req-buffer to dictionary");
-                        snprintf (msg_str, sizeof (msg_str), "Unable to decode "
-                                  "the buffer");
-                        goto out;
-
-                } else {
-                        dict->extra_stdfree = cli_req.dict.dict_val;
-
-                }
+    int ret = -1;
+    dict_t *dict = NULL;
+    dict_t *rsp_dict = NULL;
+    xlator_t *this = NULL;
+    glusterd_conf_t *priv = NULL;
+    gf_cli_rsp rsp = {
+        0,
+    };
+    gf_cli_req cli_req = {{
+        0,
+    }};
+    char err_str[64] = {
+        0,
+    };
+    char uuid_str[64] = {
+        0,
+    };
+
+    GF_ASSERT(req);
+
+    this = THIS;
+    priv = this->private;
+    GF_ASSERT(priv);
+
+    ret = xdr_to_generic(req->msg[0], &cli_req, (xdrproc_t)xdr_gf_cli_req);
+    if (ret < 0) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_REQ_DECODE_FAIL,
+               "Failed to decode "
+               "request received from cli");
+        req->rpc_err = GARBAGE_ARGS;
+        goto out;
+    }
+
+    gf_msg_debug("glusterd", 0, "Received uuid get req");
+
+    if (cli_req.dict.dict_len) {
+        dict = dict_new();
+        if (!dict) {
+            gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_CREATE_FAIL,
+                    NULL);
+            ret = -1;
+            goto out;
         }
 
-        rsp_dict = dict_new ();
-        if (!rsp_dict) {
-                ret = -1;
-                goto out;
-        }
+        ret = dict_unserialize(cli_req.dict.dict_val, cli_req.dict.dict_len,
+                               &dict);
+        if (ret < 0) {
+            gf_msg("glusterd", GF_LOG_ERROR, 0, GD_MSG_DICT_UNSERIALIZE_FAIL,
+                   "failed to "
+                   "unserialize req-buffer to dictionary");
+            snprintf(err_str, sizeof(err_str),
+                     "Unable to decode "
+                     "the buffer");
+            goto out;
 
-        uuid_utoa_r (MY_UUID, uuid_str);
-        ret = dict_set_str (rsp_dict, "uuid", uuid_str);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "Failed to set uuid in "
-                        "dictionary.");
-                goto out;
+        } else {
+            dict->extra_stdfree = cli_req.dict.dict_val;
         }
+    }
 
-        ret = dict_allocate_and_serialize (rsp_dict, &rsp.dict.dict_val,
-                                           &rsp.dict.dict_len);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "Failed to serialize "
-                        "dictionary.");
-                goto out;
-        }
-        ret = 0;
+    rsp_dict = dict_new();
+    if (!rsp_dict) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_CREATE_FAIL, NULL);
+        ret = -1;
+        goto out;
+    }
+
+    uuid_utoa_r(MY_UUID, uuid_str);
+    ret = dict_set_strn(rsp_dict, "uuid", SLEN("uuid"), uuid_str);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+               "Failed to set uuid in "
+               "dictionary.");
+        goto out;
+    }
+
+    ret = dict_allocate_and_serialize(rsp_dict, &rsp.dict.dict_val,
+                                      &rsp.dict.dict_len);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno,
+                GD_MSG_DICT_ALLOC_AND_SERL_LENGTH_GET_FAIL, NULL);
+        goto out;
+    }
+    ret = 0;
 out:
-        if (ret) {
-                rsp.op_ret = -1;
-                if (msg_str[0] == '\0')
-                        snprintf (msg_str, sizeof (msg_str), "Operation "
-                                  "failed");
-                rsp.op_errstr = msg_str;
-
-        } else {
-                rsp.op_errstr = "";
+    if (ret) {
+        rsp.op_ret = -1;
+        if (err_str[0] == '\0')
+            snprintf(err_str, sizeof(err_str),
+                     "Operation "
+                     "failed");
+        rsp.op_errstr = err_str;
+
+    } else {
+        rsp.op_errstr = "";
+    }
 
-        }
+    glusterd_to_cli(req, &rsp, NULL, 0, NULL, (xdrproc_t)xdr_gf_cli_rsp, dict);
 
-        glusterd_to_cli (req, &rsp, NULL, 0, NULL,
-                         (xdrproc_t)xdr_gf_cli_rsp, dict);
+    if (rsp_dict)
+        dict_unref(rsp_dict);
+    GF_FREE(rsp.dict.dict_val);
 
-        return 0;
+    return 0;
 }
 int
-glusterd_handle_cli_uuid_get (rpcsvc_request_t *req)
+glusterd_handle_cli_uuid_get(rpcsvc_request_t *req)
 {
-        return glusterd_big_locked_handler (req,
-                                            __glusterd_handle_cli_uuid_get);
+    return glusterd_big_locked_handler(req, __glusterd_handle_cli_uuid_get);
 }
 
 int
-__glusterd_handle_cli_list_volume (rpcsvc_request_t *req)
+__glusterd_handle_cli_list_volume(rpcsvc_request_t *req)
 {
-        int                     ret = -1;
-        dict_t                  *dict = NULL;
-        glusterd_conf_t         *priv = NULL;
-        glusterd_volinfo_t      *volinfo = NULL;
-        int                     count = 0;
-        char                    key[1024] = {0,};
-        gf_cli_rsp              rsp = {0,};
-
-        GF_ASSERT (req);
-
-        priv = THIS->private;
-        GF_ASSERT (priv);
-
-        dict = dict_new ();
-        if (!dict)
-                goto out;
-
-        cds_list_for_each_entry (volinfo, &priv->volumes, vol_list) {
-                memset (key, 0, sizeof (key));
-                snprintf (key, sizeof (key), "volume%d", count);
-                ret = dict_set_str (dict, key, volinfo->volname);
-                if (ret)
-                        goto out;
-                count++;
-        }
-
-        ret = dict_set_int32 (dict, "count", count);
+    int ret = -1;
+    dict_t *dict = NULL;
+    glusterd_conf_t *priv = NULL;
+    glusterd_volinfo_t *volinfo = NULL;
+    int count = 0;
+    char key[64] = {
+        0,
+    };
+    int keylen;
+    gf_cli_rsp rsp = {
+        0,
+    };
+
+    GF_ASSERT(req);
+
+    priv = THIS->private;
+    GF_ASSERT(priv);
+
+    dict = dict_new();
+    if (!dict) {
+        gf_smsg("glusterd", GF_LOG_ERROR, errno, GD_MSG_DICT_CREATE_FAIL, NULL);
+        goto out;
+    }
+
+    cds_list_for_each_entry(volinfo, &priv->volumes, vol_list)
+    {
+        keylen = snprintf(key, sizeof(key), "volume%d", count);
+        ret = dict_set_strn(dict, key, keylen, volinfo->volname);
         if (ret)
-                goto out;
+            goto out;
+        count++;
+    }
 
-        ret = dict_allocate_and_serialize (dict, &rsp.dict.dict_val,
-                                           &rsp.dict.dict_len);
-        if (ret)
-                goto out;
+    ret = dict_set_int32n(dict, "count", SLEN("count"), count);
+    if (ret) {
+        gf_smsg("glusterd", GF_LOG_ERROR, errno, GD_MSG_DICT_SET_FAILED,
+                "Key=count", NULL);
+        goto out;
+    }
 
-        ret = 0;
+    ret = dict_allocate_and_serialize(dict, &rsp.dict.dict_val,
+                                      &rsp.dict.dict_len);
+    if (ret)
+        goto out;
+
+    ret = 0;
 
 out:
-        rsp.op_ret = ret;
-        if (ret)
-                rsp.op_errstr = "Error listing volumes";
-        else
-                rsp.op_errstr = "";
+    rsp.op_ret = ret;
+    if (ret)
+        rsp.op_errstr = "Error listing volumes";
+    else
+        rsp.op_errstr = "";
 
-        glusterd_submit_reply (req, &rsp, NULL, 0, NULL,
-                                     (xdrproc_t)xdr_gf_cli_rsp);
-        ret = 0;
+    glusterd_submit_reply(req, &rsp, NULL, 0, NULL, (xdrproc_t)xdr_gf_cli_rsp);
+    ret = 0;
 
-        if (dict)
-                dict_unref (dict);
+    if (dict)
+        dict_unref(dict);
 
-        glusterd_friend_sm ();
-        glusterd_op_sm ();
+    GF_FREE(rsp.dict.dict_val);
 
-        return ret;
+    glusterd_friend_sm();
+    glusterd_op_sm();
+
+    return ret;
 }
 
 int
-glusterd_handle_cli_list_volume (rpcsvc_request_t *req)
+glusterd_handle_cli_list_volume(rpcsvc_request_t *req)
 {
-        return glusterd_big_locked_handler (req,
-                                            __glusterd_handle_cli_list_volume);
+    return glusterd_big_locked_handler(req, __glusterd_handle_cli_list_volume);
 }
 
 int32_t
-glusterd_op_begin (rpcsvc_request_t *req, glusterd_op_t op, void *ctx,
-                   char *err_str, size_t err_len)
+glusterd_op_begin(rpcsvc_request_t *req, glusterd_op_t op, void *ctx,
+                  char *err_str, size_t err_len)
 {
-        int             ret = -1;
+    int ret = -1;
 
-        ret = glusterd_op_txn_begin (req, op, ctx, err_str, err_len);
+    ret = glusterd_op_txn_begin(req, op, ctx, err_str, err_len);
 
-        return ret;
+    return ret;
 }
 
 int
-__glusterd_handle_ganesha_cmd (rpcsvc_request_t *req)
+__glusterd_handle_ganesha_cmd(rpcsvc_request_t *req)
 {
-        int32_t                         ret = -1;
-        gf_cli_req                      cli_req = { {0,} } ;
-        dict_t                          *dict = NULL;
-        glusterd_op_t                   cli_op = GD_OP_GANESHA;
-        char                            *volname = NULL;
-        char                            *op_errstr = NULL;
-        gf_boolean_t                    help = _gf_false;
-        char                            err_str[2048] = {0,};
-        xlator_t                        *this = NULL;
-
-        this = THIS;
-        GF_ASSERT (this);
-
-        GF_ASSERT (req);
-
-        ret = xdr_to_generic (req->msg[0], &cli_req, (xdrproc_t)xdr_gf_cli_req);
-        if (ret < 0) {
-                snprintf (err_str, sizeof (err_str), "Failed to decode "
-                          "request received from cli");
-                gf_log (this->name, GF_LOG_ERROR, "%s", err_str);
-                req->rpc_err = GARBAGE_ARGS;
-                goto out;
+    int32_t ret = -1;
+    gf_cli_req cli_req = {{
+        0,
+    }};
+    dict_t *dict = NULL;
+    glusterd_op_t cli_op = GD_OP_GANESHA;
+    char *op_errstr = NULL;
+    char err_str[2048] = {
+        0,
+    };
+    xlator_t *this = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+
+    GF_ASSERT(req);
+
+    ret = xdr_to_generic(req->msg[0], &cli_req, (xdrproc_t)xdr_gf_cli_req);
+    if (ret < 0) {
+        snprintf(err_str, sizeof(err_str),
+                 "Failed to decode "
+                 "request received from cli");
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_REQ_DECODE_FAIL, "%s",
+               err_str);
+        req->rpc_err = GARBAGE_ARGS;
+        goto out;
+    }
+
+    if (cli_req.dict.dict_len) {
+        /* Unserialize the dictionary */
+        dict = dict_new();
+        if (!dict) {
+            gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_CREATE_FAIL,
+                    NULL);
+            ret = -1;
+            goto out;
         }
 
-        if (cli_req.dict.dict_len) {
-                /* Unserialize the dictionary */
-                dict  = dict_new ();
-                if (!dict) {
-                        ret = -1;
-                        goto out;
-                }
-
-                ret = dict_unserialize (cli_req.dict.dict_val,
-                                        cli_req.dict.dict_len,
-                                        &dict);
-                if (ret < 0) {
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "failed to "
-                                "unserialize req-buffer to dictionary");
-                        snprintf (err_str, sizeof (err_str), "Unable to decode "
-                                  "the command");
-                        goto out;
-                } else {
-                        dict->extra_stdfree = cli_req.dict.dict_val;
-                }
+        ret = dict_unserialize(cli_req.dict.dict_val, cli_req.dict.dict_len,
+                               &dict);
+        if (ret < 0) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_UNSERIALIZE_FAIL,
+                   "failed to "
+                   "unserialize req-buffer to dictionary");
+            snprintf(err_str, sizeof(err_str),
+                     "Unable to decode "
+                     "the command");
+            goto out;
+        } else {
+            dict->extra_stdfree = cli_req.dict.dict_val;
         }
+    }
 
-        gf_log (this->name, GF_LOG_TRACE, "Received global option request");
+    gf_msg_trace(this->name, 0, "Received global option request");
 
-        ret = glusterd_op_begin_synctask (req, GD_OP_GANESHA, dict);
+    ret = glusterd_op_begin_synctask(req, GD_OP_GANESHA, dict);
 out:
-        if (ret) {
-                if (err_str[0] == '\0')
-                        snprintf (err_str, sizeof (err_str),
-                                  "Operation failed");
-                ret = glusterd_op_send_cli_response (cli_op, ret, 0, req,
-                                                     dict, err_str);
-        }
-        if (op_errstr)
-                GF_FREE (op_errstr);
-        if (dict)
-                dict_unref(dict);
-
-        return ret;
+    if (ret) {
+        if (err_str[0] == '\0')
+            snprintf(err_str, sizeof(err_str), "Operation failed");
+        ret = glusterd_op_send_cli_response(cli_op, ret, 0, req, dict, err_str);
+    }
+    if (op_errstr)
+        GF_FREE(op_errstr);
+    if (dict)
+        dict_unref(dict);
+
+    return ret;
 }
 
-
 int
-glusterd_handle_ganesha_cmd (rpcsvc_request_t *req)
+glusterd_handle_ganesha_cmd(rpcsvc_request_t *req)
 {
-        return glusterd_big_locked_handler (req, __glusterd_handle_ganesha_cmd);
+    return glusterd_big_locked_handler(req, __glusterd_handle_ganesha_cmd);
 }
 
-int
-__glusterd_handle_reset_volume (rpcsvc_request_t *req)
+static int
+__glusterd_handle_reset_volume(rpcsvc_request_t *req)
 {
-        int32_t                         ret = -1;
-        gf_cli_req                      cli_req = {{0,}};
-        dict_t                          *dict = NULL;
-        glusterd_op_t                   cli_op = GD_OP_RESET_VOLUME;
-        char                            *volname = NULL;
-        char                            err_str[2048] = {0,};
-        xlator_t                        *this = NULL;
-
-        GF_ASSERT (req);
-        this = THIS;
-        GF_ASSERT (this);
-
-        ret = xdr_to_generic (req->msg[0], &cli_req, (xdrproc_t)xdr_gf_cli_req);
-        if (ret < 0) {
-                snprintf (err_str, sizeof (err_str), "Failed to decode request "
-                          "received from cli");
-                gf_log (this->name, GF_LOG_ERROR, "%s", err_str);
-                req->rpc_err = GARBAGE_ARGS;
-                goto out;
-        }
-
-        if (cli_req.dict.dict_len) {
-                /* Unserialize the dictionary */
-                dict  = dict_new ();
-
-                ret = dict_unserialize (cli_req.dict.dict_val,
-                                        cli_req.dict.dict_len,
-                                        &dict);
-                if (ret < 0) {
-                        gf_log (this->name, GF_LOG_ERROR, "failed to "
-                                    "unserialize req-buffer to dictionary");
-                        snprintf (err_str, sizeof (err_str), "Unable to decode "
-                                  "the command");
-                        goto out;
-                } else {
-                        dict->extra_stdfree = cli_req.dict.dict_val;
-                }
-        }
-
-        ret = dict_get_str (dict, "volname", &volname);
-        if (ret) {
-                snprintf (err_str, sizeof (err_str), "Failed to get volume "
-                          "name");
-                gf_log (this->name, GF_LOG_ERROR, "%s", err_str);
-                goto out;
-        }
-        gf_log (this->name, GF_LOG_DEBUG, "Received volume reset request for "
-                "volume %s", volname);
+    int32_t ret = -1;
+    gf_cli_req cli_req = {{
+        0,
+    }};
+    dict_t *dict = NULL;
+    glusterd_op_t cli_op = GD_OP_RESET_VOLUME;
+    char *volname = NULL;
+    char err_str[64] = {
+        0,
+    };
+    xlator_t *this = NULL;
+
+    GF_ASSERT(req);
+    this = THIS;
+    GF_ASSERT(this);
+
+    gf_msg(this->name, GF_LOG_INFO, 0, 0, "Received reset vol req");
+
+    ret = xdr_to_generic(req->msg[0], &cli_req, (xdrproc_t)xdr_gf_cli_req);
+    if (ret < 0) {
+        snprintf(err_str, sizeof(err_str),
+                 "Failed to decode request "
+                 "received from cli");
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_REQ_DECODE_FAIL, "%s",
+               err_str);
+        req->rpc_err = GARBAGE_ARGS;
+        goto out;
+    }
+
+    if (cli_req.dict.dict_len) {
+        /* Unserialize the dictionary */
+        dict = dict_new();
 
-        ret = glusterd_op_begin_synctask (req, GD_OP_RESET_VOLUME, dict);
+        ret = dict_unserialize(cli_req.dict.dict_val, cli_req.dict.dict_len,
+                               &dict);
+        if (ret < 0) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_UNSERIALIZE_FAIL,
+                   "failed to "
+                   "unserialize req-buffer to dictionary");
+            snprintf(err_str, sizeof(err_str),
+                     "Unable to decode "
+                     "the command");
+            goto out;
+        } else {
+            dict->extra_stdfree = cli_req.dict.dict_val;
+        }
+    }
+
+    ret = dict_get_strn(dict, "volname", SLEN("volname"), &volname);
+    if (ret) {
+        snprintf(err_str, sizeof(err_str),
+                 "Failed to get volume "
+                 "name");
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_VOLNAME_NOTFOUND_IN_DICT,
+               "%s", err_str);
+        goto out;
+    }
+    gf_msg_debug(this->name, 0,
+                 "Received volume reset request for "
+                 "volume %s",
+                 volname);
+
+    ret = glusterd_op_begin_synctask(req, GD_OP_RESET_VOLUME, dict);
 
 out:
-        if (ret) {
-                if (err_str[0] == '\0')
-                        snprintf (err_str, sizeof (err_str),
-                                  "Operation failed");
-                ret = glusterd_op_send_cli_response (cli_op, ret, 0, req,
-                                                     dict, err_str);
-        }
+    if (ret) {
+        if (err_str[0] == '\0')
+            snprintf(err_str, sizeof(err_str), "Operation failed");
+        ret = glusterd_op_send_cli_response(cli_op, ret, 0, req, dict, err_str);
+    }
 
-        return ret;
+    return ret;
 }
 
 int
-glusterd_handle_reset_volume (rpcsvc_request_t *req)
+glusterd_handle_reset_volume(rpcsvc_request_t *req)
 {
-        return glusterd_big_locked_handler (req,
-                                            __glusterd_handle_reset_volume);
+    return glusterd_big_locked_handler(req, __glusterd_handle_reset_volume);
 }
 
 int
-__glusterd_handle_set_volume (rpcsvc_request_t *req)
+__glusterd_handle_set_volume(rpcsvc_request_t *req)
 {
-        int32_t                         ret = -1;
-        gf_cli_req                      cli_req = {{0,}};
-        dict_t                          *dict = NULL;
-        glusterd_op_t                   cli_op = GD_OP_SET_VOLUME;
-        char                            *key = NULL;
-        char                            *value = NULL;
-        char                            *volname = NULL;
-        char                            *op_errstr = NULL;
-        gf_boolean_t                    help = _gf_false;
-        char                            err_str[2048] = {0,};
-        xlator_t                        *this = NULL;
-
-        this = THIS;
-        GF_ASSERT (this);
-
-        GF_ASSERT (req);
-
-        ret = xdr_to_generic (req->msg[0], &cli_req, (xdrproc_t)xdr_gf_cli_req);
-        if (ret < 0) {
-                snprintf (err_str, sizeof (err_str), "Failed to decode "
-                          "request received from cli");
-                gf_log (this->name, GF_LOG_ERROR, "%s", err_str);
-                req->rpc_err = GARBAGE_ARGS;
-                goto out;
-        }
-
-        if (cli_req.dict.dict_len) {
-                /* Unserialize the dictionary */
-                dict  = dict_new ();
-
-                ret = dict_unserialize (cli_req.dict.dict_val,
-                                        cli_req.dict.dict_len,
-                                        &dict);
-                if (ret < 0) {
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "failed to "
-                                "unserialize req-buffer to dictionary");
-                        snprintf (err_str, sizeof (err_str), "Unable to decode "
-                                  "the command");
-                        goto out;
-                } else {
-                        dict->extra_stdfree = cli_req.dict.dict_val;
-                }
-        }
-
-        ret = dict_get_str (dict, "volname", &volname);
-        if (ret) {
-                snprintf (err_str, sizeof (err_str), "Failed to get volume "
-                          "name while handling volume set command");
-                gf_log (this->name, GF_LOG_ERROR, "%s", err_str);
-                goto out;
-        }
-
-        if (strcmp (volname, "help") == 0 ||
-            strcmp (volname, "help-xml") == 0) {
-                ret = glusterd_volset_help (dict, &op_errstr);
-                help = _gf_true;
-                goto out;
-        }
-
-        ret = dict_get_str (dict, "key1", &key);
-        if (ret) {
-                snprintf (err_str, sizeof (err_str), "Failed to get key while"
-                          " handling volume set for %s", volname);
-                gf_log (this->name, GF_LOG_ERROR, "%s", err_str);
-                goto out;
-        }
-
-        ret = dict_get_str (dict, "value1", &value);
-        if (ret) {
-                snprintf (err_str, sizeof (err_str), "Failed to get value while"
-                          " handling volume set for %s", volname);
-                gf_log (this->name, GF_LOG_ERROR, "%s", err_str);
-                goto out;
-        }
-        gf_log (this->name, GF_LOG_DEBUG, "Received volume set request for "
-                "volume %s", volname);
+    int32_t ret = -1;
+    gf_cli_req cli_req = {{
+        0,
+    }};
+    dict_t *dict = NULL;
+    glusterd_op_t cli_op = GD_OP_SET_VOLUME;
+    char *key = NULL;
+    char *value = NULL;
+    char *volname = NULL;
+    char *op_errstr = NULL;
+    gf_boolean_t help = _gf_false;
+    char err_str[2048] = {
+        0,
+    };
+    xlator_t *this = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+
+    GF_ASSERT(req);
+
+    ret = xdr_to_generic(req->msg[0], &cli_req, (xdrproc_t)xdr_gf_cli_req);
+    if (ret < 0) {
+        snprintf(err_str, sizeof(err_str),
+                 "Failed to decode "
+                 "request received from cli");
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_REQ_DECODE_FAIL, "%s",
+               err_str);
+        req->rpc_err = GARBAGE_ARGS;
+        goto out;
+    }
+
+    if (cli_req.dict.dict_len) {
+        /* Unserialize the dictionary */
+        dict = dict_new();
 
-        ret = glusterd_op_begin_synctask (req, GD_OP_SET_VOLUME, dict);
+        ret = dict_unserialize(cli_req.dict.dict_val, cli_req.dict.dict_len,
+                               &dict);
+        if (ret < 0) {
+            gf_msg(this->name, GF_LOG_ERROR, errno,
+                   GD_MSG_DICT_UNSERIALIZE_FAIL,
+                   "failed to "
+                   "unserialize req-buffer to dictionary");
+            snprintf(err_str, sizeof(err_str),
+                     "Unable to decode "
+                     "the command");
+            goto out;
+        } else {
+            dict->extra_stdfree = cli_req.dict.dict_val;
+        }
+    }
+
+    ret = dict_get_strn(dict, "volname", SLEN("volname"), &volname);
+    if (ret) {
+        snprintf(err_str, sizeof(err_str),
+                 "Failed to get volume "
+                 "name while handling volume set command");
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED, "%s",
+               err_str);
+        goto out;
+    }
+
+    if (strcmp(volname, "help") == 0 || strcmp(volname, "help-xml") == 0) {
+        ret = glusterd_volset_help(dict, &op_errstr);
+        help = _gf_true;
+        goto out;
+    }
+
+    ret = dict_get_strn(dict, "key1", SLEN("key1"), &key);
+    if (ret) {
+        snprintf(err_str, sizeof(err_str),
+                 "Failed to get key while"
+                 " handling volume set for %s",
+                 volname);
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED, "%s",
+               err_str);
+        goto out;
+    }
+
+    ret = dict_get_strn(dict, "value1", SLEN("value1"), &value);
+    if (ret) {
+        snprintf(err_str, sizeof(err_str),
+                 "Failed to get value while"
+                 " handling volume set for %s",
+                 volname);
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED, "%s",
+               err_str);
+        goto out;
+    }
+    gf_msg_debug(this->name, 0,
+                 "Received volume set request for "
+                 "volume %s",
+                 volname);
+
+    ret = glusterd_op_begin_synctask(req, GD_OP_SET_VOLUME, dict);
 
 out:
-        if (help)
-                ret = glusterd_op_send_cli_response (cli_op, ret, 0, req, dict,
-                                                     (op_errstr)? op_errstr:"");
-        else if (ret) {
-                if (err_str[0] == '\0')
-                        snprintf (err_str, sizeof (err_str),
-                                  "Operation failed");
-                ret = glusterd_op_send_cli_response (cli_op, ret, 0, req,
-                                                     dict, err_str);
-        }
-        if (op_errstr)
-                GF_FREE (op_errstr);
-
-        return ret;
+    if (help)
+        ret = glusterd_op_send_cli_response(cli_op, ret, 0, req, dict,
+                                            (op_errstr) ? op_errstr : "");
+    else if (ret) {
+        if (err_str[0] == '\0')
+            snprintf(err_str, sizeof(err_str), "Operation failed");
+        ret = glusterd_op_send_cli_response(cli_op, ret, 0, req, dict, err_str);
+    }
+    if (op_errstr)
+        GF_FREE(op_errstr);
+
+    return ret;
 }
 
 int
-glusterd_handle_set_volume (rpcsvc_request_t *req)
+glusterd_handle_set_volume(rpcsvc_request_t *req)
 {
-        return glusterd_big_locked_handler (req, __glusterd_handle_set_volume);
+    return glusterd_big_locked_handler(req, __glusterd_handle_set_volume);
 }
 
 int
-__glusterd_handle_sync_volume (rpcsvc_request_t *req)
+__glusterd_handle_sync_volume(rpcsvc_request_t *req)
 {
-        int32_t                          ret     = -1;
-        gf_cli_req                       cli_req = {{0,}};
-        dict_t                           *dict = NULL;
-        gf_cli_rsp                       cli_rsp = {0.};
-        char                             msg[2048] = {0,};
-        char                             *volname = NULL;
-        gf1_cli_sync_volume              flags = 0;
-        char                             *hostname = NULL;
-        xlator_t                         *this = NULL;
-
-        GF_ASSERT (req);
-        this = THIS;
-        GF_ASSERT (this);
-
-        ret = xdr_to_generic (req->msg[0], &cli_req, (xdrproc_t)xdr_gf_cli_req);
-        if (ret < 0) {
-                //failed to decode msg;
-                req->rpc_err = GARBAGE_ARGS;
-                goto out;
-        }
+    int32_t ret = -1;
+    gf_cli_req cli_req = {{
+        0,
+    }};
+    dict_t *dict = NULL;
+    gf_cli_rsp cli_rsp = {0.};
+    char msg[2048] = {
+        0,
+    };
+    char *volname = NULL;
+    gf1_cli_sync_volume flags = 0;
+    char *hostname = NULL;
+    xlator_t *this = NULL;
+
+    GF_ASSERT(req);
+    this = THIS;
+    GF_ASSERT(this);
+
+    ret = xdr_to_generic(req->msg[0], &cli_req, (xdrproc_t)xdr_gf_cli_req);
+    if (ret < 0) {
+        // failed to decode msg;
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_REQ_DECODE_FAIL, "%s",
+               "Failed to decode "
+               "request received from cli");
+        req->rpc_err = GARBAGE_ARGS;
+        goto out;
+    }
+
+    if (cli_req.dict.dict_len) {
+        /* Unserialize the dictionary */
+        dict = dict_new();
 
-        if (cli_req.dict.dict_len) {
-                /* Unserialize the dictionary */
-                dict  = dict_new ();
-
-                ret = dict_unserialize (cli_req.dict.dict_val,
-                                        cli_req.dict.dict_len,
-                                        &dict);
-                if (ret < 0) {
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "failed to "
-                                "unserialize req-buffer to dictionary");
-                        snprintf (msg, sizeof (msg), "Unable to decode the "
-                                  "command");
-                        goto out;
-                } else {
-                        dict->extra_stdfree = cli_req.dict.dict_val;
-                }
+        ret = dict_unserialize(cli_req.dict.dict_val, cli_req.dict.dict_len,
+                               &dict);
+        if (ret < 0) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_UNSERIALIZE_FAIL,
+                   "failed to "
+                   "unserialize req-buffer to dictionary");
+            snprintf(msg, sizeof(msg),
+                     "Unable to decode the "
+                     "command");
+            goto out;
+        } else {
+            dict->extra_stdfree = cli_req.dict.dict_val;
         }
+    }
 
-        ret = dict_get_str (dict, "hostname", &hostname);
-        if (ret) {
-                snprintf (msg, sizeof (msg), "Failed to get hostname");
-                gf_log (this->name, GF_LOG_ERROR, "%s", msg);
-                goto out;
-        }
+    ret = dict_get_strn(dict, "hostname", SLEN("hostname"), &hostname);
+    if (ret) {
+        snprintf(msg, sizeof(msg), "Failed to get hostname");
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_HOSTNAME_NOTFOUND_IN_DICT,
+               "%s", msg);
+        goto out;
+    }
 
-        ret = dict_get_str (dict, "volname", &volname);
+    ret = dict_get_strn(dict, "volname", SLEN("volname"), &volname);
+    if (ret) {
+        ret = dict_get_int32n(dict, "flags", SLEN("flags"), (int32_t *)&flags);
         if (ret) {
-                ret = dict_get_int32 (dict, "flags", (int32_t*)&flags);
-                if (ret) {
-                        snprintf (msg, sizeof (msg), "Failed to get volume name"
-                                  " or flags");
-                        gf_log (this->name, GF_LOG_ERROR, "%s", msg);
-                        goto out;
-                }
+            snprintf(msg, sizeof(msg), "Failed to get volume name or flags");
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_FLAGS_NOTFOUND_IN_DICT,
+                   "%s", msg);
+            goto out;
         }
+    }
 
-        gf_log (this->name, GF_LOG_INFO, "Received volume sync req "
-                "for volume %s", (flags & GF_CLI_SYNC_ALL) ? "all" : volname);
+    gf_msg(this->name, GF_LOG_INFO, 0, GD_MSG_VOL_SYNC_REQ_RCVD,
+           "Received volume sync req "
+           "for volume %s",
+           (flags & GF_CLI_SYNC_ALL) ? "all" : volname);
 
-        if (gf_is_local_addr (hostname)) {
-                ret = -1;
-                snprintf (msg, sizeof (msg), "sync from localhost"
-                          " not allowed");
-                gf_log (this->name, GF_LOG_ERROR, "%s", msg);
-                goto out;
-        }
+    if (gf_is_local_addr(hostname)) {
+        ret = -1;
+        snprintf(msg, sizeof(msg),
+                 "sync from localhost"
+                 " not allowed");
+        gf_msg(this->name, GF_LOG_ERROR, 0,
+               GD_MSG_SYNC_FROM_LOCALHOST_UNALLOWED, "%s", msg);
+        goto out;
+    }
 
-        ret = glusterd_op_begin_synctask (req, GD_OP_SYNC_VOLUME, dict);
+    ret = glusterd_op_begin_synctask(req, GD_OP_SYNC_VOLUME, dict);
 
 out:
-        if (ret) {
-                cli_rsp.op_ret = -1;
-                cli_rsp.op_errstr = msg;
-                if (msg[0] == '\0')
-                        snprintf (msg, sizeof (msg), "Operation failed");
-                glusterd_to_cli (req, &cli_rsp, NULL, 0, NULL,
-                                 (xdrproc_t)xdr_gf_cli_rsp, dict);
-
-                ret = 0; //sent error to cli, prevent second reply
-        }
-
-        return ret;
+    if (ret) {
+        cli_rsp.op_ret = -1;
+        cli_rsp.op_errstr = msg;
+        if (msg[0] == '\0')
+            snprintf(msg, sizeof(msg), "Operation failed");
+        glusterd_to_cli(req, &cli_rsp, NULL, 0, NULL, (xdrproc_t)xdr_gf_cli_rsp,
+                        dict);
+
+        ret = 0;  // sent error to cli, prevent second reply
+    }
+
+    return ret;
 }
 
 int
-glusterd_handle_sync_volume (rpcsvc_request_t *req)
+glusterd_handle_sync_volume(rpcsvc_request_t *req)
 {
-        return glusterd_big_locked_handler (req, __glusterd_handle_sync_volume);
+    return glusterd_big_locked_handler(req, __glusterd_handle_sync_volume);
 }
 
 int
-glusterd_fsm_log_send_resp (rpcsvc_request_t *req, int op_ret,
-                            char *op_errstr, dict_t *dict)
+glusterd_fsm_log_send_resp(rpcsvc_request_t *req, int op_ret, char *op_errstr,
+                           dict_t *dict)
 {
+    int ret = -1;
+    gf1_cli_fsm_log_rsp rsp = {0};
 
-        int                             ret = -1;
-        gf1_cli_fsm_log_rsp             rsp = {0};
-
-        GF_ASSERT (req);
-        GF_ASSERT (op_errstr);
+    GF_ASSERT(req);
+    GF_ASSERT(op_errstr);
 
-        rsp.op_ret = op_ret;
-        rsp.op_errstr = op_errstr;
-        if (rsp.op_ret == 0)
-                ret = dict_allocate_and_serialize (dict, &rsp.fsm_log.fsm_log_val,
-                                                &rsp.fsm_log.fsm_log_len);
+    rsp.op_ret = op_ret;
+    rsp.op_errstr = op_errstr;
+    if (rsp.op_ret == 0) {
+        ret = dict_allocate_and_serialize(dict, &rsp.fsm_log.fsm_log_val,
+                                          &rsp.fsm_log.fsm_log_len);
+        if (ret < 0) {
+            gf_smsg("glusterd", GF_LOG_ERROR, errno,
+                    GD_MSG_DICT_ALLOC_AND_SERL_LENGTH_GET_FAIL, NULL);
+            return ret;
+        }
+    }
 
-        ret = glusterd_submit_reply (req, &rsp, NULL, 0, NULL,
-                                     (xdrproc_t)xdr_gf1_cli_fsm_log_rsp);
-        GF_FREE (rsp.fsm_log.fsm_log_val);
+    ret = glusterd_submit_reply(req, &rsp, NULL, 0, NULL,
+                                (xdrproc_t)xdr_gf1_cli_fsm_log_rsp);
+    GF_FREE(rsp.fsm_log.fsm_log_val);
 
-        gf_log ("glusterd", GF_LOG_DEBUG, "Responded, ret: %d", ret);
+    gf_msg_debug("glusterd", 0, "Responded, ret: %d", ret);
 
-        return 0;
+    return 0;
 }
 
 int
-__glusterd_handle_fsm_log (rpcsvc_request_t *req)
+__glusterd_handle_fsm_log(rpcsvc_request_t *req)
 {
-        int32_t                         ret = -1;
-        gf1_cli_fsm_log_req             cli_req = {0,};
-        dict_t                          *dict = NULL;
-        glusterd_sm_tr_log_t            *log = NULL;
-        xlator_t                        *this = NULL;
-        glusterd_conf_t                 *conf = NULL;
-        char                            msg[2048] = {0};
-        glusterd_peerinfo_t             *peerinfo = NULL;
-
-        GF_ASSERT (req);
-
-        ret = xdr_to_generic (req->msg[0], &cli_req,
-                              (xdrproc_t)xdr_gf1_cli_fsm_log_req);
-        if (ret < 0) {
-                //failed to decode msg;
-                req->rpc_err = GARBAGE_ARGS;
-                snprintf (msg, sizeof (msg), "Garbage request");
-                goto out;
-        }
+    int32_t ret = -1;
+    gf1_cli_fsm_log_req cli_req = {
+        0,
+    };
+    dict_t *dict = NULL;
+    xlator_t *this = NULL;
+    glusterd_conf_t *conf = NULL;
+    char msg[2048] = {0};
+    glusterd_peerinfo_t *peerinfo = NULL;
+
+    GF_ASSERT(req);
+
+    this = THIS;
+    GF_VALIDATE_OR_GOTO("xlator", (this != NULL), out);
+
+    ret = xdr_to_generic(req->msg[0], &cli_req,
+                         (xdrproc_t)xdr_gf1_cli_fsm_log_req);
+    if (ret < 0) {
+        // failed to decode msg;
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_REQ_DECODE_FAIL,
+               "Failed to decode "
+               "request received from client.");
+        req->rpc_err = GARBAGE_ARGS;
+        snprintf(msg, sizeof(msg), "Garbage request");
+        goto out;
+    }
+
+    dict = dict_new();
+    if (!dict) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_CREATE_FAIL, NULL);
+        ret = -1;
+        goto out;
+    }
 
-        dict = dict_new ();
-        if (!dict) {
-                ret = -1;
-                goto out;
-        }
+    if (strcmp("", cli_req.name) == 0) {
+        conf = this->private;
+        ret = glusterd_sm_tr_log_add_to_dict(dict, &conf->op_sm_log);
+    } else {
+        RCU_READ_LOCK;
 
-        if (strcmp ("", cli_req.name) == 0) {
-                this = THIS;
-                conf = this->private;
-                ret = glusterd_sm_tr_log_add_to_dict (dict, &conf->op_sm_log);
+        peerinfo = glusterd_peerinfo_find_by_hostname(cli_req.name);
+        if (!peerinfo) {
+            RCU_READ_UNLOCK;
+            ret = -1;
+            snprintf(msg, sizeof(msg), "%s is not a peer", cli_req.name);
         } else {
-                rcu_read_lock ();
-
-                peerinfo = glusterd_peerinfo_find_by_hostname (cli_req.name);
-                if (!peerinfo) {
-                        ret = -1;
-                        snprintf (msg, sizeof (msg), "%s is not a peer",
-                                  cli_req.name);
-                } else {
-                        ret = glusterd_sm_tr_log_add_to_dict
-                                (dict, &peerinfo->sm_log);
-                }
-
-                rcu_read_unlock ();
+            ret = glusterd_sm_tr_log_add_to_dict(dict, &peerinfo->sm_log);
+            RCU_READ_UNLOCK;
         }
+    }
 
 out:
-        (void)glusterd_fsm_log_send_resp (req, ret, msg, dict);
-        free (cli_req.name);//malloced by xdr
-        if (dict)
-                dict_unref (dict);
+    (void)glusterd_fsm_log_send_resp(req, ret, msg, dict);
+    free(cli_req.name);  // malloced by xdr
+    if (dict)
+        dict_unref(dict);
 
-        glusterd_friend_sm ();
-        glusterd_op_sm ();
+    glusterd_friend_sm();
+    glusterd_op_sm();
 
-        return 0;//send 0 to avoid double reply
+    return 0;  // send 0 to avoid double reply
 }
 
 int
-glusterd_handle_fsm_log (rpcsvc_request_t *req)
+glusterd_handle_fsm_log(rpcsvc_request_t *req)
 {
-        return glusterd_big_locked_handler (req, __glusterd_handle_fsm_log);
+    return glusterd_big_locked_handler(req, __glusterd_handle_fsm_log);
 }
 
 int
-glusterd_op_lock_send_resp (rpcsvc_request_t *req, int32_t status)
+glusterd_op_lock_send_resp(rpcsvc_request_t *req, int32_t status)
 {
+    gd1_mgmt_cluster_lock_rsp rsp = {
+        {0},
+    };
+    int ret = -1;
 
-        gd1_mgmt_cluster_lock_rsp       rsp = {{0},};
-        int                             ret = -1;
-
-        GF_ASSERT (req);
-        glusterd_get_uuid (&rsp.uuid);
-        rsp.op_ret = status;
+    GF_ASSERT(req);
+    glusterd_get_uuid(&rsp.uuid);
+    rsp.op_ret = status;
 
-        ret = glusterd_submit_reply (req, &rsp, NULL, 0, NULL,
-                                     (xdrproc_t)xdr_gd1_mgmt_cluster_lock_rsp);
+    ret = glusterd_submit_reply(req, &rsp, NULL, 0, NULL,
+                                (xdrproc_t)xdr_gd1_mgmt_cluster_lock_rsp);
 
-        gf_log (THIS->name, GF_LOG_DEBUG, "Responded to lock, ret: %d", ret);
+    gf_msg_debug(THIS->name, 0, "Responded to lock, ret: %d", ret);
 
-        return 0;
+    return 0;
 }
 
 int
-glusterd_op_unlock_send_resp (rpcsvc_request_t *req, int32_t status)
+glusterd_op_unlock_send_resp(rpcsvc_request_t *req, int32_t status)
 {
+    gd1_mgmt_cluster_unlock_rsp rsp = {
+        {0},
+    };
+    int ret = -1;
 
-        gd1_mgmt_cluster_unlock_rsp     rsp = {{0},};
-        int                             ret = -1;
-
-        GF_ASSERT (req);
-        rsp.op_ret = status;
-        glusterd_get_uuid (&rsp.uuid);
+    GF_ASSERT(req);
+    rsp.op_ret = status;
+    glusterd_get_uuid(&rsp.uuid);
 
-        ret = glusterd_submit_reply (req, &rsp, NULL, 0, NULL,
-                                     (xdrproc_t)xdr_gd1_mgmt_cluster_unlock_rsp);
+    ret = glusterd_submit_reply(req, &rsp, NULL, 0, NULL,
+                                (xdrproc_t)xdr_gd1_mgmt_cluster_unlock_rsp);
 
-        gf_log (THIS->name, GF_LOG_DEBUG, "Responded to unlock, ret: %d", ret);
+    gf_msg_debug(THIS->name, 0, "Responded to unlock, ret: %d", ret);
 
-        return ret;
+    return ret;
 }
 
 int
-glusterd_op_mgmt_v3_lock_send_resp (rpcsvc_request_t *req, uuid_t *txn_id,
+glusterd_op_mgmt_v3_lock_send_resp(rpcsvc_request_t *req, uuid_t *txn_id,
                                    int32_t status)
 {
+    gd1_mgmt_v3_lock_rsp rsp = {
+        {0},
+    };
+    int ret = -1;
 
-        gd1_mgmt_v3_lock_rsp    rsp = {{0},};
-        int                     ret = -1;
+    GF_ASSERT(req);
+    GF_ASSERT(txn_id);
+    glusterd_get_uuid(&rsp.uuid);
+    rsp.op_ret = status;
+    if (rsp.op_ret)
+        rsp.op_errno = errno;
+    gf_uuid_copy(rsp.txn_id, *txn_id);
 
-        GF_ASSERT (req);
-        GF_ASSERT (txn_id);
-        glusterd_get_uuid (&rsp.uuid);
-        rsp.op_ret = status;
-        if (rsp.op_ret)
-               rsp.op_errno = errno;
-        gf_uuid_copy (rsp.txn_id, *txn_id);
+    ret = glusterd_submit_reply(req, &rsp, NULL, 0, NULL,
+                                (xdrproc_t)xdr_gd1_mgmt_v3_lock_rsp);
 
-        ret = glusterd_submit_reply (req, &rsp, NULL, 0, NULL,
-                                     (xdrproc_t)xdr_gd1_mgmt_v3_lock_rsp);
+    gf_msg_debug(THIS->name, 0, "Responded to mgmt_v3 lock, ret: %d", ret);
 
-        gf_log (THIS->name, GF_LOG_DEBUG, "Responded to mgmt_v3 lock, ret: %d",
-                ret);
-
-        return ret;
+    return ret;
 }
 
 int
-glusterd_op_mgmt_v3_unlock_send_resp (rpcsvc_request_t *req, uuid_t *txn_id,
+glusterd_op_mgmt_v3_unlock_send_resp(rpcsvc_request_t *req, uuid_t *txn_id,
                                      int32_t status)
 {
+    gd1_mgmt_v3_unlock_rsp rsp = {
+        {0},
+    };
+    int ret = -1;
 
-        gd1_mgmt_v3_unlock_rsp      rsp = {{0},};
-        int                             ret = -1;
+    GF_ASSERT(req);
+    GF_ASSERT(txn_id);
+    rsp.op_ret = status;
+    if (rsp.op_ret)
+        rsp.op_errno = errno;
+    glusterd_get_uuid(&rsp.uuid);
+    gf_uuid_copy(rsp.txn_id, *txn_id);
 
-        GF_ASSERT (req);
-        GF_ASSERT (txn_id);
-        rsp.op_ret = status;
-        if (rsp.op_ret)
-               rsp.op_errno = errno;
-        glusterd_get_uuid (&rsp.uuid);
-        gf_uuid_copy (rsp.txn_id, *txn_id);
+    ret = glusterd_submit_reply(req, &rsp, NULL, 0, NULL,
+                                (xdrproc_t)xdr_gd1_mgmt_v3_unlock_rsp);
 
-        ret = glusterd_submit_reply (req, &rsp, NULL, 0, NULL,
-                                     (xdrproc_t)xdr_gd1_mgmt_v3_unlock_rsp);
+    gf_msg_debug(THIS->name, 0, "Responded to mgmt_v3 unlock, ret: %d", ret);
 
-        gf_log (THIS->name, GF_LOG_DEBUG, "Responded to mgmt_v3 unlock, ret: %d",
-                ret);
-
-        return ret;
+    return ret;
 }
 
 int
-__glusterd_handle_cluster_unlock (rpcsvc_request_t *req)
+__glusterd_handle_cluster_unlock(rpcsvc_request_t *req)
 {
-        gd1_mgmt_cluster_unlock_req     unlock_req = {{0}, };
-        int32_t                         ret = -1;
-        glusterd_op_lock_ctx_t          *ctx = NULL;
-        xlator_t                        *this = NULL;
-        uuid_t                          *txn_id = NULL;
-        glusterd_conf_t                 *priv = NULL;
-
-        this = THIS;
-        GF_ASSERT (this);
-        priv = this->private;
-        GF_ASSERT (priv);
-        GF_ASSERT (req);
-
-        txn_id = &priv->global_txn_id;
-
-        ret = xdr_to_generic (req->msg[0], &unlock_req,
-                              (xdrproc_t)xdr_gd1_mgmt_cluster_unlock_req);
-        if (ret < 0) {
-                gf_log (this->name, GF_LOG_ERROR, "Failed to decode unlock "
-                        "request received from peer");
-                req->rpc_err = GARBAGE_ARGS;
-                goto out;
-        }
-
-
-        gf_log (this->name, GF_LOG_DEBUG,
-                "Received UNLOCK from uuid: %s", uuid_utoa (unlock_req.uuid));
-
-        rcu_read_lock ();
-        ret = (glusterd_peerinfo_find_by_uuid (unlock_req.uuid) == NULL);
-        rcu_read_unlock ();
-        if (ret) {
-                gf_log (this->name, GF_LOG_WARNING, "%s doesn't "
-                        "belong to the cluster. Ignoring request.",
-                        uuid_utoa (unlock_req.uuid));
-                ret = -1;
-                goto out;
-        }
+    gd1_mgmt_cluster_unlock_req unlock_req = {
+        {0},
+    };
+    int32_t ret = -1;
+    glusterd_op_lock_ctx_t *ctx = NULL;
+    xlator_t *this = NULL;
+    uuid_t *txn_id = NULL;
+    glusterd_conf_t *priv = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+    priv = this->private;
+    GF_ASSERT(priv);
+    GF_ASSERT(req);
+
+    txn_id = &priv->global_txn_id;
+
+    ret = xdr_to_generic(req->msg[0], &unlock_req,
+                         (xdrproc_t)xdr_gd1_mgmt_cluster_unlock_req);
+    if (ret < 0) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_REQ_DECODE_FAIL,
+               "Failed to decode unlock "
+               "request received from peer");
+        req->rpc_err = GARBAGE_ARGS;
+        goto out;
+    }
+
+    gf_msg_debug(this->name, 0, "Received UNLOCK from uuid: %s",
+                 uuid_utoa(unlock_req.uuid));
+
+    RCU_READ_LOCK;
+    ret = (glusterd_peerinfo_find_by_uuid(unlock_req.uuid) == NULL);
+    RCU_READ_LOCK;
+    if (ret) {
+        gf_msg(this->name, GF_LOG_WARNING, 0, GD_MSG_PEER_NOT_FOUND,
+               "%s doesn't "
+               "belong to the cluster. Ignoring request.",
+               uuid_utoa(unlock_req.uuid));
+        ret = -1;
+        goto out;
+    }
 
-        ctx = GF_CALLOC (1, sizeof (*ctx), gf_gld_mt_op_lock_ctx_t);
+    ctx = GF_CALLOC(1, sizeof(*ctx), gf_gld_mt_op_lock_ctx_t);
 
-        if (!ctx) {
-                //respond here
-                return -1;
-        }
-        gf_uuid_copy (ctx->uuid, unlock_req.uuid);
-        ctx->req = req;
-        ctx->dict = NULL;
+    if (!ctx) {
+        // respond here
+        gf_msg(this->name, GF_LOG_ERROR, ENOMEM, GD_MSG_NO_MEMORY,
+               "No memory.");
+        return -1;
+    }
+    gf_uuid_copy(ctx->uuid, unlock_req.uuid);
+    ctx->req = req;
+    ctx->dict = NULL;
 
-        ret = glusterd_op_sm_inject_event (GD_OP_EVENT_UNLOCK, txn_id, ctx);
+    ret = glusterd_op_sm_inject_event(GD_OP_EVENT_UNLOCK, txn_id, ctx);
 
 out:
-        glusterd_friend_sm ();
-        glusterd_op_sm ();
+    glusterd_friend_sm();
+    glusterd_op_sm();
 
-        return ret;
+    return ret;
 }
 
 int
-glusterd_handle_cluster_unlock (rpcsvc_request_t *req)
+glusterd_handle_cluster_unlock(rpcsvc_request_t *req)
 {
-        return glusterd_big_locked_handler (req,
-                                            __glusterd_handle_cluster_unlock);
+    return glusterd_big_locked_handler(req, __glusterd_handle_cluster_unlock);
 }
 
 int
-glusterd_op_stage_send_resp (rpcsvc_request_t   *req,
-                             int32_t op, int32_t status,
-                             char *op_errstr, dict_t *rsp_dict)
+glusterd_op_stage_send_resp(rpcsvc_request_t *req, int32_t op, int32_t status,
+                            char *op_errstr, dict_t *rsp_dict)
 {
-        gd1_mgmt_stage_op_rsp           rsp      = {{0},};
-        int                             ret      = -1;
-        xlator_t                       *this     = NULL;
-
-        this = THIS;
-        GF_ASSERT (this);
-        GF_ASSERT (req);
-
-        rsp.op_ret = status;
-        glusterd_get_uuid (&rsp.uuid);
-        rsp.op = op;
-        if (op_errstr)
-                rsp.op_errstr = op_errstr;
-        else
-                rsp.op_errstr = "";
-
-        ret = dict_allocate_and_serialize (rsp_dict, &rsp.dict.dict_val,
-                                           &rsp.dict.dict_len);
-        if (ret < 0) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "failed to get serialized length of dict");
-                return ret;
-        }
+    gd1_mgmt_stage_op_rsp rsp = {
+        {0},
+    };
+    int ret = -1;
+    xlator_t *this = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+    GF_ASSERT(req);
+
+    rsp.op_ret = status;
+    glusterd_get_uuid(&rsp.uuid);
+    rsp.op = op;
+    if (op_errstr)
+        rsp.op_errstr = op_errstr;
+    else
+        rsp.op_errstr = "";
 
-        ret = glusterd_submit_reply (req, &rsp, NULL, 0, NULL,
-                                     (xdrproc_t)xdr_gd1_mgmt_stage_op_rsp);
+    ret = dict_allocate_and_serialize(rsp_dict, &rsp.dict.dict_val,
+                                      &rsp.dict.dict_len);
+    if (ret < 0) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno,
+                GD_MSG_DICT_ALLOC_AND_SERL_LENGTH_GET_FAIL, NULL);
+        return ret;
+    }
 
-        gf_log (this->name, GF_LOG_DEBUG, "Responded to stage, ret: %d", ret);
-        GF_FREE (rsp.dict.dict_val);
+    ret = glusterd_submit_reply(req, &rsp, NULL, 0, NULL,
+                                (xdrproc_t)xdr_gd1_mgmt_stage_op_rsp);
 
-        return ret;
+    gf_msg_debug(this->name, 0, "Responded to stage, ret: %d", ret);
+    GF_FREE(rsp.dict.dict_val);
+
+    return ret;
 }
 
 int
-glusterd_op_commit_send_resp (rpcsvc_request_t *req,
-                               int32_t op, int32_t status, char *op_errstr,
-                               dict_t *rsp_dict)
+glusterd_op_commit_send_resp(rpcsvc_request_t *req, int32_t op, int32_t status,
+                             char *op_errstr, dict_t *rsp_dict)
 {
-        gd1_mgmt_commit_op_rsp          rsp      = {{0}, };
-        int                             ret      = -1;
-        xlator_t                        *this = NULL;
-
-        this = THIS;
-        GF_ASSERT (this);
-        GF_ASSERT (req);
-        rsp.op_ret = status;
-        glusterd_get_uuid (&rsp.uuid);
-        rsp.op = op;
-
-        if (op_errstr)
-                rsp.op_errstr = op_errstr;
-        else
-                rsp.op_errstr = "";
-
-        if (rsp_dict) {
-                ret = dict_allocate_and_serialize (rsp_dict, &rsp.dict.dict_val,
-                                                   &rsp.dict.dict_len);
-                if (ret < 0) {
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "failed to get serialized length of dict");
-                        goto out;
-                }
-        }
+    gd1_mgmt_commit_op_rsp rsp = {
+        {0},
+    };
+    int ret = -1;
+    xlator_t *this = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+    GF_ASSERT(req);
+    rsp.op_ret = status;
+    glusterd_get_uuid(&rsp.uuid);
+    rsp.op = op;
+
+    if (op_errstr)
+        rsp.op_errstr = op_errstr;
+    else
+        rsp.op_errstr = "";
 
+    if (rsp_dict) {
+        ret = dict_allocate_and_serialize(rsp_dict, &rsp.dict.dict_val,
+                                          &rsp.dict.dict_len);
+        if (ret < 0) {
+            gf_smsg(this->name, GF_LOG_ERROR, errno,
+                    GD_MSG_DICT_ALLOC_AND_SERL_LENGTH_GET_FAIL, NULL);
+            goto out;
+        }
+    }
 
-        ret = glusterd_submit_reply (req, &rsp, NULL, 0, NULL,
-                                     (xdrproc_t)xdr_gd1_mgmt_commit_op_rsp);
+    ret = glusterd_submit_reply(req, &rsp, NULL, 0, NULL,
+                                (xdrproc_t)xdr_gd1_mgmt_commit_op_rsp);
 
-        gf_log (this->name, GF_LOG_DEBUG, "Responded to commit, ret: %d", ret);
+    gf_msg_debug(this->name, 0, "Responded to commit, ret: %d", ret);
 
 out:
-        GF_FREE (rsp.dict.dict_val);
-        return ret;
+    GF_FREE(rsp.dict.dict_val);
+    return ret;
 }
 
 int
-__glusterd_handle_incoming_friend_req (rpcsvc_request_t *req)
+__glusterd_handle_incoming_friend_req(rpcsvc_request_t *req)
 {
-        int32_t                 ret = -1;
-        gd1_mgmt_friend_req     friend_req = {{0},};
-        gf_boolean_t            run_fsm = _gf_true;
-
-        GF_ASSERT (req);
-        ret = xdr_to_generic (req->msg[0], &friend_req,
-                              (xdrproc_t)xdr_gd1_mgmt_friend_req);
-        if (ret < 0) {
-                //failed to decode msg;
-                req->rpc_err = GARBAGE_ARGS;
-                goto out;
-        }
-
-        gf_log ("glusterd", GF_LOG_INFO,
-                "Received probe from uuid: %s", uuid_utoa (friend_req.uuid));
-        ret = glusterd_handle_friend_req (req, friend_req.uuid,
-                                          friend_req.hostname, friend_req.port,
-                                          &friend_req);
-
-        if (ret == GLUSTERD_CONNECTION_AWAITED) {
-                //fsm should be run after connection establishes
-                run_fsm = _gf_false;
-                ret = 0;
-        }
+    int32_t ret = -1;
+    gd1_mgmt_friend_req friend_req = {
+        {0},
+    };
+    gf_boolean_t run_fsm = _gf_true;
+
+    GF_ASSERT(req);
+    ret = xdr_to_generic(req->msg[0], &friend_req,
+                         (xdrproc_t)xdr_gd1_mgmt_friend_req);
+    if (ret < 0) {
+        // failed to decode msg;
+        gf_msg("glusterd", GF_LOG_ERROR, 0, GD_MSG_REQ_DECODE_FAIL,
+               "Failed to decode "
+               "request received from friend");
+        req->rpc_err = GARBAGE_ARGS;
+        goto out;
+    }
+
+    gf_msg("glusterd", GF_LOG_INFO, 0, GD_MSG_PROBE_RCVD,
+           "Received probe from uuid: %s", uuid_utoa(friend_req.uuid));
+    ret = glusterd_handle_friend_req(req, friend_req.uuid, friend_req.hostname,
+                                     friend_req.port, &friend_req);
+
+    if (ret == GLUSTERD_CONNECTION_AWAITED) {
+        // fsm should be run after connection establishes
+        run_fsm = _gf_false;
+        ret = 0;
+    }
 
 out:
-        free (friend_req.hostname);//malloced by xdr
+    free(friend_req.hostname);  // malloced by xdr
 
-        if (run_fsm) {
-                glusterd_friend_sm ();
-                glusterd_op_sm ();
-        }
+    if (run_fsm) {
+        glusterd_friend_sm();
+        glusterd_op_sm();
+    }
 
-        return ret;
+    return ret;
 }
 
 int
-glusterd_handle_incoming_friend_req (rpcsvc_request_t *req)
+glusterd_handle_incoming_friend_req(rpcsvc_request_t *req)
 {
-        return glusterd_big_locked_handler (req,
-                                            __glusterd_handle_incoming_friend_req);
+    return glusterd_big_locked_handler(req,
+                                       __glusterd_handle_incoming_friend_req);
 }
 
 int
-__glusterd_handle_incoming_unfriend_req (rpcsvc_request_t *req)
+__glusterd_handle_incoming_unfriend_req(rpcsvc_request_t *req)
 {
-        int32_t                 ret = -1;
-        gd1_mgmt_friend_req     friend_req = {{0},};
-        char               remote_hostname[UNIX_PATH_MAX + 1] = {0,};
-
-        GF_ASSERT (req);
-        ret = xdr_to_generic (req->msg[0], &friend_req,
-                              (xdrproc_t)xdr_gd1_mgmt_friend_req);
-        if (ret < 0) {
-                //failed to decode msg;
-                req->rpc_err = GARBAGE_ARGS;
-                goto out;
-        }
-
-        gf_log ("glusterd", GF_LOG_INFO,
-                "Received unfriend from uuid: %s", uuid_utoa (friend_req.uuid));
-
-        ret = glusterd_remote_hostname_get (req, remote_hostname,
-                                            sizeof (remote_hostname));
-        if (ret) {
-                gf_log ("", GF_LOG_ERROR, "Unable to get the remote hostname");
-                goto out;
-        }
-        ret = glusterd_handle_unfriend_req (req, friend_req.uuid,
-                                            remote_hostname, friend_req.port);
+    int32_t ret = -1;
+    gd1_mgmt_friend_req friend_req = {
+        {0},
+    };
+    char remote_hostname[UNIX_PATH_MAX + 1] = {
+        0,
+    };
+
+    GF_ASSERT(req);
+    ret = xdr_to_generic(req->msg[0], &friend_req,
+                         (xdrproc_t)xdr_gd1_mgmt_friend_req);
+    if (ret < 0) {
+        // failed to decode msg;
+        gf_msg("glusterd", GF_LOG_ERROR, 0, GD_MSG_REQ_DECODE_FAIL,
+               "Failed to decode "
+               "request received.");
+        req->rpc_err = GARBAGE_ARGS;
+        goto out;
+    }
+
+    gf_msg("glusterd", GF_LOG_INFO, 0, GD_MSG_UNFRIEND_REQ_RCVD,
+           "Received unfriend from uuid: %s", uuid_utoa(friend_req.uuid));
+
+    ret = glusterd_remote_hostname_get(req, remote_hostname,
+                                       sizeof(remote_hostname));
+    if (ret) {
+        gf_msg("glusterd", GF_LOG_ERROR, 0, GD_MSG_HOSTNAME_RESOLVE_FAIL,
+               "Unable to get the remote hostname");
+        goto out;
+    }
+    ret = glusterd_handle_unfriend_req(req, friend_req.uuid, remote_hostname,
+                                       friend_req.port);
 
 out:
-        free (friend_req.hostname);//malloced by xdr
-        free (friend_req.vols.vols_val);//malloced by xdr
+    free(friend_req.hostname);       // malloced by xdr
+    free(friend_req.vols.vols_val);  // malloced by xdr
 
-        glusterd_friend_sm ();
-        glusterd_op_sm ();
+    glusterd_friend_sm();
+    glusterd_op_sm();
 
-        return ret;
+    return ret;
 }
 
 int
-glusterd_handle_incoming_unfriend_req (rpcsvc_request_t *req)
+glusterd_handle_incoming_unfriend_req(rpcsvc_request_t *req)
 {
-        return glusterd_big_locked_handler (req,
-                                            __glusterd_handle_incoming_unfriend_req);
-
+    return glusterd_big_locked_handler(req,
+                                       __glusterd_handle_incoming_unfriend_req);
 }
 
 int
-glusterd_handle_friend_update_delete (dict_t *dict)
+glusterd_handle_friend_update_delete(dict_t *dict)
 {
-        char                    *hostname = NULL;
-        int32_t                 ret = -1;
+    char *hostname = NULL;
+    int32_t ret = -1;
 
-        GF_ASSERT (dict);
+    GF_ASSERT(dict);
 
-        ret = dict_get_str (dict, "hostname", &hostname);
-        if (ret)
-                goto out;
+    ret = dict_get_strn(dict, "hostname", SLEN("hostname"), &hostname);
+    if (ret)
+        goto out;
 
-        ret = glusterd_friend_remove (NULL, hostname);
+    ret = glusterd_friend_remove(NULL, hostname);
 
 out:
-        gf_log ("", GF_LOG_DEBUG, "Returning %d", ret);
-        return ret;
+    gf_msg_debug("glusterd", 0, "Returning %d", ret);
+    return ret;
 }
 
 int
-glusterd_peer_hostname_update (glusterd_peerinfo_t *peerinfo,
-                               const char *hostname, gf_boolean_t store_update)
+glusterd_peer_hostname_update(glusterd_peerinfo_t *peerinfo,
+                              const char *hostname, gf_boolean_t store_update)
 {
-        int                     ret = 0;
+    int ret = 0;
 
-        GF_ASSERT (peerinfo);
-        GF_ASSERT (hostname);
+    GF_ASSERT(peerinfo);
+    GF_ASSERT(hostname);
 
-        ret = gd_add_address_to_peer (peerinfo, hostname);
-        if (ret) {
-                gf_log (THIS->name, GF_LOG_ERROR,
-                        "Couldn't add address to the peer info");
-                goto out;
-        }
+    ret = gd_add_address_to_peer(peerinfo, hostname);
+    if (ret) {
+        gf_msg(THIS->name, GF_LOG_ERROR, 0,
+               GD_MSG_HOSTNAME_ADD_TO_PEERLIST_FAIL,
+               "Couldn't add address to the peer info");
+        goto out;
+    }
 
-        if (store_update)
-                ret = glusterd_store_peerinfo (peerinfo);
+    if (store_update)
+        ret = glusterd_store_peerinfo(peerinfo);
 out:
-        gf_log ("", GF_LOG_DEBUG, "Returning %d", ret);
-        return ret;
+    gf_msg_debug(THIS->name, 0, "Returning %d", ret);
+    return ret;
 }
 
 int
-__glusterd_handle_friend_update (rpcsvc_request_t *req)
+__glusterd_handle_friend_update(rpcsvc_request_t *req)
 {
-        int32_t                 ret = -1;
-        gd1_mgmt_friend_update     friend_req = {{0},};
-        glusterd_peerinfo_t     *peerinfo = NULL;
-        glusterd_conf_t         *priv = NULL;
-        xlator_t                *this = NULL;
-        gd1_mgmt_friend_update_rsp rsp = {{0},};
-        dict_t                  *dict = NULL;
-        char                    key[100] = {0,};
-        char                    *uuid_buf = NULL;
-        int                     i = 1;
-        int                     count = 0;
-        uuid_t                  uuid = {0,};
-        glusterd_peerctx_args_t args = {0};
-        int32_t                 op = 0;
-
-        GF_ASSERT (req);
-
-        this = THIS;
-        GF_ASSERT (this);
-        priv = this->private;
-        GF_ASSERT (priv);
-
-        ret = xdr_to_generic (req->msg[0], &friend_req,
-                              (xdrproc_t)xdr_gd1_mgmt_friend_update);
-        if (ret < 0) {
-                //failed to decode msg;
-                req->rpc_err = GARBAGE_ARGS;
-                goto out;
-        }
-
-        ret = 0;
-        rcu_read_lock ();
-        if (glusterd_peerinfo_find (friend_req.uuid, NULL) == NULL) {
-                ret = -1;
-        }
-        rcu_read_unlock ();
-        if (ret) {
-                gf_log ("", GF_LOG_CRITICAL, "Received friend update request "
-                        "from unknown peer %s", uuid_utoa (friend_req.uuid));
-                goto out;
-        }
-
-        gf_log ("glusterd", GF_LOG_INFO,
-                "Received friend update from uuid: %s", uuid_utoa (friend_req.uuid));
-
-        if (friend_req.friends.friends_len) {
-                /* Unserialize the dictionary */
-                dict  = dict_new ();
-
-                ret = dict_unserialize (friend_req.friends.friends_val,
-                                        friend_req.friends.friends_len,
-                                        &dict);
-                if (ret < 0) {
-                        gf_log ("glusterd", GF_LOG_ERROR,
-                                "failed to "
-                                "unserialize req-buffer to dictionary");
-                        goto out;
-                } else {
-                        dict->extra_stdfree = friend_req.friends.friends_val;
-                }
-        }
-
-        ret = dict_get_int32 (dict, "count", &count);
-        if (ret)
-                goto out;
+    int32_t ret = -1;
+    gd1_mgmt_friend_update friend_req = {
+        {0},
+    };
+    glusterd_peerinfo_t *peerinfo = NULL;
+    glusterd_conf_t *priv = NULL;
+    xlator_t *this = NULL;
+    gd1_mgmt_friend_update_rsp rsp = {
+        {0},
+    };
+    dict_t *dict = NULL;
+    char key[32] = {
+        0,
+    };
+    int keylen;
+    char *uuid_buf = NULL;
+    int i = 1;
+    int count = 0;
+    uuid_t uuid = {
+        0,
+    };
+    glusterd_peerctx_args_t args = {0};
+    int32_t op = 0;
+
+    GF_ASSERT(req);
+
+    this = THIS;
+    GF_ASSERT(this);
+    priv = this->private;
+    GF_ASSERT(priv);
+
+    ret = xdr_to_generic(req->msg[0], &friend_req,
+                         (xdrproc_t)xdr_gd1_mgmt_friend_update);
+    if (ret < 0) {
+        // failed to decode msg;
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_REQ_DECODE_FAIL,
+               "Failed to decode "
+               "request received");
+        req->rpc_err = GARBAGE_ARGS;
+        goto out;
+    }
+
+    ret = 0;
+    RCU_READ_LOCK;
+    if (glusterd_peerinfo_find(friend_req.uuid, NULL) == NULL) {
+        ret = -1;
+    }
+    RCU_READ_UNLOCK;
+    if (ret) {
+        gf_msg(this->name, GF_LOG_CRITICAL, 0, GD_MSG_REQ_FROM_UNKNOWN_PEER,
+               "Received friend update request "
+               "from unknown peer %s",
+               uuid_utoa(friend_req.uuid));
+        gf_event(EVENT_UNKNOWN_PEER, "peer=%s", uuid_utoa(friend_req.uuid));
+        goto out;
+    }
+
+    gf_msg("glusterd", GF_LOG_INFO, 0, GD_MSG_FRIEND_UPDATE_RCVD,
+           "Received friend update from uuid: %s", uuid_utoa(friend_req.uuid));
+
+    if (friend_req.friends.friends_len) {
+        /* Unserialize the dictionary */
+        dict = dict_new();
 
-        ret = dict_get_int32 (dict, "op", &op);
+        ret = dict_unserialize(friend_req.friends.friends_val,
+                               friend_req.friends.friends_len, &dict);
+        if (ret < 0) {
+            gf_msg("glusterd", GF_LOG_ERROR, 0, GD_MSG_DICT_UNSERIALIZE_FAIL,
+                   "failed to "
+                   "unserialize req-buffer to dictionary");
+            goto out;
+        } else {
+            dict->extra_stdfree = friend_req.friends.friends_val;
+        }
+    }
+
+    ret = dict_get_int32n(dict, "count", SLEN("count"), &count);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_GET_FAILED,
+                "Key=count", NULL);
+        goto out;
+    }
+
+    ret = dict_get_int32n(dict, "op", SLEN("op"), &op);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_GET_FAILED,
+                "Key=op", NULL);
+        goto out;
+    }
+
+    if (GD_FRIEND_UPDATE_DEL == op) {
+        (void)glusterd_handle_friend_update_delete(dict);
+        goto out;
+    }
+
+    args.mode = GD_MODE_ON;
+    while (i <= count) {
+        keylen = snprintf(key, sizeof(key), "friend%d.uuid", i);
+        ret = dict_get_strn(dict, key, keylen, &uuid_buf);
         if (ret)
-                goto out;
+            goto out;
+        gf_uuid_parse(uuid_buf, uuid);
 
-        if (GD_FRIEND_UPDATE_DEL == op) {
-                ret = glusterd_handle_friend_update_delete (dict);
-                goto out;
+        if (!gf_uuid_compare(uuid, MY_UUID)) {
+            gf_msg(this->name, GF_LOG_INFO, 0, GD_MSG_UUID_RECEIVED,
+                   "Received my uuid as Friend");
+            i++;
+            continue;
         }
 
-        args.mode = GD_MODE_ON;
-        while ( i <= count) {
-                memset (key, 0, sizeof (key));
-                snprintf (key, sizeof (key), "friend%d.uuid", i);
-                ret = dict_get_str (dict, key, &uuid_buf);
-                if (ret)
-                        goto out;
-                gf_uuid_parse (uuid_buf, uuid);
-
-                if (!gf_uuid_compare (uuid, MY_UUID)) {
-                        gf_log (this->name, GF_LOG_INFO,
-                                "Received my uuid as Friend");
-                        i++;
-                        continue;
-                }
-
-                memset (key, 0, sizeof (key));
-                snprintf (key, sizeof (key), "friend%d", i);
-
-                rcu_read_lock ();
-                peerinfo = glusterd_peerinfo_find (uuid, NULL);
-                if (peerinfo == NULL) {
-                        /* Create a new peer and add it to the list as there is
-                         * no existing peer with the uuid
-                         */
-                        peerinfo = gd_peerinfo_from_dict (dict, key);
-                        if (peerinfo == NULL) {
-                                ret = -1;
-                                gf_log (this->name, GF_LOG_ERROR,
-                                        "Could not create peerinfo from dict "
-                                        "for prefix %s", key);
-                                goto unlock;
-                        }
-
-                        /* As this is a new peer, it should be added as a
-                         * friend.  The friend state machine will take care of
-                         * correcting the state as required
-                         */
-                        peerinfo->state.state = GD_FRIEND_STATE_BEFRIENDED;
-
-                        ret = glusterd_friend_add_from_peerinfo (peerinfo, 0,
-                                                                 &args);
-                } else {
-                        /* As an existing peer was found, update it with the new
-                         * information
-                         */
-                        ret = gd_update_peerinfo_from_dict (peerinfo, dict,
-                                                            key);
-                        if (ret) {
-                                gf_log (this->name, GF_LOG_ERROR, "Failed to "
-                                        "update peer %s", peerinfo->hostname);
-                        }
-                }
-unlock:
-                rcu_read_unlock ();
-                if (ret)
-                        break;
-
-                peerinfo = NULL;
-                i++;
-        }
+        snprintf(key, sizeof(key), "friend%d", i);
 
-out:
-        gf_uuid_copy (rsp.uuid, MY_UUID);
-        ret = glusterd_submit_reply (req, &rsp, NULL, 0, NULL,
-                                     (xdrproc_t)xdr_gd1_mgmt_friend_update_rsp);
-        if (dict) {
-                if (!dict->extra_stdfree && friend_req.friends.friends_val)
-                        free (friend_req.friends.friends_val);//malloced by xdr
-                dict_unref (dict);
+        RCU_READ_LOCK;
+        peerinfo = glusterd_peerinfo_find(uuid, NULL);
+        if (peerinfo == NULL) {
+            /* Create a new peer and add it to the list as there is
+             * no existing peer with the uuid
+             */
+            peerinfo = gd_peerinfo_from_dict(dict, key);
+            if (peerinfo == NULL) {
+                ret = -1;
+                gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_PEERINFO_CREATE_FAIL,
+                       "Could not create peerinfo from dict "
+                       "for prefix %s",
+                       key);
+                goto unlock;
+            }
+
+            /* As this is a new peer, it should be added as a
+             * friend.  The friend state machine will take care of
+             * correcting the state as required
+             */
+            peerinfo->state.state = GD_FRIEND_STATE_BEFRIENDED;
+
+            ret = glusterd_friend_add_from_peerinfo(peerinfo, 0, &args);
         } else {
-                free (friend_req.friends.friends_val);//malloced by xdr
-        }
-
-        if (peerinfo)
-                glusterd_peerinfo_cleanup (peerinfo);
+            /* As an existing peer was found, update it with the new
+             * information
+             */
+            ret = gd_update_peerinfo_from_dict(peerinfo, dict, key);
+            if (ret) {
+                gf_msg(this->name, GF_LOG_ERROR, 0,
+                       GD_MSG_PEER_INFO_UPDATE_FAIL,
+                       "Failed to "
+                       "update peer %s",
+                       peerinfo->hostname);
+                goto unlock;
+            }
+            ret = glusterd_store_peerinfo(peerinfo);
+            if (ret) {
+                gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_PEERINFO_CREATE_FAIL,
+                       "Failed to store peerinfo");
+                gf_event(EVENT_PEER_STORE_FAILURE, "peer=%s",
+                         peerinfo->hostname);
+            }
+        }
+    unlock:
+        RCU_READ_UNLOCK;
+        if (ret)
+            break;
 
-        glusterd_friend_sm ();
-        glusterd_op_sm ();
+        peerinfo = NULL;
+        i++;
+    }
 
-        return ret;
+out:
+    gf_uuid_copy(rsp.uuid, MY_UUID);
+    ret = glusterd_submit_reply(req, &rsp, NULL, 0, NULL,
+                                (xdrproc_t)xdr_gd1_mgmt_friend_update_rsp);
+    if (dict) {
+        if (!dict->extra_stdfree && friend_req.friends.friends_val)
+            free(friend_req.friends.friends_val);  // malloced by xdr
+        dict_unref(dict);
+    } else {
+        free(friend_req.friends.friends_val);  // malloced by xdr
+    }
+
+    if (peerinfo)
+        glusterd_peerinfo_cleanup(peerinfo);
+
+    glusterd_friend_sm();
+    glusterd_op_sm();
+
+    return ret;
 }
 
 int
-glusterd_handle_friend_update (rpcsvc_request_t *req)
+glusterd_handle_friend_update(rpcsvc_request_t *req)
 {
-        return glusterd_big_locked_handler (req,
-                                            __glusterd_handle_friend_update);
+    return glusterd_big_locked_handler(req, __glusterd_handle_friend_update);
 }
 
 int
-__glusterd_handle_probe_query (rpcsvc_request_t *req)
+__glusterd_handle_probe_query(rpcsvc_request_t *req)
 {
-        int32_t                         ret = -1;
-        xlator_t                        *this = NULL;
-        glusterd_conf_t                 *conf = NULL;
-        gd1_mgmt_probe_req              probe_req = {{0},};
-        gd1_mgmt_probe_rsp              rsp = {{0},};
-        glusterd_peerinfo_t             *peerinfo = NULL;
-        glusterd_peerctx_args_t         args = {0};
-        int                             port = 0;
-        char               remote_hostname[UNIX_PATH_MAX + 1] = {0,};
-
-        GF_ASSERT (req);
-
-        ret = xdr_to_generic (req->msg[0], &probe_req,
-                              (xdrproc_t)xdr_gd1_mgmt_probe_req);
-        if (ret < 0) {
-                //failed to decode msg;
-                req->rpc_err = GARBAGE_ARGS;
-                goto out;
-        }
-
-        this = THIS;
-
-        conf = this->private;
-        if (probe_req.port)
-                port = probe_req.port;
-        else
-                port = GF_DEFAULT_BASE_PORT;
-
-        gf_log ("glusterd", GF_LOG_INFO,
-                "Received probe from uuid: %s", uuid_utoa (probe_req.uuid));
-
-        /* Check for uuid collision and handle it in a user friendly way by
-         * sending the error.
-         */
-        if (!gf_uuid_compare (probe_req.uuid, MY_UUID)) {
-                gf_log (THIS->name, GF_LOG_ERROR, "Peer uuid %s is same as "
-                        "local uuid. Please check the uuid of both the peers "
-                        "from %s/%s", uuid_utoa (probe_req.uuid),
-                        GLUSTERD_DEFAULT_WORKDIR, GLUSTERD_INFO_FILE);
-                rsp.op_ret = -1;
-                rsp.op_errno = GF_PROBE_SAME_UUID;
-                rsp.port = port;
-                goto respond;
-        }
-
-        ret = glusterd_remote_hostname_get (req, remote_hostname,
-                                            sizeof (remote_hostname));
+    int32_t ret = -1;
+    xlator_t *this = NULL;
+    glusterd_conf_t *conf = NULL;
+    gd1_mgmt_probe_req probe_req = {
+        {0},
+    };
+    gd1_mgmt_probe_rsp rsp = {
+        {0},
+    };
+    glusterd_peerinfo_t *peerinfo = NULL;
+    glusterd_peerctx_args_t args = {0};
+    int port = 0;
+    char remote_hostname[UNIX_PATH_MAX + 1] = {
+        0,
+    };
+
+    GF_ASSERT(req);
+
+    this = THIS;
+    GF_VALIDATE_OR_GOTO("xlator", (this != NULL), out);
+
+    ret = xdr_to_generic(req->msg[0], &probe_req,
+                         (xdrproc_t)xdr_gd1_mgmt_probe_req);
+    if (ret < 0) {
+        // failed to decode msg;
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_REQ_DECODE_FAIL,
+               "Failed to decode probe "
+               "request");
+        req->rpc_err = GARBAGE_ARGS;
+        goto out;
+    }
+
+    conf = this->private;
+    if (probe_req.port)
+        port = probe_req.port;
+    else
+        port = GF_DEFAULT_BASE_PORT;
+
+    gf_msg("glusterd", GF_LOG_INFO, 0, GD_MSG_PROBE_RCVD,
+           "Received probe from uuid: %s", uuid_utoa(probe_req.uuid));
+
+    /* Check for uuid collision and handle it in a user friendly way by
+     * sending the error.
+     */
+    if (!gf_uuid_compare(probe_req.uuid, MY_UUID)) {
+        gf_msg(THIS->name, GF_LOG_ERROR, 0, GD_MSG_UUIDS_SAME_RETRY,
+               "Peer uuid %s is same as "
+               "local uuid. Please check the uuid of both the peers "
+               "from %s/%s",
+               uuid_utoa(probe_req.uuid), GLUSTERD_DEFAULT_WORKDIR,
+               GLUSTERD_INFO_FILE);
+        rsp.op_ret = -1;
+        rsp.op_errno = GF_PROBE_SAME_UUID;
+        rsp.port = port;
+        goto respond;
+    }
+
+    ret = glusterd_remote_hostname_get(req, remote_hostname,
+                                       sizeof(remote_hostname));
+    if (ret) {
+        gf_msg("glusterd", GF_LOG_ERROR, 0, GD_MSG_HOSTNAME_RESOLVE_FAIL,
+               "Unable to get the remote hostname");
+        goto out;
+    }
+
+    RCU_READ_LOCK;
+    peerinfo = glusterd_peerinfo_find(probe_req.uuid, remote_hostname);
+    if ((peerinfo == NULL) && (!cds_list_empty(&conf->peers))) {
+        rsp.op_ret = -1;
+        rsp.op_errno = GF_PROBE_ANOTHER_CLUSTER;
+    } else if (peerinfo == NULL) {
+        gf_msg("glusterd", GF_LOG_INFO, 0, GD_MSG_PEER_NOT_FOUND,
+               "Unable to find peerinfo"
+               " for host: %s (%d)",
+               remote_hostname, port);
+        args.mode = GD_MODE_ON;
+        ret = glusterd_friend_add(remote_hostname, port,
+                                  GD_FRIEND_STATE_PROBE_RCVD, NULL, &peerinfo,
+                                  0, &args);
         if (ret) {
-                gf_log ("", GF_LOG_ERROR, "Unable to get the remote hostname");
-                goto out;
+            gf_msg("glusterd", GF_LOG_ERROR, 0, GD_MSG_PEER_ADD_FAIL,
+                   "Failed to add peer %s", remote_hostname);
+            rsp.op_errno = GF_PROBE_ADD_FAILED;
         }
-
-        rcu_read_lock ();
-        peerinfo = glusterd_peerinfo_find (probe_req.uuid, remote_hostname);
-        if ((peerinfo == NULL) && (!cds_list_empty (&conf->peers))) {
-                rsp.op_ret = -1;
-                rsp.op_errno = GF_PROBE_ANOTHER_CLUSTER;
-        } else if (peerinfo == NULL) {
-                gf_log ("glusterd", GF_LOG_INFO, "Unable to find peerinfo"
-                        " for host: %s (%d)", remote_hostname, port);
-                args.mode = GD_MODE_ON;
-                ret = glusterd_friend_add (remote_hostname, port,
-                                           GD_FRIEND_STATE_PROBE_RCVD,
-                                           NULL, &peerinfo, 0, &args);
-                if (ret) {
-                        gf_log ("", GF_LOG_ERROR, "Failed to add peer %s",
-                                remote_hostname);
-                        rsp.op_errno = GF_PROBE_ADD_FAILED;
-                }
-        }
-        rcu_read_unlock ();
+    }
+    RCU_READ_UNLOCK;
 
 respond:
-        gf_uuid_copy (rsp.uuid, MY_UUID);
+    gf_uuid_copy(rsp.uuid, MY_UUID);
 
-        rsp.hostname = probe_req.hostname;
-        rsp.op_errstr = "";
+    rsp.hostname = probe_req.hostname;
+    rsp.op_errstr = "";
 
-        glusterd_submit_reply (req, &rsp, NULL, 0, NULL,
-                               (xdrproc_t)xdr_gd1_mgmt_probe_rsp);
-        ret = 0;
+    glusterd_submit_reply(req, &rsp, NULL, 0, NULL,
+                          (xdrproc_t)xdr_gd1_mgmt_probe_rsp);
+    ret = 0;
 
-        gf_log ("glusterd", GF_LOG_INFO, "Responded to %s, op_ret: %d, "
-                "op_errno: %d, ret: %d", remote_hostname,
-                rsp.op_ret, rsp.op_errno, ret);
+    gf_msg("glusterd", GF_LOG_INFO, 0, GD_MSG_RESPONSE_INFO,
+           "Responded to %s, op_ret: %d, "
+           "op_errno: %d, ret: %d",
+           remote_hostname, rsp.op_ret, rsp.op_errno, ret);
 
 out:
-        free (probe_req.hostname);//malloced by xdr
+    free(probe_req.hostname);  // malloced by xdr
 
-        glusterd_friend_sm ();
-        glusterd_op_sm ();
+    glusterd_friend_sm();
+    glusterd_op_sm();
 
-        return ret;
+    return ret;
 }
 
-int glusterd_handle_probe_query (rpcsvc_request_t *req)
+int
+glusterd_handle_probe_query(rpcsvc_request_t *req)
 {
-        return glusterd_big_locked_handler (req, __glusterd_handle_probe_query);
+    return glusterd_big_locked_handler(req, __glusterd_handle_probe_query);
 }
 
 int
-__glusterd_handle_cli_profile_volume (rpcsvc_request_t *req)
+__glusterd_handle_cli_profile_volume(rpcsvc_request_t *req)
 {
-        int32_t                         ret     = -1;
-        gf_cli_req                      cli_req = {{0,}};
-        dict_t                          *dict = NULL;
-        glusterd_op_t                   cli_op = GD_OP_PROFILE_VOLUME;
-        char                            *volname = NULL;
-        int32_t                         op = 0;
-        char                            err_str[2048] = {0,};
-        xlator_t                        *this = NULL;
-
-        GF_ASSERT (req);
-        this = THIS;
-        GF_ASSERT (this);
-
-        ret = xdr_to_generic (req->msg[0], &cli_req, (xdrproc_t)xdr_gf_cli_req);
-        if (ret < 0) {
-                //failed to decode msg;
-                req->rpc_err = GARBAGE_ARGS;
-                goto out;
-        }
-
-        if (cli_req.dict.dict_len > 0) {
-                dict = dict_new();
-                if (!dict)
-                        goto out;
-                dict_unserialize (cli_req.dict.dict_val,
-                                  cli_req.dict.dict_len, &dict);
-        }
-
-        ret = dict_get_str (dict, "volname", &volname);
-        if (ret) {
-                snprintf (err_str, sizeof (err_str), "Unable to get volume "
-                          "name");
-                gf_log (this->name, GF_LOG_ERROR, "%s", err_str);
-                goto out;
-        }
-
-        gf_log (this->name, GF_LOG_INFO, "Received volume profile req "
-                "for volume %s", volname);
-        ret = dict_get_int32 (dict, "op", &op);
-        if (ret) {
-                snprintf (err_str, sizeof (err_str), "Unable to get operation");
-                gf_log (this->name, GF_LOG_ERROR, "%s", err_str);
-                goto out;
-        }
-
-        ret = glusterd_op_begin (req, cli_op, dict, err_str, sizeof (err_str));
+    int32_t ret = -1;
+    gf_cli_req cli_req = {{
+        0,
+    }};
+    dict_t *dict = NULL;
+    glusterd_op_t cli_op = GD_OP_PROFILE_VOLUME;
+    char *volname = NULL;
+    int32_t op = 0;
+    char err_str[64] = {
+        0,
+    };
+    xlator_t *this = NULL;
+    glusterd_conf_t *conf = NULL;
+
+    GF_ASSERT(req);
+    this = THIS;
+    GF_ASSERT(this);
+    conf = this->private;
+    GF_VALIDATE_OR_GOTO(this->name, conf, out);
+
+    ret = xdr_to_generic(req->msg[0], &cli_req, (xdrproc_t)xdr_gf_cli_req);
+    if (ret < 0) {
+        // failed to decode msg;
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_REQ_DECODE_FAIL,
+               "Failed to decode "
+               "request received from cli");
+        req->rpc_err = GARBAGE_ARGS;
+        goto out;
+    }
+
+    if (cli_req.dict.dict_len > 0) {
+        dict = dict_new();
+        if (!dict) {
+            gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_CREATE_FAIL,
+                    NULL);
+            goto out;
+        }
+        dict_unserialize(cli_req.dict.dict_val, cli_req.dict.dict_len, &dict);
+    }
+
+    ret = dict_get_strn(dict, "volname", SLEN("volname"), &volname);
+    if (ret) {
+        snprintf(err_str, sizeof(err_str),
+                 "Unable to get volume "
+                 "name");
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_VOLNAME_NOTFOUND_IN_DICT,
+               "%s", err_str);
+        goto out;
+    }
+
+    gf_msg(this->name, GF_LOG_INFO, 0, GD_MSG_VOL_PROFILE_REQ_RCVD,
+           "Received volume profile req "
+           "for volume %s",
+           volname);
+    ret = dict_get_int32n(dict, "op", SLEN("op"), &op);
+    if (ret) {
+        snprintf(err_str, sizeof(err_str), "Unable to get operation");
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED, "%s",
+               err_str);
+        goto out;
+    }
+
+    if (conf->op_version < GD_OP_VERSION_6_0) {
+        gf_msg_debug(this->name, 0,
+                     "The cluster is operating at "
+                     "version less than %d. Falling back "
+                     "to op-sm framework.",
+                     GD_OP_VERSION_6_0);
+        ret = glusterd_op_begin(req, cli_op, dict, err_str, sizeof(err_str));
+        glusterd_friend_sm();
+        glusterd_op_sm();
+    } else {
+        ret = glusterd_mgmt_v3_initiate_all_phases_with_brickop_phase(
+            req, cli_op, dict);
+    }
 
 out:
-        glusterd_friend_sm ();
-        glusterd_op_sm ();
+    free(cli_req.dict.dict_val);
 
-        free (cli_req.dict.dict_val);
+    if (ret) {
+        if (err_str[0] == '\0')
+            snprintf(err_str, sizeof(err_str), "Operation failed");
+        ret = glusterd_op_send_cli_response(cli_op, ret, 0, req, dict, err_str);
+    }
 
-        if (ret) {
-                if (err_str[0] == '\0')
-                        snprintf (err_str, sizeof (err_str),
-                                  "Operation failed");
-                ret = glusterd_op_send_cli_response (cli_op, ret, 0, req,
-                                                     dict, err_str);
-        }
-
-        gf_log (this->name, GF_LOG_DEBUG, "Returning %d", ret);
-        return ret;
+    gf_msg_debug(this->name, 0, "Returning %d", ret);
+    return ret;
 }
 
 int
-glusterd_handle_cli_profile_volume (rpcsvc_request_t *req)
+glusterd_handle_cli_profile_volume(rpcsvc_request_t *req)
 {
-        return glusterd_big_locked_handler (req,
-                                            __glusterd_handle_cli_profile_volume);
+    return glusterd_big_locked_handler(req,
+                                       __glusterd_handle_cli_profile_volume);
 }
 
 int
-__glusterd_handle_getwd (rpcsvc_request_t *req)
+__glusterd_handle_getwd(rpcsvc_request_t *req)
 {
-        int32_t                 ret = -1;
-        gf1_cli_getwd_rsp     rsp = {0,};
-        glusterd_conf_t         *priv = NULL;
+    int32_t ret = -1;
+    gf1_cli_getwd_rsp rsp = {
+        0,
+    };
+    glusterd_conf_t *priv = NULL;
 
-        GF_ASSERT (req);
+    GF_ASSERT(req);
 
-        priv = THIS->private;
-        GF_ASSERT (priv);
+    priv = THIS->private;
+    GF_ASSERT(priv);
 
-        gf_log ("glusterd", GF_LOG_INFO, "Received getwd req");
+    gf_msg("glusterd", GF_LOG_INFO, 0, GD_MSG_GETWD_REQ_RCVD,
+           "Received getwd req");
 
-        rsp.wd = priv->workdir;
+    rsp.wd = priv->workdir;
 
-        glusterd_submit_reply (req, &rsp, NULL, 0, NULL,
-                               (xdrproc_t)xdr_gf1_cli_getwd_rsp);
-        ret = 0;
+    glusterd_submit_reply(req, &rsp, NULL, 0, NULL,
+                          (xdrproc_t)xdr_gf1_cli_getwd_rsp);
+    ret = 0;
 
-        glusterd_friend_sm ();
-        glusterd_op_sm ();
+    glusterd_friend_sm();
+    glusterd_op_sm();
 
-        return ret;
+    return ret;
 }
 
 int
-glusterd_handle_getwd (rpcsvc_request_t *req)
+glusterd_handle_getwd(rpcsvc_request_t *req)
 {
-        return glusterd_big_locked_handler (req, __glusterd_handle_getwd);
+    return glusterd_big_locked_handler(req, __glusterd_handle_getwd);
 }
 
 int
-__glusterd_handle_mount (rpcsvc_request_t *req)
+__glusterd_handle_mount(rpcsvc_request_t *req)
 {
-        gf1_cli_mount_req mnt_req = {0,};
-        gf1_cli_mount_rsp rsp     = {0,};
-        dict_t *dict              = NULL;
-        int ret                   = 0;
-        glusterd_conf_t     *priv   = NULL;
-
-        GF_ASSERT (req);
-	priv = THIS->private;
+    gf1_cli_mount_req mnt_req = {
+        0,
+    };
+    gf1_cli_mount_rsp rsp = {
+        0,
+    };
+    dict_t *dict = NULL;
+    int ret = 0;
+    glusterd_conf_t *priv = NULL;
+
+    GF_ASSERT(req);
+    priv = THIS->private;
+
+    ret = xdr_to_generic(req->msg[0], &mnt_req,
+                         (xdrproc_t)xdr_gf1_cli_mount_req);
+    if (ret < 0) {
+        // failed to decode msg;
+        gf_msg("glusterd", GF_LOG_ERROR, 0, GD_MSG_REQ_DECODE_FAIL,
+               "Failed to decode mount "
+               "request received");
+        req->rpc_err = GARBAGE_ARGS;
+        rsp.op_ret = -1;
+        rsp.op_errno = EINVAL;
+        goto out;
+    }
+
+    gf_msg("glusterd", GF_LOG_INFO, 0, GD_MSG_MOUNT_REQ_RCVD,
+           "Received mount req");
+
+    if (mnt_req.dict.dict_len) {
+        /* Unserialize the dictionary */
+        dict = dict_new();
 
-        ret = xdr_to_generic (req->msg[0], &mnt_req,
-                              (xdrproc_t)xdr_gf1_cli_mount_req);
+        ret = dict_unserialize(mnt_req.dict.dict_val, mnt_req.dict.dict_len,
+                               &dict);
         if (ret < 0) {
-                //failed to decode msg;
-                req->rpc_err = GARBAGE_ARGS;
-                rsp.op_ret = -1;
-                rsp.op_errno = EINVAL;
-                goto out;
+            gf_msg("glusterd", GF_LOG_ERROR, 0, GD_MSG_DICT_UNSERIALIZE_FAIL,
+                   "failed to "
+                   "unserialize req-buffer to dictionary");
+            rsp.op_ret = -1;
+            rsp.op_errno = -EINVAL;
+            goto out;
+        } else {
+            dict->extra_stdfree = mnt_req.dict.dict_val;
         }
+    }
 
-        gf_log ("glusterd", GF_LOG_INFO, "Received mount req");
-
-        if (mnt_req.dict.dict_len) {
-                /* Unserialize the dictionary */
-                dict  = dict_new ();
-
-                ret = dict_unserialize (mnt_req.dict.dict_val,
-                                        mnt_req.dict.dict_len,
-                                        &dict);
-                if (ret < 0) {
-                        gf_log ("glusterd", GF_LOG_ERROR,
-                                "failed to "
-                                "unserialize req-buffer to dictionary");
-                        rsp.op_ret = -1;
-                        rsp.op_errno = -EINVAL;
-                        goto out;
-                } else {
-                        dict->extra_stdfree = mnt_req.dict.dict_val;
-                }
-        }
+    synclock_unlock(&priv->big_lock);
+    rsp.op_ret = glusterd_do_mount(mnt_req.label, dict, &rsp.path,
+                                   &rsp.op_errno);
+    synclock_lock(&priv->big_lock);
 
-	synclock_unlock (&priv->big_lock);
-        rsp.op_ret = glusterd_do_mount (mnt_req.label, dict,
-                                        &rsp.path, &rsp.op_errno);
-	synclock_lock (&priv->big_lock);
+out:
+    if (!rsp.path)
+        rsp.path = gf_strdup("");
 
- out:
-        if (!rsp.path)
-                rsp.path = "";
+    glusterd_submit_reply(req, &rsp, NULL, 0, NULL,
+                          (xdrproc_t)xdr_gf1_cli_mount_rsp);
+    ret = 0;
 
-        glusterd_submit_reply (req, &rsp, NULL, 0, NULL,
-                               (xdrproc_t)xdr_gf1_cli_mount_rsp);
-        ret = 0;
+    if (dict)
+        dict_unref(dict);
 
-        if (dict)
-                dict_unref (dict);
-        if (*rsp.path)
-                GF_FREE (rsp.path);
+    GF_FREE(rsp.path);
 
-        glusterd_friend_sm ();
-        glusterd_op_sm ();
+    glusterd_friend_sm();
+    glusterd_op_sm();
 
-        return ret;
+    return ret;
 }
 
 int
-glusterd_handle_mount (rpcsvc_request_t *req)
+glusterd_handle_mount(rpcsvc_request_t *req)
 {
-        return glusterd_big_locked_handler (req, __glusterd_handle_mount);
+    return glusterd_big_locked_handler(req, __glusterd_handle_mount);
 }
 
 int
-__glusterd_handle_umount (rpcsvc_request_t *req)
+__glusterd_handle_umount(rpcsvc_request_t *req)
 {
-        gf1_cli_umount_req umnt_req = {0,};
-        gf1_cli_umount_rsp rsp      = {0,};
-        char *mountbroker_root      = NULL;
-        char mntp[PATH_MAX]         = {0,};
-        char *path                  = NULL;
-        runner_t runner             = {0,};
-        int ret                     = 0;
-        xlator_t *this              = THIS;
-        gf_boolean_t dir_ok         = _gf_false;
-        char *pdir                  = NULL;
-        char *t                     = NULL;
-        glusterd_conf_t     *priv   = NULL;
-
-        GF_ASSERT (req);
-        GF_ASSERT (this);
-        priv = this->private;
-
-        ret = xdr_to_generic (req->msg[0], &umnt_req,
-                              (xdrproc_t)xdr_gf1_cli_umount_req);
-        if (ret < 0) {
-                //failed to decode msg;
-                req->rpc_err = GARBAGE_ARGS;
-                rsp.op_ret = -1;
-                goto out;
-        }
-
-        gf_log ("glusterd", GF_LOG_INFO, "Received umount req");
-
-        if (dict_get_str (this->options, "mountbroker-root",
-                          &mountbroker_root) != 0) {
-                rsp.op_errno = ENOENT;
-                goto out;
-        }
-
-        /* check if it is allowed to umount path */
-        path = gf_strdup (umnt_req.path);
-        if (!path) {
-                rsp.op_errno = ENOMEM;
-                goto out;
-        }
-        dir_ok = _gf_false;
-        pdir = dirname (path);
-        t = strtail (pdir, mountbroker_root);
-        if (t && *t == '/') {
-                t = strtail(++t, MB_HIVE);
-                if (t && !*t)
-                        dir_ok = _gf_true;
-        }
-        GF_FREE (path);
-        if (!dir_ok) {
-                rsp.op_errno = EACCES;
-                goto out;
-        }
-
-        synclock_unlock (&priv->big_lock);
-
-        if (umnt_req.lazy) {
-                rsp.op_ret = gf_umount_lazy (this->name, umnt_req.path, 0);
-        } else {
-                runinit (&runner);
-                runner_add_args (&runner, _PATH_UMOUNT, umnt_req.path, NULL);
-                rsp.op_ret = runner_run (&runner);
+    gf1_cli_umount_req umnt_req = {
+        0,
+    };
+    gf1_cli_umount_rsp rsp = {
+        0,
+    };
+    char *mountbroker_root = NULL;
+    char mntp[PATH_MAX] = {
+        0,
+    };
+    char *path = NULL;
+    runner_t runner = {
+        0,
+    };
+    int ret = 0;
+    xlator_t *this = THIS;
+    gf_boolean_t dir_ok = _gf_false;
+    char *pdir = NULL;
+    char *t = NULL;
+    glusterd_conf_t *priv = NULL;
+
+    GF_ASSERT(req);
+    GF_ASSERT(this);
+    priv = this->private;
+
+    ret = xdr_to_generic(req->msg[0], &umnt_req,
+                         (xdrproc_t)xdr_gf1_cli_umount_req);
+    if (ret < 0) {
+        // failed to decode msg;
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_REQ_DECODE_FAIL,
+               "Failed to decode umount"
+               "request");
+        req->rpc_err = GARBAGE_ARGS;
+        rsp.op_ret = -1;
+        goto out;
+    }
+
+    gf_msg("glusterd", GF_LOG_INFO, 0, GD_MSG_UMOUNT_REQ_RCVD,
+           "Received umount req");
+
+    if (dict_get_strn(this->options, "mountbroker-root",
+                      SLEN("mountbroker-root"), &mountbroker_root) != 0) {
+        rsp.op_errno = ENOENT;
+        goto out;
+    }
+
+    /* check if it is allowed to umount path */
+    path = gf_strdup(umnt_req.path);
+    if (!path) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_STRDUP_FAILED, NULL);
+        rsp.op_errno = ENOMEM;
+        goto out;
+    }
+    dir_ok = _gf_false;
+    pdir = dirname(path);
+    t = strtail(pdir, mountbroker_root);
+    if (t && *t == '/') {
+        t = strtail(++t, MB_HIVE);
+        if (t && !*t)
+            dir_ok = _gf_true;
+    }
+    GF_FREE(path);
+    if (!dir_ok) {
+        rsp.op_errno = EACCES;
+        goto out;
+    }
+
+    synclock_unlock(&priv->big_lock);
+
+    if (umnt_req.lazy) {
+        rsp.op_ret = gf_umount_lazy(this->name, umnt_req.path, 0);
+    } else {
+        runinit(&runner);
+        runner_add_args(&runner, _PATH_UMOUNT, umnt_req.path, NULL);
+        rsp.op_ret = runner_run(&runner);
+    }
+
+    synclock_lock(&priv->big_lock);
+    if (rsp.op_ret == 0) {
+        if (realpath(umnt_req.path, mntp))
+            sys_rmdir(mntp);
+        else {
+            rsp.op_ret = -1;
+            rsp.op_errno = errno;
         }
-
-        synclock_lock (&priv->big_lock);
-        if (rsp.op_ret == 0) {
-                if (realpath (umnt_req.path, mntp))
-                        rmdir (mntp);
-                else {
-                        rsp.op_ret = -1;
-                        rsp.op_errno = errno;
-                }
-                if (unlink (umnt_req.path) != 0) {
-                        rsp.op_ret = -1;
-                        rsp.op_errno = errno;
-                }
+        if (sys_unlink(umnt_req.path) != 0) {
+            rsp.op_ret = -1;
+            rsp.op_errno = errno;
         }
+    }
 
- out:
-        if (rsp.op_errno)
-                rsp.op_ret = -1;
+out:
+    if (rsp.op_errno)
+        rsp.op_ret = -1;
 
-        glusterd_submit_reply (req, &rsp, NULL, 0, NULL,
-                               (xdrproc_t)xdr_gf1_cli_umount_rsp);
-        ret = 0;
+    glusterd_submit_reply(req, &rsp, NULL, 0, NULL,
+                          (xdrproc_t)xdr_gf1_cli_umount_rsp);
+    ret = 0;
 
-        glusterd_friend_sm ();
-        glusterd_op_sm ();
+    glusterd_friend_sm();
+    glusterd_op_sm();
 
-        return ret;
+    return ret;
 }
 
 int
-glusterd_handle_umount (rpcsvc_request_t *req)
+glusterd_handle_umount(rpcsvc_request_t *req)
 {
-        return glusterd_big_locked_handler (req, __glusterd_handle_umount);
+    return glusterd_big_locked_handler(req, __glusterd_handle_umount);
 }
 
 int
-glusterd_friend_remove (uuid_t uuid, char *hostname)
+glusterd_friend_remove(uuid_t uuid, char *hostname)
 {
-        int                           ret = -1;
-        glusterd_peerinfo_t           *peerinfo = NULL;
-
-        rcu_read_lock ();
-
-        peerinfo = glusterd_peerinfo_find (uuid, hostname);
-        if (peerinfo == NULL) {
-                rcu_read_unlock ();
-                goto out;
-        }
-
-        ret = glusterd_friend_remove_cleanup_vols (peerinfo->uuid);
-        if (ret)
-                gf_log (THIS->name, GF_LOG_WARNING, "Volumes cleanup failed");
-
-        rcu_read_unlock ();
-        /* Giving up the critical section here as glusterd_peerinfo_cleanup must
-         * be called from outside a critical section
-         */
-        ret = glusterd_peerinfo_cleanup (peerinfo);
+    int ret = -1;
+    glusterd_peerinfo_t *peerinfo = NULL;
+
+    RCU_READ_LOCK;
+
+    peerinfo = glusterd_peerinfo_find(uuid, hostname);
+    if (peerinfo == NULL) {
+        RCU_READ_UNLOCK;
+        goto out;
+    }
+
+    ret = glusterd_friend_remove_cleanup_vols(peerinfo->uuid);
+    RCU_READ_UNLOCK;
+    if (ret)
+        gf_msg(THIS->name, GF_LOG_WARNING, 0, GD_MSG_VOL_CLEANUP_FAIL,
+               "Volumes cleanup failed");
+    /* Giving up the critical section here as glusterd_peerinfo_cleanup must
+     * be called from outside a critical section
+     */
+    ret = glusterd_peerinfo_cleanup(peerinfo);
 out:
-        gf_log ("", GF_LOG_DEBUG, "returning %d", ret);
-        return ret;
+    gf_msg_debug(THIS->name, 0, "returning %d", ret);
+    /* coverity[LOCK] */
+    return ret;
 }
 
 int
-glusterd_rpc_create (struct rpc_clnt **rpc,
-                     dict_t *options,
-                     rpc_clnt_notify_t notify_fn,
-                     void *notify_data)
+glusterd_rpc_create(struct rpc_clnt **rpc, dict_t *options,
+                    rpc_clnt_notify_t notify_fn, void *notify_data,
+                    gf_boolean_t force)
 {
-        struct rpc_clnt         *new_rpc = NULL;
-        int                     ret = -1;
-        xlator_t                *this = NULL;
-
-        this = THIS;
-        GF_ASSERT (this);
-
-        GF_ASSERT (options);
-
-        /* TODO: is 32 enough? or more ? */
-        new_rpc = rpc_clnt_new (options, this->ctx, this->name, 16);
-        if (!new_rpc)
-                goto out;
-
-        ret = rpc_clnt_register_notify (new_rpc, notify_fn, notify_data);
-        *rpc = new_rpc;
-        if (ret)
-                goto out;
-        ret = rpc_clnt_start (new_rpc);
+    struct rpc_clnt *new_rpc = NULL;
+    int ret = -1;
+    xlator_t *this = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+
+    GF_ASSERT(options);
+    GF_VALIDATE_OR_GOTO(this->name, rpc, out);
+
+    if (force && rpc && *rpc) {
+        (void)rpc_clnt_unref(*rpc);
+        *rpc = NULL;
+    }
+
+    /* TODO: is 32 enough? or more ? */
+    new_rpc = rpc_clnt_new(options, this, this->name, 16);
+    if (!new_rpc)
+        goto out;
+
+    ret = rpc_clnt_register_notify(new_rpc, notify_fn, notify_data);
+    if (ret)
+        goto out;
+    ret = rpc_clnt_start(new_rpc);
 out:
-        if (ret) {
-                if (new_rpc) {
-                        (void) rpc_clnt_unref (new_rpc);
-                }
+    if (ret) {
+        if (new_rpc) {
+            (void)rpc_clnt_unref(new_rpc);
         }
+    } else {
+        *rpc = new_rpc;
+    }
 
-        gf_log (this->name, GF_LOG_DEBUG, "returning %d", ret);
-        return ret;
+    gf_msg_debug(this->name, 0, "returning %d", ret);
+    return ret;
 }
 
 int
-glusterd_transport_keepalive_options_get (int *interval, int *time,
-                                          int *timeout)
+glusterd_transport_inet_options_build(dict_t *dict, const char *hostname,
+                                      int port, char *af)
 {
-        int     ret = 0;
-        xlator_t *this = NULL;
-
-        this = THIS;
-        GF_ASSERT (this);
-
-        ret = dict_get_int32 (this->options,
-                              "transport.socket.keepalive-interval",
-                              interval);
-        ret = dict_get_int32 (this->options,
-                              "transport.socket.keepalive-time",
-                              time);
-        ret = dict_get_int32 (this->options,
-                              "transport.tcp-user-timeout",
-                              timeout);
-        return 0;
+    xlator_t *this = NULL;
+    int32_t interval = -1;
+    int32_t time = -1;
+    int32_t timeout = -1;
+    int ret = 0;
+
+    this = THIS;
+    GF_ASSERT(this);
+    GF_ASSERT(dict);
+    GF_ASSERT(hostname);
+
+    if (!port)
+        port = GLUSTERD_DEFAULT_PORT;
+
+    /* Build default transport options */
+    ret = rpc_transport_inet_options_build(dict, hostname, port, af);
+    if (ret)
+        goto out;
+
+    /* Set frame-timeout to 10mins. Default timeout of 30 mins is too long
+     * when compared to 2 mins for cli timeout. This ensures users don't
+     * wait too long after cli timesout before being able to resume normal
+     * operations
+     */
+    ret = dict_set_int32n(dict, "frame-timeout", SLEN("frame-timeout"), 600);
+    if (ret) {
+        gf_msg("glusterd", GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+               "Failed to set frame-timeout");
+        goto out;
+    }
+
+    /* Set keepalive options */
+    ret = dict_get_int32n(this->options, "transport.socket.keepalive-interval",
+                          SLEN("transport.socket.keepalive-interval"),
+                          &interval);
+    if (ret) {
+        gf_msg("glusterd", GF_LOG_WARNING, 0, GD_MSG_DICT_GET_FAILED,
+               "Failed to get socket keepalive-interval");
+    }
+    ret = dict_get_int32n(this->options, "transport.socket.keepalive-time",
+                          SLEN("transport.socket.keepalive-time"), &time);
+    if (ret) {
+        gf_msg("glusterd", GF_LOG_WARNING, 0, GD_MSG_DICT_GET_FAILED,
+               "Failed to get socket keepalive-time");
+    }
+    ret = dict_get_int32n(this->options, "transport.tcp-user-timeout",
+                          SLEN("transport.tcp-user-timeout"), &timeout);
+    if (ret) {
+        gf_msg("glusterd", GF_LOG_WARNING, 0, GD_MSG_DICT_GET_FAILED,
+               "Failed to get tcp-user-timeout");
+    }
+
+    if ((interval > 0) || (time > 0))
+        ret = rpc_transport_keepalive_options_set(dict, interval, time,
+                                                  timeout);
+out:
+    gf_msg_debug("glusterd", 0, "Returning %d", ret);
+    return ret;
 }
 
 int
-glusterd_transport_inet_options_build (dict_t **options, const char *hostname,
-                                       int port)
+glusterd_friend_rpc_create(xlator_t *this, glusterd_peerinfo_t *peerinfo,
+                           glusterd_peerctx_args_t *args)
 {
-        dict_t  *dict = NULL;
-        int32_t interval = -1;
-        int32_t time     = -1;
-        int32_t timeout  = -1;
-        int     ret = 0;
-
-        GF_ASSERT (options);
-        GF_ASSERT (hostname);
-
-        if (!port)
-                port = GLUSTERD_DEFAULT_PORT;
+    dict_t *options = NULL;
+    int ret = -1;
+    glusterd_peerctx_t *peerctx = NULL;
+    data_t *data = NULL;
+    char *af = NULL;
+
+    peerctx = GF_CALLOC(1, sizeof(*peerctx), gf_gld_mt_peerctx_t);
+    if (!peerctx) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_NO_MEMORY, NULL);
+        goto out;
+    }
+
+    options = dict_new();
+    if (!options) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_CREATE_FAIL, NULL);
+        goto out;
+    }
+
+    if (args)
+        peerctx->args = *args;
+
+    gf_uuid_copy(peerctx->peerid, peerinfo->uuid);
+    peerctx->peername = gf_strdup(peerinfo->hostname);
+    peerctx->peerinfo_gen = peerinfo->generation; /* A peerinfos generation
+                                                     number can be used to
+                                                     uniquely identify a
+                                                     peerinfo */
+
+    ret = dict_get_str(this->options, "transport.address-family", &af);
+    if (ret)
+        gf_log(this->name, GF_LOG_TRACE,
+               "option transport.address-family is not set in xlator options");
+    ret = glusterd_transport_inet_options_build(options, peerinfo->hostname,
+                                                peerinfo->port, af);
+    if (ret)
+        goto out;
+
+    /*
+     * For simulated multi-node testing, we need to make sure that we
+     * create our RPC endpoint with the same address that the peer would
+     * use to reach us.
+     */
+
+    if (this->options) {
+        data = dict_getn(this->options, "transport.socket.bind-address",
+                         SLEN("transport.socket.bind-address"));
+        if (data) {
+            ret = dict_set_sizen(options, "transport.socket.source-addr", data);
+        }
+        data = dict_getn(this->options, "ping-timeout", SLEN("ping-timeout"));
+        if (data) {
+            ret = dict_set_sizen(options, "ping-timeout", data);
+        }
+    }
+
+    /* Enable encryption for the client connection if management encryption
+     * is enabled
+     */
+    if (this->ctx->secure_mgmt) {
+        ret = dict_set_nstrn(options, "transport.socket.ssl-enabled",
+                             SLEN("transport.socket.ssl-enabled"), "on",
+                             SLEN("on"));
+        if (ret) {
+            gf_msg("glusterd", GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+                   "failed to set ssl-enabled in dict");
+            goto out;
+        }
+
+        this->ctx->ssl_cert_depth = glusterfs_read_secure_access_file();
+    }
+
+    ret = glusterd_rpc_create(&peerinfo->rpc, options, glusterd_peer_rpc_notify,
+                              peerctx, _gf_false);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_RPC_CREATE_FAIL,
+               "failed to create rpc for"
+               " peer %s",
+               peerinfo->hostname);
+        gf_event(EVENT_PEER_RPC_CREATE_FAILED, "peer=%s", peerinfo->hostname);
+        goto out;
+    }
+    peerctx = NULL;
+    ret = 0;
+out:
+    if (options)
+        dict_unref(options);
 
-        /* Build default transport options */
-        ret = rpc_transport_inet_options_build (&dict, hostname, port);
-        if (ret)
-                goto out;
+    GF_FREE(peerctx);
+    return ret;
+}
 
-        /* Set frame-timeout to 10mins. Default timeout of 30 mins is too long
-         * when compared to 2 mins for cli timeout. This ensures users don't
-         * wait too long after cli timesout before being able to resume normal
-         * operations
-         */
-        ret = dict_set_int32 (dict, "frame-timeout", 600);
-        if (ret) {
-                gf_log ("glusterd", GF_LOG_ERROR,
-                        "Failed to set frame-timeout");
-                goto out;
+int
+glusterd_friend_add(const char *hoststr, int port,
+                    glusterd_friend_sm_state_t state, uuid_t *uuid,
+                    glusterd_peerinfo_t **friend, gf_boolean_t restore,
+                    glusterd_peerctx_args_t *args)
+{
+    int ret = 0;
+    xlator_t *this = NULL;
+    glusterd_conf_t *conf = NULL;
+
+    this = THIS;
+    conf = this->private;
+    GF_ASSERT(conf);
+    GF_ASSERT(hoststr);
+    GF_ASSERT(friend);
+
+    *friend = glusterd_peerinfo_new(state, uuid, hoststr, port);
+    if (*friend == NULL) {
+        ret = -1;
+        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_PEER_ADD_FAIL, NULL);
+        goto out;
+    }
+
+    /*
+     * We can't add to the list after calling glusterd_friend_rpc_create,
+     * even if it succeeds, because by then the callback to take it back
+     * off and free might have happened already (notably in the case of an
+     * invalid peer name).  That would mean we're adding something that had
+     * just been free, and we're likely to crash later.
+     */
+    cds_list_add_tail_rcu(&(*friend)->uuid_list, &conf->peers);
+
+    // restore needs to first create the list of peers, then create rpcs
+    // to keep track of quorum in race-free manner. In restore for each peer
+    // rpc-create calls rpc_notify when the friend-list is partially
+    // constructed, leading to wrong quorum calculations.
+    if (!restore) {
+        ret = glusterd_store_peerinfo(*friend);
+        if (ret == 0) {
+            ret = glusterd_friend_rpc_create(this, *friend, args);
+        } else {
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_PEERINFO_CREATE_FAIL,
+                   "Failed to store peerinfo");
+            gf_event(EVENT_PEER_STORE_FAILURE, "peer=%s", (*friend)->hostname);
         }
+    }
 
-        /* Set keepalive options */
-        glusterd_transport_keepalive_options_get (&interval, &time, &timeout);
+    if (ret) {
+        (void)glusterd_peerinfo_cleanup(*friend);
+        *friend = NULL;
+    }
 
-        if ((interval > 0) || (time > 0))
-                ret = rpc_transport_keepalive_options_set (dict, interval,
-                                                           time, timeout);
-        *options = dict;
 out:
-        gf_log ("glusterd", GF_LOG_DEBUG, "Returning %d", ret);
-        return ret;
+    gf_msg(this->name, GF_LOG_INFO, 0, GD_MSG_CONNECT_RETURNED,
+           "connect returned %d", ret);
+    return ret;
 }
 
+/* glusterd_friend_add_from_peerinfo() adds a new peer into the local friends
+ * list from a pre created @peerinfo object. It otherwise works similarly to
+ * glusterd_friend_add()
+ */
 int
-glusterd_friend_rpc_create (xlator_t *this, glusterd_peerinfo_t *peerinfo,
-                            glusterd_peerctx_args_t *args)
+glusterd_friend_add_from_peerinfo(glusterd_peerinfo_t *friend,
+                                  gf_boolean_t restore,
+                                  glusterd_peerctx_args_t *args)
 {
-        dict_t                   *options = NULL;
-        int                       ret     = -1;
-        glusterd_peerctx_t       *peerctx = NULL;
-        data_t                   *data    = NULL;
-
-        peerctx = GF_CALLOC (1, sizeof (*peerctx), gf_gld_mt_peerctx_t);
-        if (!peerctx)
-                goto out;
-
-        if (args)
-                peerctx->args = *args;
-
-        gf_uuid_copy (peerctx->peerid, peerinfo->uuid);
-        peerctx->peername = gf_strdup (peerinfo->hostname);
-
-        ret = glusterd_transport_inet_options_build (&options,
-                                                     peerinfo->hostname,
-                                                     peerinfo->port);
-        if (ret)
-                goto out;
-
-        /*
-         * For simulated multi-node testing, we need to make sure that we
-         * create our RPC endpoint with the same address that the peer would
-         * use to reach us.
-         */
-        if (this->options) {
-                data = dict_get(this->options,"transport.socket.bind-address");
-                if (data) {
-                        ret = dict_set(options,
-                                       "transport.socket.source-addr",data);
-                }
-                data = dict_get(this->options,"ping-timeout");
-                if (data) {
-                        ret = dict_set(options,
-                                       "ping-timeout",data);
-                }
+    int ret = 0;
+    xlator_t *this = NULL;
+    glusterd_conf_t *conf = NULL;
+
+    this = THIS;
+    conf = this->private;
+    GF_ASSERT(conf);
+
+    GF_VALIDATE_OR_GOTO(this->name, (friend != NULL), out);
+
+    /*
+     * We can't add to the list after calling glusterd_friend_rpc_create,
+     * even if it succeeds, because by then the callback to take it back
+     * off and free might have happened already (notably in the case of an
+     * invalid peer name).  That would mean we're adding something that had
+     * just been free, and we're likely to crash later.
+     */
+    cds_list_add_tail_rcu(&friend->uuid_list, &conf->peers);
+
+    // restore needs to first create the list of peers, then create rpcs
+    // to keep track of quorum in race-free manner. In restore for each peer
+    // rpc-create calls rpc_notify when the friend-list is partially
+    // constructed, leading to wrong quorum calculations.
+    if (!restore) {
+        ret = glusterd_store_peerinfo(friend);
+        if (ret == 0) {
+            ret = glusterd_friend_rpc_create(this, friend, args);
+        } else {
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_PEERINFO_CREATE_FAIL,
+                   "Failed to store peerinfo");
+            gf_event(EVENT_PEER_STORE_FAILURE, "peer=%s", friend->hostname);
         }
+    }
 
-        ret = glusterd_rpc_create (&peerinfo->rpc, options,
-                                   glusterd_peer_rpc_notify, peerctx);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "failed to create rpc for"
-                        " peer %s", peerinfo->hostname);
-                goto out;
-        }
-        peerctx = NULL;
-        ret = 0;
 out:
-        GF_FREE (peerctx);
-        return ret;
+    gf_msg(this->name, GF_LOG_INFO, 0, GD_MSG_CONNECT_RETURNED,
+           "connect returned %d", ret);
+    return ret;
 }
 
 int
-glusterd_friend_add (const char *hoststr, int port,
-                     glusterd_friend_sm_state_t state,
-                     uuid_t *uuid,
-                     glusterd_peerinfo_t **friend,
-                     gf_boolean_t restore,
-                     glusterd_peerctx_args_t *args)
+glusterd_probe_begin(rpcsvc_request_t *req, const char *hoststr, int port,
+                     dict_t *dict, int *op_errno)
 {
-        int                     ret = 0;
-        xlator_t               *this = NULL;
-        glusterd_conf_t        *conf = NULL;
+    int ret = -1;
+    glusterd_peerinfo_t *peerinfo = NULL;
+    glusterd_peerctx_args_t args = {0};
+    glusterd_friend_sm_event_t *event = NULL;
 
-        this = THIS;
-        conf = this->private;
-        GF_ASSERT (conf);
-        GF_ASSERT (hoststr);
-        GF_ASSERT (friend);
+    GF_ASSERT(hoststr);
 
-        *friend = glusterd_peerinfo_new (state, uuid, hoststr, port);
-        if (*friend == NULL) {
-                ret = -1;
-                goto out;
+    RCU_READ_LOCK;
+    peerinfo = glusterd_peerinfo_find(NULL, hoststr);
+
+    if (peerinfo == NULL) {
+        gf_msg("glusterd", GF_LOG_INFO, 0, GD_MSG_PEER_NOT_FOUND,
+               "Unable to find peerinfo"
+               " for host: %s (%d)",
+               hoststr, port);
+        args.mode = GD_MODE_ON;
+        args.req = req;
+        args.dict = dict;
+        ret = glusterd_friend_add(hoststr, port, GD_FRIEND_STATE_DEFAULT, NULL,
+                                  &peerinfo, 0, &args);
+        if ((!ret) && (!peerinfo->connected)) {
+            ret = GLUSTERD_CONNECTION_AWAITED;
         }
 
-        /*
-         * We can't add to the list after calling glusterd_friend_rpc_create,
-         * even if it succeeds, because by then the callback to take it back
-         * off and free might have happened already (notably in the case of an
-         * invalid peer name).  That would mean we're adding something that had
-         * just been free, and we're likely to crash later.
-         */
-        cds_list_add_tail_rcu (&(*friend)->uuid_list, &conf->peers);
-
-        //restore needs to first create the list of peers, then create rpcs
-        //to keep track of quorum in race-free manner. In restore for each peer
-        //rpc-create calls rpc_notify when the friend-list is partially
-        //constructed, leading to wrong quorum calculations.
-        if (!restore) {
-                ret = glusterd_store_peerinfo (*friend);
-                if (ret == 0) {
-                        ret = glusterd_friend_rpc_create (this, *friend, args);
-                }
-                else {
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "Failed to store peerinfo");
-                }
+    } else if (peerinfo->connected &&
+               (GD_FRIEND_STATE_BEFRIENDED == peerinfo->state.state)) {
+        if (peerinfo->detaching) {
+            ret = -1;
+            if (op_errno)
+                *op_errno = GF_PROBE_FRIEND_DETACHING;
+            goto out;
         }
+        ret = glusterd_peer_hostname_update(peerinfo, hoststr, _gf_false);
+        if (ret)
+            goto out;
+        // Injecting a NEW_NAME event to update cluster
+        ret = glusterd_friend_sm_new_event(GD_FRIEND_EVENT_NEW_NAME, &event);
+        if (!ret) {
+            event->peername = gf_strdup(peerinfo->hostname);
+            gf_uuid_copy(event->peerid, peerinfo->uuid);
 
-        if (ret) {
-                (void) glusterd_peerinfo_cleanup (*friend);
-                *friend = NULL;
+            ret = glusterd_friend_sm_inject_event(event);
+            glusterd_xfer_cli_probe_resp(req, 0, GF_PROBE_SUCCESS, NULL,
+                                         (char *)hoststr, port, dict);
         }
+    } else {
+        glusterd_xfer_cli_probe_resp(req, 0, GF_PROBE_FRIEND, NULL,
+                                     (char *)hoststr, port, dict);
+        ret = 0;
+    }
 
 out:
-        gf_log (this->name, GF_LOG_INFO, "connect returned %d", ret);
-        return ret;
+    RCU_READ_UNLOCK;
+    gf_msg_debug("glusterd", 0, "returning %d", ret);
+    return ret;
 }
 
-/* glusterd_friend_add_from_peerinfo() adds a new peer into the local friends
- * list from a pre created @peerinfo object. It otherwise works similarly to
- * glusterd_friend_add()
- */
 int
-glusterd_friend_add_from_peerinfo (glusterd_peerinfo_t *friend,
-                                   gf_boolean_t restore,
-                                   glusterd_peerctx_args_t *args)
+glusterd_deprobe_begin(rpcsvc_request_t *req, const char *hoststr, int port,
+                       uuid_t uuid, dict_t *dict, int *op_errno)
 {
-        int                     ret = 0;
-        xlator_t               *this = NULL;
-        glusterd_conf_t        *conf = NULL;
+    int ret = -1;
+    glusterd_peerinfo_t *peerinfo = NULL;
+    glusterd_friend_sm_event_t *event = NULL;
+    glusterd_probe_ctx_t *ctx = NULL;
 
-        this = THIS;
-        conf = this->private;
-        GF_ASSERT (conf);
+    GF_ASSERT(hoststr);
+    GF_ASSERT(req);
 
-        GF_VALIDATE_OR_GOTO (this->name, (friend != NULL), out);
+    RCU_READ_LOCK;
 
-        /*
-         * We can't add to the list after calling glusterd_friend_rpc_create,
-         * even if it succeeds, because by then the callback to take it back
-         * off and free might have happened already (notably in the case of an
-         * invalid peer name).  That would mean we're adding something that had
-         * just been free, and we're likely to crash later.
-         */
-        cds_list_add_tail_rcu (&friend->uuid_list, &conf->peers);
-
-        //restore needs to first create the list of peers, then create rpcs
-        //to keep track of quorum in race-free manner. In restore for each peer
-        //rpc-create calls rpc_notify when the friend-list is partially
-        //constructed, leading to wrong quorum calculations.
-        if (!restore) {
-                ret = glusterd_store_peerinfo (friend);
-                if (ret == 0) {
-                        ret = glusterd_friend_rpc_create (this, friend, args);
-                }
-                else {
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "Failed to store peerinfo");
-                }
-        }
+    peerinfo = glusterd_peerinfo_find(uuid, hoststr);
+    if (peerinfo == NULL) {
+        ret = -1;
+        gf_msg("glusterd", GF_LOG_INFO, 0, GD_MSG_PEER_NOT_FOUND,
+               "Unable to find peerinfo"
+               " for host: %s %d",
+               hoststr, port);
+        goto out;
+    }
+
+    if (!peerinfo->rpc) {
+        // handle this case
+        goto out;
+    }
+
+    if (peerinfo->detaching) {
+        ret = -1;
+        if (op_errno)
+            *op_errno = GF_DEPROBE_FRIEND_DETACHING;
+        goto out;
+    }
 
-out:
-        gf_log (this->name, GF_LOG_INFO, "connect returned %d", ret);
-        return ret;
-}
+    ret = glusterd_friend_sm_new_event(GD_FRIEND_EVENT_INIT_REMOVE_FRIEND,
+                                       &event);
 
-int
-glusterd_probe_begin (rpcsvc_request_t *req, const char *hoststr, int port,
-                      dict_t *dict, int *op_errno)
-{
-        int                             ret = -1;
-        glusterd_peerinfo_t             *peerinfo = NULL;
-        glusterd_peerctx_args_t         args = {0};
-        glusterd_friend_sm_event_t      *event = NULL;
+    if (ret) {
+        gf_msg("glusterd", GF_LOG_ERROR, 0, GD_MSG_EVENT_NEW_GET_FAIL,
+               "Unable to get new event");
+        goto out;
+    }
 
-        GF_ASSERT (hoststr);
+    ctx = GF_CALLOC(1, sizeof(*ctx), gf_gld_mt_probe_ctx_t);
 
-        rcu_read_lock ();
-        peerinfo = glusterd_peerinfo_find (NULL, hoststr);
+    if (!ctx) {
+        goto out;
+    }
 
-        if (peerinfo == NULL) {
-                gf_log ("glusterd", GF_LOG_INFO, "Unable to find peerinfo"
-                        " for host: %s (%d)", hoststr, port);
-                args.mode = GD_MODE_ON;
-                args.req  = req;
-                args.dict = dict;
-                ret = glusterd_friend_add (hoststr, port,
-                                           GD_FRIEND_STATE_DEFAULT,
-                                           NULL, &peerinfo, 0, &args);
-                if ((!ret) && (!peerinfo->connected)) {
-                        ret = GLUSTERD_CONNECTION_AWAITED;
-                }
+    ctx->hostname = gf_strdup(hoststr);
+    ctx->port = port;
+    ctx->req = req;
+    ctx->dict = dict;
 
-        } else if (peerinfo->connected &&
-                   (GD_FRIEND_STATE_BEFRIENDED == peerinfo->state.state)) {
-                if (peerinfo->detaching) {
-                        ret = -1;
-                        if (op_errno)
-                                *op_errno = GF_PROBE_FRIEND_DETACHING;
-                        goto out;
-                }
-                ret = glusterd_peer_hostname_update (peerinfo, hoststr,
-                                                     _gf_false);
-                if (ret)
-                        goto out;
-                //this is just to rename so inject local acc for cluster update
-                ret = glusterd_friend_sm_new_event (GD_FRIEND_EVENT_LOCAL_ACC,
-                                                    &event);
-                if (!ret) {
-                        event->peername = gf_strdup (peerinfo->hostname);
-                        gf_uuid_copy (event->peerid, peerinfo->uuid);
+    event->ctx = ctx;
 
-                        ret = glusterd_friend_sm_inject_event (event);
-                        glusterd_xfer_cli_probe_resp (req, 0, GF_PROBE_SUCCESS,
-                                                      NULL, (char*)hoststr,
-                                                      port, dict);
-                }
-        } else {
-                glusterd_xfer_cli_probe_resp (req, 0, GF_PROBE_FRIEND, NULL,
-                                              (char*)hoststr, port, dict);
-        }
+    event->peername = gf_strdup(hoststr);
+    gf_uuid_copy(event->peerid, uuid);
+
+    ret = glusterd_friend_sm_inject_event(event);
+
+    if (ret) {
+        gf_msg("glusterd", GF_LOG_ERROR, 0, GD_MSG_EVENT_INJECT_FAIL,
+               "Unable to inject event %d, "
+               "ret = %d",
+               event->event, ret);
+        goto out;
+    }
+    peerinfo->detaching = _gf_true;
 
 out:
-        rcu_read_unlock ();
-        gf_log ("", GF_LOG_DEBUG, "returning %d", ret);
-        return ret;
+    RCU_READ_UNLOCK;
+    return ret;
 }
 
 int
-glusterd_deprobe_begin (rpcsvc_request_t *req, const char *hoststr, int port,
-                        uuid_t uuid, dict_t *dict, int *op_errno)
+glusterd_xfer_friend_remove_resp(rpcsvc_request_t *req, char *hostname,
+                                 int port)
 {
-        int                             ret = -1;
-        glusterd_peerinfo_t             *peerinfo = NULL;
-        glusterd_friend_sm_event_t      *event = NULL;
-        glusterd_probe_ctx_t            *ctx = NULL;
-
-        GF_ASSERT (hoststr);
-        GF_ASSERT (req);
+    gd1_mgmt_friend_rsp rsp = {
+        {0},
+    };
+    int32_t ret = -1;
+    xlator_t *this = NULL;
+
+    GF_ASSERT(hostname);
+
+    rsp.op_ret = 0;
+    this = THIS;
+    GF_ASSERT(this);
+
+    gf_uuid_copy(rsp.uuid, MY_UUID);
+    rsp.hostname = hostname;
+    rsp.port = port;
+    ret = glusterd_submit_reply(req, &rsp, NULL, 0, NULL,
+                                (xdrproc_t)xdr_gd1_mgmt_friend_rsp);
+
+    gf_msg("glusterd", GF_LOG_INFO, 0, GD_MSG_RESPONSE_INFO,
+           "Responded to %s (%d), ret: %d", hostname, port, ret);
+    return ret;
+}
 
-        rcu_read_lock ();
+int
+glusterd_xfer_friend_add_resp(rpcsvc_request_t *req, char *myhostname,
+                              char *remote_hostname, int port, int32_t op_ret,
+                              int32_t op_errno)
+{
+    gd1_mgmt_friend_rsp rsp = {
+        {0},
+    };
+    int32_t ret = -1;
+    xlator_t *this = NULL;
+
+    GF_ASSERT(myhostname);
+
+    this = THIS;
+    GF_ASSERT(this);
+
+    gf_uuid_copy(rsp.uuid, MY_UUID);
+    rsp.op_ret = op_ret;
+    rsp.op_errno = op_errno;
+    rsp.hostname = gf_strdup(myhostname);
+    rsp.port = port;
+
+    ret = glusterd_submit_reply(req, &rsp, NULL, 0, NULL,
+                                (xdrproc_t)xdr_gd1_mgmt_friend_rsp);
+
+    gf_msg("glusterd", GF_LOG_INFO, 0, GD_MSG_RESPONSE_INFO,
+           "Responded to %s (%d), ret: %d, op_ret: %d", remote_hostname, port,
+           ret, op_ret);
+    GF_FREE(rsp.hostname);
+    return ret;
+}
 
-        peerinfo = glusterd_peerinfo_find (uuid, hoststr);
-        if (peerinfo == NULL) {
-                ret = -1;
-                gf_log ("glusterd", GF_LOG_INFO, "Unable to find peerinfo"
-                        " for host: %s %d", hoststr, port);
-                goto out;
-        }
+static void
+set_probe_error_str(int op_ret, int op_errno, char *op_errstr, char *errstr,
+                    size_t len, char *hostname, int port)
+{
+    if ((op_errstr) && (strcmp(op_errstr, ""))) {
+        snprintf(errstr, len, "%s", op_errstr);
+        return;
+    }
+
+    if (!op_ret) {
+        switch (op_errno) {
+            case GF_PROBE_LOCALHOST:
+                snprintf(errstr, len,
+                         "Probe on localhost not "
+                         "needed");
+                break;
 
-        if (!peerinfo->rpc) {
-                //handle this case
-                goto out;
-        }
+            case GF_PROBE_FRIEND:
+                snprintf(errstr, len,
+                         "Host %s port %d already"
+                         " in peer list",
+                         hostname, port);
+                break;
 
-        if (peerinfo->detaching) {
-                ret = -1;
-                if (op_errno)
-                        *op_errno = GF_DEPROBE_FRIEND_DETACHING;
-                goto out;
+            case GF_PROBE_FRIEND_DETACHING:
+                snprintf(errstr, len,
+                         "Peer is already being "
+                         "detached from cluster.\n"
+                         "Check peer status by running "
+                         "gluster peer status");
+                break;
+            default:
+                if (op_errno != 0)
+                    snprintf(errstr, len,
+                             "Probe returned "
+                             "with %s",
+                             strerror(op_errno));
+                break;
         }
+    } else {
+        switch (op_errno) {
+            case GF_PROBE_ANOTHER_CLUSTER:
+                snprintf(errstr, len,
+                         "%s is either already "
+                         "part of another cluster or having "
+                         "volumes configured",
+                         hostname);
+                break;
 
-        ret = glusterd_friend_sm_new_event
-                (GD_FRIEND_EVENT_INIT_REMOVE_FRIEND, &event);
-
-        if (ret) {
-                gf_log ("glusterd", GF_LOG_ERROR,
-                                "Unable to get new event");
-                goto out;
-        }
+            case GF_PROBE_VOLUME_CONFLICT:
+                snprintf(errstr, len,
+                         "At least one volume on "
+                         "%s conflicts with existing volumes "
+                         "in the cluster",
+                         hostname);
+                break;
 
-        ctx = GF_CALLOC (1, sizeof(*ctx), gf_gld_mt_probe_ctx_t);
+            case GF_PROBE_UNKNOWN_PEER:
+                snprintf(errstr, len,
+                         "%s responded with "
+                         "'unknown peer' error, this could "
+                         "happen if %s doesn't have localhost "
+                         "in its peer database",
+                         hostname, hostname);
+                break;
 
-        if (!ctx) {
-                goto out;
-        }
+            case GF_PROBE_ADD_FAILED:
+                snprintf(errstr, len,
+                         "Failed to add peer "
+                         "information on %s",
+                         hostname);
+                break;
 
-        ctx->hostname = gf_strdup (hoststr);
-        ctx->port = port;
-        ctx->req = req;
-        ctx->dict = dict;
+            case GF_PROBE_SAME_UUID:
+                snprintf(errstr, len,
+                         "Peer uuid (host %s) is "
+                         "same as local uuid",
+                         hostname);
+                break;
 
-        event->ctx = ctx;
+            case GF_PROBE_QUORUM_NOT_MET:
+                snprintf(errstr, len,
+                         "Cluster quorum is not "
+                         "met. Changing peers is not allowed "
+                         "in this state");
+                break;
 
-        event->peername = gf_strdup (hoststr);
-        gf_uuid_copy (event->peerid, uuid);
+            case GF_PROBE_MISSED_SNAP_CONFLICT:
+                snprintf(errstr, len,
+                         "Failed to update "
+                         "list of missed snapshots from "
+                         "peer %s",
+                         hostname);
+                break;
 
-        ret = glusterd_friend_sm_inject_event (event);
+            case GF_PROBE_SNAP_CONFLICT:
+                snprintf(errstr, len,
+                         "Conflict in comparing "
+                         "list of snapshots from "
+                         "peer %s",
+                         hostname);
+                break;
 
-        if (ret) {
-                gf_log ("glusterd", GF_LOG_ERROR, "Unable to inject event %d, "
-                        "ret = %d", event->event, ret);
-                goto out;
+            default:
+                snprintf(errstr, len,
+                         "Probe returned with "
+                         "%s",
+                         strerror(op_errno));
+                break;
         }
-        peerinfo->detaching = _gf_true;
-
-out:
-        rcu_read_unlock ();
-        return ret;
+    }
 }
 
-
 int
-glusterd_xfer_friend_remove_resp (rpcsvc_request_t *req, char *hostname, int port)
+glusterd_xfer_cli_probe_resp(rpcsvc_request_t *req, int32_t op_ret,
+                             int32_t op_errno, char *op_errstr, char *hostname,
+                             int port, dict_t *dict)
 {
-        gd1_mgmt_friend_rsp  rsp = {{0}, };
-        int32_t              ret = -1;
-        xlator_t             *this = NULL;
-        glusterd_conf_t      *conf = NULL;
+    gf_cli_rsp rsp = {
+        0,
+    };
+    int32_t ret = -1;
+    char errstr[2048] = {
+        0,
+    };
+    char *cmd_str = NULL;
+    xlator_t *this = THIS;
+
+    GF_ASSERT(req);
+    GF_ASSERT(this);
+
+    (void)set_probe_error_str(op_ret, op_errno, op_errstr, errstr,
+                              sizeof(errstr), hostname, port);
+
+    if (dict) {
+        ret = dict_get_strn(dict, "cmd-str", SLEN("cmd-str"), &cmd_str);
+        if (ret)
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_CMDSTR_NOTFOUND_IN_DICT,
+                   "Failed to get "
+                   "command string");
+    }
 
-        GF_ASSERT (hostname);
+    rsp.op_ret = op_ret;
+    rsp.op_errno = op_errno;
+    rsp.op_errstr = (errstr[0] != '\0') ? errstr : "";
 
-        rsp.op_ret = 0;
-        this = THIS;
-        GF_ASSERT (this);
+    gf_cmd_log("", "%s : %s %s %s", cmd_str, (op_ret) ? "FAILED" : "SUCCESS",
+               (errstr[0] != '\0') ? ":" : " ",
+               (errstr[0] != '\0') ? errstr : " ");
 
-        conf = this->private;
+    ret = glusterd_submit_reply(req, &rsp, NULL, 0, NULL,
+                                (xdrproc_t)xdr_gf_cli_rsp);
 
-        gf_uuid_copy (rsp.uuid, MY_UUID);
-        rsp.hostname = hostname;
-        rsp.port = port;
-        ret = glusterd_submit_reply (req, &rsp, NULL, 0, NULL,
-                                     (xdrproc_t)xdr_gd1_mgmt_friend_rsp);
+    if (dict)
+        dict_unref(dict);
+    gf_msg_debug(this->name, 0, "Responded to CLI, ret: %d", ret);
 
-        gf_log ("glusterd", GF_LOG_INFO,
-                "Responded to %s (%d), ret: %d", hostname, port, ret);
-        return ret;
+    return ret;
 }
 
-
-int
-glusterd_xfer_friend_add_resp (rpcsvc_request_t *req, char *myhostname,
-                               char *remote_hostname, int port, int32_t op_ret,
-                               int32_t op_errno)
+static void
+set_deprobe_error_str(int op_ret, int op_errno, char *op_errstr, char *errstr,
+                      size_t len, char *hostname)
 {
-        gd1_mgmt_friend_rsp  rsp = {{0}, };
-        int32_t              ret = -1;
-        xlator_t             *this = NULL;
-        glusterd_conf_t      *conf = NULL;
-
-        GF_ASSERT (myhostname);
-
-        this = THIS;
-        GF_ASSERT (this);
+    if ((op_errstr) && (strcmp(op_errstr, ""))) {
+        snprintf(errstr, len, "%s", op_errstr);
+        return;
+    }
+
+    if (op_ret) {
+        switch (op_errno) {
+            case GF_DEPROBE_LOCALHOST:
+                snprintf(errstr, len, "%s is localhost", hostname);
+                break;
 
-        conf = this->private;
+            case GF_DEPROBE_NOT_FRIEND:
+                snprintf(errstr, len,
+                         "%s is not part of "
+                         "cluster",
+                         hostname);
+                break;
 
-        gf_uuid_copy (rsp.uuid, MY_UUID);
-        rsp.op_ret = op_ret;
-        rsp.op_errno = op_errno;
-        rsp.hostname = gf_strdup (myhostname);
-        rsp.port = port;
+            case GF_DEPROBE_BRICK_EXIST:
+                snprintf(errstr, len,
+                         "Peer %s hosts one or more bricks. If the peer is in "
+                         "not recoverable state then use either replace-brick "
+                         "or remove-brick command with force to remove all "
+                         "bricks from the peer and attempt the peer detach "
+                         "again.",
+                         hostname);
+                break;
 
-        ret = glusterd_submit_reply (req, &rsp, NULL, 0, NULL,
-                                     (xdrproc_t)xdr_gd1_mgmt_friend_rsp);
+            case GF_DEPROBE_SNAP_BRICK_EXIST:
+                snprintf(errstr, len,
+                         "%s is part of existing "
+                         "snapshot. Remove those snapshots "
+                         "before proceeding ",
+                         hostname);
+                break;
 
-        gf_log ("glusterd", GF_LOG_INFO,
-                "Responded to %s (%d), ret: %d", remote_hostname, port, ret);
-        GF_FREE (rsp.hostname);
-        return ret;
-}
+            case GF_DEPROBE_FRIEND_DOWN:
+                snprintf(errstr, len,
+                         "One of the peers is "
+                         "probably down. Check with "
+                         "'peer status'");
+                break;
 
-static void
-set_probe_error_str (int op_ret, int op_errno, char *op_errstr, char *errstr,
-                     size_t len, char *hostname, int port)
-{
-        if ((op_errstr) && (strcmp (op_errstr, ""))) {
-                snprintf (errstr, len, "%s", op_errstr);
-                return;
-        }
+            case GF_DEPROBE_QUORUM_NOT_MET:
+                snprintf(errstr, len,
+                         "Cluster quorum is not "
+                         "met. Changing peers is not allowed "
+                         "in this state");
+                break;
 
-        if (!op_ret) {
-                switch (op_errno) {
-                        case GF_PROBE_LOCALHOST:
-                                snprintf (errstr, len, "Probe on localhost not "
-                                          "needed");
-                                break;
-
-                        case GF_PROBE_FRIEND:
-                                snprintf (errstr, len, "Host %s port %d already"
-                                          " in peer list", hostname, port);
-                                break;
-
-                        case GF_PROBE_FRIEND_DETACHING:
-                                snprintf (errstr, len, "Peer is already being "
-                                          "detached from cluster.\n"
-                                          "Check peer status by running "
-                                          "gluster peer status");
-                                break;
-                        default:
-                                if (op_errno != 0)
-                                        snprintf (errstr, len, "Probe returned "
-                                                  "with unknown errno %d",
-                                                  op_errno);
-                                break;
-                }
-        } else {
-                switch (op_errno) {
-                        case GF_PROBE_ANOTHER_CLUSTER:
-                                snprintf (errstr, len, "%s is already part of "
-                                          "another cluster", hostname);
-                                break;
-
-                        case GF_PROBE_VOLUME_CONFLICT:
-                                snprintf (errstr, len, "Atleast one volume on "
-                                          "%s conflicts with existing volumes "
-                                          "in the cluster", hostname);
-                                break;
-
-                        case GF_PROBE_UNKNOWN_PEER:
-                                snprintf (errstr, len, "%s responded with "
-                                          "'unknown peer' error, this could "
-                                          "happen if %s doesn't have localhost "
-                                          "in its peer database", hostname,
-                                          hostname);
-                                break;
-
-                        case GF_PROBE_ADD_FAILED:
-                                snprintf (errstr, len, "Failed to add peer "
-                                          "information on %s", hostname);
-                                break;
-
-                        case GF_PROBE_SAME_UUID:
-                                snprintf (errstr, len, "Peer uuid (host %s) is "
-                                          "same as local uuid", hostname);
-                                break;
-
-                        case GF_PROBE_QUORUM_NOT_MET:
-                                snprintf (errstr, len, "Cluster quorum is not "
-                                          "met. Changing peers is not allowed "
-                                          "in this state");
-                                break;
-
-                        case GF_PROBE_MISSED_SNAP_CONFLICT:
-                                snprintf (errstr, len, "Failed to update "
-                                          "list of missed snapshots from "
-                                          "peer %s", hostname);
-                                break;
-
-                        case GF_PROBE_SNAP_CONFLICT:
-                                snprintf (errstr, len, "Conflict in comparing "
-                                          "list of snapshots from "
-                                          "peer %s", hostname);
-                                break;
-
-                        default:
-                                snprintf (errstr, len, "Probe returned with "
-                                          "unknown errno %d", op_errno);
-                                break;
-                }
+            case GF_DEPROBE_FRIEND_DETACHING:
+                snprintf(errstr, len,
+                         "Peer is already being "
+                         "detached from cluster.\n"
+                         "Check peer status by running "
+                         "gluster peer status");
+                break;
+            default:
+                snprintf(errstr, len,
+                         "Detach returned with "
+                         "%s",
+                         strerror(op_errno));
+                break;
         }
+    }
 }
 
 int
-glusterd_xfer_cli_probe_resp (rpcsvc_request_t *req, int32_t op_ret,
-                              int32_t op_errno, char *op_errstr, char *hostname,
-                              int port, dict_t *dict)
+glusterd_xfer_cli_deprobe_resp(rpcsvc_request_t *req, int32_t op_ret,
+                               int32_t op_errno, char *op_errstr,
+                               char *hostname, dict_t *dict)
 {
-        gf_cli_rsp           rsp = {0,};
-        int32_t              ret = -1;
-        char        errstr[2048] = {0,};
-        char            *cmd_str = NULL;
-        xlator_t           *this = THIS;
-
-        GF_ASSERT (req);
-        GF_ASSERT (this);
-
-        (void) set_probe_error_str (op_ret, op_errno, op_errstr, errstr,
-                                    sizeof (errstr), hostname, port);
-
-        if (dict) {
-                ret = dict_get_str (dict, "cmd-str", &cmd_str);
-                if (ret)
-                        gf_log (this->name, GF_LOG_ERROR, "Failed to get "
-                                "command string");
-        }
+    gf_cli_rsp rsp = {
+        0,
+    };
+    int32_t ret = -1;
+    char *cmd_str = NULL;
+    char errstr[2048] = {
+        0,
+    };
+
+    GF_ASSERT(req);
+
+    (void)set_deprobe_error_str(op_ret, op_errno, op_errstr, errstr,
+                                sizeof(errstr), hostname);
+
+    if (dict) {
+        ret = dict_get_strn(dict, "cmd-str", SLEN("cmd-str"), &cmd_str);
+        if (ret)
+            gf_msg(THIS->name, GF_LOG_ERROR, 0, GD_MSG_CMDSTR_NOTFOUND_IN_DICT,
+                   "Failed to get "
+                   "command string");
+    }
 
-        rsp.op_ret = op_ret;
-        rsp.op_errno = op_errno;
-        rsp.op_errstr = (errstr[0] != '\0') ? errstr : "";
+    rsp.op_ret = op_ret;
+    rsp.op_errno = op_errno;
+    rsp.op_errstr = (errstr[0] != '\0') ? errstr : "";
 
-        gf_cmd_log ("", "%s : %s %s %s", cmd_str,
-                    (op_ret) ? "FAILED" : "SUCCESS",
-                    (errstr[0] != '\0') ? ":" : " ",
-                    (errstr[0] != '\0') ? errstr : " ");
+    gf_cmd_log("", "%s : %s %s %s", cmd_str, (op_ret) ? "FAILED" : "SUCCESS",
+               (errstr[0] != '\0') ? ":" : " ",
+               (errstr[0] != '\0') ? errstr : " ");
 
-        ret = glusterd_submit_reply (req, &rsp, NULL, 0, NULL,
-                                     (xdrproc_t)xdr_gf_cli_rsp);
+    ret = glusterd_submit_reply(req, &rsp, NULL, 0, NULL,
+                                (xdrproc_t)xdr_gf_cli_rsp);
 
-        if (dict)
-                dict_unref (dict);
-        gf_log (this->name, GF_LOG_DEBUG, "Responded to CLI, ret: %d",ret);
+    gf_msg_debug(THIS->name, 0, "Responded to CLI, ret: %d", ret);
 
-        return ret;
+    return ret;
 }
 
-static void
-set_deprobe_error_str (int op_ret, int op_errno, char *op_errstr, char *errstr,
-                       size_t len, char *hostname)
+int32_t
+glusterd_list_friends(rpcsvc_request_t *req, dict_t *dict, int32_t flags)
 {
-        if ((op_errstr) && (strcmp (op_errstr, ""))) {
-                snprintf (errstr, len, "%s", op_errstr);
-                return;
+    int32_t ret = -1;
+    glusterd_conf_t *priv = NULL;
+    glusterd_peerinfo_t *entry = NULL;
+    int32_t count = 0;
+    dict_t *friends = NULL;
+    gf1_cli_peer_list_rsp rsp = {
+        0,
+    };
+    char my_uuid_str[64] = {
+        0,
+    };
+    char key[64] = {
+        0,
+    };
+    int keylen;
+
+    xlator_t *this = THIS;
+    GF_ASSERT(this);
+
+    priv = this->private;
+    GF_ASSERT(priv);
+
+    friends = dict_new();
+    if (!friends) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_CREATE_FAIL, NULL);
+        goto out;
+    }
+
+    /* Reset ret to 0, needed to prevent failure in case no peers exist */
+    ret = 0;
+    RCU_READ_LOCK;
+    if (!cds_list_empty(&priv->peers)) {
+        cds_list_for_each_entry_rcu(entry, &priv->peers, uuid_list)
+        {
+            count++;
+            ret = gd_add_peer_detail_to_dict(entry, friends, count);
+            if (ret)
+                goto unlock;
         }
-
-        if (op_ret) {
-                switch (op_errno) {
-                        case GF_DEPROBE_LOCALHOST:
-                                snprintf (errstr, len, "%s is localhost",
-                                hostname);
-                                break;
-
-                        case GF_DEPROBE_NOT_FRIEND:
-                                snprintf (errstr, len, "%s is not part of "
-                                          "cluster", hostname);
-                                break;
-
-                        case GF_DEPROBE_BRICK_EXIST:
-                                snprintf (errstr, len, "Brick(s) with the peer "
-                                          "%s exist in cluster", hostname);
-                                break;
-
-                        case GF_DEPROBE_FRIEND_DOWN:
-                                snprintf (errstr, len, "One of the peers is "
-                                          "probably down. Check with "
-                                          "'peer status'");
-                                break;
-
-                        case GF_DEPROBE_QUORUM_NOT_MET:
-                                snprintf (errstr, len, "Cluster quorum is not "
-                                          "met. Changing peers is not allowed "
-                                          "in this state");
-                                break;
-
-                        case GF_DEPROBE_FRIEND_DETACHING:
-                                snprintf (errstr, len, "Peer is already being "
-                                          "detached from cluster.\n"
-                                          "Check peer status by running "
-                                          "gluster peer status");
-                                break;
-                        default:
-                                snprintf (errstr, len, "Detach returned with "
-                                          "unknown errno %d", op_errno);
-                                break;
-
-                }
+    }
+unlock:
+    RCU_READ_UNLOCK;
+    if (ret)
+        goto out;
+
+    if (flags == GF_CLI_LIST_POOL_NODES) {
+        count++;
+        keylen = snprintf(key, sizeof(key), "friend%d.uuid", count);
+        uuid_utoa_r(MY_UUID, my_uuid_str);
+        ret = dict_set_strn(friends, key, keylen, my_uuid_str);
+        if (ret) {
+            gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_SET_FAILED,
+                    "Key=%s", key, NULL);
+            goto out;
         }
-}
 
+        keylen = snprintf(key, sizeof(key), "friend%d.hostname", count);
+        ret = dict_set_nstrn(friends, key, keylen, "localhost",
+                             SLEN("localhost"));
+        if (ret) {
+            gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_SET_FAILED,
+                    "Key=%s", key, NULL);
+            goto out;
+        }
 
-int
-glusterd_xfer_cli_deprobe_resp (rpcsvc_request_t *req, int32_t op_ret,
-                                int32_t op_errno, char *op_errstr,
-                                char *hostname, dict_t *dict)
-{
-        gf_cli_rsp             rsp = {0,};
-        int32_t                ret = -1;
-        char              *cmd_str = NULL;
-        char          errstr[2048] = {0,};
+        keylen = snprintf(key, sizeof(key), "friend%d.connected", count);
+        ret = dict_set_int32n(friends, key, keylen, 1);
+        if (ret) {
+            gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_SET_FAILED,
+                    "Key=%s", key, NULL);
+            goto out;
+        }
+    }
 
-        GF_ASSERT (req);
+    ret = dict_set_int32n(friends, "count", SLEN("count"), count);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_SET_FAILED,
+                "Key=count", NULL);
+        goto out;
+    }
 
-        (void) set_deprobe_error_str (op_ret, op_errno, op_errstr, errstr,
-                                      sizeof (errstr), hostname);
+    ret = dict_allocate_and_serialize(friends, &rsp.friends.friends_val,
+                                      &rsp.friends.friends_len);
 
-        if (dict) {
-                ret = dict_get_str (dict, "cmd-str", &cmd_str);
-                if (ret)
-                        gf_log (THIS->name, GF_LOG_ERROR, "Failed to get "
-                                "command string");
-        }
+    if (ret)
+        goto out;
 
-        rsp.op_ret = op_ret;
-        rsp.op_errno = op_errno;
-        rsp.op_errstr = (errstr[0] != '\0') ? errstr : "";
+    ret = 0;
+out:
 
-        gf_cmd_log ("", "%s : %s %s %s", cmd_str,
-                    (op_ret) ? "FAILED" : "SUCCESS",
-                    (errstr[0] != '\0') ? ":" : " ",
-                    (errstr[0] != '\0') ? errstr : " ");
+    if (friends)
+        dict_unref(friends);
 
-        ret = glusterd_submit_reply (req, &rsp, NULL, 0, NULL,
-                                     (xdrproc_t)xdr_gf_cli_rsp);
+    rsp.op_ret = ret;
 
-        gf_log (THIS->name, GF_LOG_DEBUG, "Responded to CLI, ret: %d",ret);
+    glusterd_submit_reply(req, &rsp, NULL, 0, NULL,
+                          (xdrproc_t)xdr_gf1_cli_peer_list_rsp);
+    ret = 0;
+    GF_FREE(rsp.friends.friends_val);
 
-        return ret;
+    return ret;
 }
 
 int32_t
-glusterd_list_friends (rpcsvc_request_t *req, dict_t *dict, int32_t flags)
+glusterd_get_volumes(rpcsvc_request_t *req, dict_t *dict, int32_t flags)
 {
-        int32_t                 ret = -1;
-        glusterd_conf_t         *priv = NULL;
-        glusterd_peerinfo_t     *entry = NULL;
-        int32_t                 count = 0;
-        dict_t                  *friends = NULL;
-        gf1_cli_peer_list_rsp   rsp = {0,};
-        char                    my_uuid_str[64] = {0,};
-        char                    key[256] = {0,};
-
-        priv = THIS->private;
-        GF_ASSERT (priv);
-
-        friends = dict_new ();
-        if (!friends) {
-                gf_log ("", GF_LOG_WARNING, "Out of Memory");
-                goto out;
-        }
-
-        /* Reset ret to 0, needed to prevent failure incase no peers exist */
+    int32_t ret = -1;
+    int32_t ret_bkp = 0;
+    glusterd_conf_t *priv = NULL;
+    glusterd_volinfo_t *entry = NULL;
+    int32_t count = 0;
+    dict_t *volumes = NULL;
+    gf_cli_rsp rsp = {
+        0,
+    };
+    char *volname = NULL;
+
+    priv = THIS->private;
+    GF_ASSERT(priv);
+    volumes = dict_new();
+    if (!volumes) {
+        gf_msg("glusterd", GF_LOG_ERROR, ENOMEM, GD_MSG_NO_MEMORY,
+               "Out of Memory");
+        goto out;
+    }
+
+    if (cds_list_empty(&priv->volumes)) {
+        if (flags == GF_CLI_GET_VOLUME)
+            ret_bkp = -1;
         ret = 0;
-        rcu_read_lock ();
-        if (!cds_list_empty (&priv->peers)) {
-                cds_list_for_each_entry_rcu (entry, &priv->peers, uuid_list) {
-                        count++;
-                        ret = gd_add_peer_detail_to_dict (entry,
-                                                                friends, count);
-                        if (ret)
-                                goto unlock;
-                }
+        goto respond;
+    }
+    if (flags == GF_CLI_GET_VOLUME_ALL) {
+        cds_list_for_each_entry(entry, &priv->volumes, vol_list)
+        {
+            ret = glusterd_add_volume_detail_to_dict(entry, volumes, count);
+            if (ret)
+                goto respond;
+
+            count++;
         }
-unlock:
-        rcu_read_unlock ();
-        if (ret)
-                goto out;
 
-        if (flags == GF_CLI_LIST_POOL_NODES) {
-                count++;
-                snprintf (key, 256, "friend%d.uuid", count);
-                uuid_utoa_r (MY_UUID, my_uuid_str);
-                ret = dict_set_str (friends, key, my_uuid_str);
-                if (ret)
-                        goto out;
+    } else if (flags == GF_CLI_GET_NEXT_VOLUME) {
+        ret = dict_get_strn(dict, "volname", SLEN("volname"), &volname);
 
-                snprintf (key, 256, "friend%d.hostname", count);
-                ret = dict_set_str (friends, key, "localhost");
-                if (ret)
-                        goto out;
+        if (ret) {
+            if (priv->volumes.next) {
+                entry = cds_list_entry(priv->volumes.next, typeof(*entry),
+                                       vol_list);
+            }
+        } else {
+            ret = glusterd_volinfo_find(volname, &entry);
+            if (ret)
+                goto respond;
+            entry = cds_list_entry(entry->vol_list.next, typeof(*entry),
+                                   vol_list);
+        }
 
-                snprintf (key, 256, "friend%d.connected", count);
-                ret = dict_set_int32 (friends, key, 1);
-                if (ret)
-                        goto out;
+        if (&entry->vol_list == &priv->volumes) {
+            goto respond;
+        } else {
+            ret = glusterd_add_volume_detail_to_dict(entry, volumes, count);
+            if (ret)
+                goto respond;
+
+            count++;
         }
+    } else if (flags == GF_CLI_GET_VOLUME) {
+        ret = dict_get_strn(dict, "volname", SLEN("volname"), &volname);
 
-        ret = dict_set_int32 (friends, "count", count);
         if (ret)
-                goto out;
+            goto respond;
 
-        ret = dict_allocate_and_serialize (friends, &rsp.friends.friends_val,
-                                           &rsp.friends.friends_len);
+        ret = glusterd_volinfo_find(volname, &entry);
+        if (ret) {
+            ret_bkp = ret;
+            goto respond;
+        }
 
+        ret = glusterd_add_volume_detail_to_dict(entry, volumes, count);
         if (ret)
-                goto out;
+            goto respond;
 
-        ret = 0;
-out:
+        count++;
+    }
 
-        if (friends)
-                dict_unref (friends);
+respond:
+    ret = dict_set_int32n(volumes, "count", SLEN("count"), count);
+    if (ret)
+        goto out;
+    ret = dict_allocate_and_serialize(volumes, &rsp.dict.dict_val,
+                                      &rsp.dict.dict_len);
+
+    if (ret)
+        goto out;
 
+    ret = 0;
+out:
+    if (ret_bkp == -1) {
+        rsp.op_ret = ret_bkp;
+        rsp.op_errstr = "Volume does not exist";
+        rsp.op_errno = EG_NOVOL;
+    } else {
         rsp.op_ret = ret;
+        rsp.op_errstr = "";
+    }
+    glusterd_submit_reply(req, &rsp, NULL, 0, NULL, (xdrproc_t)xdr_gf_cli_rsp);
+    ret = 0;
 
-        glusterd_submit_reply (req, &rsp, NULL, 0, NULL,
-                               (xdrproc_t)xdr_gf1_cli_peer_list_rsp);
-        ret = 0;
-        GF_FREE (rsp.friends.friends_val);
+    if (volumes)
+        dict_unref(volumes);
 
-        return ret;
+    GF_FREE(rsp.dict.dict_val);
+    return ret;
 }
 
-int32_t
-glusterd_get_volumes (rpcsvc_request_t *req, dict_t *dict, int32_t flags)
+int
+__glusterd_handle_status_volume(rpcsvc_request_t *req)
 {
-        int32_t                 ret = -1;
-        glusterd_conf_t         *priv = NULL;
-        glusterd_volinfo_t      *entry = NULL;
-        int32_t                 count = 0;
-        dict_t                  *volumes = NULL;
-        gf_cli_rsp              rsp = {0,};
-        char                    *volname = NULL;
-
-        priv = THIS->private;
-        GF_ASSERT (priv);
-
-        volumes = dict_new ();
-        if (!volumes) {
-                gf_log ("", GF_LOG_WARNING, "Out of Memory");
-                goto out;
+    int32_t ret = -1;
+    uint32_t cmd = 0;
+    dict_t *dict = NULL;
+    char *volname = 0;
+    gf_cli_req cli_req = {{
+        0,
+    }};
+    glusterd_op_t cli_op = GD_OP_STATUS_VOLUME;
+    char err_str[256] = {
+        0,
+    };
+    xlator_t *this = NULL;
+    glusterd_conf_t *conf = NULL;
+
+    GF_ASSERT(req);
+    this = THIS;
+    GF_ASSERT(this);
+    conf = this->private;
+    GF_ASSERT(conf);
+
+    ret = xdr_to_generic(req->msg[0], &cli_req, (xdrproc_t)xdr_gf_cli_req);
+    if (ret < 0) {
+        // failed to decode msg;
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_REQ_DECODE_FAIL,
+               "Failed to decode "
+               "request received from cli");
+        req->rpc_err = GARBAGE_ARGS;
+        goto out;
+    }
+
+    if (cli_req.dict.dict_len > 0) {
+        dict = dict_new();
+        if (!dict) {
+            gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_CREATE_FAIL,
+                    NULL);
+            goto out;
         }
+        ret = dict_unserialize(cli_req.dict.dict_val, cli_req.dict.dict_len,
+                               &dict);
+        if (ret < 0) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_UNSERIALIZE_FAIL,
+                   "failed to "
+                   "unserialize buffer");
+            snprintf(err_str, sizeof(err_str),
+                     "Unable to decode "
+                     "the command");
+            goto out;
+        }
+    }
+
+    ret = dict_get_uint32(dict, "cmd", &cmd);
+    if (ret)
+        goto out;
+
+    if (!(cmd & GF_CLI_STATUS_ALL)) {
+        ret = dict_get_strn(dict, "volname", SLEN("volname"), &volname);
+        if (ret) {
+            snprintf(err_str, sizeof(err_str),
+                     "Unable to get "
+                     "volume name");
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_VOL_NOT_FOUND, "%s",
+                   err_str);
+            goto out;
+        }
+        gf_msg(this->name, GF_LOG_INFO, 0, GD_MSG_STATUS_VOL_REQ_RCVD,
+               "Received status volume req for volume %s", volname);
+    }
+    if ((cmd & GF_CLI_STATUS_CLIENT_LIST) &&
+        (conf->op_version < GD_OP_VERSION_3_13_0)) {
+        snprintf(err_str, sizeof(err_str),
+                 "The cluster is operating "
+                 "at version less than %d. Getting the client-list "
+                 "is not allowed in this state.",
+                 GD_OP_VERSION_3_13_0);
+        ret = -1;
+        goto out;
+    }
+
+    if ((cmd & GF_CLI_STATUS_QUOTAD) &&
+        (conf->op_version == GD_OP_VERSION_MIN)) {
+        snprintf(err_str, sizeof(err_str),
+                 "The cluster is operating "
+                 "at version 1. Getting the status of quotad is not "
+                 "allowed in this state.");
+        ret = -1;
+        goto out;
+    }
+
+    if ((cmd & GF_CLI_STATUS_SNAPD) &&
+        (conf->op_version < GD_OP_VERSION_3_6_0)) {
+        snprintf(err_str, sizeof(err_str),
+                 "The cluster is operating "
+                 "at a lesser version than %d. Getting the status of "
+                 "snapd is not allowed in this state",
+                 GD_OP_VERSION_3_6_0);
+        ret = -1;
+        goto out;
+    }
+
+    if ((cmd & GF_CLI_STATUS_BITD) &&
+        (conf->op_version < GD_OP_VERSION_3_7_0)) {
+        snprintf(err_str, sizeof(err_str),
+                 "The cluster is operating "
+                 "at a lesser version than %d. Getting the status of "
+                 "bitd is not allowed in this state",
+                 GD_OP_VERSION_3_7_0);
+        ret = -1;
+        goto out;
+    }
+
+    if ((cmd & GF_CLI_STATUS_SCRUB) &&
+        (conf->op_version < GD_OP_VERSION_3_7_0)) {
+        snprintf(err_str, sizeof(err_str),
+                 "The cluster is operating "
+                 "at a lesser version than %d. Getting the status of "
+                 "scrub is not allowed in this state",
+                 GD_OP_VERSION_3_7_0);
+        ret = -1;
+        goto out;
+    }
 
-        if (cds_list_empty (&priv->volumes)) {
-                ret = 0;
-                goto respond;
-        }
+    ret = glusterd_op_begin_synctask(req, GD_OP_STATUS_VOLUME, dict);
 
-        if (flags == GF_CLI_GET_VOLUME_ALL) {
-                cds_list_for_each_entry (entry, &priv->volumes, vol_list) {
-                        ret = glusterd_add_volume_detail_to_dict (entry,
-                                                        volumes, count);
-                        if (ret)
-                                goto respond;
+out:
 
-                        count++;
+    if (ret) {
+        if (err_str[0] == '\0')
+            snprintf(err_str, sizeof(err_str), "Operation failed");
+        ret = glusterd_op_send_cli_response(cli_op, ret, 0, req, dict, err_str);
+    }
+    free(cli_req.dict.dict_val);
 
-                }
+    return ret;
+}
 
-        } else if (flags == GF_CLI_GET_NEXT_VOLUME) {
-                ret = dict_get_str (dict, "volname", &volname);
+int
+glusterd_handle_status_volume(rpcsvc_request_t *req)
+{
+    return glusterd_big_locked_handler(req, __glusterd_handle_status_volume);
+}
 
-                if (ret) {
-                        if (priv->volumes.next) {
-                                entry = cds_list_entry (priv->volumes.next,
-                                                        typeof (*entry),
-                                                        vol_list);
-                        }
-                } else {
-                        ret = glusterd_volinfo_find (volname, &entry);
-                        if (ret)
-                                goto respond;
-                        entry = cds_list_entry (entry->vol_list.next,
-                                                typeof (*entry),
-                                                vol_list);
-                }
+int
+__glusterd_handle_cli_clearlocks_volume(rpcsvc_request_t *req)
+{
+    int32_t ret = -1;
+    gf_cli_req cli_req = {{
+        0,
+    }};
+    glusterd_op_t cli_op = GD_OP_CLEARLOCKS_VOLUME;
+    char *volname = NULL;
+    dict_t *dict = NULL;
+    char err_str[64] = {
+        0,
+    };
+    xlator_t *this = NULL;
+
+    GF_ASSERT(req);
+    this = THIS;
+    GF_ASSERT(this);
+
+    ret = -1;
+    ret = xdr_to_generic(req->msg[0], &cli_req, (xdrproc_t)xdr_gf_cli_req);
+    if (ret < 0) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_REQ_DECODE_FAIL,
+               "Failed to decode "
+               "request received from cli");
+        req->rpc_err = GARBAGE_ARGS;
+        goto out;
+    }
+
+    if (cli_req.dict.dict_len) {
+        dict = dict_new();
 
-                if (&entry->vol_list == &priv->volumes) {
-                       goto respond;
-                } else {
-                        ret = glusterd_add_volume_detail_to_dict (entry,
-                                                         volumes, count);
-                        if (ret)
-                                goto respond;
+        ret = dict_unserialize(cli_req.dict.dict_val, cli_req.dict.dict_len,
+                               &dict);
+        if (ret < 0) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_UNSERIALIZE_FAIL,
+                   "failed to unserialize req-buffer to"
+                   " dictionary");
+            snprintf(err_str, sizeof(err_str),
+                     "unable to decode "
+                     "the command");
+            goto out;
+        }
 
-                        count++;
-                }
-        } else if (flags == GF_CLI_GET_VOLUME) {
-                ret = dict_get_str (dict, "volname", &volname);
-                if (ret)
-                        goto respond;
+    } else {
+        ret = -1;
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_CLI_REQ_EMPTY,
+               "Empty cli request.");
+        goto out;
+    }
+
+    ret = dict_get_strn(dict, "volname", SLEN("volname"), &volname);
+    if (ret) {
+        snprintf(err_str, sizeof(err_str),
+                 "Unable to get volume "
+                 "name");
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_VOLNAME_NOTFOUND_IN_DICT,
+               "%s", err_str);
+        goto out;
+    }
+
+    gf_msg(this->name, GF_LOG_INFO, 0, GD_MSG_CLRCLK_VOL_REQ_RCVD,
+           "Received clear-locks volume req "
+           "for volume %s",
+           volname);
+
+    ret = glusterd_op_begin_synctask(req, GD_OP_CLEARLOCKS_VOLUME, dict);
 
-                ret = glusterd_volinfo_find (volname, &entry);
-                if (ret)
-                        goto respond;
+out:
+    if (ret) {
+        if (err_str[0] == '\0')
+            snprintf(err_str, sizeof(err_str), "Operation failed");
+        ret = glusterd_op_send_cli_response(cli_op, ret, 0, req, dict, err_str);
+    }
+    free(cli_req.dict.dict_val);
+
+    return ret;
+}
 
-                ret = glusterd_add_volume_detail_to_dict (entry,
-                                                 volumes, count);
-                if (ret)
-                        goto respond;
+int
+glusterd_handle_cli_clearlocks_volume(rpcsvc_request_t *req)
+{
+    return glusterd_big_locked_handler(req,
+                                       __glusterd_handle_cli_clearlocks_volume);
+}
 
-                count++;
+static int
+get_volinfo_from_brickid(char *brickid, glusterd_volinfo_t **volinfo)
+{
+    int ret = -1;
+    char *volid_str = NULL;
+    char *brick = NULL;
+    char *brickid_dup = NULL;
+    uuid_t volid = {0};
+    xlator_t *this = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+    GF_ASSERT(brickid);
+
+    brickid_dup = gf_strdup(brickid);
+    if (!brickid_dup)
+        goto out;
+
+    volid_str = brickid_dup;
+    brick = strchr(brickid_dup, ':');
+    if (!brick) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_BRICK_NOT_FOUND,
+               "Invalid brickid");
+        goto out;
+    }
+
+    *brick = '\0';
+    brick++;
+    gf_uuid_parse(volid_str, volid);
+    ret = glusterd_volinfo_find_by_volume_id(volid, volinfo);
+    if (ret) {
+        /* Check if it is a snapshot volume */
+        ret = glusterd_snap_volinfo_find_by_volume_id(volid, volinfo);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_WARNING, 0, GD_MSG_VOLINFO_GET_FAIL,
+                   "Failed to find volinfo");
+            goto out;
         }
+    }
 
-respond:
-        ret = dict_set_int32 (volumes, "count", count);
-        if (ret)
-                goto out;
-        ret = dict_allocate_and_serialize (volumes, &rsp.dict.dict_val,
-                                           &rsp.dict.dict_len);
-
-        if (ret)
-                goto out;
-
-        ret = 0;
+    ret = 0;
 out:
-        rsp.op_ret = ret;
+    GF_FREE(brickid_dup);
+    return ret;
+}
 
-        rsp.op_errstr = "";
-        glusterd_submit_reply (req, &rsp, NULL, 0, NULL,
-                               (xdrproc_t)xdr_gf_cli_rsp);
-        ret = 0;
+static int
+__glusterd_handle_barrier(rpcsvc_request_t *req)
+{
+    int ret = -1;
+    xlator_t *this = NULL;
+    gf_cli_req cli_req = {{
+        0,
+    }};
+    dict_t *dict = NULL;
+    char *volname = NULL;
+
+    GF_ASSERT(req);
+    this = THIS;
+    GF_ASSERT(this);
+
+    ret = xdr_to_generic(req->msg[0], &cli_req, (xdrproc_t)xdr_gf_cli_req);
+    if (ret < 0) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_REQ_DECODE_FAIL,
+               "Failed to decode "
+               "request received from cli");
+        req->rpc_err = GARBAGE_ARGS;
+        goto out;
+    }
+
+    if (!cli_req.dict.dict_len) {
+        ret = -1;
+        goto out;
+    }
 
-        if (volumes)
-                dict_unref (volumes);
+    dict = dict_new();
+    if (!dict) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_CREATE_FAIL, NULL);
+        ret = -1;
+        goto out;
+    }
+    ret = dict_unserialize(cli_req.dict.dict_val, cli_req.dict.dict_len, &dict);
+    if (ret < 0) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_UNSERIALIZE_FAIL,
+               "Failed to unserialize "
+               "request dictionary.");
+        goto out;
+    }
+
+    ret = dict_get_strn(dict, "volname", SLEN("volname"), &volname);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_VOLNAME_NOTFOUND_IN_DICT,
+               "Volname not present in "
+               "dict");
+        goto out;
+    }
+    gf_msg(this->name, GF_LOG_INFO, 0, GD_MSG_BARRIER_VOL_REQ_RCVD,
+           "Received barrier volume request for "
+           "volume %s",
+           volname);
+
+    ret = glusterd_op_begin_synctask(req, GD_OP_BARRIER, dict);
 
-        GF_FREE (rsp.dict.dict_val);
-        return ret;
+out:
+    if (ret) {
+        ret = glusterd_op_send_cli_response(GD_OP_BARRIER, ret, 0, req, dict,
+                                            "Operation failed");
+    }
+    free(cli_req.dict.dict_val);
+    return ret;
 }
 
 int
-__glusterd_handle_status_volume (rpcsvc_request_t *req)
+glusterd_handle_barrier(rpcsvc_request_t *req)
 {
-        int32_t                         ret     = -1;
-        uint32_t                        cmd     = 0;
-        dict_t                         *dict    = NULL;
-        char                           *volname = 0;
-        gf_cli_req                      cli_req = {{0,}};
-        glusterd_op_t                   cli_op  = GD_OP_STATUS_VOLUME;
-        char                            err_str[2048] = {0,};
-        xlator_t                       *this = NULL;
-        glusterd_conf_t                *conf = NULL;
-
-        GF_ASSERT (req);
-        this = THIS;
-        GF_ASSERT (this);
-        conf = this->private;
-        GF_ASSERT (conf);
+    return glusterd_big_locked_handler(req, __glusterd_handle_barrier);
+}
 
-        ret = xdr_to_generic (req->msg[0], &cli_req, (xdrproc_t)xdr_gf_cli_req);
-        if (ret < 0) {
-                //failed to decode msg;
-                req->rpc_err = GARBAGE_ARGS;
-                goto out;
-        }
+static gf_boolean_t
+gd_is_global_option(char *opt_key)
+{
+    GF_VALIDATE_OR_GOTO(THIS->name, opt_key, out);
 
-        if (cli_req.dict.dict_len > 0) {
-                dict = dict_new();
-                if (!dict)
-                        goto out;
-                ret = dict_unserialize (cli_req.dict.dict_val,
-                                        cli_req.dict.dict_len, &dict);
-                if (ret < 0) {
-                        gf_log (this->name, GF_LOG_ERROR, "failed to "
-                                "unserialize buffer");
-                        snprintf (err_str, sizeof (err_str), "Unable to decode "
-                                  "the command");
-                        goto out;
-                }
+    return (strcmp(opt_key, GLUSTERD_SHARED_STORAGE_KEY) == 0 ||
+            strcmp(opt_key, GLUSTERD_QUORUM_RATIO_KEY) == 0 ||
+            strcmp(opt_key, GLUSTERD_GLOBAL_OP_VERSION_KEY) == 0 ||
+            strcmp(opt_key, GLUSTERD_BRICK_MULTIPLEX_KEY) == 0 ||
+            strcmp(opt_key, GLUSTERD_LOCALTIME_LOGGING_KEY) == 0 ||
+            strcmp(opt_key, GLUSTERD_DAEMON_LOG_LEVEL_KEY) == 0 ||
+            strcmp(opt_key, GLUSTERD_MAX_OP_VERSION_KEY) == 0);
 
-        }
+out:
+    return _gf_false;
+}
 
-        ret = dict_get_uint32 (dict, "cmd", &cmd);
-        if (ret)
+int32_t
+glusterd_get_volume_opts(rpcsvc_request_t *req, dict_t *dict)
+{
+    int32_t ret = -1;
+    int32_t count = 1;
+    int exists = 0;
+    char *key = NULL;
+    char *orig_key = NULL;
+    char *key_fixed = NULL;
+    char *volname = NULL;
+    char *value = NULL;
+    char err_str[2048] = {
+        0,
+    };
+    char dict_key[50] = {
+        0,
+    };
+    int keylen;
+    xlator_t *this = NULL;
+    glusterd_conf_t *priv = NULL;
+    glusterd_volinfo_t *volinfo = NULL;
+    gf_cli_rsp rsp = {
+        0,
+    };
+    char op_version_buff[10] = {
+        0,
+    };
+
+    this = THIS;
+    GF_ASSERT(this);
+
+    priv = this->private;
+    GF_ASSERT(priv);
+
+    GF_ASSERT(req);
+    GF_ASSERT(dict);
+
+    ret = dict_get_strn(dict, "volname", SLEN("volname"), &volname);
+    if (ret) {
+        snprintf(err_str, sizeof(err_str),
+                 "Failed to get volume "
+                 "name while handling get volume option command");
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_VOLNAME_NOTFOUND_IN_DICT,
+               "%s", err_str);
+        goto out;
+    }
+
+    if (strcasecmp(volname, "all") == 0) {
+        ret = glusterd_get_global_options_for_all_vols(req, dict,
+                                                       &rsp.op_errstr);
+        goto out;
+    }
+
+    ret = dict_get_strn(dict, "key", SLEN("key"), &key);
+    if (ret) {
+        snprintf(err_str, sizeof(err_str),
+                 "Failed to get key "
+                 "while handling get volume option for %s",
+                 volname);
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED, "%s",
+               err_str);
+        goto out;
+    }
+    gf_msg_debug(this->name, 0,
+                 "Received get volume opt request for "
+                 "volume %s",
+                 volname);
+
+    ret = glusterd_volinfo_find(volname, &volinfo);
+    if (ret) {
+        snprintf(err_str, sizeof(err_str), FMTSTR_CHECK_VOL_EXISTS, volname);
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_VOL_NOT_FOUND,
+               FMTSTR_CHECK_VOL_EXISTS, volname);
+        goto out;
+    }
+    if (strcmp(key, "all")) {
+        if (fnmatch(GD_HOOKS_SPECIFIC_KEY, key, FNM_NOESCAPE) == 0) {
+            keylen = sprintf(dict_key, "key%d", count);
+            ret = dict_set_strn(dict, dict_key, keylen, key);
+            if (ret) {
+                gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+                       "Failed to "
+                       "set %s in dictionary",
+                       key);
+                goto out;
+            }
+            ret = dict_get_str(volinfo->dict, key, &value);
+            if (ret) {
+                gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+                       "Failed to "
+                       "get %s in dictionary",
+                       key);
+                goto out;
+            }
+            keylen = sprintf(dict_key, "value%d", count);
+            ret = dict_set_strn(dict, dict_key, keylen, value);
+            if (ret) {
+                gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+                       "Failed to "
+                       "set %s in dictionary",
+                       key);
+                goto out;
+            }
+        } else {
+            exists = glusterd_check_option_exists(key, &key_fixed);
+            if (!exists) {
+                snprintf(err_str, sizeof(err_str),
+                         "Option "
+                         "with name: %s does not exist",
+                         key);
+                gf_msg(this->name, GF_LOG_ERROR, EINVAL, GD_MSG_UNKNOWN_KEY,
+                       "%s", err_str);
+                if (key_fixed)
+                    snprintf(err_str + ret, sizeof(err_str) - ret,
+                             "Did you mean %s?", key_fixed);
+                ret = -1;
                 goto out;
-
-        if (!(cmd & GF_CLI_STATUS_ALL)) {
-                ret = dict_get_str (dict, "volname", &volname);
+            }
+            if (key_fixed) {
+                orig_key = key;
+                key = key_fixed;
+            }
+
+            if (gd_is_global_option(key)) {
+                char warn_str[] =
+                    "Warning: support to get \
+                                        global option value using volume get \
+                                        <volname>` will be deprecated from \
+                                        next release. Consider using `volume \
+                                        get all` instead for global options";
+
+                ret = dict_set_strn(dict, "warning", SLEN("warning"), warn_str);
                 if (ret) {
-                        snprintf (err_str, sizeof (err_str), "Unable to get "
-                                  "volume name");
-                        gf_log (this->name, GF_LOG_ERROR, "%s", err_str);
-                        goto out;
+                    gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+                           "Failed to set warning "
+                           "message in dictionary");
+                    goto out;
                 }
-                gf_log (this->name, GF_LOG_INFO,
-                        "Received status volume req for volume %s", volname);
+            }
 
-        }
-        if ((cmd & GF_CLI_STATUS_QUOTAD) &&
-            (conf->op_version == GD_OP_VERSION_MIN)) {
-                snprintf (err_str, sizeof (err_str), "The cluster is operating "
-                          "at version 1. Getting the status of quotad is not "
-                          "allowed in this state.");
-                ret = -1;
-                goto out;
-        }
+            if (strcmp(key, GLUSTERD_MAX_OP_VERSION_KEY) == 0) {
+                ret = glusterd_get_global_max_op_version(req, dict, 1);
+                if (ret)
+                    goto out;
+            } else if (strcmp(key, GLUSTERD_GLOBAL_OP_VERSION_KEY) == 0) {
+                keylen = sprintf(dict_key, "key%d", count);
+                ret = dict_set_strn(dict, dict_key, keylen, key);
+                if (ret) {
+                    gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+                           "Failed"
+                           "to set %s in dictionary",
+                           key);
+                    goto out;
+                }
+                keylen = sprintf(dict_key, "value%d", count);
+                sprintf(op_version_buff, "%d", priv->op_version);
+                ret = dict_set_strn(dict, dict_key, keylen, op_version_buff);
+                if (ret) {
+                    gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+                           "Failed"
+                           " to set value for key %s in "
+                           "dictionary",
+                           key);
+                    goto out;
+                }
+            } else if (strcmp(key, "config.memory-accounting") == 0) {
+                keylen = sprintf(dict_key, "key%d", count);
+                ret = dict_set_strn(dict, dict_key, keylen, key);
+                if (ret) {
+                    gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+                           "Failed"
+                           " to set %s in dictionary",
+                           key);
+                    goto out;
+                }
+                keylen = sprintf(dict_key, "value%d", count);
+
+                if (volinfo->memory_accounting)
+                    ret = dict_set_nstrn(dict, dict_key, keylen, "Enabled",
+                                         SLEN("Enabled"));
+                else
+                    ret = dict_set_nstrn(dict, dict_key, keylen, "Disabled",
+                                         SLEN("Disabled"));
+                if (ret) {
+                    gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+                           "Failed"
+                           " to set value for key %s in "
+                           "dictionary",
+                           key);
+                    goto out;
+                }
+            } else if (strcmp(key, "config.transport") == 0) {
+                keylen = sprintf(dict_key, "key%d", count);
+                ret = dict_set_strn(dict, dict_key, keylen, key);
+                if (ret) {
+                    gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+                           "Failed to set %s in "
+                           "dictionary",
+                           key);
+                    goto out;
+                }
+                keylen = sprintf(dict_key, "value%d", count);
+
+                if (volinfo->transport_type == GF_TRANSPORT_RDMA)
+                    ret = dict_set_nstrn(dict, dict_key, keylen, "rdma",
+                                         SLEN("rdma"));
+                else if (volinfo->transport_type == GF_TRANSPORT_TCP)
+                    ret = dict_set_nstrn(dict, dict_key, keylen, "tcp",
+                                         SLEN("tcp"));
+                else if (volinfo->transport_type == GF_TRANSPORT_BOTH_TCP_RDMA)
+                    ret = dict_set_nstrn(dict, dict_key, keylen, "tcp,rdma",
+                                         SLEN("tcp,rdma"));
+                else
+                    ret = dict_set_nstrn(dict, dict_key, keylen, "none",
+                                         SLEN("none"));
 
-        if ((cmd & GF_CLI_STATUS_SNAPD) &&
-            (conf->op_version < GD_OP_VERSION_3_6_0)) {
-                snprintf (err_str, sizeof (err_str), "The cluster is operating "
-                          "at a lesser version than %d. Getting the status of "
-                          "snapd is not allowed in this state",
-                          GD_OP_VERSION_3_6_0);
-                ret = -1;
-                goto out;
+                if (ret) {
+                    gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+                           "Failed to set value for key "
+                           "%s in dictionary",
+                           key);
+                    goto out;
+                }
+            } else {
+                keylen = sprintf(dict_key, "key%d", count);
+                ret = dict_set_strn(dict, dict_key, keylen, key);
+                if (ret) {
+                    gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+                           "Failed to set %s in "
+                           "dictionary",
+                           key);
+                    goto out;
+                }
+                keylen = sprintf(dict_key, "value%d", count);
+                ret = dict_get_str(priv->opts, key, &value);
+                if (!ret) {
+                    ret = dict_set_strn(dict, dict_key, keylen, value);
+                    if (ret) {
+                        gf_msg(this->name, GF_LOG_ERROR, 0,
+                               GD_MSG_DICT_SET_FAILED,
+                               "Failed to set %s in "
+                               " dictionary",
+                               key);
+                        goto out;
+                    }
+                } else {
+                    ret = glusterd_get_default_val_for_volopt(
+                        dict, _gf_false, key, orig_key, volinfo,
+                        &rsp.op_errstr);
+                    if (ret && !rsp.op_errstr) {
+                        snprintf(err_str, sizeof(err_str),
+                                 "Failed to fetch the "
+                                 "value of %s, check "
+                                 "log file for more"
+                                 " details",
+                                 key);
+                    }
+                }
+            }
         }
-
-        ret = glusterd_op_begin_synctask (req, GD_OP_STATUS_VOLUME, dict);
+        /* Request is for a single option, explicitly set count to 1
+         * in the dictionary.
+         */
+        ret = dict_set_int32n(dict, "count", SLEN("count"), 1);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_SET_FAILED,
+                   "Failed to set count "
+                   "value in the dictionary");
+            goto out;
+        }
+    } else {
+        /* Handle the "all" volume option request */
+        ret = glusterd_get_default_val_for_volopt(dict, _gf_true, NULL, NULL,
+                                                  volinfo, &rsp.op_errstr);
+        if (ret && !rsp.op_errstr) {
+            snprintf(err_str, sizeof(err_str),
+                     "Failed to fetch the value of all volume "
+                     "options, check log file for more details");
+        }
+    }
 
 out:
+    if (ret) {
+        if (!rsp.op_errstr)
+            rsp.op_errstr = err_str;
+        rsp.op_ret = ret;
+    } else {
+        rsp.op_errstr = "";
+        rsp.op_ret = 0;
+    }
 
-        if (ret) {
-                if (err_str[0] == '\0')
-                        snprintf (err_str, sizeof (err_str),
-                                  "Operation failed");
-                ret = glusterd_op_send_cli_response (cli_op, ret, 0, req,
-                                                     dict, err_str);
-        }
-        free (cli_req.dict.dict_val);
+    ret = dict_allocate_and_serialize(dict, &rsp.dict.dict_val,
+                                      &rsp.dict.dict_len);
 
-        return ret;
+    glusterd_submit_reply(req, &rsp, NULL, 0, NULL, (xdrproc_t)xdr_gf_cli_rsp);
+    GF_FREE(rsp.dict.dict_val);
+    GF_FREE(key_fixed);
+    return ret;
 }
 
 int
-glusterd_handle_status_volume (rpcsvc_request_t *req)
+__glusterd_handle_get_vol_opt(rpcsvc_request_t *req)
 {
-        return glusterd_big_locked_handler (req,
-                                            __glusterd_handle_status_volume);
+    int32_t ret = -1;
+    gf_cli_req cli_req = {{
+        0,
+    }};
+    dict_t *dict = NULL;
+    char err_str[64] = {
+        0,
+    };
+    xlator_t *this = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+
+    GF_ASSERT(req);
+
+    ret = xdr_to_generic(req->msg[0], &cli_req, (xdrproc_t)xdr_gf_cli_req);
+    if (ret < 0) {
+        snprintf(err_str, sizeof(err_str),
+                 "Failed to decode "
+                 "request received from cli");
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_REQ_DECODE_FAIL, "%s",
+               err_str);
+        req->rpc_err = GARBAGE_ARGS;
+        goto out;
+    }
+
+    if (cli_req.dict.dict_len) {
+        /* Unserialize the dictionary */
+        dict = dict_new();
+
+        ret = dict_unserialize(cli_req.dict.dict_val, cli_req.dict.dict_len,
+                               &dict);
+        if (ret < 0) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_UNSERIALIZE_FAIL,
+                   "failed to "
+                   "unserialize req-buffer to dictionary");
+            snprintf(err_str, sizeof(err_str),
+                     "Unable to decode "
+                     "the command");
+            goto out;
+        } else {
+            dict->extra_stdfree = cli_req.dict.dict_val;
+        }
+    }
+    ret = glusterd_get_volume_opts(req, dict);
+
+out:
+    if (dict)
+        dict_unref(dict);
+
+    return ret;
 }
 
 int
-__glusterd_handle_cli_clearlocks_volume (rpcsvc_request_t *req)
+glusterd_handle_get_vol_opt(rpcsvc_request_t *req)
 {
-        int32_t                         ret = -1;
-        gf_cli_req                      cli_req = {{0,}};
-        glusterd_op_t                   cli_op = GD_OP_CLEARLOCKS_VOLUME;
-        char                            *volname = NULL;
-        dict_t                          *dict = NULL;
-        char                            err_str[2048] = {0,};
-        xlator_t                        *this = NULL;
-
-        GF_ASSERT (req);
-        this = THIS;
-        GF_ASSERT (this);
+    return glusterd_big_locked_handler(req, __glusterd_handle_get_vol_opt);
+}
 
-        ret = -1;
-        ret = xdr_to_generic (req->msg[0], &cli_req, (xdrproc_t)xdr_gf_cli_req);
-        if (ret < 0) {
-                req->rpc_err = GARBAGE_ARGS;
-                goto out;
-        }
+extern struct rpc_clnt_program gd_brick_prog;
 
-        if (cli_req.dict.dict_len) {
-                dict  = dict_new ();
-
-                ret = dict_unserialize (cli_req.dict.dict_val,
-                                        cli_req.dict.dict_len,
-                                        &dict);
-                if (ret < 0) {
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "failed to unserialize req-buffer to"
-                                " dictionary");
-                        snprintf (err_str, sizeof (err_str), "unable to decode "
-                                  "the command");
-                        goto out;
-                }
+static int
+glusterd_print_global_options(dict_t *opts, char *key, data_t *val, void *data)
+{
+    FILE *fp = NULL;
 
-        } else {
-                ret = -1;
-                gf_log (this->name, GF_LOG_ERROR, "Empty cli request.");
-                goto out;
-        }
+    GF_VALIDATE_OR_GOTO(THIS->name, key, out);
+    GF_VALIDATE_OR_GOTO(THIS->name, val, out);
+    GF_VALIDATE_OR_GOTO(THIS->name, data, out);
 
-        ret = dict_get_str (dict, "volname", &volname);
-        if (ret) {
-                snprintf (err_str, sizeof (err_str), "Unable to get volume "
-                          "name");
-                gf_log (this->name, GF_LOG_ERROR, "%s", err_str);
-                goto out;
-        }
+    if (strcmp(key, GLUSTERD_GLOBAL_OPT_VERSION) == 0)
+        goto out;
 
-        gf_log (this->name, GF_LOG_INFO, "Received clear-locks volume req "
-                "for volume %s", volname);
+    fp = (FILE *)data;
+    fprintf(fp, "%s: %s\n", key, val->data);
+out:
+    return 0;
+}
+
+static int
+glusterd_print_volume_options(dict_t *opts, char *key, data_t *val, void *data)
+{
+    FILE *fp = NULL;
 
-        ret = glusterd_op_begin_synctask (req, GD_OP_CLEARLOCKS_VOLUME, dict);
+    GF_VALIDATE_OR_GOTO(THIS->name, key, out);
+    GF_VALIDATE_OR_GOTO(THIS->name, val, out);
+    GF_VALIDATE_OR_GOTO(THIS->name, data, out);
 
+    fp = (FILE *)data;
+    fprintf(fp, "Volume%d.options.%s: %s\n", volcount, key, val->data);
 out:
-        if (ret) {
-                if (err_str[0] == '\0')
-                        snprintf (err_str, sizeof (err_str),
-                                  "Operation failed");
-                ret = glusterd_op_send_cli_response (cli_op, ret, 0, req,
-                                                     dict, err_str);
-        }
-        free (cli_req.dict.dict_val);
+    return 0;
+}
 
-        return ret;
+static int
+glusterd_print_gsync_status(FILE *fp, dict_t *gsync_dict)
+{
+    int ret = -1;
+    int gsync_count = 0;
+    int i = 0;
+    gf_gsync_status_t *status_vals = NULL;
+    char status_val_name[PATH_MAX] = {
+        0,
+    };
+
+    GF_VALIDATE_OR_GOTO(THIS->name, fp, out);
+    GF_VALIDATE_OR_GOTO(THIS->name, gsync_dict, out);
+
+    ret = dict_get_int32n(gsync_dict, "gsync-count", SLEN("gsync-count"),
+                          &gsync_count);
+
+    fprintf(fp, "Volume%d.gsync_count: %d\n", volcount, gsync_count);
+
+    if (gsync_count == 0) {
+        ret = 0;
+        goto out;
+    }
+
+    for (i = 0; i < gsync_count; i++) {
+        snprintf(status_val_name, sizeof(status_val_name), "status_value%d", i);
+
+        ret = dict_get_bin(gsync_dict, status_val_name,
+                           (void **)&(status_vals));
+        if (ret)
+            goto out;
+
+        fprintf(fp, "Volume%d.pair%d.session_slave: %s\n", volcount, i + 1,
+                get_struct_variable(21, status_vals));
+        fprintf(fp, "Volume%d.pair%d.master_node: %s\n", volcount, i + 1,
+                get_struct_variable(0, status_vals));
+        fprintf(fp, "Volume%d.pair%d.master_volume: %s\n", volcount, i + 1,
+                get_struct_variable(1, status_vals));
+        fprintf(fp, "Volume%d.pair%d.master_brick: %s\n", volcount, i + 1,
+                get_struct_variable(2, status_vals));
+        fprintf(fp, "Volume%d.pair%d.slave_user: %s\n", volcount, i + 1,
+                get_struct_variable(3, status_vals));
+        fprintf(fp, "Volume%d.pair%d.slave: %s\n", volcount, i + 1,
+                get_struct_variable(4, status_vals));
+        fprintf(fp, "Volume%d.pair%d.slave_node: %s\n", volcount, i + 1,
+                get_struct_variable(5, status_vals));
+        fprintf(fp, "Volume%d.pair%d.status: %s\n", volcount, i + 1,
+                get_struct_variable(6, status_vals));
+        fprintf(fp, "Volume%d.pair%d.crawl_status: %s\n", volcount, i + 1,
+                get_struct_variable(7, status_vals));
+        fprintf(fp, "Volume%d.pair%d.last_synced: %s\n", volcount, i + 1,
+                get_struct_variable(8, status_vals));
+        fprintf(fp, "Volume%d.pair%d.entry: %s\n", volcount, i + 1,
+                get_struct_variable(9, status_vals));
+        fprintf(fp, "Volume%d.pair%d.data: %s\n", volcount, i + 1,
+                get_struct_variable(10, status_vals));
+        fprintf(fp, "Volume%d.pair%d.meta: %s\n", volcount, i + 1,
+                get_struct_variable(11, status_vals));
+        fprintf(fp, "Volume%d.pair%d.failures: %s\n", volcount, i + 1,
+                get_struct_variable(12, status_vals));
+        fprintf(fp, "Volume%d.pair%d.checkpoint_time: %s\n", volcount, i + 1,
+                get_struct_variable(13, status_vals));
+        fprintf(fp, "Volume%d.pair%d.checkpoint_completed: %s\n", volcount,
+                i + 1, get_struct_variable(14, status_vals));
+        fprintf(fp, "Volume%d.pair%d.checkpoint_completion_time: %s\n",
+                volcount, i + 1, get_struct_variable(15, status_vals));
+    }
+out:
+    return ret;
 }
 
-int
-glusterd_handle_cli_clearlocks_volume (rpcsvc_request_t *req)
+static int
+glusterd_print_gsync_status_by_vol(FILE *fp, glusterd_volinfo_t *volinfo)
 {
-        return glusterd_big_locked_handler (req,
-                                            __glusterd_handle_cli_clearlocks_volume);
+    int ret = -1;
+    dict_t *gsync_rsp_dict = NULL;
+    char my_hostname[256] = {
+        0,
+    };
+
+    xlator_t *this = THIS;
+    GF_ASSERT(this);
+
+    GF_VALIDATE_OR_GOTO(THIS->name, volinfo, out);
+    GF_VALIDATE_OR_GOTO(THIS->name, fp, out);
+
+    gsync_rsp_dict = dict_new();
+    if (!gsync_rsp_dict) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_CREATE_FAIL, NULL);
+        goto out;
+    }
+
+    ret = gethostname(my_hostname, sizeof(my_hostname));
+    if (ret) {
+        /* stick to N/A */
+        (void)strcpy(my_hostname, "N/A");
+    }
+
+    ret = glusterd_get_gsync_status_mst(volinfo, gsync_rsp_dict, my_hostname);
+    /* Ignoring ret as above function always returns ret = 0 */
+
+    ret = glusterd_print_gsync_status(fp, gsync_rsp_dict);
+out:
+    if (gsync_rsp_dict)
+        dict_unref(gsync_rsp_dict);
+    return ret;
 }
 
 static int
-get_volinfo_from_brickid (char *brickid, glusterd_volinfo_t **volinfo)
+glusterd_print_snapinfo_by_vol(FILE *fp, glusterd_volinfo_t *volinfo,
+                               int volcount)
 {
-        int             ret         = -1;
-        char           *volid_str  = NULL;
-        char           *brick      = NULL;
-        char           *brickid_dup = NULL;
-        uuid_t          volid       = {0};
-        xlator_t       *this        = NULL;
-
-        this = THIS;
-        GF_ASSERT (this);
-        GF_ASSERT (brickid);
-
-        brickid_dup = gf_strdup (brickid);
-        if (!brickid_dup)
-                goto out;
+    int ret = -1;
+    glusterd_volinfo_t *snap_vol = NULL;
+    glusterd_volinfo_t *tmp_vol = NULL;
+    glusterd_snap_t *snapinfo = NULL;
+    int snapcount = 0;
+    char timestr[GF_TIMESTR_SIZE] = {
+        0,
+    };
+    char snap_status_str[STATUS_STRLEN] = {
+        0,
+    };
+
+    GF_VALIDATE_OR_GOTO(THIS->name, volinfo, out);
+    GF_VALIDATE_OR_GOTO(THIS->name, fp, out);
+
+    cds_list_for_each_entry_safe(snap_vol, tmp_vol, &volinfo->snap_volumes,
+                                 snapvol_list)
+    {
+        snapcount++;
+        snapinfo = snap_vol->snapshot;
+
+        ret = glusterd_get_snap_status_str(snapinfo, snap_status_str);
+        if (ret) {
+            gf_msg(THIS->name, GF_LOG_ERROR, 0, GD_MSG_STATE_STR_GET_FAILED,
+                   "Failed to get status for snapshot: %s", snapinfo->snapname);
 
-        volid_str = brickid_dup;
-        brick = strchr (brickid_dup, ':');
-        if (!brick) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "Invalid brickid");
-                goto out;
+            goto out;
         }
+        gf_time_fmt(timestr, sizeof timestr, snapinfo->time_stamp,
+                    gf_timefmt_FT);
 
-        *brick = '\0';
-        brick++;
-        gf_uuid_parse (volid_str, volid);
-        ret = glusterd_volinfo_find_by_volume_id (volid, volinfo);
-        if (ret) {
-                /* Check if it is a snapshot volume */
-                ret = glusterd_snap_volinfo_find_by_volume_id (volid, volinfo);
-                if (ret) {
-                        gf_log (this->name, GF_LOG_WARNING,
-                                "Failed to find volinfo");
-                        goto out;
-                }
-        }
+        fprintf(fp, "Volume%d.snapshot%d.name: %s\n", volcount, snapcount,
+                snapinfo->snapname);
+        fprintf(fp, "Volume%d.snapshot%d.id: %s\n", volcount, snapcount,
+                uuid_utoa(snapinfo->snap_id));
+        fprintf(fp, "Volume%d.snapshot%d.time: %s\n", volcount, snapcount,
+                timestr);
 
-        ret = 0;
+        if (snapinfo->description)
+            fprintf(fp, "Volume%d.snapshot%d.description: %s\n", volcount,
+                    snapcount, snapinfo->description);
+        fprintf(fp, "Volume%d.snapshot%d.status: %s\n", volcount, snapcount,
+                snap_status_str);
+    }
+
+    ret = 0;
 out:
-        GF_FREE (brickid_dup);
-        return ret;
+    return ret;
 }
 
 static int
-__glusterd_handle_barrier (rpcsvc_request_t *req)
+glusterd_print_client_details(FILE *fp, dict_t *dict,
+                              glusterd_volinfo_t *volinfo, int volcount,
+                              glusterd_brickinfo_t *brickinfo, int brickcount)
 {
-        int          ret     = -1;
-        xlator_t     *this   = NULL;
-        gf_cli_req   cli_req = {{0,}};
-        dict_t       *dict   = NULL;
-        char *volname = NULL;
+    int ret = -1;
+    xlator_t *this = NULL;
+    int brick_index = -1;
+    int client_count = 0;
+    char key[64] = {
+        0,
+    };
+    int keylen;
+    char *clientname = NULL;
+    uint64_t bytesread = 0;
+    uint64_t byteswrite = 0;
+    uint32_t opversion = 0;
+
+    glusterd_pending_node_t *pending_node = NULL;
+    rpc_clnt_t *rpc = NULL;
+    struct syncargs args = {
+        0,
+    };
+    gd1_mgmt_brick_op_req *brick_req = NULL;
+
+    this = THIS;
+    GF_VALIDATE_OR_GOTO("glusterd", this, out);
+
+    GF_VALIDATE_OR_GOTO(this->name, dict, out);
+
+    if (gf_uuid_compare(brickinfo->uuid, MY_UUID) ||
+        !glusterd_is_brick_started(brickinfo)) {
+        ret = 0;
+        goto out;
+    }
 
-        GF_ASSERT (req);
-        this = THIS;
-        GF_ASSERT(this);
+    brick_index++;
+    pending_node = GF_CALLOC(1, sizeof(*pending_node),
+                             gf_gld_mt_pending_node_t);
+    if (!pending_node) {
+        ret = -1;
+        gf_msg(this->name, GF_LOG_ERROR, ENOMEM, GD_MSG_NO_MEMORY,
+               "Unable to allocate memory");
+        goto out;
+    }
 
-        ret = xdr_to_generic (req->msg[0], &cli_req, (xdrproc_t)xdr_gf_cli_req);
-        if (ret < 0) {
-                req->rpc_err = GARBAGE_ARGS;
-                goto out;
-        }
+    pending_node->node = brickinfo;
+    pending_node->type = GD_NODE_BRICK;
+    pending_node->index = brick_index;
 
-        if (!cli_req.dict.dict_len) {
-                ret = -1;
-                goto out;
-        }
+    rpc = glusterd_pending_node_get_rpc(pending_node);
+    if (!rpc) {
+        ret = -1;
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_RPC_FAILURE,
+               "Failed to retrieve rpc object");
+        goto out;
+    }
 
-        dict = dict_new();
-        if (!dict) {
-                ret = -1;
-                goto out;
+    brick_req = GF_CALLOC(1, sizeof(*brick_req), gf_gld_mt_mop_brick_req_t);
+    if (!brick_req) {
+        ret = -1;
+        gf_msg(this->name, GF_LOG_ERROR, ENOMEM, GD_MSG_NO_MEMORY,
+               "Unable to allocate memory");
+        goto out;
+    }
+
+    brick_req->op = GLUSTERD_BRICK_STATUS;
+    brick_req->name = "";
+    brick_req->dict.dict_val = NULL;
+    brick_req->dict.dict_len = 0;
+
+    ret = dict_set_strn(dict, "brick-name", SLEN("brick-name"),
+                        brickinfo->path);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_SET_FAILED,
+                "Key=brick-name", NULL);
+        goto out;
+    }
+
+    ret = dict_set_int32n(dict, "cmd", SLEN("cmd"), GF_CLI_STATUS_CLIENTS);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_SET_FAILED,
+                "Key=cmd", NULL);
+        goto out;
+    }
+
+    ret = dict_set_strn(dict, "volname", SLEN("volname"), volinfo->volname);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_SET_FAILED,
+                "Key=volname", NULL);
+        goto out;
+    }
+
+    ret = dict_allocate_and_serialize(dict, &brick_req->input.input_val,
+                                      &brick_req->input.input_len);
+    if (ret)
+        goto out;
+
+    GD_SYNCOP(rpc, (&args), NULL, gd_syncop_brick_op_cbk, brick_req,
+              &gd_brick_prog, brick_req->op, xdr_gd1_mgmt_brick_op_req);
+
+    if (args.op_ret)
+        goto out;
+
+    ret = dict_get_int32n(args.dict, "clientcount", SLEN("clientcount"),
+                          &client_count);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+               "Couldn't get client count");
+        goto out;
+    }
+
+    fprintf(fp, "Volume%d.Brick%d.client_count: %d\n", volcount, brickcount,
+            client_count);
+
+    if (client_count == 0) {
+        ret = 0;
+        goto out;
+    }
+
+    int i;
+    for (i = 1; i <= client_count; i++) {
+        keylen = snprintf(key, sizeof(key), "client%d.hostname", i - 1);
+        ret = dict_get_strn(args.dict, key, keylen, &clientname);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+                   "Failed to get client hostname");
+            goto out;
         }
-        ret = dict_unserialize (cli_req.dict.dict_val, cli_req.dict.dict_len,
-                                &dict);
-        if (ret < 0) {
-                gf_log (this->name, GF_LOG_ERROR, "Failed to unserialize "
-                        "request dictionary.");
-                goto out;
+
+        snprintf(key, sizeof(key), "Client%d.hostname", i);
+        fprintf(fp, "Volume%d.Brick%d.%s: %s\n", volcount, brickcount, key,
+                clientname);
+
+        snprintf(key, sizeof(key), "client%d.bytesread", i - 1);
+        ret = dict_get_uint64(args.dict, key, &bytesread);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+                   "Failed to get bytesread from client");
+            goto out;
         }
 
-        ret = dict_get_str (dict, "volname", &volname);
+        snprintf(key, sizeof(key), "Client%d.bytesread", i);
+        fprintf(fp, "Volume%d.Brick%d.%s: %" PRIu64 "\n", volcount, brickcount,
+                key, bytesread);
+
+        snprintf(key, sizeof(key), "client%d.byteswrite", i - 1);
+        ret = dict_get_uint64(args.dict, key, &byteswrite);
         if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "Volname not present in "
-                        "dict");
-                goto out;
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+                   "Failed to get byteswrite from client");
+            goto out;
         }
-        gf_log (this->name, GF_LOG_INFO, "Received barrier volume request for "
-                "volume %s", volname);
 
-        ret = glusterd_op_begin_synctask (req, GD_OP_BARRIER, dict);
+        snprintf(key, sizeof(key), "Client%d.byteswrite", i);
+        fprintf(fp, "Volume%d.Brick%d.%s: %" PRIu64 "\n", volcount, brickcount,
+                key, byteswrite);
 
-out:
+        snprintf(key, sizeof(key), "client%d.opversion", i - 1);
+        ret = dict_get_uint32(args.dict, key, &opversion);
         if (ret) {
-                ret = glusterd_op_send_cli_response (GD_OP_BARRIER, ret, 0, req,
-                                                     dict, "Operation failed");
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+                   "Failed to get client opversion");
+            goto out;
         }
-        free (cli_req.dict.dict_val);
-        return ret;
-}
 
-int
-glusterd_handle_barrier (rpcsvc_request_t *req)
-{
-        return glusterd_big_locked_handler (req, __glusterd_handle_barrier);
+        snprintf(key, sizeof(key), "Client%d.opversion", i);
+        fprintf(fp, "Volume%d.Brick%d.%s: %" PRIu32 "\n", volcount, brickcount,
+                key, opversion);
+    }
+
+out:
+    if (pending_node)
+        GF_FREE(pending_node);
+
+    if (brick_req) {
+        if (brick_req->input.input_val)
+            GF_FREE(brick_req->input.input_val);
+        GF_FREE(brick_req);
+    }
+    if (args.dict)
+        dict_unref(args.dict);
+    if (args.errstr)
+        GF_FREE(args.errstr);
+
+    return ret;
 }
 
-int32_t
-glusterd_get_volume_opts (rpcsvc_request_t *req, dict_t *dict)
+static int
+glusterd_get_state(rpcsvc_request_t *req, dict_t *dict)
 {
-        int32_t                   ret = -1;
-        int32_t                   count = 1;
-        int                       exists = 0;
-        char                      *key = NULL;
-        char                      *orig_key = NULL;
-        char                      *key_fixed = NULL;
-        char                      *volname = NULL;
-        char                      err_str[2048] = {0,};
-        char                      dict_key[50] = {0,};
-        xlator_t                  *this = NULL;
-        glusterd_conf_t           *priv = NULL;
-        glusterd_volinfo_t        *volinfo = NULL;
-        gf_cli_rsp                rsp = {0,};
-        char                      op_version_buff[10] = {0,};
-
-        this = THIS;
-        GF_ASSERT (this);
-
-        priv = this->private;
-        GF_ASSERT (priv);
-
-        GF_ASSERT (req);
-        GF_ASSERT (dict);
-
-        ret = dict_get_str (dict, "volname", &volname);
-        if (ret) {
-                snprintf (err_str, sizeof (err_str), "Failed to get volume "
-                          "name while handling get volume option command");
-                gf_log (this->name, GF_LOG_ERROR, "%s", err_str);
-                goto out;
+    int32_t ret = -1;
+    gf_cli_rsp rsp = {
+        0,
+    };
+    FILE *fp = NULL;
+    DIR *dp = NULL;
+    char err_str[2048] = {
+        0,
+    };
+    glusterd_conf_t *priv = NULL;
+    glusterd_peerinfo_t *peerinfo = NULL;
+    glusterd_peer_hostname_t *peer_hostname_info = NULL;
+    glusterd_volinfo_t *volinfo = NULL;
+    glusterd_brickinfo_t *brickinfo = NULL;
+    xlator_t *this = NULL;
+    dict_t *vol_all_opts = NULL;
+    struct statvfs brickstat = {0};
+    char *odir = NULL;
+    char *filename = NULL;
+    char *ofilepath = NULL;
+    char *tmp_str = NULL;
+    int count = 0;
+    int count_bkp = 0;
+    int odirlen = 0;
+    time_t now = 0;
+    char timestamp[16] = {
+        0,
+    };
+    uint32_t get_state_cmd = 0;
+    uint64_t memtotal = 0;
+    uint64_t memfree = 0;
+    char id_str[64] = {
+        0,
+    };
+
+    char *vol_type_str = NULL;
+
+    char transport_type_str[STATUS_STRLEN] = {
+        0,
+    };
+    char quorum_status_str[STATUS_STRLEN] = {
+        0,
+    };
+    char rebal_status_str[STATUS_STRLEN] = {
+        0,
+    };
+    char vol_status_str[STATUS_STRLEN] = {
+        0,
+    };
+    char brick_status_str[STATUS_STRLEN] = {
+        0,
+    };
+    this = THIS;
+    GF_VALIDATE_OR_GOTO(THIS->name, this, out);
+
+    priv = THIS->private;
+    GF_VALIDATE_OR_GOTO(this->name, priv, out);
+
+    GF_VALIDATE_OR_GOTO(this->name, dict, out);
+
+    ret = dict_get_strn(dict, "odir", SLEN("odir"), &tmp_str);
+    if (ret) {
+        odirlen = gf_asprintf(&odir, "%s", "/var/run/gluster/");
+        gf_msg(this->name, GF_LOG_INFO, 0, GD_MSG_DICT_GET_FAILED,
+               "Default output directory: %s", odir);
+    } else {
+        odirlen = gf_asprintf(&odir, "%s", tmp_str);
+    }
+
+    dp = sys_opendir(odir);
+    if (dp) {
+        sys_closedir(dp);
+    } else {
+        if (errno == ENOENT) {
+            snprintf(err_str, sizeof(err_str),
+                     "Output directory %s does not exist.", odir);
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED, "%s",
+                   err_str);
+        } else if (errno == ENOTDIR) {
+            snprintf(err_str, sizeof(err_str),
+                     "Output directory "
+                     "does not exist. %s points to a file.",
+                     odir);
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED, "%s",
+                   err_str);
+        }
+
+        GF_FREE(odir);
+        ret = -1;
+        goto out;
+    }
+
+    ret = dict_get_strn(dict, "filename", SLEN("filename"), &tmp_str);
+    if (ret) {
+        now = gf_time();
+        strftime(timestamp, sizeof(timestamp), "%Y%m%d_%H%M%S",
+                 localtime(&now));
+        gf_asprintf(&filename, "%s_%s", "glusterd_state", timestamp);
+
+        gf_msg(this->name, GF_LOG_INFO, 0, GD_MSG_DICT_GET_FAILED,
+               "Default filename: %s", filename);
+    } else {
+        gf_asprintf(&filename, "%s", tmp_str);
+    }
+
+    ret = gf_asprintf(&ofilepath, "%s%s%s", odir,
+                      ((odir[odirlen - 1] != '/') ? "/" : ""), filename);
+
+    if (ret < 0) {
+        GF_FREE(odir);
+        GF_FREE(filename);
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+               "Unable to get the output path");
+        ret = -1;
+        goto out;
+    }
+    GF_FREE(odir);
+    GF_FREE(filename);
+
+    ret = dict_set_dynstrn(dict, "ofilepath", SLEN("ofilepath"), ofilepath);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+               "Unable to set output path");
+        goto out;
+    }
+
+    fp = fopen(ofilepath, "w");
+    if (!fp) {
+        snprintf(err_str, sizeof(err_str), "Failed to open file at %s",
+                 ofilepath);
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED, "%s",
+               err_str);
+        ret = -1;
+        goto out;
+    }
+
+    ret = dict_get_uint32(dict, "getstate-cmd", &get_state_cmd);
+    if (ret) {
+        gf_msg_debug(this->name, 0, "get-state command type not set");
+        ret = 0;
+    }
+
+    if (get_state_cmd == GF_CLI_GET_STATE_VOLOPTS) {
+        fprintf(fp, "[Volume Options]\n");
+        cds_list_for_each_entry(volinfo, &priv->volumes, vol_list)
+        {
+            fprintf(fp, "Volume%d.name: %s\n", ++count, volinfo->volname);
+
+            volcount = count;
+            vol_all_opts = dict_new();
+
+            ret = glusterd_get_default_val_for_volopt(
+                vol_all_opts, _gf_true, NULL, NULL, volinfo, &rsp.op_errstr);
+            if (ret) {
+                gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_VOL_OPTS_IMPORT_FAIL,
+                       "Failed to "
+                       "fetch the value of all volume options "
+                       "for volume %s",
+                       volinfo->volname);
+                if (vol_all_opts)
+                    dict_unref(vol_all_opts);
+                continue;
+            }
+
+            dict_foreach(vol_all_opts, glusterd_print_volume_options, fp);
+
+            if (vol_all_opts)
+                dict_unref(vol_all_opts);
         }
+        ret = 0;
+        goto out;
+    }
 
-        ret = dict_get_str (dict, "key", &key);
-        if (ret) {
-                snprintf (err_str, sizeof (err_str), "Failed to get key "
-                          "while handling get volume option for %s", volname);
-                gf_log (this->name, GF_LOG_ERROR, "%s", err_str);
-                goto out;
+    fprintf(fp, "[Global]\n");
+
+    uuid_utoa_r(priv->uuid, id_str);
+    fprintf(fp, "MYUUID: %s\n", id_str);
+
+    fprintf(fp, "op-version: %d\n", priv->op_version);
+
+    fprintf(fp, "\n[Global options]\n");
+
+    if (priv->opts)
+        dict_foreach(priv->opts, glusterd_print_global_options, fp);
+
+    fprintf(fp, "\n[Peers]\n");
+    RCU_READ_LOCK;
+
+    cds_list_for_each_entry_rcu(peerinfo, &priv->peers, uuid_list)
+    {
+        fprintf(fp, "Peer%d.primary_hostname: %s\n", ++count,
+                peerinfo->hostname);
+        fprintf(fp, "Peer%d.uuid: %s\n", count, gd_peer_uuid_str(peerinfo));
+        fprintf(fp, "Peer%d.state: %s\n", count,
+                glusterd_friend_sm_state_name_get(peerinfo->state.state));
+        fprintf(fp, "Peer%d.connected: %s\n", count,
+                peerinfo->connected ? "Connected" : "Disconnected");
+
+        fprintf(fp, "Peer%d.othernames: ", count);
+        count_bkp = 0;
+        cds_list_for_each_entry(peer_hostname_info, &peerinfo->hostnames,
+                                hostname_list)
+        {
+            if (strcmp(peerinfo->hostname, peer_hostname_info->hostname) == 0)
+                continue;
+
+            if (count_bkp > 0)
+                fprintf(fp, ",");
+
+            fprintf(fp, "%s", peer_hostname_info->hostname);
+            count_bkp++;
         }
-        gf_log (this->name, GF_LOG_DEBUG, "Received get volume opt request for "
-                "volume %s", volname);
+        count_bkp = 0;
+        fprintf(fp, "\n");
+    }
+    RCU_READ_UNLOCK;
+
+    count = 0;
+    fprintf(fp, "\n[Volumes]\n");
 
-        ret = glusterd_volinfo_find (volname, &volinfo);
+    cds_list_for_each_entry(volinfo, &priv->volumes, vol_list)
+    {
+        ret = glusterd_volume_get_type_str(volinfo, &vol_type_str);
         if (ret) {
-                snprintf (err_str, sizeof(err_str),
-                          FMTSTR_CHECK_VOL_EXISTS, volname);
-                gf_log (this->name, GF_LOG_ERROR, FMTSTR_CHECK_VOL_EXISTS,
-                        volname);
-                goto out;
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_STATE_STR_GET_FAILED,
+                   "Failed to get type for volume: %s", volinfo->volname);
+            goto out;
         }
-        if (strcmp(key, "all")) {
-                exists = glusterd_check_option_exists (key, &key_fixed);
-                if (!exists) {
-                        snprintf (err_str, sizeof (err_str), "Option "
-                                  "with name: %s does not exist", key);
-                        gf_log (this->name, GF_LOG_ERROR, "%s",
-                                err_str);
-                        if (key_fixed)
-                                snprintf (err_str + ret,
-                                          sizeof (err_str) - ret,
-                                          "Did you mean %s?",
-                                          key_fixed);
-                        ret = -1;
-                        goto out;
-                }
-                if (key_fixed) {
-                        orig_key = key;
-                        key = key_fixed;
-                }
-                if (strcmp (key, "cluster.op-version") == 0) {
-                        sprintf (dict_key, "key%d", count);
-                        ret = dict_set_str(dict, dict_key, key);
-                        if (ret) {
-                                gf_log (this->name, GF_LOG_ERROR, "Failed to "
-                                       "set %s in dictionary", key);
-                                goto out;
-                        }
-                        sprintf (dict_key, "value%d", count);
-                        sprintf (op_version_buff, "%d", priv->op_version);
-                        ret = dict_set_str (dict, dict_key, op_version_buff);
-                        if (ret) {
-                                gf_log (this->name, GF_LOG_ERROR, "Failed to "
-                                        "set value for key %s in dictionary",
-                                        key);
-                                goto out;
-                        }
-                }
-                else if (strcmp (key, "config.memory-accounting") == 0) {
-                        sprintf (dict_key, "key%d", count);
-                        ret = dict_set_str(dict, dict_key, key);
-                        if (ret) {
-                                gf_log (this->name, GF_LOG_ERROR, "Failed to "
-                                       "set %s in dictionary", key);
-                                goto out;
-                        }
-                        sprintf (dict_key, "value%d", count);
-
-                        if (volinfo->memory_accounting)
-                                ret = dict_set_str(dict, dict_key,"Enabled");
-                        else
-                                ret = dict_set_str(dict, dict_key,"Disabled");
-                        if (ret) {
-                                gf_log (this->name, GF_LOG_ERROR, "Failed to "
-                                        "set value for key %s in dictionary",
-                                        key);
-                                goto out;
-                        }
-                }
-                else if (strcmp (key, "config.transport") == 0) {
-                        sprintf (dict_key, "key%d", count);
-                        ret = dict_set_str(dict, dict_key, key);
-                        if (ret) {
-                                gf_log (this->name, GF_LOG_ERROR, "Failed to "
-                                        "set %s in dictionary", key);
-                                goto out;
-                        }
-                        sprintf (dict_key, "value%d", count);
-
-                        if (volinfo->transport_type == GF_TRANSPORT_RDMA)
-                                ret = dict_set_str(dict, dict_key,"rdma");
-                        else if (volinfo->transport_type == GF_TRANSPORT_TCP)
-                                ret = dict_set_str(dict, dict_key,"tcp");
-                        else if (volinfo->transport_type ==
-                                 GF_TRANSPORT_BOTH_TCP_RDMA)
-                                ret = dict_set_str(dict, dict_key,"tcp,rdma");
-                        else
-                                ret = dict_set_str(dict, dict_key,"none");
-
-                        if (ret) {
-                                gf_log (this->name, GF_LOG_ERROR, "Failed to "
-                                        "set value for key %s in dictionary",
-                                        key);
-                                goto out;
-                        }
-                }
-                else {
-                        ret = glusterd_get_default_val_for_volopt (dict,
-                                                                  _gf_false,
-                                                                  key, orig_key,
-                                                                  volinfo->dict,
-                                                                  &rsp.op_errstr);
-                        if (ret && !rsp.op_errstr) {
-                                snprintf (err_str, sizeof(err_str),
-                                          "Failed to fetch the value of"
-                                          " %s, check log file for more"
-                                          " details", key);
-                        }
-                }
 
-                /* Request is for a single option, explicitly set count to 1
-                 * in the dictionary.
-                 */
-                ret = dict_set_int32 (dict, "count", 1);
-                if (ret) {
-                        gf_log (this->name, GF_LOG_ERROR, "Failed to set count "
-                                "value in the dictionary");
-                        goto out;
-                }
-        } else {
-                /* Handle the "all" volume option request */
-                ret = glusterd_get_default_val_for_volopt (dict, _gf_true, NULL,
-                                                           NULL, volinfo->dict,
-                                                           &rsp.op_errstr);
-                if (ret && !rsp.op_errstr) {
-                        snprintf (err_str, sizeof(err_str),
-                                  "Failed to fetch the value of all volume "
-                                  "options, check log file for more details");
-                }
+        ret = glusterd_volume_get_status_str(volinfo, vol_status_str);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_STATE_STR_GET_FAILED,
+                   "Failed to get status for volume: %s", volinfo->volname);
+            goto out;
+        }
 
+        ret = glusterd_volume_get_transport_type_str(volinfo,
+                                                     transport_type_str);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_STATE_STR_GET_FAILED,
+                   "Failed to get transport type for volume: %s",
+                   volinfo->volname);
+            goto out;
         }
 
-out:
+        ret = glusterd_volume_get_quorum_status_str(volinfo, quorum_status_str);
         if (ret) {
-                if (!rsp.op_errstr)
-                        rsp.op_errstr = err_str;
-                rsp.op_ret =  ret;
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_STATE_STR_GET_FAILED,
+                   "Failed to get quorum status for volume: %s",
+                   volinfo->volname);
+            goto out;
         }
-        else {
-                rsp.op_errstr = "";
-                rsp.op_ret = 0;
+
+        ret = glusterd_volume_get_rebalance_status_str(volinfo,
+                                                       rebal_status_str);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_STATE_STR_GET_FAILED,
+                   "Failed to get rebalance status for volume: %s",
+                   volinfo->volname);
+            goto out;
         }
 
-        ret = dict_allocate_and_serialize (dict, &rsp.dict.dict_val,
-                                           &rsp.dict.dict_len);
+        fprintf(fp, "Volume%d.name: %s\n", ++count, volinfo->volname);
 
-        glusterd_submit_reply (req, &rsp, NULL, 0, NULL,
-                               (xdrproc_t)xdr_gf_cli_rsp);
-        return ret;
-}
+        uuid_utoa_r(volinfo->volume_id, id_str);
+        fprintf(fp, "Volume%d.id: %s\n", count, id_str);
 
-int
-__glusterd_handle_get_vol_opt (rpcsvc_request_t *req)
-{
-        int32_t                         ret = -1;
-        gf_cli_req                      cli_req = {{0,}};
-        dict_t                          *dict = NULL;
-        char                            err_str[2048] = {0,};
-        xlator_t                        *this = NULL;
+        fprintf(fp, "Volume%d.type: %s\n", count, vol_type_str);
+        fprintf(fp, "Volume%d.transport_type: %s\n", count, transport_type_str);
+        fprintf(fp, "Volume%d.status: %s\n", count, vol_status_str);
+        fprintf(fp, "Volume%d.profile_enabled: %d\n", count,
+                glusterd_is_profile_on(volinfo));
+        fprintf(fp, "Volume%d.brickcount: %d\n", count, volinfo->brick_count);
+
+        count_bkp = count;
+        count = 0;
+        cds_list_for_each_entry(brickinfo, &volinfo->bricks, brick_list)
+        {
+            fprintf(fp, "Volume%d.Brick%d.path: %s:%s\n", count_bkp, ++count,
+                    brickinfo->hostname, brickinfo->path);
+            fprintf(fp, "Volume%d.Brick%d.hostname: %s\n", count_bkp, count,
+                    brickinfo->hostname);
+            /* Determine which one is the arbiter brick */
+            if (volinfo->arbiter_count == 1) {
+                if (count % volinfo->replica_count == 0) {
+                    fprintf(fp,
+                            "Volume%d.Brick%d."
+                            "is_arbiter: 1\n",
+                            count_bkp, count);
+                }
+            }
+            /* Add following information only for bricks
+             *  local to current node */
+            if (gf_uuid_compare(brickinfo->uuid, MY_UUID))
+                continue;
+            fprintf(fp, "Volume%d.Brick%d.port: %d\n", count_bkp, count,
+                    brickinfo->port);
+            fprintf(fp, "Volume%d.Brick%d.rdma_port: %d\n", count_bkp, count,
+                    brickinfo->rdma_port);
+            fprintf(fp, "Volume%d.Brick%d.port_registered: %d\n", count_bkp,
+                    count, brickinfo->port_registered);
+            glusterd_brick_get_status_str(brickinfo, brick_status_str);
+            fprintf(fp, "Volume%d.Brick%d.status: %s\n", count_bkp, count,
+                    brick_status_str);
+
+            ret = sys_statvfs(brickinfo->path, &brickstat);
+            if (ret) {
+                gf_msg(this->name, GF_LOG_ERROR, errno, GD_MSG_FILE_OP_FAILED,
+                       "statfs error: %s ", strerror(errno));
+                memfree = 0;
+                memtotal = 0;
+            } else {
+                memfree = brickstat.f_bfree * brickstat.f_bsize;
+                memtotal = brickstat.f_blocks * brickstat.f_bsize;
+            }
+
+            fprintf(fp, "Volume%d.Brick%d.spacefree: %" PRIu64 "Bytes\n",
+                    count_bkp, count, memfree);
+            fprintf(fp, "Volume%d.Brick%d.spacetotal: %" PRIu64 "Bytes\n",
+                    count_bkp, count, memtotal);
+
+            if (get_state_cmd != GF_CLI_GET_STATE_DETAIL)
+                continue;
+
+            ret = glusterd_print_client_details(fp, dict, volinfo, count_bkp,
+                                                brickinfo, count);
+            if (ret) {
+                gf_msg(this->name, GF_LOG_ERROR, 0,
+                       GD_MSG_CLIENTS_GET_STATE_FAILED,
+                       "Failed to get client details");
+                goto out;
+            }
+        }
+
+        count = count_bkp;
+
+        ret = glusterd_print_snapinfo_by_vol(fp, volinfo, count);
+        if (ret)
+            goto out;
+
+        fprintf(fp, "Volume%d.snap_count: %" PRIu64 "\n", count,
+                volinfo->snap_count);
+        fprintf(fp, "Volume%d.stripe_count: %d\n", count,
+                volinfo->stripe_count);
+        fprintf(fp, "Volume%d.replica_count: %d\n", count,
+                volinfo->replica_count);
+        fprintf(fp, "Volume%d.subvol_count: %d\n", count,
+                volinfo->subvol_count);
+        fprintf(fp, "Volume%d.arbiter_count: %d\n", count,
+                volinfo->arbiter_count);
+        fprintf(fp, "Volume%d.disperse_count: %d\n", count,
+                volinfo->disperse_count);
+        fprintf(fp, "Volume%d.redundancy_count: %d\n", count,
+                volinfo->redundancy_count);
+        fprintf(fp, "Volume%d.quorum_status: %s\n", count, quorum_status_str);
+
+        fprintf(fp, "Volume%d.snapd_svc.online_status: %s\n", count,
+                volinfo->snapd.svc.online ? "Online" : "Offline");
+        fprintf(fp, "Volume%d.snapd_svc.inited: %s\n", count,
+                volinfo->snapd.svc.inited ? "True" : "False");
+
+        uuid_utoa_r(volinfo->rebal.rebalance_id, id_str);
+        char *rebal_data = gf_uint64_2human_readable(
+            volinfo->rebal.rebalance_data);
+
+        fprintf(fp, "Volume%d.rebalance.id: %s\n", count, id_str);
+        fprintf(fp, "Volume%d.rebalance.status: %s\n", count, rebal_status_str);
+        fprintf(fp, "Volume%d.rebalance.failures: %" PRIu64 "\n", count,
+                volinfo->rebal.rebalance_failures);
+        fprintf(fp, "Volume%d.rebalance.skipped: %" PRIu64 "\n", count,
+                volinfo->rebal.skipped_files);
+        fprintf(fp, "Volume%d.rebalance.lookedup: %" PRIu64 "\n", count,
+                volinfo->rebal.lookedup_files);
+        fprintf(fp, "Volume%d.rebalance.files: %" PRIu64 "\n", count,
+                volinfo->rebal.rebalance_files);
+        fprintf(fp, "Volume%d.rebalance.data: %s\n", count, rebal_data);
+        fprintf(fp, "Volume%d.time_left: %" PRIu64 "\n", count,
+                volinfo->rebal.time_left);
+
+        GF_FREE(rebal_data);
+
+        fprintf(fp, "Volume%d.shd_svc.online_status: %s\n", count,
+                volinfo->shd.svc.online ? "Online" : "Offline");
+        fprintf(fp, "Volume%d.shd_svc.inited: %s\n", count,
+                volinfo->shd.svc.inited ? "True" : "False");
+
+        if (volinfo->rep_brick.src_brick && volinfo->rep_brick.dst_brick) {
+            fprintf(fp, "Volume%d.replace_brick.src: %s:%s\n", count,
+                    volinfo->rep_brick.src_brick->hostname,
+                    volinfo->rep_brick.src_brick->path);
+            fprintf(fp, "Volume%d.replace_brick.dest: %s:%s\n", count,
+                    volinfo->rep_brick.dst_brick->hostname,
+                    volinfo->rep_brick.dst_brick->path);
+        }
+
+        volcount = count;
+        ret = glusterd_print_gsync_status_by_vol(fp, volinfo);
+        if (ret)
+            goto out;
+
+        if (volinfo->dict)
+            dict_foreach(volinfo->dict, glusterd_print_volume_options, fp);
+
+        fprintf(fp, "\n");
+    }
+
+    count = 0;
+
+    fprintf(fp, "\n[Services]\n");
+#ifdef BUILD_GNFS
+    if (priv->nfs_svc.inited) {
+        fprintf(fp, "svc%d.name: %s\n", ++count, priv->nfs_svc.name);
+        fprintf(fp, "svc%d.online_status: %s\n\n", count,
+                priv->nfs_svc.online ? "Online" : "Offline");
+    }
+#endif
+    if (priv->bitd_svc.inited) {
+        fprintf(fp, "svc%d.name: %s\n", ++count, priv->bitd_svc.name);
+        fprintf(fp, "svc%d.online_status: %s\n\n", count,
+                priv->bitd_svc.online ? "Online" : "Offline");
+    }
+
+    if (priv->scrub_svc.inited) {
+        fprintf(fp, "svc%d.name: %s\n", ++count, priv->scrub_svc.name);
+        fprintf(fp, "svc%d.online_status: %s\n\n", count,
+                priv->scrub_svc.online ? "Online" : "Offline");
+    }
+
+    if (priv->quotad_svc.inited) {
+        fprintf(fp, "svc%d.name: %s\n", ++count, priv->quotad_svc.name);
+        fprintf(fp, "svc%d.online_status: %s\n\n", count,
+                priv->quotad_svc.online ? "Online" : "Offline");
+    }
+
+    fprintf(fp, "\n[Misc]\n");
+    if (priv->pmap) {
+        fprintf(fp, "Base port: %d\n", priv->pmap->base_port);
+        fprintf(fp, "Last allocated port: %d\n", priv->pmap->last_alloc);
+    }
+out:
+
+    if (fp)
+        fclose(fp);
+
+    rsp.op_ret = ret;
+    if (rsp.op_errstr == NULL)
+        rsp.op_errstr = err_str;
 
-        this = THIS;
-        GF_ASSERT (this);
+    ret = dict_allocate_and_serialize(dict, &rsp.dict.dict_val,
+                                      &rsp.dict.dict_len);
+    glusterd_to_cli(req, &rsp, NULL, 0, NULL, (xdrproc_t)xdr_gf_cli_rsp, dict);
+    GF_FREE(rsp.dict.dict_val);
 
-        GF_ASSERT (req);
+    return ret;
+}
+
+static int
+__glusterd_handle_get_state(rpcsvc_request_t *req)
+{
+    int32_t ret = -1;
+    gf_cli_req cli_req = {
+        {
+            0,
+        },
+    };
+    dict_t *dict = NULL;
+    char err_str[64] = {
+        0,
+    };
+    xlator_t *this = NULL;
+
+    this = THIS;
+    GF_VALIDATE_OR_GOTO(THIS->name, this, out);
+    GF_VALIDATE_OR_GOTO(this->name, req, out);
+
+    gf_msg(this->name, GF_LOG_INFO, 0, GD_MSG_DAEMON_STATE_REQ_RCVD,
+           "Received request to get state for glusterd");
+
+    ret = xdr_to_generic(req->msg[0], &cli_req, (xdrproc_t)xdr_gf_cli_req);
+    if (ret < 0) {
+        snprintf(err_str, sizeof(err_str),
+                 "Failed to decode "
+                 "request received from cli");
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_REQ_DECODE_FAIL, "%s",
+               err_str);
+        req->rpc_err = GARBAGE_ARGS;
+        goto out;
+    }
+
+    if (cli_req.dict.dict_len) {
+        /* Unserialize the dictionary */
+        dict = dict_new();
 
-        ret = xdr_to_generic (req->msg[0], &cli_req, (xdrproc_t)xdr_gf_cli_req);
+        ret = dict_unserialize(cli_req.dict.dict_val, cli_req.dict.dict_len,
+                               &dict);
         if (ret < 0) {
-                snprintf (err_str, sizeof (err_str), "Failed to decode "
-                          "request received from cli");
-                gf_log (this->name, GF_LOG_ERROR, "%s", err_str);
-                req->rpc_err = GARBAGE_ARGS;
-                goto out;
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_UNSERIALIZE_FAIL,
+                   "failed to "
+                   "unserialize req-buffer to dictionary");
+            snprintf(err_str, sizeof(err_str),
+                     "Unable to decode"
+                     " the command");
+            goto out;
+        } else {
+            dict->extra_stdfree = cli_req.dict.dict_val;
         }
+    }
 
-        if (cli_req.dict.dict_len) {
-                /* Unserialize the dictionary */
-                dict  = dict_new ();
-
-                ret = dict_unserialize (cli_req.dict.dict_val,
-                                        cli_req.dict.dict_len,
-                                        &dict);
-                if (ret < 0) {
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "failed to "
-                                "unserialize req-buffer to dictionary");
-                        snprintf (err_str, sizeof (err_str), "Unable to decode "
-                                  "the command");
-                        goto out;
-                } else {
-                        dict->extra_stdfree = cli_req.dict.dict_val;
-                }
-        }
-        ret = glusterd_get_volume_opts (req, dict);
+    ret = glusterd_get_state(req, dict);
 
 out:
-        if (dict)
-                dict_unref (dict);
-
-        return ret;
+    if (dict && ret) {
+        /*
+         * When glusterd_to_cli (called from glusterd_get_state)
+         * succeeds, it frees the dict for us, so this would be a
+         * double free, but in other cases it's our responsibility.
+         */
+        dict_unref(dict);
+    }
+    return ret;
 }
 
 int
-glusterd_handle_get_vol_opt (rpcsvc_request_t *req)
+glusterd_handle_get_state(rpcsvc_request_t *req)
 {
-        return glusterd_big_locked_handler (req, __glusterd_handle_get_vol_opt);
+    return glusterd_big_locked_handler(req, __glusterd_handle_get_state);
 }
+
 static int
-get_brickinfo_from_brickid (char *brickid, glusterd_brickinfo_t **brickinfo)
+get_brickinfo_from_brickid(char *brickid, glusterd_brickinfo_t **brickinfo)
 {
-        glusterd_volinfo_t      *volinfo    = NULL;
-        char                    *volid_str  = NULL;
-        char                    *brick      = NULL;
-        char                    *brickid_dup = NULL;
-        uuid_t                  volid       = {0};
-        int                     ret         = -1;
-
-        brickid_dup = gf_strdup (brickid);
-        if (!brickid_dup)
-                goto out;
-
-        volid_str = brickid_dup;
-        brick = strchr (brickid_dup, ':');
-        if (!volid_str || !brick)
-                goto out;
-
-        *brick = '\0';
-        brick++;
-        gf_uuid_parse (volid_str, volid);
-        ret = glusterd_volinfo_find_by_volume_id (volid, &volinfo);
-        if (ret) {
-                /* Check if it a snapshot volume */
-                ret = glusterd_snap_volinfo_find_by_volume_id (volid, &volinfo);
-                if (ret)
-                        goto out;
-        }
-
-        ret = glusterd_volume_brickinfo_get_by_brick (brick, volinfo,
-                                                      brickinfo);
+    glusterd_volinfo_t *volinfo = NULL;
+    char *volid_str = NULL;
+    char *brick = NULL;
+    char *brickid_dup = NULL;
+    uuid_t volid = {0};
+    int ret = -1;
+
+    xlator_t *this = THIS;
+    GF_ASSERT(this);
+
+    brickid_dup = gf_strdup(brickid);
+    if (!brickid_dup) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_STRDUP_FAILED,
+                "brick_id=%s", brickid, NULL);
+        goto out;
+    }
+
+    volid_str = brickid_dup;
+    brick = strchr(brickid_dup, ':');
+    if (!volid_str) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_STRCHR_FAIL, NULL);
+        goto out;
+    }
+
+    if (!brick) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_STRCHR_FAIL, NULL);
+        goto out;
+    }
+
+    *brick = '\0';
+    brick++;
+    gf_uuid_parse(volid_str, volid);
+    ret = glusterd_volinfo_find_by_volume_id(volid, &volinfo);
+    if (ret) {
+        /* Check if it a snapshot volume */
+        ret = glusterd_snap_volinfo_find_by_volume_id(volid, &volinfo);
         if (ret)
-                goto out;
+            goto out;
+    }
 
-        ret = 0;
+    ret = glusterd_volume_brickinfo_get_by_brick(brick, volinfo, brickinfo,
+                                                 _gf_false);
+    if (ret)
+        goto out;
+
+    ret = 0;
 out:
-        GF_FREE (brickid_dup);
-        return ret;
+    GF_FREE(brickid_dup);
+    return ret;
 }
 
+static int gd_stale_rpc_disconnect_log;
+
 int
-__glusterd_brick_rpc_notify (struct rpc_clnt *rpc, void *mydata,
-                             rpc_clnt_event_t event, void *data)
+__glusterd_brick_rpc_notify(struct rpc_clnt *rpc, void *mydata,
+                            rpc_clnt_event_t event, void *data)
 {
-        char                    *brickid           = NULL;
-        int                      ret               = 0;
-        glusterd_conf_t         *conf              = NULL;
-        glusterd_brickinfo_t    *brickinfo         = NULL;
-        glusterd_volinfo_t      *volinfo           = NULL;
-        xlator_t                *this              = NULL;
-
-        brickid = mydata;
-        if (!brickid)
-                return 0;
-
-        ret = get_brickinfo_from_brickid (brickid, &brickinfo);
-        if (ret)
-                return 0;
+    char *brickid = NULL;
+    int ret = 0;
+    glusterd_conf_t *conf = NULL;
+    glusterd_brickinfo_t *brickinfo = NULL;
+    glusterd_volinfo_t *volinfo = NULL;
+    xlator_t *this = NULL;
+    int32_t pid = -1;
+    glusterd_brickinfo_t *brickinfo_tmp = NULL;
+    glusterd_brick_proc_t *brick_proc = NULL;
+    char pidfile[PATH_MAX] = {0};
+    char *brickpath = NULL;
+    gf_boolean_t is_service_running = _gf_true;
+
+    brickid = mydata;
+    if (!brickid)
+        return 0;
 
-        this = THIS;
-        GF_ASSERT (this);
-        conf = this->private;
-        GF_ASSERT (conf);
+    ret = get_brickinfo_from_brickid(brickid, &brickinfo);
+    if (ret)
+        return 0;
+
+    this = THIS;
+    GF_ASSERT(this);
+    conf = this->private;
+    GF_ASSERT(conf);
 
-        switch (event) {
+    switch (event) {
         case RPC_CLNT_CONNECT:
-                /* If a node on coming back up, already starts a brick
-                 * before the handshake, and the notification comes after
-                 * the handshake is done, then we need to check if this
-                 * is a restored brick with a snapshot pending. If so, we
-                 * need to stop the brick
-                 */
-                if (brickinfo->snap_status == -1) {
-                        gf_log (this->name, GF_LOG_INFO,
-                                "Snapshot is pending on %s:%s. "
-                                "Hence not starting the brick",
-                                brickinfo->hostname,
-                                brickinfo->path);
-                        ret = get_volinfo_from_brickid (brickid, &volinfo);
-                        if (ret) {
-                                gf_log (this->name, GF_LOG_ERROR,
-                                        "Failed to get volinfo from "
-                                        "brickid(%s)", brickid);
-                                goto out;
-                        }
-
-                        ret = glusterd_brick_stop (volinfo, brickinfo,
-                                                   _gf_false);
-                        if (ret) {
-                                gf_log (THIS->name, GF_LOG_ERROR,
-                                        "Unable to stop %s:%s",
-                                        brickinfo->hostname, brickinfo->path);
-                                goto out;
-                        }
-
-                        break;
+            ret = get_volinfo_from_brickid(brickid, &volinfo);
+            if (ret) {
+                gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_VOLINFO_GET_FAIL,
+                       "Failed to get volinfo from "
+                       "brickid(%s)",
+                       brickid);
+                goto out;
+            }
+            /* If a node on coming back up, already starts a brick
+             * before the handshake, and the notification comes after
+             * the handshake is done, then we need to check if this
+             * is a restored brick with a snapshot pending. If so, we
+             * need to stop the brick
+             */
+            if (brickinfo->snap_status == -1) {
+                gf_msg(this->name, GF_LOG_INFO, 0, GD_MSG_SNAPSHOT_PENDING,
+                       "Snapshot is pending on %s:%s. "
+                       "Hence not starting the brick",
+                       brickinfo->hostname, brickinfo->path);
+                ret = glusterd_brick_stop(volinfo, brickinfo, _gf_false);
+                if (ret) {
+                    gf_msg(THIS->name, GF_LOG_ERROR, 0, GD_MSG_BRICK_STOP_FAIL,
+                           "Unable to stop %s:%s", brickinfo->hostname,
+                           brickinfo->path);
+                    goto out;
                 }
-                gf_log (this->name, GF_LOG_DEBUG, "Connected to %s:%s",
-                        brickinfo->hostname, brickinfo->path);
-                glusterd_set_brick_status (brickinfo, GF_BRICK_STARTED);
-                ret = default_notify (this, GF_EVENT_CHILD_UP, NULL);
 
                 break;
+            }
+            gf_msg_debug(this->name, 0, "Connected to %s:%s",
+                         brickinfo->hostname, brickinfo->path);
 
-        case RPC_CLNT_DISCONNECT:
-                if (glusterd_is_brick_started (brickinfo))
-                        gf_msg (this->name, GF_LOG_INFO, 0,
-                                GD_MSG_BRICK_DISCONNECTED,
-                                "Brick %s:%s has disconnected from glusterd.",
-                                brickinfo->hostname, brickinfo->path);
+            glusterd_set_brick_status(brickinfo, GF_BRICK_STARTED);
+
+            gf_event(EVENT_BRICK_CONNECTED, "peer=%s;volume=%s;brick=%s",
+                     brickinfo->hostname, volinfo->volname, brickinfo->path);
+
+            ret = default_notify(this, GF_EVENT_CHILD_UP, NULL);
+
+            break;
 
-                glusterd_set_brick_status (brickinfo, GF_BRICK_STOPPED);
+        case RPC_CLNT_DISCONNECT:
+            if (rpc != brickinfo->rpc) {
+                /*
+                 * There used to be a bunch of races in the volume
+                 * start/stop code that could result in us getting here
+                 * and setting the brick status incorrectly.  Many of
+                 * those have been fixed or avoided, but just in case
+                 * any are still left it doesn't hurt to keep the extra
+                 * check and avoid further damage.
+                 */
+                GF_LOG_OCCASIONALLY(gd_stale_rpc_disconnect_log, this->name,
+                                    GF_LOG_WARNING,
+                                    "got disconnect from stale rpc on "
+                                    "%s",
+                                    brickinfo->path);
                 break;
+            }
+            if (glusterd_is_brick_started(brickinfo)) {
+                gf_msg(this->name, GF_LOG_INFO, 0, GD_MSG_BRICK_DISCONNECTED,
+                       "Brick %s:%s has disconnected from glusterd.",
+                       brickinfo->hostname, brickinfo->path);
+
+                ret = get_volinfo_from_brickid(brickid, &volinfo);
+                if (ret) {
+                    gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_VOLINFO_GET_FAIL,
+                           "Failed to get volinfo from "
+                           "brickid(%s)",
+                           brickid);
+                    goto out;
+                }
+                gf_event(EVENT_BRICK_DISCONNECTED, "peer=%s;volume=%s;brick=%s",
+                         brickinfo->hostname, volinfo->volname,
+                         brickinfo->path);
+                /* In case of an abrupt shutdown of a brick PMAP_SIGNOUT
+                 * event is not received by glusterd which can lead to a
+                 * stale port entry in glusterd, so forcibly clean up
+                 * the same if the process is not running sometime
+                 * gf_is_service_running true so to ensure about brick instance
+                 * call search_brick_path_from_proc
+                 */
+                GLUSTERD_GET_BRICK_PIDFILE(pidfile, volinfo, brickinfo, conf);
+                is_service_running = gf_is_service_running(pidfile, &pid);
+                if (pid > 0)
+                    brickpath = search_brick_path_from_proc(pid,
+                                                            brickinfo->path);
+                if (!is_service_running || !brickpath) {
+                    ret = pmap_registry_remove(
+                        THIS, brickinfo->port, brickinfo->path,
+                        GF_PMAP_PORT_BRICKSERVER, NULL, _gf_true);
+                    if (ret) {
+                        gf_msg(this->name, GF_LOG_WARNING,
+                               GD_MSG_PMAP_REGISTRY_REMOVE_FAIL, 0,
+                               "Failed to remove pmap "
+                               "registry for port %d for "
+                               "brick %s",
+                               brickinfo->port, brickinfo->path);
+                        ret = 0;
+                    }
+                }
+            }
+
+            if (brickpath)
+                GF_FREE(brickpath);
+
+            if (is_brick_mx_enabled() && glusterd_is_brick_started(brickinfo)) {
+                brick_proc = brickinfo->brick_proc;
+                if (!brick_proc)
+                    break;
+                cds_list_for_each_entry(brickinfo_tmp, &brick_proc->bricks,
+                                        mux_bricks)
+                {
+                    glusterd_set_brick_status(brickinfo_tmp, GF_BRICK_STOPPED);
+                    brickinfo_tmp->start_triggered = _gf_false;
+                    /* When bricks are stopped, ports also need to
+                     * be cleaned up
+                     */
+                    pmap_registry_remove(
+                        THIS, brickinfo_tmp->port, brickinfo_tmp->path,
+                        GF_PMAP_PORT_BRICKSERVER, NULL, _gf_true);
+                }
+            } else {
+                glusterd_set_brick_status(brickinfo, GF_BRICK_STOPPED);
+                brickinfo->start_triggered = _gf_false;
+            }
+            break;
 
         case RPC_CLNT_DESTROY:
-                GF_FREE (mydata);
-                mydata = NULL;
-                break;
+            GF_FREE(mydata);
+            mydata = NULL;
+            break;
         default:
-                gf_log (this->name, GF_LOG_TRACE,
-                        "got some other RPC event %d", event);
-                break;
-        }
+            gf_msg_trace(this->name, 0, "got some other RPC event %d", event);
+            break;
+    }
 
 out:
-        return ret;
+    return ret;
 }
 
 int
-glusterd_brick_rpc_notify (struct rpc_clnt *rpc, void *mydata,
-                           rpc_clnt_event_t event, void *data)
+glusterd_brick_rpc_notify(struct rpc_clnt *rpc, void *mydata,
+                          rpc_clnt_event_t event, void *data)
 {
-        return glusterd_big_locked_notify (rpc, mydata, event, data,
-                                           __glusterd_brick_rpc_notify);
+    return glusterd_big_locked_notify(rpc, mydata, event, data,
+                                      __glusterd_brick_rpc_notify);
 }
 
 int
-glusterd_friend_remove_notify (glusterd_peerctx_t *peerctx)
+glusterd_friend_remove_notify(glusterd_peerctx_t *peerctx, int32_t op_errno)
 {
-        int                             ret = -1;
-        glusterd_friend_sm_event_t      *new_event = NULL;
-        glusterd_peerinfo_t             *peerinfo = NULL;
-        rpcsvc_request_t                *req = NULL;
-        char                            *errstr = NULL;
-        dict_t                          *dict = NULL;
+    int ret = -1;
+    glusterd_friend_sm_event_t *new_event = NULL;
+    glusterd_peerinfo_t *peerinfo = NULL;
+    rpcsvc_request_t *req = NULL;
+    char *errstr = NULL;
+    dict_t *dict = NULL;
+
+    GF_ASSERT(peerctx);
+
+    RCU_READ_LOCK;
+    peerinfo = glusterd_peerinfo_find_by_generation(peerctx->peerinfo_gen);
+    if (!peerinfo) {
+        gf_msg_debug(THIS->name, 0,
+                     "Could not find peer %s(%s). "
+                     "Peer could have been deleted.",
+                     peerctx->peername, uuid_utoa(peerctx->peerid));
+        ret = 0;
+        goto out;
+    }
 
-        GF_ASSERT (peerctx);
+    req = peerctx->args.req;
+    dict = peerctx->args.dict;
+    errstr = peerctx->errstr;
 
-        rcu_read_lock ();
-        peerinfo = glusterd_peerinfo_find (peerctx->peerid, peerctx->peername);
-        if (!peerinfo) {
-                gf_log (THIS->name, GF_LOG_DEBUG, "Could not find peer %s(%s). "
-                        "Peer could have been deleted.", peerctx->peername,
-                        uuid_utoa (peerctx->peerid));
-                ret = 0;
-                goto out;
+    ret = glusterd_friend_sm_new_event(GD_FRIEND_EVENT_REMOVE_FRIEND,
+                                       &new_event);
+    if (!ret) {
+        if (!req) {
+            gf_msg(THIS->name, GF_LOG_WARNING, 0, GD_MSG_EVENT_NEW_GET_FAIL,
+                   "Unable to find the request for responding "
+                   "to User (%s)",
+                   peerinfo->hostname);
+            goto out;
         }
 
-        req = peerctx->args.req;
-        dict = peerctx->args.dict;
-        errstr = peerctx->errstr;
-
-        ret = glusterd_friend_sm_new_event (GD_FRIEND_EVENT_REMOVE_FRIEND,
-                                            &new_event);
-        if (!ret) {
-                if (!req) {
-                        gf_log (THIS->name, GF_LOG_WARNING,
-                                "Unable to find the request for responding "
-                                "to User (%s)", peerinfo->hostname);
-                        goto out;
-                }
-
-                glusterd_xfer_cli_probe_resp (req, -1, ENOTCONN, errstr,
-                                              peerinfo->hostname,
-                                              peerinfo->port, dict);
+        glusterd_xfer_cli_probe_resp(req, -1, op_errno, errstr,
+                                     peerinfo->hostname, peerinfo->port, dict);
 
-                new_event->peername = gf_strdup (peerinfo->hostname);
-                gf_uuid_copy (new_event->peerid, peerinfo->uuid);
-                ret = glusterd_friend_sm_inject_event (new_event);
+        new_event->peername = gf_strdup(peerinfo->hostname);
+        gf_uuid_copy(new_event->peerid, peerinfo->uuid);
+        ret = glusterd_friend_sm_inject_event(new_event);
 
-        } else {
-                gf_log ("glusterd", GF_LOG_ERROR,
-                        "Unable to create event for removing peer %s",
-                        peerinfo->hostname);
-        }
+    } else {
+        gf_msg("glusterd", GF_LOG_ERROR, 0, GD_MSG_EVENT_INJECT_FAIL,
+               "Unable to create event for removing peer %s",
+               peerinfo->hostname);
+    }
 
 out:
-        rcu_read_unlock ();
-        return ret;
+    RCU_READ_UNLOCK;
+    return ret;
 }
 
 int
-__glusterd_peer_rpc_notify (struct rpc_clnt *rpc, void *mydata,
-                            rpc_clnt_event_t event, void *data)
+__glusterd_peer_rpc_notify(struct rpc_clnt *rpc, void *mydata,
+                           rpc_clnt_event_t event, void *data)
 {
-        xlator_t             *this        = NULL;
-        glusterd_conf_t      *conf        = NULL;
-        int                   ret         = 0;
-        glusterd_peerinfo_t  *peerinfo    = NULL;
-        glusterd_peerctx_t   *peerctx     = NULL;
-        gf_boolean_t         quorum_action = _gf_false;
-        glusterd_volinfo_t   *volinfo     = NULL;
-        uuid_t               uuid;
-
-        peerctx = mydata;
-        if (!peerctx)
-                return 0;
-
-        this = THIS;
-        conf = this->private;
+    xlator_t *this = NULL;
+    glusterd_conf_t *conf = NULL;
+    int ret = 0;
+    int32_t op_errno = ENOTCONN;
+    glusterd_peerinfo_t *peerinfo = NULL;
+    glusterd_peerctx_t *peerctx = NULL;
+    gf_boolean_t quorum_action = _gf_false;
+    glusterd_volinfo_t *volinfo = NULL;
+    glusterfs_ctx_t *ctx = NULL;
+
+    uuid_t uuid;
+
+    peerctx = mydata;
+    if (!peerctx)
+        return 0;
 
-        if (RPC_CLNT_DESTROY == event) {
-                GF_FREE (peerctx->errstr);
-                GF_FREE (peerctx->peername);
-                GF_FREE (peerctx);
-                return 0;
-        }
+    this = THIS;
+    conf = this->private;
 
-        rcu_read_lock ();
+    switch (event) {
+        case RPC_CLNT_DESTROY:
+            GF_FREE(peerctx->errstr);
+            GF_FREE(peerctx->peername);
+            GF_FREE(peerctx);
+            return 0;
+        case RPC_CLNT_PING:
+            return 0;
+        default:
+            break;
+    }
+    ctx = this->ctx;
+    GF_VALIDATE_OR_GOTO(this->name, ctx, out);
+    if (ctx->cleanup_started) {
+        gf_log(this->name, GF_LOG_INFO,
+               "glusterd already received a SIGTERM, "
+               "dropping the event %d for peer %s",
+               event, peerctx->peername);
+        return 0;
+    }
+    RCU_READ_LOCK;
+
+    peerinfo = glusterd_peerinfo_find_by_generation(peerctx->peerinfo_gen);
+    if (!peerinfo) {
+        /* Peerinfo should be available at this point if its a connect
+         * event. Not finding it means that something terrible has
+         * happened. For non-connect event we might end up having a null
+         * peerinfo, so log at debug level.
+         */
+        gf_msg(THIS->name,
+               (RPC_CLNT_CONNECT == event) ? GF_LOG_CRITICAL : GF_LOG_DEBUG,
+               ENOENT, GD_MSG_PEER_NOT_FOUND,
+               "Could not find peer "
+               "%s(%s)",
+               peerctx->peername, uuid_utoa(peerctx->peerid));
 
-        peerinfo = glusterd_peerinfo_find (peerctx->peerid, peerctx->peername);
-        if (!peerinfo) {
-                /* Peerinfo should be available at this point. Not finding it
-                 * means that something terrible has happened
-                 */
-                gf_log (THIS->name, GF_LOG_CRITICAL, "Could not find peer "
-                        "%s(%s)", peerctx->peername, uuid_utoa (peerctx->peerid));
-                ret = -1;
-                goto out;
+        if (RPC_CLNT_CONNECT == event) {
+            gf_event(EVENT_PEER_NOT_FOUND, "peer=%s;uuid=%s", peerctx->peername,
+                     uuid_utoa(peerctx->peerid));
         }
-
-        switch (event) {
-        case RPC_CLNT_CONNECT:
-        {
-                rpc_clnt_set_connected (&rpc->conn);
-                gf_log (this->name, GF_LOG_DEBUG, "got RPC_CLNT_CONNECT");
-                peerinfo->connected = 1;
-                peerinfo->quorum_action = _gf_true;
-
-                ret = glusterd_peer_dump_version (this, rpc, peerctx);
-                if (ret)
-                        gf_log ("", GF_LOG_ERROR, "glusterd handshake failed");
+        ret = -1;
+        goto out;
+    }
+
+    switch (event) {
+        case RPC_CLNT_CONNECT: {
+            gf_msg_debug(this->name, 0, "got RPC_CLNT_CONNECT");
+            peerinfo->connected = 1;
+            peerinfo->quorum_action = _gf_true;
+            peerinfo->generation = uatomic_add_return(&conf->generation, 1);
+            peerctx->peerinfo_gen = peerinfo->generation;
+            /* EVENT_PEER_CONNECT will only be sent if peerctx->uuid is not
+             * NULL, otherwise it indicates this RPC_CLNT_CONNECT is from a
+             * peer probe trigger and given we already generate an event for
+             * peer probe this would be unnecessary.
+             */
+            if (!gf_uuid_is_null(peerinfo->uuid)) {
+                gf_event(EVENT_PEER_CONNECT, "host=%s;uuid=%s",
+                         peerinfo->hostname, uuid_utoa(peerinfo->uuid));
+            }
+            ret = glusterd_peer_dump_version(this, rpc, peerctx);
+            if (ret)
+                gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_HANDSHAKE_FAILED,
+                       "glusterd handshake failed");
+            break;
+        }
+
+        case RPC_CLNT_DISCONNECT: {
+            /* If DISCONNECT event is already processed, skip the further
+             * ones
+             */
+            if (is_rpc_clnt_disconnected(&rpc->conn))
                 break;
-        }
-
-        case RPC_CLNT_DISCONNECT:
-        {
-                rpc_clnt_unset_connected (&rpc->conn);
-                gf_msg (this->name, GF_LOG_INFO, 0,
-                        GD_MSG_PEER_DISCONNECTED,
-                        "Peer <%s> (<%s>), in state <%s>, has disconnected "
-                        "from glusterd.",
-                        peerinfo->hostname, uuid_utoa (peerinfo->uuid),
-                        glusterd_friend_sm_state_name_get (peerinfo->state.state));
-
-                if (peerinfo->connected) {
-                        if (conf->op_version < GD_OP_VERSION_3_6_0) {
-                                glusterd_get_lock_owner (&uuid);
-                                if (!gf_uuid_is_null (uuid) &&
-                                    !gf_uuid_compare (peerinfo->uuid, uuid))
-                                        glusterd_unlock (peerinfo->uuid);
-                        } else {
-                                cds_list_for_each_entry (volinfo,
-                                                         &conf->volumes,
-                                                         vol_list) {
-                                        ret = glusterd_mgmt_v3_unlock
-                                                    (volinfo->volname,
-                                                     peerinfo->uuid,
-                                                     "vol");
-                                        if (ret)
-                                                gf_log (this->name,
-                                                        GF_LOG_TRACE,
-                                                        "Lock not released "
-                                                        "for %s",
-                                                         volinfo->volname);
-                                }
-                        }
 
-                        ret = 0;
+            gf_msg(this->name, GF_LOG_INFO, 0, GD_MSG_PEER_DISCONNECTED,
+                   "Peer <%s> (<%s>), in state <%s>, has disconnected "
+                   "from glusterd.",
+                   peerinfo->hostname, uuid_utoa(peerinfo->uuid),
+                   glusterd_friend_sm_state_name_get(peerinfo->state.state));
+            gf_event(EVENT_PEER_DISCONNECT, "peer=%s;uuid=%s;state=%s",
+                     peerinfo->hostname, uuid_utoa(peerinfo->uuid),
+                     glusterd_friend_sm_state_name_get(peerinfo->state.state));
+
+            if (peerinfo->connected) {
+                if (conf->op_version < GD_OP_VERSION_3_6_0) {
+                    glusterd_get_lock_owner(&uuid);
+                    if (!gf_uuid_is_null(uuid) &&
+                        !gf_uuid_compare(peerinfo->uuid, uuid))
+                        glusterd_unlock(peerinfo->uuid);
+                } else {
+                    cds_list_for_each_entry(volinfo, &conf->volumes, vol_list)
+                    {
+                        ret = glusterd_mgmt_v3_unlock(volinfo->volname,
+                                                      peerinfo->uuid, "vol");
+                        if (ret)
+                            gf_msg(this->name, GF_LOG_WARNING, 0,
+                                   GD_MSG_MGMTV3_UNLOCK_FAIL,
+                                   "Lock not released "
+                                   "for %s",
+                                   volinfo->volname);
+                    }
                 }
 
-                if ((peerinfo->quorum_contrib != QUORUM_DOWN) &&
-                    (peerinfo->state.state == GD_FRIEND_STATE_BEFRIENDED)) {
-                        peerinfo->quorum_contrib = QUORUM_DOWN;
-                        quorum_action = _gf_true;
-                        peerinfo->quorum_action = _gf_false;
-                }
+                op_errno = GF_PROBE_ANOTHER_CLUSTER;
+                ret = 0;
+            }
 
-                /* Remove peer if it is not a friend and connection/handshake
-                *  fails, and notify cli. Happens only during probe.
-                */
-                if (peerinfo->state.state == GD_FRIEND_STATE_DEFAULT) {
-                        glusterd_friend_remove_notify (peerctx);
-                        goto out;
-                }
+            if ((peerinfo->quorum_contrib != QUORUM_DOWN) &&
+                (peerinfo->state.state == GD_FRIEND_STATE_BEFRIENDED)) {
+                peerinfo->quorum_contrib = QUORUM_DOWN;
+                quorum_action = _gf_true;
+                peerinfo->quorum_action = _gf_false;
+            }
 
-                peerinfo->connected = 0;
-                break;
+            /* Remove peer if it is not a friend and connection/handshake
+             *  fails, and notify cli. Happens only during probe.
+             */
+            if (peerinfo->state.state == GD_FRIEND_STATE_DEFAULT) {
+                glusterd_friend_remove_notify(peerctx, op_errno);
+                goto out;
+            }
+
+            peerinfo->connected = 0;
+            break;
         }
 
         default:
-                gf_log (this->name, GF_LOG_TRACE,
-                        "got some other RPC event %d", event);
-                ret = 0;
-                break;
-        }
+            gf_msg_trace(this->name, 0, "got some other RPC event %d", event);
+            ret = 0;
+            break;
+    }
 
 out:
-        rcu_read_unlock ();
+    RCU_READ_UNLOCK;
 
-        glusterd_friend_sm ();
-        glusterd_op_sm ();
-        if (quorum_action)
-                glusterd_do_quorum_action ();
-        return ret;
+    glusterd_friend_sm();
+    glusterd_op_sm();
+    if (quorum_action)
+        glusterd_do_quorum_action();
+    return ret;
 }
 
 int
-glusterd_peer_rpc_notify (struct rpc_clnt *rpc, void *mydata,
-                          rpc_clnt_event_t event, void *data)
+glusterd_peer_rpc_notify(struct rpc_clnt *rpc, void *mydata,
+                         rpc_clnt_event_t event, void *data)
 {
-        return glusterd_big_locked_notify (rpc, mydata, event, data,
-                                           __glusterd_peer_rpc_notify);
+    return glusterd_big_locked_notify(rpc, mydata, event, data,
+                                      __glusterd_peer_rpc_notify);
 }
 
 int
-glusterd_null (rpcsvc_request_t *req)
+glusterd_null(rpcsvc_request_t *req)
 {
-
-        return 0;
+    return 0;
 }
 
-rpcsvc_actor_t gd_svc_mgmt_actors[GLUSTERD_MGMT_MAXVALUE] = {
-        [GLUSTERD_MGMT_NULL]           = { "NULL",           GLUSTERD_MGMT_NULL,           glusterd_null,                  NULL, 0, DRC_NA},
-        [GLUSTERD_MGMT_CLUSTER_LOCK]   = { "CLUSTER_LOCK",   GLUSTERD_MGMT_CLUSTER_LOCK,   glusterd_handle_cluster_lock,   NULL, 0, DRC_NA},
-        [GLUSTERD_MGMT_CLUSTER_UNLOCK] = { "CLUSTER_UNLOCK", GLUSTERD_MGMT_CLUSTER_UNLOCK, glusterd_handle_cluster_unlock, NULL, 0, DRC_NA},
-        [GLUSTERD_MGMT_STAGE_OP]       = { "STAGE_OP",       GLUSTERD_MGMT_STAGE_OP,       glusterd_handle_stage_op,       NULL, 0, DRC_NA},
-        [GLUSTERD_MGMT_COMMIT_OP]      = { "COMMIT_OP",      GLUSTERD_MGMT_COMMIT_OP,      glusterd_handle_commit_op,      NULL, 0, DRC_NA},
+static rpcsvc_actor_t gd_svc_mgmt_actors[GLUSTERD_MGMT_MAXVALUE] = {
+    [GLUSTERD_MGMT_NULL] = {"NULL", glusterd_null, NULL, GLUSTERD_MGMT_NULL,
+                            DRC_NA, 0},
+    [GLUSTERD_MGMT_CLUSTER_LOCK] = {"CLUSTER_LOCK",
+                                    glusterd_handle_cluster_lock, NULL,
+                                    GLUSTERD_MGMT_CLUSTER_LOCK, DRC_NA, 0},
+    [GLUSTERD_MGMT_CLUSTER_UNLOCK] = {"CLUSTER_UNLOCK",
+                                      glusterd_handle_cluster_unlock, NULL,
+                                      GLUSTERD_MGMT_CLUSTER_UNLOCK, DRC_NA, 0},
+    [GLUSTERD_MGMT_STAGE_OP] = {"STAGE_OP", glusterd_handle_stage_op, NULL,
+                                GLUSTERD_MGMT_STAGE_OP, DRC_NA, 0},
+    [GLUSTERD_MGMT_COMMIT_OP] =
+        {
+            "COMMIT_OP",
+            glusterd_handle_commit_op,
+            NULL,
+            GLUSTERD_MGMT_COMMIT_OP,
+            DRC_NA,
+            0,
+        },
 };
 
 struct rpcsvc_program gd_svc_mgmt_prog = {
-        .progname  = "GlusterD svc mgmt",
-        .prognum   = GD_MGMT_PROGRAM,
-        .progver   = GD_MGMT_VERSION,
-        .numactors = GLUSTERD_MGMT_MAXVALUE,
-        .actors    = gd_svc_mgmt_actors,
-	.synctask  = _gf_true,
+    .progname = "GlusterD svc mgmt",
+    .prognum = GD_MGMT_PROGRAM,
+    .progver = GD_MGMT_VERSION,
+    .numactors = GLUSTERD_MGMT_MAXVALUE,
+    .actors = gd_svc_mgmt_actors,
+    .synctask = _gf_true,
 };
 
-rpcsvc_actor_t gd_svc_peer_actors[GLUSTERD_FRIEND_MAXVALUE] = {
-        [GLUSTERD_FRIEND_NULL]    = { "NULL",          GLUSTERD_MGMT_NULL,     glusterd_null,                         NULL, 0, DRC_NA},
-        [GLUSTERD_PROBE_QUERY]    = { "PROBE_QUERY",   GLUSTERD_PROBE_QUERY,   glusterd_handle_probe_query,           NULL, 0, DRC_NA},
-        [GLUSTERD_FRIEND_ADD]     = { "FRIEND_ADD",    GLUSTERD_FRIEND_ADD,    glusterd_handle_incoming_friend_req,   NULL, 0, DRC_NA},
-        [GLUSTERD_FRIEND_REMOVE]  = { "FRIEND_REMOVE", GLUSTERD_FRIEND_REMOVE, glusterd_handle_incoming_unfriend_req, NULL, 0, DRC_NA},
-        [GLUSTERD_FRIEND_UPDATE]  = { "FRIEND_UPDATE", GLUSTERD_FRIEND_UPDATE, glusterd_handle_friend_update,         NULL, 0, DRC_NA},
+static rpcsvc_actor_t gd_svc_peer_actors[GLUSTERD_FRIEND_MAXVALUE] = {
+    [GLUSTERD_FRIEND_NULL] = {"NULL", glusterd_null, NULL, GLUSTERD_MGMT_NULL,
+                              DRC_NA, 0},
+    [GLUSTERD_PROBE_QUERY] = {"PROBE_QUERY", glusterd_handle_probe_query, NULL,
+                              GLUSTERD_PROBE_QUERY, DRC_NA, 0},
+    [GLUSTERD_FRIEND_ADD] = {"FRIEND_ADD", glusterd_handle_incoming_friend_req,
+                             NULL, GLUSTERD_FRIEND_ADD, DRC_NA, 0},
+    [GLUSTERD_FRIEND_REMOVE] = {"FRIEND_REMOVE",
+                                glusterd_handle_incoming_unfriend_req, NULL,
+                                GLUSTERD_FRIEND_REMOVE, DRC_NA, 0},
+    [GLUSTERD_FRIEND_UPDATE] = {"FRIEND_UPDATE", glusterd_handle_friend_update,
+                                NULL, GLUSTERD_FRIEND_UPDATE, DRC_NA, 0},
 };
 
 struct rpcsvc_program gd_svc_peer_prog = {
-        .progname  = "GlusterD svc peer",
-        .prognum   = GD_FRIEND_PROGRAM,
-        .progver   = GD_FRIEND_VERSION,
-        .numactors = GLUSTERD_FRIEND_MAXVALUE,
-        .actors    = gd_svc_peer_actors,
-	.synctask  = _gf_false,
+    .progname = "GlusterD svc peer",
+    .prognum = GD_FRIEND_PROGRAM,
+    .progver = GD_FRIEND_VERSION,
+    .numactors = GLUSTERD_FRIEND_MAXVALUE,
+    .actors = gd_svc_peer_actors,
+    .synctask = _gf_false,
 };
 
-
-
-rpcsvc_actor_t gd_svc_cli_actors[GLUSTER_CLI_MAXVALUE] = {
-        [GLUSTER_CLI_PROBE]              = { "CLI_PROBE",         GLUSTER_CLI_PROBE,            glusterd_handle_cli_probe,             NULL, 0, DRC_NA},
-        [GLUSTER_CLI_CREATE_VOLUME]      = { "CLI_CREATE_VOLUME", GLUSTER_CLI_CREATE_VOLUME,    glusterd_handle_create_volume,         NULL, 0, DRC_NA},
-        [GLUSTER_CLI_DEFRAG_VOLUME]      = { "CLI_DEFRAG_VOLUME", GLUSTER_CLI_DEFRAG_VOLUME,    glusterd_handle_defrag_volume,         NULL, 0, DRC_NA},
-        [GLUSTER_CLI_DEPROBE]            = { "FRIEND_REMOVE",     GLUSTER_CLI_DEPROBE,          glusterd_handle_cli_deprobe,           NULL, 0, DRC_NA},
-        [GLUSTER_CLI_LIST_FRIENDS]       = { "LIST_FRIENDS",      GLUSTER_CLI_LIST_FRIENDS,     glusterd_handle_cli_list_friends,      NULL, 0, DRC_NA},
-        [GLUSTER_CLI_UUID_RESET]         = { "UUID_RESET",        GLUSTER_CLI_UUID_RESET,       glusterd_handle_cli_uuid_reset,        NULL, 0, DRC_NA},
-        [GLUSTER_CLI_UUID_GET]           = { "UUID_GET",          GLUSTER_CLI_UUID_GET,         glusterd_handle_cli_uuid_get,          NULL, 0, DRC_NA},
-        [GLUSTER_CLI_START_VOLUME]       = { "START_VOLUME",      GLUSTER_CLI_START_VOLUME,     glusterd_handle_cli_start_volume,      NULL, 0, DRC_NA},
-        [GLUSTER_CLI_STOP_VOLUME]        = { "STOP_VOLUME",       GLUSTER_CLI_STOP_VOLUME,      glusterd_handle_cli_stop_volume,       NULL, 0, DRC_NA},
-        [GLUSTER_CLI_DELETE_VOLUME]      = { "DELETE_VOLUME",     GLUSTER_CLI_DELETE_VOLUME,    glusterd_handle_cli_delete_volume,     NULL, 0, DRC_NA},
-        [GLUSTER_CLI_GET_VOLUME]         = { "GET_VOLUME",        GLUSTER_CLI_GET_VOLUME,       glusterd_handle_cli_get_volume,        NULL, 0, DRC_NA},
-        [GLUSTER_CLI_ADD_BRICK]          = { "ADD_BRICK",         GLUSTER_CLI_ADD_BRICK,        glusterd_handle_add_brick,             NULL, 0, DRC_NA},
-        [GLUSTER_CLI_ATTACH_TIER]        = { "ATTACH_TIER",       GLUSTER_CLI_ATTACH_TIER,      glusterd_handle_attach_tier,           NULL, 0, DRC_NA},
-        [GLUSTER_CLI_DETACH_TIER]        = { "DETACH_TIER",       GLUSTER_CLI_DETACH_TIER,      glusterd_handle_detach_tier,           NULL, 0, DRC_NA},
-        [GLUSTER_CLI_REPLACE_BRICK]      = { "REPLACE_BRICK",     GLUSTER_CLI_REPLACE_BRICK,    glusterd_handle_replace_brick,         NULL, 0, DRC_NA},
-        [GLUSTER_CLI_REMOVE_BRICK]       = { "REMOVE_BRICK",      GLUSTER_CLI_REMOVE_BRICK,     glusterd_handle_remove_brick,          NULL, 0, DRC_NA},
-        [GLUSTER_CLI_LOG_ROTATE]         = { "LOG FILENAME",      GLUSTER_CLI_LOG_ROTATE,       glusterd_handle_log_rotate,            NULL, 0, DRC_NA},
-        [GLUSTER_CLI_SET_VOLUME]         = { "SET_VOLUME",        GLUSTER_CLI_SET_VOLUME,       glusterd_handle_set_volume,            NULL, 0, DRC_NA},
-        [GLUSTER_CLI_SYNC_VOLUME]        = { "SYNC_VOLUME",       GLUSTER_CLI_SYNC_VOLUME,      glusterd_handle_sync_volume,           NULL, 0, DRC_NA},
-        [GLUSTER_CLI_RESET_VOLUME]       = { "RESET_VOLUME",      GLUSTER_CLI_RESET_VOLUME,     glusterd_handle_reset_volume,          NULL, 0, DRC_NA},
-        [GLUSTER_CLI_FSM_LOG]            = { "FSM_LOG",           GLUSTER_CLI_FSM_LOG,          glusterd_handle_fsm_log,               NULL, 0, DRC_NA},
-        [GLUSTER_CLI_GSYNC_SET]          = { "GSYNC_SET",         GLUSTER_CLI_GSYNC_SET,        glusterd_handle_gsync_set,             NULL, 0, DRC_NA},
-        [GLUSTER_CLI_PROFILE_VOLUME]     = { "STATS_VOLUME",      GLUSTER_CLI_PROFILE_VOLUME,   glusterd_handle_cli_profile_volume,    NULL, 0, DRC_NA},
-        [GLUSTER_CLI_QUOTA]              = { "QUOTA",             GLUSTER_CLI_QUOTA,            glusterd_handle_quota,                 NULL, 0, DRC_NA},
-        [GLUSTER_CLI_GETWD]              = { "GETWD",             GLUSTER_CLI_GETWD,            glusterd_handle_getwd,                 NULL, 1, DRC_NA},
-        [GLUSTER_CLI_STATUS_VOLUME]      = {"STATUS_VOLUME",      GLUSTER_CLI_STATUS_VOLUME,    glusterd_handle_status_volume,         NULL, 0, DRC_NA},
-        [GLUSTER_CLI_MOUNT]              = { "MOUNT",             GLUSTER_CLI_MOUNT,            glusterd_handle_mount,                 NULL, 1, DRC_NA},
-        [GLUSTER_CLI_UMOUNT]             = { "UMOUNT",            GLUSTER_CLI_UMOUNT,           glusterd_handle_umount,                NULL, 1, DRC_NA},
-        [GLUSTER_CLI_HEAL_VOLUME]        = { "HEAL_VOLUME",       GLUSTER_CLI_HEAL_VOLUME,      glusterd_handle_cli_heal_volume,       NULL, 0, DRC_NA},
-        [GLUSTER_CLI_STATEDUMP_VOLUME]   = {"STATEDUMP_VOLUME",   GLUSTER_CLI_STATEDUMP_VOLUME, glusterd_handle_cli_statedump_volume,  NULL, 0, DRC_NA},
-        [GLUSTER_CLI_LIST_VOLUME]        = {"LIST_VOLUME",        GLUSTER_CLI_LIST_VOLUME,      glusterd_handle_cli_list_volume,       NULL, 0, DRC_NA},
-        [GLUSTER_CLI_CLRLOCKS_VOLUME]    = {"CLEARLOCKS_VOLUME",  GLUSTER_CLI_CLRLOCKS_VOLUME,  glusterd_handle_cli_clearlocks_volume, NULL, 0, DRC_NA},
-        [GLUSTER_CLI_COPY_FILE]          = {"COPY_FILE",          GLUSTER_CLI_COPY_FILE,        glusterd_handle_copy_file,             NULL, 0, DRC_NA},
-        [GLUSTER_CLI_SYS_EXEC]           = {"SYS_EXEC",           GLUSTER_CLI_SYS_EXEC,         glusterd_handle_sys_exec,              NULL, 0, DRC_NA},
-        [GLUSTER_CLI_SNAP]               = {"SNAP",               GLUSTER_CLI_SNAP,             glusterd_handle_snapshot,              NULL, 0, DRC_NA},
-        [GLUSTER_CLI_BARRIER_VOLUME]     = {"BARRIER_VOLUME",     GLUSTER_CLI_BARRIER_VOLUME,   glusterd_handle_barrier,               NULL, 0, DRC_NA},
-        [GLUSTER_CLI_GANESHA]            = { "GANESHA"  ,         GLUSTER_CLI_GANESHA,          glusterd_handle_ganesha_cmd,           NULL, 0, DRC_NA},
-        [GLUSTER_CLI_GET_VOL_OPT]        = {"GET_VOL_OPT",        GLUSTER_CLI_GET_VOL_OPT,      glusterd_handle_get_vol_opt,           NULL, 0, DRC_NA},
-        [GLUSTER_CLI_BITROT]             = {"BITROT",             GLUSTER_CLI_BITROT,           glusterd_handle_bitrot,                NULL, 0, DRC_NA},
+static rpcsvc_actor_t gd_svc_cli_actors[GLUSTER_CLI_MAXVALUE] = {
+    [GLUSTER_CLI_PROBE] = {"CLI_PROBE", glusterd_handle_cli_probe, NULL,
+                           GLUSTER_CLI_PROBE, DRC_NA, 0},
+    [GLUSTER_CLI_CREATE_VOLUME] = {"CLI_CREATE_VOLUME",
+                                   glusterd_handle_create_volume, NULL,
+                                   GLUSTER_CLI_CREATE_VOLUME, DRC_NA, 0},
+    [GLUSTER_CLI_DEFRAG_VOLUME] = {"CLI_DEFRAG_VOLUME",
+                                   glusterd_handle_defrag_volume, NULL,
+                                   GLUSTER_CLI_DEFRAG_VOLUME, DRC_NA, 0},
+    [GLUSTER_CLI_DEPROBE] = {"FRIEND_REMOVE", glusterd_handle_cli_deprobe, NULL,
+                             GLUSTER_CLI_DEPROBE, DRC_NA, 0},
+    [GLUSTER_CLI_LIST_FRIENDS] = {"LIST_FRIENDS",
+                                  glusterd_handle_cli_list_friends, NULL,
+                                  GLUSTER_CLI_LIST_FRIENDS, DRC_NA, 0},
+    [GLUSTER_CLI_UUID_RESET] = {"UUID_RESET", glusterd_handle_cli_uuid_reset,
+                                NULL, GLUSTER_CLI_UUID_RESET, DRC_NA, 0},
+    [GLUSTER_CLI_UUID_GET] = {"UUID_GET", glusterd_handle_cli_uuid_get, NULL,
+                              GLUSTER_CLI_UUID_GET, DRC_NA, 0},
+    [GLUSTER_CLI_START_VOLUME] = {"START_VOLUME",
+                                  glusterd_handle_cli_start_volume, NULL,
+                                  GLUSTER_CLI_START_VOLUME, DRC_NA, 0},
+    [GLUSTER_CLI_STOP_VOLUME] = {"STOP_VOLUME", glusterd_handle_cli_stop_volume,
+                                 NULL, GLUSTER_CLI_STOP_VOLUME, DRC_NA, 0},
+    [GLUSTER_CLI_DELETE_VOLUME] = {"DELETE_VOLUME",
+                                   glusterd_handle_cli_delete_volume, NULL,
+                                   GLUSTER_CLI_DELETE_VOLUME, DRC_NA, 0},
+    [GLUSTER_CLI_GET_VOLUME] = {"GET_VOLUME", glusterd_handle_cli_get_volume,
+                                NULL, GLUSTER_CLI_GET_VOLUME, DRC_NA, 0},
+    [GLUSTER_CLI_ADD_BRICK] = {"ADD_BRICK", glusterd_handle_add_brick, NULL,
+                               GLUSTER_CLI_ADD_BRICK, DRC_NA, 0},
+    [GLUSTER_CLI_ATTACH_TIER] = {"ATTACH_TIER", glusterd_handle_attach_tier,
+                                 NULL, GLUSTER_CLI_ATTACH_TIER, DRC_NA, 0},
+    [GLUSTER_CLI_REPLACE_BRICK] = {"REPLACE_BRICK",
+                                   glusterd_handle_replace_brick, NULL,
+                                   GLUSTER_CLI_REPLACE_BRICK, DRC_NA, 0},
+    [GLUSTER_CLI_REMOVE_BRICK] = {"REMOVE_BRICK", glusterd_handle_remove_brick,
+                                  NULL, GLUSTER_CLI_REMOVE_BRICK, DRC_NA, 0},
+    [GLUSTER_CLI_LOG_ROTATE] = {"LOG FILENAME", glusterd_handle_log_rotate,
+                                NULL, GLUSTER_CLI_LOG_ROTATE, DRC_NA, 0},
+    [GLUSTER_CLI_SET_VOLUME] = {"SET_VOLUME", glusterd_handle_set_volume, NULL,
+                                GLUSTER_CLI_SET_VOLUME, DRC_NA, 0},
+    [GLUSTER_CLI_SYNC_VOLUME] = {"SYNC_VOLUME", glusterd_handle_sync_volume,
+                                 NULL, GLUSTER_CLI_SYNC_VOLUME, DRC_NA, 0},
+    [GLUSTER_CLI_RESET_VOLUME] = {"RESET_VOLUME", glusterd_handle_reset_volume,
+                                  NULL, GLUSTER_CLI_RESET_VOLUME, DRC_NA, 0},
+    [GLUSTER_CLI_FSM_LOG] = {"FSM_LOG", glusterd_handle_fsm_log, NULL,
+                             GLUSTER_CLI_FSM_LOG, DRC_NA, 0},
+    [GLUSTER_CLI_GSYNC_SET] = {"GSYNC_SET", glusterd_handle_gsync_set, NULL,
+                               GLUSTER_CLI_GSYNC_SET, DRC_NA, 0},
+    [GLUSTER_CLI_PROFILE_VOLUME] = {"STATS_VOLUME",
+                                    glusterd_handle_cli_profile_volume, NULL,
+                                    GLUSTER_CLI_PROFILE_VOLUME, DRC_NA, 0},
+    [GLUSTER_CLI_QUOTA] = {"QUOTA", glusterd_handle_quota, NULL,
+                           GLUSTER_CLI_QUOTA, DRC_NA, 0},
+    [GLUSTER_CLI_GETWD] = {"GETWD", glusterd_handle_getwd, NULL,
+                           GLUSTER_CLI_GETWD, DRC_NA, 1},
+    [GLUSTER_CLI_STATUS_VOLUME] = {"STATUS_VOLUME",
+                                   glusterd_handle_status_volume, NULL,
+                                   GLUSTER_CLI_STATUS_VOLUME, DRC_NA, 0},
+    [GLUSTER_CLI_MOUNT] = {"MOUNT", glusterd_handle_mount, NULL,
+                           GLUSTER_CLI_MOUNT, DRC_NA, 1},
+    [GLUSTER_CLI_UMOUNT] = {"UMOUNT", glusterd_handle_umount, NULL,
+                            GLUSTER_CLI_UMOUNT, DRC_NA, 1},
+    [GLUSTER_CLI_HEAL_VOLUME] = {"HEAL_VOLUME", glusterd_handle_cli_heal_volume,
+                                 NULL, GLUSTER_CLI_HEAL_VOLUME, DRC_NA, 0},
+    [GLUSTER_CLI_STATEDUMP_VOLUME] = {"STATEDUMP_VOLUME",
+                                      glusterd_handle_cli_statedump_volume,
+                                      NULL, GLUSTER_CLI_STATEDUMP_VOLUME,
+                                      DRC_NA, 0},
+    [GLUSTER_CLI_LIST_VOLUME] = {"LIST_VOLUME", glusterd_handle_cli_list_volume,
+                                 NULL, GLUSTER_CLI_LIST_VOLUME, DRC_NA, 0},
+    [GLUSTER_CLI_CLRLOCKS_VOLUME] = {"CLEARLOCKS_VOLUME",
+                                     glusterd_handle_cli_clearlocks_volume,
+                                     NULL, GLUSTER_CLI_CLRLOCKS_VOLUME, DRC_NA,
+                                     0},
+    [GLUSTER_CLI_COPY_FILE] = {"COPY_FILE", glusterd_handle_copy_file, NULL,
+                               GLUSTER_CLI_COPY_FILE, DRC_NA, 0},
+    [GLUSTER_CLI_SYS_EXEC] = {"SYS_EXEC", glusterd_handle_sys_exec, NULL,
+                              GLUSTER_CLI_SYS_EXEC, DRC_NA, 0},
+    [GLUSTER_CLI_SNAP] = {"SNAP", glusterd_handle_snapshot, NULL,
+                          GLUSTER_CLI_SNAP, DRC_NA, 0},
+    [GLUSTER_CLI_BARRIER_VOLUME] = {"BARRIER_VOLUME", glusterd_handle_barrier,
+                                    NULL, GLUSTER_CLI_BARRIER_VOLUME, DRC_NA,
+                                    0},
+    [GLUSTER_CLI_GANESHA] = {"GANESHA", glusterd_handle_ganesha_cmd, NULL,
+                             GLUSTER_CLI_GANESHA, DRC_NA, 0},
+    [GLUSTER_CLI_GET_VOL_OPT] = {"GET_VOL_OPT", glusterd_handle_get_vol_opt,
+                                 NULL, DRC_NA, 0},
+    [GLUSTER_CLI_BITROT] = {"BITROT", glusterd_handle_bitrot, NULL,
+                            GLUSTER_CLI_BITROT, DRC_NA, 0},
+    [GLUSTER_CLI_GET_STATE] = {"GET_STATE", glusterd_handle_get_state, NULL,
+                               GLUSTER_CLI_GET_STATE, DRC_NA, 0},
+    [GLUSTER_CLI_RESET_BRICK] = {"RESET_BRICK", glusterd_handle_reset_brick,
+                                 NULL, GLUSTER_CLI_RESET_BRICK, DRC_NA, 0},
+    [GLUSTER_CLI_TIER] = {"TIER", glusterd_handle_tier, NULL, GLUSTER_CLI_TIER,
+                          DRC_NA, 0},
+    [GLUSTER_CLI_REMOVE_TIER_BRICK] = {"REMOVE_TIER_BRICK",
+                                       glusterd_handle_tier, NULL,
+                                       GLUSTER_CLI_REMOVE_TIER_BRICK, DRC_NA,
+                                       0},
+    [GLUSTER_CLI_ADD_TIER_BRICK] = {"ADD_TIER_BRICK",
+                                    glusterd_handle_add_tier_brick, NULL,
+                                    GLUSTER_CLI_ADD_TIER_BRICK, DRC_NA, 0},
 };
 
 struct rpcsvc_program gd_svc_cli_prog = {
-        .progname  = "GlusterD svc cli",
-        .prognum   = GLUSTER_CLI_PROGRAM,
-        .progver   = GLUSTER_CLI_VERSION,
-        .numactors = GLUSTER_CLI_MAXVALUE,
-        .actors    = gd_svc_cli_actors,
-	.synctask  = _gf_true,
+    .progname = "GlusterD svc cli",
+    .prognum = GLUSTER_CLI_PROGRAM,
+    .progver = GLUSTER_CLI_VERSION,
+    .numactors = GLUSTER_CLI_MAXVALUE,
+    .actors = gd_svc_cli_actors,
+    .synctask = _gf_true,
 };
 
 /**
  * This set of RPC progs are deemed to be trusted. Most of the actors support
  * read only queries, the only exception being MOUNT/UMOUNT which is required
- * by geo-replication to supprt unprivileged master -> slave sessions.
+ * by geo-replication to support unprivileged master -> slave sessions.
  */
-rpcsvc_actor_t gd_svc_cli_trusted_actors[GLUSTER_CLI_MAXVALUE] = {
-        [GLUSTER_CLI_LIST_FRIENDS]       = { "LIST_FRIENDS",      GLUSTER_CLI_LIST_FRIENDS,     glusterd_handle_cli_list_friends,      NULL, 0, DRC_NA},
-        [GLUSTER_CLI_UUID_GET]           = { "UUID_GET",          GLUSTER_CLI_UUID_GET,         glusterd_handle_cli_uuid_get,          NULL, 0, DRC_NA},
-        [GLUSTER_CLI_GET_VOLUME]         = { "GET_VOLUME",        GLUSTER_CLI_GET_VOLUME,       glusterd_handle_cli_get_volume,        NULL, 0, DRC_NA},
-        [GLUSTER_CLI_GETWD]              = { "GETWD",             GLUSTER_CLI_GETWD,            glusterd_handle_getwd,                 NULL, 1, DRC_NA},
-        [GLUSTER_CLI_STATUS_VOLUME]      = {"STATUS_VOLUME",      GLUSTER_CLI_STATUS_VOLUME,    glusterd_handle_status_volume,         NULL, 0, DRC_NA},
-        [GLUSTER_CLI_LIST_VOLUME]        = {"LIST_VOLUME",        GLUSTER_CLI_LIST_VOLUME,      glusterd_handle_cli_list_volume,       NULL, 0, DRC_NA},
-        [GLUSTER_CLI_MOUNT]              = { "MOUNT",             GLUSTER_CLI_MOUNT,            glusterd_handle_mount,                 NULL, 1, DRC_NA},
-        [GLUSTER_CLI_UMOUNT]             = { "UMOUNT",            GLUSTER_CLI_UMOUNT,           glusterd_handle_umount,                NULL, 1, DRC_NA},
+static rpcsvc_actor_t gd_svc_cli_trusted_actors[GLUSTER_CLI_MAXVALUE] = {
+    [GLUSTER_CLI_LIST_FRIENDS] = {"LIST_FRIENDS",
+                                  glusterd_handle_cli_list_friends, NULL,
+                                  GLUSTER_CLI_LIST_FRIENDS, DRC_NA, 0},
+    [GLUSTER_CLI_UUID_GET] = {"UUID_GET", glusterd_handle_cli_uuid_get, NULL,
+                              GLUSTER_CLI_UUID_GET, DRC_NA, 0},
+    [GLUSTER_CLI_GET_VOLUME] = {"GET_VOLUME", glusterd_handle_cli_get_volume,
+                                NULL, GLUSTER_CLI_GET_VOLUME, DRC_NA, 0},
+    [GLUSTER_CLI_GETWD] = {"GETWD", glusterd_handle_getwd, NULL,
+                           GLUSTER_CLI_GETWD, DRC_NA, 1},
+    [GLUSTER_CLI_STATUS_VOLUME] = {"STATUS_VOLUME",
+                                   glusterd_handle_status_volume, NULL,
+                                   GLUSTER_CLI_STATUS_VOLUME, DRC_NA, 0},
+    [GLUSTER_CLI_LIST_VOLUME] = {"LIST_VOLUME", glusterd_handle_cli_list_volume,
+                                 NULL, GLUSTER_CLI_LIST_VOLUME, DRC_NA, 0},
+    [GLUSTER_CLI_MOUNT] = {"MOUNT", glusterd_handle_mount, NULL,
+                           GLUSTER_CLI_MOUNT, DRC_NA, 1},
+    [GLUSTER_CLI_UMOUNT] = {"UMOUNT", glusterd_handle_umount, NULL,
+                            GLUSTER_CLI_UMOUNT, DRC_NA, 1},
 };
 
 struct rpcsvc_program gd_svc_cli_trusted_progs = {
-        .progname  = "GlusterD svc cli read-only",
-        .prognum   = GLUSTER_CLI_PROGRAM,
-        .progver   = GLUSTER_CLI_VERSION,
-        .numactors = GLUSTER_CLI_MAXVALUE,
-        .actors    = gd_svc_cli_trusted_actors,
-	.synctask  = _gf_true,
+    .progname = "GlusterD svc cli read-only",
+    .prognum = GLUSTER_CLI_PROGRAM,
+    .progver = GLUSTER_CLI_VERSION,
+    .numactors = GLUSTER_CLI_MAXVALUE,
+    .actors = gd_svc_cli_trusted_actors,
+    .synctask = _gf_true,
 };
+
+/* As we cant remove the handlers, I'm moving the tier based
+ * handlers to this file as we no longer have gluster-tier.c
+ * and other tier.c files
+ */
+
+int
+glusterd_handle_tier(rpcsvc_request_t *req)
+{
+    return 0;
+}
diff --git a/xlators/mgmt/glusterd/src/glusterd-handshake.c b/xlators/mgmt/glusterd/src/glusterd-handshake.c
index bc00e99b62f..d96e35503dd 100644
--- a/xlators/mgmt/glusterd/src/glusterd-handshake.c
+++ b/xlators/mgmt/glusterd/src/glusterd-handshake.c
@@ -8,15 +8,11 @@
    cases as published by the Free Software Foundation.
 */
 
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
-
-#include "xlator.h"
-#include "defaults.h"
-#include "glusterfs.h"
-#include "compat-errno.h"
+#include <glusterfs/xlator.h>
+#include <glusterfs/defaults.h>
+#include <glusterfs/glusterfs.h>
+#include <glusterfs/syscall.h>
+#include <glusterfs/compat-errno.h>
 
 #include "glusterd.h"
 #include "glusterd-utils.h"
@@ -25,317 +21,504 @@
 #include "glusterd-snapshot-utils.h"
 #include "glusterd-svc-mgmt.h"
 #include "glusterd-snapd-svc-helper.h"
+#include "glusterd-volgen.h"
 #include "glusterd-quotad-svc.h"
-
+#include "glusterd-messages.h"
 #include "glusterfs3.h"
 #include "protocol-common.h"
 #include "rpcsvc.h"
 #include "rpc-common-xdr.h"
+#include "glusterd-gfproxyd-svc-helper.h"
+#include "glusterd-shd-svc-helper.h"
 
 extern struct rpc_clnt_program gd_peer_prog;
 extern struct rpc_clnt_program gd_mgmt_prog;
 extern struct rpc_clnt_program gd_mgmt_v3_prog;
 
+#define TRUSTED_PREFIX "trusted-"
+#define GD_PEER_ID_KEY "peer-id"
 
-#define TRUSTED_PREFIX         "trusted-"
-#define GD_PEER_ID_KEY         "peer-id"
-
-typedef ssize_t (*gfs_serialize_t) (struct iovec outmsg, void *data);
+typedef ssize_t (*gfs_serialize_t)(struct iovec outmsg, void *data);
 
 static int
-get_snap_volname_and_volinfo (const char *volpath, char **volname,
-                              glusterd_volinfo_t **volinfo)
+get_snap_volname_and_volinfo(const char *volpath, char **volname,
+                             glusterd_volinfo_t **volinfo)
 {
-        int              ret            = -1;
-        char            *save_ptr       = NULL;
-        char            *str_token      = NULL;
-        char            *snapname       = NULL;
-        char            *volname_token  = NULL;
-        char            *vol            = NULL;
-        glusterd_snap_t *snap           = NULL;
-        xlator_t        *this           = NULL;
-
-        this = THIS;
-        GF_ASSERT (this);
-        GF_ASSERT (volpath);
-        GF_ASSERT (volinfo);
-
-        str_token = gf_strdup (volpath);
-        if (NULL == str_token) {
-                goto out;
+    int ret = -1;
+    char *save_ptr = NULL;
+    char *str_token = NULL;
+    char *snapname = NULL;
+    char *volname_token = NULL;
+    char *vol = NULL;
+    glusterd_snap_t *snap = NULL;
+    xlator_t *this = NULL;
+    char *tmp_str_token = NULL;
+    char *volfile_token = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+    GF_ASSERT(volpath);
+    GF_ASSERT(volinfo);
+
+    str_token = gf_strdup(volpath);
+    if (NULL == str_token) {
+        goto out;
+    }
+
+    tmp_str_token = str_token;
+
+    /* Input volname will have below formats:
+     * /snaps/<snapname>/<volname>.<hostname>
+     * or
+     * /snaps/<snapname>/<parent-volname>
+     * We need to extract snapname and parent_volname */
+
+    /*split string by "/" */
+    strtok_r(str_token, "/", &save_ptr);
+    snapname = strtok_r(NULL, "/", &save_ptr);
+    if (!snapname) {
+        gf_msg(this->name, GF_LOG_ERROR, EINVAL, GD_MSG_INVALID_ENTRY,
+               "Invalid path: %s", volpath);
+        goto out;
+    }
+
+    volname_token = strtok_r(NULL, "/", &save_ptr);
+    if (!volname_token) {
+        gf_msg(this->name, GF_LOG_ERROR, EINVAL, GD_MSG_INVALID_ENTRY,
+               "Invalid path: %s", volpath);
+        goto out;
+    }
+
+    snap = glusterd_find_snap_by_name(snapname);
+    if (!snap) {
+        gf_msg(this->name, GF_LOG_ERROR, EINVAL, GD_MSG_SNAP_NOT_FOUND,
+               "Failed to "
+               "fetch snap %s",
+               snapname);
+        goto out;
+    }
+
+    /* Find if its a parent volume name or snap volume
+     * name. This function will succeed if volname_token
+     * is a parent volname
+     */
+    ret = glusterd_volinfo_find(volname_token, volinfo);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_WARNING, 0, GD_MSG_VOLINFO_GET_FAIL,
+               "failed to get the volinfo for the volume %s", volname_token);
+
+        /* Get the actual volfile name. */
+        volfile_token = strtok_r(NULL, "/", &save_ptr);
+        *volname = gf_strdup(volfile_token);
+        if (NULL == *volname) {
+            gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_STRDUP_FAILED,
+                    "Volname=%s", volfile_token, NULL);
+            ret = -1;
+            goto out;
+        }
+
+        /*
+         * Ideally, this should succeed as volname_token now contains
+         * the name of the snap volume (i.e. name of the volume that
+         * represents the snapshot). But, if for some reason, volinfo
+         * for the snap volume is not found, then try to get from the
+         * name of the volfile. Name of the volfile is like this.
+         * <snap volume name>.<hostname>.<brick path>.vol
+         */
+        ret = glusterd_snap_volinfo_find(volname_token, snap, volinfo);
+        if (ret) {
+            /* Split the volume name */
+            vol = strtok_r(volfile_token, ".", &save_ptr);
+            if (!vol) {
+                gf_msg(this->name, GF_LOG_ERROR, EINVAL, GD_MSG_INVALID_ENTRY,
+                       "Invalid "
+                       "volname (%s)",
+                       volfile_token);
+                goto out;
+            }
+
+            ret = glusterd_snap_volinfo_find(vol, snap, volinfo);
+            if (ret) {
+                gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_SNAP_INFO_FAIL,
+                       "Failed to "
+                       "fetch snap volume from volname (%s)",
+                       vol);
+                goto out;
+            }
+        }
+    } else {
+        /*volname_token is parent volname*/
+        ret = glusterd_snap_volinfo_find_from_parent_volname(volname_token,
+                                                             snap, volinfo);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_SNAP_INFO_FAIL,
+                   "Failed to "
+                   "fetch snap volume from parent "
+                   "volname (%s)",
+                   volname_token);
+            goto out;
         }
 
-        /* Input volname will have below formats:
-         * /snaps/<snapname>/<volname>.<hostname>
-         * or
-         * /snaps/<snapname>/<parent-volname>
-         * We need to extract snapname and parent_volname */
-
-        /*split string by "/" */
-        strtok_r (str_token, "/",  &save_ptr);
-        snapname  = strtok_r(NULL, "/", &save_ptr);
-        if (!snapname) {
-                gf_log(this->name, GF_LOG_ERROR, "Invalid path: %s", volpath);
-                goto out;
+        /* Since volname_token is a parent volname we should
+         * get the snap volname here*/
+        *volname = gf_strdup((*volinfo)->volname);
+        if (NULL == *volname) {
+            ret = -1;
+            goto out;
         }
+    }
 
-        volname_token = strtok_r(NULL, "/", &save_ptr);
-        if (!volname_token) {
-                gf_log(this->name, GF_LOG_ERROR, "Invalid path: %s", volpath);
-                goto out;
-        }
+out:
+    if (ret && NULL != *volname) {
+        GF_FREE(*volname);
+        *volname = NULL;
+    }
+
+    if (tmp_str_token)
+        GF_FREE(tmp_str_token);
+    return ret;
+}
 
-        snap = glusterd_find_snap_by_name (snapname);
-        if (!snap) {
-                gf_log(this->name, GF_LOG_ERROR, "Failed to "
-                                "fetch snap %s", snapname);
-                goto out;
-        }
+int32_t
+glusterd_get_client_per_brick_volfile(glusterd_volinfo_t *volinfo,
+                                      char *filename, char *path, int path_len)
+{
+    char workdir[PATH_MAX] = {
+        0,
+    };
+    glusterd_conf_t *priv = NULL;
+    int32_t ret = -1;
 
-        /* Find if its a parent volume name or snap volume
-         * name. This function will succeed if volname_token
-         * is a parent volname
-         */
-        ret = glusterd_volinfo_find (volname_token, volinfo);
-        if (ret) {
-                *volname = gf_strdup (volname_token);
-                if (NULL == *volname) {
-                        ret = -1;
-                        goto out;
-                }
+    GF_VALIDATE_OR_GOTO("glusterd", THIS, out);
+    priv = THIS->private;
+    GF_VALIDATE_OR_GOTO(THIS->name, priv, out);
 
-                ret = glusterd_snap_volinfo_find (volname_token, snap,
-                                                  volinfo);
-                if (ret) {
-                        /* Split the volume name */
-                        vol = strtok_r (volname_token, ".", &save_ptr);
-                        if (!vol) {
-                                gf_log(this->name, GF_LOG_ERROR, "Invalid "
-                                                "volname (%s)", volname_token);
-                                goto out;
-                        }
-
-                        ret = glusterd_snap_volinfo_find (vol, snap, volinfo);
-                        if (ret) {
-                                gf_log(this->name, GF_LOG_ERROR, "Failed to "
-                                       "fetch snap volume from volname (%s)",
-                                       vol);
-                                goto out;
-                        }
-                }
-        } else {
-                /*volname_token is parent volname*/
-                ret = glusterd_snap_volinfo_find_from_parent_volname (
-                                volname_token, snap, volinfo);
-                if (ret) {
-                        gf_log(this->name, GF_LOG_ERROR, "Failed to "
-                                        "fetch snap volume from parent "
-                                        "volname (%s)", volname_token);
-                        goto out;
-                }
+    GLUSTERD_GET_VOLUME_DIR(workdir, volinfo, priv);
 
-                /* Since volname_token is a parent volname we should
-                 * get the snap volname here*/
-                *volname = gf_strdup ((*volinfo)->volname);
-                if (NULL == *volname) {
-                        ret = -1;
-                        goto out;
-                }
-        }
+    snprintf(path, path_len, "%s/%s", workdir, filename);
 
+    ret = 0;
 out:
-        if (ret && NULL != *volname) {
-                GF_FREE (*volname);
-                *volname = NULL;
-        }
-        return ret;
+    return ret;
 }
 
-static size_t
-build_volfile_path (char *volume_id, char *path,
-                    size_t path_len, char *trusted_str)
+size_t
+build_volfile_path(char *volume_id, char *path, size_t path_len,
+                   char *trusted_str, dict_t *dict)
 {
-        struct stat              stbuf                  = {0,};
-        int32_t                  ret                    = -1;
-        char                    *vol                    = NULL;
-        char                    *dup_volname            = NULL;
-        char                    *save_ptr               = NULL;
-        char                    *free_ptr               = NULL;
-        char                    *volname                = NULL;
-        char                    *volid_ptr              = NULL;
-        char                     dup_volid[PATH_MAX]    = {0,};
-        char                     path_prefix[PATH_MAX]  = {0,};
-        xlator_t                *this                   = NULL;
-        glusterd_volinfo_t      *volinfo                = NULL;
-        glusterd_conf_t         *priv                   = NULL;
-
-        this = THIS;
-        GF_ASSERT (this);
-        priv = this->private;
-        GF_ASSERT (priv);
-        GF_ASSERT (volume_id);
-        GF_ASSERT (path);
-
-        volid_ptr = strstr (volume_id, "snapd/");
-        if (volid_ptr) {
-                volid_ptr = strchr (volid_ptr, '/');
-                if (!volid_ptr) {
-                        ret = -1;
-                        goto out;
-                }
-                volid_ptr++;
-
-                ret = glusterd_volinfo_find (volid_ptr, &volinfo);
-                if (ret == -1) {
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "Couldn't find volinfo");
-                        goto out;
-                }
-                glusterd_svc_build_snapd_volfile (volinfo, path, path_len);
-                ret = 0;
-                goto out;
+    struct stat stbuf = {
+        0,
+    };
+    int32_t ret = -1;
+    char *vol = NULL;
+    char *dup_volname = NULL;
+    char *save_ptr = NULL;
+    char *free_ptr = NULL;
+    char *volname = NULL;
+    char *volid_ptr = NULL;
+    char dup_volid[PATH_MAX] = {
+        0,
+    };
+    char path_prefix[PATH_MAX] = {
+        0,
+    };
+    xlator_t *this = NULL;
+    glusterd_volinfo_t *volinfo = NULL;
+    glusterd_conf_t *priv = NULL;
+    int32_t len = 0;
+
+    this = THIS;
+    GF_ASSERT(this);
+    priv = this->private;
+    GF_ASSERT(priv);
+    GF_ASSERT(volume_id);
+    GF_ASSERT(path);
+
+    volid_ptr = strstr(volume_id, "snapd/");
+    if (volid_ptr) {
+        volid_ptr = strchr(volid_ptr, '/');
+        if (!volid_ptr) {
+            gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_STRCHR_FAIL, NULL);
+            ret = -1;
+            goto out;
+        }
+        volid_ptr++;
+
+        ret = glusterd_volinfo_find(volid_ptr, &volinfo);
+        if (ret == -1) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_VOLINFO_GET_FAIL,
+                   "Couldn't find volinfo");
+            goto out;
+        }
+        glusterd_svc_build_snapd_volfile(volinfo, path, path_len);
+        ret = 0;
+        goto out;
+    }
 
+    volid_ptr = strstr(volume_id, "gluster/");
+    if (volid_ptr) {
+        volid_ptr = strchr(volid_ptr, '/');
+        if (!volid_ptr) {
+            gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_STRCHR_FAIL, NULL);
+            ret = -1;
+            goto out;
         }
+        volid_ptr++;
 
-        volid_ptr = strstr (volume_id, "gluster/");
-        if (volid_ptr) {
-                volid_ptr = strchr (volid_ptr, '/');
-                if (!volid_ptr) {
-                        ret = -1;
-                        goto out;
-                }
-                volid_ptr++;
-
-                if (strcmp (volid_ptr, "quotad") == 0)
-                        glusterd_quotadsvc_build_volfile_path (volid_ptr,
-                                                               priv->workdir,
-                                                               path, path_len);
-                else
-                        glusterd_svc_build_volfile_path (volid_ptr,
-                                                         priv->workdir,
-                                                         path, path_len);
-                ret = 0;
-                goto out;
+        glusterd_svc_build_volfile_path(volid_ptr, priv->workdir, path,
+                                        path_len);
+        ret = 0;
+        goto out;
+    }
 
+    volid_ptr = strstr(volume_id, "gfproxy-client/");
+    if (volid_ptr) {
+        volid_ptr = strchr(volid_ptr, '/');
+        if (!volid_ptr) {
+            gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_STRCHR_FAIL, NULL);
+            ret = -1;
+            goto out;
         }
+        volid_ptr++;
 
-        volid_ptr = strstr (volume_id, "/snaps/");
-        if (volid_ptr) {
-                ret = get_snap_volname_and_volinfo (volid_ptr, &volname,
-                                                    &volinfo);
-                if (ret) {
-                        gf_log (this->name, GF_LOG_ERROR, "Failed to get snap"
-                                " volinfo from path (%s)", volume_id);
-                        ret = -1;
-                        goto out;
-                }
+        ret = glusterd_volinfo_find(volid_ptr, &volinfo);
+        if (ret == -1) {
+            gf_log(this->name, GF_LOG_ERROR, "Couldn't find volinfo");
+            goto out;
+        }
 
-                snprintf (path_prefix, sizeof (path_prefix), "%s/snaps/%s",
-                          priv->workdir, volinfo->snapshot->snapname);
+        glusterd_get_gfproxy_client_volfile(volinfo, path, path_len);
 
-                volid_ptr = volname;
-                /* this is to ensure that volname recvd from
-                   get_snap_volname_and_volinfo is free'd */
-                free_ptr = volname;
-                goto gotvolinfo;
+        ret = 0;
+        goto out;
+    }
 
+    volid_ptr = strstr(volume_id, "gfproxyd/");
+    if (volid_ptr) {
+        volid_ptr = strchr(volid_ptr, '/');
+        if (!volid_ptr) {
+            gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_STRCHR_FAIL, NULL);
+            ret = -1;
+            goto out;
         }
+        volid_ptr++;
 
-        volid_ptr = strstr (volume_id, "rebalance/");
-        if (volid_ptr) {
-                volid_ptr = strchr (volid_ptr, '/');
-                if (!volid_ptr) {
-                        ret = -1;
-                        goto out;
-                }
-                volid_ptr++;
-
-                ret = glusterd_volinfo_find (volid_ptr, &volinfo);
-                if (ret == -1) {
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "Couldn't find volinfo");
-                        goto out;
-                }
-                glusterd_get_rebalance_volfile (volinfo, path, path_len);
-                ret = 0;
-                goto out;
+        ret = glusterd_volinfo_find(volid_ptr, &volinfo);
+        if (ret == -1) {
+            gf_log(this->name, GF_LOG_ERROR, "Couldn't find volinfo");
+            goto out;
         }
 
-        if (volume_id[0] == '/') {
-                /* Normal behavior */
-                volid_ptr = volume_id;
-                volid_ptr++;
+        glusterd_svc_build_gfproxyd_volfile_path(volinfo, path, path_len);
+        ret = 0;
+        goto out;
+    }
 
-        } else {
-                /* Bringing in NFS like behavior for mount command, */
-                /* With this, one can mount a volume with below cmd */
-                /* bash# mount -t glusterfs server:/volume /mnt/pnt */
-                volid_ptr = volume_id;
+    volid_ptr = strstr(volume_id, "shd/");
+    if (volid_ptr) {
+        volid_ptr = strchr(volid_ptr, '/');
+        if (!volid_ptr) {
+            gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_STRCHR_FAIL, NULL);
+            ret = -1;
+            goto out;
         }
+        volid_ptr++;
 
-        snprintf (path_prefix, sizeof (path_prefix), "%s/vols",
-                  priv->workdir);
+        ret = glusterd_volinfo_find(volid_ptr, &volinfo);
+        if (ret == -1) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_VOLINFO_GET_FAIL,
+                   "Couldn't find volinfo for volid=%s", volid_ptr);
+            goto out;
+        }
+
+        glusterd_svc_build_shd_volfile_path(volinfo, path, path_len);
 
-        ret = glusterd_volinfo_find (volid_ptr, &volinfo);
+        ret = glusterd_svc_set_shd_pidfile(volinfo, dict);
+        if (ret == -1) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+                   "Couldn't set pidfile in dict for volid=%s", volid_ptr);
+            goto out;
+        }
+        ret = 0;
+        goto out;
+    }
 
+    volid_ptr = strstr(volume_id, "/snaps/");
+    if (volid_ptr) {
+        ret = get_snap_volname_and_volinfo(volid_ptr, &volname, &volinfo);
         if (ret) {
-                dup_volname = gf_strdup (volid_ptr);
-                if (!dup_volname) {
-                        ret = -1;
-                        goto out;
-                }
-                /* Split the volume name */
-                vol = strtok_r (dup_volname, ".", &save_ptr);
-                if (!vol) {
-                        ret = -1;
-                        goto out;
-                }
-                ret = glusterd_volinfo_find (vol, &volinfo);
-                if (ret)
-                        goto out;
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_SNAP_INFO_FAIL,
+                   "Failed to get snap"
+                   " volinfo from path (%s)",
+                   volume_id);
+            ret = -1;
+            goto out;
+        }
+
+        len = snprintf(path_prefix, sizeof(path_prefix), "%s/snaps/%s",
+                       priv->workdir, volinfo->snapshot->snapname);
+        volid_ptr = volname;
+        /* this is to ensure that volname recvd from
+           get_snap_volname_and_volinfo is free'd */
+        free_ptr = volname;
+        if ((len < 0) || (len >= sizeof(path_prefix))) {
+            ret = -1;
+            goto out;
+        }
+
+        goto gotvolinfo;
+    }
+
+    volid_ptr = strstr(volume_id, "rebalance/");
+    if (volid_ptr) {
+        volid_ptr = strchr(volid_ptr, '/');
+        if (!volid_ptr) {
+            gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_STRCHR_FAIL, NULL);
+            ret = -1;
+            goto out;
+        }
+        volid_ptr++;
+
+        ret = glusterd_volinfo_find(volid_ptr, &volinfo);
+        if (ret == -1) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_VOLINFO_GET_FAIL,
+                   "Couldn't find volinfo");
+            goto out;
+        }
+        glusterd_get_rebalance_volfile(volinfo, path, path_len);
+        ret = 0;
+        goto out;
+    }
+
+    volid_ptr = strstr(volume_id, "client_per_brick/");
+    if (volid_ptr) {
+        volid_ptr = strchr(volid_ptr, '/');
+        if (!volid_ptr) {
+            gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_STRCHR_FAIL, NULL);
+            ret = -1;
+            goto out;
+        }
+        volid_ptr++;
+
+        dup_volname = gf_strdup(volid_ptr);
+        if (!dup_volname) {
+            gf_msg(this->name, GF_LOG_ERROR, ENOMEM, GD_MSG_NO_MEMORY,
+                   "strdup failed");
+            ret = -1;
+            goto out;
+        }
+
+        /* Split the volume name */
+        vol = strtok_r(dup_volname, ".", &save_ptr);
+        if (!vol) {
+            gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_SPLIT_FAIL,
+                    "Volume name=%s", dup_volname, NULL);
+            ret = -1;
+            goto out;
+        }
+        ret = glusterd_volinfo_find(vol, &volinfo);
+        if (ret == -1) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_VOLINFO_GET_FAIL,
+                   "Couldn't find volinfo");
+            goto out;
+        }
+        ret = glusterd_get_client_per_brick_volfile(volinfo, volid_ptr, path,
+                                                    path_len);
+        if (ret < 0) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_NO_MEMORY,
+                   "failed to get volinfo path");
+            goto out;
         }
 
-gotvolinfo:
-        if (!glusterd_auth_get_username (volinfo))
-                trusted_str = NULL;
+        ret = sys_access(path, F_OK);
+        goto out;
+    }
 
-        ret = snprintf (path, path_len, "%s/%s/%s.vol", path_prefix,
-                        volinfo->volname, volid_ptr);
-        if (ret == -1)
-                goto out;
+    if (volume_id[0] == '/') {
+        /* Normal behavior */
+        volid_ptr = volume_id;
+        volid_ptr++;
 
-        ret = stat (path, &stbuf);
-
-        if ((ret == -1) && (errno == ENOENT)) {
-                strncpy (dup_volid, volid_ptr, (PATH_MAX - 1));
-                if (!strchr (dup_volid, '.')) {
-                        switch (volinfo->transport_type) {
-                        case GF_TRANSPORT_TCP:
-                                strcat (dup_volid, ".tcp");
-                                break;
-                        case GF_TRANSPORT_RDMA:
-                                strcat (dup_volid, ".rdma");
-                                break;
-                        case GF_TRANSPORT_BOTH_TCP_RDMA:
-                                strcat (dup_volid, ".tcp");
-                                break;
-                        default:
-                                ret = -1;
-                                break;
-                        }
-                }
-                snprintf (path, path_len, "%s/%s/%s%s-fuse.vol",
-                          path_prefix, volinfo->volname,
-                          (trusted_str ? trusted_str : ""),
-                          dup_volid);
-                ret = stat (path, &stbuf);
+    } else {
+        /* Bringing in NFS like behavior for mount command, */
+        /* With this, one can mount a volume with below cmd */
+        /* bash# mount -t glusterfs server:/volume /mnt/pnt */
+        volid_ptr = volume_id;
+    }
+
+    len = snprintf(path_prefix, sizeof(path_prefix), "%s/vols", priv->workdir);
+    if ((len < 0) || (len >= sizeof(path_prefix))) {
+        ret = -1;
+        goto out;
+    }
+
+    ret = glusterd_volinfo_find(volid_ptr, &volinfo);
+
+    if (ret) {
+        dup_volname = gf_strdup(volid_ptr);
+        if (!dup_volname) {
+            gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_STRDUP_FAILED,
+                    "Volume name=%s", volid_ptr, NULL);
+            ret = -1;
+            goto out;
+        }
+        /* Split the volume name */
+        vol = strtok_r(dup_volname, ".", &save_ptr);
+        if (!vol) {
+            gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_SPLIT_FAIL,
+                    "Volume name=%s", dup_volname, NULL);
+            ret = -1;
+            goto out;
+        }
+        ret = glusterd_volinfo_find(vol, &volinfo);
+        if (ret) {
+            gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_VOLINFO_GET_FAIL,
+                    NULL);
+            goto out;
         }
+    }
+
+gotvolinfo:
+    if (!glusterd_auth_get_username(volinfo))
+        trusted_str = NULL;
+
+    ret = snprintf(path, path_len, "%s/%s/%s.vol", path_prefix,
+                   volinfo->volname, volid_ptr);
+    if (ret == -1) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_COPY_FAIL, NULL);
+        goto out;
+    }
+
+    ret = sys_stat(path, &stbuf);
+
+    if ((ret == -1) && (errno == ENOENT)) {
+        if (snprintf(dup_volid, PATH_MAX, "%s", volid_ptr) >= PATH_MAX)
+            goto out;
+        if (!strchr(dup_volid, '.')) {
+            switch (volinfo->transport_type) {
+                case GF_TRANSPORT_TCP:
+                    strcat(dup_volid, ".tcp");
+                    break;
+                case GF_TRANSPORT_RDMA:
+                    strcat(dup_volid, ".rdma");
+                    break;
+                case GF_TRANSPORT_BOTH_TCP_RDMA:
+                    strcat(dup_volid, ".tcp");
+                    break;
+                default:
+                    break;
+            }
+        }
+        snprintf(path, path_len, "%s/%s/%s%s-fuse.vol", path_prefix,
+                 volinfo->volname, (trusted_str ? trusted_str : ""), dup_volid);
+        ret = sys_stat(path, &stbuf);
+    }
 out:
-        if (dup_volname)
-                GF_FREE (dup_volname);
-        if (free_ptr)
-                GF_FREE (free_ptr);
-        return ret;
+    if (dup_volname)
+        GF_FREE(dup_volname);
+    if (free_ptr)
+        GF_FREE(free_ptr);
+    return ret;
 }
 
 /* Get and store op-versions of the clients sending the getspec request
@@ -343,618 +526,762 @@ out:
  * defaulted to 1. Also fetch brick_name.
  */
 int32_t
-glusterd_get_args_from_dict (gf_getspec_req *args, peer_info_t *peerinfo,
-                             char **brick_name)
+glusterd_get_args_from_dict(gf_getspec_req *args, peer_info_t *peerinfo,
+                            char **brick_name)
 {
-        dict_t    *dict                  = NULL;
-        int        client_max_op_version = 1;
-        int        client_min_op_version = 1;
-        int32_t    ret                   = -1;
-        xlator_t  *this                  = NULL;
-
-        this = THIS;
-        GF_ASSERT (this);
-        GF_ASSERT (args);
-        GF_ASSERT (peerinfo);
-
-        if (!args->xdata.xdata_len) {
-                ret = 0;
-                goto out;
-        }
-
-        dict = dict_new ();
-        if (!dict) {
-                ret = -1;
-                goto out;
-        }
-
-        ret = dict_unserialize (args->xdata.xdata_val,
-                                args->xdata.xdata_len, &dict);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "Failed to unserialize request dictionary");
-                goto out;
-        }
-
-        ret = dict_get_int32 (dict, "min-op-version",
-                              &client_min_op_version);
-        if (ret) {
-                gf_log ("glusterd", GF_LOG_ERROR,
-                        "Failed to get client-min-op-version");
-                goto out;
-        }
-
-        ret = dict_get_int32 (dict, "max-op-version",
-                              &client_max_op_version);
-        if (ret) {
-                gf_log ("glusterd", GF_LOG_ERROR,
-                        "Failed to get client-max-op-version");
-                goto out;
-        }
+    dict_t *dict = NULL;
+    int client_max_op_version = 1;
+    int client_min_op_version = 1;
+    int32_t ret = -1;
+    xlator_t *this = NULL;
+    char *name = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+    GF_ASSERT(args);
+    GF_ASSERT(peerinfo);
+
+    if (!args->xdata.xdata_len) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_INVALID_ARGUMENT, NULL);
+        ret = 0;
+        goto out;
+    }
 
-        ret = dict_get_str (dict, "brick_name",
-                            brick_name);
-        if (ret) {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "No brick name present");
-                ret = 0;
-                goto out;
-        }
+    dict = dict_new();
+    if (!dict) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_CREATE_FAIL, NULL);
+        ret = -1;
+        goto out;
+    }
+
+    ret = dict_unserialize(args->xdata.xdata_val, args->xdata.xdata_len, &dict);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_UNSERIALIZE_FAIL,
+               "Failed to unserialize request dictionary");
+        goto out;
+    }
+
+    ret = dict_get_int32(dict, "min-op-version", &client_min_op_version);
+    if (ret) {
+        gf_msg("glusterd", GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+               "Failed to get client-min-op-version");
+        goto out;
+    }
+
+    ret = dict_get_int32(dict, "max-op-version", &client_max_op_version);
+    if (ret) {
+        gf_msg("glusterd", GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+               "Failed to get client-max-op-version");
+        goto out;
+    }
+
+    ret = dict_get_str(dict, "brick_name", &name);
+    if (ret) {
+        gf_msg_debug(this->name, 0, "No brick name present");
+        ret = 0;
+        goto out;
+    }
+    *brick_name = gf_strdup(name);
+    if (*brick_name == NULL) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_STRDUP_FAILED,
+                "Brick_name=%s", name, NULL);
+        ret = -1;
+        goto out;
+    }
 
-        gf_log (this->name, GF_LOG_DEBUG, "brick_name = %s", *brick_name);
+    gf_msg_debug(this->name, 0, "brick_name = %s", *brick_name);
 out:
-        peerinfo->max_op_version = client_max_op_version;
-        peerinfo->min_op_version = client_min_op_version;
+    peerinfo->max_op_version = client_max_op_version;
+    peerinfo->min_op_version = client_min_op_version;
 
-        return ret;
+    if (dict)
+        dict_unref(dict);
+
+    return ret;
 }
 
 /* Given the missed_snapinfo and snap_opinfo take the
  * missed lvm snapshot
  */
 int32_t
-glusterd_create_missed_snap (glusterd_missed_snap_info *missed_snapinfo,
-                             glusterd_snap_op_t *snap_opinfo)
+glusterd_create_missed_snap(glusterd_missed_snap_info *missed_snapinfo,
+                            glusterd_snap_op_t *snap_opinfo)
 {
-        char                        *device           = NULL;
-        glusterd_conf_t             *priv             = NULL;
-        glusterd_snap_t             *snap             = NULL;
-        glusterd_volinfo_t          *snap_vol         = NULL;
-        glusterd_volinfo_t          *volinfo          = NULL;
-        glusterd_brickinfo_t        *brickinfo        = NULL;
-        int32_t                      ret              = -1;
-        int32_t                      i                = 0;
-        uuid_t                       snap_uuid        = {0,};
-        xlator_t                    *this             = NULL;
-
-        this = THIS;
-        GF_ASSERT (this);
-        priv = this->private;
-        GF_ASSERT (priv);
-        GF_ASSERT (missed_snapinfo);
-        GF_ASSERT (snap_opinfo);
-
-        gf_uuid_parse (missed_snapinfo->snap_uuid, snap_uuid);
-
-        /* Find the snap-object */
-        snap = glusterd_find_snap_by_id (snap_uuid);
-        if (!snap) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "Unable to find the snap with snap_uuid %s",
-                        missed_snapinfo->snap_uuid);
-                ret = -1;
-                goto out;
-        }
-
-        /* Find the snap_vol */
-        cds_list_for_each_entry (volinfo, &snap->volumes, vol_list) {
-                if (!strcmp (volinfo->volname,
-                             snap_opinfo->snap_vol_id)) {
-                        snap_vol = volinfo;
-                        break;
-                }
-        }
-
-        if (!snap_vol) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "Unable to find the snap_vol(%s) "
-                        "for snap(%s)", snap_opinfo->snap_vol_id,
-                        snap->snapname);
-                ret = -1;
-                goto out;
-        }
-
-        /* Find the missed brick in the snap volume */
-        cds_list_for_each_entry (brickinfo, &snap_vol->bricks, brick_list) {
-                i++;
-                if (i == snap_opinfo->brick_num)
-                        break;
-        }
-
-        if (brickinfo->snap_status != -1) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "The snap status of the missed "
-                        "brick(%s) is not pending", brickinfo->path);
-                goto out;
-        }
-
-        /* Fetch the device path */
-        device = glusterd_get_brick_mount_device (snap_opinfo->brick_path);
-        if (!device) {
-                gf_log (this->name, GF_LOG_ERROR, "Getting device name for the"
-                        "brick %s:%s failed", brickinfo->hostname,
-                        snap_opinfo->brick_path);
-                ret = -1;
-                goto out;
-        }
-
-        device = glusterd_build_snap_device_path (device, snap_vol->volname,
-                                                  snap_opinfo->brick_num - 1);
-        if (!device) {
-                gf_log (this->name, GF_LOG_ERROR, "cannot copy the snapshot "
-                        "device name (volname: %s, snapname: %s)",
-                         snap_vol->volname, snap->snapname);
-                ret = -1;
-                goto out;
-        }
-        strncpy (brickinfo->device_path, device,
-                 sizeof(brickinfo->device_path));
-
-        /* Update the backend file-system type of snap brick in
-         * snap volinfo. */
-        ret = glusterd_update_mntopts (snap_opinfo->brick_path, brickinfo);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "Failed to update "
-                        "mount options for %s brick", brickinfo->path);
-                /* We should not fail snapshot operation if we fail to get
-                 * the file-system type */
-        }
-
-        ret = glusterd_take_lvm_snapshot (brickinfo, snap_opinfo->brick_path);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "Failed to take snapshot of %s",
-                        snap_opinfo->brick_path);
-                goto out;
-        }
-
-        /* After the snapshot both the origin brick (LVM brick) and
-         * the snapshot brick will have the same file-system label. This
-         * will cause lot of problems at mount time. Therefore we must
-         * generate a new label for the snapshot brick
+    char *device = NULL;
+    glusterd_conf_t *priv = NULL;
+    glusterd_snap_t *snap = NULL;
+    glusterd_volinfo_t *snap_vol = NULL;
+    glusterd_volinfo_t *volinfo = NULL;
+    glusterd_brickinfo_t *brickinfo = NULL;
+    int32_t ret = -1;
+    int32_t i = 0;
+    uuid_t snap_uuid = {
+        0,
+    };
+    xlator_t *this = NULL;
+    char *mnt_device = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+    priv = this->private;
+    GF_ASSERT(priv);
+    GF_ASSERT(missed_snapinfo);
+    GF_ASSERT(snap_opinfo);
+
+    gf_uuid_parse(missed_snapinfo->snap_uuid, snap_uuid);
+
+    /* Find the snap-object */
+    snap = glusterd_find_snap_by_id(snap_uuid);
+    if (!snap) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_SNAP_NOT_FOUND,
+               "Unable to find the snap with snap_uuid %s",
+               missed_snapinfo->snap_uuid);
+        ret = -1;
+        goto out;
+    }
+
+    /* Find the snap_vol */
+    cds_list_for_each_entry(volinfo, &snap->volumes, vol_list)
+    {
+        if (!strcmp(volinfo->volname, snap_opinfo->snap_vol_id)) {
+            snap_vol = volinfo;
+            break;
+        }
+    }
+
+    if (!snap_vol) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_VOL_NOT_FOUND,
+               "Unable to find the snap_vol(%s) "
+               "for snap(%s)",
+               snap_opinfo->snap_vol_id, snap->snapname);
+        ret = -1;
+        goto out;
+    }
+
+    /* Find the missed brick in the snap volume */
+    cds_list_for_each_entry(brickinfo, &snap_vol->bricks, brick_list)
+    {
+        i++;
+        if (i == snap_opinfo->brick_num)
+            break;
+    }
+
+    if (brickinfo->snap_status != -1) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_SNAP_STATUS_NOT_PENDING,
+               "The snap status of the missed "
+               "brick(%s) is not pending",
+               brickinfo->path);
+        goto out;
+    }
+
+    /* Fetch the device path */
+    mnt_device = glusterd_get_brick_mount_device(snap_opinfo->brick_path);
+    if (!mnt_device) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_BRICK_GET_INFO_FAIL,
+               "Getting device name for the"
+               "brick %s:%s failed",
+               brickinfo->hostname, snap_opinfo->brick_path);
+        ret = -1;
+        goto out;
+    }
+
+    device = glusterd_build_snap_device_path(mnt_device, snap_vol->volname,
+                                             snap_opinfo->brick_num - 1);
+    if (!device) {
+        gf_msg(this->name, GF_LOG_ERROR, ENXIO,
+               GD_MSG_SNAP_DEVICE_NAME_GET_FAIL,
+               "cannot copy the snapshot "
+               "device name (volname: %s, snapname: %s)",
+               snap_vol->volname, snap->snapname);
+        ret = -1;
+        goto out;
+    }
+    if (snprintf(brickinfo->device_path, sizeof(brickinfo->device_path), "%s",
+                 device) >= sizeof(brickinfo->device_path)) {
+        gf_msg(this->name, GF_LOG_ERROR, ENXIO,
+               GD_MSG_SNAP_DEVICE_NAME_GET_FAIL,
+               "cannot copy the device_path "
+               "(device_path: %s)",
+               brickinfo->device_path);
+        ret = -1;
+        goto out;
+    }
+
+    /* Update the backend file-system type of snap brick in
+     * snap volinfo. */
+    ret = glusterd_update_mntopts(snap_opinfo->brick_path, brickinfo);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_BRK_MOUNTOPTS_FAIL,
+               "Failed to update "
+               "mount options for %s brick",
+               brickinfo->path);
+        /* We should not fail snapshot operation if we fail to get
+         * the file-system type */
+    }
+
+    ret = glusterd_take_lvm_snapshot(brickinfo, snap_opinfo->brick_path);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_SNAPSHOT_OP_FAILED,
+               "Failed to take snapshot of %s", snap_opinfo->brick_path);
+        goto out;
+    }
+
+    /* After the snapshot both the origin brick (LVM brick) and
+     * the snapshot brick will have the same file-system label. This
+     * will cause lot of problems at mount time. Therefore we must
+     * generate a new label for the snapshot brick
+     */
+    ret = glusterd_update_fs_label(brickinfo);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_BRICK_SET_INFO_FAIL,
+               "Failed to update "
+               "file-system label for %s brick",
+               brickinfo->path);
+        /* Failing to update label should not cause snapshot failure.
+         * Currently label is updated only for XFS and ext2/ext3/ext4
+         * file-system.
          */
-        ret = glusterd_update_fs_label (brickinfo);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "Failed to update "
-                        "file-system label for %s brick", brickinfo->path);
-                /* Failing to update label should not cause snapshot failure.
-                 * Currently label is updated only for XFS and ext2/ext3/ext4
-                 * file-system.
-                 */
-        }
+    }
+
+    /* Create and mount the snap brick */
+    ret = glusterd_snap_brick_create(snap_vol, brickinfo,
+                                     snap_opinfo->brick_num - 1, 0);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_BRICK_CREATION_FAIL,
+               "Failed to "
+               " create and mount the brick(%s) for the snap %s",
+               snap_opinfo->brick_path, snap_vol->snapshot->snapname);
+        goto out;
+    }
+
+    brickinfo->snap_status = 0;
+    ret = glusterd_brick_start(snap_vol, brickinfo, _gf_false, _gf_false);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_WARNING, 0, GD_MSG_BRICK_DISCONNECTED,
+               "starting the "
+               "brick %s:%s for the snap %s failed",
+               brickinfo->hostname, brickinfo->path, snap->snapname);
+        goto out;
+    }
+    ret = glusterd_store_volinfo(snap_vol, GLUSTERD_VOLINFO_VER_AC_NONE);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_VOLINFO_STORE_FAIL,
+               "Failed to store snapshot "
+               "volinfo (%s) for snap %s",
+               snap_vol->volname, snap->snapname);
+        goto out;
+    }
 
-        /* Create and mount the snap brick */
-        ret = glusterd_snap_brick_create (snap_vol, brickinfo,
-                                          snap_opinfo->brick_num - 1);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "Failed to "
-                        " create and mount the brick(%s) for the snap %s",
-                        snap_opinfo->brick_path,
-                        snap_vol->snapshot->snapname);
-                goto out;
-        }
-
-        brickinfo->snap_status = 0;
-        ret = glusterd_store_volinfo (snap_vol,
-                                      GLUSTERD_VOLINFO_VER_AC_NONE);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "Failed to store snapshot "
-                        "volinfo (%s) for snap %s", snap_vol->volname,
-                        snap->snapname);
-                goto out;
-        }
-
-        ret = glusterd_brick_start (snap_vol, brickinfo, _gf_false);
-        if (ret) {
-                gf_log (this->name, GF_LOG_WARNING, "starting the "
-                        "brick %s:%s for the snap %s failed",
-                        brickinfo->hostname, brickinfo->path,
-                        snap->snapname);
-                goto out;
-        }
 out:
-        if (device)
-                GF_FREE (device);
+    if (mnt_device)
+        GF_FREE(mnt_device);
+    if (device)
+        GF_FREE(device);
 
-        return ret;
+    return ret;
 }
 
 /* Look into missed_snap_list, to see it the given brick_name,
  * has any missed snap creates for the local node */
 int32_t
-glusterd_take_missing_brick_snapshots (char *brick_name)
+glusterd_take_missing_brick_snapshots(char *brick_name)
 {
-        char                        *my_node_uuid     = NULL;
-        glusterd_conf_t             *priv             = NULL;
-        glusterd_missed_snap_info   *missed_snapinfo  = NULL;
-        glusterd_snap_op_t          *snap_opinfo      = NULL;
-        int32_t                      ret              = -1;
-        gf_boolean_t                 update_list      = _gf_false;
-        xlator_t                    *this             = NULL;
-
-        this = THIS;
-        GF_ASSERT (this);
-        priv = this->private;
-        GF_ASSERT (priv);
-        GF_ASSERT (brick_name);
-
-        my_node_uuid = uuid_utoa (MY_UUID);
-
-        cds_list_for_each_entry (missed_snapinfo, &priv->missed_snaps_list,
-                                 missed_snaps) {
-                /* If the missed snap op is not for the local node
-                 * then continue
+    char *my_node_uuid = NULL;
+    glusterd_conf_t *priv = NULL;
+    glusterd_missed_snap_info *missed_snapinfo = NULL;
+    glusterd_snap_op_t *snap_opinfo = NULL;
+    int32_t ret = -1;
+    gf_boolean_t update_list = _gf_false;
+    xlator_t *this = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+    priv = this->private;
+    GF_ASSERT(priv);
+    GF_ASSERT(brick_name);
+
+    my_node_uuid = uuid_utoa(MY_UUID);
+
+    cds_list_for_each_entry(missed_snapinfo, &priv->missed_snaps_list,
+                            missed_snaps)
+    {
+        /* If the missed snap op is not for the local node
+         * then continue
+         */
+        if (strcmp(my_node_uuid, missed_snapinfo->node_uuid))
+            continue;
+
+        cds_list_for_each_entry(snap_opinfo, &missed_snapinfo->snap_ops,
+                                snap_ops_list)
+        {
+            /* Check if the missed snap's op is a create for
+             * the brick name in question
+             */
+            if ((snap_opinfo->op == GF_SNAP_OPTION_TYPE_CREATE) &&
+                (!strcmp(brick_name, snap_opinfo->brick_path))) {
+                /* Perform a snap create if the
+                 * op is still pending
                  */
-                if (strcmp (my_node_uuid, missed_snapinfo->node_uuid))
-                        continue;
-
-                cds_list_for_each_entry (snap_opinfo,
-                                         &missed_snapinfo->snap_ops,
-                                         snap_ops_list) {
-                        /* Check if the missed snap's op is a create for
-                         * the brick name in question
+                if (snap_opinfo->status == GD_MISSED_SNAP_PENDING) {
+                    ret = glusterd_create_missed_snap(missed_snapinfo,
+                                                      snap_opinfo);
+                    if (ret) {
+                        gf_msg(this->name, GF_LOG_ERROR, 0,
+                               GD_MSG_MISSED_SNAP_CREATE_FAIL,
+                               "Failed to create "
+                               "missed snap for %s",
+                               brick_name);
+                        /* At this stage, we will mark
+                         * the entry as done. Because
+                         * of the failure other
+                         * snapshots will not be
+                         * affected, and neither the
+                         * brick. Only the current snap
+                         * brick will always remain as
+                         * pending.
                          */
-                        if ((snap_opinfo->op == GF_SNAP_OPTION_TYPE_CREATE) &&
-                            (!strcmp (brick_name, snap_opinfo->brick_path))) {
-                                /* Perform a snap create if the
-                                 * op is still pending
-                                 */
-                                if (snap_opinfo->status ==
-                                                 GD_MISSED_SNAP_PENDING) {
-                                        ret = glusterd_create_missed_snap
-                                                              (missed_snapinfo,
-                                                               snap_opinfo);
-                                        if (ret) {
-                                                gf_log (this->name,
-                                                        GF_LOG_ERROR,
-                                                        "Failed to create "
-                                                        "missed snap for %s",
-                                                        brick_name);
-                                                /* At this stage, we will mark
-                                                 * the entry as done. Because
-                                                 * of the failure other
-                                                 * snapshots will not be
-                                                 * affected, and neither the
-                                                 * brick. Only the current snap
-                                                 * brick will always remain as
-                                                 * pending.
-                                                 */
-                                        }
-                                        snap_opinfo->status =
-                                                 GD_MISSED_SNAP_DONE;
-                                        update_list = _gf_true;
-                                }
-                                /* One snap-id won't have more than one missed
-                                 * create for the same brick path. Hence
-                                 * breaking in search of another missed create
-                                 * for the same brick path in the local node
-                                 */
-                                break;
-                        }
+                    }
+                    snap_opinfo->status = GD_MISSED_SNAP_DONE;
+                    update_list = _gf_true;
                 }
+                /* One snap-id won't have more than one missed
+                 * create for the same brick path. Hence
+                 * breaking in search of another missed create
+                 * for the same brick path in the local node
+                 */
+                break;
+            }
         }
+    }
 
-        if (update_list == _gf_true) {
-                ret = glusterd_store_update_missed_snaps ();
-                if (ret) {
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "Failed to update missed_snaps_list");
-                        goto out;
-                }
+    if (update_list == _gf_true) {
+        ret = glusterd_store_update_missed_snaps();
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, 0,
+                   GD_MSG_MISSED_SNAP_LIST_STORE_FAIL,
+                   "Failed to update missed_snaps_list");
+            goto out;
         }
+    }
 
-        ret = 0;
+    ret = 0;
 out:
-        return ret;
+    return ret;
 }
 
 /* Checks if the client supports the volume, ie. client can understand all the
  * options in the volfile
  */
 static gf_boolean_t
-_client_supports_volume (peer_info_t *peerinfo, int32_t *op_errno)
+_client_supports_volume(peer_info_t *peerinfo, int32_t *op_errno)
 {
-        gf_boolean_t       ret       = _gf_true;
-        glusterd_volinfo_t *volinfo  = NULL;
-
-        GF_ASSERT (peerinfo);
-        GF_ASSERT (op_errno);
-
-
-        /* Only check when the volfile being requested is a volume. Not finding
-         * a volinfo implies that the volfile requested for is not of a gluster
-         * volume. A non volume volfile is requested by the local gluster
-         * services like shd and nfs-server. These need not be checked as they
-         * will be running at the same op-version as glusterd and will be able
-         * to support all the features
-         */
-        if ((glusterd_volinfo_find (peerinfo->volname, &volinfo) == 0) &&
-            ((peerinfo->min_op_version > volinfo->client_op_version) ||
-             (peerinfo->max_op_version < volinfo->client_op_version))) {
-                ret = _gf_false;
-                *op_errno = ENOTSUP;
-                gf_log ("glusterd", GF_LOG_INFO,
-                        "Client %s (%d -> %d) doesn't support required "
-                        "op-version (%d). Rejecting volfile request.",
-                        peerinfo->identifier, peerinfo->min_op_version,
-                        peerinfo->max_op_version, volinfo->client_op_version);
-        }
-
-        return ret;
+    gf_boolean_t ret = _gf_true;
+    glusterd_volinfo_t *volinfo = NULL;
+
+    GF_ASSERT(peerinfo);
+    GF_ASSERT(op_errno);
+
+    /* Only check when the volfile being requested is a volume. Not finding
+     * a volinfo implies that the volfile requested for is not of a gluster
+     * volume. A non volume volfile is requested by the local gluster
+     * services like shd and nfs-server. These need not be checked as they
+     * will be running at the same op-version as glusterd and will be able
+     * to support all the features
+     */
+    if ((glusterd_volinfo_find(peerinfo->volname, &volinfo) == 0) &&
+        ((peerinfo->min_op_version > volinfo->client_op_version) ||
+         (peerinfo->max_op_version < volinfo->client_op_version))) {
+        ret = _gf_false;
+        *op_errno = ENOTSUP;
+        gf_msg("glusterd", GF_LOG_INFO, ENOTSUP, GD_MSG_UNSUPPORTED_VERSION,
+               "Client %s (%d -> %d) doesn't support required "
+               "op-version (%d). Rejecting volfile request.",
+               peerinfo->identifier, peerinfo->min_op_version,
+               peerinfo->max_op_version, volinfo->client_op_version);
+    }
+
+    return ret;
 }
 
 int
-__server_getspec (rpcsvc_request_t *req)
+__server_getspec(rpcsvc_request_t *req)
 {
-        int32_t               ret                    = -1;
-        int32_t               op_ret                 = -1;
-        int32_t               op_errno               = 0;
-        int32_t               spec_fd                = -1;
-        size_t                file_len               = 0;
-        char                  filename[PATH_MAX]  = {0,};
-        struct stat           stbuf                  = {0,};
-        char                 *brick_name             = NULL;
-        char                 *volume                 = NULL;
-        char                 *tmp                    = NULL;
-        int                   cookie                 = 0;
-        rpc_transport_t      *trans                  = NULL;
-        gf_getspec_req        args                   = {0,};
-        gf_getspec_rsp        rsp                    = {0,};
-        char                  addrstr[RPCSVC_PEER_STRLEN] = {0};
-        peer_info_t          *peerinfo               = NULL;
-        xlator_t             *this                   = NULL;
-
-        this = THIS;
-        GF_ASSERT (this);
-
-        ret = xdr_to_generic (req->msg[0], &args,
-                              (xdrproc_t)xdr_gf_getspec_req);
-        if (ret < 0) {
-                //failed to decode msg;
-                req->rpc_err = GARBAGE_ARGS;
-                goto fail;
-        }
-
-        peerinfo = &req->trans->peerinfo;
-
-        volume = args.key;
-        /* Need to strip leading '/' from volnames. This was introduced to
-         * support nfs style mount parameters for native gluster mount
-         */
-        if (volume[0] == '/')
-                strncpy (peerinfo->volname, &volume[1], strlen(&volume[1]));
-        else
-                strncpy (peerinfo->volname, volume, strlen(volume));
-
-        ret = glusterd_get_args_from_dict (&args, peerinfo, &brick_name);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "Failed to get args from dict");
-                goto fail;
-        }
-
-        if (!_client_supports_volume (peerinfo, &op_errno)) {
-                ret = -1;
-                goto fail;
-        }
-
-        trans = req->trans;
-        /* addrstr will be empty for cli socket connections */
-        ret = rpcsvc_transport_peername (trans, (char *)&addrstr,
-                                         sizeof (addrstr));
-        if (ret)
-                goto fail;
-
-        tmp  = strrchr (addrstr, ':');
-        if (tmp)
-                *tmp = '\0';
+    int32_t ret = -1;
+    int32_t op_ret = -1;
+    int32_t op_errno = 0;
+    int32_t spec_fd = -1;
+    size_t file_len = 0;
+    char filename[PATH_MAX] = {
+        0,
+    };
+    struct stat stbuf = {
+        0,
+    };
+    char *brick_name = NULL;
+    char *volume = NULL;
+    char *tmp = NULL;
+    rpc_transport_t *trans = NULL;
+    gf_getspec_req args = {
+        0,
+    };
+    gf_getspec_rsp rsp = {
+        0,
+    };
+    char addrstr[RPCSVC_PEER_STRLEN] = {0};
+    peer_info_t *peerinfo = NULL;
+    xlator_t *this = NULL;
+    dict_t *dict = NULL;
+    glusterd_peerinfo_t *peer = NULL;
+    glusterd_conf_t *conf = NULL;
+    int peer_cnt = 0;
+    char *peer_hosts = NULL;
+    char *tmp_str = NULL;
+    char portstr[10] = {
+        0,
+    };
+    int len = 0;
+
+    this = THIS;
+    GF_ASSERT(this);
+
+    conf = this->private;
+    ret = xdr_to_generic(req->msg[0], &args, (xdrproc_t)xdr_gf_getspec_req);
+    if (ret < 0) {
+        // failed to decode msg;
+        req->rpc_err = GARBAGE_ARGS;
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_REQ_DECODE_FAIL,
+               "Failed to decode the message");
+        goto fail;
+    }
+
+    peerinfo = &req->trans->peerinfo;
+
+    volume = args.key;
+
+    if (strlen(volume) >= (NAME_MAX)) {
+        op_errno = EINVAL;
+        gf_msg(this->name, GF_LOG_ERROR, EINVAL, GD_MSG_NAME_TOO_LONG,
+               "volume name too long (%s)", volume);
+        goto fail;
+    }
+
+    gf_msg(this->name, GF_LOG_INFO, 0, GD_MSG_MOUNT_REQ_RCVD,
+           "Received mount request for volume %s", volume);
+
+    /* Need to strip leading '/' from volnames. This was introduced to
+     * support nfs style mount parameters for native gluster mount
+     */
+    if (volume[0] == '/')
+        ret = snprintf(peerinfo->volname, sizeof(peerinfo->volname), "%s",
+                       &volume[1]);
+    else
+        ret = snprintf(peerinfo->volname, sizeof(peerinfo->volname), "%s",
+                       volume);
+    if (ret < 0 || ret >= sizeof(peerinfo->volname)) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_VOLINFO_GET_FAIL,
+               "peerinfo->volname %s truncated or error occurred: "
+               "(ret: %d)",
+               peerinfo->volname, ret);
+        ret = -1;
+        goto fail;
+    }
 
-        /* The trusted volfiles are given to the glusterd owned process like NFS
-         * server, self-heal daemon etc., so that they are not inadvertently
-         * blocked by a auth.{allow,reject} setting. The trusted volfile is not
-         * meant for external users.
-         */
-        if (strlen (addrstr) && gf_is_local_addr (addrstr)) {
+    ret = glusterd_get_args_from_dict(&args, peerinfo, &brick_name);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+               "Failed to get args from dict");
+        goto fail;
+    }
 
-                ret = build_volfile_path (volume, filename,
-                                          sizeof (filename),
-                                          TRUSTED_PREFIX);
+    if (!_client_supports_volume(peerinfo, &op_errno)) {
+        ret = -1;
+        goto fail;
+    }
+
+    dict = dict_new();
+    if (!dict) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_CREATE_FAIL, NULL);
+        ret = -ENOMEM;
+        goto fail;
+    }
+
+    trans = req->trans;
+    /* addrstr will be empty for cli socket connections */
+    ret = rpcsvc_transport_peername(trans, (char *)&addrstr, sizeof(addrstr));
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0,
+               GD_MSG_RPC_TRANSPORT_GET_PEERNAME_FAIL,
+               "Failed to get the peername");
+        goto fail;
+    }
+
+    tmp = strrchr(addrstr, ':');
+    if (tmp)
+        *tmp = '\0';
+
+    /* The trusted volfiles are given to the glusterd owned process like NFS
+     * server, self-heal daemon etc., so that they are not inadvertently
+     * blocked by a auth.{allow,reject} setting. The trusted volfile is not
+     * meant for external users.
+     * For unix domain socket, address will be empty.
+     */
+    if (strlen(addrstr) == 0 || gf_is_local_addr(addrstr)) {
+        ret = build_volfile_path(volume, filename, sizeof(filename),
+                                 TRUSTED_PREFIX, dict);
+    } else {
+        ret = build_volfile_path(volume, filename, sizeof(filename), NULL,
+                                 dict);
+    }
+
+    RCU_READ_LOCK;
+    cds_list_for_each_entry_rcu(peer, &conf->peers, uuid_list)
+    {
+        if (!peer->connected)
+            continue;
+        if (!peer_hosts) {
+            if (peer->port) {
+                snprintf(portstr, sizeof(portstr), "%d", peer->port);
+            } else {
+                snprintf(portstr, sizeof(portstr), "%d", GLUSTERD_DEFAULT_PORT);
+            }
+            len = strlen(peer->hostname) + strlen(portstr) + 3;
+            tmp_str = GF_CALLOC(1, len, gf_gld_mt_char);
+            snprintf(tmp_str, len, "%s%s%s%s", peer->hostname, ":", portstr,
+                     " ");
+            peer_hosts = tmp_str;
         } else {
-                ret = build_volfile_path (volume, filename,
-                                          sizeof (filename), NULL);
+            len = strlen(peer_hosts) + strlen(peer->hostname) +
+                  strlen(portstr) + 3;
+            tmp_str = GF_CALLOC(1, len, gf_gld_mt_char);
+            snprintf(tmp_str, len, "%s%s%s%s%s", peer_hosts, peer->hostname,
+                     ":", portstr, " ");
+            GF_FREE(peer_hosts);
+            peer_hosts = tmp_str;
+        }
+        peer_cnt++;
+    }
+    RCU_READ_UNLOCK;
+    if (peer_cnt) {
+        op_ret = dict_set_str(dict, GLUSTERD_BRICK_SERVERS, peer_hosts);
+        if (op_ret) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+                   "failed to set peer_host in dict");
+            ret = op_ret;
+            goto fail;
+        }
+    }
+
+    if (ret == 0) {
+        if (dict->count > 0) {
+            ret = dict_allocate_and_serialize(dict, &rsp.xdata.xdata_val,
+                                              &rsp.xdata.xdata_len);
+            if (ret) {
+                gf_smsg(this->name, GF_LOG_ERROR, errno,
+                        GD_MSG_DICT_ALLOC_AND_SERL_LENGTH_GET_FAIL, NULL);
+                goto fail;
+            }
         }
 
-        if (ret == 0) {
-                /* to allocate the proper buffer to hold the file data */
-                ret = stat (filename, &stbuf);
-                if (ret < 0){
-                        gf_log ("glusterd", GF_LOG_ERROR,
-                                "Unable to stat %s (%s)",
-                                filename, strerror (errno));
-                        goto fail;
-                }
+        /* to allocate the proper buffer to hold the file data */
+        ret = sys_stat(filename, &stbuf);
+        if (ret < 0) {
+            gf_msg("glusterd", GF_LOG_ERROR, errno, GD_MSG_FILE_OP_FAILED,
+                   "Unable to stat %s (%s)", filename, strerror(errno));
+            goto fail;
+        }
+
+        spec_fd = open(filename, O_RDONLY);
+        if (spec_fd < 0) {
+            gf_msg("glusterd", GF_LOG_ERROR, errno, GD_MSG_FILE_OP_FAILED,
+                   "Unable to open %s (%s)", filename, strerror(errno));
+            goto fail;
+        }
+        ret = file_len = stbuf.st_size;
+    } else {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_PEER_NOT_FOUND, NULL);
+        op_errno = ENOENT;
+        goto fail;
+    }
+
+    if (file_len) {
+        rsp.spec = CALLOC(file_len + 1, sizeof(char));
+        if (!rsp.spec) {
+            gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_NO_MEMORY, NULL);
+            ret = -1;
+            op_errno = ENOMEM;
+            goto fail;
+        }
+        ret = sys_read(spec_fd, rsp.spec, file_len);
+    }
+
+    if (brick_name) {
+        gf_msg_debug(this->name, 0, "Look for missing snap creates for %s",
+                     brick_name);
+        op_ret = glusterd_take_missing_brick_snapshots(brick_name);
+        if (op_ret) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_MISSED_SNAP_CREATE_FAIL,
+                   "Failed to take missing brick snapshots");
+            ret = -1;
+            goto fail;
+        }
+    }
+    /* convert to XDR */
+fail:
+    if (spec_fd >= 0)
+        sys_close(spec_fd);
 
-                spec_fd = open (filename, O_RDONLY);
-                if (spec_fd < 0) {
-                        gf_log ("glusterd", GF_LOG_ERROR,
-                                "Unable to open %s (%s)",
-                                filename, strerror (errno));
-                        goto fail;
-                }
-                ret = file_len = stbuf.st_size;
-        } else {
-                op_errno = ENOENT;
-                goto fail;
-        }
+    GF_FREE(brick_name);
 
-        if (file_len) {
-                rsp.spec = CALLOC (file_len+1, sizeof (char));
-                if (!rsp.spec) {
-                        ret = -1;
-                        op_errno = ENOMEM;
-                        goto fail;
-                }
-                ret = read (spec_fd, rsp.spec, file_len);
-        }
+    rsp.op_ret = ret;
+    if (rsp.op_ret < 0)
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_MOUNT_REQ_FAIL,
+               "Failed to mount the volume");
 
-        if (brick_name) {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "Look for missing snap creates for %s", brick_name);
-                op_ret = glusterd_take_missing_brick_snapshots (brick_name);
-                if (op_ret) {
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "Failed to take missing brick snapshots");
-                        ret = -1;
-                        goto fail;
-                }
-        }
+    if (op_errno)
+        rsp.op_errno = gf_errno_to_error(op_errno);
 
-        /* convert to XDR */
-fail:
-        if (spec_fd > 0)
-                close (spec_fd);
+    if (!rsp.spec)
+        rsp.spec = strdup("");
 
-        rsp.op_ret   = ret;
+    glusterd_submit_reply(req, &rsp, NULL, 0, NULL,
+                          (xdrproc_t)xdr_gf_getspec_rsp);
+    free(args.key);  // malloced by xdr
+    free(rsp.spec);
 
-        if (op_errno)
-                rsp.op_errno = gf_errno_to_error (op_errno);
-        if (cookie)
-                rsp.op_errno = cookie;
+    if (peer_hosts)
+        GF_FREE(peer_hosts);
+    if (dict)
+        dict_unref(dict);
 
-        if (!rsp.spec)
-                rsp.spec = strdup ("");
+    if (args.xdata.xdata_val)
+        free(args.xdata.xdata_val);
 
-        glusterd_submit_reply (req, &rsp, NULL, 0, NULL,
-                               (xdrproc_t)xdr_gf_getspec_rsp);
-        free (args.key);//malloced by xdr
-        free (rsp.spec);
+    if (rsp.xdata.xdata_val)
+        GF_FREE(rsp.xdata.xdata_val);
 
-        return 0;
+    return 0;
 }
 
 int
-server_getspec (rpcsvc_request_t *req)
+server_getspec(rpcsvc_request_t *req)
 {
-        return glusterd_big_locked_handler (req, __server_getspec);
+    return glusterd_big_locked_handler(req, __server_getspec);
 }
 
 int32_t
-__server_event_notify (rpcsvc_request_t *req)
+__server_event_notify(rpcsvc_request_t *req)
 {
-        int32_t                 ret             = -1;
-        int32_t                 op_errno        =  0;
-        gf_event_notify_req     args            = {0,};
-        gf_event_notify_rsp     rsp             = {0,};
-        dict_t                 *dict            = NULL;
-        gf_boolean_t            need_rsp        = _gf_true;
-
-        ret = xdr_to_generic (req->msg[0], &args,
-                              (xdrproc_t)xdr_gf_event_notify_req);
-        if (ret < 0) {
-                req->rpc_err = GARBAGE_ARGS;
-                goto fail;
+    int32_t ret = -1;
+    gf_event_notify_req args = {
+        0,
+    };
+    gf_event_notify_rsp rsp = {
+        0,
+    };
+    dict_t *dict = NULL;
+    gf_boolean_t need_rsp = _gf_true;
+
+    ret = xdr_to_generic(req->msg[0], &args,
+                         (xdrproc_t)xdr_gf_event_notify_req);
+    if (ret < 0) {
+        req->rpc_err = GARBAGE_ARGS;
+        gf_smsg("glusterd", GF_LOG_ERROR, errno, GD_MSG_GARBAGE_ARGS, NULL);
+        goto fail;
+    }
+
+    if (args.dict.dict_len) {
+        dict = dict_new();
+        if (!dict) {
+            gf_smsg("glusterd", GF_LOG_ERROR, errno, GD_MSG_DICT_CREATE_FAIL,
+                    NULL);
+            return ret;
         }
-
-        if (args.dict.dict_len) {
-                dict = dict_new ();
-                if (!dict)
-                        return ret;
-                ret = dict_unserialize (args.dict.dict_val,
-                                        args.dict.dict_len, &dict);
-                if (ret) {
-                        gf_log ("", GF_LOG_ERROR, "Failed to unserialize req");
-                        goto fail;
-                }
+        ret = dict_unserialize(args.dict.dict_val, args.dict.dict_len, &dict);
+        if (ret) {
+            gf_msg("glusterd", GF_LOG_ERROR, 0, GD_MSG_DICT_UNSERIALIZE_FAIL,
+                   "Failed to unserialize req");
+            goto fail;
         }
+    }
 
-        switch (args.op) {
+    switch (args.op) {
         case GF_EN_DEFRAG_STATUS:
-                gf_log ("", GF_LOG_INFO,
-                        "received defrag status updated");
-                if (dict) {
-                        glusterd_defrag_event_notify_handle (dict);
-                        need_rsp = _gf_false;
-                }
-                break;
+            gf_msg("glusterd", GF_LOG_INFO, 0, GD_MSG_DEFRAG_STATUS_UPDATED,
+                   "received defrag status updated");
+            if (dict) {
+                glusterd_defrag_event_notify_handle(dict);
+                need_rsp = _gf_false;
+            }
+            break;
         default:
-                gf_log ("", GF_LOG_ERROR, "Unknown op received in event "
-                        "notify");
-                ret = -1;
-                break;
-        }
+            gf_msg("glusterd", GF_LOG_ERROR, EINVAL, GD_MSG_OP_UNSUPPORTED,
+                   "Unknown op received in event "
+                   "notify");
+            gf_event(EVENT_NOTIFY_UNKNOWN_OP, "op=%d", args.op);
+            ret = -1;
+            break;
+    }
 
 fail:
-        rsp.op_ret   = ret;
+    rsp.op_ret = ret;
 
-        if (op_errno)
-                rsp.op_errno = gf_errno_to_error (op_errno);
+    if (need_rsp)
+        glusterd_submit_reply(req, &rsp, NULL, 0, NULL,
+                              (xdrproc_t)xdr_gf_event_notify_rsp);
+    if (dict)
+        dict_unref(dict);
+    free(args.dict.dict_val);  // malloced by xdr
 
-        if (need_rsp)
-                glusterd_submit_reply (req, &rsp, NULL, 0, NULL,
-                                       (xdrproc_t)xdr_gf_event_notify_rsp);
-        if (dict)
-                dict_unref (dict);
-        free (args.dict.dict_val);//malloced by xdr
-
-        return 0;
+    return 0;
 }
 
 int32_t
-server_event_notify (rpcsvc_request_t *req)
+server_event_notify(rpcsvc_request_t *req)
 {
-        return glusterd_big_locked_handler (req, __server_event_notify);
+    return glusterd_big_locked_handler(req, __server_event_notify);
 }
 
 int
-gd_validate_cluster_op_version (xlator_t *this, int cluster_op_version,
-                                char *peerid)
+gd_validate_cluster_op_version(xlator_t *this, int cluster_op_version,
+                               char *peerid)
 {
-        int              ret  = -1;
-        glusterd_conf_t *conf = NULL;
-
-        conf = this->private;
-
-        if (cluster_op_version > GD_OP_VERSION_MAX) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "operating version %d is more than the maximum "
-                        "supported (%d) on the machine (as per peer request "
-                        "from %s)", cluster_op_version, GD_OP_VERSION_MAX,
-                        peerid);
-                goto out;
-        }
-
-        /* The peer can only reduce its op-version when it doesn't have any
-         * volumes. Reducing op-version when it already contains volumes can
-         * lead to inconsistencies in the cluster
-         */
-        if ((cluster_op_version < conf->op_version) &&
-            !cds_list_empty (&conf->volumes)) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "cannot reduce operating version to %d from current "
-                        "version %d as volumes exist (as per peer request from "
-                        "%s)", cluster_op_version, conf->op_version, peerid);
-                goto out;
-        }
-
-        ret = 0;
+    int ret = -1;
+    glusterd_conf_t *conf = NULL;
+
+    conf = this->private;
+
+    if (cluster_op_version > GD_OP_VERSION_MAX) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_OP_VERSION_MISMATCH,
+               "operating version %d is more than the maximum "
+               "supported (%d) on the machine (as per peer request "
+               "from %s)",
+               cluster_op_version, GD_OP_VERSION_MAX, peerid);
+        goto out;
+    }
+
+    /* The peer can only reduce its op-version when it doesn't have any
+     * volumes. Reducing op-version when it already contains volumes can
+     * lead to inconsistencies in the cluster
+     */
+    if ((cluster_op_version < conf->op_version) &&
+        !cds_list_empty(&conf->volumes)) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_OP_VERS_ADJUST_FAIL,
+               "cannot reduce operating version to %d from current "
+               "version %d as volumes exist (as per peer request from "
+               "%s)",
+               cluster_op_version, conf->op_version, peerid);
+        goto out;
+    }
+
+    ret = 0;
 out:
-        return ret;
+    return ret;
 }
 
 /* Validate if glusterd can serve the management handshake request
  *
  * Requests are allowed if,
- *  - glusterd has no peers, or
+ *  - glusterd has no peers & no volumes, or
  *  - the request came from a known peer
  * A known peer is identified using the following steps
  *  - the dict is checked for a peer uuid, which if present is matched with the
@@ -962,1169 +1289,1292 @@ out:
  *  - the incoming request address is matched with the peer list
  */
 gf_boolean_t
-gd_validate_mgmt_hndsk_req (rpcsvc_request_t *req, dict_t *dict)
+gd_validate_mgmt_hndsk_req(rpcsvc_request_t *req, dict_t *dict)
 {
-        int                  ret                         = -1;
-        char                 hostname[UNIX_PATH_MAX + 1] = {0,};
-        glusterd_peerinfo_t *peer                        = NULL;
-        xlator_t            *this                        = NULL;
-        char                *uuid_str                    = NULL;
-        uuid_t               peer_uuid                   = {0,};
-
-        this = THIS;
-        GF_ASSERT (this);
-
-        if (!glusterd_have_peers ())
-                return _gf_true;
-
-        ret = dict_get_str (dict, GD_PEER_ID_KEY, &uuid_str);
-        /* Try to match uuid only if available, don't fail as older peers will
-         * not send a uuid
-         */
-        if (!ret) {
-                gf_uuid_parse (uuid_str, peer_uuid);
-                rcu_read_lock ();
-                ret = (glusterd_peerinfo_find (peer_uuid, NULL) != NULL);
-                rcu_read_unlock ();
-                if (ret)
-                        return _gf_true;
-        }
+    int ret = -1;
+    char hostname[UNIX_PATH_MAX + 1] = {
+        0,
+    };
+    glusterd_peerinfo_t *peer = NULL;
+    xlator_t *this = NULL;
+    char *uuid_str = NULL;
+    uuid_t peer_uuid = {
+        0,
+    };
+
+    this = THIS;
+    GF_ASSERT(this);
+
+    if (!glusterd_have_peers() && !glusterd_have_volumes())
+        return _gf_true;
 
-        /* If you cannot get the hostname, you cannot authenticate */
-        ret = glusterd_remote_hostname_get (req, hostname, sizeof (hostname));
+    ret = dict_get_str(dict, GD_PEER_ID_KEY, &uuid_str);
+    /* Try to match uuid only if available, don't fail as older peers will
+     * not send a uuid
+     */
+    if (!ret) {
+        gf_uuid_parse(uuid_str, peer_uuid);
+        RCU_READ_LOCK;
+        ret = (glusterd_peerinfo_find(peer_uuid, NULL) != NULL);
+        RCU_READ_UNLOCK;
         if (ret)
-                return _gf_false;
-
-        rcu_read_lock ();
-        ret = (glusterd_peerinfo_find (NULL, hostname) == NULL);
-        rcu_read_unlock ();
-
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "Rejecting management "
-                        "handshake request from unknown peer %s",
-                        req->trans->peerinfo.identifier);
-                return _gf_false;
-        }
-
-        return _gf_true;
+            return _gf_true;
+    }
+
+    /* If you cannot get the hostname, you cannot authenticate */
+    ret = glusterd_remote_hostname_get(req, hostname, sizeof(hostname));
+    if (ret)
+        return _gf_false;
+
+    /* If peer object is not found it indicates that request is from an
+     * unknown peer, if its found, validate whether its uuid is also
+     * available in the peerinfo list. There could be a case where hostname
+     * is available in the peerinfo list but the uuid has changed of the
+     * node due to a reinstall, in that case the validation should fail!
+     */
+    RCU_READ_LOCK;
+    if (!uuid_str) {
+        ret = (glusterd_peerinfo_find(NULL, hostname) == NULL);
+    } else {
+        peer = glusterd_peerinfo_find(NULL, hostname);
+        if (!peer) {
+            ret = -1;
+        } else if (peer && glusterd_peerinfo_find(peer_uuid, NULL) != NULL) {
+            ret = 0;
+        } else {
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_HANDSHAKE_REQ_REJECTED,
+                   "Request from "
+                   "peer %s has an entry in peerinfo, but uuid "
+                   "does not match",
+                   req->trans->peerinfo.identifier);
+            ret = -1;
+        }
+    }
+    RCU_READ_UNLOCK;
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_HANDSHAKE_REQ_REJECTED,
+               "Rejecting management "
+               "handshake request from unknown peer %s",
+               req->trans->peerinfo.identifier);
+        gf_event(EVENT_PEER_REJECT, "peer=%s", req->trans->peerinfo.identifier);
+        return _gf_false;
+    }
+
+    return _gf_true;
 }
 
 int
-__glusterd_mgmt_hndsk_versions (rpcsvc_request_t *req)
+__glusterd_mgmt_hndsk_versions(rpcsvc_request_t *req)
 {
-        dict_t            *dict            = NULL;
-        xlator_t          *this            = NULL;
-        glusterd_conf_t   *conf            = NULL;
-        int                ret             = -1;
-        int                op_errno        = EINVAL;
-        gf_mgmt_hndsk_req  args            = {{0,},};
-        gf_mgmt_hndsk_rsp  rsp             = {0,};
-        dict_t            *args_dict       = NULL;
-
-        this = THIS;
-        conf = this->private;
-
-        ret = xdr_to_generic (req->msg[0], &args,
-                              (xdrproc_t)xdr_gf_mgmt_hndsk_req);
-        if (ret < 0) {
-                //failed to decode msg;
-                req->rpc_err = GARBAGE_ARGS;
-                goto out;
-        }
-
-        GF_PROTOCOL_DICT_UNSERIALIZE (this, args_dict, args.hndsk.hndsk_val,
-                                      (args.hndsk.hndsk_len), ret, op_errno,
-                                      out);
-
-        /* Check if we can service the request */
-        if (!gd_validate_mgmt_hndsk_req (req, args_dict)) {
-                ret = -1;
-                goto out;
-        }
-
-        dict = dict_new ();
-        if (!dict)
-                goto out;
-
-        ret = dict_set_int32 (dict, GD_OP_VERSION_KEY, conf->op_version);
-        if (ret) {
-                gf_log (this->name, GF_LOG_WARNING,
-                        "failed to set operating version");
-                rsp.op_ret = ret;
-                goto out;
-        }
+    dict_t *dict = NULL;
+    xlator_t *this = NULL;
+    glusterd_conf_t *conf = NULL;
+    int ret = -1;
+    int op_errno = EINVAL;
+    gf_mgmt_hndsk_req args = {
+        {
+            0,
+        },
+    };
+    gf_mgmt_hndsk_rsp rsp = {
+        0,
+    };
+    dict_t *args_dict = NULL;
+
+    this = THIS;
+    conf = this->private;
+
+    ret = xdr_to_generic(req->msg[0], &args, (xdrproc_t)xdr_gf_mgmt_hndsk_req);
+    if (ret < 0) {
+        // failed to decode msg;
+        req->rpc_err = GARBAGE_ARGS;
+        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_GARBAGE_ARGS, NULL);
+        goto out;
+    }
+
+    GF_PROTOCOL_DICT_UNSERIALIZE(this, args_dict, args.hndsk.hndsk_val,
+                                 (args.hndsk.hndsk_len), ret, op_errno, out);
+
+    /* Check if we can service the request */
+    if (!gd_validate_mgmt_hndsk_req(req, args_dict)) {
+        ret = -1;
+        goto out;
+    }
+
+    dict = dict_new();
+    if (!dict) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_CREATE_FAIL, NULL);
+        goto out;
+    }
+
+    ret = dict_set_int32(dict, GD_OP_VERSION_KEY, conf->op_version);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_WARNING, 0, GD_MSG_DICT_SET_FAILED,
+               "failed to set operating version");
+        rsp.op_ret = ret;
+        goto out;
+    }
 
-        ret = dict_set_int32 (dict, GD_MIN_OP_VERSION_KEY, GD_OP_VERSION_MIN);
-        if (ret) {
-                gf_log (this->name, GF_LOG_WARNING,
-                        "failed to set %s", GD_MIN_OP_VERSION_KEY);
-                rsp.op_ret = ret;
-                goto out;
-        }
+    ret = dict_set_int32(dict, GD_MIN_OP_VERSION_KEY, GD_OP_VERSION_MIN);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_WARNING, 0, GD_MSG_DICT_SET_FAILED,
+               "failed to set %s", GD_MIN_OP_VERSION_KEY);
+        rsp.op_ret = ret;
+        goto out;
+    }
 
-        ret = dict_set_int32 (dict, GD_MAX_OP_VERSION_KEY, GD_OP_VERSION_MAX);
-        if (ret) {
-                gf_log (this->name, GF_LOG_WARNING,
-                        "failed to set %s", GD_MAX_OP_VERSION_KEY);
-                rsp.op_ret = ret;
-                goto out;
-        }
+    ret = dict_set_int32(dict, GD_MAX_OP_VERSION_KEY, GD_OP_VERSION_MAX);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_WARNING, 0, GD_MSG_DICT_SET_FAILED,
+               "failed to set %s", GD_MAX_OP_VERSION_KEY);
+        rsp.op_ret = ret;
+        goto out;
+    }
 
-        ret = 0;
+    ret = 0;
 
-        GF_PROTOCOL_DICT_SERIALIZE (this, dict, (&rsp.hndsk.hndsk_val),
-                                    rsp.hndsk.hndsk_len, op_errno, out);
+    GF_PROTOCOL_DICT_SERIALIZE(this, dict, (&rsp.hndsk.hndsk_val),
+                               rsp.hndsk.hndsk_len, op_errno, out);
 out:
 
-        rsp.op_ret = ret;
-        rsp.op_errno = op_errno;
+    rsp.op_ret = ret;
+    rsp.op_errno = op_errno;
 
-        glusterd_submit_reply (req, &rsp, NULL, 0, NULL,
-                               (xdrproc_t)xdr_gf_mgmt_hndsk_rsp);
+    glusterd_submit_reply(req, &rsp, NULL, 0, NULL,
+                          (xdrproc_t)xdr_gf_mgmt_hndsk_rsp);
 
-        ret = 0;
+    ret = 0;
+
+    if (dict)
+        dict_unref(dict);
 
-        if (dict)
-                dict_unref (dict);
+    if (args.hndsk.hndsk_val)
+        free(args.hndsk.hndsk_val);
 
-        if (args.hndsk.hndsk_val)
-                free (args.hndsk.hndsk_val);
+    if (rsp.hndsk.hndsk_val)
+        GF_FREE(rsp.hndsk.hndsk_val);
 
-        if (rsp.hndsk.hndsk_val)
-                GF_FREE (rsp.hndsk.hndsk_val);
+    if (args_dict)
+        dict_unref(args_dict);
 
-        return ret;
+    return ret;
 }
 
 int
-glusterd_mgmt_hndsk_versions (rpcsvc_request_t *req)
+glusterd_mgmt_hndsk_versions(rpcsvc_request_t *req)
 {
-        return glusterd_big_locked_handler (req,
-                                            __glusterd_mgmt_hndsk_versions);
+    return glusterd_big_locked_handler(req, __glusterd_mgmt_hndsk_versions);
 }
 
 int
-__glusterd_mgmt_hndsk_versions_ack (rpcsvc_request_t *req)
+__glusterd_mgmt_hndsk_versions_ack(rpcsvc_request_t *req)
 {
-        dict_t            *clnt_dict       = NULL;
-        xlator_t          *this            = NULL;
-        glusterd_conf_t   *conf            = NULL;
-        int                ret             = -1;
-        int                op_errno        = EINVAL;
-        int                peer_op_version = 0;
-        gf_mgmt_hndsk_req  args            = {{0,},};
-        gf_mgmt_hndsk_rsp  rsp             = {0,};
-
-        this = THIS;
-        conf = this->private;
-
-        ret = xdr_to_generic (req->msg[0], &args,
-                              (xdrproc_t)xdr_gf_mgmt_hndsk_req);
-        if (ret < 0) {
-                //failed to decode msg;
-                req->rpc_err = GARBAGE_ARGS;
-                goto out;
-        }
-
-        GF_PROTOCOL_DICT_UNSERIALIZE (this, clnt_dict, args.hndsk.hndsk_val,
-                                      (args.hndsk.hndsk_len), ret, op_errno,
-                                      out);
-
-        ret = dict_get_int32 (clnt_dict, GD_OP_VERSION_KEY, &peer_op_version);
-        if (ret) {
-                gf_log (this->name, GF_LOG_WARNING,
-                        "failed to get the op-version key peer=%s",
-                        req->trans->peerinfo.identifier);
-                goto out;
-        }
-
-        ret = gd_validate_cluster_op_version (this, peer_op_version,
-                                              req->trans->peerinfo.identifier);
-        if (ret)
-                goto out;
-
-
-        /* As this is ACK from the Cluster for the versions supported,
-           can set the op-version of 'this' glusterd to the one
-           received. */
-        gf_log (this->name, GF_LOG_INFO, "using the op-version %d",
-                peer_op_version);
-        conf->op_version = peer_op_version;
-        ret = glusterd_store_global_info (this);
-        if (ret)
-                gf_log (this->name, GF_LOG_ERROR, "Failed to store op-version");
+    dict_t *clnt_dict = NULL;
+    xlator_t *this = NULL;
+    glusterd_conf_t *conf = NULL;
+    int ret = -1;
+    int op_errno = EINVAL;
+    int peer_op_version = 0;
+    gf_mgmt_hndsk_req args = {
+        {
+            0,
+        },
+    };
+    gf_mgmt_hndsk_rsp rsp = {
+        0,
+    };
+
+    this = THIS;
+    conf = this->private;
+
+    ret = xdr_to_generic(req->msg[0], &args, (xdrproc_t)xdr_gf_mgmt_hndsk_req);
+    if (ret < 0) {
+        // failed to decode msg;
+        req->rpc_err = GARBAGE_ARGS;
+        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_GARBAGE_ARGS, NULL);
+        goto out;
+    }
+
+    GF_PROTOCOL_DICT_UNSERIALIZE(this, clnt_dict, args.hndsk.hndsk_val,
+                                 (args.hndsk.hndsk_len), ret, op_errno, out);
+
+    ret = dict_get_int32(clnt_dict, GD_OP_VERSION_KEY, &peer_op_version);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_WARNING, 0, GD_MSG_DICT_GET_FAILED,
+               "failed to get the op-version key peer=%s",
+               req->trans->peerinfo.identifier);
+        goto out;
+    }
+
+    ret = gd_validate_cluster_op_version(this, peer_op_version,
+                                         req->trans->peerinfo.identifier);
+    if (ret)
+        goto out;
+
+    /* As this is ACK from the Cluster for the versions supported,
+       can set the op-version of 'this' glusterd to the one
+       received. */
+    gf_msg(this->name, GF_LOG_INFO, 0, GD_MSG_VERS_INFO,
+           "using the op-version %d", peer_op_version);
+    conf->op_version = peer_op_version;
+    ret = glusterd_store_global_info(this);
+    if (ret)
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_GLOBAL_OP_VERSION_SET_FAIL,
+               "Failed to store op-version");
 
 out:
-        rsp.op_ret = ret;
-        rsp.op_errno = op_errno;
+    rsp.op_ret = ret;
+    rsp.op_errno = op_errno;
 
-        glusterd_submit_reply (req, &rsp, NULL, 0, NULL,
-                               (xdrproc_t)xdr_gf_mgmt_hndsk_rsp);
+    glusterd_submit_reply(req, &rsp, NULL, 0, NULL,
+                          (xdrproc_t)xdr_gf_mgmt_hndsk_rsp);
 
-        ret = 0;
+    ret = 0;
 
-        if (clnt_dict)
-                dict_unref (clnt_dict);
+    if (clnt_dict)
+        dict_unref(clnt_dict);
 
-        if (args.hndsk.hndsk_val)
-                free (args.hndsk.hndsk_val);
+    if (args.hndsk.hndsk_val)
+        free(args.hndsk.hndsk_val);
 
-        return ret;
+    return ret;
 }
 
 int
-glusterd_mgmt_hndsk_versions_ack (rpcsvc_request_t *req)
+glusterd_mgmt_hndsk_versions_ack(rpcsvc_request_t *req)
 {
-        return glusterd_big_locked_handler (req,
-                                            __glusterd_mgmt_hndsk_versions_ack);
+    return glusterd_big_locked_handler(req, __glusterd_mgmt_hndsk_versions_ack);
 }
 
 int
-__server_get_volume_info (rpcsvc_request_t *req)
+__server_get_volume_info(rpcsvc_request_t *req)
 {
-        int                     ret             = -1;
-        int32_t                 op_errno        = ENOENT;
-        gf_get_volume_info_req  vol_info_req    = {{0,}};
-        gf_get_volume_info_rsp  vol_info_rsp    = {0,};
-        char                    *volname        = NULL;
-        glusterd_volinfo_t      *volinfo        = NULL;
-        dict_t                  *dict           = NULL;
-        dict_t                  *dict_rsp       = NULL;
-        char                    *volume_id_str  = NULL;
-        int32_t                 flags           = 0;
-
-        ret = xdr_to_generic (req->msg[0], &vol_info_req,
-                             (xdrproc_t)xdr_gf_get_volume_info_req);
-        if (ret < 0) {
-                /* failed to decode msg */
-                req->rpc_err = GARBAGE_ARGS;
-                goto out;
-        }
-        gf_log ("glusterd", GF_LOG_INFO, "Received get volume info req");
-
-        if (vol_info_req.dict.dict_len) {
-                /* Unserialize the dictionary */
-                dict  = dict_new ();
-                if (!dict) {
-                        gf_log ("", GF_LOG_WARNING, "Out of Memory");
-                        op_errno = ENOMEM;
-                        ret = -1;
-                        goto out;
-                }
-
-                ret = dict_unserialize (vol_info_req.dict.dict_val,
-                                        vol_info_req.dict.dict_len,
-                                        &dict);
-                if (ret < 0) {
-                        gf_log ("glusterd", GF_LOG_ERROR,
-                                "failed to "
-                                "unserialize req-buffer to dictionary");
-                        op_errno = -ret;
-                        ret = -1;
-                        goto out;
-                } else {
-                        dict->extra_stdfree = vol_info_req.dict.dict_val;
-                }
+    int ret = -1;
+    int32_t op_errno = ENOENT;
+    gf_get_volume_info_req vol_info_req = {{
+        0,
+    }};
+    gf_get_volume_info_rsp vol_info_rsp = {
+        0,
+    };
+    char *volname = NULL;
+    glusterd_volinfo_t *volinfo = NULL;
+    dict_t *dict = NULL;
+    dict_t *dict_rsp = NULL;
+    char *volume_id_str = NULL;
+    int32_t flags = 0;
+
+    xlator_t *this = THIS;
+    GF_ASSERT(this);
+
+    ret = xdr_to_generic(req->msg[0], &vol_info_req,
+                         (xdrproc_t)xdr_gf_get_volume_info_req);
+    if (ret < 0) {
+        /* failed to decode msg */
+        req->rpc_err = GARBAGE_ARGS;
+        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_GARBAGE_ARGS, NULL);
+        goto out;
+    }
+    gf_smsg(this->name, GF_LOG_INFO, 0, GD_MSG_VOL_INFO_REQ_RECVD, NULL);
+
+    if (vol_info_req.dict.dict_len) {
+        /* Unserialize the dictionary */
+        dict = dict_new();
+        if (!dict) {
+            gf_smsg(this->name, GF_LOG_WARNING, ENOMEM, GD_MSG_DICT_CREATE_FAIL,
+                    NULL);
+            op_errno = ENOMEM;
+            ret = -1;
+            goto out;
         }
 
-        ret = dict_get_int32 (dict, "flags", &flags);
-        if (ret) {
-                gf_log (THIS->name, GF_LOG_ERROR, "failed to get flags");
-                op_errno = -ret;
-                ret = -1;
-                goto out;
+        ret = dict_unserialize(vol_info_req.dict.dict_val,
+                               vol_info_req.dict.dict_len, &dict);
+        if (ret < 0) {
+            gf_smsg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_UNSERIALIZE_FAIL,
+                    NULL);
+            op_errno = -ret;
+            ret = -1;
+            goto out;
+        } else {
+            dict->extra_stdfree = vol_info_req.dict.dict_val;
         }
+    }
 
-        if (!flags) {
-                /* Nothing to query about. Just return success */
-                gf_log (THIS->name, GF_LOG_ERROR, "No flags set");
-                ret = 0;
-                goto out;
-        }
+    ret = dict_get_int32(dict, "flags", &flags);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_GET_FAILED,
+                "Key=flags", NULL);
+        op_errno = -ret;
+        ret = -1;
+        goto out;
+    }
 
-        ret = dict_get_str (dict, "volname", &volname);
-        if (ret) {
-                op_errno = EINVAL;
-                ret = -1;
-                goto out;
-        }
+    if (!flags) {
+        /* Nothing to query about. Just return success */
+        gf_smsg(this->name, GF_LOG_ERROR, 0, GD_MSG_NO_FLAG_SET, NULL);
+        ret = 0;
+        goto out;
+    }
+
+    ret = dict_get_str(dict, "volname", &volname);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_GET_FAILED,
+                "Key=volname", NULL);
+        op_errno = EINVAL;
+        ret = -1;
+        goto out;
+    }
+
+    ret = glusterd_volinfo_find(volname, &volinfo);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_VOLINFO_GET_FAIL,
+                "Volname=%s", volname, NULL);
+        op_errno = EINVAL;
+        ret = -1;
+        goto out;
+    }
 
-        ret = glusterd_volinfo_find (volname, &volinfo);
-        if (ret) {
-                op_errno = EINVAL;
-                ret = -1;
-                goto out;
+    if (flags & (int32_t)GF_GET_VOLUME_UUID) {
+        volume_id_str = gf_strdup(uuid_utoa(volinfo->volume_id));
+        if (!volume_id_str) {
+            gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_STRDUP_FAILED,
+                    NULL);
+            op_errno = ENOMEM;
+            ret = -1;
+            goto out;
         }
 
-        if (flags | (int32_t)GF_GET_VOLUME_UUID) {
-                volume_id_str = gf_strdup (uuid_utoa (volinfo->volume_id));
-                if (!volume_id_str) {
-                        op_errno = ENOMEM;
-                        ret = -1;
-                        goto out;
-                }
-
-                dict_rsp = dict_new ();
-                if (!dict_rsp) {
-                        gf_log ("", GF_LOG_WARNING, "Out of Memory");
-                        op_errno = ENOMEM;
-                        ret = -1;
-                        goto out;
-                }
-                ret = dict_set_dynstr (dict_rsp, "volume_id", volume_id_str);
-                if (ret) {
-                        op_errno = -ret;
-                        ret = -1;
-                        goto out;
-                }
-        }
-        ret = dict_allocate_and_serialize (dict_rsp, &vol_info_rsp.dict.dict_val,
-                                           &vol_info_rsp.dict.dict_len);
+        dict_rsp = dict_new();
+        if (!dict_rsp) {
+            gf_smsg(this->name, GF_LOG_WARNING, ENOMEM, GD_MSG_DICT_CREATE_FAIL,
+                    NULL);
+            op_errno = ENOMEM;
+            GF_FREE(volume_id_str);
+            ret = -1;
+            goto out;
+        }
+        ret = dict_set_dynstr(dict_rsp, "volume_id", volume_id_str);
         if (ret) {
-                op_errno = -ret;
-                ret = -1;
-                goto out;
-        }
+            gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_SET_FAILED,
+                    "Key=volume_id", NULL);
+            op_errno = -ret;
+            ret = -1;
+            goto out;
+        }
+    }
+    ret = dict_allocate_and_serialize(dict_rsp, &vol_info_rsp.dict.dict_val,
+                                      &vol_info_rsp.dict.dict_len);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno,
+                GD_MSG_DICT_ALLOC_AND_SERL_LENGTH_GET_FAIL, NULL);
+        op_errno = -ret;
+        ret = -1;
+        goto out;
+    }
 
 out:
-        vol_info_rsp.op_ret = ret;
-        vol_info_rsp.op_errno = op_errno;
-        vol_info_rsp.op_errstr = "";
-        glusterd_submit_reply (req, &vol_info_rsp, NULL, 0, NULL,
-                               (xdrproc_t)xdr_gf_get_volume_info_rsp);
-        ret = 0;
-
-        if (dict) {
-                dict_unref (dict);
-        }
-
-        if (dict_rsp) {
-                dict_unref (dict_rsp);
-        }
-
-        if (vol_info_rsp.dict.dict_val) {
-                GF_FREE (vol_info_rsp.dict.dict_val);
-        }
-        return ret;
+    vol_info_rsp.op_ret = ret;
+    vol_info_rsp.op_errno = op_errno;
+    vol_info_rsp.op_errstr = "";
+    glusterd_submit_reply(req, &vol_info_rsp, NULL, 0, NULL,
+                          (xdrproc_t)xdr_gf_get_volume_info_rsp);
+    ret = 0;
+
+    if (dict) {
+        dict_unref(dict);
+    }
+
+    if (dict_rsp) {
+        dict_unref(dict_rsp);
+    }
+
+    if (vol_info_rsp.dict.dict_val) {
+        GF_FREE(vol_info_rsp.dict.dict_val);
+    }
+    return ret;
 }
 
 int
-server_get_volume_info (rpcsvc_request_t *req)
+server_get_volume_info(rpcsvc_request_t *req)
 {
-        return glusterd_big_locked_handler (req,
-                                            __server_get_volume_info);
+    return glusterd_big_locked_handler(req, __server_get_volume_info);
 }
 
-
 /*
  * glusterd function to get the list of snapshot names and uuids
  */
 int
-__server_get_snap_info (rpcsvc_request_t *req)
+__server_get_snap_info(rpcsvc_request_t *req)
 {
-        int                             ret             = -1;
-        int                             op_errno        = ENOENT;
-        gf_getsnap_name_uuid_req        snap_info_req   = {{0,}};
-        gf_getsnap_name_uuid_rsp        snap_info_rsp   = {0,};
-        dict_t                          *dict           = NULL;
-        dict_t                          *dict_rsp       = NULL;
-        glusterd_volinfo_t              *volinfo        = NULL;
-        char                            *volname        = NULL;
-
-        GF_ASSERT (req);
-
-        ret = xdr_to_generic (req->msg[0], &snap_info_req,
-                              (xdrproc_t)xdr_gf_getsnap_name_uuid_req);
-        if (ret < 0) {
-                req->rpc_err = GARBAGE_ARGS;
-                gf_log ("glusterd", GF_LOG_ERROR,
-                        "Failed to decode management handshake response");
-                goto out;
-        }
-
-        if (snap_info_req.dict.dict_len) {
-                dict = dict_new ();
-                if (!dict) {
-                        op_errno = ENOMEM;
-                        ret = -1;
-                        goto out;
-                }
-
-                ret = dict_unserialize (snap_info_req.dict.dict_val,
-                                        snap_info_req.dict.dict_len,
-                                        &dict);
-                if (ret < 0) {
-                        gf_log ("glusterd", GF_LOG_ERROR,
-                                "Failed to unserialize dictionary");
-                        op_errno = EINVAL;
-                        ret = -1;
-                        goto out;
-                } else {
-                        dict->extra_stdfree = snap_info_req.dict.dict_val;
-                }
+    int ret = -1;
+    int op_errno = ENOENT;
+    gf_getsnap_name_uuid_req snap_info_req = {{
+        0,
+    }};
+    gf_getsnap_name_uuid_rsp snap_info_rsp = {
+        0,
+    };
+    dict_t *dict = NULL;
+    dict_t *dict_rsp = NULL;
+    char *volname = NULL;
+
+    GF_ASSERT(req);
+
+    ret = xdr_to_generic(req->msg[0], &snap_info_req,
+                         (xdrproc_t)xdr_gf_getsnap_name_uuid_req);
+    if (ret < 0) {
+        req->rpc_err = GARBAGE_ARGS;
+        gf_msg("glusterd", GF_LOG_ERROR, 0, GD_MSG_REQ_DECODE_FAIL,
+               "Failed to decode management handshake response");
+        goto out;
+    }
+
+    if (snap_info_req.dict.dict_len) {
+        dict = dict_new();
+        if (!dict) {
+            gf_smsg("glusterd", GF_LOG_WARNING, ENOMEM, GD_MSG_DICT_CREATE_FAIL,
+                    NULL);
+            op_errno = ENOMEM;
+            ret = -1;
+            goto out;
         }
 
-        ret = dict_get_str (dict, "volname", &volname);
-        if (ret) {
-                op_errno = EINVAL;
-                gf_log ("glusterd", GF_LOG_ERROR,
-                        "Failed to retrieve volname");
-                ret = -1;
-                goto out;
+        ret = dict_unserialize(snap_info_req.dict.dict_val,
+                               snap_info_req.dict.dict_len, &dict);
+        if (ret < 0) {
+            gf_msg("glusterd", GF_LOG_ERROR, EINVAL,
+                   GD_MSG_DICT_UNSERIALIZE_FAIL,
+                   "Failed to unserialize dictionary");
+            op_errno = EINVAL;
+            ret = -1;
+            goto out;
+        } else {
+            dict->extra_stdfree = snap_info_req.dict.dict_val;
         }
+    }
 
-        dict_rsp = dict_new ();
-        if (!dict_rsp) {
-                op_errno = ENOMEM;
-                ret = -1;
-                goto out;
-        }
+    ret = dict_get_str(dict, "volname", &volname);
+    if (ret) {
+        op_errno = EINVAL;
+        gf_msg("glusterd", GF_LOG_ERROR, EINVAL, GD_MSG_DICT_GET_FAILED,
+               "Failed to retrieve volname");
+        ret = -1;
+        goto out;
+    }
+
+    dict_rsp = dict_new();
+    if (!dict_rsp) {
+        gf_smsg("glusterd", GF_LOG_WARNING, ENOMEM, GD_MSG_DICT_CREATE_FAIL,
+                NULL);
+        op_errno = ENOMEM;
+        ret = -1;
+        goto out;
+    }
 
-        ret = glusterd_snapshot_get_volnames_uuids (dict_rsp, volname,
-                                                    &snap_info_rsp);
+    ret = glusterd_snapshot_get_volnames_uuids(dict_rsp, volname,
+                                               &snap_info_rsp);
 
-        if (ret) {
-                gf_log ("glusterd", GF_LOG_ERROR,
-                        "Error getting snapshot volume names and uuids : %s",
-                        volname);
-                op_errno = EINVAL;
-        }
+    if (ret) {
+        gf_msg("glusterd", GF_LOG_ERROR, EINVAL, GD_MSG_VOL_NOT_FOUND,
+               "Error getting snapshot volume names and uuids : %s", volname);
+        op_errno = EINVAL;
+    }
 
 out:
-        snap_info_rsp.op_ret = ret;
-        snap_info_rsp.op_errno = op_errno;
-        snap_info_rsp.op_errstr = "";
-        glusterd_submit_reply (req, &snap_info_rsp, NULL, 0, NULL,
-                               (xdrproc_t)xdr_gf_getsnap_name_uuid_rsp);
-
-        if (dict) {
-                dict_unref (dict);
-        }
+    snap_info_rsp.op_ret = ret;
+    snap_info_rsp.op_errno = op_errno;
+    snap_info_rsp.op_errstr = "";
+    glusterd_submit_reply(req, &snap_info_rsp, NULL, 0, NULL,
+                          (xdrproc_t)xdr_gf_getsnap_name_uuid_rsp);
 
-        if (dict_rsp) {
-                dict_unref (dict_rsp);
-        }
+    if (dict) {
+        dict_unref(dict);
+    }
 
-        if (snap_info_rsp.dict.dict_val) {
-                GF_FREE (snap_info_rsp.dict.dict_val);
-        }
+    if (dict_rsp) {
+        dict_unref(dict_rsp);
+    }
 
-        return 0;
+    if (snap_info_rsp.dict.dict_val) {
+        GF_FREE(snap_info_rsp.dict.dict_val);
+    }
+
+    return 0;
 }
 
 int
-server_get_snap_info (rpcsvc_request_t *req)
+server_get_snap_info(rpcsvc_request_t *req)
 {
-        return glusterd_big_locked_handler (req,
-                                            __server_get_snap_info);
+    return glusterd_big_locked_handler(req, __server_get_snap_info);
 }
 
-rpcsvc_actor_t gluster_handshake_actors[GF_HNDSK_MAXVALUE] = {
-        [GF_HNDSK_NULL]         = {"NULL",        GF_HNDSK_NULL,         NULL,                NULL, 0, DRC_NA},
-        [GF_HNDSK_GETSPEC]      = {"GETSPEC",     GF_HNDSK_GETSPEC,      server_getspec,      NULL, 0, DRC_NA},
-        [GF_HNDSK_EVENT_NOTIFY] = {"EVENTNOTIFY", GF_HNDSK_EVENT_NOTIFY, server_event_notify, NULL, 0, DRC_NA},
-        [GF_HNDSK_GET_VOLUME_INFO] = {"GETVOLUMEINFO", GF_HNDSK_GET_VOLUME_INFO, server_get_volume_info, NULL, 0, DRC_NA},
-        [GF_HNDSK_GET_SNAPSHOT_INFO] = {"GETSNAPINFO", GF_HNDSK_GET_SNAPSHOT_INFO, server_get_snap_info, NULL, 0, DRC_NA},
+static rpcsvc_actor_t gluster_handshake_actors[GF_HNDSK_MAXVALUE] = {
+    [GF_HNDSK_NULL] = {"NULL", NULL, NULL, GF_HNDSK_NULL, DRC_NA, 0},
+    [GF_HNDSK_GETSPEC] = {"GETSPEC", server_getspec, NULL, GF_HNDSK_GETSPEC,
+                          DRC_NA, 0},
+    [GF_HNDSK_EVENT_NOTIFY] = {"EVENTNOTIFY", server_event_notify, NULL,
+                               GF_HNDSK_EVENT_NOTIFY, DRC_NA, 0},
+    [GF_HNDSK_GET_VOLUME_INFO] = {"GETVOLUMEINFO", server_get_volume_info, NULL,
+                                  GF_HNDSK_GET_VOLUME_INFO, DRC_NA, 0},
+    [GF_HNDSK_GET_SNAPSHOT_INFO] = {"GETSNAPINFO", server_get_snap_info, NULL,
+                                    GF_HNDSK_GET_SNAPSHOT_INFO, DRC_NA, 0},
 };
 
-
 struct rpcsvc_program gluster_handshake_prog = {
-        .progname  = "Gluster Handshake",
-        .prognum   = GLUSTER_HNDSK_PROGRAM,
-        .progver   = GLUSTER_HNDSK_VERSION,
-        .actors    = gluster_handshake_actors,
-        .numactors = GF_HNDSK_MAXVALUE,
+    .progname = "Gluster Handshake",
+    .prognum = GLUSTER_HNDSK_PROGRAM,
+    .progver = GLUSTER_HNDSK_VERSION,
+    .actors = gluster_handshake_actors,
+    .numactors = GF_HNDSK_MAXVALUE,
 };
 
 /* A minimal RPC program just for the cli getspec command */
-rpcsvc_actor_t gluster_cli_getspec_actors[GF_HNDSK_MAXVALUE] = {
-        [GF_HNDSK_GETSPEC]      = {"GETSPEC",     GF_HNDSK_GETSPEC,      server_getspec,      NULL, 0, DRC_NA},
+static rpcsvc_actor_t gluster_cli_getspec_actors[GF_HNDSK_MAXVALUE] = {
+    [GF_HNDSK_GETSPEC] = {"GETSPEC", server_getspec, NULL, GF_HNDSK_GETSPEC,
+                          DRC_NA, 0},
 };
 
 struct rpcsvc_program gluster_cli_getspec_prog = {
-        .progname  = "Gluster Handshake (CLI Getspec)",
-        .prognum   = GLUSTER_HNDSK_PROGRAM,
-        .progver   = GLUSTER_HNDSK_VERSION,
-        .actors    = gluster_cli_getspec_actors,
-        .numactors = GF_HNDSK_MAXVALUE,
+    .progname = "Gluster Handshake (CLI Getspec)",
+    .prognum = GLUSTER_HNDSK_PROGRAM,
+    .progver = GLUSTER_HNDSK_VERSION,
+    .actors = gluster_cli_getspec_actors,
+    .numactors = GF_HNDSK_MAXVALUE,
 };
 
-
-char *glusterd_dump_proc[GF_DUMP_MAXVALUE] = {
-        [GF_DUMP_NULL] = "NULL",
-        [GF_DUMP_DUMP] = "DUMP",
-        [GF_DUMP_PING] = "PING",
+static char *glusterd_dump_proc[GF_DUMP_MAXVALUE] = {
+    [GF_DUMP_NULL] = "NULL",
+    [GF_DUMP_DUMP] = "DUMP",
+    [GF_DUMP_PING] = "PING",
 };
 
-rpc_clnt_prog_t glusterd_dump_prog = {
-        .progname       = "GLUSTERD-DUMP",
-        .prognum        = GLUSTER_DUMP_PROGRAM,
-        .progver        = GLUSTER_DUMP_VERSION,
-        .procnames      = glusterd_dump_proc,
+static rpc_clnt_prog_t glusterd_dump_prog = {
+    .progname = "GLUSTERD-DUMP",
+    .prognum = GLUSTER_DUMP_PROGRAM,
+    .progver = GLUSTER_DUMP_VERSION,
+    .procnames = glusterd_dump_proc,
 };
 
-
-rpcsvc_actor_t glusterd_mgmt_hndsk_actors[GD_MGMT_HNDSK_MAXVALUE] = {
-        [GD_MGMT_HNDSK_NULL]            = {"NULL", GD_MGMT_HNDSK_NULL, NULL,
-                                           NULL, 0, DRC_NA},
-        [GD_MGMT_HNDSK_VERSIONS]        = {"MGMT-VERS", GD_MGMT_HNDSK_VERSIONS,
-                                           glusterd_mgmt_hndsk_versions, NULL,
-                                           0, DRC_NA},
-        [GD_MGMT_HNDSK_VERSIONS_ACK]    = {"MGMT-VERS-ACK",
-                                           GD_MGMT_HNDSK_VERSIONS_ACK,
-                                           glusterd_mgmt_hndsk_versions_ack,
-                                           NULL, 0, DRC_NA},
+static rpcsvc_actor_t glusterd_mgmt_hndsk_actors[GD_MGMT_HNDSK_MAXVALUE] = {
+    [GD_MGMT_HNDSK_NULL] = {"NULL", NULL, NULL, GD_MGMT_HNDSK_NULL, DRC_NA, 0},
+    [GD_MGMT_HNDSK_VERSIONS] = {"MGMT-VERS", glusterd_mgmt_hndsk_versions, NULL,
+                                GD_MGMT_HNDSK_VERSIONS, DRC_NA, 0},
+    [GD_MGMT_HNDSK_VERSIONS_ACK] = {"MGMT-VERS-ACK",
+                                    glusterd_mgmt_hndsk_versions_ack, NULL,
+                                    GD_MGMT_HNDSK_VERSIONS_ACK, DRC_NA, 0},
 };
 
 struct rpcsvc_program glusterd_mgmt_hndsk_prog = {
-        .progname       = "Gluster MGMT Handshake",
-        .prognum        = GD_MGMT_HNDSK_PROGRAM,
-        .progver        = GD_MGMT_HNDSK_VERSION,
-        .actors         = glusterd_mgmt_hndsk_actors,
-        .numactors      = GD_MGMT_HNDSK_MAXVALUE,
+    .progname = "Gluster MGMT Handshake",
+    .prognum = GD_MGMT_HNDSK_PROGRAM,
+    .progver = GD_MGMT_HNDSK_VERSION,
+    .actors = glusterd_mgmt_hndsk_actors,
+    .numactors = GD_MGMT_HNDSK_MAXVALUE,
 };
 
-char *glusterd_mgmt_hndsk_proc[GD_MGMT_HNDSK_MAXVALUE] = {
-        [GD_MGMT_HNDSK_NULL]          = "NULL",
-        [GD_MGMT_HNDSK_VERSIONS] = "MGMT-VERS",
-        [GD_MGMT_HNDSK_VERSIONS_ACK] = "MGMT-VERS-ACK",
+static char *glusterd_mgmt_hndsk_proc[GD_MGMT_HNDSK_MAXVALUE] = {
+    [GD_MGMT_HNDSK_NULL] = "NULL",
+    [GD_MGMT_HNDSK_VERSIONS] = "MGMT-VERS",
+    [GD_MGMT_HNDSK_VERSIONS_ACK] = "MGMT-VERS-ACK",
 };
 
-rpc_clnt_prog_t gd_clnt_mgmt_hndsk_prog = {
-        .progname  = "Gluster MGMT Handshake",
-        .prognum   = GD_MGMT_HNDSK_PROGRAM,
-        .progver   = GD_MGMT_HNDSK_VERSION,
-        .procnames = glusterd_mgmt_hndsk_proc,
+static rpc_clnt_prog_t gd_clnt_mgmt_hndsk_prog = {
+    .progname = "Gluster MGMT Handshake",
+    .prognum = GD_MGMT_HNDSK_PROGRAM,
+    .progver = GD_MGMT_HNDSK_VERSION,
+    .procnames = glusterd_mgmt_hndsk_proc,
 };
 
-
 static int
-glusterd_event_connected_inject (glusterd_peerctx_t *peerctx)
+glusterd_event_connected_inject(glusterd_peerctx_t *peerctx)
 {
-        GF_ASSERT (peerctx);
+    GF_ASSERT(peerctx);
 
-        glusterd_friend_sm_event_t      *event = NULL;
-        glusterd_probe_ctx_t            *ctx = NULL;
-        int                             ret = -1;
-        glusterd_peerinfo_t             *peerinfo = NULL;
+    glusterd_friend_sm_event_t *event = NULL;
+    glusterd_probe_ctx_t *ctx = NULL;
+    int ret = -1;
+    glusterd_peerinfo_t *peerinfo = NULL;
 
+    ret = glusterd_friend_sm_new_event(GD_FRIEND_EVENT_CONNECTED, &event);
 
-        ret = glusterd_friend_sm_new_event
-                        (GD_FRIEND_EVENT_CONNECTED, &event);
-
-        if (ret) {
-                gf_log ("", GF_LOG_ERROR, "Unable to get new event");
-                goto out;
-        }
+    if (ret) {
+        gf_msg("glusterd", GF_LOG_ERROR, 0, GD_MSG_EVENT_NEW_GET_FAIL,
+               "Unable to get new event");
+        goto out;
+    }
 
-        ctx = GF_CALLOC (1, sizeof(*ctx), gf_gld_mt_probe_ctx_t);
+    ctx = GF_CALLOC(1, sizeof(*ctx), gf_gld_mt_probe_ctx_t);
 
-        if (!ctx) {
-                ret = -1;
-                gf_log ("", GF_LOG_ERROR, "Memory not available");
-                goto out;
-        }
-
-        rcu_read_lock ();
-
-        peerinfo = glusterd_peerinfo_find (peerctx->peerid, peerctx->peername);
-        if (!peerinfo) {
-                ret = -1;
-                gf_log (THIS->name, GF_LOG_ERROR, "Could not find peer %s(%s)",
-                        peerctx->peername, uuid_utoa (peerctx->peerid));
-                goto unlock;
-        }
-        ctx->hostname = gf_strdup (peerinfo->hostname);
-        ctx->port = peerinfo->port;
-        ctx->req = peerctx->args.req;
-        ctx->dict = peerctx->args.dict;
-
-        event->peername = gf_strdup (peerinfo->hostname);
-        gf_uuid_copy (event->peerid, peerinfo->uuid);
-        event->ctx = ctx;
+    if (!ctx) {
+        ret = -1;
+        gf_msg("glusterd", GF_LOG_ERROR, ENOMEM, GD_MSG_NO_MEMORY,
+               "Memory not available");
+        goto out;
+    }
 
-        ret = glusterd_friend_sm_inject_event (event);
+    RCU_READ_LOCK;
 
-        if (ret)
-                gf_log ("glusterd", GF_LOG_ERROR, "Unable to inject "
-                        "EVENT_CONNECTED ret = %d", ret);
-unlock:
-        rcu_read_unlock ();
+    peerinfo = glusterd_peerinfo_find_by_generation(peerctx->peerinfo_gen);
+    if (!peerinfo) {
+        RCU_READ_UNLOCK;
+        ret = -1;
+        gf_msg(THIS->name, GF_LOG_ERROR, 0, GD_MSG_PEER_NOT_FOUND,
+               "Could not find peer %s(%s)", peerctx->peername,
+               uuid_utoa(peerctx->peerid));
+        GF_FREE(ctx);
+        goto out;
+    }
+    ctx->hostname = gf_strdup(peerinfo->hostname);
+    ctx->port = peerinfo->port;
+    ctx->req = peerctx->args.req;
+    ctx->dict = peerctx->args.dict;
+
+    event->peername = gf_strdup(peerinfo->hostname);
+    gf_uuid_copy(event->peerid, peerinfo->uuid);
+    event->ctx = ctx;
+
+    ret = glusterd_friend_sm_inject_event(event);
+
+    RCU_READ_UNLOCK;
+
+    if (ret)
+        gf_msg("glusterd", GF_LOG_ERROR, 0, GD_MSG_EVENT_INJECT_FAIL,
+               "Unable to inject "
+               "EVENT_CONNECTED ret = %d",
+               ret);
 
 out:
-        gf_log ("", GF_LOG_DEBUG, "returning %d", ret);
-        return ret;
+    gf_msg_debug("glusterd", 0, "returning %d", ret);
+    return ret;
 }
 
-
 int
-gd_validate_peer_op_version (xlator_t *this, glusterd_peerinfo_t *peerinfo,
-                             dict_t *dict, char **errstr)
+gd_validate_peer_op_version(xlator_t *this, glusterd_peerinfo_t *peerinfo,
+                            dict_t *dict, char **errstr)
 {
-        int              ret  = -1;
-        glusterd_conf_t *conf = NULL;
-        int32_t          peer_op_version = 0;
-        int32_t          peer_min_op_version = 0;
-        int32_t          peer_max_op_version = 0;
-
-        if (!dict && !this && !peerinfo)
-                goto out;
-
-        conf = this->private;
-
-        ret = dict_get_int32 (dict, GD_OP_VERSION_KEY, &peer_op_version);
-        if (ret)
-                goto out;
-
-        ret = dict_get_int32 (dict, GD_MAX_OP_VERSION_KEY,
-                              &peer_max_op_version);
-        if (ret)
-                goto out;
-
-        ret = dict_get_int32 (dict, GD_MIN_OP_VERSION_KEY,
-                              &peer_min_op_version);
-        if (ret)
-                goto out;
-
+    int ret = -1;
+    glusterd_conf_t *conf = NULL;
+    int32_t peer_op_version = 0;
+    int32_t peer_min_op_version = 0;
+    int32_t peer_max_op_version = 0;
+
+    if (!dict) {
+        gf_smsg("glusterd", GF_LOG_WARNING, ENOMEM, GD_MSG_DICT_CREATE_FAIL,
+                NULL);
+        goto out;
+    }
+
+    if (!this) {
+        gf_smsg("glusterd", GF_LOG_ERROR, errno, GD_MSG_XLATOR_NOT_DEFINED,
+                NULL);
+        goto out;
+    }
+
+    if (!peerinfo) {
+        gf_smsg("glusterd", GF_LOG_ERROR, errno, GD_MSG_INVALID_ARGUMENT, NULL);
+        goto out;
+    }
+
+    conf = this->private;
+
+    ret = dict_get_int32(dict, GD_OP_VERSION_KEY, &peer_op_version);
+    if (ret) {
+        gf_smsg("glusterd", GF_LOG_ERROR, errno, GD_MSG_DICT_GET_FAILED,
+                "Key=%s", GD_OP_VERSION_KEY, NULL);
+        goto out;
+    }
+
+    ret = dict_get_int32(dict, GD_MAX_OP_VERSION_KEY, &peer_max_op_version);
+    if (ret) {
+        gf_smsg("glusterd", GF_LOG_ERROR, errno, GD_MSG_DICT_GET_FAILED,
+                "Key=%s", GD_MAX_OP_VERSION_KEY, NULL);
+        goto out;
+    }
+
+    ret = dict_get_int32(dict, GD_MIN_OP_VERSION_KEY, &peer_min_op_version);
+    if (ret) {
+        gf_smsg("glusterd", GF_LOG_ERROR, errno, GD_MSG_DICT_GET_FAILED,
+                "Key=%s", GD_MIN_OP_VERSION_KEY, NULL);
+        goto out;
+    }
+
+    ret = -1;
+    /* Check if peer can support our op_version */
+    if ((peer_max_op_version < conf->op_version) ||
+        (peer_min_op_version > conf->op_version)) {
+        ret = gf_asprintf(errstr,
+                          "Peer %s does not support required "
+                          "op-version",
+                          peerinfo->hostname);
         ret = -1;
-        /* Check if peer can support our op_version */
-        if ((peer_max_op_version < conf->op_version) ||
-            (peer_min_op_version > conf->op_version)) {
-                ret = gf_asprintf (errstr, "Peer %s does not support required "
-                                   "op-version", peerinfo->hostname);
-                ret = -1;
-                goto out;
-        }
+        goto out;
+    }
 
-        ret = 0;
+    ret = 0;
 out:
-        gf_log (this->name , GF_LOG_DEBUG, "Peer %s %s", peerinfo->hostname,
-                ((ret < 0) ? "rejected" : "accepted"));
-        return ret;
+    if (peerinfo)
+        gf_msg_debug((this ? this->name : "glusterd"), 0, "Peer %s %s",
+                     peerinfo->hostname, ((ret < 0) ? "rejected" : "accepted"));
+    return ret;
 }
 
 int
-__glusterd_mgmt_hndsk_version_ack_cbk (struct rpc_req *req, struct iovec *iov,
-                                     int count, void *myframe)
+__glusterd_mgmt_hndsk_version_ack_cbk(struct rpc_req *req, struct iovec *iov,
+                                      int count, void *myframe)
 {
-        int                  ret      = -1;
-        int                  op_errno = EINVAL;
-        gf_mgmt_hndsk_rsp    rsp      = {0,};
-        xlator_t            *this     = NULL;
-        call_frame_t        *frame    = NULL;
-        glusterd_peerinfo_t *peerinfo = NULL;
-        glusterd_peerctx_t  *peerctx  = NULL;
-        char                msg[1024] = {0,};
-
-        this = THIS;
-        frame = myframe;
-        peerctx = frame->local;
-
-        rcu_read_lock ();
-        peerinfo = glusterd_peerinfo_find (peerctx->peerid, peerctx->peername);
-        if (!peerinfo) {
-                gf_log (this->name, GF_LOG_DEBUG, "Could not find peer %s(%s)",
-                        peerctx->peername, uuid_utoa (peerctx->peerid));
-                ret = -1;
-                goto out;
-        }
-
-        if (-1 == req->rpc_status) {
-                snprintf (msg, sizeof (msg),
-                          "Error through RPC layer, retry again later");
-                gf_log ("", GF_LOG_ERROR, "%s", msg);
-                peerctx->errstr = gf_strdup (msg);
-                goto out;
-        }
-
-        ret = xdr_to_generic (*iov, &rsp, (xdrproc_t)xdr_gf_mgmt_hndsk_rsp);
-        if (ret < 0) {
-                snprintf (msg, sizeof (msg), "Failed to decode XDR");
-                gf_log ("", GF_LOG_ERROR, "%s", msg);
-                peerctx->errstr = gf_strdup (msg);
-                goto out;
-        }
-
-        op_errno = rsp.op_errno;
-        if (-1 == rsp.op_ret) {
-                ret = -1;
-                snprintf (msg, sizeof (msg),
-                          "Failed to get handshake ack from remote server");
-                gf_log (frame->this->name, GF_LOG_ERROR, "%s", msg);
-                peerctx->errstr = gf_strdup (msg);
-                goto out;
-        }
-
-        /* TODO: this is hardcoded as of now, but I don't forsee any problems
-         * with this as long as we are properly handshaking operating versions
-         */
-        peerinfo->mgmt = &gd_mgmt_prog;
-        peerinfo->peer = &gd_peer_prog;
-        peerinfo->mgmt_v3 = &gd_mgmt_v3_prog;
-
-        ret = default_notify (this, GF_EVENT_CHILD_UP, NULL);
-
-        if (GD_MODE_ON == peerctx->args.mode) {
-                ret = glusterd_event_connected_inject (peerctx);
-                peerctx->args.req = NULL;
-        } else if (GD_MODE_SWITCH_ON == peerctx->args.mode) {
-                peerctx->args.mode = GD_MODE_ON;
-        } else {
-                gf_log (this->name, GF_LOG_WARNING, "unknown mode %d",
-                        peerctx->args.mode);
-        }
-
-        ret = 0;
+    int ret = -1;
+    gf_mgmt_hndsk_rsp rsp = {
+        0,
+    };
+    xlator_t *this = NULL;
+    call_frame_t *frame = NULL;
+    glusterd_peerinfo_t *peerinfo = NULL;
+    glusterd_peerctx_t *peerctx = NULL;
+    char msg[64] = {
+        0,
+    };
+
+    this = THIS;
+    frame = myframe;
+    peerctx = frame->local;
+
+    RCU_READ_LOCK;
+    peerinfo = glusterd_peerinfo_find_by_generation(peerctx->peerinfo_gen);
+    if (!peerinfo) {
+        gf_msg_debug(this->name, 0, "Could not find peer %s(%s)",
+                     peerctx->peername, uuid_utoa(peerctx->peerid));
+        ret = -1;
+        goto out;
+    }
+
+    if (-1 == req->rpc_status) {
+        snprintf(msg, sizeof(msg),
+                 "Error through RPC layer, retry again later");
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_RPC_LAYER_ERROR, "%s", msg);
+        peerctx->errstr = gf_strdup(msg);
+        goto out;
+    }
+
+    ret = xdr_to_generic(*iov, &rsp, (xdrproc_t)xdr_gf_mgmt_hndsk_rsp);
+    if (ret < 0) {
+        snprintf(msg, sizeof(msg), "Failed to decode XDR");
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_REQ_DECODE_FAIL, "%s", msg);
+        peerctx->errstr = gf_strdup(msg);
+        goto out;
+    }
+
+    if (-1 == rsp.op_ret) {
+        ret = -1;
+        snprintf(msg, sizeof(msg),
+                 "Failed to get handshake ack from remote server");
+        gf_msg(frame->this->name, GF_LOG_ERROR, 0, GD_MSG_NO_HANDSHAKE_ACK,
+               "%s", msg);
+        peerctx->errstr = gf_strdup(msg);
+        goto out;
+    }
+
+    /* TODO: this is hardcoded as of now, but I don't forsee any problems
+     * with this as long as we are properly handshaking operating versions
+     */
+    peerinfo->mgmt = &gd_mgmt_prog;
+    peerinfo->peer = &gd_peer_prog;
+    peerinfo->mgmt_v3 = &gd_mgmt_v3_prog;
+
+    ret = default_notify(this, GF_EVENT_CHILD_UP, NULL);
+
+    if (GD_MODE_ON == peerctx->args.mode) {
+        (void)glusterd_event_connected_inject(peerctx);
+        peerctx->args.req = NULL;
+    } else if (GD_MODE_SWITCH_ON == peerctx->args.mode) {
+        peerctx->args.mode = GD_MODE_ON;
+    } else {
+        gf_msg(this->name, GF_LOG_WARNING, 0, GD_MSG_UNKNOWN_MODE,
+               "unknown mode %d", peerctx->args.mode);
+    }
+
+    ret = 0;
 out:
 
-        if (ret != 0 && peerinfo)
-                rpc_transport_disconnect (peerinfo->rpc->conn.trans);
+    if (ret != 0 && peerinfo)
+        rpc_transport_disconnect(peerinfo->rpc->conn.trans, _gf_false);
 
-        rcu_read_unlock ();
+    RCU_READ_UNLOCK;
 
-        frame->local = NULL;
-        STACK_DESTROY (frame->root);
+    frame->local = NULL;
+    STACK_DESTROY(frame->root);
 
-        if (rsp.hndsk.hndsk_val)
-                free (rsp.hndsk.hndsk_val);
+    if (rsp.hndsk.hndsk_val)
+        free(rsp.hndsk.hndsk_val);
 
-        glusterd_friend_sm ();
+    glusterd_friend_sm();
 
-        return 0;
+    return 0;
 }
 
 int
-glusterd_mgmt_hndsk_version_ack_cbk (struct rpc_req *req, struct iovec *iov,
-                                     int count, void *myframe)
+glusterd_mgmt_hndsk_version_ack_cbk(struct rpc_req *req, struct iovec *iov,
+                                    int count, void *myframe)
 {
-        return glusterd_big_locked_cbk (req, iov, count, myframe,
-                                        __glusterd_mgmt_hndsk_version_ack_cbk);
+    return glusterd_big_locked_cbk(req, iov, count, myframe,
+                                   __glusterd_mgmt_hndsk_version_ack_cbk);
 }
 
 int
-__glusterd_mgmt_hndsk_version_cbk (struct rpc_req *req, struct iovec *iov,
-                                 int count, void *myframe)
+__glusterd_mgmt_hndsk_version_cbk(struct rpc_req *req, struct iovec *iov,
+                                  int count, void *myframe)
 {
-        int                  ret       = -1;
-        int                  op_errno  = EINVAL;
-        gf_mgmt_hndsk_rsp    rsp       = {0,};
-        gf_mgmt_hndsk_req    arg       = {{0,}};
-        xlator_t            *this      = NULL;
-        call_frame_t        *frame     = NULL;
-        glusterd_peerinfo_t *peerinfo  = NULL;
-        glusterd_peerctx_t  *peerctx   = NULL;
-        dict_t              *dict      = NULL;
-        dict_t              *rsp_dict  = NULL;
-        glusterd_conf_t     *conf      = NULL;
-        char                 msg[1024] = {0,};
-
-        this = THIS;
-        conf = this->private;
-        frame = myframe;
-        peerctx = frame->local;
-
-        rcu_read_lock ();
-
-        peerinfo = glusterd_peerinfo_find (peerctx->peerid, peerctx->peername);
-        if (!peerinfo) {
-                ret = -1;
-                gf_log (this->name, GF_LOG_DEBUG, "Could not find peer %s(%s)",
-                        peerctx->peername, uuid_utoa (peerctx->peerid));
-                goto out;
-        }
-
-        if (-1 == req->rpc_status) {
-                ret = -1;
-                snprintf (msg, sizeof (msg),
-                          "Error through RPC layer, retry again later");
-                gf_log (this->name, GF_LOG_ERROR, "%s", msg);
-                peerctx->errstr = gf_strdup (msg);
-                goto out;
-        }
-
-        ret = xdr_to_generic (*iov, &rsp, (xdrproc_t)xdr_gf_mgmt_hndsk_rsp);
-        if (ret < 0) {
-                snprintf (msg, sizeof (msg), "Failed to decode management "
-                          "handshake response");
-                gf_log (this->name, GF_LOG_ERROR, "%s", msg);
-                peerctx->errstr = gf_strdup (msg);
-                goto out;
-        }
-
-        GF_PROTOCOL_DICT_UNSERIALIZE (this, dict, rsp.hndsk.hndsk_val,
-                                      rsp.hndsk.hndsk_len, ret, op_errno,
-                                      out);
-
-        op_errno = rsp.op_errno;
-        if (-1 == rsp.op_ret) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "failed to get the 'versions' from peer (%s)",
-                        req->conn->trans->peerinfo.identifier);
-                goto out;
-        }
-
-        /* Check if peer can be part of cluster */
-        ret = gd_validate_peer_op_version (this, peerinfo, dict,
-                                           &peerctx->errstr);
-        if (ret < 0) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "failed to validate the operating version of peer (%s)",
-                        peerinfo->hostname);
-                goto out;
-        }
-
-        rsp_dict = dict_new ();
-        if (!rsp_dict)
-                goto out;
-
-        ret = dict_set_int32 (rsp_dict, GD_OP_VERSION_KEY, conf->op_version);
-        if (ret) {
-                gf_log(this->name, GF_LOG_ERROR,
-                       "failed to set operating version in dict");
-                goto out;
-        }
-
-        GF_PROTOCOL_DICT_SERIALIZE (this, rsp_dict, (&arg.hndsk.hndsk_val),
-                                    arg.hndsk.hndsk_len, op_errno, out);
+    int ret = -1;
+    int op_errno = EINVAL;
+    gf_mgmt_hndsk_rsp rsp = {
+        0,
+    };
+    gf_mgmt_hndsk_req arg = {{
+        0,
+    }};
+    xlator_t *this = NULL;
+    call_frame_t *frame = NULL;
+    glusterd_peerinfo_t *peerinfo = NULL;
+    glusterd_peerctx_t *peerctx = NULL;
+    dict_t *dict = NULL;
+    dict_t *rsp_dict = NULL;
+    glusterd_conf_t *conf = NULL;
+    char msg[64] = {
+        0,
+    };
+
+    this = THIS;
+    conf = this->private;
+    frame = myframe;
+    peerctx = frame->local;
+
+    RCU_READ_LOCK;
+
+    peerinfo = glusterd_peerinfo_find_by_generation(peerctx->peerinfo_gen);
+    if (!peerinfo) {
+        ret = -1;
+        gf_msg_debug(this->name, 0, "Could not find peer %s(%s)",
+                     peerctx->peername, uuid_utoa(peerctx->peerid));
+        goto out;
+    }
 
-        ret = glusterd_submit_request (peerinfo->rpc, &arg, frame,
-                                       &gd_clnt_mgmt_hndsk_prog,
-                                       GD_MGMT_HNDSK_VERSIONS_ACK, NULL, this,
-                                       glusterd_mgmt_hndsk_version_ack_cbk,
-                                       (xdrproc_t)xdr_gf_mgmt_hndsk_req);
+    if (-1 == req->rpc_status) {
+        ret = -1;
+        snprintf(msg, sizeof(msg),
+                 "Error through RPC layer, retry again later");
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_RPC_LAYER_ERROR, "%s", msg);
+        peerctx->errstr = gf_strdup(msg);
+        goto out;
+    }
+
+    ret = xdr_to_generic(*iov, &rsp, (xdrproc_t)xdr_gf_mgmt_hndsk_rsp);
+    if (ret < 0) {
+        snprintf(msg, sizeof(msg),
+                 "Failed to decode management "
+                 "handshake response");
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_REQ_DECODE_FAIL, "%s", msg);
+        peerctx->errstr = gf_strdup(msg);
+        goto out;
+    }
+
+    GF_PROTOCOL_DICT_UNSERIALIZE(this, dict, rsp.hndsk.hndsk_val,
+                                 rsp.hndsk.hndsk_len, ret, op_errno, out);
+
+    op_errno = rsp.op_errno;
+    if (-1 == rsp.op_ret) {
+        gf_msg(this->name, GF_LOG_ERROR, op_errno, GD_MSG_VERS_GET_FAIL,
+               "failed to get the 'versions' from peer (%s)",
+               req->conn->trans->peerinfo.identifier);
+        goto out;
+    }
+
+    /* Check if peer can be part of cluster */
+    ret = gd_validate_peer_op_version(this, peerinfo, dict, &peerctx->errstr);
+    if (ret < 0) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_OP_VERSION_MISMATCH,
+               "failed to validate the operating version of peer (%s)",
+               peerinfo->hostname);
+        goto out;
+    }
+
+    rsp_dict = dict_new();
+    if (!rsp_dict)
+        goto out;
+
+    ret = dict_set_int32(rsp_dict, GD_OP_VERSION_KEY, conf->op_version);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+               "failed to set operating version in dict");
+        goto out;
+    }
+
+    GF_PROTOCOL_DICT_SERIALIZE(this, rsp_dict, (&arg.hndsk.hndsk_val),
+                               arg.hndsk.hndsk_len, op_errno, out);
+
+    ret = glusterd_submit_request(
+        peerinfo->rpc, &arg, frame, &gd_clnt_mgmt_hndsk_prog,
+        GD_MGMT_HNDSK_VERSIONS_ACK, NULL, this,
+        glusterd_mgmt_hndsk_version_ack_cbk, (xdrproc_t)xdr_gf_mgmt_hndsk_req);
 
 out:
-        if (ret) {
-                frame->local = NULL;
-                STACK_DESTROY (frame->root);
-                if (peerinfo)
-                        rpc_transport_disconnect (peerinfo->rpc->conn.trans);
-        }
+    if (ret) {
+        frame->local = NULL;
+        STACK_DESTROY(frame->root);
+        if (peerinfo)
+            rpc_transport_disconnect(peerinfo->rpc->conn.trans, _gf_false);
+    }
 
-        rcu_read_unlock ();
+    RCU_READ_UNLOCK;
 
-        if (rsp.hndsk.hndsk_val)
-                free (rsp.hndsk.hndsk_val);
+    if (rsp.hndsk.hndsk_val)
+        free(rsp.hndsk.hndsk_val);
 
-        if (arg.hndsk.hndsk_val)
-                GF_FREE (arg.hndsk.hndsk_val);
+    if (arg.hndsk.hndsk_val)
+        GF_FREE(arg.hndsk.hndsk_val);
 
-        if (dict)
-                dict_unref (dict);
+    if (dict)
+        dict_unref(dict);
 
-        if (rsp_dict)
-                dict_unref (rsp_dict);
+    if (rsp_dict)
+        dict_unref(rsp_dict);
 
-        return 0;
+    return 0;
 }
 
 int
-glusterd_mgmt_hndsk_version_cbk (struct rpc_req *req, struct iovec *iov,
-                                     int count, void *myframe)
+glusterd_mgmt_hndsk_version_cbk(struct rpc_req *req, struct iovec *iov,
+                                int count, void *myframe)
 {
-        return glusterd_big_locked_cbk (req, iov, count, myframe,
-                                        __glusterd_mgmt_hndsk_version_cbk);
+    return glusterd_big_locked_cbk(req, iov, count, myframe,
+                                   __glusterd_mgmt_hndsk_version_cbk);
 }
 
 int
-glusterd_mgmt_handshake (xlator_t *this, glusterd_peerctx_t *peerctx)
+glusterd_mgmt_handshake(xlator_t *this, glusterd_peerctx_t *peerctx)
 {
-        call_frame_t        *frame    = NULL;
-        gf_mgmt_hndsk_req    req      = {{0,},};
-        glusterd_peerinfo_t *peerinfo = NULL;
-        dict_t              *req_dict = NULL;
-        int                  ret      = -1;
-        int                  op_errno = EINVAL;
-
-        frame = create_frame (this, this->ctx->pool);
-        if (!frame)
-                goto out;
-
-        frame->local = peerctx;
+    call_frame_t *frame = NULL;
+    gf_mgmt_hndsk_req req = {
+        {
+            0,
+        },
+    };
+    glusterd_peerinfo_t *peerinfo = NULL;
+    dict_t *req_dict = NULL;
+    int ret = -1;
+
+    frame = create_frame(this, this->ctx->pool);
+    if (!frame) {
+        gf_smsg("glusterd", GF_LOG_WARNING, errno, GD_MSG_FRAME_CREATE_FAIL,
+                NULL);
+        goto out;
+    }
+
+    frame->local = peerctx;
+
+    req_dict = dict_new();
+    if (!req_dict) {
+        gf_smsg("glusterd", GF_LOG_WARNING, ENOMEM, GD_MSG_DICT_CREATE_FAIL,
+                NULL);
+        goto out;
+    }
+
+    ret = dict_set_dynstr(req_dict, GD_PEER_ID_KEY,
+                          gf_strdup(uuid_utoa(MY_UUID)));
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_SET_FAILED,
+               "failed to set peer ID in dict");
+        goto out;
+    }
+
+    GF_PROTOCOL_DICT_SERIALIZE(this, req_dict, (&req.hndsk.hndsk_val),
+                               req.hndsk.hndsk_len, ret, out);
+
+    RCU_READ_LOCK;
+
+    peerinfo = glusterd_peerinfo_find_by_generation(peerctx->peerinfo_gen);
+    if (!peerinfo) {
+        RCU_READ_UNLOCK;
+        gf_msg_debug(THIS->name, 0, "Could not find peer %s(%s)",
+                     peerctx->peername, uuid_utoa(peerctx->peerid));
+        goto out;
+    }
+
+    ret = glusterd_submit_request(
+        peerinfo->rpc, &req, frame, &gd_clnt_mgmt_hndsk_prog,
+        GD_MGMT_HNDSK_VERSIONS, NULL, this, glusterd_mgmt_hndsk_version_cbk,
+        (xdrproc_t)xdr_gf_mgmt_hndsk_req);
+
+    RCU_READ_UNLOCK;
+
+    ret = 0;
 
-        req_dict = dict_new ();
-        if (!req_dict)
-                goto out;
-
-        ret = dict_set_dynstr (req_dict, GD_PEER_ID_KEY,
-                               gf_strdup (uuid_utoa (MY_UUID)));
-        if (ret) {
-                gf_log(this->name, GF_LOG_ERROR,
-                       "failed to set peer ID in dict");
-                goto out;
-        }
-
-        GF_PROTOCOL_DICT_SERIALIZE (this, req_dict, (&req.hndsk.hndsk_val),
-                                    req.hndsk.hndsk_len, op_errno, out);
-
-        rcu_read_lock ();
-
-        peerinfo = glusterd_peerinfo_find (peerctx->peerid, peerctx->peername);
-        if (!peerinfo) {
-                gf_log (THIS->name, GF_LOG_DEBUG, "Could not find peer %s(%s)",
-                        peerctx->peername, uuid_utoa (peerctx->peerid));
-                goto unlock;
-        }
-
-        ret = glusterd_submit_request (peerinfo->rpc, &req, frame,
-                                       &gd_clnt_mgmt_hndsk_prog,
-                                       GD_MGMT_HNDSK_VERSIONS, NULL, this,
-                                       glusterd_mgmt_hndsk_version_cbk,
-                                       (xdrproc_t)xdr_gf_mgmt_hndsk_req);
-        ret = 0;
-unlock:
-        rcu_read_unlock ();
 out:
-        if (ret && frame)
-                STACK_DESTROY (frame->root);
+    if (req_dict)
+        dict_unref(req_dict);
+
+    if (ret && frame)
+        STACK_DESTROY(frame->root);
 
-        return ret;
+    return ret;
 }
 
 int
-glusterd_set_clnt_mgmt_program (glusterd_peerinfo_t *peerinfo,
-                                gf_prog_detail *prog)
+glusterd_set_clnt_mgmt_program(glusterd_peerinfo_t *peerinfo,
+                               gf_prog_detail *prog)
 {
-        gf_prog_detail  *trav   = NULL;
-        int             ret     = -1;
+    gf_prog_detail *trav = NULL;
+    int ret = -1;
 
-        if (!peerinfo || !prog)
-                goto out;
+    if (!peerinfo || !prog)
+        goto out;
 
-        trav = prog;
+    trav = prog;
 
-        while (trav) {
-                ret = -1;
-                if ((gd_mgmt_prog.prognum == trav->prognum) &&
-                    (gd_mgmt_prog.progver == trav->progver)) {
-                        peerinfo->mgmt = &gd_mgmt_prog;
-                        ret = 0;
-                }
-
-                if ((gd_peer_prog.prognum == trav->prognum) &&
-                    (gd_peer_prog.progver == trav->progver)) {
-                        peerinfo->peer = &gd_peer_prog;
-                        ret = 0;
-                }
-
-                if (ret) {
-                        gf_log ("", GF_LOG_DEBUG,
-                                "%s (%"PRId64":%"PRId64") not supported",
-                                trav->progname, trav->prognum,
-                                trav->progver);
-                }
-
-                trav = trav->next;
-        }
-
-        if (peerinfo->mgmt) {
-                 gf_log ("", GF_LOG_INFO,
-                         "Using Program %s, Num (%d), Version (%d)",
-                         peerinfo->mgmt->progname, peerinfo->mgmt->prognum,
-                         peerinfo->mgmt->progver);
-        }
-
-        if (peerinfo->peer) {
-                 gf_log ("", GF_LOG_INFO,
-                         "Using Program %s, Num (%d), Version (%d)",
-                         peerinfo->peer->progname, peerinfo->peer->prognum,
-                         peerinfo->peer->progver);
+    while (trav) {
+        ret = -1;
+        if ((gd_mgmt_prog.prognum == trav->prognum) &&
+            (gd_mgmt_prog.progver == trav->progver)) {
+            peerinfo->mgmt = &gd_mgmt_prog;
+            ret = 0;
         }
 
-        if (peerinfo->mgmt_v3) {
-                 gf_log ("", GF_LOG_INFO,
-                         "Using Program %s, Num (%d), Version (%d)",
-                         peerinfo->mgmt_v3->progname,
-                         peerinfo->mgmt_v3->prognum,
-                         peerinfo->mgmt_v3->progver);
+        if ((gd_peer_prog.prognum == trav->prognum) &&
+            (gd_peer_prog.progver == trav->progver)) {
+            peerinfo->peer = &gd_peer_prog;
+            ret = 0;
         }
 
-        ret = 0;
+        if (ret) {
+            gf_msg_debug("glusterd", 0,
+                         "%s (%" PRId64 ":%" PRId64 ") not supported",
+                         trav->progname, trav->prognum, trav->progver);
+        }
+
+        trav = trav->next;
+    }
+
+    if (peerinfo->mgmt) {
+        gf_msg("glusterd", GF_LOG_INFO, 0, GD_MSG_VERS_INFO,
+               "Using Program %s, Num (%d), Version (%d)",
+               peerinfo->mgmt->progname, peerinfo->mgmt->prognum,
+               peerinfo->mgmt->progver);
+    }
+
+    if (peerinfo->peer) {
+        gf_msg("glusterd", GF_LOG_INFO, 0, GD_MSG_VERS_INFO,
+               "Using Program %s, Num (%d), Version (%d)",
+               peerinfo->peer->progname, peerinfo->peer->prognum,
+               peerinfo->peer->progver);
+    }
+
+    if (peerinfo->mgmt_v3) {
+        gf_msg("glusterd", GF_LOG_INFO, 0, GD_MSG_VERS_INFO,
+               "Using Program %s, Num (%d), Version (%d)",
+               peerinfo->mgmt_v3->progname, peerinfo->mgmt_v3->prognum,
+               peerinfo->mgmt_v3->progver);
+    }
+
+    ret = 0;
 out:
-        return ret;
-
+    return ret;
 }
 
 static gf_boolean_t
-_mgmt_hndsk_prog_present (gf_prog_detail *prog) {
-        gf_boolean_t    ret = _gf_false;
-        gf_prog_detail  *trav = NULL;
+_mgmt_hndsk_prog_present(gf_prog_detail *prog)
+{
+    gf_boolean_t ret = _gf_false;
+    gf_prog_detail *trav = NULL;
 
-        GF_ASSERT (prog);
+    GF_ASSERT(prog);
 
-        trav = prog;
+    trav = prog;
 
-        while (trav) {
-                if ((trav->prognum == GD_MGMT_HNDSK_PROGRAM) &&
-                    (trav->progver == GD_MGMT_HNDSK_VERSION)) {
-                        ret = _gf_true;
-                        goto out;
-                }
-                trav = trav->next;
+    while (trav) {
+        if ((trav->prognum == GD_MGMT_HNDSK_PROGRAM) &&
+            (trav->progver == GD_MGMT_HNDSK_VERSION)) {
+            ret = _gf_true;
+            goto out;
         }
+        trav = trav->next;
+    }
 out:
-        return ret;
+    return ret;
 }
 
 int
-__glusterd_peer_dump_version_cbk (struct rpc_req *req, struct iovec *iov,
-                                int count, void *myframe)
+__glusterd_peer_dump_version_cbk(struct rpc_req *req, struct iovec *iov,
+                                 int count, void *myframe)
 {
-        int                  ret      = -1;
-        gf_dump_rsp          rsp      = {0,};
-        xlator_t            *this     = NULL;
-        gf_prog_detail      *trav     = NULL;
-        gf_prog_detail      *next     = NULL;
-        call_frame_t        *frame    = NULL;
-        glusterd_peerinfo_t *peerinfo = NULL;
-        glusterd_peerctx_t  *peerctx  = NULL;
-        glusterd_conf_t     *conf     = NULL;
-        char                msg[1024] = {0,};
-
-        this = THIS;
-        conf = this->private;
-        frame = myframe;
-        peerctx = frame->local;
-
-        rcu_read_lock ();
-
-        peerinfo = glusterd_peerinfo_find (peerctx->peerid, peerctx->peername);
-        if (!peerinfo) {
-                gf_log (this->name, GF_LOG_DEBUG, "Couldn't find peer %s(%s)",
-                        peerctx->peername, uuid_utoa (peerctx->peerid));
-                goto out;
-        }
-
-        if (-1 == req->rpc_status) {
-                snprintf (msg, sizeof (msg),
-                          "Error through RPC layer, retry again later");
-                gf_log ("", GF_LOG_ERROR, "%s", msg);
-                peerctx->errstr = gf_strdup (msg);
-                goto out;
-        }
-
-        ret = xdr_to_generic (*iov, &rsp, (xdrproc_t)xdr_gf_dump_rsp);
-        if (ret < 0) {
-                snprintf (msg, sizeof (msg), "Failed to decode XDR");
-                gf_log ("", GF_LOG_ERROR, "%s", msg);
-                peerctx->errstr = gf_strdup (msg);
-                goto out;
-        }
-        if (-1 == rsp.op_ret) {
-                snprintf (msg, sizeof (msg),
-                          "Failed to get the 'versions' from remote server");
-                gf_log (frame->this->name, GF_LOG_ERROR, "%s", msg);
-                peerctx->errstr = gf_strdup (msg);
-                goto out;
-        }
-
-        if (_mgmt_hndsk_prog_present (rsp.prog)) {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "Proceeding to op-version handshake with peer %s",
-                        peerinfo->hostname);
-                ret = glusterd_mgmt_handshake (this, peerctx);
-                goto out;
-        } else if (conf->op_version > 1) {
-                ret = -1;
-                snprintf (msg, sizeof (msg),
-                          "Peer %s does not support required op-version",
-                          peerinfo->hostname);
-                peerctx->errstr = gf_strdup (msg);
-                gf_log (this->name, GF_LOG_ERROR, "%s", msg);
-                goto out;
-        }
-
-        /* Make sure we assign the proper program to peer */
-        ret = glusterd_set_clnt_mgmt_program (peerinfo, rsp.prog);
-        if (ret) {
-                gf_log ("", GF_LOG_WARNING, "failed to set the mgmt program");
-                goto out;
-        }
+    int ret = -1;
+    gf_dump_rsp rsp = {
+        0,
+    };
+    xlator_t *this = NULL;
+    gf_prog_detail *trav = NULL;
+    gf_prog_detail *next = NULL;
+    call_frame_t *frame = NULL;
+    glusterd_peerinfo_t *peerinfo = NULL;
+    glusterd_peerctx_t *peerctx = NULL;
+    glusterd_conf_t *conf = NULL;
+    char msg[1024] = {
+        0,
+    };
+
+    this = THIS;
+    conf = this->private;
+    frame = myframe;
+    peerctx = frame->local;
+
+    RCU_READ_LOCK;
+
+    peerinfo = glusterd_peerinfo_find_by_generation(peerctx->peerinfo_gen);
+    if (!peerinfo) {
+        gf_msg_debug(this->name, 0, "Couldn't find peer %s(%s)",
+                     peerctx->peername, uuid_utoa(peerctx->peerid));
+        goto out;
+    }
+
+    if (-1 == req->rpc_status) {
+        snprintf(msg, sizeof(msg),
+                 "Error through RPC layer, retry again later");
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_RPC_LAYER_ERROR, "%s", msg);
+        peerctx->errstr = gf_strdup(msg);
+        goto out;
+    }
+
+    ret = xdr_to_generic(*iov, &rsp, (xdrproc_t)xdr_gf_dump_rsp);
+    if (ret < 0) {
+        snprintf(msg, sizeof(msg), "Failed to decode XDR");
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_REQ_DECODE_FAIL, "%s", msg);
+        peerctx->errstr = gf_strdup(msg);
+        goto out;
+    }
+    if (-1 == rsp.op_ret) {
+        snprintf(msg, sizeof(msg),
+                 "Failed to get the 'versions' from remote server");
+        gf_msg(frame->this->name, GF_LOG_ERROR, 0, GD_MSG_VERS_GET_FAIL, "%s",
+               msg);
+        peerctx->errstr = gf_strdup(msg);
+        goto out;
+    }
+
+    if (_mgmt_hndsk_prog_present(rsp.prog)) {
+        gf_msg_debug(this->name, 0,
+                     "Proceeding to op-version handshake with peer %s",
+                     peerinfo->hostname);
+        ret = glusterd_mgmt_handshake(this, peerctx);
+        goto out;
+    } else if (conf->op_version > 1) {
+        ret = -1;
+        snprintf(msg, sizeof(msg),
+                 "Peer %s does not support required op-version",
+                 peerinfo->hostname);
+        peerctx->errstr = gf_strdup(msg);
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_VERSION_UNSUPPORTED, "%s",
+               msg);
+        goto out;
+    }
+
+    /* Make sure we assign the proper program to peer */
+    ret = glusterd_set_clnt_mgmt_program(peerinfo, rsp.prog);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_WARNING, 0, GD_MSG_MGMT_PGM_SET_FAIL,
+               "failed to set the mgmt program");
+        goto out;
+    }
+
+    ret = default_notify(this, GF_EVENT_CHILD_UP, NULL);
+
+    if (GD_MODE_ON == peerctx->args.mode) {
+        (void)glusterd_event_connected_inject(peerctx);
+        peerctx->args.req = NULL;
+    } else if (GD_MODE_SWITCH_ON == peerctx->args.mode) {
+        peerctx->args.mode = GD_MODE_ON;
+    } else {
+        gf_msg(this->name, GF_LOG_WARNING, 0, GD_MSG_UNKNOWN_MODE,
+               "unknown mode %d", peerctx->args.mode);
+    }
+
+    ret = 0;
 
-        ret = default_notify (this, GF_EVENT_CHILD_UP, NULL);
+out:
+    if (ret != 0 && peerinfo)
+        rpc_transport_disconnect(peerinfo->rpc->conn.trans, _gf_false);
 
-        if (GD_MODE_ON == peerctx->args.mode) {
-                ret = glusterd_event_connected_inject (peerctx);
-                peerctx->args.req = NULL;
-        } else if (GD_MODE_SWITCH_ON == peerctx->args.mode) {
-                peerctx->args.mode = GD_MODE_ON;
-        } else {
-                gf_log ("", GF_LOG_WARNING, "unknown mode %d",
-                        peerctx->args.mode);
-        }
+    RCU_READ_UNLOCK;
 
-        ret = 0;
+    glusterd_friend_sm();
+    glusterd_op_sm();
 
-out:
-        if (ret != 0 && peerinfo)
-                rpc_transport_disconnect (peerinfo->rpc->conn.trans);
-
-        rcu_read_unlock ();
-
-        glusterd_friend_sm ();
-        glusterd_op_sm ();
-
-        /* don't use GF_FREE, buffer was allocated by libc */
-        if (rsp.prog) {
-                trav = rsp.prog;
-                while (trav) {
-                        next = trav->next;
-                        free (trav->progname);
-                        free (trav);
-                        trav = next;
-                }
+    /* don't use GF_FREE, buffer was allocated by libc */
+    if (rsp.prog) {
+        trav = rsp.prog;
+        while (trav) {
+            next = trav->next;
+            free(trav->progname);
+            free(trav);
+            trav = next;
         }
+    }
 
-        frame->local = NULL;
-        STACK_DESTROY (frame->root);
+    frame->local = NULL;
+    STACK_DESTROY(frame->root);
 
-        return 0;
+    return 0;
 }
 
-
 int
-glusterd_peer_dump_version_cbk (struct rpc_req *req, struct iovec *iov,
-                                int count, void *myframe)
+glusterd_peer_dump_version_cbk(struct rpc_req *req, struct iovec *iov,
+                               int count, void *myframe)
 {
-        return glusterd_big_locked_cbk (req, iov, count, myframe,
-                                        __glusterd_peer_dump_version_cbk);
+    return glusterd_big_locked_cbk(req, iov, count, myframe,
+                                   __glusterd_peer_dump_version_cbk);
 }
 
 int
-glusterd_peer_dump_version (xlator_t *this, struct rpc_clnt *rpc,
-                            glusterd_peerctx_t *peerctx)
+glusterd_peer_dump_version(xlator_t *this, struct rpc_clnt *rpc,
+                           glusterd_peerctx_t *peerctx)
 {
-        call_frame_t        *frame    = NULL;
-        gf_dump_req          req      = {0,};
-        glusterd_peerinfo_t *peerinfo = NULL;
-        int                  ret      = -1;
-
-        frame = create_frame (this, this->ctx->pool);
-        if (!frame)
-                goto out;
-
-        frame->local = peerctx;
-        if (!peerctx)
-                goto out;
-
-        rcu_read_lock ();
-
-        peerinfo = glusterd_peerinfo_find (peerctx->peerid, peerctx->peername);
-        if (!peerinfo) {
-                gf_log (this->name, GF_LOG_DEBUG, "Couldn't find peer %s(%s)",
-                        peerctx->peername, uuid_utoa (peerctx->peerid));
-                goto unlock;
-        }
-
-        req.gfs_id = 0xcafe;
-
-        ret = glusterd_submit_request (peerinfo->rpc, &req, frame,
-                                       &glusterd_dump_prog, GF_DUMP_DUMP,
-                                       NULL, this,
-                                       glusterd_peer_dump_version_cbk,
-                                       (xdrproc_t)xdr_gf_dump_req);
-unlock:
-        rcu_read_unlock ();
+    call_frame_t *frame = NULL;
+    gf_dump_req req = {
+        0,
+    };
+    glusterd_peerinfo_t *peerinfo = NULL;
+    int ret = -1;
+
+    frame = create_frame(this, this->ctx->pool);
+    if (!frame) {
+        gf_smsg(this->name, GF_LOG_WARNING, errno, GD_MSG_FRAME_CREATE_FAIL,
+                NULL);
+        goto out;
+    }
+
+    frame->local = peerctx;
+    if (!peerctx) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_INVALID_ARGUMENT, NULL);
+        goto out;
+    }
+
+    RCU_READ_LOCK;
+
+    peerinfo = glusterd_peerinfo_find_by_generation(peerctx->peerinfo_gen);
+    if (!peerinfo) {
+        RCU_READ_UNLOCK;
+        gf_msg_debug(this->name, 0, "Couldn't find peer %s(%s)",
+                     peerctx->peername, uuid_utoa(peerctx->peerid));
+        goto out;
+    }
+
+    req.gfs_id = 0xcafe;
+
+    ret = glusterd_submit_request(
+        peerinfo->rpc, &req, frame, &glusterd_dump_prog, GF_DUMP_DUMP, NULL,
+        this, glusterd_peer_dump_version_cbk, (xdrproc_t)xdr_gf_dump_req);
+
+    RCU_READ_UNLOCK;
 out:
-        return ret;
+    if (ret && frame)
+        STACK_DESTROY(frame->root);
+
+    return ret;
 }
diff --git a/xlators/mgmt/glusterd/src/glusterd-hooks.c b/xlators/mgmt/glusterd/src/glusterd-hooks.c
index 569f2743dda..61c0f1c946f 100644
--- a/xlators/mgmt/glusterd/src/glusterd-hooks.c
+++ b/xlators/mgmt/glusterd/src/glusterd-hooks.c
@@ -8,560 +8,634 @@
    cases as published by the Free Software Foundation.
 */
 
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
-
-#include "globals.h"
-#include "glusterfs.h"
-#include "dict.h"
-#include "xlator.h"
-#include "logging.h"
-#include "run.h"
-#include "defaults.h"
-#include "compat.h"
-#include "compat-errno.h"
+#include <glusterfs/glusterfs.h>
+#include <glusterfs/dict.h>
+#include <glusterfs/xlator.h>
+#include <glusterfs/logging.h>
+#include <glusterfs/run.h>
+#include <glusterfs/defaults.h>
+#include <glusterfs/syscall.h>
+#include <glusterfs/compat.h>
+#include <glusterfs/compat-errno.h>
 #include "glusterd.h"
 #include "glusterd-sm.h"
 #include "glusterd-op-sm.h"
 #include "glusterd-utils.h"
 #include "glusterd-store.h"
 #include "glusterd-hooks.h"
+#include "glusterd-messages.h"
 
 #include <fnmatch.h>
 
 #define EMPTY ""
-char glusterd_hook_dirnames[GD_OP_MAX][256] =
-{
-        [GD_OP_NONE]                    = EMPTY,
-        [GD_OP_CREATE_VOLUME]           = "create",
-        [GD_OP_START_BRICK]             = EMPTY,
-        [GD_OP_STOP_BRICK]              = EMPTY,
-        [GD_OP_DELETE_VOLUME]           = "delete",
-        [GD_OP_START_VOLUME]            = "start",
-        [GD_OP_STOP_VOLUME]             = "stop",
-        [GD_OP_DEFRAG_VOLUME]           = EMPTY,
-        [GD_OP_ADD_BRICK]               = "add-brick",
-        [GD_OP_REMOVE_BRICK]            = "remove-brick",
-        [GD_OP_REPLACE_BRICK]           = EMPTY,
-        [GD_OP_SET_VOLUME]              = "set",
-        [GD_OP_RESET_VOLUME]            = "reset",
-        [GD_OP_SYNC_VOLUME]             = EMPTY,
-        [GD_OP_LOG_ROTATE]              = EMPTY,
-        [GD_OP_GSYNC_CREATE]            = "gsync-create",
-        [GD_OP_GSYNC_SET]               = EMPTY,
-        [GD_OP_PROFILE_VOLUME]          = EMPTY,
-        [GD_OP_QUOTA]                   = EMPTY,
-        [GD_OP_STATUS_VOLUME]           = EMPTY,
-        [GD_OP_REBALANCE]               = EMPTY,
-        [GD_OP_HEAL_VOLUME]             = EMPTY,
-        [GD_OP_STATEDUMP_VOLUME]        = EMPTY,
-        [GD_OP_LIST_VOLUME]             = EMPTY,
-        [GD_OP_CLEARLOCKS_VOLUME]       = EMPTY,
-        [GD_OP_DEFRAG_BRICK_VOLUME]     = EMPTY,
+char glusterd_hook_dirnames[GD_OP_MAX][256] = {
+    [GD_OP_NONE] = EMPTY,
+    [GD_OP_CREATE_VOLUME] = "create",
+    [GD_OP_START_BRICK] = EMPTY,
+    [GD_OP_STOP_BRICK] = EMPTY,
+    [GD_OP_DELETE_VOLUME] = "delete",
+    [GD_OP_START_VOLUME] = "start",
+    [GD_OP_STOP_VOLUME] = "stop",
+    [GD_OP_DEFRAG_VOLUME] = EMPTY,
+    [GD_OP_ADD_BRICK] = "add-brick",
+    [GD_OP_REMOVE_BRICK] = "remove-brick",
+    [GD_OP_REPLACE_BRICK] = EMPTY,
+    [GD_OP_SET_VOLUME] = "set",
+    [GD_OP_RESET_VOLUME] = "reset",
+    [GD_OP_SYNC_VOLUME] = EMPTY,
+    [GD_OP_LOG_ROTATE] = EMPTY,
+    [GD_OP_GSYNC_CREATE] = "gsync-create",
+    [GD_OP_GSYNC_SET] = EMPTY,
+    [GD_OP_PROFILE_VOLUME] = EMPTY,
+    [GD_OP_QUOTA] = EMPTY,
+    [GD_OP_STATUS_VOLUME] = EMPTY,
+    [GD_OP_REBALANCE] = EMPTY,
+    [GD_OP_HEAL_VOLUME] = EMPTY,
+    [GD_OP_STATEDUMP_VOLUME] = EMPTY,
+    [GD_OP_LIST_VOLUME] = EMPTY,
+    [GD_OP_CLEARLOCKS_VOLUME] = EMPTY,
+    [GD_OP_DEFRAG_BRICK_VOLUME] = EMPTY,
+    [GD_OP_RESET_BRICK] = EMPTY,
 };
 #undef EMPTY
 
-static inline gf_boolean_t
-glusterd_is_hook_enabled (char *script)
+static gf_boolean_t
+glusterd_is_hook_enabled(char *script)
 {
-        return (script[0] == 'S' && (fnmatch ("*.rpmsave", script, 0) != 0)
-                                 && (fnmatch ("*.rpmnew", script, 0) != 0));
+    return (script[0] == 'S' && (fnmatch("*.rpmsave", script, 0) != 0) &&
+            (fnmatch("*.rpmnew", script, 0) != 0));
 }
 
 int
-glusterd_hooks_create_hooks_directory (char *basedir)
+glusterd_hooks_create_hooks_directory(char *basedir)
 {
-        int  ret                                        = -1;
-        int  op                                         = GD_OP_NONE;
-        int  type                                       = GD_COMMIT_HOOK_NONE;
-        char version_dir[PATH_MAX]                      = {0, };
-        char path[PATH_MAX]                             = {0, };
-        char *cmd_subdir                                = NULL;
-        char type_subdir[GD_COMMIT_HOOK_MAX][256]       = {{0, },
-                                                           "pre",
-                                                           "post"};
-        glusterd_conf_t *priv                           = NULL;
-
-        priv = THIS->private;
-
-        snprintf (path, sizeof (path), "%s/hooks", basedir);
-        ret = mkdir_p (path, 0777, _gf_true);
-        if (ret) {
-                gf_log (THIS->name, GF_LOG_CRITICAL, "Unable to create %s due"
-                         "to %s", path, strerror (errno));
-                goto out;
+    int ret = -1;
+    int op = GD_OP_NONE;
+    int type = GD_COMMIT_HOOK_NONE;
+    char version_dir[PATH_MAX] = {
+        0,
+    };
+    char path[PATH_MAX] = {
+        0,
+    };
+    char *cmd_subdir = NULL;
+    char type_subdir[GD_COMMIT_HOOK_MAX][256] = {{
+                                                     0,
+                                                 },
+                                                 "pre",
+                                                 "post"};
+    glusterd_conf_t *priv = NULL;
+    int32_t len = 0;
+
+    xlator_t *this = NULL;
+    this = THIS;
+    GF_ASSERT(this);
+    priv = this->private;
+
+    snprintf(path, sizeof(path), "%s/hooks", basedir);
+    ret = mkdir_p(path, 0755, _gf_true);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_CRITICAL, errno, GD_MSG_CREATE_DIR_FAILED,
+                "Path=%s", path, NULL);
+        goto out;
+    }
+
+    GLUSTERD_GET_HOOKS_DIR(version_dir, GLUSTERD_HOOK_VER, priv);
+    ret = mkdir_p(version_dir, 0755, _gf_true);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_CRITICAL, errno, GD_MSG_CREATE_DIR_FAILED,
+                "Directory=%s", version_dir, NULL);
+        goto out;
+    }
+
+    for (op = GD_OP_NONE + 1; op < GD_OP_MAX; op++) {
+        cmd_subdir = glusterd_hooks_get_hooks_cmd_subdir(op);
+        if (strlen(cmd_subdir) == 0)
+            continue;
+
+        len = snprintf(path, sizeof(path), "%s/%s", version_dir, cmd_subdir);
+        if ((len < 0) || (len >= sizeof(path))) {
+            gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_COPY_FAIL, NULL);
+            ret = -1;
+            goto out;
         }
-
-        GLUSTERD_GET_HOOKS_DIR (version_dir, GLUSTERD_HOOK_VER, priv);
-        ret = mkdir_p (version_dir, 0777, _gf_true);
+        ret = mkdir_p(path, 0755, _gf_true);
         if (ret) {
-                gf_log (THIS->name, GF_LOG_CRITICAL, "Unable to create %s due "
-                        "to %s", version_dir, strerror (errno));
-                goto out;
+            gf_smsg(this->name, GF_LOG_CRITICAL, errno,
+                    GD_MSG_CREATE_DIR_FAILED, "Path=%s", path, NULL);
+            goto out;
         }
 
-        for (op = GD_OP_NONE+1; op < GD_OP_MAX; op++) {
-                cmd_subdir = glusterd_hooks_get_hooks_cmd_subdir (op);
-                if (strlen (cmd_subdir) == 0)
-                        continue;
-
-                snprintf (path, sizeof (path), "%s/%s", version_dir,
-                          cmd_subdir);
-                ret = mkdir_p (path, 0777, _gf_true);
-                if (ret) {
-                        gf_log (THIS->name, GF_LOG_CRITICAL,
-                                "Unable to create %s due to %s",
-                                path, strerror (errno));
-                        goto out;
-                }
-
-                for (type = GD_COMMIT_HOOK_PRE; type < GD_COMMIT_HOOK_MAX;
-                     type++) {
-                        snprintf (path, sizeof (path), "%s/%s/%s",
-                                  version_dir, cmd_subdir, type_subdir[type]);
-                        ret = mkdir_p (path, 0777, _gf_true);
-                        if (ret) {
-                                gf_log (THIS->name, GF_LOG_CRITICAL,
-                                        "Unable to create %s due to %s",
-                                        path, strerror (errno));
-                                goto out;
-                        }
-                }
+        for (type = GD_COMMIT_HOOK_PRE; type < GD_COMMIT_HOOK_MAX; type++) {
+            len = snprintf(path, sizeof(path), "%s/%s/%s", version_dir,
+                           cmd_subdir, type_subdir[type]);
+            if ((len < 0) || (len >= sizeof(path))) {
+                gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_COPY_FAIL,
+                        NULL);
+                ret = -1;
+                goto out;
+            }
+            ret = mkdir_p(path, 0755, _gf_true);
+            if (ret) {
+                gf_smsg(this->name, GF_LOG_CRITICAL, errno,
+                        GD_MSG_CREATE_DIR_FAILED, "Path=%s", path, NULL);
+                goto out;
+            }
         }
+    }
 
-        ret = 0;
+    ret = 0;
 out:
-        return ret;
+    return ret;
 }
 
-char*
-glusterd_hooks_get_hooks_cmd_subdir (glusterd_op_t op)
+char *
+glusterd_hooks_get_hooks_cmd_subdir(glusterd_op_t op)
 {
-        GF_ASSERT ((op > GD_OP_NONE) && (op < GD_OP_MAX));
+    GF_ASSERT((op > GD_OP_NONE) && (op < GD_OP_MAX));
 
-        return glusterd_hook_dirnames[op];
+    return glusterd_hook_dirnames[op];
 }
 
 void
-glusterd_hooks_add_working_dir (runner_t *runner, glusterd_conf_t *priv)
+glusterd_hooks_add_working_dir(runner_t *runner, glusterd_conf_t *priv)
 {
-        runner_argprintf (runner, "--gd-workdir=%s", priv->workdir);
+    runner_argprintf(runner, "--gd-workdir=%s", priv->workdir);
 }
 
 void
-glusterd_hooks_add_op (runner_t *runner, char *op)
+glusterd_hooks_add_op(runner_t *runner, char *op)
 {
-        runner_argprintf (runner, "--volume-op=%s", op);
+    runner_argprintf(runner, "--volume-op=%s", op);
 }
 
 void
-glusterd_hooks_add_hooks_version (runner_t* runner)
+glusterd_hooks_add_hooks_version(runner_t *runner)
 {
-        runner_argprintf (runner, "--version=%d", GLUSTERD_HOOK_VER);
+    runner_argprintf(runner, "--version=%d", GLUSTERD_HOOK_VER);
 }
 
-int
-glusterd_hooks_set_volume_args (dict_t *dict, runner_t *runner)
+static void
+glusterd_hooks_add_custom_args(dict_t *dict, runner_t *runner)
 {
-        int     i               = 0;
-        int     count           = 0;
-        int     ret             = -1;
-        char    query[1024]     = {0,};
-        char    *key            = NULL;
-        char    *value          = NULL;
-
-        ret = dict_get_int32 (dict, "count", &count);
-        if (ret)
-                goto out;
+    char *hooks_args = NULL;
+    int32_t ret = -1;
+    xlator_t *this = NULL;
 
-        /* This will not happen unless op_ctx
-         * is corrupted*/
-        if (!count)
-                goto out;
+    this = THIS;
+    GF_VALIDATE_OR_GOTO("glusterd", this, out);
+    GF_VALIDATE_OR_GOTO(this->name, dict, out);
+    GF_VALIDATE_OR_GOTO(this->name, runner, out);
 
-        runner_add_arg (runner, "-o");
-        for (i = 1; ret == 0; i++) {
-                snprintf (query, sizeof (query), "key%d", i);
-                ret = dict_get_str (dict, query, &key);
-                if (ret)
-                       continue;
+    ret = dict_get_str(dict, "hooks_args", &hooks_args);
+    if (ret)
+        gf_msg_debug(this->name, 0, "No Hooks Arguments.");
+    else
+        gf_msg_debug(this->name, 0, "Hooks Args = %s", hooks_args);
 
-                snprintf (query, sizeof (query), "value%d", i);
-                ret = dict_get_str (dict, query, &value);
-                if (ret)
-                       continue;
+    if (hooks_args)
+        runner_argprintf(runner, "%s", hooks_args);
 
-                runner_argprintf (runner, "%s=%s", key, value);
-        }
-
-        ret = 0;
 out:
-        return ret;
+    return;
 }
 
-static int
-glusterd_hooks_add_op_args (runner_t *runner, glusterd_op_t op,
-                            dict_t *op_ctx, glusterd_commit_hook_type_t type)
+int
+glusterd_hooks_set_volume_args(dict_t *dict, runner_t *runner)
 {
-        char                   *hooks_args       = NULL;
-        int                     vol_count        = 0;
-        gf_boolean_t            truth            = _gf_false;
-        glusterd_volinfo_t      *voliter         = NULL;
-        glusterd_conf_t         *priv            = NULL;
-        int                     ret              = -1;
-
-        priv = THIS->private;
-        cds_list_for_each_entry (voliter, &priv->volumes, vol_list) {
-                if (glusterd_is_volume_started (voliter))
-                        vol_count++;
-        }
-
-        ret = 0;
-        switch (op) {
-                case GD_OP_START_VOLUME:
-                        if (type == GD_COMMIT_HOOK_PRE &&
-                            vol_count == 0)
-                                truth = _gf_true;
-
-                        else if (type == GD_COMMIT_HOOK_POST &&
-                                 vol_count == 1)
-                                truth = _gf_true;
-
-                        else
-                                truth = _gf_false;
-
-                        runner_argprintf (runner, "--first=%s",
-                                          truth? "yes":"no");
-
-                        glusterd_hooks_add_hooks_version (runner);
-                        glusterd_hooks_add_op (runner, "start");
-                        glusterd_hooks_add_working_dir (runner, priv);
-
-                        break;
-
-                case GD_OP_STOP_VOLUME:
-                        if (type == GD_COMMIT_HOOK_PRE &&
-                            vol_count == 1)
-                                truth = _gf_true;
-
-                        else if (type == GD_COMMIT_HOOK_POST &&
-                                 vol_count == 0)
-                                truth = _gf_true;
-
-                        else
-                                truth = _gf_false;
-
-                        runner_argprintf (runner, "--last=%s",
-                                          truth? "yes":"no");
-                        break;
-
-                case GD_OP_SET_VOLUME:
-                        ret = glusterd_hooks_set_volume_args (op_ctx, runner);
-                        glusterd_hooks_add_working_dir (runner, priv);
-                        break;
-
-                case GD_OP_GSYNC_CREATE:
-                        ret = dict_get_str (op_ctx, "hooks_args", &hooks_args);
-                        if (ret)
-                                gf_log ("", GF_LOG_DEBUG,
-                                        "No Hooks Arguments.");
-                        else
-                                gf_log ("", GF_LOG_DEBUG,
-                                        "Hooks Args = %s", hooks_args);
-                        if (hooks_args)
-                                runner_argprintf (runner, "%s", hooks_args);
-                        break;
-
-                case GD_OP_ADD_BRICK:
-                        glusterd_hooks_add_hooks_version (runner);
-                        glusterd_hooks_add_op (runner, "add-brick");
-                        glusterd_hooks_add_working_dir (runner, priv);
-                        break;
-
-               case GD_OP_RESET_VOLUME:
-                        glusterd_hooks_add_hooks_version (runner);
-                        glusterd_hooks_add_op (runner, "reset");
-                        glusterd_hooks_add_working_dir (runner, priv);
-                        break;
-
-                default:
-                        break;
+    int i = 0;
+    int count = 0;
+    int ret = -1;
+    int flag = 0;
+    char query[1024] = {
+        0,
+    };
+    char *key = NULL;
+    char *value = NULL;
+    char *inet_family = NULL;
+    xlator_t *this = NULL;
+    this = THIS;
+    GF_ASSERT(this);
+
+    ret = dict_get_int32(dict, "count", &count);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_GET_FAILED,
+                "Key=count", NULL);
+        goto out;
+    }
+
+    /* This will not happen unless op_ctx
+     * is corrupted*/
+    if (!count) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_INVALID_ENTRY, "count",
+                NULL);
+        goto out;
+    }
+
+    runner_add_arg(runner, "-o");
+    for (i = 1; ret == 0; i++) {
+        snprintf(query, sizeof(query), "key%d", i);
+        ret = dict_get_str(dict, query, &key);
+        if (ret)
+            continue;
 
+        snprintf(query, sizeof(query), "value%d", i);
+        ret = dict_get_str(dict, query, &value);
+        if (ret)
+            continue;
+
+        runner_argprintf(runner, "%s=%s", key, value);
+        if ((strncmp(key, "cluster.enable-shared-storage",
+                     SLEN("cluster.enable-shared-storage")) == 0 ||
+             strncmp(key, "enable-shared-storage",
+                     SLEN("enable-shared-storage")) == 0) &&
+            strncmp(value, "enable", SLEN("enable")) == 0)
+            flag = 1;
+    }
+
+    glusterd_hooks_add_custom_args(dict, runner);
+    if (flag == 1) {
+        ret = dict_get_str_sizen(this->options, "transport.address-family",
+                                 &inet_family);
+        if (!ret) {
+            runner_argprintf(runner, "transport.address-family=%s",
+                             inet_family);
         }
+    }
 
-        return ret;
+    ret = 0;
+out:
+    return ret;
+}
+
+static int
+glusterd_hooks_add_op_args(runner_t *runner, glusterd_op_t op, dict_t *op_ctx,
+                           glusterd_commit_hook_type_t type)
+{
+    int vol_count = 0;
+    gf_boolean_t truth = _gf_false;
+    glusterd_volinfo_t *voliter = NULL;
+    glusterd_conf_t *priv = NULL;
+    int ret = -1;
+
+    priv = THIS->private;
+    cds_list_for_each_entry(voliter, &priv->volumes, vol_list)
+    {
+        if (glusterd_is_volume_started(voliter))
+            vol_count++;
+    }
+
+    ret = 0;
+    switch (op) {
+        case GD_OP_START_VOLUME:
+            if (type == GD_COMMIT_HOOK_PRE && vol_count == 0)
+                truth = _gf_true;
+
+            else if (type == GD_COMMIT_HOOK_POST && vol_count == 1)
+                truth = _gf_true;
+
+            else
+                truth = _gf_false;
+
+            runner_argprintf(runner, "--first=%s", truth ? "yes" : "no");
+
+            glusterd_hooks_add_hooks_version(runner);
+            glusterd_hooks_add_op(runner, "start");
+            glusterd_hooks_add_working_dir(runner, priv);
+
+            break;
+
+        case GD_OP_STOP_VOLUME:
+            if (type == GD_COMMIT_HOOK_PRE && vol_count == 1)
+                truth = _gf_true;
+
+            else if (type == GD_COMMIT_HOOK_POST && vol_count == 0)
+                truth = _gf_true;
+
+            else
+                truth = _gf_false;
+
+            runner_argprintf(runner, "--last=%s", truth ? "yes" : "no");
+            break;
+
+        case GD_OP_SET_VOLUME:
+            ret = glusterd_hooks_set_volume_args(op_ctx, runner);
+            glusterd_hooks_add_working_dir(runner, priv);
+            break;
+
+        case GD_OP_GSYNC_CREATE:
+            glusterd_hooks_add_custom_args(op_ctx, runner);
+            break;
+
+        case GD_OP_ADD_BRICK:
+            glusterd_hooks_add_hooks_version(runner);
+            glusterd_hooks_add_op(runner, "add-brick");
+            glusterd_hooks_add_working_dir(runner, priv);
+            break;
+
+        case GD_OP_RESET_VOLUME:
+            glusterd_hooks_add_hooks_version(runner);
+            glusterd_hooks_add_op(runner, "reset");
+            glusterd_hooks_add_working_dir(runner, priv);
+            break;
+
+        default:
+            break;
+    }
+
+    return ret;
 }
 
 int
-glusterd_hooks_run_hooks (char *hooks_path, glusterd_op_t op, dict_t *op_ctx,
-                          glusterd_commit_hook_type_t type)
+glusterd_hooks_run_hooks(char *hooks_path, glusterd_op_t op, dict_t *op_ctx,
+                         glusterd_commit_hook_type_t type)
 {
-        xlator_t        *this                   = NULL;
-        glusterd_conf_t *priv                   = NULL;
-        runner_t        runner                  = {0, };
-        struct dirent   *entry                  = NULL;
-        DIR             *hookdir                = NULL;
-        char            *volname                = NULL;
-        char            **lines                 = NULL;
-        int             N                       = 8; /*arbitrary*/
-        int             lineno                  = 0;
-        int             line_count              = 0;
-        int             ret                     = -1;
-
-        this = THIS;
-        priv = this->private;
-
-        ret = dict_get_str (op_ctx, "volname", &volname);
-        if (ret) {
-                gf_log (this->name, GF_LOG_CRITICAL, "Failed to get volname "
-                        "from operation context");
+    xlator_t *this = NULL;
+    runner_t runner = {
+        0,
+    };
+    DIR *hookdir = NULL;
+    struct dirent *entry = NULL;
+    struct dirent scratch[2] = {
+        {
+            0,
+        },
+    };
+    char *volname = NULL;
+    char **lines = NULL;
+    int N = 8; /*arbitrary*/
+    int lineno = 0;
+    int line_count = 0;
+    int ret = -1;
+
+    this = THIS;
+
+    ret = dict_get_str(op_ctx, "volname", &volname);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_CRITICAL, errno, GD_MSG_DICT_GET_FAILED,
+               "Failed to get volname "
+               "from operation context");
+        goto out;
+    }
+
+    hookdir = sys_opendir(hooks_path);
+    if (!hookdir) {
+        ret = -1;
+        gf_msg(this->name, GF_LOG_ERROR, errno, GD_MSG_DIR_OP_FAILED,
+               "Failed to open dir %s", hooks_path);
+        goto out;
+    }
+
+    lines = GF_CALLOC(1, N * sizeof(*lines), gf_gld_mt_charptr);
+    if (!lines) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_NO_MEMORY, NULL);
+        ret = -1;
+        goto out;
+    }
+
+    ret = -1;
+    line_count = 0;
+
+    while ((entry = sys_readdir(hookdir, scratch))) {
+        if (gf_irrelevant_entry(entry))
+            continue;
+        if (line_count == N - 1) {
+            N *= 2;
+            lines = GF_REALLOC(lines, N * sizeof(char *));
+            if (!lines) {
+                gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_NO_MEMORY,
+                        NULL);
                 goto out;
+            }
         }
 
-        hookdir = opendir (hooks_path);
-        if (!hookdir) {
-                ret = -1;
-                gf_log (this->name, GF_LOG_ERROR, "Failed to open dir %s, due "
-                        "to %s", hooks_path, strerror (errno));
-                goto out;
+        if (glusterd_is_hook_enabled(entry->d_name)) {
+            lines[line_count] = gf_strdup(entry->d_name);
+            line_count++;
         }
+    }
 
-        lines = GF_CALLOC (1, N * sizeof (*lines), gf_gld_mt_charptr);
-        if (!lines) {
-                ret = -1;
-                goto out;
-        }
+    lines[line_count] = NULL;
+    lines = GF_REALLOC(lines, (line_count + 1) * sizeof(char *));
+    if (!lines)
+        goto out;
 
-        ret = -1;
-        line_count = 0;
-        GF_FOR_EACH_ENTRY_IN_DIR (entry, hookdir);
-        while (entry) {
-                if (line_count == N-1) {
-                        N *= 2;
-                        lines = GF_REALLOC (lines, N * sizeof (char *));
-                        if (!lines)
-                                goto out;
-                }
-
-                if (glusterd_is_hook_enabled (entry->d_name)) {
-                        lines[line_count] = gf_strdup (entry->d_name);
-                        line_count++;
-                }
-
-                GF_FOR_EACH_ENTRY_IN_DIR (entry, hookdir);
-        }
+    qsort(lines, line_count, sizeof(*lines), glusterd_compare_lines);
 
-        lines[line_count] = NULL;
-        lines = GF_REALLOC (lines, (line_count + 1) * sizeof (char *));
-        if (!lines)
-                goto out;
+    for (lineno = 0; lineno < line_count; lineno++) {
+        runinit(&runner);
+        runner_argprintf(&runner, "%s/%s", hooks_path, lines[lineno]);
+        /*Add future command line arguments to hook scripts below*/
+        runner_argprintf(&runner, "--volname=%s", volname);
+        ret = glusterd_hooks_add_op_args(&runner, op, op_ctx, type);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_ADD_OP_ARGS_FAIL,
+                   "Failed to add "
+                   "command specific arguments");
+            goto out;
+        }
 
-        qsort (lines, line_count, sizeof (*lines), glusterd_compare_lines);
-
-        for (lineno = 0; lineno < line_count; lineno++) {
-
-                runinit (&runner);
-                runner_argprintf (&runner, "%s/%s", hooks_path, lines[lineno]);
-                /*Add future command line arguments to hook scripts below*/
-                runner_argprintf (&runner, "--volname=%s", volname);
-                ret = glusterd_hooks_add_op_args (&runner, op, op_ctx, type);
-                if (ret) {
-                        gf_log (this->name, GF_LOG_ERROR, "Failed to add "
-                                "command specific arguments");
-                        goto out;
-                }
-
-                ret = runner_run_reuse (&runner);
-                if (ret) {
-                        runner_log (&runner, this->name, GF_LOG_ERROR,
-                                    "Failed to execute script");
-                } else {
-                        runner_log (&runner, this->name, GF_LOG_INFO,
-                                    "Ran script");
-                }
-                runner_end (&runner);
+        ret = runner_run_reuse(&runner);
+        if (ret) {
+            runner_log(&runner, this->name, GF_LOG_ERROR,
+                       "Failed to execute script");
+        } else {
+            runner_log(&runner, this->name, GF_LOG_INFO, "Ran script");
         }
+        runner_end(&runner);
+    }
 
-        ret = 0;
+    ret = 0;
 out:
-        if (lines) {
-                for (lineno = 0; lineno < line_count+1; lineno++)
-                        GF_FREE (lines[lineno]);
+    if (lines) {
+        for (lineno = 0; lineno < line_count + 1; lineno++)
+            GF_FREE(lines[lineno]);
 
-                GF_FREE (lines);
-        }
+        GF_FREE(lines);
+    }
 
-        if (hookdir)
-                closedir (hookdir);
+    if (hookdir)
+        sys_closedir(hookdir);
 
-        return ret;
+    return ret;
 }
 
 int
-glusterd_hooks_post_stub_enqueue (char *scriptdir, glusterd_op_t op,
-                                  dict_t *op_ctx)
+glusterd_hooks_post_stub_enqueue(char *scriptdir, glusterd_op_t op,
+                                 dict_t *op_ctx)
 {
-        int                     ret     = -1;
-        glusterd_hooks_stub_t   *stub   = NULL;
-        glusterd_hooks_private_t *hooks_priv = NULL;
-        glusterd_conf_t         *conf   = NULL;
-
-        conf = THIS->private;
-        hooks_priv = conf->hooks_priv;
-
-        ret = glusterd_hooks_stub_init (&stub, scriptdir, op, op_ctx);
-        if (ret)
-                goto out;
-
-        pthread_mutex_lock (&hooks_priv->mutex);
-        {
-                hooks_priv->waitcount++;
-                cds_list_add_tail (&stub->all_hooks, &hooks_priv->list);
-                pthread_cond_signal (&hooks_priv->cond);
-        }
-        pthread_mutex_unlock (&hooks_priv->mutex);
-
-        ret = 0;
+    int ret = -1;
+    glusterd_hooks_stub_t *stub = NULL;
+    glusterd_hooks_private_t *hooks_priv = NULL;
+    glusterd_conf_t *conf = NULL;
+
+    conf = THIS->private;
+    hooks_priv = conf->hooks_priv;
+
+    ret = glusterd_hooks_stub_init(&stub, scriptdir, op, op_ctx);
+    if (ret)
+        goto out;
+
+    pthread_mutex_lock(&hooks_priv->mutex);
+    {
+        hooks_priv->waitcount++;
+        cds_list_add_tail(&stub->all_hooks, &hooks_priv->list);
+        pthread_cond_signal(&hooks_priv->cond);
+    }
+    pthread_mutex_unlock(&hooks_priv->mutex);
+
+    ret = 0;
 out:
-        return ret;
+    return ret;
 }
 
 int
-glusterd_hooks_stub_init (glusterd_hooks_stub_t **stub, char *scriptdir,
-                          glusterd_op_t op, dict_t *op_ctx)
+glusterd_hooks_stub_init(glusterd_hooks_stub_t **stub, char *scriptdir,
+                         glusterd_op_t op, dict_t *op_ctx)
 {
-        int                     ret             = -1;
-        glusterd_hooks_stub_t   *hooks_stub     = NULL;
-
-        GF_ASSERT (stub);
-        if (!stub)
-                goto out;
-
-        hooks_stub = GF_CALLOC (1, sizeof (*hooks_stub),
-                                gf_gld_mt_hooks_stub_t);
-        if (!hooks_stub)
-                goto out;
-
-        CDS_INIT_LIST_HEAD (&hooks_stub->all_hooks);
-        hooks_stub->op = op;
-        hooks_stub->scriptdir = gf_strdup (scriptdir);
-        if (!hooks_stub->scriptdir)
-                goto out;
-
-        hooks_stub->op_ctx = dict_copy_with_ref (op_ctx, hooks_stub->op_ctx);
-        if (!hooks_stub->op_ctx)
-                goto out;
-
-        *stub = hooks_stub;
-        ret = 0;
+    int ret = -1;
+    glusterd_hooks_stub_t *hooks_stub = NULL;
+
+    xlator_t *this = NULL;
+    this = THIS;
+    GF_ASSERT(this);
+    GF_ASSERT(stub);
+    if (!stub)
+        goto out;
+
+    hooks_stub = GF_CALLOC(1, sizeof(*hooks_stub), gf_gld_mt_hooks_stub_t);
+    if (!hooks_stub) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_NO_MEMORY, NULL);
+        goto out;
+    }
+
+    CDS_INIT_LIST_HEAD(&hooks_stub->all_hooks);
+    hooks_stub->op = op;
+    hooks_stub->scriptdir = gf_strdup(scriptdir);
+    if (!hooks_stub->scriptdir) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_STRDUP_FAILED,
+                "scriptdir=%s", scriptdir, NULL);
+        goto out;
+    }
+
+    hooks_stub->op_ctx = dict_copy_with_ref(op_ctx, hooks_stub->op_ctx);
+    if (!hooks_stub->op_ctx) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_COPY_FAIL, NULL);
+        goto out;
+    }
+
+    *stub = hooks_stub;
+    ret = 0;
 out:
-        if (ret) {
-                gf_log (THIS->name, GF_LOG_ERROR, "Failed to initialize "
-                        "post hooks stub");
-                glusterd_hooks_stub_cleanup (hooks_stub);
-        }
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_ERROR, 0, GD_MSG_POST_HOOK_STUB_INIT_FAIL,
+                NULL);
+        glusterd_hooks_stub_cleanup(hooks_stub);
+    }
 
-        return ret;
+    return ret;
 }
 
 void
-glusterd_hooks_stub_cleanup (glusterd_hooks_stub_t *stub)
+glusterd_hooks_stub_cleanup(glusterd_hooks_stub_t *stub)
 {
-        if (!stub) {
-                gf_log_callingfn (THIS->name, GF_LOG_WARNING,
-                                  "hooks_stub is NULL");
-                return;
-        }
+    if (!stub) {
+        gf_msg_callingfn(THIS->name, GF_LOG_WARNING, 0, GD_MSG_HOOK_STUB_NULL,
+                         "hooks_stub is NULL");
+        return;
+    }
 
-        if (stub->op_ctx)
-                dict_unref (stub->op_ctx);
+    if (stub->op_ctx)
+        dict_unref(stub->op_ctx);
 
-        GF_FREE (stub->scriptdir);
+    GF_FREE(stub->scriptdir);
 
-        GF_FREE (stub);
+    GF_FREE(stub);
 }
 
-static void*
-hooks_worker (void *args)
+static void *
+hooks_worker(void *args)
 {
-        glusterd_conf_t *conf = NULL;
-        glusterd_hooks_private_t *hooks_priv = NULL;
-        glusterd_hooks_stub_t *stub = NULL;
-
-        THIS = args;
-        conf = THIS->private;
-        hooks_priv = conf->hooks_priv;
-
-        for (;;) {
-                pthread_mutex_lock (&hooks_priv->mutex);
-                {
-                        while (cds_list_empty (&hooks_priv->list)) {
-                                pthread_cond_wait (&hooks_priv->cond,
-                                                   &hooks_priv->mutex);
-                        }
-                        stub = cds_list_entry (hooks_priv->list.next,
-                                               glusterd_hooks_stub_t,
-                                               all_hooks);
-                        cds_list_del_init (&stub->all_hooks);
-                        hooks_priv->waitcount--;
-
-                }
-                pthread_mutex_unlock (&hooks_priv->mutex);
-
-                glusterd_hooks_run_hooks (stub->scriptdir, stub->op,
-                                          stub->op_ctx, GD_COMMIT_HOOK_POST);
-                glusterd_hooks_stub_cleanup (stub);
+    glusterd_conf_t *conf = NULL;
+    glusterd_hooks_private_t *hooks_priv = NULL;
+    glusterd_hooks_stub_t *stub = NULL;
+
+    THIS = args;
+    conf = THIS->private;
+    hooks_priv = conf->hooks_priv;
+
+    for (;;) {
+        pthread_mutex_lock(&hooks_priv->mutex);
+        {
+            while (cds_list_empty(&hooks_priv->list)) {
+                pthread_cond_wait(&hooks_priv->cond, &hooks_priv->mutex);
+            }
+            stub = cds_list_entry(hooks_priv->list.next, glusterd_hooks_stub_t,
+                                  all_hooks);
+            cds_list_del_init(&stub->all_hooks);
+            hooks_priv->waitcount--;
         }
+        pthread_mutex_unlock(&hooks_priv->mutex);
+
+        glusterd_hooks_run_hooks(stub->scriptdir, stub->op, stub->op_ctx,
+                                 GD_COMMIT_HOOK_POST);
+        glusterd_hooks_stub_cleanup(stub);
+    }
 
-        return NULL;
+    return NULL;
 }
 
 int
-glusterd_hooks_priv_init (glusterd_hooks_private_t **new)
+glusterd_hooks_priv_init(glusterd_hooks_private_t **new)
 {
-        int                      ret            = -1;
-        glusterd_hooks_private_t *hooks_priv    = NULL;
-
-        if (!new)
-                goto out;
-
-        hooks_priv = GF_CALLOC (1, sizeof (*hooks_priv),
-                                gf_gld_mt_hooks_priv_t);
-        if (!hooks_priv)
-                goto out;
-
-        pthread_mutex_init (&hooks_priv->mutex, NULL);
-        pthread_cond_init (&hooks_priv->cond, NULL);
-        CDS_INIT_LIST_HEAD (&hooks_priv->list);
-        hooks_priv->waitcount = 0;
-
-        *new = hooks_priv;
-        ret = 0;
+    int ret = -1;
+    glusterd_hooks_private_t *hooks_priv = NULL;
+
+    xlator_t *this = NULL;
+    this = THIS;
+    GF_ASSERT(this);
+
+    if (!new) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_INVALID_ARGUMENT, NULL);
+        goto out;
+    }
+
+    hooks_priv = GF_CALLOC(1, sizeof(*hooks_priv), gf_gld_mt_hooks_priv_t);
+    if (!hooks_priv) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_NO_MEMORY, NULL);
+        goto out;
+    }
+
+    pthread_mutex_init(&hooks_priv->mutex, NULL);
+    pthread_cond_init(&hooks_priv->cond, NULL);
+    CDS_INIT_LIST_HEAD(&hooks_priv->list);
+    hooks_priv->waitcount = 0;
+
+    *new = hooks_priv;
+    ret = 0;
 out:
-        return ret;
+    return ret;
 }
 
 int
-glusterd_hooks_spawn_worker (xlator_t *this)
+glusterd_hooks_spawn_worker(xlator_t *this)
 {
-        int                       ret          = -1;
-        glusterd_conf_t           *conf        = NULL;
-        glusterd_hooks_private_t  *hooks_priv  = NULL;
-
-
-        ret = glusterd_hooks_priv_init (&hooks_priv);
-        if (ret)
-                goto out;
-
-        conf = this->private;
-        conf->hooks_priv = hooks_priv;
-        ret = pthread_create (&hooks_priv->worker, NULL, hooks_worker,
-                              (void *)this);
-        if (ret)
-                gf_log (this->name, GF_LOG_CRITICAL, "Failed to spawn post "
-                        "hooks worker thread");
+    int ret = -1;
+    glusterd_conf_t *conf = NULL;
+    glusterd_hooks_private_t *hooks_priv = NULL;
+
+    ret = glusterd_hooks_priv_init(&hooks_priv);
+    if (ret)
+        goto out;
+
+    conf = this->private;
+    conf->hooks_priv = hooks_priv;
+    ret = gf_thread_create(&hooks_priv->worker, NULL, hooks_worker,
+                           (void *)this, "gdhooks");
+    if (ret)
+        gf_msg(this->name, GF_LOG_CRITICAL, errno, GD_MSG_SPAWN_THREADS_FAIL,
+               "Failed to spawn post "
+               "hooks worker thread");
 out:
-        return ret;
+    return ret;
 }
diff --git a/xlators/mgmt/glusterd/src/glusterd-hooks.h b/xlators/mgmt/glusterd/src/glusterd-hooks.h
index 1e99f3472fb..f8b887b9bd7 100644
--- a/xlators/mgmt/glusterd/src/glusterd-hooks.h
+++ b/xlators/mgmt/glusterd/src/glusterd-hooks.h
@@ -10,80 +10,79 @@
 #ifndef _GLUSTERD_HOOKS_H_
 #define _GLUSTERD_HOOKS_H_
 
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
-
 #include <fnmatch.h>
 
-#define GLUSTERD_GET_HOOKS_DIR(path, version, priv) \
-        snprintf (path, PATH_MAX, "%s/hooks/%d", priv->workdir,\
-                  version);
+#define GLUSTERD_GET_HOOKS_DIR(path, version, priv)                            \
+    do {                                                                       \
+        int32_t len;                                                           \
+        len = snprintf(path, PATH_MAX, "%s/hooks/%d", priv->workdir, version); \
+        if (len < 0) {                                                         \
+            path[0] = 0;                                                       \
+        }                                                                      \
+    } while (0)
 
-#define GLUSTERD_HOOK_VER       1
+#define GLUSTERD_HOOK_VER 1
 
-#define GD_HOOKS_SPECIFIC_KEY   "user.*"
+#define GD_HOOKS_SPECIFIC_KEY "user.*"
 
 typedef enum glusterd_commit_hook_type {
-        GD_COMMIT_HOOK_NONE = 0,
-        GD_COMMIT_HOOK_PRE,
-        GD_COMMIT_HOOK_POST,
-        GD_COMMIT_HOOK_MAX
+    GD_COMMIT_HOOK_NONE = 0,
+    GD_COMMIT_HOOK_PRE,
+    GD_COMMIT_HOOK_POST,
+    GD_COMMIT_HOOK_MAX
 } glusterd_commit_hook_type_t;
 
 typedef struct hooks_private {
-        struct cds_list_head    list;
-        int                     waitcount; //debug purposes
-        pthread_mutex_t         mutex;
-        pthread_cond_t          cond;
-        pthread_t               worker;
+    struct cds_list_head list;
+    pthread_mutex_t mutex;
+    pthread_cond_t cond;
+    pthread_t worker;
+    int waitcount;  // debug purposes
 } glusterd_hooks_private_t;
 
 typedef struct hooks_stub {
-        struct cds_list_head    all_hooks;
-        char                    *scriptdir;
-        glusterd_op_t           op;
-        dict_t                  *op_ctx;
+    struct cds_list_head all_hooks;
+    char *scriptdir;
+    dict_t *op_ctx;
+    glusterd_op_t op;
 
 } glusterd_hooks_stub_t;
 
-
 static inline gf_boolean_t
-is_key_glusterd_hooks_friendly (char *key)
+is_key_glusterd_hooks_friendly(char *key)
 {
-        gf_boolean_t is_friendly = _gf_false;
+    gf_boolean_t is_friendly = _gf_false;
 
-        /* This is very specific to hooks friendly behavior */
-        if (fnmatch (GD_HOOKS_SPECIFIC_KEY, key, FNM_NOESCAPE) == 0) {
-                gf_log (THIS->name, GF_LOG_DEBUG, "user namespace key %s", key);
-                is_friendly = _gf_true;
-        }
+    /* This is very specific to hooks friendly behavior */
+    if (fnmatch(GD_HOOKS_SPECIFIC_KEY, key, FNM_NOESCAPE) == 0) {
+        gf_msg_debug(THIS->name, 0, "user namespace key %s", key);
+        is_friendly = _gf_true;
+    }
 
-        return is_friendly;
+    return is_friendly;
 }
 
 int
-glusterd_hooks_create_hooks_directory (char *basedir);
+glusterd_hooks_create_hooks_directory(char *basedir);
 
 char *
-glusterd_hooks_get_hooks_cmd_subdir (glusterd_op_t op);
+glusterd_hooks_get_hooks_cmd_subdir(glusterd_op_t op);
 
 int
-glusterd_hooks_run_hooks (char *hooks_path, glusterd_op_t op, dict_t *op_ctx,
-                          glusterd_commit_hook_type_t type);
+glusterd_hooks_run_hooks(char *hooks_path, glusterd_op_t op, dict_t *op_ctx,
+                         glusterd_commit_hook_type_t type);
 int
-glusterd_hooks_spawn_worker (xlator_t *this);
+glusterd_hooks_spawn_worker(xlator_t *this);
 
 int
-glusterd_hooks_stub_init (glusterd_hooks_stub_t **stub, char *scriptdir,
-                          glusterd_op_t op, dict_t *op_ctx);
+glusterd_hooks_stub_init(glusterd_hooks_stub_t **stub, char *scriptdir,
+                         glusterd_op_t op, dict_t *op_ctx);
 void
-glusterd_hooks_stub_cleanup (glusterd_hooks_stub_t *stub);
+glusterd_hooks_stub_cleanup(glusterd_hooks_stub_t *stub);
 
 int
-glusterd_hooks_post_stub_enqueue (char *scriptdir, glusterd_op_t op,
-                                      dict_t *op_ctx);
+glusterd_hooks_post_stub_enqueue(char *scriptdir, glusterd_op_t op,
+                                 dict_t *op_ctx);
 int
-glusterd_hooks_priv_init (glusterd_hooks_private_t **new);
+glusterd_hooks_priv_init(glusterd_hooks_private_t **new);
 #endif
diff --git a/xlators/mgmt/glusterd/src/glusterd-locks.c b/xlators/mgmt/glusterd/src/glusterd-locks.c
index 0703777bdcb..11523f2854b 100644
--- a/xlators/mgmt/glusterd/src/glusterd-locks.c
+++ b/xlators/mgmt/glusterd/src/glusterd-locks.c
@@ -7,12 +7,7 @@
    later), or the GNU General Public License, version 2 (GPLv2), in all
    cases as published by the Free Software Foundation.
 */
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
-
-#include "common-utils.h"
+#include <glusterfs/common-utils.h>
 #include "cli1-xdr.h"
 #include "xdr-generic.h"
 #include "glusterd.h"
@@ -21,167 +16,206 @@
 #include "glusterd-utils.h"
 #include "glusterd-volgen.h"
 #include "glusterd-locks.h"
-#include "run.h"
-#include "syscall.h"
+#include "glusterd-errno.h"
+#include <glusterfs/run.h>
+#include <glusterfs/syscall.h>
+#include "glusterd-messages.h"
 
 #include <signal.h>
 
-#define GF_MAX_LOCKING_ENTITIES 2
+#define GF_MAX_LOCKING_ENTITIES 3
 
 /* Valid entities that the mgmt_v3 lock can hold locks upon    *
  * To add newer entities to be locked, we can just add more    *
  * entries to this table along with the type and default value */
-glusterd_valid_entities   valid_types[] = {
-        { "vol",  _gf_true  },
-        { "snap", _gf_false },
-        { NULL              },
+glusterd_valid_entities valid_types[] = {
+    {"vol", _gf_true},
+    {"snap", _gf_false},
+    {"global", _gf_false},
+    {NULL},
 };
 
 /* Checks if the lock request is for a valid entity */
-gf_boolean_t
-glusterd_mgmt_v3_is_type_valid (char *type)
+static gf_boolean_t
+glusterd_mgmt_v3_is_type_valid(char *type)
 {
-        int32_t         i   = 0;
-        gf_boolean_t    ret = _gf_false;
+    int i = 0;
 
-        GF_ASSERT (type);
+    GF_ASSERT(type);
 
-        for (i = 0; valid_types[i].type; i++) {
-                if (!strcmp (type, valid_types[i].type)) {
-                        ret = _gf_true;
-                        break;
-                }
+    for (i = 0; valid_types[i].type; i++) {
+        if (!strcmp(type, valid_types[i].type)) {
+            return _gf_true;
         }
+    }
 
-        return ret;
+    return _gf_false;
 }
 
 /* Initialize the global mgmt_v3 lock list(dict) when
  * glusterd is spawned */
 int32_t
-glusterd_mgmt_v3_lock_init ()
+glusterd_mgmt_v3_lock_init()
 {
-        int32_t             ret = -1;
-        xlator_t           *this   = NULL;
-        glusterd_conf_t    *priv   = NULL;
+    int32_t ret = -1;
+    xlator_t *this = NULL;
+    glusterd_conf_t *priv = NULL;
 
-        this = THIS;
-        GF_ASSERT (this);
-        priv = this->private;
-        GF_ASSERT (priv);
+    this = THIS;
+    GF_ASSERT(this);
+    priv = this->private;
+    GF_ASSERT(priv);
 
-        priv->mgmt_v3_lock = dict_new ();
-        if (!priv->mgmt_v3_lock)
-                goto out;
+    priv->mgmt_v3_lock = dict_new();
+    if (!priv->mgmt_v3_lock)
+        goto out;
 
-        ret = 0;
+    ret = 0;
 out:
-        return ret;
+    return ret;
 }
 
 /* Destroy the global mgmt_v3 lock list(dict) when
  * glusterd cleanup is performed */
 void
-glusterd_mgmt_v3_lock_fini ()
+glusterd_mgmt_v3_lock_fini()
 {
-        xlator_t           *this   = NULL;
-        glusterd_conf_t    *priv   = NULL;
+    xlator_t *this = NULL;
+    glusterd_conf_t *priv = NULL;
 
-        this = THIS;
-        GF_ASSERT (this);
-        priv = this->private;
-        GF_ASSERT (priv);
+    this = THIS;
+    GF_ASSERT(this);
+    priv = this->private;
+    GF_ASSERT(priv);
 
-        if (priv->mgmt_v3_lock)
-                dict_unref (priv->mgmt_v3_lock);
+    if (priv->mgmt_v3_lock)
+        dict_unref(priv->mgmt_v3_lock);
 }
 
+/* Initialize the global mgmt_v3_timer lock list(dict) when
+ * glusterd is spawned */
 int32_t
-glusterd_get_mgmt_v3_lock_owner (char *key, uuid_t *uuid)
+glusterd_mgmt_v3_lock_timer_init()
 {
-        int32_t                         ret      = -1;
-        glusterd_mgmt_v3_lock_obj      *lock_obj = NULL;
-        glusterd_conf_t                *priv     = NULL;
-        uuid_t                          no_owner = {0,};
-        xlator_t                       *this     = NULL;
-
-        this = THIS;
-        GF_ASSERT (this);
-        priv = this->private;
-        GF_ASSERT (priv);
-
-        if (!key || !uuid) {
-                gf_log (this->name, GF_LOG_ERROR, "key or uuid is null.");
-                ret = -1;
-                goto out;
-        }
+    int32_t ret = -1;
+    xlator_t *this = NULL;
+    glusterd_conf_t *priv = NULL;
 
-        ret = dict_get_bin (priv->mgmt_v3_lock, key, (void **) &lock_obj);
-        if (!ret)
-                gf_uuid_copy (*uuid, lock_obj->lock_owner);
-        else
-                gf_uuid_copy (*uuid, no_owner);
+    this = THIS;
+    GF_VALIDATE_OR_GOTO("glusterd", this, out);
 
-        ret = 0;
+    priv = this->private;
+    GF_VALIDATE_OR_GOTO(this->name, priv, out);
+
+    priv->mgmt_v3_lock_timer = dict_new();
+    if (!priv->mgmt_v3_lock_timer)
+        goto out;
+
+    ret = 0;
 out:
-        gf_log (this->name, GF_LOG_TRACE, "Returning %d", ret);
-        return ret;
+    return ret;
+}
+
+/* Destroy the global mgmt_v3_timer lock list(dict) when
+ * glusterd cleanup is performed */
+void
+glusterd_mgmt_v3_lock_timer_fini()
+{
+    xlator_t *this = NULL;
+    glusterd_conf_t *priv = NULL;
+
+    this = THIS;
+    GF_VALIDATE_OR_GOTO("glusterd", this, out);
+
+    priv = this->private;
+    GF_VALIDATE_OR_GOTO(this->name, priv, out);
+
+    if (priv->mgmt_v3_lock_timer)
+        dict_unref(priv->mgmt_v3_lock_timer);
+out:
+    return;
+}
+
+static int32_t
+glusterd_get_mgmt_v3_lock_owner(char *key, uuid_t *uuid)
+{
+    int32_t ret = -1;
+    glusterd_mgmt_v3_lock_obj *lock_obj = NULL;
+    glusterd_conf_t *priv = NULL;
+    xlator_t *this = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+    priv = this->private;
+    GF_ASSERT(priv);
+
+    if (!key || !uuid) {
+        gf_msg(this->name, GF_LOG_ERROR, EINVAL, GD_MSG_INVALID_ENTRY,
+               "key or uuid is null.");
+        ret = -1;
+        goto out;
+    }
+
+    ret = dict_get_bin(priv->mgmt_v3_lock, key, (void **)&lock_obj);
+    if (!ret)
+        gf_uuid_copy(*uuid, lock_obj->lock_owner);
+
+    ret = 0;
+out:
+    gf_msg_trace(this->name, 0, "Returning %d", ret);
+    return ret;
 }
 
 /* This function is called with the locked_count and type, to   *
  * release all the acquired locks. */
 static int32_t
-glusterd_release_multiple_locks_per_entity (dict_t *dict, uuid_t uuid,
-                                            int32_t locked_count,
-                                            char *type)
+glusterd_release_multiple_locks_per_entity(dict_t *dict, uuid_t uuid,
+                                           int32_t locked_count, char *type)
 {
-        char           name_buf[PATH_MAX]    = "";
-        char          *name                  = NULL;
-        int32_t        i                     = -1;
-        int32_t        op_ret                = 0;
-        int32_t        ret                   = -1;
-        xlator_t      *this                  = NULL;
-
-        this = THIS;
-        GF_ASSERT(this);
-        GF_ASSERT (dict);
-        GF_ASSERT (type);
-
-        if (locked_count == 0) {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "No %s locked as part of this transaction",
-                        type);
-                goto out;
+    char name_buf[PATH_MAX] = "";
+    char *name = NULL;
+    int32_t i = -1;
+    int32_t op_ret = 0;
+    int32_t ret = -1;
+    xlator_t *this = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+    GF_ASSERT(dict);
+    GF_ASSERT(type);
+
+    if (locked_count == 0) {
+        gf_msg_debug(this->name, 0, "No %s locked as part of this transaction",
+                     type);
+        goto out;
+    }
+
+    /* Release all the locks held */
+    for (i = 0; i < locked_count; i++) {
+        ret = snprintf(name_buf, sizeof(name_buf), "%sname%d", type, i + 1);
+
+        /* Looking for volname1, volname2 or snapname1, *
+         * as key in the dict snapname2 */
+        ret = dict_get_strn(dict, name_buf, ret, &name);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+                   "Unable to get %s locked_count = %d", name_buf,
+                   locked_count);
+            op_ret = ret;
+            continue;
         }
 
-        /* Release all the locks held */
-        for (i = 0; i < locked_count; i++) {
-                snprintf (name_buf, sizeof(name_buf),
-                          "%sname%d", type, i+1);
-
-                /* Looking for volname1, volname2 or snapname1, *
-                 * as key in the dict snapname2 */
-                ret = dict_get_str (dict, name_buf, &name);
-                if (ret) {
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "Unable to get %s locked_count = %d",
-                                name_buf, locked_count);
-                        op_ret = ret;
-                        continue;
-                }
-
-                ret = glusterd_mgmt_v3_unlock (name, uuid, type);
-                if (ret) {
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "Failed to release lock for %s.",
-                                name);
-                        op_ret = ret;
-                }
+        ret = glusterd_mgmt_v3_unlock(name, uuid, type);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_MGMTV3_UNLOCK_FAIL,
+                   "Failed to release lock for %s.", name);
+            op_ret = ret;
         }
+    }
 
 out:
-        gf_log (this->name, GF_LOG_TRACE, "Returning %d", op_ret);
-        return op_ret;
+    gf_msg_trace(this->name, 0, "Returning %d", op_ret);
+    return op_ret;
 }
 
 /* Given the count and type of the entity this function acquires     *
@@ -189,67 +223,64 @@ out:
  * If type is "vol" this function tries to acquire locks on multiple *
  * volumes */
 static int32_t
-glusterd_acquire_multiple_locks_per_entity (dict_t *dict, uuid_t uuid,
-                                            int32_t count, char *type)
+glusterd_acquire_multiple_locks_per_entity(dict_t *dict, uuid_t uuid,
+                                           uint32_t *op_errno, int32_t count,
+                                           char *type)
 {
-        char           name_buf[PATH_MAX]    = "";
-        char          *name                  = NULL;
-        int32_t        i                     = -1;
-        int32_t        ret                   = -1;
-        int32_t        locked_count          = 0;
-        xlator_t      *this                  = NULL;
-
-        this = THIS;
-        GF_ASSERT(this);
-        GF_ASSERT (dict);
-        GF_ASSERT (type);
-
-        /* Locking one element after other */
-        for (i = 0; i < count; i++) {
-                snprintf (name_buf, sizeof(name_buf),
-                          "%sname%d", type, i+1);
-
-                /* Looking for volname1, volname2 or snapname1, *
-                 * as key in the dict snapname2 */
-                ret = dict_get_str (dict, name_buf, &name);
-                if (ret) {
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "Unable to get %s count = %d",
-                                name_buf, count);
-                        break;
-                }
-
-                ret = glusterd_mgmt_v3_lock (name, uuid, type);
-                if (ret) {
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "Failed to acquire lock for %s %s "
-                                "on behalf of %s. Reversing "
-                                "this transaction", type, name,
-                                uuid_utoa(uuid));
-                        break;
-                }
-                locked_count++;
-        }
-
-        if (count == locked_count) {
-                /* If all locking ops went successfuly, return as success */
-                ret = 0;
-                goto out;
+    char name_buf[PATH_MAX] = "";
+    char *name = NULL;
+    int32_t i = -1;
+    int32_t ret = -1;
+    int32_t locked_count = 0;
+    xlator_t *this = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+    GF_ASSERT(dict);
+    GF_ASSERT(type);
+
+    /* Locking one element after other */
+    for (i = 0; i < count; i++) {
+        ret = snprintf(name_buf, sizeof(name_buf), "%sname%d", type, i + 1);
+
+        /* Looking for volname1, volname2 or snapname1, *
+         * as key in the dict snapname2 */
+        ret = dict_get_strn(dict, name_buf, ret, &name);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+                   "Unable to get %s count = %d", name_buf, count);
+            break;
         }
 
-        /* If we failed to lock one element, unlock others and return failure */
-        ret = glusterd_release_multiple_locks_per_entity (dict, uuid,
-                                                          locked_count,
-                                                          type);
+        ret = glusterd_mgmt_v3_lock(name, uuid, op_errno, type);
         if (ret) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "Failed to release multiple %s locks",
-                        type);
-        }
-        ret = -1;
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_MGMTV3_LOCK_GET_FAIL,
+                   "Failed to acquire lock for %s %s "
+                   "on behalf of %s. Reversing "
+                   "this transaction",
+                   type, name, uuid_utoa(uuid));
+            break;
+        }
+        locked_count++;
+    }
+
+    if (count == locked_count) {
+        /* If all locking ops went successfully, return as success */
+        ret = 0;
+        goto out;
+    }
+
+    /* If we failed to lock one element, unlock others and return failure */
+    ret = glusterd_release_multiple_locks_per_entity(dict, uuid, locked_count,
+                                                     type);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_MULTIPLE_LOCK_RELEASE_FAIL,
+               "Failed to release multiple %s locks", type);
+    }
+    ret = -1;
 out:
-        gf_log (this->name, GF_LOG_TRACE, "Returning %d", ret);
-        return ret;
+    gf_msg_trace(this->name, 0, "Returning %d", ret);
+    return ret;
 }
 
 /* Given the type of entity, this function figures out if it should unlock a   *
@@ -257,71 +288,69 @@ out:
  * if the type is "vol", this function will accordingly unlock a single volume *
  * or multiple volumes */
 static int32_t
-glusterd_mgmt_v3_unlock_entity (dict_t *dict, uuid_t uuid, char *type,
-                                gf_boolean_t default_value)
+glusterd_mgmt_v3_unlock_entity(dict_t *dict, uuid_t uuid, char *type,
+                               gf_boolean_t default_value)
 {
-        char           name_buf[PATH_MAX]    = "";
-        char          *name                  = NULL;
-        int32_t        count                 = -1;
-        int32_t        ret                   = -1;
-        gf_boolean_t   hold_locks            = _gf_false;
-        xlator_t      *this                  = NULL;
-
-        this = THIS;
-        GF_ASSERT(this);
-        GF_ASSERT (dict);
-        GF_ASSERT (type);
-
-        snprintf (name_buf, sizeof(name_buf), "hold_%s_locks", type);
-        hold_locks = dict_get_str_boolean (dict, name_buf, default_value);
-
-        if (hold_locks == _gf_false) {
-                /* Locks were not held for this particular entity *
-                 * Hence nothing to release */
-                ret = 0;
-                goto out;
+    char name_buf[PATH_MAX] = "";
+    char *name = NULL;
+    int32_t count = -1;
+    int32_t ret = -1;
+    gf_boolean_t hold_locks = _gf_false;
+    xlator_t *this = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+    GF_ASSERT(dict);
+    GF_ASSERT(type);
+
+    snprintf(name_buf, sizeof(name_buf), "hold_%s_locks", type);
+    hold_locks = dict_get_str_boolean(dict, name_buf, default_value);
+
+    if (hold_locks == _gf_false) {
+        /* Locks were not held for this particular entity *
+         * Hence nothing to release */
+        ret = 0;
+        goto out;
+    }
+
+    /* Looking for volcount or snapcount in the dict */
+    ret = snprintf(name_buf, sizeof(name_buf), "%scount", type);
+    ret = dict_get_int32n(dict, name_buf, ret, &count);
+    if (ret) {
+        /* count is not present. Only one *
+         * element name needs to be unlocked */
+        ret = snprintf(name_buf, sizeof(name_buf), "%sname", type);
+        ret = dict_get_strn(dict, name_buf, ret, &name);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+                   "Unable to fetch %sname", type);
+            goto out;
         }
 
-        /* Looking for volcount or snapcount in the dict */
-        snprintf (name_buf, sizeof(name_buf), "%scount", type);
-        ret = dict_get_int32 (dict, name_buf, &count);
+        ret = glusterd_mgmt_v3_unlock(name, uuid, type);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_MGMTV3_UNLOCK_FAIL,
+                   "Failed to release lock for %s %s "
+                   "on behalf of %s.",
+                   type, name, uuid_utoa(uuid));
+            goto out;
+        }
+    } else {
+        /* Unlocking one element name after another */
+        ret = glusterd_release_multiple_locks_per_entity(dict, uuid, count,
+                                                         type);
         if (ret) {
-                /* count is not present. Only one *
-                 * element name needs to be unlocked */
-                snprintf (name_buf, sizeof(name_buf), "%sname",
-                          type);
-                ret = dict_get_str (dict, name_buf, &name);
-                if (ret) {
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "Unable to fetch %sname", type);
-                        goto out;
-                }
-
-                ret = glusterd_mgmt_v3_unlock (name, uuid, type);
-                if (ret) {
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "Failed to release lock for %s %s "
-                                "on behalf of %s.", type, name,
-                                uuid_utoa(uuid));
-                        goto out;
-                }
-        } else {
-                /* Unlocking one element name after another */
-                ret = glusterd_release_multiple_locks_per_entity (dict,
-                                                                  uuid,
-                                                                  count,
-                                                                  type);
-                if (ret) {
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "Failed to release all %s locks", type);
-                        goto out;
-                }
+            gf_msg(this->name, GF_LOG_ERROR, 0,
+                   GD_MSG_MULTIPLE_LOCK_RELEASE_FAIL,
+                   "Failed to release all %s locks", type);
+            goto out;
         }
+    }
 
-        ret = 0;
+    ret = 0;
 out:
-        gf_log (this->name, GF_LOG_TRACE, "Returning %d", ret);
-        return ret;
+    gf_msg_trace(this->name, 0, "Returning %d", ret);
+    return ret;
 }
 
 /* Given the type of entity, this function figures out if it should lock a   *
@@ -329,356 +358,513 @@ out:
  * if the type is "vol", this function will accordingly lock a single volume *
  * or multiple volumes */
 static int32_t
-glusterd_mgmt_v3_lock_entity (dict_t *dict, uuid_t uuid, char *type,
-                              gf_boolean_t default_value)
+glusterd_mgmt_v3_lock_entity(dict_t *dict, uuid_t uuid, uint32_t *op_errno,
+                             char *type, gf_boolean_t default_value)
 {
-        char           name_buf[PATH_MAX]    = "";
-        char          *name                  = NULL;
-        int32_t        count                 = -1;
-        int32_t        ret                   = -1;
-        gf_boolean_t   hold_locks            = _gf_false;
-        xlator_t      *this                  = NULL;
-
-        this = THIS;
-        GF_ASSERT(this);
-        GF_ASSERT (dict);
-        GF_ASSERT (type);
-
-        snprintf (name_buf, sizeof(name_buf), "hold_%s_locks", type);
-        hold_locks = dict_get_str_boolean (dict, name_buf, default_value);
-
-        if (hold_locks == _gf_false) {
-                /* Not holding locks for this particular entity */
-                ret = 0;
-                goto out;
+    char name_buf[PATH_MAX] = "";
+    char *name = NULL;
+    int32_t count = -1;
+    int32_t ret = -1;
+    gf_boolean_t hold_locks = _gf_false;
+    xlator_t *this = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+    GF_ASSERT(dict);
+    GF_ASSERT(type);
+
+    snprintf(name_buf, sizeof(name_buf), "hold_%s_locks", type);
+    hold_locks = dict_get_str_boolean(dict, name_buf, default_value);
+
+    if (hold_locks == _gf_false) {
+        /* Not holding locks for this particular entity */
+        ret = 0;
+        goto out;
+    }
+
+    /* Looking for volcount or snapcount in the dict */
+    ret = snprintf(name_buf, sizeof(name_buf), "%scount", type);
+    ret = dict_get_int32n(dict, name_buf, ret, &count);
+    if (ret) {
+        /* count is not present. Only one *
+         * element name needs to be locked */
+        ret = snprintf(name_buf, sizeof(name_buf), "%sname", type);
+        ret = dict_get_strn(dict, name_buf, ret, &name);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+                   "Unable to fetch %sname", type);
+            goto out;
         }
 
-        /* Looking for volcount or snapcount in the dict */
-        snprintf (name_buf, sizeof(name_buf), "%scount", type);
-        ret = dict_get_int32 (dict, name_buf, &count);
+        ret = glusterd_mgmt_v3_lock(name, uuid, op_errno, type);
         if (ret) {
-                /* count is not present. Only one *
-                 * element name needs to be locked */
-                snprintf (name_buf, sizeof(name_buf), "%sname",
-                          type);
-                ret = dict_get_str (dict, name_buf, &name);
-                if (ret) {
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "Unable to fetch %sname", type);
-                        goto out;
-                }
-
-                ret = glusterd_mgmt_v3_lock (name, uuid, type);
-                if (ret) {
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "Failed to acquire lock for %s %s "
-                                "on behalf of %s.", type, name,
-                                uuid_utoa(uuid));
-                        goto out;
-                }
-        } else {
-                /* Locking one element name after another */
-                ret = glusterd_acquire_multiple_locks_per_entity (dict,
-                                                                  uuid,
-                                                                  count,
-                                                                  type);
-                if (ret) {
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "Failed to acquire all %s locks", type);
-                        goto out;
-                }
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_MGMTV3_LOCK_GET_FAIL,
+                   "Failed to acquire lock for %s %s "
+                   "on behalf of %s.",
+                   type, name, uuid_utoa(uuid));
+            goto out;
+        }
+    } else {
+        /* Locking one element name after another */
+        ret = glusterd_acquire_multiple_locks_per_entity(dict, uuid, op_errno,
+                                                         count, type);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, 0,
+                   GD_MSG_MULTIPLE_LOCK_ACQUIRE_FAIL,
+                   "Failed to acquire all %s locks", type);
+            goto out;
         }
+    }
 
-        ret = 0;
+    ret = 0;
 out:
-        gf_log (this->name, GF_LOG_TRACE, "Returning %d", ret);
-        return ret;
+    gf_msg_trace(this->name, 0, "Returning %d", ret);
+    return ret;
 }
 
 /* Try to release locks of multiple entities like *
  * volume, snaps etc. */
 int32_t
-glusterd_multiple_mgmt_v3_unlock (dict_t *dict, uuid_t uuid)
+glusterd_multiple_mgmt_v3_unlock(dict_t *dict, uuid_t uuid)
 {
-        int32_t        i                     = -1;
-        int32_t        ret                   = -1;
-        int32_t        op_ret                = 0;
-        xlator_t      *this                  = NULL;
-
-        this = THIS;
-        GF_ASSERT(this);
-
-        if (!dict) {
-                gf_log (this->name, GF_LOG_ERROR, "dict is null.");
-                ret = -1;
-                goto out;
-        }
+    int32_t i = -1;
+    int32_t ret = -1;
+    int32_t op_ret = 0;
+    xlator_t *this = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+
+    if (!dict) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_EMPTY, "dict is null.");
+        ret = -1;
+        goto out;
+    }
 
-        for (i = 0; valid_types[i].type; i++) {
-                ret = glusterd_mgmt_v3_unlock_entity
-                                            (dict, uuid,
-                                             valid_types[i].type,
+    for (i = 0; valid_types[i].type; i++) {
+        ret = glusterd_mgmt_v3_unlock_entity(dict, uuid, valid_types[i].type,
                                              valid_types[i].default_value);
-                if (ret) {
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "Unable to unlock all %s",
-                                valid_types[i].type);
-                        op_ret = ret;
-                }
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, 0,
+                   GD_MSG_MULTIPLE_LOCK_RELEASE_FAIL, "Unable to unlock all %s",
+                   valid_types[i].type);
+            op_ret = ret;
         }
+    }
 
-        ret = op_ret;
+    ret = op_ret;
 out:
-        gf_log (this->name, GF_LOG_DEBUG, "Returning %d", ret);
-        return ret;
+    gf_msg_debug(this->name, 0, "Returning %d", ret);
+    return ret;
 }
 
 /* Try to acquire locks on multiple entities like *
  * volume, snaps etc. */
 int32_t
-glusterd_multiple_mgmt_v3_lock (dict_t *dict, uuid_t uuid)
+glusterd_multiple_mgmt_v3_lock(dict_t *dict, uuid_t uuid, uint32_t *op_errno)
 {
-        int32_t        i                     = -1;
-        int32_t        ret                   = -1;
-        int32_t        locked_count          = 0;
-        xlator_t      *this                  = NULL;
-
-        this = THIS;
-        GF_ASSERT(this);
-
-        if (!dict) {
-                gf_log (this->name, GF_LOG_ERROR, "dict is null.");
-                ret = -1;
-                goto out;
-        }
+    int32_t i = -1;
+    int32_t ret = -1;
+    int32_t locked_count = 0;
+    xlator_t *this = NULL;
 
-        /* Locking one entity after other */
-        for (i = 0; valid_types[i].type; i++) {
-                ret = glusterd_mgmt_v3_lock_entity
-                                            (dict, uuid,
-                                             valid_types[i].type,
-                                             valid_types[i].default_value);
-                if (ret) {
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "Unable to lock all %s",
-                                valid_types[i].type);
-                        break;
-                }
-                locked_count++;
-        }
+    this = THIS;
+    GF_ASSERT(this);
 
-        if (locked_count == GF_MAX_LOCKING_ENTITIES) {
-                /* If all locking ops went successfuly, return as success */
-                ret = 0;
-                goto out;
+    if (!dict) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_EMPTY, "dict is null.");
+        ret = -1;
+        goto out;
+    }
+
+    /* Locking one entity after other */
+    for (i = 0; valid_types[i].type; i++) {
+        ret = glusterd_mgmt_v3_lock_entity(dict, uuid, op_errno,
+                                           valid_types[i].type,
+                                           valid_types[i].default_value);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, 0,
+                   GD_MSG_MULTIPLE_LOCK_ACQUIRE_FAIL, "Unable to lock all %s",
+                   valid_types[i].type);
+            break;
         }
+        locked_count++;
+    }
 
-        /* If we failed to lock one entity, unlock others and return failure */
-        for (i = 0; i < locked_count; i++) {
-                ret = glusterd_mgmt_v3_unlock_entity
-                                              (dict, uuid,
-                                               valid_types[i].type,
-                                               valid_types[i].default_value);
-                if (ret) {
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "Unable to unlock all %s",
-                                valid_types[i].type);
-                }
+    if (locked_count == GF_MAX_LOCKING_ENTITIES) {
+        /* If all locking ops went successfully, return as success */
+        ret = 0;
+        goto out;
+    }
+
+    /* If we failed to lock one entity, unlock others and return failure */
+    for (i = 0; i < locked_count; i++) {
+        ret = glusterd_mgmt_v3_unlock_entity(dict, uuid, valid_types[i].type,
+                                             valid_types[i].default_value);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, 0,
+                   GD_MSG_MULTIPLE_LOCK_RELEASE_FAIL, "Unable to unlock all %s",
+                   valid_types[i].type);
         }
-        ret = -1;
+    }
+    ret = -1;
 out:
-        gf_log (this->name, GF_LOG_DEBUG, "Returning %d", ret);
-        return ret;
+    gf_msg_debug(this->name, 0, "Returning %d", ret);
+    return ret;
 }
 
-
 int32_t
-glusterd_mgmt_v3_lock (const char *name, uuid_t uuid, char *type)
+glusterd_mgmt_v3_lock(const char *name, uuid_t uuid, uint32_t *op_errno,
+                      char *type)
 {
-        char                            key[PATH_MAX]   = "";
-        int32_t                         ret             = -1;
-        glusterd_mgmt_v3_lock_obj      *lock_obj        = NULL;
-        glusterd_conf_t                *priv            = NULL;
-        gf_boolean_t                    is_valid        = _gf_true;
-        uuid_t                          owner           = {0};
-        xlator_t                       *this            = NULL;
-        char                           *bt              = NULL;
-
-        this = THIS;
-        GF_ASSERT (this);
-        priv = this->private;
-        GF_ASSERT (priv);
-
-        if (!name || !type) {
-                gf_log (this->name, GF_LOG_ERROR, "name or type is null.");
-                ret = -1;
-                goto out;
-        }
-
-        is_valid = glusterd_mgmt_v3_is_type_valid (type);
-        if (is_valid != _gf_true) {
-                gf_log_callingfn (this->name, GF_LOG_ERROR,
-                        "Invalid entity. Cannot perform locking "
-                        "operation on %s types", type);
-                ret = -1;
-                goto out;
-        }
+    char key[PATH_MAX] = "";
+    int32_t ret = -1;
+    glusterd_mgmt_v3_lock_obj *lock_obj = NULL;
+    glusterd_mgmt_v3_lock_timer *mgmt_lock_timer = NULL;
+    glusterd_conf_t *priv = NULL;
+    gf_boolean_t is_valid = _gf_true;
+    uuid_t owner = {0};
+    xlator_t *this = NULL;
+    char *bt = NULL;
+    struct timespec delay = {0};
+    char *key_dup = NULL;
+    glusterfs_ctx_t *mgmt_lock_timer_ctx = NULL;
+    xlator_t *mgmt_lock_timer_xl = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+
+    priv = this->private;
+    GF_ASSERT(priv);
+
+    if (!name || !type) {
+        gf_msg(this->name, GF_LOG_ERROR, EINVAL, GD_MSG_INVALID_ENTRY,
+               "name or type is null.");
+        ret = -1;
+        goto out;
+    }
+
+    is_valid = glusterd_mgmt_v3_is_type_valid(type);
+    if (is_valid != _gf_true) {
+        gf_msg_callingfn(this->name, GF_LOG_ERROR, EINVAL, GD_MSG_INVALID_ENTRY,
+                         "Invalid entity. Cannot perform locking "
+                         "operation on %s types",
+                         type);
+        ret = -1;
+        goto out;
+    }
 
-        ret = snprintf (key, sizeof(key), "%s_%s", name, type);
-        if (ret != strlen(name) + 1 + strlen(type)) {
-                ret = -1;
-                gf_log (this->name, GF_LOG_ERROR, "Unable to create key");
-                goto out;
-        }
+    ret = snprintf(key, sizeof(key), "%s_%s", name, type);
+    if (ret != strlen(name) + 1 + strlen(type)) {
+        ret = -1;
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_CREATE_KEY_FAIL,
+               "Unable to create key");
+        goto out;
+    }
+
+    gf_msg_debug(this->name, 0, "Trying to acquire lock of %s for %s", key,
+                 uuid_utoa(uuid));
+
+    ret = glusterd_get_mgmt_v3_lock_owner(key, &owner);
+    if (ret) {
+        gf_msg_debug(this->name, 0, "Unable to get mgmt_v3 lock owner");
+        goto out;
+    }
+
+    /* If the lock has already been held for the given volume
+     * we fail */
+    if (!gf_uuid_is_null(owner)) {
+        gf_msg_callingfn(this->name, GF_LOG_WARNING, 0,
+                         GD_MSG_LOCK_ALREADY_HELD, "Lock for %s held by %s",
+                         name, uuid_utoa(owner));
+        ret = -1;
+        *op_errno = EG_ANOTRANS;
+        goto out;
+    }
 
-        gf_log (this->name, GF_LOG_DEBUG,
-                "Trying to acquire lock of %s %s for %s as %s",
-                type, name, uuid_utoa (uuid), key);
+    lock_obj = GF_MALLOC(sizeof(glusterd_mgmt_v3_lock_obj),
+                         gf_common_mt_mgmt_v3_lock_obj_t);
+    if (!lock_obj) {
+        ret = -1;
+        goto out;
+    }
 
-        ret = glusterd_get_mgmt_v3_lock_owner (key, &owner);
-        if (ret) {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "Unable to get mgmt_v3 lock owner");
-                goto out;
-        }
+    gf_uuid_copy(lock_obj->lock_owner, uuid);
 
-        /* If the lock has already been held for the given volume
-         * we fail */
-        if (!gf_uuid_is_null (owner)) {
-                gf_log_callingfn (this->name, GF_LOG_WARNING,
-                                  "Lock for %s held by %s",
-                                  name, uuid_utoa (owner));
-                ret = -1;
-                goto out;
-        }
+    ret = dict_set_bin(priv->mgmt_v3_lock, key, lock_obj,
+                       sizeof(glusterd_mgmt_v3_lock_obj));
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+               "Unable to set lock owner in mgmt_v3 lock");
+        GF_FREE(lock_obj);
+        goto out;
+    }
 
-        lock_obj = GF_CALLOC (1, sizeof(glusterd_mgmt_v3_lock_obj),
-                              gf_common_mt_mgmt_v3_lock_obj_t);
-        if (!lock_obj) {
-                ret = -1;
-                goto out;
-        }
+    mgmt_lock_timer = GF_CALLOC(1, sizeof(glusterd_mgmt_v3_lock_timer),
+                                gf_common_mt_mgmt_v3_lock_timer_t);
 
-        gf_uuid_copy (lock_obj->lock_owner, uuid);
+    if (!mgmt_lock_timer) {
+        ret = -1;
+        goto out;
+    }
+
+    mgmt_lock_timer->xl = THIS;
+    /*changing to default timeout value*/
+    priv->mgmt_v3_lock_timeout = GF_LOCK_TIMER;
+
+    ret = -1;
+    mgmt_lock_timer_xl = mgmt_lock_timer->xl;
+    if (!mgmt_lock_timer_xl) {
+        GF_FREE(mgmt_lock_timer);
+        goto out;
+    }
+
+    mgmt_lock_timer_ctx = mgmt_lock_timer_xl->ctx;
+    if (!mgmt_lock_timer_ctx) {
+        GF_FREE(mgmt_lock_timer);
+        goto out;
+    }
+
+    key_dup = gf_strdup(key);
+    delay.tv_sec = priv->mgmt_v3_lock_timeout;
+    delay.tv_nsec = 0;
+
+    mgmt_lock_timer->timer = gf_timer_call_after(
+        mgmt_lock_timer_ctx, delay, gd_mgmt_v3_unlock_timer_cbk, key_dup);
+
+    ret = dict_set_bin(priv->mgmt_v3_lock_timer, key, mgmt_lock_timer,
+                       sizeof(glusterd_mgmt_v3_lock_timer));
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+               "Unable to set timer in mgmt_v3 lock");
+        GF_FREE(key_dup);
+        GF_FREE(mgmt_lock_timer);
+        goto out;
+    }
+
+    /* Saving the backtrace into the pre-allocated buffer, ctx->btbuf*/
+    if ((bt = gf_backtrace_save(NULL))) {
+        snprintf(key, sizeof(key), "debug.last-success-bt-%s", key_dup);
+        ret = dict_set_dynstr_with_alloc(priv->mgmt_v3_lock, key, bt);
+        if (ret)
+            gf_msg(this->name, GF_LOG_WARNING, 0, GD_MSG_DICT_SET_FAILED,
+                   "Failed to save "
+                   "the back trace for lock %s granted to %s",
+                   key_dup, uuid_utoa(uuid));
+        ret = 0;
+    }
 
-        ret = dict_set_bin (priv->mgmt_v3_lock, key, lock_obj,
-                            sizeof(glusterd_mgmt_v3_lock_obj));
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "Unable to set lock owner in mgmt_v3 lock");
-                if (lock_obj)
-                        GF_FREE (lock_obj);
-                goto out;
-        }
+    gf_msg_debug(this->name, 0, "Lock for %s successfully held by %s", key_dup,
+                 uuid_utoa(uuid));
 
-        /* Saving the backtrace into the pre-allocated buffer, ctx->btbuf*/
-        if ((bt = gf_backtrace_save (NULL))) {
-                snprintf (key, sizeof (key), "debug.last-success-bt-%s-%s",
-                          name, type);
-                ret = dict_set_dynstr_with_alloc (priv->mgmt_v3_lock, key, bt);
-                if (ret)
-                        gf_log (this->name, GF_LOG_WARNING, "Failed to save "
-                                "the back trace for lock %s-%s granted to %s",
-                                name, type, uuid_utoa (uuid));
-                ret = 0;
-        }
+    ret = 0;
+out:
+    gf_msg_trace(this->name, 0, "Returning %d", ret);
+    return ret;
+}
 
-        gf_log (this->name, GF_LOG_DEBUG,
-                "Lock for %s %s successfully held by %s",
-                type, name, uuid_utoa (uuid));
+/*
+ * This call back will ensure to unlock the lock_obj, in case we hit a situation
+ * where unlocking failed and stale lock exist*/
+void
+gd_mgmt_v3_unlock_timer_cbk(void *data)
+{
+    xlator_t *this = NULL;
+    glusterd_conf_t *conf = NULL;
+    glusterd_mgmt_v3_lock_timer *mgmt_lock_timer = NULL;
+    char *key = NULL;
+    int keylen;
+    char bt_key[PATH_MAX] = "";
+    int bt_key_len = 0;
+    int32_t ret = -1;
+    glusterfs_ctx_t *mgmt_lock_timer_ctx = NULL;
+    xlator_t *mgmt_lock_timer_xl = NULL;
+    gf_timer_t *timer = NULL;
+
+    this = THIS;
+    GF_VALIDATE_OR_GOTO("glusterd", this, out);
+
+    conf = this->private;
+    GF_VALIDATE_OR_GOTO(this->name, conf, out);
+
+    GF_ASSERT(NULL != data);
+    key = (char *)data;
+
+    keylen = strlen(key);
+    dict_deln(conf->mgmt_v3_lock, key, keylen);
+
+    bt_key_len = snprintf(bt_key, PATH_MAX, "debug.last-success-bt-%s", key);
+    if (bt_key_len != SLEN("debug.last-success-bt-") + keylen) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_CREATE_KEY_FAIL,
+               "Unable to create backtrace "
+               "key");
+        goto out;
+    }
+
+    dict_deln(conf->mgmt_v3_lock, bt_key, bt_key_len);
+
+    ret = dict_get_bin(conf->mgmt_v3_lock_timer, key,
+                       (void **)&mgmt_lock_timer);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+               "Unable to get lock owner in mgmt_v3 lock");
+    }
 
-        ret = 0;
 out:
-        gf_log (this->name, GF_LOG_TRACE, "Returning %d", ret);
-        return ret;
+    if (mgmt_lock_timer && mgmt_lock_timer->timer) {
+        mgmt_lock_timer_xl = mgmt_lock_timer->xl;
+        GF_VALIDATE_OR_GOTO(this->name, mgmt_lock_timer_xl, ret_function);
+
+        mgmt_lock_timer_ctx = mgmt_lock_timer_xl->ctx;
+        GF_VALIDATE_OR_GOTO(this->name, mgmt_lock_timer_ctx, ret_function);
+
+        timer = mgmt_lock_timer->timer;
+        GF_FREE(timer->data);
+        gf_timer_call_cancel(mgmt_lock_timer_ctx, mgmt_lock_timer->timer);
+        dict_deln(conf->mgmt_v3_lock_timer, bt_key, bt_key_len);
+        mgmt_lock_timer->timer = NULL;
+        gf_log(this->name, GF_LOG_INFO,
+               "unlock timer is cancelled for volume_type"
+               " %s",
+               key);
+    }
+
+ret_function:
+
+    return;
 }
 
 int32_t
-glusterd_mgmt_v3_unlock (const char *name, uuid_t uuid, char *type)
+glusterd_mgmt_v3_unlock(const char *name, uuid_t uuid, char *type)
 {
-        char                    key[PATH_MAX]   = "";
-        int32_t                 ret             = -1;
-        gf_boolean_t            is_valid        = _gf_true;
-        glusterd_conf_t        *priv            = NULL;
-        uuid_t                  owner           = {0};
-        xlator_t               *this            = NULL;
-
-        this = THIS;
-        GF_ASSERT (this);
-        priv = this->private;
-        GF_ASSERT (priv);
-
-        if (!name || !type) {
-                gf_log (this->name, GF_LOG_ERROR, "name is null.");
-                ret = -1;
-                goto out;
-        }
-
-        is_valid = glusterd_mgmt_v3_is_type_valid (type);
-        if (is_valid != _gf_true) {
-                gf_log_callingfn (this->name, GF_LOG_ERROR,
-                        "Invalid entity. Cannot perform unlocking "
-                        "operation on %s types", type);
-                ret = -1;
-                goto out;
-        }
-
-        ret = snprintf (key, sizeof(key), "%s_%s",
-                        name, type);
-        if (ret != strlen(name) + 1 + strlen(type)) {
-                gf_log (this->name, GF_LOG_ERROR, "Unable to create key");
-                ret = -1;
-                goto out;
-        }
+    char key[PATH_MAX] = "";
+    char key_dup[PATH_MAX] = "";
+    int keylen;
+    int32_t ret = -1;
+    gf_boolean_t is_valid = _gf_true;
+    glusterd_conf_t *priv = NULL;
+    glusterd_volinfo_t *volinfo = NULL;
+    glusterd_mgmt_v3_lock_timer *mgmt_lock_timer = NULL;
+    uuid_t owner = {0};
+    xlator_t *this = NULL;
+    glusterfs_ctx_t *mgmt_lock_timer_ctx = NULL;
+    xlator_t *mgmt_lock_timer_xl = NULL;
+    gf_timer_t *timer = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+
+    priv = this->private;
+    GF_ASSERT(priv);
+
+    if (!name || !type) {
+        gf_msg(this->name, GF_LOG_ERROR, EINVAL, GD_MSG_INVALID_ENTRY,
+               "name is null.");
+        ret = -1;
+        goto out;
+    }
+
+    is_valid = glusterd_mgmt_v3_is_type_valid(type);
+    if (is_valid != _gf_true) {
+        gf_msg_callingfn(this->name, GF_LOG_ERROR, EINVAL, GD_MSG_INVALID_ENTRY,
+                         "Invalid entity. Cannot perform unlocking "
+                         "operation on %s types",
+                         type);
+        ret = -1;
+        goto out;
+    }
 
-        gf_log (this->name, GF_LOG_DEBUG,
-                "Trying to release lock of %s %s for %s as %s",
-                type, name, uuid_utoa (uuid), key);
+    keylen = snprintf(key, sizeof(key), "%s_%s", name, type);
+    if (keylen != strlen(name) + 1 + strlen(type)) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_CREATE_KEY_FAIL,
+               "Unable to create key");
+        ret = -1;
+        goto out;
+    }
 
-        ret = glusterd_get_mgmt_v3_lock_owner (key, &owner);
-        if (ret) {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "Unable to get mgmt_v3 lock owner");
-                goto out;
-        }
+    gf_msg_debug(this->name, 0, "Trying to release lock of %s %s for %s as %s",
+                 type, name, uuid_utoa(uuid), key);
 
-        if (gf_uuid_is_null (owner)) {
-                gf_log_callingfn (this->name, GF_LOG_WARNING,
-                        "Lock for %s %s not held", type, name);
-                ret = -1;
-                goto out;
-        }
+    ret = glusterd_get_mgmt_v3_lock_owner(key, &owner);
+    if (ret) {
+        gf_msg_debug(this->name, 0, "Unable to get mgmt_v3 lock owner");
+        goto out;
+    }
 
-        ret = gf_uuid_compare (uuid, owner);
-        if (ret) {
-                gf_log_callingfn (this->name, GF_LOG_WARNING,
-                                  "Lock owner mismatch. "
-                                  "Lock for %s %s held by %s",
-                                  type, name, uuid_utoa (owner));
-                goto out;
-        }
+    if (gf_uuid_is_null(owner)) {
+        gf_msg_callingfn(this->name, GF_LOG_WARNING, 0, GD_MSG_LOCK_NOT_HELD,
+                         "Lock for %s %s not held", type, name);
+        ret = -1;
+        goto out;
+    }
+
+    ret = gf_uuid_compare(uuid, owner);
+    if (ret) {
+        gf_msg_callingfn(this->name, GF_LOG_WARNING, 0,
+                         GD_MSG_LOCK_OWNER_MISMATCH,
+                         "Lock owner mismatch. "
+                         "Lock for %s %s held by %s",
+                         type, name, uuid_utoa(owner));
+        goto out;
+    }
+
+    /* Removing the mgmt_v3 lock from the global list */
+    dict_deln(priv->mgmt_v3_lock, key, keylen);
+
+    ret = dict_get_bin(priv->mgmt_v3_lock_timer, key,
+                       (void **)&mgmt_lock_timer);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+               "Unable to get mgmt lock key in mgmt_v3 lock");
+        goto out;
+    }
+
+    (void)snprintf(key_dup, sizeof(key_dup), "%s", key);
+
+    /* Remove the backtrace key as well */
+    ret = snprintf(key, sizeof(key), "debug.last-success-bt-%s", key_dup);
+    if (ret != SLEN("debug.last-success-bt-") + keylen) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_CREATE_KEY_FAIL,
+               "Unable to create backtrace "
+               "key");
+        ret = -1;
+        goto out;
+    }
+    dict_deln(priv->mgmt_v3_lock, key, ret);
 
-        /* Removing the mgmt_v3 lock from the global list */
-        dict_del (priv->mgmt_v3_lock, key);
-
-        /* Remove the backtrace key as well */
-        ret = snprintf (key, sizeof(key), "debug.last-success-bt-%s-%s", name,
-                        type);
-        if (ret != strlen ("debug.last-success-bt-") + strlen (name) +
-                   strlen (type) + 1) {
-                gf_log (this->name, GF_LOG_ERROR, "Unable to create backtrace "
-                        "key");
-                ret = -1;
-                goto out;
-        }
-        dict_del (priv->mgmt_v3_lock, key);
+    gf_msg_debug(this->name, 0, "Lock for %s %s successfully released", type,
+                 name);
 
-        gf_log (this->name, GF_LOG_DEBUG,
-                "Lock for %s %s successfully released",
-                type, name);
+    /* Release owner reference which was held during lock */
+    if (mgmt_lock_timer && mgmt_lock_timer->timer) {
+        ret = -1;
+        mgmt_lock_timer_xl = mgmt_lock_timer->xl;
+        GF_VALIDATE_OR_GOTO(this->name, mgmt_lock_timer_xl, out);
 
+        mgmt_lock_timer_ctx = mgmt_lock_timer_xl->ctx;
+        GF_VALIDATE_OR_GOTO(this->name, mgmt_lock_timer_ctx, out);
         ret = 0;
+
+        timer = mgmt_lock_timer->timer;
+        GF_FREE(timer->data);
+        gf_timer_call_cancel(mgmt_lock_timer_ctx, mgmt_lock_timer->timer);
+        dict_deln(priv->mgmt_v3_lock_timer, key_dup, keylen);
+    }
+    ret = glusterd_volinfo_find(name, &volinfo);
+    if (volinfo && volinfo->stage_deleted) {
+        /* this indicates a volume still exists and the volume delete
+         * operation has failed in some of the phases, need to ensure
+         * stage_deleted flag is set back to false
+         */
+        volinfo->stage_deleted = _gf_false;
+        gf_log(this->name, GF_LOG_INFO,
+               "Volume %s still exist, setting "
+               "stage deleted flag to false for the volume",
+               volinfo->volname);
+    }
+    ret = 0;
 out:
-        gf_log (this->name, GF_LOG_TRACE, "Returning %d", ret);
-        return ret;
+
+    gf_msg_trace(this->name, 0, "Returning %d", ret);
+    return ret;
 }
diff --git a/xlators/mgmt/glusterd/src/glusterd-locks.h b/xlators/mgmt/glusterd/src/glusterd-locks.h
index b9cc8c0d1e4..44667cebd3d 100644
--- a/xlators/mgmt/glusterd/src/glusterd-locks.h
+++ b/xlators/mgmt/glusterd/src/glusterd-locks.h
@@ -10,42 +10,48 @@
 #ifndef _GLUSTERD_LOCKS_H_
 #define _GLUSTERD_LOCKS_H_
 
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
-
 typedef struct glusterd_mgmt_v3_lock_object_ {
-        uuid_t              lock_owner;
+    uuid_t lock_owner;
 } glusterd_mgmt_v3_lock_obj;
 
+typedef struct glusterd_mgmt_v3_lock_timer_ {
+    gf_timer_t *timer;
+    xlator_t *xl;
+} glusterd_mgmt_v3_lock_timer;
+
 typedef struct glusterd_mgmt_v3_lock_valid_entities {
-        char          *type;          /* Entity type like vol, snap */
-        gf_boolean_t   default_value; /* The default value that  *
-                                       * determines if the locks *
-                                       * should be held for that *
-                                       * entity */
+    char *type;                 /* Entity type like vol, snap */
+    gf_boolean_t default_value; /* The default value that  *
+                                 * determines if the locks *
+                                 * should be held for that *
+                                 * entity */
 } glusterd_valid_entities;
 
 int32_t
-glusterd_mgmt_v3_lock_init ();
+glusterd_mgmt_v3_lock_init();
 
 void
-glusterd_mgmt_v3_lock_fini ();
+glusterd_mgmt_v3_lock_fini();
 
 int32_t
-glusterd_get_mgmt_v3_lock_owner (char *volname, uuid_t *uuid);
+glusterd_mgmt_v3_lock_timer_init();
+
+void
+glusterd_mgmt_v3_lock_timer_fini();
 
 int32_t
-glusterd_mgmt_v3_lock (const char *key, uuid_t uuid, char *type);
+glusterd_mgmt_v3_lock(const char *key, uuid_t uuid, uint32_t *op_errno,
+                      char *type);
 
 int32_t
-glusterd_mgmt_v3_unlock (const char *key, uuid_t uuid, char *type);
+glusterd_mgmt_v3_unlock(const char *key, uuid_t uuid, char *type);
 
 int32_t
-glusterd_multiple_mgmt_v3_lock (dict_t *dict, uuid_t uuid);
+glusterd_multiple_mgmt_v3_lock(dict_t *dict, uuid_t uuid, uint32_t *op_errno);
 
 int32_t
-glusterd_multiple_mgmt_v3_unlock (dict_t *dict, uuid_t uuid);
+glusterd_multiple_mgmt_v3_unlock(dict_t *dict, uuid_t uuid);
 
+void
+gd_mgmt_v3_unlock_timer_cbk(void *data);
 #endif
diff --git a/xlators/mgmt/glusterd/src/glusterd-log-ops.c b/xlators/mgmt/glusterd/src/glusterd-log-ops.c
index 449b28cdbf1..34abf35cb00 100644
--- a/xlators/mgmt/glusterd/src/glusterd-log-ops.c
+++ b/xlators/mgmt/glusterd/src/glusterd-log-ops.c
@@ -7,12 +7,7 @@
    later), or the GNU General Public License, version 2 (GPLv2), in all
    cases as published by the Free Software Foundation.
 */
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
-
-#include "common-utils.h"
+#include <glusterfs/common-utils.h>
 #include "cli1-xdr.h"
 #include "xdr-generic.h"
 #include "glusterd.h"
@@ -20,252 +15,276 @@
 #include "glusterd-store.h"
 #include "glusterd-utils.h"
 #include "glusterd-volgen.h"
+#include "glusterd-messages.h"
+#include <glusterfs/syscall.h>
 
 #include <signal.h>
 
 int
-__glusterd_handle_log_rotate (rpcsvc_request_t *req)
+__glusterd_handle_log_rotate(rpcsvc_request_t *req)
 {
-        int32_t                 ret     = -1;
-        gf_cli_req              cli_req = {{0,}};
-        dict_t                 *dict    = NULL;
-        glusterd_op_t           cli_op = GD_OP_LOG_ROTATE;
-        char                   *volname = NULL;
-        char                   msg[2048] = {0,};
-        xlator_t               *this = NULL;
-
-        GF_ASSERT (req);
-        this = THIS;
-        GF_ASSERT (this);
-
-        ret = xdr_to_generic (req->msg[0], &cli_req, (xdrproc_t)xdr_gf_cli_req);
+    int32_t ret = -1;
+    gf_cli_req cli_req = {{
+        0,
+    }};
+    dict_t *dict = NULL;
+    glusterd_op_t cli_op = GD_OP_LOG_ROTATE;
+    char *volname = NULL;
+    char msg[64] = {
+        0,
+    };
+    xlator_t *this = NULL;
+
+    GF_ASSERT(req);
+    this = THIS;
+    GF_ASSERT(this);
+
+    ret = xdr_to_generic(req->msg[0], &cli_req, (xdrproc_t)xdr_gf_cli_req);
+    if (ret < 0) {
+        // failed to decode msg;
+        req->rpc_err = GARBAGE_ARGS;
+        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_GARBAGE_ARGS, NULL);
+        goto out;
+    }
+
+    if (cli_req.dict.dict_len) {
+        /* Unserialize the dictionary */
+        dict = dict_new();
+
+        ret = dict_unserialize(cli_req.dict.dict_val, cli_req.dict.dict_len,
+                               &dict);
         if (ret < 0) {
-                //failed to decode msg;
-                req->rpc_err = GARBAGE_ARGS;
-                goto out;
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_UNSERIALIZE_FAIL,
+                   "failed to "
+                   "unserialize req-buffer to dictionary");
+            snprintf(msg, sizeof(msg),
+                     "Unable to decode the "
+                     "command");
+            goto out;
         }
+    }
 
-        if (cli_req.dict.dict_len) {
-                /* Unserialize the dictionary */
-                dict  = dict_new ();
-
-                ret = dict_unserialize (cli_req.dict.dict_val,
-                                        cli_req.dict.dict_len,
-                                        &dict);
-                if (ret < 0) {
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "failed to "
-                                "unserialize req-buffer to dictionary");
-                        snprintf (msg, sizeof (msg), "Unable to decode the "
-                                  "command");
-                        goto out;
-                }
-        }
+    ret = dict_get_str(dict, "volname", &volname);
+    if (ret) {
+        snprintf(msg, sizeof(msg), "Failed to get volume name");
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED, "%s", msg);
+        goto out;
+    }
 
-        ret = dict_get_str (dict, "volname", &volname);
-        if (ret) {
-                snprintf (msg, sizeof (msg), "Failed to get volume name");
-                gf_log (this->name, GF_LOG_ERROR, "%s", msg);
-                goto out;
-        }
+    gf_msg(this->name, GF_LOG_INFO, 0, GD_MSG_LOG_ROTATE_REQ_RECVD,
+           "Received log rotate req "
+           "for volume %s",
+           volname);
 
-        gf_log (this->name, GF_LOG_INFO, "Received log rotate req "
-                "for volume %s", volname);
+    ret = dict_set_uint64(dict, "rotate-key", (uint64_t)gf_time());
+    if (ret)
+        goto out;
 
-        ret = dict_set_uint64 (dict, "rotate-key", (uint64_t)time (NULL));
-        if (ret)
-                goto out;
-
-        ret = glusterd_op_begin_synctask (req, GD_OP_LOG_ROTATE, dict);
+    ret = glusterd_op_begin_synctask(req, GD_OP_LOG_ROTATE, dict);
 
 out:
-        if (ret) {
-                if (msg[0] == '\0')
-                        snprintf (msg, sizeof (msg), "Operation failed");
-                ret = glusterd_op_send_cli_response (cli_op, ret, 0, req,
-                                                     dict, msg);
-        }
-
-        free (cli_req.dict.dict_val);
-        return ret;
+    if (ret) {
+        if (msg[0] == '\0')
+            snprintf(msg, sizeof(msg), "Operation failed");
+        ret = glusterd_op_send_cli_response(cli_op, ret, 0, req, dict, msg);
+    }
+
+    free(cli_req.dict.dict_val);
+    return ret;
 }
 
 int
-glusterd_handle_log_rotate (rpcsvc_request_t *req)
+glusterd_handle_log_rotate(rpcsvc_request_t *req)
 {
-        return glusterd_big_locked_handler (req,
-                                            __glusterd_handle_log_rotate);
+    return glusterd_big_locked_handler(req, __glusterd_handle_log_rotate);
 }
 
 /* op-sm */
 int
-glusterd_op_stage_log_rotate (dict_t *dict, char **op_errstr)
+glusterd_op_stage_log_rotate(dict_t *dict, char **op_errstr)
 {
-        int                                     ret = -1;
-        char                                    *volname = NULL;
-        glusterd_volinfo_t                      *volinfo = NULL;
-        gf_boolean_t                            exists = _gf_false;
-        char                                    msg[2048] = {0};
-        char                                    *brick = NULL;
-
-        ret = dict_get_str (dict, "volname", &volname);
-        if (ret) {
-                gf_log ("", GF_LOG_ERROR, "Unable to get volume name");
-                goto out;
-        }
-
-        exists = glusterd_check_volume_exists (volname);
-        ret = glusterd_volinfo_find (volname, &volinfo);
-        if (!exists) {
-                snprintf (msg, sizeof (msg), "Volume %s does not exist",
-                          volname);
-                gf_log ("", GF_LOG_ERROR, "%s", msg);
-                *op_errstr = gf_strdup (msg);
-                ret = -1;
-                goto out;
-        }
-
-        if (_gf_false == glusterd_is_volume_started (volinfo)) {
-                snprintf (msg, sizeof (msg), "Volume %s needs to be started before"
-                          " log rotate.", volname);
-                gf_log ("", GF_LOG_ERROR, "%s", msg);
-                *op_errstr = gf_strdup (msg);
-                ret = -1;
-                goto out;
-        }
-
-        ret = dict_get_str (dict, "brick", &brick);
-        /* If no brick is specified, do log-rotate for
-           all the bricks in the volume */
-        if (ret) {
-                ret = 0;
-                goto out;
-        }
-
-        ret = glusterd_volume_brickinfo_get_by_brick (brick, volinfo, NULL);
-        if (ret) {
-                snprintf (msg, sizeof (msg), "Incorrect brick %s "
-                          "for volume %s", brick, volname);
-                gf_log ("", GF_LOG_ERROR, "%s", msg);
-                *op_errstr = gf_strdup (msg);
-                goto out;
-        }
+    int ret = -1;
+    char *volname = NULL;
+    glusterd_volinfo_t *volinfo = NULL;
+    char msg[2048] = {0};
+    char *brick = NULL;
+
+    ret = dict_get_str(dict, "volname", &volname);
+    if (ret) {
+        gf_msg("glusterd", GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+               "Unable to get volume name");
+        goto out;
+    }
+
+    ret = glusterd_volinfo_find(volname, &volinfo);
+    if (ret) {
+        snprintf(msg, sizeof(msg), "Volume %s does not exist", volname);
+        gf_msg("glusterd", GF_LOG_ERROR, 0, GD_MSG_VOL_NOT_FOUND, "%s", msg);
+        *op_errstr = gf_strdup(msg);
+        goto out;
+    }
+
+    if (_gf_false == glusterd_is_volume_started(volinfo)) {
+        snprintf(msg, sizeof(msg),
+                 "Volume %s needs to be started before"
+                 " log rotate.",
+                 volname);
+        gf_msg("glusterd", GF_LOG_ERROR, 0, GD_MSG_VOL_NOT_STARTED, "%s", msg);
+        *op_errstr = gf_strdup(msg);
+        ret = -1;
+        goto out;
+    }
+
+    ret = dict_get_str(dict, "brick", &brick);
+    /* If no brick is specified, do log-rotate for
+       all the bricks in the volume */
+    if (ret) {
+        gf_smsg("glusterd", GF_LOG_ERROR, errno, GD_MSG_DICT_GET_FAILED,
+                "Key=brick", NULL);
+        ret = 0;
+        goto out;
+    }
+
+    ret = glusterd_volume_brickinfo_get_by_brick(brick, volinfo, NULL,
+                                                 _gf_false);
+    if (ret) {
+        snprintf(msg, sizeof(msg),
+                 "Incorrect brick %s "
+                 "for volume %s",
+                 brick, volname);
+        gf_msg("glusterd", GF_LOG_ERROR, EINVAL, GD_MSG_INVALID_ENTRY, "%s",
+               msg);
+        *op_errstr = gf_strdup(msg);
+        goto out;
+    }
 out:
-        gf_log ("", GF_LOG_DEBUG, "Returning %d", ret);
+    gf_msg_debug("glusterd", 0, "Returning %d", ret);
 
-        return ret;
+    return ret;
 }
 
-
 int
-glusterd_op_log_rotate (dict_t *dict)
+glusterd_op_log_rotate(dict_t *dict)
 {
-        int                   ret                = -1;
-        glusterd_conf_t      *priv               = NULL;
-        glusterd_volinfo_t   *volinfo            = NULL;
-        glusterd_brickinfo_t *brickinfo          = NULL;
-        xlator_t             *this               = NULL;
-        char                 *volname            = NULL;
-        char                 *brick              = NULL;
-        char                  logfile[PATH_MAX]  = {0,};
-        char                  pidfile[PATH_MAX]  = {0,};
-        FILE                 *file               = NULL;
-        pid_t                 pid                = 0;
-        uint64_t              key                = 0;
-        int                   valid_brick        = 0;
-        glusterd_brickinfo_t *tmpbrkinfo         = NULL;
-
-        this = THIS;
-        GF_ASSERT (this);
-        priv = this->private;
-        GF_ASSERT (priv);
-
-        ret = dict_get_str (dict, "volname", &volname);
-        if (ret) {
-                gf_log ("", GF_LOG_ERROR, "volname not found");
-                goto out;
+    int ret = -1;
+    glusterd_conf_t *priv = NULL;
+    glusterd_volinfo_t *volinfo = NULL;
+    glusterd_brickinfo_t *brickinfo = NULL;
+    xlator_t *this = NULL;
+    char *volname = NULL;
+    char *brick = NULL;
+    char logfile[PATH_MAX] = {
+        0,
+    };
+    char pidfile[PATH_MAX] = {
+        0,
+    };
+    FILE *file = NULL;
+    pid_t pid = 0;
+    uint64_t key = 0;
+    int valid_brick = 0;
+    glusterd_brickinfo_t *tmpbrkinfo = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+    priv = this->private;
+    GF_ASSERT(priv);
+
+    ret = dict_get_str(dict, "volname", &volname);
+    if (ret) {
+        gf_msg("glusterd", GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+               "volname not found");
+        goto out;
+    }
+
+    ret = dict_get_uint64(dict, "rotate-key", &key);
+    if (ret) {
+        gf_msg("glusterd", GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+               "rotate key not found");
+        goto out;
+    }
+
+    ret = dict_get_str(dict, "brick", &brick);
+    /* If no brick is specified, do log-rotate for
+       all the bricks in the volume */
+    if (ret) {
+        gf_smsg("glusterd", GF_LOG_ERROR, errno, GD_MSG_DICT_GET_FAILED,
+                "Key=brick", NULL);
+        goto cont;
+    }
+
+    ret = glusterd_brickinfo_new_from_brick(brick, &tmpbrkinfo, _gf_false,
+                                            NULL);
+    if (ret) {
+        gf_msg("glusterd", GF_LOG_ERROR, 0, GD_MSG_BRICK_NOT_FOUND,
+               "cannot get brickinfo from brick");
+        goto out;
+    }
+
+cont:
+    ret = glusterd_volinfo_find(volname, &volinfo);
+    if (ret)
+        goto out;
+
+    ret = -1;
+    cds_list_for_each_entry(brickinfo, &volinfo->bricks, brick_list)
+    {
+        if (gf_uuid_compare(brickinfo->uuid, MY_UUID))
+            continue;
+
+        if (tmpbrkinfo && brick &&
+            (strcmp(tmpbrkinfo->hostname, brickinfo->hostname) ||
+             strcmp(tmpbrkinfo->path, brickinfo->path)))
+            continue;
+
+        valid_brick = 1;
+
+        GLUSTERD_GET_BRICK_PIDFILE(pidfile, volinfo, brickinfo, priv);
+        file = fopen(pidfile, "r+");
+        if (!file) {
+            gf_msg("glusterd", GF_LOG_ERROR, errno, GD_MSG_FILE_OP_FAILED,
+                   "Unable to open pidfile: %s", pidfile);
+            ret = -1;
+            goto out;
         }
 
-        ret = dict_get_uint64 (dict, "rotate-key", &key);
-        if (ret) {
-                gf_log ("", GF_LOG_ERROR, "rotate key not found");
-                goto out;
+        ret = fscanf(file, "%d", &pid);
+        if (ret <= 0) {
+            fclose(file);
+            gf_msg("glusterd", GF_LOG_ERROR, errno, GD_MSG_FILE_OP_FAILED,
+                   "Unable to read pidfile: %s", pidfile);
+            ret = -1;
+            goto out;
         }
+        fclose(file);
+        file = NULL;
 
-        ret = dict_get_str (dict, "brick", &brick);
-        /* If no brick is specified, do log-rotate for
-           all the bricks in the volume */
+        snprintf(logfile, PATH_MAX, "%s.%" PRIu64, brickinfo->logfile, key);
+
+        ret = sys_rename(brickinfo->logfile, logfile);
         if (ret)
-                goto cont;
+            gf_msg("glusterd", GF_LOG_WARNING, errno, GD_MSG_FILE_OP_FAILED,
+                   "rename failed");
 
-        ret = glusterd_brickinfo_new_from_brick (brick, &tmpbrkinfo);
+        ret = kill(pid, SIGHUP);
         if (ret) {
-                gf_log ("glusterd", GF_LOG_ERROR,
-                        "cannot get brickinfo from brick");
-                goto out;
+            gf_msg("glusterd", GF_LOG_ERROR, errno, GD_MSG_PID_KILL_FAIL,
+                   "Unable to SIGHUP to %d", pid);
+            goto out;
         }
+        ret = 0;
 
-cont:
-        ret = glusterd_volinfo_find (volname, &volinfo);
-        if (ret)
-                goto out;
-
-        ret = -1;
-        cds_list_for_each_entry (brickinfo, &volinfo->bricks, brick_list) {
-                if (gf_uuid_compare (brickinfo->uuid, MY_UUID))
-                        continue;
-
-                if (brick &&
-                    (strcmp (tmpbrkinfo->hostname, brickinfo->hostname) ||
-                     strcmp (tmpbrkinfo->path,brickinfo->path)))
-                        continue;
-
-                valid_brick = 1;
-
-                GLUSTERD_GET_BRICK_PIDFILE (pidfile, volinfo, brickinfo, priv);
-                file = fopen (pidfile, "r+");
-                if (!file) {
-                        gf_log ("", GF_LOG_ERROR, "Unable to open pidfile: %s",
-                                pidfile);
-                        ret = -1;
-                        goto out;
-                }
-
-                ret = fscanf (file, "%d", &pid);
-                if (ret <= 0) {
-                        gf_log ("", GF_LOG_ERROR, "Unable to read pidfile: %s",
-                                pidfile);
-                        ret = -1;
-                        goto out;
-                }
-                fclose (file);
-                file = NULL;
-
-                snprintf (logfile, PATH_MAX, "%s.%"PRIu64,
-                          brickinfo->logfile, key);
-
-                ret = rename (brickinfo->logfile, logfile);
-                if (ret)
-                        gf_log ("", GF_LOG_WARNING, "rename failed");
-
-                ret = kill (pid, SIGHUP);
-                if (ret) {
-                        gf_log ("", GF_LOG_ERROR, "Unable to SIGHUP to %d", pid);
-                        goto out;
-                }
-                ret = 0;
-
-                /* If request was for brick, only one iteration is enough */
-                if (brick)
-                        break;
-        }
+        /* If request was for brick, only one iteration is enough */
+        if (brick)
+            break;
+    }
 
-        if (ret && !valid_brick)
-                ret = 0;
+    if (ret && !valid_brick)
+        ret = 0;
 
 out:
-        if (tmpbrkinfo)
-                glusterd_brickinfo_delete (tmpbrkinfo);
+    if (tmpbrkinfo)
+        glusterd_brickinfo_delete(tmpbrkinfo);
 
-        return ret;
+    return ret;
 }
diff --git a/xlators/mgmt/glusterd/src/glusterd-mem-types.h b/xlators/mgmt/glusterd/src/glusterd-mem-types.h
index ed171b69b66..d7257e1a7b5 100644
--- a/xlators/mgmt/glusterd/src/glusterd-mem-types.h
+++ b/xlators/mgmt/glusterd/src/glusterd-mem-types.h
@@ -11,67 +11,48 @@
 #ifndef __GLUSTERD_MEM_TYPES_H__
 #define __GLUSTERD_MEM_TYPES_H__
 
-#include "mem-types.h"
+#include <glusterfs/mem-types.h>
 
 typedef enum gf_gld_mem_types_ {
-        gf_gld_mt_dir_entry_t                   = gf_common_mt_end + 1,
-        gf_gld_mt_volfile_ctx                   = gf_common_mt_end + 2,
-        gf_gld_mt_glusterd_state_t              = gf_common_mt_end + 3,
-        gf_gld_mt_glusterd_conf_t               = gf_common_mt_end + 4,
-        gf_gld_mt_locker                        = gf_common_mt_end + 5,
-        gf_gld_mt_string                        = gf_common_mt_end + 6,
-        gf_gld_mt_lock_table                    = gf_common_mt_end + 7,
-        gf_gld_mt_char                          = gf_common_mt_end + 8,
-        gf_gld_mt_glusterd_connection_t         = gf_common_mt_end + 9,
-        gf_gld_mt_resolve_comp                  = gf_common_mt_end + 10,
-        gf_gld_mt_peerinfo_t                    = gf_common_mt_end + 11,
-        gf_gld_mt_friend_sm_event_t             = gf_common_mt_end + 12,
-        gf_gld_mt_friend_req_ctx_t              = gf_common_mt_end + 13,
-        gf_gld_mt_friend_update_ctx_t           = gf_common_mt_end + 14,
-        gf_gld_mt_op_sm_event_t                 = gf_common_mt_end + 15,
-        gf_gld_mt_op_lock_ctx_t                 = gf_common_mt_end + 16,
-        gf_gld_mt_op_stage_ctx_t                = gf_common_mt_end + 17,
-        gf_gld_mt_op_commit_ctx_t               = gf_common_mt_end + 18,
-        gf_gld_mt_mop_stage_req_t               = gf_common_mt_end + 19,
-        gf_gld_mt_probe_ctx_t                   = gf_common_mt_end + 20,
-        gf_gld_mt_create_volume_ctx_t           = gf_common_mt_end + 21,
-        gf_gld_mt_start_volume_ctx_t            = gf_common_mt_end + 22,
-        gf_gld_mt_stop_volume_ctx_t             = gf_common_mt_end + 23,
-        gf_gld_mt_delete_volume_ctx_t           = gf_common_mt_end + 24,
-        gf_gld_mt_glusterd_volinfo_t            = gf_common_mt_end + 25,
-        gf_gld_mt_glusterd_brickinfo_t          = gf_common_mt_end + 26,
-        gf_gld_mt_peer_hostname_t               = gf_common_mt_end + 27,
-        gf_gld_mt_ifreq                         = gf_common_mt_end + 28,
-        gf_gld_mt_store_handle_t                = gf_common_mt_end + 29,
-        gf_gld_mt_store_iter_t                  = gf_common_mt_end + 30,
-        gf_gld_mt_defrag_info                   = gf_common_mt_end + 31,
-        gf_gld_mt_log_filename_ctx_t            = gf_common_mt_end + 32,
-        gf_gld_mt_log_locate_ctx_t              = gf_common_mt_end + 33,
-        gf_gld_mt_log_rotate_ctx_t              = gf_common_mt_end + 34,
-        gf_gld_mt_peerctx_t                     = gf_common_mt_end + 35,
-        gf_gld_mt_sm_tr_log_t                   = gf_common_mt_end + 36,
-        gf_gld_mt_pending_node_t                = gf_common_mt_end + 37,
-        gf_gld_mt_brick_rsp_ctx_t               = gf_common_mt_end + 38,
-        gf_gld_mt_mop_brick_req_t               = gf_common_mt_end + 39,
-        gf_gld_mt_op_allack_ctx_t               = gf_common_mt_end + 40,
-        gf_gld_mt_linearr                       = gf_common_mt_end + 41,
-        gf_gld_mt_linebuf                       = gf_common_mt_end + 42,
-        gf_gld_mt_mount_pattern                 = gf_common_mt_end + 43,
-        gf_gld_mt_mount_comp_container          = gf_common_mt_end + 44,
-        gf_gld_mt_mount_component               = gf_common_mt_end + 45,
-        gf_gld_mt_mount_spec                    = gf_common_mt_end + 46,
-        gf_gld_mt_georep_meet_spec              = gf_common_mt_end + 47,
-        gf_gld_mt_nodesrv_t                     = gf_common_mt_end + 48,
-        gf_gld_mt_charptr                       = gf_common_mt_end + 49,
-        gf_gld_mt_hooks_stub_t                  = gf_common_mt_end + 50,
-        gf_gld_mt_hooks_priv_t                  = gf_common_mt_end + 51,
-        gf_gld_mt_mop_commit_req_t              = gf_common_mt_end + 52,
-        gf_gld_mt_int                           = gf_common_mt_end + 53,
-        gf_gld_mt_snap_t                        = gf_common_mt_end + 54,
-        gf_gld_mt_missed_snapinfo_t             = gf_common_mt_end + 55,
-        gf_gld_mt_snap_create_args_t            = gf_common_mt_end + 56,
-        gf_gld_mt_local_peers_t                 = gf_common_mt_end + 57,
-        gf_gld_mt_end                           = gf_common_mt_end + 58,
+    gf_gld_mt_glusterd_conf_t = gf_common_mt_end + 1,
+    gf_gld_mt_char,
+    gf_gld_mt_peerinfo_t,
+    gf_gld_mt_friend_sm_event_t,
+    gf_gld_mt_friend_req_ctx_t,
+    gf_gld_mt_friend_update_ctx_t,
+    gf_gld_mt_op_sm_event_t,
+    gf_gld_mt_op_lock_ctx_t,
+    gf_gld_mt_op_stage_ctx_t,
+    gf_gld_mt_op_commit_ctx_t,
+    gf_gld_mt_mop_stage_req_t,
+    gf_gld_mt_probe_ctx_t,
+    gf_gld_mt_glusterd_volinfo_t,
+    gf_gld_mt_volinfo_dict_data_t,
+    gf_gld_mt_glusterd_brickinfo_t,
+    gf_gld_mt_peer_hostname_t,
+    gf_gld_mt_defrag_info,
+    gf_gld_mt_peerctx_t,
+    gf_gld_mt_sm_tr_log_t,
+    gf_gld_mt_pending_node_t,
+    gf_gld_mt_brick_rsp_ctx_t,
+    gf_gld_mt_mop_brick_req_t,
+    gf_gld_mt_op_allack_ctx_t,
+    gf_gld_mt_linearr,
+    gf_gld_mt_linebuf,
+    gf_gld_mt_mount_pattern,
+    gf_gld_mt_mount_comp_container,
+    gf_gld_mt_mount_spec,
+    gf_gld_mt_georep_meet_spec,
+    gf_gld_mt_charptr,
+    gf_gld_mt_hooks_stub_t,
+    gf_gld_mt_hooks_priv_t,
+    gf_gld_mt_mop_commit_req_t,
+    gf_gld_mt_int,
+    gf_gld_mt_snap_t,
+    gf_gld_mt_missed_snapinfo_t,
+    gf_gld_mt_snap_create_args_t,
+    gf_gld_mt_glusterd_brick_proc_t,
+    gf_gld_mt_glusterd_svc_proc_t,
+    gf_gld_mt_end,
 } gf_gld_mem_types_t;
 #endif
-
diff --git a/xlators/mgmt/glusterd/src/glusterd-messages.h b/xlators/mgmt/glusterd/src/glusterd-messages.h
index 7af73907e49..3a1e600fb03 100644
--- a/xlators/mgmt/glusterd/src/glusterd-messages.h
+++ b/xlators/mgmt/glusterd/src/glusterd-messages.h
@@ -11,211 +11,441 @@
 #ifndef _GLUSTERD_MESSAGES_H_
 #define _GLUSTERD_MESSAGES_H_
 
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
+#include <glusterfs/glfs-message-id.h>
 
-#include "glfs-message-id.h"
-
-/*! \file glusterd-messages.h
- *  \brief Glusterd log-message IDs and their descriptions
- */
-
-/* NOTE: Rules for message additions
- * 1) Each instance of a message is _better_ left with a unique message ID, even
- *    if the message format is the same. Reasoning is that, if the message
- *    format needs to change in one instance, the other instances are not
- *    impacted or the new change does not change the ID of the instance being
- *    modified.
- * 2) Addition of a message,
- *       - Should increment the GLFS_NUM_MESSAGES
- *       - Append to the list of messages defined, towards the end
- *       - Retain macro naming as glfs_msg_X (for redability across developers)
- * NOTE: Rules for message format modifications
- * 3) Check acorss the code if the message ID macro in question is reused
- *    anywhere. If reused then then the modifications should ensure correctness
- *    everywhere, or needs a new message ID as (1) above was not adhered to. If
- *    not used anywhere, proceed with the required modification.
- * NOTE: Rules for message deletion
- * 4) Check (3) and if used anywhere else, then cannot be deleted. If not used
- *    anywhere, then can be deleted, but will leave a hole by design, as
- *    addition rules specify modification to the end of the list and not filling
- *    holes.
- */
-
-#define GLUSTERD_COMP_BASE      GLFS_MSGID_GLUSTERD
-#define GLFS_NUM_MESSAGES       20
-#define GLFS_MSGID_END          (GLUSTERD_COMP_BASE + GLFS_NUM_MESSAGES + 1)
-/* Messaged with message IDs */
-#define glfs_msg_start_x GLFS_COMP_BASE, "Invalid: Start of messages"
-/*------------*/
-
-/*!
- * @messageid 106001
- * @diagnosis Operation could not be performed because the server quorum was not
- *            met
- * @recommendedaction Ensure that other peer nodes are online and reachable from
- *                    the local peer node
- */
-#define GD_MSG_SERVER_QUORUM_NOT_MET (GLUSTERD_COMP_BASE + 1)
-
-/*!
- * @messageid 106002
- * @diagnosis The local bricks belonging to the volume were killed because
- *            the server-quorum was not met
- * @recommendedaction Ensure that other peer nodes are online and reachable from
- *                    the local peer node
- */
-#define GD_MSG_SERVER_QUORUM_LOST_STOPPING_BRICKS (GLUSTERD_COMP_BASE + 2)
-
-/*!
- * @messageid 106003
- * @diagnosis The local bricks belonging to the named volume were (re)started
- *            because the server-quorum was met
- * @recommendedaction  None
- */
-#define GD_MSG_SERVER_QUORUM_MET_STARTING_BRICKS (GLUSTERD_COMP_BASE + 3)
-
-/*!
- * @messageid 106004
- * @diagnosis Glusterd on the peer might be down or unreachable
- * @recommendedaction  Check if glusterd is running on the peer node or if
- *                     the firewall rules are not blocking port 24007
- */
-#define GD_MSG_PEER_DISCONNECTED (GLUSTERD_COMP_BASE + 4)
-
-/*!
- * @messageid 106005
- * @diagnosis Brick process might be down
- * @recommendedaction Check brick log files to get more information on the cause
- *                    for the brick's offline status. To bring the brick back
- *                    online,run gluster volume start <VOLNAME> force
- */
-#define GD_MSG_BRICK_DISCONNECTED (GLUSTERD_COMP_BASE + 5)
-
-/*!
- * @messageid 106006
- * @diagnosis NFS Server or Self-heal daemon might be down
- * @recommendedaction Check nfs or self-heal daemon log files to get more
- *                    information on the cause for the brick's offline status.
- *                    To bring the brick back online, run gluster volume
- *                    start <VOLNAME> force
- */
-#define GD_MSG_NODE_DISCONNECTED (GLUSTERD_COMP_BASE + 6)
-
-/*!
- * @messageid 106007
- * @diagnosis Rebalance process might be down
- * @recommendedaction None
- */
-#define GD_MSG_REBALANCE_DISCONNECTED (GLUSTERD_COMP_BASE + 7)
-
-/*!
- * @messageid 106008
- * @diagnosis Volume cleanup failed
- * @recommendedaction None
- */
-#define GD_MSG_VOL_CLEANUP_FAIL (GLUSTERD_COMP_BASE + 8)
-
-/*!
- * @messageid 106009
- * @diagnosis Volume version mismatch while adding a peer
- * @recommendedaction None
+/* To add new message IDs, append new identifiers at the end of the list.
+ *
+ * Never remove a message ID. If it's not used anymore, you can rename it or
+ * leave it as it is, but not delete it. This is to prevent reutilization of
+ * IDs by other messages.
+ *
+ * The component name must match one of the entries defined in
+ * glfs-message-id.h.
  */
-#define GD_MSG_VOL_VERS_MISMATCH (GLUSTERD_COMP_BASE + 9)
 
-/*!
- * @messageid 106010
- * @diagnosis Volume checksum mismatch while adding a peer
- * @recommendedaction Check for which node the checksum mismatch happens
- *                    and delete the volume configuration files from it andi
- *                    restart glusterd
- */
-#define GD_MSG_CKSUM_VERS_MISMATCH (GLUSTERD_COMP_BASE + 10)
-
-/*!
- * @messageid 106011
- * @diagnosis A volume quota-conf version mismatch occurred while adding a peer
- * @recommendedaction None
- */
-#define GD_MSG_QUOTA_CONFIG_VERS_MISMATCH (GLUSTERD_COMP_BASE + 11)
-
-/*!
- * @messageid 106012
- * @diagnosis A quota-conf checksum mismatch occurred while adding a peer
- * @recommendedaction Check for which node the checksum mismatch happens
- *                    and delete the volume configuration files from it and
- *                    restart glusterd
- */
-#define GD_MSG_QUOTA_CONFIG_CKSUM_MISMATCH (GLUSTERD_COMP_BASE + 12)
-
-/*!
- * @messageid 106013
- * @diagnosis Brick process could not be terminated
- * @recommendedaction Find the pid of the brick process from the log file and
- *                    manually kill it
- */
-#define GD_MSG_BRICK_STOP_FAIL (GLUSTERD_COMP_BASE + 13)
-
-/*!
- * @messageid 106014
- * @diagnosis One of the listed services:NFS Server, Quota Daemon, Self Heal
- *            Daemon, or brick process could not be brought offline
- * @recommendedaction Find the pid of the process from the log file and
- *                    manually kill it
- */
-#define GD_MSG_SVC_KILL_FAIL (GLUSTERD_COMP_BASE + 14)
-
-/*!
- * @messageid 106015
- * @diagnosis The process could not be killed with the specified PID
- * @recommendedaction None
- */
-#define GD_MSG_PID_KILL_FAIL (GLUSTERD_COMP_BASE + 15)
-
-/*!
- * @messageid 106016
- * @diagnosis Rebalance socket file is not found
- * @recommendedaction Rebalance failed as the socket file for rebalance is
- *                    missing. Restart the rebalance process
- */
-#define GD_MSG_REBAL_NO_SOCK_FILE (GLUSTERD_COMP_BASE + 16)
-
-/*!
- * @messageid 106017
- * @diagnosis Unix options could not be set
- * @recommendedaction Server is out of memory and needs a restart
- */
-#define GD_MSG_UNIX_OP_BUILD_FAIL (GLUSTERD_COMP_BASE + 17)
-
-/*!
- * @messageid 106018
- * @diagnosis RPC creation failed
- * @recommendedaction Rebalance failed as glusterd could not establish an RPC
- *                    connection. Check the log file for the exact reason of the
- *                    failure and then restart the rebalance process
- */
-#define GD_MSG_RPC_CREATE_FAIL (GLUSTERD_COMP_BASE + 18)
-
-/*!
- * @messageid 106019
- * @diagnosis The default options on volume could not be set with the volume
- *            create and volume reset commands
- * @recommendedaction Check glusterd log files to see the exact reason for
- *                    failure to set default options
- */
-#define GD_MSG_FAIL_DEFAULT_OPT_SET (GLUSTERD_COMP_BASE + 19)
-
-/*!
- * @messageid 106020
- * @diagnosis Failed to release cluster wide lock for one of the peer
- * @recommendedaction Restart the glusterd service on the node where the command
- * was issued
- */
-#define GD_MSG_CLUSTER_UNLOCK_FAILED (GLUSTERD_COMP_BASE + 20)
+GLFS_MSGID(
+    GLUSTERD, GD_MSG_SERVER_QUORUM_NOT_MET,
+    GD_MSG_SERVER_QUORUM_LOST_STOPPING_BRICKS,
+    GD_MSG_SERVER_QUORUM_MET_STARTING_BRICKS, GD_MSG_PEER_DISCONNECTED,
+    GD_MSG_BRICK_DISCONNECTED, GD_MSG_NODE_DISCONNECTED,
+    GD_MSG_REBALANCE_DISCONNECTED, GD_MSG_VOL_CLEANUP_FAIL,
+    GD_MSG_VOL_VERS_MISMATCH, GD_MSG_CKSUM_VERS_MISMATCH,
+    GD_MSG_QUOTA_CONFIG_VERS_MISMATCH, GD_MSG_QUOTA_CONFIG_CKSUM_MISMATCH,
+    GD_MSG_BRICK_STOP_FAIL, GD_MSG_SVC_KILL_FAIL, GD_MSG_PID_KILL_FAIL,
+    GD_MSG_REBAL_NO_SOCK_FILE, GD_MSG_UNIX_OP_BUILD_FAIL,
+    GD_MSG_RPC_CREATE_FAIL, GD_MSG_FAIL_DEFAULT_OPT_SET,
+    GD_MSG_CLUSTER_UNLOCK_FAILED, GD_MSG_NO_MEMORY, GD_MSG_UNSUPPORTED_VERSION,
+    GD_MSG_COMMAND_NOT_FOUND, GD_MSG_SNAPSHOT_OP_FAILED, GD_MSG_INVALID_ENTRY,
+    GD_MSG_VOL_NOT_FOUND, GD_MSG_REG_COMPILE_FAILED, GD_MSG_FILE_OP_FAILED,
+    GD_MSG_SNAP_CREATION_FAIL, GD_MSG_VOL_OP_FAILED, GD_MSG_CREATE_DIR_FAILED,
+    GD_MSG_DIR_OP_FAILED, GD_MSG_VOL_STOP_FAILED, GD_MSG_NO_CLI_RESP,
+    GD_MSG_LOCK_INIT_FAILED, GD_MSG_SNAP_LIST_GET_FAIL, GD_MSG_UNOUNT_FAILED,
+    GD_MSG_LOCK_DESTROY_FAILED, GD_MSG_SNAP_CLEANUP_FAIL,
+    GD_MSG_SNAP_ACTIVATE_FAIL, GD_MSG_SNAP_DEACTIVATE_FAIL,
+    GD_MSG_SNAP_RESTORE_FAIL, GD_MSG_SNAP_REMOVE_FAIL, GD_MSG_SNAP_CONFIG_FAIL,
+    GD_MSG_SNAP_STATUS_FAIL, GD_MSG_SNAP_INIT_FAIL, GD_MSG_VOLINFO_SET_FAIL,
+    GD_MSG_VOLINFO_GET_FAIL, GD_MSG_BRICK_CREATION_FAIL,
+    GD_MSG_BRICK_GET_INFO_FAIL, GD_MSG_BRICK_NEW_INFO_FAIL, GD_MSG_LVS_FAIL,
+    GD_MSG_SET_XATTR_FAIL, GD_MSG_UMOUNTING_SNAP_BRICK, GD_MSG_OP_UNSUPPORTED,
+    GD_MSG_SNAP_NOT_FOUND, GD_MSG_FS_LABEL_UPDATE_FAIL, GD_MSG_LVM_MOUNT_FAILED,
+    GD_MSG_DICT_SET_FAILED, GD_MSG_CANONICALIZE_FAIL, GD_MSG_DICT_GET_FAILED,
+    GD_MSG_SNAP_INFO_FAIL, GD_MSG_SNAP_VOL_CONFIG_FAIL,
+    GD_MSG_SNAP_OBJECT_STORE_FAIL, GD_MSG_DICT_UNSERIALIZE_FAIL,
+    GD_MSG_SNAP_RESTORE_REVERT_FAIL, GD_MSG_SNAP_LIST_SET_FAIL,
+    GD_MSG_VOLFILE_CREATE_FAIL, GD_MSG_VOLINFO_REMOVE_FAIL,
+    GD_MSG_VOL_DELETE_FAIL, GD_MSG_SNAPSHOT_PENDING,
+    GD_MSG_BRICK_PATH_UNMOUNTED, GD_MSG_BRICK_ADD_FAIL,
+    GD_MSG_BRICK_SET_INFO_FAIL, GD_MSG_LVCREATE_FAIL, GD_MSG_VG_GET_FAIL,
+    GD_MSG_TPOOL_GET_FAIL, GD_MSG_LVM_REMOVE_FAILED,
+    GD_MSG_MISSEDSNAP_INFO_SET_FAIL, GD_MSG_BRK_MOUNTOPTS_FAIL,
+    GD_MSG_MISSED_SNAP_LIST_STORE_FAIL, GD_MSG_INVALID_MISSED_SNAP_ENTRY,
+    GD_MSG_MISSED_SNAP_GET_FAIL, GD_MSG_MISSED_SNAP_CREATE_FAIL,
+    GD_MSG_DUP_ENTRY, GD_MSG_MISSED_SNAP_STATUS_DONE, GD_MSG_NO_EXEC_PERMS,
+    GD_MSG_GLOBAL_OP_VERSION_SET_FAIL, GD_MSG_HARD_LIMIT_SET_FAIL,
+    GD_MSG_OP_SUCCESS, GD_MSG_STORE_FAIL, GD_MSG_GLOBAL_OP_VERSION_GET_FAIL,
+    GD_MSG_GEOREP_GET_FAILED, GD_MSG_GLUSTERD_UMOUNT_FAIL,
+    GD_MSG_QUORUM_CHECK_FAIL, GD_MSG_QUORUM_COUNT_IGNORED,
+    GD_MSG_SNAP_MOUNT_FAIL, GD_MSG_RSP_DICT_USE_FAIL, GD_MSG_SNAP_IMPORT_FAIL,
+    GD_MSG_SNAP_CONFLICT, GD_MSG_MISSED_SNAP_DELETE,
+    GD_MSG_QUOTA_CONFIG_IMPORT_FAIL, GD_MSG_SNAPDIR_CREATE_FAIL,
+    GD_MSG_MISSED_SNAP_PRESENT, GD_MSG_UUID_NULL, GD_MSG_TSTAMP_SET_FAIL,
+    GD_MSG_RESP_AGGR_FAIL, GD_MSG_DICT_EMPTY, GD_MSG_DICT_CREATE_FAIL,
+    GD_MSG_SNAPD_STOP_FAIL, GD_MSG_SOFT_LIMIT_REACHED, GD_MSG_SNAPD_START_FAIL,
+    GD_MSG_SNAPD_CREATE_FAIL, GD_MSG_SNAPD_INIT_FAIL, GD_MSG_MGMTV3_OP_FAIL,
+    GD_MSG_MGMTV3_PAYLOAD_BUILD_FAIL, GD_MSG_MGMTV3_UNLOCK_FAIL,
+    GD_MSG_MGMTV3_LOCK_GET_FAIL, GD_MSG_MGMTV3_LOCKDOWN_FAIL,
+    GD_MSG_POST_VALIDATION_FAIL, GD_MSG_PRE_VALIDATION_FAIL,
+    GD_MSG_COMMIT_OP_FAIL, GD_MSG_PEER_LIST_CREATE_FAIL, GD_MSG_BRICK_OP_FAIL,
+    GD_MSG_OPINFO_SET_FAIL, GD_MSG_OP_EVENT_UNLOCK_FAIL,
+    GD_MSG_MGMTV3_OP_RESP_FAIL, GD_MSG_PEER_NOT_FOUND, GD_MSG_REQ_DECODE_FAIL,
+    GD_MSG_DICT_ALLOC_AND_SERL_LENGTH_GET_FAIL, GD_MSG_ALREADY_STOPPED,
+    GD_MSG_PRE_VALD_RESP_FAIL, GD_MSG_SVC_GET_FAIL, GD_MSG_VOLFILE_NOT_FOUND,
+    GD_MSG_OP_EVENT_LOCK_FAIL, GD_MSG_NON_STRIPE_VOL, GD_MSG_SNAPD_OBJ_GET_FAIL,
+    GD_MSG_QUOTA_DISABLED, GD_MSG_CACHE_MINMAX_SIZE_INVALID,
+    GD_MSG_QUOTA_GET_STAT_FAIL, GD_MSG_SUBVOLUMES_EXCEED, GD_MSG_BRICK_ADD,
+    GD_MSG_BRICK_REMOVE, GD_MSG_CREATE_KEY_FAIL,
+    GD_MSG_MULTIPLE_LOCK_ACQUIRE_FAIL, GD_MSG_MULTIPLE_LOCK_RELEASE_FAIL,
+    GD_MSG_RESP_FROM_UNKNOWN_PEER, GD_MSG_BRICK_MOUNDIRS_AGGR_FAIL,
+    GD_MSG_GFID_VALIDATE_SET_FAIL, GD_MSG_PEER_LOCK_FAIL,
+    GD_MSG_PEER_UNLOCK_FAIL, GD_MSG_MGMT_OP_FAIL,
+    GD_MSG_TRANS_OPINFO_CLEAR_FAIL, GD_MSG_GLUSTERD_LOCK_FAIL,
+    GD_MSG_TRANS_OPINFO_SET_FAIL, GD_MSG_TRANS_IDGEN_FAIL, GD_MSG_RPC_FAILURE,
+    GD_MSG_OP_VERS_ADJUST_FAIL, GD_MSG_SNAP_DEVICE_NAME_GET_FAIL,
+    GD_MSG_SNAP_STATUS_NOT_PENDING, GD_MSG_MGMT_PGM_SET_FAIL,
+    GD_MSG_EVENT_INJECT_FAIL, GD_MSG_VERS_INFO, GD_MSG_VOL_INFO_REQ_RECVD,
+    GD_MSG_VERS_GET_FAIL, GD_MSG_EVENT_NEW_GET_FAIL, GD_MSG_RPC_LAYER_ERROR,
+    GD_MSG_NO_HANDSHAKE_ACK, GD_MSG_OP_VERSION_MISMATCH,
+    GD_MSG_HANDSHAKE_REQ_REJECTED, GD_MSG_UNKNOWN_MODE,
+    GD_MSG_DEFRAG_STATUS_UPDATED, GD_MSG_NO_FLAG_SET,
+    GD_MSG_VERSION_UNSUPPORTED, GD_MSG_UUID_SET_FAIL, GD_MSG_MOUNT_REQ_FAIL,
+    GD_MSG_GLUSTERD_GLOBAL_INFO_STORE_FAIL, GD_MSG_OP_VERS_STORE_FAIL,
+    GD_MSG_SNAP_AUTOMIC_UPDATE_FAIL, GD_MSG_SNAPINFO_WRITE_FAIL,
+    GD_MSG_SNAPINFO_CREATE_FAIL, GD_MSG_SNAPD_INFO_STORE_FAIL,
+    GD_MSG_BRK_MNTPATH_MOUNT_FAIL, GD_MSG_BRK_MNTPATH_GET_FAIL,
+    GD_MSG_SNAP_BRK_MNT_RECREATE_FAIL, GD_MSG_SNAP_RESOLVE_BRICK_FAIL,
+    GD_MSG_RESOLVE_BRICK_FAIL, GD_MSG_BRK_MNT_RECREATE_FAIL,
+    GD_MSG_TMP_FILE_UNLINK_FAIL, GD_MSG_VOL_VALS_WRITE_FAIL,
+    GD_MSG_STORE_HANDLE_GET_FAIL, GD_MSG_STORE_HANDLE_WRITE_FAIL,
+    GD_MSG_MISSED_SNAP_LIST_STORE_HANDLE_GET_FAIL,
+    GD_MSG_MISSED_SNAP_LIST_EMPTY, GD_MSG_SNAP_VOL_RETRIEVE_FAIL,
+    GD_MSG_SNAPSHOT_UPDATE_FAIL, GD_MSG_SNAPD_PORT_STORE_FAIL,
+    GD_MSG_CKSUM_STORE_FAIL, GD_MSG_STORE_HANDLE_CREATE_FAIL,
+    GD_MSG_HANDLE_NULL, GD_MSG_VOL_RESTORE_FAIL, GD_MSG_NAME_TOO_LONG,
+    GD_MSG_UUID_PARSE_FAIL, GD_MSG_UNKNOWN_KEY, GD_MSG_STORE_ITER_DESTROY_FAIL,
+    GD_MSG_STORE_ITER_GET_FAIL, GD_MSG_VOLINFO_UPDATE_FAIL,
+    GD_MSG_PARSE_BRICKINFO_FAIL, GD_MSG_VERS_STORE_FAIL, GD_MSG_HEADER_ADD_FAIL,
+    GD_MSG_QUOTA_CONF_WRITE_FAIL, GD_MSG_QUOTA_CONF_CORRUPT, GD_MSG_FORK_FAIL,
+    GD_MSG_CKSUM_COMPUTE_FAIL, GD_MSG_VERS_CKSUM_STORE_FAIL,
+    GD_MSG_GET_XATTR_FAIL, GD_MSG_CONVERSION_FAILED, GD_MSG_VOL_NOT_DISTRIBUTE,
+    GD_MSG_VOL_STOPPED, GD_MSG_OPCTX_GET_FAIL, GD_MSG_TASKID_GEN_FAIL,
+    GD_MSG_REBALANCE_ID_MISSING, GD_MSG_NO_REBALANCE_PFX_IN_VOLNAME,
+    GD_MSG_DEFRAG_STATUS_UPDATE_FAIL, GD_MSG_UUID_GEN_STORE_FAIL,
+    GD_MSG_UUID_STORE_FAIL, GD_MSG_NO_INIT, GD_MSG_MODULE_NOT_INSTALLED,
+    GD_MSG_MODULE_NOT_WORKING, GD_MSG_WRITE_ACCESS_GRANT_FAIL,
+    GD_MSG_DIRPATH_TOO_LONG, GD_MSG_LOGGROUP_INVALID, GD_MSG_DIR_PERM_LIBERAL,
+    GD_MSG_DIR_PERM_STRICT, GD_MSG_MOUNT_SPEC_INSTALL_FAIL,
+    GD_MSG_GLUSTERD_SOCK_LISTENER_START_FAIL, GD_MSG_DIR_NOT_FOUND,
+    GD_MSG_FAILED_INIT_SHDSVC, GD_MSG_FAILED_INIT_NFSSVC,
+    GD_MSG_FAILED_INIT_QUOTASVC, GD_MSG_RPC_INIT_FAIL,
+    GD_MSG_RPCSVC_REG_NOTIFY_RETURNED, GD_MSG_RPC_TRANSPORT_COUNT_GET_FAIL,
+    GD_MSG_RPC_LISTENER_CREATE_FAIL, GD_MSG_OP_VERS_RESTORE_FAIL,
+    GD_MSG_SELF_HEALD_DISABLED, GD_MSG_PRIV_NULL, GD_MSG_GSYNC_VALIDATION_FAIL,
+    GD_MSG_SLAVE_CONFPATH_DETAILS_FETCH_FAIL, GD_MSG_OP_NOT_PERMITTED_AC_REQD,
+    GD_MSG_OP_NOT_PERMITTED, GD_MSG_REBALANCE_START_FAIL,
+    GD_MSG_NFS_RECONF_FAIL, GD_MSG_REMOVE_BRICK_ID_SET_FAIL,
+    GD_MSG_BRICK_MOUNTDIR_GET_FAIL, GD_MSG_BRICK_NOT_FOUND,
+    GD_MSG_BRKPATH_TOO_LONG, GD_MSG_CLRLOCKS_CLNT_UMOUNT_FAIL,
+    GD_MSG_CLRLOCKS_CLNT_MOUNT_FAIL, GD_MSG_CLRLOCKS_MOUNTDIR_CREATE_FAIL,
+    GD_MSG_BRK_PORT_NUM_GET_FAIL, GD_MSG_BRK_STATEDUMP_FAIL,
+    GD_MSG_VOL_GRAPH_CHANGE_NOTIFY_FAIL, GD_MSG_INVALID_VG,
+    GD_MSG_GLUSTERD_OP_FAILED, GD_MSG_HOSTNAME_ADD_TO_PEERLIST_FAIL,
+    GD_MSG_STALE_PEERINFO_REMOVE_FAIL, GD_MSG_TRANS_ID_GET_FAIL,
+    GD_MSG_RES_DECODE_FAIL, GD_MSG_VOL_ALREADY_EXIST, GD_MSG_BAD_BRKORDER,
+    GD_MSG_BAD_BRKORDER_CHECK_FAIL, GD_MSG_BRICK_SELECT_FAIL,
+    GD_MSG_NO_LOCK_RESP_FROM_PEER, GD_MSG_MGMTV3_LOCK_FROM_UUID_REJCT,
+    GD_MSG_STAGE_FROM_UUID_REJCT, GD_MSG_UNLOCK_FROM_UUID_REJCT,
+    GD_MSG_MGMTV3_UNLOCK_FROM_UUID_REJCT, GD_MSG_COMMIT_FROM_UUID_REJCT,
+    GD_MSG_VOL_NOT_STARTED, GD_MSG_VOL_NOT_REPLICA, GD_MSG_VOL_NOT_DISPERSE,
+    GD_MSG_OLD_REMOVE_BRICK_EXISTS, GD_MSG_USE_THE_FORCE, GD_MSG_OIP,
+    GD_MSG_OIP_RETRY_LATER, GD_MSG_GSYNC_RESTART_FAIL,
+    GD_MSG_LOCK_FROM_UUID_REJCT, GD_MSG_BRICK_OP_PAYLOAD_BUILD_FAIL,
+    GD_MSG_HOSTNAME_RESOLVE_FAIL, GD_MSG_COUNT_VALIDATE_FAILED,
+    GD_MSG_SPAWNING_CHILD_FAILED, GD_MSG_READ_CHILD_DATA_FAILED,
+    GD_MSG_DEFAULT_TEMP_CONFIG, GD_MSG_PIDFILE_CREATE_FAILED,
+    GD_MSG_GSYNCD_SPAWN_FAILED, GD_MSG_SUBOP_NOT_FOUND, GD_MSG_RESERVED_OPTION,
+    GD_MSG_GLUSTERD_PRIV_NOT_FOUND, GD_MSG_SLAVEINFO_FETCH_ERROR,
+    GD_MSG_VALIDATE_FAILED, GD_MSG_INVOKE_ERROR, GD_MSG_SESSION_CREATE_ERROR,
+    GD_MSG_STOP_FORCE, GD_MSG_GET_CONFIG_INFO_FAILED,
+    GD_MSG_STAT_FILE_READ_FAILED, GD_MSG_CONF_PATH_ASSIGN_FAILED,
+    GD_MSG_SESSION_INACTIVE, GD_MSG_PIDFILE_NOT_FOUND, GD_MSG_PEER_CMD_ERROR,
+    GD_MSG_SRC_FILE_ERROR, GD_MSG_GET_STATEFILE_NAME_FAILED, GD_MSG_STATUS_NULL,
+    GD_MSG_STATUSFILE_CREATE_FAILED, GD_MSG_SLAVE_URL_INVALID,
+    GD_MSG_INVALID_SLAVE, GD_MSG_READ_ERROR, GD_MSG_ARG_FETCH_ERROR,
+    GD_MSG_REG_FILE_MISSING, GD_MSG_STATEFILE_NAME_NOT_FOUND,
+    GD_MSG_GEO_REP_START_FAILED, GD_MSG_GSYNCD_ERROR,
+    GD_MSG_UPDATE_STATEFILE_FAILED, GD_MSG_STATUS_UPDATE_FAILED,
+    GD_MSG_GSYNCD_OP_SET_FAILED, GD_MSG_BUFFER_EMPTY, GD_MSG_CONFIG_INFO,
+    GD_MSG_FETCH_CONFIG_VAL_FAILED, GD_MSG_GSYNCD_PARSE_ERROR,
+    GD_MSG_SESSION_ALREADY_EXIST, GD_MSG_FORCE_CREATE_SESSION,
+    GD_MSG_GET_KEY_FAILED, GD_MSG_SESSION_DEL_FAILED, GD_MSG_CMD_EXEC_FAIL,
+    GD_MSG_STRDUP_FAILED, GD_MSG_UNABLE_TO_END, GD_MSG_PAUSE_FAILED,
+    GD_MSG_NORMALIZE_URL_FAIL, GD_MSG_MODULE_ERROR,
+    GD_MSG_SLAVEINFO_STORE_ERROR, GD_MSG_MARKER_START_FAIL,
+    GD_MSG_RESUME_FAILED, GD_MSG_GLUSTERFS_START_FAIL,
+    GD_MSG_GLUSTERFS_STOP_FAIL, GD_MSG_RBOP_STATE_STORE_FAIL,
+    GD_MSG_PUMP_XLATOR_DISABLED, GD_MSG_ABORT_OP_FAIL, GD_MSG_PAUSE_OP_FAIL,
+    GD_MSG_GLUSTER_SERVICE_START_FAIL, GD_MSG_HANDSHAKE_FAILED,
+    GD_MSG_CLI_REQ_EMPTY, GD_MSG_PEER_ADD_FAIL,
+    GD_MSG_SYNC_FROM_LOCALHOST_UNALLOWED, GD_MSG_UUIDS_SAME_RETRY,
+    GD_MSG_TSP_ALREADY_FORMED, GD_MSG_VOLS_ALREADY_PRESENT,
+    GD_MSG_REQ_CTX_CREATE_FAIL, GD_MSG_PEER_INFO_UPDATE_FAIL,
+    GD_MSG_PEERINFO_CREATE_FAIL, GD_MSG_REQ_FROM_UNKNOWN_PEER,
+    GD_MSG_STATUS_REPLY_STRING_CREATE_FAIL, GD_MSG_TOKENIZE_FAIL,
+    GD_MSG_LAZY_UMOUNT_FAIL, GD_MSG_NFS_SERVER_START_FAIL,
+    GD_MSG_GLUSTER_SERVICES_STOP_FAIL, GD_MSG_BRK_CLEANUP_FAIL,
+    GD_MSG_RB_ALREADY_STARTED, GD_MSG_RB_BRICKINFO_GET_FAIL, GD_MSG_BAD_FORMAT,
+    GD_MSG_RB_CMD_FAIL, GD_MSG_RB_NOT_STARTED_OR_PAUSED, GD_MSG_RB_NOT_STARTED,
+    GD_MSG_RB_PAUSED_ALREADY, GD_MSG_NO_FREE_PORTS,
+    GD_MSG_EVENT_STATE_TRANSITION_FAIL, GD_MSG_HANDLER_RETURNED,
+    GD_MSG_SNAP_COMPARE_CONFLICT, GD_MSG_PEER_DETACH_CLEANUP_FAIL,
+    GD_MSG_STALE_VOL_REMOVE_FAIL, GD_MSG_AC_ERROR, GD_MSG_LOCK_FAIL,
+    GD_MSG_MGMTV3_LOCK_REQ_SEND_FAIL, GD_MSG_GLUSTERD_UNLOCK_FAIL,
+    GD_MSG_RBOP_START_FAIL, GD_MSG_UNKNOWN_RESPONSE,
+    GD_MSG_COMMIT_REQ_SEND_FAIL, GD_MSG_OPCTX_UPDATE_FAIL, GD_MSG_OPCTX_NULL,
+    GD_MSG_DICT_COPY_FAIL, GD_MSG_SHD_STATUS_SET_FAIL,
+    GD_MSG_REPLICA_INDEX_GET_FAIL, GD_MSG_NFS_SERVER_NOT_RUNNING,
+    GD_MSG_STAGE_REQ_SEND_FAIL, GD_MSG_LOCK_REQ_SEND_FAIL,
+    GD_MSG_VOLNAMES_GET_FAIL, GD_MSG_NO_TASK_ID, GD_MSG_ADD_REMOVE_BRICK_FAIL,
+    GD_MSG_SVC_RESTART_FAIL, GD_MSG_VOL_SET_FAIL, GD_MSG_QUOTAD_NOT_RUNNING,
+    GD_MSG_XLATOR_COUNT_GET_FAIL, GD_MSG_TRANS_OPINFO_GET_FAIL,
+    GD_MSG_TRANS_ID_INVALID, GD_MSG_NO_OPTIONS_GIVEN, GD_MSG_SNAPD_NOT_RUNNING,
+    GD_MSG_ADD_ADDRESS_TO_PEER_FAIL, GD_MSG_PEER_ADDRESS_GET_FAIL,
+    GD_MSG_GETADDRINFO_FAIL, GD_MSG_PEERINFO_DELETE_FAIL, GD_MSG_KEY_NULL,
+    GD_MSG_SPAWN_SVCS_FAIL, GD_MSG_DICT_ITER_FAIL,
+    GD_MSG_TASK_STATUS_UPDATE_FAIL, GD_MSG_VOL_ID_MISMATCH,
+    GD_MSG_STR_TO_BOOL_FAIL, GD_MSG_RB_MNT_BRICKS_MISMATCH,
+    GD_MSG_RB_SRC_BRICKS_MISMATCH, GD_MSG_MNTENTRY_GET_FAIL,
+    GD_MSG_INODE_SIZE_GET_FAIL, GD_MSG_NO_STATEFILE_ENTRY,
+    GD_MSG_PMAP_UNSET_FAIL, GD_MSG_GLOBAL_OPT_IMPORT_FAIL,
+    GD_MSD_BRICK_DISCONNECT_FAIL, GD_MSG_SNAP_DETAILS_IMPORT_FAIL,
+    GD_MSG_BRICKINFO_CREATE_FAIL, GD_MSG_QUOTA_CKSUM_VER_STORE_FAIL,
+    GD_MSG_CKSUM_GET_FAIL, GD_MSG_BRICKPATH_ROOT_GET_FAIL,
+    GD_MSG_HOSTNAME_TO_UUID_FAIL, GD_MSG_REPLY_SUBMIT_FAIL,
+    GD_MSG_SERIALIZE_MSG_FAIL, GD_MSG_ENCODE_FAIL,
+    GD_MSG_RB_DST_BRICKS_MISMATCH, GD_MSG_XLATOR_VOLOPT_DYNLOAD_ERROR,
+    GD_MSG_VOLNAME_NOTFOUND_IN_DICT, GD_MSG_FLAGS_NOTFOUND_IN_DICT,
+    GD_MSG_HOSTNAME_NOTFOUND_IN_DICT, GD_MSG_PORT_NOTFOUND_IN_DICT,
+    GD_MSG_CMDSTR_NOTFOUND_IN_DICT, GD_MSG_SNAP_OBJ_NEW_FAIL,
+    GD_MSG_SNAP_BACKEND_MAKE_FAIL, GD_MSG_SNAP_CLONE_FAILED,
+    GD_MSG_SNAP_CLONE_PREVAL_FAILED, GD_MSG_SNAP_CLONE_POSTVAL_FAILED,
+    GD_MSG_VOLINFO_STORE_FAIL, GD_MSG_NEW_FRIEND_SM_EVENT_GET_FAIL,
+    GD_MSG_VOL_TYPE_CHANGING_INFO, GD_MSG_BRKPATH_MNTPNT_MISMATCH,
+    GD_MSG_TASKS_COUNT_MISMATCH, GD_MSG_WRONG_OPTS_SETTING,
+    GD_MSG_PATH_ALREADY_PART_OF_VOL, GD_MSG_BRICK_VALIDATE_FAIL,
+    GD_MSG_READIN_FILE_FAILED, GD_MSG_IMPORT_PRDICT_DICT,
+    GD_MSG_VOL_OPTS_IMPORT_FAIL, GD_MSG_BRICK_IMPORT_FAIL,
+    GD_MSG_VOLINFO_IMPORT_FAIL, GD_MSG_BRICK_ID_GEN_FAILED,
+    GD_MSG_GET_STATUS_DATA_FAIL, GD_MSG_BITROT_NOT_RUNNING,
+    GD_MSG_SCRUBBER_NOT_RUNNING, GD_MSG_SRC_BRICK_PORT_UNAVAIL,
+    GD_MSG_BITD_INIT_FAIL, GD_MSG_SCRUB_INIT_FAIL, GD_MSG_VAR_RUN_DIR_INIT_FAIL,
+    GD_MSG_VAR_RUN_DIR_FIND_FAIL, GD_MSG_SCRUBSVC_RECONF_FAIL,
+    GD_MSG_BITDSVC_RECONF_FAIL, GD_MSG_NFS_GNS_START_FAIL,
+    GD_MSG_NFS_GNS_SETUP_FAIL, GD_MSG_UNRECOGNIZED_SVC_MNGR,
+    GD_MSG_NFS_GNS_OP_HANDLE_FAIL, GD_MSG_EXPORT_FILE_CREATE_FAIL,
+    GD_MSG_NFS_GNS_HOST_FOUND, GD_MSG_REBALANCE_CMD_IN_TIER_VOL,
+    GD_MSG_INCOMPATIBLE_VALUE, GD_MSG_GENERATED_UUID,
+    GD_MSG_FILE_DESC_LIMIT_SET, GD_MSG_CURR_WORK_DIR_INFO,
+    GD_MSG_STRIPE_COUNT_CHANGE_INFO, GD_MSG_REPLICA_COUNT_CHANGE_INFO,
+    GD_MSG_ADD_BRICK_REQ_RECVD, GD_MSG_VOL_ALREADY_TIER,
+    GD_MSG_REM_BRICK_REQ_RECVD, GD_MSG_VOL_NOT_TIER,
+    GD_MSG_LOG_ROTATE_REQ_RECVD, GD_MSG_CLI_REQ_RECVD, GD_MSG_GET_VOL_REQ_RCVD,
+    GD_MSG_VOL_SYNC_REQ_RCVD, GD_MSG_PROBE_RCVD, GD_MSG_UNFRIEND_REQ_RCVD,
+    GD_MSG_FRIEND_UPDATE_RCVD, GD_MSG_RESPONSE_INFO,
+    GD_MSG_VOL_PROFILE_REQ_RCVD, GD_MSG_GETWD_REQ_RCVD, GD_MSG_MOUNT_REQ_RCVD,
+    GD_MSG_UMOUNT_REQ_RCVD, GD_MSG_CONNECT_RETURNED, GD_MSG_STATUS_VOL_REQ_RCVD,
+    GD_MSG_CLRCLK_VOL_REQ_RCVD, GD_MSG_BARRIER_VOL_REQ_RCVD,
+    GD_MSG_UUID_RECEIVED, GD_MSG_REPLACE_BRK_COMMIT_FORCE_REQ_RCVD,
+    GD_MSG_BRK_PORT_NO_ADD_INDO, GD_MSG_REPLACE_BRK_REQ_RCVD,
+    GD_MSG_ADD_OP_ARGS_FAIL, GD_MSG_POST_HOOK_STUB_INIT_FAIL,
+    GD_MSG_HOOK_STUB_NULL, GD_MSG_SPAWN_THREADS_FAIL,
+    GD_MSG_STALE_VOL_DELETE_INFO, GD_MSG_PROBE_REQ_RESP_RCVD,
+    GD_MSG_HOST_PRESENT_ALREADY, GD_MSG_OP_VERS_INFO, GD_MSG_OP_VERS_SET_INFO,
+    GD_MSG_NEW_NODE_STATE_CREATION, GD_MSG_ALREADY_MOUNTED,
+    GD_MSG_SHARED_STRG_VOL_OPT_VALIDATE_FAIL, GD_MSG_NFS_GNS_STOP_FAIL,
+    GD_MSG_NFS_GNS_RESET_FAIL, GD_MSG_SHARED_STRG_SET_FAIL,
+    GD_MSG_VOL_TRANSPORT_TYPE_CHANGE, GD_MSG_PEER_COUNT_GET_FAIL,
+    GD_MSG_INSUFFICIENT_UP_NODES, GD_MSG_OP_STAGE_STATS_VOL_FAIL,
+    GD_MSG_VOL_ID_SET_FAIL, GD_MSG_OP_STAGE_RESET_VOL_FAIL,
+    GD_MSG_OP_STAGE_BITROT_FAIL, GD_MSG_OP_STAGE_QUOTA_FAIL,
+    GD_MSG_OP_STAGE_DELETE_VOL_FAIL, GD_MSG_HANDLE_HEAL_CMD_FAIL,
+    GD_MSG_CLRCLK_SND_CMD_FAIL, GD_MSG_DISPERSE_CLUSTER_FOUND,
+    GD_MSG_HEAL_VOL_REQ_RCVD, GD_MSG_STATEDUMP_VOL_REQ_RCVD,
+    GD_MSG_THINPOOLS_FOR_THINLVS, GD_MSG_OP_STAGE_CREATE_VOL_FAIL,
+    GD_MSG_OP_STAGE_START_VOL_FAIL, GD_MSG_NFS_GNS_UNEXPRT_VOL_FAIL,
+    GD_MSG_TASK_ID_INFO, GD_MSG_DEREGISTER_SUCCESS, GD_MSG_STATEDUMP_OPTS_RCVD,
+    GD_MSG_STATEDUMP_INFO, GD_MSG_RECOVERING_CORRUPT_CONF,
+    GD_MSG_RETRIEVED_UUID, GD_MSG_XLATOR_CREATE_FAIL,
+    GD_MSG_GRAPH_ENTRY_ADD_FAIL, GD_MSG_ERROR_ENCOUNTERED,
+    GD_MSG_FILTER_RUN_FAILED, GD_MSG_DEFAULT_OPT_INFO,
+    GD_MSG_MARKER_STATUS_GET_FAIL, GD_MSG_MARKER_DISABLE_FAIL,
+    GD_MSG_GRAPH_FEATURE_ADD_FAIL, GD_MSG_XLATOR_SET_OPT_FAIL,
+    GD_MSG_BUILD_GRAPH_FAILED, GD_MSG_XML_TEXT_WRITE_FAIL,
+    GD_MSG_XML_DOC_START_FAIL, GD_MSG_XML_ELE_CREATE_FAIL,
+    GD_MSG_VOLUME_INCONSISTENCY, GD_MSG_XLATOR_LINK_FAIL,
+    GD_MSG_REMOTE_HOST_GET_FAIL, GD_MSG_GRAPH_SET_OPT_FAIL,
+    GD_MSG_ROOT_SQUASH_ENABLED, GD_MSG_ROOT_SQUASH_FAILED,
+    GD_MSG_LOCK_OWNER_MISMATCH, GD_MSG_LOCK_NOT_HELD, GD_MSG_LOCK_ALREADY_HELD,
+    GD_MSG_SVC_START_SUCCESS, GD_MSG_SVC_STOP_SUCCESS, GD_MSG_PARAM_NULL,
+    GD_MSG_SVC_STOP_FAIL, GD_MSG_SHARED_STORAGE_DOES_NOT_EXIST,
+    GD_MSG_SNAP_PAUSE_TIER_FAIL, GD_MSG_SNAP_RESUME_TIER_FAIL,
+    GD_MSG_FILE_NOT_FOUND, GD_MSG_RETRY_WITH_NEW_PORT,
+    GD_MSG_REMOTE_VOL_UUID_FAIL, GD_MSG_SLAVE_VOL_PARSE_FAIL,
+    GD_MSG_DICT_GET_SUCCESS, GD_MSG_PMAP_REGISTRY_REMOVE_FAIL,
+    GD_MSG_MNTBROKER_LABEL_NULL, GD_MSG_MNTBROKER_LABEL_MISS,
+    GD_MSG_MNTBROKER_SPEC_MISMATCH, GD_MSG_SYSCALL_FAIL,
+    GD_MSG_DAEMON_STATE_REQ_RCVD, GD_MSG_BRICK_CLEANUP_SUCCESS,
+    GD_MSG_STATE_STR_GET_FAILED, GD_MSG_RESET_BRICK_COMMIT_FORCE_REQ_RCVD,
+    GD_MSG_RESET_BRICK_CMD_FAIL, GD_MSG_TIERD_STOP_FAIL,
+    GD_MSG_TIERD_CREATE_FAIL, GD_MSG_TIERD_START_FAIL,
+    GD_MSG_TIERD_OBJ_GET_FAIL, GD_MSG_TIERD_NOT_RUNNING, GD_MSG_TIERD_INIT_FAIL,
+    GD_MSG_BRICK_MX_SET_FAIL, GD_MSG_NO_SIG_TO_PID_ZERO,
+    GD_MSG_TIER_WATERMARK_RESET_FAIL, GD_MSG_CLIENTS_GET_STATE_FAILED,
+    GD_MSG_GNFS_XLATOR_NOT_INSTALLED, GD_MSG_PIDFILE_UNLINKING,
+    GD_MSG_VOL_SET_VALIDATION_INFO, GD_MSG_NO_MUX_LIMIT,
+    GD_MSG_BRICKPROC_REM_BRICK_FAILED, GD_MSG_BRICKPROC_ADD_BRICK_FAILED,
+    GD_MSG_BRICKPROC_NEW_FAILED, GD_MSG_STATVFS_FAILED, GD_MSG_GARBAGE_ARGS,
+    GD_MSG_LOCALTIME_LOGGING_VOL_OPT_VALIDATE_FAIL,
+    GD_MSG_LOCALTIME_LOGGING_ENABLE, GD_MSG_LOCALTIME_LOGGING_DISABLE,
+    GD_MSG_PORTS_EXHAUSTED, GD_MSG_CHANGELOG_GET_FAIL,
+    GD_MSG_MANAGER_FUNCTION_FAILED,
+    GD_MSG_DAEMON_LOG_LEVEL_VOL_OPT_VALIDATE_FAIL, GD_MSG_SHD_START_FAIL,
+    GD_MSG_SHD_OBJ_GET_FAIL, GD_MSG_SVC_ATTACH_FAIL, GD_MSG_ATTACH_INFO,
+    GD_MSG_DETACH_INFO, GD_MSG_SVC_DETACH_FAIL,
+    GD_MSG_RPC_TRANSPORT_GET_PEERNAME_FAIL, GD_MSG_CLUSTER_RC_ENABLE,
+    GD_MSG_NFS_GANESHA_DISABLED, GD_MSG_GANESHA_NOT_RUNNING, GD_MSG_SNAP_WARN,
+    GD_MSG_BRICK_SUBVOL_VERIFY_FAIL, GD_MSG_REMOVE_ARBITER_BRICK,
+    GD_MSG_BRICK_NOT_DECOM, GD_MSG_BRICK_STOPPED, GD_MSG_BRICK_DEAD,
+    GD_MSG_BRICK_HOST_NOT_FOUND, GD_MSG_BRICK_HOST_DOWN, GD_MSG_BRICK_DELETE,
+    GD_MSG_BRICK_NO_REMOVE_CMD, GD_MSG_MIGRATION_PROG, GD_MSG_MIGRATION_FAIL,
+    GD_MSG_COPY_FAIL, GD_MSG_REALPATH_GET_FAIL,
+    GD_MSG_ARBITER_BRICK_SET_INFO_FAIL, GD_MSG_STRCHR_FAIL, GD_MSG_SPLIT_FAIL,
+    GD_MSG_ALLOC_AND_COPY_UUID_FAIL, GD_MSG_VOL_SHD_NOT_COMP,
+    GD_MSG_BITROT_NOT_ENABLED, GD_MSG_CREATE_BRICK_DIR_FAILED,
+    GD_MSG_CREATE_GLUSTER_DIR_FAILED, GD_MSG_BRICK_CREATE_MNTPNT,
+    GD_MSG_BRICK_CREATE_ROOT, GD_MSG_SET_XATTR_BRICK_FAIL,
+    GD_MSG_REMOVE_XATTR_FAIL, GD_MSG_XLATOR_NOT_DEFINED,
+    GD_MSG_BRICK_NOT_RUNNING, GD_MSG_INCORRECT_BRICK, GD_MSG_UUID_GET_FAIL,
+    GD_MSG_INVALID_ARGUMENT, GD_MSG_FRAME_CREATE_FAIL,
+    GD_MSG_SNAPSHOT_NOT_THIN_PROVISIONED, GD_MSG_VOL_STOP_ARGS_GET_FAILED,
+    GD_MSG_LSTAT_FAIL, GD_MSG_VOLUME_NOT_IMPORTED,
+    GD_MSG_ADD_BRICK_MNT_INFO_FAIL, GD_MSG_GET_MNT_ENTRY_INFO_FAIL,
+    GD_MSG_QUORUM_CLUSTER_COUNT_GET_FAIL, GD_MSG_POST_COMMIT_OP_FAIL,
+    GD_MSG_POST_COMMIT_FROM_UUID_REJCT, GD_MSG_POST_COMMIT_REQ_SEND_FAIL);
 
-/*------------*/
-#define glfs_msg_end_x GLFS_MSGID_END, "Invalid: End of messages"
+#define GD_MSG_INVALID_ENTRY_STR "Invalid data entry"
+#define GD_MSG_INVALID_ARGUMENT_STR                                            \
+    "Invalid arguments have been given to function"
+#define GD_MSG_GARBAGE_ARGS_STR "Garbage args received"
+#define GD_MSG_BRICK_SUBVOL_VERIFY_FAIL_STR "Brick's subvol verification fail"
+#define GD_MSG_REMOVE_ARBITER_BRICK_STR "Failed to remove arbiter bricks"
+#define GD_MSG_DICT_GET_FAILED_STR "Dict get failed"
+#define GD_MSG_DICT_SET_FAILED_STR "Dict set failed"
+#define GD_MSG_BRICK_NOT_FOUND_STR "Brick not found in volume"
+#define GD_MSG_BRICK_NOT_DECOM_STR "Brick is not decommissoned"
+#define GD_MSG_BRICK_STOPPED_STR "Found stopped brick"
+#define GD_MSG_BRICK_DEAD_STR "Found dead brick"
+#define GD_MSG_BRICK_HOST_NOT_FOUND_STR                                        \
+    "Host node of the brick is not a part of cluster"
+#define GD_MSG_BRICK_HOST_DOWN_STR "Host node of the brick is down"
+#define GD_MSG_BRICK_DELETE_STR                                                \
+    "Deleting all the bricks of the volume is not allowed"
+#define GD_MSG_BRICK_NO_REMOVE_CMD_STR "No remove-brick command issued"
+#define GD_MSG_INCORRECT_BRICK_STR "Incorrect brick for volume"
+#define GD_MSG_MIGRATION_PROG_STR "Migration is in progress"
+#define GD_MSG_MIGRATION_FAIL_STR "Migration has failed"
+#define GD_MSG_XLATOR_NOT_DEFINED_STR "Xlator not defined"
+#define GD_MSG_DICT_CREATE_FAIL_STR "Failed to create dictionary"
+#define GD_MSG_COPY_FAIL_STR "Failed to copy"
+#define GD_MSG_UUID_GET_FAIL_STR "Failed to get the uuid of local glusterd"
+#define GD_MSG_GEO_REP_START_FAILED_STR "Georep start failed for volume"
+#define GD_MSG_REALPATH_GET_FAIL_STR "Failed to get realpath"
+#define GD_MSG_FILE_NOT_FOUND_STR "File not found in directory"
+#define GD_MSG_SRC_FILE_ERROR_STR "Error in source file"
+#define GD_MSG_DICT_UNSERIALIZE_FAIL_STR "Failed to unserialize dict"
+#define GD_MSG_VOL_ID_SET_FAIL_STR "Failed to set volume id"
+#define GD_MSG_ARBITER_BRICK_SET_INFO_FAIL_STR                                 \
+    "Failed to add arbiter info to brick"
+#define GD_MSG_NO_MEMORY_STR "Out of memory"
+#define GD_MSG_GLUSTERD_UMOUNT_FAIL_STR "Failed to unmount path"
+#define GD_MSG_PEER_ADD_FAIL_STR "Failed to add new peer"
+#define GD_MSG_BRICK_GET_INFO_FAIL_STR "Failed to get brick info"
+#define GD_MSG_STRCHR_FAIL_STR "Failed to get the character"
+#define GD_MSG_SPLIT_FAIL_STR "Failed to split"
+#define GD_MSG_VOLINFO_GET_FAIL_STR "Failed to get volinfo"
+#define GD_MSG_PEER_NOT_FOUND_STR "Failed to find peer info"
+#define GD_MSG_DICT_COPY_FAIL_STR "Failed to copy values from dictionary"
+#define GD_MSG_ALLOC_AND_COPY_UUID_FAIL_STR                                    \
+    "Failed to allocate memory or copy uuid"
+#define GD_MSG_VOL_NOT_FOUND_STR "Volume not found"
+#define GD_MSG_PEER_DISCONNECTED_STR "Peer is disconnected"
+#define GD_MSG_QUOTA_GET_STAT_FAIL_STR "Failed to get quota status"
+#define GD_MSG_SNAP_STATUS_FAIL_STR "Failed to get status of snapd"
+#define GD_MSG_VALIDATE_FAILED_STR "Failed to validate volume"
+#define GD_MSG_VOL_NOT_STARTED_STR "Volume is not started"
+#define GD_MSG_VOL_SHD_NOT_COMP_STR "Volume is not Self-heal compatible"
+#define GD_MSG_SELF_HEALD_DISABLED_STR "Self-heal daemon is disabled"
+#define GD_MSG_NFS_GANESHA_DISABLED_STR "NFS server is disabled"
+#define GD_MSG_QUOTA_DISABLED_STR "Quota is disabled"
+#define GD_MSG_BITROT_NOT_RUNNING_STR "Bitrot is not enabled"
+#define GD_MSG_BITROT_NOT_ENABLED_STR "Volume does not have bitrot enabled"
+#define GD_MSG_SNAPD_NOT_RUNNING_STR "Snapd is not enabled"
+#define GD_MSG_STRDUP_FAILED_STR "Strdup operation failed"
+#define GD_MSG_QUORUM_CLUSTER_COUNT_GET_FAIL_STR                               \
+    "Failed to get quorum cluster counts"
+#define GD_MSG_GLUSTER_SERVICE_START_FAIL_STR "Failed to start glusterd service"
+#define GD_MSG_PEER_ADDRESS_GET_FAIL_STR "Failed to get the address of peer"
+#define GD_MSG_INVALID_SLAVE_STR "Volume is not a slave volume"
+#define GD_MSG_BRICK_NOT_RUNNING_STR "One or more bricks are not running"
+#define GD_MSG_BRK_MNTPATH_GET_FAIL_STR "Failed to get brick mount device"
+#define GD_MSG_SNAPSHOT_NOT_THIN_PROVISIONED_STR                               \
+    "Snapshot is supported only for thin provisioned LV."
+#define GD_MSG_SNAP_DEVICE_NAME_GET_FAIL_STR                                   \
+    "Failed to copy snapshot device name"
+#define GD_MSG_SNAP_NOT_FOUND_STR "Snapshot does not exist"
+#define GD_MSG_CREATE_BRICK_DIR_FAILED_STR "Failed to create brick directory"
+#define GD_MSG_LSTAT_FAIL_STR "Lstat operation failed"
+#define GD_MSG_DIR_OP_FAILED_STR                                               \
+    "The provided path is already present. It is not a directory"
+#define GD_MSG_BRICK_CREATION_FAIL_STR                                         \
+    "Brick isn't allowed to be created inside glusterd's working directory."
+#define GD_MSG_BRICK_CREATE_ROOT_STR                                           \
+    "The brick is being created in the root partition. It is recommended "     \
+    "that you don't use the system's root partition for storage backend."
+#define GD_MSG_BRICK_CREATE_MNTPNT_STR                                         \
+    "The brick is a mount point. Please create a sub-directory under the "     \
+    "mount point and use that as the brick directory."
+#define GD_MSG_CREATE_GLUSTER_DIR_FAILED_STR                                   \
+    "Failed to create glusterfs directory"
+#define GD_MSG_VOLINFO_IMPORT_FAIL_STR "Volume is not yet imported"
+#define GD_MSG_BRICK_SET_INFO_FAIL_STR                                         \
+    "Failed to add brick mount details to dict"
+#define GD_MSG_SET_XATTR_BRICK_FAIL_STR                                        \
+    "Glusterfs is not supported on brick. Setting extended attribute failed"
+#define GD_MSG_SET_XATTR_FAIL_STR "Failed to set extended attribute"
+#define GD_MSG_REMOVE_XATTR_FAIL_STR "Failed to remove extended attribute"
+#define GD_MSG_XLATOR_SET_OPT_FAIL_STR "Failed to set xlator type"
+#define GD_MSG_XLATOR_LINK_FAIL_STR                                            \
+    "Failed to do the link of xlator with children"
+#define GD_MSG_READ_ERROR_STR "Failed to read directory"
+#define GD_MSG_INCOMPATIBLE_VALUE_STR "Incompatible transport type"
+#define GD_MSG_VOL_STOP_ARGS_GET_FAILED_STR "Failed to get volume stop args"
+#define GD_MSG_FRAME_CREATE_FAIL_STR "Failed to create frame"
+#define GD_MSG_VOLUME_NOT_IMPORTED_STR "Volume has not been imported"
+#define GD_MSG_ADD_BRICK_MNT_INFO_FAIL_STR                                     \
+    "Failed to add brick mount details to dict"
+#define GD_MSG_GET_MNT_ENTRY_INFO_FAIL_STR "Failed to get mount entry details"
+#define GD_MSG_BRICKPATH_ROOT_GET_FAIL_STR "failed to get brick root details"
+#define GD_MSG_VOL_INFO_REQ_RECVD_STR "Received get volume info req"
+#define GD_MSG_NO_FLAG_SET_STR "No flags set"
+#define GD_MSG_CREATE_DIR_FAILED_STR "Failed to create directory"
+#define GD_MSG_POST_HOOK_STUB_INIT_FAIL_STR                                    \
+    "Failed to initialize post hooks stub"
+#define GD_MSG_FILE_OP_FAILED_STR "File operation failed"
+#define GD_MSG_INODE_SIZE_GET_FAIL_STR "Failed to get inode size"
+#define GD_MSG_CMD_EXEC_FAIL_STR "Command execution failed"
+#define GD_MSG_XLATOR_CREATE_FAIL_STR "Failed to create xlator"
+#define GD_MSG_CLRCLK_VOL_REQ_RCVD_STR "Received clear-locks request for volume"
+#define GD_MSG_BRK_PORT_NUM_GET_FAIL_STR                                       \
+    "Couldn't get port number of local bricks"
+#define GD_MSG_CLRLOCKS_MOUNTDIR_CREATE_FAIL_STR                               \
+    "Creating mount directory for clear-locks failed"
+#define GD_MSG_CLRLOCKS_CLNT_MOUNT_FAIL_STR                                    \
+    "Failed to mount clear-locks maintenance client"
+#define GD_MSG_CLRLOCKS_CLNT_UMOUNT_FAIL_STR                                   \
+    "Failed to unmount clear-locks mount point"
+#define GD_MSG_CLRCLK_SND_CMD_FAIL_STR "Failed to send command for clear-locks"
+#define GD_MSG_DICT_ALLOC_AND_SERL_LENGTH_GET_FAIL_STR                         \
+    "Failed to allocate memory or get serialized length of dict"
+#define GD_MSG_GET_XATTR_FAIL_STR "Failed to get extended attribute"
 
 #endif /* !_GLUSTERD_MESSAGES_H_ */
diff --git a/xlators/mgmt/glusterd/src/glusterd-mgmt-handler.c b/xlators/mgmt/glusterd/src/glusterd-mgmt-handler.c
index a200c8e1230..1069688a89d 100644
--- a/xlators/mgmt/glusterd/src/glusterd-mgmt-handler.c
+++ b/xlators/mgmt/glusterd/src/glusterd-mgmt-handler.c
@@ -19,946 +19,1126 @@
 #include "glusterd-locks.h"
 #include "glusterd-mgmt.h"
 #include "glusterd-op-sm.h"
+#include "glusterd-messages.h"
 
 static int
-glusterd_mgmt_v3_null (rpcsvc_request_t *req)
+glusterd_mgmt_v3_null(rpcsvc_request_t *req)
 {
-        return 0;
+    return 0;
 }
 
 static int
-glusterd_mgmt_v3_lock_send_resp (rpcsvc_request_t *req, int32_t status)
+glusterd_mgmt_v3_lock_send_resp(rpcsvc_request_t *req, int32_t status,
+                                uint32_t op_errno)
 {
+    gd1_mgmt_v3_lock_rsp rsp = {
+        {0},
+    };
+    int ret = -1;
+    xlator_t *this = NULL;
 
-        gd1_mgmt_v3_lock_rsp          rsp   = {{0},};
-        int                           ret   = -1;
-        xlator_t                     *this  = NULL;
+    this = THIS;
+    GF_ASSERT(this);
+    GF_ASSERT(req);
 
-        this = THIS;
-        GF_ASSERT (this);
-        GF_ASSERT (req);
+    rsp.op_ret = status;
+    if (rsp.op_ret)
+        rsp.op_errno = op_errno;
 
-        rsp.op_ret = status;
-        if (rsp.op_ret)
-                rsp.op_errno = errno;
+    glusterd_get_uuid(&rsp.uuid);
 
-        glusterd_get_uuid (&rsp.uuid);
+    ret = glusterd_submit_reply(req, &rsp, NULL, 0, NULL,
+                                (xdrproc_t)xdr_gd1_mgmt_v3_lock_rsp);
 
-        ret = glusterd_submit_reply (req, &rsp, NULL, 0, NULL,
-                                     (xdrproc_t)xdr_gd1_mgmt_v3_lock_rsp);
+    gf_msg_debug(this->name, 0, "Responded to mgmt_v3 lock, ret: %d", ret);
 
-        gf_log (this->name, GF_LOG_DEBUG,
-                "Responded to mgmt_v3 lock, ret: %d", ret);
-
-        return ret;
+    return ret;
 }
 
 static int
-glusterd_synctasked_mgmt_v3_lock (rpcsvc_request_t *req,
-                                gd1_mgmt_v3_lock_req *lock_req,
-                                glusterd_op_lock_ctx_t *ctx)
+glusterd_synctasked_mgmt_v3_lock(rpcsvc_request_t *req,
+                                 gd1_mgmt_v3_lock_req *lock_req,
+                                 glusterd_op_lock_ctx_t *ctx)
 {
-        int32_t                         ret         = -1;
-        xlator_t                       *this        = NULL;
-
-        this = THIS;
-        GF_ASSERT (this);
-        GF_ASSERT (req);
-        GF_ASSERT (ctx);
-        GF_ASSERT (ctx->dict);
-
-        /* Trying to acquire multiple mgmt_v3 locks */
-        ret = glusterd_multiple_mgmt_v3_lock (ctx->dict, ctx->uuid);
-        if (ret)
-                gf_log (this->name, GF_LOG_ERROR,
-                        "Failed to acquire mgmt_v3 locks for %s",
-                         uuid_utoa (ctx->uuid));
-
-        ret = glusterd_mgmt_v3_lock_send_resp (req, ret);
-
-        gf_log (this->name, GF_LOG_TRACE, "Returning %d", ret);
-        return ret;
+    int32_t ret = -1;
+    xlator_t *this = NULL;
+    uint32_t op_errno = 0;
+
+    this = THIS;
+    GF_ASSERT(this);
+    GF_ASSERT(req);
+    GF_ASSERT(ctx);
+    GF_ASSERT(ctx->dict);
+
+    /* Trying to acquire multiple mgmt_v3 locks */
+    ret = glusterd_multiple_mgmt_v3_lock(ctx->dict, ctx->uuid, &op_errno);
+    if (ret)
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_MGMTV3_LOCK_GET_FAIL,
+               "Failed to acquire mgmt_v3 locks for %s", uuid_utoa(ctx->uuid));
+
+    ret = glusterd_mgmt_v3_lock_send_resp(req, ret, op_errno);
+
+    gf_msg_trace(this->name, 0, "Returning %d", ret);
+    return ret;
 }
 
 static int
-glusterd_op_state_machine_mgmt_v3_lock (rpcsvc_request_t *req,
+glusterd_op_state_machine_mgmt_v3_lock(rpcsvc_request_t *req,
                                        gd1_mgmt_v3_lock_req *lock_req,
                                        glusterd_op_lock_ctx_t *ctx)
 {
-        int32_t                         ret         = -1;
-        xlator_t                       *this        = NULL;
-        glusterd_op_info_t              txn_op_info = {{0},};
-
-        this = THIS;
-        GF_ASSERT (this);
-        GF_ASSERT (req);
-
-        glusterd_txn_opinfo_init (&txn_op_info, NULL, &lock_req->op,
-                                  ctx->dict, req);
-
-        ret = glusterd_set_txn_opinfo (&lock_req->txn_id, &txn_op_info);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "Unable to set transaction's opinfo");
-                goto out;
-        }
-
-        ret = glusterd_op_sm_inject_event (GD_OP_EVENT_LOCK,
-                                           &lock_req->txn_id, ctx);
-        if (ret)
-                gf_log (this->name, GF_LOG_ERROR,
-                        "Failed to inject event GD_OP_EVENT_LOCK");
+    int32_t ret = -1;
+    xlator_t *this = NULL;
+    glusterd_op_info_t txn_op_info = {
+        {0},
+    };
+
+    this = THIS;
+    GF_ASSERT(this);
+    GF_ASSERT(req);
+
+    glusterd_txn_opinfo_init(&txn_op_info, NULL, &lock_req->op, ctx->dict, req);
+
+    ret = glusterd_set_txn_opinfo(&lock_req->txn_id, &txn_op_info);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_OPINFO_SET_FAIL,
+               "Unable to set transaction's opinfo");
+        goto out;
+    }
+
+    ret = glusterd_op_sm_inject_event(GD_OP_EVENT_LOCK, &lock_req->txn_id, ctx);
+    if (ret)
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_OP_EVENT_LOCK_FAIL,
+               "Failed to inject event GD_OP_EVENT_LOCK");
 
 out:
-        glusterd_friend_sm ();
-        glusterd_op_sm ();
+    glusterd_friend_sm();
+    glusterd_op_sm();
 
-        gf_log (this->name, GF_LOG_TRACE, "Returning %d", ret);
-        return ret;
+    gf_msg_trace(this->name, 0, "Returning %d", ret);
+    return ret;
 }
 
 static int
-glusterd_handle_mgmt_v3_lock_fn (rpcsvc_request_t *req)
+glusterd_handle_mgmt_v3_lock_fn(rpcsvc_request_t *req)
 {
-        gd1_mgmt_v3_lock_req    lock_req        = {{0},};
-        int32_t                 ret             = -1;
-        glusterd_op_lock_ctx_t *ctx             = NULL;
-        xlator_t               *this            = NULL;
-        gf_boolean_t            is_synctasked   = _gf_false;
-        gf_boolean_t            free_ctx        = _gf_false;
-
-        this = THIS;
-        GF_ASSERT (this);
-        GF_ASSERT (req);
-
-        ret = xdr_to_generic (req->msg[0], &lock_req,
-                              (xdrproc_t)xdr_gd1_mgmt_v3_lock_req);
-        if (ret < 0) {
-                gf_log (this->name, GF_LOG_ERROR, "Failed to decode lock "
-                        "request received from peer");
-                req->rpc_err = GARBAGE_ARGS;
-                goto out;
-        }
-
-        gf_log (this->name, GF_LOG_DEBUG, "Received mgmt_v3 lock req "
-                "from uuid: %s", uuid_utoa (lock_req.uuid));
-
-        if (glusterd_peerinfo_find_by_uuid (lock_req.uuid) == NULL) {
-                gf_log (this->name, GF_LOG_WARNING, "%s doesn't "
-                        "belong to the cluster. Ignoring request.",
-                        uuid_utoa (lock_req.uuid));
-                ret = -1;
-                goto out;
-        }
-
-        ctx = GF_CALLOC (1, sizeof (*ctx), gf_gld_mt_op_lock_ctx_t);
-        if (!ctx) {
-                ret = -1;
-                goto out;
-        }
-
-        gf_uuid_copy (ctx->uuid, lock_req.uuid);
-        ctx->req = req;
-
-        ctx->dict = dict_new ();
-        if (!ctx->dict) {
-                ret = -1;
-                goto out;
-        }
-
-        ret = dict_unserialize (lock_req.dict.dict_val,
-                                lock_req.dict.dict_len, &ctx->dict);
+    gd1_mgmt_v3_lock_req lock_req = {
+        {0},
+    };
+    int32_t ret = -1;
+    glusterd_op_lock_ctx_t *ctx = NULL;
+    xlator_t *this = NULL;
+    gf_boolean_t is_synctasked = _gf_false;
+    gf_boolean_t free_ctx = _gf_false;
+    glusterd_conf_t *conf = NULL;
+    uint32_t timeout = 0;
+
+    this = THIS;
+    conf = this->private;
+    GF_ASSERT(conf);
+    GF_ASSERT(this);
+    GF_ASSERT(req);
+
+    ret = xdr_to_generic(req->msg[0], &lock_req,
+                         (xdrproc_t)xdr_gd1_mgmt_v3_lock_req);
+    if (ret < 0) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_REQ_DECODE_FAIL,
+               "Failed to decode lock "
+               "request received from peer");
+        req->rpc_err = GARBAGE_ARGS;
+        goto out;
+    }
+
+    gf_msg_debug(this->name, 0,
+                 "Received mgmt_v3 lock req "
+                 "from uuid: %s",
+                 uuid_utoa(lock_req.uuid));
+
+    if (glusterd_peerinfo_find_by_uuid(lock_req.uuid) == NULL) {
+        gf_msg(this->name, GF_LOG_WARNING, 0, GD_MSG_PEER_NOT_FOUND,
+               "%s doesn't "
+               "belong to the cluster. Ignoring request.",
+               uuid_utoa(lock_req.uuid));
+        ret = -1;
+        goto out;
+    }
+
+    ctx = GF_CALLOC(1, sizeof(*ctx), gf_gld_mt_op_lock_ctx_t);
+    if (!ctx) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_NO_MEMORY, NULL);
+        ret = -1;
+        goto out;
+    }
+
+    gf_uuid_copy(ctx->uuid, lock_req.uuid);
+    ctx->req = req;
+
+    ctx->dict = dict_new();
+    if (!ctx->dict) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_CREATE_FAIL, NULL);
+        ret = -1;
+        goto out;
+    }
+
+    ret = dict_unserialize(lock_req.dict.dict_val, lock_req.dict.dict_len,
+                           &ctx->dict);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_WARNING, 0, GD_MSG_DICT_UNSERIALIZE_FAIL,
+                NULL);
+        goto out;
+    }
+
+    /* Cli will add timeout key to dict if the default timeout is
+     * other than 2 minutes. Here we use this value to check whether
+     * mgmt_v3_lock_timeout should be set to default value or we
+     * need to change the value according to timeout value
+     * i.e, timeout + 120 seconds. */
+    ret = dict_get_uint32(ctx->dict, "timeout", &timeout);
+    if (!ret)
+        conf->mgmt_v3_lock_timeout = timeout + 120;
+
+    is_synctasked = dict_get_str_boolean(ctx->dict, "is_synctasked", _gf_false);
+    if (is_synctasked) {
+        ret = glusterd_synctasked_mgmt_v3_lock(req, &lock_req, ctx);
         if (ret) {
-                gf_log (this->name, GF_LOG_WARNING,
-                        "failed to unserialize the dictionary");
-                goto out;
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_MGMTV3_LOCK_GET_FAIL,
+                   "Failed to acquire mgmt_v3_locks");
+            /* Ignore the return code, as it shouldn't be propagated
+             * from the handler function so as to avoid double
+             * deletion of the req
+             */
+            ret = 0;
         }
 
-        is_synctasked = dict_get_str_boolean (ctx->dict,
-                                              "is_synctasked", _gf_false);
-        if (is_synctasked) {
-                ret = glusterd_synctasked_mgmt_v3_lock (req, &lock_req, ctx);
-                if (ret) {
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "Failed to acquire mgmt_v3_locks");
-                        /* Ignore the return code, as it shouldn't be propagated
-                         * from the handler function so as to avoid double
-                         * deletion of the req
-                         */
-                        ret = 0;
-                }
-
-                /* The above function does not take ownership of ctx.
-                 * Therefore we need to free the ctx explicitly. */
-                free_ctx = _gf_true;
-        }
-        else {
-                /* Shouldn't ignore the return code here, and it should
-                 * be propagated from the handler function as in failure
-                 * case it doesn't delete the req object
-                 */
-                ret = glusterd_op_state_machine_mgmt_v3_lock (req, &lock_req,
-                                                              ctx);
-                if (ret)
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "Failed to acquire mgmt_v3_locks");
-        }
+        /* The above function does not take ownership of ctx.
+         * Therefore we need to free the ctx explicitly. */
+        free_ctx = _gf_true;
+    } else {
+        /* Shouldn't ignore the return code here, and it should
+         * be propagated from the handler function as in failure
+         * case it doesn't delete the req object
+         */
+        ret = glusterd_op_state_machine_mgmt_v3_lock(req, &lock_req, ctx);
+        if (ret)
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_MGMTV3_LOCK_GET_FAIL,
+                   "Failed to acquire mgmt_v3_locks");
+    }
 
 out:
 
-        if (ctx && (ret || free_ctx)) {
-                if (ctx->dict)
-                        dict_unref (ctx->dict);
+    if (ctx && (ret || free_ctx)) {
+        if (ctx->dict)
+            dict_unref(ctx->dict);
 
-                GF_FREE (ctx);
-        }
+        GF_FREE(ctx);
+    }
 
-        free (lock_req.dict.dict_val);
+    free(lock_req.dict.dict_val);
 
-        gf_log (this->name, GF_LOG_TRACE, "Returning %d", ret);
-        return ret;
+    gf_msg_trace(this->name, 0, "Returning %d", ret);
+    return ret;
 }
 
 static int
-glusterd_mgmt_v3_pre_validate_send_resp (rpcsvc_request_t   *req,
-                                        int32_t op, int32_t status,
-                                        char *op_errstr, dict_t *rsp_dict)
+glusterd_mgmt_v3_pre_validate_send_resp(rpcsvc_request_t *req, int32_t op,
+                                        int32_t status, char *op_errstr,
+                                        dict_t *rsp_dict, uint32_t op_errno)
 {
-        gd1_mgmt_v3_pre_val_rsp          rsp      = {{0},};
-        int                             ret      = -1;
-        xlator_t                       *this     = NULL;
-
-        this = THIS;
-        GF_ASSERT (this);
-        GF_ASSERT (req);
-
-        rsp.op_ret = status;
-        glusterd_get_uuid (&rsp.uuid);
-        rsp.op = op;
-        if (op_errstr)
-                rsp.op_errstr = op_errstr;
-        else
-                rsp.op_errstr = "";
-
-        ret = dict_allocate_and_serialize (rsp_dict, &rsp.dict.dict_val,
-                                           &rsp.dict.dict_len);
-        if (ret < 0) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "failed to get serialized length of dict");
-                goto out;
-        }
-
-        ret = glusterd_submit_reply (req, &rsp, NULL, 0, NULL,
-                                     (xdrproc_t)xdr_gd1_mgmt_v3_pre_val_rsp);
-
-        GF_FREE (rsp.dict.dict_val);
+    gd1_mgmt_v3_pre_val_rsp rsp = {
+        {0},
+    };
+    int ret = -1;
+    xlator_t *this = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+    GF_ASSERT(req);
+
+    rsp.op_ret = status;
+    glusterd_get_uuid(&rsp.uuid);
+    rsp.op = op;
+    rsp.op_errno = op_errno;
+    if (op_errstr)
+        rsp.op_errstr = op_errstr;
+    else
+        rsp.op_errstr = "";
+
+    ret = dict_allocate_and_serialize(rsp_dict, &rsp.dict.dict_val,
+                                      &rsp.dict.dict_len);
+    if (ret < 0) {
+        gf_smsg(this->name, GF_LOG_ERROR, 0,
+                GD_MSG_DICT_ALLOC_AND_SERL_LENGTH_GET_FAIL, NULL);
+        goto out;
+    }
+
+    ret = glusterd_submit_reply(req, &rsp, NULL, 0, NULL,
+                                (xdrproc_t)xdr_gd1_mgmt_v3_pre_val_rsp);
+
+    GF_FREE(rsp.dict.dict_val);
 out:
-        gf_log (this->name, GF_LOG_DEBUG,
-                "Responded to pre validation, ret: %d", ret);
-        return ret;
+    gf_msg_debug(this->name, 0, "Responded to pre validation, ret: %d", ret);
+    return ret;
 }
 
 static int
-glusterd_handle_pre_validate_fn (rpcsvc_request_t *req)
+glusterd_handle_pre_validate_fn(rpcsvc_request_t *req)
 {
-        int32_t                         ret       = -1;
-        gd1_mgmt_v3_pre_val_req          op_req    = {{0},};
-        xlator_t                       *this      = NULL;
-        char                           *op_errstr = NULL;
-        dict_t                         *dict      = NULL;
-        dict_t                         *rsp_dict  = NULL;
-
-        this = THIS;
-        GF_ASSERT (this);
-        GF_ASSERT (req);
-
-        ret = xdr_to_generic (req->msg[0], &op_req,
-                              (xdrproc_t)xdr_gd1_mgmt_v3_pre_val_req);
-        if (ret < 0) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "Failed to decode pre validation "
-                        "request received from peer");
-                req->rpc_err = GARBAGE_ARGS;
-                goto out;
-        }
-
-        if (glusterd_peerinfo_find_by_uuid (op_req.uuid) == NULL) {
-                gf_log (this->name, GF_LOG_WARNING, "%s doesn't "
-                        "belong to the cluster. Ignoring request.",
-                        uuid_utoa (op_req.uuid));
-                ret = -1;
-                goto out;
-        }
-
-        dict = dict_new ();
-        if (!dict)
-                goto out;
-
-        ret = dict_unserialize (op_req.dict.dict_val,
-                                op_req.dict.dict_len, &dict);
-        if (ret) {
-                gf_log (this->name, GF_LOG_WARNING,
-                        "failed to unserialize the dictionary");
-                goto out;
-        }
-
-        rsp_dict = dict_new ();
-        if (!rsp_dict) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "Failed to get new dictionary");
-                return -1;
-        }
-
-        ret = gd_mgmt_v3_pre_validate_fn (op_req.op, dict, &op_errstr,
-                                         rsp_dict);
-
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "Pre Validation failed on operation %s",
-                        gd_op_list[op_req.op]);
-        }
-
-        ret = glusterd_mgmt_v3_pre_validate_send_resp (req, op_req.op,
-                                                      ret, op_errstr,
-                                                      rsp_dict);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "Failed to send Pre Validation "
-                        "response for operation %s",
-                        gd_op_list[op_req.op]);
-                goto out;
-        }
+    int32_t ret = -1;
+    gd1_mgmt_v3_pre_val_req op_req = {
+        {0},
+    };
+    xlator_t *this = NULL;
+    char *op_errstr = NULL;
+    dict_t *dict = NULL;
+    dict_t *rsp_dict = NULL;
+    uint32_t op_errno = 0;
+
+    this = THIS;
+    GF_ASSERT(this);
+    GF_ASSERT(req);
+
+    ret = xdr_to_generic(req->msg[0], &op_req,
+                         (xdrproc_t)xdr_gd1_mgmt_v3_pre_val_req);
+    if (ret < 0) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_REQ_DECODE_FAIL,
+               "Failed to decode pre validation "
+               "request received from peer");
+        req->rpc_err = GARBAGE_ARGS;
+        goto out;
+    }
+
+    if (glusterd_peerinfo_find_by_uuid(op_req.uuid) == NULL) {
+        gf_msg(this->name, GF_LOG_WARNING, 0, GD_MSG_PEER_NOT_FOUND,
+               "%s doesn't "
+               "belong to the cluster. Ignoring request.",
+               uuid_utoa(op_req.uuid));
+        ret = -1;
+        goto out;
+    }
+
+    dict = dict_new();
+    if (!dict) {
+        gf_smsg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_CREATE_FAIL, NULL);
+        goto out;
+    }
+
+    ret = dict_unserialize(op_req.dict.dict_val, op_req.dict.dict_len, &dict);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_WARNING, 0, GD_MSG_DICT_UNSERIALIZE_FAIL,
+                NULL);
+        goto out;
+    }
+
+    rsp_dict = dict_new();
+    if (!rsp_dict) {
+        gf_smsg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_CREATE_FAIL, NULL);
+        return -1;
+    }
+
+    ret = gd_mgmt_v3_pre_validate_fn(op_req.op, dict, &op_errstr, rsp_dict,
+                                     &op_errno);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_PRE_VALIDATION_FAIL,
+               "Pre Validation failed on operation %s", gd_op_list[op_req.op]);
+    }
+
+    ret = glusterd_mgmt_v3_pre_validate_send_resp(
+        req, op_req.op, ret, op_errstr, rsp_dict, op_errno);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_MGMTV3_OP_RESP_FAIL,
+               "Failed to send Pre Validation "
+               "response for operation %s",
+               gd_op_list[op_req.op]);
+        goto out;
+    }
 
 out:
-        if (op_errstr && (strcmp (op_errstr, "")))
-                GF_FREE (op_errstr);
+    if (op_errstr && (strcmp(op_errstr, "")))
+        GF_FREE(op_errstr);
 
-        free (op_req.dict.dict_val);
+    free(op_req.dict.dict_val);
 
-        if (dict)
-                dict_unref (dict);
+    if (dict)
+        dict_unref(dict);
 
-        if (rsp_dict)
-                dict_unref (rsp_dict);
+    if (rsp_dict)
+        dict_unref(rsp_dict);
 
-        /* Return 0 from handler to avoid double deletion of req obj */
-        return 0;
+    /* Return 0 from handler to avoid double deletion of req obj */
+    return 0;
 }
 
 static int
-glusterd_mgmt_v3_brick_op_send_resp (rpcsvc_request_t   *req,
-                                    int32_t op, int32_t status,
-                                    char *op_errstr, dict_t *rsp_dict)
+glusterd_mgmt_v3_brick_op_send_resp(rpcsvc_request_t *req, int32_t op,
+                                    int32_t status, char *op_errstr,
+                                    dict_t *rsp_dict)
 {
-        gd1_mgmt_v3_brick_op_rsp         rsp      = {{0},};
-        int                             ret      = -1;
-        xlator_t                       *this     = NULL;
-
-        this = THIS;
-        GF_ASSERT (this);
-        GF_ASSERT (req);
-
-        rsp.op_ret = status;
-        glusterd_get_uuid (&rsp.uuid);
-        rsp.op = op;
-        if (op_errstr)
-                rsp.op_errstr = op_errstr;
-        else
-                rsp.op_errstr = "";
-
-        ret = dict_allocate_and_serialize (rsp_dict, &rsp.dict.dict_val,
-                                           &rsp.dict.dict_len);
-        if (ret < 0) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "failed to get serialized length of dict");
-                goto out;
-        }
-
-        ret = glusterd_submit_reply (req, &rsp, NULL, 0, NULL,
-                                     (xdrproc_t)xdr_gd1_mgmt_v3_brick_op_rsp);
-
-        GF_FREE (rsp.dict.dict_val);
+    gd1_mgmt_v3_brick_op_rsp rsp = {
+        {0},
+    };
+    int ret = -1;
+    xlator_t *this = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+    GF_ASSERT(req);
+
+    rsp.op_ret = status;
+    glusterd_get_uuid(&rsp.uuid);
+    rsp.op = op;
+    if (op_errstr)
+        rsp.op_errstr = op_errstr;
+    else
+        rsp.op_errstr = "";
+
+    ret = dict_allocate_and_serialize(rsp_dict, &rsp.dict.dict_val,
+                                      &rsp.dict.dict_len);
+    if (ret < 0) {
+        gf_smsg(this->name, GF_LOG_ERROR, 0,
+                GD_MSG_DICT_ALLOC_AND_SERL_LENGTH_GET_FAIL, NULL);
+        goto out;
+    }
+
+    ret = glusterd_submit_reply(req, &rsp, NULL, 0, NULL,
+                                (xdrproc_t)xdr_gd1_mgmt_v3_brick_op_rsp);
+
+    GF_FREE(rsp.dict.dict_val);
 out:
-        gf_log (this->name, GF_LOG_DEBUG,
-                "Responded to brick op, ret: %d", ret);
-        return ret;
+    gf_msg_debug(this->name, 0, "Responded to brick op, ret: %d", ret);
+    return ret;
 }
 
 static int
-glusterd_handle_brick_op_fn (rpcsvc_request_t *req)
+glusterd_handle_brick_op_fn(rpcsvc_request_t *req)
 {
-        int32_t                         ret       = -1;
-        gd1_mgmt_v3_brick_op_req         op_req    = {{0},};
-        xlator_t                       *this      = NULL;
-        char                           *op_errstr = NULL;
-        dict_t                         *dict      = NULL;
-        dict_t                         *rsp_dict  = NULL;
-
-        this = THIS;
-        GF_ASSERT (this);
-        GF_ASSERT (req);
-
-        ret = xdr_to_generic (req->msg[0], &op_req,
-                              (xdrproc_t)xdr_gd1_mgmt_v3_brick_op_req);
-        if (ret < 0) {
-                gf_log (this->name, GF_LOG_ERROR, "Failed to decode brick op "
-                        "request received from peer");
-                req->rpc_err = GARBAGE_ARGS;
-                goto out;
-        }
-
-        if (glusterd_peerinfo_find_by_uuid (op_req.uuid) == NULL) {
-                gf_log (this->name, GF_LOG_WARNING, "%s doesn't "
-                        "belong to the cluster. Ignoring request.",
-                        uuid_utoa (op_req.uuid));
-                ret = -1;
-                goto out;
-        }
-
-        dict = dict_new ();
-        if (!dict)
-                goto out;
-
-        ret = dict_unserialize (op_req.dict.dict_val,
-                                op_req.dict.dict_len, &dict);
-        if (ret) {
-                gf_log (this->name, GF_LOG_WARNING,
-                        "failed to unserialize the dictionary");
-                goto out;
-        }
-
-        rsp_dict = dict_new ();
-        if (!rsp_dict) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "Failed to get new dictionary");
-                return -1;
-        }
-
-        ret = gd_mgmt_v3_brick_op_fn (op_req.op, dict, &op_errstr,
-                                     rsp_dict);
-
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "Brick Op failed on operation %s",
-                        gd_op_list[op_req.op]);
-        }
-
-        ret = glusterd_mgmt_v3_brick_op_send_resp (req, op_req.op,
-                                                  ret, op_errstr,
-                                                  rsp_dict);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "Failed to send brick op "
-                        "response for operation %s",
-                        gd_op_list[op_req.op]);
-                goto out;
-        }
+    int32_t ret = -1;
+    gd1_mgmt_v3_brick_op_req op_req = {
+        {0},
+    };
+    xlator_t *this = NULL;
+    char *op_errstr = NULL;
+    dict_t *dict = NULL;
+    dict_t *rsp_dict = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+    GF_ASSERT(req);
+
+    ret = xdr_to_generic(req->msg[0], &op_req,
+                         (xdrproc_t)xdr_gd1_mgmt_v3_brick_op_req);
+    if (ret < 0) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_REQ_DECODE_FAIL,
+               "Failed to decode brick op "
+               "request received from peer");
+        req->rpc_err = GARBAGE_ARGS;
+        goto out;
+    }
+
+    if (glusterd_peerinfo_find_by_uuid(op_req.uuid) == NULL) {
+        gf_msg(this->name, GF_LOG_WARNING, 0, GD_MSG_PEER_NOT_FOUND,
+               "%s doesn't "
+               "belong to the cluster. Ignoring request.",
+               uuid_utoa(op_req.uuid));
+        ret = -1;
+        goto out;
+    }
+
+    dict = dict_new();
+    if (!dict) {
+        gf_smsg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_CREATE_FAIL, NULL);
+        goto out;
+    }
+
+    ret = dict_unserialize(op_req.dict.dict_val, op_req.dict.dict_len, &dict);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_WARNING, 0, GD_MSG_DICT_UNSERIALIZE_FAIL,
+                NULL);
+        goto out;
+    }
+
+    rsp_dict = dict_new();
+    if (!rsp_dict) {
+        gf_smsg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_CREATE_FAIL, NULL);
+        return -1;
+    }
+
+    ret = gd_mgmt_v3_brick_op_fn(op_req.op, dict, &op_errstr, rsp_dict);
+
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_BRICK_OP_FAIL,
+               "Brick Op failed on operation %s", gd_op_list[op_req.op]);
+    }
+
+    ret = glusterd_mgmt_v3_brick_op_send_resp(req, op_req.op, ret, op_errstr,
+                                              rsp_dict);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_PRE_VALD_RESP_FAIL,
+               "Failed to send brick op "
+               "response for operation %s",
+               gd_op_list[op_req.op]);
+        goto out;
+    }
 
 out:
-        if (op_errstr && (strcmp (op_errstr, "")))
-                GF_FREE (op_errstr);
+    if (op_errstr && (strcmp(op_errstr, "")))
+        GF_FREE(op_errstr);
 
-        free (op_req.dict.dict_val);
+    free(op_req.dict.dict_val);
 
-        if (dict)
-                dict_unref (dict);
+    if (dict)
+        dict_unref(dict);
 
-        if (rsp_dict)
-                dict_unref (rsp_dict);
+    if (rsp_dict)
+        dict_unref(rsp_dict);
 
-        /* Return 0 from handler to avoid double deletion of req obj */
-        return 0;
+    /* Return 0 from handler to avoid double deletion of req obj */
+    return 0;
 }
 
 static int
-glusterd_mgmt_v3_commit_send_resp (rpcsvc_request_t   *req,
-                                  int32_t op, int32_t status,
-                                  char *op_errstr, dict_t *rsp_dict)
+glusterd_mgmt_v3_commit_send_resp(rpcsvc_request_t *req, int32_t op,
+                                  int32_t status, char *op_errstr,
+                                  uint32_t op_errno, dict_t *rsp_dict)
 {
-        gd1_mgmt_v3_commit_rsp           rsp      = {{0},};
-        int                             ret      = -1;
-        xlator_t                       *this     = NULL;
-
-        this = THIS;
-        GF_ASSERT (this);
-        GF_ASSERT (req);
-
-        rsp.op_ret = status;
-        glusterd_get_uuid (&rsp.uuid);
-        rsp.op = op;
-        if (op_errstr)
-                rsp.op_errstr = op_errstr;
-        else
-                rsp.op_errstr = "";
-
-        ret = dict_allocate_and_serialize (rsp_dict, &rsp.dict.dict_val,
-                                           &rsp.dict.dict_len);
-        if (ret < 0) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "failed to get serialized length of dict");
-                goto out;
-        }
-
-        ret = glusterd_submit_reply (req, &rsp, NULL, 0, NULL,
-                                     (xdrproc_t)xdr_gd1_mgmt_v3_commit_rsp);
-
-        GF_FREE (rsp.dict.dict_val);
+    gd1_mgmt_v3_commit_rsp rsp = {
+        {0},
+    };
+    int ret = -1;
+    xlator_t *this = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+    GF_ASSERT(req);
+
+    rsp.op_ret = status;
+    glusterd_get_uuid(&rsp.uuid);
+    rsp.op = op;
+    rsp.op_errno = op_errno;
+    if (op_errstr)
+        rsp.op_errstr = op_errstr;
+    else
+        rsp.op_errstr = "";
+
+    ret = dict_allocate_and_serialize(rsp_dict, &rsp.dict.dict_val,
+                                      &rsp.dict.dict_len);
+    if (ret < 0) {
+        gf_smsg(this->name, GF_LOG_ERROR, 0,
+                GD_MSG_DICT_ALLOC_AND_SERL_LENGTH_GET_FAIL, NULL);
+        goto out;
+    }
+
+    ret = glusterd_submit_reply(req, &rsp, NULL, 0, NULL,
+                                (xdrproc_t)xdr_gd1_mgmt_v3_commit_rsp);
+
+    GF_FREE(rsp.dict.dict_val);
 out:
-        gf_log (this->name, GF_LOG_DEBUG, "Responded to commit, ret: %d", ret);
-        return ret;
+    gf_msg_debug(this->name, 0, "Responded to commit, ret: %d", ret);
+    return ret;
 }
 
 static int
-glusterd_handle_commit_fn (rpcsvc_request_t *req)
+glusterd_handle_commit_fn(rpcsvc_request_t *req)
 {
-        int32_t                         ret       = -1;
-        gd1_mgmt_v3_commit_req           op_req    = {{0},};
-        xlator_t                       *this      = NULL;
-        char                           *op_errstr = NULL;
-        dict_t                         *dict      = NULL;
-        dict_t                         *rsp_dict  = NULL;
-
-        this = THIS;
-        GF_ASSERT (this);
-        GF_ASSERT (req);
-
-        ret = xdr_to_generic (req->msg[0], &op_req,
-                              (xdrproc_t)xdr_gd1_mgmt_v3_commit_req);
-        if (ret < 0) {
-                gf_log (this->name, GF_LOG_ERROR, "Failed to decode commit "
-                        "request received from peer");
-                req->rpc_err = GARBAGE_ARGS;
-                goto out;
-        }
-
-        if (glusterd_peerinfo_find_by_uuid (op_req.uuid) == NULL) {
-                gf_log (this->name, GF_LOG_WARNING, "%s doesn't "
-                        "belong to the cluster. Ignoring request.",
-                        uuid_utoa (op_req.uuid));
-                ret = -1;
-                goto out;
-        }
-
-        dict = dict_new ();
-        if (!dict)
-                goto out;
-
-        ret = dict_unserialize (op_req.dict.dict_val,
-                                op_req.dict.dict_len, &dict);
-        if (ret) {
-                gf_log (this->name, GF_LOG_WARNING,
-                        "failed to unserialize the dictionary");
-                goto out;
-        }
-
-        rsp_dict = dict_new ();
-        if (!rsp_dict) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "Failed to get new dictionary");
-                return -1;
-        }
-
-        ret = gd_mgmt_v3_commit_fn (op_req.op, dict, &op_errstr,
-                                   rsp_dict);
-
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "commit failed on operation %s",
-                        gd_op_list[op_req.op]);
-        }
-
-        ret = glusterd_mgmt_v3_commit_send_resp (req, op_req.op,
-                                                ret, op_errstr,
-                                                rsp_dict);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "Failed to send commit "
-                        "response for operation %s",
-                        gd_op_list[op_req.op]);
-                goto out;
-        }
+    int32_t ret = -1;
+    gd1_mgmt_v3_commit_req op_req = {
+        {0},
+    };
+    xlator_t *this = NULL;
+    char *op_errstr = NULL;
+    dict_t *dict = NULL;
+    dict_t *rsp_dict = NULL;
+    uint32_t op_errno = 0;
+
+    this = THIS;
+    GF_ASSERT(this);
+    GF_ASSERT(req);
+
+    ret = xdr_to_generic(req->msg[0], &op_req,
+                         (xdrproc_t)xdr_gd1_mgmt_v3_commit_req);
+    if (ret < 0) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_REQ_DECODE_FAIL,
+               "Failed to decode commit "
+               "request received from peer");
+        req->rpc_err = GARBAGE_ARGS;
+        goto out;
+    }
+
+    if (glusterd_peerinfo_find_by_uuid(op_req.uuid) == NULL) {
+        gf_msg(this->name, GF_LOG_WARNING, 0, GD_MSG_PEER_NOT_FOUND,
+               "%s doesn't "
+               "belong to the cluster. Ignoring request.",
+               uuid_utoa(op_req.uuid));
+        ret = -1;
+        goto out;
+    }
+
+    dict = dict_new();
+    if (!dict) {
+        gf_smsg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_CREATE_FAIL, NULL);
+        goto out;
+    }
+
+    ret = dict_unserialize(op_req.dict.dict_val, op_req.dict.dict_len, &dict);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_WARNING, 0, GD_MSG_DICT_UNSERIALIZE_FAIL,
+                NULL);
+        goto out;
+    }
+
+    rsp_dict = dict_new();
+    if (!rsp_dict) {
+        gf_smsg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_CREATE_FAIL, NULL);
+        return -1;
+    }
+
+    ret = gd_mgmt_v3_commit_fn(op_req.op, dict, &op_errstr, &op_errno,
+                               rsp_dict);
+
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_COMMIT_OP_FAIL,
+               "commit failed on operation %s", gd_op_list[op_req.op]);
+    }
+
+    ret = glusterd_mgmt_v3_commit_send_resp(req, op_req.op, ret, op_errstr,
+                                            op_errno, rsp_dict);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_MGMTV3_OP_RESP_FAIL,
+               "Failed to send commit "
+               "response for operation %s",
+               gd_op_list[op_req.op]);
+        goto out;
+    }
 
 out:
-        if (op_errstr && (strcmp (op_errstr, "")))
-                GF_FREE (op_errstr);
+    if (op_errstr && (strcmp(op_errstr, "")))
+        GF_FREE(op_errstr);
 
-        free (op_req.dict.dict_val);
+    free(op_req.dict.dict_val);
 
-        if (dict)
-                dict_unref (dict);
+    if (dict)
+        dict_unref(dict);
 
-        if (rsp_dict)
-                dict_unref (rsp_dict);
+    if (rsp_dict)
+        dict_unref(rsp_dict);
 
-        /* Return 0 from handler to avoid double deletion of req obj */
-        return 0;
+    /* Return 0 from handler to avoid double deletion of req obj */
+    return 0;
 }
 
 static int
-glusterd_mgmt_v3_post_validate_send_resp (rpcsvc_request_t   *req,
-                                         int32_t op, int32_t status,
-                                         char *op_errstr, dict_t *rsp_dict)
+glusterd_mgmt_v3_post_commit_send_resp(rpcsvc_request_t *req, int32_t op,
+                                       int32_t status, char *op_errstr,
+                                       uint32_t op_errno, dict_t *rsp_dict)
 {
-        gd1_mgmt_v3_post_val_rsp         rsp      = {{0},};
-        int                             ret      = -1;
-        xlator_t                       *this     = NULL;
-
-        this = THIS;
-        GF_ASSERT (this);
-        GF_ASSERT (req);
-
-        rsp.op_ret = status;
-        glusterd_get_uuid (&rsp.uuid);
-        rsp.op = op;
-        if (op_errstr)
-                rsp.op_errstr = op_errstr;
-        else
-                rsp.op_errstr = "";
-
-        ret = dict_allocate_and_serialize (rsp_dict, &rsp.dict.dict_val,
-                                           &rsp.dict.dict_len);
-        if (ret < 0) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "failed to get serialized length of dict");
-                goto out;
-        }
-
-        ret = glusterd_submit_reply (req, &rsp, NULL, 0, NULL,
-                                     (xdrproc_t)xdr_gd1_mgmt_v3_post_val_rsp);
-
-        GF_FREE (rsp.dict.dict_val);
+    gd1_mgmt_v3_post_commit_rsp rsp = {
+        {0},
+    };
+    int ret = -1;
+    xlator_t *this = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+    GF_ASSERT(req);
+
+    rsp.op_ret = status;
+    glusterd_get_uuid(&rsp.uuid);
+    rsp.op = op;
+    rsp.op_errno = op_errno;
+    if (op_errstr)
+        rsp.op_errstr = op_errstr;
+    else
+        rsp.op_errstr = "";
+
+    ret = dict_allocate_and_serialize(rsp_dict, &rsp.dict.dict_val,
+                                      &rsp.dict.dict_len);
+    if (ret < 0) {
+        gf_smsg(this->name, GF_LOG_ERROR, 0,
+                GD_MSG_DICT_ALLOC_AND_SERL_LENGTH_GET_FAIL, NULL);
+        goto out;
+    }
+
+    ret = glusterd_submit_reply(req, &rsp, NULL, 0, NULL,
+                                (xdrproc_t)xdr_gd1_mgmt_v3_post_commit_rsp);
+
+    GF_FREE(rsp.dict.dict_val);
 out:
-        gf_log (this->name, GF_LOG_DEBUG,
-                "Responded to post validation, ret: %d", ret);
-        return ret;
+    gf_msg_debug(this->name, 0, "Responded to post commit, ret: %d", ret);
+    return ret;
 }
 
 static int
-glusterd_handle_post_validate_fn (rpcsvc_request_t *req)
+glusterd_handle_post_commit_fn(rpcsvc_request_t *req)
 {
-        int32_t                         ret       = -1;
-        gd1_mgmt_v3_post_val_req         op_req    = {{0},};
-        xlator_t                       *this      = NULL;
-        char                           *op_errstr = NULL;
-        dict_t                         *dict      = NULL;
-        dict_t                         *rsp_dict  = NULL;
-
-        this = THIS;
-        GF_ASSERT (this);
-        GF_ASSERT (req);
-
-        ret = xdr_to_generic (req->msg[0], &op_req,
-                              (xdrproc_t)xdr_gd1_mgmt_v3_post_val_req);
-        if (ret < 0) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "Failed to decode post validation "
-                        "request received from peer");
-                req->rpc_err = GARBAGE_ARGS;
-                goto out;
-        }
+    int32_t ret = -1;
+    gd1_mgmt_v3_post_commit_req op_req = {
+        {0},
+    };
+    xlator_t *this = NULL;
+    char *op_errstr = NULL;
+    dict_t *dict = NULL;
+    dict_t *rsp_dict = NULL;
+    uint32_t op_errno = 0;
+
+    this = THIS;
+    GF_ASSERT(this);
+    GF_ASSERT(req);
+
+    ret = xdr_to_generic(req->msg[0], &op_req,
+                         (xdrproc_t)xdr_gd1_mgmt_v3_post_commit_req);
+    if (ret < 0) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_REQ_DECODE_FAIL,
+               "Failed to decode post commit "
+               "request received from peer");
+        req->rpc_err = GARBAGE_ARGS;
+        goto out;
+    }
+
+    if (glusterd_peerinfo_find_by_uuid(op_req.uuid) == NULL) {
+        gf_msg(this->name, GF_LOG_WARNING, 0, GD_MSG_PEER_NOT_FOUND,
+               "%s doesn't "
+               "belong to the cluster. Ignoring request.",
+               uuid_utoa(op_req.uuid));
+        ret = -1;
+        goto out;
+    }
+
+    dict = dict_new();
+    if (!dict) {
+        gf_smsg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_CREATE_FAIL, NULL);
+        goto out;
+    }
+
+    ret = dict_unserialize(op_req.dict.dict_val, op_req.dict.dict_len, &dict);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_WARNING, 0, GD_MSG_DICT_UNSERIALIZE_FAIL,
+                NULL);
+        goto out;
+    }
+
+    rsp_dict = dict_new();
+    if (!rsp_dict) {
+        gf_smsg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_CREATE_FAIL, NULL);
+        return -1;
+    }
+
+    ret = gd_mgmt_v3_post_commit_fn(op_req.op, dict, &op_errstr, &op_errno,
+                                    rsp_dict);
+
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_POST_COMMIT_OP_FAIL,
+               "post commit failed on operation %s", gd_op_list[op_req.op]);
+    }
+
+    ret = glusterd_mgmt_v3_post_commit_send_resp(req, op_req.op, ret, op_errstr,
+                                                 op_errno, rsp_dict);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_MGMTV3_OP_RESP_FAIL,
+               "Failed to send post commit "
+               "response for operation %s",
+               gd_op_list[op_req.op]);
+        goto out;
+    }
 
-        if (glusterd_peerinfo_find_by_uuid (op_req.uuid) == NULL) {
-                gf_log (this->name, GF_LOG_WARNING, "%s doesn't "
-                        "belong to the cluster. Ignoring request.",
-                        uuid_utoa (op_req.uuid));
-                ret = -1;
-                goto out;
-        }
+out:
+    if (op_errstr && (strcmp(op_errstr, "")))
+        GF_FREE(op_errstr);
 
-        dict = dict_new ();
-        if (!dict)
-                goto out;
+    free(op_req.dict.dict_val);
 
-        ret = dict_unserialize (op_req.dict.dict_val,
-                                op_req.dict.dict_len, &dict);
-        if (ret) {
-                gf_log (this->name, GF_LOG_WARNING,
-                        "failed to unserialize the dictionary");
-                goto out;
-        }
+    if (dict)
+        dict_unref(dict);
 
-        rsp_dict = dict_new ();
-        if (!rsp_dict) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "Failed to get new dictionary");
-                return -1;
-        }
+    if (rsp_dict)
+        dict_unref(rsp_dict);
 
-        ret = gd_mgmt_v3_post_validate_fn (op_req.op, op_req.op_ret, dict,
-                                           &op_errstr, rsp_dict);
+    /* Return 0 from handler to avoid double deletion of req obj */
+    return 0;
+}
 
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "Post Validation failed on operation %s",
-                        gd_op_list[op_req.op]);
-        }
+static int
+glusterd_mgmt_v3_post_validate_send_resp(rpcsvc_request_t *req, int32_t op,
+                                         int32_t status, char *op_errstr,
+                                         dict_t *rsp_dict)
+{
+    gd1_mgmt_v3_post_val_rsp rsp = {
+        {0},
+    };
+    int ret = -1;
+    xlator_t *this = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+    GF_ASSERT(req);
+
+    rsp.op_ret = status;
+    glusterd_get_uuid(&rsp.uuid);
+    rsp.op = op;
+    if (op_errstr)
+        rsp.op_errstr = op_errstr;
+    else
+        rsp.op_errstr = "";
+
+    ret = dict_allocate_and_serialize(rsp_dict, &rsp.dict.dict_val,
+                                      &rsp.dict.dict_len);
+    if (ret < 0) {
+        gf_smsg(this->name, GF_LOG_ERROR, 0,
+                GD_MSG_DICT_ALLOC_AND_SERL_LENGTH_GET_FAIL, NULL);
+        goto out;
+    }
+
+    ret = glusterd_submit_reply(req, &rsp, NULL, 0, NULL,
+                                (xdrproc_t)xdr_gd1_mgmt_v3_post_val_rsp);
+
+    GF_FREE(rsp.dict.dict_val);
+out:
+    gf_msg_debug(this->name, 0, "Responded to post validation, ret: %d", ret);
+    return ret;
+}
 
-        ret = glusterd_mgmt_v3_post_validate_send_resp (req, op_req.op,
-                                                       ret, op_errstr,
-                                                       rsp_dict);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "Failed to send Post Validation "
-                        "response for operation %s",
-                        gd_op_list[op_req.op]);
-                goto out;
-        }
+static int
+glusterd_handle_post_validate_fn(rpcsvc_request_t *req)
+{
+    int32_t ret = -1;
+    gd1_mgmt_v3_post_val_req op_req = {
+        {0},
+    };
+    xlator_t *this = NULL;
+    char *op_errstr = NULL;
+    dict_t *dict = NULL;
+    dict_t *rsp_dict = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+    GF_ASSERT(req);
+
+    ret = xdr_to_generic(req->msg[0], &op_req,
+                         (xdrproc_t)xdr_gd1_mgmt_v3_post_val_req);
+    if (ret < 0) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_REQ_DECODE_FAIL,
+               "Failed to decode post validation "
+               "request received from peer");
+        req->rpc_err = GARBAGE_ARGS;
+        goto out;
+    }
+
+    if (glusterd_peerinfo_find_by_uuid(op_req.uuid) == NULL) {
+        gf_msg(this->name, GF_LOG_WARNING, 0, GD_MSG_PEER_NOT_FOUND,
+               "%s doesn't "
+               "belong to the cluster. Ignoring request.",
+               uuid_utoa(op_req.uuid));
+        ret = -1;
+        goto out;
+    }
+
+    dict = dict_new();
+    if (!dict) {
+        gf_smsg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_CREATE_FAIL, NULL);
+        goto out;
+    }
+
+    ret = dict_unserialize(op_req.dict.dict_val, op_req.dict.dict_len, &dict);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_WARNING, 0, GD_MSG_DICT_UNSERIALIZE_FAIL,
+                NULL);
+        goto out;
+    }
+
+    rsp_dict = dict_new();
+    if (!rsp_dict) {
+        gf_smsg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_CREATE_FAIL, NULL);
+        return -1;
+    }
+
+    ret = gd_mgmt_v3_post_validate_fn(op_req.op, op_req.op_ret, dict,
+                                      &op_errstr, rsp_dict);
+
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_POST_VALIDATION_FAIL,
+               "Post Validation failed on operation %s", gd_op_list[op_req.op]);
+    }
+
+    ret = glusterd_mgmt_v3_post_validate_send_resp(req, op_req.op, ret,
+                                                   op_errstr, rsp_dict);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_MGMTV3_OP_RESP_FAIL,
+               "Failed to send Post Validation "
+               "response for operation %s",
+               gd_op_list[op_req.op]);
+        goto out;
+    }
 
 out:
-        if (op_errstr && (strcmp (op_errstr, "")))
-                GF_FREE (op_errstr);
+    if (op_errstr && (strcmp(op_errstr, "")))
+        GF_FREE(op_errstr);
 
-        free (op_req.dict.dict_val);
+    free(op_req.dict.dict_val);
 
-        if (dict)
-                dict_unref (dict);
+    if (dict)
+        dict_unref(dict);
 
-        if (rsp_dict)
-                dict_unref (rsp_dict);
+    if (rsp_dict)
+        dict_unref(rsp_dict);
 
-        /* Return 0 from handler to avoid double deletion of req obj */
-        return 0;
+    /* Return 0 from handler to avoid double deletion of req obj */
+    return 0;
 }
 
 static int
-glusterd_mgmt_v3_unlock_send_resp (rpcsvc_request_t *req, int32_t status)
+glusterd_mgmt_v3_unlock_send_resp(rpcsvc_request_t *req, int32_t status)
 {
+    gd1_mgmt_v3_unlock_rsp rsp = {
+        {0},
+    };
+    int ret = -1;
+    xlator_t *this = NULL;
 
-        gd1_mgmt_v3_unlock_rsp          rsp   = {{0},};
-        int                             ret   = -1;
-        xlator_t                       *this  = NULL;
+    this = THIS;
+    GF_ASSERT(this);
+    GF_ASSERT(req);
 
-        this = THIS;
-        GF_ASSERT (this);
-        GF_ASSERT (req);
+    rsp.op_ret = status;
+    if (rsp.op_ret)
+        rsp.op_errno = errno;
 
-        rsp.op_ret = status;
-        if (rsp.op_ret)
-                rsp.op_errno = errno;
+    glusterd_get_uuid(&rsp.uuid);
 
-        glusterd_get_uuid (&rsp.uuid);
+    ret = glusterd_submit_reply(req, &rsp, NULL, 0, NULL,
+                                (xdrproc_t)xdr_gd1_mgmt_v3_unlock_rsp);
 
-        ret = glusterd_submit_reply (req, &rsp, NULL, 0, NULL,
-                                     (xdrproc_t)xdr_gd1_mgmt_v3_unlock_rsp);
+    gf_msg_debug(this->name, 0, "Responded to mgmt_v3 unlock, ret: %d", ret);
 
-        gf_log (this->name, GF_LOG_DEBUG,
-                "Responded to mgmt_v3 unlock, ret: %d", ret);
-
-        return ret;
+    return ret;
 }
 
 static int
-glusterd_syctasked_mgmt_v3_unlock (rpcsvc_request_t *req,
+glusterd_syctasked_mgmt_v3_unlock(rpcsvc_request_t *req,
                                   gd1_mgmt_v3_unlock_req *unlock_req,
                                   glusterd_op_lock_ctx_t *ctx)
 {
-        int32_t                         ret         = -1;
-        xlator_t                       *this        = NULL;
+    int32_t ret = -1;
+    xlator_t *this = NULL;
 
-        this = THIS;
-        GF_ASSERT (this);
-        GF_ASSERT (req);
-        GF_ASSERT (ctx);
+    this = THIS;
+    GF_ASSERT(this);
+    GF_ASSERT(req);
+    GF_ASSERT(ctx);
 
-        /* Trying to release multiple mgmt_v3 locks */
-        ret = glusterd_multiple_mgmt_v3_unlock (ctx->dict, ctx->uuid);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "Failed to release mgmt_v3 locks for %s",
-                        uuid_utoa(ctx->uuid));
-        }
+    /* Trying to release multiple mgmt_v3 locks */
+    ret = glusterd_multiple_mgmt_v3_unlock(ctx->dict, ctx->uuid);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_MGMTV3_UNLOCK_FAIL,
+               "Failed to release mgmt_v3 locks for %s", uuid_utoa(ctx->uuid));
+    }
 
-        ret = glusterd_mgmt_v3_unlock_send_resp (req, ret);
+    ret = glusterd_mgmt_v3_unlock_send_resp(req, ret);
 
-        gf_log (this->name, GF_LOG_TRACE, "Returning %d", ret);
-        return ret;
+    gf_msg_trace(this->name, 0, "Returning %d", ret);
+    return ret;
 }
 
-
 static int
-glusterd_op_state_machine_mgmt_v3_unlock (rpcsvc_request_t *req,
+glusterd_op_state_machine_mgmt_v3_unlock(rpcsvc_request_t *req,
                                          gd1_mgmt_v3_unlock_req *lock_req,
                                          glusterd_op_lock_ctx_t *ctx)
 {
-        int32_t                           ret      = -1;
-        xlator_t                         *this     = NULL;
+    int32_t ret = -1;
+    xlator_t *this = NULL;
 
-        this = THIS;
-        GF_ASSERT (this);
-        GF_ASSERT (req);
+    this = THIS;
+    GF_ASSERT(this);
+    GF_ASSERT(req);
 
-        ret = glusterd_op_sm_inject_event (GD_OP_EVENT_UNLOCK,
-                                           &lock_req->txn_id, ctx);
-        if (ret)
-                gf_log (this->name, GF_LOG_ERROR,
-                        "Failed to inject event GD_OP_EVENT_UNLOCK");
+    ret = glusterd_op_sm_inject_event(GD_OP_EVENT_UNLOCK, &lock_req->txn_id,
+                                      ctx);
+    if (ret)
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_OP_EVENT_UNLOCK_FAIL,
+               "Failed to inject event GD_OP_EVENT_UNLOCK");
 
-        glusterd_friend_sm ();
-        glusterd_op_sm ();
+    glusterd_friend_sm();
+    glusterd_op_sm();
 
-        gf_log (this->name, GF_LOG_TRACE, "Returning %d", ret);
-        return ret;
+    gf_msg_trace(this->name, 0, "Returning %d", ret);
+    return ret;
 }
 
 static int
-glusterd_handle_mgmt_v3_unlock_fn (rpcsvc_request_t *req)
+glusterd_handle_mgmt_v3_unlock_fn(rpcsvc_request_t *req)
 {
-        gd1_mgmt_v3_unlock_req  lock_req        = {{0},};
-        int32_t                 ret             = -1;
-        glusterd_op_lock_ctx_t *ctx             = NULL;
-        xlator_t               *this            = NULL;
-        gf_boolean_t            is_synctasked   = _gf_false;
-        gf_boolean_t            free_ctx        = _gf_false;
-
-        this = THIS;
-        GF_ASSERT (this);
-        GF_ASSERT (req);
-
-        ret = xdr_to_generic (req->msg[0], &lock_req,
-                              (xdrproc_t)xdr_gd1_mgmt_v3_unlock_req);
-        if (ret < 0) {
-                gf_log (this->name, GF_LOG_ERROR, "Failed to decode unlock "
-                        "request received from peer");
-                req->rpc_err = GARBAGE_ARGS;
-                goto out;
-        }
-
-        gf_log (this->name, GF_LOG_DEBUG, "Received volume unlock req "
-                "from uuid: %s", uuid_utoa (lock_req.uuid));
-
-        if (glusterd_peerinfo_find_by_uuid (lock_req.uuid) == NULL) {
-                gf_log (this->name, GF_LOG_WARNING, "%s doesn't "
-                        "belong to the cluster. Ignoring request.",
-                        uuid_utoa (lock_req.uuid));
-                ret = -1;
-                goto out;
-        }
-
-        ctx = GF_CALLOC (1, sizeof (*ctx), gf_gld_mt_op_lock_ctx_t);
-        if (!ctx) {
-                ret = -1;
-                goto out;
-        }
-
-        gf_uuid_copy (ctx->uuid, lock_req.uuid);
-        ctx->req = req;
-
-        ctx->dict = dict_new ();
-        if (!ctx->dict) {
-                ret = -1;
-                goto out;
-        }
-
-        ret = dict_unserialize (lock_req.dict.dict_val,
-                                lock_req.dict.dict_len, &ctx->dict);
+    gd1_mgmt_v3_unlock_req lock_req = {
+        {0},
+    };
+    int32_t ret = -1;
+    glusterd_op_lock_ctx_t *ctx = NULL;
+    xlator_t *this = NULL;
+    gf_boolean_t is_synctasked = _gf_false;
+    gf_boolean_t free_ctx = _gf_false;
+
+    this = THIS;
+    GF_ASSERT(this);
+    GF_ASSERT(req);
+
+    ret = xdr_to_generic(req->msg[0], &lock_req,
+                         (xdrproc_t)xdr_gd1_mgmt_v3_unlock_req);
+    if (ret < 0) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_REQ_DECODE_FAIL,
+               "Failed to decode unlock "
+               "request received from peer");
+        req->rpc_err = GARBAGE_ARGS;
+        goto out;
+    }
+
+    gf_msg_debug(this->name, 0,
+                 "Received volume unlock req "
+                 "from uuid: %s",
+                 uuid_utoa(lock_req.uuid));
+
+    if (glusterd_peerinfo_find_by_uuid(lock_req.uuid) == NULL) {
+        gf_msg(this->name, GF_LOG_WARNING, 0, GD_MSG_PEER_NOT_FOUND,
+               "%s doesn't "
+               "belong to the cluster. Ignoring request.",
+               uuid_utoa(lock_req.uuid));
+        ret = -1;
+        goto out;
+    }
+
+    ctx = GF_CALLOC(1, sizeof(*ctx), gf_gld_mt_op_lock_ctx_t);
+    if (!ctx) {
+        gf_smsg(this->name, GF_LOG_ERROR, 0, GD_MSG_NO_MEMORY, NULL);
+        ret = -1;
+        goto out;
+    }
+
+    gf_uuid_copy(ctx->uuid, lock_req.uuid);
+    ctx->req = req;
+
+    ctx->dict = dict_new();
+    if (!ctx->dict) {
+        gf_smsg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_CREATE_FAIL, NULL);
+        ret = -1;
+        goto out;
+    }
+
+    ret = dict_unserialize(lock_req.dict.dict_val, lock_req.dict.dict_len,
+                           &ctx->dict);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_WARNING, 0, GD_MSG_DICT_UNSERIALIZE_FAIL,
+                NULL);
+        goto out;
+    }
+
+    is_synctasked = dict_get_str_boolean(ctx->dict, "is_synctasked", _gf_false);
+    if (is_synctasked) {
+        ret = glusterd_syctasked_mgmt_v3_unlock(req, &lock_req, ctx);
         if (ret) {
-                gf_log (this->name, GF_LOG_WARNING,
-                        "failed to unserialize the dictionary");
-                goto out;
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_MGMTV3_UNLOCK_FAIL,
+                   "Failed to release mgmt_v3_locks");
+            /* Ignore the return code, as it shouldn't be propagated
+             * from the handler function so as to avoid double
+             * deletion of the req
+             */
+            ret = 0;
         }
 
-        is_synctasked = dict_get_str_boolean (ctx->dict,
-                                              "is_synctasked", _gf_false);
-        if (is_synctasked) {
-                ret = glusterd_syctasked_mgmt_v3_unlock (req, &lock_req, ctx);
-                if (ret) {
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "Failed to release mgmt_v3_locks");
-                        /* Ignore the return code, as it shouldn't be propagated
-                         * from the handler function so as to avoid double
-                         * deletion of the req
-                         */
-                        ret = 0;
-                }
-
-                /* The above function does not take ownership of ctx.
-                 * Therefore we need to free the ctx explicitly. */
-                free_ctx = _gf_true;
-        }
-        else {
-                /* Shouldn't ignore the return code here, and it should
-                 * be propagated from the handler function as in failure
-                 * case it doesn't delete the req object
-                 */
-                ret = glusterd_op_state_machine_mgmt_v3_unlock (req, &lock_req,
-                                                                ctx);
-                if (ret)
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "Failed to release mgmt_v3_locks");
-        }
+        /* The above function does not take ownership of ctx.
+         * Therefore we need to free the ctx explicitly. */
+        free_ctx = _gf_true;
+    } else {
+        /* Shouldn't ignore the return code here, and it should
+         * be propagated from the handler function as in failure
+         * case it doesn't delete the req object
+         */
+        ret = glusterd_op_state_machine_mgmt_v3_unlock(req, &lock_req, ctx);
+        if (ret)
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_MGMTV3_UNLOCK_FAIL,
+                   "Failed to release mgmt_v3_locks");
+    }
 
 out:
 
-        if (ctx && (ret || free_ctx)) {
-                if (ctx->dict)
-                        dict_unref (ctx->dict);
+    if (ctx && (ret || free_ctx)) {
+        if (ctx->dict)
+            dict_unref(ctx->dict);
 
-                GF_FREE (ctx);
-        }
+        GF_FREE(ctx);
+    }
 
-        free (lock_req.dict.dict_val);
+    free(lock_req.dict.dict_val);
 
-        gf_log (this->name, GF_LOG_TRACE, "Returning %d", ret);
-        return ret;
+    gf_msg_trace(this->name, 0, "Returning %d", ret);
+    return ret;
 }
 
 int
-glusterd_handle_mgmt_v3_lock (rpcsvc_request_t *req)
+glusterd_handle_mgmt_v3_lock(rpcsvc_request_t *req)
+{
+    return glusterd_big_locked_handler(req, glusterd_handle_mgmt_v3_lock_fn);
+}
+
+static int
+glusterd_handle_pre_validate(rpcsvc_request_t *req)
 {
-        return glusterd_big_locked_handler (req,
-                                            glusterd_handle_mgmt_v3_lock_fn);
+    return glusterd_big_locked_handler(req, glusterd_handle_pre_validate_fn);
 }
 
 static int
-glusterd_handle_pre_validate (rpcsvc_request_t *req)
+glusterd_handle_brick_op(rpcsvc_request_t *req)
 {
-        return glusterd_big_locked_handler (req,
-                                            glusterd_handle_pre_validate_fn);
+    return glusterd_big_locked_handler(req, glusterd_handle_brick_op_fn);
 }
 
 static int
-glusterd_handle_brick_op (rpcsvc_request_t *req)
+glusterd_handle_commit(rpcsvc_request_t *req)
 {
-        return glusterd_big_locked_handler (req,
-                                            glusterd_handle_brick_op_fn);
+    return glusterd_big_locked_handler(req, glusterd_handle_commit_fn);
 }
 
 static int
-glusterd_handle_commit (rpcsvc_request_t *req)
+glusterd_handle_post_commit(rpcsvc_request_t *req)
 {
-        return glusterd_big_locked_handler (req,
-                                            glusterd_handle_commit_fn);
+    return glusterd_big_locked_handler(req, glusterd_handle_post_commit_fn);
 }
 
 static int
-glusterd_handle_post_validate (rpcsvc_request_t *req)
+glusterd_handle_post_validate(rpcsvc_request_t *req)
 {
-        return glusterd_big_locked_handler (req,
-                                            glusterd_handle_post_validate_fn);
+    return glusterd_big_locked_handler(req, glusterd_handle_post_validate_fn);
 }
 
 int
-glusterd_handle_mgmt_v3_unlock (rpcsvc_request_t *req)
+glusterd_handle_mgmt_v3_unlock(rpcsvc_request_t *req)
 {
-        return glusterd_big_locked_handler (req,
-                                            glusterd_handle_mgmt_v3_unlock_fn);
+    return glusterd_big_locked_handler(req, glusterd_handle_mgmt_v3_unlock_fn);
 }
 
-rpcsvc_actor_t gd_svc_mgmt_v3_actors[GLUSTERD_MGMT_V3_MAXVALUE] = {
-        [GLUSTERD_MGMT_V3_NULL]          = { "NULL",           GLUSTERD_MGMT_V3_NULL,          glusterd_mgmt_v3_null,         NULL, 0, DRC_NA},
-        [GLUSTERD_MGMT_V3_LOCK]          = { "MGMT_V3_LOCK",   GLUSTERD_MGMT_V3_LOCK,          glusterd_handle_mgmt_v3_lock,   NULL, 0, DRC_NA},
-        [GLUSTERD_MGMT_V3_PRE_VALIDATE]  = { "PRE_VAL",        GLUSTERD_MGMT_V3_PRE_VALIDATE,  glusterd_handle_pre_validate,  NULL, 0, DRC_NA},
-        [GLUSTERD_MGMT_V3_BRICK_OP]      = { "BRCK_OP",        GLUSTERD_MGMT_V3_BRICK_OP,      glusterd_handle_brick_op,      NULL, 0, DRC_NA},
-        [GLUSTERD_MGMT_V3_COMMIT]        = { "COMMIT",         GLUSTERD_MGMT_V3_COMMIT,        glusterd_handle_commit,        NULL, 0, DRC_NA},
-        [GLUSTERD_MGMT_V3_POST_VALIDATE] = { "POST_VAL",       GLUSTERD_MGMT_V3_POST_VALIDATE, glusterd_handle_post_validate, NULL, 0, DRC_NA},
-        [GLUSTERD_MGMT_V3_UNLOCK]        = { "MGMT_V3_UNLOCK", GLUSTERD_MGMT_V3_UNLOCK,        glusterd_handle_mgmt_v3_unlock, NULL, 0, DRC_NA},
+static rpcsvc_actor_t gd_svc_mgmt_v3_actors[GLUSTERD_MGMT_V3_MAXVALUE] = {
+    [GLUSTERD_MGMT_V3_NULL] = {"NULL", glusterd_mgmt_v3_null, NULL,
+                               GLUSTERD_MGMT_V3_NULL, DRC_NA, 0},
+    [GLUSTERD_MGMT_V3_LOCK] = {"MGMT_V3_LOCK", glusterd_handle_mgmt_v3_lock,
+                               NULL, GLUSTERD_MGMT_V3_LOCK, DRC_NA, 0},
+    [GLUSTERD_MGMT_V3_PRE_VALIDATE] = {"PRE_VAL", glusterd_handle_pre_validate,
+                                       NULL, GLUSTERD_MGMT_V3_PRE_VALIDATE,
+                                       DRC_NA, 0},
+    [GLUSTERD_MGMT_V3_BRICK_OP] = {"BRCK_OP", glusterd_handle_brick_op, NULL,
+                                   GLUSTERD_MGMT_V3_BRICK_OP, DRC_NA, 0},
+    [GLUSTERD_MGMT_V3_COMMIT] = {"COMMIT", glusterd_handle_commit, NULL,
+                                 GLUSTERD_MGMT_V3_COMMIT, DRC_NA, 0},
+    [GLUSTERD_MGMT_V3_POST_COMMIT] = {"POST_COMMIT",
+                                      glusterd_handle_post_commit, NULL,
+                                      GLUSTERD_MGMT_V3_POST_COMMIT, DRC_NA, 0},
+    [GLUSTERD_MGMT_V3_POST_VALIDATE] = {"POST_VAL",
+                                        glusterd_handle_post_validate, NULL,
+                                        GLUSTERD_MGMT_V3_POST_VALIDATE, DRC_NA,
+                                        0},
+    [GLUSTERD_MGMT_V3_UNLOCK] = {"MGMT_V3_UNLOCK",
+                                 glusterd_handle_mgmt_v3_unlock, NULL,
+                                 GLUSTERD_MGMT_V3_UNLOCK, DRC_NA, 0},
 };
 
 struct rpcsvc_program gd_svc_mgmt_v3_prog = {
-        .progname  = "GlusterD svc mgmt v3",
-        .prognum   = GD_MGMT_PROGRAM,
-        .progver   = GD_MGMT_V3_VERSION,
-        .numactors = GLUSTERD_MGMT_V3_MAXVALUE,
-        .actors    = gd_svc_mgmt_v3_actors,
-        .synctask  = _gf_true,
+    .progname = "GlusterD svc mgmt v3",
+    .prognum = GD_MGMT_PROGRAM,
+    .progver = GD_MGMT_V3_VERSION,
+    .numactors = GLUSTERD_MGMT_V3_MAXVALUE,
+    .actors = gd_svc_mgmt_v3_actors,
+    .synctask = _gf_true,
 };
diff --git a/xlators/mgmt/glusterd/src/glusterd-mgmt.c b/xlators/mgmt/glusterd/src/glusterd-mgmt.c
index 2a34c78b35b..bca7221062b 100644
--- a/xlators/mgmt/glusterd/src/glusterd-mgmt.c
+++ b/xlators/mgmt/glusterd/src/glusterd-mgmt.c
@@ -19,1981 +19,3096 @@
 #include "glusterd-locks.h"
 #include "glusterd-mgmt.h"
 #include "glusterd-op-sm.h"
+#include "glusterd-server-quorum.h"
 #include "glusterd-volgen.h"
 #include "glusterd-store.h"
 #include "glusterd-snapshot-utils.h"
+#include "glusterd-messages.h"
+#include "glusterd-errno.h"
+#include "glusterd-hooks.h"
 
 extern struct rpc_clnt_program gd_mgmt_v3_prog;
 
-
 void
-gd_mgmt_v3_collate_errors (struct syncargs *args, int op_ret, int op_errno,
-                           char *op_errstr, int op_code,
-                           glusterd_peerinfo_t *peerinfo, u_char *uuid)
+gd_mgmt_v3_collate_errors(struct syncargs *args, int op_ret, int op_errno,
+                          char *op_errstr, int op_code, uuid_t peerid,
+                          u_char *uuid)
 {
-        char      *peer_str          = NULL;
-        char       err_str[PATH_MAX] = "Please check log file for details.";
-        char       op_err[PATH_MAX]  = "";
-        int32_t    len               = -1;
-        xlator_t  *this              = NULL;
-        int        is_operrstr_blk   = 0;
-        char       *err_string       = NULL;
-        char       *cli_err_str      = NULL;
-
-        this = THIS;
-        GF_ASSERT (this);
-        GF_ASSERT (args);
-        GF_ASSERT (uuid);
-
-        if (op_ret) {
-                args->op_ret = op_ret;
-                args->op_errno = op_errno;
-
-                if (peerinfo)
-                        peer_str = peerinfo->hostname;
-                else
-                        peer_str = uuid_utoa (uuid);
-
-                is_operrstr_blk = (op_errstr && strcmp (op_errstr, ""));
-                err_string     = (is_operrstr_blk) ? op_errstr : err_str;
-
-                switch (op_code) {
-                case GLUSTERD_MGMT_V3_LOCK:
-                        {
-                                len = snprintf (op_err, sizeof(op_err),
-                                                "Locking failed "
-                                                "on %s. %s", peer_str,
-                                                err_string);
-                                break;
-                        }
-                case GLUSTERD_MGMT_V3_PRE_VALIDATE:
-                        {
-                                len = snprintf (op_err, sizeof(op_err),
-                                                "Pre Validation failed "
-                                                "on %s. %s", peer_str,
-                                                err_string);
-                                break;
-                        }
-                case GLUSTERD_MGMT_V3_BRICK_OP:
-                        {
-                                len = snprintf (op_err, sizeof(op_err),
-                                                "Brick ops failed "
-                                                "on %s. %s", peer_str,
-                                                err_string);
-                                break;
-                        }
-                case GLUSTERD_MGMT_V3_COMMIT:
-                        {
-                                len = snprintf (op_err, sizeof(op_err),
-                                                "Commit failed"
-                                                " on %s. %s", peer_str,
-                                                err_string);
-                                break;
-                        }
-                case GLUSTERD_MGMT_V3_POST_VALIDATE:
-                        {
-                                len = snprintf (op_err, sizeof(op_err),
-                                                "Post Validation failed "
-                                                "on %s. %s", peer_str,
-                                                err_string);
-                                break;
-                        }
-                case GLUSTERD_MGMT_V3_UNLOCK:
-                        {
-                                len = snprintf (op_err, sizeof(op_err),
-                                                "Unlocking failed "
-                                                "on %s. %s", peer_str,
-                                                err_string);
-                                break;
-                        }
-                default :
-                        len = snprintf (op_err, sizeof(op_err),
-                                       "Unknown error! "
-                                       "on %s. %s", peer_str,
-                                        err_string);
-                }
-
-                cli_err_str = ((is_operrstr_blk) ? op_errstr : op_err);
-
-                if (args->errstr) {
-                        len = snprintf (err_str, sizeof(err_str),
-                                      "%s\n%s", args->errstr,
-                                      cli_err_str);
-                        GF_FREE (args->errstr);
-                        args->errstr = NULL;
-                } else
-                        len = snprintf (err_str, sizeof(err_str),
-                                "%s", cli_err_str);
-
-                gf_log (this->name, GF_LOG_ERROR, "%s", op_err);
-                args->errstr = gf_strdup (err_str);
-        }
-
-        return;
+    char *peer_str = NULL;
+    char err_str[PATH_MAX] = "Please check log file for details.";
+    char op_err[PATH_MAX] = "";
+    xlator_t *this = NULL;
+    int is_operrstr_blk = 0;
+    char *err_string = NULL;
+    glusterd_peerinfo_t *peerinfo = NULL;
+    int32_t len = 0;
+
+    this = THIS;
+    GF_ASSERT(this);
+    GF_ASSERT(args);
+    GF_ASSERT(uuid);
+
+    if (op_ret) {
+        args->op_ret = op_ret;
+        args->op_errno = op_errno;
+
+        RCU_READ_LOCK;
+        peerinfo = glusterd_peerinfo_find(peerid, NULL);
+        if (peerinfo)
+            peer_str = gf_strdup(peerinfo->hostname);
+        else
+            peer_str = gf_strdup(uuid_utoa(uuid));
+
+        RCU_READ_UNLOCK;
+
+        is_operrstr_blk = (op_errstr && strcmp(op_errstr, ""));
+        err_string = (is_operrstr_blk) ? op_errstr : err_str;
+
+        switch (op_code) {
+            case GLUSTERD_MGMT_V3_LOCK: {
+                snprintf(op_err, sizeof(op_err), "Locking failed on %s. %s",
+                         peer_str, err_string);
+                break;
+            }
+            case GLUSTERD_MGMT_V3_PRE_VALIDATE: {
+                snprintf(op_err, sizeof(op_err),
+                         "Pre Validation failed on %s. %s", peer_str,
+                         err_string);
+                break;
+            }
+            case GLUSTERD_MGMT_V3_BRICK_OP: {
+                snprintf(op_err, sizeof(op_err), "Brick ops failed on %s. %s",
+                         peer_str, err_string);
+                break;
+            }
+            case GLUSTERD_MGMT_V3_COMMIT: {
+                snprintf(op_err, sizeof(op_err), "Commit failed on %s. %s",
+                         peer_str, err_string);
+                break;
+            }
+            case GLUSTERD_MGMT_V3_POST_COMMIT: {
+                snprintf(op_err, sizeof(op_err), "Post commit failed on %s. %s",
+                         peer_str, err_string);
+                break;
+            }
+            case GLUSTERD_MGMT_V3_POST_VALIDATE: {
+                snprintf(op_err, sizeof(op_err),
+                         "Post Validation failed on %s. %s", peer_str,
+                         err_string);
+                break;
+            }
+            case GLUSTERD_MGMT_V3_UNLOCK: {
+                snprintf(op_err, sizeof(op_err), "Unlocking failed on %s. %s",
+                         peer_str, err_string);
+                break;
+            }
+            default:
+                snprintf(op_err, sizeof(op_err), "Unknown error! on %s. %s",
+                         peer_str, err_string);
+        }
+
+        if (args->errstr) {
+            len = snprintf(err_str, sizeof(err_str), "%s\n%s", args->errstr,
+                           op_err);
+            if (len < 0) {
+                strcpy(err_str, "<error>");
+            }
+            GF_FREE(args->errstr);
+            args->errstr = NULL;
+        } else
+            snprintf(err_str, sizeof(err_str), "%s", op_err);
+
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_MGMTV3_OP_FAIL, "%s",
+               op_err);
+        args->errstr = gf_strdup(err_str);
+    }
+
+    GF_FREE(peer_str);
+
+    return;
 }
 
 int32_t
-gd_mgmt_v3_pre_validate_fn (glusterd_op_t op, dict_t *dict,
-                            char **op_errstr, dict_t *rsp_dict)
+gd_mgmt_v3_pre_validate_fn(glusterd_op_t op, dict_t *dict, char **op_errstr,
+                           dict_t *rsp_dict, uint32_t *op_errno)
 {
-        int32_t       ret = -1;
-        xlator_t     *this = NULL;
+    int32_t ret = -1;
+    xlator_t *this = NULL;
 
-        this = THIS;
-        GF_ASSERT (this);
-        GF_ASSERT (dict);
-        GF_ASSERT (op_errstr);
-        GF_ASSERT (rsp_dict);
+    this = THIS;
+    GF_ASSERT(this);
+    GF_ASSERT(dict);
+    GF_ASSERT(op_errstr);
+    GF_ASSERT(rsp_dict);
+    GF_VALIDATE_OR_GOTO(this->name, op_errno, out);
 
-        switch (op) {
+    switch (op) {
         case GD_OP_SNAP:
-                ret = glusterd_snapshot_prevalidate (dict, op_errstr,
-                                                     rsp_dict);
+            ret = glusterd_snapshot_prevalidate(dict, op_errstr, rsp_dict,
+                                                op_errno);
 
-                if (ret) {
-                        gf_log (this->name, GF_LOG_WARNING,
-                                "Snapshot Prevalidate Failed");
-                        goto out;
-                }
+            if (ret) {
+                gf_msg(this->name, GF_LOG_WARNING, 0,
+                       GD_MSG_PRE_VALIDATION_FAIL,
+                       "Snapshot Prevalidate Failed");
+                goto out;
+            }
 
-                break;
+            break;
+
+        case GD_OP_REPLACE_BRICK:
+            ret = glusterd_op_stage_replace_brick(dict, op_errstr, rsp_dict);
+            if (ret) {
+                gf_msg(this->name, GF_LOG_WARNING, 0,
+                       GD_MSG_PRE_VALIDATION_FAIL,
+                       "Replace-brick prevalidation failed.");
+                goto out;
+            }
+            break;
+        case GD_OP_ADD_BRICK:
+            ret = glusterd_op_stage_add_brick(dict, op_errstr, rsp_dict);
+            if (ret) {
+                gf_msg(this->name, GF_LOG_WARNING, 0,
+                       GD_MSG_PRE_VALIDATION_FAIL,
+                       "ADD-brick prevalidation failed.");
+                goto out;
+            }
+            break;
+        case GD_OP_START_VOLUME:
+            ret = glusterd_op_stage_start_volume(dict, op_errstr, rsp_dict);
+            if (ret) {
+                gf_msg(this->name, GF_LOG_WARNING, 0,
+                       GD_MSG_PRE_VALIDATION_FAIL,
+                       "Volume start prevalidation failed.");
+                goto out;
+            }
+            break;
+        case GD_OP_STOP_VOLUME:
+            ret = glusterd_op_stage_stop_volume(dict, op_errstr);
+            if (ret) {
+                gf_msg(this->name, GF_LOG_WARNING, 0,
+                       GD_MSG_PRE_VALIDATION_FAIL,
+                       "Volume stop prevalidation failed.");
+                goto out;
+            }
+            break;
+        case GD_OP_REMOVE_BRICK:
+            ret = glusterd_op_stage_remove_brick(dict, op_errstr);
+            if (ret) {
+                gf_msg(this->name, GF_LOG_WARNING, 0,
+                       GD_MSG_PRE_VALIDATION_FAIL,
+                       "Remove brick prevalidation failed.");
+                goto out;
+            }
+            break;
+
+        case GD_OP_RESET_BRICK:
+            ret = glusterd_reset_brick_prevalidate(dict, op_errstr, rsp_dict);
+            if (ret) {
+                gf_msg(this->name, GF_LOG_WARNING, 0,
+                       GD_MSG_PRE_VALIDATION_FAIL,
+                       "Reset brick prevalidation failed.");
+                goto out;
+            }
+            break;
+
+        case GD_OP_PROFILE_VOLUME:
+            ret = glusterd_op_stage_stats_volume(dict, op_errstr);
+            if (ret) {
+                gf_msg(this->name, GF_LOG_WARNING, 0,
+                       GD_MSG_PRE_VALIDATION_FAIL,
+                       "prevalidation failed for profile operation.");
+                goto out;
+            }
+            break;
+        case GD_OP_REBALANCE:
+        case GD_OP_DEFRAG_BRICK_VOLUME:
+            ret = glusterd_mgmt_v3_op_stage_rebalance(dict, op_errstr);
+            if (ret) {
+                gf_log(this->name, GF_LOG_WARNING,
+                       "Rebalance Prevalidate Failed");
+                goto out;
+            }
+            break;
+
+        case GD_OP_MAX_OPVERSION:
+            ret = 0;
+            break;
 
         default:
-                break;
-        }
+            break;
+    }
 
-        ret = 0;
+    ret = 0;
 out:
-        gf_log (this->name, GF_LOG_DEBUG, "OP = %d. Returning %d", op, ret);
-        return ret;
+    gf_msg_debug(this->name, 0, "OP = %d. Returning %d", op, ret);
+    return ret;
 }
 
 int32_t
-gd_mgmt_v3_brick_op_fn (glusterd_op_t op, dict_t *dict,
-                        char **op_errstr, dict_t *rsp_dict)
+gd_mgmt_v3_brick_op_fn(glusterd_op_t op, dict_t *dict, char **op_errstr,
+                       dict_t *rsp_dict)
 {
-        int32_t       ret = -1;
-        xlator_t     *this = NULL;
-
-        this = THIS;
-        GF_ASSERT (this);
-        GF_ASSERT (dict);
-        GF_ASSERT (op_errstr);
-        GF_ASSERT (rsp_dict);
-
-        switch (op) {
-        case GD_OP_SNAP:
-        {
-                ret = glusterd_snapshot_brickop (dict, op_errstr, rsp_dict);
-                if (ret) {
-                        gf_log (this->name, GF_LOG_WARNING, "snapshot brickop "
-                                "failed");
-                        goto out;
-                }
-                break;
+    int32_t ret = -1;
+    xlator_t *this = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+    GF_ASSERT(dict);
+    GF_ASSERT(op_errstr);
+    GF_ASSERT(rsp_dict);
+
+    switch (op) {
+        case GD_OP_SNAP: {
+            ret = glusterd_snapshot_brickop(dict, op_errstr, rsp_dict);
+            if (ret) {
+                gf_msg(this->name, GF_LOG_WARNING, 0, GD_MSG_BRICK_OP_FAIL,
+                       "snapshot brickop failed");
+                goto out;
+            }
+            break;
+        }
+        case GD_OP_PROFILE_VOLUME:
+        case GD_OP_REBALANCE:
+        case GD_OP_DEFRAG_BRICK_VOLUME: {
+            ret = gd_brick_op_phase(op, rsp_dict, dict, op_errstr);
+            if (ret) {
+                gf_log(this->name, GF_LOG_WARNING,
+                       "%s brickop "
+                       "failed",
+                       gd_op_list[op]);
+                goto out;
+            }
+            break;
         }
         default:
-                break;
-        }
+            break;
+    }
 
-        ret = 0;
+    ret = 0;
 out:
-        gf_log (this->name, GF_LOG_TRACE, "OP = %d. Returning %d", op, ret);
-        return ret;
+    gf_msg_trace(this->name, 0, "OP = %d. Returning %d", op, ret);
+    return ret;
 }
 
 int32_t
-gd_mgmt_v3_commit_fn (glusterd_op_t op, dict_t *dict,
-                      char **op_errstr, dict_t *rsp_dict)
+gd_mgmt_v3_commit_fn(glusterd_op_t op, dict_t *dict, char **op_errstr,
+                     uint32_t *op_errno, dict_t *rsp_dict)
 {
-        int32_t       ret = -1;
-        xlator_t     *this = NULL;
-
-        this = THIS;
-        GF_ASSERT (this);
-        GF_ASSERT (dict);
-        GF_ASSERT (op_errstr);
-        GF_ASSERT (rsp_dict);
-
-        switch (op) {
-               case GD_OP_SNAP:
-               {
-                       ret = glusterd_snapshot (dict, op_errstr, rsp_dict);
-                       if (ret) {
-                               gf_log (this->name, GF_LOG_WARNING,
-                                       "Snapshot Commit Failed");
-                               goto out;
-                       }
-                       break;
-               }
-               default:
-                       break;
+    int32_t ret = -1;
+    xlator_t *this = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+    GF_ASSERT(dict);
+    GF_ASSERT(op_errstr);
+    GF_VALIDATE_OR_GOTO(this->name, op_errno, out);
+    GF_ASSERT(rsp_dict);
+
+    glusterd_op_commit_hook(op, dict, GD_COMMIT_HOOK_PRE);
+    switch (op) {
+        case GD_OP_SNAP: {
+            ret = glusterd_snapshot(dict, op_errstr, op_errno, rsp_dict);
+            if (ret) {
+                gf_msg(this->name, GF_LOG_WARNING, 0, GD_MSG_COMMIT_OP_FAIL,
+                       "Snapshot Commit Failed");
+                goto out;
+            }
+            break;
+        }
+        case GD_OP_REPLACE_BRICK: {
+            ret = glusterd_op_replace_brick(dict, rsp_dict);
+            if (ret) {
+                gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_COMMIT_OP_FAIL,
+                       "Replace-brick commit failed.");
+                goto out;
+            }
+            break;
+        }
+        case GD_OP_ADD_BRICK: {
+            ret = glusterd_op_add_brick(dict, op_errstr);
+            if (ret) {
+                gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_COMMIT_OP_FAIL,
+                       "Add-brick commit failed.");
+                goto out;
+            }
+            break;
+        }
+        case GD_OP_START_VOLUME: {
+            ret = glusterd_op_start_volume(dict, op_errstr);
+            if (ret) {
+                gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_COMMIT_OP_FAIL,
+                       "Volume start commit failed.");
+                goto out;
+            }
+            break;
+        }
+        case GD_OP_STOP_VOLUME: {
+            ret = glusterd_op_stop_volume(dict);
+            if (ret) {
+                gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_COMMIT_OP_FAIL,
+                       "Volume stop commit failed.");
+                goto out;
+            }
+            break;
+        }
+        case GD_OP_REMOVE_BRICK: {
+            ret = glusterd_op_remove_brick(dict, op_errstr);
+            if (ret) {
+                gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_COMMIT_OP_FAIL,
+                       "Remove-brick commit failed.");
+                goto out;
+            }
+            break;
+        }
+        case GD_OP_RESET_BRICK: {
+            ret = glusterd_op_reset_brick(dict, rsp_dict);
+            if (ret) {
+                gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_COMMIT_OP_FAIL,
+                       "Reset-brick commit failed.");
+                goto out;
+            }
+            break;
+        }
+        case GD_OP_MAX_OPVERSION: {
+            ret = glusterd_op_get_max_opversion(op_errstr, rsp_dict);
+            if (ret) {
+                gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_COMMIT_OP_FAIL,
+                       "Commit failed.");
+                goto out;
+            }
+            break;
+        }
+        case GD_OP_PROFILE_VOLUME: {
+            ret = glusterd_op_stats_volume(dict, op_errstr, rsp_dict);
+            if (ret) {
+                gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_COMMIT_OP_FAIL,
+                       "commit failed for volume profile operation.");
+                goto out;
+            }
+            break;
+        }
+        case GD_OP_REBALANCE:
+        case GD_OP_DEFRAG_BRICK_VOLUME: {
+            ret = glusterd_mgmt_v3_op_rebalance(dict, op_errstr, rsp_dict);
+            if (ret) {
+                gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_COMMIT_OP_FAIL,
+                       "Rebalance Commit Failed");
+                goto out;
+            }
+            break;
         }
 
-        ret = 0;
+        default:
+            break;
+    }
+
+    ret = 0;
 out:
-        gf_log (this->name, GF_LOG_DEBUG, "OP = %d. Returning %d", op, ret);
-        return ret;
+    gf_msg_debug(this->name, 0, "OP = %d. Returning %d", op, ret);
+    return ret;
 }
 
 int32_t
-gd_mgmt_v3_post_validate_fn (glusterd_op_t op, int32_t op_ret, dict_t *dict,
-                             char **op_errstr, dict_t *rsp_dict)
+gd_mgmt_v3_post_commit_fn(glusterd_op_t op, dict_t *dict, char **op_errstr,
+                          uint32_t *op_errno, dict_t *rsp_dict)
 {
-        int32_t       ret = -1;
-        xlator_t     *this = NULL;
-
-        this = THIS;
-        GF_ASSERT (this);
-        GF_ASSERT (dict);
-        GF_ASSERT (op_errstr);
-        GF_ASSERT (rsp_dict);
-
-        switch (op) {
-               case GD_OP_SNAP:
-               {
-                       ret = glusterd_snapshot_postvalidate (dict, op_ret,
-                                                             op_errstr,
-                                                             rsp_dict);
-                       if (ret) {
-                               gf_log (this->name, GF_LOG_WARNING,
-                                       "postvalidate operation failed");
-                               goto out;
-                       }
-                       break;
-               }
-               default:
-                       break;
-        }
-
-        ret = 0;
+    int32_t ret = -1;
+    xlator_t *this = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+    GF_ASSERT(dict);
+    GF_ASSERT(op_errstr);
+    GF_VALIDATE_OR_GOTO(this->name, op_errno, out);
+    GF_ASSERT(rsp_dict);
+
+    switch (op) {
+        case GD_OP_ADD_BRICK:
+            ret = glusterd_post_commit_add_brick(dict, op_errstr);
+            if (ret) {
+                gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_POST_COMMIT_OP_FAIL,
+                       "Add-brick post commit failed.");
+                goto out;
+            }
+            break;
+        case GD_OP_REPLACE_BRICK:
+            ret = glusterd_post_commit_replace_brick(dict, op_errstr);
+            if (ret) {
+                gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_POST_COMMIT_OP_FAIL,
+                       "Replace-brick post commit failed.");
+                goto out;
+            }
+            break;
+        default:
+            break;
+    }
 
+    ret = 0;
 out:
-        gf_log (this->name, GF_LOG_TRACE, "OP = %d. Returning %d", op, ret);
-        return ret;
+    gf_msg_debug(this->name, 0, "OP = %d. Returning %d", op, ret);
+    return ret;
 }
 
 int32_t
-gd_mgmt_v3_lock_cbk_fn (struct rpc_req *req, struct iovec *iov,
-                                int count, void *myframe)
+gd_mgmt_v3_post_validate_fn(glusterd_op_t op, int32_t op_ret, dict_t *dict,
+                            char **op_errstr, dict_t *rsp_dict)
 {
-        int32_t                     ret           = -1;
-        struct syncargs            *args          = NULL;
-        glusterd_peerinfo_t        *peerinfo      = NULL;
-        gd1_mgmt_v3_lock_rsp        rsp           = {{0},};
-        call_frame_t               *frame         = NULL;
-        int32_t                     op_ret        = -1;
-        int32_t                     op_errno      = -1;
-        xlator_t                   *this          = NULL;
-
-        this = THIS;
-        GF_ASSERT (this);
-        GF_ASSERT (req);
-        GF_ASSERT (myframe);
-
-        /* Even though the lock command has failed, while collating the errors
-           (gd_mgmt_v3_collate_errors), args->op_ret and args->op_errno will be
-           used. @args is obtained from frame->local. So before checking the
-           status of the request and going out if its a failure, args should be
-           set to frame->local. Otherwise, while collating args will be NULL.
-           This applies to other phases such as prevalidate, brickop, commit and
-           postvalidate also.
-        */
-        frame  = myframe;
-        args   = frame->local;
-        peerinfo = frame->cookie;
-        frame->local = NULL;
-        frame->cookie = NULL;
-
-        if (-1 == req->rpc_status) {
-                op_errno = ENOTCONN;
+    int32_t ret = -1;
+    xlator_t *this = NULL;
+    char *volname = NULL;
+    glusterd_volinfo_t *volinfo = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+    GF_ASSERT(dict);
+    GF_ASSERT(op_errstr);
+    GF_ASSERT(rsp_dict);
+
+    if (op_ret == 0)
+        glusterd_op_commit_hook(op, dict, GD_COMMIT_HOOK_POST);
+
+    switch (op) {
+        case GD_OP_SNAP: {
+            ret = glusterd_snapshot_postvalidate(dict, op_ret, op_errstr,
+                                                 rsp_dict);
+            if (ret) {
+                gf_msg(this->name, GF_LOG_WARNING, 0,
+                       GD_MSG_POST_VALIDATION_FAIL,
+                       "postvalidate operation failed");
                 goto out;
-        }
+            }
+            break;
+        }
+        case GD_OP_ADD_BRICK: {
+            ret = dict_get_strn(dict, "volname", SLEN("volname"), &volname);
+            if (ret) {
+                gf_msg("glusterd", GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+                       "Unable to get"
+                       " volume name");
+                goto out;
+            }
+
+            ret = glusterd_volinfo_find(volname, &volinfo);
+            if (ret) {
+                gf_msg("glusterd", GF_LOG_ERROR, EINVAL, GD_MSG_VOL_NOT_FOUND,
+                       "Unable to "
+                       "allocate memory");
+                goto out;
+            }
+            ret = glusterd_create_volfiles_and_notify_services(volinfo);
+            if (ret)
+                goto out;
+            ret = glusterd_store_volinfo(volinfo,
+                                         GLUSTERD_VOLINFO_VER_AC_INCREMENT);
+            if (ret)
+                goto out;
+            break;
+        }
+        case GD_OP_START_VOLUME: {
+            ret = dict_get_strn(dict, "volname", SLEN("volname"), &volname);
+            if (ret) {
+                gf_msg("glusterd", GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+                       "Unable to get"
+                       " volume name");
+                goto out;
+            }
 
-        GF_VALIDATE_OR_GOTO_WITH_ERROR (this->name, iov, out, op_errno,
-                                        EINVAL);
+            ret = glusterd_volinfo_find(volname, &volinfo);
+            if (ret) {
+                gf_msg("glusterd", GF_LOG_ERROR, EINVAL, GD_MSG_VOL_NOT_FOUND,
+                       "Unable to "
+                       "allocate memory");
+                goto out;
+            }
 
-        ret = xdr_to_generic (*iov, &rsp,
-                              (xdrproc_t)xdr_gd1_mgmt_v3_lock_rsp);
-        if (ret < 0)
+            break;
+        }
+        case GD_OP_STOP_VOLUME: {
+            ret = dict_get_strn(dict, "volname", SLEN("volname"), &volname);
+            if (ret) {
+                gf_msg("glusterd", GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+                       "Unable to get"
+                       " volume name");
                 goto out;
+            }
 
-        gf_uuid_copy (args->uuid, rsp.uuid);
+            ret = glusterd_volinfo_find(volname, &volinfo);
+            if (ret) {
+                gf_msg("glusterd", GF_LOG_ERROR, EINVAL, GD_MSG_VOL_NOT_FOUND,
+                       "Unable to "
+                       "allocate memory");
+                goto out;
+            }
+            break;
+        }
 
-        op_ret = rsp.op_ret;
-        op_errno = rsp.op_errno;
+        default:
+            break;
+    }
+
+    ret = 0;
 
 out:
-        gd_mgmt_v3_collate_errors (args, op_ret, op_errno, NULL,
-                                   GLUSTERD_MGMT_V3_LOCK,
-                                   peerinfo, rsp.uuid);
-        if (rsp.dict.dict_val)
-                free (rsp.dict.dict_val);
-        STACK_DESTROY (frame->root);
-        synctask_barrier_wake(args);
-        return 0;
+    gf_msg_trace(this->name, 0, "OP = %d. Returning %d", op, ret);
+    return ret;
 }
 
 int32_t
-gd_mgmt_v3_lock_cbk (struct rpc_req *req, struct iovec *iov,
-                             int count, void *myframe)
+gd_mgmt_v3_lock_cbk_fn(struct rpc_req *req, struct iovec *iov, int count,
+                       void *myframe)
 {
-        return glusterd_big_locked_cbk (req, iov, count, myframe,
-                                        gd_mgmt_v3_lock_cbk_fn);
+    int32_t ret = -1;
+    struct syncargs *args = NULL;
+    gd1_mgmt_v3_lock_rsp rsp = {
+        {0},
+    };
+    call_frame_t *frame = NULL;
+    int32_t op_ret = -1;
+    int32_t op_errno = -1;
+    xlator_t *this = NULL;
+    uuid_t *peerid = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+    GF_ASSERT(req);
+    GF_ASSERT(myframe);
+
+    /* Even though the lock command has failed, while collating the errors
+       (gd_mgmt_v3_collate_errors), args->op_ret and args->op_errno will be
+       used. @args is obtained from frame->local. So before checking the
+       status of the request and going out if its a failure, args should be
+       set to frame->local. Otherwise, while collating args will be NULL.
+       This applies to other phases such as prevalidate, brickop, commit and
+       postvalidate also.
+    */
+    frame = myframe;
+    args = frame->local;
+    peerid = frame->cookie;
+    frame->local = NULL;
+    frame->cookie = NULL;
+
+    if (-1 == req->rpc_status) {
+        op_errno = ENOTCONN;
+        goto out;
+    }
+
+    GF_VALIDATE_OR_GOTO_WITH_ERROR(this->name, iov, out, op_errno, EINVAL);
+
+    ret = xdr_to_generic(*iov, &rsp, (xdrproc_t)xdr_gd1_mgmt_v3_lock_rsp);
+    if (ret < 0)
+        goto out;
+
+    gf_uuid_copy(args->uuid, rsp.uuid);
+
+    op_ret = rsp.op_ret;
+    op_errno = rsp.op_errno;
+
+out:
+    gd_mgmt_v3_collate_errors(args, op_ret, op_errno, NULL,
+                              GLUSTERD_MGMT_V3_LOCK, *peerid, rsp.uuid);
+    GF_FREE(peerid);
+
+    if (rsp.dict.dict_val)
+        free(rsp.dict.dict_val);
+    /* req->rpc_status set to -1 means, STACK_DESTROY will be called from
+     * the caller function.
+     */
+    if (req->rpc_status != -1)
+        STACK_DESTROY(frame->root);
+    synctask_barrier_wake(args);
+    return 0;
 }
 
-int
-gd_mgmt_v3_lock (glusterd_op_t op, dict_t *op_ctx,
-                 glusterd_peerinfo_t *peerinfo,
-                 struct syncargs *args, uuid_t my_uuid,
-                 uuid_t recv_uuid)
+int32_t
+gd_mgmt_v3_lock_cbk(struct rpc_req *req, struct iovec *iov, int count,
+                    void *myframe)
 {
-        gd1_mgmt_v3_lock_req     req  = {{0},};
-        glusterd_conf_t         *conf = THIS->private;
-        int32_t                  ret  = -1;
-        xlator_t                *this = NULL;
-
-        this = THIS;
-        GF_ASSERT (this);
-        GF_ASSERT (op_ctx);
-        GF_ASSERT (peerinfo);
-        GF_ASSERT (args);
-
-        ret = dict_allocate_and_serialize (op_ctx,
-                                           &req.dict.dict_val,
-                                           &req.dict.dict_len);
-        if (ret)
-                goto out;
-
-        gf_uuid_copy (req.uuid, my_uuid);
-        req.op = op;
-        synclock_unlock (&conf->big_lock);
+    return glusterd_big_locked_cbk(req, iov, count, myframe,
+                                   gd_mgmt_v3_lock_cbk_fn);
+}
 
-        ret = gd_syncop_submit_request (peerinfo->rpc, &req, args, peerinfo,
-                                        &gd_mgmt_v3_prog,
-                                        GLUSTERD_MGMT_V3_LOCK,
-                                        gd_mgmt_v3_lock_cbk,
-                                        (xdrproc_t) xdr_gd1_mgmt_v3_lock_req);
-        synclock_lock (&conf->big_lock);
+int
+gd_mgmt_v3_lock(glusterd_op_t op, dict_t *op_ctx, glusterd_peerinfo_t *peerinfo,
+                struct syncargs *args, uuid_t my_uuid, uuid_t recv_uuid)
+{
+    gd1_mgmt_v3_lock_req req = {
+        {0},
+    };
+    int32_t ret = -1;
+    xlator_t *this = NULL;
+    uuid_t *peerid = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+    GF_ASSERT(op_ctx);
+    GF_ASSERT(peerinfo);
+    GF_ASSERT(args);
+
+    ret = dict_allocate_and_serialize(op_ctx, &req.dict.dict_val,
+                                      &req.dict.dict_len);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno,
+                GD_MSG_DICT_ALLOC_AND_SERL_LENGTH_GET_FAIL, NULL);
+        goto out;
+    }
+
+    gf_uuid_copy(req.uuid, my_uuid);
+    req.op = op;
+
+    GD_ALLOC_COPY_UUID(peerid, peerinfo->uuid, ret);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno,
+                GD_MSG_ALLOC_AND_COPY_UUID_FAIL, NULL);
+        goto out;
+    }
+
+    ret = gd_syncop_submit_request(peerinfo->rpc, &req, args, peerid,
+                                   &gd_mgmt_v3_prog, GLUSTERD_MGMT_V3_LOCK,
+                                   gd_mgmt_v3_lock_cbk,
+                                   (xdrproc_t)xdr_gd1_mgmt_v3_lock_req);
 out:
-        GF_FREE (req.dict.dict_val);
-        gf_log (this->name, GF_LOG_TRACE, "Returning %d", ret);
-        return ret;
+    GF_FREE(req.dict.dict_val);
+    gf_msg_trace(this->name, 0, "Returning %d", ret);
+    return ret;
 }
 
 int
-glusterd_mgmt_v3_initiate_lockdown (glusterd_op_t op, dict_t *dict,
-                                    char **op_errstr, int npeers,
-                                    gf_boolean_t  *is_acquired,
-                                    struct cds_list_head *peers)
+glusterd_mgmt_v3_initiate_lockdown(glusterd_op_t op, dict_t *dict,
+                                   char **op_errstr, uint32_t *op_errno,
+                                   gf_boolean_t *is_acquired,
+                                   uint32_t txn_generation)
 {
-        char                *volname    = NULL;
-        glusterd_peerinfo_t *peerinfo   = NULL;
-        int32_t              ret        = -1;
-        int32_t              peer_cnt   = 0;
-        struct syncargs      args       = {0};
-        uuid_t               peer_uuid  = {0};
-        xlator_t            *this       = NULL;
-
-        this = THIS;
-        GF_ASSERT (this);
-        GF_ASSERT (dict);
-        GF_ASSERT (op_errstr);
-        GF_ASSERT (is_acquired);
-
-        /* Trying to acquire multiple mgmt_v3 locks on local node */
-        ret = glusterd_multiple_mgmt_v3_lock (dict, MY_UUID);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "Failed to acquire mgmt_v3 locks on localhost");
-                goto out;
-        }
+    glusterd_peerinfo_t *peerinfo = NULL;
+    int32_t ret = -1;
+    int32_t peer_cnt = 0;
+    struct syncargs args = {0};
+    uuid_t peer_uuid = {0};
+    xlator_t *this = NULL;
+    glusterd_conf_t *conf = NULL;
+    uint32_t timeout = 0;
+
+    this = THIS;
+    GF_ASSERT(this);
+    conf = this->private;
+    GF_ASSERT(conf);
+
+    GF_ASSERT(dict);
+    GF_ASSERT(op_errstr);
+    GF_ASSERT(is_acquired);
+
+    /* Cli will add timeout key to dict if the default timeout is
+     * other than 2 minutes. Here we use this value to check whether
+     * mgmt_v3_lock_timeout should be set to default value or we
+     * need to change the value according to timeout value
+     * i.e, timeout + 120 seconds. */
+    ret = dict_get_uint32(dict, "timeout", &timeout);
+    if (!ret)
+        conf->mgmt_v3_lock_timeout = timeout + 120;
+
+    /* Trying to acquire multiple mgmt_v3 locks on local node */
+    ret = glusterd_multiple_mgmt_v3_lock(dict, MY_UUID, op_errno);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_MGMTV3_LOCK_GET_FAIL,
+               "Failed to acquire mgmt_v3 locks on localhost");
+        goto out;
+    }
+
+    *is_acquired = _gf_true;
+
+    /* Sending mgmt_v3 lock req to other nodes in the cluster */
+    gd_syncargs_init(&args, NULL);
+    ret = synctask_barrier_init((&args));
+    if (ret)
+        goto out;
+
+    peer_cnt = 0;
+
+    RCU_READ_LOCK;
+    cds_list_for_each_entry_rcu(peerinfo, &conf->peers, uuid_list)
+    {
+        /* Only send requests to peers who were available before the
+         * transaction started
+         */
+        if (peerinfo->generation > txn_generation)
+            continue;
 
-        *is_acquired = _gf_true;
+        if (!peerinfo->connected)
+            continue;
+        if (op != GD_OP_SYNC_VOLUME &&
+            peerinfo->state.state != GD_FRIEND_STATE_BEFRIENDED)
+            continue;
 
-        if (!npeers) {
-                ret = 0;
-                goto out;
-        }
+        gd_mgmt_v3_lock(op, dict, peerinfo, &args, MY_UUID, peer_uuid);
+        peer_cnt++;
+    }
+    RCU_READ_UNLOCK;
 
-        /* Sending mgmt_v3 lock req to other nodes in the cluster */
-        gd_syncargs_init (&args, NULL);
-        synctask_barrier_init((&args));
-        peer_cnt = 0;
-        list_for_each_local_xaction_peers (peerinfo, peers) {
-                gd_mgmt_v3_lock (op, dict, peerinfo, &args,
-                                 MY_UUID, peer_uuid);
-                peer_cnt++;
-        }
-        gd_synctask_barrier_wait((&args), peer_cnt);
+    if (0 == peer_cnt) {
+        ret = 0;
+        goto out;
+    }
 
-        if (args.errstr)
-                *op_errstr = gf_strdup (args.errstr);
+    gd_synctask_barrier_wait((&args), peer_cnt);
 
-        ret = args.op_ret;
+    if (args.errstr)
+        *op_errstr = gf_strdup(args.errstr);
 
-        gf_log (this->name, GF_LOG_DEBUG, "Sent lock op req for %s "
-                "to %d peers. Returning %d", gd_op_list[op], peer_cnt, ret);
+    ret = args.op_ret;
+    *op_errno = args.op_errno;
+
+    gf_msg_debug(this->name, 0,
+                 "Sent lock op req for %s "
+                 "to %d peers. Returning %d",
+                 gd_op_list[op], peer_cnt, ret);
 out:
-        if (ret) {
-                if (*op_errstr)
-                        gf_log (this->name, GF_LOG_ERROR, "%s",
-                                *op_errstr);
-
-                if (volname)
-                        ret = gf_asprintf (op_errstr,
-                                           "Another transaction is in progress "
-                                           "for %s. Please try again after "
-                                           "sometime.", volname);
-                else
-                        ret = gf_asprintf (op_errstr,
-                                           "Another transaction is in progress "
-                                           "Please try again after sometime.");
-
-                if (ret == -1)
-                        *op_errstr = NULL;
+    if (ret) {
+        if (*op_errstr)
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_MGMTV3_LOCK_GET_FAIL,
+                   "%s", *op_errstr);
 
-                ret = -1;
-        }
+        ret = gf_asprintf(op_errstr,
+                          "Another transaction is in progress. "
+                          "Please try again after some time.");
+
+        if (ret == -1)
+            *op_errstr = NULL;
+
+        ret = -1;
+    }
 
-        return ret;
+    return ret;
 }
 
 int
-glusterd_pre_validate_aggr_rsp_dict (glusterd_op_t op,
-                                     dict_t *aggr, dict_t *rsp)
+glusterd_pre_validate_aggr_rsp_dict(glusterd_op_t op, dict_t *aggr, dict_t *rsp)
 {
-        int32_t              ret      = 0;
-        xlator_t            *this     = NULL;
+    int32_t ret = 0;
+    xlator_t *this = NULL;
 
-        this = THIS;
-        GF_ASSERT (this);
-        GF_ASSERT (aggr);
-        GF_ASSERT (rsp);
+    this = THIS;
+    GF_ASSERT(this);
+    GF_ASSERT(aggr);
+    GF_ASSERT(rsp);
 
-        switch (op) {
+    switch (op) {
         case GD_OP_SNAP:
-                ret = glusterd_snap_pre_validate_use_rsp_dict (aggr, rsp);
-                if (ret) {
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "Failed to aggregate prevalidate "
-                                "response dictionaries.");
-                        goto out;
-                }
-                break;
+            ret = glusterd_snap_pre_validate_use_rsp_dict(aggr, rsp);
+            if (ret) {
+                gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_PRE_VALIDATION_FAIL,
+                       "Failed to aggregate prevalidate "
+                       "response dictionaries.");
+                goto out;
+            }
+            break;
+        case GD_OP_REPLACE_BRICK:
+            ret = glusterd_rb_use_rsp_dict(aggr, rsp);
+            if (ret) {
+                gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_PRE_VALIDATION_FAIL,
+                       "Failed to aggregate prevalidate "
+                       "response dictionaries.");
+                goto out;
+            }
+            break;
+        case GD_OP_START_VOLUME:
+        case GD_OP_ADD_BRICK:
+            ret = glusterd_aggr_brick_mount_dirs(aggr, rsp);
+            if (ret) {
+                gf_msg(this->name, GF_LOG_ERROR, 0,
+                       GD_MSG_BRICK_MOUNDIRS_AGGR_FAIL,
+                       "Failed to "
+                       "aggregate brick mount dirs");
+                goto out;
+            }
+            break;
+        case GD_OP_RESET_BRICK:
+            ret = glusterd_rb_use_rsp_dict(aggr, rsp);
+            if (ret) {
+                gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_PRE_VALIDATION_FAIL,
+                       "Failed to aggregate prevalidate "
+                       "response dictionaries.");
+                goto out;
+            }
+        case GD_OP_STOP_VOLUME:
+        case GD_OP_REMOVE_BRICK:
+        case GD_OP_PROFILE_VOLUME:
+        case GD_OP_DEFRAG_BRICK_VOLUME:
+        case GD_OP_REBALANCE:
+            break;
+        case GD_OP_MAX_OPVERSION:
+            break;
         default:
-                ret = -1;
-                gf_log (this->name, GF_LOG_ERROR, "Invalid op (%s)",
-                        gd_op_list[op]);
+            ret = -1;
+            gf_msg(this->name, GF_LOG_ERROR, EINVAL, GD_MSG_INVALID_ENTRY,
+                   "Invalid op (%s)", gd_op_list[op]);
 
-                break;
-        }
+            break;
+    }
 out:
-        return ret;
+    return ret;
 }
 
 int32_t
-gd_mgmt_v3_pre_validate_cbk_fn (struct rpc_req *req, struct iovec *iov,
-                                int count, void *myframe)
+gd_mgmt_v3_pre_validate_cbk_fn(struct rpc_req *req, struct iovec *iov,
+                               int count, void *myframe)
 {
-        int32_t                     ret           = -1;
-        struct syncargs            *args          = NULL;
-        glusterd_peerinfo_t        *peerinfo      = NULL;
-        gd1_mgmt_v3_pre_val_rsp     rsp           = {{0},};
-        call_frame_t               *frame         = NULL;
-        int32_t                     op_ret        = -1;
-        int32_t                     op_errno      = -1;
-        dict_t                     *rsp_dict      = NULL;
-        xlator_t                   *this          = NULL;
-
-        this = THIS;
-        GF_ASSERT (this);
-        GF_ASSERT (req);
-        GF_ASSERT (myframe);
-
-        frame  = myframe;
-        args   = frame->local;
-        peerinfo = frame->cookie;
-        frame->local = NULL;
-        frame->cookie = NULL;
-
-        if (-1 == req->rpc_status) {
-                op_errno = ENOTCONN;
-                goto out;
-        }
-
-        GF_VALIDATE_OR_GOTO_WITH_ERROR (this->name, iov, out, op_errno,
-                                        EINVAL);
-
-        ret = xdr_to_generic (*iov, &rsp,
-                              (xdrproc_t)xdr_gd1_mgmt_v3_pre_val_rsp);
-        if (ret < 0)
-                goto out;
-
-        if (rsp.dict.dict_len) {
-                /* Unserialize the dictionary */
-                rsp_dict  = dict_new ();
-
-                ret = dict_unserialize (rsp.dict.dict_val,
-                                        rsp.dict.dict_len,
-                                        &rsp_dict);
-                if (ret < 0) {
-                        free (rsp.dict.dict_val);
-                        goto out;
-                } else {
-                        rsp_dict->extra_stdfree = rsp.dict.dict_val;
-                }
-        }
-
-        gf_uuid_copy (args->uuid, rsp.uuid);
-        pthread_mutex_lock (&args->lock_dict);
-        {
-                ret = glusterd_pre_validate_aggr_rsp_dict (rsp.op, args->dict,
-                                                           rsp_dict);
-        }
-        pthread_mutex_unlock (&args->lock_dict);
-
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "%s",
-                        "Failed to aggregate response from "
-                        " node/brick");
-                if (!rsp.op_ret)
-                        op_ret = ret;
-                else {
-                        op_ret = rsp.op_ret;
-                        op_errno = rsp.op_errno;
-                }
+    int32_t ret = -1;
+    struct syncargs *args = NULL;
+    gd1_mgmt_v3_pre_val_rsp rsp = {
+        {0},
+    };
+    call_frame_t *frame = NULL;
+    int32_t op_ret = -1;
+    int32_t op_errno = -1;
+    dict_t *rsp_dict = NULL;
+    xlator_t *this = NULL;
+    uuid_t *peerid = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+    GF_ASSERT(req);
+    GF_ASSERT(myframe);
+
+    frame = myframe;
+    args = frame->local;
+    peerid = frame->cookie;
+    frame->local = NULL;
+    frame->cookie = NULL;
+
+    if (-1 == req->rpc_status) {
+        op_errno = ENOTCONN;
+        goto out;
+    }
+
+    GF_VALIDATE_OR_GOTO_WITH_ERROR(this->name, iov, out, op_errno, EINVAL);
+
+    ret = xdr_to_generic(*iov, &rsp, (xdrproc_t)xdr_gd1_mgmt_v3_pre_val_rsp);
+    if (ret < 0)
+        goto out;
+
+    if (rsp.dict.dict_len) {
+        /* Unserialize the dictionary */
+        rsp_dict = dict_new();
+
+        ret = dict_unserialize(rsp.dict.dict_val, rsp.dict.dict_len, &rsp_dict);
+        if (ret < 0) {
+            free(rsp.dict.dict_val);
+            goto out;
         } else {
-                op_ret = rsp.op_ret;
-                op_errno = rsp.op_errno;
-        }
+            rsp_dict->extra_stdfree = rsp.dict.dict_val;
+        }
+    }
+
+    gf_uuid_copy(args->uuid, rsp.uuid);
+    pthread_mutex_lock(&args->lock_dict);
+    {
+        ret = glusterd_pre_validate_aggr_rsp_dict(rsp.op, args->dict, rsp_dict);
+    }
+    pthread_mutex_unlock(&args->lock_dict);
+
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_RESP_AGGR_FAIL, "%s",
+               "Failed to aggregate response from "
+               " node/brick");
+        if (!rsp.op_ret)
+            op_ret = ret;
+        else {
+            op_ret = rsp.op_ret;
+            op_errno = rsp.op_errno;
+        }
+    } else {
+        op_ret = rsp.op_ret;
+        op_errno = rsp.op_errno;
+    }
 
 out:
-        if (rsp_dict)
-                dict_unref (rsp_dict);
-
-        gd_mgmt_v3_collate_errors (args, op_ret, op_errno, rsp.op_errstr,
-                                  GLUSTERD_MGMT_V3_PRE_VALIDATE,
-                                  peerinfo, rsp.uuid);
-
-        if (rsp.op_errstr)
-                free (rsp.op_errstr);
-
-        STACK_DESTROY (frame->root);
-        synctask_barrier_wake(args);
-        return 0;
+    if (rsp_dict)
+        dict_unref(rsp_dict);
+
+    gd_mgmt_v3_collate_errors(args, op_ret, op_errno, rsp.op_errstr,
+                              GLUSTERD_MGMT_V3_PRE_VALIDATE, *peerid, rsp.uuid);
+
+    if (rsp.op_errstr)
+        free(rsp.op_errstr);
+    GF_FREE(peerid);
+    /* req->rpc_status set to -1 means, STACK_DESTROY will be called from
+     * the caller function.
+     */
+    if (req->rpc_status != -1)
+        STACK_DESTROY(frame->root);
+    synctask_barrier_wake(args);
+    return 0;
 }
 
 int32_t
-gd_mgmt_v3_pre_validate_cbk (struct rpc_req *req, struct iovec *iov,
-                             int count, void *myframe)
+gd_mgmt_v3_pre_validate_cbk(struct rpc_req *req, struct iovec *iov, int count,
+                            void *myframe)
 {
-        return glusterd_big_locked_cbk (req, iov, count, myframe,
-                                        gd_mgmt_v3_pre_validate_cbk_fn);
+    return glusterd_big_locked_cbk(req, iov, count, myframe,
+                                   gd_mgmt_v3_pre_validate_cbk_fn);
 }
 
 int
-gd_mgmt_v3_pre_validate_req (glusterd_op_t op, dict_t *op_ctx,
-                             glusterd_peerinfo_t *peerinfo,
-                             struct syncargs *args, uuid_t my_uuid,
-                             uuid_t recv_uuid)
+gd_mgmt_v3_pre_validate_req(glusterd_op_t op, dict_t *op_ctx,
+                            glusterd_peerinfo_t *peerinfo,
+                            struct syncargs *args, uuid_t my_uuid,
+                            uuid_t recv_uuid)
 {
-        int32_t                  ret   = -1;
-        gd1_mgmt_v3_pre_val_req  req   = {{0},};
-        glusterd_conf_t         *conf  = THIS->private;
-        xlator_t                *this  = NULL;
-
-        this = THIS;
-        GF_ASSERT (this);
-        GF_ASSERT (op_ctx);
-        GF_ASSERT (peerinfo);
-        GF_ASSERT (args);
-
-        ret = dict_allocate_and_serialize (op_ctx,
-                                           &req.dict.dict_val,
-                                           &req.dict.dict_len);
-        if (ret)
-                goto out;
-
-        gf_uuid_copy (req.uuid, my_uuid);
-        req.op = op;
-        synclock_unlock (&conf->big_lock);
-
-        ret = gd_syncop_submit_request (peerinfo->rpc, &req, args, peerinfo,
-                                        &gd_mgmt_v3_prog,
-                                        GLUSTERD_MGMT_V3_PRE_VALIDATE,
-                                        gd_mgmt_v3_pre_validate_cbk,
-                                        (xdrproc_t) xdr_gd1_mgmt_v3_pre_val_req);
-        synclock_lock (&conf->big_lock);
+    int32_t ret = -1;
+    gd1_mgmt_v3_pre_val_req req = {
+        {0},
+    };
+    xlator_t *this = NULL;
+    uuid_t *peerid = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+    GF_ASSERT(op_ctx);
+    GF_ASSERT(peerinfo);
+    GF_ASSERT(args);
+
+    ret = dict_allocate_and_serialize(op_ctx, &req.dict.dict_val,
+                                      &req.dict.dict_len);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno,
+                GD_MSG_DICT_ALLOC_AND_SERL_LENGTH_GET_FAIL, NULL);
+        goto out;
+    }
+
+    gf_uuid_copy(req.uuid, my_uuid);
+    req.op = op;
+
+    GD_ALLOC_COPY_UUID(peerid, peerinfo->uuid, ret);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno,
+                GD_MSG_ALLOC_AND_COPY_UUID_FAIL, NULL);
+        goto out;
+    }
+
+    ret = gd_syncop_submit_request(
+        peerinfo->rpc, &req, args, peerid, &gd_mgmt_v3_prog,
+        GLUSTERD_MGMT_V3_PRE_VALIDATE, gd_mgmt_v3_pre_validate_cbk,
+        (xdrproc_t)xdr_gd1_mgmt_v3_pre_val_req);
 out:
-        GF_FREE (req.dict.dict_val);
-        gf_log (this->name, GF_LOG_TRACE, "Returning %d", ret);
-        return ret;
+    GF_FREE(req.dict.dict_val);
+    gf_msg_trace(this->name, 0, "Returning %d", ret);
+    return ret;
 }
 
 int
-glusterd_mgmt_v3_pre_validate (glusterd_op_t op, dict_t *req_dict,
-                               char **op_errstr, int npeers,
-                               struct cds_list_head *peers)
+glusterd_mgmt_v3_pre_validate(glusterd_op_t op, dict_t *req_dict,
+                              char **op_errstr, uint32_t *op_errno,
+                              uint32_t txn_generation)
 {
-        int32_t              ret        = -1;
-        int32_t              peer_cnt   = 0;
-        dict_t              *rsp_dict   = NULL;
-        glusterd_peerinfo_t *peerinfo   = NULL;
-        struct syncargs      args       = {0};
-        uuid_t               peer_uuid  = {0};
-        xlator_t            *this       = NULL;
-
-        this = THIS;
-        GF_ASSERT (this);
-        GF_ASSERT (req_dict);
-        GF_ASSERT (op_errstr);
-
-        rsp_dict = dict_new ();
-        if (!rsp_dict) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "Failed to create response dictionary");
-                goto out;
+    int32_t ret = -1;
+    int32_t peer_cnt = 0;
+    dict_t *rsp_dict = NULL;
+    glusterd_peerinfo_t *peerinfo = NULL;
+    struct syncargs args = {0};
+    uuid_t peer_uuid = {0};
+    xlator_t *this = NULL;
+    glusterd_conf_t *conf = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+    conf = this->private;
+    GF_ASSERT(conf);
+
+    GF_ASSERT(req_dict);
+    GF_ASSERT(op_errstr);
+    GF_VALIDATE_OR_GOTO(this->name, op_errno, out);
+
+    rsp_dict = dict_new();
+    if (!rsp_dict) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_CREATE_FAIL,
+               "Failed to create response dictionary");
+        goto out;
+    }
+
+    if (op == GD_OP_PROFILE_VOLUME || op == GD_OP_STOP_VOLUME ||
+        op == GD_OP_REBALANCE || op == GD_OP_REMOVE_BRICK) {
+        ret = glusterd_validate_quorum(this, op, req_dict, op_errstr);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_SERVER_QUORUM_NOT_MET,
+                   "Server quorum not met. Rejecting operation.");
+            goto out;
         }
+    }
 
-        /* Pre Validation on local node */
-        ret = gd_mgmt_v3_pre_validate_fn (op, req_dict, op_errstr,
-                                          rsp_dict);
+    /* Pre Validation on local node */
+    ret = gd_mgmt_v3_pre_validate_fn(op, req_dict, op_errstr, rsp_dict,
+                                     op_errno);
 
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "Pre Validation failed for "
-                        "operation %s on local node",
-                        gd_op_list[op]);
-
-                if (*op_errstr == NULL) {
-                        ret = gf_asprintf (op_errstr,
-                                           "Pre-validation failed "
-                                           "on localhost. Please "
-                                           "check log file for details");
-                        if (ret == -1)
-                                *op_errstr = NULL;
-
-                        ret = -1;
-                }
-                goto out;
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_PRE_VALIDATION_FAIL,
+               "Pre Validation failed for "
+               "operation %s on local node",
+               gd_op_list[op]);
+
+        if (*op_errstr == NULL) {
+            ret = gf_asprintf(op_errstr,
+                              "Pre-validation failed "
+                              "on localhost. Please "
+                              "check log file for details");
+            if (ret == -1)
+                *op_errstr = NULL;
+
+            ret = -1;
         }
+        goto out;
+    }
 
-        ret = glusterd_pre_validate_aggr_rsp_dict (op, req_dict,
-                                                   rsp_dict);
+    if (op != GD_OP_MAX_OPVERSION) {
+        ret = glusterd_pre_validate_aggr_rsp_dict(op, req_dict, rsp_dict);
         if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "%s",
-                        "Failed to aggregate response from "
-                        " node/brick");
-                goto out;
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_PRE_VALIDATION_FAIL,
+                   "%s",
+                   "Failed to aggregate response from "
+                   " node/brick");
+            goto out;
         }
 
-        dict_unref (rsp_dict);
+        dict_unref(rsp_dict);
         rsp_dict = NULL;
+    }
 
-        if (!npeers) {
-                ret = 0;
-                goto out;
-        }
+    /* Sending Pre Validation req to other nodes in the cluster */
+    gd_syncargs_init(&args, req_dict);
+    ret = synctask_barrier_init((&args));
+    if (ret)
+        goto out;
 
-        /* Sending Pre Validation req to other nodes in the cluster */
-        gd_syncargs_init (&args, req_dict);
-        synctask_barrier_init((&args));
-        peer_cnt = 0;
-        list_for_each_local_xaction_peers (peerinfo, peers) {
-                gd_mgmt_v3_pre_validate_req (op, req_dict, peerinfo, &args,
-                                             MY_UUID, peer_uuid);
-                peer_cnt++;
-        }
-        gd_synctask_barrier_wait((&args), peer_cnt);
+    peer_cnt = 0;
 
-        if (args.op_ret) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "Pre Validation failed on peers");
+    RCU_READ_LOCK;
+    cds_list_for_each_entry_rcu(peerinfo, &conf->peers, uuid_list)
+    {
+        /* Only send requests to peers who were available before the
+         * transaction started
+         */
+        if (peerinfo->generation > txn_generation)
+            continue;
+
+        if (!peerinfo->connected)
+            continue;
+        if (op != GD_OP_SYNC_VOLUME &&
+            peerinfo->state.state != GD_FRIEND_STATE_BEFRIENDED)
+            continue;
+
+        gd_mgmt_v3_pre_validate_req(op, req_dict, peerinfo, &args, MY_UUID,
+                                    peer_uuid);
+        peer_cnt++;
+    }
+    RCU_READ_UNLOCK;
+
+    if (0 == peer_cnt) {
+        ret = 0;
+        goto out;
+    }
 
-                if (args.errstr)
-                         *op_errstr = gf_strdup (args.errstr);
-        }
+    gd_synctask_barrier_wait((&args), peer_cnt);
 
-        ret = args.op_ret;
+    if (args.op_ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_PRE_VALIDATION_FAIL,
+               "Pre Validation failed on peers");
 
-        gf_log (this->name, GF_LOG_DEBUG, "Sent pre valaidation req for %s "
-                "to %d peers. Returning %d", gd_op_list[op], peer_cnt, ret);
+        if (args.errstr)
+            *op_errstr = gf_strdup(args.errstr);
+    }
+
+    ret = args.op_ret;
+    *op_errno = args.op_errno;
+
+    gf_msg_debug(this->name, 0,
+                 "Sent pre valaidation req for %s "
+                 "to %d peers. Returning %d",
+                 gd_op_list[op], peer_cnt, ret);
 out:
-        return ret;
+    return ret;
 }
 
 int
-glusterd_mgmt_v3_build_payload (dict_t **req, char **op_errstr, dict_t *dict,
-                                glusterd_op_t op)
+glusterd_mgmt_v3_build_payload(dict_t **req, char **op_errstr, dict_t *dict,
+                               glusterd_op_t op)
 {
-        int32_t                 ret      = -1;
-        dict_t                 *req_dict = NULL;
-        xlator_t               *this     = NULL;
-
-        this = THIS;
-        GF_ASSERT (this);
-        GF_ASSERT (req);
-        GF_ASSERT (op_errstr);
-        GF_ASSERT (dict);
-
-        req_dict = dict_new ();
-        if (!req_dict)
+    int32_t ret = -1;
+    dict_t *req_dict = NULL;
+    xlator_t *this = NULL;
+    char *volname = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+    GF_ASSERT(req);
+    GF_ASSERT(op_errstr);
+    GF_ASSERT(dict);
+
+    req_dict = dict_new();
+    if (!req_dict)
+        goto out;
+
+    switch (op) {
+        case GD_OP_MAX_OPVERSION:
+        case GD_OP_SNAP:
+            dict_copy(dict, req_dict);
+            break;
+        case GD_OP_START_VOLUME:
+        case GD_OP_STOP_VOLUME:
+        case GD_OP_ADD_BRICK:
+        case GD_OP_REMOVE_BRICK:
+        case GD_OP_DEFRAG_BRICK_VOLUME:
+        case GD_OP_REPLACE_BRICK:
+        case GD_OP_RESET_BRICK:
+        case GD_OP_PROFILE_VOLUME: {
+            ret = dict_get_strn(dict, "volname", SLEN("volname"), &volname);
+            if (ret) {
+                gf_msg(this->name, GF_LOG_CRITICAL, errno,
+                       GD_MSG_DICT_GET_FAILED,
+                       "volname is not present in "
+                       "operation ctx");
+                goto out;
+            }
+
+            if (strcasecmp(volname, "all")) {
+                ret = glusterd_dict_set_volid(dict, volname, op_errstr);
+                if (ret)
+                    goto out;
+            }
+            dict_copy(dict, req_dict);
+        } break;
+
+        case GD_OP_REBALANCE: {
+            if (gd_set_commit_hash(dict) != 0) {
+                ret = -1;
                 goto out;
+            }
+            ret = dict_get_strn(dict, "volname", SLEN("volname"), &volname);
+            if (ret) {
+                gf_msg(this->name, GF_LOG_CRITICAL, errno,
+                       GD_MSG_DICT_GET_FAILED,
+                       "volname is not present in "
+                       "operation ctx");
+                goto out;
+            }
 
-        switch (op) {
-                case GD_OP_SNAP:
-                        dict_copy (dict, req_dict);
-                        break;
-                default:
-                        break;
-        }
+            if (strcasecmp(volname, "all")) {
+                ret = glusterd_dict_set_volid(dict, volname, op_errstr);
+                if (ret)
+                    goto out;
+            }
+            dict_copy(dict, req_dict);
+        } break;
 
-        *req = req_dict;
-        ret = 0;
+        default:
+            break;
+    }
+
+    *req = req_dict;
+    ret = 0;
 out:
-        return ret;
+    return ret;
 }
 
 int32_t
-gd_mgmt_v3_brick_op_cbk_fn (struct rpc_req *req, struct iovec *iov,
-                            int count, void *myframe)
+gd_mgmt_v3_brick_op_cbk_fn(struct rpc_req *req, struct iovec *iov, int count,
+                           void *myframe)
 {
-        int32_t                     ret           = -1;
-        struct syncargs            *args          = NULL;
-        glusterd_peerinfo_t        *peerinfo      = NULL;
-        gd1_mgmt_v3_brick_op_rsp     rsp          = {{0},};
-        call_frame_t               *frame         = NULL;
-        int32_t                     op_ret        = -1;
-        int32_t                     op_errno      = -1;
-        xlator_t                   *this          = NULL;
-
-        this = THIS;
-        GF_ASSERT (this);
-        GF_ASSERT (req);
-        GF_ASSERT (myframe);
-
-        frame  = myframe;
-        args   = frame->local;
-        peerinfo = frame->cookie;
-        frame->local = NULL;
-        frame->cookie = NULL;
-
-        /* If the operation failed, then iov can be NULL. So better check the
-           status of the operation and then worry about iov (if the status of
-           the command is success)
-        */
-        if (-1 == req->rpc_status) {
-                op_errno = ENOTCONN;
-                goto out;
-        }
-
-        GF_VALIDATE_OR_GOTO_WITH_ERROR (this->name, iov, out, op_errno,
-                                        EINVAL);
-
-        ret = xdr_to_generic (*iov, &rsp,
-                              (xdrproc_t)xdr_gd1_mgmt_v3_brick_op_rsp);
-        if (ret < 0)
-                goto out;
-
-        gf_uuid_copy (args->uuid, rsp.uuid);
-
+    int32_t ret = -1;
+    struct syncargs *args = NULL;
+    gd1_mgmt_v3_brick_op_rsp rsp = {
+        {0},
+    };
+    call_frame_t *frame = NULL;
+    int32_t op_ret = -1;
+    int32_t op_errno = -1;
+    dict_t *rsp_dict = NULL;
+    xlator_t *this = NULL;
+    uuid_t *peerid = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+    GF_ASSERT(req);
+    GF_ASSERT(myframe);
+
+    frame = myframe;
+    args = frame->local;
+    peerid = frame->cookie;
+    frame->local = NULL;
+    frame->cookie = NULL;
+
+    /* If the operation failed, then iov can be NULL. So better check the
+       status of the operation and then worry about iov (if the status of
+       the command is success)
+    */
+    if (-1 == req->rpc_status) {
+        op_errno = ENOTCONN;
+        goto out;
+    }
+
+    GF_VALIDATE_OR_GOTO_WITH_ERROR(this->name, iov, out, op_errno, EINVAL);
+
+    ret = xdr_to_generic(*iov, &rsp, (xdrproc_t)xdr_gd1_mgmt_v3_brick_op_rsp);
+    if (ret < 0)
+        goto out;
+
+    if (rsp.dict.dict_len) {
+        /* Unserialize the dictionary */
+        rsp_dict = dict_new();
+
+        ret = dict_unserialize(rsp.dict.dict_val, rsp.dict.dict_len, &rsp_dict);
+        if (ret < 0) {
+            goto out;
+        } else {
+            rsp_dict->extra_stdfree = rsp.dict.dict_val;
+        }
+    }
+
+    gf_uuid_copy(args->uuid, rsp.uuid);
+    pthread_mutex_lock(&args->lock_dict);
+    {
+        if (rsp.op == GD_OP_DEFRAG_BRICK_VOLUME ||
+            rsp.op == GD_OP_PROFILE_VOLUME)
+            ret = glusterd_syncop_aggr_rsp_dict(rsp.op, args->dict, rsp_dict);
+    }
+    pthread_mutex_unlock(&args->lock_dict);
+
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_RESP_AGGR_FAIL, "%s",
+               "Failed to aggregate response from "
+               " node/brick");
+        if (!rsp.op_ret)
+            op_ret = ret;
+        else {
+            op_ret = rsp.op_ret;
+            op_errno = rsp.op_errno;
+        }
+    } else {
         op_ret = rsp.op_ret;
         op_errno = rsp.op_errno;
+    }
 
 out:
-        gd_mgmt_v3_collate_errors (args, op_ret, op_errno, rsp.op_errstr,
-                                   GLUSTERD_MGMT_V3_BRICK_OP,
-                                   peerinfo, rsp.uuid);
 
-        if (rsp.op_errstr)
-                free (rsp.op_errstr);
+    gd_mgmt_v3_collate_errors(args, op_ret, op_errno, rsp.op_errstr,
+                              GLUSTERD_MGMT_V3_BRICK_OP, *peerid, rsp.uuid);
+
+    if (rsp.op_errstr)
+        free(rsp.op_errstr);
 
-        if (rsp.dict.dict_val)
-                free (rsp.dict.dict_val);
+    if (rsp_dict)
+        dict_unref(rsp_dict);
 
-        STACK_DESTROY (frame->root);
-        synctask_barrier_wake(args);
-        return 0;
+    GF_FREE(peerid);
+    /* req->rpc_status set to -1 means, STACK_DESTROY will be called from
+     * the caller function.
+     */
+    if (req->rpc_status != -1)
+        STACK_DESTROY(frame->root);
+    synctask_barrier_wake(args);
+    return 0;
 }
 
 int32_t
-gd_mgmt_v3_brick_op_cbk (struct rpc_req *req, struct iovec *iov,
-                         int count, void *myframe)
+gd_mgmt_v3_brick_op_cbk(struct rpc_req *req, struct iovec *iov, int count,
+                        void *myframe)
 {
-        return glusterd_big_locked_cbk (req, iov, count, myframe,
-                                        gd_mgmt_v3_brick_op_cbk_fn);
+    return glusterd_big_locked_cbk(req, iov, count, myframe,
+                                   gd_mgmt_v3_brick_op_cbk_fn);
 }
 
 int
-gd_mgmt_v3_brick_op_req (glusterd_op_t op, dict_t *op_ctx,
-                         glusterd_peerinfo_t *peerinfo,
-                         struct syncargs *args, uuid_t my_uuid,
-                         uuid_t recv_uuid)
+gd_mgmt_v3_brick_op_req(glusterd_op_t op, dict_t *op_ctx,
+                        glusterd_peerinfo_t *peerinfo, struct syncargs *args,
+                        uuid_t my_uuid, uuid_t recv_uuid)
 {
-        int32_t                   ret  = -1;
-        gd1_mgmt_v3_brick_op_req  req  = {{0},};
-        glusterd_conf_t          *conf = THIS->private;
-        xlator_t                 *this = NULL;
-
-        this = THIS;
-        GF_ASSERT (this);
-        GF_ASSERT (op_ctx);
-        GF_ASSERT (peerinfo);
-        GF_ASSERT (args);
-
-        ret = dict_allocate_and_serialize (op_ctx,
-                                           &req.dict.dict_val,
-                                           &req.dict.dict_len);
-        if (ret)
-                goto out;
-
-        gf_uuid_copy (req.uuid, my_uuid);
-        req.op = op;
-        synclock_unlock (&conf->big_lock);
-
-        ret = gd_syncop_submit_request (peerinfo->rpc, &req, args, peerinfo,
-                                        &gd_mgmt_v3_prog,
-                                        GLUSTERD_MGMT_V3_BRICK_OP,
-                                        gd_mgmt_v3_brick_op_cbk,
-                                        (xdrproc_t) xdr_gd1_mgmt_v3_brick_op_req);
-        synclock_lock (&conf->big_lock);
+    int32_t ret = -1;
+    gd1_mgmt_v3_brick_op_req req = {
+        {0},
+    };
+    xlator_t *this = NULL;
+    uuid_t *peerid = {
+        0,
+    };
+
+    this = THIS;
+    GF_ASSERT(this);
+    GF_ASSERT(op_ctx);
+    GF_ASSERT(peerinfo);
+    GF_ASSERT(args);
+
+    ret = dict_allocate_and_serialize(op_ctx, &req.dict.dict_val,
+                                      &req.dict.dict_len);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno,
+                GD_MSG_DICT_ALLOC_AND_SERL_LENGTH_GET_FAIL, NULL);
+        goto out;
+    }
+
+    gf_uuid_copy(req.uuid, my_uuid);
+    req.op = op;
+
+    GD_ALLOC_COPY_UUID(peerid, peerinfo->uuid, ret);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno,
+                GD_MSG_ALLOC_AND_COPY_UUID_FAIL, NULL);
+        goto out;
+    }
+
+    ret = gd_syncop_submit_request(peerinfo->rpc, &req, args, peerid,
+                                   &gd_mgmt_v3_prog, GLUSTERD_MGMT_V3_BRICK_OP,
+                                   gd_mgmt_v3_brick_op_cbk,
+                                   (xdrproc_t)xdr_gd1_mgmt_v3_brick_op_req);
 out:
-        GF_FREE (req.dict.dict_val);
-        gf_log (this->name, GF_LOG_TRACE, "Returning %d", ret);
-        return ret;
+    GF_FREE(req.dict.dict_val);
+    gf_msg_trace(this->name, 0, "Returning %d", ret);
+    return ret;
 }
 
 int
-glusterd_mgmt_v3_brick_op (glusterd_op_t op, dict_t *req_dict,
-                           char **op_errstr, int npeers,
-                           struct cds_list_head *peers)
+glusterd_mgmt_v3_brick_op(glusterd_op_t op, dict_t *op_ctx, dict_t *req_dict,
+                          char **op_errstr, uint32_t txn_generation)
 {
-        int32_t              ret        = -1;
-        int32_t              peer_cnt   = 0;
-        dict_t              *rsp_dict   = NULL;
-        glusterd_peerinfo_t *peerinfo   = NULL;
-        struct syncargs      args       = {0};
-        uuid_t               peer_uuid  = {0};
-        xlator_t            *this       = NULL;
-
-        this = THIS;
-        GF_ASSERT (this);
-        GF_ASSERT (req_dict);
-        GF_ASSERT (op_errstr);
-
-        rsp_dict = dict_new ();
-        if (!rsp_dict) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "Failed to create response dictionary");
-                goto out;
+    int32_t ret = -1;
+    int32_t peer_cnt = 0;
+    dict_t *rsp_dict = NULL;
+    glusterd_peerinfo_t *peerinfo = NULL;
+    struct syncargs args = {0};
+    uuid_t peer_uuid = {0};
+    xlator_t *this = NULL;
+    glusterd_conf_t *conf = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+    conf = this->private;
+    GF_ASSERT(conf);
+
+    GF_ASSERT(req_dict);
+    GF_ASSERT(op_errstr);
+
+    rsp_dict = dict_new();
+    if (!rsp_dict) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_CREATE_FAIL,
+               "Failed to create response dictionary");
+        goto out;
+    }
+
+    /* Perform brick op on local node */
+    ret = gd_mgmt_v3_brick_op_fn(op, req_dict, op_errstr, rsp_dict);
+
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_BRICK_OP_FAIL,
+               "Brick ops failed for "
+               "operation %s on local node",
+               gd_op_list[op]);
+
+        if (*op_errstr == NULL) {
+            ret = gf_asprintf(op_errstr,
+                              "Brick ops failed "
+                              "on localhost. Please "
+                              "check log file for details");
+            if (ret == -1)
+                *op_errstr = NULL;
+
+            ret = -1;
+        }
+        goto out;
+    }
+    if (op == GD_OP_DEFRAG_BRICK_VOLUME || op == GD_OP_PROFILE_VOLUME) {
+        ret = glusterd_syncop_aggr_rsp_dict(op, op_ctx, rsp_dict);
+        if (ret) {
+            gf_log(this->name, GF_LOG_ERROR, "%s",
+                   "Failed to aggregate response from "
+                   " node/brick");
+            goto out;
         }
+    }
 
-        /* Perform brick op on local node */
-        ret = gd_mgmt_v3_brick_op_fn (op, req_dict, op_errstr,
-                                     rsp_dict);
+    dict_unref(rsp_dict);
+    rsp_dict = NULL;
 
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "Brick ops failed for "
-                        "operation %s on local node",
-                        gd_op_list[op]);
-
-                if (*op_errstr == NULL) {
-                        ret = gf_asprintf (op_errstr,
-                                           "Brick ops failed "
-                                           "on localhost. Please "
-                                           "check log file for details");
-                        if (ret == -1)
-                                *op_errstr = NULL;
-
-                        ret = -1;
-                }
-                goto out;
-        }
+    /* Sending brick op req to other nodes in the cluster */
+    gd_syncargs_init(&args, op_ctx);
+    ret = synctask_barrier_init((&args));
+    if (ret)
+        goto out;
 
-        dict_unref (rsp_dict);
-        rsp_dict = NULL;
+    peer_cnt = 0;
 
-        if (!npeers) {
-                ret = 0;
-                goto out;
-        }
+    RCU_READ_LOCK;
+    cds_list_for_each_entry_rcu(peerinfo, &conf->peers, uuid_list)
+    {
+        /* Only send requests to peers who were available before the
+         * transaction started
+         */
+        if (peerinfo->generation > txn_generation)
+            continue;
+
+        if (!peerinfo->connected)
+            continue;
+        if (op != GD_OP_SYNC_VOLUME &&
+            peerinfo->state.state != GD_FRIEND_STATE_BEFRIENDED)
+            continue;
+
+        gd_mgmt_v3_brick_op_req(op, req_dict, peerinfo, &args, MY_UUID,
+                                peer_uuid);
+        peer_cnt++;
+    }
+    RCU_READ_UNLOCK;
+
+    if (0 == peer_cnt) {
+        ret = 0;
+        goto out;
+    }
 
-        /* Sending brick op req to other nodes in the cluster */
-        gd_syncargs_init (&args, NULL);
-        synctask_barrier_init((&args));
-        peer_cnt = 0;
-        list_for_each_local_xaction_peers (peerinfo, peers) {
-                gd_mgmt_v3_brick_op_req (op, req_dict, peerinfo, &args,
-                                         MY_UUID, peer_uuid);
-                peer_cnt++;
-        }
-        gd_synctask_barrier_wait((&args), peer_cnt);
+    gd_synctask_barrier_wait((&args), peer_cnt);
 
-        if (args.op_ret) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "Brick ops failed on peers");
+    if (args.op_ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_BRICK_OP_FAIL,
+               "Brick ops failed on peers");
 
-                if (args.errstr)
-                         *op_errstr = gf_strdup (args.errstr);
-        }
+        if (args.errstr)
+            *op_errstr = gf_strdup(args.errstr);
+    }
 
-        ret = args.op_ret;
+    ret = args.op_ret;
 
-        gf_log (this->name, GF_LOG_DEBUG, "Sent brick op req for %s "
-                "to %d peers. Returning %d", gd_op_list[op], peer_cnt, ret);
+    gf_msg_debug(this->name, 0,
+                 "Sent brick op req for %s "
+                 "to %d peers. Returning %d",
+                 gd_op_list[op], peer_cnt, ret);
 out:
-        return ret;
+    return ret;
 }
 
 int32_t
-gd_mgmt_v3_commit_cbk_fn (struct rpc_req *req, struct iovec *iov,
-                          int count, void *myframe)
+gd_mgmt_v3_commit_cbk_fn(struct rpc_req *req, struct iovec *iov, int count,
+                         void *myframe)
 {
-        int32_t                     ret           = -1;
-        struct syncargs            *args          = NULL;
-        glusterd_peerinfo_t        *peerinfo      = NULL;
-        gd1_mgmt_v3_commit_rsp       rsp          = {{0},};
-        call_frame_t               *frame         = NULL;
-        int32_t                     op_ret        = -1;
-        int32_t                     op_errno      = -1;
-        dict_t                     *rsp_dict      = NULL;
-        xlator_t                   *this          = NULL;
-
-        this = THIS;
-        GF_ASSERT (this);
-        GF_ASSERT (req);
-        GF_ASSERT (myframe);
-
-        frame  = myframe;
-        args   = frame->local;
-        peerinfo = frame->cookie;
-        frame->local = NULL;
-        frame->cookie = NULL;
-
-        if (-1 == req->rpc_status) {
-                op_errno = ENOTCONN;
-                goto out;
-        }
-
-        GF_VALIDATE_OR_GOTO_WITH_ERROR (this->name, iov, out, op_errno,
-                                        EINVAL);
-
-        ret = xdr_to_generic (*iov, &rsp,
-                              (xdrproc_t)xdr_gd1_mgmt_v3_commit_rsp);
-        if (ret < 0)
-                goto out;
-
-        if (rsp.dict.dict_len) {
-                /* Unserialize the dictionary */
-                rsp_dict  = dict_new ();
-
-                ret = dict_unserialize (rsp.dict.dict_val,
-                                        rsp.dict.dict_len,
-                                        &rsp_dict);
-                if (ret < 0) {
-                        free (rsp.dict.dict_val);
-                        goto out;
-                } else {
-                        rsp_dict->extra_stdfree = rsp.dict.dict_val;
-                }
-        }
-
-        gf_uuid_copy (args->uuid, rsp.uuid);
-        pthread_mutex_lock (&args->lock_dict);
-        {
-                ret = glusterd_syncop_aggr_rsp_dict (rsp.op, args->dict,
-                                                     rsp_dict);
-        }
-        pthread_mutex_unlock (&args->lock_dict);
-
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "%s",
-                        "Failed to aggregate response from "
-                        " node/brick");
-                if (!rsp.op_ret)
-                        op_ret = ret;
-                else {
-                        op_ret = rsp.op_ret;
-                        op_errno = rsp.op_errno;
-                }
+    int32_t ret = -1;
+    struct syncargs *args = NULL;
+    gd1_mgmt_v3_commit_rsp rsp = {
+        {0},
+    };
+    call_frame_t *frame = NULL;
+    int32_t op_ret = -1;
+    int32_t op_errno = -1;
+    dict_t *rsp_dict = NULL;
+    xlator_t *this = NULL;
+    uuid_t *peerid = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+    GF_ASSERT(req);
+    GF_ASSERT(myframe);
+
+    frame = myframe;
+    args = frame->local;
+    peerid = frame->cookie;
+    frame->local = NULL;
+    frame->cookie = NULL;
+
+    if (-1 == req->rpc_status) {
+        op_errno = ENOTCONN;
+        goto out;
+    }
+
+    GF_VALIDATE_OR_GOTO_WITH_ERROR(this->name, iov, out, op_errno, EINVAL);
+
+    ret = xdr_to_generic(*iov, &rsp, (xdrproc_t)xdr_gd1_mgmt_v3_commit_rsp);
+    if (ret < 0)
+        goto out;
+
+    if (rsp.dict.dict_len) {
+        /* Unserialize the dictionary */
+        rsp_dict = dict_new();
+
+        ret = dict_unserialize(rsp.dict.dict_val, rsp.dict.dict_len, &rsp_dict);
+        if (ret < 0) {
+            free(rsp.dict.dict_val);
+            goto out;
         } else {
-                op_ret = rsp.op_ret;
-                op_errno = rsp.op_errno;
-        }
+            rsp_dict->extra_stdfree = rsp.dict.dict_val;
+        }
+    }
+
+    gf_uuid_copy(args->uuid, rsp.uuid);
+    pthread_mutex_lock(&args->lock_dict);
+    {
+        ret = glusterd_syncop_aggr_rsp_dict(rsp.op, args->dict, rsp_dict);
+    }
+    pthread_mutex_unlock(&args->lock_dict);
+
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_RESP_AGGR_FAIL, "%s",
+               "Failed to aggregate response from "
+               " node/brick");
+        if (!rsp.op_ret)
+            op_ret = ret;
+        else {
+            op_ret = rsp.op_ret;
+            op_errno = rsp.op_errno;
+        }
+    } else {
+        op_ret = rsp.op_ret;
+        op_errno = rsp.op_errno;
+    }
 
 out:
-        if (rsp_dict)
-                dict_unref (rsp_dict);
-
-        gd_mgmt_v3_collate_errors (args, op_ret, op_errno, rsp.op_errstr,
-                                  GLUSTERD_MGMT_V3_COMMIT,
-                                  peerinfo, rsp.uuid);
-
-        STACK_DESTROY (frame->root);
-        synctask_barrier_wake(args);
-        return 0;
+    if (rsp_dict)
+        dict_unref(rsp_dict);
+
+    gd_mgmt_v3_collate_errors(args, op_ret, op_errno, rsp.op_errstr,
+                              GLUSTERD_MGMT_V3_COMMIT, *peerid, rsp.uuid);
+    GF_FREE(peerid);
+
+    if (rsp.op_errstr)
+        free(rsp.op_errstr);
+
+    /* req->rpc_status set to -1 means, STACK_DESTROY will be called from
+     * the caller function.
+     */
+    if (req->rpc_status != -1)
+        STACK_DESTROY(frame->root);
+    synctask_barrier_wake(args);
+    return 0;
 }
 
 int32_t
-gd_mgmt_v3_commit_cbk (struct rpc_req *req, struct iovec *iov,
-                       int count, void *myframe)
+gd_mgmt_v3_commit_cbk(struct rpc_req *req, struct iovec *iov, int count,
+                      void *myframe)
 {
-        return glusterd_big_locked_cbk (req, iov, count, myframe,
-                                        gd_mgmt_v3_commit_cbk_fn);
+    return glusterd_big_locked_cbk(req, iov, count, myframe,
+                                   gd_mgmt_v3_commit_cbk_fn);
 }
 
 int
-gd_mgmt_v3_commit_req (glusterd_op_t op, dict_t *op_ctx,
-                       glusterd_peerinfo_t *peerinfo,
-                       struct syncargs *args, uuid_t my_uuid,
-                       uuid_t recv_uuid)
+gd_mgmt_v3_commit_req(glusterd_op_t op, dict_t *op_ctx,
+                      glusterd_peerinfo_t *peerinfo, struct syncargs *args,
+                      uuid_t my_uuid, uuid_t recv_uuid)
 {
-        int32_t                  ret  = -1;
-        gd1_mgmt_v3_commit_req   req  = {{0},};
-        glusterd_conf_t         *conf = THIS->private;
-        xlator_t                *this = NULL;
-
-        this = THIS;
-        GF_ASSERT (this);
-        GF_ASSERT (op_ctx);
-        GF_ASSERT (peerinfo);
-        GF_ASSERT (args);
-
-        ret = dict_allocate_and_serialize (op_ctx,
-                                           &req.dict.dict_val,
-                                           &req.dict.dict_len);
-        if (ret)
-                goto out;
-
-        gf_uuid_copy (req.uuid, my_uuid);
-        req.op = op;
-        synclock_unlock (&conf->big_lock);
-
-        ret = gd_syncop_submit_request (peerinfo->rpc, &req, args, peerinfo,
-                                        &gd_mgmt_v3_prog,
-                                        GLUSTERD_MGMT_V3_COMMIT,
-                                        gd_mgmt_v3_commit_cbk,
-                                        (xdrproc_t) xdr_gd1_mgmt_v3_commit_req);
-        synclock_lock (&conf->big_lock);
+    int32_t ret = -1;
+    gd1_mgmt_v3_commit_req req = {
+        {0},
+    };
+    xlator_t *this = NULL;
+    uuid_t *peerid = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+    GF_ASSERT(op_ctx);
+    GF_ASSERT(peerinfo);
+    GF_ASSERT(args);
+
+    ret = dict_allocate_and_serialize(op_ctx, &req.dict.dict_val,
+                                      &req.dict.dict_len);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno,
+                GD_MSG_DICT_ALLOC_AND_SERL_LENGTH_GET_FAIL, NULL);
+        goto out;
+    }
+
+    gf_uuid_copy(req.uuid, my_uuid);
+    req.op = op;
+
+    GD_ALLOC_COPY_UUID(peerid, peerinfo->uuid, ret);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno,
+                GD_MSG_ALLOC_AND_COPY_UUID_FAIL, NULL);
+        goto out;
+    }
+
+    ret = gd_syncop_submit_request(peerinfo->rpc, &req, args, peerid,
+                                   &gd_mgmt_v3_prog, GLUSTERD_MGMT_V3_COMMIT,
+                                   gd_mgmt_v3_commit_cbk,
+                                   (xdrproc_t)xdr_gd1_mgmt_v3_commit_req);
 out:
-        GF_FREE (req.dict.dict_val);
-        gf_log (this->name, GF_LOG_TRACE, "Returning %d", ret);
-        return ret;
+    GF_FREE(req.dict.dict_val);
+    gf_msg_trace(this->name, 0, "Returning %d", ret);
+    return ret;
 }
 
 int
-glusterd_mgmt_v3_commit (glusterd_op_t op, dict_t *op_ctx,
-                         dict_t *req_dict, char **op_errstr,
-                         int npeers, struct cds_list_head *peers)
+glusterd_mgmt_v3_commit(glusterd_op_t op, dict_t *op_ctx, dict_t *req_dict,
+                        char **op_errstr, uint32_t *op_errno,
+                        uint32_t txn_generation)
 {
-        int32_t              ret        = -1;
-        int32_t              peer_cnt   = 0;
-        dict_t              *rsp_dict   = NULL;
-        glusterd_peerinfo_t *peerinfo   = NULL;
-        struct syncargs      args       = {0};
-        uuid_t               peer_uuid  = {0};
-        xlator_t            *this       = NULL;
-
-        this = THIS;
-        GF_ASSERT (this);
-        GF_ASSERT (op_ctx);
-        GF_ASSERT (req_dict);
-        GF_ASSERT (op_errstr);
-
-        rsp_dict = dict_new ();
-        if (!rsp_dict) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "Failed to create response dictionary");
-                goto out;
-        }
-
-        /* Commit on local node */
-        ret = gd_mgmt_v3_commit_fn (op, req_dict, op_errstr,
-                                   rsp_dict);
-
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "Commit failed for "
-                        "operation %s on local node",
-                        gd_op_list[op]);
-
-                if (*op_errstr == NULL) {
-                        ret = gf_asprintf (op_errstr,
-                                           "Commit failed "
-                                           "on localhost. Please "
-                                           "check log file for details.");
-                        if (ret == -1)
-                                *op_errstr = NULL;
-
-                        ret = -1;
-                }
-                goto out;
-        }
-
-        ret = glusterd_syncop_aggr_rsp_dict (op, op_ctx,
-                                             rsp_dict);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "%s",
-                        "Failed to aggregate response from "
-                        " node/brick");
-                goto out;
-        }
-
-        dict_unref (rsp_dict);
-        rsp_dict = NULL;
-
-        if (!npeers) {
-                ret = 0;
-                goto out;
-        }
+    int32_t ret = -1;
+    int32_t peer_cnt = 0;
+    dict_t *rsp_dict = NULL;
+    glusterd_peerinfo_t *peerinfo = NULL;
+    struct syncargs args = {0};
+    uuid_t peer_uuid = {0};
+    xlator_t *this = NULL;
+    glusterd_conf_t *conf = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+    conf = this->private;
+    GF_ASSERT(conf);
+
+    GF_ASSERT(op_ctx);
+    GF_ASSERT(req_dict);
+    GF_ASSERT(op_errstr);
+    GF_VALIDATE_OR_GOTO(this->name, op_errno, out);
+
+    switch (op) {
+        case GD_OP_REBALANCE:
+        case GD_OP_DEFRAG_BRICK_VOLUME:
+
+            ret = glusterd_set_rebalance_id_in_rsp_dict(req_dict, op_ctx);
+            if (ret) {
+                gf_log(this->name, GF_LOG_WARNING,
+                       "Failed to set rebalance id in dict.");
+            }
+            break;
+        case GD_OP_REMOVE_BRICK:
+            ret = glusterd_set_rebalance_id_for_remove_brick(req_dict, op_ctx);
+            if (ret) {
+                gf_log(this->name, GF_LOG_WARNING,
+                       "Failed to set rebalance id for remove-brick in dict.");
+            }
+            break;
+        default:
+            break;
+    }
+    rsp_dict = dict_new();
+    if (!rsp_dict) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_CREATE_FAIL,
+               "Failed to create response dictionary");
+        goto out;
+    }
+
+    /* Commit on local node */
+    ret = gd_mgmt_v3_commit_fn(op, req_dict, op_errstr, op_errno, rsp_dict);
+
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_COMMIT_OP_FAIL,
+               "Commit failed for "
+               "operation %s on local node",
+               gd_op_list[op]);
+
+        if (*op_errstr == NULL) {
+            ret = gf_asprintf(op_errstr,
+                              "Commit failed "
+                              "on localhost. Please "
+                              "check log file for details.");
+            if (ret == -1)
+                *op_errstr = NULL;
+
+            ret = -1;
+        }
+        goto out;
+    }
+
+    ret = glusterd_syncop_aggr_rsp_dict(op, op_ctx, rsp_dict);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_RESP_AGGR_FAIL, "%s",
+               "Failed to aggregate response from "
+               " node/brick");
+        goto out;
+    }
+
+    dict_unref(rsp_dict);
+    rsp_dict = NULL;
+
+    /* Sending commit req to other nodes in the cluster */
+    gd_syncargs_init(&args, op_ctx);
+    ret = synctask_barrier_init((&args));
+    if (ret)
+        goto out;
+    peer_cnt = 0;
+
+    RCU_READ_LOCK;
+    cds_list_for_each_entry_rcu(peerinfo, &conf->peers, uuid_list)
+    {
+        /* Only send requests to peers who were available before the
+         * transaction started
+         */
+        if (peerinfo->generation > txn_generation)
+            continue;
+        if (!peerinfo->connected)
+            continue;
+
+        if (op != GD_OP_SYNC_VOLUME &&
+            peerinfo->state.state != GD_FRIEND_STATE_BEFRIENDED)
+            continue;
+
+        gd_mgmt_v3_commit_req(op, req_dict, peerinfo, &args, MY_UUID,
+                              peer_uuid);
+        peer_cnt++;
+    }
+    RCU_READ_UNLOCK;
+
+    if (0 == peer_cnt) {
+        ret = 0;
+        goto out;
+    }
 
-        /* Sending commit req to other nodes in the cluster */
-        gd_syncargs_init (&args, op_ctx);
-        synctask_barrier_init((&args));
-        peer_cnt = 0;
-        list_for_each_local_xaction_peers (peerinfo, peers) {
-                gd_mgmt_v3_commit_req (op, req_dict, peerinfo, &args,
-                                       MY_UUID, peer_uuid);
-                peer_cnt++;
-        }
-        gd_synctask_barrier_wait((&args), peer_cnt);
+    gd_synctask_barrier_wait((&args), peer_cnt);
 
-        if (args.op_ret) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "Commit failed on peers");
+    if (args.op_ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_COMMIT_OP_FAIL,
+               "Commit failed on peers");
 
-                if (args.errstr)
-                         *op_errstr = gf_strdup (args.errstr);
-        }
+        if (args.errstr)
+            *op_errstr = gf_strdup(args.errstr);
+    }
 
-        ret = args.op_ret;
+    ret = args.op_ret;
+    *op_errno = args.op_errno;
 
-        gf_log (this->name, GF_LOG_DEBUG, "Sent commit req for %s to %d "
-                "peers. Returning %d", gd_op_list[op], peer_cnt, ret);
+    gf_msg_debug(this->name, 0,
+                 "Sent commit req for %s to %d "
+                 "peers. Returning %d",
+                 gd_op_list[op], peer_cnt, ret);
 out:
-        return ret;
+    glusterd_op_modify_op_ctx(op, op_ctx);
+    return ret;
 }
 
 int32_t
-gd_mgmt_v3_post_validate_cbk_fn (struct rpc_req *req, struct iovec *iov,
-                                 int count, void *myframe)
+gd_mgmt_v3_post_commit_cbk_fn(struct rpc_req *req, struct iovec *iov, int count,
+                              void *myframe)
 {
-        int32_t                     ret           = -1;
-        struct syncargs            *args          = NULL;
-        glusterd_peerinfo_t        *peerinfo      = NULL;
-        gd1_mgmt_v3_post_val_rsp    rsp           = {{0},};
-        call_frame_t               *frame         = NULL;
-        int32_t                     op_ret        = -1;
-        int32_t                     op_errno      = -1;
-        xlator_t                   *this          = NULL;
-
-        this = THIS;
-        GF_ASSERT (this);
-        GF_ASSERT (req);
-        GF_ASSERT (myframe);
-
-        frame  = myframe;
-        args   = frame->local;
-        peerinfo = frame->cookie;
-        frame->local = NULL;
-        frame->cookie = NULL;
-
-        if (-1 == req->rpc_status) {
-                op_errno = ENOTCONN;
-                goto out;
-        }
-
-        GF_VALIDATE_OR_GOTO_WITH_ERROR (this->name, iov, out, op_errno,
-                                        EINVAL);
-
-        ret = xdr_to_generic (*iov, &rsp,
-                              (xdrproc_t)xdr_gd1_mgmt_v3_post_val_rsp);
-        if (ret < 0)
-                goto out;
-
-        gf_uuid_copy (args->uuid, rsp.uuid);
-
+    int32_t ret = -1;
+    struct syncargs *args = NULL;
+    gd1_mgmt_v3_post_commit_rsp rsp = {
+        {0},
+    };
+    call_frame_t *frame = NULL;
+    int32_t op_ret = -1;
+    int32_t op_errno = -1;
+    dict_t *rsp_dict = NULL;
+    xlator_t *this = NULL;
+    uuid_t *peerid = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+    GF_ASSERT(req);
+    GF_ASSERT(myframe);
+
+    frame = myframe;
+    args = frame->local;
+    peerid = frame->cookie;
+    frame->local = NULL;
+    frame->cookie = NULL;
+
+    if (-1 == req->rpc_status) {
+        op_errno = ENOTCONN;
+        goto out;
+    }
+
+    GF_VALIDATE_OR_GOTO_WITH_ERROR(this->name, iov, out, op_errno, EINVAL);
+
+    ret = xdr_to_generic(*iov, &rsp,
+                         (xdrproc_t)xdr_gd1_mgmt_v3_post_commit_rsp);
+    if (ret < 0)
+        goto out;
+
+    if (rsp.dict.dict_len) {
+        /* Unserialize the dictionary */
+        rsp_dict = dict_new();
+
+        ret = dict_unserialize(rsp.dict.dict_val, rsp.dict.dict_len, &rsp_dict);
+        if (ret < 0) {
+            free(rsp.dict.dict_val);
+            goto out;
+        } else {
+            rsp_dict->extra_stdfree = rsp.dict.dict_val;
+        }
+    }
+
+    gf_uuid_copy(args->uuid, rsp.uuid);
+    pthread_mutex_lock(&args->lock_dict);
+    {
+        ret = glusterd_syncop_aggr_rsp_dict(rsp.op, args->dict, rsp_dict);
+    }
+    pthread_mutex_unlock(&args->lock_dict);
+
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_RESP_AGGR_FAIL, "%s",
+               "Failed to aggregate response from "
+               " node/brick");
+        if (!rsp.op_ret)
+            op_ret = ret;
+        else {
+            op_ret = rsp.op_ret;
+            op_errno = rsp.op_errno;
+        }
+    } else {
         op_ret = rsp.op_ret;
         op_errno = rsp.op_errno;
+    }
 
 out:
-        gd_mgmt_v3_collate_errors (args, op_ret, op_errno, rsp.op_errstr,
-                                  GLUSTERD_MGMT_V3_POST_VALIDATE,
-                                  peerinfo, rsp.uuid);
-        if (rsp.op_errstr)
-                free (rsp.op_errstr);
-
-        if (rsp.dict.dict_val)
-                free (rsp.dict.dict_val);
-        STACK_DESTROY (frame->root);
-        synctask_barrier_wake(args);
-        return 0;
+    if (rsp_dict)
+        dict_unref(rsp_dict);
+
+    gd_mgmt_v3_collate_errors(args, op_ret, op_errno, rsp.op_errstr,
+                              GLUSTERD_MGMT_V3_POST_COMMIT, *peerid, rsp.uuid);
+    GF_FREE(peerid);
+
+    if (rsp.op_errstr)
+        free(rsp.op_errstr);
+
+    /* req->rpc_status set to -1 means, STACK_DESTROY will be called from
+     * the caller function.
+     */
+    if (req->rpc_status != -1)
+        STACK_DESTROY(frame->root);
+    synctask_barrier_wake(args);
+    return 0;
 }
 
 int32_t
-gd_mgmt_v3_post_validate_cbk (struct rpc_req *req, struct iovec *iov,
-                              int count, void *myframe)
+gd_mgmt_v3_post_commit_cbk(struct rpc_req *req, struct iovec *iov, int count,
+                           void *myframe)
 {
-        return glusterd_big_locked_cbk (req, iov, count, myframe,
-                                        gd_mgmt_v3_post_validate_cbk_fn);
+    return glusterd_big_locked_cbk(req, iov, count, myframe,
+                                   gd_mgmt_v3_post_commit_cbk_fn);
 }
 
 int
-gd_mgmt_v3_post_validate_req (glusterd_op_t op, int32_t op_ret, dict_t *op_ctx,
-                              glusterd_peerinfo_t *peerinfo,
-                              struct syncargs *args, uuid_t my_uuid,
-                              uuid_t recv_uuid)
+gd_mgmt_v3_post_commit_req(glusterd_op_t op, dict_t *op_ctx,
+                           glusterd_peerinfo_t *peerinfo, struct syncargs *args,
+                           uuid_t my_uuid, uuid_t recv_uuid)
 {
-        int32_t                   ret  = -1;
-        gd1_mgmt_v3_post_val_req  req  = {{0},};
-        glusterd_conf_t          *conf = THIS->private;
-        xlator_t                 *this = NULL;
-
-        this = THIS;
-        GF_ASSERT (this);
-        GF_ASSERT (op_ctx);
-        GF_ASSERT (peerinfo);
-        GF_ASSERT (args);
-
-        ret = dict_allocate_and_serialize (op_ctx,
-                                           &req.dict.dict_val,
-                                           &req.dict.dict_len);
-        if (ret)
-                goto out;
-
-        gf_uuid_copy (req.uuid, my_uuid);
-        req.op = op;
-        req.op_ret = op_ret;
-        synclock_unlock (&conf->big_lock);
-
-        ret = gd_syncop_submit_request (peerinfo->rpc, &req, args, peerinfo,
-                                        &gd_mgmt_v3_prog,
-                                        GLUSTERD_MGMT_V3_POST_VALIDATE,
-                                        gd_mgmt_v3_post_validate_cbk,
-                                        (xdrproc_t) xdr_gd1_mgmt_v3_post_val_req);
-        synclock_lock (&conf->big_lock);
+    int32_t ret = -1;
+    gd1_mgmt_v3_post_commit_req req = {
+        {0},
+    };
+    xlator_t *this = NULL;
+    uuid_t *peerid = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+    GF_ASSERT(op_ctx);
+    GF_ASSERT(peerinfo);
+    GF_ASSERT(args);
+
+    ret = dict_allocate_and_serialize(op_ctx, &req.dict.dict_val,
+                                      &req.dict.dict_len);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno,
+                GD_MSG_DICT_ALLOC_AND_SERL_LENGTH_GET_FAIL, NULL);
+        goto out;
+    }
+
+    gf_uuid_copy(req.uuid, my_uuid);
+    req.op = op;
+
+    GD_ALLOC_COPY_UUID(peerid, peerinfo->uuid, ret);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno,
+                GD_MSG_ALLOC_AND_COPY_UUID_FAIL, NULL);
+        goto out;
+    }
+
+    ret = gd_syncop_submit_request(
+        peerinfo->rpc, &req, args, peerid, &gd_mgmt_v3_prog,
+        GLUSTERD_MGMT_V3_POST_COMMIT, gd_mgmt_v3_post_commit_cbk,
+        (xdrproc_t)xdr_gd1_mgmt_v3_post_commit_req);
 out:
-        GF_FREE (req.dict.dict_val);
-        gf_log (this->name, GF_LOG_TRACE, "Returning %d", ret);
-        return ret;
+    GF_FREE(req.dict.dict_val);
+    gf_msg_trace(this->name, 0, "Returning %d", ret);
+    return ret;
 }
 
 int
-glusterd_mgmt_v3_post_validate (glusterd_op_t op, int32_t op_ret, dict_t *dict,
-                                dict_t *req_dict, char **op_errstr, int npeers,
-                                struct cds_list_head *peers)
+glusterd_mgmt_v3_post_commit(glusterd_op_t op, dict_t *op_ctx, dict_t *req_dict,
+                             char **op_errstr, uint32_t *op_errno,
+                             uint32_t txn_generation)
 {
-        int32_t              ret        = -1;
-        int32_t              peer_cnt   = 0;
-        dict_t              *rsp_dict   = NULL;
-        glusterd_peerinfo_t *peerinfo   = NULL;
-        struct syncargs      args       = {0};
-        uuid_t               peer_uuid  = {0};
-        xlator_t            *this       = NULL;
-
-        this = THIS;
-        GF_ASSERT (this);
-        GF_ASSERT (dict);
-        GF_VALIDATE_OR_GOTO (this->name, req_dict, out);
-        GF_ASSERT (op_errstr);
-
-        rsp_dict = dict_new ();
-        if (!rsp_dict) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "Failed to create response dictionary");
-                goto out;
-        }
-
-        /* Copy the contents of dict like missed snaps info to req_dict */
-        dict_copy (dict, req_dict);
-
-        /* Post Validation on local node */
-        ret = gd_mgmt_v3_post_validate_fn (op, op_ret, req_dict, op_errstr,
-                                           rsp_dict);
-
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "Post Validation failed for "
-                        "operation %s on local node",
-                        gd_op_list[op]);
-
-                if (*op_errstr == NULL) {
-                        ret = gf_asprintf (op_errstr,
-                                           "Post-validation failed "
-                                           "on localhost. Please check "
-                                           "log file for details");
-                        if (ret == -1)
-                                *op_errstr = NULL;
-
-                        ret = -1;
-                }
-                goto out;
-        }
-
-        dict_unref (rsp_dict);
-        rsp_dict = NULL;
-
-        if (!npeers) {
-                ret = 0;
-                goto out;
-        }
+    int32_t ret = -1;
+    int32_t peer_cnt = 0;
+    dict_t *rsp_dict = NULL;
+    glusterd_peerinfo_t *peerinfo = NULL;
+    struct syncargs args = {0};
+    uuid_t peer_uuid = {0};
+    xlator_t *this = NULL;
+    glusterd_conf_t *conf = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+    conf = this->private;
+    GF_ASSERT(conf);
+
+    GF_ASSERT(op_ctx);
+    GF_ASSERT(req_dict);
+    GF_ASSERT(op_errstr);
+    GF_VALIDATE_OR_GOTO(this->name, op_errno, out);
+
+    rsp_dict = dict_new();
+    if (!rsp_dict) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_CREATE_FAIL,
+               "Failed to create response dictionary");
+        goto out;
+    }
+
+    /* Post commit on local node */
+    ret = gd_mgmt_v3_post_commit_fn(op, req_dict, op_errstr, op_errno,
+                                    rsp_dict);
+
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_POST_COMMIT_OP_FAIL,
+               "Post commit failed for "
+               "operation %s on local node",
+               gd_op_list[op]);
+
+        if (*op_errstr == NULL) {
+            ret = gf_asprintf(op_errstr,
+                              "Post commit failed "
+                              "on localhost. Please "
+                              "check log file for details.");
+            if (ret == -1)
+                *op_errstr = NULL;
+
+            ret = -1;
+        }
+        goto out;
+    }
+
+    ret = glusterd_syncop_aggr_rsp_dict(op, op_ctx, rsp_dict);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_RESP_AGGR_FAIL, "%s",
+               "Failed to aggregate response from "
+               " node/brick");
+        goto out;
+    }
+
+    dict_unref(rsp_dict);
+    rsp_dict = NULL;
+
+    /* Sending post commit req to other nodes in the cluster */
+    gd_syncargs_init(&args, op_ctx);
+    ret = synctask_barrier_init((&args));
+    if (ret)
+        goto out;
+    peer_cnt = 0;
+
+    RCU_READ_LOCK;
+    cds_list_for_each_entry_rcu(peerinfo, &conf->peers, uuid_list)
+    {
+        /* Only send requests to peers who were available before the
+         * transaction started
+         */
+        if (peerinfo->generation > txn_generation)
+            continue;
+        if (!peerinfo->connected)
+            continue;
+
+        if (op != GD_OP_SYNC_VOLUME &&
+            peerinfo->state.state != GD_FRIEND_STATE_BEFRIENDED)
+            continue;
+
+        gd_mgmt_v3_post_commit_req(op, req_dict, peerinfo, &args, MY_UUID,
+                                   peer_uuid);
+        peer_cnt++;
+    }
+    RCU_READ_UNLOCK;
+
+    if (0 == peer_cnt) {
+        ret = 0;
+        goto out;
+    }
 
-        /* Sending Post Validation req to other nodes in the cluster */
-        gd_syncargs_init (&args, req_dict);
-        synctask_barrier_init((&args));
-        peer_cnt = 0;
-        list_for_each_local_xaction_peers (peerinfo, peers) {
-                gd_mgmt_v3_post_validate_req (op, op_ret, req_dict, peerinfo,
-                                              &args, MY_UUID, peer_uuid);
-                peer_cnt++;
-        }
-        gd_synctask_barrier_wait((&args), peer_cnt);
+    gd_synctask_barrier_wait((&args), peer_cnt);
 
-        if (args.op_ret) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "Post Validation failed on peers");
+    if (args.op_ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_POST_COMMIT_OP_FAIL,
+               "Post commit failed on peers");
 
-                if (args.errstr)
-                         *op_errstr = gf_strdup (args.errstr);
-        }
+        if (args.errstr)
+            *op_errstr = gf_strdup(args.errstr);
+    }
 
-        ret = args.op_ret;
+    ret = args.op_ret;
+    *op_errno = args.op_errno;
 
-        gf_log (this->name, GF_LOG_DEBUG, "Sent post valaidation req for %s "
-                "to %d peers. Returning %d", gd_op_list[op], peer_cnt, ret);
+    gf_msg_debug(this->name, 0,
+                 "Sent post commit req for %s to %d "
+                 "peers. Returning %d",
+                 gd_op_list[op], peer_cnt, ret);
 out:
-        return ret;
+    glusterd_op_modify_op_ctx(op, op_ctx);
+    return ret;
 }
 
 int32_t
-gd_mgmt_v3_unlock_cbk_fn (struct rpc_req *req, struct iovec *iov,
-                          int count, void *myframe)
+gd_mgmt_v3_post_validate_cbk_fn(struct rpc_req *req, struct iovec *iov,
+                                int count, void *myframe)
 {
-        int32_t                     ret           = -1;
-        struct syncargs            *args          = NULL;
-        glusterd_peerinfo_t        *peerinfo      = NULL;
-        gd1_mgmt_v3_unlock_rsp      rsp           = {{0},};
-        call_frame_t               *frame         = NULL;
-        int32_t                     op_ret        = -1;
-        int32_t                     op_errno      = -1;
-        xlator_t                   *this          = NULL;
-
-        this = THIS;
-        GF_ASSERT (this);
-        GF_ASSERT (req);
-        GF_ASSERT (myframe);
-
-        frame  = myframe;
-        args   = frame->local;
-        peerinfo = frame->cookie;
-        frame->local = NULL;
-        frame->cookie = NULL;
-
-        if (-1 == req->rpc_status) {
-                op_errno = ENOTCONN;
-                goto out;
-        }
-
-        GF_VALIDATE_OR_GOTO_WITH_ERROR (this->name, iov, out, op_errno,
-                                        EINVAL);
-
-        ret = xdr_to_generic (*iov, &rsp,
-                              (xdrproc_t)xdr_gd1_mgmt_v3_unlock_rsp);
-        if (ret < 0)
-                goto out;
-
-        gf_uuid_copy (args->uuid, rsp.uuid);
-
-        op_ret = rsp.op_ret;
-        op_errno = rsp.op_errno;
+    int32_t ret = -1;
+    struct syncargs *args = NULL;
+    gd1_mgmt_v3_post_val_rsp rsp = {
+        {0},
+    };
+    call_frame_t *frame = NULL;
+    int32_t op_ret = -1;
+    int32_t op_errno = -1;
+    xlator_t *this = NULL;
+    uuid_t *peerid = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+    GF_ASSERT(req);
+    GF_ASSERT(myframe);
+
+    frame = myframe;
+    args = frame->local;
+    peerid = frame->cookie;
+    frame->local = NULL;
+    frame->cookie = NULL;
+
+    if (-1 == req->rpc_status) {
+        op_errno = ENOTCONN;
+        goto out;
+    }
+
+    GF_VALIDATE_OR_GOTO_WITH_ERROR(this->name, iov, out, op_errno, EINVAL);
+
+    ret = xdr_to_generic(*iov, &rsp, (xdrproc_t)xdr_gd1_mgmt_v3_post_val_rsp);
+    if (ret < 0)
+        goto out;
+
+    gf_uuid_copy(args->uuid, rsp.uuid);
+
+    op_ret = rsp.op_ret;
+    op_errno = rsp.op_errno;
 
 out:
-        gd_mgmt_v3_collate_errors (args, op_ret, op_errno, NULL,
-                                  GLUSTERD_MGMT_V3_UNLOCK,
-                                  peerinfo, rsp.uuid);
-        if (rsp.dict.dict_val)
-                free (rsp.dict.dict_val);
-        STACK_DESTROY (frame->root);
-        synctask_barrier_wake(args);
-        return 0;
+    gd_mgmt_v3_collate_errors(args, op_ret, op_errno, rsp.op_errstr,
+                              GLUSTERD_MGMT_V3_POST_VALIDATE, *peerid,
+                              rsp.uuid);
+    if (rsp.op_errstr)
+        free(rsp.op_errstr);
+
+    if (rsp.dict.dict_val)
+        free(rsp.dict.dict_val);
+    GF_FREE(peerid);
+    /* req->rpc_status set to -1 means, STACK_DESTROY will be called from
+     * the caller function.
+     */
+    if (req->rpc_status != -1)
+        STACK_DESTROY(frame->root);
+    synctask_barrier_wake(args);
+    return 0;
 }
 
 int32_t
-gd_mgmt_v3_unlock_cbk (struct rpc_req *req, struct iovec *iov,
-                       int count, void *myframe)
+gd_mgmt_v3_post_validate_cbk(struct rpc_req *req, struct iovec *iov, int count,
+                             void *myframe)
 {
-        return glusterd_big_locked_cbk (req, iov, count, myframe,
-                                        gd_mgmt_v3_unlock_cbk_fn);
+    return glusterd_big_locked_cbk(req, iov, count, myframe,
+                                   gd_mgmt_v3_post_validate_cbk_fn);
 }
 
 int
-gd_mgmt_v3_unlock (glusterd_op_t op, dict_t *op_ctx,
-                   glusterd_peerinfo_t *peerinfo,
-                   struct syncargs *args, uuid_t my_uuid,
-                   uuid_t recv_uuid)
+gd_mgmt_v3_post_validate_req(glusterd_op_t op, int32_t op_ret, dict_t *op_ctx,
+                             glusterd_peerinfo_t *peerinfo,
+                             struct syncargs *args, uuid_t my_uuid,
+                             uuid_t recv_uuid)
 {
-        int32_t                  ret  = -1;
-        gd1_mgmt_v3_unlock_req   req  = {{0},};
-        glusterd_conf_t         *conf = THIS->private;
-        xlator_t                *this = NULL;
-
-        this = THIS;
-        GF_ASSERT (this);
-        GF_ASSERT (op_ctx);
-        GF_ASSERT (peerinfo);
-        GF_ASSERT (args);
-
-        ret = dict_allocate_and_serialize (op_ctx,
-                                           &req.dict.dict_val,
-                                           &req.dict.dict_len);
-        if (ret)
-                goto out;
-
-        gf_uuid_copy (req.uuid, my_uuid);
-        req.op = op;
-        synclock_unlock (&conf->big_lock);
-
-        ret = gd_syncop_submit_request (peerinfo->rpc, &req, args, peerinfo,
-                                        &gd_mgmt_v3_prog,
-                                        GLUSTERD_MGMT_V3_UNLOCK,
-                                        gd_mgmt_v3_unlock_cbk,
-                                        (xdrproc_t) xdr_gd1_mgmt_v3_unlock_req);
-        synclock_lock (&conf->big_lock);
+    int32_t ret = -1;
+    gd1_mgmt_v3_post_val_req req = {
+        {0},
+    };
+    xlator_t *this = NULL;
+    uuid_t *peerid = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+    GF_ASSERT(op_ctx);
+    GF_ASSERT(peerinfo);
+    GF_ASSERT(args);
+
+    ret = dict_allocate_and_serialize(op_ctx, &req.dict.dict_val,
+                                      &req.dict.dict_len);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno,
+                GD_MSG_DICT_ALLOC_AND_SERL_LENGTH_GET_FAIL, NULL);
+        goto out;
+    }
+
+    gf_uuid_copy(req.uuid, my_uuid);
+    req.op = op;
+    req.op_ret = op_ret;
+
+    GD_ALLOC_COPY_UUID(peerid, peerinfo->uuid, ret);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno,
+                GD_MSG_ALLOC_AND_COPY_UUID_FAIL, NULL);
+        goto out;
+    }
+
+    ret = gd_syncop_submit_request(
+        peerinfo->rpc, &req, args, peerid, &gd_mgmt_v3_prog,
+        GLUSTERD_MGMT_V3_POST_VALIDATE, gd_mgmt_v3_post_validate_cbk,
+        (xdrproc_t)xdr_gd1_mgmt_v3_post_val_req);
 out:
-        GF_FREE (req.dict.dict_val);
-        gf_log (this->name, GF_LOG_TRACE, "Returning %d", ret);
-        return ret;
+    GF_FREE(req.dict.dict_val);
+    gf_msg_trace(this->name, 0, "Returning %d", ret);
+    return ret;
 }
 
 int
-glusterd_mgmt_v3_release_peer_locks (glusterd_op_t op,
-                                     dict_t *dict, int32_t op_ret,
-                                     char **op_errstr, int npeers,
-                                     gf_boolean_t  is_acquired,
-                                     struct cds_list_head *peers)
+glusterd_mgmt_v3_post_validate(glusterd_op_t op, int32_t op_ret, dict_t *dict,
+                               dict_t *req_dict, char **op_errstr,
+                               uint32_t txn_generation)
 {
-        int32_t              ret        = -1;
-        int32_t              peer_cnt   = 0;
-        uuid_t               peer_uuid  = {0};
-        xlator_t            *this       = NULL;
-        glusterd_peerinfo_t *peerinfo   = NULL;
-        struct syncargs      args       = {0};
-
-        this = THIS;
-        GF_ASSERT (this);
-        GF_ASSERT (dict);
-        GF_ASSERT (op_errstr);
-
-        /* If the lock has not been held during this
-         * transaction, do not send unlock requests */
-        if (!is_acquired)
-                goto out;
-
-        if (!npeers) {
-                ret = 0;
-                goto out;
-        }
-
-        /* Sending mgmt_v3 unlock req to other nodes in the cluster */
-        gd_syncargs_init (&args, NULL);
-        synctask_barrier_init((&args));
-        peer_cnt = 0;
-        list_for_each_local_xaction_peers (peerinfo, peers) {
-                gd_mgmt_v3_unlock (op, dict, peerinfo, &args,
-                                   MY_UUID, peer_uuid);
-                peer_cnt++;
-        }
-        gd_synctask_barrier_wait((&args), peer_cnt);
+    int32_t ret = -1;
+    int32_t peer_cnt = 0;
+    dict_t *rsp_dict = NULL;
+    glusterd_peerinfo_t *peerinfo = NULL;
+    struct syncargs args = {0};
+    uuid_t peer_uuid = {0};
+    xlator_t *this = NULL;
+    glusterd_conf_t *conf = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+    conf = this->private;
+    GF_ASSERT(conf);
+
+    GF_ASSERT(dict);
+    GF_VALIDATE_OR_GOTO(this->name, req_dict, out);
+    GF_ASSERT(op_errstr);
+
+    rsp_dict = dict_new();
+    if (!rsp_dict) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_CREATE_FAIL,
+               "Failed to create response dictionary");
+        goto out;
+    }
+
+    /* Post Validation on local node */
+    ret = gd_mgmt_v3_post_validate_fn(op, op_ret, req_dict, op_errstr,
+                                      rsp_dict);
+
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_POST_VALIDATION_FAIL,
+               "Post Validation failed for "
+               "operation %s on local node",
+               gd_op_list[op]);
+
+        if (*op_errstr == NULL) {
+            ret = gf_asprintf(op_errstr,
+                              "Post-validation failed "
+                              "on localhost. Please check "
+                              "log file for details");
+            if (ret == -1)
+                *op_errstr = NULL;
+
+            ret = -1;
+        }
+        goto out;
+    }
+
+    dict_unref(rsp_dict);
+    rsp_dict = NULL;
+
+    /* Sending Post Validation req to other nodes in the cluster */
+    gd_syncargs_init(&args, req_dict);
+    ret = synctask_barrier_init((&args));
+    if (ret)
+        goto out;
+
+    peer_cnt = 0;
+
+    RCU_READ_LOCK;
+    cds_list_for_each_entry_rcu(peerinfo, &conf->peers, uuid_list)
+    {
+        /* Only send requests to peers who were available before the
+         * transaction started
+         */
+        if (peerinfo->generation > txn_generation)
+            continue;
+
+        if (!peerinfo->connected)
+            continue;
+        if (op != GD_OP_SYNC_VOLUME &&
+            peerinfo->state.state != GD_FRIEND_STATE_BEFRIENDED)
+            continue;
+
+        gd_mgmt_v3_post_validate_req(op, op_ret, req_dict, peerinfo, &args,
+                                     MY_UUID, peer_uuid);
+        peer_cnt++;
+    }
+    RCU_READ_UNLOCK;
+
+    if (0 == peer_cnt) {
+        ret = 0;
+        goto out;
+    }
 
-        if (args.op_ret) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "Unlock failed on peers");
+    gd_synctask_barrier_wait((&args), peer_cnt);
 
-                if (!op_ret && args.errstr)
-                         *op_errstr = gf_strdup (args.errstr);
-        }
+    if (args.op_ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_POST_VALIDATION_FAIL,
+               "Post Validation failed on peers");
 
-        ret = args.op_ret;
+        if (args.errstr)
+            *op_errstr = gf_strdup(args.errstr);
+    }
 
-        gf_log (this->name, GF_LOG_DEBUG, "Sent unlock op req for %s "
-                "to %d peers. Returning %d", gd_op_list[op], peer_cnt, ret);
+    ret = args.op_ret;
 
+    gf_msg_debug(this->name, 0,
+                 "Sent post valaidation req for %s "
+                 "to %d peers. Returning %d",
+                 gd_op_list[op], peer_cnt, ret);
 out:
-        return ret;
+    return ret;
 }
 
 int32_t
-glusterd_mgmt_v3_initiate_all_phases (rpcsvc_request_t *req, glusterd_op_t op,
-                                      dict_t *dict)
+gd_mgmt_v3_unlock_cbk_fn(struct rpc_req *req, struct iovec *iov, int count,
+                         void *myframe)
 {
-        int32_t                     ret              = -1;
-        int32_t                     op_ret           = -1;
-        int32_t                     npeers           = 0;
-        dict_t                      *req_dict        = NULL;
-        dict_t                      *tmp_dict        = NULL;
-        glusterd_conf_t             *conf            = NULL;
-        char                        *op_errstr       = NULL;
-        xlator_t                    *this            = NULL;
-        gf_boolean_t                is_acquired      = _gf_false;
-        uuid_t                      *originator_uuid = NULL;
-        struct cds_list_head        xaction_peers   = {0,};
-
-        this = THIS;
-        GF_ASSERT (this);
-        GF_ASSERT (req);
-        GF_ASSERT (dict);
-        conf = this->private;
-        GF_ASSERT (conf);
-
-        CDS_INIT_LIST_HEAD (&xaction_peers);
-        npeers = gd_build_local_xaction_peers_list  (&conf->peers,
-                                                     &xaction_peers, op);
-        if (npeers == -1) {
-                gf_log (this->name, GF_LOG_ERROR, "building local peers list "
-                        "failed");
-                goto rsp;
-        }
+    int32_t ret = -1;
+    struct syncargs *args = NULL;
+    gd1_mgmt_v3_unlock_rsp rsp = {
+        {0},
+    };
+    call_frame_t *frame = NULL;
+    int32_t op_ret = -1;
+    int32_t op_errno = -1;
+    xlator_t *this = NULL;
+    uuid_t *peerid = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+    GF_ASSERT(req);
+    GF_ASSERT(myframe);
+
+    frame = myframe;
+    args = frame->local;
+    peerid = frame->cookie;
+    frame->local = NULL;
+    frame->cookie = NULL;
+
+    if (-1 == req->rpc_status) {
+        op_errno = ENOTCONN;
+        goto out;
+    }
+
+    GF_VALIDATE_OR_GOTO_WITH_ERROR(this->name, iov, out, op_errno, EINVAL);
+
+    ret = xdr_to_generic(*iov, &rsp, (xdrproc_t)xdr_gd1_mgmt_v3_unlock_rsp);
+    if (ret < 0)
+        goto out;
+
+    gf_uuid_copy(args->uuid, rsp.uuid);
+
+    op_ret = rsp.op_ret;
+    op_errno = rsp.op_errno;
 
-        /* Save the MY_UUID as the originator_uuid. This originator_uuid
-         * will be used by is_origin_glusterd() to determine if a node
-         * is the originator node for a command. */
-        originator_uuid = GF_CALLOC (1, sizeof(uuid_t),
-                                     gf_common_mt_uuid_t);
-        if (!originator_uuid) {
-                ret = -1;
-                goto out;
-        }
-
-        gf_uuid_copy (*originator_uuid, MY_UUID);
-        ret = dict_set_bin (dict, "originator_uuid",
-                            originator_uuid, sizeof (uuid_t));
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "Failed to set originator_uuid.");
-                goto out;
-        }
-
-        /* Marking the operation as complete synctasked */
-        ret = dict_set_int32 (dict, "is_synctasked", _gf_true);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "Failed to set synctasked flag.");
-                goto out;
-        }
-
-        /* Use a copy at local unlock as cli response will be sent before
-         * the unlock and the volname in the dict might be removed */
-        tmp_dict = dict_new();
-        if (!tmp_dict) {
-                gf_log (this->name, GF_LOG_ERROR, "Unable to create dict");
-                goto out;
-        }
-        dict_copy (dict, tmp_dict);
+out:
+    gd_mgmt_v3_collate_errors(args, op_ret, op_errno, NULL,
+                              GLUSTERD_MGMT_V3_UNLOCK, *peerid, rsp.uuid);
+    if (rsp.dict.dict_val)
+        free(rsp.dict.dict_val);
+    GF_FREE(peerid);
+    /* req->rpc_status set to -1 means, STACK_DESTROY will be called from
+     * the caller function.
+     */
+    if (req->rpc_status != -1)
+        STACK_DESTROY(frame->root);
+    synctask_barrier_wake(args);
+    return 0;
+}
 
-        /* LOCKDOWN PHASE - Acquire mgmt_v3 locks */
-        ret = glusterd_mgmt_v3_initiate_lockdown (op, dict, &op_errstr,
-                                                  npeers, &is_acquired,
-                                                  &xaction_peers);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "mgmt_v3 lockdown failed.");
-                goto out;
-        }
+int32_t
+gd_mgmt_v3_unlock_cbk(struct rpc_req *req, struct iovec *iov, int count,
+                      void *myframe)
+{
+    return glusterd_big_locked_cbk(req, iov, count, myframe,
+                                   gd_mgmt_v3_unlock_cbk_fn);
+}
 
-        /* BUILD PAYLOAD */
-        ret = glusterd_mgmt_v3_build_payload (&req_dict, &op_errstr, dict, op);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, LOGSTR_BUILD_PAYLOAD,
-                        gd_op_list[op]);
-                if (op_errstr == NULL)
-                        gf_asprintf (&op_errstr, OPERRSTR_BUILD_PAYLOAD);
-                goto out;
-        }
+int
+gd_mgmt_v3_unlock(glusterd_op_t op, dict_t *op_ctx,
+                  glusterd_peerinfo_t *peerinfo, struct syncargs *args,
+                  uuid_t my_uuid, uuid_t recv_uuid)
+{
+    int32_t ret = -1;
+    gd1_mgmt_v3_unlock_req req = {
+        {0},
+    };
+    xlator_t *this = NULL;
+    uuid_t *peerid = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+    GF_ASSERT(op_ctx);
+    GF_ASSERT(peerinfo);
+    GF_ASSERT(args);
+
+    ret = dict_allocate_and_serialize(op_ctx, &req.dict.dict_val,
+                                      &req.dict.dict_len);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno,
+                GD_MSG_DICT_ALLOC_AND_SERL_LENGTH_GET_FAIL, NULL);
+        goto out;
+    }
+
+    gf_uuid_copy(req.uuid, my_uuid);
+    req.op = op;
+
+    GD_ALLOC_COPY_UUID(peerid, peerinfo->uuid, ret);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno,
+                GD_MSG_ALLOC_AND_COPY_UUID_FAIL, NULL);
+        goto out;
+    }
+
+    ret = gd_syncop_submit_request(peerinfo->rpc, &req, args, peerid,
+                                   &gd_mgmt_v3_prog, GLUSTERD_MGMT_V3_UNLOCK,
+                                   gd_mgmt_v3_unlock_cbk,
+                                   (xdrproc_t)xdr_gd1_mgmt_v3_unlock_req);
+out:
+    GF_FREE(req.dict.dict_val);
+    gf_msg_trace(this->name, 0, "Returning %d", ret);
+    return ret;
+}
 
-        /* PRE-COMMIT VALIDATE PHASE */
-        ret = glusterd_mgmt_v3_pre_validate (op, req_dict,
-                                             &op_errstr, npeers,
-                                             &xaction_peers);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "Pre Validation Failed");
-                goto out;
-        }
+int
+glusterd_mgmt_v3_release_peer_locks(glusterd_op_t op, dict_t *dict,
+                                    int32_t op_ret, char **op_errstr,
+                                    gf_boolean_t is_acquired,
+                                    uint32_t txn_generation)
+{
+    int32_t ret = -1;
+    int32_t peer_cnt = 0;
+    uuid_t peer_uuid = {0};
+    xlator_t *this = NULL;
+    glusterd_peerinfo_t *peerinfo = NULL;
+    struct syncargs args = {0};
+    glusterd_conf_t *conf = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+    conf = this->private;
+    GF_ASSERT(conf);
+
+    GF_ASSERT(dict);
+    GF_ASSERT(op_errstr);
+
+    /* If the lock has not been held during this
+     * transaction, do not send unlock requests */
+    if (!is_acquired)
+        goto out;
+
+    /* Sending mgmt_v3 unlock req to other nodes in the cluster */
+    gd_syncargs_init(&args, NULL);
+    ret = synctask_barrier_init((&args));
+    if (ret)
+        goto out;
+    peer_cnt = 0;
+    RCU_READ_LOCK;
+    cds_list_for_each_entry_rcu(peerinfo, &conf->peers, uuid_list)
+    {
+        /* Only send requests to peers who were available before the
+         * transaction started
+         */
+        if (peerinfo->generation > txn_generation)
+            continue;
 
-        /* COMMIT OP PHASE */
-        ret = glusterd_mgmt_v3_commit (op, dict, req_dict,
-                                       &op_errstr, npeers, &xaction_peers);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "Commit Op Failed");
-                goto out;
-        }
+        if (!peerinfo->connected)
+            continue;
+        if (op != GD_OP_SYNC_VOLUME &&
+            peerinfo->state.state != GD_FRIEND_STATE_BEFRIENDED)
+            continue;
 
-        /* POST-COMMIT VALIDATE PHASE */
-        /* As of now, post_validate is not handling any other
-           commands other than snapshot. So as of now, I am
-           sending 0 (op_ret as 0).
-        */
-        ret = glusterd_mgmt_v3_post_validate (op, 0, dict, req_dict,
-                                              &op_errstr, npeers,
-                                              &xaction_peers);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "Post Validation Failed");
-                goto out;
-        }
+        gd_mgmt_v3_unlock(op, dict, peerinfo, &args, MY_UUID, peer_uuid);
+        peer_cnt++;
+    }
+    RCU_READ_UNLOCK;
 
+    if (0 == peer_cnt) {
         ret = 0;
-out:
-        op_ret = ret;
-        /* UNLOCK PHASE FOR PEERS*/
-        (void) glusterd_mgmt_v3_release_peer_locks (op, dict,
-                                                    op_ret, &op_errstr,
-                                                    npeers, is_acquired,
-                                                    &xaction_peers);
-
-        /* LOCAL VOLUME(S) UNLOCK */
-        if (is_acquired) {
-                /* Trying to release multiple mgmt_v3 locks */
-                ret = glusterd_multiple_mgmt_v3_unlock (tmp_dict, MY_UUID);
-                if (ret) {
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "Failed to release mgmt_v3 locks on localhost");
-                        op_ret = ret;
-                }
-        }
-rsp:
-        /* SEND CLI RESPONSE */
-        glusterd_op_send_cli_response (op, op_ret, 0, req, dict, op_errstr);
+        goto out;
+    }
 
-        gd_cleanup_local_xaction_peers_list (&xaction_peers);
+    gd_synctask_barrier_wait((&args), peer_cnt);
 
-        if (req_dict)
-                dict_unref (req_dict);
+    if (args.op_ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_MGMTV3_UNLOCK_FAIL,
+               "Unlock failed on peers");
 
-        if (tmp_dict)
-                dict_unref (tmp_dict);
+        if (!op_ret && args.errstr)
+            *op_errstr = gf_strdup(args.errstr);
+    }
 
-        if (op_errstr) {
-                GF_FREE (op_errstr);
-                op_errstr = NULL;
-        }
+    ret = args.op_ret;
 
-        return 0;
+    gf_msg_debug(this->name, 0,
+                 "Sent unlock op req for %s "
+                 "to %d peers. Returning %d",
+                 gd_op_list[op], peer_cnt, ret);
+
+out:
+    return ret;
 }
 
 int32_t
-glusterd_set_barrier_value (dict_t *dict, char *option)
+glusterd_mgmt_v3_initiate_all_phases_with_brickop_phase(rpcsvc_request_t *req,
+                                                        glusterd_op_t op,
+                                                        dict_t *dict)
 {
-        int32_t                 ret             = -1;
-        xlator_t                *this           = NULL;
-        glusterd_volinfo_t     *vol             = NULL;
-        char                    *volname        = NULL;
-
-        this = THIS;
-        GF_ASSERT (this);
-
-        GF_ASSERT (dict);
-        GF_ASSERT (option);
-
-        /* TODO : Change this when we support multiple volume.
-         * As of now only snapshot of single volume is supported,
-         * Hence volname1 is directly fetched
-         */
-        ret = dict_get_str (dict, "volname1", &volname);
+    int32_t ret = -1;
+    int32_t op_ret = -1;
+    dict_t *req_dict = NULL;
+    dict_t *tmp_dict = NULL;
+    glusterd_conf_t *conf = NULL;
+    char *op_errstr = NULL;
+    xlator_t *this = NULL;
+    gf_boolean_t is_acquired = _gf_false;
+    uuid_t *originator_uuid = NULL;
+    uint32_t txn_generation = 0;
+    uint32_t op_errno = 0;
+
+    this = THIS;
+    GF_ASSERT(this);
+    GF_ASSERT(req);
+    GF_ASSERT(dict);
+    conf = this->private;
+    GF_ASSERT(conf);
+
+    /* Save the peer list generation */
+    txn_generation = conf->generation;
+    cmm_smp_rmb();
+    /* This read memory barrier makes sure that this assignment happens here
+     * only and is not reordered and optimized by either the compiler or the
+     * processor.
+     */
+
+    /* Save the MY_UUID as the originator_uuid. This originator_uuid
+     * will be used by is_origin_glusterd() to determine if a node
+     * is the originator node for a command. */
+    originator_uuid = GF_MALLOC(sizeof(uuid_t), gf_common_mt_uuid_t);
+    if (!originator_uuid) {
+        ret = -1;
+        goto out;
+    }
+
+    gf_uuid_copy(*originator_uuid, MY_UUID);
+    ret = dict_set_bin(dict, "originator_uuid", originator_uuid,
+                       sizeof(uuid_t));
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+               "Failed to set originator_uuid.");
+        GF_FREE(originator_uuid);
+        goto out;
+    }
+
+    /* Marking the operation as complete synctasked */
+    ret = dict_set_int32(dict, "is_synctasked", _gf_true);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+               "Failed to set synctasked flag.");
+        goto out;
+    }
+
+    /* Use a copy at local unlock as cli response will be sent before
+     * the unlock and the volname in the dict might be removed */
+    tmp_dict = dict_new();
+    if (!tmp_dict) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_CREATE_FAIL,
+               "Unable to create dict");
+        goto out;
+    }
+    dict_copy(dict, tmp_dict);
+
+    /* LOCKDOWN PHASE - Acquire mgmt_v3 locks */
+    ret = glusterd_mgmt_v3_initiate_lockdown(op, dict, &op_errstr, &op_errno,
+                                             &is_acquired, txn_generation);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_MGMTV3_LOCKDOWN_FAIL,
+               "mgmt_v3 lockdown failed.");
+        goto out;
+    }
+
+    /* BUILD PAYLOAD */
+    ret = glusterd_mgmt_v3_build_payload(&req_dict, &op_errstr, dict, op);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_MGMTV3_PAYLOAD_BUILD_FAIL,
+               LOGSTR_BUILD_PAYLOAD, gd_op_list[op]);
+        if (op_errstr == NULL)
+            gf_asprintf(&op_errstr, OPERRSTR_BUILD_PAYLOAD);
+        goto out;
+    }
+
+    /* PRE-COMMIT VALIDATE PHASE */
+    ret = glusterd_mgmt_v3_pre_validate(op, req_dict, &op_errstr, &op_errno,
+                                        txn_generation);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_PRE_VALIDATION_FAIL,
+               "Pre Validation Failed");
+        goto out;
+    }
+
+    /* BRICK-OPS */
+    ret = glusterd_mgmt_v3_brick_op(op, dict, req_dict, &op_errstr,
+                                    txn_generation);
+    if (ret) {
+        gf_log(this->name, GF_LOG_ERROR, "Brick Op Failed");
+        goto out;
+    }
+
+    /* COMMIT OP PHASE */
+    ret = glusterd_mgmt_v3_commit(op, dict, req_dict, &op_errstr, &op_errno,
+                                  txn_generation);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_COMMIT_OP_FAIL,
+               "Commit Op Failed");
+        goto out;
+    }
+
+    /* POST-COMMIT VALIDATE PHASE */
+    /* As of now, post_validate is not trying to cleanup any failed
+       commands. So as of now, I am sending 0 (op_ret as 0).
+    */
+    ret = glusterd_mgmt_v3_post_validate(op, 0, dict, req_dict, &op_errstr,
+                                         txn_generation);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_POST_VALIDATION_FAIL,
+               "Post Validation Failed");
+        goto out;
+    }
+
+    ret = 0;
+out:
+    op_ret = ret;
+    /* UNLOCK PHASE FOR PEERS*/
+    (void)glusterd_mgmt_v3_release_peer_locks(op, dict, op_ret, &op_errstr,
+                                              is_acquired, txn_generation);
+
+    /* LOCAL VOLUME(S) UNLOCK */
+    if (is_acquired) {
+        /* Trying to release multiple mgmt_v3 locks */
+        ret = glusterd_multiple_mgmt_v3_unlock(tmp_dict, MY_UUID);
         if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "Volname not present in "
-                        "dict");
-                goto out;
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_MGMTV3_UNLOCK_FAIL,
+                   "Failed to release mgmt_v3 locks on localhost");
+            op_ret = ret;
         }
+    }
 
-        ret = glusterd_volinfo_find (volname, &vol);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "Volume %s not found ",
-                        volname);
-                goto out;
-        }
+    if (op_ret && (op_errno == 0))
+        op_errno = EG_INTRNL;
 
-        ret = dict_set_dynstr_with_alloc (dict, "barrier", option);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "Failed to set barrier op "
-                        "in request dictionary");
-                goto out;
-        }
-
-        ret = dict_set_dynstr_with_alloc (vol->dict, "features.barrier",
-                                          option);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "Failed to set barrier op "
-                        "in volume option dict");
-                goto out;
-        }
+    if (op != GD_OP_MAX_OPVERSION) {
+        /* SEND CLI RESPONSE */
+        glusterd_op_send_cli_response(op, op_ret, op_errno, req, dict,
+                                      op_errstr);
+    }
 
-        gd_update_volume_op_versions (vol);
+    if (req_dict)
+        dict_unref(req_dict);
 
-        ret = glusterd_create_volfiles (vol);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "Failed to create volfiles");
-                goto out;
-        }
+    if (tmp_dict)
+        dict_unref(tmp_dict);
 
-        ret = glusterd_store_volinfo (vol, GLUSTERD_VOLINFO_VER_AC_INCREMENT);
+    if (op_errstr) {
+        GF_FREE(op_errstr);
+        op_errstr = NULL;
+    }
 
-out:
-        gf_log (this->name, GF_LOG_DEBUG, "Returning %d", ret);
-        return ret;
+    return 0;
 }
 
 int32_t
-glusterd_mgmt_v3_initiate_snap_phases (rpcsvc_request_t *req, glusterd_op_t op,
-                                       dict_t *dict)
+glusterd_mgmt_v3_initiate_all_phases(rpcsvc_request_t *req, glusterd_op_t op,
+                                     dict_t *dict)
 {
-        int32_t                     ret              = -1;
-        int32_t                     op_ret           = -1;
-        int32_t                     npeers           = 0;
-        dict_t                      *req_dict        = NULL;
-        dict_t                      *tmp_dict        = NULL;
-        glusterd_conf_t             *conf            = NULL;
-        char                        *op_errstr       = NULL;
-        xlator_t                    *this            = NULL;
-        gf_boolean_t                is_acquired      = _gf_false;
-        uuid_t                      *originator_uuid = NULL;
-        gf_boolean_t                success          = _gf_false;
-        char                        *cli_errstr      = NULL;
-        struct cds_list_head        xaction_peers    = {0,};
-
-        this = THIS;
-        GF_ASSERT (this);
-        GF_ASSERT (req);
-        GF_ASSERT (dict);
-        conf = this->private;
-        GF_ASSERT (conf);
-
-        CDS_INIT_LIST_HEAD (&xaction_peers);
-        npeers = gd_build_local_xaction_peers_list  (&conf->peers,
-                                                     &xaction_peers, op);
-        if (npeers == -1) {
-                gf_log (this->name, GF_LOG_ERROR, "building local peers list "
-                        "failed");
-                goto rsp;
-        }
-
-        /* Save the MY_UUID as the originator_uuid. This originator_uuid
-         * will be used by is_origin_glusterd() to determine if a node
-         * is the originator node for a command. */
-        originator_uuid = GF_CALLOC (1, sizeof(uuid_t),
-                                     gf_common_mt_uuid_t);
-        if (!originator_uuid) {
-                ret = -1;
-                goto out;
-        }
-
-        gf_uuid_copy (*originator_uuid, MY_UUID);
-        ret = dict_set_bin (dict, "originator_uuid",
-                            originator_uuid, sizeof (uuid_t));
+    int32_t ret = -1;
+    int32_t op_ret = -1;
+    dict_t *req_dict = NULL;
+    dict_t *tmp_dict = NULL;
+    glusterd_conf_t *conf = NULL;
+    char *op_errstr = NULL;
+    xlator_t *this = NULL;
+    gf_boolean_t is_acquired = _gf_false;
+    uuid_t *originator_uuid = NULL;
+    uint32_t txn_generation = 0;
+    uint32_t op_errno = 0;
+
+    this = THIS;
+    GF_ASSERT(this);
+    GF_ASSERT(req);
+    GF_ASSERT(dict);
+    conf = this->private;
+    GF_ASSERT(conf);
+
+    /* Save the peer list generation */
+    txn_generation = conf->generation;
+    cmm_smp_rmb();
+    /* This read memory barrier makes sure that this assignment happens here
+     * only and is not reordered and optimized by either the compiler or the
+     * processor.
+     */
+
+    /* Save the MY_UUID as the originator_uuid. This originator_uuid
+     * will be used by is_origin_glusterd() to determine if a node
+     * is the originator node for a command. */
+    originator_uuid = GF_MALLOC(sizeof(uuid_t), gf_common_mt_uuid_t);
+    if (!originator_uuid) {
+        ret = -1;
+        goto out;
+    }
+
+    gf_uuid_copy(*originator_uuid, MY_UUID);
+    ret = dict_set_bin(dict, "originator_uuid", originator_uuid,
+                       sizeof(uuid_t));
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+               "Failed to set originator_uuid.");
+        GF_FREE(originator_uuid);
+        goto out;
+    }
+
+    /* Marking the operation as complete synctasked */
+    ret = dict_set_int32(dict, "is_synctasked", _gf_true);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+               "Failed to set synctasked flag.");
+        goto out;
+    }
+
+    /* Use a copy at local unlock as cli response will be sent before
+     * the unlock and the volname in the dict might be removed */
+    tmp_dict = dict_new();
+    if (!tmp_dict) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_CREATE_FAIL,
+               "Unable to create dict");
+        goto out;
+    }
+    dict_copy(dict, tmp_dict);
+
+    /* LOCKDOWN PHASE - Acquire mgmt_v3 locks */
+    ret = glusterd_mgmt_v3_initiate_lockdown(op, dict, &op_errstr, &op_errno,
+                                             &is_acquired, txn_generation);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_MGMTV3_LOCKDOWN_FAIL,
+               "mgmt_v3 lockdown failed.");
+        goto out;
+    }
+
+    /* BUILD PAYLOAD */
+    ret = glusterd_mgmt_v3_build_payload(&req_dict, &op_errstr, dict, op);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_MGMTV3_PAYLOAD_BUILD_FAIL,
+               LOGSTR_BUILD_PAYLOAD, gd_op_list[op]);
+        if (op_errstr == NULL)
+            gf_asprintf(&op_errstr, OPERRSTR_BUILD_PAYLOAD);
+        goto out;
+    }
+
+    /* PRE-COMMIT VALIDATE PHASE */
+    ret = glusterd_mgmt_v3_pre_validate(op, req_dict, &op_errstr, &op_errno,
+                                        txn_generation);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_PRE_VALIDATION_FAIL,
+               "Pre Validation Failed");
+        goto out;
+    }
+
+    /* COMMIT OP PHASE */
+    ret = glusterd_mgmt_v3_commit(op, dict, req_dict, &op_errstr, &op_errno,
+                                  txn_generation);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_COMMIT_OP_FAIL,
+               "Commit Op Failed");
+        goto out;
+    }
+
+    /* POST COMMIT OP PHASE */
+    ret = glusterd_mgmt_v3_post_commit(op, dict, req_dict, &op_errstr,
+                                       &op_errno, txn_generation);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_POST_COMMIT_OP_FAIL,
+               "Post commit Op Failed");
+        goto out;
+    }
+
+    /* POST-COMMIT VALIDATE PHASE */
+    /* As of now, post_validate is not trying to cleanup any failed
+       commands. So as of now, I am sending 0 (op_ret as 0).
+    */
+    ret = glusterd_mgmt_v3_post_validate(op, 0, dict, req_dict, &op_errstr,
+                                         txn_generation);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_POST_VALIDATION_FAIL,
+               "Post Validation Failed");
+        goto out;
+    }
+
+    ret = 0;
+out:
+    op_ret = ret;
+    /* UNLOCK PHASE FOR PEERS*/
+    (void)glusterd_mgmt_v3_release_peer_locks(op, dict, op_ret, &op_errstr,
+                                              is_acquired, txn_generation);
+
+    /* LOCAL VOLUME(S) UNLOCK */
+    if (is_acquired) {
+        /* Trying to release multiple mgmt_v3 locks */
+        ret = glusterd_multiple_mgmt_v3_unlock(tmp_dict, MY_UUID);
         if (ret) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "Failed to set originator_uuid.");
-                goto out;
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_MGMTV3_UNLOCK_FAIL,
+                   "Failed to release mgmt_v3 locks on localhost");
+            op_ret = ret;
         }
+    }
 
-        /* Marking the operation as complete synctasked */
-        ret = dict_set_int32 (dict, "is_synctasked", _gf_true);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "Failed to set synctasked flag.");
-                goto out;
-        }
+    if (op_ret && (op_errno == 0))
+        op_errno = EG_INTRNL;
 
-        /* Use a copy at local unlock as cli response will be sent before
-         * the unlock and the volname in the dict might be removed */
-        tmp_dict = dict_new();
-        if (!tmp_dict) {
-                gf_log (this->name, GF_LOG_ERROR, "Unable to create dict");
-                goto out;
-        }
-        dict_copy (dict, tmp_dict);
+    if (op != GD_OP_MAX_OPVERSION) {
+        /* SEND CLI RESPONSE */
+        glusterd_op_send_cli_response(op, op_ret, op_errno, req, dict,
+                                      op_errstr);
+    }
 
-        /* LOCKDOWN PHASE - Acquire mgmt_v3 locks */
-        ret = glusterd_mgmt_v3_initiate_lockdown (op, dict, &op_errstr,
-                                                  npeers, &is_acquired,
-                                                  &xaction_peers);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "mgmt_v3 lockdown failed.");
-                goto out;
-        }
+    if (req_dict)
+        dict_unref(req_dict);
 
-        /* BUILD PAYLOAD */
-        ret = glusterd_mgmt_v3_build_payload (&req_dict, &op_errstr, dict, op);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, LOGSTR_BUILD_PAYLOAD,
-                        gd_op_list[op]);
-                if (op_errstr == NULL)
-                        gf_asprintf (&op_errstr, OPERRSTR_BUILD_PAYLOAD);
-                goto out;
-        }
+    if (tmp_dict)
+        dict_unref(tmp_dict);
 
-        /* PRE-COMMIT VALIDATE PHASE */
-        ret = glusterd_mgmt_v3_pre_validate (op, req_dict,
-                                            &op_errstr, npeers, &xaction_peers);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "Pre Validation Failed");
-                goto out;
-        }
+    if (op_errstr) {
+        GF_FREE(op_errstr);
+        op_errstr = NULL;
+    }
 
-        /* quorum check of the volume is done here */
-        ret = glusterd_snap_quorum_check (req_dict, _gf_false, &op_errstr);
-        if (ret) {
-                gf_log (this->name, GF_LOG_WARNING,
-                                "Volume quorum check failed");
-                goto out;
-        }
+    return 0;
+}
 
-        /* Set the operation type as pre, so that differentiation can be
-         * made whether the brickop is sent during pre-commit or post-commit
-         */
-        ret = dict_set_dynstr_with_alloc (req_dict, "operation-type", "pre");
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "Failed to set "
-                        "operation-type in dictionary");
-                goto out;
-        }
+int32_t
+glusterd_set_barrier_value(dict_t *dict, char *option)
+{
+    int32_t ret = -1;
+    xlator_t *this = NULL;
+    glusterd_volinfo_t *vol = NULL;
+    char *volname = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+
+    GF_ASSERT(dict);
+    GF_ASSERT(option);
+
+    /* TODO : Change this when we support multiple volume.
+     * As of now only snapshot of single volume is supported,
+     * Hence volname1 is directly fetched
+     */
+    ret = dict_get_strn(dict, "volname1", SLEN("volname1"), &volname);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+               "Volname not present in "
+               "dict");
+        goto out;
+    }
+
+    ret = glusterd_volinfo_find(volname, &vol);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_VOL_NOT_FOUND,
+               "Volume %s not found ", volname);
+        goto out;
+    }
+
+    ret = dict_set_dynstr_with_alloc(dict, "barrier", option);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+               "Failed to set barrier op "
+               "in request dictionary");
+        goto out;
+    }
+
+    ret = dict_set_dynstr_with_alloc(vol->dict, "features.barrier", option);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+               "Failed to set barrier op "
+               "in volume option dict");
+        goto out;
+    }
+
+    gd_update_volume_op_versions(vol);
+
+    ret = glusterd_create_volfiles(vol);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_VOLFILE_CREATE_FAIL,
+               "Failed to create volfiles");
+        goto out;
+    }
+
+    ret = glusterd_store_volinfo(vol, GLUSTERD_VOLINFO_VER_AC_INCREMENT);
 
-        ret = glusterd_mgmt_v3_brick_op (op, req_dict,
-                                        &op_errstr, npeers, &xaction_peers);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "Brick Ops Failed");
-                goto unbarrier;
-        }
+out:
+    gf_msg_debug(this->name, 0, "Returning %d", ret);
+    return ret;
+}
 
-        /* COMMIT OP PHASE */
-        /* TODO: As of now, the plan is to do quorum check before sending the
-           commit fop and if the quorum succeeds, then commit is sent to all
-           the other glusterds.
-           snap create functionality now creates the in memory and on disk
-           objects for the snapshot (marking them as incomplete), takes the lvm
-           snapshot and then updates the status of the in memory and on disk
-           snap objects as complete. Suppose one of the glusterds goes down
-           after taking the lvm snapshot, but before updating the snap object,
-           then treat it as a snapshot create failure and trigger cleanup.
-           i.e the number of commit responses received by the originator
-           glusterd shold be the same as the number of peers it has sent the
-           request to (i.e npeers variable). If not, then originator glusterd
-           will initiate cleanup in post-validate fop.
-           Question: What if one of the other glusterds goes down as explained
-           above and along with it the originator glusterd also goes down?
-           Who will initiate the cleanup?
+int32_t
+glusterd_mgmt_v3_initiate_snap_phases(rpcsvc_request_t *req, glusterd_op_t op,
+                                      dict_t *dict)
+{
+    int32_t ret = -1;
+    int32_t op_ret = -1;
+    dict_t *req_dict = NULL;
+    dict_t *tmp_dict = NULL;
+    glusterd_conf_t *conf = NULL;
+    char *op_errstr = NULL;
+    xlator_t *this = NULL;
+    gf_boolean_t is_acquired = _gf_false;
+    uuid_t *originator_uuid = NULL;
+    gf_boolean_t success = _gf_false;
+    char *cli_errstr = NULL;
+    uint32_t txn_generation = 0;
+    uint32_t op_errno = 0;
+
+    this = THIS;
+    GF_ASSERT(this);
+    GF_ASSERT(req);
+    GF_ASSERT(dict);
+    conf = this->private;
+    GF_ASSERT(conf);
+
+    /* Save the peer list generation */
+    txn_generation = conf->generation;
+    cmm_smp_rmb();
+    /* This read memory barrier makes sure that this assignment happens here
+     * only and is not reordered and optimized by either the compiler or the
+     * processor.
+     */
+
+    /* Save the MY_UUID as the originator_uuid. This originator_uuid
+     * will be used by is_origin_glusterd() to determine if a node
+     * is the originator node for a command. */
+    originator_uuid = GF_MALLOC(sizeof(uuid_t), gf_common_mt_uuid_t);
+    if (!originator_uuid) {
+        ret = -1;
+        goto out;
+    }
+
+    gf_uuid_copy(*originator_uuid, MY_UUID);
+    ret = dict_set_bin(dict, "originator_uuid", originator_uuid,
+                       sizeof(uuid_t));
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+               "Failed to set originator_uuid.");
+        GF_FREE(originator_uuid);
+        goto out;
+    }
+
+    /* Marking the operation as complete synctasked */
+    ret = dict_set_int32n(dict, "is_synctasked", SLEN("is_synctasked"),
+                          _gf_true);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+               "Failed to set synctasked flag.");
+        goto out;
+    }
+
+    /* Use a copy at local unlock as cli response will be sent before
+     * the unlock and the volname in the dict might be removed */
+    tmp_dict = dict_new();
+    if (!tmp_dict) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_CREATE_FAIL,
+               "Unable to create dict");
+        goto out;
+    }
+    dict_copy(dict, tmp_dict);
+
+    /* LOCKDOWN PHASE - Acquire mgmt_v3 locks */
+    ret = glusterd_mgmt_v3_initiate_lockdown(op, dict, &op_errstr, &op_errno,
+                                             &is_acquired, txn_generation);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_MGMTV3_LOCKDOWN_FAIL,
+               "mgmt_v3 lockdown failed.");
+        goto out;
+    }
+
+    /* BUILD PAYLOAD */
+    ret = glusterd_mgmt_v3_build_payload(&req_dict, &op_errstr, dict, op);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_MGMTV3_PAYLOAD_BUILD_FAIL,
+               LOGSTR_BUILD_PAYLOAD, gd_op_list[op]);
+        if (op_errstr == NULL)
+            gf_asprintf(&op_errstr, OPERRSTR_BUILD_PAYLOAD);
+        goto out;
+    }
+
+    /* PRE-COMMIT VALIDATE PHASE */
+    ret = glusterd_mgmt_v3_pre_validate(op, req_dict, &op_errstr, &op_errno,
+                                        txn_generation);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_PRE_VALIDATION_FAIL,
+               "Pre Validation Failed");
+        goto out;
+    }
+
+    /* quorum check of the volume is done here */
+    ret = glusterd_snap_quorum_check(req_dict, _gf_false, &op_errstr,
+                                     &op_errno);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_WARNING, 0, GD_MSG_QUORUM_CHECK_FAIL,
+               "Volume quorum check failed");
+        goto out;
+    }
+
+    /* Set the operation type as pre, so that differentiation can be
+     * made whether the brickop is sent during pre-commit or post-commit
+     */
+    ret = dict_set_dynstr_with_alloc(req_dict, "operation-type", "pre");
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+               "Failed to set "
+               "operation-type in dictionary");
+        goto out;
+    }
+
+    ret = glusterd_mgmt_v3_brick_op(op, dict, req_dict, &op_errstr,
+                                    txn_generation);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_BRICK_OP_FAIL,
+               "Brick Ops Failed");
+        goto unbarrier;
+    }
+
+    /* COMMIT OP PHASE */
+    /* TODO: As of now, the plan is to do quorum check before sending the
+       commit fop and if the quorum succeeds, then commit is sent to all
+       the other glusterds.
+       snap create functionality now creates the in memory and on disk
+       objects for the snapshot (marking them as incomplete), takes the lvm
+       snapshot and then updates the status of the in memory and on disk
+       snap objects as complete. Suppose one of the glusterds goes down
+       after taking the lvm snapshot, but before updating the snap object,
+       then treat it as a snapshot create failure and trigger cleanup.
+       i.e the number of commit responses received by the originator
+       glusterd shold be the same as the number of peers it has sent the
+       request to (i.e npeers variable). If not, then originator glusterd
+       will initiate cleanup in post-validate fop.
+       Question: What if one of the other glusterds goes down as explained
+       above and along with it the originator glusterd also goes down?
+       Who will initiate the cleanup?
+    */
+    ret = dict_set_int32n(req_dict, "cleanup", SLEN("cleanup"), 1);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+               "failed to set dict");
+        goto unbarrier;
+    }
+
+    ret = glusterd_mgmt_v3_commit(op, dict, req_dict, &op_errstr, &op_errno,
+                                  txn_generation);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_COMMIT_OP_FAIL,
+               "Commit Op Failed");
+        /* If the main op fails, we should save the error string.
+           Because, op_errstr will be used for unbarrier and
+           unlock ops also. We might lose the actual error that
+           caused the failure.
         */
-        ret = dict_set_int32 (req_dict, "cleanup", 1);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "failed to set dict");
-                goto unbarrier;
-        }
-
-        ret = glusterd_mgmt_v3_commit (op, dict, req_dict,
-                                       &op_errstr, npeers, &xaction_peers);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "Commit Op Failed");
-                /* If the main op fails, we should save the error string.
-                   Because, op_errstr will be used for unbarrier and
-                   unlock ops also. We might lose the actual error that
-                   caused the failure.
-                */
-                cli_errstr = op_errstr;
-                op_errstr = NULL;
-                goto unbarrier;
-        }
+        cli_errstr = op_errstr;
+        op_errstr = NULL;
+        goto unbarrier;
+    }
 
-        success = _gf_true;
+    success = _gf_true;
 unbarrier:
-        /* Set the operation type as post, so that differentiation can be
-         * made whether the brickop is sent during pre-commit or post-commit
-         */
-        ret = dict_set_dynstr_with_alloc (req_dict, "operation-type", "post");
+    /* Set the operation type as post, so that differentiation can be
+     * made whether the brickop is sent during pre-commit or post-commit
+     */
+    ret = dict_set_dynstr_with_alloc(req_dict, "operation-type", "post");
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+               "Failed to set "
+               "operation-type in dictionary");
+        goto out;
+    }
+
+    ret = glusterd_mgmt_v3_brick_op(op, dict, req_dict, &op_errstr,
+                                    txn_generation);
+
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_BRICK_OP_FAIL,
+               "Brick Ops Failed");
+        goto out;
+    }
+
+    /*Do a quorum check if the commit phase is successful*/
+    if (success) {
+        // quorum check of the snapshot volume
+        ret = glusterd_snap_quorum_check(dict, _gf_true, &op_errstr, &op_errno);
         if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "Failed to set "
-                        "operation-type in dictionary");
-                goto out;
+            gf_msg(this->name, GF_LOG_WARNING, 0, GD_MSG_QUORUM_CHECK_FAIL,
+                   "Snapshot Volume quorum check failed");
+            goto out;
         }
+    }
 
-        ret = glusterd_mgmt_v3_brick_op (op, req_dict,
-                                         &op_errstr, npeers, &xaction_peers);
-
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "Brick Ops Failed");
-                goto out;
-        }
-
-        /*Do a quorum check if the commit phase is successful*/
-        if (success) {
-                //quorum check of the snapshot volume
-                ret = glusterd_snap_quorum_check (dict, _gf_true, &op_errstr);
-                if (ret) {
-                        gf_log (this->name, GF_LOG_WARNING, 
-                                "Snapshot Volume quorum check failed");
-                        goto out;
-                }
-        }
-
-        ret = 0;
+    ret = 0;
 
 out:
-        op_ret = ret;
-
-        if (success == _gf_false)
-                op_ret = -1;
-
-        /* POST-COMMIT VALIDATE PHASE */
-        ret = glusterd_mgmt_v3_post_validate (op, op_ret, dict, req_dict,
-                                              &op_errstr, npeers,
-                                              &xaction_peers);
+    op_ret = ret;
+
+    if (success == _gf_false)
+        op_ret = -1;
+
+    /* POST-COMMIT VALIDATE PHASE */
+    ret = glusterd_mgmt_v3_post_validate(op, op_ret, dict, req_dict, &op_errstr,
+                                         txn_generation);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_POST_VALIDATION_FAIL,
+               "Post Validation Failed");
+        op_ret = -1;
+    }
+
+    /* UNLOCK PHASE FOR PEERS*/
+    (void)glusterd_mgmt_v3_release_peer_locks(op, dict, op_ret, &op_errstr,
+                                              is_acquired, txn_generation);
+
+    /* If the commit op (snapshot taking) failed, then the error is stored
+       in cli_errstr and unbarrier is called. Suppose, if unbarrier also
+       fails, then the error happened in unbarrier is logged and freed.
+       The error happened in commit op, which is stored in cli_errstr
+       is sent to cli.
+    */
+    if (cli_errstr) {
+        GF_FREE(op_errstr);
+        op_errstr = NULL;
+        op_errstr = cli_errstr;
+    }
+
+    /* LOCAL VOLUME(S) UNLOCK */
+    if (is_acquired) {
+        /* Trying to release multiple mgmt_v3 locks */
+        ret = glusterd_multiple_mgmt_v3_unlock(tmp_dict, MY_UUID);
         if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "Post Validation Failed");
-                op_ret = -1;
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_MGMTV3_UNLOCK_FAIL,
+                   "Failed to release mgmt_v3 locks on localhost");
+            op_ret = ret;
         }
+    }
 
-        /* UNLOCK PHASE FOR PEERS*/
-        (void) glusterd_mgmt_v3_release_peer_locks (op, dict,
-                                                    op_ret, &op_errstr,
-                                                    npeers, is_acquired,
-                                                    &xaction_peers);
-
-        /* If the commit op (snapshot taking) failed, then the error is stored
-           in cli_errstr and unbarrier is called. Suppose, if unbarrier also
-           fails, then the error happened in unbarrier is logged and freed.
-           The error happened in commit op, which is stored in cli_errstr
-           is sent to cli.
-        */
-        if (cli_errstr) {
-                GF_FREE (op_errstr);
-                op_errstr = NULL;
-                op_errstr = cli_errstr;
-        }
-
-        /* LOCAL VOLUME(S) UNLOCK */
-        if (is_acquired) {
-                /* Trying to release multiple mgmt_v3 locks */
-                ret = glusterd_multiple_mgmt_v3_unlock (tmp_dict, MY_UUID);
-                if (ret) {
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "Failed to release mgmt_v3 locks on localhost");
-                        op_ret = ret;
-                }
-        }
-rsp:
-        /* SEND CLI RESPONSE */
-        glusterd_op_send_cli_response (op, op_ret, 0, req, dict, op_errstr);
+    if (op_ret && (op_errno == 0))
+        op_errno = EG_INTRNL;
 
-        gd_cleanup_local_xaction_peers_list (&xaction_peers);
+    /* SEND CLI RESPONSE */
+    glusterd_op_send_cli_response(op, op_ret, op_errno, req, dict, op_errstr);
 
-        if (req_dict)
-                dict_unref (req_dict);
+    if (req_dict)
+        dict_unref(req_dict);
 
-        if (tmp_dict)
-                dict_unref (tmp_dict);
+    if (tmp_dict)
+        dict_unref(tmp_dict);
 
-        if (op_errstr) {
-                GF_FREE (op_errstr);
-                op_errstr = NULL;
-        }
+    if (op_errstr) {
+        GF_FREE(op_errstr);
+        op_errstr = NULL;
+    }
 
-        return 0;
+    return 0;
 }
diff --git a/xlators/mgmt/glusterd/src/glusterd-mgmt.h b/xlators/mgmt/glusterd/src/glusterd-mgmt.h
index ab9265c75c4..27dd1849519 100644
--- a/xlators/mgmt/glusterd/src/glusterd-mgmt.h
+++ b/xlators/mgmt/glusterd/src/glusterd-mgmt.h
@@ -10,42 +10,88 @@
 #ifndef _GLUSTERD_MGMT_H_
 #define _GLUSTERD_MGMT_H_
 
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
+void
+gd_mgmt_v3_collate_errors(struct syncargs *args, int op_ret, int op_errno,
+                          char *op_errstr, int op_code, uuid_t peerid,
+                          u_char *uuid);
 
-void gd_mgmt_v3_collate_errors (struct syncargs *args, int op_ret, int op_errno,
-                                char *op_errstr, int op_code,
-                                glusterd_peerinfo_t *peerinfo, u_char *uuid);
+int32_t
+gd_mgmt_v3_pre_validate_fn(glusterd_op_t op, dict_t *dict, char **op_errstr,
+                           dict_t *rsp_dict, uint32_t *op_errno);
 
 int32_t
-gd_mgmt_v3_pre_validate_fn (glusterd_op_t op, dict_t *dict,
-                           char **op_errstr, dict_t *rsp_dict);
+gd_mgmt_v3_brick_op_fn(glusterd_op_t op, dict_t *dict, char **op_errstr,
+                       dict_t *rsp_dict);
 
 int32_t
-gd_mgmt_v3_brick_op_fn (glusterd_op_t op, dict_t *dict,
-                       char **op_errstr, dict_t *rsp_dict);
+gd_mgmt_v3_commit_fn(glusterd_op_t op, dict_t *dict, char **op_errstr,
+                     uint32_t *op_errno, dict_t *rsp_dict);
 
 int32_t
-gd_mgmt_v3_commit_fn (glusterd_op_t op, dict_t *dict,
-                     char **op_errstr, dict_t *rsp_dict);
+gd_mgmt_v3_post_commit_fn(glusterd_op_t op, dict_t *dict, char **op_errstr,
+                          uint32_t *op_errno, dict_t *rsp_dict);
 
 int32_t
-gd_mgmt_v3_post_validate_fn (glusterd_op_t op, int32_t op_ret, dict_t *dict,
+gd_mgmt_v3_post_validate_fn(glusterd_op_t op, int32_t op_ret, dict_t *dict,
                             char **op_errstr, dict_t *rsp_dict);
 
 int32_t
-glusterd_mgmt_v3_initiate_all_phases (rpcsvc_request_t *req, glusterd_op_t op,
+glusterd_mgmt_v3_initiate_all_phases(rpcsvc_request_t *req, glusterd_op_t op,
                                      dict_t *dict);
 
 int32_t
-glusterd_mgmt_v3_initiate_snap_phases (rpcsvc_request_t *req, glusterd_op_t op,
+glusterd_mgmt_v3_initiate_all_phases_with_brickop_phase(rpcsvc_request_t *req,
+                                                        glusterd_op_t op,
+                                                        dict_t *dict);
+
+int32_t
+glusterd_mgmt_v3_initiate_snap_phases(rpcsvc_request_t *req, glusterd_op_t op,
                                       dict_t *dict);
 
 int
-glusterd_snap_pre_validate_use_rsp_dict (dict_t *dst, dict_t *src);
+glusterd_snap_pre_validate_use_rsp_dict(dict_t *dst, dict_t *src);
+
+int32_t
+glusterd_set_barrier_value(dict_t *dict, char *option);
+int
+
+glusterd_mgmt_v3_initiate_lockdown(glusterd_op_t op, dict_t *dict,
+                                   char **op_errstr, uint32_t *op_errno,
+                                   gf_boolean_t *is_acquired,
+                                   uint32_t txn_generation);
+
+int
+glusterd_mgmt_v3_build_payload(dict_t **req, char **op_errstr, dict_t *dict,
+                               glusterd_op_t op);
+
+int
+glusterd_mgmt_v3_pre_validate(glusterd_op_t op, dict_t *req_dict,
+                              char **op_errstr, uint32_t *op_errno,
+                              uint32_t txn_generation);
+
+int
+glusterd_mgmt_v3_commit(glusterd_op_t op, dict_t *op_ctx, dict_t *req_dict,
+                        char **op_errstr, uint32_t *op_errno,
+                        uint32_t txn_generation);
+
+int
+glusterd_mgmt_v3_release_peer_locks(glusterd_op_t op, dict_t *dict,
+                                    int32_t op_ret, char **op_errstr,
+                                    gf_boolean_t is_acquired,
+                                    uint32_t txn_generation);
 
 int32_t
-glusterd_set_barrier_value (dict_t *dict, char *option);
+glusterd_multiple_mgmt_v3_unlock(dict_t *dict, uuid_t uuid);
+
+int
+glusterd_reset_brick_prevalidate(dict_t *dict, char **op_errstr,
+                                 dict_t *rsp_dict);
+int
+glusterd_op_reset_brick(dict_t *dict, dict_t *rsp_dict);
+
+int
+glusterd_post_commit_add_brick(dict_t *dict, char **op_errstr);
+
+int
+glusterd_post_commit_replace_brick(dict_t *dict, char **op_errstr);
 #endif /* _GLUSTERD_MGMT_H_ */
diff --git a/xlators/mgmt/glusterd/src/glusterd-mountbroker.c b/xlators/mgmt/glusterd/src/glusterd-mountbroker.c
index 3d26b8745cc..645d845ee76 100644
--- a/xlators/mgmt/glusterd/src/glusterd-mountbroker.c
+++ b/xlators/mgmt/glusterd/src/glusterd-mountbroker.c
@@ -7,687 +7,715 @@
    later), or the GNU General Public License, version 2 (GPLv2), in all
    cases as published by the Free Software Foundation.
 */
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
 #include <inttypes.h>
 #include <fnmatch.h>
 #include <pwd.h>
 
-#include "globals.h"
-#include "glusterfs.h"
-#include "compat.h"
-#include "dict.h"
-#include "list.h"
-#include "logging.h"
-#include "defaults.h"
-#include "compat.h"
-#include "compat-errno.h"
-#include "run.h"
+#include <glusterfs/globals.h>
+#include <glusterfs/glusterfs.h>
+#include <glusterfs/compat.h>
+#include <glusterfs/dict.h>
+#include <glusterfs/list.h>
+#include <glusterfs/logging.h>
+#include <glusterfs/syscall.h>
+#include <glusterfs/defaults.h>
+#include <glusterfs/compat.h>
+#include <glusterfs/compat-errno.h>
+#include <glusterfs/run.h>
 #include "glusterd-mem-types.h"
 #include "glusterd.h"
 #include "glusterd-utils.h"
-#include "common-utils.h"
+#include <glusterfs/common-utils.h>
 #include "glusterd-mountbroker.h"
 #include "glusterd-op-sm.h"
+#include "glusterd-messages.h"
 
 static int
-seq_dict_foreach (dict_t *dict,
-                  int (*fn)(char *str, void *data),
-                  void *data)
+seq_dict_foreach(dict_t *dict, int (*fn)(char *str, void *data), void *data)
 {
-        char index[] = "4294967296"; // 1<<32
-        int        i = 0;
-        char    *val = NULL;
-        int      ret = 0;
-
-        for (;;i++) {
-                snprintf(index, sizeof(index), "%d", i);
-                ret = dict_get_str (dict, index, &val);
-                if (ret != 0)
-                        return ret == -ENOENT ? 0 : ret;
-                ret = fn (val, data);
-                if (ret != 0)
-                        return ret;
-        }
+    char index[] = "4294967296";  // 1<<32
+    int i = 0;
+    char *val = NULL;
+    int ret = 0;
+
+    for (;; i++) {
+        snprintf(index, sizeof(index), "%d", i);
+        ret = dict_get_str(dict, index, &val);
+        if (ret != 0)
+            return ret == -ENOENT ? 0 : ret;
+        ret = fn(val, data);
+        if (ret != 0)
+            return ret;
+    }
 }
 
 int
-parse_mount_pattern_desc (gf_mount_spec_t *mspec, char *pdesc)
+parse_mount_pattern_desc(gf_mount_spec_t *mspec, char *pdesc)
 #define SYNTAX_ERR -2
 {
-        char *curs              = NULL;
-        char *c2                = NULL;
-        char sc                 = '\0';
-        char **cc               = NULL;
-        gf_mount_pattern_t *pat = NULL;
-        int pnum                = 0;
-        int ret                 = 0;
-        int lastsup             = -1;
-        int incl                = -1;
-        char **pcc              = NULL;
-        int pnc                 = 0;
-
-        skipwhite (&pdesc);
-
-        /* a bow to theory */
-        if (!*pdesc)
-                return 0;
-
-        /* count number of components, separated by '&' */
-        mspec->len = 0;
-        for (curs = pdesc; *curs; curs++) {
-                if (*curs == ')')
-                        mspec->len++;
+    char *curs = NULL;
+    char *c2 = NULL;
+    char sc = '\0';
+    char **cc = NULL;
+    gf_mount_pattern_t *pat = NULL;
+    int pnum = 0;
+    int ret = 0;
+    int lastsup = -1;
+    int incl = -1;
+    char **pcc = NULL;
+    int pnc = 0;
+
+    skipwhite(&pdesc);
+
+    /* a bow to theory */
+    if (!*pdesc)
+        return 0;
+
+    /* count number of components, separated by '&' */
+    mspec->len = 0;
+    for (curs = pdesc; *curs; curs++) {
+        if (*curs == ')')
+            mspec->len++;
+    }
+
+    mspec->patterns = GF_CALLOC(mspec->len, sizeof(*mspec->patterns),
+                                gf_gld_mt_mount_pattern);
+    if (!mspec->patterns) {
+        gf_smsg("glusterd", GF_LOG_ERROR, errno, GD_MSG_NO_MEMORY, NULL);
+        ret = -1;
+        goto out;
+    }
+
+    pat = mspec->patterns;
+    curs = pdesc;
+    skipwhite(&curs);
+    for (;;) {
+        incl = -1;
+
+        /* check for pattern signedness modifier */
+        if (*curs == '-') {
+            pat->negative = _gf_true;
+            curs++;
         }
 
-        mspec->patterns = GF_CALLOC (mspec->len, sizeof (*mspec->patterns),
-                                     gf_gld_mt_mount_pattern);
-        if (!mspec->patterns) {
-                ret = -1;
-                goto out;
+        /* now should come condition specifier,
+         * then opening paren
+         */
+        c2 = nwstrtail(curs, "SUB(");
+        if (c2) {
+            pat->condition = SET_SUB;
+            goto got_cond;
+        }
+        c2 = nwstrtail(curs, "SUP(");
+        if (c2) {
+            pat->condition = SET_SUPER;
+            lastsup = pat - mspec->patterns;
+            goto got_cond;
+        }
+        c2 = nwstrtail(curs, "EQL(");
+        if (c2) {
+            pat->condition = SET_EQUAL;
+            goto got_cond;
+        }
+        c2 = nwstrtail(curs, "MEET(");
+        if (c2) {
+            pat->condition = SET_INTERSECT;
+            goto got_cond;
+        }
+        c2 = nwstrtail(curs, "SUB+(");
+        if (c2) {
+            pat->condition = SET_SUB;
+            incl = lastsup;
+            goto got_cond;
         }
 
-        pat = mspec->patterns;
-        curs = pdesc;
-        skipwhite (&curs);
-        for (;;) {
-                incl = -1;
-
-                /* check for pattern signedness modifier */
-                if (*curs == '-') {
-                        pat->negative = _gf_true;
-                        curs++;
-                }
-
-                /* now should come condition specifier,
-                 * then opening paren
-                 */
-                c2 = nwstrtail (curs, "SUB(");
-                if (c2) {
-                        pat->condition = SET_SUB;
-                        goto got_cond;
-                }
-                c2 = nwstrtail (curs, "SUP(");
-                if (c2) {
-                        pat->condition = SET_SUPER;
-                        lastsup = pat - mspec->patterns;
-                        goto got_cond;
-                }
-                c2 = nwstrtail (curs, "EQL(");
-                if (c2) {
-                        pat->condition = SET_EQUAL;
-                        goto got_cond;
-                }
-                c2 = nwstrtail (curs, "MEET(");
-                if (c2) {
-                        pat->condition = SET_INTERSECT;
-                        goto got_cond;
-                }
-                c2 = nwstrtail (curs, "SUB+(");
-                if (c2) {
-                        pat->condition = SET_SUB;
-                        incl = lastsup;
-                        goto got_cond;
-                }
+        ret = SYNTAX_ERR;
+        goto out;
 
+    got_cond:
+        curs = c2;
+        skipwhite(&curs);
+        /* count the number of components for pattern */
+        pnum = *curs == ')' ? 0 : 1;
+        for (c2 = curs; *c2 != ')';) {
+            if (strchr("&|", *c2)) {
                 ret = SYNTAX_ERR;
                 goto out;
-
- got_cond:
-                curs = c2;
-                skipwhite (&curs);
-                /* count the number of components for pattern */
-                pnum = *curs == ')' ? 0 : 1;
-                for (c2 = curs ;*c2 != ')';) {
-                        if (strchr ("&|", *c2)) {
-                                ret = SYNTAX_ERR;
-                                goto out;
-                        }
-                        while (!strchr ("|&)", *c2) && !isspace (*c2))
-                                c2++;
-                        skipwhite (&c2);
-                        switch (*c2) {
-                        case ')':
-                                break;
-                        case '\0':
-                        case '&':
-                                ret = SYNTAX_ERR;
-                                goto out;
-                        case '|':
-                                *c2 = ' ';
-                                skipwhite (&c2);
-                                /* fall through */
-                        default:
-                                pnum++;
-                        }
-                }
-                if (incl >= 0) {
-                        pnc = 0;
-                        for (pcc = mspec->patterns[incl].components; *pcc; pcc++)
-                                pnc++;
-                        pnum += pnc;
-                }
-                pat->components = GF_CALLOC (pnum + 1, sizeof (*pat->components),
-                                             gf_gld_mt_mount_comp_container);
-                if (!pat->components) {
-                        ret = -1;
-                        goto out;
-                }
-
-                cc = pat->components;
-                /* copy over included component set */
-                if (incl >= 0) {
-                        memcpy (pat->components,
-                                mspec->patterns[incl].components,
-                                pnc * sizeof (*pat->components));
-                        cc += pnc;
-                }
-                /* parse and add components */
-                c2 = ""; /* reset c2 */
-                while (*c2 != ')') {
-                        c2 = curs;
-                        while (!isspace (*c2) && *c2 != ')')
-                                c2++;
-                        sc = *c2;
-                        *c2 = '\0';;
-                        *cc = gf_strdup (curs);
-                        if (!*cc) {
-                                ret = -1;
-                                goto out;
-                        }
-                        *c2 = sc;
-                        skipwhite (&c2);
-                        curs = c2;
-                        cc++;
-                }
-
-                curs++;
-                skipwhite (&curs);
-                if (*curs == '&') {
-                        curs++;
-                        skipwhite (&curs);
-                }
-
-                if (!*curs)
-                        break;
-                pat++;
+            }
+            while (!strchr("|&)", *c2) && !isspace(*c2))
+                c2++;
+            skipwhite(&c2);
+            switch (*c2) {
+                case ')':
+                    break;
+                case '\0':
+                case '&':
+                    ret = SYNTAX_ERR;
+                    goto out;
+                case '|':
+                    *c2 = ' ';
+                    skipwhite(&c2);
+                    /* fall through */
+                default:
+                    pnum++;
+            }
         }
-
- out:
-        if (ret == SYNTAX_ERR) {
-                gf_log ("", GF_LOG_ERROR, "cannot parse mount patterns %s",
-                        pdesc);
+        if (incl >= 0) {
+            pnc = 0;
+            for (pcc = mspec->patterns[incl].components; *pcc; pcc++)
+                pnc++;
+            pnum += pnc;
         }
-
-        /* We've allocted a lotta stuff here but don't bother with freeing
-         * on error, in that case we'll terminate anyway
-         */
-        return ret ? -1 : 0;
-}
-#undef SYNTAX_ERR
-
-
-const char *georep_mnt_desc_template =
-        "SUP("
-                "aux-gfid-mount "
-                "volfile-server=localhost "
-                "client-pid=%d "
-                "user-map-root=%s "
-        ")"
-        "SUB+("
-                "log-file="DEFAULT_LOG_FILE_DIRECTORY"/"GEOREP"*/* "
-                "log-level=* "
-                "volfile-id=* "
-        ")"
-        "MEET("
-                "%s"
-        ")";
-
-const char *hadoop_mnt_desc_template =
-        "SUP("
-                "volfile-server=%s "
-                "client-pid=%d "
-                "volfile-id=%s "
-                "user-map-root=%s "
-        ")"
-        "SUB+("
-                "log-file="DEFAULT_LOG_FILE_DIRECTORY"/"GHADOOP"*/* "
-                "log-level=* "
-        ")";
-
-int
-make_georep_mountspec (gf_mount_spec_t *mspec, const char *volnames,
-                       char *user)
-{
-        char *georep_mnt_desc = NULL;
-        char *meetspec        = NULL;
-        char *vols            = NULL;
-        char *vol             = NULL;
-        char *p               = NULL;
-        char *savetok         = NULL;
-        char *fa[3]           = {0,};
-        size_t siz            = 0;
-        int vc                = 0;
-        int i                 = 0;
-        int ret               = 0;
-
-        vols = gf_strdup ((char *)volnames);
-        if (!vols)
-                goto out;
-
-        for (vc = 1, p = vols; *p; p++) {
-                if (*p == ',')
-                        vc++;
+        pat->components = GF_CALLOC(pnum + 1, sizeof(*pat->components),
+                                    gf_gld_mt_mount_comp_container);
+        if (!pat->components) {
+            ret = -1;
+            goto out;
         }
-        siz = strlen (volnames) + vc * strlen("volfile-id=");
-        meetspec = GF_CALLOC (1, siz + 1, gf_gld_mt_georep_meet_spec);
-        if (!meetspec)
-                goto out;
 
-        for (p = vols;;) {
-                vol = strtok_r (p, ",", &savetok);
-                if (!vol) {
-                        GF_ASSERT (vc == 0);
-                        break;
-                }
-                p = NULL;
-                strcat (meetspec, "volfile-id=");
-                strcat (meetspec, vol);
-                if (--vc > 0)
-                        strcat (meetspec, " ");
+        cc = pat->components;
+        /* copy over included component set */
+        if (incl >= 0) {
+            memcpy(pat->components, mspec->patterns[incl].components,
+                   pnc * sizeof(*pat->components));
+            cc += pnc;
         }
-
-        ret = gf_asprintf (&georep_mnt_desc, georep_mnt_desc_template,
-                           GF_CLIENT_PID_GSYNCD, user, meetspec);
-        if (ret == -1) {
-                georep_mnt_desc = NULL;
+        /* parse and add components */
+        c2 = ""; /* reset c2 */
+        while (*c2 != ')') {
+            c2 = curs;
+            while (!isspace(*c2) && *c2 != ')')
+                c2++;
+            sc = *c2;
+            *c2 = '\0';
+            ;
+            *cc = gf_strdup(curs);
+            if (!*cc) {
+                ret = -1;
                 goto out;
+            }
+            *c2 = sc;
+            skipwhite(&c2);
+            curs = c2;
+            cc++;
         }
 
-        ret = parse_mount_pattern_desc (mspec, georep_mnt_desc);
-
- out:
-        fa[0] = meetspec;
-        fa[1] = vols;
-        fa[2] = georep_mnt_desc;
-
-        for (i = 0; i < 3; i++) {
-                if (fa[i] == NULL)
-                        ret = -1;
-                else
-                        GF_FREE (fa[i]);
+        curs++;
+        skipwhite(&curs);
+        if (*curs == '&') {
+            curs++;
+            skipwhite(&curs);
         }
 
-        return ret;
+        if (!*curs)
+            break;
+        pat++;
+    }
+
+out:
+    if (ret == SYNTAX_ERR) {
+        gf_msg("glusterd", GF_LOG_ERROR, EINVAL, GD_MSG_INVALID_ENTRY,
+               "cannot parse mount patterns %s", pdesc);
+    }
+
+    /* We've allocted a lotta stuff here but don't bother with freeing
+     * on error, in that case we'll terminate anyway
+     */
+    return ret ? -1 : 0;
 }
+#undef SYNTAX_ERR
+
+const char *georep_mnt_desc_template =
+    "SUP("
+    "aux-gfid-mount "
+    "acl "
+    "volfile-server=localhost "
+    "client-pid=%d "
+    "user-map-root=%s "
+    ")"
+    "SUB+("
+    "log-file=%s/" GEOREP
+    "*/* "
+    "log-level=* "
+    "volfile-id=* "
+    ")"
+    "MEET("
+    "%s"
+    ")";
 
 int
-make_ghadoop_mountspec (gf_mount_spec_t *mspec, const char *volname,
-                        char *user, char *server)
+make_georep_mountspec(gf_mount_spec_t *mspec, const char *volnames, char *user,
+                      char *logdir)
 {
-        char *hadoop_mnt_desc = NULL;
-        int   ret             = 0;
-
-        ret = gf_asprintf (&hadoop_mnt_desc, hadoop_mnt_desc_template,
-                           server, GF_CLIENT_PID_HADOOP, volname, user);
-        if (ret == -1)
-                return ret;
-
-        return parse_mount_pattern_desc (mspec, hadoop_mnt_desc);
+    char *georep_mnt_desc = NULL;
+    char *meetspec = NULL;
+    char *vols = NULL;
+    char *vol = NULL;
+    char *p = NULL;
+    char *savetok = NULL;
+    char *fa[3] = {
+        0,
+    };
+    size_t siz = 0;
+    int vc = 0;
+    int i = 0;
+    int ret = 0;
+
+    vols = gf_strdup((char *)volnames);
+    if (!vols) {
+        gf_smsg(THIS->name, GF_LOG_ERROR, errno, GD_MSG_STRDUP_FAILED,
+                "Volume name=%s", volnames, NULL);
+        goto out;
+    }
+
+    for (vc = 1, p = vols; *p; p++) {
+        if (*p == ',')
+            vc++;
+    }
+    siz = strlen(volnames) + vc * SLEN("volfile-id=");
+    meetspec = GF_CALLOC(1, siz + 1, gf_gld_mt_georep_meet_spec);
+    if (!meetspec) {
+        gf_smsg(THIS->name, GF_LOG_ERROR, errno, GD_MSG_NO_MEMORY, NULL);
+        goto out;
+    }
+
+    for (p = vols;;) {
+        vol = strtok_r(p, ",", &savetok);
+        if (!vol) {
+            GF_ASSERT(vc == 0);
+            break;
+        }
+        p = NULL;
+        strcat(meetspec, "volfile-id=");
+        strcat(meetspec, vol);
+        if (--vc > 0)
+            strcat(meetspec, " ");
+    }
+
+    ret = gf_asprintf(&georep_mnt_desc, georep_mnt_desc_template,
+                      GF_CLIENT_PID_GSYNCD, user, logdir, meetspec);
+    if (ret == -1) {
+        georep_mnt_desc = NULL;
+        goto out;
+    }
+
+    ret = parse_mount_pattern_desc(mspec, georep_mnt_desc);
+
+out:
+    fa[0] = meetspec;
+    fa[1] = vols;
+    fa[2] = georep_mnt_desc;
+
+    for (i = 0; i < 3; i++) {
+        if (fa[i] == NULL)
+            ret = -1;
+        else
+            GF_FREE(fa[i]);
+    }
+
+    return ret;
 }
 
 static gf_boolean_t
-match_comp (char *str, char *patcomp)
+match_comp(char *str, char *patcomp)
 {
-        char *c1 = patcomp;
-        char *c2 = str;
-
-        GF_ASSERT (c1);
-        GF_ASSERT (c2);
-
-        while (*c1 == *c2) {
-                if (!*c1)
-                        return _gf_true;
-                c1++;
-                c2++;
-                if (c1[-1] == '=')
-                        break;
-        }
-
-        return fnmatch (c1, c2, 0) == 0 ? _gf_true : _gf_false;
+    char *c1 = patcomp;
+    char *c2 = str;
+
+    GF_ASSERT(c1);
+    GF_ASSERT(c2);
+
+    while (*c1 == *c2) {
+        if (!*c1)
+            return _gf_true;
+        c1++;
+        c2++;
+        if (c1[-1] == '=')
+            break;
+    }
+
+    return fnmatch(c1, c2, 0) == 0 ? _gf_true : _gf_false;
 }
 
 struct gf_set_descriptor {
-        gf_boolean_t priv[2];
-        gf_boolean_t common;
+    gf_boolean_t priv[2];
+    gf_boolean_t common;
 };
 
 static int
-_gf_set_dict_iter1 (char *val, void *data)
+_gf_set_dict_iter1(char *val, void *data)
 {
-        void **dataa                 = data;
-        struct gf_set_descriptor *sd = dataa[0];
-        char **curs                  = dataa[1];
-        gf_boolean_t priv            = _gf_true;
-
-        while (*curs) {
-                if (match_comp (val, *curs)) {
-                        priv = _gf_false;
-                        sd->common = _gf_true;
-                }
-                curs++;
+    void **dataa = data;
+    struct gf_set_descriptor *sd = dataa[0];
+    char **curs = dataa[1];
+    gf_boolean_t priv = _gf_true;
+
+    while (*curs) {
+        if (match_comp(val, *curs)) {
+            priv = _gf_false;
+            sd->common = _gf_true;
         }
+        curs++;
+    }
 
-        if (priv)
-                sd->priv[0] = _gf_true;
+    if (priv)
+        sd->priv[0] = _gf_true;
 
-        return 0;
+    return 0;
 }
 
 static int
-_gf_set_dict_iter2 (char *val, void *data)
+_gf_set_dict_iter2(char *val, void *data)
 {
-        void **dataa      = data;
-        gf_boolean_t *boo = dataa[0];
-        char *comp        = dataa[1];
+    void **dataa = data;
+    gf_boolean_t *boo = dataa[0];
+    char *comp = dataa[1];
 
-        if (match_comp (val, comp))
-                *boo = _gf_true;
+    if (match_comp(val, comp))
+        *boo = _gf_true;
 
-        return 0;
+    return 0;
 }
 
 static void
-relate_sets (struct gf_set_descriptor *sd, dict_t *argdict, char **complist)
+relate_sets(struct gf_set_descriptor *sd, dict_t *argdict, char **complist)
 {
-        void *dataa[] = {NULL, NULL};
-        gf_boolean_t boo = _gf_false;
+    void *dataa[] = {NULL, NULL};
+    gf_boolean_t boo = _gf_false;
 
-        memset (sd, 0, sizeof (*sd));
+    memset(sd, 0, sizeof(*sd));
 
-        dataa[0] = sd;
-        dataa[1] = complist;
-        seq_dict_foreach (argdict, _gf_set_dict_iter1, dataa);
+    dataa[0] = sd;
+    dataa[1] = complist;
+    seq_dict_foreach(argdict, _gf_set_dict_iter1, dataa);
 
-        while (*complist) {
-                boo = _gf_false;
-                dataa[0] = &boo;
-                dataa[1] = *complist;
-                seq_dict_foreach (argdict, _gf_set_dict_iter2, dataa);
+    while (*complist) {
+        boo = _gf_false;
+        dataa[0] = &boo;
+        dataa[1] = *complist;
+        seq_dict_foreach(argdict, _gf_set_dict_iter2, dataa);
 
-                if (boo)
-                        sd->common = _gf_true;
-                else
-                        sd->priv[1] = _gf_true;
+        if (boo)
+            sd->common = _gf_true;
+        else
+            sd->priv[1] = _gf_true;
 
-                complist++;
-        }
+        complist++;
+    }
 }
 
 static int
-_arg_parse_uid (char *val, void *data)
+_arg_parse_uid(char *val, void *data)
 {
-        char *user        = strtail (val, "user-map-root=");
-        struct passwd *pw = NULL;
+    char *user = strtail(val, "user-map-root=");
+    struct passwd *pw = NULL;
 
-        if (!user)
-                return 0;
-        pw = getpwnam (user);
-        if (!pw)
-                return -EINVAL;
+    if (!user)
+        return 0;
+    pw = getpwnam(user);
+    if (!pw)
+        return -EINVAL;
 
-        if (*(int *)data >= 0)
-                /* uid ambiguity, already found */
-                return -EINVAL;
+    if (*(int *)data >= 0)
+        /* uid ambiguity, already found */
+        return -EINVAL;
 
-        *(int *)data = pw->pw_uid;
-        return 0;
+    *(int *)data = pw->pw_uid;
+    return 0;
 }
 
 static int
-evaluate_mount_request (gf_mount_spec_t *mspec, dict_t *argdict)
+evaluate_mount_request(xlator_t *this, gf_mount_spec_t *mspec, dict_t *argdict)
 {
-        struct gf_set_descriptor sd = {{0,},};
-        int i                       = 0;
-        int uid                     = -1;
-        int ret                     = 0;
-        gf_boolean_t match          = _gf_false;
-
-        for (i = 0; i < mspec->len; i++) {
-                relate_sets (&sd, argdict, mspec->patterns[i].components);
-                switch (mspec->patterns[i].condition) {
-                case SET_SUB:
-                        match = !sd.priv[0];
-                        break;
-                case SET_SUPER:
-                        match = !sd.priv[1];
-                        break;
-                case SET_EQUAL:
-                        match = (!sd.priv[0] && !sd.priv[1]);
-                        break;
-                case SET_INTERSECT:
-                        match = sd.common;
-                        break;
-                default:
-                        GF_ASSERT(!"unreached");
-                }
-                if (mspec->patterns[i].negative)
-                        match = !match;
-
-                if (!match)
-                        return -EPERM;
+    struct gf_set_descriptor sd = {
+        {
+            0,
+        },
+    };
+    int i = 0;
+    int uid = -1;
+    int ret = 0;
+    gf_boolean_t match = _gf_false;
+
+    for (i = 0; i < mspec->len; i++) {
+        relate_sets(&sd, argdict, mspec->patterns[i].components);
+        switch (mspec->patterns[i].condition) {
+            case SET_SUB:
+                match = !sd.priv[0];
+                break;
+            case SET_SUPER:
+                match = !sd.priv[1];
+                break;
+            case SET_EQUAL:
+                match = (!sd.priv[0] && !sd.priv[1]);
+                break;
+            case SET_INTERSECT:
+                match = sd.common;
+                break;
+            default:
+                GF_ASSERT(!"unreached");
         }
+        if (mspec->patterns[i].negative)
+            match = !match;
+
+        if (!match) {
+            gf_msg(this->name, GF_LOG_ERROR, EPERM,
+                   GD_MSG_MNTBROKER_SPEC_MISMATCH,
+                   "Mountbroker spec mismatch!!! SET: %d "
+                   "COMPONENT: %d. Review the mount args passed",
+                   mspec->patterns[i].condition, i);
+            return -EPERM;
+        }
+    }
 
-        ret = seq_dict_foreach (argdict, _arg_parse_uid, &uid);
-        if (ret != 0)
-                return ret;
+    ret = seq_dict_foreach(argdict, _arg_parse_uid, &uid);
+    if (ret != 0)
+        return ret;
 
-        return uid;
+    return uid;
 }
 
 static int
-_volname_get (char *val, void *data)
+_volname_get(char *val, void *data)
 {
-        char **volname = data;
+    char **volname = data;
 
-        *volname = strtail (val, "volfile-id=");
+    *volname = strtail(val, "volfile-id=");
 
-        return *volname ? 1 : 0;
+    return *volname ? 1 : 0;
 }
 
 static int
-_runner_add (char *val, void *data)
+_runner_add(char *val, void *data)
 {
-        runner_t *runner = data;
+    runner_t *runner = data;
 
-        runner_argprintf (runner, "--%s", val);
+    runner_argprintf(runner, "--%s", val);
 
-        return 0;
+    return 0;
 }
 
 int
-glusterd_do_mount (char *label, dict_t *argdict, char **path, int *op_errno)
+glusterd_do_mount(char *label, dict_t *argdict, char **path, int *op_errno)
 {
-        glusterd_conf_t *priv      = NULL;
-        char *mountbroker_root     = NULL;
-        gf_mount_spec_t *mspec     = NULL;
-        int uid                    = -ENOENT;
-        char *volname              = NULL;
-        glusterd_volinfo_t *vol    = NULL;
-        char *mtptemp              = NULL;
-        char *mntlink              = NULL;
-        char *cookieswitch         = NULL;
-        char *cookie               = NULL;
-        char *sla                  = NULL;
-        struct stat st             = {0,};
-        runner_t runner            = {0,};
-        int ret                    = 0;
-        xlator_t *this             = THIS;
-
-        priv = this->private;
-        GF_ASSERT (priv);
-
-        GF_ASSERT (op_errno);
-        *op_errno = 0;
-
-        if (dict_get_str (this->options, "mountbroker-root",
-                          &mountbroker_root) != 0) {
-                *op_errno = ENOENT;
-                goto out;
-        }
-
-        GF_ASSERT (label);
-        if (!*label) {
-                *op_errno = EINVAL;
-                goto out;
-        }
-
-        /* look up spec for label */
-        cds_list_for_each_entry (mspec, &priv->mount_specs,
-                                 speclist) {
-                if (strcmp (mspec->label, label) != 0)
-                        continue;
-                uid = evaluate_mount_request (mspec, argdict);
-                break;
-        }
-        if (uid < 0) {
-                *op_errno = -uid;
-                goto out;
-        }
-
-        /* some sanity check on arguments */
-        seq_dict_foreach (argdict, _volname_get, &volname);
-        if (!volname) {
-                *op_errno = EINVAL;
-                goto out;
-        }
-        if (glusterd_volinfo_find (volname, &vol) != 0 ||
-            !glusterd_is_volume_started (vol)) {
-                *op_errno = ENOENT;
-                goto out;
-        }
-
-        /* go do mount */
-
-        /** create actual mount dir */
-
-        /*** "overload" string name to be possible to used for cookie
-             creation, see below */
-        ret = gf_asprintf (&mtptemp, "%s/user%d/mtpt-%s-XXXXXX/cookie",
-                           mountbroker_root, uid, label);
-        if (ret == -1) {
-                mtptemp = NULL;
-                *op_errno = ENOMEM;
-                goto out;
-        }
-        /*** hide cookie part */
-        cookieswitch = strrchr (mtptemp, '/');
-        *cookieswitch = '\0';
-
-        sla = strrchr (mtptemp, '/');
-        *sla = '\0';
-        ret = mkdir (mtptemp, 0700);
-        if (ret == 0)
-                ret = chown (mtptemp, uid, 0);
-        else if (errno == EEXIST)
-                ret = 0;
-        if (ret == -1) {
-                *op_errno = errno;
-                goto out;
-        }
-        ret = lstat (mtptemp, &st);
-        if (ret == -1) {
-                *op_errno = errno;
-                goto out;
-        }
-        if (!(S_ISDIR (st.st_mode) && (st.st_mode & ~S_IFMT) == 0700 &&
-              st.st_uid == uid && st.st_gid == 0)) {
-                *op_errno = EACCES;
-                goto out;
-        }
-        *sla = '/';
-
-        if (!mkdtemp (mtptemp)) {
-                *op_errno = errno;
-                goto out;
-        }
-
-        /** create private "cookie" symlink */
-
-        /*** occupy an entry in the hive dir via mkstemp */
-        ret = gf_asprintf (&cookie, "%s/"MB_HIVE"/mntXXXXXX",
-                           mountbroker_root);
-        if (ret == -1) {
-                cookie = NULL;
-                *op_errno = ENOMEM;
-                goto out;
+    glusterd_conf_t *priv = NULL;
+    char *mountbroker_root = NULL;
+    gf_mount_spec_t *mspec = NULL;
+    int uid = -ENOENT;
+    char *volname = NULL;
+    glusterd_volinfo_t *vol = NULL;
+    char *mtptemp = NULL;
+    char *mntlink = NULL;
+    char *cookieswitch = NULL;
+    char *cookie = NULL;
+    char *sla = NULL;
+    struct stat st = {
+        0,
+    };
+    runner_t runner = {
+        0,
+    };
+    int ret = 0;
+    xlator_t *this = THIS;
+    mode_t orig_umask = 0;
+    gf_boolean_t found_label = _gf_false;
+
+    priv = this->private;
+    GF_ASSERT(priv);
+
+    GF_ASSERT(op_errno);
+    *op_errno = 0;
+
+    if (dict_get_strn(this->options, "mountbroker-root",
+                      SLEN("mountbroker-root"), &mountbroker_root) != 0) {
+        *op_errno = ENOENT;
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+               "'option mountbroker-root' "
+               "missing in glusterd vol file");
+        goto out;
+    }
+
+    GF_ASSERT(label);
+    if (!*label) {
+        *op_errno = EINVAL;
+        gf_msg(this->name, GF_LOG_ERROR, *op_errno, GD_MSG_MNTBROKER_LABEL_NULL,
+               "label is NULL (%s)", strerror(*op_errno));
+        goto out;
+    }
+
+    /* look up spec for label */
+    cds_list_for_each_entry(mspec, &priv->mount_specs, speclist)
+    {
+        if (strcmp(mspec->label, label) != 0)
+            continue;
+
+        found_label = _gf_true;
+        uid = evaluate_mount_request(this, mspec, argdict);
+        break;
+    }
+    if (uid < 0) {
+        *op_errno = -uid;
+        if (!found_label) {
+            gf_msg(this->name, GF_LOG_ERROR, *op_errno,
+                   GD_MSG_MNTBROKER_LABEL_MISS,
+                   "Missing mspec: Check the corresponding option "
+                   "in glusterd vol file for mountbroker user: %s",
+                   label);
         }
-        ret = mkstemp (cookie);
-        if (ret == -1) {
-                *op_errno = errno;
-                goto out;
+        goto out;
+    }
+
+    /* some sanity check on arguments */
+    seq_dict_foreach(argdict, _volname_get, &volname);
+    if (!volname) {
+        *op_errno = EINVAL;
+        gf_msg(this->name, GF_LOG_ERROR, EINVAL, GD_MSG_DICT_GET_FAILED,
+               "Dict get failed for the key 'volname'");
+        goto out;
+    }
+    if (glusterd_volinfo_find(volname, &vol) != 0 ||
+        !glusterd_is_volume_started(vol)) {
+        *op_errno = ENOENT;
+        gf_msg(this->name, GF_LOG_ERROR, *op_errno, GD_MSG_MOUNT_REQ_FAIL,
+               "Either volume is not started or volinfo not found");
+        goto out;
+    }
+
+    /* go do mount */
+
+    /** create actual mount dir */
+
+    /*** "overload" string name to be possible to used for cookie
+         creation, see below */
+    ret = gf_asprintf(&mtptemp, "%s/user%d/mtpt-%s-XXXXXX/cookie",
+                      mountbroker_root, uid, label);
+    if (ret == -1) {
+        mtptemp = NULL;
+        *op_errno = ENOMEM;
+        goto out;
+    }
+    /*** hide cookie part */
+    cookieswitch = strrchr(mtptemp, '/');
+    *cookieswitch = '\0';
+
+    sla = strrchr(mtptemp, '/');
+    *sla = '\0';
+    ret = sys_mkdir(mtptemp, 0700);
+    if (ret == 0)
+        ret = sys_chown(mtptemp, uid, 0);
+    else if (errno == EEXIST)
+        ret = 0;
+    if (ret == -1) {
+        *op_errno = errno;
+        gf_msg(this->name, GF_LOG_ERROR, *op_errno, GD_MSG_SYSCALL_FAIL,
+               "Mountbroker User directory creation failed");
+        goto out;
+    }
+    ret = sys_lstat(mtptemp, &st);
+    if (ret == -1) {
+        *op_errno = errno;
+        gf_msg(this->name, GF_LOG_ERROR, *op_errno, GD_MSG_SYSCALL_FAIL,
+               "stat on mountbroker user directory failed");
+        goto out;
+    }
+    if (!(S_ISDIR(st.st_mode) && (st.st_mode & ~S_IFMT) == 0700 &&
+          st.st_uid == uid && st.st_gid == 0)) {
+        *op_errno = EACCES;
+        gf_msg(this->name, GF_LOG_ERROR, *op_errno, GD_MSG_MOUNT_REQ_FAIL,
+               "Incorrect mountbroker user directory attributes");
+        goto out;
+    }
+    *sla = '/';
+
+    if (!mkdtemp(mtptemp)) {
+        *op_errno = errno;
+        gf_msg(this->name, GF_LOG_ERROR, *op_errno, GD_MSG_SYSCALL_FAIL,
+               "Mountbroker mount directory creation failed");
+        goto out;
+    }
+
+    /** create private "cookie" symlink */
+
+    /*** occupy an entry in the hive dir via mkstemp */
+    ret = gf_asprintf(&cookie, "%s/" MB_HIVE "/mntXXXXXX", mountbroker_root);
+    if (ret == -1) {
+        cookie = NULL;
+        *op_errno = ENOMEM;
+        goto out;
+    }
+    orig_umask = umask(S_IRWXG | S_IRWXO);
+    ret = mkstemp(cookie);
+    umask(orig_umask);
+    if (ret == -1) {
+        *op_errno = errno;
+        gf_msg(this->name, GF_LOG_ERROR, *op_errno, GD_MSG_SYSCALL_FAIL,
+               "Mountbroker cookie file creation failed");
+        goto out;
+    }
+    sys_close(ret);
+
+    /*** assembly the path from cookie to mountpoint */
+    sla = strchr(sla - 1, '/');
+    GF_ASSERT(sla);
+    ret = gf_asprintf(&mntlink, "../user%d%s", uid, sla);
+    if (ret == -1) {
+        *op_errno = ENOMEM;
+        goto out;
+    }
+
+    /*** create cookie link in (to-be) mountpoint,
+         move it over to the final place */
+    *cookieswitch = '/';
+    ret = sys_symlink(mntlink, mtptemp);
+    if (ret != -1)
+        ret = sys_rename(mtptemp, cookie);
+    *cookieswitch = '\0';
+    if (ret == -1) {
+        *op_errno = errno;
+        gf_msg(this->name, GF_LOG_ERROR, *op_errno, GD_MSG_SYSCALL_FAIL,
+               "symlink or rename failed");
+        goto out;
+    }
+
+    /** invoke glusterfs on the mountpoint */
+
+    runinit(&runner);
+    runner_add_arg(&runner, SBIN_DIR "/glusterfs");
+    seq_dict_foreach(argdict, _runner_add, &runner);
+    runner_add_arg(&runner, mtptemp);
+    ret = runner_run_reuse(&runner);
+    if (ret == -1) {
+        *op_errno = EIO; /* XXX hacky fake */
+        runner_log(&runner, "", GF_LOG_ERROR, "command failed");
+    }
+    runner_end(&runner);
+
+out:
+
+    if (*op_errno) {
+        ret = -1;
+        gf_msg(this->name, GF_LOG_WARNING, *op_errno, GD_MSG_MOUNT_REQ_FAIL,
+               "unsuccessful mount request");
+        if (mtptemp) {
+            *cookieswitch = '/';
+            sys_unlink(mtptemp);
+            *cookieswitch = '\0';
+            sys_rmdir(mtptemp);
         }
-        close (ret);
-
-        /*** assembly the path from cookie to mountpoint */
-        sla = strchr (sla - 1, '/');
-        GF_ASSERT (sla);
-        ret = gf_asprintf (&mntlink, "../user%d%s", uid, sla);
-        if (ret == -1) {
-                *op_errno = ENOMEM;
-                goto out;
+        if (cookie) {
+            sys_unlink(cookie);
+            GF_FREE(cookie);
         }
 
-        /*** create cookie link in (to-be) mountpoint,
-             move it over to the final place */
-        *cookieswitch = '/';
-        ret = symlink (mntlink, mtptemp);
-        if (ret != -1)
-                ret = rename (mtptemp, cookie);
-        *cookieswitch = '\0';
-        if (ret == -1) {
-                *op_errno = errno;
-                goto out;
-        }
-
-        /** invoke glusterfs on the mountpoint */
+    } else {
+        ret = 0;
+        *path = cookie;
+    }
 
-        runinit (&runner);
-        runner_add_arg (&runner, SBIN_DIR"/glusterfs");
-        seq_dict_foreach (argdict, _runner_add, &runner);
-        runner_add_arg (&runner, mtptemp);
-        ret = runner_run_reuse (&runner);
-        if (ret == -1) {
-                *op_errno = EIO; /* XXX hacky fake */
-                runner_log (&runner, "", GF_LOG_ERROR, "command failed");
-        }
-        runner_end (&runner);
+    if (mtptemp)
+        GF_FREE(mtptemp);
+    if (mntlink)
+        GF_FREE(mntlink);
 
- out:
-
-        if (*op_errno) {
-                ret = -1;
-                gf_log ("", GF_LOG_WARNING, "unsuccessful mount request (%s)",
-                        strerror (*op_errno));
-                if (mtptemp) {
-                        *cookieswitch = '/';
-                        unlink (mtptemp);
-                        *cookieswitch = '\0';
-                        rmdir (mtptemp);
-                }
-                if (cookie) {
-                        unlink (cookie);
-                        GF_FREE (cookie);
-                }
-
-        } else {
-                ret = 0;
-                *path = cookie;
-        }
-
-        GF_FREE (mtptemp);
-
-        return ret;
+    return ret;
 }
diff --git a/xlators/mgmt/glusterd/src/glusterd-mountbroker.h b/xlators/mgmt/glusterd/src/glusterd-mountbroker.h
index 83267c203ca..20c1347f52f 100644
--- a/xlators/mgmt/glusterd/src/glusterd-mountbroker.h
+++ b/xlators/mgmt/glusterd/src/glusterd-mountbroker.h
@@ -9,34 +9,29 @@
 */
 #define MB_HIVE "mb_hive"
 
-typedef enum {
-        SET_SUB = 1,
-        SET_SUPER,
-        SET_EQUAL,
-        SET_INTERSECT
-} gf_setrel_t;
+typedef enum { SET_SUB = 1, SET_SUPER, SET_EQUAL, SET_INTERSECT } gf_setrel_t;
 
 struct gf_mount_pattern {
-        char **components;
-        gf_setrel_t condition;
-        gf_boolean_t negative;
+    char **components;
+    gf_setrel_t condition;
+    gf_boolean_t negative;
 };
 typedef struct gf_mount_pattern gf_mount_pattern_t;
 
 struct gf_mount_spec {
-        struct cds_list_head speclist;
-        char *label;
-        gf_mount_pattern_t *patterns;
-        size_t len;
+    struct cds_list_head speclist;
+    char *label;
+    gf_mount_pattern_t *patterns;
+    size_t len;
 };
 typedef struct gf_mount_spec gf_mount_spec_t;
 
+int
+parse_mount_pattern_desc(gf_mount_spec_t *mspec, char *pdesc);
 
-int parse_mount_pattern_desc (gf_mount_spec_t *mspec, char *pdesc);
+int
+make_georep_mountspec(gf_mount_spec_t *mspec, const char *volname, char *user,
+                      char *logdir);
 
-int make_georep_mountspec (gf_mount_spec_t *mspec, const char *volname,
-                           char *user);
-int make_ghadoop_mountspec (gf_mount_spec_t *mspec, const char *volname,
-                            char *user, char *server);
-
-int glusterd_do_mount (char *label, dict_t *argdict, char **path, int *op_errno);
+int
+glusterd_do_mount(char *label, dict_t *argdict, char **path, int *op_errno);
diff --git a/xlators/mgmt/glusterd/src/glusterd-nfs-svc.c b/xlators/mgmt/glusterd/src/glusterd-nfs-svc.c
index 49b1b561701..4908dbbc213 100644
--- a/xlators/mgmt/glusterd/src/glusterd-nfs-svc.c
+++ b/xlators/mgmt/glusterd/src/glusterd-nfs-svc.c
@@ -8,275 +8,221 @@
    cases as published by the Free Software Foundation.
 */
 
-#include "globals.h"
-#include "run.h"
+#ifdef BUILD_GNFS
+
+#include <glusterfs/globals.h>
+#include <glusterfs/run.h>
+#include <glusterfs/syscall.h>
 #include "glusterd.h"
 #include "glusterd-utils.h"
 #include "glusterd-volgen.h"
 #include "glusterd-nfs-svc.h"
-
-char *nfs_svc_name = "nfs";
+#include "glusterd-messages.h"
+#include "glusterd-svc-helper.h"
 
 static gf_boolean_t
-glusterd_nfssvc_need_start ()
+glusterd_nfssvc_need_start()
 {
-        glusterd_conf_t    *priv    = NULL;
-        gf_boolean_t       start   = _gf_false;
-        glusterd_volinfo_t *volinfo = NULL;
+    glusterd_conf_t *priv = NULL;
+    gf_boolean_t start = _gf_false;
+    glusterd_volinfo_t *volinfo = NULL;
 
-        priv = THIS->private;
+    priv = THIS->private;
 
-        cds_list_for_each_entry (volinfo, &priv->volumes, vol_list) {
-                if (!glusterd_is_volume_started (volinfo))
-                        continue;
+    cds_list_for_each_entry(volinfo, &priv->volumes, vol_list)
+    {
+        if (!glusterd_is_volume_started(volinfo))
+            continue;
 
-                if (dict_get_str_boolean (volinfo->dict, "nfs.disable", 0))
-                        continue;
-                start = _gf_true;
-                break;
-        }
+        if (dict_get_str_boolean(volinfo->dict, NFS_DISABLE_MAP_KEY, 1))
+            continue;
+        start = _gf_true;
+        break;
+    }
 
-        return start;
-}
-
-int
-glusterd_nfssvc_init (glusterd_svc_t *svc)
-{
-        return glusterd_svc_init (svc, nfs_svc_name,
-                                  glusterd_nfssvc_manager,
-                                  glusterd_nfssvc_start,
-                                  glusterd_nfssvc_stop);
+    return start;
 }
 
 static int
-glusterd_nfssvc_create_volfile ()
+glusterd_nfssvc_create_volfile()
 {
-        char            filepath[PATH_MAX] = {0,};
-        glusterd_conf_t *conf = THIS->private;
-
-        glusterd_svc_build_volfile_path (nfs_svc_name, conf->workdir,
-                                         filepath, sizeof (filepath));
-        return glusterd_create_global_volfile (build_nfs_graph,
-                                               filepath, NULL);
+    char filepath[PATH_MAX] = {
+        0,
+    };
+    glusterd_conf_t *conf = THIS->private;
+
+    glusterd_svc_build_volfile_path(conf->nfs_svc.name, conf->workdir, filepath,
+                                    sizeof(filepath));
+    return glusterd_create_global_volfile(build_nfs_graph, filepath, NULL);
 }
 
 static int
-glusterd_nfssvc_check_volfile_identical (gf_boolean_t *identical)
+glusterd_nfssvc_manager(glusterd_svc_t *svc, void *data, int flags)
 {
-        char            nfsvol[PATH_MAX]        = {0,};
-        char            tmpnfsvol[PATH_MAX]     = {0,};
-        glusterd_conf_t *conf                   = NULL;
-        xlator_t        *this                   = NULL;
-        int             ret                     = -1;
-        int             need_unlink             = 0;
-        int             tmp_fd                  = -1;
-
-        this = THIS;
-
-        GF_ASSERT (this);
-        GF_ASSERT (identical);
-        conf = this->private;
-
-        glusterd_svc_build_volfile_path (nfs_svc_name, conf->workdir,
-                                         nfsvol, sizeof (nfsvol));
-
-        snprintf (tmpnfsvol, sizeof (tmpnfsvol), "/tmp/gnfs-XXXXXX");
-
-        tmp_fd = mkstemp (tmpnfsvol);
-        if (tmp_fd < 0) {
-                gf_log (this->name, GF_LOG_WARNING, "Unable to create temp file"
-                        " %s:(%s)", tmpnfsvol, strerror (errno));
-                goto out;
+    int ret = -1;
+
+    if (!svc->inited) {
+        ret = glusterd_svc_init(svc, "nfs");
+        if (ret) {
+            gf_msg(THIS->name, GF_LOG_ERROR, 0, GD_MSG_FAILED_INIT_NFSSVC,
+                   "Failed to init nfs service");
+            goto out;
+        } else {
+            svc->inited = _gf_true;
+            gf_msg_debug(THIS->name, 0, "nfs service initialized");
         }
+    }
 
-        need_unlink = 1;
+    ret = svc->stop(svc, SIGKILL);
+    if (ret)
+        goto out;
 
-        ret = glusterd_create_global_volfile (build_nfs_graph,
-                                              tmpnfsvol, NULL);
-        if (ret)
-                goto out;
+    /* not an error, or a (very) soft error at best */
+    if (sys_access(XLATORDIR "/nfs/server.so", R_OK) != 0) {
+        gf_msg(THIS->name, GF_LOG_INFO, 0, GD_MSG_GNFS_XLATOR_NOT_INSTALLED,
+               "nfs/server.so xlator is not installed");
+        goto out;
+    }
+
+    ret = glusterd_nfssvc_create_volfile();
+    if (ret)
+        goto out;
 
-        ret = glusterd_check_files_identical (nfsvol, tmpnfsvol,
-                                              identical);
+    if (glusterd_nfssvc_need_start()) {
+        ret = svc->start(svc, flags);
         if (ret)
-                goto out;
+            goto out;
 
+        ret = glusterd_conn_connect(&(svc->conn));
+        if (ret)
+            goto out;
+    }
 out:
-        if (need_unlink)
-                unlink (tmpnfsvol);
+    if (ret)
+        gf_event(EVENT_SVC_MANAGER_FAILED, "svc_name=%s", svc->name);
 
-        if (tmp_fd >= 0)
-                close (tmp_fd);
+    gf_msg_debug(THIS->name, 0, "Returning %d", ret);
 
-        return ret;
+    return ret;
 }
 
 static int
-glusterd_nfssvc_check_topology_identical (gf_boolean_t *identical)
+glusterd_nfssvc_start(glusterd_svc_t *svc, int flags)
 {
-        char            nfsvol[PATH_MAX]        = {0,};
-        char            tmpnfsvol[PATH_MAX]     = {0,};
-        glusterd_conf_t *conf                   = NULL;
-        xlator_t        *this                   = THIS;
-        int             ret                     = -1;
-        int             tmpclean                = 0;
-        int             tmpfd                   = -1;
-
-        if ((!identical) || (!this) || (!this->private))
-                goto out;
-
-        conf = (glusterd_conf_t *) this->private;
-        GF_ASSERT (conf);
-
-        /* Fetch the original NFS volfile */
-        glusterd_svc_build_volfile_path (conf->nfs_svc.name, conf->workdir,
-                                         nfsvol, sizeof (nfsvol));
-
-        /* Create the temporary NFS volfile */
-        snprintf (tmpnfsvol, sizeof (tmpnfsvol), "/tmp/gnfs-XXXXXX");
-        tmpfd = mkstemp (tmpnfsvol);
-        if (tmpfd < 0) {
-                gf_log (this->name, GF_LOG_WARNING, "Unable to create temp file"
-                        " %s: (%s)", tmpnfsvol, strerror (errno));
-                goto out;
-        }
-
-        tmpclean = 1; /* SET the flag to unlink() tmpfile */
-
-        ret = glusterd_create_global_volfile (build_nfs_graph,
-                                              tmpnfsvol, NULL);
-        if (ret)
-                goto out;
-
-        /* Compare the topology of volfiles */
-        ret = glusterd_check_topology_identical (nfsvol, tmpnfsvol,
-                                                 identical);
-out:
-        if (tmpfd >= 0)
-                close (tmpfd);
-        if (tmpclean)
-                unlink (tmpnfsvol);
-        return ret;
+    return glusterd_svc_start(svc, flags, NULL);
 }
 
-int
-glusterd_nfssvc_manager (glusterd_svc_t *svc, void *data, int flags)
+static int
+glusterd_nfssvc_stop(glusterd_svc_t *svc, int sig)
 {
-        int                 ret     = -1;
+    int ret = -1;
+    gf_boolean_t deregister = _gf_false;
 
-        if (glusterd_are_all_volumes_stopped ()) {
-                ret = svc->stop (svc, SIGKILL);
+    if (glusterd_proc_is_running(&(svc->proc)))
+        deregister = _gf_true;
 
-        } else {
-                ret = glusterd_nfssvc_create_volfile ();
-                if (ret)
-                        goto out;
+    ret = glusterd_svc_stop(svc, sig);
+    if (ret)
+        goto out;
+    if (deregister)
+        glusterd_nfs_pmap_deregister();
 
-                ret = svc->stop (svc, SIGKILL);
-                if (ret)
-                        goto out;
-
-                ret = svc->start (svc, flags);
-                if (ret)
-                        goto out;
-
-                ret = glusterd_conn_connect (&(svc->conn));
-                if (ret)
-                        goto out;
-        }
 out:
-        gf_log (THIS->name, GF_LOG_DEBUG, "Returning %d", ret);
-
-        return ret;
-}
+    gf_msg_debug(THIS->name, 0, "Returning %d", ret);
 
-int
-glusterd_nfssvc_start (glusterd_svc_t *svc, int flags)
-{
-        if (glusterd_nfssvc_need_start ())
-                return glusterd_svc_start (svc, flags, NULL);
-
-        return 0;
+    return ret;
 }
 
-int
-glusterd_nfssvc_stop (glusterd_svc_t *svc, int sig)
+void
+glusterd_nfssvc_build(glusterd_svc_t *svc)
 {
-        int                    ret        = -1;
-        gf_boolean_t           deregister = _gf_false;
-
-        if (glusterd_proc_is_running (&(svc->proc)))
-                deregister = _gf_true;
-
-        ret = glusterd_svc_stop (svc, sig);
-        if (ret)
-                goto out;
-        if (deregister)
-                glusterd_nfs_pmap_deregister ();
-
-out:
-        gf_log (THIS->name, GF_LOG_DEBUG, "Returning %d", ret);
-
-        return ret;
+    svc->manager = glusterd_nfssvc_manager;
+    svc->start = glusterd_nfssvc_start;
+    svc->stop = glusterd_nfssvc_stop;
 }
 
 int
-glusterd_nfssvc_reconfigure ()
+glusterd_nfssvc_reconfigure()
 {
-        int              ret             = -1;
-        xlator_t        *this            = NULL;
-        glusterd_conf_t *priv            = NULL;
-        gf_boolean_t     identical       = _gf_false;
-
-        this = THIS;
-        GF_ASSERT (this);
-
-        priv = this->private;
-        GF_ASSERT (priv);
-
-        /*
-         * Check both OLD and NEW volfiles, if they are SAME by size
-         * and cksum i.e. "character-by-character". If YES, then
-         * NOTHING has been changed, just return.
-         */
-        ret = glusterd_nfssvc_check_volfile_identical (&identical);
-        if (ret)
-                goto out;
-
-        if (identical) {
-                ret = 0;
-                goto out;
+    int ret = -1;
+    xlator_t *this = NULL;
+    glusterd_conf_t *priv = NULL;
+    gf_boolean_t identical = _gf_false;
+    gf_boolean_t vol_started = _gf_false;
+    glusterd_volinfo_t *volinfo = NULL;
+
+    this = THIS;
+    GF_VALIDATE_OR_GOTO("glusterd", this, out);
+
+    priv = this->private;
+    GF_VALIDATE_OR_GOTO(this->name, priv, out);
+
+    /* not an error, or a (very) soft error at best */
+    if (sys_access(XLATORDIR "/nfs/server.so", R_OK) != 0) {
+        gf_msg(THIS->name, GF_LOG_INFO, 0, GD_MSG_GNFS_XLATOR_NOT_INSTALLED,
+               "nfs/server.so xlator is not installed");
+        ret = 0;
+        goto out;
+    }
+
+    cds_list_for_each_entry(volinfo, &priv->volumes, vol_list)
+    {
+        if (GLUSTERD_STATUS_STARTED == volinfo->status) {
+            vol_started = _gf_true;
+            break;
         }
-
-        /*
-         * They are not identical. Find out if the topology is changed
-         * OR just the volume options. If just the options which got
-         * changed, then inform the xlator to reconfigure the options.
-         */
-        identical = _gf_false; /* RESET the FLAG */
-        ret = glusterd_nfssvc_check_topology_identical (&identical);
-        if (ret)
-                goto out;
-
-        /* Topology is not changed, but just the options. But write the
-         * options to NFS volfile, so that NFS will be reconfigured.
-         */
-        if (identical) {
-                ret = glusterd_nfssvc_create_volfile();
-                if (ret == 0) {/* Only if above PASSES */
-                        ret = glusterd_fetchspec_notify (THIS);
-                }
-                goto out;
+    }
+    if (!vol_started) {
+        ret = 0;
+        goto out;
+    }
+
+    /*
+     * Check both OLD and NEW volfiles, if they are SAME by size
+     * and cksum i.e. "character-by-character". If YES, then
+     * NOTHING has been changed, just return.
+     */
+
+    ret = glusterd_svc_check_volfile_identical(priv->nfs_svc.name,
+                                               build_nfs_graph, &identical);
+    if (ret)
+        goto out;
+
+    if (identical) {
+        ret = 0;
+        goto out;
+    }
+
+    /*
+     * They are not identical. Find out if the topology is changed
+     * OR just the volume options. If just the options which got
+     * changed, then inform the xlator to reconfigure the options.
+     */
+    identical = _gf_false; /* RESET the FLAG */
+    ret = glusterd_svc_check_topology_identical(priv->nfs_svc.name,
+                                                build_nfs_graph, &identical);
+    if (ret)
+        goto out;
+
+    /* Topology is not changed, but just the options. But write the
+     * options to NFS volfile, so that NFS will be reconfigured.
+     */
+    if (identical) {
+        ret = glusterd_nfssvc_create_volfile();
+        if (ret == 0) { /* Only if above PASSES */
+            ret = glusterd_fetchspec_notify(THIS);
         }
+        goto out;
+    }
 
-        /*
-         * NFS volfile's topology has been changed. NFS server needs
-         * to be RESTARTED to ACT on the changed volfile.
-         */
-        ret = priv->nfs_svc.manager (&(priv->nfs_svc), NULL,
-                                     PROC_START_NO_WAIT);
+    /*
+     * NFS volfile's topology has been changed. NFS server needs
+     * to be RESTARTED to ACT on the changed volfile.
+     */
+    ret = priv->nfs_svc.manager(&(priv->nfs_svc), NULL, PROC_START_NO_WAIT);
 
 out:
-        return ret;
-
+    gf_msg_debug(this ? this->name : "glusterd", 0, "Returning %d", ret);
+    return ret;
 }
+#endif
diff --git a/xlators/mgmt/glusterd/src/glusterd-nfs-svc.h b/xlators/mgmt/glusterd/src/glusterd-nfs-svc.h
index 210336cde83..6bfdde95749 100644
--- a/xlators/mgmt/glusterd/src/glusterd-nfs-svc.h
+++ b/xlators/mgmt/glusterd/src/glusterd-nfs-svc.h
@@ -11,26 +11,17 @@
 #ifndef _GLUSTERD_NFS_SVC_H_
 #define _GLUSTERD_NFS_SVC_H_
 
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
-
 #include "glusterd-svc-mgmt.h"
 
-int
-glusterd_nfssvc_init (glusterd_svc_t *svc);
-
-int
-glusterd_nfssvc_manager (glusterd_svc_t *svc, void *data, int flags);
-
-int
-glusterd_nfssvc_start (glusterd_svc_t *svc, int flags);
+#ifdef BUILD_GNFS
+void
+glusterd_nfssvc_build(glusterd_svc_t *svc);
 
 int
-glusterd_nfssvc_stop (glusterd_svc_t *svc, int sig);
+glusterd_nfssvc_init(glusterd_svc_t *svc);
 
 int
-glusterd_nfssvc_reconfigure ();
+glusterd_nfssvc_reconfigure();
 
+#endif /* BUILD_GNFS */
 #endif
diff --git a/xlators/mgmt/glusterd/src/glusterd-op-sm.c b/xlators/mgmt/glusterd/src/glusterd-op-sm.c
index d7694258301..c537fc33a85 100644
--- a/xlators/mgmt/glusterd/src/glusterd-op-sm.c
+++ b/xlators/mgmt/glusterd/src/glusterd-op-sm.c
@@ -8,3771 +8,4702 @@
    cases as published by the Free Software Foundation.
 */
 
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
 #include <time.h>
 #include <sys/uio.h>
 #include <sys/resource.h>
 #include <sys/mount.h>
 
 #include <libgen.h>
-#include "compat-uuid.h"
+#include <glusterfs/compat-uuid.h>
 
 #include "fnmatch.h"
-#include "xlator.h"
+#include <glusterfs/xlator.h>
 #include "protocol-common.h"
 #include "glusterd.h"
-#include "call-stub.h"
-#include "defaults.h"
-#include "list.h"
-#include "dict.h"
-#include "compat.h"
-#include "compat-errno.h"
-#include "statedump.h"
-#include "glusterd-sm.h"
+#include <glusterfs/call-stub.h>
+#include <glusterfs/list.h>
+#include <glusterfs/dict.h>
+#include <glusterfs/compat.h>
+#include <glusterfs/compat-errno.h>
+#include <glusterfs/statedump.h>
 #include "glusterd-op-sm.h"
 #include "glusterd-utils.h"
 #include "glusterd-store.h"
-#include "glusterd-hooks.h"
-#include "glusterd-volgen.h"
 #include "glusterd-locks.h"
-#include "glusterd-messages.h"
-#include "syscall.h"
+#include "glusterd-quota.h"
+#include <glusterfs/syscall.h>
 #include "cli1-xdr.h"
-#include "common-utils.h"
-#include "run.h"
 #include "glusterd-snapshot-utils.h"
 #include "glusterd-svc-mgmt.h"
 #include "glusterd-svc-helper.h"
+#include "glusterd-shd-svc-helper.h"
 #include "glusterd-shd-svc.h"
-#include "glusterd-nfs-svc.h"
 #include "glusterd-quotad-svc.h"
 #include "glusterd-server-quorum.h"
-
 #include <sys/types.h>
 #include <signal.h>
 #include <sys/wait.h>
+#include "glusterd-gfproxyd-svc-helper.h"
+
+#define len_strcmp(key, len, str)                                              \
+    ((len == SLEN(str)) && (strcmp(key, str) == 0))
+
+extern char local_node_hostname[PATH_MAX];
+static int
+glusterd_set_shared_storage(dict_t *dict, char *key, char *value,
+                            char **op_errstr);
 
-#define ALL_VOLUME_OPTION_CHECK(volname, key, ret, op_errstr, label)           \
-        do {                                                                   \
-                gf_boolean_t    _all = !strcmp ("all", volname);               \
-                gf_boolean_t    _ratio = !strcmp (key,                         \
-                                                  GLUSTERD_QUORUM_RATIO_KEY);  \
-                if (_all && !_ratio) {                                         \
-                        ret = -1;                                              \
-                        *op_errstr = gf_strdup ("Not a valid option for all "  \
-                                                "volumes");                    \
-                        goto label;                                            \
-                } else if (!_all && _ratio) {                                  \
-                        ret = -1;                                              \
-                        *op_errstr = gf_strdup ("Not a valid option for "      \
-                                                "single volume");              \
-                        goto label;                                            \
-                }                                                              \
-         } while (0)
+/*
+ * Valid options for all volumes to be listed in the valid_all_vol_opts table.
+ * To add newer options to all volumes, we can just add more entries to this
+ * table.
+ *
+ * It's important that every value have a default, or have a special handler
+ * in glusterd_get_global_options_for_all_vols, or else we might crash there.
+ */
+const glusterd_all_vol_opts valid_all_vol_opts[] = {
+    {GLUSTERD_QUORUM_RATIO_KEY, "51"},
+    {GLUSTERD_SHARED_STORAGE_KEY, "disable"},
+    /* This one actually gets filled in dynamically. */
+    {GLUSTERD_GLOBAL_OP_VERSION_KEY, "BUG_NO_OP_VERSION"},
+    /*
+     * This one should be filled in dynamically, but it didn't used to be
+     * (before the defaults were added here) so the value is unclear.
+     *
+     * TBD: add a dynamic handler to set the appropriate value
+     */
+    {GLUSTERD_MAX_OP_VERSION_KEY, "BUG_NO_MAX_OP_VERSION"},
+    {GLUSTERD_BRICK_MULTIPLEX_KEY, "disable"},
+    /* Set this value to 0 by default implying brick-multiplexing
+     * behaviour with no limit set on the number of brick instances that
+     * can be attached per process.
+     * TBD: Discuss the default value for this. Maybe this should be a
+     * dynamic value depending on the memory specifications per node */
+    {GLUSTERD_BRICKMUX_LIMIT_KEY, GLUSTERD_BRICKMUX_LIMIT_DFLT_VALUE},
+    {GLUSTERD_VOL_CNT_PER_THRD, GLUSTERD_VOL_CNT_PER_THRD_DEFAULT_VALUE},
+    {GLUSTERD_LOCALTIME_LOGGING_KEY, "disable"},
+    {GLUSTERD_DAEMON_LOG_LEVEL_KEY, "INFO"},
+    {NULL},
+};
 
 static struct cds_list_head gd_op_sm_queue;
 synclock_t gd_op_sm_lock;
-glusterd_op_info_t    opinfo = {{0},};
+glusterd_op_info_t opinfo = {
+    {0},
+};
 
 int32_t
-glusterd_txn_opinfo_dict_init ()
+glusterd_txn_opinfo_dict_init()
 {
-        int32_t             ret    = -1;
-        xlator_t           *this   = NULL;
-        glusterd_conf_t    *priv   = NULL;
-
-        this = THIS;
-        GF_ASSERT (this);
-        priv = this->private;
-        GF_ASSERT (priv);
-
-        priv->glusterd_txn_opinfo = dict_new ();
-        if (!priv->glusterd_txn_opinfo) {
-                ret = -1;
-                goto out;
-        }
+    int32_t ret = -1;
+    xlator_t *this = NULL;
+    glusterd_conf_t *priv = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+    priv = this->private;
+    GF_ASSERT(priv);
+
+    priv->glusterd_txn_opinfo = dict_new();
+    if (!priv->glusterd_txn_opinfo) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_CREATE_FAIL, NULL);
+        ret = -1;
+        goto out;
+    }
 
-        memset (priv->global_txn_id, '\0', sizeof(uuid_t));
+    memset(priv->global_txn_id, '\0', sizeof(uuid_t));
 
-        ret = 0;
+    ret = 0;
 out:
-        return ret;
+    return ret;
 }
 
 void
-glusterd_txn_opinfo_dict_fini ()
+glusterd_txn_opinfo_dict_fini()
 {
-        xlator_t           *this   = NULL;
-        glusterd_conf_t    *priv   = NULL;
+    xlator_t *this = NULL;
+    glusterd_conf_t *priv = NULL;
 
-        this = THIS;
-        GF_ASSERT (this);
-        priv = this->private;
-        GF_ASSERT (priv);
+    this = THIS;
+    GF_ASSERT(this);
+    priv = this->private;
+    GF_ASSERT(priv);
 
-        if (priv->glusterd_txn_opinfo)
-                dict_unref (priv->glusterd_txn_opinfo);
+    if (priv->glusterd_txn_opinfo)
+        dict_unref(priv->glusterd_txn_opinfo);
 }
 
 void
-glusterd_txn_opinfo_init (glusterd_op_info_t  *opinfo,
-                          glusterd_op_sm_state_info_t *state,
-                          glusterd_op_t *op,
-                          dict_t *op_ctx,
-                          rpcsvc_request_t *req)
+glusterd_txn_opinfo_init(glusterd_op_info_t *opinfo,
+                         glusterd_op_sm_state_info_t *state, int *op,
+                         dict_t *op_ctx, rpcsvc_request_t *req)
 {
-        GF_ASSERT (opinfo);
+    glusterd_conf_t *conf = NULL;
 
-        if (state)
-                opinfo->state = *state;
+    GF_ASSERT(opinfo);
 
-        if (op)
-                opinfo->op = *op;
+    conf = THIS->private;
+    GF_ASSERT(conf);
 
-        if (op_ctx)
-                opinfo->op_ctx = dict_ref(op_ctx);
-        else
-                opinfo->op_ctx = NULL;
+    if (state)
+        opinfo->state = *state;
 
-        if (req)
-                opinfo->req = req;
+    if (op)
+        opinfo->op = *op;
 
-        return;
-}
+    if (op_ctx)
+        opinfo->op_ctx = dict_ref(op_ctx);
+    else
+        opinfo->op_ctx = NULL;
 
-int32_t
-glusterd_generate_txn_id (dict_t *dict, uuid_t **txn_id)
-{
-        int32_t             ret      = -1;
-        glusterd_conf_t    *priv     = NULL;
-        xlator_t           *this     = NULL;
-
-        this = THIS;
-        GF_ASSERT (this);
-        priv = this->private;
-        GF_ASSERT (priv);
-        GF_ASSERT (dict);
-
-        *txn_id = GF_CALLOC (1, sizeof(uuid_t), gf_common_mt_uuid_t);
-        if (!*txn_id)
-                goto out;
+    if (req)
+        opinfo->req = req;
 
-        if (priv->op_version < GD_OP_VERSION_3_6_0)
-                gf_uuid_copy (**txn_id, priv->global_txn_id);
-        else
-                gf_uuid_generate (**txn_id);
+    opinfo->txn_generation = conf->generation;
+    cmm_smp_rmb();
 
-        ret = dict_set_bin (dict, "transaction_id",
-                            *txn_id, sizeof (**txn_id));
-        if (ret) {
-                gf_log ("", GF_LOG_ERROR,
-                       "Failed to set transaction id.");
-                goto out;
-        }
+    return;
+}
 
-        gf_log ("", GF_LOG_DEBUG,
-                "Transaction_id = %s", uuid_utoa (**txn_id));
+int32_t
+glusterd_generate_txn_id(dict_t *dict, uuid_t **txn_id)
+{
+    int32_t ret = -1;
+    glusterd_conf_t *priv = NULL;
+    xlator_t *this = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+    priv = this->private;
+    GF_ASSERT(priv);
+    GF_ASSERT(dict);
+
+    *txn_id = GF_MALLOC(sizeof(uuid_t), gf_common_mt_uuid_t);
+    if (!*txn_id) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_NO_MEMORY, NULL);
+        goto out;
+    }
+
+    if (priv->op_version < GD_OP_VERSION_3_6_0)
+        gf_uuid_copy(**txn_id, priv->global_txn_id);
+    else
+        gf_uuid_generate(**txn_id);
+
+    ret = dict_set_bin(dict, "transaction_id", *txn_id, sizeof(**txn_id));
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+               "Failed to set transaction id.");
+        goto out;
+    }
+
+    gf_msg_debug(this->name, 0, "Transaction_id = %s", uuid_utoa(**txn_id));
 out:
-        if (ret && *txn_id) {
-                GF_FREE (*txn_id);
-                *txn_id = NULL;
-        }
+    if (ret && *txn_id) {
+        GF_FREE(*txn_id);
+        *txn_id = NULL;
+    }
 
-        return ret;
+    return ret;
 }
 
 int32_t
-glusterd_get_txn_opinfo (uuid_t *txn_id, glusterd_op_info_t  *opinfo)
+glusterd_get_txn_opinfo(uuid_t *txn_id, glusterd_op_info_t *opinfo)
 {
-        int32_t                   ret         = -1;
-        glusterd_txn_opinfo_obj  *opinfo_obj  = NULL;
-        glusterd_conf_t          *priv        = NULL;
-        xlator_t                 *this        = NULL;
-
-        this = THIS;
-        GF_ASSERT (this);
-        priv = this->private;
-        GF_ASSERT (priv);
-
-        if (!txn_id || !opinfo) {
-                gf_log_callingfn (this->name, GF_LOG_ERROR,
-                                  "Empty transaction id or opinfo received.");
-                ret = -1;
-                goto out;
-        }
+    int32_t ret = -1;
+    glusterd_txn_opinfo_obj *opinfo_obj = NULL;
+    glusterd_conf_t *priv = NULL;
+    xlator_t *this = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+    priv = this->private;
+    GF_ASSERT(priv);
+
+    if (!txn_id || !opinfo) {
+        gf_msg_callingfn(this->name, GF_LOG_ERROR, 0, GD_MSG_TRANS_ID_GET_FAIL,
+                         "Empty transaction id or opinfo received.");
+        ret = -1;
+        goto out;
+    }
 
-        ret = dict_get_bin(priv->glusterd_txn_opinfo,
-                           uuid_utoa (*txn_id),
-                           (void **) &opinfo_obj);
-        if (ret) {
-                gf_log_callingfn (this->name, GF_LOG_ERROR,
-                        "Unable to get transaction opinfo "
-                        "for transaction ID : %s",
-                        uuid_utoa (*txn_id));
-                goto out;
-        }
+    ret = dict_get_bin(priv->glusterd_txn_opinfo, uuid_utoa(*txn_id),
+                       (void **)&opinfo_obj);
+    if (ret)
+        goto out;
 
-        (*opinfo) = opinfo_obj->opinfo;
+    (*opinfo) = opinfo_obj->opinfo;
 
-        gf_log (this->name, GF_LOG_DEBUG,
-                "Successfully got opinfo for transaction ID : %s",
-                uuid_utoa (*txn_id));
+    gf_msg_debug(this->name, 0,
+                 "Successfully got opinfo for transaction ID : %s",
+                 uuid_utoa(*txn_id));
 
-        ret = 0;
+    ret = 0;
 out:
-        gf_log (this->name, GF_LOG_DEBUG, "Returning %d", ret);
-        return ret;
+    gf_msg_debug(this->name, 0, "Returning %d", ret);
+    return ret;
 }
 
 int32_t
-glusterd_set_txn_opinfo (uuid_t *txn_id, glusterd_op_info_t  *opinfo)
+glusterd_set_txn_opinfo(uuid_t *txn_id, glusterd_op_info_t *opinfo)
 {
-        int32_t                   ret        = -1;
-        glusterd_txn_opinfo_obj  *opinfo_obj = NULL;
-        glusterd_conf_t          *priv        = NULL;
-        xlator_t                 *this        = NULL;
-
-        this = THIS;
-        GF_ASSERT (this);
-        priv = this->private;
-        GF_ASSERT (priv);
-
-        if (!txn_id) {
-                gf_log_callingfn (this->name, GF_LOG_ERROR,
-                                  "Empty transaction id received.");
-                ret = -1;
-                goto out;
+    int32_t ret = -1;
+    glusterd_txn_opinfo_obj *opinfo_obj = NULL;
+    glusterd_conf_t *priv = NULL;
+    xlator_t *this = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+    priv = this->private;
+    GF_ASSERT(priv);
+
+    if (!txn_id) {
+        gf_msg_callingfn(this->name, GF_LOG_ERROR, 0, GD_MSG_TRANS_ID_GET_FAIL,
+                         "Empty transaction id received.");
+        ret = -1;
+        goto out;
+    }
+
+    ret = dict_get_bin(priv->glusterd_txn_opinfo, uuid_utoa(*txn_id),
+                       (void **)&opinfo_obj);
+    if (ret) {
+        opinfo_obj = GF_CALLOC(1, sizeof(glusterd_txn_opinfo_obj),
+                               gf_common_mt_txn_opinfo_obj_t);
+        if (!opinfo_obj) {
+            ret = -1;
+            goto out;
         }
 
-        ret = dict_get_bin(priv->glusterd_txn_opinfo,
-                           uuid_utoa (*txn_id),
-                           (void **) &opinfo_obj);
+        ret = dict_set_bin(priv->glusterd_txn_opinfo, uuid_utoa(*txn_id),
+                           opinfo_obj, sizeof(glusterd_txn_opinfo_obj));
         if (ret) {
-                opinfo_obj = GF_CALLOC (1, sizeof(glusterd_txn_opinfo_obj),
-                                        gf_common_mt_txn_opinfo_obj_t);
-                if (!opinfo_obj) {
-                        ret = -1;
-                        goto out;
-                }
-
-                ret = dict_set_bin(priv->glusterd_txn_opinfo,
-                                   uuid_utoa (*txn_id), opinfo_obj,
-                                   sizeof(glusterd_txn_opinfo_obj));
-                if (ret) {
-                        gf_log_callingfn (this->name, GF_LOG_ERROR,
-                                          "Unable to set opinfo for transaction"
-                                          " ID : %s", uuid_utoa (*txn_id));
-                        goto out;
-                }
+            gf_msg_callingfn(this->name, GF_LOG_ERROR, errno,
+                             GD_MSG_DICT_SET_FAILED,
+                             "Unable to set opinfo for transaction"
+                             " ID : %s",
+                             uuid_utoa(*txn_id));
+            goto out;
         }
+    }
 
-        opinfo_obj->opinfo = (*opinfo);
+    opinfo_obj->opinfo = (*opinfo);
 
-        gf_log (this->name, GF_LOG_DEBUG,
-                "Successfully set opinfo for transaction ID : %s",
-                uuid_utoa (*txn_id));
-        ret = 0;
+    gf_msg_debug(this->name, 0,
+                 "Successfully set opinfo for transaction ID : %s",
+                 uuid_utoa(*txn_id));
+    ret = 0;
 out:
-        if (ret)
-                if (opinfo_obj)
-                        GF_FREE (opinfo_obj);
+    if (ret)
+        if (opinfo_obj)
+            GF_FREE(opinfo_obj);
 
-        gf_log (this->name, GF_LOG_DEBUG, "Returning %d", ret);
-        return ret;
+    gf_msg_debug(this->name, 0, "Returning %d", ret);
+    return ret;
 }
 
 int32_t
-glusterd_clear_txn_opinfo (uuid_t *txn_id)
+glusterd_clear_txn_opinfo(uuid_t *txn_id)
 {
-        int32_t               ret         = -1;
-        glusterd_op_info_t    txn_op_info = {{0},};
-        glusterd_conf_t      *priv        = NULL;
-        xlator_t             *this        = NULL;
-
-        this = THIS;
-        GF_ASSERT (this);
-        priv = this->private;
-        GF_ASSERT (priv);
-
-        if (!txn_id) {
-                gf_log ("", GF_LOG_ERROR, "Empty transaction id received.");
-                ret = -1;
-                goto out;
-        }
-
-        ret = glusterd_get_txn_opinfo (txn_id, &txn_op_info);
-        if (ret) {
-                gf_log ("", GF_LOG_ERROR, "Transaction opinfo not found");
-                goto out;
-        }
+    int32_t ret = -1;
+    glusterd_op_info_t txn_op_info = {
+        {0},
+    };
+    glusterd_conf_t *priv = NULL;
+    xlator_t *this = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+    priv = this->private;
+    GF_ASSERT(priv);
+
+    if (!txn_id) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_TRANS_ID_GET_FAIL,
+               "Empty transaction id received.");
+        ret = -1;
+        goto out;
+    }
 
-        if (txn_op_info.op_ctx)
-                dict_unref (txn_op_info.op_ctx);
+    ret = glusterd_get_txn_opinfo(txn_id, &txn_op_info);
+    if (ret) {
+        gf_msg_callingfn(this->name, GF_LOG_ERROR, 0,
+                         GD_MSG_TRANS_OPINFO_GET_FAIL,
+                         "Unable to get transaction opinfo "
+                         "for transaction ID : %s",
+                         uuid_utoa(*txn_id));
+        goto out;
+    }
 
-        dict_del(priv->glusterd_txn_opinfo, uuid_utoa (*txn_id));
+    if (txn_op_info.op_ctx)
+        dict_unref(txn_op_info.op_ctx);
 
-        if (txn_op_info.local_xaction_peers)
-                GF_FREE (txn_op_info.local_xaction_peers);
+    dict_del(priv->glusterd_txn_opinfo, uuid_utoa(*txn_id));
 
-        gf_log ("", GF_LOG_DEBUG,
-                "Successfully cleared opinfo for transaction ID : %s",
-                uuid_utoa (*txn_id));
+    gf_msg_debug(this->name, 0,
+                 "Successfully cleared opinfo for transaction ID : %s",
+                 uuid_utoa(*txn_id));
 
-        ret = 0;
+    ret = 0;
 out:
-        gf_log ("", GF_LOG_DEBUG, "Returning %d", ret);
-        return ret;
+    gf_msg_debug(this->name, 0, "Returning %d", ret);
+    return ret;
 }
 
 static int glusterfs_port = GLUSTERD_DEFAULT_PORT;
 static char *glusterd_op_sm_state_names[] = {
-        "Default",
-        "Lock sent",
-        "Locked",
-        "Stage op sent",
-        "Staged",
-        "Commit op sent",
-        "Committed",
-        "Unlock sent",
-        "Stage op failed",
-        "Commit op failed",
-        "Brick op sent",
-        "Brick op failed",
-        "Brick op Committed",
-        "Brick op Commit failed",
-        "Ack drain",
-        "Invalid",
+    "Default",
+    "Lock sent",
+    "Locked",
+    "Stage op sent",
+    "Staged",
+    "Commit op sent",
+    "Committed",
+    "Unlock sent",
+    "Stage op failed",
+    "Commit op failed",
+    "Brick op sent",
+    "Brick op failed",
+    "Brick op Committed",
+    "Brick op Commit failed",
+    "Ack drain",
+    "Invalid",
 };
 
 static char *glusterd_op_sm_event_names[] = {
-        "GD_OP_EVENT_NONE",
-        "GD_OP_EVENT_START_LOCK",
-        "GD_OP_EVENT_LOCK",
-        "GD_OP_EVENT_RCVD_ACC",
-        "GD_OP_EVENT_ALL_ACC",
-        "GD_OP_EVENT_STAGE_ACC",
-        "GD_OP_EVENT_COMMIT_ACC",
-        "GD_OP_EVENT_RCVD_RJT",
-        "GD_OP_EVENT_STAGE_OP",
-        "GD_OP_EVENT_COMMIT_OP",
-        "GD_OP_EVENT_UNLOCK",
-        "GD_OP_EVENT_START_UNLOCK",
-        "GD_OP_EVENT_ALL_ACK",
-        "GD_OP_EVENT_LOCAL_UNLOCK_NO_RESP",
-        "GD_OP_EVENT_INVALID"
-};
-
-extern struct volopt_map_entry glusterd_volopt_map[];
+    "GD_OP_EVENT_NONE",       "GD_OP_EVENT_START_LOCK",
+    "GD_OP_EVENT_LOCK",       "GD_OP_EVENT_RCVD_ACC",
+    "GD_OP_EVENT_ALL_ACC",    "GD_OP_EVENT_STAGE_ACC",
+    "GD_OP_EVENT_COMMIT_ACC", "GD_OP_EVENT_RCVD_RJT",
+    "GD_OP_EVENT_STAGE_OP",   "GD_OP_EVENT_COMMIT_OP",
+    "GD_OP_EVENT_UNLOCK",     "GD_OP_EVENT_START_UNLOCK",
+    "GD_OP_EVENT_ALL_ACK",    "GD_OP_EVENT_LOCAL_UNLOCK_NO_RESP",
+    "GD_OP_EVENT_INVALID"};
+
+char *
+glusterd_op_sm_state_name_get(int state)
+{
+    if (state < 0 || state >= GD_OP_STATE_MAX)
+        return glusterd_op_sm_state_names[GD_OP_STATE_MAX];
+    return glusterd_op_sm_state_names[state];
+}
 
-char*
-glusterd_op_sm_state_name_get (int state)
+char *
+glusterd_op_sm_event_name_get(int event)
 {
-        if (state < 0 || state >= GD_OP_STATE_MAX)
-                return glusterd_op_sm_state_names[GD_OP_STATE_MAX];
-        return glusterd_op_sm_state_names[state];
+    if (event < 0 || event >= GD_OP_EVENT_MAX)
+        return glusterd_op_sm_event_names[GD_OP_EVENT_MAX];
+    return glusterd_op_sm_event_names[event];
 }
 
-char*
-glusterd_op_sm_event_name_get (int event)
+static void
+glusterd_destroy_lock_ctx(glusterd_op_lock_ctx_t *ctx)
 {
-        if (event < 0 || event >= GD_OP_EVENT_MAX)
-                return glusterd_op_sm_event_names[GD_OP_EVENT_MAX];
-        return glusterd_op_sm_event_names[event];
+    if (!ctx)
+        return;
+    GF_FREE(ctx);
 }
 
 void
-glusterd_destroy_lock_ctx (glusterd_op_lock_ctx_t *ctx)
+glusterd_set_volume_status(glusterd_volinfo_t *volinfo,
+                           glusterd_volume_status status)
 {
-        if (!ctx)
-                return;
-        GF_FREE (ctx);
+    GF_ASSERT(volinfo);
+    volinfo->status = status;
 }
 
-void
-glusterd_set_volume_status (glusterd_volinfo_t  *volinfo,
-                            glusterd_volume_status status)
+static int
+glusterd_op_sm_inject_all_acc(uuid_t *txn_id)
 {
-        GF_ASSERT (volinfo);
-        volinfo->status = status;
+    int ret = -1;
+    ret = glusterd_op_sm_inject_event(GD_OP_EVENT_ALL_ACC, txn_id, NULL);
+    gf_msg_debug("glusterd", 0, "Returning %d", ret);
+    return ret;
 }
 
 static int
-glusterd_op_sm_inject_all_acc (uuid_t *txn_id)
+glusterd_check_bitrot_cmd(char *key, const int keylen, char *errstr,
+                          const size_t size)
 {
-        int32_t                 ret = -1;
-        ret = glusterd_op_sm_inject_event (GD_OP_EVENT_ALL_ACC, txn_id, NULL);
-        gf_log ("", GF_LOG_DEBUG, "Returning %d", ret);
-        return ret;
+    int ret = -1;
+
+    if (len_strcmp(key, keylen, "bitrot") ||
+        len_strcmp(key, keylen, "features.bitrot")) {
+        snprintf(errstr, size,
+                 " 'gluster volume set <VOLNAME> %s' is invalid command."
+                 " Use 'gluster volume bitrot <VOLNAME> {enable|disable}'"
+                 " instead.",
+                 key);
+        goto out;
+    } else if (len_strcmp(key, keylen, "scrub-freq") ||
+               len_strcmp(key, keylen, "features.scrub-freq")) {
+        snprintf(errstr, size,
+                 " 'gluster volume set <VOLNAME> %s' is invalid command."
+                 " Use 'gluster volume bitrot <VOLNAME> scrub-frequency"
+                 " {hourly|daily|weekly|biweekly|monthly}' instead.",
+                 key);
+        goto out;
+    } else if (len_strcmp(key, keylen, "scrub") ||
+               len_strcmp(key, keylen, "features.scrub")) {
+        snprintf(errstr, size,
+                 " 'gluster volume set <VOLNAME> %s' is invalid command."
+                 " Use 'gluster volume bitrot <VOLNAME> scrub {pause|resume}'"
+                 " instead.",
+                 key);
+        goto out;
+    } else if (len_strcmp(key, keylen, "scrub-throttle") ||
+               len_strcmp(key, keylen, "features.scrub-throttle")) {
+        snprintf(errstr, size,
+                 " 'gluster volume set <VOLNAME> %s' is invalid command."
+                 " Use 'gluster volume bitrot <VOLNAME> scrub-throttle "
+                 " {lazy|normal|aggressive}' instead.",
+                 key);
+        goto out;
+    }
+
+    ret = 0;
+out:
+    return ret;
 }
 
 static int
-glusterd_check_quota_cmd (char *key, char *value, char *errstr, size_t size)
+glusterd_check_quota_cmd(char *key, const int keylen, char *value, char *errstr,
+                         size_t size)
 {
-        int                ret = -1;
-        gf_boolean_t       b   = _gf_false;
+    int ret = -1;
+    gf_boolean_t b = _gf_false;
 
-        if ((strcmp (key, "quota") == 0) ||
-           (strcmp (key, "features.quota") == 0)) {
-                ret = gf_string2boolean (value, &b);
-                if (ret)
-                        goto out;
-                if (b) {
-                          snprintf (errstr, size," 'gluster "
-                                    "volume set <VOLNAME> %s %s' is "
-                                    "deprecated. Use 'gluster volume "
-                                    "quota <VOLNAME> enable' instead.",
-                                     key, value);
-                          ret = -1;
-                          goto out;
-                } else {
-                          snprintf (errstr, size, " 'gluster "
-                                    "volume set <VOLNAME> %s %s' is "
-                                    "deprecated. Use 'gluster volume "
-                                    "quota <VOLNAME> disable' instead.",
-                                     key, value);
-                          ret = -1;
-                          goto out;
-                }
-        }
-        ret = 0;
+    if (len_strcmp(key, keylen, "quota") ||
+        len_strcmp(key, keylen, "features.quota")) {
+        ret = gf_string2boolean(value, &b);
+        if (ret)
+            goto out;
+        ret = -1;
+        if (b) {
+            snprintf(errstr, size,
+                     " 'gluster volume set <VOLNAME> %s %s' is deprecated."
+                     " Use 'gluster volume quota <VOLNAME> enable' instead.",
+                     key, value);
+        } else {
+            snprintf(errstr, size,
+                     " 'gluster volume set <VOLNAME> %s %s' is deprecated."
+                     " Use 'gluster volume quota <VOLNAME> disable' instead.",
+                     key, value);
+        }
+        goto out;
+    } else if (len_strcmp(key, keylen, "inode-quota") ||
+               len_strcmp(key, keylen, "features.inode-quota")) {
+        ret = gf_string2boolean(value, &b);
+        if (ret)
+            goto out;
+        ret = -1;
+        if (b) {
+            snprintf(
+                errstr, size,
+                " 'gluster volume set <VOLNAME> %s %s' is deprecated."
+                " Use 'gluster volume inode-quota <VOLNAME> enable' instead.",
+                key, value);
+        } else {
+            /* inode-quota disable not supported,
+             * use quota disable
+             */
+            snprintf(errstr, size,
+                     " 'gluster volume set <VOLNAME> %s %s' is deprecated."
+                     " Use 'gluster volume quota <VOLNAME> disable' instead.",
+                     key, value);
+        }
+        goto out;
+    }
+
+    ret = 0;
 out:
-        return ret;
+    return ret;
 }
 
 int
-glusterd_brick_op_build_payload (glusterd_op_t op, glusterd_brickinfo_t *brickinfo,
-                                 gd1_mgmt_brick_op_req **req, dict_t *dict)
+glusterd_brick_op_build_payload(glusterd_op_t op,
+                                glusterd_brickinfo_t *brickinfo,
+                                gd1_mgmt_brick_op_req **req, dict_t *dict)
 {
-        int                     ret = -1;
-        gd1_mgmt_brick_op_req   *brick_req = NULL;
-        char                    *volname = NULL;
-        char                    name[1024] = {0,};
-        gf_xl_afr_op_t          heal_op = GF_SHD_OP_INVALID;
-        xlator_t                *this = NULL;
-        glusterd_volinfo_t        *volinfo      = NULL;
-
-        this = THIS;
-        GF_ASSERT (this);
-
-        GF_ASSERT (op < GD_OP_MAX);
-        GF_ASSERT (op > GD_OP_NONE);
-        GF_ASSERT (req);
-
-
-        switch (op) {
+    int ret = -1;
+    gd1_mgmt_brick_op_req *brick_req = NULL;
+    char *volname = NULL;
+    char name[1024] = {
+        0,
+    };
+    gf_xl_afr_op_t heal_op = GF_SHD_OP_INVALID;
+    xlator_t *this = NULL;
+    glusterd_volinfo_t *volinfo = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+
+    GF_ASSERT(op < GD_OP_MAX);
+    GF_ASSERT(op > GD_OP_NONE);
+    GF_ASSERT(req);
+
+    switch (op) {
         case GD_OP_REMOVE_BRICK:
         case GD_OP_STOP_VOLUME:
-                brick_req = GF_CALLOC (1, sizeof (*brick_req),
-                                       gf_gld_mt_mop_brick_req_t);
-                if (!brick_req)
-                        goto out;
-                brick_req->op = GLUSTERD_BRICK_TERMINATE;
-                brick_req->name = "";
-        break;
+            brick_req = GF_CALLOC(1, sizeof(*brick_req),
+                                  gf_gld_mt_mop_brick_req_t);
+            if (!brick_req) {
+                gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_NO_MEMORY,
+                        NULL);
+                goto out;
+            }
+            brick_req->op = GLUSTERD_BRICK_TERMINATE;
+            brick_req->name = brickinfo->path;
+            glusterd_set_brick_status(brickinfo, GF_BRICK_STOPPING);
+            break;
         case GD_OP_PROFILE_VOLUME:
-                brick_req = GF_CALLOC (1, sizeof (*brick_req),
-                                       gf_gld_mt_mop_brick_req_t);
-
-                if (!brick_req)
-                        goto out;
-
-                brick_req->op = GLUSTERD_BRICK_XLATOR_INFO;
-                brick_req->name = brickinfo->path;
-
-                break;
-        case GD_OP_HEAL_VOLUME:
-        {
-                brick_req = GF_CALLOC (1, sizeof (*brick_req),
-                                       gf_gld_mt_mop_brick_req_t);
-                if (!brick_req)
-                        goto out;
-
-                brick_req->op = GLUSTERD_BRICK_XLATOR_OP;
-                brick_req->name = "";
-                ret = dict_get_int32 (dict, "heal-op", (int32_t*)&heal_op);
-                if (ret)
-                        goto out;
-                ret = dict_set_int32 (dict, "xl-op", heal_op);
-        }
-                break;
-        case GD_OP_STATUS_VOLUME:
-        {
-                brick_req = GF_CALLOC (1, sizeof (*brick_req),
-                                       gf_gld_mt_mop_brick_req_t);
-                if (!brick_req)
-                        goto out;
-                brick_req->op = GLUSTERD_BRICK_STATUS;
-                brick_req->name = "";
-        }
-                break;
+            brick_req = GF_CALLOC(1, sizeof(*brick_req),
+                                  gf_gld_mt_mop_brick_req_t);
+
+            if (!brick_req) {
+                gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_NO_MEMORY,
+                        NULL);
+                goto out;
+            }
+
+            brick_req->op = GLUSTERD_BRICK_XLATOR_INFO;
+            brick_req->name = brickinfo->path;
+
+            break;
+        case GD_OP_HEAL_VOLUME: {
+            brick_req = GF_CALLOC(1, sizeof(*brick_req),
+                                  gf_gld_mt_mop_brick_req_t);
+            if (!brick_req) {
+                gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_NO_MEMORY,
+                        NULL);
+                goto out;
+            }
+
+            brick_req->op = GLUSTERD_BRICK_XLATOR_OP;
+            brick_req->name = "";
+            ret = dict_get_int32n(dict, "heal-op", SLEN("heal-op"),
+                                  (int32_t *)&heal_op);
+            if (ret) {
+                gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_GET_FAILED,
+                        "Key=heal-op", NULL);
+                goto out;
+            }
+            ret = dict_set_int32n(dict, "xl-op", SLEN("xl-op"), heal_op);
+            if (ret) {
+                gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_SET_FAILED,
+                        "Key=xl-op", NULL);
+                goto out;
+            }
+        } break;
+        case GD_OP_STATUS_VOLUME: {
+            brick_req = GF_CALLOC(1, sizeof(*brick_req),
+                                  gf_gld_mt_mop_brick_req_t);
+            if (!brick_req) {
+                gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_NO_MEMORY,
+                        NULL);
+                goto out;
+            }
+            brick_req->op = GLUSTERD_BRICK_STATUS;
+            brick_req->name = "";
+            ret = dict_set_strn(dict, "brick-name", SLEN("brick-name"),
+                                brickinfo->path);
+            if (ret) {
+                gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_SET_FAILED,
+                        "Key=brick-name", NULL);
+                goto out;
+            }
+        } break;
         case GD_OP_REBALANCE:
         case GD_OP_DEFRAG_BRICK_VOLUME:
-                brick_req = GF_CALLOC (1, sizeof (*brick_req),
-                                       gf_gld_mt_mop_brick_req_t);
-                if (!brick_req)
-                        goto out;
-
-                brick_req->op = GLUSTERD_BRICK_XLATOR_DEFRAG;
-                ret = dict_get_str (dict, "volname", &volname);
-                if (ret)
-                        goto out;
-                ret = glusterd_volinfo_find (volname, &volinfo);
-                if (volinfo->type == GF_CLUSTER_TYPE_TIER)
-                        snprintf (name, 1024, "tier-dht");
-                else
-                        snprintf (name, 1024, "%s-dht", volname);
-                brick_req->name = gf_strdup (name);
-
-                break;
+            brick_req = GF_CALLOC(1, sizeof(*brick_req),
+                                  gf_gld_mt_mop_brick_req_t);
+            if (!brick_req) {
+                gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_NO_MEMORY,
+                        NULL);
+                goto out;
+            }
+
+            brick_req->op = GLUSTERD_BRICK_XLATOR_DEFRAG;
+            ret = dict_get_strn(dict, "volname", SLEN("volname"), &volname);
+            if (ret) {
+                gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_GET_FAILED,
+                        "Key=volname", NULL);
+                goto out;
+            }
+            ret = glusterd_volinfo_find(volname, &volinfo);
+            if (ret) {
+                gf_smsg(this->name, GF_LOG_ERROR, errno,
+                        GD_MSG_VOLINFO_GET_FAIL, "Volume=%s", volname, NULL);
+                goto out;
+            }
+            snprintf(name, sizeof(name), "%s-dht", volname);
+            brick_req->name = gf_strdup(name);
+
+            break;
         case GD_OP_SNAP:
-                brick_req = GF_CALLOC (1, sizeof (*brick_req),
-                                       gf_gld_mt_mop_brick_req_t);
-                if (!brick_req)
-                        goto out;
-
-                brick_req->op = GLUSTERD_BRICK_BARRIER;
-                ret = dict_get_str (dict, "volname", &volname);
-                if (ret)
-                        goto out;
-                brick_req->name = gf_strdup (volname);
-
-                break;
         case GD_OP_BARRIER:
-                brick_req = GF_CALLOC (1, sizeof(*brick_req),
-                                       gf_gld_mt_mop_brick_req_t);
-                if (!brick_req)
-                        goto out;
-                brick_req->op = GLUSTERD_BRICK_BARRIER;
-                ret = dict_get_str(dict, "volname", &volname);
-                if (ret)
-                        goto out;
-                brick_req->name = gf_strdup (volname);
-                break;
-
-        default:
+            brick_req = GF_CALLOC(1, sizeof(*brick_req),
+                                  gf_gld_mt_mop_brick_req_t);
+            if (!brick_req) {
+                gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_NO_MEMORY,
+                        NULL);
                 goto out;
-        break;
-        }
+            }
+            brick_req->op = GLUSTERD_BRICK_BARRIER;
+            brick_req->name = brickinfo->path;
+            break;
 
-        ret = dict_allocate_and_serialize (dict, &brick_req->input.input_val,
-                                           &brick_req->input.input_len);
-        if (ret)
-                goto out;
-        *req = brick_req;
-        ret = 0;
+        default:
+            goto out;
+            break;
+    }
+
+    brick_req->dict.dict_len = 0;
+    brick_req->dict.dict_val = NULL;
+    ret = dict_allocate_and_serialize(dict, &brick_req->input.input_val,
+                                      &brick_req->input.input_len);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno,
+                GD_MSG_DICT_ALLOC_AND_SERL_LENGTH_GET_FAIL, NULL);
+        goto out;
+    }
+    *req = brick_req;
+    ret = 0;
 
 out:
-        if (ret && brick_req)
-                GF_FREE (brick_req);
-        gf_log (this->name, GF_LOG_DEBUG, "Returning %d", ret);
-        return ret;
+    if (ret && brick_req)
+        GF_FREE(brick_req);
+    gf_msg_debug(this->name, 0, "Returning %d", ret);
+    return ret;
 }
 
 int
-glusterd_node_op_build_payload (glusterd_op_t op, gd1_mgmt_brick_op_req **req,
-                                dict_t *dict)
+glusterd_node_op_build_payload(glusterd_op_t op, gd1_mgmt_brick_op_req **req,
+                               dict_t *dict)
 {
-        int                     ret = -1;
-        gd1_mgmt_brick_op_req   *brick_req = NULL;
-
-        GF_ASSERT (op < GD_OP_MAX);
-        GF_ASSERT (op > GD_OP_NONE);
-        GF_ASSERT (req);
-
-        switch (op) {
+    int ret = -1;
+    gd1_mgmt_brick_op_req *brick_req = NULL;
+    char *volname = NULL;
+
+    GF_ASSERT(op < GD_OP_MAX);
+    GF_ASSERT(op > GD_OP_NONE);
+    GF_ASSERT(req);
+    xlator_t *this = NULL;
+    this = THIS;
+    GF_ASSERT(this);
+
+    switch (op) {
         case GD_OP_PROFILE_VOLUME:
-                brick_req = GF_CALLOC (1, sizeof (*brick_req),
-                                       gf_gld_mt_mop_brick_req_t);
-                if (!brick_req)
-                        goto out;
+            brick_req = GF_CALLOC(1, sizeof(*brick_req),
+                                  gf_gld_mt_mop_brick_req_t);
+            if (!brick_req) {
+                gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_NO_MEMORY,
+                        NULL);
+                goto out;
+            }
 
-                brick_req->op = GLUSTERD_NODE_PROFILE;
-                brick_req->name = "";
+            brick_req->op = GLUSTERD_NODE_PROFILE;
+            brick_req->name = "";
 
-                break;
+            break;
 
         case GD_OP_STATUS_VOLUME:
-                brick_req = GF_CALLOC (1, sizeof (*brick_req),
-                                       gf_gld_mt_mop_brick_req_t);
-                if (!brick_req)
-                        goto out;
+            brick_req = GF_CALLOC(1, sizeof(*brick_req),
+                                  gf_gld_mt_mop_brick_req_t);
+            if (!brick_req) {
+                gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_NO_MEMORY,
+                        NULL);
+                goto out;
+            }
 
-                brick_req->op = GLUSTERD_NODE_STATUS;
-                brick_req->name = "";
+            brick_req->op = GLUSTERD_NODE_STATUS;
+            brick_req->name = "";
 
-                break;
+            break;
 
-        default:
+        case GD_OP_SCRUB_STATUS:
+        case GD_OP_SCRUB_ONDEMAND:
+            brick_req = GF_CALLOC(1, sizeof(*brick_req),
+                                  gf_gld_mt_mop_brick_req_t);
+            if (!brick_req) {
+                gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_NO_MEMORY,
+                        NULL);
                 goto out;
-        }
+            }
 
-        ret = dict_allocate_and_serialize (dict, &brick_req->input.input_val,
-                                           &brick_req->input.input_len);
+            brick_req->op = GLUSTERD_NODE_BITROT;
 
-        if (ret)
+            ret = dict_get_strn(dict, "volname", SLEN("volname"), &volname);
+            if (ret) {
+                gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_GET_FAILED,
+                        "Key=volname", NULL);
                 goto out;
+            }
 
-        *req = brick_req;
-        ret = 0;
+            brick_req->name = gf_strdup(volname);
+            break;
+        default:
+            goto out;
+    }
+
+    brick_req->dict.dict_len = 0;
+    brick_req->dict.dict_val = NULL;
+    ret = dict_allocate_and_serialize(dict, &brick_req->input.input_val,
+                                      &brick_req->input.input_len);
+
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno,
+                GD_MSG_DICT_ALLOC_AND_SERL_LENGTH_GET_FAIL, NULL);
+        goto out;
+    }
+
+    *req = brick_req;
+    ret = 0;
 
 out:
-        if (ret && brick_req)
-                GF_FREE (brick_req);
-        gf_log (THIS->name, GF_LOG_DEBUG, "Returning %d", ret);
-        return ret;
+    if (ret && brick_req)
+        GF_FREE(brick_req);
+    gf_msg_debug(this->name, 0, "Returning %d", ret);
+    return ret;
 }
 
 static int
-glusterd_validate_quorum_options (xlator_t *this, char *fullkey, char *value,
-                                  char **op_errstr)
+glusterd_validate_quorum_options(xlator_t *this, char *fullkey, char *value,
+                                 char **op_errstr)
 {
-        int             ret = 0;
-        char            *key = NULL;
-        volume_option_t *opt = NULL;
-
-        if (!glusterd_is_quorum_option (fullkey))
-                goto out;
-        key = strchr (fullkey, '.');
-        if (key == NULL) {
-                ret = -1;
-                goto out;
-        }
-        key++;
-        opt = xlator_volume_option_get (this, key);
-        ret = xlator_option_validate (this, key, value, opt, op_errstr);
+    int ret = 0;
+    char *key = NULL;
+    volume_option_t *opt = NULL;
+
+    if (!glusterd_is_quorum_option(fullkey))
+        goto out;
+    key = strchr(fullkey, '.');
+    if (key == NULL) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_STRCHR_FAIL, NULL);
+        ret = -1;
+        goto out;
+    }
+    key++;
+    opt = xlator_volume_option_get(this, key);
+    if (!opt) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_VOLINFO_GET_FAIL, NULL);
+        ret = -1;
+        goto out;
+    }
+    ret = xlator_option_validate(this, key, value, opt, op_errstr);
 out:
-        return ret;
+    return ret;
 }
 
 static int
-glusterd_op_stage_set_volume (dict_t *dict, char **op_errstr)
+glusterd_validate_brick_mx_options(xlator_t *this, char *fullkey, char *value,
+                                   char **op_errstr)
 {
-        int                             ret                     = -1;
-        char                            *volname                = NULL;
-        int                             exists                  = 0;
-        char                            *key                    = NULL;
-        char                            *key_fixed              = NULL;
-        char                            *value                  = NULL;
-        char                            *val_dup                = NULL;
-        char                            str[100]                = {0, };
-        char                            *trash_path             = NULL;
-        int                             count                   = 0;
-        int                             dict_count              = 0;
-        char                            errstr[2048]            = {0, };
-        glusterd_volinfo_t              *volinfo                = NULL;
-        glusterd_brickinfo_t            *brickinfo              = NULL;
-        dict_t                          *val_dict               = NULL;
-        gf_boolean_t                    global_opt              = _gf_false;
-        glusterd_volinfo_t              *voliter                = NULL;
-        glusterd_conf_t                 *priv                   = NULL;
-        xlator_t                        *this                   = NULL;
-        uint32_t                        new_op_version          = 0;
-        uint32_t                        local_new_op_version    = 0;
-        uint32_t                        key_op_version          = 0;
-        uint32_t                        local_key_op_version    = 0;
-        gf_boolean_t                    origin_glusterd         = _gf_true;
-        gf_boolean_t                    check_op_version        = _gf_true;
-        gf_boolean_t                    trash_enabled           = _gf_false;
-        gf_boolean_t                    all_vol                 = _gf_false;
-        struct          stat            stbuf                   = {0, };
-
-        GF_ASSERT (dict);
-        this = THIS;
-        GF_ASSERT (this);
-        priv = this->private;
-        GF_ASSERT (priv);
-
-        val_dict = dict_new();
-        if (!val_dict)
-                goto out;
-
-        /* Check if we can support the required op-version
-         * This check is not done on the originator glusterd. The originator
-         * glusterd sets this value.
-         */
-        origin_glusterd = is_origin_glusterd (dict);
-
-        if (!origin_glusterd) {
-                /* Check for v3.3.x origin glusterd */
-                check_op_version = dict_get_str_boolean (dict,
-                                                         "check-op-version",
-                                                         _gf_false);
-
-                if (check_op_version) {
-                        ret = dict_get_uint32 (dict, "new-op-version",
-                                               &new_op_version);
-                        if (ret) {
-                                gf_log (this->name, GF_LOG_ERROR,
-                                        "Failed to get new_op_version");
-                                goto out;
-                        }
-
-                        if ((new_op_version > GD_OP_VERSION_MAX) ||
-                            (new_op_version < GD_OP_VERSION_MIN)) {
-                                ret = -1;
-                                snprintf (errstr, sizeof (errstr),
-                                          "Required op_version (%d) is not "
-                                          "supported", new_op_version);
-                                gf_log (this->name, GF_LOG_ERROR, "%s", errstr);
-                                goto out;
-                        }
-                }
-        }
-
-        ret = dict_get_int32 (dict, "count", &dict_count);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "Count(dict),not set in Volume-Set");
-                goto out;
-        }
-
-        if (dict_count == 0) {
-        /*No options would be specified of volume set help */
-                if (dict_get (dict, "help" ))  {
-                        ret = 0;
-                        goto out;
-                }
-
-                if (dict_get (dict, "help-xml" )) {
-#if (HAVE_LIB_XML)
-                        ret = 0;
-                        goto out;
-#else
-                        ret  = -1;
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "libxml not present in the system");
-                        *op_errstr = gf_strdup ("Error: xml libraries not "
-                                                "present to produce xml-output");
-                        goto out;
-#endif
-                }
-                gf_log (this->name, GF_LOG_ERROR, "No options received ");
-                *op_errstr = gf_strdup ("Options not specified");
-                ret = -1;
-                goto out;
-        }
-
-        ret = dict_get_str (dict, "volname", &volname);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "Unable to get volume name");
-                goto out;
-        }
+    int ret = 0;
 
-        if (strcasecmp (volname, "all") != 0) {
-                exists = glusterd_check_volume_exists (volname);
-                if (!exists) {
-                        snprintf (errstr, sizeof (errstr),
-                                  FMTSTR_CHECK_VOL_EXISTS, volname);
-                        gf_log (this->name, GF_LOG_ERROR, "%s", errstr);
-                        ret = -1;
-                        goto out;
-                }
-
-                ret = glusterd_volinfo_find (volname, &volinfo);
-                if (ret) {
-                        gf_log (this->name, GF_LOG_ERROR,
-                                FMTSTR_CHECK_VOL_EXISTS, volname);
-                        goto out;
-                }
+    // Placeholder function for now
 
-                ret = glusterd_validate_volume_id (dict, volinfo);
-                if (ret)
-                        goto out;
-        } else {
-                all_vol = _gf_true;
-        }
-
-        local_new_op_version = priv->op_version;
-
-        for ( count = 1; ret != 1 ; count++ ) {
-                global_opt = _gf_false;
-                sprintf (str, "key%d", count);
-                ret = dict_get_str (dict, str, &key);
-                if (ret)
-                        break;
-
-                sprintf (str, "value%d", count);
-                ret = dict_get_str (dict, str, &value);
-                if (ret) {
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "invalid key,value pair in 'volume set'");
-                        ret = -1;
-                        goto out;
-                }
-
-                if (strcmp (key, "config.memory-accounting") == 0) {
-                        gf_log (this->name, GF_LOG_DEBUG,
-                                "enabling memory accounting for volume %s",
-                                volname);
-                        ret = 0;
-                }
-
-                if (strcmp (key, "config.transport") == 0) {
-                        gf_log (this->name, GF_LOG_DEBUG,
-                                "changing transport-type for volume %s",
-                                volname);
-                        ret = 0;
-                        /* if value is none of 'tcp/rdma/tcp,rdma' error out */
-                        if (!((strcasecmp (value, "rdma") == 0) ||
-                              (strcasecmp (value, "tcp") == 0) ||
-                              (strcasecmp (value, "tcp,rdma") == 0) ||
-                              (strcasecmp (value, "rdma,tcp") == 0))) {
-                                ret = snprintf (errstr, sizeof (errstr),
-                                                "transport-type %s does "
-                                                "not exist", value);
-                                /* lets not bother about above return value,
-                                   its a failure anyways */
-                                ret = -1;
-                                goto out;
-                        }
-                }
-
-                ret = glusterd_check_quota_cmd (key, value, errstr, sizeof (errstr));
-                if (ret)
-                        goto out;
-
-                if (is_key_glusterd_hooks_friendly (key))
-                        continue;
-
-                ret = glusterd_volopt_validate (volinfo, dict, key, value,
-                                                op_errstr);
-                if (ret)
-                        goto out;
-
-                exists = glusterd_check_option_exists (key, &key_fixed);
-                if (exists == -1) {
-                        ret = -1;
-                        goto out;
-                }
-
-                if (!exists) {
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "Option with name: %s does not exist", key);
-                        ret = snprintf (errstr, sizeof (errstr),
-                                       "option : %s does not exist",
-                                       key);
-                        if (key_fixed)
-                                snprintf (errstr + ret, sizeof (errstr) - ret,
-                                          "\nDid you mean %s?", key_fixed);
-                        ret = -1;
-                        goto out;
-                }
-
-                if (key_fixed)
-                        key = key_fixed;
-
-                /* Check if the key is cluster.op-version and set
-                 * local_new_op_version to the value given if possible.
-                 */
-                if (strcmp (key, "cluster.op-version") == 0) {
-                        if (!all_vol) {
-                                ret = -1;
-                                snprintf (errstr, sizeof (errstr), "Option \""
-                                          "%s\" is not valid for a single "
-                                          "volume", key);
-                                goto out;
-                        }
-                        /* Check if cluster.op-version is the only option being
-                         * set
-                         */
-                        if (count != 1) {
-                                ret = -1;
-                                snprintf (errstr, sizeof (errstr), "Option \""
-                                          "%s\" cannot be set along with other "
-                                          "options", key);
-                                goto out;
-                        }
-                        /* Just reusing the variable, but I'm using it for
-                         * storing the op-version from value
-                         */
-                        ret = gf_string2uint (value, &local_key_op_version);
-                        if (ret) {
-                                snprintf (errstr, sizeof (errstr), "invalid "
-                                          "number format \"%s\" in option "
-                                          "\"%s\"", value, key);
-                                gf_log (this->name, GF_LOG_ERROR, "%s", errstr);
-                                goto out;
-                        }
-
-                        if (local_key_op_version > GD_OP_VERSION_MAX ||
-                            local_key_op_version < GD_OP_VERSION_MIN) {
-                                ret = -1;
-                                snprintf (errstr, sizeof (errstr),
-                                          "Required op_version (%d) is not "
-                                          "supported", local_key_op_version);
-                                gf_log (this->name, GF_LOG_ERROR, "%s", errstr);
-                                goto out;
-                        }
-                        if (local_key_op_version > local_new_op_version)
-                                local_new_op_version = local_key_op_version;
-                        goto cont;
-                }
-
-                ALL_VOLUME_OPTION_CHECK (volname, key, ret, op_errstr, out);
-                ret = glusterd_validate_quorum_options (this, key, value,
-                                                        op_errstr);
-                if (ret)
-                        goto out;
+    return ret;
+}
 
-                local_key_op_version = glusterd_get_op_version_for_key (key);
-                if (local_key_op_version > local_new_op_version)
-                        local_new_op_version = local_key_op_version;
-
-                sprintf (str, "op-version%d", count);
-                if (origin_glusterd) {
-                        ret = dict_set_uint32 (dict, str, local_key_op_version);
-                        if (ret) {
-                                gf_log (this->name, GF_LOG_ERROR,
-                                        "Failed to set key-op-version in dict");
-                                goto out;
-                        }
-                } else if (check_op_version) {
-                        ret = dict_get_uint32 (dict, str, &key_op_version);
-                        if (ret) {
-                                gf_log (this->name, GF_LOG_ERROR,
-                                        "Failed to get key-op-version from"
-                                        " dict");
-                                goto out;
-                        }
-                        if (local_key_op_version != key_op_version) {
-                                ret = -1;
-                                snprintf (errstr, sizeof (errstr),
-                                          "option: %s op-version mismatch",
-                                          key);
-                                gf_log (this->name, GF_LOG_ERROR,
-                                        "%s, required op-version = %"PRIu32", "
-                                        "available op-version = %"PRIu32,
-                                        errstr, key_op_version,
-                                        local_key_op_version);
-                                goto out;
-                        }
-                }
+static int
+glusterd_validate_shared_storage(char *value, char *errstr)
+{
+    int32_t ret = -1;
+    int32_t count = -1;
+    char *op = NULL;
+    char hook_script[PATH_MAX] = "";
+    xlator_t *this = NULL;
+    glusterd_conf_t *conf = NULL;
+    int32_t len = 0;
+    glusterd_volinfo_t *volinfo = NULL;
+
+    this = THIS;
+    GF_VALIDATE_OR_GOTO("glusterd", this, out);
+
+    conf = this->private;
+    GF_VALIDATE_OR_GOTO(this->name, conf, out);
+
+    GF_VALIDATE_OR_GOTO(this->name, value, out);
+    GF_VALIDATE_OR_GOTO(this->name, errstr, out);
+
+    if ((strcmp(value, "enable")) && (strcmp(value, "disable"))) {
+        snprintf(errstr, PATH_MAX,
+                 "Invalid option(%s). Valid options "
+                 "are 'enable' and 'disable'",
+                 value);
+        gf_msg(this->name, GF_LOG_ERROR, EINVAL, GD_MSG_INVALID_ENTRY, "%s",
+               errstr);
+        ret = -1;
+        goto out;
+    }
 
-                if (glusterd_check_globaloption (key))
-                        global_opt = _gf_true;
-
-                if (volinfo) {
-                        ret = glusterd_volinfo_get (volinfo,
-                                                VKEY_FEATURES_TRASH, &val_dup);
-                        if (val_dup) {
-                                ret = gf_string2boolean (val_dup,
-                                                        &trash_enabled);
-                                if (ret)
-                                        goto out;
-                        }
-                }
+    len = snprintf(hook_script, sizeof(hook_script),
+                   "%s" GLUSTERD_SHRD_STRG_HOOK_SCRIPT, conf->workdir);
+    if ((len < 0) || (len >= sizeof(hook_script))) {
+        ret = -1;
+        goto out;
+    }
+
+    ret = sys_access(hook_script, R_OK | X_OK);
+    if (ret) {
+        len = snprintf(errstr, PATH_MAX,
+                       "The hook-script (%s) required "
+                       "for this operation is not present. "
+                       "Please install the hook-script "
+                       "and retry",
+                       hook_script);
+        if (len < 0) {
+            strncpy(errstr, "<error>", PATH_MAX);
+        }
+        gf_msg(this->name, GF_LOG_ERROR, ENOENT, GD_MSG_FILE_OP_FAILED, "%s",
+               errstr);
+        goto out;
+    }
+
+    if (!strncmp(value, "disable", SLEN("disable"))) {
+        ret = dict_get_strn(conf->opts, GLUSTERD_SHARED_STORAGE_KEY,
+                            SLEN(GLUSTERD_SHARED_STORAGE_KEY), &op);
+        if (ret || !strncmp(op, "disable", SLEN("disable"))) {
+            snprintf(errstr, PATH_MAX,
+                     "Shared storage volume "
+                     "does not exist. Please enable shared storage"
+                     " for creating shared storage volume.");
+            gf_msg(this->name, GF_LOG_ERROR, 0,
+                   GD_MSG_SHARED_STORAGE_DOES_NOT_EXIST, "%s", errstr);
+            ret = -1;
+            goto out;
+        }
+        goto out;
+    }
+
+    ret = glusterd_volinfo_find(GLUSTER_SHARED_STORAGE, &volinfo);
+    if (!ret) {
+        snprintf(errstr, PATH_MAX,
+                 "Shared storage volume(" GLUSTER_SHARED_STORAGE
+                 ") already exists.");
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_VOL_ALREADY_EXIST, "%s",
+               errstr);
+        ret = -1;
+        goto out;
+    }
+
+    ret = glusterd_count_connected_peers(&count);
+    if (ret) {
+        snprintf(errstr, PATH_MAX,
+                 "Failed to calculate number of connected peers.");
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_PEER_COUNT_GET_FAIL, "%s",
+               errstr);
+        goto out;
+    }
+
+    if (count <= 1) {
+        snprintf(errstr, PATH_MAX,
+                 "More than one node should "
+                 "be up/present in the cluster to enable this option");
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_INSUFFICIENT_UP_NODES, "%s",
+               errstr);
+        ret = -1;
+        goto out;
+    }
 
-                if (!strcmp(key, "features.trash-dir") && trash_enabled) {
-                        if (strchr (value, '/')) {
-                                snprintf (errstr, sizeof (errstr),
-                                          "Path is not allowed as option");
-                                gf_log (this->name, GF_LOG_ERROR,
-                                        "Unable to set the options in 'volume "
-                                        "set': %s", errstr);
-                                ret = -1;
-                                goto out;
-                        }
+out:
+    return ret;
+}
 
-                        list_for_each_entry (brickinfo, &volinfo->bricks,
-                                             brick_list) {
-                                /* Check for local brick */
-                                if (!gf_uuid_compare (brickinfo->uuid, MY_UUID)) {
-                                        trash_path = gf_strdup (brickinfo->path);
-                                        strcat(trash_path, "/");
-                                        strcat(trash_path, value);
-
-                                        /* Checks whether a directory with
-                                           given option exists or not */
-                                        if (!stat(trash_path, &stbuf)) {
-                                                snprintf (errstr, sizeof (errstr),
-                                                          "Path %s exists", value);
-                                                gf_log (this->name, GF_LOG_ERROR,
-                                                        "Unable to set the options in "
-                                                        "'volume set': %s", errstr);
-                                                ret = -1;
-                                                goto out;
-                                        } else {
-                                                gf_log (this->name, GF_LOG_DEBUG,
-                                                        "Directory with given name "
-                                                        "does not exists, continuing");
-                                        }
-
-                                        if (volinfo->status == GLUSTERD_STATUS_STARTED
-                                                && brickinfo->status != GF_BRICK_STARTED) {
-                                                /* If volume is in started state , checks
-                                                   whether bricks are online */
-                                                snprintf (errstr, sizeof (errstr),
-                                                          "One or more bricks are down");
-                                                gf_log (this->name, GF_LOG_ERROR,
-                                                        "Unable to set the options in "
-                                                        "'volume set': %s", errstr);
-                                                ret = -1;
-                                                goto out;
-                                        }
-                                }
-                        }
-                } else if (!strcmp(key, "features.trash-dir") && !trash_enabled) {
-                        snprintf (errstr, sizeof (errstr),
-                                        "Trash translator is not enabled. Use "
-                                        "volume set %s trash on", volname);
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "Unable to set the options in 'volume "
-                                "set': %s", errstr);
-                        ret = -1;
-                        goto out;
-                }
-                ret = dict_set_str (val_dict, key, value);
+static int
+glusterd_validate_localtime_logging(char *value, char *errstr)
+{
+    int32_t ret = -1;
+    xlator_t *this = NULL;
+    glusterd_conf_t *conf = NULL;
+    int already_enabled = 0;
+
+    this = THIS;
+    GF_VALIDATE_OR_GOTO("glusterd", this, out);
+
+    conf = this->private;
+    GF_VALIDATE_OR_GOTO(this->name, conf, out);
+    GF_VALIDATE_OR_GOTO(this->name, value, out);
+
+    already_enabled = gf_log_get_localtime();
+
+    ret = 0;
+    if (strcmp(value, "enable") == 0) {
+        gf_log_set_localtime(1);
+        if (!already_enabled)
+            gf_msg(this->name, GF_LOG_INFO, 0, GD_MSG_LOCALTIME_LOGGING_ENABLE,
+                   "localtime logging enable");
+    } else if (strcmp(value, "disable") == 0) {
+        gf_log_set_localtime(0);
+        if (already_enabled)
+            gf_msg(this->name, GF_LOG_INFO, 0, GD_MSG_LOCALTIME_LOGGING_DISABLE,
+                   "localtime logging disable");
+    } else {
+        ret = -1;
+        GF_VALIDATE_OR_GOTO(this->name, errstr, out);
+        snprintf(errstr, PATH_MAX,
+                 "Invalid option(%s). Valid options "
+                 "are 'enable' and 'disable'",
+                 value);
+        gf_msg(this->name, GF_LOG_ERROR, EINVAL, GD_MSG_INVALID_ENTRY, "%s",
+               errstr);
+    }
 
-                if (ret) {
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "Unable to set the options in 'volume set'");
-                        ret = -1;
-                        goto out;
-                }
+out:
+    return ret;
+}
 
-                *op_errstr = NULL;
-                if (!global_opt && !all_vol)
-                        ret = glusterd_validate_reconfopts (volinfo, val_dict, op_errstr);
-                else if (!all_vol) {
-                        voliter = NULL;
-                        cds_list_for_each_entry (voliter, &priv->volumes,
-                                                 vol_list) {
-                                ret = glusterd_validate_globalopts (voliter,
-                                                                    val_dict,
-                                                                    op_errstr);
-                                if (ret)
-                                        break;
-                        }
-                }
+static int
+glusterd_validate_daemon_log_level(char *value, char *errstr)
+{
+    int32_t ret = -1;
+    xlator_t *this = NULL;
+    glusterd_conf_t *conf = NULL;
 
-                if (ret) {
-                        gf_log (this->name, GF_LOG_ERROR, "Could not create "
-                                "temp volfile, some option failed: %s",
-                                *op_errstr);
-                        goto out;
-                }
-                dict_del (val_dict, key);
+    this = THIS;
+    GF_VALIDATE_OR_GOTO("glusterd", this, out);
 
-                if (key_fixed) {
-                        GF_FREE (key_fixed);
-                        key_fixed = NULL;
-                }
-        }
+    conf = this->private;
+    GF_VALIDATE_OR_GOTO(this->name, conf, out);
 
-        // Check if all the connected clients support the new op-version
-        ret = glusterd_check_client_op_version_support (volname,
-                                                        local_new_op_version,
-                                                        op_errstr);
-        if (ret)
-                goto out;
+    GF_VALIDATE_OR_GOTO(this->name, value, out);
 
-cont:
-        if (origin_glusterd) {
-                ret = dict_set_uint32 (dict, "new-op-version",
-                                       local_new_op_version);
-                if (ret) {
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "Failed to set new-op-version in dict");
-                        goto out;
-                }
-                /* Set this value in dict so other peers know to check for
-                 * op-version. This is a hack for 3.3.x compatibility
-                 *
-                 * TODO: Remove this and the other places this is referred once
-                 * 3.3.x compatibility is not required
-                 */
-                ret = dict_set_uint32 (dict, "check-op-version",
-                                       _gf_true);
-                if (ret) {
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "Failed to set check-op-version in dict");
-                        goto out;
-                }
-        }
+    ret = 0;
 
-        ret = 0;
+    if ((strcmp(value, "INFO")) && (strcmp(value, "WARNING")) &&
+        (strcmp(value, "DEBUG")) && (strcmp(value, "TRACE")) &&
+        (strcmp(value, "ERROR"))) {
+        ret = -1;
+        GF_VALIDATE_OR_GOTO(this->name, errstr, out);
+        snprintf(errstr, PATH_MAX,
+                 "Invalid option(%s). Valid options "
+                 "are 'INFO' or 'WARNING' or 'ERROR' or 'DEBUG' or "
+                 " 'TRACE'",
+                 value);
+        gf_msg(this->name, GF_LOG_ERROR, EINVAL, GD_MSG_INVALID_ENTRY, "%s",
+               errstr);
+    }
 
 out:
-        if (val_dict)
-                dict_unref (val_dict);
-
-        if (trash_path)
-                GF_FREE (trash_path);
-
-        GF_FREE (key_fixed);
-        if (errstr[0] != '\0')
-                *op_errstr = gf_strdup (errstr);
-
-        if (ret) {
-                if (!(*op_errstr)) {
-                        *op_errstr = gf_strdup ("Error, Validation Failed");
-                        gf_log (this->name, GF_LOG_DEBUG,
-                                "Error, Cannot Validate option :%s",
-                                *op_errstr);
-                } else {
-                        gf_log (this->name, GF_LOG_DEBUG,
-                                "Error, Cannot Validate option");
-                }
-        }
-        return ret;
+    return ret;
 }
 
 static int
-glusterd_op_stage_reset_volume (dict_t *dict, char **op_errstr)
+glusterd_op_stage_set_volume(dict_t *dict, char **op_errstr)
 {
-        int                                      ret           = 0;
-        char                                    *volname       = NULL;
-        int                                     exists         = 0;
-        char                                    msg[2048]      = {0};
-        char                                    *key = NULL;
-        char                                    *key_fixed = NULL;
-        glusterd_volinfo_t                      *volinfo       = NULL;
-        xlator_t                                *this = NULL;
+    int ret = -1;
+    char *volname = NULL;
+    int exists = 0;
+    char *key = NULL;
+    char *key_fixed = NULL;
+    char *value = NULL;
+    char *val_dup = NULL;
+    char keystr[100] = {
+        0,
+    };
+    int keystr_len;
+    int keylen;
+    char *trash_path = NULL;
+    int trash_path_len = 0;
+    int count = 0;
+    int dict_count = 0;
+    char errstr[PATH_MAX] = {
+        0,
+    };
+    glusterd_volinfo_t *volinfo = NULL;
+    glusterd_brickinfo_t *brickinfo = NULL;
+    dict_t *val_dict = NULL;
+    gf_boolean_t global_opt = _gf_false;
+    gf_boolean_t key_matched = _gf_false; /* if a key was processed or not*/
+    glusterd_volinfo_t *voliter = NULL;
+    glusterd_conf_t *priv = NULL;
+    xlator_t *this = NULL;
+    uint32_t new_op_version = GD_OP_VERSION_MIN;
+    uint32_t local_new_op_version = GD_OP_VERSION_MIN;
+    uint32_t local_new_client_op_version = GD_OP_VERSION_MIN;
+    uint32_t key_op_version = GD_OP_VERSION_MIN;
+    uint32_t local_key_op_version = GD_OP_VERSION_MIN;
+    gf_boolean_t origin_glusterd = _gf_true;
+    gf_boolean_t check_op_version = _gf_true;
+    gf_boolean_t trash_enabled = _gf_false;
+    gf_boolean_t all_vol = _gf_false;
+    struct volopt_map_entry *vmep = NULL;
+
+    GF_ASSERT(dict);
+    this = THIS;
+    GF_ASSERT(this);
+    priv = this->private;
+    GF_ASSERT(priv);
+
+    /* Check if we can support the required op-version
+     * This check is not done on the originator glusterd. The originator
+     * glusterd sets this value.
+     */
+    origin_glusterd = is_origin_glusterd(dict);
+
+    if (!origin_glusterd) {
+        /* Check for v3.3.x origin glusterd */
+        check_op_version = dict_get_str_boolean(dict, "check-op-version",
+                                                _gf_false);
 
-        this = THIS;
-        GF_ASSERT (this);
-
-        ret = dict_get_str (dict, "volname", &volname);
+        if (check_op_version) {
+            ret = dict_get_uint32(dict, "new-op-version", &new_op_version);
+            if (ret) {
+                gf_smsg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+                        "Key=new-op-version", NULL);
+                goto out;
+            }
 
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "Unable to get volume name");
+            if ((new_op_version > GD_OP_VERSION_MAX) ||
+                (new_op_version < GD_OP_VERSION_MIN)) {
+                ret = -1;
+                snprintf(errstr, sizeof(errstr),
+                         "Required op_version (%d) is not supported."
+                         " Max supported op version is %d",
+                         new_op_version, priv->op_version);
+                gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_UNSUPPORTED_VERSION,
+                       "%s", errstr);
                 goto out;
+            }
         }
+    }
 
-        if (strcasecmp (volname, "all") != 0) {
-                exists = glusterd_check_volume_exists (volname);
-                if (!exists) {
-                        snprintf (msg, sizeof (msg), FMTSTR_CHECK_VOL_EXISTS,
-                                  volname);
-                        ret = -1;
-                        goto out;
-                }
-                ret = glusterd_volinfo_find (volname, &volinfo);
-                if (ret) {
-                        snprintf (msg, sizeof (msg), FMTSTR_CHECK_VOL_EXISTS,
-                                  volname);
-                        goto out;
-                }
+    ret = dict_get_int32_sizen(dict, "count", &dict_count);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+               "Count(dict),not set in Volume-Set");
+        goto out;
+    }
 
-                ret = glusterd_validate_volume_id (dict, volinfo);
-                if (ret)
-                        goto out;
+    if (dict_count == 0) {
+        /*No options would be specified of volume set help */
+        if (dict_get_sizen(dict, "help")) {
+            ret = 0;
+            goto out;
         }
 
-        ret = dict_get_str (dict, "key", &key);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "Unable to get option key");
-                goto out;
+        if (dict_get_sizen(dict, "help-xml")) {
+#if (HAVE_LIB_XML)
+            ret = 0;
+            goto out;
+#else
+            ret = -1;
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_MODULE_NOT_INSTALLED,
+                   "libxml not present in the system");
+            *op_errstr = gf_strdup(
+                "Error: xml libraries not present to produce xml-output");
+            goto out;
+#endif
         }
-        if (strcmp(key, "all")) {
-                exists = glusterd_check_option_exists (key, &key_fixed);
-                if (exists == -1) {
-                        ret = -1;
-                        goto out;
-                }
-
-                if (!exists) {
-                        ret = snprintf (msg, sizeof (msg),
-                                        "Option %s does not exist", key);
-                        if (key_fixed)
-                                snprintf (msg + ret, sizeof (msg) - ret,
-                                          "\nDid you mean %s?", key_fixed);
-                        ret = -1;
-                        goto out;
-                } else if (exists > 0) {
-                        if (key_fixed)
-                                key = key_fixed;
-                        ALL_VOLUME_OPTION_CHECK (volname, key, ret,
-                                                 op_errstr, out);
-                }
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_NO_OPTIONS_GIVEN,
+               "No options received ");
+        *op_errstr = gf_strdup("Options not specified");
+        ret = -1;
+        goto out;
+    }
+
+    ret = dict_get_str_sizen(dict, "volname", &volname);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+                "Key=volname", NULL);
+        goto out;
+    }
+
+    if (strcasecmp(volname, "all") != 0) {
+        ret = glusterd_volinfo_find(volname, &volinfo);
+        if (ret) {
+            snprintf(errstr, sizeof(errstr), FMTSTR_CHECK_VOL_EXISTS, volname);
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_VOL_NOT_FOUND,
+                   FMTSTR_CHECK_VOL_EXISTS, volname);
+            goto out;
         }
 
-out:
-        GF_FREE (key_fixed);
+        ret = glusterd_validate_volume_id(dict, volinfo);
+        if (ret)
+            goto out;
 
-        if (msg[0] != '\0') {
-                gf_log (this->name, GF_LOG_ERROR, "%s", msg);
-                *op_errstr = gf_strdup (msg);
-        }
+        local_new_op_version = volinfo->op_version;
+        local_new_client_op_version = volinfo->client_op_version;
 
-        gf_log (this->name, GF_LOG_DEBUG, "Returning %d", ret);
-
-        return ret;
-}
+    } else {
+        all_vol = _gf_true;
+    }
 
+    val_dict = dict_new();
+    if (!val_dict) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_CREATE_FAIL, NULL);
+        goto out;
+    }
 
+    for (count = 1; ret != 1; count++) {
+        keystr_len = sprintf(keystr, "key%d", count);
+        ret = dict_get_strn(dict, keystr, keystr_len, &key);
+        if (ret) {
+            gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_GET_FAILED,
+                    "Key=%s", keystr, NULL);
+            break;
+        }
 
-static int
-glusterd_op_stage_sync_volume (dict_t *dict, char **op_errstr)
-{
-        int                                     ret = -1;
-        char                                    *volname = NULL;
-        char                                    *hostname = NULL;
-        gf_boolean_t                            exists = _gf_false;
-        glusterd_peerinfo_t                     *peerinfo = NULL;
-        char                                    msg[2048] = {0,};
-        glusterd_volinfo_t                      *volinfo  = NULL;
-
-        ret = dict_get_str (dict, "hostname", &hostname);
+        keystr_len = sprintf(keystr, "value%d", count);
+        ret = dict_get_strn(dict, keystr, keystr_len, &value);
         if (ret) {
-                snprintf (msg, sizeof (msg), "hostname couldn't be "
-                          "retrieved from msg");
-                *op_errstr = gf_strdup (msg);
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+                   "invalid key,value pair in 'volume set'");
+            ret = -1;
+            goto out;
+        }
+
+        key_matched = _gf_false;
+        keylen = strlen(key);
+        if (len_strcmp(key, keylen, "config.memory-accounting")) {
+            key_matched = _gf_true;
+            gf_msg_debug(this->name, 0,
+                         "enabling memory accounting for volume %s", volname);
+            ret = 0;
+        } else if (len_strcmp(key, keylen, "config.transport")) {
+            key_matched = _gf_true;
+            gf_msg_debug(this->name, 0, "changing transport-type for volume %s",
+                         volname);
+            ret = 0;
+            /* if value is none of 'tcp/rdma/tcp,rdma' error out */
+            if (!((strcasecmp(value, "rdma") == 0) ||
+                  (strcasecmp(value, "tcp") == 0) ||
+                  (strcasecmp(value, "tcp,rdma") == 0) ||
+                  (strcasecmp(value, "rdma,tcp") == 0))) {
+                ret = snprintf(errstr, sizeof(errstr),
+                               "transport-type %s does not exist", value);
+                /* lets not bother about above return value,
+                   its a failure anyways */
+                ret = -1;
                 goto out;
+            }
+        } else if (len_strcmp(key, keylen, "ganesha.enable")) {
+            key_matched = _gf_true;
+            if (!strcmp(value, "off") == 0) {
+                ret = ganesha_manage_export(dict, "off", _gf_true, op_errstr);
+                if (ret)
+                    goto out;
+            }
         }
 
-        if (gf_is_local_addr (hostname)) {
-                //volname is not present in case of sync all
-                ret = dict_get_str (dict, "volname", &volname);
-                if (!ret) {
-                        exists = glusterd_check_volume_exists (volname);
-                        if (!exists) {
-                                snprintf (msg, sizeof (msg), "Volume %s "
-                                         "does not exist", volname);
-                                *op_errstr = gf_strdup (msg);
-                                ret = -1;
-                                goto out;
-                        }
-                        ret = glusterd_volinfo_find (volname, &volinfo);
-                        if (ret)
-                                goto out;
-
-                } else {
-                        ret = 0;
-                }
-         } else {
-                 rcu_read_lock ();
-
-                 peerinfo = glusterd_peerinfo_find (NULL, hostname);
-                 if (peerinfo == NULL) {
-                         ret = -1;
-                         snprintf (msg, sizeof (msg), "%s, is not a friend",
-                                         hostname);
-                         *op_errstr = gf_strdup (msg);
-
-                 } else if (!peerinfo->connected) {
-                         snprintf (msg, sizeof (msg), "%s, is not connected at "
-                                         "the moment", hostname);
-                         *op_errstr = gf_strdup (msg);
-                         ret = -1;
-                 }
-
-                 rcu_read_unlock ();
+        if (!key_matched) {
+            ret = glusterd_check_bitrot_cmd(key, keylen, errstr,
+                                            sizeof(errstr));
+            if (ret)
+                goto out;
+            ret = glusterd_check_quota_cmd(key, keylen, value, errstr,
+                                           sizeof(errstr));
+            if (ret)
+                goto out;
         }
 
-out:
-        gf_log ("", GF_LOG_DEBUG, "Returning %d", ret);
-
-        return ret;
-}
+        if (is_key_glusterd_hooks_friendly(key))
+            continue;
 
-static int
-glusterd_op_stage_status_volume (dict_t *dict, char **op_errstr)
-{
-        int                    ret            = -1;
-        uint32_t               cmd            = 0;
-        char                   msg[2048]      = {0,};
-        char                  *volname        = NULL;
-        char                  *brick          = NULL;
-        xlator_t              *this           = NULL;
-        glusterd_conf_t       *priv           = NULL;
-        glusterd_brickinfo_t  *brickinfo      = NULL;
-        glusterd_volinfo_t    *volinfo        = NULL;
-        dict_t                *vol_opts       = NULL;
-        gf_boolean_t           nfs_disabled   = _gf_false;
-        gf_boolean_t           shd_enabled    = _gf_true;
-
-        GF_ASSERT (dict);
-        this = THIS;
-        GF_ASSERT (this);
-        priv = this->private;
-        GF_ASSERT(priv);
-
-        ret = dict_get_uint32 (dict, "cmd", &cmd);
+        ret = glusterd_volopt_validate(volinfo, dict, key, value, op_errstr);
         if (ret)
-                goto out;
-
-        if (cmd & GF_CLI_STATUS_ALL)
-                goto out;
-
-        if ((cmd & GF_CLI_STATUS_QUOTAD) &&
-            (priv->op_version == GD_OP_VERSION_MIN)) {
-                snprintf (msg, sizeof (msg), "The cluster is operating at "
-                          "version 1. Getting the status of quotad is not "
-                          "allowed in this state.");
+            goto out;
+
+        exists = glusterd_check_option_exists(key, &key_fixed);
+        if (exists == -1) {
+            ret = -1;
+            goto out;
+        }
+
+        if (!exists) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_INVALID_ENTRY,
+                   "Option with name: %s does not exist", key);
+            ret = snprintf(errstr, sizeof(errstr), "option : %s does not exist",
+                           key);
+            if (key_fixed)
+                snprintf(errstr + ret, sizeof(errstr) - ret,
+                         "\nDid you mean %s?", key_fixed);
+            ret = -1;
+            goto out;
+        }
+
+        if (key_fixed) {
+            key = key_fixed;
+            keylen = strlen(key_fixed);
+        }
+
+        if (len_strcmp(key, keylen, "cluster.granular-entry-heal")) {
+            /* For granular entry-heal, if the set command was
+             * invoked through volume-set CLI, then allow the
+             * command only if the volume is still in 'Created'
+             * state
+             */
+            if (volinfo && volinfo->status != GLUSTERD_STATUS_NONE &&
+                (dict_get_sizen(dict, "is-special-key") == NULL)) {
+                snprintf(errstr, sizeof(errstr),
+                         " 'gluster volume set <VOLNAME> %s {enable, disable}'"
+                         " is not supported."
+                         " Use 'gluster volume heal <VOLNAME> "
+                         "granular-entry-heal {enable, disable}' instead.",
+                         key);
                 ret = -1;
                 goto out;
-        }
-
-        if ((cmd & GF_CLI_STATUS_SNAPD) &&
-            (priv->op_version < GD_OP_VERSION_3_6_0)) {
-                snprintf (msg, sizeof (msg), "The cluster is operating at "
-                          "version less than %d. Getting the "
-                          "status of snapd is not allowed in this state.",
-                          GD_OP_VERSION_3_6_0);
+            }
+        } else if (len_strcmp(key, keylen, GLUSTERD_GLOBAL_OP_VERSION_KEY)) {
+            /* Check if the key is cluster.op-version and set
+             * local_new_op_version to the value given if possible.
+             */
+            if (!all_vol) {
                 ret = -1;
-                goto out;
-        }
-
-        ret = dict_get_str (dict, "volname", &volname);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "Unable to get volume name");
-                goto out;
-        }
-
-        ret = glusterd_volinfo_find (volname, &volinfo);
-        if (ret) {
-                snprintf (msg, sizeof(msg), FMTSTR_CHECK_VOL_EXISTS, volname);
+                snprintf(errstr, sizeof(errstr),
+                         "Option \"%s\" is not valid for a single volume", key);
+                goto out;
+            }
+            /* Check if cluster.op-version is the only option being
+             * set
+             */
+            if (count != 1) {
                 ret = -1;
+                snprintf(errstr, sizeof(errstr),
+                         "Option \"%s\" cannot be set along with other options",
+                         key);
+                goto out;
+            }
+            /* Just reusing the variable, but I'm using it for
+             * storing the op-version from value
+             */
+            ret = gf_string2uint(value, &local_key_op_version);
+            if (ret) {
+                snprintf(errstr, sizeof(errstr),
+                         "invalid number format \"%s\" in option \"%s\"", value,
+                         key);
+                gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_INVALID_ENTRY, "%s",
+                       errstr);
+                goto out;
+            }
+
+            if (local_key_op_version > GD_OP_VERSION_MAX ||
+                local_key_op_version < GD_OP_VERSION_MIN) {
+                ret = -1;
+                snprintf(errstr, sizeof(errstr),
+                         "Required op_version (%d) is not supported."
+                         " Max supported op version is %d",
+                         local_key_op_version, priv->op_version);
+                gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_VERSION_UNSUPPORTED,
+                       "%s", errstr);
+                goto out;
+            }
+            if (local_key_op_version > priv->op_version) {
+                local_new_op_version = local_key_op_version;
+            } else {
+                ret = -1;
+                snprintf(errstr, sizeof(errstr),
+                         "Required op-version (%d) should"
+                         " not be equal or lower than current"
+                         " cluster op-version (%d).",
+                         local_key_op_version, priv->op_version);
+                gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_VERSION_UNSUPPORTED,
+                       "%s", errstr);
                 goto out;
+            }
+
+            goto cont;
         }
 
-        ret = glusterd_validate_volume_id (dict, volinfo);
+        ALL_VOLUME_OPTION_CHECK(volname, _gf_false, key, ret, op_errstr, out);
+        ret = glusterd_validate_quorum_options(this, key, value, op_errstr);
         if (ret)
-                goto out;
+            goto out;
 
-        ret = glusterd_is_volume_started (volinfo);
-        if (!ret) {
-                snprintf (msg, sizeof (msg), "Volume %s is not started",
-                          volname);
-                ret = -1;
-                goto out;
-        }
+        ret = glusterd_validate_brick_mx_options(this, key, value, op_errstr);
+        if (ret)
+            goto out;
 
-        vol_opts = volinfo->dict;
+        vmep = gd_get_vmep(key);
+        local_key_op_version = glusterd_get_op_version_from_vmep(vmep);
+        if (local_key_op_version > local_new_op_version)
+            local_new_op_version = local_key_op_version;
+        if (gd_is_client_option(vmep) &&
+            (local_key_op_version > local_new_client_op_version))
+            local_new_client_op_version = local_key_op_version;
 
-        if ((cmd & GF_CLI_STATUS_NFS) != 0) {
-                nfs_disabled = dict_get_str_boolean (vol_opts, "nfs.disable",
-                                                     _gf_false);
-                if (nfs_disabled) {
-                        ret = -1;
-                        snprintf (msg, sizeof (msg),
-                                  "NFS server is disabled for volume %s",
-                                  volname);
-                        goto out;
-                }
-        } else if ((cmd & GF_CLI_STATUS_SHD) != 0) {
-                if (!glusterd_is_volume_replicate (volinfo)) {
-                        ret = -1;
-                        snprintf (msg, sizeof (msg),
-                                  "Volume %s is not of type replicate",
-                                  volname);
+        sprintf(keystr, "op-version%d", count);
+        if (origin_glusterd) {
+            ret = dict_set_uint32(dict, keystr, local_key_op_version);
+            if (ret) {
+                gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+                       "Failed to set key-op-version in dict");
+                goto out;
+            }
+        } else if (check_op_version) {
+            ret = dict_get_uint32(dict, keystr, &key_op_version);
+            if (ret) {
+                gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+                       "Failed to get key-op-version from dict");
+                goto out;
+            }
+            if (local_key_op_version != key_op_version) {
+                ret = -1;
+                snprintf(errstr, sizeof(errstr),
+                         "option: %s op-version mismatch", key);
+                gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_OP_VERSION_MISMATCH,
+                       "%s, required op-version = %" PRIu32
+                       ", available op-version = %" PRIu32,
+                       errstr, key_op_version, local_key_op_version);
+                goto out;
+            }
+        }
+
+        global_opt = glusterd_check_globaloption(key);
+
+        if (len_strcmp(key, keylen, GLUSTERD_SHARED_STORAGE_KEY)) {
+            ret = glusterd_validate_shared_storage(value, errstr);
+            if (ret) {
+                gf_msg(this->name, GF_LOG_ERROR, 0,
+                       GD_MSG_SHARED_STRG_VOL_OPT_VALIDATE_FAIL,
+                       "Failed to validate shared storage volume options");
+                goto out;
+            }
+        } else if (len_strcmp(key, keylen, GLUSTERD_LOCALTIME_LOGGING_KEY)) {
+            ret = glusterd_validate_localtime_logging(value, errstr);
+            if (ret) {
+                gf_msg(this->name, GF_LOG_ERROR, 0,
+                       GD_MSG_LOCALTIME_LOGGING_VOL_OPT_VALIDATE_FAIL,
+                       "Failed to validate localtime logging volume options");
+                goto out;
+            }
+        } else if (len_strcmp(key, keylen, GLUSTERD_DAEMON_LOG_LEVEL_KEY)) {
+            ret = glusterd_validate_daemon_log_level(value, errstr);
+            if (ret) {
+                gf_msg(this->name, GF_LOG_ERROR, 0,
+                       GD_MSG_DAEMON_LOG_LEVEL_VOL_OPT_VALIDATE_FAIL,
+                       "Failed to validate daemon-log-level volume options");
+                goto out;
+            }
+        } else if (len_strcmp(key, keylen, "features.trash-dir")) {
+            if (volinfo) {
+                ret = glusterd_volinfo_get(volinfo, VKEY_FEATURES_TRASH,
+                                           &val_dup);
+                if (!ret && val_dup) {
+                    ret = gf_string2boolean(val_dup, &trash_enabled);
+                    if (ret)
                         goto out;
                 }
-
-                shd_enabled = dict_get_str_boolean (vol_opts,
-                                                    "cluster.self-heal-daemon",
-                                                    _gf_true);
-                if (!shd_enabled) {
+            }
+            if (!trash_enabled) {
+                snprintf(errstr, sizeof(errstr),
+                         "Trash translator is not enabled. "
+                         "Use volume set %s trash on",
+                         volname);
+                gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_VOL_SET_FAIL,
+                       "Unable to set the options in 'volume set': %s", errstr);
+                ret = -1;
+                goto out;
+            }
+            if (strchr(value, '/')) {
+                snprintf(errstr, sizeof(errstr),
+                         "Path is not allowed as option");
+                gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_VOL_SET_FAIL,
+                       "Unable to set the options in 'volume set': %s", errstr);
+                ret = -1;
+                goto out;
+            }
+
+            list_for_each_entry(brickinfo, &volinfo->bricks, brick_list)
+            {
+                /* Check for local brick */
+                if (!gf_uuid_compare(brickinfo->uuid, MY_UUID)) {
+                    trash_path_len = strlen(value) + strlen(brickinfo->path) +
+                                     2;
+                    trash_path = GF_MALLOC(trash_path_len, gf_common_mt_char);
+                    snprintf(trash_path, trash_path_len, "%s/%s",
+                             brickinfo->path, value);
+
+                    /* Checks whether a directory with
+                       given option exists or not */
+                    if (!sys_access(trash_path, R_OK)) {
+                        snprintf(errstr, sizeof(errstr), "Path %s exists",
+                                 value);
+                        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_VOL_SET_FAIL,
+                               "Unable to set the options in 'volume set': %s",
+                               errstr);
                         ret = -1;
-                        snprintf (msg, sizeof (msg),
-                                  "Self-heal Daemon is disabled for volume %s",
-                                  volname);
                         goto out;
-                }
-        } else if ((cmd & GF_CLI_STATUS_QUOTAD) != 0) {
-                if (!glusterd_is_volume_quota_enabled (volinfo)) {
+                    } else {
+                        gf_msg_debug(this->name, 0,
+                                     "Directory with given name does not exist,"
+                                     " continuing");
+                    }
+
+                    if (volinfo->status == GLUSTERD_STATUS_STARTED &&
+                        brickinfo->status != GF_BRICK_STARTED) {
+                        /* If volume is in started state , checks
+                           whether bricks are online */
+                        snprintf(errstr, sizeof(errstr),
+                                 "One or more bricks are down");
+                        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_VOL_SET_FAIL,
+                               "Unable to set the options in 'volume set': %s",
+                               errstr);
                         ret = -1;
-                        snprintf (msg, sizeof (msg), "Volume %s does not have "
-                                  "quota enabled", volname);
                         goto out;
+                    }
                 }
-        } else if ((cmd & GF_CLI_STATUS_SNAPD) != 0) {
-                if (!glusterd_is_snapd_enabled (volinfo)) {
-                        ret = -1;
-                        snprintf (msg, sizeof (msg), "Volume %s does not have "
-                                  "uss enabled", volname);
-                        goto out;
+                if (trash_path) {
+                    GF_FREE(trash_path);
+                    trash_path = NULL;
                 }
-        } else if ((cmd & GF_CLI_STATUS_BRICK) != 0) {
-                ret = dict_get_str (dict, "brick", &brick);
+            }
+        }
+
+        ret = dict_set_strn(val_dict, key, keylen, value);
+
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+                   "Unable to set the options in 'volume set'");
+            ret = -1;
+            goto out;
+        }
+
+        *op_errstr = NULL;
+        if (!global_opt && !all_vol)
+            ret = glusterd_validate_reconfopts(volinfo, val_dict, op_errstr);
+        else if (!all_vol) {
+            voliter = NULL;
+            cds_list_for_each_entry(voliter, &priv->volumes, vol_list)
+            {
+                ret = glusterd_validate_globalopts(voliter, val_dict,
+                                                   op_errstr);
                 if (ret)
-                        goto out;
+                    break;
+            }
+        }
 
-                ret = glusterd_volume_brickinfo_get_by_brick (brick, volinfo,
-                                                              &brickinfo);
-                if (ret) {
-                        snprintf (msg, sizeof(msg), "No brick %s in"
-                                  " volume %s", brick, volname);
-                        ret = -1;
-                        goto out;
-                }
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_VOLFILE_CREATE_FAIL,
+                   "Could not create temp volfile, some option failed: %s",
+                   *op_errstr);
+            goto out;
         }
+        dict_deln(val_dict, key, keylen);
 
-        ret = 0;
+        if (key_fixed) {
+            GF_FREE(key_fixed);
+            key_fixed = NULL;
+        }
+    }
 
- out:
+    /* Check if all the connected clients support the new client-op-version
+     */
+    ret = glusterd_check_client_op_version_support(
+        volname, local_new_client_op_version, op_errstr);
+    if (ret)
+        goto out;
+cont:
+    if (origin_glusterd) {
+        ret = dict_set_uint32(dict, "new-op-version", local_new_op_version);
         if (ret) {
-                if (msg[0] != '\0')
-                        *op_errstr = gf_strdup (msg);
-                else
-                        *op_errstr = gf_strdup ("Validation Failed for Status");
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+                   "Failed to set new-op-version in dict");
+            goto out;
+        }
+        /* Set this value in dict so other peers know to check for
+         * op-version. This is a hack for 3.3.x compatibility
+         *
+         * TODO: Remove this and the other places this is referred once
+         * 3.3.x compatibility is not required
+         */
+        ret = dict_set_int32_sizen(dict, "check-op-version", 1);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+                   "Failed to set check-op-version in dict");
+            goto out;
         }
+    }
 
-        gf_log (this->name, GF_LOG_DEBUG, "Returning: %d", ret);
-        return ret;
-}
+    ret = 0;
 
+out:
+    if (val_dict)
+        dict_unref(val_dict);
 
-static gf_boolean_t
-glusterd_is_profile_on (glusterd_volinfo_t *volinfo)
-{
-        int                                     ret = -1;
-        gf_boolean_t                            is_latency_on = _gf_false;
-        gf_boolean_t                            is_fd_stats_on = _gf_false;
-
-        GF_ASSERT (volinfo);
-
-        ret = glusterd_volinfo_get_boolean (volinfo, VKEY_DIAG_CNT_FOP_HITS);
-        if (ret != -1)
-                is_fd_stats_on = ret;
-        ret = glusterd_volinfo_get_boolean (volinfo, VKEY_DIAG_LAT_MEASUREMENT);
-        if (ret != -1)
-                is_latency_on = ret;
-        if ((_gf_true == is_latency_on) &&
-            (_gf_true == is_fd_stats_on))
-                return _gf_true;
-        return _gf_false;
+    if (trash_path)
+        GF_FREE(trash_path);
+
+    GF_FREE(key_fixed);
+    if (errstr[0] != '\0')
+        *op_errstr = gf_strdup(errstr);
+
+    if (ret) {
+        if (!(*op_errstr)) {
+            *op_errstr = gf_strdup("Error, Validation Failed");
+            gf_msg_debug(this->name, 0, "Error, Cannot Validate option :%s",
+                         *op_errstr);
+        } else {
+            gf_msg_debug(this->name, 0, "Error, Cannot Validate option");
+        }
+    }
+    return ret;
 }
 
 static int
-glusterd_op_stage_stats_volume (dict_t *dict, char **op_errstr)
+glusterd_op_stage_reset_volume(dict_t *dict, char **op_errstr)
 {
-        int                                     ret = -1;
-        char                                    *volname = NULL;
-        gf_boolean_t                            exists = _gf_false;
-        char                                    msg[2048] = {0,};
-        int32_t                                 stats_op = GF_CLI_STATS_NONE;
-        glusterd_volinfo_t                      *volinfo = NULL;
-
-        ret = dict_get_str (dict, "volname", &volname);
+    int ret = 0;
+    char *volname = NULL;
+    int exists = 0;
+    char msg[2048] = {0};
+    char *key = NULL;
+    char *key_fixed = NULL;
+    glusterd_volinfo_t *volinfo = NULL;
+    xlator_t *this = NULL;
+    glusterd_conf_t *priv = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+
+    priv = this->private;
+    GF_ASSERT(priv);
+
+    ret = dict_get_strn(dict, "volname", SLEN("volname"), &volname);
+
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+               "Unable to get volume name");
+        goto out;
+    }
+
+    if (strcasecmp(volname, "all") != 0) {
+        ret = glusterd_volinfo_find(volname, &volinfo);
         if (ret) {
-                snprintf (msg, sizeof (msg), "Volume name get failed");
-                goto out;
+            snprintf(msg, sizeof(msg), FMTSTR_CHECK_VOL_EXISTS, volname);
+            goto out;
         }
 
-        exists = glusterd_check_volume_exists (volname);
-        ret = glusterd_volinfo_find (volname, &volinfo);
-        if ((!exists) || (ret < 0)) {
-                snprintf (msg, sizeof (msg), "Volume %s, "
-                         "doesn't exist", volname);
-                ret = -1;
-                goto out;
-        }
-
-        ret = glusterd_validate_volume_id (dict, volinfo);
+        ret = glusterd_validate_volume_id(dict, volinfo);
         if (ret)
-                goto out;
+            goto out;
+    }
+
+    ret = dict_get_strn(dict, "key", SLEN("key"), &key);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+               "Unable to get option key");
+        goto out;
+    }
+
+    /* *
+     * If key ganesha.enable is set, then volume should be unexported from
+     * ganesha server. Also it is a volume-level option, perform only when
+     * volume name not equal to "all"(in other words if volinfo != NULL)
+     */
+    if (volinfo && (!strcmp(key, "all") || !strcmp(key, "ganesha.enable"))) {
+        if (glusterd_check_ganesha_export(volinfo)) {
+            ret = ganesha_manage_export(dict, "off", _gf_true, op_errstr);
+            if (ret)
+                gf_msg(this->name, GF_LOG_WARNING, 0, GD_MSG_NFS_GNS_RESET_FAIL,
+                       "Could not reset ganesha.enable key");
+        }
+    }
+
+    if (strcmp(key, "all")) {
+        exists = glusterd_check_option_exists(key, &key_fixed);
+        if (exists == -1) {
+            ret = -1;
+            goto out;
+        }
+
+        if (!exists) {
+            ret = snprintf(msg, sizeof(msg), "Option %s does not exist", key);
+            if (key_fixed)
+                snprintf(msg + ret, sizeof(msg) - ret, "\nDid you mean %s?",
+                         key_fixed);
+            ret = -1;
+            goto out;
+        } else if (exists > 0) {
+            if (key_fixed)
+                key = key_fixed;
 
-        ret = dict_get_int32 (dict, "op", &stats_op);
-        if (ret) {
-                snprintf (msg, sizeof (msg), "Volume profile op get failed");
+            /* 'gluster volume set/reset <VOLNAME>
+             * features.quota/features.inode-quota' should
+             * not be allowed as it is deprecated.
+             * Setting and resetting quota/inode-quota features
+             * should be allowed only through 'gluster volume quota
+             * <VOLNAME> enable/disable'.
+             * But, 'gluster volume set features.quota-deem-statfs'
+             * can be turned on/off when quota is enabled.
+             */
+
+            if (strcmp(VKEY_FEATURES_INODE_QUOTA, key) == 0 ||
+                strcmp(VKEY_FEATURES_QUOTA, key) == 0) {
+                snprintf(msg, sizeof(msg),
+                         "'gluster volume "
+                         "reset <VOLNAME> %s' is deprecated. "
+                         "Use 'gluster volume quota <VOLNAME> "
+                         "disable' instead.",
+                         key);
+                ret = -1;
                 goto out;
+            }
+            ALL_VOLUME_OPTION_CHECK(volname, _gf_false, key, ret, op_errstr,
+                                    out);
         }
+    }
 
-        if (GF_CLI_STATS_START == stats_op) {
-                if (_gf_true == glusterd_is_profile_on (volinfo)) {
-                        snprintf (msg, sizeof (msg), "Profile on Volume %s is"
-                                  " already started", volinfo->volname);
-                        ret = -1;
-                        goto out;
-                }
-
-        }
-        if ((GF_CLI_STATS_STOP == stats_op) ||
-            (GF_CLI_STATS_INFO == stats_op)) {
-                if (_gf_false == glusterd_is_profile_on (volinfo)) {
-                        snprintf (msg, sizeof (msg), "Profile on Volume %s is"
-                                  " not started", volinfo->volname);
-                        ret = -1;
-
-                        goto out;
-                }
-        }
-        if ((GF_CLI_STATS_TOP == stats_op) ||
-            (GF_CLI_STATS_INFO == stats_op)) {
-                if (_gf_false == glusterd_is_volume_started (volinfo)) {
-                        snprintf (msg, sizeof (msg), "Volume %s is not started.",
-                                  volinfo->volname);
-                        gf_log ("glusterd", GF_LOG_ERROR, "%s", msg);
-                        ret = -1;
-                        goto out;
-                }
-        }
-        ret = 0;
 out:
-        if (msg[0] != '\0') {
-                gf_log ("glusterd", GF_LOG_ERROR, "%s", msg);
-                *op_errstr = gf_strdup (msg);
-        }
-        gf_log ("", GF_LOG_DEBUG, "Returning %d", ret);
-        return ret;
-}
+    GF_FREE(key_fixed);
 
+    if (msg[0] != '\0') {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_OP_STAGE_RESET_VOL_FAIL,
+               "%s", msg);
+        *op_errstr = gf_strdup(msg);
+    }
 
-static int
-_delete_reconfig_opt (dict_t *this, char *key, data_t *value, void *data)
-{
-        int32_t        *is_force = 0;
-
-        GF_ASSERT (data);
-        is_force = (int32_t*)data;
-
-        if (*is_force != 1) {
-                if (_gf_true == glusterd_check_voloption_flags (key,
-                                                         OPT_FLAG_FORCE)) {
-                /* indicate to caller that we don't set the option
-                 * due to being protected
-                 */
-                        *is_force = *is_force | GD_OP_PROTECTED;
-                        goto out;
-                } else {
-                        *is_force = *is_force | GD_OP_UNPROTECTED;
-                }
-        }
+    gf_msg_debug(this->name, 0, "Returning %d", ret);
 
-        gf_log ("", GF_LOG_DEBUG, "deleting dict with key=%s,value=%s",
-                key, value->data);
-        dict_del (this, key);
-out:
-        return 0;
+    return ret;
 }
 
 static int
-_delete_reconfig_global_opt (dict_t *this, char *key, data_t *value, void *data)
+glusterd_op_stage_sync_volume(dict_t *dict, char **op_errstr)
 {
-        int32_t        *is_force = 0;
-
-        GF_ASSERT (data);
-        is_force = (int32_t*)data;
-
-        if (strcmp (GLUSTERD_GLOBAL_OPT_VERSION, key) == 0)
-                goto out;
+    int ret = -1;
+    char *volname = NULL;
+    char *hostname = NULL;
+    glusterd_peerinfo_t *peerinfo = NULL;
+    char msg[2048] = {
+        0,
+    };
+    glusterd_volinfo_t *volinfo = NULL;
+    xlator_t *this = NULL;
+    this = THIS;
+    GF_ASSERT(this);
+
+    ret = dict_get_strn(dict, "hostname", SLEN("hostname"), &hostname);
+    if (ret) {
+        snprintf(msg, sizeof(msg),
+                 "hostname couldn't be "
+                 "retrieved from msg");
+        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_GET_FAILED,
+                "Key=hostname", NULL);
+        *op_errstr = gf_strdup(msg);
+        goto out;
+    }
+
+    if (gf_is_local_addr(hostname)) {
+        // volname is not present in case of sync all
+        ret = dict_get_strn(dict, "volname", SLEN("volname"), &volname);
+        if (!ret) {
+            ret = glusterd_volinfo_find(volname, &volinfo);
+            if (ret) {
+                snprintf(msg, sizeof(msg),
+                         "Volume %s "
+                         "does not exist",
+                         volname);
+                gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_VOL_NOT_FOUND,
+                        "Volume=%s", volname, NULL);
+                *op_errstr = gf_strdup(msg);
+                goto out;
+            }
+        }
+    } else {
+        RCU_READ_LOCK;
+
+        peerinfo = glusterd_peerinfo_find(NULL, hostname);
+        if (peerinfo == NULL) {
+            RCU_READ_UNLOCK;
+            ret = -1;
+            snprintf(msg, sizeof(msg), "%s, is not a friend", hostname);
+            gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_PEER_NOT_FOUND,
+                    "Peer_name=%s", hostname, NULL);
+            *op_errstr = gf_strdup(msg);
+            goto out;
+
+        } else if (!peerinfo->connected) {
+            RCU_READ_UNLOCK;
+            ret = -1;
+            snprintf(msg, sizeof(msg),
+                     "%s, is not connected at "
+                     "the moment",
+                     hostname);
+            gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_PEER_DISCONNECTED,
+                    "Peer_name=%s", hostname, NULL);
+            *op_errstr = gf_strdup(msg);
+            goto out;
+        }
+
+        RCU_READ_UNLOCK;
+    }
 
-        _delete_reconfig_opt (this, key, value, data);
 out:
-        return 0;
+    gf_msg_debug("glusterd", 0, "Returning %d", ret);
+
+    return ret;
 }
 
 static int
-glusterd_options_reset (glusterd_volinfo_t *volinfo, char *key,
-                        int32_t *is_force)
+glusterd_op_stage_status_volume(dict_t *dict, char **op_errstr)
 {
-        int                      ret = 0;
-        data_t                  *value = NULL;
-        char                    *key_fixed = NULL;
-        xlator_t                *this = NULL;
-
-        this = THIS;
-        GF_ASSERT (this);
-        GF_ASSERT (volinfo->dict);
-        GF_ASSERT (key);
-
-        if (!strncmp(key, "all", 3)) {
-                dict_foreach (volinfo->dict, _delete_reconfig_opt, is_force);
-                ret = glusterd_enable_default_options (volinfo, NULL);
-                if (ret) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                GD_MSG_FAIL_DEFAULT_OPT_SET, "Failed to set "
-                                "default options on reset for volume %s",
-                                volinfo->volname);
-                        goto out;
-                }
+    int ret = -1;
+    uint32_t cmd = 0;
+    char msg[2048] = {
+        0,
+    };
+    char *volname = NULL;
+    char *brick = NULL;
+    xlator_t *this = NULL;
+    glusterd_conf_t *priv = NULL;
+    glusterd_brickinfo_t *brickinfo = NULL;
+    glusterd_volinfo_t *volinfo = NULL;
+    dict_t *vol_opts = NULL;
+#ifdef BUILD_GNFS
+    gf_boolean_t nfs_disabled = _gf_false;
+#endif
+    gf_boolean_t shd_enabled = _gf_false;
+
+    GF_ASSERT(dict);
+    this = THIS;
+    GF_ASSERT(this);
+    priv = this->private;
+    GF_ASSERT(priv);
+
+    ret = dict_get_uint32(dict, "cmd", &cmd);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_GET_FAILED,
+                "Key=cmd", NULL);
+        goto out;
+    }
+
+    if (cmd & GF_CLI_STATUS_ALL)
+        goto out;
+
+    if ((cmd & GF_CLI_STATUS_QUOTAD) &&
+        (priv->op_version == GD_OP_VERSION_MIN)) {
+        snprintf(msg, sizeof(msg),
+                 "The cluster is operating at "
+                 "version 1. Getting the status of quotad is not "
+                 "allowed in this state.");
+        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_QUOTA_GET_STAT_FAIL,
+                msg, NULL);
+        ret = -1;
+        goto out;
+    }
+
+    if ((cmd & GF_CLI_STATUS_SNAPD) &&
+        (priv->op_version < GD_OP_VERSION_3_6_0)) {
+        snprintf(msg, sizeof(msg),
+                 "The cluster is operating at "
+                 "version less than %d. Getting the "
+                 "status of snapd is not allowed in this state.",
+                 GD_OP_VERSION_3_6_0);
+        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_SNAP_STATUS_FAIL, msg,
+                NULL);
+        ret = -1;
+        goto out;
+    }
+
+    ret = dict_get_strn(dict, "volname", SLEN("volname"), &volname);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+               "Unable to get volume name");
+        goto out;
+    }
+
+    ret = glusterd_volinfo_find(volname, &volinfo);
+    if (ret) {
+        snprintf(msg, sizeof(msg), FMTSTR_CHECK_VOL_EXISTS, volname);
+        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_VOLINFO_GET_FAIL,
+                "Volume=%s", volname, NULL);
+        ret = -1;
+        goto out;
+    }
+
+    ret = glusterd_validate_volume_id(dict, volinfo);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_VALIDATE_FAILED, NULL);
+        goto out;
+    }
+
+    ret = glusterd_is_volume_started(volinfo);
+    if (!ret) {
+        snprintf(msg, sizeof(msg), "Volume %s is not started", volname);
+        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_VOL_NOT_STARTED,
+                "Volume=%s", volname, NULL);
+        ret = -1;
+        goto out;
+    }
+
+    vol_opts = volinfo->dict;
+
+    if ((cmd & GF_CLI_STATUS_SHD) != 0) {
+        if (glusterd_is_shd_compatible_volume(volinfo)) {
+            shd_enabled = gd_is_self_heal_enabled(volinfo, vol_opts);
         } else {
-                value = dict_get (volinfo->dict, key);
-                if (!value) {
-                        gf_log (this->name, GF_LOG_DEBUG,
-                                "no value set for option %s", key);
-                        goto out;
-                }
-                _delete_reconfig_opt (volinfo->dict, key, value, is_force);
-                ret = glusterd_enable_default_options (volinfo, key);
-                if (ret) {
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                GD_MSG_FAIL_DEFAULT_OPT_SET, "Failed to set "
-                                "default value for option '%s' on reset for "
-                                "volume %s", key, volinfo->volname);
-                        goto out;
-                }
+            ret = -1;
+            snprintf(msg, sizeof(msg), "Volume %s is not Self-heal compatible",
+                     volname);
+            gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_VOL_SHD_NOT_COMP,
+                    "Volume=%s", volname, NULL);
+            goto out;
+        }
+        if (!shd_enabled) {
+            ret = -1;
+            snprintf(msg, sizeof(msg),
+                     "Self-heal Daemon is disabled for volume %s", volname);
+            gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_SELF_HEALD_DISABLED,
+                    "Volume=%s", volname, NULL);
+            goto out;
+        }
+#ifdef BUILD_GNFS
+    } else if ((cmd & GF_CLI_STATUS_NFS) != 0) {
+        nfs_disabled = dict_get_str_boolean(vol_opts, NFS_DISABLE_MAP_KEY,
+                                            _gf_false);
+        if (nfs_disabled) {
+            ret = -1;
+            snprintf(msg, sizeof(msg), "NFS server is disabled for volume %s",
+                     volname);
+            gf_smsg(this->name, GF_LOG_ERROR, errno,
+                    GD_MSG_NFS_GANESHA_DISABLED, "Volume=%s", volname, NULL);
+            goto out;
+        }
+#endif
+    } else if ((cmd & GF_CLI_STATUS_QUOTAD) != 0) {
+        if (!glusterd_is_volume_quota_enabled(volinfo)) {
+            ret = -1;
+            snprintf(msg, sizeof(msg),
+                     "Volume %s does not have "
+                     "quota enabled",
+                     volname);
+            gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_QUOTA_DISABLED,
+                    "Volume=%s", volname, NULL);
+            goto out;
+        }
+    } else if ((cmd & GF_CLI_STATUS_BITD) != 0) {
+        if (!glusterd_is_bitrot_enabled(volinfo)) {
+            ret = -1;
+            snprintf(msg, sizeof(msg),
+                     "Volume %s does not have "
+                     "bitrot enabled",
+                     volname);
+            gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_BITROT_NOT_ENABLED,
+                    "Volume=%s", volname, NULL);
+            goto out;
+        }
+    } else if ((cmd & GF_CLI_STATUS_SCRUB) != 0) {
+        if (!glusterd_is_bitrot_enabled(volinfo)) {
+            ret = -1;
+            snprintf(msg, sizeof(msg),
+                     "Volume %s does not have "
+                     "bitrot enabled. Scrubber will be enabled "
+                     "automatically if bitrot is enabled",
+                     volname);
+            gf_smsg(
+                this->name, GF_LOG_ERROR, errno, GD_MSG_BITROT_NOT_ENABLED,
+                "Scrubber will be enabled automatically if bitrot is enabled",
+                "Volume=%s", volname, NULL);
+            goto out;
+        }
+    } else if ((cmd & GF_CLI_STATUS_SNAPD) != 0) {
+        if (!glusterd_is_snapd_enabled(volinfo)) {
+            ret = -1;
+            snprintf(msg, sizeof(msg),
+                     "Volume %s does not have "
+                     "uss enabled",
+                     volname);
+            gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_SNAPD_NOT_RUNNING,
+                    "Volume=%s", volname, NULL);
+            goto out;
+        }
+    } else if ((cmd & GF_CLI_STATUS_BRICK) != 0) {
+        ret = dict_get_strn(dict, "brick", SLEN("brick"), &brick);
+        if (ret) {
+            gf_smsg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+                    "Key=brick", NULL);
+            goto out;
         }
 
-        gd_update_volume_op_versions (volinfo);
-
-        ret = glusterd_create_volfiles_and_notify_services (volinfo);
+        ret = glusterd_volume_brickinfo_get_by_brick(brick, volinfo, &brickinfo,
+                                                     _gf_false);
         if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "Unable to create volfile for"
-                        " 'volume reset'");
-                ret = -1;
-                goto out;
+            snprintf(msg, sizeof(msg),
+                     "No brick %s in"
+                     " volume %s",
+                     brick, volname);
+            gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_BRICK_NOT_FOUND,
+                    "Brick=%s, Volume=%s", brick, volname, NULL);
+            ret = -1;
+            goto out;
         }
+    }
 
-        ret = glusterd_store_volinfo (volinfo, GLUSTERD_VOLINFO_VER_AC_INCREMENT);
-        if (ret)
-                goto out;
+    ret = 0;
 
-        if (GLUSTERD_STATUS_STARTED == volinfo->status) {
-                ret = glusterd_svcs_reconfigure (volinfo);
-                if (ret)
-                        goto out;
-        }
+out:
+    if (ret) {
+        if (msg[0] != '\0')
+            *op_errstr = gf_strdup(msg);
+        else
+            *op_errstr = gf_strdup("Validation Failed for Status");
+    }
 
-        ret = 0;
+    gf_msg_debug(this->name, 0, "Returning: %d", ret);
+    return ret;
+}
 
+int
+glusterd_op_stage_stats_volume(dict_t *dict, char **op_errstr)
+{
+    int ret = -1;
+    char *volname = NULL;
+    char msg[2048] = {
+        0,
+    };
+    int32_t stats_op = GF_CLI_STATS_NONE;
+    glusterd_volinfo_t *volinfo = NULL;
+
+    ret = dict_get_strn(dict, "volname", SLEN("volname"), &volname);
+    if (ret) {
+        snprintf(msg, sizeof(msg), "Volume name get failed");
+        goto out;
+    }
+
+    ret = glusterd_volinfo_find(volname, &volinfo);
+    if (ret) {
+        snprintf(msg, sizeof(msg),
+                 "Volume %s, "
+                 "doesn't exist",
+                 volname);
+        goto out;
+    }
+
+    ret = glusterd_validate_volume_id(dict, volinfo);
+    if (ret)
+        goto out;
+
+    ret = dict_get_int32n(dict, "op", SLEN("op"), &stats_op);
+    if (ret) {
+        snprintf(msg, sizeof(msg), "Volume profile op get failed");
+        goto out;
+    }
+
+    if (GF_CLI_STATS_START == stats_op) {
+        if (_gf_true == glusterd_is_profile_on(volinfo)) {
+            snprintf(msg, sizeof(msg),
+                     "Profile on Volume %s is"
+                     " already started",
+                     volinfo->volname);
+            ret = -1;
+            goto out;
+        }
+    } else if ((GF_CLI_STATS_STOP == stats_op) ||
+               (GF_CLI_STATS_INFO == stats_op)) {
+        if (_gf_false == glusterd_is_profile_on(volinfo)) {
+            snprintf(msg, sizeof(msg),
+                     "Profile on Volume %s is"
+                     " not started",
+                     volinfo->volname);
+            ret = -1;
+
+            goto out;
+        }
+    }
+    if ((GF_CLI_STATS_TOP == stats_op) || (GF_CLI_STATS_INFO == stats_op)) {
+        if (_gf_false == glusterd_is_volume_started(volinfo)) {
+            snprintf(msg, sizeof(msg), "Volume %s is not started.",
+                     volinfo->volname);
+            gf_msg("glusterd", GF_LOG_ERROR, 0, GD_MSG_VOL_NOT_STARTED, "%s",
+                   msg);
+            ret = -1;
+            goto out;
+        }
+    }
+    ret = 0;
 out:
-        GF_FREE (key_fixed);
-        gf_log (this->name, GF_LOG_DEBUG, "Returning %d", ret);
-        return ret;
+    if (msg[0] != '\0') {
+        gf_msg("glusterd", GF_LOG_ERROR, 0, GD_MSG_OP_STAGE_STATS_VOL_FAIL,
+               "%s", msg);
+        *op_errstr = gf_strdup(msg);
+    }
+    gf_msg_debug("glusterd", 0, "Returning %d", ret);
+    return ret;
 }
 
 static int
-glusterd_op_reset_all_volume_options (xlator_t *this, dict_t *dict)
+_delete_reconfig_opt(dict_t *this, char *key, data_t *value, void *data)
 {
-        char            *key            = NULL;
-        char            *key_fixed      = NULL;
-        int             ret             = -1;
-        int32_t         is_force        = 0;
-        glusterd_conf_t *conf           = NULL;
-        dict_t          *dup_opt        = NULL;
-        gf_boolean_t    all             = _gf_false;
-        char            *next_version   = NULL;
-        gf_boolean_t    quorum_action   = _gf_false;
-
-        conf = this->private;
-        ret = dict_get_str (dict, "key", &key);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "Failed to get key");
-                goto out;
-        }
-
-        ret = dict_get_int32 (dict, "force", &is_force);
-        if (ret)
-                is_force = 0;
-
-        if (strcmp (key, "all")) {
-                ret = glusterd_check_option_exists (key, &key_fixed);
-                if (ret <= 0) {
-                        gf_log (this->name, GF_LOG_ERROR, "Option %s does not "
-                                "exist", key);
-                        ret = -1;
-                        goto out;
-                }
+    int32_t *is_force = 0;
+
+    GF_ASSERT(data);
+    is_force = (int32_t *)data;
+
+    /* Keys which has the flag VOLOPT_FLAG_NEVER_RESET
+     * should not be deleted
+     */
+
+    if (_gf_true ==
+        glusterd_check_voloption_flags(key, VOLOPT_FLAG_NEVER_RESET)) {
+        if (*is_force != 1)
+            *is_force = *is_force | GD_OP_PROTECTED;
+        goto out;
+    }
+
+    if (*is_force != 1) {
+        if (_gf_true ==
+            glusterd_check_voloption_flags(key, VOLOPT_FLAG_FORCE)) {
+            /* indicate to caller that we don't set the option
+             * due to being protected
+             */
+            *is_force = *is_force | GD_OP_PROTECTED;
+            goto out;
         } else {
-                all = _gf_true;
-        }
-
-        if (key_fixed)
-                key = key_fixed;
-
-        ret = -1;
-        dup_opt = dict_new ();
-        if (!dup_opt)
-                goto out;
-        if (!all) {
-                dict_copy (conf->opts, dup_opt);
-                dict_del (dup_opt, key);
-        }
-        ret = glusterd_get_next_global_opt_version_str (conf->opts,
-                                                        &next_version);
-        if (ret)
-                goto out;
-
-        ret = dict_set_str (dup_opt, GLUSTERD_GLOBAL_OPT_VERSION, next_version);
-        if (ret)
-                goto out;
-
-        ret = glusterd_store_options (this, dup_opt);
-        if (ret)
-                goto out;
+            *is_force = *is_force | GD_OP_UNPROTECTED;
+        }
+    }
+
+    gf_msg_debug("glusterd", 0, "deleting dict with key=%s,value=%s", key,
+                 value->data);
+    dict_del(this, key);
+    /**Delete scrubber (pause/resume) option from the dictionary if bitrot
+     * option is going to be reset
+     * */
+    if (!strncmp(key, VKEY_FEATURES_BITROT, strlen(VKEY_FEATURES_BITROT))) {
+        dict_del_sizen(this, VKEY_FEATURES_SCRUB);
+    }
+out:
+    return 0;
+}
 
-        if (glusterd_is_quorum_changed (conf->opts, key, NULL))
-                quorum_action = _gf_true;
+static int
+_delete_reconfig_global_opt(dict_t *this, char *key, data_t *value, void *data)
+{
+    GF_ASSERT(data);
 
-        ret = dict_set_dynstr (conf->opts, GLUSTERD_GLOBAL_OPT_VERSION,
-                               next_version);
-        if (ret)
-                goto out;
-        else
-                next_version = NULL;
+    if (strcmp(GLUSTERD_GLOBAL_OPT_VERSION, key) == 0)
+        goto out;
 
-        if (!all) {
-                dict_del (conf->opts, key);
-        } else {
-                dict_foreach (conf->opts, _delete_reconfig_global_opt,
-                              &is_force);
-        }
+    _delete_reconfig_opt(this, key, value, data);
 out:
-        GF_FREE (key_fixed);
-        if (dup_opt)
-                dict_unref (dup_opt);
-
-        gf_log (this->name, GF_LOG_DEBUG, "returning %d", ret);
-        if (quorum_action)
-                glusterd_do_quorum_action ();
-        GF_FREE (next_version);
-        return ret;
+    return 0;
 }
 
 static int
-glusterd_op_reset_volume (dict_t *dict, char **op_rspstr)
+glusterd_options_reset(glusterd_volinfo_t *volinfo, char *key,
+                       int32_t *is_force)
 {
-        glusterd_volinfo_t      *volinfo    = NULL;
-        int                     ret         = -1;
-        char                    *volname    = NULL;
-        char                    *key        = NULL;
-        char                    *key_fixed  = NULL;
-        int32_t                 is_force    = 0;
-        gf_boolean_t            quorum_action = _gf_false;
-        xlator_t                *this         = NULL;
-
-        this = THIS;
-        ret = dict_get_str (dict, "volname", &volname);
+    int ret = 0;
+    data_t *value = NULL;
+    char *key_fixed = NULL;
+    xlator_t *this = NULL;
+    glusterd_svc_t *svc = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+    GF_ASSERT(volinfo->dict);
+    GF_ASSERT(key);
+
+    if (!strncmp(key, "all", 3)) {
+        dict_foreach(volinfo->dict, _delete_reconfig_opt, is_force);
+        ret = glusterd_enable_default_options(volinfo, NULL);
         if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "Unable to get volume name" );
-                goto out;
-        }
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_FAIL_DEFAULT_OPT_SET,
+                   "Failed to set "
+                   "default options on reset for volume %s",
+                   volinfo->volname);
+            goto out;
+        }
+    } else {
+        value = dict_get(volinfo->dict, key);
+        if (!value) {
+            gf_msg_debug(this->name, 0, "no value set for option %s", key);
+            goto out;
+        }
+        _delete_reconfig_opt(volinfo->dict, key, value, is_force);
+        ret = glusterd_enable_default_options(volinfo, key);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_FAIL_DEFAULT_OPT_SET,
+                   "Failed to set "
+                   "default value for option '%s' on reset for "
+                   "volume %s",
+                   key, volinfo->volname);
+            goto out;
+        }
+    }
+
+    gd_update_volume_op_versions(volinfo);
+    if (!volinfo->is_snap_volume) {
+        svc = &(volinfo->snapd.svc);
+        ret = svc->manager(svc, volinfo, PROC_START_NO_WAIT);
+        if (ret)
+            goto out;
+    }
+    svc = &(volinfo->gfproxyd.svc);
+    ret = svc->reconfigure(volinfo);
+    if (ret)
+        goto out;
+
+    svc = &(volinfo->shd.svc);
+    ret = svc->reconfigure(volinfo);
+    if (ret)
+        goto out;
+
+    ret = glusterd_create_volfiles_and_notify_services(volinfo);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_VOLFILE_CREATE_FAIL,
+               "Unable to create volfile for"
+               " 'volume reset'");
+        ret = -1;
+        goto out;
+    }
 
-        if (strcasecmp (volname, "all") == 0) {
-                ret = glusterd_op_reset_all_volume_options (this, dict);
-                goto out;
-        }
+    ret = glusterd_store_volinfo(volinfo, GLUSTERD_VOLINFO_VER_AC_INCREMENT);
+    if (ret)
+        goto out;
 
-        ret = dict_get_int32 (dict, "force", &is_force);
+    if (GLUSTERD_STATUS_STARTED == volinfo->status) {
+        ret = glusterd_svcs_reconfigure(volinfo);
         if (ret)
-                is_force = 0;
+            goto out;
+    }
 
-        ret = dict_get_str (dict, "key", &key);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "Unable to get option key");
-                goto out;
-        }
+    ret = 0;
 
-        ret = glusterd_volinfo_find (volname, &volinfo);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, FMTSTR_CHECK_VOL_EXISTS,
-                        volname);
-                goto out;
-        }
+out:
+    GF_FREE(key_fixed);
+    gf_msg_debug(this->name, 0, "Returning %d", ret);
+    return ret;
+}
 
-        if (strcmp (key, "all") &&
-            glusterd_check_option_exists (key, &key_fixed) != 1) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "volinfo dict inconsistency: option %s not found",
+static int
+glusterd_op_reset_all_volume_options(xlator_t *this, dict_t *dict)
+{
+    char *key = NULL;
+    char *key_fixed = NULL;
+    int ret = -1;
+    int32_t is_force = 0;
+    glusterd_conf_t *conf = NULL;
+    dict_t *dup_opt = NULL;
+    gf_boolean_t all = _gf_false;
+    char *next_version = NULL;
+    gf_boolean_t quorum_action = _gf_false;
+
+    conf = this->private;
+    ret = dict_get_strn(dict, "key", SLEN("key"), &key);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+               "Failed to get key");
+        goto out;
+    }
+
+    ret = dict_get_int32n(dict, "force", SLEN("force"), &is_force);
+    if (ret)
+        is_force = 0;
+
+    if (strcmp(key, "all")) {
+        ret = glusterd_check_option_exists(key, &key_fixed);
+        if (ret <= 0) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_INVALID_ENTRY,
+                   "Option %s does not "
+                   "exist",
+                   key);
+            ret = -1;
+            goto out;
+        }
+    } else {
+        all = _gf_true;
+    }
+
+    if (key_fixed)
+        key = key_fixed;
+
+    ret = -1;
+    dup_opt = dict_new();
+    if (!dup_opt) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_CREATE_FAIL, NULL);
+        goto out;
+    }
+    if (!all) {
+        dict_copy(conf->opts, dup_opt);
+        dict_del(dup_opt, key);
+    }
+    ret = glusterd_get_next_global_opt_version_str(conf->opts, &next_version);
+    if (ret)
+        goto out;
+
+    ret = dict_set_strn(dup_opt, GLUSTERD_GLOBAL_OPT_VERSION,
+                        SLEN(GLUSTERD_GLOBAL_OPT_VERSION), next_version);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_SET_FAILED,
+                "Key=%s", GLUSTERD_GLOBAL_OPT_VERSION, NULL);
+        goto out;
+    }
+
+    ret = glusterd_store_options(this, dup_opt);
+    if (ret)
+        goto out;
+
+    if (glusterd_is_quorum_changed(conf->opts, key, NULL))
+        quorum_action = _gf_true;
+
+    ret = dict_set_dynstrn(conf->opts, GLUSTERD_GLOBAL_OPT_VERSION,
+                           SLEN(GLUSTERD_GLOBAL_OPT_VERSION), next_version);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_SET_FAILED,
+                "Key=%s", GLUSTERD_GLOBAL_OPT_VERSION, NULL);
+        goto out;
+    } else
+        next_version = NULL;
+
+    if (!all) {
+        dict_del(conf->opts, key);
+    } else {
+        dict_foreach(conf->opts, _delete_reconfig_global_opt, &is_force);
+    }
+out:
+    GF_FREE(key_fixed);
+    if (dup_opt)
+        dict_unref(dup_opt);
+
+    gf_msg_debug(this->name, 0, "returning %d", ret);
+    if (quorum_action)
+        glusterd_do_quorum_action();
+    GF_FREE(next_version);
+    return ret;
+}
+
+static int
+glusterd_op_reset_volume(dict_t *dict, char **op_rspstr)
+{
+    glusterd_volinfo_t *volinfo = NULL;
+    int ret = -1;
+    char *volname = NULL;
+    char *key = NULL;
+    char *key_fixed = NULL;
+    int32_t is_force = 0;
+    gf_boolean_t quorum_action = _gf_false;
+    xlator_t *this = NULL;
+
+    this = THIS;
+    ret = dict_get_strn(dict, "volname", SLEN("volname"), &volname);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+               "Unable to get volume name");
+        goto out;
+    }
+
+    if (strcasecmp(volname, "all") == 0) {
+        ret = glusterd_op_reset_all_volume_options(this, dict);
+        goto out;
+    }
+
+    ret = dict_get_int32n(dict, "force", SLEN("force"), &is_force);
+    if (ret)
+        is_force = 0;
+
+    ret = dict_get_strn(dict, "key", SLEN("key"), &key);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+               "Unable to get option key");
+        goto out;
+    }
+
+    ret = glusterd_volinfo_find(volname, &volinfo);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_VOL_NOT_FOUND,
+               FMTSTR_CHECK_VOL_EXISTS, volname);
+        goto out;
+    }
+
+    if (strcmp(key, "all") &&
+        glusterd_check_option_exists(key, &key_fixed) != 1) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_INVALID_ENTRY,
+               "volinfo dict inconsistency: option %s not found", key);
+        ret = -1;
+        goto out;
+    }
+    if (key_fixed)
+        key = key_fixed;
+
+    if (glusterd_is_quorum_changed(volinfo->dict, key, NULL))
+        quorum_action = _gf_true;
+
+    ret = glusterd_options_reset(volinfo, key, &is_force);
+    if (ret == -1) {
+        gf_asprintf(op_rspstr, "Volume reset : failed");
+    } else if (is_force & GD_OP_PROTECTED) {
+        if (is_force & GD_OP_UNPROTECTED) {
+            gf_asprintf(op_rspstr,
+                        "All unprotected fields were"
+                        " reset. To reset the protected fields,"
+                        " use 'force'.");
+        } else {
+            ret = -1;
+            gf_asprintf(op_rspstr,
+                        "'%s' is protected. To reset"
+                        " use 'force'.",
                         key);
-                ret = -1;
-                goto out;
         }
-        if (key_fixed)
-                key = key_fixed;
+    }
 
-        if (glusterd_is_quorum_changed (volinfo->dict, key, NULL))
-                quorum_action = _gf_true;
-
-        ret = glusterd_options_reset (volinfo, key, &is_force);
-        if (ret == -1) {
-                gf_asprintf(op_rspstr, "Volume reset : failed");
-        } else if (is_force & GD_OP_PROTECTED) {
-                if (is_force & GD_OP_UNPROTECTED) {
-                        gf_asprintf (op_rspstr, "All unprotected fields were"
-                                     " reset. To reset the protected fields,"
-                                     " use 'force'.");
-                } else {
-                        ret = -1;
-                        gf_asprintf (op_rspstr, "'%s' is protected. To reset"
-                                     " use 'force'.", key);
-                }
+    if (!strcmp(key, "ganesha.enable") || !strcmp(key, "all")) {
+        if (glusterd_check_ganesha_export(volinfo) &&
+            is_origin_glusterd(dict)) {
+            ret = manage_export_config(volname, "off", op_rspstr);
+            if (ret)
+                gf_msg(this->name, GF_LOG_WARNING, 0, GD_MSG_NFS_GNS_RESET_FAIL,
+                       "Could not reset ganesha.enable key");
         }
+    }
 
 out:
-        GF_FREE (key_fixed);
-        if (quorum_action)
-                glusterd_do_quorum_action ();
+    GF_FREE(key_fixed);
+    if (quorum_action)
+        glusterd_do_quorum_action();
 
-        gf_log (this->name, GF_LOG_DEBUG, "'volume reset' returning %d", ret);
-        return ret;
+    gf_msg_debug(this->name, 0, "'volume reset' returning %d", ret);
+    return ret;
 }
 
 int
-glusterd_stop_bricks (glusterd_volinfo_t *volinfo)
+glusterd_stop_bricks(glusterd_volinfo_t *volinfo)
 {
-        glusterd_brickinfo_t                    *brickinfo = NULL;
+    glusterd_brickinfo_t *brickinfo = NULL;
 
-        cds_list_for_each_entry (brickinfo, &volinfo->bricks, brick_list) {
-            /*TODO: Need to change @del_brick in brick_stop to _gf_true
-             * once we enable synctask in peer rpc prog */
-                if (glusterd_brick_stop (volinfo, brickinfo, _gf_false))
-                        return -1;
+    cds_list_for_each_entry(brickinfo, &volinfo->bricks, brick_list)
+    {
+        /*TODO: Need to change @del_brick in brick_stop to _gf_true
+         * once we enable synctask in peer rpc prog */
+        if (glusterd_brick_stop(volinfo, brickinfo, _gf_false)) {
+            gf_event(EVENT_BRICK_STOP_FAILED, "peer=%s;volume=%s;brick=%s",
+                     brickinfo->hostname, volinfo->volname, brickinfo->path);
+            return -1;
         }
+    }
 
-        return 0;
+    return 0;
 }
 
 int
-glusterd_start_bricks (glusterd_volinfo_t *volinfo)
-{
-        int                      ret            = -1;
-        glusterd_brickinfo_t    *brickinfo      = NULL;
+glusterd_start_bricks(glusterd_volinfo_t *volinfo)
 
-        GF_ASSERT (volinfo);
+{
+    int ret = -1;
+    glusterd_brickinfo_t *brickinfo = NULL;
+
+    GF_ASSERT(volinfo);
+
+    cds_list_for_each_entry(brickinfo, &volinfo->bricks, brick_list)
+    {
+        if (!brickinfo->start_triggered) {
+            pthread_mutex_lock(&brickinfo->restart_mutex);
+            {
+                /* coverity[SLEEP] */
+                ret = glusterd_brick_start(volinfo, brickinfo, _gf_false,
+                                           _gf_false);
+            }
+            pthread_mutex_unlock(&brickinfo->restart_mutex);
+            if (ret) {
+                gf_msg(THIS->name, GF_LOG_ERROR, 0, GD_MSG_BRICK_DISCONNECTED,
+                       "Failed to start %s:%s for %s", brickinfo->hostname,
+                       brickinfo->path, volinfo->volname);
+                gf_event(EVENT_BRICK_START_FAILED, "peer=%s;volume=%s;brick=%s",
+                         brickinfo->hostname, volinfo->volname,
+                         brickinfo->path);
+                goto out;
+            }
+        }
+    }
+    ret = 0;
+out:
+    return ret;
+}
 
-        cds_list_for_each_entry (brickinfo, &volinfo->bricks, brick_list) {
-                ret = glusterd_brick_start (volinfo, brickinfo, _gf_false);
+static int
+glusterd_update_volumes_dict(glusterd_volinfo_t *volinfo)
+{
+    int ret = -1;
+    xlator_t *this = NULL;
+    glusterd_conf_t *conf = NULL;
+    char *address_family_str = NULL;
+
+    this = THIS;
+    GF_VALIDATE_OR_GOTO("glusterd", this, out);
+
+    conf = this->private;
+    GF_VALIDATE_OR_GOTO(this->name, conf, out);
+
+    /* 3.9.0 onwards gNFS will be disabled by default. In case of an upgrade
+     * from anything below than 3.9.0 to 3.9.x the volume's dictionary will
+     * not have 'nfs.disable' key set which means the same will not be set
+     * to on until explicitly done. setnfs.disable to 'on' at op-version
+     * bump up flow is the ideal way here. The same is also applicable for
+     * transport.address-family where if the transport type is set to tcp
+     * then transport.address-family is defaulted to 'inet'.
+     */
+    if (conf->op_version >= GD_OP_VERSION_3_9_0) {
+        if (dict_get_str_boolean(volinfo->dict, NFS_DISABLE_MAP_KEY, 1)) {
+            ret = dict_set_dynstr_with_alloc(volinfo->dict, NFS_DISABLE_MAP_KEY,
+                                             "on");
+            if (ret) {
+                gf_msg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_SET_FAILED,
+                       "Failed to set "
+                       "option ' NFS_DISABLE_MAP_KEY ' on "
+                       "volume %s",
+                       volinfo->volname);
+                goto out;
+            }
+        }
+        ret = dict_get_strn(volinfo->dict, "transport.address-family",
+                            SLEN("transport.address-family"),
+                            &address_family_str);
+        if (ret) {
+            if (volinfo->transport_type == GF_TRANSPORT_TCP) {
+                ret = dict_set_dynstr_with_alloc(
+                    volinfo->dict, "transport.address-family", "inet");
                 if (ret) {
-                        gf_log (THIS->name, GF_LOG_ERROR,
-                                "Failed to start %s:%s for %s",
-                                brickinfo->hostname, brickinfo->path,
-                                volinfo->volname);
-                        goto out;
+                    gf_msg(this->name, GF_LOG_ERROR, errno,
+                           GD_MSG_DICT_SET_FAILED,
+                           "failed to set transport."
+                           "address-family on %s",
+                           volinfo->volname);
+                    goto out;
                 }
+            }
         }
+    }
+    ret = glusterd_store_volinfo(volinfo, GLUSTERD_VOLINFO_VER_AC_INCREMENT);
 
-        ret = 0;
 out:
-        return ret;
+    return ret;
 }
 
 static int
-glusterd_op_set_all_volume_options (xlator_t *this, dict_t *dict)
+glusterd_set_brick_mx_opts(dict_t *dict, char *key, char *value,
+                           char **op_errstr)
 {
-        char            *key            = NULL;
-        char            *key_fixed      = NULL;
-        char            *value          = NULL;
-        char            *dup_value      = NULL;
-        int             ret             = -1;
-        glusterd_conf_t *conf           = NULL;
-        dict_t          *dup_opt        = NULL;
-        char            *next_version   = NULL;
-        gf_boolean_t    quorum_action   = _gf_false;
-        uint32_t        op_version      = 0;
-
-        conf = this->private;
-        ret = dict_get_str (dict, "key1", &key);
-        if (ret)
-                goto out;
+    int32_t ret = -1;
+    xlator_t *this = NULL;
+    glusterd_conf_t *priv = NULL;
 
-        ret = dict_get_str (dict, "value1", &value);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "invalid key,value pair in 'volume set'");
-                goto out;
-        }
-        ret = glusterd_check_option_exists (key, &key_fixed);
-        if (ret <= 0) {
-                gf_log (this->name, GF_LOG_ERROR, "Invalid key %s", key);
-                ret = -1;
-                goto out;
-        }
+    this = THIS;
+    GF_VALIDATE_OR_GOTO("glusterd", this, out);
+    GF_VALIDATE_OR_GOTO(this->name, dict, out);
+    GF_VALIDATE_OR_GOTO(this->name, key, out);
+    GF_VALIDATE_OR_GOTO(this->name, value, out);
+    GF_VALIDATE_OR_GOTO(this->name, op_errstr, out);
 
-        if (key_fixed)
-                key = key_fixed;
+    ret = 0;
 
-        /* If the key is cluster.op-version, set conf->op_version to the value
-         * if needed and save it.
-         */
-        if (strcmp(key, "cluster.op-version") == 0) {
-                ret = 0;
+    priv = this->private;
+
+    if (!strcmp(key, GLUSTERD_BRICK_MULTIPLEX_KEY)) {
+        ret = dict_set_dynstrn(priv->opts, GLUSTERD_BRICK_MULTIPLEX_KEY,
+                               SLEN(GLUSTERD_BRICK_MULTIPLEX_KEY),
+                               gf_strdup(value));
+    }
+
+out:
+    return ret;
+}
+
+/* This is a hack to prevent client-io-threads from being loaded in the graph
+ * when the cluster-op-version is bumped up from 3.8.x to 3.13.x. The key is
+ * deleted subsequently in glusterd_create_volfiles(). */
+static int
+glusterd_dict_set_skip_cliot_key(glusterd_volinfo_t *volinfo)
+{
+    return dict_set_int32n(volinfo->dict, "skip-CLIOT", SLEN("skip-CLIOT"), 1);
+}
+
+static int
+glusterd_op_set_all_volume_options(xlator_t *this, dict_t *dict,
+                                   char **op_errstr)
+{
+    char *key = NULL;
+    char *key_fixed = NULL;
+    char *value = NULL;
+    char *dup_value = NULL;
+    int ret = -1;
+    glusterd_conf_t *conf = NULL;
+    dict_t *dup_opt = NULL;
+    char *next_version = NULL;
+    gf_boolean_t quorum_action = _gf_false;
+    uint32_t op_version = 0;
+    glusterd_volinfo_t *volinfo = NULL;
+    glusterd_svc_t *svc = NULL;
+    gf_boolean_t svcs_reconfigure = _gf_false;
+
+    conf = this->private;
+    ret = dict_get_strn(dict, "key1", SLEN("key1"), &key);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_GET_FAILED,
+                "Key=key1", NULL);
+        goto out;
+    }
+
+    ret = dict_get_strn(dict, "value1", SLEN("value1"), &value);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+               "invalid key,value pair in 'volume set'");
+        goto out;
+    }
+
+    ret = glusterd_check_option_exists(key, &key_fixed);
+    if (ret <= 0) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_UNKNOWN_KEY,
+               "Invalid key %s", key);
+        ret = -1;
+        goto out;
+    }
+
+    if (key_fixed)
+        key = key_fixed;
+
+    ret = glusterd_set_shared_storage(dict, key, value, op_errstr);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_SHARED_STRG_SET_FAIL,
+               "Failed to set shared storage option");
+        goto out;
+    }
+
+    ret = glusterd_set_brick_mx_opts(dict, key, value, op_errstr);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_BRICK_MX_SET_FAIL,
+               "Failed to set brick multiplexing option");
+        goto out;
+    }
+
+    /* If the key is cluster.op-version, set conf->op_version to the value
+     * if needed and save it.
+     */
+    if (strcmp(key, GLUSTERD_GLOBAL_OP_VERSION_KEY) == 0) {
+        ret = 0;
 
-                ret = gf_string2uint (value, &op_version);
+        ret = gf_string2uint(value, &op_version);
+        if (ret)
+            goto out;
+
+        if (op_version >= conf->op_version) {
+            conf->op_version = op_version;
+
+            /* When a bump up happens, update the quota.conf file
+             * as well. This is because, till 3.7 we had a quota
+             * conf version v1.1 in quota.conf. When inode-quota
+             * feature is introduced, this needs to be changed to
+             * v1.2 in quota.conf and 16 bytes uuid in quota.conf
+             * needs to be changed to 17 bytes. Look
+             * glusterd_store_quota_config for more details.
+             */
+            cds_list_for_each_entry(volinfo, &conf->volumes, vol_list)
+            {
+                ret = glusterd_store_quota_config(
+                    volinfo, NULL, NULL, GF_QUOTA_OPTION_TYPE_UPGRADE, NULL);
                 if (ret)
+                    goto out;
+                ret = glusterd_update_volumes_dict(volinfo);
+                if (ret)
+                    goto out;
+
+                if (glusterd_dict_set_skip_cliot_key(volinfo))
+                    goto out;
+
+                if (!volinfo->is_snap_volume) {
+                    svc = &(volinfo->snapd.svc);
+                    ret = svc->manager(svc, volinfo, PROC_START_NO_WAIT);
+                    if (ret)
                         goto out;
+                }
 
-                if (op_version >= conf->op_version) {
-                        conf->op_version = op_version;
-                        ret = glusterd_store_global_info (this);
-                        if (ret) {
-                                gf_log (this->name, GF_LOG_ERROR,
-                                        "Failed to store op-version.");
-                        }
+                svc = &(volinfo->gfproxyd.svc);
+                ret = svc->reconfigure(volinfo);
+                if (ret)
+                    goto out;
+
+                svc = &(volinfo->shd.svc);
+                ret = svc->reconfigure(volinfo);
+                if (ret)
+                    goto out;
+
+                ret = glusterd_create_volfiles_and_notify_services(volinfo);
+                if (ret) {
+                    gf_msg(this->name, GF_LOG_ERROR, 0,
+                           GD_MSG_VOLFILE_CREATE_FAIL,
+                           "Unable to create volfile for"
+                           " 'volume set'");
+                    goto out;
                 }
-                /* No need to save cluster.op-version in conf->opts
-                 */
-                goto out;
+                if (GLUSTERD_STATUS_STARTED == volinfo->status) {
+                    svcs_reconfigure = _gf_true;
+                }
+            }
+            if (svcs_reconfigure) {
+                ret = glusterd_svcs_reconfigure(NULL);
+                if (ret) {
+                    gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_SVC_RESTART_FAIL,
+                           "Unable to restart "
+                           "services");
+                    goto out;
+                }
+            }
+
+            ret = glusterd_store_global_info(this);
+            if (ret) {
+                gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_OP_VERS_STORE_FAIL,
+                       "Failed to store op-version.");
+            }
         }
-        ret = -1;
-        dup_opt = dict_new ();
-        if (!dup_opt)
-                goto out;
-        dict_copy (conf->opts, dup_opt);
-        ret = dict_set_str (dup_opt, key, value);
-        if (ret)
-                goto out;
+        /* No need to save cluster.op-version in conf->opts
+         */
+        goto out;
+    }
+    ret = -1;
+    dup_opt = dict_new();
+    if (!dup_opt) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_CREATE_FAIL, NULL);
+        goto out;
+    }
+    dict_copy(conf->opts, dup_opt);
+    ret = dict_set_str(dup_opt, key, value);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_SET_FAILED,
+                "Key=%s", key, NULL);
+        goto out;
+    }
+
+    ret = glusterd_get_next_global_opt_version_str(conf->opts, &next_version);
+    if (ret)
+        goto out;
+
+    ret = dict_set_strn(dup_opt, GLUSTERD_GLOBAL_OPT_VERSION,
+                        SLEN(GLUSTERD_GLOBAL_OPT_VERSION), next_version);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_SET_FAILED,
+                "Key=%s", GLUSTERD_GLOBAL_OPT_VERSION, NULL);
+        goto out;
+    }
+
+    ret = glusterd_store_options(this, dup_opt);
+    if (ret)
+        goto out;
+
+    if (glusterd_is_quorum_changed(conf->opts, key, value))
+        quorum_action = _gf_true;
+
+    ret = dict_set_dynstrn(conf->opts, GLUSTERD_GLOBAL_OPT_VERSION,
+                           SLEN(GLUSTERD_GLOBAL_OPT_VERSION), next_version);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_SET_FAILED,
+                "Key=%s", GLUSTERD_GLOBAL_OPT_VERSION, NULL);
+        goto out;
+    } else
+        next_version = NULL;
+
+    dup_value = gf_strdup(value);
+    if (!dup_value)
+        goto out;
+
+    ret = dict_set_dynstr(conf->opts, key, dup_value);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_SET_FAILED,
+                "Key=%s", key, NULL);
+        goto out;
+    } else
+        dup_value = NULL; /* Protect the allocation from GF_FREE */
 
-        ret = glusterd_get_next_global_opt_version_str (conf->opts,
-                                                        &next_version);
-        if (ret)
-                goto out;
+out:
+    GF_FREE(dup_value);
+    GF_FREE(key_fixed);
+    if (dup_opt)
+        dict_unref(dup_opt);
+
+    gf_msg_debug(this->name, 0, "returning %d", ret);
+    if (quorum_action)
+        glusterd_do_quorum_action();
+    GF_FREE(next_version);
+    return ret;
+}
 
-        ret = dict_set_str (dup_opt, GLUSTERD_GLOBAL_OPT_VERSION, next_version);
-        if (ret)
-                goto out;
+int
+glusterd_op_get_max_opversion(char **op_errstr, dict_t *rsp_dict)
+{
+    int ret = -1;
 
-        ret = glusterd_store_options (this, dup_opt);
-        if (ret)
-                goto out;
+    GF_VALIDATE_OR_GOTO(THIS->name, rsp_dict, out);
 
-        if (glusterd_is_quorum_changed (conf->opts, key, value))
-                quorum_action = _gf_true;
+    ret = dict_set_int32n(rsp_dict, "max-opversion", SLEN("max-opversion"),
+                          GD_OP_VERSION_MAX);
+    if (ret) {
+        gf_msg(THIS->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+               "Setting value for max-opversion to dict failed");
+        goto out;
+    }
 
-        ret = dict_set_dynstr (conf->opts, GLUSTERD_GLOBAL_OPT_VERSION,
-                               next_version);
-        if (ret)
-                goto out;
-        else
-                next_version = NULL;
+out:
+    gf_msg_debug(THIS->name, 0, "Returning %d", ret);
+    return ret;
+}
 
-        dup_value = gf_strdup (value);
-        if (!dup_value)
-                goto out;
+static int
+glusterd_set_shared_storage(dict_t *dict, char *key, char *value,
+                            char **op_errstr)
+{
+    int32_t ret = -1;
+    char hooks_args[PATH_MAX] = {
+        0,
+    };
+    char errstr[PATH_MAX] = {
+        0,
+    };
+    xlator_t *this = NULL;
+    int32_t len = 0;
+
+    this = THIS;
+    GF_VALIDATE_OR_GOTO("glusterd", this, out);
+    GF_VALIDATE_OR_GOTO(this->name, dict, out);
+    GF_VALIDATE_OR_GOTO(this->name, key, out);
+    GF_VALIDATE_OR_GOTO(this->name, value, out);
+    GF_VALIDATE_OR_GOTO(this->name, op_errstr, out);
+
+    ret = 0;
+
+    if (strcmp(key, GLUSTERD_SHARED_STORAGE_KEY)) {
+        goto out;
+    }
+
+    /* Re-create the brick path so as to be *
+     * able to re-use it                    *
+     */
+    ret = recursive_rmdir(GLUSTER_SHARED_STORAGE_BRICK_DIR);
+    if (ret) {
+        snprintf(errstr, PATH_MAX,
+                 "Failed to remove shared "
+                 "storage brick(%s). "
+                 "Reason: %s",
+                 GLUSTER_SHARED_STORAGE_BRICK_DIR, strerror(errno));
+        gf_msg(this->name, GF_LOG_ERROR, errno, GD_MSG_DIR_OP_FAILED, "%s",
+               errstr);
+        ret = -1;
+        goto out;
+    }
+
+    ret = mkdir_p(GLUSTER_SHARED_STORAGE_BRICK_DIR, 0755, _gf_true);
+    if (-1 == ret) {
+        snprintf(errstr, PATH_MAX,
+                 "Failed to create shared "
+                 "storage brick(%s). "
+                 "Reason: %s",
+                 GLUSTER_SHARED_STORAGE_BRICK_DIR, strerror(errno));
+        gf_msg(this->name, GF_LOG_ERROR, errno, GD_MSG_CREATE_DIR_FAILED, "%s",
+               errstr);
+        goto out;
+    }
+
+    if (is_origin_glusterd(dict)) {
+        len = snprintf(hooks_args, sizeof(hooks_args),
+                       "is_originator=1,local_node_hostname=%s",
+                       local_node_hostname);
+    } else {
+        len = snprintf(hooks_args, sizeof(hooks_args),
+                       "is_originator=0,local_node_hostname=%s",
+                       local_node_hostname);
+    }
+    if ((len < 0) || (len >= sizeof(hooks_args))) {
+        ret = -1;
+        goto out;
+    }
 
-        ret = dict_set_dynstr (conf->opts, key, dup_value);
-        if (ret)
-                goto out;
-        else
-                dup_value = NULL; /* Protect the allocation from GF_FREE */
+    ret = dict_set_dynstr_with_alloc(dict, "hooks_args", hooks_args);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_SET_FAILED,
+               "Failed to set"
+               " hooks_args in dict.");
+        goto out;
+    }
 
 out:
-        GF_FREE (dup_value);
-        GF_FREE (key_fixed);
-        if (dup_opt)
-                dict_unref (dup_opt);
-
-        gf_log (this->name, GF_LOG_DEBUG, "returning %d", ret);
-        if (quorum_action)
-                glusterd_do_quorum_action ();
-        GF_FREE (next_version);
-        return ret;
+    if (ret && strlen(errstr)) {
+        *op_errstr = gf_strdup(errstr);
+    }
+
+    return ret;
 }
 
 static int
-glusterd_op_set_volume (dict_t *dict, char **errstr)
+glusterd_op_set_volume(dict_t *dict, char **errstr)
 {
-        int                                      ret = 0;
-        glusterd_volinfo_t                      *volinfo = NULL;
-        char                                    *volname = NULL;
-        xlator_t                                *this = NULL;
-        glusterd_conf_t                         *priv = NULL;
-        int                                      count = 1;
-        char                                    *key = NULL;
-        char                                    *key_fixed = NULL;
-        char                                    *value = NULL;
-        char                                     str[50] = {0, };
-        char                                    *op_errstr = NULL;
-        gf_boolean_t                             global_opt    = _gf_false;
-        gf_boolean_t                             global_opts_set = _gf_false;
-        glusterd_volinfo_t                      *voliter = NULL;
-        int32_t                                  dict_count = 0;
-        gf_boolean_t                             check_op_version = _gf_false;
-        uint32_t                                 new_op_version = 0;
-        gf_boolean_t                             quorum_action  = _gf_false;
-        glusterd_svc_t                          *svc            = NULL;
-
-        this = THIS;
-        GF_ASSERT (this);
-
-        priv = this->private;
-        GF_ASSERT (priv);
-
-        ret = dict_get_int32 (dict, "count", &dict_count);
+    int ret = 0;
+    glusterd_volinfo_t *volinfo = NULL;
+    char *volname = NULL;
+    xlator_t *this = NULL;
+    glusterd_conf_t *priv = NULL;
+    int count = 1;
+    char *key = NULL;
+    char *key_fixed = NULL;
+    char *value = NULL;
+    char keystr[50] = {
+        0,
+    };
+    int keylen;
+    gf_boolean_t global_opt = _gf_false;
+    gf_boolean_t global_opts_set = _gf_false;
+    glusterd_volinfo_t *voliter = NULL;
+    int32_t dict_count = 0;
+    gf_boolean_t check_op_version = _gf_false;
+    uint32_t new_op_version = 0;
+    gf_boolean_t quorum_action = _gf_false;
+    glusterd_svc_t *svc = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+
+    priv = this->private;
+    GF_ASSERT(priv);
+
+    ret = dict_get_int32n(dict, "count", SLEN("count"), &dict_count);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+               "Count(dict),not set in Volume-Set");
+        goto out;
+    }
+
+    if (dict_count == 0) {
+        ret = glusterd_volset_help(NULL, errstr);
+        goto out;
+    }
+
+    ret = dict_get_strn(dict, "volname", SLEN("volname"), &volname);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+               "Unable to get volume name");
+        goto out;
+    }
+
+    if (strcasecmp(volname, "all") == 0) {
+        ret = glusterd_op_set_all_volume_options(this, dict, errstr);
+        goto out;
+    }
+
+    ret = glusterd_volinfo_find(volname, &volinfo);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_VOL_NOT_FOUND,
+               FMTSTR_CHECK_VOL_EXISTS, volname);
+        goto out;
+    }
+
+    /* TODO: Remove this once v3.3 compatibility is not required */
+    check_op_version = dict_get_str_boolean(dict, "check-op-version",
+                                            _gf_false);
+
+    if (check_op_version) {
+        ret = dict_get_uint32(dict, "new-op-version", &new_op_version);
         if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "Count(dict),not set in Volume-Set");
-                goto out;
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+                   "Unable to get new op-version from dict");
+            goto out;
         }
+    }
 
-        if (dict_count == 0) {
-                ret = glusterd_volset_help (NULL, &op_errstr);
-                if (ret) {
-                        gf_log (this->name, GF_LOG_ERROR, "%s",
-                                       (op_errstr)? op_errstr:
-                                       "Volume set help internal error");
-                }
-
-                GF_FREE(op_errstr);
-                goto out;
-         }
+    for (count = 1; ret != -1; count++) {
+        keylen = snprintf(keystr, sizeof(keystr), "key%d", count);
+        ret = dict_get_strn(dict, keystr, keylen, &key);
+        if (ret)
+            break;
 
-        ret = dict_get_str (dict, "volname", &volname);
+        keylen = snprintf(keystr, sizeof(keystr), "value%d", count);
+        ret = dict_get_strn(dict, keystr, keylen, &value);
         if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "Unable to get volume name");
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+                   "invalid key,value pair in 'volume set'");
+            ret = -1;
+            goto out;
+        }
+
+        if (strcmp(key, "config.memory-accounting") == 0) {
+            ret = gf_string2boolean(value, &volinfo->memory_accounting);
+            if (ret == -1) {
+                gf_msg(this->name, GF_LOG_ERROR, EINVAL, GD_MSG_INVALID_ENTRY,
+                       "Invalid value in key-value pair.");
+                goto out;
+            }
+        }
+
+        if (strcmp(key, "config.transport") == 0) {
+            gf_msg(this->name, GF_LOG_INFO, 0, GD_MSG_VOL_TRANSPORT_TYPE_CHANGE,
+                   "changing transport-type for volume %s to %s", volname,
+                   value);
+            ret = 0;
+            if (strcasecmp(value, "rdma") == 0) {
+                volinfo->transport_type = GF_TRANSPORT_RDMA;
+            } else if (strcasecmp(value, "tcp") == 0) {
+                volinfo->transport_type = GF_TRANSPORT_TCP;
+            } else if ((strcasecmp(value, "tcp,rdma") == 0) ||
+                       (strcasecmp(value, "rdma,tcp") == 0)) {
+                volinfo->transport_type = GF_TRANSPORT_BOTH_TCP_RDMA;
+            } else {
+                ret = -1;
                 goto out;
+            }
         }
 
-        if (strcasecmp (volname, "all") == 0) {
-                ret = glusterd_op_set_all_volume_options (this, dict);
+        ret = glusterd_check_ganesha_cmd(key, value, errstr, dict);
+        if (ret == -1)
+            goto out;
+
+        if (!is_key_glusterd_hooks_friendly(key)) {
+            ret = glusterd_check_option_exists(key, &key_fixed);
+            GF_ASSERT(ret);
+            if (ret <= 0) {
+                key_fixed = NULL;
                 goto out;
+            }
         }
 
-        ret = glusterd_volinfo_find (volname, &volinfo);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, FMTSTR_CHECK_VOL_EXISTS,
-                        volname);
-                goto out;
+        global_opt = _gf_false;
+        if (glusterd_check_globaloption(key)) {
+            global_opt = _gf_true;
+            global_opts_set = _gf_true;
         }
 
-        /* TODO: Remove this once v3.3 compatibility is not required */
-        check_op_version = dict_get_str_boolean (dict, "check-op-version",
-                                                 _gf_false);
+        if (!global_opt)
+            value = gf_strdup(value);
 
-        if (check_op_version) {
-                ret = dict_get_uint32 (dict, "new-op-version", &new_op_version);
-                if (ret) {
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "Unable to get new op-version from dict");
-                        goto out;
-                }
+        if (!value) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_VOL_SET_FAIL,
+                   "Unable to set the options in 'volume set'");
+            ret = -1;
+            goto out;
         }
 
-        for (count = 1; ret != -1 ; count++) {
+        if (key_fixed)
+            key = key_fixed;
+
+        if (glusterd_is_quorum_changed(volinfo->dict, key, value))
+            quorum_action = _gf_true;
 
-                sprintf (str, "key%d", count);
-                ret = dict_get_str (dict, str, &key);
+        if (global_opt) {
+            cds_list_for_each_entry(voliter, &priv->volumes, vol_list)
+            {
+                value = gf_strdup(value);
+                ret = dict_set_dynstr(voliter->dict, key, value);
                 if (ret)
-                        break;
+                    goto out;
+            }
+        } else {
+            ret = dict_set_dynstr(volinfo->dict, key, value);
+            if (ret)
+                goto out;
+        }
 
-                sprintf (str, "value%d", count);
-                ret = dict_get_str (dict, str, &value);
-                if (ret) {
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "invalid key,value pair in 'volume set'");
-                        ret = -1;
-                        goto out;
-                }
+        if (key_fixed) {
+            GF_FREE(key_fixed);
+            key_fixed = NULL;
+        }
+    }
 
-                if (strcmp (key, "config.memory-accounting") == 0) {
-                        ret = gf_string2boolean (value,
-                                                 &volinfo->memory_accounting);
-                }
+    if (count == 1) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_NO_OPTIONS_GIVEN,
+               "No options received ");
+        ret = -1;
+        goto out;
+    }
+
+    /* Update the cluster op-version before regenerating volfiles so that
+     * correct volfiles are generated
+     */
+    if (new_op_version > priv->op_version) {
+        priv->op_version = new_op_version;
+        ret = glusterd_store_global_info(this);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_OP_VERS_STORE_FAIL,
+                   "Failed to store op-version");
+            goto out;
+        }
+    }
+    if (!global_opts_set) {
+        gd_update_volume_op_versions(volinfo);
 
-                if (strcmp (key, "config.transport") == 0) {
-                        gf_log (this->name, GF_LOG_INFO,
-                                "changing transport-type for volume %s to %s",
-                                volname, value);
-                        ret = 0;
-                        if (strcasecmp (value, "rdma") == 0) {
-                                volinfo->transport_type = GF_TRANSPORT_RDMA;
-                        } else if (strcasecmp (value, "tcp") == 0) {
-                                volinfo->transport_type = GF_TRANSPORT_TCP;
-                        } else if ((strcasecmp (value, "tcp,rdma") == 0) ||
-                                   (strcasecmp (value, "rdma,tcp") == 0)) {
-                                volinfo->transport_type =
-                                        GF_TRANSPORT_BOTH_TCP_RDMA;
-                        } else {
-                                ret = -1;
-                                goto out;
-                        }
-                }
+        if (!volinfo->is_snap_volume) {
+            svc = &(volinfo->snapd.svc);
+            ret = svc->manager(svc, volinfo, PROC_START_NO_WAIT);
+            if (ret)
+                goto out;
+        }
+        svc = &(volinfo->gfproxyd.svc);
+        ret = svc->reconfigure(volinfo);
+        if (ret)
+            goto out;
 
-                ret =  glusterd_check_ganesha_cmd (key, value, errstr, dict);
-                if (ret == -1)
-                        goto out;
-                if (!is_key_glusterd_hooks_friendly (key)) {
-                        ret = glusterd_check_option_exists (key, &key_fixed);
-                        GF_ASSERT (ret);
-                        if (ret <= 0) {
-                                key_fixed = NULL;
-                                goto out;
-                        }
-                }
+        svc = &(volinfo->shd.svc);
+        ret = svc->reconfigure(volinfo);
+        if (ret)
+            goto out;
 
-                global_opt = _gf_false;
-                if (glusterd_check_globaloption (key)) {
-                        global_opt = _gf_true;
-                        global_opts_set = _gf_true;
-                }
+        ret = glusterd_create_volfiles_and_notify_services(volinfo);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_VOLFILE_CREATE_FAIL,
+                   "Unable to create volfile for"
+                   " 'volume set'");
+            ret = -1;
+            goto out;
+        }
 
-                if (!global_opt)
-                        value = gf_strdup (value);
+        ret = glusterd_store_volinfo(volinfo,
+                                     GLUSTERD_VOLINFO_VER_AC_INCREMENT);
+        if (ret)
+            goto out;
 
-                if (!value) {
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "Unable to set the options in 'volume set'");
-                        ret = -1;
-                        goto out;
-                }
+        if (GLUSTERD_STATUS_STARTED == volinfo->status) {
+            ret = glusterd_svcs_reconfigure(volinfo);
+            if (ret) {
+                gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_SVC_RESTART_FAIL,
+                       "Unable to restart services");
+                goto out;
+            }
+        }
 
-                if (key_fixed)
-                        key = key_fixed;
+    } else {
+        cds_list_for_each_entry(voliter, &priv->volumes, vol_list)
+        {
+            volinfo = voliter;
+            gd_update_volume_op_versions(volinfo);
 
-                if (glusterd_is_quorum_changed (volinfo->dict, key, value))
-                        quorum_action = _gf_true;
+            if (!volinfo->is_snap_volume) {
+                svc = &(volinfo->snapd.svc);
+                ret = svc->manager(svc, volinfo, PROC_START_NO_WAIT);
+                if (ret)
+                    goto out;
+            }
 
-                if (global_opt) {
-                        cds_list_for_each_entry (voliter, &priv->volumes,
-                                                 vol_list) {
-                                value = gf_strdup (value);
-                                ret = dict_set_dynstr (voliter->dict, key,
-                                                       value);
-                                if (ret)
-                                        goto out;
-                        }
-                } else {
-                        ret = dict_set_dynstr (volinfo->dict, key, value);
-                        if (ret)
-                                goto out;
-                }
+            svc = &(volinfo->gfproxyd.svc);
+            ret = svc->reconfigure(volinfo);
+            if (ret)
+                goto out;
 
-                if (key_fixed) {
-                        GF_FREE (key_fixed);
-                        key_fixed = NULL;
-                }
-        }
+            svc = &(volinfo->shd.svc);
+            ret = svc->reconfigure(volinfo);
+            if (ret)
+                goto out;
 
-        if (count == 1) {
-                gf_log (this->name, GF_LOG_ERROR, "No options received ");
+            ret = glusterd_create_volfiles_and_notify_services(volinfo);
+            if (ret) {
+                gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_VOLFILE_CREATE_FAIL,
+                       "Unable to create volfile for"
+                       " 'volume set'");
                 ret = -1;
                 goto out;
-        }
+            }
 
-        /* Update the cluster op-version before regenerating volfiles so that
-         * correct volfiles are generated
-         */
-        if (new_op_version > priv->op_version) {
-                priv->op_version = new_op_version;
-                ret = glusterd_store_global_info (this);
-                if (ret) {
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "Failed to store op-version");
-                        goto out;
-                }
-        }
-        if (!global_opts_set) {
-                gd_update_volume_op_versions (volinfo);
+            ret = glusterd_store_volinfo(volinfo,
+                                         GLUSTERD_VOLINFO_VER_AC_INCREMENT);
+            if (ret)
+                goto out;
 
-                if (!volinfo->is_snap_volume) {
-                        svc = &(volinfo->snapd.svc);
-                        ret = svc->manager (svc, volinfo, PROC_START_NO_WAIT);
-                        if (ret)
-                                goto out;
-                }
-                ret = glusterd_create_volfiles_and_notify_services (volinfo);
+            if (GLUSTERD_STATUS_STARTED == volinfo->status) {
+                ret = glusterd_svcs_reconfigure(volinfo);
                 if (ret) {
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "Unable to create volfile for"
-                                " 'volume set'");
-                        ret = -1;
-                        goto out;
-                }
-
-                ret = glusterd_store_volinfo (volinfo, GLUSTERD_VOLINFO_VER_AC_INCREMENT);
-                if (ret)
-                        goto out;
-
-                if (GLUSTERD_STATUS_STARTED == volinfo->status) {
-                        ret = glusterd_svcs_reconfigure (volinfo);
-                        if (ret) {
-                                gf_log (this->name, GF_LOG_ERROR,
-                                         "Unable to restart services");
-                                goto out;
-                        }
-                }
-
-        } else {
-                cds_list_for_each_entry (voliter, &priv->volumes, vol_list) {
-                        volinfo = voliter;
-                        gd_update_volume_op_versions (volinfo);
-
-                        if (!volinfo->is_snap_volume) {
-                                svc = &(volinfo->snapd.svc);
-                                ret = svc->manager (svc, volinfo,
-                                                    PROC_START_NO_WAIT);
-                                if (ret)
-                                        goto out;
-                        }
-
-                        ret = glusterd_create_volfiles_and_notify_services (volinfo);
-                        if (ret) {
-                                gf_log (this->name, GF_LOG_ERROR,
-                                        "Unable to create volfile for"
-                                        " 'volume set'");
-                                ret = -1;
-                                goto out;
-                        }
-
-                        ret = glusterd_store_volinfo (volinfo,
-                                      GLUSTERD_VOLINFO_VER_AC_INCREMENT);
-                        if (ret)
-                                goto out;
-
-                        if (GLUSTERD_STATUS_STARTED == volinfo->status) {
-                                ret = glusterd_svcs_reconfigure (volinfo);
-                                if (ret) {
-                                        gf_log (this->name, GF_LOG_WARNING,
-                                                "Unable to restart NFS-Server");
-                                        goto out;
-                                }
-                        }
+                    gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_SVC_RESTART_FAIL,
+                           "Unable to restart services");
+                    goto out;
                 }
+            }
         }
+    }
 
- out:
-        GF_FREE (key_fixed);
-        gf_log (this->name, GF_LOG_DEBUG, "returning %d", ret);
-        if (quorum_action)
-                glusterd_do_quorum_action ();
-        return ret;
+out:
+    GF_FREE(key_fixed);
+    gf_msg_debug(this->name, 0, "returning %d", ret);
+    if (quorum_action)
+        glusterd_do_quorum_action();
+    return ret;
 }
 
-
 static int
-glusterd_op_sync_volume (dict_t *dict, char **op_errstr,
-                         dict_t *rsp_dict)
+glusterd_op_sync_volume(dict_t *dict, char **op_errstr, dict_t *rsp_dict)
 {
-        int                                     ret = -1;
-        char                                    *volname = NULL;
-        char                                    *hostname = NULL;
-        char                                    msg[2048] = {0,};
-        int                                     count = 1;
-        int                                     vol_count = 0;
-        glusterd_conf_t                         *priv = NULL;
-        glusterd_volinfo_t                      *volinfo = NULL;
-        xlator_t                                *this = NULL;
-
-        this = THIS;
-        GF_ASSERT (this);
-        priv = this->private;
-        GF_ASSERT (priv);
-
-        ret = dict_get_str (dict, "hostname", &hostname);
-        if (ret) {
-                snprintf (msg, sizeof (msg), "hostname couldn't be "
-                          "retrieved from msg");
-                *op_errstr = gf_strdup (msg);
-                goto out;
-        }
+    int ret = -1;
+    char *volname = NULL;
+    char *hostname = NULL;
+    char msg[2048] = {
+        0,
+    };
+    int count = 1;
+    int vol_count = 0;
+    glusterd_conf_t *priv = NULL;
+    glusterd_volinfo_t *volinfo = NULL;
+    xlator_t *this = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+    priv = this->private;
+    GF_ASSERT(priv);
+
+    ret = dict_get_strn(dict, "hostname", SLEN("hostname"), &hostname);
+    if (ret) {
+        snprintf(msg, sizeof(msg),
+                 "hostname couldn't be "
+                 "retrieved from msg");
+        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_GET_FAILED,
+                "Key=hostname", NULL);
+        *op_errstr = gf_strdup(msg);
+        goto out;
+    }
+
+    if (!gf_is_local_addr(hostname)) {
+        ret = 0;
+        goto out;
+    }
 
-        if (!gf_is_local_addr (hostname)) {
-                ret = 0;
-                goto out;
+    // volname is not present in case of sync all
+    ret = dict_get_strn(dict, "volname", SLEN("volname"), &volname);
+    if (!ret) {
+        ret = glusterd_volinfo_find(volname, &volinfo);
+        if (ret) {
+            gf_msg("glusterd", GF_LOG_ERROR, 0, GD_MSG_VOL_NOT_FOUND,
+                   "Volume with name: %s "
+                   "not exists",
+                   volname);
+            goto out;
         }
+    }
 
-        //volname is not present in case of sync all
-        ret = dict_get_str (dict, "volname", &volname);
-        if (!ret) {
-                ret = glusterd_volinfo_find (volname, &volinfo);
-                if (ret) {
-                        gf_log ("", GF_LOG_ERROR, "Volume with name: %s "
-                                "not exists", volname);
-                        goto out;
-                }
-        }
+    if (!rsp_dict) {
+        // this should happen only on source
+        gf_smsg(this->name, GF_LOG_INFO, errno, GD_MSG_INVALID_ARGUMENT, NULL);
+        ret = 0;
+        goto out;
+    }
 
-        if (!rsp_dict) {
-                //this should happen only on source
-                ret = 0;
+    if (volname) {
+        ret = glusterd_add_volume_to_dict(volinfo, rsp_dict, 1, "volume");
+        if (ret)
+            goto out;
+        vol_count = 1;
+    } else {
+        cds_list_for_each_entry(volinfo, &priv->volumes, vol_list)
+        {
+            ret = glusterd_add_volume_to_dict(volinfo, rsp_dict, count,
+                                              "volume");
+            if (ret)
                 goto out;
-        }
 
-        if (volname) {
-                ret = glusterd_add_volume_to_dict (volinfo, rsp_dict,
-                                                   1, "volume");
-                vol_count = 1;
-        } else {
-                cds_list_for_each_entry (volinfo, &priv->volumes, vol_list) {
-                        ret = glusterd_add_volume_to_dict (volinfo, rsp_dict,
-                                                           count, "volume");
-                        if (ret)
-                                goto out;
-
-                        vol_count = count++;
-                }
+            vol_count = count++;
         }
-        ret = dict_set_int32 (rsp_dict, "count", vol_count);
+    }
+    ret = dict_set_int32n(rsp_dict, "count", SLEN("count"), vol_count);
 
 out:
-        gf_log ("", GF_LOG_DEBUG, "Returning %d", ret);
+    gf_msg_debug("glusterd", 0, "Returning %d", ret);
 
-        return ret;
+    return ret;
 }
 
 static int
-glusterd_add_profile_volume_options (glusterd_volinfo_t *volinfo)
+glusterd_add_profile_volume_options(glusterd_volinfo_t *volinfo)
 {
-        int                                     ret = -1;
-        char                                    *latency_key = NULL;
-        char                                    *fd_stats_key = NULL;
-
-        GF_ASSERT (volinfo);
-
-        latency_key = VKEY_DIAG_LAT_MEASUREMENT;
-        fd_stats_key = VKEY_DIAG_CNT_FOP_HITS;
-
-        ret = dict_set_str (volinfo->dict, latency_key, "on");
-        if (ret) {
-                gf_log ("glusterd", GF_LOG_ERROR, "failed to set the volume %s "
-                        "option %s value %s",
-                        volinfo->volname, latency_key, "on");
-                goto out;
-        }
-
-        ret = dict_set_str (volinfo->dict, fd_stats_key, "on");
-        if (ret) {
-                gf_log ("glusterd", GF_LOG_ERROR, "failed to set the volume %s "
-                        "option %s value %s",
-                        volinfo->volname, fd_stats_key, "on");
-                goto out;
-        }
+    int ret = -1;
+
+    GF_ASSERT(volinfo);
+
+    ret = dict_set_nstrn(volinfo->dict, VKEY_DIAG_LAT_MEASUREMENT,
+                         SLEN(VKEY_DIAG_LAT_MEASUREMENT), "on", SLEN("on"));
+    if (ret) {
+        gf_msg("glusterd", GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+               "failed to set the volume %s "
+               "option %s value %s",
+               volinfo->volname, VKEY_DIAG_LAT_MEASUREMENT, "on");
+        goto out;
+    }
+
+    ret = dict_set_nstrn(volinfo->dict, VKEY_DIAG_CNT_FOP_HITS,
+                         SLEN(VKEY_DIAG_CNT_FOP_HITS), "on", SLEN("on"));
+    if (ret) {
+        gf_msg("glusterd", GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+               "failed to set the volume %s "
+               "option %s value %s",
+               volinfo->volname, VKEY_DIAG_CNT_FOP_HITS, "on");
+        goto out;
+    }
 out:
-        gf_log ("", GF_LOG_DEBUG, "Returning %d", ret);
-        return ret;
+    gf_msg_debug("glusterd", 0, "Returning %d", ret);
+    return ret;
 }
 
 static void
-glusterd_remove_profile_volume_options (glusterd_volinfo_t *volinfo)
+glusterd_remove_profile_volume_options(glusterd_volinfo_t *volinfo)
 {
-        char                                    *latency_key = NULL;
-        char                                    *fd_stats_key = NULL;
+    GF_ASSERT(volinfo);
 
-        GF_ASSERT (volinfo);
-
-        latency_key = VKEY_DIAG_LAT_MEASUREMENT;
-        fd_stats_key = VKEY_DIAG_CNT_FOP_HITS;
-        dict_del (volinfo->dict, latency_key);
-        dict_del (volinfo->dict, fd_stats_key);
+    dict_del_sizen(volinfo->dict, VKEY_DIAG_LAT_MEASUREMENT);
+    dict_del_sizen(volinfo->dict, VKEY_DIAG_CNT_FOP_HITS);
 }
 
-static int
-glusterd_op_stats_volume (dict_t *dict, char **op_errstr,
-                          dict_t *rsp_dict)
+int
+glusterd_op_stats_volume(dict_t *dict, char **op_errstr, dict_t *rsp_dict)
 {
-        int                                     ret = -1;
-        char                                    *volname = NULL;
-        char                                    msg[2048] = {0,};
-        glusterd_volinfo_t                      *volinfo = NULL;
-        int32_t                                 stats_op = GF_CLI_STATS_NONE;
-
-        ret = dict_get_str (dict, "volname", &volname);
-        if (ret) {
-                gf_log ("glusterd", GF_LOG_ERROR, "volume name get failed");
-                goto out;
-        }
-
-        ret = glusterd_volinfo_find (volname, &volinfo);
-        if (ret) {
-                snprintf (msg, sizeof (msg), "Volume %s does not exists",
-                          volname);
-
-                gf_log ("", GF_LOG_ERROR, "%s", msg);
-                goto out;
-        }
-
-        ret = dict_get_int32 (dict, "op", &stats_op);
-        if (ret) {
-                gf_log ("glusterd", GF_LOG_ERROR, "volume profile op get failed");
-                goto out;
-        }
-
-        switch (stats_op) {
+    int ret = -1;
+    char *volname = NULL;
+    char msg[2048] = {
+        0,
+    };
+    glusterd_volinfo_t *volinfo = NULL;
+    int32_t stats_op = GF_CLI_STATS_NONE;
+
+    ret = dict_get_strn(dict, "volname", SLEN("volname"), &volname);
+    if (ret) {
+        gf_msg("glusterd", GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+               "volume name get failed");
+        goto out;
+    }
+
+    ret = glusterd_volinfo_find(volname, &volinfo);
+    if (ret) {
+        snprintf(msg, sizeof(msg), "Volume %s does not exists", volname);
+
+        gf_msg("glusterd", GF_LOG_ERROR, 0, GD_MSG_VOL_NOT_FOUND, "%s", msg);
+        goto out;
+    }
+
+    ret = dict_get_int32n(dict, "op", SLEN("op"), &stats_op);
+    if (ret) {
+        gf_msg("glusterd", GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+               "volume profile op get failed");
+        goto out;
+    }
+
+    switch (stats_op) {
         case GF_CLI_STATS_START:
-                ret = glusterd_add_profile_volume_options (volinfo);
-                if (ret)
-                        goto out;
-                break;
+            ret = glusterd_add_profile_volume_options(volinfo);
+            if (ret)
+                goto out;
+            break;
         case GF_CLI_STATS_STOP:
-                glusterd_remove_profile_volume_options (volinfo);
-                break;
+            glusterd_remove_profile_volume_options(volinfo);
+            break;
         case GF_CLI_STATS_INFO:
         case GF_CLI_STATS_TOP:
-                //info is already collected in brick op.
-                //just goto out;
-                ret = 0;
-                goto out;
-                break;
+            // info is already collected in brick op.
+            // just goto out;
+            ret = 0;
+            goto out;
+            break;
         default:
-                GF_ASSERT (0);
-                gf_log ("glusterd", GF_LOG_ERROR, "Invalid profile op: %d",
-                        stats_op);
-                ret = -1;
-                goto out;
-                break;
-        }
-        ret = glusterd_create_volfiles_and_notify_services (volinfo);
+            GF_ASSERT(0);
+            gf_msg("glusterd", GF_LOG_ERROR, 0, GD_MSG_INVALID_ENTRY,
+                   "Invalid profile op: %d", stats_op);
+            ret = -1;
+            goto out;
+            break;
+    }
+    ret = glusterd_create_volfiles_and_notify_services(volinfo);
+
+    if (ret) {
+        gf_msg("glusterd", GF_LOG_ERROR, 0, GD_MSG_VOLFILE_CREATE_FAIL,
+               "Unable to create volfile for"
+               " 'volume set'");
+        ret = -1;
+        goto out;
+    }
 
-        if (ret) {
-                gf_log ("", GF_LOG_ERROR, "Unable to create volfile for"
-                                          " 'volume set'");
-                ret = -1;
-                goto out;
-        }
+    ret = glusterd_store_volinfo(volinfo, GLUSTERD_VOLINFO_VER_AC_INCREMENT);
+    if (ret)
+        goto out;
 
-        ret = glusterd_store_volinfo (volinfo,
-                                      GLUSTERD_VOLINFO_VER_AC_INCREMENT);
+    if (GLUSTERD_STATUS_STARTED == volinfo->status) {
+        ret = glusterd_svcs_reconfigure(volinfo);
         if (ret)
-                goto out;
-
-        if (GLUSTERD_STATUS_STARTED == volinfo->status)
-                ret = glusterd_svcs_reconfigure (volinfo);
+            goto out;
+    }
 
-        ret = 0;
+    ret = 0;
 
 out:
-        gf_log ("", GF_LOG_DEBUG, "Returning %d", ret);
+    gf_msg_debug("glusterd", 0, "Returning %d", ret);
 
-        return ret;
+    return ret;
 }
 
 static int
-_add_brick_name_to_dict (dict_t *dict, char *key, glusterd_brickinfo_t *brick)
+_add_remove_bricks_to_dict(dict_t *dict, glusterd_volinfo_t *volinfo,
+                           char *prefix)
 {
-        int     ret = -1;
-        char    tmp[1024] = {0,};
-        char    *brickname = NULL;
-        xlator_t *this = NULL;
-
-        GF_ASSERT (dict);
-        GF_ASSERT (key);
-        GF_ASSERT (brick);
-
-        this = THIS;
-        GF_ASSERT (this);
-
-        snprintf (tmp, sizeof (tmp), "%s:%s", brick->hostname, brick->path);
-        brickname = gf_strdup (tmp);
-        if (!brickname) {
-                gf_log (this->name, GF_LOG_ERROR, "Failed to dup brick name");
-                goto out;
-        }
-
-        ret = dict_set_dynstr (dict, key, brickname);
+    int ret = -1;
+    int count = 0;
+    int i = 0;
+    char brick_key[16] = {
+        0,
+    };
+    char dict_key[64] = {
+        /* dict_key is small as prefix is up to 32 chars */
+        0,
+    };
+    int keylen;
+    char *brick = NULL;
+    xlator_t *this = NULL;
+
+    GF_ASSERT(dict);
+    GF_ASSERT(volinfo);
+    GF_ASSERT(prefix);
+
+    this = THIS;
+    GF_ASSERT(this);
+
+    ret = dict_get_int32n(volinfo->rebal.dict, "count", SLEN("count"), &count);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+               "Failed to get brick count");
+        goto out;
+    }
+
+    keylen = snprintf(dict_key, sizeof(dict_key), "%s.count", prefix);
+    ret = dict_set_int32n(dict, dict_key, keylen, count);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+               "Failed to set brick count in dict");
+        goto out;
+    }
+
+    for (i = 1; i <= count; i++) {
+        keylen = snprintf(brick_key, sizeof(brick_key), "brick%d", i);
+
+        ret = dict_get_strn(volinfo->rebal.dict, brick_key, keylen, &brick);
         if (ret) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "Failed to add brick name to dict");
-                goto out;
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+                   "Unable to get %s", brick_key);
+            goto out;
         }
-        brickname = NULL;
-out:
-        if (brickname)
-                GF_FREE (brickname);
-        return ret;
-}
 
-static int
-_add_remove_bricks_to_dict (dict_t *dict, glusterd_volinfo_t *volinfo,
-                            char *prefix)
-{
-        int             ret = -1;
-        int             count = 0;
-        int             i = 0;
-        char            brick_key[1024] = {0,};
-        char            dict_key[1024] ={0,};
-        char            *brick = NULL;
-        xlator_t        *this = NULL;
-
-        GF_ASSERT (dict);
-        GF_ASSERT (volinfo);
-        GF_ASSERT (prefix);
-
-        this = THIS;
-        GF_ASSERT (this);
-
-        ret = dict_get_int32 (volinfo->rebal.dict, "count", &count);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "Failed to get brick count");
-                goto out;
+        keylen = snprintf(dict_key, sizeof(dict_key), "%s.%s", prefix,
+                          brick_key);
+        if ((keylen < 0) || (keylen >= sizeof(dict_key))) {
+            ret = -1;
+            goto out;
         }
-
-        snprintf (dict_key, sizeof (dict_key), "%s.count", prefix);
-        ret = dict_set_int32 (dict, dict_key, count);
+        ret = dict_set_strn(dict, dict_key, keylen, brick);
         if (ret) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "Failed to set brick count in dict");
-                goto out;
-        }
-
-        for (i = 1; i <= count; i++) {
-                memset (brick_key, 0, sizeof (brick_key));
-                snprintf (brick_key, sizeof (brick_key), "brick%d", i);
-
-                ret = dict_get_str (volinfo->rebal.dict, brick_key, &brick);
-                if (ret) {
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "Unable to get %s", brick_key);
-                        goto out;
-                }
-
-                memset (dict_key, 0, sizeof (dict_key));
-                snprintf (dict_key, sizeof (dict_key), "%s.%s", prefix,
-                          brick_key);
-                ret = dict_set_str (dict, dict_key, brick);
-                if (ret) {
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "Failed to add brick to dict");
-                        goto out;
-                }
-                brick = NULL;
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+                   "Failed to add brick to dict");
+            goto out;
         }
+        brick = NULL;
+    }
 
 out:
-        return ret;
+    return ret;
 }
 
 /* This adds the respective task-id and all available parameters of a task into
  * a dictionary
  */
 static int
-_add_task_to_dict (dict_t *dict, glusterd_volinfo_t *volinfo, int op, int index)
+_add_task_to_dict(dict_t *dict, glusterd_volinfo_t *volinfo, int op, int index)
 {
-
-        int             ret = -1;
-        char            key[128] = {0,};
-        char            *uuid_str = NULL;
-        int             status = 0;
-        xlator_t        *this = NULL;
-
-        GF_ASSERT (dict);
-        GF_ASSERT (volinfo);
-
-        this = THIS;
-        GF_ASSERT (this);
-
-        switch (op) {
+    int ret = -1;
+    char key[32] = {
+        0,
+    };
+    int keylen;
+    char *uuid_str = NULL;
+    int status = 0;
+    xlator_t *this = NULL;
+
+    GF_ASSERT(dict);
+    GF_ASSERT(volinfo);
+
+    this = THIS;
+    GF_ASSERT(this);
+
+    switch (op) {
         case GD_OP_REMOVE_BRICK:
-                snprintf (key, sizeof (key), "task%d", index);
-                ret = _add_remove_bricks_to_dict (dict, volinfo, key);
-                if (ret) {
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "Failed to add remove bricks to dict");
-                        goto out;
-                }
+            snprintf(key, sizeof(key), "task%d", index);
+            ret = _add_remove_bricks_to_dict(dict, volinfo, key);
+            if (ret) {
+                gf_msg(this->name, GF_LOG_ERROR, 0,
+                       GD_MSG_ADD_REMOVE_BRICK_FAIL,
+                       "Failed to add remove bricks to dict");
+                goto out;
+            }
         case GD_OP_REBALANCE:
-                uuid_str = gf_strdup (uuid_utoa (volinfo->rebal.rebalance_id));
-                status = volinfo->rebal.defrag_status;
-                break;
-
-        case GD_OP_REPLACE_BRICK:
-                snprintf (key, sizeof (key), "task%d.src-brick", index);
-                ret = _add_brick_name_to_dict (dict, key,
-                                               volinfo->rep_brick.src_brick);
-                if (ret)
-                        goto out;
-                memset (key, 0, sizeof (key));
-
-                snprintf (key, sizeof (key), "task%d.dst-brick", index);
-                ret = _add_brick_name_to_dict (dict, key,
-                                               volinfo->rep_brick.dst_brick);
-                if (ret)
-                        goto out;
-                memset (key, 0, sizeof (key));
-
-                uuid_str = gf_strdup (uuid_utoa (volinfo->rep_brick.rb_id));
-                status = volinfo->rep_brick.rb_status;
-                break;
+            uuid_str = gf_strdup(uuid_utoa(volinfo->rebal.rebalance_id));
+            status = volinfo->rebal.defrag_status;
+            break;
 
         default:
-                ret = -1;
-                gf_log (this->name, GF_LOG_ERROR, "%s operation doesn't have a"
-                        " task_id", gd_op_list[op]);
-                goto out;
-        }
+            ret = -1;
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_NO_TASK_ID,
+                   "%s operation doesn't have a"
+                   " task_id",
+                   gd_op_list[op]);
+            goto out;
+    }
+
+    keylen = snprintf(key, sizeof(key), "task%d.type", index);
+    ret = dict_set_strn(dict, key, keylen, (char *)gd_op_list[op]);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+               "Error setting task type in dict");
+        goto out;
+    }
+
+    keylen = snprintf(key, sizeof(key), "task%d.id", index);
+
+    if (!uuid_str)
+        goto out;
+    ret = dict_set_dynstrn(dict, key, keylen, uuid_str);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+               "Error setting task id in dict");
+        goto out;
+    }
+    uuid_str = NULL;
+
+    keylen = snprintf(key, sizeof(key), "task%d.status", index);
+    ret = dict_set_int32n(dict, key, keylen, status);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+               "Error setting task status in dict");
+        goto out;
+    }
 
-        snprintf (key, sizeof (key), "task%d.type", index);
-        ret = dict_set_str (dict, key, (char *)gd_op_list[op]);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "Error setting task type in dict");
-                goto out;
-        }
+out:
+    if (uuid_str)
+        GF_FREE(uuid_str);
+    return ret;
+}
+
+static int
+glusterd_aggregate_task_status(dict_t *rsp_dict, glusterd_volinfo_t *volinfo)
+{
+    int ret = -1;
+    int tasks = 0;
+    xlator_t *this = NULL;
 
-        memset (key, 0, sizeof (key));
-        snprintf (key, sizeof (key), "task%d.id", index);
+    this = THIS;
+    GF_ASSERT(this);
 
-        if (!uuid_str)
-                goto out;
-        ret = dict_set_dynstr (dict, key, uuid_str);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "Error setting task id in dict");
-                goto out;
-        }
-        uuid_str = NULL;
+    if (!gf_uuid_is_null(volinfo->rebal.rebalance_id)) {
+        ret = _add_task_to_dict(rsp_dict, volinfo, volinfo->rebal.op, tasks);
 
-        memset (key, 0, sizeof (key));
-        snprintf (key, sizeof (key), "task%d.status", index);
-        ret = dict_set_int32 (dict, key, status);
         if (ret) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "Error setting task status in dict");
-                goto out;
-        }
-
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+                   "Failed to add task details to dict");
+            goto out;
+        }
+        tasks++;
+    }
+    ret = dict_set_int32n(rsp_dict, "tasks", SLEN("tasks"), tasks);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+               "Error setting tasks count in dict");
+        goto out;
+    }
 out:
-        if (uuid_str)
-                GF_FREE (uuid_str);
-        return ret;
+    return ret;
 }
 
 static int
-glusterd_aggregate_task_status (dict_t *rsp_dict, glusterd_volinfo_t *volinfo)
+glusterd_op_status_volume(dict_t *dict, char **op_errstr, dict_t *rsp_dict)
 {
-        int        ret   = -1;
-        int        tasks = 0;
-        xlator_t  *this  = NULL;
+    int ret = -1;
+    int node_count = 0;
+    int brick_index = -1;
+    int other_count = 0;
+    int other_index = 0;
+    uint32_t cmd = 0;
+    char *volname = NULL;
+    char *brick = NULL;
+    xlator_t *this = NULL;
+    glusterd_volinfo_t *volinfo = NULL;
+    glusterd_brickinfo_t *brickinfo = NULL;
+    glusterd_conf_t *priv = NULL;
+    dict_t *vol_opts = NULL;
+#ifdef BUILD_GNFS
+    gf_boolean_t nfs_disabled = _gf_false;
+#endif
+    gf_boolean_t shd_enabled = _gf_false;
+    gf_boolean_t origin_glusterd = _gf_false;
+    int snapd_enabled, bitrot_enabled, volume_quota_enabled;
 
-        this = THIS;
-        GF_ASSERT (this);
+    this = THIS;
+    GF_ASSERT(this);
+    priv = this->private;
 
-        if (!gf_uuid_is_null (volinfo->rebal.rebalance_id)) {
-                ret = _add_task_to_dict (rsp_dict, volinfo, volinfo->rebal.op,
-                                         tasks);
-                if (ret) {
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "Failed to add task details to dict");
-                        goto out;
-                }
-                tasks++;
-        }
+    GF_ASSERT(priv);
 
-        if (!gf_uuid_is_null (volinfo->rep_brick.rb_id)) {
-                ret = _add_task_to_dict (rsp_dict, volinfo, GD_OP_REPLACE_BRICK,
-                                         tasks);
-                if (ret) {
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "Failed to add task details to dict");
-                        goto out;
-                }
-                tasks++;
-        }
+    GF_ASSERT(dict);
 
-        ret = dict_set_int32 (rsp_dict, "tasks", tasks);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "Error setting tasks count in dict");
-                goto out;
-        }
-        ret = 0;
+    origin_glusterd = is_origin_glusterd(dict);
 
-out:
-        return ret;
-}
+    ret = dict_get_uint32(dict, "cmd", &cmd);
+    if (ret)
+        goto out;
 
-static int
-glusterd_op_status_volume (dict_t *dict, char **op_errstr,
-                           dict_t *rsp_dict)
-{
-        int                     ret             = -1;
-        int                     node_count      = 0;
-        int                     brick_index     = -1;
-        int                     other_count     = 0;
-        int                     other_index     = 0;
-        uint32_t                cmd             = 0;
-        char                   *volname         = NULL;
-        char                   *brick           = NULL;
-        xlator_t               *this            = NULL;
-        glusterd_volinfo_t     *volinfo         = NULL;
-        glusterd_brickinfo_t   *brickinfo       = NULL;
-        glusterd_conf_t        *priv            = NULL;
-        dict_t                 *vol_opts        = NULL;
-        gf_boolean_t            nfs_disabled    = _gf_false;
-        gf_boolean_t            shd_enabled     = _gf_true;
-        gf_boolean_t            origin_glusterd = _gf_false;
-
-        this = THIS;
-        GF_ASSERT (this);
-        priv = this->private;
-
-        GF_ASSERT (priv);
-
-        GF_ASSERT (dict);
-
-        origin_glusterd = is_origin_glusterd (dict);
-
-        ret = dict_get_uint32 (dict, "cmd", &cmd);
+    if (origin_glusterd) {
+        ret = 0;
+        if ((cmd & GF_CLI_STATUS_ALL)) {
+            ret = glusterd_get_all_volnames(rsp_dict);
+            if (ret)
+                gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_VOLNAMES_GET_FAIL,
+                       "failed to get all volume "
+                       "names for status");
+        }
+    }
+
+    ret = dict_set_uint32(rsp_dict, "cmd", cmd);
+    if (ret)
+        goto out;
+
+    if (cmd & GF_CLI_STATUS_ALL)
+        goto out;
+
+    ret = dict_get_strn(dict, "volname", SLEN("volname"), &volname);
+    if (ret)
+        goto out;
+
+    ret = glusterd_volinfo_find(volname, &volinfo);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_VOL_NOT_FOUND,
+               "Volume with name: %s "
+               "does not exist",
+               volname);
+        goto out;
+    }
+    vol_opts = volinfo->dict;
+
+    if ((cmd & GF_CLI_STATUS_QUOTAD) != 0) {
+        ret = glusterd_add_node_to_dict(priv->quotad_svc.name, rsp_dict, 0,
+                                        vol_opts);
         if (ret)
-                goto out;
-
-        if (origin_glusterd) {
-                ret = 0;
-                if ((cmd & GF_CLI_STATUS_ALL)) {
-                        ret = glusterd_get_all_volnames (rsp_dict);
-                        if (ret)
-                                gf_log (this->name, GF_LOG_ERROR,
-                                        "failed to get all volume "
-                                        "names for status");
-                }
-        }
+            goto out;
+        other_count++;
+        node_count++;
+#ifdef BUILD_GNFS
+    } else if ((cmd & GF_CLI_STATUS_NFS) != 0) {
+        ret = glusterd_add_node_to_dict(priv->nfs_svc.name, rsp_dict, 0,
+                                        vol_opts);
+        if (ret)
+            goto out;
+        other_count++;
+        node_count++;
+#endif
+    } else if ((cmd & GF_CLI_STATUS_BITD) != 0) {
+        ret = glusterd_add_node_to_dict(priv->bitd_svc.name, rsp_dict, 0,
+                                        vol_opts);
+        if (ret)
+            goto out;
+        other_count++;
+        node_count++;
+    } else if ((cmd & GF_CLI_STATUS_SCRUB) != 0) {
+        ret = glusterd_add_node_to_dict(priv->scrub_svc.name, rsp_dict, 0,
+                                        vol_opts);
+        if (ret)
+            goto out;
+        other_count++;
+        node_count++;
+    } else if ((cmd & GF_CLI_STATUS_SNAPD) != 0) {
+        ret = glusterd_add_snapd_to_dict(volinfo, rsp_dict, other_index);
+        if (ret)
+            goto out;
+        other_count++;
+        node_count++;
+    } else if ((cmd & GF_CLI_STATUS_SHD) != 0) {
+        ret = glusterd_add_shd_to_dict(volinfo, rsp_dict, other_index);
+        if (ret)
+            goto out;
+        other_count++;
+        node_count++;
+    } else if ((cmd & GF_CLI_STATUS_BRICK) != 0) {
+        ret = dict_get_strn(dict, "brick", SLEN("brick"), &brick);
+        if (ret)
+            goto out;
 
-        ret = dict_set_uint32 (rsp_dict, "cmd", cmd);
+        ret = glusterd_volume_brickinfo_get_by_brick(brick, volinfo, &brickinfo,
+                                                     _gf_false);
         if (ret)
-                goto out;
+            goto out;
+
+        if (gf_uuid_compare(brickinfo->uuid, MY_UUID))
+            goto out;
+
+        glusterd_add_brick_to_dict(volinfo, brickinfo, rsp_dict, ++brick_index);
+        if (cmd & GF_CLI_STATUS_DETAIL)
+            glusterd_add_brick_detail_to_dict(volinfo, brickinfo, rsp_dict,
+                                              brick_index);
+        node_count++;
+
+    } else if ((cmd & GF_CLI_STATUS_TASKS) != 0) {
+        ret = glusterd_aggregate_task_status(rsp_dict, volinfo);
+        goto out;
+
+    } else {
+        snapd_enabled = glusterd_is_snapd_enabled(volinfo);
+        shd_enabled = gd_is_self_heal_enabled(volinfo, vol_opts);
+#ifdef BUILD_GNFS
+        nfs_disabled = dict_get_str_boolean(vol_opts, NFS_DISABLE_MAP_KEY,
+                                            _gf_false);
+#endif
+        volume_quota_enabled = glusterd_is_volume_quota_enabled(volinfo);
+        bitrot_enabled = glusterd_is_bitrot_enabled(volinfo);
 
-        if (cmd & GF_CLI_STATUS_ALL)
-                goto out;
+        cds_list_for_each_entry(brickinfo, &volinfo->bricks, brick_list)
+        {
+            brick_index++;
+            if (gf_uuid_compare(brickinfo->uuid, MY_UUID))
+                continue;
 
-        ret = dict_get_str (dict, "volname", &volname);
-        if (ret)
-                goto out;
+            glusterd_add_brick_to_dict(volinfo, brickinfo, rsp_dict,
+                                       brick_index);
 
-        ret = glusterd_volinfo_find (volname, &volinfo);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "Volume with name: %s "
-                        "does not exist", volname);
-                goto out;
+            if (cmd & GF_CLI_STATUS_DETAIL) {
+                glusterd_add_brick_detail_to_dict(volinfo, brickinfo, rsp_dict,
+                                                  brick_index);
+            }
+            node_count++;
         }
-        vol_opts = volinfo->dict;
 
-        if ((cmd & GF_CLI_STATUS_NFS) != 0) {
-                ret = glusterd_add_node_to_dict (priv->nfs_svc.name, rsp_dict,
-                                                 0, vol_opts);
+        if ((cmd & GF_CLI_STATUS_MASK) == GF_CLI_STATUS_NONE) {
+            other_index = brick_index + 1;
+            if (snapd_enabled) {
+                ret = glusterd_add_snapd_to_dict(volinfo, rsp_dict,
+                                                 other_index);
                 if (ret)
-                        goto out;
+                    goto out;
                 other_count++;
+                other_index++;
                 node_count++;
+            }
 
-        } else if ((cmd & GF_CLI_STATUS_SHD) != 0) {
-                ret = glusterd_add_node_to_dict (priv->shd_svc.name, rsp_dict,
-                                                 0, vol_opts);
-                if (ret)
+            if (glusterd_is_shd_compatible_volume(volinfo)) {
+                if (shd_enabled) {
+                    ret = glusterd_add_shd_to_dict(volinfo, rsp_dict,
+                                                   other_index);
+                    if (ret)
                         goto out;
+                    other_count++;
+                    other_index++;
+                    node_count++;
+                }
+            }
+#ifdef BUILD_GNFS
+            if (!nfs_disabled) {
+                ret = glusterd_add_node_to_dict(priv->nfs_svc.name, rsp_dict,
+                                                other_index, vol_opts);
+                if (ret)
+                    goto out;
+                other_index++;
                 other_count++;
                 node_count++;
-
-        } else if ((cmd & GF_CLI_STATUS_QUOTAD) != 0) {
-                ret = glusterd_add_node_to_dict (priv->quotad_svc.name,
-                                                 rsp_dict, 0, vol_opts);
+            }
+#endif
+            if (volume_quota_enabled) {
+                ret = glusterd_add_node_to_dict(priv->quotad_svc.name, rsp_dict,
+                                                other_index, vol_opts);
                 if (ret)
-                        goto out;
+                    goto out;
                 other_count++;
                 node_count++;
-        } else if ((cmd & GF_CLI_STATUS_SNAPD) != 0) {
-                ret = glusterd_add_node_to_dict ("snapd", rsp_dict, 0,
-                                                 vol_opts);
+                other_index++;
+            }
+
+            if (bitrot_enabled) {
+                ret = glusterd_add_node_to_dict(priv->bitd_svc.name, rsp_dict,
+                                                other_index, vol_opts);
                 if (ret)
-                        goto out;
+                    goto out;
                 other_count++;
                 node_count++;
-        } else if ((cmd & GF_CLI_STATUS_BRICK) != 0) {
-                ret = dict_get_str (dict, "brick", &brick);
-                if (ret)
-                        goto out;
-
-                ret = glusterd_volume_brickinfo_get_by_brick (brick,
-                                                              volinfo,
-                                                              &brickinfo);
+                other_index++;
+                /* For handling scrub status. Scrub daemon will be
+                 * running automatically when bitrot is enable */
+                ret = glusterd_add_node_to_dict(priv->scrub_svc.name, rsp_dict,
+                                                other_index, vol_opts);
                 if (ret)
-                        goto out;
-
-                if (gf_uuid_compare (brickinfo->uuid, MY_UUID))
-                        goto out;
-
-                glusterd_add_brick_to_dict (volinfo, brickinfo, rsp_dict,
-                                            ++brick_index);
-                if (cmd & GF_CLI_STATUS_DETAIL)
-                        glusterd_add_brick_detail_to_dict (volinfo, brickinfo,
-                                                           rsp_dict,
-                                                           brick_index);
+                    goto out;
+                other_count++;
                 node_count++;
-
-        } else if ((cmd & GF_CLI_STATUS_TASKS) != 0) {
-                ret = glusterd_aggregate_task_status (rsp_dict, volinfo);
-                goto out;
-
-        } else {
-                cds_list_for_each_entry (brickinfo, &volinfo->bricks,
-                                         brick_list) {
-                        brick_index++;
-                        if (gf_uuid_compare (brickinfo->uuid, MY_UUID))
-                                continue;
-
-                        glusterd_add_brick_to_dict (volinfo, brickinfo,
-                                                    rsp_dict, brick_index);
-
-                        if (cmd & GF_CLI_STATUS_DETAIL) {
-                                glusterd_add_brick_detail_to_dict (volinfo,
-                                                                   brickinfo,
-                                                                   rsp_dict,
-                                                                   brick_index);
-                        }
-                        node_count++;
-                }
-
-                if ((cmd & GF_CLI_STATUS_MASK) == GF_CLI_STATUS_NONE) {
-                        other_index = brick_index + 1;
-                        if (glusterd_is_snapd_enabled (volinfo)) {
-                                ret = glusterd_add_snapd_to_dict (volinfo,
-                                                                  rsp_dict,
-                                                                  other_index);
-                                if (ret)
-                                        goto out;
-                                other_count++;
-                                other_index++;
-                                node_count++;
-                        }
-
-                        nfs_disabled = dict_get_str_boolean (vol_opts,
-                                                             "nfs.disable",
-                                                             _gf_false);
-                        if (!nfs_disabled) {
-                                ret = glusterd_add_node_to_dict
-                                                           (priv->nfs_svc.name,
-                                                            rsp_dict,
-                                                            other_index,
-                                                            vol_opts);
-                                if (ret)
-                                        goto out;
-                                other_index++;
-                                other_count++;
-                                node_count++;
-                        }
-
-                        shd_enabled = dict_get_str_boolean
-                                        (vol_opts, "cluster.self-heal-daemon",
-                                         _gf_true);
-                        if (glusterd_is_volume_replicate (volinfo)
-                            && shd_enabled) {
-                                ret = glusterd_add_node_to_dict
-                                        (priv->shd_svc.name, rsp_dict,
-                                         other_index, vol_opts);
-                                if (ret)
-                                        goto out;
-                                other_count++;
-                                node_count++;
-                                other_index++;
-                        }
-                        if (glusterd_is_volume_quota_enabled (volinfo)) {
-                                ret = glusterd_add_node_to_dict
-                                                        (priv->quotad_svc.name,
-                                                         rsp_dict,
-                                                         other_index,
-                                                         vol_opts);
-                                if (ret)
-                                        goto out;
-                                other_count++;
-                                node_count++;
-                        }
-                }
-        }
-
-        ret = dict_set_int32 (rsp_dict, "brick-index-max", brick_index);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "Error setting brick-index-max to dict");
-                goto out;
-        }
-        ret = dict_set_int32 (rsp_dict, "other-count", other_count);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "Error setting other-count to dict");
-                goto out;
-        }
-        ret = dict_set_int32 (rsp_dict, "count", node_count);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "Error setting node count to dict");
-                goto out;
-        }
-
-        /* Active tasks */
-        /* Tasks are added only for normal volume status request for either a
-         * single volume or all volumes
-         */
-        if (!glusterd_status_has_tasks (cmd))
-                goto out;
-
-        ret = glusterd_aggregate_task_status (rsp_dict, volinfo);
-        if (ret)
-                goto out;
-        ret = 0;
+            }
+        }
+    }
+
+    ret = dict_set_int32n(rsp_dict, "type", SLEN("type"), volinfo->type);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_SET_FAILED,
+                "Key=type", NULL);
+        goto out;
+    }
+
+    ret = dict_set_int32n(rsp_dict, "brick-index-max", SLEN("brick-index-max"),
+                          brick_index);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+                "Key=brick-index-max", NULL);
+        goto out;
+    }
+    ret = dict_set_int32n(rsp_dict, "other-count", SLEN("other-count"),
+                          other_count);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+                "Key=other-count", NULL);
+        goto out;
+    }
+    ret = dict_set_int32n(rsp_dict, "count", SLEN("count"), node_count);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+                "Key=count", NULL);
+        goto out;
+    }
+
+    /* Active tasks */
+    /* Tasks are added only for normal volume status request for either a
+     * single volume or all volumes
+     */
+    if (!glusterd_status_has_tasks(cmd))
+        goto out;
+
+    ret = glusterd_aggregate_task_status(rsp_dict, volinfo);
+    if (ret)
+        goto out;
+    ret = 0;
 
 out:
-        gf_log (this->name, GF_LOG_DEBUG, "Returning %d", ret);
+    gf_msg_debug(this->name, 0, "Returning %d", ret);
 
-        return ret;
+    return ret;
 }
 
 static int
-glusterd_op_ac_none (glusterd_op_sm_event_t *event, void *ctx)
+glusterd_op_ac_none(glusterd_op_sm_event_t *event, void *ctx)
 {
-        int ret = 0;
+    int ret = 0;
 
-        gf_log (THIS->name, GF_LOG_DEBUG, "Returning with %d", ret);
+    gf_msg_debug(THIS->name, 0, "Returning with %d", ret);
 
-        return ret;
+    return ret;
 }
 
 static int
-glusterd_op_sm_locking_failed (uuid_t *txn_id)
+glusterd_op_sm_locking_failed(uuid_t *txn_id)
 {
-        int ret = -1;
+    int ret = -1;
 
-        opinfo.op_ret = -1;
-        opinfo.op_errstr = gf_strdup ("locking failed for one of the peer.");
+    opinfo.op_ret = -1;
+    opinfo.op_errstr = gf_strdup("locking failed for one of the peer.");
 
-        /* Inject a reject event such that unlocking gets triggered right away*/
-        ret = glusterd_op_sm_inject_event (GD_OP_EVENT_RCVD_RJT, txn_id, NULL);
+    ret = glusterd_set_txn_opinfo(txn_id, &opinfo);
+    if (ret)
+        gf_msg(THIS->name, GF_LOG_ERROR, 0, GD_MSG_TRANS_OPINFO_SET_FAIL,
+               "Unable to set "
+               "transaction's opinfo");
+    /* Inject a reject event such that unlocking gets triggered right away*/
+    ret = glusterd_op_sm_inject_event(GD_OP_EVENT_RCVD_RJT, txn_id, NULL);
 
-        return ret;
+    return ret;
 }
 
 static int
-glusterd_op_ac_send_lock (glusterd_op_sm_event_t *event, void *ctx)
+glusterd_op_ac_send_lock(glusterd_op_sm_event_t *event, void *ctx)
 {
-        int                   ret      = 0;
-        rpc_clnt_procedure_t *proc     = NULL;
-        glusterd_conf_t      *priv     = NULL;
-        xlator_t             *this     = NULL;
-        glusterd_peerinfo_t  *peerinfo = NULL;
-        uint32_t             pending_count = 0;
-        dict_t               *dict     = NULL;
-
-        this = THIS;
-        priv = this->private;
-        GF_ASSERT (priv);
-
-        list_for_each_local_xaction_peers (peerinfo,
-                                           opinfo.local_xaction_peers) {
-                GF_ASSERT (peerinfo);
-
-                if (!peerinfo->connected || !peerinfo->mgmt)
-                        continue;
-                if ((peerinfo->state.state != GD_FRIEND_STATE_BEFRIENDED) &&
-                    (glusterd_op_get_op() != GD_OP_SYNC_VOLUME))
-                        continue;
+    int ret = 0;
+    rpc_clnt_procedure_t *proc = NULL;
+    glusterd_conf_t *priv = NULL;
+    xlator_t *this = NULL;
+    glusterd_peerinfo_t *peerinfo = NULL;
+    uint32_t pending_count = 0;
+    dict_t *dict = NULL;
+
+    this = THIS;
+    priv = this->private;
+    GF_ASSERT(priv);
+
+    RCU_READ_LOCK;
+    cds_list_for_each_entry_rcu(peerinfo, &priv->peers, uuid_list)
+    {
+        /* Only send requests to peers who were available before the
+         * transaction started
+         */
+        if (peerinfo->generation > opinfo.txn_generation)
+            continue;
 
-                /* Based on the op_version, acquire a cluster or mgmt_v3 lock */
-                if (priv->op_version < GD_OP_VERSION_3_6_0) {
-                        proc = &peerinfo->mgmt->proctable
-                                          [GLUSTERD_MGMT_CLUSTER_LOCK];
-                        if (proc->fn) {
-                                ret = proc->fn (NULL, this, peerinfo);
-                                if (ret) {
-                                        gf_log (this->name, GF_LOG_WARNING,
-                                                "Failed to send lock request "
-                                                "for operation 'Volume %s' to "
-                                                "peer %s",
-                                                gd_op_list[opinfo.op],
-                                                peerinfo->hostname);
-                                        goto out;
-                                }
-                                /* Mark the peer as locked*/
-                                peerinfo->locked = _gf_true;
-                                pending_count++;
-                        }
-                } else {
-                        dict = glusterd_op_get_ctx ();
-                        dict_ref (dict);
-
-                        proc = &peerinfo->mgmt_v3->proctable
-                                          [GLUSTERD_MGMT_V3_LOCK];
-                        if (proc->fn) {
-                                ret = dict_set_static_ptr (dict, "peerinfo",
-                                                           peerinfo);
-                                if (ret) {
-                                        gf_log (this->name, GF_LOG_ERROR,
-                                                "failed to set peerinfo");
-                                        dict_unref (dict);
-                                        goto out;
-                                }
-
-                                ret = proc->fn (NULL, this, dict);
-                                if (ret) {
-                                        gf_log (this->name, GF_LOG_WARNING,
-                                                "Failed to send mgmt_v3 lock "
-                                                "request for operation "
-                                                "'Volume %s' to peer %s",
-                                                gd_op_list[opinfo.op],
-                                                peerinfo->hostname);
-                                        dict_unref (dict);
-                                        goto out;
-                                }
-                                /* Mark the peer as locked*/
-                                peerinfo->locked = _gf_true;
-                                pending_count++;
-                        }
+        if (!peerinfo->connected || !peerinfo->mgmt)
+            continue;
+        if ((peerinfo->state.state != GD_FRIEND_STATE_BEFRIENDED) &&
+            (glusterd_op_get_op() != GD_OP_SYNC_VOLUME))
+            continue;
+
+        /* Based on the op_version, acquire a cluster or mgmt_v3 lock */
+        if (priv->op_version < GD_OP_VERSION_3_6_0) {
+            proc = &peerinfo->mgmt->proctable[GLUSTERD_MGMT_CLUSTER_LOCK];
+            if (proc->fn) {
+                ret = proc->fn(NULL, this, peerinfo);
+                if (ret) {
+                    RCU_READ_UNLOCK;
+                    gf_msg(this->name, GF_LOG_WARNING, 0,
+                           GD_MSG_LOCK_REQ_SEND_FAIL,
+                           "Failed to send lock request "
+                           "for operation 'Volume %s' to "
+                           "peer %s",
+                           gd_op_list[opinfo.op], peerinfo->hostname);
+                    goto out;
+                }
+                /* Mark the peer as locked*/
+                peerinfo->locked = _gf_true;
+                pending_count++;
+            }
+        } else {
+            dict = glusterd_op_get_ctx();
+            dict_ref(dict);
+
+            proc = &peerinfo->mgmt_v3->proctable[GLUSTERD_MGMT_V3_LOCK];
+            if (proc->fn) {
+                ret = dict_set_static_ptr(dict, "peerinfo", peerinfo);
+                if (ret) {
+                    RCU_READ_UNLOCK;
+                    gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+                           "failed to set peerinfo");
+                    dict_unref(dict);
+                    goto out;
                 }
-        }
 
-        opinfo.pending_count = pending_count;
-        if (!opinfo.pending_count)
-                ret = glusterd_op_sm_inject_all_acc (&event->txn_id);
+                ret = proc->fn(NULL, this, dict);
+                if (ret) {
+                    RCU_READ_UNLOCK;
+                    gf_msg(this->name, GF_LOG_WARNING, 0,
+                           GD_MSG_MGMTV3_LOCK_REQ_SEND_FAIL,
+                           "Failed to send mgmt_v3 lock "
+                           "request for operation "
+                           "'Volume %s' to peer %s",
+                           gd_op_list[opinfo.op], peerinfo->hostname);
+                    dict_unref(dict);
+                    goto out;
+                }
+                /* Mark the peer as locked*/
+                peerinfo->locked = _gf_true;
+                pending_count++;
+            }
+        }
+    }
+    RCU_READ_UNLOCK;
+
+    opinfo.pending_count = pending_count;
+
+    ret = glusterd_set_txn_opinfo(&event->txn_id, &opinfo);
+    if (ret)
+        gf_msg(THIS->name, GF_LOG_ERROR, 0, GD_MSG_TRANS_OPINFO_SET_FAIL,
+               "Unable to set "
+               "transaction's opinfo");
+
+    if (!opinfo.pending_count)
+        ret = glusterd_op_sm_inject_all_acc(&event->txn_id);
 
 out:
-        if (ret)
-                ret = glusterd_op_sm_locking_failed (&event->txn_id);
+    if (ret)
+        ret = glusterd_op_sm_locking_failed(&event->txn_id);
 
-        gf_log (this->name, GF_LOG_DEBUG, "Returning with %d", ret);
-        return ret;
+    gf_msg_debug(this->name, 0, "Returning with %d", ret);
+    return ret;
 }
 
 static int
-glusterd_op_ac_send_unlock (glusterd_op_sm_event_t *event, void *ctx)
+glusterd_op_ac_send_unlock(glusterd_op_sm_event_t *event, void *ctx)
 {
-        int                   ret           = 0;
-        rpc_clnt_procedure_t *proc          = NULL;
-        glusterd_conf_t      *priv          = NULL;
-        xlator_t             *this          = NULL;
-        glusterd_peerinfo_t  *peerinfo      = NULL;
-        uint32_t              pending_count = 0;
-        dict_t               *dict          = NULL;
-
-        this = THIS;
-        priv = this->private;
-        GF_ASSERT (priv);
-
-        list_for_each_local_xaction_peers (peerinfo,
-                                           opinfo.local_xaction_peers) {
-                GF_ASSERT (peerinfo);
-
-                if (!peerinfo->connected || !peerinfo->mgmt ||
-                    !peerinfo->locked)
-                        continue;
-                if ((peerinfo->state.state != GD_FRIEND_STATE_BEFRIENDED) &&
-                    (glusterd_op_get_op() != GD_OP_SYNC_VOLUME))
-                        continue;
-                /* Based on the op_version,
-                 * release the cluster or mgmt_v3 lock */
-                if (priv->op_version < GD_OP_VERSION_3_6_0) {
-                        proc = &peerinfo->mgmt->proctable
-                                          [GLUSTERD_MGMT_CLUSTER_UNLOCK];
-                        if (proc->fn) {
-                                ret = proc->fn (NULL, this, peerinfo);
-                                if (ret) {
-                                        opinfo.op_errstr = gf_strdup
-                                               ("Unlocking failed for one of "
-                                                "the peer.");
-                                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                                GD_MSG_CLUSTER_UNLOCK_FAILED,
-                                                "Unlocking failed for operation"
-                                                " volume %s on peer %s",
-                                                gd_op_list[opinfo.op],
-                                                peerinfo->hostname);
-                                        continue;
-                                }
-                                pending_count++;
-                                peerinfo->locked = _gf_false;
-                        }
-                } else {
-                        dict = glusterd_op_get_ctx ();
-                        dict_ref (dict);
-
-                        proc = &peerinfo->mgmt_v3->proctable
-                                          [GLUSTERD_MGMT_V3_UNLOCK];
-                        if (proc->fn) {
-                                ret = dict_set_static_ptr (dict, "peerinfo",
-                                                           peerinfo);
-                                if (ret) {
-                                        opinfo.op_errstr = gf_strdup
-                                          ("Unlocking failed for one of the "
-                                           "peer.");
-                                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                                GD_MSG_CLUSTER_UNLOCK_FAILED,
-                                                "Unlocking failed for operation"
-                                                " volume %s on peer %s",
-                                                gd_op_list[opinfo.op],
-                                                peerinfo->hostname);
-                                        dict_unref (dict);
-                                        continue;
-                                }
-
-                                ret = proc->fn (NULL, this, dict);
-                                if (ret) {
-                                        opinfo.op_errstr = gf_strdup
-                                          ("Unlocking failed for one of the "
-                                           "peer.");
-                                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                                GD_MSG_CLUSTER_UNLOCK_FAILED,
-                                                "Unlocking failed for operation"
-                                                " volume %s on peer %s",
-                                                gd_op_list[opinfo.op],
-                                                peerinfo->hostname);
-                                        dict_unref (dict);
-                                        continue;
-                                }
-                                pending_count++;
-                                peerinfo->locked = _gf_false;
-                        }
-                }
-        }
-
-        opinfo.pending_count = pending_count;
-        if (!opinfo.pending_count)
-                ret = glusterd_op_sm_inject_all_acc (&event->txn_id);
+    int ret = 0;
+    rpc_clnt_procedure_t *proc = NULL;
+    glusterd_conf_t *priv = NULL;
+    xlator_t *this = NULL;
+    glusterd_peerinfo_t *peerinfo = NULL;
+    uint32_t pending_count = 0;
+    dict_t *dict = NULL;
+
+    this = THIS;
+    priv = this->private;
+    GF_ASSERT(priv);
+
+    RCU_READ_LOCK;
+    cds_list_for_each_entry_rcu(peerinfo, &priv->peers, uuid_list)
+    {
+        /* Only send requests to peers who were available before the
+         * transaction started
+         */
+        if (peerinfo->generation > opinfo.txn_generation)
+            continue;
+
+        if (!peerinfo->connected || !peerinfo->mgmt || !peerinfo->locked)
+            continue;
+        if ((peerinfo->state.state != GD_FRIEND_STATE_BEFRIENDED) &&
+            (glusterd_op_get_op() != GD_OP_SYNC_VOLUME))
+            continue;
+        /* Based on the op_version,
+         * release the cluster or mgmt_v3 lock */
+        if (priv->op_version < GD_OP_VERSION_3_6_0) {
+            proc = &peerinfo->mgmt->proctable[GLUSTERD_MGMT_CLUSTER_UNLOCK];
+            if (proc->fn) {
+                ret = proc->fn(NULL, this, peerinfo);
+                if (ret) {
+                    opinfo.op_errstr = gf_strdup(
+                        "Unlocking failed for one of "
+                        "the peer.");
+                    gf_msg(this->name, GF_LOG_ERROR, 0,
+                           GD_MSG_CLUSTER_UNLOCK_FAILED,
+                           "Unlocking failed for operation"
+                           " volume %s on peer %s",
+                           gd_op_list[opinfo.op], peerinfo->hostname);
+                    continue;
+                }
+                pending_count++;
+                peerinfo->locked = _gf_false;
+            }
+        } else {
+            dict = glusterd_op_get_ctx();
+            dict_ref(dict);
 
-        gf_log (this->name, GF_LOG_DEBUG, "Returning with %d", ret);
-        return ret;
+            proc = &peerinfo->mgmt_v3->proctable[GLUSTERD_MGMT_V3_UNLOCK];
+            if (proc->fn) {
+                ret = dict_set_static_ptr(dict, "peerinfo", peerinfo);
+                if (ret) {
+                    opinfo.op_errstr = gf_strdup(
+                        "Unlocking failed for one of the "
+                        "peer.");
+                    gf_msg(this->name, GF_LOG_ERROR, 0,
+                           GD_MSG_CLUSTER_UNLOCK_FAILED,
+                           "Unlocking failed for operation"
+                           " volume %s on peer %s",
+                           gd_op_list[opinfo.op], peerinfo->hostname);
+                    dict_unref(dict);
+                    continue;
+                }
+
+                ret = proc->fn(NULL, this, dict);
+                if (ret) {
+                    opinfo.op_errstr = gf_strdup(
+                        "Unlocking failed for one of the "
+                        "peer.");
+                    gf_msg(this->name, GF_LOG_ERROR, 0,
+                           GD_MSG_CLUSTER_UNLOCK_FAILED,
+                           "Unlocking failed for operation"
+                           " volume %s on peer %s",
+                           gd_op_list[opinfo.op], peerinfo->hostname);
+                    dict_unref(dict);
+                    continue;
+                }
+                pending_count++;
+                peerinfo->locked = _gf_false;
+            }
+        }
+    }
+    RCU_READ_UNLOCK;
+
+    opinfo.pending_count = pending_count;
+
+    ret = glusterd_set_txn_opinfo(&event->txn_id, &opinfo);
+    if (ret)
+        gf_msg(THIS->name, GF_LOG_ERROR, 0, GD_MSG_TRANS_OPINFO_SET_FAIL,
+               "Unable to set "
+               "transaction's opinfo");
+
+    if (!opinfo.pending_count)
+        ret = glusterd_op_sm_inject_all_acc(&event->txn_id);
+
+    gf_msg_debug(this->name, 0, "Returning with %d", ret);
+    return ret;
 }
 
 static int
-glusterd_op_ac_ack_drain (glusterd_op_sm_event_t *event, void *ctx)
+glusterd_op_ac_ack_drain(glusterd_op_sm_event_t *event, void *ctx)
 {
-        int ret = 0;
+    int ret = 0;
 
-        if (opinfo.pending_count > 0)
-                opinfo.pending_count--;
+    if (opinfo.pending_count > 0)
+        opinfo.pending_count--;
 
-        if (!opinfo.pending_count)
-                ret = glusterd_op_sm_inject_event (GD_OP_EVENT_ALL_ACK,
-                                                   &event->txn_id, NULL);
+    ret = glusterd_set_txn_opinfo(&event->txn_id, &opinfo);
+    if (ret)
+        gf_msg(THIS->name, GF_LOG_ERROR, 0, GD_MSG_TRANS_OPINFO_SET_FAIL,
+               "Unable to set "
+               "transaction's opinfo");
 
-        gf_log (THIS->name, GF_LOG_DEBUG, "Returning with %d", ret);
+    if (!opinfo.pending_count)
+        ret = glusterd_op_sm_inject_event(GD_OP_EVENT_ALL_ACK, &event->txn_id,
+                                          NULL);
 
-        return ret;
+    gf_msg_debug(THIS->name, 0, "Returning with %d", ret);
+
+    return ret;
 }
 
 static int
-glusterd_op_ac_send_unlock_drain (glusterd_op_sm_event_t *event, void *ctx)
+glusterd_op_ac_send_unlock_drain(glusterd_op_sm_event_t *event, void *ctx)
 {
-        return glusterd_op_ac_ack_drain (event, ctx);
+    return glusterd_op_ac_ack_drain(event, ctx);
 }
 
 static int
-glusterd_op_ac_lock (glusterd_op_sm_event_t *event, void *ctx)
+glusterd_op_ac_lock(glusterd_op_sm_event_t *event, void *ctx)
 {
-        int32_t                         ret             = 0;
-        char                           *volname         = NULL;
-        glusterd_op_lock_ctx_t         *lock_ctx        = NULL;
-        glusterd_conf_t                *priv            = NULL;
-        xlator_t                       *this            = NULL;
-
-        GF_ASSERT (event);
-        GF_ASSERT (ctx);
-
-        this = THIS;
-        priv = this->private;
-
-        lock_ctx = (glusterd_op_lock_ctx_t *)ctx;
-
-        /* If the req came from a node running on older op_version
-         * the dict won't be present. Based on it acquiring a cluster
-         * or mgmt_v3 lock */
-        if (lock_ctx->dict == NULL) {
-                ret = glusterd_lock (lock_ctx->uuid);
-                glusterd_op_lock_send_resp (lock_ctx->req, ret);
-        } else {
-                ret = dict_get_str (lock_ctx->dict, "volname", &volname);
-                if (ret)
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "Unable to acquire volname");
-                else {
-                        ret = glusterd_mgmt_v3_lock (volname, lock_ctx->uuid,
-                                                     "vol");
-                        if (ret)
-                                gf_log (this->name, GF_LOG_ERROR,
-                                        "Unable to acquire lock for %s",
-                                        volname);
-                }
-
-                glusterd_op_mgmt_v3_lock_send_resp (lock_ctx->req,
-                                                   &event->txn_id, ret);
-
-                dict_unref (lock_ctx->dict);
+    int32_t ret = 0;
+    char *volname = NULL;
+    char *globalname = NULL;
+    glusterd_op_lock_ctx_t *lock_ctx = NULL;
+    xlator_t *this = NULL;
+    uint32_t op_errno = 0;
+    glusterd_conf_t *conf = NULL;
+    uint32_t timeout = 0;
+
+    GF_ASSERT(event);
+    GF_ASSERT(ctx);
+
+    this = THIS;
+    GF_ASSERT(this);
+    conf = this->private;
+    GF_ASSERT(conf);
+
+    lock_ctx = (glusterd_op_lock_ctx_t *)ctx;
+
+    /* If the req came from a node running on older op_version
+     * the dict won't be present. Based on it acquiring a cluster
+     * or mgmt_v3 lock */
+    if (lock_ctx->dict == NULL) {
+        ret = glusterd_lock(lock_ctx->uuid);
+        glusterd_op_lock_send_resp(lock_ctx->req, ret);
+    } else {
+        /* Cli will add timeout key to dict if the default timeout is
+         * other than 2 minutes. Here we use this value to check whether
+         * mgmt_v3_lock_timeout should be set to default value or we
+         * need to change the value according to timeout value
+         * i.e, timeout + 120 seconds. */
+        ret = dict_get_uint32(lock_ctx->dict, "timeout", &timeout);
+        if (!ret)
+            conf->mgmt_v3_lock_timeout = timeout + 120;
+
+        ret = dict_get_strn(lock_ctx->dict, "volname", SLEN("volname"),
+                            &volname);
+        if (ret)
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+                   "Unable to acquire volname");
+        else {
+            ret = glusterd_mgmt_v3_lock(volname, lock_ctx->uuid, &op_errno,
+                                        "vol");
+            if (ret)
+                gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_MGMTV3_LOCK_GET_FAIL,
+                       "Unable to acquire lock for %s", volname);
+            goto out;
+        }
+        ret = dict_get_strn(lock_ctx->dict, "globalname", SLEN("globalname"),
+                            &globalname);
+        if (!ret) {
+            ret = glusterd_mgmt_v3_lock(globalname, lock_ctx->uuid, &op_errno,
+                                        "global");
+            if (ret)
+                gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_MGMTV3_LOCK_GET_FAIL,
+                       "Unable to acquire lock for %s", globalname);
         }
+    out:
+        glusterd_op_mgmt_v3_lock_send_resp(lock_ctx->req, &event->txn_id, ret);
 
-        gf_log (THIS->name, GF_LOG_DEBUG, "Lock Returned %d", ret);
-        return ret;
+        dict_unref(lock_ctx->dict);
+    }
+
+    gf_msg_debug(THIS->name, 0, "Lock Returned %d", ret);
+    return ret;
 }
 
 static int
-glusterd_op_ac_unlock (glusterd_op_sm_event_t *event, void *ctx)
+glusterd_op_ac_unlock(glusterd_op_sm_event_t *event, void *ctx)
 {
-        int32_t                         ret             = 0;
-        char                           *volname         = NULL;
-        glusterd_op_lock_ctx_t         *lock_ctx        = NULL;
-        glusterd_conf_t                *priv            = NULL;
-        xlator_t                       *this            = NULL;
-
-
-        GF_ASSERT (event);
-        GF_ASSERT (ctx);
-
-        this = THIS;
-        priv = this->private;
-
-        lock_ctx = (glusterd_op_lock_ctx_t *)ctx;
-
-        /* If the req came from a node running on older op_version
-         * the dict won't be present. Based on it releasing the cluster
-         * or mgmt_v3 lock */
-        if (lock_ctx->dict == NULL) {
-                ret = glusterd_unlock (lock_ctx->uuid);
-                glusterd_op_unlock_send_resp (lock_ctx->req, ret);
-        } else {
-                ret = dict_get_str (lock_ctx->dict, "volname", &volname);
-                if (ret)
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "Unable to acquire volname");
-                else {
-                        ret = glusterd_mgmt_v3_unlock (volname, lock_ctx->uuid,
-                                                       "vol");
-                        if (ret)
-                                gf_log (this->name, GF_LOG_ERROR,
-                                        "Unable to release lock for %s",
-                                        volname);
-                }
-
-                glusterd_op_mgmt_v3_unlock_send_resp (lock_ctx->req,
-                                                     &event->txn_id, ret);
+    int32_t ret = 0;
+    char *volname = NULL;
+    char *globalname = NULL;
+    glusterd_op_lock_ctx_t *lock_ctx = NULL;
+    glusterd_conf_t *priv = NULL;
+    xlator_t *this = NULL;
+
+    GF_ASSERT(event);
+    GF_ASSERT(ctx);
+
+    this = THIS;
+    priv = this->private;
+
+    lock_ctx = (glusterd_op_lock_ctx_t *)ctx;
+
+    /* If the req came from a node running on older op_version
+     * the dict won't be present. Based on it releasing the cluster
+     * or mgmt_v3 lock */
+    if (lock_ctx->dict == NULL) {
+        ret = glusterd_unlock(lock_ctx->uuid);
+        glusterd_op_unlock_send_resp(lock_ctx->req, ret);
+    } else {
+        ret = dict_get_strn(lock_ctx->dict, "volname", SLEN("volname"),
+                            &volname);
+        if (ret)
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+                   "Unable to acquire volname");
+        else {
+            ret = glusterd_mgmt_v3_unlock(volname, lock_ctx->uuid, "vol");
+            if (ret)
+                gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_MGMTV3_UNLOCK_FAIL,
+                       "Unable to release lock for %s", volname);
+            goto out;
+        }
 
-                dict_unref (lock_ctx->dict);
+        ret = dict_get_strn(lock_ctx->dict, "globalname", SLEN("globalname"),
+                            &globalname);
+        if (!ret) {
+            ret = glusterd_mgmt_v3_unlock(globalname, lock_ctx->uuid, "global");
+            if (ret)
+                gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_MGMTV3_UNLOCK_FAIL,
+                       "Unable to release lock for %s", globalname);
         }
+    out:
+        glusterd_op_mgmt_v3_unlock_send_resp(lock_ctx->req, &event->txn_id,
+                                             ret);
 
-        gf_log (this->name, GF_LOG_DEBUG, "Unlock Returned %d", ret);
+        dict_unref(lock_ctx->dict);
+    }
 
-        if (priv->pending_quorum_action)
-                glusterd_do_quorum_action ();
-        return ret;
+    gf_msg_debug(this->name, 0, "Unlock Returned %d", ret);
+
+    if (priv->pending_quorum_action)
+        glusterd_do_quorum_action();
+    return ret;
 }
 
 static int
-glusterd_op_ac_local_unlock (glusterd_op_sm_event_t *event, void *ctx)
+glusterd_op_ac_local_unlock(glusterd_op_sm_event_t *event, void *ctx)
 {
-        int              ret          = 0;
-        uuid_t          *originator   = NULL;
+    int ret = 0;
+    uuid_t *originator = NULL;
 
-        GF_ASSERT (event);
-        GF_ASSERT (ctx);
+    GF_ASSERT(event);
+    GF_ASSERT(ctx);
 
-        originator = (uuid_t *) ctx;
+    originator = (uuid_t *)ctx;
 
-        ret = glusterd_unlock (*originator);
+    ret = glusterd_unlock(*originator);
 
-        gf_log (THIS->name, GF_LOG_DEBUG, "Unlock Returned %d", ret);
+    gf_msg_debug(THIS->name, 0, "Unlock Returned %d", ret);
 
-        return ret;
+    return ret;
 }
 
 static int
-glusterd_op_ac_rcvd_lock_acc (glusterd_op_sm_event_t *event, void *ctx)
+glusterd_op_ac_rcvd_lock_acc(glusterd_op_sm_event_t *event, void *ctx)
 {
-        int                     ret = 0;
+    int ret = 0;
 
-        GF_ASSERT (event);
+    GF_ASSERT(event);
 
-        if (opinfo.pending_count > 0)
-                opinfo.pending_count--;
+    if (opinfo.pending_count > 0)
+        opinfo.pending_count--;
 
-        if (opinfo.pending_count > 0)
-                goto out;
+    ret = glusterd_set_txn_opinfo(&event->txn_id, &opinfo);
+    if (ret)
+        gf_msg(THIS->name, GF_LOG_ERROR, 0, GD_MSG_TRANS_OPINFO_SET_FAIL,
+               "Unable to set "
+               "transaction's opinfo");
 
-        ret = glusterd_op_sm_inject_event (GD_OP_EVENT_ALL_ACC,
-                                           &event->txn_id, NULL);
+    if (opinfo.pending_count > 0)
+        goto out;
 
-        gf_log (THIS->name, GF_LOG_DEBUG, "Returning %d", ret);
+    ret = glusterd_op_sm_inject_event(GD_OP_EVENT_ALL_ACC, &event->txn_id,
+                                      NULL);
+
+    gf_msg_debug(THIS->name, 0, "Returning %d", ret);
 
 out:
-        return ret;
+    return ret;
 }
 
-static int
-glusterd_dict_set_volid (dict_t *dict, char *volname, char **op_errstr)
+int
+glusterd_dict_set_volid(dict_t *dict, char *volname, char **op_errstr)
 {
-        int                     ret = -1;
-        glusterd_volinfo_t      *volinfo = NULL;
-        char                    *volid = NULL;
-        char                    msg[1024] = {0,};
-        xlator_t                *this = NULL;
-
-        this = THIS;
-        GF_ASSERT (this);
-
-        if (!dict || !volname)
-                goto out;
-
-        ret = glusterd_volinfo_find (volname, &volinfo);
-        if (ret) {
-                snprintf (msg, sizeof (msg), FMTSTR_CHECK_VOL_EXISTS, volname);
-                goto out;
-        }
-        volid = gf_strdup (uuid_utoa (volinfo->volume_id));
-        if (!volid) {
-                ret = -1;
-                goto out;
-        }
-        ret = dict_set_dynstr (dict, "vol-id", volid);
-        if (ret) {
-                snprintf (msg, sizeof (msg), "Failed to set volume id of volume"
-                          " %s", volname);
-                goto out;
-        }
+    int ret = -1;
+    glusterd_volinfo_t *volinfo = NULL;
+    char *volid = NULL;
+    char msg[1024] = {
+        0,
+    };
+    xlator_t *this = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+
+    if (!dict || !volname) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_INVALID_ARGUMENT, NULL);
+        goto out;
+    }
+
+    ret = glusterd_volinfo_find(volname, &volinfo);
+    if (ret) {
+        snprintf(msg, sizeof(msg), FMTSTR_CHECK_VOL_EXISTS, volname);
+        goto out;
+    }
+    volid = gf_strdup(uuid_utoa(volinfo->volume_id));
+    if (!volid) {
+        ret = -1;
+        goto out;
+    }
+    ret = dict_set_dynstrn(dict, "vol-id", SLEN("vol-id"), volid);
+    if (ret) {
+        snprintf(msg, sizeof(msg),
+                 "Failed to set volume id of volume"
+                 " %s",
+                 volname);
+        GF_FREE(volid);
+        goto out;
+    }
 out:
-        if (msg[0] != '\0') {
-                gf_log (this->name, GF_LOG_ERROR, "%s", msg);
-                *op_errstr = gf_strdup (msg);
-        }
-        return ret;
+    if (msg[0] != '\0') {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_VOL_ID_SET_FAIL, "%s", msg);
+        *op_errstr = gf_strdup(msg);
+    }
+    return ret;
 }
 
 int
-glusterd_op_build_payload (dict_t **req, char **op_errstr, dict_t *op_ctx)
+gd_set_commit_hash(dict_t *dict)
 {
-        int                     ret = -1;
-        void                    *ctx = NULL;
-        dict_t                  *dict = NULL;
-        dict_t                  *req_dict = NULL;
-        glusterd_op_t           op = GD_OP_NONE;
-        char                    *volname = NULL;
-        uint32_t                status_cmd = GF_CLI_STATUS_NONE;
-        char                    *errstr = NULL;
-        xlator_t                *this = NULL;
-
-        GF_ASSERT (req);
-
-        this = THIS;
-        GF_ASSERT (this);
-
-        req_dict = dict_new ();
-        if (!req_dict)
-                goto out;
-
-        if (!op_ctx) {
-                op  = glusterd_op_get_op ();
-                ctx = (void*)glusterd_op_get_ctx ();
-                if (!ctx) {
-                        gf_log (this->name, GF_LOG_ERROR, "Null Context for "
-                                "op %d", op);
-                        ret = -1;
-                        goto out;
-                }
-
-        } else {
-#define GD_SYNC_OPCODE_KEY "sync-mgmt-operation"
-                ret = dict_get_int32 (op_ctx, GD_SYNC_OPCODE_KEY, (int32_t*)&op);
-                if (ret) {
-                        gf_log (this->name, GF_LOG_ERROR, "Failed to get volume"
-                                " operation");
-                        goto out;
-                }
-                ctx = op_ctx;
-#undef GD_SYNC_OPCODE_KEY
-        }
-
-        dict = ctx;
-        switch (op) {
-                case GD_OP_CREATE_VOLUME:
-                        {
-                                ++glusterfs_port;
-                                ret = dict_set_int32 (dict, "port",
-                                                      glusterfs_port);
-                                if (ret) {
-                                        gf_log (this->name, GF_LOG_ERROR,
-                                                "Failed to set port in "
-                                                "dictionary");
-                                        goto out;
-                                }
-                                dict_copy (dict, req_dict);
-                        }
-                        break;
-
-                case GD_OP_GSYNC_CREATE:
-                case GD_OP_GSYNC_SET:
-                        {
-                                ret = glusterd_op_gsync_args_get (dict,
-                                                                  &errstr,
-                                                                  &volname,
-                                                                  NULL, NULL);
-                                if (ret == 0) {
-                                        ret = glusterd_dict_set_volid
-                                                (dict, volname, op_errstr);
-                                        if (ret)
-                                                goto out;
-                                }
-                                dict_copy (dict, req_dict);
-                        }
-                        break;
-
-                case GD_OP_SET_VOLUME:
-                        {
-                                ret = dict_get_str (dict, "volname", &volname);
-                                if (ret) {
-                                        gf_log (this->name, GF_LOG_CRITICAL,
-                                                "volname is not present in "
-                                                "operation ctx");
-                                        goto out;
-                                }
-                                if (strcmp (volname, "help") &&
-                                    strcmp (volname, "help-xml") &&
-                                    strcasecmp (volname, "all")) {
-                                        ret = glusterd_dict_set_volid
-                                                 (dict, volname, op_errstr);
-                                        if (ret)
-                                                goto out;
-                                }
-                                dict_destroy (req_dict);
-                                req_dict = dict_ref (dict);
-                        }
-                        break;
-
-                case GD_OP_SYNC_VOLUME:
-                        {
-                                dict_copy (dict, req_dict);
-                                break;
-                        }
-
-                case GD_OP_REMOVE_BRICK:
-                        {
-                                dict_t *dict = ctx;
-                                ret = dict_get_str (dict, "volname", &volname);
-                                if (ret) {
-                                        gf_log (this->name, GF_LOG_CRITICAL,
-                                                "volname is not present in "
-                                                "operation ctx");
-                                        goto out;
-                                }
-
-                                ret = glusterd_dict_set_volid (dict, volname,
-                                                               op_errstr);
-                                if (ret)
-                                        goto out;
-
-                                dict_destroy (req_dict);
-                                req_dict = dict_ref (dict);
-                        }
-                        break;
-
-                case GD_OP_STATUS_VOLUME:
-                        {
-                                ret = dict_get_uint32 (dict, "cmd",
-                                                       &status_cmd);
-                                if (ret) {
-                                        gf_log (this->name, GF_LOG_ERROR,
-                                                "Status command not present "
-                                                "in op ctx");
-                                        goto out;
-                                }
-                                if (GF_CLI_STATUS_ALL & status_cmd) {
-                                        dict_copy (dict, req_dict);
-                                        break;
-                                }
-                        }
-                        /*fall-through*/
-                case GD_OP_DELETE_VOLUME:
-                case GD_OP_START_VOLUME:
-                case GD_OP_STOP_VOLUME:
-                case GD_OP_ADD_BRICK:
-                case GD_OP_REPLACE_BRICK:
-                case GD_OP_RESET_VOLUME:
-                case GD_OP_LOG_ROTATE:
-                case GD_OP_QUOTA:
-                case GD_OP_PROFILE_VOLUME:
-                case GD_OP_REBALANCE:
-                case GD_OP_HEAL_VOLUME:
-                case GD_OP_STATEDUMP_VOLUME:
-                case GD_OP_CLEARLOCKS_VOLUME:
-                case GD_OP_DEFRAG_BRICK_VOLUME:
-                case GD_OP_BARRIER:
-                case GD_OP_BITROT:
-                        {
-                                ret = dict_get_str (dict, "volname", &volname);
-                                if (ret) {
-                                        gf_log (this->name, GF_LOG_CRITICAL,
-                                                "volname is not present in "
-                                                "operation ctx");
-                                        goto out;
-                                }
-
-                                if (strcasecmp (volname, "all")) {
-                                        ret = glusterd_dict_set_volid (dict,
-                                                                       volname,
-                                                                     op_errstr);
-                                        if (ret)
-                                                goto out;
-                                }
-                                dict_copy (dict, req_dict);
-                        }
-                        break;
-
-                case GD_OP_COPY_FILE:
-                        {
-                                dict_copy (dict, req_dict);
-                                break;
-                        }
-
-                case GD_OP_SYS_EXEC:
-                        {
-                                dict_copy (dict, req_dict);
-                                break;
-                        }
-
-                case GD_OP_GANESHA:
-                        {
-                                dict_copy (dict, req_dict);
-                                break;
-                        }
-
-                default:
-                        break;
-        }
-
-        *req = req_dict;
-        ret = 0;
-
-out:
-        return ret;
+    struct timeval tv;
+    uint32_t hash;
+
+    /*
+     * We need a commit hash that won't conflict with others we might have
+     * set, or zero which is the implicit value if we never have.  Using
+     * seconds<<3 like this ensures that we'll only get a collision if two
+     * consecutive rebalances are separated by exactly 2^29 seconds - about
+     * 17 years - and even then there's only a 1/8 chance of a collision in
+     * the low order bits.  It's far more likely that this code will have
+     * changed completely by then.  If not, call me in 2031.
+     *
+     * P.S. Time zone changes?  Yeah, right.
+     */
+    gettimeofday(&tv, NULL);
+    hash = tv.tv_sec << 3;
+
+    /*
+     * Make sure at least one of those low-order bits is set.  The extra
+     * shifting is because not all machines have sub-millisecond time
+     * resolution.
+     */
+    hash |= 1 << ((tv.tv_usec >> 10) % 3);
+
+    return dict_set_uint32(dict, "commit-hash", hash);
 }
 
-static int
-glusterd_op_ac_send_stage_op (glusterd_op_sm_event_t *event, void *ctx)
+int
+glusterd_op_build_payload(dict_t **req, char **op_errstr, dict_t *op_ctx)
 {
-        int                     ret = 0;
-        rpc_clnt_procedure_t    *proc = NULL;
-        glusterd_conf_t         *priv = NULL;
-        xlator_t                *this = NULL;
-        glusterd_peerinfo_t     *peerinfo = NULL;
-        dict_t                  *dict = NULL;
-        dict_t                  *rsp_dict = NULL;
-        char                    *op_errstr  = NULL;
-        glusterd_op_t           op = GD_OP_NONE;
-        uint32_t                pending_count = 0;
-
-        this = THIS;
-        GF_ASSERT (this);
-        priv = this->private;
-        GF_ASSERT (priv);
-
-        op = glusterd_op_get_op ();
-
-        rsp_dict = dict_new();
-        if (!rsp_dict) {
-                gf_log (this->name, GF_LOG_ERROR, "Failed to create rsp_dict");
-                ret = -1;
-                goto out;
-        }
-
-        ret = glusterd_op_build_payload (&dict, &op_errstr, NULL);
+    int ret = -1;
+    void *ctx = NULL;
+    dict_t *dict = NULL;
+    dict_t *req_dict = NULL;
+    glusterd_op_t op = GD_OP_NONE;
+    char *volname = NULL;
+    uint32_t status_cmd = GF_CLI_STATUS_NONE;
+    xlator_t *this = NULL;
+    gf_boolean_t do_common = _gf_false;
+
+    GF_ASSERT(req);
+
+    this = THIS;
+    GF_ASSERT(this);
+
+    req_dict = dict_new();
+    if (!req_dict)
+        goto out;
+
+    if (!op_ctx) {
+        op = glusterd_op_get_op();
+        ctx = (void *)glusterd_op_get_ctx();
+        if (!ctx) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_NO_OPTIONS_GIVEN,
+                   "Null Context for "
+                   "op %d",
+                   op);
+            ret = -1;
+            goto out;
+        }
+
+    } else {
+#define GD_SYNC_OPCODE_KEY "sync-mgmt-operation"
+        ret = dict_get_int32(op_ctx, GD_SYNC_OPCODE_KEY, (int32_t *)&op);
         if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, LOGSTR_BUILD_PAYLOAD,
-                        gd_op_list[op]);
-                if (op_errstr == NULL)
-                        gf_asprintf (&op_errstr, OPERRSTR_BUILD_PAYLOAD);
-                opinfo.op_errstr = op_errstr;
-                goto out;
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+                   "Failed to get volume"
+                   " operation");
+            goto out;
         }
+        ctx = op_ctx;
+#undef GD_SYNC_OPCODE_KEY
+    }
+
+    dict = ctx;
+    switch (op) {
+        case GD_OP_CREATE_VOLUME: {
+            ++glusterfs_port;
+            ret = dict_set_int32n(dict, "port", SLEN("port"), glusterfs_port);
+            if (ret) {
+                gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+                       "Failed to set port in "
+                       "dictionary");
+                goto out;
+            }
+            dict_copy(dict, req_dict);
+        } break;
+
+        case GD_OP_GSYNC_CREATE:
+        case GD_OP_GSYNC_SET: {
+            ret = glusterd_op_gsync_args_get(dict, op_errstr, &volname, NULL,
+                                             NULL);
+            if (ret == 0) {
+                ret = glusterd_dict_set_volid(dict, volname, op_errstr);
+                if (ret)
+                    goto out;
+            }
+            dict_copy(dict, req_dict);
+        } break;
+
+        case GD_OP_SET_VOLUME: {
+            ret = dict_get_strn(dict, "volname", SLEN("volname"), &volname);
+            if (ret) {
+                gf_msg(this->name, GF_LOG_CRITICAL, 0, GD_MSG_DICT_GET_FAILED,
+                       "volname is not present in "
+                       "operation ctx");
+                goto out;
+            }
+            if (strcmp(volname, "help") && strcmp(volname, "help-xml") &&
+                strcasecmp(volname, "all")) {
+                ret = glusterd_dict_set_volid(dict, volname, op_errstr);
+                if (ret)
+                    goto out;
+            }
+            dict_unref(req_dict);
+            req_dict = dict_ref(dict);
+        } break;
 
-        ret = glusterd_validate_quorum (this, op, dict, &op_errstr);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_CRITICAL, 0,
-                        GD_MSG_SERVER_QUORUM_NOT_MET,
-                        "Server quorum not met. Rejecting operation.");
-                opinfo.op_errstr = op_errstr;
+        case GD_OP_REMOVE_BRICK: {
+            dict_t *dict = ctx;
+            ret = dict_get_strn(dict, "volname", SLEN("volname"), &volname);
+            if (ret) {
+                gf_msg(this->name, GF_LOG_CRITICAL, 0, GD_MSG_DICT_GET_FAILED,
+                       "volname is not present in "
+                       "operation ctx");
                 goto out;
-        }
+            }
 
-        ret = glusterd_op_stage_validate (op, dict, &op_errstr, rsp_dict);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, LOGSTR_STAGE_FAIL,
-                        gd_op_list[op], "localhost",
-                        (op_errstr) ? ":" : " ", (op_errstr) ? op_errstr : " ");
-                if (op_errstr == NULL)
-                        gf_asprintf (&op_errstr, OPERRSTR_STAGE_FAIL,
-                                     "localhost");
-                opinfo.op_errstr = op_errstr;
+            ret = glusterd_dict_set_volid(dict, volname, op_errstr);
+            if (ret)
                 goto out;
-        }
-
-        if (op == GD_OP_REPLACE_BRICK)
-                glusterd_rb_use_rsp_dict (NULL, rsp_dict);
 
-        list_for_each_local_xaction_peers (peerinfo,
-                                           opinfo.local_xaction_peers) {
-                GF_ASSERT (peerinfo);
-
-                if (!peerinfo->connected || !peerinfo->mgmt)
-                        continue;
-                if ((peerinfo->state.state != GD_FRIEND_STATE_BEFRIENDED) &&
-                    (glusterd_op_get_op() != GD_OP_SYNC_VOLUME))
-                        continue;
+            if (gd_set_commit_hash(dict) != 0) {
+                goto out;
+            }
 
-                proc = &peerinfo->mgmt->proctable[GLUSTERD_MGMT_STAGE_OP];
-                GF_ASSERT (proc);
-                if (proc->fn) {
-                        ret = dict_set_static_ptr (dict, "peerinfo", peerinfo);
-                        if (ret) {
-                                gf_log (this->name, GF_LOG_ERROR, "failed to "
-                                        "set peerinfo");
-                                goto out;
-                        }
+            dict_unref(req_dict);
+            req_dict = dict_ref(dict);
+        } break;
 
-                        ret = proc->fn (NULL, this, dict);
-                        if (ret) {
-                                gf_log (this->name, GF_LOG_WARNING, "Failed to "
-                                        "send stage request for operation "
-                                        "'Volume %s' to peer %s",
-                                        gd_op_list[op], peerinfo->hostname);
-                                continue;
-                        }
-                        pending_count++;
-                }
-        }
+        case GD_OP_STATUS_VOLUME: {
+            ret = dict_get_uint32(dict, "cmd", &status_cmd);
+            if (ret) {
+                gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+                       "Status command not present "
+                       "in op ctx");
+                goto out;
+            }
+            if (GF_CLI_STATUS_ALL & status_cmd) {
+                dict_copy(dict, req_dict);
+                break;
+            }
+            do_common = _gf_true;
+        } break;
 
-        opinfo.pending_count = pending_count;
-out:
-        if (rsp_dict)
-                dict_unref (rsp_dict);
+        case GD_OP_DELETE_VOLUME:
+        case GD_OP_START_VOLUME:
+        case GD_OP_STOP_VOLUME:
+        case GD_OP_ADD_BRICK:
+        case GD_OP_REPLACE_BRICK:
+        case GD_OP_RESET_VOLUME:
+        case GD_OP_LOG_ROTATE:
+        case GD_OP_QUOTA:
+        case GD_OP_PROFILE_VOLUME:
+        case GD_OP_HEAL_VOLUME:
+        case GD_OP_STATEDUMP_VOLUME:
+        case GD_OP_CLEARLOCKS_VOLUME:
+        case GD_OP_DEFRAG_BRICK_VOLUME:
+        case GD_OP_BARRIER:
+        case GD_OP_BITROT:
+        case GD_OP_SCRUB_STATUS:
+        case GD_OP_SCRUB_ONDEMAND:
+        case GD_OP_RESET_BRICK: {
+            do_common = _gf_true;
+        } break;
+
+        case GD_OP_REBALANCE: {
+            if (gd_set_commit_hash(dict) != 0) {
+                goto out;
+            }
+            do_common = _gf_true;
+        } break;
+
+        case GD_OP_SYNC_VOLUME:
+        case GD_OP_COPY_FILE:
+        case GD_OP_SYS_EXEC:
+        case GD_OP_GANESHA: {
+            dict_copy(dict, req_dict);
+        } break;
 
-        if (dict)
-                dict_unref (dict);
+        default:
+            break;
+    }
+
+    /*
+     * This has been moved out of the switch so that multiple ops with
+     * other special needs can all "fall through" to it.
+     */
+    if (do_common) {
+        ret = dict_get_strn(dict, "volname", SLEN("volname"), &volname);
         if (ret) {
-                glusterd_op_sm_inject_event (GD_OP_EVENT_RCVD_RJT,
-                                             &event->txn_id, NULL);
-                opinfo.op_ret = ret;
+            gf_msg(this->name, GF_LOG_CRITICAL, -ret, GD_MSG_DICT_GET_FAILED,
+                   "volname is not present in "
+                   "operation ctx");
+            goto out;
         }
 
-        gf_log (this->name, GF_LOG_DEBUG, "Sent stage op request for "
-                "'Volume %s' to %d peers", gd_op_list[op],
-                opinfo.pending_count);
-
-        if (!opinfo.pending_count)
-                ret = glusterd_op_sm_inject_all_acc (&event->txn_id);
-
-        gf_log (this->name, GF_LOG_DEBUG, "Returning with %d", ret);
+        if (strcasecmp(volname, "all")) {
+            ret = glusterd_dict_set_volid(dict, volname, op_errstr);
+            if (ret)
+                goto out;
+        }
+        dict_copy(dict, req_dict);
+    }
 
-        return ret;
+    *req = req_dict;
+    ret = 0;
 
+out:
+    return ret;
 }
 
-static int32_t
-glusterd_op_start_rb_timer (dict_t *dict, uuid_t *txn_id)
+static int
+glusterd_op_ac_send_stage_op(glusterd_op_sm_event_t *event, void *ctx)
 {
-        int32_t         op = 0;
-        struct timespec timeout = {0, };
-        glusterd_conf_t *priv = NULL;
-        int32_t         ret = -1;
-        dict_t          *rb_ctx = NULL;
-        uuid_t          *rb_txn_id = NULL;
+    int ret = 0;
+    int ret1 = 0;
+    rpc_clnt_procedure_t *proc = NULL;
+    glusterd_conf_t *priv = NULL;
+    xlator_t *this = NULL;
+    glusterd_peerinfo_t *peerinfo = NULL;
+    dict_t *dict = NULL;
+    dict_t *rsp_dict = NULL;
+    char *op_errstr = NULL;
+    glusterd_op_t op = GD_OP_NONE;
+    uint32_t pending_count = 0;
+
+    this = THIS;
+    GF_ASSERT(this);
+    priv = this->private;
+    GF_ASSERT(priv);
+
+    op = glusterd_op_get_op();
+
+    rsp_dict = dict_new();
+    if (!rsp_dict) {
+        gf_msg(this->name, GF_LOG_ERROR, ENOMEM, GD_MSG_DICT_CREATE_FAIL,
+               "Failed to create rsp_dict");
+        ret = -1;
+        goto out;
+    }
+
+    ret = glusterd_op_build_payload(&dict, &op_errstr, NULL);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_BRICK_OP_PAYLOAD_BUILD_FAIL,
+               LOGSTR_BUILD_PAYLOAD, gd_op_list[op]);
+        if (op_errstr == NULL)
+            gf_asprintf(&op_errstr, OPERRSTR_BUILD_PAYLOAD);
+        opinfo.op_errstr = op_errstr;
+        goto out;
+    }
+
+    ret = glusterd_validate_quorum(this, op, dict, &op_errstr);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_CRITICAL, 0, GD_MSG_SERVER_QUORUM_NOT_MET,
+               "Server quorum not met. Rejecting operation.");
+        opinfo.op_errstr = op_errstr;
+        goto out;
+    }
+
+    ret = glusterd_op_stage_validate(op, dict, &op_errstr, rsp_dict);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_VALIDATE_FAILED,
+               LOGSTR_STAGE_FAIL, gd_op_list[op], "localhost",
+               (op_errstr) ? ":" : " ", (op_errstr) ? op_errstr : " ");
+        if (op_errstr == NULL)
+            gf_asprintf(&op_errstr, OPERRSTR_STAGE_FAIL, "localhost");
+        opinfo.op_errstr = op_errstr;
+        goto out;
+    }
+
+    RCU_READ_LOCK;
+    cds_list_for_each_entry_rcu(peerinfo, &priv->peers, uuid_list)
+    {
+        /* Only send requests to peers who were available before the
+         * transaction started
+         */
+        if (peerinfo->generation > opinfo.txn_generation)
+            continue;
 
-        GF_ASSERT (dict);
-        priv = THIS->private;
+        if (!peerinfo->connected || !peerinfo->mgmt)
+            continue;
+        if ((peerinfo->state.state != GD_FRIEND_STATE_BEFRIENDED) &&
+            (glusterd_op_get_op() != GD_OP_SYNC_VOLUME))
+            continue;
 
-        ret = dict_get_int32 (dict, "operation", &op);
-        if (ret) {
-                gf_log ("", GF_LOG_DEBUG,
-                        "dict_get on operation failed");
-                goto out;
-        }
+        proc = &peerinfo->mgmt->proctable[GLUSTERD_MGMT_STAGE_OP];
+        GF_ASSERT(proc);
+        if (proc->fn) {
+            ret = dict_set_static_ptr(dict, "peerinfo", peerinfo);
+            if (ret) {
+                RCU_READ_UNLOCK;
+                gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+                       "failed to "
+                       "set peerinfo");
+                goto out;
+            }
+
+            ret = proc->fn(NULL, this, dict);
+            if (ret) {
+                gf_msg(this->name, GF_LOG_WARNING, 0,
+                       GD_MSG_STAGE_REQ_SEND_FAIL,
+                       "Failed to "
+                       "send stage request for operation "
+                       "'Volume %s' to peer %s",
+                       gd_op_list[op], peerinfo->hostname);
+                continue;
+            }
+            pending_count++;
+        }
+    }
+    RCU_READ_UNLOCK;
+
+    opinfo.pending_count = pending_count;
+out:
+    if (ret)
+        opinfo.op_ret = ret;
 
-        if (op != GF_REPLACE_OP_START) {
-                ret = glusterd_op_sm_inject_all_acc (txn_id);
-                goto out;
-        }
+    ret1 = glusterd_set_txn_opinfo(&event->txn_id, &opinfo);
+    if (ret1)
+        gf_msg(THIS->name, GF_LOG_ERROR, 0, GD_MSG_TRANS_OPINFO_SET_FAIL,
+               "Unable to set "
+               "transaction's opinfo");
 
-        rb_txn_id = GF_CALLOC (1, sizeof(uuid_t), gf_common_mt_uuid_t);
-        if (!rb_txn_id)
-                goto out;
+    if (rsp_dict)
+        dict_unref(rsp_dict);
 
-        gf_uuid_copy (*rb_txn_id, *txn_id);
-        timeout.tv_sec  = 5;
-        timeout.tv_nsec = 0;
+    if (dict)
+        dict_unref(dict);
+    if (ret) {
+        glusterd_op_sm_inject_event(GD_OP_EVENT_RCVD_RJT, &event->txn_id, NULL);
+        opinfo.op_ret = ret;
+    }
 
-        rb_ctx = dict_copy (dict, rb_ctx);
-        if (!rb_ctx) {
-                gf_log (THIS->name, GF_LOG_ERROR, "Couldn't copy "
-                        "replace brick context. Can't start replace brick");
-                ret = -1;
-                goto out;
-        }
-
-        ret = dict_set_bin (rb_ctx, "transaction_id",
-                            rb_txn_id, sizeof (*rb_txn_id));
-        if (ret) {
-                gf_log ("", GF_LOG_ERROR,
-                       "Failed to set transaction id.");
-                goto out;
-        } else
-               gf_log ("", GF_LOG_DEBUG,
-                        "transaction_id = %s", uuid_utoa (*rb_txn_id));
+    gf_msg_debug(this->name, 0,
+                 "Sent stage op request for "
+                 "'Volume %s' to %d peers",
+                 gd_op_list[op], opinfo.pending_count);
 
-        priv->timer = gf_timer_call_after (THIS->ctx, timeout,
-                                           glusterd_do_replace_brick,
-                                           (void *) rb_ctx);
+    if (!opinfo.pending_count)
+        ret = glusterd_op_sm_inject_all_acc(&event->txn_id);
 
-        ret = 0;
+    gf_msg_debug(this->name, 0, "Returning with %d", ret);
 
-out:
-        return ret;
+    return ret;
 }
 
 /* This function takes a dict and converts the uuid values of key specified
  * into hostnames
  */
 static int
-glusterd_op_volume_dict_uuid_to_hostname (dict_t *dict, const char *key_fmt,
-                                          int idx_min, int idx_max)
+glusterd_op_volume_dict_uuid_to_hostname(dict_t *dict, const char *key_fmt,
+                                         int idx_min, int idx_max)
 {
-        int             ret = -1;
-        int             i = 0;
-        char            key[1024];
-        char            *uuid_str = NULL;
-        uuid_t          uuid = {0,};
-        char            *hostname = NULL;
-        xlator_t        *this = NULL;
-
-        this = THIS;
-        GF_ASSERT (this);
-
-        GF_ASSERT (dict);
-        GF_ASSERT (key_fmt);
-
-        for (i = idx_min; i < idx_max; i++) {
-                memset (key, 0, sizeof (key));
-                snprintf (key, sizeof (key), key_fmt, i);
-                ret = dict_get_str (dict, key, &uuid_str);
-                if (ret)
-                        continue;
+    int ret = -1;
+    int i = 0;
+    char key[128];
+    int keylen;
+    char *uuid_str = NULL;
+    uuid_t uuid = {
+        0,
+    };
+    char *hostname = NULL;
+    xlator_t *this = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+
+    GF_ASSERT(dict);
+    GF_ASSERT(key_fmt);
+
+    for (i = idx_min; i < idx_max; i++) {
+        keylen = snprintf(key, sizeof(key), key_fmt, i);
+        ret = dict_get_strn(dict, key, keylen, &uuid_str);
+        if (ret) {
+            ret = 0;
+            continue;
+        }
 
-                gf_log (this->name, GF_LOG_DEBUG, "Got uuid %s",
-                        uuid_str);
+        gf_msg_debug(this->name, 0, "Got uuid %s", uuid_str);
 
-                ret = gf_uuid_parse (uuid_str, uuid);
-                /* if parsing fails don't error out
-                 * let the original value be retained
-                 */
-                if (ret)
-                        continue;
+        ret = gf_uuid_parse(uuid_str, uuid);
+        /* if parsing fails don't error out
+         * let the original value be retained
+         */
+        if (ret) {
+            ret = 0;
+            continue;
+        }
 
-                hostname = glusterd_uuid_to_hostname (uuid);
-                if (hostname) {
-                        gf_log (this->name, GF_LOG_DEBUG, "%s -> %s",
-                                uuid_str, hostname);
-                        ret = dict_set_dynstr (dict, key, hostname);
-                        if (ret) {
-                                gf_log (this->name, GF_LOG_ERROR,
-                                        "Error setting hostname %s to dict",
-                                        hostname);
-                                GF_FREE (hostname);
-                                goto out;
-                        }
-                }
+        hostname = glusterd_uuid_to_hostname(uuid);
+        if (hostname) {
+            gf_msg_debug(this->name, 0, "%s -> %s", uuid_str, hostname);
+            ret = dict_set_dynstrn(dict, key, keylen, hostname);
+            if (ret) {
+                gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+                       "Error setting hostname %s to dict", hostname);
+                GF_FREE(hostname);
+                goto out;
+            }
         }
+    }
 
 out:
-        gf_log (this->name, GF_LOG_DEBUG, "Returning %d", ret);
-        return ret;
+    gf_msg_debug(this->name, 0, "Returning %d", ret);
+    return ret;
 }
 
 static int
-reassign_defrag_status (dict_t *dict, char *key, gf_defrag_status_t *status)
+reassign_defrag_status(dict_t *dict, char *key, int keylen,
+                       gf_defrag_status_t *status)
 {
-        int ret = 0;
+    int ret = 0;
 
-        if (!*status)
-                return ret;
+    if (!*status)
+        return ret;
 
-        switch (*status) {
+    switch (*status) {
         case GF_DEFRAG_STATUS_STARTED:
-                *status = GF_DEFRAG_STATUS_LAYOUT_FIX_STARTED;
-                break;
+            *status = GF_DEFRAG_STATUS_LAYOUT_FIX_STARTED;
+            break;
 
         case GF_DEFRAG_STATUS_STOPPED:
-                *status = GF_DEFRAG_STATUS_LAYOUT_FIX_STOPPED;
-                break;
+            *status = GF_DEFRAG_STATUS_LAYOUT_FIX_STOPPED;
+            break;
 
         case GF_DEFRAG_STATUS_COMPLETE:
-                *status = GF_DEFRAG_STATUS_LAYOUT_FIX_COMPLETE;
-                break;
+            *status = GF_DEFRAG_STATUS_LAYOUT_FIX_COMPLETE;
+            break;
 
         case GF_DEFRAG_STATUS_FAILED:
-                *status = GF_DEFRAG_STATUS_LAYOUT_FIX_FAILED;
-                break;
+            *status = GF_DEFRAG_STATUS_LAYOUT_FIX_FAILED;
+            break;
         default:
-                break;
-         }
+            break;
+    }
 
-        ret = dict_set_int32(dict, key, *status);
-        if (ret)
-                gf_log (THIS->name, GF_LOG_WARNING,
-                        "failed to reset defrag %s in dict", key);
+    ret = dict_set_int32n(dict, key, keylen, *status);
+    if (ret)
+        gf_msg(THIS->name, GF_LOG_WARNING, 0, GD_MSG_DICT_SET_FAILED,
+               "failed to reset defrag %s in dict", key);
 
-        return ret;
+    return ret;
 }
 
 /* Check and reassign the defrag_status enum got from the rebalance process
@@ -3780,54 +4711,56 @@ reassign_defrag_status (dict_t *dict, char *key, gf_defrag_status_t *status)
  * full-rebalance or just a fix-layout was carried out.
  */
 static int
-glusterd_op_check_peer_defrag_status (dict_t *dict, int count)
+glusterd_op_check_peer_defrag_status(dict_t *dict, int count)
 {
-        glusterd_volinfo_t *volinfo  = NULL;
-        gf_defrag_status_t status    = GF_DEFRAG_STATUS_NOT_STARTED;
-        char               key[256]  = {0,};
-        char               *volname  = NULL;
-        int                ret       = -1;
-        int                i         = 1;
-
-        ret = dict_get_str (dict, "volname", &volname);
-        if (ret) {
-                gf_log (THIS->name, GF_LOG_WARNING, "Unable to get volume name");
-                goto out;
-        }
+    glusterd_volinfo_t *volinfo = NULL;
+    gf_defrag_status_t status = GF_DEFRAG_STATUS_NOT_STARTED;
+    char key[64] = {
+        0,
+    };
+    int keylen;
+    char *volname = NULL;
+    int ret = -1;
+    int i = 1;
+
+    ret = dict_get_strn(dict, "volname", SLEN("volname"), &volname);
+    if (ret) {
+        gf_msg(THIS->name, GF_LOG_WARNING, 0, GD_MSG_DICT_GET_FAILED,
+               "Unable to get volume name");
+        goto out;
+    }
+
+    ret = glusterd_volinfo_find(volname, &volinfo);
+    if (ret) {
+        gf_msg(THIS->name, GF_LOG_WARNING, 0, GD_MSG_VOL_NOT_FOUND,
+               FMTSTR_CHECK_VOL_EXISTS, volname);
+        goto out;
+    }
+
+    if (volinfo->rebal.defrag_cmd != GF_DEFRAG_CMD_START_LAYOUT_FIX) {
+        /* Fix layout was not issued; we don't need to reassign
+           the status */
+        ret = 0;
+        goto out;
+    }
 
-        ret = glusterd_volinfo_find (volname, &volinfo);
+    do {
+        keylen = snprintf(key, sizeof(key), "status-%d", i);
+        ret = dict_get_int32n(dict, key, keylen, (int32_t *)&status);
         if (ret) {
-                gf_log (THIS->name, GF_LOG_WARNING, FMTSTR_CHECK_VOL_EXISTS,
-                        volname);
-                goto out;
-        }
-
-        if (volinfo->rebal.defrag_cmd != GF_DEFRAG_CMD_START_LAYOUT_FIX) {
-                /* Fix layout was not issued; we don't need to reassign
-                   the status */
-                ret = 0;
-                goto out;
+            gf_msg(THIS->name, GF_LOG_WARNING, 0, GD_MSG_DICT_GET_FAILED,
+                   "failed to get defrag %s", key);
+            goto out;
         }
+        ret = reassign_defrag_status(dict, key, keylen, &status);
+        if (ret)
+            goto out;
+        i++;
+    } while (i <= count);
 
-        do {
-                memset (key, 0, 256);
-                snprintf (key, 256, "status-%d", i);
-                ret = dict_get_int32 (dict, key, (int32_t *)&status);
-                if (ret) {
-                        gf_log (THIS->name, GF_LOG_WARNING,
-                                "failed to get defrag %s", key);
-                        goto out;
-                }
-                ret = reassign_defrag_status (dict, key, &status);
-                if (ret)
-                        goto out;
-                i++;
-        } while (i <= count);
-
-        ret = 0;
+    ret = 0;
 out:
-        return ret;
-
+    return ret;
 }
 
 /* This function is used to verify if op_ctx indeed
@@ -3851,52 +4784,49 @@ out:
 */
 
 static gf_boolean_t
-glusterd_is_volume_status_modify_op_ctx (uint32_t cmd)
+glusterd_is_volume_status_modify_op_ctx(uint32_t cmd)
 {
-        if ((cmd & GF_CLI_STATUS_MASK) == GF_CLI_STATUS_NONE) {
-                if (cmd & GF_CLI_STATUS_BRICK)
-                        return _gf_false;
-                if (cmd & GF_CLI_STATUS_ALL)
-                        return _gf_false;
-                return _gf_true;
-        }
-        return _gf_false;
+    if ((cmd & GF_CLI_STATUS_MASK) == GF_CLI_STATUS_NONE) {
+        if (cmd & GF_CLI_STATUS_BRICK)
+            return _gf_false;
+        if (cmd & GF_CLI_STATUS_ALL)
+            return _gf_false;
+        return _gf_true;
+    }
+    return _gf_false;
 }
 
 int
-glusterd_op_modify_port_key (dict_t *op_ctx, int brick_index_max)
+glusterd_op_modify_port_key(dict_t *op_ctx, int brick_index_max)
 {
-        char      *port         = NULL;
-        int       i             = 0;
-        int       ret = -1;
-        char      key[1024]     = {0};
-        char      old_key[1024] = {0};
-
-        for (i = 0; i <= brick_index_max; i++) {
-
-               memset (key, 0, sizeof (key));
-               snprintf (key, sizeof (key), "brick%d.rdma_port", i);
-               ret = dict_get_str (op_ctx, key, &port);
-
-               if (ret) {
-
-                       memset (old_key, 0, sizeof (old_key));
-                       snprintf (old_key, sizeof (old_key),
-                                           "brick%d.port", i);
-                       ret = dict_get_str (op_ctx, old_key, &port);
-                       if (ret)
-                               goto out;
-
-                       ret = dict_set_str (op_ctx, key, port);
-                       if (ret)
-                               goto out;
-                       ret = dict_set_str (op_ctx, old_key, "\0");
-                       if (ret)
-                               goto out;
-               }
+    char *port = NULL;
+    int i = 0;
+    int ret = -1;
+    char key[64] = {0};
+    int keylen;
+    char old_key[64] = {0};
+    int old_keylen;
+
+    for (i = 0; i <= brick_index_max; i++) {
+        keylen = snprintf(key, sizeof(key), "brick%d.rdma_port", i);
+        ret = dict_get_strn(op_ctx, key, keylen, &port);
+
+        if (ret) {
+            old_keylen = snprintf(old_key, sizeof(old_key), "brick%d.port", i);
+            ret = dict_get_strn(op_ctx, old_key, old_keylen, &port);
+            if (ret)
+                goto out;
+
+            ret = dict_set_strn(op_ctx, key, keylen, port);
+            if (ret)
+                goto out;
+            ret = dict_set_nstrn(op_ctx, old_key, old_keylen, "\0", SLEN("\0"));
+            if (ret)
+                goto out;
         }
+    }
 out:
-        return ret;
+    return ret;
 }
 
 /* This function is used to modify the op_ctx dict before sending it back
@@ -3904,2016 +4834,2223 @@ out:
  * hostnames etc.
  */
 void
-glusterd_op_modify_op_ctx (glusterd_op_t op, void *ctx)
+glusterd_op_modify_op_ctx(glusterd_op_t op, void *ctx)
 {
-        int             ret = -1;
-        dict_t          *op_ctx = NULL;
-        int             brick_index_max = -1;
-        int             other_count = 0;
-        int             count = 0;
-        uint32_t        cmd = GF_CLI_STATUS_NONE;
-        xlator_t        *this = NULL;
-        glusterd_conf_t    *conf    = NULL;
-        char               *volname = NULL;
-        glusterd_volinfo_t *volinfo = NULL;
-
-        this = THIS;
-        GF_ASSERT (this);
-        conf = this->private;
-
-        if (ctx)
-                op_ctx = ctx;
-        else
-                op_ctx = glusterd_op_get_ctx();
+    int ret = -1;
+    dict_t *op_ctx = NULL;
+    int brick_index_max = -1;
+    int other_count = 0;
+    int count = 0;
+    uint32_t cmd = GF_CLI_STATUS_NONE;
+    xlator_t *this = NULL;
+    glusterd_conf_t *conf = NULL;
+    char *volname = NULL;
+    glusterd_volinfo_t *volinfo = NULL;
+    char *port = 0;
+    int i = 0;
+    char key[64] = {
+        0,
+    };
+    int keylen;
+
+    this = THIS;
+    GF_ASSERT(this);
+    conf = this->private;
+
+    if (ctx)
+        op_ctx = ctx;
+    else
+        op_ctx = glusterd_op_get_ctx();
+
+    if (!op_ctx) {
+        gf_msg(this->name, GF_LOG_CRITICAL, 0, GD_MSG_OPCTX_NULL,
+               "Operation context is not present.");
+        goto out;
+    }
+
+    switch (op) {
+        case GD_OP_STATUS_VOLUME:
+            ret = dict_get_uint32(op_ctx, "cmd", &cmd);
+            if (ret) {
+                gf_msg_debug(this->name, 0, "Failed to get status cmd");
+                goto out;
+            }
 
-        if (!op_ctx) {
-                gf_log (this->name, GF_LOG_CRITICAL,
-                        "Operation context is not present.");
+            if (!glusterd_is_volume_status_modify_op_ctx(cmd)) {
+                gf_msg_debug(this->name, 0,
+                             "op_ctx modification not required for status "
+                             "operation being performed");
                 goto out;
-        }
+            }
 
-        switch (op) {
-        case GD_OP_STATUS_VOLUME:
-                ret = dict_get_uint32 (op_ctx, "cmd", &cmd);
-                if (ret) {
-                        gf_log (this->name, GF_LOG_DEBUG,
-                                "Failed to get status cmd");
-                        goto out;
-                }
+            ret = dict_get_int32n(op_ctx, "brick-index-max",
+                                  SLEN("brick-index-max"), &brick_index_max);
+            if (ret) {
+                gf_msg_debug(this->name, 0, "Failed to get brick-index-max");
+                goto out;
+            }
 
-                if (!glusterd_is_volume_status_modify_op_ctx (cmd)) {
-                        gf_log (this->name, GF_LOG_DEBUG,
-                                "op_ctx modification not required for status "
-                                "operation being performed");
-                        goto out;
-                }
+            ret = dict_get_int32n(op_ctx, "other-count", SLEN("other-count"),
+                                  &other_count);
+            if (ret) {
+                gf_msg_debug(this->name, 0, "Failed to get other-count");
+                goto out;
+            }
 
-                ret = dict_get_int32 (op_ctx, "brick-index-max",
-                                      &brick_index_max);
-                if (ret) {
-                        gf_log (this->name, GF_LOG_DEBUG,
-                                "Failed to get brick-index-max");
-                        goto out;
-                }
+            count = brick_index_max + other_count + 1;
+
+            /*
+             * a glusterd lesser than version 3.7 will be sending the
+             * rdma port in older key. Changing that value from here
+             * to support backward compatibility
+             */
+            ret = dict_get_strn(op_ctx, "volname", SLEN("volname"), &volname);
+            if (ret)
+                goto out;
 
-                ret = dict_get_int32 (op_ctx, "other-count", &other_count);
+            for (i = 0; i <= brick_index_max; i++) {
+                keylen = snprintf(key, sizeof(key), "brick%d.rdma_port", i);
+                ret = dict_get_strn(op_ctx, key, keylen, &port);
                 if (ret) {
-                        gf_log (this->name, GF_LOG_DEBUG,
-                                "Failed to get other-count");
+                    ret = dict_set_nstrn(op_ctx, key, keylen, "\0", SLEN("\0"));
+                    if (ret)
                         goto out;
                 }
-
-                count = brick_index_max + other_count + 1;
-
-                /*
-                 * a glusterd lesser than version 3.7 will be sending the
-                 * rdma port in older key. Changing that value from here
-                 * to support backward compatibility
-                 */
-                 ret = dict_get_str (op_ctx, "volname", &volname);
-                 if (ret)
-                        goto out;
-
-                 glusterd_volinfo_find (volname, &volinfo);
-                 if (conf->op_version < GD_OP_VERSION_3_7_0 &&
-                     volinfo->transport_type == GF_TRANSPORT_RDMA) {
-                         ret = glusterd_op_modify_port_key (op_ctx,
-                                                            brick_index_max);
-                         if (ret)
-                                 goto out;
-                 }
-                /* add 'brick%d.peerid' into op_ctx with value of 'brick%d.path'.
-                   nfs/sshd like services have this additional uuid */
-                {
-                        char  key[1024];
-                        char *uuid_str = NULL;
-                        char *uuid = NULL;
-                        int   i;
-
-                        for (i = brick_index_max + 1; i < count; i++) {
-                                memset (key, 0, sizeof (key));
-                                snprintf (key, sizeof (key), "brick%d.path", i);
-                                ret = dict_get_str (op_ctx, key, &uuid_str);
-                                if (!ret) {
-                                        memset (key, 0, sizeof (key));
-                                        snprintf (key, sizeof (key),
-                                                  "brick%d.peerid", i);
-                                        uuid = gf_strdup (uuid_str);
-                                        if (!uuid) {
-                                                gf_log (this->name, GF_LOG_DEBUG,
-                                                        "unable to create dup of"
-                                                        " uuid_str");
-                                                continue;
-                                        }
-                                        ret = dict_set_dynstr (op_ctx, key,
-                                                               uuid);
-                                        if (ret != 0) {
-                                                GF_FREE (uuid);
-                                        }
-                                }
+            }
+            ret = glusterd_volinfo_find(volname, &volinfo);
+            if (ret)
+                goto out;
+            if (conf->op_version < GD_OP_VERSION_3_7_0 &&
+                volinfo->transport_type == GF_TRANSPORT_RDMA) {
+                ret = glusterd_op_modify_port_key(op_ctx, brick_index_max);
+                if (ret)
+                    goto out;
+            }
+            /* add 'brick%d.peerid' into op_ctx with value of 'brick%d.path'.
+               nfs/sshd like services have this additional uuid */
+            {
+                char *uuid_str = NULL;
+                char *uuid = NULL;
+                int i;
+
+                for (i = brick_index_max + 1; i < count; i++) {
+                    keylen = snprintf(key, sizeof(key), "brick%d.path", i);
+                    ret = dict_get_strn(op_ctx, key, keylen, &uuid_str);
+                    if (!ret) {
+                        keylen = snprintf(key, sizeof(key), "brick%d.peerid",
+                                          i);
+                        uuid = gf_strdup(uuid_str);
+                        if (!uuid) {
+                            gf_msg_debug(this->name, 0,
+                                         "unable to create dup of"
+                                         " uuid_str");
+                            continue;
+                        }
+                        ret = dict_set_dynstrn(op_ctx, key, keylen, uuid);
+                        if (ret != 0) {
+                            GF_FREE(uuid);
                         }
+                    }
                 }
+            }
 
-                ret = glusterd_op_volume_dict_uuid_to_hostname (op_ctx,
-                                                                "brick%d.path",
-                                                                0, count);
-                if (ret)
-                        gf_log (this->name, GF_LOG_WARNING,
-                                "Failed uuid to hostname conversion");
+            ret = glusterd_op_volume_dict_uuid_to_hostname(
+                op_ctx, "brick%d.path", 0, count);
+            if (ret)
+                gf_msg(this->name, GF_LOG_WARNING, 0, GD_MSG_CONVERSION_FAILED,
+                       "Failed uuid to hostname conversion");
 
-                break;
+            break;
 
         case GD_OP_PROFILE_VOLUME:
-                ret = dict_get_str_boolean (op_ctx, "nfs", _gf_false);
-                if (!ret)
-                        goto out;
+            ret = dict_get_str_boolean(op_ctx, "nfs", _gf_false);
+            if (!ret)
+                goto out;
 
-                ret = dict_get_int32 (op_ctx, "count", &count);
-                if (ret) {
-                        gf_log (this->name, GF_LOG_DEBUG,
-                                "Failed to get brick count");
-                        goto out;
-                }
+            ret = dict_get_int32n(op_ctx, "count", SLEN("count"), &count);
+            if (ret) {
+                gf_msg_debug(this->name, 0, "Failed to get brick count");
+                goto out;
+            }
 
-                ret = glusterd_op_volume_dict_uuid_to_hostname (op_ctx,
-                                                                "%d-brick",
-                                                                1, (count + 1));
-                if (ret)
-                        gf_log (this->name, GF_LOG_WARNING,
-                                "Failed uuid to hostname conversion");
+            ret = glusterd_op_volume_dict_uuid_to_hostname(op_ctx, "%d-brick",
+                                                           1, (count + 1));
+            if (ret)
+                gf_msg(this->name, GF_LOG_WARNING, 0, GD_MSG_CONVERSION_FAILED,
+                       "Failed uuid to hostname conversion");
 
-                break;
+            break;
 
         /* For both rebalance and remove-brick status, the glusterd op is the
          * same
          */
         case GD_OP_DEFRAG_BRICK_VOLUME:
-                ret = dict_get_int32 (op_ctx, "count", &count);
-                if (ret) {
-                        gf_log (this->name, GF_LOG_DEBUG,
-                                "Failed to get count");
-                        goto out;
-                }
-
-                /* add 'node-name-%d' into op_ctx with value uuid_str.
-                   this will be used to convert to hostname later */
-                {
-                        char  key[1024];
-                        char *uuid_str = NULL;
-                        char *uuid = NULL;
-                        int   i;
-
-                        for (i = 1; i <= count; i++) {
-                                memset (key, 0, sizeof (key));
-                                snprintf (key, sizeof (key), "node-uuid-%d", i);
-                                ret = dict_get_str (op_ctx, key, &uuid_str);
-                                if (!ret) {
-                                        memset (key, 0, sizeof (key));
-                                        snprintf (key, sizeof (key),
-                                                  "node-name-%d", i);
-                                        uuid = gf_strdup (uuid_str);
-                                        if (!uuid) {
-                                                gf_log (this->name, GF_LOG_DEBUG,
-                                                        "unable to create dup of"
-                                                        " uuid_str");
-                                                continue;
-                                        }
-                                        ret = dict_set_dynstr (op_ctx, key,
-                                                               uuid);
-                                        if (ret != 0) {
-                                                GF_FREE (uuid);
-                                        }
-                                }
+        case GD_OP_SCRUB_STATUS:
+        case GD_OP_SCRUB_ONDEMAND:
+            ret = dict_get_int32n(op_ctx, "count", SLEN("count"), &count);
+            if (ret) {
+                gf_msg_debug(this->name, 0, "Failed to get count");
+                goto out;
+            }
+
+            /* add 'node-name-%d' into op_ctx with value uuid_str.
+               this will be used to convert to hostname later */
+            {
+                char *uuid_str = NULL;
+                char *uuid = NULL;
+                int i;
+
+                for (i = 1; i <= count; i++) {
+                    keylen = snprintf(key, sizeof(key), "node-uuid-%d", i);
+                    ret = dict_get_strn(op_ctx, key, keylen, &uuid_str);
+                    if (!ret) {
+                        keylen = snprintf(key, sizeof(key), "node-name-%d", i);
+                        uuid = gf_strdup(uuid_str);
+                        if (!uuid) {
+                            gf_msg_debug(this->name, 0,
+                                         "unable to create dup of"
+                                         " uuid_str");
+                            continue;
+                        }
+                        ret = dict_set_dynstrn(op_ctx, key, keylen, uuid);
+                        if (ret != 0) {
+                            GF_FREE(uuid);
                         }
+                    }
                 }
+            }
 
-                ret = glusterd_op_volume_dict_uuid_to_hostname (op_ctx,
-                                                                "node-name-%d",
-                                                                1, (count + 1));
-                if (ret)
-                        gf_log (this->name, GF_LOG_WARNING,
-                                "Failed uuid to hostname conversion");
+            ret = glusterd_op_volume_dict_uuid_to_hostname(
+                op_ctx, "node-name-%d", 1, (count + 1));
+            if (ret)
+                gf_msg(this->name, GF_LOG_WARNING, 0, GD_MSG_CONVERSION_FAILED,
+                       "Failed uuid to hostname conversion");
 
-                ret = glusterd_op_check_peer_defrag_status (op_ctx, count);
-                if (ret)
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "Failed to reset defrag status for fix-layout");
+            /* Since Both rebalance and bitrot scrub status/ondemand
+             * are going to use same code path till here, we should
+             * break in case of scrub status.
+             */
+            if (op == GD_OP_SCRUB_STATUS || op == GD_OP_SCRUB_ONDEMAND) {
                 break;
+            }
 
-        default:
-                ret = 0;
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "op_ctx modification not required");
-                break;
+            ret = glusterd_op_check_peer_defrag_status(op_ctx, count);
+            if (ret)
+                gf_msg(this->name, GF_LOG_ERROR, 0,
+                       GD_MSG_DEFRAG_STATUS_UPDATE_FAIL,
+                       "Failed to reset defrag status for fix-layout");
+            break;
 
-        }
+        default:
+            ret = 0;
+            gf_msg_debug(this->name, 0, "op_ctx modification not required");
+            break;
+    }
 
 out:
-        if (ret)
-                gf_log (this->name, GF_LOG_WARNING,
-                        "op_ctx modification failed");
-        return;
+    if (ret)
+        gf_msg(this->name, GF_LOG_WARNING, 0, GD_MSG_OPCTX_UPDATE_FAIL,
+               "op_ctx modification failed");
+    return;
 }
 
-static int
-glusterd_op_commit_hook (glusterd_op_t op, dict_t *op_ctx,
-                         glusterd_commit_hook_type_t type)
+int
+glusterd_op_commit_hook(glusterd_op_t op, dict_t *op_ctx,
+                        glusterd_commit_hook_type_t type)
 {
-        glusterd_conf_t *priv                   = NULL;
-        char            hookdir[PATH_MAX]       = {0, };
-        char            scriptdir[PATH_MAX]     = {0, };
-        char            type_subdir[256]        = {0, };
-        char            *cmd_subdir             = NULL;
-        int             ret                     = -1;
-
-        priv = THIS->private;
-        switch (type) {
-                case GD_COMMIT_HOOK_NONE:
-                case GD_COMMIT_HOOK_MAX:
-                        /*Won't be called*/
-                        break;
-
-                case GD_COMMIT_HOOK_PRE:
-                        strcpy (type_subdir, "pre");
-                        break;
-                case GD_COMMIT_HOOK_POST:
-                        strcpy (type_subdir, "post");
-                        break;
-        }
-
-        cmd_subdir = glusterd_hooks_get_hooks_cmd_subdir (op);
-        if (strlen (cmd_subdir) == 0)
-                return -1;
-
-        GLUSTERD_GET_HOOKS_DIR (hookdir, GLUSTERD_HOOK_VER, priv);
-        snprintf (scriptdir, sizeof (scriptdir), "%s/%s/%s",
-                  hookdir, cmd_subdir, type_subdir);
-
-        switch (type) {
-                case GD_COMMIT_HOOK_NONE:
-                case GD_COMMIT_HOOK_MAX:
-                        /*Won't be called*/
-                        break;
-
-                case GD_COMMIT_HOOK_PRE:
-                        ret = glusterd_hooks_run_hooks (scriptdir, op, op_ctx,
-                                                        type);
-                        break;
-                case GD_COMMIT_HOOK_POST:
-                        ret = glusterd_hooks_post_stub_enqueue (scriptdir, op,
-                                                                op_ctx);
-                        break;
-        }
-
-        return ret;
+    glusterd_conf_t *priv = NULL;
+    char hookdir[PATH_MAX] = {
+        0,
+    };
+    char scriptdir[PATH_MAX] = {
+        0,
+    };
+    char *type_subdir = "";
+    char *cmd_subdir = NULL;
+    int ret = -1;
+    int32_t len = 0;
+
+    priv = THIS->private;
+    switch (type) {
+        case GD_COMMIT_HOOK_NONE:
+        case GD_COMMIT_HOOK_MAX:
+            /*Won't be called*/
+            break;
+
+        case GD_COMMIT_HOOK_PRE:
+            type_subdir = "pre";
+            break;
+        case GD_COMMIT_HOOK_POST:
+            type_subdir = "post";
+            break;
+    }
+
+    cmd_subdir = glusterd_hooks_get_hooks_cmd_subdir(op);
+    if (strlen(cmd_subdir) == 0)
+        return -1;
+
+    GLUSTERD_GET_HOOKS_DIR(hookdir, GLUSTERD_HOOK_VER, priv);
+    len = snprintf(scriptdir, sizeof(scriptdir), "%s/%s/%s", hookdir,
+                   cmd_subdir, type_subdir);
+    if ((len < 0) || (len >= sizeof(scriptdir))) {
+        return -1;
+    }
+
+    switch (type) {
+        case GD_COMMIT_HOOK_NONE:
+        case GD_COMMIT_HOOK_MAX:
+            /*Won't be called*/
+            break;
+
+        case GD_COMMIT_HOOK_PRE:
+            ret = glusterd_hooks_run_hooks(scriptdir, op, op_ctx, type);
+            break;
+        case GD_COMMIT_HOOK_POST:
+            ret = glusterd_hooks_post_stub_enqueue(scriptdir, op, op_ctx);
+            break;
+    }
+
+    return ret;
 }
 
 static int
-glusterd_op_ac_send_commit_op (glusterd_op_sm_event_t *event, void *ctx)
+glusterd_op_ac_send_commit_op(glusterd_op_sm_event_t *event, void *ctx)
 {
-        int                     ret = 0;
-        rpc_clnt_procedure_t    *proc = NULL;
-        glusterd_conf_t         *priv = NULL;
-        xlator_t                *this = NULL;
-        dict_t                  *dict = NULL;
-        dict_t                  *op_dict = NULL;
-        glusterd_peerinfo_t     *peerinfo = NULL;
-        char                    *op_errstr  = NULL;
-        glusterd_op_t           op = GD_OP_NONE;
-        uint32_t                pending_count = 0;
-
-        this = THIS;
-        GF_ASSERT (this);
-        priv = this->private;
-        GF_ASSERT (priv);
-
-        op      = glusterd_op_get_op ();
-        op_dict = glusterd_op_get_ctx ();
-
-        ret = glusterd_op_build_payload (&dict, &op_errstr, NULL);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, LOGSTR_BUILD_PAYLOAD,
-                        gd_op_list[op]);
-                if (op_errstr == NULL)
-                        gf_asprintf (&op_errstr, OPERRSTR_BUILD_PAYLOAD);
-                opinfo.op_errstr = op_errstr;
-                goto out;
-        }
-
-        ret = glusterd_op_commit_perform (op, dict, &op_errstr, NULL); //rsp_dict invalid for source
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, LOGSTR_COMMIT_FAIL,
-                        gd_op_list[op], "localhost", (op_errstr) ? ":" : " ",
-                        (op_errstr) ? op_errstr : " ");
-                if (op_errstr == NULL)
-                        gf_asprintf (&op_errstr, OPERRSTR_COMMIT_FAIL,
-                                     "localhost");
-                opinfo.op_errstr = op_errstr;
-                goto out;
-        }
+    int ret = 0;
+    int ret1 = 0;
+    rpc_clnt_procedure_t *proc = NULL;
+    glusterd_conf_t *priv = NULL;
+    xlator_t *this = NULL;
+    dict_t *dict = NULL;
+    glusterd_peerinfo_t *peerinfo = NULL;
+    char *op_errstr = NULL;
+    glusterd_op_t op = GD_OP_NONE;
+    uint32_t pending_count = 0;
+
+    this = THIS;
+    GF_ASSERT(this);
+    priv = this->private;
+    GF_ASSERT(priv);
+
+    op = glusterd_op_get_op();
+
+    ret = glusterd_op_build_payload(&dict, &op_errstr, NULL);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_BRICK_OP_PAYLOAD_BUILD_FAIL,
+               LOGSTR_BUILD_PAYLOAD, gd_op_list[op]);
+        if (op_errstr == NULL)
+            gf_asprintf(&op_errstr, OPERRSTR_BUILD_PAYLOAD);
+        opinfo.op_errstr = op_errstr;
+        goto out;
+    }
+
+    ret = glusterd_op_commit_perform(op, dict, &op_errstr,
+                                     NULL);  // rsp_dict invalid for source
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_COMMIT_OP_FAIL,
+               LOGSTR_COMMIT_FAIL, gd_op_list[op], "localhost",
+               (op_errstr) ? ":" : " ", (op_errstr) ? op_errstr : " ");
+        if (op_errstr == NULL)
+            gf_asprintf(&op_errstr, OPERRSTR_COMMIT_FAIL, "localhost");
+        opinfo.op_errstr = op_errstr;
+        goto out;
+    }
+
+    RCU_READ_LOCK;
+    cds_list_for_each_entry_rcu(peerinfo, &priv->peers, uuid_list)
+    {
+        /* Only send requests to peers who were available before the
+         * transaction started
+         */
+        if (peerinfo->generation > opinfo.txn_generation)
+            continue;
 
-        list_for_each_local_xaction_peers (peerinfo,
-                                           opinfo.local_xaction_peers) {
-                GF_ASSERT (peerinfo);
+        if (!peerinfo->connected || !peerinfo->mgmt)
+            continue;
+        if ((peerinfo->state.state != GD_FRIEND_STATE_BEFRIENDED) &&
+            (glusterd_op_get_op() != GD_OP_SYNC_VOLUME))
+            continue;
 
-                if (!peerinfo->connected || !peerinfo->mgmt)
-                        continue;
-                if ((peerinfo->state.state != GD_FRIEND_STATE_BEFRIENDED) &&
-                    (glusterd_op_get_op() != GD_OP_SYNC_VOLUME))
-                        continue;
+        proc = &peerinfo->mgmt->proctable[GLUSTERD_MGMT_COMMIT_OP];
+        GF_ASSERT(proc);
+        if (proc->fn) {
+            ret = dict_set_static_ptr(dict, "peerinfo", peerinfo);
+            if (ret) {
+                RCU_READ_UNLOCK;
+                gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+                       "failed to set peerinfo");
+                goto out;
+            }
+            ret = proc->fn(NULL, this, dict);
+            if (ret) {
+                gf_msg(this->name, GF_LOG_WARNING, 0,
+                       GD_MSG_COMMIT_REQ_SEND_FAIL,
+                       "Failed to "
+                       "send commit request for operation "
+                       "'Volume %s' to peer %s",
+                       gd_op_list[op], peerinfo->hostname);
+                continue;
+            }
+            pending_count++;
+        }
+    }
+    RCU_READ_UNLOCK;
+
+    opinfo.pending_count = pending_count;
+    gf_msg_debug(this->name, 0,
+                 "Sent commit op req for 'Volume %s' "
+                 "to %d peers",
+                 gd_op_list[op], opinfo.pending_count);
+out:
+    if (dict)
+        dict_unref(dict);
 
-                proc = &peerinfo->mgmt->proctable[GLUSTERD_MGMT_COMMIT_OP];
-                GF_ASSERT (proc);
-                if (proc->fn) {
-                        ret = dict_set_static_ptr (dict, "peerinfo", peerinfo);
-                        if (ret) {
-                                gf_log (this->name, GF_LOG_ERROR,
-                                        "failed to set peerinfo");
-                                goto out;
-                        }
-                        ret = proc->fn (NULL, this, dict);
-                        if (ret) {
-                                gf_log (this->name, GF_LOG_WARNING, "Failed to "
-                                        "send commit request for operation "
-                                        "'Volume %s' to peer %s",
-                                        gd_op_list[op], peerinfo->hostname);
-                                continue;
-                        }
-                        pending_count++;
-                }
-        }
+    if (ret)
+        opinfo.op_ret = ret;
 
-        opinfo.pending_count = pending_count;
-        gf_log (this->name, GF_LOG_DEBUG, "Sent commit op req for 'Volume %s' "
-                "to %d peers", gd_op_list[op], opinfo.pending_count);
-out:
-        if (dict)
-                dict_unref (dict);
-        if (ret) {
-                glusterd_op_sm_inject_event (GD_OP_EVENT_RCVD_RJT,
-                                             &event->txn_id, NULL);
-                opinfo.op_ret = ret;
-        }
+    ret1 = glusterd_set_txn_opinfo(&event->txn_id, &opinfo);
+    if (ret1)
+        gf_msg(THIS->name, GF_LOG_ERROR, 0, GD_MSG_TRANS_OPINFO_SET_FAIL,
+               "Unable to set "
+               "transaction's opinfo");
 
-        if (!opinfo.pending_count) {
-                if (op == GD_OP_REPLACE_BRICK) {
-                        ret = glusterd_op_start_rb_timer (op_dict,
-                                                          &event->txn_id);
+    if (ret) {
+        glusterd_op_sm_inject_event(GD_OP_EVENT_RCVD_RJT, &event->txn_id, NULL);
+        opinfo.op_ret = ret;
+    }
 
-                } else {
-                        glusterd_op_modify_op_ctx (op, NULL);
-                        ret = glusterd_op_sm_inject_all_acc (&event->txn_id);
-                }
-                goto err;
+    if (!opinfo.pending_count) {
+        if (op == GD_OP_REPLACE_BRICK) {
+            ret = glusterd_op_sm_inject_all_acc(&event->txn_id);
+        } else {
+            glusterd_op_modify_op_ctx(op, NULL);
+            ret = glusterd_op_sm_inject_all_acc(&event->txn_id);
         }
+        goto err;
+    }
 
 err:
-        gf_log (this->name, GF_LOG_DEBUG, "Returning with %d", ret);
-
-        return ret;
+    gf_msg_debug(this->name, 0, "Returning with %d", ret);
 
+    return ret;
 }
 
 static int
-glusterd_op_ac_rcvd_stage_op_acc (glusterd_op_sm_event_t *event, void *ctx)
+glusterd_op_ac_rcvd_stage_op_acc(glusterd_op_sm_event_t *event, void *ctx)
 {
-        int                     ret = 0;
+    int ret = 0;
 
-        GF_ASSERT (event);
+    GF_ASSERT(event);
 
-        if (opinfo.pending_count > 0)
-                opinfo.pending_count--;
+    if (opinfo.pending_count > 0)
+        opinfo.pending_count--;
 
-        if (opinfo.pending_count > 0)
-                goto out;
+    ret = glusterd_set_txn_opinfo(&event->txn_id, &opinfo);
+    if (ret)
+        gf_msg(THIS->name, GF_LOG_ERROR, 0, GD_MSG_TRANS_OPINFO_SET_FAIL,
+               "Unable to set "
+               "transaction's opinfo");
 
-        ret = glusterd_op_sm_inject_event (GD_OP_EVENT_STAGE_ACC,
-                                           &event->txn_id, NULL);
+    if (opinfo.pending_count > 0)
+        goto out;
+
+    ret = glusterd_op_sm_inject_event(GD_OP_EVENT_STAGE_ACC, &event->txn_id,
+                                      NULL);
 
 out:
-        gf_log (THIS->name, GF_LOG_DEBUG, "Returning %d", ret);
+    gf_msg_debug(THIS->name, 0, "Returning %d", ret);
 
-        return ret;
+    return ret;
 }
 
 static int
-glusterd_op_ac_stage_op_failed (glusterd_op_sm_event_t *event, void *ctx)
+glusterd_op_ac_stage_op_failed(glusterd_op_sm_event_t *event, void *ctx)
 {
-        int                     ret = 0;
+    int ret = 0;
 
-        GF_ASSERT (event);
+    GF_ASSERT(event);
 
-        if (opinfo.pending_count > 0)
-                opinfo.pending_count--;
+    if (opinfo.pending_count > 0)
+        opinfo.pending_count--;
 
-        if (opinfo.pending_count > 0)
-                goto out;
+    ret = glusterd_set_txn_opinfo(&event->txn_id, &opinfo);
+    if (ret)
+        gf_msg(THIS->name, GF_LOG_ERROR, 0, GD_MSG_TRANS_OPINFO_SET_FAIL,
+               "Unable to set "
+               "transaction's opinfo");
+
+    if (opinfo.pending_count > 0)
+        goto out;
 
-        ret = glusterd_op_sm_inject_event (GD_OP_EVENT_ALL_ACK,
-                                           &event->txn_id, NULL);
+    ret = glusterd_op_sm_inject_event(GD_OP_EVENT_ALL_ACK, &event->txn_id,
+                                      NULL);
 
 out:
-        gf_log (THIS->name, GF_LOG_DEBUG, "Returning %d", ret);
+    gf_msg_debug(THIS->name, 0, "Returning %d", ret);
 
-        return ret;
+    return ret;
 }
 
 static int
-glusterd_op_ac_commit_op_failed (glusterd_op_sm_event_t *event, void *ctx)
+glusterd_op_ac_commit_op_failed(glusterd_op_sm_event_t *event, void *ctx)
 {
-        int                     ret = 0;
+    int ret = 0;
 
-        GF_ASSERT (event);
+    GF_ASSERT(event);
 
-        if (opinfo.pending_count > 0)
-                opinfo.pending_count--;
+    if (opinfo.pending_count > 0)
+        opinfo.pending_count--;
 
-        if (opinfo.pending_count > 0)
-                goto out;
+    ret = glusterd_set_txn_opinfo(&event->txn_id, &opinfo);
+    if (ret)
+        gf_msg(THIS->name, GF_LOG_ERROR, 0, GD_MSG_TRANS_OPINFO_SET_FAIL,
+               "Unable to set "
+               "transaction's opinfo");
 
-        ret = glusterd_op_sm_inject_event (GD_OP_EVENT_ALL_ACK,
-                                           &event->txn_id, NULL);
+    if (opinfo.pending_count > 0)
+        goto out;
+
+    ret = glusterd_op_sm_inject_event(GD_OP_EVENT_ALL_ACK, &event->txn_id,
+                                      NULL);
 
 out:
-        gf_log (THIS->name, GF_LOG_DEBUG, "Returning %d", ret);
+    gf_msg_debug(THIS->name, 0, "Returning %d", ret);
 
-        return ret;
+    return ret;
 }
 
 static int
-glusterd_op_ac_brick_op_failed (glusterd_op_sm_event_t *event, void *ctx)
+glusterd_op_ac_brick_op_failed(glusterd_op_sm_event_t *event, void *ctx)
 {
-        int                     ret = 0;
-        glusterd_op_brick_rsp_ctx_t *ev_ctx = NULL;
-        gf_boolean_t                free_errstr = _gf_false;
-        xlator_t                    *this = NULL;
-
-        this = THIS;
-        GF_ASSERT (this);
-
-        GF_ASSERT (event);
-        GF_ASSERT (ctx);
-        ev_ctx = ctx;
-
-        ret = glusterd_remove_pending_entry (&opinfo.pending_bricks, ev_ctx->pending_node->node);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "unknown response received ");
-                ret = -1;
-                free_errstr = _gf_true;
-                goto out;
-        }
-        if (opinfo.brick_pending_count > 0)
-                opinfo.brick_pending_count--;
-        if (opinfo.op_ret == 0)
-                opinfo.op_ret = ev_ctx->op_ret;
-
-        if (opinfo.op_errstr == NULL)
-                opinfo.op_errstr = ev_ctx->op_errstr;
-        else
-                free_errstr = _gf_true;
-
-        if (opinfo.brick_pending_count > 0)
-                goto out;
-
-        ret = glusterd_op_sm_inject_event (GD_OP_EVENT_ALL_ACK,
-                                           &event->txn_id, ev_ctx->commit_ctx);
+    int ret = 0;
+    glusterd_op_brick_rsp_ctx_t *ev_ctx = NULL;
+    gf_boolean_t free_errstr = _gf_false;
+    xlator_t *this = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+
+    GF_ASSERT(event);
+    GF_ASSERT(ctx);
+    ev_ctx = ctx;
+
+    ret = glusterd_remove_pending_entry(&opinfo.pending_bricks,
+                                        ev_ctx->pending_node->node);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_UNKNOWN_RESPONSE,
+               "unknown response received ");
+        ret = -1;
+        free_errstr = _gf_true;
+        goto out;
+    }
+    if (opinfo.brick_pending_count > 0)
+        opinfo.brick_pending_count--;
+    if (opinfo.op_ret == 0)
+        opinfo.op_ret = ev_ctx->op_ret;
+
+    if (opinfo.op_errstr == NULL)
+        opinfo.op_errstr = ev_ctx->op_errstr;
+    else
+        free_errstr = _gf_true;
+
+    ret = glusterd_set_txn_opinfo(&event->txn_id, &opinfo);
+    if (ret)
+        gf_msg(THIS->name, GF_LOG_ERROR, 0, GD_MSG_TRANS_OPINFO_SET_FAIL,
+               "Unable to set "
+               "transaction's opinfo");
+
+    if (opinfo.brick_pending_count > 0)
+        goto out;
+
+    ret = glusterd_op_sm_inject_event(GD_OP_EVENT_ALL_ACK, &event->txn_id,
+                                      ev_ctx->commit_ctx);
 
 out:
-        if (ev_ctx->rsp_dict)
-                dict_unref (ev_ctx->rsp_dict);
-        if (free_errstr && ev_ctx->op_errstr)
-                GF_FREE (ev_ctx->op_errstr);
-        GF_FREE (ctx);
-        gf_log (this->name, GF_LOG_DEBUG, "Returning %d", ret);
-
-        return ret;
+    if (ev_ctx->rsp_dict)
+        dict_unref(ev_ctx->rsp_dict);
+    if (free_errstr && ev_ctx->op_errstr)
+        GF_FREE(ev_ctx->op_errstr);
+    GF_FREE(ctx);
+    gf_msg_debug(this->name, 0, "Returning %d", ret);
+
+    return ret;
 }
 
 static int
-glusterd_op_ac_rcvd_commit_op_acc (glusterd_op_sm_event_t *event, void *ctx)
+glusterd_op_ac_rcvd_commit_op_acc(glusterd_op_sm_event_t *event, void *ctx)
 {
-        dict_t                 *op_ctx            = NULL;
-        int                     ret               = 0;
-        gf_boolean_t            commit_ack_inject = _gf_true;
-        glusterd_op_t           op                = GD_OP_NONE;
-        xlator_t               *this = NULL;
-
-        this = THIS;
-        GF_ASSERT (this);
-        op = glusterd_op_get_op ();
-        GF_ASSERT (event);
-
-        if (opinfo.pending_count > 0)
-                opinfo.pending_count--;
-
-        if (opinfo.pending_count > 0)
-                goto out;
-
-        if (op == GD_OP_REPLACE_BRICK) {
-                op_ctx = glusterd_op_get_ctx ();
-                if (!op_ctx) {
-                        gf_log (this->name, GF_LOG_CRITICAL, "Operation "
-                                "context is not present.");
-                        ret = -1;
-                        goto out;
-                }
-
-                ret = glusterd_op_start_rb_timer (op_ctx, &event->txn_id);
-                if (ret) {
-                        gf_log (this->name, GF_LOG_ERROR, "Couldn't start "
-                                "replace-brick operation.");
-                        goto out;
-                }
-
-                commit_ack_inject = _gf_false;
-                goto out;
+    int ret = 0;
+    gf_boolean_t commit_ack_inject = _gf_true;
+    glusterd_op_t op = GD_OP_NONE;
+    xlator_t *this = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+    op = glusterd_op_get_op();
+    GF_ASSERT(event);
+
+    if (opinfo.pending_count > 0)
+        opinfo.pending_count--;
+
+    ret = glusterd_set_txn_opinfo(&event->txn_id, &opinfo);
+    if (ret)
+        gf_msg(THIS->name, GF_LOG_ERROR, 0, GD_MSG_TRANS_OPINFO_SET_FAIL,
+               "Unable to set "
+               "transaction's opinfo");
+
+    if (opinfo.pending_count > 0)
+        goto out;
+
+    if (op == GD_OP_REPLACE_BRICK) {
+        ret = glusterd_op_sm_inject_all_acc(&event->txn_id);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_RBOP_START_FAIL,
+                   "Couldn't start "
+                   "replace-brick operation.");
+            goto out;
         }
 
+        commit_ack_inject = _gf_false;
+        goto out;
+    }
 
 out:
-        if (commit_ack_inject) {
-                if (ret)
-                        ret = glusterd_op_sm_inject_event (GD_OP_EVENT_RCVD_RJT,
-                                                           &event->txn_id,
-                                                           NULL);
-                else if (!opinfo.pending_count) {
-                        glusterd_op_modify_op_ctx (op, NULL);
-                        ret = glusterd_op_sm_inject_event
-                                                  (GD_OP_EVENT_COMMIT_ACC,
-                                                   &event->txn_id, NULL);
-                }
-                /*else do nothing*/
+    if (commit_ack_inject) {
+        if (ret)
+            ret = glusterd_op_sm_inject_event(GD_OP_EVENT_RCVD_RJT,
+                                              &event->txn_id, NULL);
+        else if (!opinfo.pending_count) {
+            glusterd_op_modify_op_ctx(op, NULL);
+            ret = glusterd_op_sm_inject_event(GD_OP_EVENT_COMMIT_ACC,
+                                              &event->txn_id, NULL);
         }
+        /*else do nothing*/
+    }
 
-        return ret;
+    return ret;
 }
 
 static int
-glusterd_op_ac_rcvd_unlock_acc (glusterd_op_sm_event_t *event, void *ctx)
+glusterd_op_ac_rcvd_unlock_acc(glusterd_op_sm_event_t *event, void *ctx)
 {
-        int                     ret = 0;
+    int ret = 0;
 
-        GF_ASSERT (event);
+    GF_ASSERT(event);
 
-        if (opinfo.pending_count > 0)
-                opinfo.pending_count--;
+    if (opinfo.pending_count > 0)
+        opinfo.pending_count--;
 
-        if (opinfo.pending_count > 0)
-                goto out;
+    ret = glusterd_set_txn_opinfo(&event->txn_id, &opinfo);
+    if (ret)
+        gf_msg(THIS->name, GF_LOG_ERROR, 0, GD_MSG_TRANS_OPINFO_SET_FAIL,
+               "Unable to set "
+               "transaction's opinfo");
 
-        ret = glusterd_op_sm_inject_event (GD_OP_EVENT_ALL_ACC,
-                                           &event->txn_id, NULL);
+    if (opinfo.pending_count > 0)
+        goto out;
 
-        gf_log (THIS->name, GF_LOG_DEBUG, "Returning %d", ret);
+    ret = glusterd_op_sm_inject_event(GD_OP_EVENT_ALL_ACC, &event->txn_id,
+                                      NULL);
+
+    gf_msg_debug(THIS->name, 0, "Returning %d", ret);
 
 out:
-        return ret;
+    return ret;
 }
 
 int32_t
-glusterd_op_clear_errstr() {
-        opinfo.op_errstr = NULL;
-        return 0;
+glusterd_op_clear_errstr()
+{
+    opinfo.op_errstr = NULL;
+    return 0;
 }
 
 int32_t
-glusterd_op_set_ctx (void *ctx)
+glusterd_op_set_ctx(void *ctx)
 {
+    opinfo.op_ctx = ctx;
 
-        opinfo.op_ctx = ctx;
-
-        return 0;
-
+    return 0;
 }
 
 int32_t
-glusterd_op_reset_ctx ()
+glusterd_op_reset_ctx()
 {
+    glusterd_op_set_ctx(NULL);
 
-        glusterd_op_set_ctx (NULL);
-
-        return 0;
+    return 0;
 }
 
 int32_t
-glusterd_op_txn_complete (uuid_t *txn_id)
+glusterd_op_txn_complete(uuid_t *txn_id)
 {
-        int32_t                 ret = -1;
-        glusterd_conf_t         *priv = NULL;
-        int32_t                 op = -1;
-        int32_t                 op_ret = 0;
-        int32_t                 op_errno = 0;
-        rpcsvc_request_t        *req = NULL;
-        void                    *ctx = NULL;
-        char                    *op_errstr = NULL;
-        char                    *volname = NULL;
-        xlator_t                *this = NULL;
-
-        this = THIS;
-        GF_ASSERT (this);
-
-        priv = this->private;
-        GF_ASSERT (priv);
-
-        op  = glusterd_op_get_op ();
-        ctx = glusterd_op_get_ctx ();
-        op_ret = opinfo.op_ret;
-        op_errno = opinfo.op_errno;
-        req = opinfo.req;
-        if (opinfo.op_errstr)
-                op_errstr = opinfo.op_errstr;
-
-        opinfo.op_ret = 0;
-        opinfo.op_errno = 0;
-        glusterd_op_clear_op ();
-        glusterd_op_reset_ctx ();
-        glusterd_op_clear_errstr ();
-        gd_cleanup_local_xaction_peers_list (opinfo.local_xaction_peers);
-
-        /* Based on the op-version, we release the cluster or mgmt_v3 lock */
-        if (priv->op_version < GD_OP_VERSION_3_6_0) {
-                ret = glusterd_unlock (MY_UUID);
-                /* unlock cant/shouldnt fail here!! */
-                if (ret)
-                        gf_log (this->name, GF_LOG_CRITICAL,
-                                "Unable to clear local lock, ret: %d", ret);
-                else
-                        gf_log (this->name, GF_LOG_DEBUG, "Cleared local lock");
-        } else {
-                ret = dict_get_str (ctx, "volname", &volname);
-                if (ret)
-                        gf_log ("", GF_LOG_INFO,
-                                "No Volume name present. "
-                                "Locks have not been held.");
-
-                if (volname) {
-                        ret = glusterd_mgmt_v3_unlock (volname, MY_UUID,
-                                                       "vol");
-                        if (ret)
-                                gf_log (this->name, GF_LOG_ERROR,
-                                        "Unable to release lock for %s",
-                                        volname);
-                }
-        }
-
-        ret = glusterd_op_send_cli_response (op, op_ret,
-                                             op_errno, req, ctx, op_errstr);
-
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "Responding to cli failed, "
-                        "ret: %d", ret);
-                //Ignore this error, else state machine blocks
-                ret = 0;
-        }
+    int32_t ret = -1;
+    glusterd_conf_t *priv = NULL;
+    int32_t op = -1;
+    int32_t op_ret = 0;
+    int32_t op_errno = 0;
+    rpcsvc_request_t *req = NULL;
+    void *ctx = NULL;
+    char *op_errstr = NULL;
+    char *volname = NULL;
+    xlator_t *this = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+
+    priv = this->private;
+    GF_ASSERT(priv);
+
+    op = glusterd_op_get_op();
+    ctx = glusterd_op_get_ctx();
+    op_ret = opinfo.op_ret;
+    op_errno = opinfo.op_errno;
+    req = opinfo.req;
+    if (opinfo.op_errstr)
+        op_errstr = opinfo.op_errstr;
+
+    opinfo.op_ret = 0;
+    opinfo.op_errno = 0;
+    glusterd_op_clear_op();
+    glusterd_op_reset_ctx();
+    glusterd_op_clear_errstr();
+
+    /* Based on the op-version, we release the cluster or mgmt_v3 lock */
+    if (priv->op_version < GD_OP_VERSION_3_6_0) {
+        ret = glusterd_unlock(MY_UUID);
+        /* unlock can't/shouldn't fail here!! */
+        if (ret)
+            gf_msg(this->name, GF_LOG_CRITICAL, 0, GD_MSG_GLUSTERD_UNLOCK_FAIL,
+                   "Unable to clear local lock, ret: %d", ret);
+        else
+            gf_msg_debug(this->name, 0, "Cleared local lock");
+    } else {
+        ret = dict_get_strn(ctx, "volname", SLEN("volname"), &volname);
+        if (ret)
+            gf_msg(this->name, GF_LOG_INFO, 0, GD_MSG_DICT_GET_FAILED,
+                   "No Volume name present. "
+                   "Locks have not been held.");
 
-        if (op_errstr && (strcmp (op_errstr, "")))
-                GF_FREE (op_errstr);
+        if (volname) {
+            ret = glusterd_mgmt_v3_unlock(volname, MY_UUID, "vol");
+            if (ret)
+                gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_MGMTV3_UNLOCK_FAIL,
+                       "Unable to release lock for %s", volname);
+        }
+    }
+
+    ret = glusterd_op_send_cli_response(op, op_ret, op_errno, req, ctx,
+                                        op_errstr);
+
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_NO_CLI_RESP,
+               "Responding to cli failed, "
+               "ret: %d",
+               ret);
+        // Ignore this error, else state machine blocks
+        ret = 0;
+    }
 
+    if (op_errstr && (strcmp(op_errstr, "")))
+        GF_FREE(op_errstr);
 
-        if (priv->pending_quorum_action)
-                glusterd_do_quorum_action ();
+    if (priv->pending_quorum_action)
+        glusterd_do_quorum_action();
 
-        /* Clearing the transaction opinfo */
-        ret = glusterd_clear_txn_opinfo (txn_id);
-        if (ret)
-                gf_log (this->name, GF_LOG_ERROR,
-                        "Unable to clear transaction's opinfo");
+    /* Clearing the transaction opinfo */
+    ret = glusterd_clear_txn_opinfo(txn_id);
+    if (ret)
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_TRANS_OPINFO_CLEAR_FAIL,
+               "Unable to clear transaction's opinfo");
 
-        gf_log (this->name, GF_LOG_DEBUG, "Returning %d", ret);
-        return ret;
+    gf_msg_debug(this->name, 0, "Returning %d", ret);
+    return ret;
 }
 
 static int
-glusterd_op_ac_unlocked_all (glusterd_op_sm_event_t *event, void *ctx)
+glusterd_op_ac_unlocked_all(glusterd_op_sm_event_t *event, void *ctx)
 {
-        int                     ret = 0;
+    int ret = 0;
 
-        GF_ASSERT (event);
+    GF_ASSERT(event);
 
-        ret = glusterd_op_txn_complete (&event->txn_id);
+    ret = glusterd_op_txn_complete(&event->txn_id);
 
-        gf_log (THIS->name, GF_LOG_DEBUG, "Returning %d", ret);
+    gf_msg_debug(THIS->name, 0, "Returning %d", ret);
 
-        return ret;
+    return ret;
 }
 
 static int
-glusterd_op_ac_stage_op (glusterd_op_sm_event_t *event, void *ctx)
+glusterd_op_ac_stage_op(glusterd_op_sm_event_t *event, void *ctx)
 {
-        int                     ret = -1;
-        glusterd_req_ctx_t      *req_ctx = NULL;
-        int32_t                 status = 0;
-        dict_t                  *rsp_dict  = NULL;
-        char                    *op_errstr = NULL;
-        dict_t                  *dict = NULL;
-        xlator_t                *this = NULL;
-        uuid_t                  *txn_id = NULL;
-
-        this = THIS;
-        GF_ASSERT (this);
-        GF_ASSERT (ctx);
-
-        req_ctx = ctx;
-
-        dict = req_ctx->dict;
-
-        rsp_dict = dict_new ();
-        if (!rsp_dict) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "Failed to get new dictionary");
-                return -1;
-        }
-
-        status = glusterd_op_stage_validate (req_ctx->op, dict, &op_errstr,
-                                             rsp_dict);
-
-        if (status) {
-                gf_log (this->name, GF_LOG_ERROR, "Stage failed on operation"
-                        " 'Volume %s', Status : %d", gd_op_list[req_ctx->op],
-                        status);
-        }
-
-        txn_id = GF_CALLOC (1, sizeof(uuid_t), gf_common_mt_uuid_t);
-
-        if (txn_id)
-                gf_uuid_copy (*txn_id, event->txn_id);
-        else {
-                ret = -1;
-                goto out;
-        }
-
-        ret = dict_set_bin (rsp_dict, "transaction_id",
-                            txn_id, sizeof(*txn_id));
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR,
-                       "Failed to set transaction id.");
-                goto out;
-        }
-
-        ret = glusterd_op_stage_send_resp (req_ctx->req, req_ctx->op,
-                                           status, op_errstr, rsp_dict);
+    int ret = -1;
+    glusterd_req_ctx_t *req_ctx = NULL;
+    int32_t status = 0;
+    dict_t *rsp_dict = NULL;
+    char *op_errstr = NULL;
+    dict_t *dict = NULL;
+    xlator_t *this = NULL;
+    uuid_t *txn_id = NULL;
+    glusterd_op_info_t txn_op_info = {
+        {0},
+    };
+    glusterd_conf_t *priv = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+
+    priv = this->private;
+    GF_ASSERT(priv);
+
+    GF_ASSERT(ctx);
+
+    req_ctx = ctx;
+
+    dict = req_ctx->dict;
+
+    rsp_dict = dict_new();
+    if (!rsp_dict) {
+        gf_msg(this->name, GF_LOG_ERROR, ENOMEM, GD_MSG_DICT_CREATE_FAIL,
+               "Failed to get new dictionary");
+        return -1;
+    }
+
+    status = glusterd_op_stage_validate(req_ctx->op, dict, &op_errstr,
+                                        rsp_dict);
+
+    if (status) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_VALIDATE_FAILED,
+               "Stage failed on operation"
+               " 'Volume %s', Status : %d",
+               gd_op_list[req_ctx->op], status);
+    }
+
+    txn_id = GF_MALLOC(sizeof(uuid_t), gf_common_mt_uuid_t);
+
+    if (txn_id)
+        gf_uuid_copy(*txn_id, event->txn_id);
+    else {
+        ret = -1;
+        goto out;
+    }
+    ret = glusterd_get_txn_opinfo(&event->txn_id, &txn_op_info);
+
+    ret = dict_set_bin(rsp_dict, "transaction_id", txn_id, sizeof(*txn_id));
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+               "Failed to set transaction id.");
+        GF_FREE(txn_id);
+        txn_id = NULL;
+        goto out;
+    }
+
+    ret = glusterd_op_stage_send_resp(req_ctx->req, req_ctx->op, status,
+                                      op_errstr, rsp_dict);
 
 out:
-        if (op_errstr && (strcmp (op_errstr, "")))
-                GF_FREE (op_errstr);
+    if (op_errstr && (strcmp(op_errstr, "")))
+        GF_FREE(op_errstr);
 
-        gf_log (this->name, GF_LOG_DEBUG, "Returning with %d", ret);
+    gf_msg_debug(this->name, 0, "Returning with %d", ret);
 
-        if (rsp_dict)
-                dict_unref (rsp_dict);
+    /* for no volname transactions, the txn_opinfo needs to be cleaned up
+     * as there's no unlock event triggered. However if the originator node of
+     * this transaction is still running with a version lower than 60000,
+     * txn_opinfo can't be cleared as that'll lead to a race of referring op_ctx
+     * after it's being freed.
+     */
+    if (txn_op_info.skip_locking && priv->op_version >= GD_OP_VERSION_6_0 &&
+        txn_id)
+        ret = glusterd_clear_txn_opinfo(txn_id);
 
-        return ret;
+    if (rsp_dict)
+        dict_unref(rsp_dict);
+
+    return ret;
 }
 
 static gf_boolean_t
-glusterd_need_brick_op (glusterd_op_t op)
+glusterd_need_brick_op(glusterd_op_t op)
 {
-        gf_boolean_t ret        = _gf_false;
+    gf_boolean_t ret = _gf_false;
 
-        GF_ASSERT (GD_OP_NONE < op && op < GD_OP_MAX);
+    GF_ASSERT(GD_OP_NONE < op && op < GD_OP_MAX);
 
-        switch (op) {
+    switch (op) {
         case GD_OP_PROFILE_VOLUME:
         case GD_OP_STATUS_VOLUME:
         case GD_OP_DEFRAG_BRICK_VOLUME:
         case GD_OP_HEAL_VOLUME:
-                ret = _gf_true;
-                break;
+        case GD_OP_SCRUB_STATUS:
+        case GD_OP_SCRUB_ONDEMAND:
+            ret = _gf_true;
+            break;
         default:
-                ret = _gf_false;
-        }
+            ret = _gf_false;
+    }
 
-        return ret;
+    return ret;
 }
 
-dict_t*
-glusterd_op_init_commit_rsp_dict (glusterd_op_t op)
+dict_t *
+glusterd_op_init_commit_rsp_dict(glusterd_op_t op)
 {
-        dict_t                  *rsp_dict = NULL;
-        dict_t                  *op_ctx   = NULL;
+    dict_t *rsp_dict = NULL;
+    dict_t *op_ctx = NULL;
 
-        GF_ASSERT (GD_OP_NONE < op && op < GD_OP_MAX);
+    GF_ASSERT(GD_OP_NONE < op && op < GD_OP_MAX);
 
-        if (glusterd_need_brick_op (op)) {
-                op_ctx = glusterd_op_get_ctx ();
-                GF_ASSERT (op_ctx);
-                rsp_dict = dict_ref (op_ctx);
-        } else {
-                rsp_dict = dict_new ();
-        }
+    if (glusterd_need_brick_op(op)) {
+        op_ctx = glusterd_op_get_ctx();
+        GF_ASSERT(op_ctx);
+        rsp_dict = dict_ref(op_ctx);
+    } else {
+        rsp_dict = dict_new();
+    }
 
-        return rsp_dict;
+    return rsp_dict;
 }
 
 static int
-glusterd_op_ac_commit_op (glusterd_op_sm_event_t *event, void *ctx)
+glusterd_op_ac_commit_op(glusterd_op_sm_event_t *event, void *ctx)
 {
-        int                       ret        = 0;
-        glusterd_req_ctx_t       *req_ctx    = NULL;
-        int32_t                   status     = 0;
-        char                     *op_errstr  = NULL;
-        dict_t                   *dict       = NULL;
-        dict_t                   *rsp_dict   = NULL;
-        xlator_t                 *this       = NULL;
-        uuid_t                   *txn_id     = NULL;
-
-        this = THIS;
-        GF_ASSERT (this);
-        GF_ASSERT (ctx);
-
-        req_ctx = ctx;
-
-        dict = req_ctx->dict;
-
-        rsp_dict = glusterd_op_init_commit_rsp_dict (req_ctx->op);
-        if (NULL == rsp_dict)
-                return -1;
-
-
-        if (GD_OP_CLEARLOCKS_VOLUME == req_ctx->op) {
-                /*clear locks should be run only on
-                 * originator glusterd*/
-                status = 0;
-
-        } else {
-                status = glusterd_op_commit_perform (req_ctx->op, dict,
-                                                     &op_errstr, rsp_dict);
-        }
-
-        if (status)
-                gf_log (this->name, GF_LOG_ERROR, "Commit of operation "
-                        "'Volume %s' failed: %d", gd_op_list[req_ctx->op],
-                        status);
-
-        txn_id = GF_CALLOC (1, sizeof(uuid_t), gf_common_mt_uuid_t);
-
-        if (txn_id)
-                gf_uuid_copy (*txn_id, event->txn_id);
-        else {
-                ret = -1;
-                goto out;
-        }
-
-        ret = dict_set_bin (rsp_dict, "transaction_id",
-                            txn_id, sizeof(*txn_id));
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR,
-                       "Failed to set transaction id.");
-                goto out;
-        }
-
-        ret = glusterd_op_commit_send_resp (req_ctx->req, req_ctx->op,
-                                            status, op_errstr, rsp_dict);
+    int ret = 0;
+    glusterd_req_ctx_t *req_ctx = NULL;
+    int32_t status = 0;
+    char *op_errstr = NULL;
+    dict_t *dict = NULL;
+    dict_t *rsp_dict = NULL;
+    xlator_t *this = NULL;
+    uuid_t *txn_id = NULL;
+    glusterd_op_info_t txn_op_info = {
+        {0},
+    };
+    gf_boolean_t need_cleanup = _gf_true;
+
+    this = THIS;
+    GF_ASSERT(this);
+    GF_ASSERT(ctx);
+
+    req_ctx = ctx;
+
+    dict = req_ctx->dict;
+
+    rsp_dict = glusterd_op_init_commit_rsp_dict(req_ctx->op);
+    if (NULL == rsp_dict)
+        return -1;
+
+    if (GD_OP_CLEARLOCKS_VOLUME == req_ctx->op) {
+        /*clear locks should be run only on
+         * originator glusterd*/
+        status = 0;
+
+    } else {
+        status = glusterd_op_commit_perform(req_ctx->op, dict, &op_errstr,
+                                            rsp_dict);
+    }
+
+    if (status)
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_COMMIT_OP_FAIL,
+               "Commit of operation "
+               "'Volume %s' failed: %d",
+               gd_op_list[req_ctx->op], status);
+
+    txn_id = GF_MALLOC(sizeof(uuid_t), gf_common_mt_uuid_t);
+
+    if (txn_id)
+        gf_uuid_copy(*txn_id, event->txn_id);
+    else {
+        ret = -1;
+        goto out;
+    }
+    ret = glusterd_get_txn_opinfo(&event->txn_id, &txn_op_info);
+    if (ret) {
+        gf_msg_callingfn(this->name, GF_LOG_ERROR, 0,
+                         GD_MSG_TRANS_OPINFO_GET_FAIL,
+                         "Unable to get transaction opinfo "
+                         "for transaction ID : %s",
+                         uuid_utoa(event->txn_id));
+        goto out;
+    }
+
+    ret = dict_set_bin(rsp_dict, "transaction_id", txn_id, sizeof(*txn_id));
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+               "Failed to set transaction id.");
+        if (txn_op_info.skip_locking)
+            ret = glusterd_clear_txn_opinfo(txn_id);
+        need_cleanup = _gf_false;
+        GF_FREE(txn_id);
+        goto out;
+    }
+
+    ret = glusterd_op_commit_send_resp(req_ctx->req, req_ctx->op, status,
+                                       op_errstr, rsp_dict);
 
 out:
-        if (op_errstr && (strcmp (op_errstr, "")))
-                GF_FREE (op_errstr);
-
-        if (rsp_dict)
-                dict_unref (rsp_dict);
-
-        gf_log (this->name, GF_LOG_DEBUG, "Returning with %d", ret);
-
-        return ret;
+    if (op_errstr && (strcmp(op_errstr, "")))
+        GF_FREE(op_errstr);
+
+    if (rsp_dict)
+        dict_unref(rsp_dict);
+    /* for no volname transactions, the txn_opinfo needs to be cleaned up
+     * as there's no unlock event triggered
+     */
+    if (need_cleanup && txn_id && txn_op_info.skip_locking)
+        ret = glusterd_clear_txn_opinfo(txn_id);
+    gf_msg_debug(this->name, 0, "Returning with %d", ret);
+
+    return ret;
 }
 
 static int
-glusterd_op_ac_send_commit_failed (glusterd_op_sm_event_t *event, void *ctx)
+glusterd_op_ac_send_commit_failed(glusterd_op_sm_event_t *event, void *ctx)
 {
-        int                             ret = 0;
-        glusterd_req_ctx_t              *req_ctx = NULL;
-        dict_t                          *op_ctx = NULL;
+    int ret = 0;
+    glusterd_req_ctx_t *req_ctx = NULL;
+    dict_t *op_ctx = NULL;
 
-        GF_ASSERT (ctx);
+    GF_ASSERT(ctx);
 
-        req_ctx = ctx;
+    req_ctx = ctx;
 
-        op_ctx = glusterd_op_get_ctx ();
+    op_ctx = glusterd_op_get_ctx();
 
-        ret = glusterd_op_commit_send_resp (req_ctx->req, req_ctx->op,
-                                            opinfo.op_ret, opinfo.op_errstr,
-                                            op_ctx);
+    ret = glusterd_op_commit_send_resp(req_ctx->req, req_ctx->op, opinfo.op_ret,
+                                       opinfo.op_errstr, op_ctx);
 
-        if (opinfo.op_errstr && (strcmp (opinfo.op_errstr, ""))) {
-                GF_FREE (opinfo.op_errstr);
-                opinfo.op_errstr = NULL;
-        }
+    if (opinfo.op_errstr && (strcmp(opinfo.op_errstr, ""))) {
+        GF_FREE(opinfo.op_errstr);
+        opinfo.op_errstr = NULL;
+    }
 
-        gf_log (THIS->name, GF_LOG_DEBUG, "Returning with %d", ret);
-        return ret;
+    ret = glusterd_set_txn_opinfo(&event->txn_id, &opinfo);
+    if (ret)
+        gf_msg(THIS->name, GF_LOG_ERROR, 0, GD_MSG_TRANS_OPINFO_SET_FAIL,
+               "Unable to set "
+               "transaction's opinfo");
+
+    gf_msg_debug(THIS->name, 0, "Returning with %d", ret);
+    return ret;
 }
 
 static int
-glusterd_op_sm_transition_state (glusterd_op_info_t *opinfo,
-                                 glusterd_op_sm_t *state,
-                                 glusterd_op_sm_event_type_t event_type)
+glusterd_op_sm_transition_state(glusterd_op_info_t *opinfo,
+                                glusterd_op_sm_t *state,
+                                glusterd_op_sm_event_type_t event_type)
 {
-        glusterd_conf_t         *conf = NULL;
+    glusterd_conf_t *conf = NULL;
 
-        GF_ASSERT (state);
-        GF_ASSERT (opinfo);
+    GF_ASSERT(state);
+    GF_ASSERT(opinfo);
 
-        conf = THIS->private;
-        GF_ASSERT (conf);
+    conf = THIS->private;
+    GF_ASSERT(conf);
 
-        (void) glusterd_sm_tr_log_transition_add (&conf->op_sm_log,
-                                           opinfo->state.state,
-                                           state[event_type].next_state,
-                                           event_type);
+    (void)glusterd_sm_tr_log_transition_add(
+        &conf->op_sm_log, opinfo->state.state, state[event_type].next_state,
+        event_type);
 
-        opinfo->state.state = state[event_type].next_state;
-        return 0;
+    opinfo->state.state = state[event_type].next_state;
+    return 0;
 }
 
 int32_t
-glusterd_op_stage_validate (glusterd_op_t op, dict_t *dict, char **op_errstr,
-                            dict_t *rsp_dict)
+glusterd_op_stage_validate(glusterd_op_t op, dict_t *dict, char **op_errstr,
+                           dict_t *rsp_dict)
 {
-        int ret = -1;
-        xlator_t *this = THIS;
+    int ret = -1;
+    xlator_t *this = THIS;
 
-        switch (op) {
-                case GD_OP_CREATE_VOLUME:
-                        ret = glusterd_op_stage_create_volume (dict, op_errstr,
-                                                               rsp_dict);
-                        break;
-
-                case GD_OP_START_VOLUME:
-                        ret = glusterd_op_stage_start_volume (dict, op_errstr,
-                                                              rsp_dict);
-                        break;
-
-                case GD_OP_STOP_VOLUME:
-                        ret = glusterd_op_stage_stop_volume (dict, op_errstr);
-                        break;
-
-                case GD_OP_DELETE_VOLUME:
-                        ret = glusterd_op_stage_delete_volume (dict, op_errstr);
-                        break;
-
-                case GD_OP_ADD_BRICK:
-                        ret = glusterd_op_stage_add_brick (dict, op_errstr,
-                                                           rsp_dict);
-                        break;
-
-                case GD_OP_REPLACE_BRICK:
-                        ret = glusterd_op_stage_replace_brick (dict, op_errstr,
-                                                               rsp_dict);
-                        break;
-
-                case GD_OP_SET_VOLUME:
-                        ret = glusterd_op_stage_set_volume (dict, op_errstr);
-                        break;
-
-                case GD_OP_GANESHA:
-                        ret = glusterd_op_stage_set_ganesha (dict, op_errstr);
-                        break;
-
-                case GD_OP_RESET_VOLUME:
-                        ret = glusterd_op_stage_reset_volume (dict, op_errstr);
-                        break;
-
-                case GD_OP_REMOVE_BRICK:
-                        ret = glusterd_op_stage_remove_brick (dict, op_errstr);
-                        break;
-
-                case GD_OP_LOG_ROTATE:
-                        ret = glusterd_op_stage_log_rotate (dict, op_errstr);
-                        break;
-
-                case GD_OP_SYNC_VOLUME:
-                        ret = glusterd_op_stage_sync_volume (dict, op_errstr);
-                        break;
-
-                case GD_OP_GSYNC_CREATE:
-                        ret = glusterd_op_stage_gsync_create (dict, op_errstr);
-                        break;
-
-                case GD_OP_GSYNC_SET:
-                        ret = glusterd_op_stage_gsync_set (dict, op_errstr);
-                        break;
-
-                case GD_OP_PROFILE_VOLUME:
-                        ret = glusterd_op_stage_stats_volume (dict, op_errstr);
-                        break;
-
-                case GD_OP_QUOTA:
-                        ret = glusterd_op_stage_quota (dict, op_errstr,
-                                                       rsp_dict);
-                        break;
-
-                case GD_OP_STATUS_VOLUME:
-                        ret = glusterd_op_stage_status_volume (dict, op_errstr);
-                        break;
-
-                case GD_OP_REBALANCE:
-                case GD_OP_DEFRAG_BRICK_VOLUME:
-                        ret = glusterd_op_stage_rebalance (dict, op_errstr);
-                        break;
-
-                case GD_OP_HEAL_VOLUME:
-                        ret = glusterd_op_stage_heal_volume (dict, op_errstr);
-                        break;
-
-                case GD_OP_STATEDUMP_VOLUME:
-                        ret = glusterd_op_stage_statedump_volume (dict,
-                                                                  op_errstr);
-                        break;
-                case GD_OP_CLEARLOCKS_VOLUME:
-                        ret = glusterd_op_stage_clearlocks_volume (dict,
-                                                                   op_errstr);
-                        break;
-
-                case GD_OP_COPY_FILE:
-                        ret = glusterd_op_stage_copy_file (dict, op_errstr);
-                        break;
-
-                case GD_OP_SYS_EXEC:
-                        ret = glusterd_op_stage_sys_exec (dict, op_errstr);
-                        break;
-
-                case GD_OP_BARRIER:
-                        ret = glusterd_op_stage_barrier (dict, op_errstr);
-                        break;
-
-                case GD_OP_BITROT:
-                        ret = glusterd_op_stage_bitrot (dict, op_errstr,
-                                                        rsp_dict);
-                        break;
-
-                default:
-                        gf_log (this->name, GF_LOG_ERROR, "Unknown op %s",
-                                gd_op_list[op]);
-        }
-
-        gf_log (this->name, GF_LOG_DEBUG, "OP = %d. Returning %d", op, ret);
-        return ret;
-}
+    switch (op) {
+        case GD_OP_CREATE_VOLUME:
+            ret = glusterd_op_stage_create_volume(dict, op_errstr, rsp_dict);
+            break;
 
+        case GD_OP_START_VOLUME:
+            ret = glusterd_op_stage_start_volume(dict, op_errstr, rsp_dict);
+            break;
 
-int32_t
-glusterd_op_commit_perform (glusterd_op_t op, dict_t *dict, char **op_errstr,
-                            dict_t *rsp_dict)
-{
-        int ret = -1;
-        xlator_t *this = THIS;
+        case GD_OP_STOP_VOLUME:
+            ret = glusterd_op_stage_stop_volume(dict, op_errstr);
+            break;
 
-        glusterd_op_commit_hook (op, dict, GD_COMMIT_HOOK_PRE);
-        switch (op) {
-                case GD_OP_CREATE_VOLUME:
-                        ret = glusterd_op_create_volume (dict, op_errstr);
-                        break;
-
-                case GD_OP_START_VOLUME:
-                        ret = glusterd_op_start_volume (dict, op_errstr);
-                        break;
-
-                case GD_OP_STOP_VOLUME:
-                        ret = glusterd_op_stop_volume (dict);
-                        break;
-
-                case GD_OP_DELETE_VOLUME:
-                        ret = glusterd_op_delete_volume (dict);
-                        break;
-
-                case GD_OP_ADD_BRICK:
-                        ret = glusterd_op_add_brick (dict, op_errstr);
-                        break;
-
-                case GD_OP_REPLACE_BRICK:
-                        ret = glusterd_op_replace_brick (dict, rsp_dict);
-                        break;
-
-                case GD_OP_SET_VOLUME:
-                        ret = glusterd_op_set_volume (dict, op_errstr);
-                        break;
-                case GD_OP_GANESHA:
-                        ret = glusterd_op_set_ganesha (dict, op_errstr);
-                        break;
-
-                case GD_OP_RESET_VOLUME:
-                        ret = glusterd_op_reset_volume (dict, op_errstr);
-                        break;
-
-                case GD_OP_REMOVE_BRICK:
-                        ret = glusterd_op_remove_brick (dict, op_errstr);
-                        break;
-
-                case GD_OP_LOG_ROTATE:
-                        ret = glusterd_op_log_rotate (dict);
-                        break;
-
-                case GD_OP_SYNC_VOLUME:
-                        ret = glusterd_op_sync_volume (dict, op_errstr, rsp_dict);
-                        break;
-
-                case GD_OP_GSYNC_CREATE:
-                        ret = glusterd_op_gsync_create (dict, op_errstr,
-                                                        rsp_dict);
-                        break;
-
-                case GD_OP_GSYNC_SET:
-                        ret = glusterd_op_gsync_set (dict, op_errstr, rsp_dict);
-                        break;
-
-                case GD_OP_PROFILE_VOLUME:
-                        ret = glusterd_op_stats_volume (dict, op_errstr,
-                                                        rsp_dict);
-                        break;
-
-                case GD_OP_QUOTA:
-                        ret = glusterd_op_quota (dict, op_errstr, rsp_dict);
-                        break;
-
-                case GD_OP_STATUS_VOLUME:
-                        ret = glusterd_op_status_volume (dict, op_errstr, rsp_dict);
-                        break;
-
-                case GD_OP_REBALANCE:
-                case GD_OP_DEFRAG_BRICK_VOLUME:
-                        ret = glusterd_op_rebalance (dict, op_errstr, rsp_dict);
-                        break;
-
-                case GD_OP_HEAL_VOLUME:
-                        ret = glusterd_op_heal_volume (dict, op_errstr);
-                        break;
-
-                case GD_OP_STATEDUMP_VOLUME:
-                        ret = glusterd_op_statedump_volume (dict, op_errstr);
-                        break;
-
-                case GD_OP_CLEARLOCKS_VOLUME:
-                        ret = glusterd_op_clearlocks_volume (dict, op_errstr,
-                                                             rsp_dict);
-                        break;
-
-                case GD_OP_COPY_FILE:
-                        ret = glusterd_op_copy_file (dict, op_errstr);
-                        break;
-
-                case GD_OP_SYS_EXEC:
-                        ret = glusterd_op_sys_exec (dict, op_errstr, rsp_dict);
-                        break;
-
-                case GD_OP_BARRIER:
-                        ret = glusterd_op_barrier (dict, op_errstr);
-                        break;
-
-                case GD_OP_BITROT:
-                        ret = glusterd_op_bitrot (dict, op_errstr, rsp_dict);
-                        break;
+        case GD_OP_DELETE_VOLUME:
+            ret = glusterd_op_stage_delete_volume(dict, op_errstr);
+            break;
 
-                default:
-                        gf_log (this->name, GF_LOG_ERROR, "Unknown op %s",
-                                gd_op_list[op]);
-                        break;
-        }
-
-        if (ret == 0)
-            glusterd_op_commit_hook (op, dict, GD_COMMIT_HOOK_POST);
+        case GD_OP_ADD_BRICK:
+            ret = glusterd_op_stage_add_brick(dict, op_errstr, rsp_dict);
+            break;
 
-        gf_log (this->name, GF_LOG_DEBUG, "Returning %d", ret);
-        return ret;
-}
+        case GD_OP_REPLACE_BRICK:
+            ret = glusterd_op_stage_replace_brick(dict, op_errstr, rsp_dict);
+            break;
 
+        case GD_OP_SET_VOLUME:
+            ret = glusterd_op_stage_set_volume(dict, op_errstr);
+            break;
 
-static int
-glusterd_bricks_select_stop_volume (dict_t *dict, char **op_errstr,
-                                    struct cds_list_head *selected)
-{
-        int                                     ret = 0;
-        int                                     flags = 0;
-        char                                    *volname = NULL;
-        glusterd_volinfo_t                      *volinfo = NULL;
-        glusterd_brickinfo_t                    *brickinfo = NULL;
-        glusterd_pending_node_t                 *pending_node = NULL;
-
-        ret = glusterd_op_stop_volume_args_get (dict, &volname, &flags);
-        if (ret)
-                goto out;
+        case GD_OP_GANESHA:
+            ret = glusterd_op_stage_set_ganesha(dict, op_errstr);
+            break;
 
-        ret  = glusterd_volinfo_find (volname, &volinfo);
-        if (ret) {
-                gf_log (THIS->name, GF_LOG_ERROR, FMTSTR_CHECK_VOL_EXISTS,
-                        volname);
-                gf_asprintf (op_errstr, FMTSTR_CHECK_VOL_EXISTS, volname);
-                goto out;
-        }
+        case GD_OP_RESET_VOLUME:
+            ret = glusterd_op_stage_reset_volume(dict, op_errstr);
+            break;
+        case GD_OP_REMOVE_BRICK:
+            ret = glusterd_op_stage_remove_brick(dict, op_errstr);
+            break;
 
-        cds_list_for_each_entry (brickinfo, &volinfo->bricks, brick_list) {
-                if (glusterd_is_brick_started (brickinfo)) {
-                        pending_node = GF_CALLOC (1, sizeof (*pending_node),
-                                                  gf_gld_mt_pending_node_t);
-                        if (!pending_node) {
-                                ret = -1;
-                                goto out;
-                        } else {
-                                pending_node->node = brickinfo;
-                                pending_node->type = GD_NODE_BRICK;
-                                cds_list_add_tail (&pending_node->list,
-                                                   selected);
-                                pending_node = NULL;
-                        }
-                }
-        }
+        case GD_OP_LOG_ROTATE:
+            ret = glusterd_op_stage_log_rotate(dict, op_errstr);
+            break;
 
-out:
-        return ret;
-}
+        case GD_OP_SYNC_VOLUME:
+            ret = glusterd_op_stage_sync_volume(dict, op_errstr);
+            break;
 
-static int
-glusterd_bricks_select_remove_brick (dict_t *dict, char **op_errstr,
-                                     struct cds_list_head *selected)
-{
-        int                                     ret = -1;
-        char                                    *volname = NULL;
-        glusterd_volinfo_t                      *volinfo = NULL;
-        glusterd_brickinfo_t                    *brickinfo = NULL;
-        char                                    *brick = NULL;
-        int32_t                                 count = 0;
-        int32_t                                 i = 1;
-        char                                    key[256] = {0,};
-        glusterd_pending_node_t                 *pending_node = NULL;
-        int32_t                                 force = 0;
+        case GD_OP_GSYNC_CREATE:
+            ret = glusterd_op_stage_gsync_create(dict, op_errstr);
+            break;
 
+        case GD_OP_GSYNC_SET:
+            ret = glusterd_op_stage_gsync_set(dict, op_errstr);
+            break;
 
+        case GD_OP_PROFILE_VOLUME:
+            ret = glusterd_op_stage_stats_volume(dict, op_errstr);
+            break;
 
-        ret = dict_get_str (dict, "volname", &volname);
+        case GD_OP_QUOTA:
+            ret = glusterd_op_stage_quota(dict, op_errstr, rsp_dict);
+            break;
 
-        if (ret) {
-                gf_log ("", GF_LOG_ERROR, "Unable to get volume name");
-                goto out;
-        }
+        case GD_OP_STATUS_VOLUME:
+            ret = glusterd_op_stage_status_volume(dict, op_errstr);
+            break;
 
-        ret = glusterd_volinfo_find (volname, &volinfo);
+        case GD_OP_REBALANCE:
+        case GD_OP_DEFRAG_BRICK_VOLUME:
+            ret = glusterd_op_stage_rebalance(dict, op_errstr);
+            break;
 
-        if (ret) {
-                gf_log ("", GF_LOG_ERROR, "Unable to allocate memory");
-                goto out;
-        }
+        case GD_OP_HEAL_VOLUME:
+            ret = glusterd_op_stage_heal_volume(dict, op_errstr);
+            break;
 
-        ret = dict_get_int32 (dict, "count", &count);
-        if (ret) {
-                gf_log ("", GF_LOG_ERROR, "Unable to get count");
-                goto out;
-        }
+        case GD_OP_STATEDUMP_VOLUME:
+            ret = glusterd_op_stage_statedump_volume(dict, op_errstr);
+            break;
+        case GD_OP_CLEARLOCKS_VOLUME:
+            ret = glusterd_op_stage_clearlocks_volume(dict, op_errstr);
+            break;
 
-        ret = dict_get_int32 (dict, "force", &force);
-        if (ret) {
-                gf_log (THIS->name, GF_LOG_INFO, "force flag is not set");
-                ret = 0;
-                goto out;
-        }
+        case GD_OP_COPY_FILE:
+            ret = glusterd_op_stage_copy_file(dict, op_errstr);
+            break;
 
-        while ( i <= count) {
-                snprintf (key, 256, "brick%d", i);
+        case GD_OP_SYS_EXEC:
+            ret = glusterd_op_stage_sys_exec(dict, op_errstr);
+            break;
 
-                ret = dict_get_str (dict, key, &brick);
-                if (ret) {
-                        gf_log ("glusterd", GF_LOG_ERROR, "Unable to get brick");
-                        goto out;
-                }
+        case GD_OP_BARRIER:
+            ret = glusterd_op_stage_barrier(dict, op_errstr);
+            break;
 
-                ret = glusterd_volume_brickinfo_get_by_brick (brick, volinfo,
-                                                              &brickinfo);
+        case GD_OP_BITROT:
+        case GD_OP_SCRUB_STATUS:
+        case GD_OP_SCRUB_ONDEMAND:
+            ret = glusterd_op_stage_bitrot(dict, op_errstr, rsp_dict);
+            break;
 
-                if (ret)
-                        goto out;
+        default:
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_INVALID_ENTRY,
+                   "Unknown op %s", gd_op_list[op]);
+    }
 
-                if (glusterd_is_brick_started (brickinfo)) {
-                        pending_node = GF_CALLOC (1, sizeof (*pending_node),
-                                                  gf_gld_mt_pending_node_t);
-                        if (!pending_node) {
-                                ret = -1;
-                                goto out;
-                        } else {
-                                pending_node->node = brickinfo;
-                                pending_node->type = GD_NODE_BRICK;
-                                cds_list_add_tail (&pending_node->list,
-                                                   selected);
-                                pending_node = NULL;
-                        }
-                }
-                i++;
-        }
+    gf_msg_debug(this->name, 0, "OP = %d. Returning %d", op, ret);
+    return ret;
+}
 
-out:
-        return ret;
+static void
+glusterd_wait_for_blockers(glusterd_conf_t *priv)
+{
+    while (GF_ATOMIC_GET(priv->blockers)) {
+        synccond_wait(&priv->cond_blockers, &priv->big_lock);
+    }
 }
 
-static int
-glusterd_bricks_select_profile_volume (dict_t *dict, char **op_errstr,
-                                       struct cds_list_head *selected)
+int32_t
+glusterd_op_commit_perform(glusterd_op_t op, dict_t *dict, char **op_errstr,
+                           dict_t *rsp_dict)
 {
-        int                                     ret = -1;
-        char                                    *volname = NULL;
-        char                                    msg[2048] = {0,};
-        glusterd_conf_t                         *priv = NULL;
-        glusterd_volinfo_t                      *volinfo = NULL;
-        xlator_t                                *this = NULL;
-        int32_t                                 stats_op = GF_CLI_STATS_NONE;
-        glusterd_brickinfo_t                    *brickinfo = NULL;
-        glusterd_pending_node_t                 *pending_node = NULL;
-        char                                    *brick = NULL;
+    int ret = -1;
+    xlator_t *this = THIS;
 
+    glusterd_op_commit_hook(op, dict, GD_COMMIT_HOOK_PRE);
+    switch (op) {
+        case GD_OP_CREATE_VOLUME:
+            ret = glusterd_op_create_volume(dict, op_errstr);
+            break;
 
+        case GD_OP_START_VOLUME:
+            ret = glusterd_op_start_volume(dict, op_errstr);
+            break;
 
-        this = THIS;
-        GF_ASSERT (this);
-        priv = this->private;
-        GF_ASSERT (priv);
+        case GD_OP_STOP_VOLUME:
+            ret = glusterd_op_stop_volume(dict);
+            break;
 
+        case GD_OP_DELETE_VOLUME:
+            glusterd_wait_for_blockers(this->private);
+            ret = glusterd_op_delete_volume(dict);
+            break;
 
-        ret = dict_get_str (dict, "volname", &volname);
-        if (ret) {
-                gf_log ("glusterd", GF_LOG_ERROR, "volume name get failed");
-                goto out;
-        }
+        case GD_OP_ADD_BRICK:
+            glusterd_wait_for_blockers(this->private);
+            ret = glusterd_op_add_brick(dict, op_errstr);
+            break;
 
-        ret = glusterd_volinfo_find (volname, &volinfo);
-        if (ret) {
-                snprintf (msg, sizeof (msg), "Volume %s does not exists",
-                          volname);
+        case GD_OP_REPLACE_BRICK:
+            glusterd_wait_for_blockers(this->private);
+            ret = glusterd_op_replace_brick(dict, rsp_dict);
+            break;
+
+        case GD_OP_SET_VOLUME:
+            ret = glusterd_op_set_volume(dict, op_errstr);
+            break;
+        case GD_OP_GANESHA:
+            ret = glusterd_op_set_ganesha(dict, op_errstr);
+            break;
+        case GD_OP_RESET_VOLUME:
+            ret = glusterd_op_reset_volume(dict, op_errstr);
+            break;
 
-                *op_errstr = gf_strdup (msg);
-                gf_log ("", GF_LOG_ERROR, "%s", msg);
-                goto out;
-        }
+        case GD_OP_REMOVE_BRICK:
+            glusterd_wait_for_blockers(this->private);
+            ret = glusterd_op_remove_brick(dict, op_errstr);
+            break;
 
-        ret = dict_get_int32 (dict, "op", &stats_op);
-        if (ret) {
-                gf_log ("glusterd", GF_LOG_ERROR, "volume profile op get failed");
-                goto out;
-        }
+        case GD_OP_LOG_ROTATE:
+            ret = glusterd_op_log_rotate(dict);
+            break;
 
-        switch (stats_op) {
-        case GF_CLI_STATS_START:
-        case GF_CLI_STATS_STOP:
-                goto out;
-                break;
-        case GF_CLI_STATS_INFO:
-                ret = dict_get_str_boolean (dict, "nfs", _gf_false);
-                if (ret) {
-                        if (!priv->nfs_svc.online) {
-                                ret = -1;
-                                gf_log (this->name, GF_LOG_ERROR, "NFS server"
-                                        " is not running");
-                                goto out;
-                        }
-                        pending_node = GF_CALLOC (1, sizeof (*pending_node),
-                                                  gf_gld_mt_pending_node_t);
-                        if (!pending_node) {
-                                ret = -1;
-                                goto out;
-                        }
-                        pending_node->node = &(priv->nfs_svc);
-                        pending_node->type = GD_NODE_NFS;
-                        cds_list_add_tail (&pending_node->list, selected);
-                        pending_node = NULL;
+        case GD_OP_SYNC_VOLUME:
+            ret = glusterd_op_sync_volume(dict, op_errstr, rsp_dict);
+            break;
 
-                        ret = 0;
-                        goto out;
+        case GD_OP_GSYNC_CREATE:
+            ret = glusterd_op_gsync_create(dict, op_errstr, rsp_dict);
+            break;
 
-                }
-                cds_list_for_each_entry (brickinfo, &volinfo->bricks,
-                                         brick_list) {
-                        if (glusterd_is_brick_started (brickinfo)) {
-                                pending_node = GF_CALLOC (1, sizeof (*pending_node),
-                                                          gf_gld_mt_pending_node_t);
-                                if (!pending_node) {
-                                        ret = -1;
-                                        goto out;
-                                } else {
-                                        pending_node->node = brickinfo;
-                                        pending_node->type = GD_NODE_BRICK;
-                                        cds_list_add_tail (&pending_node->list,
-                                                           selected);
-                                        pending_node = NULL;
-                                }
-                        }
-                }
-                break;
+        case GD_OP_GSYNC_SET:
+            ret = glusterd_op_gsync_set(dict, op_errstr, rsp_dict);
+            break;
 
-        case GF_CLI_STATS_TOP:
-                ret = dict_get_str_boolean (dict, "nfs", _gf_false);
-                if (ret) {
-                        if (!priv->nfs_svc.online) {
-                                ret = -1;
-                                gf_log (this->name, GF_LOG_ERROR, "NFS server"
-                                        " is not running");
-                                goto out;
-                        }
-                        pending_node = GF_CALLOC (1, sizeof (*pending_node),
-                                                  gf_gld_mt_pending_node_t);
-                        if (!pending_node) {
-                                ret = -1;
-                                goto out;
-                        }
-                        pending_node->node = &(priv->nfs_svc);
-                        pending_node->type = GD_NODE_NFS;
-                        cds_list_add_tail (&pending_node->list, selected);
-                        pending_node = NULL;
+        case GD_OP_PROFILE_VOLUME:
+            ret = glusterd_op_stats_volume(dict, op_errstr, rsp_dict);
+            break;
 
-                        ret = 0;
-                        goto out;
+        case GD_OP_QUOTA:
+            ret = glusterd_op_quota(dict, op_errstr, rsp_dict);
+            break;
 
-                }
-                ret = dict_get_str (dict, "brick", &brick);
-                if (!ret) {
-                        ret = glusterd_volume_brickinfo_get_by_brick (brick, volinfo,
-                                                                      &brickinfo);
-                        if (ret)
-                                goto out;
-
-                        if (!glusterd_is_brick_started (brickinfo))
-                                goto out;
-
-                        pending_node = GF_CALLOC (1, sizeof (*pending_node),
-                                                  gf_gld_mt_pending_node_t);
-                        if (!pending_node) {
-                                ret = -1;
-                                goto out;
-                        } else {
-                                pending_node->node = brickinfo;
-                                pending_node->type = GD_NODE_BRICK;
-                                cds_list_add_tail (&pending_node->list,
-                                                   selected);
-                                pending_node = NULL;
-                                goto out;
-                        }
-                }
-                ret = 0;
-                cds_list_for_each_entry (brickinfo, &volinfo->bricks,
-                                         brick_list) {
-                        if (glusterd_is_brick_started (brickinfo)) {
-                                pending_node = GF_CALLOC (1, sizeof (*pending_node),
-                                                          gf_gld_mt_pending_node_t);
-                                if (!pending_node) {
-                                        ret = -1;
-                                        goto out;
-                                } else {
-                                        pending_node->node = brickinfo;
-                                        pending_node->type = GD_NODE_BRICK;
-                                        cds_list_add_tail (&pending_node->list,
-                                                           selected);
-                                        pending_node = NULL;
-                                }
-                        }
-                }
-                break;
+        case GD_OP_STATUS_VOLUME:
+            ret = glusterd_op_status_volume(dict, op_errstr, rsp_dict);
+            break;
 
-        default:
-                GF_ASSERT (0);
-                gf_log ("glusterd", GF_LOG_ERROR, "Invalid profile op: %d",
-                        stats_op);
-                ret = -1;
-                goto out;
-                break;
-        }
+        case GD_OP_REBALANCE:
+        case GD_OP_DEFRAG_BRICK_VOLUME:
+            ret = glusterd_op_rebalance(dict, op_errstr, rsp_dict);
+            break;
 
+        case GD_OP_HEAL_VOLUME:
+            ret = glusterd_op_heal_volume(dict, op_errstr);
+            break;
 
-out:
-        gf_log ("", GF_LOG_DEBUG, "Returning %d", ret);
+        case GD_OP_STATEDUMP_VOLUME:
+            ret = glusterd_op_statedump_volume(dict, op_errstr);
+            break;
 
-        return ret;
+        case GD_OP_CLEARLOCKS_VOLUME:
+            ret = glusterd_op_clearlocks_volume(dict, op_errstr, rsp_dict);
+            break;
+
+        case GD_OP_COPY_FILE:
+            ret = glusterd_op_copy_file(dict, op_errstr);
+            break;
+
+        case GD_OP_SYS_EXEC:
+            ret = glusterd_op_sys_exec(dict, op_errstr, rsp_dict);
+            break;
+
+        case GD_OP_BARRIER:
+            ret = glusterd_op_barrier(dict, op_errstr);
+            break;
+
+        case GD_OP_BITROT:
+        case GD_OP_SCRUB_STATUS:
+        case GD_OP_SCRUB_ONDEMAND:
+            ret = glusterd_op_bitrot(dict, op_errstr, rsp_dict);
+            break;
+
+        default:
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_INVALID_ENTRY,
+                   "Unknown op %s", gd_op_list[op]);
+            break;
+    }
+
+    if (ret == 0)
+        glusterd_op_commit_hook(op, dict, GD_COMMIT_HOOK_POST);
+
+    gf_msg_debug(this->name, 0, "Returning %d", ret);
+    return ret;
 }
 
-int
-_get_hxl_children_count (glusterd_volinfo_t *volinfo)
+static int
+glusterd_bricks_select_stop_volume(dict_t *dict, char **op_errstr,
+                                   struct cds_list_head *selected)
 {
-        if (volinfo->type == GF_CLUSTER_TYPE_DISPERSE) {
-                return volinfo->disperse_count;
-        } else {
-                return volinfo->replica_count;
+    int ret = 0;
+    int flags = 0;
+    char *volname = NULL;
+    glusterd_volinfo_t *volinfo = NULL;
+    glusterd_brickinfo_t *brickinfo = NULL;
+    glusterd_pending_node_t *pending_node = NULL;
+
+    ret = glusterd_op_stop_volume_args_get(dict, &volname, &flags);
+    if (ret)
+        goto out;
+
+    ret = glusterd_volinfo_find(volname, &volinfo);
+    if (ret) {
+        gf_msg(THIS->name, GF_LOG_ERROR, 0, GD_MSG_VOL_NOT_FOUND,
+               FMTSTR_CHECK_VOL_EXISTS, volname);
+        gf_asprintf(op_errstr, FMTSTR_CHECK_VOL_EXISTS, volname);
+        goto out;
+    }
+
+    cds_list_for_each_entry(brickinfo, &volinfo->bricks, brick_list)
+    {
+        if (glusterd_is_brick_started(brickinfo)) {
+            pending_node = GF_CALLOC(1, sizeof(*pending_node),
+                                     gf_gld_mt_pending_node_t);
+            if (!pending_node) {
+                ret = -1;
+                goto out;
+            } else {
+                pending_node->node = brickinfo;
+                pending_node->type = GD_NODE_BRICK;
+                cds_list_add_tail(&pending_node->list, selected);
+                pending_node = NULL;
+            }
+            /*
+             * This is not really the right place to do it, but
+             * it's the most convenient.
+             * TBD: move this to *after* the RPC
+             */
+            brickinfo->status = GF_BRICK_STOPPED;
         }
+    }
+
+out:
+    return ret;
 }
 
 static int
-_add_hxlator_to_dict (dict_t *dict, glusterd_volinfo_t *volinfo, int index,
-                      int count)
+glusterd_bricks_select_remove_brick(dict_t *dict, char **op_errstr,
+                                    struct cds_list_head *selected)
 {
-        int     ret             = -1;
-        char    key[128]        = {0,};
-        char    *xname          = NULL;
-        char    *xl_type        = 0;
+    int ret = -1;
+    char *volname = NULL;
+    glusterd_volinfo_t *volinfo = NULL;
+    glusterd_brickinfo_t *brickinfo = NULL;
+    char *brick = NULL;
+    int32_t count = 0;
+    int32_t i = 1;
+    char key[64] = {
+        0,
+    };
+    int keylen;
+    glusterd_pending_node_t *pending_node = NULL;
+    int32_t command = 0;
+    int32_t force = 0;
+
+    ret = dict_get_strn(dict, "volname", SLEN("volname"), &volname);
+
+    if (ret) {
+        gf_msg("glusterd", GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+               "Unable to get volume name");
+        goto out;
+    }
+
+    ret = glusterd_volinfo_find(volname, &volinfo);
+
+    if (ret) {
+        gf_msg("glusterd", GF_LOG_ERROR, 0, GD_MSG_VOL_NOT_FOUND,
+               "Unable to allocate memory");
+        goto out;
+    }
+
+    ret = dict_get_int32n(dict, "count", SLEN("count"), &count);
+    if (ret) {
+        gf_msg("glusterd", GF_LOG_ERROR, -ret, GD_MSG_DICT_GET_FAILED,
+               "Unable to get count");
+        goto out;
+    }
+
+    ret = dict_get_int32n(dict, "command", SLEN("command"), &command);
+    if (ret) {
+        gf_msg("glusterd", GF_LOG_ERROR, -ret, GD_MSG_DICT_GET_FAILED,
+               "Unable to get command");
+        goto out;
+    }
+
+    ret = dict_get_int32n(dict, "force", SLEN("force"), &force);
+    if (ret) {
+        gf_msg(THIS->name, GF_LOG_INFO, 0, GD_MSG_DICT_GET_FAILED,
+               "force flag is not set");
+        ret = 0;
+        goto out;
+    }
 
-        if (volinfo->type == GF_CLUSTER_TYPE_DISPERSE) {
-                xl_type = "disperse";
-        } else {
-                xl_type = "replicate";
+    while (i <= count) {
+        keylen = snprintf(key, sizeof(key), "brick%d", i);
+
+        ret = dict_get_strn(dict, key, keylen, &brick);
+        if (ret) {
+            gf_msg("glusterd", GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+                   "Unable to get brick");
+            goto out;
         }
-        snprintf (key, sizeof (key), "xl-%d", count);
-        ret = gf_asprintf (&xname, "%s-%s-%d", volinfo->volname, xl_type,
-                           index);
-        if (ret == -1)
-                goto out;
 
-        ret = dict_set_dynstr (dict, key, xname);
+        ret = glusterd_volume_brickinfo_get_by_brick(brick, volinfo, &brickinfo,
+                                                     _gf_false);
+
         if (ret)
+            goto out;
+
+        if (glusterd_is_brick_started(brickinfo)) {
+            pending_node = GF_CALLOC(1, sizeof(*pending_node),
+                                     gf_gld_mt_pending_node_t);
+            if (!pending_node) {
+                ret = -1;
                 goto out;
+            } else {
+                pending_node->node = brickinfo;
+                pending_node->type = GD_NODE_BRICK;
+                cds_list_add_tail(&pending_node->list, selected);
+                pending_node = NULL;
+            }
+            /*
+             * This is not really the right place to do it, but
+             * it's the most convenient.
+             * TBD: move this to *after* the RPC
+             */
+            brickinfo->status = GF_BRICK_STOPPED;
+        }
+        i++;
+    }
 
-        ret = dict_set_int32 (dict, xname, index);
 out:
-        return ret;
+    return ret;
 }
 
-int
-get_replica_index_for_per_replica_cmd (glusterd_volinfo_t *volinfo,
-                                       dict_t *dict)
+static int
+glusterd_bricks_select_profile_volume(dict_t *dict, char **op_errstr,
+                                      struct cds_list_head *selected)
 {
-        int                     ret = 0;
-        char                    *hostname = NULL;
-        char                    *path = NULL;
-        int                     index = 0;
-        glusterd_brickinfo_t   *brickinfo = NULL;
-        int                     cmd_replica_index = -1;
-        int                     replica_count = -1;
-
+    int ret = -1;
+    char *volname = NULL;
+    char msg[2048] = {
+        0,
+    };
+    glusterd_conf_t *priv = NULL;
+    glusterd_volinfo_t *volinfo = NULL;
+    xlator_t *this = NULL;
+    int32_t stats_op = GF_CLI_STATS_NONE;
+    glusterd_brickinfo_t *brickinfo = NULL;
+    glusterd_pending_node_t *pending_node = NULL;
+    char *brick = NULL;
+    int32_t pid = -1;
+    char pidfile[PATH_MAX] = {0};
+
+    this = THIS;
+    GF_ASSERT(this);
+    priv = this->private;
+    GF_ASSERT(priv);
+
+    ret = dict_get_strn(dict, "volname", SLEN("volname"), &volname);
+    if (ret) {
+        gf_msg("glusterd", GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+               "volume name get failed");
+        goto out;
+    }
+
+    ret = glusterd_volinfo_find(volname, &volinfo);
+    if (ret) {
+        snprintf(msg, sizeof(msg), "Volume %s does not exists", volname);
+
+        *op_errstr = gf_strdup(msg);
+        gf_msg("glusterd", GF_LOG_ERROR, 0, GD_MSG_VOL_NOT_FOUND, "%s", msg);
+        goto out;
+    }
+
+    ret = dict_get_int32n(dict, "op", SLEN("op"), &stats_op);
+    if (ret) {
+        gf_msg("glusterd", GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+               "volume profile op get failed");
+        goto out;
+    }
+
+    switch (stats_op) {
+        case GF_CLI_STATS_START:
+        case GF_CLI_STATS_STOP:
+            goto out;
+            break;
+        case GF_CLI_STATS_INFO:
+#ifdef BUILD_GNFS
+            ret = dict_get_str_boolean(dict, "nfs", _gf_false);
+            if (ret) {
+                if (!priv->nfs_svc.online) {
+                    ret = -1;
+                    gf_msg(this->name, GF_LOG_ERROR, 0,
+                           GD_MSG_NFS_SERVER_NOT_RUNNING,
+                           "NFS server"
+                           " is not running");
+                    goto out;
+                }
+                pending_node = GF_CALLOC(1, sizeof(*pending_node),
+                                         gf_gld_mt_pending_node_t);
+                if (!pending_node) {
+                    ret = -1;
+                    goto out;
+                }
+                pending_node->node = &(priv->nfs_svc);
+                pending_node->type = GD_NODE_NFS;
+                cds_list_add_tail(&pending_node->list, selected);
+                pending_node = NULL;
 
-        if (!dict)  {
-                ret = -1;
+                ret = 0;
                 goto out;
-        }
+            }
+#endif
+            cds_list_for_each_entry(brickinfo, &volinfo->bricks, brick_list)
+            {
+                if (glusterd_is_brick_started(brickinfo)) {
+                    /*
+                     * In normal use, glusterd_is_brick_started
+                     * will give us the answer we need.  However,
+                     * in our tests the brick gets detached behind
+                     * our back, so we need to double-check this
+                     * way.
+                     */
+                    GLUSTERD_GET_BRICK_PIDFILE(pidfile, volinfo, brickinfo,
+                                               priv);
+                    if (!gf_is_service_running(pidfile, &pid)) {
+                        continue;
+                    }
+                    pending_node = GF_CALLOC(1, sizeof(*pending_node),
+                                             gf_gld_mt_pending_node_t);
+                    if (!pending_node) {
+                        ret = -1;
+                        goto out;
+                    } else {
+                        pending_node->node = brickinfo;
+                        pending_node->type = GD_NODE_BRICK;
+                        cds_list_add_tail(&pending_node->list, selected);
+                        pending_node = NULL;
+                    }
+                }
+            }
+            break;
 
-        ret = dict_get_str (dict, "per-replica-cmd-hostname", &hostname);
-        if (ret)
-                goto out;
-        ret = dict_get_str (dict, "per-replica-cmd-path", &path);
-        if (ret)
+        case GF_CLI_STATS_TOP:
+#ifdef BUILD_GNFS
+            ret = dict_get_str_boolean(dict, "nfs", _gf_false);
+            if (ret) {
+                if (!priv->nfs_svc.online) {
+                    ret = -1;
+                    gf_msg(this->name, GF_LOG_ERROR, 0,
+                           GD_MSG_NFS_SERVER_NOT_RUNNING,
+                           "NFS server"
+                           " is not running");
+                    goto out;
+                }
+                pending_node = GF_CALLOC(1, sizeof(*pending_node),
+                                         gf_gld_mt_pending_node_t);
+                if (!pending_node) {
+                    ret = -1;
+                    goto out;
+                }
+                pending_node->node = &(priv->nfs_svc);
+                pending_node->type = GD_NODE_NFS;
+                cds_list_add_tail(&pending_node->list, selected);
+                pending_node = NULL;
+
+                ret = 0;
                 goto out;
+            }
+#endif
+            ret = dict_get_strn(dict, "brick", SLEN("brick"), &brick);
+            if (!ret) {
+                ret = glusterd_volume_brickinfo_get_by_brick(
+                    brick, volinfo, &brickinfo, _gf_true);
+                if (ret)
+                    goto out;
 
-        replica_count = volinfo->replica_count;
+                if (!glusterd_is_brick_started(brickinfo))
+                    goto out;
 
-        cds_list_for_each_entry (brickinfo, &volinfo->bricks, brick_list) {
-                if (gf_uuid_is_null (brickinfo->uuid))
-                        (void)glusterd_resolve_brick (brickinfo);
-                if (!strcmp (brickinfo->path,  path) &&
-                    !strcmp (brickinfo->hostname, hostname)) {
-                        cmd_replica_index = index/(replica_count);
+                pending_node = GF_CALLOC(1, sizeof(*pending_node),
+                                         gf_gld_mt_pending_node_t);
+                if (!pending_node) {
+                    ret = -1;
+                    goto out;
+                } else {
+                    pending_node->node = brickinfo;
+                    pending_node->type = GD_NODE_BRICK;
+                    cds_list_add_tail(&pending_node->list, selected);
+                    pending_node = NULL;
+                    goto out;
+                }
+            }
+            ret = 0;
+            cds_list_for_each_entry(brickinfo, &volinfo->bricks, brick_list)
+            {
+                if (glusterd_is_brick_started(brickinfo)) {
+                    pending_node = GF_CALLOC(1, sizeof(*pending_node),
+                                             gf_gld_mt_pending_node_t);
+                    if (!pending_node) {
+                        ret = -1;
                         goto out;
+                    } else {
+                        pending_node->node = brickinfo;
+                        pending_node->type = GD_NODE_BRICK;
+                        cds_list_add_tail(&pending_node->list, selected);
+                        pending_node = NULL;
+                    }
                 }
-                index++;
-        }
+            }
+            break;
 
+        default:
+            GF_ASSERT(0);
+            gf_msg("glusterd", GF_LOG_ERROR, 0, GD_MSG_INVALID_ENTRY,
+                   "Invalid profile op: %d", stats_op);
+            ret = -1;
+            goto out;
+            break;
+    }
 
 out:
-        if (ret)
-                cmd_replica_index = -1;
+    gf_msg_debug("glusterd", 0, "Returning %d", ret);
 
-        return cmd_replica_index;
+    return ret;
 }
 
 int
-_select_hxlator_with_matching_brick (xlator_t *this,
-                                     glusterd_volinfo_t *volinfo, dict_t *dict)
+_get_hxl_children_count(glusterd_volinfo_t *volinfo)
 {
-        char                    *hostname = NULL;
-        char                    *path = NULL;
-        glusterd_brickinfo_t    *brickinfo = NULL;
-        glusterd_conf_t         *priv   = NULL;
-        int                     index = 1;
-        int                     hxl_children = 0;
-
-        priv = this->private;
-        if (!dict ||
-            dict_get_str (dict, "per-replica-cmd-hostname", &hostname) ||
-            dict_get_str (dict, "per-replica-cmd-path", &path))
-                return -1;
-
-        hxl_children = _get_hxl_children_count (volinfo);
-
-        cds_list_for_each_entry (brickinfo, &volinfo->bricks, brick_list) {
-                if (gf_uuid_is_null (brickinfo->uuid))
-                        (void)glusterd_resolve_brick (brickinfo);
-
-                if (!gf_uuid_compare (MY_UUID, brickinfo->uuid)) {
-                        _add_hxlator_to_dict (dict, volinfo,
-                                              (index - 1)/hxl_children, 0);
-                        return 1;
-                }
-                index++;
-        }
+    if (volinfo->type == GF_CLUSTER_TYPE_DISPERSE) {
+        return volinfo->disperse_count;
+    } else {
+        return volinfo->replica_count;
+    }
+}
 
-        return 0;
+static int
+_add_hxlator_to_dict(dict_t *dict, glusterd_volinfo_t *volinfo, int index,
+                     int count)
+{
+    int ret = -1;
+    char key[64] = {
+        0,
+    };
+    int keylen;
+    char *xname = NULL;
+    char *xl_type = 0;
+
+    if (volinfo->type == GF_CLUSTER_TYPE_DISPERSE) {
+        xl_type = "disperse";
+    } else {
+        xl_type = "replicate";
+    }
+    keylen = snprintf(key, sizeof(key), "xl-%d", count);
+    ret = gf_asprintf(&xname, "%s-%s-%d", volinfo->volname, xl_type, index);
+    if (ret == -1)
+        goto out;
+
+    ret = dict_set_dynstrn(dict, key, keylen, xname);
+    if (ret)
+        goto out;
+
+    ret = dict_set_int32(dict, xname, index);
+out:
+    return ret;
 }
+
 int
-_select_hxlators_with_local_bricks (xlator_t *this, glusterd_volinfo_t *volinfo,
-                                    dict_t *dict)
+get_replica_index_for_per_replica_cmd(glusterd_volinfo_t *volinfo, dict_t *dict)
 {
-        glusterd_brickinfo_t    *brickinfo = NULL;
-        glusterd_conf_t         *priv   = NULL;
-        int                     index = 1;
-        int                     hxlator_count = 0;
-        int                     hxl_children = 0;
-        gf_boolean_t            add     = _gf_false;
-        int                     cmd_replica_index = -1;
-
-        priv = this->private;
-        hxl_children = _get_hxl_children_count (volinfo);
-
-        cds_list_for_each_entry (brickinfo, &volinfo->bricks, brick_list) {
-                if (gf_uuid_is_null (brickinfo->uuid))
-                        (void)glusterd_resolve_brick (brickinfo);
-
-                if (!gf_uuid_compare (MY_UUID, brickinfo->uuid))
-                        add = _gf_true;
-
-                if (index % hxl_children == 0) {
-                        if (add) {
-                                _add_hxlator_to_dict (dict, volinfo,
-                                                      (index - 1)/hxl_children,
-                                                      hxlator_count);
-                                hxlator_count++;
-                        }
-                        add = _gf_false;
-                }
+    int ret = 0;
+    char *hostname = NULL;
+    char *path = NULL;
+    int index = 0;
+    glusterd_brickinfo_t *brickinfo = NULL;
+    int cmd_replica_index = -1;
+    int replica_count = -1;
+
+    if (!dict) {
+        ret = -1;
+        goto out;
+    }
+
+    ret = dict_get_strn(dict, "per-replica-cmd-hostname",
+                        SLEN("per-replica-cmd-hostname"), &hostname);
+    if (ret)
+        goto out;
+    ret = dict_get_strn(dict, "per-replica-cmd-path",
+                        SLEN("per-replica-cmd-path"), &path);
+    if (ret)
+        goto out;
+
+    replica_count = volinfo->replica_count;
+
+    cds_list_for_each_entry(brickinfo, &volinfo->bricks, brick_list)
+    {
+        if (gf_uuid_is_null(brickinfo->uuid))
+            (void)glusterd_resolve_brick(brickinfo);
+        if (!strcmp(brickinfo->path, path) &&
+            !strcmp(brickinfo->hostname, hostname)) {
+            cmd_replica_index = index / (replica_count);
+            goto out;
+        }
+        index++;
+    }
 
-                index++;
-        }
+out:
+    if (ret)
+        cmd_replica_index = -1;
 
-        return hxlator_count;
+    return cmd_replica_index;
 }
 
 int
-_select_hxlators_for_full_self_heal (xlator_t *this,
-                                     glusterd_volinfo_t *volinfo,
-                                     dict_t *dict)
+_select_hxlator_with_matching_brick(xlator_t *this, glusterd_volinfo_t *volinfo,
+                                    dict_t *dict, int *index)
 {
-        glusterd_brickinfo_t    *brickinfo = NULL;
-        glusterd_conf_t         *priv   = NULL;
-        int                     index = 1;
-        int                     hxlator_count = 0;
-        int                     hxl_children = 0;
-        uuid_t                  candidate = {0};
-
-        priv = this->private;
-        if (volinfo->type == GF_CLUSTER_TYPE_DISPERSE) {
-                hxl_children = volinfo->disperse_count;
-        } else {
-                hxl_children = volinfo->replica_count;
-        }
+    char *path = NULL;
+    glusterd_brickinfo_t *brickinfo = NULL;
+    int hxl_children = 0;
 
-        cds_list_for_each_entry (brickinfo, &volinfo->bricks, brick_list) {
-                if (gf_uuid_is_null (brickinfo->uuid))
-                        (void)glusterd_resolve_brick (brickinfo);
+    if (!dict || dict_get_strn(dict, "per-replica-cmd-path",
+                               SLEN("per-replica-cmd-path"), &path))
+        return -1;
 
-                if (gf_uuid_compare (brickinfo->uuid, candidate) > 0)
-                        gf_uuid_copy (candidate, brickinfo->uuid);
+    hxl_children = _get_hxl_children_count(volinfo);
+    if ((*index) == 0)
+        (*index)++;
 
-                if (index % hxl_children == 0) {
-                        if (!gf_uuid_compare (MY_UUID, candidate)) {
-                                _add_hxlator_to_dict (dict, volinfo,
-                                                      (index-1)/hxl_children,
-                                                      hxlator_count);
-                                hxlator_count++;
-                        }
-                        gf_uuid_clear (candidate);
-                }
+    cds_list_for_each_entry(brickinfo, &volinfo->bricks, brick_list)
+    {
+        if (gf_uuid_is_null(brickinfo->uuid))
+            (void)glusterd_resolve_brick(brickinfo);
 
-                index++;
+        if ((!gf_uuid_compare(MY_UUID, brickinfo->uuid)) &&
+            (!strncmp(brickinfo->path, path, strlen(path)))) {
+            _add_hxlator_to_dict(dict, volinfo, ((*index) - 1) / hxl_children,
+                                 0);
+            return 1;
         }
-        return hxlator_count;
+        (*index)++;
+    }
+
+    return 0;
 }
+void
+_select_hxlators_with_local_bricks(xlator_t *this, glusterd_volinfo_t *volinfo,
+                                   dict_t *dict, int *index, int *hxlator_count)
+{
+    glusterd_brickinfo_t *brickinfo = NULL;
+    int hxl_children = 0;
+    gf_boolean_t add = _gf_false;
 
+    hxl_children = _get_hxl_children_count(volinfo);
 
-static int
-glusterd_bricks_select_snap (dict_t *dict, char **op_errstr,
-                             struct cds_list_head *selected)
-{
-        int                        ret           = -1;
-        glusterd_conf_t            *priv         = NULL;
-        xlator_t                   *this         = NULL;
-        glusterd_pending_node_t    *pending_node = NULL;
-        glusterd_volinfo_t         *volinfo      = NULL;
-        char                       *volname      = NULL;
-        glusterd_brickinfo_t       *brickinfo    = NULL;
-        int                         brick_index  = -1;
-
-        this = THIS;
-        GF_ASSERT (this);
-        priv = this->private;
-        GF_ASSERT (priv);
-
-        ret = dict_get_str (dict, "volname", &volname);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "Unable to get"
-                        " volname");
-                goto out;
-        }
-        ret = glusterd_volinfo_find (volname, &volinfo);
-        if (ret)
-                goto out;
+    if ((*index) == 0)
+        (*index)++;
 
-        cds_list_for_each_entry (brickinfo, &volinfo->bricks, brick_list) {
-                brick_index++;
-                if (gf_uuid_compare (brickinfo->uuid, MY_UUID) ||
-                    !glusterd_is_brick_started (brickinfo)) {
-                        continue;
-                }
-                pending_node = GF_CALLOC (1, sizeof (*pending_node),
-                                          gf_gld_mt_pending_node_t);
-                if (!pending_node) {
-                        ret = -1;
-                        goto out;
-                }
-                pending_node->node = brickinfo;
-                pending_node->type = GD_NODE_BRICK;
-                pending_node->index = brick_index;
-                cds_list_add_tail (&pending_node->list, selected);
-                pending_node = NULL;
+    cds_list_for_each_entry(brickinfo, &volinfo->bricks, brick_list)
+    {
+        if (gf_uuid_is_null(brickinfo->uuid))
+            (void)glusterd_resolve_brick(brickinfo);
+
+        if (!gf_uuid_compare(MY_UUID, brickinfo->uuid))
+            add = _gf_true;
+
+        if ((*index) % hxl_children == 0) {
+            if (add) {
+                _add_hxlator_to_dict(dict, volinfo,
+                                     ((*index) - 1) / hxl_children,
+                                     (*hxlator_count));
+                (*hxlator_count)++;
+            }
+            add = _gf_false;
         }
 
-        ret = 0;
+        (*index)++;
+    }
+}
 
-out:
-        gf_log (THIS->name, GF_LOG_DEBUG, "Returning ret %d", ret);
-        return ret;
+int
+_select_hxlators_for_full_self_heal(xlator_t *this, glusterd_volinfo_t *volinfo,
+                                    dict_t *dict, int *index,
+                                    int *hxlator_count)
+{
+    glusterd_brickinfo_t *brickinfo = NULL;
+    int hxl_children = 0;
+    uuid_t candidate = {0};
+    int brick_index = 0;
+    glusterd_peerinfo_t *peerinfo = NULL;
+    int delta = 0;
+    uuid_t candidate_max = {0};
+
+    if ((*index) == 0)
+        (*index)++;
+    if (volinfo->type == GF_CLUSTER_TYPE_DISPERSE) {
+        hxl_children = volinfo->disperse_count;
+    } else {
+        hxl_children = volinfo->replica_count;
+    }
+
+    cds_list_for_each_entry(brickinfo, &volinfo->bricks, brick_list)
+    {
+        if (gf_uuid_compare(brickinfo->uuid, candidate_max) > 0) {
+            if (!gf_uuid_compare(MY_UUID, brickinfo->uuid)) {
+                gf_uuid_copy(candidate_max, brickinfo->uuid);
+            } else {
+                peerinfo = glusterd_peerinfo_find(brickinfo->uuid, NULL);
+                if (peerinfo && peerinfo->connected) {
+                    gf_uuid_copy(candidate_max, brickinfo->uuid);
+                }
+            }
+        }
+    }
+
+    cds_list_for_each_entry(brickinfo, &volinfo->bricks, brick_list)
+    {
+        if (gf_uuid_is_null(brickinfo->uuid))
+            (void)glusterd_resolve_brick(brickinfo);
+
+        delta %= hxl_children;
+        if ((*index + delta) == (brick_index + hxl_children)) {
+            if (!gf_uuid_compare(MY_UUID, brickinfo->uuid)) {
+                gf_uuid_copy(candidate, brickinfo->uuid);
+            } else {
+                peerinfo = glusterd_peerinfo_find(brickinfo->uuid, NULL);
+                if (peerinfo && peerinfo->connected) {
+                    gf_uuid_copy(candidate, brickinfo->uuid);
+                } else if (peerinfo &&
+                           (!gf_uuid_compare(candidate_max, MY_UUID))) {
+                    _add_hxlator_to_dict(dict, volinfo,
+                                         ((*index) - 1) / hxl_children,
+                                         (*hxlator_count));
+                    (*hxlator_count)++;
+                }
+            }
+
+            if (!gf_uuid_compare(MY_UUID, candidate)) {
+                _add_hxlator_to_dict(dict, volinfo,
+                                     ((*index) - 1) / hxl_children,
+                                     (*hxlator_count));
+                (*hxlator_count)++;
+            }
+            gf_uuid_clear(candidate);
+            brick_index += hxl_children;
+            delta++;
+        }
+
+        (*index)++;
+    }
+    return *hxlator_count;
 }
 
 static int
-fill_shd_status_for_local_bricks (dict_t *dict, glusterd_volinfo_t *volinfo,
-                                  cli_cmd_type type, dict_t *req_dict)
+glusterd_bricks_select_snap(dict_t *dict, char **op_errstr,
+                            struct cds_list_head *selected)
 {
-        glusterd_brickinfo_t    *brickinfo = NULL;
-        char                    msg[1024] = {0,};
-        char                    key[1024]  = {0,};
-        char                    value[1024] = {0,};
-        int                     index = 0;
-        int                     ret = 0;
-        xlator_t               *this = NULL;
-        int                     cmd_replica_index = -1;
-
-        this = THIS;
-        snprintf (msg, sizeof (msg), "self-heal-daemon is not running on");
-
-        if (type == PER_HEAL_XL) {
-                cmd_replica_index = get_replica_index_for_per_replica_cmd
-                                    (volinfo, req_dict);
-                if (cmd_replica_index == -1) {
-                        gf_log (THIS->name, GF_LOG_ERROR, "Could not find the "
-                                "replica index for per replica type command");
-                        ret = -1;
-                        goto out;
-                }
+    int ret = -1;
+    glusterd_conf_t *priv = NULL;
+    xlator_t *this = NULL;
+    glusterd_pending_node_t *pending_node = NULL;
+    glusterd_volinfo_t *volinfo = NULL;
+    char *volname = NULL;
+    glusterd_brickinfo_t *brickinfo = NULL;
+    int brick_index = -1;
+
+    this = THIS;
+    GF_ASSERT(this);
+    priv = this->private;
+    GF_ASSERT(priv);
+
+    ret = dict_get_strn(dict, "volname", SLEN("volname"), &volname);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+               "Unable to get"
+               " volname");
+        goto out;
+    }
+    ret = glusterd_volinfo_find(volname, &volinfo);
+    if (ret)
+        goto out;
+
+    cds_list_for_each_entry(brickinfo, &volinfo->bricks, brick_list)
+    {
+        brick_index++;
+        if (gf_uuid_compare(brickinfo->uuid, MY_UUID) ||
+            !glusterd_is_brick_started(brickinfo)) {
+            continue;
+        }
+        pending_node = GF_CALLOC(1, sizeof(*pending_node),
+                                 gf_gld_mt_pending_node_t);
+        if (!pending_node) {
+            ret = -1;
+            goto out;
         }
+        pending_node->node = brickinfo;
+        pending_node->type = GD_NODE_BRICK;
+        pending_node->index = brick_index;
+        cds_list_add_tail(&pending_node->list, selected);
+        pending_node = NULL;
+    }
 
-        cds_list_for_each_entry (brickinfo, &volinfo->bricks, brick_list) {
-                if (gf_uuid_is_null (brickinfo->uuid))
-                        (void)glusterd_resolve_brick (brickinfo);
-
-                if (gf_uuid_compare (MY_UUID, brickinfo->uuid)) {
-                        index++;
-                        continue;
-                }
-
-                if (type == PER_HEAL_XL) {
-                      if (cmd_replica_index != (index/volinfo->replica_count)) {
-                              index++;
-                              continue;
-                        }
-
-                }
-                snprintf (key, sizeof (key), "%d-status",index);
-                snprintf (value, sizeof (value), "%s %s",msg,
-                          uuid_utoa(MY_UUID));
-                ret = dict_set_dynstr (dict, key, gf_strdup(value));
-                if (ret) {
-                        gf_log (this->name, GF_LOG_ERROR, "Unable to"
-                                "set the dictionary for shd status msg");
-                        goto out;
-                }
-                snprintf (key, sizeof (key), "%d-shd-status",index);
-                ret = dict_set_str (dict, key, "off");
-                if (ret) {
-                        gf_log (this->name, GF_LOG_ERROR, "Unable to"
-                                " set dictionary for shd status msg");
-                        goto out;
-                }
-
-                index++;
-        }
+    ret = 0;
 
 out:
-        return ret;
-
+    gf_msg_debug(THIS->name, 0, "Returning ret %d", ret);
+    return ret;
 }
 
 static int
-glusterd_bricks_select_heal_volume (dict_t *dict, char **op_errstr,
-                                    struct cds_list_head *selected,
-                                    dict_t *rsp_dict)
+fill_shd_status_for_local_bricks(dict_t *dict, glusterd_volinfo_t *volinfo,
+                                 cli_cmd_type type, int *index,
+                                 dict_t *req_dict)
 {
-        int                                     ret = -1;
-        char                                    *volname = NULL;
-        glusterd_conf_t                         *priv = NULL;
-        glusterd_volinfo_t                      *volinfo = NULL;
-        xlator_t                                *this = NULL;
-        char                                    msg[2048] = {0,};
-        glusterd_pending_node_t                 *pending_node = NULL;
-        gf_xl_afr_op_t                          heal_op = GF_SHD_OP_INVALID;
-        int                                     hxlator_count = 0;
-
-        this = THIS;
-        GF_ASSERT (this);
-        priv = this->private;
-        GF_ASSERT (priv);
-
-        ret = dict_get_str (dict, "volname", &volname);
-        if (ret) {
-                gf_log ("glusterd", GF_LOG_ERROR, "volume name get failed");
-                goto out;
+    glusterd_brickinfo_t *brickinfo = NULL;
+    static char *msg = "self-heal-daemon is not running on";
+    char key[32] = {
+        0,
+    };
+    int keylen;
+    char value[128] = {
+        0,
+    };
+    int ret = 0;
+    xlator_t *this = NULL;
+    int cmd_replica_index = -1;
+
+    this = THIS;
+
+    if (type == PER_HEAL_XL) {
+        cmd_replica_index = get_replica_index_for_per_replica_cmd(volinfo,
+                                                                  req_dict);
+        if (cmd_replica_index == -1) {
+            gf_msg(THIS->name, GF_LOG_ERROR, 0, GD_MSG_REPLICA_INDEX_GET_FAIL,
+                   "Could not find the "
+                   "replica index for per replica type command");
+            ret = -1;
+            goto out;
+        }
+    }
+
+    cds_list_for_each_entry(brickinfo, &volinfo->bricks, brick_list)
+    {
+        if (gf_uuid_is_null(brickinfo->uuid))
+            (void)glusterd_resolve_brick(brickinfo);
+
+        if (gf_uuid_compare(MY_UUID, brickinfo->uuid)) {
+            (*index)++;
+            continue;
         }
 
-        ret = glusterd_volinfo_find (volname, &volinfo);
+        if (type == PER_HEAL_XL) {
+            if (cmd_replica_index != ((*index) / volinfo->replica_count)) {
+                (*index)++;
+                continue;
+            }
+        }
+        keylen = snprintf(key, sizeof(key), "%d-status", (*index));
+        snprintf(value, sizeof(value), "%s %s", msg, uuid_utoa(MY_UUID));
+        ret = dict_set_dynstrn(dict, key, keylen, gf_strdup(value));
         if (ret) {
-                snprintf (msg, sizeof (msg), "Volume %s does not exist",
-                          volname);
-
-                *op_errstr = gf_strdup (msg);
-                gf_log ("", GF_LOG_ERROR, "%s", msg);
-                goto out;
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+                   "Unable to"
+                   "set the dictionary for shd status msg");
+            goto out;
         }
-
-        ret = dict_get_int32 (dict, "heal-op", (int32_t*)&heal_op);
-        if (ret || (heal_op == GF_SHD_OP_INVALID)) {
-                gf_log ("glusterd", GF_LOG_ERROR, "heal op invalid");
-                goto out;
+        keylen = snprintf(key, sizeof(key), "%d-shd-status", (*index));
+        ret = dict_set_nstrn(dict, key, keylen, "off", SLEN("off"));
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+                   "Unable to"
+                   " set dictionary for shd status msg");
+            goto out;
         }
 
-        switch (heal_op) {
+        (*index)++;
+    }
+
+out:
+    return ret;
+}
+int
+glusterd_shd_select_brick_xlator(dict_t *dict, gf_xl_afr_op_t heal_op,
+                                 glusterd_volinfo_t *volinfo, int *index,
+                                 int *hxlator_count, dict_t *rsp_dict)
+{
+    int ret = -1;
+    glusterd_conf_t *priv = NULL;
+    xlator_t *this = NULL;
+    glusterd_svc_t *svc = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+    priv = this->private;
+    GF_ASSERT(priv);
+    svc = &(volinfo->shd.svc);
+
+    switch (heal_op) {
         case GF_SHD_OP_INDEX_SUMMARY:
         case GF_SHD_OP_STATISTICS_HEAL_COUNT:
-        if (!priv->shd_svc.online) {
+            if (!svc->online) {
                 if (!rsp_dict) {
-                        gf_log (this->name, GF_LOG_ERROR, "Received "
-                                "empty ctx.");
-                        goto out;
+                    gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_OPCTX_NULL,
+                           "Received "
+                           "empty ctx.");
+                    goto out;
                 }
 
-                ret = fill_shd_status_for_local_bricks (rsp_dict,
-                                                        volinfo,
-                                                        ALL_HEAL_XL,
-                                                        dict);
+                ret = fill_shd_status_for_local_bricks(
+                    rsp_dict, volinfo, ALL_HEAL_XL, index, dict);
                 if (ret)
-                        gf_log (this->name, GF_LOG_ERROR, "Unable to "
-                                "fill the shd status for the local "
-                                "bricks");
+                    gf_msg(this->name, GF_LOG_ERROR, 0,
+                           GD_MSG_SHD_STATUS_SET_FAIL,
+                           "Unable to "
+                           "fill the shd status for the local "
+                           "bricks");
                 goto out;
-        }
-        break;
+            }
+            break;
 
         case GF_SHD_OP_STATISTICS_HEAL_COUNT_PER_REPLICA:
-        if (!priv->shd_svc.online) {
+            if (!svc->online) {
                 if (!rsp_dict) {
-                        gf_log (this->name, GF_LOG_ERROR, "Received "
-                                "empty ctx.");
-                        goto out;
+                    gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_OPCTX_NULL,
+                           "Received "
+                           "empty ctx.");
+                    goto out;
                 }
-                ret = fill_shd_status_for_local_bricks (rsp_dict,
-                                                        volinfo,
-                                                        PER_HEAL_XL,
-                                                        dict);
+                ret = fill_shd_status_for_local_bricks(
+                    rsp_dict, volinfo, PER_HEAL_XL, index, dict);
                 if (ret)
-                        gf_log (this->name, GF_LOG_ERROR, "Unable to "
-                                "fill the shd status for the local"
-                                " bricks.");
+                    gf_msg(this->name, GF_LOG_ERROR, 0,
+                           GD_MSG_SHD_STATUS_SET_FAIL,
+                           "Unable to "
+                           "fill the shd status for the local"
+                           " bricks.");
                 goto out;
-
-        }
-        break;
+            }
+            break;
 
         default:
-                break;
-        }
-
+            break;
+    }
 
-        switch (heal_op) {
+    switch (heal_op) {
         case GF_SHD_OP_HEAL_FULL:
-                hxlator_count = _select_hxlators_for_full_self_heal (this,
-                                                                     volinfo,
-                                                                     dict);
-                break;
+            _select_hxlators_for_full_self_heal(this, volinfo, dict, index,
+                                                hxlator_count);
+            break;
         case GF_SHD_OP_STATISTICS_HEAL_COUNT_PER_REPLICA:
-                hxlator_count = _select_hxlator_with_matching_brick (this,
-                                                                     volinfo,
-                                                                     dict);
-                break;
+            (*hxlator_count) += _select_hxlator_with_matching_brick(
+                this, volinfo, dict, index);
+            break;
         default:
-                hxlator_count = _select_hxlators_with_local_bricks (this,
-                                                                    volinfo,
-                                                                    dict);
-                break;
-        }
-
-        if (!hxlator_count)
-                goto out;
-        if (hxlator_count == -1) {
-                gf_log (this->name, GF_LOG_ERROR, "Could not determine the"
-                        "translator count");
-                ret = -1;
-                goto out;
-        }
-
-        ret = dict_set_int32 (dict, "count", hxlator_count);
-        if (ret)
-                goto out;
-
-        pending_node = GF_CALLOC (1, sizeof (*pending_node),
-                                  gf_gld_mt_pending_node_t);
-        if (!pending_node) {
-                ret = -1;
-                goto out;
-        } else {
-                pending_node->node = &(priv->shd_svc);
-                pending_node->type = GD_NODE_SHD;
-                cds_list_add_tail (&pending_node->list, selected);
-                pending_node = NULL;
-        }
-
+            _select_hxlators_with_local_bricks(this, volinfo, dict, index,
+                                               hxlator_count);
+            break;
+    }
+    ret = (*hxlator_count);
 out:
-        gf_log (THIS->name, GF_LOG_DEBUG, "Returning ret %d", ret);
-        return ret;
-
+    return ret;
 }
 
 static int
-glusterd_bricks_select_rebalance_volume (dict_t *dict, char **op_errstr,
-                                         struct cds_list_head *selected)
+glusterd_bricks_select_heal_volume(dict_t *dict, char **op_errstr,
+                                   struct cds_list_head *selected,
+                                   dict_t *rsp_dict)
 {
-        int                                     ret = -1;
-        char                                    *volname = NULL;
-        glusterd_volinfo_t                      *volinfo = NULL;
-        xlator_t                                *this = NULL;
-        char                                    msg[2048] = {0,};
-        glusterd_pending_node_t                 *pending_node = NULL;
-
-        this = THIS;
-        GF_ASSERT (this);
-
-        ret = dict_get_str (dict, "volname", &volname);
-        if (ret) {
-                gf_log ("glusterd", GF_LOG_ERROR, "volume name get failed");
-                goto out;
-        }
-
-        ret = glusterd_volinfo_find (volname, &volinfo);
-        if (ret) {
-                snprintf (msg, sizeof (msg), "Volume %s does not exist",
-                          volname);
-
-                *op_errstr = gf_strdup (msg);
-                gf_log ("", GF_LOG_ERROR, "%s", msg);
-                goto out;
-        }
-        pending_node = GF_CALLOC (1, sizeof (*pending_node),
-                                  gf_gld_mt_pending_node_t);
-        if (!pending_node) {
-                ret = -1;
-                goto out;
-        } else {
-                pending_node->node = volinfo;
-                pending_node->type = GD_NODE_REBALANCE;
-                cds_list_add_tail (&pending_node->list, &opinfo.pending_bricks);
-                pending_node = NULL;
-        }
+    int ret = -1;
+    char *volname = NULL;
+    glusterd_conf_t *priv = NULL;
+    glusterd_volinfo_t *volinfo = NULL;
+    xlator_t *this = NULL;
+    char msg[2048] = {
+        0,
+    };
+    glusterd_pending_node_t *pending_node = NULL;
+    gf_xl_afr_op_t heal_op = GF_SHD_OP_INVALID;
+    int hxlator_count = 0;
+    int index = 0;
+
+    this = THIS;
+    GF_ASSERT(this);
+    priv = this->private;
+    GF_ASSERT(priv);
+
+    ret = dict_get_strn(dict, "volname", SLEN("volname"), &volname);
+    if (ret) {
+        gf_msg("glusterd", GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+               "volume name get failed");
+        goto out;
+    }
+
+    ret = glusterd_volinfo_find(volname, &volinfo);
+    if (ret) {
+        snprintf(msg, sizeof(msg), "Volume %s does not exist", volname);
+
+        *op_errstr = gf_strdup(msg);
+        gf_msg("glusterd", GF_LOG_ERROR, 0, GD_MSG_VOL_NOT_FOUND, "%s", msg);
+        goto out;
+    }
+
+    ret = dict_get_int32n(dict, "heal-op", SLEN("heal-op"),
+                          (int32_t *)&heal_op);
+    if (ret || (heal_op == GF_SHD_OP_INVALID)) {
+        gf_msg("glusterd", GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+               "heal op invalid");
+        goto out;
+    }
+    ret = glusterd_shd_select_brick_xlator(dict, heal_op, volinfo, &index,
+                                           &hxlator_count, rsp_dict);
+    if (ret < 0) {
+        goto out;
+    }
+
+    if (!hxlator_count)
+        goto out;
+    if (hxlator_count == -1) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_XLATOR_COUNT_GET_FAIL,
+               "Could not determine the"
+               "translator count");
+        ret = -1;
+        goto out;
+    }
+
+    ret = dict_set_int32n(dict, "count", SLEN("count"), hxlator_count);
+    if (ret)
+        goto out;
+    pending_node = GF_CALLOC(1, sizeof(*pending_node),
+                             gf_gld_mt_pending_node_t);
+    if (!pending_node) {
+        ret = -1;
+        goto out;
+    } else {
+        pending_node->node = &(volinfo->shd.svc);
+        pending_node->type = GD_NODE_SHD;
+        cds_list_add_tail(&pending_node->list, selected);
+        pending_node = NULL;
+    }
 
 out:
-        return ret;
+    gf_msg_debug(THIS->name, 0, "Returning ret %d", ret);
+    return ret;
 }
 
 static int
-glusterd_bricks_select_status_volume (dict_t *dict, char **op_errstr,
-                                      struct cds_list_head *selected)
+glusterd_bricks_select_rebalance_volume(dict_t *dict, char **op_errstr,
+                                        struct cds_list_head *selected)
 {
-        int                     ret = -1;
-        int                     cmd = 0;
-        int                     brick_index = -1;
-        char                    *volname = NULL;
-        char                    *brickname = NULL;
-        glusterd_volinfo_t      *volinfo = NULL;
-        glusterd_brickinfo_t    *brickinfo = NULL;
-        glusterd_pending_node_t *pending_node = NULL;
-        xlator_t                *this = NULL;
-        glusterd_conf_t         *priv = NULL;
-        glusterd_snapdsvc_t     *snapd = NULL;
-
-        GF_ASSERT (dict);
-
-        this = THIS;
-        GF_ASSERT (this);
-        priv = this->private;
-        GF_ASSERT (priv);
-
-        ret = dict_get_int32 (dict, "cmd", &cmd);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "Unable to get status type");
-                goto out;
-        }
+    int ret = -1;
+    char *volname = NULL;
+    glusterd_volinfo_t *volinfo = NULL;
+    xlator_t *this = NULL;
+    char msg[2048] = {
+        0,
+    };
+    glusterd_pending_node_t *pending_node = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+
+    ret = dict_get_strn(dict, "volname", SLEN("volname"), &volname);
+    if (ret) {
+        gf_msg("glusterd", GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+               "volume name get failed");
+        goto out;
+    }
+
+    ret = glusterd_volinfo_find(volname, &volinfo);
+    if (ret) {
+        snprintf(msg, sizeof(msg), "Volume %s does not exist", volname);
+
+        *op_errstr = gf_strdup(msg);
+        gf_msg("glusterd", GF_LOG_ERROR, 0, GD_MSG_VOL_NOT_FOUND, "%s", msg);
+        goto out;
+    }
+    pending_node = GF_CALLOC(1, sizeof(*pending_node),
+                             gf_gld_mt_pending_node_t);
+    if (!pending_node) {
+        ret = -1;
+        goto out;
+    } else {
+        pending_node->node = volinfo;
+        pending_node->type = GD_NODE_REBALANCE;
+        cds_list_add_tail(&pending_node->list, selected);
+        pending_node = NULL;
+    }
 
-        if (cmd & GF_CLI_STATUS_ALL)
-                goto out;
+out:
+    return ret;
+}
 
-        switch (cmd & GF_CLI_STATUS_MASK) {
+static int
+glusterd_bricks_select_status_volume(dict_t *dict, char **op_errstr,
+                                     struct cds_list_head *selected)
+{
+    int ret = -1;
+    int cmd = 0;
+    int brick_index = -1;
+    char *volname = NULL;
+    char *brickname = NULL;
+    glusterd_volinfo_t *volinfo = NULL;
+    glusterd_brickinfo_t *brickinfo = NULL;
+    glusterd_pending_node_t *pending_node = NULL;
+    xlator_t *this = NULL;
+    glusterd_conf_t *priv = NULL;
+    glusterd_svc_t *svc = NULL;
+
+    GF_ASSERT(dict);
+
+    this = THIS;
+    GF_ASSERT(this);
+    priv = this->private;
+    GF_ASSERT(priv);
+
+    ret = dict_get_int32n(dict, "cmd", SLEN("cmd"), &cmd);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+               "Unable to get status type");
+        goto out;
+    }
+
+    if (cmd & GF_CLI_STATUS_ALL)
+        goto out;
+
+    switch (cmd & GF_CLI_STATUS_MASK) {
         case GF_CLI_STATUS_MEM:
         case GF_CLI_STATUS_CLIENTS:
         case GF_CLI_STATUS_INODE:
@@ -5923,955 +7060,1105 @@ glusterd_bricks_select_status_volume (dict_t *dict, char **op_errstr,
         case GF_CLI_STATUS_SHD:
         case GF_CLI_STATUS_QUOTAD:
         case GF_CLI_STATUS_SNAPD:
-                break;
+        case GF_CLI_STATUS_BITD:
+        case GF_CLI_STATUS_SCRUB:
+        case GF_CLI_STATUS_CLIENT_LIST:
+            break;
         default:
-                goto out;
-        }
-        ret = dict_get_str (dict, "volname", &volname);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "Unable to get volname");
-                goto out;
-        }
-        ret = glusterd_volinfo_find (volname, &volinfo);
+            goto out;
+    }
+    ret = dict_get_strn(dict, "volname", SLEN("volname"), &volname);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+               "Unable to get volname");
+        goto out;
+    }
+    ret = glusterd_volinfo_find(volname, &volinfo);
+    if (ret) {
+        goto out;
+    }
+
+    if ((cmd & GF_CLI_STATUS_BRICK) != 0) {
+        ret = dict_get_strn(dict, "brick", SLEN("brick"), &brickname);
         if (ret) {
-                goto out;
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+                   "Unable to get brick");
+            goto out;
         }
+        ret = glusterd_volume_brickinfo_get_by_brick(brickname, volinfo,
+                                                     &brickinfo, _gf_false);
+        if (ret)
+            goto out;
 
-        if ( (cmd & GF_CLI_STATUS_BRICK) != 0) {
-                ret = dict_get_str (dict, "brick", &brickname);
-                if (ret) {
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "Unable to get brick");
-                        goto out;
-                }
-                ret = glusterd_volume_brickinfo_get_by_brick (brickname,
-                                                              volinfo,
-                                                              &brickinfo);
-                if (ret)
-                        goto out;
+        if (gf_uuid_compare(brickinfo->uuid, MY_UUID) ||
+            !glusterd_is_brick_started(brickinfo))
+            goto out;
 
-                if (gf_uuid_compare (brickinfo->uuid, MY_UUID)||
-                    !glusterd_is_brick_started (brickinfo))
-                        goto out;
+        pending_node = GF_CALLOC(1, sizeof(*pending_node),
+                                 gf_gld_mt_pending_node_t);
+        if (!pending_node) {
+            ret = -1;
+            goto out;
+        }
+        pending_node->node = brickinfo;
+        pending_node->type = GD_NODE_BRICK;
+        pending_node->index = 0;
+        cds_list_add_tail(&pending_node->list, selected);
 
-                pending_node = GF_CALLOC (1, sizeof (*pending_node),
-                                          gf_gld_mt_pending_node_t);
-                if (!pending_node) {
-                        ret = -1;
-                        goto out;
-                }
-                pending_node->node = brickinfo;
-                pending_node->type = GD_NODE_BRICK;
-                pending_node->index = 0;
-                cds_list_add_tail (&pending_node->list, selected);
+        ret = 0;
+#ifdef BUILD_GNFS
+    } else if ((cmd & GF_CLI_STATUS_NFS) != 0) {
+        if (!priv->nfs_svc.online) {
+            ret = -1;
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_NFS_SERVER_NOT_RUNNING,
+                   "NFS server is not running");
+            goto out;
+        }
+        pending_node = GF_CALLOC(1, sizeof(*pending_node),
+                                 gf_gld_mt_pending_node_t);
+        if (!pending_node) {
+            ret = -1;
+            goto out;
+        }
+        pending_node->node = &(priv->nfs_svc);
+        pending_node->type = GD_NODE_NFS;
+        pending_node->index = 0;
+        cds_list_add_tail(&pending_node->list, selected);
 
-                ret = 0;
-        } else if ((cmd & GF_CLI_STATUS_NFS) != 0) {
-                if (!priv->nfs_svc.online) {
-                        ret = -1;
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "NFS server is not running");
-                        goto out;
-                }
-                pending_node = GF_CALLOC (1, sizeof (*pending_node),
-                                          gf_gld_mt_pending_node_t);
-                if (!pending_node) {
-                        ret = -1;
-                        goto out;
-                }
-                pending_node->node = &(priv->nfs_svc);
-                pending_node->type = GD_NODE_NFS;
-                pending_node->index = 0;
-                cds_list_add_tail (&pending_node->list, selected);
+        ret = 0;
+#endif
+    } else if ((cmd & GF_CLI_STATUS_SHD) != 0) {
+        svc = &(volinfo->shd.svc);
+        if (!svc->online) {
+            ret = -1;
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_SELF_HEALD_DISABLED,
+                   "Self-heal daemon is not running");
+            goto out;
+        }
+        pending_node = GF_CALLOC(1, sizeof(*pending_node),
+                                 gf_gld_mt_pending_node_t);
+        if (!pending_node) {
+            ret = -1;
+            goto out;
+        }
+        pending_node->node = svc;
+        pending_node->type = GD_NODE_SHD;
+        pending_node->index = 0;
+        cds_list_add_tail(&pending_node->list, selected);
 
-                ret = 0;
-        } else if ((cmd & GF_CLI_STATUS_SHD) != 0) {
-                if (!priv->shd_svc.online) {
-                        ret = -1;
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "Self-heal daemon is not running");
-                        goto out;
-                }
-                pending_node = GF_CALLOC (1, sizeof (*pending_node),
-                                          gf_gld_mt_pending_node_t);
-                if (!pending_node) {
-                        ret = -1;
-                        goto out;
-                }
-                pending_node->node = &(priv->shd_svc);
-                pending_node->type = GD_NODE_SHD;
-                pending_node->index = 0;
-                cds_list_add_tail (&pending_node->list, selected);
+        ret = 0;
+    } else if ((cmd & GF_CLI_STATUS_QUOTAD) != 0) {
+        if (!priv->quotad_svc.online) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_QUOTAD_NOT_RUNNING,
+                   "Quotad is not "
+                   "running");
+            ret = -1;
+            goto out;
+        }
+        pending_node = GF_CALLOC(1, sizeof(*pending_node),
+                                 gf_gld_mt_pending_node_t);
+        if (!pending_node) {
+            ret = -1;
+            goto out;
+        }
+        pending_node->node = &(priv->quotad_svc);
+        pending_node->type = GD_NODE_QUOTAD;
+        pending_node->index = 0;
+        cds_list_add_tail(&pending_node->list, selected);
 
-                ret = 0;
-        } else if ((cmd & GF_CLI_STATUS_QUOTAD) != 0) {
-                if (!priv->quotad_svc.online) {
-                        gf_log (this->name, GF_LOG_ERROR, "Quotad is not "
-                                "running");
-                        ret = -1;
-                        goto out;
-                }
-                pending_node = GF_CALLOC (1, sizeof (*pending_node),
-                                          gf_gld_mt_pending_node_t);
-                if (!pending_node) {
-                        ret = -1;
-                        goto out;
-                }
-                pending_node->node = &(priv->quotad_svc);
-                pending_node->type = GD_NODE_QUOTAD;
-                pending_node->index = 0;
-                cds_list_add_tail (&pending_node->list, selected);
+        ret = 0;
+    } else if ((cmd & GF_CLI_STATUS_BITD) != 0) {
+        if (!priv->bitd_svc.online) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_BITROT_NOT_RUNNING,
+                   "Bitrot is not "
+                   "running");
+            ret = -1;
+            goto out;
+        }
+        pending_node = GF_CALLOC(1, sizeof(*pending_node),
+                                 gf_gld_mt_pending_node_t);
+        if (!pending_node) {
+            ret = -1;
+            goto out;
+        }
+        pending_node->node = &(priv->bitd_svc);
+        pending_node->type = GD_NODE_BITD;
+        pending_node->index = 0;
+        cds_list_add_tail(&pending_node->list, selected);
 
-                ret = 0;
-        } else if ((cmd & GF_CLI_STATUS_SNAPD) != 0) {
-                if (!volinfo->snapd.svc.online) {
-                        gf_log (this->name, GF_LOG_ERROR, "snapd is not "
-                                "running");
-                        ret = -1;
-                        goto out;
-                }
-                pending_node = GF_CALLOC (1, sizeof (*pending_node),
-                                          gf_gld_mt_pending_node_t);
-                if (!pending_node) {
-                        gf_log (this->name, GF_LOG_ERROR, "failed to allocate "
-                                "memory for pending node");
-                        ret = -1;
-                        goto out;
-                }
+        ret = 0;
+    } else if ((cmd & GF_CLI_STATUS_SCRUB) != 0) {
+        if (!priv->scrub_svc.online) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_SCRUBBER_NOT_RUNNING,
+                   "Scrubber is not "
+                   "running");
+            ret = -1;
+            goto out;
+        }
+        pending_node = GF_CALLOC(1, sizeof(*pending_node),
+                                 gf_gld_mt_pending_node_t);
+        if (!pending_node) {
+            ret = -1;
+            goto out;
+        }
+        pending_node->node = &(priv->scrub_svc);
+        pending_node->type = GD_NODE_SCRUB;
+        pending_node->index = 0;
+        cds_list_add_tail(&pending_node->list, selected);
 
-                pending_node->node = (void *)(&volinfo->snapd);
-                pending_node->type = GD_NODE_SNAPD;
-                pending_node->index = 0;
-                cds_list_add_tail (&pending_node->list, selected);
+        ret = 0;
+    } else if ((cmd & GF_CLI_STATUS_SNAPD) != 0) {
+        if (!volinfo->snapd.svc.online) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_SNAPD_NOT_RUNNING,
+                   "snapd is not "
+                   "running");
+            ret = -1;
+            goto out;
+        }
+        pending_node = GF_CALLOC(1, sizeof(*pending_node),
+                                 gf_gld_mt_pending_node_t);
+        if (!pending_node) {
+            gf_msg(this->name, GF_LOG_ERROR, ENOMEM, GD_MSG_NO_MEMORY,
+                   "failed to allocate "
+                   "memory for pending node");
+            ret = -1;
+            goto out;
+        }
 
-                ret = 0;
-        } else {
-                cds_list_for_each_entry (brickinfo, &volinfo->bricks,
-                                         brick_list) {
-                        brick_index++;
-                        if (gf_uuid_compare (brickinfo->uuid, MY_UUID) ||
-                            !glusterd_is_brick_started (brickinfo)) {
-                                continue;
-                        }
-                        pending_node = GF_CALLOC (1, sizeof (*pending_node),
-                                                  gf_gld_mt_pending_node_t);
-                        if (!pending_node) {
-                                ret = -1;
-                                gf_log (THIS->name ,GF_LOG_ERROR,
-                                        "Unable to allocate memory");
-                                goto out;
-                        }
-                        pending_node->node = brickinfo;
-                        pending_node->type = GD_NODE_BRICK;
-                        pending_node->index = brick_index;
-                        cds_list_add_tail (&pending_node->list, selected);
-                        pending_node = NULL;
-                }
+        pending_node->node = (void *)(&volinfo->snapd);
+        pending_node->type = GD_NODE_SNAPD;
+        pending_node->index = 0;
+        cds_list_add_tail(&pending_node->list, selected);
+
+        ret = 0;
+    } else {
+        cds_list_for_each_entry(brickinfo, &volinfo->bricks, brick_list)
+        {
+            brick_index++;
+            if (gf_uuid_compare(brickinfo->uuid, MY_UUID) ||
+                !glusterd_is_brick_started(brickinfo)) {
+                continue;
+            }
+            pending_node = GF_CALLOC(1, sizeof(*pending_node),
+                                     gf_gld_mt_pending_node_t);
+            if (!pending_node) {
+                ret = -1;
+                gf_msg(THIS->name, GF_LOG_ERROR, ENOMEM, GD_MSG_NO_MEMORY,
+                       "Unable to allocate memory");
+                goto out;
+            }
+            pending_node->node = brickinfo;
+            pending_node->type = GD_NODE_BRICK;
+            pending_node->index = brick_index;
+            cds_list_add_tail(&pending_node->list, selected);
+            pending_node = NULL;
         }
+    }
 out:
-        return ret;
+    return ret;
 }
 
+static int
+glusterd_bricks_select_scrub(dict_t *dict, char **op_errstr,
+                             struct cds_list_head *selected)
+{
+    int ret = -1;
+    char *volname = NULL;
+    char msg[2048] = {
+        0,
+    };
+    xlator_t *this = NULL;
+    glusterd_conf_t *priv = NULL;
+    glusterd_volinfo_t *volinfo = NULL;
+    glusterd_pending_node_t *pending_node = NULL;
+
+    this = THIS;
+    priv = this->private;
+    GF_ASSERT(this);
+    GF_ASSERT(priv);
+
+    GF_ASSERT(dict);
+
+    ret = dict_get_strn(dict, "volname", SLEN("volname"), &volname);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+               "Unable to get"
+               " volname");
+        goto out;
+    }
+
+    ret = glusterd_volinfo_find(volname, &volinfo);
+    if (ret) {
+        snprintf(msg, sizeof(msg), "Volume %s does not exist", volname);
+
+        *op_errstr = gf_strdup(msg);
+        gf_msg(this->name, GF_LOG_ERROR, EINVAL, GD_MSG_VOL_NOT_FOUND, "%s",
+               msg);
+        goto out;
+    }
+
+    if (!priv->scrub_svc.online) {
+        ret = 0;
+        snprintf(msg, sizeof(msg), "Scrubber daemon is not running");
+
+        gf_msg_debug(this->name, 0, "%s", msg);
+        goto out;
+    }
+
+    pending_node = GF_CALLOC(1, sizeof(*pending_node),
+                             gf_gld_mt_pending_node_t);
+    if (!pending_node) {
+        ret = -1;
+        goto out;
+    }
+
+    pending_node->node = &(priv->scrub_svc);
+    pending_node->type = GD_NODE_SCRUB;
+    cds_list_add_tail(&pending_node->list, selected);
+    pending_node = NULL;
+out:
+    gf_msg_debug(this->name, 0, "Returning %d", ret);
+    return ret;
+}
 /* Select the bricks to send the barrier request to.
  * This selects the bricks of the given volume which are present on this peer
  * and are running
  */
 static int
-glusterd_bricks_select_barrier (dict_t *dict, struct cds_list_head *selected)
+glusterd_bricks_select_barrier(dict_t *dict, struct cds_list_head *selected)
 {
-        int                       ret           = -1;
-        char                      *volname      = NULL;
-        glusterd_volinfo_t        *volinfo      = NULL;
-        glusterd_brickinfo_t      *brickinfo    = NULL;
-        glusterd_pending_node_t   *pending_node = NULL;
-
-        GF_ASSERT (dict);
-
-        ret = dict_get_str (dict, "volname", &volname);
-        if (ret) {
-                gf_log (THIS->name, GF_LOG_ERROR, "Failed to get volname");
-                goto out;
-        }
-
-        ret = glusterd_volinfo_find (volname, &volinfo);
-        if (ret) {
-                gf_log (THIS->name, GF_LOG_ERROR, "Failed to find volume %s",
-                        volname);
-                goto out;
-        }
-
-        cds_list_for_each_entry (brickinfo, &volinfo->bricks, brick_list) {
-                if (gf_uuid_compare (brickinfo->uuid, MY_UUID) ||
-                    !glusterd_is_brick_started (brickinfo)) {
-                        continue;
-                }
-                pending_node = GF_CALLOC (1, sizeof (*pending_node),
-                                gf_gld_mt_pending_node_t);
-                if (!pending_node) {
-                        ret = -1;
-                        goto out;
-                }
-                pending_node->node = brickinfo;
-                pending_node->type = GD_NODE_BRICK;
-                cds_list_add_tail (&pending_node->list, selected);
-                pending_node = NULL;
+    int ret = -1;
+    char *volname = NULL;
+    glusterd_volinfo_t *volinfo = NULL;
+    glusterd_brickinfo_t *brickinfo = NULL;
+    glusterd_pending_node_t *pending_node = NULL;
+
+    GF_ASSERT(dict);
+
+    ret = dict_get_strn(dict, "volname", SLEN("volname"), &volname);
+    if (ret) {
+        gf_msg(THIS->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+               "Failed to get volname");
+        goto out;
+    }
+
+    ret = glusterd_volinfo_find(volname, &volinfo);
+    if (ret) {
+        gf_msg(THIS->name, GF_LOG_ERROR, 0, GD_MSG_VOL_NOT_FOUND,
+               "Failed to find volume %s", volname);
+        goto out;
+    }
+
+    cds_list_for_each_entry(brickinfo, &volinfo->bricks, brick_list)
+    {
+        if (gf_uuid_compare(brickinfo->uuid, MY_UUID) ||
+            !glusterd_is_brick_started(brickinfo)) {
+            continue;
+        }
+        pending_node = GF_CALLOC(1, sizeof(*pending_node),
+                                 gf_gld_mt_pending_node_t);
+        if (!pending_node) {
+            ret = -1;
+            goto out;
         }
+        pending_node->node = brickinfo;
+        pending_node->type = GD_NODE_BRICK;
+        cds_list_add_tail(&pending_node->list, selected);
+        pending_node = NULL;
+    }
 
 out:
-        gf_log(THIS->name, GF_LOG_DEBUG, "Returning %d", ret);
-        return ret;
+    gf_msg_debug(THIS->name, 0, "Returning %d", ret);
+    return ret;
 }
 
 static int
-glusterd_op_ac_send_brick_op (glusterd_op_sm_event_t *event, void *ctx)
+glusterd_op_ac_send_brick_op(glusterd_op_sm_event_t *event, void *ctx)
 {
-        int                             ret = 0;
-        rpc_clnt_procedure_t            *proc = NULL;
-        glusterd_conf_t                 *priv = NULL;
-        xlator_t                        *this = NULL;
-        glusterd_op_t                   op = GD_OP_NONE;
-        glusterd_req_ctx_t              *req_ctx = NULL;
-        char                            *op_errstr = NULL;
-
-        this = THIS;
-        priv = this->private;
-
-        if (ctx) {
-                req_ctx = ctx;
-        } else {
-                req_ctx = GF_CALLOC (1, sizeof (*req_ctx),
-                                     gf_gld_mt_op_allack_ctx_t);
-                op = glusterd_op_get_op ();
-                req_ctx->op = op;
-                gf_uuid_copy (req_ctx->uuid, MY_UUID);
-                ret = glusterd_op_build_payload (&req_ctx->dict, &op_errstr,
-                                                 NULL);
-                if (ret) {
-                        gf_log (this->name, GF_LOG_ERROR, LOGSTR_BUILD_PAYLOAD,
-                                gd_op_list[op]);
-                        if (op_errstr == NULL)
-                                gf_asprintf (&op_errstr,
-                                             OPERRSTR_BUILD_PAYLOAD);
-                        opinfo.op_errstr = op_errstr;
-                        goto out;
-                }
-        }
-
-        proc = &priv->gfs_mgmt->proctable[GLUSTERD_BRICK_OP];
-        if (proc->fn) {
-                ret = proc->fn (NULL, this, req_ctx);
-                if (ret)
-                        goto out;
-        }
+    int ret = 0;
+    rpc_clnt_procedure_t *proc = NULL;
+    glusterd_conf_t *priv = NULL;
+    xlator_t *this = NULL;
+    glusterd_op_t op = GD_OP_NONE;
+    glusterd_req_ctx_t *req_ctx = NULL;
+    char *op_errstr = NULL;
+    gf_boolean_t free_req_ctx = _gf_false;
+
+    this = THIS;
+    priv = this->private;
+
+    if (ctx) {
+        req_ctx = ctx;
+    } else {
+        req_ctx = GF_CALLOC(1, sizeof(*req_ctx), gf_gld_mt_op_allack_ctx_t);
+        if (!req_ctx)
+            goto out;
+        free_req_ctx = _gf_true;
+        op = glusterd_op_get_op();
+        req_ctx->op = op;
+        gf_uuid_copy(req_ctx->uuid, MY_UUID);
+        ret = glusterd_op_build_payload(&req_ctx->dict, &op_errstr, NULL);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, 0,
+                   GD_MSG_BRICK_OP_PAYLOAD_BUILD_FAIL, LOGSTR_BUILD_PAYLOAD,
+                   gd_op_list[op]);
+            if (op_errstr == NULL)
+                gf_asprintf(&op_errstr, OPERRSTR_BUILD_PAYLOAD);
+            opinfo.op_errstr = op_errstr;
+            goto out;
+        }
+    }
+
+    proc = &priv->gfs_mgmt->proctable[GLUSTERD_BRICK_OP];
+    if (proc->fn) {
+        ret = proc->fn(NULL, this, req_ctx);
+        if (ret)
+            goto out;
+    }
 
-        if (!opinfo.pending_count && !opinfo.brick_pending_count) {
-                glusterd_clear_pending_nodes (&opinfo.pending_bricks);
-                ret = glusterd_op_sm_inject_event (GD_OP_EVENT_ALL_ACK,
-                                                   &event->txn_id, req_ctx);
-        }
+    if (!opinfo.pending_count && !opinfo.brick_pending_count) {
+        glusterd_clear_pending_nodes(&opinfo.pending_bricks);
+        ret = glusterd_op_sm_inject_event(GD_OP_EVENT_ALL_ACK, &event->txn_id,
+                                          req_ctx);
+    }
 
 out:
-        gf_log (this->name, GF_LOG_DEBUG, "Returning with %d", ret);
+    if (ret && free_req_ctx)
+        GF_FREE(req_ctx);
+    gf_msg_debug(this->name, 0, "Returning with %d", ret);
 
-        return ret;
+    return ret;
 }
 
-
 static int
-glusterd_op_ac_rcvd_brick_op_acc (glusterd_op_sm_event_t *event, void *ctx)
+glusterd_op_ac_rcvd_brick_op_acc(glusterd_op_sm_event_t *event, void *ctx)
 {
-        int                     ret = 0;
-        glusterd_op_brick_rsp_ctx_t *ev_ctx = NULL;
-        char                        *op_errstr = NULL;
-        glusterd_op_t               op = GD_OP_NONE;
-        gd_node_type                type = GD_NODE_NONE;
-        dict_t                      *op_ctx = NULL;
-        glusterd_req_ctx_t          *req_ctx = NULL;
-        void                        *pending_entry = NULL;
-        xlator_t                    *this = NULL;
-
-        this = THIS;
-        GF_ASSERT (this);
-        GF_ASSERT (event);
-        GF_ASSERT (ctx);
-        ev_ctx = ctx;
-
-        req_ctx = ev_ctx->commit_ctx;
-        GF_ASSERT (req_ctx);
-
-        op = req_ctx->op;
-        op_ctx = glusterd_op_get_ctx ();
-        pending_entry = ev_ctx->pending_node->node;
-        type = ev_ctx->pending_node->type;
-
-        ret = glusterd_remove_pending_entry (&opinfo.pending_bricks,
-                                             pending_entry);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "unknown response received ");
-                ret = -1;
-                goto out;
-        }
+    int ret = -1;
+    glusterd_op_brick_rsp_ctx_t *ev_ctx = NULL;
+    char *op_errstr = NULL;
+    glusterd_op_t op = GD_OP_NONE;
+    gd_node_type type = GD_NODE_NONE;
+    dict_t *op_ctx = NULL;
+    glusterd_req_ctx_t *req_ctx = NULL;
+    void *pending_entry = NULL;
+    xlator_t *this = NULL;
+
+    this = THIS;
+    GF_VALIDATE_OR_GOTO("glusterd", this, out);
+    GF_VALIDATE_OR_GOTO(this->name, event, out);
+    GF_VALIDATE_OR_GOTO(this->name, ctx, out);
+    ev_ctx = ctx;
+    GF_VALIDATE_OR_GOTO(this->name, ev_ctx, out);
+
+    req_ctx = ev_ctx->commit_ctx;
+    GF_VALIDATE_OR_GOTO(this->name, req_ctx, out);
+
+    op = req_ctx->op;
+    op_ctx = glusterd_op_get_ctx();
+    pending_entry = ev_ctx->pending_node->node;
+    type = ev_ctx->pending_node->type;
+
+    ret = glusterd_remove_pending_entry(&opinfo.pending_bricks, pending_entry);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_UNKNOWN_RESPONSE,
+               "unknown response received ");
+        ret = -1;
+        goto out;
+    }
 
-        if (opinfo.brick_pending_count > 0)
-                opinfo.brick_pending_count--;
+    if (opinfo.brick_pending_count > 0)
+        opinfo.brick_pending_count--;
 
-        glusterd_handle_node_rsp (req_ctx->dict, pending_entry, op, ev_ctx->rsp_dict,
-                                  op_ctx, &op_errstr, type);
+    ret = glusterd_set_txn_opinfo(&event->txn_id, &opinfo);
+    if (ret)
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_TRANS_OPINFO_SET_FAIL,
+               "Unable to set "
+               "transaction's opinfo");
 
-        if (opinfo.brick_pending_count > 0)
-                goto out;
+    glusterd_handle_node_rsp(req_ctx->dict, pending_entry, op, ev_ctx->rsp_dict,
+                             op_ctx, &op_errstr, type);
 
-        ret = glusterd_op_sm_inject_event (GD_OP_EVENT_ALL_ACK, &event->txn_id,
-                                           ev_ctx->commit_ctx);
+    if (opinfo.brick_pending_count > 0)
+        goto out;
 
-out:
-        if (ev_ctx->rsp_dict)
-                dict_unref (ev_ctx->rsp_dict);
-        GF_FREE (ev_ctx);
-        gf_log (this->name, GF_LOG_DEBUG, "Returning %d", ret);
+    ret = glusterd_op_sm_inject_event(GD_OP_EVENT_ALL_ACK, &event->txn_id,
+                                      ev_ctx->commit_ctx);
 
-        return ret;
+out:
+    if (ev_ctx && ev_ctx->rsp_dict)
+        dict_unref(ev_ctx->rsp_dict);
+    GF_FREE(ev_ctx);
+    gf_msg_debug(this ? this->name : "glusterd", 0, "Returning %d", ret);
+    return ret;
 }
 
 int32_t
-glusterd_op_bricks_select (glusterd_op_t op, dict_t *dict, char **op_errstr,
-                           struct cds_list_head *selected, dict_t *rsp_dict)
+glusterd_op_bricks_select(glusterd_op_t op, dict_t *dict, char **op_errstr,
+                          struct cds_list_head *selected, dict_t *rsp_dict)
 {
-        int     ret = 0;
+    int ret = 0;
 
-        GF_ASSERT (dict);
-        GF_ASSERT (op_errstr);
-        GF_ASSERT (op > GD_OP_NONE);
-        GF_ASSERT (op < GD_OP_MAX);
+    GF_ASSERT(dict);
+    GF_ASSERT(op_errstr);
+    GF_ASSERT(op > GD_OP_NONE);
+    GF_ASSERT(op < GD_OP_MAX);
 
-        switch (op) {
+    switch (op) {
         case GD_OP_STOP_VOLUME:
-                ret = glusterd_bricks_select_stop_volume (dict, op_errstr,
-                                                          selected);
-                break;
-
+            ret = glusterd_bricks_select_stop_volume(dict, op_errstr, selected);
+            break;
         case GD_OP_REMOVE_BRICK:
-                ret = glusterd_bricks_select_remove_brick (dict, op_errstr,
-                                                           selected);
-                break;
+            ret = glusterd_bricks_select_remove_brick(dict, op_errstr,
+                                                      selected);
+            break;
 
         case GD_OP_PROFILE_VOLUME:
-                ret = glusterd_bricks_select_profile_volume (dict, op_errstr,
-                                                             selected);
-                break;
+            ret = glusterd_bricks_select_profile_volume(dict, op_errstr,
+                                                        selected);
+            break;
 
         case GD_OP_HEAL_VOLUME:
-                ret = glusterd_bricks_select_heal_volume (dict, op_errstr,
-                                                          selected, rsp_dict);
-                break;
+            ret = glusterd_bricks_select_heal_volume(dict, op_errstr, selected,
+                                                     rsp_dict);
+            break;
 
         case GD_OP_STATUS_VOLUME:
-                ret = glusterd_bricks_select_status_volume (dict, op_errstr,
-                                                            selected);
-                break;
-
+            ret = glusterd_bricks_select_status_volume(dict, op_errstr,
+                                                       selected);
+            break;
         case GD_OP_DEFRAG_BRICK_VOLUME:
-                ret = glusterd_bricks_select_rebalance_volume (dict, op_errstr,
-                                                               selected);
-                break;
+            ret = glusterd_bricks_select_rebalance_volume(dict, op_errstr,
+                                                          selected);
+            break;
 
         case GD_OP_BARRIER:
-                ret = glusterd_bricks_select_barrier (dict, selected);
-                break;
+            ret = glusterd_bricks_select_barrier(dict, selected);
+            break;
         case GD_OP_SNAP:
-                ret = glusterd_bricks_select_snap (dict, op_errstr, selected);
-                break;
+            ret = glusterd_bricks_select_snap(dict, op_errstr, selected);
+            break;
+        case GD_OP_SCRUB_STATUS:
+        case GD_OP_SCRUB_ONDEMAND:
+            ret = glusterd_bricks_select_scrub(dict, op_errstr, selected);
+            break;
         default:
-                break;
-         }
+            break;
+    }
 
-        gf_log (THIS->name, GF_LOG_DEBUG, "Returning %d", ret);
+    gf_msg_debug(THIS->name, 0, "Returning %d", ret);
 
-        return ret;
+    return ret;
 }
 
-glusterd_op_sm_t glusterd_op_state_default [] = {
-        {GD_OP_STATE_DEFAULT, glusterd_op_ac_none}, //EVENT_NONE
-        {GD_OP_STATE_LOCK_SENT, glusterd_op_ac_send_lock},//EVENT_START_LOCK
-        {GD_OP_STATE_LOCKED, glusterd_op_ac_lock}, //EVENT_LOCK
-        {GD_OP_STATE_DEFAULT, glusterd_op_ac_none}, //EVENT_RCVD_ACC
-        {GD_OP_STATE_DEFAULT, glusterd_op_ac_none}, //EVENT_ALL_ACC
-        {GD_OP_STATE_DEFAULT, glusterd_op_ac_none}, //EVENT_STAGE_ACC
-        {GD_OP_STATE_DEFAULT, glusterd_op_ac_none}, //EVENT_COMMIT_ACC
-        {GD_OP_STATE_DEFAULT, glusterd_op_ac_none}, //EVENT_RCVD_RJT
-        {GD_OP_STATE_DEFAULT, glusterd_op_ac_none}, //EVENT_STAGE_OP
-        {GD_OP_STATE_DEFAULT, glusterd_op_ac_none}, //EVENT_COMMIT_OP
-        {GD_OP_STATE_DEFAULT, glusterd_op_ac_unlock}, //EVENT_UNLOCK
-        {GD_OP_STATE_DEFAULT, glusterd_op_ac_none}, //EVENT_START_UNLOCK
-        {GD_OP_STATE_DEFAULT, glusterd_op_ac_none}, //EVENT_ALL_ACK
-        {GD_OP_STATE_DEFAULT, glusterd_op_ac_none}, //EVENT_LOCAL_UNLOCK_NO_RESP
-        {GD_OP_STATE_DEFAULT, glusterd_op_ac_none}, //EVENT_MAX
+glusterd_op_sm_t glusterd_op_state_default[] = {
+    {GD_OP_STATE_DEFAULT, glusterd_op_ac_none},         // EVENT_NONE
+    {GD_OP_STATE_LOCK_SENT, glusterd_op_ac_send_lock},  // EVENT_START_LOCK
+    {GD_OP_STATE_LOCKED, glusterd_op_ac_lock},          // EVENT_LOCK
+    {GD_OP_STATE_DEFAULT, glusterd_op_ac_none},         // EVENT_RCVD_ACC
+    {GD_OP_STATE_DEFAULT, glusterd_op_ac_none},         // EVENT_ALL_ACC
+    {GD_OP_STATE_DEFAULT, glusterd_op_ac_none},         // EVENT_STAGE_ACC
+    {GD_OP_STATE_DEFAULT, glusterd_op_ac_none},         // EVENT_COMMIT_ACC
+    {GD_OP_STATE_DEFAULT, glusterd_op_ac_none},         // EVENT_RCVD_RJT
+    {GD_OP_STATE_DEFAULT, glusterd_op_ac_none},         // EVENT_STAGE_OP
+    {GD_OP_STATE_DEFAULT, glusterd_op_ac_none},         // EVENT_COMMIT_OP
+    {GD_OP_STATE_DEFAULT, glusterd_op_ac_unlock},       // EVENT_UNLOCK
+    {GD_OP_STATE_DEFAULT, glusterd_op_ac_none},         // EVENT_START_UNLOCK
+    {GD_OP_STATE_DEFAULT, glusterd_op_ac_none},         // EVENT_ALL_ACK
+    {GD_OP_STATE_DEFAULT, glusterd_op_ac_none},  // EVENT_LOCAL_UNLOCK_NO_RESP
+    {GD_OP_STATE_DEFAULT, glusterd_op_ac_none},  // EVENT_MAX
 };
 
-glusterd_op_sm_t glusterd_op_state_lock_sent [] = {
-        {GD_OP_STATE_LOCK_SENT, glusterd_op_ac_none}, //EVENT_NONE
-        {GD_OP_STATE_LOCK_SENT, glusterd_op_ac_none},//EVENT_START_LOCK
-        {GD_OP_STATE_LOCK_SENT, glusterd_op_ac_lock}, //EVENT_LOCK
-        {GD_OP_STATE_LOCK_SENT, glusterd_op_ac_rcvd_lock_acc}, //EVENT_RCVD_ACC
-        {GD_OP_STATE_STAGE_OP_SENT, glusterd_op_ac_send_stage_op}, //EVENT_ALL_ACC
-        {GD_OP_STATE_LOCK_SENT, glusterd_op_ac_none}, //EVENT_STAGE_ACC
-        {GD_OP_STATE_LOCK_SENT, glusterd_op_ac_none}, //EVENT_COMMIT_ACC
-        {GD_OP_STATE_ACK_DRAIN, glusterd_op_ac_send_unlock_drain}, //EVENT_RCVD_RJT
-        {GD_OP_STATE_LOCK_SENT, glusterd_op_ac_none}, //EVENT_STAGE_OP
-        {GD_OP_STATE_LOCK_SENT, glusterd_op_ac_none}, //EVENT_COMMIT_OP
-        {GD_OP_STATE_DEFAULT, glusterd_op_ac_unlock}, //EVENT_UNLOCK
-        {GD_OP_STATE_ACK_DRAIN, glusterd_op_ac_none}, //EVENT_START_UNLOCK
-        {GD_OP_STATE_LOCK_SENT, glusterd_op_ac_none}, //EVENT_ALL_ACK
-        {GD_OP_STATE_LOCK_SENT, glusterd_op_ac_none}, //EVENT_LOCAL_UNLOCK_NO_RESP
-        {GD_OP_STATE_LOCK_SENT, glusterd_op_ac_none}, //EVENT_MAX
+glusterd_op_sm_t glusterd_op_state_lock_sent[] = {
+    {GD_OP_STATE_LOCK_SENT, glusterd_op_ac_none},           // EVENT_NONE
+    {GD_OP_STATE_LOCK_SENT, glusterd_op_ac_none},           // EVENT_START_LOCK
+    {GD_OP_STATE_LOCK_SENT, glusterd_op_ac_lock},           // EVENT_LOCK
+    {GD_OP_STATE_LOCK_SENT, glusterd_op_ac_rcvd_lock_acc},  // EVENT_RCVD_ACC
+    {GD_OP_STATE_STAGE_OP_SENT, glusterd_op_ac_send_stage_op},  // EVENT_ALL_ACC
+    {GD_OP_STATE_LOCK_SENT, glusterd_op_ac_none},  // EVENT_STAGE_ACC
+    {GD_OP_STATE_LOCK_SENT, glusterd_op_ac_none},  // EVENT_COMMIT_ACC
+    {GD_OP_STATE_ACK_DRAIN,
+     glusterd_op_ac_send_unlock_drain},            // EVENT_RCVD_RJT
+    {GD_OP_STATE_LOCK_SENT, glusterd_op_ac_none},  // EVENT_STAGE_OP
+    {GD_OP_STATE_LOCK_SENT, glusterd_op_ac_none},  // EVENT_COMMIT_OP
+    {GD_OP_STATE_DEFAULT, glusterd_op_ac_unlock},  // EVENT_UNLOCK
+    {GD_OP_STATE_ACK_DRAIN, glusterd_op_ac_none},  // EVENT_START_UNLOCK
+    {GD_OP_STATE_LOCK_SENT, glusterd_op_ac_none},  // EVENT_ALL_ACK
+    {GD_OP_STATE_LOCK_SENT, glusterd_op_ac_none},  // EVENT_LOCAL_UNLOCK_NO_RESP
+    {GD_OP_STATE_LOCK_SENT, glusterd_op_ac_none},  // EVENT_MAX
 };
 
-glusterd_op_sm_t glusterd_op_state_locked [] = {
-        {GD_OP_STATE_LOCKED, glusterd_op_ac_none}, //EVENT_NONE
-        {GD_OP_STATE_LOCKED, glusterd_op_ac_none},//EVENT_START_LOCK
-        {GD_OP_STATE_LOCKED, glusterd_op_ac_lock}, //EVENT_LOCK
-        {GD_OP_STATE_LOCKED, glusterd_op_ac_none}, //EVENT_RCVD_ACC
-        {GD_OP_STATE_LOCKED, glusterd_op_ac_none}, //EVENT_ALL_ACC
-        {GD_OP_STATE_LOCKED, glusterd_op_ac_none}, //EVENT_STAGE_ACC
-        {GD_OP_STATE_LOCKED, glusterd_op_ac_none}, //EVENT_COMMIT_ACC
-        {GD_OP_STATE_LOCKED, glusterd_op_ac_none}, //EVENT_RCVD_RJT
-        {GD_OP_STATE_STAGED, glusterd_op_ac_stage_op}, //EVENT_STAGE_OP
-        {GD_OP_STATE_LOCKED, glusterd_op_ac_none}, //EVENT_COMMIT_OP
-        {GD_OP_STATE_DEFAULT, glusterd_op_ac_unlock}, //EVENT_UNLOCK
-        {GD_OP_STATE_LOCKED, glusterd_op_ac_none}, //EVENT_START_UNLOCK
-        {GD_OP_STATE_LOCKED, glusterd_op_ac_none}, //EVENT_ALL_ACK
-        {GD_OP_STATE_DEFAULT, glusterd_op_ac_local_unlock}, //EVENT_LOCAL_UNLOCK_NO_RESP
-        {GD_OP_STATE_LOCKED, glusterd_op_ac_none}, //EVENT_MAX
+glusterd_op_sm_t glusterd_op_state_locked[] = {
+    {GD_OP_STATE_LOCKED, glusterd_op_ac_none},      // EVENT_NONE
+    {GD_OP_STATE_LOCKED, glusterd_op_ac_none},      // EVENT_START_LOCK
+    {GD_OP_STATE_LOCKED, glusterd_op_ac_lock},      // EVENT_LOCK
+    {GD_OP_STATE_LOCKED, glusterd_op_ac_none},      // EVENT_RCVD_ACC
+    {GD_OP_STATE_LOCKED, glusterd_op_ac_none},      // EVENT_ALL_ACC
+    {GD_OP_STATE_LOCKED, glusterd_op_ac_none},      // EVENT_STAGE_ACC
+    {GD_OP_STATE_LOCKED, glusterd_op_ac_none},      // EVENT_COMMIT_ACC
+    {GD_OP_STATE_LOCKED, glusterd_op_ac_none},      // EVENT_RCVD_RJT
+    {GD_OP_STATE_STAGED, glusterd_op_ac_stage_op},  // EVENT_STAGE_OP
+    {GD_OP_STATE_LOCKED, glusterd_op_ac_none},      // EVENT_COMMIT_OP
+    {GD_OP_STATE_DEFAULT, glusterd_op_ac_unlock},   // EVENT_UNLOCK
+    {GD_OP_STATE_LOCKED, glusterd_op_ac_none},      // EVENT_START_UNLOCK
+    {GD_OP_STATE_LOCKED, glusterd_op_ac_none},      // EVENT_ALL_ACK
+    {GD_OP_STATE_DEFAULT,
+     glusterd_op_ac_local_unlock},              // EVENT_LOCAL_UNLOCK_NO_RESP
+    {GD_OP_STATE_LOCKED, glusterd_op_ac_none},  // EVENT_MAX
 };
 
-glusterd_op_sm_t glusterd_op_state_stage_op_sent [] = {
-        {GD_OP_STATE_STAGE_OP_SENT, glusterd_op_ac_none}, //EVENT_NONE
-        {GD_OP_STATE_STAGE_OP_SENT, glusterd_op_ac_none},//EVENT_START_LOCK
-        {GD_OP_STATE_STAGE_OP_SENT, glusterd_op_ac_lock}, //EVENT_LOCK
-        {GD_OP_STATE_STAGE_OP_SENT, glusterd_op_ac_rcvd_stage_op_acc}, //EVENT_RCVD_ACC
-        {GD_OP_STATE_BRICK_OP_SENT, glusterd_op_ac_send_brick_op}, //EVENT_ALL_ACC
-        {GD_OP_STATE_BRICK_OP_SENT, glusterd_op_ac_send_brick_op}, //EVENT_STAGE_ACC
-        {GD_OP_STATE_STAGE_OP_SENT, glusterd_op_ac_none}, //EVENT_COMMIT_ACC
-        {GD_OP_STATE_STAGE_OP_FAILED,   glusterd_op_ac_stage_op_failed}, //EVENT_RCVD_RJT
-        {GD_OP_STATE_STAGE_OP_SENT, glusterd_op_ac_none}, //EVENT_STAGE_OP
-        {GD_OP_STATE_STAGE_OP_SENT, glusterd_op_ac_none}, //EVENT_COMMIT_OP
-        {GD_OP_STATE_DEFAULT, glusterd_op_ac_unlock}, //EVENT_UNLOCK
-        {GD_OP_STATE_ACK_DRAIN, glusterd_op_ac_none}, //EVENT_START_UNLOCK
-        {GD_OP_STATE_STAGE_OP_SENT, glusterd_op_ac_none}, //EVENT_ALL_ACK
-        {GD_OP_STATE_STAGE_OP_SENT, glusterd_op_ac_none}, //EVENT_LOCAL_UNLOCK_NO_RESP
-        {GD_OP_STATE_STAGE_OP_SENT, glusterd_op_ac_none}, //EVENT_MAX
+glusterd_op_sm_t glusterd_op_state_stage_op_sent[] = {
+    {GD_OP_STATE_STAGE_OP_SENT, glusterd_op_ac_none},  // EVENT_NONE
+    {GD_OP_STATE_STAGE_OP_SENT, glusterd_op_ac_none},  // EVENT_START_LOCK
+    {GD_OP_STATE_STAGE_OP_SENT, glusterd_op_ac_lock},  // EVENT_LOCK
+    {GD_OP_STATE_STAGE_OP_SENT,
+     glusterd_op_ac_rcvd_stage_op_acc},  // EVENT_RCVD_ACC
+    {GD_OP_STATE_BRICK_OP_SENT, glusterd_op_ac_send_brick_op},  // EVENT_ALL_ACC
+    {GD_OP_STATE_BRICK_OP_SENT,
+     glusterd_op_ac_send_brick_op},                    // EVENT_STAGE_ACC
+    {GD_OP_STATE_STAGE_OP_SENT, glusterd_op_ac_none},  // EVENT_COMMIT_ACC
+    {GD_OP_STATE_STAGE_OP_FAILED,
+     glusterd_op_ac_stage_op_failed},                  // EVENT_RCVD_RJT
+    {GD_OP_STATE_STAGE_OP_SENT, glusterd_op_ac_none},  // EVENT_STAGE_OP
+    {GD_OP_STATE_STAGE_OP_SENT, glusterd_op_ac_none},  // EVENT_COMMIT_OP
+    {GD_OP_STATE_DEFAULT, glusterd_op_ac_unlock},      // EVENT_UNLOCK
+    {GD_OP_STATE_ACK_DRAIN, glusterd_op_ac_none},      // EVENT_START_UNLOCK
+    {GD_OP_STATE_STAGE_OP_SENT, glusterd_op_ac_none},  // EVENT_ALL_ACK
+    {GD_OP_STATE_STAGE_OP_SENT,
+     glusterd_op_ac_none},  // EVENT_LOCAL_UNLOCK_NO_RESP
+    {GD_OP_STATE_STAGE_OP_SENT, glusterd_op_ac_none},  // EVENT_MAX
 };
 
-glusterd_op_sm_t glusterd_op_state_stage_op_failed [] = {
-        {GD_OP_STATE_STAGE_OP_FAILED, glusterd_op_ac_none}, //EVENT_NONE
-        {GD_OP_STATE_STAGE_OP_FAILED, glusterd_op_ac_none},//EVENT_START_LOCK
-        {GD_OP_STATE_STAGE_OP_FAILED, glusterd_op_ac_lock}, //EVENT_LOCK
-        {GD_OP_STATE_STAGE_OP_FAILED, glusterd_op_ac_stage_op_failed}, //EVENT_RCVD_ACC
-        {GD_OP_STATE_STAGE_OP_FAILED, glusterd_op_ac_none}, //EVENT_ALL_ACC
-        {GD_OP_STATE_STAGE_OP_FAILED, glusterd_op_ac_none}, //EVENT_STAGE_ACC
-        {GD_OP_STATE_STAGE_OP_FAILED, glusterd_op_ac_none}, //EVENT_COMMIT_ACC
-        {GD_OP_STATE_STAGE_OP_FAILED, glusterd_op_ac_stage_op_failed}, //EVENT_RCVD_RJT
-        {GD_OP_STATE_STAGE_OP_FAILED, glusterd_op_ac_none}, //EVENT_STAGE_OP
-        {GD_OP_STATE_STAGE_OP_FAILED, glusterd_op_ac_none}, //EVENT_COMMIT_OP
-        {GD_OP_STATE_DEFAULT, glusterd_op_ac_unlock}, //EVENT_UNLOCK
-        {GD_OP_STATE_ACK_DRAIN, glusterd_op_ac_none}, //EVENT_START_UNLOCK
-        {GD_OP_STATE_UNLOCK_SENT, glusterd_op_ac_send_unlock}, //EVENT_ALL_ACK
-        {GD_OP_STATE_STAGE_OP_FAILED, glusterd_op_ac_none}, //EVENT_LOCAL_UNLOCK_NO_RESP
-        {GD_OP_STATE_STAGE_OP_FAILED, glusterd_op_ac_none}, //EVENT_MAX
+glusterd_op_sm_t glusterd_op_state_stage_op_failed[] = {
+    {GD_OP_STATE_STAGE_OP_FAILED, glusterd_op_ac_none},  // EVENT_NONE
+    {GD_OP_STATE_STAGE_OP_FAILED, glusterd_op_ac_none},  // EVENT_START_LOCK
+    {GD_OP_STATE_STAGE_OP_FAILED, glusterd_op_ac_lock},  // EVENT_LOCK
+    {GD_OP_STATE_STAGE_OP_FAILED,
+     glusterd_op_ac_stage_op_failed},                    // EVENT_RCVD_ACC
+    {GD_OP_STATE_STAGE_OP_FAILED, glusterd_op_ac_none},  // EVENT_ALL_ACC
+    {GD_OP_STATE_STAGE_OP_FAILED, glusterd_op_ac_none},  // EVENT_STAGE_ACC
+    {GD_OP_STATE_STAGE_OP_FAILED, glusterd_op_ac_none},  // EVENT_COMMIT_ACC
+    {GD_OP_STATE_STAGE_OP_FAILED,
+     glusterd_op_ac_stage_op_failed},                    // EVENT_RCVD_RJT
+    {GD_OP_STATE_STAGE_OP_FAILED, glusterd_op_ac_none},  // EVENT_STAGE_OP
+    {GD_OP_STATE_STAGE_OP_FAILED, glusterd_op_ac_none},  // EVENT_COMMIT_OP
+    {GD_OP_STATE_DEFAULT, glusterd_op_ac_unlock},        // EVENT_UNLOCK
+    {GD_OP_STATE_ACK_DRAIN, glusterd_op_ac_none},        // EVENT_START_UNLOCK
+    {GD_OP_STATE_UNLOCK_SENT, glusterd_op_ac_send_unlock},  // EVENT_ALL_ACK
+    {GD_OP_STATE_STAGE_OP_FAILED,
+     glusterd_op_ac_none},  // EVENT_LOCAL_UNLOCK_NO_RESP
+    {GD_OP_STATE_STAGE_OP_FAILED, glusterd_op_ac_none},  // EVENT_MAX
 };
 
-glusterd_op_sm_t glusterd_op_state_staged [] = {
-        {GD_OP_STATE_STAGED, glusterd_op_ac_none}, //EVENT_NONE
-        {GD_OP_STATE_STAGED, glusterd_op_ac_none},//EVENT_START_LOCK
-        {GD_OP_STATE_STAGED, glusterd_op_ac_lock}, //EVENT_LOCK
-        {GD_OP_STATE_STAGED, glusterd_op_ac_none}, //EVENT_RCVD_ACC
-        {GD_OP_STATE_STAGED, glusterd_op_ac_none}, //EVENT_ALL_ACC
-        {GD_OP_STATE_STAGED, glusterd_op_ac_none}, //EVENT_STAGE_ACC
-        {GD_OP_STATE_STAGED, glusterd_op_ac_none}, //EVENT_COMMIT_ACC
-        {GD_OP_STATE_STAGED, glusterd_op_ac_none}, //EVENT_RCVD_RJT
-        {GD_OP_STATE_STAGED, glusterd_op_ac_none}, //EVENT_STAGE_OP
-        {GD_OP_STATE_BRICK_COMMITTED, glusterd_op_ac_send_brick_op}, //EVENT_COMMIT_OP
-        {GD_OP_STATE_DEFAULT, glusterd_op_ac_unlock}, //EVENT_UNLOCK
-        {GD_OP_STATE_STAGED, glusterd_op_ac_none}, //EVENT_START_UNLOCK
-        {GD_OP_STATE_STAGED, glusterd_op_ac_none}, //EVENT_ALL_ACK
-        {GD_OP_STATE_DEFAULT, glusterd_op_ac_local_unlock}, //EVENT_LOCAL_UNLOCK_NO_RESP
-        {GD_OP_STATE_STAGED, glusterd_op_ac_none}, //EVENT_MAX
+glusterd_op_sm_t glusterd_op_state_staged[] = {
+    {GD_OP_STATE_STAGED, glusterd_op_ac_none},  // EVENT_NONE
+    {GD_OP_STATE_STAGED, glusterd_op_ac_none},  // EVENT_START_LOCK
+    {GD_OP_STATE_STAGED, glusterd_op_ac_lock},  // EVENT_LOCK
+    {GD_OP_STATE_STAGED, glusterd_op_ac_none},  // EVENT_RCVD_ACC
+    {GD_OP_STATE_STAGED, glusterd_op_ac_none},  // EVENT_ALL_ACC
+    {GD_OP_STATE_STAGED, glusterd_op_ac_none},  // EVENT_STAGE_ACC
+    {GD_OP_STATE_STAGED, glusterd_op_ac_none},  // EVENT_COMMIT_ACC
+    {GD_OP_STATE_STAGED, glusterd_op_ac_none},  // EVENT_RCVD_RJT
+    {GD_OP_STATE_STAGED, glusterd_op_ac_none},  // EVENT_STAGE_OP
+    {GD_OP_STATE_BRICK_COMMITTED,
+     glusterd_op_ac_send_brick_op},                // EVENT_COMMIT_OP
+    {GD_OP_STATE_DEFAULT, glusterd_op_ac_unlock},  // EVENT_UNLOCK
+    {GD_OP_STATE_STAGED, glusterd_op_ac_none},     // EVENT_START_UNLOCK
+    {GD_OP_STATE_STAGED, glusterd_op_ac_none},     // EVENT_ALL_ACK
+    {GD_OP_STATE_DEFAULT,
+     glusterd_op_ac_local_unlock},              // EVENT_LOCAL_UNLOCK_NO_RESP
+    {GD_OP_STATE_STAGED, glusterd_op_ac_none},  // EVENT_MAX
 };
 
-glusterd_op_sm_t glusterd_op_state_brick_op_sent [] = {
-        {GD_OP_STATE_BRICK_OP_SENT, glusterd_op_ac_none}, //EVENT_NONE
-        {GD_OP_STATE_BRICK_OP_SENT, glusterd_op_ac_none},//EVENT_START_LOCK
-        {GD_OP_STATE_BRICK_OP_SENT, glusterd_op_ac_lock}, //EVENT_LOCK
-        {GD_OP_STATE_BRICK_OP_SENT, glusterd_op_ac_rcvd_brick_op_acc}, //EVENT_RCVD_ACC
-        {GD_OP_STATE_BRICK_OP_SENT, glusterd_op_ac_none}, //EVENT_ALL_ACC
-        {GD_OP_STATE_BRICK_OP_SENT, glusterd_op_ac_none}, //EVENT_STAGE_ACC
-        {GD_OP_STATE_BRICK_OP_SENT, glusterd_op_ac_none}, //EVENT_COMMIT_ACC
-        {GD_OP_STATE_BRICK_OP_FAILED,   glusterd_op_ac_brick_op_failed}, //EVENT_RCVD_RJT
-        {GD_OP_STATE_BRICK_OP_SENT, glusterd_op_ac_none}, //EVENT_BRICK_OP
-        {GD_OP_STATE_BRICK_OP_SENT, glusterd_op_ac_none}, //EVENT_COMMIT_OP
-        {GD_OP_STATE_DEFAULT, glusterd_op_ac_unlock}, //EVENT_UNLOCK
-        {GD_OP_STATE_ACK_DRAIN, glusterd_op_ac_none}, //EVENT_START_UNLOCK
-        {GD_OP_STATE_COMMIT_OP_SENT, glusterd_op_ac_send_commit_op}, //EVENT_ALL_ACK
-        {GD_OP_STATE_BRICK_OP_SENT, glusterd_op_ac_none}, //EVENT_LOCAL_UNLOCK_NO_RESP
-        {GD_OP_STATE_BRICK_OP_SENT, glusterd_op_ac_none}, //EVENT_MAX
+glusterd_op_sm_t glusterd_op_state_brick_op_sent[] = {
+    {GD_OP_STATE_BRICK_OP_SENT, glusterd_op_ac_none},  // EVENT_NONE
+    {GD_OP_STATE_BRICK_OP_SENT, glusterd_op_ac_none},  // EVENT_START_LOCK
+    {GD_OP_STATE_BRICK_OP_SENT, glusterd_op_ac_lock},  // EVENT_LOCK
+    {GD_OP_STATE_BRICK_OP_SENT,
+     glusterd_op_ac_rcvd_brick_op_acc},                // EVENT_RCVD_ACC
+    {GD_OP_STATE_BRICK_OP_SENT, glusterd_op_ac_none},  // EVENT_ALL_ACC
+    {GD_OP_STATE_BRICK_OP_SENT, glusterd_op_ac_none},  // EVENT_STAGE_ACC
+    {GD_OP_STATE_BRICK_OP_SENT, glusterd_op_ac_none},  // EVENT_COMMIT_ACC
+    {GD_OP_STATE_BRICK_OP_FAILED,
+     glusterd_op_ac_brick_op_failed},                  // EVENT_RCVD_RJT
+    {GD_OP_STATE_BRICK_OP_SENT, glusterd_op_ac_none},  // EVENT_BRICK_OP
+    {GD_OP_STATE_BRICK_OP_SENT, glusterd_op_ac_none},  // EVENT_COMMIT_OP
+    {GD_OP_STATE_DEFAULT, glusterd_op_ac_unlock},      // EVENT_UNLOCK
+    {GD_OP_STATE_ACK_DRAIN, glusterd_op_ac_none},      // EVENT_START_UNLOCK
+    {GD_OP_STATE_COMMIT_OP_SENT,
+     glusterd_op_ac_send_commit_op},  // EVENT_ALL_ACK
+    {GD_OP_STATE_BRICK_OP_SENT,
+     glusterd_op_ac_none},  // EVENT_LOCAL_UNLOCK_NO_RESP
+    {GD_OP_STATE_BRICK_OP_SENT, glusterd_op_ac_none},  // EVENT_MAX
 };
 
-glusterd_op_sm_t glusterd_op_state_brick_op_failed [] = {
-        {GD_OP_STATE_BRICK_OP_FAILED, glusterd_op_ac_none}, //EVENT_NONE
-        {GD_OP_STATE_BRICK_OP_FAILED, glusterd_op_ac_none},//EVENT_START_LOCK
-        {GD_OP_STATE_BRICK_OP_FAILED, glusterd_op_ac_lock}, //EVENT_LOCK
-        {GD_OP_STATE_BRICK_OP_FAILED, glusterd_op_ac_brick_op_failed}, //EVENT_RCVD_ACC
-        {GD_OP_STATE_BRICK_OP_FAILED, glusterd_op_ac_none}, //EVENT_ALL_ACC
-        {GD_OP_STATE_BRICK_OP_FAILED, glusterd_op_ac_none}, //EVENT_STAGE_ACC
-        {GD_OP_STATE_BRICK_OP_FAILED, glusterd_op_ac_none}, //EVENT_COMMIT_ACC
-        {GD_OP_STATE_BRICK_OP_FAILED, glusterd_op_ac_brick_op_failed}, //EVENT_RCVD_RJT
-        {GD_OP_STATE_BRICK_OP_FAILED, glusterd_op_ac_none}, //EVENT_BRICK_OP
-        {GD_OP_STATE_BRICK_OP_FAILED, glusterd_op_ac_none}, //EVENT_COMMIT_OP
-        {GD_OP_STATE_DEFAULT, glusterd_op_ac_unlock}, //EVENT_UNLOCK
-        {GD_OP_STATE_ACK_DRAIN, glusterd_op_ac_none}, //EVENT_START_UNLOCK
-        {GD_OP_STATE_UNLOCK_SENT, glusterd_op_ac_send_unlock}, //EVENT_ALL_ACK
-        {GD_OP_STATE_BRICK_OP_FAILED, glusterd_op_ac_none}, //EVENT_LOCAL_UNLOCK_NO_RESP
-        {GD_OP_STATE_BRICK_OP_FAILED, glusterd_op_ac_none}, //EVENT_MAX
+glusterd_op_sm_t glusterd_op_state_brick_op_failed[] = {
+    {GD_OP_STATE_BRICK_OP_FAILED, glusterd_op_ac_none},  // EVENT_NONE
+    {GD_OP_STATE_BRICK_OP_FAILED, glusterd_op_ac_none},  // EVENT_START_LOCK
+    {GD_OP_STATE_BRICK_OP_FAILED, glusterd_op_ac_lock},  // EVENT_LOCK
+    {GD_OP_STATE_BRICK_OP_FAILED,
+     glusterd_op_ac_brick_op_failed},                    // EVENT_RCVD_ACC
+    {GD_OP_STATE_BRICK_OP_FAILED, glusterd_op_ac_none},  // EVENT_ALL_ACC
+    {GD_OP_STATE_BRICK_OP_FAILED, glusterd_op_ac_none},  // EVENT_STAGE_ACC
+    {GD_OP_STATE_BRICK_OP_FAILED, glusterd_op_ac_none},  // EVENT_COMMIT_ACC
+    {GD_OP_STATE_BRICK_OP_FAILED,
+     glusterd_op_ac_brick_op_failed},                    // EVENT_RCVD_RJT
+    {GD_OP_STATE_BRICK_OP_FAILED, glusterd_op_ac_none},  // EVENT_BRICK_OP
+    {GD_OP_STATE_BRICK_OP_FAILED, glusterd_op_ac_none},  // EVENT_COMMIT_OP
+    {GD_OP_STATE_DEFAULT, glusterd_op_ac_unlock},        // EVENT_UNLOCK
+    {GD_OP_STATE_ACK_DRAIN, glusterd_op_ac_none},        // EVENT_START_UNLOCK
+    {GD_OP_STATE_UNLOCK_SENT, glusterd_op_ac_send_unlock},  // EVENT_ALL_ACK
+    {GD_OP_STATE_BRICK_OP_FAILED,
+     glusterd_op_ac_none},  // EVENT_LOCAL_UNLOCK_NO_RESP
+    {GD_OP_STATE_BRICK_OP_FAILED, glusterd_op_ac_none},  // EVENT_MAX
 };
 
-glusterd_op_sm_t glusterd_op_state_brick_committed [] = {
-        {GD_OP_STATE_BRICK_COMMITTED, glusterd_op_ac_none}, //EVENT_NONE
-        {GD_OP_STATE_BRICK_COMMITTED, glusterd_op_ac_none},//EVENT_START_LOCK
-        {GD_OP_STATE_BRICK_COMMITTED, glusterd_op_ac_lock}, //EVENT_LOCK
-        {GD_OP_STATE_BRICK_COMMITTED, glusterd_op_ac_rcvd_brick_op_acc}, //EVENT_RCVD_ACC
-        {GD_OP_STATE_BRICK_COMMITTED, glusterd_op_ac_none}, //EVENT_ALL_ACC
-        {GD_OP_STATE_BRICK_COMMITTED, glusterd_op_ac_none}, //EVENT_STAGE_ACC
-        {GD_OP_STATE_BRICK_COMMITTED, glusterd_op_ac_none}, //EVENT_COMMIT_ACC
-        {GD_OP_STATE_BRICK_COMMIT_FAILED, glusterd_op_ac_brick_op_failed}, //EVENT_RCVD_RJT
-        {GD_OP_STATE_BRICK_COMMITTED, glusterd_op_ac_none}, //EVENT_STAGE_OP
-        {GD_OP_STATE_BRICK_COMMITTED, glusterd_op_ac_none}, //EVENT_COMMIT_OP
-        {GD_OP_STATE_DEFAULT, glusterd_op_ac_unlock}, //EVENT_UNLOCK
-        {GD_OP_STATE_BRICK_COMMITTED, glusterd_op_ac_none}, //EVENT_START_UNLOCK
-        {GD_OP_STATE_COMMITED, glusterd_op_ac_commit_op}, //EVENT_ALL_ACK
-        {GD_OP_STATE_DEFAULT, glusterd_op_ac_local_unlock}, //EVENT_LOCAL_UNLOCK_NO_RESP
-        {GD_OP_STATE_BRICK_COMMITTED, glusterd_op_ac_none}, //EVENT_MAX
+glusterd_op_sm_t glusterd_op_state_brick_committed[] = {
+    {GD_OP_STATE_BRICK_COMMITTED, glusterd_op_ac_none},  // EVENT_NONE
+    {GD_OP_STATE_BRICK_COMMITTED, glusterd_op_ac_none},  // EVENT_START_LOCK
+    {GD_OP_STATE_BRICK_COMMITTED, glusterd_op_ac_lock},  // EVENT_LOCK
+    {GD_OP_STATE_BRICK_COMMITTED,
+     glusterd_op_ac_rcvd_brick_op_acc},                  // EVENT_RCVD_ACC
+    {GD_OP_STATE_BRICK_COMMITTED, glusterd_op_ac_none},  // EVENT_ALL_ACC
+    {GD_OP_STATE_BRICK_COMMITTED, glusterd_op_ac_none},  // EVENT_STAGE_ACC
+    {GD_OP_STATE_BRICK_COMMITTED, glusterd_op_ac_none},  // EVENT_COMMIT_ACC
+    {GD_OP_STATE_BRICK_COMMIT_FAILED,
+     glusterd_op_ac_brick_op_failed},                    // EVENT_RCVD_RJT
+    {GD_OP_STATE_BRICK_COMMITTED, glusterd_op_ac_none},  // EVENT_STAGE_OP
+    {GD_OP_STATE_BRICK_COMMITTED, glusterd_op_ac_none},  // EVENT_COMMIT_OP
+    {GD_OP_STATE_DEFAULT, glusterd_op_ac_unlock},        // EVENT_UNLOCK
+    {GD_OP_STATE_BRICK_COMMITTED, glusterd_op_ac_none},  // EVENT_START_UNLOCK
+    {GD_OP_STATE_COMMITED, glusterd_op_ac_commit_op},    // EVENT_ALL_ACK
+    {GD_OP_STATE_DEFAULT,
+     glusterd_op_ac_local_unlock},  // EVENT_LOCAL_UNLOCK_NO_RESP
+    {GD_OP_STATE_BRICK_COMMITTED, glusterd_op_ac_none},  // EVENT_MAX
 };
 
-glusterd_op_sm_t glusterd_op_state_brick_commit_failed [] = {
-        {GD_OP_STATE_BRICK_COMMIT_FAILED, glusterd_op_ac_none}, //EVENT_NONE
-        {GD_OP_STATE_BRICK_COMMIT_FAILED, glusterd_op_ac_none},//EVENT_START_LOCK
-        {GD_OP_STATE_BRICK_COMMIT_FAILED, glusterd_op_ac_lock}, //EVENT_LOCK
-        {GD_OP_STATE_BRICK_COMMIT_FAILED, glusterd_op_ac_brick_op_failed}, //EVENT_RCVD_ACC
-        {GD_OP_STATE_BRICK_COMMIT_FAILED, glusterd_op_ac_none}, //EVENT_ALL_ACC
-        {GD_OP_STATE_BRICK_COMMIT_FAILED, glusterd_op_ac_none}, //EVENT_STAGE_ACC
-        {GD_OP_STATE_BRICK_COMMIT_FAILED, glusterd_op_ac_none}, //EVENT_COMMIT_ACC
-        {GD_OP_STATE_BRICK_COMMIT_FAILED, glusterd_op_ac_brick_op_failed}, //EVENT_RCVD_RJT
-        {GD_OP_STATE_BRICK_COMMIT_FAILED, glusterd_op_ac_none}, //EVENT_STAGE_OP
-        {GD_OP_STATE_BRICK_COMMIT_FAILED, glusterd_op_ac_none}, //EVENT_COMMIT_OP
-        {GD_OP_STATE_DEFAULT, glusterd_op_ac_unlock}, //EVENT_UNLOCK
-        {GD_OP_STATE_BRICK_COMMIT_FAILED, glusterd_op_ac_none}, //EVENT_START_UNLOCK
-        {GD_OP_STATE_BRICK_COMMIT_FAILED, glusterd_op_ac_send_commit_failed}, //EVENT_ALL_ACK
-        {GD_OP_STATE_DEFAULT, glusterd_op_ac_local_unlock}, //EVENT_LOCAL_UNLOCK_NO_RESP
-        {GD_OP_STATE_BRICK_COMMIT_FAILED, glusterd_op_ac_none}, //EVENT_MAX
+glusterd_op_sm_t glusterd_op_state_brick_commit_failed[] = {
+    {GD_OP_STATE_BRICK_COMMIT_FAILED, glusterd_op_ac_none},  // EVENT_NONE
+    {GD_OP_STATE_BRICK_COMMIT_FAILED, glusterd_op_ac_none},  // EVENT_START_LOCK
+    {GD_OP_STATE_BRICK_COMMIT_FAILED, glusterd_op_ac_lock},  // EVENT_LOCK
+    {GD_OP_STATE_BRICK_COMMIT_FAILED,
+     glusterd_op_ac_brick_op_failed},                        // EVENT_RCVD_ACC
+    {GD_OP_STATE_BRICK_COMMIT_FAILED, glusterd_op_ac_none},  // EVENT_ALL_ACC
+    {GD_OP_STATE_BRICK_COMMIT_FAILED, glusterd_op_ac_none},  // EVENT_STAGE_ACC
+    {GD_OP_STATE_BRICK_COMMIT_FAILED, glusterd_op_ac_none},  // EVENT_COMMIT_ACC
+    {GD_OP_STATE_BRICK_COMMIT_FAILED,
+     glusterd_op_ac_brick_op_failed},                        // EVENT_RCVD_RJT
+    {GD_OP_STATE_BRICK_COMMIT_FAILED, glusterd_op_ac_none},  // EVENT_STAGE_OP
+    {GD_OP_STATE_BRICK_COMMIT_FAILED, glusterd_op_ac_none},  // EVENT_COMMIT_OP
+    {GD_OP_STATE_DEFAULT, glusterd_op_ac_unlock},            // EVENT_UNLOCK
+    {GD_OP_STATE_BRICK_COMMIT_FAILED,
+     glusterd_op_ac_none},  // EVENT_START_UNLOCK
+    {GD_OP_STATE_BRICK_COMMIT_FAILED,
+     glusterd_op_ac_send_commit_failed},  // EVENT_ALL_ACK
+    {GD_OP_STATE_DEFAULT,
+     glusterd_op_ac_local_unlock},  // EVENT_LOCAL_UNLOCK_NO_RESP
+    {GD_OP_STATE_BRICK_COMMIT_FAILED, glusterd_op_ac_none},  // EVENT_MAX
 };
 
-glusterd_op_sm_t glusterd_op_state_commit_op_failed [] = {
-        {GD_OP_STATE_COMMIT_OP_FAILED, glusterd_op_ac_none}, //EVENT_NONE
-        {GD_OP_STATE_COMMIT_OP_FAILED, glusterd_op_ac_none},//EVENT_START_LOCK
-        {GD_OP_STATE_COMMIT_OP_FAILED, glusterd_op_ac_lock}, //EVENT_LOCK
-        {GD_OP_STATE_COMMIT_OP_FAILED, glusterd_op_ac_commit_op_failed}, //EVENT_RCVD_ACC
-        {GD_OP_STATE_COMMIT_OP_FAILED, glusterd_op_ac_none}, //EVENT_ALL_ACC
-        {GD_OP_STATE_COMMIT_OP_FAILED, glusterd_op_ac_none}, //EVENT_STAGE_ACC
-        {GD_OP_STATE_COMMIT_OP_FAILED, glusterd_op_ac_none}, //EVENT_COMMIT_ACC
-        {GD_OP_STATE_COMMIT_OP_FAILED, glusterd_op_ac_commit_op_failed}, //EVENT_RCVD_RJT
-        {GD_OP_STATE_COMMIT_OP_FAILED, glusterd_op_ac_none}, //EVENT_STAGE_OP
-        {GD_OP_STATE_COMMIT_OP_FAILED, glusterd_op_ac_none}, //EVENT_COMMIT_OP
-        {GD_OP_STATE_DEFAULT, glusterd_op_ac_unlock}, //EVENT_UNLOCK
-        {GD_OP_STATE_ACK_DRAIN, glusterd_op_ac_none}, //EVENT_START_UNLOCK
-        {GD_OP_STATE_UNLOCK_SENT, glusterd_op_ac_send_unlock}, //EVENT_ALL_ACK
-        {GD_OP_STATE_COMMIT_OP_FAILED, glusterd_op_ac_none}, //EVENT_LOCAL_UNLOCK_NO_RESP
-        {GD_OP_STATE_COMMIT_OP_FAILED, glusterd_op_ac_none}, //EVENT_MAX
+glusterd_op_sm_t glusterd_op_state_commit_op_failed[] = {
+    {GD_OP_STATE_COMMIT_OP_FAILED, glusterd_op_ac_none},  // EVENT_NONE
+    {GD_OP_STATE_COMMIT_OP_FAILED, glusterd_op_ac_none},  // EVENT_START_LOCK
+    {GD_OP_STATE_COMMIT_OP_FAILED, glusterd_op_ac_lock},  // EVENT_LOCK
+    {GD_OP_STATE_COMMIT_OP_FAILED,
+     glusterd_op_ac_commit_op_failed},                    // EVENT_RCVD_ACC
+    {GD_OP_STATE_COMMIT_OP_FAILED, glusterd_op_ac_none},  // EVENT_ALL_ACC
+    {GD_OP_STATE_COMMIT_OP_FAILED, glusterd_op_ac_none},  // EVENT_STAGE_ACC
+    {GD_OP_STATE_COMMIT_OP_FAILED, glusterd_op_ac_none},  // EVENT_COMMIT_ACC
+    {GD_OP_STATE_COMMIT_OP_FAILED,
+     glusterd_op_ac_commit_op_failed},                    // EVENT_RCVD_RJT
+    {GD_OP_STATE_COMMIT_OP_FAILED, glusterd_op_ac_none},  // EVENT_STAGE_OP
+    {GD_OP_STATE_COMMIT_OP_FAILED, glusterd_op_ac_none},  // EVENT_COMMIT_OP
+    {GD_OP_STATE_DEFAULT, glusterd_op_ac_unlock},         // EVENT_UNLOCK
+    {GD_OP_STATE_ACK_DRAIN, glusterd_op_ac_none},         // EVENT_START_UNLOCK
+    {GD_OP_STATE_UNLOCK_SENT, glusterd_op_ac_send_unlock},  // EVENT_ALL_ACK
+    {GD_OP_STATE_COMMIT_OP_FAILED,
+     glusterd_op_ac_none},  // EVENT_LOCAL_UNLOCK_NO_RESP
+    {GD_OP_STATE_COMMIT_OP_FAILED, glusterd_op_ac_none},  // EVENT_MAX
 };
 
-glusterd_op_sm_t glusterd_op_state_commit_op_sent [] = {
-        {GD_OP_STATE_COMMIT_OP_SENT, glusterd_op_ac_none}, //EVENT_NONE
-        {GD_OP_STATE_COMMIT_OP_SENT, glusterd_op_ac_none},//EVENT_START_LOCK
-        {GD_OP_STATE_COMMIT_OP_SENT, glusterd_op_ac_lock}, //EVENT_LOCK
-        {GD_OP_STATE_COMMIT_OP_SENT, glusterd_op_ac_rcvd_commit_op_acc}, //EVENT_RCVD_ACC
-        {GD_OP_STATE_UNLOCK_SENT,    glusterd_op_ac_send_unlock}, //EVENT_ALL_ACC
-        {GD_OP_STATE_COMMIT_OP_SENT, glusterd_op_ac_none}, //EVENT_STAGE_ACC
-        {GD_OP_STATE_UNLOCK_SENT,    glusterd_op_ac_send_unlock}, //EVENT_COMMIT_ACC
-        {GD_OP_STATE_COMMIT_OP_FAILED, glusterd_op_ac_commit_op_failed}, //EVENT_RCVD_RJT
-        {GD_OP_STATE_COMMIT_OP_SENT, glusterd_op_ac_none}, //EVENT_STAGE_OP
-        {GD_OP_STATE_COMMIT_OP_SENT, glusterd_op_ac_none}, //EVENT_COMMIT_OP
-        {GD_OP_STATE_DEFAULT,        glusterd_op_ac_unlock}, //EVENT_UNLOCK
-        {GD_OP_STATE_ACK_DRAIN, glusterd_op_ac_none}, //EVENT_START_UNLOCK
-        {GD_OP_STATE_COMMIT_OP_SENT, glusterd_op_ac_none}, //EVENT_ALL_ACK
-        {GD_OP_STATE_COMMIT_OP_SENT, glusterd_op_ac_none}, //EVENT_LOCAL_UNLOCK_NO_RESP
-        {GD_OP_STATE_COMMIT_OP_SENT, glusterd_op_ac_none}, //EVENT_MAX
+glusterd_op_sm_t glusterd_op_state_commit_op_sent[] = {
+    {GD_OP_STATE_COMMIT_OP_SENT, glusterd_op_ac_none},  // EVENT_NONE
+    {GD_OP_STATE_COMMIT_OP_SENT, glusterd_op_ac_none},  // EVENT_START_LOCK
+    {GD_OP_STATE_COMMIT_OP_SENT, glusterd_op_ac_lock},  // EVENT_LOCK
+    {GD_OP_STATE_COMMIT_OP_SENT,
+     glusterd_op_ac_rcvd_commit_op_acc},                    // EVENT_RCVD_ACC
+    {GD_OP_STATE_UNLOCK_SENT, glusterd_op_ac_send_unlock},  // EVENT_ALL_ACC
+    {GD_OP_STATE_COMMIT_OP_SENT, glusterd_op_ac_none},      // EVENT_STAGE_ACC
+    {GD_OP_STATE_UNLOCK_SENT, glusterd_op_ac_send_unlock},  // EVENT_COMMIT_ACC
+    {GD_OP_STATE_COMMIT_OP_FAILED,
+     glusterd_op_ac_commit_op_failed},                  // EVENT_RCVD_RJT
+    {GD_OP_STATE_COMMIT_OP_SENT, glusterd_op_ac_none},  // EVENT_STAGE_OP
+    {GD_OP_STATE_COMMIT_OP_SENT, glusterd_op_ac_none},  // EVENT_COMMIT_OP
+    {GD_OP_STATE_DEFAULT, glusterd_op_ac_unlock},       // EVENT_UNLOCK
+    {GD_OP_STATE_ACK_DRAIN, glusterd_op_ac_none},       // EVENT_START_UNLOCK
+    {GD_OP_STATE_COMMIT_OP_SENT, glusterd_op_ac_none},  // EVENT_ALL_ACK
+    {GD_OP_STATE_COMMIT_OP_SENT,
+     glusterd_op_ac_none},  // EVENT_LOCAL_UNLOCK_NO_RESP
+    {GD_OP_STATE_COMMIT_OP_SENT, glusterd_op_ac_none},  // EVENT_MAX
 };
 
-glusterd_op_sm_t glusterd_op_state_committed [] = {
-        {GD_OP_STATE_COMMITED, glusterd_op_ac_none}, //EVENT_NONE
-        {GD_OP_STATE_COMMITED, glusterd_op_ac_none},//EVENT_START_LOCK
-        {GD_OP_STATE_COMMITED, glusterd_op_ac_lock}, //EVENT_LOCK
-        {GD_OP_STATE_COMMITED, glusterd_op_ac_none}, //EVENT_RCVD_ACC
-        {GD_OP_STATE_COMMITED, glusterd_op_ac_none}, //EVENT_ALL_ACC
-        {GD_OP_STATE_COMMITED, glusterd_op_ac_none}, //EVENT_STAGE_ACC
-        {GD_OP_STATE_COMMITED, glusterd_op_ac_none}, //EVENT_COMMIT_ACC
-        {GD_OP_STATE_COMMITED, glusterd_op_ac_none}, //EVENT_RCVD_RJT
-        {GD_OP_STATE_COMMITED, glusterd_op_ac_none}, //EVENT_STAGE_OP
-        {GD_OP_STATE_COMMITED, glusterd_op_ac_none}, //EVENT_COMMIT_OP
-        {GD_OP_STATE_DEFAULT, glusterd_op_ac_unlock}, //EVENT_UNLOCK
-        {GD_OP_STATE_COMMITED, glusterd_op_ac_none}, //EVENT_START_UNLOCK
-        {GD_OP_STATE_COMMITED, glusterd_op_ac_none}, //EVENT_ALL_ACK
-        {GD_OP_STATE_DEFAULT, glusterd_op_ac_local_unlock}, //EVENT_LOCAL_UNLOCK_NO_RESP
-        {GD_OP_STATE_COMMITED, glusterd_op_ac_none}, //EVENT_MAX
+glusterd_op_sm_t glusterd_op_state_committed[] = {
+    {GD_OP_STATE_COMMITED, glusterd_op_ac_none},   // EVENT_NONE
+    {GD_OP_STATE_COMMITED, glusterd_op_ac_none},   // EVENT_START_LOCK
+    {GD_OP_STATE_COMMITED, glusterd_op_ac_lock},   // EVENT_LOCK
+    {GD_OP_STATE_COMMITED, glusterd_op_ac_none},   // EVENT_RCVD_ACC
+    {GD_OP_STATE_COMMITED, glusterd_op_ac_none},   // EVENT_ALL_ACC
+    {GD_OP_STATE_COMMITED, glusterd_op_ac_none},   // EVENT_STAGE_ACC
+    {GD_OP_STATE_COMMITED, glusterd_op_ac_none},   // EVENT_COMMIT_ACC
+    {GD_OP_STATE_COMMITED, glusterd_op_ac_none},   // EVENT_RCVD_RJT
+    {GD_OP_STATE_COMMITED, glusterd_op_ac_none},   // EVENT_STAGE_OP
+    {GD_OP_STATE_COMMITED, glusterd_op_ac_none},   // EVENT_COMMIT_OP
+    {GD_OP_STATE_DEFAULT, glusterd_op_ac_unlock},  // EVENT_UNLOCK
+    {GD_OP_STATE_COMMITED, glusterd_op_ac_none},   // EVENT_START_UNLOCK
+    {GD_OP_STATE_COMMITED, glusterd_op_ac_none},   // EVENT_ALL_ACK
+    {GD_OP_STATE_DEFAULT,
+     glusterd_op_ac_local_unlock},                // EVENT_LOCAL_UNLOCK_NO_RESP
+    {GD_OP_STATE_COMMITED, glusterd_op_ac_none},  // EVENT_MAX
 };
 
-glusterd_op_sm_t glusterd_op_state_unlock_sent [] = {
-        {GD_OP_STATE_UNLOCK_SENT, glusterd_op_ac_none}, //EVENT_NONE
-        {GD_OP_STATE_UNLOCK_SENT, glusterd_op_ac_none},//EVENT_START_LOCK
-        {GD_OP_STATE_UNLOCK_SENT, glusterd_op_ac_lock}, //EVENT_LOCK
-        {GD_OP_STATE_UNLOCK_SENT, glusterd_op_ac_rcvd_unlock_acc}, //EVENT_RCVD_ACC
-        {GD_OP_STATE_DEFAULT, glusterd_op_ac_unlocked_all}, //EVENT_ALL_ACC
-        {GD_OP_STATE_UNLOCK_SENT, glusterd_op_ac_none}, //EVENT_STAGE_ACC
-        {GD_OP_STATE_UNLOCK_SENT, glusterd_op_ac_none}, //EVENT_COMMIT_ACC
-        {GD_OP_STATE_UNLOCK_SENT, glusterd_op_ac_rcvd_unlock_acc}, //EVENT_RCVD_RJT
-        {GD_OP_STATE_UNLOCK_SENT, glusterd_op_ac_none}, //EVENT_STAGE_OP
-        {GD_OP_STATE_UNLOCK_SENT, glusterd_op_ac_none}, //EVENT_COMMIT_OP
-        {GD_OP_STATE_DEFAULT, glusterd_op_ac_unlock}, //EVENT_UNLOCK
-        {GD_OP_STATE_ACK_DRAIN, glusterd_op_ac_none}, //EVENT_START_UNLOCK
-        {GD_OP_STATE_UNLOCK_SENT, glusterd_op_ac_none}, //EVENT_ALL_ACK
-        {GD_OP_STATE_UNLOCK_SENT, glusterd_op_ac_none}, //EVENT_LOCAL_UNLOCK_NO_RESP
-        {GD_OP_STATE_UNLOCK_SENT, glusterd_op_ac_none}, //EVENT_MAX
+glusterd_op_sm_t glusterd_op_state_unlock_sent[] = {
+    {GD_OP_STATE_UNLOCK_SENT, glusterd_op_ac_none},  // EVENT_NONE
+    {GD_OP_STATE_UNLOCK_SENT, glusterd_op_ac_none},  // EVENT_START_LOCK
+    {GD_OP_STATE_UNLOCK_SENT, glusterd_op_ac_lock},  // EVENT_LOCK
+    {GD_OP_STATE_UNLOCK_SENT,
+     glusterd_op_ac_rcvd_unlock_acc},                    // EVENT_RCVD_ACC
+    {GD_OP_STATE_DEFAULT, glusterd_op_ac_unlocked_all},  // EVENT_ALL_ACC
+    {GD_OP_STATE_UNLOCK_SENT, glusterd_op_ac_none},      // EVENT_STAGE_ACC
+    {GD_OP_STATE_UNLOCK_SENT, glusterd_op_ac_none},      // EVENT_COMMIT_ACC
+    {GD_OP_STATE_UNLOCK_SENT,
+     glusterd_op_ac_rcvd_unlock_acc},                // EVENT_RCVD_RJT
+    {GD_OP_STATE_UNLOCK_SENT, glusterd_op_ac_none},  // EVENT_STAGE_OP
+    {GD_OP_STATE_UNLOCK_SENT, glusterd_op_ac_none},  // EVENT_COMMIT_OP
+    {GD_OP_STATE_DEFAULT, glusterd_op_ac_unlock},    // EVENT_UNLOCK
+    {GD_OP_STATE_ACK_DRAIN, glusterd_op_ac_none},    // EVENT_START_UNLOCK
+    {GD_OP_STATE_UNLOCK_SENT, glusterd_op_ac_none},  // EVENT_ALL_ACK
+    {GD_OP_STATE_UNLOCK_SENT,
+     glusterd_op_ac_none},  // EVENT_LOCAL_UNLOCK_NO_RESP
+    {GD_OP_STATE_UNLOCK_SENT, glusterd_op_ac_none},  // EVENT_MAX
 };
 
-glusterd_op_sm_t glusterd_op_state_ack_drain [] = {
-        {GD_OP_STATE_ACK_DRAIN, glusterd_op_ac_none}, //EVENT_NONE
-        {GD_OP_STATE_ACK_DRAIN, glusterd_op_ac_none},//EVENT_START_LOCK
-        {GD_OP_STATE_ACK_DRAIN, glusterd_op_ac_lock}, //EVENT_LOCK
-        {GD_OP_STATE_ACK_DRAIN, glusterd_op_ac_send_unlock_drain}, //EVENT_RCVD_ACC
-        {GD_OP_STATE_ACK_DRAIN, glusterd_op_ac_none}, //EVENT_ALL_ACC
-        {GD_OP_STATE_ACK_DRAIN, glusterd_op_ac_none}, //EVENT_STAGE_ACC
-        {GD_OP_STATE_ACK_DRAIN, glusterd_op_ac_none}, //EVENT_COMMIT_ACC
-        {GD_OP_STATE_ACK_DRAIN, glusterd_op_ac_send_unlock_drain}, //EVENT_RCVD_RJT
-        {GD_OP_STATE_ACK_DRAIN, glusterd_op_ac_none}, //EVENT_STAGE_OP
-        {GD_OP_STATE_ACK_DRAIN, glusterd_op_ac_none}, //EVENT_COMMIT_OP
-        {GD_OP_STATE_DEFAULT, glusterd_op_ac_unlock}, //EVENT_UNLOCK
-        {GD_OP_STATE_ACK_DRAIN, glusterd_op_ac_none}, //EVENT_START_UNLOCK
-        {GD_OP_STATE_UNLOCK_SENT, glusterd_op_ac_send_unlock}, //EVENT_ALL_ACK
-        {GD_OP_STATE_ACK_DRAIN, glusterd_op_ac_none}, //EVENT_LOCAL_UNLOCK_NO_RESP
-        {GD_OP_STATE_ACK_DRAIN, glusterd_op_ac_none}, //EVENT_MAX
+glusterd_op_sm_t glusterd_op_state_ack_drain[] = {
+    {GD_OP_STATE_ACK_DRAIN, glusterd_op_ac_none},  // EVENT_NONE
+    {GD_OP_STATE_ACK_DRAIN, glusterd_op_ac_none},  // EVENT_START_LOCK
+    {GD_OP_STATE_ACK_DRAIN, glusterd_op_ac_lock},  // EVENT_LOCK
+    {GD_OP_STATE_ACK_DRAIN,
+     glusterd_op_ac_send_unlock_drain},            // EVENT_RCVD_ACC
+    {GD_OP_STATE_ACK_DRAIN, glusterd_op_ac_none},  // EVENT_ALL_ACC
+    {GD_OP_STATE_ACK_DRAIN, glusterd_op_ac_none},  // EVENT_STAGE_ACC
+    {GD_OP_STATE_ACK_DRAIN, glusterd_op_ac_none},  // EVENT_COMMIT_ACC
+    {GD_OP_STATE_ACK_DRAIN,
+     glusterd_op_ac_send_unlock_drain},            // EVENT_RCVD_RJT
+    {GD_OP_STATE_ACK_DRAIN, glusterd_op_ac_none},  // EVENT_STAGE_OP
+    {GD_OP_STATE_ACK_DRAIN, glusterd_op_ac_none},  // EVENT_COMMIT_OP
+    {GD_OP_STATE_DEFAULT, glusterd_op_ac_unlock},  // EVENT_UNLOCK
+    {GD_OP_STATE_ACK_DRAIN, glusterd_op_ac_none},  // EVENT_START_UNLOCK
+    {GD_OP_STATE_UNLOCK_SENT, glusterd_op_ac_send_unlock},  // EVENT_ALL_ACK
+    {GD_OP_STATE_ACK_DRAIN, glusterd_op_ac_none},  // EVENT_LOCAL_UNLOCK_NO_RESP
+    {GD_OP_STATE_ACK_DRAIN, glusterd_op_ac_none},  // EVENT_MAX
 };
 
-glusterd_op_sm_t *glusterd_op_state_table [] = {
-        glusterd_op_state_default,
-        glusterd_op_state_lock_sent,
-        glusterd_op_state_locked,
-        glusterd_op_state_stage_op_sent,
-        glusterd_op_state_staged,
-        glusterd_op_state_commit_op_sent,
-        glusterd_op_state_committed,
-        glusterd_op_state_unlock_sent,
-        glusterd_op_state_stage_op_failed,
-        glusterd_op_state_commit_op_failed,
-        glusterd_op_state_brick_op_sent,
-        glusterd_op_state_brick_op_failed,
-        glusterd_op_state_brick_committed,
-        glusterd_op_state_brick_commit_failed,
-        glusterd_op_state_ack_drain
-};
+glusterd_op_sm_t *glusterd_op_state_table[] = {
+    glusterd_op_state_default,         glusterd_op_state_lock_sent,
+    glusterd_op_state_locked,          glusterd_op_state_stage_op_sent,
+    glusterd_op_state_staged,          glusterd_op_state_commit_op_sent,
+    glusterd_op_state_committed,       glusterd_op_state_unlock_sent,
+    glusterd_op_state_stage_op_failed, glusterd_op_state_commit_op_failed,
+    glusterd_op_state_brick_op_sent,   glusterd_op_state_brick_op_failed,
+    glusterd_op_state_brick_committed, glusterd_op_state_brick_commit_failed,
+    glusterd_op_state_ack_drain};
 
 int
-glusterd_op_sm_new_event (glusterd_op_sm_event_type_t event_type,
-                          glusterd_op_sm_event_t **new_event)
+glusterd_op_sm_new_event(glusterd_op_sm_event_type_t event_type,
+                         glusterd_op_sm_event_t **new_event)
 {
-        glusterd_op_sm_event_t      *event = NULL;
+    glusterd_op_sm_event_t *event = NULL;
 
-        GF_ASSERT (new_event);
-        GF_ASSERT (GD_OP_EVENT_NONE <= event_type &&
-                        GD_OP_EVENT_MAX > event_type);
+    GF_ASSERT(new_event);
+    GF_ASSERT(GD_OP_EVENT_NONE <= event_type && GD_OP_EVENT_MAX > event_type);
 
-        event = GF_CALLOC (1, sizeof (*event), gf_gld_mt_op_sm_event_t);
+    event = GF_CALLOC(1, sizeof(*event), gf_gld_mt_op_sm_event_t);
 
-        if (!event)
-                return -1;
+    if (!event)
+        return -1;
 
-        *new_event = event;
-        event->event = event_type;
-        CDS_INIT_LIST_HEAD (&event->list);
+    *new_event = event;
+    event->event = event_type;
+    CDS_INIT_LIST_HEAD(&event->list);
 
-        return 0;
+    return 0;
 }
 
 int
-glusterd_op_sm_inject_event (glusterd_op_sm_event_type_t event_type,
-                             uuid_t *txn_id, void *ctx)
+glusterd_op_sm_inject_event(glusterd_op_sm_event_type_t event_type,
+                            uuid_t *txn_id, void *ctx)
 {
-        int32_t                 ret = -1;
-        glusterd_op_sm_event_t  *event = NULL;
+    int32_t ret = -1;
+    glusterd_op_sm_event_t *event = NULL;
 
-        GF_ASSERT (event_type < GD_OP_EVENT_MAX &&
-                        event_type >= GD_OP_EVENT_NONE);
+    GF_ASSERT(event_type < GD_OP_EVENT_MAX && event_type >= GD_OP_EVENT_NONE);
 
-        ret = glusterd_op_sm_new_event (event_type, &event);
+    ret = glusterd_op_sm_new_event(event_type, &event);
 
-        if (ret)
-                goto out;
+    if (ret)
+        goto out;
 
-        event->ctx = ctx;
+    event->ctx = ctx;
 
-        if (txn_id)
-                gf_uuid_copy (event->txn_id, *txn_id);
+    if (txn_id)
+        gf_uuid_copy(event->txn_id, *txn_id);
 
-        gf_log (THIS->name, GF_LOG_DEBUG, "Enqueue event: '%s'",
-                glusterd_op_sm_event_name_get (event->event));
-        cds_list_add_tail (&event->list, &gd_op_sm_queue);
+    gf_msg_debug(THIS->name, 0, "Enqueue event: '%s'",
+                 glusterd_op_sm_event_name_get(event->event));
+    cds_list_add_tail(&event->list, &gd_op_sm_queue);
 
 out:
-        return ret;
+    return ret;
 }
 
 void
-glusterd_destroy_req_ctx (glusterd_req_ctx_t *ctx)
+glusterd_destroy_req_ctx(glusterd_req_ctx_t *ctx)
 {
-        if (!ctx)
-                return;
-        if (ctx->dict)
-                dict_unref (ctx->dict);
-        GF_FREE (ctx);
+    if (!ctx)
+        return;
+    if (ctx->dict)
+        dict_unref(ctx->dict);
+    GF_FREE(ctx);
 }
 
 void
-glusterd_destroy_local_unlock_ctx (uuid_t *ctx)
+glusterd_destroy_local_unlock_ctx(uuid_t *ctx)
 {
-        if (!ctx)
-                return;
-        GF_FREE (ctx);
+    if (!ctx)
+        return;
+    GF_FREE(ctx);
 }
 
 void
-glusterd_destroy_op_event_ctx (glusterd_op_sm_event_t *event)
+glusterd_destroy_op_event_ctx(glusterd_op_sm_event_t *event)
 {
-        if (!event)
-                return;
+    if (!event)
+        return;
 
-        switch (event->event) {
+    switch (event->event) {
         case GD_OP_EVENT_LOCK:
         case GD_OP_EVENT_UNLOCK:
-                glusterd_destroy_lock_ctx (event->ctx);
-                break;
+            glusterd_destroy_lock_ctx(event->ctx);
+            break;
         case GD_OP_EVENT_STAGE_OP:
         case GD_OP_EVENT_ALL_ACK:
-                glusterd_destroy_req_ctx (event->ctx);
-                break;
+            glusterd_destroy_req_ctx(event->ctx);
+            break;
         case GD_OP_EVENT_LOCAL_UNLOCK_NO_RESP:
-                glusterd_destroy_local_unlock_ctx (event->ctx);
-                break;
+            glusterd_destroy_local_unlock_ctx(event->ctx);
+            break;
         default:
-                break;
-        }
+            break;
+    }
 }
 
 int
-glusterd_op_sm ()
+glusterd_op_sm()
 {
-        glusterd_op_sm_event_t          *event = NULL;
-        glusterd_op_sm_event_t          *tmp = NULL;
-        int                             ret = -1;
-        int                             lock_err = 0;
-        glusterd_op_sm_ac_fn            handler = NULL;
-        glusterd_op_sm_t                *state = NULL;
-        glusterd_op_sm_event_type_t     event_type = GD_OP_EVENT_NONE;
-        xlator_t                        *this = NULL;
-        glusterd_op_info_t              txn_op_info;
-
-        this = THIS;
-        GF_ASSERT (this);
-
-        ret = synclock_trylock (&gd_op_sm_lock);
-        if (ret) {
-                lock_err = errno;
-                gf_log (this->name, GF_LOG_ERROR, "lock failed due to %s",
-                        strerror (lock_err));
-                goto lock_failed;
-        }
-
-        while (!cds_list_empty (&gd_op_sm_queue)) {
-
-                cds_list_for_each_entry_safe (event, tmp, &gd_op_sm_queue,
-                                              list) {
-
-                        cds_list_del_init (&event->list);
-                        event_type = event->event;
-                        gf_log (this->name, GF_LOG_DEBUG, "Dequeued event of "
-                                "type: '%s'",
-                                glusterd_op_sm_event_name_get(event_type));
-
-                        gf_log (this->name, GF_LOG_DEBUG, "transaction ID = %s",
-                                uuid_utoa (event->txn_id));
-
-                        ret = glusterd_get_txn_opinfo (&event->txn_id,
-                                                       &txn_op_info);
-                        if (ret) {
-                                gf_log (this->name, GF_LOG_ERROR,
-                                        "Unable to get transaction's opinfo");
-                                glusterd_destroy_op_event_ctx (event);
-                                GF_FREE (event);
-                                continue;
-                        } else
-                                opinfo = txn_op_info;
-
-                        state = glusterd_op_state_table[opinfo.state.state];
-
-                        GF_ASSERT (state);
-
-                        handler = state[event_type].handler;
-                        GF_ASSERT (handler);
-
-                        ret = handler (event, event->ctx);
-
-                        if (ret) {
-                                gf_log (this->name, GF_LOG_ERROR,
-                                        "handler returned: %d", ret);
-                                glusterd_destroy_op_event_ctx (event);
-                                GF_FREE (event);
-                                continue;
-                        }
-
-                        ret = glusterd_op_sm_transition_state (&opinfo, state,
-                                                                event_type);
-
-                        if (ret) {
-                                gf_log (this->name, GF_LOG_ERROR,
-                                        "Unable to transition"
-                                        "state from '%s' to '%s'",
-                         glusterd_op_sm_state_name_get(opinfo.state.state),
-                         glusterd_op_sm_state_name_get(state[event_type].next_state));
-                                (void) synclock_unlock (&gd_op_sm_lock);
-                                return ret;
-                        }
-
-                        if ((state[event_type].next_state ==
-                             GD_OP_STATE_DEFAULT) &&
-                           (event_type == GD_OP_EVENT_UNLOCK)) {
-                                /* Clearing the transaction opinfo */
-                                ret = glusterd_clear_txn_opinfo(&event->txn_id);
-                                if (ret)
-                                        gf_log (this->name, GF_LOG_ERROR,
-                                                "Unable to clear "
-                                                "transaction's opinfo");
-                        } else {
-                                ret = glusterd_set_txn_opinfo (&event->txn_id,
-                                                               &opinfo);
-                                if (ret)
-                                        gf_log (this->name, GF_LOG_ERROR,
-                                                "Unable to set "
-                                                "transaction's opinfo");
-                        }
-
-                        glusterd_destroy_op_event_ctx (event);
-                        GF_FREE (event);
-
-                }
-        }
-
+    glusterd_op_sm_event_t *event = NULL;
+    glusterd_op_sm_event_t *tmp = NULL;
+    int ret = -1;
+    int lock_err = 0;
+    glusterd_op_sm_ac_fn handler = NULL;
+    glusterd_op_sm_t *state = NULL;
+    glusterd_op_sm_event_type_t event_type = GD_OP_EVENT_NONE;
+    xlator_t *this = NULL;
+    glusterd_op_info_t txn_op_info;
+    glusterd_conf_t *priv = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+    priv = this->private;
+    GF_ASSERT(priv);
+
+    ret = synclock_trylock(&gd_op_sm_lock);
+    if (ret) {
+        lock_err = errno;
+        gf_msg(this->name, GF_LOG_ERROR, errno, GD_MSG_LOCK_FAIL,
+               "lock failed due to %s", strerror(lock_err));
+        goto lock_failed;
+    }
+
+    while (!cds_list_empty(&gd_op_sm_queue)) {
+        cds_list_for_each_entry_safe(event, tmp, &gd_op_sm_queue, list)
+        {
+            cds_list_del_init(&event->list);
+            event_type = event->event;
+            gf_msg_debug(this->name, 0,
+                         "Dequeued event of "
+                         "type: '%s'",
+                         glusterd_op_sm_event_name_get(event_type));
+
+            gf_msg_debug(this->name, 0, "transaction ID = %s",
+                         uuid_utoa(event->txn_id));
+
+            ret = glusterd_get_txn_opinfo(&event->txn_id, &txn_op_info);
+            if (ret) {
+                gf_msg_callingfn(this->name, GF_LOG_ERROR, 0,
+                                 GD_MSG_TRANS_OPINFO_GET_FAIL,
+                                 "Unable to get transaction "
+                                 "opinfo for transaction ID :"
+                                 "%s",
+                                 uuid_utoa(event->txn_id));
+                glusterd_destroy_op_event_ctx(event);
+                GF_FREE(event);
+                continue;
+            } else
+                opinfo = txn_op_info;
+
+            state = glusterd_op_state_table[opinfo.state.state];
+
+            GF_ASSERT(state);
+
+            handler = state[event_type].handler;
+            GF_ASSERT(handler);
+
+            ret = handler(event, event->ctx);
+
+            if (ret) {
+                gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_HANDLER_RETURNED,
+                       "handler returned: %d", ret);
+                glusterd_destroy_op_event_ctx(event);
+                GF_FREE(event);
+                continue;
+            }
+
+            ret = glusterd_op_sm_transition_state(&opinfo, state, event_type);
+
+            if (ret) {
+                gf_msg(this->name, GF_LOG_ERROR, 0,
+                       GD_MSG_EVENT_STATE_TRANSITION_FAIL,
+                       "Unable to transition"
+                       "state from '%s' to '%s'",
+                       glusterd_op_sm_state_name_get(opinfo.state.state),
+                       glusterd_op_sm_state_name_get(
+                           state[event_type].next_state));
+                (void)synclock_unlock(&gd_op_sm_lock);
+                return ret;
+            }
 
-        (void) synclock_unlock (&gd_op_sm_lock);
-        ret = 0;
+            if ((state[event_type].next_state == GD_OP_STATE_DEFAULT) &&
+                (event_type == GD_OP_EVENT_UNLOCK)) {
+                /* Clearing the transaction opinfo */
+                ret = glusterd_clear_txn_opinfo(&event->txn_id);
+                if (ret)
+                    gf_msg(this->name, GF_LOG_ERROR, 0,
+                           GD_MSG_TRANS_OPINFO_CLEAR_FAIL,
+                           "Unable to clear "
+                           "transaction's opinfo");
+            } else {
+                if ((priv->op_version < GD_OP_VERSION_6_0) ||
+                    !(event_type == GD_OP_EVENT_STAGE_OP &&
+                      opinfo.state.state == GD_OP_STATE_STAGED &&
+                      opinfo.skip_locking)) {
+                    ret = glusterd_set_txn_opinfo(&event->txn_id, &opinfo);
+                    if (ret)
+                        gf_msg(this->name, GF_LOG_ERROR, 0,
+                               GD_MSG_TRANS_OPINFO_SET_FAIL,
+                               "Unable to set "
+                               "transaction's opinfo");
+                }
+            }
+
+            glusterd_destroy_op_event_ctx(event);
+            GF_FREE(event);
+        }
+    }
+
+    (void)synclock_unlock(&gd_op_sm_lock);
+    ret = 0;
 
 lock_failed:
 
-        return ret;
+    return ret;
 }
 
 int32_t
-glusterd_op_set_op (glusterd_op_t op)
+glusterd_op_set_op(glusterd_op_t op)
 {
+    GF_ASSERT(op < GD_OP_MAX);
+    GF_ASSERT(op > GD_OP_NONE);
 
-        GF_ASSERT (op < GD_OP_MAX);
-        GF_ASSERT (op > GD_OP_NONE);
-
-        opinfo.op = op;
-
-        return 0;
+    opinfo.op = op;
 
+    return 0;
 }
 
 int32_t
-glusterd_op_get_op ()
+glusterd_op_get_op()
 {
-
-        return opinfo.op;
-
+    return opinfo.op;
 }
 
 int32_t
-glusterd_op_set_req (rpcsvc_request_t *req)
+glusterd_op_set_req(rpcsvc_request_t *req)
 {
-
-        GF_ASSERT (req);
-        opinfo.req = req;
-        return 0;
+    GF_ASSERT(req);
+    opinfo.req = req;
+    return 0;
 }
 
 int32_t
-glusterd_op_clear_op (glusterd_op_t op)
+glusterd_op_clear_op(glusterd_op_t op)
 {
+    opinfo.op = GD_OP_NONE;
 
-        opinfo.op = GD_OP_NONE;
-
-        return 0;
-
+    return 0;
 }
 
 int32_t
-glusterd_op_free_ctx (glusterd_op_t op, void *ctx)
+glusterd_op_free_ctx(glusterd_op_t op, void *ctx)
 {
-
-        if (ctx) {
-                switch (op) {
-                case GD_OP_CREATE_VOLUME:
-                case GD_OP_DELETE_VOLUME:
-                case GD_OP_STOP_VOLUME:
-                case GD_OP_ADD_BRICK:
-                case GD_OP_REMOVE_BRICK:
-                case GD_OP_REPLACE_BRICK:
-                case GD_OP_LOG_ROTATE:
-                case GD_OP_SYNC_VOLUME:
-                case GD_OP_SET_VOLUME:
-                case GD_OP_START_VOLUME:
-                case GD_OP_RESET_VOLUME:
-                case GD_OP_GSYNC_SET:
-                case GD_OP_QUOTA:
-                case GD_OP_PROFILE_VOLUME:
-                case GD_OP_STATUS_VOLUME:
-                case GD_OP_REBALANCE:
-                case GD_OP_HEAL_VOLUME:
-                case GD_OP_STATEDUMP_VOLUME:
-                case GD_OP_CLEARLOCKS_VOLUME:
-                case GD_OP_DEFRAG_BRICK_VOLUME:
-                        dict_unref (ctx);
-                        break;
-                default:
-                        GF_ASSERT (0);
-                        break;
-                }
+    if (ctx) {
+        switch (op) {
+            case GD_OP_CREATE_VOLUME:
+            case GD_OP_DELETE_VOLUME:
+            case GD_OP_STOP_VOLUME:
+            case GD_OP_ADD_BRICK:
+            case GD_OP_REMOVE_BRICK:
+            case GD_OP_REPLACE_BRICK:
+            case GD_OP_LOG_ROTATE:
+            case GD_OP_SYNC_VOLUME:
+            case GD_OP_SET_VOLUME:
+            case GD_OP_START_VOLUME:
+            case GD_OP_RESET_VOLUME:
+            case GD_OP_GSYNC_SET:
+            case GD_OP_QUOTA:
+            case GD_OP_PROFILE_VOLUME:
+            case GD_OP_STATUS_VOLUME:
+            case GD_OP_REBALANCE:
+            case GD_OP_HEAL_VOLUME:
+            case GD_OP_STATEDUMP_VOLUME:
+            case GD_OP_CLEARLOCKS_VOLUME:
+            case GD_OP_DEFRAG_BRICK_VOLUME:
+            case GD_OP_MAX_OPVERSION:
+                dict_unref(ctx);
+                break;
+            default:
+                GF_ASSERT(0);
+                break;
         }
+    }
 
-        glusterd_op_reset_ctx ();
-        return 0;
-
+    glusterd_op_reset_ctx();
+    return 0;
 }
 
 void *
-glusterd_op_get_ctx ()
+glusterd_op_get_ctx()
 {
-
-        return opinfo.op_ctx;
-
+    return opinfo.op_ctx;
 }
 
 int
-glusterd_op_sm_init ()
+glusterd_op_sm_init()
 {
-        CDS_INIT_LIST_HEAD (&gd_op_sm_queue);
-        synclock_init (&gd_op_sm_lock);
-        return 0;
+    CDS_INIT_LIST_HEAD(&gd_op_sm_queue);
+    synclock_init(&gd_op_sm_lock, SYNC_LOCK_DEFAULT);
+    return 0;
 }
diff --git a/xlators/mgmt/glusterd/src/glusterd-op-sm.h b/xlators/mgmt/glusterd/src/glusterd-op-sm.h
index 69bfd4c92a5..8a24b16612a 100644
--- a/xlators/mgmt/glusterd/src/glusterd-op-sm.h
+++ b/xlators/mgmt/glusterd/src/glusterd-op-sm.h
@@ -10,289 +10,304 @@
 #ifndef _GLUSTERD_OP_SM_H_
 #define _GLUSTERD_OP_SM_H_
 
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
-
-
 #include <pthread.h>
-#include "compat-uuid.h"
-
-#include "glusterfs.h"
-#include "xlator.h"
-#include "logging.h"
-#include "call-stub.h"
-#include "fd.h"
-#include "byte-order.h"
+#include <glusterfs/compat-uuid.h>
+
+#include <glusterfs/glusterfs.h>
+#include <glusterfs/xlator.h>
+#include <glusterfs/logging.h>
+#include <glusterfs/call-stub.h>
+#include <glusterfs/byte-order.h>
 #include "glusterd.h"
 #include "protocol-common.h"
+#include "glusterd-hooks.h"
 
-#define GD_OP_PROTECTED    (0x02)
-#define GD_OP_UNPROTECTED  (0x04)
+#define GD_OP_PROTECTED (0x02)
+#define GD_OP_UNPROTECTED (0x04)
 
 typedef enum glusterd_op_sm_state_ {
-        GD_OP_STATE_DEFAULT = 0,
-        GD_OP_STATE_LOCK_SENT,
-        GD_OP_STATE_LOCKED,
-        GD_OP_STATE_STAGE_OP_SENT,
-        GD_OP_STATE_STAGED,
-        GD_OP_STATE_COMMIT_OP_SENT,
-        GD_OP_STATE_COMMITED,
-        GD_OP_STATE_UNLOCK_SENT,
-        GD_OP_STATE_STAGE_OP_FAILED,
-        GD_OP_STATE_COMMIT_OP_FAILED,
-        GD_OP_STATE_BRICK_OP_SENT,
-        GD_OP_STATE_BRICK_OP_FAILED,
-        GD_OP_STATE_BRICK_COMMITTED,
-        GD_OP_STATE_BRICK_COMMIT_FAILED,
-        GD_OP_STATE_ACK_DRAIN,
-        GD_OP_STATE_MAX,
+    GD_OP_STATE_DEFAULT = 0,
+    GD_OP_STATE_LOCK_SENT,
+    GD_OP_STATE_LOCKED,
+    GD_OP_STATE_STAGE_OP_SENT,
+    GD_OP_STATE_STAGED,
+    GD_OP_STATE_COMMIT_OP_SENT,
+    GD_OP_STATE_COMMITED,
+    GD_OP_STATE_UNLOCK_SENT,
+    GD_OP_STATE_STAGE_OP_FAILED,
+    GD_OP_STATE_COMMIT_OP_FAILED,
+    GD_OP_STATE_BRICK_OP_SENT,
+    GD_OP_STATE_BRICK_OP_FAILED,
+    GD_OP_STATE_BRICK_COMMITTED,
+    GD_OP_STATE_BRICK_COMMIT_FAILED,
+    GD_OP_STATE_ACK_DRAIN,
+    GD_OP_STATE_MAX,
 } glusterd_op_sm_state_t;
 
 typedef enum glusterd_op_sm_event_type_ {
-        GD_OP_EVENT_NONE = 0,
-        GD_OP_EVENT_START_LOCK,
-        GD_OP_EVENT_LOCK,
-        GD_OP_EVENT_RCVD_ACC,
-        GD_OP_EVENT_ALL_ACC,
-        GD_OP_EVENT_STAGE_ACC,
-        GD_OP_EVENT_COMMIT_ACC,
-        GD_OP_EVENT_RCVD_RJT,
-        GD_OP_EVENT_STAGE_OP,
-        GD_OP_EVENT_COMMIT_OP,
-        GD_OP_EVENT_UNLOCK,
-        GD_OP_EVENT_START_UNLOCK,
-        GD_OP_EVENT_ALL_ACK,
-        GD_OP_EVENT_LOCAL_UNLOCK_NO_RESP,
-        GD_OP_EVENT_MAX
+    GD_OP_EVENT_NONE = 0,
+    GD_OP_EVENT_START_LOCK,
+    GD_OP_EVENT_LOCK,
+    GD_OP_EVENT_RCVD_ACC,
+    GD_OP_EVENT_ALL_ACC,
+    GD_OP_EVENT_STAGE_ACC,
+    GD_OP_EVENT_COMMIT_ACC,
+    GD_OP_EVENT_RCVD_RJT,
+    GD_OP_EVENT_STAGE_OP,
+    GD_OP_EVENT_COMMIT_OP,
+    GD_OP_EVENT_UNLOCK,
+    GD_OP_EVENT_START_UNLOCK,
+    GD_OP_EVENT_ALL_ACK,
+    GD_OP_EVENT_LOCAL_UNLOCK_NO_RESP,
+    GD_OP_EVENT_MAX
 } glusterd_op_sm_event_type_t;
 
-
 struct glusterd_op_sm_event_ {
-        struct cds_list_head            list;
-        void                            *ctx;
-        glusterd_op_sm_event_type_t     event;
-        uuid_t                          txn_id;
+    struct cds_list_head list;
+    void *ctx;
+    glusterd_op_sm_event_type_t event;
+    uuid_t txn_id;
 };
 
 typedef struct glusterd_op_sm_event_ glusterd_op_sm_event_t;
 
-typedef int (*glusterd_op_sm_ac_fn) (glusterd_op_sm_event_t *, void *);
+typedef int (*glusterd_op_sm_ac_fn)(glusterd_op_sm_event_t *, void *);
 
 typedef struct glusterd_op_sm_ {
-        glusterd_op_sm_state_t      next_state;
-        glusterd_op_sm_ac_fn        handler;
+    glusterd_op_sm_state_t next_state;
+    glusterd_op_sm_ac_fn handler;
 } glusterd_op_sm_t;
 
 typedef struct glusterd_op_sm_state_info_ {
-        glusterd_op_sm_state_t          state;
-        struct timeval                  time;
+    glusterd_op_sm_state_t state;
+    struct timeval time;
 } glusterd_op_sm_state_info_t;
 
 struct glusterd_op_info_ {
-        glusterd_op_sm_state_info_t     state;
-        int32_t                         pending_count;
-        int32_t                         brick_pending_count;
-        int32_t                         op_count;
-        glusterd_op_t                   op;
-        struct cds_list_head            op_peers;
-        void                            *op_ctx;
-        rpcsvc_request_t                *req;
-        int32_t                         op_ret;
-        int32_t                         op_errno;
-        char                            *op_errstr;
-        struct  cds_list_head           pending_bricks;
-        struct  cds_list_head           *local_xaction_peers;
-
+    glusterd_op_sm_state_info_t state;
+    int32_t pending_count;
+    int32_t brick_pending_count;
+    int32_t op_count;
+    /* op is an enum, glusterd_op_t or glusterd_op_sm_state_info_t */
+    int op;
+    struct cds_list_head op_peers;
+    void *op_ctx;
+    rpcsvc_request_t *req;
+    int32_t op_ret;
+    int32_t op_errno;
+    char *op_errstr;
+    struct cds_list_head pending_bricks;
+    uint32_t txn_generation;
+    gf_boolean_t skip_locking;
 };
 
 typedef struct glusterd_op_info_ glusterd_op_info_t;
 
 struct glusterd_op_log_filename_ctx_ {
-        char                    volume_name[GD_VOLUME_NAME_MAX];
-        char                    brick[GD_VOLUME_NAME_MAX];
-        char                    path[PATH_MAX];
+    char volume_name[GD_VOLUME_NAME_MAX];
+    char brick[GD_VOLUME_NAME_MAX];
+    char path[PATH_MAX];
 };
 typedef struct glusterd_op_log_filename_ctx_ glusterd_op_log_filename_ctx_t;
 
 struct glusterd_op_lock_ctx_ {
-        uuid_t                  uuid;
-        dict_t                 *dict;
-        rpcsvc_request_t        *req;
+    uuid_t uuid;
+    dict_t *dict;
+    rpcsvc_request_t *req;
 };
 
 typedef struct glusterd_op_lock_ctx_ glusterd_op_lock_ctx_t;
 
 struct glusterd_req_ctx_ {
-        rpcsvc_request_t *req;
-	u_char            uuid[16];
-	int               op;
-        dict_t           *dict;
+    rpcsvc_request_t *req;
+    u_char uuid[16];
+    int op;
+    dict_t *dict;
 };
 
 typedef struct glusterd_req_ctx_ glusterd_req_ctx_t;
 
 typedef struct glusterd_op_brick_rsp_ctx_ {
-        int  op_ret;
-        char *op_errstr;
-        dict_t *rsp_dict;
-        glusterd_req_ctx_t *commit_ctx;
-        glusterd_pending_node_t *pending_node;
+    int op_ret;
+    char *op_errstr;
+    dict_t *rsp_dict;
+    glusterd_req_ctx_t *commit_ctx;
+    glusterd_pending_node_t *pending_node;
 } glusterd_op_brick_rsp_ctx_t;
 
 typedef struct glusterd_pr_brick_rsp_conv_t {
-        int count;
-        dict_t *dict;
+    int count;
+    dict_t *dict;
 } glusterd_pr_brick_rsp_conv_t;
 
 typedef struct glusterd_heal_rsp_conv_ {
-        dict_t *dict;
-        glusterd_volinfo_t *volinfo;
-        xlator_t *this;
+    dict_t *dict;
+    glusterd_volinfo_t *volinfo;
+    xlator_t *this;
 } glusterd_heal_rsp_conv_t;
 
 typedef struct glusterd_status_rsp_conv_ {
-        int count;
-        int brick_index_max;
-        int other_count;
-        dict_t *dict;
+    int count;
+    int brick_index_max;
+    int other_count;
+    dict_t *dict;
 } glusterd_status_rsp_conv_t;
 
-
 typedef struct glusterd_txn_opinfo_object_ {
-        glusterd_op_info_t    opinfo;
+    glusterd_op_info_t opinfo;
 } glusterd_txn_opinfo_obj;
 
 typedef enum cli_cmd_type_ {
-        PER_HEAL_XL,
-        ALL_HEAL_XL,
- } cli_cmd_type;
+    PER_HEAL_XL,
+    ALL_HEAL_XL,
+} cli_cmd_type;
+
+typedef struct glusterd_all_volume_options {
+    char *option;
+    char *dflt_val;
+} glusterd_all_vol_opts;
+
+int
+glusterd_op_commit_hook(glusterd_op_t op, dict_t *op_ctx,
+                        glusterd_commit_hook_type_t type);
 
 int
-glusterd_op_sm_new_event (glusterd_op_sm_event_type_t event_type,
-                          glusterd_op_sm_event_t **new_event);
+glusterd_op_sm_new_event(glusterd_op_sm_event_type_t event_type,
+                         glusterd_op_sm_event_t **new_event);
 int
-glusterd_op_sm_inject_event (glusterd_op_sm_event_type_t event_type,
-                             uuid_t *txn_id, void *ctx);
+glusterd_op_sm_inject_event(glusterd_op_sm_event_type_t event_type,
+                            uuid_t *txn_id, void *ctx);
 
 int
-glusterd_op_sm_init ();
+glusterd_op_sm_init();
 
 int
-glusterd_op_sm ();
+glusterd_op_sm();
 
 int32_t
-glusterd_op_set_ctx (void *ctx);
+glusterd_op_set_ctx(void *ctx);
 
 int32_t
-glusterd_op_set_op (glusterd_op_t op);
+glusterd_op_set_op(glusterd_op_t op);
 
 int
-glusterd_op_build_payload (dict_t **req, char **op_errstr, dict_t *op_ctx);
+glusterd_op_build_payload(dict_t **req, char **op_errstr, dict_t *op_ctx);
 
 int32_t
-glusterd_op_stage_validate (glusterd_op_t op, dict_t *req, char **op_errstr,
-                            dict_t *rsp_dict);
+glusterd_op_stage_validate(glusterd_op_t op, dict_t *req, char **op_errstr,
+                           dict_t *rsp_dict);
 
 int32_t
-glusterd_op_commit_perform (glusterd_op_t op, dict_t *req, char **op_errstr,
-                            dict_t* dict);
+glusterd_op_commit_perform(glusterd_op_t op, dict_t *req, char **op_errstr,
+                           dict_t *dict);
 
 int32_t
-glusterd_op_txn_begin (rpcsvc_request_t *req, glusterd_op_t op, void *ctx,
-                       char *err_str, size_t err_len);
+glusterd_op_txn_begin(rpcsvc_request_t *req, glusterd_op_t op, void *ctx,
+                      char *err_str, size_t err_len);
 
 int32_t
-glusterd_op_txn_complete ();
+glusterd_op_txn_complete();
 
 void *
-glusterd_op_get_ctx ();
+glusterd_op_get_ctx();
 
 int32_t
-glusterd_op_set_req (rpcsvc_request_t *req);
+glusterd_op_set_req(rpcsvc_request_t *req);
 
 int32_t
-glusterd_op_send_cli_response (glusterd_op_t op, int32_t op_ret,
-                               int32_t op_errno, rpcsvc_request_t *req,
-                               void *ctx, char *op_errstr);
+glusterd_op_send_cli_response(glusterd_op_t op, int32_t op_ret,
+                              int32_t op_errno, rpcsvc_request_t *req,
+                              void *ctx, char *op_errstr);
 int32_t
-glusterd_op_get_op ();
+glusterd_op_get_op();
 
 int32_t
-glusterd_op_clear_op ();
+glusterd_op_clear_op();
 
 int32_t
-glusterd_op_free_ctx (glusterd_op_t op, void *ctx);
+glusterd_op_free_ctx(glusterd_op_t op, void *ctx);
 
 int
 glusterd_check_option_exists(char *optstring, char **completion);
 
 int
-set_xlator_option (dict_t *dict, char *key, char *value);
+set_xlator_option(dict_t *dict, char *key, char *value);
 
-void
-glusterd_do_replace_brick (void *data);
-
-char*
-glusterd_op_sm_state_name_get (int state);
+char *
+glusterd_op_sm_state_name_get(int state);
 
-char*
-glusterd_op_sm_event_name_get (int event);
+char *
+glusterd_op_sm_event_name_get(int event);
 int32_t
-glusterd_op_bricks_select (glusterd_op_t op, dict_t *dict, char **op_errstr,
-                           struct cds_list_head *selected, dict_t *rsp_dict);
+glusterd_op_bricks_select(glusterd_op_t op, dict_t *dict, char **op_errstr,
+                          struct cds_list_head *selected, dict_t *rsp_dict);
 int
-glusterd_brick_op_build_payload (glusterd_op_t op, glusterd_brickinfo_t *brickinfo,
-                                 gd1_mgmt_brick_op_req **req, dict_t *dict);
+glusterd_brick_op_build_payload(glusterd_op_t op,
+                                glusterd_brickinfo_t *brickinfo,
+                                gd1_mgmt_brick_op_req **req, dict_t *dict);
 int
-glusterd_node_op_build_payload (glusterd_op_t op, gd1_mgmt_brick_op_req **req,
+glusterd_node_op_build_payload(glusterd_op_t op, gd1_mgmt_brick_op_req **req,
                                dict_t *dict);
 int32_t
-glusterd_handle_brick_rsp (void *pending_entry, glusterd_op_t op,
-                           dict_t *rsp_dict, dict_t *ctx_dict, char **op_errstr,
-                           gd_node_type type);
+glusterd_handle_brick_rsp(void *pending_entry, glusterd_op_t op,
+                          dict_t *rsp_dict, dict_t *ctx_dict, char **op_errstr,
+                          gd_node_type type);
 
-dict_t*
-glusterd_op_init_commit_rsp_dict (glusterd_op_t op);
+dict_t *
+glusterd_op_init_commit_rsp_dict(glusterd_op_t op);
 
 void
-glusterd_op_modify_op_ctx (glusterd_op_t op, void *op_ctx);
+glusterd_op_modify_op_ctx(glusterd_op_t op, void *op_ctx);
+
+int
+glusterd_set_detach_bricks(dict_t *dict, glusterd_volinfo_t *volinfo);
 
 int32_t
-glusterd_volume_stats_read_perf (char *brick_path, int32_t blk_size,
-                int32_t blk_count, double *throughput, double *time);
+glusterd_volume_stats_read_perf(char *brick_path, int32_t blk_size,
+                                int32_t blk_count, double *throughput,
+                                double *time);
 int32_t
-glusterd_volume_stats_write_perf (char *brick_path, int32_t blk_size,
-                int32_t blk_count, double *throughput, double *time);
+glusterd_volume_stats_write_perf(char *brick_path, int32_t blk_size,
+                                 int32_t blk_count, double *throughput,
+                                 double *time);
 gf_boolean_t
-glusterd_is_volume_started (glusterd_volinfo_t  *volinfo);
+glusterd_is_volume_started(glusterd_volinfo_t *volinfo);
+
 int
-glusterd_start_bricks (glusterd_volinfo_t *volinfo);
+glusterd_start_bricks(glusterd_volinfo_t *volinfo);
+
 gf_boolean_t
-glusterd_are_all_volumes_stopped ();
-int
-glusterd_stop_bricks (glusterd_volinfo_t *volinfo);
+glusterd_are_all_volumes_stopped();
 int
-glusterd_defrag_volume_node_rsp (dict_t *req_dict, dict_t *rsp_dict,
-                                 dict_t *op_ctx);
-#ifdef HAVE_BD_XLATOR
+glusterd_stop_bricks(glusterd_volinfo_t *volinfo);
 int
-glusterd_is_valid_vg (glusterd_brickinfo_t *brick, int check_tag, char *msg);
-#endif
+glusterd_defrag_volume_node_rsp(dict_t *req_dict, dict_t *rsp_dict,
+                                dict_t *op_ctx);
 
 int32_t
-glusterd_get_txn_opinfo (uuid_t *txn_id, glusterd_op_info_t  *opinfo);
+glusterd_get_txn_opinfo(uuid_t *txn_id, glusterd_op_info_t *opinfo);
 
 int32_t
-glusterd_set_txn_opinfo (uuid_t *txn_id, glusterd_op_info_t  *opinfo);
+glusterd_set_txn_opinfo(uuid_t *txn_id, glusterd_op_info_t *opinfo);
 
 int32_t
-glusterd_clear_txn_opinfo (uuid_t *txn_id);
+glusterd_clear_txn_opinfo(uuid_t *txn_id);
 
 int32_t
-glusterd_generate_txn_id (dict_t *dict, uuid_t **txn_id);
+glusterd_generate_txn_id(dict_t *dict, uuid_t **txn_id);
 
 void
-glusterd_set_opinfo (char *errstr, int32_t op_errno, int32_t op_ret);
+glusterd_set_opinfo(char *errstr, int32_t op_errno, int32_t op_ret);
+
+int
+glusterd_dict_set_volid(dict_t *dict, char *volname, char **op_errstr);
+
+int
+glusterd_op_stats_volume(dict_t *dict, char **op_errstr, dict_t *rsp_dict);
+
+int
+glusterd_op_stage_stats_volume(dict_t *dict, char **op_errstr);
+
+int
+gd_set_commit_hash(dict_t *dict);
 #endif
diff --git a/xlators/mgmt/glusterd/src/glusterd-peer-utils.c b/xlators/mgmt/glusterd/src/glusterd-peer-utils.c
index 5f098839398..18d355cb186 100644
--- a/xlators/mgmt/glusterd/src/glusterd-peer-utils.c
+++ b/xlators/mgmt/glusterd/src/glusterd-peer-utils.c
@@ -11,75 +11,181 @@
 #include "glusterd-peer-utils.h"
 #include "glusterd-store.h"
 #include "glusterd-server-quorum.h"
-#include "common-utils.h"
+#include "glusterd-messages.h"
+#include <glusterfs/common-utils.h>
+#include "glusterd-utils.h"
 
 void
-glusterd_peerinfo_destroy (struct rcu_head *head)
+glusterd_peerinfo_destroy(struct rcu_head *head)
 {
-        int32_t                   ret      = -1;
-        glusterd_peerinfo_t      *peerinfo = NULL;
-        glusterd_peer_hostname_t *hostname = NULL;
-        glusterd_peer_hostname_t *tmp      = NULL;
+    int32_t ret = -1;
+    glusterd_peerinfo_t *peerinfo = NULL;
+    glusterd_peer_hostname_t *hostname = NULL;
+    glusterd_peer_hostname_t *tmp = NULL;
 
-        /* This works as rcu_head is the first member of gd_rcu_head */
-        peerinfo = caa_container_of ((gd_rcu_head *)head, glusterd_peerinfo_t,
-                                      rcu_head);
+    /* This works as rcu_head is the first member of gd_rcu_head */
+    peerinfo = caa_container_of((gd_rcu_head *)head, glusterd_peerinfo_t,
+                                rcu_head);
 
-        /* Set THIS to the saved this. Needed by some functions below */
-        THIS = peerinfo->rcu_head.this;
+    /* Set THIS to the saved this. Needed by some functions below */
+    THIS = peerinfo->rcu_head.this;
 
-        CDS_INIT_LIST_HEAD (&peerinfo->uuid_list);
+    CDS_INIT_LIST_HEAD(&peerinfo->uuid_list);
 
-        ret = glusterd_store_delete_peerinfo (peerinfo);
-        if (ret) {
-                gf_log ("glusterd", GF_LOG_ERROR, "Deleting peer info failed");
-        }
+    ret = glusterd_store_delete_peerinfo(peerinfo);
+    if (ret) {
+        gf_msg("glusterd", GF_LOG_ERROR, errno, GD_MSG_PEERINFO_DELETE_FAIL,
+               "Deleting peer info failed");
+    }
 
-        GF_FREE (peerinfo->hostname);
-        peerinfo->hostname = NULL;
+    GF_FREE(peerinfo->hostname);
+    peerinfo->hostname = NULL;
 
-        cds_list_for_each_entry_safe (hostname, tmp, &peerinfo->hostnames,
-                                      hostname_list) {
-                glusterd_peer_hostname_free (hostname);
-        }
+    cds_list_for_each_entry_safe(hostname, tmp, &peerinfo->hostnames,
+                                 hostname_list)
+    {
+        glusterd_peer_hostname_free(hostname);
+    }
 
-        glusterd_sm_tr_log_delete (&peerinfo->sm_log);
-        pthread_mutex_destroy (&peerinfo->delete_lock);
-        GF_FREE (peerinfo);
+    glusterd_sm_tr_log_delete(&peerinfo->sm_log);
+    pthread_mutex_unlock(&peerinfo->delete_lock);
+    pthread_mutex_destroy(&peerinfo->delete_lock);
+    GF_FREE(peerinfo);
 
-        peerinfo = NULL;
+    peerinfo = NULL;
 
-        return;
+    return;
 }
 
 int32_t
-glusterd_peerinfo_cleanup (glusterd_peerinfo_t *peerinfo)
+glusterd_peerinfo_cleanup(glusterd_peerinfo_t *peerinfo)
 {
-        GF_ASSERT (peerinfo);
-        glusterd_peerctx_t      *peerctx = NULL;
-        gf_boolean_t            quorum_action = _gf_false;
-        glusterd_conf_t         *priv = THIS->private;
-
-        if (pthread_mutex_trylock (&peerinfo->delete_lock)) {
-                /* Someone else is already deleting the peer, so give up */
-                return 0;
-        }
+    GF_ASSERT(peerinfo);
+    gf_boolean_t quorum_action = _gf_false;
+    glusterd_conf_t *priv = THIS->private;
 
-        if (peerinfo->quorum_contrib != QUORUM_NONE)
-                quorum_action = _gf_true;
-        if (peerinfo->rpc) {
-                peerinfo->rpc = glusterd_rpc_clnt_unref (priv, peerinfo->rpc);
-                peerinfo->rpc = NULL;
+    if (pthread_mutex_trylock(&peerinfo->delete_lock)) {
+        /* Someone else is already deleting the peer, so give up */
+        return 0;
+    }
+
+    if (peerinfo->quorum_contrib != QUORUM_NONE)
+        quorum_action = _gf_true;
+    if (peerinfo->rpc) {
+        peerinfo->rpc = glusterd_rpc_clnt_unref(priv, peerinfo->rpc);
+        peerinfo->rpc = NULL;
+    }
+
+    cds_list_del_rcu(&peerinfo->uuid_list);
+    /* Saving THIS, as it is needed by the callback function */
+    peerinfo->rcu_head.this = THIS;
+    call_rcu(&peerinfo->rcu_head.head, glusterd_peerinfo_destroy);
+
+    if (quorum_action)
+        /* coverity[SLEEP] */
+        glusterd_do_quorum_action();
+    return 0;
+}
+
+/* gd_peerinfo_find_from_hostname iterates over all the addresses saved for each
+ * peer and matches it to @hoststr.
+ * Returns the matched peer if found else returns NULL
+ */
+static glusterd_peerinfo_t *
+gd_peerinfo_find_from_hostname(const char *hoststr)
+{
+    xlator_t *this = THIS;
+    glusterd_conf_t *priv = NULL;
+    glusterd_peerinfo_t *peer = NULL;
+    glusterd_peerinfo_t *found = NULL;
+    glusterd_peer_hostname_t *tmphost = NULL;
+
+    GF_ASSERT(this != NULL);
+    priv = this->private;
+    GF_VALIDATE_OR_GOTO(this->name, (priv != NULL), out);
+
+    GF_VALIDATE_OR_GOTO(this->name, (hoststr != NULL), out);
+
+    RCU_READ_LOCK;
+    cds_list_for_each_entry_rcu(peer, &priv->peers, uuid_list)
+    {
+        cds_list_for_each_entry_rcu(tmphost, &peer->hostnames, hostname_list)
+        {
+            if (!strncasecmp(tmphost->hostname, hoststr, 1024)) {
+                gf_msg_debug(this->name, 0, "Friend %s found.. state: %d",
+                             tmphost->hostname, peer->state.state);
+                found = peer; /* Probably needs to be
+                                 dereferenced*/
+                goto unlock;
+            }
         }
+    }
+unlock:
+    RCU_READ_UNLOCK;
+out:
+    return found;
+}
 
-        cds_list_del_rcu (&peerinfo->uuid_list);
-        /* Saving THIS, as it is needed by the callback function */
-        peerinfo->rcu_head.this = THIS;
-        call_rcu (&peerinfo->rcu_head.head, glusterd_peerinfo_destroy);
+/* gd_peerinfo_find_from_addrinfo iterates over all the addresses saved for each
+ * peer, resolves them and compares them to @addr.
+ *
+ *
+ * NOTE: As getaddrinfo is a blocking call and is being performed multiple times
+ * in this function, it could lead to the calling thread to be blocked for
+ * significant amounts of time.
+ *
+ * Returns the matched peer if found else returns NULL
+ */
+static glusterd_peerinfo_t *
+gd_peerinfo_find_from_addrinfo(const struct addrinfo *addr)
+{
+    xlator_t *this = THIS;
+    glusterd_conf_t *conf = NULL;
+    glusterd_peerinfo_t *peer = NULL;
+    glusterd_peerinfo_t *found = NULL;
+    glusterd_peer_hostname_t *address = NULL;
+    int ret = 0;
+    struct addrinfo *paddr = NULL;
+    struct addrinfo *tmp = NULL;
+
+    GF_ASSERT(this != NULL);
+    conf = this->private;
+    GF_VALIDATE_OR_GOTO(this->name, (conf != NULL), out);
+
+    RCU_READ_LOCK;
+    cds_list_for_each_entry_rcu(peer, &conf->peers, uuid_list)
+    {
+        cds_list_for_each_entry_rcu(address, &peer->hostnames, hostname_list)
+        {
+            /* TODO: Cache the resolved addrinfos to improve
+             * performance
+             */
+            ret = getaddrinfo(address->hostname, NULL, NULL, &paddr);
+            if (ret) {
+                /* Don't fail if getaddrinfo fails, continue
+                 * onto the next address
+                 */
+                gf_msg_trace(this->name, 0, "getaddrinfo for %s failed (%s)",
+                             address->hostname, gai_strerror(ret));
+                continue;
+            }
+
+            for (tmp = paddr; tmp != NULL; tmp = tmp->ai_next) {
+                if (gf_compare_sockaddr(addr->ai_addr, tmp->ai_addr)) {
+                    found = peer; /* (de)referenced? */
+                    break;
+                }
+            }
 
-        if (quorum_action)
-                glusterd_do_quorum_action ();
-        return 0;
+            freeaddrinfo(paddr);
+            if (found)
+                goto unlock;
+        }
+    }
+unlock:
+    RCU_READ_UNLOCK;
+out:
+    return found;
 }
 
 /* glusterd_peerinfo_find_by_hostname searches for a peer which matches the
@@ -91,78 +197,73 @@ glusterd_peerinfo_cleanup (glusterd_peerinfo_t *peerinfo)
  * the resolved addrinfos.
  */
 glusterd_peerinfo_t *
-glusterd_peerinfo_find_by_hostname (const char *hoststr)
+glusterd_peerinfo_find_by_hostname(const char *hoststr)
 {
-        int                  ret      = -1;
-        struct addrinfo     *addr     = NULL;
-        struct addrinfo     *p        = NULL;
-        xlator_t            *this     = NULL;
-        glusterd_peerinfo_t *peerinfo = NULL;
-
-
-        this = THIS;
-        GF_ASSERT (hoststr);
-
-        peerinfo = NULL;
-
-        peerinfo = gd_peerinfo_find_from_hostname (hoststr);
-        if (peerinfo)
-                return peerinfo;
-
-        ret = getaddrinfo (hoststr, NULL, NULL, &addr);
-        if (ret != 0) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "error in getaddrinfo: %s\n",
-                        gai_strerror(ret));
-                goto out;
-        }
-
-        for (p = addr; p != NULL; p = p->ai_next) {
-                peerinfo = gd_peerinfo_find_from_addrinfo (p);
-                if (peerinfo) {
-                        freeaddrinfo (addr);
-                        return peerinfo;
-                }
+    int ret = -1;
+    struct addrinfo *addr = NULL;
+    struct addrinfo *p = NULL;
+    xlator_t *this = THIS;
+    glusterd_peerinfo_t *peerinfo = NULL;
+
+    GF_ASSERT(hoststr);
+
+    peerinfo = gd_peerinfo_find_from_hostname(hoststr);
+    if (peerinfo)
+        return peerinfo;
+
+    ret = getaddrinfo(hoststr, NULL, NULL, &addr);
+    if (ret != 0) {
+        gf_msg(this->name, GF_LOG_ERROR, ret, GD_MSG_GETADDRINFO_FAIL,
+               "error in getaddrinfo: %s\n", gai_strerror(ret));
+        goto out;
+    }
+
+    for (p = addr; p != NULL; p = p->ai_next) {
+        peerinfo = gd_peerinfo_find_from_addrinfo(p);
+        if (peerinfo) {
+            freeaddrinfo(addr);
+            return peerinfo;
         }
+    }
 
 out:
-        gf_log (this->name, GF_LOG_DEBUG, "Unable to find friend: %s", hoststr);
-        if (addr)
-                freeaddrinfo (addr);
-        return NULL;
+    gf_msg_debug(this->name, 0, "Unable to find friend: %s", hoststr);
+    if (addr)
+        freeaddrinfo(addr);
+    return NULL;
 }
 
 int
-glusterd_hostname_to_uuid (char *hostname, uuid_t uuid)
+glusterd_hostname_to_uuid(char *hostname, uuid_t uuid)
 {
-        GF_ASSERT (hostname);
-        GF_ASSERT (uuid);
+    GF_ASSERT(hostname);
+    GF_ASSERT(uuid);
 
-        glusterd_peerinfo_t     *peerinfo = NULL;
-        glusterd_conf_t         *priv = NULL;
-        int                     ret = -1;
-        xlator_t                *this = NULL;
+    glusterd_peerinfo_t *peerinfo = NULL;
+    glusterd_conf_t *priv = NULL;
+    int ret = -1;
+    xlator_t *this = NULL;
 
-        this = THIS;
-        GF_ASSERT (this);
-        priv = this->private;
-        GF_ASSERT (priv);
+    this = THIS;
+    GF_ASSERT(this);
+    priv = this->private;
+    GF_ASSERT(priv);
 
-        peerinfo = glusterd_peerinfo_find_by_hostname (hostname);
-        if (peerinfo) {
-                ret = 0;
-                gf_uuid_copy (uuid, peerinfo->uuid);
+    peerinfo = glusterd_peerinfo_find_by_hostname(hostname);
+    if (peerinfo) {
+        ret = 0;
+        gf_uuid_copy(uuid, peerinfo->uuid);
+    } else {
+        if (gf_is_local_addr(hostname)) {
+            gf_uuid_copy(uuid, MY_UUID);
+            ret = 0;
         } else {
-                if (gf_is_local_addr (hostname)) {
-                        gf_uuid_copy (uuid, MY_UUID);
-                        ret = 0;
-                } else {
-                        ret = -1;
-                }
+            ret = -1;
         }
+    }
 
-        gf_log (this->name, GF_LOG_DEBUG, "returning %d", ret);
-        return ret;
+    gf_msg_debug(this->name, 0, "returning %d", ret);
+    return ret;
 }
 
 /* glusterd_peerinfo_find_by_uuid searches for a peer which matches the
@@ -170,40 +271,41 @@ glusterd_hostname_to_uuid (char *hostname, uuid_t uuid)
  * Returns NULL otherwise.
  */
 glusterd_peerinfo_t *
-glusterd_peerinfo_find_by_uuid (uuid_t uuid)
+glusterd_peerinfo_find_by_uuid(uuid_t uuid)
 {
-        glusterd_conf_t         *priv = NULL;
-        glusterd_peerinfo_t     *entry = NULL;
-        glusterd_peerinfo_t     *found = NULL;
-        xlator_t                *this = NULL;
-
-        this = THIS;
-        GF_ASSERT (this);
+    glusterd_conf_t *priv = NULL;
+    glusterd_peerinfo_t *entry = NULL;
+    glusterd_peerinfo_t *found = NULL;
+    xlator_t *this = THIS;
+    glusterd_friend_sm_state_t state;
 
-        priv    = this->private;
+    GF_ASSERT(this);
 
-        GF_ASSERT (priv);
+    if (gf_uuid_is_null(uuid))
+        return NULL;
 
-        if (gf_uuid_is_null (uuid))
-                return NULL;
+    priv = this->private;
 
-        rcu_read_lock ();
-        cds_list_for_each_entry_rcu (entry, &priv->peers, uuid_list) {
-                if (!gf_uuid_compare (entry->uuid, uuid)) {
+    GF_ASSERT(priv);
 
-                        gf_log (this->name, GF_LOG_DEBUG,
-                                 "Friend found... state: %s",
-                        glusterd_friend_sm_state_name_get (entry->state.state));
-                        found = entry; /* Probably should be rcu_dereferenced */
-                        break;
-                }
+    RCU_READ_LOCK;
+    cds_list_for_each_entry_rcu(entry, &priv->peers, uuid_list)
+    {
+        if (!gf_uuid_compare(entry->uuid, uuid)) {
+            found = entry; /* Probably should be rcu_dereferenced */
+            state = found->state.state;
+            break;
         }
-        rcu_read_unlock ();
-
-        if (!found)
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "Friend with uuid: %s, not found", uuid_utoa (uuid));
-        return found;
+    }
+    RCU_READ_UNLOCK;
+
+    if (found)
+        gf_msg_debug(this->name, 0, "Friend found... state: %s",
+                     glusterd_friend_sm_state_name_get(state));
+    else
+        gf_msg_debug(this->name, 0, "Friend with uuid: %s, not found",
+                     uuid_utoa(uuid));
+    return found;
 }
 
 /* glusterd_peerinfo_find will search for a peer matching either @uuid or
@@ -211,39 +313,35 @@ glusterd_peerinfo_find_by_uuid (uuid_t uuid)
  * Returns NULL otherwise.
  */
 glusterd_peerinfo_t *
-glusterd_peerinfo_find (uuid_t uuid, const char *hostname)
+glusterd_peerinfo_find(uuid_t uuid, const char *hostname)
 {
-        glusterd_peerinfo_t *peerinfo = NULL;
-        xlator_t            *this     = NULL;
+    glusterd_peerinfo_t *peerinfo = NULL;
+    xlator_t *this = THIS;
 
-        this = THIS;
-        GF_ASSERT (this);
+    GF_ASSERT(this);
 
+    if (uuid) {
+        peerinfo = glusterd_peerinfo_find_by_uuid(uuid);
 
-        if (uuid) {
-                peerinfo = glusterd_peerinfo_find_by_uuid (uuid);
-
-                if (peerinfo) {
-                        return peerinfo;
-                } else {
-                        gf_log (this->name, GF_LOG_DEBUG,
-                                 "Unable to find peer by uuid: %s",
-                                 uuid_utoa (uuid));
-                }
-
+        if (peerinfo) {
+            return peerinfo;
+        } else {
+            gf_msg_debug(this->name, 0, "Unable to find peer by uuid: %s",
+                         uuid_utoa(uuid));
         }
+    }
 
-        if (hostname) {
-                peerinfo = glusterd_peerinfo_find_by_hostname (hostname);
+    if (hostname) {
+        peerinfo = glusterd_peerinfo_find_by_hostname(hostname);
 
-                if (peerinfo) {
-                        return peerinfo;
-                } else {
-                        gf_log (this->name, GF_LOG_DEBUG,
-                                "Unable to find hostname: %s", hostname);
-                }
+        if (peerinfo) {
+            return peerinfo;
+        } else {
+            gf_msg_debug(this->name, 0, "Unable to find hostname: %s",
+                         hostname);
         }
-        return NULL;
+    }
+    return NULL;
 }
 
 /* glusterd_peerinfo_new will create a new peerinfo object and set it's members
@@ -255,258 +353,299 @@ glusterd_peerinfo_find (uuid_t uuid, const char *hostname)
  * object.
  */
 glusterd_peerinfo_t *
-glusterd_peerinfo_new (glusterd_friend_sm_state_t state, uuid_t *uuid,
-                       const char *hostname, int port)
+glusterd_peerinfo_new(glusterd_friend_sm_state_t state, uuid_t *uuid,
+                      const char *hostname, int port)
 {
-        glusterd_peerinfo_t      *new_peer = NULL;
-        int                      ret = -1;
-
-        new_peer = GF_CALLOC (1, sizeof (*new_peer), gf_gld_mt_peerinfo_t);
-        if (!new_peer)
-                goto out;
+    glusterd_peerinfo_t *new_peer = NULL;
+    int ret = -1;
+    xlator_t *this = NULL;
+    glusterd_conf_t *conf = NULL;
 
-        CDS_INIT_LIST_HEAD (&new_peer->uuid_list);
+    this = THIS;
+    GF_ASSERT(this);
+    conf = this->private;
+    GF_ASSERT(conf);
 
-        new_peer->state.state = state;
+    new_peer = GF_CALLOC(1, sizeof(*new_peer), gf_gld_mt_peerinfo_t);
+    if (!new_peer) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_NO_MEMORY, NULL);
+        goto out;
+    }
 
-        CDS_INIT_LIST_HEAD (&new_peer->hostnames);
-        if (hostname) {
-                ret = gd_add_address_to_peer (new_peer, hostname);
-                if (ret)
-                        goto out;
-                /* Also set it to peerinfo->hostname. Doing this as we use
-                 * peerinfo->hostname in a lot of places and is really hard to
-                 * get everything right
-                 */
-                new_peer->hostname = gf_strdup (hostname);
-        }
+    CDS_INIT_LIST_HEAD(&new_peer->uuid_list);
 
-        if (uuid) {
-                gf_uuid_copy (new_peer->uuid, *uuid);
-        }
+    new_peer->state.state = state;
 
-        ret = glusterd_sm_tr_log_init (&new_peer->sm_log,
-                                       glusterd_friend_sm_state_name_get,
-                                       glusterd_friend_sm_event_name_get,
-                                       GLUSTERD_TR_LOG_SIZE);
+    CDS_INIT_LIST_HEAD(&new_peer->hostnames);
+    if (hostname) {
+        ret = gd_add_address_to_peer(new_peer, hostname);
         if (ret)
-                goto out;
+            goto out;
+        /* Also set it to peerinfo->hostname. Doing this as we use
+         * peerinfo->hostname in a lot of places and is really hard to
+         * get everything right
+         */
+        new_peer->hostname = gf_strdup(hostname);
+    }
+
+    if (uuid) {
+        gf_uuid_copy(new_peer->uuid, *uuid);
+    }
 
-        if (new_peer->state.state == GD_FRIEND_STATE_BEFRIENDED)
-                new_peer->quorum_contrib = QUORUM_WAITING;
-        new_peer->port = port;
+    ret = glusterd_sm_tr_log_init(
+        &new_peer->sm_log, glusterd_friend_sm_state_name_get,
+        glusterd_friend_sm_event_name_get, GLUSTERD_TR_LOG_SIZE);
+    if (ret)
+        goto out;
 
-        pthread_mutex_init (&new_peer->delete_lock, NULL);
+    if (new_peer->state.state == GD_FRIEND_STATE_BEFRIENDED)
+        new_peer->quorum_contrib = QUORUM_WAITING;
+    new_peer->port = port;
+
+    pthread_mutex_init(&new_peer->delete_lock, NULL);
+
+    new_peer->generation = uatomic_add_return(&conf->generation, 1);
 out:
-        if (ret && new_peer) {
-                glusterd_peerinfo_cleanup (new_peer);
-                new_peer = NULL;
-        }
-        return new_peer;
+    if (ret && new_peer) {
+        glusterd_peerinfo_cleanup(new_peer);
+        new_peer = NULL;
+    }
+    return new_peer;
 }
 
 /* Check if the all peers are connected and befriended, except the peer
  * specified (the peer being detached)
  */
 gf_boolean_t
-glusterd_chk_peers_connected_befriended (uuid_t skip_uuid)
+glusterd_chk_peers_connected_befriended(uuid_t skip_uuid)
 {
-        gf_boolean_t            ret = _gf_true;
-        glusterd_peerinfo_t     *peerinfo = NULL;
-        glusterd_conf_t         *priv = NULL;
-
-        priv= THIS->private;
-        GF_ASSERT (priv);
-
-        rcu_read_lock ();
-        cds_list_for_each_entry_rcu (peerinfo, &priv->peers, uuid_list) {
-
-                if (!gf_uuid_is_null (skip_uuid) && !gf_uuid_compare (skip_uuid,
-                                                           peerinfo->uuid))
-                        continue;
-
-                if ((GD_FRIEND_STATE_BEFRIENDED != peerinfo->state.state)
-                    || !(peerinfo->connected)) {
-                        ret = _gf_false;
-                        break;
-                }
+    gf_boolean_t ret = _gf_true;
+    glusterd_peerinfo_t *peerinfo = NULL;
+    glusterd_conf_t *priv = NULL;
+
+    priv = THIS->private;
+    GF_ASSERT(priv);
+
+    RCU_READ_LOCK;
+    cds_list_for_each_entry_rcu(peerinfo, &priv->peers, uuid_list)
+    {
+        if (!gf_uuid_is_null(skip_uuid) &&
+            !gf_uuid_compare(skip_uuid, peerinfo->uuid))
+            continue;
+
+        if ((GD_FRIEND_STATE_BEFRIENDED != peerinfo->state.state) ||
+            !(peerinfo->connected)) {
+            ret = _gf_false;
+            break;
         }
-        rcu_read_unlock ();
+    }
+    RCU_READ_UNLOCK;
 
-        gf_log (THIS->name, GF_LOG_DEBUG, "Returning %s",
-                (ret?"TRUE":"FALSE"));
-        return ret;
+    gf_msg_debug(THIS->name, 0, "Returning %s", (ret ? "TRUE" : "FALSE"));
+    return ret;
 }
 
 /* Return hostname for given uuid if it exists
  * else return NULL
  */
 char *
-glusterd_uuid_to_hostname (uuid_t uuid)
+glusterd_uuid_to_hostname(uuid_t uuid)
 {
-        char                    *hostname = NULL;
-        glusterd_conf_t         *priv = NULL;
-        glusterd_peerinfo_t     *entry = NULL;
+    char *hostname = NULL;
+    glusterd_conf_t *priv = NULL;
+    glusterd_peerinfo_t *entry = NULL;
 
-        priv = THIS->private;
-        GF_ASSERT (priv);
+    priv = THIS->private;
+    GF_ASSERT(priv);
 
-        if (!gf_uuid_compare (MY_UUID, uuid)) {
-                hostname = gf_strdup ("localhost");
-        }
-        rcu_read_lock ();
-        if (!cds_list_empty (&priv->peers)) {
-                cds_list_for_each_entry_rcu (entry, &priv->peers, uuid_list) {
-                        if (!gf_uuid_compare (entry->uuid, uuid)) {
-                                hostname = gf_strdup (entry->hostname);
-                                break;
-                        }
-                }
+    if (!gf_uuid_compare(MY_UUID, uuid)) {
+        hostname = gf_strdup("localhost");
+        return hostname;
+    }
+    RCU_READ_LOCK;
+    if (!cds_list_empty(&priv->peers)) {
+        cds_list_for_each_entry_rcu(entry, &priv->peers, uuid_list)
+        {
+            if (!gf_uuid_compare(entry->uuid, uuid)) {
+                hostname = gf_strdup(entry->hostname);
+                break;
+            }
         }
-        rcu_read_unlock ();
+    }
+    RCU_READ_UNLOCK;
 
-        return hostname;
+    return hostname;
 }
 
-char*
-gd_peer_uuid_str (glusterd_peerinfo_t *peerinfo)
+char *
+gd_peer_uuid_str(glusterd_peerinfo_t *peerinfo)
 {
-        if ((peerinfo == NULL) || gf_uuid_is_null (peerinfo->uuid))
-                return NULL;
+    if ((peerinfo == NULL) || gf_uuid_is_null(peerinfo->uuid))
+        return NULL;
 
-        if (peerinfo->uuid_str[0] == '\0')
-                uuid_utoa_r (peerinfo->uuid, peerinfo->uuid_str);
+    if (peerinfo->uuid_str[0] == '\0')
+        uuid_utoa_r(peerinfo->uuid, peerinfo->uuid_str);
 
-        return peerinfo->uuid_str;
+    return peerinfo->uuid_str;
 }
 
 gf_boolean_t
-glusterd_are_vol_all_peers_up (glusterd_volinfo_t *volinfo,
-                               struct cds_list_head *peers,
-                               char **down_peerstr)
+glusterd_are_all_peers_up()
 {
-        glusterd_peerinfo_t   *peerinfo  = NULL;
-        glusterd_brickinfo_t  *brickinfo = NULL;
-        gf_boolean_t           ret       = _gf_false;
-
-        cds_list_for_each_entry (brickinfo, &volinfo->bricks, brick_list) {
-                if (!gf_uuid_compare (brickinfo->uuid, MY_UUID))
-                        continue;
-
-                rcu_read_lock ();
-                cds_list_for_each_entry_rcu (peerinfo, peers, uuid_list) {
-                        if (gf_uuid_compare (peerinfo->uuid, brickinfo->uuid))
-                                continue;
-
-                        /*Found peer who owns the brick, return false
-                         * if peer is not connected or not friend */
-                        if (!(peerinfo->connected) ||
-                           (peerinfo->state.state !=
-                             GD_FRIEND_STATE_BEFRIENDED)) {
-                                *down_peerstr = gf_strdup (peerinfo->hostname);
-                                gf_log ("", GF_LOG_DEBUG, "Peer %s is down. ",
-                                        peerinfo->hostname);
-                                rcu_read_unlock ();
-                                goto out;
-                        }
-                }
-                rcu_read_unlock ();
+    glusterd_peerinfo_t *peerinfo = NULL;
+    xlator_t *this = NULL;
+    glusterd_conf_t *conf = NULL;
+    gf_boolean_t peers_up = _gf_false;
+
+    this = THIS;
+    GF_VALIDATE_OR_GOTO("glusterd", this, out);
+
+    conf = this->private;
+    GF_VALIDATE_OR_GOTO(this->name, conf, out);
+
+    RCU_READ_LOCK;
+    cds_list_for_each_entry_rcu(peerinfo, &conf->peers, uuid_list)
+    {
+        if (!peerinfo->connected) {
+            RCU_READ_UNLOCK;
+            goto out;
         }
+    }
+    RCU_READ_UNLOCK;
+
+    peers_up = _gf_true;
 
-        ret = _gf_true;
 out:
-        gf_log ("glusterd", GF_LOG_DEBUG, "Returning %d", ret);
-        return ret;
+    return peers_up;
+}
+
+gf_boolean_t
+glusterd_are_vol_all_peers_up(glusterd_volinfo_t *volinfo,
+                              struct cds_list_head *peers, char **down_peerstr)
+{
+    glusterd_peerinfo_t *peerinfo = NULL;
+    glusterd_brickinfo_t *brickinfo = NULL;
+    gf_boolean_t ret = _gf_false;
+
+    cds_list_for_each_entry(brickinfo, &volinfo->bricks, brick_list)
+    {
+        if (!gf_uuid_compare(brickinfo->uuid, MY_UUID))
+            continue;
+
+        RCU_READ_LOCK;
+        cds_list_for_each_entry_rcu(peerinfo, peers, uuid_list)
+        {
+            if (gf_uuid_compare(peerinfo->uuid, brickinfo->uuid))
+                continue;
+
+            /*Found peer who owns the brick, return false
+             * if peer is not connected or not friend */
+            if (!(peerinfo->connected) ||
+                (peerinfo->state.state != GD_FRIEND_STATE_BEFRIENDED)) {
+                *down_peerstr = gf_strdup(peerinfo->hostname);
+                RCU_READ_UNLOCK;
+                gf_msg_debug(THIS->name, 0, "Peer %s is down. ", *down_peerstr);
+                goto out;
+            }
+        }
+        RCU_READ_UNLOCK;
+    }
+
+    ret = _gf_true;
+out:
+    gf_msg_debug("glusterd", 0, "Returning %d", ret);
+    return ret;
 }
 
 int32_t
-glusterd_peer_hostname_new (const char *hostname,
-                            glusterd_peer_hostname_t **name)
+glusterd_peer_hostname_new(const char *hostname,
+                           glusterd_peer_hostname_t **name)
 {
-        glusterd_peer_hostname_t        *peer_hostname = NULL;
-        int32_t                         ret = -1;
+    glusterd_peer_hostname_t *peer_hostname = NULL;
+    int32_t ret = -1;
 
-        GF_ASSERT (hostname);
-        GF_ASSERT (name);
+    GF_ASSERT(hostname);
+    GF_ASSERT(name);
+    xlator_t *this = THIS;
+    GF_ASSERT(this);
 
-        peer_hostname = GF_CALLOC (1, sizeof (*peer_hostname),
-                                   gf_gld_mt_peer_hostname_t);
+    peer_hostname = GF_CALLOC(1, sizeof(*peer_hostname),
+                              gf_gld_mt_peer_hostname_t);
 
-        if (!peer_hostname)
-                goto out;
+    if (!peer_hostname) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_NO_MEMORY, NULL);
+        goto out;
+    }
 
-        peer_hostname->hostname = gf_strdup (hostname);
-        CDS_INIT_LIST_HEAD (&peer_hostname->hostname_list);
+    peer_hostname->hostname = gf_strdup(hostname);
+    CDS_INIT_LIST_HEAD(&peer_hostname->hostname_list);
 
-        *name = peer_hostname;
-        ret = 0;
+    *name = peer_hostname;
+    ret = 0;
 
 out:
-        gf_log ("glusterd", GF_LOG_DEBUG, "Returning %d", ret);
-        return ret;
+    gf_msg_debug("glusterd", 0, "Returning %d", ret);
+    return ret;
 }
 
 void
-glusterd_peer_hostname_free (glusterd_peer_hostname_t *name)
+glusterd_peer_hostname_free(glusterd_peer_hostname_t *name)
 {
-        if (!name)
-                return;
+    if (!name)
+        return;
 
-        cds_list_del_init (&name->hostname_list);
+    cds_list_del_init(&name->hostname_list);
 
-        GF_FREE (name->hostname);
-        name->hostname = NULL;
+    GF_FREE(name->hostname);
+    name->hostname = NULL;
 
-        GF_FREE (name);
+    GF_FREE(name);
 
-        return;
+    return;
 }
 
 gf_boolean_t
-gd_peer_has_address (glusterd_peerinfo_t *peerinfo, const char *address)
+gd_peer_has_address(glusterd_peerinfo_t *peerinfo, const char *address)
 {
-        gf_boolean_t ret = _gf_false;
-        glusterd_peer_hostname_t *hostname = NULL;
+    glusterd_peer_hostname_t *hostname = NULL;
 
-        GF_VALIDATE_OR_GOTO ("glusterd", (peerinfo != NULL), out);
-        GF_VALIDATE_OR_GOTO ("glusterd", (address != NULL), out);
+    GF_VALIDATE_OR_GOTO("glusterd", (peerinfo != NULL), out);
+    GF_VALIDATE_OR_GOTO("glusterd", (address != NULL), out);
 
-        cds_list_for_each_entry (hostname, &peerinfo->hostnames,
-                                 hostname_list) {
-                if (strcmp (hostname->hostname, address) == 0) {
-                        ret = _gf_true;
-                        break;
-                }
+    cds_list_for_each_entry(hostname, &peerinfo->hostnames, hostname_list)
+    {
+        if (strcmp(hostname->hostname, address) == 0) {
+            return _gf_true;
         }
+    }
 
 out:
-        return ret;
+    return _gf_false;
 }
 
 int
-gd_add_address_to_peer (glusterd_peerinfo_t *peerinfo, const char *address)
+gd_add_address_to_peer(glusterd_peerinfo_t *peerinfo, const char *address)
 {
+    int ret = -1;
+    glusterd_peer_hostname_t *hostname = NULL;
 
-        int ret = -1;
-        glusterd_peer_hostname_t *hostname = NULL;
-
-        GF_VALIDATE_OR_GOTO ("glusterd", (peerinfo != NULL), out);
-        GF_VALIDATE_OR_GOTO ("glusterd", (address != NULL), out);
+    GF_VALIDATE_OR_GOTO("glusterd", (peerinfo != NULL), out);
+    GF_VALIDATE_OR_GOTO("glusterd", (address != NULL), out);
 
-        if (gd_peer_has_address (peerinfo, address)) {
-                ret = 0;
-                goto out;
-        }
+    if (gd_peer_has_address(peerinfo, address)) {
+        ret = 0;
+        goto out;
+    }
 
-        ret = glusterd_peer_hostname_new (address, &hostname);
-        if (ret)
-                goto out;
+    ret = glusterd_peer_hostname_new(address, &hostname);
+    if (ret)
+        goto out;
 
-        cds_list_add_tail_rcu (&hostname->hostname_list, &peerinfo->hostnames);
+    cds_list_add_tail_rcu(&hostname->hostname_list, &peerinfo->hostnames);
 
-        ret = 0;
+    ret = 0;
 out:
-        return ret;
+    return ret;
 }
 
 /* gd_add_friend_to_dict() adds details of @friend into @dict with the given
@@ -516,193 +655,78 @@ out:
  * is >= GD_OP_VERSION_3_6_0
  */
 int
-gd_add_friend_to_dict (glusterd_peerinfo_t *friend, dict_t *dict,
-                       const char *prefix)
+gd_add_friend_to_dict(glusterd_peerinfo_t *friend, dict_t *dict,
+                      const char *prefix)
 {
-        int                       ret      = -1;
-        xlator_t                 *this     = NULL;
-        glusterd_conf_t          *conf     = NULL;
-        char                      key[100] = {0,};
-        glusterd_peer_hostname_t *address  = NULL;
-        int                       count    = 0;
-
-        this = THIS;
-        GF_VALIDATE_OR_GOTO ("glusterd", (this != NULL), out);
-
-        conf = this->private;
-        GF_VALIDATE_OR_GOTO (this->name, (conf != NULL), out);
+    int ret = -1;
+    xlator_t *this = NULL;
+    glusterd_conf_t *conf = NULL;
+    char key[100] = {
+        0,
+    };
+    glusterd_peer_hostname_t *address = NULL;
+    int count = 0;
+
+    this = THIS;
+    GF_VALIDATE_OR_GOTO("glusterd", (this != NULL), out);
+
+    conf = this->private;
+    GF_VALIDATE_OR_GOTO(this->name, (conf != NULL), out);
+
+    GF_VALIDATE_OR_GOTO(this->name, (friend != NULL), out);
+    GF_VALIDATE_OR_GOTO(this->name, (dict != NULL), out);
+    GF_VALIDATE_OR_GOTO(this->name, (prefix != NULL), out);
+
+    snprintf(key, sizeof(key), "%s.uuid", prefix);
+    ret = dict_set_dynstr_with_alloc(dict, key, uuid_utoa(friend->uuid));
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+               "Failed to set key %s in dict", key);
+        goto out;
+    }
+
+    /* Setting the first hostname from the list with this key for backward
+     * compatibility
+     */
+    snprintf(key, sizeof(key), "%s.hostname", prefix);
+    address = cds_list_entry(&friend->hostnames, glusterd_peer_hostname_t,
+                             hostname_list);
+    ret = dict_set_dynstr_with_alloc(dict, key, address->hostname);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+               "Failed to set key %s in dict", key);
+        goto out;
+    }
+
+    if (conf->op_version < GD_OP_VERSION_3_6_0) {
+        ret = 0;
+        goto out;
+    }
 
-        GF_VALIDATE_OR_GOTO (this->name, (friend != NULL), out);
-        GF_VALIDATE_OR_GOTO (this->name, (dict != NULL), out);
-        GF_VALIDATE_OR_GOTO (this->name, (prefix != NULL), out);
+    address = NULL;
+    count = 0;
+    cds_list_for_each_entry(address, &friend->hostnames, hostname_list)
+    {
+        GF_VALIDATE_OR_GOTO(this->name, (address != NULL), out);
 
-        snprintf (key, sizeof (key), "%s.uuid", prefix);
-        ret = dict_set_dynstr_with_alloc (dict, key, uuid_utoa (friend->uuid));
+        snprintf(key, sizeof(key), "%s.hostname%d", prefix, count);
+        ret = dict_set_dynstr_with_alloc(dict, key, address->hostname);
         if (ret) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "Failed to set key %s in dict", key);
-                goto out;
-        }
-
-        /* Setting the first hostname from the list with this key for backward
-         * compatibility
-         */
-        memset (key, 0, sizeof (key));
-        snprintf (key, sizeof (key), "%s.hostname", prefix);
-        address = cds_list_entry (&friend->hostnames, glusterd_peer_hostname_t,
-                                  hostname_list);
-        if (!address) {
-                ret = -1;
-                gf_log (this->name, GF_LOG_ERROR, "Could not retrieve first "
-                        "address for peer");
-                goto out;
-        }
-        ret = dict_set_dynstr_with_alloc (dict, key, address->hostname);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "Failed to set key %s in dict", key);
-                goto out;
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+                   "Failed to set key %s in dict", key);
+            goto out;
         }
-
-        if (conf->op_version < GD_OP_VERSION_3_6_0) {
-                ret = 0;
-                goto out;
-        }
-
-        address = NULL;
-        count = 0;
-        cds_list_for_each_entry (address, &friend->hostnames, hostname_list) {
-                GF_VALIDATE_OR_GOTO (this->name, (address != NULL), out);
-
-                memset (key, 0, sizeof (key));
-                snprintf (key, sizeof (key), "%s.hostname%d", prefix, count);
-                ret = dict_set_dynstr_with_alloc (dict, key, address->hostname);
-                if (ret) {
-                        gf_log (this->name, GF_LOG_ERROR,
-                                        "Failed to set key %s in dict", key);
-                        goto out;
-                }
-                count++;
-        }
-        memset (key, 0, sizeof (key));
-        snprintf (key, sizeof (key), "%s.address-count", prefix);
-        ret = dict_set_int32 (dict, key, count);
-        if (ret)
-                gf_log (this->name, GF_LOG_ERROR,
-                        "Failed to set key %s in dict", key);
+        count++;
+    }
+    ret = snprintf(key, sizeof(key), "%s.address-count", prefix);
+    ret = dict_set_int32n(dict, key, ret, count);
+    if (ret)
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+               "Failed to set key %s in dict", key);
 
 out:
-        gf_log (this ? this->name : "glusterd", GF_LOG_DEBUG, "Returning %d",
-                ret);
-        return ret;
-}
-
-/* gd_peerinfo_find_from_hostname iterates over all the addresses saved for each
- * peer and matches it to @hoststr.
- * Returns the matched peer if found else returns NULL
- */
-glusterd_peerinfo_t *
-gd_peerinfo_find_from_hostname (const char *hoststr)
-{
-        xlator_t                 *this    = NULL;
-        glusterd_conf_t          *priv    = NULL;
-        glusterd_peerinfo_t      *peer    = NULL;
-        glusterd_peerinfo_t      *found   = NULL;
-        glusterd_peer_hostname_t *tmphost = NULL;
-
-        this = THIS;
-        GF_ASSERT (this != NULL);
-        priv = this->private;
-        GF_VALIDATE_OR_GOTO (this->name, (priv != NULL), out);
-
-        GF_VALIDATE_OR_GOTO (this->name, (hoststr != NULL), out);
-
-        rcu_read_lock ();
-        cds_list_for_each_entry_rcu (peer, &priv->peers, uuid_list) {
-                cds_list_for_each_entry_rcu (tmphost, &peer->hostnames,
-                                             hostname_list) {
-                        if (!strncasecmp (tmphost->hostname, hoststr, 1024)) {
-                                gf_log (this->name, GF_LOG_DEBUG,
-                                        "Friend %s found.. state: %d",
-                                        tmphost->hostname, peer->state.state);
-                                found = peer; /* Probably needs to be
-                                                 dereferenced*/
-                                goto unlock;
-                        }
-                }
-        }
-unlock:
-        rcu_read_unlock ();
-out:
-        return found;
-}
-
-/* gd_peerinfo_find_from_addrinfo iterates over all the addresses saved for each
- * peer, resolves them and compares them to @addr.
- *
- *
- * NOTE: As getaddrinfo is a blocking call and is being performed multiple times
- * in this function, it could lead to the calling thread to be blocked for
- * significant amounts of time.
- *
- * Returns the matched peer if found else returns NULL
- */
-glusterd_peerinfo_t *
-gd_peerinfo_find_from_addrinfo (const struct addrinfo *addr)
-{
-        xlator_t                 *this    = NULL;
-        glusterd_conf_t          *conf    = NULL;
-        glusterd_peerinfo_t      *peer    = NULL;
-        glusterd_peerinfo_t      *found   = NULL;
-        glusterd_peer_hostname_t *address = NULL;
-        int                       ret     = 0;
-        struct addrinfo          *paddr   = NULL;
-        struct addrinfo          *tmp     = NULL;
-
-        this = THIS;
-        GF_ASSERT (this != NULL);
-        conf = this->private;
-        GF_VALIDATE_OR_GOTO (this->name, (conf != NULL), out);
-
-        GF_VALIDATE_OR_GOTO (this->name, (addr != NULL), out);
-
-        rcu_read_lock ();
-        cds_list_for_each_entry_rcu (peer, &conf->peers, uuid_list) {
-                cds_list_for_each_entry_rcu (address, &peer->hostnames,
-                                             hostname_list) {
-                        /* TODO: Cache the resolved addrinfos to improve
-                         * performance
-                         */
-                        ret = getaddrinfo (address->hostname, NULL, NULL,
-                                           &paddr);
-                        if (ret) {
-                                /* Don't fail if getaddrinfo fails, continue
-                                 * onto the next address
-                                 */
-                                gf_log (this->name, GF_LOG_TRACE,
-                                        "getaddrinfo for %s failed (%s)",
-                                        address->hostname, gai_strerror (ret));
-                                ret = 0;
-                                continue;
-                        }
-
-                        for (tmp = paddr; tmp != NULL; tmp = tmp->ai_next) {
-                                if (gf_compare_sockaddr (addr->ai_addr,
-                                                         tmp->ai_addr)) {
-                                        found = peer; /* (de)referenced? */
-                                        break;
-                                }
-                        }
-
-                        freeaddrinfo (paddr);
-                        if (found)
-                                goto unlock;
-                }
-        }
-unlock:
-        rcu_read_unlock ();
-out:
-        return found;
+    gf_msg_debug(this ? this->name : "glusterd", 0, "Returning %d", ret);
+    return ret;
 }
 
 /* gd_update_peerinfo_from_dict will update the hostnames for @peerinfo from
@@ -710,82 +734,87 @@ out:
  * Returns 0 on success and -1 on failure.
  */
 int
-gd_update_peerinfo_from_dict (glusterd_peerinfo_t *peerinfo, dict_t *dict,
-                              const char *prefix)
+gd_update_peerinfo_from_dict(glusterd_peerinfo_t *peerinfo, dict_t *dict,
+                             const char *prefix)
 {
-        int                  ret      = -1;
-        xlator_t            *this     = NULL;
-        glusterd_conf_t     *conf     = NULL;
-        char                 key[100] = {0,};
-        char                *hostname = NULL;
-        int                  count    = 0;
-        int                  i        = 0;
-
-        this = THIS;
-        GF_ASSERT (this != NULL);
-
-        conf = this->private;
-        GF_VALIDATE_OR_GOTO (this->name, (conf != NULL), out);
-
-        GF_VALIDATE_OR_GOTO (this->name, (peerinfo != NULL), out);
-        GF_VALIDATE_OR_GOTO (this->name, (dict != NULL), out);
-        GF_VALIDATE_OR_GOTO (this->name, (prefix != NULL), out);
-
-        memset (key, 0, sizeof (key));
-        snprintf (key, sizeof (key), "%s.hostname", prefix);
-        ret = dict_get_str (dict, key, &hostname);
+    int ret = -1;
+    xlator_t *this = NULL;
+    glusterd_conf_t *conf = NULL;
+    char key[100] = {
+        0,
+    };
+    char *hostname = NULL;
+    int count = 0;
+    int i = 0;
+
+    this = THIS;
+    GF_ASSERT(this != NULL);
+
+    conf = this->private;
+    GF_VALIDATE_OR_GOTO(this->name, (conf != NULL), out);
+
+    GF_VALIDATE_OR_GOTO(this->name, (peerinfo != NULL), out);
+    GF_VALIDATE_OR_GOTO(this->name, (dict != NULL), out);
+    GF_VALIDATE_OR_GOTO(this->name, (prefix != NULL), out);
+
+    ret = snprintf(key, sizeof(key), "%s.hostname", prefix);
+    ret = dict_get_strn(dict, key, ret, &hostname);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+               "Key %s not present in "
+               "dictionary",
+               key);
+        goto out;
+    }
+    ret = gd_add_address_to_peer(peerinfo, hostname);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_ADD_ADDRESS_TO_PEER_FAIL,
+               "Could not add address to peer");
+        goto out;
+    }
+    /* Also set peerinfo->hostname to the first address */
+    if (peerinfo->hostname != NULL)
+        GF_FREE(peerinfo->hostname);
+    peerinfo->hostname = gf_strdup(hostname);
+
+    if (conf->op_version < GD_OP_VERSION_3_6_0) {
+        ret = 0;
+        goto out;
+    }
+
+    ret = snprintf(key, sizeof(key), "%s.address-count", prefix);
+    ret = dict_get_int32n(dict, key, ret, &count);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+               "Key %s not present in "
+               "dictionary",
+               key);
+        goto out;
+    }
+    hostname = NULL;
+    for (i = 0; i < count; i++) {
+        ret = snprintf(key, sizeof(key), "%s.hostname%d", prefix, i);
+        ret = dict_get_strn(dict, key, ret, &hostname);
         if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "Key %s not present in "
-                        "dictionary", key);
-                goto out;
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+                   "Key %s not present "
+                   "in dictionary",
+                   key);
+            goto out;
         }
-        ret = gd_add_address_to_peer (peerinfo, hostname);
+        ret = gd_add_address_to_peer(peerinfo, hostname);
         if (ret) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "Could not add address to peer");
-                goto out;
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_ADD_ADDRESS_TO_PEER_FAIL,
+                   "Could not add address to peer");
+            goto out;
         }
-        /* Also set peerinfo->hostname to the first address */
-        if (peerinfo->hostname != NULL)
-                GF_FREE (peerinfo->hostname);
-        peerinfo->hostname = gf_strdup (hostname);
 
-        if (conf->op_version < GD_OP_VERSION_3_6_0) {
-                ret = 0;
-                goto out;
-        }
-
-        memset (key, 0, sizeof (key));
-        snprintf (key, sizeof (key), "%s.address-count", prefix);
-        ret = dict_get_int32 (dict, key, &count);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "Key %s not present in "
-                        "dictionary", key);
-                goto out;
-        }
         hostname = NULL;
-        for (i = 0; i < count; i++) {
-                memset (key, 0, sizeof (key));
-                snprintf (key, sizeof (key), "%s.hostname%d",prefix, i);
-                ret = dict_get_str (dict, key, &hostname);
-                if (ret) {
-                        gf_log (this->name, GF_LOG_ERROR, "Key %s not present "
-                                "in dictionary", key);
-                        goto out;
-                }
-                ret = gd_add_address_to_peer (peerinfo, hostname);
-                if (ret) {
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "Could not add address to peer");
-                        goto out;
-                }
-
-                hostname = NULL;
-        }
+    }
 
 out:
-        gf_log (this->name, GF_LOG_DEBUG, "Returning %d", ret);
-        return ret;
+    gf_msg_debug(this->name, 0, "Returning %d", ret);
+    return ret;
 }
 
 /* gd_peerinfo_from_dict creates a peerinfo object from details of peer with
@@ -794,149 +823,236 @@ out:
  * failure.
  */
 glusterd_peerinfo_t *
-gd_peerinfo_from_dict (dict_t *dict, const char *prefix)
+gd_peerinfo_from_dict(dict_t *dict, const char *prefix)
 {
-        int                  ret      = -1;
-        xlator_t            *this     = NULL;
-        glusterd_conf_t     *conf     = NULL;
-        glusterd_peerinfo_t *new_peer = NULL;
-        char                 key[100] = {0,};
-        char                *uuid_str = NULL;
-
-        this = THIS;
-        GF_VALIDATE_OR_GOTO ("glusterd", (this != NULL), out);
-
-        conf = this->private;
-        GF_VALIDATE_OR_GOTO (this->name, (conf != NULL), out);
-
-        GF_VALIDATE_OR_GOTO (this->name, (dict != NULL), out);
-        GF_VALIDATE_OR_GOTO (this->name, (prefix != NULL), out);
-
-        new_peer = glusterd_peerinfo_new (GD_FRIEND_STATE_DEFAULT, NULL, NULL,
-                                          0);
-        if (new_peer == NULL) {
-                ret = -1;
-                gf_log (this->name, GF_LOG_ERROR, "Could not create peerinfo "
-                        "object");
-                goto out;
-        }
-
-        snprintf (key, sizeof (key), "%s.uuid", prefix);
-        ret = dict_get_str (dict, key, &uuid_str);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "Key %s not present in "
-                        "dictionary", key);
-                goto out;
-        }
-        gf_uuid_parse (uuid_str, new_peer->uuid);
-
-        ret = gd_update_peerinfo_from_dict (new_peer, dict, prefix);
+    int ret = -1;
+    xlator_t *this = NULL;
+    glusterd_conf_t *conf = NULL;
+    glusterd_peerinfo_t *new_peer = NULL;
+    char key[64] = {
+        0,
+    };
+    char *uuid_str = NULL;
+
+    this = THIS;
+    GF_VALIDATE_OR_GOTO("glusterd", (this != NULL), out);
+
+    conf = this->private;
+    GF_VALIDATE_OR_GOTO(this->name, (conf != NULL), out);
+
+    GF_VALIDATE_OR_GOTO(this->name, (dict != NULL), out);
+    GF_VALIDATE_OR_GOTO(this->name, (prefix != NULL), out);
+
+    new_peer = glusterd_peerinfo_new(GD_FRIEND_STATE_DEFAULT, NULL, NULL, 0);
+    if (new_peer == NULL) {
+        ret = -1;
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_PEERINFO_CREATE_FAIL,
+               "Could not create peerinfo "
+               "object");
+        goto out;
+    }
+
+    ret = snprintf(key, sizeof(key), "%s.uuid", prefix);
+    ret = dict_get_strn(dict, key, ret, &uuid_str);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+               "Key %s not present in "
+               "dictionary",
+               key);
+        goto out;
+    }
+    gf_uuid_parse(uuid_str, new_peer->uuid);
+
+    ret = gd_update_peerinfo_from_dict(new_peer, dict, prefix);
 
 out:
-        if ((ret != 0) && (new_peer != NULL)) {
-                glusterd_peerinfo_cleanup (new_peer);
-                new_peer = NULL;
-        }
+    if ((ret != 0) && (new_peer != NULL)) {
+        glusterd_peerinfo_cleanup(new_peer);
+        new_peer = NULL;
+    }
 
-        return new_peer;
+    return new_peer;
 }
 
-int
-gd_add_peer_hostnames_to_dict (glusterd_peerinfo_t *peerinfo, dict_t *dict,
-                               const char *prefix)
+static int
+gd_add_peer_hostnames_to_dict(glusterd_peerinfo_t *peerinfo, dict_t *dict,
+                              const char *prefix)
 {
-        int                       ret      = -1;
-        xlator_t                 *this     = NULL;
-        glusterd_conf_t          *conf     = NULL;
-        char                      key[256] = {0,};
-        glusterd_peer_hostname_t *addr     = NULL;
-        int                       count    = 0;
-
-        this = THIS;
-        GF_ASSERT (this != NULL);
+    int ret = -1;
+    xlator_t *this = NULL;
+    glusterd_conf_t *conf = NULL;
+    char key[64] = {
+        0,
+    };
+    glusterd_peer_hostname_t *addr = NULL;
+    int count = 0;
+
+    this = THIS;
+    GF_ASSERT(this != NULL);
+
+    conf = this->private;
+    GF_VALIDATE_OR_GOTO(this->name, (conf != NULL), out);
+
+    if (conf->op_version < GD_OP_VERSION_3_6_0) {
+        ret = 0;
+        goto out;
+    }
 
-        conf = this->private;
-        GF_VALIDATE_OR_GOTO (this->name, (conf != NULL), out);
+    GF_VALIDATE_OR_GOTO(this->name, (peerinfo != NULL), out);
+    GF_VALIDATE_OR_GOTO(this->name, (dict != NULL), out);
+    GF_VALIDATE_OR_GOTO(this->name, (prefix != NULL), out);
 
-        if (conf->op_version < GD_OP_VERSION_3_6_0) {
-                ret = 0;
-                goto out;
-        }
-
-        GF_VALIDATE_OR_GOTO (this->name, (peerinfo != NULL), out);
-        GF_VALIDATE_OR_GOTO (this->name, (dict != NULL), out);
-        GF_VALIDATE_OR_GOTO (this->name, (prefix != NULL), out);
-
-        cds_list_for_each_entry (addr, &peerinfo->hostnames, hostname_list) {
-                memset (key, 0, sizeof (key));
-                snprintf (key, sizeof (key), "%s.hostname%d", prefix, count);
-                ret = dict_set_dynstr_with_alloc (dict, key, addr->hostname);
-                if (ret)
-                        goto out;
-                count++;
+    cds_list_for_each_entry(addr, &peerinfo->hostnames, hostname_list)
+    {
+        snprintf(key, sizeof(key), "%s.hostname%d", prefix, count);
+        ret = dict_set_dynstr_with_alloc(dict, key, addr->hostname);
+        if (ret) {
+            gf_smsg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+                    "Key=%s", key, NULL);
+            goto out;
         }
+        count++;
+    }
 
-        memset (key, 0, sizeof (key));
-        snprintf (key, sizeof (key), "%s.hostname_count", prefix);
-        ret = dict_set_int32 (dict, key, count);
+    ret = snprintf(key, sizeof(key), "%s.hostname_count", prefix);
+    ret = dict_set_int32n(dict, key, ret, count);
 
 out:
-        return ret;
+    return ret;
 }
 
 int
-gd_add_peer_detail_to_dict (glusterd_peerinfo_t *peerinfo, dict_t *friends,
-                            int count)
+gd_add_peer_detail_to_dict(glusterd_peerinfo_t *peerinfo, dict_t *friends,
+                           int count)
 {
+    int ret = -1;
+    char key[32] = {
+        0,
+    };
+    int keylen;
+    char *peer_uuid_str = NULL;
+
+    xlator_t *this = THIS;
+    GF_ASSERT(this);
+    GF_ASSERT(peerinfo);
+    GF_ASSERT(friends);
+
+    peer_uuid_str = gd_peer_uuid_str(peerinfo);
+    keylen = snprintf(key, sizeof(key), "friend%d.uuid", count);
+    ret = dict_set_strn(friends, key, keylen, peer_uuid_str);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED, "Key=%s",
+                key, NULL);
+        goto out;
+    }
+
+    keylen = snprintf(key, sizeof(key), "friend%d.hostname", count);
+    ret = dict_set_strn(friends, key, keylen, peerinfo->hostname);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED, "Key=%s",
+                key, NULL);
+        goto out;
+    }
+
+    keylen = snprintf(key, sizeof(key), "friend%d.port", count);
+    ret = dict_set_int32n(friends, key, keylen, peerinfo->port);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED, "Key=%s",
+                key, NULL);
+        goto out;
+    }
+
+    keylen = snprintf(key, sizeof(key), "friend%d.stateId", count);
+    ret = dict_set_int32n(friends, key, keylen, peerinfo->state.state);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+                "Key=%s in dict", key, NULL);
+        goto out;
+    }
+
+    keylen = snprintf(key, sizeof(key), "friend%d.state", count);
+    ret = dict_set_strn(
+        friends, key, keylen,
+        glusterd_friend_sm_state_name_get(peerinfo->state.state));
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED, "key=%s",
+                key, NULL);
+        goto out;
+    }
+
+    keylen = snprintf(key, sizeof(key), "friend%d.connected", count);
+    ret = dict_set_int32n(friends, key, keylen, (int32_t)peerinfo->connected);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED, "Key=%s",
+                key, NULL);
+        goto out;
+    }
+
+    snprintf(key, sizeof(key), "friend%d", count);
+    ret = gd_add_peer_hostnames_to_dict(peerinfo, friends, key);
 
-        int             ret = -1;
-        char            key[256] = {0, };
-        char           *peer_uuid_str = NULL;
+out:
+    return ret;
+}
 
-        GF_ASSERT (peerinfo);
-        GF_ASSERT (friends);
+/* glusterd_peerinfo_find_by_generation searches for a peer which has the
+ * generation number @generation and if found returns the pointer to peerinfo
+ * object. Returns NULL otherwise.
+ */
+glusterd_peerinfo_t *
+glusterd_peerinfo_find_by_generation(uint32_t generation)
+{
+    glusterd_conf_t *priv = NULL;
+    glusterd_peerinfo_t *entry = NULL;
+    glusterd_peerinfo_t *found = NULL;
+    xlator_t *this = THIS;
+    glusterd_friend_sm_state_t state;
 
-        snprintf (key, sizeof (key), "friend%d.uuid", count);
-        peer_uuid_str = gd_peer_uuid_str (peerinfo);
-        ret = dict_set_str (friends, key, peer_uuid_str);
-        if (ret)
-                goto out;
+    GF_ASSERT(this);
 
-        memset (key, 0, sizeof (key));
-        snprintf (key, sizeof (key), "friend%d.hostname", count);
-        ret = dict_set_str (friends, key, peerinfo->hostname);
-        if (ret)
-                goto out;
+    priv = this->private;
 
-        memset (key, 0, sizeof (key));
-        snprintf (key, sizeof (key), "friend%d.port", count);
-        ret = dict_set_int32 (friends, key, peerinfo->port);
-        if (ret)
-                goto out;
+    GF_ASSERT(priv);
 
-        memset (key, 0, sizeof (key));
-        snprintf (key, sizeof (key), "friend%d.stateId", count);
-        ret = dict_set_int32 (friends, key, peerinfo->state.state);
-        if (ret)
-                goto out;
+    RCU_READ_LOCK;
+    cds_list_for_each_entry_rcu(entry, &priv->peers, uuid_list)
+    {
+        if (entry->generation == generation) {
+            found = entry; /* Probably should be rcu_dereferenced */
+            state = found->state.state;
+            break;
+        }
+    }
+    RCU_READ_UNLOCK;
+
+    if (found)
+        gf_msg_debug(this->name, 0, "Friend found... state: %s",
+                     glusterd_friend_sm_state_name_get(state));
+    else
+        gf_msg_debug(this->name, 0,
+                     "Friend with generation: %" PRIu32 ", not found",
+                     generation);
+    return found;
+}
 
-        memset (key, 0, sizeof (key));
-        snprintf (key, sizeof (key), "friend%d.state", count);
-        ret = dict_set_str (friends, key,
-                    glusterd_friend_sm_state_name_get(peerinfo->state.state));
-        if (ret)
-                goto out;
+int
+glusterd_get_peers_count()
+{
+    int count = 0;
+    xlator_t *this = NULL;
+    glusterd_conf_t *conf = NULL;
+    glusterd_peerinfo_t *peer = NULL;
 
-        memset (key, 0, sizeof (key));
-        snprintf (key, sizeof (key), "friend%d.connected", count);
-        ret = dict_set_int32 (friends, key, (int32_t)peerinfo->connected);
-        if (ret)
-                goto out;
+    this = THIS;
+    GF_VALIDATE_OR_GOTO("glusterd", this, out);
+
+    conf = this->private;
+    GF_VALIDATE_OR_GOTO(this->name, conf, out);
 
-        memset (key, 0, sizeof (key));
-        snprintf (key, sizeof (key), "friend%d", count);
-        ret = gd_add_peer_hostnames_to_dict (peerinfo, friends, key);
+    RCU_READ_LOCK;
+    cds_list_for_each_entry_rcu(peer, &conf->peers, uuid_list) count++;
+    RCU_READ_UNLOCK;
 
 out:
-        return ret;
+    return count;
 }
diff --git a/xlators/mgmt/glusterd/src/glusterd-peer-utils.h b/xlators/mgmt/glusterd/src/glusterd-peer-utils.h
index 3a1aee7cd15..fd254d57391 100644
--- a/xlators/mgmt/glusterd/src/glusterd-peer-utils.h
+++ b/xlators/mgmt/glusterd/src/glusterd-peer-utils.h
@@ -12,74 +12,71 @@
 #define _GLUSTERD_PEER_UTILS_H
 
 #include "glusterd.h"
-#include "glusterd-utils.h"
 
 int32_t
-glusterd_peerinfo_cleanup (glusterd_peerinfo_t *peerinfo);
+glusterd_peerinfo_cleanup(glusterd_peerinfo_t *peerinfo);
 
 glusterd_peerinfo_t *
-glusterd_peerinfo_find_by_hostname (const char *hoststr);
+glusterd_peerinfo_find_by_hostname(const char *hoststr);
 
 int
-glusterd_hostname_to_uuid (char *hostname, uuid_t uuid);
+glusterd_hostname_to_uuid(char *hostname, uuid_t uuid);
 
 glusterd_peerinfo_t *
-glusterd_peerinfo_find_by_uuid (uuid_t uuid);
+glusterd_peerinfo_find_by_uuid(uuid_t uuid);
 
 glusterd_peerinfo_t *
-glusterd_peerinfo_find (uuid_t uuid, const char *hostname);
+glusterd_peerinfo_find(uuid_t uuid, const char *hostname);
 
 glusterd_peerinfo_t *
-glusterd_peerinfo_new (glusterd_friend_sm_state_t state, uuid_t *uuid,
-                       const char *hostname, int port);
+glusterd_peerinfo_new(glusterd_friend_sm_state_t state, uuid_t *uuid,
+                      const char *hostname, int port);
 
 gf_boolean_t
-glusterd_chk_peers_connected_befriended (uuid_t skip_uuid);
+glusterd_chk_peers_connected_befriended(uuid_t skip_uuid);
 
 char *
-glusterd_uuid_to_hostname (uuid_t uuid);
+glusterd_uuid_to_hostname(uuid_t uuid);
 
-char*
-gd_peer_uuid_str (glusterd_peerinfo_t *peerinfo);
+char *
+gd_peer_uuid_str(glusterd_peerinfo_t *peerinfo);
+
+gf_boolean_t
+glusterd_are_all_peers_up();
 
 gf_boolean_t
-glusterd_are_vol_all_peers_up (glusterd_volinfo_t *volinfo,
-                               struct cds_list_head *peers,
-                               char **down_peerstr);
+glusterd_are_vol_all_peers_up(glusterd_volinfo_t *volinfo,
+                              struct cds_list_head *peers, char **down_peerstr);
 
 int32_t
-glusterd_peer_hostname_new (const char *hostname,
-                            glusterd_peer_hostname_t **name);
+glusterd_peer_hostname_new(const char *hostname,
+                           glusterd_peer_hostname_t **name);
 void
-glusterd_peer_hostname_free (glusterd_peer_hostname_t *name);
+glusterd_peer_hostname_free(glusterd_peer_hostname_t *name);
 
 gf_boolean_t
-gd_peer_has_address (glusterd_peerinfo_t *peerinfo, const char *address);
+gd_peer_has_address(glusterd_peerinfo_t *peerinfo, const char *address);
 
 int
-gd_add_address_to_peer (glusterd_peerinfo_t *peerinfo, const char *address);
+gd_add_address_to_peer(glusterd_peerinfo_t *peerinfo, const char *address);
 
 int
-gd_add_friend_to_dict (glusterd_peerinfo_t *friend, dict_t *dict,
-                       const char *prefix);
+gd_add_friend_to_dict(glusterd_peerinfo_t *friend, dict_t *dict,
+                      const char *prefix);
 
-glusterd_peerinfo_t *
-gd_peerinfo_find_from_hostname (const char *hoststr);
+int
+gd_update_peerinfo_from_dict(glusterd_peerinfo_t *peerinfo, dict_t *dict,
+                             const char *prefix);
 
 glusterd_peerinfo_t *
-gd_peerinfo_find_from_addrinfo (const struct addrinfo *addr);
+gd_peerinfo_from_dict(dict_t *dict, const char *prefix);
 
 int
-gd_update_peerinfo_from_dict (glusterd_peerinfo_t *peerinfo, dict_t *dict,
-                              const char *prefix);
-
+gd_add_peer_detail_to_dict(glusterd_peerinfo_t *peerinfo, dict_t *friends,
+                           int count);
 glusterd_peerinfo_t *
-gd_peerinfo_from_dict (dict_t *dict, const char *prefix);
+glusterd_peerinfo_find_by_generation(uint32_t generation);
 
 int
-gd_add_peer_hostnames_to_dict (glusterd_peerinfo_t *peerinfo, dict_t *dict,
-                               const char *prefix);
-int
-gd_add_peer_detail_to_dict (glusterd_peerinfo_t *peerinfo, dict_t *friends,
-                            int count);
+glusterd_get_peers_count();
 #endif /* _GLUSTERD_PEER_UTILS_H */
diff --git a/xlators/mgmt/glusterd/src/glusterd-pmap.c b/xlators/mgmt/glusterd/src/glusterd-pmap.c
index e4dacb80c4a..16ac628ab82 100644
--- a/xlators/mgmt/glusterd/src/glusterd-pmap.c
+++ b/xlators/mgmt/glusterd/src/glusterd-pmap.c
@@ -8,14 +8,10 @@
    cases as published by the Free Software Foundation.
 */
 
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
-
-#include "xlator.h"
-#include "glusterfs.h"
-#include "compat-errno.h"
+#include <glusterfs/xlator.h>
+#include <glusterfs/glusterfs.h>
+#include <glusterfs/syscall.h>
+#include <glusterfs/compat-errno.h>
 
 #include "glusterd.h"
 #include "glusterd-utils.h"
@@ -23,478 +19,648 @@
 #include "portmap-xdr.h"
 #include "xdr-generic.h"
 #include "protocol-common.h"
+#include "glusterd-messages.h"
 #include "rpcsvc.h"
 
 #include <sys/socket.h>
 #include <sys/types.h>
 #include <netinet/in.h>
 
-
-int
-pmap_port_isfree (int port)
+static int
+pmap_port_isfree(int port)
 {
-        struct sockaddr_in sin;
-        int                sock = -1;
-        int                ret = 0;
+    struct sockaddr_in sin;
+    int sock = -1;
+    int ret = 0;
 
-        memset (&sin, 0, sizeof (sin));
-        sin.sin_family = PF_INET;
-        sin.sin_port = hton16 (port);
+    memset(&sin, 0, sizeof(sin));
+    sin.sin_family = PF_INET;
+    sin.sin_port = hton16(port);
 
-        sock = socket (PF_INET, SOCK_STREAM, 0);
-        if (sock == -1)
-                return -1;
+    sock = socket(PF_INET, SOCK_STREAM, 0);
+    if (sock == -1)
+        return -1;
 
-        ret = bind (sock, (struct sockaddr *)&sin, sizeof (sin));
-        close (sock);
+    ret = bind(sock, (struct sockaddr *)&sin, sizeof(sin));
+    sys_close(sock);
 
-        return (ret == 0) ? 1 : 0;
+    return (ret == 0) ? 1 : 0;
 }
 
-
 static struct pmap_registry *
-pmap_registry_new (xlator_t *this)
+pmap_registry_new(xlator_t *this)
 {
-        struct pmap_registry *pmap = NULL;
-        int                   i = 0;
-
-        pmap = CALLOC (sizeof (*pmap), 1);
-        if (!pmap)
-                return NULL;
-
-        for (i = 0; i < 65536; i++) {
-                if (pmap_port_isfree (i))
-                        pmap->ports[i].type = GF_PMAP_PORT_FREE;
-                else
-                        pmap->ports[i].type = GF_PMAP_PORT_FOREIGN;
-        }
-
-        pmap->base_port = pmap->last_alloc =
-                ((glusterd_conf_t *)(this->private))->base_port;
-
-        return pmap;
+    struct pmap_registry *pmap = NULL;
+    int i = 0;
+
+    pmap = CALLOC(sizeof(*pmap), 1);
+    if (!pmap)
+        return NULL;
+
+    pmap->base_port = pmap->last_alloc = ((glusterd_conf_t *)(this->private))
+                                             ->base_port;
+    pmap->max_port = ((glusterd_conf_t *)(this->private))->max_port;
+    for (i = pmap->base_port; i <= pmap->max_port; i++) {
+        if (pmap_port_isfree(i))
+            pmap->ports[i].type = GF_PMAP_PORT_FREE;
+        else
+            pmap->ports[i].type = GF_PMAP_PORT_FOREIGN;
+    }
+
+    return pmap;
 }
 
-
 struct pmap_registry *
-pmap_registry_get (xlator_t *this)
+pmap_registry_get(xlator_t *this)
 {
-        glusterd_conf_t      *priv = NULL;
-        struct pmap_registry *pmap = NULL;
+    glusterd_conf_t *priv = NULL;
+    struct pmap_registry *pmap = NULL;
 
-        priv = this->private;
+    priv = this->private;
 
-        pmap = priv->pmap;
-        if (!pmap) {
-                pmap = pmap_registry_new (this);
-                if (!pmap)
-                        return NULL;
-                priv->pmap = pmap;
-        }
-
-        return pmap;
-}
-
-
-static char*
-nextword (char *str)
-{
-        while (*str && !isspace (*str))
-                str++;
-        while (*str && isspace (*str))
-                str++;
+    pmap = priv->pmap;
+    if (!pmap) {
+        pmap = pmap_registry_new(this);
+        if (!pmap)
+            return NULL;
+        priv->pmap = pmap;
+    }
 
-        return str;
+    return pmap;
 }
 
+/*
+ * The "destroy" argument avoids a double search in pmap_registry_remove - one
+ * to find the entry in the table, and the other to find the particular
+ * brickname within that entry (which might cover multiple bricks).  We do the
+ * actual deletion here by "whiting out" the brick name with spaces.  It's up
+ * to pmap_registry_remove to figure out what to do from there.
+ */
 int
-pmap_registry_search (xlator_t *this, const char *brickname,
-                      gf_pmap_port_type_t type)
+pmap_registry_search(xlator_t *this, const char *brickname,
+                     gf_pmap_port_type_t type, gf_boolean_t destroy)
 {
-        struct pmap_registry *pmap = NULL;
-        int                   p = 0;
-        char                 *brck = NULL;
-        char                 *nbrck = NULL;
-
-        pmap = pmap_registry_get (this);
-
-        for (p = pmap->base_port; p <= pmap->last_alloc; p++) {
-                if (!pmap->ports[p].brickname || pmap->ports[p].type != type)
-                        continue;
-
-                for (brck = pmap->ports[p].brickname;;) {
-                        nbrck = strtail (brck, brickname);
-                        if (nbrck && (!*nbrck || isspace (*nbrck)))
-                                return p;
-                        brck = nextword (brck);
-                        if (!*brck)
-                                break;
+    struct pmap_registry *pmap = NULL;
+    int p = 0;
+    char *brck = NULL;
+    size_t i;
+
+    pmap = pmap_registry_get(this);
+
+    for (p = pmap->last_alloc; p >= pmap->base_port; p--) {
+        if (!pmap->ports[p].brickname || pmap->ports[p].type != type)
+            continue;
+
+        brck = pmap->ports[p].brickname;
+        for (;;) {
+            for (i = 0; brck[i] && !isspace(brck[i]); ++i)
+                ;
+            if (i == 0 && brck[i] == '\0')
+                break;
+
+            if (strncmp(brck, brickname, i) == 0) {
+                /*
+                 * Without this check, we'd break when brck
+                 * is merely a substring of brickname.
+                 */
+                if (brickname[i] == '\0') {
+                    if (destroy)
+                        do {
+                            *(brck++) = ' ';
+                        } while (--i);
+                    return p;
                 }
+            }
+
+            brck += i;
+
+            /*
+             * Skip over *any* amount of whitespace, including
+             * none (if we're already at the end of the string).
+             */
+            while (isspace(*brck))
+                ++brck;
+            /*
+             * We're either at the end of the string (which will be
+             * handled above strncmp on the next iteration) or at
+             * the next non-whitespace substring (which will be
+             * handled by strncmp itself).
+             */
         }
+    }
 
-        return 0;
+    return 0;
 }
 
-int
-pmap_registry_search_by_xprt (xlator_t *this, void *xprt,
-                              gf_pmap_port_type_t type)
+static int
+pmap_registry_search_by_xprt(xlator_t *this, void *xprt,
+                             gf_pmap_port_type_t type)
 {
-        struct pmap_registry *pmap = NULL;
-        int                   p    = 0;
-        int                   port = 0;
-
-        pmap = pmap_registry_get (this);
-
-        for (p = pmap->base_port; p <= pmap->last_alloc; p++) {
-                if (!pmap->ports[p].xprt)
-                        continue;
-                if (pmap->ports[p].xprt == xprt &&
-                    pmap->ports[p].type == type) {
-                        port = p;
-                        break;
-                }
+    struct pmap_registry *pmap = NULL;
+    int p = 0;
+    int port = 0;
+
+    pmap = pmap_registry_get(this);
+
+    for (p = pmap->last_alloc; p >= pmap->base_port; p--) {
+        if (!pmap->ports[p].xprt)
+            continue;
+        if (pmap->ports[p].xprt == xprt) {
+            if (pmap->ports[p].type == type || type == GF_PMAP_PORT_ANY) {
+                port = p;
+                break;
+            }
         }
+    }
 
-        return port;
+    return port;
 }
 
-
-char *
-pmap_registry_search_by_port (xlator_t *this, int port)
+static char *
+pmap_registry_search_by_port(xlator_t *this, int port)
 {
-        struct pmap_registry *pmap = NULL;
-        char *brickname = NULL;
+    struct pmap_registry *pmap = NULL;
+    char *brickname = NULL;
+    int max_port = 0;
 
-        if (port > 65535)
-                goto out;
+    max_port = ((glusterd_conf_t *)(this->private))->max_port;
+    if (port > max_port)
+        goto out;
 
-        pmap = pmap_registry_get (this);
+    pmap = pmap_registry_get(this);
 
-        if (pmap->ports[port].type == GF_PMAP_PORT_BRICKSERVER)
-                brickname = pmap->ports[port].brickname;
+    if (pmap->ports[port].type == GF_PMAP_PORT_BRICKSERVER)
+        brickname = pmap->ports[port].brickname;
 
 out:
-        return brickname;
+    return brickname;
 }
 
-
 int
-pmap_registry_alloc (xlator_t *this)
+pmap_registry_alloc(xlator_t *this)
 {
-        struct pmap_registry *pmap = NULL;
-        int                   p = 0;
-        int                   port = 0;
-
-        pmap = pmap_registry_get (this);
-
-        for (p = pmap->last_alloc; p < 65535; p++) {
-                if (pmap->ports[p].type != GF_PMAP_PORT_FREE)
-                        continue;
-
-                if (pmap_port_isfree (p)) {
-                        pmap->ports[p].type = GF_PMAP_PORT_LEASED;
-                        port = p;
-                        break;
-                }
+    struct pmap_registry *pmap = NULL;
+    int p = 0;
+    int port = 0;
+
+    pmap = pmap_registry_get(this);
+
+    for (p = pmap->base_port; p <= pmap->max_port; p++) {
+        /* GF_PMAP_PORT_FOREIGN may be freed up ? */
+        if ((pmap->ports[p].type == GF_PMAP_PORT_FREE) ||
+            (pmap->ports[p].type == GF_PMAP_PORT_FOREIGN)) {
+            if (pmap_port_isfree(p)) {
+                pmap->ports[p].type = GF_PMAP_PORT_LEASED;
+                port = p;
+                break;
+            }
         }
+    }
 
-        if (port)
-                pmap->last_alloc = port;
+    if (port > pmap->last_alloc)
+        pmap->last_alloc = port;
 
-        return port;
+    return port;
 }
 
+/* pmap_assign_port does a pmap_registry_remove followed by pmap_registry_alloc,
+ * the reason for the former is to ensure we don't end up with stale ports
+ */
 int
-pmap_registry_bind (xlator_t *this, int port, const char *brickname,
-                    gf_pmap_port_type_t type, void *xprt)
+pmap_assign_port(xlator_t *this, int old_port, const char *path)
 {
-        struct pmap_registry *pmap = NULL;
-        int                   p = 0;
-
-        pmap = pmap_registry_get (this);
+    int ret = -1;
+    int new_port = 0;
+
+    if (old_port) {
+        ret = pmap_registry_remove(this, 0, path, GF_PMAP_PORT_BRICKSERVER,
+                                   NULL, _gf_false);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_WARNING, GD_MSG_PMAP_REGISTRY_REMOVE_FAIL,
+                   0,
+                   "Failed to"
+                   "remove pmap registry for older signin for path"
+                   " %s",
+                   path);
+        }
+    }
+    new_port = pmap_registry_alloc(this);
+    return new_port;
+}
 
-        if (port > 65535)
-                goto out;
+int
+pmap_registry_bind(xlator_t *this, int port, const char *brickname,
+                   gf_pmap_port_type_t type, void *xprt)
+{
+    struct pmap_registry *pmap = NULL;
+    int p = 0;
 
-        p = port;
-        pmap->ports[p].type = type;
-        free (pmap->ports[p].brickname);
-        pmap->ports[p].brickname = strdup (brickname);
-        pmap->ports[p].type = type;
-        pmap->ports[p].xprt = xprt;
+    pmap = pmap_registry_get(this);
 
-        gf_log ("pmap", GF_LOG_INFO, "adding brick %s on port %d",
-                brickname, port);
+    if (port > pmap->max_port)
+        goto out;
 
-        if (pmap->last_alloc < p)
-                pmap->last_alloc = p;
+    p = port;
+    if (pmap->ports[p].type == GF_PMAP_PORT_FREE) {
+        /* Because of some crazy race in volume start code path because
+         * of friend handshaking with volumes with quorum enabled we
+         * might end up into a situation where glusterd would start a
+         * brick and get a disconnect and then immediately try to start
+         * the same brick instance based on another friend update
+         * request. And then if for the very first brick even if the
+         * process doesn't come up at the end sign in event gets sent
+         * and we end up having two duplicate portmap entries for the
+         * same brick. Since in brick start we mark the previous port as
+         * free, its better to consider a sign in request as no op if
+         * the corresponding port type is marked as free
+         */
+        goto out;
+    }
+    if (pmap->ports[p].brickname) {
+        char *tmp = pmap->ports[p].brickname;
+        asprintf(&pmap->ports[p].brickname, "%s %s", tmp, brickname);
+        free(tmp);
+    } else {
+        pmap->ports[p].brickname = strdup(brickname);
+    }
+    pmap->ports[p].type = type;
+    pmap->ports[p].xprt = xprt;
+
+    gf_msg("pmap", GF_LOG_INFO, 0, GD_MSG_BRICK_ADD,
+           "adding brick %s on port %d", brickname, port);
+
+    if (pmap->last_alloc < p)
+        pmap->last_alloc = p;
 out:
-        return 0;
+    return 0;
 }
 
 int
-pmap_registry_remove (xlator_t *this, int port, const char *brickname,
-                      gf_pmap_port_type_t type, void *xprt)
+pmap_registry_extend(xlator_t *this, int port, const char *brickname)
 {
-        struct pmap_registry *pmap = NULL;
-        int                   p = 0;
-        glusterd_conf_t      *priv = NULL;
-
-        priv = this->private;
-        pmap = priv->pmap;
-        if (!pmap)
-                goto out;
-
-        if (port) {
-                if (port > 65535)
-                        goto out;
-
-                p = port;
-                goto remove;
+    struct pmap_registry *pmap = NULL;
+    char *old_bn;
+    char *new_bn;
+    size_t bn_len;
+    char *entry;
+    int found = 0;
+
+    pmap = pmap_registry_get(this);
+
+    if (port > pmap->max_port) {
+        return -1;
+    }
+
+    switch (pmap->ports[port].type) {
+        case GF_PMAP_PORT_LEASED:
+        case GF_PMAP_PORT_BRICKSERVER:
+            break;
+        default:
+            return -1;
+    }
+
+    old_bn = pmap->ports[port].brickname;
+    if (old_bn) {
+        bn_len = strlen(brickname);
+        entry = strstr(old_bn, brickname);
+        while (entry) {
+            found = 1;
+            if ((entry != old_bn) && (entry[-1] != ' ')) {
+                found = 0;
+            }
+            if ((entry[bn_len] != ' ') && (entry[bn_len] != '\0')) {
+                found = 0;
+            }
+            if (found) {
+                return 0;
+            }
+            entry = strstr(entry + bn_len, brickname);
         }
+        asprintf(&new_bn, "%s %s", old_bn, brickname);
+    } else {
+        new_bn = strdup(brickname);
+    }
 
-        if (brickname && strchr (brickname, '/')) {
-                p = pmap_registry_search (this, brickname, type);
-                if (p)
-                        goto remove;
-        }
-
-        if (xprt) {
-                p = pmap_registry_search_by_xprt (this, xprt, type);
-                if (p)
-                        goto remove;
-        }
-
-        goto out;
-remove:
-        gf_log ("pmap", GF_LOG_INFO, "removing brick %s on port %d",
-                pmap->ports[p].brickname, p);
-
-        free (pmap->ports[p].brickname);
+    if (!new_bn) {
+        return -1;
+    }
 
-        pmap->ports[p].brickname = NULL;
-        pmap->ports[p].xprt = NULL;
+    pmap->ports[port].brickname = new_bn;
+    free(old_bn);
 
-out:
-        return 0;
+    return 0;
 }
 
 int
-__gluster_pmap_portbybrick (rpcsvc_request_t *req)
+pmap_registry_remove(xlator_t *this, int port, const char *brickname,
+                     gf_pmap_port_type_t type, void *xprt,
+                     gf_boolean_t brick_disconnect)
 {
-        pmap_port_by_brick_req    args = {0,};
-        pmap_port_by_brick_rsp    rsp  = {0,};
-        char                     *brick = NULL;
-        int                       port = 0;
-        int                       ret = -1;
-
-        ret = xdr_to_generic (req->msg[0], &args,
-                              (xdrproc_t)xdr_pmap_port_by_brick_req);
-        if (ret < 0) {
-                req->rpc_err = GARBAGE_ARGS;
-                goto fail;
-        }
-
-        brick = args.brick;
-
-        port = pmap_registry_search (THIS, brick, GF_PMAP_PORT_BRICKSERVER);
+    struct pmap_registry *pmap = NULL;
+    int p = 0;
+    glusterd_conf_t *priv = NULL;
+    char *brick_str;
+
+    priv = this->private;
+    pmap = priv->pmap;
+    if (!pmap)
+        goto out;
 
-        if (!port)
-                rsp.op_ret = -1;
+    if (port) {
+        if (port > pmap->max_port)
+            goto out;
+    }
 
-        rsp.port = port;
+    if (brickname) {
+        p = pmap_registry_search(this, brickname, type, _gf_true);
+        if (p)
+            goto remove;
+    }
 
-fail:
-        glusterd_submit_reply (req, &rsp, NULL, 0, NULL,
-                               (xdrproc_t)xdr_pmap_port_by_brick_rsp);
-        free (args.brick);//malloced by xdr
+    if (xprt) {
+        p = pmap_registry_search_by_xprt(this, xprt, type);
+        if (p)
+            goto remove;
+    }
 
-        return 0;
-}
+    goto out;
+remove:
+    gf_msg("pmap", GF_LOG_INFO, 0, GD_MSG_BRICK_REMOVE,
+           "removing brick %s on port %d", brickname, p);
 
+    if (xprt && (xprt == pmap->ports[p].xprt)) {
+        pmap->ports[p].xprt = NULL;
+    }
+
+    /*
+     * This is where we garbage-collect.  If all of the brick names have
+     * been "whited out" by pmap_registry_search(...,destroy=_gf_true) and
+     * there's no xprt either, then we have nothing left worth saving and
+     * can delete the entire entry.
+     */
+    if (brick_disconnect || !pmap->ports[p].xprt) {
+        /* If the signout call is being triggered by brick disconnect
+         * then clean up all the bricks (in case of brick mux)
+         */
+        if (!brick_disconnect) {
+            brick_str = pmap->ports[p].brickname;
+            if (brick_str) {
+                while (*brick_str != '\0') {
+                    if (*(brick_str++) != ' ') {
+                        goto out;
+                    }
+                }
+            }
+        }
+        free(pmap->ports[p].brickname);
+        pmap->ports[p].brickname = NULL;
+        pmap->ports[p].type = GF_PMAP_PORT_FREE;
+    }
 
-int
-gluster_pmap_portbybrick (rpcsvc_request_t *req)
-{
-        return glusterd_big_locked_handler (req, __gluster_pmap_portbybrick);
+out:
+    return 0;
 }
 
-
 int
-__gluster_pmap_brickbyport (rpcsvc_request_t *req)
+__gluster_pmap_portbybrick(rpcsvc_request_t *req)
 {
-        pmap_brick_by_port_req    args = {0,};
-        pmap_brick_by_port_rsp    rsp  = {0,};
-        int                       ret = -1;
-
-        ret = xdr_to_generic (req->msg[0], &args,
-                              (xdrproc_t)xdr_pmap_brick_by_port_req);
-        if (ret < 0) {
-                req->rpc_err = GARBAGE_ARGS;
-                goto fail;
-        }
+    pmap_port_by_brick_req args = {
+        0,
+    };
+    pmap_port_by_brick_rsp rsp = {
+        0,
+    };
+    char *brick = NULL;
+    int port = 0;
+    int ret = -1;
+    xlator_t *this = THIS;
+    GF_ASSERT(this);
+
+    ret = xdr_to_generic(req->msg[0], &args,
+                         (xdrproc_t)xdr_pmap_port_by_brick_req);
+    if (ret < 0) {
+        req->rpc_err = GARBAGE_ARGS;
+        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_GARBAGE_ARGS, NULL);
+        goto fail;
+    }
+
+    brick = args.brick;
+
+    port = pmap_registry_search(this, brick, GF_PMAP_PORT_BRICKSERVER,
+                                _gf_false);
+
+    if (!port)
+        rsp.op_ret = -1;
+
+    rsp.port = port;
 
-        rsp.brick = pmap_registry_search_by_port (THIS, args.port);
-        if (!rsp.brick) {
-                rsp.op_ret = -1;
-                rsp.brick = "";
-        }
 fail:
+    glusterd_submit_reply(req, &rsp, NULL, 0, NULL,
+                          (xdrproc_t)xdr_pmap_port_by_brick_rsp);
+    free(args.brick);  // malloced by xdr
 
-        glusterd_submit_reply (req, &rsp, NULL, 0, NULL,
-                               (xdrproc_t)xdr_pmap_brick_by_port_rsp);
-
-        return 0;
+    return 0;
 }
 
-
 int
-gluster_pmap_brickbyport (rpcsvc_request_t *req)
-{
-        return glusterd_big_locked_handler (req, __gluster_pmap_brickbyport);
-}
-
-
-static int
-glusterd_brick_update_signin (glusterd_brickinfo_t *brickinfo,
-                              gf_boolean_t value)
+gluster_pmap_portbybrick(rpcsvc_request_t *req)
 {
-        brickinfo->signed_in = value;
-
-        return 0;
+    return glusterd_big_locked_handler(req, __gluster_pmap_portbybrick);
 }
 
 int
-__gluster_pmap_signup (rpcsvc_request_t *req)
+__gluster_pmap_brickbyport(rpcsvc_request_t *req)
 {
-        pmap_signup_req    args = {0,};
-        pmap_signup_rsp    rsp  = {0,};
-        int                ret = -1;
-
-
-        ret = xdr_to_generic (req->msg[0], &args,
-                              (xdrproc_t)xdr_pmap_signup_req);
-        if (ret < 0) {
-                req->rpc_err = GARBAGE_ARGS;
-                goto fail;
-        }
-
-        rsp.op_ret = pmap_registry_bind (THIS, args.port, args.brick,
-                                         GF_PMAP_PORT_BRICKSERVER, req->trans);
-
+    pmap_brick_by_port_req args = {
+        0,
+    };
+    pmap_brick_by_port_rsp rsp = {
+        0,
+    };
+    int ret = -1;
+    xlator_t *this = THIS;
+    GF_ASSERT(this);
+
+    ret = xdr_to_generic(req->msg[0], &args,
+                         (xdrproc_t)xdr_pmap_brick_by_port_req);
+    if (ret < 0) {
+        req->rpc_err = GARBAGE_ARGS;
+        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_GARBAGE_ARGS, NULL);
+        goto fail;
+    }
+
+    rsp.brick = pmap_registry_search_by_port(THIS, args.port);
+    if (!rsp.brick) {
+        rsp.op_ret = -1;
+        rsp.brick = "";
+    }
 fail:
-        glusterd_submit_reply (req, &rsp, NULL, 0, NULL,
-                               (xdrproc_t)xdr_pmap_signup_rsp);
-        free (args.brick);//malloced by xdr
 
-        return 0;
+    glusterd_submit_reply(req, &rsp, NULL, 0, NULL,
+                          (xdrproc_t)xdr_pmap_brick_by_port_rsp);
+
+    return 0;
 }
 
 int
-gluster_pmap_signup (rpcsvc_request_t *req)
+gluster_pmap_brickbyport(rpcsvc_request_t *req)
 {
-        return glusterd_big_locked_handler (req, __gluster_pmap_signup);
+    return glusterd_big_locked_handler(req, __gluster_pmap_brickbyport);
 }
 
 int
-__gluster_pmap_signin (rpcsvc_request_t *req)
+__gluster_pmap_signin(rpcsvc_request_t *req)
 {
-        pmap_signin_req    args = {0,};
-        pmap_signin_rsp    rsp  = {0,};
-        glusterd_brickinfo_t *brickinfo = NULL;
-        int                ret = -1;
-
-        ret = xdr_to_generic (req->msg[0], &args,
-                              (xdrproc_t)xdr_pmap_signin_req);
-        if (ret < 0) {
-                req->rpc_err = GARBAGE_ARGS;
-                goto fail;
-        }
-
-        rsp.op_ret = pmap_registry_bind (THIS, args.port, args.brick,
-                                         GF_PMAP_PORT_BRICKSERVER, req->trans);
+    pmap_signin_req args = {
+        0,
+    };
+    pmap_signin_rsp rsp = {
+        0,
+    };
+    int ret = -1;
+    glusterd_brickinfo_t *brickinfo = NULL;
+    xlator_t *this = THIS;
+    GF_ASSERT(this);
+
+    ret = xdr_to_generic(req->msg[0], &args, (xdrproc_t)xdr_pmap_signin_req);
+    if (ret < 0) {
+        req->rpc_err = GARBAGE_ARGS;
+        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_GARBAGE_ARGS, NULL);
+        goto fail;
+    }
+
+    rsp.op_ret = pmap_registry_bind(THIS, args.port, args.brick,
+                                    GF_PMAP_PORT_BRICKSERVER, req->trans);
+
+    ret = glusterd_get_brickinfo(THIS, args.brick, args.port, &brickinfo);
+    /* Update portmap status in brickinfo */
+    if (brickinfo)
+        brickinfo->port_registered = _gf_true;
 
-        ret = glusterd_get_brickinfo (THIS, args.brick, args.port, _gf_true,
-                                      &brickinfo);
 fail:
-        glusterd_submit_reply (req, &rsp, NULL, 0, NULL,
-                               (xdrproc_t)xdr_pmap_signin_rsp);
-        free (args.brick);//malloced by xdr
+    glusterd_submit_reply(req, &rsp, NULL, 0, NULL,
+                          (xdrproc_t)xdr_pmap_signin_rsp);
+    free(args.brick);  // malloced by xdr
 
-        if (!ret)
-                glusterd_brick_update_signin (brickinfo, _gf_true);
-
-        return 0;
+    return 0;
 }
 
-
 int
-gluster_pmap_signin (rpcsvc_request_t *req)
+gluster_pmap_signin(rpcsvc_request_t *req)
 {
-        return glusterd_big_locked_handler (req, __gluster_pmap_signin);
+    return glusterd_big_locked_handler(req, __gluster_pmap_signin);
 }
 
-
 int
-__gluster_pmap_signout (rpcsvc_request_t *req)
+__gluster_pmap_signout(rpcsvc_request_t *req)
 {
-        pmap_signout_req    args                 = {0,};
-        pmap_signout_rsp    rsp                  = {0,};
-        int                 ret                  = -1;
-        char                brick_path[PATH_MAX] = {0,};
-        glusterd_brickinfo_t *brickinfo = NULL;
-
-        ret = xdr_to_generic (req->msg[0], &args,
-                              (xdrproc_t)xdr_pmap_signout_req);
-        if (ret < 0) {
-                //failed to decode msg;
-                req->rpc_err = GARBAGE_ARGS;
-                goto fail;
+    pmap_signout_req args = {
+        0,
+    };
+    pmap_signout_rsp rsp = {
+        0,
+    };
+    int ret = -1;
+    xlator_t *this = NULL;
+    glusterd_conf_t *conf = NULL;
+    glusterd_volinfo_t *volinfo = NULL;
+    glusterd_brickinfo_t *brickinfo = NULL;
+    char pidfile[PATH_MAX] = {0};
+    char brick_path[PATH_MAX] = {
+        0,
+    };
+
+    this = THIS;
+    GF_VALIDATE_OR_GOTO("glusterd", this, fail);
+    conf = this->private;
+    GF_VALIDATE_OR_GOTO(this->name, conf, fail);
+
+    ret = xdr_to_generic(req->msg[0], &args, (xdrproc_t)xdr_pmap_signout_req);
+    if (ret < 0) {
+        // failed to decode msg;
+        req->rpc_err = GARBAGE_ARGS;
+        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_GARBAGE_ARGS, NULL);
+        goto fail;
+    }
+    rsp.op_ret = pmap_registry_remove(THIS, args.port, args.brick,
+                                      GF_PMAP_PORT_BRICKSERVER, req->trans,
+                                      _gf_false);
+
+    ret = glusterd_get_brickinfo(THIS, args.brick, args.port, &brickinfo);
+    if (args.rdma_port) {
+        snprintf(brick_path, PATH_MAX, "%s.rdma", args.brick);
+        rsp.op_ret = pmap_registry_remove(THIS, args.rdma_port, brick_path,
+                                          GF_PMAP_PORT_BRICKSERVER, req->trans,
+                                          _gf_false);
+    }
+    /* Update portmap status on brickinfo */
+    if (brickinfo)
+        brickinfo->port_registered = _gf_false;
+
+    /* Clean up the pidfile for this brick given glusterfsd doesn't clean it
+     * any more. This is required to ensure we don't end up with having
+     * stale pid files in case a brick is killed from the backend
+     */
+    ret = glusterd_get_volinfo_from_brick(args.brick, &volinfo);
+    if (!ret) {
+        if (volinfo && brickinfo) {
+            GLUSTERD_GET_BRICK_PIDFILE(pidfile, volinfo, brickinfo, conf);
+            sys_unlink(pidfile);
+
+            /* Setting the brick status to GF_BRICK_STOPPED to
+             * ensure correct brick status is maintained on the
+             * glusterd end when a brick is killed from the
+             * backend */
+            brickinfo->status = GF_BRICK_STOPPED;
+
+            /* Remove brick from brick process if not already
+             * removed in the brick op phase. This situation would
+             * arise when the brick is killed explicitly from the
+             * backend */
+            ret = glusterd_brick_process_remove_brick(brickinfo, NULL);
+            if (ret) {
+                gf_msg_debug(this->name, 0,
+                             "Couldn't remove "
+                             "brick %s:%s from brick process",
+                             brickinfo->hostname, brickinfo->path);
+                /* Ignore 'ret' here since the brick might
+                 * have already been deleted in brick op phase
+                 */
+                ret = 0;
+            }
         }
-
-        rsp.op_ret = pmap_registry_remove (THIS, args.port, args.brick,
-                                           GF_PMAP_PORT_BRICKSERVER, req->trans);
-
-        ret = glusterd_get_brickinfo (THIS, args.brick, args.port, _gf_true,
-                                      &brickinfo);
-        if (args.rdma_port) {
-                snprintf(brick_path, PATH_MAX, "%s.rdma", args.brick);
-                rsp.op_ret = pmap_registry_remove (THIS, args.rdma_port,
-                                brick_path, GF_PMAP_PORT_BRICKSERVER,
-                                req->trans);
-        }
-
-        if (!ret)
-                glusterd_brick_update_signin (brickinfo, _gf_false);
+    }
 
 fail:
-        glusterd_submit_reply (req, &rsp, NULL, 0, NULL,
-                               (xdrproc_t)xdr_pmap_signout_rsp);
-        free (args.brick);//malloced by xdr
+    glusterd_submit_reply(req, &rsp, NULL, 0, NULL,
+                          (xdrproc_t)xdr_pmap_signout_rsp);
+    free(args.brick);  // malloced by xdr
 
-        return 0;
+    return 0;
 }
 
 int
-gluster_pmap_signout (rpcsvc_request_t *req)
+gluster_pmap_signout(rpcsvc_request_t *req)
 {
-        return glusterd_big_locked_handler (req, __gluster_pmap_signout);
+    return glusterd_big_locked_handler(req, __gluster_pmap_signout);
 }
 
-rpcsvc_actor_t gluster_pmap_actors[GF_PMAP_MAXVALUE] = {
-        [GF_PMAP_NULL]        = {"NULL",        GF_PMAP_NULL,        NULL,                     NULL, 0, DRC_NA},
-        [GF_PMAP_PORTBYBRICK] = {"PORTBYBRICK", GF_PMAP_PORTBYBRICK, gluster_pmap_portbybrick, NULL, 0, DRC_NA},
-        [GF_PMAP_BRICKBYPORT] = {"BRICKBYPORT", GF_PMAP_BRICKBYPORT, gluster_pmap_brickbyport, NULL, 0, DRC_NA},
-        [GF_PMAP_SIGNUP]      = {"SIGNUP",      GF_PMAP_SIGNUP,      gluster_pmap_signup,      NULL, 0, DRC_NA},
-        [GF_PMAP_SIGNIN]      = {"SIGNIN",      GF_PMAP_SIGNIN,      gluster_pmap_signin,      NULL, 0, DRC_NA},
-        [GF_PMAP_SIGNOUT]     = {"SIGNOUT",     GF_PMAP_SIGNOUT,     gluster_pmap_signout,     NULL, 0, DRC_NA},
+static rpcsvc_actor_t gluster_pmap_actors[GF_PMAP_MAXVALUE] = {
+    [GF_PMAP_NULL] = {"NULL", NULL, NULL, GF_PMAP_NULL, DRC_NA, 0},
+    [GF_PMAP_PORTBYBRICK] = {"PORTBYBRICK", gluster_pmap_portbybrick, NULL,
+                             GF_PMAP_PORTBYBRICK, DRC_NA, 0},
+    [GF_PMAP_BRICKBYPORT] = {"BRICKBYPORT", gluster_pmap_brickbyport, NULL,
+                             GF_PMAP_BRICKBYPORT, DRC_NA, 0},
+    [GF_PMAP_SIGNIN] = {"SIGNIN", gluster_pmap_signin, NULL, GF_PMAP_SIGNIN,
+                        DRC_NA, 0},
+    [GF_PMAP_SIGNOUT] = {"SIGNOUT", gluster_pmap_signout, NULL, GF_PMAP_SIGNOUT,
+                         DRC_NA, 0},
 };
 
-
 struct rpcsvc_program gluster_pmap_prog = {
-        .progname  = "Gluster Portmap",
-        .prognum   = GLUSTER_PMAP_PROGRAM,
-        .progver   = GLUSTER_PMAP_VERSION,
-        .actors    = gluster_pmap_actors,
-        .numactors = GF_PMAP_MAXVALUE,
+    .progname = "Gluster Portmap",
+    .prognum = GLUSTER_PMAP_PROGRAM,
+    .progver = GLUSTER_PMAP_VERSION,
+    .actors = gluster_pmap_actors,
+    .numactors = GF_PMAP_MAXVALUE,
 };
diff --git a/xlators/mgmt/glusterd/src/glusterd-pmap.h b/xlators/mgmt/glusterd/src/glusterd-pmap.h
index 47434390303..51d75361431 100644
--- a/xlators/mgmt/glusterd/src/glusterd-pmap.h
+++ b/xlators/mgmt/glusterd/src/glusterd-pmap.h
@@ -10,45 +10,48 @@
 #ifndef _GLUSTERD_PMAP_H_
 #define _GLUSTERD_PMAP_H_
 
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
-
 #include <pthread.h>
-#include "compat-uuid.h"
+#include <glusterfs/compat-uuid.h>
 
-#include "glusterfs.h"
-#include "xlator.h"
-#include "logging.h"
-#include "call-stub.h"
-#include "fd.h"
-#include "byte-order.h"
-#include "glusterd.h"
+#include <glusterfs/glusterfs.h>
+#include <glusterfs/xlator.h>
+#include <glusterfs/logging.h>
+#include <glusterfs/call-stub.h>
+#include <glusterfs/byte-order.h>
 #include "rpcsvc.h"
 
-
-#define GF_IANA_PRIV_PORTS_START 49152 /* RFC 6335 */
-
 struct pmap_port_status {
-        gf_pmap_port_type_t type;
-        char  *brickname;
-        void  *xprt;
+    char *brickname;
+    void *xprt;
+    gf_pmap_port_type_t type;
 };
 
 struct pmap_registry {
-        int     base_port;
-        int     last_alloc;
-        struct  pmap_port_status ports[65536];
+    struct pmap_port_status ports[GF_PORT_MAX + 1];
+    int base_port;
+    int max_port;
+    int last_alloc;
 };
 
-int pmap_registry_alloc (xlator_t *this);
-int pmap_registry_bind (xlator_t *this, int port, const char *brickname,
-                        gf_pmap_port_type_t type, void *xprt);
-int pmap_registry_remove (xlator_t *this, int port, const char *brickname,
-                          gf_pmap_port_type_t type, void *xprt);
-int pmap_registry_search (xlator_t *this, const char *brickname,
-                          gf_pmap_port_type_t type);
-struct pmap_registry *pmap_registry_get (xlator_t *this);
+int
+pmap_assign_port(xlator_t *this, int port, const char *path);
+int
+pmap_mark_port_leased(xlator_t *this, int port);
+int
+pmap_registry_alloc(xlator_t *this);
+int
+pmap_registry_bind(xlator_t *this, int port, const char *brickname,
+                   gf_pmap_port_type_t type, void *xprt);
+int
+pmap_registry_extend(xlator_t *this, int port, const char *brickname);
+int
+pmap_registry_remove(xlator_t *this, int port, const char *brickname,
+                     gf_pmap_port_type_t type, void *xprt,
+                     gf_boolean_t brick_disconnect);
+int
+pmap_registry_search(xlator_t *this, const char *brickname,
+                     gf_pmap_port_type_t type, gf_boolean_t destroy);
+struct pmap_registry *
+pmap_registry_get(xlator_t *this);
 
 #endif
diff --git a/xlators/mgmt/glusterd/src/glusterd-proc-mgmt.c b/xlators/mgmt/glusterd/src/glusterd-proc-mgmt.c
index d6088a398fa..a05c90d7b10 100644
--- a/xlators/mgmt/glusterd/src/glusterd-proc-mgmt.c
+++ b/xlators/mgmt/glusterd/src/glusterd-proc-mgmt.c
@@ -12,123 +12,141 @@
 #include <limits.h>
 #include <signal.h>
 
-#include "common-utils.h"
-#include "xlator.h"
-#include "logging.h"
+#include "glusterd.h"
+#include "glusterd-utils.h"
+#include <glusterfs/common-utils.h>
+#include <glusterfs/xlator.h>
+#include <glusterfs/logging.h>
 #include "glusterd-messages.h"
 #include "glusterd-proc-mgmt.h"
 
 int
-glusterd_proc_init (glusterd_proc_t *proc, char *name, char *pidfile,
-                    char *logdir, char *logfile, char *volfile, char *volfileid,
-                    char *volfileserver)
+glusterd_proc_init(glusterd_proc_t *proc, char *name, char *pidfile,
+                   char *logdir, char *logfile, char *volfile, char *volfileid,
+                   char *volfileserver)
 {
-        int ret = -1;
+    int ret = -1;
 
-        ret = snprintf (proc->name, sizeof (proc->name), "%s", name);
-        if (ret < 0)
-                goto out;
+    ret = snprintf(proc->name, sizeof(proc->name), "%s", name);
+    if (ret < 0)
+        goto out;
 
-        ret = snprintf (proc->pidfile, sizeof (proc->pidfile), "%s", pidfile);
-        if (ret < 0)
-                goto out;
+    ret = snprintf(proc->pidfile, sizeof(proc->pidfile), "%s", pidfile);
+    if (ret < 0)
+        goto out;
 
-        ret = snprintf (proc->logdir, sizeof (proc->logdir), "%s", logdir);
-        if (ret < 0)
-                goto out;
+    ret = snprintf(proc->logdir, sizeof(proc->logdir), "%s", logdir);
+    if (ret < 0)
+        goto out;
 
-        ret = snprintf (proc->logfile, sizeof (proc->logfile), "%s", logfile);
-        if (ret < 0)
-                goto out;
+    ret = snprintf(proc->logfile, sizeof(proc->logfile), "%s", logfile);
+    if (ret < 0)
+        goto out;
 
-        ret = snprintf (proc->volfile, sizeof (proc->volfile), "%s", volfile);
-        if (ret < 0)
-                goto out;
+    ret = snprintf(proc->volfile, sizeof(proc->volfile), "%s", volfile);
+    if (ret < 0)
+        goto out;
 
-        ret = snprintf (proc->volfileid, sizeof (proc->volfileid), "%s",
-                        volfileid);
-        if (ret < 0)
-                goto out;
+    ret = snprintf(proc->volfileid, sizeof(proc->volfileid), "%s", volfileid);
+    if (ret < 0)
+        goto out;
 
-        ret = snprintf (proc->volfileserver, sizeof (proc->volfileserver), "%s",
-                        volfileserver);
-        if (ret < 0)
-                goto out;
+    ret = snprintf(proc->volfileserver, sizeof(proc->volfileserver), "%s",
+                   volfileserver);
+    if (ret < 0)
+        goto out;
 
 out:
-        if (ret > 0)
-                ret = 0;
+    if (ret > 0)
+        ret = 0;
 
-        return ret;
+    return ret;
 }
 
 int
-glusterd_proc_stop (glusterd_proc_t *proc, int sig, int flags)
+glusterd_proc_stop(glusterd_proc_t *proc, int sig, int flags)
 {
+    /* NB: Copy-paste code from glusterd_service_stop, the source may be
+     * removed once all daemon management use proc */
 
-        /* NB: Copy-paste code from glusterd_service_stop, the source may be
-         * removed once all daemon management use proc */
+    int32_t ret = -1;
+    pid_t pid = -1;
+    xlator_t *this = NULL;
+    glusterd_conf_t *conf = NULL;
 
-        int32_t  ret    = -1;
-        pid_t    pid    = -1;
-        xlator_t *this  = NULL;
+    this = THIS;
+    GF_ASSERT(this);
 
-        this = THIS;
-        GF_ASSERT (this);
+    conf = this->private;
+    GF_ASSERT(conf);
 
-        if (!gf_is_service_running (proc->pidfile, &pid)) {
+    if (!gf_is_service_running(proc->pidfile, &pid)) {
+        ret = 0;
+        gf_msg(this->name, GF_LOG_INFO, 0, GD_MSG_ALREADY_STOPPED,
+               "%s already stopped", proc->name);
+        goto out;
+    }
+    gf_msg(this->name, GF_LOG_INFO, 0, GD_MSG_SVC_STOP_SUCCESS,
+           "Stopping %s daemon running in pid: "
+           "%d",
+           proc->name, pid);
+
+    ret = kill(pid, sig);
+    if (ret) {
+        switch (errno) {
+            case ESRCH:
+                gf_msg_debug(this->name, 0,
+                             "%s is already "
+                             "stopped",
+                             proc->name);
                 ret = 0;
-                gf_log (this->name, GF_LOG_INFO, "%s already stopped",
-                        proc->name);
                 goto out;
+            default:
+                gf_msg(this->name, GF_LOG_ERROR, errno, GD_MSG_SVC_KILL_FAIL,
+                       "Unable to kill %s "
+                       "service, reason:%s",
+                       proc->name, strerror(errno));
         }
-        gf_log (this->name, GF_LOG_DEBUG, "Stopping %s daemon running in pid: "
-                "%d", proc->name, pid);
-
-        ret = kill (pid, sig);
+    } else {
+        (void)glusterd_unlink_file(proc->pidfile);
+    }
+    if (flags != PROC_STOP_FORCE)
+        goto out;
+
+    synclock_unlock(&conf->big_lock);
+    synctask_sleep(1);
+    synclock_lock(&conf->big_lock);
+    if (gf_is_service_running(proc->pidfile, &pid)) {
+        ret = kill(pid, SIGKILL);
         if (ret) {
-                switch (errno) {
-                case ESRCH:
-                        gf_log (this->name, GF_LOG_DEBUG, "%s is already "
-                                "stopped", proc->name);
-                        ret = 0;
-                        goto out;
-                default:
-                        gf_msg (this->name, GF_LOG_ERROR, errno,
-                                GD_MSG_SVC_KILL_FAIL, "Unable to kill %s "
-                                "service, reason:%s", proc->name,
-                                strerror (errno));
-                }
-        }
-        if (flags != PROC_STOP_FORCE)
-                goto out;
-
-        sleep (1);
-        if (gf_is_service_running (proc->pidfile, NULL)) {
-                ret = kill (pid, SIGKILL);
-                if (ret) {
-                        gf_msg (this->name, GF_LOG_ERROR, errno,
-                                GD_MSG_PID_KILL_FAIL, "Unable to kill pid:%d, "
-                                "reason:%s", pid, strerror(errno));
-                        goto out;
-                }
+            gf_msg(this->name, GF_LOG_ERROR, errno, GD_MSG_PID_KILL_FAIL,
+                   "Unable to kill pid:%d, "
+                   "reason:%s",
+                   pid, strerror(errno));
+            goto out;
         }
+        ret = glusterd_unlink_file(proc->pidfile);
+        if (ret)
+            goto out;
+    }
 
-        ret = 0;
+    ret = 0;
 out:
-        return ret;
+    return ret;
 }
 
 int
-glusterd_proc_get_pid (glusterd_proc_t *proc)
+glusterd_proc_get_pid(glusterd_proc_t *proc)
 {
-        int pid = -1;
-        (void) gf_is_service_running (proc->pidfile, &pid);
-        return pid;
+    int pid = -1;
+    (void)gf_is_service_running(proc->pidfile, &pid);
+    return pid;
 }
 
 int
-glusterd_proc_is_running (glusterd_proc_t *proc)
+glusterd_proc_is_running(glusterd_proc_t *proc)
 {
-        return gf_is_service_running (proc->pidfile, NULL);
+    int pid = -1;
+
+    return gf_is_service_running(proc->pidfile, &pid);
 }
diff --git a/xlators/mgmt/glusterd/src/glusterd-proc-mgmt.h b/xlators/mgmt/glusterd/src/glusterd-proc-mgmt.h
index 9485d78d03e..e8e9ffc5082 100644
--- a/xlators/mgmt/glusterd/src/glusterd-proc-mgmt.h
+++ b/xlators/mgmt/glusterd/src/glusterd-proc-mgmt.h
@@ -11,39 +11,34 @@
 #ifndef _GLUSTERD_PROC_MGMT_H_
 #define _GLUSTERD_PROC_MGMT_H_
 
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
-
 typedef struct glusterd_proc_ glusterd_proc_t;
 
 enum proc_flags {
-        PROC_NONE = 0,
-        PROC_START,
-        PROC_START_NO_WAIT,
-        PROC_STOP,
-        PROC_STOP_FORCE
+    PROC_NONE = 0,
+    PROC_START,
+    PROC_START_NO_WAIT,
+    PROC_STOP,
+    PROC_STOP_FORCE
 };
 
 struct glusterd_proc_ {
-        char name[PATH_MAX];
-        char pidfile[PATH_MAX];
-        char logdir[PATH_MAX];
-        char logfile[PATH_MAX];
-        char volfile[PATH_MAX];
-        char volfileserver[PATH_MAX];
-        char volfileid[256];
+    char name[NAME_MAX];
+    char pidfile[PATH_MAX];
+    char logdir[PATH_MAX];
+    char logfile[PATH_MAX];
+    char volfile[PATH_MAX];
+    char volfileserver[PATH_MAX];
+    char volfileid[256];
 };
 
 int
-glusterd_proc_init (glusterd_proc_t *proc, char *name, char *pidfile,
-                    char *logdir, char *logfile, char *volfile, char *volfileid,
-                    char *volfileserver);
+glusterd_proc_init(glusterd_proc_t *proc, char *name, char *pidfile,
+                   char *logdir, char *logfile, char *volfile, char *volfileid,
+                   char *volfileserver);
 
 int
-glusterd_proc_stop (glusterd_proc_t *proc, int sig, int flags);
+glusterd_proc_stop(glusterd_proc_t *proc, int sig, int flags);
 
 int
-glusterd_proc_is_running (glusterd_proc_t *proc);
+glusterd_proc_is_running(glusterd_proc_t *proc);
 #endif
diff --git a/xlators/mgmt/glusterd/src/glusterd-quota.c b/xlators/mgmt/glusterd/src/glusterd-quota.c
index a137e82b230..8370c174ce3 100644
--- a/xlators/mgmt/glusterd/src/glusterd-quota.c
+++ b/xlators/mgmt/glusterd/src/glusterd-quota.c
@@ -7,523 +7,886 @@
    later), or the GNU General Public License, version 2 (GPLv2), in all
    cases as published by the Free Software Foundation.
 */
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
-
-#include "common-utils.h"
+#include <glusterfs/common-utils.h>
 #include "cli1-xdr.h"
 #include "xdr-generic.h"
 #include "glusterd.h"
 #include "glusterd-op-sm.h"
 #include "glusterd-store.h"
 #include "glusterd-utils.h"
-#include "glusterd-nfs-svc.h"
 #include "glusterd-quotad-svc.h"
 #include "glusterd-volgen.h"
-#include "run.h"
-#include "syscall.h"
-#include "byte-order.h"
-#include "compat-errno.h"
-#include "quota-common-utils.h"
+#include "glusterd-messages.h"
+#include <glusterfs/run.h>
+#include <glusterfs/syscall.h>
+#include <glusterfs/byte-order.h>
+#include <glusterfs/compat-errno.h>
+#include <glusterfs/quota-common-utils.h>
+#include "glusterd-quota.h"
 
 #include <sys/wait.h>
 #include <dlfcn.h>
 
 #ifndef _PATH_SETFATTR
-# ifdef GF_LINUX_HOST_OS
-#  define _PATH_SETFATTR "/usr/bin/setfattr"
-# endif
-# ifdef __NetBSD__
-#  define _PATH_SETFATTR "/usr/pkg/bin/setfattr"
-# endif
+#ifdef GF_LINUX_HOST_OS
+#define _PATH_SETFATTR "setfattr"
+#endif
+#ifdef __NetBSD__
+#define _PATH_SETFATTR "/usr/pkg/bin/setfattr"
+#endif
 #endif
 
 /* Any negative pid to make it special client */
 #define QUOTA_CRAWL_PID "-100"
 
+#define GLUSTERFS_GET_QUOTA_LIMIT_MOUNT_PIDFILE(pidfile, volname)              \
+    {                                                                          \
+        snprintf(pidfile, PATH_MAX - 1,                                        \
+                 DEFAULT_VAR_RUN_DIRECTORY "/%s_quota_limit.pid", volname);    \
+    }
+
+#define GLUSTERFS_GET_QUOTA_LIST_MOUNT_PIDFILE(pidfile, volname)               \
+    {                                                                          \
+        snprintf(pidfile, PATH_MAX - 1,                                        \
+                 DEFAULT_VAR_RUN_DIRECTORY "/%s_quota_list.pid", volname);     \
+    }
+
+#define GLUSTERD_GET_QUOTA_CRAWL_PIDDIR(piddir, volinfo, type)                 \
+    do {                                                                       \
+        char _volpath[PATH_MAX] = {                                            \
+            0,                                                                 \
+        };                                                                     \
+        int32_t _crawl_pid_len;                                                \
+        GLUSTERD_GET_VOLUME_DIR(_volpath, volinfo, priv);                      \
+        if (type == GF_QUOTA_OPTION_TYPE_ENABLE ||                             \
+            type == GF_QUOTA_OPTION_TYPE_ENABLE_OBJECTS)                       \
+            _crawl_pid_len = snprintf(piddir, PATH_MAX, "%s/run/quota/enable", \
+                                      _volpath);                               \
+        else                                                                   \
+            _crawl_pid_len = snprintf(piddir, PATH_MAX,                        \
+                                      "%s/run/quota/disable", _volpath);       \
+        if ((_crawl_pid_len < 0) || (_crawl_pid_len >= PATH_MAX)) {            \
+            piddir[0] = 0;                                                     \
+        }                                                                      \
+    } while (0)
+
+#define GLUSTERD_GET_TMP_PATH(abspath, path)                                   \
+    do {                                                                       \
+        snprintf(abspath, sizeof(abspath) - 1,                                 \
+                 DEFAULT_VAR_RUN_DIRECTORY "/tmp%s", path);                    \
+    } while (0)
+
+#define GLUSTERD_GET_QUOTA_LIST_MOUNT_PATH(abspath, volname, path)             \
+    do {                                                                       \
+        snprintf(abspath, sizeof(abspath) - 1,                                 \
+                 DEFAULT_VAR_RUN_DIRECTORY "/%s_quota_list%s", volname, path); \
+    } while (0)
+
 const char *gd_quota_op_list[GF_QUOTA_OPTION_TYPE_MAX + 1] = {
-        [GF_QUOTA_OPTION_TYPE_NONE]               = "none",
-        [GF_QUOTA_OPTION_TYPE_ENABLE]             = "enable",
-        [GF_QUOTA_OPTION_TYPE_DISABLE]            = "disable",
-        [GF_QUOTA_OPTION_TYPE_LIMIT_USAGE]        = "limit-usage",
-        [GF_QUOTA_OPTION_TYPE_REMOVE]             = "remove",
-        [GF_QUOTA_OPTION_TYPE_LIST]               = "list",
-        [GF_QUOTA_OPTION_TYPE_VERSION]            = "version",
-        [GF_QUOTA_OPTION_TYPE_ALERT_TIME]         = "alert-time",
-        [GF_QUOTA_OPTION_TYPE_SOFT_TIMEOUT]       = "soft-timeout",
-        [GF_QUOTA_OPTION_TYPE_HARD_TIMEOUT]       = "hard-timeout",
-        [GF_QUOTA_OPTION_TYPE_DEFAULT_SOFT_LIMIT] = "default-soft-limit",
-        [GF_QUOTA_OPTION_TYPE_LIMIT_OBJECTS]      = "limit-objects",
-        [GF_QUOTA_OPTION_TYPE_LIST_OBJECTS]       = "list-objects",
-        [GF_QUOTA_OPTION_TYPE_REMOVE_OBJECTS]     = "remove-objetcs",
-        [GF_QUOTA_OPTION_TYPE_MAX]                = NULL
-};
+    [GF_QUOTA_OPTION_TYPE_NONE] = "none",
+    [GF_QUOTA_OPTION_TYPE_ENABLE] = "enable",
+    [GF_QUOTA_OPTION_TYPE_DISABLE] = "disable",
+    [GF_QUOTA_OPTION_TYPE_LIMIT_USAGE] = "limit-usage",
+    [GF_QUOTA_OPTION_TYPE_REMOVE] = "remove",
+    [GF_QUOTA_OPTION_TYPE_LIST] = "list",
+    [GF_QUOTA_OPTION_TYPE_VERSION] = "version",
+    [GF_QUOTA_OPTION_TYPE_ALERT_TIME] = "alert-time",
+    [GF_QUOTA_OPTION_TYPE_SOFT_TIMEOUT] = "soft-timeout",
+    [GF_QUOTA_OPTION_TYPE_HARD_TIMEOUT] = "hard-timeout",
+    [GF_QUOTA_OPTION_TYPE_DEFAULT_SOFT_LIMIT] = "default-soft-limit",
+    [GF_QUOTA_OPTION_TYPE_LIMIT_OBJECTS] = "limit-objects",
+    [GF_QUOTA_OPTION_TYPE_LIST_OBJECTS] = "list-objects",
+    [GF_QUOTA_OPTION_TYPE_REMOVE_OBJECTS] = "remove-objects",
+    [GF_QUOTA_OPTION_TYPE_ENABLE_OBJECTS] = "enable-objects",
+    [GF_QUOTA_OPTION_TYPE_UPGRADE] = "upgrade",
+    [GF_QUOTA_OPTION_TYPE_MAX] = NULL};
+
+gf_boolean_t
+glusterd_is_quota_supported(int32_t type, char **op_errstr)
+{
+    xlator_t *this = NULL;
+    glusterd_conf_t *conf = NULL;
+    gf_boolean_t supported = _gf_false;
+
+    this = THIS;
+    GF_VALIDATE_OR_GOTO("glusterd", this, out);
+
+    conf = this->private;
+    GF_VALIDATE_OR_GOTO(this->name, conf, out);
+
+    if ((conf->op_version == GD_OP_VERSION_MIN) &&
+        (type > GF_QUOTA_OPTION_TYPE_VERSION))
+        goto out;
+
+    if ((conf->op_version < GD_OP_VERSION_3_7_0) &&
+        (type > GF_QUOTA_OPTION_TYPE_VERSION_OBJECTS))
+        goto out;
+
+    /* Quota Operations that change quota.conf shouldn't
+     * be allowed as the quota.conf format changes in 3.7
+     */
+    if ((conf->op_version < GD_OP_VERSION_3_7_0) &&
+        (type == GF_QUOTA_OPTION_TYPE_ENABLE ||
+         type == GF_QUOTA_OPTION_TYPE_LIMIT_USAGE ||
+         type == GF_QUOTA_OPTION_TYPE_REMOVE))
+        goto out;
+
+    /* Quota xattr version implemented in 3.7.6
+     * quota-version is incremented when quota is enabled
+     * Quota enable and disable performance enhancement has been done
+     * in version 3.7.12.
+     * so don't allow enabling/disabling quota in heterogeneous
+     * cluster during upgrade
+     */
+    if (type == GF_QUOTA_OPTION_TYPE_ENABLE ||
+        type == GF_QUOTA_OPTION_TYPE_ENABLE_OBJECTS ||
+        type == GF_QUOTA_OPTION_TYPE_DISABLE) {
+        if (conf->op_version < GD_OP_VERSION_3_7_12)
+            goto out;
+    }
+
+    supported = _gf_true;
+
+out:
+    if (!supported && op_errstr != NULL && conf)
+        gf_asprintf(op_errstr,
+                    "Volume quota failed. The cluster is "
+                    "operating at version %d. Quota command"
+                    " %s is unavailable in this version.",
+                    conf->op_version, gd_quota_op_list[type]);
+
+    return supported;
+}
 
 int
-glusterd_store_quota_config (glusterd_volinfo_t *volinfo, char *path,
-                             char *gfid_str, int opcode, char **op_errstr);
-int
-__glusterd_handle_quota (rpcsvc_request_t *req)
+__glusterd_handle_quota(rpcsvc_request_t *req)
 {
-        int32_t                         ret = -1;
-        gf_cli_req                      cli_req = {{0,}};
-        dict_t                         *dict = NULL;
-        glusterd_op_t                   cli_op = GD_OP_QUOTA;
-        char                           *volname = NULL;
-        int32_t                         type = 0;
-        char                            msg[2048] = {0,};
-        xlator_t                       *this = NULL;
-        glusterd_conf_t                *conf = NULL;
-
-        GF_ASSERT (req);
-        this = THIS;
-        GF_ASSERT (this);
-        conf = this->private;
-        GF_ASSERT (conf);
-
-        ret = xdr_to_generic (req->msg[0], &cli_req, (xdrproc_t)xdr_gf_cli_req);
+    int32_t ret = -1;
+    gf_cli_req cli_req = {{
+        0,
+    }};
+    dict_t *dict = NULL;
+    glusterd_op_t cli_op = GD_OP_QUOTA;
+    char *volname = NULL;
+    int32_t type = 0;
+    char msg[2048] = {
+        0,
+    };
+    xlator_t *this = NULL;
+    glusterd_conf_t *conf = NULL;
+
+    GF_ASSERT(req);
+    this = THIS;
+    GF_ASSERT(this);
+    conf = this->private;
+    GF_ASSERT(conf);
+
+    ret = xdr_to_generic(req->msg[0], &cli_req, (xdrproc_t)xdr_gf_cli_req);
+    if (ret < 0) {
+        // failed to decode msg;
+        req->rpc_err = GARBAGE_ARGS;
+        goto out;
+    }
+
+    if (cli_req.dict.dict_len) {
+        /* Unserialize the dictionary */
+        dict = dict_new();
+
+        ret = dict_unserialize(cli_req.dict.dict_val, cli_req.dict.dict_len,
+                               &dict);
         if (ret < 0) {
-                //failed to decode msg;
-                req->rpc_err = GARBAGE_ARGS;
-                goto out;
-        }
-
-        if (cli_req.dict.dict_len) {
-                /* Unserialize the dictionary */
-                dict  = dict_new ();
-
-                ret = dict_unserialize (cli_req.dict.dict_val,
-                                        cli_req.dict.dict_len,
-                                        &dict);
-                if (ret < 0) {
-                        gf_log (this->name, GF_LOG_ERROR, "failed to "
-                                    "unserialize req-buffer to dictionary");
-                        snprintf (msg, sizeof (msg), "Unable to decode the "
-                                  "command");
-                        goto out;
-                } else {
-                        dict->extra_stdfree = cli_req.dict.dict_val;
-                }
-        }
-
-        ret = dict_get_str (dict, "volname", &volname);
-        if (ret) {
-                snprintf (msg, sizeof (msg), "Unable to get volume name");
-                gf_log (this->name, GF_LOG_ERROR, "Unable to get volume name, "
-                        "while handling quota command");
-                goto out;
-        }
-
-        ret = dict_get_int32 (dict, "type", &type);
-        if (ret) {
-                snprintf (msg, sizeof (msg), "Unable to get type of command");
-                gf_log (this->name, GF_LOG_ERROR, "Unable to get type of cmd, "
-                        "while handling quota command");
-               goto out;
-        }
-
-        if ((conf->op_version == GD_OP_VERSION_MIN) &&
-            (type > GF_QUOTA_OPTION_TYPE_VERSION)) {
-                snprintf (msg, sizeof (msg), "Cannot execute command. The "
-                         "cluster is operating at version %d. Quota command %s "
-                         "is unavailable in this version", conf->op_version,
-                         gd_quota_op_list[type]);
-                ret = -1;
-                goto out;
-       }
-
-        ret = glusterd_op_begin_synctask (req, GD_OP_QUOTA, dict);
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_UNSERIALIZE_FAIL,
+                   "failed to "
+                   "unserialize req-buffer to dictionary");
+            snprintf(msg, sizeof(msg),
+                     "Unable to decode the "
+                     "command");
+            goto out;
+        } else {
+            dict->extra_stdfree = cli_req.dict.dict_val;
+        }
+    }
+
+    ret = dict_get_strn(dict, "volname", SLEN("volname"), &volname);
+    if (ret) {
+        snprintf(msg, sizeof(msg), "Unable to get volume name");
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+               "Unable to get volume name, "
+               "while handling quota command");
+        goto out;
+    }
+
+    ret = dict_get_int32n(dict, "type", SLEN("type"), &type);
+    if (ret) {
+        snprintf(msg, sizeof(msg), "Unable to get type of command");
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+               "Unable to get type of cmd, "
+               "while handling quota command");
+        goto out;
+    }
+
+    if (!glusterd_is_quota_supported(type, NULL)) {
+        snprintf(msg, sizeof(msg),
+                 "Volume quota failed. The cluster "
+                 "is operating at version %d. Quota command"
+                 " %s is unavailable in this version.",
+                 conf->op_version, gd_quota_op_list[type]);
+        ret = -1;
+        goto out;
+    }
+
+    ret = glusterd_op_begin_synctask(req, GD_OP_QUOTA, dict);
 
 out:
-        if (ret) {
-                if (msg[0] == '\0')
-                        snprintf (msg, sizeof (msg), "Operation failed");
-                ret = glusterd_op_send_cli_response (cli_op, ret, 0, req,
-                                                     dict, msg);
-        }
+    if (ret) {
+        if (msg[0] == '\0')
+            snprintf(msg, sizeof(msg), "Operation failed");
+        ret = glusterd_op_send_cli_response(cli_op, ret, 0, req, dict, msg);
+    }
 
-        return ret;
+    return ret;
 }
 
 int
-glusterd_handle_quota (rpcsvc_request_t *req)
+glusterd_handle_quota(rpcsvc_request_t *req)
 {
-        return glusterd_big_locked_handler (req, __glusterd_handle_quota);
+    return glusterd_big_locked_handler(req, __glusterd_handle_quota);
 }
 
 int32_t
-glusterd_check_if_quota_trans_enabled (glusterd_volinfo_t *volinfo)
+glusterd_check_if_quota_trans_enabled(glusterd_volinfo_t *volinfo)
 {
-        int32_t  ret           = 0;
-        int      flag          = _gf_false;
-
-        flag = glusterd_volinfo_get_boolean (volinfo, VKEY_FEATURES_QUOTA);
-        if (flag == -1) {
-                gf_log ("", GF_LOG_ERROR, "failed to get the quota status");
-                ret = -1;
-                goto out;
-        }
-
-        if (flag == _gf_false) {
-                ret = -1;
-                goto out;
-        }
-        ret = 0;
+    int32_t ret = 0;
+    int flag = _gf_false;
+
+    flag = glusterd_volinfo_get_boolean(volinfo, VKEY_FEATURES_QUOTA);
+    if (flag == -1) {
+        gf_msg("glusterd", GF_LOG_ERROR, 0, GD_MSG_QUOTA_GET_STAT_FAIL,
+               "failed to get the quota status");
+        ret = -1;
+        goto out;
+    }
+
+    if (flag == _gf_false) {
+        ret = -1;
+        goto out;
+    }
+    ret = 0;
 out:
-        return ret;
+    return ret;
 }
 
 int32_t
-glusterd_quota_initiate_fs_crawl (glusterd_conf_t *priv, char *volname,
-                                  int type)
+_glusterd_quota_initiate_fs_crawl(glusterd_conf_t *priv,
+                                  glusterd_volinfo_t *volinfo,
+                                  glusterd_brickinfo_t *brick, int type,
+                                  char *pid_dir)
 {
-        pid_t                      pid;
-        int32_t                    ret               = 0;
-        int                        status            = 0;
-        char                       mountdir[]        = "/tmp/mntXXXXXX";
-        char                       logfile[PATH_MAX] = {0,};
-        runner_t                   runner            = {0};
-
-        if (mkdtemp (mountdir) == NULL) {
-                gf_log ("glusterd", GF_LOG_DEBUG,
-                        "failed to create a temporary mount directory");
-                ret = -1;
-                goto out;
-        }
-        snprintf (logfile, sizeof (logfile),
-                  DEFAULT_LOG_FILE_DIRECTORY"/%s-quota-crawl.log", volname);
-
-        runinit (&runner);
-        runner_add_args (&runner, SBIN_DIR"/glusterfs",
-                         "-s", "localhost",
-                         "--volfile-id", volname,
-			 "--use-readdirp=no",
-                         "--client-pid", QUOTA_CRAWL_PID,
-                         "-l", logfile, mountdir, NULL);
-
-        synclock_unlock (&priv->big_lock);
-        ret = runner_run_reuse (&runner);
-        synclock_lock (&priv->big_lock);
+    pid_t pid;
+    int32_t ret = -1;
+    int status = 0;
+    char mountdir[PATH_MAX] = {
+        0,
+    };
+    char logfile[PATH_MAX] = {
+        0,
+    };
+    char brickpath[PATH_MAX] = {
+        0,
+    };
+    char vol_id[PATH_MAX] = {
+        0,
+    };
+    char pidfile[PATH_MAX] = {
+        0,
+    };
+    runner_t runner = {0};
+    char *volfileserver = NULL;
+    FILE *pidfp = NULL;
+    int32_t len = 0;
+
+    GF_VALIDATE_OR_GOTO("glusterd", THIS, out);
+
+    GLUSTERD_GET_TMP_PATH(mountdir, "/");
+    ret = sys_mkdir(mountdir, 0755);
+    if (ret && errno != EEXIST) {
+        gf_msg(THIS->name, GF_LOG_WARNING, errno, GD_MSG_MOUNT_REQ_FAIL,
+               "failed to create temporary "
+               "directory %s",
+               mountdir);
+        ret = -1;
+        goto out;
+    }
+
+    strcat(mountdir, "mntXXXXXX");
+    if (mkdtemp(mountdir) == NULL) {
+        gf_msg(THIS->name, GF_LOG_WARNING, errno, GD_MSG_MOUNT_REQ_FAIL,
+               "failed to create a temporary "
+               "mount directory: %s",
+               mountdir);
+        ret = -1;
+        goto out;
+    }
+
+    GLUSTERD_REMOVE_SLASH_FROM_PATH(brick->path, brickpath);
+    len = snprintf(logfile, sizeof(logfile),
+                   DEFAULT_QUOTA_CRAWL_LOG_DIRECTORY "/%s.log", brickpath);
+    if ((len < 0) || (len >= sizeof(vol_id))) {
+        ret = -1;
+        goto out;
+    }
+
+    if (dict_get_strn(THIS->options, "transport.socket.bind-address",
+                      SLEN("transport.socket.bind-address"),
+                      &volfileserver) != 0)
+        volfileserver = "localhost";
+
+    len = snprintf(vol_id, sizeof(vol_id), "client_per_brick/%s.%s.%s.%s.vol",
+                   volinfo->volname, "client", brick->hostname, brickpath);
+    if ((len < 0) || (len >= sizeof(vol_id))) {
+        ret = -1;
+        goto out;
+    }
+
+    runinit(&runner);
+
+    if (type == GF_QUOTA_OPTION_TYPE_ENABLE ||
+        type == GF_QUOTA_OPTION_TYPE_ENABLE_OBJECTS)
+        runner_add_args(&runner, SBIN_DIR "/glusterfs", "-s", volfileserver,
+                        "--volfile-id", vol_id, "--use-readdirp=yes",
+                        "--client-pid", QUOTA_CRAWL_PID, "-l", logfile,
+                        mountdir, NULL);
+    else
+        runner_add_args(&runner, SBIN_DIR "/glusterfs", "-s", volfileserver,
+                        "--volfile-id", vol_id, "--use-readdirp=no",
+                        "--client-pid", QUOTA_CRAWL_PID, "-l", logfile,
+                        mountdir, NULL);
+
+    synclock_unlock(&priv->big_lock);
+    ret = runner_run_reuse(&runner);
+    synclock_lock(&priv->big_lock);
+    if (ret == -1) {
+        runner_log(&runner, "glusterd", GF_LOG_DEBUG, "command failed");
+        runner_end(&runner);
+        goto out;
+    }
+    runner_end(&runner);
+
+    if ((pid = fork()) < 0) {
+        gf_msg(THIS->name, GF_LOG_WARNING, 0, GD_MSG_FORK_FAIL,
+               "fork from parent failed");
+        gf_umount_lazy("glusterd", mountdir, 1);
+        ret = -1;
+        goto out;
+    } else if (pid == 0) {  // first child
+        /* fork one more to not hold back main process on
+         * blocking call below
+         */
+        pid = fork();
+        if (pid < 0) {
+            gf_umount_lazy("glusterd", mountdir, 1);
+            _exit(EXIT_FAILURE);
+        } else if (pid > 0) {
+            _exit(EXIT_SUCCESS);
+        }
+
+        ret = chdir(mountdir);
         if (ret == -1) {
-                runner_log (&runner, "glusterd", GF_LOG_DEBUG, "command failed");
-                runner_end (&runner);
-                goto out;
+            gf_msg(THIS->name, GF_LOG_WARNING, errno, GD_MSG_DIR_OP_FAILED,
+                   "chdir %s failed", mountdir);
+            gf_umount_lazy("glusterd", mountdir, 1);
+            exit(EXIT_FAILURE);
         }
-        runner_end (&runner);
-
-        if ((pid = fork ()) < 0) {
-                gf_log ("glusterd", GF_LOG_WARNING, "fork from parent failed");
-                ret = -1;
-                goto out;
-        } else if (pid == 0) {//first child
-                /* fork one more to not hold back main process on
-                 * blocking call below
-                 */
-                pid = fork ();
-                if (pid)
-                        _exit (pid > 0 ? EXIT_SUCCESS : EXIT_FAILURE);
-
-                ret = chdir (mountdir);
-                if (ret == -1) {
-                        gf_log ("glusterd", GF_LOG_WARNING, "chdir %s failed, "
-                                "reason: %s", mountdir, strerror (errno));
-                        exit (EXIT_FAILURE);
-                }
-                runinit (&runner);
-
-                if (type == GF_QUOTA_OPTION_TYPE_ENABLE)
-                        runner_add_args (&runner, "/usr/bin/find", ".",
-                                         "-exec", "/usr/bin/stat",
-                                         "{}", "\\", ";", NULL);
+        runinit(&runner);
 
-                else if (type == GF_QUOTA_OPTION_TYPE_DISABLE) {
+        if (type == GF_QUOTA_OPTION_TYPE_ENABLE ||
+            type == GF_QUOTA_OPTION_TYPE_ENABLE_OBJECTS)
+            runner_add_args(&runner, "/usr/bin/find", ".", "-exec",
+                            "/usr/bin/stat", "{}", "\\", ";", NULL);
 
+        else if (type == GF_QUOTA_OPTION_TYPE_DISABLE) {
 #if defined(GF_DARWIN_HOST_OS)
-                        runner_add_args (&runner, "/usr/bin/find", ".",
-                                         "-exec", "/usr/bin/xattr", "-w",
-                                         VIRTUAL_QUOTA_XATTR_CLEANUP_KEY, "1",
-                                         "{}", "\\", ";", NULL);
+            runner_add_args(
+                &runner, "/usr/bin/find", ".", "-exec", "/usr/bin/xattr", "-w",
+                VIRTUAL_QUOTA_XATTR_CLEANUP_KEY, "1", "{}", "\\", ";", NULL);
 #elif defined(__FreeBSD__)
-                        runner_add_args (&runner, "/usr/bin/find", ".",
-                                         "-exec", "/usr/sbin/setextattr",
-                                         EXTATTR_NAMESPACE_USER,
-                                         VIRTUAL_QUOTA_XATTR_CLEANUP_KEY, "1",
-                                         "{}", "\\", ";", NULL);
+            runner_add_args(&runner, "/usr/bin/find", ".", "-exec",
+                            "/usr/sbin/setextattr", EXTATTR_NAMESPACE_USER,
+                            VIRTUAL_QUOTA_XATTR_CLEANUP_KEY, "1", "{}", "\\",
+                            ";", NULL);
 #else
-                        runner_add_args (&runner, "/usr/bin/find", ".",
-                                         "-exec", _PATH_SETFATTR, "-n",
-                                         VIRTUAL_QUOTA_XATTR_CLEANUP_KEY, "-v",
-                                         "1", "{}", "\\", ";", NULL);
+            runner_add_args(&runner, "find", ".", "-exec", _PATH_SETFATTR, "-n",
+                            VIRTUAL_QUOTA_XATTR_CLEANUP_KEY, "-v", "1", "{}",
+                            "\\", ";", NULL);
 #endif
+        }
 
-                }
+        if (runner_start(&runner) == -1) {
+            gf_umount_lazy("glusterd", mountdir, 1);
+            _exit(EXIT_FAILURE);
+        }
 
-                if (runner_start (&runner) == -1)
-                        _exit (EXIT_FAILURE);
+        len = snprintf(pidfile, sizeof(pidfile), "%s/%s.pid", pid_dir,
+                       brickpath);
+        if ((len >= 0) && (len < sizeof(pidfile))) {
+            pidfp = fopen(pidfile, "w");
+            if (pidfp != NULL) {
+                fprintf(pidfp, "%d\n", runner.chpid);
+                fflush(pidfp);
+                fclose(pidfp);
+            }
+        }
 
 #ifndef GF_LINUX_HOST_OS
-                runner_end (&runner); /* blocks in waitpid */
+        runner_end(&runner); /* blocks in waitpid */
 #endif
-                gf_umount_lazy ("glusterd", mountdir, 1);
+        gf_umount_lazy("glusterd", mountdir, 1);
 
-                _exit (EXIT_SUCCESS);
-        }
-        ret = (waitpid (pid, &status, 0) == pid &&
-               WIFEXITED (status) && WEXITSTATUS (status) == EXIT_SUCCESS) ? 0 : -1;
+        _exit(EXIT_SUCCESS);
+    }
+    ret = (waitpid(pid, &status, 0) == pid && WIFEXITED(status) &&
+           WEXITSTATUS(status) == EXIT_SUCCESS)
+              ? 0
+              : -1;
 
 out:
-        return ret;
+    return ret;
 }
 
-int32_t
-glusterd_quota_get_default_soft_limit (glusterd_volinfo_t *volinfo,
-                                       dict_t *rsp_dict)
+void
+glusterd_stop_all_quota_crawl_service(glusterd_conf_t *priv,
+                                      glusterd_volinfo_t *volinfo, int type)
 {
-        int32_t            ret             = 0;
-        xlator_t          *this            = NULL;
-        glusterd_conf_t   *conf            = NULL;
-        char              *default_limit   = NULL;
-        char              *val             = NULL;
-
-        if (rsp_dict == NULL)
-                return -1;
-
-        this = THIS;
-        GF_ASSERT (this);
-        conf = this->private;
-        GF_ASSERT (conf);
-
-        ret = glusterd_volinfo_get (volinfo, "features.default-soft-limit",
-                                    &default_limit);
-        if (default_limit)
-                val = gf_strdup (default_limit);
-        else
-                val = gf_strdup ("80%");
-
-        ret = dict_set_dynstr (rsp_dict, "default-soft-limit", val);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "Failed to set default "
-                        "soft-limit into dict");
-                goto out;
-        }
-        ret = 0;
-
-out:
-        return ret;
+    DIR *dir = NULL;
+    struct dirent *entry = NULL;
+    struct dirent scratch[2] = {
+        {
+            0,
+        },
+    };
+    char pid_dir[PATH_MAX] = {
+        0,
+    };
+    char pidfile[PATH_MAX] = {
+        0,
+    };
+    int32_t len = 0;
+
+    GLUSTERD_GET_QUOTA_CRAWL_PIDDIR(pid_dir, volinfo, type);
+
+    dir = sys_opendir(pid_dir);
+    if (dir == NULL)
+        return;
+
+    while ((entry = sys_readdir(dir, scratch))) {
+        if (gf_irrelevant_entry(entry))
+            continue;
+        len = snprintf(pidfile, sizeof(pidfile), "%s/%s", pid_dir,
+                       entry->d_name);
+        if ((len >= 0) && (len < sizeof(pidfile))) {
+            glusterd_service_stop_nolock("quota_crawl", pidfile, SIGKILL,
+                                         _gf_true);
+            sys_unlink(pidfile);
+        }
+    }
+    sys_closedir(dir);
 }
 
 int32_t
-glusterd_quota_enable (glusterd_volinfo_t *volinfo, char **op_errstr,
-                       gf_boolean_t *crawl)
+glusterd_quota_initiate_fs_crawl(glusterd_conf_t *priv,
+                                 glusterd_volinfo_t *volinfo, int type)
 {
-        int32_t         ret     = -1;
-        char            *quota_status = NULL;
-        xlator_t        *this         = NULL;
+    int32_t ret = -1;
+    glusterd_brickinfo_t *brick = NULL;
+    char pid_dir[PATH_MAX] = {
+        0,
+    };
+
+    GF_VALIDATE_OR_GOTO("glusterd", THIS, out);
+
+    ret = glusterd_generate_client_per_brick_volfile(volinfo);
+    if (ret) {
+        gf_msg(THIS->name, GF_LOG_ERROR, 0, GD_MSG_GLUSTERD_OP_FAILED,
+               "failed to generate client volume file");
+        goto out;
+    }
+
+    ret = mkdir_p(DEFAULT_QUOTA_CRAWL_LOG_DIRECTORY, 0755, _gf_true);
+    if (ret) {
+        gf_msg(THIS->name, GF_LOG_ERROR, errno, GD_MSG_GLUSTERD_OP_FAILED,
+               "failed to create dir %s: %s", DEFAULT_QUOTA_CRAWL_LOG_DIRECTORY,
+               strerror(errno));
+        goto out;
+    }
+
+    GLUSTERD_GET_QUOTA_CRAWL_PIDDIR(pid_dir, volinfo, type);
+    ret = mkdir_p(pid_dir, 0755, _gf_true);
+    if (ret) {
+        gf_msg(THIS->name, GF_LOG_ERROR, errno, GD_MSG_GLUSTERD_OP_FAILED,
+               "failed to create dir %s: %s", pid_dir, strerror(errno));
+        goto out;
+    }
+
+    /* When quota enable is performed, stop alreday running enable crawl
+     * process and start fresh crawl process. let disable process continue
+     * if running to cleanup the older xattrs
+     * When quota disable is performed, stop both enable/disable crawl
+     * process and start fresh crawl process to cleanup the xattrs
+     */
+    glusterd_stop_all_quota_crawl_service(priv, volinfo,
+                                          GF_QUOTA_OPTION_TYPE_ENABLE);
+    if (type == GF_QUOTA_OPTION_TYPE_DISABLE)
+        glusterd_stop_all_quota_crawl_service(priv, volinfo,
+                                              GF_QUOTA_OPTION_TYPE_DISABLE);
+
+    cds_list_for_each_entry(brick, &volinfo->bricks, brick_list)
+    {
+        if (gf_uuid_compare(brick->uuid, MY_UUID))
+            continue;
+
+        ret = _glusterd_quota_initiate_fs_crawl(priv, volinfo, brick, type,
+                                                pid_dir);
 
-        this = THIS;
-        GF_ASSERT (this);
-
-        GF_VALIDATE_OR_GOTO (this->name, volinfo, out);
-        GF_VALIDATE_OR_GOTO (this->name, crawl, out);
-        GF_VALIDATE_OR_GOTO (this->name, op_errstr, out);
-
-        if (glusterd_is_volume_started (volinfo) == 0) {
-                *op_errstr = gf_strdup ("Volume is stopped, start volume "
-                                        "to enable quota.");
-                ret = -1;
-                goto out;
-        }
-
-        ret = glusterd_check_if_quota_trans_enabled (volinfo);
-        if (ret == 0) {
-                *op_errstr = gf_strdup ("Quota is already enabled");
-                ret = -1;
-                goto out;
-        }
-
-        quota_status = gf_strdup ("on");
-        if (!quota_status) {
-                gf_log (this->name, GF_LOG_ERROR, "memory allocation failed");
-                ret = -1;
-                goto out;
-        }
-
-        ret = dict_set_dynstr (volinfo->dict, VKEY_FEATURES_QUOTA,
-                               quota_status);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "dict set failed");
-                goto out;
-        }
+        if (ret)
+            goto out;
+    }
 
-        *crawl = _gf_true;
+    ret = 0;
+out:
+    return ret;
+}
 
-        ret = glusterd_store_quota_config (volinfo, NULL, NULL,
-                                           GF_QUOTA_OPTION_TYPE_ENABLE,
-                                           op_errstr);
+int32_t
+glusterd_quota_get_default_soft_limit(glusterd_volinfo_t *volinfo,
+                                      dict_t *rsp_dict)
+{
+    int32_t ret = 0;
+    xlator_t *this = NULL;
+    glusterd_conf_t *conf = NULL;
+    char *default_limit = NULL;
+    char *val = NULL;
+
+    if (rsp_dict == NULL)
+        return -1;
+
+    this = THIS;
+    GF_ASSERT(this);
+    conf = this->private;
+    GF_ASSERT(conf);
+
+    ret = glusterd_volinfo_get(volinfo, "features.default-soft-limit",
+                               &default_limit);
+    if (default_limit)
+        val = gf_strdup(default_limit);
+    else
+        val = gf_strdup("80%");
+
+    ret = dict_set_dynstr_sizen(rsp_dict, "default-soft-limit", val);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+               "Failed to set default "
+               "soft-limit into dict");
+        goto out;
+    }
+    ret = 0;
 
-        ret = 0;
 out:
-        if (ret && op_errstr && !*op_errstr)
-                gf_asprintf (op_errstr, "Enabling quota on volume %s has been "
-                             "unsuccessful", volinfo->volname);
-        return ret;
+    return ret;
 }
 
 int32_t
-glusterd_quota_disable (glusterd_volinfo_t *volinfo, char **op_errstr,
-                        gf_boolean_t *crawl)
+glusterd_inode_quota_enable(glusterd_volinfo_t *volinfo, char **op_errstr,
+                            gf_boolean_t *crawl)
 {
-        int32_t    ret            = -1;
-        int        i              =  0;
-        char      *quota_status   = NULL;
-        char      *value          = NULL;
-        xlator_t  *this           = NULL;
-        glusterd_conf_t *conf     = NULL;
-        char *quota_options[]     = {"features.soft-timeout",
-                                     "features.hard-timeout",
-                                     "features.alert-time",
-                                     "features.default-soft-limit",
-                                     "features.quota-deem-statfs",
-                                     "features.quota-timeout", NULL};
-
-        this = THIS;
-        GF_ASSERT (this);
-        conf = this->private;
-        GF_ASSERT (conf);
-
-        GF_VALIDATE_OR_GOTO (this->name, volinfo, out);
-        GF_VALIDATE_OR_GOTO (this->name, op_errstr, out);
-
-        ret = glusterd_check_if_quota_trans_enabled (volinfo);
-        if (ret == -1) {
-                *op_errstr = gf_strdup ("Quota is already disabled");
-                goto out;
-        }
+    int32_t ret = -1;
+    xlator_t *this = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+
+    GF_VALIDATE_OR_GOTO(this->name, volinfo, out);
+    GF_VALIDATE_OR_GOTO(this->name, crawl, out);
+    GF_VALIDATE_OR_GOTO(this->name, op_errstr, out);
+
+    if (glusterd_is_volume_started(volinfo) == 0) {
+        *op_errstr = gf_strdup(
+            "Volume is stopped, start volume "
+            "to enable inode quota.");
+        ret = -1;
+        goto out;
+    }
+
+    ret = glusterd_check_if_quota_trans_enabled(volinfo);
+    if (ret != 0) {
+        *op_errstr = gf_strdup(
+            "Quota is disabled. Enabling quota "
+            "will enable inode quota");
+        ret = -1;
+        goto out;
+    }
+
+    if (glusterd_is_volume_inode_quota_enabled(volinfo)) {
+        *op_errstr = gf_strdup("Inode Quota is already enabled");
+        ret = -1;
+        goto out;
+    }
+
+    ret = dict_set_dynstr_with_alloc(volinfo->dict, VKEY_FEATURES_INODE_QUOTA,
+                                     "on");
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_SET_FAILED,
+               "dict set failed");
+        goto out;
+    }
+
+    *crawl = _gf_true;
+
+    ret = glusterd_store_quota_config(
+        volinfo, NULL, NULL, GF_QUOTA_OPTION_TYPE_ENABLE_OBJECTS, op_errstr);
+
+    ret = 0;
+out:
+    if (ret && op_errstr && !*op_errstr)
+        gf_asprintf(op_errstr,
+                    "Enabling inode quota on volume %s has "
+                    "been unsuccessful",
+                    volinfo->volname);
+    return ret;
+}
 
-        quota_status = gf_strdup ("off");
-        if (!quota_status) {
-                gf_log (this->name, GF_LOG_ERROR, "memory allocation failed");
-                ret = -1;
-                goto out;
-        }
+int32_t
+glusterd_quota_enable(glusterd_volinfo_t *volinfo, char **op_errstr,
+                      gf_boolean_t *crawl)
+{
+    int32_t ret = -1;
+    xlator_t *this = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+
+    GF_VALIDATE_OR_GOTO(this->name, volinfo, out);
+    GF_VALIDATE_OR_GOTO(this->name, crawl, out);
+    GF_VALIDATE_OR_GOTO(this->name, op_errstr, out);
+
+    if (glusterd_is_volume_started(volinfo) == 0) {
+        *op_errstr = gf_strdup(
+            "Volume is stopped, start volume "
+            "to enable quota.");
+        ret = -1;
+        goto out;
+    }
+
+    ret = glusterd_check_if_quota_trans_enabled(volinfo);
+    if (ret == 0) {
+        *op_errstr = gf_strdup("Quota is already enabled");
+        ret = -1;
+        goto out;
+    }
+
+    ret = dict_set_dynstr_with_alloc(volinfo->dict, VKEY_FEATURES_QUOTA, "on");
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_SET_FAILED,
+               "dict set failed");
+        goto out;
+    }
+
+    ret = dict_set_dynstr_with_alloc(volinfo->dict, VKEY_FEATURES_INODE_QUOTA,
+                                     "on");
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+               "dict set failed");
+        goto out;
+    }
+
+    ret = dict_set_dynstr_with_alloc(volinfo->dict,
+                                     "features.quota-deem-statfs", "on");
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_SET_FAILED,
+               "setting quota-deem-statfs"
+               "in volinfo failed");
+        goto out;
+    }
+
+    *crawl = _gf_true;
+
+    ret = glusterd_store_quota_config(volinfo, NULL, NULL,
+                                      GF_QUOTA_OPTION_TYPE_ENABLE, op_errstr);
+
+    ret = 0;
+out:
+    if (ret && op_errstr && !*op_errstr)
+        gf_asprintf(op_errstr,
+                    "Enabling quota on volume %s has been "
+                    "unsuccessful",
+                    volinfo->volname);
+    return ret;
+}
 
-        ret = dict_set_dynstr (volinfo->dict, VKEY_FEATURES_QUOTA, quota_status);
+int32_t
+glusterd_quota_disable(glusterd_volinfo_t *volinfo, char **op_errstr,
+                       gf_boolean_t *crawl)
+{
+    int32_t ret = -1;
+    int i = 0;
+    char *value = NULL;
+    xlator_t *this = NULL;
+    glusterd_conf_t *conf = NULL;
+    char *quota_options[] = {"features.soft-timeout",
+                             "features.hard-timeout",
+                             "features.alert-time",
+                             "features.default-soft-limit",
+                             "features.quota-deem-statfs",
+                             "features.quota-timeout",
+                             NULL};
+
+    this = THIS;
+    GF_ASSERT(this);
+    conf = this->private;
+    GF_ASSERT(conf);
+
+    GF_VALIDATE_OR_GOTO(this->name, volinfo, out);
+    GF_VALIDATE_OR_GOTO(this->name, op_errstr, out);
+
+    ret = glusterd_check_if_quota_trans_enabled(volinfo);
+    if (ret == -1) {
+        *op_errstr = gf_strdup("Quota is already disabled");
+        goto out;
+    }
+
+    ret = dict_set_dynstr_with_alloc(volinfo->dict, VKEY_FEATURES_QUOTA, "off");
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_SET_FAILED,
+               "dict set failed");
+        goto out;
+    }
+
+    ret = dict_set_dynstr_with_alloc(volinfo->dict, VKEY_FEATURES_INODE_QUOTA,
+                                     "off");
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+               "dict set failed");
+        goto out;
+    }
+
+    for (i = 0; quota_options[i]; i++) {
+        ret = glusterd_volinfo_get(volinfo, quota_options[i], &value);
         if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "dict set failed");
-                goto out;
-        }
-
-        for (i = 0; quota_options [i]; i++) {
-                ret = glusterd_volinfo_get (volinfo, quota_options[i], &value);
-                if (ret) {
-                        gf_log (this->name, GF_LOG_INFO, "failed to get option"
-                                                         " %s",
-                                                         quota_options[i]);
-                } else {
-                dict_del (volinfo->dict, quota_options[i]);
-                }
+            gf_msg(this->name, GF_LOG_INFO, 0, GD_MSG_VOLINFO_GET_FAIL,
+                   "failed to get option"
+                   " %s",
+                   quota_options[i]);
+        } else {
+            dict_del(volinfo->dict, quota_options[i]);
         }
+    }
 
-        //Remove aux mount of the volume on every node in the cluster
-        ret = glusterd_remove_auxiliary_mount (volinfo->volname);
-        if (ret)
-                goto out;
-
-        *crawl = _gf_true;
+    *crawl = _gf_true;
 
-        (void) glusterd_clean_up_quota_store (volinfo);
+    (void)glusterd_clean_up_quota_store(volinfo);
 
-        ret = 0;
+    ret = 0;
 out:
-        if (ret && op_errstr && !*op_errstr)
-                gf_asprintf (op_errstr, "Disabling quota on volume %s has been "
-                             "unsuccessful", volinfo->volname);
-        return ret;
+    if (ret && op_errstr && !*op_errstr)
+        gf_asprintf(op_errstr,
+                    "Disabling quota on volume %s has been "
+                    "unsuccessful",
+                    volinfo->volname);
+    return ret;
 }
 
 static int
-glusterd_set_quota_limit (char *volname, char *path, char *hard_limit,
-                          char *soft_limit, char *key, char **op_errstr)
+glusterd_set_quota_limit(char *volname, char *path, char *hard_limit,
+                         char *soft_limit, char *key, char **op_errstr)
 {
-        int               ret                = -1;
-        xlator_t         *this               = NULL;
-        char              abspath[PATH_MAX]  = {0,};
-        glusterd_conf_t  *priv               = NULL;
-	quota_limits_t    existing_limit     = {0,};
-	quota_limits_t    new_limit          = {0,};
-        double            soft_limit_double  = 0;
-
-        this = THIS;
-        GF_ASSERT (this);
-        priv = this->private;
-        GF_ASSERT (priv);
-
-        GLUSTERD_GET_QUOTA_AUX_MOUNT_PATH (abspath, volname, path);
-        ret = gf_lstat_dir (abspath, NULL);
-        if (ret) {
-                gf_asprintf (op_errstr, "Failed to find the directory %s. "
-                             "Reason : %s", abspath, strerror (errno));
-                goto out;
-        }
-
-        if (!soft_limit) {
-                ret = sys_lgetxattr (abspath, key, (void *)&existing_limit,
-                                     sizeof (existing_limit));
-                if (ret < 0) {
-                        switch (errno) {
+    int ret = -1;
+    xlator_t *this = NULL;
+    char abspath[PATH_MAX] = {
+        0,
+    };
+    glusterd_conf_t *priv = NULL;
+    quota_limits_t existing_limit = {
+        0,
+    };
+    quota_limits_t new_limit = {
+        0,
+    };
+    double soft_limit_double = 0;
+    int64_t local_hl = 0;
+
+    this = THIS;
+    GF_ASSERT(this);
+    priv = this->private;
+    GF_ASSERT(priv);
+
+    GLUSTERD_GET_QUOTA_LIMIT_MOUNT_PATH(abspath, volname, path);
+    ret = gf_lstat_dir(abspath, NULL);
+    if (ret) {
+        gf_asprintf(op_errstr,
+                    "Failed to find the directory %s. "
+                    "Reason : %s",
+                    abspath, strerror(errno));
+        goto out;
+    }
+
+    if (!soft_limit) {
+        ret = sys_lgetxattr(abspath, key, (void *)&existing_limit,
+                            sizeof(existing_limit));
+        if (ret < 0) {
+            switch (errno) {
 #if defined(ENOATTR) && (ENOATTR != ENODATA)
-                        case ENODATA: /* FALLTHROUGH */
+                case ENODATA: /* FALLTHROUGH */
 #endif
-                        case ENOATTR:
-                                existing_limit.sl = -1;
-                            break;
-                        default:
-                                gf_asprintf (op_errstr, "Failed to get the "
-                                             "xattr %s from %s. Reason : %s",
-                                             key, abspath, strerror (errno));
-                                goto out;
-                        }
-                } else {
-                        existing_limit.hl = ntoh64 (existing_limit.hl);
-                        existing_limit.sl = ntoh64 (existing_limit.sl);
-                }
-                new_limit.sl = existing_limit.sl;
-
+                case ENOATTR:
+                    existing_limit.sl = -1;
+                    break;
+                default:
+                    gf_asprintf(op_errstr,
+                                "Failed to get the "
+                                "xattr %s from %s. Reason : %s",
+                                key, abspath, strerror(errno));
+                    goto out;
+            }
         } else {
-                ret = gf_string2percent (soft_limit, &soft_limit_double);
-                if (ret)
-                        goto out;
-                new_limit.sl = soft_limit_double;
+            existing_limit.hl = ntoh64(existing_limit.hl);
+            existing_limit.sl = ntoh64(existing_limit.sl);
         }
+        new_limit.sl = existing_limit.sl;
 
-        new_limit.sl = hton64 (new_limit.sl);
-
-        ret = gf_string2bytesize_int64 (hard_limit, &new_limit.hl);
+    } else {
+        ret = gf_string2percent(soft_limit, &soft_limit_double);
         if (ret)
-                goto out;
+            goto out;
+        new_limit.sl = soft_limit_double;
+    }
 
-        new_limit.hl = hton64 (new_limit.hl);
+    new_limit.sl = hton64(new_limit.sl);
 
-        ret = sys_lsetxattr (abspath, key, (char *)(void *)&new_limit,
-                             sizeof (new_limit), 0);
-        if (ret == -1) {
-                gf_asprintf (op_errstr, "setxattr of %s failed on %s."
-                             " Reason : %s", key, abspath, strerror (errno));
-                goto out;
-        }
-        ret = 0;
+    ret = gf_string2bytesize_int64(hard_limit, &local_hl);
+    if (ret)
+        goto out;
+
+    new_limit.hl = hton64(local_hl);
+
+    ret = sys_lsetxattr(abspath, key, (char *)(void *)&new_limit,
+                        sizeof(new_limit), 0);
+    if (ret == -1) {
+        gf_asprintf(op_errstr,
+                    "setxattr of %s failed on %s."
+                    " Reason : %s",
+                    key, abspath, strerror(errno));
+        goto out;
+    }
+    ret = 0;
 
 out:
-        return ret;
+    return ret;
 }
 
 static int
-glusterd_update_quota_conf_version (glusterd_volinfo_t *volinfo)
+glusterd_update_quota_conf_version(glusterd_volinfo_t *volinfo)
 {
-        volinfo->quota_conf_version++;
-        return 0;
+    volinfo->quota_conf_version++;
+    return 0;
 }
 
 /*The function glusterd_find_gfid_match () does the following:
@@ -545,670 +908,928 @@ glusterd_update_quota_conf_version (glusterd_volinfo_t *volinfo)
  * and continue the search.
  */
 static gf_boolean_t
-glusterd_find_gfid_match (uuid_t gfid, unsigned char *buf, size_t bytes_read,
-                          int opcode, size_t *write_byte_count)
+glusterd_find_gfid_match_3_6(uuid_t gfid, unsigned char *buf, size_t bytes_read,
+                             int opcode, size_t *write_byte_count)
 {
-        int           gfid_index  = 0;
-        int           shift_count = 0;
-        unsigned char tmp_buf[17] = {0,};
-
-        while (gfid_index != bytes_read) {
-                memcpy ((void *)tmp_buf, (void *)&buf[gfid_index], 16);
-                if (!gf_uuid_compare (gfid, tmp_buf)) {
-                        if (opcode == GF_QUOTA_OPTION_TYPE_REMOVE) {
-                                shift_count = bytes_read - (gfid_index + 16);
-                                memmove ((void *)&buf[gfid_index],
-                                         (void *)&buf[gfid_index+16],
-                                         shift_count);
-                                *write_byte_count = bytes_read - 16;
-                        } else {
-                                *write_byte_count = bytes_read;
-                        }
-                        return _gf_true;
-                } else {
-                        gfid_index+=16;
-                }
+    int gfid_index = 0;
+    int shift_count = 0;
+    unsigned char tmp_buf[17] = {
+        0,
+    };
+
+    /* This function if for backward compatibility */
+
+    while (gfid_index != bytes_read) {
+        memcpy((void *)tmp_buf, (void *)&buf[gfid_index], 16);
+        if (!gf_uuid_compare(gfid, tmp_buf)) {
+            if (opcode == GF_QUOTA_OPTION_TYPE_REMOVE) {
+                shift_count = bytes_read - (gfid_index + 16);
+                memmove((void *)&buf[gfid_index], (void *)&buf[gfid_index + 16],
+                        shift_count);
+                *write_byte_count = bytes_read - 16;
+            } else {
+                *write_byte_count = bytes_read;
+            }
+            return _gf_true;
+        } else {
+            gfid_index += 16;
         }
-        if (gfid_index == bytes_read)
+    }
+    if (gfid_index == bytes_read)
+        *write_byte_count = bytes_read;
+
+    return _gf_false;
+}
+
+static gf_boolean_t
+glusterd_find_gfid_match(uuid_t gfid, char gfid_type, unsigned char *buf,
+                         size_t bytes_read, int opcode,
+                         size_t *write_byte_count)
+{
+    int gfid_index = 0;
+    int shift_count = 0;
+    unsigned char tmp_buf[17] = {
+        0,
+    };
+    char type = 0;
+    xlator_t *this = NULL;
+    glusterd_conf_t *conf = NULL;
+
+    this = THIS;
+    GF_VALIDATE_OR_GOTO("glusterd", this, out);
+
+    conf = this->private;
+    GF_VALIDATE_OR_GOTO(this->name, conf, out);
+
+    if (conf->op_version < GD_OP_VERSION_3_7_0)
+        return glusterd_find_gfid_match_3_6(gfid, buf, bytes_read, opcode,
+                                            write_byte_count);
+
+    while (gfid_index != bytes_read) {
+        memcpy((void *)tmp_buf, (void *)&buf[gfid_index], 16);
+        type = buf[gfid_index + 16];
+
+        if (!gf_uuid_compare(gfid, tmp_buf) && type == gfid_type) {
+            if (opcode == GF_QUOTA_OPTION_TYPE_REMOVE ||
+                opcode == GF_QUOTA_OPTION_TYPE_REMOVE_OBJECTS) {
+                shift_count = bytes_read - (gfid_index + 17);
+                memmove((void *)&buf[gfid_index], (void *)&buf[gfid_index + 17],
+                        shift_count);
+                *write_byte_count = bytes_read - 17;
+            } else {
                 *write_byte_count = bytes_read;
+            }
+            return _gf_true;
+        } else {
+            gfid_index += 17;
+        }
+    }
+    if (gfid_index == bytes_read)
+        *write_byte_count = bytes_read;
+
+out:
 
-        return _gf_false;
+    return _gf_false;
 }
 
 /* The function glusterd_copy_to_tmp_file() reads the "remaining" bytes from
- * the source fd and writes them to destination fd, at the rate of 128K bytes
- * of read+write at a time.
+ * the source fd and writes them to destination fd, at the rate of 1000 entries
+ * a time (qconf_line_sz is the size of an entry)
  */
 
 static int
-glusterd_copy_to_tmp_file (int src_fd, int dst_fd)
+glusterd_copy_to_tmp_file(int src_fd, int dst_fd, int qconf_line_sz)
 {
-        int            ret         = 0;
-        size_t         entry_sz    = 131072;
-        ssize_t        bytes_read  = 0;
-        unsigned char  buf[131072] = {0,};
-        xlator_t      *this        = NULL;
-
-        this = THIS;
-        GF_ASSERT (this);
-
-        while ((bytes_read = read (src_fd, (void *)&buf, entry_sz)) > 0) {
-                if (bytes_read % 16 != 0) {
-                        gf_log (this->name, GF_LOG_ERROR, "quota.conf "
-                                "corrupted");
-                        ret = -1;
-                        goto out;
-                }
-                ret = write (dst_fd, (void *) buf, bytes_read);
-                if (ret == -1) {
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "write into quota.conf failed. Reason : %s",
-                                strerror (errno));
-                        goto out;
-                }
+    int ret = 0;
+    ssize_t bytes_read = 0;
+    xlator_t *this = NULL;
+    unsigned char *buf = 0;
+    int buf_sz = qconf_line_sz * 1000;
+
+    this = THIS;
+    GF_ASSERT(this);
+    GF_ASSERT(buf_sz > 0);
+
+    buf = GF_CALLOC(buf_sz, 1, gf_common_mt_char);
+    if (!buf) {
+        ret = -1;
+        goto out;
+    }
+
+    while ((bytes_read = sys_read(src_fd, buf, buf_sz)) > 0) {
+        if (bytes_read % qconf_line_sz != 0) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_QUOTA_CONF_CORRUPT,
+                   "quota.conf "
+                   "corrupted");
+            ret = -1;
+            goto out;
+        }
+        ret = sys_write(dst_fd, (void *)buf, bytes_read);
+        if (ret == -1) {
+            gf_msg(this->name, GF_LOG_ERROR, errno,
+                   GD_MSG_QUOTA_CONF_WRITE_FAIL,
+                   "write into quota.conf failed.");
+            goto out;
         }
-        ret = 0;
+    }
+    ret = 0;
 
 out:
-        return ret;
+    if (buf)
+        GF_FREE(buf);
+    return ret;
 }
 
 int
-glusterd_store_quota_config (glusterd_volinfo_t *volinfo, char *path,
-                             char *gfid_str, int opcode, char **op_errstr)
+glusterd_store_quota_conf_upgrade(glusterd_volinfo_t *volinfo)
 {
-        int                ret                   = -1;
-        int                fd                    = -1;
-        int                conf_fd               = -1;
-        size_t             entry_sz              = 131072;
-        ssize_t            bytes_read            = 0;
-        size_t            bytes_to_write         = 0;
-        unsigned char      buf[131072]           = {0,};
-        uuid_t             gfid                  = {0,};
-        xlator_t          *this                  = NULL;
-        gf_boolean_t       found                 = _gf_false;
-        gf_boolean_t       modified              = _gf_false;
-        gf_boolean_t       is_file_empty         = _gf_false;
-        gf_boolean_t       is_first_read         = _gf_true;
-        glusterd_conf_t   *conf                  = NULL;
-
-        this = THIS;
-        GF_ASSERT (this);
-        conf = this->private;
-        GF_ASSERT (conf);
-
-        glusterd_store_create_quota_conf_sh_on_absence (volinfo);
-
-        fd = gf_store_mkstemp (volinfo->quota_conf_shandle);
-        if (fd < 0) {
-                ret = -1;
-                goto out;
-        }
+    int ret = -1;
+    int fd = -1;
+    int conf_fd = -1;
+    unsigned char gfid[17] = {
+        0,
+    };
+    xlator_t *this = NULL;
+    char type = 0;
+
+    this = THIS;
+    GF_ASSERT(this);
+
+    fd = gf_store_mkstemp(volinfo->quota_conf_shandle);
+    if (fd < 0) {
+        ret = -1;
+        goto out;
+    }
+
+    conf_fd = open(volinfo->quota_conf_shandle->path, O_RDONLY);
+    if (conf_fd == -1) {
+        ret = -1;
+        goto out;
+    }
+
+    ret = quota_conf_skip_header(conf_fd);
+    if (ret)
+        goto out;
+
+    ret = glusterd_quota_conf_write_header(fd);
+    if (ret)
+        goto out;
+
+    while (1) {
+        ret = quota_conf_read_gfid(conf_fd, gfid, &type, 1.1);
+        if (ret == 0)
+            break;
+        else if (ret < 0)
+            goto out;
+
+        ret = glusterd_quota_conf_write_gfid(fd, gfid,
+                                             GF_QUOTA_CONF_TYPE_USAGE);
+        if (ret < 0)
+            goto out;
+    }
 
-        conf_fd = open (volinfo->quota_conf_shandle->path, O_RDONLY);
-        if (conf_fd == -1) {
-                ret = -1;
-                goto out;
-        }
+out:
+    if (conf_fd != -1)
+        sys_close(conf_fd);
 
-        ret = glusterd_store_quota_conf_skip_header (this, conf_fd);
+    if (ret && (fd > 0)) {
+        gf_store_unlink_tmppath(volinfo->quota_conf_shandle);
+    } else if (!ret) {
+        ret = gf_store_rename_tmppath(volinfo->quota_conf_shandle);
         if (ret) {
-                goto out;
+            gf_msg(this->name, GF_LOG_ERROR, errno, GD_MSG_FILE_OP_FAILED,
+                   "Failed to rename "
+                   "quota conf file");
+            return ret;
         }
 
-        ret = glusterd_store_quota_conf_stamp_header (this, fd);
+        ret = glusterd_compute_cksum(volinfo, _gf_true);
         if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "Failed to add header to tmp "
-                        "file.");
-                goto out;
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_CKSUM_COMPUTE_FAIL,
+                   "Failed to "
+                   "compute cksum for quota conf file");
+            return ret;
         }
 
-        /* Just create empty quota.conf file if create */
-        if (GF_QUOTA_OPTION_TYPE_ENABLE == opcode) {
-                modified = _gf_true;
-                goto out;
-        }
+        ret = glusterd_store_save_quota_version_and_cksum(volinfo);
+        if (ret)
+            gf_msg(this->name, GF_LOG_ERROR, 0,
+                   GD_MSG_QUOTA_CKSUM_VER_STORE_FAIL,
+                   "Failed to "
+                   "store quota version and cksum");
+    }
 
-        /* Check if gfid_str is given for opts other than ENABLE */
-        if (!gfid_str) {
-                ret = -1;
-                goto out;
+    return ret;
+}
+
+int
+glusterd_store_quota_config(glusterd_volinfo_t *volinfo, char *path,
+                            char *gfid_str, int opcode, char **op_errstr)
+{
+    int ret = -1;
+    int fd = -1;
+    int conf_fd = -1;
+    ssize_t bytes_read = 0;
+    size_t bytes_to_write = 0;
+    uuid_t gfid = {
+        0,
+    };
+    xlator_t *this = NULL;
+    gf_boolean_t found = _gf_false;
+    gf_boolean_t modified = _gf_false;
+    gf_boolean_t is_file_empty = _gf_false;
+    gf_boolean_t is_first_read = _gf_true;
+    glusterd_conf_t *conf = NULL;
+    float version = 0.0f;
+    char type = 0;
+    int quota_conf_line_sz = 16;
+    unsigned char *buf = 0;
+    int buf_sz = 0;
+
+    this = THIS;
+    GF_ASSERT(this);
+    conf = this->private;
+    GF_ASSERT(conf);
+
+    glusterd_store_create_quota_conf_sh_on_absence(volinfo);
+
+    conf_fd = open(volinfo->quota_conf_shandle->path, O_RDONLY);
+    if (conf_fd == -1) {
+        ret = -1;
+        goto out;
+    }
+
+    ret = quota_conf_read_version(conf_fd, &version);
+    if (ret)
+        goto out;
+
+    if (version < 1.2f && conf->op_version >= GD_OP_VERSION_3_7_0) {
+        /* Upgrade quota.conf file to newer format */
+        sys_close(conf_fd);
+        conf_fd = -1;
+
+        ret = glusterd_store_quota_conf_upgrade(volinfo);
+        if (ret)
+            goto out;
+
+        if (GF_QUOTA_OPTION_TYPE_UPGRADE == opcode) {
+            /* Nothing more to be done here */
+            goto out;
         }
-        gf_uuid_parse (gfid_str, gfid);
-
-        for (;;) {
-                bytes_read = read (conf_fd, (void*)&buf, entry_sz);
-                if (bytes_read <= 0) {
-                        /*The flag @is_first_read is TRUE when the loop is
-                         * entered, and is set to false if the first read
-                         * reads non-zero bytes of data. The flag is used to
-                         * detect if quota.conf is an empty file, but for the
-                         * header. This is done to log appropriate error message
-                         * when 'quota remove' is attempted when there are no
-                         * limits set on the given volume.
-                         */
-                        if (is_first_read)
-                                is_file_empty = _gf_true;
-                        break;
-                }
-                if ((bytes_read % 16) != 0) {
-                        gf_log (this->name, GF_LOG_ERROR, "quota.conf "
-                                "corrupted");
-                        ret = -1;
-                        goto out;
-                }
-                found = glusterd_find_gfid_match (gfid, buf, bytes_read, opcode,
-                                                  &bytes_to_write);
 
-                ret = write (fd, (void *) buf, bytes_to_write);
-                if (ret == -1) {
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "write into quota.conf failed. Reason : %s",
-                                strerror (errno));
-                        goto out;
-                }
+        conf_fd = open(volinfo->quota_conf_shandle->path, O_RDONLY);
+        if (conf_fd == -1) {
+            ret = -1;
+            goto out;
+        }
 
-                /*If the match is found in this iteration, copy the rest of
-                 * quota.conf into quota.conf.tmp and break.
-                 * Else continue with the search.
-                 */
-                if (found) {
-                        ret = glusterd_copy_to_tmp_file (conf_fd, fd);
-                        if (ret)
-                                goto out;
-                        break;
-                }
-                is_first_read = _gf_false;
+        ret = quota_conf_skip_header(conf_fd);
+        if (ret)
+            goto out;
+    } else if (GF_QUOTA_OPTION_TYPE_UPGRADE == opcode) {
+        /* No change to be done in quota_conf*/
+        goto out;
+    }
+
+    /* If op-ver is gt 3.7, then quota.conf will be upgraded, and 17 bytes
+     * storted in the new format. 16 bytes uuid and
+     * 1 byte type (usage/object)
+     */
+    if (conf->op_version >= GD_OP_VERSION_3_7_0)
+        quota_conf_line_sz++;
+
+    buf_sz = quota_conf_line_sz * 1000;
+
+    buf = GF_CALLOC(buf_sz, 1, gf_common_mt_char);
+    if (!buf) {
+        ret = -1;
+        goto out;
+    }
+
+    fd = gf_store_mkstemp(volinfo->quota_conf_shandle);
+    if (fd < 0) {
+        ret = -1;
+        goto out;
+    }
+
+    ret = glusterd_quota_conf_write_header(fd);
+    if (ret)
+        goto out;
+
+    /* Just create empty quota.conf file if create */
+    if (GF_QUOTA_OPTION_TYPE_ENABLE == opcode ||
+        GF_QUOTA_OPTION_TYPE_ENABLE_OBJECTS == opcode) {
+        modified = _gf_true;
+        goto out;
+    }
+
+    /* Check if gfid_str is given for opts other than ENABLE */
+    if (!gfid_str) {
+        ret = -1;
+        goto out;
+    }
+    gf_uuid_parse(gfid_str, gfid);
+
+    if (opcode > GF_QUOTA_OPTION_TYPE_VERSION_OBJECTS)
+        type = GF_QUOTA_CONF_TYPE_OBJECTS;
+    else
+        type = GF_QUOTA_CONF_TYPE_USAGE;
+
+    for (;;) {
+        bytes_read = sys_read(conf_fd, buf, buf_sz);
+        if (bytes_read <= 0) {
+            /*The flag @is_first_read is TRUE when the loop is
+             * entered, and is set to false if the first read
+             * reads non-zero bytes of data. The flag is used to
+             * detect if quota.conf is an empty file, but for the
+             * header. This is done to log appropriate error message
+             * when 'quota remove' is attempted when there are no
+             * limits set on the given volume.
+             */
+            if (is_first_read)
+                is_file_empty = _gf_true;
+            break;
+        }
+        if ((bytes_read % quota_conf_line_sz) != 0) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_QUOTA_CONF_CORRUPT,
+                   "quota.conf "
+                   "corrupted");
+            ret = -1;
+            goto out;
+        }
+        found = glusterd_find_gfid_match(gfid, type, buf, bytes_read, opcode,
+                                         &bytes_to_write);
+
+        ret = sys_write(fd, (void *)buf, bytes_to_write);
+        if (ret == -1) {
+            gf_msg(this->name, GF_LOG_ERROR, errno,
+                   GD_MSG_QUOTA_CONF_WRITE_FAIL,
+                   "write into quota.conf failed.");
+            goto out;
+        }
+
+        /*If the match is found in this iteration, copy the rest of
+         * quota.conf into quota.conf.tmp and break.
+         * Else continue with the search.
+         */
+        if (found) {
+            ret = glusterd_copy_to_tmp_file(conf_fd, fd, quota_conf_line_sz);
+            if (ret)
+                goto out;
+            break;
         }
+        is_first_read = _gf_false;
+    }
 
-        switch (opcode) {
+    switch (opcode) {
         case GF_QUOTA_OPTION_TYPE_LIMIT_USAGE:
+            if (!found) {
+                ret = glusterd_quota_conf_write_gfid(fd, gfid,
+                                                     GF_QUOTA_CONF_TYPE_USAGE);
+                if (ret == -1) {
+                    gf_msg(this->name, GF_LOG_ERROR, errno,
+                           GD_MSG_QUOTA_CONF_WRITE_FAIL,
+                           "write into quota.conf failed. ");
+                    goto out;
+                }
+                modified = _gf_true;
+            }
+            break;
         case GF_QUOTA_OPTION_TYPE_LIMIT_OBJECTS:
-                if (!found) {
-                        ret = write (fd, gfid, 16);
-                        if (ret == -1) {
-                                gf_log (this->name, GF_LOG_ERROR,
-                                        "write into quota.conf failed. "
-                                        "Reason : %s",
-                                        strerror (errno));
-                                goto out;
-                        }
-                        modified = _gf_true;
+            if (!found) {
+                ret = glusterd_quota_conf_write_gfid(
+                    fd, gfid, GF_QUOTA_CONF_TYPE_OBJECTS);
+                if (ret == -1) {
+                    gf_msg(this->name, GF_LOG_ERROR, errno,
+                           GD_MSG_QUOTA_CONF_WRITE_FAIL,
+                           "write into quota.conf failed. ");
+                    goto out;
                 }
-                break;
+                modified = _gf_true;
+            }
+            break;
 
         case GF_QUOTA_OPTION_TYPE_REMOVE:
         case GF_QUOTA_OPTION_TYPE_REMOVE_OBJECTS:
-                if (is_file_empty) {
-                        gf_asprintf (op_errstr, "Cannot remove limit on"
-                                     " %s. The quota configuration file"
-                                     " for volume %s is empty.", path,
-                                     volinfo->volname);
-                        ret = -1;
-                        goto out;
+            if (is_file_empty) {
+                gf_asprintf(op_errstr,
+                            "Cannot remove limit on"
+                            " %s. The quota configuration file"
+                            " for volume %s is empty.",
+                            path, volinfo->volname);
+                ret = -1;
+                goto out;
+            } else {
+                if (!found) {
+                    gf_asprintf(op_errstr,
+                                "Error. gfid %s"
+                                " for path %s not found in"
+                                " store",
+                                gfid_str, path);
+                    ret = -1;
+                    goto out;
                 } else {
-                        if (!found) {
-                                gf_asprintf (op_errstr, "Error. gfid %s"
-                                             " for path %s not found in"
-                                             " store", gfid_str, path);
-                                ret = -1;
-                                goto out;
-                        } else {
-                                modified = _gf_true;
-                        }
+                    modified = _gf_true;
                 }
-                break;
+            }
+            break;
 
         default:
-                ret = 0;
-                break;
-        }
+            ret = 0;
+            break;
+    }
 
-        if (modified)
-                glusterd_update_quota_conf_version (volinfo);
+    if (modified)
+        glusterd_update_quota_conf_version(volinfo);
 
-        ret = 0;
+    ret = 0;
 out:
-        if (conf_fd != -1) {
-                close (conf_fd);
-        }
-
-        if (ret && (fd > 0)) {
-                gf_store_unlink_tmppath (volinfo->quota_conf_shandle);
-        } else if (!ret) {
-                ret = gf_store_rename_tmppath (volinfo->quota_conf_shandle);
-                if (modified) {
-                        ret = glusterd_compute_cksum (volinfo, _gf_true);
-                        if (ret) {
-                                gf_log (this->name, GF_LOG_ERROR, "Failed to "
-                                        "compute cksum for quota conf file");
-                                return ret;
-                        }
-
-                        ret = glusterd_store_save_quota_version_and_cksum
-                                                                      (volinfo);
-                        if (ret)
-                                gf_log (this->name, GF_LOG_ERROR, "Failed to "
-                                        "store quota version and cksum");
-                }
-        }
-
-        return ret;
+    if (conf_fd != -1) {
+        sys_close(conf_fd);
+    }
+
+    if (buf)
+        GF_FREE(buf);
+
+    if (ret && (fd > 0)) {
+        gf_store_unlink_tmppath(volinfo->quota_conf_shandle);
+    } else if (!ret && GF_QUOTA_OPTION_TYPE_UPGRADE != opcode) {
+        ret = gf_store_rename_tmppath(volinfo->quota_conf_shandle);
+        if (modified) {
+            ret = glusterd_compute_cksum(volinfo, _gf_true);
+            if (ret) {
+                gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_CKSUM_COMPUTE_FAIL,
+                       "Failed to "
+                       "compute cksum for quota conf file");
+                return ret;
+            }
+
+            ret = glusterd_store_save_quota_version_and_cksum(volinfo);
+            if (ret)
+                gf_msg(this->name, GF_LOG_ERROR, 0,
+                       GD_MSG_VERS_CKSUM_STORE_FAIL,
+                       "Failed to "
+                       "store quota version and cksum");
+        }
+    }
+    return ret;
 }
 
 int32_t
-glusterd_quota_limit_usage (glusterd_volinfo_t *volinfo, dict_t *dict,
-                            int opcode, char **op_errstr)
+glusterd_quota_limit_usage(glusterd_volinfo_t *volinfo, dict_t *dict,
+                           int opcode, char **op_errstr)
 {
-        int32_t          ret                = -1;
-        char            *path               = NULL;
-        char            *hard_limit         = NULL;
-        char            *soft_limit         = NULL;
-        char            *gfid_str           = NULL;
-        xlator_t        *this               = NULL;
-
-        this = THIS;
-        GF_ASSERT (this);
-
-        GF_VALIDATE_OR_GOTO (this->name, dict, out);
-        GF_VALIDATE_OR_GOTO (this->name, volinfo, out);
-        GF_VALIDATE_OR_GOTO (this->name, op_errstr, out);
-
-        ret = glusterd_check_if_quota_trans_enabled (volinfo);
-        if (ret == -1) {
-                *op_errstr = gf_strdup ("Quota is disabled, please enable "
-                                        "quota");
-                goto out;
-        }
-
-        ret = dict_get_str (dict, "path", &path);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "Unable to fetch path");
-                goto out;
-        }
-        ret = gf_canonicalize_path (path);
-        if (ret)
-                goto out;
-
-        ret = dict_get_str (dict, "hard-limit", &hard_limit);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "Unable to fetch hard limit");
-                goto out;
-        }
-
-        if (dict_get (dict, "soft-limit")) {
-                ret = dict_get_str (dict, "soft-limit", &soft_limit);
-                if (ret) {
-                        gf_log (this->name, GF_LOG_ERROR, "Unable to fetch "
-                                "soft limit");
-                        goto out;
-                }
-        }
-
-        if (is_origin_glusterd (dict)) {
-                if (opcode == GF_QUOTA_OPTION_TYPE_LIMIT_USAGE) {
-                        ret = glusterd_set_quota_limit (volinfo->volname, path,
-                                                        hard_limit, soft_limit,
-                                                        QUOTA_LIMIT_KEY,
-                                                        op_errstr);
-                } else {
-                        ret = glusterd_set_quota_limit (volinfo->volname, path,
-                                                        hard_limit, soft_limit,
-                                                        QUOTA_LIMIT_OBJECTS_KEY,
-                                                        op_errstr);
-                }
-                if (ret)
-                        goto out;
-        }
-
-        ret = dict_get_str (dict, "gfid", &gfid_str);
+    int32_t ret = -1;
+    char *path = NULL;
+    char *hard_limit = NULL;
+    char *soft_limit = NULL;
+    char *gfid_str = NULL;
+    xlator_t *this = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+
+    GF_VALIDATE_OR_GOTO(this->name, dict, out);
+    GF_VALIDATE_OR_GOTO(this->name, volinfo, out);
+    GF_VALIDATE_OR_GOTO(this->name, op_errstr, out);
+
+    ret = glusterd_check_if_quota_trans_enabled(volinfo);
+    if (ret == -1) {
+        *op_errstr = gf_strdup(
+            "Quota is disabled, please enable "
+            "quota");
+        goto out;
+    }
+
+    ret = dict_get_strn(dict, "path", SLEN("path"), &path);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+               "Unable to fetch path");
+        goto out;
+    }
+    ret = gf_canonicalize_path(path);
+    if (ret)
+        goto out;
+
+    ret = dict_get_strn(dict, "hard-limit", SLEN("hard-limit"), &hard_limit);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+               "Unable to fetch hard limit");
+        goto out;
+    }
+
+    if (dict_getn(dict, "soft-limit", SLEN("soft-limit"))) {
+        ret = dict_get_strn(dict, "soft-limit", SLEN("soft-limit"),
+                            &soft_limit);
         if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "Failed to get gfid of path "
-                        "%s", path);
-                goto out;
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+                   "Unable to fetch "
+                   "soft limit");
+            goto out;
         }
+    }
 
-        ret = glusterd_store_quota_config (volinfo, path, gfid_str, opcode,
+    if (is_origin_glusterd(dict)) {
+        if (opcode == GF_QUOTA_OPTION_TYPE_LIMIT_USAGE) {
+            ret = glusterd_set_quota_limit(volinfo->volname, path, hard_limit,
+                                           soft_limit, QUOTA_LIMIT_KEY,
+                                           op_errstr);
+        } else {
+            ret = glusterd_set_quota_limit(volinfo->volname, path, hard_limit,
+                                           soft_limit, QUOTA_LIMIT_OBJECTS_KEY,
                                            op_errstr);
+        }
         if (ret)
-                goto out;
-
-        ret = 0;
+            goto out;
+    }
+
+    ret = dict_get_strn(dict, "gfid", SLEN("gfid"), &gfid_str);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+               "Failed to get gfid of path "
+               "%s",
+               path);
+        goto out;
+    }
+
+    ret = glusterd_store_quota_config(volinfo, path, gfid_str, opcode,
+                                      op_errstr);
+    if (ret)
+        goto out;
+
+    ret = 0;
 out:
 
-        if (ret && op_errstr && !*op_errstr)
-                gf_asprintf (op_errstr, "Failed to set hard limit on path %s "
-                             "for volume %s", path, volinfo->volname);
-        return ret;
+    if (ret && op_errstr && !*op_errstr)
+        gf_asprintf(op_errstr,
+                    "Failed to set hard limit on path %s "
+                    "for volume %s",
+                    path, volinfo->volname);
+    return ret;
 }
 
 static int
-glusterd_remove_quota_limit (char *volname, char *path, char **op_errstr,
-                             int type)
+glusterd_remove_quota_limit(char *volname, char *path, char **op_errstr,
+                            int type)
 {
-        int               ret                = -1;
-        xlator_t         *this               = NULL;
-        char              abspath[PATH_MAX]  = {0,};
-        glusterd_conf_t  *priv               = NULL;
-
-        this = THIS;
-        GF_ASSERT (this);
-        priv = this->private;
-        GF_ASSERT (priv);
-
-        GLUSTERD_GET_QUOTA_AUX_MOUNT_PATH (abspath, volname, path);
-        ret = gf_lstat_dir (abspath, NULL);
+    int ret = -1;
+    xlator_t *this = NULL;
+    char abspath[PATH_MAX] = {
+        0,
+    };
+    glusterd_conf_t *priv = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+    priv = this->private;
+    GF_ASSERT(priv);
+
+    GLUSTERD_GET_QUOTA_LIMIT_MOUNT_PATH(abspath, volname, path);
+    ret = gf_lstat_dir(abspath, NULL);
+    if (ret) {
+        gf_asprintf(op_errstr,
+                    "Failed to find the directory %s. "
+                    "Reason : %s",
+                    abspath, strerror(errno));
+        goto out;
+    }
+
+    if (type == GF_QUOTA_OPTION_TYPE_REMOVE) {
+        ret = sys_lremovexattr(abspath, QUOTA_LIMIT_KEY);
         if (ret) {
-                gf_asprintf (op_errstr, "Failed to find the directory %s. "
-                             "Reason : %s", abspath, strerror (errno));
-                goto out;
-        }
-
-        if (type == GF_QUOTA_OPTION_TYPE_REMOVE) {
-                ret = sys_lremovexattr (abspath,
-                                        "trusted.glusterfs.quota.limit-set");
-                if (ret) {
-                        gf_asprintf (op_errstr, "removexattr failed on %s. "
-                                     "Reason : %s", abspath, strerror (errno));
-                        goto out;
-                }
+            gf_asprintf(op_errstr,
+                        "removexattr failed on %s. "
+                        "Reason : %s",
+                        abspath, strerror(errno));
+            goto out;
         }
+    }
 
-        if (type == GF_QUOTA_OPTION_TYPE_REMOVE_OBJECTS) {
-                ret = sys_lremovexattr (abspath,
-                                "trusted.glusterfs.quota.limit-objects");
-                if (ret) {
-                        gf_asprintf (op_errstr, "removexattr failed on %s. "
-                                     "Reason : %s", abspath, strerror (errno));
-                        goto out;
-                }
+    if (type == GF_QUOTA_OPTION_TYPE_REMOVE_OBJECTS) {
+        ret = sys_lremovexattr(abspath, QUOTA_LIMIT_OBJECTS_KEY);
+        if (ret) {
+            gf_asprintf(op_errstr,
+                        "removexattr failed on %s. "
+                        "Reason : %s",
+                        abspath, strerror(errno));
+            goto out;
         }
-        ret = 0;
+    }
+    ret = 0;
 
 out:
-        return ret;
+    return ret;
 }
 
 int32_t
-glusterd_quota_remove_limits (glusterd_volinfo_t *volinfo, dict_t *dict,
-                              int opcode, char **op_errstr, int type)
+glusterd_quota_remove_limits(glusterd_volinfo_t *volinfo, dict_t *dict,
+                             int opcode, char **op_errstr, int type)
 {
-        int32_t         ret                   = -1;
-        char            *path                 = NULL;
-        char            *gfid_str             = NULL;
-        xlator_t        *this                 = NULL;
-
-        this = THIS;
-        GF_ASSERT (this);
-
-        GF_VALIDATE_OR_GOTO (this->name, dict, out);
-        GF_VALIDATE_OR_GOTO (this->name, volinfo, out);
-        GF_VALIDATE_OR_GOTO (this->name, op_errstr, out);
-
-        ret = glusterd_check_if_quota_trans_enabled (volinfo);
-        if (ret == -1) {
-                *op_errstr = gf_strdup ("Quota is disabled, please enable "
-                                        "quota");
-                goto out;
-        }
-
-        ret = dict_get_str (dict, "path", &path);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "Unable to fetch path");
-                goto out;
-        }
-
-        ret = gf_canonicalize_path (path);
+    int32_t ret = -1;
+    char *path = NULL;
+    char *gfid_str = NULL;
+    xlator_t *this = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+
+    GF_VALIDATE_OR_GOTO(this->name, dict, out);
+    GF_VALIDATE_OR_GOTO(this->name, volinfo, out);
+    GF_VALIDATE_OR_GOTO(this->name, op_errstr, out);
+
+    ret = glusterd_check_if_quota_trans_enabled(volinfo);
+    if (ret == -1) {
+        *op_errstr = gf_strdup(
+            "Quota is disabled, please enable "
+            "quota");
+        goto out;
+    }
+
+    ret = dict_get_strn(dict, "path", SLEN("path"), &path);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+               "Unable to fetch path");
+        goto out;
+    }
+
+    ret = gf_canonicalize_path(path);
+    if (ret)
+        goto out;
+
+    if (is_origin_glusterd(dict)) {
+        ret = glusterd_remove_quota_limit(volinfo->volname, path, op_errstr,
+                                          type);
         if (ret)
-                goto out;
-
-        if (is_origin_glusterd (dict)) {
-                ret = glusterd_remove_quota_limit (volinfo->volname, path,
-                                                   op_errstr, type);
-                if (ret)
-                        goto out;
-        }
-
-        ret = dict_get_str (dict, "gfid", &gfid_str);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "Failed to get gfid of path "
-                        "%s", path);
-                goto out;
-        }
+            goto out;
+    }
 
-        ret = glusterd_store_quota_config (volinfo, path, gfid_str, opcode,
-                                           op_errstr);
-        if (ret)
-                goto out;
+    ret = dict_get_strn(dict, "gfid", SLEN("gfid"), &gfid_str);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+               "Failed to get gfid of path "
+               "%s",
+               path);
+        goto out;
+    }
 
+    ret = glusterd_store_quota_config(volinfo, path, gfid_str, opcode,
+                                      op_errstr);
+    if (ret)
+        goto out;
 
-        ret = 0;
+    ret = 0;
 
 out:
-        return ret;
+    return ret;
 }
 
 int
-glusterd_set_quota_option (glusterd_volinfo_t *volinfo, dict_t *dict,
-                           char *key, char **op_errstr)
+glusterd_set_quota_option(glusterd_volinfo_t *volinfo, dict_t *dict, char *key,
+                          char **op_errstr)
 {
-        int        ret    = 0;
-        char      *value  = NULL;
-        xlator_t  *this   = NULL;
-        char      *option = NULL;
-
-        this = THIS;
-        GF_ASSERT (this);
-
-        ret = glusterd_check_if_quota_trans_enabled (volinfo);
-        if (ret == -1) {
-                gf_asprintf (op_errstr, "Cannot set %s. Quota on volume %s is "
-                                        "disabled", key, volinfo->volname);
-                return -1;
-        }
-
-        ret = dict_get_str (dict, "value", &value);
-        if(ret) {
-                gf_log (this->name, GF_LOG_ERROR, "Option value absent.");
-                return -1;
-        }
-
-        option = gf_strdup (value);
-        ret = dict_set_dynstr (volinfo->dict, key, option);
-        if(ret) {
-                gf_log (this->name, GF_LOG_ERROR, "Failed to set option %s",
-                        key);
-                return -1;
-        }
-
-        return 0;
+    int ret = 0;
+    char *value = NULL;
+    xlator_t *this = NULL;
+    char *option = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+
+    ret = glusterd_check_if_quota_trans_enabled(volinfo);
+    if (ret == -1) {
+        gf_asprintf(op_errstr,
+                    "Cannot set %s. Quota on volume %s is "
+                    "disabled",
+                    key, volinfo->volname);
+        return -1;
+    }
+
+    ret = dict_get_strn(dict, "value", SLEN("value"), &value);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+               "Option value absent.");
+        return -1;
+    }
+
+    option = gf_strdup(value);
+    ret = dict_set_dynstr(volinfo->dict, key, option);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+               "Failed to set option %s", key);
+        return -1;
+    }
+
+    return 0;
 }
 
 static int
-glusterd_quotad_op (int opcode)
+glusterd_quotad_op(int opcode)
 {
-        int              ret  = -1;
-        xlator_t        *this = NULL;
-        glusterd_conf_t *priv = NULL;
+    int ret = -1;
+    xlator_t *this = NULL;
+    glusterd_conf_t *priv = NULL;
 
-        this = THIS;
-        GF_ASSERT (this);
+    this = THIS;
+    GF_ASSERT(this);
 
-        priv = this->private;
-        GF_ASSERT (priv);
+    priv = this->private;
+    GF_ASSERT(priv);
 
-        switch (opcode) {
-                case GF_QUOTA_OPTION_TYPE_ENABLE:
-                case GF_QUOTA_OPTION_TYPE_DISABLE:
+    switch (opcode) {
+        case GF_QUOTA_OPTION_TYPE_ENABLE:
+        case GF_QUOTA_OPTION_TYPE_DISABLE:
 
-                        if (glusterd_all_volumes_with_quota_stopped ())
-                                ret = glusterd_svc_stop (&(priv->quotad_svc),
-                                                         SIGTERM);
-                        else
-                                ret = priv->quotad_svc.manager
-                                                (&(priv->quotad_svc), NULL,
-                                                 PROC_START);
-                        break;
+            if (glusterd_all_volumes_with_quota_stopped())
+                ret = glusterd_svc_stop(&(priv->quotad_svc), SIGTERM);
+            else
+                ret = priv->quotad_svc.manager(&(priv->quotad_svc), NULL,
+                                               PROC_START);
+            break;
 
-                default:
-                        ret = 0;
-                        break;
-        }
-        return ret;
+        default:
+            ret = 0;
+            break;
+    }
+    return ret;
 }
 
 int
-glusterd_op_quota (dict_t *dict, char **op_errstr, dict_t *rsp_dict)
+glusterd_op_quota(dict_t *dict, char **op_errstr, dict_t *rsp_dict)
 {
-        glusterd_volinfo_t     *volinfo      = NULL;
-        int32_t                 ret          = -1;
-        char                   *volname      = NULL;
-        int                     type         = -1;
-        gf_boolean_t            start_crawl  = _gf_false;
-        glusterd_conf_t        *priv         = NULL;
-        xlator_t               *this         = NULL;
-
-        GF_ASSERT (dict);
-        GF_ASSERT (op_errstr);
-
-        this = THIS;
-        GF_ASSERT (this);
-        priv = this->private;
-        GF_ASSERT (priv);
-
-        ret = dict_get_str (dict, "volname", &volname);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "Unable to get volume name");
+    glusterd_volinfo_t *volinfo = NULL;
+    int32_t ret = -1;
+    char *volname = NULL;
+    int type = -1;
+    gf_boolean_t start_crawl = _gf_false;
+    glusterd_conf_t *priv = NULL;
+    xlator_t *this = NULL;
+
+    GF_ASSERT(dict);
+    GF_ASSERT(op_errstr);
+
+    this = THIS;
+    GF_ASSERT(this);
+    priv = this->private;
+    GF_ASSERT(priv);
+
+    ret = dict_get_strn(dict, "volname", SLEN("volname"), &volname);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+               "Unable to get volume name");
+        goto out;
+    }
+
+    ret = glusterd_volinfo_find(volname, &volinfo);
+    if (ret) {
+        gf_asprintf(op_errstr, FMTSTR_CHECK_VOL_EXISTS, volname);
+        goto out;
+    }
+
+    ret = dict_get_int32n(dict, "type", SLEN("type"), &type);
+
+    if (!glusterd_is_quota_supported(type, op_errstr)) {
+        ret = -1;
+        goto out;
+    }
+
+    switch (type) {
+        case GF_QUOTA_OPTION_TYPE_ENABLE:
+            ret = glusterd_quota_enable(volinfo, op_errstr, &start_crawl);
+            if (ret < 0)
                 goto out;
-        }
+            break;
 
-        ret = glusterd_volinfo_find (volname, &volinfo);
-        if (ret) {
-                gf_asprintf (op_errstr, FMTSTR_CHECK_VOL_EXISTS, volname);
+        case GF_QUOTA_OPTION_TYPE_ENABLE_OBJECTS:
+            ret = glusterd_inode_quota_enable(volinfo, op_errstr, &start_crawl);
+            if (ret < 0)
                 goto out;
-        }
-
-        ret = dict_get_int32 (dict, "type", &type);
+            break;
 
-        if ((priv->op_version == GD_OP_VERSION_MIN) &&
-            (type > GF_QUOTA_OPTION_TYPE_VERSION)) {
-                gf_asprintf (op_errstr, "Volume quota failed. The cluster is "
-                                        "operating at version %d. Quota command"
-                                        " %s is unavailable in this version.",
-                                        priv->op_version,
-                                        gd_quota_op_list[type]);
-                ret = -1;
+        case GF_QUOTA_OPTION_TYPE_DISABLE:
+            ret = glusterd_quota_disable(volinfo, op_errstr, &start_crawl);
+            if (ret < 0)
                 goto out;
-        }
 
-        switch (type) {
-                case GF_QUOTA_OPTION_TYPE_ENABLE:
-                        ret = glusterd_quota_enable (volinfo, op_errstr,
-                                                     &start_crawl);
-                        if (ret < 0)
-                                goto out;
-                        break;
-
-                case GF_QUOTA_OPTION_TYPE_DISABLE:
-                        ret = glusterd_quota_disable (volinfo, op_errstr,
-                                                      &start_crawl);
-                        if (ret < 0)
-                                goto out;
-
-                        break;
-
-                case GF_QUOTA_OPTION_TYPE_LIMIT_USAGE:
-                case GF_QUOTA_OPTION_TYPE_LIMIT_OBJECTS:
-                        ret = glusterd_quota_limit_usage (volinfo, dict, type,
-                                                          op_errstr);
-                        goto out;
-
-                case GF_QUOTA_OPTION_TYPE_REMOVE:
-                case GF_QUOTA_OPTION_TYPE_REMOVE_OBJECTS:
-                        ret = glusterd_quota_remove_limits (volinfo, dict, type,
-                                                            op_errstr, type);
-                        goto out;
-
-                case GF_QUOTA_OPTION_TYPE_LIST:
-                case GF_QUOTA_OPTION_TYPE_LIST_OBJECTS:
-                        ret = glusterd_check_if_quota_trans_enabled (volinfo);
-                        if (ret == -1) {
-                                *op_errstr = gf_strdup ("Cannot list limits, "
-                                                        "quota is disabled");
-                                goto out;
-                        }
-                        ret = glusterd_quota_get_default_soft_limit (volinfo,
-                                                               rsp_dict);
-                        goto out;
-
-                case GF_QUOTA_OPTION_TYPE_SOFT_TIMEOUT:
-                        ret = glusterd_set_quota_option (volinfo, dict,
-                                                        "features.soft-timeout",
-                                                         op_errstr);
-                        if (ret)
-                                goto out;
-                        break;
-
-                case GF_QUOTA_OPTION_TYPE_HARD_TIMEOUT:
-                        ret = glusterd_set_quota_option (volinfo, dict,
-                                                        "features.hard-timeout",
-                                                         op_errstr);
-                        if (ret)
-                                goto out;
-                        break;
-
-                case GF_QUOTA_OPTION_TYPE_ALERT_TIME:
-                        ret = glusterd_set_quota_option (volinfo, dict,
-                                                         "features.alert-time",
-                                                         op_errstr);
-                        if (ret)
-                                goto out;
-                        break;
-
-                case GF_QUOTA_OPTION_TYPE_DEFAULT_SOFT_LIMIT:
-                        ret = glusterd_set_quota_option (volinfo, dict,
-                                                  "features.default-soft-limit",
-                                                  op_errstr);
-                        if (ret)
-                                goto out;
-                        break;
+            break;
 
-                default:
-                        gf_asprintf (op_errstr, "Quota command failed. Invalid "
-                                     "opcode");
-                        ret = -1;
-                        goto out;
-        }
+        case GF_QUOTA_OPTION_TYPE_LIMIT_USAGE:
+        case GF_QUOTA_OPTION_TYPE_LIMIT_OBJECTS:
+            ret = glusterd_quota_limit_usage(volinfo, dict, type, op_errstr);
+            goto out;
 
-        if (priv->op_version > GD_OP_VERSION_MIN) {
-                ret = glusterd_quotad_op (type);
-                if (ret)
-                        goto out;
-        }
+        case GF_QUOTA_OPTION_TYPE_REMOVE:
+        case GF_QUOTA_OPTION_TYPE_REMOVE_OBJECTS:
+            ret = glusterd_quota_remove_limits(volinfo, dict, type, op_errstr,
+                                               type);
+            goto out;
 
-        ret = glusterd_create_volfiles_and_notify_services (volinfo);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "Unable to re-create "
-                                                  "volfiles");
-                ret = -1;
+        case GF_QUOTA_OPTION_TYPE_LIST:
+        case GF_QUOTA_OPTION_TYPE_LIST_OBJECTS:
+            ret = glusterd_check_if_quota_trans_enabled(volinfo);
+            if (ret == -1) {
+                *op_errstr = gf_strdup(
+                    "Cannot list limits, "
+                    "quota is disabled");
                 goto out;
-        }
+            }
+            ret = glusterd_quota_get_default_soft_limit(volinfo, rsp_dict);
+            goto out;
 
-        ret = glusterd_store_volinfo (volinfo, GLUSTERD_VOLINFO_VER_AC_INCREMENT);
-        if (ret)
+        case GF_QUOTA_OPTION_TYPE_SOFT_TIMEOUT:
+            ret = glusterd_set_quota_option(volinfo, dict,
+                                            "features.soft-timeout", op_errstr);
+            if (ret)
                 goto out;
+            break;
 
-        if (GLUSTERD_STATUS_STARTED == volinfo->status) {
-                if (priv->op_version == GD_OP_VERSION_MIN)
-                        ret = priv->nfs_svc.manager (&(priv->nfs_svc), NULL, 0);
-        }
+        case GF_QUOTA_OPTION_TYPE_HARD_TIMEOUT:
+            ret = glusterd_set_quota_option(volinfo, dict,
+                                            "features.hard-timeout", op_errstr);
+            if (ret)
+                goto out;
+            break;
+
+        case GF_QUOTA_OPTION_TYPE_ALERT_TIME:
+            ret = glusterd_set_quota_option(volinfo, dict,
+                                            "features.alert-time", op_errstr);
+            if (ret)
+                goto out;
+            break;
+
+        case GF_QUOTA_OPTION_TYPE_DEFAULT_SOFT_LIMIT:
+            ret = glusterd_set_quota_option(
+                volinfo, dict, "features.default-soft-limit", op_errstr);
+            if (ret)
+                goto out;
+            break;
+
+        default:
+            gf_asprintf(op_errstr,
+                        "Quota command failed. Invalid "
+                        "opcode");
+            ret = -1;
+            goto out;
+    }
+
+    if (priv->op_version > GD_OP_VERSION_MIN) {
+        ret = glusterd_quotad_op(type);
+        if (ret)
+            goto out;
+    }
+
+    if (GF_QUOTA_OPTION_TYPE_ENABLE == type)
+        volinfo->quota_xattr_version++;
+    ret = glusterd_store_volinfo(volinfo, GLUSTERD_VOLINFO_VER_AC_INCREMENT);
+    if (ret) {
+        if (GF_QUOTA_OPTION_TYPE_ENABLE == type)
+            volinfo->quota_xattr_version--;
+        goto out;
+    }
+
+    ret = glusterd_create_volfiles_and_notify_services(volinfo);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_VOLFILE_CREATE_FAIL,
+               "Unable to re-create "
+               "volfiles");
+        if (GF_QUOTA_OPTION_TYPE_ENABLE == type) {
+            /* rollback volinfo */
+            volinfo->quota_xattr_version--;
+            ret = glusterd_store_volinfo(volinfo,
+                                         GLUSTERD_VOLINFO_VER_AC_INCREMENT);
+            if (ret) {
+                gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_VOLINFO_SET_FAIL,
+                       "Failed to store volinfo for volume %s",
+                       volinfo->volname);
+            }
+        }
+
+        ret = -1;
+        goto out;
+    }
+
+#if BUILD_GNFS
+    if (GLUSTERD_STATUS_STARTED == volinfo->status) {
+        if (priv->op_version == GD_OP_VERSION_MIN)
+            (void)priv->nfs_svc.manager(&(priv->nfs_svc), NULL, 0);
+    }
+#endif
 
-        if (rsp_dict && start_crawl == _gf_true)
-                glusterd_quota_initiate_fs_crawl (priv, volname, type);
+    if (rsp_dict && start_crawl == _gf_true)
+        glusterd_quota_initiate_fs_crawl(priv, volinfo, type);
 
-        ret = 0;
+    ret = 0;
 out:
-        return ret;
+    if (type == GF_QUOTA_OPTION_TYPE_LIMIT_USAGE ||
+        type == GF_QUOTA_OPTION_TYPE_LIMIT_OBJECTS ||
+        type == GF_QUOTA_OPTION_TYPE_REMOVE ||
+        type == GF_QUOTA_OPTION_TYPE_REMOVE_OBJECTS) {
+        /* During a list operation we need the aux mount to be
+         * accessible until the listing is done at the cli
+         */
+        glusterd_remove_auxiliary_mount(volinfo->volname);
+    }
+
+    return ret;
 }
 
 /*
@@ -1218,297 +1839,421 @@ out:
  * is not treated as error.
  */
 static int
-glusterd_get_gfid_from_brick (dict_t *dict, glusterd_volinfo_t *volinfo,
-                              dict_t *rsp_dict, char **op_errstr)
+glusterd_get_gfid_from_brick(dict_t *dict, glusterd_volinfo_t *volinfo,
+                             dict_t *rsp_dict, char **op_errstr)
 {
-        int                    ret                    = -1;
-        int                    count                  = 0;
-        char                  *path                   = NULL;
-        char                   backend_path[PATH_MAX] = {0,};
-        xlator_t              *this                   = NULL;
-        glusterd_conf_t       *priv                   = NULL;
-        glusterd_brickinfo_t  *brickinfo              = NULL;
-        char                   key[256]               = {0,};
-        char                  *gfid_str               = NULL;
-        uuid_t                 gfid;
-
-        this = THIS;
-        GF_ASSERT (this);
-        priv = this->private;
-        GF_ASSERT (priv);
-
-        ret = dict_get_str (dict, "path", &path);
+    int ret = -1;
+    int count = 0;
+    char *path = NULL;
+    char backend_path[PATH_MAX] = {
+        0,
+    };
+    xlator_t *this = NULL;
+    glusterd_conf_t *priv = NULL;
+    glusterd_brickinfo_t *brickinfo = NULL;
+    char key[64] = {
+        0,
+    };
+    int keylen;
+    char *gfid_str = NULL;
+    uuid_t gfid;
+
+    this = THIS;
+    GF_ASSERT(this);
+    priv = this->private;
+    GF_ASSERT(priv);
+
+    ret = dict_get_strn(dict, "path", SLEN("path"), &path);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+               "Failed to get path");
+        goto out;
+    }
+
+    cds_list_for_each_entry(brickinfo, &volinfo->bricks, brick_list)
+    {
+        ret = glusterd_resolve_brick(brickinfo);
         if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "Failed to get path");
-                goto out;
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_RESOLVE_BRICK_FAIL,
+                   FMTSTR_RESOLVE_BRICK, brickinfo->hostname, brickinfo->path);
+            goto out;
         }
 
-        cds_list_for_each_entry (brickinfo, &volinfo->bricks, brick_list) {
-                ret = glusterd_resolve_brick (brickinfo);
-                if (ret) {
-                        gf_log (this->name, GF_LOG_ERROR, FMTSTR_RESOLVE_BRICK,
-                                brickinfo->hostname, brickinfo->path);
-                        goto out;
-                }
-
-                if (gf_uuid_compare (brickinfo->uuid, MY_UUID))
-                        continue;
-
-                if (brickinfo->vg[0])
-                        continue;
-
-                snprintf (backend_path, sizeof (backend_path), "%s%s",
-                          brickinfo->path, path);
+        if (gf_uuid_compare(brickinfo->uuid, MY_UUID))
+            continue;
 
-                ret = gf_lstat_dir (backend_path, NULL);
-                if (ret) {
-                        gf_log (this->name, GF_LOG_INFO, "Failed to find "
-                                "directory %s. Reason : %s", backend_path,
-                                strerror (errno));
-                        ret = 0;
-                        continue;
-                }
-                ret = sys_lgetxattr (backend_path, GFID_XATTR_KEY, gfid, 16);
-                if (ret < 0) {
-                        gf_log (this->name, GF_LOG_INFO, "Failed to get "
-                                "extended attribute %s for directory %s. "
-                                "Reason : %s", GFID_XATTR_KEY, backend_path,
-                                     strerror (errno));
-                        ret = 0;
-                        continue;
-                }
-                snprintf (key, sizeof (key), "gfid%d", count);
+        if (brickinfo->vg[0])
+            continue;
 
-                gfid_str = gf_strdup (uuid_utoa (gfid));
-                if (!gfid_str) {
-                        ret = -1;
-                        goto out;
-                }
+        snprintf(backend_path, sizeof(backend_path), "%s%s", brickinfo->path,
+                 path);
 
-                ret = dict_set_dynstr (rsp_dict, key, gfid_str);
-                if (ret) {
-                        gf_log (this->name, GF_LOG_ERROR, "Failed to place "
-                                "gfid of %s in dict", backend_path);
-                        GF_FREE (gfid_str);
-                        goto out;
-                }
-                count++;
+        ret = gf_lstat_dir(backend_path, NULL);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_INFO, errno, GD_MSG_DIR_OP_FAILED,
+                   "Failed to find "
+                   "directory %s.",
+                   backend_path);
+            ret = 0;
+            continue;
+        }
+        ret = sys_lgetxattr(backend_path, GFID_XATTR_KEY, gfid, 16);
+        if (ret < 0) {
+            gf_smsg(this->name, GF_LOG_INFO, errno, GD_MSG_GET_XATTR_FAIL,
+                    "Attribute=%s, Directory=%s", GFID_XATTR_KEY, backend_path,
+                    NULL);
+            ret = 0;
+            continue;
         }
+        keylen = snprintf(key, sizeof(key), "gfid%d", count);
 
-        ret = dict_set_int32 (rsp_dict, "count", count);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "Failed to set count");
-                goto out;
+        gfid_str = gf_strdup(uuid_utoa(gfid));
+        if (!gfid_str) {
+            ret = -1;
+            goto out;
         }
 
-        ret = 0;
+        ret = dict_set_dynstrn(rsp_dict, key, keylen, gfid_str);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+                   "Failed to place "
+                   "gfid of %s in dict",
+                   backend_path);
+            GF_FREE(gfid_str);
+            goto out;
+        }
+        count++;
+    }
+
+    ret = dict_set_int32n(rsp_dict, "count", SLEN("count"), count);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+               "Failed to set count");
+        goto out;
+    }
+
+    ret = 0;
 out:
-        return ret;
+    return ret;
 }
 
 static int
-_glusterd_validate_quota_opts (dict_t *dict, int type, char **errstr)
+_glusterd_validate_quota_opts(dict_t *dict, int type, char **errstr)
 {
-        int                     ret = -1;
-        xlator_t                *this = THIS;
-        void                    *quota_xl = NULL;
-        volume_opt_list_t       opt_list = {{0},};
-        volume_option_t         *opt = NULL;
-        char                    *key = NULL;
-        char                    *value = NULL;
-
-        GF_ASSERT (dict);
-        GF_ASSERT (this);
-
-        ret = xlator_volopt_dynload ("features/quota", &quota_xl, &opt_list);
-        if (ret)
-                goto out;
-
-        switch (type) {
+    int ret = -1;
+    xlator_t *this = THIS;
+    void *quota_xl = NULL;
+    volume_opt_list_t opt_list = {
+        {0},
+    };
+    volume_option_t *opt = NULL;
+    char *key = NULL;
+    char *value = NULL;
+
+    GF_ASSERT(dict);
+    GF_ASSERT(this);
+
+    ret = xlator_volopt_dynload("features/quota", &quota_xl, &opt_list);
+    if (ret)
+        goto out;
+
+    switch (type) {
         case GF_QUOTA_OPTION_TYPE_SOFT_TIMEOUT:
         case GF_QUOTA_OPTION_TYPE_HARD_TIMEOUT:
         case GF_QUOTA_OPTION_TYPE_ALERT_TIME:
         case GF_QUOTA_OPTION_TYPE_DEFAULT_SOFT_LIMIT:
-                key = (char *)gd_quota_op_list[type];
-                break;
+            key = (char *)gd_quota_op_list[type];
+            break;
         default:
-                ret = -1;
-                goto out;
-        }
+            ret = -1;
+            goto out;
+    }
+
+    opt = xlator_volume_option_get_list(&opt_list, key);
+    if (!opt) {
+        ret = -1;
+        gf_msg(this->name, GF_LOG_ERROR, EINVAL, GD_MSG_UNKNOWN_KEY,
+               "Unknown option: %s", key);
+        goto out;
+    }
+    ret = dict_get_strn(dict, "value", SLEN("value"), &value);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+               "Value not found for key %s", key);
+        goto out;
+    }
+
+    ret = xlator_option_validate(this, key, value, opt, errstr);
 
-        opt = xlator_volume_option_get_list (&opt_list, key);
-        if (!opt) {
-                ret = -1;
-                gf_log (this->name, GF_LOG_ERROR, "Unknown option: %s", key);
-                goto out;
-        }
-        ret = dict_get_str (dict, "value", &value);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "Value not found for key %s",
-                        key);
-                goto out;
-        }
+out:
+    if (quota_xl) {
+        dlclose(quota_xl);
+        quota_xl = NULL;
+    }
+    return ret;
+}
 
-        ret = xlator_option_validate (this, key, value, opt, errstr);
+static int
+glusterd_create_quota_auxiliary_mount(xlator_t *this, char *volname, int type)
+{
+    int ret = -1;
+    char mountdir[PATH_MAX] = {
+        0,
+    };
+    char pidfile_path[PATH_MAX] = {
+        0,
+    };
+    char logfile[PATH_MAX] = {
+        0,
+    };
+    char qpid[16] = {
+        0,
+    };
+    char *volfileserver = NULL;
+    glusterd_conf_t *priv = NULL;
+    struct stat buf = {
+        0,
+    };
+    FILE *file = NULL;
+
+    GF_VALIDATE_OR_GOTO("glusterd", this, out);
+    priv = this->private;
+    GF_VALIDATE_OR_GOTO(this->name, priv, out);
+
+    if (type == GF_QUOTA_OPTION_TYPE_LIST ||
+        type == GF_QUOTA_OPTION_TYPE_LIST_OBJECTS) {
+        GLUSTERFS_GET_QUOTA_LIST_MOUNT_PIDFILE(pidfile_path, volname);
+        GLUSTERD_GET_QUOTA_LIST_MOUNT_PATH(mountdir, volname, "/");
+    } else {
+        GLUSTERFS_GET_QUOTA_LIMIT_MOUNT_PIDFILE(pidfile_path, volname);
+        GLUSTERD_GET_QUOTA_LIMIT_MOUNT_PATH(mountdir, volname, "/");
+    }
+
+    file = fopen(pidfile_path, "r");
+    if (file) {
+        /* Previous command did not clean up pid file.
+         * remove aux mount if it exists*/
+        gf_umount_lazy(this->name, mountdir, 1);
+        fclose(file);
+    }
+
+    ret = sys_mkdir(mountdir, 0755);
+    if (ret && errno != EEXIST) {
+        gf_msg(this->name, GF_LOG_ERROR, errno, GD_MSG_MOUNT_REQ_FAIL,
+               "Failed to create auxiliary "
+               "mount directory %s",
+               mountdir);
+        goto out;
+    }
+    snprintf(logfile, PATH_MAX - 1, "%s/quota-mount-%s.log", priv->logdir,
+             volname);
+    snprintf(qpid, 15, "%d", GF_CLIENT_PID_QUOTA_MOUNT);
+
+    if (dict_get_strn(this->options, "transport.socket.bind-address",
+                      SLEN("transport.socket.bind-address"),
+                      &volfileserver) != 0)
+        volfileserver = "localhost";
+
+    synclock_unlock(&priv->big_lock);
+    ret = runcmd(SBIN_DIR "/glusterfs", "--volfile-server", volfileserver,
+                 "--volfile-id", volname, "-l", logfile, "-p", pidfile_path,
+                 "--client-pid", qpid, mountdir, NULL);
+    if (ret == 0) {
+        /* Block here till mount process is ready to accept FOPs.
+         * Else, if glusterd acquires biglock below before
+         * mount process is ready, then glusterd and mount process
+         * can get into a deadlock situation.
+         */
+        ret = sys_stat(mountdir, &buf);
+        if (ret < 0)
+            ret = -errno;
+    } else {
+        ret = -errno;
+    }
+
+    synclock_lock(&priv->big_lock);
+
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, -ret, GD_MSG_MOUNT_REQ_FAIL,
+               "Failed to mount glusterfs "
+               "client. Please check the log file %s for more details",
+               logfile);
+        ret = -1;
+        goto out;
+    }
+
+    ret = 0;
 
 out:
-        if (quota_xl) {
-                dlclose (quota_xl);
-                quota_xl = NULL;
-        }
-        return ret;
+    return ret;
 }
 
 int
-glusterd_op_stage_quota (dict_t *dict, char **op_errstr, dict_t *rsp_dict)
+glusterd_op_stage_quota(dict_t *dict, char **op_errstr, dict_t *rsp_dict)
 {
-        int                 ret            = 0;
-        char               *volname        = NULL;
-        gf_boolean_t        exists         = _gf_false;
-        int                 type           = 0;
-        xlator_t           *this           = NULL;
-        glusterd_conf_t    *priv           = NULL;
-        glusterd_volinfo_t *volinfo        = NULL;
-        char               *hard_limit_str = NULL;
-        int64_t             hard_limit     = 0;
-        gf_boolean_t        get_gfid       = _gf_false;
-
-        this = THIS;
-        GF_ASSERT (this);
-        priv = this->private;
-        GF_ASSERT (priv);
-
-        GF_ASSERT (dict);
-        GF_ASSERT (op_errstr);
-
-        ret = dict_get_str (dict, "volname", &volname);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "Unable to get volume name");
-                goto out;
-        }
-
-        exists = glusterd_check_volume_exists (volname);
-        if (!exists) {
-                gf_asprintf (op_errstr, FMTSTR_CHECK_VOL_EXISTS, volname);
-                ret = -1;
-                goto out;
-        }
-        ret = glusterd_volinfo_find (volname, &volinfo);
-        if (ret) {
-                gf_asprintf (op_errstr, FMTSTR_CHECK_VOL_EXISTS, volname);
-                goto out;
-        }
-
-        if (!glusterd_is_volume_started (volinfo)) {
-                *op_errstr = gf_strdup ("Volume is stopped, start volume "
-                                        "before executing quota command.");
-                ret = -1;
-                goto out;
-        }
-
-        ret = dict_get_int32 (dict, "type", &type);
-        if (ret) {
-                *op_errstr = gf_strdup ("Volume quota failed, internal error, "
-                                        "unable to get type of operation");
-                goto out;
-        }
-
-        if ((!glusterd_is_volume_quota_enabled (volinfo)) &&
-            (type != GF_QUOTA_OPTION_TYPE_ENABLE)) {
-                *op_errstr = gf_strdup ("Quota is disabled, please enable "
-                                        "quota");
-                ret = -1;
-                goto out;
-        }
-
-        if ((priv->op_version == GD_OP_VERSION_MIN) &&
-            (type > GF_QUOTA_OPTION_TYPE_VERSION)) {
-                gf_asprintf (op_errstr, "Volume quota failed. The cluster is "
-                                        "operating at version %d. Quota command"
-                                        " %s is unavailable in this version.",
-                                         priv->op_version,
-                                         gd_quota_op_list[type]);
-                ret = -1;
-                goto out;
-        }
-
-        if ((GF_QUOTA_OPTION_TYPE_ENABLE != type) &&
-            (glusterd_check_if_quota_trans_enabled (volinfo) != 0)) {
-                ret = -1;
-                gf_asprintf (op_errstr, "Quota is not enabled on volume %s",
-                             volname);
-                goto out;
-        }
-
-        switch (type) {
-        case GF_QUOTA_OPTION_TYPE_ENABLE:
+    int ret = 0;
+    char *volname = NULL;
+    int type = 0;
+    xlator_t *this = NULL;
+    glusterd_conf_t *priv = NULL;
+    glusterd_volinfo_t *volinfo = NULL;
+    char *hard_limit_str = NULL;
+    int64_t hard_limit = 0;
+    gf_boolean_t get_gfid = _gf_false;
+
+    this = THIS;
+    GF_ASSERT(this);
+    priv = this->private;
+    GF_ASSERT(priv);
+
+    GF_ASSERT(dict);
+    GF_ASSERT(op_errstr);
+
+    ret = dict_get_strn(dict, "volname", SLEN("volname"), &volname);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+               "Unable to get volume name");
+        goto out;
+    }
+
+    ret = glusterd_volinfo_find(volname, &volinfo);
+    if (ret) {
+        gf_asprintf(op_errstr, FMTSTR_CHECK_VOL_EXISTS, volname);
+        goto out;
+    }
+
+    if (!glusterd_is_volume_started(volinfo)) {
+        *op_errstr = gf_strdup(
+            "Volume is stopped, start volume "
+            "before executing quota command.");
+        ret = -1;
+        goto out;
+    }
+
+    ret = dict_get_int32n(dict, "type", SLEN("type"), &type);
+    if (ret) {
+        *op_errstr = gf_strdup(
+            "Volume quota failed, internal error, "
+            "unable to get type of operation");
+        goto out;
+    }
+
+    if ((!glusterd_is_volume_quota_enabled(volinfo)) &&
+        (type != GF_QUOTA_OPTION_TYPE_ENABLE)) {
+        *op_errstr = gf_strdup(
+            "Quota is disabled, please enable "
+            "quota");
+        ret = -1;
+        goto out;
+    }
+
+    if (type > GF_QUOTA_OPTION_TYPE_VERSION_OBJECTS) {
+        if (!glusterd_is_volume_inode_quota_enabled(volinfo) &&
+            type != GF_QUOTA_OPTION_TYPE_ENABLE_OBJECTS) {
+            *op_errstr = gf_strdup(
+                "Inode Quota is disabled, "
+                "please enable inode quota");
+            ret = -1;
+            goto out;
+        }
+    }
+
+    if (!glusterd_is_quota_supported(type, op_errstr)) {
+        ret = -1;
+        goto out;
+    }
+
+    if ((GF_QUOTA_OPTION_TYPE_ENABLE != type) &&
+        (glusterd_check_if_quota_trans_enabled(volinfo) != 0)) {
+        ret = -1;
+        gf_asprintf(op_errstr, "Quota is not enabled on volume %s", volname);
+        goto out;
+    }
+
+    switch (type) {
         case GF_QUOTA_OPTION_TYPE_LIST:
         case GF_QUOTA_OPTION_TYPE_LIST_OBJECTS:
-                /* Fuse mount req. only for enable & list-usage options*/
-                if (is_origin_glusterd (dict) &&
-                    !glusterd_is_fuse_available ()) {
-                        *op_errstr = gf_strdup ("Fuse unavailable");
-                        ret = -1;
-                        goto out;
-                }
-                break;
-
         case GF_QUOTA_OPTION_TYPE_LIMIT_USAGE:
-                ret = dict_get_str (dict, "hard-limit", &hard_limit_str);
-                if (ret) {
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "Faild to get hard-limit from dict");
-                        goto out;
-                }
-                ret = gf_string2bytesize_int64 (hard_limit_str, &hard_limit);
+        case GF_QUOTA_OPTION_TYPE_LIMIT_OBJECTS:
+        case GF_QUOTA_OPTION_TYPE_REMOVE:
+        case GF_QUOTA_OPTION_TYPE_REMOVE_OBJECTS:
+            /* Quota auxiliary mount is needed by CLI
+             * for list command and need by glusterd for
+             * setting/removing limit
+             */
+            if (is_origin_glusterd(dict)) {
+                ret = glusterd_create_quota_auxiliary_mount(this, volname,
+                                                            type);
                 if (ret) {
-                        if (errno == ERANGE || hard_limit < 0)
-                                gf_asprintf (op_errstr, "Hard-limit "
-                                        "value out of range (0 - %"PRId64
-                                        "): %s", hard_limit_str);
-                        else
-                                gf_log (this->name, GF_LOG_ERROR,
-                                        "Failed to convert hard-limit "
-                                        "from string to bytes");
-                        goto out;
+                    *op_errstr = gf_strdup(
+                        "Failed to start aux "
+                        "mount");
+                    goto out;
                 }
-                get_gfid = _gf_true;
-                break;
+            }
+            break;
+    }
+
+    switch (type) {
+        case GF_QUOTA_OPTION_TYPE_LIMIT_USAGE:
+            ret = dict_get_strn(dict, "hard-limit", SLEN("hard-limit"),
+                                &hard_limit_str);
+            if (ret) {
+                gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+                       "Failed to get hard-limit from dict");
+                goto out;
+            }
+            ret = gf_string2bytesize_int64(hard_limit_str, &hard_limit);
+            if (ret) {
+                if (errno == ERANGE || hard_limit < 0)
+                    gf_asprintf(op_errstr,
+                                "Hard-limit "
+                                "value out of range (0 - %" PRId64 "): %s",
+                                hard_limit, hard_limit_str);
+                else
+                    gf_msg(this->name, GF_LOG_ERROR, errno,
+                           GD_MSG_CONVERSION_FAILED,
+                           "Failed to convert hard-limit "
+                           "string to value");
+                goto out;
+            }
+            get_gfid = _gf_true;
+            break;
         case GF_QUOTA_OPTION_TYPE_LIMIT_OBJECTS:
-                get_gfid = _gf_true;
-                break;
+            get_gfid = _gf_true;
+            break;
 
         case GF_QUOTA_OPTION_TYPE_REMOVE:
         case GF_QUOTA_OPTION_TYPE_REMOVE_OBJECTS:
-                get_gfid = _gf_true;
-                break;
+            get_gfid = _gf_true;
+            break;
 
         case GF_QUOTA_OPTION_TYPE_SOFT_TIMEOUT:
         case GF_QUOTA_OPTION_TYPE_HARD_TIMEOUT:
         case GF_QUOTA_OPTION_TYPE_ALERT_TIME:
         case GF_QUOTA_OPTION_TYPE_DEFAULT_SOFT_LIMIT:
-                ret = _glusterd_validate_quota_opts (dict, type, op_errstr);
-                if (ret)
-                        goto out;
-                break;
+            ret = _glusterd_validate_quota_opts(dict, type, op_errstr);
+            if (ret)
+                goto out;
+            break;
 
         default:
-                break;
-        }
+            break;
+    }
 
-        if (get_gfid == _gf_true) {
-                ret = glusterd_get_gfid_from_brick (dict, volinfo, rsp_dict,
-                                                    op_errstr);
-                if (ret)
-                        goto out;
-        }
+    if (get_gfid == _gf_true) {
+        ret = glusterd_get_gfid_from_brick(dict, volinfo, rsp_dict, op_errstr);
+        if (ret)
+            goto out;
+    }
 
-        ret = 0;
+    ret = 0;
 
- out:
-        if (ret && op_errstr && *op_errstr)
-                gf_log (this->name, GF_LOG_ERROR, "%s", *op_errstr);
-        gf_log (this->name, GF_LOG_DEBUG, "Returning %d", ret);
+out:
+    if (ret && op_errstr && *op_errstr)
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_OP_STAGE_QUOTA_FAIL, "%s",
+               *op_errstr);
+    gf_msg_debug(this->name, 0, "Returning %d", ret);
 
-         return ret;
+    return ret;
 }
diff --git a/xlators/features/ganesha/src/ganesha.h b/xlators/mgmt/glusterd/src/glusterd-quota.h
index f30723da46d..ab2092a9c6a 100644
--- a/xlators/features/ganesha/src/ganesha.h
+++ b/xlators/mgmt/glusterd/src/glusterd-quota.h
@@ -1,5 +1,5 @@
 /*
-   Copyright (c) 2015 Red Hat, Inc. <http://www.redhat.com>
+   Copyright (c) 2016 Red Hat, Inc. <http://www.redhat.com>
    This file is part of GlusterFS.
 
    This file is licensed to you under your choice of the GNU Lesser
@@ -7,17 +7,11 @@
    later), or the GNU General Public License, version 2 (GPLv2), in all
    cases as published by the Free Software Foundation.
 */
+#ifndef _GLUSTERD_QUOTA_
+#define _GLUSTERD_QUOTA_
 
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
-
-#include "xlator.h"
-#include "ganesha-mem-types.h"
-
-typedef struct {
-        char *host_name;
-} ganesha_priv_t;
-
+int
+glusterd_store_quota_config(glusterd_volinfo_t *volinfo, char *path,
+                            char *gfid_str, int opcode, char **op_errstr);
 
+#endif
diff --git a/xlators/mgmt/glusterd/src/glusterd-quotad-svc.c b/xlators/mgmt/glusterd/src/glusterd-quotad-svc.c
index bd77a72c5ab..f26d832a06d 100644
--- a/xlators/mgmt/glusterd/src/glusterd-quotad-svc.c
+++ b/xlators/mgmt/glusterd/src/glusterd-quotad-svc.c
@@ -8,151 +8,210 @@
    cases as published by the Free Software Foundation.
 */
 
-#include "globals.h"
-#include "run.h"
+#include <glusterfs/globals.h>
+#include <glusterfs/run.h>
 #include "glusterd.h"
 #include "glusterd-utils.h"
 #include "glusterd-volgen.h"
 #include "glusterd-quotad-svc.h"
+#include "glusterd-messages.h"
+#include "glusterd-svc-helper.h"
 
 char *quotad_svc_name = "quotad";
 
-int glusterd_quotadsvc_init (glusterd_svc_t *svc)
+void
+glusterd_quotadsvc_build(glusterd_svc_t *svc)
 {
-        int              ret                = -1;
-        char             volfile[PATH_MAX]  = {0,};
-        glusterd_conf_t *conf               = THIS->private;
-
-        ret = glusterd_svc_init (svc, quotad_svc_name,
-                                 glusterd_quotadsvc_manager,
-                                 glusterd_quotadsvc_start,
-                                 glusterd_svc_stop);
-        if (ret)
-                goto out;
+    svc->manager = glusterd_quotadsvc_manager;
+    svc->start = glusterd_quotadsvc_start;
+    svc->stop = glusterd_svc_stop;
+}
+
+int
+glusterd_quotadsvc_init(glusterd_svc_t *svc)
+{
+    int ret = -1;
+
+    ret = glusterd_svc_init(svc, quotad_svc_name);
+    if (ret)
+        goto out;
 
-        /* glusterd_svc_build_volfile_path () doesn't put correct quotad volfile
-         * path in proc object at service initialization. Re-initialize
-         * the correct path
-         */
-        glusterd_quotadsvc_build_volfile_path (quotad_svc_name, conf->workdir,
-                                               volfile, sizeof (volfile));
-        snprintf (svc->proc.volfile, sizeof (svc->proc.volfile), "%s", volfile);
 out:
-        return ret;
+    return ret;
 }
 
 static int
-glusterd_quotadsvc_create_volfile ()
+glusterd_quotadsvc_create_volfile()
 {
-        char             filepath[PATH_MAX] = {0,};
-        glusterd_conf_t *conf               = THIS->private;
-
-        glusterd_quotadsvc_build_volfile_path (quotad_svc_name, conf->workdir,
-                                               filepath, sizeof (filepath));
-        return glusterd_create_global_volfile (build_quotad_graph,
-                                               filepath, NULL);
+    char filepath[PATH_MAX] = {
+        0,
+    };
+    glusterd_conf_t *conf = THIS->private;
+
+    glusterd_svc_build_volfile_path(quotad_svc_name, conf->workdir, filepath,
+                                    sizeof(filepath));
+    return glusterd_create_global_volfile(build_quotad_graph, filepath, NULL);
 }
 
 int
-glusterd_quotadsvc_manager (glusterd_svc_t *svc, void *data, int flags)
+glusterd_quotadsvc_manager(glusterd_svc_t *svc, void *data, int flags)
 {
-        int                 ret     = 0;
-        glusterd_volinfo_t *volinfo = NULL;
-
-        volinfo = data;
-
-        /* If all the volumes are stopped or all shd compatible volumes
-         * are stopped then stop the service if:
-         * - volinfo is NULL or
-         * - volinfo is present and volume is shd compatible
-         * Otherwise create volfile and restart service if:
-         * - volinfo is NULL or
-         * - volinfo is present and volume is shd compatible
-         */
-        if (glusterd_are_all_volumes_stopped () ||
-            glusterd_all_volumes_with_quota_stopped ()) {
-                if (!(volinfo && !glusterd_is_volume_quota_enabled (volinfo))) {
-                        ret = svc->stop (svc, SIGTERM);
-                }
+    int ret = 0;
+    glusterd_volinfo_t *volinfo = NULL;
+
+    if (!svc->inited) {
+        ret = glusterd_quotadsvc_init(svc);
+        if (ret) {
+            gf_msg(THIS->name, GF_LOG_ERROR, 0, GD_MSG_FAILED_INIT_QUOTASVC,
+                   "Failed to init "
+                   "quotad service");
+            goto out;
         } else {
-                if (!(volinfo && !glusterd_is_volume_quota_enabled (volinfo))) {
-                        ret = glusterd_quotadsvc_create_volfile ();
-                        if (ret)
-                                goto out;
-
-                        ret = svc->stop (svc, SIGTERM);
-                        if (ret)
-                                goto out;
-
-                        ret = svc->start (svc, flags);
-                        if (ret)
-                                goto out;
-
-                        ret = glusterd_conn_connect (&(svc->conn));
-                        if (ret)
-                                goto out;
-                }
+            svc->inited = _gf_true;
+            gf_msg_debug(THIS->name, 0,
+                         "quotad service "
+                         "initialized");
         }
-out:
-        gf_log (THIS->name, GF_LOG_DEBUG, "Returning %d", ret);
+    }
+
+    volinfo = data;
+
+    /* If all the volumes are stopped or all shd compatible volumes
+     * are stopped then stop the service if:
+     * - volinfo is NULL or
+     * - volinfo is present and volume is shd compatible
+     * Otherwise create volfile and restart service if:
+     * - volinfo is NULL or
+     * - volinfo is present and volume is shd compatible
+     */
+    if (glusterd_are_all_volumes_stopped() ||
+        glusterd_all_volumes_with_quota_stopped()) {
+        if (!(volinfo && !glusterd_is_volume_quota_enabled(volinfo))) {
+            ret = svc->stop(svc, SIGTERM);
+        }
+    } else {
+        if (!(volinfo && !glusterd_is_volume_quota_enabled(volinfo))) {
+            ret = glusterd_quotadsvc_create_volfile();
+            if (ret)
+                goto out;
 
-        return ret;
-}
+            ret = svc->stop(svc, SIGTERM);
+            if (ret)
+                goto out;
 
-int
-glusterd_quotadsvc_start (glusterd_svc_t *svc, int flags)
-{
-        int              i         = 0;
-        int              ret       = -1;
-        dict_t          *cmdline   = NULL;
-        char             key[16]   = {0};
-        char            *options[] = {
-                                      "*replicate*.entry-self-heal=off",
-                                      "--xlator-option",
-                                      "*replicate*.metadata-self-heal=off",
-                                      "--xlator-option",
-                                      "*replicate*.data-self-heal=off",
-                                      "--xlator-option",
-                                      NULL
-                                      };
-
-        cmdline = dict_new ();
-        if (!cmdline)
+            ret = svc->start(svc, flags);
+            if (ret)
                 goto out;
 
-        for (i = 0; options[i]; i++) {
-                memset (key, 0, sizeof (key));
-                snprintf (key, sizeof (key), "arg%d", i);
-                ret = dict_set_str (cmdline, key, options[i]);
-                if (ret)
-                        goto out;
+            ret = glusterd_conn_connect(&(svc->conn));
+            if (ret)
+                goto out;
         }
-
-        ret = glusterd_svc_start (svc, flags, cmdline);
-
+    }
 out:
-        if (cmdline)
-                dict_unref (cmdline);
+    if (ret)
+        gf_event(EVENT_SVC_MANAGER_FAILED, "svc_name=%s", svc->name);
 
-        gf_log (THIS->name, GF_LOG_DEBUG, "Returning %d", ret);
+    gf_msg_debug(THIS->name, 0, "Returning %d", ret);
 
-        return ret;
+    return ret;
 }
 
 int
-glusterd_quotadsvc_reconfigure ()
+glusterd_quotadsvc_start(glusterd_svc_t *svc, int flags)
 {
-        return glusterd_svc_reconfigure (glusterd_quotadsvc_create_volfile);
+    int i = 0;
+    int ret = -1;
+    dict_t *cmdline = NULL;
+    char key[16] = {0};
+    char *options[] = {svc->name, "--process-name", NULL};
+
+    cmdline = dict_new();
+    if (!cmdline) {
+        gf_smsg(THIS->name, GF_LOG_ERROR, errno, GD_MSG_DICT_CREATE_FAIL, NULL);
+        goto out;
+    }
+
+    for (i = 0; options[i]; i++) {
+        ret = snprintf(key, sizeof(key), "arg%d", i);
+        ret = dict_set_strn(cmdline, key, ret, options[i]);
+        if (ret)
+            goto out;
+    }
+
+    ret = glusterd_svc_start(svc, flags, cmdline);
+
+out:
+    if (cmdline)
+        dict_unref(cmdline);
+
+    gf_msg_debug(THIS->name, 0, "Returning %d", ret);
+
+    return ret;
 }
 
-void
-glusterd_quotadsvc_build_volfile_path (char *server, char *workdir,
-                                       char *volfile, size_t len)
+int
+glusterd_quotadsvc_reconfigure()
 {
-        char  dir[PATH_MAX] = {0,};
-
-        GF_ASSERT (len == PATH_MAX);
+    int ret = -1;
+    xlator_t *this = NULL;
+    glusterd_conf_t *priv = NULL;
+    gf_boolean_t identical = _gf_false;
+
+    this = THIS;
+    GF_VALIDATE_OR_GOTO("glusterd", this, out);
+
+    priv = this->private;
+    GF_VALIDATE_OR_GOTO(this->name, priv, out);
+
+    if (glusterd_all_volumes_with_quota_stopped())
+        goto manager;
+
+    /*
+     * Check both OLD and NEW volfiles, if they are SAME by size
+     * and cksum i.e. "character-by-character". If YES, then
+     * NOTHING has been changed, just return.
+     */
+    ret = glusterd_svc_check_volfile_identical(priv->quotad_svc.name,
+                                               build_quotad_graph, &identical);
+    if (ret)
+        goto out;
+
+    if (identical) {
+        ret = 0;
+        goto out;
+    }
+
+    /*
+     * They are not identical. Find out if the topology is changed
+     * OR just the volume options. If just the options which got
+     * changed, then inform the xlator to reconfigure the options.
+     */
+    identical = _gf_false; /* RESET the FLAG */
+    ret = glusterd_svc_check_topology_identical(priv->quotad_svc.name,
+                                                build_quotad_graph, &identical);
+    if (ret)
+        goto out;
+
+    /* Topology is not changed, but just the options. But write the
+     * options to quotad volfile, so that quotad will be reconfigured.
+     */
+    if (identical) {
+        ret = glusterd_quotadsvc_create_volfile();
+        if (ret == 0) { /* Only if above PASSES */
+            ret = glusterd_fetchspec_notify(THIS);
+        }
+        goto out;
+    }
+manager:
+    /*
+     * quotad volfile's topology has been changed. quotad server needs
+     * to be RESTARTED to ACT on the changed volfile.
+     */
+    ret = priv->quotad_svc.manager(&(priv->quotad_svc), NULL,
+                                   PROC_START_NO_WAIT);
 
-        glusterd_svc_build_svcdir (server, workdir, dir, sizeof (dir));
-        snprintf (volfile, len, "%s/%s.vol", dir, server);
+out:
+    gf_msg_debug(this ? this->name : "Quotad", 0, "Returning %d", ret);
+    return ret;
 }
diff --git a/xlators/mgmt/glusterd/src/glusterd-quotad-svc.h b/xlators/mgmt/glusterd/src/glusterd-quotad-svc.h
index 945d47e68f6..e8d9bbee964 100644
--- a/xlators/mgmt/glusterd/src/glusterd-quotad-svc.h
+++ b/xlators/mgmt/glusterd/src/glusterd-quotad-svc.h
@@ -11,26 +11,21 @@
 #ifndef _GLUSTERD_QUOTAD_SVC_H_
 #define _GLUSTERD_QUOTAD_SVC_H_
 
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
-
 #include "glusterd-svc-mgmt.h"
 
+void
+glusterd_quotadsvc_build(glusterd_svc_t *svc);
+
 int
-glusterd_quotadsvc_init (glusterd_svc_t *svc);
+glusterd_quotadsvc_init(glusterd_svc_t *svc);
 
 int
-glusterd_quotadsvc_start (glusterd_svc_t *svc, int flags);
+glusterd_quotadsvc_start(glusterd_svc_t *svc, int flags);
 
 int
-glusterd_quotadsvc_manager (glusterd_svc_t *svc, void *data, int flags);
+glusterd_quotadsvc_manager(glusterd_svc_t *svc, void *data, int flags);
 
 int
-glusterd_quotadsvc_reconfigure ();
+glusterd_quotadsvc_reconfigure();
 
-void
-glusterd_quotadsvc_build_volfile_path (char *server, char *workdir,
-                                       char *volfile, size_t len);
 #endif
diff --git a/xlators/mgmt/glusterd/src/glusterd-rcu.h b/xlators/mgmt/glusterd/src/glusterd-rcu.h
index d4e78753e69..c85f9bea8f8 100644
--- a/xlators/mgmt/glusterd/src/glusterd-rcu.h
+++ b/xlators/mgmt/glusterd/src/glusterd-rcu.h
@@ -17,11 +17,11 @@
 #include <urcu/uatomic.h>
 #include <urcu-call-rcu.h>
 
-#ifdef URCU_0_7
+#ifdef URCU_OLD
 #include "rculist-extra.h"
 #endif
 
-#include "xlator.h"
+#include <glusterfs/xlator.h>
 
 /* gd_rcu_head is a composite struct, composed of struct rcu_head and a this
  * pointer, which is used to pass the THIS pointer to call_rcu callbacks.
@@ -29,8 +29,8 @@
  * Use this in place of struct rcu_head when embedding into another struct
  */
 typedef struct glusterd_rcu_head_ {
-        struct rcu_head head;
-        xlator_t *this;
+    struct rcu_head head;
+    xlator_t *this;
 } gd_rcu_head;
 
 #endif /* _GLUSTERD_RCU_H */
diff --git a/xlators/mgmt/glusterd/src/glusterd-rebalance.c b/xlators/mgmt/glusterd/src/glusterd-rebalance.c
index f5bb319cb7d..458bf168ede 100644
--- a/xlators/mgmt/glusterd/src/glusterd-rebalance.c
+++ b/xlators/mgmt/glusterd/src/glusterd-rebalance.c
@@ -7,904 +7,1416 @@
    later), or the GNU General Public License, version 2 (GPLv2), in all
    cases as published by the Free Software Foundation.
 */
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
-
 #include <inttypes.h>
 #include <sys/types.h>
 #include <unistd.h>
 #include <sys/resource.h>
 #include <sys/statvfs.h>
 
-#include "globals.h"
-#include "compat.h"
+#include <glusterfs/compat.h>
 #include "protocol-common.h"
-#include "xlator.h"
-#include "logging.h"
-#include "timer.h"
+#include <glusterfs/xlator.h>
+#include <glusterfs/logging.h>
+#include <glusterfs/timer.h>
 #include "glusterd-mem-types.h"
 #include "glusterd.h"
 #include "glusterd-sm.h"
 #include "glusterd-op-sm.h"
 #include "glusterd-utils.h"
+#include "glusterd-mgmt.h"
 #include "glusterd-messages.h"
 #include "glusterd-store.h"
-#include "run.h"
+#include <glusterfs/run.h>
 #include "glusterd-volgen.h"
 #include "glusterd-messages.h"
 
-#include "syscall.h"
+#include <glusterfs/syscall.h>
 #include "cli1-xdr.h"
 #include "xdr-generic.h"
 
+#define GLUSTERD_GET_DEFRAG_SOCK_FILE(path, volinfo)                           \
+    do {                                                                       \
+        int32_t _defrag_sockfile_len;                                          \
+        char tmppath[PATH_MAX] = {                                             \
+            0,                                                                 \
+        };                                                                     \
+        _defrag_sockfile_len = snprintf(                                       \
+            tmppath, PATH_MAX,                                                 \
+            DEFAULT_VAR_RUN_DIRECTORY "/gluster-%s-%s-%s.sock", "rebalance",   \
+            volinfo->volname, uuid_utoa(MY_UUID));                             \
+        if ((_defrag_sockfile_len < 0) ||                                      \
+            (_defrag_sockfile_len >= PATH_MAX)) {                              \
+            path[0] = 0;                                                       \
+        } else {                                                               \
+            glusterd_set_socket_filepath(tmppath, path, sizeof(path));         \
+        }                                                                      \
+    } while (0)
+
 int32_t
-glusterd_brick_op_cbk (struct rpc_req *req, struct iovec *iov,
-                          int count, void *myframe);
+glusterd_brick_op_cbk(struct rpc_req *req, struct iovec *iov, int count,
+                      void *myframe);
 int
-glusterd_defrag_start_validate (glusterd_volinfo_t *volinfo, char *op_errstr,
-                                size_t len, glusterd_op_t op)
+glusterd_defrag_start_validate(glusterd_volinfo_t *volinfo, char *op_errstr,
+                               size_t len, glusterd_op_t op)
 {
-        int      ret = -1;
-        xlator_t *this = NULL;
-
-        this = THIS;
-        GF_ASSERT (this);
-
-        /* Check only if operation is not remove-brick */
-        if ((GD_OP_REMOVE_BRICK != op) &&
-            !gd_is_remove_brick_committed (volinfo)) {
-                gf_log (this->name, GF_LOG_DEBUG, "A remove-brick task on "
-                        "volume %s is not yet committed", volinfo->volname);
-                snprintf (op_errstr, len, "A remove-brick task on volume %s is"
-                          " not yet committed. Either commit or stop the "
-                          "remove-brick task.", volinfo->volname);
-                goto out;
-        }
-
-        if (glusterd_is_defrag_on (volinfo)) {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "rebalance on volume %s already started",
-                        volinfo->volname);
-                snprintf (op_errstr, len, "Rebalance on %s is already started",
-                          volinfo->volname);
-                goto out;
-        }
-
-        if (glusterd_is_rb_started (volinfo) ||
-            glusterd_is_rb_paused (volinfo)) {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "Rebalance failed as replace brick is in progress on volume %s",
-                        volinfo->volname);
-                snprintf (op_errstr, len, "Rebalance failed as replace brick is in progress on "
-                          "volume %s", volinfo->volname);
-                goto out;
-        }
-        ret = 0;
+    int ret = -1;
+    xlator_t *this = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+
+    /* Check only if operation is not remove-brick */
+    if ((GD_OP_REMOVE_BRICK != op) && !gd_is_remove_brick_committed(volinfo)) {
+        gf_msg_debug(this->name, 0,
+                     "A remove-brick task on "
+                     "volume %s is not yet committed",
+                     volinfo->volname);
+        snprintf(op_errstr, len,
+                 "A remove-brick task on volume %s is"
+                 " not yet committed. Either commit or stop the "
+                 "remove-brick task.",
+                 volinfo->volname);
+        goto out;
+    }
+
+    if (glusterd_is_defrag_on(volinfo)) {
+        gf_msg_debug(this->name, 0, "rebalance on volume %s already started",
+                     volinfo->volname);
+        snprintf(op_errstr, len, "Rebalance on %s is already started",
+                 volinfo->volname);
+        goto out;
+    }
+
+    ret = 0;
 out:
-        gf_log (this->name, GF_LOG_DEBUG, "Returning %d", ret);
-        return ret;
+    gf_msg_debug(this->name, 0, "Returning %d", ret);
+    return ret;
 }
 
-
 int32_t
-__glusterd_defrag_notify (struct rpc_clnt *rpc, void *mydata,
-                          rpc_clnt_event_t event, void *data)
+__glusterd_defrag_notify(struct rpc_clnt *rpc, void *mydata,
+                         rpc_clnt_event_t event, void *data)
 {
-        glusterd_volinfo_t      *volinfo = NULL;
-        glusterd_defrag_info_t  *defrag  = NULL;
-        int                     ret      = 0;
-        char                    pidfile[PATH_MAX];
-        glusterd_conf_t        *priv    = NULL;
-        xlator_t               *this    = NULL;
-
-        this = THIS;
-        if (!this)
-                return 0;
+    glusterd_volinfo_t *volinfo = NULL;
+    glusterd_defrag_info_t *defrag = NULL;
+    int ret = 0;
+    char pidfile[PATH_MAX];
+    glusterd_conf_t *priv = NULL;
+    xlator_t *this = NULL;
+    int pid = -1;
+
+    this = THIS;
+    if (!this)
+        return 0;
 
-        priv = this->private;
-        if (!priv)
-                return 0;
+    priv = this->private;
+    if (!priv)
+        return 0;
 
-        volinfo = mydata;
-        if (!volinfo)
-                return 0;
+    volinfo = mydata;
+    if (!volinfo)
+        return 0;
 
-        defrag = volinfo->rebal.defrag;
-        if (!defrag)
-                return 0;
+    defrag = volinfo->rebal.defrag;
+    if (!defrag)
+        return 0;
 
-        if ((event == RPC_CLNT_DISCONNECT) && defrag->connected)
-                volinfo->rebal.defrag = NULL;
+    if ((event == RPC_CLNT_DISCONNECT) && defrag->connected)
+        volinfo->rebal.defrag = NULL;
 
-        GLUSTERD_GET_DEFRAG_PID_FILE(pidfile, volinfo, priv);
+    GLUSTERD_GET_DEFRAG_PID_FILE(pidfile, volinfo, priv);
 
-        switch (event) {
-        case RPC_CLNT_CONNECT:
-        {
-                if (defrag->connected)
-                        return 0;
+    switch (event) {
+        case RPC_CLNT_CONNECT: {
+            if (defrag->connected)
+                return 0;
 
-                LOCK (&defrag->lock);
-                {
-                        defrag->connected = 1;
-                }
-                UNLOCK (&defrag->lock);
+            LOCK(&defrag->lock);
+            {
+                defrag->connected = 1;
+            }
+            UNLOCK(&defrag->lock);
 
-               gf_log ("", GF_LOG_DEBUG, "%s got RPC_CLNT_CONNECT",
-                        rpc->conn.name);
-               break;
+            gf_msg_debug(this->name, 0, "%s got RPC_CLNT_CONNECT",
+                         rpc->conn.name);
+            break;
         }
 
-        case RPC_CLNT_DISCONNECT:
-        {
-                if (!defrag->connected)
-                        return 0;
+        case RPC_CLNT_DISCONNECT: {
+            if (!defrag->connected)
+                return 0;
+
+            LOCK(&defrag->lock);
+            {
+                defrag->connected = 0;
+            }
+            UNLOCK(&defrag->lock);
 
-                LOCK (&defrag->lock);
-                {
-                        defrag->connected = 0;
+            if (!gf_is_service_running(pidfile, &pid)) {
+                if (volinfo->rebal.defrag_status == GF_DEFRAG_STATUS_STARTED) {
+                    volinfo->rebal.defrag_status = GF_DEFRAG_STATUS_FAILED;
                 }
-                UNLOCK (&defrag->lock);
+            }
 
-                if (!gf_is_service_running (pidfile, NULL)) {
-                        if (volinfo->rebal.defrag_status ==
-                                                GF_DEFRAG_STATUS_STARTED) {
-                                volinfo->rebal.defrag_status =
-                                                   GF_DEFRAG_STATUS_FAILED;
-                        }
-                 }
+            glusterd_store_perform_node_state_store(volinfo);
 
-                glusterd_store_perform_node_state_store (volinfo);
+            rpc_clnt_disable(defrag->rpc);
+            glusterd_defrag_rpc_put(defrag);
+            if (defrag->cbk_fn)
+                defrag->cbk_fn(volinfo, volinfo->rebal.defrag_status);
 
-                if (defrag->rpc) {
-                        glusterd_rpc_clnt_unref (priv, defrag->rpc);
-                        defrag->rpc = NULL;
-                }
-                if (defrag->cbk_fn)
-                        defrag->cbk_fn (volinfo,
-                                        volinfo->rebal.defrag_status);
-
-                GF_FREE (defrag);
-                gf_msg (this->name, GF_LOG_INFO, 0,
-                        GD_MSG_REBALANCE_DISCONNECTED,
-                        "Rebalance process for volume %s has disconnected.",
-                        volinfo->volname);
-                break;
+            GF_FREE(defrag);
+            gf_msg(this->name, GF_LOG_INFO, 0, GD_MSG_REBALANCE_DISCONNECTED,
+                   "Rebalance process for volume %s has disconnected.",
+                   volinfo->volname);
+            break;
         }
         case RPC_CLNT_DESTROY:
-                glusterd_volinfo_unref (volinfo);
-                break;
+            glusterd_volinfo_unref(volinfo);
+            break;
         default:
-                gf_log ("", GF_LOG_TRACE,
-                        "got some other RPC event %d", event);
-                ret = 0;
-                break;
-        }
+            gf_msg_trace(this->name, 0, "got some other RPC event %d", event);
+            ret = 0;
+            break;
+    }
 
-        return ret;
+    return ret;
 }
 
 int32_t
-glusterd_defrag_notify (struct rpc_clnt *rpc, void *mydata,
-                        rpc_clnt_event_t event, void *data)
+glusterd_defrag_notify(struct rpc_clnt *rpc, void *mydata,
+                       rpc_clnt_event_t event, void *data)
 {
-        return glusterd_big_locked_notify (rpc, mydata, event,
-                                           data, __glusterd_defrag_notify);
+    return glusterd_big_locked_notify(rpc, mydata, event, data,
+                                      __glusterd_defrag_notify);
 }
 
 int
-glusterd_handle_defrag_start (glusterd_volinfo_t *volinfo, char *op_errstr,
-                              size_t len, int cmd, defrag_cbk_fn_t cbk,
-                              glusterd_op_t op)
+glusterd_handle_defrag_start(glusterd_volinfo_t *volinfo, char *op_errstr,
+                             size_t len, int cmd, defrag_cbk_fn_t cbk,
+                             glusterd_op_t op)
 {
-        int                    ret = -1;
-        glusterd_defrag_info_t *defrag =  NULL;
-        runner_t               runner = {0,};
-        glusterd_conf_t        *priv = NULL;
-        char                   defrag_path[PATH_MAX];
-        char                   sockfile[PATH_MAX] = {0,};
-        char                   pidfile[PATH_MAX] = {0,};
-        char                   logfile[PATH_MAX] = {0,};
-        char                   volname[PATH_MAX] = {0,};
-        char                   valgrind_logfile[PATH_MAX] = {0,};
-
-        priv    = THIS->private;
-
-        GF_ASSERT (volinfo);
-        GF_ASSERT (op_errstr);
-
-        ret = glusterd_defrag_start_validate (volinfo, op_errstr, len, op);
-        if (ret)
-                goto out;
-        if (!volinfo->rebal.defrag)
-                volinfo->rebal.defrag =
-                        GF_CALLOC (1, sizeof (*volinfo->rebal.defrag),
-                                   gf_gld_mt_defrag_info);
-        if (!volinfo->rebal.defrag)
-                goto out;
+    xlator_t *this = NULL;
+    int ret = -1;
+    glusterd_defrag_info_t *defrag = NULL;
+    runner_t runner = {
+        0,
+    };
+    glusterd_conf_t *priv = NULL;
+    char defrag_path[PATH_MAX];
+    char sockfile[PATH_MAX] = {
+        0,
+    };
+    char pidfile[PATH_MAX] = {
+        0,
+    };
+    char logfile[PATH_MAX] = {
+        0,
+    };
+    char volname[PATH_MAX] = {
+        0,
+    };
+    char valgrind_logfile[PATH_MAX] = {
+        0,
+    };
+    char msg[1024] = {
+        0,
+    };
+    char *volfileserver = NULL;
+    char *localtime_logging = NULL;
+
+    this = THIS;
+    GF_VALIDATE_OR_GOTO("glusterd", this, out);
+
+    priv = this->private;
+    GF_VALIDATE_OR_GOTO("glusterd", priv, out);
+
+    GF_ASSERT(volinfo);
+    GF_ASSERT(op_errstr);
+
+    ret = glusterd_defrag_start_validate(volinfo, op_errstr, len, op);
+    if (ret)
+        goto out;
+    if (!volinfo->rebal.defrag)
+        volinfo->rebal.defrag = GF_CALLOC(1, sizeof(*volinfo->rebal.defrag),
+                                          gf_gld_mt_defrag_info);
+    if (!volinfo->rebal.defrag)
+        goto out;
+
+    defrag = volinfo->rebal.defrag;
+
+    defrag->cmd = cmd;
+
+    volinfo->rebal.defrag_cmd = cmd;
+    volinfo->rebal.op = op;
+
+    LOCK_INIT(&defrag->lock);
+
+    volinfo->rebal.defrag_status = GF_DEFRAG_STATUS_STARTED;
+
+    glusterd_volinfo_reset_defrag_stats(volinfo);
+    glusterd_store_perform_node_state_store(volinfo);
+
+    GLUSTERD_GET_DEFRAG_DIR(defrag_path, volinfo, priv);
+    ret = mkdir_p(defrag_path, 0755, _gf_true);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, errno, GD_MSG_CREATE_DIR_FAILED,
+               "Failed to create "
+               "directory %s",
+               defrag_path);
+        goto out;
+    }
+
+    GLUSTERD_GET_DEFRAG_SOCK_FILE(sockfile, volinfo);
+    GLUSTERD_GET_DEFRAG_PID_FILE(pidfile, volinfo, priv);
+    snprintf(logfile, PATH_MAX, "%s/%s-%s.log", priv->logdir, volinfo->volname,
+             "rebalance");
+    runinit(&runner);
+
+    if (this->ctx->cmd_args.vgtool != _gf_none) {
+        snprintf(valgrind_logfile, PATH_MAX, "%s/valgrind-%s-rebalance.log",
+                 priv->logdir, volinfo->volname);
+
+        if (this->ctx->cmd_args.vgtool == _gf_memcheck)
+            runner_add_args(&runner, "valgrind", "--leak-check=full",
+                            "--trace-children=yes", "--track-origins=yes",
+                            NULL);
+        else
+            runner_add_args(&runner, "valgrind", "--tool=drd", NULL);
+
+        runner_argprintf(&runner, "--log-file=%s", valgrind_logfile);
+    }
+
+    snprintf(volname, sizeof(volname), "rebalance/%s", volinfo->volname);
+
+    if (dict_get_strn(this->options, "transport.socket.bind-address",
+                      SLEN("transport.socket.bind-address"),
+                      &volfileserver) != 0) {
+        volfileserver = "localhost";
+    }
+
+    runner_add_args(
+        &runner, SBIN_DIR "/glusterfs", "-s", volfileserver, "--volfile-id",
+        volname, "--xlator-option", "*dht.use-readdirp=yes", "--xlator-option",
+        "*dht.lookup-unhashed=yes", "--xlator-option",
+        "*dht.assert-no-child-down=yes", "--xlator-option",
+        "*dht.readdir-optimize=on", "--process-name", "rebalance", NULL);
+
+    runner_add_arg(&runner, "--xlator-option");
+    runner_argprintf(&runner, "*dht.rebalance-cmd=%d", cmd);
+    runner_add_arg(&runner, "--xlator-option");
+    runner_argprintf(&runner, "*dht.node-uuid=%s", uuid_utoa(MY_UUID));
+    runner_add_arg(&runner, "--xlator-option");
+    runner_argprintf(&runner, "*dht.commit-hash=%u",
+                     volinfo->rebal.commit_hash);
+    runner_add_arg(&runner, "--socket-file");
+    runner_argprintf(&runner, "%s", sockfile);
+    runner_add_arg(&runner, "--pid-file");
+    runner_argprintf(&runner, "%s", pidfile);
+    runner_add_arg(&runner, "-l");
+    runner_argprintf(&runner, "%s", logfile);
+    if (volinfo->memory_accounting)
+        runner_add_arg(&runner, "--mem-accounting");
+    if (dict_get_strn(priv->opts, GLUSTERD_LOCALTIME_LOGGING_KEY,
+                      SLEN(GLUSTERD_LOCALTIME_LOGGING_KEY),
+                      &localtime_logging) == 0) {
+        if (strcmp(localtime_logging, "enable") == 0)
+            runner_add_arg(&runner, "--localtime-logging");
+    }
+
+    snprintf(msg, sizeof(msg), "Starting the rebalance service for volume %s",
+             volinfo->volname);
+    runner_log(&runner, this->name, GF_LOG_DEBUG, msg);
+
+    ret = runner_run_nowait(&runner);
+    if (ret) {
+        gf_msg_debug("glusterd", 0, "rebalance command failed");
+        goto out;
+    }
+
+    sleep(5);
+
+    ret = glusterd_rebalance_rpc_create(volinfo);
+
+    // FIXME: this cbk is passed as NULL in all occurrences. May be
+    // we never needed it.
+    if (cbk)
+        defrag->cbk_fn = cbk;
 
-        defrag = volinfo->rebal.defrag;
-
-        defrag->cmd = cmd;
-
-        volinfo->rebal.defrag_cmd = cmd;
-        volinfo->rebal.op = op;
-
-        LOCK_INIT (&defrag->lock);
+out:
+    gf_msg_debug("glusterd", 0, "Returning %d", ret);
+    return ret;
+}
 
-        volinfo->rebal.defrag_status = GF_DEFRAG_STATUS_STARTED;
+int
+glusterd_rebalance_defrag_init(glusterd_volinfo_t *volinfo, defrag_cbk_fn_t cbk)
 
-        glusterd_volinfo_reset_defrag_stats (volinfo);
-        glusterd_store_perform_node_state_store (volinfo);
+{
+    glusterd_defrag_info_t *defrag = NULL;
+    int ret = -1;
+
+    if (!volinfo->rebal.defrag) {
+        volinfo->rebal.defrag = GF_CALLOC(1, sizeof(*volinfo->rebal.defrag),
+                                          gf_gld_mt_defrag_info);
+    } else {
+        /*
+         * if defrag variable is already initialized,
+         * we skip the initialization.
+         */
+        ret = 0;
+        goto out;
+    }
+
+    if (!volinfo->rebal.defrag)
+        goto out;
+    defrag = volinfo->rebal.defrag;
+
+    defrag->cmd = volinfo->rebal.defrag_cmd;
+    LOCK_INIT(&defrag->lock);
+    if (cbk)
+        defrag->cbk_fn = cbk;
+    ret = 0;
+out:
+    return ret;
+}
 
-        GLUSTERD_GET_DEFRAG_DIR (defrag_path, volinfo, priv);
-        ret = mkdir_p (defrag_path, 0777, _gf_true);
-        if (ret) {
-                gf_log (THIS->name, GF_LOG_ERROR, "Failed to create "
-                        "directory %s", defrag_path);
-                goto out;
-        }
+int
+glusterd_rebalance_rpc_create(glusterd_volinfo_t *volinfo)
+{
+    dict_t *options = NULL;
+    char sockfile[PATH_MAX] = {
+        0,
+    };
+    int ret = -1;
+    glusterd_defrag_info_t *defrag = volinfo->rebal.defrag;
+    glusterd_conf_t *priv = NULL;
+    xlator_t *this = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+    priv = this->private;
+    GF_ASSERT(priv);
+
+    // rebalance process is not started
+    if (!defrag)
+        goto out;
+
+    options = dict_new();
+    if (!options) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_CREATE_FAIL, NULL);
+        goto out;
+    }
+
+    GLUSTERD_GET_DEFRAG_SOCK_FILE(sockfile, volinfo);
+
+    /* Setting frame-timeout to 10mins (600seconds).
+     * Unix domain sockets ensures that the connection is reliable. The
+     * default timeout of 30mins used for unreliable network connections is
+     * too long for unix domain socket connections.
+     */
+    ret = rpc_transport_unix_options_build(options, sockfile, 600);
+    if (ret) {
+        gf_msg(THIS->name, GF_LOG_ERROR, 0, GD_MSG_UNIX_OP_BUILD_FAIL,
+               "Unix options build failed");
+        goto out;
+    }
+
+    glusterd_volinfo_ref(volinfo);
+    ret = glusterd_rpc_create(&defrag->rpc, options, glusterd_defrag_notify,
+                              volinfo, _gf_true);
+    if (ret) {
+        gf_msg(THIS->name, GF_LOG_ERROR, 0, GD_MSG_RPC_CREATE_FAIL,
+               "Glusterd RPC creation failed");
+        goto out;
+    }
+    ret = 0;
+out:
+    if (options)
+        dict_unref(options);
+    return ret;
+}
 
-        GLUSTERD_GET_DEFRAG_SOCK_FILE (sockfile, volinfo);
-        GLUSTERD_GET_DEFRAG_PID_FILE (pidfile, volinfo, priv);
-        snprintf (logfile, PATH_MAX, "%s/%s-rebalance.log",
-                    DEFAULT_LOG_FILE_DIRECTORY, volinfo->volname);
-        runinit (&runner);
-
-        if (priv->valgrind) {
-                snprintf (valgrind_logfile, PATH_MAX,
-                          "%s/valgrind-%s-rebalance.log",
-                          DEFAULT_LOG_FILE_DIRECTORY,
-                          volinfo->volname);
-
-                runner_add_args (&runner, "valgrind", "--leak-check=full",
-                                 "--trace-children=yes", "--track-origins=yes",
-                                 NULL);
-                runner_argprintf (&runner, "--log-file=%s", valgrind_logfile);
-        }
+int
+glusterd_rebalance_cmd_validate(int cmd, char *volname,
+                                glusterd_volinfo_t **volinfo, char *op_errstr,
+                                size_t len)
+{
+    int ret = -1;
+
+    if (glusterd_volinfo_find(volname, volinfo)) {
+        gf_msg("glusterd", GF_LOG_ERROR, EINVAL, GD_MSG_VOL_NOT_FOUND,
+               "Received rebalance on invalid"
+               " volname %s",
+               volname);
+        snprintf(op_errstr, len, "Volume %s does not exist", volname);
+        goto out;
+    }
+    if ((*volinfo)->brick_count <= (*volinfo)->dist_leaf_count) {
+        gf_msg("glusterd", GF_LOG_ERROR, 0, GD_MSG_VOL_NOT_DISTRIBUTE,
+               "Volume %s is not a "
+               "distribute type or contains only 1 brick",
+               volname);
+        snprintf(op_errstr, len,
+                 "Volume %s is not a distribute "
+                 "volume or contains only 1 brick.\n"
+                 "Not performing rebalance",
+                 volname);
+        goto out;
+    }
+
+    if ((*volinfo)->status != GLUSTERD_STATUS_STARTED) {
+        gf_msg("glusterd", GF_LOG_ERROR, 0, GD_MSG_VOL_STOPPED,
+               "Received rebalance on stopped"
+               " volname %s",
+               volname);
+        snprintf(op_errstr, len,
+                 "Volume %s needs to "
+                 "be started to perform rebalance",
+                 volname);
+        goto out;
+    }
+
+    ret = 0;
 
-        snprintf (volname, sizeof(volname), "rebalance/%s", volinfo->volname);
-        runner_add_args (&runner, SBIN_DIR"/glusterfs",
-                         "-s", "localhost", "--volfile-id", volname,
-                         "--xlator-option", "*dht.use-readdirp=yes",
-                         "--xlator-option", "*dht.lookup-unhashed=yes",
-                         "--xlator-option", "*dht.assert-no-child-down=yes",
-                         "--xlator-option", "*replicate*.data-self-heal=off",
-                         "--xlator-option",
-                         "*replicate*.metadata-self-heal=off",
-                         "--xlator-option", "*replicate*.entry-self-heal=off",
-                         "--xlator-option", "*replicate*.readdir-failover=off",
-                         "--xlator-option", "*dht.readdir-optimize=on",
-                         NULL);
-
-        if (volinfo->type == GF_CLUSTER_TYPE_TIER) {
-                runner_add_arg (&runner, "--xlator-option");
-                runner_argprintf (&runner,
-                                  "*tier-dht.xattr-name=trusted.tier-gfid");
-        }
+out:
+    gf_msg_debug("glusterd", 0, "Returning %d", ret);
+    return ret;
+}
 
-        runner_add_arg (&runner, "--xlator-option");
-        runner_argprintf ( &runner, "*dht.rebalance-cmd=%d",cmd);
-        runner_add_arg (&runner, "--xlator-option");
-        runner_argprintf (&runner, "*dht.node-uuid=%s", uuid_utoa(MY_UUID));
-        runner_add_arg (&runner, "--socket-file");
-        runner_argprintf (&runner, "%s",sockfile);
-        runner_add_arg (&runner, "--pid-file");
-        runner_argprintf (&runner, "%s",pidfile);
-        runner_add_arg (&runner, "-l");
-        runner_argprintf (&runner, logfile);
-        if (volinfo->memory_accounting)
-                runner_add_arg (&runner, "--mem-accounting");
-
-        ret = runner_run_nowait (&runner);
-        if (ret) {
-                gf_log ("glusterd", GF_LOG_DEBUG, "rebalance command failed");
-                goto out;
+int
+__glusterd_handle_defrag_volume(rpcsvc_request_t *req)
+{
+    int32_t ret = -1;
+    gf_cli_req cli_req = {{
+        0,
+    }};
+    glusterd_conf_t *priv = NULL;
+    int32_t op = GD_OP_NONE;
+    dict_t *dict = NULL;
+    char *volname = NULL;
+    gf_cli_defrag_type cmd = 0;
+    char msg[2048] = {
+        0,
+    };
+    xlator_t *this = NULL;
+
+    GF_ASSERT(req);
+    this = THIS;
+    GF_ASSERT(this);
+
+    priv = this->private;
+    GF_ASSERT(priv);
+
+    ret = xdr_to_generic(req->msg[0], &cli_req, (xdrproc_t)xdr_gf_cli_req);
+    if (ret < 0) {
+        // failed to decode msg;
+        req->rpc_err = GARBAGE_ARGS;
+        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_GARBAGE_ARGS, NULL);
+        goto out;
+    }
+
+    if (cli_req.dict.dict_len) {
+        /* Unserialize the dictionary */
+        dict = dict_new();
+
+        ret = dict_unserialize(cli_req.dict.dict_val, cli_req.dict.dict_len,
+                               &dict);
+        if (ret < 0) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_UNSERIALIZE_FAIL,
+                   "failed to "
+                   "unserialize req-buffer to dictionary");
+            snprintf(msg, sizeof(msg),
+                     "Unable to decode the "
+                     "command");
+            goto out;
         }
+    }
+
+    ret = dict_get_strn(dict, "volname", SLEN("volname"), &volname);
+    if (ret) {
+        snprintf(msg, sizeof(msg), "Failed to get volume name");
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED, "%s", msg);
+        goto out;
+    }
+
+    ret = dict_get_int32n(dict, "rebalance-command", SLEN("rebalance-command"),
+                          (int32_t *)&cmd);
+    if (ret) {
+        snprintf(msg, sizeof(msg), "Failed to get command");
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED, "%s", msg);
+        goto out;
+    }
+
+    ret = dict_set_static_bin(dict, "node-uuid", MY_UUID, 16);
+    if (ret)
+        goto out;
+
+    if ((cmd == GF_DEFRAG_CMD_STATUS) || (cmd == GF_DEFRAG_CMD_STOP)) {
+        op = GD_OP_DEFRAG_BRICK_VOLUME;
+    } else
+        op = GD_OP_REBALANCE;
+
+    if (priv->op_version < GD_OP_VERSION_6_0) {
+        gf_msg_debug(this->name, 0,
+                     "The cluster is operating at "
+                     "version less than %d. Falling back "
+                     "to op-sm framework.",
+                     GD_OP_VERSION_6_0);
+        ret = glusterd_op_begin(req, op, dict, msg, sizeof(msg));
+        glusterd_friend_sm();
+        glusterd_op_sm();
+    } else {
+        ret = glusterd_mgmt_v3_initiate_all_phases_with_brickop_phase(req, op,
+                                                                      dict);
+    }
+out:
+    if (ret) {
+        if (msg[0] == '\0')
+            snprintf(msg, sizeof(msg), "Operation failed");
+        ret = glusterd_op_send_cli_response(GD_OP_REBALANCE, ret, 0, req, dict,
+                                            msg);
+    }
+
+    free(cli_req.dict.dict_val);  // malloced by xdr
+    gf_msg_debug(this->name, 0, "Returning %d", ret);
+    return ret;
+}
 
-        sleep (5);
-
-        ret = glusterd_rebalance_rpc_create (volinfo, _gf_false);
-
-        //FIXME: this cbk is passed as NULL in all occurrences. May be
-        //we never needed it.
-        if (cbk)
-                defrag->cbk_fn = cbk;
+int
+glusterd_handle_defrag_volume(rpcsvc_request_t *req)
+{
+    return glusterd_big_locked_handler(req, __glusterd_handle_defrag_volume);
+}
 
-out:
-        gf_log ("", GF_LOG_DEBUG, "Returning %d", ret);
+static int
+glusterd_brick_validation(dict_t *dict, char *key, data_t *value, void *data)
+{
+    int32_t ret = -1;
+    xlator_t *this = NULL;
+    glusterd_volinfo_t *volinfo = data;
+    glusterd_brickinfo_t *brickinfo = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+
+    ret = glusterd_volume_brickinfo_get_by_brick(value->data, volinfo,
+                                                 &brickinfo, _gf_false);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, EINVAL, GD_MSG_BRICK_NOT_FOUND,
+               "Incorrect brick %s for "
+               "volume %s",
+               value->data, volinfo->volname);
         return ret;
-}
+    }
+
+    if (!brickinfo->decommissioned) {
+        gf_msg(this->name, GF_LOG_ERROR, EINVAL, GD_MSG_BRICK_NOT_FOUND,
+               "Incorrect brick %s for "
+               "volume %s",
+               value->data, volinfo->volname);
+        ret = -1;
+        return ret;
+    }
 
+    return ret;
+}
 
 int
-glusterd_rebalance_rpc_create (glusterd_volinfo_t *volinfo,
-                               gf_boolean_t reconnect)
+glusterd_set_rebalance_id_in_rsp_dict(dict_t *req_dict, dict_t *rsp_dict)
 {
-        dict_t                  *options = NULL;
-        char                     sockfile[PATH_MAX] = {0,};
-        int                      ret = -1;
-        glusterd_defrag_info_t  *defrag = volinfo->rebal.defrag;
-        glusterd_conf_t         *priv = NULL;
-        xlator_t                *this = NULL;
-        struct stat             buf = {0,};
-
-        this = THIS;
-        GF_ASSERT (this);
-        priv = this->private;
-        GF_ASSERT (priv);
-
-        //rebalance process is not started
-        if (!defrag)
-                goto out;
-
-        //rpc obj for rebalance process already in place.
-        if (defrag->rpc) {
+    int ret = -1;
+    int32_t cmd = 0;
+    char *volname = NULL;
+    glusterd_volinfo_t *volinfo = NULL;
+    char msg[2048] = {0};
+    char *task_id_str = NULL;
+    xlator_t *this = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+
+    GF_ASSERT(rsp_dict);
+    GF_ASSERT(req_dict);
+
+    ret = dict_get_strn(rsp_dict, "volname", SLEN("volname"), &volname);
+    if (ret) {
+        gf_msg_debug(this->name, 0, "volname not found");
+        goto out;
+    }
+
+    ret = dict_get_int32n(rsp_dict, "rebalance-command",
+                          SLEN("rebalance-command"), &cmd);
+    if (ret) {
+        gf_msg_debug(this->name, 0, "cmd not found");
+        goto out;
+    }
+
+    ret = glusterd_rebalance_cmd_validate(cmd, volname, &volinfo, msg,
+                                          sizeof(msg));
+    if (ret) {
+        gf_msg_debug(this->name, 0, "failed to validate");
+        goto out;
+    }
+
+    /* reblance id is generted in glusterd_mgmt_v3_op_stage_rebalance(), but
+     * rsp_dict is unavailable there. So copying it to rsp_dict from req_dict
+     * here. So that cli can display the rebalance id.*/
+    if ((cmd == GF_DEFRAG_CMD_START) ||
+        (cmd == GF_DEFRAG_CMD_START_LAYOUT_FIX) ||
+        (cmd == GF_DEFRAG_CMD_START_FORCE)) {
+        if (is_origin_glusterd(rsp_dict)) {
+            ret = dict_get_strn(req_dict, GF_REBALANCE_TID_KEY,
+                                SLEN(GF_REBALANCE_TID_KEY), &task_id_str);
+            if (ret) {
+                snprintf(msg, sizeof(msg), "Missing rebalance-id");
+                gf_msg(this->name, GF_LOG_WARNING, 0,
+                       GD_MSG_REBALANCE_ID_MISSING, "%s", msg);
                 ret = 0;
-                goto out;
-        }
-        GLUSTERD_GET_DEFRAG_SOCK_FILE (sockfile, volinfo);
-        /* If reconnecting check if defrag sockfile exists in the new location
-         * in /var/run/ , if it does not try the old location
-         */
-        if (reconnect) {
-                ret = sys_stat (sockfile, &buf);
-                /* TODO: Remove this once we don't need backward compatibility
-                 * with the older path
-                 */
-                if (ret && (errno == ENOENT)) {
-                        gf_log (this->name, GF_LOG_WARNING, "Rebalance sockfile "
-                                "%s does not exist. Trying old path.",
-                                sockfile);
-                        GLUSTERD_GET_DEFRAG_SOCK_FILE_OLD (sockfile, volinfo,
-                                                           priv);
-                        ret =sys_stat (sockfile, &buf);
-                        if (ret && (ENOENT == errno)) {
-                                gf_msg (this->name, GF_LOG_ERROR, 0,
-                                        GD_MSG_REBAL_NO_SOCK_FILE, "Rebalance "
-                                        "sockfile %s does not exist", sockfile);
-                                goto out;
-                        }
+            } else {
+                gf_uuid_parse(task_id_str, volinfo->rebal.rebalance_id);
+                ret = glusterd_copy_uuid_to_dict(volinfo->rebal.rebalance_id,
+                                                 rsp_dict, GF_REBALANCE_TID_KEY,
+                                                 SLEN(GF_REBALANCE_TID_KEY));
+                if (ret) {
+                    snprintf(msg, sizeof(msg),
+                             "Failed to set rebalance id for volume %s",
+                             volname);
+                    gf_msg(this->name, GF_LOG_WARNING, 0,
+                           GD_MSG_DICT_SET_FAILED, "%s", msg);
                 }
+            }
         }
-
-        /* Setting frame-timeout to 10mins (600seconds).
-         * Unix domain sockets ensures that the connection is reliable. The
-         * default timeout of 30mins used for unreliable network connections is
-         * too long for unix domain socket connections.
-         */
-        ret = rpc_transport_unix_options_build (&options, sockfile, 600);
-        if (ret) {
-                gf_msg (THIS->name, GF_LOG_ERROR, 0, GD_MSG_UNIX_OP_BUILD_FAIL,
-                        "Unix options build failed");
+    }
+
+    /* Set task-id, if available, in rsp_dict for operations other than
+     * start. This is needed when we want rebalance id in xml output
+     */
+    if (cmd == GF_DEFRAG_CMD_STATUS || cmd == GF_DEFRAG_CMD_STOP) {
+        if (!gf_uuid_is_null(volinfo->rebal.rebalance_id)) {
+            if (GD_OP_REMOVE_BRICK == volinfo->rebal.op)
+                ret = glusterd_copy_uuid_to_dict(
+                    volinfo->rebal.rebalance_id, rsp_dict,
+                    GF_REMOVE_BRICK_TID_KEY, SLEN(GF_REMOVE_BRICK_TID_KEY));
+            else
+                ret = glusterd_copy_uuid_to_dict(volinfo->rebal.rebalance_id,
+                                                 rsp_dict, GF_REBALANCE_TID_KEY,
+                                                 SLEN(GF_REBALANCE_TID_KEY));
+            if (ret) {
+                gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+                       "Failed to set task-id for volume %s", volname);
                 goto out;
+            }
         }
-
-        glusterd_volinfo_ref (volinfo);
-        ret = glusterd_rpc_create (&defrag->rpc, options,
-                                   glusterd_defrag_notify, volinfo);
-        if (ret) {
-                gf_msg (THIS->name, GF_LOG_ERROR, 0, GD_MSG_RPC_CREATE_FAIL,
-                        "Glusterd RPC creation failed");
-                goto out;
-        }
-        ret = 0;
+    }
 out:
-        return ret;
+    return ret;
 }
 
 int
-glusterd_rebalance_cmd_validate (int cmd, char *volname,
-                                 glusterd_volinfo_t **volinfo,
-                                 char *op_errstr, size_t len)
+glusterd_mgmt_v3_op_stage_rebalance(dict_t *dict, char **op_errstr)
 {
-        int ret = -1;
-
-        if (glusterd_volinfo_find(volname, volinfo)) {
-                gf_log ("glusterd", GF_LOG_ERROR, "Received rebalance on invalid"
-                        " volname %s", volname);
-                snprintf (op_errstr, len, "Volume %s does not exist",
-                          volname);
+    char *volname = NULL;
+    char *cmd_str = NULL;
+    int ret = 0;
+    int32_t cmd = 0;
+    char msg[2048] = {0};
+    glusterd_volinfo_t *volinfo = NULL;
+    char *task_id_str = NULL;
+    xlator_t *this = 0;
+
+    this = THIS;
+    GF_ASSERT(this);
+
+    ret = dict_get_strn(dict, "volname", SLEN("volname"), &volname);
+    if (ret) {
+        gf_msg_debug(this->name, 0, "volname not found");
+        goto out;
+    }
+
+    ret = dict_get_int32n(dict, "rebalance-command", SLEN("rebalance-command"),
+                          &cmd);
+    if (ret) {
+        gf_msg_debug(this->name, 0, "cmd not found");
+        goto out;
+    }
+
+    ret = glusterd_rebalance_cmd_validate(cmd, volname, &volinfo, msg,
+                                          sizeof(msg));
+    if (ret) {
+        gf_msg_debug(this->name, 0, "failed to validate");
+        goto out;
+    }
+    switch (cmd) {
+        case GF_DEFRAG_CMD_START:
+        case GF_DEFRAG_CMD_START_LAYOUT_FIX:
+            /* Check if the connected clients are all of version
+             * glusterfs-3.6 and higher. This is needed to prevent some data
+             * loss issues that could occur when older clients are connected
+             * when rebalance is run. This check can be bypassed by using
+             * 'force'
+             */
+            ret = glusterd_check_client_op_version_support(
+                volname, GD_OP_VERSION_3_6_0, NULL);
+            if (ret) {
+                ret = gf_asprintf(op_errstr,
+                                  "Volume %s has one or "
+                                  "more connected clients of a version"
+                                  " lower than GlusterFS-v3.6.0. "
+                                  "Starting rebalance in this state "
+                                  "could lead to data loss.\nPlease "
+                                  "disconnect those clients before "
+                                  "attempting this command again.",
+                                  volname);
                 goto out;
-        }
-        if ((*volinfo)->brick_count <= (*volinfo)->dist_leaf_count) {
-                gf_log ("glusterd", GF_LOG_ERROR, "Volume %s is not a "
-                "distribute type or contains only 1 brick", volname);
-                snprintf (op_errstr, len, "Volume %s is not a distribute "
-                          "volume or contains only 1 brick.\n"
-                          "Not performing rebalance", volname);
+            }
+            /* Fall through */
+        case GF_DEFRAG_CMD_START_FORCE:
+            if (is_origin_glusterd(dict)) {
+                ret = glusterd_generate_and_set_task_id(
+                    dict, GF_REBALANCE_TID_KEY, SLEN(GF_REBALANCE_TID_KEY));
+                if (ret) {
+                    gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_TASKID_GEN_FAIL,
+                           "Failed to generate task-id");
+                    goto out;
+                }
+            } else {
+                ret = dict_get_strn(dict, GF_REBALANCE_TID_KEY,
+                                    SLEN(GF_REBALANCE_TID_KEY), &task_id_str);
+                if (ret) {
+                    snprintf(msg, sizeof(msg), "Missing rebalance-id");
+                    gf_msg(this->name, GF_LOG_WARNING, 0,
+                           GD_MSG_REBALANCE_ID_MISSING, "%s", msg);
+                    ret = 0;
+                }
+            }
+            ret = glusterd_defrag_start_validate(volinfo, msg, sizeof(msg),
+                                                 GD_OP_REBALANCE);
+            if (ret) {
+                gf_msg_debug(this->name, 0,
+                             "defrag start validate "
+                             "failed for volume %s.",
+                             volinfo->volname);
                 goto out;
-        }
+            }
+            break;
+        case GF_DEFRAG_CMD_STATUS:
+        case GF_DEFRAG_CMD_STOP:
 
-        if ((*volinfo)->status != GLUSTERD_STATUS_STARTED) {
-                gf_log ("glusterd", GF_LOG_ERROR, "Received rebalance on stopped"
-                        " volname %s", volname);
-                snprintf (op_errstr, len, "Volume %s needs to "
-                          "be started to perform rebalance", volname);
+            ret = dict_get_strn(dict, "cmd-str", SLEN("cmd-str"), &cmd_str);
+            if (ret) {
+                gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+                       "Failed to get "
+                       "command string");
+                ret = -1;
                 goto out;
-        }
+            }
+            if ((strstr(cmd_str, "rebalance") != NULL) &&
+                (volinfo->rebal.op != GD_OP_REBALANCE)) {
+                snprintf(msg, sizeof(msg),
+                         "Rebalance not started "
+                         "for volume %s.",
+                         volinfo->volname);
+                ret = -1;
+                goto out;
+            }
+
+            if (strstr(cmd_str, "remove-brick") != NULL) {
+                if (volinfo->rebal.op != GD_OP_REMOVE_BRICK) {
+                    snprintf(msg, sizeof(msg),
+                             "remove-brick not "
+                             "started for volume %s.",
+                             volinfo->volname);
+                    ret = -1;
+                    goto out;
+                }
 
-        ret = 0;
+                /* For remove-brick status/stop command check whether
+                 * given input brick is part of volume or not.*/
+
+                ret = dict_foreach_fnmatch(dict, "brick*",
+                                           glusterd_brick_validation, volinfo);
+                if (ret == -1) {
+                    snprintf(msg, sizeof(msg),
+                             "Incorrect brick"
+                             " for volume %s",
+                             volinfo->volname);
+                    goto out;
+                }
+            }
+            break;
+
+        default:
+            break;
+    }
 
+    ret = 0;
 out:
-        gf_log ("glusterd", GF_LOG_DEBUG, "Returning %d", ret);
-        return ret;
+    if (ret && op_errstr && msg[0])
+        *op_errstr = gf_strdup(msg);
+
+    return ret;
 }
 
 int
-__glusterd_handle_defrag_volume (rpcsvc_request_t *req)
+glusterd_mgmt_v3_op_rebalance(dict_t *dict, char **op_errstr, dict_t *rsp_dict)
 {
-        int32_t                 ret       = -1;
-        gf_cli_req              cli_req   = {{0,}};
-        glusterd_conf_t        *priv      = NULL;
-        dict_t                 *dict      = NULL;
-        char                   *volname   = NULL;
-        gf_cli_defrag_type      cmd       = 0;
-        char                    msg[2048] = {0,};
-        xlator_t               *this      = NULL;
-
-        GF_ASSERT (req);
-        this = THIS;
-        GF_ASSERT (this);
-
-        priv = this->private;
-        GF_ASSERT (priv);
-
-        ret = xdr_to_generic (req->msg[0], &cli_req, (xdrproc_t)xdr_gf_cli_req);
-        if (ret < 0) {
-                //failed to decode msg;
-                req->rpc_err = GARBAGE_ARGS;
-                goto out;
-        }
+    char *volname = NULL;
+    int ret = 0;
+    int32_t cmd = 0;
+    char msg[2048] = {0};
+    glusterd_volinfo_t *volinfo = NULL;
+    glusterd_brickinfo_t *brickinfo = NULL;
+    glusterd_brickinfo_t *tmp = NULL;
+    gf_boolean_t volfile_update = _gf_false;
+    char *task_id_str = NULL;
+    xlator_t *this = NULL;
+    uint32_t commit_hash;
+    int32_t is_force = 0;
+
+    this = THIS;
+    GF_ASSERT(this);
+
+    ret = dict_get_strn(dict, "volname", SLEN("volname"), &volname);
+    if (ret) {
+        gf_msg_debug(this->name, 0, "volname not given");
+        goto out;
+    }
+
+    ret = dict_get_int32n(dict, "rebalance-command", SLEN("rebalance-command"),
+                          &cmd);
+    if (ret) {
+        gf_msg_debug(this->name, 0, "command not given");
+        goto out;
+    }
+
+    ret = glusterd_rebalance_cmd_validate(cmd, volname, &volinfo, msg,
+                                          sizeof(msg));
+    if (ret) {
+        gf_msg_debug(this->name, 0, "cmd validate failed");
+        goto out;
+    }
+
+    switch (cmd) {
+        case GF_DEFRAG_CMD_START:
+        case GF_DEFRAG_CMD_START_LAYOUT_FIX:
+        case GF_DEFRAG_CMD_START_FORCE:
+
+            ret = dict_get_int32n(dict, "force", SLEN("force"), &is_force);
+            if (ret)
+                is_force = 0;
+            if (!is_force) {
+                /* Reset defrag status to 'NOT STARTED' whenever a
+                 * remove-brick/rebalance command is issued to remove
+                 * stale information from previous run.
+                 */
+                volinfo->rebal.defrag_status = GF_DEFRAG_STATUS_NOT_STARTED;
 
-        if (cli_req.dict.dict_len) {
-                /* Unserialize the dictionary */
-                dict  = dict_new ();
-
-                ret = dict_unserialize (cli_req.dict.dict_val,
-                                        cli_req.dict.dict_len,
-                                        &dict);
-                if (ret < 0) {
-                        gf_log (this->name, GF_LOG_ERROR, "failed to "
-                                "unserialize req-buffer to dictionary");
-                        snprintf (msg, sizeof (msg), "Unable to decode the "
-                                  "command");
-                        goto out;
+                ret = dict_get_strn(dict, GF_REBALANCE_TID_KEY,
+                                    SLEN(GF_REBALANCE_TID_KEY), &task_id_str);
+                if (ret) {
+                    gf_msg_debug(this->name, 0,
+                                 "Missing rebalance"
+                                 " id");
+                    ret = 0;
+                } else {
+                    gf_uuid_parse(task_id_str, volinfo->rebal.rebalance_id);
+                    volinfo->rebal.op = GD_OP_REBALANCE;
                 }
-        }
+                if (!gd_should_i_start_rebalance(volinfo)) {
+                    /* Store the rebalance-id and rebalance command
+                     * even if the peer isn't starting a rebalance
+                     * process. On peers where a rebalance process
+                     * is started, glusterd_handle_defrag_start
+                     * performs the storing.
+                     * Storing this is needed for having
+                     * 'volume status' work correctly.
+                     */
+                    glusterd_store_perform_node_state_store(volinfo);
+                    break;
+                }
+                if (dict_get_uint32(dict, "commit-hash", &commit_hash) == 0) {
+                    volinfo->rebal.commit_hash = commit_hash;
+                }
+                ret = glusterd_handle_defrag_start(volinfo, msg, sizeof(msg),
+                                                   cmd, NULL, GD_OP_REBALANCE);
+                break;
+            } else {
+                /* Reset defrag status to 'STARTED' so that the
+                 * pid is checked and restarted accordingly.
+                 * If the pid is not running it executes the
+                 * "NOT_STARTED" case and restarts the process
+                 */
+                volinfo->rebal.defrag_status = GF_DEFRAG_STATUS_STARTED;
+                volinfo->rebal.defrag_cmd = cmd;
+                volinfo->rebal.op = GD_OP_REBALANCE;
 
-        ret = dict_get_str (dict, "volname", &volname);
-        if (ret) {
-                snprintf (msg, sizeof (msg), "Failed to get volume name");
-                gf_log (this->name, GF_LOG_ERROR, "%s", msg);
-                goto out;
-        }
+                ret = dict_get_strn(dict, GF_REBALANCE_TID_KEY,
+                                    SLEN(GF_REBALANCE_TID_KEY), &task_id_str);
+                if (ret) {
+                    gf_msg_debug(this->name, 0,
+                                 "Missing rebalance"
+                                 " id");
+                    ret = 0;
+                } else {
+                    gf_uuid_parse(task_id_str, volinfo->rebal.rebalance_id);
+                    volinfo->rebal.op = GD_OP_REBALANCE;
+                }
+                if (dict_get_uint32(dict, "commit-hash", &commit_hash) == 0) {
+                    volinfo->rebal.commit_hash = commit_hash;
+                }
+                ret = glusterd_restart_rebalance_for_volume(volinfo);
+                break;
+            }
+        case GF_DEFRAG_CMD_STOP:
+            /* Clear task-id only on explicitly stopping rebalance.
+             * Also clear the stored operation, so it doesn't cause trouble
+             * with future rebalance/remove-brick starts
+             */
+            gf_uuid_clear(volinfo->rebal.rebalance_id);
+            volinfo->rebal.op = GD_OP_NONE;
+
+            /* Fall back to the old volume file in case of decommission*/
+            cds_list_for_each_entry_safe(brickinfo, tmp, &volinfo->bricks,
+                                         brick_list)
+            {
+                if (!brickinfo->decommissioned)
+                    continue;
+                brickinfo->decommissioned = 0;
+                volfile_update = _gf_true;
+            }
+
+            if (volfile_update == _gf_false) {
+                ret = 0;
+                break;
+            }
 
-        ret = dict_get_int32 (dict, "rebalance-command", (int32_t*)&cmd);
-        if (ret) {
-                snprintf (msg, sizeof (msg), "Failed to get command");
-                gf_log (this->name, GF_LOG_ERROR, "%s", msg);
+            ret = glusterd_create_volfiles_and_notify_services(volinfo);
+            if (ret) {
+                gf_msg(this->name, GF_LOG_WARNING, 0,
+                       GD_MSG_VOLFILE_CREATE_FAIL, "failed to create volfiles");
                 goto out;
-        }
+            }
 
-        ret = dict_set_static_bin (dict, "node-uuid", MY_UUID, 16);
-        if (ret)
+            ret = glusterd_store_volinfo(volinfo,
+                                         GLUSTERD_VOLINFO_VER_AC_INCREMENT);
+            if (ret) {
+                gf_msg(this->name, GF_LOG_WARNING, 0, GD_MSG_VOLINFO_SET_FAIL,
+                       "failed to store volinfo");
                 goto out;
+            }
 
-        if ((cmd == GF_DEFRAG_CMD_STATUS) ||
-            (cmd == GF_DEFRAG_CMD_STATUS_TIER) ||
-              (cmd == GF_DEFRAG_CMD_STOP)) {
-                ret = glusterd_op_begin (req, GD_OP_DEFRAG_BRICK_VOLUME,
-                                         dict, msg, sizeof (msg));
-        } else
-                ret = glusterd_op_begin (req, GD_OP_REBALANCE, dict,
-                                         msg, sizeof (msg));
-
-out:
-
-        glusterd_friend_sm ();
-        glusterd_op_sm ();
-
-        if (ret) {
-                if (msg[0] == '\0')
-                        snprintf (msg, sizeof (msg), "Operation failed");
-                ret = glusterd_op_send_cli_response (GD_OP_REBALANCE, ret, 0,
-                                                     req, dict, msg);
-
-        }
-
-        free (cli_req.dict.dict_val);//malloced by xdr
-
-        return 0;
-}
-
-int
-glusterd_handle_defrag_volume (rpcsvc_request_t *req)
-{
-        return glusterd_big_locked_handler (req, __glusterd_handle_defrag_volume);
-}
+            ret = 0;
+            break;
 
-static int
-glusterd_brick_validation  (dict_t *dict, char *key, data_t *value,
-                            void *data)
-{
-        int32_t                   ret              = -1;
-        xlator_t                 *this             = NULL;
-        glusterd_volinfo_t       *volinfo          = data;
-        glusterd_brickinfo_t     *brickinfo        = NULL;
-
-        this = THIS;
-        GF_ASSERT (this);
-
-        ret = glusterd_volume_brickinfo_get_by_brick (value->data, volinfo,
-                                                      &brickinfo);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "Incorrect brick %s for "
-                        "volume %s", value->data, volinfo->volname);
-                return ret;
-        }
+        case GF_DEFRAG_CMD_STATUS:
+            break;
+        default:
+            break;
+    }
 
-        if (!brickinfo->decommissioned) {
-                gf_log (this->name, GF_LOG_ERROR, "Incorrect brick %s for "
-                        "volume %s", value->data, volinfo->volname);
-                ret = -1;
-                return ret;
-        }
+out:
+    if (ret && op_errstr && msg[0])
+        *op_errstr = gf_strdup(msg);
 
-        return ret;
+    return ret;
 }
 
 int
-glusterd_op_stage_rebalance (dict_t *dict, char **op_errstr)
+glusterd_op_stage_rebalance(dict_t *dict, char **op_errstr)
 {
-        char                    *volname     = NULL;
-        char                    *cmd_str     = NULL;
-        int                     ret          = 0;
-        int32_t                 cmd          = 0;
-        char                    msg[2048]    = {0};
-        glusterd_volinfo_t      *volinfo     = NULL;
-        char                    *task_id_str = NULL;
-        dict_t                  *op_ctx      = NULL;
-        xlator_t                *this        = 0;
-
-        this = THIS;
-        GF_ASSERT (this);
-
-        ret = dict_get_str (dict, "volname", &volname);
-        if (ret) {
-                gf_log (this->name, GF_LOG_DEBUG, "volname not found");
-                goto out;
-        }
-
-        ret = dict_get_int32 (dict, "rebalance-command", &cmd);
-        if (ret) {
-                gf_log (this->name, GF_LOG_DEBUG, "cmd not found");
-                goto out;
-        }
-
-        ret = glusterd_rebalance_cmd_validate (cmd, volname, &volinfo,
-                                               msg, sizeof (msg));
-        if (ret) {
-                gf_log (this->name, GF_LOG_DEBUG, "failed to validate");
-                goto out;
-        }
-        switch (cmd) {
+    char *volname = NULL;
+    char *cmd_str = NULL;
+    int ret = 0;
+    int32_t cmd = 0;
+    char msg[2048] = {0};
+    glusterd_volinfo_t *volinfo = NULL;
+    char *task_id_str = NULL;
+    dict_t *op_ctx = NULL;
+    xlator_t *this = 0;
+
+    this = THIS;
+    GF_ASSERT(this);
+
+    ret = dict_get_strn(dict, "volname", SLEN("volname"), &volname);
+    if (ret) {
+        gf_msg_debug(this->name, 0, "volname not found");
+        goto out;
+    }
+
+    ret = dict_get_int32n(dict, "rebalance-command", SLEN("rebalance-command"),
+                          &cmd);
+    if (ret) {
+        gf_msg_debug(this->name, 0, "cmd not found");
+        goto out;
+    }
+
+    ret = glusterd_rebalance_cmd_validate(cmd, volname, &volinfo, msg,
+                                          sizeof(msg));
+    if (ret) {
+        gf_msg_debug(this->name, 0, "failed to validate");
+        goto out;
+    }
+    switch (cmd) {
         case GF_DEFRAG_CMD_START:
         case GF_DEFRAG_CMD_START_LAYOUT_FIX:
-        case GF_DEFRAG_CMD_START_TIER:
-                /* Check if the connected clients are all of version
-                 * glusterfs-3.6 and higher. This is needed to prevent some data
-                 * loss issues that could occur when older clients are connected
-                 * when rebalance is run. This check can be bypassed by using
-                 * 'force'
-                 */
-                ret = glusterd_check_client_op_version_support
-                        (volname, GD_OP_VERSION_3_6_0, NULL);
-                if (ret) {
-                        ret = gf_asprintf (op_errstr, "Volume %s has one or "
-                                           "more connected clients of a version"
-                                           " lower than GlusterFS-v3.6.0. "
-                                           "Starting rebalance in this state "
-                                           "could lead to data loss.\nPlease "
-                                           "disconnect those clients before "
-                                           "attempting this command again.",
-                                           volname);
-                        goto out;
+            /* Check if the connected clients are all of version
+             * glusterfs-3.6 and higher. This is needed to prevent some data
+             * loss issues that could occur when older clients are connected
+             * when rebalance is run. This check can be bypassed by using
+             * 'force'
+             */
+            ret = glusterd_check_client_op_version_support(
+                volname, GD_OP_VERSION_3_6_0, NULL);
+            if (ret) {
+                ret = gf_asprintf(op_errstr,
+                                  "Volume %s has one or "
+                                  "more connected clients of a version"
+                                  " lower than GlusterFS-v3.6.0. "
+                                  "Starting rebalance in this state "
+                                  "could lead to data loss.\nPlease "
+                                  "disconnect those clients before "
+                                  "attempting this command again.",
+                                  volname);
+                goto out;
+            }
+            /* Fall through */
+        case GF_DEFRAG_CMD_START_FORCE:
+            if (is_origin_glusterd(dict)) {
+                op_ctx = glusterd_op_get_ctx();
+                if (!op_ctx) {
+                    ret = -1;
+                    gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_OPCTX_GET_FAIL,
+                           "Failed to get op_ctx");
+                    goto out;
                 }
 
-        case GF_DEFRAG_CMD_START_FORCE:
-                if (is_origin_glusterd (dict)) {
-                        op_ctx = glusterd_op_get_ctx ();
-                        if (!op_ctx) {
-                                ret = -1;
-                                gf_log (this->name, GF_LOG_ERROR,
-                                        "Failed to get op_ctx");
-                                goto out;
-                        }
-
-                        ret = glusterd_generate_and_set_task_id
-                                (op_ctx, GF_REBALANCE_TID_KEY);
-                        if (ret) {
-                                gf_log (this->name, GF_LOG_ERROR,
-                                        "Failed to generate task-id");
-                                goto out;
-                        }
-                } else {
-                        ret = dict_get_str (dict, GF_REBALANCE_TID_KEY,
-                                            &task_id_str);
-                        if (ret) {
-                                snprintf (msg, sizeof (msg),
-                                          "Missing rebalance-id");
-                                gf_log (this->name, GF_LOG_WARNING, "%s", msg);
-                                ret = 0;
-                        }
+                ret = glusterd_generate_and_set_task_id(
+                    op_ctx, GF_REBALANCE_TID_KEY, SLEN(GF_REBALANCE_TID_KEY));
+                if (ret) {
+                    gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_TASKID_GEN_FAIL,
+                           "Failed to generate task-id");
+                    goto out;
                 }
-                ret = glusterd_defrag_start_validate (volinfo, msg,
-                                                      sizeof (msg),
-                                                      GD_OP_REBALANCE);
+            } else {
+                ret = dict_get_strn(dict, GF_REBALANCE_TID_KEY,
+                                    SLEN(GF_REBALANCE_TID_KEY), &task_id_str);
                 if (ret) {
-                        gf_log (this->name, GF_LOG_DEBUG,
-                                        "start validate failed");
-                        goto out;
+                    snprintf(msg, sizeof(msg), "Missing rebalance-id");
+                    gf_msg(this->name, GF_LOG_WARNING, 0,
+                           GD_MSG_REBALANCE_ID_MISSING, "%s", msg);
+                    ret = 0;
                 }
-                break;
+            }
+            ret = glusterd_defrag_start_validate(volinfo, msg, sizeof(msg),
+                                                 GD_OP_REBALANCE);
+            if (ret) {
+                gf_msg_debug(this->name, 0,
+                             "defrag start validate "
+                             "failed for volume %s.",
+                             volinfo->volname);
+                goto out;
+            }
+            break;
         case GF_DEFRAG_CMD_STATUS:
         case GF_DEFRAG_CMD_STOP:
-                ret = dict_get_str (dict, "cmd-str", &cmd_str);
-                if (ret) {
-                     gf_log (this->name, GF_LOG_ERROR, "Failed to get "
-                             "command string");
-                     ret = -1;
-                     goto out;
-                }
-                if ((strstr(cmd_str, "rebalance") != NULL) &&
-                    (volinfo->rebal.op != GD_OP_REBALANCE)) {
-                        snprintf (msg, sizeof(msg), "Rebalance not started.");
-                        ret = -1;
-                        goto out;
+
+            ret = dict_get_strn(dict, "cmd-str", SLEN("cmd-str"), &cmd_str);
+            if (ret) {
+                gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+                       "Failed to get "
+                       "command string");
+                ret = -1;
+                goto out;
+            }
+            if ((strstr(cmd_str, "rebalance") != NULL) &&
+                (volinfo->rebal.op != GD_OP_REBALANCE)) {
+                snprintf(msg, sizeof(msg),
+                         "Rebalance not started "
+                         "for volume %s.",
+                         volinfo->volname);
+                ret = -1;
+                goto out;
+            }
+
+            if (strstr(cmd_str, "remove-brick") != NULL) {
+                if (volinfo->rebal.op != GD_OP_REMOVE_BRICK) {
+                    snprintf(msg, sizeof(msg),
+                             "remove-brick not "
+                             "started for volume %s.",
+                             volinfo->volname);
+                    ret = -1;
+                    goto out;
                 }
-                if (strstr(cmd_str, "remove-brick") != NULL) {
-                        if (volinfo->rebal.op != GD_OP_REMOVE_BRICK) {
-                                snprintf (msg, sizeof(msg), "remove-brick not "
-                                          "started.");
-                                ret = -1;
-                                goto out;
-                        }
-
-                        /* For remove-brick status/stop command check whether
-                         * given input brick is part of volume or not.*/
-
-                        ret = dict_foreach_fnmatch (dict, "brick*",
-                                                    glusterd_brick_validation,
-                                                    volinfo);
-                        if (ret == -1) {
-                                snprintf (msg, sizeof (msg), "Incorrect brick"
-                                          " for volume %s", volinfo->volname);
-                                goto out;
-                        }
+
+                /* For remove-brick status/stop command check whether
+                 * given input brick is part of volume or not.*/
+
+                ret = dict_foreach_fnmatch(dict, "brick*",
+                                           glusterd_brick_validation, volinfo);
+                if (ret == -1) {
+                    snprintf(msg, sizeof(msg),
+                             "Incorrect brick"
+                             " for volume %s",
+                             volinfo->volname);
+                    goto out;
                 }
-                break;
+            }
+            break;
+
         default:
-                break;
-        }
+            break;
+    }
 
-        ret = 0;
+    ret = 0;
 out:
-        if (ret && op_errstr && msg[0])
-                *op_errstr = gf_strdup (msg);
+    if (ret && op_errstr && msg[0])
+        *op_errstr = gf_strdup(msg);
 
-        return ret;
+    return ret;
 }
 
 int
-glusterd_op_rebalance (dict_t *dict, char **op_errstr, dict_t *rsp_dict)
+glusterd_op_rebalance(dict_t *dict, char **op_errstr, dict_t *rsp_dict)
 {
-        char                    *volname   = NULL;
-        int                     ret       = 0;
-        int32_t                 cmd       = 0;
-        char                    msg[2048] = {0};
-        glusterd_volinfo_t      *volinfo   = NULL;
-        glusterd_conf_t         *priv      = NULL;
-        glusterd_brickinfo_t    *brickinfo = NULL;
-        glusterd_brickinfo_t    *tmp      = NULL;
-        gf_boolean_t            volfile_update = _gf_false;
-        char                    *task_id_str = NULL;
-        dict_t                  *ctx = NULL;
-        xlator_t                *this = NULL;
-
-        this = THIS;
-        GF_ASSERT (this);
-        priv = this->private;
-
-        ret = dict_get_str (dict, "volname", &volname);
-        if (ret) {
-                gf_log (this->name, GF_LOG_DEBUG, "volname not given");
-                goto out;
-        }
-
-        ret = dict_get_int32 (dict, "rebalance-command", &cmd);
-        if (ret) {
-                gf_log (this->name, GF_LOG_DEBUG, "command not given");
+    char *volname = NULL;
+    int ret = 0;
+    int32_t cmd = 0;
+    char msg[2048] = {0};
+    glusterd_volinfo_t *volinfo = NULL;
+    glusterd_brickinfo_t *brickinfo = NULL;
+    glusterd_brickinfo_t *tmp = NULL;
+    gf_boolean_t volfile_update = _gf_false;
+    char *task_id_str = NULL;
+    dict_t *ctx = NULL;
+    xlator_t *this = NULL;
+    uint32_t commit_hash;
+    int32_t is_force = 0;
+
+    this = THIS;
+    GF_ASSERT(this);
+
+    ret = dict_get_strn(dict, "volname", SLEN("volname"), &volname);
+    if (ret) {
+        gf_msg_debug(this->name, 0, "volname not given");
+        goto out;
+    }
+
+    ret = dict_get_int32n(dict, "rebalance-command", SLEN("rebalance-command"),
+                          &cmd);
+    if (ret) {
+        gf_msg_debug(this->name, 0, "command not given");
+        goto out;
+    }
+
+    ret = glusterd_rebalance_cmd_validate(cmd, volname, &volinfo, msg,
+                                          sizeof(msg));
+    if (ret) {
+        gf_msg_debug(this->name, 0, "cmd validate failed");
+        goto out;
+    }
+
+    /* Set task-id, if available, in op_ctx dict for operations other than
+     * start
+     */
+    if (cmd == GF_DEFRAG_CMD_STATUS || cmd == GF_DEFRAG_CMD_STOP) {
+        if (!gf_uuid_is_null(volinfo->rebal.rebalance_id)) {
+            ctx = glusterd_op_get_ctx();
+            if (!ctx) {
+                gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_OPCTX_GET_FAIL,
+                       "Failed to get op_ctx");
+                ret = -1;
                 goto out;
-        }
-
-
-        ret = glusterd_rebalance_cmd_validate (cmd, volname, &volinfo,
-                                               msg, sizeof (msg));
-        if (ret) {
-                gf_log (this->name, GF_LOG_DEBUG, "cmd validate failed");
+            }
+
+            if (GD_OP_REMOVE_BRICK == volinfo->rebal.op)
+                ret = glusterd_copy_uuid_to_dict(volinfo->rebal.rebalance_id,
+                                                 ctx, GF_REMOVE_BRICK_TID_KEY,
+                                                 SLEN(GF_REMOVE_BRICK_TID_KEY));
+            else
+                ret = glusterd_copy_uuid_to_dict(volinfo->rebal.rebalance_id,
+                                                 ctx, GF_REBALANCE_TID_KEY,
+                                                 SLEN(GF_REBALANCE_TID_KEY));
+            if (ret) {
+                gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_TASKID_GEN_FAIL,
+                       "Failed to set task-id");
                 goto out;
+            }
         }
+    }
 
-        /* Set task-id, if available, in op_ctx dict for operations other than
-         * start
-         */
-        if (cmd == GF_DEFRAG_CMD_STATUS ||
-            cmd == GF_DEFRAG_CMD_STOP ||
-            cmd == GF_DEFRAG_CMD_STATUS_TIER) {
-                if (!gf_uuid_is_null (volinfo->rebal.rebalance_id)) {
-                        ctx = glusterd_op_get_ctx ();
-                        if (!ctx) {
-                                gf_log (this->name, GF_LOG_ERROR,
-                                        "Failed to get op_ctx");
-                                ret = -1;
-                                goto out;
-                        }
-
-                        if (GD_OP_REMOVE_BRICK == volinfo->rebal.op)
-                                ret = glusterd_copy_uuid_to_dict
-                                        (volinfo->rebal.rebalance_id, ctx,
-                                         GF_REMOVE_BRICK_TID_KEY);
-                        else
-                                ret = glusterd_copy_uuid_to_dict
-                                        (volinfo->rebal.rebalance_id, ctx,
-                                         GF_REBALANCE_TID_KEY);
-                        if (ret) {
-                                gf_log (this->name, GF_LOG_ERROR,
-                                        "Failed to set task-id");
-                                goto out;
-                        }
-                }
-        }
-
-        switch (cmd) {
+    switch (cmd) {
         case GF_DEFRAG_CMD_START:
         case GF_DEFRAG_CMD_START_LAYOUT_FIX:
         case GF_DEFRAG_CMD_START_FORCE:
-        case GF_DEFRAG_CMD_START_TIER:
+
+            ret = dict_get_int32n(dict, "force", SLEN("force"), &is_force);
+            if (ret)
+                is_force = 0;
+            if (!is_force) {
                 /* Reset defrag status to 'NOT STARTED' whenever a
                  * remove-brick/rebalance command is issued to remove
                  * stale information from previous run.
                  */
                 volinfo->rebal.defrag_status = GF_DEFRAG_STATUS_NOT_STARTED;
 
-                ret = dict_get_str (dict, GF_REBALANCE_TID_KEY, &task_id_str);
+                ret = dict_get_strn(dict, GF_REBALANCE_TID_KEY,
+                                    SLEN(GF_REBALANCE_TID_KEY), &task_id_str);
                 if (ret) {
-                        gf_log (this->name, GF_LOG_DEBUG, "Missing rebalance "
-                                "id");
-                        ret = 0;
+                    gf_msg_debug(this->name, 0,
+                                 "Missing rebalance"
+                                 " id");
+                    ret = 0;
                 } else {
-                        gf_uuid_parse (task_id_str, volinfo->rebal.rebalance_id) ;
-                        volinfo->rebal.op = GD_OP_REBALANCE;
+                    gf_uuid_parse(task_id_str, volinfo->rebal.rebalance_id);
+                    volinfo->rebal.op = GD_OP_REBALANCE;
+                }
+                if (!gd_should_i_start_rebalance(volinfo)) {
+                    /* Store the rebalance-id and rebalance command
+                     * even if the peer isn't starting a rebalance
+                     * process. On peers where a rebalance process
+                     * is started, glusterd_handle_defrag_start
+                     * performs the storing.
+                     * Storing this is needed for having
+                     * 'volume status' work correctly.
+                     */
+                    glusterd_store_perform_node_state_store(volinfo);
+                    break;
                 }
-                if (!gd_should_i_start_rebalance (volinfo)) {
-                        /* Store the rebalance-id and rebalance command even if
-                         * the peer isn't starting a rebalance process. On peers
-                         * where a rebalance process is started,
-                         * glusterd_handle_defrag_start performs the storing.
-                         *
-                         * Storing this is needed for having 'volume status'
-                         * work correctly.
-                         */
-                        glusterd_store_perform_node_state_store (volinfo);
-                        break;
+                if (dict_get_uint32(dict, "commit-hash", &commit_hash) == 0) {
+                    volinfo->rebal.commit_hash = commit_hash;
                 }
-                ret = glusterd_handle_defrag_start (volinfo, msg, sizeof (msg),
-                                                    cmd, NULL, GD_OP_REBALANCE);
+                ret = glusterd_handle_defrag_start(volinfo, msg, sizeof(msg),
+                                                   cmd, NULL, GD_OP_REBALANCE);
                 break;
-        case GF_DEFRAG_CMD_STOP:
-                /* Clear task-id only on explicitly stopping rebalance.
-                 * Also clear the stored operation, so it doesn't cause trouble
-                 * with future rebalance/remove-brick starts
+            } else {
+                /* Reset defrag status to 'STARTED' so that the
+                 * pid is checked and restarted accordingly.
+                 * If the pid is not running it executes the
+                 * "NOT_STARTED" case and restarts the process
                  */
-                gf_uuid_clear (volinfo->rebal.rebalance_id);
-                volinfo->rebal.op = GD_OP_NONE;
-
-                /* Fall back to the old volume file in case of decommission*/
-                cds_list_for_each_entry_safe (brickinfo, tmp, &volinfo->bricks,
-                                              brick_list) {
-                        if (!brickinfo->decommissioned)
-                                continue;
-                        brickinfo->decommissioned = 0;
-                        volfile_update = _gf_true;
-                }
-
-                if (volfile_update == _gf_false) {
-                        ret = 0;
-                        break;
-                }
+                volinfo->rebal.defrag_status = GF_DEFRAG_STATUS_STARTED;
+                volinfo->rebal.defrag_cmd = cmd;
+                volinfo->rebal.op = GD_OP_REBALANCE;
 
-                ret = glusterd_create_volfiles_and_notify_services (volinfo);
+                ret = dict_get_strn(dict, GF_REBALANCE_TID_KEY,
+                                    SLEN(GF_REBALANCE_TID_KEY), &task_id_str);
                 if (ret) {
-                        gf_log (this->name, GF_LOG_WARNING,
-                                "failed to create volfiles");
-                        goto out;
+                    gf_msg_debug(this->name, 0,
+                                 "Missing rebalance"
+                                 " id");
+                    ret = 0;
+                } else {
+                    gf_uuid_parse(task_id_str, volinfo->rebal.rebalance_id);
+                    volinfo->rebal.op = GD_OP_REBALANCE;
                 }
-
-                ret = glusterd_store_volinfo (volinfo,
-                                             GLUSTERD_VOLINFO_VER_AC_INCREMENT);
-                if (ret) {
-                        gf_log (this->name, GF_LOG_WARNING,
-                                "failed to store volinfo");
-                        goto out;
+                if (dict_get_uint32(dict, "commit-hash", &commit_hash) == 0) {
+                    volinfo->rebal.commit_hash = commit_hash;
                 }
-
+                ret = glusterd_restart_rebalance_for_volume(volinfo);
+                break;
+            }
+        case GF_DEFRAG_CMD_STOP:
+            /* Clear task-id only on explicitly stopping rebalance.
+             * Also clear the stored operation, so it doesn't cause trouble
+             * with future rebalance/remove-brick starts
+             */
+            gf_uuid_clear(volinfo->rebal.rebalance_id);
+            volinfo->rebal.op = GD_OP_NONE;
+
+            /* Fall back to the old volume file in case of decommission*/
+            cds_list_for_each_entry_safe(brickinfo, tmp, &volinfo->bricks,
+                                         brick_list)
+            {
+                if (!brickinfo->decommissioned)
+                    continue;
+                brickinfo->decommissioned = 0;
+                volfile_update = _gf_true;
+            }
+
+            if (volfile_update == _gf_false) {
                 ret = 0;
                 break;
+            }
+
+            ret = glusterd_create_volfiles_and_notify_services(volinfo);
+            if (ret) {
+                gf_msg(this->name, GF_LOG_WARNING, 0,
+                       GD_MSG_VOLFILE_CREATE_FAIL, "failed to create volfiles");
+                goto out;
+            }
+
+            ret = glusterd_store_volinfo(volinfo,
+                                         GLUSTERD_VOLINFO_VER_AC_INCREMENT);
+            if (ret) {
+                gf_msg(this->name, GF_LOG_WARNING, 0, GD_MSG_VOLINFO_SET_FAIL,
+                       "failed to store volinfo");
+                goto out;
+            }
+
+            ret = 0;
+            break;
 
         case GF_DEFRAG_CMD_STATUS:
-        case GF_DEFRAG_CMD_STATUS_TIER:
-                break;
+            break;
         default:
-                break;
-        }
+            break;
+    }
 
 out:
-        if (ret && op_errstr && msg[0])
-                *op_errstr = gf_strdup (msg);
+    if (ret && op_errstr && msg[0])
+        *op_errstr = gf_strdup(msg);
 
-        return ret;
+    return ret;
 }
 
 int32_t
-glusterd_defrag_event_notify_handle (dict_t *dict)
+glusterd_defrag_event_notify_handle(dict_t *dict)
 {
-        glusterd_volinfo_t      *volinfo     = NULL;
-        char                    *volname     = NULL;
-        char                    *volname_ptr = NULL;
-        int32_t                  ret         = -1;
-        xlator_t                *this        = NULL;
-
-        this = THIS;
-        GF_ASSERT (this);
-        GF_ASSERT (dict);
-
-        ret = dict_get_str (dict, "volname", &volname);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "Failed to get volname");
-                return ret;
-        }
-
-        volname_ptr = strstr (volname, "rebalance/");
-        if (volname_ptr) {
-                volname_ptr = strchr (volname_ptr, '/');
-                if (!volname_ptr) {
-                        ret = -1;
-                        goto out;
-                }
-                volname = volname_ptr + 1;
-        } else {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "volname received (%s) is not prefixed with rebalance.",
-                        volname);
-                ret = -1;
-                goto out;
-        }
-
-        ret = glusterd_volinfo_find (volname, &volinfo);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "Failed to get volinfo for %s"
-                        , volname);
-                return ret;
-        }
+    glusterd_volinfo_t *volinfo = NULL;
+    char *volname = NULL;
+    char *volname_ptr = NULL;
+    int32_t ret = -1;
+    xlator_t *this = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+    GF_ASSERT(dict);
+
+    ret = dict_get_strn(dict, "volname", SLEN("volname"), &volname);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+               "Failed to get volname");
+        return ret;
+    }
+
+    volname_ptr = strstr(volname, "rebalance/");
+    if (volname_ptr) {
+        volname_ptr = strchr(volname_ptr, '/');
+        volname = volname_ptr + 1;
+    } else {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_NO_REBALANCE_PFX_IN_VOLNAME,
+               "volname received (%s) is not prefixed with rebalance.",
+               volname);
+        ret = -1;
+        goto out;
+    }
+
+    ret = glusterd_volinfo_find(volname, &volinfo);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_VOLINFO_GET_FAIL,
+               "Failed to get volinfo for %s", volname);
+        return ret;
+    }
 
-        ret = glusterd_defrag_volume_status_update (volinfo, dict);
+    ret = glusterd_defrag_volume_status_update(volinfo, dict, 0);
 
-        if (ret)
-                gf_log (this->name, GF_LOG_ERROR, "Failed to update status");
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DEFRAG_STATUS_UPDATE_FAIL,
+               "Failed to update status");
+        gf_event(EVENT_REBALANCE_STATUS_UPDATE_FAILED, "volume=%s",
+                 volinfo->volname);
+    }
 
 out:
-        return ret;
+    return ret;
 }
diff --git a/xlators/mgmt/glusterd/src/glusterd-replace-brick.c b/xlators/mgmt/glusterd/src/glusterd-replace-brick.c
index 92cfcd2fa95..43c2f4373e0 100644
--- a/xlators/mgmt/glusterd/src/glusterd-replace-brick.c
+++ b/xlators/mgmt/glusterd/src/glusterd-replace-brick.c
@@ -7,15 +7,10 @@
    later), or the GNU General Public License, version 2 (GPLv2), in all
    cases as published by the Free Software Foundation.
 */
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
-
-#include "common-utils.h"
+#include <glusterfs/common-utils.h>
 #include "cli1-xdr.h"
 #include "xdr-generic.h"
-#include "glusterfs.h"
+#include <glusterfs/glusterfs.h>
 #include "glusterd.h"
 #include "glusterd-op-sm.h"
 #include "glusterd-geo-rep.h"
@@ -23,2051 +18,699 @@
 #include "glusterd-utils.h"
 #include "glusterd-svc-mgmt.h"
 #include "glusterd-svc-helper.h"
-#include "glusterd-nfs-svc.h"
 #include "glusterd-volgen.h"
-#include "run.h"
-#include "syscall.h"
+#include "glusterd-messages.h"
+#include "glusterd-server-quorum.h"
+#include "glusterd-mgmt.h"
+#include <glusterfs/run.h>
+#include <glusterfs/syscall.h>
 
 #include <signal.h>
 
-#define GLUSTERD_GET_RB_MNTPT(path, len, volinfo)                           \
-        snprintf (path, len,                                                \
-                  DEFAULT_VAR_RUN_DIRECTORY"/%s-"RB_CLIENT_MOUNTPOINT,      \
-                  volinfo->volname);
-
-extern uuid_t global_txn_id;
-
 int
-glusterd_get_replace_op_str (gf1_cli_replace_op op, char *op_str)
-{
-        int     ret = -1;
-
-        if (!op_str)
-                goto out;
-
-        switch (op) {
-                case GF_REPLACE_OP_START:
-                        strcpy (op_str, "start");
-                        break;
-                case GF_REPLACE_OP_COMMIT:
-                        strcpy (op_str, "commit");
-                        break;
-                case GF_REPLACE_OP_PAUSE:
-                        strcpy (op_str, "pause");
-                        break;
-                case GF_REPLACE_OP_ABORT:
-                        strcpy (op_str, "abort");
-                        break;
-                case GF_REPLACE_OP_STATUS:
-                        strcpy (op_str, "status");
-                        break;
-                case GF_REPLACE_OP_COMMIT_FORCE:
-                        strcpy (op_str, "commit-force");
-                        break;
-                default:
-                        strcpy (op_str, "unknown");
-                        break;
-        }
-
-        ret = 0;
-out:
-        return ret;
-}
-
+glusterd_mgmt_v3_initiate_replace_brick_cmd_phases(rpcsvc_request_t *req,
+                                                   glusterd_op_t op,
+                                                   dict_t *dict);
 int
-__glusterd_handle_replace_brick (rpcsvc_request_t *req)
+__glusterd_handle_replace_brick(rpcsvc_request_t *req)
 {
-        int32_t                         ret = -1;
-        gf_cli_req                      cli_req = {{0,}};
-        dict_t                          *dict = NULL;
-        char                            *src_brick = NULL;
-        char                            *dst_brick = NULL;
-        int32_t                         op = 0;
-        char                            operation[256];
-        glusterd_op_t                   cli_op = GD_OP_REPLACE_BRICK;
-        char                            *volname = NULL;
-        char                            msg[2048] = {0,};
-        xlator_t                        *this = NULL;
-
-        GF_ASSERT (req);
-        this = THIS;
-        GF_ASSERT (this);
-
-        ret = xdr_to_generic (req->msg[0], &cli_req, (xdrproc_t)xdr_gf_cli_req);
+    int32_t ret = -1;
+    gf_cli_req cli_req = {{
+        0,
+    }};
+    dict_t *dict = NULL;
+    char *src_brick = NULL;
+    char *dst_brick = NULL;
+    char *cli_op = NULL;
+    glusterd_op_t op = -1;
+    char *volname = NULL;
+    char msg[256] = {
+        0,
+    };
+    xlator_t *this = NULL;
+    glusterd_conf_t *conf = NULL;
+
+    GF_ASSERT(req);
+    this = THIS;
+    GF_ASSERT(this);
+    conf = this->private;
+
+    ret = xdr_to_generic(req->msg[0], &cli_req, (xdrproc_t)xdr_gf_cli_req);
+    if (ret < 0) {
+        // failed to decode msg;
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_REQ_DECODE_FAIL,
+               "Failed to decode "
+               "request received from cli");
+        req->rpc_err = GARBAGE_ARGS;
+        goto out;
+    }
+
+    gf_msg(this->name, GF_LOG_INFO, 0, GD_MSG_REPLACE_BRK_REQ_RCVD,
+           "Received replace brick req");
+
+    if (cli_req.dict.dict_len) {
+        /* Unserialize the dictionary */
+        dict = dict_new();
+
+        ret = dict_unserialize(cli_req.dict.dict_val, cli_req.dict.dict_len,
+                               &dict);
         if (ret < 0) {
-                //failed to decode msg;
-                req->rpc_err = GARBAGE_ARGS;
-                goto out;
-        }
-
-        gf_log (this->name, GF_LOG_INFO, "Received replace brick req");
-
-        if (cli_req.dict.dict_len) {
-                /* Unserialize the dictionary */
-                dict  = dict_new ();
-
-                ret = dict_unserialize (cli_req.dict.dict_val,
-                                        cli_req.dict.dict_len,
-                                        &dict);
-                if (ret < 0) {
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "failed to "
-                                "unserialize req-buffer to dictionary");
-                        snprintf (msg, sizeof (msg), "Unable to decode the "
-                                  "command");
-                        goto out;
-                }
-        }
-
-        ret = dict_get_str (dict, "volname", &volname);
-        if (ret) {
-                snprintf (msg, sizeof (msg), "Could not get volume name");
-                gf_log (this->name, GF_LOG_ERROR, "%s", msg);
-                goto out;
-        }
-
-        ret = dict_get_int32 (dict, "operation", &op);
-        if (ret) {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "dict_get on operation failed");
-                snprintf (msg, sizeof (msg), "Could not get operation");
-                goto out;
-        }
-
-        ret = dict_get_str (dict, "src-brick", &src_brick);
-
-        if (ret) {
-                snprintf (msg, sizeof (msg), "Failed to get src brick");
-                gf_log (this->name, GF_LOG_ERROR, "%s", msg);
-                goto out;
-        }
-        gf_log (this->name, GF_LOG_DEBUG,
-                "src brick=%s", src_brick);
-
-        ret = dict_get_str (dict, "dst-brick", &dst_brick);
-
-        if (ret) {
-                snprintf (msg, sizeof (msg), "Failed to get dest brick");
-                gf_log (this->name, GF_LOG_ERROR, "%s", msg);
-                goto out;
-        }
-
-        (void) glusterd_get_replace_op_str (op, operation);
-        gf_log (this->name, GF_LOG_DEBUG, "dst brick=%s", dst_brick);
-        gf_log (this->name, GF_LOG_INFO, "Received replace brick %s request",
-                operation);
-
-        ret = glusterd_op_begin (req, GD_OP_REPLACE_BRICK, dict,
-                                 msg, sizeof (msg));
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_UNSERIALIZE_FAIL,
+                   "failed to "
+                   "unserialize req-buffer to dictionary");
+            snprintf(msg, sizeof(msg),
+                     "Unable to decode the "
+                     "command");
+            goto out;
+        }
+    }
+
+    ret = dict_get_strn(dict, "volname", SLEN("volname"), &volname);
+    if (ret) {
+        snprintf(msg, sizeof(msg), "Could not get volume name");
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED, "%s", msg);
+        goto out;
+    }
+
+    ret = dict_get_strn(dict, "operation", SLEN("operation"), &cli_op);
+    if (ret) {
+        gf_msg_debug(this->name, 0, "dict_get on operation failed");
+        snprintf(msg, sizeof(msg), "Could not get operation");
+        goto out;
+    }
+
+    op = gd_cli_to_gd_op(cli_op);
+
+    if (conf->op_version < GD_OP_VERSION_3_9_0 &&
+        strcmp(cli_op, "GF_REPLACE_OP_COMMIT_FORCE")) {
+        snprintf(msg, sizeof(msg),
+                 "Cannot execute command. The "
+                 "cluster is operating at version %d. reset-brick "
+                 "command %s is unavailable in this version.",
+                 conf->op_version, gd_rb_op_to_str(cli_op));
+        ret = -1;
+        goto out;
+    }
+
+    ret = dict_get_strn(dict, "src-brick", SLEN("src-brick"), &src_brick);
+
+    if (ret) {
+        snprintf(msg, sizeof(msg), "Failed to get src brick");
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED, "%s", msg);
+        goto out;
+    }
+    gf_msg_debug(this->name, 0, "src brick=%s", src_brick);
+
+    if (!strcmp(cli_op, "GF_RESET_OP_COMMIT") ||
+        !strcmp(cli_op, "GF_RESET_OP_COMMIT_FORCE") ||
+        !strcmp(cli_op, "GF_REPLACE_OP_COMMIT_FORCE")) {
+        ret = dict_get_strn(dict, "dst-brick", SLEN("dst-brick"), &dst_brick);
+
+        if (ret) {
+            snprintf(msg, sizeof(msg),
+                     "Failed to get"
+                     "dest brick");
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED, "%s",
+                   msg);
+            goto out;
+        }
+
+        gf_msg_debug(this->name, 0, "dst brick=%s", dst_brick);
+    }
+
+    gf_msg(this->name, GF_LOG_INFO, 0,
+           (op == GD_OP_REPLACE_BRICK)
+               ? GD_MSG_REPLACE_BRK_COMMIT_FORCE_REQ_RCVD
+               : GD_MSG_RESET_BRICK_COMMIT_FORCE_REQ_RCVD,
+           "Received %s request.", gd_rb_op_to_str(cli_op));
+
+    ret = glusterd_mgmt_v3_initiate_replace_brick_cmd_phases(req, op, dict);
 
 out:
-        free (cli_req.dict.dict_val);//malloced by xdr
-
-        glusterd_friend_sm ();
-        glusterd_op_sm ();
+    if (ret) {
+        glusterd_op_send_cli_response(op, ret, 0, req, dict, msg);
+    }
+    ret = 0;
+    free(cli_req.dict.dict_val);  // malloced by xdr
 
-        if (ret) {
-                if (msg[0] == '\0')
-                        snprintf (msg, sizeof (msg), "Operation failed");
-                ret = glusterd_op_send_cli_response (cli_op, ret, 0, req,
-                                                     dict, msg);
-        }
-
-        return ret;
+    return ret;
 }
 
 int
-glusterd_handle_replace_brick (rpcsvc_request_t *req)
+glusterd_handle_reset_brick(rpcsvc_request_t *req)
 {
-        return glusterd_big_locked_handler (req,
-                                            __glusterd_handle_replace_brick);
-}
-
-static int
-glusterd_get_rb_dst_brickinfo (glusterd_volinfo_t *volinfo,
-                               glusterd_brickinfo_t **brickinfo)
-{
-        int32_t                 ret = -1;
-
-        if (!volinfo || !brickinfo)
-                goto out;
-
-        *brickinfo = volinfo->rep_brick.dst_brick;
-
-        ret = 0;
-
-out:
-        return ret;
+    return glusterd_big_locked_handler(req, __glusterd_handle_replace_brick);
 }
 
 int
-glusterd_op_stage_replace_brick (dict_t *dict, char **op_errstr,
-                                 dict_t *rsp_dict)
-{
-        int                                      ret                = 0;
-        int32_t                                  port               = 0;
-        char                                    *src_brick          = NULL;
-        char                                    *dst_brick          = NULL;
-        char                                    *volname            = NULL;
-        int                                      replace_op         = 0;
-        glusterd_volinfo_t                      *volinfo            = NULL;
-        glusterd_brickinfo_t                    *src_brickinfo      = NULL;
-        char                                    *host               = NULL;
-        char                                    *path               = NULL;
-        char                                     msg[2048]          = {0};
-        char                                    *dup_dstbrick       = NULL;
-        glusterd_peerinfo_t                     *peerinfo           = NULL;
-        glusterd_brickinfo_t                    *dst_brickinfo      = NULL;
-        gf_boolean_t                             enabled            = _gf_false;
-        dict_t                                  *ctx                = NULL;
-        glusterd_conf_t                         *priv               = NULL;
-        char                                    *savetok            = NULL;
-        char                                     pidfile[PATH_MAX]  = {0};
-        char                                    *task_id_str        = NULL;
-        xlator_t                                *this               = NULL;
-        gf_boolean_t                             is_force           = _gf_false;
-        gsync_status_param_t                     param              = {0,};
-
-        this = THIS;
-        GF_ASSERT (this);
-
-        priv = this->private;
-        GF_ASSERT (priv);
-
-        ret = dict_get_str (dict, "src-brick", &src_brick);
-
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "Unable to get src brick");
-                goto out;
-        }
-
-        gf_log (this->name, GF_LOG_DEBUG, "src brick=%s", src_brick);
-
-        ret = dict_get_str (dict, "dst-brick", &dst_brick);
-
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "Unable to get dest brick");
-                goto out;
-        }
-
-        gf_log (this->name, GF_LOG_DEBUG, "dst brick=%s", dst_brick);
-
-        ret = dict_get_str (dict, "volname", &volname);
-
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "Unable to get volume name");
-                goto out;
-        }
-
-        ret = dict_get_int32 (dict, "operation", (int32_t *)&replace_op);
-        if (ret) {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "dict get on replace-brick operation failed");
-                goto out;
-        }
-
-        ret = glusterd_volinfo_find (volname, &volinfo);
-        if (ret) {
-                snprintf (msg, sizeof (msg), "volume: %s does not exist",
-                          volname);
-                *op_errstr = gf_strdup (msg);
-                goto out;
-        }
-
-        if (GLUSTERD_STATUS_STARTED != volinfo->status) {
-                ret = -1;
-                snprintf (msg, sizeof (msg), "volume: %s is not started",
-                          volname);
-                *op_errstr = gf_strdup (msg);
-                goto out;
-        }
-
-        if (!glusterd_store_is_valid_brickpath (volname, dst_brick) ||
-                !glusterd_is_valid_volfpath (volname, dst_brick)) {
-                snprintf (msg, sizeof (msg), "brick path %s is too "
-                          "long.", dst_brick);
-                gf_log (this->name, GF_LOG_ERROR, "%s", msg);
-                *op_errstr = gf_strdup (msg);
-
-                ret = -1;
-                goto out;
-        }
-
-        /* If geo-rep is configured, for this volume, it should be stopped. */
-        param.volinfo = volinfo;
-        ret = glusterd_check_geo_rep_running (&param, op_errstr);
-        if (ret || param.is_active) {
-                ret = -1;
-                goto out;
-        }
-
-        if (glusterd_is_defrag_on(volinfo)) {
-                snprintf (msg, sizeof(msg), "Volume name %s rebalance is in "
-                          "progress. Please retry after completion", volname);
-                gf_log (this->name, GF_LOG_ERROR, "%s", msg);
-                *op_errstr = gf_strdup (msg);
-                ret = -1;
-                goto out;
-        }
-
-        ctx = glusterd_op_get_ctx();
-
-        switch (replace_op) {
-        case GF_REPLACE_OP_START:
-                if (glusterd_is_rb_started (volinfo)) {
-                        snprintf (msg, sizeof (msg), "Replace brick is already "
-                                  "started for volume");
-                        gf_log (this->name, GF_LOG_ERROR, "%s", msg);
-                        *op_errstr = gf_strdup (msg);
-                        ret = -1;
-                        goto out;
-                }
-                if (is_origin_glusterd (dict)) {
-                        if (!ctx) {
-                                ret = -1;
-                                gf_log (this->name, GF_LOG_ERROR,
-                                        "Failed to get op_ctx");
-                                goto out;
-                        }
-
-                        ret = glusterd_generate_and_set_task_id
-                                (ctx, GF_REPLACE_BRICK_TID_KEY);
-                        if (ret) {
-                                gf_log (this->name, GF_LOG_ERROR,
-                                        "Failed to generate task-id");
-                                goto out;
-                        }
-
-                } else {
-                        ret = dict_get_str (dict, GF_REPLACE_BRICK_TID_KEY,
-                                            &task_id_str);
-                        if (ret) {
-                                gf_log (this->name, GF_LOG_WARNING,
-                                        "Missing replace-brick-id");
-                                ret = 0;
-                        }
-                }
-                is_force = dict_get_str_boolean (dict, "force", _gf_false);
-
-                break;
-
-        case GF_REPLACE_OP_PAUSE:
-                if (glusterd_is_rb_paused (volinfo)) {
-                        gf_log (this->name, GF_LOG_ERROR, "Replace brick is "
-                                "already paused for volume ");
-                        ret = -1;
-                        goto out;
-                } else if (!glusterd_is_rb_started(volinfo)) {
-                        gf_log (this->name, GF_LOG_ERROR, "Replace brick is not"
-                                " started for volume ");
-                        ret = -1;
-                        goto out;
-                }
-                break;
-
-        case GF_REPLACE_OP_ABORT:
-                if (!glusterd_is_rb_ongoing (volinfo)) {
-                        gf_log (this->name, GF_LOG_ERROR, "Replace brick is not"
-                                " started or paused for volume ");
-                        ret = -1;
-                        goto out;
-                }
-                break;
-
-        case GF_REPLACE_OP_COMMIT:
-                if (!glusterd_is_rb_ongoing (volinfo)) {
-                        gf_log (this->name, GF_LOG_ERROR, "Replace brick is not "
-                                "started for volume ");
-                        ret = -1;
-                        goto out;
-                }
-                break;
-
-        case GF_REPLACE_OP_COMMIT_FORCE:
-                is_force = _gf_true;
-                break;
-
-        case GF_REPLACE_OP_STATUS:
-
-                if (glusterd_is_rb_ongoing (volinfo) == _gf_false) {
-                        ret = gf_asprintf (op_errstr, "replace-brick not"
-                                           " started on volume %s",
-                                           volinfo->volname);
-                        if (ret < 0) {
-                                *op_errstr = NULL;
-                                goto out;
-                        }
-
-                        gf_log (this->name, GF_LOG_ERROR, "%s", *op_errstr);
-                        ret = -1;
-                        goto out;
-                }
-                break;
-
-        default:
-                ret = -1;
-                goto out;
-        }
-
-        ret = glusterd_volume_brickinfo_get_by_brick (src_brick, volinfo,
-                                                      &src_brickinfo);
-        if (ret) {
-                snprintf (msg, sizeof (msg), "brick: %s does not exist in "
-                          "volume: %s", src_brick, volname);
-                *op_errstr = gf_strdup (msg);
-                goto out;
-        }
-
-        if (ctx) {
-                if (!glusterd_is_fuse_available ()) {
-                        gf_log (this->name, GF_LOG_ERROR, "Unable to open /dev/"
-                                "fuse (%s), replace-brick command failed",
-                                strerror (errno));
-                        snprintf (msg, sizeof(msg), "Fuse unavailable\n "
-                                "Replace-brick failed");
-                        *op_errstr = gf_strdup (msg);
-                        ret = -1;
-                        goto out;
-                }
-        }
-
-        if (gf_is_local_addr (src_brickinfo->hostname)) {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "I AM THE SOURCE HOST");
-                if (src_brickinfo->port && rsp_dict) {
-                        ret = dict_set_int32 (rsp_dict, "src-brick-port",
-                                              src_brickinfo->port);
-                        if (ret) {
-                                gf_log ("", GF_LOG_DEBUG,
-                                        "Could not set src-brick-port=%d",
-                                        src_brickinfo->port);
-                        }
-                }
-
-                GLUSTERD_GET_BRICK_PIDFILE (pidfile, volinfo, src_brickinfo,
-                                            priv);
-                if ((replace_op != GF_REPLACE_OP_COMMIT_FORCE) &&
-                    !gf_is_service_running (pidfile, NULL)) {
-                        snprintf(msg, sizeof(msg), "Source brick %s:%s "
-                                 "is not online.", src_brickinfo->hostname,
-                                 src_brickinfo->path);
-                        *op_errstr = gf_strdup (msg);
-                        ret = -1;
-                        goto out;
-                }
-
-
-        }
-
-        dup_dstbrick = gf_strdup (dst_brick);
-        if (!dup_dstbrick) {
-                ret = -1;
-                gf_log (this->name, GF_LOG_ERROR, "Memory allocation failed");
-                goto out;
-        }
-        host = strtok_r (dup_dstbrick, ":", &savetok);
-        path = strtok_r (NULL, ":", &savetok);
-
-        if (!host || !path) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "dst brick %s is not of form <HOSTNAME>:<export-dir>",
-                        dst_brick);
-                ret = -1;
-                goto out;
-        }
-
-        ret = glusterd_brickinfo_new_from_brick (dst_brick, &dst_brickinfo);
-        if (ret)
-                goto out;
-
-        ret = glusterd_new_brick_validate (dst_brick, dst_brickinfo,
-                                           msg, sizeof (msg));
-        if (ret) {
-                *op_errstr = gf_strdup (msg);
-                ret = -1;
-                gf_log (this->name, GF_LOG_ERROR, "%s", *op_errstr);
-                goto out;
-        }
-
-       if (!glusterd_is_rb_ongoing (volinfo) &&
-            (replace_op == GF_REPLACE_OP_START ||
-             replace_op == GF_REPLACE_OP_COMMIT_FORCE)) {
-
-                volinfo->rep_brick.src_brick = src_brickinfo;
-                volinfo->rep_brick.dst_brick = dst_brickinfo;
-        }
-
-        if (glusterd_rb_check_bricks (volinfo, src_brickinfo, dst_brickinfo)) {
-
-                ret = -1;
-                *op_errstr = gf_strdup ("Incorrect source or "
-                                        "destination brick");
-                if (*op_errstr)
-                        gf_log (this->name, GF_LOG_ERROR, "%s", *op_errstr);
-                goto out;
-       }
-
-        if (!glusterd_is_rb_ongoing (volinfo) &&
-            gf_is_local_addr (host)) {
-                ret = glusterd_validate_and_create_brickpath (dst_brickinfo,
-                                                  volinfo->volume_id,
-                                                  op_errstr, is_force);
-                if (ret)
-                        goto out;
-        }
-
-        if (!gf_is_local_addr (host)) {
-                rcu_read_lock ();
-
-                peerinfo = glusterd_peerinfo_find (NULL, host);
-                if (peerinfo == NULL) {
-                        ret = -1;
-                        snprintf (msg, sizeof (msg), "%s, is not a friend",
-                                  host);
-                        *op_errstr = gf_strdup (msg);
-
-                } else if (!peerinfo->connected) {
-                        snprintf (msg, sizeof (msg), "%s, is not connected at "
-                                  "the moment", host);
-                        *op_errstr = gf_strdup (msg);
-                        ret = -1;
-
-                } else if (GD_FRIEND_STATE_BEFRIENDED !=
-                                peerinfo->state.state) {
-                        snprintf (msg, sizeof (msg), "%s, is not befriended "
-                                  "at the moment", host);
-                        *op_errstr = gf_strdup (msg);
-                        ret = -1;
-                }
-                rcu_read_unlock ();
-
-                if (ret)
-                        goto out;
-
-        } else if (priv->op_version >= GD_OP_VERSION_3_6_0) {
-                /* A bricks mount dir is required only by snapshots which were
-                 * introduced in gluster-3.6.0
-                 */
-                ret = glusterd_get_brick_mount_dir (dst_brickinfo->path,
-                                                    dst_brickinfo->hostname,
-                                                    dst_brickinfo->mount_dir);
-                if (ret) {
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "Failed to get brick mount_dir");
-                        goto out;
-                }
-
-                ret = dict_set_dynstr_with_alloc (rsp_dict, "brick1.mount_dir",
-                                                  dst_brickinfo->mount_dir);
-                if (ret) {
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "Failed to set brick1.mount_dir");
-                        goto out;
-                }
-
-                ret = dict_set_int32 (rsp_dict, "brick_count", 1);
-                if (ret) {
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "Failed to set local_brick_count");
-                        goto out;
-                }
-        }
-
-        if (replace_op == GF_REPLACE_OP_START &&
-            gf_is_local_addr (volinfo->rep_brick.dst_brick->hostname)) {
-                port = pmap_registry_alloc (THIS);
-                if (!port) {
-                        gf_log (THIS->name, GF_LOG_CRITICAL,
-                                "No free ports available");
-                        ret = -1;
-                        goto out;
-                }
-
-                ctx = glusterd_op_get_ctx();
-                ret = dict_set_int32 ((ctx)?ctx:rsp_dict, "dst-brick-port",
-                                      port);
-                if (ret) {
-                        gf_log (THIS->name, GF_LOG_ERROR, "Failed to set dst "
-                                "brick port");
-                        goto out;
-                }
-                volinfo->rep_brick.dst_brick->port = port;
-        }
-
-        ret = 0;
-
-out:
-        GF_FREE (dup_dstbrick);
-        gf_log (this->name, GF_LOG_DEBUG, "Returning %d", ret);
-
-        return ret;
-}
-
-static int
-rb_set_mntfd (int mntfd)
-{
-        int     ret = -1;
-        dict_t *ctx = NULL;
-
-        ctx = glusterd_op_get_ctx ();
-        if (!ctx) {
-                gf_log (THIS->name, GF_LOG_CRITICAL, "Failed to get op ctx");
-                goto out;
-        }
-        ret = dict_set_int32 (ctx, "mntfd", mntfd);
-        if (ret)
-                gf_log (THIS->name, GF_LOG_DEBUG, "Failed to set mnt fd "
-                        "in op ctx");
-out:
-        return ret;
-}
-
-static int
-rb_get_mntfd (int *mntfd)
-{
-        int     ret = -1;
-        dict_t *ctx = NULL;
-
-        ctx = glusterd_op_get_ctx ();
-        if (!ctx) {
-                gf_log (THIS->name, GF_LOG_CRITICAL, "Failed to get op ctx");
-                goto out;
-        }
-        ret = dict_get_int32 (ctx, "mntfd", mntfd);
-        if (ret)
-                gf_log (THIS->name, GF_LOG_DEBUG, "Failed to get mnt fd "
-                        "from op ctx");
-out:
-        return ret;
-}
-
-static int
-rb_regenerate_volfiles (glusterd_volinfo_t *volinfo,
-                        glusterd_brickinfo_t *brickinfo,
-                        int32_t pump_needed)
-{
-        dict_t *dict = NULL;
-        int ret = 0;
-
-        dict = volinfo->dict;
-
-        gf_log ("", GF_LOG_DEBUG,
-                "attempting to set pump value=%d", pump_needed);
-
-        ret = dict_set_int32 (dict, "enable-pump", pump_needed);
-        if (ret) {
-                gf_log ("", GF_LOG_DEBUG,
-                        "could not dict_set enable-pump");
-                goto out;
-        }
-
-        ret = glusterd_create_rb_volfiles (volinfo, brickinfo);
-
-        dict_del (dict, "enable-pump");
-
-out:
-        return ret;
-}
-
-static int
-rb_src_brick_restart (glusterd_volinfo_t *volinfo,
-                      glusterd_brickinfo_t *src_brickinfo,
-                      int activate_pump)
-{
-        int              ret  = 0;
-        xlator_t        *this = NULL;
-        glusterd_conf_t *priv = NULL;
-
-        this = THIS;
-        GF_ASSERT (this);
-
-        priv = this->private;
-        GF_ASSERT (priv);
-
-        gf_log ("", GF_LOG_DEBUG,
-                "Attempting to kill src");
-
-        ret = priv->nfs_svc.stop (&(priv->nfs_svc), SIGKILL);
-
-        if (ret) {
-                gf_log ("", GF_LOG_ERROR, "Unable to stop nfs, ret: %d",
-                        ret);
-        }
-
-        ret = glusterd_volume_stop_glusterfs (volinfo, src_brickinfo,
-                                              _gf_false);
-        if (ret) {
-                gf_log ("", GF_LOG_ERROR, "Unable to stop "
-                        "glusterfs, ret: %d", ret);
-                goto out;
-        }
-
-        glusterd_delete_volfile (volinfo, src_brickinfo);
-
-        if (activate_pump) {
-                ret = rb_regenerate_volfiles (volinfo, src_brickinfo, 1);
-                if (ret) {
-                        gf_log ("", GF_LOG_DEBUG,
-                                "Could not regenerate volfiles with pump");
-                        goto out;
-                }
-        } else {
-                ret = rb_regenerate_volfiles (volinfo, src_brickinfo, 0);
-                if (ret) {
-                        gf_log ("", GF_LOG_DEBUG,
-                                "Could not regenerate volfiles without pump");
-                        goto out;
-                }
-
-        }
-
-        sleep (2);
-        ret = glusterd_volume_start_glusterfs (volinfo, src_brickinfo,
-                                               _gf_false);
-        if (ret) {
-                gf_log ("", GF_LOG_ERROR, "Unable to start "
-                        "glusterfs, ret: %d", ret);
-                goto out;
-        }
-
-out:
-        ret = priv->nfs_svc.start (&(priv->nfs_svc), PROC_START_NO_WAIT);
-        if (ret) {
-                gf_log ("", GF_LOG_ERROR, "Unable to start nfs, ret: %d",
-                        ret);
-        }
-        return ret;
-}
-
-static int
-rb_send_xattr_command (glusterd_volinfo_t *volinfo,
-                       glusterd_brickinfo_t *src_brickinfo,
-                       glusterd_brickinfo_t *dst_brickinfo,
-                       const char *xattr_key, const char *value)
-{
-        int             ret   = -1;
-        int             mntfd = -1;
-
-        ret = rb_get_mntfd (&mntfd);
-        if (ret)
-                goto out;
-
-        ret = sys_fsetxattr (mntfd, xattr_key, value, strlen (value) + 1, 0);
-        if (ret)
-                gf_log (THIS->name, GF_LOG_DEBUG, "setxattr on key: "
-                        "%s, reason: %s", xattr_key, strerror (errno));
-
-out:
-        return ret;
-}
-
-static int
-rb_spawn_dst_brick (glusterd_volinfo_t *volinfo,
-                    glusterd_brickinfo_t *brickinfo)
-{
-        glusterd_conf_t    *priv           = NULL;
-        runner_t            runner         = {0,};
-        int                 ret            = -1;
-        int32_t             port           = 0;
-
-        priv = THIS->private;
-
-        port = brickinfo->port;
-        GF_ASSERT (port);
-
-        runinit (&runner);
-        runner_add_arg (&runner, SBIN_DIR"/glusterfs");
-        runner_argprintf (&runner, "-f" "%s/vols/%s/"RB_DSTBRICKVOL_FILENAME,
-                          priv->workdir, volinfo->volname);
-        runner_argprintf (&runner, "-p" "%s/vols/%s/"RB_DSTBRICK_PIDFILE,
-                          priv->workdir, volinfo->volname);
-        runner_add_arg (&runner, "--xlator-option");
-        runner_argprintf (&runner, "src-server.listen-port=%d", port);
-        if (volinfo->memory_accounting)
-                runner_add_arg (&runner, "--mem-accounting");
-
-        ret = runner_run_nowait (&runner);
-        if (ret) {
-                pmap_registry_remove (THIS, 0, brickinfo->path,
-                                      GF_PMAP_PORT_BRICKSERVER, NULL);
-                gf_log ("", GF_LOG_DEBUG,
-                        "Could not start glusterfs");
-                goto out;
-        }
-
-        gf_log ("", GF_LOG_DEBUG,
-                "Successfully started glusterfs: brick=%s:%s",
-                brickinfo->hostname, brickinfo->path);
-
-        ret = 0;
-
-out:
-        return ret;
-}
-
-static int
-rb_spawn_glusterfs_client (glusterd_volinfo_t *volinfo,
-                           glusterd_brickinfo_t *brickinfo)
+glusterd_handle_replace_brick(rpcsvc_request_t *req)
 {
-        xlator_t           *this            = NULL;
-        glusterd_conf_t    *priv            = NULL;
-        runner_t            runner          = {0,};
-        struct stat         buf             = {0,};
-        char                mntpt[PATH_MAX] = {0,};
-        int                 mntfd           = -1;
-        int                 ret             = -1;
-
-        this = THIS;
-        priv = this->private;
-
-        GLUSTERD_GET_RB_MNTPT (mntpt, sizeof (mntpt), volinfo);
-        runinit (&runner);
-        runner_add_arg (&runner, SBIN_DIR"/glusterfs");
-        runner_argprintf (&runner, "-f" "%s/vols/%s/"RB_CLIENTVOL_FILENAME,
-                          priv->workdir, volinfo->volname);
-        runner_add_arg (&runner, mntpt);
-        if (volinfo->memory_accounting)
-                runner_add_arg (&runner, "--mem-accounting");
-
-        ret = runner_run_reuse (&runner);
-        if (ret) {
-                runner_log (&runner, this->name, GF_LOG_DEBUG,
-                            "Could not start glusterfs");
-                runner_end (&runner);
-                goto out;
-        } else {
-                runner_log (&runner, this->name, GF_LOG_DEBUG,
-                            "Successfully started  glusterfs");
-                runner_end (&runner);
-        }
-
-        ret = stat (mntpt, &buf);
-        if (ret) {
-                 gf_log (this->name, GF_LOG_DEBUG, "stat on mount point %s "
-                         "failed", mntpt);
-                 goto out;
-        }
-
-        mntfd = open (mntpt, O_DIRECTORY);
-        if (mntfd == -1)
-                goto out;
-
-        ret = rb_set_mntfd (mntfd);
-        if (ret)
-                goto out;
-
-
-out:
-
-        return ret;
+    return glusterd_big_locked_handler(req, __glusterd_handle_replace_brick);
 }
 
-static const char *client_volfile_str =  "volume mnt-client\n"
-        " type protocol/client\n"
-        " option remote-host %s\n"
-        " option remote-subvolume %s\n"
-        " option remote-port %d\n"
-        " option transport-type %s\n"
-        " option username %s\n"
-        " option password %s\n"
-        "end-volume\n"
-        "volume mnt-wb\n"
-        " type performance/write-behind\n"
-        " subvolumes mnt-client\n"
-        "end-volume\n";
-
-static int
-rb_generate_client_volfile (glusterd_volinfo_t *volinfo,
-                            glusterd_brickinfo_t *src_brickinfo)
-{
-        glusterd_conf_t  *priv                  = NULL;
-        xlator_t         *this                  = NULL;
-        FILE             *file                  = NULL;
-        char              filename[PATH_MAX]    = {0, };
-        int               ret                   = -1;
-        int               fd                    = -1;
-        char             *ttype                 = NULL;
-
-        this = THIS;
-        priv = this->private;
-
-        gf_log (this->name, GF_LOG_DEBUG, "Creating volfile");
-
-        snprintf (filename, PATH_MAX, "%s/vols/%s/%s",
-                  priv->workdir, volinfo->volname,
-                  RB_CLIENTVOL_FILENAME);
-
-        fd = open (filename, O_CREAT | O_RDONLY, S_IRUSR | S_IWUSR);
-        if (fd < 0) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "%s", strerror (errno));
-                goto out;
-        }
-        sys_close (fd);
-
-        file = fopen (filename, "w+");
-        if (!file) {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "Open of volfile failed");
-                ret = -1;
-                goto out;
-        }
-
-        GF_ASSERT (src_brickinfo->port);
-
-	ttype = glusterd_get_trans_type_rb (volinfo->transport_type);
-	if (NULL == ttype){
-		ret = -1;
-		goto out;
-	}
-
-        fprintf (file, client_volfile_str, src_brickinfo->hostname,
-                 src_brickinfo->path,
-                 src_brickinfo->port, ttype,
-                 glusterd_auth_get_username (volinfo),
-                 glusterd_auth_get_password (volinfo));
-
-        GF_FREE (ttype);
-
-        ret = 0;
-
-out:
-        if (file)
-                fclose (file);
-
-        return ret;
-}
-
-static const char *dst_brick_volfile_str = "volume src-posix\n"
-        " type storage/posix\n"
-        " option directory %s\n"
-        " option volume-id %s\n"
-        "end-volume\n"
-        "volume %s\n"
-        " type features/locks\n"
-        " subvolumes src-posix\n"
-        "end-volume\n"
-        "volume src-server\n"
-        " type protocol/server\n"
-        " option auth.login.%s.allow %s\n"
-        " option auth.login.%s.password %s\n"
-        " option auth.addr.%s.allow *\n"
-        " option transport-type %s\n"
-        " subvolumes %s\n"
-        "end-volume\n";
-
-static int
-rb_generate_dst_brick_volfile (glusterd_volinfo_t *volinfo,
-                               glusterd_brickinfo_t *dst_brickinfo)
-{
-        glusterd_conf_t    *priv                = NULL;
-        xlator_t           *this                = NULL;
-        FILE               *file                = NULL;
-        char                filename[PATH_MAX]  = {0, };
-        int                 ret                 = -1;
-        int                 fd                  = -1;
-        char               *trans_type          = NULL;
-
-        this = THIS;
-        priv = this->private;
-
-        gf_log (this->name, GF_LOG_DEBUG,
-                "Creating volfile");
-
-        snprintf (filename, PATH_MAX, "%s/vols/%s/%s",
-                  priv->workdir, volinfo->volname,
-                  RB_DSTBRICKVOL_FILENAME);
-
-        fd = sys_creat (filename, S_IRUSR | S_IWUSR);
-        if (fd < 0) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "%s", strerror (errno));
-                goto out;
-        }
-        sys_close (fd);
-
-        file = fopen (filename, "w+");
-        if (!file) {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "Open of volfile failed");
-                ret = -1;
-                goto out;
-        }
-
-	trans_type = glusterd_get_trans_type_rb (volinfo->transport_type);
-	if (NULL == trans_type){
-		ret = -1;
-		goto out;
-	}
-
-        fprintf (file, dst_brick_volfile_str,
-                 dst_brickinfo->path,
-                 uuid_utoa (volinfo->volume_id),
-                 dst_brickinfo->path,
-                 dst_brickinfo->path,
-                 glusterd_auth_get_username (volinfo),
-                 glusterd_auth_get_username (volinfo),
-                 glusterd_auth_get_password (volinfo),
-                 dst_brickinfo->path,
-                 trans_type,
-                 dst_brickinfo->path);
-
-	GF_FREE (trans_type);
-
-        ret = 0;
-
-out:
-        if (file)
-                fclose (file);
-
-        return ret;
-}
-
-
-static int
-rb_mountpoint_mkdir (glusterd_volinfo_t *volinfo,
-                     glusterd_brickinfo_t *src_brickinfo)
-{
-        char             mntpt[PATH_MAX]        = {0,};
-        int              ret                    = -1;
-
-        GLUSTERD_GET_RB_MNTPT (mntpt, sizeof (mntpt), volinfo);
-        ret = mkdir (mntpt, 0777);
-        if (ret && (errno != EEXIST)) {
-                gf_log ("", GF_LOG_DEBUG, "mkdir failed, due to %s",
-                        strerror (errno));
-                goto out;
-        }
-
-        ret = 0;
-
-out:
-        return ret;
-}
-
-static int
-rb_destroy_maintenance_client (glusterd_volinfo_t *volinfo,
-                               glusterd_brickinfo_t *src_brickinfo)
+int
+glusterd_op_stage_replace_brick(dict_t *dict, char **op_errstr,
+                                dict_t *rsp_dict)
 {
-        xlator_t         *this                        = NULL;
-        glusterd_conf_t  *priv                        = NULL;
-        char              volfile[PATH_MAX]           = {0,};
-        int               ret                         = -1;
-        int               mntfd                       = -1;
-        char              mntpt[PATH_MAX]             = {0,};
-
-        this = THIS;
-        priv = this->private;
-
-        ret = rb_get_mntfd (&mntfd);
+    int ret = 0;
+    char *src_brick = NULL;
+    char *dst_brick = NULL;
+    char *volname = NULL;
+    char *op = NULL;
+    glusterd_op_t gd_op = -1;
+    glusterd_volinfo_t *volinfo = NULL;
+    glusterd_brickinfo_t *src_brickinfo = NULL;
+    char *host = NULL;
+    char msg[2048] = {0};
+    glusterd_peerinfo_t *peerinfo = NULL;
+    glusterd_brickinfo_t *dst_brickinfo = NULL;
+    glusterd_conf_t *priv = NULL;
+    char pidfile[PATH_MAX] = {0};
+    xlator_t *this = NULL;
+    gf_boolean_t is_force = _gf_false;
+    char *dup_dstbrick = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+
+    priv = this->private;
+    GF_ASSERT(priv);
+
+    ret = glusterd_brick_op_prerequisites(dict, &op, &gd_op, &volname, &volinfo,
+                                          &src_brick, &src_brickinfo, pidfile,
+                                          op_errstr, rsp_dict);
+    if (ret)
+        goto out;
+
+    if (volinfo->type == GF_CLUSTER_TYPE_NONE) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_OP_NOT_PERMITTED,
+               "replace-brick is not permitted on distribute only "
+               "volumes");
+        gf_asprintf(op_errstr,
+                    "replace-brick is not permitted on "
+                    "distribute only volumes. Please use add-brick "
+                    "and remove-brick operations instead.");
+        ret = -1;
+        goto out;
+    }
+    ret = glusterd_validate_quorum(this, gd_op, dict, op_errstr);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_SERVER_QUORUM_NOT_MET,
+               "Server quorum not met. Rejecting operation.");
+        goto out;
+    }
+
+    if (strcmp(op, "GF_REPLACE_OP_COMMIT_FORCE")) {
+        ret = -1;
+        goto out;
+    } else {
+        is_force = _gf_true;
+    }
+
+    if (volinfo->snap_count > 0 || !cds_list_empty(&volinfo->snap_volumes)) {
+        snprintf(msg, sizeof(msg),
+                 "Volume %s  has %" PRIu64
+                 " snapshots. "
+                 "Changing the volume configuration will not effect snapshots."
+                 "But the snapshot brick mount should be intact to "
+                 "make them function.",
+                 volname, volinfo->snap_count);
+        gf_msg("glusterd", GF_LOG_WARNING, 0, GD_MSG_SNAP_WARN, "%s", msg);
+        msg[0] = '\0';
+    }
+
+    glusterd_add_peers_to_auth_list(volname);
+
+    ret = glusterd_get_dst_brick_info(&dst_brick, volname, op_errstr,
+                                      &dst_brickinfo, &host, dict,
+                                      &dup_dstbrick);
+    if (ret)
+        goto out;
+
+    ret = glusterd_new_brick_validate(dst_brick, dst_brickinfo, msg,
+                                      sizeof(msg), op);
+    /* fail if brick being replaced with itself */
+    if (ret) {
+        *op_errstr = gf_strdup(msg);
+        ret = -1;
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_BRICK_VALIDATE_FAIL, "%s",
+               *op_errstr);
+        goto out;
+    }
+
+    volinfo->rep_brick.src_brick = src_brickinfo;
+    volinfo->rep_brick.dst_brick = dst_brickinfo;
+
+    if (glusterd_rb_check_bricks(volinfo, src_brickinfo, dst_brickinfo)) {
+        ret = -1;
+        *op_errstr = gf_strdup(
+            "Incorrect source or "
+            "destination brick");
+        if (*op_errstr)
+            gf_msg(this->name, GF_LOG_ERROR, EINVAL, GD_MSG_BRICK_NOT_FOUND,
+                   "%s", *op_errstr);
+        goto out;
+    }
+
+    if (gf_is_local_addr(host)) {
+        ret = glusterd_validate_and_create_brickpath(
+            dst_brickinfo, volinfo->volume_id, volinfo->volname, op_errstr,
+            is_force, _gf_false);
         if (ret)
-                goto out;
-
-        ret = close (mntfd);
-        if (ret) {
-                gf_log (this->name, GF_LOG_DEBUG, "Failed to close mount "
-                        "point directory");
-                goto out;
-        }
-
-        GLUSTERD_GET_RB_MNTPT (mntpt, sizeof (mntpt), volinfo);
-        ret = gf_umount_lazy (this->name, mntpt, 1);
-        if (ret) {
-                gf_log (this->name, GF_LOG_WARNING,
-                        "Lazy unmount failed on maintenance client");
-        } else {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "Successfully unmounted  maintenance client");
-        }
-
-        snprintf (volfile, PATH_MAX, "%s/vols/%s/%s", priv->workdir,
-                  volinfo->volname, RB_CLIENTVOL_FILENAME);
-
-        ret = unlink (volfile);
-        if (ret) {
-                gf_log ("", GF_LOG_DEBUG, "unlink of %s failed, reason: %s",
-                        volfile, strerror (errno));
-                goto out;
-        }
-
-        ret = 0;
-
-out:
-        return ret;
-}
-
-static int
-rb_spawn_maintenance_client (glusterd_volinfo_t *volinfo,
-                             glusterd_brickinfo_t *src_brickinfo)
-{
-        int ret = -1;
-
-        ret = rb_generate_client_volfile (volinfo, src_brickinfo);
-        if (ret) {
-                gf_log ("", GF_LOG_DEBUG, "Unable to generate client "
-                        "volfile");
-                goto out;
-        }
+            goto out;
+    }
+
+    if (!gf_is_local_addr(host)) {
+        RCU_READ_LOCK;
+
+        peerinfo = glusterd_peerinfo_find(NULL, host);
+        if (peerinfo == NULL) {
+            RCU_READ_UNLOCK;
+            ret = -1;
+            snprintf(msg, sizeof(msg), "%s, is not a friend", host);
+            *op_errstr = gf_strdup(msg);
+            goto out;
+
+        } else if (!peerinfo->connected) {
+            RCU_READ_UNLOCK;
+            ret = -1;
+            snprintf(msg, sizeof(msg),
+                     "%s, is not connected at "
+                     "the moment",
+                     host);
+            *op_errstr = gf_strdup(msg);
+            goto out;
+
+        } else if (GD_FRIEND_STATE_BEFRIENDED != peerinfo->state.state) {
+            RCU_READ_UNLOCK;
+            ret = -1;
+            snprintf(msg, sizeof(msg),
+                     "%s, is not befriended "
+                     "at the moment",
+                     host);
+            *op_errstr = gf_strdup(msg);
+            goto out;
+        }
+        RCU_READ_UNLOCK;
+
+    } else if (priv->op_version >= GD_OP_VERSION_3_6_0) {
+        /* A bricks mount dir is required only by snapshots which were
+         * introduced in gluster-3.6.0
+         */
 
-        ret = rb_mountpoint_mkdir (volinfo, src_brickinfo);
-        if (ret) {
-                gf_log ("", GF_LOG_DEBUG, "Unable to mkdir "
-                        "mountpoint");
+        if (!(gf_uuid_compare(dst_brickinfo->uuid, MY_UUID))) {
+            ret = glusterd_get_brick_mount_dir(dst_brickinfo->path,
+                                               dst_brickinfo->hostname,
+                                               dst_brickinfo->mount_dir);
+            if (ret) {
+                gf_msg(this->name, GF_LOG_ERROR, 0,
+                       GD_MSG_BRICK_MOUNTDIR_GET_FAIL,
+                       "Failed to get brick mount_dir");
                 goto out;
-        }
-
-        ret = rb_spawn_glusterfs_client (volinfo, src_brickinfo);
-        if (ret) {
-                gf_log ("", GF_LOG_DEBUG, "Unable to start glusterfs");
+            }
+            ret = dict_set_dynstr_with_alloc(rsp_dict, "brick1.mount_dir",
+                                             dst_brickinfo->mount_dir);
+            if (ret) {
+                gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+                       "Failed to set brick.mount_dir");
                 goto out;
+            }
         }
 
-        ret = 0;
-out:
-        return ret;
-}
-
-static int
-rb_spawn_destination_brick (glusterd_volinfo_t *volinfo,
-                            glusterd_brickinfo_t *dst_brickinfo)
-
-{
-        int ret = -1;
-
-        ret = rb_generate_dst_brick_volfile (volinfo, dst_brickinfo);
+        ret = dict_set_int32n(rsp_dict, "brick_count", SLEN("brick_count"), 1);
         if (ret) {
-                gf_log ("", GF_LOG_DEBUG, "Unable to generate client "
-                        "volfile");
-                goto out;
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+                   "Failed to set local_brick_count");
+            goto out;
         }
+    }
 
-        ret = rb_spawn_dst_brick (volinfo, dst_brickinfo);
-        if (ret) {
-                gf_log ("", GF_LOG_DEBUG, "Unable to start glusterfs");
-                goto out;
-        }
+    ret = 0;
 
-        ret = 0;
 out:
-        return ret;
-}
-
-static int
-rb_kill_destination_brick (glusterd_volinfo_t *volinfo,
-                           glusterd_brickinfo_t *dst_brickinfo)
-{
-        glusterd_conf_t  *priv               = NULL;
-        char              pidfile[PATH_MAX]  = {0,};
-
-        priv = THIS->private;
+    GF_FREE(dup_dstbrick);
+    gf_msg_debug(this->name, 0, "Returning %d", ret);
 
-        snprintf (pidfile, PATH_MAX, "%s/vols/%s/%s",
-                  priv->workdir, volinfo->volname,
-                  RB_DSTBRICK_PIDFILE);
-
-        return glusterd_service_stop ("brick", pidfile, SIGTERM, _gf_true);
+    return ret;
 }
 
-static int
-rb_get_xattr_command (glusterd_volinfo_t *volinfo,
-                      glusterd_brickinfo_t *src_brickinfo,
-                      glusterd_brickinfo_t *dst_brickinfo,
-                      const char *xattr_key,
-                      char *value)
+int
+glusterd_op_perform_replace_brick(glusterd_volinfo_t *volinfo, char *old_brick,
+                                  char *new_brick, dict_t *dict)
 {
-        int             ret    = -1;
-        int             mntfd  = -1;
-
-        ret = rb_get_mntfd (&mntfd);
+    char *brick_mount_dir = NULL;
+    glusterd_brickinfo_t *old_brickinfo = NULL;
+    glusterd_brickinfo_t *new_brickinfo = NULL;
+    int32_t ret = -1;
+    xlator_t *this = NULL;
+    glusterd_conf_t *conf = NULL;
+    struct statvfs brickstat = {
+        0,
+    };
+
+    this = THIS;
+    GF_ASSERT(this);
+    GF_ASSERT(dict);
+    GF_ASSERT(volinfo);
+
+    conf = this->private;
+    GF_ASSERT(conf);
+
+    ret = glusterd_brickinfo_new_from_brick(new_brick, &new_brickinfo, _gf_true,
+                                            NULL);
+    if (ret)
+        goto out;
+
+    ret = glusterd_resolve_brick(new_brickinfo);
+    if (ret)
+        goto out;
+
+    if (!gf_uuid_compare(new_brickinfo->uuid, MY_UUID)) {
+        ret = sys_statvfs(new_brickinfo->path, &brickstat);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, errno, GD_MSG_STATVFS_FAILED,
+                   "Failed to fetch disk utilization "
+                   "from the brick (%s:%s). Please check the health of "
+                   "the brick. Error code was %s",
+                   new_brickinfo->hostname, new_brickinfo->path,
+                   strerror(errno));
+
+            goto out;
+        }
+        new_brickinfo->statfs_fsid = brickstat.f_fsid;
+    }
+
+    ret = glusterd_volume_brickinfo_get_by_brick(old_brick, volinfo,
+                                                 &old_brickinfo, _gf_false);
+    if (ret)
+        goto out;
+
+    (void)snprintf(new_brickinfo->brick_id, sizeof(new_brickinfo->brick_id),
+                   "%s", old_brickinfo->brick_id);
+    new_brickinfo->port = old_brickinfo->port;
+
+    /* A bricks mount dir is required only by snapshots which were
+     * introduced in gluster-3.6.0
+     */
+    if (conf->op_version >= GD_OP_VERSION_3_6_0) {
+        ret = dict_get_strn(dict, "brick1.mount_dir", SLEN("brick1.mount_dir"),
+                            &brick_mount_dir);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, errno,
+                   GD_MSG_BRICK_MOUNTDIR_GET_FAIL,
+                   "brick1.mount_dir not present");
+            goto out;
+        }
+        (void)snprintf(new_brickinfo->mount_dir,
+                       sizeof(new_brickinfo->mount_dir), "%s", brick_mount_dir);
+    }
+
+    cds_list_add(&new_brickinfo->brick_list, &old_brickinfo->brick_list);
+
+    volinfo->brick_count++;
+
+    ret = glusterd_op_perform_remove_brick(volinfo, old_brick, 1, NULL);
+    if (ret)
+        goto out;
+
+    /* if the volume is a replicate volume, do: */
+    if (glusterd_is_volume_replicate(volinfo)) {
+        if (!gf_uuid_compare(new_brickinfo->uuid, MY_UUID)) {
+            ret = glusterd_handle_replicate_brick_ops(volinfo, new_brickinfo,
+                                                      GD_OP_REPLACE_BRICK);
+            if (ret < 0)
+                goto out;
+        }
+    }
+
+    ret = glusterd_create_volfiles_and_notify_services(volinfo);
+    if (ret)
+        goto out;
+
+    if (GLUSTERD_STATUS_STARTED == volinfo->status) {
+        ret = glusterd_brick_start(volinfo, new_brickinfo, _gf_false,
+                                   _gf_false);
         if (ret)
-                goto out;
-
-        ret = sys_fgetxattr (mntfd, xattr_key, value, 8192);
-
-        if (ret < 0) {
-                gf_log (THIS->name, GF_LOG_DEBUG, "getxattr on key: %s "
-                        "failed, reason: %s", xattr_key, strerror (errno));
-                goto out;
-        }
+            goto out;
+    }
 
-        ret = 0;
-out:
-        return ret;
-}
-
-static int
-rb_send_cmd (glusterd_volinfo_t *volinfo,
-             glusterd_brickinfo_t *src,
-             glusterd_brickinfo_t *dst,
-             gf1_cli_replace_op op)
-{
-        char         start_value[8192]          = {0,};
-        char         status_str[8192]           = {0,};
-        char        *status_reply               = NULL;
-        char        *tmp                        = NULL;
-        char        *save_ptr                   = NULL;
-        char         filename[PATH_MAX]         = {0,};
-        char        *current_file               = NULL;
-        uint64_t     files                      = 0;
-        int          status                     = 0;
-        dict_t      *ctx                        = NULL;
-        int          ret                        = 0;
-
-        GF_ASSERT (volinfo);
-        GF_ASSERT (src);
-        GF_ASSERT (dst);
-        GF_ASSERT ((op > GF_REPLACE_OP_NONE)
-                   && (op <= GF_REPLACE_OP_COMMIT_FORCE));
-
-        switch (op) {
-                case GF_REPLACE_OP_START:
-                {
-                        snprintf (start_value, sizeof (start_value),
-                                  "%s:%s:%d", dst->hostname, dst->path,
-                                  dst->port);
-                        ret = rb_send_xattr_command (volinfo, src, dst,
-                                                     RB_PUMP_CMD_START,
-                                                     start_value);
-                }
-                break;
-                case GF_REPLACE_OP_PAUSE:
-                {
-                        ret = rb_send_xattr_command (volinfo, src, dst,
-                                                     RB_PUMP_CMD_PAUSE,
-                                                     RB_PUMP_DEF_ARG);
-                }
-                break;
-                case GF_REPLACE_OP_ABORT:
-                {
-                        ret = rb_send_xattr_command (volinfo, src, dst,
-                                                     RB_PUMP_CMD_ABORT,
-                                                     RB_PUMP_DEF_ARG);
-                }
-                break;
-                case GF_REPLACE_OP_COMMIT:
-                {
-                        ret = rb_send_xattr_command (volinfo, src, dst,
-                                                     RB_PUMP_CMD_COMMIT,
-                                                     RB_PUMP_DEF_ARG);
-                }
-                break;
-                case GF_REPLACE_OP_STATUS:
-                {
-                        ret = rb_get_xattr_command (volinfo, src, dst,
-                                                    RB_PUMP_CMD_STATUS,
-                                                    status_str);
-                        if (ret)
-                                goto out;
-
-                        ctx = glusterd_op_get_ctx ();
-                        GF_ASSERT (ctx);
-                        if (!ctx) {
-                                ret = -1;
-                                gf_log (THIS->name, GF_LOG_CRITICAL,
-                                        "ctx is not present.");
-                                goto out;
-                        }
-
-                        /* Split status reply into different parts */
-                        tmp = strtok_r (status_str, ":", &save_ptr);
-                        if (!tmp) {
-                                ret = -1;
-                                gf_log (THIS->name, GF_LOG_ERROR,
-                                        "Couldn't tokenize status string");
-                                goto out;
-                        }
-                        sscanf (tmp, "status=%d", &status);
-                        ret = dict_set_int32 (ctx, "status", status);
-                        if (ret) {
-                                gf_log (THIS->name, GF_LOG_ERROR, "Couldn't "
-                                        "set rb status in context");
-                                goto out;
-                        }
-
-                        tmp = NULL;
-                        tmp = strtok_r (NULL, ":", &save_ptr);
-                        if (!tmp) {
-                                ret = -1;
-                                gf_log (THIS->name, GF_LOG_ERROR,
-                                        "Couldn't tokenize status string");
-                                goto out;
-                        }
-                        sscanf (tmp, "no_of_files=%"SCNu64, &files);
-                        ret = dict_set_uint64 (ctx, "files", files);
-                        if (ret) {
-                                gf_log (THIS->name, GF_LOG_ERROR, "Couldn't "
-                                        "set rb files in context");
-                                goto out;
-                        }
-
-                        if (status == 0) {
-                                tmp = NULL;
-                                tmp = strtok_r (NULL, ":", &save_ptr);
-                                if (!tmp) {
-                                        ret = -1;
-                                        gf_log (THIS->name, GF_LOG_ERROR,
-                                                "Couldn't tokenize status "
-                                                "string");
-                                        goto out;
-                                }
-                                sscanf (tmp, "current_file=%s", filename);
-                                current_file = gf_strdup (filename);
-                                ret = dict_set_dynstr (ctx, "current_file",
-                                                       current_file);
-                                if (ret) {
-                                        GF_FREE (current_file);
-                                        gf_log (THIS->name, GF_LOG_ERROR,
-                                                "Couldn't set rb current file "
-                                                "in context");
-                                        goto out;
-                                }
-                        }
-                        if (status) {
-                                ret = gf_asprintf (&status_reply,
-                                                  "Number of files migrated = %"
-                                                  PRIu64"\tMigration complete",
-                                                  files);
-                        } else {
-                                ret = gf_asprintf (&status_reply,
-                                                  "Number of files migrated = %"
-                                                  PRIu64"\tCurrent file = %s",
-                                                  files, filename);
-                        }
-                        if (ret == -1) {
-                                gf_log (THIS->name, GF_LOG_ERROR,
-                                        "Failed to create status_reply string");
-                                goto out;
-                        }
-                        ret = dict_set_dynstr (ctx, "status-reply",
-                                               status_reply);
-                        if (ret) {
-                                GF_FREE (status_reply);
-                                gf_log (THIS->name, GF_LOG_ERROR, "Couldn't "
-                                        "set rb status response in context.");
-                                goto out;
-                        }
-                }
-                break;
-                default:
-                {
-                        GF_ASSERT (0);
-                        ret = -1;
-                        gf_log (THIS->name, GF_LOG_CRITICAL, "Invalid replace"
-                                " brick subcommand.");
-                }
-                break;
-        }
 out:
-        return ret;
-}
-
-static int
-rb_do_operation (glusterd_volinfo_t *volinfo,
-                 glusterd_brickinfo_t *src_brickinfo,
-                 glusterd_brickinfo_t *dst_brickinfo,
-                 gf1_cli_replace_op op)
-{
-
-        int             ret             = -1;
-        char            op_str[256]     = {0, };
-        xlator_t        *this           = NULL;
-
-        this = THIS;
-
-        ret = rb_spawn_maintenance_client (volinfo, src_brickinfo);
-        if (ret) {
-                gf_log (this->name, GF_LOG_DEBUG, "Could not spawn "
-                        "maintenance client");
-                goto umount;
-        }
 
-        ret = rb_send_cmd (volinfo, src_brickinfo, dst_brickinfo, op);
-        if (ret) {
-                (void) glusterd_get_replace_op_str (op, op_str);
-                gf_log (this->name, GF_LOG_DEBUG, "Sending replace-brick "
-                        "sub-command %s failed.", op_str);
-        }
-
-umount:
-        if (rb_destroy_maintenance_client (volinfo, src_brickinfo))
-                gf_log (this->name, GF_LOG_DEBUG, "Failed to destroy "
-                        "maintenance client");
-
-         return ret;
+    gf_msg_debug("glusterd", 0, "Returning %d", ret);
+    return ret;
 }
 
-/* Set src-brick's port number to be used in the maintenance mount
- * after all commit acks are received.
- */
-static int
-rb_update_srcbrick_port (glusterd_volinfo_t *volinfo,
-                         glusterd_brickinfo_t *src_brickinfo,
-                         dict_t *rsp_dict, dict_t *req_dict, int32_t replace_op)
+int
+glusterd_op_replace_brick(dict_t *dict, dict_t *rsp_dict)
 {
-        xlator_t *this                  = NULL;
-        dict_t   *ctx                   = NULL;
-        int       ret                   = 0;
-        int       dict_ret              = 0;
-        int       src_port              = 0;
-        char      brickname[PATH_MAX]   = {0,};
-
-        this = THIS;
-
-        dict_ret = dict_get_int32 (req_dict, "src-brick-port", &src_port);
-        if (src_port)
-                src_brickinfo->port = src_port;
-
-        if (gf_is_local_addr (src_brickinfo->hostname)) {
-                gf_log ("", GF_LOG_INFO,
-                        "adding src-brick port no");
-
-                if (volinfo->transport_type == GF_TRANSPORT_RDMA) {
-                        snprintf (brickname, sizeof(brickname), "%s.rdma",
-                                  src_brickinfo->path);
-                } else
-                        snprintf (brickname, sizeof(brickname), "%s",
-                                  src_brickinfo->path);
-
-                src_brickinfo->port = pmap_registry_search (this,
-                                      brickname, GF_PMAP_PORT_BRICKSERVER);
-                if (!src_brickinfo->port &&
-                    replace_op != GF_REPLACE_OP_COMMIT_FORCE ) {
-                        gf_log ("", GF_LOG_ERROR,
-                                "Src brick port not available");
-                        ret = -1;
-                        goto out;
-                }
-
-                if (rsp_dict) {
-                        ret = dict_set_int32 (rsp_dict, "src-brick-port", src_brickinfo->port);
-                        if (ret) {
-                                gf_log ("", GF_LOG_DEBUG,
-                                        "Could not set src-brick port no");
-                                goto out;
-                        }
-                }
-
-                ctx = glusterd_op_get_ctx ();
-                if (ctx) {
-                        ret = dict_set_int32 (ctx, "src-brick-port", src_brickinfo->port);
-                        if (ret) {
-                                gf_log ("", GF_LOG_DEBUG,
-                                        "Could not set src-brick port no");
-                                goto out;
-                        }
-                }
-
-        }
+    int ret = 0;
+    char *replace_op = NULL;
+    glusterd_volinfo_t *volinfo = NULL;
+    char *volname = NULL;
+    xlator_t *this = NULL;
+    glusterd_conf_t *priv = NULL;
+    char *src_brick = NULL;
+    char *dst_brick = NULL;
+    glusterd_brickinfo_t *src_brickinfo = NULL;
+    glusterd_brickinfo_t *dst_brickinfo = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+
+    priv = this->private;
+    GF_ASSERT(priv);
+
+    ret = dict_get_strn(dict, "src-brick", SLEN("src-brick"), &src_brick);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+               "Unable to get src brick");
+        goto out;
+    }
+
+    gf_msg_debug(this->name, 0, "src brick=%s", src_brick);
+
+    ret = dict_get_strn(dict, "dst-brick", SLEN("dst-brick"), &dst_brick);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+               "Unable to get dst brick");
+        goto out;
+    }
+
+    gf_msg_debug(this->name, 0, "dst brick=%s", dst_brick);
+
+    ret = dict_get_strn(dict, "volname", SLEN("volname"), &volname);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+               "Unable to get volume name");
+        goto out;
+    }
+
+    ret = dict_get_strn(dict, "operation", SLEN("operation"), &replace_op);
+    if (ret) {
+        gf_msg_debug(this->name, 0, "dict_get on operation failed");
+        goto out;
+    }
+
+    ret = glusterd_volinfo_find(volname, &volinfo);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, ENOMEM, GD_MSG_NO_MEMORY,
+               "Unable to allocate memory");
+        goto out;
+    }
+
+    ret = glusterd_volume_brickinfo_get_by_brick(src_brick, volinfo,
+                                                 &src_brickinfo, _gf_false);
+    if (ret) {
+        gf_msg_debug(this->name, 0, "Unable to get src-brickinfo");
+        goto out;
+    }
+
+    ret = glusterd_get_rb_dst_brickinfo(volinfo, &dst_brickinfo);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_RB_BRICKINFO_GET_FAIL,
+               "Unable to get "
+               "replace brick destination brickinfo");
+        goto out;
+    }
+
+    ret = glusterd_resolve_brick(dst_brickinfo);
+    if (ret) {
+        gf_msg_debug(this->name, 0, "Unable to resolve dst-brickinfo");
+        goto out;
+    }
+
+    ret = rb_update_dstbrick_port(dst_brickinfo, rsp_dict, dict);
+    if (ret)
+        goto out;
+
+    if (strcmp(replace_op, "GF_REPLACE_OP_COMMIT_FORCE")) {
+        ret = -1;
+        goto out;
+    }
+
+    ret = glusterd_svcs_stop(volinfo);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_GLUSTER_SERVICES_STOP_FAIL,
+               "Unable to stop gluster services, ret: %d", ret);
+    }
+
+    ret = glusterd_op_perform_replace_brick(volinfo, src_brick, dst_brick,
+                                            dict);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_CRITICAL, 0, GD_MSG_BRICK_ADD_FAIL,
+               "Unable to add dst-brick: "
+               "%s to volume: %s",
+               dst_brick, volinfo->volname);
+        (void)glusterd_svcs_manager(volinfo);
+        goto out;
+    }
+
+    volinfo->rebal.defrag_status = 0;
+
+    ret = glusterd_svcs_manager(volinfo);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_CRITICAL, 0,
+               GD_MSG_GLUSTER_SERVICE_START_FAIL,
+               "Failed to start one or more gluster services.");
+    }
+
+    ret = glusterd_fetchspec_notify(THIS);
+    glusterd_brickinfo_delete(volinfo->rep_brick.dst_brick);
+    volinfo->rep_brick.src_brick = NULL;
+    volinfo->rep_brick.dst_brick = NULL;
+
+    if (!ret)
+        ret = glusterd_store_volinfo(volinfo,
+                                     GLUSTERD_VOLINFO_VER_AC_INCREMENT);
+    if (ret)
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_RBOP_STATE_STORE_FAIL,
+               "Couldn't store"
+               " replace brick operation's state");
 
 out:
-        return ret;
-
-}
-
-static int
-rb_update_dstbrick_port (glusterd_brickinfo_t *dst_brickinfo, dict_t *rsp_dict,
-                         dict_t *req_dict, int32_t replace_op)
-{
-        dict_t *ctx           = NULL;
-        int     ret           = 0;
-        int     dict_ret      = 0;
-        int     dst_port      = 0;
-
-        dict_ret = dict_get_int32 (req_dict, "dst-brick-port", &dst_port);
-        if (!dict_ret)
-                dst_brickinfo->port = dst_port;
-
-
-        if (gf_is_local_addr (dst_brickinfo->hostname)) {
-                gf_log ("", GF_LOG_INFO,
-                        "adding dst-brick port no");
-
-                if (rsp_dict) {
-                        ret = dict_set_int32 (rsp_dict, "dst-brick-port",
-                                              dst_brickinfo->port);
-                        if (ret) {
-                                gf_log ("", GF_LOG_DEBUG,
-                                        "Could not set dst-brick port no in rsp dict");
-                                goto out;
-                        }
-                }
-
-                ctx = glusterd_op_get_ctx ();
-                if (ctx) {
-                        ret = dict_set_int32 (ctx, "dst-brick-port",
-                                              dst_brickinfo->port);
-                        if (ret) {
-                                gf_log ("", GF_LOG_DEBUG,
-                                        "Could not set dst-brick port no");
-                                goto out;
-                        }
-                }
-        }
-out:
-        return ret;
-}
-
-static int
-glusterd_op_perform_replace_brick (glusterd_volinfo_t  *volinfo,
-                                   char *old_brick, char *new_brick,
-                                   dict_t *dict)
-{
-        char                                    *brick_mount_dir = NULL;
-        glusterd_brickinfo_t                    *old_brickinfo = NULL;
-        glusterd_brickinfo_t                    *new_brickinfo = NULL;
-        int32_t                                 ret = -1;
-        xlator_t                                *this = NULL;
-        glusterd_conf_t                         *conf = NULL;
-
-        this = THIS;
-        GF_ASSERT (this);
-        GF_ASSERT (dict);
-        GF_ASSERT (volinfo);
-
-        conf = this->private;
-        GF_ASSERT (conf);
-
-        ret = glusterd_brickinfo_new_from_brick (new_brick,
-                                                 &new_brickinfo);
-        if (ret)
-                goto out;
-
-        ret = glusterd_resolve_brick (new_brickinfo);
-
-        if (ret)
-                goto out;
-
-        ret = glusterd_volume_brickinfo_get_by_brick (old_brick,
-                                                      volinfo, &old_brickinfo);
-        if (ret)
-                goto out;
-
-        strncpy (new_brickinfo->brick_id, old_brickinfo->brick_id,
-                 sizeof (new_brickinfo->brick_id));
-
-        /* A bricks mount dir is required only by snapshots which were
-         * introduced in gluster-3.6.0
-         */
-        if (conf->op_version >= GD_OP_VERSION_3_6_0) {
-                ret = dict_get_str (dict, "brick1.mount_dir", &brick_mount_dir);
-                if (ret) {
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "brick1.mount_dir not present");
-                        goto out;
-                }
-                strncpy (new_brickinfo->mount_dir, brick_mount_dir,
-                         sizeof(new_brickinfo->mount_dir));
-        }
-
-        cds_list_add_tail (&new_brickinfo->brick_list,
-                           &old_brickinfo->brick_list);
-
-        volinfo->brick_count++;
-
-        ret = glusterd_op_perform_remove_brick (volinfo, old_brick, 1, NULL);
-        if (ret)
-                goto out;
-
-        ret = glusterd_create_volfiles_and_notify_services (volinfo);
-        if (ret)
-                goto out;
-
-        if (GLUSTERD_STATUS_STARTED == volinfo->status) {
-                ret = glusterd_brick_start (volinfo, new_brickinfo, _gf_false);
-                if (ret)
-                        goto out;
-        }
-
-out:
-
-        gf_log ("", GF_LOG_DEBUG, "Returning %d", ret);
-        return ret;
+    return ret;
 }
 
 int
-glusterd_op_replace_brick (dict_t *dict, dict_t *rsp_dict)
+glusterd_mgmt_v3_initiate_replace_brick_cmd_phases(rpcsvc_request_t *req,
+                                                   glusterd_op_t op,
+                                                   dict_t *dict)
 {
-        int                                      ret = 0;
-        dict_t                                  *ctx  = NULL;
-        int                                      replace_op = 0;
-        glusterd_volinfo_t                      *volinfo = NULL;
-        char                                    *volname = NULL;
-        xlator_t                                *this = NULL;
-        glusterd_conf_t                         *priv = NULL;
-        char                                    *src_brick = NULL;
-        char                                    *dst_brick = NULL;
-        glusterd_brickinfo_t                    *src_brickinfo = NULL;
-        glusterd_brickinfo_t                    *dst_brickinfo = NULL;
-        char                                    *task_id_str = NULL;
-
-        this = THIS;
-        GF_ASSERT (this);
-
-        priv = this->private;
-        GF_ASSERT (priv);
-
-        ret = dict_get_str (dict, "src-brick", &src_brick);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "Unable to get src brick");
-                goto out;
-        }
-
-        gf_log (this->name, GF_LOG_DEBUG, "src brick=%s", src_brick);
-
-        ret = dict_get_str (dict, "dst-brick", &dst_brick);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "Unable to get dst brick");
-                goto out;
-        }
-
-        gf_log (this->name, GF_LOG_DEBUG, "dst brick=%s", dst_brick);
-
-        ret = dict_get_str (dict, "volname", &volname);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "Unable to get volume name");
-                goto out;
-        }
-
-        ret = dict_get_int32 (dict, "operation", (int32_t *)&replace_op);
-        if (ret) {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "dict_get on operation failed");
-                goto out;
-        }
-
-        ret = glusterd_volinfo_find (volname, &volinfo);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "Unable to allocate memory");
-                goto out;
-        }
-
-        ret = glusterd_volume_brickinfo_get_by_brick (src_brick, volinfo,
-                                                      &src_brickinfo);
-        if (ret) {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "Unable to get src-brickinfo");
-                goto out;
-        }
-
-
-        ret = glusterd_get_rb_dst_brickinfo (volinfo, &dst_brickinfo);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "Unable to get "
-                         "replace brick destination brickinfo");
-                goto out;
-        }
-
-        ret = glusterd_resolve_brick (dst_brickinfo);
-        if (ret) {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "Unable to resolve dst-brickinfo");
-                goto out;
-        }
-
-        ret = rb_update_srcbrick_port (volinfo, src_brickinfo, rsp_dict,
-                                       dict, replace_op);
-        if (ret)
-                goto out;
-
-	if ((GF_REPLACE_OP_START != replace_op)) {
-
-                /* Set task-id, if available, in op_ctx dict for operations
-                 * other than start
-                 */
-                if  (is_origin_glusterd (dict)) {
-                        ctx = glusterd_op_get_ctx();
-                        if (!ctx) {
-                                gf_log (this->name, GF_LOG_ERROR, "Failed to "
-                                        "get op_ctx");
-                                ret = -1;
-                                goto out;
-                        }
-                        if (!gf_uuid_is_null (volinfo->rep_brick.rb_id)) {
-                                ret = glusterd_copy_uuid_to_dict
-                                        (volinfo->rep_brick.rb_id, ctx,
-                                         GF_REPLACE_BRICK_TID_KEY);
-                                if (ret) {
-                                        gf_log (this->name, GF_LOG_ERROR,
-                                                "Failed to set "
-                                                "replace-brick-id");
-                                        goto out;
-                                }
-                        }
-                }
-	}
-        ret = rb_update_dstbrick_port (dst_brickinfo, rsp_dict,
-                                       dict, replace_op);
-        if (ret)
-                goto out;
-
-        switch (replace_op) {
-        case GF_REPLACE_OP_START:
-        {
-                ret = dict_get_str (dict, GF_REPLACE_BRICK_TID_KEY, &task_id_str);
-                if (ret) {
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "Missing replace-brick-id");
-                        ret = 0;
-                } else {
-                        gf_uuid_parse (task_id_str, volinfo->rep_brick.rb_id);
-                }
-
-                if (gf_is_local_addr (dst_brickinfo->hostname)) {
-                        gf_log (this->name, GF_LOG_INFO,
-                                "I AM THE DESTINATION HOST");
-                        if (!glusterd_is_rb_paused (volinfo)) {
-                                ret = rb_spawn_destination_brick
-                                        (volinfo, dst_brickinfo);
-                                if (ret) {
-                                        gf_log (this->name, GF_LOG_DEBUG,
-                                                "Failed to spawn destination "
-                                                "brick");
-                                        goto out;
-                                }
-                        } else {
-                                gf_log (this->name, GF_LOG_ERROR,
-                                        "Replace brick is already started=> no "
-                                        "need to restart dst brick ");
-                        }
-		}
-
-
-		if (gf_is_local_addr (src_brickinfo->hostname)) {
-		        ret = rb_src_brick_restart (volinfo, src_brickinfo,
-				                    1);
-		        if (ret) {
-				gf_log (this->name, GF_LOG_DEBUG,
-	                                "Could not restart src-brick");
-			        goto out;
-			}
-		}
-
-		if (gf_is_local_addr (dst_brickinfo->hostname)) {
-			gf_log (this->name, GF_LOG_INFO,
-				"adding dst-brick port no");
-
-                        ret = rb_update_dstbrick_port (dst_brickinfo, rsp_dict,
-                                                       dict, replace_op);
-                        if (ret)
-                                goto out;
-                }
-
-                glusterd_set_rb_status (volinfo, GF_RB_STATUS_STARTED);
-		break;
-	}
-
-        case GF_REPLACE_OP_COMMIT:
-        {
-                ctx = glusterd_op_get_ctx ();
-                if (ctx) {
-                        ret = rb_do_operation (volinfo, src_brickinfo,
-                                               dst_brickinfo,
-                                               GF_REPLACE_OP_COMMIT);
-                        if (ret) {
-                                gf_log (this->name, GF_LOG_ERROR,
-                                        "Commit operation failed");
-                                goto out;
-                        }
-                }
-        }
-                /* fall through */
-        case GF_REPLACE_OP_COMMIT_FORCE:
-        {
-                if (gf_is_local_addr (dst_brickinfo->hostname)) {
-                        gf_log (this->name, GF_LOG_DEBUG,
-                                "I AM THE DESTINATION HOST");
-                        ret = rb_kill_destination_brick (volinfo,
-                                                         dst_brickinfo);
-                        if (ret) {
-                                gf_log (this->name, GF_LOG_CRITICAL,
-                                        "Unable to cleanup dst brick");
-                                goto out;
-                        }
-                }
-
-                ret = glusterd_svcs_stop (volinfo);
-                if (ret) {
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "Unable to stop nfs server, ret: %d", ret);
-                }
-
-		ret = glusterd_op_perform_replace_brick (volinfo, src_brick,
-							 dst_brick, dict);
-		if (ret) {
-			gf_log (this->name, GF_LOG_CRITICAL, "Unable to add "
-				"dst-brick: %s to volume: %s", dst_brick,
-                                volinfo->volname);
-		        (void) glusterd_svcs_manager (volinfo);
-			goto out;
-		}
-
-		volinfo->rebal.defrag_status = 0;
-
-		ret = glusterd_svcs_manager (volinfo);
-		if (ret) {
-                        gf_log (this->name, GF_LOG_CRITICAL,
-                                "Failed to generate nfs volume file");
-		}
-
-
-		ret = glusterd_fetchspec_notify (THIS);
-                glusterd_set_rb_status (volinfo, GF_RB_STATUS_NONE);
-                glusterd_brickinfo_delete (volinfo->rep_brick.dst_brick);
-                volinfo->rep_brick.src_brick = NULL;
-                volinfo->rep_brick.dst_brick = NULL;
-                gf_uuid_clear (volinfo->rep_brick.rb_id);
-        }
-        break;
-
-        case GF_REPLACE_OP_PAUSE:
-        {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "Received pause - doing nothing");
-                ctx = glusterd_op_get_ctx ();
-                if (ctx) {
-                        ret = rb_do_operation (volinfo, src_brickinfo,
-                                               dst_brickinfo,
-                                               GF_REPLACE_OP_PAUSE);
-                        if (ret) {
-                                gf_log (this->name, GF_LOG_ERROR,
-                                        "Pause operation failed");
-                                goto out;
-                        }
-                }
-
-                glusterd_set_rb_status (volinfo, GF_RB_STATUS_PAUSED);
-        }
-                break;
-
-        case GF_REPLACE_OP_ABORT:
-        {
-
-                ctx = glusterd_op_get_ctx ();
-                if (ctx) {
-                        ret = rb_do_operation (volinfo, src_brickinfo,
-                                               dst_brickinfo,
-                                               GF_REPLACE_OP_ABORT);
-                        if (ret) {
-                                gf_log (this->name, GF_LOG_ERROR,
-                                        "Abort operation failed");
-                                goto out;
-                        }
-                }
-
-                if (gf_is_local_addr (src_brickinfo->hostname)) {
-                        ret = rb_src_brick_restart (volinfo, src_brickinfo,
-                                                    0);
-                        if (ret) {
-                                gf_log (this->name, GF_LOG_ERROR,
-                                        "Couldn't restart src brick "
-                                        "with pump xlator disabled.");
-                                goto out;
-                        }
-                }
-
-                if (gf_is_local_addr (dst_brickinfo->hostname)) {
-                        gf_log (this->name, GF_LOG_INFO,
-                                "I AM THE DESTINATION HOST");
-                        ret = rb_kill_destination_brick (volinfo, dst_brickinfo);
-                        if (ret) {
-                                gf_log ("", GF_LOG_DEBUG,
-                                        "Failed to kill destination brick");
-                                goto out;
-                        }
-                }
-                glusterd_set_rb_status (volinfo, GF_RB_STATUS_NONE);
-                glusterd_brickinfo_delete (volinfo->rep_brick.dst_brick);
-                volinfo->rep_brick.src_brick = NULL;
-                volinfo->rep_brick.dst_brick = NULL;
-        }
-        break;
-
-        case GF_REPLACE_OP_STATUS:
-        {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "received status - doing nothing");
-                ctx = glusterd_op_get_ctx ();
-                if (ctx) {
-                        if (glusterd_is_rb_paused (volinfo)) {
-                                ret = dict_set_str (ctx, "status-reply",
-                                                 "replace brick has been paused");
-                                if (ret)
-                                        gf_log (this->name, GF_LOG_ERROR,
-                                                "failed to set pump status"
-                                                " in ctx");
-                                goto out;
-                        }
-
-                        ret = rb_do_operation (volinfo, src_brickinfo,
-                                               dst_brickinfo,
-                                               GF_REPLACE_OP_STATUS);
-                        if (ret)
-                                goto out;
-                }
-
-        }
-                break;
-
-        default:
-                ret = -1;
-                goto out;
-        }
-        if (!ret && replace_op != GF_REPLACE_OP_STATUS)
-		ret = glusterd_store_volinfo (volinfo,
-                                              GLUSTERD_VOLINFO_VER_AC_INCREMENT);
-        if (ret)
-                gf_log (this->name, GF_LOG_ERROR, "Couldn't store"
-                        " replace brick operation's state");
+    int32_t ret = -1;
+    int32_t op_ret = -1;
+    uint32_t txn_generation = 0;
+    uint32_t op_errno = 0;
+    char *op_errstr = NULL;
+    dict_t *req_dict = NULL;
+    dict_t *tmp_dict = NULL;
+    uuid_t *originator_uuid = NULL;
+    xlator_t *this = NULL;
+    glusterd_conf_t *conf = NULL;
+    gf_boolean_t is_acquired = _gf_false;
+
+    this = THIS;
+    GF_ASSERT(this);
+    GF_ASSERT(req);
+    GF_ASSERT(dict);
+    conf = this->private;
+    GF_ASSERT(conf);
+
+    txn_generation = conf->generation;
+    originator_uuid = GF_MALLOC(sizeof(uuid_t), gf_common_mt_uuid_t);
+    if (!originator_uuid) {
+        ret = -1;
+        goto out;
+    }
+
+    gf_uuid_copy(*originator_uuid, MY_UUID);
+    ret = dict_set_bin(dict, "originator_uuid", originator_uuid,
+                       sizeof(uuid_t));
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+               "Failed to set originator_uuid.");
+        GF_FREE(originator_uuid);
+        goto out;
+    }
+
+    ret = dict_set_int32n(dict, "is_synctasked", SLEN("is_synctasked"),
+                          _gf_true);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+               "Failed to set synctasked flag to true.");
+        goto out;
+    }
+
+    tmp_dict = dict_new();
+    if (!tmp_dict) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_CREATE_FAIL,
+               "Unable to create dict");
+        goto out;
+    }
+    dict_copy(dict, tmp_dict);
+
+    ret = glusterd_mgmt_v3_initiate_lockdown(op, dict, &op_errstr, &op_errno,
+                                             &is_acquired, txn_generation);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_MGMTV3_LOCKDOWN_FAIL,
+               "mgmt_v3 lockdown failed.");
+        goto out;
+    }
+
+    ret = glusterd_mgmt_v3_build_payload(&req_dict, &op_errstr, dict, op);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_MGMTV3_PAYLOAD_BUILD_FAIL,
+               LOGSTR_BUILD_PAYLOAD, gd_op_list[op]);
+        if (op_errstr == NULL)
+            gf_asprintf(&op_errstr, OPERRSTR_BUILD_PAYLOAD);
+        goto out;
+    }
+
+    ret = glusterd_mgmt_v3_pre_validate(op, req_dict, &op_errstr, &op_errno,
+                                        txn_generation);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_PRE_VALIDATION_FAIL,
+               "Pre Validation Failed");
+        goto out;
+    }
+
+    ret = glusterd_mgmt_v3_commit(op, dict, req_dict, &op_errstr, &op_errno,
+                                  txn_generation);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_COMMIT_OP_FAIL,
+               "Commit Op Failed");
+        goto out;
+    }
+
+    ret = 0;
 
 out:
-        return ret;
-}
-
-void
-glusterd_do_replace_brick (void *data)
-{
-        glusterd_volinfo_t     *volinfo = NULL;
-        int32_t                 op      = 0;
-        int32_t                 src_port = 0;
-        int32_t                 dst_port = 0;
-        int32_t                 ret      = 0;
-        dict_t                 *dict    = NULL;
-        char                   *src_brick = NULL;
-        char                   *dst_brick = NULL;
-        char                   *volname   = NULL;
-        glusterd_brickinfo_t   *src_brickinfo = NULL;
-        glusterd_brickinfo_t   *dst_brickinfo = NULL;
-	glusterd_conf_t	       *priv = NULL;
-        uuid_t                 *txn_id = NULL;
-        xlator_t               *this = NULL;
-
-        this = THIS;
-	GF_ASSERT (this);
-	priv = this->private;
-        GF_ASSERT (priv);
-        GF_ASSERT (data);
-
-        txn_id = &priv->global_txn_id;
-        dict = data;
-
-	if (priv->timer) {
-		gf_timer_call_cancel (THIS->ctx, priv->timer);
-		priv->timer = NULL;
-                gf_log ("", GF_LOG_DEBUG,
-                        "Cancelling timer thread");
-	}
-
-        gf_log ("", GF_LOG_DEBUG,
-                "Replace brick operation detected");
-
-        ret = dict_get_bin (dict, "transaction_id", (void **)&txn_id);
-        gf_log (this->name, GF_LOG_DEBUG, "transaction ID = %s",
-                uuid_utoa (*txn_id));
-
-        ret = dict_get_int32 (dict, "operation", &op);
-        if (ret) {
-                gf_log ("", GF_LOG_DEBUG,
-                        "dict_get on operation failed");
-                goto out;
-        }
-        ret = dict_get_str (dict, "src-brick", &src_brick);
-        if (ret) {
-                gf_log ("", GF_LOG_ERROR, "Unable to get src brick");
-                goto out;
-        }
+    op_ret = ret;
 
-        gf_log ("", GF_LOG_DEBUG,
-                "src brick=%s", src_brick);
+    (void)glusterd_mgmt_v3_release_peer_locks(op, dict, op_ret, &op_errstr,
+                                              is_acquired, txn_generation);
 
-        ret = dict_get_str (dict, "dst-brick", &dst_brick);
+    if (is_acquired) {
+        ret = glusterd_multiple_mgmt_v3_unlock(tmp_dict, MY_UUID);
         if (ret) {
-                gf_log ("", GF_LOG_ERROR, "Unable to get dst brick");
-                goto out;
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_MGMTV3_UNLOCK_FAIL,
+                   "Failed to release mgmt_v3 locks on "
+                   "localhost.");
+            op_ret = ret;
         }
+    }
+    /* SEND CLI RESPONSE */
+    glusterd_op_send_cli_response(op, op_ret, op_errno, req, dict, op_errstr);
 
-        gf_log ("", GF_LOG_DEBUG,
-                "dst brick=%s", dst_brick);
+    if (req_dict)
+        dict_unref(req_dict);
 
-        ret = dict_get_str (dict, "volname", &volname);
+    if (tmp_dict)
+        dict_unref(tmp_dict);
 
-        if (ret) {
-                gf_log ("", GF_LOG_ERROR, "Unable to get volume name");
-                goto out;
-        }
+    if (op_errstr) {
+        GF_FREE(op_errstr);
+        op_errstr = NULL;
+    }
 
-        ret = glusterd_volinfo_find (volname, &volinfo);
-        if (ret) {
-                gf_log ("", GF_LOG_ERROR, "Unable to allocate memory");
-                goto out;
-        }
-
-        ret = glusterd_volume_brickinfo_get_by_brick (src_brick, volinfo,
-                                                      &src_brickinfo);
-        if (ret) {
-                gf_log ("", GF_LOG_DEBUG, "Unable to get src-brickinfo");
-                goto out;
-        }
-
-        ret = glusterd_get_rb_dst_brickinfo (volinfo, &dst_brickinfo);
-        if (!dst_brickinfo) {
-                gf_log ("", GF_LOG_DEBUG, "Unable to get dst-brickinfo");
-                goto out;
-        }
-
-        ret = glusterd_resolve_brick (dst_brickinfo);
-        if (ret) {
-                gf_log ("", GF_LOG_DEBUG, "Unable to resolve dst-brickinfo");
-                goto out;
-        }
-
-        ret = dict_get_int32 (dict, "src-brick-port", &src_port);
-        if (ret) {
-                gf_log ("", GF_LOG_ERROR, "Unable to get src-brick port");
-                goto out;
-        }
-
-        ret = dict_get_int32 (dict, "dst-brick-port", &dst_port);
-        if (ret) {
-                gf_log ("", GF_LOG_ERROR, "Unable to get dst-brick port");
-        }
-
-        dst_brickinfo->port = dst_port;
-        src_brickinfo->port = src_port;
-
-        switch (op) {
-        case GF_REPLACE_OP_START:
-                if (!dst_port) {
-                        ret = -1;
-                        goto out;
-                }
-
-                ret = rb_do_operation (volinfo, src_brickinfo, dst_brickinfo,
-                                       GF_REPLACE_OP_START);
-                if (ret)
-                        goto out;
-                break;
-        case GF_REPLACE_OP_PAUSE:
-        case GF_REPLACE_OP_ABORT:
-        case GF_REPLACE_OP_COMMIT:
-        case GF_REPLACE_OP_COMMIT_FORCE:
-        case GF_REPLACE_OP_STATUS:
-                break;
-        default:
-                ret = -1;
-                goto out;
-        }
-
-out:
-        if (ret)
-                ret = glusterd_op_sm_inject_event (GD_OP_EVENT_RCVD_RJT,
-                                                   txn_id, NULL);
-        else
-                ret = glusterd_op_sm_inject_event (GD_OP_EVENT_COMMIT_ACC,
-                                                   txn_id, NULL);
-
-        synclock_lock (&priv->big_lock);
-        {
-                glusterd_op_sm ();
-        }
-        synclock_unlock (&priv->big_lock);
+    return 0;
 }
diff --git a/xlators/mgmt/glusterd/src/glusterd-reset-brick.c b/xlators/mgmt/glusterd/src/glusterd-reset-brick.c
new file mode 100644
index 00000000000..e4d247a1d6c
--- /dev/null
+++ b/xlators/mgmt/glusterd/src/glusterd-reset-brick.c
@@ -0,0 +1,376 @@
+/*
+   Copyright (c) 2016 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
+
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
+#include <glusterfs/common-utils.h>
+#include "cli1-xdr.h"
+#include "xdr-generic.h"
+#include <glusterfs/glusterfs.h>
+#include "glusterd.h"
+#include "glusterd-op-sm.h"
+#include "glusterd-geo-rep.h"
+#include "glusterd-store.h"
+#include "glusterd-utils.h"
+#include "glusterd-svc-mgmt.h"
+#include "glusterd-svc-helper.h"
+#include "glusterd-volgen.h"
+#include "glusterd-messages.h"
+#include "glusterd-mgmt.h"
+#include <glusterfs/run.h>
+#include <glusterfs/syscall.h>
+
+#include <signal.h>
+
+int
+glusterd_reset_brick_prevalidate(dict_t *dict, char **op_errstr,
+                                 dict_t *rsp_dict)
+{
+    int ret = 0;
+    char *src_brick = NULL;
+    char *dst_brick = NULL;
+    char *volname = NULL;
+    char *op = NULL;
+    glusterd_op_t gd_op = -1;
+    glusterd_volinfo_t *volinfo = NULL;
+    glusterd_brickinfo_t *src_brickinfo = NULL;
+    char *host = NULL;
+    char msg[2048] = {0};
+    glusterd_peerinfo_t *peerinfo = NULL;
+    glusterd_brickinfo_t *dst_brickinfo = NULL;
+    glusterd_conf_t *priv = NULL;
+    char pidfile[PATH_MAX] = {0};
+    xlator_t *this = NULL;
+    gf_boolean_t is_force = _gf_false;
+    int32_t ignore_partition = 0;
+    pid_t pid = -1;
+    uuid_t volume_id = {
+        0,
+    };
+    char *dup_dstbrick = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+
+    priv = this->private;
+    GF_ASSERT(priv);
+
+    ret = glusterd_brick_op_prerequisites(dict, &op, &gd_op, &volname, &volinfo,
+                                          &src_brick, &src_brickinfo, pidfile,
+                                          op_errstr, rsp_dict);
+    if (ret)
+        goto out;
+
+    if (!strcmp(op, "GF_RESET_OP_START"))
+        goto done;
+
+    if (!strcmp(op, "GF_RESET_OP_COMMIT_FORCE"))
+        is_force = _gf_true;
+
+    ret = glusterd_get_dst_brick_info(&dst_brick, volname, op_errstr,
+                                      &dst_brickinfo, &host, dict,
+                                      &dup_dstbrick);
+    if (ret)
+        goto out;
+
+    ret = glusterd_new_brick_validate(dst_brick, dst_brickinfo, msg,
+                                      sizeof(msg), op);
+    /* if bricks are not same and reset brick was used, fail command.
+     * Only replace brick should be used to replace with new bricks
+     * to the volume.
+     */
+    if (ret == 0) {
+        if (!gf_uuid_compare(MY_UUID, dst_brickinfo->uuid)) {
+            ret = -1;
+            *op_errstr = gf_strdup(
+                "When destination brick is new,"
+                " please use"
+                " gluster volume "
+                "replace-brick <volname> "
+                "<src-brick> <dst-brick> "
+                "commit force");
+            if (*op_errstr)
+                gf_msg(this->name, GF_LOG_ERROR, EPERM,
+                       GD_MSG_BRICK_VALIDATE_FAIL, "%s", *op_errstr);
+            goto out;
+        }
+    } else if (ret == 1) {
+        if (gf_is_service_running(pidfile, &pid)) {
+            ret = -1;
+            *op_errstr = gf_strdup(
+                "Source brick"
+                " must be stopped."
+                " Please use "
+                "gluster volume "
+                "reset-brick <volname> "
+                "<dst-brick> start.");
+            if (*op_errstr)
+                gf_msg(this->name, GF_LOG_ERROR, EPERM,
+                       GD_MSG_BRICK_VALIDATE_FAIL, "%s", *op_errstr);
+            goto out;
+        }
+        ret = sys_lgetxattr(dst_brickinfo->path, GF_XATTR_VOL_ID_KEY, volume_id,
+                            16);
+        if (gf_uuid_compare(dst_brickinfo->uuid, src_brickinfo->uuid) ||
+            (ret >= 0 && is_force == _gf_false)) {
+            ret = -1;
+            *op_errstr = gf_strdup(
+                "Brick not available."
+                "It may be containing "
+                "or be contained "
+                "by an existing brick."
+                "Use 'force' option to "
+                "override this.");
+            if (*op_errstr)
+                gf_msg(this->name, GF_LOG_ERROR, EPERM,
+                       GD_MSG_BRICK_VALIDATE_FAIL, "%s", *op_errstr);
+            goto out;
+        }
+        ret = 0;
+    } else {
+        *op_errstr = gf_strdup(msg);
+        ret = -1;
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_BRICK_VALIDATE_FAIL, "%s",
+               *op_errstr);
+        goto out;
+    }
+
+    volinfo->rep_brick.src_brick = src_brickinfo;
+    volinfo->rep_brick.dst_brick = dst_brickinfo;
+
+    ret = dict_get_int32n(dict, "ignore-partition", SLEN("ignore-partition"),
+                          &ignore_partition);
+    ret = 0;
+    if (gf_is_local_addr(host)) {
+        ret = glusterd_validate_and_create_brickpath(
+            dst_brickinfo, volinfo->volume_id, volinfo->volname, op_errstr,
+            is_force, ignore_partition);
+        if (ret)
+            goto out;
+    } else {
+        RCU_READ_LOCK;
+
+        peerinfo = glusterd_peerinfo_find(NULL, host);
+        if (peerinfo == NULL) {
+            RCU_READ_UNLOCK;
+            ret = -1;
+            snprintf(msg, sizeof(msg), "%s, is not a friend.", host);
+            *op_errstr = gf_strdup(msg);
+            goto out;
+
+        } else if (!peerinfo->connected) {
+            RCU_READ_UNLOCK;
+            ret = -1;
+            snprintf(msg, sizeof(msg),
+                     "%s,"
+                     "is not connected at "
+                     "the moment.",
+                     host);
+            *op_errstr = gf_strdup(msg);
+            goto out;
+
+        } else if (GD_FRIEND_STATE_BEFRIENDED != peerinfo->state.state) {
+            RCU_READ_UNLOCK;
+            ret = -1;
+            snprintf(msg, sizeof(msg),
+                     "%s, is not befriended "
+                     "at the moment.",
+                     host);
+            *op_errstr = gf_strdup(msg);
+            goto out;
+        }
+        RCU_READ_UNLOCK;
+    }
+
+    if (!(gf_uuid_compare(dst_brickinfo->uuid, MY_UUID))) {
+        ret = glusterd_get_brick_mount_dir(dst_brickinfo->path,
+                                           dst_brickinfo->hostname,
+                                           dst_brickinfo->mount_dir);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_BRICK_MOUNTDIR_GET_FAIL,
+                   "Failed to get brick mount_dir");
+            goto out;
+        }
+        ret = dict_set_dynstr_with_alloc(rsp_dict, "brick1.mount_dir",
+                                         dst_brickinfo->mount_dir);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+                   "Failed to set brick.mount_dir");
+            goto out;
+        }
+    }
+
+    ret = dict_set_int32n(rsp_dict, "brick_count", SLEN("brick_count"), 1);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+               "Failed to set local_brick_count.");
+        goto out;
+    }
+
+done:
+    ret = 0;
+out:
+    GF_FREE(dup_dstbrick);
+    gf_msg_debug(this->name, 0, "Returning %d.", ret);
+
+    return ret;
+}
+
+int
+glusterd_op_reset_brick(dict_t *dict, dict_t *rsp_dict)
+{
+    int ret = 0;
+    char *op = NULL;
+    glusterd_volinfo_t *volinfo = NULL;
+    char *volname = NULL;
+    xlator_t *this = NULL;
+    glusterd_conf_t *priv = NULL;
+    char *src_brick = NULL;
+    char *dst_brick = NULL;
+    glusterd_brickinfo_t *src_brickinfo = NULL;
+    glusterd_brickinfo_t *dst_brickinfo = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+
+    priv = this->private;
+    GF_ASSERT(priv);
+
+    ret = dict_get_strn(dict, "operation", SLEN("operation"), &op);
+    if (ret) {
+        gf_msg_debug(this->name, 0, "dict_get on operation failed");
+        goto out;
+    }
+
+    ret = dict_get_strn(dict, "volname", SLEN("volname"), &volname);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+               "Unable to get volume name");
+        goto out;
+    }
+
+    ret = glusterd_volinfo_find(volname, &volinfo);
+    if (ret)
+        goto out;
+
+    ret = dict_get_strn(dict, "src-brick", SLEN("src-brick"), &src_brick);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+               "Unable to get src brick");
+        goto out;
+    }
+
+    gf_msg_debug(this->name, 0, "src brick=%s", src_brick);
+
+    ret = glusterd_volume_brickinfo_get_by_brick(src_brick, volinfo,
+                                                 &src_brickinfo, _gf_false);
+    if (ret) {
+        gf_msg_debug(this->name, 0, "Unable to get src-brickinfo");
+        goto out;
+    }
+
+    if (!strcmp(op, "GF_RESET_OP_START")) {
+        ret = glusterd_volume_stop_glusterfs(volinfo, src_brickinfo, _gf_false);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_CRITICAL, 0, GD_MSG_BRICK_STOP_FAIL,
+                   "Unable to stop"
+                   " brick: %s:%s",
+                   src_brickinfo->hostname, src_brickinfo->path);
+        }
+
+        goto out;
+
+    } else if (!strcmp(op, "GF_RESET_OP_COMMIT") ||
+               !strcmp(op, "GF_RESET_OP_COMMIT_FORCE")) {
+        ret = dict_get_strn(dict, "dst-brick", SLEN("dst-brick"), &dst_brick);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+                   "Unable to get dst brick");
+            goto out;
+        }
+
+        gf_msg_debug(this->name, 0, "dst brick=%s", dst_brick);
+
+        ret = glusterd_get_rb_dst_brickinfo(volinfo, &dst_brickinfo);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_RB_BRICKINFO_GET_FAIL,
+                   "Unable to get "
+                   "reset brick "
+                   "destination brickinfo");
+            goto out;
+        }
+
+        ret = glusterd_resolve_brick(dst_brickinfo);
+        if (ret) {
+            gf_msg_debug(this->name, 0, "Unable to resolve dst-brickinfo");
+            goto out;
+        }
+
+        ret = rb_update_dstbrick_port(dst_brickinfo, rsp_dict, dict);
+        if (ret)
+            goto out;
+
+        if (gf_uuid_compare(dst_brickinfo->uuid, MY_UUID)) {
+            gf_msg_debug(this->name, 0, "I AM THE DESTINATION HOST");
+            ret = glusterd_volume_stop_glusterfs(volinfo, src_brickinfo,
+                                                 _gf_false);
+            if (ret) {
+                gf_msg(this->name, GF_LOG_CRITICAL, 0, GD_MSG_BRICK_STOP_FAIL,
+                       "Unable to stop brick: %s:%s", src_brickinfo->hostname,
+                       src_brickinfo->path);
+                goto out;
+            }
+        }
+
+        ret = glusterd_svcs_stop(volinfo);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, 0,
+                   GD_MSG_GLUSTER_SERVICES_STOP_FAIL,
+                   "Unable to stop gluster services, ret: %d", ret);
+            goto out;
+        }
+        ret = glusterd_op_perform_replace_brick(volinfo, src_brick, dst_brick,
+                                                dict);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_CRITICAL, 0, GD_MSG_BRICK_ADD_FAIL,
+                   "Unable to add dst-brick: "
+                   "%s to volume: %s",
+                   dst_brick, volinfo->volname);
+            (void)glusterd_svcs_manager(volinfo);
+            goto out;
+        }
+
+        volinfo->rebal.defrag_status = 0;
+
+        ret = glusterd_svcs_manager(volinfo);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_CRITICAL, 0,
+                   GD_MSG_GLUSTER_SERVICE_START_FAIL,
+                   "Failed to start one or more gluster services.");
+        }
+
+        ret = glusterd_fetchspec_notify(THIS);
+        glusterd_brickinfo_delete(volinfo->rep_brick.dst_brick);
+        volinfo->rep_brick.src_brick = NULL;
+        volinfo->rep_brick.dst_brick = NULL;
+
+        if (!ret)
+            ret = glusterd_store_volinfo(volinfo,
+                                         GLUSTERD_VOLINFO_VER_AC_INCREMENT);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_RBOP_STATE_STORE_FAIL,
+                   "Couldn't store"
+                   " reset brick operation's state.");
+        }
+    } else {
+        ret = -1;
+        goto out;
+    }
+
+out:
+    return ret;
+}
diff --git a/xlators/mgmt/glusterd/src/glusterd-rpc-ops.c b/xlators/mgmt/glusterd/src/glusterd-rpc-ops.c
index 59b0e5f072e..88662e3bbae 100644
--- a/xlators/mgmt/glusterd/src/glusterd-rpc-ops.c
+++ b/xlators/mgmt/glusterd/src/glusterd-rpc-ops.c
@@ -8,120 +8,113 @@
    cases as published by the Free Software Foundation.
 */
 
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
-
 #include "rpc-clnt.h"
 #include "glusterd1-xdr.h"
 #include "cli1-xdr.h"
 
 #include "xdr-generic.h"
 
-#include "compat-errno.h"
+#include <glusterfs/compat-errno.h>
 #include "glusterd-op-sm.h"
 #include "glusterd-sm.h"
 #include "glusterd.h"
 #include "protocol-common.h"
 #include "glusterd-utils.h"
-#include "common-utils.h"
+#include <glusterfs/common-utils.h>
 #include "glusterd-messages.h"
 #include "glusterd-snapshot-utils.h"
 #include <sys/uio.h>
 
+#define SERVER_PATH_MAX (16 * 1024)
 
-#define SERVER_PATH_MAX  (16 * 1024)
-
+#define GLUSTERD_STACK_DESTROY(frame)                                          \
+    do {                                                                       \
+        frame->local = NULL;                                                   \
+        STACK_DESTROY(frame->root);                                            \
+    } while (0)
 
 extern glusterd_op_info_t opinfo;
 extern uuid_t global_txn_id;
 
 int32_t
-glusterd_op_send_cli_response (glusterd_op_t op, int32_t op_ret,
-                               int32_t op_errno, rpcsvc_request_t *req,
-                               void *op_ctx, char *op_errstr)
+glusterd_op_send_cli_response(glusterd_op_t op, int32_t op_ret,
+                              int32_t op_errno, rpcsvc_request_t *req,
+                              void *op_ctx, char *op_errstr)
 {
-        int32_t         ret = -1;
-        void            *cli_rsp = NULL;
-        dict_t          *ctx = NULL;
-        char            *free_ptr = NULL;
-        glusterd_conf_t *conf = NULL;
-        xdrproc_t       xdrproc = NULL;
-        char            *errstr = NULL;
-        int32_t         status = 0;
-        int32_t         count = 0;
-        gf_cli_rsp      rsp = {0,};
-        xlator_t        *this = NULL;
-
-        this = THIS;
-        GF_ASSERT (this);
-        conf = this->private;
-
-        GF_ASSERT (conf);
-
-        ctx = op_ctx;
-
-        switch (op) {
-        case GD_OP_REMOVE_BRICK:
-        {
-                if (ctx)
-                        ret = dict_get_str (ctx, "errstr", &errstr);
-                break;
-        }
-        case GD_OP_RESET_VOLUME:
-        {
-                if (op_ret && !op_errstr)
-                        errstr = "Error while resetting options";
-                break;
+    int32_t ret = -1;
+    void *cli_rsp = NULL;
+    dict_t *ctx = NULL;
+    char *free_ptr = NULL;
+    glusterd_conf_t *conf = NULL;
+    xdrproc_t xdrproc = NULL;
+    char *errstr = NULL;
+    int32_t status = 0;
+    int32_t count = 0;
+    gf_cli_rsp rsp = {
+        0,
+    };
+    xlator_t *this = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+    conf = this->private;
+
+    GF_ASSERT(conf);
+
+    ctx = op_ctx;
+
+    switch (op) {
+        case GD_OP_REMOVE_BRICK: {
+            if (ctx)
+                ret = dict_get_strn(ctx, "errstr", SLEN("errstr"), &errstr);
+            break;
+        }
+        case GD_OP_RESET_VOLUME: {
+            if (op_ret && !op_errstr)
+                errstr = "Error while resetting options";
+            break;
         }
         case GD_OP_REBALANCE:
-        case GD_OP_DEFRAG_BRICK_VOLUME:
-        {
-                if (ctx) {
-                        ret = dict_get_int32 (ctx, "status", &status);
-                        if (ret) {
-                                gf_log (this->name, GF_LOG_TRACE,
-                                        "failed to get status");
-                        }
+        case GD_OP_DEFRAG_BRICK_VOLUME: {
+            if (ctx) {
+                ret = dict_get_int32n(ctx, "status", SLEN("status"), &status);
+                if (ret) {
+                    gf_msg_trace(this->name, 0, "failed to get status");
                 }
-                break;
+            }
+            break;
         }
         case GD_OP_GSYNC_CREATE:
-        case GD_OP_GSYNC_SET:
-        {
-               if (ctx) {
-                        ret = dict_get_str (ctx, "errstr", &errstr);
-                        ret = dict_set_str (ctx, "glusterd_workdir", conf->workdir);
-                        /* swallow error here, that will be re-triggered in cli */
-
-               }
-               break;
-
-        }
-        case GD_OP_PROFILE_VOLUME:
-        {
-                if (ctx && dict_get_int32 (ctx, "count", &count)) {
-                        ret = dict_set_int32 (ctx, "count", 0);
-                        if (ret) {
-                                gf_log (this->name, GF_LOG_ERROR,
-                                        "failed to set count in dictionary");
-                        }
+        case GD_OP_GSYNC_SET: {
+            if (ctx) {
+                ret = dict_get_strn(ctx, "errstr", SLEN("errstr"), &errstr);
+                ret = dict_set_strn(ctx, "glusterd_workdir",
+                                    SLEN("glusterd_workdir"), conf->workdir);
+                /* swallow error here, that will be re-triggered in cli */
+            }
+            break;
+        }
+        case GD_OP_PROFILE_VOLUME: {
+            if (ctx && dict_get_int32n(ctx, "count", SLEN("count"), &count)) {
+                ret = dict_set_int32n(ctx, "count", SLEN("count"), 0);
+                if (ret) {
+                    gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+                           "failed to set count in dictionary");
                 }
-                break;
+            }
+            break;
         }
         case GD_OP_START_BRICK:
-        case GD_OP_STOP_BRICK:
-        {
-                gf_log (this->name, GF_LOG_DEBUG, "op '%s' not supported",
-                        gd_op_list[op]);
-                break;
+        case GD_OP_STOP_BRICK: {
+            gf_msg_debug(this->name, 0, "op '%s' not supported",
+                         gd_op_list[op]);
+            break;
         }
         case GD_OP_NONE:
-        case GD_OP_MAX:
-        {
-                gf_log (this->name, GF_LOG_ERROR, "invalid operation");
-                break;
+        case GD_OP_MAX: {
+            gf_msg(this->name, GF_LOG_ERROR, EINVAL, GD_MSG_OP_UNSUPPORTED,
+                   "invalid operation");
+            break;
         }
         case GD_OP_CREATE_VOLUME:
         case GD_OP_START_VOLUME:
@@ -135,7 +128,6 @@ glusterd_op_send_cli_response (glusterd_op_t op, int32_t op_ret,
         case GD_OP_REPLACE_BRICK:
         case GD_OP_STATUS_VOLUME:
         case GD_OP_SET_VOLUME:
-        case GD_OP_GANESHA:
         case GD_OP_LIST_VOLUME:
         case GD_OP_CLEARLOCKS_VOLUME:
         case GD_OP_HEAL_VOLUME:
@@ -143,2108 +135,2314 @@ glusterd_op_send_cli_response (glusterd_op_t op, int32_t op_ret,
         case GD_OP_SNAP:
         case GD_OP_BARRIER:
         case GD_OP_BITROT:
-        {
-                /*nothing specific to be done*/
-                break;
-        }
-        case GD_OP_COPY_FILE:
-        {
-               if (ctx)
-                        ret = dict_get_str (ctx, "errstr", &errstr);
-               break;
-        }
-        case GD_OP_SYS_EXEC:
-        {
-               if (ctx) {
-                        ret = dict_get_str (ctx, "errstr", &errstr);
-                        ret = dict_set_str (ctx, "glusterd_workdir",
-                                            conf->workdir);
-               }
-               break;
-        }
-        }
+        case GD_OP_SCRUB_STATUS:
+        case GD_OP_SCRUB_ONDEMAND:
+        case GD_OP_RESET_BRICK:
+        case GD_OP_MAX_OPVERSION:
+        case GD_OP_DETACH_NOT_STARTED:
+        case GD_OP_GANESHA:
+        case GD_OP_DETACH_TIER:
+        case GD_OP_TIER_MIGRATE:
+        case GD_OP_TIER_START_STOP:
+        case GD_OP_TIER_STATUS:
+        case GD_OP_DETACH_TIER_STATUS:
+        case GD_OP_REMOVE_TIER_BRICK:
+        case GD_OP_ADD_TIER_BRICK:
 
-        rsp.op_ret = op_ret;
-        rsp.op_errno = errno;
-        if (errstr)
-                rsp.op_errstr = errstr;
-        else if (op_errstr)
-                rsp.op_errstr = op_errstr;
-
-        if (!rsp.op_errstr)
-                rsp.op_errstr = "";
-
-        if (ctx) {
-                ret = dict_allocate_and_serialize (ctx, &rsp.dict.dict_val,
-                                                   &rsp.dict.dict_len);
-                if (ret < 0 )
-                        gf_log (this->name, GF_LOG_ERROR, "failed to "
-                                "serialize buffer");
-                else
-                        free_ptr = rsp.dict.dict_val;
-        }
+        {
+            /*nothing specific to be done*/
+            break;
+        }
+        case GD_OP_COPY_FILE: {
+            if (ctx)
+                ret = dict_get_strn(ctx, "errstr", SLEN("errstr"), &errstr);
+            break;
+        }
+        case GD_OP_SYS_EXEC: {
+            if (ctx) {
+                ret = dict_get_strn(ctx, "errstr", SLEN("errstr"), &errstr);
+                ret = dict_set_strn(ctx, "glusterd_workdir",
+                                    SLEN("glusterd_workdir"), conf->workdir);
+            }
+            break;
+        }
+    }
+
+    rsp.op_ret = op_ret;
+    rsp.op_errno = op_errno;
+
+    if (errstr)
+        rsp.op_errstr = errstr;
+    else if (op_errstr)
+        rsp.op_errstr = op_errstr;
+
+    if (!rsp.op_errstr)
+        rsp.op_errstr = "";
+
+    if (ctx) {
+        ret = dict_allocate_and_serialize(ctx, &rsp.dict.dict_val,
+                                          &rsp.dict.dict_len);
+        if (ret < 0)
+            gf_smsg(this->name, GF_LOG_ERROR, errno,
+                    GD_MSG_DICT_ALLOC_AND_SERL_LENGTH_GET_FAIL, NULL);
+        else
+            free_ptr = rsp.dict.dict_val;
+    }
 
-        /* needed by 'rebalance status' */
-        if (status)
-                rsp.op_errno = status;
+    /* needed by 'rebalance status' */
+    if (status)
+        rsp.op_errno = status;
 
-        cli_rsp = &rsp;
-        xdrproc = (xdrproc_t) xdr_gf_cli_rsp;
+    cli_rsp = &rsp;
+    xdrproc = (xdrproc_t)xdr_gf_cli_rsp;
 
-        glusterd_to_cli (req, cli_rsp, NULL, 0, NULL,
-                         xdrproc, ctx);
-        ret = 0;
+    glusterd_to_cli(req, cli_rsp, NULL, 0, NULL, xdrproc, ctx);
+    ret = 0;
 
-        GF_FREE (free_ptr);
-        gf_log (this->name, GF_LOG_DEBUG, "Returning %d", ret);
-        return ret;
+    GF_FREE(free_ptr);
+    gf_msg_debug(this->name, 0, "Returning %d", ret);
+    return ret;
 }
 
 int
-glusterd_big_locked_cbk (struct rpc_req *req, struct iovec *iov,
-                         int count, void *myframe, fop_cbk_fn_t fn)
+glusterd_big_locked_cbk(struct rpc_req *req, struct iovec *iov, int count,
+                        void *myframe, fop_cbk_fn_t fn)
 {
-        glusterd_conf_t *priv = THIS->private;
-        int             ret   = -1;
+    glusterd_conf_t *priv = THIS->private;
+    int ret = -1;
 
-        synclock_lock (&priv->big_lock);
-        ret = fn (req, iov, count, myframe);
-        synclock_unlock (&priv->big_lock);
+    synclock_lock(&priv->big_lock);
+    ret = fn(req, iov, count, myframe);
+    synclock_unlock(&priv->big_lock);
 
-        return ret;
+    return ret;
 }
 
 int
-__glusterd_probe_cbk (struct rpc_req *req, struct iovec *iov,
-                        int count, void *myframe)
+__glusterd_probe_cbk(struct rpc_req *req, struct iovec *iov, int count,
+                     void *myframe)
 {
-        gd1_mgmt_probe_rsp          rsp      = {{0},};
-        int                         ret      = 0;
-        glusterd_peerinfo_t        *peerinfo = NULL;
-        glusterd_friend_sm_event_t *event    = NULL;
-        glusterd_probe_ctx_t       *ctx      = NULL;
-        xlator_t                   *this     = NULL;
-        glusterd_conf_t            *conf     = NULL;
-
-        if (-1 == req->rpc_status) {
-                goto out;
-        }
+    gd1_mgmt_probe_rsp rsp = {
+        {0},
+    };
+    int ret = 0;
+    glusterd_peerinfo_t *peerinfo = NULL;
+    glusterd_friend_sm_event_t *event = NULL;
+    glusterd_probe_ctx_t *ctx = NULL;
+    xlator_t *this = NULL;
+    glusterd_conf_t *conf = NULL;
+
+    if (-1 == req->rpc_status) {
+        goto out;
+    }
+
+    this = THIS;
+    GF_ASSERT(this != NULL);
+    conf = this->private;
+    GF_VALIDATE_OR_GOTO(this->name, (conf != NULL), out);
+
+    ret = xdr_to_generic(*iov, &rsp, (xdrproc_t)xdr_gd1_mgmt_probe_rsp);
+    if (ret < 0) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_RES_DECODE_FAIL, "error");
+        // rsp.op_ret   = -1;
+        // rsp.op_errno = EINVAL;
+        goto out;
+    }
+
+    gf_msg(this->name, GF_LOG_INFO, 0, GD_MSG_PROBE_REQ_RESP_RCVD,
+           "Received probe resp from uuid: %s, host: %s", uuid_utoa(rsp.uuid),
+           rsp.hostname);
+    if (rsp.op_ret != 0) {
+        ctx = ((call_frame_t *)myframe)->local;
+        ((call_frame_t *)myframe)->local = NULL;
 
-        this = THIS;
-        GF_ASSERT (this != NULL);
-        conf = this->private;
-        GF_VALIDATE_OR_GOTO (this->name, (conf != NULL), out);
+        GF_ASSERT(ctx);
+
+        if (ctx->req) {
+            glusterd_xfer_cli_probe_resp(ctx->req, rsp.op_ret, rsp.op_errno,
+                                         rsp.op_errstr, ctx->hostname,
+                                         ctx->port, ctx->dict);
+        }
+
+        glusterd_destroy_probe_ctx(ctx);
+        (void)glusterd_friend_remove(rsp.uuid, rsp.hostname);
+        ret = rsp.op_ret;
+        goto out;
+    }
+
+    RCU_READ_LOCK;
+    peerinfo = glusterd_peerinfo_find(rsp.uuid, rsp.hostname);
+    if (peerinfo == NULL) {
+        RCU_READ_UNLOCK
+        ret = -1;
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_PEER_NOT_FOUND,
+               "Could not find peerd %s(%s)", rsp.hostname,
+               uuid_utoa(rsp.uuid));
+        goto out;
+    }
+
+    /*
+     * In the case of a fresh probe rsp.uuid and peerinfo.uuid will not
+     * match, as peerinfo->uuid will be NULL.
+     *
+     * In the case of a peer probe being done to add a new network to a
+     * peer, rsp.uuid will match an existing peerinfo.uuid. If we have this
+     * stage it means that the current address/hostname being used isn't
+     * present in the found peerinfo. If it were, we would have found out
+     * earlier in the probe process and wouldn't even reach till here. So,
+     * we need to add the new hostname to the peer.
+     *
+     * This addition should only be done for cluster op-version >=
+     * GD_OP_VERSION_3_6_0 as address lists are only supported from then on.
+     * Also, this update should only be done when an explicit CLI probe
+     * command was used to begin the probe process.
+     */
+    if ((conf->op_version >= GD_OP_VERSION_3_6_0) &&
+        (gf_uuid_compare(rsp.uuid, peerinfo->uuid) == 0)) {
+        ctx = ((call_frame_t *)myframe)->local;
+        /* Presence of ctx->req implies this probe was started by a cli
+         * probe command
+         */
+        if (ctx->req == NULL)
+            goto cont;
 
-        ret = xdr_to_generic (*iov, &rsp, (xdrproc_t)xdr_gd1_mgmt_probe_rsp);
-        if (ret < 0) {
-                gf_log (this->name, GF_LOG_ERROR, "error");
-                //rsp.op_ret   = -1;
-                //rsp.op_errno = EINVAL;
-                goto out;
-        }
+        gf_msg_debug(this->name, 0,
+                     "Adding address '%s' to "
+                     "existing peer %s",
+                     rsp.hostname, uuid_utoa(rsp.uuid));
 
-        gf_log (this->name, GF_LOG_INFO,
-                "Received probe resp from uuid: %s, host: %s",
-                uuid_utoa (rsp.uuid), rsp.hostname);
-        if (rsp.op_ret != 0) {
-                ctx = ((call_frame_t *)myframe)->local;
-                ((call_frame_t *)myframe)->local = NULL;
-
-                GF_ASSERT (ctx);
-
-                if (ctx->req) {
-                        glusterd_xfer_cli_probe_resp (ctx->req, rsp.op_ret,
-                                                      rsp.op_errno,
-                                                      rsp.op_errstr,
-                                                      ctx->hostname, ctx->port,
-                                                      ctx->dict);
-                }
-
-                glusterd_destroy_probe_ctx (ctx);
-                (void) glusterd_friend_remove (rsp.uuid, rsp.hostname);
-                ret = rsp.op_ret;
-                goto out;
+        ret = glusterd_friend_remove(NULL, rsp.hostname);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, 0,
+                   GD_MSG_STALE_PEERINFO_REMOVE_FAIL,
+                   "Could not remove "
+                   "stale peerinfo with name %s",
+                   rsp.hostname);
+            goto reply;
         }
 
-        rcu_read_lock ();
-        peerinfo = glusterd_peerinfo_find (rsp.uuid, rsp.hostname);
-        if (peerinfo == NULL) {
-                ret = -1;
-                gf_log (this->name, GF_LOG_ERROR, "Could not find peerd %s(%s)",
-                        rsp.hostname, uuid_utoa (rsp.uuid));
-                goto unlock;
+        ret = gd_add_address_to_peer(peerinfo, rsp.hostname);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, 0,
+                   GD_MSG_HOSTNAME_ADD_TO_PEERLIST_FAIL,
+                   "Couldn't add hostname to peer list");
+            goto reply;
         }
 
-        /*
-         * In the case of a fresh probe rsp.uuid and peerinfo.uuid will not
-         * match, as peerinfo->uuid will be NULL.
-         *
-         * In the case of a peer probe being done to add a new network to a
-         * peer, rsp.uuid will match an existing peerinfo.uuid. If we have this
-         * stage it means that the current address/hostname being used isn't
-         * present in the found peerinfo. If it were, we would have found out
-         * earlier in the probe process and wouldn't even reach till here. So,
-         * we need to add the new hostname to the peer.
-         *
-         * This addition should only be done for cluster op-version >=
-         * GD_OP_VERSION_3_6_0 as address lists are only supported from then on.
-         * Also, this update should only be done when an explicit CLI probe
-         * command was used to begin the probe process.
-         */
-        if ((conf->op_version >= GD_OP_VERSION_3_6_0) &&
-            (gf_uuid_compare (rsp.uuid, peerinfo->uuid) == 0)) {
-                ctx = ((call_frame_t *)myframe)->local;
-                /* Presence of ctx->req implies this probe was started by a cli
-                 * probe command
-                 */
-                if (ctx->req == NULL)
-                        goto cont;
-
-                gf_log (this->name, GF_LOG_DEBUG, "Adding address '%s' to "
-                        "existing peer %s", rsp.hostname, uuid_utoa (rsp.uuid));
-
-                ret = glusterd_friend_remove (NULL, rsp.hostname);
-                if (ret) {
-                        gf_log (this->name, GF_LOG_ERROR, "Could not remove "
-                                "stale peerinfo with name %s", rsp.hostname);
-                        goto reply;
-                }
-
-                ret = gd_add_address_to_peer (peerinfo, rsp.hostname);
-                if (ret) {
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "Couldn't add hostname to peer list");
-                        goto reply;
-                }
-
-                /* Injecting LOCAL_ACC to send update */
-                ret = glusterd_friend_sm_new_event (GD_FRIEND_EVENT_LOCAL_ACC,
-                                                    &event);
-                if (!ret) {
-                        event->peername = gf_strdup (peerinfo->hostname);
-                        gf_uuid_copy (event->peerid, peerinfo->uuid);
+        /* Injecting EVENT_NEW_NAME to send update */
+        ret = glusterd_friend_sm_new_event(GD_FRIEND_EVENT_NEW_NAME, &event);
+        if (!ret) {
+            event->peername = gf_strdup(peerinfo->hostname);
+            gf_uuid_copy(event->peerid, peerinfo->uuid);
 
-                        ret = glusterd_friend_sm_inject_event (event);
-                }
-                rsp.op_errno = GF_PROBE_FRIEND;
+            ret = glusterd_friend_sm_inject_event(event);
+        }
+        rsp.op_errno = GF_PROBE_FRIEND;
 
-reply:
-                ctx = ((call_frame_t *)myframe)->local;
-                ((call_frame_t *)myframe)->local = NULL;
+    reply:
+        ctx = ((call_frame_t *)myframe)->local;
+        ((call_frame_t *)myframe)->local = NULL;
 
-                if (!ctx) {
-                        ret = -1;
-                        goto unlock;
-                }
+        if (!ctx) {
+            ret = -1;
+            goto unlock;
+        }
 
-                if (ctx->req) {
-                        glusterd_xfer_cli_probe_resp (ctx->req, ret,
-                                                      rsp.op_errno,
-                                                      rsp.op_errstr,
-                                                      ctx->hostname, ctx->port,
-                                                      ctx->dict);
-                }
+        if (ctx->req) {
+            glusterd_xfer_cli_probe_resp(ctx->req, ret, rsp.op_errno,
+                                         rsp.op_errstr, ctx->hostname,
+                                         ctx->port, ctx->dict);
+        }
 
-                glusterd_destroy_probe_ctx (ctx);
+        glusterd_destroy_probe_ctx(ctx);
 
-                goto unlock;
+        goto unlock;
 
-        } else if (strncasecmp (rsp.hostname, peerinfo->hostname, 1024)) {
-                gf_log (THIS->name, GF_LOG_INFO, "Host: %s  with uuid: %s "
-                        "already present in cluster with alias hostname: %s",
-                        rsp.hostname, uuid_utoa (rsp.uuid), peerinfo->hostname);
+    } else if (strncasecmp(rsp.hostname, peerinfo->hostname, 1024)) {
+        gf_msg(THIS->name, GF_LOG_INFO, 0, GD_MSG_HOST_PRESENT_ALREADY,
+               "Host: %s  with uuid: %s "
+               "already present in cluster with alias hostname: %s",
+               rsp.hostname, uuid_utoa(rsp.uuid), peerinfo->hostname);
 
-                ctx = ((call_frame_t *)myframe)->local;
-                ((call_frame_t *)myframe)->local = NULL;
+        ctx = ((call_frame_t *)myframe)->local;
+        ((call_frame_t *)myframe)->local = NULL;
 
-                if (!ctx) {
-                        ret = -1;
-                        goto unlock;
-                }
+        if (!ctx) {
+            ret = -1;
+            goto unlock;
+        }
 
-                rsp.op_errno = GF_PROBE_FRIEND;
-                if (ctx->req) {
-                        glusterd_xfer_cli_probe_resp (ctx->req, rsp.op_ret,
-                                                      rsp.op_errno,
-                                                      rsp.op_errstr,
-                                                      ctx->hostname, ctx->port,
-                                                      ctx->dict);
-                }
+        rsp.op_errno = GF_PROBE_FRIEND;
+        if (ctx->req) {
+            glusterd_xfer_cli_probe_resp(ctx->req, rsp.op_ret, rsp.op_errno,
+                                         rsp.op_errstr, ctx->hostname,
+                                         ctx->port, ctx->dict);
+        }
 
-                glusterd_destroy_probe_ctx (ctx);
-                (void) glusterd_friend_remove (NULL, rsp.hostname);
-                ret = rsp.op_ret;
+        glusterd_destroy_probe_ctx(ctx);
+        (void)glusterd_friend_remove(NULL, rsp.hostname);
+        ret = rsp.op_ret;
 
-                goto unlock;
-        }
+        goto unlock;
+    }
 
 cont:
-        gf_uuid_copy (peerinfo->uuid, rsp.uuid);
+    gf_uuid_copy(peerinfo->uuid, rsp.uuid);
 
-        ret = glusterd_friend_sm_new_event
-                        (GD_FRIEND_EVENT_INIT_FRIEND_REQ, &event);
+    ret = glusterd_friend_sm_new_event(GD_FRIEND_EVENT_INIT_FRIEND_REQ, &event);
 
-        if (ret) {
-                gf_log ("glusterd", GF_LOG_ERROR,
-                         "Unable to get event");
-                goto unlock;
-        }
+    if (ret) {
+        RCU_READ_UNLOCK;
+        gf_msg("glusterd", GF_LOG_ERROR, 0, GD_MSG_NEW_FRIEND_SM_EVENT_GET_FAIL,
+               "Unable to get event");
+        goto out;
+    }
 
-        event->peername = gf_strdup (peerinfo->hostname);
-        gf_uuid_copy (event->peerid, peerinfo->uuid);
-
-        event->ctx      = ((call_frame_t *)myframe)->local;
-        ((call_frame_t *)myframe)->local = NULL;
-        ret = glusterd_friend_sm_inject_event (event);
+    event->peername = gf_strdup(peerinfo->hostname);
+    gf_uuid_copy(event->peerid, peerinfo->uuid);
 
+    event->ctx = ((call_frame_t *)myframe)->local;
+    ((call_frame_t *)myframe)->local = NULL;
+    ret = glusterd_friend_sm_inject_event(event);
 
-        gf_log ("glusterd", GF_LOG_INFO, "Received resp to probe req");
+    gf_msg("glusterd", GF_LOG_INFO, 0, GD_MSG_PROBE_REQ_RESP_RCVD,
+           "Received resp to probe req");
 
 unlock:
-        rcu_read_unlock ();
+    RCU_READ_UNLOCK;
 
 out:
-        free (rsp.hostname);//malloced by xdr
-        GLUSTERD_STACK_DESTROY (((call_frame_t *)myframe));
-
-        /* Attempt to start the state machine. Needed as no state machine could
-         * be running at time this RPC reply was recieved
-         */
-        if (!ret) {
-                glusterd_friend_sm ();
-                glusterd_op_sm ();
-        }
-
-        return ret;
+    free(rsp.hostname);  // malloced by xdr
+    GLUSTERD_STACK_DESTROY(((call_frame_t *)myframe));
+
+    /* Attempt to start the state machine. Needed as no state machine could
+     * be running at time this RPC reply was received
+     */
+    if (!ret) {
+        glusterd_friend_sm();
+        glusterd_op_sm();
+    }
+
+    return ret;
 }
 
 int
-glusterd_probe_cbk (struct rpc_req *req, struct iovec *iov,
-                    int count, void *myframe)
+glusterd_probe_cbk(struct rpc_req *req, struct iovec *iov, int count,
+                   void *myframe)
 {
-        return glusterd_big_locked_cbk (req, iov, count, myframe,
-                                        __glusterd_probe_cbk);
+    return glusterd_big_locked_cbk(req, iov, count, myframe,
+                                   __glusterd_probe_cbk);
 }
 
-
 int
-__glusterd_friend_add_cbk (struct rpc_req * req, struct iovec *iov,
-                            int count, void *myframe)
+__glusterd_friend_add_cbk(struct rpc_req *req, struct iovec *iov, int count,
+                          void *myframe)
 {
-        gd1_mgmt_friend_rsp           rsp   = {{0},};
-        int                           ret   = -1;
-        glusterd_friend_sm_event_t        *event = NULL;
-        glusterd_friend_sm_event_type_t    event_type = GD_FRIEND_EVENT_NONE;
-        glusterd_peerinfo_t           *peerinfo = NULL;
-        int32_t                       op_ret = -1;
-        int32_t                       op_errno = -1;
-        glusterd_probe_ctx_t          *ctx = NULL;
-        glusterd_friend_update_ctx_t  *ev_ctx = NULL;
-
-        if (-1 == req->rpc_status) {
-                rsp.op_ret   = -1;
-                rsp.op_errno = EINVAL;
-                goto out;
-        }
-
-        ret = xdr_to_generic (*iov, &rsp, (xdrproc_t)xdr_gd1_mgmt_friend_rsp);
-        if (ret < 0) {
-                gf_log ("", GF_LOG_ERROR, "error");
-                rsp.op_ret   = -1;
-                rsp.op_errno = EINVAL;
-                goto out;
-        }
-
-        op_ret = rsp.op_ret;
-        op_errno = rsp.op_errno;
-
-        gf_log ("glusterd", GF_LOG_INFO,
-                "Received %s from uuid: %s, host: %s, port: %d",
-                (op_ret)?"RJT":"ACC", uuid_utoa (rsp.uuid), rsp.hostname, rsp.port);
-
-        rcu_read_lock ();
-
-        peerinfo = glusterd_peerinfo_find (rsp.uuid, rsp.hostname);
-        if (peerinfo == NULL) {
-                ret = -1;
-                gf_log ("", GF_LOG_ERROR, "received friend add response from"
-                        " unknown peer uuid: %s", uuid_utoa (rsp.uuid));
-                goto unlock;
-        }
-
-        if (op_ret)
-                event_type = GD_FRIEND_EVENT_RCVD_RJT;
-        else
-                event_type = GD_FRIEND_EVENT_RCVD_ACC;
-
-        ret = glusterd_friend_sm_new_event (event_type, &event);
-
-        if (ret) {
-                gf_log ("glusterd", GF_LOG_ERROR,
-                         "Unable to get event");
-                goto unlock;
-        }
-
-        ev_ctx = GF_CALLOC (1, sizeof (*ev_ctx),
-                                gf_gld_mt_friend_update_ctx_t);
-        if (!ev_ctx) {
-                ret = -1;
-                goto unlock;
-        }
-
-        gf_uuid_copy (ev_ctx->uuid, rsp.uuid);
-        ev_ctx->hostname = gf_strdup (rsp.hostname);
-
-        event->peername = gf_strdup (peerinfo->hostname);
-        gf_uuid_copy (event->peerid, peerinfo->uuid);
-        event->ctx = ev_ctx;
-        ret = glusterd_friend_sm_inject_event (event);
+    gd1_mgmt_friend_rsp rsp = {
+        {0},
+    };
+    int ret = -1;
+    glusterd_friend_sm_event_t *event = NULL;
+    glusterd_friend_sm_event_type_t event_type = GD_FRIEND_EVENT_NONE;
+    glusterd_peerinfo_t *peerinfo = NULL;
+    int32_t op_ret = -1;
+    int32_t op_errno = EINVAL;
+    glusterd_probe_ctx_t *ctx = NULL;
+    glusterd_friend_update_ctx_t *ev_ctx = NULL;
+
+    if (-1 == req->rpc_status) {
+        rsp.op_ret = -1;
+        rsp.op_errno = EINVAL;
+        goto out;
+    }
+
+    ret = xdr_to_generic(*iov, &rsp, (xdrproc_t)xdr_gd1_mgmt_friend_rsp);
+    if (ret < 0) {
+        gf_msg("glusterd", GF_LOG_ERROR, errno, GD_MSG_RES_DECODE_FAIL,
+               "error");
+        rsp.op_ret = -1;
+        rsp.op_errno = EINVAL;
+        goto out;
+    }
+
+    op_ret = rsp.op_ret;
+    op_errno = rsp.op_errno;
+
+    gf_msg("glusterd", GF_LOG_INFO, 0, GD_MSG_RESPONSE_INFO,
+           "Received %s from uuid: %s, host: %s, port: %d",
+           (op_ret) ? "RJT" : "ACC", uuid_utoa(rsp.uuid), rsp.hostname,
+           rsp.port);
+
+    RCU_READ_LOCK;
+
+    peerinfo = glusterd_peerinfo_find(rsp.uuid, rsp.hostname);
+    if (peerinfo == NULL) {
+        RCU_READ_UNLOCK
+        ret = -1;
+        gf_msg("glusterd", GF_LOG_ERROR, 0, GD_MSG_RESP_FROM_UNKNOWN_PEER,
+               "received friend add response from"
+               " unknown peer uuid: %s",
+               uuid_utoa(rsp.uuid));
+        goto out;
+    }
+
+    if (op_ret)
+        event_type = GD_FRIEND_EVENT_RCVD_RJT;
+    else
+        event_type = GD_FRIEND_EVENT_RCVD_ACC;
+
+    ret = glusterd_friend_sm_new_event(event_type, &event);
+
+    if (ret) {
+        gf_msg("glusterd", GF_LOG_ERROR, 0, GD_MSG_EVENT_NEW_GET_FAIL,
+               "Unable to get event");
+        goto unlock;
+    }
+
+    ev_ctx = GF_CALLOC(1, sizeof(*ev_ctx), gf_gld_mt_friend_update_ctx_t);
+    if (!ev_ctx) {
+        ret = -1;
+        goto unlock;
+    }
+
+    gf_uuid_copy(ev_ctx->uuid, rsp.uuid);
+    ev_ctx->hostname = gf_strdup(rsp.hostname);
+
+    event->peername = gf_strdup(peerinfo->hostname);
+    gf_uuid_copy(event->peerid, peerinfo->uuid);
+    event->ctx = ev_ctx;
+    ret = glusterd_friend_sm_inject_event(event);
 
 unlock:
-        rcu_read_unlock ();
+    RCU_READ_UNLOCK;
 out:
-        ctx = ((call_frame_t *)myframe)->local;
-        ((call_frame_t *)myframe)->local = NULL;
-
-        GF_ASSERT (ctx);
-
-        if (ctx->req)//reverse probe doesn't have req
-                ret = glusterd_xfer_cli_probe_resp (ctx->req, op_ret, op_errno,
-                                                    NULL, ctx->hostname,
-                                                    ctx->port, ctx->dict);
-        if (!ret) {
-                glusterd_friend_sm ();
-                glusterd_op_sm ();
-        }
-        if (ctx)
-                glusterd_destroy_probe_ctx (ctx);
-        free (rsp.hostname);//malloced by xdr
-        GLUSTERD_STACK_DESTROY (((call_frame_t *)myframe));
-        return ret;
+    ctx = ((call_frame_t *)myframe)->local;
+    ((call_frame_t *)myframe)->local = NULL;
+
+    if (ctx && ctx->req) {
+        /*reverse probe doesn't have req*/
+        ret = glusterd_xfer_cli_probe_resp(ctx->req, op_ret, op_errno, NULL,
+                                           ctx->hostname, ctx->port, ctx->dict);
+    }
+    if (!ret) {
+        glusterd_friend_sm();
+        glusterd_op_sm();
+    }
+
+    if (ctx)
+        glusterd_destroy_probe_ctx(ctx);
+    free(rsp.hostname);  // malloced by xdr
+    GLUSTERD_STACK_DESTROY(((call_frame_t *)myframe));
+    return ret;
 }
 
 int
-glusterd_friend_add_cbk (struct rpc_req *req, struct iovec *iov,
-                    int count, void *myframe)
+glusterd_friend_add_cbk(struct rpc_req *req, struct iovec *iov, int count,
+                        void *myframe)
 {
-        return glusterd_big_locked_cbk (req, iov, count, myframe,
-                                        __glusterd_friend_add_cbk);
+    return glusterd_big_locked_cbk(req, iov, count, myframe,
+                                   __glusterd_friend_add_cbk);
 }
 
 int
-__glusterd_friend_remove_cbk (struct rpc_req * req, struct iovec *iov,
-                               int count, void *myframe)
+__glusterd_friend_remove_cbk(struct rpc_req *req, struct iovec *iov, int count,
+                             void *myframe)
 {
-        gd1_mgmt_friend_rsp             rsp   = {{0},};
-        glusterd_conf_t                 *conf = NULL;
-        int                             ret   = -1;
-        glusterd_friend_sm_event_t      *event = NULL;
-        glusterd_friend_sm_event_type_t event_type = GD_FRIEND_EVENT_NONE;
-        glusterd_peerinfo_t             *peerinfo = NULL;
-        int32_t                         op_ret = -1;
-        int32_t                         op_errno = -1;
-        glusterd_probe_ctx_t            *ctx = NULL;
-        gf_boolean_t                    move_sm_now = _gf_true;
-
-        conf  = THIS->private;
-        GF_ASSERT (conf);
-
-        ctx = ((call_frame_t *)myframe)->local;
-        ((call_frame_t *)myframe)->local = NULL;
-        GF_ASSERT (ctx);
-
-        if (-1 == req->rpc_status) {
-                rsp.op_ret   = -1;
-                rsp.op_errno = EINVAL;
-                move_sm_now = _gf_false;
-                goto inject;
-        }
-
-        ret = xdr_to_generic (*iov, &rsp, (xdrproc_t)xdr_gd1_mgmt_friend_rsp);
-        if (ret < 0) {
-                gf_log ("", GF_LOG_ERROR, "error");
-                rsp.op_ret   = -1;
-                rsp.op_errno = EINVAL;
-                goto respond;
-        }
-
-        op_ret = rsp.op_ret;
-        op_errno = rsp.op_errno;
-
-        gf_log ("glusterd", GF_LOG_INFO,
-                "Received %s from uuid: %s, host: %s, port: %d",
-                (op_ret)?"RJT":"ACC", uuid_utoa (rsp.uuid), rsp.hostname, rsp.port);
+    gd1_mgmt_friend_rsp rsp = {
+        {0},
+    };
+    glusterd_conf_t *conf = NULL;
+    int ret = -1;
+    glusterd_friend_sm_event_t *event = NULL;
+    glusterd_friend_sm_event_type_t event_type = GD_FRIEND_EVENT_NONE;
+    glusterd_peerinfo_t *peerinfo = NULL;
+    int32_t op_ret = -1;
+    int32_t op_errno = 0;
+    glusterd_probe_ctx_t *ctx = NULL;
+    gf_boolean_t move_sm_now = _gf_true;
+
+    conf = THIS->private;
+    GF_ASSERT(conf);
+
+    ctx = ((call_frame_t *)myframe)->local;
+    ((call_frame_t *)myframe)->local = NULL;
+    if (!ctx) {
+        gf_msg(THIS->name, GF_LOG_ERROR, 0, GD_MSG_EVENT_NEW_GET_FAIL,
+               "Unable to get glusterd probe context");
+        goto out;
+    }
+    if (-1 == req->rpc_status) {
+        rsp.op_ret = -1;
+        rsp.op_errno = EINVAL;
+        move_sm_now = _gf_false;
+        goto inject;
+    }
+
+    ret = xdr_to_generic(*iov, &rsp, (xdrproc_t)xdr_gd1_mgmt_friend_rsp);
+    if (ret < 0) {
+        gf_msg("glusterd", GF_LOG_ERROR, errno, GD_MSG_RES_DECODE_FAIL,
+               "error");
+        rsp.op_ret = -1;
+        rsp.op_errno = EINVAL;
+        goto respond;
+    }
+
+    op_ret = rsp.op_ret;
+    op_errno = rsp.op_errno;
+
+    gf_msg("glusterd", GF_LOG_INFO, 0, GD_MSG_RESPONSE_INFO,
+           "Received %s from uuid: %s, host: %s, port: %d",
+           (op_ret) ? "RJT" : "ACC", uuid_utoa(rsp.uuid), rsp.hostname,
+           rsp.port);
 
 inject:
-        rcu_read_lock ();
-
-        peerinfo = glusterd_peerinfo_find (rsp.uuid, ctx->hostname);
-        if (peerinfo == NULL) {
-                //can happen as part of rpc clnt connection cleanup
-                //when the frame timeout happens after 30 minutes
-                ret = -1;
-                goto unlock;
-        }
+    RCU_READ_LOCK;
 
-        event_type = GD_FRIEND_EVENT_REMOVE_FRIEND;
+    peerinfo = glusterd_peerinfo_find(rsp.uuid, ctx->hostname);
+    if (peerinfo == NULL) {
+        // can happen as part of rpc clnt connection cleanup
+        // when the frame timeout happens after 30 minutes
+        goto unlock;
+    }
 
-        ret = glusterd_friend_sm_new_event (event_type, &event);
+    event_type = GD_FRIEND_EVENT_REMOVE_FRIEND;
 
-        if (ret) {
-                gf_log ("glusterd", GF_LOG_ERROR,
-                         "Unable to get event");
-                goto unlock;
-        }
-        event->peername = gf_strdup (peerinfo->hostname);
-        gf_uuid_copy (event->peerid, peerinfo->uuid);
+    ret = glusterd_friend_sm_new_event(event_type, &event);
+
+    if (ret) {
+        gf_msg("glusterd", GF_LOG_ERROR, 0, GD_MSG_EVENT_NEW_GET_FAIL,
+               "Unable to get event");
+        goto unlock;
+    }
+    event->peername = gf_strdup(peerinfo->hostname);
+    gf_uuid_copy(event->peerid, peerinfo->uuid);
 
-        ret = glusterd_friend_sm_inject_event (event);
+    ret = glusterd_friend_sm_inject_event(event);
 
-        if (ret)
-                goto unlock;
+    if (ret)
+        goto unlock;
 
-        /*friend_sm would be moved on CLNT_DISCONNECT, consequently
-          cleaning up peerinfo. Else, we run the risk of triggering
-          a clnt_destroy within saved_frames_unwind.
-        */
-        op_ret = 0;
+    /*friend_sm would be moved on CLNT_DISCONNECT, consequently
+      cleaning up peerinfo. Else, we run the risk of triggering
+      a clnt_destroy within saved_frames_unwind.
+    */
+    op_ret = 0;
 
 unlock:
-        rcu_read_unlock ();
+    RCU_READ_UNLOCK;
 
 respond:
-        ret = glusterd_xfer_cli_deprobe_resp (ctx->req, op_ret, op_errno, NULL,
-                                              ctx->hostname, ctx->dict);
-        if (!ret && move_sm_now) {
-                glusterd_friend_sm ();
-                glusterd_op_sm ();
-        }
-
-        if (ctx) {
-                glusterd_broadcast_friend_delete (ctx->hostname, NULL);
-                glusterd_destroy_probe_ctx (ctx);
-        }
-
-        free (rsp.hostname);//malloced by xdr
-        GLUSTERD_STACK_DESTROY (((call_frame_t *)myframe));
-        return ret;
+    ret = glusterd_xfer_cli_deprobe_resp(ctx->req, op_ret, op_errno, NULL,
+                                         ctx->hostname, ctx->dict);
+    if (!ret && move_sm_now) {
+        glusterd_friend_sm();
+        glusterd_op_sm();
+    }
+
+    glusterd_broadcast_friend_delete(ctx->hostname, NULL);
+    glusterd_destroy_probe_ctx(ctx);
+out:
+    free(rsp.hostname);  // malloced by xdr
+    GLUSTERD_STACK_DESTROY(((call_frame_t *)myframe));
+    return ret;
 }
 
 int
-glusterd_friend_remove_cbk (struct rpc_req *req, struct iovec *iov,
-                    int count, void *myframe)
+glusterd_friend_remove_cbk(struct rpc_req *req, struct iovec *iov, int count,
+                           void *myframe)
 {
-        return glusterd_big_locked_cbk (req, iov, count, myframe,
-                                        __glusterd_friend_remove_cbk);
+    return glusterd_big_locked_cbk(req, iov, count, myframe,
+                                   __glusterd_friend_remove_cbk);
 }
 
 int32_t
-__glusterd_friend_update_cbk (struct rpc_req *req, struct iovec *iov,
-                              int count, void *myframe)
+__glusterd_friend_update_cbk(struct rpc_req *req, struct iovec *iov, int count,
+                             void *myframe)
 {
-        int                           ret    = -1;
-        gd1_mgmt_friend_update_rsp    rsp    = {{0}, };
-        xlator_t                      *this  = NULL;
-
-        GF_ASSERT (req);
-        this = THIS;
-
-        if (-1 == req->rpc_status) {
-                gf_log (this->name, GF_LOG_ERROR, "RPC Error");
-                goto out;
-        }
-
-        ret = xdr_to_generic (*iov, &rsp,
-                              (xdrproc_t)xdr_gd1_mgmt_friend_update_rsp);
-        if (ret < 0) {
-                gf_log (this->name, GF_LOG_ERROR, "Failed to serialize friend"
-                        " update repsonse");
-                goto out;
-        }
-
-        ret = 0;
+    int ret = -1;
+    gd1_mgmt_friend_update_rsp rsp = {
+        {0},
+    };
+    xlator_t *this = NULL;
+
+    GF_ASSERT(req);
+    this = THIS;
+
+    if (-1 == req->rpc_status) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_RPC_FAILURE, "RPC Error");
+        goto out;
+    }
+
+    ret = xdr_to_generic(*iov, &rsp, (xdrproc_t)xdr_gd1_mgmt_friend_update_rsp);
+    if (ret < 0) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_RES_DECODE_FAIL,
+               "Failed to serialize friend"
+               " update response");
+        goto out;
+    }
+
+    ret = 0;
 out:
-        gf_log (this->name, GF_LOG_INFO, "Received %s from uuid: %s",
-                (ret)?"RJT":"ACC", uuid_utoa (rsp.uuid));
+    gf_msg(this->name, GF_LOG_INFO, 0, GD_MSG_RESPONSE_INFO,
+           "Received %s from uuid: %s", (ret) ? "RJT" : "ACC",
+           uuid_utoa(rsp.uuid));
 
-        GLUSTERD_STACK_DESTROY (((call_frame_t *)myframe));
-        return ret;
+    GLUSTERD_STACK_DESTROY(((call_frame_t *)myframe));
+    return ret;
 }
 
 int
-glusterd_friend_update_cbk (struct rpc_req *req, struct iovec *iov,
-                    int count, void *myframe)
+glusterd_friend_update_cbk(struct rpc_req *req, struct iovec *iov, int count,
+                           void *myframe)
 {
-        return glusterd_big_locked_cbk (req, iov, count, myframe,
-                                        __glusterd_friend_update_cbk);
+    return glusterd_big_locked_cbk(req, iov, count, myframe,
+                                   __glusterd_friend_update_cbk);
 }
 
 int32_t
-__glusterd_cluster_lock_cbk (struct rpc_req *req, struct iovec *iov,
-                              int count, void *myframe)
+__glusterd_cluster_lock_cbk(struct rpc_req *req, struct iovec *iov, int count,
+                            void *myframe)
 {
-        gd1_mgmt_cluster_lock_rsp     rsp   = {{0},};
-        int                           ret   = -1;
-        int32_t                       op_ret = -1;
-        glusterd_op_sm_event_type_t   event_type = GD_OP_EVENT_NONE;
-        glusterd_peerinfo_t           *peerinfo = NULL;
-        xlator_t                      *this = NULL;
-        uuid_t                        *txn_id = NULL;
-        glusterd_conf_t               *priv = NULL;
-        char                          *err_str = NULL;
-
-        this = THIS;
-        GF_ASSERT (this);
-        priv = this->private;
-        GF_ASSERT (priv);
-        GF_ASSERT (req);
-
-        txn_id = &priv->global_txn_id;
-
-        if (-1 == req->rpc_status) {
-                gf_log (this->name, GF_LOG_ERROR, "Lock response is not "
-                        "received from one of the peer");
-                err_str = "Lock response is not received from one of the peer";
-                glusterd_set_opinfo (err_str, ENETRESET, -1);
-                event_type = GD_OP_EVENT_RCVD_RJT;
-                goto out;
-        }
-
-        ret = xdr_to_generic (*iov, &rsp,
-                              (xdrproc_t)xdr_gd1_mgmt_cluster_lock_rsp);
-        if (ret < 0) {
-                gf_log (this->name, GF_LOG_ERROR, "Failed to decode "
-                        "cluster lock response received from peer");
-                err_str = "Failed to decode cluster lock response received from"
-                          " peer";
-                glusterd_set_opinfo (err_str, EINVAL, -1);
-                event_type = GD_OP_EVENT_RCVD_RJT;
-                goto out;
-        }
-
-        op_ret = rsp.op_ret;
-
-        gf_log (this->name, (op_ret) ? GF_LOG_ERROR : GF_LOG_DEBUG,
-                "Received lock %s from uuid: %s", (op_ret) ? "RJT" : "ACC",
-                uuid_utoa (rsp.uuid));
-
-        rcu_read_lock ();
-        ret = (glusterd_peerinfo_find (rsp.uuid, NULL) == NULL);
-        rcu_read_unlock ();
+    gd1_mgmt_cluster_lock_rsp rsp = {
+        {0},
+    };
+    int ret = -1;
+    int32_t op_ret = -1;
+    glusterd_op_sm_event_type_t event_type = GD_OP_EVENT_NONE;
+    xlator_t *this = NULL;
+    uuid_t *txn_id = NULL;
+    glusterd_conf_t *priv = NULL;
+    char *err_str = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+    priv = this->private;
+    GF_ASSERT(priv);
+    GF_ASSERT(req);
+
+    txn_id = &priv->global_txn_id;
+
+    if (-1 == req->rpc_status) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_NO_LOCK_RESP_FROM_PEER,
+               "Lock response is not "
+               "received from one of the peer");
+        err_str = "Lock response is not received from one of the peer";
+        glusterd_set_opinfo(err_str, ENETRESET, -1);
+        event_type = GD_OP_EVENT_RCVD_RJT;
+        goto out;
+    }
+
+    ret = xdr_to_generic(*iov, &rsp, (xdrproc_t)xdr_gd1_mgmt_cluster_lock_rsp);
+    if (ret < 0) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_RES_DECODE_FAIL,
+               "Failed to decode "
+               "cluster lock response received from peer");
+        err_str =
+            "Failed to decode cluster lock response received from"
+            " peer";
+        glusterd_set_opinfo(err_str, EINVAL, -1);
+        event_type = GD_OP_EVENT_RCVD_RJT;
+        goto out;
+    }
+
+    op_ret = rsp.op_ret;
+
+    if (op_ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_LOCK_FROM_UUID_REJCT,
+               "Received lock RJT from uuid: %s", uuid_utoa(rsp.uuid));
+    } else {
+        gf_msg_debug(this->name, 0, "Received lock ACC from uuid: %s",
+                     uuid_utoa(rsp.uuid));
+    }
+
+    RCU_READ_LOCK;
+    ret = (glusterd_peerinfo_find(rsp.uuid, NULL) == NULL);
+    RCU_READ_UNLOCK;
+
+    if (ret) {
+        gf_msg(this->name, GF_LOG_CRITICAL, 0, GD_MSG_RESP_FROM_UNKNOWN_PEER,
+               "cluster lock response received from unknown peer: %s."
+               "Ignoring response",
+               uuid_utoa(rsp.uuid));
+        err_str = "cluster lock response received from unknown peer";
+        goto out;
+    }
+
+    if (op_ret) {
+        event_type = GD_OP_EVENT_RCVD_RJT;
+        opinfo.op_ret = op_ret;
+        opinfo.op_errstr = gf_strdup(
+            "Another transaction could be in "
+            "progress. Please try again after"
+            " some time.");
+    } else {
+        event_type = GD_OP_EVENT_RCVD_ACC;
+    }
 
-        if (ret) {
-                gf_log (this->name, GF_LOG_CRITICAL,
-                        "cluster lock response received from unknown peer: %s."
-                        "Ignoring response", uuid_utoa (rsp.uuid));
-                err_str = "cluster lock response received from unknown peer";
-                goto out;
+out:
 
-        }
+    ret = glusterd_set_txn_opinfo(txn_id, &opinfo);
+    if (ret)
+        gf_msg(THIS->name, GF_LOG_ERROR, 0, GD_MSG_TRANS_OPINFO_SET_FAIL,
+               "Unable to set "
+               "transaction's opinfo");
 
-        if (op_ret) {
-                event_type = GD_OP_EVENT_RCVD_RJT;
-                opinfo.op_ret = op_ret;
-                opinfo.op_errstr = gf_strdup ("Another transaction could be in "
-                                              "progress. Please try again after"
-                                              " sometime.");
-        } else {
-                event_type = GD_OP_EVENT_RCVD_ACC;
-        }
+    ret = glusterd_op_sm_inject_event(event_type, txn_id, NULL);
 
-out:
-        ret = glusterd_op_sm_inject_event (event_type, txn_id, NULL);
-
-        if (!ret) {
-                glusterd_friend_sm ();
-                glusterd_op_sm ();
-        }
+    if (!ret) {
+        glusterd_friend_sm();
+        glusterd_op_sm();
+    }
 
-        GLUSTERD_STACK_DESTROY (((call_frame_t *)myframe));
-        return ret;
+    GLUSTERD_STACK_DESTROY(((call_frame_t *)myframe));
+    return ret;
 }
 
 int32_t
-glusterd_cluster_lock_cbk (struct rpc_req *req, struct iovec *iov,
-                              int count, void *myframe)
+glusterd_cluster_lock_cbk(struct rpc_req *req, struct iovec *iov, int count,
+                          void *myframe)
 {
-        return glusterd_big_locked_cbk (req, iov, count, myframe,
-                                        __glusterd_cluster_lock_cbk);
+    return glusterd_big_locked_cbk(req, iov, count, myframe,
+                                   __glusterd_cluster_lock_cbk);
 }
 
 void
-glusterd_set_opinfo (char *errstr, int32_t op_errno, int32_t op_ret)
+glusterd_set_opinfo(char *errstr, int32_t op_errno, int32_t op_ret)
 {
-        opinfo.op_errstr = gf_strdup (errstr);
-        opinfo.op_errno = op_errno;
-        opinfo.op_ret = op_ret;
+    opinfo.op_errstr = gf_strdup(errstr);
+    opinfo.op_errno = op_errno;
+    opinfo.op_ret = op_ret;
 }
 
 static int32_t
-glusterd_mgmt_v3_lock_peers_cbk_fn (struct rpc_req *req, struct iovec *iov,
-                                    int count, void *myframe)
+glusterd_mgmt_v3_lock_peers_cbk_fn(struct rpc_req *req, struct iovec *iov,
+                                   int count, void *myframe)
 {
-        gd1_mgmt_v3_lock_rsp          rsp   = {{0},};
-        int                           ret   = -1;
-        int32_t                       op_ret = -1;
-        glusterd_op_sm_event_type_t   event_type = GD_OP_EVENT_NONE;
-        xlator_t                      *this = NULL;
-        call_frame_t                  *frame  = NULL;
-        uuid_t                        *txn_id = NULL;
-        char                          *err_str = NULL;
-
-        this = THIS;
-        GF_ASSERT (this);
-        GF_ASSERT (req);
-
-        frame = myframe;
-        txn_id = frame->cookie;
-        frame->cookie = NULL;
-
-        if (-1 == req->rpc_status) {
-                gf_log (this->name, GF_LOG_ERROR, "Lock response is not "
-                        "received from one of the peer");
-                err_str = "Lock response is not received from one of the peer";
-                glusterd_set_opinfo (err_str, ENETRESET, -1);
-                event_type = GD_OP_EVENT_RCVD_RJT;
-                goto out;
-        }
-
-        ret = xdr_to_generic (*iov, &rsp,
-                              (xdrproc_t)xdr_gd1_mgmt_v3_lock_rsp);
-        if (ret < 0) {
-                gf_log (this->name, GF_LOG_ERROR, "Failed to decode "
-                        "mgmt_v3 lock response received from peer");
-                err_str = "Failed to decode mgmt_v3 lock response received from"
-                          " peer";
-                glusterd_set_opinfo (err_str, EINVAL, -1);
-                event_type = GD_OP_EVENT_RCVD_RJT;
-                goto out;
-        }
-
-        op_ret = rsp.op_ret;
-
-        txn_id = &rsp.txn_id;
-
-        gf_log (this->name, (op_ret) ? GF_LOG_ERROR : GF_LOG_DEBUG,
-                "Received mgmt_v3 lock %s from uuid: %s",
-                (op_ret) ? "RJT" : "ACC", uuid_utoa (rsp.uuid));
-
-        rcu_read_lock ();
-        ret = (glusterd_peerinfo_find (rsp.uuid, NULL) == NULL);
-        rcu_read_unlock ();
-
-        if (ret) {
-                gf_log (this->name, GF_LOG_CRITICAL,
-                        "mgmt_v3 lock response received "
-                        "from unknown peer: %s. Ignoring response",
-                         uuid_utoa (rsp.uuid));
-                goto out;
-        }
-
-        if (op_ret) {
-                event_type = GD_OP_EVENT_RCVD_RJT;
-                opinfo.op_ret = op_ret;
-                opinfo.op_errstr = gf_strdup ("Another transaction could be in "
-                                              "progress. Please try again after"
-                                              " sometime.");
-        } else {
-                event_type = GD_OP_EVENT_RCVD_ACC;
-        }
+    gd1_mgmt_v3_lock_rsp rsp = {
+        {0},
+    };
+    int ret = -1;
+    int32_t op_ret = -1;
+    glusterd_op_sm_event_type_t event_type = GD_OP_EVENT_NONE;
+    xlator_t *this = NULL;
+    call_frame_t *frame = NULL;
+    uuid_t *txn_id = NULL;
+    char *err_str = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+    GF_ASSERT(req);
+
+    frame = myframe;
+    txn_id = frame->cookie;
+    frame->cookie = NULL;
+
+    if (-1 == req->rpc_status) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_NO_LOCK_RESP_FROM_PEER,
+               "Lock response is not "
+               "received from one of the peer");
+        err_str = "Lock response is not received from one of the peer";
+        glusterd_set_opinfo(err_str, ENETRESET, -1);
+        event_type = GD_OP_EVENT_RCVD_RJT;
+        goto out;
+    }
+
+    ret = xdr_to_generic(*iov, &rsp, (xdrproc_t)xdr_gd1_mgmt_v3_lock_rsp);
+    if (ret < 0) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_RES_DECODE_FAIL,
+               "Failed to decode "
+               "mgmt_v3 lock response received from peer");
+        err_str =
+            "Failed to decode mgmt_v3 lock response received from"
+            " peer";
+        glusterd_set_opinfo(err_str, EINVAL, -1);
+        event_type = GD_OP_EVENT_RCVD_RJT;
+        goto out;
+    }
+
+    op_ret = rsp.op_ret;
+
+    txn_id = &rsp.txn_id;
+
+    if (op_ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_MGMTV3_LOCK_FROM_UUID_REJCT,
+               "Received mgmt_v3 lock RJT from uuid: %s", uuid_utoa(rsp.uuid));
+    } else {
+        gf_msg_debug(this->name, 0, "Received mgmt_v3 lock ACC from uuid: %s",
+                     uuid_utoa(rsp.uuid));
+    }
+
+    RCU_READ_LOCK;
+    ret = (glusterd_peerinfo_find(rsp.uuid, NULL) == NULL);
+    RCU_READ_UNLOCK;
+
+    if (ret) {
+        gf_msg(this->name, GF_LOG_CRITICAL, 0, GD_MSG_RESP_FROM_UNKNOWN_PEER,
+               "mgmt_v3 lock response received "
+               "from unknown peer: %s. Ignoring response",
+               uuid_utoa(rsp.uuid));
+        goto out;
+    }
+
+    if (op_ret) {
+        event_type = GD_OP_EVENT_RCVD_RJT;
+        opinfo.op_ret = op_ret;
+        opinfo.op_errstr = gf_strdup(
+            "Another transaction could be in "
+            "progress. Please try again after"
+            " some time.");
+    } else {
+        event_type = GD_OP_EVENT_RCVD_ACC;
+    }
 
 out:
-        ret = glusterd_op_sm_inject_event (event_type, txn_id, NULL);
-        if (!ret) {
-                glusterd_friend_sm ();
-                glusterd_op_sm ();
-        }
 
-        GF_FREE (frame->cookie);
-        GLUSTERD_STACK_DESTROY (frame);
-        return ret;
+    ret = glusterd_set_txn_opinfo(txn_id, &opinfo);
+    if (ret)
+        gf_msg(THIS->name, GF_LOG_ERROR, 0, GD_MSG_TRANS_OPINFO_SET_FAIL,
+               "Unable to set "
+               "transaction's opinfo");
+
+    ret = glusterd_op_sm_inject_event(event_type, txn_id, NULL);
+    if (!ret) {
+        glusterd_friend_sm();
+        glusterd_op_sm();
+    }
+
+    GF_FREE(frame->cookie);
+    GLUSTERD_STACK_DESTROY(frame);
+    return ret;
 }
 
 int32_t
-glusterd_mgmt_v3_lock_peers_cbk (struct rpc_req *req, struct iovec *iov,
-                                 int count, void *myframe)
+glusterd_mgmt_v3_lock_peers_cbk(struct rpc_req *req, struct iovec *iov,
+                                int count, void *myframe)
 {
-        return glusterd_big_locked_cbk (req, iov, count, myframe,
-                                        glusterd_mgmt_v3_lock_peers_cbk_fn);
+    return glusterd_big_locked_cbk(req, iov, count, myframe,
+                                   glusterd_mgmt_v3_lock_peers_cbk_fn);
 }
 
 static int32_t
-glusterd_mgmt_v3_unlock_peers_cbk_fn (struct rpc_req *req, struct iovec *iov,
-                                      int count, void *myframe)
+glusterd_mgmt_v3_unlock_peers_cbk_fn(struct rpc_req *req, struct iovec *iov,
+                                     int count, void *myframe)
 {
-        gd1_mgmt_v3_unlock_rsp        rsp   = {{0},};
-        int                           ret   = -1;
-        int32_t                       op_ret = -1;
-        glusterd_op_sm_event_type_t   event_type = GD_OP_EVENT_NONE;
-        xlator_t                      *this = NULL;
-        call_frame_t                  *frame = NULL;
-        uuid_t                        *txn_id = NULL;
-        char                          *err_str = NULL;
-
-        this = THIS;
-        GF_ASSERT (this);
-        GF_ASSERT (req);
-
-        frame = myframe;
-        txn_id = frame->cookie;
-        frame->cookie = NULL;
-
-        if (-1 == req->rpc_status) {
-                err_str = "Unlock response not received from one of the peer.";
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_CLUSTER_UNLOCK_FAILED,
-                        "UnLock response is not received from one of the peer");
-                glusterd_set_opinfo (err_str, 0, 0);
-                event_type = GD_OP_EVENT_RCVD_RJT;
-                goto out;
-        }
-
-        ret = xdr_to_generic (*iov, &rsp,
-                              (xdrproc_t)xdr_gd1_mgmt_v3_unlock_rsp);
-        if (ret < 0) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_CLUSTER_UNLOCK_FAILED,
-                        "Failed to decode mgmt_v3 unlock response received from"
-                        "peer");
-                err_str = "Failed to decode mgmt_v3 unlock response received "
-                          "from peer";
-                glusterd_set_opinfo (err_str, 0, 0);
-                event_type = GD_OP_EVENT_RCVD_RJT;
-                goto out;
-        }
-
-        op_ret = rsp.op_ret;
-
-        txn_id = &rsp.txn_id;
-
-        gf_log (this->name, (op_ret) ? GF_LOG_ERROR : GF_LOG_DEBUG,
-                "Received mgmt_v3 unlock %s from uuid: %s",
-                (op_ret) ? "RJT" : "ACC",
-                uuid_utoa (rsp.uuid));
-
-        rcu_read_lock ();
-        ret = (glusterd_peerinfo_find (rsp.uuid, NULL) == NULL);
-        rcu_read_unlock ();
+    gd1_mgmt_v3_unlock_rsp rsp = {
+        {0},
+    };
+    int ret = -1;
+    int32_t op_ret = -1;
+    glusterd_op_sm_event_type_t event_type = GD_OP_EVENT_NONE;
+    xlator_t *this = NULL;
+    call_frame_t *frame = NULL;
+    uuid_t *txn_id = NULL;
+    char *err_str = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+    GF_ASSERT(req);
+
+    frame = myframe;
+    txn_id = frame->cookie;
+    frame->cookie = NULL;
+
+    if (-1 == req->rpc_status) {
+        err_str = "Unlock response not received from one of the peer.";
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_CLUSTER_UNLOCK_FAILED,
+               "UnLock response is not received from one of the peer");
+        glusterd_set_opinfo(err_str, 0, 0);
+        event_type = GD_OP_EVENT_RCVD_RJT;
+        goto out;
+    }
+
+    ret = xdr_to_generic(*iov, &rsp, (xdrproc_t)xdr_gd1_mgmt_v3_unlock_rsp);
+    if (ret < 0) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_CLUSTER_UNLOCK_FAILED,
+               "Failed to decode mgmt_v3 unlock response received from"
+               "peer");
+        err_str =
+            "Failed to decode mgmt_v3 unlock response received "
+            "from peer";
+        glusterd_set_opinfo(err_str, 0, 0);
+        event_type = GD_OP_EVENT_RCVD_RJT;
+        goto out;
+    }
+
+    op_ret = rsp.op_ret;
+
+    txn_id = &rsp.txn_id;
+
+    if (op_ret) {
+        gf_msg(
+            this->name, GF_LOG_ERROR, 0, GD_MSG_MGMTV3_UNLOCK_FROM_UUID_REJCT,
+            "Received mgmt_v3 unlock RJT from uuid: %s", uuid_utoa(rsp.uuid));
+    } else {
+        gf_msg_debug(this->name, 0, "Received mgmt_v3 unlock ACC from uuid: %s",
+                     uuid_utoa(rsp.uuid));
+    }
+
+    RCU_READ_LOCK;
+    ret = (glusterd_peerinfo_find(rsp.uuid, NULL) == NULL);
+    RCU_READ_UNLOCK;
+
+    if (ret) {
+        gf_msg(this->name, GF_LOG_CRITICAL, 0, GD_MSG_CLUSTER_UNLOCK_FAILED,
+               "mgmt_v3 unlock response received "
+               "from unknown peer: %s. Ignoring response",
+               uuid_utoa(rsp.uuid));
+        goto out;
+    }
+
+    if (op_ret) {
+        event_type = GD_OP_EVENT_RCVD_RJT;
+        opinfo.op_ret = op_ret;
+        opinfo.op_errstr = gf_strdup(
+            "Another transaction could be in "
+            "progress. Please try again after"
+            " some time.");
+    } else {
+        event_type = GD_OP_EVENT_RCVD_ACC;
+    }
 
-        if (ret) {
-                gf_msg (this->name, GF_LOG_CRITICAL, 0,
-                        GD_MSG_CLUSTER_UNLOCK_FAILED,
-                        "mgmt_v3 unlock response received "
-                        "from unknown peer: %s. Ignoring response",
-                        uuid_utoa (rsp.uuid));
-                goto out;
-        }
+out:
 
-        if (op_ret) {
-                event_type = GD_OP_EVENT_RCVD_RJT;
-                opinfo.op_ret = op_ret;
-                opinfo.op_errstr = gf_strdup ("Another transaction could be in "
-                                              "progress. Please try again after"
-                                              " sometime.");
-        } else {
-                event_type = GD_OP_EVENT_RCVD_ACC;
-        }
+    ret = glusterd_set_txn_opinfo(txn_id, &opinfo);
+    if (ret)
+        gf_msg(THIS->name, GF_LOG_ERROR, 0, GD_MSG_TRANS_OPINFO_SET_FAIL,
+               "Unable to set "
+               "transaction's opinfo");
 
-out:
-        ret = glusterd_op_sm_inject_event (event_type, txn_id, NULL);
+    ret = glusterd_op_sm_inject_event(event_type, txn_id, NULL);
 
-        if (!ret) {
-                glusterd_friend_sm ();
-                glusterd_op_sm ();
-        }
+    if (!ret) {
+        glusterd_friend_sm();
+        glusterd_op_sm();
+    }
 
-        GF_FREE (frame->cookie);
-        GLUSTERD_STACK_DESTROY (frame);
-        return ret;
+    GF_FREE(frame->cookie);
+    GLUSTERD_STACK_DESTROY(frame);
+    return ret;
 }
 
 int32_t
-glusterd_mgmt_v3_unlock_peers_cbk (struct rpc_req *req, struct iovec *iov,
-                                   int count, void *myframe)
+glusterd_mgmt_v3_unlock_peers_cbk(struct rpc_req *req, struct iovec *iov,
+                                  int count, void *myframe)
 {
-        return glusterd_big_locked_cbk (req, iov, count, myframe,
-                                        glusterd_mgmt_v3_unlock_peers_cbk_fn);
+    return glusterd_big_locked_cbk(req, iov, count, myframe,
+                                   glusterd_mgmt_v3_unlock_peers_cbk_fn);
 }
 
 int32_t
-__glusterd_cluster_unlock_cbk (struct rpc_req *req, struct iovec *iov,
-                                 int count, void *myframe)
+__glusterd_cluster_unlock_cbk(struct rpc_req *req, struct iovec *iov, int count,
+                              void *myframe)
 {
-        gd1_mgmt_cluster_lock_rsp     rsp   = {{0},};
-        int                           ret   = -1;
-        int32_t                       op_ret = -1;
-        glusterd_op_sm_event_type_t   event_type = GD_OP_EVENT_NONE;
-        glusterd_peerinfo_t           *peerinfo = NULL;
-        xlator_t                      *this = NULL;
-        uuid_t                        *txn_id = NULL;
-        glusterd_conf_t               *priv = NULL;
-        char                          *err_str = NULL;
-
-        this = THIS;
-        GF_ASSERT (this);
-        priv = this->private;
-        GF_ASSERT (priv);
-        GF_ASSERT (req);
-
-        txn_id = &priv->global_txn_id;
-
-        if (-1 == req->rpc_status) {
-                err_str = "Unlock response not received from one of the peer.";
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_CLUSTER_UNLOCK_FAILED,
-                        "UnLock response is not received from one of the peer");
-                glusterd_set_opinfo (err_str, 0, 0);
-                event_type = GD_OP_EVENT_RCVD_RJT;
-                goto out;
-        }
-
-        ret = xdr_to_generic (*iov, &rsp,
-                              (xdrproc_t)xdr_gd1_mgmt_cluster_unlock_rsp);
-        if (ret < 0) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_CLUSTER_UNLOCK_FAILED,
-                        "Failed to decode unlock response received from peer");
-                err_str = "Failed to decode cluster unlock response received "
-                          "from peer";
-                glusterd_set_opinfo (err_str, 0, 0);
-                event_type = GD_OP_EVENT_RCVD_RJT;
-                goto out;
-        }
-
-        op_ret = rsp.op_ret;
-
-        gf_log (this->name, (op_ret) ? GF_LOG_ERROR : GF_LOG_DEBUG,
-                "Received unlock %s from uuid: %s",
-                (op_ret)?"RJT":"ACC", uuid_utoa (rsp.uuid));
-
-        rcu_read_lock ();
-        ret = (glusterd_peerinfo_find (rsp.uuid, NULL) == NULL);
-        rcu_read_unlock ();
+    gd1_mgmt_cluster_lock_rsp rsp = {
+        {0},
+    };
+    int ret = -1;
+    int32_t op_ret = -1;
+    glusterd_op_sm_event_type_t event_type = GD_OP_EVENT_NONE;
+    xlator_t *this = NULL;
+    uuid_t *txn_id = NULL;
+    glusterd_conf_t *priv = NULL;
+    char *err_str = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+    priv = this->private;
+    GF_ASSERT(priv);
+    GF_ASSERT(req);
+
+    txn_id = &priv->global_txn_id;
+
+    if (-1 == req->rpc_status) {
+        err_str = "Unlock response not received from one of the peer.";
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_CLUSTER_UNLOCK_FAILED,
+               "UnLock response is not received from one of the peer");
+        glusterd_set_opinfo(err_str, 0, 0);
+        event_type = GD_OP_EVENT_RCVD_RJT;
+        goto out;
+    }
+
+    ret = xdr_to_generic(*iov, &rsp,
+                         (xdrproc_t)xdr_gd1_mgmt_cluster_unlock_rsp);
+    if (ret < 0) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_CLUSTER_UNLOCK_FAILED,
+               "Failed to decode unlock response received from peer");
+        err_str =
+            "Failed to decode cluster unlock response received "
+            "from peer";
+        glusterd_set_opinfo(err_str, 0, 0);
+        event_type = GD_OP_EVENT_RCVD_RJT;
+        goto out;
+    }
+
+    op_ret = rsp.op_ret;
+
+    if (op_ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_UNLOCK_FROM_UUID_REJCT,
+               "Received unlock RJT from uuid: %s", uuid_utoa(rsp.uuid));
+    } else {
+        gf_msg_debug(this->name, 0, "Received unlock ACC from uuid: %s",
+                     uuid_utoa(rsp.uuid));
+    }
+
+    RCU_READ_LOCK;
+    ret = (glusterd_peerinfo_find(rsp.uuid, NULL) == NULL);
+    RCU_READ_UNLOCK;
+
+    if (ret) {
+        gf_msg(this->name, GF_LOG_CRITICAL, 0, GD_MSG_CLUSTER_UNLOCK_FAILED,
+               "Unlock response received from unknown peer %s",
+               uuid_utoa(rsp.uuid));
+        goto out;
+    }
+
+    if (op_ret) {
+        event_type = GD_OP_EVENT_RCVD_RJT;
+        opinfo.op_ret = op_ret;
+    } else {
+        event_type = GD_OP_EVENT_RCVD_ACC;
+    }
 
-        if (ret) {
-                gf_msg (this->name, GF_LOG_CRITICAL, 0,
-                        GD_MSG_CLUSTER_UNLOCK_FAILED,
-                        "Unlock response received from unknown peer %s",
-                        uuid_utoa (rsp.uuid));
-                goto out;
-        }
+out:
 
-        if (op_ret) {
-                event_type = GD_OP_EVENT_RCVD_RJT;
-                opinfo.op_ret = op_ret;
-        } else {
-                event_type = GD_OP_EVENT_RCVD_ACC;
-        }
+    ret = glusterd_set_txn_opinfo(txn_id, &opinfo);
+    if (ret)
+        gf_msg(THIS->name, GF_LOG_ERROR, 0, GD_MSG_TRANS_OPINFO_SET_FAIL,
+               "Unable to set "
+               "transaction's opinfo");
 
-out:
-        ret = glusterd_op_sm_inject_event (event_type, txn_id, NULL);
+    ret = glusterd_op_sm_inject_event(event_type, txn_id, NULL);
 
-        if (!ret) {
-                glusterd_friend_sm ();
-                glusterd_op_sm ();
-        }
+    if (!ret) {
+        glusterd_friend_sm();
+        glusterd_op_sm();
+    }
 
-        GLUSTERD_STACK_DESTROY (((call_frame_t *)myframe));
-        return ret;
+    GLUSTERD_STACK_DESTROY(((call_frame_t *)myframe));
+    return ret;
 }
 
 int32_t
-glusterd_cluster_unlock_cbk (struct rpc_req *req, struct iovec *iov,
-                                 int count, void *myframe)
+glusterd_cluster_unlock_cbk(struct rpc_req *req, struct iovec *iov, int count,
+                            void *myframe)
 {
-        return glusterd_big_locked_cbk (req, iov, count, myframe,
-                                        __glusterd_cluster_unlock_cbk);
+    return glusterd_big_locked_cbk(req, iov, count, myframe,
+                                   __glusterd_cluster_unlock_cbk);
 }
 
 int32_t
-__glusterd_stage_op_cbk (struct rpc_req *req, struct iovec *iov,
-                          int count, void *myframe)
+__glusterd_stage_op_cbk(struct rpc_req *req, struct iovec *iov, int count,
+                        void *myframe)
 {
-        gd1_mgmt_stage_op_rsp         rsp   = {{0},};
-        int                           ret   = -1;
-        int32_t                       op_ret = -1;
-        glusterd_op_sm_event_type_t   event_type = GD_OP_EVENT_NONE;
-        glusterd_peerinfo_t           *peerinfo = NULL;
-        dict_t                        *dict   = NULL;
-        char                          err_str[2048] = {0};
-        char                          *peer_str = NULL;
-        xlator_t                      *this = NULL;
-        glusterd_conf_t               *priv = NULL;
-        uuid_t                        *txn_id = NULL;
-
-        this = THIS;
-        GF_ASSERT (this);
-        GF_ASSERT (req);
-        priv = this->private;
-        GF_ASSERT (priv);
-
-        txn_id = &priv->global_txn_id;
-
-        if (-1 == req->rpc_status) {
-                rsp.op_ret   = -1;
-                rsp.op_errno = EINVAL;
-                /* use standard allocation because to keep uniformity
-                   in freeing it */
-                rsp.op_errstr = strdup ("error");
-                goto out;
-        }
-
-        ret = xdr_to_generic (*iov, &rsp, (xdrproc_t)xdr_gd1_mgmt_stage_op_rsp);
+    gd1_mgmt_stage_op_rsp rsp = {
+        {0},
+    };
+    int ret = -1;
+    int32_t op_ret = -1;
+    glusterd_op_sm_event_type_t event_type = GD_OP_EVENT_NONE;
+    glusterd_peerinfo_t *peerinfo = NULL;
+    dict_t *dict = NULL;
+    char *peer_str = NULL;
+    xlator_t *this = NULL;
+    glusterd_conf_t *priv = NULL;
+    uuid_t *txn_id = NULL;
+    call_frame_t *frame = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+    GF_ASSERT(req);
+    priv = this->private;
+    GF_ASSERT(priv);
+    GF_ASSERT(myframe);
+
+    frame = myframe;
+    txn_id = frame->cookie;
+
+    if (-1 == req->rpc_status) {
+        rsp.op_ret = -1;
+        rsp.op_errno = EINVAL;
+        /* use standard allocation because to keep uniformity
+           in freeing it */
+        rsp.op_errstr = strdup("error");
+        goto out;
+    }
+
+    ret = xdr_to_generic(*iov, &rsp, (xdrproc_t)xdr_gd1_mgmt_stage_op_rsp);
+    if (ret < 0) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_RES_DECODE_FAIL,
+               "Failed to decode stage "
+               "response received from peer");
+        rsp.op_ret = -1;
+        rsp.op_errno = EINVAL;
+        /* use standard allocation because to keep uniformity
+           in freeing it */
+        rsp.op_errstr = strdup(
+            "Failed to decode stage response "
+            "received from peer.");
+        goto out;
+    }
+
+    if (rsp.dict.dict_len) {
+        /* Unserialize the dictionary */
+        dict = dict_new();
+
+        ret = dict_unserialize(rsp.dict.dict_val, rsp.dict.dict_len, &dict);
         if (ret < 0) {
-                gf_log (this->name, GF_LOG_ERROR, "Failed to decode stage "
-                        "response received from peer");
-                rsp.op_ret   = -1;
-                rsp.op_errno = EINVAL;
-                /* use standard allocation because to keep uniformity
-                   in freeing it */
-                rsp.op_errstr = strdup ("Failed to decode stage response "
-                                        "received from peer.");
-                goto out;
-        }
-
-        if (rsp.dict.dict_len) {
-                /* Unserialize the dictionary */
-                dict  = dict_new ();
-
-                ret = dict_unserialize (rsp.dict.dict_val,
-                                        rsp.dict.dict_len,
-                                        &dict);
-                if (ret < 0) {
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "failed to "
-                                "unserialize rsp-buffer to dictionary");
-			event_type = GD_OP_EVENT_RCVD_RJT;
-                        goto out;
-                } else {
-                        dict->extra_stdfree = rsp.dict.dict_val;
-                }
-        }
-
-out:
-        op_ret = rsp.op_ret;
-
-        gf_log (this->name, (op_ret) ? GF_LOG_ERROR : GF_LOG_DEBUG,
-                "Received stage %s from uuid: %s",
-                (op_ret) ? "RJT" : "ACC", uuid_utoa (rsp.uuid));
-
-        ret = dict_get_bin (dict, "transaction_id", (void **)&txn_id);
-        gf_log (this->name, GF_LOG_DEBUG, "transaction ID = %s",
-                uuid_utoa (*txn_id));
-
-        rcu_read_lock ();
-        peerinfo = glusterd_peerinfo_find (rsp.uuid, NULL);
-        if (peerinfo == NULL) {
-                gf_log (this->name, GF_LOG_CRITICAL, "Stage response received "
-                        "from unknown peer: %s. Ignoring response.",
-                        uuid_utoa (rsp.uuid));
-        }
-
-        if (op_ret) {
-                event_type = GD_OP_EVENT_RCVD_RJT;
-                opinfo.op_ret = op_ret;
-                if (strcmp ("", rsp.op_errstr)) {
-                        opinfo.op_errstr = gf_strdup (rsp.op_errstr);
-                } else {
-                        if (peerinfo)
-                                peer_str = peerinfo->hostname;
-                        else
-                                peer_str = uuid_utoa (rsp.uuid);
-                        snprintf (err_str, sizeof (err_str),
-                                  OPERRSTR_STAGE_FAIL, peer_str);
-                        opinfo.op_errstr = gf_strdup (err_str);
-                }
-                if (!opinfo.op_errstr)
-                        ret = -1;
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_UNSERIALIZE_FAIL,
+                   "failed to "
+                   "unserialize rsp-buffer to dictionary");
+            event_type = GD_OP_EVENT_RCVD_RJT;
+            goto out;
         } else {
-                event_type = GD_OP_EVENT_RCVD_ACC;
-        }
-
-        rcu_read_unlock ();
-
-        switch (rsp.op) {
-        case GD_OP_REPLACE_BRICK:
-                glusterd_rb_use_rsp_dict (NULL, dict);
-                break;
-        }
-
-        ret = glusterd_op_sm_inject_event (event_type, txn_id, NULL);
-
-        if (!ret) {
-                glusterd_friend_sm ();
-                glusterd_op_sm ();
+            dict->extra_stdfree = rsp.dict.dict_val;
         }
+    }
 
-        free (rsp.op_errstr); //malloced by xdr
-        if (dict) {
-                if (!dict->extra_stdfree && rsp.dict.dict_val)
-                        free (rsp.dict.dict_val); //malloced by xdr
-                dict_unref (dict);
+out:
+    op_ret = rsp.op_ret;
+
+    if (op_ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_STAGE_FROM_UUID_REJCT,
+               "Received stage RJT from uuid: %s", uuid_utoa(rsp.uuid));
+    } else {
+        gf_msg_debug(this->name, 0, "Received stage ACC from uuid: %s",
+                     uuid_utoa(rsp.uuid));
+    }
+
+    RCU_READ_LOCK;
+    peerinfo = glusterd_peerinfo_find(rsp.uuid, NULL);
+    if (peerinfo == NULL) {
+        gf_msg(this->name, GF_LOG_CRITICAL, 0, GD_MSG_RESP_FROM_UNKNOWN_PEER,
+               "Stage response received "
+               "from unknown peer: %s. Ignoring response.",
+               uuid_utoa(rsp.uuid));
+    }
+
+    if (op_ret) {
+        event_type = GD_OP_EVENT_RCVD_RJT;
+        opinfo.op_ret = op_ret;
+        if (strcmp("", rsp.op_errstr)) {
+            opinfo.op_errstr = gf_strdup(rsp.op_errstr);
         } else {
-                free (rsp.dict.dict_val); //malloced by xdr
-        }
-        GLUSTERD_STACK_DESTROY (((call_frame_t *)myframe));
-        return ret;
+            if (peerinfo)
+                peer_str = peerinfo->hostname;
+            else
+                peer_str = uuid_utoa(rsp.uuid);
+            char err_str[2048];
+            snprintf(err_str, sizeof(err_str), OPERRSTR_STAGE_FAIL, peer_str);
+            opinfo.op_errstr = gf_strdup(err_str);
+        }
+    } else {
+        event_type = GD_OP_EVENT_RCVD_ACC;
+    }
+
+    RCU_READ_UNLOCK;
+
+    ret = glusterd_set_txn_opinfo(txn_id, &opinfo);
+    if (ret)
+        gf_msg(THIS->name, GF_LOG_ERROR, 0, GD_MSG_TRANS_OPINFO_SET_FAIL,
+               "Unable to set "
+               "transaction's opinfo");
+
+    ret = glusterd_op_sm_inject_event(event_type, txn_id, NULL);
+
+    if (!ret) {
+        glusterd_friend_sm();
+        glusterd_op_sm();
+    }
+
+    free(rsp.op_errstr);  // malloced by xdr
+    if (dict) {
+        if (!dict->extra_stdfree && rsp.dict.dict_val)
+            free(rsp.dict.dict_val);  // malloced by xdr
+        dict_unref(dict);
+    } else {
+        free(rsp.dict.dict_val);  // malloced by xdr
+    }
+    GF_FREE(frame->cookie);
+    GLUSTERD_STACK_DESTROY(((call_frame_t *)myframe));
+    return ret;
 }
 
 int32_t
-glusterd_stage_op_cbk (struct rpc_req *req, struct iovec *iov,
-                          int count, void *myframe)
+glusterd_stage_op_cbk(struct rpc_req *req, struct iovec *iov, int count,
+                      void *myframe)
 {
-        return glusterd_big_locked_cbk (req, iov, count, myframe,
-                                        __glusterd_stage_op_cbk);
+    return glusterd_big_locked_cbk(req, iov, count, myframe,
+                                   __glusterd_stage_op_cbk);
 }
 
 int32_t
-__glusterd_commit_op_cbk (struct rpc_req *req, struct iovec *iov,
-                          int count, void *myframe)
+__glusterd_commit_op_cbk(struct rpc_req *req, struct iovec *iov, int count,
+                         void *myframe)
 {
-        gd1_mgmt_commit_op_rsp         rsp   = {{0},};
-        int                           ret   = -1;
-        int32_t                       op_ret = -1;
-        glusterd_op_sm_event_type_t   event_type = GD_OP_EVENT_NONE;
-        glusterd_peerinfo_t           *peerinfo = NULL;
-        dict_t                        *dict = NULL;
-        char                          err_str[2048] = {0};
-        char                          *peer_str = NULL;
-        xlator_t                      *this = NULL;
-        glusterd_conf_t               *priv = NULL;
-        uuid_t                        *txn_id = NULL;
-
-        this = THIS;
-        GF_ASSERT (this);
-        GF_ASSERT (req);
-        priv = this->private;
-        GF_ASSERT (priv);
-
-        txn_id = &priv->global_txn_id;
-
-        if (-1 == req->rpc_status) {
-                rsp.op_ret   = -1;
-                rsp.op_errno = EINVAL;
-                /* use standard allocation because to keep uniformity
-                   in freeing it */
-                rsp.op_errstr = strdup ("error");
-		event_type = GD_OP_EVENT_RCVD_RJT;
-                goto out;
-        }
-
-        ret = xdr_to_generic (*iov, &rsp, (xdrproc_t)xdr_gd1_mgmt_commit_op_rsp);
+    gd1_mgmt_commit_op_rsp rsp = {
+        {0},
+    };
+    int ret = -1;
+    int32_t op_ret = -1;
+    glusterd_op_sm_event_type_t event_type = GD_OP_EVENT_NONE;
+    glusterd_peerinfo_t *peerinfo = NULL;
+    dict_t *dict = NULL;
+    char *peer_str = NULL;
+    xlator_t *this = NULL;
+    glusterd_conf_t *priv = NULL;
+    uuid_t *txn_id = NULL;
+    glusterd_op_info_t txn_op_info = {
+        {0},
+    };
+    call_frame_t *frame = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+    GF_ASSERT(req);
+    priv = this->private;
+    GF_ASSERT(priv);
+    GF_ASSERT(myframe);
+
+    frame = myframe;
+    txn_id = frame->cookie;
+
+    if (-1 == req->rpc_status) {
+        rsp.op_ret = -1;
+        rsp.op_errno = EINVAL;
+        /* use standard allocation because to keep uniformity
+           in freeing it */
+        rsp.op_errstr = strdup("error");
+        event_type = GD_OP_EVENT_RCVD_RJT;
+        goto out;
+    }
+
+    ret = xdr_to_generic(*iov, &rsp, (xdrproc_t)xdr_gd1_mgmt_commit_op_rsp);
+    if (ret < 0) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_RES_DECODE_FAIL,
+               "Failed to decode commit "
+               "response received from peer");
+        rsp.op_ret = -1;
+        rsp.op_errno = EINVAL;
+        /* use standard allocation because to keep uniformity
+           in freeing it */
+        rsp.op_errstr = strdup(
+            "Failed to decode commit response "
+            "received from peer.");
+        event_type = GD_OP_EVENT_RCVD_RJT;
+        goto out;
+    }
+
+    if (rsp.dict.dict_len) {
+        /* Unserialize the dictionary */
+        dict = dict_new();
+
+        ret = dict_unserialize(rsp.dict.dict_val, rsp.dict.dict_len, &dict);
         if (ret < 0) {
-                gf_log (this->name, GF_LOG_ERROR, "Failed to decode commit "
-                        "response received from peer");
-                rsp.op_ret   = -1;
-                rsp.op_errno = EINVAL;
-                /* use standard allocation because to keep uniformity
-                   in freeing it */
-                rsp.op_errstr = strdup ("Failed to decode commit response "
-                                        "received from peer.");
-		event_type = GD_OP_EVENT_RCVD_RJT;
-                goto out;
-        }
-
-        if (rsp.dict.dict_len) {
-                /* Unserialize the dictionary */
-                dict  = dict_new ();
-
-                ret = dict_unserialize (rsp.dict.dict_val,
-                                        rsp.dict.dict_len,
-                                        &dict);
-                if (ret < 0) {
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "failed to "
-                                "unserialize rsp-buffer to dictionary");
-			event_type = GD_OP_EVENT_RCVD_RJT;
-                        goto out;
-                } else {
-                        dict->extra_stdfree = rsp.dict.dict_val;
-                }
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_UNSERIALIZE_FAIL,
+                   "failed to "
+                   "unserialize rsp-buffer to dictionary");
+            event_type = GD_OP_EVENT_RCVD_RJT;
+            goto out;
+        } else {
+            dict->extra_stdfree = rsp.dict.dict_val;
+        }
+    }
+
+    op_ret = rsp.op_ret;
+
+    if (op_ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_COMMIT_FROM_UUID_REJCT,
+               "Received commit RJT from uuid: %s", uuid_utoa(rsp.uuid));
+    } else {
+        gf_msg_debug(this->name, 0, "Received commit ACC from uuid: %s",
+                     uuid_utoa(rsp.uuid));
+    }
+
+    ret = glusterd_get_txn_opinfo(txn_id, &txn_op_info);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_TRANS_OPINFO_GET_FAIL,
+               "Failed to get txn_op_info "
+               "for txn_id = %s",
+               uuid_utoa(*txn_id));
+    }
+
+    RCU_READ_LOCK;
+    peerinfo = glusterd_peerinfo_find(rsp.uuid, NULL);
+    if (peerinfo == NULL) {
+        gf_msg(this->name, GF_LOG_CRITICAL, 0, GD_MSG_RESP_FROM_UNKNOWN_PEER,
+               "Commit response for "
+               "'Volume %s' received from unknown peer: %s",
+               gd_op_list[opinfo.op], uuid_utoa(rsp.uuid));
+    }
+
+    if (op_ret) {
+        event_type = GD_OP_EVENT_RCVD_RJT;
+        opinfo.op_ret = op_ret;
+        if (strcmp("", rsp.op_errstr)) {
+            opinfo.op_errstr = gf_strdup(rsp.op_errstr);
+        } else {
+            if (peerinfo)
+                peer_str = peerinfo->hostname;
+            else
+                peer_str = uuid_utoa(rsp.uuid);
+            char err_str[2048];
+            snprintf(err_str, sizeof(err_str), OPERRSTR_COMMIT_FAIL, peer_str);
+            opinfo.op_errstr = gf_strdup(err_str);
         }
-
-        op_ret = rsp.op_ret;
-
-        gf_log (this->name, (op_ret) ? GF_LOG_ERROR : GF_LOG_DEBUG,
-                "Received commit %s from uuid: %s",
-                (op_ret)?"RJT":"ACC", uuid_utoa (rsp.uuid));
-
-        ret = dict_get_bin (dict, "transaction_id", (void **)&txn_id);
-        gf_log (this->name, GF_LOG_DEBUG, "transaction ID = %s",
-                uuid_utoa (*txn_id));
-
-        rcu_read_lock ();
-        peerinfo = glusterd_peerinfo_find (rsp.uuid, NULL);
-        if (peerinfo == NULL) {
-                gf_log (this->name, GF_LOG_CRITICAL, "Commit response for "
-                        "'Volume %s' received from unknown peer: %s",
-                        gd_op_list[opinfo.op], uuid_utoa (rsp.uuid));
+        if (!opinfo.op_errstr) {
+            goto unlock;
         }
+    } else {
+        event_type = GD_OP_EVENT_RCVD_ACC;
+        GF_ASSERT(rsp.op == txn_op_info.op);
 
-        if (op_ret) {
-                event_type = GD_OP_EVENT_RCVD_RJT;
-                opinfo.op_ret = op_ret;
-                if (strcmp ("", rsp.op_errstr)) {
-                        opinfo.op_errstr = gf_strdup(rsp.op_errstr);
-                } else {
-                        if (peerinfo)
-                                peer_str = peerinfo->hostname;
-                        else
-                                peer_str = uuid_utoa (rsp.uuid);
-                        snprintf (err_str, sizeof (err_str),
-                                  OPERRSTR_COMMIT_FAIL, peer_str);
-                        opinfo.op_errstr = gf_strdup (err_str);
-                }
-                if (!opinfo.op_errstr) {
-                        ret = -1;
-                        goto unlock;
-                }
-        } else {
-                event_type = GD_OP_EVENT_RCVD_ACC;
-                switch (rsp.op) {
-                case GD_OP_REPLACE_BRICK:
-                        ret = glusterd_rb_use_rsp_dict (NULL, dict);
-                        if (ret)
-                                goto unlock;
-                break;
-
-                case GD_OP_SYNC_VOLUME:
-                        ret = glusterd_sync_use_rsp_dict (NULL, dict);
-                        if (ret)
-                                goto unlock;
-                break;
-
-                case GD_OP_PROFILE_VOLUME:
-                        ret = glusterd_profile_volume_use_rsp_dict (NULL, dict);
-                        if (ret)
-                                goto unlock;
-                break;
-
-                case GD_OP_GSYNC_SET:
-                        ret = glusterd_gsync_use_rsp_dict (NULL, dict, rsp.op_errstr);
-                        if (ret)
-                                goto unlock;
-                break;
-
-                case GD_OP_STATUS_VOLUME:
-                        ret = glusterd_volume_status_copy_to_op_ctx_dict (NULL, dict);
-                        if (ret)
-                                goto unlock;
+        switch (rsp.op) {
+            case GD_OP_PROFILE_VOLUME:
+                ret = glusterd_profile_volume_use_rsp_dict(txn_op_info.op_ctx,
+                                                           dict);
+                if (ret)
+                    goto unlock;
                 break;
 
-                case GD_OP_REBALANCE:
-                case GD_OP_DEFRAG_BRICK_VOLUME:
-                        ret = glusterd_volume_rebalance_use_rsp_dict (NULL, dict);
-                        if (ret)
-                                goto unlock;
+            case GD_OP_REBALANCE:
+            case GD_OP_DEFRAG_BRICK_VOLUME:
+                ret = glusterd_volume_rebalance_use_rsp_dict(txn_op_info.op_ctx,
+                                                             dict);
+                if (ret)
+                    goto unlock;
                 break;
 
-                case GD_OP_HEAL_VOLUME:
-                        ret = glusterd_volume_heal_use_rsp_dict (NULL, dict);
-                        if (ret)
-                                goto unlock;
-
+            default:
                 break;
-
-                default:
-                break;
-                }
         }
+    }
 unlock:
-        rcu_read_unlock ();
+    RCU_READ_UNLOCK;
 
 out:
-        ret = glusterd_op_sm_inject_event (event_type, txn_id, NULL);
-
-        if (!ret) {
-                glusterd_friend_sm ();
-                glusterd_op_sm ();
-        }
 
-        if (dict)
-                dict_unref (dict);
-        free (rsp.op_errstr); //malloced by xdr
-        GLUSTERD_STACK_DESTROY (((call_frame_t *)myframe));
-        return ret;
+    ret = glusterd_set_txn_opinfo(txn_id, &opinfo);
+    if (ret)
+        gf_msg(THIS->name, GF_LOG_ERROR, 0, GD_MSG_TRANS_OPINFO_SET_FAIL,
+               "Unable to set "
+               "transaction's opinfo");
+
+    ret = glusterd_op_sm_inject_event(event_type, txn_id, NULL);
+
+    if (!ret) {
+        glusterd_friend_sm();
+        glusterd_op_sm();
+    }
+
+    if (dict)
+        dict_unref(dict);
+    free(rsp.op_errstr);  // malloced by xdr
+    GF_FREE(frame->cookie);
+    GLUSTERD_STACK_DESTROY(((call_frame_t *)myframe));
+    return ret;
 }
 
 int32_t
-glusterd_commit_op_cbk (struct rpc_req *req, struct iovec *iov,
-                          int count, void *myframe)
+glusterd_commit_op_cbk(struct rpc_req *req, struct iovec *iov, int count,
+                       void *myframe)
 {
-        return glusterd_big_locked_cbk (req, iov, count, myframe,
-                                        __glusterd_commit_op_cbk);
+    return glusterd_big_locked_cbk(req, iov, count, myframe,
+                                   __glusterd_commit_op_cbk);
 }
 
 int32_t
-glusterd_rpc_probe (call_frame_t *frame, xlator_t *this,
-                    void *data)
+glusterd_rpc_probe(call_frame_t *frame, xlator_t *this, void *data)
 {
-        gd1_mgmt_probe_req      req = {{0},};
-        int                     ret = 0;
-        int                     port = 0;
-        char                    *hostname = NULL;
-        glusterd_peerinfo_t     *peerinfo = NULL;
-        glusterd_conf_t         *priv = NULL;
-        dict_t                  *dict = NULL;
-
-        if (!frame || !this ||  !data) {
-                ret = -1;
-                goto out;
-        }
-
-        dict = data;
-        priv = this->private;
-
-        GF_ASSERT (priv);
-        ret = dict_get_str (dict, "hostname", &hostname);
-        if (ret)
-                goto out;
-        ret = dict_get_int32 (dict, "port", &port);
-        if (ret)
-                port = GF_DEFAULT_BASE_PORT;
-
-        ret = dict_get_ptr (dict, "peerinfo", VOID (&peerinfo));
-        if (ret)
-                goto out;
-
-        gf_uuid_copy (req.uuid, MY_UUID);
-        req.hostname = gf_strdup (hostname);
-        req.port = port;
-
-        ret = glusterd_submit_request (peerinfo->rpc, &req, frame, peerinfo->peer,
-                                       GLUSTERD_PROBE_QUERY,
-                                       NULL, this, glusterd_probe_cbk,
-                                       (xdrproc_t)xdr_gd1_mgmt_probe_req);
+    gd1_mgmt_probe_req req = {
+        {0},
+    };
+    int ret = 0;
+    int port = 0;
+    char *hostname = NULL;
+    glusterd_peerinfo_t *peerinfo = NULL;
+    glusterd_conf_t *priv = NULL;
+    dict_t *dict = NULL;
+
+    if (!frame || !this || !data) {
+        gf_smsg(THIS->name, GF_LOG_ERROR, errno, GD_MSG_INVALID_ARGUMENT, NULL);
+        ret = -1;
+        goto out;
+    }
+
+    dict = data;
+    priv = this->private;
+
+    GF_ASSERT(priv);
+    ret = dict_get_strn(dict, "hostname", SLEN("hostname"), &hostname);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_GET_FAILED,
+                "Key=hostname", NULL);
+        goto out;
+    }
+    ret = dict_get_int32n(dict, "port", SLEN("port"), &port);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_DEBUG, errno, GD_MSG_DICT_GET_FAILED,
+                "Key=port", NULL);
+        port = GF_DEFAULT_BASE_PORT;
+    }
+
+    ret = dict_get_ptr(dict, "peerinfo", VOID(&peerinfo));
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_GET_FAILED,
+                "Key=peerinfo", NULL);
+        goto out;
+    }
+
+    gf_uuid_copy(req.uuid, MY_UUID);
+    req.hostname = gf_strdup(hostname);
+    req.port = port;
+
+    ret = glusterd_submit_request(
+        peerinfo->rpc, &req, frame, peerinfo->peer, GLUSTERD_PROBE_QUERY, NULL,
+        this, glusterd_probe_cbk, (xdrproc_t)xdr_gd1_mgmt_probe_req);
 
 out:
-        GF_FREE (req.hostname);
-        gf_log ("glusterd", GF_LOG_DEBUG, "Returning %d", ret);
-        return ret;
+    GF_FREE(req.hostname);
+    gf_msg_debug(this ? this->name : "glusterd", 0, "Returning %d", ret);
+    return ret;
 }
 
-
 int32_t
-glusterd_rpc_friend_add (call_frame_t *frame, xlator_t *this,
-                         void *data)
+glusterd_rpc_friend_add(call_frame_t *frame, xlator_t *this, void *data)
 {
-        gd1_mgmt_friend_req         req       = {{0},};
-        int                         ret       = 0;
-        glusterd_peerinfo_t        *peerinfo  = NULL;
-        glusterd_conf_t            *priv      = NULL;
-        glusterd_friend_sm_event_t *event     = NULL;
-        dict_t                     *peer_data = NULL;
-
-
-        if (!frame || !this || !data) {
-                ret = -1;
-                goto out;
-        }
-
-        event = data;
-        priv = this->private;
-
-        GF_ASSERT (priv);
-
-        rcu_read_lock ();
-
-        peerinfo = glusterd_peerinfo_find (event->peerid, event->peername);
-        if (!peerinfo) {
-                rcu_read_unlock ();
-                ret = -1;
-                gf_log (this->name, GF_LOG_ERROR, "Could not find peer %s(%s)",
-                        event->peername, uuid_utoa (event->peerid));
-                goto out;
-        }
-
-        gf_uuid_copy (req.uuid, MY_UUID);
-        req.hostname = gf_strdup (peerinfo->hostname);
-        req.port = peerinfo->port;
-
-        rcu_read_unlock ();
-
-        ret = glusterd_add_volumes_to_export_dict (&peer_data);
+    gd1_mgmt_friend_req req = {
+        {0},
+    };
+    int ret = 0;
+    glusterd_peerinfo_t *peerinfo = NULL;
+    glusterd_conf_t *priv = NULL;
+    glusterd_friend_sm_event_t *event = NULL;
+    dict_t *peer_data = NULL;
+
+    if (!frame || !this || !data) {
+        gf_smsg(THIS->name, GF_LOG_ERROR, errno, GD_MSG_INVALID_ARGUMENT, NULL);
+        ret = -1;
+        goto out;
+    }
+
+    event = data;
+    priv = this->private;
+
+    GF_ASSERT(priv);
+
+    RCU_READ_LOCK;
+
+    peerinfo = glusterd_peerinfo_find(event->peerid, event->peername);
+    if (!peerinfo) {
+        RCU_READ_UNLOCK;
+        ret = -1;
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_PEER_NOT_FOUND,
+               "Could not find peer %s(%s)", event->peername,
+               uuid_utoa(event->peerid));
+        goto out;
+    }
+
+    req.hostname = gf_strdup(peerinfo->hostname);
+    req.port = peerinfo->port;
+
+    RCU_READ_UNLOCK;
+
+    gf_uuid_copy(req.uuid, MY_UUID);
+
+    peer_data = dict_new();
+    if (!peer_data) {
+        gf_smsg(this->name, GF_LOG_ERROR, ENOMEM, GD_MSG_DICT_CREATE_FAIL,
+                NULL);
+        errno = ENOMEM;
+        goto out;
+    }
+
+    ret = dict_set_dynstr_with_alloc(peer_data, "hostname_in_cluster",
+                                     peerinfo->hostname);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_SET_FAILED,
+               "Unable to add hostname of the peer");
+        goto out;
+    }
+
+    if (priv->op_version >= GD_OP_VERSION_3_6_0) {
+        ret = glusterd_add_missed_snaps_to_export_dict(peer_data);
         if (ret) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "Unable to add list of volumes "
-                        "in the peer_data dict for handshake");
-                goto out;
+            gf_msg(this->name, GF_LOG_ERROR, 0,
+                   GD_MSG_MISSED_SNAP_LIST_STORE_FAIL,
+                   "Unable to add list of missed snapshots "
+                   "in the peer_data dict for handshake");
+            goto out;
         }
 
-        if (priv->op_version >= GD_OP_VERSION_3_6_0) {
-                ret = glusterd_add_missed_snaps_to_export_dict (peer_data);
-                if (ret) {
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "Unable to add list of missed snapshots "
-                                "in the peer_data dict for handshake");
-                        goto out;
-                }
-
-                ret = glusterd_add_snapshots_to_export_dict (peer_data);
-                if (ret) {
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "Unable to add list of snapshots "
-                                "in the peer_data dict for handshake");
-                        goto out;
-                }
+        ret = glusterd_add_snapshots_to_export_dict(peer_data);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_SNAP_LIST_SET_FAIL,
+                   "Unable to add list of snapshots "
+                   "in the peer_data dict for handshake");
+            goto out;
+        }
+    }
+
+    /* Don't add any key-value in peer_data dictionary after call this function
+     */
+    ret = glusterd_add_volumes_to_export_dict(peer_data, &req.vols.vols_val,
+                                              &req.vols.vols_len);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+               "Unable to add list of volumes "
+               "in the peer_data dict for handshake");
+        goto out;
+    }
+
+    if (!req.vols.vols_len) {
+        ret = dict_allocate_and_serialize(peer_data, &req.vols.vols_val,
+                                          &req.vols.vols_len);
+        if (ret) {
+            gf_smsg(this->name, GF_LOG_ERROR, errno,
+                    GD_MSG_DICT_ALLOC_AND_SERL_LENGTH_GET_FAIL, NULL);
+            goto out;
         }
+    }
 
-        ret = dict_allocate_and_serialize (peer_data, &req.vols.vols_val,
-                                           &req.vols.vols_len);
-        if (ret)
-                goto out;
-
-        ret = glusterd_submit_request (peerinfo->rpc, &req, frame, peerinfo->peer,
-                                       GLUSTERD_FRIEND_ADD,
-                                       NULL, this, glusterd_friend_add_cbk,
-                                       (xdrproc_t)xdr_gd1_mgmt_friend_req);
-
+    ret = glusterd_submit_request(
+        peerinfo->rpc, &req, frame, peerinfo->peer, GLUSTERD_FRIEND_ADD, NULL,
+        this, glusterd_friend_add_cbk, (xdrproc_t)xdr_gd1_mgmt_friend_req);
 
 out:
-        GF_FREE (req.vols.vols_val);
-        GF_FREE (req.hostname);
+    GF_FREE(req.vols.vols_val);
+    GF_FREE(req.hostname);
 
-        if (peer_data)
-                dict_unref (peer_data);
+    if (peer_data)
+        dict_unref(peer_data);
 
-        gf_log ("glusterd", GF_LOG_DEBUG, "Returning %d", ret);
-        return ret;
+    gf_msg_debug(this ? this->name : "glusterd", 0, "Returning %d", ret);
+    return ret;
 }
 
 int32_t
-glusterd_rpc_friend_remove (call_frame_t *frame, xlator_t *this,
-                            void *data)
+glusterd_rpc_friend_remove(call_frame_t *frame, xlator_t *this, void *data)
 {
-        gd1_mgmt_friend_req             req = {{0},};
-        int                             ret = 0;
-        glusterd_peerinfo_t             *peerinfo = NULL;
-        glusterd_conf_t                 *priv = NULL;
-        glusterd_friend_sm_event_t      *event = NULL;
-
-        if (!frame || !this || !data) {
-                ret = -1;
-                goto out;
-        }
-
-        event = data;
-        priv = this->private;
-
-        GF_ASSERT (priv);
-
-        rcu_read_lock ();
-
-        peerinfo = glusterd_peerinfo_find (event->peerid, event->peername);
-        if (!peerinfo) {
-                rcu_read_unlock ();
-                ret = -1;
-                gf_log (this->name, GF_LOG_ERROR, "Could not find peer %s(%s)",
-                        event->peername, uuid_utoa (event->peerid));
-                goto out;
-        }
-
-        gf_uuid_copy (req.uuid, MY_UUID);
-        req.hostname = gf_strdup (peerinfo->hostname);
-        req.port = peerinfo->port;
-
-        rcu_read_unlock ();
-
-        ret = glusterd_submit_request (peerinfo->rpc, &req, frame, peerinfo->peer,
-                                       GLUSTERD_FRIEND_REMOVE, NULL,
-                                       this, glusterd_friend_remove_cbk,
-                                       (xdrproc_t)xdr_gd1_mgmt_friend_req);
-
+    gd1_mgmt_friend_req req = {
+        {0},
+    };
+    int ret = 0;
+    glusterd_peerinfo_t *peerinfo = NULL;
+    glusterd_conf_t *priv = NULL;
+    glusterd_friend_sm_event_t *event = NULL;
+
+    if (!frame || !this || !data) {
+        ret = -1;
+        goto out;
+    }
+
+    event = data;
+    priv = this->private;
+
+    GF_ASSERT(priv);
+
+    RCU_READ_LOCK;
+
+    peerinfo = glusterd_peerinfo_find(event->peerid, event->peername);
+    if (!peerinfo) {
+        RCU_READ_UNLOCK;
+        ret = -1;
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_PEER_NOT_FOUND,
+               "Could not find peer %s(%s)", event->peername,
+               uuid_utoa(event->peerid));
+        goto out;
+    }
+
+    gf_uuid_copy(req.uuid, MY_UUID);
+    req.hostname = gf_strdup(peerinfo->hostname);
+    req.port = peerinfo->port;
+
+    ret = glusterd_submit_request(peerinfo->rpc, &req, frame, peerinfo->peer,
+                                  GLUSTERD_FRIEND_REMOVE, NULL, this,
+                                  glusterd_friend_remove_cbk,
+                                  (xdrproc_t)xdr_gd1_mgmt_friend_req);
+
+    RCU_READ_UNLOCK;
 out:
-        GF_FREE (req.hostname);
+    GF_FREE(req.hostname);
 
-        gf_log ("glusterd", GF_LOG_DEBUG, "Returning %d", ret);
-        return ret;
+    gf_msg_debug(this ? this->name : "glusterd", 0, "Returning %d", ret);
+    return ret;
 }
 
-
 int32_t
-glusterd_rpc_friend_update (call_frame_t *frame, xlator_t *this,
-                            void *data)
+glusterd_rpc_friend_update(call_frame_t *frame, xlator_t *this, void *data)
 {
-        gd1_mgmt_friend_update  req         = {{0},};
-        int                     ret         = 0;
-        glusterd_conf_t        *priv        = NULL;
-        dict_t                 *friends     = NULL;
-        call_frame_t           *dummy_frame = NULL;
-        glusterd_peerinfo_t    *peerinfo    = NULL;
-
-        priv = this->private;
-        GF_ASSERT (priv);
-
-        friends = data;
-        if (!friends)
-                goto out;
-
-        ret = dict_get_ptr (friends, "peerinfo", VOID(&peerinfo));
-        if (ret)
-                goto out;
-        /* Don't want to send the pointer over */
-        dict_del (friends, "peerinfo");
-
-        ret = dict_allocate_and_serialize (friends, &req.friends.friends_val,
-                                           &req.friends.friends_len);
-        if (ret)
-                goto out;
-
-        gf_uuid_copy (req.uuid, MY_UUID);
-
-        dummy_frame = create_frame (this, this->ctx->pool);
-        ret = glusterd_submit_request (peerinfo->rpc, &req, dummy_frame,
-                                       peerinfo->peer,
-                                       GLUSTERD_FRIEND_UPDATE, NULL,
-                                       this, glusterd_friend_update_cbk,
-                                       (xdrproc_t)xdr_gd1_mgmt_friend_update);
+    gd1_mgmt_friend_update req = {
+        {0},
+    };
+    int ret = 0;
+    glusterd_conf_t *priv = NULL;
+    dict_t *friends = NULL;
+    call_frame_t *dummy_frame = NULL;
+    glusterd_peerinfo_t *peerinfo = NULL;
+
+    priv = this->private;
+    GF_ASSERT(priv);
+
+    friends = data;
+    if (!friends)
+        goto out;
+
+    ret = dict_get_ptr(friends, "peerinfo", VOID(&peerinfo));
+    if (ret)
+        goto out;
+    /* Don't want to send the pointer over */
+    dict_deln(friends, "peerinfo", SLEN("peerinfo"));
+
+    ret = dict_allocate_and_serialize(friends, &req.friends.friends_val,
+                                      &req.friends.friends_len);
+    if (ret)
+        goto out;
+
+    gf_uuid_copy(req.uuid, MY_UUID);
+
+    dummy_frame = create_frame(this, this->ctx->pool);
+    ret = glusterd_submit_request(peerinfo->rpc, &req, dummy_frame,
+                                  peerinfo->peer, GLUSTERD_FRIEND_UPDATE, NULL,
+                                  this, glusterd_friend_update_cbk,
+                                  (xdrproc_t)xdr_gd1_mgmt_friend_update);
 
 out:
-        GF_FREE (req.friends.friends_val);
+    GF_FREE(req.friends.friends_val);
+
+    if (ret && dummy_frame)
+        STACK_DESTROY(dummy_frame->root);
 
-        gf_log ("glusterd", GF_LOG_DEBUG, "Returning %d", ret);
-        return ret;
+    gf_msg_debug(this ? this->name : "glusterd", 0, "Returning %d", ret);
+    return ret;
 }
 
 int32_t
-glusterd_cluster_lock (call_frame_t *frame, xlator_t *this,
-                          void *data)
+glusterd_cluster_lock(call_frame_t *frame, xlator_t *this, void *data)
 {
-        gd1_mgmt_cluster_lock_req       req = {{0},};
-        int                             ret = -1;
-        glusterd_peerinfo_t             *peerinfo = NULL;
-        glusterd_conf_t                 *priv = NULL;
-        call_frame_t                    *dummy_frame = NULL;
+    gd1_mgmt_cluster_lock_req req = {
+        {0},
+    };
+    int ret = -1;
+    glusterd_peerinfo_t *peerinfo = NULL;
+    glusterd_conf_t *priv = NULL;
+    call_frame_t *dummy_frame = NULL;
 
-        if (!this)
-                goto out;
+    if (!this)
+        goto out;
 
-        peerinfo = data;
+    peerinfo = data;
 
-        priv = this->private;
-        GF_ASSERT (priv);
+    priv = this->private;
+    GF_ASSERT(priv);
 
-        glusterd_get_uuid (&req.uuid);
+    glusterd_get_uuid(&req.uuid);
 
-        dummy_frame = create_frame (this, this->ctx->pool);
-        if (!dummy_frame)
-                goto out;
+    dummy_frame = create_frame(this, this->ctx->pool);
+    if (!dummy_frame)
+        goto out;
 
-        ret = glusterd_submit_request (peerinfo->rpc, &req, dummy_frame,
-                                       peerinfo->mgmt, GLUSTERD_MGMT_CLUSTER_LOCK,
-                                       NULL,
-                                       this, glusterd_cluster_lock_cbk,
-                                       (xdrproc_t)xdr_gd1_mgmt_cluster_lock_req);
+    ret = glusterd_submit_request(peerinfo->rpc, &req, dummy_frame,
+                                  peerinfo->mgmt, GLUSTERD_MGMT_CLUSTER_LOCK,
+                                  NULL, this, glusterd_cluster_lock_cbk,
+                                  (xdrproc_t)xdr_gd1_mgmt_cluster_lock_req);
 out:
-        gf_log (this->name, GF_LOG_DEBUG, "Returning %d", ret);
-        return ret;
+    gf_msg_debug(this ? this->name : "glusterd", 0, "Returning %d", ret);
+
+    if (ret && dummy_frame)
+        STACK_DESTROY(dummy_frame->root);
+    return ret;
 }
 
 int32_t
-glusterd_mgmt_v3_lock_peers (call_frame_t *frame, xlator_t *this,
-                             void *data)
+glusterd_mgmt_v3_lock_peers(call_frame_t *frame, xlator_t *this, void *data)
 {
-        gd1_mgmt_v3_lock_req             req         = {{0},};
-        int                              ret         = -1;
-        glusterd_peerinfo_t             *peerinfo    = NULL;
-        glusterd_conf_t                 *priv        = NULL;
-        dict_t                          *dict        = NULL;
-        uuid_t                          *txn_id      = NULL;
-
-        if (!this)
-                goto out;
-
-        dict = data;
-
-        priv = this->private;
-        GF_ASSERT (priv);
-
-        ret = dict_get_ptr (dict, "peerinfo", VOID (&peerinfo));
-        if (ret)
-                goto out;
-
-        //peerinfo should not be in payload
-        dict_del (dict, "peerinfo");
-
-        glusterd_get_uuid (&req.uuid);
-
-        ret = dict_allocate_and_serialize (dict, &req.dict.dict_val,
-                                           &req.dict.dict_len);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "Failed to serialize dict "
-                        "to request buffer");
-                goto out;
-        }
-
-        /* Sending valid transaction ID to peers */
-        ret = dict_get_bin (dict, "transaction_id",
-                            (void **)&txn_id);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR,
-                       "Failed to get transaction id.");
-                goto out;
-        } else {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "Transaction_id = %s", uuid_utoa (*txn_id));
-                gf_uuid_copy (req.txn_id, *txn_id);
-        }
-
-        if (!frame)
-                frame = create_frame (this, this->ctx->pool);
-
-        if (!frame) {
-                ret = -1;
-                goto out;
-        }
-        frame->cookie = GF_CALLOC (1, sizeof(uuid_t), gf_common_mt_uuid_t);
-        if (!frame->cookie) {
-                ret = -1;
-                goto out;
-        }
-        gf_uuid_copy (frame->cookie, req.txn_id);
-
-        ret = glusterd_submit_request (peerinfo->rpc, &req, frame,
-                                       peerinfo->mgmt_v3,
-                                       GLUSTERD_MGMT_V3_LOCK, NULL,
-                                       this, glusterd_mgmt_v3_lock_peers_cbk,
-                                       (xdrproc_t)xdr_gd1_mgmt_v3_lock_req);
+    gd1_mgmt_v3_lock_req req = {
+        {0},
+    };
+    int ret = -1;
+    glusterd_peerinfo_t *peerinfo = NULL;
+    glusterd_conf_t *priv = NULL;
+    dict_t *dict = NULL;
+    uuid_t *txn_id = NULL;
+
+    if (!this)
+        goto out;
+
+    dict = data;
+
+    priv = this->private;
+    GF_ASSERT(priv);
+
+    ret = dict_get_ptr(dict, "peerinfo", VOID(&peerinfo));
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_GET_FAILED,
+                "Key=peerinfo", NULL);
+        goto out;
+    }
+
+    // peerinfo should not be in payload
+    dict_deln(dict, "peerinfo", SLEN("peerinfo"));
+
+    glusterd_get_uuid(&req.uuid);
+
+    ret = dict_allocate_and_serialize(dict, &req.dict.dict_val,
+                                      &req.dict.dict_len);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_ERROR, 0,
+                GD_MSG_DICT_ALLOC_AND_SERL_LENGTH_GET_FAIL, NULL);
+        goto out;
+    }
+
+    /* Sending valid transaction ID to peers */
+    ret = dict_get_bin(dict, "transaction_id", (void **)&txn_id);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_TRANS_ID_GET_FAIL,
+               "Failed to get transaction id.");
+        goto out;
+    } else {
+        gf_msg_debug(this->name, 0, "Transaction_id = %s", uuid_utoa(*txn_id));
+        gf_uuid_copy(req.txn_id, *txn_id);
+    }
+
+    if (!frame)
+        frame = create_frame(this, this->ctx->pool);
+
+    if (!frame) {
+        ret = -1;
+        goto out;
+    }
+    frame->cookie = GF_MALLOC(sizeof(uuid_t), gf_common_mt_uuid_t);
+    if (!frame->cookie) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_NO_MEMORY, NULL);
+        ret = -1;
+        goto out;
+    }
+    gf_uuid_copy(frame->cookie, req.txn_id);
+
+    ret = glusterd_submit_request(peerinfo->rpc, &req, frame, peerinfo->mgmt_v3,
+                                  GLUSTERD_MGMT_V3_LOCK, NULL, this,
+                                  glusterd_mgmt_v3_lock_peers_cbk,
+                                  (xdrproc_t)xdr_gd1_mgmt_v3_lock_req);
 out:
-        gf_log (this->name, GF_LOG_DEBUG, "Returning %d", ret);
-        return ret;
+    gf_msg_debug(this ? this->name : "glusterd", 0, "Returning %d", ret);
+    if (dict)
+        dict_unref(dict);
+    if (req.dict.dict_val)
+        GF_FREE(req.dict.dict_val);
+    return ret;
 }
 
 int32_t
-glusterd_mgmt_v3_unlock_peers (call_frame_t *frame, xlator_t *this,
-                   void *data)
+glusterd_mgmt_v3_unlock_peers(call_frame_t *frame, xlator_t *this, void *data)
 {
-        gd1_mgmt_v3_unlock_req           req         = {{0},};
-        int                              ret         = -1;
-        glusterd_peerinfo_t             *peerinfo    = NULL;
-        glusterd_conf_t                 *priv        = NULL;
-        dict_t                          *dict        = NULL;
-        uuid_t                          *txn_id      = NULL;
-
-        if (!this)
-                goto out;
-
-        dict = data;
-
-        priv = this->private;
-        GF_ASSERT (priv);
-
-        ret = dict_get_ptr (dict, "peerinfo", VOID (&peerinfo));
-        if (ret)
-                goto out;
-
-        //peerinfo should not be in payload
-        dict_del (dict, "peerinfo");
-
-        glusterd_get_uuid (&req.uuid);
-
-        ret = dict_allocate_and_serialize (dict, &req.dict.dict_val,
-                                           &req.dict.dict_len);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "Failed to serialize dict "
-                        "to request buffer");
-                goto out;
-        }
-
-        /* Sending valid transaction ID to peers */
-        ret = dict_get_bin (dict, "transaction_id",
-                            (void **)&txn_id);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR,
-                       "Failed to get transaction id.");
-                goto out;
-        } else {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "Transaction_id = %s", uuid_utoa (*txn_id));
-                gf_uuid_copy (req.txn_id, *txn_id);
-        }
-
-        if (!frame)
-                frame = create_frame (this, this->ctx->pool);
-
-        if (!frame) {
-                ret = -1;
-                goto out;
-        }
-        frame->cookie = GF_CALLOC (1, sizeof(uuid_t), gf_common_mt_uuid_t);
-        if (!frame->cookie) {
-                ret = -1;
-                goto out;
-        }
-        gf_uuid_copy (frame->cookie, req.txn_id);
-
-        ret = glusterd_submit_request (peerinfo->rpc, &req, frame,
-                                       peerinfo->mgmt_v3,
-                                       GLUSTERD_MGMT_V3_UNLOCK, NULL,
-                                       this, glusterd_mgmt_v3_unlock_peers_cbk,
-                                       (xdrproc_t)
-                                       xdr_gd1_mgmt_v3_unlock_req);
+    gd1_mgmt_v3_unlock_req req = {
+        {0},
+    };
+    int ret = -1;
+    glusterd_peerinfo_t *peerinfo = NULL;
+    glusterd_conf_t *priv = NULL;
+    dict_t *dict = NULL;
+    uuid_t *txn_id = NULL;
+
+    if (!this)
+        goto out;
+
+    dict = data;
+
+    priv = this->private;
+    GF_ASSERT(priv);
+
+    ret = dict_get_ptr(dict, "peerinfo", VOID(&peerinfo));
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_GET_FAILED,
+                "Key=peerinfo", NULL);
+        goto out;
+    }
+
+    // peerinfo should not be in payload
+    dict_deln(dict, "peerinfo", SLEN("peerinfo"));
+
+    glusterd_get_uuid(&req.uuid);
+
+    ret = dict_allocate_and_serialize(dict, &req.dict.dict_val,
+                                      &req.dict.dict_len);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno,
+                GD_MSG_DICT_ALLOC_AND_SERL_LENGTH_GET_FAIL, NULL);
+        goto out;
+    }
+
+    /* Sending valid transaction ID to peers */
+    ret = dict_get_bin(dict, "transaction_id", (void **)&txn_id);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_TRANS_ID_GET_FAIL,
+               "Failed to get transaction id.");
+        goto out;
+    } else {
+        gf_msg_debug(this->name, 0, "Transaction_id = %s", uuid_utoa(*txn_id));
+        gf_uuid_copy(req.txn_id, *txn_id);
+    }
+
+    if (!frame)
+        frame = create_frame(this, this->ctx->pool);
+
+    if (!frame) {
+        ret = -1;
+        goto out;
+    }
+    frame->cookie = GF_MALLOC(sizeof(uuid_t), gf_common_mt_uuid_t);
+    if (!frame->cookie) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_NO_MEMORY, NULL);
+        ret = -1;
+        goto out;
+    }
+    gf_uuid_copy(frame->cookie, req.txn_id);
+
+    ret = glusterd_submit_request(peerinfo->rpc, &req, frame, peerinfo->mgmt_v3,
+                                  GLUSTERD_MGMT_V3_UNLOCK, NULL, this,
+                                  glusterd_mgmt_v3_unlock_peers_cbk,
+                                  (xdrproc_t)xdr_gd1_mgmt_v3_unlock_req);
 out:
-        gf_log (this->name, GF_LOG_DEBUG, "Returning %d", ret);
-        return ret;
+    gf_msg_debug(this ? this->name : "glusterd", 0, "Returning %d", ret);
+    if (dict)
+        dict_unref(dict);
+
+    if (req.dict.dict_val)
+        GF_FREE(req.dict.dict_val);
+    return ret;
 }
 
 int32_t
-glusterd_cluster_unlock (call_frame_t *frame, xlator_t *this,
-                            void *data)
+glusterd_cluster_unlock(call_frame_t *frame, xlator_t *this, void *data)
 {
-        gd1_mgmt_cluster_lock_req       req = {{0},};
-        int                             ret = -1;
-        glusterd_peerinfo_t             *peerinfo = NULL;
-        glusterd_conf_t                 *priv = NULL;
-        call_frame_t                    *dummy_frame = NULL;
-
-        if (!this ) {
-                ret = -1;
-                goto out;
-        }
-        peerinfo = data;
-        priv = this->private;
-        GF_ASSERT (priv);
+    gd1_mgmt_cluster_lock_req req = {
+        {0},
+    };
+    int ret = -1;
+    glusterd_peerinfo_t *peerinfo = NULL;
+    glusterd_conf_t *priv = NULL;
+    call_frame_t *dummy_frame = NULL;
+
+    if (!this) {
+        ret = -1;
+        goto out;
+    }
+    peerinfo = data;
+    priv = this->private;
+    GF_ASSERT(priv);
+
+    glusterd_get_uuid(&req.uuid);
+
+    dummy_frame = create_frame(this, this->ctx->pool);
+    if (!dummy_frame)
+        goto out;
+
+    ret = glusterd_submit_request(peerinfo->rpc, &req, dummy_frame,
+                                  peerinfo->mgmt, GLUSTERD_MGMT_CLUSTER_UNLOCK,
+                                  NULL, this, glusterd_cluster_unlock_cbk,
+                                  (xdrproc_t)xdr_gd1_mgmt_cluster_unlock_req);
+out:
+    gf_msg_debug(this ? this->name : "glusterd", 0, "Returning %d", ret);
 
-        glusterd_get_uuid (&req.uuid);
+    if (ret && dummy_frame)
+        STACK_DESTROY(dummy_frame->root);
 
-        dummy_frame = create_frame (this, this->ctx->pool);
-        if (!dummy_frame)
-                goto out;
-
-        ret = glusterd_submit_request (peerinfo->rpc, &req, dummy_frame,
-                                       peerinfo->mgmt, GLUSTERD_MGMT_CLUSTER_UNLOCK,
-                                       NULL,
-                                       this, glusterd_cluster_unlock_cbk,
-                                       (xdrproc_t)xdr_gd1_mgmt_cluster_unlock_req);
-out:
-        gf_log (this->name, GF_LOG_DEBUG, "Returning %d", ret);
-        return ret;
+    return ret;
 }
 
 int32_t
-glusterd_stage_op (call_frame_t *frame, xlator_t *this,
-                      void *data)
+glusterd_stage_op(call_frame_t *frame, xlator_t *this, void *data)
 {
-        gd1_mgmt_stage_op_req           req = {{0,},};
-        int                             ret = -1;
-        glusterd_peerinfo_t             *peerinfo = NULL;
-        glusterd_conf_t                 *priv = NULL;
-        call_frame_t                    *dummy_frame = NULL;
-        dict_t                          *dict = NULL;
-        gf_boolean_t                    is_alloc = _gf_true;
-
-        if (!this) {
-                goto out;
-        }
-
-        dict = data;
-
-        priv = this->private;
-        GF_ASSERT (priv);
-
-        ret = dict_get_ptr (dict, "peerinfo", VOID (&peerinfo));
-        if (ret)
-                goto out;
-
-        //peerinfo should not be in payload
-        dict_del (dict, "peerinfo");
-
-        glusterd_get_uuid (&req.uuid);
-        req.op = glusterd_op_get_op ();
-
-        ret = dict_allocate_and_serialize (dict, &req.buf.buf_val,
-                                           &req.buf.buf_len);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "Failed to serialize dict "
-                        "to request buffer");
-                goto out;
-        }
-
-
-        dummy_frame = create_frame (this, this->ctx->pool);
-        if (!dummy_frame)
-                goto out;
-
-        ret = glusterd_submit_request (peerinfo->rpc, &req, dummy_frame,
-                                       peerinfo->mgmt, GLUSTERD_MGMT_STAGE_OP,
-                                       NULL,
-                                       this, glusterd_stage_op_cbk,
-                                       (xdrproc_t)xdr_gd1_mgmt_stage_op_req);
+    gd1_mgmt_stage_op_req req = {
+        {
+            0,
+        },
+    };
+    int ret = -1;
+    glusterd_peerinfo_t *peerinfo = NULL;
+    glusterd_conf_t *priv = NULL;
+    dict_t *dict = NULL;
+    uuid_t *txn_id = NULL;
+
+    if (!this) {
+        goto out;
+    }
+
+    dict = data;
+
+    priv = this->private;
+    GF_ASSERT(priv);
+
+    ret = dict_get_ptr(dict, "peerinfo", VOID(&peerinfo));
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_GET_FAILED,
+                "Key=peerinfo", NULL);
+        goto out;
+    }
+
+    // peerinfo should not be in payload
+    dict_deln(dict, "peerinfo", SLEN("peerinfo"));
+
+    glusterd_get_uuid(&req.uuid);
+    req.op = glusterd_op_get_op();
+
+    ret = dict_allocate_and_serialize(dict, &req.buf.buf_val, &req.buf.buf_len);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno,
+                GD_MSG_DICT_ALLOC_AND_SERL_LENGTH_GET_FAIL, NULL);
+        goto out;
+    }
+    /* Sending valid transaction ID to peers */
+    ret = dict_get_bin(dict, "transaction_id", (void **)&txn_id);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_TRANS_ID_GET_FAIL,
+               "Failed to get transaction id.");
+        goto out;
+    } else {
+        gf_msg_debug(this->name, 0, "Transaction_id = %s", uuid_utoa(*txn_id));
+    }
+
+    if (!frame)
+        frame = create_frame(this, this->ctx->pool);
+
+    if (!frame) {
+        ret = -1;
+        goto out;
+    }
+    frame->cookie = GF_MALLOC(sizeof(uuid_t), gf_common_mt_uuid_t);
+    if (!frame->cookie) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_NO_MEMORY, NULL);
+        ret = -1;
+        goto out;
+    }
+    gf_uuid_copy(frame->cookie, *txn_id);
+
+    ret = glusterd_submit_request(peerinfo->rpc, &req, frame, peerinfo->mgmt,
+                                  GLUSTERD_MGMT_STAGE_OP, NULL, this,
+                                  glusterd_stage_op_cbk,
+                                  (xdrproc_t)xdr_gd1_mgmt_stage_op_req);
 
 out:
-        if ((_gf_true == is_alloc) && req.buf.buf_val)
-                GF_FREE (req.buf.buf_val);
+    if (req.buf.buf_val)
+        GF_FREE(req.buf.buf_val);
 
-        gf_log (this->name, GF_LOG_DEBUG, "Returning %d", ret);
-        return ret;
+    gf_msg_debug(this ? this->name : "glusterd", 0, "Returning %d", ret);
+    return ret;
 }
 
 int32_t
-glusterd_commit_op (call_frame_t *frame, xlator_t *this,
-                      void *data)
+glusterd_commit_op(call_frame_t *frame, xlator_t *this, void *data)
 {
-        gd1_mgmt_commit_op_req  req         = {{0,},};
-        int                     ret         = -1;
-        glusterd_peerinfo_t    *peerinfo    = NULL;
-        glusterd_conf_t        *priv        = NULL;
-        call_frame_t           *dummy_frame = NULL;
-        dict_t                 *dict        = NULL;
-        gf_boolean_t            is_alloc    = _gf_true;
-
-        if (!this) {
-                goto out;
-        }
-
-        dict = data;
-        priv = this->private;
-        GF_ASSERT (priv);
-
-        ret = dict_get_ptr (dict, "peerinfo", VOID (&peerinfo));
-        if (ret)
-                goto out;
-
-        //peerinfo should not be in payload
-        dict_del (dict, "peerinfo");
-
-        glusterd_get_uuid (&req.uuid);
-        req.op = glusterd_op_get_op ();
-
-        ret = dict_allocate_and_serialize (dict, &req.buf.buf_val,
-                                           &req.buf.buf_len);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "Failed to serialize dict to "
-                        "request buffer");
-                goto out;
-        }
-
-        dummy_frame = create_frame (this, this->ctx->pool);
-        if (!dummy_frame)
-                goto out;
-
-        ret = glusterd_submit_request (peerinfo->rpc, &req, dummy_frame,
-                                       peerinfo->mgmt, GLUSTERD_MGMT_COMMIT_OP,
-                                       NULL,
-                                       this, glusterd_commit_op_cbk,
-                                       (xdrproc_t)xdr_gd1_mgmt_commit_op_req);
+    gd1_mgmt_commit_op_req req = {
+        {
+            0,
+        },
+    };
+    int ret = -1;
+    glusterd_peerinfo_t *peerinfo = NULL;
+    glusterd_conf_t *priv = NULL;
+    dict_t *dict = NULL;
+    uuid_t *txn_id = NULL;
+
+    if (!this) {
+        goto out;
+    }
+
+    dict = data;
+    priv = this->private;
+    GF_ASSERT(priv);
+
+    ret = dict_get_ptr(dict, "peerinfo", VOID(&peerinfo));
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_GET_FAILED,
+                "Key=peerinfo", NULL);
+        goto out;
+    }
+
+    // peerinfo should not be in payload
+    dict_deln(dict, "peerinfo", SLEN("peerinfo"));
+
+    glusterd_get_uuid(&req.uuid);
+    req.op = glusterd_op_get_op();
+
+    ret = dict_allocate_and_serialize(dict, &req.buf.buf_val, &req.buf.buf_len);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno,
+                GD_MSG_DICT_ALLOC_AND_SERL_LENGTH_GET_FAIL, NULL);
+        goto out;
+    }
+    /* Sending valid transaction ID to peers */
+    ret = dict_get_bin(dict, "transaction_id", (void **)&txn_id);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_TRANS_ID_GET_FAIL,
+               "Failed to get transaction id.");
+        goto out;
+    } else {
+        gf_msg_debug(this->name, 0, "Transaction_id = %s", uuid_utoa(*txn_id));
+    }
+
+    if (!frame)
+        frame = create_frame(this, this->ctx->pool);
+
+    if (!frame) {
+        ret = -1;
+        goto out;
+    }
+    frame->cookie = GF_MALLOC(sizeof(uuid_t), gf_common_mt_uuid_t);
+    if (!frame->cookie) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_NO_MEMORY, NULL);
+        ret = -1;
+        goto out;
+    }
+    gf_uuid_copy(frame->cookie, *txn_id);
+
+    ret = glusterd_submit_request(peerinfo->rpc, &req, frame, peerinfo->mgmt,
+                                  GLUSTERD_MGMT_COMMIT_OP, NULL, this,
+                                  glusterd_commit_op_cbk,
+                                  (xdrproc_t)xdr_gd1_mgmt_commit_op_req);
 
 out:
-        if ((_gf_true == is_alloc) && req.buf.buf_val)
-                GF_FREE (req.buf.buf_val);
+    if (req.buf.buf_val)
+        GF_FREE(req.buf.buf_val);
 
-        gf_log (this->name, GF_LOG_DEBUG, "Returning %d", ret);
-        return ret;
+    gf_msg_debug(this ? this->name : "glusterd", 0, "Returning %d", ret);
+    return ret;
 }
 
 int32_t
-__glusterd_brick_op_cbk (struct rpc_req *req, struct iovec *iov,
-                          int count, void *myframe)
+__glusterd_brick_op_cbk(struct rpc_req *req, struct iovec *iov, int count,
+                        void *myframe)
 {
-        gd1_mgmt_brick_op_rsp         rsp   = {0};
-        int                           ret   = -1;
-        int32_t                       op_ret = -1;
-        glusterd_op_sm_event_type_t   event_type = GD_OP_EVENT_NONE;
-        call_frame_t                  *frame = NULL;
-        glusterd_op_brick_rsp_ctx_t   *ev_ctx = NULL;
-        dict_t                        *dict = NULL;
-        int                           index = 0;
-        glusterd_req_ctx_t            *req_ctx = NULL;
-        glusterd_pending_node_t       *node = NULL;
-        xlator_t                      *this = NULL;
-        uuid_t                        *txn_id = NULL;
-        glusterd_conf_t               *priv = NULL;
-
-        this = THIS;
-        GF_ASSERT (this);
-        priv = this->private;
-        GF_ASSERT (priv);
-        GF_ASSERT (req);
-
-        txn_id = &priv->global_txn_id;
-        frame = myframe;
-        req_ctx = frame->local;
-
-        if (-1 == req->rpc_status) {
-                rsp.op_ret   = -1;
-                rsp.op_errno = EINVAL;
-                /* use standard allocation because to keep uniformity
-                   in freeing it */
-                rsp.op_errstr = strdup ("error");
-		event_type = GD_OP_EVENT_RCVD_RJT;
-                goto out;
-        }
-
-        ret =  xdr_to_generic (*iov, &rsp, (xdrproc_t)xdr_gd1_mgmt_brick_op_rsp);
+    gd1_mgmt_brick_op_rsp rsp = {0};
+    int ret = -1;
+    int32_t op_ret = -1;
+    glusterd_op_sm_event_type_t event_type = GD_OP_EVENT_NONE;
+    call_frame_t *frame = NULL;
+    glusterd_op_brick_rsp_ctx_t *ev_ctx = NULL;
+    dict_t *dict = NULL;
+    int index = 0;
+    glusterd_req_ctx_t *req_ctx = NULL;
+    glusterd_pending_node_t *node = NULL;
+    xlator_t *this = NULL;
+    uuid_t *txn_id = NULL;
+    glusterd_conf_t *priv = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+    priv = this->private;
+    GF_ASSERT(priv);
+    GF_ASSERT(req);
+
+    txn_id = &priv->global_txn_id;
+    frame = myframe;
+    req_ctx = frame->local;
+
+    if (-1 == req->rpc_status) {
+        rsp.op_ret = -1;
+        rsp.op_errno = EINVAL;
+        /* use standard allocation because to keep uniformity
+           in freeing it */
+        rsp.op_errstr = strdup("error");
+        event_type = GD_OP_EVENT_RCVD_RJT;
+        goto out;
+    }
+
+    ret = xdr_to_generic(*iov, &rsp, (xdrproc_t)xdr_gd1_mgmt_brick_op_rsp);
+    if (ret < 0) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_RES_DECODE_FAIL,
+               "Failed to decode brick op "
+               "response received");
+        rsp.op_ret = -1;
+        rsp.op_errno = EINVAL;
+        rsp.op_errstr = strdup("Unable to decode brick op response");
+        event_type = GD_OP_EVENT_RCVD_RJT;
+        goto out;
+    }
+
+    if (rsp.output.output_len) {
+        /* Unserialize the dictionary */
+        dict = dict_new();
+
+        ret = dict_unserialize(rsp.output.output_val, rsp.output.output_len,
+                               &dict);
         if (ret < 0) {
-                gf_log (this->name, GF_LOG_ERROR, "Failed to decode brick op "
-                        "response received");
-                rsp.op_ret   = -1;
-                rsp.op_errno = EINVAL;
-                rsp.op_errstr = strdup ("Unable to decode brick op response");
-		event_type = GD_OP_EVENT_RCVD_RJT;
-                goto out;
-        }
-
-        if (rsp.output.output_len) {
-                /* Unserialize the dictionary */
-                dict  = dict_new ();
-
-                ret = dict_unserialize (rsp.output.output_val,
-                                        rsp.output.output_len,
-                                        &dict);
-                if (ret < 0) {
-                        gf_log (this->name, GF_LOG_ERROR, "Failed to "
-                                "unserialize rsp-buffer to dictionary");
-			event_type = GD_OP_EVENT_RCVD_RJT;
-                        goto out;
-                } else {
-                        dict->extra_stdfree = rsp.output.output_val;
-                }
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_UNSERIALIZE_FAIL,
+                   "Failed to "
+                   "unserialize rsp-buffer to dictionary");
+            event_type = GD_OP_EVENT_RCVD_RJT;
+            goto out;
+        } else {
+            dict->extra_stdfree = rsp.output.output_val;
         }
+    }
 
-        op_ret = rsp.op_ret;
+    op_ret = rsp.op_ret;
 
-        /* Add index to rsp_dict for GD_OP_STATUS_VOLUME */
-        if (GD_OP_STATUS_VOLUME == req_ctx->op) {
-                node = frame->cookie;
-                index = node->index;
-                ret = dict_set_int32 (dict, "index", index);
-                if (ret) {
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "Error setting index on brick status rsp dict");
-                        rsp.op_ret = -1;
-                        event_type = GD_OP_EVENT_RCVD_RJT;
-                        goto out;
-                }
+    /* Add index to rsp_dict for GD_OP_STATUS_VOLUME */
+    if (GD_OP_STATUS_VOLUME == req_ctx->op) {
+        node = frame->cookie;
+        index = node->index;
+        ret = dict_set_int32n(dict, "index", SLEN("index"), index);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+                   "Error setting index on brick status rsp dict");
+            rsp.op_ret = -1;
+            event_type = GD_OP_EVENT_RCVD_RJT;
+            goto out;
         }
+    }
 out:
 
-        if (req_ctx && req_ctx->dict) {
-                ret = dict_get_bin (req_ctx->dict, "transaction_id",
-                                    (void **)&txn_id);
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "transaction ID = %s", uuid_utoa (*txn_id));
-        }
+    if (req_ctx && req_ctx->dict) {
+        ret = dict_get_bin(req_ctx->dict, "transaction_id", (void **)&txn_id);
+        gf_msg_debug(this->name, -ret, "transaction ID = %s",
+                     uuid_utoa(*txn_id));
+    }
 
-        ev_ctx = GF_CALLOC (1, sizeof (*ev_ctx), gf_gld_mt_brick_rsp_ctx_t);
-        GF_ASSERT (ev_ctx);
+    ev_ctx = GF_CALLOC(1, sizeof(*ev_ctx), gf_gld_mt_brick_rsp_ctx_t);
+    if (ev_ctx) {
         if (op_ret) {
-                event_type = GD_OP_EVENT_RCVD_RJT;
-                ev_ctx->op_ret = op_ret;
-                ev_ctx->op_errstr = gf_strdup(rsp.op_errstr);
+            event_type = GD_OP_EVENT_RCVD_RJT;
+            ev_ctx->op_ret = op_ret;
+            ev_ctx->op_errstr = gf_strdup(rsp.op_errstr);
         } else {
-                event_type = GD_OP_EVENT_RCVD_ACC;
+            event_type = GD_OP_EVENT_RCVD_ACC;
         }
         ev_ctx->pending_node = frame->cookie;
-        ev_ctx->rsp_dict  = dict;
+        ev_ctx->rsp_dict = dict;
         ev_ctx->commit_ctx = frame->local;
-        ret = glusterd_op_sm_inject_event (event_type, txn_id, ev_ctx);
-        if (!ret) {
-                glusterd_friend_sm ();
-                glusterd_op_sm ();
+        ret = glusterd_op_sm_inject_event(event_type, txn_id, ev_ctx);
+    }
+    if (!ret) {
+        glusterd_friend_sm();
+        glusterd_op_sm();
+    }
+
+    if (ret) {
+        if (dict) {
+            dict_unref(dict);
         }
-
-        if (ret && dict)
-                dict_unref (dict);
-        free (rsp.op_errstr); //malloced by xdr
-        GLUSTERD_STACK_DESTROY (frame);
-        return ret;
+        if (ev_ctx) {
+            GF_FREE(ev_ctx->op_errstr);
+            GF_FREE(ev_ctx);
+        }
+    }
+    free(rsp.op_errstr);  // malloced by xdr
+    GLUSTERD_STACK_DESTROY(frame);
+    return ret;
 }
 
 int32_t
-glusterd_brick_op_cbk (struct rpc_req *req, struct iovec *iov,
-                          int count, void *myframe)
+glusterd_brick_op_cbk(struct rpc_req *req, struct iovec *iov, int count,
+                      void *myframe)
 {
-        return glusterd_big_locked_cbk (req, iov, count, myframe,
-                                        __glusterd_brick_op_cbk);
+    return glusterd_big_locked_cbk(req, iov, count, myframe,
+                                   __glusterd_brick_op_cbk);
 }
 
 int32_t
-glusterd_brick_op (call_frame_t *frame, xlator_t *this,
-                   void *data)
+glusterd_brick_op(call_frame_t *frame, xlator_t *this, void *data)
 {
+    gd1_mgmt_brick_op_req *req = NULL;
+    int ret = 0;
+    int ret1 = 0;
+    glusterd_conf_t *priv = NULL;
+    call_frame_t *dummy_frame = NULL;
+    char *op_errstr = NULL;
+    int pending_bricks = 0;
+    glusterd_pending_node_t *pending_node;
+    glusterd_req_ctx_t *req_ctx = NULL;
+    struct rpc_clnt *rpc = NULL;
+    dict_t *op_ctx = NULL;
+    uuid_t *txn_id = NULL;
+
+    if (!this) {
+        ret = -1;
+        goto out;
+    }
+    priv = this->private;
+    GF_ASSERT(priv);
+
+    txn_id = &priv->global_txn_id;
+
+    req_ctx = data;
+    GF_ASSERT(req_ctx);
+    CDS_INIT_LIST_HEAD(&opinfo.pending_bricks);
+
+    ret = dict_get_bin(req_ctx->dict, "transaction_id", (void **)&txn_id);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_WARNING, 0, GD_MSG_BRICK_SELECT_FAIL,
+               "Could not get transaction ID from dict, global"
+               "transaction ID = %s",
+               uuid_utoa(*txn_id));
+    } else {
+        gf_msg_debug(this->name, 0, "transaction ID = %s", uuid_utoa(*txn_id));
+    }
+    ret = glusterd_op_bricks_select(req_ctx->op, req_ctx->dict, &op_errstr,
+                                    &opinfo.pending_bricks, NULL);
+
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_BRICK_SELECT_FAIL,
+               "Failed to select bricks "
+               "while performing brick op during 'Volume %s'",
+               gd_op_list[opinfo.op]);
+        opinfo.op_errstr = op_errstr;
+        goto out;
+    }
+
+    cds_list_for_each_entry(pending_node, &opinfo.pending_bricks, list)
+    {
+        dummy_frame = create_frame(this, this->ctx->pool);
+        if (!dummy_frame)
+            continue;
+
+        if ((pending_node->type == GD_NODE_NFS) ||
+            (pending_node->type == GD_NODE_QUOTAD) ||
+            (pending_node->type == GD_NODE_SNAPD) ||
+            (pending_node->type == GD_NODE_SCRUB) ||
+            ((pending_node->type == GD_NODE_SHD) &&
+             (req_ctx->op == GD_OP_STATUS_VOLUME))) {
+            ret = glusterd_node_op_build_payload(
+                req_ctx->op, (gd1_mgmt_brick_op_req **)&req, req_ctx->dict);
+        } else {
+            ret = glusterd_brick_op_build_payload(
+                req_ctx->op, pending_node->node, (gd1_mgmt_brick_op_req **)&req,
+                req_ctx->dict);
+        }
+        if (ret || !req) {
+            gf_msg(this->name, GF_LOG_ERROR, 0,
+                   GD_MSG_BRICK_OP_PAYLOAD_BUILD_FAIL,
+                   "Failed to "
+                   "build op payload during "
+                   "'Volume %s'",
+                   gd_op_list[req_ctx->op]);
+            goto out;
+        }
+
+        dummy_frame->local = data;
+        dummy_frame->cookie = pending_node;
+
+        rpc = glusterd_pending_node_get_rpc(pending_node);
+        if (!rpc) {
+            if (pending_node->type == GD_NODE_REBALANCE) {
+                opinfo.brick_pending_count = 0;
+                ret = 0;
+                GF_FREE(req->input.input_val);
+                GF_FREE(req);
+                req = NULL;
+                GLUSTERD_STACK_DESTROY(dummy_frame);
+
+                op_ctx = glusterd_op_get_ctx();
+                if (!op_ctx)
+                    goto out;
+                glusterd_defrag_volume_node_rsp(req_ctx->dict, NULL, op_ctx);
 
-        gd1_mgmt_brick_op_req           *req = NULL;
-        int                             ret = 0;
-        glusterd_conf_t                 *priv = NULL;
-        call_frame_t                    *dummy_frame = NULL;
-        char                            *op_errstr = NULL;
-        int                             pending_bricks = 0;
-        glusterd_pending_node_t         *pending_node;
-        glusterd_req_ctx_t              *req_ctx = NULL;
-        struct rpc_clnt                 *rpc = NULL;
-        dict_t                          *op_ctx = NULL;
-        uuid_t                          *txn_id = NULL;
-
-        if (!this) {
-                ret = -1;
                 goto out;
+            }
+
+            ret = -1;
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_RPC_FAILURE,
+                   "Brick Op failed "
+                   "due to rpc failure.");
+            goto out;
         }
-        priv = this->private;
-        GF_ASSERT (priv);
 
-        txn_id = &priv->global_txn_id;
+        ret = glusterd_submit_request(
+            rpc, req, dummy_frame, priv->gfs_mgmt, req->op, NULL, this,
+            glusterd_brick_op_cbk, (xdrproc_t)xdr_gd1_mgmt_brick_op_req);
+        GF_FREE(req->input.input_val);
+        GF_FREE(req);
+        req = NULL;
 
-        req_ctx = data;
-        GF_ASSERT (req_ctx);
-        CDS_INIT_LIST_HEAD (&opinfo.pending_bricks);
+        if (!ret)
+            pending_bricks++;
 
-        ret = dict_get_bin (req_ctx->dict, "transaction_id", (void **)&txn_id);
-        gf_log (this->name, GF_LOG_DEBUG, "transaction ID = %s",
-                uuid_utoa (*txn_id));
+        glusterd_pending_node_put_rpc(pending_node);
+    }
 
-        ret = glusterd_op_bricks_select (req_ctx->op, req_ctx->dict, &op_errstr,
-                                         &opinfo.pending_bricks, NULL);
+    gf_msg_trace(this->name, 0,
+                 "Sent brick op req for operation "
+                 "'Volume %s' to %d bricks",
+                 gd_op_list[req_ctx->op], pending_bricks);
+    opinfo.brick_pending_count = pending_bricks;
 
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "Failed to select bricks "
-                        "while performing brick op during 'Volume %s'",
-                        gd_op_list[opinfo.op]);
-                opinfo.op_errstr = op_errstr;
-                goto out;
-        }
-
-        cds_list_for_each_entry (pending_node, &opinfo.pending_bricks, list) {
-                dummy_frame = create_frame (this, this->ctx->pool);
-                if (!dummy_frame)
-                        continue;
-
-                if ((pending_node->type == GD_NODE_NFS) ||
-                    (pending_node->type == GD_NODE_QUOTAD) ||
-                    (pending_node->type == GD_NODE_SNAPD) ||
-                    ((pending_node->type == GD_NODE_SHD) &&
-                     (req_ctx->op == GD_OP_STATUS_VOLUME)))
-                        ret = glusterd_node_op_build_payload
-                                (req_ctx->op,
-                                 (gd1_mgmt_brick_op_req **)&req,
-                                 req_ctx->dict);
-                else {
-                        ret = glusterd_brick_op_build_payload
-                                (req_ctx->op, pending_node->node,
-                                 (gd1_mgmt_brick_op_req **)&req,
-                                 req_ctx->dict);
-
-                        if (ret) {
-                                gf_log (this->name, GF_LOG_ERROR, "Failed to "
-                                        "build brick op payload during "
-                                        "'Volume %s'", gd_op_list[req_ctx->op]);
-                                goto out;
-                        }
-                }
+out:
 
-                dummy_frame->local = data;
-                dummy_frame->cookie = pending_node;
-
-                rpc = glusterd_pending_node_get_rpc (pending_node);
-                if (!rpc) {
-                        if (pending_node->type == GD_NODE_REBALANCE) {
-                                opinfo.brick_pending_count = 0;
-                                ret = 0;
-                                if (req) {
-                                        GF_FREE (req->input.input_val);
-                                        GF_FREE (req);
-                                        req = NULL;
-                                }
-                                GLUSTERD_STACK_DESTROY (dummy_frame);
-
-                                op_ctx = glusterd_op_get_ctx ();
-                                if (!op_ctx)
-                                        goto out;
-                                glusterd_defrag_volume_node_rsp (req_ctx->dict,
-                                                                 NULL, op_ctx);
-
-                                goto out;
-                        }
-
-                        ret = -1;
-                        gf_log (this->name, GF_LOG_ERROR, "Brick Op failed "
-                                "due to rpc failure.");
-                        goto out;
-                }
+    if (ret)
+        opinfo.op_ret = ret;
 
-                ret = glusterd_submit_request (rpc, req, dummy_frame,
-                                               priv->gfs_mgmt,
-                                               req->op, NULL,
-                                               this, glusterd_brick_op_cbk,
-                                               (xdrproc_t)xdr_gd1_mgmt_brick_op_req);
-                if (req) {
-                        GF_FREE (req->input.input_val);
-                        GF_FREE (req);
-                        req = NULL;
-                }
-                if (!ret)
-                        pending_bricks++;
-        }
+    ret1 = glusterd_set_txn_opinfo(txn_id, &opinfo);
+    if (ret1)
+        gf_msg(THIS->name, GF_LOG_ERROR, 0, GD_MSG_TRANS_OPINFO_SET_FAIL,
+               "Unable to set "
+               "transaction's opinfo");
 
-        gf_log (this->name, GF_LOG_DEBUG, "Sent brick op req for operation "
-                "'Volume %s' to %d bricks", gd_op_list[req_ctx->op],
-                pending_bricks);
-        opinfo.brick_pending_count = pending_bricks;
+    if (ret) {
+        glusterd_op_sm_inject_event(GD_OP_EVENT_RCVD_RJT, txn_id, data);
+        opinfo.op_ret = ret;
+    }
 
-out:
-        if (ret) {
-                glusterd_op_sm_inject_event (GD_OP_EVENT_RCVD_RJT,
-                                             txn_id, data);
-                opinfo.op_ret = ret;
-        }
-        gf_log (this->name, GF_LOG_DEBUG, "Returning %d", ret);
-        return ret;
+    gf_msg_debug(this ? this->name : "glusterd", 0, "Returning %d", ret);
+    return ret;
 }
 
 struct rpc_clnt_procedure gd_brick_actors[GLUSTERD_BRICK_MAXVALUE] = {
-        [GLUSTERD_BRICK_NULL] = {"NULL", NULL },
-        [GLUSTERD_BRICK_OP]   = {"BRICK_OP", glusterd_brick_op},
+    [GLUSTERD_BRICK_NULL] = {"NULL", NULL},
+    [GLUSTERD_BRICK_OP] = {"BRICK_OP", glusterd_brick_op},
 };
 
 struct rpc_clnt_procedure gd_peer_actors[GLUSTERD_FRIEND_MAXVALUE] = {
-        [GLUSTERD_FRIEND_NULL]   = {"NULL", NULL },
-        [GLUSTERD_PROBE_QUERY]   = {"PROBE_QUERY", glusterd_rpc_probe},
-        [GLUSTERD_FRIEND_ADD]    = {"FRIEND_ADD", glusterd_rpc_friend_add},
-        [GLUSTERD_FRIEND_REMOVE] = {"FRIEND_REMOVE", glusterd_rpc_friend_remove},
-        [GLUSTERD_FRIEND_UPDATE] = {"FRIEND_UPDATE", glusterd_rpc_friend_update},
+    [GLUSTERD_FRIEND_NULL] = {"NULL", NULL},
+    [GLUSTERD_PROBE_QUERY] = {"PROBE_QUERY", glusterd_rpc_probe},
+    [GLUSTERD_FRIEND_ADD] = {"FRIEND_ADD", glusterd_rpc_friend_add},
+    [GLUSTERD_FRIEND_REMOVE] = {"FRIEND_REMOVE", glusterd_rpc_friend_remove},
+    [GLUSTERD_FRIEND_UPDATE] = {"FRIEND_UPDATE", glusterd_rpc_friend_update},
 };
 
 struct rpc_clnt_procedure gd_mgmt_actors[GLUSTERD_MGMT_MAXVALUE] = {
-        [GLUSTERD_MGMT_NULL]           = {"NULL", NULL },
-        [GLUSTERD_MGMT_CLUSTER_LOCK]   = {"CLUSTER_LOCK", glusterd_cluster_lock},
-        [GLUSTERD_MGMT_CLUSTER_UNLOCK] = {"CLUSTER_UNLOCK", glusterd_cluster_unlock},
-        [GLUSTERD_MGMT_STAGE_OP]       = {"STAGE_OP", glusterd_stage_op},
-        [GLUSTERD_MGMT_COMMIT_OP]      = {"COMMIT_OP", glusterd_commit_op},
+    [GLUSTERD_MGMT_NULL] = {"NULL", NULL},
+    [GLUSTERD_MGMT_CLUSTER_LOCK] = {"CLUSTER_LOCK", glusterd_cluster_lock},
+    [GLUSTERD_MGMT_CLUSTER_UNLOCK] = {"CLUSTER_UNLOCK",
+                                      glusterd_cluster_unlock},
+    [GLUSTERD_MGMT_STAGE_OP] = {"STAGE_OP", glusterd_stage_op},
+    [GLUSTERD_MGMT_COMMIT_OP] = {"COMMIT_OP", glusterd_commit_op},
 };
 
 struct rpc_clnt_procedure gd_mgmt_v3_actors[GLUSTERD_MGMT_V3_MAXVALUE] = {
-        [GLUSTERD_MGMT_V3_NULL]    = {"NULL", NULL },
-        [GLUSTERD_MGMT_V3_LOCK]    = {"MGMT_V3_LOCK", glusterd_mgmt_v3_lock_peers},
-        [GLUSTERD_MGMT_V3_UNLOCK]  = {"MGMT_V3_UNLOCK", glusterd_mgmt_v3_unlock_peers},
+    [GLUSTERD_MGMT_V3_NULL] = {"NULL", NULL},
+    [GLUSTERD_MGMT_V3_LOCK] = {"MGMT_V3_LOCK", glusterd_mgmt_v3_lock_peers},
+    [GLUSTERD_MGMT_V3_UNLOCK] = {"MGMT_V3_UNLOCK",
+                                 glusterd_mgmt_v3_unlock_peers},
 };
 
 struct rpc_clnt_program gd_mgmt_prog = {
-        .progname  = "glusterd mgmt",
-        .prognum   = GD_MGMT_PROGRAM,
-        .progver   = GD_MGMT_VERSION,
-        .proctable = gd_mgmt_actors,
-        .numproc   = GLUSTERD_MGMT_MAXVALUE,
+    .progname = "glusterd mgmt",
+    .prognum = GD_MGMT_PROGRAM,
+    .progver = GD_MGMT_VERSION,
+    .proctable = gd_mgmt_actors,
+    .numproc = GLUSTERD_MGMT_MAXVALUE,
 };
 
 struct rpc_clnt_program gd_brick_prog = {
-        .progname  = "brick operations",
-        .prognum   = GD_BRICK_PROGRAM,
-        .progver   = GD_BRICK_VERSION,
-        .proctable = gd_brick_actors,
-        .numproc   = GLUSTERD_BRICK_MAXVALUE,
+    .progname = "brick operations",
+    .prognum = GD_BRICK_PROGRAM,
+    .progver = GD_BRICK_VERSION,
+    .proctable = gd_brick_actors,
+    .numproc = GLUSTERD_BRICK_MAXVALUE,
 };
 
 struct rpc_clnt_program gd_peer_prog = {
-        .progname  = "Peer mgmt",
-        .prognum   = GD_FRIEND_PROGRAM,
-        .progver   = GD_FRIEND_VERSION,
-        .proctable = gd_peer_actors,
-        .numproc   = GLUSTERD_FRIEND_MAXVALUE,
+    .progname = "Peer mgmt",
+    .prognum = GD_FRIEND_PROGRAM,
+    .progver = GD_FRIEND_VERSION,
+    .proctable = gd_peer_actors,
+    .numproc = GLUSTERD_FRIEND_MAXVALUE,
 };
 
 struct rpc_clnt_program gd_mgmt_v3_prog = {
-        .progname  = "glusterd mgmt v3",
-        .prognum   = GD_MGMT_PROGRAM,
-        .progver   = GD_MGMT_V3_VERSION,
-        .proctable = gd_mgmt_v3_actors,
-        .numproc   = GLUSTERD_MGMT_V3_MAXVALUE,
+    .progname = "glusterd mgmt v3",
+    .prognum = GD_MGMT_PROGRAM,
+    .progver = GD_MGMT_V3_VERSION,
+    .proctable = gd_mgmt_v3_actors,
+    .numproc = GLUSTERD_MGMT_V3_MAXVALUE,
 };
diff --git a/xlators/mgmt/glusterd/src/glusterd-scrub-svc.c b/xlators/mgmt/glusterd/src/glusterd-scrub-svc.c
index bd1ed390f46..c49a0eefba5 100644
--- a/xlators/mgmt/glusterd/src/glusterd-scrub-svc.c
+++ b/xlators/mgmt/glusterd/src/glusterd-scrub-svc.c
@@ -8,112 +8,200 @@
   cases as published by the Free Software Foundation.
 */
 
-#include "globals.h"
-#include "run.h"
+#include <glusterfs/globals.h>
+#include <glusterfs/run.h>
 #include "glusterd.h"
 #include "glusterd-utils.h"
 #include "glusterd-volgen.h"
 #include "glusterd-scrub-svc.h"
+#include "glusterd-svc-helper.h"
 
 char *scrub_svc_name = "scrub";
 
+void
+glusterd_scrubsvc_build(glusterd_svc_t *svc)
+{
+    svc->manager = glusterd_scrubsvc_manager;
+    svc->start = glusterd_scrubsvc_start;
+    svc->stop = glusterd_scrubsvc_stop;
+}
+
 int
-glusterd_scrubsvc_init (glusterd_svc_t *svc)
+glusterd_scrubsvc_init(glusterd_svc_t *svc)
 {
-        return glusterd_svc_init (svc, scrub_svc_name,
-                                  glusterd_scrubsvc_manager,
-                                  glusterd_scrubsvc_start,
-                                  glusterd_scrubsvc_stop);
+    return glusterd_svc_init(svc, scrub_svc_name);
 }
 
 static int
-glusterd_scrubsvc_create_volfile ()
+glusterd_scrubsvc_create_volfile()
 {
-        char              filepath[PATH_MAX] = {0,};
-        int               ret                = -1;
-        glusterd_conf_t   *conf              = NULL;
-        dict_t            *mod_dict          = NULL;
-        xlator_t          *this              = NULL;
-
-        this = THIS;
-        conf = this->private;
-        GF_ASSERT (conf);
-
-
-        mod_dict = dict_new ();
-        if (!mod_dict) {
-                gf_log (this->name, GF_LOG_ERROR, "failed to allocate new "
-                        "dict");
-                goto out;
-        }
-
-        ret = dict_set_uint32 (mod_dict, "trusted-client", GF_CLIENT_TRUSTED);
-        if (ret)
-                goto free_dict;
-
-        glusterd_svc_build_volfile_path (scrub_svc_name, conf->workdir,
-                                         filepath, sizeof (filepath));
+    char filepath[PATH_MAX] = {
+        0,
+    };
+    int ret = -1;
+    glusterd_conf_t *conf = NULL;
+    xlator_t *this = NULL;
+
+    this = THIS;
+    conf = this->private;
+    GF_ASSERT(conf);
+
+    glusterd_svc_build_volfile_path(scrub_svc_name, conf->workdir, filepath,
+                                    sizeof(filepath));
+
+    ret = glusterd_create_global_volfile(build_scrub_graph, filepath, NULL);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_VOLFILE_CREATE_FAIL,
+               "Failed to create volfile");
+        goto out;
+    }
 
-        ret = glusterd_create_global_volfile (build_scrub_graph,
-                                              filepath, mod_dict);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "Failed to create volfile");
-                goto free_dict;
-        }
-
-free_dict:
-        dict_unref (mod_dict);
 out:
-        gf_log (this->name, GF_LOG_DEBUG, "Returning %d", ret);
+    gf_msg_debug(this->name, 0, "Returning %d", ret);
 
-        return ret;
+    return ret;
 }
 
 int
-glusterd_scrubsvc_manager (glusterd_svc_t *svc, void *data, int flags)
+glusterd_scrubsvc_manager(glusterd_svc_t *svc, void *data, int flags)
 {
-        int          ret    = -EINVAL;
+    int ret = -EINVAL;
 
-        if (glusterd_all_volumes_with_bitrot_stopped ()) {
-                ret = svc->stop (svc, SIGTERM);
+    if (!svc->inited) {
+        ret = glusterd_scrubsvc_init(svc);
+        if (ret) {
+            gf_msg(THIS->name, GF_LOG_ERROR, 0, GD_MSG_SCRUB_INIT_FAIL,
+                   "Failed to init "
+                   "scrub service");
+            goto out;
         } else {
-                ret = glusterd_scrubsvc_create_volfile ();
-                if (ret)
-                        goto out;
+            svc->inited = _gf_true;
+            gf_msg_debug(THIS->name, 0,
+                         "scrub service "
+                         "initialized");
+        }
+    }
+
+    if (glusterd_should_i_stop_bitd()) {
+        ret = svc->stop(svc, SIGTERM);
+    } else {
+        ret = glusterd_scrubsvc_create_volfile();
+        if (ret)
+            goto out;
 
-                ret = svc->stop (svc, SIGKILL);
-                if (ret)
-                        goto out;
+        ret = svc->stop(svc, SIGKILL);
+        if (ret)
+            goto out;
 
-                ret = svc->start (svc, flags);
-                if (ret)
-                        goto out;
+        ret = svc->start(svc, flags);
+        if (ret)
+            goto out;
 
-                ret = glusterd_conn_connect (&(svc->conn));
-                if (ret)
-                        goto out;
-        }
+        ret = glusterd_conn_connect(&(svc->conn));
+        if (ret)
+            goto out;
+    }
 
 out:
-        gf_log (THIS->name, GF_LOG_DEBUG, "Returning %d", ret);
+    if (ret)
+        gf_event(EVENT_SVC_MANAGER_FAILED, "svc_name=%s", svc->name);
+    gf_msg_debug(THIS->name, 0, "Returning %d", ret);
 
-        return ret;
+    return ret;
 }
 
 int
-glusterd_scrubsvc_start (glusterd_svc_t *svc, int flags)
+glusterd_scrubsvc_start(glusterd_svc_t *svc, int flags)
 {
-        return glusterd_svc_start (svc, flags, NULL);
+    int ret = -1;
+    dict_t *cmdict = NULL;
+
+    cmdict = dict_new();
+    if (!cmdict) {
+        gf_smsg(THIS->name, GF_LOG_ERROR, errno, GD_MSG_DICT_CREATE_FAIL, NULL);
+        goto error_return;
+    }
+
+    ret = dict_set_str(cmdict, "cmdarg0", "--global-timer-wheel");
+    if (ret)
+        goto dealloc_dict;
+
+    ret = glusterd_svc_start(svc, flags, cmdict);
+
+dealloc_dict:
+    dict_unref(cmdict);
+error_return:
+    return ret;
 }
 
 int
-glusterd_scrubsvc_stop (glusterd_svc_t *svc, int sig)
+glusterd_scrubsvc_stop(glusterd_svc_t *svc, int sig)
 {
-        return glusterd_svc_stop (svc, sig);
+    return glusterd_svc_stop(svc, sig);
 }
 
 int
-glusterd_scrubsvc_reconfigure ()
+glusterd_scrubsvc_reconfigure()
 {
-        return glusterd_svc_reconfigure (glusterd_scrubsvc_create_volfile);
+    int ret = -1;
+    xlator_t *this = NULL;
+    glusterd_conf_t *priv = NULL;
+    gf_boolean_t identical = _gf_false;
+
+    this = THIS;
+    GF_VALIDATE_OR_GOTO("glusterd", this, out);
+
+    priv = this->private;
+    GF_VALIDATE_OR_GOTO(this->name, priv, out);
+
+    if (glusterd_should_i_stop_bitd())
+        goto manager;
+
+    /*
+     * Check both OLD and NEW volfiles, if they are SAME by size
+     * and cksum i.e. "character-by-character". If YES, then
+     * NOTHING has been changed, just return.
+     */
+    ret = glusterd_svc_check_volfile_identical(priv->scrub_svc.name,
+                                               build_scrub_graph, &identical);
+    if (ret)
+        goto out;
+
+    if (identical) {
+        ret = 0;
+        goto out;
+    }
+
+    /*
+     * They are not identical. Find out if the topology is changed
+     * OR just the volume options. If just the options which got
+     * changed, then inform the xlator to reconfigure the options.
+     */
+    identical = _gf_false; /* RESET the FLAG */
+    ret = glusterd_svc_check_topology_identical(priv->scrub_svc.name,
+                                                build_scrub_graph, &identical);
+    if (ret)
+        goto out;
+
+    /* Topology is not changed, but just the options. But write the
+     * options to scrub volfile, so that scrub will be reconfigured.
+     */
+    if (identical) {
+        ret = glusterd_scrubsvc_create_volfile();
+        if (ret == 0) { /* Only if above PASSES */
+            ret = glusterd_fetchspec_notify(THIS);
+        }
+        goto out;
+    }
+
+manager:
+    /*
+     * scrub volfile's topology has been changed. scrub server needs
+     * to be RESTARTED to ACT on the changed volfile.
+     */
+    ret = priv->scrub_svc.manager(&(priv->scrub_svc), NULL, PROC_START_NO_WAIT);
+
+out:
+    gf_msg_debug(this ? this->name : "glusterd", 0, "Returning %d", ret);
+    return ret;
 }
diff --git a/xlators/mgmt/glusterd/src/glusterd-scrub-svc.h b/xlators/mgmt/glusterd/src/glusterd-scrub-svc.h
index e2e3b0d2ba2..514b1de96a0 100644
--- a/xlators/mgmt/glusterd/src/glusterd-scrub-svc.h
+++ b/xlators/mgmt/glusterd/src/glusterd-scrub-svc.h
@@ -11,37 +11,35 @@
 #ifndef _GLUSTERD_SCRUB_SVC_H_
 #define _GLUSTERD_SCRUB_SVC_H_
 
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
-
 #include "glusterd-svc-mgmt.h"
 
 typedef struct glusterd_scrubsvc_ glusterd_scrubsvc_t;
 
-struct glusterd_scrubsvc_{
-        glusterd_svc_t      svc;
-        gf_store_handle_t   *handle;
+struct glusterd_scrubsvc_ {
+    glusterd_svc_t svc;
+    gf_store_handle_t *handle;
 };
 
+void
+glusterd_scrubsvc_build(glusterd_svc_t *svc);
+
 int
-glusterd_scrubsvc_init (glusterd_svc_t *svc);
+glusterd_scrubsvc_init(glusterd_svc_t *svc);
 
 int
-glusterd_scrubsvc_manager (glusterd_svc_t *svc, void *data, int flags);
+glusterd_scrubsvc_manager(glusterd_svc_t *svc, void *data, int flags);
 
 int
-glusterd_scrubsvc_start (glusterd_svc_t *svc, int flags);
+glusterd_scrubsvc_start(glusterd_svc_t *svc, int flags);
 
 int
-glusterd_scrubsvc_stop (glusterd_svc_t *svc, int sig);
+glusterd_scrubsvc_stop(glusterd_svc_t *svc, int sig);
 
 int
-glusterd_scrubsvc_reconfigure ();
+glusterd_scrubsvc_reconfigure();
 
 void
-glusterd_scrubsvc_build_volfile_path (char *server, char *workdir,
-                                     char *volfile, size_t len);
+glusterd_scrubsvc_build_volfile_path(char *server, char *workdir, char *volfile,
+                                     size_t len);
 
 #endif
diff --git a/xlators/mgmt/glusterd/src/glusterd-server-quorum.c b/xlators/mgmt/glusterd/src/glusterd-server-quorum.c
index 8d8acb17513..b0b8a2e4018 100644
--- a/xlators/mgmt/glusterd/src/glusterd-server-quorum.c
+++ b/xlators/mgmt/glusterd/src/glusterd-server-quorum.c
@@ -7,402 +7,480 @@
    later), or the GNU General Public License, version 2 (GPLv2), in all
    cases as published by the Free Software Foundation.
 */
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
-
-#include "common-utils.h"
+#include <glusterfs/common-utils.h>
 #include "glusterd.h"
 #include "glusterd-utils.h"
 #include "glusterd-messages.h"
 #include "glusterd-server-quorum.h"
+#include "glusterd-store.h"
 #include "glusterd-syncop.h"
 #include "glusterd-op-sm.h"
 
-#define CEILING_POS(X) (((X)-(int)(X)) > 0 ? (int)((X)+1) : (int)(X))
+#define CEILING_POS(X) (((X) - (int)(X)) > 0 ? (int)((X) + 1) : (int)(X))
 
 static gf_boolean_t
-glusterd_is_get_op (xlator_t *this, glusterd_op_t op, dict_t *dict)
+glusterd_is_get_op(xlator_t *this, glusterd_op_t op, dict_t *dict)
 {
-        char            *key            = NULL;
-        char            *volname        = NULL;
-        int             ret             = 0;
-
-        if (op == GD_OP_STATUS_VOLUME)
+    char *key = NULL;
+    char *volname = NULL;
+    int ret = 0;
+
+    if (op == GD_OP_STATUS_VOLUME)
+        return _gf_true;
+
+    if (op == GD_OP_SET_VOLUME) {
+        /*check for set volume help*/
+        ret = dict_get_str(dict, "volname", &volname);
+        if (volname && ((strcmp(volname, "help") == 0) ||
+                        (strcmp(volname, "help-xml") == 0))) {
+            ret = dict_get_str(dict, "key1", &key);
+            if (ret < 0)
                 return _gf_true;
-
-        if (op == GD_OP_SET_VOLUME) {
-                /*check for set volume help*/
-                ret = dict_get_str (dict, "volname", &volname);
-                if (volname &&
-                    ((strcmp (volname, "help") == 0) ||
-                     (strcmp (volname, "help-xml") == 0))) {
-                        ret = dict_get_str (dict, "key1", &key);
-                        if (ret < 0)
-                                return _gf_true;
-                }
         }
-        return _gf_false;
+    }
+    return _gf_false;
 }
 
 gf_boolean_t
-glusterd_is_quorum_validation_required (xlator_t *this, glusterd_op_t op,
-                                        dict_t *dict)
+glusterd_is_quorum_validation_required(xlator_t *this, glusterd_op_t op,
+                                       dict_t *dict)
 {
-        gf_boolean_t    required        = _gf_true;
-        char            *key            = NULL;
-        char            *key_fixed      = NULL;
-        int             ret             = -1;
-
-        if (glusterd_is_get_op (this, op, dict)) {
-                required = _gf_false;
-                goto out;
-        }
-        if ((op != GD_OP_SET_VOLUME) && (op != GD_OP_RESET_VOLUME))
-                goto out;
-        if (op == GD_OP_SET_VOLUME)
-                ret = dict_get_str (dict, "key1", &key);
-        else if (op == GD_OP_RESET_VOLUME)
-                ret = dict_get_str (dict, "key", &key);
-        if (ret)
-                goto out;
-        ret = glusterd_check_option_exists (key, &key_fixed);
-        if (ret <= 0)
-                goto out;
-        if (key_fixed)
-                key = key_fixed;
-        if (glusterd_is_quorum_option (key))
-                required = _gf_false;
+    gf_boolean_t required = _gf_true;
+    char *key = NULL;
+    char *key_fixed = NULL;
+    int ret = -1;
+
+    if (glusterd_is_get_op(this, op, dict)) {
+        required = _gf_false;
+        goto out;
+    }
+    if ((op != GD_OP_SET_VOLUME) && (op != GD_OP_RESET_VOLUME))
+        goto out;
+    if (op == GD_OP_SET_VOLUME)
+        ret = dict_get_str(dict, "key1", &key);
+    else if (op == GD_OP_RESET_VOLUME)
+        ret = dict_get_str(dict, "key", &key);
+    if (ret)
+        goto out;
+    ret = glusterd_check_option_exists(key, &key_fixed);
+    if (ret <= 0)
+        goto out;
+    if (key_fixed)
+        key = key_fixed;
+    if (glusterd_is_quorum_option(key))
+        required = _gf_false;
 out:
-        GF_FREE (key_fixed);
-        return required;
+    GF_FREE(key_fixed);
+    return required;
 }
 
 int
-glusterd_validate_quorum (xlator_t *this, glusterd_op_t op,
-                             dict_t *dict, char **op_errstr)
+glusterd_validate_quorum(xlator_t *this, glusterd_op_t op, dict_t *dict,
+                         char **op_errstr)
 {
-        int                      ret     = 0;
-        char                    *volname = NULL;
-        glusterd_volinfo_t      *volinfo = NULL;
-        char                    *errstr  = NULL;
+    int ret = 0;
+    char *volname = NULL;
+    glusterd_volinfo_t *volinfo = NULL;
+    char *errstr = NULL;
+
+    errstr = "Quorum not met. Volume operation not allowed.";
+    if (!glusterd_is_quorum_validation_required(this, op, dict))
+        goto out;
+
+    ret = dict_get_str(dict, "volname", &volname);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_GET_FAILED,
+                "Key=volname", NULL);
+        ret = 0;
+        goto out;
+    }
 
-        errstr = "Quorum not met. Volume operation not allowed.";
-        if (!glusterd_is_quorum_validation_required (this, op, dict))
-                goto out;
+    ret = glusterd_volinfo_find(volname, &volinfo);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_VOLINFO_GET_FAIL, NULL);
+        ret = 0;
+        goto out;
+    }
 
-        ret = dict_get_str (dict, "volname", &volname);
-        if (ret) {
-                ret = 0;
-                goto out;
-        }
+    if (!glusterd_is_volume_in_server_quorum(volinfo)) {
+        ret = 0;
+        goto out;
+    }
 
-        ret = glusterd_volinfo_find (volname, &volinfo);
-        if (ret) {
-                ret = 0;
-                goto out;
-        }
+    if (does_gd_meet_server_quorum(this)) {
+        ret = 0;
+        goto out;
+    }
 
-        if (does_gd_meet_server_quorum (this)) {
-                ret = 0;
-                goto out;
-        }
+    ret = -1;
+    *op_errstr = gf_strdup(errstr);
 
-        if (glusterd_is_volume_in_server_quorum (volinfo)) {
-                ret = -1;
-                *op_errstr = gf_strdup (errstr);
-                goto out;
-        }
-        ret = 0;
 out:
-        return ret;
+    return ret;
 }
 
 gf_boolean_t
-glusterd_is_quorum_option (char *option)
+glusterd_is_quorum_option(char *option)
 {
-        gf_boolean_t    res     = _gf_false;
-        int             i       = 0;
-        static const char * const keys[] = {GLUSTERD_QUORUM_TYPE_KEY,
-                                            GLUSTERD_QUORUM_RATIO_KEY,
-                                            NULL};
-
-        for (i = 0; keys[i]; i++) {
-                if (strcmp (option, keys[i]) == 0) {
-                        res = _gf_true;
-                        break;
-                }
+    gf_boolean_t res = _gf_false;
+    int i = 0;
+    static const char *const keys[] = {GLUSTERD_QUORUM_TYPE_KEY,
+                                       GLUSTERD_QUORUM_RATIO_KEY, NULL};
+
+    for (i = 0; keys[i]; i++) {
+        if (strcmp(option, keys[i]) == 0) {
+            res = _gf_true;
+            break;
         }
-        return res;
+    }
+    return res;
 }
 
 gf_boolean_t
-glusterd_is_quorum_changed (dict_t *options, char *option, char *value)
+glusterd_is_quorum_changed(dict_t *options, char *option, char *value)
 {
-        int             ret             = 0;
-        gf_boolean_t    reconfigured    = _gf_false;
-        gf_boolean_t    all             = _gf_false;
-        char            *oldquorum      = NULL;
-        char            *newquorum      = NULL;
-        char            *oldratio       = NULL;
-        char            *newratio       = NULL;
-
-        if ((strcmp ("all", option) != 0) &&
-            !glusterd_is_quorum_option (option))
-                goto out;
-
-        if (strcmp ("all", option) == 0)
-                all = _gf_true;
-
-        if (all || (strcmp (GLUSTERD_QUORUM_TYPE_KEY, option) == 0)) {
-                newquorum = value;
-                ret = dict_get_str (options, GLUSTERD_QUORUM_TYPE_KEY,
-                                    &oldquorum);
-        }
+    int ret = 0;
+    gf_boolean_t reconfigured = _gf_false;
+    gf_boolean_t all = _gf_false;
+    char *oldquorum = NULL;
+    char *newquorum = NULL;
+    char *oldratio = NULL;
+    char *newratio = NULL;
+    xlator_t *this = NULL;
+
+    this = THIS;
+
+    if ((strcmp("all", option) != 0) && !glusterd_is_quorum_option(option))
+        goto out;
+
+    if (strcmp("all", option) == 0)
+        all = _gf_true;
+
+    if (all || (strcmp(GLUSTERD_QUORUM_TYPE_KEY, option) == 0)) {
+        newquorum = value;
+        ret = dict_get_str(options, GLUSTERD_QUORUM_TYPE_KEY, &oldquorum);
+        if (ret)
+            gf_msg(this->name, GF_LOG_DEBUG, 0, GD_MSG_DICT_GET_FAILED,
+                   "dict_get_str failed on %s", GLUSTERD_QUORUM_TYPE_KEY);
+    }
 
-        if (all || (strcmp (GLUSTERD_QUORUM_RATIO_KEY, option) == 0)) {
-                newratio = value;
-                ret = dict_get_str (options, GLUSTERD_QUORUM_RATIO_KEY,
-                                    &oldratio);
-        }
+    if (all || (strcmp(GLUSTERD_QUORUM_RATIO_KEY, option) == 0)) {
+        newratio = value;
+        ret = dict_get_str(options, GLUSTERD_QUORUM_RATIO_KEY, &oldratio);
+        if (ret)
+            gf_msg(this->name, GF_LOG_DEBUG, 0, GD_MSG_DICT_GET_FAILED,
+                   "dict_get_str failed on %s", GLUSTERD_QUORUM_RATIO_KEY);
+    }
 
-        reconfigured = _gf_true;
+    reconfigured = _gf_true;
 
-        if (oldquorum && newquorum && (strcmp (oldquorum, newquorum) == 0))
-                reconfigured = _gf_false;
-        if (oldratio && newratio && (strcmp (oldratio, newratio) == 0))
-                reconfigured = _gf_false;
+    if (oldquorum && newquorum && (strcmp(oldquorum, newquorum) == 0))
+        reconfigured = _gf_false;
+    if (oldratio && newratio && (strcmp(oldratio, newratio) == 0))
+        reconfigured = _gf_false;
 
-        if ((oldratio == NULL) && (newratio == NULL) && (oldquorum == NULL) &&
-            (newquorum == NULL))
-                reconfigured = _gf_false;
+    if ((oldratio == NULL) && (newratio == NULL) && (oldquorum == NULL) &&
+        (newquorum == NULL))
+        reconfigured = _gf_false;
 out:
-        return reconfigured;
+    return reconfigured;
 }
 
-static inline gf_boolean_t
-_is_contributing_to_quorum (gd_quorum_contrib_t contrib)
+static gf_boolean_t
+_is_contributing_to_quorum(gd_quorum_contrib_t contrib)
 {
-        if ((contrib == QUORUM_UP) || (contrib == QUORUM_DOWN))
-                return _gf_true;
-        return _gf_false;
+    if ((contrib == QUORUM_UP) || (contrib == QUORUM_DOWN))
+        return _gf_true;
+    return _gf_false;
 }
 
-static inline gf_boolean_t
-_does_quorum_meet (int active_count, int quorum_count)
+gf_boolean_t
+does_quorum_meet(int active_count, int quorum_count)
 {
-        return (active_count >= quorum_count);
+    return (active_count >= quorum_count);
 }
 
 int
-glusterd_get_quorum_cluster_counts (xlator_t *this, int *active_count,
-                                    int *quorum_count)
+glusterd_get_quorum_cluster_counts(xlator_t *this, int *active_count,
+                                   int *quorum_count)
 {
-        glusterd_peerinfo_t *peerinfo      = NULL;
-        glusterd_conf_t     *conf          = NULL;
-        int                 ret            = -1;
-        int                 inquorum_count = 0;
-        char                *val           = NULL;
-        double              quorum_percentage = 0.0;
-        gf_boolean_t        ratio          = _gf_false;
-        int                 count          = 0;
-
-        conf = this->private;
-
-        /* Start with counting self */
-        inquorum_count = 1;
-        if (active_count)
-                *active_count = 1;
-
-        rcu_read_lock ();
-        cds_list_for_each_entry_rcu (peerinfo, &conf->peers, uuid_list) {
-                if (peerinfo->quorum_contrib == QUORUM_WAITING) {
-                        rcu_read_unlock ();
-                        goto out;
-                }
-
-                if (_is_contributing_to_quorum (peerinfo->quorum_contrib))
-                        inquorum_count = inquorum_count + 1;
-                if (active_count && (peerinfo->quorum_contrib == QUORUM_UP))
-                        *active_count = *active_count + 1;
-        }
-        rcu_read_unlock ();
-
-        ret = dict_get_str (conf->opts, GLUSTERD_QUORUM_RATIO_KEY, &val);
-        if (ret == 0) {
-                ratio = _gf_true;
-                ret = gf_string2percent (val, &quorum_percentage);
-                if (!ret)
-                        ratio = _gf_true;
-        }
-        if (ratio)
-                count = CEILING_POS (inquorum_count *
-                                     quorum_percentage / 100.0);
-        else
-                count = (inquorum_count * 50 / 100) + 1;
-
-        *quorum_count = count;
-        ret = 0;
-out:
-        return ret;
+    glusterd_peerinfo_t *peerinfo = NULL;
+    glusterd_conf_t *conf = NULL;
+    int ret = -1;
+    int inquorum_count = 0;
+    char *val = NULL;
+    double quorum_percentage = 0.0;
+    gf_boolean_t ratio = _gf_false;
+    int count = 0;
+
+    conf = this->private;
+
+    /* Start with counting self */
+    inquorum_count = 1;
+    if (active_count)
+        *active_count = 1;
+
+    RCU_READ_LOCK;
+    cds_list_for_each_entry_rcu(peerinfo, &conf->peers, uuid_list)
+    {
+        if (_is_contributing_to_quorum(peerinfo->quorum_contrib))
+            inquorum_count = inquorum_count + 1;
+        if (active_count && (peerinfo->quorum_contrib == QUORUM_UP))
+            *active_count = *active_count + 1;
+    }
+    RCU_READ_UNLOCK;
+
+    ret = dict_get_str(conf->opts, GLUSTERD_QUORUM_RATIO_KEY, &val);
+    if (ret == 0) {
+        ret = gf_string2percent(val, &quorum_percentage);
+        if (ret == 0)
+            ratio = _gf_true;
+    }
+    if (ratio)
+        count = CEILING_POS(inquorum_count * quorum_percentage / 100.0);
+    else
+        count = (inquorum_count * 50 / 100) + 1;
+
+    *quorum_count = count;
+    ret = 0;
+
+    return ret;
 }
 
 gf_boolean_t
-glusterd_is_volume_in_server_quorum (glusterd_volinfo_t *volinfo)
+glusterd_is_volume_in_server_quorum(glusterd_volinfo_t *volinfo)
 {
-        gf_boolean_t    res             = _gf_false;
-        char            *quorum_type    = NULL;
-        int             ret             = 0;
-
-        ret = dict_get_str (volinfo->dict, GLUSTERD_QUORUM_TYPE_KEY,
-                            &quorum_type);
-        if (ret)
-                goto out;
-
-        if (strcmp (quorum_type, GLUSTERD_SERVER_QUORUM) == 0)
-                res = _gf_true;
+    gf_boolean_t res = _gf_false;
+    char *quorum_type = NULL;
+    int ret = 0;
+
+    ret = dict_get_str(volinfo->dict, GLUSTERD_QUORUM_TYPE_KEY, &quorum_type);
+    if (ret) {
+        gf_smsg(THIS->name, GF_LOG_ERROR, errno, GD_MSG_DICT_GET_FAILED,
+                "Key=%s", GLUSTERD_QUORUM_TYPE_KEY, NULL);
+        goto out;
+    }
+
+    if (strcmp(quorum_type, GLUSTERD_SERVER_QUORUM) == 0)
+        res = _gf_true;
 out:
-        return res;
+    return res;
 }
 
 gf_boolean_t
-glusterd_is_any_volume_in_server_quorum (xlator_t *this)
+glusterd_is_any_volume_in_server_quorum(xlator_t *this)
 {
-        glusterd_conf_t     *conf     = NULL;
-        glusterd_volinfo_t  *volinfo  = NULL;
-
-        conf = this->private;
-        list_for_each_entry (volinfo, &conf->volumes, vol_list) {
-                if (glusterd_is_volume_in_server_quorum (volinfo)) {
-                        return _gf_true;
-                }
+    glusterd_conf_t *conf = NULL;
+    glusterd_volinfo_t *volinfo = NULL;
+
+    conf = this->private;
+    list_for_each_entry(volinfo, &conf->volumes, vol_list)
+    {
+        if (glusterd_is_volume_in_server_quorum(volinfo)) {
+            return _gf_true;
         }
-        return _gf_false;
+    }
+    return _gf_false;
 }
 
 gf_boolean_t
-does_gd_meet_server_quorum (xlator_t *this)
+does_gd_meet_server_quorum(xlator_t *this)
 {
-        int                     quorum_count    = 0;
-        int                     active_count    = 0;
-        gf_boolean_t            in              = _gf_false;
-        glusterd_conf_t         *conf           = NULL;
-        int                     ret             = -1;
-
-        conf = this->private;
-        ret = glusterd_get_quorum_cluster_counts (this, &active_count,
-                                                  &quorum_count);
-        if (ret)
-                goto out;
-
-        if (!_does_quorum_meet (active_count, quorum_count)) {
-                goto out;
-        }
-
-        in = _gf_true;
+    int quorum_count = 0;
+    int active_count = 0;
+    gf_boolean_t in = _gf_false;
+    int ret = -1;
+
+    ret = glusterd_get_quorum_cluster_counts(this, &active_count,
+                                             &quorum_count);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno,
+                GD_MSG_QUORUM_CLUSTER_COUNT_GET_FAIL, NULL);
+        goto out;
+    }
+
+    if (!does_quorum_meet(active_count, quorum_count)) {
+        goto out;
+    }
+
+    in = _gf_true;
 out:
-        return in;
+    return in;
 }
 
 void
-glusterd_do_volume_quorum_action (xlator_t *this, glusterd_volinfo_t *volinfo,
-                                  gf_boolean_t meets_quorum)
+glusterd_do_volume_quorum_action(xlator_t *this, glusterd_volinfo_t *volinfo,
+                                 gf_boolean_t meets_quorum)
 {
-        glusterd_brickinfo_t *brickinfo     = NULL;
-        glusterd_conf_t      *conf          = NULL;
-        gd_quorum_status_t   quorum_status  = NOT_APPLICABLE_QUORUM;
-        gf_boolean_t         follows_quorum = _gf_false;
-
-        conf = this->private;
-        if (volinfo->status != GLUSTERD_STATUS_STARTED) {
-                volinfo->quorum_status = NOT_APPLICABLE_QUORUM;
-                goto out;
-        }
-
-        follows_quorum = glusterd_is_volume_in_server_quorum (volinfo);
-        if (follows_quorum) {
-                if (meets_quorum)
-                        quorum_status = MEETS_QUORUM;
-                else
-                        quorum_status = DOESNT_MEET_QUORUM;
+    int ret = -1;
+    glusterd_brickinfo_t *brickinfo = NULL;
+    gd_quorum_status_t quorum_status = NOT_APPLICABLE_QUORUM;
+    gf_boolean_t follows_quorum = _gf_false;
+    gf_boolean_t quorum_status_unchanged = _gf_false;
+
+    if (volinfo->status != GLUSTERD_STATUS_STARTED) {
+        volinfo->quorum_status = NOT_APPLICABLE_QUORUM;
+        goto out;
+    }
+
+    follows_quorum = glusterd_is_volume_in_server_quorum(volinfo);
+    if (follows_quorum) {
+        if (meets_quorum)
+            quorum_status = MEETS_QUORUM;
+        else
+            quorum_status = DOESNT_MEET_QUORUM;
+    } else {
+        quorum_status = NOT_APPLICABLE_QUORUM;
+    }
+
+    /*
+     * The following check is added to prevent spurious brick starts when
+     * events occur that affect quorum.
+     * Example:
+     * There is a cluster of 10 peers. Volume is in quorum. User
+     * takes down one brick from the volume to perform maintenance.
+     * Suddenly one of the peers go down. Cluster is still in quorum. But
+     * because of this 'peer going down' event, quorum is calculated and
+     * the bricks that are down are brought up again. In this process it
+     * also brings up the brick that is purposefully taken down.
+     */
+    if (volinfo->quorum_status == quorum_status) {
+        quorum_status_unchanged = _gf_true;
+        goto out;
+    }
+
+    if (quorum_status == MEETS_QUORUM) {
+        gf_msg(this->name, GF_LOG_CRITICAL, 0,
+               GD_MSG_SERVER_QUORUM_MET_STARTING_BRICKS,
+               "Server quorum regained for volume %s. Starting local "
+               "bricks.",
+               volinfo->volname);
+        gf_event(EVENT_QUORUM_REGAINED, "volume=%s", volinfo->volname);
+    } else if (quorum_status == DOESNT_MEET_QUORUM) {
+        gf_msg(this->name, GF_LOG_CRITICAL, 0,
+               GD_MSG_SERVER_QUORUM_LOST_STOPPING_BRICKS,
+               "Server quorum lost for volume %s. Stopping local "
+               "bricks.",
+               volinfo->volname);
+        gf_event(EVENT_QUORUM_LOST, "volume=%s", volinfo->volname);
+    }
+
+    list_for_each_entry(brickinfo, &volinfo->bricks, brick_list)
+    {
+        if (!glusterd_is_local_brick(this, volinfo, brickinfo))
+            continue;
+        if (quorum_status == DOESNT_MEET_QUORUM) {
+            ret = glusterd_brick_stop(volinfo, brickinfo, _gf_false);
+            if (ret) {
+                gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_BRICK_STOP_FAIL,
+                       "Failed to "
+                       "stop brick %s:%s",
+                       brickinfo->hostname, brickinfo->path);
+            }
         } else {
-                quorum_status = NOT_APPLICABLE_QUORUM;
+            if (!brickinfo->start_triggered) {
+                pthread_mutex_lock(&brickinfo->restart_mutex);
+                {
+                    /* coverity[SLEEP] */
+                    ret = glusterd_brick_start(volinfo, brickinfo, _gf_false,
+                                               _gf_false);
+                }
+                pthread_mutex_unlock(&brickinfo->restart_mutex);
+                if (ret) {
+                    gf_msg(this->name, GF_LOG_ERROR, 0,
+                           GD_MSG_BRICK_DISCONNECTED, "Failed to start %s:%s",
+                           brickinfo->hostname, brickinfo->path);
+                }
+            }
         }
-
-        /*
-         * The following check is added to prevent spurious brick starts when
-         * events occur that affect quorum.
-         * Example:
-         * There is a cluster of 10 peers. Volume is in quorum. User
-         * takes down one brick from the volume to perform maintenance.
-         * Suddenly one of the peers go down. Cluster is still in quorum. But
-         * because of this 'peer going down' event, quorum is calculated and
-         * the bricks that are down are brought up again. In this process it
-         * also brings up the brick that is purposefully taken down.
+    }
+    volinfo->quorum_status = quorum_status;
+    if (quorum_status == MEETS_QUORUM) {
+        /* bricks might have been restarted and so as the port change
+         * might have happened
          */
-        if (volinfo->quorum_status == quorum_status)
-                goto out;
-
-        if (quorum_status == MEETS_QUORUM) {
-                gf_msg (this->name, GF_LOG_CRITICAL, 0,
-                        GD_MSG_SERVER_QUORUM_MET_STARTING_BRICKS,
-                        "Server quorum regained for volume %s. Starting local "
-                        "bricks.", volinfo->volname);
-        } else if (quorum_status == DOESNT_MEET_QUORUM) {
-                gf_msg (this->name, GF_LOG_CRITICAL, 0,
-                        GD_MSG_SERVER_QUORUM_LOST_STOPPING_BRICKS,
-                        "Server quorum lost for volume %s. Stopping local "
-                        "bricks.", volinfo->volname);
-        }
-
-        list_for_each_entry (brickinfo, &volinfo->bricks, brick_list) {
-                if (!glusterd_is_local_brick (this, volinfo, brickinfo))
-                        continue;
-                if (quorum_status == DOESNT_MEET_QUORUM)
-                        glusterd_brick_stop (volinfo, brickinfo, _gf_false);
-                else
-                        glusterd_brick_start (volinfo, brickinfo, _gf_false);
+        ret = glusterd_store_volinfo(volinfo, GLUSTERD_VOLINFO_VER_AC_NONE);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_VOLINFO_STORE_FAIL,
+                   "Failed to write volinfo for volume %s", volinfo->volname);
+            goto out;
         }
-        volinfo->quorum_status = quorum_status;
+    }
 out:
-        return;
+    if (quorum_status_unchanged) {
+        list_for_each_entry(brickinfo, &volinfo->bricks, brick_list)
+        {
+            if (!glusterd_is_local_brick(this, volinfo, brickinfo))
+                continue;
+            ret = glusterd_brick_start(volinfo, brickinfo, _gf_false, _gf_true);
+            if (ret) {
+                gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_BRICK_DISCONNECTED,
+                       "Failed to "
+                       "connect to %s:%s",
+                       brickinfo->hostname, brickinfo->path);
+            }
+        }
+    }
+    return;
 }
 
 int
-glusterd_do_quorum_action ()
+glusterd_do_quorum_action()
 {
-        xlator_t            *this          = NULL;
-        glusterd_conf_t     *conf          = NULL;
-        glusterd_volinfo_t  *volinfo       = NULL;
-        int                 ret            = 0;
-        int                 active_count   = 0;
-        int                 quorum_count   = 0;
-        gf_boolean_t        meets          = _gf_false;
-
-        this = THIS;
-        conf = this->private;
-
-        conf->pending_quorum_action = _gf_true;
-        ret = glusterd_lock (conf->uuid);
+    xlator_t *this = NULL;
+    glusterd_conf_t *conf = NULL;
+    glusterd_volinfo_t *volinfo = NULL;
+    int ret = 0;
+    int active_count = 0;
+    int quorum_count = 0;
+    gf_boolean_t meets = _gf_false;
+
+    this = THIS;
+    conf = this->private;
+
+    conf->pending_quorum_action = _gf_true;
+    ret = glusterd_lock(conf->uuid);
+    if (ret)
+        goto out;
+
+    {
+        ret = glusterd_get_quorum_cluster_counts(this, &active_count,
+                                                 &quorum_count);
         if (ret)
-                goto out;
+            goto unlock;
 
+        if (does_quorum_meet(active_count, quorum_count))
+            meets = _gf_true;
+        list_for_each_entry(volinfo, &conf->volumes, vol_list)
         {
-                ret = glusterd_get_quorum_cluster_counts (this, &active_count,
-                                                          &quorum_count);
-                if (ret)
-                        goto unlock;
-
-                if (_does_quorum_meet (active_count, quorum_count))
-                        meets = _gf_true;
-                list_for_each_entry (volinfo, &conf->volumes, vol_list) {
-                        glusterd_do_volume_quorum_action (this, volinfo, meets);
-                }
+            glusterd_do_volume_quorum_action(this, volinfo, meets);
         }
+    }
 unlock:
-        (void)glusterd_unlock (conf->uuid);
-        conf->pending_quorum_action = _gf_false;
+    (void)glusterd_unlock(conf->uuid);
+    conf->pending_quorum_action = _gf_false;
 out:
-        return ret;
+    return ret;
+}
+
+/* ret = 0 represents quorum is not met
+ * ret = 1 represents quorum is met
+ * ret = 2 represents quorum not applicable
+ */
+
+int
+check_quorum_for_brick_start(glusterd_volinfo_t *volinfo,
+                             gf_boolean_t node_quorum)
+{
+    gf_boolean_t volume_quorum = _gf_false;
+    int ret = 0;
+
+    volume_quorum = glusterd_is_volume_in_server_quorum(volinfo);
+    if (volume_quorum) {
+        if (node_quorum)
+            ret = 1;
+    } else {
+        ret = 2;
+    }
+    return ret;
 }
diff --git a/xlators/mgmt/glusterd/src/glusterd-server-quorum.h b/xlators/mgmt/glusterd/src/glusterd-server-quorum.h
index f43cdbbca10..e11bf1a9206 100644
--- a/xlators/mgmt/glusterd/src/glusterd-server-quorum.h
+++ b/xlators/mgmt/glusterd/src/glusterd-server-quorum.h
@@ -10,34 +10,37 @@
 #ifndef _GLUSTERD_SERVER_QUORUM_H
 #define _GLUSTERD_SERVER_QUORUM_H
 
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
-
 int
-glusterd_validate_quorum (xlator_t *this, glusterd_op_t op, dict_t *dict,
-                          char **op_errstr);
+glusterd_validate_quorum(xlator_t *this, glusterd_op_t op, dict_t *dict,
+                         char **op_errstr);
 
 gf_boolean_t
-glusterd_is_quorum_changed (dict_t *options, char *option, char *value);
+glusterd_is_quorum_changed(dict_t *options, char *option, char *value);
 
 int
-glusterd_do_quorum_action ();
+glusterd_do_quorum_action();
 
 int
-glusterd_get_quorum_cluster_counts (xlator_t *this, int *active_count,
-                                    int *quorum_count);
+glusterd_get_quorum_cluster_counts(xlator_t *this, int *active_count,
+                                   int *quorum_count);
+
+gf_boolean_t
+glusterd_is_quorum_option(char *option);
 
 gf_boolean_t
-glusterd_is_quorum_option (char *option);
+glusterd_is_volume_in_server_quorum(glusterd_volinfo_t *volinfo);
 
 gf_boolean_t
-glusterd_is_volume_in_server_quorum (glusterd_volinfo_t *volinfo);
+glusterd_is_any_volume_in_server_quorum(xlator_t *this);
 
 gf_boolean_t
-glusterd_is_any_volume_in_server_quorum (xlator_t *this);
+does_gd_meet_server_quorum(xlator_t *this);
+
+int
+check_quorum_for_brick_start(glusterd_volinfo_t *volinfo,
+                             gf_boolean_t node_quorum);
 
 gf_boolean_t
-does_gd_meet_server_quorum (xlator_t *this);
+does_quorum_meet(int active_count, int quorum_count);
+
 #endif /* _GLUSTERD_SERVER_QUORUM_H */
diff --git a/xlators/mgmt/glusterd/src/glusterd-shd-svc-helper.c b/xlators/mgmt/glusterd/src/glusterd-shd-svc-helper.c
new file mode 100644
index 00000000000..5661e391a9c
--- /dev/null
+++ b/xlators/mgmt/glusterd/src/glusterd-shd-svc-helper.c
@@ -0,0 +1,153 @@
+/*
+   Copyright (c) 2016 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
+
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
+
+#include "glusterd.h"
+#include "glusterd-utils.h"
+#include "glusterd-shd-svc-helper.h"
+#include "glusterd-messages.h"
+#include "glusterd-volgen.h"
+
+void
+glusterd_svc_build_shd_socket_filepath(glusterd_volinfo_t *volinfo, char *path,
+                                       int path_len)
+{
+    char sockfilepath[PATH_MAX] = {
+        0,
+    };
+    char rundir[PATH_MAX] = {
+        0,
+    };
+    int32_t len = 0;
+    glusterd_conf_t *priv = THIS->private;
+
+    if (!priv)
+        return;
+
+    GLUSTERD_GET_SHD_RUNDIR(rundir, volinfo, priv);
+    len = snprintf(sockfilepath, sizeof(sockfilepath), "%s/run-%s", rundir,
+                   uuid_utoa(MY_UUID));
+    if ((len < 0) || (len >= sizeof(sockfilepath))) {
+        sockfilepath[0] = 0;
+    }
+
+    glusterd_set_socket_filepath(sockfilepath, path, path_len);
+}
+
+void
+glusterd_svc_build_shd_pidfile(glusterd_volinfo_t *volinfo, char *path,
+                               int path_len)
+{
+    char rundir[PATH_MAX] = {
+        0,
+    };
+    glusterd_conf_t *priv = THIS->private;
+
+    if (!priv)
+        return;
+
+    GLUSTERD_GET_SHD_RUNDIR(rundir, volinfo, priv);
+
+    snprintf(path, path_len, "%s/%s-shd.pid", rundir, volinfo->volname);
+}
+
+void
+glusterd_svc_build_shd_volfile_path(glusterd_volinfo_t *volinfo, char *path,
+                                    int path_len)
+{
+    char workdir[PATH_MAX] = {
+        0,
+    };
+    glusterd_conf_t *priv = THIS->private;
+
+    if (!priv)
+        return;
+
+    GLUSTERD_GET_VOLUME_DIR(workdir, volinfo, priv);
+
+    snprintf(path, path_len, "%s/%s-shd.vol", workdir, volinfo->volname);
+}
+
+void
+glusterd_shd_svcproc_cleanup(glusterd_shdsvc_t *shd)
+{
+    glusterd_svc_proc_t *svc_proc = NULL;
+    glusterd_svc_t *svc = NULL;
+    glusterd_conf_t *conf = NULL;
+    gf_boolean_t need_unref = _gf_false;
+    rpc_clnt_t *rpc = NULL;
+
+    conf = THIS->private;
+    if (!conf)
+        return;
+
+    GF_VALIDATE_OR_GOTO(THIS->name, conf, out);
+    GF_VALIDATE_OR_GOTO(THIS->name, shd, out);
+
+    svc = &shd->svc;
+    shd->attached = _gf_false;
+
+    if (svc->conn.rpc) {
+        rpc_clnt_unref(svc->conn.rpc);
+        svc->conn.rpc = NULL;
+    }
+
+    pthread_mutex_lock(&conf->attach_lock);
+    {
+        svc_proc = svc->svc_proc;
+        svc->svc_proc = NULL;
+        svc->inited = _gf_false;
+        cds_list_del_init(&svc->mux_svc);
+        glusterd_unlink_file(svc->proc.pidfile);
+
+        if (svc_proc && cds_list_empty(&svc_proc->svcs)) {
+            cds_list_del_init(&svc_proc->svc_proc_list);
+            /* We cannot free svc_proc list from here. Because
+             * if there are pending events on the rpc, it will
+             * try to access the corresponding svc_proc, so unrefing
+             * rpc request and then cleaning up the memory is carried
+             * from the notify function upon RPC_CLNT_DESTROY destroy.
+             */
+            need_unref = _gf_true;
+            rpc = svc_proc->rpc;
+            svc_proc->rpc = NULL;
+        }
+    }
+    pthread_mutex_unlock(&conf->attach_lock);
+    /*rpc unref has to be performed outside the lock*/
+    if (need_unref && rpc)
+        rpc_clnt_unref(rpc);
+out:
+    return;
+}
+
+int
+glusterd_svc_set_shd_pidfile(glusterd_volinfo_t *volinfo, dict_t *dict)
+{
+    int ret = -1;
+    glusterd_svc_t *svc = NULL;
+    xlator_t *this = NULL;
+
+    this = THIS;
+    GF_VALIDATE_OR_GOTO("glusterd", this, out);
+    GF_VALIDATE_OR_GOTO(this->name, volinfo, out);
+    GF_VALIDATE_OR_GOTO(this->name, dict, out);
+
+    svc = &(volinfo->shd.svc);
+
+    ret = dict_set_dynstr_with_alloc(dict, "pidfile", svc->proc.pidfile);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+               "Failed to set pidfile %s in dict", svc->proc.pidfile);
+        goto out;
+    }
+    ret = 0;
+out:
+    return ret;
+}
diff --git a/xlators/mgmt/glusterd/src/glusterd-shd-svc-helper.h b/xlators/mgmt/glusterd/src/glusterd-shd-svc-helper.h
new file mode 100644
index 00000000000..1f0984ba857
--- /dev/null
+++ b/xlators/mgmt/glusterd/src/glusterd-shd-svc-helper.h
@@ -0,0 +1,42 @@
+/*
+   Copyright (c) 2016 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
+
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
+
+#ifndef _GLUSTERD_SHD_SVC_HELPER_H_
+#define _GLUSTERD_SHD_SVC_HELPER_H_
+
+#include "glusterd.h"
+#include "glusterd-svc-mgmt.h"
+
+void
+glusterd_svc_build_shd_socket_filepath(glusterd_volinfo_t *volinfo, char *path,
+                                       int path_len);
+
+void
+glusterd_svc_build_shd_pidfile(glusterd_volinfo_t *volinfo, char *path,
+                               int path_len);
+
+void
+glusterd_svc_build_shd_volfile_path(glusterd_volinfo_t *volinfo, char *path,
+                                    int path_len);
+
+void
+glusterd_shd_svcproc_cleanup(glusterd_shdsvc_t *shd);
+
+int
+glusterd_recover_shd_attach_failure(glusterd_volinfo_t *volinfo,
+                                    glusterd_svc_t *svc, int flags);
+
+int
+glusterd_shdsvc_create_volfile(glusterd_volinfo_t *volinfo);
+
+int
+glusterd_svc_set_shd_pidfile(glusterd_volinfo_t *volinfo, dict_t *dict);
+
+#endif
diff --git a/xlators/mgmt/glusterd/src/glusterd-shd-svc.c b/xlators/mgmt/glusterd/src/glusterd-shd-svc.c
index dd85d1e3638..1c56384a14b 100644
--- a/xlators/mgmt/glusterd/src/glusterd-shd-svc.c
+++ b/xlators/mgmt/glusterd/src/glusterd-shd-svc.c
@@ -8,160 +8,789 @@
    cases as published by the Free Software Foundation.
 */
 
-#include "globals.h"
-#include "run.h"
+#include <glusterfs/globals.h>
+#include <glusterfs/run.h>
 #include "glusterd.h"
 #include "glusterd-utils.h"
 #include "glusterd-volgen.h"
-#include "glusterd-svc-mgmt.h"
 #include "glusterd-shd-svc.h"
+#include "glusterd-shd-svc-helper.h"
+#include "glusterd-svc-helper.h"
+#include "glusterd-store.h"
 
+#define GD_SHD_PROCESS_NAME "--process-name"
 char *shd_svc_name = "glustershd";
 
-int
-glusterd_shdsvc_init (glusterd_svc_t *svc)
+void
+glusterd_shdsvc_build(glusterd_svc_t *svc)
 {
-        return glusterd_svc_init (svc, shd_svc_name,
-                                  glusterd_shdsvc_manager,
-                                  glusterd_shdsvc_start,
-                                  glusterd_svc_stop);
+    int ret = -1;
+    ret = snprintf(svc->name, sizeof(svc->name), "%s", shd_svc_name);
+    if (ret < 0)
+        return;
+
+    CDS_INIT_LIST_HEAD(&svc->mux_svc);
+    svc->manager = glusterd_shdsvc_manager;
+    svc->start = glusterd_shdsvc_start;
+    svc->stop = glusterd_shdsvc_stop;
+    svc->reconfigure = glusterd_shdsvc_reconfigure;
 }
 
-static int
-glusterd_shdsvc_create_volfile ()
+int
+glusterd_shdsvc_init(void *data, glusterd_conn_t *mux_conn,
+                     glusterd_svc_proc_t *mux_svc)
 {
-        char            filepath[PATH_MAX] = {0,};
-        int             ret = -1;
-        glusterd_conf_t *conf = THIS->private;
-        dict_t          *mod_dict = NULL;
+    int ret = -1;
+    char rundir[PATH_MAX] = {
+        0,
+    };
+    char sockpath[PATH_MAX] = {
+        0,
+    };
+    char pidfile[PATH_MAX] = {
+        0,
+    };
+    char volfile[PATH_MAX] = {
+        0,
+    };
+    char logdir[PATH_MAX] = {
+        0,
+    };
+    char logfile[PATH_MAX] = {
+        0,
+    };
+    char volfileid[256] = {0};
+    glusterd_svc_t *svc = NULL;
+    glusterd_volinfo_t *volinfo = NULL;
+    glusterd_conf_t *priv = NULL;
+    glusterd_muxsvc_conn_notify_t notify = NULL;
+    xlator_t *this = NULL;
+    char *volfileserver = NULL;
+    int32_t len = 0;
 
-        mod_dict = dict_new ();
-        if (!mod_dict)
-                goto out;
+    this = THIS;
+    GF_VALIDATE_OR_GOTO(THIS->name, this, out);
 
-        ret = dict_set_uint32 (mod_dict, "cluster.background-self-heal-count",
-                               0);
-        if (ret)
-                goto out;
+    priv = this->private;
+    GF_VALIDATE_OR_GOTO(this->name, priv, out);
 
-        ret = dict_set_str (mod_dict, "cluster.data-self-heal", "on");
-        if (ret)
-                goto out;
+    volinfo = data;
+    GF_VALIDATE_OR_GOTO(this->name, data, out);
+    GF_VALIDATE_OR_GOTO(this->name, mux_svc, out);
 
-        ret = dict_set_str (mod_dict, "cluster.metadata-self-heal", "on");
-        if (ret)
-                goto out;
+    svc = &(volinfo->shd.svc);
 
-        ret = dict_set_str (mod_dict, "cluster.entry-self-heal", "on");
-        if (ret)
-                goto out;
+    ret = snprintf(svc->name, sizeof(svc->name), "%s", shd_svc_name);
+    if (ret < 0)
+        goto out;
 
-        glusterd_svc_build_volfile_path (shd_svc_name, conf->workdir,
-                                         filepath, sizeof (filepath));
-        ret = glusterd_create_global_volfile (build_shd_graph, filepath,
-                                              mod_dict);
-        if (ret) {
-                gf_log (THIS->name, GF_LOG_ERROR, "Failed to create volfile");
-                goto out;
+    notify = glusterd_muxsvc_common_rpc_notify;
+    glusterd_store_perform_node_state_store(volinfo);
+
+    GLUSTERD_GET_SHD_RUNDIR(rundir, volinfo, priv);
+    glusterd_svc_create_rundir(rundir);
+
+    glusterd_svc_build_logfile_path(shd_svc_name, priv->logdir, logfile,
+                                    sizeof(logfile));
+
+    /* Initialize the connection mgmt */
+    if (mux_conn && mux_svc->rpc) {
+        /* multiplexed svc */
+        svc->conn.frame_timeout = mux_conn->frame_timeout;
+        /* This will be unrefed from glusterd_shd_svcproc_cleanup*/
+        svc->conn.rpc = rpc_clnt_ref(mux_svc->rpc);
+        ret = snprintf(svc->conn.sockpath, sizeof(svc->conn.sockpath), "%s",
+                       mux_conn->sockpath);
+        if (ret < 0)
+            goto out;
+    } else {
+        ret = mkdir_p(priv->logdir, 0755, _gf_true);
+        if ((ret == -1) && (EEXIST != errno)) {
+            gf_msg(this->name, GF_LOG_ERROR, errno, GD_MSG_CREATE_DIR_FAILED,
+                   "Unable to create logdir %s", logdir);
+            goto out;
         }
 
+        glusterd_svc_build_shd_socket_filepath(volinfo, sockpath,
+                                               sizeof(sockpath));
+        ret = glusterd_muxsvc_conn_init(&(svc->conn), mux_svc, sockpath, 600,
+                                        notify);
+        if (ret)
+            goto out;
+        /* This will be unrefed when the last svcs is detached from the list */
+        if (!mux_svc->rpc)
+            mux_svc->rpc = rpc_clnt_ref(svc->conn.rpc);
+    }
+
+    /* Initialize the process mgmt */
+    glusterd_svc_build_shd_pidfile(volinfo, pidfile, sizeof(pidfile));
+    glusterd_svc_build_shd_volfile_path(volinfo, volfile, PATH_MAX);
+    len = snprintf(volfileid, sizeof(volfileid), "shd/%s", volinfo->volname);
+    if ((len < 0) || (len >= sizeof(volfileid))) {
+        ret = -1;
+        goto out;
+    }
+
+    if (dict_get_strn(this->options, "transport.socket.bind-address",
+                      SLEN("transport.socket.bind-address"),
+                      &volfileserver) != 0) {
+        volfileserver = "localhost";
+    }
+    ret = glusterd_proc_init(&(svc->proc), shd_svc_name, pidfile, logdir,
+                             logfile, volfile, volfileid, volfileserver);
+    if (ret)
+        goto out;
+
+out:
+    gf_msg_debug(this ? this->name : "glusterd", 0, "Returning %d", ret);
+    return ret;
+}
+
+int
+glusterd_shdsvc_create_volfile(glusterd_volinfo_t *volinfo)
+{
+    char filepath[PATH_MAX] = {
+        0,
+    };
+
+    int ret = -1;
+    dict_t *mod_dict = NULL;
+    xlator_t *this = THIS;
+    GF_ASSERT(this);
+
+    glusterd_svc_build_shd_volfile_path(volinfo, filepath, PATH_MAX);
+    if (!glusterd_is_shd_compatible_volume(volinfo)) {
+        /* If volfile exist, delete it. This case happens when we
+         * change from replica/ec to distribute.
+         */
+        (void)glusterd_unlink_file(filepath);
+        ret = 0;
+        goto out;
+    }
+    mod_dict = dict_new();
+    if (!mod_dict) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_CREATE_FAIL, NULL);
+        goto out;
+    }
+
+    ret = dict_set_uint32(mod_dict, "cluster.background-self-heal-count", 0);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_SET_FAILED,
+                "Key=cluster.background-self-heal-count", NULL);
+        goto out;
+    }
+
+    ret = dict_set_str(mod_dict, "cluster.data-self-heal", "on");
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_SET_FAILED,
+                "Key=cluster.data-self-heal", NULL);
+        goto out;
+    }
+
+    ret = dict_set_str(mod_dict, "cluster.metadata-self-heal", "on");
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_SET_FAILED,
+                "Key=cluster.metadata-self-heal", NULL);
+        goto out;
+    }
+
+    ret = dict_set_str(mod_dict, "cluster.entry-self-heal", "on");
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_SET_FAILED,
+                "Key=cluster.entry-self-heal", NULL);
+        goto out;
+    }
+
+    ret = glusterd_shdsvc_generate_volfile(volinfo, filepath, mod_dict);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_VOLFILE_CREATE_FAIL,
+               "Failed to create volfile");
+        goto out;
+    }
+
 out:
-        if (mod_dict)
-                dict_unref (mod_dict);
-        gf_log (THIS->name, GF_LOG_DEBUG, "Returning %d", ret);
+    if (mod_dict)
+        dict_unref(mod_dict);
+    gf_msg_debug(this->name, 0, "Returning %d", ret);
+
+    return ret;
+}
+
+gf_boolean_t
+glusterd_svcs_shd_compatible_volumes_stopped(glusterd_svc_t *svc)
+{
+    glusterd_svc_proc_t *svc_proc = NULL;
+    glusterd_shdsvc_t *shd = NULL;
+    glusterd_svc_t *temp_svc = NULL;
+    glusterd_volinfo_t *volinfo = NULL;
+    gf_boolean_t comp = _gf_false;
+    glusterd_conf_t *conf = THIS->private;
 
-        return ret;
+    GF_VALIDATE_OR_GOTO("glusterd", conf, out);
+    GF_VALIDATE_OR_GOTO("glusterd", svc, out);
+    pthread_mutex_lock(&conf->attach_lock);
+    {
+        svc_proc = svc->svc_proc;
+        if (!svc_proc)
+            goto unlock;
+        cds_list_for_each_entry(temp_svc, &svc_proc->svcs, mux_svc)
+        {
+            /* Get volinfo->shd from svc object */
+            shd = cds_list_entry(svc, glusterd_shdsvc_t, svc);
+            if (!shd) {
+                gf_msg("glusterd", GF_LOG_ERROR, 0, GD_MSG_SHD_OBJ_GET_FAIL,
+                       "Failed to get shd object "
+                       "from shd service");
+                goto unlock;
+            }
+
+            /* Get volinfo from shd */
+            volinfo = cds_list_entry(shd, glusterd_volinfo_t, shd);
+            if (!volinfo) {
+                gf_msg("glusterd", GF_LOG_ERROR, 0, GD_MSG_VOLINFO_GET_FAIL,
+                       "Failed to get volinfo from "
+                       "from shd");
+                goto unlock;
+            }
+            if (!glusterd_is_shd_compatible_volume(volinfo))
+                continue;
+            if (volinfo->status == GLUSTERD_STATUS_STARTED)
+                goto unlock;
+        }
+        comp = _gf_true;
+    }
+unlock:
+    pthread_mutex_unlock(&conf->attach_lock);
+out:
+    return comp;
 }
 
 int
-glusterd_shdsvc_manager (glusterd_svc_t *svc, void *data, int flags)
+glusterd_shdsvc_manager(glusterd_svc_t *svc, void *data, int flags)
 {
-        int                 ret     = 0;
-        glusterd_volinfo_t *volinfo = NULL;
-
-        volinfo = data;
-
-        /* If all the volumes are stopped or all shd compatible volumes
-         * are stopped then stop the service if:
-         * - volinfo is NULL or
-         * - volinfo is present and volume is shd compatible
-         * Otherwise create volfile and restart service if:
-         * - volinfo is NULL or
-         * - volinfo is present and volume is shd compatible
+    int ret = -1;
+    glusterd_volinfo_t *volinfo = NULL;
+    glusterd_conf_t *conf = NULL;
+    gf_boolean_t shd_restart = _gf_false;
+
+    conf = THIS->private;
+    GF_VALIDATE_OR_GOTO("glusterd", conf, out);
+    GF_VALIDATE_OR_GOTO("glusterd", svc, out);
+    volinfo = data;
+    GF_VALIDATE_OR_GOTO("glusterd", volinfo, out);
+
+    if (volinfo->is_snap_volume) {
+        /* healing of a snap volume is not supported yet*/
+        ret = 0;
+        goto out;
+    }
+
+    while (conf->restart_shd) {
+        synccond_wait(&conf->cond_restart_shd, &conf->big_lock);
+    }
+    conf->restart_shd = _gf_true;
+    shd_restart = _gf_true;
+
+    if (volinfo)
+        glusterd_volinfo_ref(volinfo);
+
+    if (!glusterd_is_shd_compatible_volume(volinfo)) {
+        ret = 0;
+        if (svc->inited) {
+            /* This means glusterd was running for this volume and now
+             * it was converted to a non-shd volume. So just stop the shd
+             */
+            ret = svc->stop(svc, SIGTERM);
+        }
+        goto out;
+    }
+    ret = glusterd_shdsvc_create_volfile(volinfo);
+    if (ret)
+        goto out;
+
+    ret = glusterd_shd_svc_mux_init(volinfo, svc);
+    if (ret) {
+        gf_msg(THIS->name, GF_LOG_ERROR, 0, GD_MSG_FAILED_INIT_SHDSVC,
+               "Failed to init shd service");
+        goto out;
+    }
+
+    /* If all the volumes are stopped or all shd compatible volumes
+     * are stopped then stop the service if:
+     * - volinfo is NULL or
+     * - volinfo is present and volume is shd compatible
+     * Otherwise create volfile and restart service if:
+     * - volinfo is NULL or
+     * - volinfo is present and volume is shd compatible
+     */
+    if (glusterd_svcs_shd_compatible_volumes_stopped(svc)) {
+        /* TODO
+         * Take a lock and detach all svc's to stop the process
+         * also reset the init flag
          */
-        if (glusterd_are_all_volumes_stopped () ||
-            glusterd_all_shd_compatible_volumes_stopped ()) {
-                if (!(volinfo &&
-                      !glusterd_is_shd_compatible_volume (volinfo))) {
-                        ret = svc->stop (svc, SIGTERM);
-                }
-        } else {
-                if (!(volinfo &&
-                      !glusterd_is_shd_compatible_volume (volinfo))) {
-                        ret = glusterd_shdsvc_create_volfile ();
-                        if (ret)
-                                goto out;
-
-                        ret = svc->stop (svc, SIGTERM);
-                        if (ret)
-                                goto out;
-
-                        ret = svc->start (svc, flags);
-                        if (ret)
-                                goto out;
-
-                        ret = glusterd_conn_connect (&(svc->conn));
-                        if (ret)
-                                goto out;
-                }
+        ret = svc->stop(svc, SIGTERM);
+    } else if (volinfo) {
+        if (volinfo->status != GLUSTERD_STATUS_STARTED) {
+            ret = svc->stop(svc, SIGTERM);
+            if (ret)
+                goto out;
         }
+        if (volinfo->status == GLUSTERD_STATUS_STARTED) {
+            ret = svc->start(svc, flags);
+            if (ret)
+                goto out;
+        }
+    }
 out:
-        gf_log (THIS->name, GF_LOG_DEBUG, "Returning %d", ret);
+    if (shd_restart) {
+        conf->restart_shd = _gf_false;
+        synccond_broadcast(&conf->cond_restart_shd);
+    }
+    if (volinfo)
+        glusterd_volinfo_unref(volinfo);
+    if (ret)
+        gf_event(EVENT_SVC_MANAGER_FAILED, "svc_name=%s", svc->name);
+    gf_msg_debug(THIS->name, 0, "Returning %d", ret);
 
-        return ret;
+    return ret;
 }
 
 int
-glusterd_shdsvc_start (glusterd_svc_t *svc, int flags)
+glusterd_new_shd_svc_start(glusterd_svc_t *svc, int flags)
 {
-        int              ret                            = -1;
-        char             glusterd_uuid_option[PATH_MAX] = {0};
-        dict_t          *cmdline                        = NULL;
+    int ret = -1;
+    char glusterd_uuid_option[PATH_MAX] = {0};
+    char client_pid[32] = {0};
+    dict_t *cmdline = NULL;
+    xlator_t *this = THIS;
+    GF_ASSERT(this);
 
-        cmdline = dict_new ();
-        if (!cmdline)
-                goto out;
+    cmdline = dict_new();
+    if (!cmdline)
+        goto out;
 
-        ret = snprintf (glusterd_uuid_option, sizeof (glusterd_uuid_option),
-                        "*replicate*.node-uuid=%s", uuid_utoa (MY_UUID));
-        if (ret < 0)
-                goto out;
+    ret = snprintf(glusterd_uuid_option, sizeof(glusterd_uuid_option),
+                   "*replicate*.node-uuid=%s", uuid_utoa(MY_UUID));
+    if (ret < 0)
+        goto out;
 
-        /* Pass cmdline arguments as key-value pair. The key is merely
-         * a carrier and is not used. Since dictionary follows LIFO the value
-         * should be put in reverse order*/
-        ret = dict_set_str (cmdline, "arg2", glusterd_uuid_option);
-        if (ret)
-                goto out;
+    ret = snprintf(client_pid, sizeof(client_pid), "--client-pid=%d",
+                   GF_CLIENT_PID_SELF_HEALD);
+    if (ret < 0)
+        goto out;
 
-        ret = dict_set_str (cmdline, "arg1", "--xlator-option");
-        if (ret)
-                goto out;
+    ret = dict_set_str(cmdline, "arg", client_pid);
+    if (ret < 0) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_SET_FAILED,
+                "Key=arg", NULL);
+        goto out;
+    }
+
+    /* Pass cmdline arguments as key-value pair. The key is merely
+     * a carrier and is not used. Since dictionary follows LIFO the value
+     * should be put in reverse order*/
+    ret = dict_set_str(cmdline, "arg4", svc->name);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_SET_FAILED,
+                "Key=arg4", NULL);
+        goto out;
+    }
+
+    ret = dict_set_str(cmdline, "arg3", GD_SHD_PROCESS_NAME);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_SET_FAILED,
+                "Key=arg3", NULL);
+        goto out;
+    }
+
+    ret = dict_set_str(cmdline, "arg2", glusterd_uuid_option);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_SET_FAILED,
+                "Key=arg2", NULL);
+        goto out;
+    }
 
-        ret = glusterd_svc_start (svc, flags, cmdline);
+    ret = dict_set_str(cmdline, "arg1", "--xlator-option");
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_SET_FAILED,
+                "Key=arg1", NULL);
+        goto out;
+    }
 
+    ret = glusterd_svc_start(svc, flags, cmdline);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno,
+                GD_MSG_GLUSTER_SERVICE_START_FAIL, NULL);
+        goto out;
+    }
+
+    ret = glusterd_conn_connect(&(svc->conn));
+out:
+    if (cmdline)
+        dict_unref(cmdline);
+    return ret;
+}
+
+int
+glusterd_recover_shd_attach_failure(glusterd_volinfo_t *volinfo,
+                                    glusterd_svc_t *svc, int flags)
+{
+    int ret = -1;
+    glusterd_svc_proc_t *mux_proc = NULL;
+    glusterd_conf_t *conf = NULL;
+
+    conf = THIS->private;
+
+    if (!conf || !volinfo || !svc)
+        return -1;
+    glusterd_shd_svcproc_cleanup(&volinfo->shd);
+    mux_proc = glusterd_svcprocess_new();
+    if (!mux_proc) {
+        return -1;
+    }
+    ret = glusterd_shdsvc_init(volinfo, NULL, mux_proc);
+    if (ret)
+        return -1;
+    pthread_mutex_lock(&conf->attach_lock);
+    {
+        cds_list_add_tail(&mux_proc->svc_proc_list, &conf->shd_procs);
+        svc->svc_proc = mux_proc;
+        cds_list_del_init(&svc->mux_svc);
+        cds_list_add_tail(&svc->mux_svc, &mux_proc->svcs);
+    }
+    pthread_mutex_unlock(&conf->attach_lock);
+
+    ret = glusterd_new_shd_svc_start(svc, flags);
+    if (!ret) {
+        volinfo->shd.attached = _gf_true;
+    }
+    return ret;
+}
+
+int
+glusterd_shdsvc_start(glusterd_svc_t *svc, int flags)
+{
+    int ret = -1;
+    glusterd_shdsvc_t *shd = NULL;
+    glusterd_volinfo_t *volinfo = NULL;
+    glusterd_conf_t *conf = NULL;
+
+    GF_VALIDATE_OR_GOTO("glusterd", svc, out);
+    conf = THIS->private;
+    GF_VALIDATE_OR_GOTO("glusterd", conf, out);
+
+    /* Get volinfo->shd from svc object */
+    shd = cds_list_entry(svc, glusterd_shdsvc_t, svc);
+    if (!shd) {
+        gf_msg("glusterd", GF_LOG_ERROR, 0, GD_MSG_SHD_OBJ_GET_FAIL,
+               "Failed to get shd object "
+               "from shd service");
+        return -1;
+    }
+
+    /* Get volinfo from shd */
+    volinfo = cds_list_entry(shd, glusterd_volinfo_t, shd);
+    if (!volinfo) {
+        gf_msg("glusterd", GF_LOG_ERROR, 0, GD_MSG_VOLINFO_GET_FAIL,
+               "Failed to get volinfo from "
+               "from shd");
+        return -1;
+    }
+
+    if (volinfo->status != GLUSTERD_STATUS_STARTED)
+        return -1;
+
+    glusterd_volinfo_ref(volinfo);
+
+    if (!svc->inited) {
+        ret = glusterd_shd_svc_mux_init(volinfo, svc);
+        if (ret)
+            goto out;
+    }
+
+    if (shd->attached) {
+        glusterd_volinfo_ref(volinfo);
+        /* Unref will happen from glusterd_svc_attach_cbk */
+        ret = glusterd_attach_svc(svc, volinfo, flags);
+        if (ret) {
+            gf_msg("glusterd", GF_LOG_ERROR, 0, GD_MSG_VOLINFO_GET_FAIL,
+                   "Failed to attach shd svc(volume=%s) to pid=%d",
+                   volinfo->volname, glusterd_proc_get_pid(&svc->proc));
+            glusterd_shd_svcproc_cleanup(&volinfo->shd);
+            glusterd_volinfo_unref(volinfo);
+            goto out1;
+        }
+        goto out;
+    }
+    ret = glusterd_new_shd_svc_start(svc, flags);
+    if (!ret) {
+        shd->attached = _gf_true;
+    }
 out:
-        if (cmdline)
-                dict_unref (cmdline);
+    if (ret && volinfo)
+        glusterd_shd_svcproc_cleanup(&volinfo->shd);
+    if (volinfo)
+        glusterd_volinfo_unref(volinfo);
+out1:
+    gf_msg_debug(THIS->name, 0, "Returning %d", ret);
+
+    return ret;
+}
+
+int
+glusterd_shdsvc_reconfigure(glusterd_volinfo_t *volinfo)
+{
+    int ret = -1;
+    xlator_t *this = NULL;
+    gf_boolean_t identical = _gf_false;
+    dict_t *mod_dict = NULL;
+    glusterd_svc_t *svc = NULL;
+
+    this = THIS;
+    GF_VALIDATE_OR_GOTO("glusterd", this, out);
+
+    if (!volinfo) {
+        /* reconfigure will be called separately*/
+        ret = 0;
+        goto out;
+    }
+
+    glusterd_volinfo_ref(volinfo);
+    svc = &(volinfo->shd.svc);
+    if (glusterd_svcs_shd_compatible_volumes_stopped(svc))
+        goto manager;
+
+    /*
+     * Check both OLD and NEW volfiles, if they are SAME by size
+     * and cksum i.e. "character-by-character". If YES, then
+     * NOTHING has been changed, just return.
+     */
 
-        gf_log (THIS->name, GF_LOG_DEBUG, "Returning %d", ret);
+    if (!glusterd_is_shd_compatible_volume(volinfo)) {
+        if (svc->inited)
+            goto manager;
 
-        return ret;
+        /* Nothing to do if not shd compatible */
+        ret = 0;
+        goto out;
+    }
+    mod_dict = dict_new();
+    if (!mod_dict) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_CREATE_FAIL, NULL);
+        goto out;
+    }
+
+    ret = dict_set_uint32(mod_dict, "cluster.background-self-heal-count", 0);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_SET_FAILED,
+                "Key=cluster.background-self-heal-count", NULL);
+        goto out;
+    }
+
+    ret = dict_set_str(mod_dict, "cluster.data-self-heal", "on");
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_SET_FAILED,
+                "Key=cluster.data-self-heal", NULL);
+        goto out;
+    }
+
+    ret = dict_set_str(mod_dict, "cluster.metadata-self-heal", "on");
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_SET_FAILED,
+                "Key=cluster.metadata-self-heal", NULL);
+        goto out;
+    }
+
+    ret = dict_set_int32(mod_dict, "graph-check", 1);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_SET_FAILED,
+                "Key=graph-check", NULL);
+        goto out;
+    }
+
+    ret = dict_set_str(mod_dict, "cluster.entry-self-heal", "on");
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_SET_FAILED,
+                "Key=cluster.entry-self-heal", NULL);
+        goto out;
+    }
+
+    ret = glusterd_volume_svc_check_volfile_identical(
+        "glustershd", mod_dict, volinfo, glusterd_shdsvc_generate_volfile,
+        &identical);
+    if (ret)
+        goto out;
+
+    if (identical) {
+        ret = 0;
+        goto out;
+    }
+
+    /*
+     * They are not identical. Find out if the topology is changed
+     * OR just the volume options. If just the options which got
+     * changed, then inform the xlator to reconfigure the options.
+     */
+    identical = _gf_false; /* RESET the FLAG */
+    ret = glusterd_volume_svc_check_topology_identical(
+        "glustershd", mod_dict, volinfo, glusterd_shdsvc_generate_volfile,
+        &identical);
+    if (ret)
+        goto out;
+
+    /* Topology is not changed, but just the options. But write the
+     * options to shd volfile, so that shd will be reconfigured.
+     */
+    if (identical) {
+        ret = glusterd_shdsvc_create_volfile(volinfo);
+        if (ret == 0) { /* Only if above PASSES */
+            ret = glusterd_fetchspec_notify(THIS);
+        }
+        goto out;
+    }
+manager:
+    /*
+     * shd volfile's topology has been changed. volfile needs
+     * to be RECONFIGURED to ACT on the changed volfile.
+     */
+    ret = svc->manager(svc, volinfo, PROC_START_NO_WAIT);
+
+out:
+    if (volinfo)
+        glusterd_volinfo_unref(volinfo);
+    if (mod_dict)
+        dict_unref(mod_dict);
+    gf_msg_debug(this ? this->name : "glusterd", 0, "Returning %d", ret);
+    return ret;
 }
 
+int
+glusterd_shdsvc_restart()
+{
+    glusterd_volinfo_t *volinfo = NULL;
+    glusterd_volinfo_t *tmp = NULL;
+    int ret = -1;
+    xlator_t *this = THIS;
+    glusterd_conf_t *conf = NULL;
+    glusterd_svc_t *svc = NULL;
+
+    GF_VALIDATE_OR_GOTO("glusterd", this, out);
+
+    conf = this->private;
+    GF_VALIDATE_OR_GOTO(this->name, conf, out);
+
+    pthread_mutex_lock(&conf->volume_lock);
+    cds_list_for_each_entry_safe(volinfo, tmp, &conf->volumes, vol_list)
+    {
+        glusterd_volinfo_ref(volinfo);
+        pthread_mutex_unlock(&conf->volume_lock);
+        /* Start per volume shd svc */
+        if (volinfo->status == GLUSTERD_STATUS_STARTED) {
+            svc = &(volinfo->shd.svc);
+            ret = svc->manager(svc, volinfo, PROC_START_NO_WAIT);
+            if (ret) {
+                gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_SHD_START_FAIL,
+                       "Couldn't start shd for "
+                       "vol: %s on restart",
+                       volinfo->volname);
+                gf_event(EVENT_SVC_MANAGER_FAILED, "volume=%s;svc_name=%s",
+                         volinfo->volname, svc->name);
+                glusterd_volinfo_unref(volinfo);
+                goto out;
+            }
+        }
+        glusterd_volinfo_unref(volinfo);
+        pthread_mutex_lock(&conf->volume_lock);
+    }
+    pthread_mutex_unlock(&conf->volume_lock);
+out:
+    return ret;
+}
 
 int
-glusterd_shdsvc_reconfigure ()
+glusterd_shdsvc_stop(glusterd_svc_t *svc, int sig)
 {
-        return glusterd_svc_reconfigure (glusterd_shdsvc_create_volfile);
+    int ret = -1;
+    glusterd_svc_proc_t *svc_proc = NULL;
+    glusterd_shdsvc_t *shd = NULL;
+    glusterd_volinfo_t *volinfo = NULL;
+    gf_boolean_t empty = _gf_false;
+    glusterd_conf_t *conf = NULL;
+    int pid = -1;
+
+    conf = THIS->private;
+    GF_VALIDATE_OR_GOTO("glusterd", conf, out);
+    GF_VALIDATE_OR_GOTO("glusterd", svc, out);
+    svc_proc = svc->svc_proc;
+    if (!svc_proc) {
+        /*
+         * This can happen when stop was called on a volume that is not shd
+         * compatible.
+         */
+        gf_msg_debug("glusterd", 0, "svc_proc is null, ie shd already stopped");
+        ret = 0;
+        goto out;
+    }
+
+    /* Get volinfo->shd from svc object */
+    shd = cds_list_entry(svc, glusterd_shdsvc_t, svc);
+    if (!shd) {
+        gf_msg("glusterd", GF_LOG_ERROR, 0, GD_MSG_SHD_OBJ_GET_FAIL,
+               "Failed to get shd object "
+               "from shd service");
+        return -1;
+    }
+
+    /* Get volinfo from shd */
+    volinfo = cds_list_entry(shd, glusterd_volinfo_t, shd);
+    if (!volinfo) {
+        gf_msg("glusterd", GF_LOG_ERROR, 0, GD_MSG_VOLINFO_GET_FAIL,
+               "Failed to get volinfo from "
+               "from shd");
+        return -1;
+    }
+
+    glusterd_volinfo_ref(volinfo);
+    pthread_mutex_lock(&conf->attach_lock);
+    {
+        if (!gf_is_service_running(svc->proc.pidfile, &pid)) {
+            gf_msg_debug(THIS->name, 0, "shd isn't running");
+        }
+        cds_list_del_init(&svc->mux_svc);
+        empty = cds_list_empty(&svc_proc->svcs);
+        if (empty) {
+            svc_proc->status = GF_SVC_STOPPING;
+            cds_list_del_init(&svc_proc->svc_proc_list);
+        }
+    }
+    pthread_mutex_unlock(&conf->attach_lock);
+    if (empty) {
+        /* Unref will happen when destroying the connection */
+        glusterd_volinfo_ref(volinfo);
+        svc_proc->data = volinfo;
+        ret = glusterd_svc_stop(svc, sig);
+        if (ret) {
+            glusterd_volinfo_unref(volinfo);
+            goto out;
+        }
+    }
+    if (!empty && pid != -1) {
+        ret = glusterd_detach_svc(svc, volinfo, sig);
+        if (ret)
+            gf_msg(THIS->name, GF_LOG_ERROR, 0, GD_MSG_SVC_STOP_FAIL,
+                   "shd service is failed to detach volume %s from pid %d",
+                   volinfo->volname, glusterd_proc_get_pid(&svc->proc));
+        else
+            gf_msg(THIS->name, GF_LOG_INFO, 0, GD_MSG_SVC_STOP_SUCCESS,
+                   "Shd service is detached for volume %s from pid %d",
+                   volinfo->volname, glusterd_proc_get_pid(&svc->proc));
+    }
+    svc->online = _gf_false;
+    (void)glusterd_unlink_file((char *)svc->proc.pidfile);
+    glusterd_shd_svcproc_cleanup(shd);
+    ret = 0;
+    glusterd_volinfo_unref(volinfo);
+out:
+    gf_msg_debug(THIS->name, 0, "Returning %d", ret);
+    return ret;
 }
diff --git a/xlators/mgmt/glusterd/src/glusterd-shd-svc.h b/xlators/mgmt/glusterd/src/glusterd-shd-svc.h
index 469ed5d0af2..55b409f4b69 100644
--- a/xlators/mgmt/glusterd/src/glusterd-shd-svc.h
+++ b/xlators/mgmt/glusterd/src/glusterd-shd-svc.h
@@ -11,22 +11,35 @@
 #ifndef _GLUSTERD_SHD_SVC_H_
 #define _GLUSTERD_SHD_SVC_H_
 
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
-
 #include "glusterd-svc-mgmt.h"
+#include "glusterd.h"
+
+typedef struct glusterd_shdsvc_ glusterd_shdsvc_t;
+struct glusterd_shdsvc_ {
+    glusterd_svc_t svc;
+    gf_boolean_t attached;
+};
+
+void
+glusterd_shdsvc_build(glusterd_svc_t *svc);
 
 int
-glusterd_shdsvc_init (glusterd_svc_t *svc);
+glusterd_shdsvc_init(void *data, glusterd_conn_t *mux_conn,
+                     glusterd_svc_proc_t *svc_proc);
 
 int
-glusterd_shdsvc_manager (glusterd_svc_t *svc, void *data, int flags);
+glusterd_shdsvc_manager(glusterd_svc_t *svc, void *data, int flags);
 
 int
-glusterd_shdsvc_start (glusterd_svc_t *svc, int flags);
+glusterd_shdsvc_start(glusterd_svc_t *svc, int flags);
 
 int
-glusterd_shdsvc_reconfigure ();
+glusterd_shdsvc_reconfigure();
+
+int
+glusterd_shdsvc_restart();
+
+int
+glusterd_shdsvc_stop(glusterd_svc_t *svc, int sig);
+
 #endif
diff --git a/xlators/mgmt/glusterd/src/glusterd-sm.c b/xlators/mgmt/glusterd/src/glusterd-sm.c
index f8228b0ab83..bf2d81b644a 100644
--- a/xlators/mgmt/glusterd/src/glusterd-sm.c
+++ b/xlators/mgmt/glusterd/src/glusterd-sm.c
@@ -8,704 +8,926 @@
    cases as published by the Free Software Foundation.
 */
 
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
 #include <time.h>
 #include <sys/uio.h>
 #include <sys/resource.h>
 
 #include <libgen.h>
-#include "compat-uuid.h"
+#include <glusterfs/compat-uuid.h>
 
 #include "fnmatch.h"
-#include "xlator.h"
+#include <glusterfs/xlator.h>
 #include "protocol-common.h"
 #include "glusterd.h"
-#include "call-stub.h"
-#include "defaults.h"
-#include "list.h"
+#include <glusterfs/call-stub.h>
+#include <glusterfs/defaults.h>
+#include <glusterfs/list.h>
 #include "glusterd-messages.h"
-#include "dict.h"
-#include "compat.h"
-#include "compat-errno.h"
-#include "statedump.h"
+#include <glusterfs/dict.h>
+#include <glusterfs/compat.h>
+#include <glusterfs/compat-errno.h>
+#include <glusterfs/statedump.h>
 #include "glusterd-sm.h"
 #include "glusterd-op-sm.h"
 #include "glusterd-utils.h"
 #include "glusterd-store.h"
+#include "glusterd-svc-helper.h"
 #include "glusterd-snapshot-utils.h"
 #include "glusterd-server-quorum.h"
+#include "glusterd-gfproxyd-svc-helper.h"
+
+char local_node_hostname[PATH_MAX] = {
+    0,
+};
 
 static struct cds_list_head gd_friend_sm_queue;
 
-static  char *glusterd_friend_sm_state_names[] = {
-        "Establishing Connection",
-        "Probe Sent to Peer",
-        "Probe Received from Peer",
-        "Peer in Cluster",
-        "Accepted peer request",
-        "Sent and Received peer request",
-        "Peer Rejected",
-        "Peer detach in progress",
-        "Probe Received from peer",
-        "Connected to Peer",
-        "Peer is connected and Accepted",
-        "Invalid State"
-};
+static char *glusterd_friend_sm_state_names[] = {
+    "Establishing Connection",
+    "Probe Sent to Peer",
+    "Probe Received from Peer",
+    "Peer in Cluster",
+    "Accepted peer request",
+    "Sent and Received peer request",
+    "Peer Rejected",
+    "Peer detach in progress",
+    "Probe Received from peer",
+    "Connected to Peer",
+    "Peer is connected and Accepted",
+    "Invalid State"};
 
 static char *glusterd_friend_sm_event_names[] = {
-        "GD_FRIEND_EVENT_NONE",
-        "GD_FRIEND_EVENT_PROBE",
-        "GD_FRIEND_EVENT_INIT_FRIEND_REQ",
-        "GD_FRIEND_EVENT_RCVD_ACC",
-        "GD_FRIEND_EVENT_LOCAL_ACC",
-        "GD_FRIEND_EVENT_RCVD_RJT",
-        "GD_FRIEND_EVENT_LOCAL_RJT",
-        "GD_FRIEND_EVENT_RCVD_FRIEND_REQ",
-        "GD_FRIEND_EVENT_INIT_REMOVE_FRIEND",
-        "GD_FRIEND_EVENT_RCVD_REMOVE_FRIEND",
-        "GD_FRIEND_EVENT_REMOVE_FRIEND",
-        "GD_FRIEND_EVENT_CONNECTED",
-        "GD_FRIEND_EVENT_MAX"
-};
-
-char*
-glusterd_friend_sm_state_name_get (int state)
+    "GD_FRIEND_EVENT_NONE",
+    "GD_FRIEND_EVENT_PROBE",
+    "GD_FRIEND_EVENT_INIT_FRIEND_REQ",
+    "GD_FRIEND_EVENT_RCVD_ACC",
+    "GD_FRIEND_EVENT_LOCAL_ACC",
+    "GD_FRIEND_EVENT_RCVD_RJT",
+    "GD_FRIEND_EVENT_LOCAL_RJT",
+    "GD_FRIEND_EVENT_RCVD_FRIEND_REQ",
+    "GD_FRIEND_EVENT_INIT_REMOVE_FRIEND",
+    "GD_FRIEND_EVENT_RCVD_REMOVE_FRIEND",
+    "GD_FRIEND_EVENT_REMOVE_FRIEND",
+    "GD_FRIEND_EVENT_CONNECTED",
+    "GD_FRIEND_EVENT_NEW_NAME",
+    "GD_FRIEND_EVENT_MAX"};
+
+char *
+glusterd_friend_sm_state_name_get(int state)
 {
-        if (state < 0 || state >= GD_FRIEND_STATE_MAX)
-                return glusterd_friend_sm_state_names[GD_FRIEND_STATE_MAX];
-        return glusterd_friend_sm_state_names[state];
+    if (state < 0 || state >= GD_FRIEND_STATE_MAX)
+        return glusterd_friend_sm_state_names[GD_FRIEND_STATE_MAX];
+    return glusterd_friend_sm_state_names[state];
 }
 
-char*
-glusterd_friend_sm_event_name_get (int event)
+char *
+glusterd_friend_sm_event_name_get(int event)
 {
-        if (event < 0 || event >= GD_FRIEND_EVENT_MAX)
-                return glusterd_friend_sm_event_names[GD_FRIEND_EVENT_MAX];
-        return glusterd_friend_sm_event_names[event];
+    if (event < 0 || event >= GD_FRIEND_EVENT_MAX)
+        return glusterd_friend_sm_event_names[GD_FRIEND_EVENT_MAX];
+    return glusterd_friend_sm_event_names[event];
 }
 
 void
-glusterd_destroy_probe_ctx (glusterd_probe_ctx_t *ctx)
+glusterd_destroy_probe_ctx(glusterd_probe_ctx_t *ctx)
 {
-        if (!ctx)
-                return;
+    if (!ctx)
+        return;
 
-        GF_FREE (ctx->hostname);
-        GF_FREE (ctx);
+    GF_FREE(ctx->hostname);
+    GF_FREE(ctx);
 }
 
 void
-glusterd_destroy_friend_req_ctx (glusterd_friend_req_ctx_t *ctx)
+glusterd_destroy_friend_req_ctx(glusterd_friend_req_ctx_t *ctx)
 {
-        if (!ctx)
-                return;
+    if (!ctx)
+        return;
 
-        if (ctx->vols)
-                dict_unref (ctx->vols);
-        GF_FREE (ctx->hostname);
-        GF_FREE (ctx);
+    if (ctx->vols)
+        dict_unref(ctx->vols);
+    GF_FREE(ctx->hostname);
+    GF_FREE(ctx);
 }
 
 void
-glusterd_destroy_friend_update_ctx (glusterd_friend_update_ctx_t *ctx)
+glusterd_destroy_friend_update_ctx(glusterd_friend_update_ctx_t *ctx)
 {
-        if (!ctx)
-                return;
-        GF_FREE (ctx->hostname);
-        GF_FREE (ctx);
+    if (!ctx)
+        return;
+    GF_FREE(ctx->hostname);
+    GF_FREE(ctx);
 }
 
 int
-glusterd_broadcast_friend_delete (char *hostname, uuid_t uuid)
+glusterd_broadcast_friend_delete(char *hostname, uuid_t uuid)
 {
-        int                             ret = 0;
-        rpc_clnt_procedure_t            *proc = NULL;
-        xlator_t                        *this = NULL;
-        glusterd_friend_update_ctx_t    ctx = {{0},};
-        glusterd_peerinfo_t             *peerinfo = NULL;
-        glusterd_conf_t                 *priv = NULL;
-        dict_t                          *friends = NULL;
-        char                            key[100] = {0,};
-        int32_t                         count = 0;
-
-        this = THIS;
-        priv = this->private;
-
-        GF_ASSERT (priv);
-
-        ctx.hostname = hostname;
-        ctx.op = GD_FRIEND_UPDATE_DEL;
-
-        friends = dict_new ();
-        if (!friends)
-                goto out;
-
-        snprintf (key, sizeof (key), "op");
-        ret = dict_set_int32 (friends, key, ctx.op);
-        if (ret)
-                goto out;
-
-        snprintf (key, sizeof (key), "hostname");
-        ret = dict_set_str (friends, key, hostname);
-        if (ret)
-                goto out;
-
-        ret = dict_set_int32 (friends, "count", count);
-        if (ret)
-                goto out;
-
-        rcu_read_lock ();
-        cds_list_for_each_entry_rcu (peerinfo, &priv->peers, uuid_list) {
-                if (!peerinfo->connected || !peerinfo->peer)
-                        continue;
-
-                /* Setting a direct reference to peerinfo in the dict is okay as
-                 * it is only going to be used within this read critical section
-                 * (in glusterd_rpc_friend_update)
-                 */
-                ret = dict_set_static_ptr (friends, "peerinfo", peerinfo);
-                if (ret) {
-                        gf_log ("", GF_LOG_ERROR, "failed to set peerinfo");
-                        goto unlock;
-                }
-
-                proc = &peerinfo->peer->proctable[GLUSTERD_FRIEND_UPDATE];
-                if (proc->fn) {
-                        ret = proc->fn (NULL, this, friends);
-                }
+    int ret = 0;
+    rpc_clnt_procedure_t *proc = NULL;
+    xlator_t *this = NULL;
+    glusterd_friend_update_ctx_t ctx = {
+        {0},
+    };
+    glusterd_peerinfo_t *peerinfo = NULL;
+    glusterd_conf_t *priv = NULL;
+    dict_t *friends = NULL;
+    char key[64] = {
+        0,
+    };
+    int keylen;
+    int32_t count = 0;
+
+    this = THIS;
+    priv = this->private;
+
+    GF_ASSERT(priv);
+
+    ctx.hostname = hostname;
+    ctx.op = GD_FRIEND_UPDATE_DEL;
+
+    friends = dict_new();
+    if (!friends) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_CREATE_FAIL, NULL);
+        goto out;
+    }
+
+    keylen = snprintf(key, sizeof(key), "op");
+    ret = dict_set_int32n(friends, key, keylen, ctx.op);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_SET_FAILED,
+                "Key=%s", key, NULL);
+        goto out;
+    }
+
+    keylen = snprintf(key, sizeof(key), "hostname");
+    ret = dict_set_strn(friends, key, keylen, hostname);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_SET_FAILED,
+                "Key=%s", key, NULL);
+        goto out;
+    }
+
+    ret = dict_set_int32n(friends, "count", SLEN("count"), count);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_SET_FAILED,
+                "Key=%s", key, NULL);
+        goto out;
+    }
+
+    RCU_READ_LOCK;
+    cds_list_for_each_entry_rcu(peerinfo, &priv->peers, uuid_list)
+    {
+        if (!peerinfo->connected || !peerinfo->peer)
+            continue;
+
+        /* Setting a direct reference to peerinfo in the dict is okay as
+         * it is only going to be used within this read critical section
+         * (in glusterd_rpc_friend_update)
+         */
+        ret = dict_set_static_ptr(friends, "peerinfo", peerinfo);
+        if (ret) {
+            RCU_READ_UNLOCK;
+            gf_msg("glusterd", GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+                   "failed to set peerinfo");
+            goto out;
         }
-unlock:
-        rcu_read_unlock ();
 
-        gf_log ("", GF_LOG_DEBUG, "Returning with %d", ret);
+        proc = &peerinfo->peer->proctable[GLUSTERD_FRIEND_UPDATE];
+        if (proc->fn) {
+            ret = proc->fn(NULL, this, friends);
+        }
+    }
 
+    RCU_READ_UNLOCK;
 out:
-        if (friends)
-                dict_unref (friends);
+    if (friends)
+        dict_unref(friends);
 
-        return ret;
+    gf_msg_debug("glusterd", 0, "Returning with %d", ret);
+    return ret;
 }
 
-
 static int
-glusterd_ac_none (glusterd_friend_sm_event_t *event, void *ctx)
+glusterd_ac_none(glusterd_friend_sm_event_t *event, void *ctx)
 {
-        int ret = 0;
+    int ret = 0;
 
-        gf_log ("", GF_LOG_DEBUG, "Returning with %d", ret);
+    gf_msg_debug("glusterd", 0, "Returning with %d", ret);
 
-        return ret;
+    return ret;
 }
 
 static int
-glusterd_ac_error (glusterd_friend_sm_event_t *event, void *ctx)
+glusterd_ac_error(glusterd_friend_sm_event_t *event, void *ctx)
 {
-        int ret = 0;
+    int ret = 0;
 
-        gf_log ("", GF_LOG_ERROR, "Received event %d ", event->event);
+    gf_msg("glusterd", GF_LOG_ERROR, 0, GD_MSG_AC_ERROR, "Received event %d ",
+           event->event);
 
-        return ret;
+    return ret;
 }
 
 static int
-glusterd_ac_reverse_probe_begin (glusterd_friend_sm_event_t *event, void *ctx)
+glusterd_ac_reverse_probe_begin(glusterd_friend_sm_event_t *event, void *ctx)
 {
-        int ret = 0;
-        glusterd_peerinfo_t             *peerinfo = NULL;
-        glusterd_friend_sm_event_t      *new_event = NULL;
-        glusterd_probe_ctx_t            *new_ev_ctx = NULL;
-
-        GF_ASSERT (event);
-        GF_ASSERT (ctx);
-
-        rcu_read_lock ();
-
-        peerinfo = glusterd_peerinfo_find (event->peerid, event->peername);
-        if (!peerinfo) {
-                gf_log (THIS->name, GF_LOG_ERROR, "Could not find peer %s(%s)",
-                        event->peername, uuid_utoa (event->peerid));
-                ret = -1;
-                goto out;
-        }
-
-        ret = glusterd_friend_sm_new_event
-                (GD_FRIEND_EVENT_PROBE, &new_event);
-
-        if (ret) {
-                gf_log ("glusterd", GF_LOG_ERROR, "Unable to get new new_event");
-                ret = -1;
-                goto out;
-        }
-
-        new_ev_ctx = GF_CALLOC (1, sizeof(*new_ev_ctx), gf_gld_mt_probe_ctx_t);
-
-        if (!new_ev_ctx) {
-                ret = -1;
-                goto out;
-        }
-
-        new_ev_ctx->hostname = gf_strdup (peerinfo->hostname);
-        new_ev_ctx->port = peerinfo->port;
-        new_ev_ctx->req = NULL;
-
-        new_event->peername = gf_strdup (peerinfo->hostname);
-        gf_uuid_copy (new_event->peerid, peerinfo->uuid);
-        new_event->ctx = new_ev_ctx;
-
-        ret = glusterd_friend_sm_inject_event (new_event);
-
-        if (ret) {
-                gf_log ("glusterd", GF_LOG_ERROR, "Unable to inject new_event %d, "
-                                "ret = %d", new_event->event, ret);
-        }
+    int ret = 0;
+    glusterd_peerinfo_t *peerinfo = NULL;
+    glusterd_friend_sm_event_t *new_event = NULL;
+    glusterd_probe_ctx_t *new_ev_ctx = NULL;
+
+    GF_ASSERT(event);
+    GF_ASSERT(ctx);
+
+    new_ev_ctx = GF_CALLOC(1, sizeof(*new_ev_ctx), gf_gld_mt_probe_ctx_t);
+
+    RCU_READ_LOCK;
+
+    peerinfo = glusterd_peerinfo_find(event->peerid, event->peername);
+    if (!peerinfo) {
+        RCU_READ_UNLOCK;
+        ret = -1;
+        gf_msg(THIS->name, GF_LOG_ERROR, 0, GD_MSG_PEER_NOT_FOUND,
+               "Could not find peer %s(%s)", event->peername,
+               uuid_utoa(event->peerid));
+        goto out;
+    }
+
+    ret = glusterd_friend_sm_new_event(GD_FRIEND_EVENT_PROBE, &new_event);
+
+    if (ret) {
+        RCU_READ_UNLOCK;
+        gf_msg("glusterd", GF_LOG_ERROR, 0, GD_MSG_EVENT_NEW_GET_FAIL,
+               "Unable to get new new_event");
+        ret = -1;
+        goto out;
+    }
+
+    if (!new_ev_ctx) {
+        RCU_READ_UNLOCK;
+        ret = -1;
+        goto out;
+    }
+
+    new_ev_ctx->hostname = gf_strdup(peerinfo->hostname);
+    new_ev_ctx->port = peerinfo->port;
+    new_ev_ctx->req = NULL;
+
+    new_event->peername = gf_strdup(peerinfo->hostname);
+    gf_uuid_copy(new_event->peerid, peerinfo->uuid);
+    new_event->ctx = new_ev_ctx;
+
+    ret = glusterd_friend_sm_inject_event(new_event);
+
+    RCU_READ_UNLOCK;
+
+    if (ret) {
+        gf_msg("glusterd", GF_LOG_ERROR, 0, GD_MSG_EVENT_INJECT_FAIL,
+               "Unable to inject new_event %d, "
+               "ret = %d",
+               new_event->event, ret);
+    }
 
 out:
-        rcu_read_unlock ();
-
-        if (ret) {
-                if (new_event)
-                        GF_FREE (new_event->peername);
-                GF_FREE (new_event);
-                if (new_ev_ctx)
-                        GF_FREE (new_ev_ctx->hostname);
-                GF_FREE (new_ev_ctx);
-        }
-        gf_log ("", GF_LOG_DEBUG, "returning with %d", ret);
-        return ret;
+    if (ret) {
+        if (new_event)
+            GF_FREE(new_event->peername);
+        GF_FREE(new_event);
+        if (new_ev_ctx)
+            GF_FREE(new_ev_ctx->hostname);
+        GF_FREE(new_ev_ctx);
+    }
+    gf_msg_debug("glusterd", 0, "returning with %d", ret);
+    return ret;
 }
 
 static int
-glusterd_ac_friend_add (glusterd_friend_sm_event_t *event, void *ctx)
+glusterd_ac_friend_add(glusterd_friend_sm_event_t *event, void *ctx)
 {
-        int                     ret = 0;
-        glusterd_peerinfo_t     *peerinfo = NULL;
-        rpc_clnt_procedure_t    *proc = NULL;
-        call_frame_t            *frame = NULL;
-        glusterd_conf_t         *conf = NULL;
-        xlator_t                *this = NULL;
-
-        GF_ASSERT (event);
-
-        this = THIS;
-        conf = this->private;
-
-        GF_ASSERT (conf);
-
-        rcu_read_lock ();
-
-        peerinfo = glusterd_peerinfo_find (event->peerid, event->peername);
-        if (!peerinfo) {
-                gf_log (this->name, GF_LOG_ERROR, "Could not find peer %s(%s)",
-                        event->peername, uuid_utoa (event->peerid));
-                goto out;
-        }
-
-        if (!peerinfo->peer)
-                goto out;
-        proc = &peerinfo->peer->proctable[GLUSTERD_FRIEND_ADD];
-        if (proc->fn) {
-                frame = create_frame (this, this->ctx->pool);
-                if (!frame) {
-                        goto out;
-                }
-                frame->local = ctx;
-                ret = proc->fn (frame, this, event);
+    int ret = 0;
+    glusterd_peerinfo_t *peerinfo = NULL;
+    rpc_clnt_procedure_t *proc = NULL;
+    call_frame_t *frame = NULL;
+    glusterd_conf_t *conf = NULL;
+    xlator_t *this = NULL;
+
+    GF_ASSERT(event);
+
+    this = THIS;
+    conf = this->private;
+
+    GF_ASSERT(conf);
+
+    RCU_READ_LOCK;
+
+    peerinfo = glusterd_peerinfo_find(event->peerid, event->peername);
+    if (!peerinfo) {
+        RCU_READ_UNLOCK;
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_PEER_NOT_FOUND,
+               "Could not find peer %s(%s)", event->peername,
+               uuid_utoa(event->peerid));
+        goto out;
+    }
+
+    if (!peerinfo->peer) {
+        RCU_READ_UNLOCK;
+        goto out;
+    }
+    proc = &peerinfo->peer->proctable[GLUSTERD_FRIEND_ADD];
+    if (proc->fn) {
+        frame = create_frame(this, this->ctx->pool);
+        if (!frame) {
+            RCU_READ_UNLOCK;
+            goto out;
         }
+        frame->local = ctx;
+        ret = proc->fn(frame, this, event);
+    }
 
+    RCU_READ_UNLOCK;
 out:
-        rcu_read_unlock ();
+    if (ret && frame)
+        STACK_DESTROY(frame->root);
 
-        gf_log ("", GF_LOG_DEBUG, "Returning with %d", ret);
-        return ret;
+    gf_msg_debug("glusterd", 0, "Returning with %d", ret);
+    return ret;
 }
 
 static int
-glusterd_ac_friend_probe (glusterd_friend_sm_event_t *event, void *ctx)
+glusterd_ac_friend_probe(glusterd_friend_sm_event_t *event, void *ctx)
 {
-        int                     ret = -1;
-        rpc_clnt_procedure_t    *proc = NULL;
-        call_frame_t            *frame = NULL;
-        glusterd_conf_t         *conf = NULL;
-        xlator_t                *this = NULL;
-        glusterd_probe_ctx_t    *probe_ctx = NULL;
-        glusterd_peerinfo_t     *peerinfo = NULL;
-        dict_t                  *dict = NULL;
-
-        GF_ASSERT (ctx);
-
-        probe_ctx = ctx;
-
-        this = THIS;
-
-        GF_ASSERT (this);
-
-        conf = this->private;
-
-        GF_ASSERT (conf);
-
-        rcu_read_lock ();
-        peerinfo = glusterd_peerinfo_find (NULL, probe_ctx->hostname);
-        if (peerinfo == NULL) {
-                //We should not reach this state ideally
-                ret = -1;
-                goto out;
+    int ret = -1;
+    rpc_clnt_procedure_t *proc = NULL;
+    call_frame_t *frame = NULL;
+    glusterd_conf_t *conf = NULL;
+    xlator_t *this = NULL;
+    glusterd_probe_ctx_t *probe_ctx = NULL;
+    glusterd_peerinfo_t *peerinfo = NULL;
+    dict_t *dict = NULL;
+
+    GF_ASSERT(ctx);
+
+    probe_ctx = ctx;
+
+    this = THIS;
+
+    GF_ASSERT(this);
+
+    conf = this->private;
+
+    GF_ASSERT(conf);
+
+    RCU_READ_LOCK;
+    peerinfo = glusterd_peerinfo_find(NULL, probe_ctx->hostname);
+    if (peerinfo == NULL) {
+        // We should not reach this state ideally
+        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_PEER_NOT_FOUND, NULL);
+        ret = -1;
+        goto unlock;
+    }
+
+    if (!peerinfo->peer) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_PEER_ADDRESS_GET_FAIL,
+                NULL);
+        goto unlock;
+    }
+    proc = &peerinfo->peer->proctable[GLUSTERD_PROBE_QUERY];
+    if (proc->fn) {
+        frame = create_frame(this, this->ctx->pool);
+        if (!frame) {
+            gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_FRAME_CREATE_FAIL,
+                    NULL);
+            goto unlock;
+        }
+        frame->local = ctx;
+        dict = dict_new();
+        if (!dict) {
+            gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_CREATE_FAIL,
+                    NULL);
+            goto unlock;
+        }
+        ret = dict_set_strn(dict, "hostname", SLEN("hostname"),
+                            probe_ctx->hostname);
+        if (ret) {
+            gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_SET_FAILED,
+                    "Key=hostname", NULL);
+            goto unlock;
         }
 
-        if (!peerinfo->peer)
-                goto out;
-        proc = &peerinfo->peer->proctable[GLUSTERD_PROBE_QUERY];
-        if (proc->fn) {
-                frame = create_frame (this, this->ctx->pool);
-                if (!frame) {
-                        goto out;
-                }
-                frame->local = ctx;
-                dict = dict_new ();
-                if (!dict)
-                        goto out;
-                ret = dict_set_str (dict, "hostname", probe_ctx->hostname);
-                if (ret)
-                        goto out;
-
-                ret = dict_set_int32 (dict, "port", probe_ctx->port);
-                if (ret)
-                        goto out;
-
-                /* The peerinfo reference being set here is going to be used
-                 * only within this critical section, in glusterd_rpc_probe
-                 * (ie. proc->fn).
-                 */
-                ret = dict_set_static_ptr (dict, "peerinfo", peerinfo);
-                if (ret) {
-                        gf_log ("", GF_LOG_ERROR, "failed to set peerinfo");
-                        goto out;
-                }
-
-                ret = proc->fn (frame, this, dict);
-                if (ret)
-                        goto out;
+        ret = dict_set_int32n(dict, "port", SLEN("port"), probe_ctx->port);
+        if (ret) {
+            gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_SET_FAILED,
+                    "Key=port", NULL);
+            goto unlock;
+        }
 
+        /* The peerinfo reference being set here is going to be used
+         * only within this critical section, in glusterd_rpc_probe
+         * (ie. proc->fn).
+         */
+        ret = dict_set_static_ptr(dict, "peerinfo", peerinfo);
+        if (ret) {
+            RCU_READ_UNLOCK;
+            gf_msg("glusterd", GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+                   "failed to set peerinfo");
+            goto out;
         }
 
+        ret = proc->fn(frame, this, dict);
+        if (ret)
+            goto unlock;
+    }
+unlock:
+    RCU_READ_UNLOCK;
 out:
-        rcu_read_unlock ();
 
-        if (dict)
-                dict_unref (dict);
-        gf_log ("", GF_LOG_DEBUG, "Returning with %d", ret);
+    if (dict)
+        dict_unref(dict);
+    gf_msg_debug("glusterd", 0, "Returning with %d", ret);
 
-        return ret;
+    if (ret && frame)
+        STACK_DESTROY(frame->root);
+
+    return ret;
 }
 
 static int
-glusterd_ac_send_friend_remove_req (glusterd_friend_sm_event_t *event,
-                                    void *data)
+glusterd_ac_send_friend_remove_req(glusterd_friend_sm_event_t *event,
+                                   void *data)
 {
-        int                     ret = 0;
-        glusterd_peerinfo_t     *peerinfo = NULL;
-        rpc_clnt_procedure_t    *proc = NULL;
-        call_frame_t            *frame = NULL;
-        glusterd_conf_t         *conf = NULL;
-        xlator_t                *this = NULL;
-        glusterd_friend_sm_event_type_t event_type = GD_FRIEND_EVENT_NONE;
-        glusterd_probe_ctx_t            *ctx = NULL;
-        glusterd_friend_sm_event_t      *new_event = NULL;
-
-        GF_ASSERT (event);
-
-        this = THIS;
-        conf = this->private;
-
-        GF_ASSERT (conf);
-
-        rcu_read_lock ();
-
-        peerinfo = glusterd_peerinfo_find (event->peerid, event->peername);
-        if (!peerinfo) {
-                gf_log (this->name, GF_LOG_ERROR, "Could not find peer %s(%s)",
-                        event->peername, uuid_utoa (event->peerid));
-                goto out;
+    int ret = 0;
+    glusterd_peerinfo_t *peerinfo = NULL;
+    rpc_clnt_procedure_t *proc = NULL;
+    call_frame_t *frame = NULL;
+    glusterd_conf_t *conf = NULL;
+    xlator_t *this = NULL;
+    glusterd_friend_sm_event_type_t event_type = GD_FRIEND_EVENT_NONE;
+    glusterd_probe_ctx_t *ctx = NULL;
+    glusterd_friend_sm_event_t *new_event = NULL;
+
+    GF_ASSERT(event);
+
+    this = THIS;
+    conf = this->private;
+
+    GF_ASSERT(conf);
+
+    RCU_READ_LOCK;
+
+    peerinfo = glusterd_peerinfo_find(event->peerid, event->peername);
+    if (!peerinfo) {
+        RCU_READ_UNLOCK;
+        ret = -1;
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_PEER_NOT_FOUND,
+               "Could not find peer %s(%s)", event->peername,
+               uuid_utoa(event->peerid));
+        goto out;
+    }
+    ctx = event->ctx;
+
+    if (!peerinfo->connected) {
+        event_type = GD_FRIEND_EVENT_REMOVE_FRIEND;
+
+        ret = glusterd_friend_sm_new_event(event_type, &new_event);
+
+        if (!ret) {
+            new_event->peername = peerinfo->hostname;
+            gf_uuid_copy(new_event->peerid, peerinfo->uuid);
+            ret = glusterd_friend_sm_inject_event(new_event);
+        } else {
+            gf_msg("glusterd", GF_LOG_ERROR, 0, GD_MSG_EVENT_NEW_GET_FAIL,
+                   "Unable to get event");
         }
-        ctx = event->ctx;
-
-        if (!peerinfo->connected) {
-                event_type = GD_FRIEND_EVENT_REMOVE_FRIEND;
-
-                ret = glusterd_friend_sm_new_event (event_type, &new_event);
 
-                if (!ret) {
-                        new_event->peername = peerinfo->hostname;
-                        gf_uuid_copy (new_event->peerid, peerinfo->uuid);
-                        ret = glusterd_friend_sm_inject_event (new_event);
-                } else {
-                        gf_log ("glusterd", GF_LOG_ERROR,
-                                 "Unable to get event");
-                }
-
-                if (ctx) {
-                        ret = glusterd_xfer_cli_deprobe_resp (ctx->req, ret, 0,
-                                                              NULL,
-                                                              ctx->hostname,
-                                                              ctx->dict);
-                        glusterd_broadcast_friend_delete (ctx->hostname, NULL);
-                        glusterd_destroy_probe_ctx (ctx);
-                }
-                goto out;
+        if (ctx) {
+            ret = glusterd_xfer_cli_deprobe_resp(ctx->req, ret, 0, NULL,
+                                                 ctx->hostname, ctx->dict);
+            glusterd_broadcast_friend_delete(ctx->hostname, NULL);
+            glusterd_destroy_probe_ctx(ctx);
         }
-
-        if (!peerinfo->peer)
-                goto out;
-        proc = &peerinfo->peer->proctable[GLUSTERD_FRIEND_REMOVE];
-        if (proc->fn) {
-                frame = create_frame (this, this->ctx->pool);
-                if (!frame) {
-                        goto out;
-                }
-                frame->local = data;
-                ret = proc->fn (frame, this, event);
+        goto unlock;
+    }
+
+    if (!peerinfo->peer) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_PEER_ADDRESS_GET_FAIL,
+                NULL);
+        goto unlock;
+    }
+    proc = &peerinfo->peer->proctable[GLUSTERD_FRIEND_REMOVE];
+    if (proc->fn) {
+        frame = create_frame(this, this->ctx->pool);
+        if (!frame) {
+            gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_FRAME_CREATE_FAIL,
+                    NULL);
+            goto unlock;
         }
+        frame->local = data;
+        ret = proc->fn(frame, this, event);
+    }
 
+unlock:
+    RCU_READ_UNLOCK;
 out:
-        rcu_read_unlock ();
 
-        gf_log ("", GF_LOG_DEBUG, "Returning with %d", ret);
+    gf_msg_debug("glusterd", 0, "Returning with %d", ret);
 
-        return ret;
+    if (ret && frame)
+        STACK_DESTROY(frame->root);
+
+    return ret;
 }
 
 static gf_boolean_t
-glusterd_should_update_peer (glusterd_peerinfo_t *peerinfo,
-                             glusterd_peerinfo_t *cur_peerinfo)
+glusterd_should_update_peer(glusterd_peerinfo_t *peerinfo,
+                            glusterd_peerinfo_t *cur_peerinfo)
 {
-        gf_boolean_t is_valid = _gf_false;
+    if ((peerinfo == cur_peerinfo) ||
+        (peerinfo->state.state == GD_FRIEND_STATE_BEFRIENDED))
+        return _gf_true;
 
-        if ((peerinfo == cur_peerinfo) ||
-            (peerinfo->state.state == GD_FRIEND_STATE_BEFRIENDED))
-                is_valid = _gf_true;
-
-        return is_valid;
+    return _gf_false;
 }
 
 static int
-glusterd_ac_send_friend_update (glusterd_friend_sm_event_t *event, void *ctx)
+glusterd_ac_send_friend_update(glusterd_friend_sm_event_t *event, void *ctx)
 {
-        int                           ret               = 0;
-        glusterd_peerinfo_t          *cur_peerinfo      = NULL;
-        glusterd_peerinfo_t          *peerinfo          = NULL;
-        rpc_clnt_procedure_t         *proc              = NULL;
-        xlator_t                     *this              = NULL;
-        glusterd_friend_update_ctx_t  ev_ctx            = {{0}};
-        glusterd_conf_t              *priv              = NULL;
-        dict_t                       *friends           = NULL;
-        char                          key[100]          = {0,};
-        int32_t                       count             = 0;
-
-        GF_ASSERT (event);
-
-        this = THIS;
-        priv = this->private;
-
-        GF_ASSERT (priv);
-
-        rcu_read_lock ();
-
-        cur_peerinfo = glusterd_peerinfo_find (event->peerid, event->peername);
-        if (!cur_peerinfo) {
-                gf_log (this->name, GF_LOG_ERROR, "Could not find peer %s(%s)",
-                        event->peername, uuid_utoa (event->peerid));
-                ret = -1;
-                goto out;
-        }
-
-        ev_ctx.op = GD_FRIEND_UPDATE_ADD;
-
-        friends = dict_new ();
-        if (!friends)
-                goto out;
-
-        snprintf (key, sizeof (key), "op");
-        ret = dict_set_int32 (friends, key, ev_ctx.op);
+    int ret = 0;
+    glusterd_peerinfo_t *cur_peerinfo = NULL;
+    glusterd_peerinfo_t *peerinfo = NULL;
+    rpc_clnt_procedure_t *proc = NULL;
+    xlator_t *this = NULL;
+    glusterd_friend_update_ctx_t ev_ctx = {{0}};
+    glusterd_conf_t *priv = NULL;
+    dict_t *friends = NULL;
+    char key[64] = {
+        0,
+    };
+    int keylen;
+    int32_t count = 0;
+
+    GF_ASSERT(event);
+
+    this = THIS;
+    priv = this->private;
+
+    GF_ASSERT(priv);
+
+    keylen = snprintf(key, sizeof(key), "op");
+    friends = dict_new();
+
+    RCU_READ_LOCK;
+
+    cur_peerinfo = glusterd_peerinfo_find(event->peerid, event->peername);
+    if (!cur_peerinfo) {
+        RCU_READ_UNLOCK;
+        ret = -1;
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_PEER_NOT_FOUND,
+               "Could not find peer %s(%s)", event->peername,
+               uuid_utoa(event->peerid));
+        goto out;
+    }
+
+    if (!friends) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_CREATE_FAIL, NULL);
+        goto unlock;
+    }
+
+    ev_ctx.op = GD_FRIEND_UPDATE_ADD;
+    ret = dict_set_int32n(friends, key, keylen, ev_ctx.op);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_SET_FAILED,
+                "Key=%s", key, NULL);
+        goto unlock;
+    }
+
+    cds_list_for_each_entry_rcu(peerinfo, &priv->peers, uuid_list)
+    {
+        if (!glusterd_should_update_peer(peerinfo, cur_peerinfo))
+            continue;
+
+        count++;
+
+        snprintf(key, sizeof(key), "friend%d", count);
+        ret = gd_add_friend_to_dict(peerinfo, friends, key);
         if (ret)
-                goto out;
-
-        cds_list_for_each_entry_rcu (peerinfo, &priv->peers, uuid_list) {
-                if (!glusterd_should_update_peer (peerinfo, cur_peerinfo))
-                        continue;
+            goto unlock;
+    }
 
-                count++;
+    ret = dict_set_int32n(friends, "count", SLEN("count"), count);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_SET_FAILED,
+                "Key=count", NULL);
+        goto unlock;
+    }
 
-                memset (key, 0, sizeof (key));
-                snprintf (key, sizeof (key), "friend%d", count);
-                ret = gd_add_friend_to_dict (peerinfo, friends, key);
-                if (ret)
-                        goto out;
-        }
+    cds_list_for_each_entry_rcu(peerinfo, &priv->peers, uuid_list)
+    {
+        if (!peerinfo->connected || !peerinfo->peer)
+            continue;
 
-        ret = dict_set_int32 (friends, "count", count);
-        if (ret)
-                goto out;
+        if (!glusterd_should_update_peer(peerinfo, cur_peerinfo))
+            continue;
 
-        cds_list_for_each_entry_rcu (peerinfo, &priv->peers, uuid_list) {
-                if (!peerinfo->connected || !peerinfo->peer)
-                        continue;
-
-                if (!glusterd_should_update_peer (peerinfo, cur_peerinfo))
-                        continue;
-
-                ret = dict_set_static_ptr (friends, "peerinfo", peerinfo);
-                if (ret) {
-                        gf_log ("", GF_LOG_ERROR, "failed to set peerinfo");
-                        goto out;
-                }
-
-                proc = &peerinfo->peer->proctable[GLUSTERD_FRIEND_UPDATE];
-                if (proc->fn) {
-                        ret = proc->fn (NULL, this, friends);
-                }
+        ret = dict_set_static_ptr(friends, "peerinfo", peerinfo);
+        if (ret) {
+            RCU_READ_UNLOCK;
+            gf_msg("glusterd", GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+                   "failed to set peerinfo");
+            goto out;
         }
 
-        gf_log ("", GF_LOG_DEBUG, "Returning with %d", ret);
+        proc = &peerinfo->peer->proctable[GLUSTERD_FRIEND_UPDATE];
+        if (proc->fn) {
+            ret = proc->fn(NULL, this, friends);
+        }
+    }
 
+unlock:
+    RCU_READ_UNLOCK;
 out:
-        rcu_read_unlock ();
 
-        if (friends)
-                dict_unref (friends);
+    if (friends)
+        dict_unref(friends);
 
-        return ret;
+    gf_msg_debug("glusterd", 0, "Returning with %d", ret);
+    return ret;
 }
 
-/* Clean up stale volumes on the peer being detached. The volumes which have
- * bricks on other peers are stale with respect to the detached peer.
+/* ac_update_friend only sends friend update to the friend that caused this
+ * event to happen
  */
 static int
-glusterd_peer_detach_cleanup (glusterd_conf_t *priv)
+glusterd_ac_update_friend(glusterd_friend_sm_event_t *event, void *ctx)
 {
-        int                     ret = -1;
-        glusterd_volinfo_t      *volinfo = NULL;
-        glusterd_volinfo_t      *tmp_volinfo = NULL;
-
-        GF_ASSERT (priv);
-
-        cds_list_for_each_entry_safe (volinfo, tmp_volinfo, &priv->volumes,
-                                      vol_list) {
-                /* The peer detach checks make sure that, at this point in the
-                 * detach process, there are only volumes contained completely
-                 * within or completely outside the detached peer.
-                 * The only stale volumes at this point are the ones
-                 * completely outside the peer and can be safely deleted.
-                 */
-                if (!glusterd_friend_contains_vol_bricks (volinfo,
-                                                          MY_UUID)) {
-                        gf_log (THIS->name, GF_LOG_INFO,
-                                "Deleting stale volume %s", volinfo->volname);
-                        ret = glusterd_delete_volume (volinfo);
-                        if (ret) {
-                                gf_log (THIS->name, GF_LOG_ERROR,
-                                        "Error deleting stale volume");
-                                goto out;
-                        }
-                }
-        }
+    int ret = 0;
+    glusterd_peerinfo_t *cur_peerinfo = NULL;
+    glusterd_peerinfo_t *peerinfo = NULL;
+    rpc_clnt_procedure_t *proc = NULL;
+    xlator_t *this = NULL;
+    glusterd_friend_update_ctx_t ev_ctx = {{0}};
+    glusterd_conf_t *priv = NULL;
+    dict_t *friends = NULL;
+    char key[64] = {
+        0,
+    };
+    int keylen;
+    int32_t count = 0;
+
+    GF_ASSERT(event);
+
+    this = THIS;
+    priv = this->private;
+
+    GF_ASSERT(priv);
+
+    friends = dict_new();
+    keylen = snprintf(key, sizeof(key), "op");
+
+    RCU_READ_LOCK;
+
+    cur_peerinfo = glusterd_peerinfo_find(event->peerid, event->peername);
+    if (!cur_peerinfo) {
+        RCU_READ_UNLOCK;
+        ret = -1;
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_PEER_NOT_FOUND,
+               "Could not find peer %s(%s)", event->peername,
+               uuid_utoa(event->peerid));
+        goto out;
+    }
+
+    /* Bail out early if peer is not connected.
+     * We cannot send requests to the peer until we have established our
+     * client connection to it.
+     */
+    if (!cur_peerinfo->connected || !cur_peerinfo->peer) {
         ret = 0;
-out:
-        gf_log (THIS->name, GF_LOG_DEBUG, "Returning %d", ret);
-        return ret;
-}
-
-static int
-glusterd_ac_handle_friend_remove_req (glusterd_friend_sm_event_t *event,
-                                      void *ctx)
-{
-        int                             ret = 0;
-        glusterd_peerinfo_t             *peerinfo = NULL;
-        glusterd_friend_req_ctx_t       *ev_ctx = NULL;
-        glusterd_friend_sm_event_t      *new_event = NULL;
-        glusterd_conf_t                 *priv = NULL;
-
-        GF_ASSERT (ctx);
-        ev_ctx = ctx;
+        goto unlock;
+    }
+
+    if (!friends) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_CREATE_FAIL, NULL);
+        goto out;
+    }
+
+    ev_ctx.op = GD_FRIEND_UPDATE_ADD;
+    ret = dict_set_int32n(friends, key, keylen, ev_ctx.op);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_SET_FAILED,
+                "Key=%s", key, NULL);
+        goto unlock;
+    }
+
+    cds_list_for_each_entry_rcu(peerinfo, &priv->peers, uuid_list)
+    {
+        if (!glusterd_should_update_peer(peerinfo, cur_peerinfo))
+            continue;
+
+        count++;
+
+        snprintf(key, sizeof(key), "friend%d", count);
+        ret = gd_add_friend_to_dict(peerinfo, friends, key);
+        if (ret)
+            goto unlock;
+    }
+
+    ret = dict_set_int32n(friends, "count", SLEN("count"), count);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_SET_FAILED,
+                "Key=count", NULL);
+        goto unlock;
+    }
+
+    ret = dict_set_static_ptr(friends, "peerinfo", cur_peerinfo);
+    if (ret) {
+        RCU_READ_UNLOCK;
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+               "failed to set peerinfo");
+        goto out;
+    }
+
+    proc = &cur_peerinfo->peer->proctable[GLUSTERD_FRIEND_UPDATE];
+    if (proc->fn)
+        ret = proc->fn(NULL, this, friends);
+
+    gf_msg_debug(this->name, 0, "Returning with %d", ret);
 
-        priv = THIS->private;
-        GF_ASSERT (priv);
+unlock:
+    RCU_READ_UNLOCK;
+out:
 
-        ret = glusterd_xfer_friend_remove_resp (ev_ctx->req, ev_ctx->hostname,
-                                                ev_ctx->port);
+    if (friends)
+        dict_unref(friends);
 
-        rcu_read_lock ();
-        cds_list_for_each_entry_rcu (peerinfo, &priv->peers, uuid_list) {
+    return ret;
+}
 
-                ret = glusterd_friend_sm_new_event (GD_FRIEND_EVENT_REMOVE_FRIEND,
-                                                    &new_event);
+/* Clean up stale volumes on the peer being detached. The volumes which have
+ * bricks on other peers are stale with respect to the detached peer.
+ */
+static void
+glusterd_peer_detach_cleanup(glusterd_conf_t *priv)
+{
+    int ret = -1;
+    glusterd_volinfo_t *volinfo = NULL;
+    glusterd_volinfo_t *tmp_volinfo = NULL;
+    glusterd_svc_t *svc = NULL;
+
+    GF_ASSERT(priv);
+
+    cds_list_for_each_entry_safe(volinfo, tmp_volinfo, &priv->volumes, vol_list)
+    {
+        /* The peer detach checks make sure that, at this point in the
+         * detach process, there are only volumes contained completely
+         * within or completely outside the detached peer.
+         * The only stale volumes at this point are the ones
+         * completely outside the peer and can be safely deleted.
+         */
+        if (!glusterd_friend_contains_vol_bricks(volinfo, MY_UUID)) {
+            gf_msg(THIS->name, GF_LOG_INFO, 0, GD_MSG_STALE_VOL_DELETE_INFO,
+                   "Deleting stale volume %s", volinfo->volname);
+
+            /*Stop snapd daemon service if snapd daemon is running*/
+            if (!volinfo->is_snap_volume) {
+                svc = &(volinfo->snapd.svc);
+                ret = svc->stop(svc, SIGTERM);
                 if (ret) {
-                        rcu_read_unlock ();
-                        goto out;
+                    gf_msg(THIS->name, GF_LOG_ERROR, 0, GD_MSG_SVC_STOP_FAIL,
+                           "Failed "
+                           "to stop snapd daemon service");
                 }
+            }
 
-                new_event->peername = gf_strdup (peerinfo->hostname);
-                gf_uuid_copy (new_event->peerid, peerinfo->uuid);
+            if (glusterd_is_shd_compatible_volume(volinfo)) {
+                svc = &(volinfo->shd.svc);
+                ret = svc->stop(svc, SIGTERM);
+                if (ret) {
+                    gf_msg(THIS->name, GF_LOG_ERROR, 0, GD_MSG_SVC_STOP_FAIL,
+                           "Failed "
+                           "to stop shd daemon service");
+                }
+            }
 
-                ret = glusterd_friend_sm_inject_event (new_event);
+            if (glusterd_is_gfproxyd_enabled(volinfo)) {
+                svc = &(volinfo->gfproxyd.svc);
+                ret = svc->stop(svc, SIGTERM);
                 if (ret) {
-                        rcu_read_unlock ();
-                        goto out;
+                    gf_msg(THIS->name, GF_LOG_ERROR, 0, GD_MSG_SVC_STOP_FAIL,
+                           "Failed "
+                           "to stop gfproxyd daemon service");
                 }
+            }
+
+            ret = glusterd_cleanup_snaps_for_volume(volinfo);
+            if (ret) {
+                gf_msg(THIS->name, GF_LOG_ERROR, 0, GD_MSG_VOL_DELETE_FAIL,
+                       "Error deleting snapshots for volume %s",
+                       volinfo->volname);
+            }
+
+            ret = glusterd_delete_volume(volinfo);
+            if (ret) {
+                gf_msg(THIS->name, GF_LOG_ERROR, 0,
+                       GD_MSG_STALE_VOL_REMOVE_FAIL,
+                       "Error deleting stale volume");
+            }
+        }
+    }
+
+    /*Reconfigure all daemon services upon peer detach*/
+    ret = glusterd_svcs_reconfigure(NULL);
+    if (ret) {
+        gf_msg(THIS->name, GF_LOG_ERROR, 0, GD_MSG_SVC_STOP_FAIL,
+               "Failed to reconfigure all daemon services.");
+    }
+}
 
-                new_event = NULL;
+static int
+glusterd_ac_handle_friend_remove_req(glusterd_friend_sm_event_t *event,
+                                     void *ctx)
+{
+    int ret = 0;
+    glusterd_peerinfo_t *peerinfo = NULL;
+    glusterd_friend_req_ctx_t *ev_ctx = NULL;
+    glusterd_friend_sm_event_t *new_event = NULL;
+    glusterd_conf_t *priv = NULL;
+
+    GF_ASSERT(ctx);
+    ev_ctx = ctx;
+
+    priv = THIS->private;
+    GF_ASSERT(priv);
+
+    ret = glusterd_xfer_friend_remove_resp(ev_ctx->req, ev_ctx->hostname,
+                                           ev_ctx->port);
+
+    RCU_READ_LOCK;
+    cds_list_for_each_entry_rcu(peerinfo, &priv->peers, uuid_list)
+    {
+        ret = glusterd_friend_sm_new_event(GD_FRIEND_EVENT_REMOVE_FRIEND,
+                                           &new_event);
+        if (ret) {
+            RCU_READ_UNLOCK;
+            goto out;
         }
-        rcu_read_unlock ();
 
-        ret = glusterd_peer_detach_cleanup (priv);
+        new_event->peername = gf_strdup(peerinfo->hostname);
+        gf_uuid_copy(new_event->peerid, peerinfo->uuid);
+
+        ret = glusterd_friend_sm_inject_event(new_event);
         if (ret) {
-                gf_log (THIS->name, GF_LOG_WARNING,
-                        "Peer detach cleanup was not successful");
-                ret = 0;
+            RCU_READ_UNLOCK;
+            goto out;
         }
+
+        new_event = NULL;
+    }
+    RCU_READ_UNLOCK;
+
+    glusterd_peer_detach_cleanup(priv);
 out:
-        if (new_event)
-                GF_FREE (new_event->peername);
-        GF_FREE (new_event);
+    if (new_event)
+        GF_FREE(new_event->peername);
+    GF_FREE(new_event);
 
-        gf_log (THIS->name, GF_LOG_DEBUG, "Returning with %d", ret);
-        return ret;
+    gf_msg_debug(THIS->name, 0, "Returning with %d", ret);
+    return ret;
 }
 
 static int
-glusterd_ac_friend_remove (glusterd_friend_sm_event_t *event, void *ctx)
+glusterd_ac_friend_remove(glusterd_friend_sm_event_t *event, void *ctx)
 {
-        int                  ret = -1;
-        glusterd_peerinfo_t *peerinfo = NULL;
-
-        GF_ASSERT (event);
-
-        rcu_read_lock ();
-
-        peerinfo = glusterd_peerinfo_find (event->peerid, event->peername);
-        if (!peerinfo) {
-                gf_log (THIS->name, GF_LOG_ERROR, "Could not find peer %s(%s)",
-                        event->peername, uuid_utoa (event->peerid));
-                rcu_read_unlock ();
-                goto out;
-        }
-        ret = glusterd_friend_remove_cleanup_vols (peerinfo->uuid);
-        if (ret)
-                gf_msg (THIS->name, GF_LOG_WARNING, 0, GD_MSG_VOL_CLEANUP_FAIL,
-                        "Volumes cleanup failed");
-
-        rcu_read_unlock ();
-        /* Exiting read critical section as glusterd_peerinfo_cleanup calls
-         * synchronize_rcu before freeing the peerinfo
-         */
-
-        ret = glusterd_peerinfo_cleanup (peerinfo);
-        if (ret) {
-                gf_log (THIS->name, GF_LOG_ERROR, "Cleanup returned: %d", ret);
-        }
+    int ret = -1;
+    glusterd_peerinfo_t *peerinfo = NULL;
+
+    GF_ASSERT(event);
+
+    RCU_READ_LOCK;
+
+    peerinfo = glusterd_peerinfo_find(event->peerid, event->peername);
+    if (!peerinfo) {
+        RCU_READ_UNLOCK;
+        gf_msg(THIS->name, GF_LOG_ERROR, 0, GD_MSG_PEER_NOT_FOUND,
+               "Could not find peer %s(%s)", event->peername,
+               uuid_utoa(event->peerid));
+        goto out;
+    }
+    ret = glusterd_friend_remove_cleanup_vols(peerinfo->uuid);
+    RCU_READ_UNLOCK;
+    if (ret)
+        gf_msg(THIS->name, GF_LOG_WARNING, 0, GD_MSG_VOL_CLEANUP_FAIL,
+               "Volumes cleanup failed");
+
+    /* Exiting read critical section as glusterd_peerinfo_cleanup calls
+     * synchronize_rcu before freeing the peerinfo
+     */
+
+    ret = glusterd_peerinfo_cleanup(peerinfo);
+    if (ret) {
+        gf_msg(THIS->name, GF_LOG_ERROR, 0, GD_MSG_PEER_DETACH_CLEANUP_FAIL,
+               "Cleanup returned: %d", ret);
+    }
 out:
-        return 0;
+    return 0;
 }
 
 /*static int
@@ -719,606 +941,682 @@ glusterd_ac_none (void *ctx)
 }*/
 
 static int
-glusterd_ac_handle_friend_add_req (glusterd_friend_sm_event_t *event, void *ctx)
+glusterd_ac_handle_friend_add_req(glusterd_friend_sm_event_t *event, void *ctx)
 {
-        int                             ret = 0;
-        uuid_t                          uuid;
-        glusterd_peerinfo_t             *peerinfo = NULL;
-        glusterd_friend_req_ctx_t       *ev_ctx = NULL;
-        glusterd_friend_update_ctx_t    *new_ev_ctx = NULL;
-        glusterd_friend_sm_event_t      *new_event = NULL;
-        glusterd_friend_sm_event_type_t event_type = GD_FRIEND_EVENT_NONE;
-        glusterd_conf_t                *conf       = NULL;
-        int                             status = 0;
-        int32_t                         op_ret = -1;
-        int32_t                         op_errno = 0;
-        xlator_t                       *this       = NULL;
-
-        this = THIS;
-        GF_ASSERT (this);
-
-        GF_ASSERT (ctx);
-        ev_ctx = ctx;
-        gf_uuid_copy (uuid, ev_ctx->uuid);
-
-        rcu_read_lock ();
-        peerinfo = glusterd_peerinfo_find (event->peerid, event->peername);
-        if (!peerinfo) {
-                gf_log (this->name, GF_LOG_ERROR, "Could not find peer %s(%s)",
-                        event->peername, uuid_utoa (event->peerid));
-                ret = -1;
-                rcu_read_unlock ();
-                goto out;
+    int ret = 0;
+    uuid_t uuid;
+    glusterd_peerinfo_t *peerinfo = NULL;
+    glusterd_friend_req_ctx_t *ev_ctx = NULL;
+    glusterd_friend_update_ctx_t *new_ev_ctx = NULL;
+    glusterd_friend_sm_event_t *new_event = NULL;
+    glusterd_friend_sm_event_type_t event_type = GD_FRIEND_EVENT_NONE;
+    glusterd_conf_t *conf = NULL;
+    int status = 0;
+    int32_t op_ret = -1;
+    int32_t op_errno = 0;
+    xlator_t *this = NULL;
+    char *hostname = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+
+    GF_ASSERT(ctx);
+    ev_ctx = ctx;
+    gf_uuid_copy(uuid, ev_ctx->uuid);
+
+    RCU_READ_LOCK;
+    peerinfo = glusterd_peerinfo_find(event->peerid, event->peername);
+    if (!peerinfo) {
+        RCU_READ_UNLOCK;
+        ret = -1;
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_PEER_NOT_FOUND,
+               "Could not find peer %s(%s)", event->peername,
+               uuid_utoa(event->peerid));
+        goto out;
+    }
+
+    /* TODO: How do you do an atomic copy of uuid_t */
+    /* TODO: Updating within a read-critical section is also invalid
+     *       Update properly with updater synchronization
+     */
+    gf_uuid_copy(peerinfo->uuid, ev_ctx->uuid);
+
+    RCU_READ_UNLOCK;
+
+    conf = this->private;
+    GF_ASSERT(conf);
+
+    /* Passing the peername from the event. glusterd_compare_friend_data
+     * updates volumes and will use synchronize_rcu. If we were to pass
+     * peerinfo->hostname, we would have to do it under a read critical
+     * section which would lead to a deadlock
+     */
+
+    // Build comparison logic here.
+    pthread_mutex_lock(&conf->import_volumes);
+    {
+        ret = glusterd_compare_friend_data(ev_ctx->vols, &status,
+                                           event->peername);
+        if (ret) {
+            pthread_mutex_unlock(&conf->import_volumes);
+            goto out;
         }
 
-        /* TODO: How do you do an atomic copy of uuid_t */
-        /* TODO: Updating within a read-critical section is also invalid
-         *       Update properly with updater synchronization
-         */
-        gf_uuid_copy (peerinfo->uuid, ev_ctx->uuid);
-
-        rcu_read_unlock ();
-
-        conf = this->private;
-        GF_ASSERT (conf);
-
-        /* Passing the peername from the event. glusterd_compare_friend_data
-         * updates volumes and will use synchronize_rcu. If we were to pass
-         * peerinfo->hostname, we would have to do it under a read critical
-         * section which would lead to a deadlock
-         */
-
-        //Build comparison logic here.
-        ret = glusterd_compare_friend_data (ev_ctx->vols, &status,
-                                            event->peername);
-        if (ret)
-                goto out;
-
         if (GLUSTERD_VOL_COMP_RJT != status) {
-                event_type = GD_FRIEND_EVENT_LOCAL_ACC;
-                op_ret = 0;
+            event_type = GD_FRIEND_EVENT_LOCAL_ACC;
+            op_ret = 0;
         } else {
-                event_type = GD_FRIEND_EVENT_LOCAL_RJT;
-                op_errno = GF_PROBE_VOLUME_CONFLICT;
-                op_ret = -1;
+            event_type = GD_FRIEND_EVENT_LOCAL_RJT;
+            op_errno = GF_PROBE_VOLUME_CONFLICT;
+            op_ret = -1;
         }
 
         /* Compare missed_snapshot list with the peer *
          * if volume comparison is successful */
-        if ((op_ret == 0) &&
-            (conf->op_version >= GD_OP_VERSION_3_6_0)) {
-                ret = glusterd_import_friend_missed_snap_list (ev_ctx->vols);
-                if (ret) {
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "Failed to import peer's "
-                                "missed_snaps_list.");
-                        event_type = GD_FRIEND_EVENT_LOCAL_RJT;
-                        op_errno = GF_PROBE_MISSED_SNAP_CONFLICT;
-                        op_ret = -1;
-                }
-
-                /* glusterd_compare_friend_snapshots and functions only require
-                 * a peers hostname and uuid. It also does updates, which
-                 * require use of synchronize_rcu. So we pass the hostname and
-                 * id from the event instead of the peerinfo object to prevent
-                 * deadlocks as above.
-                 */
-                ret = glusterd_compare_friend_snapshots (ev_ctx->vols,
-                                                         event->peername,
-                                                         event->peerid);
-                if (ret) {
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "Conflict in comparing peer's snapshots");
-                        event_type = GD_FRIEND_EVENT_LOCAL_RJT;
-                        op_errno = GF_PROBE_SNAP_CONFLICT;
-                        op_ret = -1;
-                }
-        }
-
-        ret = glusterd_friend_sm_new_event (event_type, &new_event);
-
-        if (ret) {
-                gf_log ("", GF_LOG_ERROR, "Out of Memory");
-        }
-
-        new_event->peername = gf_strdup (event->peername);
-        gf_uuid_copy (new_event->peerid, event->peerid);
-
-        new_ev_ctx = GF_CALLOC (1, sizeof (*new_ev_ctx),
-                                gf_gld_mt_friend_update_ctx_t);
-        if (!new_ev_ctx) {
-                ret = -1;
-                goto out;
+        if ((op_ret == 0) && (conf->op_version >= GD_OP_VERSION_3_6_0)) {
+            ret = glusterd_import_friend_missed_snap_list(ev_ctx->vols);
+            if (ret) {
+                gf_msg(this->name, GF_LOG_ERROR, 0,
+                       GD_MSG_MISSED_SNAP_LIST_STORE_FAIL,
+                       "Failed to import peer's "
+                       "missed_snaps_list.");
+                event_type = GD_FRIEND_EVENT_LOCAL_RJT;
+                op_errno = GF_PROBE_MISSED_SNAP_CONFLICT;
+                op_ret = -1;
+            }
+
+            /* glusterd_compare_friend_snapshots and functions only require
+             * a peers hostname and uuid. It also does updates, which
+             * require use of synchronize_rcu. So we pass the hostname and
+             * id from the event instead of the peerinfo object to prevent
+             * deadlocks as above.
+             */
+            ret = glusterd_compare_friend_snapshots(
+                ev_ctx->vols, event->peername, event->peerid);
+            if (ret) {
+                gf_msg(this->name, GF_LOG_ERROR, 0,
+                       GD_MSG_SNAP_COMPARE_CONFLICT,
+                       "Conflict in comparing peer's snapshots");
+                event_type = GD_FRIEND_EVENT_LOCAL_RJT;
+                op_errno = GF_PROBE_SNAP_CONFLICT;
+                op_ret = -1;
+            }
         }
-
-        gf_uuid_copy (new_ev_ctx->uuid, ev_ctx->uuid);
-        new_ev_ctx->hostname = gf_strdup (ev_ctx->hostname);
-        new_ev_ctx->op = GD_FRIEND_UPDATE_ADD;
-
-        new_event->ctx = new_ev_ctx;
-
-        glusterd_friend_sm_inject_event (new_event);
-        new_event = NULL;
-
-        ret = glusterd_xfer_friend_add_resp (ev_ctx->req, ev_ctx->hostname,
-                                             event->peername, ev_ctx->port,
-                                             op_ret, op_errno);
+    }
+    pthread_mutex_unlock(&conf->import_volumes);
+    ret = glusterd_friend_sm_new_event(event_type, &new_event);
+
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, ENOMEM, GD_MSG_NO_MEMORY,
+               "Out of Memory");
+        goto out;
+    }
+
+    new_event->peername = gf_strdup(event->peername);
+    gf_uuid_copy(new_event->peerid, event->peerid);
+
+    new_ev_ctx = GF_CALLOC(1, sizeof(*new_ev_ctx),
+                           gf_gld_mt_friend_update_ctx_t);
+    if (!new_ev_ctx) {
+        ret = -1;
+        goto out;
+    }
+
+    gf_uuid_copy(new_ev_ctx->uuid, ev_ctx->uuid);
+    new_ev_ctx->hostname = gf_strdup(ev_ctx->hostname);
+    new_ev_ctx->op = GD_FRIEND_UPDATE_ADD;
+
+    new_event->ctx = new_ev_ctx;
+
+    ret = dict_get_strn(ev_ctx->vols, "hostname_in_cluster",
+                        SLEN("hostname_in_cluster"), &hostname);
+    if (ret || !hostname) {
+        gf_msg_debug(this->name, 0, "Unable to fetch local hostname from peer");
+    } else if (snprintf(local_node_hostname, sizeof(local_node_hostname), "%s",
+                        hostname) >= sizeof(local_node_hostname)) {
+        gf_msg_debug(this->name, 0, "local_node_hostname truncated");
+        ret = -1;
+        goto out;
+    }
+
+    glusterd_friend_sm_inject_event(new_event);
+    new_event = NULL;
+
+    ret = glusterd_xfer_friend_add_resp(ev_ctx->req, ev_ctx->hostname,
+                                        event->peername, ev_ctx->port, op_ret,
+                                        op_errno);
 
 out:
-        if (new_event)
-                GF_FREE (new_event->peername);
-        GF_FREE (new_event);
+    if (new_event)
+        GF_FREE(new_event->peername);
+    GF_FREE(new_event);
 
-        gf_log ("", GF_LOG_DEBUG, "Returning with %d", ret);
-        return ret;
+    gf_msg_debug("glusterd", 0, "Returning with %d", ret);
+    return ret;
 }
 
 static int
-glusterd_friend_sm_transition_state (uuid_t peerid, char *peername,
-                                     glusterd_sm_t *state,
-                                     glusterd_friend_sm_event_type_t event_type)
+glusterd_friend_sm_transition_state(uuid_t peerid, char *peername,
+                                    glusterd_sm_t *state,
+                                    glusterd_friend_sm_event_type_t event_type)
 {
-        int ret = -1;
-        glusterd_peerinfo_t *peerinfo = NULL;
+    int ret = -1;
+    glusterd_peerinfo_t *peerinfo = NULL;
 
-        GF_ASSERT (state);
-        GF_ASSERT (peername);
+    GF_ASSERT(state);
+    GF_ASSERT(peername);
 
-        rcu_read_lock ();
-        peerinfo = glusterd_peerinfo_find (peerid, peername);
-        if (!peerinfo) {
-                goto out;
-        }
+    RCU_READ_LOCK;
+    peerinfo = glusterd_peerinfo_find(peerid, peername);
+    if (!peerinfo) {
+        gf_smsg(THIS->name, GF_LOG_ERROR, errno, GD_MSG_PEER_NOT_FOUND, NULL);
+        goto out;
+    }
 
-        (void) glusterd_sm_tr_log_transition_add (&peerinfo->sm_log,
-                                           peerinfo->state.state,
-                                           state[event_type].next_state,
-                                           event_type);
+    (void)glusterd_sm_tr_log_transition_add(
+        &peerinfo->sm_log, peerinfo->state.state, state[event_type].next_state,
+        event_type);
 
-        uatomic_set (&peerinfo->state.state, state[event_type].next_state);
+    uatomic_set(&peerinfo->state.state, state[event_type].next_state);
 
-        ret = 0;
+    ret = 0;
 out:
-        rcu_read_unlock ();
-        return ret;
+    RCU_READ_UNLOCK;
+    return ret;
 }
 
-
-glusterd_sm_t glusterd_state_default [] = {
-        {GD_FRIEND_STATE_DEFAULT, glusterd_ac_none},
-        {GD_FRIEND_STATE_DEFAULT, glusterd_ac_friend_probe},//EV_PROBE
-        {GD_FRIEND_STATE_REQ_SENT, glusterd_ac_friend_add}, //EV_INIT_FRIEND_REQ
-        {GD_FRIEND_STATE_DEFAULT, glusterd_ac_none}, //EVENT_RCVD_ACC
-        {GD_FRIEND_STATE_DEFAULT, glusterd_ac_none}, //EVENT_RCVD_LOCAL_ACC
-        {GD_FRIEND_STATE_DEFAULT, glusterd_ac_none}, //EVENT_RCVD_RJT
-        {GD_FRIEND_STATE_DEFAULT, glusterd_ac_none}, //EVENT_RCVD_LOCAL_RJT
-        {GD_FRIEND_STATE_REQ_RCVD, glusterd_ac_handle_friend_add_req}, //EVENT_RCV_FRIEND_REQ
-        {GD_FRIEND_STATE_DEFAULT, glusterd_ac_send_friend_remove_req}, //EV_INIT_REMOVE_FRIEND
-        {GD_FRIEND_STATE_DEFAULT, glusterd_ac_none}, //EVENT_RCVD_REMOVE_FRIEND
-        {GD_FRIEND_STATE_DEFAULT, glusterd_ac_friend_remove}, //EVENT_REMOVE_FRIEND
-        {GD_FRIEND_STATE_DEFAULT, glusterd_ac_friend_probe}, //EVENT_CONNECTED
-        {GD_FRIEND_STATE_DEFAULT, glusterd_ac_none}, //EVENT_MAX
+glusterd_sm_t glusterd_state_default[] = {
+    {GD_FRIEND_STATE_DEFAULT, glusterd_ac_none},
+    {GD_FRIEND_STATE_DEFAULT, glusterd_ac_friend_probe},  // EV_PROBE
+    {GD_FRIEND_STATE_REQ_SENT, glusterd_ac_friend_add},   // EV_INIT_FRIEND_REQ
+    {GD_FRIEND_STATE_DEFAULT, glusterd_ac_none},          // EVENT_RCVD_ACC
+    {GD_FRIEND_STATE_DEFAULT, glusterd_ac_none},  // EVENT_RCVD_LOCAL_ACC
+    {GD_FRIEND_STATE_DEFAULT, glusterd_ac_none},  // EVENT_RCVD_RJT
+    {GD_FRIEND_STATE_DEFAULT, glusterd_ac_none},  // EVENT_RCVD_LOCAL_RJT
+    {GD_FRIEND_STATE_REQ_RCVD,
+     glusterd_ac_handle_friend_add_req},  // EVENT_RCV_FRIEND_REQ
+    {GD_FRIEND_STATE_DEFAULT,
+     glusterd_ac_send_friend_remove_req},         // EV_INIT_REMOVE_FRIEND
+    {GD_FRIEND_STATE_DEFAULT, glusterd_ac_none},  // EVENT_RCVD_REMOVE_FRIEND
+    {GD_FRIEND_STATE_DEFAULT,
+     glusterd_ac_friend_remove},                          // EVENT_REMOVE_FRIEND
+    {GD_FRIEND_STATE_DEFAULT, glusterd_ac_friend_probe},  // EVENT_CONNECTED
+    {GD_FRIEND_STATE_DEFAULT, glusterd_ac_none},          // EVENT_NEW_NAME
+    {GD_FRIEND_STATE_DEFAULT, glusterd_ac_none},          // EVENT_MAX
 };
 
-glusterd_sm_t glusterd_state_probe_rcvd [] = {
-        {GD_FRIEND_STATE_PROBE_RCVD, glusterd_ac_none},
-        {GD_FRIEND_STATE_PROBE_RCVD, glusterd_ac_none}, //EV_PROBE
-        {GD_FRIEND_STATE_PROBE_RCVD, glusterd_ac_none}, //EV_INIT_FRIEND_REQ
-        {GD_FRIEND_STATE_PROBE_RCVD, glusterd_ac_none}, //EVENT_RCVD_ACC
-        {GD_FRIEND_STATE_PROBE_RCVD, glusterd_ac_none}, //EVENT_RCVD_LOCAL_ACC
-        {GD_FRIEND_STATE_PROBE_RCVD, glusterd_ac_none}, //EVENT_RCVD_RJT
-        {GD_FRIEND_STATE_PROBE_RCVD, glusterd_ac_none}, //EVENT_RCVD_LOCAL_RJT
-        {GD_FRIEND_STATE_REQ_RCVD, glusterd_ac_handle_friend_add_req}, //EVENT_RCV_FRIEND_REQ
-        {GD_FRIEND_STATE_DEFAULT, glusterd_ac_send_friend_remove_req}, //EV_INIT_REMOVE_FRIEND
-        {GD_FRIEND_STATE_DEFAULT, glusterd_ac_none}, //EVENT_RCVD_REMOVE_FRIEND
-        {GD_FRIEND_STATE_DEFAULT, glusterd_ac_friend_remove}, //EVENT_REMOVE_FRIEND
-        {GD_FRIEND_STATE_CONNECTED_RCVD, glusterd_ac_none}, //EVENT_CONNECTED
-        {GD_FRIEND_STATE_DEFAULT, glusterd_ac_none}, //EVENT_MAX
+glusterd_sm_t glusterd_state_probe_rcvd[] = {
+    {GD_FRIEND_STATE_PROBE_RCVD, glusterd_ac_none},
+    {GD_FRIEND_STATE_PROBE_RCVD, glusterd_ac_none},  // EV_PROBE
+    {GD_FRIEND_STATE_PROBE_RCVD, glusterd_ac_none},  // EV_INIT_FRIEND_REQ
+    {GD_FRIEND_STATE_PROBE_RCVD, glusterd_ac_none},  // EVENT_RCVD_ACC
+    {GD_FRIEND_STATE_PROBE_RCVD, glusterd_ac_none},  // EVENT_RCVD_LOCAL_ACC
+    {GD_FRIEND_STATE_PROBE_RCVD, glusterd_ac_none},  // EVENT_RCVD_RJT
+    {GD_FRIEND_STATE_PROBE_RCVD, glusterd_ac_none},  // EVENT_RCVD_LOCAL_RJT
+    {GD_FRIEND_STATE_REQ_RCVD,
+     glusterd_ac_handle_friend_add_req},  // EVENT_RCV_FRIEND_REQ
+    {GD_FRIEND_STATE_DEFAULT,
+     glusterd_ac_send_friend_remove_req},         // EV_INIT_REMOVE_FRIEND
+    {GD_FRIEND_STATE_DEFAULT, glusterd_ac_none},  // EVENT_RCVD_REMOVE_FRIEND
+    {GD_FRIEND_STATE_DEFAULT,
+     glusterd_ac_friend_remove},                         // EVENT_REMOVE_FRIEND
+    {GD_FRIEND_STATE_CONNECTED_RCVD, glusterd_ac_none},  // EVENT_CONNECTED
+    {GD_FRIEND_STATE_DEFAULT, glusterd_ac_none},         // EVENT_NEW_NAME
+    {GD_FRIEND_STATE_DEFAULT, glusterd_ac_none},         // EVENT_MAX
 };
 
-glusterd_sm_t glusterd_state_connected_rcvd [] = {
-        {GD_FRIEND_STATE_CONNECTED_RCVD, glusterd_ac_none},
-        {GD_FRIEND_STATE_CONNECTED_RCVD, glusterd_ac_none}, //EV_PROBE
-        {GD_FRIEND_STATE_CONNECTED_RCVD, glusterd_ac_none}, //EV_INIT_FRIEND_REQ
-        {GD_FRIEND_STATE_CONNECTED_RCVD, glusterd_ac_none}, //EVENT_RCVD_ACC
-        {GD_FRIEND_STATE_CONNECTED_ACCEPTED, glusterd_ac_reverse_probe_begin}, //EVENT_RCVD_LOCAL_ACC
-        {GD_FRIEND_STATE_CONNECTED_RCVD, glusterd_ac_none}, //EVENT_RCVD_RJT
-        {GD_FRIEND_STATE_REJECTED, glusterd_ac_none}, //EVENT_RCVD_LOCAL_RJT
-        {GD_FRIEND_STATE_CONNECTED_RCVD, glusterd_ac_handle_friend_add_req}, //EVENT_RCV_FRIEND_REQ
-        {GD_FRIEND_STATE_DEFAULT, glusterd_ac_send_friend_remove_req}, //EV_INIT_REMOVE_FRIEND
-        {GD_FRIEND_STATE_DEFAULT, glusterd_ac_none}, //EVENT_RCVD_REMOVE_FRIEND
-        {GD_FRIEND_STATE_DEFAULT, glusterd_ac_friend_remove}, //EVENT_REMOVE_FRIEND
-        {GD_FRIEND_STATE_CONNECTED_RCVD, glusterd_ac_none}, //EVENT_CONNECTED
-        {GD_FRIEND_STATE_CONNECTED_RCVD, glusterd_ac_none}, //EVENT_MAX
+glusterd_sm_t glusterd_state_connected_rcvd[] = {
+    {GD_FRIEND_STATE_CONNECTED_RCVD, glusterd_ac_none},
+    {GD_FRIEND_STATE_CONNECTED_RCVD, glusterd_ac_none},  // EV_PROBE
+    {GD_FRIEND_STATE_CONNECTED_RCVD, glusterd_ac_none},  // EV_INIT_FRIEND_REQ
+    {GD_FRIEND_STATE_CONNECTED_RCVD, glusterd_ac_none},  // EVENT_RCVD_ACC
+    {GD_FRIEND_STATE_CONNECTED_ACCEPTED,
+     glusterd_ac_reverse_probe_begin},                   // EVENT_RCVD_LOCAL_ACC
+    {GD_FRIEND_STATE_CONNECTED_RCVD, glusterd_ac_none},  // EVENT_RCVD_RJT
+    {GD_FRIEND_STATE_REJECTED, glusterd_ac_none},        // EVENT_RCVD_LOCAL_RJT
+    {GD_FRIEND_STATE_CONNECTED_RCVD,
+     glusterd_ac_handle_friend_add_req},  // EVENT_RCV_FRIEND_REQ
+    {GD_FRIEND_STATE_DEFAULT,
+     glusterd_ac_send_friend_remove_req},         // EV_INIT_REMOVE_FRIEND
+    {GD_FRIEND_STATE_DEFAULT, glusterd_ac_none},  // EVENT_RCVD_REMOVE_FRIEND
+    {GD_FRIEND_STATE_DEFAULT,
+     glusterd_ac_friend_remove},                         // EVENT_REMOVE_FRIEND
+    {GD_FRIEND_STATE_CONNECTED_RCVD, glusterd_ac_none},  // EVENT_CONNECTED
+    {GD_FRIEND_STATE_CONNECTED_RCVD, glusterd_ac_none},  // EVENT_NEW_NAME
+    {GD_FRIEND_STATE_CONNECTED_RCVD, glusterd_ac_none},  // EVENT_MAX
 };
 
-glusterd_sm_t glusterd_state_connected_accepted [] = {
-        {GD_FRIEND_STATE_CONNECTED_ACCEPTED, glusterd_ac_none},
-        {GD_FRIEND_STATE_CONNECTED_ACCEPTED, glusterd_ac_friend_probe}, //EV_PROBE
-        {GD_FRIEND_STATE_REQ_SENT_RCVD, glusterd_ac_friend_add}, //EV_INIT_FRIEND_REQ
-        {GD_FRIEND_STATE_CONNECTED_ACCEPTED, glusterd_ac_none}, //EVENT_RCVD_ACC
-        {GD_FRIEND_STATE_CONNECTED_ACCEPTED, glusterd_ac_none}, //EVENT_RCVD_LOCAL_ACC
-        {GD_FRIEND_STATE_CONNECTED_ACCEPTED, glusterd_ac_none}, //EVENT_RCVD_RJT
-        {GD_FRIEND_STATE_CONNECTED_ACCEPTED, glusterd_ac_none}, //EVENT_RCVD_LOCAL_RJT
-        {GD_FRIEND_STATE_CONNECTED_ACCEPTED, glusterd_ac_none}, //EVENT_RCV_FRIEND_REQ
-        {GD_FRIEND_STATE_DEFAULT, glusterd_ac_send_friend_remove_req}, //EV_INIT_REMOVE_FRIEND
-        {GD_FRIEND_STATE_DEFAULT, glusterd_ac_none}, //EVENT_RCVD_REMOVE_FRIEND
-        {GD_FRIEND_STATE_DEFAULT, glusterd_ac_friend_remove}, //EVENT_REMOVE_FRIEND
-        {GD_FRIEND_STATE_CONNECTED_ACCEPTED, glusterd_ac_none}, //EVENT_CONNECTED
-        {GD_FRIEND_STATE_CONNECTED_ACCEPTED, glusterd_ac_none}, //EVENT_MAX
+glusterd_sm_t glusterd_state_connected_accepted[] = {
+    {GD_FRIEND_STATE_CONNECTED_ACCEPTED, glusterd_ac_none},
+    {GD_FRIEND_STATE_CONNECTED_ACCEPTED, glusterd_ac_friend_probe},  // EV_PROBE
+    {GD_FRIEND_STATE_REQ_SENT_RCVD,
+     glusterd_ac_friend_add},  // EV_INIT_FRIEND_REQ
+    {GD_FRIEND_STATE_CONNECTED_ACCEPTED, glusterd_ac_none},  // EVENT_RCVD_ACC
+    {GD_FRIEND_STATE_CONNECTED_ACCEPTED,
+     glusterd_ac_none},  // EVENT_RCVD_LOCAL_ACC
+    {GD_FRIEND_STATE_CONNECTED_ACCEPTED, glusterd_ac_none},  // EVENT_RCVD_RJT
+    {GD_FRIEND_STATE_CONNECTED_ACCEPTED,
+     glusterd_ac_none},  // EVENT_RCVD_LOCAL_RJT
+    {GD_FRIEND_STATE_CONNECTED_ACCEPTED,
+     glusterd_ac_none},  // EVENT_RCV_FRIEND_REQ
+    {GD_FRIEND_STATE_DEFAULT,
+     glusterd_ac_send_friend_remove_req},         // EV_INIT_REMOVE_FRIEND
+    {GD_FRIEND_STATE_DEFAULT, glusterd_ac_none},  // EVENT_RCVD_REMOVE_FRIEND
+    {GD_FRIEND_STATE_DEFAULT,
+     glusterd_ac_friend_remove},  // EVENT_REMOVE_FRIEND
+    {GD_FRIEND_STATE_CONNECTED_ACCEPTED, glusterd_ac_none},  // EVENT_CONNECTED
+    {GD_FRIEND_STATE_CONNECTED_ACCEPTED, glusterd_ac_none},  // EVENT_NEW_NAME
+    {GD_FRIEND_STATE_CONNECTED_ACCEPTED, glusterd_ac_none},  // EVENT_MAX
 };
 
-glusterd_sm_t  glusterd_state_req_sent [] = {
-        {GD_FRIEND_STATE_REQ_SENT, glusterd_ac_none}, //EVENT_NONE,
-        {GD_FRIEND_STATE_REQ_SENT, glusterd_ac_none}, //EVENT_PROBE,
-        {GD_FRIEND_STATE_REQ_SENT, glusterd_ac_none}, //EVENT_INIT_FRIEND_REQ,
-        {GD_FRIEND_STATE_REQ_ACCEPTED, glusterd_ac_none}, //EVENT_RCVD_ACC
-        {GD_FRIEND_STATE_REQ_SENT, glusterd_ac_none}, //EVENT_RCVD_LOCAL_ACC
-        {GD_FRIEND_STATE_REJECTED, glusterd_ac_none}, //EVENT_RCVD_RJT
-        {GD_FRIEND_STATE_REQ_SENT, glusterd_ac_none}, //EVENT_RCVD_LOCAL_RJT
-        {GD_FRIEND_STATE_REQ_SENT_RCVD, glusterd_ac_handle_friend_add_req}, //EVENT_RCV_FRIEND_REQ
-        {GD_FRIEND_STATE_UNFRIEND_SENT, glusterd_ac_send_friend_remove_req}, //EVENT_INIT_REMOVE_FRIEND,
-        {GD_FRIEND_STATE_REQ_SENT, glusterd_ac_none}, //EVENT_RCVD_REMOVE_FRIEND
-        {GD_FRIEND_STATE_DEFAULT, glusterd_ac_friend_remove}, //EVENT_REMOVE_FRIEND
-        {GD_FRIEND_STATE_REQ_SENT, glusterd_ac_none},//EVENT_CONNECTED
-        {GD_FRIEND_STATE_REQ_SENT, glusterd_ac_none},//EVENT_MAX
+glusterd_sm_t glusterd_state_req_sent[] = {
+    {GD_FRIEND_STATE_REQ_SENT, glusterd_ac_none},      // EVENT_NONE,
+    {GD_FRIEND_STATE_REQ_SENT, glusterd_ac_none},      // EVENT_PROBE,
+    {GD_FRIEND_STATE_REQ_SENT, glusterd_ac_none},      // EVENT_INIT_FRIEND_REQ,
+    {GD_FRIEND_STATE_REQ_ACCEPTED, glusterd_ac_none},  // EVENT_RCVD_ACC
+    {GD_FRIEND_STATE_REQ_SENT, glusterd_ac_none},      // EVENT_RCVD_LOCAL_ACC
+    {GD_FRIEND_STATE_REJECTED, glusterd_ac_none},      // EVENT_RCVD_RJT
+    {GD_FRIEND_STATE_REQ_SENT, glusterd_ac_none},      // EVENT_RCVD_LOCAL_RJT
+    {GD_FRIEND_STATE_REQ_SENT_RCVD,
+     glusterd_ac_handle_friend_add_req},  // EVENT_RCV_FRIEND_REQ
+    {GD_FRIEND_STATE_UNFRIEND_SENT,
+     glusterd_ac_send_friend_remove_req},          // EVENT_INIT_REMOVE_FRIEND,
+    {GD_FRIEND_STATE_REQ_SENT, glusterd_ac_none},  // EVENT_RCVD_REMOVE_FRIEND
+    {GD_FRIEND_STATE_DEFAULT,
+     glusterd_ac_friend_remove},                   // EVENT_REMOVE_FRIEND
+    {GD_FRIEND_STATE_REQ_SENT, glusterd_ac_none},  // EVENT_CONNECTED
+    {GD_FRIEND_STATE_REQ_SENT, glusterd_ac_none},  // EVENT_NEW_NAME
+    {GD_FRIEND_STATE_REQ_SENT, glusterd_ac_none},  // EVENT_MAX
 };
 
-glusterd_sm_t  glusterd_state_req_rcvd [] = {
-        {GD_FRIEND_STATE_REQ_RCVD, glusterd_ac_none}, //EVENT_NONE,
-        {GD_FRIEND_STATE_REQ_RCVD, glusterd_ac_none}, //EVENT_PROBE,
-        {GD_FRIEND_STATE_REQ_SENT_RCVD, glusterd_ac_none}, //EVENT_INIT_FRIEND_REQ,
-        {GD_FRIEND_STATE_REQ_RCVD, glusterd_ac_none}, //EVENT_RCVD_ACC
-        {GD_FRIEND_STATE_REQ_ACCEPTED, glusterd_ac_none}, //EVENT_RCVD_LOCAL_ACC
-        {GD_FRIEND_STATE_REQ_RCVD, glusterd_ac_none}, //EVENT_RCVD_RJT
-        {GD_FRIEND_STATE_REJECTED, glusterd_ac_none}, //EVENT_RCVD_LOCAL_RJT
-        {GD_FRIEND_STATE_REQ_RCVD, glusterd_ac_none}, //EVENT_RCV_FRIEND_REQ
-        {GD_FRIEND_STATE_DEFAULT, glusterd_ac_send_friend_remove_req}, //EVENT_INIT_REMOVE_FRIEND,
-        {GD_FRIEND_STATE_DEFAULT, glusterd_ac_handle_friend_remove_req}, //EVENT_RCVD_REMOVE_FRIEND
-        {GD_FRIEND_STATE_DEFAULT, glusterd_ac_friend_remove}, //EVENT_REMOVE_FRIEND
-        {GD_FRIEND_STATE_CONNECTED_RCVD, glusterd_ac_none},//EVENT_CONNECTED
-        {GD_FRIEND_STATE_REQ_RCVD, glusterd_ac_none},//EVENT_MAX
+glusterd_sm_t glusterd_state_req_rcvd[] = {
+    {GD_FRIEND_STATE_REQ_RCVD, glusterd_ac_none},  // EVENT_NONE,
+    {GD_FRIEND_STATE_REQ_RCVD, glusterd_ac_none},  // EVENT_PROBE,
+    {GD_FRIEND_STATE_REQ_SENT_RCVD,
+     glusterd_ac_none},                                // EVENT_INIT_FRIEND_REQ,
+    {GD_FRIEND_STATE_REQ_RCVD, glusterd_ac_none},      // EVENT_RCVD_ACC
+    {GD_FRIEND_STATE_REQ_ACCEPTED, glusterd_ac_none},  // EVENT_RCVD_LOCAL_ACC
+    {GD_FRIEND_STATE_REQ_RCVD, glusterd_ac_none},      // EVENT_RCVD_RJT
+    {GD_FRIEND_STATE_REJECTED, glusterd_ac_none},      // EVENT_RCVD_LOCAL_RJT
+    {GD_FRIEND_STATE_REQ_RCVD, glusterd_ac_none},      // EVENT_RCV_FRIEND_REQ
+    {GD_FRIEND_STATE_DEFAULT,
+     glusterd_ac_send_friend_remove_req},  // EVENT_INIT_REMOVE_FRIEND,
+    {GD_FRIEND_STATE_DEFAULT,
+     glusterd_ac_handle_friend_remove_req},  // EVENT_RCVD_REMOVE_FRIEND
+    {GD_FRIEND_STATE_DEFAULT,
+     glusterd_ac_friend_remove},                         // EVENT_REMOVE_FRIEND
+    {GD_FRIEND_STATE_CONNECTED_RCVD, glusterd_ac_none},  // EVENT_CONNECTED
+    {GD_FRIEND_STATE_CONNECTED_RCVD, glusterd_ac_none},  // EVENT_NEW_NAME
+    {GD_FRIEND_STATE_REQ_RCVD, glusterd_ac_none},        // EVENT_MAX
 };
 
-glusterd_sm_t  glusterd_state_befriended [] = {
-        {GD_FRIEND_STATE_BEFRIENDED, glusterd_ac_none}, //EVENT_NONE,
-        {GD_FRIEND_STATE_BEFRIENDED, glusterd_ac_none}, //EVENT_PROBE,
-        {GD_FRIEND_STATE_BEFRIENDED, glusterd_ac_none}, //EVENT_INIT_FRIEND_REQ,
-        {GD_FRIEND_STATE_BEFRIENDED, glusterd_ac_none}, //EVENT_RCVD_ACC
-        {GD_FRIEND_STATE_BEFRIENDED, glusterd_ac_send_friend_update}, //EVENT_RCVD_LOCAL_ACC
-        {GD_FRIEND_STATE_REJECTED, glusterd_ac_none}, //EVENT_RCVD_RJT
-        {GD_FRIEND_STATE_REJECTED, glusterd_ac_none}, //EVENT_RCVD_LOCAL_RJT
-        {GD_FRIEND_STATE_BEFRIENDED, glusterd_ac_handle_friend_add_req}, //EVENT_RCV_FRIEND_REQ
-        {GD_FRIEND_STATE_UNFRIEND_SENT, glusterd_ac_send_friend_remove_req}, //EVENT_INIT_REMOVE_FRIEND,
-        {GD_FRIEND_STATE_DEFAULT, glusterd_ac_handle_friend_remove_req}, //EVENT_RCVD_REMOVE_FRIEND
-        {GD_FRIEND_STATE_DEFAULT, glusterd_ac_friend_remove}, //EVENT_REMOVE_FRIEND
-        {GD_FRIEND_STATE_BEFRIENDED, glusterd_ac_friend_add},//EVENT_CONNECTED
-        {GD_FRIEND_STATE_BEFRIENDED, glusterd_ac_none},//EVENT_MAX
+glusterd_sm_t glusterd_state_befriended[] = {
+    {GD_FRIEND_STATE_BEFRIENDED, glusterd_ac_none},  // EVENT_NONE,
+    {GD_FRIEND_STATE_BEFRIENDED, glusterd_ac_none},  // EVENT_PROBE,
+    {GD_FRIEND_STATE_BEFRIENDED, glusterd_ac_none},  // EVENT_INIT_FRIEND_REQ,
+    {GD_FRIEND_STATE_BEFRIENDED, glusterd_ac_update_friend},  // EVENT_RCVD_ACC
+    {GD_FRIEND_STATE_BEFRIENDED,
+     glusterd_ac_update_friend},                   // EVENT_RCVD_LOCAL_ACC
+    {GD_FRIEND_STATE_REJECTED, glusterd_ac_none},  // EVENT_RCVD_RJT
+    {GD_FRIEND_STATE_REJECTED, glusterd_ac_none},  // EVENT_RCVD_LOCAL_RJT
+    {GD_FRIEND_STATE_BEFRIENDED,
+     glusterd_ac_handle_friend_add_req},  // EVENT_RCV_FRIEND_REQ
+    {GD_FRIEND_STATE_UNFRIEND_SENT,
+     glusterd_ac_send_friend_remove_req},  // EVENT_INIT_REMOVE_FRIEND,
+    {GD_FRIEND_STATE_DEFAULT,
+     glusterd_ac_handle_friend_remove_req},  // EVENT_RCVD_REMOVE_FRIEND
+    {GD_FRIEND_STATE_DEFAULT,
+     glusterd_ac_friend_remove},  // EVENT_REMOVE_FRIEND
+    {GD_FRIEND_STATE_BEFRIENDED, glusterd_ac_friend_add},  // EVENT_CONNECTED
+    {GD_FRIEND_STATE_BEFRIENDED,
+     glusterd_ac_send_friend_update},                // EVENT_NEW_NAME
+    {GD_FRIEND_STATE_BEFRIENDED, glusterd_ac_none},  // EVENT_MAX
 };
 
-glusterd_sm_t  glusterd_state_req_sent_rcvd [] = {
-        {GD_FRIEND_STATE_REQ_SENT_RCVD, glusterd_ac_none}, //EVENT_NONE,
-        {GD_FRIEND_STATE_REQ_SENT_RCVD, glusterd_ac_none}, //EVENT_PROBE,
-        {GD_FRIEND_STATE_REQ_SENT_RCVD, glusterd_ac_none}, //EVENT_INIT_FRIEND_REQ,
-        {GD_FRIEND_STATE_BEFRIENDED, glusterd_ac_send_friend_update}, //EVENT_RCVD_ACC
-        {GD_FRIEND_STATE_REQ_SENT_RCVD, glusterd_ac_none}, //EVENT_RCVD_LOCAL_ACC
-        {GD_FRIEND_STATE_REJECTED, glusterd_ac_none}, //EVENT_RCVD_RJT
-        {GD_FRIEND_STATE_REQ_SENT_RCVD, glusterd_ac_none}, //EVENT_RCVD_LOCAL_RJT
-        {GD_FRIEND_STATE_REQ_SENT_RCVD, glusterd_ac_none}, //EVENT_RCV_FRIEND_REQ
-        {GD_FRIEND_STATE_UNFRIEND_SENT, glusterd_ac_send_friend_remove_req}, //EVENT_INIT_REMOVE_FRIEND,
-        {GD_FRIEND_STATE_DEFAULT, glusterd_ac_handle_friend_remove_req}, //EVENT_RCVD_REMOVE_FRIEND
-        {GD_FRIEND_STATE_DEFAULT, glusterd_ac_friend_remove}, //EVENT_REMOVE_FRIEND
-        {GD_FRIEND_STATE_REQ_SENT_RCVD, glusterd_ac_none},//EVENT_CONNECTED
-        {GD_FRIEND_STATE_REQ_SENT_RCVD, glusterd_ac_none},//EVENT_MAX
+glusterd_sm_t glusterd_state_req_sent_rcvd[] = {
+    {GD_FRIEND_STATE_REQ_SENT_RCVD, glusterd_ac_none},  // EVENT_NONE,
+    {GD_FRIEND_STATE_REQ_SENT_RCVD, glusterd_ac_none},  // EVENT_PROBE,
+    {GD_FRIEND_STATE_REQ_SENT_RCVD,
+     glusterd_ac_none},  // EVENT_INIT_FRIEND_REQ,
+    {GD_FRIEND_STATE_BEFRIENDED,
+     glusterd_ac_send_friend_update},                   // EVENT_RCVD_ACC
+    {GD_FRIEND_STATE_REQ_SENT_RCVD, glusterd_ac_none},  // EVENT_RCVD_LOCAL_ACC
+    {GD_FRIEND_STATE_REJECTED, glusterd_ac_none},       // EVENT_RCVD_RJT
+    {GD_FRIEND_STATE_REQ_SENT_RCVD, glusterd_ac_none},  // EVENT_RCVD_LOCAL_RJT
+    {GD_FRIEND_STATE_REQ_SENT_RCVD, glusterd_ac_none},  // EVENT_RCV_FRIEND_REQ
+    {GD_FRIEND_STATE_UNFRIEND_SENT,
+     glusterd_ac_send_friend_remove_req},  // EVENT_INIT_REMOVE_FRIEND,
+    {GD_FRIEND_STATE_DEFAULT,
+     glusterd_ac_handle_friend_remove_req},  // EVENT_RCVD_REMOVE_FRIEND
+    {GD_FRIEND_STATE_DEFAULT,
+     glusterd_ac_friend_remove},                        // EVENT_REMOVE_FRIEND
+    {GD_FRIEND_STATE_REQ_SENT_RCVD, glusterd_ac_none},  // EVENT_CONNECTED
+    {GD_FRIEND_STATE_REQ_SENT_RCVD, glusterd_ac_none},  // EVENT_NEW_NAME
+    {GD_FRIEND_STATE_REQ_SENT_RCVD, glusterd_ac_none},  // EVENT_MAX
 };
 
-glusterd_sm_t  glusterd_state_rejected [] = {
-        {GD_FRIEND_STATE_REJECTED, glusterd_ac_none}, //EVENT_NONE,
-        {GD_FRIEND_STATE_REJECTED, glusterd_ac_friend_probe}, //EVENT_PROBE,
-        {GD_FRIEND_STATE_REQ_SENT, glusterd_ac_friend_add}, //EVENT_INIT_FRIEND_REQ,
-        {GD_FRIEND_STATE_BEFRIENDED, glusterd_ac_none}, //EVENT_RCVD_ACC
-        {GD_FRIEND_STATE_BEFRIENDED, glusterd_ac_none}, //EVENT_RCVD_LOCAL_ACC
-        {GD_FRIEND_STATE_REJECTED, glusterd_ac_none}, //EVENT_RCVD_RJT
-        {GD_FRIEND_STATE_REJECTED, glusterd_ac_none}, //EVENT_RCVD_LOCAL_RJT
-        {GD_FRIEND_STATE_REQ_RCVD, glusterd_ac_handle_friend_add_req}, //EVENT_RCV_FRIEND_REQ
-        {GD_FRIEND_STATE_DEFAULT, glusterd_ac_send_friend_remove_req}, //EVENT_INIT_REMOVE_FRIEND
-        {GD_FRIEND_STATE_DEFAULT, glusterd_ac_handle_friend_remove_req}, //EVENT_RCVD_REMOVE_FRIEND
-        {GD_FRIEND_STATE_DEFAULT, glusterd_ac_friend_remove}, //EVENT_REMOVE_FRIEND
-        {GD_FRIEND_STATE_REJECTED, glusterd_ac_friend_add},//EVENT_CONNECTED
-        {GD_FRIEND_STATE_REQ_RCVD, glusterd_ac_none},//EVENT_MAX
+glusterd_sm_t glusterd_state_rejected[] = {
+    {GD_FRIEND_STATE_REJECTED, glusterd_ac_none},          // EVENT_NONE,
+    {GD_FRIEND_STATE_REJECTED, glusterd_ac_friend_probe},  // EVENT_PROBE,
+    {GD_FRIEND_STATE_REQ_SENT,
+     glusterd_ac_friend_add},                        // EVENT_INIT_FRIEND_REQ,
+    {GD_FRIEND_STATE_BEFRIENDED, glusterd_ac_none},  // EVENT_RCVD_ACC
+    {GD_FRIEND_STATE_BEFRIENDED, glusterd_ac_none},  // EVENT_RCVD_LOCAL_ACC
+    {GD_FRIEND_STATE_REJECTED, glusterd_ac_none},    // EVENT_RCVD_RJT
+    {GD_FRIEND_STATE_REJECTED, glusterd_ac_none},    // EVENT_RCVD_LOCAL_RJT
+    {GD_FRIEND_STATE_REQ_RCVD,
+     glusterd_ac_handle_friend_add_req},  // EVENT_RCV_FRIEND_REQ
+    {GD_FRIEND_STATE_DEFAULT,
+     glusterd_ac_send_friend_remove_req},  // EVENT_INIT_REMOVE_FRIEND
+    {GD_FRIEND_STATE_DEFAULT,
+     glusterd_ac_handle_friend_remove_req},  // EVENT_RCVD_REMOVE_FRIEND
+    {GD_FRIEND_STATE_DEFAULT,
+     glusterd_ac_friend_remove},                         // EVENT_REMOVE_FRIEND
+    {GD_FRIEND_STATE_REJECTED, glusterd_ac_friend_add},  // EVENT_CONNECTED
+    {GD_FRIEND_STATE_REJECTED, glusterd_ac_none},        // EVENT_NEW_NAME
+    {GD_FRIEND_STATE_REQ_RCVD, glusterd_ac_none},        // EVENT_MAX
 };
 
-glusterd_sm_t  glusterd_state_req_accepted [] = {
-        {GD_FRIEND_STATE_REQ_ACCEPTED, glusterd_ac_none}, //EVENT_NONE,
-        {GD_FRIEND_STATE_REQ_ACCEPTED, glusterd_ac_none}, //EVENT_PROBE,
-        {GD_FRIEND_STATE_REQ_ACCEPTED, glusterd_ac_none}, //EVENT_INIT_FRIEND_REQ,
-        {GD_FRIEND_STATE_BEFRIENDED, glusterd_ac_send_friend_update}, //EVENT_RCVD_ACC
-        {GD_FRIEND_STATE_BEFRIENDED, glusterd_ac_send_friend_update}, //EVENT_RCVD_LOCAL_ACC
-        {GD_FRIEND_STATE_REJECTED, glusterd_ac_none}, //EVENT_RCVD_RJT
-        {GD_FRIEND_STATE_REJECTED, glusterd_ac_none}, //EVENT_RCVD_LOCAL_RJT
-        {GD_FRIEND_STATE_REQ_ACCEPTED, glusterd_ac_handle_friend_add_req}, //EVENT_RCV_FRIEND_REQ
-        {GD_FRIEND_STATE_REQ_ACCEPTED, glusterd_ac_send_friend_remove_req}, //EVENT_INIT_REMOVE_FRIEND
-        {GD_FRIEND_STATE_DEFAULT, glusterd_ac_handle_friend_remove_req}, //EVENT_RCVD_REMOVE_FRIEND
-        {GD_FRIEND_STATE_DEFAULT, glusterd_ac_friend_remove}, //EVENT_REMOVE_FRIEND
-        {GD_FRIEND_STATE_CONNECTED_ACCEPTED, glusterd_ac_reverse_probe_begin},//EVENT_CONNECTED
-        {GD_FRIEND_STATE_REQ_SENT, glusterd_ac_none},//EVENT_MAX
+glusterd_sm_t glusterd_state_req_accepted[] = {
+    {GD_FRIEND_STATE_REQ_ACCEPTED, glusterd_ac_none},  // EVENT_NONE,
+    {GD_FRIEND_STATE_REQ_ACCEPTED, glusterd_ac_none},  // EVENT_PROBE,
+    {GD_FRIEND_STATE_REQ_ACCEPTED, glusterd_ac_none},  // EVENT_INIT_FRIEND_REQ,
+    {GD_FRIEND_STATE_BEFRIENDED,
+     glusterd_ac_send_friend_update},  // EVENT_RCVD_ACC
+    {GD_FRIEND_STATE_BEFRIENDED,
+     glusterd_ac_send_friend_update},              // EVENT_RCVD_LOCAL_ACC
+    {GD_FRIEND_STATE_REJECTED, glusterd_ac_none},  // EVENT_RCVD_RJT
+    {GD_FRIEND_STATE_REJECTED, glusterd_ac_none},  // EVENT_RCVD_LOCAL_RJT
+    {GD_FRIEND_STATE_REQ_ACCEPTED,
+     glusterd_ac_handle_friend_add_req},  // EVENT_RCV_FRIEND_REQ
+    {GD_FRIEND_STATE_REQ_ACCEPTED,
+     glusterd_ac_send_friend_remove_req},  // EVENT_INIT_REMOVE_FRIEND
+    {GD_FRIEND_STATE_DEFAULT,
+     glusterd_ac_handle_friend_remove_req},  // EVENT_RCVD_REMOVE_FRIEND
+    {GD_FRIEND_STATE_DEFAULT,
+     glusterd_ac_friend_remove},  // EVENT_REMOVE_FRIEND
+    {GD_FRIEND_STATE_CONNECTED_ACCEPTED,
+     glusterd_ac_reverse_probe_begin},                 // EVENT_CONNECTED
+    {GD_FRIEND_STATE_REQ_ACCEPTED, glusterd_ac_none},  // EVENT_NEW_NAME
+    {GD_FRIEND_STATE_REQ_SENT, glusterd_ac_none},      // EVENT_MAX
 };
 
-glusterd_sm_t  glusterd_state_unfriend_sent [] = {
-        {GD_FRIEND_STATE_UNFRIEND_SENT, glusterd_ac_none}, //EVENT_NONE,
-        {GD_FRIEND_STATE_UNFRIEND_SENT, glusterd_ac_error}, //EVENT_PROBE,
-        {GD_FRIEND_STATE_UNFRIEND_SENT, glusterd_ac_none}, //EVENT_INIT_FRIEND_REQ,
-        {GD_FRIEND_STATE_UNFRIEND_SENT, glusterd_ac_none}, //EVENT_RCVD_ACC
-        {GD_FRIEND_STATE_UNFRIEND_SENT, glusterd_ac_none}, //EVENT_RCVD_LOCAL_ACC
-        {GD_FRIEND_STATE_UNFRIEND_SENT, glusterd_ac_error}, //EVENT_RCVD_RJT
-        {GD_FRIEND_STATE_UNFRIEND_SENT, glusterd_ac_error}, //EVENT_RCVD_LOCAL_RJT
-        {GD_FRIEND_STATE_UNFRIEND_SENT, glusterd_ac_error}, //EVENT_RCV_FRIEND_REQ
-        {GD_FRIEND_STATE_UNFRIEND_SENT, glusterd_ac_none}, //EVENT_INIT_REMOVE_FRIEND
-        {GD_FRIEND_STATE_UNFRIEND_SENT, glusterd_ac_none}, //EVENT_RCVD_REMOVE_FRIEND
-        {GD_FRIEND_STATE_DEFAULT, glusterd_ac_friend_remove}, //EVENT_REMOVE_FRIEND
-        {GD_FRIEND_STATE_UNFRIEND_SENT, glusterd_ac_none},//EVENT_CONNECTED
-        {GD_FRIEND_STATE_UNFRIEND_SENT, glusterd_ac_none},//EVENT_MAX
+glusterd_sm_t glusterd_state_unfriend_sent[] = {
+    {GD_FRIEND_STATE_UNFRIEND_SENT, glusterd_ac_none},   // EVENT_NONE,
+    {GD_FRIEND_STATE_UNFRIEND_SENT, glusterd_ac_error},  // EVENT_PROBE,
+    {GD_FRIEND_STATE_UNFRIEND_SENT,
+     glusterd_ac_none},  // EVENT_INIT_FRIEND_REQ,
+    {GD_FRIEND_STATE_UNFRIEND_SENT, glusterd_ac_none},   // EVENT_RCVD_ACC
+    {GD_FRIEND_STATE_UNFRIEND_SENT, glusterd_ac_none},   // EVENT_RCVD_LOCAL_ACC
+    {GD_FRIEND_STATE_UNFRIEND_SENT, glusterd_ac_error},  // EVENT_RCVD_RJT
+    {GD_FRIEND_STATE_UNFRIEND_SENT, glusterd_ac_error},  // EVENT_RCVD_LOCAL_RJT
+    {GD_FRIEND_STATE_UNFRIEND_SENT, glusterd_ac_error},  // EVENT_RCV_FRIEND_REQ
+    {GD_FRIEND_STATE_UNFRIEND_SENT,
+     glusterd_ac_none},  // EVENT_INIT_REMOVE_FRIEND
+    {GD_FRIEND_STATE_UNFRIEND_SENT,
+     glusterd_ac_none},  // EVENT_RCVD_REMOVE_FRIEND
+    {GD_FRIEND_STATE_DEFAULT,
+     glusterd_ac_friend_remove},                        // EVENT_REMOVE_FRIEND
+    {GD_FRIEND_STATE_UNFRIEND_SENT, glusterd_ac_none},  // EVENT_CONNECTED
+    {GD_FRIEND_STATE_UNFRIEND_SENT, glusterd_ac_none},  // EVENT_NEW_NAME
+    {GD_FRIEND_STATE_UNFRIEND_SENT, glusterd_ac_none},  // EVENT_MAX
 };
 
-glusterd_sm_t *glusterd_friend_state_table [] = {
-        glusterd_state_default,
-        glusterd_state_req_sent,
-        glusterd_state_req_rcvd,
-        glusterd_state_befriended,
-        glusterd_state_req_accepted,
-        glusterd_state_req_sent_rcvd,
-        glusterd_state_rejected,
-        glusterd_state_unfriend_sent,
-        glusterd_state_probe_rcvd,
-        glusterd_state_connected_rcvd,
-        glusterd_state_connected_accepted
-};
+glusterd_sm_t *glusterd_friend_state_table[] = {
+    glusterd_state_default,           glusterd_state_req_sent,
+    glusterd_state_req_rcvd,          glusterd_state_befriended,
+    glusterd_state_req_accepted,      glusterd_state_req_sent_rcvd,
+    glusterd_state_rejected,          glusterd_state_unfriend_sent,
+    glusterd_state_probe_rcvd,        glusterd_state_connected_rcvd,
+    glusterd_state_connected_accepted};
 
 int
-glusterd_friend_sm_new_event (glusterd_friend_sm_event_type_t event_type,
-                              glusterd_friend_sm_event_t **new_event)
+glusterd_friend_sm_new_event(glusterd_friend_sm_event_type_t event_type,
+                             glusterd_friend_sm_event_t **new_event)
 {
-        glusterd_friend_sm_event_t      *event = NULL;
+    glusterd_friend_sm_event_t *event = NULL;
 
-        GF_ASSERT (new_event);
-        GF_ASSERT (GD_FRIEND_EVENT_NONE <= event_type &&
-                        GD_FRIEND_EVENT_MAX > event_type);
+    GF_ASSERT(new_event);
+    GF_ASSERT(GD_FRIEND_EVENT_NONE <= event_type &&
+              GD_FRIEND_EVENT_MAX > event_type);
 
-        event = GF_CALLOC (1, sizeof (*event), gf_gld_mt_friend_sm_event_t);
+    event = GF_CALLOC(1, sizeof(*event), gf_gld_mt_friend_sm_event_t);
 
-        if (!event)
-                return -1;
+    if (!event)
+        return -1;
 
-        *new_event = event;
-        event->event = event_type;
-        CDS_INIT_LIST_HEAD (&event->list);
+    *new_event = event;
+    event->event = event_type;
+    CDS_INIT_LIST_HEAD(&event->list);
 
-        return 0;
+    return 0;
 }
 
 int
-glusterd_friend_sm_inject_event (glusterd_friend_sm_event_t *event)
+glusterd_friend_sm_inject_event(glusterd_friend_sm_event_t *event)
 {
-        GF_ASSERT (event);
-        gf_log ("glusterd", GF_LOG_DEBUG, "Enqueue event: '%s'",
-                glusterd_friend_sm_event_name_get (event->event));
-        cds_list_add_tail (&event->list, &gd_friend_sm_queue);
+    GF_ASSERT(event);
+    gf_msg_debug("glusterd", 0, "Enqueue event: '%s'",
+                 glusterd_friend_sm_event_name_get(event->event));
+    cds_list_add_tail(&event->list, &gd_friend_sm_queue);
 
-        return 0;
+    return 0;
 }
 
 void
-glusterd_destroy_friend_event_context (glusterd_friend_sm_event_t *event)
+glusterd_destroy_friend_event_context(glusterd_friend_sm_event_t *event)
 {
-        if (!event)
-                return;
+    if (!event)
+        return;
 
-        switch (event->event) {
+    switch (event->event) {
         case GD_FRIEND_EVENT_RCVD_FRIEND_REQ:
         case GD_FRIEND_EVENT_RCVD_REMOVE_FRIEND:
-                glusterd_destroy_friend_req_ctx (event->ctx);
-                break;
+            glusterd_destroy_friend_req_ctx(event->ctx);
+            break;
         case GD_FRIEND_EVENT_LOCAL_ACC:
         case GD_FRIEND_EVENT_LOCAL_RJT:
         case GD_FRIEND_EVENT_RCVD_ACC:
         case GD_FRIEND_EVENT_RCVD_RJT:
-                glusterd_destroy_friend_update_ctx (event->ctx);
-                break;
+            glusterd_destroy_friend_update_ctx(event->ctx);
+            break;
         default:
-                break;
-        }
+            break;
+    }
 }
 
 gf_boolean_t
-gd_does_peer_affect_quorum (glusterd_friend_sm_state_t old_state,
-                            glusterd_friend_sm_event_type_t event_type,
-                            glusterd_peerinfo_t *peerinfo)
+gd_does_peer_affect_quorum(glusterd_friend_sm_state_t old_state,
+                           glusterd_friend_sm_event_type_t event_type,
+                           glusterd_peerinfo_t *peerinfo)
 {
-        gf_boolean_t    affects = _gf_false;
-
-        //When glusterd comes up with friends in BEFRIENDED state in store,
-        //wait until compare-data happens.
-        if ((old_state == GD_FRIEND_STATE_BEFRIENDED) &&
-            (event_type != GD_FRIEND_EVENT_RCVD_ACC) &&
-            (event_type != GD_FRIEND_EVENT_LOCAL_ACC))
-                goto out;
-        if ((peerinfo->state.state == GD_FRIEND_STATE_BEFRIENDED)
-            && peerinfo->connected) {
-                affects = _gf_true;
-        }
+    gf_boolean_t affects = _gf_false;
+
+    // When glusterd comes up with friends in BEFRIENDED state in store,
+    // wait until compare-data happens.
+    if ((old_state == GD_FRIEND_STATE_BEFRIENDED) &&
+        (event_type != GD_FRIEND_EVENT_RCVD_ACC) &&
+        (event_type != GD_FRIEND_EVENT_LOCAL_ACC))
+        goto out;
+    if ((peerinfo->state.state == GD_FRIEND_STATE_BEFRIENDED) &&
+        peerinfo->connected) {
+        affects = _gf_true;
+    }
 out:
-        return affects;
+    return affects;
 }
 
 int
-glusterd_friend_sm ()
+glusterd_friend_sm()
 {
-        glusterd_friend_sm_event_t      *event      = NULL;
-        glusterd_friend_sm_event_t      *tmp        = NULL;
-        int                              ret        = -1;
-        glusterd_friend_sm_ac_fn         handler    = NULL;
-        glusterd_sm_t                   *state      = NULL;
-        glusterd_peerinfo_t             *peerinfo   = NULL;
-        glusterd_friend_sm_event_type_t  event_type = 0;
-        gf_boolean_t                     is_await_conn = _gf_false;
-        gf_boolean_t                     quorum_action = _gf_false;
-        glusterd_friend_sm_state_t       old_state = GD_FRIEND_STATE_DEFAULT;
-        xlator_t                        *this = NULL;
-        glusterd_conf_t                 *priv = NULL;
-
-        this = THIS;
-        GF_ASSERT (this);
-        priv = this->private;
-        GF_ASSERT (priv);
-
-        while (!cds_list_empty (&gd_friend_sm_queue)) {
-                cds_list_for_each_entry_safe (event, tmp, &gd_friend_sm_queue,
-                                              list) {
-
-                        cds_list_del_init (&event->list);
-                        event_type = event->event;
-
-                        rcu_read_lock ();
-
-                        peerinfo = glusterd_peerinfo_find (event->peerid,
-                                                           event->peername);
-                        if (!peerinfo) {
-                                gf_log ("glusterd", GF_LOG_CRITICAL, "Received"
-                                        " event %s with empty peer info",
-                                glusterd_friend_sm_event_name_get (event_type));
-
-                                GF_FREE (event);
-                                rcu_read_unlock ();
-                                continue;
-                        }
-                        gf_log ("", GF_LOG_DEBUG, "Dequeued event of type: '%s'",
-                                glusterd_friend_sm_event_name_get (event_type));
-
-
-                        old_state = peerinfo->state.state;
-
-                        rcu_read_unlock ();
-                        /* Giving up read-critical section here as we only need
-                         * the current state to call the handler.
-                         *
-                         * We cannot continue into the handler in a read
-                         * critical section as there are handlers who do
-                         * updates, and could cause deadlocks.
-                         */
-
-                        state = glusterd_friend_state_table[old_state];
-
-                        GF_ASSERT (state);
-
-                        handler = state[event_type].handler;
-                        GF_ASSERT (handler);
-
-                        ret = handler (event, event->ctx);
-                        if (ret == GLUSTERD_CONNECTION_AWAITED) {
-                                is_await_conn = _gf_true;
-                                ret = 0;
-                        }
-
-                        if (ret) {
-                                gf_log ("glusterd", GF_LOG_ERROR, "handler returned: "
-                                                "%d", ret);
-                                glusterd_destroy_friend_event_context (event);
-                                GF_FREE (event);
-                                continue;
-                        }
-
-                        if ((GD_FRIEND_EVENT_REMOVE_FRIEND == event_type) ||
-                           (GD_FRIEND_EVENT_INIT_REMOVE_FRIEND == event_type)){
-                                glusterd_destroy_friend_event_context (event);
-                                GF_FREE (event);
-                                continue;
-                        }
-
-                        ret = glusterd_friend_sm_transition_state
-                                (event->peerid, event->peername, state,
-                                 event_type);
-
-                        if (ret) {
-                                gf_log ("glusterd", GF_LOG_ERROR, "Unable to transition"
-                                        " state from '%s' to '%s' for event '%s'",
-                        glusterd_friend_sm_state_name_get(old_state),
-                        glusterd_friend_sm_state_name_get(state[event_type].next_state),
-                                glusterd_friend_sm_event_name_get(event_type));
-                                goto out;
-                        }
-
-                        peerinfo = NULL;
-                        /* We need to obtain peerinfo reference once again as we
-                         * had exited the read critical section above.
-                         */
-                        rcu_read_lock ();
-                        peerinfo = glusterd_peerinfo_find (event->peerid,
-                                        event->peername);
-                        if (!peerinfo) {
-                                rcu_read_unlock ();
-                                /* A peer can only be deleted as a effect of
-                                 * this state machine, and two such state
-                                 * machines can never run at the same time.
-                                 * So if we cannot find the peerinfo here,
-                                 * something has gone terribly wrong.
-                                 */
-                                ret = -1;
-                                gf_log ("glusterd", GF_LOG_ERROR,
-                                        "Cannot find peer %s(%s)",
-                                        event->peername, uuid_utoa (event->peerid));
-                                goto out;
-                        }
-                        if (gd_does_peer_affect_quorum (old_state, event_type,
-                                                        peerinfo)) {
-                                peerinfo->quorum_contrib = QUORUM_UP;
-                                if (peerinfo->quorum_action) {
-                                        peerinfo->quorum_action = _gf_false;
-                                        quorum_action = _gf_true;
-                                }
-                        }
-
-                        ret = glusterd_store_peerinfo (peerinfo);
-                        rcu_read_unlock ();
-
-                        glusterd_destroy_friend_event_context (event);
-                        GF_FREE (event);
-                        if (is_await_conn)
-                                break;
+    glusterd_friend_sm_event_t *event = NULL;
+    glusterd_friend_sm_event_t *tmp = NULL;
+    int ret = -1;
+    glusterd_friend_sm_ac_fn handler = NULL;
+    glusterd_sm_t *state = NULL;
+    glusterd_peerinfo_t *peerinfo = NULL;
+    glusterd_friend_sm_event_type_t event_type = 0;
+    gf_boolean_t is_await_conn = _gf_false;
+    gf_boolean_t quorum_action = _gf_false;
+    glusterd_friend_sm_state_t old_state = GD_FRIEND_STATE_DEFAULT;
+    xlator_t *this = NULL;
+    glusterd_conf_t *priv = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+    priv = this->private;
+    GF_ASSERT(priv);
+
+    while (!cds_list_empty(&gd_friend_sm_queue)) {
+        cds_list_for_each_entry_safe(event, tmp, &gd_friend_sm_queue, list)
+        {
+            cds_list_del_init(&event->list);
+            event_type = event->event;
+
+            RCU_READ_LOCK;
+
+            peerinfo = glusterd_peerinfo_find(event->peerid, event->peername);
+            if (!peerinfo) {
+                RCU_READ_UNLOCK;
+                gf_msg("glusterd", GF_LOG_CRITICAL, 0, GD_MSG_PEER_NOT_FOUND,
+                       "Received"
+                       " event %s with empty peer info",
+                       glusterd_friend_sm_event_name_get(event_type));
+
+                GF_FREE(event);
+                continue;
+            }
+            old_state = peerinfo->state.state;
+            RCU_READ_UNLOCK;
+            gf_msg_debug("glusterd", 0, "Dequeued event of type: '%s'",
+                         glusterd_friend_sm_event_name_get(event_type));
+
+            /* Giving up read-critical section here as we only need
+             * the current state to call the handler.
+             *
+             * We cannot continue into the handler in a read
+             * critical section as there are handlers who do
+             * updates, and could cause deadlocks.
+             */
+
+            state = glusterd_friend_state_table[old_state];
+
+            GF_ASSERT(state);
+
+            handler = state[event_type].handler;
+            GF_ASSERT(handler);
+
+            ret = handler(event, event->ctx);
+            if (ret == GLUSTERD_CONNECTION_AWAITED) {
+                is_await_conn = _gf_true;
+                ret = 0;
+            }
+
+            if (ret) {
+                gf_msg("glusterd", GF_LOG_ERROR, 0, GD_MSG_HANDLER_RETURNED,
+                       "handler returned: "
+                       "%d",
+                       ret);
+                glusterd_destroy_friend_event_context(event);
+                GF_FREE(event);
+                continue;
+            }
+
+            if ((GD_FRIEND_EVENT_REMOVE_FRIEND == event_type) ||
+                (GD_FRIEND_EVENT_INIT_REMOVE_FRIEND == event_type)) {
+                glusterd_destroy_friend_event_context(event);
+                GF_FREE(event);
+                continue;
+            }
+
+            ret = glusterd_friend_sm_transition_state(
+                event->peerid, event->peername, state, event_type);
+
+            if (ret) {
+                gf_msg("glusterd", GF_LOG_ERROR, 0,
+                       GD_MSG_EVENT_STATE_TRANSITION_FAIL,
+                       "Unable to transition"
+                       " state from '%s' to '%s' for event '%s'",
+                       glusterd_friend_sm_state_name_get(old_state),
+                       glusterd_friend_sm_state_name_get(
+                           state[event_type].next_state),
+                       glusterd_friend_sm_event_name_get(event_type));
+                goto out;
+            }
+
+            peerinfo = NULL;
+            /* We need to obtain peerinfo reference once again as we
+             * had exited the read critical section above.
+             */
+            RCU_READ_LOCK;
+            peerinfo = glusterd_peerinfo_find(event->peerid, event->peername);
+            if (!peerinfo) {
+                RCU_READ_UNLOCK;
+                /* A peer can only be deleted as a effect of
+                 * this state machine, and two such state
+                 * machines can never run at the same time.
+                 * So if we cannot find the peerinfo here,
+                 * something has gone terribly wrong.
+                 */
+                ret = -1;
+                gf_msg("glusterd", GF_LOG_ERROR, 0, GD_MSG_PEER_NOT_FOUND,
+                       "Cannot find peer %s(%s)", event->peername,
+                       uuid_utoa(event->peerid));
+                goto out;
+            }
+            if (gd_does_peer_affect_quorum(old_state, event_type, peerinfo)) {
+                peerinfo->quorum_contrib = QUORUM_UP;
+                if (peerinfo->quorum_action) {
+                    peerinfo->quorum_action = _gf_false;
+                    quorum_action = _gf_true;
                 }
-                if (is_await_conn)
-                        break;
+            }
+
+            ret = glusterd_store_peerinfo(peerinfo);
+            RCU_READ_UNLOCK;
+            if (ret) {
+                gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_PEERINFO_CREATE_FAIL,
+                       "Failed to store peerinfo");
+            }
+
+            glusterd_destroy_friend_event_context(event);
+            GF_FREE(event);
+            if (is_await_conn)
+                break;
         }
+        if (is_await_conn)
+            break;
+    }
 
-        ret = 0;
+    ret = 0;
 out:
-        if (quorum_action) {
-            /* When glusterd is restarted, it needs to wait until the 'friends' view
-             * of the volumes settle, before it starts any of the internal daemons.
-             *
-             * Every friend that was part of the cluster, would send its
-             * cluster-view, 'our' way. For every friend, who belongs to
-             * a partition which has a different cluster-view from our
-             * partition, we may update our cluster-view. For subsequent
-             * friends from that partition would agree with us, if the first
-             * friend wasn't rejected. For every first friend, whom we agreed with,
-             * we would need to start internal daemons/bricks belonging to the
-             * new volumes.
-             * glusterd_spawn_daemons calls functions that are idempotent. ie,
-             * the functions spawn process(es) only if they are not started yet.
-             *
-             * */
-                synclock_unlock (&priv->big_lock);
-                glusterd_launch_synctask (glusterd_spawn_daemons, NULL);
-                synclock_lock (&priv->big_lock);
-                glusterd_do_quorum_action ();
-        }
-        return ret;
+    if (quorum_action) {
+        /* When glusterd is restarted, it needs to wait until the 'friends' view
+         * of the volumes settle, before it starts any of the internal daemons.
+         *
+         * Every friend that was part of the cluster, would send its
+         * cluster-view, 'our' way. For every friend, who belongs to
+         * a partition which has a different cluster-view from our
+         * partition, we may update our cluster-view. For subsequent
+         * friends from that partition would agree with us, if the first
+         * friend wasn't rejected. For every first friend, whom we agreed with,
+         * we would need to start internal daemons/bricks belonging to the
+         * new volumes.
+         * glusterd_spawn_daemons calls functions that are idempotent. ie,
+         * the functions spawn process(es) only if they are not started yet.
+         *
+         * */
+        synclock_unlock(&priv->big_lock);
+        glusterd_launch_synctask(glusterd_spawn_daemons, NULL);
+        synclock_lock(&priv->big_lock);
+        glusterd_do_quorum_action();
+    }
+    return ret;
 }
 
-
 int
-glusterd_friend_sm_init ()
+glusterd_friend_sm_init()
 {
-        CDS_INIT_LIST_HEAD (&gd_friend_sm_queue);
-        return 0;
+    CDS_INIT_LIST_HEAD(&gd_friend_sm_queue);
+    return 0;
 }
diff --git a/xlators/mgmt/glusterd/src/glusterd-sm.h b/xlators/mgmt/glusterd/src/glusterd-sm.h
index be137802a64..11cbd85b3e3 100644
--- a/xlators/mgmt/glusterd/src/glusterd-sm.h
+++ b/xlators/mgmt/glusterd/src/glusterd-sm.h
@@ -10,215 +10,207 @@
 #ifndef _GLUSTERD_SM_H_
 #define _GLUSTERD_SM_H_
 
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
-
 #include <pthread.h>
-#include "compat-uuid.h"
+#include <glusterfs/compat-uuid.h>
 
 #include "rpc-clnt.h"
-#include "glusterfs.h"
-#include "xlator.h"
-#include "logging.h"
-#include "call-stub.h"
-#include "fd.h"
-#include "byte-order.h"
-//#include "glusterd.h"
+#include <glusterfs/glusterfs.h>
+#include <glusterfs/call-stub.h>
+#include <glusterfs/byte-order.h>
 #include "rpcsvc.h"
-#include "store.h"
+#include <glusterfs/store.h>
 
 #include "glusterd-rcu.h"
 
 typedef enum gd_quorum_contribution_ {
-        QUORUM_NONE,
-        QUORUM_WAITING,
-        QUORUM_DOWN,
-        QUORUM_UP
+    QUORUM_NONE,
+    QUORUM_WAITING,
+    QUORUM_DOWN,
+    QUORUM_UP
 } gd_quorum_contrib_t;
 
 typedef enum glusterd_friend_sm_state_ {
-        GD_FRIEND_STATE_DEFAULT = 0,
-        GD_FRIEND_STATE_REQ_SENT,
-        GD_FRIEND_STATE_REQ_RCVD,
-        GD_FRIEND_STATE_BEFRIENDED,
-        GD_FRIEND_STATE_REQ_ACCEPTED,
-        GD_FRIEND_STATE_REQ_SENT_RCVD,
-        GD_FRIEND_STATE_REJECTED,
-        GD_FRIEND_STATE_UNFRIEND_SENT,
-        GD_FRIEND_STATE_PROBE_RCVD,
-        GD_FRIEND_STATE_CONNECTED_RCVD,
-        GD_FRIEND_STATE_CONNECTED_ACCEPTED,
-        GD_FRIEND_STATE_MAX
+    GD_FRIEND_STATE_DEFAULT = 0,
+    GD_FRIEND_STATE_REQ_SENT,
+    GD_FRIEND_STATE_REQ_RCVD,
+    GD_FRIEND_STATE_BEFRIENDED,
+    GD_FRIEND_STATE_REQ_ACCEPTED,
+    GD_FRIEND_STATE_REQ_SENT_RCVD,
+    GD_FRIEND_STATE_REJECTED,
+    GD_FRIEND_STATE_UNFRIEND_SENT,
+    GD_FRIEND_STATE_PROBE_RCVD,
+    GD_FRIEND_STATE_CONNECTED_RCVD,
+    GD_FRIEND_STATE_CONNECTED_ACCEPTED,
+    GD_FRIEND_STATE_MAX
 } glusterd_friend_sm_state_t;
 
 typedef struct glusterd_peer_state_info_ {
-        glusterd_friend_sm_state_t   state;
-        struct timeval          transition_time;
-}glusterd_peer_state_info_t;
+    glusterd_friend_sm_state_t state;
+    struct timeval transition_time;
+} glusterd_peer_state_info_t;
 
 typedef struct glusterd_peer_hostname_ {
-        char                    *hostname;
-        struct cds_list_head     hostname_list;
+    char *hostname;
+    struct cds_list_head hostname_list;
 } glusterd_peer_hostname_t;
 
 typedef struct glusterd_sm_transition_ {
-        int             old_state;
-        int             event;
-        int             new_state;
-        time_t          time;
+    int old_state;
+    int event;
+    int new_state;
+    time_t time;
 } glusterd_sm_transition_t;
 
 typedef struct glusterd_sm_tr_log_ {
-        glusterd_sm_transition_t    *transitions;
-        size_t                      current;
-        size_t                      size;
-        size_t                      count;
-        char*                       (*state_name_get) (int);
-        char*                       (*event_name_get) (int);
+    glusterd_sm_transition_t *transitions;
+    size_t current;
+    size_t size;
+    size_t count;
+    char *(*state_name_get)(int);
+    char *(*event_name_get)(int);
 } glusterd_sm_tr_log_t;
 
 struct glusterd_peerinfo_ {
-        uuid_t                          uuid;
-        char                            uuid_str[50]; /* Retrieve this using
-                                                       * gd_peer_uuid_str ()
-                                                       */
-        glusterd_peer_state_info_t      state;
-        char                            *hostname;
-        struct cds_list_head            hostnames;
-        int                             port;
-        struct cds_list_head            uuid_list;
-        struct cds_list_head            op_peers_list;
-        struct rpc_clnt                 *rpc;
-        rpc_clnt_prog_t                 *mgmt;
-        rpc_clnt_prog_t                 *peer;
-        rpc_clnt_prog_t                 *mgmt_v3;
-        int                             connected;
-        gf_store_handle_t               *shandle;
-        glusterd_sm_tr_log_t            sm_log;
-        gf_boolean_t                    quorum_action;
-        gd_quorum_contrib_t             quorum_contrib;
-        gf_boolean_t                    locked;
-        gf_boolean_t                    detaching;
-        /* Members required for proper cleanup using RCU */
-        gd_rcu_head                     rcu_head;
-        pthread_mutex_t                 delete_lock;
+    uuid_t uuid;
+    char uuid_str[50]; /* Retrieve this using
+                        * gd_peer_uuid_str ()
+                        */
+    glusterd_peer_state_info_t state;
+    char *hostname;
+    struct cds_list_head hostnames;
+    int port;
+    struct cds_list_head uuid_list;
+    struct cds_list_head op_peers_list;
+    struct rpc_clnt *rpc;
+    rpc_clnt_prog_t *mgmt;
+    rpc_clnt_prog_t *peer;
+    rpc_clnt_prog_t *mgmt_v3;
+    int connected;
+    gf_store_handle_t *shandle;
+    glusterd_sm_tr_log_t sm_log;
+    gf_boolean_t quorum_action;
+    gd_quorum_contrib_t quorum_contrib;
+    gf_boolean_t locked;
+    gf_boolean_t detaching;
+    /* Members required for proper cleanup using RCU */
+    gd_rcu_head rcu_head;
+    pthread_mutex_t delete_lock;
+    uint32_t generation;
 };
 
 typedef struct glusterd_peerinfo_ glusterd_peerinfo_t;
 
 typedef struct glusterd_local_peers_ {
-        glusterd_peerinfo_t   *peerinfo;
-        struct cds_list_head  op_peers_list;
+    glusterd_peerinfo_t *peerinfo;
+    struct cds_list_head op_peers_list;
 } glusterd_local_peers_t;
 
 typedef enum glusterd_ev_gen_mode_ {
-        GD_MODE_OFF,
-        GD_MODE_ON,
-        GD_MODE_SWITCH_ON
+    GD_MODE_OFF,
+    GD_MODE_ON,
+    GD_MODE_SWITCH_ON
 } glusterd_ev_gen_mode_t;
 
 typedef struct glusterd_peer_ctx_args_ {
-        rpcsvc_request_t        *req;
-        glusterd_ev_gen_mode_t  mode;
-        dict_t                  *dict;
+    rpcsvc_request_t *req;
+    glusterd_ev_gen_mode_t mode;
+    dict_t *dict;
 } glusterd_peerctx_args_t;
 
 typedef struct glusterd_peer_ctx_ {
-        glusterd_peerctx_args_t        args;
-        uuid_t                         peerid;
-        char                           *peername;
-        char                           *errstr;
+    glusterd_peerctx_args_t args;
+    uuid_t peerid;
+    char *peername;
+    uint32_t peerinfo_gen;
+    char *errstr;
 } glusterd_peerctx_t;
 
 typedef enum glusterd_friend_sm_event_type_ {
-        GD_FRIEND_EVENT_NONE = 0,
-        GD_FRIEND_EVENT_PROBE,
-        GD_FRIEND_EVENT_INIT_FRIEND_REQ,
-        GD_FRIEND_EVENT_RCVD_ACC,
-        GD_FRIEND_EVENT_LOCAL_ACC,
-        GD_FRIEND_EVENT_RCVD_RJT,
-        GD_FRIEND_EVENT_LOCAL_RJT,
-        GD_FRIEND_EVENT_RCVD_FRIEND_REQ,
-        GD_FRIEND_EVENT_INIT_REMOVE_FRIEND,
-        GD_FRIEND_EVENT_RCVD_REMOVE_FRIEND,
-        GD_FRIEND_EVENT_REMOVE_FRIEND,
-        GD_FRIEND_EVENT_CONNECTED,
-        GD_FRIEND_EVENT_MAX
+    GD_FRIEND_EVENT_NONE = 0,
+    GD_FRIEND_EVENT_PROBE,
+    GD_FRIEND_EVENT_INIT_FRIEND_REQ,
+    GD_FRIEND_EVENT_RCVD_ACC,
+    GD_FRIEND_EVENT_LOCAL_ACC,
+    GD_FRIEND_EVENT_RCVD_RJT,
+    GD_FRIEND_EVENT_LOCAL_RJT,
+    GD_FRIEND_EVENT_RCVD_FRIEND_REQ,
+    GD_FRIEND_EVENT_INIT_REMOVE_FRIEND,
+    GD_FRIEND_EVENT_RCVD_REMOVE_FRIEND,
+    GD_FRIEND_EVENT_REMOVE_FRIEND,
+    GD_FRIEND_EVENT_CONNECTED,
+    GD_FRIEND_EVENT_NEW_NAME,
+    GD_FRIEND_EVENT_MAX
 } glusterd_friend_sm_event_type_t;
 
-
 typedef enum glusterd_friend_update_op_ {
-        GD_FRIEND_UPDATE_NONE = 0,
-        GD_FRIEND_UPDATE_ADD,
-        GD_FRIEND_UPDATE_DEL,
+    GD_FRIEND_UPDATE_NONE = 0,
+    GD_FRIEND_UPDATE_ADD,
+    GD_FRIEND_UPDATE_DEL,
 } glusterd_friend_update_op_t;
 
-
 struct glusterd_friend_sm_event_ {
-        struct cds_list_head             list;
-        uuid_t                           peerid;
-        char                            *peername;
-        void                            *ctx;
-        glusterd_friend_sm_event_type_t  event;
+    struct cds_list_head list;
+    uuid_t peerid;
+    char *peername;
+    void *ctx;
+    glusterd_friend_sm_event_type_t event;
 };
 
 typedef struct glusterd_friend_sm_event_ glusterd_friend_sm_event_t;
 
-typedef int (*glusterd_friend_sm_ac_fn) (glusterd_friend_sm_event_t *, void *);
+typedef int (*glusterd_friend_sm_ac_fn)(glusterd_friend_sm_event_t *, void *);
 
 typedef struct glusterd_sm_ {
-        glusterd_friend_sm_state_t      next_state;
-        glusterd_friend_sm_ac_fn        handler;
+    glusterd_friend_sm_state_t next_state;
+    glusterd_friend_sm_ac_fn handler;
 } glusterd_sm_t;
 
 typedef struct glusterd_friend_req_ctx_ {
-        uuid_t                  uuid;
-        char                    *hostname;
-        rpcsvc_request_t        *req;
-        int                      port;
-        dict_t                  *vols;
+    uuid_t uuid;
+    char *hostname;
+    rpcsvc_request_t *req;
+    int port;
+    dict_t *vols;
 } glusterd_friend_req_ctx_t;
 
 typedef struct glusterd_friend_update_ctx_ {
-        uuid_t                  uuid;
-        char                    *hostname;
-        int                     op;
+    uuid_t uuid;
+    char *hostname;
+    int op;
 } glusterd_friend_update_ctx_t;
 
 typedef struct glusterd_probe_ctx_ {
-        char                    *hostname;
-        rpcsvc_request_t        *req;
-        int                      port;
-        dict_t                  *dict;
+    char *hostname;
+    rpcsvc_request_t *req;
+    int port;
+    dict_t *dict;
 } glusterd_probe_ctx_t;
 int
-glusterd_friend_sm_new_event (glusterd_friend_sm_event_type_t event_type,
-                              glusterd_friend_sm_event_t **new_event);
+glusterd_friend_sm_new_event(glusterd_friend_sm_event_type_t event_type,
+                             glusterd_friend_sm_event_t **new_event);
 int
-glusterd_friend_sm_inject_event (glusterd_friend_sm_event_t *event);
+glusterd_friend_sm_inject_event(glusterd_friend_sm_event_t *event);
 
 int
-glusterd_friend_sm_init ();
+glusterd_friend_sm_init();
 
 int
-glusterd_friend_sm ();
+glusterd_friend_sm();
 
 void
-glusterd_destroy_probe_ctx (glusterd_probe_ctx_t *ctx);
+glusterd_destroy_probe_ctx(glusterd_probe_ctx_t *ctx);
 
 void
-glusterd_destroy_friend_req_ctx (glusterd_friend_req_ctx_t *ctx);
+glusterd_destroy_friend_req_ctx(glusterd_friend_req_ctx_t *ctx);
 
-char*
-glusterd_friend_sm_state_name_get (int state);
+char *
+glusterd_friend_sm_state_name_get(int state);
 
-char*
-glusterd_friend_sm_event_name_get (int event);
+char *
+glusterd_friend_sm_event_name_get(int event);
 
 int
-glusterd_broadcast_friend_delete (char *hostname, uuid_t uuid);
+glusterd_broadcast_friend_delete(char *hostname, uuid_t uuid);
 void
-glusterd_destroy_friend_update_ctx (glusterd_friend_update_ctx_t *ctx);
+glusterd_destroy_friend_update_ctx(glusterd_friend_update_ctx_t *ctx);
 #endif
diff --git a/xlators/mgmt/glusterd/src/glusterd-snapd-svc-helper.c b/xlators/mgmt/glusterd/src/glusterd-snapd-svc-helper.c
index 826b4ca7463..42ef51b01b4 100644
--- a/xlators/mgmt/glusterd/src/glusterd-snapd-svc-helper.c
+++ b/xlators/mgmt/glusterd/src/glusterd-snapd-svc-helper.c
@@ -13,51 +13,63 @@
 #include "glusterd-snapd-svc-helper.h"
 
 void
-glusterd_svc_build_snapd_rundir (glusterd_volinfo_t *volinfo,
-                                 char *path, int path_len)
+glusterd_svc_build_snapd_rundir(glusterd_volinfo_t *volinfo, char *path,
+                                int path_len)
 {
-        char                    workdir[PATH_MAX]      = {0,};
-        glusterd_conf_t        *priv                   = THIS->private;
+    char workdir[PATH_MAX] = {
+        0,
+    };
+    glusterd_conf_t *priv = THIS->private;
 
-        GLUSTERD_GET_VOLUME_DIR (workdir, volinfo, priv);
-
-        snprintf (path, path_len, "%s/run", workdir);
+    GLUSTERD_GET_VOLUME_PID_DIR(workdir, volinfo, priv);
+    snprintf(path, path_len, "%s", workdir);
 }
 
 void
-glusterd_svc_build_snapd_socket_filepath (glusterd_volinfo_t *volinfo,
-                                          char *path, int path_len)
+glusterd_svc_build_snapd_socket_filepath(glusterd_volinfo_t *volinfo,
+                                         char *path, int path_len)
 {
-        char                    sockfilepath[PATH_MAX] = {0,};
-        char                    rundir[PATH_MAX]       = {0,};
+    char sockfilepath[PATH_MAX] = {
+        0,
+    };
+    char rundir[PATH_MAX] = {
+        0,
+    };
+    int32_t len = 0;
 
-        glusterd_svc_build_snapd_rundir (volinfo, rundir, sizeof (rundir));
-        snprintf (sockfilepath, sizeof (sockfilepath), "%s/run-%s",
-                  rundir, uuid_utoa (MY_UUID));
+    glusterd_svc_build_snapd_rundir(volinfo, rundir, sizeof(rundir));
+    len = snprintf(sockfilepath, sizeof(sockfilepath), "%s/run-%s", rundir,
+                   uuid_utoa(MY_UUID));
+    if ((len < 0) || (len >= sizeof(sockfilepath))) {
+        sockfilepath[0] = 0;
+    }
 
-        glusterd_set_socket_filepath (sockfilepath, path, path_len);
+    glusterd_set_socket_filepath(sockfilepath, path, path_len);
 }
 
 void
-glusterd_svc_build_snapd_pidfile (glusterd_volinfo_t *volinfo,
-                                  char *path, int path_len)
+glusterd_svc_build_snapd_pidfile(glusterd_volinfo_t *volinfo, char *path,
+                                 int path_len)
 {
-        char            rundir[PATH_MAX]      = {0,};
+    char rundir[PATH_MAX] = {
+        0,
+    };
 
-        glusterd_svc_build_snapd_rundir (volinfo, rundir, sizeof (rundir));
+    glusterd_svc_build_snapd_rundir(volinfo, rundir, sizeof(rundir));
 
-        snprintf (path, path_len, "%s/%s-snapd.pid", rundir, volinfo->volname);
+    snprintf(path, path_len, "%s/%s-snapd.pid", rundir, volinfo->volname);
 }
 
 void
-glusterd_svc_build_snapd_volfile (glusterd_volinfo_t *volinfo,
-                                  char *path, int path_len)
+glusterd_svc_build_snapd_volfile(glusterd_volinfo_t *volinfo, char *path,
+                                 int path_len)
 {
-        char                    workdir[PATH_MAX]      = {0,};
-        glusterd_conf_t        *priv                   = THIS->private;
+    char workdir[PATH_MAX] = {
+        0,
+    };
+    glusterd_conf_t *priv = THIS->private;
 
-        GLUSTERD_GET_VOLUME_DIR (workdir, volinfo, priv);
+    GLUSTERD_GET_VOLUME_DIR(workdir, volinfo, priv);
 
-        snprintf (path, path_len, "%s/%s-snapd.vol", workdir,
-                  volinfo->volname);
+    snprintf(path, path_len, "%s/%s-snapd.vol", workdir, volinfo->volname);
 }
diff --git a/xlators/mgmt/glusterd/src/glusterd-snapd-svc-helper.h b/xlators/mgmt/glusterd/src/glusterd-snapd-svc-helper.h
index 5c15e1a82c7..3e23c2ce942 100644
--- a/xlators/mgmt/glusterd/src/glusterd-snapd-svc-helper.h
+++ b/xlators/mgmt/glusterd/src/glusterd-snapd-svc-helper.h
@@ -11,27 +11,22 @@
 #ifndef _GLUSTERD_SNAPD_SVC_HELPER_H_
 #define _GLUSTERD_SNAPD_SVC_HELPER_H_
 
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
-
 #include "glusterd.h"
 
 void
-glusterd_svc_build_snapd_rundir (glusterd_volinfo_t *volinfo,
-                                 char *path, int path_len);
+glusterd_svc_build_snapd_rundir(glusterd_volinfo_t *volinfo, char *path,
+                                int path_len);
 
 void
-glusterd_svc_build_snapd_socket_filepath (glusterd_volinfo_t *volinfo,
-                                          char *path, int path_len);
+glusterd_svc_build_snapd_socket_filepath(glusterd_volinfo_t *volinfo,
+                                         char *path, int path_len);
 
 void
-glusterd_svc_build_snapd_pidfile (glusterd_volinfo_t *volinfo,
-                                  char *path, int path_len);
+glusterd_svc_build_snapd_pidfile(glusterd_volinfo_t *volinfo, char *path,
+                                 int path_len);
 
 void
-glusterd_svc_build_snapd_volfile (glusterd_volinfo_t *volinfo,
-                                  char *path, int path_len);
+glusterd_svc_build_snapd_volfile(glusterd_volinfo_t *volinfo, char *path,
+                                 int path_len);
 
 #endif
diff --git a/xlators/mgmt/glusterd/src/glusterd-snapd-svc.c b/xlators/mgmt/glusterd/src/glusterd-snapd-svc.c
index d1a7da7e0ae..d75f249b29e 100644
--- a/xlators/mgmt/glusterd/src/glusterd-snapd-svc.c
+++ b/xlators/mgmt/glusterd/src/glusterd-snapd-svc.c
@@ -8,8 +8,8 @@
    cases as published by the Free Software Foundation.
 */
 
-#include "globals.h"
-#include "run.h"
+#include <glusterfs/globals.h>
+#include <glusterfs/run.h>
 #include "glusterd-utils.h"
 #include "glusterd-volgen.h"
 #include "glusterd-messages.h"
@@ -20,388 +20,459 @@
 #include "glusterd-snapd-svc.h"
 #include "glusterd-snapd-svc-helper.h"
 #include "glusterd-snapshot-utils.h"
+#include <glusterfs/syscall.h>
 
 char *snapd_svc_name = "snapd";
 
 static void
-glusterd_svc_build_snapd_logdir (char *logdir, char *volname, size_t len)
+glusterd_svc_build_snapd_logdir(char *logdir, char *volname, size_t len)
 {
-        snprintf (logdir, len, "%s/snaps/%s", DEFAULT_LOG_FILE_DIRECTORY,
-                  volname);
+    glusterd_conf_t *priv = THIS->private;
+    snprintf(logdir, len, "%s/snaps/%s", priv->logdir, volname);
 }
 
 static void
-glusterd_svc_build_snapd_logfile (char *logfile, char *logdir, size_t len)
+glusterd_svc_build_snapd_logfile(char *logfile, char *logdir, size_t len)
 {
-        snprintf (logfile, len, "%s/snapd.log", logdir);
+    snprintf(logfile, len, "%s/snapd.log", logdir);
 }
 
+void
+glusterd_snapdsvc_build(glusterd_svc_t *svc)
+{
+    svc->manager = glusterd_snapdsvc_manager;
+    svc->start = glusterd_snapdsvc_start;
+    svc->stop = glusterd_svc_stop;
+}
 
 int
-glusterd_snapdsvc_init (void *data)
+glusterd_snapdsvc_init(void *data)
 {
-        int                     ret                = -1;
-        char                    rundir[PATH_MAX]   = {0,};
-        char                    sockpath[PATH_MAX] = {0,};
-        char                    pidfile[PATH_MAX]  = {0,};
-        char                    volfile[PATH_MAX]  = {0,};
-        char                    logdir[PATH_MAX]   = {0,};
-        char                    logfile[PATH_MAX]  = {0,};
-        char                    volfileid[256]     = {0};
-        glusterd_svc_t         *svc                = NULL;
-        glusterd_volinfo_t     *volinfo            = NULL;
-        glusterd_conf_t        *priv               = NULL;
-        glusterd_conn_notify_t  notify             = NULL;
-        xlator_t               *this               = NULL;
-        char                    *volfileserver     = NULL;
-
-        this = THIS;
-        GF_ASSERT (this);
-
-        priv = this->private;
-        GF_ASSERT (priv);
-
-        volinfo = data;
-
-        svc = &(volinfo->snapd.svc);
-
-        ret = snprintf (svc->name, sizeof (svc->name), "%s", snapd_svc_name);
-        if (ret < 0)
-                goto out;
+    int ret = -1;
+    char rundir[PATH_MAX] = {
+        0,
+    };
+    char sockpath[PATH_MAX] = {
+        0,
+    };
+    char pidfile[PATH_MAX] = {
+        0,
+    };
+    char volfile[PATH_MAX] = {
+        0,
+    };
+    char logdir[PATH_MAX] = {
+        0,
+    };
+    char logfile[PATH_MAX] = {
+        0,
+    };
+    char volfileid[256] = {0};
+    glusterd_svc_t *svc = NULL;
+    glusterd_volinfo_t *volinfo = NULL;
+    glusterd_conf_t *priv = NULL;
+    glusterd_conn_notify_t notify = NULL;
+    xlator_t *this = NULL;
+    char *volfileserver = NULL;
+    int32_t len = 0;
+
+    this = THIS;
+    GF_ASSERT(this);
+
+    priv = this->private;
+    GF_ASSERT(priv);
+
+    volinfo = data;
+
+    svc = &(volinfo->snapd.svc);
+
+    ret = snprintf(svc->name, sizeof(svc->name), "%s", snapd_svc_name);
+    if (ret < 0) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_COPY_FAIL, NULL);
+        goto out;
+    }
+
+    notify = glusterd_snapdsvc_rpc_notify;
+
+    glusterd_svc_build_snapd_rundir(volinfo, rundir, sizeof(rundir));
+    glusterd_svc_create_rundir(rundir);
+
+    /* Initialize the connection mgmt */
+    glusterd_svc_build_snapd_socket_filepath(volinfo, sockpath,
+                                             sizeof(sockpath));
+    ret = glusterd_conn_init(&(svc->conn), sockpath, 600, notify);
+    if (ret)
+        goto out;
+
+    /* Initialize the process mgmt */
+    glusterd_svc_build_snapd_pidfile(volinfo, pidfile, sizeof(pidfile));
+    glusterd_svc_build_snapd_volfile(volinfo, volfile, sizeof(volfile));
+    glusterd_svc_build_snapd_logdir(logdir, volinfo->volname, sizeof(logdir));
+    ret = mkdir_p(logdir, 0755, _gf_true);
+    if ((ret == -1) && (EEXIST != errno)) {
+        gf_msg(this->name, GF_LOG_ERROR, errno, GD_MSG_CREATE_DIR_FAILED,
+               "Unable to create logdir %s", logdir);
+        goto out;
+    }
+    glusterd_svc_build_snapd_logfile(logfile, logdir, sizeof(logfile));
+    len = snprintf(volfileid, sizeof(volfileid), "snapd/%s", volinfo->volname);
+    if ((len < 0) || (len >= sizeof(volfileid))) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_COPY_FAIL, NULL);
+        ret = -1;
+        goto out;
+    }
+
+    if (dict_get_str(this->options, "transport.socket.bind-address",
+                     &volfileserver) != 0) {
+        volfileserver = "localhost";
+    }
+    ret = glusterd_proc_init(&(svc->proc), snapd_svc_name, pidfile, logdir,
+                             logfile, volfile, volfileid, volfileserver);
+    if (ret)
+        goto out;
 
-        svc->manager = glusterd_snapdsvc_manager;
-        svc->start = glusterd_snapdsvc_start;
-        svc->stop = glusterd_svc_stop;
-
-        notify = glusterd_snapdsvc_rpc_notify;
+out:
+    gf_msg_debug(this->name, 0, "Returning %d", ret);
+    return ret;
+}
 
-        glusterd_svc_build_snapd_rundir (volinfo, rundir, sizeof (rundir));
-        glusterd_svc_create_rundir (rundir);
+int
+glusterd_snapdsvc_manager(glusterd_svc_t *svc, void *data, int flags)
+{
+    int ret = 0;
+    xlator_t *this = THIS;
+    glusterd_volinfo_t *volinfo = NULL;
 
-        /* Initialize the connection mgmt */
-        glusterd_svc_build_snapd_socket_filepath (volinfo, sockpath,
-                                                  sizeof (sockpath));
-        ret = glusterd_conn_init (&(svc->conn), sockpath, 600, notify);
-        if (ret)
-                goto out;
+    volinfo = data;
 
-        /* Initialize the process mgmt */
-        glusterd_svc_build_snapd_pidfile (volinfo, pidfile, sizeof (pidfile));
-        glusterd_svc_build_snapd_volfile (volinfo, volfile, sizeof (volfile));
-        glusterd_svc_build_snapd_logdir (logdir, volinfo->volname,
-                                         sizeof (logdir));
-        ret = mkdir_p (logdir, 0755, _gf_true);
-        if ((ret == -1) && (EEXIST != errno)) {
-                gf_log (this->name, GF_LOG_ERROR, "Unable to create logdir %s",
-                        logdir);
-                goto out;
+    if (!svc->inited) {
+        ret = glusterd_snapdsvc_init(volinfo);
+        if (ret) {
+            gf_msg(THIS->name, GF_LOG_ERROR, 0, GD_MSG_SNAPD_INIT_FAIL,
+                   "Failed to initialize "
+                   "snapd service for volume %s",
+                   volinfo->volname);
+            goto out;
+        } else {
+            svc->inited = _gf_true;
+            gf_msg_debug(THIS->name, 0,
+                         "snapd service "
+                         "initialized");
         }
-        glusterd_svc_build_snapd_logfile (logfile, logdir, sizeof (logfile));
-        snprintf (volfileid, sizeof (volfileid), "snapd/%s", volinfo->volname);
-
-        if (dict_get_str (this->options, "transport.socket.bind-address",
-                          &volfileserver) != 0) {
-                volfileserver = "localhost";
+    }
+
+    ret = glusterd_is_snapd_enabled(volinfo);
+    if (ret == -1) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_VOLINFO_GET_FAIL,
+               "Failed to read volume "
+               "options");
+        goto out;
+    }
+
+    if (ret) {
+        if (!glusterd_is_volume_started(volinfo)) {
+            if (glusterd_proc_is_running(&svc->proc)) {
+                ret = svc->stop(svc, SIGTERM);
+                if (ret)
+                    gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_SNAPD_STOP_FAIL,
+                           "Couldn't stop snapd for "
+                           "volume: %s",
+                           volinfo->volname);
+            } else {
+                /* Since snapd is not running set ret to 0 */
+                ret = 0;
+            }
+            goto out;
         }
-        ret = glusterd_proc_init (&(svc->proc), snapd_svc_name, pidfile, logdir,
-                                  logfile, volfile, volfileid, volfileserver);
-        if (ret)
-                goto out;
 
-out:
-        gf_log (this->name, GF_LOG_DEBUG, "Returning %d", ret);
-        return ret;
-}
-
-int
-glusterd_snapdsvc_manager (glusterd_svc_t *svc, void *data, int flags)
-{
-        int                 ret     = 0;
-        xlator_t           *this    = THIS;
-        glusterd_volinfo_t *volinfo = NULL;
+        ret = glusterd_snapdsvc_create_volfile(volinfo);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_SNAPD_CREATE_FAIL,
+                   "Couldn't create "
+                   "snapd volfile for volume: %s",
+                   volinfo->volname);
+            goto out;
+        }
 
-        volinfo = data;
+        ret = svc->start(svc, flags);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_SNAPD_START_FAIL,
+                   "Couldn't start "
+                   "snapd for volume: %s",
+                   volinfo->volname);
+            goto out;
+        }
 
-        ret = glusterd_is_snapd_enabled (volinfo);
-        if (ret == -1) {
-                gf_log (this->name, GF_LOG_ERROR, "Failed to read volume "
-                        "options");
-                goto out;
+        glusterd_volinfo_ref(volinfo);
+        ret = glusterd_conn_connect(&(svc->conn));
+        if (ret) {
+            glusterd_volinfo_unref(volinfo);
+            goto out;
         }
 
+    } else if (glusterd_proc_is_running(&svc->proc)) {
+        ret = svc->stop(svc, SIGTERM);
         if (ret) {
-                if (!glusterd_is_volume_started (volinfo)) {
-                        if (glusterd_proc_is_running (&svc->proc)) {
-                                ret = svc->stop (svc, SIGTERM);
-                                if (ret)
-                                        gf_log (this->name, GF_LOG_ERROR,
-                                                "Couldn't stop snapd for "
-                                                "volume: %s",
-                                                volinfo->volname);
-                        } else {
-                                /* Since snapd is not running set ret to 0 */
-                                ret = 0;
-                        }
-                        goto out;
-                }
-
-                ret = glusterd_snapdsvc_create_volfile (volinfo);
-                if (ret) {
-                        gf_log (this->name, GF_LOG_ERROR, "Couldn't create "
-                                "snapd volfile for volume: %s",
-                                volinfo->volname);
-                        goto out;
-                }
-
-                ret = svc->start (svc, flags);
-                if (ret) {
-                        gf_log (this->name, GF_LOG_ERROR, "Couldn't start "
-                                "snapd for volume: %s", volinfo->volname);
-                        goto out;
-                }
-
-                glusterd_volinfo_ref (volinfo);
-                ret = glusterd_conn_connect (&(svc->conn));
-                if (ret) {
-                        glusterd_volinfo_unref (volinfo);
-                        goto out;
-                }
-
-        } else if (glusterd_proc_is_running (&svc->proc)) {
-                ret = svc->stop (svc, SIGTERM);
-                if (ret) {
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "Couldn't stop snapd for volume: %s",
-                                volinfo->volname);
-                        goto out;
-                }
-                volinfo->snapd.port = 0;
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_SNAPD_STOP_FAIL,
+                   "Couldn't stop snapd for volume: %s", volinfo->volname);
+            goto out;
         }
+        volinfo->snapd.port = 0;
+    }
 
 out:
-        gf_log (THIS->name, GF_LOG_DEBUG, "Returning %d", ret);
+    if (ret) {
+        gf_event(EVENT_SVC_MANAGER_FAILED, "volume=%s;svc_name=%s",
+                 volinfo->volname, svc->name);
+    }
+    gf_msg_debug(THIS->name, 0, "Returning %d", ret);
 
-        return ret;
+    return ret;
 }
 
 int32_t
-glusterd_snapdsvc_start (glusterd_svc_t *svc, int flags)
+glusterd_snapdsvc_start(glusterd_svc_t *svc, int flags)
 {
-        int                  ret                        = -1;
-        runner_t             runner                     = {0,};
-        glusterd_conf_t     *priv                       = NULL;
-        xlator_t            *this                       = NULL;
-        char                 valgrind_logfile[PATH_MAX] = {0};
-        int                  snapd_port                 = 0;
-        char                 msg[1024]                  = {0,};
-        char                 snapd_id[PATH_MAX]         = {0,};
-        glusterd_volinfo_t  *volinfo                    = NULL;
-        glusterd_snapdsvc_t *snapd                      = NULL;
-
-        this = THIS;
-        GF_ASSERT(this);
-
-        priv = this->private;
-        GF_ASSERT (priv);
-
-        if (glusterd_proc_is_running (&svc->proc)) {
-                ret = 0;
-                goto out;
-        }
-
-        /* Get volinfo->snapd from svc object */
-        snapd = cds_list_entry (svc, glusterd_snapdsvc_t, svc);
-        if (!snapd) {
-                gf_log (this->name, GF_LOG_ERROR, "Failed to get snapd object "
-                        "from snapd service");
-                goto out;
-        }
-
-        /* Get volinfo from snapd */
-        volinfo = cds_list_entry (snapd, glusterd_volinfo_t, snapd);
-        if (!volinfo) {
-                gf_log (this->name, GF_LOG_ERROR, "Failed to get volinfo from "
-                        "from snapd");
-                goto out;
-        }
-
-        ret = access (svc->proc.volfile, F_OK);
+    int ret = -1;
+    runner_t runner = {
+        0,
+    };
+    glusterd_conf_t *priv = NULL;
+    xlator_t *this = NULL;
+    char valgrind_logfile[PATH_MAX] = {0};
+    int snapd_port = 0;
+    char msg[1024] = {
+        0,
+    };
+    char snapd_id[PATH_MAX] = {
+        0,
+    };
+    glusterd_volinfo_t *volinfo = NULL;
+    glusterd_snapdsvc_t *snapd = NULL;
+    char *localtime_logging = NULL;
+    int32_t len = 0;
+
+    this = THIS;
+    GF_ASSERT(this);
+
+    priv = this->private;
+    GF_ASSERT(priv);
+
+    if (glusterd_proc_is_running(&svc->proc)) {
+        ret = 0;
+        goto out;
+    }
+
+    /* Get volinfo->snapd from svc object */
+    snapd = cds_list_entry(svc, glusterd_snapdsvc_t, svc);
+    if (!snapd) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_SNAPD_OBJ_GET_FAIL,
+               "Failed to get snapd object "
+               "from snapd service");
+        goto out;
+    }
+
+    /* Get volinfo from snapd */
+    volinfo = cds_list_entry(snapd, glusterd_volinfo_t, snapd);
+    if (!volinfo) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_VOLINFO_GET_FAIL,
+               "Failed to get volinfo from "
+               "from snapd");
+        goto out;
+    }
+
+    ret = sys_access(svc->proc.volfile, F_OK);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_DEBUG, 0, GD_MSG_VOLINFO_GET_FAIL,
+               "snapd Volfile %s is not present", svc->proc.volfile);
+        /* If glusterd is down on one of the nodes and during
+         * that time "USS is enabled" for the first time. After some
+         * time when the glusterd which was down comes back it tries
+         * to look for the snapd volfile and it does not find snapd
+         * volfile and because of this starting of snapd fails.
+         * Therefore, if volfile is not present then create a fresh
+         * volfile.
+         */
+        ret = glusterd_snapdsvc_create_volfile(volinfo);
         if (ret) {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "snapd Volfile %s is not present", svc->proc.volfile);
-                /* If glusterd is down on one of the nodes and during
-                 * that time "USS is enabled" for the first time. After some
-                 * time when the glusterd which was down comes back it tries
-                 * to look for the snapd volfile and it does not find snapd
-                 * volfile and because of this starting of snapd fails.
-                 * Therefore, if volfile is not present then create a fresh
-                 * volfile.
-                 */
-                ret = glusterd_snapdsvc_create_volfile (volinfo);
-                if (ret) {
-                        gf_log (this->name, GF_LOG_ERROR, "Couldn't create "
-                                "snapd volfile for volume: %s",
-                                volinfo->volname);
-                        goto out;
-                }
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_VOLFILE_CREATE_FAIL,
+                   "Couldn't create "
+                   "snapd volfile for volume: %s",
+                   volinfo->volname);
+            goto out;
         }
-        runinit (&runner);
-
-        if (priv->valgrind) {
-                snprintf (valgrind_logfile, PATH_MAX, "%s/valgrind-snapd.log",
-                          svc->proc.logdir);
-
-                runner_add_args (&runner, "valgrind", "--leak-check=full",
-                                 "--trace-children=yes", "--track-origins=yes",
-                                 NULL);
-                runner_argprintf (&runner, "--log-file=%s", valgrind_logfile);
+    }
+    runinit(&runner);
+
+    if (this->ctx->cmd_args.vgtool != _gf_none) {
+        len = snprintf(valgrind_logfile, PATH_MAX, "%s/valgrind-snapd.log",
+                       svc->proc.logdir);
+        if ((len < 0) || (len >= PATH_MAX)) {
+            gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_COPY_FAIL, NULL);
+            ret = -1;
+            goto out;
         }
 
-        snprintf (snapd_id, sizeof (snapd_id), "snapd-%s", volinfo->volname);
-        runner_add_args (&runner, SBIN_DIR"/glusterfsd",
-                         "-s", svc->proc.volfileserver,
-                         "--volfile-id", svc->proc.volfileid,
-                         "-p", svc->proc.pidfile,
-                         "-l", svc->proc.logfile,
-                         "--brick-name", snapd_id,
-                         "-S", svc->conn.sockpath, NULL);
-
-        snapd_port = volinfo->snapd.port;
-        if (!snapd_port) {
-                snapd_port = pmap_registry_alloc (THIS);
-                if (!snapd_port) {
-                        snprintf (msg, sizeof (msg), "Could not allocate port "
-                                  "for snapd service for volume %s",
-                                  volinfo->volname);
-                        runner_log (&runner, this->name, GF_LOG_DEBUG, msg);
-                        ret = -1;
-                        goto out;
-                }
+        if (this->ctx->cmd_args.vgtool == _gf_memcheck)
+            runner_add_args(&runner, "valgrind", "--leak-check=full",
+                            "--trace-children=yes", "--track-origins=yes",
+                            NULL);
+        else
+            runner_add_args(&runner, "valgrind", "--tool=drd", NULL);
+
+        runner_argprintf(&runner, "--log-file=%s", valgrind_logfile);
+    }
+
+    snprintf(snapd_id, sizeof(snapd_id), "snapd-%s", volinfo->volname);
+    runner_add_args(&runner, SBIN_DIR "/glusterfsd", "-s",
+                    svc->proc.volfileserver, "--volfile-id",
+                    svc->proc.volfileid, "-p", svc->proc.pidfile, "-l",
+                    svc->proc.logfile, "--brick-name", snapd_id, "-S",
+                    svc->conn.sockpath, "--process-name", svc->name, NULL);
+    if (dict_get_str(priv->opts, GLUSTERD_LOCALTIME_LOGGING_KEY,
+                     &localtime_logging) == 0) {
+        if (strcmp(localtime_logging, "enable") == 0)
+            runner_add_arg(&runner, "--localtime-logging");
+    }
+
+    snapd_port = pmap_assign_port(THIS, volinfo->snapd.port, snapd_id);
+    if (!snapd_port) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_PORTS_EXHAUSTED,
+               "All the ports in the range are exhausted, can't start "
+               "snapd for volume %s",
+               volinfo->volname);
+        ret = -1;
+        goto out;
+    }
+
+    volinfo->snapd.port = snapd_port;
+
+    runner_add_arg(&runner, "--brick-port");
+    runner_argprintf(&runner, "%d", snapd_port);
+    runner_add_arg(&runner, "--xlator-option");
+    runner_argprintf(&runner, "%s-server.listen-port=%d", volinfo->volname,
+                     snapd_port);
+    runner_add_arg(&runner, "--no-mem-accounting");
+
+    snprintf(msg, sizeof(msg), "Starting the snapd service for volume %s",
+             volinfo->volname);
+    runner_log(&runner, this->name, GF_LOG_DEBUG, msg);
+
+    if (flags == PROC_START_NO_WAIT) {
+        ret = runner_run_nowait(&runner);
+    } else {
+        synclock_unlock(&priv->big_lock);
+        {
+            ret = runner_run(&runner);
         }
-        runner_add_arg (&runner, "--brick-port");
-        runner_argprintf (&runner, "%d", snapd_port);
-        runner_add_arg (&runner, "--xlator-option");
-        runner_argprintf (&runner, "%s-server.listen-port=%d",
-                         volinfo->volname, snapd_port);
-        runner_add_arg (&runner, "--no-mem-accounting");
-
-        snprintf (msg, sizeof (msg),
-                  "Starting the snapd service for volume %s", volinfo->volname);
-        runner_log (&runner, this->name, GF_LOG_DEBUG, msg);
-
-        if (flags == PROC_START_NO_WAIT) {
-                ret = runner_run_nowait (&runner);
-        } else {
-                synclock_unlock (&priv->big_lock);
-                {
-                        ret = runner_run (&runner);
-                }
-                synclock_lock (&priv->big_lock);
-        }
-        volinfo->snapd.port = snapd_port;
+        synclock_lock(&priv->big_lock);
+    }
 
 out:
-        return ret;
+    return ret;
 }
 
 int
-glusterd_snapdsvc_restart ()
+glusterd_snapdsvc_restart()
 {
-        glusterd_volinfo_t      *volinfo        = NULL;
-        int                     ret             = 0;
-        xlator_t                *this           = THIS;
-        glusterd_conf_t         *conf           = NULL;
-        glusterd_svc_t          *svc            = NULL;
-
-        GF_ASSERT (this);
-
-        conf = this->private;
-        GF_ASSERT (conf);
-
-        cds_list_for_each_entry (volinfo, &conf->volumes, vol_list) {
-                /* Init per volume snapd svc */
-                ret = glusterd_snapdsvc_init (volinfo);
-                if (ret) {
-                        gf_log (this->name, GF_LOG_ERROR, "snapd service "
-                                "initialization failed for volume %s",
-                                volinfo->volname);
-                        goto out;
-                }
-                gf_log (this->name, GF_LOG_DEBUG, "snapd service initialized "
-                        "for %s", volinfo->volname);
-
-                /* Start per volume snapd svc */
-                if (volinfo->status == GLUSTERD_STATUS_STARTED &&
-                    glusterd_is_snapd_enabled (volinfo)) {
-                        svc = &(volinfo->snapd.svc);
-                        ret = svc->start (svc, PROC_START_NO_WAIT);
-                        if (ret) {
-                                gf_log (this->name, GF_LOG_ERROR,
-                                        "Couldn't start snapd for "
-                                        "vol: %s", volinfo->volname);
-                                goto out;
-                        }
-                }
+    glusterd_volinfo_t *volinfo = NULL;
+    glusterd_volinfo_t *tmp = NULL;
+    int ret = 0;
+    xlator_t *this = THIS;
+    glusterd_conf_t *conf = NULL;
+    glusterd_svc_t *svc = NULL;
+
+    GF_ASSERT(this);
+
+    conf = this->private;
+    GF_ASSERT(conf);
+
+    cds_list_for_each_entry_safe(volinfo, tmp, &conf->volumes, vol_list)
+    {
+        /* Start per volume snapd svc */
+        if (volinfo->status == GLUSTERD_STATUS_STARTED) {
+            svc = &(volinfo->snapd.svc);
+            ret = svc->manager(svc, volinfo, PROC_START_NO_WAIT);
+            if (ret) {
+                gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_SNAPD_START_FAIL,
+                       "Couldn't resolve snapd for "
+                       "vol: %s on restart",
+                       volinfo->volname);
+                gf_event(EVENT_SVC_MANAGER_FAILED, "volume=%s;svc_name=%s",
+                         volinfo->volname, svc->name);
+                goto out;
+            }
         }
+    }
 out:
-        return ret;
+    return ret;
 }
 
 int
-glusterd_snapdsvc_rpc_notify (glusterd_conn_t *conn, rpc_clnt_event_t event)
+glusterd_snapdsvc_rpc_notify(glusterd_conn_t *conn, rpc_clnt_event_t event)
 {
-        int                  ret     = 0;
-        glusterd_svc_t      *svc     = NULL;
-        xlator_t            *this    = NULL;
-        glusterd_volinfo_t  *volinfo = NULL;
-        glusterd_snapdsvc_t *snapd   = NULL;
-
-        this = THIS;
-        GF_ASSERT (this);
-
-        svc = cds_list_entry (conn, glusterd_svc_t, conn);
-        if (!svc) {
-                gf_log (this->name, GF_LOG_ERROR, "Failed to get the service");
-                return -1;
-        }
-
-        switch (event) {
+    int ret = 0;
+    glusterd_svc_t *svc = NULL;
+    xlator_t *this = NULL;
+    glusterd_volinfo_t *volinfo = NULL;
+    glusterd_snapdsvc_t *snapd = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+
+    svc = cds_list_entry(conn, glusterd_svc_t, conn);
+    if (!svc) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_SVC_GET_FAIL,
+               "Failed to get the service");
+        return -1;
+    }
+    snapd = cds_list_entry(svc, glusterd_snapdsvc_t, svc);
+    if (!snapd) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_SNAPD_OBJ_GET_FAIL,
+               "Failed to get the "
+               "snapd object");
+        return -1;
+    }
+
+    volinfo = cds_list_entry(snapd, glusterd_volinfo_t, snapd);
+    if (!volinfo) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_VOLINFO_GET_FAIL,
+               "Failed to get the "
+               "volinfo object");
+        return -1;
+    }
+
+    switch (event) {
         case RPC_CLNT_CONNECT:
-                gf_log (this->name, GF_LOG_DEBUG, "%s has connected with "
-                        "glusterd.", svc->name);
-                svc->online =  _gf_true;
-                break;
+            gf_msg_debug(this->name, 0,
+                         "%s has connected with "
+                         "glusterd.",
+                         svc->name);
+            gf_event(EVENT_SVC_CONNECTED, "volume=%s;svc_name=%s",
+                     volinfo->volname, svc->name);
+            svc->online = _gf_true;
+            break;
 
         case RPC_CLNT_DISCONNECT:
-                if (svc->online) {
-                        gf_msg (this->name, GF_LOG_INFO, 0,
-                                GD_MSG_NODE_DISCONNECTED, "%s has disconnected "
-                                "from glusterd.", svc->name);
-                        svc->online =  _gf_false;
-                }
-                break;
+            if (svc->online) {
+                gf_msg(this->name, GF_LOG_INFO, 0, GD_MSG_NODE_DISCONNECTED,
+                       "%s has disconnected "
+                       "from glusterd.",
+                       svc->name);
+                gf_event(EVENT_SVC_DISCONNECTED, "volume=%s;svc_name=%s",
+                         volinfo->volname, svc->name);
+                svc->online = _gf_false;
+            }
+            break;
 
         case RPC_CLNT_DESTROY:
-                snapd = cds_list_entry (svc, glusterd_snapdsvc_t, svc);
-                if (!snapd) {
-                        gf_log (this->name, GF_LOG_ERROR, "Failed to get the "
-                                "snapd object");
-                        return -1;
-                }
-
-                volinfo = cds_list_entry (snapd, glusterd_volinfo_t, snapd);
-                if (!volinfo) {
-                        gf_log (this->name, GF_LOG_ERROR, "Failed to get the "
-                                "volinfo object");
-                        return -1;
-                }
-                glusterd_volinfo_unref (volinfo);
+            glusterd_volinfo_unref(volinfo);
+            break;
 
         default:
-                gf_log (this->name, GF_LOG_TRACE,
-                        "got some other RPC event %d", event);
-                break;
-        }
+            gf_msg_trace(this->name, 0, "got some other RPC event %d", event);
+            break;
+    }
 
-        return ret;
+    return ret;
 }
diff --git a/xlators/mgmt/glusterd/src/glusterd-snapd-svc.h b/xlators/mgmt/glusterd/src/glusterd-snapd-svc.h
index bc5d39f7ffb..e15dbf54315 100644
--- a/xlators/mgmt/glusterd/src/glusterd-snapd-svc.h
+++ b/xlators/mgmt/glusterd/src/glusterd-snapd-svc.h
@@ -11,34 +11,32 @@
 #ifndef _GLUSTERD_SNAPD_SVC_H_
 #define _GLUSTERD_SNAPD_SVC_H_
 
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
-
 #include "glusterd-svc-mgmt.h"
 
 typedef struct glusterd_snapdsvc_ glusterd_snapdsvc_t;
 
-struct glusterd_snapdsvc_{
-        glusterd_svc_t     svc;
-        int                port;
-        gf_store_handle_t *handle;
+struct glusterd_snapdsvc_ {
+    glusterd_svc_t svc;
+    gf_store_handle_t *handle;
+    int port;
 };
 
+void
+glusterd_snapdsvc_build(glusterd_svc_t *svc);
+
 int
-glusterd_snapdsvc_init (void *data);
+glusterd_snapdsvc_init(void *data);
 
 int
-glusterd_snapdsvc_manager (glusterd_svc_t *svc, void *data, int flags);
+glusterd_snapdsvc_manager(glusterd_svc_t *svc, void *data, int flags);
 
 int
-glusterd_snapdsvc_start (glusterd_svc_t *svc, int flags);
+glusterd_snapdsvc_start(glusterd_svc_t *svc, int flags);
 
 int
-glusterd_snapdsvc_restart ();
+glusterd_snapdsvc_restart();
 
 int
-glusterd_snapdsvc_rpc_notify (glusterd_conn_t *conn, rpc_clnt_event_t event);
+glusterd_snapdsvc_rpc_notify(glusterd_conn_t *conn, rpc_clnt_event_t event);
 
 #endif
diff --git a/xlators/mgmt/glusterd/src/glusterd-snapshot-utils.c b/xlators/mgmt/glusterd/src/glusterd-snapshot-utils.c
index 1da8a5b4490..995268b796d 100644
--- a/xlators/mgmt/glusterd/src/glusterd-snapshot-utils.c
+++ b/xlators/mgmt/glusterd/src/glusterd-snapshot-utils.c
@@ -7,10 +7,6 @@
    later), or the GNU General Public License, version 2 (GPLv2), in all
    cases as published by the Free Software Foundation.
 */
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
 #include <inttypes.h>
 
 #if defined(GF_LINUX_HOST_OS)
@@ -20,10 +16,11 @@
 #endif
 #include <dlfcn.h>
 
-#include "dict.h"
-#include "syscall.h"
+#include <glusterfs/dict.h>
+#include <glusterfs/syscall.h>
 #include "glusterd-op-sm.h"
 #include "glusterd-utils.h"
+#include "glusterd-messages.h"
 #include "glusterd-store.h"
 #include "glusterd-volgen.h"
 #include "glusterd-snapd-svc.h"
@@ -31,61 +28,158 @@
 #include "glusterd-snapd-svc-helper.h"
 #include "glusterd-snapshot-utils.h"
 #include "glusterd-server-quorum.h"
+#include "glusterd-messages.h"
+#include "glusterd-errno.h"
 
 /*
  *  glusterd_snap_geo_rep_restore:
  *      This function restores the atime and mtime of marker.tstamp
  *      if present from snapped marker.tstamp file.
  */
+
+int32_t
+glusterd_snapobject_delete(glusterd_snap_t *snap)
+{
+    if (snap == NULL) {
+        gf_msg(THIS->name, GF_LOG_WARNING, 0, GD_MSG_PARAM_NULL,
+               "snap is NULL");
+        return -1;
+    }
+
+    cds_list_del_init(&snap->snap_list);
+    cds_list_del_init(&snap->volumes);
+    if (LOCK_DESTROY(&snap->lock))
+        gf_msg(THIS->name, GF_LOG_WARNING, 0, GD_MSG_LOCK_DESTROY_FAILED,
+               "Failed destroying lock"
+               "of snap %s",
+               snap->snapname);
+
+    GF_FREE(snap->description);
+    GF_FREE(snap);
+
+    return 0;
+}
+
+/*
+ * This function is to be called only from glusterd_peer_detach_cleanup()
+ * as this continues to delete snaps in spite of faiure while deleting
+ * one, as we don't want to fail peer_detach in such a case.
+ */
 int
-glusterd_snap_geo_rep_restore (glusterd_volinfo_t *snap_volinfo,
-                               glusterd_volinfo_t *new_volinfo)
+glusterd_cleanup_snaps_for_volume(glusterd_volinfo_t *volinfo)
 {
-        char                    vol_tstamp_file[PATH_MAX]  = {0,};
-        char                    snap_tstamp_file[PATH_MAX] = {0,};
-        glusterd_conf_t         *priv                      = NULL;
-        xlator_t                *this                      = NULL;
-        int                     geo_rep_indexing_on        = 0;
-        int                     ret                        = 0;
-
-        this = THIS;
-        GF_ASSERT (this);
-        GF_ASSERT (snap_volinfo);
-        GF_ASSERT (new_volinfo);
-
-        priv = this->private;
-        GF_ASSERT (priv);
-
-        /* Check if geo-rep indexing is enabled, if yes, we need restore
-         * back the mtime of 'marker.tstamp' file.
-         */
-        geo_rep_indexing_on = glusterd_volinfo_get_boolean (new_volinfo,
-                                                            VKEY_MARKER_XTIME);
-        if (geo_rep_indexing_on == -1) {
-                gf_log (this->name, GF_LOG_DEBUG, "Failed"
-                        " to check whether geo-rep-indexing enabled or not");
-                ret = 0;
-                goto out;
+    int32_t op_ret = 0;
+    int32_t ret = 0;
+    xlator_t *this = NULL;
+    glusterd_volinfo_t *snap_vol = NULL;
+    glusterd_volinfo_t *dummy_snap_vol = NULL;
+    glusterd_snap_t *snap = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+
+    cds_list_for_each_entry_safe(snap_vol, dummy_snap_vol,
+                                 &volinfo->snap_volumes, snapvol_list)
+    {
+        snap = snap_vol->snapshot;
+        ret = glusterd_store_delete_snap(snap);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_WARNING, 0, GD_MSG_VOL_DELETE_FAIL,
+                   "Failed to remove "
+                   "snap %s from store",
+                   snap->snapname);
+            op_ret = ret;
+            continue;
         }
 
-        if (geo_rep_indexing_on == 1) {
-                GLUSTERD_GET_VOLUME_DIR (vol_tstamp_file, new_volinfo, priv);
-                strncat (vol_tstamp_file, "/marker.tstamp",
-                         PATH_MAX - strlen(vol_tstamp_file) - 1);
-                GLUSTERD_GET_VOLUME_DIR (snap_tstamp_file, snap_volinfo, priv);
-                strncat (snap_tstamp_file, "/marker.tstamp",
-                         PATH_MAX - strlen(snap_tstamp_file) - 1);
-                ret = gf_set_timestamp (snap_tstamp_file, vol_tstamp_file);
-                if (ret) {
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "Unable to set atime and mtime of %s as of %s",
-                                vol_tstamp_file, snap_tstamp_file);
-                        goto out;
-                }
+        ret = glusterd_snapobject_delete(snap);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_WARNING, 0, GD_MSG_VOL_DELETE_FAIL,
+                   "Failed to delete "
+                   "snap object %s",
+                   snap->snapname);
+            op_ret = ret;
+            continue;
+        }
+
+        ret = glusterd_store_delete_volume(snap_vol);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_WARNING, 0, GD_MSG_VOL_DELETE_FAIL,
+                   "Failed to remove "
+                   "volume %s from store",
+                   snap_vol->volname);
+            op_ret = ret;
+            continue;
         }
 
+        ret = glusterd_volinfo_delete(snap_vol);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_WARNING, 0, GD_MSG_VOL_DELETE_FAIL,
+                   "Failed to remove "
+                   "volinfo %s ",
+                   snap_vol->volname);
+            op_ret = ret;
+            continue;
+        }
+    }
+
+    return op_ret;
+}
+
+int
+glusterd_snap_geo_rep_restore(glusterd_volinfo_t *snap_volinfo,
+                              glusterd_volinfo_t *new_volinfo)
+{
+    char vol_tstamp_file[PATH_MAX] = {
+        0,
+    };
+    char snap_tstamp_file[PATH_MAX] = {
+        0,
+    };
+    glusterd_conf_t *priv = NULL;
+    xlator_t *this = NULL;
+    int geo_rep_indexing_on = 0;
+    int ret = 0;
+
+    this = THIS;
+    GF_ASSERT(this);
+    GF_ASSERT(snap_volinfo);
+    GF_ASSERT(new_volinfo);
+
+    priv = this->private;
+    GF_ASSERT(priv);
+
+    /* Check if geo-rep indexing is enabled, if yes, we need restore
+     * back the mtime of 'marker.tstamp' file.
+     */
+    geo_rep_indexing_on = glusterd_volinfo_get_boolean(new_volinfo,
+                                                       VKEY_MARKER_XTIME);
+    if (geo_rep_indexing_on == -1) {
+        gf_msg_debug(this->name, 0,
+                     "Failed"
+                     " to check whether geo-rep-indexing enabled or not");
+        ret = 0;
+        goto out;
+    }
+
+    if (geo_rep_indexing_on == 1) {
+        GLUSTERD_GET_VOLUME_DIR(vol_tstamp_file, new_volinfo, priv);
+        strncat(vol_tstamp_file, "/marker.tstamp",
+                PATH_MAX - strlen(vol_tstamp_file) - 1);
+        GLUSTERD_GET_VOLUME_DIR(snap_tstamp_file, snap_volinfo, priv);
+        strncat(snap_tstamp_file, "/marker.tstamp",
+                PATH_MAX - strlen(snap_tstamp_file) - 1);
+        ret = gf_set_timestamp(snap_tstamp_file, vol_tstamp_file);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_TSTAMP_SET_FAIL,
+                   "Unable to set atime and mtime of %s as of %s",
+                   vol_tstamp_file, snap_tstamp_file);
+            goto out;
+        }
+    }
+
 out:
-        return ret;
+    return ret;
 }
 
 /* This function will copy snap volinfo to the new
@@ -100,328 +194,330 @@ out:
  * TODO: Duplicate all members of volinfo, e.g. geo-rep sync slaves
  */
 int32_t
-glusterd_snap_volinfo_restore (dict_t *dict, dict_t *rsp_dict,
-                               glusterd_volinfo_t *new_volinfo,
-                               glusterd_volinfo_t *snap_volinfo,
-                               int32_t volcount)
+glusterd_snap_volinfo_restore(dict_t *dict, dict_t *rsp_dict,
+                              glusterd_volinfo_t *new_volinfo,
+                              glusterd_volinfo_t *snap_volinfo,
+                              int32_t volcount)
 {
-        char                    *value          = NULL;
-        char                    key[PATH_MAX]   = "";
-        int32_t                 brick_count     = -1;
-        int32_t                 ret             = -1;
-        xlator_t                *this           = NULL;
-        glusterd_brickinfo_t    *brickinfo      = NULL;
-        glusterd_brickinfo_t    *new_brickinfo  = NULL;
-
-        this = THIS;
-        GF_ASSERT (this);
-        GF_ASSERT (dict);
-        GF_ASSERT (rsp_dict);
-
-        GF_VALIDATE_OR_GOTO (this->name, new_volinfo, out);
-        GF_VALIDATE_OR_GOTO (this->name, snap_volinfo, out);
-
-        brick_count = 0;
-        cds_list_for_each_entry (brickinfo, &snap_volinfo->bricks, brick_list) {
-                brick_count++;
-                ret = glusterd_brickinfo_new (&new_brickinfo);
-                if (ret) {
-                        gf_log (this->name, GF_LOG_ERROR, "Failed to create "
-                                "new brickinfo");
-                        goto out;
-                }
-
-                /* Duplicate brickinfo */
-                ret = glusterd_brickinfo_dup (brickinfo, new_brickinfo);
-                if (ret) {
-                        gf_log (this->name, GF_LOG_ERROR, "Failed to dup "
-                                "brickinfo");
-                        goto out;
-                }
-
-                /* Fetch values if present in dict These values won't
-                 * be present in case of a missed restore. In that case
-                 * it's fine to use the local node's value
-                 */
-                snprintf (key, sizeof (key), "snap%d.brick%d.path",
-                          volcount, brick_count);
-                ret = dict_get_str (dict, key, &value);
-                if (!ret)
-                        strncpy (new_brickinfo->path, value,
-                                 sizeof(new_brickinfo->path));
-
-                snprintf (key, sizeof (key), "snap%d.brick%d.snap_status",
-                          volcount, brick_count);
-                ret = dict_get_int32 (dict, key, &new_brickinfo->snap_status);
-
-                snprintf (key, sizeof (key), "snap%d.brick%d.device_path",
-                          volcount, brick_count);
-                ret = dict_get_str (dict, key, &value);
-                if (!ret)
-                        strncpy (new_brickinfo->device_path, value,
-                                 sizeof(new_brickinfo->device_path));
-
-                snprintf (key, sizeof (key), "snap%d.brick%d.fs_type",
-                          volcount, brick_count);
-                ret = dict_get_str (dict, key, &value);
-                if (!ret)
-                        strncpy (new_brickinfo->fstype, value,
-                                 sizeof(new_brickinfo->fstype));
-
-                snprintf (key, sizeof (key), "snap%d.brick%d.mnt_opts",
-                          volcount, brick_count);
-                ret = dict_get_str (dict, key, &value);
-                if (!ret)
-                        strncpy (new_brickinfo->mnt_opts, value,
-                                 sizeof(new_brickinfo->mnt_opts));
-
-                /* If the brick is not of this peer, or snapshot is missed *
-                 * for the brick do not replace the xattr for it */
-                if ((!gf_uuid_compare (brickinfo->uuid, MY_UUID)) &&
-                    (brickinfo->snap_status != -1)) {
-                        /* We need to replace the volume id of all the bricks
-                         * to the volume id of the origin volume. new_volinfo
-                         * has the origin volume's volume id*/
-                        ret = sys_lsetxattr (new_brickinfo->path,
-                                             GF_XATTR_VOL_ID_KEY,
-                                             new_volinfo->volume_id,
-                                             sizeof (new_volinfo->volume_id),
-                                             XATTR_REPLACE);
-                        if (ret == -1) {
-                                gf_log (this->name, GF_LOG_ERROR, "Failed to "
-                                        "set extended attribute %s on %s. "
-                                        "Reason: %s, snap: %s",
-                                        GF_XATTR_VOL_ID_KEY,
-                                        new_brickinfo->path, strerror (errno),
-                                        new_volinfo->volname);
-                                goto out;
-                        }
-                }
-
-                /* If a snapshot is pending for this brick then
-                 * restore should also be pending
-                 */
-                if (brickinfo->snap_status == -1) {
-                        /* Adding missed delete to the dict */
-                        ret = glusterd_add_missed_snaps_to_dict
-                                                (rsp_dict,
-                                                 snap_volinfo,
-                                                 brickinfo,
-                                                 brick_count,
-                                                 GF_SNAP_OPTION_TYPE_RESTORE);
-                        if (ret) {
-                                gf_log (this->name, GF_LOG_ERROR,
-                                        "Failed to add missed snapshot info "
-                                        "for %s:%s in the rsp_dict",
-                                        brickinfo->hostname,
-                                        brickinfo->path);
-                                goto out;
-                        }
-                }
-
-                cds_list_add_tail (&new_brickinfo->brick_list,
-                                   &new_volinfo->bricks);
-                /* ownership of new_brickinfo is passed to new_volinfo */
-                new_brickinfo = NULL;
+    char *value = NULL;
+    char key[64] = "";
+    int32_t brick_count = -1;
+    int32_t ret = -1;
+    xlator_t *this = NULL;
+    glusterd_brickinfo_t *brickinfo = NULL;
+    glusterd_brickinfo_t *new_brickinfo = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+    GF_ASSERT(dict);
+    GF_ASSERT(rsp_dict);
+
+    GF_VALIDATE_OR_GOTO(this->name, new_volinfo, out);
+    GF_VALIDATE_OR_GOTO(this->name, snap_volinfo, out);
+
+    brick_count = 0;
+    cds_list_for_each_entry(brickinfo, &snap_volinfo->bricks, brick_list)
+    {
+        brick_count++;
+        ret = glusterd_brickinfo_new(&new_brickinfo);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_BRICK_NEW_INFO_FAIL,
+                   "Failed to create "
+                   "new brickinfo");
+            goto out;
         }
 
-        /* Regenerate all volfiles */
-        ret = glusterd_create_volfiles_and_notify_services (new_volinfo);
+        /* Duplicate brickinfo */
+        ret = glusterd_brickinfo_dup(brickinfo, new_brickinfo);
         if (ret) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "Failed to regenerate volfiles");
-                goto out;
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_BRICK_SET_INFO_FAIL,
+                   "Failed to dup "
+                   "brickinfo");
+            goto out;
         }
 
-        /* Restore geo-rep marker.tstamp's timestamp */
-        ret = glusterd_snap_geo_rep_restore (snap_volinfo, new_volinfo);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "Geo-rep: marker.tstamp's timestamp restoration failed");
+        /* Fetch values if present in dict These values won't
+         * be present in case of a missed restore. In that case
+         * it's fine to use the local node's value
+         */
+        snprintf(key, sizeof(key), "snap%d.brick%d.path", volcount,
+                 brick_count);
+        ret = dict_get_str(dict, key, &value);
+        if (!ret)
+            gf_strncpy(new_brickinfo->path, value, sizeof(new_brickinfo->path));
+
+        snprintf(key, sizeof(key), "snap%d.brick%d.snap_status", volcount,
+                 brick_count);
+        ret = dict_get_int32(dict, key, &new_brickinfo->snap_status);
+
+        snprintf(key, sizeof(key), "snap%d.brick%d.device_path", volcount,
+                 brick_count);
+        ret = dict_get_str(dict, key, &value);
+        if (!ret)
+            gf_strncpy(new_brickinfo->device_path, value,
+                       sizeof(new_brickinfo->device_path));
+
+        snprintf(key, sizeof(key), "snap%d.brick%d.fs_type", volcount,
+                 brick_count);
+        ret = dict_get_str(dict, key, &value);
+        if (!ret)
+            gf_strncpy(new_brickinfo->fstype, value,
+                       sizeof(new_brickinfo->fstype));
+
+        snprintf(key, sizeof(key), "snap%d.brick%d.mnt_opts", volcount,
+                 brick_count);
+        ret = dict_get_str(dict, key, &value);
+        if (!ret)
+            gf_strncpy(new_brickinfo->mnt_opts, value,
+                       sizeof(new_brickinfo->mnt_opts));
+
+        /* If the brick is not of this peer, or snapshot is missed *
+         * for the brick do not replace the xattr for it */
+        if ((!gf_uuid_compare(brickinfo->uuid, MY_UUID)) &&
+            (brickinfo->snap_status != -1)) {
+            /* We need to replace the volume id of all the bricks
+             * to the volume id of the origin volume. new_volinfo
+             * has the origin volume's volume id*/
+            ret = sys_lsetxattr(new_brickinfo->path, GF_XATTR_VOL_ID_KEY,
+                                new_volinfo->volume_id,
+                                sizeof(new_volinfo->volume_id), XATTR_REPLACE);
+            if (ret == -1) {
+                gf_smsg(this->name, GF_LOG_ERROR, 0, GD_MSG_SET_XATTR_FAIL,
+                        "Attribute=%s, Path=%s, Reason=%s, Snap=%s",
+                        GF_XATTR_VOL_ID_KEY, new_brickinfo->path,
+                        strerror(errno), new_volinfo->volname, NULL);
                 goto out;
+            }
         }
 
+        /* If a snapshot is pending for this brick then
+         * restore should also be pending
+         */
+        if (brickinfo->snap_status == -1) {
+            /* Adding missed delete to the dict */
+            ret = glusterd_add_missed_snaps_to_dict(
+                rsp_dict, snap_volinfo, brickinfo, brick_count,
+                GF_SNAP_OPTION_TYPE_RESTORE);
+            if (ret) {
+                gf_msg(this->name, GF_LOG_ERROR, 0,
+                       GD_MSG_MISSEDSNAP_INFO_SET_FAIL,
+                       "Failed to add missed snapshot info "
+                       "for %s:%s in the rsp_dict",
+                       brickinfo->hostname, brickinfo->path);
+                goto out;
+            }
+        }
+
+        cds_list_add_tail(&new_brickinfo->brick_list, &new_volinfo->bricks);
+        /* ownership of new_brickinfo is passed to new_volinfo */
+        new_brickinfo = NULL;
+    }
+
+    /* Regenerate all volfiles */
+    ret = glusterd_create_volfiles_and_notify_services(new_volinfo);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_VOLFILE_CREATE_FAIL,
+               "Failed to regenerate volfiles");
+        goto out;
+    }
+
+    /* Restore geo-rep marker.tstamp's timestamp */
+    ret = glusterd_snap_geo_rep_restore(snap_volinfo, new_volinfo);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_TSTAMP_SET_FAIL,
+               "Geo-rep: marker.tstamp's timestamp restoration failed");
+        goto out;
+    }
+
 out:
-        if (ret && (NULL != new_brickinfo)) {
-                (void) glusterd_brickinfo_delete (new_brickinfo);
-        }
+    if (ret && (NULL != new_brickinfo)) {
+        (void)glusterd_brickinfo_delete(new_brickinfo);
+    }
 
-        return ret;
+    return ret;
 }
 
 int
-glusterd_snap_volinfo_find_by_volume_id (uuid_t volume_id,
-                                         glusterd_volinfo_t **volinfo)
+glusterd_snap_volinfo_find_by_volume_id(uuid_t volume_id,
+                                        glusterd_volinfo_t **volinfo)
 {
-        int32_t                  ret     = -1;
-        xlator_t                *this    = NULL;
-        glusterd_volinfo_t      *voliter = NULL;
-        glusterd_snap_t         *snap    = NULL;
-        glusterd_conf_t         *priv    = NULL;
-
-        this = THIS;
-        priv = this->private;
-        GF_ASSERT (priv);
-        GF_ASSERT (volinfo);
-
-        if (gf_uuid_is_null(volume_id)) {
-                gf_log (this->name, GF_LOG_WARNING, "Volume UUID is NULL");
-                goto out;
-        }
-
-        cds_list_for_each_entry (snap, &priv->snapshots, snap_list) {
-                cds_list_for_each_entry (voliter, &snap->volumes, vol_list) {
-                        if (gf_uuid_compare (volume_id, voliter->volume_id))
-                                continue;
-                        *volinfo = voliter;
-                        ret = 0;
-                        goto out;
-                }
-        }
-
-        gf_log (this->name, GF_LOG_WARNING, "Snap volume not found");
+    int32_t ret = -1;
+    xlator_t *this = NULL;
+    glusterd_volinfo_t *voliter = NULL;
+    glusterd_snap_t *snap = NULL;
+    glusterd_conf_t *priv = NULL;
+
+    this = THIS;
+    priv = this->private;
+    GF_ASSERT(priv);
+    GF_ASSERT(volinfo);
+
+    if (gf_uuid_is_null(volume_id)) {
+        gf_msg(this->name, GF_LOG_WARNING, 0, GD_MSG_UUID_NULL,
+               "Volume UUID is NULL");
+        goto out;
+    }
+
+    cds_list_for_each_entry(snap, &priv->snapshots, snap_list)
+    {
+        cds_list_for_each_entry(voliter, &snap->volumes, vol_list)
+        {
+            if (gf_uuid_compare(volume_id, voliter->volume_id))
+                continue;
+            *volinfo = voliter;
+            ret = 0;
+            goto out;
+        }
+    }
+
+    gf_msg(this->name, GF_LOG_WARNING, 0, GD_MSG_SNAP_NOT_FOUND,
+           "Snap volume not found");
 out:
-        gf_log (this->name, GF_LOG_TRACE, "Returning %d", ret);
-        return ret;
+    gf_msg_trace(this->name, 0, "Returning %d", ret);
+    return ret;
 }
 
 int32_t
-glusterd_snap_volinfo_find (char *snap_volname, glusterd_snap_t *snap,
-                            glusterd_volinfo_t **volinfo)
+glusterd_snap_volinfo_find(char *snap_volname, glusterd_snap_t *snap,
+                           glusterd_volinfo_t **volinfo)
 {
-        int32_t                  ret         = -1;
-        xlator_t                *this        = NULL;
-        glusterd_volinfo_t      *snap_vol    = NULL;
-        glusterd_conf_t         *priv        = NULL;
-
-        this = THIS;
-        priv = this->private;
-        GF_ASSERT (priv);
-        GF_ASSERT (snap);
-        GF_ASSERT (snap_volname);
-
-        cds_list_for_each_entry (snap_vol, &snap->volumes, vol_list) {
-                if (!strcmp (snap_vol->volname, snap_volname)) {
-                        ret = 0;
-                        *volinfo = snap_vol;
-                        goto out;
-                }
-        }
-
-        gf_log (this->name, GF_LOG_WARNING, "Snap volume %s not found",
-                snap_volname);
+    int32_t ret = -1;
+    xlator_t *this = NULL;
+    glusterd_volinfo_t *snap_vol = NULL;
+    glusterd_conf_t *priv = NULL;
+
+    this = THIS;
+    priv = this->private;
+    GF_ASSERT(priv);
+    GF_ASSERT(snap);
+    GF_ASSERT(snap_volname);
+
+    cds_list_for_each_entry(snap_vol, &snap->volumes, vol_list)
+    {
+        if (!strcmp(snap_vol->volname, snap_volname)) {
+            ret = 0;
+            *volinfo = snap_vol;
+            goto out;
+        }
+    }
+
+    gf_msg(this->name, GF_LOG_WARNING, EINVAL, GD_MSG_SNAP_NOT_FOUND,
+           "Snap volume %s not found", snap_volname);
 out:
-        gf_log (this->name, GF_LOG_TRACE, "Returning %d", ret);
-        return ret;
+    gf_msg_trace(this->name, 0, "Returning %d", ret);
+    return ret;
 }
 
 int32_t
-glusterd_snap_volinfo_find_from_parent_volname (char *origin_volname,
-                                      glusterd_snap_t *snap,
-                                      glusterd_volinfo_t **volinfo)
+glusterd_snap_volinfo_find_from_parent_volname(char *origin_volname,
+                                               glusterd_snap_t *snap,
+                                               glusterd_volinfo_t **volinfo)
 {
-        int32_t                  ret         = -1;
-        xlator_t                *this        = NULL;
-        glusterd_volinfo_t      *snap_vol    = NULL;
-        glusterd_conf_t         *priv        = NULL;
-
-        this = THIS;
-        priv = this->private;
-        GF_ASSERT (priv);
-        GF_ASSERT (snap);
-        GF_ASSERT (origin_volname);
-
-        cds_list_for_each_entry (snap_vol, &snap->volumes, vol_list) {
-                if (!strcmp (snap_vol->parent_volname, origin_volname)) {
-                        ret = 0;
-                        *volinfo = snap_vol;
-                        goto out;
-                }
-        }
-
-        gf_log (this->name, GF_LOG_DEBUG, "Snap volume not found(snap: %s, "
-                "origin-volume: %s", snap->snapname, origin_volname);
+    int32_t ret = -1;
+    xlator_t *this = NULL;
+    glusterd_volinfo_t *snap_vol = NULL;
+    glusterd_conf_t *priv = NULL;
+
+    this = THIS;
+    priv = this->private;
+    GF_ASSERT(priv);
+    GF_ASSERT(snap);
+    GF_ASSERT(origin_volname);
+
+    cds_list_for_each_entry(snap_vol, &snap->volumes, vol_list)
+    {
+        if (!strcmp(snap_vol->parent_volname, origin_volname)) {
+            ret = 0;
+            *volinfo = snap_vol;
+            goto out;
+        }
+    }
+
+    gf_msg_debug(this->name, 0,
+                 "Snap volume not found(snap: %s, "
+                 "origin-volume: %s",
+                 snap->snapname, origin_volname);
 
 out:
-        gf_log (this->name, GF_LOG_TRACE, "Returning %d", ret);
-        return ret;
+    gf_msg_trace(this->name, 0, "Returning %d", ret);
+    return ret;
 }
 
 /* Exports a bricks snapshot details only if required
  *
- * The details will be exported only if the cluster op-version is greather than
+ * The details will be exported only if the cluster op-version is greater than
  * 4, ie. snapshot is supported in the cluster
  */
 int
-gd_add_brick_snap_details_to_dict (dict_t *dict, char *prefix,
-                                   glusterd_brickinfo_t *brickinfo)
+gd_add_brick_snap_details_to_dict(dict_t *dict, char *prefix,
+                                  glusterd_brickinfo_t *brickinfo)
 {
-        int ret = -1;
-        xlator_t *this = NULL;
-        glusterd_conf_t *conf = NULL;
-        char key[256] = {0,};
-
-        this = THIS;
-        GF_ASSERT (this != NULL);
-        conf = this->private;
-        GF_VALIDATE_OR_GOTO (this->name, (conf != NULL), out);
-
-        GF_VALIDATE_OR_GOTO (this->name, (dict != NULL), out);
-        GF_VALIDATE_OR_GOTO (this->name, (prefix != NULL), out);
-        GF_VALIDATE_OR_GOTO (this->name, (brickinfo != NULL), out);
-
-        if (conf->op_version < GD_OP_VERSION_3_6_0) {
-                ret = 0;
-                goto out;
-        }
-
-        snprintf (key, sizeof (key), "%s.snap_status", prefix);
-        ret = dict_set_int32 (dict, key, brickinfo->snap_status);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "Failed to set snap_status for %s:%s",
-                        brickinfo->hostname, brickinfo->path);
-                goto out;
-        }
-
-        memset (key, 0, sizeof (key));
-        snprintf (key, sizeof (key), "%s.device_path", prefix);
-        ret = dict_set_str (dict, key, brickinfo->device_path);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "Failed to set snap_device for %s:%s",
-                         brickinfo->hostname, brickinfo->path);
-                goto out;
-        }
-
-        snprintf (key, sizeof (key), "%s.fs_type", prefix);
-        ret = dict_set_str (dict, key, brickinfo->fstype);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "Failed to set fstype for %s:%s",
-                         brickinfo->hostname, brickinfo->path);
-                goto out;
-        }
-
-        snprintf (key, sizeof (key), "%s.mnt_opts", prefix);
-        ret = dict_set_str (dict, key, brickinfo->mnt_opts);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "Failed to set mnt_opts for %s:%s",
-                         brickinfo->hostname, brickinfo->path);
-                goto out;
-        }
-
-        memset (key, 0, sizeof (key));
-        snprintf (key, sizeof (key), "%s.mount_dir", prefix);
-        ret = dict_set_str (dict, key, brickinfo->mount_dir);
-        if (ret)
-                gf_log (this->name, GF_LOG_ERROR,
-                        "Failed to set mount_dir for %s:%s",
-                         brickinfo->hostname, brickinfo->path);
+    int ret = -1;
+    xlator_t *this = NULL;
+    glusterd_conf_t *conf = NULL;
+    char key[256] = {
+        0,
+    };
+
+    this = THIS;
+    GF_ASSERT(this != NULL);
+    conf = this->private;
+    GF_VALIDATE_OR_GOTO(this->name, (conf != NULL), out);
+
+    GF_VALIDATE_OR_GOTO(this->name, (dict != NULL), out);
+    GF_VALIDATE_OR_GOTO(this->name, (prefix != NULL), out);
+    GF_VALIDATE_OR_GOTO(this->name, (brickinfo != NULL), out);
+
+    if (conf->op_version < GD_OP_VERSION_3_6_0) {
+        ret = 0;
+        goto out;
+    }
+
+    snprintf(key, sizeof(key), "%s.snap_status", prefix);
+    ret = dict_set_int32(dict, key, brickinfo->snap_status);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_SNAP_STATUS_FAIL,
+               "Failed to set snap_status for %s:%s", brickinfo->hostname,
+               brickinfo->path);
+        goto out;
+    }
+
+    snprintf(key, sizeof(key), "%s.device_path", prefix);
+    ret = dict_set_str(dict, key, brickinfo->device_path);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+               "Failed to set snap_device for %s:%s", brickinfo->hostname,
+               brickinfo->path);
+        goto out;
+    }
+
+    snprintf(key, sizeof(key), "%s.fs_type", prefix);
+    ret = dict_set_str(dict, key, brickinfo->fstype);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+               "Failed to set fstype for %s:%s", brickinfo->hostname,
+               brickinfo->path);
+        goto out;
+    }
+
+    snprintf(key, sizeof(key), "%s.mnt_opts", prefix);
+    ret = dict_set_str(dict, key, brickinfo->mnt_opts);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_BRK_MOUNTOPTS_FAIL,
+               "Failed to set mnt_opts for %s:%s", brickinfo->hostname,
+               brickinfo->path);
+        goto out;
+    }
+
+    snprintf(key, sizeof(key), "%s.mount_dir", prefix);
+    ret = dict_set_str(dict, key, brickinfo->mount_dir);
+    if (ret)
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+               "Failed to set mount_dir for %s:%s", brickinfo->hostname,
+               brickinfo->path);
 
 out:
-        return ret;
+    return ret;
 }
 
 /* Exports a volumes snapshot details only if required.
@@ -430,305 +526,294 @@ out:
  * greater than 4, ie. snapshot is supported in the cluster
  */
 int
-gd_add_vol_snap_details_to_dict (dict_t *dict, char *prefix,
-                                 glusterd_volinfo_t *volinfo)
+gd_add_vol_snap_details_to_dict(dict_t *dict, char *prefix,
+                                glusterd_volinfo_t *volinfo)
 {
-        int ret = -1;
-        xlator_t *this = NULL;
-        glusterd_conf_t *conf = NULL;
-        char key[256] = {0,};
-
-        this = THIS;
-        GF_ASSERT (this != NULL);
-        conf = this->private;
-        GF_VALIDATE_OR_GOTO (this->name, (conf != NULL), out);
-
-        GF_VALIDATE_OR_GOTO (this->name, (dict != NULL), out);
-        GF_VALIDATE_OR_GOTO (this->name, (volinfo != NULL), out);
-        GF_VALIDATE_OR_GOTO (this->name, (prefix != NULL), out);
-
-        if (conf->op_version < GD_OP_VERSION_3_6_0) {
-                ret = 0;
-                goto out;
-        }
-
-        snprintf (key, sizeof (key), "%s.restored_from_snap", prefix);
-        ret = dict_set_dynstr_with_alloc
-                                  (dict, key,
-                                   uuid_utoa (volinfo->restored_from_snap));
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "Unable to set %s for volume"
-                        "%s", key, volinfo->volname);
-                goto out;
-        }
-
-        if (strlen (volinfo->parent_volname) > 0) {
-                memset (key, 0, sizeof (key));
-                snprintf (key, sizeof (key), "%s.parent_volname", prefix);
-                ret = dict_set_dynstr_with_alloc (dict, key,
-                                                  volinfo->parent_volname);
-                if (ret) {
-                        gf_log (this->name, GF_LOG_ERROR, "Unable to set %s "
-                                "for volume %s", key, volinfo->volname);
-                        goto out;
-                }
-        }
-
-        memset (key, 0, sizeof (key));
-        snprintf (key, sizeof (key), "%s.is_snap_volume", prefix);
-        ret = dict_set_uint32 (dict, key, volinfo->is_snap_volume);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "Unable to set %s for volume"
-                        "%s", key, volinfo->volname);
-                goto out;
-        }
-
-        memset (key, 0, sizeof (key));
-        snprintf (key, sizeof (key), "%s.snap-max-hard-limit", prefix);
-        ret = dict_set_uint64 (dict, key, volinfo->snap_max_hard_limit);
+    int ret = -1;
+    xlator_t *this = NULL;
+    glusterd_conf_t *conf = NULL;
+    char key[256] = {
+        0,
+    };
+
+    this = THIS;
+    GF_ASSERT(this != NULL);
+    conf = this->private;
+    GF_VALIDATE_OR_GOTO(this->name, (conf != NULL), out);
+
+    GF_VALIDATE_OR_GOTO(this->name, (dict != NULL), out);
+    GF_VALIDATE_OR_GOTO(this->name, (volinfo != NULL), out);
+    GF_VALIDATE_OR_GOTO(this->name, (prefix != NULL), out);
+
+    if (conf->op_version < GD_OP_VERSION_3_6_0) {
+        ret = 0;
+        goto out;
+    }
+
+    snprintf(key, sizeof(key), "%s.restored_from_snap", prefix);
+    ret = dict_set_dynstr_with_alloc(dict, key,
+                                     uuid_utoa(volinfo->restored_from_snap));
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+               "Unable to set %s for volume"
+               "%s",
+               key, volinfo->volname);
+        goto out;
+    }
+
+    if (strlen(volinfo->parent_volname) > 0) {
+        snprintf(key, sizeof(key), "%s.parent_volname", prefix);
+        ret = dict_set_dynstr_with_alloc(dict, key, volinfo->parent_volname);
         if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "Unable to set %s for volume"
-                        "%s", key, volinfo->volname);
-        }
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+                   "Unable to set %s "
+                   "for volume %s",
+                   key, volinfo->volname);
+            goto out;
+        }
+    }
+
+    snprintf(key, sizeof(key), "%s.is_snap_volume", prefix);
+    ret = dict_set_uint32(dict, key, volinfo->is_snap_volume);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+               "Unable to set %s for volume"
+               "%s",
+               key, volinfo->volname);
+        goto out;
+    }
+
+    snprintf(key, sizeof(key), "%s.snap-max-hard-limit", prefix);
+    ret = dict_set_uint64(dict, key, volinfo->snap_max_hard_limit);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+               "Unable to set %s for volume"
+               "%s",
+               key, volinfo->volname);
+    }
 
 out:
-        return ret;
+    return ret;
 }
 
 int32_t
-glusterd_add_missed_snaps_to_export_dict (dict_t *peer_data)
+glusterd_add_missed_snaps_to_export_dict(dict_t *peer_data)
 {
-        char                           name_buf[PATH_MAX]   = "";
-        char                           value[PATH_MAX]      = "";
-        int32_t                        missed_snap_count    = 0;
-        int32_t                        ret                  = -1;
-        glusterd_conf_t               *priv                 = NULL;
-        glusterd_missed_snap_info     *missed_snapinfo      = NULL;
-        glusterd_snap_op_t            *snap_opinfo          = NULL;
-        xlator_t                      *this                 = NULL;
-
-        this = THIS;
-        GF_ASSERT (this);
-        GF_ASSERT (peer_data);
-
-        priv = this->private;
-        GF_ASSERT (priv);
-
-        /* Add the missed_entries in the dict */
-        cds_list_for_each_entry (missed_snapinfo, &priv->missed_snaps_list,
-                                 missed_snaps) {
-                cds_list_for_each_entry (snap_opinfo,
-                                         &missed_snapinfo->snap_ops,
-                                         snap_ops_list) {
-                        snprintf (name_buf, sizeof(name_buf),
-                                  "missed_snaps_%d", missed_snap_count);
-                        snprintf (value, sizeof(value), "%s:%s=%s:%d:%s:%d:%d",
-                                  missed_snapinfo->node_uuid,
-                                  missed_snapinfo->snap_uuid,
-                                  snap_opinfo->snap_vol_id,
-                                  snap_opinfo->brick_num,
-                                  snap_opinfo->brick_path,
-                                  snap_opinfo->op,
-                                  snap_opinfo->status);
-
-                        ret = dict_set_dynstr_with_alloc (peer_data, name_buf,
-                                                          value);
-                        if (ret) {
-                                gf_log (this->name, GF_LOG_ERROR,
-                                        "Unable to set %s",
-                                        name_buf);
-                                goto out;
-                        }
-                        missed_snap_count++;
-                }
-        }
-
-        ret = dict_set_int32 (peer_data, "missed_snap_count",
-                              missed_snap_count);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "Unable to set missed_snap_count");
+    char name_buf[PATH_MAX] = "";
+    char value[PATH_MAX] = "";
+    int32_t missed_snap_count = 0;
+    int32_t ret = -1;
+    glusterd_conf_t *priv = NULL;
+    glusterd_missed_snap_info *missed_snapinfo = NULL;
+    glusterd_snap_op_t *snap_opinfo = NULL;
+    xlator_t *this = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+    GF_ASSERT(peer_data);
+
+    priv = this->private;
+    GF_ASSERT(priv);
+
+    /* Add the missed_entries in the dict */
+    cds_list_for_each_entry(missed_snapinfo, &priv->missed_snaps_list,
+                            missed_snaps)
+    {
+        cds_list_for_each_entry(snap_opinfo, &missed_snapinfo->snap_ops,
+                                snap_ops_list)
+        {
+            snprintf(name_buf, sizeof(name_buf), "missed_snaps_%d",
+                     missed_snap_count);
+            snprintf(value, sizeof(value), "%s:%s=%s:%d:%s:%d:%d",
+                     missed_snapinfo->node_uuid, missed_snapinfo->snap_uuid,
+                     snap_opinfo->snap_vol_id, snap_opinfo->brick_num,
+                     snap_opinfo->brick_path, snap_opinfo->op,
+                     snap_opinfo->status);
+
+            ret = dict_set_dynstr_with_alloc(peer_data, name_buf, value);
+            if (ret) {
+                gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+                       "Unable to set %s", name_buf);
                 goto out;
+            }
+            missed_snap_count++;
         }
+    }
+
+    ret = dict_set_int32(peer_data, "missed_snap_count", missed_snap_count);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+               "Unable to set missed_snap_count");
+        goto out;
+    }
 
 out:
-        gf_log (this->name, GF_LOG_TRACE, "Returning %d", ret);
-        return ret;
+    gf_msg_trace(this->name, 0, "Returning %d", ret);
+    return ret;
 }
 
 int32_t
-glusterd_add_snap_to_dict (glusterd_snap_t *snap, dict_t *peer_data,
-                           int32_t snap_count)
+glusterd_add_snap_to_dict(glusterd_snap_t *snap, dict_t *peer_data,
+                          int32_t snap_count)
 {
-        char                    buf[NAME_MAX]    = "";
-        char                    prefix[NAME_MAX] = "";
-        int32_t                 ret              = -1;
-        int32_t                 volcount         = 0;
-        glusterd_volinfo_t     *volinfo          = NULL;
-        glusterd_brickinfo_t   *brickinfo        = NULL;
-        gf_boolean_t            host_bricks      = _gf_false;
-        xlator_t               *this             = NULL;
-
-        this = THIS;
-        GF_ASSERT (this);
-        GF_ASSERT (snap);
-        GF_ASSERT (peer_data);
-
-        snprintf (prefix, sizeof(prefix), "snap%d", snap_count);
-
-        cds_list_for_each_entry (volinfo, &snap->volumes, vol_list) {
-                volcount++;
-                ret = glusterd_add_volume_to_dict (volinfo, peer_data,
-                                                   volcount, prefix);
-                if (ret) {
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "Failed to add snap:%s volume:%s "
-                                "to peer_data dict for handshake",
-                                snap->snapname, volinfo->volname);
-                        goto out;
-                }
-
-                if (glusterd_is_volume_quota_enabled (volinfo)) {
-
-                        ret = glusterd_vol_add_quota_conf_to_dict (volinfo,
-                                                                   peer_data,
-                                                                   volcount,
-                                                                   prefix);
-                        if (ret) {
-                                gf_log (this->name, GF_LOG_ERROR,
-                                        "Failed to add quota conf for "
-                                        "snap:%s volume:%s to peer_data "
-                                        "dict for handshake", snap->snapname,
-                                        volinfo->volname);
-                                goto out;
-                        }
-                }
-
-                cds_list_for_each_entry (brickinfo, &volinfo->bricks,
-                                         brick_list) {
-                        if (!gf_uuid_compare (brickinfo->uuid, MY_UUID)) {
-                                host_bricks = _gf_true;
-                                break;
-                        }
-                }
-        }
-
-        snprintf (buf, sizeof(buf), "%s.host_bricks", prefix);
-        ret = dict_set_int8 (peer_data, buf, (int8_t) host_bricks);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "Unable to set host_bricks for snap %s",
-                        snap->snapname);
-                goto out;
-        }
-
-        snprintf (buf, sizeof(buf), "%s.volcount", prefix);
-        ret = dict_set_int32 (peer_data, buf, volcount);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "Unable to set volcount for snap %s",
-                        snap->snapname);
-                goto out;
-        }
-
-        snprintf (buf, sizeof(buf), "%s.snapname", prefix);
-        ret = dict_set_dynstr_with_alloc (peer_data, buf, snap->snapname);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "Unable to set snapname for snap %s",
-                        snap->snapname);
-                goto out;
-        }
-
-        snprintf (buf, sizeof(buf), "%s.snap_id", prefix);
-        ret = dict_set_dynstr_with_alloc (peer_data, buf,
-                                          uuid_utoa (snap->snap_id));
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "Unable to set snap_id for snap %s",
-                        snap->snapname);
-                goto out;
-        }
-
-        if (snap->description) {
-                snprintf (buf, sizeof(buf), "%s.snapid", prefix);
-                ret = dict_set_dynstr_with_alloc (peer_data, buf,
-                                                  snap->description);
-                if (ret) {
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "Unable to set description for snap %s",
-                                snap->snapname);
-                        goto out;
-                }
-        }
-
-        snprintf (buf, sizeof(buf), "%s.time_stamp", prefix);
-        ret = dict_set_int64 (peer_data, buf, (int64_t)snap->time_stamp);
+    char buf[64] = "";
+    char prefix[32] = "";
+    int32_t ret = -1;
+    int32_t volcount = 0;
+    glusterd_volinfo_t *volinfo = NULL;
+    glusterd_brickinfo_t *brickinfo = NULL;
+    gf_boolean_t host_bricks = _gf_false;
+    xlator_t *this = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+    GF_ASSERT(snap);
+    GF_ASSERT(peer_data);
+
+    snprintf(prefix, sizeof(prefix), "snap%d", snap_count);
+
+    cds_list_for_each_entry(volinfo, &snap->volumes, vol_list)
+    {
+        volcount++;
+        ret = glusterd_add_volume_to_dict(volinfo, peer_data, volcount, prefix);
         if (ret) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "Unable to set time_stamp for snap %s",
-                        snap->snapname);
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+                   "Failed to add snap:%s volume:%s "
+                   "to peer_data dict for handshake",
+                   snap->snapname, volinfo->volname);
+            goto out;
+        }
+
+        if (glusterd_is_volume_quota_enabled(volinfo)) {
+            ret = glusterd_vol_add_quota_conf_to_dict(volinfo, peer_data,
+                                                      volcount, prefix);
+            if (ret) {
+                gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+                       "Failed to add quota conf for "
+                       "snap:%s volume:%s to peer_data "
+                       "dict for handshake",
+                       snap->snapname, volinfo->volname);
                 goto out;
+            }
         }
 
-        snprintf (buf, sizeof(buf), "%s.snap_restored", prefix);
-        ret = dict_set_int8 (peer_data, buf, snap->snap_restored);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "Unable to set snap_restored for snap %s",
-                        snap->snapname);
-                goto out;
-        }
-
-        snprintf (buf, sizeof(buf), "%s.snap_status", prefix);
-        ret = dict_set_int32 (peer_data, buf, snap->snap_status);
+        cds_list_for_each_entry(brickinfo, &volinfo->bricks, brick_list)
+        {
+            if (!gf_uuid_compare(brickinfo->uuid, MY_UUID)) {
+                host_bricks = _gf_true;
+                break;
+            }
+        }
+    }
+
+    snprintf(buf, sizeof(buf), "%s.host_bricks", prefix);
+    ret = dict_set_int8(peer_data, buf, (int8_t)host_bricks);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+               "Unable to set host_bricks for snap %s", snap->snapname);
+        goto out;
+    }
+
+    snprintf(buf, sizeof(buf), "%s.volcount", prefix);
+    ret = dict_set_int32(peer_data, buf, volcount);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+               "Unable to set volcount for snap %s", snap->snapname);
+        goto out;
+    }
+
+    snprintf(buf, sizeof(buf), "%s.snapname", prefix);
+    ret = dict_set_dynstr_with_alloc(peer_data, buf, snap->snapname);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+               "Unable to set snapname for snap %s", snap->snapname);
+        goto out;
+    }
+
+    snprintf(buf, sizeof(buf), "%s.snap_id", prefix);
+    ret = dict_set_dynstr_with_alloc(peer_data, buf, uuid_utoa(snap->snap_id));
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+               "Unable to set snap_id for snap %s", snap->snapname);
+        goto out;
+    }
+
+    if (snap->description) {
+        snprintf(buf, sizeof(buf), "%s.description", prefix);
+        ret = dict_set_dynstr_with_alloc(peer_data, buf, snap->description);
         if (ret) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "Unable to set snap_status for snap %s",
-                        snap->snapname);
-                goto out;
-        }
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+                   "Unable to set description for snap %s", snap->snapname);
+            goto out;
+        }
+    }
+
+    snprintf(buf, sizeof(buf), "%s.time_stamp", prefix);
+    ret = dict_set_int64(peer_data, buf, (int64_t)snap->time_stamp);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+               "Unable to set time_stamp for snap %s", snap->snapname);
+        goto out;
+    }
+
+    snprintf(buf, sizeof(buf), "%s.snap_restored", prefix);
+    ret = dict_set_int8(peer_data, buf, snap->snap_restored);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+               "Unable to set snap_restored for snap %s", snap->snapname);
+        goto out;
+    }
+
+    snprintf(buf, sizeof(buf), "%s.snap_status", prefix);
+    ret = dict_set_int32(peer_data, buf, snap->snap_status);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+               "Unable to set snap_status for snap %s", snap->snapname);
+        goto out;
+    }
 out:
-        gf_log (this->name, GF_LOG_TRACE, "Returning %d", ret);
-        return ret;
+    gf_msg_trace(this->name, 0, "Returning %d", ret);
+    return ret;
 }
 
 int32_t
-glusterd_add_snapshots_to_export_dict (dict_t *peer_data)
+glusterd_add_snapshots_to_export_dict(dict_t *peer_data)
 {
-        int32_t                 snap_count = 0;
-        int32_t                 ret        = -1;
-        glusterd_conf_t        *priv       = NULL;
-        glusterd_snap_t        *snap       = NULL;
-        xlator_t               *this       = NULL;
-
-        this = THIS;
-        GF_ASSERT (this);
-        priv = this->private;
-        GF_ASSERT (priv);
-        GF_ASSERT (peer_data);
-
-        cds_list_for_each_entry (snap, &priv->snapshots, snap_list) {
-                snap_count++;
-                ret = glusterd_add_snap_to_dict (snap, peer_data, snap_count);
-                if (ret) {
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "Failed to add snap(%s) to the "
-                                " peer_data dict for handshake",
-                                snap->snapname);
-                        goto out;
-                }
-        }
-
-        ret = dict_set_int32 (peer_data, "snap_count", snap_count);
+    int32_t snap_count = 0;
+    int32_t ret = -1;
+    glusterd_conf_t *priv = NULL;
+    glusterd_snap_t *snap = NULL;
+    xlator_t *this = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+    priv = this->private;
+    GF_ASSERT(priv);
+    GF_ASSERT(peer_data);
+
+    cds_list_for_each_entry(snap, &priv->snapshots, snap_list)
+    {
+        snap_count++;
+        ret = glusterd_add_snap_to_dict(snap, peer_data, snap_count);
         if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "Failed to set snap_count");
-                goto out;
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+                   "Failed to add snap(%s) to the "
+                   " peer_data dict for handshake",
+                   snap->snapname);
+            goto out;
         }
+    }
+
+    ret = dict_set_int32(peer_data, "snap_count", snap_count);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+               "Failed to set snap_count");
+        goto out;
+    }
 
 out:
-        gf_log (this->name, GF_LOG_TRACE, "Returning %d", ret);
-        return ret;
+    gf_msg_trace(this->name, 0, "Returning %d", ret);
+    return ret;
 }
 
 /* Imports the snapshot details of a brick if required and available
@@ -736,75 +821,80 @@ out:
  * Snapshot details will be imported only if the cluster op-version is >= 4
  */
 int
-gd_import_new_brick_snap_details (dict_t *dict, char *prefix,
-                                  glusterd_brickinfo_t *brickinfo)
+gd_import_new_brick_snap_details(dict_t *dict, char *prefix,
+                                 glusterd_brickinfo_t *brickinfo)
 {
-        int              ret         = -1;
-        xlator_t        *this        = NULL;
-        glusterd_conf_t *conf        = NULL;
-        char             key[512]    = {0,};
-        char            *snap_device = NULL;
-        char            *fs_type     = NULL;
-        char            *mnt_opts    = NULL;
-        char            *mount_dir   = NULL;
-
-        this = THIS;
-        GF_ASSERT (this != NULL);
-        conf = this->private;
-        GF_VALIDATE_OR_GOTO (this->name, (conf != NULL), out);
-
-        GF_VALIDATE_OR_GOTO (this->name, (dict != NULL), out);
-        GF_VALIDATE_OR_GOTO (this->name, (prefix != NULL), out);
-        GF_VALIDATE_OR_GOTO (this->name, (brickinfo != NULL), out);
-
-        if (conf->op_version < GD_OP_VERSION_3_6_0) {
-                ret = 0;
-                goto out;
-        }
-
-        snprintf (key, sizeof (key), "%s.snap_status", prefix);
-        ret = dict_get_int32 (dict, key, &brickinfo->snap_status);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "%s missing in payload", key);
-                goto out;
-        }
-
-        memset (key, 0, sizeof (key));
-        snprintf (key, sizeof (key), "%s.device_path", prefix);
-        ret = dict_get_str (dict, key, &snap_device);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "%s missing in payload", key);
-                goto out;
-        }
-        strcpy (brickinfo->device_path, snap_device);
-
-        snprintf (key, sizeof (key), "%s.fs_type", prefix);
-        ret = dict_get_str (dict, key, &fs_type);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "%s missing in payload", key);
-                goto out;
-        }
-        strcpy (brickinfo->fstype, fs_type);
-
-        snprintf (key, sizeof (key), "%s.mnt_opts", prefix);
-        ret = dict_get_str (dict, key, &mnt_opts);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "%s missing in payload", key);
-                goto out;
-        }
-        strcpy (brickinfo->mnt_opts, mnt_opts);
-
-        memset (key, 0, sizeof (key));
-        snprintf (key, sizeof (key), "%s.mount_dir", prefix);
-        ret = dict_get_str (dict, key, &mount_dir);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "%s missing in payload", key);
-                goto out;
-        }
-        strcpy (brickinfo->mount_dir, mount_dir);
+    int ret = -1;
+    xlator_t *this = NULL;
+    glusterd_conf_t *conf = NULL;
+    char key[512] = {
+        0,
+    };
+    char *snap_device = NULL;
+    char *fs_type = NULL;
+    char *mnt_opts = NULL;
+    char *mount_dir = NULL;
+
+    this = THIS;
+    GF_ASSERT(this != NULL);
+    conf = this->private;
+    GF_VALIDATE_OR_GOTO(this->name, (conf != NULL), out);
+
+    GF_VALIDATE_OR_GOTO(this->name, (dict != NULL), out);
+    GF_VALIDATE_OR_GOTO(this->name, (prefix != NULL), out);
+    GF_VALIDATE_OR_GOTO(this->name, (brickinfo != NULL), out);
+
+    if (conf->op_version < GD_OP_VERSION_3_6_0) {
+        ret = 0;
+        goto out;
+    }
+
+    snprintf(key, sizeof(key), "%s.snap_status", prefix);
+    ret = dict_get_int32(dict, key, &brickinfo->snap_status);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+               "%s missing in payload", key);
+        goto out;
+    }
+
+    snprintf(key, sizeof(key), "%s.device_path", prefix);
+    ret = dict_get_str(dict, key, &snap_device);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+               "%s missing in payload", key);
+        goto out;
+    }
+    gf_strncpy(brickinfo->device_path, snap_device,
+               sizeof(brickinfo->device_path));
+    snprintf(key, sizeof(key), "%s.fs_type", prefix);
+    ret = dict_get_str(dict, key, &fs_type);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+               "%s missing in payload", key);
+        goto out;
+    }
+    gf_strncpy(brickinfo->fstype, fs_type, sizeof(brickinfo->fstype));
+
+    snprintf(key, sizeof(key), "%s.mnt_opts", prefix);
+    ret = dict_get_str(dict, key, &mnt_opts);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+               "%s missing in payload", key);
+        goto out;
+    }
+    gf_strncpy(brickinfo->mnt_opts, mnt_opts, sizeof(brickinfo->mnt_opts));
+
+    snprintf(key, sizeof(key), "%s.mount_dir", prefix);
+    ret = dict_get_str(dict, key, &mount_dir);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+               "%s missing in payload", key);
+        goto out;
+    }
+    gf_strncpy(brickinfo->mount_dir, mount_dir, sizeof(brickinfo->mount_dir));
 
 out:
-        return ret;
+    return ret;
 }
 
 /*
@@ -815,319 +905,391 @@ out:
  * supported.
  */
 int
-gd_import_volume_snap_details (dict_t *dict, glusterd_volinfo_t *volinfo,
-                               char *prefix, char *volname)
+gd_import_volume_snap_details(dict_t *dict, glusterd_volinfo_t *volinfo,
+                              char *prefix, char *volname)
 {
-        int              ret           = -1;
-        xlator_t        *this          = NULL;
-        glusterd_conf_t *conf          = NULL;
-        char             key[256]      = {0,};
-        char            *restored_snap = NULL;
-
-        this = THIS;
-        GF_ASSERT (this != NULL);
-        conf = this->private;
-        GF_VALIDATE_OR_GOTO (this->name, (conf != NULL), out);
-
-        GF_VALIDATE_OR_GOTO (this->name, (dict != NULL), out);
-        GF_VALIDATE_OR_GOTO (this->name, (volinfo != NULL), out);
-        GF_VALIDATE_OR_GOTO (this->name, (prefix != NULL), out);
-        GF_VALIDATE_OR_GOTO (this->name, (volname != NULL), out);
-
-        if (conf->op_version < GD_OP_VERSION_3_6_0) {
-                ret = 0;
-                goto out;
-        }
-
-        snprintf (key, sizeof (key), "%s.is_snap_volume", prefix);
-        ret = dict_get_uint32 (dict, key, &volinfo->is_snap_volume);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "%s missing in payload "
-                        "for %s", key, volname);
-                goto out;
-        }
-
-        memset (key, 0, sizeof (key));
-        snprintf (key, sizeof (key), "%s.restored_from_snap", prefix);
-        ret = dict_get_str (dict, key, &restored_snap);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "%s missing in payload "
-                        "for %s", key, volname);
-                goto out;
-        }
-
-        gf_uuid_parse (restored_snap, volinfo->restored_from_snap);
-
-        memset (key, 0, sizeof (key));
-        snprintf (key, sizeof (key), "%s.snap-max-hard-limit", prefix);
-        ret = dict_get_uint64 (dict, key,
-                               &volinfo->snap_max_hard_limit);
-        if (ret)
-                gf_log (this->name, GF_LOG_ERROR, "%s missing in payload "
-                        "for %s", key, volname);
+    int ret = -1;
+    xlator_t *this = NULL;
+    glusterd_conf_t *conf = NULL;
+    char key[256] = {
+        0,
+    };
+    char *restored_snap = NULL;
+
+    this = THIS;
+    GF_ASSERT(this != NULL);
+    conf = this->private;
+    GF_VALIDATE_OR_GOTO(this->name, (conf != NULL), out);
+
+    GF_VALIDATE_OR_GOTO(this->name, (dict != NULL), out);
+    GF_VALIDATE_OR_GOTO(this->name, (volinfo != NULL), out);
+    GF_VALIDATE_OR_GOTO(this->name, (prefix != NULL), out);
+    GF_VALIDATE_OR_GOTO(this->name, (volname != NULL), out);
+
+    if (conf->op_version < GD_OP_VERSION_3_6_0) {
+        ret = 0;
+        goto out;
+    }
+
+    snprintf(key, sizeof(key), "%s.is_snap_volume", prefix);
+    uint32_t is_snap_int;
+    ret = dict_get_uint32(dict, key, &is_snap_int);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+               "%s missing in payload "
+               "for %s",
+               key, volname);
+        goto out;
+    }
+    volinfo->is_snap_volume = (is_snap_int != 0);
+
+    snprintf(key, sizeof(key), "%s.restored_from_snap", prefix);
+    ret = dict_get_str(dict, key, &restored_snap);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+               "%s missing in payload "
+               "for %s",
+               key, volname);
+        goto out;
+    }
+
+    gf_uuid_parse(restored_snap, volinfo->restored_from_snap);
+
+    snprintf(key, sizeof(key), "%s.snap-max-hard-limit", prefix);
+    ret = dict_get_uint64(dict, key, &volinfo->snap_max_hard_limit);
+    if (ret)
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+               "%s missing in payload "
+               "for %s",
+               key, volname);
 out:
-        return ret;
+    return ret;
 }
 
 int32_t
-glusterd_perform_missed_op (glusterd_snap_t *snap, int32_t op)
+glusterd_perform_missed_op(glusterd_snap_t *snap, int32_t op)
 {
-        dict_t                  *dict           = NULL;
-        int32_t                  ret            = -1;
-        glusterd_conf_t         *priv           = NULL;
-        glusterd_volinfo_t      *snap_volinfo   = NULL;
-        glusterd_volinfo_t      *volinfo        = NULL;
-        glusterd_volinfo_t      *tmp            = NULL;
-        xlator_t                *this           = NULL;
-        uuid_t                   null_uuid      = {0};
-        char                    *parent_volname = NULL;
-
-        this = THIS;
-        GF_ASSERT (this);
-
-        priv = this->private;
-        GF_ASSERT (priv);
-        GF_ASSERT (snap);
-
-        dict = dict_new();
-        if (!dict) {
-                gf_log (this->name, GF_LOG_ERROR, "Unable to create dict");
-                ret = -1;
+    dict_t *dict = NULL;
+    int32_t ret = -1;
+    glusterd_conf_t *priv = NULL;
+    glusterd_volinfo_t *snap_volinfo = NULL;
+    glusterd_volinfo_t *volinfo = NULL;
+    glusterd_volinfo_t *tmp = NULL;
+    xlator_t *this = NULL;
+    uuid_t null_uuid = {0};
+    char *parent_volname = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+
+    priv = this->private;
+    GF_ASSERT(priv);
+    GF_ASSERT(snap);
+
+    dict = dict_new();
+    if (!dict) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_CREATE_FAIL,
+               "Unable to create dict");
+        ret = -1;
+        goto out;
+    }
+
+    switch (op) {
+        case GF_SNAP_OPTION_TYPE_DELETE:
+            ret = glusterd_snap_remove(dict, snap, _gf_true, _gf_false,
+                                       _gf_false);
+            if (ret) {
+                gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_SNAP_REMOVE_FAIL,
+                       "Failed to remove snap");
                 goto out;
-        }
+            }
 
-        switch (op) {
-        case GF_SNAP_OPTION_TYPE_DELETE:
-                ret = glusterd_snap_remove (dict, snap, _gf_true, _gf_false);
+            break;
+        case GF_SNAP_OPTION_TYPE_RESTORE:
+            cds_list_for_each_entry_safe(snap_volinfo, tmp, &snap->volumes,
+                                         vol_list)
+            {
+                parent_volname = gf_strdup(snap_volinfo->parent_volname);
+                if (!parent_volname)
+                    goto out;
+
+                ret = glusterd_volinfo_find(parent_volname, &volinfo);
                 if (ret) {
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "Failed to remove snap");
-                        goto out;
+                    gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_VOLINFO_GET_FAIL,
+                           "Could not get volinfo of %s", parent_volname);
+                    goto out;
                 }
 
-                break;
-        case GF_SNAP_OPTION_TYPE_RESTORE:
-                cds_list_for_each_entry_safe (snap_volinfo, tmp, &snap->volumes,
-                                              vol_list) {
-                        parent_volname = gf_strdup
-                                            (snap_volinfo->parent_volname);
-                        if (!parent_volname)
-                                goto out;
-
-                        ret = glusterd_volinfo_find (parent_volname, &volinfo);
-                        if (ret) {
-                                gf_log (this->name, GF_LOG_ERROR,
-                                        "Could not get volinfo of %s",
-                                        parent_volname);
-                                goto out;
-                        }
-
-                        volinfo->version--;
-                        gf_uuid_copy (volinfo->restored_from_snap, null_uuid);
-
-                        /* gd_restore_snap_volume() uses the dict and volcount
-                         * to fetch snap brick info from other nodes, which were
-                         * collected during prevalidation. As this is an ad-hoc
-                         * op and only local node's data matter, hence sending
-                         * volcount as 0 and re-using the same dict because we
-                         * need not record any missed creates in the rsp_dict.
-                         */
-                        ret = gd_restore_snap_volume (dict, dict, volinfo,
-                                                      snap_volinfo, 0);
-                        if (ret) {
-                                gf_log (this->name, GF_LOG_ERROR,
-                                        "Failed to restore snap for %s",
-                                        snap->snapname);
-                                volinfo->version++;
-                                goto out;
-                        }
-
-                        /* Restore is successful therefore delete the original
-                         * volume's volinfo. If the volinfo is already restored
-                         * then we should delete the backend LVMs */
-                        if (!gf_uuid_is_null (volinfo->restored_from_snap)) {
-                                ret = glusterd_lvm_snapshot_remove (dict,
-                                                                    volinfo);
-                                if (ret) {
-                                        gf_log (this->name, GF_LOG_ERROR,
-                                                "Failed to remove LVM backend");
-                                        goto out;
-                                }
-                        }
-
-                        /* Detach the volinfo from priv->volumes, so that no new
-                         * command can ref it any more and then unref it.
-                         */
-                        cds_list_del_init (&volinfo->vol_list);
-                        glusterd_volinfo_unref (volinfo);
-
-                        ret = glusterd_snapshot_restore_cleanup (dict,
-                                                                 parent_volname,
-                                                                 snap);
-                        if (ret) {
-                                gf_log (this->name, GF_LOG_ERROR,
-                                        "Failed to perform snapshot restore "
-                                        "cleanup for %s volume",
-                                        parent_volname);
-                                goto out;
-                        }
-
-                        GF_FREE (parent_volname);
-                        parent_volname = NULL;
+                volinfo->version--;
+                gf_uuid_copy(volinfo->restored_from_snap, null_uuid);
+
+                /* gd_restore_snap_volume() uses the dict and volcount
+                 * to fetch snap brick info from other nodes, which were
+                 * collected during prevalidation. As this is an ad-hoc
+                 * op and only local node's data matter, hence sending
+                 * volcount as 0 and re-using the same dict because we
+                 * need not record any missed creates in the rsp_dict.
+                 */
+                ret = gd_restore_snap_volume(dict, dict, volinfo, snap_volinfo,
+                                             0);
+                if (ret) {
+                    gf_msg(this->name, GF_LOG_ERROR, 0,
+                           GD_MSG_SNAP_RESTORE_FAIL,
+                           "Failed to restore snap for %s", snap->snapname);
+                    volinfo->version++;
+                    goto out;
                 }
 
-                break;
-        default:
-                /* The entry must be a create, delete, or
-                 * restore entry
+                /* Restore is successful therefore delete the original
+                 * volume's volinfo. If the volinfo is already restored
+                 * then we should delete the backend LVMs */
+                if (!gf_uuid_is_null(volinfo->restored_from_snap)) {
+                    ret = glusterd_lvm_snapshot_remove(dict, volinfo);
+                    if (ret) {
+                        gf_msg(this->name, GF_LOG_ERROR, 0,
+                               GD_MSG_SNAP_REMOVE_FAIL,
+                               "Failed to remove LVM backend");
+                        goto out;
+                    }
+                }
+
+                /* Detach the volinfo from priv->volumes, so that no new
+                 * command can ref it any more and then unref it.
                  */
-                gf_log (this->name, GF_LOG_ERROR, "Invalid missed snap entry");
-                ret = -1;
-                goto out;
-        }
+                cds_list_del_init(&volinfo->vol_list);
+                glusterd_volinfo_unref(volinfo);
 
-out:
-        dict_unref (dict);
-        if (parent_volname) {
-                GF_FREE (parent_volname);
+                ret = glusterd_snapshot_restore_cleanup(dict, parent_volname,
+                                                        snap);
+                if (ret) {
+                    gf_msg(this->name, GF_LOG_ERROR, 0,
+                           GD_MSG_SNAP_CLEANUP_FAIL,
+                           "Failed to perform snapshot restore "
+                           "cleanup for %s volume",
+                           parent_volname);
+                    goto out;
+                }
+
+                GF_FREE(parent_volname);
                 parent_volname = NULL;
-        }
+            }
+
+            break;
+        default:
+            /* The entry must be a create, delete, or
+             * restore entry
+             */
+            gf_msg(this->name, GF_LOG_ERROR, EINVAL, GD_MSG_INVALID_ENTRY,
+                   "Invalid missed snap entry");
+            ret = -1;
+            goto out;
+    }
 
-        gf_log (this->name, GF_LOG_TRACE, "Returning %d", ret);
-        return ret;
+out:
+    dict_unref(dict);
+    if (parent_volname) {
+        GF_FREE(parent_volname);
+        parent_volname = NULL;
+    }
+
+    gf_msg_trace(this->name, 0, "Returning %d", ret);
+    return ret;
 }
 
 /* Perform missed deletes and restores on this node */
 int32_t
-glusterd_perform_missed_snap_ops ()
+glusterd_perform_missed_snap_ops()
 {
-        int32_t                      ret                 = -1;
-        int32_t                      op_status           = -1;
-        glusterd_conf_t             *priv                = NULL;
-        glusterd_missed_snap_info   *missed_snapinfo     = NULL;
-        glusterd_snap_op_t          *snap_opinfo         = NULL;
-        glusterd_snap_t             *snap                = NULL;
-        uuid_t                       snap_uuid           = {0,};
-        xlator_t                    *this                = NULL;
-
-        this = THIS;
-        GF_ASSERT (this);
-
-        priv = this->private;
-        GF_ASSERT (priv);
-
-        cds_list_for_each_entry (missed_snapinfo, &priv->missed_snaps_list,
-                                 missed_snaps) {
-                /* If the pending snap_op is not for this node then continue */
-                if (strcmp (missed_snapinfo->node_uuid, uuid_utoa (MY_UUID)))
-                        continue;
-
-                /* Find the snap id */
-                gf_uuid_parse (missed_snapinfo->snap_uuid, snap_uuid);
-                snap = NULL;
-                snap = glusterd_find_snap_by_id (snap_uuid);
-                if (!snap) {
-                        /* If the snap is not found, then a delete or a
-                         * restore can't be pending on that snap_uuid.
-                         */
-                        gf_log (this->name, GF_LOG_DEBUG,
-                                "Not a pending delete or restore op");
-                        continue;
+    int32_t ret = -1;
+    int32_t op_status = -1;
+    glusterd_conf_t *priv = NULL;
+    glusterd_missed_snap_info *missed_snapinfo = NULL;
+    glusterd_snap_op_t *snap_opinfo = NULL;
+    glusterd_snap_t *snap = NULL;
+    uuid_t snap_uuid = {
+        0,
+    };
+    xlator_t *this = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+
+    priv = this->private;
+    GF_ASSERT(priv);
+
+    cds_list_for_each_entry(missed_snapinfo, &priv->missed_snaps_list,
+                            missed_snaps)
+    {
+        /* If the pending snap_op is not for this node then continue */
+        if (strcmp(missed_snapinfo->node_uuid, uuid_utoa(MY_UUID)))
+            continue;
+
+        /* Find the snap id */
+        gf_uuid_parse(missed_snapinfo->snap_uuid, snap_uuid);
+        snap = NULL;
+        snap = glusterd_find_snap_by_id(snap_uuid);
+        if (!snap) {
+            /* If the snap is not found, then a delete or a
+             * restore can't be pending on that snap_uuid.
+             */
+            gf_msg_debug(this->name, 0, "Not a pending delete or restore op");
+            continue;
+        }
+
+        op_status = GD_MISSED_SNAP_PENDING;
+        cds_list_for_each_entry(snap_opinfo, &missed_snapinfo->snap_ops,
+                                snap_ops_list)
+        {
+            /* If the snap_op is create or its status is
+             * GD_MISSED_SNAP_DONE then continue
+             */
+            if ((snap_opinfo->status == GD_MISSED_SNAP_DONE) ||
+                (snap_opinfo->op == GF_SNAP_OPTION_TYPE_CREATE))
+                continue;
+
+            /* Perform the actual op for the first time for
+             * this snap, and mark the snap_status as
+             * GD_MISSED_SNAP_DONE. For other entries for the same
+             * snap, just mark the entry as done.
+             */
+            if (op_status == GD_MISSED_SNAP_PENDING) {
+                ret = glusterd_perform_missed_op(snap, snap_opinfo->op);
+                if (ret) {
+                    gf_msg(this->name, GF_LOG_ERROR, 0,
+                           GD_MSG_SNAPSHOT_OP_FAILED,
+                           "Failed to perform missed snap op");
+                    goto out;
                 }
+                op_status = GD_MISSED_SNAP_DONE;
+            }
 
-                op_status = GD_MISSED_SNAP_PENDING;
-                cds_list_for_each_entry (snap_opinfo,
-                                         &missed_snapinfo->snap_ops,
-                                         snap_ops_list) {
-                        /* If the snap_op is create or its status is
-                         * GD_MISSED_SNAP_DONE then continue
-                         */
-                        if ((snap_opinfo->status == GD_MISSED_SNAP_DONE) ||
-                            (snap_opinfo->op == GF_SNAP_OPTION_TYPE_CREATE))
-                                continue;
-
-                        /* Perform the actual op for the first time for
-                         * this snap, and mark the snap_status as
-                         * GD_MISSED_SNAP_DONE. For other entries for the same
-                         * snap, just mark the entry as done.
-                         */
-                        if (op_status == GD_MISSED_SNAP_PENDING) {
-                                ret = glusterd_perform_missed_op
-                                                             (snap,
-                                                              snap_opinfo->op);
-                                if (ret) {
-                                        gf_log (this->name, GF_LOG_ERROR,
-                                                "Failed to perform missed snap op");
-                                        goto out;
-                                }
-                                op_status = GD_MISSED_SNAP_DONE;
-                        }
-
-                        snap_opinfo->status = GD_MISSED_SNAP_DONE;
-                }
+            snap_opinfo->status = GD_MISSED_SNAP_DONE;
         }
+    }
 
-        ret = 0;
+    ret = 0;
 out:
-        gf_log (this->name, GF_LOG_TRACE, "Returning %d", ret);
-        return ret;
+    gf_msg_trace(this->name, 0, "Returning %d", ret);
+    return ret;
 }
 
 /* Import friend volumes missed_snap_list and update *
  * missed_snap_list if need be */
 int32_t
-glusterd_import_friend_missed_snap_list (dict_t *peer_data)
+glusterd_import_friend_missed_snap_list(dict_t *peer_data)
 {
-        int32_t                      missed_snap_count     = -1;
-        int32_t                      ret                   = -1;
-        glusterd_conf_t             *priv                  = NULL;
-        xlator_t                    *this                  = NULL;
-
-        this = THIS;
-        GF_ASSERT (this);
-        GF_ASSERT (peer_data);
-
-        priv = this->private;
-        GF_ASSERT (priv);
+    int32_t missed_snap_count = -1;
+    int32_t ret = -1;
+    glusterd_conf_t *priv = NULL;
+    xlator_t *this = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+    GF_ASSERT(peer_data);
+
+    priv = this->private;
+    GF_ASSERT(priv);
+
+    /* Add the friends missed_snaps entries to the in-memory list */
+    ret = dict_get_int32(peer_data, "missed_snap_count", &missed_snap_count);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_INFO, 0, GD_MSG_MISSED_SNAP_GET_FAIL,
+               "No missed snaps");
+        ret = 0;
+        goto out;
+    }
+
+    ret = glusterd_add_missed_snaps_to_list(peer_data, missed_snap_count);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_MISSED_SNAP_LIST_STORE_FAIL,
+               "Failed to add missed snaps to list");
+        goto out;
+    }
+
+    ret = glusterd_perform_missed_snap_ops();
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_SNAPSHOT_OP_FAILED,
+               "Failed to perform snap operations");
+        /* Not going to out at this point coz some *
+         * missed ops might have been performed. We *
+         * need to persist the current list *
+         */
+    }
 
-        /* Add the friends missed_snaps entries to the in-memory list */
-        ret = dict_get_int32 (peer_data, "missed_snap_count",
-                              &missed_snap_count);
-        if (ret) {
-                gf_log (this->name, GF_LOG_INFO,
-                        "No missed snaps");
-                ret = 0;
-                goto out;
-        }
+    ret = glusterd_store_update_missed_snaps();
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_MISSED_SNAP_LIST_STORE_FAIL,
+               "Failed to update missed_snaps_list");
+        goto out;
+    }
 
-        ret = glusterd_add_missed_snaps_to_list (peer_data,
-                                                 missed_snap_count);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "Failed to add missed snaps to list");
-                goto out;
-        }
+out:
+    gf_msg_trace(this->name, 0, "Returning %d", ret);
+    return ret;
+}
 
-        ret = glusterd_perform_missed_snap_ops ();
+/*
+ * This function will set boolean "conflict" to true if peer snap
+ * has a version greater than snap version of local node. Otherwise
+ * boolean "conflict" will be set to false.
+ */
+int
+glusterd_check_peer_has_higher_snap_version(dict_t *peer_data,
+                                            char *peer_snap_name, int volcount,
+                                            gf_boolean_t *conflict,
+                                            char *prefix, glusterd_snap_t *snap,
+                                            char *hostname)
+{
+    glusterd_volinfo_t *snap_volinfo = NULL;
+    char key[256] = {0};
+    int version = 0, i = 0;
+    int ret = 0;
+    xlator_t *this = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+    GF_ASSERT(snap);
+    GF_ASSERT(peer_data);
+
+    for (i = 1; i <= volcount; i++) {
+        snprintf(key, sizeof(key), "%s%d.version", prefix, i);
+        ret = dict_get_int32(peer_data, key, &version);
         if (ret) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "Failed to perform snap operations");
-                /* Not going to out at this point coz some *
-                 * missed ops might have been performed. We *
-                 * need to persist the current list *
-                 */
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+                   "failed to get "
+                   "version of snap volume = %s",
+                   peer_snap_name);
+            return -1;
         }
 
-        ret = glusterd_store_update_missed_snaps ();
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "Failed to update missed_snaps_list");
-                goto out;
+        /* TODO : As of now there is only one volume in snapshot.
+         * Change this when multiple volume snapshot is introduced
+         */
+        snap_volinfo = cds_list_entry(snap->volumes.next, glusterd_volinfo_t,
+                                      vol_list);
+        if (!snap_volinfo) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_VOLINFO_GET_FAIL,
+                   "Failed to get snap "
+                   "volinfo %s",
+                   snap->snapname);
+            return -1;
+        }
+
+        if (version > snap_volinfo->version) {
+            /* Mismatch detected */
+            gf_msg(this->name, GF_LOG_INFO, 0, GD_MSG_VOL_VERS_MISMATCH,
+                   "Version of volume %s differ. "
+                   "local version = %d, remote version = %d "
+                   "on peer %s",
+                   snap_volinfo->volname, snap_volinfo->version, version,
+                   hostname);
+            *conflict = _gf_true;
+            break;
+        } else {
+            *conflict = _gf_false;
         }
-
-out:
-        gf_log (this->name, GF_LOG_TRACE, "Returning %d", ret);
-        return ret;
+    }
+    return 0;
 }
 
 /* Check for the peer_snap_name in the list of existing snapshots.
@@ -1140,355 +1302,404 @@ out:
  * and snap to NULL.
  */
 void
-glusterd_is_peer_snap_conflicting (char *peer_snap_name, char *peer_snap_id,
-                                   gf_boolean_t *conflict,
-                                   glusterd_snap_t **snap, char *hostname)
+glusterd_is_peer_snap_conflicting(char *peer_snap_name, char *peer_snap_id,
+                                  gf_boolean_t *conflict,
+                                  glusterd_snap_t **snap, char *hostname)
 {
-        uuid_t       peer_snap_uuid = {0,};
-        xlator_t    *this           = NULL;
-
-        this = THIS;
-        GF_ASSERT (this);
-        GF_ASSERT (peer_snap_name);
-        GF_ASSERT (peer_snap_id);
-        GF_ASSERT (conflict);
-        GF_ASSERT (snap);
-        GF_ASSERT (hostname);
-
-        *snap = glusterd_find_snap_by_name (peer_snap_name);
-        if (*snap) {
-                gf_uuid_parse (peer_snap_id, peer_snap_uuid);
-                if (!gf_uuid_compare (peer_snap_uuid, (*snap)->snap_id)) {
-                        /* Current node contains the same snap having
-                         * the same snapname and snap_id
-                         */
-                        gf_log (this->name, GF_LOG_DEBUG,
-                                "Snapshot %s from peer %s present in "
-                                "localhost", peer_snap_name, hostname);
-                        *conflict = _gf_false;
-                } else {
-                        /* Current node contains the same snap having
-                         * the same snapname but different snap_id
-                         */
-                        gf_log (this->name, GF_LOG_DEBUG,
-                                "Snapshot %s from peer %s conflicts with "
-                                "snapshot in localhost", peer_snap_name,
-                                hostname);
-                        *conflict = _gf_true;
-                }
+    uuid_t peer_snap_uuid = {
+        0,
+    };
+    xlator_t *this = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+    GF_ASSERT(peer_snap_name);
+    GF_ASSERT(peer_snap_id);
+    GF_ASSERT(conflict);
+    GF_ASSERT(snap);
+    GF_ASSERT(hostname);
+
+    *snap = glusterd_find_snap_by_name(peer_snap_name);
+    if (*snap) {
+        gf_uuid_parse(peer_snap_id, peer_snap_uuid);
+        if (!gf_uuid_compare(peer_snap_uuid, (*snap)->snap_id)) {
+            /* Current node contains the same snap having
+             * the same snapname and snap_id
+             */
+            gf_msg_debug(this->name, 0,
+                         "Snapshot %s from peer %s present in "
+                         "localhost",
+                         peer_snap_name, hostname);
+            *conflict = _gf_false;
         } else {
-                /* Peer contains snapshots missing on the current node */
-                gf_log (this->name, GF_LOG_INFO,
-                        "Snapshot %s from peer %s missing on localhost",
-                        peer_snap_name, hostname);
-                *conflict = _gf_false;
-        }
+            /* Current node contains the same snap having
+             * the same snapname but different snap_id
+             */
+            gf_msg_debug(this->name, 0,
+                         "Snapshot %s from peer %s conflicts with "
+                         "snapshot in localhost",
+                         peer_snap_name, hostname);
+            *conflict = _gf_true;
+        }
+    } else {
+        /* Peer contains snapshots missing on the current node */
+        gf_msg(this->name, GF_LOG_INFO, 0, GD_MSG_MISSED_SNAP_PRESENT,
+               "Snapshot %s from peer %s missing on localhost", peer_snap_name,
+               hostname);
+        *conflict = _gf_false;
+    }
 }
 
 /* Check if the local node is hosting any bricks for the given snapshot */
 gf_boolean_t
-glusterd_are_snap_bricks_local (glusterd_snap_t *snap)
+glusterd_are_snap_bricks_local(glusterd_snap_t *snap)
 {
-        gf_boolean_t            is_local   = _gf_false;
-        glusterd_volinfo_t     *volinfo    = NULL;
-        glusterd_brickinfo_t   *brickinfo  = NULL;
-        xlator_t               *this       = NULL;
-
-        this = THIS;
-        GF_ASSERT (this);
-        GF_ASSERT (snap);
-
-        cds_list_for_each_entry (volinfo, &snap->volumes, vol_list) {
-                cds_list_for_each_entry (brickinfo, &volinfo->bricks,
-                                         brick_list) {
-                        if (!gf_uuid_compare (brickinfo->uuid, MY_UUID)) {
-                                is_local = _gf_true;
-                                goto out;
-                        }
-                }
+    gf_boolean_t is_local = _gf_false;
+    glusterd_volinfo_t *volinfo = NULL;
+    glusterd_brickinfo_t *brickinfo = NULL;
+    xlator_t *this = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+    GF_ASSERT(snap);
+
+    cds_list_for_each_entry(volinfo, &snap->volumes, vol_list)
+    {
+        cds_list_for_each_entry(brickinfo, &volinfo->bricks, brick_list)
+        {
+            if (!gf_uuid_compare(brickinfo->uuid, MY_UUID)) {
+                is_local = _gf_true;
+                goto out;
+            }
         }
+    }
 
 out:
-        gf_log (this->name, GF_LOG_TRACE, "Returning %d", is_local);
-        return is_local;
+    gf_msg_trace(this->name, 0, "Returning %d", is_local);
+    return is_local;
 }
 
 /* Check if the peer has missed any snap delete
  * or restore for the given snap_id
  */
 gf_boolean_t
-glusterd_peer_has_missed_snap_delete (uuid_t peerid, char *peer_snap_id)
+glusterd_peer_has_missed_snap_delete(uuid_t peerid, char *peer_snap_id)
 {
-        char                        *peer_uuid           = NULL;
-        gf_boolean_t                 missed_delete       = _gf_false;
-        glusterd_conf_t             *priv                = NULL;
-        glusterd_missed_snap_info   *missed_snapinfo     = NULL;
-        glusterd_snap_op_t          *snap_opinfo         = NULL;
-        xlator_t                    *this                = NULL;
-
-        this = THIS;
-        GF_ASSERT (this);
-        priv = this->private;
-        GF_ASSERT (priv);
-        GF_ASSERT (peer_snap_id);
-
-        peer_uuid = uuid_utoa (peerid);
-
-        cds_list_for_each_entry (missed_snapinfo, &priv->missed_snaps_list,
-                                 missed_snaps) {
-                /* Look for missed snap for the same peer, and
-                 * the same snap_id
-                 */
-                if ((!strcmp (peer_uuid, missed_snapinfo->node_uuid)) &&
-                    (!strcmp (peer_snap_id, missed_snapinfo->snap_uuid))) {
-                        /* Check if the missed snap's op is delete and the
-                         * status is pending
-                         */
-                        cds_list_for_each_entry (snap_opinfo,
-                                                 &missed_snapinfo->snap_ops,
-                                                 snap_ops_list) {
-                                if (((snap_opinfo->op ==
-                                              GF_SNAP_OPTION_TYPE_DELETE) ||
-                                     (snap_opinfo->op ==
-                                              GF_SNAP_OPTION_TYPE_RESTORE)) &&
-                                    (snap_opinfo->status ==
-                                             GD_MISSED_SNAP_PENDING)) {
-                                        missed_delete = _gf_true;
-                                        goto out;
-                                }
-                        }
+    char *peer_uuid = NULL;
+    gf_boolean_t missed_delete = _gf_false;
+    glusterd_conf_t *priv = NULL;
+    glusterd_missed_snap_info *missed_snapinfo = NULL;
+    glusterd_snap_op_t *snap_opinfo = NULL;
+    xlator_t *this = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+    priv = this->private;
+    GF_ASSERT(priv);
+    GF_ASSERT(peer_snap_id);
+
+    peer_uuid = uuid_utoa(peerid);
+
+    cds_list_for_each_entry(missed_snapinfo, &priv->missed_snaps_list,
+                            missed_snaps)
+    {
+        /* Look for missed snap for the same peer, and
+         * the same snap_id
+         */
+        if ((!strcmp(peer_uuid, missed_snapinfo->node_uuid)) &&
+            (!strcmp(peer_snap_id, missed_snapinfo->snap_uuid))) {
+            /* Check if the missed snap's op is delete and the
+             * status is pending
+             */
+            cds_list_for_each_entry(snap_opinfo, &missed_snapinfo->snap_ops,
+                                    snap_ops_list)
+            {
+                if (((snap_opinfo->op == GF_SNAP_OPTION_TYPE_DELETE) ||
+                     (snap_opinfo->op == GF_SNAP_OPTION_TYPE_RESTORE)) &&
+                    (snap_opinfo->status == GD_MISSED_SNAP_PENDING)) {
+                    missed_delete = _gf_true;
+                    goto out;
                 }
+            }
         }
+    }
 
 out:
-        gf_log (this->name, GF_LOG_TRACE, "Returning %d", missed_delete);
-        return missed_delete;
+    gf_msg_trace(this->name, 0, "Returning %d", missed_delete);
+    return missed_delete;
 }
 
-/* Genrate and store snap volfiles for imported snap object */
+/* Generate and store snap volfiles for imported snap object */
 int32_t
-glusterd_gen_snap_volfiles (glusterd_volinfo_t *snap_vol, char *peer_snap_name)
+glusterd_gen_snap_volfiles(glusterd_volinfo_t *snap_vol, char *peer_snap_name)
 {
-        int32_t                 ret              = -1;
-        xlator_t               *this             = NULL;
-        glusterd_volinfo_t     *parent_volinfo   = NULL;
-        glusterd_brickinfo_t   *brickinfo        = NULL;
-
-        this = THIS;
-        GF_ASSERT (this);
-        GF_ASSERT (snap_vol);
-        GF_ASSERT (peer_snap_name);
-
-        ret = glusterd_store_volinfo (snap_vol, GLUSTERD_VOLINFO_VER_AC_NONE);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "Failed to store snapshot "
-                        "volinfo (%s) for snap %s", snap_vol->volname,
-                        peer_snap_name);
-                goto out;
-        }
-
-        ret = generate_brick_volfiles (snap_vol);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "generating the brick volfiles for the "
-                        "snap %s failed", peer_snap_name);
-                goto out;
-        }
-
-        ret = generate_client_volfiles (snap_vol, GF_CLIENT_TRUSTED);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "generating the trusted client volfiles for "
-                        "the snap %s failed", peer_snap_name);
-                goto out;
-        }
-
-        ret = generate_client_volfiles (snap_vol, GF_CLIENT_OTHER);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "generating the client volfiles for the "
-                        "snap %s failed", peer_snap_name);
-                goto out;
-        }
-
-        ret = glusterd_volinfo_find (snap_vol->parent_volname,
-                                     &parent_volinfo);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "Parent volinfo "
-                        "not found for %s volume of snap %s",
-                        snap_vol->volname, peer_snap_name);
-                goto out;
-        }
-
-        glusterd_list_add_snapvol (parent_volinfo, snap_vol);
-
-        snap_vol->status = GLUSTERD_STATUS_STARTED;
-
-        ret = glusterd_store_volinfo (snap_vol, GLUSTERD_VOLINFO_VER_AC_NONE);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "Failed to store snap volinfo");
-                goto out;
-        }
+    int32_t ret = -1;
+    xlator_t *this = NULL;
+    glusterd_volinfo_t *parent_volinfo = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+    GF_ASSERT(snap_vol);
+    GF_ASSERT(peer_snap_name);
+
+    ret = glusterd_store_volinfo(snap_vol, GLUSTERD_VOLINFO_VER_AC_NONE);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_VOLINFO_SET_FAIL,
+               "Failed to store snapshot "
+               "volinfo (%s) for snap %s",
+               snap_vol->volname, peer_snap_name);
+        goto out;
+    }
+
+    ret = generate_brick_volfiles(snap_vol);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_VOLFILE_CREATE_FAIL,
+               "generating the brick volfiles for the "
+               "snap %s failed",
+               peer_snap_name);
+        goto out;
+    }
+
+    ret = generate_client_volfiles(snap_vol, GF_CLIENT_TRUSTED);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_VOLFILE_CREATE_FAIL,
+               "generating the trusted client volfiles for "
+               "the snap %s failed",
+               peer_snap_name);
+        goto out;
+    }
+
+    ret = generate_client_volfiles(snap_vol, GF_CLIENT_OTHER);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_VOLFILE_CREATE_FAIL,
+               "generating the client volfiles for the "
+               "snap %s failed",
+               peer_snap_name);
+        goto out;
+    }
+
+    ret = glusterd_volinfo_find(snap_vol->parent_volname, &parent_volinfo);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_VOLINFO_GET_FAIL,
+               "Parent volinfo "
+               "not found for %s volume of snap %s",
+               snap_vol->volname, peer_snap_name);
+        goto out;
+    }
+
+    glusterd_list_add_snapvol(parent_volinfo, snap_vol);
+
+    ret = glusterd_store_volinfo(snap_vol, GLUSTERD_VOLINFO_VER_AC_NONE);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_VOLINFO_SET_FAIL,
+               "Failed to store snap volinfo");
+        goto out;
+    }
 out:
-        gf_log (this->name, GF_LOG_TRACE, "Returning %d", ret);
-        return ret;
+    gf_msg_trace(this->name, 0, "Returning %d", ret);
+    return ret;
 }
 
 /* Import snapshot info from peer_data and add it to priv */
 int32_t
-glusterd_import_friend_snap (dict_t *peer_data, int32_t snap_count,
-                             char *peer_snap_name, char *peer_snap_id)
+glusterd_import_friend_snap(dict_t *peer_data, int32_t snap_count,
+                            char *peer_snap_name, char *peer_snap_id)
 {
-        char                 buf[NAME_MAX]    = "";
-        char                 prefix[NAME_MAX] = "";
-        dict_t              *dict             = NULL;
-        glusterd_snap_t     *snap             = NULL;
-        glusterd_volinfo_t  *snap_vol         = NULL;
-        glusterd_conf_t     *priv             = NULL;
-        int32_t              ret              = -1;
-        int32_t              volcount         = -1;
-        int32_t              i                = -1;
-        xlator_t            *this             = NULL;
-
-        this = THIS;
-        GF_ASSERT (this);
-        priv = this->private;
-        GF_ASSERT (priv);
-        GF_ASSERT (peer_data);
-        GF_ASSERT (peer_snap_name);
-        GF_ASSERT (peer_snap_id);
-
-        snprintf (prefix, sizeof(prefix), "snap%d", snap_count);
-
-        snap = glusterd_new_snap_object ();
-        if (!snap) {
-                gf_log (this->name, GF_LOG_ERROR, "Could not create "
-                        "the snap object for snap %s", peer_snap_name);
-                goto out;
-        }
-
-        dict = dict_new ();
-        if (!dict) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "Failed to create dict");
-                ret = -1;
-                goto out;
-        }
-
-        strcpy (snap->snapname, peer_snap_name);
-        gf_uuid_parse (peer_snap_id, snap->snap_id);
-
-        snprintf (buf, sizeof(buf), "%s.snapid", prefix);
-        ret = dict_get_str (peer_data, buf, &snap->description);
-
-        snprintf (buf, sizeof(buf), "%s.time_stamp", prefix);
-        ret = dict_get_int64 (peer_data, buf, &snap->time_stamp);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "Unable to get time_stamp for snap %s",
-                        peer_snap_name);
-                goto out;
-        }
-
-        snprintf (buf, sizeof(buf), "%s.snap_restored", prefix);
-        ret = dict_get_int8 (peer_data, buf, (int8_t *) &snap->snap_restored);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "Unable to get snap_restored for snap %s",
-                        peer_snap_name);
-                goto out;
-        }
-
-        snprintf (buf, sizeof(buf), "%s.snap_status", prefix);
-        ret = dict_get_int32 (peer_data, buf, (int32_t *) &snap->snap_status);
+    char buf[64] = "";
+    char prefix[32] = "";
+    char *description = NULL;
+    dict_t *dict = NULL;
+    glusterd_snap_t *snap = NULL;
+    glusterd_volinfo_t *snap_vol = NULL;
+    glusterd_conf_t *priv = NULL;
+    int32_t ret = -1;
+    int32_t volcount = -1;
+    int32_t i = -1;
+    xlator_t *this = NULL;
+    int64_t time_stamp;
+
+    this = THIS;
+    GF_ASSERT(this);
+    priv = this->private;
+    GF_ASSERT(priv);
+    GF_ASSERT(peer_data);
+    GF_ASSERT(peer_snap_name);
+    GF_ASSERT(peer_snap_id);
+
+    snprintf(prefix, sizeof(prefix), "snap%d", snap_count);
+
+    snap = glusterd_new_snap_object();
+    if (!snap) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_SNAP_CREATION_FAIL,
+               "Could not create "
+               "the snap object for snap %s",
+               peer_snap_name);
+        goto out;
+    }
+
+    dict = dict_new();
+    if (!dict) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_CREATE_FAIL,
+               "Failed to create dict");
+        ret = -1;
+        goto out;
+    }
+
+    gf_strncpy(snap->snapname, peer_snap_name, sizeof(snap->snapname));
+    gf_uuid_parse(peer_snap_id, snap->snap_id);
+
+    snprintf(buf, sizeof(buf), "%s.description", prefix);
+    ret = dict_get_str(peer_data, buf, &description);
+    if (ret == 0 && description) {
+        snap->description = gf_strdup(description);
+        if (snap->description == NULL) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_SNAP_CREATION_FAIL,
+                   "Saving the Snapshot Description Failed");
+            ret = -1;
+            goto out;
+        }
+    }
+
+    snprintf(buf, sizeof(buf), "%s.time_stamp", prefix);
+    ret = dict_get_int64(peer_data, buf, &time_stamp);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+               "Unable to get time_stamp for snap %s", peer_snap_name);
+        goto out;
+    }
+    snap->time_stamp = (time_t)time_stamp;
+
+    snprintf(buf, sizeof(buf), "%s.snap_restored", prefix);
+    ret = dict_get_int8(peer_data, buf, (int8_t *)&snap->snap_restored);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+               "Unable to get snap_restored for snap %s", peer_snap_name);
+        goto out;
+    }
+
+    snprintf(buf, sizeof(buf), "%s.snap_status", prefix);
+    ret = dict_get_int32(peer_data, buf, (int32_t *)&snap->snap_status);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+               "Unable to get snap_status for snap %s", peer_snap_name);
+        goto out;
+    }
+
+    /* If the snap is scheduled to be decommissioned, then
+     * don't accept the snap */
+    if (snap->snap_status == GD_SNAP_STATUS_DECOMMISSION) {
+        gf_msg_debug(this->name, 0,
+                     "The snap(%s) is scheduled to be decommissioned "
+                     "Not accepting the snap.",
+                     peer_snap_name);
+        glusterd_snap_remove(dict, snap, _gf_true, _gf_true, _gf_false);
+        ret = 0;
+        goto out;
+    }
+
+    snprintf(buf, sizeof(buf), "%s.volcount", prefix);
+    ret = dict_get_int32(peer_data, buf, &volcount);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+               "Unable to get volcount for snap %s", peer_snap_name);
+        goto out;
+    }
+
+    ret = glusterd_store_create_snap_dir(snap);
+    if (ret) {
+        gf_msg(THIS->name, GF_LOG_ERROR, 0, GD_MSG_SNAPDIR_CREATE_FAIL,
+               "Failed to create snap dir");
+        goto out;
+    }
+
+    glusterd_list_add_order(&snap->snap_list, &priv->snapshots,
+                            glusterd_compare_snap_time);
+
+    for (i = 1; i <= volcount; i++) {
+        ret = glusterd_import_volinfo(peer_data, i, &snap_vol, prefix);
         if (ret) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "Unable to get snap_status for snap %s",
-                        peer_snap_name);
-                goto out;
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_VOLINFO_SET_FAIL,
+                   "Failed to import snap volinfo for "
+                   "snap %s",
+                   peer_snap_name);
+            goto out;
         }
 
-        /* If the snap is scheduled to be decommissioned, then
-         * don't accept the snap */
-        if (snap->snap_status == GD_SNAP_STATUS_DECOMMISSION) {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "The snap(%s) is scheduled to be decommissioned "
-                        "Not accepting the snap.", peer_snap_name);
-                glusterd_snap_remove (dict, snap,
-                                      _gf_true, _gf_true);
-                ret = 0;
-                goto out;
-        }
+        snap_vol->snapshot = snap;
 
-        snprintf (buf, sizeof(buf), "%s.volcount", prefix);
-        ret = dict_get_int32 (peer_data, buf, &volcount);
+        ret = glusterd_gen_snap_volfiles(snap_vol, peer_snap_name);
         if (ret) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "Unable to get volcount for snap %s",
-                        peer_snap_name);
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_VOLFILE_CREATE_FAIL,
+                   "Failed to generate snap vol files "
+                   "for snap %s",
+                   peer_snap_name);
+            goto out;
+        }
+        /* During handshake, after getting updates from friend mount
+         * point for activated snapshot should exist and should not
+         * for deactivated snapshot.
+         */
+        if (glusterd_is_volume_started(snap_vol)) {
+            ret = glusterd_recreate_vol_brick_mounts(this, snap_vol);
+            if (ret) {
+                gf_msg(this->name, GF_LOG_ERROR, 0,
+                       GD_MSG_BRK_MNT_RECREATE_FAIL,
+                       "Failed to recreate brick mounts"
+                       " for %s",
+                       snap->snapname);
                 goto out;
-        }
-
-        ret = glusterd_store_create_snap_dir (snap);
-        if (ret) {
-                gf_log (THIS->name, GF_LOG_ERROR, "Failed to create snap dir");
+            }
+
+            (void)glusterd_start_bricks(snap_vol);
+            ret = glusterd_store_volinfo(snap_vol,
+                                         GLUSTERD_VOLINFO_VER_AC_NONE);
+            if (ret) {
+                gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_VOLINFO_STORE_FAIL,
+                       "Failed to "
+                       "write volinfo for volume %s",
+                       snap_vol->volname);
                 goto out;
+            }
+        } else {
+            (void)glusterd_stop_bricks(snap_vol);
+            ret = glusterd_snap_unmount(this, snap_vol);
+            if (ret) {
+                gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_GLUSTERD_UMOUNT_FAIL,
+                       "Failed to unmounts for %s", snap->snapname);
+            }
         }
 
-        glusterd_list_add_order (&snap->snap_list, &priv->snapshots,
-                                 glusterd_compare_snap_time);
-
-
-        for (i = 1; i <= volcount; i++) {
-                ret = glusterd_import_volinfo (peer_data, i,
-                                               &snap_vol, prefix);
-                if (ret) {
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "Failed to import snap volinfo for "
-                                "snap %s", peer_snap_name);
-                        goto out;
-                }
-
-                snap_vol->snapshot = snap;
-
-                ret = glusterd_gen_snap_volfiles (snap_vol, peer_snap_name);
-                if (ret) {
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "Failed to generate snap vol files "
-                                "for snap %s", peer_snap_name);
-                        goto out;
-                }
-
-                ret = glusterd_import_quota_conf (peer_data, i,
-                                                  snap_vol, prefix);
-                if (ret) {
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "Failed to import quota conf "
-                                "for snap %s", peer_snap_name);
-                        goto out;
-                }
-
-                snap_vol = NULL;
-        }
-
-        ret = glusterd_store_snap (snap);
+        ret = glusterd_import_quota_conf(peer_data, i, snap_vol, prefix);
         if (ret) {
-                gf_log (this->name, GF_LOG_WARNING, "Could not store snap"
-                        "object %s", peer_snap_name);
-                goto out;
-        }
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_QUOTA_CONFIG_IMPORT_FAIL,
+                   "Failed to import quota conf "
+                   "for snap %s",
+                   peer_snap_name);
+            goto out;
+        }
+
+        snap_vol = NULL;
+    }
+
+    ret = glusterd_store_snap(snap);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_WARNING, 0, GD_MSG_SNAP_CREATION_FAIL,
+               "Could not store snap"
+               "object %s",
+               peer_snap_name);
+        goto out;
+    }
+    glusterd_fetchsnap_notify(this);
 
 out:
-        if (ret)
-                glusterd_snap_remove (dict, snap,
-                                      _gf_true, _gf_true);
+    if (ret)
+        glusterd_snap_remove(dict, snap, _gf_true, _gf_true, _gf_false);
 
-        if (dict)
-                dict_unref (dict);
+    if (dict)
+        dict_unref(dict);
 
-        gf_log (this->name, GF_LOG_TRACE, "Returning %d", ret);
-        return ret;
+    gf_msg_trace(this->name, 0, "Returning %d", ret);
+    return ret;
 }
 
 /* During a peer-handshake, after the volumes have synced, and the list of
@@ -1500,8 +1711,11 @@ out:
  * state, i.e either both would be hosting bricks or both would not be hosting
  * bricks, then a decision can't be taken and a peer-reject will happen.
  *
- * glusterd_compare_and_update_snap() implements the following algorithm to
- * perform the above task:
+ * glusterd_compare_snap()  & glusterd_update_snaps () implement the following
+ * algorithm to perform the above task. Please note the former function tries to
+ * iterate over the snaps one at a time and updating the relevant fields in the
+ * dictionary and then glusterd_update_snaps () go over all the snaps and update
+ * them at one go as part of a synctask.
  * Step  1: Start.
  * Step  2: Check if the peer is missing a delete or restore on the said snap.
  *          If yes, goto step 6.
@@ -1510,1777 +1724,2492 @@ out:
  * Step  4: As there is a conflict, check if both the peer and the local nodes
  *          are hosting bricks. Based on the results perform the following:
  *          Peer Hosts Bricks    Local Node Hosts Bricks       Action
- *                Yes                     Yes                Goto Step 7
- *                No                      No                 Goto Step 7
- *                Yes                     No                 Goto Step 8
- *                No                      Yes                Goto Step 6
+ *                Yes                     Yes                Goto Step 8
+ *                No                      No                 Goto Step 8
+ *                Yes                     No                 Goto Step 9
+ *                No                      Yes                Goto Step 7
  * Step  5: Check if the local node is missing the peer's data.
- *          If yes, goto step 9.
- * Step  6: It's a no-op. Goto step 10
- * Step  7: Peer Reject. Goto step 10
- * Step  8: Delete local node's data.
- * Step  9: Accept Peer Data.
- * Step 10: Stop
+ *          If yes, goto step 10.
+ * Step  6: Check if the snap volume version is lesser than peer_data
+ *          if yes goto step 9
+ * Step  7: It's a no-op. Goto step 11
+ * Step  8: Peer Reject. Goto step 11
+ * Step  9: Delete local node's data.
+ * Step 10: Accept Peer Data.
+ * Step 11: Stop
  *
  */
 int32_t
-glusterd_compare_and_update_snap (dict_t *peer_data, int32_t snap_count,
-                                  char *peername, uuid_t peerid)
+glusterd_compare_snap(dict_t *peer_data, int32_t snap_count, char *peername,
+                      uuid_t peerid)
 {
-        char              buf[NAME_MAX]    = "";
-        char              prefix[NAME_MAX] = "";
-        char             *peer_snap_name   = NULL;
-        char             *peer_snap_id     = NULL;
-        dict_t           *dict             = NULL;
-        glusterd_snap_t  *snap             = NULL;
-        gf_boolean_t      conflict         = _gf_false;
-        gf_boolean_t      is_local         = _gf_false;
-        gf_boolean_t      is_hosted        = _gf_false;
-        gf_boolean_t      missed_delete    = _gf_false;
-        int32_t           ret              = -1;
-        xlator_t         *this             = NULL;
-
-        this = THIS;
-        GF_ASSERT (this);
-        GF_ASSERT (peer_data);
-        GF_ASSERT (peername);
-
-        snprintf (prefix, sizeof(prefix), "snap%d", snap_count);
-
-        /* Fetch the peer's snapname */
-        snprintf (buf, sizeof(buf), "%s.snapname", prefix);
-        ret = dict_get_str (peer_data, buf, &peer_snap_name);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "Unable to fetch snapname from peer: %s", peername);
-                goto out;
-        }
-
-        /* Fetch the peer's snap_id */
-        snprintf (buf, sizeof(buf), "%s.snap_id", prefix);
-        ret = dict_get_str (peer_data, buf, &peer_snap_id);
+    char buf[64] = "";
+    char prefix[32] = "";
+    char *peer_snap_name = NULL;
+    char *peer_snap_id = NULL;
+    glusterd_snap_t *snap = NULL;
+    gf_boolean_t conflict = _gf_false;
+    gf_boolean_t is_local = _gf_false;
+    gf_boolean_t is_hosted = _gf_false;
+    gf_boolean_t missed_delete = _gf_false;
+    int32_t ret = -1;
+    int32_t volcount = 0;
+    xlator_t *this = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+    GF_ASSERT(peer_data);
+    GF_ASSERT(peername);
+
+    snprintf(prefix, sizeof(prefix), "snap%d", snap_count);
+
+    ret = dict_set_uint32(peer_data, buf, 0);
+    snprintf(buf, sizeof(buf), "%s.accept_peer_data", prefix);
+    ret = dict_set_uint32(peer_data, buf, 0);
+    snprintf(buf, sizeof(buf), "%s.remove_lvm", prefix);
+    ret = dict_set_uint32(peer_data, buf, 0);
+    snprintf(buf, sizeof(buf), "%s.remove_my_data", prefix);
+    ret = dict_set_uint32(peer_data, buf, 0);
+
+    /* Fetch the peer's snapname */
+    snprintf(buf, sizeof(buf), "%s.snapname", prefix);
+    ret = dict_get_str(peer_data, buf, &peer_snap_name);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+               "Unable to fetch snapname from peer: %s", peername);
+        goto out;
+    }
+
+    /* Fetch the peer's snap_id */
+    snprintf(buf, sizeof(buf), "%s.snap_id", prefix);
+    ret = dict_get_str(peer_data, buf, &peer_snap_id);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+               "Unable to fetch snap_id from peer: %s", peername);
+        goto out;
+    }
+
+    snprintf(buf, sizeof(buf), "%s.volcount", prefix);
+    ret = dict_get_int32(peer_data, buf, &volcount);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+               "Unable to get volcount for snap %s", peer_snap_name);
+        goto out;
+    }
+
+    /* Check if the peer has missed a snap delete or restore
+     * resulting in stale data for the snap in question
+     */
+    missed_delete = glusterd_peer_has_missed_snap_delete(peerid, peer_snap_id);
+    if (missed_delete == _gf_true) {
+        /* Peer has missed delete on the missing/conflicting snap_id */
+        gf_msg(this->name, GF_LOG_INFO, 0, GD_MSG_MISSED_SNAP_DELETE,
+               "Peer %s has missed a delete "
+               "on snap %s",
+               peername, peer_snap_name);
+        ret = 0;
+        goto out;
+    }
+
+    /* Check if there is a conflict, and if the
+     * peer data is already present
+     */
+    glusterd_is_peer_snap_conflicting(peer_snap_name, peer_snap_id, &conflict,
+                                      &snap, peername);
+    if (conflict == _gf_false) {
+        if (!snap) {
+            /* Peer has snap with the same snapname
+             * and snap_id, which local node doesn't have.
+             */
+            snprintf(buf, sizeof(buf), "%s.accept_peer_data", prefix);
+            ret = dict_set_uint32(peer_data, buf, 1);
+            goto out;
+        }
+        /* Peer has snap with the same snapname
+         * and snap_id. Now check if peer has a
+         * snap with higher snap version than local
+         * node has.
+         */
+        ret = glusterd_check_peer_has_higher_snap_version(
+            peer_data, peer_snap_name, volcount, &conflict, prefix, snap,
+            peername);
         if (ret) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "Unable to fetch snap_id from peer: %s", peername);
-                goto out;
-        }
+            gf_msg(this->name, GF_LOG_WARNING, 0, GD_MSG_VOL_VERS_MISMATCH,
+                   "Failed "
+                   "to check version of snap volume");
+            goto out;
+        }
+        if (conflict == _gf_true) {
+            /*
+             * Snap version of peer is higher than snap
+             * version of local node.
+             *
+             * Remove data in local node and accept peer data.
+             * We just need to heal snap info of local node, So
+             * When removing data from local node, make sure
+             * we are not removing backend lvm of the snap.
+             */
+            snprintf(buf, sizeof(buf), "%s.remove_lvm", prefix);
+            ret = dict_set_uint32(peer_data, buf, 0);
+            snprintf(buf, sizeof(buf), "%s.remove_my_data", prefix);
+            ret = dict_set_uint32(peer_data, buf, 1);
+            snprintf(buf, sizeof(buf), "%s.accept_peer_data", prefix);
+            ret = dict_set_uint32(peer_data, buf, 1);
 
-        /* Check if the peer has missed a snap delete or restore
-         * resulting in stale data for the snap in question
+        } else {
+            ret = 0;
+        }
+        goto out;
+    }
+
+    /* There is a conflict. Check if the current node is
+     * hosting bricks for the conflicted snap.
+     */
+    is_local = glusterd_are_snap_bricks_local(snap);
+
+    /* Check if the peer is hosting any bricks for the
+     * conflicting snap
+     */
+    snprintf(buf, sizeof(buf), "%s.host_bricks", prefix);
+    ret = dict_get_int8(peer_data, buf, (int8_t *)&is_hosted);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+               "Unable to fetch host_bricks from peer: %s "
+               "for %s",
+               peername, peer_snap_name);
+        goto out;
+    }
+
+    /* As there is a conflict at this point of time, the data of the
+     * node that hosts a brick takes precedence. If both the local
+     * node and the peer are in the same state, i.e if both of them
+     * are either hosting or not hosting the bricks, for the snap,
+     * then it's a peer reject
+     */
+    if (is_hosted == is_local) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_SNAP_CONFLICT,
+               "Conflict in snapshot %s with peer %s", peer_snap_name,
+               peername);
+        ret = -1;
+        goto out;
+    }
+
+    if (is_hosted == _gf_false) {
+        /* If there was a conflict, and the peer is not hosting
+         * any brick, then don't accept peer data
          */
-        missed_delete = glusterd_peer_has_missed_snap_delete (peerid,
-                                                              peer_snap_id);
-        if (missed_delete == _gf_true) {
-                /* Peer has missed delete on the missing/conflicting snap_id */
-                gf_log (this->name, GF_LOG_INFO, "Peer %s has missed a delete "
-                        "on snap %s", peername, peer_snap_name);
-                ret = 0;
-                goto out;
-        }
+        gf_msg_debug(this->name, 0,
+                     "Peer doesn't hosts bricks for conflicting "
+                     "snap(%s). Not accepting peer data.",
+                     peer_snap_name);
+        ret = 0;
+        goto out;
+    }
+
+    /* The peer is hosting a brick in case of conflict
+     * And local node isn't. Hence remove local node's
+     * data and accept peer data
+     */
+    gf_msg_debug(this->name, 0,
+                 "Peer hosts bricks for conflicting "
+                 "snap(%s). Removing local data. Accepting peer data.",
+                 peer_snap_name);
+    snprintf(buf, sizeof(buf), "%s.remove_lvm", prefix);
+    ret = dict_set_uint32(peer_data, buf, 1);
+    snprintf(buf, sizeof(buf), "%s.remove_my_data", prefix);
+    ret = dict_set_uint32(peer_data, buf, 1);
+    snprintf(buf, sizeof(buf), "%s.accept_peer_data", prefix);
+    ret = dict_set_uint32(peer_data, buf, 1);
 
-        /* Check if there is a conflict, and if the
-         * peer data is already present
-         */
-        glusterd_is_peer_snap_conflicting (peer_snap_name, peer_snap_id,
-                                           &conflict, &snap, peername);
-        if (conflict == _gf_false) {
-                if (snap) {
-                        /* Peer has snap with the same snapname
-                         * and snap_id. No need to accept peer data
-                         */
-                        ret = 0;
-                        goto out;
-                } else {
-                        /* Peer has snap with the same snapname
-                         * and snap_id, which local node doesn't have.
-                         */
-                        goto accept_peer_data;
-                }
-        }
+out:
+    gf_msg_trace(this->name, 0, "Returning %d", ret);
+    return ret;
+}
 
-        /* There is a conflict. Check if the current node is
-         * hosting bricks for the conflicted snap.
-         */
-        is_local = glusterd_are_snap_bricks_local (snap);
+int32_t
+glusterd_update_snaps_synctask(void *opaque)
+{
+    int32_t ret = -1;
+    int32_t snap_count = 0;
+    int i = 1;
+    xlator_t *this = NULL;
+    dict_t *peer_data = NULL;
+    char buf[64] = "";
+    char prefix[32] = "";
+    char *peer_snap_name = NULL;
+    char *peer_snap_id = NULL;
+    char *peername = NULL;
+    gf_boolean_t remove_lvm = _gf_false;
+    gf_boolean_t remove_my_data = _gf_false;
+    gf_boolean_t accept_peer_data = _gf_false;
+    int32_t val = 0;
+    glusterd_snap_t *snap = NULL;
+    dict_t *dict = NULL;
+    glusterd_conf_t *conf = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+
+    conf = this->private;
+    GF_ASSERT(conf);
+
+    peer_data = (dict_t *)opaque;
+    GF_ASSERT(peer_data);
+
+    synclock_lock(&conf->big_lock);
+
+    while (conf->restart_bricks) {
+        synccond_wait(&conf->cond_restart_bricks, &conf->big_lock);
+    }
+    conf->restart_bricks = _gf_true;
+
+    ret = dict_get_int32(peer_data, "snap_count", &snap_count);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+               "Failed to fetch snap_count");
+        goto out;
+    }
+    ret = dict_get_str(peer_data, "peername", &peername);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+               "Failed to fetch peername");
+        goto out;
+    }
+
+    for (i = 1; i <= snap_count; i++) {
+        snprintf(prefix, sizeof(prefix), "snap%d", i);
 
-        /* Check if the peer is hosting any bricks for the
-         * conflicting snap
-         */
-        snprintf (buf, sizeof(buf), "%s.host_bricks", prefix);
-        ret = dict_get_int8 (peer_data, buf, (int8_t *) &is_hosted);
+        /* Fetch the peer's snapname */
+        snprintf(buf, sizeof(buf), "%s.snapname", prefix);
+        ret = dict_get_str(peer_data, buf, &peer_snap_name);
         if (ret) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "Unable to fetch host_bricks from peer: %s "
-                        "for %s", peername, peer_snap_name);
-                goto out;
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+                   "Unable to fetch snapname from peer: %s", peername);
+            goto out;
         }
 
-        /* As there is a conflict at this point of time, the data of the
-         * node that hosts a brick takes precedence. If both the local
-         * node and the peer are in the same state, i.e if both of them
-         * are either hosting or not hosting the bricks, for the snap,
-         * then it's a peer reject
-         */
-        if (is_hosted == is_local) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "Conflict in snapshot %s with peer %s", peer_snap_name,
-                        peername);
-                ret = -1;
-                goto out;
+        /* Fetch the peer's snap_id */
+        snprintf(buf, sizeof(buf), "%s.snap_id", prefix);
+        ret = dict_get_str(peer_data, buf, &peer_snap_id);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+                   "Unable to fetch snap_id from peer: %s", peername);
+            goto out;
         }
 
-        if (is_hosted == _gf_false) {
-                /* If there was a conflict, and the peer is not hosting
-                 * any brick, then don't accept peer data
-                 */
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "Peer doesn't hosts bricks for conflicting "
-                        "snap(%s). Not accepting peer data.",
-                        peer_snap_name);
-                ret = 0;
+        /* remove_my_data */
+        snprintf(buf, sizeof(buf), "%s.remove_my_data", prefix);
+        ret = dict_get_int32(peer_data, buf, &val);
+        if (val)
+            remove_my_data = _gf_true;
+        else
+            remove_my_data = _gf_false;
+
+        if (remove_my_data) {
+            snprintf(buf, sizeof(buf), "%s.remove_lvm", prefix);
+            ret = dict_get_int32(peer_data, buf, &val);
+            if (val)
+                remove_lvm = _gf_true;
+            else
+                remove_lvm = _gf_false;
+
+            dict = dict_new();
+            if (!dict) {
+                gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_CREATE_FAIL,
+                       "Unable to create dict");
+                ret = -1;
                 goto out;
-        }
-
-        /* The peer is hosting a brick in case of conflict
-         * And local node isn't. Hence remove local node's
-         * data and accept peer data
-         */
-
-        gf_log (this->name, GF_LOG_DEBUG, "Peer hosts bricks for conflicting "
-                "snap(%s). Removing local data. Accepting peer data.",
-                peer_snap_name);
-
-        dict = dict_new();
-        if (!dict) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "Unable to create dict");
+            }
+            snap = glusterd_find_snap_by_name(peer_snap_name);
+            if (!snap) {
+                gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_MISSED_SNAP_PRESENT,
+                       "Snapshot %s from peer %s missing on "
+                       "localhost",
+                       peer_snap_name, peername);
                 ret = -1;
                 goto out;
-        }
+            }
 
-        ret = glusterd_snap_remove (dict, snap, _gf_true, _gf_false);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "Failed to remove snap %s", snap->snapname);
+            ret = glusterd_snap_remove(dict, snap, remove_lvm, _gf_false,
+                                       _gf_false);
+            if (ret) {
+                gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_SNAP_REMOVE_FAIL,
+                       "Failed to remove snap %s", snap->snapname);
                 goto out;
-        }
+            }
 
-accept_peer_data:
-
-        /* Accept Peer Data */
-        ret = glusterd_import_friend_snap (peer_data, snap_count,
-                                           peer_snap_name, peer_snap_id);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "Failed to import snap %s from peer %s", peer_snap_name,
-                        peername);
+            dict_unref(dict);
+            dict = NULL;
+        }
+        snprintf(buf, sizeof(buf), "%s.accept_peer_data", prefix);
+        ret = dict_get_int32(peer_data, buf, &val);
+        if (val)
+            accept_peer_data = _gf_true;
+        else
+            accept_peer_data = _gf_false;
+
+        if (accept_peer_data) {
+            /* Accept Peer Data */
+            ret = glusterd_import_friend_snap(peer_data, i, peer_snap_name,
+                                              peer_snap_id);
+            if (ret) {
+                gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_SNAP_IMPORT_FAIL,
+                       "Failed to import snap %s from peer %s", peer_snap_name,
+                       peername);
                 goto out;
+            }
         }
+    }
 
 out:
-        if (dict)
-                dict_unref (dict);
-
-        gf_log (this->name, GF_LOG_TRACE, "Returning %d", ret);
-        return ret;
+    if (peer_data)
+        dict_unref(peer_data);
+    if (dict)
+        dict_unref(dict);
+    conf->restart_bricks = _gf_false;
+    synccond_broadcast(&conf->cond_restart_bricks);
+
+    return ret;
 }
 
 /* Compare snapshots present in peer_data, with the snapshots in
  * the current node
  */
 int32_t
-glusterd_compare_friend_snapshots (dict_t *peer_data, char *peername,
-                                   uuid_t peerid)
+glusterd_compare_friend_snapshots(dict_t *peer_data, char *peername,
+                                  uuid_t peerid)
 {
-        int32_t          ret          = -1;
-        int32_t          snap_count   = 0;
-        int              i            = 1;
-        xlator_t        *this         = NULL;
-
-        this = THIS;
-        GF_ASSERT (this);
-        GF_ASSERT (peer_data);
-        GF_ASSERT (peername);
-
-        ret = dict_get_int32 (peer_data, "snap_count", &snap_count);
+    int32_t ret = -1;
+    int32_t snap_count = 0;
+    int i = 1;
+    xlator_t *this = NULL;
+    dict_t *peer_data_copy = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+    GF_ASSERT(peer_data);
+    GF_ASSERT(peername);
+
+    ret = dict_get_int32(peer_data, "snap_count", &snap_count);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+               "Failed to fetch snap_count");
+        goto out;
+    }
+
+    if (!snap_count)
+        goto out;
+
+    for (i = 1; i <= snap_count; i++) {
+        /* Compare one snapshot from peer_data at a time */
+        ret = glusterd_compare_snap(peer_data, i, peername, peerid);
         if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "Failed to fetch snap_count");
-                goto out;
-        }
-
-        for (i = 1; i <= snap_count; i++) {
-                /* Compare one snapshot from peer_data at a time */
-                ret = glusterd_compare_and_update_snap (peer_data, i, peername,
-                                                        peerid);
-                if (ret) {
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "Failed to compare snapshots with peer %s",
-                                peername);
-                        goto out;
-                }
-        }
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_SNAPSHOT_OP_FAILED,
+                   "Failed to compare snapshots with peer %s", peername);
+            goto out;
+        }
+    }
+    /* Update the snaps at one go */
+    peer_data_copy = dict_copy_with_ref(peer_data, NULL);
+    ret = dict_set_str(peer_data_copy, "peername", peername);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+               "Failed to set peername into the dict");
+        if (peer_data_copy)
+            dict_unref(peer_data_copy);
+        goto out;
+    }
+    glusterd_launch_synctask(glusterd_update_snaps_synctask, peer_data_copy);
 
 out:
-        gf_log (this->name, GF_LOG_TRACE, "Returning %d", ret);
-        return ret;
+    gf_msg_trace(this->name, 0, "Returning %d", ret);
+    return ret;
 }
 
 int32_t
-glusterd_add_snapd_to_dict (glusterd_volinfo_t *volinfo,
-                            dict_t  *dict, int32_t count)
+glusterd_add_snapd_to_dict(glusterd_volinfo_t *volinfo, dict_t *dict,
+                           int32_t count)
 {
+    int ret = -1;
+    int32_t pid = -1;
+    int32_t brick_online = -1;
+    char key[64] = {0};
+    char base_key[32] = {0};
+    char pidfile[PATH_MAX] = {0};
+    xlator_t *this = NULL;
+
+    GF_ASSERT(volinfo);
+    GF_ASSERT(dict);
+
+    this = THIS;
+    GF_ASSERT(this);
+
+    snprintf(base_key, sizeof(base_key), "brick%d", count);
+    snprintf(key, sizeof(key), "%s.hostname", base_key);
+    ret = dict_set_str(dict, key, "Snapshot Daemon");
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED, "Key=%s",
+                key, NULL);
+        goto out;
+    }
+
+    snprintf(key, sizeof(key), "%s.path", base_key);
+    ret = dict_set_dynstr(dict, key, gf_strdup(uuid_utoa(MY_UUID)));
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED, "Key=%s",
+                key, NULL);
+        goto out;
+    }
+
+    snprintf(key, sizeof(key), "%s.port", base_key);
+    ret = dict_set_int32(dict, key, volinfo->snapd.port);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED, "Key=%s",
+                key, NULL);
+        goto out;
+    }
+
+    glusterd_svc_build_snapd_pidfile(volinfo, pidfile, sizeof(pidfile));
+
+    brick_online = gf_is_service_running(pidfile, &pid);
+    if (brick_online == _gf_false)
+        pid = -1;
+
+    snprintf(key, sizeof(key), "%s.pid", base_key);
+    ret = dict_set_int32(dict, key, pid);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED, "Key=%s",
+                key, NULL);
+        goto out;
+    }
+
+    snprintf(key, sizeof(key), "%s.status", base_key);
+    ret = dict_set_int32(dict, key, brick_online);
 
-        int             ret                   = -1;
-        int32_t         pid                   = -1;
-        int32_t         brick_online          = -1;
-        char            key[1024]             = {0};
-        char            base_key[1024]        = {0};
-        char            pidfile[PATH_MAX]     = {0};
-        xlator_t        *this                 = NULL;
-        glusterd_conf_t *priv                 = NULL;
-
-
-        GF_ASSERT (volinfo);
-        GF_ASSERT (dict);
-
-        this = THIS;
-        GF_ASSERT (this);
+out:
+    if (ret)
+        gf_msg_debug(this->name, 0, "Returning %d", ret);
 
-        priv = this->private;
+    return ret;
+}
 
-        snprintf (base_key, sizeof (base_key), "brick%d", count);
-        snprintf (key, sizeof (key), "%s.hostname", base_key);
-        ret = dict_set_str (dict, key, "Snapshot Daemon");
-        if (ret)
+int
+glusterd_snap_config_use_rsp_dict(dict_t *dst, dict_t *src)
+{
+    char buf[PATH_MAX] = "";
+    char *volname = NULL;
+    int ret = -1;
+    int config_command = 0;
+    uint64_t i = 0;
+    uint64_t hard_limit = GLUSTERD_SNAPS_MAX_HARD_LIMIT;
+    uint64_t soft_limit = GLUSTERD_SNAPS_DEF_SOFT_LIMIT_PERCENT;
+    uint64_t value = 0;
+    uint64_t voldisplaycount = 0;
+
+    if (!dst || !src) {
+        gf_msg("glusterd", GF_LOG_ERROR, 0, GD_MSG_DICT_EMPTY,
+               "Source or Destination "
+               "dict is empty.");
+        goto out;
+    }
+
+    ret = dict_get_int32(dst, "config-command", &config_command);
+    if (ret) {
+        gf_msg("glusterd", GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+               "failed to get config-command type");
+        goto out;
+    }
+
+    switch (config_command) {
+        case GF_SNAP_CONFIG_DISPLAY:
+            ret = dict_get_uint64(src, GLUSTERD_STORE_KEY_SNAP_MAX_HARD_LIMIT,
+                                  &hard_limit);
+            if (!ret) {
+                ret = dict_set_uint64(
+                    dst, GLUSTERD_STORE_KEY_SNAP_MAX_HARD_LIMIT, hard_limit);
+                if (ret) {
+                    gf_msg("glusterd", GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+                           "Unable to set snap_max_hard_limit");
+                    goto out;
+                }
+            } else {
+                /* Received dummy response from other nodes */
+                ret = 0;
                 goto out;
+            }
 
-        snprintf (key, sizeof (key), "%s.path", base_key);
-        ret = dict_set_dynstr (dict, key, gf_strdup (uuid_utoa (MY_UUID)));
-        if (ret)
+            ret = dict_get_uint64(src, GLUSTERD_STORE_KEY_SNAP_MAX_SOFT_LIMIT,
+                                  &soft_limit);
+            if (ret) {
+                gf_msg("glusterd", GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+                       "Unable to get snap_max_soft_limit");
                 goto out;
+            }
 
-        memset (key, 0, sizeof (key));
-        snprintf (key, sizeof (key), "%s.port", base_key);
-        ret = dict_set_int32 (dict, key, volinfo->snapd.port);
-        if (ret)
+            ret = dict_set_uint64(dst, GLUSTERD_STORE_KEY_SNAP_MAX_SOFT_LIMIT,
+                                  soft_limit);
+            if (ret) {
+                gf_msg("glusterd", GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+                       "Unable to set snap_max_soft_limit");
                 goto out;
+            }
 
-        glusterd_svc_build_snapd_pidfile (volinfo, pidfile, sizeof (pidfile));
-
-        brick_online = gf_is_service_running (pidfile, &pid);
-
-        memset (key, 0, sizeof (key));
-        snprintf (key, sizeof (key), "%s.pid", base_key);
-        ret = dict_set_int32 (dict, key, pid);
-        if (ret)
+            ret = dict_get_uint64(src, "voldisplaycount", &voldisplaycount);
+            if (ret) {
+                gf_msg("glusterd", GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+                       "Unable to get voldisplaycount");
                 goto out;
+            }
 
-        memset (key, 0, sizeof (key));
-        snprintf (key, sizeof (key), "%s.status", base_key);
-        ret = dict_set_int32 (dict, key, brick_online);
-
-out:
-        if (ret)
-                gf_log (this->name, GF_LOG_DEBUG, "Returning %d", ret);
-
-        return ret;
-}
-
-int
-glusterd_snap_config_use_rsp_dict (dict_t *dst, dict_t *src)
-{
-        char       buf[PATH_MAX]    = "";
-        char      *volname          = NULL;
-        int        ret              = -1;
-        int        config_command   = 0;
-        uint64_t   i                = 0;
-        uint64_t   hard_limit       = GLUSTERD_SNAPS_MAX_HARD_LIMIT;
-        uint64_t   soft_limit       = GLUSTERD_SNAPS_DEF_SOFT_LIMIT_PERCENT;
-        uint64_t   value            = 0;
-        uint64_t   voldisplaycount  = 0;
-
-        if (!dst || !src) {
-                gf_log ("", GF_LOG_ERROR, "Source or Destination "
-                        "dict is empty.");
+            ret = dict_set_uint64(dst, "voldisplaycount", voldisplaycount);
+            if (ret) {
+                gf_msg("glusterd", GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+                       "Unable to set voldisplaycount");
                 goto out;
-        }
-
-        ret = dict_get_int32 (dst, "config-command", &config_command);
-        if (ret) {
-                gf_log ("", GF_LOG_ERROR,
-                        "failed to get config-command type");
-                goto out;
-        }
+            }
 
-        switch (config_command) {
-        case GF_SNAP_CONFIG_DISPLAY:
-                ret = dict_get_uint64 (src,
-                                       GLUSTERD_STORE_KEY_SNAP_MAX_HARD_LIMIT,
-                                       &hard_limit);
-                if (!ret) {
-                        ret = dict_set_uint64 (dst,
-                                         GLUSTERD_STORE_KEY_SNAP_MAX_HARD_LIMIT,
-                                         hard_limit);
-                        if (ret) {
-                                gf_log ("", GF_LOG_ERROR,
-                                        "Unable to set snap_max_hard_limit");
-                                goto out;
-                        }
-                } else {
-                        /* Received dummy response from other nodes */
-                        ret = 0;
-                        goto out;
+            for (i = 0; i < voldisplaycount; i++) {
+                snprintf(buf, sizeof(buf), "volume%" PRIu64 "-volname", i);
+                ret = dict_get_str(src, buf, &volname);
+                if (ret) {
+                    gf_msg("glusterd", GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+                           "Unable to get %s", buf);
+                    goto out;
                 }
-
-                ret = dict_get_uint64 (src,
-                                       GLUSTERD_STORE_KEY_SNAP_MAX_SOFT_LIMIT,
-                                       &soft_limit);
+                ret = dict_set_str(dst, buf, volname);
                 if (ret) {
-                        gf_log ("", GF_LOG_ERROR,
-                                "Unable to get snap_max_soft_limit");
-                        goto out;
+                    gf_msg("glusterd", GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+                           "Unable to set %s", buf);
+                    goto out;
                 }
 
-                ret = dict_set_uint64 (dst,
-                                       GLUSTERD_STORE_KEY_SNAP_MAX_SOFT_LIMIT,
-                                       soft_limit);
+                snprintf(buf, sizeof(buf),
+                         "volume%" PRIu64 "-snap-max-hard-limit", i);
+                ret = dict_get_uint64(src, buf, &value);
                 if (ret) {
-                        gf_log ("", GF_LOG_ERROR,
-                                "Unable to set snap_max_soft_limit");
-                        goto out;
+                    gf_msg("glusterd", GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+                           "Unable to get %s", buf);
+                    goto out;
                 }
-
-                ret = dict_get_uint64 (src, "voldisplaycount",
-                                       &voldisplaycount);
+                ret = dict_set_uint64(dst, buf, value);
                 if (ret) {
-                        gf_log ("", GF_LOG_ERROR,
-                                "Unable to get voldisplaycount");
-                        goto out;
+                    gf_msg("glusterd", GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+                           "Unable to set %s", buf);
+                    goto out;
                 }
 
-                ret = dict_set_uint64 (dst, "voldisplaycount",
-                                       voldisplaycount);
+                snprintf(buf, sizeof(buf),
+                         "volume%" PRIu64 "-active-hard-limit", i);
+                ret = dict_get_uint64(src, buf, &value);
                 if (ret) {
-                        gf_log ("", GF_LOG_ERROR,
-                                "Unable to set voldisplaycount");
-                        goto out;
+                    gf_msg("glusterd", GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+                           "Unable to get %s", buf);
+                    goto out;
+                }
+                ret = dict_set_uint64(dst, buf, value);
+                if (ret) {
+                    gf_msg("glusterd", GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+                           "Unable to set %s", buf);
+                    goto out;
                 }
 
-                for (i = 0; i < voldisplaycount; i++) {
-                        snprintf (buf, sizeof(buf),
-                                  "volume%"PRIu64"-volname", i);
-                        ret = dict_get_str (src, buf, &volname);
-                        if (ret) {
-                                gf_log ("", GF_LOG_ERROR,
-                                        "Unable to get %s", buf);
-                                goto out;
-                        }
-                        ret = dict_set_str (dst, buf, volname);
-                        if (ret) {
-                                gf_log ("", GF_LOG_ERROR,
-                                        "Unable to set %s", buf);
-                                goto out;
-                        }
-
-                        snprintf (buf, sizeof(buf),
-                                  "volume%"PRIu64"-snap-max-hard-limit", i);
-                        ret = dict_get_uint64 (src, buf, &value);
-                        if (ret) {
-                                gf_log ("", GF_LOG_ERROR,
-                                        "Unable to get %s", buf);
-                                goto out;
-                        }
-                        ret = dict_set_uint64 (dst, buf, value);
-                        if (ret) {
-                                gf_log ("", GF_LOG_ERROR,
-                                        "Unable to set %s", buf);
-                                goto out;
-                        }
-
-                        snprintf (buf, sizeof(buf),
-                                  "volume%"PRIu64"-active-hard-limit", i);
-                        ret = dict_get_uint64 (src, buf, &value);
-                        if (ret) {
-                                gf_log ("", GF_LOG_ERROR,
-                                        "Unable to get %s", buf);
-                                goto out;
-                        }
-                        ret = dict_set_uint64 (dst, buf, value);
-                        if (ret) {
-                                gf_log ("", GF_LOG_ERROR,
-                                        "Unable to set %s", buf);
-                                goto out;
-                        }
-
-                        snprintf (buf, sizeof(buf),
-                                  "volume%"PRIu64"-snap-max-soft-limit", i);
-                        ret = dict_get_uint64 (src, buf, &value);
-                        if (ret) {
-                                gf_log ("", GF_LOG_ERROR,
-                                        "Unable to get %s", buf);
-                                goto out;
-                        }
-                        ret = dict_set_uint64 (dst, buf, value);
-                        if (ret) {
-                                gf_log ("", GF_LOG_ERROR,
-                                        "Unable to set %s", buf);
-                                goto out;
-                        }
+                snprintf(buf, sizeof(buf),
+                         "volume%" PRIu64 "-snap-max-soft-limit", i);
+                ret = dict_get_uint64(src, buf, &value);
+                if (ret) {
+                    gf_msg("glusterd", GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+                           "Unable to get %s", buf);
+                    goto out;
+                }
+                ret = dict_set_uint64(dst, buf, value);
+                if (ret) {
+                    gf_msg("glusterd", GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+                           "Unable to set %s", buf);
+                    goto out;
                 }
+            }
 
-                break;
+            break;
         default:
-                break;
-        }
+            break;
+    }
 
-        ret = 0;
+    ret = 0;
 out:
-        gf_log ("", GF_LOG_DEBUG, "Returning %d", ret);
-        return ret;
+    gf_msg_debug("glusterd", 0, "Returning %d", ret);
+    return ret;
 }
 
 int
-glusterd_merge_brick_status (dict_t *dst, dict_t *src)
+glusterd_merge_brick_status(dict_t *dst, dict_t *src)
 {
-        int64_t        volume_count             = 0;
-        int64_t        index                    = 0;
-        int64_t        j                        = 0;
-        int64_t        brick_count              = 0;
-        int64_t        brick_order              = 0;
-        char           key[PATH_MAX]            = {0, };
-        char           snapbrckcnt[PATH_MAX]    = {0, };
-        char           snapbrckord[PATH_MAX]    = {0, };
-        int            ret                      = -1;
-        int32_t        brick_online             = 0;
-        xlator_t      *this                     = NULL;
-        int32_t        snap_command             = 0;
-
-        this = THIS;
-        GF_ASSERT (this);
-
-        if (!dst || !src) {
-                gf_log (this->name, GF_LOG_ERROR, "Source or Destination "
-                        "dict is empty.");
-                goto out;
-        }
-
-        ret = dict_get_int32 (dst, "type", &snap_command);
+    int64_t volume_count = 0;
+    int64_t index = 0;
+    int64_t j = 0;
+    int64_t brick_count = 0;
+    int64_t brick_order = 0;
+    char key[64] = {
+        0,
+    };
+    char key_prefix[16] = {
+        0,
+    };
+    char snapbrckcnt[PATH_MAX] = {
+        0,
+    };
+    char snapbrckord[PATH_MAX] = {
+        0,
+    };
+    char *clonename = NULL;
+    int ret = -1;
+    int32_t brick_online = 0;
+    xlator_t *this = NULL;
+    int32_t snap_command = 0;
+
+    this = THIS;
+    GF_ASSERT(this);
+
+    if (!dst || !src) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_EMPTY,
+               "Source or Destination "
+               "dict is empty.");
+        goto out;
+    }
+
+    ret = dict_get_int32(dst, "type", &snap_command);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+               "unable to get the type of "
+               "the snapshot command");
+        goto out;
+    }
+
+    if (snap_command == GF_SNAP_OPTION_TYPE_DELETE) {
+        gf_msg_debug(this->name, 0,
+                     "snapshot delete command."
+                     " Need not merge the status of the bricks");
+        ret = 0;
+        goto out;
+    }
+
+    /* Try and fetch clonename. If present set status with clonename *
+     * else do so as snap-vol */
+    ret = dict_get_str(dst, "clonename", &clonename);
+    if (ret) {
+        snprintf(key_prefix, sizeof(key_prefix), "snap-vol");
+    } else
+        snprintf(key_prefix, sizeof(key_prefix), "clone");
+
+    ret = dict_get_int64(src, "volcount", &volume_count);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+               "failed to "
+               "get the volume count");
+        goto out;
+    }
+
+    for (index = 0; index < volume_count; index++) {
+        ret = snprintf(snapbrckcnt, sizeof(snapbrckcnt) - 1,
+                       "snap-vol%" PRId64 "_brickcount", index + 1);
+        ret = dict_get_int64(src, snapbrckcnt, &brick_count);
         if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "unable to get the type of "
-                        "the snapshot command");
+            gf_msg_trace(this->name, 0,
+                         "No bricks for this volume in this dict (%s)",
+                         snapbrckcnt);
+            continue;
+        }
+
+        for (j = 0; j < brick_count; j++) {
+            /* Fetching data from source dict */
+            snprintf(snapbrckord, sizeof(snapbrckord) - 1,
+                     "snap-vol%" PRId64 ".brick%" PRId64 ".order", index + 1,
+                     j);
+
+            ret = dict_get_int64(src, snapbrckord, &brick_order);
+            if (ret) {
+                gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+                       "Failed to get brick order (%s)", snapbrckord);
                 goto out;
-        }
-
-        if (snap_command == GF_SNAP_OPTION_TYPE_DELETE) {
-                gf_log (this->name, GF_LOG_DEBUG, "snapshot delete command."
-                        " Need not merge the status of the bricks");
-                ret = 0;
+            }
+
+            snprintf(key, sizeof(key), "%s%" PRId64 ".brick%" PRId64 ".status",
+                     key_prefix, index + 1, brick_order);
+            ret = dict_get_int32(src, key, &brick_online);
+            if (ret) {
+                gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+                       "failed to "
+                       "get the brick status (%s)",
+                       key);
                 goto out;
-        }
-
-        ret = dict_get_int64 (src, "volcount", &volume_count);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "failed to "
-                        "get the volume count");
+            }
+
+            ret = dict_set_int32(dst, key, brick_online);
+            if (ret) {
+                gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+                       "failed to "
+                       "set the brick status (%s)",
+                       key);
                 goto out;
+            }
+            brick_online = 0;
         }
+    }
 
-        for (index = 0; index < volume_count; index++) {
-                ret = snprintf (snapbrckcnt, sizeof(snapbrckcnt) - 1,
-                                "snap-vol%"PRId64"_brickcount", index+1);
-                ret = dict_get_int64 (src, snapbrckcnt, &brick_count);
-                if (ret) {
-                        gf_log (this->name, GF_LOG_TRACE,
-                                "No bricks for this volume in this dict (%s)",
-                                snapbrckcnt);
-                        continue;
-                }
-
-                for (j = 0; j < brick_count; j++) {
-                        /* Fetching data from source dict */
-                        snprintf (snapbrckord, sizeof(snapbrckord) - 1,
-                                  "snap-vol%"PRId64".brick%"PRId64".order",
-                                  index+1, j);
-
-                        ret = dict_get_int64 (src, snapbrckord, &brick_order);
-                        if (ret) {
-                                gf_log (this->name, GF_LOG_ERROR,
-                                        "Failed to get brick order (%s)",
-                                        snapbrckord);
-                                goto out;
-                        }
-
-                        snprintf (key, sizeof (key) - 1,
-                                  "snap-vol%"PRId64".brick%"PRId64".status",
-                                  index+1, brick_order);
-                        ret = dict_get_int32 (src, key, &brick_online);
-                        if (ret) {
-                                gf_log (this->name, GF_LOG_ERROR, "failed to "
-                                        "get the brick status (%s)", key);
-                                goto out;
-                        }
-
-                        ret = dict_set_int32 (dst, key, brick_online);
-                        if (ret) {
-                                gf_log (this->name, GF_LOG_ERROR, "failed to "
-                                        "set the brick status (%s)", key);
-                                goto out;
-                        }
-                        brick_online = 0;
-                }
-        }
-
-        ret = 0;
+    ret = 0;
 
 out:
-        return ret;
+    return ret;
 }
 
 /* Aggregate missed_snap_counts from different nodes and save it *
  * in the req_dict of the originator node */
 int
-glusterd_snap_create_use_rsp_dict (dict_t *dst, dict_t *src)
+glusterd_snap_create_use_rsp_dict(dict_t *dst, dict_t *src)
 {
-        char          *buf                      = NULL;
-        char          *tmp_str                  = NULL;
-        char           name_buf[PATH_MAX]       = "";
-        int32_t        i                        = -1;
-        int32_t        ret                      = -1;
-        int32_t        src_missed_snap_count    = -1;
-        int32_t        dst_missed_snap_count    = -1;
-        xlator_t      *this                     = NULL;
-        int8_t         soft_limit_flag          = -1;
-
-        this = THIS;
-        GF_ASSERT (this);
-
-        if (!dst || !src) {
-                gf_log (this->name, GF_LOG_ERROR, "Source or Destination "
-                        "dict is empty.");
-                goto out;
-        }
-
-        ret = glusterd_merge_brick_status (dst, src);
+    char *buf = NULL;
+    char *tmp_str = NULL;
+    char name_buf[PATH_MAX] = "";
+    int32_t i = -1;
+    int32_t ret = -1;
+    int32_t src_missed_snap_count = -1;
+    int32_t dst_missed_snap_count = -1;
+    xlator_t *this = NULL;
+    int8_t soft_limit_flag = -1;
+
+    this = THIS;
+    GF_ASSERT(this);
+
+    if (!dst || !src) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_EMPTY,
+               "Source or Destination "
+               "dict is empty.");
+        goto out;
+    }
+
+    ret = glusterd_merge_brick_status(dst, src);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_BRICK_SET_INFO_FAIL,
+               "failed to merge brick "
+               "status");
+        goto out;
+    }
+
+    ret = dict_get_str(src, "snapuuid", &buf);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+               "failed to get snap UUID");
+        goto out;
+    }
+
+    ret = dict_set_dynstr_with_alloc(dst, "snapuuid", buf);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+               "Failed to set snap uuid in dict");
+        goto out;
+    }
+
+    /* set in dst dictionary soft-limit-reach only if soft-limit-reach
+     * is present src dictionary */
+    ret = dict_get_int8(src, "soft-limit-reach", &soft_limit_flag);
+    if (!ret) {
+        ret = dict_set_int8(dst, "soft-limit-reach", soft_limit_flag);
         if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "failed to merge brick "
-                        "status");
-                goto out;
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+                   "Failed to set "
+                   "soft_limit_flag");
+            goto out;
         }
+    }
 
-        ret = dict_get_str (src, "snapuuid", &buf);
+    ret = dict_get_int32(src, "missed_snap_count", &src_missed_snap_count);
+    if (ret) {
+        gf_msg_debug(this->name, 0, "No missed snaps");
+        ret = 0;
+        goto out;
+    }
+
+    ret = dict_get_int32(dst, "missed_snap_count", &dst_missed_snap_count);
+    if (ret) {
+        /* Initialize dst_missed_count for the first time */
+        dst_missed_snap_count = 0;
+    }
+
+    for (i = 0; i < src_missed_snap_count; i++) {
+        snprintf(name_buf, sizeof(name_buf), "missed_snaps_%d", i);
+        ret = dict_get_str(src, name_buf, &buf);
         if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "failed to get snap UUID");
-                goto out;
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+                   "Unable to fetch %s", name_buf);
+            goto out;
         }
 
-        ret = dict_set_dynstr_with_alloc (dst, "snapuuid", buf);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "Failed to set snap uuid in dict");
-                goto out;
-        }
+        snprintf(name_buf, sizeof(name_buf), "missed_snaps_%d",
+                 dst_missed_snap_count);
 
-        /* set in dst dictionary soft-limit-reach only if soft-limit-reach
-         * is present src dictionary */
-        ret = dict_get_int8 (src, "soft-limit-reach", &soft_limit_flag);
-        if (!ret) {
-                ret = dict_set_int8 (dst, "soft-limit-reach", soft_limit_flag);
-                if (ret) {
-                        gf_log (this->name, GF_LOG_ERROR, "Failed to set "
-                                "soft_limit_flag");
-                        goto out;
-                }
-        }
-
-        ret = dict_get_int32 (src, "missed_snap_count",
-                              &src_missed_snap_count);
-        if (ret) {
-                gf_log (this->name, GF_LOG_DEBUG, "No missed snaps");
-                ret = 0;
-                goto out;
+        tmp_str = gf_strdup(buf);
+        if (!tmp_str) {
+            ret = -1;
+            goto out;
         }
 
-        ret = dict_get_int32 (dst, "missed_snap_count",
-                              &dst_missed_snap_count);
+        ret = dict_set_dynstr(dst, name_buf, tmp_str);
         if (ret) {
-                /* Initialize dst_missed_count for the first time */
-                dst_missed_snap_count = 0;
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+                   "Unable to set %s", name_buf);
+            goto out;
         }
 
-        for (i = 0; i < src_missed_snap_count; i++) {
-                snprintf (name_buf, sizeof(name_buf), "missed_snaps_%d", i);
-                ret = dict_get_str (src, name_buf, &buf);
-                if (ret) {
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "Unable to fetch %s", name_buf);
-                        goto out;
-                }
-
-                snprintf (name_buf, sizeof(name_buf), "missed_snaps_%d",
-                          dst_missed_snap_count);
-
-                tmp_str = gf_strdup (buf);
-                if (!tmp_str) {
-                        ret = -1;
-                        goto out;
-                }
-
-                ret = dict_set_dynstr (dst, name_buf, tmp_str);
-                if (ret) {
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "Unable to set %s", name_buf);
-                        goto out;
-                }
-
-                tmp_str = NULL;
-                dst_missed_snap_count++;
-        }
+        tmp_str = NULL;
+        dst_missed_snap_count++;
+    }
 
-        ret = dict_set_int32 (dst, "missed_snap_count", dst_missed_snap_count);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "Unable to set dst_missed_snap_count");
-                goto out;
-        }
+    ret = dict_set_int32(dst, "missed_snap_count", dst_missed_snap_count);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+               "Unable to set dst_missed_snap_count");
+        goto out;
+    }
 
 out:
-        if (ret && tmp_str)
-                GF_FREE(tmp_str);
+    if (ret && tmp_str)
+        GF_FREE(tmp_str);
 
-        gf_log (this->name, GF_LOG_TRACE, "Returning %d", ret);
-        return ret;
+    gf_msg_trace(this->name, 0, "Returning %d", ret);
+    return ret;
 }
 
 int
-glusterd_snap_use_rsp_dict (dict_t *dst, dict_t *src)
+glusterd_snap_use_rsp_dict(dict_t *dst, dict_t *src)
 {
-        int            ret            = -1;
-        int32_t        snap_command   = 0;
-
-        if (!dst || !src) {
-                gf_log ("", GF_LOG_ERROR, "Source or Destination "
-                        "dict is empty.");
-                goto out;
-        }
-
-        ret = dict_get_int32 (dst, "type", &snap_command);
-        if (ret) {
-                gf_log ("", GF_LOG_ERROR, "unable to get the type of "
-                        "the snapshot command");
-                goto out;
-        }
-
-        switch (snap_command) {
+    int ret = -1;
+    int32_t snap_command = 0;
+
+    if (!dst || !src) {
+        gf_msg("glusterd", GF_LOG_ERROR, 0, GD_MSG_DICT_EMPTY,
+               "Source or Destination "
+               "dict is empty.");
+        goto out;
+    }
+
+    ret = dict_get_int32(dst, "type", &snap_command);
+    if (ret) {
+        gf_msg("glusterd", GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+               "unable to get the type of "
+               "the snapshot command");
+        goto out;
+    }
+
+    switch (snap_command) {
         case GF_SNAP_OPTION_TYPE_CREATE:
         case GF_SNAP_OPTION_TYPE_DELETE:
         case GF_SNAP_OPTION_TYPE_CLONE:
-                ret = glusterd_snap_create_use_rsp_dict (dst, src);
-                if (ret) {
-                        gf_log ("", GF_LOG_ERROR, "Unable to use rsp dict");
-                        goto out;
-                }
-                break;
+            ret = glusterd_snap_create_use_rsp_dict(dst, src);
+            if (ret) {
+                gf_msg("glusterd", GF_LOG_ERROR, 0, GD_MSG_RSP_DICT_USE_FAIL,
+                       "Unable to use rsp dict");
+                goto out;
+            }
+            break;
         case GF_SNAP_OPTION_TYPE_CONFIG:
-                ret = glusterd_snap_config_use_rsp_dict (dst, src);
-                if (ret) {
-                        gf_log ("", GF_LOG_ERROR, "Unable to use rsp dict");
-                        goto out;
-                }
-                break;
+            ret = glusterd_snap_config_use_rsp_dict(dst, src);
+            if (ret) {
+                gf_msg("glusterd", GF_LOG_ERROR, 0, GD_MSG_RSP_DICT_USE_FAIL,
+                       "Unable to use rsp dict");
+                goto out;
+            }
+            break;
         default:
-                /* copy the response dictinary's contents to the dict to be
-                 * sent back to the cli */
-                dict_copy (src, dst);
-                break;
-        }
+            /* copy the response dictinary's contents to the dict to be
+             * sent back to the cli */
+            dict_copy(src, dst);
+            break;
+    }
 
-        ret = 0;
+    ret = 0;
 out:
-        gf_log ("", GF_LOG_DEBUG, "Returning %d", ret);
-        return ret;
+    gf_msg_debug("glusterd", 0, "Returning %d", ret);
+    return ret;
 }
 
 int
-glusterd_compare_snap_time (struct cds_list_head *list1,
-                            struct cds_list_head *list2)
+glusterd_compare_snap_time(struct cds_list_head *list1,
+                           struct cds_list_head *list2)
 {
-        glusterd_snap_t *snap1 = NULL;
-        glusterd_snap_t *snap2 = NULL;
-        double diff_time       = 0;
+    glusterd_snap_t *snap1 = NULL;
+    glusterd_snap_t *snap2 = NULL;
+    double diff_time = 0;
 
-        GF_ASSERT (list1);
-        GF_ASSERT (list2);
+    GF_ASSERT(list1);
+    GF_ASSERT(list2);
 
-        snap1 = cds_list_entry (list1, glusterd_snap_t, snap_list);
-        snap2 = cds_list_entry (list2, glusterd_snap_t, snap_list);
-        diff_time = difftime(snap1->time_stamp, snap2->time_stamp);
+    snap1 = cds_list_entry(list1, glusterd_snap_t, snap_list);
+    snap2 = cds_list_entry(list2, glusterd_snap_t, snap_list);
+    diff_time = difftime(snap1->time_stamp, snap2->time_stamp);
 
-        return (int)diff_time;
+    return (int)diff_time;
 }
 
 int
-glusterd_compare_snap_vol_time (struct cds_list_head *list1,
-                                struct cds_list_head *list2)
+glusterd_compare_snap_vol_time(struct cds_list_head *list1,
+                               struct cds_list_head *list2)
 {
-        glusterd_volinfo_t *snapvol1 = NULL;
-        glusterd_volinfo_t *snapvol2 = NULL;
-        double diff_time             = 0;
+    glusterd_volinfo_t *snapvol1 = NULL;
+    glusterd_volinfo_t *snapvol2 = NULL;
+    double diff_time = 0;
 
-        GF_ASSERT (list1);
-        GF_ASSERT (list2);
+    GF_ASSERT(list1);
+    GF_ASSERT(list2);
 
-        snapvol1 = cds_list_entry (list1, glusterd_volinfo_t, snapvol_list);
-        snapvol2 = cds_list_entry (list2, glusterd_volinfo_t, snapvol_list);
-        diff_time = difftime(snapvol1->snapshot->time_stamp,
-                             snapvol2->snapshot->time_stamp);
+    snapvol1 = cds_list_entry(list1, glusterd_volinfo_t, snapvol_list);
+    snapvol2 = cds_list_entry(list2, glusterd_volinfo_t, snapvol_list);
+    diff_time = difftime(snapvol1->snapshot->time_stamp,
+                         snapvol2->snapshot->time_stamp);
 
-        return (int)diff_time;
+    return (int)diff_time;
 }
 
 int32_t
-glusterd_missed_snapinfo_new (glusterd_missed_snap_info **missed_snapinfo)
+glusterd_missed_snapinfo_new(glusterd_missed_snap_info **missed_snapinfo)
 {
-        glusterd_missed_snap_info      *new_missed_snapinfo = NULL;
-        int32_t                         ret                 = -1;
-        xlator_t                       *this                = NULL;
+    glusterd_missed_snap_info *new_missed_snapinfo = NULL;
+    int32_t ret = -1;
+    xlator_t *this = NULL;
 
-        this = THIS;
-        GF_ASSERT (this);
-        GF_ASSERT (missed_snapinfo);
+    this = THIS;
+    GF_ASSERT(this);
+    GF_ASSERT(missed_snapinfo);
 
-        new_missed_snapinfo = GF_CALLOC (1, sizeof(*new_missed_snapinfo),
-                                         gf_gld_mt_missed_snapinfo_t);
+    new_missed_snapinfo = GF_CALLOC(1, sizeof(*new_missed_snapinfo),
+                                    gf_gld_mt_missed_snapinfo_t);
 
-        if (!new_missed_snapinfo)
-                goto out;
+    if (!new_missed_snapinfo) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_NO_MEMORY, NULL);
+        goto out;
+    }
 
-        CDS_INIT_LIST_HEAD (&new_missed_snapinfo->missed_snaps);
-        CDS_INIT_LIST_HEAD (&new_missed_snapinfo->snap_ops);
+    CDS_INIT_LIST_HEAD(&new_missed_snapinfo->missed_snaps);
+    CDS_INIT_LIST_HEAD(&new_missed_snapinfo->snap_ops);
 
-        *missed_snapinfo = new_missed_snapinfo;
+    *missed_snapinfo = new_missed_snapinfo;
 
-        ret = 0;
+    ret = 0;
 
 out:
-        gf_log (this->name, GF_LOG_TRACE, "Returning %d", ret);
-        return ret;
+    gf_msg_trace(this->name, 0, "Returning %d", ret);
+    return ret;
 }
 
 int32_t
-glusterd_missed_snap_op_new (glusterd_snap_op_t **snap_op)
+glusterd_missed_snap_op_new(glusterd_snap_op_t **snap_op)
 {
-        glusterd_snap_op_t      *new_snap_op = NULL;
-        int32_t                  ret         = -1;
-        xlator_t                *this        = NULL;
+    glusterd_snap_op_t *new_snap_op = NULL;
+    int32_t ret = -1;
+    xlator_t *this = NULL;
 
-        this = THIS;
-        GF_ASSERT (this);
-        GF_ASSERT (snap_op);
+    this = THIS;
+    GF_ASSERT(this);
+    GF_ASSERT(snap_op);
 
-        new_snap_op = GF_CALLOC (1, sizeof(*new_snap_op),
-                                 gf_gld_mt_missed_snapinfo_t);
+    new_snap_op = GF_CALLOC(1, sizeof(*new_snap_op),
+                            gf_gld_mt_missed_snapinfo_t);
 
-        if (!new_snap_op)
-                goto out;
+    if (!new_snap_op) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_NO_MEMORY, NULL);
+        goto out;
+    }
 
-        new_snap_op->brick_num = -1;
-        new_snap_op->op = -1;
-        new_snap_op->status = -1;
-        CDS_INIT_LIST_HEAD (&new_snap_op->snap_ops_list);
+    new_snap_op->brick_num = -1;
+    new_snap_op->op = -1;
+    new_snap_op->status = -1;
+    CDS_INIT_LIST_HEAD(&new_snap_op->snap_ops_list);
 
-        *snap_op = new_snap_op;
+    *snap_op = new_snap_op;
 
-        ret = 0;
+    ret = 0;
 out:
-        gf_log (this->name, GF_LOG_TRACE, "Returning %d", ret);
-        return ret;
+    gf_msg_trace(this->name, 0, "Returning %d", ret);
+    return ret;
 }
 
 gf_boolean_t
-mntopts_exists (const char *str, const char *opts)
+mntopts_exists(const char *str, const char *opts)
 {
-        char          *dup_val     = NULL;
-        char          *savetok     = NULL;
-        char          *token       = NULL;
-        gf_boolean_t   exists      = _gf_false;
+    char *dup_val = NULL;
+    char *savetok = NULL;
+    char *token = NULL;
+    gf_boolean_t exists = _gf_false;
 
-        GF_ASSERT (opts);
+    GF_ASSERT(opts);
 
-        if (!str || !strlen(str))
-                goto out;
+    if (!str || !strlen(str))
+        goto out;
 
-        dup_val = gf_strdup (str);
-        if (!dup_val)
-                goto out;
+    dup_val = gf_strdup(str);
+    if (!dup_val)
+        goto out;
 
-        token = strtok_r (dup_val, ",", &savetok);
-        while (token) {
-                if (!strcmp (token, opts)) {
-                        exists = _gf_true;
-                        goto out;
-                }
-                token = strtok_r (NULL, ",", &savetok);
+    token = strtok_r(dup_val, ",", &savetok);
+    while (token) {
+        if (!strcmp(token, opts)) {
+            exists = _gf_true;
+            goto out;
         }
+        token = strtok_r(NULL, ",", &savetok);
+    }
 
 out:
-        GF_FREE (dup_val);
-        return exists;
+    GF_FREE(dup_val);
+    return exists;
 }
 
 int32_t
-glusterd_mount_lvm_snapshot (glusterd_brickinfo_t *brickinfo,
-                             char *brick_mount_path)
+glusterd_mount_lvm_snapshot(glusterd_brickinfo_t *brickinfo,
+                            char *brick_mount_path)
 {
-        char               msg[NAME_MAX]  = "";
-        char               mnt_opts[1024] = "";
-        int32_t            ret            = -1;
-        runner_t           runner         = {0, };
-        xlator_t          *this           = NULL;
-
-        this = THIS;
-        GF_ASSERT (this);
-        GF_ASSERT (brick_mount_path);
-        GF_ASSERT (brickinfo);
-
-
-        runinit (&runner);
-        snprintf (msg, sizeof (msg), "mount %s %s",
-                  brickinfo->device_path, brick_mount_path);
-
-        strcpy (mnt_opts, brickinfo->mnt_opts);
-
-        /* XFS file-system does not allow to mount file-system with duplicate
-         * UUID. File-system UUID of snapshot and its origin volume is same.
-         * Therefore to mount such a snapshot in XFS we need to pass nouuid
-         * option
-         */
-        if (!strcmp (brickinfo->fstype, "xfs") &&
-            !mntopts_exists (mnt_opts, "nouuid")) {
-                if (strlen (mnt_opts) > 0)
-                        strcat (mnt_opts, ",");
-                strcat (mnt_opts, "nouuid");
-        }
-
-
-        if (strlen (mnt_opts) > 0) {
-                runner_add_args (&runner, "mount", "-o", mnt_opts,
-                                brickinfo->device_path, brick_mount_path, NULL);
-        } else {
-                runner_add_args (&runner, "mount", brickinfo->device_path,
-                                 brick_mount_path, NULL);
-        }
-
-        runner_log (&runner, this->name, GF_LOG_DEBUG, msg);
-        ret = runner_run (&runner);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "mounting the snapshot "
-                        "logical device %s failed (error: %s)",
-                        brickinfo->device_path, strerror (errno));
-                goto out;
-        } else
-                gf_log (this->name, GF_LOG_DEBUG, "mounting the snapshot "
-                        "logical device %s successful", brickinfo->device_path);
+    char msg[NAME_MAX] = "";
+    char mnt_opts[1024] = "";
+    int32_t ret = -1;
+    runner_t runner = {
+        0,
+    };
+    xlator_t *this = NULL;
+    int32_t len = 0;
+
+    this = THIS;
+    GF_ASSERT(this);
+    GF_ASSERT(brick_mount_path);
+    GF_ASSERT(brickinfo);
+
+    runinit(&runner);
+    len = snprintf(msg, sizeof(msg), "mount %s %s", brickinfo->device_path,
+                   brick_mount_path);
+    if (len < 0) {
+        strcpy(msg, "<error>");
+    }
+
+    gf_strncpy(mnt_opts, brickinfo->mnt_opts, sizeof(mnt_opts));
+
+    /* XFS file-system does not allow to mount file-system with duplicate
+     * UUID. File-system UUID of snapshot and its origin volume is same.
+     * Therefore to mount such a snapshot in XFS we need to pass nouuid
+     * option
+     */
+    if (!strcmp(brickinfo->fstype, "xfs") &&
+        !mntopts_exists(mnt_opts, "nouuid")) {
+        if (strlen(mnt_opts) > 0)
+            strcat(mnt_opts, ",");
+        strcat(mnt_opts, "nouuid");
+    }
+
+    if (strlen(mnt_opts) > 0) {
+        runner_add_args(&runner, "mount", "-o", mnt_opts,
+                        brickinfo->device_path, brick_mount_path, NULL);
+    } else {
+        runner_add_args(&runner, "mount", brickinfo->device_path,
+                        brick_mount_path, NULL);
+    }
+
+    runner_log(&runner, this->name, GF_LOG_DEBUG, msg);
+    ret = runner_run(&runner);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_SNAP_MOUNT_FAIL,
+               "mounting the snapshot "
+               "logical device %s failed (error: %s)",
+               brickinfo->device_path, strerror(errno));
+        goto out;
+    } else
+        gf_msg_debug(this->name, 0,
+                     "mounting the snapshot "
+                     "logical device %s successful",
+                     brickinfo->device_path);
 
 out:
-        gf_log (this->name, GF_LOG_TRACE, "Returning with %d", ret);
-        return ret;
+    gf_msg_trace(this->name, 0, "Returning with %d", ret);
+    return ret;
 }
 
 gf_boolean_t
-glusterd_volume_quorum_calculate (glusterd_volinfo_t *volinfo, dict_t *dict,
-                                  int down_count, gf_boolean_t first_brick_on,
-                                  int8_t snap_force, int quorum_count,
-                                  char *quorum_type, char **op_errstr)
+glusterd_volume_quorum_calculate(glusterd_volinfo_t *volinfo, dict_t *dict,
+                                 int down_count, gf_boolean_t first_brick_on,
+                                 int8_t snap_force, int quorum_count,
+                                 char *quorum_type, char **op_errstr,
+                                 uint32_t *op_errno)
 {
-        gf_boolean_t  quorum_met        = _gf_false;
-        char          err_str[PATH_MAX] = {0, };
-        xlator_t     *this              = NULL;
-        int           up_count          = 0;
-
-        this = THIS;
-        GF_ASSERT (this);
+    gf_boolean_t quorum_met = _gf_false;
+    const char err_str[] = "One or more bricks may be down.";
+    xlator_t *this = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+    GF_VALIDATE_OR_GOTO(this->name, op_errno, out);
+
+    if (!volinfo || !dict) {
+        gf_msg(this->name, GF_LOG_WARNING, 0, GD_MSG_INVALID_ENTRY,
+               "input parameters NULL");
+        goto out;
+    }
+
+    /* In a n-way replication where n >= 3 we should not take a snapshot
+     * if even one brick is down, irrespective of the quorum being met.
+     * TODO: Remove this restriction once n-way replication is
+     * supported with snapshot.
+     */
+    if (down_count) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_BRICK_DISCONNECTED, "%s",
+               err_str);
+        *op_errstr = gf_strdup(err_str);
+        *op_errno = EG_BRCKDWN;
+    } else {
+        quorum_met = _gf_true;
+    }
+
+    /* TODO : Support for n-way relication in snapshot*/
+out:
+    return quorum_met;
+}
 
-        if (!volinfo || !dict) {
-                gf_log (this->name, GF_LOG_WARNING, "input parameters NULL");
+static int32_t
+glusterd_volume_quorum_check(glusterd_volinfo_t *volinfo, int64_t index,
+                             dict_t *dict, const char *key_prefix,
+                             int8_t snap_force, int quorum_count,
+                             char *quorum_type, char **op_errstr,
+                             uint32_t *op_errno)
+{
+    int ret = 0;
+    xlator_t *this = NULL;
+    int64_t i = 0;
+    int64_t j = 0;
+    char key[128] = {
+        0,
+    }; /* key_prefix is passed from above, but is really quite small */
+    int keylen;
+    int down_count = 0;
+    gf_boolean_t first_brick_on = _gf_true;
+    glusterd_conf_t *priv = NULL;
+    gf_boolean_t quorum_met = _gf_false;
+    int distribute_subvols = 0;
+    int32_t brick_online = 0;
+    const char err_str[] = "quorum is not met";
+
+    this = THIS;
+    GF_ASSERT(this);
+    priv = this->private;
+    GF_ASSERT(priv);
+    GF_VALIDATE_OR_GOTO(this->name, op_errno, out);
+
+    if (!volinfo || !dict) {
+        gf_msg(this->name, GF_LOG_WARNING, 0, GD_MSG_INVALID_ENTRY,
+               "input parameters NULL");
+        goto out;
+    }
+
+    if ((!glusterd_is_volume_replicate(volinfo) ||
+         volinfo->replica_count < 3) &&
+        (GF_CLUSTER_TYPE_DISPERSE != volinfo->type)) {
+        for (i = 0; i < volinfo->brick_count; i++) {
+            /* for a pure distribute volume, and replica volume
+               with replica count 2, quorum is not met if even
+               one of its subvolumes is down
+            */
+            keylen = snprintf(key, sizeof(key),
+                              "%s%" PRId64 ".brick%" PRId64 ".status",
+                              key_prefix, index, i);
+            ret = dict_get_int32n(dict, key, keylen, &brick_online);
+            if (ret || !brick_online) {
+                ret = 1;
+                gf_msg(this->name, GF_LOG_ERROR, 0,
+                       GD_MSG_SERVER_QUORUM_NOT_MET, "%s", err_str);
+                *op_errstr = gf_strdup(err_str);
+                *op_errno = EG_BRCKDWN;
                 goto out;
+            }
         }
+        ret = 0;
+        quorum_met = _gf_true;
+    } else {
+        distribute_subvols = volinfo->brick_count / volinfo->dist_leaf_count;
+        for (j = 0; j < distribute_subvols; j++) {
+            /* by default assume quorum is not met
+               TODO: Handle distributed striped replicate volumes
+               Currently only distributed replicate volumes are
+               handled.
+            */
+            ret = 1;
+            quorum_met = _gf_false;
+            for (i = 0; i < volinfo->dist_leaf_count; i++) {
+                keylen = snprintf(
+                    key, sizeof(key), "%s%" PRId64 ".brick%" PRId64 ".status",
+                    key_prefix, index, (j * volinfo->dist_leaf_count) + i);
+                ret = dict_get_int32n(dict, key, keylen, &brick_online);
+                if (ret || !brick_online) {
+                    if (i == 0)
+                        first_brick_on = _gf_false;
+                    down_count++;
+                }
+            }
 
-        /* In a n-way replication where n >= 3 we should not take a snapshot
-         * if even one brick is down, irrespective of the quorum being met.
-         * TODO: Remove this restriction once n-way replication is
-         * supported with snapshot.
-         */
-        if (down_count) {
-                snprintf (err_str, sizeof (err_str), "One or more bricks may "
-                          "be down.");
-                gf_log (this->name, GF_LOG_ERROR, "%s", err_str);
-                *op_errstr = gf_strdup (err_str);
+            quorum_met = glusterd_volume_quorum_calculate(
+                volinfo, dict, down_count, first_brick_on, snap_force,
+                quorum_count, quorum_type, op_errstr, op_errno);
+            /* goto out if quorum is not met */
+            if (!quorum_met) {
+                ret = -1;
                 goto out;
+            }
+
+            down_count = 0;
+            first_brick_on = _gf_true;
         }
+    }
 
-        up_count = volinfo->dist_leaf_count - down_count;
+    if (quorum_met) {
+        gf_msg_debug(this->name, 0, "volume %s is in quorum", volinfo->volname);
+        ret = 0;
+    }
 
-        if (quorum_type && !strcmp (quorum_type, "fixed")) {
-                if (up_count >= quorum_count) {
-                        quorum_met = _gf_true;
-                        goto out;
-                }
-        } else {
-                if ((GF_CLUSTER_TYPE_DISPERSE != volinfo->type) &&
-                    (volinfo->dist_leaf_count % 2 == 0)) {
-                        if ((up_count > quorum_count) ||
-                            ((up_count == quorum_count) && first_brick_on)) {
-                                quorum_met = _gf_true;
-                                goto out;
-                        }
-                } else {
-                        if (up_count >= quorum_count) {
-                                quorum_met = _gf_true;
-                                goto out;
-                        }
-                }
-        }
+out:
+    return ret;
+}
 
-        if (!quorum_met) {
-                snprintf (err_str, sizeof (err_str), "quorum is not met");
-                gf_log (this->name, GF_LOG_WARNING, "%s", err_str);
-                *op_errstr = gf_strdup (err_str);
-        }
+static int32_t
+glusterd_snap_common_quorum_calculate(glusterd_volinfo_t *volinfo, dict_t *dict,
+                                      int64_t index, const char *key_prefix,
+                                      int8_t snap_force,
+                                      gf_boolean_t snap_volume,
+                                      char **op_errstr, uint32_t *op_errno)
+{
+    int quorum_count = 0;
+    char *quorum_type = NULL;
+    int32_t tmp = 0;
+    int32_t ret = -1;
+    xlator_t *this = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+    GF_VALIDATE_OR_GOTO(this->name, op_errno, out);
+    GF_VALIDATE_OR_GOTO(this->name, volinfo, out);
+
+    /* for replicate volumes with replica count equal to or
+       greater than 3, do quorum check by getting what type
+       of quorum rule has been set by getting the volume
+       option set. If getting the option fails, then assume
+       default.
+       AFR does this:
+       if quorum type is "auto":
+       - for odd number of bricks (n), n/2 + 1
+       bricks should be present
+       - for even number of bricks n, n/2 bricks
+       should be present along with the 1st
+       subvolume
+       if quorum type is not "auto":
+       - get the quorum count from dict with the
+       help of the option "cluster.quorum-count"
+       if the option is not there in the dict,
+       then assume quorum type is auto and follow
+       the above method.
+       For non replicate volumes quorum is met only if all
+       the bricks of the volume are online
+     */
+
+    if (GF_CLUSTER_TYPE_REPLICATE == volinfo->type) {
+        if (volinfo->replica_count % 2 == 0)
+            quorum_count = volinfo->replica_count / 2;
+        else
+            quorum_count = volinfo->replica_count / 2 + 1;
+    } else if (GF_CLUSTER_TYPE_DISPERSE == volinfo->type) {
+        quorum_count = volinfo->disperse_count - volinfo->redundancy_count;
+    } else {
+        quorum_count = volinfo->brick_count;
+    }
+
+    ret = dict_get_str_sizen(volinfo->dict, "cluster.quorum-type",
+                             &quorum_type);
+    if (!ret && !strcmp(quorum_type, "fixed")) {
+        ret = dict_get_int32_sizen(volinfo->dict, "cluster.quorum-count", &tmp);
+        /* if quorum-type option is not found in the
+           dict assume auto quorum type. i.e n/2 + 1.
+           The same assumption is made when quorum-count
+           option cannot be obtained from the dict (even
+           if the quorum-type option is not set to auto,
+           the behavior is set to the default behavior)
+         */
+        if (!ret) {
+            /* for dispersed volumes, only allow quorums
+               equal or larger than minimum functional
+               value.
+             */
+            if ((GF_CLUSTER_TYPE_DISPERSE != volinfo->type) ||
+                (tmp >= quorum_count)) {
+                quorum_count = tmp;
+            } else {
+                gf_msg(this->name, GF_LOG_INFO, 0, GD_MSG_QUORUM_COUNT_IGNORED,
+                       "Ignoring small quorum-count "
+                       "(%d) on dispersed volume",
+                       tmp);
+                quorum_type = NULL;
+            }
+        } else
+            quorum_type = NULL;
+    }
+
+    ret = glusterd_volume_quorum_check(volinfo, index, dict, key_prefix,
+                                       snap_force, quorum_count, quorum_type,
+                                       op_errstr, op_errno);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_WARNING, 0, GD_MSG_VOL_NOT_FOUND,
+               "volume %s "
+               "is not in quorum",
+               volinfo->volname);
+        goto out;
+    }
 
 out:
-        return quorum_met;
+    return ret;
 }
 
-int32_t
-glusterd_volume_quorum_check (glusterd_volinfo_t *volinfo, int64_t index,
-                              dict_t *dict, char *key_prefix,
-                              int8_t snap_force, int quorum_count,
-                              char *quorum_type, char **op_errstr)
+static int32_t
+glusterd_snap_quorum_check_for_clone(dict_t *dict, gf_boolean_t snap_volume,
+                                     char **op_errstr, uint32_t *op_errno)
 {
-        int                      ret                = 0;
-        xlator_t                *this               = NULL;
-        int64_t                  i                  = 0;
-        int64_t                  j                  = 0;
-        char                     key[1024]          = {0, };
-        int                      down_count         = 0;
-        gf_boolean_t             first_brick_on     = _gf_true;
-        glusterd_conf_t         *priv               = NULL;
-        gf_boolean_t             quorum_met         = _gf_false;
-        int                      distribute_subvols = 0;
-        int32_t                  brick_online       = 0;
-        char                     err_str[PATH_MAX]  = {0, };
-
-        this = THIS;
-        GF_ASSERT (this);
-        priv = this->private;
-        GF_ASSERT (priv);
-
-        if (!volinfo || !dict) {
-                gf_log (this->name, GF_LOG_WARNING, "input parameters NULL");
-                goto out;
+    const char err_str[] = "glusterds are not in quorum";
+    char key_prefix[16] = {
+        0,
+    };
+    char *snapname = NULL;
+    glusterd_snap_t *snap = NULL;
+    glusterd_volinfo_t *volinfo = NULL;
+    glusterd_volinfo_t *tmp_volinfo = NULL;
+    char *volname = NULL;
+    int64_t volcount = 0;
+    int64_t i = 0;
+    int32_t ret = -1;
+    xlator_t *this = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+    GF_VALIDATE_OR_GOTO(this->name, op_errno, out);
+
+    if (!dict) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_EMPTY, "dict is NULL");
+        goto out;
+    }
+
+    if (snap_volume) {
+        ret = dict_get_str_sizen(dict, "snapname", &snapname);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+                   "failed to "
+                   "get snapname");
+            goto out;
         }
 
-        if ((!glusterd_is_volume_replicate (volinfo) ||
-             volinfo->replica_count < 3) &&
-            (GF_CLUSTER_TYPE_DISPERSE != volinfo->type)) {
-                for (i = 0; i < volinfo->brick_count ; i++) {
-                        /* for a pure distribute volume, and replica volume
-                           with replica count 2, quorum is not met if even
-                           one of its subvolumes is down
-                        */
-                        snprintf (key, sizeof (key),
-                                  "%s%"PRId64".brick%"PRId64".status",
-                                  key_prefix, index, i);
-                        ret = dict_get_int32 (dict, key, &brick_online);
-                        if (ret || !brick_online) {
-                                ret = 1;
-                                snprintf (err_str, sizeof (err_str), "quorum "
-                                          "is not met");
-                                gf_log (this->name, GF_LOG_ERROR, "%s",
-                                        err_str);
-                                *op_errstr = gf_strdup (err_str);
-                                goto out;
-                        }
+        snap = glusterd_find_snap_by_name(snapname);
+        if (!snap) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_SNAP_NOT_FOUND,
+                   "failed to "
+                   "get the snapshot %s",
+                   snapname);
+            ret = -1;
+            goto out;
+        }
+    }
+
+    /* Do a quorum check of glusterds also. Because, the missed snapshot
+     * information will be saved by glusterd and if glusterds are not in
+     * quorum, then better fail the snapshot
+     */
+    if (!does_gd_meet_server_quorum(this)) {
+        gf_msg(this->name, GF_LOG_WARNING, 0, GD_MSG_SERVER_QUORUM_NOT_MET,
+               "%s", err_str);
+        *op_errstr = gf_strdup(err_str);
+        *op_errno = EG_NODEDWN;
+        ret = -1;
+        goto out;
+    } else
+        gf_msg_debug(this->name, 0, "glusterds are in quorum");
+
+    ret = dict_get_int64(dict, "volcount", &volcount);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+               "failed to get "
+               "volcount");
+        goto out;
+    }
+
+    for (i = 1; i <= volcount; i++) {
+        ret = dict_get_str_sizen(dict, "clonename", &volname);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+                   "failed to "
+                   "get clonename");
+            goto out;
+        }
+
+        if (snap_volume && snap) {
+            cds_list_for_each_entry(tmp_volinfo, &snap->volumes, vol_list)
+            {
+                if (!tmp_volinfo) {
+                    gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_SNAP_NOT_FOUND,
+                           "failed to get snap volume "
+                           "for snap %s",
+                           snapname);
+                    ret = -1;
+                    goto out;
                 }
-                ret = 0;
-                quorum_met = _gf_true;
+                volinfo = tmp_volinfo;
+            }
         } else {
-             distribute_subvols = volinfo->brick_count /
-                                  volinfo->dist_leaf_count;
-             for (j = 0; j < distribute_subvols; j++) {
-                        /* by default assume quorum is not met
-                           TODO: Handle distributed striped replicate volumes
-                           Currently only distributed replicate volumes are
-                           handled.
-                        */
-                        ret = 1;
-                        quorum_met = _gf_false;
-                        for (i = 0; i < volinfo->dist_leaf_count; i++) {
-                                snprintf (key, sizeof (key),
-                                          "%s%"PRId64".brick%"PRId64".status",
-                                          key_prefix, index,
-                                          (j * volinfo->dist_leaf_count) + i);
-                                ret = dict_get_int32 (dict, key, &brick_online);
-                                if (ret || !brick_online) {
-                                        if (i == 0)
-                                                first_brick_on = _gf_false;
-                                        down_count++;
-                                }
-                        }
-
-                        quorum_met = glusterd_volume_quorum_calculate (volinfo,
-                                                                       dict,
-                                                                    down_count,
-                                                                first_brick_on,
-                                                                    snap_force,
-                                                                  quorum_count,
-                                                                   quorum_type,
-                                                                   op_errstr);
-                        /* goto out if quorum is not met */
-                        if (!quorum_met) {
-                                ret = -1;
-                                goto out;
-                        }
-
-                        down_count = 0;
-                        first_brick_on = _gf_true;
-                }
+            ret = glusterd_volinfo_find(volname, &volinfo);
+            if (ret) {
+                gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_VOL_NOT_FOUND,
+                       "failed to find the volume %s", volname);
+                goto out;
+            }
         }
 
-        if (quorum_met) {
-                gf_log (this->name, GF_LOG_DEBUG, "volume %s is in quorum",
-                        volinfo->volname);
-                ret = 0;
-        }
+        snprintf(key_prefix, sizeof(key_prefix), "%s",
+                 snap_volume ? "vol" : "clone");
 
+        ret = glusterd_snap_common_quorum_calculate(
+            volinfo, dict, i, key_prefix, 0, snap_volume, op_errstr, op_errno);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_WARNING, 0, GD_MSG_VOL_NOT_FOUND,
+                   "volume %s "
+                   "is not in quorum",
+                   volname);
+            goto out;
+        }
+    }
 out:
-        return ret;
+    return ret;
 }
 
-int32_t
-glusterd_snap_quorum_check_for_create (dict_t *dict, gf_boolean_t snap_volume,
-                                       char **op_errstr)
+static int32_t
+glusterd_snap_quorum_check_for_create(dict_t *dict, gf_boolean_t snap_volume,
+                                      char **op_errstr, uint32_t *op_errno)
 {
-        int8_t              snap_force        = 0;
-        int32_t             force             = 0;
-        char                err_str[PATH_MAX] = {0, };
-        int                 quorum_count      = 0;
-        char               *quorum_type       = NULL;
-        int32_t             tmp               = 0;
-        char                key_prefix[PATH_MAX] = {0, };
-        char               *snapname          = NULL;
-        glusterd_snap_t    *snap              = NULL;
-        glusterd_volinfo_t *volinfo           = NULL;
-        char               *volname           = NULL;
-        int64_t             volcount          = 0;
-        char                key[PATH_MAX]     = {0, };
-        int64_t             i                 = 0;
-        int32_t             ret               = -1;
-        xlator_t           *this              = NULL;
-
-        this = THIS;
-        GF_ASSERT (this);
-
-        if (!dict) {
-                gf_log (this->name, GF_LOG_ERROR, "dict is NULL");
-                goto out;
+    int8_t snap_force = 0;
+    int32_t force = 0;
+    const char err_str[] = "glusterds are not in quorum";
+    char key_prefix[16] = {
+        0,
+    };
+    char *snapname = NULL;
+    glusterd_snap_t *snap = NULL;
+    glusterd_volinfo_t *volinfo = NULL;
+    char *volname = NULL;
+    int64_t volcount = 0;
+    char key[32] = {
+        0,
+    };
+    int64_t i = 0;
+    int32_t ret = -1;
+    xlator_t *this = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+    GF_VALIDATE_OR_GOTO(this->name, op_errno, out);
+
+    if (!dict) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_EMPTY, "dict is NULL");
+        goto out;
+    }
+
+    if (snap_volume) {
+        ret = dict_get_str(dict, "snapname", &snapname);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+                   "failed to "
+                   "get snapname");
+            goto out;
         }
 
-        if (snap_volume) {
-                ret = dict_get_str (dict, "snapname", &snapname);
-                if (ret) {
-                        gf_log (this->name, GF_LOG_ERROR, "failed to "
-                                "get snapname");
-                        goto out;
-                }
-
-                snap = glusterd_find_snap_by_name (snapname);
-                if (!snap) {
-                        gf_log (this->name, GF_LOG_ERROR, "failed to "
-                                "get the snapshot %s", snapname);
-                        ret = -1;
-                        goto out;
-                }
+        snap = glusterd_find_snap_by_name(snapname);
+        if (!snap) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_SNAP_NOT_FOUND,
+                   "failed to "
+                   "get the snapshot %s",
+                   snapname);
+            ret = -1;
+            goto out;
+        }
+    }
+
+    ret = dict_get_int32(dict, "flags", &force);
+    if (!ret && (force & GF_CLI_FLAG_OP_FORCE))
+        snap_force = 1;
+
+    /* Do a quorum check of glusterds also. Because, the missed snapshot
+     * information will be saved by glusterd and if glusterds are not in
+     * quorum, then better fail the snapshot
+     */
+    if (!does_gd_meet_server_quorum(this)) {
+        gf_msg(this->name, GF_LOG_WARNING, 0, GD_MSG_SERVER_QUORUM_NOT_MET,
+               "%s", err_str);
+        *op_errstr = gf_strdup(err_str);
+        *op_errno = EG_NODEDWN;
+        ret = -1;
+        goto out;
+    } else
+        gf_msg_debug(this->name, 0, "glusterds are in quorum");
+
+    ret = dict_get_int64(dict, "volcount", &volcount);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+               "failed to get "
+               "volcount");
+        goto out;
+    }
+
+    for (i = 1; i <= volcount; i++) {
+        snprintf(key, sizeof(key), "%s%" PRId64,
+                 snap_volume ? "snap-volname" : "volname", i);
+        ret = dict_get_str(dict, key, &volname);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+                   "failed to "
+                   "get volname");
+            goto out;
         }
 
-        ret = dict_get_int32 (dict, "flags", &force);
-        if (!ret && (force & GF_CLI_FLAG_OP_FORCE))
-                snap_force = 1;
-
-        /* Do a quorum check of glusterds also. Because, the missed snapshot
-         * information will be saved by glusterd and if glusterds are not in
-         * quorum, then better fail the snapshot
-         */
-        if (!does_gd_meet_server_quorum (this)) {
-                snprintf (err_str, sizeof (err_str),
-                          "glusterds are not in quorum");
-                gf_log (this->name, GF_LOG_WARNING, "%s", err_str);
-                *op_errstr = gf_strdup (err_str);
-                ret = -1;
+        if (snap_volume) {
+            ret = glusterd_snap_volinfo_find(volname, snap, &volinfo);
+            if (ret) {
+                gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_SNAP_NOT_FOUND,
+                       "failed to get snap volume %s "
+                       "for snap %s",
+                       volname, snapname);
                 goto out;
-        } else
-                gf_log (this->name, GF_LOG_DEBUG, "glusterds are in quorum");
-
-        ret = dict_get_int64 (dict, "volcount", &volcount);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "failed to get "
-                        "volcount");
+            }
+        } else {
+            ret = glusterd_volinfo_find(volname, &volinfo);
+            if (ret) {
+                gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_VOL_NOT_FOUND,
+                       "failed to find the volume %s", volname);
                 goto out;
+            }
         }
 
-        for (i = 1; i <= volcount; i++) {
-                snprintf (key, sizeof (key), "%s%"PRId64,
-                          snap_volume?"snap-volname":"volname", i);
-                ret = dict_get_str (dict, key, &volname);
-                if (ret) {
-                        gf_log (this->name, GF_LOG_ERROR, "failed to "
-                                "get volname");
-                        goto out;
-                }
-
-                if (snap_volume) {
-                        ret = glusterd_snap_volinfo_find (volname, snap,
-                                                          &volinfo);
-                        if (ret) {
-                                gf_log (this->name, GF_LOG_ERROR,
-                                        "failed to get snap volume %s "
-                                        "for snap %s", volname,
-                                        snapname);
-                                goto out;
-                        }
-                } else {
-                        ret = glusterd_volinfo_find (volname, &volinfo);
-                        if (ret) {
-                                gf_log (this->name, GF_LOG_ERROR,
-                                        "failed to find the volume %s",
-                                        volname);
-                                goto out;
-                        }
-                }
+        snprintf(key_prefix, sizeof(key_prefix), "%s",
+                 snap_volume ? "snap-vol" : "vol");
 
-                /* for replicate volumes with replica count equal to or
-                   greater than 3, do quorum check by getting what type
-                   of quorum rule has been set by getting the volume
-                   option set. If getting the option fails, then assume
-                   default.
-                   AFR does this:
-                   if quorum type is "auto":
-                   - for odd numner of bricks (n), n/2 + 1
-                   bricks should be present
-                   - for even number of bricks n, n/2 bricks
-                   should be present along with the 1st
-                   subvolume
-                   if quorum type is not "auto":
-                   - get the quorum count from dict with the
-                   help of the option "cluster.quorum-count"
-                   if the option is not there in the dict,
-                   then assume quorum type is auto and follow
-                   the above method.
-                   For non replicate volumes quorum is met only if all
-                   the bricks of the volume are online
-                */
-
-                if (GF_CLUSTER_TYPE_REPLICATE == volinfo->type) {
-                        if (volinfo->replica_count % 2 == 0)
-                                quorum_count = volinfo->replica_count/2;
-                        else
-                                quorum_count =
-                                        volinfo->replica_count/2 + 1;
-                } else if (GF_CLUSTER_TYPE_DISPERSE == volinfo->type) {
-                        quorum_count = volinfo->disperse_count -
-                                       volinfo->redundancy_count;
-                } else {
-                        quorum_count = volinfo->brick_count;
-                }
-
-                ret = dict_get_str (volinfo->dict,
-                                    "cluster.quorum-type",
-                                    &quorum_type);
-                if (!ret && !strcmp (quorum_type, "fixed")) {
-                        ret = dict_get_int32 (volinfo->dict,
-                                              "cluster.quorum-count",
-                                              &tmp);
-                        /* if quorum-type option is not found in the
-                           dict assume auto quorum type. i.e n/2 + 1.
-                           The same assumption is made when quorum-count
-                           option cannot be obtained from the dict (even
-                           if the quorum-type option is not set to auto,
-                           the behavior is set to the default behavior)
-                        */
-                        if (!ret) {
-                                /* for dispersed volumes, only allow quorums
-                                   equal or larger than minimum functional
-                                   value.
-                                */
-                                if ((GF_CLUSTER_TYPE_DISPERSE !=
-                                                              volinfo->type) ||
-                                    (tmp >= quorum_count)) {
-                                        quorum_count = tmp;
-                                } else {
-                                        gf_log(this->name, GF_LOG_INFO,
-                                               "Ignoring small quorum-count "
-                                               "(%d) on dispersed volume", tmp);
-                                        quorum_type = NULL;
-                                }
-                        } else
-                                quorum_type = NULL;
-                }
-
-                snprintf (key_prefix, sizeof (key_prefix),
-                          "%s", snap_volume?"snap-vol":"vol");
-
-                ret = glusterd_volume_quorum_check (volinfo, i, dict,
-                                                    key_prefix,
-                                                    snap_force,
-                                                    quorum_count,
-                                                    quorum_type,
-                                                    op_errstr);
-                if (ret) {
-                        gf_log (this->name, GF_LOG_WARNING, "volume %s "
-                                "is not in quorum", volinfo->volname);
-                        goto out;
-                }
+        ret = glusterd_snap_common_quorum_calculate(
+            volinfo, dict, i, key_prefix, snap_force, snap_volume, op_errstr,
+            op_errno);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_WARNING, 0, GD_MSG_VOL_NOT_FOUND,
+                   "volume %s "
+                   "is not in quorum",
+                   volinfo->volname);
+            goto out;
         }
+    }
 out:
-        return ret;
+    return ret;
 }
 
 int32_t
-glusterd_snap_quorum_check (dict_t *dict, gf_boolean_t snap_volume,
-                            char **op_errstr)
+glusterd_snap_quorum_check(dict_t *dict, gf_boolean_t snap_volume,
+                           char **op_errstr, uint32_t *op_errno)
 {
-        int32_t             ret               = -1;
-        xlator_t           *this              = NULL;
-        int32_t             snap_command      = 0;
-        char                err_str[PATH_MAX] = {0, };
-
-        this = THIS;
-        GF_ASSERT (this);
-
-        if (!dict) {
-                gf_log (this->name, GF_LOG_ERROR, "dict is NULL");
-                goto out;
-        }
-
-
-        ret = dict_get_int32 (dict, "type", &snap_command);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "unable to get the type of "
-                        "the snapshot command");
-                goto out;
-        }
-
-        switch (snap_command) {
+    int32_t ret = -1;
+    xlator_t *this = NULL;
+    int32_t snap_command = 0;
+    const char err_str[] = "glusterds are not in quorum";
+
+    this = THIS;
+    GF_ASSERT(this);
+    GF_VALIDATE_OR_GOTO(this->name, op_errno, out);
+
+    if (!dict) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_EMPTY, "dict is NULL");
+        goto out;
+    }
+
+    ret = dict_get_int32_sizen(dict, "type", &snap_command);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+               "unable to get the type of "
+               "the snapshot command");
+        goto out;
+    }
+
+    switch (snap_command) {
         case GF_SNAP_OPTION_TYPE_CREATE:
-                ret = glusterd_snap_quorum_check_for_create (dict, snap_volume,
-                                                             op_errstr);
-                if (ret) {
-                        gf_log (this->name, GF_LOG_WARNING, "Quorum check"
-                                "failed during snapshot create command");
-                        goto out;
-                }
-                break;
+            ret = glusterd_snap_quorum_check_for_create(dict, snap_volume,
+                                                        op_errstr, op_errno);
+            if (ret) {
+                gf_msg(this->name, GF_LOG_WARNING, 0, GD_MSG_QUORUM_CHECK_FAIL,
+                       "Quorum check"
+                       "failed during snapshot create command");
+                goto out;
+            }
+            break;
         case GF_SNAP_OPTION_TYPE_CLONE:
-
-                if (!does_gd_meet_server_quorum (this)) {
-                        ret = -1;
-                        snprintf (err_str, sizeof (err_str),
-                                  "glusterds are not in quorum");
-                        gf_log (this->name, GF_LOG_WARNING, "%s",
-                                err_str);
-                        *op_errstr = gf_strdup (err_str);
-                        goto out;
-                }
-
-                gf_log (this->name, GF_LOG_DEBUG, "glusterds are in "
-                        "quorum");
-                break;
+            ret = glusterd_snap_quorum_check_for_clone(dict, !snap_volume,
+                                                       op_errstr, op_errno);
+            if (ret) {
+                gf_msg(this->name, GF_LOG_WARNING, 0, GD_MSG_QUORUM_CHECK_FAIL,
+                       "Quorum check"
+                       "failed during snapshot clone command");
+                goto out;
+            }
+            break;
         case GF_SNAP_OPTION_TYPE_DELETE:
         case GF_SNAP_OPTION_TYPE_RESTORE:
-                if (!does_gd_meet_server_quorum (this)) {
-                        ret = -1;
-                        snprintf (err_str, sizeof (err_str),
-                                  "glusterds are not in quorum");
-                        gf_log (this->name, GF_LOG_WARNING, "%s",
-                                err_str);
-                        *op_errstr = gf_strdup (err_str);
-                        goto out;
-                }
+            if (!does_gd_meet_server_quorum(this)) {
+                ret = -1;
+                gf_msg(this->name, GF_LOG_WARNING, 0,
+                       GD_MSG_SERVER_QUORUM_NOT_MET, "%s", err_str);
+                *op_errstr = gf_strdup(err_str);
+                *op_errno = EG_NODEDWN;
+                goto out;
+            }
 
-                gf_log (this->name, GF_LOG_DEBUG, "glusterds are in "
-                        "quorum");
-                break;
+            gf_msg_debug(this->name, 0,
+                         "glusterds are in "
+                         "quorum");
+            break;
         default:
-                break;
-        }
+            break;
+    }
 
-        ret = 0;
+    ret = 0;
 
 out:
-        return ret;
+    return ret;
 }
 
-int32_t
-glusterd_umount (const char *path)
+int
+glusterd_is_path_mounted(const char *path)
 {
-        char               msg[NAME_MAX] = "";
-        int32_t            ret           = -1;
-        runner_t           runner        = {0, };
-        xlator_t          *this          = NULL;
-
-        this = THIS;
-        GF_ASSERT (this);
-        GF_ASSERT (path);
-
-        runinit (&runner);
-        snprintf (msg, sizeof (msg), "umount path %s", path);
-        runner_add_args (&runner, _PATH_UMOUNT, "-f", path, NULL);
-        runner_log (&runner, this->name, GF_LOG_DEBUG, msg);
-        ret = runner_run (&runner);
-        if (ret)
-                gf_log (this->name, GF_LOG_ERROR, "umounting %s failed (%s)",
-                        path, strerror (errno));
-
-        gf_log (this->name, GF_LOG_TRACE, "Returning with %d", ret);
-        return ret;
+    FILE *mtab = NULL;
+    struct mntent *part = NULL;
+    int is_mounted = 0;
+
+    if ((mtab = setmntent("/etc/mtab", "r")) != NULL) {
+        while ((part = getmntent(mtab)) != NULL) {
+            if ((part->mnt_fsname != NULL) &&
+                (strcmp(part->mnt_dir, path)) == 0) {
+                is_mounted = 1;
+                break;
+            }
+        }
+        endmntent(mtab);
+    }
+    return is_mounted;
 }
-
+/* This function will do unmount for snaps.
+ */
 int32_t
-glusterd_copy_file (const char *source, const char *destination)
+glusterd_snap_unmount(xlator_t *this, glusterd_volinfo_t *volinfo)
 {
-        int32_t         ret             =       -1;
-        xlator_t        *this           =       NULL;
-        char            buffer[1024]    =       "";
-        int             src_fd          =       -1;
-        int             dest_fd         =       -1;
-        int             read_len        =       -1;
-        struct  stat    stbuf           =       {0,};
-        mode_t          dest_mode       =       0;
-
-        this = THIS;
-        GF_ASSERT (this);
-
-        GF_ASSERT (source);
-        GF_ASSERT (destination);
-
-        /* Here is stat is made to get the file permission of source file*/
-        ret = lstat (source, &stbuf);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "%s not found", source);
-                goto out;
-        }
+    char *brick_mount_path = NULL;
+    glusterd_brickinfo_t *brickinfo = NULL;
+    int32_t ret = -1;
+    int retry_count = 0;
 
-        dest_mode = stbuf.st_mode & 0777;
+    GF_ASSERT(this);
+    GF_ASSERT(volinfo);
 
-        src_fd = open (source, O_RDONLY);
-        if (src_fd < 0) {
-                ret = -1;
-                gf_log (this->name, GF_LOG_ERROR, "Unable to open file %s",
-                        source);
-                goto out;
+    cds_list_for_each_entry(brickinfo, &volinfo->bricks, brick_list)
+    {
+        /* If the brick is not of this node, we continue */
+        if (gf_uuid_compare(brickinfo->uuid, MY_UUID)) {
+            continue;
+        }
+        /* If snapshot is pending, we continue */
+        if (brickinfo->snap_status == -1) {
+            continue;
         }
 
-        dest_fd = open (destination, O_CREAT | O_RDWR, dest_mode);
-        if (dest_fd < 0) {
-                ret = -1;
-                gf_log (this->name, GF_LOG_ERROR,
-                        "Unble to open a file %s", destination);
-                goto out;
+        ret = glusterd_find_brick_mount_path(brickinfo->path,
+                                             &brick_mount_path);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_BRK_MNTPATH_GET_FAIL,
+                   "Failed to find brick_mount_path for %s", brickinfo->path);
+            goto out;
         }
+        /* unmount cannot be done when the brick process is still in
+         * the process of shutdown, so give three re-tries
+         */
+        retry_count = 0;
+        while (retry_count <= 2) {
+            retry_count++;
+            /* umount2 system call doesn't cleanup mtab entry
+             * after un-mount, using external umount command.
+             */
+            ret = glusterd_umount(brick_mount_path);
+            if (!ret)
+                break;
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_GLUSTERD_UMOUNT_FAIL,
+                   "umount failed "
+                   "for path %s (brick: %s): %s. Retry(%d)",
+                   brick_mount_path, brickinfo->path, strerror(errno),
+                   retry_count);
+            sleep(3);
+        }
+    }
 
-        do {
-                ret = read (src_fd, buffer, sizeof (buffer));
-                if (ret ==  -1) {
-                        gf_log (this->name, GF_LOG_ERROR, "Error reading file "
-                                "%s", source);
-                        goto out;
-                }
-                read_len = ret;
-                if (read_len == 0)
-                        break;
-
-                ret = write (dest_fd, buffer, read_len);
-                if (ret != read_len) {
-                        gf_log (this->name, GF_LOG_ERROR, "Error writing in "
-                                "file %s", destination);
-                        goto out;
-                }
-        } while (ret > 0);
 out:
-        if (src_fd > 0)
-                close (src_fd);
+    if (brick_mount_path)
+        GF_FREE(brick_mount_path);
 
-        if (dest_fd > 0)
-                close (dest_fd);
-        return ret;
+    return ret;
 }
 
 int32_t
-glusterd_copy_folder (const char *source, const char *destination)
+glusterd_umount(const char *path)
 {
-        DIR             *dir_ptr                =       NULL;
-        struct dirent   *direntp                =       NULL;
-        int32_t         ret                     =       -1;
-        char            src_path[PATH_MAX]      =       "";
-        char            dest_path[PATH_MAX]     =       "";
-        xlator_t        *this                   =       NULL;
-
-        this = THIS;
-        GF_ASSERT (this);
-
-        GF_ASSERT (source);
-        GF_ASSERT (destination);
-
-        dir_ptr = opendir (source);
-        if (!dir_ptr) {
-                gf_log (this->name, GF_LOG_ERROR, "Unable to open %s", source);
-                goto out;
-        }
-
-        while ((direntp = readdir (dir_ptr)) != NULL) {
-                if (strcmp (direntp->d_name, ".") == 0 ||
-                    strcmp (direntp->d_name, "..") == 0)
-                        continue;
-                ret = snprintf (src_path, sizeof (src_path), "%s/%s",
-                                source, direntp->d_name);
-                if (ret < 0)
-                        goto out;
-
-                ret = snprintf (dest_path, sizeof (dest_path), "%s/%s",
-                                destination, direntp->d_name);
-                if (ret < 0)
-                        goto out;
+    char msg[NAME_MAX] = "";
+    int32_t ret = -1;
+    runner_t runner = {
+        0,
+    };
+    xlator_t *this = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+    GF_ASSERT(path);
+
+    if (!glusterd_is_path_mounted(path)) {
+        return 0;
+    }
+
+    runinit(&runner);
+    snprintf(msg, sizeof(msg), "umount path %s", path);
+    runner_add_args(&runner, _PATH_UMOUNT, "-f", path, NULL);
+    runner_log(&runner, this->name, GF_LOG_DEBUG, msg);
+    ret = runner_run(&runner);
+    if (ret)
+        gf_msg(this->name, GF_LOG_ERROR, errno, GD_MSG_GLUSTERD_UMOUNT_FAIL,
+               "umounting %s failed (%s)", path, strerror(errno));
+
+    gf_msg_trace(this->name, 0, "Returning with %d", ret);
+    return ret;
+}
 
-                ret = glusterd_copy_file (src_path, dest_path);
-                if (ret) {
-                        gf_log (this->name, GF_LOG_ERROR, "Could not copy "
-                                "%s to %s", src_path, dest_path);
-                        goto out;
-                }
-        }
+int32_t
+glusterd_copy_file(const char *source, const char *destination)
+{
+    int32_t ret = -1;
+    xlator_t *this = NULL;
+    char buffer[1024] = "";
+    int src_fd = -1;
+    int dest_fd = -1;
+    int read_len = -1;
+    struct stat stbuf = {
+        0,
+    };
+    mode_t dest_mode = 0;
+
+    this = THIS;
+    GF_ASSERT(this);
+
+    GF_ASSERT(source);
+    GF_ASSERT(destination);
+
+    /* Here is stat is made to get the file permission of source file*/
+    ret = sys_lstat(source, &stbuf);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, errno, GD_MSG_FILE_OP_FAILED,
+               "%s not found", source);
+        goto out;
+    }
+
+    dest_mode = stbuf.st_mode & 0777;
+
+    src_fd = open(source, O_RDONLY);
+    if (src_fd == -1) {
+        ret = -1;
+        gf_msg(this->name, GF_LOG_ERROR, errno, GD_MSG_FILE_OP_FAILED,
+               "Unable to open file %s", source);
+        goto out;
+    }
+
+    dest_fd = sys_creat(destination, dest_mode);
+    if (dest_fd < 0) {
+        ret = -1;
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_FILE_OP_FAILED,
+               "Unble to open a file %s", destination);
+        goto out;
+    }
+
+    do {
+        ret = sys_read(src_fd, buffer, sizeof(buffer));
+        if (ret == -1) {
+            gf_msg(this->name, GF_LOG_ERROR, errno, GD_MSG_FILE_OP_FAILED,
+                   "Error reading file "
+                   "%s",
+                   source);
+            goto out;
+        }
+        read_len = ret;
+        if (read_len == 0)
+            break;
+
+        ret = sys_write(dest_fd, buffer, read_len);
+        if (ret != read_len) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_FILE_OP_FAILED,
+                   "Writing in "
+                   "file %s failed with error %s",
+                   destination, strerror(errno));
+            goto out;
+        }
+    } while (ret > 0);
 out:
-        if (dir_ptr)
-                closedir (dir_ptr);
+    if (src_fd != -1)
+        sys_close(src_fd);
 
-        return ret;
+    if (dest_fd > 0)
+        sys_close(dest_fd);
+    return ret;
 }
 
 int32_t
-glusterd_get_geo_rep_session (char *slave_key, char *origin_volname,
-                              dict_t *gsync_slaves_dict, char *session,
-                              char *slave)
+glusterd_copy_folder(const char *source, const char *destination)
 {
-        int32_t         ret             =       -1;
-        char            *token          =       NULL;
-        char            *temp           =       NULL;
-        char            *ip             =       NULL;
-        char            *buffer         =       NULL;
-        xlator_t        *this           =       NULL;
-        char            *slave_temp     =       NULL;
-        char            *save_ptr       =       NULL;
-
-        this = THIS;
-        GF_ASSERT (this);
-
-        GF_ASSERT (slave_key);
-        GF_ASSERT (origin_volname);
-        GF_ASSERT (gsync_slaves_dict);
-
-        ret = dict_get_str (gsync_slaves_dict, slave_key, &buffer);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "Failed to "
-                        "get value for key %s", slave_key);
-                goto out;
-        }
-
-        temp = gf_strdup (buffer);
-        if (!temp) {
-                ret = -1;
-                goto out;
-        }
-
-        token = strtok_r (temp, "/", &save_ptr);
-
-        token = strtok_r (NULL, ":", &save_ptr);
-        if (!token) {
-                ret = -1;
-                goto out;
-        }
-        token++;
-
-        ip = gf_strdup (token);
-        if (!ip) {
-                ret = -1;
-                goto out;
-        }
-
-        token = strtok_r (NULL, "\0", &save_ptr);
-        if (!token) {
-                ret = -1;
-                goto out;
-        }
-        token++;
-
-        slave_temp = gf_strdup (token);
-        if (!slave) {
-                ret = -1;
-                goto out;
+    int32_t ret = -1;
+    xlator_t *this = NULL;
+    DIR *dir_ptr = NULL;
+    struct dirent *entry = NULL;
+    struct dirent scratch[2] = {
+        {
+            0,
+        },
+    };
+    char src_path[PATH_MAX] = {
+        0,
+    };
+    char dest_path[PATH_MAX] = {
+        0,
+    };
+
+    this = THIS;
+    GF_ASSERT(this);
+
+    GF_ASSERT(source);
+    GF_ASSERT(destination);
+
+    dir_ptr = sys_opendir(source);
+    if (!dir_ptr) {
+        gf_msg(this->name, GF_LOG_ERROR, errno, GD_MSG_DIR_OP_FAILED,
+               "Unable to open %s", source);
+        goto out;
+    }
+
+    for (;;) {
+        errno = 0;
+        entry = sys_readdir(dir_ptr, scratch);
+        if (!entry || errno != 0)
+            break;
+
+        if (strcmp(entry->d_name, ".") == 0 || strcmp(entry->d_name, "..") == 0)
+            continue;
+        ret = snprintf(src_path, sizeof(src_path), "%s/%s", source,
+                       entry->d_name);
+        if (ret < 0) {
+            gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_COPY_FAIL, NULL);
+            goto out;
         }
 
-        ret = snprintf (session, PATH_MAX, "%s_%s_%s",
-                        origin_volname, ip, slave_temp);
-        if (ret < 0) /* Negative value is an error */
-                goto out;
-
-        ret = snprintf  (slave, PATH_MAX, "%s::%s", ip, slave_temp);
+        ret = snprintf(dest_path, sizeof(dest_path), "%s/%s", destination,
+                       entry->d_name);
         if (ret < 0) {
-                goto out;
+            gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_COPY_FAIL, NULL);
+            goto out;
         }
 
-        ret = 0; /* Success */
-
+        ret = glusterd_copy_file(src_path, dest_path);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, ENOMEM, GD_MSG_NO_MEMORY,
+                   "Could not copy "
+                   "%s to %s",
+                   src_path, dest_path);
+            goto out;
+        }
+    }
 out:
-        if (temp)
-                GF_FREE (temp);
-
-        if (ip)
-                GF_FREE (ip);
+    if (dir_ptr)
+        (void)sys_closedir(dir_ptr);
 
-        if (slave_temp)
-                GF_FREE (slave_temp);
-
-        return ret;
+    return ret;
 }
 
 int32_t
-glusterd_copy_quota_files (glusterd_volinfo_t *src_vol,
-                           glusterd_volinfo_t *dest_vol,
-                           gf_boolean_t *conf_present) {
+glusterd_get_geo_rep_session(char *slave_key, char *origin_volname,
+                             dict_t *gsync_slaves_dict, char *session,
+                             char *slave)
+{
+    int32_t ret = -1;
+    int32_t len = 0;
+    char *token = NULL;
+    char *tok = NULL;
+    char *temp = NULL;
+    char *ip = NULL;
+    char *ip_i = NULL;
+    char *ip_temp = NULL;
+    char *buffer = NULL;
+    xlator_t *this = NULL;
+    char *slave_temp = NULL;
+    char *save_ptr = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+
+    GF_ASSERT(slave_key);
+    GF_ASSERT(origin_volname);
+    GF_ASSERT(gsync_slaves_dict);
+
+    ret = dict_get_str(gsync_slaves_dict, slave_key, &buffer);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+               "Failed to "
+               "get value for key %s",
+               slave_key);
+        goto out;
+    }
+
+    temp = gf_strdup(buffer);
+    if (!temp) {
+        ret = -1;
+        goto out;
+    }
+
+    /* geo-rep session string format being parsed:
+     * "master_node_uuid:ssh://slave_host::slave_vol:slave_voluuid"
+     */
+    token = strtok_r(temp, "/", &save_ptr);
+
+    token = strtok_r(NULL, ":", &save_ptr);
+    if (!token) {
+        ret = -1;
+        goto out;
+    }
+    token++;
+
+    ip = gf_strdup(token);
+    if (!ip) {
+        ret = -1;
+        goto out;
+    }
+    ip_i = ip;
+
+    token = strtok_r(NULL, ":", &save_ptr);
+    if (!token) {
+        ret = -1;
+        goto out;
+    }
+
+    slave_temp = gf_strdup(token);
+    if (!slave) {
+        ret = -1;
+        goto out;
+    }
+
+    /* If 'ip' has 'root@slavehost', point to 'slavehost' as
+     * working directory for root users are created without
+     * 'root@' */
+    ip_temp = gf_strdup(ip);
+    tok = strtok_r(ip_temp, "@", &save_ptr);
+    len = strlen(tok);
+    tok = strtok_r(NULL, "@", &save_ptr);
+    if (tok != NULL)
+        ip_i = ip + len + 1;
+
+    ret = snprintf(session, PATH_MAX, "%s_%s_%s", origin_volname, ip_i,
+                   slave_temp);
+    if (ret < 0) /* Negative value is an error */
+        goto out;
+
+    ret = snprintf(slave, PATH_MAX, "%s::%s", ip, slave_temp);
+    if (ret < 0) {
+        goto out;
+    }
+
+    ret = 0; /* Success */
 
-        int32_t         ret                     = -1;
-        char            src_dir[PATH_MAX]       = "";
-        char            dest_dir[PATH_MAX]      = "";
-        char            src_path[PATH_MAX]      = "";
-        char            dest_path[PATH_MAX]     = "";
-        xlator_t        *this                   = NULL;
-        glusterd_conf_t *priv                   = NULL;
-        struct  stat    stbuf                   = {0,};
+out:
+    if (temp)
+        GF_FREE(temp);
 
-        this = THIS;
-        GF_ASSERT (this);
-        priv = this->private;
-        GF_ASSERT (priv);
+    if (ip)
+        GF_FREE(ip);
 
-        GF_ASSERT (src_vol);
-        GF_ASSERT (dest_vol);
+    if (ip_temp)
+        GF_FREE(ip_temp);
 
-        GLUSTERD_GET_VOLUME_DIR (src_dir, src_vol, priv);
+    if (slave_temp)
+        GF_FREE(slave_temp);
 
-        GLUSTERD_GET_VOLUME_DIR (dest_dir, dest_vol, priv);
+    return ret;
+}
 
-        ret = snprintf (src_path, sizeof (src_path), "%s/quota.conf",
-                        src_dir);
-        if (ret < 0)
-                goto out;
+int32_t
+glusterd_copy_quota_files(glusterd_volinfo_t *src_vol,
+                          glusterd_volinfo_t *dest_vol,
+                          gf_boolean_t *conf_present)
+{
+    int32_t ret = -1;
+    char src_dir[PATH_MAX] = "";
+    char dest_dir[PATH_MAX] = "";
+    char src_path[PATH_MAX] = "";
+    char dest_path[PATH_MAX] = "";
+    xlator_t *this = NULL;
+    glusterd_conf_t *priv = NULL;
+    struct stat stbuf = {
+        0,
+    };
+
+    this = THIS;
+    GF_ASSERT(this);
+    priv = this->private;
+    GF_ASSERT(priv);
+
+    GF_ASSERT(src_vol);
+    GF_ASSERT(dest_vol);
+
+    GLUSTERD_GET_VOLUME_DIR(src_dir, src_vol, priv);
+
+    GLUSTERD_GET_VOLUME_DIR(dest_dir, dest_vol, priv);
+
+    ret = snprintf(src_path, sizeof(src_path), "%s/quota.conf", src_dir);
+    if (ret < 0) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_COPY_FAIL, NULL);
+        goto out;
+    }
+
+    /* quota.conf is not present if quota is not enabled, Hence ignoring
+     * the absence of this file
+     */
+    ret = sys_lstat(src_path, &stbuf);
+    if (ret) {
+        ret = 0;
+        gf_msg_debug(this->name, 0, "%s not found", src_path);
+        goto out;
+    }
+
+    ret = snprintf(dest_path, sizeof(dest_path), "%s/quota.conf", dest_dir);
+    if (ret < 0) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_COPY_FAIL, NULL);
+        goto out;
+    }
+
+    ret = glusterd_copy_file(src_path, dest_path);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, ENOMEM, GD_MSG_NO_MEMORY,
+               "Failed to copy %s in %s", src_path, dest_path);
+        goto out;
+    }
+
+    ret = snprintf(src_path, sizeof(src_path), "%s/quota.cksum", src_dir);
+    if (ret < 0)
+        goto out;
+
+    /* if quota.conf is present, quota.cksum has to be present. *
+     * Fail snapshot operation if file is absent                *
+     */
+    ret = sys_lstat(src_path, &stbuf);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_FILE_NOT_FOUND,
+               "%s not found", src_path);
+        goto out;
+    }
+
+    ret = snprintf(dest_path, sizeof(dest_path), "%s/quota.cksum", dest_dir);
+    if (ret < 0) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_COPY_FAIL, NULL);
+        goto out;
+    }
+
+    ret = glusterd_copy_file(src_path, dest_path);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, ENOMEM, GD_MSG_NO_MEMORY,
+               "Failed to copy %s in %s", src_path, dest_path);
+        goto out;
+    }
+
+    *conf_present = _gf_true;
+out:
+    return ret;
+}
 
-        /* quota.conf is not present if quota is not enabled, Hence ignoring
-         * the absence of this file
+/* *
+ * Here there are two possibilities, either destination is snaphot or
+ * clone. In the case of snapshot nfs_ganesha export file will be copied
+ * to snapdir. If it is clone , then new export file will be created for
+ * the clone in the GANESHA_EXPORT_DIRECTORY, replacing occurences of
+ * volname with clonename
+ */
+int
+glusterd_copy_nfs_ganesha_file(glusterd_volinfo_t *src_vol,
+                               glusterd_volinfo_t *dest_vol)
+{
+    int32_t ret = -1;
+    char snap_dir[PATH_MAX] = {
+        0,
+    };
+    char src_path[PATH_MAX] = {
+        0,
+    };
+    char dest_path[PATH_MAX] = {
+        0,
+    };
+    char buffer[BUFSIZ] = {
+        0,
+    };
+    char *find_ptr = NULL;
+    char *buff_ptr = NULL;
+    char *tmp_ptr = NULL;
+    xlator_t *this = NULL;
+    glusterd_conf_t *priv = NULL;
+    struct stat stbuf = {
+        0,
+    };
+    FILE *src = NULL;
+    FILE *dest = NULL;
+
+    this = THIS;
+    GF_VALIDATE_OR_GOTO("snapshot", this, out);
+    priv = this->private;
+    GF_VALIDATE_OR_GOTO(this->name, priv, out);
+
+    GF_VALIDATE_OR_GOTO(this->name, src_vol, out);
+    GF_VALIDATE_OR_GOTO(this->name, dest_vol, out);
+
+    if (glusterd_check_ganesha_export(src_vol) == _gf_false) {
+        gf_msg_debug(this->name, 0,
+                     "%s is not exported via "
+                     "NFS-Ganesha. Skipping copy of export conf.",
+                     src_vol->volname);
+        ret = 0;
+        goto out;
+    }
+
+    if (src_vol->is_snap_volume) {
+        GLUSTERD_GET_SNAP_DIR(snap_dir, src_vol->snapshot, priv);
+        ret = snprintf(src_path, PATH_MAX, "%s/export.%s.conf", snap_dir,
+                       src_vol->snapshot->snapname);
+    } else {
+        ret = snprintf(src_path, PATH_MAX, "%s/export.%s.conf",
+                       GANESHA_EXPORT_DIRECTORY, src_vol->volname);
+    }
+    if (ret < 0 || ret >= PATH_MAX)
+        goto out;
+
+    ret = sys_lstat(src_path, &stbuf);
+    if (ret) {
+        /*
+         * This code path is hit, only when the src_vol is being *
+         * exported via NFS-Ganesha. So if the conf file is not  *
+         * available, we fail the snapshot operation.            *
          */
-        ret = lstat (src_path, &stbuf);
+        gf_msg(this->name, GF_LOG_ERROR, errno, GD_MSG_FILE_OP_FAILED,
+               "Stat on %s failed with %s", src_path, strerror(errno));
+        goto out;
+    }
+
+    if (dest_vol->is_snap_volume) {
+        memset(snap_dir, 0, PATH_MAX);
+        GLUSTERD_GET_SNAP_DIR(snap_dir, dest_vol->snapshot, priv);
+        ret = snprintf(dest_path, sizeof(dest_path), "%s/export.%s.conf",
+                       snap_dir, dest_vol->snapshot->snapname);
+        if (ret < 0)
+            goto out;
+
+        ret = glusterd_copy_file(src_path, dest_path);
         if (ret) {
-                ret = 0;
-                gf_log (this->name, GF_LOG_DEBUG, "%s not found", src_path);
-                goto out;
+            gf_msg(this->name, GF_LOG_ERROR, ENOMEM, GD_MSG_NO_MEMORY,
+                   "Failed to copy %s in %s", src_path, dest_path);
+            goto out;
         }
 
-        ret = snprintf (dest_path, sizeof (dest_path), "%s/quota.conf",
-                       dest_dir);
+    } else {
+        ret = snprintf(dest_path, sizeof(dest_path), "%s/export.%s.conf",
+                       GANESHA_EXPORT_DIRECTORY, dest_vol->volname);
         if (ret < 0)
-                goto out;
+            goto out;
 
-        ret = glusterd_copy_file (src_path, dest_path);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "Failed to copy %s in %s",
-                        src_path, dest_path);
-                goto out;
+        src = fopen(src_path, "r");
+        dest = fopen(dest_path, "w");
+
+        if (!src || !dest) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_FILE_OP_FAILED,
+                   "Failed to open %s", dest ? src_path : dest_path);
+            ret = -1;
+            goto out;
         }
 
-        *conf_present = _gf_true;
+        /* *
+         * if the source volume is snapshot, the export conf file
+         * consists of orginal volname
+         */
+        if (src_vol->is_snap_volume)
+            find_ptr = gf_strdup(src_vol->parent_volname);
+        else
+            find_ptr = gf_strdup(src_vol->volname);
+
+        if (!find_ptr)
+            goto out;
+
+        /* Replacing volname with clonename */
+        while (fgets(buffer, BUFSIZ, src)) {
+            buff_ptr = buffer;
+            while ((tmp_ptr = strstr(buff_ptr, find_ptr))) {
+                while (buff_ptr < tmp_ptr)
+                    fputc((int)*buff_ptr++, dest);
+                fputs(dest_vol->volname, dest);
+                buff_ptr += strlen(find_ptr);
+            }
+            fputs(buff_ptr, dest);
+            memset(buffer, 0, BUFSIZ);
+        }
+    }
 out:
-        return ret;
-
+    if (src)
+        fclose(src);
+    if (dest)
+        fclose(dest);
+    if (find_ptr)
+        GF_FREE(find_ptr);
+
+    return ret;
 }
 
 int32_t
-glusterd_restore_geo_rep_files (glusterd_volinfo_t *snap_vol)
+glusterd_restore_geo_rep_files(glusterd_volinfo_t *snap_vol)
 {
-        int32_t                 ret                     =       -1;
-        char                    src_path[PATH_MAX]      =       "";
-        char                    dest_path[PATH_MAX]     =       "";
-        xlator_t                *this                   =       NULL;
-        char                    *origin_volname         =       NULL;
-        glusterd_volinfo_t      *origin_vol             =       NULL;
-        int                     i                       =       0;
-        char                    key[PATH_MAX]           =       "";
-        char                    session[PATH_MAX]       =       "";
-        char                    slave[PATH_MAX]         =       "";
-        char                    snapgeo_dir[PATH_MAX]   =       "";
-        glusterd_conf_t         *priv                   =       NULL;
-
-        this = THIS;
-        GF_ASSERT (this);
-        priv = this->private;
-        GF_ASSERT (priv);
-
-        GF_ASSERT (snap_vol);
-
-        origin_volname = gf_strdup (snap_vol->parent_volname);
-        if (!origin_volname) {
-                ret = -1;
-                goto out;
+    int32_t ret = -1;
+    char src_path[PATH_MAX] = "";
+    char dest_path[PATH_MAX] = "";
+    xlator_t *this = NULL;
+    char *origin_volname = NULL;
+    glusterd_volinfo_t *origin_vol = NULL;
+    int i = 0;
+    char key[32] = "";
+    char session[PATH_MAX] = "";
+    char slave[PATH_MAX] = "";
+    char snapgeo_dir[PATH_MAX] = "";
+    glusterd_conf_t *priv = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+    priv = this->private;
+    GF_ASSERT(priv);
+
+    GF_ASSERT(snap_vol);
+
+    origin_volname = gf_strdup(snap_vol->parent_volname);
+    if (!origin_volname) {
+        ret = -1;
+        goto out;
+    }
+
+    ret = glusterd_volinfo_find(origin_volname, &origin_vol);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_VOL_NOT_FOUND,
+               "Unable to fetch "
+               "volinfo for volname %s",
+               origin_volname);
+        goto out;
+    }
+
+    for (i = 1; i <= snap_vol->gsync_slaves->count; i++) {
+        ret = snprintf(key, sizeof(key), "slave%d", i);
+        if (ret < 0) {
+            goto out;
         }
 
-        ret = glusterd_volinfo_find (origin_volname, &origin_vol);
+        /* "origin_vol" is used here because geo-replication saves
+         * the session in the form of master_ip_slave.
+         * As we need the master volume to be same even after
+         * restore, we are passing the origin volume name.
+         *
+         * "snap_vol->gsync_slaves" contain the slave information
+         * when the snapshot was taken, hence we have to restore all
+         * those slaves information when we do snapshot restore.
+         */
+        ret = glusterd_get_geo_rep_session(
+            key, origin_vol->volname, snap_vol->gsync_slaves, session, slave);
         if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "Unable to fetch "
-                        "volinfo for volname %s", origin_volname);
-                goto out;
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_GEOREP_GET_FAILED,
+                   "Failed to get geo-rep session");
+            goto out;
         }
 
-        for (i = 1 ; i <= snap_vol->gsync_slaves->count; i++) {
-                ret = snprintf (key, sizeof (key), "slave%d", i);
-                if (ret < 0) {
-                        goto out;
-                }
-
-                /* "origin_vol" is used here because geo-replication saves
-                 * the session in the form of master_ip_slave.
-                 * As we need the master volume to be same even after
-                 * restore, we are passing the origin volume name.
-                 *
-                 * "snap_vol->gsync_slaves" contain the slave information
-                 * when the snapshot was taken, hence we have to restore all
-                 * those slaves information when we do snapshot restore.
-                 */
-                ret = glusterd_get_geo_rep_session (key, origin_vol->volname,
-                                                    snap_vol->gsync_slaves,
-                                                    session, slave);
-                if (ret) {
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "Failed to get geo-rep session");
-                        goto out;
-                }
-
-                GLUSTERD_GET_SNAP_GEO_REP_DIR(snapgeo_dir, snap_vol->snapshot,
-                                              priv);
-                ret = snprintf (src_path, sizeof (src_path),
-                                "%s/%s", snapgeo_dir, session);
-                if (ret < 0)
-                        goto out;
+        GLUSTERD_GET_SNAP_GEO_REP_DIR(snapgeo_dir, snap_vol->snapshot, priv);
+        ret = snprintf(src_path, sizeof(src_path), "%s/%s", snapgeo_dir,
+                       session);
+        if (ret < 0)
+            goto out;
 
-                ret = snprintf (dest_path, sizeof (dest_path),
-                                "%s/%s/%s", priv->workdir, GEOREP,
-                                session);
-                if (ret < 0)
-                        goto out;
+        ret = snprintf(dest_path, sizeof(dest_path), "%s/%s/%s", priv->workdir,
+                       GEOREP, session);
+        if (ret < 0)
+            goto out;
 
-                ret = glusterd_copy_folder (src_path, dest_path);
-                if (ret) {
-                        gf_log (this->name, GF_LOG_ERROR, "Could not copy "
-                                "%s to %s", src_path, dest_path);
-                        goto out;
-                }
+        ret = glusterd_copy_folder(src_path, dest_path);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DIR_OP_FAILED,
+                   "Could not copy "
+                   "%s to %s",
+                   src_path, dest_path);
+            goto out;
         }
+    }
 out:
-        if (origin_volname)
-                GF_ASSERT (origin_volname);
+    if (origin_volname)
+        GF_FREE(origin_volname);
 
-        return ret;
+    return ret;
 }
 
-/* Snapd functions */
 int
-glusterd_is_snapd_enabled (glusterd_volinfo_t *volinfo)
+glusterd_restore_nfs_ganesha_file(glusterd_volinfo_t *src_vol,
+                                  glusterd_snap_t *snap)
 {
-        int              ret    = 0;
-        xlator_t        *this   = THIS;
-
-        ret = dict_get_str_boolean (volinfo->dict, "features.uss", -2);
-        if (ret == -2) {
-                gf_log (this->name, GF_LOG_DEBUG, "Key features.uss not "
-                        "present in the dict for volume %s", volinfo->volname);
-                ret = 0;
-
-        } else if (ret == -1) {
-                gf_log (this->name, GF_LOG_ERROR, "Failed to get 'features.uss'"
-                        " from dict for volume %s", volinfo->volname);
-        }
+    int32_t ret = -1;
+    char snap_dir[PATH_MAX] = "";
+    char src_path[PATH_MAX] = "";
+    char dest_path[PATH_MAX] = "";
+    xlator_t *this = NULL;
+    glusterd_conf_t *priv = NULL;
+    struct stat stbuf = {
+        0,
+    };
+
+    this = THIS;
+    GF_VALIDATE_OR_GOTO("snapshot", this, out);
+    priv = this->private;
+    GF_VALIDATE_OR_GOTO(this->name, priv, out);
+
+    GF_VALIDATE_OR_GOTO(this->name, src_vol, out);
+    GF_VALIDATE_OR_GOTO(this->name, snap, out);
+
+    GLUSTERD_GET_SNAP_DIR(snap_dir, snap, priv);
+
+    ret = snprintf(src_path, sizeof(src_path), "%s/export.%s.conf", snap_dir,
+                   snap->snapname);
+    if (ret < 0) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_COPY_FAIL, NULL);
+        goto out;
+    }
+
+    ret = sys_lstat(src_path, &stbuf);
+    if (ret) {
+        if (errno == ENOENT) {
+            ret = 0;
+            gf_msg_debug(this->name, 0, "%s not found", src_path);
+        } else
+            gf_msg(this->name, GF_LOG_WARNING, errno, GD_MSG_FILE_OP_FAILED,
+                   "Stat on %s failed with %s", src_path, strerror(errno));
+        goto out;
+    }
+
+    ret = snprintf(dest_path, sizeof(dest_path), "%s/export.%s.conf",
+                   GANESHA_EXPORT_DIRECTORY, src_vol->volname);
+    if (ret < 0) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_COPY_FAIL, NULL);
+        goto out;
+    }
+
+    ret = glusterd_copy_file(src_path, dest_path);
+    if (ret)
+        gf_msg(this->name, GF_LOG_ERROR, ENOMEM, GD_MSG_NO_MEMORY,
+               "Failed to copy %s in %s", src_path, dest_path);
 
-        return ret;
+out:
+    return ret;
 }
 
-
-int32_t
-glusterd_is_snap_soft_limit_reached (glusterd_volinfo_t *volinfo, dict_t *dict)
+/* Snapd functions */
+int
+glusterd_is_snapd_enabled(glusterd_volinfo_t *volinfo)
 {
-        int32_t         ret            = -1;
-        uint64_t        opt_max_hard   = GLUSTERD_SNAPS_MAX_HARD_LIMIT;
-        uint64_t        opt_max_soft   = GLUSTERD_SNAPS_DEF_SOFT_LIMIT_PERCENT;
-        uint64_t        limit          = 0;
-        int             auto_delete    = 0;
-        uint64_t        effective_max_limit = 0;
-        xlator_t        *this          = NULL;
-        glusterd_conf_t *priv          = NULL;
-
-        GF_ASSERT (volinfo);
-        GF_ASSERT (dict);
-
-        this = THIS;
-        GF_ASSERT (this);
-        priv = this->private;
-        GF_ASSERT (priv);
-
-        /* config values snap-max-hard-limit and snap-max-soft-limit are
-         * optional and hence we are not erroring out if values are not
-         * present
-         */
-        gd_get_snap_conf_values_if_present (priv->opts, &opt_max_hard,
-                                            &opt_max_soft);
-
-        /* "auto-delete" might not be set by user explicitly,
-         * in that case it's better to consider the default value.
-         * Hence not erroring out if Key is not found.
-         */
-        auto_delete = dict_get_str_boolean (priv->opts,
-                                GLUSTERD_STORE_KEY_SNAP_AUTO_DELETE,
-                                _gf_false);
-
-        if (volinfo->snap_max_hard_limit < opt_max_hard)
-                effective_max_limit = volinfo->snap_max_hard_limit;
-        else
-                effective_max_limit = opt_max_hard;
+    int ret = 0;
+    xlator_t *this = THIS;
+
+    ret = dict_get_str_boolean(volinfo->dict, "features.uss", -2);
+    if (ret == -2) {
+        gf_msg_debug(this->name, 0,
+                     "Key features.uss not "
+                     "present in the dict for volume %s",
+                     volinfo->volname);
+        ret = 0;
 
-        limit = (opt_max_soft * effective_max_limit)/100;
+    } else if (ret == -1) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+               "Failed to get 'features.uss'"
+               " from dict for volume %s",
+               volinfo->volname);
+    }
 
-        if (volinfo->snap_count >= limit && auto_delete != _gf_true) {
-                gf_log (this->name, GF_LOG_WARNING, "Soft-limit "
-                        "(value = %"PRIu64") of volume %s is reached. "
-                        "Snapshot creation is not possible once effective "
-                        "hard-limit (value = %"PRIu64") is reached.",
-                        limit, volinfo->volname, effective_max_limit);
+    return ret;
+}
 
-                ret = dict_set_int8 (dict, "soft-limit-reach",
-                                     _gf_true);
-                if (ret) {
-                        gf_log (this->name, GF_LOG_ERROR, "Failed to "
-                                "set soft limit exceed flag in "
-                                "response dictionary");
-                }
-                goto out;
+int32_t
+glusterd_is_snap_soft_limit_reached(glusterd_volinfo_t *volinfo, dict_t *dict)
+{
+    int32_t ret = -1;
+    uint64_t opt_max_hard = GLUSTERD_SNAPS_MAX_HARD_LIMIT;
+    uint64_t opt_max_soft = GLUSTERD_SNAPS_DEF_SOFT_LIMIT_PERCENT;
+    uint64_t limit = 0;
+    int auto_delete = 0;
+    uint64_t effective_max_limit = 0;
+    xlator_t *this = NULL;
+    glusterd_conf_t *priv = NULL;
+
+    GF_ASSERT(volinfo);
+    GF_ASSERT(dict);
+
+    this = THIS;
+    GF_ASSERT(this);
+    priv = this->private;
+    GF_ASSERT(priv);
+
+    /* config values snap-max-hard-limit and snap-max-soft-limit are
+     * optional and hence we are not erroring out if values are not
+     * present
+     */
+    gd_get_snap_conf_values_if_present(priv->opts, &opt_max_hard,
+                                       &opt_max_soft);
+
+    /* "auto-delete" might not be set by user explicitly,
+     * in that case it's better to consider the default value.
+     * Hence not erroring out if Key is not found.
+     */
+    auto_delete = dict_get_str_boolean(
+        priv->opts, GLUSTERD_STORE_KEY_SNAP_AUTO_DELETE, _gf_false);
+
+    if (volinfo->snap_max_hard_limit < opt_max_hard)
+        effective_max_limit = volinfo->snap_max_hard_limit;
+    else
+        effective_max_limit = opt_max_hard;
+
+    limit = (opt_max_soft * effective_max_limit) / 100;
+
+    if (volinfo->snap_count >= limit && auto_delete != _gf_true) {
+        gf_msg(this->name, GF_LOG_WARNING, 0, GD_MSG_SOFT_LIMIT_REACHED,
+               "Soft-limit "
+               "(value = %" PRIu64
+               ") of volume %s is reached. "
+               "Snapshot creation is not possible once effective "
+               "hard-limit (value = %" PRIu64 ") is reached.",
+               limit, volinfo->volname, effective_max_limit);
+
+        ret = dict_set_int8(dict, "soft-limit-reach", _gf_true);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+                   "Failed to "
+                   "set soft limit exceed flag in "
+                   "response dictionary");
         }
-        ret = 0;
+
+        goto out;
+    }
+    ret = 0;
 out:
-        return ret;
+    return ret;
 }
 
 /* This function initializes the parameter sys_hard_limit,
@@ -3290,35 +4219,72 @@ out:
  * return any values.
  */
 void
-gd_get_snap_conf_values_if_present (dict_t *dict, uint64_t *sys_hard_limit,
-                                    uint64_t *sys_soft_limit)
+gd_get_snap_conf_values_if_present(dict_t *dict, uint64_t *sys_hard_limit,
+                                   uint64_t *sys_soft_limit)
 {
-        xlator_t        *this   = NULL;
-
-        this = THIS;
-        GF_ASSERT (this);
-
-        GF_ASSERT (dict);
-
-        /* "snap-max-hard-limit" might not be set by user explicitly,
-         * in that case it's better to consider the default value.
-         * Hence not erroring out if Key is not found.
-         */
-        if (dict_get_uint64 (dict, GLUSTERD_STORE_KEY_SNAP_MAX_HARD_LIMIT,
-                               sys_hard_limit)) {
-                gf_log (this->name, GF_LOG_DEBUG, "%s is not present in"
-                        "dictionary",
-                        GLUSTERD_STORE_KEY_SNAP_MAX_HARD_LIMIT);
-        }
+    xlator_t *this = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+
+    GF_ASSERT(dict);
+
+    /* "snap-max-hard-limit" might not be set by user explicitly,
+     * in that case it's better to consider the default value.
+     * Hence not erroring out if Key is not found.
+     */
+    if (dict_get_uint64(dict, GLUSTERD_STORE_KEY_SNAP_MAX_HARD_LIMIT,
+                        sys_hard_limit)) {
+        gf_msg_debug(this->name, 0,
+                     "%s is not present in"
+                     "dictionary",
+                     GLUSTERD_STORE_KEY_SNAP_MAX_HARD_LIMIT);
+    }
+
+    /* "snap-max-soft-limit" might not be set by user explicitly,
+     * in that case it's better to consider the default value.
+     * Hence not erroring out if Key is not found.
+     */
+    if (dict_get_uint64(dict, GLUSTERD_STORE_KEY_SNAP_MAX_SOFT_LIMIT,
+                        sys_soft_limit)) {
+        gf_msg_debug(this->name, 0,
+                     "%s is not present in"
+                     "dictionary",
+                     GLUSTERD_STORE_KEY_SNAP_MAX_SOFT_LIMIT);
+    }
+}
 
-        /* "snap-max-soft-limit" might not be set by user explicitly,
-         * in that case it's better to consider the default value.
-         * Hence not erroring out if Key is not found.
-         */
-        if (dict_get_uint64 (dict, GLUSTERD_STORE_KEY_SNAP_MAX_SOFT_LIMIT,
-                              sys_soft_limit)) {
-                gf_log (this->name, GF_LOG_DEBUG, "%s is not present in"
-                        "dictionary",
-                        GLUSTERD_STORE_KEY_SNAP_MAX_SOFT_LIMIT);
-        }
+int
+glusterd_get_snap_status_str(glusterd_snap_t *snapinfo, char *snap_status_str)
+{
+    int ret = -1;
+
+    GF_VALIDATE_OR_GOTO(THIS->name, snapinfo, out);
+    GF_VALIDATE_OR_GOTO(THIS->name, snap_status_str, out);
+
+    switch (snapinfo->snap_status) {
+        case GD_SNAP_STATUS_NONE:
+            sprintf(snap_status_str, "%s", "none");
+            break;
+        case GD_SNAP_STATUS_INIT:
+            sprintf(snap_status_str, "%s", "init");
+            break;
+        case GD_SNAP_STATUS_IN_USE:
+            sprintf(snap_status_str, "%s", "in_use");
+            break;
+        case GD_SNAP_STATUS_DECOMMISSION:
+            sprintf(snap_status_str, "%s", "decommissioned");
+            break;
+        case GD_SNAP_STATUS_UNDER_RESTORE:
+            sprintf(snap_status_str, "%s", "under_restore");
+            break;
+        case GD_SNAP_STATUS_RESTORED:
+            sprintf(snap_status_str, "%s", "restored");
+            break;
+        default:
+            goto out;
+    }
+    ret = 0;
+out:
+    return ret;
 }
diff --git a/xlators/mgmt/glusterd/src/glusterd-snapshot-utils.h b/xlators/mgmt/glusterd/src/glusterd-snapshot-utils.h
index 68906dcf59e..5762999bba7 100644
--- a/xlators/mgmt/glusterd/src/glusterd-snapshot-utils.h
+++ b/xlators/mgmt/glusterd/src/glusterd-snapshot-utils.h
@@ -10,149 +10,160 @@
 #ifndef _GLUSTERD_SNAP_UTILS_H
 #define _GLUSTERD_SNAP_UTILS_H
 
+#define GLUSTERD_GET_SNAP_DIR(path, snap, priv)                                \
+    do {                                                                       \
+        int32_t _snap_dir_len;                                                 \
+        _snap_dir_len = snprintf(path, PATH_MAX, "%s/snaps/%s", priv->workdir, \
+                                 snap->snapname);                              \
+        if ((_snap_dir_len < 0) || (_snap_dir_len >= PATH_MAX)) {              \
+            path[0] = 0;                                                       \
+        }                                                                      \
+    } while (0)
+
 int32_t
-glusterd_snap_volinfo_find (char *volname, glusterd_snap_t *snap,
-                            glusterd_volinfo_t **volinfo);
+glusterd_snap_volinfo_find(char *volname, glusterd_snap_t *snap,
+                           glusterd_volinfo_t **volinfo);
 
 int32_t
-glusterd_snap_volinfo_find_from_parent_volname (char *origin_volname,
-                                      glusterd_snap_t *snap,
-                                      glusterd_volinfo_t **volinfo);
+glusterd_snap_volinfo_find_from_parent_volname(char *origin_volname,
+                                               glusterd_snap_t *snap,
+                                               glusterd_volinfo_t **volinfo);
 
 int
-glusterd_snap_volinfo_find_by_volume_id (uuid_t volume_id,
-                                         glusterd_volinfo_t **volinfo);
+glusterd_snap_volinfo_find_by_volume_id(uuid_t volume_id,
+                                        glusterd_volinfo_t **volinfo);
 
 int32_t
-glusterd_add_snapd_to_dict (glusterd_volinfo_t *volinfo,
-                            dict_t  *dict, int32_t count);
+glusterd_add_snapd_to_dict(glusterd_volinfo_t *volinfo, dict_t *dict,
+                           int32_t count);
 
 int
-glusterd_compare_snap_time (struct cds_list_head *, struct cds_list_head *);
+glusterd_compare_snap_time(struct cds_list_head *, struct cds_list_head *);
 
 int
-glusterd_compare_snap_vol_time (struct cds_list_head *, struct cds_list_head *);
+glusterd_compare_snap_vol_time(struct cds_list_head *, struct cds_list_head *);
 
 int32_t
-glusterd_snap_volinfo_restore (dict_t *dict, dict_t *rsp_dict,
-                               glusterd_volinfo_t *new_volinfo,
-                               glusterd_volinfo_t *snap_volinfo,
-                               int32_t volcount);
+glusterd_snap_volinfo_restore(dict_t *dict, dict_t *rsp_dict,
+                              glusterd_volinfo_t *new_volinfo,
+                              glusterd_volinfo_t *snap_volinfo,
+                              int32_t volcount);
+int32_t
+glusterd_snapobject_delete(glusterd_snap_t *snap);
 
 int32_t
-glusterd_missed_snapinfo_new (glusterd_missed_snap_info **missed_snapinfo);
+glusterd_cleanup_snaps_for_volume(glusterd_volinfo_t *volinfo);
 
 int32_t
-glusterd_missed_snap_op_new (glusterd_snap_op_t **snap_op);
+glusterd_missed_snapinfo_new(glusterd_missed_snap_info **missed_snapinfo);
 
 int32_t
-glusterd_add_missed_snaps_to_dict (dict_t *rsp_dict,
-                                   glusterd_volinfo_t *snap_vol,
-                                   glusterd_brickinfo_t *brickinfo,
-                                   int32_t brick_number, int32_t op);
+glusterd_missed_snap_op_new(glusterd_snap_op_t **snap_op);
 
 int32_t
-glusterd_add_missed_snaps_to_export_dict (dict_t *peer_data);
+glusterd_add_missed_snaps_to_dict(dict_t *rsp_dict,
+                                  glusterd_volinfo_t *snap_vol,
+                                  glusterd_brickinfo_t *brickinfo,
+                                  int32_t brick_number, int32_t op);
 
 int32_t
-glusterd_import_friend_missed_snap_list (dict_t *peer_data);
+glusterd_add_missed_snaps_to_export_dict(dict_t *peer_data);
+
+int32_t
+glusterd_import_friend_missed_snap_list(dict_t *peer_data);
 
 int
-gd_restore_snap_volume (dict_t *dict, dict_t *rsp_dict,
-                        glusterd_volinfo_t *orig_vol,
-                        glusterd_volinfo_t *snap_vol,
-                        int32_t volcount);
+gd_restore_snap_volume(dict_t *dict, dict_t *rsp_dict,
+                       glusterd_volinfo_t *orig_vol,
+                       glusterd_volinfo_t *snap_vol, int32_t volcount);
 
 int32_t
-glusterd_mount_lvm_snapshot (glusterd_brickinfo_t *brickinfo,
-                             char *brick_mount_path);
+glusterd_mount_lvm_snapshot(glusterd_brickinfo_t *brickinfo,
+                            char *brick_mount_path);
 
 int32_t
-glusterd_umount (const char *path);
+glusterd_umount(const char *path);
 
 int32_t
-glusterd_add_snapshots_to_export_dict (dict_t *peer_data);
+glusterd_snap_unmount(xlator_t *this, glusterd_volinfo_t *volinfo);
 
 int32_t
-glusterd_compare_friend_snapshots (dict_t *peer_data, char *peername,
-                                   uuid_t peerid);
+glusterd_add_snapshots_to_export_dict(dict_t *peer_data);
 
 int32_t
-glusterd_store_create_snap_dir (glusterd_snap_t *snap);
+glusterd_compare_friend_snapshots(dict_t *peer_data, char *peername,
+                                  uuid_t peerid);
 
 int32_t
-glusterd_copy_file (const char *source, const char *destination);
+glusterd_store_create_snap_dir(glusterd_snap_t *snap);
 
 int32_t
-glusterd_copy_folder (const char *source, const char *destination);
+glusterd_copy_file(const char *source, const char *destination);
 
 int32_t
-glusterd_get_geo_rep_session (char *slave_key, char *origin_volname,
-                              dict_t *gsync_slaves_dict, char *session,
-                              char *slave);
+glusterd_copy_folder(const char *source, const char *destination);
 
 int32_t
-glusterd_restore_geo_rep_files (glusterd_volinfo_t *snap_vol);
+glusterd_get_geo_rep_session(char *slave_key, char *origin_volname,
+                             dict_t *gsync_slaves_dict, char *session,
+                             char *slave);
 
 int32_t
-glusterd_copy_quota_files (glusterd_volinfo_t *src_vol,
-                           glusterd_volinfo_t *dest_vol,
-                           gf_boolean_t *conf_present);
+glusterd_restore_geo_rep_files(glusterd_volinfo_t *snap_vol);
 
-int
-glusterd_snap_use_rsp_dict (dict_t *aggr, dict_t *rsp_dict);
+int32_t
+glusterd_copy_quota_files(glusterd_volinfo_t *src_vol,
+                          glusterd_volinfo_t *dest_vol,
+                          gf_boolean_t *conf_present);
 
 int
-gd_add_vol_snap_details_to_dict (dict_t *dict, char *prefix,
-                                 glusterd_volinfo_t *volinfo);
+glusterd_snap_use_rsp_dict(dict_t *aggr, dict_t *rsp_dict);
 
 int
-gd_add_brick_snap_details_to_dict (dict_t *dict, char *prefix,
-                                   glusterd_brickinfo_t *brickinfo);
+gd_add_vol_snap_details_to_dict(dict_t *dict, char *prefix,
+                                glusterd_volinfo_t *volinfo);
 
 int
-gd_import_new_brick_snap_details (dict_t *dict, char *prefix,
+gd_add_brick_snap_details_to_dict(dict_t *dict, char *prefix,
                                   glusterd_brickinfo_t *brickinfo);
 
 int
-gd_import_volume_snap_details (dict_t *dict, glusterd_volinfo_t *volinfo,
-                               char *prefix, char *volname);
+gd_import_new_brick_snap_details(dict_t *dict, char *prefix,
+                                 glusterd_brickinfo_t *brickinfo);
 
-int32_t
-glusterd_snap_quorum_check (dict_t *dict, gf_boolean_t snap_volume,
-                            char **op_errstr);
+int
+gd_import_volume_snap_details(dict_t *dict, glusterd_volinfo_t *volinfo,
+                              char *prefix, char *volname);
 
 int32_t
-glusterd_snap_quorum_check_for_create (dict_t *dict, gf_boolean_t snap_volume,
-                                       char **op_errstr);
+glusterd_snap_quorum_check(dict_t *dict, gf_boolean_t snap_volume,
+                           char **op_errstr, uint32_t *op_errno);
 
 int32_t
-glusterd_snap_brick_create (glusterd_volinfo_t *snap_volinfo,
-                            glusterd_brickinfo_t *brickinfo,
-                            int32_t brick_count);
+glusterd_snap_brick_create(glusterd_volinfo_t *snap_volinfo,
+                           glusterd_brickinfo_t *brickinfo, int32_t brick_count,
+                           int32_t clone);
 
 int
-glusterd_snapshot_restore_cleanup (dict_t *rsp_dict,
-                                   char *volname,
-                                   glusterd_snap_t *snap);
+glusterd_snapshot_restore_cleanup(dict_t *rsp_dict, char *volname,
+                                  glusterd_snap_t *snap);
 
 void
-glusterd_get_snapd_dir (glusterd_volinfo_t *volinfo,
-                        char *path, int path_len);
+glusterd_get_snapd_dir(glusterd_volinfo_t *volinfo, char *path, int path_len);
 
 int
-glusterd_is_snapd_enabled (glusterd_volinfo_t *volinfo);
+glusterd_is_snapd_enabled(glusterd_volinfo_t *volinfo);
 
 int32_t
-glusterd_check_and_set_config_limit (glusterd_conf_t *priv);
+glusterd_check_and_set_config_limit(glusterd_conf_t *priv);
 
 int32_t
-glusterd_is_snap_soft_limit_reached (glusterd_volinfo_t *volinfo,
-                                     dict_t *dict);
+glusterd_is_snap_soft_limit_reached(glusterd_volinfo_t *volinfo, dict_t *dict);
 
 void
-gd_get_snap_conf_values_if_present (dict_t *opts, uint64_t *sys_hard_limit,
-                                    uint64_t *sys_soft_limit);
+gd_get_snap_conf_values_if_present(dict_t *opts, uint64_t *sys_hard_limit,
+                                   uint64_t *sys_soft_limit);
+int
+glusterd_get_snap_status_str(glusterd_snap_t *snapinfo, char *snap_status_str);
 
 #endif
-
diff --git a/xlators/mgmt/glusterd/src/glusterd-snapshot.c b/xlators/mgmt/glusterd/src/glusterd-snapshot.c
index cdd19cd354d..aeaa8d15214 100644
--- a/xlators/mgmt/glusterd/src/glusterd-snapshot.c
+++ b/xlators/mgmt/glusterd/src/glusterd-snapshot.c
@@ -7,11 +7,6 @@
    later), or the GNU General Public License, version 2 (GPLv2), in all
    cases as published by the Free Software Foundation.
 */
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
-
 #include <inttypes.h>
 #include <sys/types.h>
 #include <unistd.h>
@@ -19,6 +14,8 @@
 #include <sys/statvfs.h>
 #include <sys/mount.h>
 #include <signal.h>
+#include "glusterd-messages.h"
+#include "glusterd-errno.h"
 
 #if defined(GF_LINUX_HOST_OS)
 #include <mntent.h>
@@ -38,19 +35,18 @@
 
 #include <regex.h>
 
-#include "globals.h"
-#include "compat.h"
+#include <glusterfs/compat.h>
 #include "protocol-common.h"
-#include "xlator.h"
-#include "logging.h"
-#include "timer.h"
+#include <glusterfs/xlator.h>
+#include <glusterfs/logging.h>
+#include <glusterfs/timer.h>
 #include "glusterd-mem-types.h"
 #include "glusterd.h"
 #include "glusterd-sm.h"
 #include "glusterd-op-sm.h"
 #include "glusterd-utils.h"
 #include "glusterd-store.h"
-#include "run.h"
+#include <glusterfs/run.h>
 #include "glusterd-volgen.h"
 #include "glusterd-mgmt.h"
 #include "glusterd-syncop.h"
@@ -59,24 +55,49 @@
 
 #include "glusterfs3.h"
 
-#include "syscall.h"
+#include <glusterfs/syscall.h>
 #include "cli1-xdr.h"
 #include "xdr-generic.h"
 
-#include "lvm-defaults.h"
-
-char snap_mount_folder[PATH_MAX];
+#include <glusterfs/lvm-defaults.h>
+#include <glusterfs/events.h>
+
+#define GLUSTERD_GET_UUID_NOHYPHEN(ret_string, uuid)                           \
+    do {                                                                       \
+        char *snap_volname_ptr = ret_string;                                   \
+        char tmp_uuid[64];                                                     \
+        char *snap_volid_ptr = uuid_utoa_r(uuid, tmp_uuid);                    \
+        while (*snap_volid_ptr) {                                              \
+            if (*snap_volid_ptr == '-') {                                      \
+                snap_volid_ptr++;                                              \
+            } else {                                                           \
+                (*snap_volname_ptr++) = (*snap_volid_ptr++);                   \
+            }                                                                  \
+        }                                                                      \
+        *snap_volname_ptr = '\0';                                              \
+    } while (0)
+
+char snap_mount_dir[VALID_GLUSTERD_PATHMAX];
 struct snap_create_args_ {
-        xlator_t             *this;
-        dict_t               *dict;
-        dict_t               *rsp_dict;
-        glusterd_volinfo_t   *snap_vol;
-        glusterd_brickinfo_t *brickinfo;
-        struct syncargs      *args;
-        int32_t               volcount;
-        int32_t               brickcount;
-        int32_t               brickorder;
+    xlator_t *this;
+    dict_t *dict;
+    dict_t *rsp_dict;
+    glusterd_volinfo_t *snap_vol;
+    glusterd_brickinfo_t *brickinfo;
+    struct syncargs *args;
+    int32_t volcount;
+    int32_t brickcount;
+    int32_t brickorder;
+};
+
+/* This structure is used to store unsupported options and their values
+ * for snapshotted volume.
+ */
+struct gd_snap_unsupported_opt_t {
+    char *key;
+    char *value;
 };
+
 typedef struct snap_create_args_ snap_create_args_t;
 
 /* This function is called to get the device path of the snap lvm. Usually
@@ -84,477 +105,503 @@ typedef struct snap_create_args_ snap_create_args_t;
    then the snap device will be /dev/<group-name>/<snapname>.
    This function takes care of building the path for the snap device.
 */
+
 char *
-glusterd_build_snap_device_path (char *device, char *snapname,
-                                 int32_t brickcount)
+glusterd_build_snap_device_path(char *device, char *snapname,
+                                int32_t brickcount)
 {
-        char        snap[PATH_MAX]      = "";
-        char        msg[1024]           = "";
-        char        volgroup[PATH_MAX]  = "";
-        char       *snap_device         = NULL;
-        xlator_t   *this                = NULL;
-        runner_t    runner              = {0,};
-        char       *ptr                 = NULL;
-        int         ret                 = -1;
-
-        this = THIS;
-        GF_ASSERT (this);
-        if (!device) {
-                gf_log (this->name, GF_LOG_ERROR, "device is NULL");
-                goto out;
-        }
-        if (!snapname) {
-                gf_log (this->name, GF_LOG_ERROR, "snapname is NULL");
-                goto out;
-        }
-
-        runinit (&runner);
-        runner_add_args (&runner, "/sbin/lvs", "--noheadings", "-o", "vg_name",
-                         device, NULL);
-        runner_redir (&runner, STDOUT_FILENO, RUN_PIPE);
-        snprintf (msg, sizeof (msg), "Get volume group for device %s", device);
-        runner_log (&runner, this->name, GF_LOG_DEBUG, msg);
-        ret = runner_start (&runner);
-        if (ret == -1) {
-                gf_log (this->name, GF_LOG_ERROR, "Failed to get volume group "
-                        "for device %s", device);
-                runner_end (&runner);
-                goto out;
-        }
-        ptr = fgets(volgroup, sizeof(volgroup),
-                    runner_chio (&runner, STDOUT_FILENO));
-        if (!ptr || !strlen(volgroup)) {
-                gf_log (this->name, GF_LOG_ERROR, "Failed to get volume group "
-                        "for snap %s", snapname);
-                runner_end (&runner);
-                ret = -1;
-                goto out;
-        }
-        runner_end (&runner);
-
-        snprintf (snap, sizeof(snap), "/dev/%s/%s_%d", gf_trim(volgroup),
-                  snapname, brickcount);
-        snap_device = gf_strdup (snap);
-        if (!snap_device) {
-                gf_log (this->name, GF_LOG_WARNING, "Cannot copy the "
-                        "snapshot device name for snapname: %s", snapname);
-        }
+    char snap[PATH_MAX] = "";
+    char msg[1024] = "";
+    char volgroup[PATH_MAX] = "";
+    char *snap_device = NULL;
+    xlator_t *this = NULL;
+    runner_t runner = {
+        0,
+    };
+    char *ptr = NULL;
+    int ret = -1;
+
+    this = THIS;
+    GF_ASSERT(this);
+    if (!device) {
+        gf_msg(this->name, GF_LOG_ERROR, EINVAL, GD_MSG_INVALID_ENTRY,
+               "device is NULL");
+        goto out;
+    }
+    if (!snapname) {
+        gf_msg(this->name, GF_LOG_ERROR, EINVAL, GD_MSG_INVALID_ENTRY,
+               "snapname is NULL");
+        goto out;
+    }
+
+    runinit(&runner);
+    runner_add_args(&runner, "lvs", "--noheadings", "-o", "vg_name", device,
+                    NULL);
+    runner_redir(&runner, STDOUT_FILENO, RUN_PIPE);
+    snprintf(msg, sizeof(msg), "Get volume group for device %s", device);
+    runner_log(&runner, this->name, GF_LOG_DEBUG, msg);
+    ret = runner_start(&runner);
+    if (ret == -1) {
+        gf_msg(this->name, GF_LOG_ERROR, errno, GD_MSG_VG_GET_FAIL,
+               "Failed to get volume group "
+               "for device %s",
+               device);
+        runner_end(&runner);
+        goto out;
+    }
+    ptr = fgets(volgroup, sizeof(volgroup),
+                runner_chio(&runner, STDOUT_FILENO));
+    if (!ptr || !strlen(volgroup)) {
+        gf_msg(this->name, GF_LOG_ERROR, errno, GD_MSG_VG_GET_FAIL,
+               "Failed to get volume group "
+               "for snap %s",
+               snapname);
+        runner_end(&runner);
+        ret = -1;
+        goto out;
+    }
+    runner_end(&runner);
+
+    snprintf(snap, sizeof(snap), "/dev/%s/%s_%d", gf_trim(volgroup), snapname,
+             brickcount);
+    snap_device = gf_strdup(snap);
+    if (!snap_device) {
+        gf_msg(this->name, GF_LOG_WARNING, ENOMEM, GD_MSG_NO_MEMORY,
+               "Cannot copy the snapshot device name for snapname: %s",
+               snapname);
+    }
 
 out:
-        return snap_device;
+    return snap_device;
 }
 
 /* Look for disconnected peers, for missed snap creates or deletes */
 static int32_t
-glusterd_find_missed_snap (dict_t *rsp_dict, glusterd_volinfo_t *vol,
-                           struct cds_list_head *peers, int32_t op)
+glusterd_find_missed_snap(dict_t *rsp_dict, glusterd_volinfo_t *vol,
+                          struct cds_list_head *peers, int32_t op)
 {
-        int32_t                   brick_count          = -1;
-        int32_t                   ret                  = -1;
-        xlator_t                 *this                 = NULL;
-        glusterd_peerinfo_t      *peerinfo             = NULL;
-        glusterd_brickinfo_t     *brickinfo            = NULL;
-
-        this = THIS;
-        GF_ASSERT (this);
-        GF_ASSERT (rsp_dict);
-        GF_ASSERT (peers);
-        GF_ASSERT (vol);
-
-        brick_count = 0;
-        cds_list_for_each_entry (brickinfo, &vol->bricks, brick_list) {
-                if (!gf_uuid_compare (brickinfo->uuid, MY_UUID)) {
-                        /* If the brick belongs to the same node */
-                        brick_count++;
-                        continue;
-                }
-
-                rcu_read_lock ();
-                cds_list_for_each_entry_rcu (peerinfo, peers, uuid_list) {
-                        if (gf_uuid_compare (peerinfo->uuid, brickinfo->uuid)) {
-                                /* If the brick doesnt belong to this peer */
-                                continue;
-                        }
-
-                        /* Found peer who owns the brick,    *
-                         * if peer is not connected or not   *
-                         * friend add it to missed snap list */
-                        if (!(peerinfo->connected) ||
-                           (peerinfo->state.state !=
-                             GD_FRIEND_STATE_BEFRIENDED)) {
-                                ret = glusterd_add_missed_snaps_to_dict
-                                                   (rsp_dict,
-                                                    vol, brickinfo,
-                                                    brick_count + 1,
-                                                    op);
-                                if (ret) {
-                                        gf_log (this->name, GF_LOG_ERROR,
-                                                "Failed to add missed snapshot "
-                                                "info for %s:%s in the "
-                                                "rsp_dict", brickinfo->hostname,
-                                                brickinfo->path);
-                                        rcu_read_unlock ();
-                                        goto out;
-                                }
-                        }
-                }
-                rcu_read_unlock ();
-                brick_count++;
-        }
-
-        ret = 0;
+    int32_t brick_count = -1;
+    int32_t ret = -1;
+    xlator_t *this = NULL;
+    glusterd_peerinfo_t *peerinfo = NULL;
+    glusterd_brickinfo_t *brickinfo = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+    GF_ASSERT(rsp_dict);
+    GF_ASSERT(peers);
+    GF_ASSERT(vol);
+
+    brick_count = 0;
+    cds_list_for_each_entry(brickinfo, &vol->bricks, brick_list)
+    {
+        if (!gf_uuid_compare(brickinfo->uuid, MY_UUID)) {
+            /* If the brick belongs to the same node */
+            brick_count++;
+            continue;
+        }
+
+        RCU_READ_LOCK;
+        cds_list_for_each_entry_rcu(peerinfo, peers, uuid_list)
+        {
+            if (gf_uuid_compare(peerinfo->uuid, brickinfo->uuid)) {
+                /* If the brick doesn't belong to this peer */
+                continue;
+            }
+
+            /* Found peer who owns the brick,    *
+             * if peer is not connected or not   *
+             * friend add it to missed snap list */
+            if (!(peerinfo->connected) ||
+                (peerinfo->state.state != GD_FRIEND_STATE_BEFRIENDED)) {
+                ret = glusterd_add_missed_snaps_to_dict(
+                    rsp_dict, vol, brickinfo, brick_count + 1, op);
+                if (ret) {
+                    RCU_READ_UNLOCK;
+                    gf_msg(this->name, GF_LOG_ERROR, 0,
+                           GD_MSG_MISSED_SNAP_CREATE_FAIL,
+                           "Failed to add missed snapshot "
+                           "info for %s:%s in the "
+                           "rsp_dict",
+                           brickinfo->hostname, brickinfo->path);
+                    goto out;
+                }
+            }
+        }
+        RCU_READ_UNLOCK;
+        brick_count++;
+    }
+
+    ret = 0;
 out:
-        gf_log (this->name, GF_LOG_TRACE, "Returning %d", ret);
-        return ret;
+    gf_msg_trace(this->name, 0, "Returning %d", ret);
+    return ret;
 }
 
 int
-snap_max_limits_display_commit (dict_t *rsp_dict, char *volname,
-                                char *op_errstr, int len)
+snap_max_limits_display_commit(dict_t *rsp_dict, char *volname, char *op_errstr,
+                               int len)
 {
-        char          err_str[PATH_MAX] = "";
-        char          buf[PATH_MAX]     = "";
-        glusterd_conf_t    *conf        = NULL;
-        glusterd_volinfo_t *volinfo     = NULL;
-        int           ret               = -1;
-        uint64_t      active_hard_limit = 0;
-        uint64_t      snap_max_limit    = 0;
-        uint64_t      soft_limit_value  = -1;
-        uint64_t      count             = 0;
-        xlator_t     *this              = NULL;
-        uint64_t      opt_hard_max      = GLUSTERD_SNAPS_MAX_HARD_LIMIT;
-        uint64_t      opt_soft_max      = GLUSTERD_SNAPS_DEF_SOFT_LIMIT_PERCENT;
-        char         *auto_delete       = "disable";
-        char         *snap_activate     = "disable";
-
-        this = THIS;
-
-        GF_ASSERT (this);
-        GF_ASSERT (rsp_dict);
-        GF_ASSERT (op_errstr);
-
-        conf = this->private;
-
-        GF_ASSERT (conf);
-
-
-        /* config values snap-max-hard-limit and snap-max-soft-limit are
-         * optional and hence we are not erroring out if values are not
-         * present
-         */
-        gd_get_snap_conf_values_if_present (conf->opts, &opt_hard_max,
-                                            &opt_soft_max);
-
-        if (!volname) {
-                /* For system limit */
-                cds_list_for_each_entry (volinfo, &conf->volumes, vol_list) {
-                        if (volinfo->is_snap_volume == _gf_true)
-                                continue;
-
-                        snap_max_limit = volinfo->snap_max_hard_limit;
-                        if (snap_max_limit > opt_hard_max)
-                                active_hard_limit = opt_hard_max;
-                        else
-                                active_hard_limit = snap_max_limit;
-
-                        soft_limit_value = (opt_soft_max *
-                                            active_hard_limit) / 100;
-
-                        snprintf (buf, sizeof(buf), "volume%"PRId64"-volname",
-                                  count);
-                        ret = dict_set_str (rsp_dict, buf, volinfo->volname);
-                        if (ret) {
-                                snprintf (err_str, PATH_MAX,
-                                          "Failed to set %s", buf);
-                                goto out;
-                        }
-
-                        snprintf (buf, sizeof(buf),
-                                  "volume%"PRId64"-snap-max-hard-limit", count);
-                        ret = dict_set_uint64 (rsp_dict, buf, snap_max_limit);
-                        if (ret) {
-                                snprintf (err_str, PATH_MAX,
-                                          "Failed to set %s", buf);
-                                goto out;
-                        }
-
-                        snprintf (buf, sizeof(buf),
-                                  "volume%"PRId64"-active-hard-limit", count);
-                        ret = dict_set_uint64 (rsp_dict, buf,
-                                               active_hard_limit);
-                        if (ret) {
-                                snprintf (err_str, PATH_MAX,
-                                          "Failed to set %s", buf);
-                                goto out;
-                        }
-
-                        snprintf (buf, sizeof(buf),
-                                  "volume%"PRId64"-snap-max-soft-limit", count);
-                        ret = dict_set_uint64 (rsp_dict, buf, soft_limit_value);
-                        if (ret) {
-                                snprintf (err_str, PATH_MAX,
-                                          "Failed to set %s", buf);
-                                goto out;
-                        }
-                        count++;
-                }
-
-                ret = dict_set_uint64 (rsp_dict, "voldisplaycount", count);
-                if (ret) {
-                        snprintf (err_str, PATH_MAX,
-                                  "Failed to set voldisplaycount");
-                        goto out;
-                }
-        } else {
-                /*  For one volume */
-                ret = glusterd_volinfo_find (volname, &volinfo);
-                if (ret) {
-                        snprintf (err_str, PATH_MAX, "Volume (%s) does not "
-                                  "exist", volname);
-                        goto out;
-                }
-
-                snap_max_limit = volinfo->snap_max_hard_limit;
-                if (snap_max_limit > opt_hard_max)
-                        active_hard_limit = opt_hard_max;
-                else
-                        active_hard_limit = snap_max_limit;
+    char err_str[PATH_MAX] = "";
+    char key[64] = "";
+    int keylen;
+    glusterd_conf_t *conf = NULL;
+    glusterd_volinfo_t *volinfo = NULL;
+    int ret = -1;
+    uint64_t active_hard_limit = 0;
+    uint64_t snap_max_limit = 0;
+    uint64_t soft_limit_value = -1;
+    uint64_t count = 0;
+    xlator_t *this = NULL;
+    uint64_t opt_hard_max = GLUSTERD_SNAPS_MAX_HARD_LIMIT;
+    uint64_t opt_soft_max = GLUSTERD_SNAPS_DEF_SOFT_LIMIT_PERCENT;
+    char *auto_delete = "disable";
+    char *snap_activate = "disable";
+
+    this = THIS;
+
+    GF_ASSERT(this);
+    GF_ASSERT(rsp_dict);
+    GF_ASSERT(op_errstr);
+
+    conf = this->private;
+
+    GF_ASSERT(conf);
+
+    /* config values snap-max-hard-limit and snap-max-soft-limit are
+     * optional and hence we are not erroring out if values are not
+     * present
+     */
+    gd_get_snap_conf_values_if_present(conf->opts, &opt_hard_max,
+                                       &opt_soft_max);
+
+    if (!volname) {
+        /* For system limit */
+        cds_list_for_each_entry(volinfo, &conf->volumes, vol_list)
+        {
+            if (volinfo->is_snap_volume == _gf_true)
+                continue;
 
-                soft_limit_value = (opt_soft_max *
-                                    active_hard_limit) / 100;
+            snap_max_limit = volinfo->snap_max_hard_limit;
+            if (snap_max_limit > opt_hard_max)
+                active_hard_limit = opt_hard_max;
+            else
+                active_hard_limit = snap_max_limit;
 
-                snprintf (buf, sizeof(buf), "volume%"PRId64"-volname", count);
-                ret = dict_set_str (rsp_dict, buf, volinfo->volname);
-                if (ret) {
-                        snprintf (err_str, PATH_MAX,
-                                  "Failed to set %s", buf);
-                        goto out;
-                }
+            soft_limit_value = (opt_soft_max * active_hard_limit) / 100;
 
-                snprintf (buf, sizeof(buf),
-                          "volume%"PRId64"-snap-max-hard-limit", count);
-                ret = dict_set_uint64 (rsp_dict, buf, snap_max_limit);
-                if (ret) {
-                        snprintf (err_str, PATH_MAX,
-                                  "Failed to set %s", buf);
-                        goto out;
+            keylen = snprintf(key, sizeof(key), "volume%" PRId64 "-volname",
+                              count);
+            ret = dict_set_strn(rsp_dict, key, keylen, volinfo->volname);
+            if (ret) {
+                len = snprintf(err_str, PATH_MAX, "Failed to set %s", key);
+                if (len < 0) {
+                    strcpy(err_str, "<error>");
                 }
+                goto out;
+            }
 
-                snprintf (buf, sizeof(buf),
-                          "volume%"PRId64"-active-hard-limit", count);
-                ret = dict_set_uint64 (rsp_dict, buf, active_hard_limit);
-                if (ret) {
-                        snprintf (err_str, PATH_MAX,
-                                  "Failed to set %s", buf);
-                        goto out;
+            snprintf(key, sizeof(key), "volume%" PRId64 "-snap-max-hard-limit",
+                     count);
+            ret = dict_set_uint64(rsp_dict, key, snap_max_limit);
+            if (ret) {
+                len = snprintf(err_str, PATH_MAX, "Failed to set %s", key);
+                if (len < 0) {
+                    strcpy(err_str, "<error>");
                 }
+                goto out;
+            }
 
-                snprintf (buf, sizeof(buf),
-                          "volume%"PRId64"-snap-max-soft-limit", count);
-                ret = dict_set_uint64 (rsp_dict, buf, soft_limit_value);
-                if (ret) {
-                        snprintf (err_str, PATH_MAX,
-                                  "Failed to set %s", buf);
-                        goto out;
+            snprintf(key, sizeof(key), "volume%" PRId64 "-active-hard-limit",
+                     count);
+            ret = dict_set_uint64(rsp_dict, key, active_hard_limit);
+            if (ret) {
+                len = snprintf(err_str, PATH_MAX, "Failed to set %s", key);
+                if (len < 0) {
+                    strcpy(err_str, "<error>");
                 }
+                goto out;
+            }
 
-                count++;
-
-                ret = dict_set_uint64 (rsp_dict, "voldisplaycount", count);
-                if (ret) {
-                        snprintf (err_str, PATH_MAX,
-                                  "Failed to set voldisplaycount");
-                        goto out;
+            snprintf(key, sizeof(key), "volume%" PRId64 "-snap-max-soft-limit",
+                     count);
+            ret = dict_set_uint64(rsp_dict, key, soft_limit_value);
+            if (ret) {
+                len = snprintf(err_str, PATH_MAX, "Failed to set %s", key);
+                if (len < 0) {
+                    strcpy(err_str, "<error>");
                 }
-
-        }
-
-        ret = dict_set_uint64 (rsp_dict,
-                               GLUSTERD_STORE_KEY_SNAP_MAX_HARD_LIMIT,
-                               opt_hard_max);
-        if (ret) {
-                snprintf (err_str, PATH_MAX,
-                          "Failed to set %s in response dictionary",
-                          GLUSTERD_STORE_KEY_SNAP_MAX_HARD_LIMIT);
                 goto out;
+            }
+            count++;
         }
 
-        ret = dict_set_uint64 (rsp_dict,
-                               GLUSTERD_STORE_KEY_SNAP_MAX_SOFT_LIMIT,
-                               opt_soft_max);
+        ret = dict_set_uint64(rsp_dict, "voldisplaycount", count);
         if (ret) {
-                snprintf (err_str, PATH_MAX,
-                         "Failed to set %s in response dictionary",
-                         GLUSTERD_STORE_KEY_SNAP_MAX_SOFT_LIMIT);
-                goto out;
+            snprintf(err_str, PATH_MAX, "Failed to set voldisplaycount");
+            goto out;
         }
-
-        /* "auto-delete" might not be set by user explicitly,
-         * in that case it's better to consider the default value.
-         * Hence not erroring out if Key is not found.
-         */
-        ret = dict_get_str (conf->opts, GLUSTERD_STORE_KEY_SNAP_AUTO_DELETE,
-                            &auto_delete);
-
-        ret = dict_set_dynstr_with_alloc (rsp_dict,
-                                     GLUSTERD_STORE_KEY_SNAP_AUTO_DELETE,
-                                     auto_delete);
+    } else {
+        /*  For one volume */
+        ret = glusterd_volinfo_find(volname, &volinfo);
         if (ret) {
-                snprintf (err_str, PATH_MAX,
-                          "Failed to set %s in response dictionary",
-                          GLUSTERD_STORE_KEY_SNAP_AUTO_DELETE);
-                goto out;
+            snprintf(err_str, PATH_MAX,
+                     "Volume (%s) does not "
+                     "exist",
+                     volname);
+            goto out;
         }
 
-        /* "snap-activate-on-create" might not be set by user explicitly,
-         * in that case it's better to consider the default value.
-         * Hence not erroring out if Key is not found.
-         */
-        ret = dict_get_str (conf->opts, GLUSTERD_STORE_KEY_SNAP_ACTIVATE,
-                            &snap_activate);
-
-        ret = dict_set_dynstr_with_alloc (rsp_dict,
-                                     GLUSTERD_STORE_KEY_SNAP_ACTIVATE,
+        snap_max_limit = volinfo->snap_max_hard_limit;
+        if (snap_max_limit > opt_hard_max)
+            active_hard_limit = opt_hard_max;
+        else
+            active_hard_limit = snap_max_limit;
+
+        soft_limit_value = (opt_soft_max * active_hard_limit) / 100;
+
+        keylen = snprintf(key, sizeof(key), "volume%" PRId64 "-volname", count);
+        ret = dict_set_strn(rsp_dict, key, keylen, volinfo->volname);
+        if (ret) {
+            len = snprintf(err_str, PATH_MAX, "Failed to set %s", key);
+            if (len < 0) {
+                strcpy(err_str, "<error>");
+            }
+            goto out;
+        }
+
+        snprintf(key, sizeof(key), "volume%" PRId64 "-snap-max-hard-limit",
+                 count);
+        ret = dict_set_uint64(rsp_dict, key, snap_max_limit);
+        if (ret) {
+            len = snprintf(err_str, PATH_MAX, "Failed to set %s", key);
+            if (len < 0) {
+                strcpy(err_str, "<error>");
+            }
+            goto out;
+        }
+
+        snprintf(key, sizeof(key), "volume%" PRId64 "-active-hard-limit",
+                 count);
+        ret = dict_set_uint64(rsp_dict, key, active_hard_limit);
+        if (ret) {
+            len = snprintf(err_str, PATH_MAX, "Failed to set %s", key);
+            if (len < 0) {
+                strcpy(err_str, "<error>");
+            }
+            goto out;
+        }
+
+        snprintf(key, sizeof(key), "volume%" PRId64 "-snap-max-soft-limit",
+                 count);
+        ret = dict_set_uint64(rsp_dict, key, soft_limit_value);
+        if (ret) {
+            len = snprintf(err_str, PATH_MAX, "Failed to set %s", key);
+            if (len < 0) {
+                strcpy(err_str, "<error>");
+            }
+            goto out;
+        }
+
+        count++;
+
+        ret = dict_set_uint64(rsp_dict, "voldisplaycount", count);
+        if (ret) {
+            snprintf(err_str, PATH_MAX, "Failed to set voldisplaycount");
+            goto out;
+        }
+    }
+
+    ret = dict_set_uint64(rsp_dict, GLUSTERD_STORE_KEY_SNAP_MAX_HARD_LIMIT,
+                          opt_hard_max);
+    if (ret) {
+        snprintf(err_str, PATH_MAX, "Failed to set %s in response dictionary",
+                 GLUSTERD_STORE_KEY_SNAP_MAX_HARD_LIMIT);
+        goto out;
+    }
+
+    ret = dict_set_uint64(rsp_dict, GLUSTERD_STORE_KEY_SNAP_MAX_SOFT_LIMIT,
+                          opt_soft_max);
+    if (ret) {
+        snprintf(err_str, PATH_MAX, "Failed to set %s in response dictionary",
+                 GLUSTERD_STORE_KEY_SNAP_MAX_SOFT_LIMIT);
+        goto out;
+    }
+
+    /* "auto-delete" might not be set by user explicitly,
+     * in that case it's better to consider the default value.
+     * Hence not erroring out if Key is not found.
+     */
+    ret = dict_get_strn(conf->opts, GLUSTERD_STORE_KEY_SNAP_AUTO_DELETE,
+                        SLEN(GLUSTERD_STORE_KEY_SNAP_AUTO_DELETE),
+                        &auto_delete);
+
+    ret = dict_set_dynstr_with_alloc(
+        rsp_dict, GLUSTERD_STORE_KEY_SNAP_AUTO_DELETE, auto_delete);
+    if (ret) {
+        snprintf(err_str, PATH_MAX, "Failed to set %s in response dictionary",
+                 GLUSTERD_STORE_KEY_SNAP_AUTO_DELETE);
+        goto out;
+    }
+
+    /* "snap-activate-on-create" might not be set by user explicitly,
+     * in that case it's better to consider the default value.
+     * Hence not erroring out if Key is not found.
+     */
+    ret = dict_get_strn(conf->opts, GLUSTERD_STORE_KEY_SNAP_ACTIVATE,
+                        SLEN(GLUSTERD_STORE_KEY_SNAP_ACTIVATE), &snap_activate);
+
+    ret = dict_set_dynstr_with_alloc(rsp_dict, GLUSTERD_STORE_KEY_SNAP_ACTIVATE,
                                      snap_activate);
-        if (ret) {
-                snprintf (err_str, PATH_MAX,
-                          "Failed to set %s in response dictionary",
-                          GLUSTERD_STORE_KEY_SNAP_ACTIVATE);
-                goto out;
-        }
+    if (ret) {
+        snprintf(err_str, PATH_MAX, "Failed to set %s in response dictionary",
+                 GLUSTERD_STORE_KEY_SNAP_ACTIVATE);
+        goto out;
+    }
 
-        ret = 0;
+    ret = 0;
 out:
-        if (ret) {
-                strncpy (op_errstr, err_str, len);
-                gf_log (this->name, GF_LOG_ERROR, "%s", err_str);
-        }
-        return ret;
+    if (ret) {
+        strncpy(op_errstr, err_str, len);
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED, "%s",
+               err_str);
+    }
+    return ret;
 }
 
-
 /* Third argument of scandir(used in glusterd_copy_geo_rep_session_files)
- * is filter function. As we dont want "." and ".." files present in the
+ * is filter function. As we don't want "." and ".." files present in the
  * directory, we are excliding these 2 files.
  * "file_select" function here does the job of filtering.
  */
 int
-file_select (const struct dirent *entry)
+file_select(const struct dirent *entry)
 {
-        if (entry == NULL)
-                return (FALSE);
+    if (entry == NULL)
+        return (FALSE);
 
-        if ((strcmp(entry->d_name, ".") == 0) ||
-            (strcmp(entry->d_name, "..") == 0))
-                return (FALSE);
-        else
-                return (TRUE);
+    if ((strcmp(entry->d_name, ".") == 0) || (strcmp(entry->d_name, "..") == 0))
+        return (FALSE);
+    else
+        return (TRUE);
 }
 
 int32_t
-glusterd_copy_geo_rep_session_files (char *session,
-                                     glusterd_volinfo_t *snap_vol)
+glusterd_copy_geo_rep_session_files(char *session, glusterd_volinfo_t *snap_vol)
 {
-        int32_t         ret                             = -1;
-        char            snap_session_dir[PATH_MAX]      = "";
-        char            georep_session_dir[PATH_MAX]    = "";
-        regex_t         *reg_exp                        = NULL;
-        int             file_count                      = -1;
-        struct  dirent  **files                         = {0,};
-        xlator_t        *this                           = NULL;
-        int             i                               = 0;
-        char            src_path[PATH_MAX]              = "";
-        char            dest_path[PATH_MAX]             = "";
-        glusterd_conf_t *priv                           = NULL;
-
-        this = THIS;
-        GF_ASSERT (this);
-        priv = this->private;
-        GF_ASSERT (priv);
-
-        GF_ASSERT (session);
-        GF_ASSERT (snap_vol);
-
-        ret = snprintf (georep_session_dir, sizeof (georep_session_dir),
-                        "%s/%s/%s", priv->workdir, GEOREP,
-                        session);
-        if (ret < 0) { /* Negative value is an error */
-                goto out;
-        }
-
-        ret = snprintf (snap_session_dir, sizeof (snap_session_dir),
-                        "%s/%s/%s/%s/%s", priv->workdir,
-                        GLUSTERD_VOL_SNAP_DIR_PREFIX,
-                        snap_vol->snapshot->snapname, GEOREP, session);
-        if (ret < 0) { /* Negative value is an error */
-                goto out;
-        }
-
-        ret = mkdir_p (snap_session_dir, 0777, _gf_true);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "Creating directory %s failed", snap_session_dir);
-                goto out;
+    int32_t ret = -1;
+    char snap_session_dir[PATH_MAX] = "";
+    char georep_session_dir[PATH_MAX] = "";
+    regex_t *reg_exp = NULL;
+    int file_count = -1;
+    struct dirent **files = {
+        0,
+    };
+    xlator_t *this = NULL;
+    int i = 0;
+    char src_path[PATH_MAX] = "";
+    char dest_path[PATH_MAX] = "";
+    glusterd_conf_t *priv = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+    priv = this->private;
+    GF_ASSERT(priv);
+
+    GF_ASSERT(session);
+    GF_ASSERT(snap_vol);
+
+    ret = snprintf(georep_session_dir, sizeof(georep_session_dir), "%s/%s/%s",
+                   priv->workdir, GEOREP, session);
+    if (ret < 0) { /* Negative value is an error */
+        gf_smsg(this->name, GF_LOG_ERROR, 0, GD_MSG_COPY_FAIL, NULL);
+        goto out;
+    }
+
+    ret = snprintf(snap_session_dir, sizeof(snap_session_dir), "%s/%s/%s/%s/%s",
+                   priv->workdir, GLUSTERD_VOL_SNAP_DIR_PREFIX,
+                   snap_vol->snapshot->snapname, GEOREP, session);
+    if (ret < 0) { /* Negative value is an error */
+        gf_smsg(this->name, GF_LOG_ERROR, 0, GD_MSG_COPY_FAIL, NULL);
+        goto out;
+    }
+
+    ret = mkdir_p(snap_session_dir, 0755, _gf_true);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, errno, GD_MSG_DIR_OP_FAILED,
+               "Creating directory %s failed", snap_session_dir);
+        goto out;
+    }
+
+    /* TODO : good to have - Allocate in stack instead of heap */
+    reg_exp = GF_CALLOC(1, sizeof(regex_t), gf_common_mt_regex_t);
+    if (!reg_exp) {
+        ret = -1;
+        gf_msg(this->name, GF_LOG_ERROR, ENOMEM, GD_MSG_NO_MEMORY,
+               "Failed to allocate memory for regular expression");
+        goto out;
+    }
+
+    ret = regcomp(reg_exp, "(.*status$)|(.*conf$)\0", REG_EXTENDED);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_REG_COMPILE_FAILED,
+               "Failed to compile the regular expression");
+        goto out;
+    }
+
+    /* If there are no files in a particular session then fail it*/
+    file_count = scandir(georep_session_dir, &files, file_select, alphasort);
+    if (file_count <= 0) {
+        ret = -1;
+        gf_msg(this->name, GF_LOG_ERROR, ENOENT, GD_MSG_FILE_OP_FAILED,
+               "Session files not present "
+               "in %s",
+               georep_session_dir);
+        goto out;
+    }
+
+    /* Now compare the file name with regular expression to see if
+     * there is a match
+     */
+    for (i = 0; i < file_count; i++) {
+        if (regexec(reg_exp, files[i]->d_name, 0, NULL, 0))
+            continue;
+
+        ret = snprintf(src_path, sizeof(src_path), "%s/%s", georep_session_dir,
+                       files[i]->d_name);
+        if (ret < 0) {
+            gf_smsg(this->name, GF_LOG_ERROR, 0, GD_MSG_COPY_FAIL, NULL);
+            goto out;
         }
 
-        /* TODO : good to have - Allocate in stack instead of heap */
-        reg_exp = GF_CALLOC (1, sizeof (regex_t), gf_common_mt_regex_t);
-        if (!reg_exp) {
-                ret = -1;
-                gf_log (this->name, GF_LOG_ERROR, "Failed to allocate "
-                        "memory for regular expression");
-                goto out;
+        ret = snprintf(dest_path, sizeof(dest_path), "%s/%s", snap_session_dir,
+                       files[i]->d_name);
+        if (ret < 0) {
+            gf_smsg(this->name, GF_LOG_ERROR, 0, GD_MSG_COPY_FAIL, NULL);
+            goto out;
         }
 
-        ret = regcomp (reg_exp, "(.*status$)|(.*conf$)\0", REG_EXTENDED);
+        ret = glusterd_copy_file(src_path, dest_path);
         if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "Failed to "
-                        "compile the regular expression");
-                goto out;
+            gf_msg(this->name, GF_LOG_ERROR, ENOMEM, GD_MSG_NO_MEMORY,
+                   "Could not copy file %s of session %s", files[i]->d_name,
+                   session);
+            goto out;
         }
-
-        /* If there are no files in a particular session then fail it*/
-        file_count = scandir (georep_session_dir, &files, file_select,
-                              alphasort);
-        if (file_count <= 0) {
-                ret = -1;
-                gf_log (this->name, GF_LOG_ERROR, "Session files not present "
-                        "in %s", georep_session_dir);
-                goto out;
+    }
+out:
+    /* files are malloc'd by scandir, free them */
+    if (file_count > 0) {
+        while (file_count--) {
+            free(files[file_count]);
         }
+        free(files);
+    }
 
-        /* Now compare the file name with regular expression to see if
-         * there is a match
-         */
-        for (i = 0 ; i < file_count; i++) {
-                if (regexec (reg_exp, files[i]->d_name, 0, NULL, 0))
-                        continue;
-
-                ret = snprintf (src_path, sizeof (src_path), "%s/%s",
-                                georep_session_dir, files[i]->d_name);
-                if (ret < 0) {
-                        goto out;
-                }
+    if (reg_exp)
+        GF_FREE(reg_exp);
 
-                ret = snprintf (dest_path , sizeof (dest_path), "%s/%s",
-                                snap_session_dir, files[i]->d_name);
-                if (ret < 0) {
-                        goto out;
-                }
-
-                ret = glusterd_copy_file (src_path, dest_path);
-                if (ret) {
-                        gf_log (this->name, GF_LOG_ERROR, "Could not "
-                                "copy file %s of session %s",
-                                files[i]->d_name, session);
-                        goto out;
-                }
-        }
-out:
-        if (reg_exp)
-                GF_FREE (reg_exp);
-
-        return ret;
+    return ret;
 }
 
 /* This function will take backup of the volume store
@@ -566,155 +613,168 @@ out:
  * @return 0 on success and -1 on failure
  */
 int
-glusterd_snapshot_backup_vol (glusterd_volinfo_t *volinfo)
+glusterd_snapshot_backup_vol(glusterd_volinfo_t *volinfo)
 {
-        char             pathname[PATH_MAX]    = {0,};
-        int              ret                   = -1;
-        int              op_ret                = 0;
-        char             delete_path[PATH_MAX] = {0,};
-        char             trashdir[PATH_MAX]    = {0,};
-        glusterd_conf_t *priv                  = NULL;
-        xlator_t        *this                  = NULL;
-
-        this = THIS;
-        GF_ASSERT (this);
-        priv = this->private;
-        GF_ASSERT (priv);
-        GF_ASSERT (volinfo);
-
-        GLUSTERD_GET_VOLUME_DIR (pathname, volinfo, priv);
-
-        snprintf (delete_path, sizeof (delete_path),
-                  "%s/"GLUSTERD_TRASH"/vols-%s.deleted", priv->workdir,
-                  volinfo->volname);
-
-        snprintf (trashdir, sizeof (trashdir), "%s/"GLUSTERD_TRASH,
-                  priv->workdir);
-
-        /* Create trash folder if it is not there */
-        ret = mkdir (trashdir, 0777);
-        if (ret && errno != EEXIST) {
-                gf_log (this->name, GF_LOG_ERROR, "Failed to create trash "
-                        "directory, reason : %s", strerror (errno));
-                ret = -1;
-                goto out;
-        }
-
-        /* Move the origin volume volder to the backup location */
-        ret = rename (pathname, delete_path);
+    char pathname[PATH_MAX] = "";
+    int ret = -1;
+    int op_ret = 0;
+    char delete_path[PATH_MAX] = "";
+    char trashdir[PATH_MAX] = "";
+    glusterd_conf_t *priv = NULL;
+    xlator_t *this = NULL;
+    int32_t len = 0;
+
+    this = THIS;
+    GF_ASSERT(this);
+    priv = this->private;
+    GF_ASSERT(priv);
+    GF_ASSERT(volinfo);
+
+    GLUSTERD_GET_VOLUME_DIR(pathname, volinfo, priv);
+
+    len = snprintf(delete_path, sizeof(delete_path),
+                   "%s/" GLUSTERD_TRASH "/vols-%s.deleted", priv->workdir,
+                   volinfo->volname);
+    if ((len < 0) || (len >= sizeof(delete_path))) {
+        gf_smsg(this->name, GF_LOG_ERROR, 0, GD_MSG_COPY_FAIL, NULL);
+        goto out;
+    }
+
+    len = snprintf(trashdir, sizeof(trashdir), "%s/" GLUSTERD_TRASH,
+                   priv->workdir);
+    if ((len < 0) || (len >= sizeof(trashdir))) {
+        gf_smsg(this->name, GF_LOG_ERROR, 0, GD_MSG_COPY_FAIL, NULL);
+        goto out;
+    }
+
+    /* Create trash folder if it is not there */
+    ret = sys_mkdir(trashdir, 0755);
+    if (ret && errno != EEXIST) {
+        gf_msg(this->name, GF_LOG_ERROR, errno, GD_MSG_DIR_OP_FAILED,
+               "Failed to create trash directory, reason : %s",
+               strerror(errno));
+        ret = -1;
+        goto out;
+    }
+
+    /* Move the origin volume volder to the backup location */
+    ret = sys_rename(pathname, delete_path);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, errno, GD_MSG_FILE_OP_FAILED,
+               "Failed to rename snap "
+               "directory %s to %s",
+               pathname, delete_path);
+        goto out;
+    }
+
+    /* Re-create an empty origin volume folder so that restore can
+     * happen. */
+    ret = sys_mkdir(pathname, 0755);
+    if (ret && errno != EEXIST) {
+        gf_msg(this->name, GF_LOG_ERROR, errno, GD_MSG_DIR_OP_FAILED,
+               "Failed to create origin "
+               "volume directory (%s), reason : %s",
+               pathname, strerror(errno));
+        ret = -1;
+        goto out;
+    }
+
+    ret = 0;
+out:
+    /* Save the actual return value */
+    op_ret = ret;
+    if (ret) {
+        /* Revert the changes in case of failure */
+        ret = sys_rmdir(pathname);
         if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "Failed to rename snap "
-                        "directory %s to %s", pathname, delete_path);
-                goto out;
+            gf_msg_debug(this->name, 0, "Failed to rmdir: %s,err: %s", pathname,
+                         strerror(errno));
         }
 
-        /* Re-create an empty origin volume folder so that restore can
-         * happen. */
-        ret = mkdir (pathname, 0777);
-        if (ret && errno != EEXIST) {
-                gf_log (this->name, GF_LOG_ERROR, "Failed to create origin "
-                        "volume directory (%s), reason : %s",
-                        pathname, strerror (errno));
-                ret = -1;
-                goto out;
+        ret = sys_rename(delete_path, pathname);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, errno, GD_MSG_FILE_OP_FAILED,
+                   "Failed to rename directory %s to %s", delete_path,
+                   pathname);
         }
 
-        ret = 0;
-out:
-        /* Save the actual return value */
-        op_ret = ret;
+        ret = sys_rmdir(trashdir);
         if (ret) {
-                /* Revert the changes in case of failure */
-                ret = rmdir (pathname);
-                if (ret) {
-                        gf_log (this->name, GF_LOG_DEBUG,
-                                "Failed to rmdir: %s,err: %s",
-                                pathname, strerror (errno));
-                }
-
-                ret = rename (delete_path, pathname);
-                if (ret) {
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "Failed to rename directory %s to %s",
-                                delete_path, pathname);
-                }
-
-                ret = rmdir (trashdir);
-                if (ret) {
-                        gf_log (this->name, GF_LOG_DEBUG,
-                                "Failed to rmdir: %s, Reason: %s",
-                                trashdir, strerror (errno));
-                }
+            gf_msg_debug(this->name, 0, "Failed to rmdir: %s, Reason: %s",
+                         trashdir, strerror(errno));
         }
+    }
 
-        gf_log (this->name, GF_LOG_TRACE, "Returning %d", op_ret);
+    gf_msg_trace(this->name, 0, "Returning %d", op_ret);
 
-        return op_ret;
+    return op_ret;
 }
 
-int32_t
-glusterd_copy_geo_rep_files (glusterd_volinfo_t *origin_vol,
-                             glusterd_volinfo_t *snap_vol, dict_t *rsp_dict)
+static int32_t
+glusterd_copy_geo_rep_files(glusterd_volinfo_t *origin_vol,
+                            glusterd_volinfo_t *snap_vol, dict_t *rsp_dict)
 {
-        int32_t         ret                     =       -1;
-        int             i                       =       0;
-        xlator_t        *this                   =       NULL;
-        char            key[PATH_MAX]           =       "";
-        char            session[PATH_MAX]       =       "";
-        char            slave[PATH_MAX]         =       "";
-        char            snapgeo_dir[PATH_MAX]   =       "";
-        glusterd_conf_t *priv                   =       NULL;
-
-        this = THIS;
-        GF_ASSERT (this);
-        priv = this->private;
-        GF_ASSERT (priv);
-
-        GF_ASSERT (origin_vol);
-        GF_ASSERT (snap_vol);
-        GF_ASSERT (rsp_dict);
-
-        /* This condition is not satisfied if the volume
-         * is slave volume.
-         */
-        if (!origin_vol->gsync_slaves) {
-                ret = 0;
-                goto out;
-        }
+    int32_t ret = -1;
+    int i = 0;
+    xlator_t *this = NULL;
+    char key[32] = "";
+    char session[PATH_MAX] = "";
+    char slave[PATH_MAX] = "";
+    char snapgeo_dir[PATH_MAX] = "";
+    glusterd_conf_t *priv = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+    priv = this->private;
+    GF_ASSERT(priv);
+
+    GF_ASSERT(origin_vol);
+    GF_ASSERT(snap_vol);
+    GF_ASSERT(rsp_dict);
+
+    /* This condition is not satisfied if the volume
+     * is slave volume.
+     */
+    if (!origin_vol->gsync_slaves) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_INVALID_SLAVE, NULL);
+        ret = 0;
+        goto out;
+    }
 
-        GLUSTERD_GET_SNAP_GEO_REP_DIR(snapgeo_dir, snap_vol->snapshot, priv);
+    GLUSTERD_GET_SNAP_GEO_REP_DIR(snapgeo_dir, snap_vol->snapshot, priv);
 
-        ret = mkdir (snapgeo_dir, 0777);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "Creating directory %s failed", snapgeo_dir);
-                goto out;
-        }
+    ret = sys_mkdir(snapgeo_dir, 0755);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, errno, GD_MSG_DIR_OP_FAILED,
+               "Creating directory %s failed", snapgeo_dir);
+        goto out;
+    }
 
-        for (i = 1 ; i <= origin_vol->gsync_slaves->count ; i++) {
-                ret = snprintf (key, sizeof (key), "slave%d", i);
-                if (ret < 0) /* Negative value is an error */
-                        goto out;
+    for (i = 1; i <= origin_vol->gsync_slaves->count; i++) {
+        ret = snprintf(key, sizeof(key), "slave%d", i);
+        if (ret < 0) /* Negative value is an error */
+            goto out;
 
-                ret = glusterd_get_geo_rep_session (key, origin_vol->volname,
-                                                    origin_vol->gsync_slaves,
-                                                    session, slave);
-                if (ret) {
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "Failed to get geo-rep session");
-                        goto out;
-                }
+        ret = glusterd_get_geo_rep_session(
+            key, origin_vol->volname, origin_vol->gsync_slaves, session, slave);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_GEOREP_GET_FAILED,
+                   "Failed to get geo-rep session");
+            goto out;
+        }
 
-                ret = glusterd_copy_geo_rep_session_files (session, snap_vol);
-                if (ret) {
-                        gf_log (this->name, GF_LOG_ERROR, "Failed to copy files"
-                                " related to session %s", session);
-                        goto out;
-                }
+        ret = glusterd_copy_geo_rep_session_files(session, snap_vol);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_FILE_OP_FAILED,
+                   "Failed to copy files"
+                   " related to session %s",
+                   session);
+            goto out;
         }
+    }
 
 out:
-        return ret;
+    return ret;
 }
 
 /* This function will restore a snapshot volumes
@@ -725,151 +785,140 @@ out:
  * @return              Negative value on Failure and 0 in success
  */
 int
-glusterd_snapshot_restore (dict_t *dict, char **op_errstr, dict_t *rsp_dict)
+glusterd_snapshot_restore(dict_t *dict, char **op_errstr, dict_t *rsp_dict)
 {
-        int                     ret             = -1;
-        int32_t                 volcount        = -1;
-        char                    *snapname       = NULL;
-        xlator_t                *this           = NULL;
-        glusterd_volinfo_t      *snap_volinfo   = NULL;
-        glusterd_volinfo_t      *tmp            = NULL;
-        glusterd_volinfo_t      *parent_volinfo = NULL;
-        glusterd_snap_t         *snap           = NULL;
-        glusterd_conf_t         *priv           = NULL;
-
-        this = THIS;
-
-        GF_ASSERT (this);
-        GF_ASSERT (dict);
-        GF_ASSERT (op_errstr);
-        GF_ASSERT (rsp_dict);
-
-        priv = this->private;
-        GF_ASSERT (priv);
-
-        ret = dict_get_str (dict, "snapname", &snapname);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "Failed to get "
-                        "snap name");
-                goto out;
+    int ret = -1;
+    int32_t volcount = -1;
+    char *snapname = NULL;
+    xlator_t *this = NULL;
+    glusterd_volinfo_t *snap_volinfo = NULL;
+    glusterd_volinfo_t *tmp = NULL;
+    glusterd_volinfo_t *parent_volinfo = NULL;
+    glusterd_snap_t *snap = NULL;
+    glusterd_conf_t *priv = NULL;
+
+    this = THIS;
+
+    GF_ASSERT(this);
+    GF_ASSERT(dict);
+    GF_ASSERT(op_errstr);
+    GF_ASSERT(rsp_dict);
+
+    priv = this->private;
+    GF_ASSERT(priv);
+
+    ret = dict_get_strn(dict, "snapname", SLEN("snapname"), &snapname);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+               "Failed to get snap name");
+        goto out;
+    }
+
+    snap = glusterd_find_snap_by_name(snapname);
+    if (NULL == snap) {
+        ret = gf_asprintf(op_errstr, "Snapshot (%s) does not exist", snapname);
+        if (ret < 0) {
+            goto out;
         }
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_SNAP_NOT_FOUND, "%s",
+               *op_errstr);
+        ret = -1;
+        goto out;
+    }
 
-        snap = glusterd_find_snap_by_name (snapname);
-        if (NULL == snap) {
-                ret = gf_asprintf (op_errstr, "Snapshot (%s) does not exist",
-                                   snapname);
-                if (ret < 0) {
-                        goto out;
-                }
-                gf_log (this->name, GF_LOG_ERROR, "%s", *op_errstr);
-                ret = -1;
-                goto out;
+    volcount = 0;
+    cds_list_for_each_entry_safe(snap_volinfo, tmp, &snap->volumes, vol_list)
+    {
+        volcount++;
+        ret = glusterd_volinfo_find(snap_volinfo->parent_volname,
+                                    &parent_volinfo);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, EINVAL, GD_MSG_VOL_NOT_FOUND,
+                   "Could not get volinfo of %s", snap_volinfo->parent_volname);
+            goto out;
         }
 
-        volcount = 0;
-        cds_list_for_each_entry_safe (snap_volinfo, tmp, &snap->volumes,
-                                      vol_list) {
-                volcount++;
-                ret = glusterd_volinfo_find (snap_volinfo->parent_volname,
-                                             &parent_volinfo);
-                if (ret) {
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "Could not get volinfo of %s",
-                                snap_volinfo->parent_volname);
-                        goto out;
-                }
-
-                ret = dict_set_dynstr_with_alloc (rsp_dict, "snapuuid",
-                                                  uuid_utoa (snap->snap_id));
-                if (ret) {
-                        gf_log (this->name, GF_LOG_ERROR, "Failed to set snap "
-                                "uuid in response dictionary for %s snapshot",
-                                snap->snapname);
-                        goto out;
-                }
-
-
-                ret = dict_set_dynstr_with_alloc (rsp_dict, "volname",
-                                                  snap_volinfo->parent_volname);
-                if (ret) {
-                        gf_log (this->name, GF_LOG_ERROR, "Failed to set snap "
-                                "uuid in response dictionary for %s snapshot",
-                                snap->snapname);
-                        goto out;
-                }
-
-                ret = dict_set_dynstr_with_alloc (rsp_dict, "volid",
-                                        uuid_utoa (parent_volinfo->volume_id));
-                if (ret) {
-                        gf_log (this->name, GF_LOG_ERROR, "Failed to set snap "
-                                "uuid in response dictionary for %s snapshot",
-                                snap->snapname);
-                        goto out;
-                }
-
-                /* Take backup of the volinfo folder */
-                ret = glusterd_snapshot_backup_vol (parent_volinfo);
-                if (ret) {
-                        gf_log (this->name, GF_LOG_ERROR, "Failed to backup "
-                                "volume backend files for %s volume",
-                                parent_volinfo->volname);
-                        goto out;
-                }
-
-                if (is_origin_glusterd (dict) == _gf_true) {
-                        /* From origin glusterd check if      *
-                         * any peers with snap bricks is down */
-                        ret = glusterd_find_missed_snap
-                                               (rsp_dict, snap_volinfo,
-                                                &priv->peers,
-                                                GF_SNAP_OPTION_TYPE_RESTORE);
-                        if (ret) {
-                                gf_log (this->name, GF_LOG_ERROR,
-                                        "Failed to find missed snap restores");
-                                goto out;
-                        }
-                }
+        ret = dict_set_dynstr_with_alloc(rsp_dict, "snapuuid",
+                                         uuid_utoa(snap->snap_id));
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+                   "Failed to set snap "
+                   "uuid in response dictionary for %s snapshot",
+                   snap->snapname);
+            goto out;
+        }
 
-                ret = gd_restore_snap_volume (dict, rsp_dict, parent_volinfo,
-                                              snap_volinfo, volcount);
-                if (ret) {
-                        /* No need to update op_errstr because it is assumed
-                         * that the called function will do that in case of
-                         * failure.
-                         */
-                        gf_log (this->name, GF_LOG_ERROR, "Failed to restore "
-                                "snap for %s", snapname);
-                        goto out;
-                }
+        ret = dict_set_dynstr_with_alloc(rsp_dict, "volname",
+                                         snap_volinfo->parent_volname);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+                   "Failed to set snap "
+                   "uuid in response dictionary for %s snapshot",
+                   snap->snapname);
+            goto out;
+        }
 
-                /* Restore is successful therefore delete the original volume's
-                 * volinfo. If the volinfo is already restored then we should
-                 * delete the backend LVMs */
-                if (!gf_uuid_is_null (parent_volinfo->restored_from_snap)) {
-                        ret = glusterd_lvm_snapshot_remove (rsp_dict,
-                                                            parent_volinfo);
-                        if (ret) {
-                                gf_log (this->name, GF_LOG_ERROR,
-                                        "Failed to remove LVM backend");
-                                goto out;
-                        }
-                }
+        ret = dict_set_dynstr_with_alloc(rsp_dict, "volid",
+                                         uuid_utoa(parent_volinfo->volume_id));
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+                   "Failed to set snap "
+                   "uuid in response dictionary for %s snapshot",
+                   snap->snapname);
+            goto out;
+        }
 
-                /* Detach the volinfo from priv->volumes, so that no new
-                 * command can ref it any more and then unref it.
-                 */
-                cds_list_del_init (&parent_volinfo->vol_list);
-                glusterd_volinfo_unref (parent_volinfo);
+        if (is_origin_glusterd(dict) == _gf_true) {
+            /* From origin glusterd check if      *
+             * any peers with snap bricks is down */
+            ret = glusterd_find_missed_snap(rsp_dict, snap_volinfo,
+                                            &priv->peers,
+                                            GF_SNAP_OPTION_TYPE_RESTORE);
+            if (ret) {
+                gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_MISSED_SNAP_GET_FAIL,
+                       "Failed to find missed snap restores");
+                goto out;
+            }
+        }
+        /* During snapshot restore, mount point for stopped snap
+         * should exist as it is required to set extended attribute.
+         */
+        ret = glusterd_recreate_vol_brick_mounts(this, snap_volinfo);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_BRK_MNT_RECREATE_FAIL,
+                   "Failed to recreate brick mounts for %s", snap->snapname);
+            goto out;
         }
 
-        ret = 0;
+        ret = gd_restore_snap_volume(dict, rsp_dict, parent_volinfo,
+                                     snap_volinfo, volcount);
+        if (ret) {
+            /* No need to update op_errstr because it is assumed
+             * that the called function will do that in case of
+             * failure.
+             */
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_SNAP_RESTORE_FAIL,
+                   "Failed to restore "
+                   "snap for %s",
+                   snapname);
+            goto out;
+        }
 
-        /* TODO: Need to check if we need to delete the snap after the
-         * operation is successful or not. Also need to persist the state
-         * of restore operation in the store.
+        /* Detach the volinfo from priv->volumes, so that no new
+         * command can ref it any more and then unref it.
          */
+        cds_list_del_init(&parent_volinfo->vol_list);
+        glusterd_volinfo_unref(parent_volinfo);
+    }
+
+    ret = 0;
+
+    /* TODO: Need to check if we need to delete the snap after the
+     * operation is successful or not. Also need to persist the state
+     * of restore operation in the store.
+     */
 out:
-        return ret;
+    return ret;
 }
 
 /* This function is called before actual restore is taken place. This function
@@ -882,426 +931,466 @@ out:
  * @return              Negative value on Failure and 0 in success
  */
 int32_t
-glusterd_snapshot_restore_prevalidate (dict_t *dict, char **op_errstr,
-                                       dict_t *rsp_dict)
+glusterd_snapshot_restore_prevalidate(dict_t *dict, char **op_errstr,
+                                      uint32_t *op_errno, dict_t *rsp_dict)
 {
-        int                     ret             = -1;
-        int32_t                 i               = 0;
-        int32_t                 volcount        = 0;
-        int32_t                 brick_count     = 0;
-        gf_boolean_t            snap_restored   = _gf_false;
-        char                    key[PATH_MAX]   = {0, };
-        char                    *volname        = NULL;
-        char                    *snapname       = NULL;
-        glusterd_volinfo_t      *volinfo        = NULL;
-        glusterd_brickinfo_t    *brickinfo      = NULL;
-        glusterd_snap_t         *snap           = NULL;
-        xlator_t                *this           = NULL;
-
-        this = THIS;
-
-        GF_ASSERT (this);
-        GF_ASSERT (dict);
-        GF_ASSERT (op_errstr);
-        GF_ASSERT (rsp_dict);
-
-        ret = dict_get_str (dict, "snapname", &snapname);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "Failed to get "
-                        "snap name");
-                goto out;
+    int ret = -1;
+    int32_t i = 0;
+    int32_t volcount = 0;
+    int32_t brick_count = 0;
+    gf_boolean_t snap_restored = _gf_false;
+    char key[64] = "";
+    int keylen;
+    char *volname = NULL;
+    char *snapname = NULL;
+    glusterd_volinfo_t *volinfo = NULL;
+    glusterd_brickinfo_t *brickinfo = NULL;
+    glusterd_snap_t *snap = NULL;
+    xlator_t *this = NULL;
+
+    this = THIS;
+
+    GF_ASSERT(this);
+    GF_ASSERT(dict);
+    GF_ASSERT(op_errstr);
+    GF_VALIDATE_OR_GOTO(this->name, op_errno, out);
+    GF_ASSERT(rsp_dict);
+
+    ret = dict_get_strn(dict, "snapname", SLEN("snapname"), &snapname);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+               "Failed to get "
+               "snap name");
+        goto out;
+    }
+
+    snap = glusterd_find_snap_by_name(snapname);
+    if (NULL == snap) {
+        ret = gf_asprintf(op_errstr, "Snapshot (%s) does not exist", snapname);
+        *op_errno = EG_SNAPEXST;
+        if (ret < 0) {
+            goto out;
         }
+        gf_msg(this->name, GF_LOG_ERROR, EINVAL, GD_MSG_SNAP_NOT_FOUND, "%s",
+               *op_errstr);
+        ret = -1;
+        goto out;
+    }
 
-        snap = glusterd_find_snap_by_name (snapname);
-        if (NULL == snap) {
-                ret = gf_asprintf (op_errstr, "Snapshot (%s) does not exist",
-                                snapname);
-                if (ret < 0) {
-                        goto out;
-                }
-                gf_log (this->name, GF_LOG_ERROR, "%s", *op_errstr);
-                ret = -1;
-                goto out;
-        }
+    snap_restored = snap->snap_restored;
 
-        snap_restored = snap->snap_restored;
+    if (snap_restored) {
+        ret = gf_asprintf(op_errstr,
+                          "Snapshot (%s) is already "
+                          "restored",
+                          snapname);
+        if (ret < 0) {
+            goto out;
+        }
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_SNAPSHOT_OP_FAILED, "%s",
+               *op_errstr);
+        ret = -1;
+        goto out;
+    }
+
+    ret = dict_set_strn(rsp_dict, "snapname", SLEN("snapname"), snapname);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+               "Failed to set "
+               "snap name(%s)",
+               snapname);
+        goto out;
+    }
+
+    ret = dict_get_int32n(dict, "volcount", SLEN("volcount"), &volcount);
+
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+               "Failed to get volume count");
+        goto out;
+    }
+
+    /* Snapshot restore will only work if all the volumes,
+       that are part of the snapshot, are stopped. */
+    for (i = 1; i <= volcount; ++i) {
+        keylen = snprintf(key, sizeof(key), "volname%d", i);
+        ret = dict_get_strn(dict, key, keylen, &volname);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+                   "Failed to "
+                   "get volume name");
+            goto out;
+        }
+
+        ret = glusterd_volinfo_find(volname, &volinfo);
+        if (ret) {
+            ret = gf_asprintf(op_errstr,
+                              "Volume (%s) "
+                              "does not exist",
+                              volname);
+            *op_errno = EG_NOVOL;
+            if (ret < 0) {
+                goto out;
+            }
+            gf_msg(this->name, GF_LOG_ERROR, EINVAL, GD_MSG_VOL_NOT_FOUND, "%s",
+                   *op_errstr);
+            ret = -1;
+            goto out;
+        }
+
+        if (glusterd_is_volume_started(volinfo)) {
+            ret = gf_asprintf(
+                op_errstr,
+                "Volume (%s) has been "
+                "started. Volume needs to be stopped before restoring "
+                "a snapshot.",
+                volname);
+            *op_errno = EG_VOLRUN;
+            if (ret < 0) {
+                goto out;
+            }
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_SNAPSHOT_OP_FAILED, "%s",
+                   *op_errstr);
+            ret = -1;
+            goto out;
+        }
+
+        /* Take backup of the volinfo folder */
+        ret = glusterd_snapshot_backup_vol(volinfo);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_VOL_OP_FAILED,
+                   "Failed to backup "
+                   "volume backend files for %s volume",
+                   volinfo->volname);
+            goto out;
+        }
+    }
+
+    /* Get brickinfo for snap_volumes */
+    volcount = 0;
+    cds_list_for_each_entry(volinfo, &snap->volumes, vol_list)
+    {
+        volcount++;
+        brick_count = 0;
 
-        if (snap_restored) {
-                ret = gf_asprintf (op_errstr, "Snapshot (%s) is already "
-                                  "restored", snapname);
-                if (ret < 0) {
-                        goto out;
-                }
-                gf_log (this->name, GF_LOG_ERROR, "%s", *op_errstr);
-                ret = -1;
-                goto out;
-        }
+        cds_list_for_each_entry(brickinfo, &volinfo->bricks, brick_list)
+        {
+            brick_count++;
+            if (gf_uuid_compare(brickinfo->uuid, MY_UUID))
+                continue;
 
-        ret = dict_set_str (rsp_dict, "snapname", snapname);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "Failed to set "
-                        "snap name(%s)", snapname);
+            keylen = snprintf(key, sizeof(key), "snap%d.brick%d.path", volcount,
+                              brick_count);
+            ret = dict_set_strn(rsp_dict, key, keylen, brickinfo->path);
+            if (ret) {
+                gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+                       "Failed to set %s", key);
                 goto out;
-        }
-
-        ret = dict_get_int32 (dict, "volcount", &volcount);
+            }
 
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "Failed to get volume count");
+            keylen = snprintf(key, sizeof(key), "snap%d.brick%d.snap_status",
+                              volcount, brick_count);
+            ret = dict_set_int32n(rsp_dict, key, keylen,
+                                  brickinfo->snap_status);
+            if (ret) {
+                gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+                       "Failed to set %s", key);
                 goto out;
-        }
-
-        /* Snapshot restore will only work if all the volumes,
-           that are part of the snapshot, are stopped. */
-        for (i = 1; i <= volcount; ++i) {
-                snprintf (key, sizeof (key), "volname%d", i);
-                ret = dict_get_str (dict, key, &volname);
-                if (ret) {
-                        gf_log (this->name, GF_LOG_ERROR, "Failed to "
-                                "get volume name");
-                        goto out;
-                }
-
-                ret = glusterd_volinfo_find (volname, &volinfo);
-                if (ret) {
-                        ret = gf_asprintf (op_errstr, "Volume (%s) "
-                                           "does not exist", volname);
-                        if (ret < 0) {
-                                goto out;
-                        }
-                        gf_log (this->name, GF_LOG_ERROR, "%s", *op_errstr);
-                        ret = -1;
-                        goto out;
-                }
+            }
 
-                if (glusterd_is_volume_started (volinfo)) {
-                        ret = gf_asprintf (op_errstr, "Volume (%s) has been "
-                        "started. Volume needs to be stopped before restoring "
-                        "a snapshot.", volname);
-                        if (ret < 0) {
-                                goto out;
-                        }
-                        gf_log (this->name, GF_LOG_ERROR, "%s", *op_errstr);
-                        ret = -1;
-                        goto out;
-                }
-        }
+            keylen = snprintf(key, sizeof(key), "snap%d.brick%d.device_path",
+                              volcount, brick_count);
+            ret = dict_set_strn(rsp_dict, key, keylen, brickinfo->device_path);
+            if (ret) {
+                gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+                       "Failed to set %s", key);
+                goto out;
+            }
 
-        /* Get brickinfo for snap_volumes */
-        volcount = 0;
-        cds_list_for_each_entry (volinfo, &snap->volumes, vol_list) {
-                volcount++;
-                brick_count = 0;
-                cds_list_for_each_entry (brickinfo, &volinfo->bricks,
-                                         brick_list) {
-                        brick_count++;
-                        if (gf_uuid_compare (brickinfo->uuid, MY_UUID))
-                                continue;
-
-                        snprintf (key, sizeof (key), "snap%d.brick%d.path",
-                                  volcount, brick_count);
-                        ret = dict_set_str (rsp_dict, key, brickinfo->path);
-                        if (ret) {
-                                gf_log (this->name, GF_LOG_ERROR,
-                                        "Failed to set %s", key);
-                                goto out;
-                        }
-
-                        snprintf (key, sizeof (key),
-                                  "snap%d.brick%d.snap_status",
-                                  volcount, brick_count);
-                        ret = dict_set_int32 (rsp_dict, key,
-                                              brickinfo->snap_status);
-                        if (ret) {
-                                gf_log (this->name, GF_LOG_ERROR,
-                                        "Failed to set %s", key);
-                                goto out;
-                        }
-
-                        snprintf (key, sizeof (key),
-                                  "snap%d.brick%d.device_path",
-                                  volcount, brick_count);
-                        ret = dict_set_str (rsp_dict, key,
-                                            brickinfo->device_path);
-                        if (ret) {
-                                gf_log (this->name, GF_LOG_ERROR,
-                                        "Failed to set %s", key);
-                                goto out;
-                        }
-
-                        snprintf (key, sizeof (key),
-                                  "snap%d.brick%d.fs_type",
-                                  volcount, brick_count);
-                        ret = dict_set_str (rsp_dict, key,
-                                            brickinfo->fstype);
-                        if (ret) {
-                                gf_log (this->name, GF_LOG_ERROR,
-                                        "Failed to set %s", key);
-                                goto out;
-                        }
-
-                        snprintf (key, sizeof (key),
-                                  "snap%d.brick%d.mnt_opts",
-                                  volcount, brick_count);
-                        ret = dict_set_str (rsp_dict, key,
-                                            brickinfo->mnt_opts);
-                        if (ret) {
-                                gf_log (this->name, GF_LOG_ERROR,
-                                        "Failed to set %s", key);
-                                goto out;
-                        }
-                }
+            keylen = snprintf(key, sizeof(key), "snap%d.brick%d.fs_type",
+                              volcount, brick_count);
+            ret = dict_set_strn(rsp_dict, key, keylen, brickinfo->fstype);
+            if (ret) {
+                gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+                       "Failed to set %s", key);
+                goto out;
+            }
 
-                snprintf (key, sizeof (key), "snap%d.brick_count", volcount);
-                ret = dict_set_int32 (rsp_dict, key, brick_count);
-                if (ret) {
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "Failed to set %s", key);
-                        goto out;
-                }
+            keylen = snprintf(key, sizeof(key), "snap%d.brick%d.mnt_opts",
+                              volcount, brick_count);
+            ret = dict_set_strn(rsp_dict, key, keylen, brickinfo->mnt_opts);
+            if (ret) {
+                gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+                       "Failed to set %s", key);
+                goto out;
+            }
         }
 
-        ret = dict_set_int32 (rsp_dict, "volcount", volcount);
+        keylen = snprintf(key, sizeof(key), "snap%d.brick_count", volcount);
+        ret = dict_set_int32n(rsp_dict, key, keylen, brick_count);
         if (ret) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "Failed to set %s", key);
-                goto out;
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+                   "Failed to set %s", key);
+            goto out;
         }
+    }
+
+    ret = dict_set_int32n(rsp_dict, "volcount", SLEN("volcount"), volcount);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+               "Failed to set %s", key);
+        goto out;
+    }
 
 out:
-        return ret;
+    return ret;
 }
 
 int
-snap_max_hard_limits_validate (dict_t *dict, char *volname,
-                               uint64_t value, char **op_errstr)
+snap_max_hard_limits_validate(dict_t *dict, char *volname, uint64_t value,
+                              char **op_errstr)
 {
-        char                err_str[PATH_MAX] = "";
-        glusterd_conf_t    *conf              = NULL;
-        glusterd_volinfo_t *volinfo           = NULL;
-        int                 ret               = -1;
-        uint64_t            max_limit         = GLUSTERD_SNAPS_MAX_HARD_LIMIT;
-        xlator_t           *this              = NULL;
-        uint64_t            opt_hard_max      = GLUSTERD_SNAPS_MAX_HARD_LIMIT;
-
-        this = THIS;
-
-        GF_ASSERT (this);
-        GF_ASSERT (dict);
-        GF_ASSERT (op_errstr);
-
-        conf = this->private;
-
-        GF_ASSERT (conf);
-
-        if (volname) {
-                ret = glusterd_volinfo_find (volname, &volinfo);
-                if (!ret) {
-                        if (volinfo->is_snap_volume) {
-                                ret = -1;
-                                snprintf (err_str, PATH_MAX,
-                                         "%s is a snap volume. Configuring "
-                                         "snap-max-hard-limit for a snap "
-                                         "volume is prohibited.", volname);
-                                goto out;
-                        }
-                }
-        }
+    char err_str[PATH_MAX] = "";
+    glusterd_conf_t *conf = NULL;
+    glusterd_volinfo_t *volinfo = NULL;
+    int ret = -1;
+    uint64_t max_limit = GLUSTERD_SNAPS_MAX_HARD_LIMIT;
+    xlator_t *this = NULL;
+    uint64_t opt_hard_max = GLUSTERD_SNAPS_MAX_HARD_LIMIT;
 
-        /* "snap-max-hard-limit" might not be set by user explicitly,
-         * in that case it's better to use the default value.
-         * Hence not erroring out if Key is not found.
-         */
-        ret = dict_get_uint64 (conf->opts,
-                               GLUSTERD_STORE_KEY_SNAP_MAX_HARD_LIMIT,
-                               &opt_hard_max);
-        if (ret) {
-                ret = 0;
-                gf_log (this->name, GF_LOG_DEBUG, "%s is not present in "
-                        "opts dictionary",
-                        GLUSTERD_STORE_KEY_SNAP_MAX_HARD_LIMIT);
-        }
+    this = THIS;
 
-        /* volume snap-max-hard-limit cannot exceed system snap-max-hard-limit.
-         * Hence during prevalidate following checks are made to ensure the
-         * snap-max-hard-limit set on one particular volume does not
-         * exceed snap-max-hard-limit set globally (system limit).
-         */
-        if (value && volname) {
-                max_limit = opt_hard_max;
-        }
+    GF_ASSERT(this);
+    GF_ASSERT(dict);
+    GF_ASSERT(op_errstr);
 
-        if ((value < 0) || (value > max_limit)) {
-                ret = -1;
-                snprintf (err_str, PATH_MAX, "Invalid snap-max-hard-limit "
-                          "%"PRIu64 ". Expected range 1 - %"PRIu64,
-                          value, max_limit);
-                goto out;
-        }
+    conf = this->private;
 
+    GF_ASSERT(conf);
+
+    if (volname) {
+        ret = glusterd_volinfo_find(volname, &volinfo);
+        if (!ret) {
+            if (volinfo->is_snap_volume) {
+                ret = -1;
+                snprintf(err_str, PATH_MAX,
+                         "%s is a snap volume. Configuring "
+                         "snap-max-hard-limit for a snap "
+                         "volume is prohibited.",
+                         volname);
+                goto out;
+            }
+        }
+    }
+
+    /* "snap-max-hard-limit" might not be set by user explicitly,
+     * in that case it's better to use the default value.
+     * Hence not erroring out if Key is not found.
+     */
+    ret = dict_get_uint64(conf->opts, GLUSTERD_STORE_KEY_SNAP_MAX_HARD_LIMIT,
+                          &opt_hard_max);
+    if (ret) {
         ret = 0;
+        gf_msg_debug(this->name, 0,
+                     "%s is not present in "
+                     "opts dictionary",
+                     GLUSTERD_STORE_KEY_SNAP_MAX_HARD_LIMIT);
+    }
+
+    /* volume snap-max-hard-limit cannot exceed system snap-max-hard-limit.
+     * Hence during prevalidate following checks are made to ensure the
+     * snap-max-hard-limit set on one particular volume does not
+     * exceed snap-max-hard-limit set globally (system limit).
+     */
+    if (value && volname) {
+        max_limit = opt_hard_max;
+    }
+
+    if (value > max_limit) {
+        ret = -1;
+        snprintf(err_str, PATH_MAX,
+                 "Invalid snap-max-hard-limit "
+                 "%" PRIu64 ". Expected range 1 - %" PRIu64,
+                 value, max_limit);
+        goto out;
+    }
+
+    ret = 0;
 out:
-        if (ret) {
-                *op_errstr = gf_strdup (err_str);
-                gf_log (this->name, GF_LOG_ERROR, "%s", err_str);
-        }
-        return ret;
+    if (ret) {
+        *op_errstr = gf_strdup(err_str);
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_SNAPSHOT_OP_FAILED, "%s",
+               err_str);
+    }
+    return ret;
 }
 
 int
-glusterd_snapshot_config_prevalidate (dict_t *dict, char **op_errstr)
+glusterd_snapshot_config_prevalidate(dict_t *dict, char **op_errstr,
+                                     uint32_t *op_errno)
 {
-        char               *volname             = NULL;
-        glusterd_volinfo_t *volinfo             = NULL;
-        xlator_t           *this                = NULL;
-        int                 ret                 = -1;
-        int                 config_command      = 0;
-        char                err_str[PATH_MAX]   = {0,};
-        glusterd_conf_t    *conf                = NULL;
-        uint64_t            hard_limit          = 0;
-        uint64_t            soft_limit          = 0;
-        gf_loglevel_t       loglevel            = GF_LOG_ERROR;
-        uint64_t            max_limit           = GLUSTERD_SNAPS_MAX_HARD_LIMIT;
-        int32_t             cur_auto_delete     = 0;
-        int32_t             req_auto_delete     = 0;
-        int32_t             cur_snap_activate   = 0;
-        int32_t             req_snap_activate   = 0;
-
-        this = THIS;
-
-        GF_ASSERT (this);
-        GF_ASSERT (dict);
-        GF_ASSERT (op_errstr);
-
-        conf = this->private;
-
-        GF_ASSERT (conf);
-
-        ret = dict_get_int32 (dict, "config-command", &config_command);
-        if (ret) {
-                snprintf (err_str, sizeof (err_str),
-                          "failed to get config-command type");
-                goto out;
-        }
-
-        if (config_command != GF_SNAP_CONFIG_TYPE_SET) {
-                ret = 0;
-                goto out;
-        }
-
-        ret = dict_get_str (dict, "volname", &volname);
-        if (volname) {
-                ret = glusterd_volinfo_find (volname, &volinfo);
-                if (ret) {
-                        snprintf (err_str, sizeof (err_str),
-                                  "Volume (%s) does not exist.", volname);
-                        goto out;
-                }
-        }
-
-        /* config values snap-max-hard-limit and snap-max-soft-limit are
-         * optional and hence we are not erroring out if values are not
-         * present
-         */
-        gd_get_snap_conf_values_if_present (dict, &hard_limit, &soft_limit);
-
-        if (hard_limit) {
-                /* Validations for snap-max-hard-limits */
-                ret = snap_max_hard_limits_validate (dict, volname,
-                                                     hard_limit, op_errstr);
-                if (ret) {
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "snap-max-hard-limit validation failed.");
-                        goto out;
-                }
-        }
-
-        if (soft_limit) {
-                max_limit = GLUSTERD_SNAPS_MAX_SOFT_LIMIT_PERCENT;
-                if ((soft_limit < 0) || (soft_limit > max_limit)) {
-                        ret = -1;
-                        snprintf (err_str, PATH_MAX, "Invalid "
-                                 "snap-max-soft-limit ""%"
-                                 PRIu64 ". Expected range 1 - %"PRIu64,
-                                 soft_limit, max_limit);
-                        goto out;
-                }
-        }
-
-        if (hard_limit || soft_limit) {
-                ret = 0;
-                goto out;
-        }
-
-        if (dict_get(dict, GLUSTERD_STORE_KEY_SNAP_AUTO_DELETE)) {
-                req_auto_delete = dict_get_str_boolean (dict,
-                                        GLUSTERD_STORE_KEY_SNAP_AUTO_DELETE,
-                                        _gf_false);
-                if (req_auto_delete < 0) {
-                        ret = -1;
-                        snprintf (err_str, sizeof (err_str), "Please enter a "
-                                  "valid boolean value for auto-delete");
-                        goto out;
-                }
-
-                /* Ignoring the error as the auto-delete is optional and
-                   might not be present in the options dictionary.*/
-                cur_auto_delete = dict_get_str_boolean (conf->opts,
-                                        GLUSTERD_STORE_KEY_SNAP_AUTO_DELETE,
-                                        _gf_false);
-
-                if (cur_auto_delete == req_auto_delete) {
-                        ret = -1;
-                        if (cur_auto_delete == _gf_true)
-                                snprintf (err_str, sizeof (err_str),
-                                          "auto-delete is already enabled");
-                        else
-                                snprintf (err_str, sizeof (err_str),
-                                          "auto-delete is already disabled");
-                        goto out;
-                }
-        } else if (dict_get(dict, GLUSTERD_STORE_KEY_SNAP_ACTIVATE)) {
-                req_snap_activate = dict_get_str_boolean (dict,
-                                        GLUSTERD_STORE_KEY_SNAP_ACTIVATE,
-                                        _gf_false);
-                if (req_snap_activate < 0) {
-                        ret = -1;
-                        snprintf (err_str, sizeof (err_str), "Please enter a "
-                                  "valid boolean value for activate-on-create");
-                        goto out;
-                }
-
-                /* Ignoring the error as the activate-on-create is optional and
-                   might not be present in the options dictionary.*/
-                cur_snap_activate = dict_get_str_boolean (conf->opts,
-                                        GLUSTERD_STORE_KEY_SNAP_ACTIVATE,
-                                        _gf_false);
-
-                if (cur_snap_activate == req_snap_activate) {
-                        ret = -1;
-                        if (cur_snap_activate == _gf_true)
-                                snprintf (err_str, sizeof (err_str),
-                                      "activate-on-create is already enabled");
-                        else
-                                snprintf (err_str, sizeof (err_str),
-                                      "activate-on-create is already disabled");
-                        goto out;
-                }
-        } else {
-                ret = -1;
-                snprintf (err_str, sizeof (err_str), "Invalid option");
-                goto out;
-        }
-
+    char *volname = NULL;
+    glusterd_volinfo_t *volinfo = NULL;
+    xlator_t *this = NULL;
+    int ret = -1;
+    int config_command = 0;
+    char err_str[PATH_MAX] = "";
+    glusterd_conf_t *conf = NULL;
+    uint64_t hard_limit = 0;
+    uint64_t soft_limit = 0;
+    gf_loglevel_t loglevel = GF_LOG_ERROR;
+    uint64_t max_limit = GLUSTERD_SNAPS_MAX_HARD_LIMIT;
+    int32_t cur_auto_delete = 0;
+    int32_t req_auto_delete = 0;
+    int32_t cur_snap_activate = 0;
+    int32_t req_snap_activate = 0;
+
+    this = THIS;
+
+    GF_ASSERT(this);
+    GF_ASSERT(dict);
+    GF_ASSERT(op_errstr);
+    GF_VALIDATE_OR_GOTO(this->name, op_errno, out);
+
+    conf = this->private;
+
+    GF_ASSERT(conf);
+
+    ret = dict_get_int32n(dict, "config-command", SLEN("config-command"),
+                          &config_command);
+    if (ret) {
+        snprintf(err_str, sizeof(err_str), "failed to get config-command type");
+        goto out;
+    }
+
+    if (config_command != GF_SNAP_CONFIG_TYPE_SET) {
         ret = 0;
+        goto out;
+    }
+
+    ret = dict_get_strn(dict, "volname", SLEN("volname"), &volname);
+    if (volname) {
+        ret = glusterd_volinfo_find(volname, &volinfo);
+        if (ret) {
+            snprintf(err_str, sizeof(err_str), "Volume (%s) does not exist.",
+                     volname);
+            *op_errno = EG_NOVOL;
+            goto out;
+        }
+    }
+
+    /* config values snap-max-hard-limit and snap-max-soft-limit are
+     * optional and hence we are not erroring out if values are not
+     * present
+     */
+    gd_get_snap_conf_values_if_present(dict, &hard_limit, &soft_limit);
+
+    if (hard_limit) {
+        /* Validations for snap-max-hard-limits */
+        ret = snap_max_hard_limits_validate(dict, volname, hard_limit,
+                                            op_errstr);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_HARD_LIMIT_SET_FAIL,
+                   "snap-max-hard-limit validation failed.");
+            *op_errno = EINVAL;
+            goto out;
+        }
+    }
+
+    if (soft_limit) {
+        max_limit = GLUSTERD_SNAPS_MAX_SOFT_LIMIT_PERCENT;
+        if (soft_limit > max_limit) {
+            ret = -1;
+            snprintf(err_str, PATH_MAX,
+                     "Invalid "
+                     "snap-max-soft-limit "
+                     "%" PRIu64 ". Expected range 1 - %" PRIu64,
+                     soft_limit, max_limit);
+            *op_errno = EINVAL;
+            goto out;
+        }
+    }
+
+    if (hard_limit || soft_limit) {
+        ret = 0;
+        goto out;
+    }
+
+    if (dict_getn(dict, GLUSTERD_STORE_KEY_SNAP_AUTO_DELETE,
+                  SLEN(GLUSTERD_STORE_KEY_SNAP_AUTO_DELETE))) {
+        req_auto_delete = dict_get_str_boolean(
+            dict, GLUSTERD_STORE_KEY_SNAP_AUTO_DELETE, _gf_false);
+        if (req_auto_delete < 0) {
+            ret = -1;
+            snprintf(err_str, sizeof(err_str),
+                     "Please enter a "
+                     "valid boolean value for auto-delete");
+            *op_errno = EINVAL;
+            goto out;
+        }
+
+        /* Ignoring the error as the auto-delete is optional and
+           might not be present in the options dictionary.*/
+        cur_auto_delete = dict_get_str_boolean(
+            conf->opts, GLUSTERD_STORE_KEY_SNAP_AUTO_DELETE, _gf_false);
+
+        if (cur_auto_delete == req_auto_delete) {
+            ret = -1;
+            if (cur_auto_delete == _gf_true)
+                snprintf(err_str, sizeof(err_str),
+                         "auto-delete is already enabled");
+            else
+                snprintf(err_str, sizeof(err_str),
+                         "auto-delete is already disabled");
+            *op_errno = EINVAL;
+            goto out;
+        }
+    } else if (dict_getn(dict, GLUSTERD_STORE_KEY_SNAP_ACTIVATE,
+                         SLEN(GLUSTERD_STORE_KEY_SNAP_ACTIVATE))) {
+        req_snap_activate = dict_get_str_boolean(
+            dict, GLUSTERD_STORE_KEY_SNAP_ACTIVATE, _gf_false);
+        if (req_snap_activate < 0) {
+            ret = -1;
+            snprintf(err_str, sizeof(err_str),
+                     "Please enter a "
+                     "valid boolean value for activate-on-create");
+            *op_errno = EINVAL;
+            goto out;
+        }
+
+        /* Ignoring the error as the activate-on-create is optional and
+           might not be present in the options dictionary.*/
+        cur_snap_activate = dict_get_str_boolean(
+            conf->opts, GLUSTERD_STORE_KEY_SNAP_ACTIVATE, _gf_false);
+
+        if (cur_snap_activate == req_snap_activate) {
+            ret = -1;
+            if (cur_snap_activate == _gf_true)
+                snprintf(err_str, sizeof(err_str),
+                         "activate-on-create is already enabled");
+            else
+                snprintf(err_str, sizeof(err_str),
+                         "activate-on-create is already disabled");
+            *op_errno = EINVAL;
+            goto out;
+        }
+    } else {
+        ret = -1;
+        snprintf(err_str, sizeof(err_str), "Invalid option");
+        *op_errno = EINVAL;
+        goto out;
+    }
+
+    ret = 0;
 out:
 
-        if (ret && err_str[0] != '\0') {
-                gf_log (this->name, loglevel, "%s", err_str);
-                *op_errstr = gf_strdup (err_str);
-        }
+    if (ret && err_str[0] != '\0') {
+        gf_msg(this->name, loglevel, 0, GD_MSG_SNAPSHOT_OP_FAILED, "%s",
+               err_str);
+        *op_errstr = gf_strdup(err_str);
+    }
 
-        return ret;
+    return ret;
 }
 
 /* This function will be called from RPC handler routine.
@@ -1315,456 +1404,452 @@ out:
  * @return -1 on error and 0 on success
  */
 int
-glusterd_handle_snapshot_config (rpcsvc_request_t *req, glusterd_op_t op,
-                               dict_t *dict, char *err_str, size_t len)
+glusterd_handle_snapshot_config(rpcsvc_request_t *req, glusterd_op_t op,
+                                dict_t *dict, char *err_str, size_t len)
 {
-        int32_t         ret             = -1;
-        char            *volname        = NULL;
-        xlator_t        *this           = NULL;
-        int             config_command  = 0;
-
-        this = THIS;
-        GF_ASSERT (this);
-
-        GF_VALIDATE_OR_GOTO (this->name, req, out);
-        GF_VALIDATE_OR_GOTO (this->name, dict, out);
-
-        /* TODO : Type of lock to be taken when we are setting
-         * limits system wide
-         */
-        ret = dict_get_int32 (dict, "config-command", &config_command);
-        if (ret) {
-                snprintf (err_str, len,
-                         "Failed to get config-command type");
-                goto out;
-        }
-
-        ret = dict_get_str (dict, "volname", &volname);
-
-        switch (config_command) {
+    int32_t ret = -1;
+    char *volname = NULL;
+    xlator_t *this = NULL;
+    int config_command = 0;
+
+    this = THIS;
+    GF_ASSERT(this);
+
+    GF_VALIDATE_OR_GOTO(this->name, req, out);
+    GF_VALIDATE_OR_GOTO(this->name, dict, out);
+
+    /* TODO : Type of lock to be taken when we are setting
+     * limits system wide
+     */
+    ret = dict_get_int32n(dict, "config-command", SLEN("config-command"),
+                          &config_command);
+    if (ret) {
+        snprintf(err_str, len, "Failed to get config-command type");
+        gf_smsg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+                "Key=config-command", NULL);
+        goto out;
+    }
+
+    ret = dict_get_strn(dict, "volname", SLEN("volname"), &volname);
+
+    switch (config_command) {
         case GF_SNAP_CONFIG_TYPE_SET:
-                if (!volname) {
-                        ret = dict_set_int32 (dict, "hold_vol_locks",
-                                              _gf_false);
-                        if (ret) {
-                                gf_log (this->name, GF_LOG_ERROR,
-                                        "Unable to set hold_vol_locks value "
-                                        "as _gf_false");
-                                goto out;
-                        }
-
-                }
-                ret = glusterd_mgmt_v3_initiate_all_phases (req, op, dict);
-                break;
+            if (!volname) {
+                ret = dict_set_int32n(dict, "hold_vol_locks",
+                                      SLEN("hold_vol_locks"), _gf_false);
+                if (ret) {
+                    gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+                           "Unable to set hold_vol_locks value "
+                           "as _gf_false");
+                    goto out;
+                }
+            }
+            ret = glusterd_mgmt_v3_initiate_all_phases(req, op, dict);
+            break;
         case GF_SNAP_CONFIG_DISPLAY:
-                /* Reading data from local node only */
-                ret = snap_max_limits_display_commit (dict, volname,
-                                                      err_str, len);
-                if (ret) {
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "snap-max-limit "
-                                "display commit failed.");
-                        goto out;
-                }
-
-                /* If everything is successful then send the response
-                 * back to cli
-                 */
-                ret = glusterd_op_send_cli_response (op, 0, 0, req, dict,
-                                                     err_str);
-                if (ret) {
-                        gf_log (this->name, GF_LOG_ERROR, "Failed to send cli "
-                                        "response");
-                        goto out;
-                }
-
-                break;
+            /* Reading data from local node only */
+            ret = snap_max_limits_display_commit(dict, volname, err_str, len);
+            if (ret) {
+                gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_HARD_LIMIT_SET_FAIL,
+                       "snap-max-limit "
+                       "display commit failed.");
+                goto out;
+            }
+
+            /* If everything is successful then send the response
+             * back to cli
+             */
+            ret = glusterd_op_send_cli_response(op, 0, 0, req, dict, err_str);
+            if (ret) {
+                gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_NO_CLI_RESP,
+                       "Failed to send cli "
+                       "response");
+                goto out;
+            }
+
+            break;
         default:
-                gf_log (this->name, GF_LOG_ERROR, "Unknown config type");
-                ret = -1;
-                break;
-        }
+            gf_msg(this->name, GF_LOG_ERROR, EINVAL, GD_MSG_COMMAND_NOT_FOUND,
+                   "Unknown config type");
+            ret = -1;
+            break;
+    }
 out:
-        return ret;
+    return ret;
 }
 int
-glusterd_snap_create_clone_pre_val_use_rsp_dict (dict_t *dst, dict_t *src)
+glusterd_snap_create_clone_pre_val_use_rsp_dict(dict_t *dst, dict_t *src)
 {
-        char                  *snap_brick_dir        = NULL;
-        char                  *snap_device           = NULL;
-        char                   key[PATH_MAX]         = "";
-        char                  *value                 = "";
-        char                   snapbrckcnt[PATH_MAX] = "";
-        char                   snapbrckord[PATH_MAX] = "";
-        int                    ret                   = -1;
-        int64_t                i                     = -1;
-        int64_t                j                     = -1;
-        int64_t                volume_count          = 0;
-        int64_t                brick_count           = 0;
-        int64_t                brick_order           = 0;
-        xlator_t              *this                  = NULL;
-        int32_t                brick_online          = 0;
-
-        this = THIS;
-        GF_ASSERT (this);
-        GF_ASSERT (dst);
-        GF_ASSERT (src);
-
-        ret = dict_get_int64 (src, "volcount", &volume_count);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "failed to "
-                        "get the volume count");
-                goto out;
-        }
-
-        for (i = 0; i < volume_count; i++) {
-                memset (snapbrckcnt, '\0', sizeof(snapbrckcnt));
-                ret = snprintf (snapbrckcnt, sizeof(snapbrckcnt) - 1,
-                                "vol%"PRId64"_brickcount", i+1);
-                ret = dict_get_int64 (src, snapbrckcnt, &brick_count);
-                if (ret) {
-                        gf_log (this->name, GF_LOG_TRACE,
-                                "No bricks for this volume in this dict");
-                        continue;
-                }
-
-                for (j = 0; j < brick_count; j++) {
-                        /* Fetching data from source dict */
-                        snprintf (key, sizeof(key) - 1,
-                                  "vol%"PRId64".brickdir%"PRId64, i+1, j);
-                        ret = dict_get_ptr (src, key,
-                                            (void **)&snap_brick_dir);
-                        if (ret) {
-                                gf_log (this->name, GF_LOG_WARNING,
-                                        "Unable to fetch %s", key);
-                                continue;
-                        }
-
-                        /* Fetching brick order from source dict */
-                        snprintf (snapbrckord, sizeof(snapbrckord) - 1,
-                                  "vol%"PRId64".brick%"PRId64".order", i+1, j);
-                        ret = dict_get_int64 (src, snapbrckord, &brick_order);
-                        if (ret) {
-                                gf_log (this->name, GF_LOG_ERROR,
-                                        "Failed to get brick order");
-                                goto out;
-                        }
-
-                        snprintf (key, sizeof(key) - 1,
-                                  "vol%"PRId64".brickdir%"PRId64, i+1,
-                                  brick_order);
-                        ret = dict_set_dynstr_with_alloc (dst, key,
-                                                          snap_brick_dir);
-                        if (ret) {
-                                gf_log (this->name, GF_LOG_ERROR,
-                                        "Failed to set %s", key);
-                                goto out;
-                        }
-
-                        snprintf (key, sizeof(key) - 1,
-                                  "vol%"PRId64".fstype%"PRId64, i+1, j);
-                        ret = dict_get_str (src, key, &value);
-                        if (ret) {
-                                gf_log (this->name, GF_LOG_WARNING,
-                                        "Unable to fetch %s", key);
-                                continue;
-                        }
-
-                        snprintf (key, sizeof(key) - 1,
-                                  "vol%"PRId64".fstype%"PRId64, i+1,
-                                  brick_order);
-                        ret = dict_set_dynstr_with_alloc (dst, key, value);
-                        if (ret) {
-                                gf_log (this->name, GF_LOG_ERROR,
-                                        "Failed to set %s", key);
-                                goto out;
-                        }
-
-                        snprintf (key, sizeof(key) - 1,
-                                  "vol%"PRId64".mnt_opts%"PRId64, i+1, j);
-                        ret = dict_get_str (src, key, &value);
-                        if (ret) {
-                                gf_log (this->name, GF_LOG_WARNING,
-                                        "Unable to fetch %s", key);
-                                continue;
-                        }
-
-                        snprintf (key, sizeof(key) - 1,
-                                  "vol%"PRId64".mnt_opts%"PRId64, i+1,
-                                  brick_order);
-                        ret = dict_set_dynstr_with_alloc (dst, key, value);
-                        if (ret) {
-                                gf_log (this->name, GF_LOG_ERROR,
-                                        "Failed to set %s", key);
-                                goto out;
-                        }
-
-                        snprintf (key, sizeof(key) - 1,
-                                  "vol%"PRId64".brick_snapdevice%"PRId64,
-                                  i+1, j);
-                        ret = dict_get_ptr (src, key,
-                                            (void **)&snap_device);
-                        if (ret) {
-                                gf_log (this->name, GF_LOG_ERROR,
-                                        "Unable to fetch snap_device");
-                                goto out;
-                        }
-
-                        snprintf (key, sizeof(key) - 1,
-                                  "vol%"PRId64".brick_snapdevice%"PRId64,
-                                  i+1, brick_order);
-                        ret = dict_set_dynstr_with_alloc (dst, key,
-                                                          snap_device);
-                        if (ret) {
-                                gf_log (this->name, GF_LOG_ERROR,
-                                        "Failed to set %s", key);
-                                goto out;
-                        }
-
-                        snprintf (key, sizeof (key),
-                                  "vol%"PRId64".brick%"PRId64".status", i+1, brick_order);
-                        ret = dict_get_int32 (src, key, &brick_online);
-                        if (ret) {
-                                gf_log (this->name, GF_LOG_ERROR, "failed to "
-                                        "get the brick status");
-                                goto out;
-                        }
-
-                        ret = dict_set_int32 (dst, key, brick_online);
-                        if (ret) {
-                                gf_log (this->name, GF_LOG_ERROR, "failed to "
-                                        "set the brick status");
-                                goto out;
-                        }
-                        brick_online = 0;
-                }
-        }
-        ret = 0;
+    char *snap_brick_dir = NULL;
+    char *snap_device = NULL;
+    char key[64] = "";
+    int keylen;
+    char *value = "";
+    char snapbrckcnt[PATH_MAX] = "";
+    char snapbrckord[PATH_MAX] = "";
+    int ret = -1;
+    int64_t i = -1;
+    int64_t j = -1;
+    int64_t volume_count = 0;
+    int64_t brick_count = 0;
+    int64_t brick_order = 0;
+    xlator_t *this = NULL;
+    int32_t brick_online = 0;
+
+    this = THIS;
+    GF_ASSERT(this);
+    GF_ASSERT(dst);
+    GF_ASSERT(src);
+
+    ret = dict_get_int64(src, "volcount", &volume_count);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+               "failed to "
+               "get the volume count");
+        goto out;
+    }
+
+    for (i = 0; i < volume_count; i++) {
+        ret = snprintf(snapbrckcnt, sizeof(snapbrckcnt) - 1,
+                       "vol%" PRId64 "_brickcount", i + 1);
+        ret = dict_get_int64(src, snapbrckcnt, &brick_count);
+        if (ret) {
+            gf_msg_trace(this->name, 0,
+                         "No bricks for this volume in this dict");
+            continue;
+        }
+
+        for (j = 0; j < brick_count; j++) {
+            /* Fetching data from source dict */
+            snprintf(key, sizeof(key), "vol%" PRId64 ".brickdir%" PRId64, i + 1,
+                     j);
+            ret = dict_get_ptr(src, key, (void **)&snap_brick_dir);
+            if (ret) {
+                gf_msg(this->name, GF_LOG_WARNING, 0, GD_MSG_DICT_GET_FAILED,
+                       "Unable to fetch %s", key);
+                continue;
+            }
+
+            /* Fetching brick order from source dict */
+            snprintf(snapbrckord, sizeof(snapbrckord) - 1,
+                     "vol%" PRId64 ".brick%" PRId64 ".order", i + 1, j);
+            ret = dict_get_int64(src, snapbrckord, &brick_order);
+            if (ret) {
+                gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+                       "Failed to get brick order");
+                goto out;
+            }
+
+            snprintf(key, sizeof(key), "vol%" PRId64 ".brickdir%" PRId64, i + 1,
+                     brick_order);
+            ret = dict_set_dynstr_with_alloc(dst, key, snap_brick_dir);
+            if (ret) {
+                gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+                       "Failed to set %s", key);
+                goto out;
+            }
+
+            keylen = snprintf(key, sizeof(key), "vol%" PRId64 ".fstype%" PRId64,
+                              i + 1, j);
+            ret = dict_get_strn(src, key, keylen, &value);
+            if (ret) {
+                gf_msg(this->name, GF_LOG_WARNING, 0, GD_MSG_DICT_GET_FAILED,
+                       "Unable to fetch %s", key);
+                continue;
+            }
+
+            snprintf(key, sizeof(key), "vol%" PRId64 ".fstype%" PRId64, i + 1,
+                     brick_order);
+            ret = dict_set_dynstr_with_alloc(dst, key, value);
+            if (ret) {
+                gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+                       "Failed to set %s", key);
+                goto out;
+            }
+
+            keylen = snprintf(key, sizeof(key),
+                              "vol%" PRId64 ".mnt_opts%" PRId64, i + 1, j);
+            ret = dict_get_strn(src, key, keylen, &value);
+            if (ret) {
+                gf_msg(this->name, GF_LOG_WARNING, 0, GD_MSG_DICT_GET_FAILED,
+                       "Unable to fetch %s", key);
+                continue;
+            }
+
+            snprintf(key, sizeof(key), "vol%" PRId64 ".mnt_opts%" PRId64, i + 1,
+                     brick_order);
+            ret = dict_set_dynstr_with_alloc(dst, key, value);
+            if (ret) {
+                gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+                       "Failed to set %s", key);
+                goto out;
+            }
+
+            snprintf(key, sizeof(key),
+                     "vol%" PRId64 ".brick_snapdevice%" PRId64, i + 1, j);
+            ret = dict_get_ptr(src, key, (void **)&snap_device);
+            if (ret) {
+                gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+                       "Unable to fetch snap_device");
+                goto out;
+            }
+
+            snprintf(key, sizeof(key),
+                     "vol%" PRId64 ".brick_snapdevice%" PRId64, i + 1,
+                     brick_order);
+            ret = dict_set_dynstr_with_alloc(dst, key, snap_device);
+            if (ret) {
+                gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+                       "Failed to set %s", key);
+                goto out;
+            }
+
+            keylen = snprintf(key, sizeof(key),
+                              "vol%" PRId64 ".brick%" PRId64 ".status", i + 1,
+                              brick_order);
+            ret = dict_get_int32n(src, key, keylen, &brick_online);
+            if (ret) {
+                gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+                       "failed to "
+                       "get the brick status");
+                goto out;
+            }
+
+            ret = dict_set_int32n(dst, key, keylen, brick_online);
+            if (ret) {
+                gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+                       "failed to "
+                       "set the brick status");
+                goto out;
+            }
+            brick_online = 0;
+        }
+    }
+    ret = 0;
 out:
 
-        gf_log (this->name, GF_LOG_TRACE, "Returning %d", ret);
-        return ret;
+    gf_msg_trace(this->name, 0, "Returning %d", ret);
+    return ret;
 }
 
 /* Aggregate brickinfo's of the snap volumes to be restored from */
 int32_t
-glusterd_snap_restore_use_rsp_dict (dict_t *dst, dict_t *src)
+glusterd_snap_restore_use_rsp_dict(dict_t *dst, dict_t *src)
 {
-        char           key[PATH_MAX]  = "";
-        char          *strvalue       = NULL;
-        int32_t        value          = -1;
-        int32_t        i              = -1;
-        int32_t        j              = -1;
-        int32_t        vol_count       = -1;
-        int32_t        brickcount    = -1;
-        int32_t        ret            = -1;
-        xlator_t      *this           = NULL;
-
-        this = THIS;
-        GF_ASSERT (this);
-
-        if (!dst || !src) {
-                gf_log (this->name, GF_LOG_ERROR, "Source or Destination "
-                        "dict is empty.");
-                goto out;
-        }
+    char key[64] = "";
+    int keylen;
+    char *strvalue = NULL;
+    int32_t value = -1;
+    int32_t i = -1;
+    int32_t j = -1;
+    int32_t vol_count = -1;
+    int32_t brickcount = -1;
+    int32_t ret = -1;
+    xlator_t *this = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+
+    if (!dst || !src) {
+        gf_msg(this->name, GF_LOG_ERROR, EINVAL, GD_MSG_INVALID_ENTRY,
+               "Source or Destination "
+               "dict is empty.");
+        goto out;
+    }
+
+    ret = dict_get_int32(src, "volcount", &vol_count);
+    if (ret) {
+        gf_msg_debug(this->name, 0, "No volumes");
+        ret = 0;
+        goto out;
+    }
 
-        ret = dict_get_int32 (src, "volcount", &vol_count);
+    for (i = 1; i <= vol_count; i++) {
+        keylen = snprintf(key, sizeof(key), "snap%d.brick_count", i);
+        ret = dict_get_int32n(src, key, keylen, &brickcount);
         if (ret) {
-                gf_log (this->name, GF_LOG_DEBUG, "No volumes");
-                ret = 0;
-                goto out;
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+                   "Failed to get %s", key);
+            goto out;
         }
 
-        for (i = 1; i <= vol_count; i++) {
-                snprintf (key, sizeof (key), "snap%d.brick_count", i);
-                ret = dict_get_int32 (src, key, &brickcount);
-                if (ret) {
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "Failed to get %s", key);
-                        goto out;
-                }
-
-                for (j = 1; j <= brickcount; j++) {
-                        snprintf (key, sizeof (key), "snap%d.brick%d.path",
-                                  i, j);
-                        ret = dict_get_str (src, key, &strvalue);
-                        if (ret) {
-                                /* The brickinfo will be present in
-                                 * another rsp_dict */
-                                gf_log (this->name, GF_LOG_DEBUG,
-                                        "%s not present", key);
-                                ret = 0;
-                                continue;
-                        }
-                        ret = dict_set_dynstr_with_alloc (dst, key, strvalue);
-                        if (ret) {
-                                gf_log (this->name, GF_LOG_DEBUG,
-                                        "Failed to set %s", key);
-                                goto out;
-                        }
-
-                        snprintf (key, sizeof (key),
-                                  "snap%d.brick%d.snap_status", i, j);
-                        ret = dict_get_int32 (src, key, &value);
-                        if (ret) {
-                                gf_log (this->name, GF_LOG_ERROR,
-                                        "Failed to get %s", key);
-                                goto out;
-                        }
-                        ret = dict_set_int32 (dst, key, value);
-                        if (ret) {
-                                gf_log (this->name, GF_LOG_ERROR,
-                                        "Failed to set %s", key);
-                                goto out;
-                        }
-
-                        snprintf (key, sizeof (key),
-                                  "snap%d.brick%d.device_path", i, j);
-                        ret = dict_get_str (src, key, &strvalue);
-                        if (ret) {
-                                gf_log (this->name, GF_LOG_ERROR,
-                                        "Failed to get %s", key);
-                                goto out;
-                        }
-                        ret = dict_set_dynstr_with_alloc (dst, key, strvalue);
-                        if (ret) {
-                                gf_log (this->name, GF_LOG_DEBUG,
-                                        "Failed to set %s", key);
-                                goto out;
-                        }
-
-                        snprintf (key, sizeof (key),
-                                  "snap%d.brick%d.fs_type", i, j);
-                        ret = dict_get_str (src, key, &strvalue);
-                        if (ret) {
-                                gf_log (this->name, GF_LOG_ERROR,
-                                        "Failed to get %s", key);
-                                goto out;
-                        }
-                        ret = dict_set_dynstr_with_alloc (dst, key, strvalue);
-                        if (ret) {
-                                gf_log (this->name, GF_LOG_DEBUG,
-                                        "Failed to set %s", key);
-                                goto out;
-                        }
-
-                        snprintf (key, sizeof (key),
-                                  "snap%d.brick%d.mnt_opts", i, j);
-                        ret = dict_get_str (src, key, &strvalue);
-                        if (ret) {
-                                gf_log (this->name, GF_LOG_ERROR,
-                                        "Failed to get %s", key);
-                                goto out;
-                        }
-                        ret = dict_set_dynstr_with_alloc (dst, key, strvalue);
-                        if (ret) {
-                                gf_log (this->name, GF_LOG_DEBUG,
-                                        "Failed to set %s", key);
-                                goto out;
-                        }
-                }
-        }
+        for (j = 1; j <= brickcount; j++) {
+            keylen = snprintf(key, sizeof(key), "snap%d.brick%d.path", i, j);
+            ret = dict_get_strn(src, key, keylen, &strvalue);
+            if (ret) {
+                /* The brickinfo will be present in
+                 * another rsp_dict */
+                gf_msg_debug(this->name, 0, "%s not present", key);
+                ret = 0;
+                continue;
+            }
+            ret = dict_set_dynstr_with_alloc(dst, key, strvalue);
+            if (ret) {
+                gf_msg_debug(this->name, 0, "Failed to set %s", key);
+                goto out;
+            }
+
+            keylen = snprintf(key, sizeof(key), "snap%d.brick%d.snap_status", i,
+                              j);
+            ret = dict_get_int32n(src, key, keylen, &value);
+            if (ret) {
+                gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+                       "Failed to get %s", key);
+                goto out;
+            }
+            ret = dict_set_int32n(dst, key, keylen, value);
+            if (ret) {
+                gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+                       "Failed to set %s", key);
+                goto out;
+            }
+
+            keylen = snprintf(key, sizeof(key), "snap%d.brick%d.device_path", i,
+                              j);
+            ret = dict_get_strn(src, key, keylen, &strvalue);
+            if (ret) {
+                gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+                       "Failed to get %s", key);
+                goto out;
+            }
+            ret = dict_set_dynstr_with_alloc(dst, key, strvalue);
+            if (ret) {
+                gf_msg_debug(this->name, 0, "Failed to set %s", key);
+                goto out;
+            }
+
+            keylen = snprintf(key, sizeof(key), "snap%d.brick%d.fs_type", i, j);
+            ret = dict_get_strn(src, key, keylen, &strvalue);
+            if (ret) {
+                gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+                       "Failed to get %s", key);
+                goto out;
+            }
+            ret = dict_set_dynstr_with_alloc(dst, key, strvalue);
+            if (ret) {
+                gf_msg_debug(this->name, 0, "Failed to set %s", key);
+                goto out;
+            }
+
+            keylen = snprintf(key, sizeof(key), "snap%d.brick%d.mnt_opts", i,
+                              j);
+            ret = dict_get_strn(src, key, keylen, &strvalue);
+            if (ret) {
+                gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+                       "Failed to get %s", key);
+                goto out;
+            }
+            ret = dict_set_dynstr_with_alloc(dst, key, strvalue);
+            if (ret) {
+                gf_msg_debug(this->name, 0, "Failed to set %s", key);
+                goto out;
+            }
+        }
+    }
 
 out:
-        gf_log (this->name, GF_LOG_TRACE, "Returning %d", ret);
-        return ret;
+    gf_msg_trace(this->name, 0, "Returning %d", ret);
+    return ret;
 }
 
 int
-glusterd_snap_pre_validate_use_rsp_dict (dict_t *dst, dict_t *src)
+glusterd_snap_pre_validate_use_rsp_dict(dict_t *dst, dict_t *src)
 {
-        int             ret             = -1;
-        int32_t         snap_command    = 0;
-        xlator_t       *this            = NULL;
-
-        this = THIS;
-        GF_ASSERT (this);
-
-        if (!dst || !src) {
-                gf_log (this->name, GF_LOG_ERROR, "Source or Destination "
-                        "dict is empty.");
-                goto out;
-        }
-
-        ret = dict_get_int32 (dst, "type", &snap_command);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "unable to get the type of "
-                        "the snapshot command");
-                goto out;
-        }
-
-        switch (snap_command) {
+    int ret = -1;
+    int32_t snap_command = 0;
+    xlator_t *this = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+
+    if (!dst || !src) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_INVALID_ENTRY,
+               "Source or Destination "
+               "dict is empty.");
+        goto out;
+    }
+
+    ret = dict_get_int32n(dst, "type", SLEN("type"), &snap_command);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+               "unable to get the type of "
+               "the snapshot command");
+        goto out;
+    }
+
+    switch (snap_command) {
         case GF_SNAP_OPTION_TYPE_CREATE:
         case GF_SNAP_OPTION_TYPE_CLONE:
-                ret = glusterd_snap_create_clone_pre_val_use_rsp_dict (dst,
-                                                                       src);
-                if (ret) {
-                        gf_log (this->name, GF_LOG_ERROR, "Unable to use "
-                                "rsp dict");
-                        goto out;
-                }
-                break;
+            ret = glusterd_snap_create_clone_pre_val_use_rsp_dict(dst, src);
+            if (ret) {
+                gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+                       "Unable to use "
+                       "rsp dict");
+                goto out;
+            }
+            break;
         case GF_SNAP_OPTION_TYPE_RESTORE:
-                ret = glusterd_snap_restore_use_rsp_dict (dst, src);
-                if (ret) {
-                        gf_log (this->name, GF_LOG_ERROR, "Unable to use "
-                                "rsp dict");
-                        goto out;
-                }
-                break;
+            ret = glusterd_snap_restore_use_rsp_dict(dst, src);
+            if (ret) {
+                gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_RSP_DICT_USE_FAIL,
+                       "Unable to use "
+                       "rsp dict");
+                goto out;
+            }
+            break;
         default:
-                break;
-        }
+            break;
+    }
 
-        ret = 0;
+    ret = 0;
 out:
-        gf_log (this->name, GF_LOG_DEBUG, "Returning %d", ret);
-        return ret;
+    gf_msg_debug(this->name, 0, "Returning %d", ret);
+    return ret;
 }
 
 int
-glusterd_add_brick_status_to_dict (dict_t *dict, glusterd_volinfo_t *volinfo,
-                                   glusterd_brickinfo_t *brickinfo,
-                                   char *key_prefix)
+glusterd_add_brick_status_to_dict(dict_t *dict, glusterd_volinfo_t *volinfo,
+                                  glusterd_brickinfo_t *brickinfo,
+                                  char *key_prefix)
 {
-        char                   pidfile[PATH_MAX]         = {0, };
-        int32_t                brick_online              = 0;
-        pid_t                  pid                       = 0;
-        xlator_t              *this                      = NULL;
-        glusterd_conf_t       *conf                      = NULL;
-        int                    ret                       = -1;
-
-        GF_ASSERT (dict);
-        GF_ASSERT (volinfo);
-        GF_ASSERT (brickinfo);
-
-        this = THIS;
-        GF_ASSERT (this);
-        conf = this->private;
-        GF_ASSERT (conf);
-
-        if (!key_prefix) {
-                gf_log (this->name, GF_LOG_ERROR, "key prefix is NULL");
-                goto out;
-        }
-
-        GLUSTERD_GET_BRICK_PIDFILE (pidfile, volinfo, brickinfo, conf);
-
-        brick_online = gf_is_service_running (pidfile, &pid);
-
-        ret = dict_set_int32 (dict, key_prefix, brick_online);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "Failed to set %s", key_prefix);
-                goto out;
-        }
-        brick_online = 0;
-
-        ret = 0;
+    char pidfile[PATH_MAX] = "";
+    int32_t brick_online = 0;
+    pid_t pid = 0;
+    xlator_t *this = NULL;
+    glusterd_conf_t *conf = NULL;
+    int ret = -1;
+
+    GF_ASSERT(dict);
+    GF_ASSERT(volinfo);
+    GF_ASSERT(brickinfo);
+
+    this = THIS;
+    GF_ASSERT(this);
+    conf = this->private;
+    GF_ASSERT(conf);
+
+    if (!key_prefix) {
+        gf_msg(this->name, GF_LOG_ERROR, EINVAL, GD_MSG_INVALID_ENTRY,
+               "key prefix is NULL");
+        goto out;
+    }
+
+    GLUSTERD_GET_BRICK_PIDFILE(pidfile, volinfo, brickinfo, conf);
+
+    brick_online = gf_is_service_running(pidfile, &pid);
+
+    ret = dict_set_int32(dict, key_prefix, brick_online);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+               "Failed to set %s", key_prefix);
+        goto out;
+    }
+    brick_online = 0;
+
+    ret = 0;
 
 out:
-        return ret;
+    return ret;
 }
 
 /* This function will check whether the given device
@@ -1775,1542 +1860,1719 @@ out:
  * @return              _gf_true if LV is thin else _gf_false
  */
 gf_boolean_t
-glusterd_is_thinp_brick (char *device)
+glusterd_is_thinp_brick(char *device, uint32_t *op_errno)
 {
-        int             ret                     = -1;
-        char            msg [1024]              = "";
-        char            pool_name [PATH_MAX]    = "";
-        char           *ptr                     = NULL;
-        xlator_t       *this                    = NULL;
-        runner_t        runner                  = {0,};
-        gf_boolean_t    is_thin                 = _gf_false;
-
-        this = THIS;
-
-        GF_VALIDATE_OR_GOTO ("glusterd", this, out);
-        GF_VALIDATE_OR_GOTO (this->name, device, out);
-
-        snprintf (msg, sizeof (msg), "Get thin pool name for device %s",
-                  device);
-
-        runinit (&runner);
-
-        runner_add_args (&runner, "/sbin/lvs", "--noheadings", "-o", "pool_lv",
-                         device, NULL);
-        runner_redir (&runner, STDOUT_FILENO, RUN_PIPE);
-        runner_log (&runner, this->name, GF_LOG_DEBUG, msg);
-
-        ret = runner_start (&runner);
-        if (ret == -1) {
-                gf_log (this->name, GF_LOG_ERROR, "Failed to get thin pool "
-                        "name for device %s", device);
-                runner_end (&runner);
-                goto out;
-        }
-
-        ptr = fgets(pool_name, sizeof(pool_name),
-                    runner_chio (&runner, STDOUT_FILENO));
-        if (!ptr || !strlen(pool_name)) {
-                gf_log (this->name, GF_LOG_ERROR, "Failed to get pool name "
-                        "for device %s", device);
-                runner_end (&runner);
-                ret = -1;
-                goto out;
-        }
-
-        runner_end (&runner);
-
-        /* Trim all the whitespaces. */
-        ptr = gf_trim (pool_name);
-
-        /* If the LV has thin pool associated with this
-         * then it is a thinly provisioned LV else it is
-         * regular LV */
-        if (0 != ptr [0]) {
-                is_thin = _gf_true;
-        }
+    int ret = -1;
+    char msg[1024] = "";
+    char pool_name[PATH_MAX] = "";
+    char *ptr = NULL;
+    xlator_t *this = NULL;
+    runner_t runner = {
+        0,
+    };
+    gf_boolean_t is_thin = _gf_false;
+
+    this = THIS;
+
+    GF_VALIDATE_OR_GOTO("glusterd", this, out);
+    GF_VALIDATE_OR_GOTO(this->name, device, out);
+    GF_VALIDATE_OR_GOTO(this->name, op_errno, out);
+
+    snprintf(msg, sizeof(msg), "Get thin pool name for device %s", device);
+
+    runinit(&runner);
+
+    runner_add_args(&runner, "lvs", "--noheadings", "-o", "pool_lv", device,
+                    NULL);
+    runner_redir(&runner, STDOUT_FILENO, RUN_PIPE);
+    runner_log(&runner, this->name, GF_LOG_DEBUG, msg);
+
+    ret = runner_start(&runner);
+    if (ret == -1) {
+        gf_msg(this->name, GF_LOG_ERROR, errno, GD_MSG_TPOOL_GET_FAIL,
+               "Failed to get thin pool "
+               "name for device %s",
+               device);
+        runner_end(&runner);
+        goto out;
+    }
+
+    ptr = fgets(pool_name, sizeof(pool_name),
+                runner_chio(&runner, STDOUT_FILENO));
+    if (!ptr || !strlen(pool_name)) {
+        gf_msg(this->name, GF_LOG_ERROR, errno, GD_MSG_TPOOL_GET_FAIL,
+               "Failed to get pool name "
+               "for device %s",
+               device);
+        runner_end(&runner);
+        ret = -1;
+        goto out;
+    }
+
+    runner_end(&runner);
+
+    /* Trim all the whitespaces. */
+    ptr = gf_trim(pool_name);
+
+    /* If the LV has thin pool associated with this
+     * then it is a thinly provisioned LV else it is
+     * regular LV */
+    if (0 != ptr[0]) {
+        is_thin = _gf_true;
+    }
 
 out:
-        return is_thin;
+    if (!is_thin)
+        *op_errno = EG_NOTTHINP;
+
+    return is_thin;
 }
 
 int
-glusterd_snap_create_clone_common_prevalidate (dict_t *rsp_dict, int flags,
-            char *snapname, char *err_str, char *snap_volname, int64_t volcount,
-            glusterd_volinfo_t *volinfo, gf_loglevel_t *loglevel, int clone)  {
-
-        char                  *device            = NULL;
-        char                   key[PATH_MAX]     = "";
-        int                    ret               = -1;
-        int64_t                i                 = 1;
-        int64_t                brick_order       = 0;
-        int64_t                brick_count       = 0;
-        xlator_t              *this              = NULL;
-        glusterd_conf_t       *conf              = NULL;
-        glusterd_brickinfo_t  *brickinfo         = NULL;
-
-        this = THIS;
-        conf = this->private;
-        GF_ASSERT (conf);
-
-        if (!snapname || !volinfo) {
-                gf_log (this->name, GF_LOG_ERROR, "Failed to validate "
-                        "snapname or volume information");
+glusterd_snap_create_clone_common_prevalidate(
+    dict_t *rsp_dict, int flags, char *snapname, char *err_str,
+    char *snap_volname, int64_t volcount, glusterd_volinfo_t *volinfo,
+    gf_loglevel_t *loglevel, int clone, uint32_t *op_errno)
+{
+    char *device = NULL;
+    char *orig_device = NULL;
+    char key[128] = "";
+    int ret = -1;
+    int64_t i = 1;
+    int64_t brick_order = 0;
+    int64_t brick_count = 0;
+    xlator_t *this = NULL;
+    glusterd_conf_t *conf = NULL;
+    glusterd_brickinfo_t *brickinfo = NULL;
+    int32_t len = 0;
+
+    this = THIS;
+    conf = this->private;
+    GF_ASSERT(conf);
+    GF_VALIDATE_OR_GOTO(this->name, op_errno, out);
+
+    if (!snapname || !volinfo) {
+        gf_msg(this->name, GF_LOG_ERROR, EINVAL, GD_MSG_INVALID_ENTRY,
+               "Failed to validate "
+               "snapname or volume information");
+        ret = -1;
+        goto out;
+    }
+
+    cds_list_for_each_entry(brickinfo, &volinfo->bricks, brick_list)
+    {
+        if (gf_uuid_compare(brickinfo->uuid, MY_UUID)) {
+            brick_order++;
+            continue;
+        }
+
+        if (!glusterd_is_brick_started(brickinfo)) {
+            if (!clone && (flags & GF_CLI_FLAG_OP_FORCE)) {
+                gf_msg(this->name, GF_LOG_WARNING, 0, GD_MSG_BRICK_DISCONNECTED,
+                       "brick %s:%s is not started", brickinfo->hostname,
+                       brickinfo->path);
+                brick_order++;
+                brick_count++;
+                continue;
+            }
+            if (!clone) {
+                snprintf(err_str, PATH_MAX,
+                         "One or more bricks are not running. "
+                         "Please run volume status command to see "
+                         "brick status.\n"
+                         "Please start the stopped brick "
+                         "and then issue snapshot create "
+                         "command or use [force] option in "
+                         "snapshot create to override this "
+                         "behavior.");
+                gf_smsg(this->name, GF_LOG_ERROR, errno,
+                        GD_MSG_BRICK_NOT_RUNNING,
+                        "Please run volume status command to see brick "
+                        "status.Please start the stopped brick and then issue "
+                        "snapshot create command or use 'force' option in "
+                        "snapshot create to override this behavior.",
+                        NULL);
+            } else {
+                snprintf(err_str, PATH_MAX,
+                         "One or more bricks are not running. "
+                         "Please run snapshot status command to see "
+                         "brick status.\n"
+                         "Please start the stopped brick "
+                         "and then issue snapshot clone "
+                         "command ");
+                gf_smsg(this->name, GF_LOG_ERROR, errno,
+                        GD_MSG_BRICK_NOT_RUNNING,
+                        "Please run snapshot status command to see brick "
+                        "status. Please start the stopped brick and then issue "
+                        "snapshot clone command.",
+                        NULL);
+            }
+            *op_errno = EG_BRCKDWN;
+            ret = -1;
+            goto out;
+        }
+
+        orig_device = glusterd_get_brick_mount_device(brickinfo->path);
+        if (!orig_device) {
+            len = snprintf(err_str, PATH_MAX,
+                           "getting device name for the brick "
+                           "%s:%s failed",
+                           brickinfo->hostname, brickinfo->path);
+            if (len < 0) {
+                strcpy(err_str, "<error>");
+            }
+            gf_smsg(this->name, GF_LOG_ERROR, errno,
+                    GD_MSG_BRK_MNTPATH_GET_FAIL,
+                    "Brick_hostname=%s, Brick_path=%s", brickinfo->hostname,
+                    brickinfo->path, NULL);
+            ret = -1;
+            goto out;
+        }
+        if (!clone) {
+            if (!glusterd_is_thinp_brick(orig_device, op_errno)) {
+                snprintf(err_str, PATH_MAX,
+                         "Snapshot is supported only for "
+                         "thin provisioned LV. Ensure that "
+                         "all bricks of %s are thinly "
+                         "provisioned LV.",
+                         volinfo->volname);
+                gf_smsg(this->name, GF_LOG_ERROR, errno,
+                        GD_MSG_SNAPSHOT_NOT_THIN_PROVISIONED,
+                        "Ensure that all bricks of volume are thinly "
+                        "provisioned LV, Volume=%s",
+                        volinfo->volname, NULL);
                 ret = -1;
                 goto out;
+            }
         }
 
-        cds_list_for_each_entry (brickinfo, &volinfo->bricks,
-                                 brick_list) {
-                if (gf_uuid_compare (brickinfo->uuid, MY_UUID)) {
-                        brick_order++;
-                        continue;
-                }
-
-                if (!glusterd_is_brick_started (brickinfo)) {
-                        if (!clone && (flags & GF_CLI_FLAG_OP_FORCE)) {
-                                gf_log (this->name, GF_LOG_WARNING,
-                                        "brick %s:%s is not started",
-                                        brickinfo->hostname,
-                                        brickinfo->path);
-                                brick_order++;
-                                brick_count++;
-                                continue;
-                        }
-                        if (!clone) {
-                        snprintf (err_str, PATH_MAX,
-                                  "One or more bricks are not running. "
-                                  "Please run volume status command to see "
-                                  "brick status.\n"
-                                  "Please start the stopped brick "
-                                  "and then issue snapshot create "
-                                  "command or use [force] option in "
-                                  "snapshot create to override this "
-                                  "behavior.");
-                        } else {
-                        snprintf (err_str, PATH_MAX,
-                                  "One or more bricks are not running. "
-                                  "Please run snapshot status command to see "
-                                  "brick status.\n"
-                                  "Please start the stopped brick "
-                                  "and then issue snapshot clone "
-                                  "command ");
-                        }
-                        ret = -1;
-                        goto out;
-                }
-
-
-                device = glusterd_get_brick_mount_device
-                                          (brickinfo->path);
-                if (!device) {
-                        snprintf (err_str, PATH_MAX,
-                                  "getting device name for the brick "
-                                 "%s:%s failed", brickinfo->hostname,
-                                  brickinfo->path);
-                        ret = -1;
-                        goto out;
-                }
-                if (!clone) {
-                     if (!glusterd_is_thinp_brick (device)) {
-                                snprintf (err_str, PATH_MAX,
-                                          "Snapshot is supported only for "
-                                          "thin provisioned LV. Ensure that "
-                                          "all bricks of %s are thinly "
-                                          "provisioned LV.", volinfo->volname);
-                                ret = -1;
-                                goto out;
-                     }
-                }
-
-                device = glusterd_build_snap_device_path (device,
-                                                  snap_volname,
-                                                  brick_count);
-                if (!device) {
-                        snprintf (err_str, PATH_MAX,
-                                  "cannot copy the snapshot device "
-                                "name (volname: %s, snapname: %s)",
-                                volinfo->volname, snapname);
-                        *loglevel = GF_LOG_WARNING;
-                        ret = -1;
-                        goto out;
-                }
-
-                snprintf (key, sizeof(key),
-                        "vol%"PRId64".brick_snapdevice%"PRId64,
-                        i, brick_count);
-                ret = dict_set_dynstr (rsp_dict, key, device);
-                if (ret) {
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "Failed to set %s", key);
-                        GF_FREE (device);
-                        goto out;
-                }
-                device = NULL;
-
-                ret = glusterd_update_mntopts (brickinfo->path,
-                                               brickinfo);
-                if (ret) {
-                         gf_log (this->name, GF_LOG_ERROR, "Failed to "
-                                "update mount options for %s brick",
-                                brickinfo->path);
-                }
-
-                snprintf (key, sizeof(key), "vol%"PRId64".fstype%"
-                          PRId64, i, brick_count);
-                ret = dict_set_dynstr_with_alloc (rsp_dict, key,
-                                                  brickinfo->fstype);
-                if (ret) {
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "Failed to set %s", key);
-                        goto out;
-                }
-
-                snprintf (key, sizeof(key), "vol%"PRId64".mnt_opts%"
-                          PRId64, i, brick_count);
-                ret = dict_set_dynstr_with_alloc (rsp_dict, key,
-                                                  brickinfo->mnt_opts);
-                if (ret) {
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "Failed to set %s", key);
-                        goto out;
-                }
-
-                snprintf (key, sizeof(key), "vol%"PRId64".brickdir%"PRId64, i,
-                          brick_count);
-                ret = dict_set_dynstr_with_alloc (rsp_dict, key,
-                                                  brickinfo->mount_dir);
-                if (ret) {
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "Failed to set %s", key);
-                        goto out;
-                }
-
-                snprintf (key, sizeof(key) - 1,
-                          "vol%"PRId64".brick%"PRId64".order", i, brick_count);
-                ret = dict_set_int64 (rsp_dict, key, brick_order);
-                if (ret) {
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "Failed to set %s", key);
-                        goto out;
-                }
+        device = glusterd_build_snap_device_path(orig_device, snap_volname,
+                                                 brick_count);
+        if (!device) {
+            snprintf(err_str, PATH_MAX,
+                     "cannot copy the snapshot device "
+                     "name (volname: %s, snapname: %s)",
+                     volinfo->volname, snapname);
+            gf_smsg(this->name, GF_LOG_ERROR, errno,
+                    GD_MSG_SNAP_DEVICE_NAME_GET_FAIL, "Volname=%s, Snapname=%s",
+                    volinfo->volname, snapname, NULL);
+            *loglevel = GF_LOG_WARNING;
+            ret = -1;
+            goto out;
+        }
 
-                snprintf (key, sizeof (key),
-                          "vol%"PRId64".brick%"PRId64".status", i, brick_order);
+        GF_FREE(orig_device);
+        orig_device = NULL;
 
-                ret = glusterd_add_brick_status_to_dict (rsp_dict,
-                                                         volinfo,
-                                                         brickinfo,
-                                                         key);
-                if (ret) {
-                        gf_log (this->name, GF_LOG_ERROR, "failed to "
-                                "add brick status to dict");
-                        goto out;
-                }
-                brick_count++;
-                brick_order++;
-        }
-        snprintf (key, sizeof(key) - 1, "vol%"PRId64"_brickcount", volcount);
-        ret = dict_set_int64 (rsp_dict, key, brick_count);
+        snprintf(key, sizeof(key), "vol%" PRId64 ".brick_snapdevice%" PRId64, i,
+                 brick_count);
+        ret = dict_set_dynstr_with_alloc(rsp_dict, key, device);
         if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "Failed to set %s",
-                        key);
-                goto out;
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+                   "Failed to set %s", key);
+            goto out;
         }
-        ret = 0;
-out:
-        if (device)
-               GF_FREE (device);
-
-       return ret;
-
-}
-
 
-int
-glusterd_snapshot_clone_prevalidate (dict_t *dict, char **op_errstr,
-                                      dict_t *rsp_dict)
-{
-        char                  *clonename         = NULL;
-        char                  *snapname          = NULL;
-        char                   key[PATH_MAX]     = "";
-        glusterd_snap_t       *snap              = NULL;
-        char                   err_str[PATH_MAX] = "";
-        int                    ret               = -1;
-        int64_t                volcount          = 1;
-        glusterd_volinfo_t    *snap_vol          = NULL;
-        xlator_t              *this              = NULL;
-        uuid_t                *snap_volid        = NULL;
-        gf_loglevel_t          loglevel          = GF_LOG_ERROR;
-
-        this = THIS;
-        GF_ASSERT (op_errstr);
-        GF_ASSERT (dict);
-
-        ret = dict_get_str (dict, "clonename", &clonename);
+        ret = glusterd_update_mntopts(brickinfo->path, brickinfo);
         if (ret) {
-                snprintf (err_str, sizeof (err_str), "Failed to "
-                          "get the clone name");
-                goto out;
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_BRK_MOUNTOPTS_FAIL,
+                   "Failed to "
+                   "update mount options for %s brick",
+                   brickinfo->path);
         }
 
-        ret = dict_get_str (dict, "snapname", &snapname);
+        snprintf(key, sizeof(key), "vol%" PRId64 ".fstype%" PRId64, i,
+                 brick_count);
+        ret = dict_set_dynstr_with_alloc(rsp_dict, key, brickinfo->fstype);
         if (ret) {
-                snprintf (err_str, sizeof (err_str), "Failed to get snapname");
-                goto out;
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+                   "Failed to set %s", key);
+            goto out;
         }
 
-        if (glusterd_check_volume_exists(clonename)) {
-                ret = -1;
-                snprintf (err_str, sizeof (err_str), "Volume with name:%s "
-                          "already exists", clonename);
-                goto out;
-        }
-        /* need to find snap volinfo*/
-        snap = glusterd_find_snap_by_name (snapname);
-        if (!snap) {
-                ret = -1;
-                snprintf (err_str, sizeof (err_str), "Failed to find :%s "
-                          "snap", snapname);
-                goto out;
+        snprintf(key, sizeof(key), "vol%" PRId64 ".mnt_opts%" PRId64, i,
+                 brick_count);
+        ret = dict_set_dynstr_with_alloc(rsp_dict, key, brickinfo->mnt_opts);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+                   "Failed to set %s", key);
+            goto out;
         }
 
-        /* TODO : As of now there is only one volume in snapshot.
-         * Change this when multiple volume snapshot is introduced
-         */
-        snap_vol = list_entry (snap->volumes.next,
-                                   glusterd_volinfo_t, vol_list);
-        if (!snap_vol) {
-                gf_log ("glusterd", GF_LOG_ERROR, "Failed to get snap "
-                "volinfo %s", snap->snapname);
-                goto out;
+        snprintf(key, sizeof(key), "vol%" PRId64 ".brickdir%" PRId64, i,
+                 brick_count);
+        ret = dict_set_dynstr_with_alloc(rsp_dict, key, brickinfo->mount_dir);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+                   "Failed to set %s", key);
+            goto out;
         }
 
-        snprintf (key, sizeof(key) - 1, "vol1_volid");
-        ret = dict_get_bin (dict, key, (void **)&snap_volid);
+        snprintf(key, sizeof(key) - 1, "vol%" PRId64 ".brick%" PRId64 ".order",
+                 i, brick_count);
+        ret = dict_set_int64(rsp_dict, key, brick_order);
         if (ret) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "Unable to fetch snap_volid");
-                goto out;
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+                   "Failed to set %s", key);
+            goto out;
         }
 
+        snprintf(key, sizeof(key), "vol%" PRId64 ".brick%" PRId64 ".status", i,
+                 brick_order);
 
-        /* Adding snap bricks mount paths to the dict */
-        ret = glusterd_snap_create_clone_common_prevalidate (rsp_dict, 0,
-                        snapname, err_str, clonename, 1, snap_vol,
-                        &loglevel, 1);
+        ret = glusterd_add_brick_status_to_dict(rsp_dict, volinfo, brickinfo,
+                                                key);
         if (ret) {
-                goto out;
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+                   "failed to "
+                   "add brick status to dict");
+            goto out;
         }
-        ret = dict_set_int64 (rsp_dict, "volcount", volcount);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "Failed to set volcount");
-                goto out;
+        brick_count++;
+        brick_order++;
+        if (device) {
+            GF_FREE(device);
+            device = NULL;
         }
-
-        ret = 0;
+    }
+    snprintf(key, sizeof(key) - 1, "vol%" PRId64 "_brickcount", volcount);
+    ret = dict_set_int64(rsp_dict, key, brick_count);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+               "Failed to set %s", key);
+        goto out;
+    }
+    ret = 0;
 out:
+    if (orig_device)
+        GF_FREE(orig_device);
 
-        if (ret && err_str[0] != '\0') {
-                gf_log (this->name, loglevel, "%s", err_str);
-                *op_errstr = gf_strdup (err_str);
-        }
+    if (device)
+        GF_FREE(device);
 
-        gf_log (this->name, GF_LOG_TRACE, "Returning %d", ret);
-        return ret;
+    return ret;
 }
 
+int
+glusterd_snapshot_clone_prevalidate(dict_t *dict, char **op_errstr,
+                                    dict_t *rsp_dict, uint32_t *op_errno)
+{
+    char *clonename = NULL;
+    char *snapname = NULL;
+    char device_name[64] = "";
+    glusterd_snap_t *snap = NULL;
+    char err_str[PATH_MAX] = "";
+    int ret = -1;
+    int64_t volcount = 1;
+    glusterd_volinfo_t *snap_vol = NULL;
+    xlator_t *this = NULL;
+    uuid_t *snap_volid = NULL;
+    gf_loglevel_t loglevel = GF_LOG_ERROR;
+    glusterd_volinfo_t *volinfo = NULL;
+
+    this = THIS;
+    GF_ASSERT(op_errstr);
+    GF_ASSERT(dict);
+    GF_VALIDATE_OR_GOTO(this->name, op_errno, out);
+
+    ret = dict_get_strn(dict, "clonename", SLEN("clonename"), &clonename);
+    if (ret) {
+        snprintf(err_str, sizeof(err_str),
+                 "Failed to "
+                 "get the clone name");
+        goto out;
+    }
+
+    ret = dict_get_strn(dict, "snapname", SLEN("snapname"), &snapname);
+    if (ret) {
+        snprintf(err_str, sizeof(err_str), "Failed to get snapname");
+        goto out;
+    }
+
+    ret = glusterd_volinfo_find(clonename, &volinfo);
+    if (!ret) {
+        ret = -1;
+        snprintf(err_str, sizeof(err_str),
+                 "Volume with name:%s "
+                 "already exists",
+                 clonename);
+        *op_errno = EG_VOLEXST;
+        goto out;
+    }
+    /* need to find snap volinfo*/
+    snap = glusterd_find_snap_by_name(snapname);
+    if (!snap) {
+        ret = -1;
+        snprintf(err_str, sizeof(err_str),
+                 "Failed to find :%s "
+                 "snap",
+                 snapname);
+        goto out;
+    }
+
+    /* TODO : As of now there is only one volume in snapshot.
+     * Change this when multiple volume snapshot is introduced
+     */
+    snap_vol = list_entry(snap->volumes.next, glusterd_volinfo_t, vol_list);
+    if (!snap_vol) {
+        gf_msg("glusterd", GF_LOG_ERROR, 0, GD_MSG_VOLINFO_GET_FAIL,
+               "Failed to get snap "
+               "volinfo %s",
+               snap->snapname);
+        goto out;
+    }
+
+    if (!glusterd_is_volume_started(snap_vol)) {
+        snprintf(err_str, sizeof(err_str),
+                 "Snapshot %s is "
+                 "not activated",
+                 snap->snapname);
+        loglevel = GF_LOG_WARNING;
+        *op_errno = EG_VOLSTP;
+        goto out;
+    }
+
+    ret = dict_get_bin(dict, "vol1_volid", (void **)&snap_volid);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+               "Unable to fetch snap_volid");
+        goto out;
+    }
+
+    GLUSTERD_GET_UUID_NOHYPHEN(device_name, *snap_volid);
+
+    /* Adding snap bricks mount paths to the dict */
+    ret = glusterd_snap_create_clone_common_prevalidate(
+        rsp_dict, 0, snapname, err_str, device_name, 1, snap_vol, &loglevel, 1,
+        op_errno);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_PRE_VALIDATION_FAIL,
+               "Failed to pre validate");
+        goto out;
+    }
+
+    ret = dict_set_int64(rsp_dict, "volcount", volcount);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+               "Failed to set volcount");
+        goto out;
+    }
+
+out:
+
+    if (ret && err_str[0] != '\0') {
+        gf_msg(this->name, loglevel, 0, GD_MSG_SNAP_CLONE_PREVAL_FAILED, "%s",
+               err_str);
+        *op_errstr = gf_strdup(err_str);
+    }
+
+    gf_msg_trace(this->name, 0, "Returning %d", ret);
+    return ret;
+}
 
 int
-glusterd_snapshot_create_prevalidate (dict_t *dict, char **op_errstr,
-                                      dict_t *rsp_dict)
+glusterd_snapshot_create_prevalidate(dict_t *dict, char **op_errstr,
+                                     dict_t *rsp_dict, uint32_t *op_errno)
 {
-        char                  *volname           = NULL;
-        char                  *snapname          = NULL;
-        char                   key[PATH_MAX]     = "";
-        char                   snap_volname[64]  = "";
-        char                   err_str[PATH_MAX] = "";
-        int                    ret               = -1;
-        int64_t                i                 = 0;
-        int64_t                volcount          = 0;
-        glusterd_volinfo_t    *volinfo           = NULL;
-        xlator_t              *this              = NULL;
-        uuid_t                *snap_volid        = NULL;
-        gf_loglevel_t          loglevel          = GF_LOG_ERROR;
-        glusterd_conf_t       *conf              = NULL;
-        int64_t                effective_max_limit = 0;
-        int                    flags             = 0;
-        uint64_t               opt_hard_max      = GLUSTERD_SNAPS_MAX_HARD_LIMIT;
-
-        this = THIS;
-        GF_ASSERT (op_errstr);
-        conf = this->private;
-        GF_ASSERT (conf);
-
-        ret = dict_get_int64 (dict, "volcount", &volcount);
-        if (ret) {
-                snprintf (err_str, sizeof (err_str), "Failed to "
-                          "get the volume count");
-                goto out;
-        }
-        if (volcount <= 0) {
-                snprintf (err_str, sizeof (err_str),
-                          "Invalid volume count %"PRId64" supplied", volcount);
-                ret = -1;
-                goto out;
+    char *volname = NULL;
+    char *snapname = NULL;
+    char key[64] = "";
+    int keylen;
+    char snap_volname[64] = "";
+    char err_str[PATH_MAX] = "";
+    int ret = -1;
+    int64_t i = 0;
+    int64_t volcount = 0;
+    glusterd_volinfo_t *volinfo = NULL;
+    xlator_t *this = NULL;
+    uuid_t *snap_volid = NULL;
+    gf_loglevel_t loglevel = GF_LOG_ERROR;
+    glusterd_conf_t *conf = NULL;
+    int64_t effective_max_limit = 0;
+    int flags = 0;
+    uint64_t opt_hard_max = GLUSTERD_SNAPS_MAX_HARD_LIMIT;
+    char *description = NULL;
+
+    this = THIS;
+    GF_ASSERT(op_errstr);
+    conf = this->private;
+    GF_ASSERT(conf);
+    GF_VALIDATE_OR_GOTO(this->name, op_errno, out);
+
+    ret = dict_get_int64(dict, "volcount", &volcount);
+    if (ret) {
+        snprintf(err_str, sizeof(err_str),
+                 "Failed to "
+                 "get the volume count");
+        goto out;
+    }
+    if (volcount <= 0) {
+        snprintf(err_str, sizeof(err_str),
+                 "Invalid volume count %" PRId64 " supplied", volcount);
+        ret = -1;
+        goto out;
+    }
+
+    ret = dict_get_strn(dict, "snapname", SLEN("snapname"), &snapname);
+    if (ret) {
+        snprintf(err_str, sizeof(err_str), "Failed to get snapname");
+        goto out;
+    }
+
+    ret = dict_get_strn(dict, "description", SLEN("description"), &description);
+    if (description && !(*description)) {
+        /* description should have a non-null value */
+        ret = -1;
+        snprintf(err_str, sizeof(err_str),
+                 "Snapshot cannot be "
+                 "created with empty description");
+        goto out;
+    }
+
+    ret = dict_get_int32n(dict, "flags", SLEN("flags"), &flags);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+               "Unable to get flags");
+        goto out;
+    }
+
+    if (glusterd_find_snap_by_name(snapname)) {
+        ret = -1;
+        snprintf(err_str, sizeof(err_str),
+                 "Snapshot %s already "
+                 "exists",
+                 snapname);
+        *op_errno = EG_SNAPEXST;
+        goto out;
+    }
+
+    for (i = 1; i <= volcount; i++) {
+        keylen = snprintf(key, sizeof(key), "volname%" PRId64, i);
+        ret = dict_get_strn(dict, key, keylen, &volname);
+        if (ret) {
+            snprintf(err_str, sizeof(err_str), "failed to get volume name");
+            goto out;
+        }
+        ret = glusterd_volinfo_find(volname, &volinfo);
+        if (ret) {
+            snprintf(err_str, sizeof(err_str), "Volume (%s) does not exist ",
+                     volname);
+            *op_errno = EG_NOVOL;
+            goto out;
+        }
+
+        ret = -1;
+        if (!glusterd_is_volume_started(volinfo)) {
+            snprintf(err_str, sizeof(err_str),
+                     "volume %s is "
+                     "not started",
+                     volinfo->volname);
+            loglevel = GF_LOG_WARNING;
+            *op_errno = EG_VOLSTP;
+            goto out;
+        }
+
+        if (glusterd_is_defrag_on(volinfo)) {
+            snprintf(err_str, sizeof(err_str),
+                     "rebalance process is running for the "
+                     "volume %s",
+                     volname);
+            loglevel = GF_LOG_WARNING;
+            *op_errno = EG_RBALRUN;
+            goto out;
+        }
+
+        if (gd_vol_is_geo_rep_active(volinfo)) {
+            snprintf(err_str, sizeof(err_str),
+                     "geo-replication session is running for "
+                     "the volume %s. Session needs to be "
+                     "stopped before taking a snapshot.",
+                     volname);
+            loglevel = GF_LOG_WARNING;
+            *op_errno = EG_GEOREPRUN;
+            goto out;
+        }
+
+        if (volinfo->is_snap_volume == _gf_true) {
+            snprintf(err_str, sizeof(err_str), "Volume %s is a snap volume",
+                     volname);
+            loglevel = GF_LOG_WARNING;
+            *op_errno = EG_ISSNAP;
+            goto out;
         }
 
-        ret = dict_get_str (dict, "snapname", &snapname);
+        /* "snap-max-hard-limit" might not be set by user explicitly,
+         * in that case it's better to consider the default value.
+         * Hence not erroring out if Key is not found.
+         */
+        ret = dict_get_uint64(
+            conf->opts, GLUSTERD_STORE_KEY_SNAP_MAX_HARD_LIMIT, &opt_hard_max);
         if (ret) {
-                snprintf (err_str, sizeof (err_str), "Failed to get snapname");
-                goto out;
+            ret = 0;
+            gf_msg_debug(this->name, 0,
+                         "%s is not present "
+                         "in opts dictionary",
+                         GLUSTERD_STORE_KEY_SNAP_MAX_HARD_LIMIT);
         }
 
-        ret = dict_get_int32 (dict, "flags", &flags);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "Unable to get flags");
-                goto out;
-        }
+        if (volinfo->snap_max_hard_limit < opt_hard_max)
+            effective_max_limit = volinfo->snap_max_hard_limit;
+        else
+            effective_max_limit = opt_hard_max;
 
-        if (glusterd_find_snap_by_name (snapname)) {
-                ret = -1;
-                snprintf (err_str, sizeof (err_str), "Snapshot %s already "
-                          "exists", snapname);
-                goto out;
+        if (volinfo->snap_count >= effective_max_limit) {
+            ret = -1;
+            snprintf(err_str, sizeof(err_str),
+                     "The number of existing snaps has reached "
+                     "the effective maximum limit of %" PRIu64
+                     ", "
+                     "for the volume (%s). Please delete few "
+                     "snapshots before taking further snapshots.",
+                     effective_max_limit, volname);
+            loglevel = GF_LOG_WARNING;
+            *op_errno = EG_HRDLMT;
+            goto out;
         }
 
-        for (i = 1; i <= volcount; i++) {
-                snprintf (key, sizeof (key), "volname%"PRId64, i);
-                ret = dict_get_str (dict, key, &volname);
-                if (ret) {
-                        snprintf (err_str, sizeof (err_str),
-                                  "failed to get volume name");
-                        goto out;
-                }
-                ret = glusterd_volinfo_find (volname, &volinfo);
-                if (ret) {
-                        snprintf (err_str, sizeof (err_str),
-                                  "Volume (%s) does not exist ", volname);
-                        goto out;
-                }
-
-                ret = -1;
-                if (!glusterd_is_volume_started (volinfo)) {
-                        snprintf (err_str, sizeof (err_str), "volume %s is "
-                                  "not started", volinfo->volname);
-                        loglevel = GF_LOG_WARNING;
-                        goto out;
-                }
-
-                if (glusterd_is_defrag_on (volinfo)) {
-                        snprintf (err_str, sizeof (err_str),
-                                  "rebalance process is running for the "
-                                  "volume %s", volname);
-                        loglevel = GF_LOG_WARNING;
-                        goto out;
-                }
-
-                if (gd_vol_is_geo_rep_active (volinfo)) {
-                         snprintf (err_str, sizeof (err_str),
-                                   "geo-replication session is running for "
-                                   "the volume %s. Session needs to be "
-                                   "stopped before taking a snapshot.",
-                                   volname);
-                         loglevel = GF_LOG_WARNING;
-                         goto out;
-                }
-
-                if (volinfo->is_snap_volume == _gf_true) {
-                        snprintf (err_str, sizeof (err_str),
-                                  "Volume %s is a snap volume", volname);
-                        loglevel = GF_LOG_WARNING;
-                        goto out;
-                }
-
-                /* "snap-max-hard-limit" might not be set by user explicitly,
-                 * in that case it's better to consider the default value.
-                 * Hence not erroring out if Key is not found.
-                 */
-                ret = dict_get_uint64 (conf->opts,
-                                      GLUSTERD_STORE_KEY_SNAP_MAX_HARD_LIMIT,
-                                      &opt_hard_max);
-                if (ret) {
-                        ret = 0;
-                        gf_log (this->name, GF_LOG_DEBUG, "%s is not present "
-                                "in opts dictionary",
-                                GLUSTERD_STORE_KEY_SNAP_MAX_HARD_LIMIT);
-                }
-
-                if (volinfo->snap_max_hard_limit < opt_hard_max)
-                        effective_max_limit = volinfo->snap_max_hard_limit;
-                else
-                        effective_max_limit = opt_hard_max;
-
-                if (volinfo->snap_count >= effective_max_limit) {
-                        ret = -1;
-                        snprintf (err_str, sizeof (err_str),
-                                  "The number of existing snaps has reached "
-                                  "the effective maximum limit of %"PRIu64", "
-                                  "for the volume (%s). Please delete few "
-                                  "snapshots before taking further snapshots.",
-                                  effective_max_limit, volname);
-                        loglevel = GF_LOG_WARNING;
-                        goto out;
-                }
-
-                snprintf (key, sizeof(key) - 1, "vol%"PRId64"_volid", i);
-                ret = dict_get_bin (dict, key, (void **)&snap_volid);
-                if (ret) {
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "Unable to fetch snap_volid");
-                        goto out;
-                }
-
-                /* snap volume uuid is used as lvm snapshot name.
-                   This will avoid restrictions on snapshot names
-                   provided by user */
-                GLUSTERD_GET_UUID_NOHYPHEN (snap_volname, *snap_volid);
-
-                ret = glusterd_snap_create_clone_common_prevalidate (rsp_dict,
-                                flags, snapname, err_str, snap_volname, i,
-                                volinfo, &loglevel, 0);
-                if (ret) {
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "Failed to pre validate");
-                        goto out;
-                }
-
+        snprintf(key, sizeof(key), "vol%" PRId64 "_volid", i);
+        ret = dict_get_bin(dict, key, (void **)&snap_volid);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+                   "Unable to fetch snap_volid");
+            goto out;
         }
 
-        ret = dict_set_int64 (rsp_dict, "volcount", volcount);
+        /* snap volume uuid is used as lvm snapshot name.
+           This will avoid restrictions on snapshot names
+           provided by user */
+        GLUSTERD_GET_UUID_NOHYPHEN(snap_volname, *snap_volid);
+
+        ret = glusterd_snap_create_clone_common_prevalidate(
+            rsp_dict, flags, snapname, err_str, snap_volname, i, volinfo,
+            &loglevel, 0, op_errno);
         if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "Failed to set volcount");
-                goto out;
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_PRE_VALIDATION_FAIL,
+                   "Failed to pre validate");
+            goto out;
         }
+    }
 
-        ret = 0;
-out:
-        if (ret && err_str[0] != '\0') {
-                gf_log (this->name, loglevel, "%s", err_str);
-                *op_errstr = gf_strdup (err_str);
-        }
+    ret = dict_set_int64(rsp_dict, "volcount", volcount);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+               "Failed to set volcount");
+        goto out;
+    }
 
-        gf_log (this->name, GF_LOG_TRACE, "Returning %d", ret);
-        return ret;
+    ret = 0;
+
+out:
+    if (ret && err_str[0] != '\0') {
+        gf_msg(this->name, loglevel, 0, GD_MSG_SNAPSHOT_OP_FAILED, "%s",
+               err_str);
+        *op_errstr = gf_strdup(err_str);
+    }
+
+    gf_msg_trace(this->name, 0, "Returning %d", ret);
+    return ret;
 }
 
-glusterd_snap_t*
+glusterd_snap_t *
 glusterd_new_snap_object()
 {
-        glusterd_snap_t        *snap    = NULL;
+    glusterd_snap_t *snap = NULL;
 
-        snap = GF_CALLOC (1, sizeof (*snap), gf_gld_mt_snap_t);
+    snap = GF_CALLOC(1, sizeof(*snap), gf_gld_mt_snap_t);
 
-        if (snap) {
-                if (LOCK_INIT (&snap->lock)) {
-                        gf_log (THIS->name, GF_LOG_ERROR, "Failed initiating"
-                                " snap lock");
-                        GF_FREE (snap);
-                        return NULL;
-                }
-
-                CDS_INIT_LIST_HEAD (&snap->snap_list);
-                CDS_INIT_LIST_HEAD (&snap->volumes);
-                snap->snapname[0] = 0;
-                snap->snap_status = GD_SNAP_STATUS_INIT;
+    if (snap) {
+        if (LOCK_INIT(&snap->lock)) {
+            gf_msg(THIS->name, GF_LOG_ERROR, 0, GD_MSG_LOCK_INIT_FAILED,
+                   "Failed initiating"
+                   " snap lock");
+            GF_FREE(snap);
+            return NULL;
         }
 
-        return snap;
+        CDS_INIT_LIST_HEAD(&snap->snap_list);
+        CDS_INIT_LIST_HEAD(&snap->volumes);
+        snap->snapname[0] = 0;
+        snap->snap_status = GD_SNAP_STATUS_INIT;
+    }
 
+    return snap;
 };
 
 /* Function glusterd_list_add_snapvol adds the volinfo object (snapshot volume)
    to the snapshot object list and to the parent volume list */
 int32_t
-glusterd_list_add_snapvol (glusterd_volinfo_t *origin_vol,
-                           glusterd_volinfo_t *snap_vol)
+glusterd_list_add_snapvol(glusterd_volinfo_t *origin_vol,
+                          glusterd_volinfo_t *snap_vol)
 {
-        int ret               = -1;
-        glusterd_snap_t *snap = NULL;
+    int ret = -1;
+    glusterd_snap_t *snap = NULL;
 
-        GF_VALIDATE_OR_GOTO ("glusterd", origin_vol, out);
-        GF_VALIDATE_OR_GOTO ("glusterd", snap_vol, out);
+    GF_VALIDATE_OR_GOTO("glusterd", origin_vol, out);
+    GF_VALIDATE_OR_GOTO("glusterd", snap_vol, out);
 
-        snap = snap_vol->snapshot;
-        GF_ASSERT (snap);
+    snap = snap_vol->snapshot;
+    GF_ASSERT(snap);
 
-        cds_list_add_tail (&snap_vol->vol_list, &snap->volumes);
-        LOCK (&origin_vol->lock);
-        {
-                glusterd_list_add_order (&snap_vol->snapvol_list,
-                                         &origin_vol->snap_volumes,
-                                         glusterd_compare_snap_vol_time);
+    cds_list_add_tail(&snap_vol->vol_list, &snap->volumes);
+    LOCK(&origin_vol->lock);
+    {
+        glusterd_list_add_order(&snap_vol->snapvol_list,
+                                &origin_vol->snap_volumes,
+                                glusterd_compare_snap_vol_time);
 
-                origin_vol->snap_count++;
-        }
-        UNLOCK (&origin_vol->lock);
+        origin_vol->snap_count++;
+    }
+    UNLOCK(&origin_vol->lock);
 
-        gf_log (THIS->name, GF_LOG_DEBUG, "Snapshot %s added to the list",
-                snap->snapname);
-        ret = 0;
- out:
-        return ret;
+    gf_msg_debug(THIS->name, 0, "Snapshot %s added to the list",
+                 snap->snapname);
+    ret = 0;
+out:
+    return ret;
 }
 
-glusterd_snap_t*
-glusterd_find_snap_by_name (char *snapname)
+glusterd_snap_t *
+glusterd_find_snap_by_name(char *snapname)
 {
-        glusterd_snap_t *snap  = NULL;
-        glusterd_conf_t *priv  = NULL;
-
-        priv = THIS->private;
-        GF_ASSERT (priv);
-        GF_ASSERT (snapname);
-
-        cds_list_for_each_entry (snap, &priv->snapshots, snap_list) {
-                if (!strcmp (snap->snapname, snapname)) {
-                        gf_log (THIS->name, GF_LOG_DEBUG, "Found "
-                                "snap %s (%s)", snap->snapname,
-                                uuid_utoa (snap->snap_id));
-                        goto out;
-                }
-        }
-        snap = NULL;
+    glusterd_snap_t *snap = NULL;
+    glusterd_conf_t *priv = NULL;
+
+    priv = THIS->private;
+    GF_ASSERT(priv);
+    GF_ASSERT(snapname);
+
+    cds_list_for_each_entry(snap, &priv->snapshots, snap_list)
+    {
+        if (!strcmp(snap->snapname, snapname)) {
+            gf_msg_debug(THIS->name, 0,
+                         "Found "
+                         "snap %s (%s)",
+                         snap->snapname, uuid_utoa(snap->snap_id));
+            goto out;
+        }
+    }
+    snap = NULL;
 out:
-        return snap;
+    return snap;
 }
 
-glusterd_snap_t*
-glusterd_find_snap_by_id (uuid_t snap_id)
+glusterd_snap_t *
+glusterd_find_snap_by_id(uuid_t snap_id)
 {
-        glusterd_snap_t *snap  = NULL;
-        glusterd_conf_t *priv  = NULL;
-
-        priv = THIS->private;
-        GF_ASSERT (priv);
-
-        if (gf_uuid_is_null(snap_id))
-                goto out;
-
-        cds_list_for_each_entry (snap, &priv->snapshots, snap_list) {
-                if (!gf_uuid_compare (snap->snap_id, snap_id)) {
-                        gf_log (THIS->name, GF_LOG_DEBUG, "Found "
-                                "snap %s (%s)", snap->snapname,
-                                uuid_utoa (snap->snap_id));
-                        goto out;
-                }
-        }
-        snap = NULL;
+    glusterd_snap_t *snap = NULL;
+    glusterd_conf_t *priv = NULL;
+
+    priv = THIS->private;
+    GF_ASSERT(priv);
+
+    if (gf_uuid_is_null(snap_id))
+        goto out;
+
+    cds_list_for_each_entry(snap, &priv->snapshots, snap_list)
+    {
+        if (!gf_uuid_compare(snap->snap_id, snap_id)) {
+            gf_msg_debug(THIS->name, 0,
+                         "Found "
+                         "snap %s (%s)",
+                         snap->snapname, uuid_utoa(snap->snap_id));
+            goto out;
+        }
+    }
+    snap = NULL;
 out:
-        return snap;
+    return snap;
 }
 
 int
-glusterd_do_lvm_snapshot_remove (glusterd_volinfo_t *snap_vol,
-                                 glusterd_brickinfo_t *brickinfo,
-                                 const char *mount_pt, const char *snap_device)
+glusterd_do_lvm_snapshot_remove(glusterd_volinfo_t *snap_vol,
+                                glusterd_brickinfo_t *brickinfo,
+                                const char *mount_pt, const char *snap_device)
 {
-        int                     ret               = -1;
-        xlator_t               *this              = NULL;
-        glusterd_conf_t        *priv              = NULL;
-        runner_t                runner            = {0,};
-        char                    msg[1024]         = {0, };
-        char                    pidfile[PATH_MAX] = {0, };
-        pid_t                   pid               = -1;
-        int                     retry_count       = 0;
-        char                   *mnt_pt            = NULL;
-        struct mntent          *entry             = NULL;
-        gf_boolean_t            unmount           = _gf_true;
-
-        this = THIS;
-        GF_ASSERT (this);
-        priv = this->private;
-        GF_ASSERT (priv);
-
-        if (!brickinfo) {
-                gf_log (this->name, GF_LOG_ERROR, "brickinfo NULL");
-                goto out;
-        }
-        GF_ASSERT (snap_vol);
-        GF_ASSERT (mount_pt);
-        GF_ASSERT (snap_device);
-
-        GLUSTERD_GET_BRICK_PIDFILE (pidfile, snap_vol, brickinfo, priv);
-        if (gf_is_service_running (pidfile, &pid)) {
-                ret = kill (pid, SIGKILL);
-                if (ret && errno != ESRCH) {
-                        gf_log (this->name, GF_LOG_ERROR, "Unable to kill pid "
-                                "%d reason : %s", pid, strerror(errno));
-                        goto out;
-                }
-        }
+    int ret = -1;
+    xlator_t *this = NULL;
+    glusterd_conf_t *priv = NULL;
+    runner_t runner = {
+        0,
+    };
+    char msg[1024] = "";
+    char pidfile[PATH_MAX] = "";
+    pid_t pid = -1;
+    int retry_count = 0;
+    char *mnt_pt = NULL;
+    gf_boolean_t unmount = _gf_true;
+    int32_t len = 0;
+
+    this = THIS;
+    GF_ASSERT(this);
+    priv = this->private;
+    GF_ASSERT(priv);
+
+    if (!brickinfo) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_INVALID_ENTRY,
+               "brickinfo NULL");
+        goto out;
+    }
+    GF_ASSERT(snap_vol);
+    GF_ASSERT(mount_pt);
+    GF_ASSERT(snap_device);
+
+    GLUSTERD_GET_BRICK_PIDFILE(pidfile, snap_vol, brickinfo, priv);
+    if (gf_is_service_running(pidfile, &pid)) {
+        (void)send_attach_req(this, brickinfo->rpc, brickinfo->path, NULL, NULL,
+                              GLUSTERD_BRICK_TERMINATE);
+        brickinfo->status = GF_BRICK_STOPPED;
+    }
+
+    /* Check if the brick is mounted and then try unmounting the brick */
+    ret = glusterd_get_brick_root(brickinfo->path, &mnt_pt);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_WARNING, 0, GD_MSG_BRICK_PATH_UNMOUNTED,
+               "Getting the root "
+               "of the brick for volume %s (snap %s) failed. "
+               "Removing lv (%s).",
+               snap_vol->volname, snap_vol->snapshot->snapname, snap_device);
+        /* The brick path is already unmounted. Remove the lv only *
+         * Need not fail the operation */
+        ret = 0;
+        unmount = _gf_false;
+    }
+
+    if ((unmount == _gf_true) && (strcmp(mnt_pt, mount_pt))) {
+        gf_msg(this->name, GF_LOG_WARNING, 0, GD_MSG_BRICK_PATH_UNMOUNTED,
+               "Lvm is not mounted for brick %s:%s. "
+               "Removing lv (%s).",
+               brickinfo->hostname, brickinfo->path, snap_device);
+        /* The brick path is already unmounted. Remove the lv only *
+         * Need not fail the operation */
+        unmount = _gf_false;
+    }
+
+    /* umount cannot be done when the brick process is still in the process
+       of shutdown, so give three re-tries */
+    while ((unmount == _gf_true) && (retry_count < 3)) {
+        retry_count++;
+        /*umount2 system call doesn't cleanup mtab entry after un-mount.
+          So use external umount command*/
+        ret = glusterd_umount(mount_pt);
+        if (!ret)
+            break;
+
+        gf_msg_debug(this->name, 0,
+                     "umount failed for "
+                     "path %s (brick: %s): %s. Retry(%d)",
+                     mount_pt, brickinfo->path, strerror(errno), retry_count);
 
-        /* Check if the brick is mounted and then try unmounting the brick */
-        ret = glusterd_get_brick_root (brickinfo->path, &mnt_pt);
-        if (ret) {
-                gf_log (this->name, GF_LOG_WARNING, "Getting the root "
-                        "of the brick for volume %s (snap %s) failed. "
-                        "Removing lv (%s).", snap_vol->volname,
-                         snap_vol->snapshot->snapname, snap_device);
-                /* The brick path is already unmounted. Remove the lv only *
-                 * Need not fail the operation */
-                ret = 0;
-                unmount = _gf_false;
-        }
+        /*
+         * This used to be one second, but that wasn't long enough
+         * to get past the spurious EPERM errors that prevent some
+         * tests (especially bug-1162462.t) from passing reliably.
+         *
+         * TBD: figure out where that garbage is coming from
+         */
+        sleep(3);
+    }
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_UNOUNT_FAILED,
+               "umount failed for "
+               "path %s (brick: %s): %s.",
+               mount_pt, brickinfo->path, strerror(errno));
+        /*
+         * This is cheating, but necessary until we figure out how to
+         * shut down a brick within a still-living brick daemon so that
+         * random translators aren't keeping the mountpoint alive.
+         *
+         * TBD: figure out a real solution
+         */
+        ret = 0;
+        goto out;
+    }
+
+    runinit(&runner);
+    len = snprintf(msg, sizeof(msg),
+                   "remove snapshot of the brick %s:%s, "
+                   "device: %s",
+                   brickinfo->hostname, brickinfo->path, snap_device);
+    if (len < 0) {
+        strcpy(msg, "<error>");
+    }
+    runner_add_args(&runner, LVM_REMOVE, "-f", snap_device, NULL);
+    runner_log(&runner, "", GF_LOG_DEBUG, msg);
+
+    ret = runner_run(&runner);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_SNAP_REMOVE_FAIL,
+               "removing snapshot of the "
+               "brick (%s:%s) of device %s failed",
+               brickinfo->hostname, brickinfo->path, snap_device);
+        goto out;
+    }
 
-        if ((unmount == _gf_true) && (strcmp (mnt_pt, mount_pt))) {
-                gf_log (this->name, GF_LOG_WARNING,
-                        "Lvm is not mounted for brick %s:%s. "
-                        "Removing lv (%s).", brickinfo->hostname,
-                        brickinfo->path, snap_device);
-                /* The brick path is already unmounted. Remove the lv only *
-                 * Need not fail the operation */
-                unmount = _gf_false;
-        }
+out:
+    if (mnt_pt)
+        GF_FREE(mnt_pt);
 
-        /* umount cannot be done when the brick process is still in the process
-           of shutdown, so give three re-tries */
-        while ((unmount == _gf_true) && (retry_count < 3)) {
-                retry_count++;
-                /*umount2 system call doesn't cleanup mtab entry after un-mount.
-                  So use external umount command*/
-                ret = glusterd_umount(mount_pt);
-                if (!ret)
-                        break;
+    return ret;
+}
 
-                gf_log (this->name, GF_LOG_DEBUG, "umount failed for "
-                        "path %s (brick: %s): %s. Retry(%d)", mount_pt,
-                        brickinfo->path, strerror (errno), retry_count);
+int32_t
+glusterd_lvm_snapshot_remove(dict_t *rsp_dict, glusterd_volinfo_t *snap_vol)
+{
+    int32_t brick_count = -1;
+    int32_t ret = -1;
+    int32_t err = 0;
+    glusterd_brickinfo_t *brickinfo = NULL;
+    xlator_t *this = NULL;
+    char brick_dir[PATH_MAX] = "";
+    char snap_path[PATH_MAX] = "";
+    char *tmp = NULL;
+    char *brick_mount_path = NULL;
+    gf_boolean_t is_brick_dir_present = _gf_false;
+    struct stat stbuf = {
+        0,
+    };
+
+    this = THIS;
+    GF_ASSERT(this);
+    GF_ASSERT(snap_vol);
+
+    if ((snap_vol->is_snap_volume == _gf_false) &&
+        (gf_uuid_is_null(snap_vol->restored_from_snap))) {
+        gf_msg_debug(this->name, 0,
+                     "Not a snap volume, or a restored snap volume.");
+        ret = 0;
+        goto out;
+    }
 
-                sleep (1);
-        }
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "umount failed for "
-                        "path %s (brick: %s): %s.", mount_pt,
-                        brickinfo->path, strerror (errno));
-                goto out;
+    brick_count = -1;
+    cds_list_for_each_entry(brickinfo, &snap_vol->bricks, brick_list)
+    {
+        brick_count++;
+        if (gf_uuid_compare(brickinfo->uuid, MY_UUID)) {
+            gf_msg_debug(this->name, 0, "%s:%s belongs to a different node",
+                         brickinfo->hostname, brickinfo->path);
+            continue;
         }
 
-        runinit (&runner);
-        snprintf (msg, sizeof(msg), "remove snapshot of the brick %s:%s, "
-                  "device: %s", brickinfo->hostname, brickinfo->path,
-                  snap_device);
-        runner_add_args (&runner, LVM_REMOVE, "-f", snap_device, NULL);
-        runner_log (&runner, "", GF_LOG_DEBUG, msg);
-
-        ret = runner_run (&runner);
+        /* Fetch the brick mount path from the brickinfo->path */
+        ret = glusterd_find_brick_mount_path(brickinfo->path,
+                                             &brick_mount_path);
         if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "removing snapshot of the "
-                        "brick (%s:%s) of device %s failed",
-                        brickinfo->hostname, brickinfo->path, snap_device);
-                goto out;
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_BRICK_GET_INFO_FAIL,
+                   "Failed to find brick_mount_path for %s", brickinfo->path);
+            ret = 0;
+            continue;
         }
 
-out:
-        return ret;
-}
-
-int32_t
-glusterd_lvm_snapshot_remove (dict_t *rsp_dict, glusterd_volinfo_t *snap_vol)
-{
-        struct mntent         save_entry           = {0,};
-        int32_t               brick_count          = -1;
-        int32_t               ret                  = -1;
-        int32_t               err                  = 0;
-        glusterd_brickinfo_t *brickinfo            = NULL;
-        xlator_t             *this                 = NULL;
-        char                  buff[PATH_MAX]       = "";
-        char                  brick_dir[PATH_MAX]  = "";
-        char                 *tmp                  = NULL;
-        char                 *brick_mount_path     = NULL;
-        gf_boolean_t          is_brick_dir_present = _gf_false;
-        struct stat           stbuf                = {0,};
-
-        this = THIS;
-        GF_ASSERT (this);
-        GF_ASSERT (snap_vol);
-
-        if ((snap_vol->is_snap_volume == _gf_false) &&
-            (gf_uuid_is_null (snap_vol->restored_from_snap))) {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "Not a snap volume, or a restored snap volume.");
+        /* As deactivated snapshot have no active mount point we
+         * check only for activated snapshot.
+         */
+        if (snap_vol->status == GLUSTERD_STATUS_STARTED) {
+            ret = sys_lstat(brick_mount_path, &stbuf);
+            if (ret) {
+                gf_msg_debug(this->name, 0, "Brick %s:%s already deleted.",
+                             brickinfo->hostname, brickinfo->path);
                 ret = 0;
-                goto out;
+                continue;
+            }
         }
 
-        brick_count = -1;
-        cds_list_for_each_entry (brickinfo, &snap_vol->bricks, brick_list) {
-                brick_count++;
-                if (gf_uuid_compare (brickinfo->uuid, MY_UUID)) {
-                        gf_log (this->name, GF_LOG_DEBUG,
-                                "%s:%s belongs to a different node",
-                                brickinfo->hostname, brickinfo->path);
-                        continue;
-                }
-
-                /* Fetch the brick mount path from the brickinfo->path */
-                ret = glusterd_find_brick_mount_path (brickinfo->path,
-                                                      brick_count + 1,
-                                                      &brick_mount_path);
-                if (ret) {
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "Failed to find brick_mount_path for %s",
-                                brickinfo->path);
-                        ret = 0;
-                        continue;
-                }
-
-                ret = lstat (brick_mount_path, &stbuf);
-                if (ret) {
-                        gf_log (this->name, GF_LOG_DEBUG,
-                                "Brick %s:%s already deleted.",
-                                brickinfo->hostname, brickinfo->path);
-                        ret = 0;
-                        continue;
-                }
-
-                if (brickinfo->snap_status == -1) {
-                        gf_log (this->name, GF_LOG_INFO,
-                                "snapshot was pending. lvm not present "
-                                "for brick %s:%s of the snap %s.",
-                                brickinfo->hostname, brickinfo->path,
-                                snap_vol->snapshot->snapname);
-
-                        if (rsp_dict &&
-                            (snap_vol->is_snap_volume == _gf_true)) {
-                                /* Adding missed delete to the dict */
-                                ret = glusterd_add_missed_snaps_to_dict
-                                                   (rsp_dict,
-                                                    snap_vol,
-                                                    brickinfo,
-                                                    brick_count + 1,
-                                                    GF_SNAP_OPTION_TYPE_DELETE);
-                                if (ret) {
-                                        gf_log (this->name, GF_LOG_ERROR,
-                                                "Failed to add missed snapshot "
-                                                "info for %s:%s in the "
-                                                "rsp_dict", brickinfo->hostname,
-                                                brickinfo->path);
-                                        goto out;
-                                }
-                        }
-
-                        continue;
-                }
-
-                /* Check if the brick has a LV associated with it */
-                if (strlen(brickinfo->device_path) == 0) {
-                        gf_log (this->name, GF_LOG_DEBUG,
-                                "Brick (%s:%s) does not have a LV "
-                                "associated with it. Removing the brick path",
-                                brickinfo->hostname, brickinfo->path);
-                        goto remove_brick_path;
-                }
-
-                /* Verify if the device path exists or not */
-                ret = stat (brickinfo->device_path, &stbuf);
-                if (ret) {
-                        gf_log (this->name, GF_LOG_DEBUG,
-                                "LV (%s) for brick (%s:%s) not present. "
-                                "Removing the brick path",
-                                brickinfo->device_path,
-                                brickinfo->hostname, brickinfo->path);
-                        /* Making ret = 0 as absence of device path should *
-                         * not fail the remove operation */
-                        ret = 0;
-                        goto remove_brick_path;
-                }
+        if (brickinfo->snap_status == -1) {
+            gf_msg(this->name, GF_LOG_INFO, 0, GD_MSG_SNAPSHOT_PENDING,
+                   "snapshot was pending. lvm not present "
+                   "for brick %s:%s of the snap %s.",
+                   brickinfo->hostname, brickinfo->path,
+                   snap_vol->snapshot->snapname);
 
-                ret = glusterd_do_lvm_snapshot_remove (snap_vol, brickinfo,
-                                                       brick_mount_path,
-                                                       brickinfo->device_path);
+            if (rsp_dict && (snap_vol->is_snap_volume == _gf_true)) {
+                /* Adding missed delete to the dict */
+                ret = glusterd_add_missed_snaps_to_dict(
+                    rsp_dict, snap_vol, brickinfo, brick_count + 1,
+                    GF_SNAP_OPTION_TYPE_DELETE);
                 if (ret) {
-                        gf_log (this->name, GF_LOG_ERROR, "Failed to "
-                                "remove the snapshot %s (%s)",
-                                brickinfo->path, brickinfo->device_path);
-                        err = -1; /* We need to record this failure */
-                }
-
-remove_brick_path:
-                /* After removing the brick dir fetch the parent path
-                 * i.e /var/run/gluster/snaps/<snap-vol-id>/
-                 */
-                if (is_brick_dir_present == _gf_false) {
-                        /* Need to fetch brick_dir to be removed from
-                         * brickinfo->path, as in a restored volume,
-                         * snap_vol won't have the non-hyphenated snap_vol_id
-                         */
-                        tmp = strstr (brick_mount_path, "brick");
-                        if (!tmp) {
-                                gf_log (this->name, GF_LOG_ERROR,
-                                        "Invalid brick %s", brickinfo->path);
-                                GF_FREE (brick_mount_path);
-                                brick_mount_path = NULL;
-                                continue;
-                        }
-
-                        strncpy (brick_dir, brick_mount_path,
-                                 (size_t) (tmp - brick_mount_path));
-
-                        /* Peers not hosting bricks will have _gf_false */
-                        is_brick_dir_present = _gf_true;
+                    gf_msg(this->name, GF_LOG_ERROR, 0,
+                           GD_MSG_MISSED_SNAP_CREATE_FAIL,
+                           "Failed to add missed snapshot "
+                           "info for %s:%s in the "
+                           "rsp_dict",
+                           brickinfo->hostname, brickinfo->path);
+                    goto out;
                 }
+            }
 
-                GF_FREE (brick_mount_path);
-                brick_mount_path = NULL;
+            continue;
         }
 
-        if (is_brick_dir_present == _gf_true) {
-                ret = recursive_rmdir (brick_dir);
-                if (ret) {
-                        if (errno == ENOTEMPTY) {
-                                /* Will occur when multiple glusterds
-                                 * are running in the same node
-                                 */
-                                gf_log (this->name, GF_LOG_WARNING,
-                                        "Failed to rmdir: %s, err: %s. "
-                                        "More than one glusterd running "
-                                        "on this node.",
-                                        brick_dir, strerror (errno));
-                                ret = 0;
-                                goto out;
-                        } else
-                                gf_log (this->name, GF_LOG_ERROR,
-                                        "Failed to rmdir: %s, err: %s",
-                                        brick_dir, strerror (errno));
-                                goto out;
-                }
+        /* Check if the brick has a LV associated with it */
+        if (strlen(brickinfo->device_path) == 0) {
+            gf_msg_debug(this->name, 0,
+                         "Brick (%s:%s) does not have a LV "
+                         "associated with it. Removing the brick path",
+                         brickinfo->hostname, brickinfo->path);
+            goto remove_brick_path;
         }
 
-        ret = 0;
-out:
-        if (err) {
-                ret = err;
+        /* Verify if the device path exists or not */
+        ret = sys_stat(brickinfo->device_path, &stbuf);
+        if (ret) {
+            gf_msg_debug(this->name, 0,
+                         "LV (%s) for brick (%s:%s) not present. "
+                         "Removing the brick path",
+                         brickinfo->device_path, brickinfo->hostname,
+                         brickinfo->path);
+            /* Making ret = 0 as absence of device path should *
+             * not fail the remove operation */
+            ret = 0;
+            goto remove_brick_path;
         }
-        GF_FREE (brick_mount_path);
-        gf_log (this->name, GF_LOG_TRACE, "Returning %d", ret);
-        return ret;
-}
 
+        ret = glusterd_do_lvm_snapshot_remove(
+            snap_vol, brickinfo, brick_mount_path, brickinfo->device_path);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_SNAP_REMOVE_FAIL,
+                   "Failed to "
+                   "remove the snapshot %s (%s)",
+                   brickinfo->path, brickinfo->device_path);
+            err = -1; /* We need to record this failure */
+        }
 
-int32_t
-glusterd_snap_volume_remove (dict_t *rsp_dict,
-                             glusterd_volinfo_t *snap_vol,
-                             gf_boolean_t remove_lvm,
-                             gf_boolean_t force)
-{
-        int                   ret         = -1;
-        int                   save_ret    =  0;
-        glusterd_brickinfo_t *brickinfo   = NULL;
-        glusterd_volinfo_t   *origin_vol  = NULL;
-        xlator_t             *this        = NULL;
+    remove_brick_path:
+        /* After removing the brick dir fetch the parent path
+         * i.e /var/run/gluster/snaps/<snap-vol-id>/
+         */
+        if (is_brick_dir_present == _gf_false) {
+            /* Need to fetch brick_dir to be removed from
+             * brickinfo->path, as in a restored volume,
+             * snap_vol won't have the non-hyphenated snap_vol_id
+             */
+            tmp = strstr(brick_mount_path, "brick");
+            if (!tmp) {
+                gf_msg(this->name, GF_LOG_ERROR, EINVAL, GD_MSG_INVALID_ENTRY,
+                       "Invalid brick %s", brickinfo->path);
+                GF_FREE(brick_mount_path);
+                brick_mount_path = NULL;
+                continue;
+            }
 
-        this = THIS;
-        GF_ASSERT (this);
-        GF_ASSERT (rsp_dict);
-        GF_ASSERT (snap_vol);
+            strncpy(brick_dir, brick_mount_path,
+                    (size_t)(tmp - brick_mount_path));
 
-        if (!snap_vol) {
-                gf_log(this->name, GF_LOG_WARNING, "snap_vol in NULL");
-                ret = -1;
-                goto out;
+            /* Peers not hosting bricks will have _gf_false */
+            is_brick_dir_present = _gf_true;
         }
 
-        cds_list_for_each_entry (brickinfo, &snap_vol->bricks, brick_list) {
-                if (gf_uuid_compare (brickinfo->uuid, MY_UUID))
-                        continue;
+        GF_FREE(brick_mount_path);
+        brick_mount_path = NULL;
+    }
 
-                ret = glusterd_brick_stop (snap_vol, brickinfo, _gf_false);
-                if (ret) {
-                        gf_log(this->name, GF_LOG_WARNING, "Failed to stop "
-                               "brick for volume %s", snap_vol->volname);
-                        save_ret = ret;
-
-                        /* Don't clean up the snap on error when
-                           force flag is disabled */
-                        if (!force)
-                                goto out;
-                }
+    if (is_brick_dir_present == _gf_true) {
+        ret = recursive_rmdir(brick_dir);
+        if (ret) {
+            if (errno == ENOTEMPTY) {
+                /* Will occur when multiple glusterds
+                 * are running in the same node
+                 */
+                gf_msg(this->name, GF_LOG_WARNING, errno, GD_MSG_DIR_OP_FAILED,
+                       "Failed to rmdir: %s, err: %s. "
+                       "More than one glusterd running "
+                       "on this node.",
+                       brick_dir, strerror(errno));
+                ret = 0;
+                goto out;
+            } else
+                gf_msg(this->name, GF_LOG_ERROR, errno, GD_MSG_DIR_OP_FAILED,
+                       "Failed to rmdir: %s, err: %s", brick_dir,
+                       strerror(errno));
+            goto out;
         }
 
-        /* Only remove the backend lvm when required */
-        if (remove_lvm) {
-                ret = glusterd_lvm_snapshot_remove (rsp_dict, snap_vol);
-                if (ret) {
-                        gf_log(this->name, GF_LOG_WARNING, "Failed to remove "
-                               "lvm snapshot volume %s", snap_vol->volname);
-                        save_ret = ret;
-                        if (!force)
-                                goto out;
-                }
+        /* After removing brick_dir, fetch and remove snap path
+         * i.e. /var/run/gluster/snaps/<snap-name>.
+         */
+        if (!snap_vol->snapshot) {
+            gf_msg(this->name, GF_LOG_WARNING, EINVAL, GD_MSG_INVALID_ENTRY,
+                   "snapshot not"
+                   "present in snap_vol");
+            ret = -1;
+            goto out;
         }
 
-        ret = glusterd_store_delete_volume (snap_vol);
+        snprintf(snap_path, sizeof(snap_path), "%s/%s", snap_mount_dir,
+                 snap_vol->snapshot->snapname);
+        ret = recursive_rmdir(snap_path);
         if (ret) {
-                gf_log(this->name, GF_LOG_WARNING, "Failed to remove volume %s "
-                       "from store", snap_vol->volname);
-                save_ret = ret;
-                if (!force)
-                        goto out;
-        }
-
-        if (!cds_list_empty (&snap_vol->snapvol_list)) {
-                ret = glusterd_volinfo_find (snap_vol->parent_volname,
-                                             &origin_vol);
-                if (ret) {
-                        gf_log (this->name, GF_LOG_ERROR, "Failed to get "
-                                "parent volinfo %s  for volume  %s",
-                                snap_vol->parent_volname, snap_vol->volname);
-                        save_ret = ret;
-                        if (!force)
-                                goto out;
-                }
-                origin_vol->snap_count--;
+            gf_msg(this->name, GF_LOG_ERROR, errno, GD_MSG_DIR_OP_FAILED,
+                   "Failed to remove "
+                   "%s directory : error : %s",
+                   snap_path, strerror(errno));
+            goto out;
         }
+    }
 
-        glusterd_volinfo_unref (snap_vol);
-
-        if (save_ret)
-                ret = save_ret;
+    ret = 0;
 out:
-        gf_log (this->name, GF_LOG_TRACE, "returning %d", ret);
-        return ret;
+    if (err) {
+        ret = err;
+    }
+    GF_FREE(brick_mount_path);
+    gf_msg_trace(this->name, 0, "Returning %d", ret);
+    return ret;
 }
 
 int32_t
-glusterd_snapobject_delete (glusterd_snap_t *snap)
+glusterd_snap_volume_remove(dict_t *rsp_dict, glusterd_volinfo_t *snap_vol,
+                            gf_boolean_t remove_lvm, gf_boolean_t force)
 {
-        if (snap == NULL) {
-                gf_log(THIS->name, GF_LOG_WARNING, "snap is NULL");
-                return -1;
-        }
-
-        cds_list_del_init (&snap->snap_list);
-        cds_list_del_init (&snap->volumes);
-        if (LOCK_DESTROY(&snap->lock))
-                gf_log (THIS->name, GF_LOG_WARNING, "Failed destroying lock"
-                        "of snap %s", snap->snapname);
-
-        GF_FREE (snap->description);
-        GF_FREE (snap);
-
-        return 0;
+    int ret = -1;
+    int save_ret = 0;
+    glusterd_brickinfo_t *brickinfo = NULL;
+    glusterd_volinfo_t *origin_vol = NULL;
+    xlator_t *this = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+    GF_ASSERT(rsp_dict);
+    GF_ASSERT(snap_vol);
+
+    if (!snap_vol) {
+        gf_msg(this->name, GF_LOG_WARNING, EINVAL, GD_MSG_INVALID_ENTRY,
+               "snap_vol in NULL");
+        ret = -1;
+        goto out;
+    }
+
+    cds_list_for_each_entry(brickinfo, &snap_vol->bricks, brick_list)
+    {
+        if (gf_uuid_compare(brickinfo->uuid, MY_UUID))
+            continue;
+
+        ret = glusterd_brick_stop(snap_vol, brickinfo, _gf_false);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_WARNING, 0, GD_MSG_BRICK_STOP_FAIL,
+                   "Failed to stop "
+                   "brick for volume %s",
+                   snap_vol->volname);
+            save_ret = ret;
+
+            /* Don't clean up the snap on error when
+               force flag is disabled */
+            if (!force)
+                goto out;
+        }
+    }
+
+    /* Only remove the backend lvm when required */
+    if (remove_lvm) {
+        ret = glusterd_lvm_snapshot_remove(rsp_dict, snap_vol);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_WARNING, 0, GD_MSG_SNAP_REMOVE_FAIL,
+                   "Failed to remove "
+                   "lvm snapshot volume %s",
+                   snap_vol->volname);
+            save_ret = ret;
+            if (!force)
+                goto out;
+        }
+    }
+
+    ret = glusterd_store_delete_volume(snap_vol);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_WARNING, 0, GD_MSG_VOL_DELETE_FAIL,
+               "Failed to remove volume %s "
+               "from store",
+               snap_vol->volname);
+        save_ret = ret;
+        if (!force)
+            goto out;
+    }
+
+    if (!cds_list_empty(&snap_vol->snapvol_list)) {
+        ret = glusterd_volinfo_find(snap_vol->parent_volname, &origin_vol);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, EINVAL, GD_MSG_VOL_NOT_FOUND,
+                   "Failed to get "
+                   "parent volinfo %s  for volume  %s",
+                   snap_vol->parent_volname, snap_vol->volname);
+            save_ret = ret;
+            if (!force)
+                goto out;
+        }
+        origin_vol->snap_count--;
+    }
+
+    glusterd_volinfo_unref(snap_vol);
+
+    if (save_ret)
+        ret = save_ret;
+out:
+    gf_msg_trace(this->name, 0, "returning %d", ret);
+    return ret;
 }
 
 int32_t
-glusterd_snap_remove (dict_t *rsp_dict,
-                      glusterd_snap_t *snap,
-                      gf_boolean_t remove_lvm,
-                      gf_boolean_t force)
+glusterd_snap_remove(dict_t *rsp_dict, glusterd_snap_t *snap,
+                     gf_boolean_t remove_lvm, gf_boolean_t force,
+                     gf_boolean_t is_clone)
 {
-        int                 ret       = -1;
-        int                 save_ret  =  0;
-        glusterd_volinfo_t *snap_vol  = NULL;
-        glusterd_volinfo_t *tmp       = NULL;
-        xlator_t           *this      = NULL;
-
-        this = THIS;
-        GF_ASSERT (this);
-        GF_ASSERT (rsp_dict);
-        GF_ASSERT (snap);
-
-        if (!snap) {
-                gf_log(this->name, GF_LOG_WARNING, "snap is NULL");
-                ret = -1;
-                goto out;
-        }
-
-        cds_list_for_each_entry_safe (snap_vol, tmp, &snap->volumes, vol_list) {
-                ret = glusterd_snap_volume_remove (rsp_dict, snap_vol,
-                                                   remove_lvm, force);
-                if (ret && !force) {
-                        /* Don't clean up the snap on error when
-                           force flag is disabled */
-                        gf_log(this->name, GF_LOG_WARNING, "Failed to remove "
-                               "volinfo %s for snap %s", snap_vol->volname,
-                               snap->snapname);
-                        save_ret = ret;
-                        goto out;
-                }
-        }
-
-        ret = glusterd_store_delete_snap (snap);
-        if (ret) {
-                gf_log(this->name, GF_LOG_WARNING, "Failed to remove snap %s "
-                       "from store", snap->snapname);
-                save_ret = ret;
-                if (!force)
-                        goto out;
-        }
-
-        ret = glusterd_snapobject_delete (snap);
-        if (ret)
-                gf_log (this->name, GF_LOG_WARNING, "Failed to delete "
-                        "snap object %s", snap->snapname);
-
-        if (save_ret)
-                ret = save_ret;
+    int ret = -1;
+    int save_ret = 0;
+    glusterd_volinfo_t *snap_vol = NULL;
+    glusterd_volinfo_t *tmp = NULL;
+    xlator_t *this = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+    GF_ASSERT(rsp_dict);
+    GF_ASSERT(snap);
+
+    if (!snap) {
+        gf_msg(this->name, GF_LOG_WARNING, EINVAL, GD_MSG_INVALID_ENTRY,
+               "snap is NULL");
+        ret = -1;
+        goto out;
+    }
+
+    cds_list_for_each_entry_safe(snap_vol, tmp, &snap->volumes, vol_list)
+    {
+        ret = glusterd_snap_volume_remove(rsp_dict, snap_vol, remove_lvm,
+                                          force);
+        if (ret && !force) {
+            /* Don't clean up the snap on error when
+               force flag is disabled */
+            gf_msg(this->name, GF_LOG_WARNING, 0, GD_MSG_SNAP_REMOVE_FAIL,
+                   "Failed to remove "
+                   "volinfo %s for snap %s",
+                   snap_vol->volname, snap->snapname);
+            save_ret = ret;
+            goto out;
+        }
+    }
+
+    /* A clone does not persist snap info in /var/lib/glusterd/snaps/ *
+     * and hence there is no snap info to be deleted from there       *
+     */
+    if (!is_clone) {
+        ret = glusterd_store_delete_snap(snap);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_WARNING, 0, GD_MSG_SNAP_REMOVE_FAIL,
+                   "Failed to remove snap %s from store", snap->snapname);
+            save_ret = ret;
+            if (!force)
+                goto out;
+        }
+    }
+
+    ret = glusterd_snapobject_delete(snap);
+    if (ret)
+        gf_msg(this->name, GF_LOG_WARNING, 0, GD_MSG_SNAP_REMOVE_FAIL,
+               "Failed to delete "
+               "snap object %s",
+               snap->snapname);
+
+    if (save_ret)
+        ret = save_ret;
 out:
-        gf_log (THIS->name, GF_LOG_TRACE, "returning %d", ret);
-        return ret;
+    gf_msg_trace(THIS->name, 0, "returning %d", ret);
+    return ret;
 }
 
 static int
-glusterd_snapshot_get_snapvol_detail (dict_t *dict,
-                                      glusterd_volinfo_t *snap_vol,
-                                      char *keyprefix, int detail)
+glusterd_snapshot_get_snapvol_detail(dict_t *dict, glusterd_volinfo_t *snap_vol,
+                                     const char *keyprefix, const int detail)
 {
-        int                 ret           = -1;
-        int                 snap_limit    = 0;
-        char                key[PATH_MAX] = {0,};
-        char               *value         = NULL;
-        glusterd_volinfo_t *origin_vol    = NULL;
-        glusterd_conf_t    *conf          = NULL;
-        xlator_t           *this          = NULL;
-        uint64_t           opt_hard_max   = GLUSTERD_SNAPS_MAX_HARD_LIMIT;
-
-        this = THIS;
-        conf = this->private;
-        GF_ASSERT (conf);
-
-        GF_ASSERT (dict);
-        GF_ASSERT (snap_vol);
-        GF_ASSERT (keyprefix);
-
-        /* Volume Name */
-        value = gf_strdup (snap_vol->volname);
-        if (!value)
-                goto out;
-
-        snprintf (key, sizeof (key), "%s.volname", keyprefix);
-        ret = dict_set_dynstr (dict, key, value);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "Failed to set "
-                        "volume name in dictionary: %s", key);
-                goto out;
-        }
-
-        /* Volume ID */
-        value = gf_strdup (uuid_utoa (snap_vol->volume_id));
-        if (NULL == value) {
-                ret = -1;
-                goto out;
-        }
-
-        snprintf (key, sizeof (key), "%s.vol-id", keyprefix);
-        ret = dict_set_dynstr (dict, key, value);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "Failed to set "
-                        "volume id in dictionary: %s", key);
-                goto out;
-        }
-        value = NULL;
-
-        /* volume status */
-        snprintf (key, sizeof (key), "%s.vol-status", keyprefix);
-        switch (snap_vol->status) {
+    int ret = -1;
+    int snap_limit = 0;
+    char key[64] = ""; /* keyprefix is quite small, up to 32 byts */
+    int keylen;
+    char *value = NULL;
+    glusterd_volinfo_t *origin_vol = NULL;
+    glusterd_conf_t *conf = NULL;
+    xlator_t *this = NULL;
+    uint64_t opt_hard_max = GLUSTERD_SNAPS_MAX_HARD_LIMIT;
+
+    this = THIS;
+    conf = this->private;
+    GF_ASSERT(conf);
+
+    GF_ASSERT(dict);
+    GF_ASSERT(snap_vol);
+    GF_ASSERT(keyprefix);
+
+    /* Volume Name */
+    value = gf_strdup(snap_vol->volname);
+    if (!value)
+        goto out;
+
+    keylen = snprintf(key, sizeof(key), "%s.volname", keyprefix);
+    ret = dict_set_dynstrn(dict, key, keylen, value);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+               "Failed to set "
+               "volume name in dictionary: %s",
+               key);
+        goto out;
+    }
+
+    /* Volume ID */
+    value = gf_strdup(uuid_utoa(snap_vol->volume_id));
+    if (NULL == value) {
+        ret = -1;
+        goto out;
+    }
+
+    keylen = snprintf(key, sizeof(key), "%s.vol-id", keyprefix);
+    ret = dict_set_dynstrn(dict, key, keylen, value);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_NO_MEMORY,
+               "Failed to set "
+               "volume id in dictionary: %s",
+               key);
+        goto out;
+    }
+    value = NULL;
+
+    /* volume status */
+    keylen = snprintf(key, sizeof(key), "%s.vol-status", keyprefix);
+    switch (snap_vol->status) {
         case GLUSTERD_STATUS_STARTED:
-                ret = dict_set_str (dict, key, "Started");
-                break;
+            ret = dict_set_nstrn(dict, key, keylen, "Started", SLEN("Started"));
+            break;
         case GLUSTERD_STATUS_STOPPED:
-                ret = dict_set_str (dict, key, "Stopped");
-                break;
+            ret = dict_set_nstrn(dict, key, keylen, "Stopped", SLEN("Stopped"));
+            break;
         case GD_SNAP_STATUS_NONE:
-                ret = dict_set_str (dict, key, "None");
-                break;
+            ret = dict_set_nstrn(dict, key, keylen, "None", SLEN("None"));
+            break;
         default:
-                gf_log (this->name, GF_LOG_ERROR, "Invalid volume status");
-                ret = -1;
-                goto out;
-        }
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "Failed to set volume status"
-                        " in dictionary: %s", key);
-                goto out;
-        }
-
-
-        ret = glusterd_volinfo_find (snap_vol->parent_volname, &origin_vol);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "failed to get the parent "
-                        "volinfo for the volume %s", snap_vol->volname);
-                goto out;
-        }
-
-        /* "snap-max-hard-limit" might not be set by user explicitly,
-         * in that case it's better to consider the default value.
-         * Hence not erroring out if Key is not found.
-         */
-        ret = dict_get_uint64 (conf->opts,
-                               GLUSTERD_STORE_KEY_SNAP_MAX_HARD_LIMIT,
-                               &opt_hard_max);
-        if (ret) {
-                ret = 0;
-                gf_log (this->name, GF_LOG_DEBUG, "%s is not present in "
-                        "opts dictionary",
-                        GLUSTERD_STORE_KEY_SNAP_MAX_HARD_LIMIT);
-        }
-
-        if (opt_hard_max < origin_vol->snap_max_hard_limit) {
-                snap_limit = opt_hard_max;
-                gf_log(this->name, GF_LOG_DEBUG, "system snap-max-hard-limit is"
-                       " lesser than volume snap-max-hard-limit, "
-                       "snap-max-hard-limit value is set to %d", snap_limit);
-        } else {
-                snap_limit = origin_vol->snap_max_hard_limit;
-                gf_log(this->name, GF_LOG_DEBUG, "volume snap-max-hard-limit is"
-                       " lesser than system snap-max-hard-limit, "
-                       "snap-max-hard-limit value is set to %d", snap_limit);
-        }
-
-        snprintf (key, sizeof (key), "%s.snaps-available", keyprefix);
-        if (snap_limit > origin_vol->snap_count)
-                ret = dict_set_int32 (dict, key,
-                        snap_limit - origin_vol->snap_count);
-        else
-                ret = dict_set_int32 (dict, key, 0);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "Failed to set available snaps");
-                goto out;
-        }
-
-        snprintf (key, sizeof (key), "%s.snapcount", keyprefix);
-        ret = dict_set_int32 (dict, key, origin_vol->snap_count);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "Could not save snapcount");
-                goto out;
-        }
-
-        if (!detail)
-                goto out;
-
-        /* Parent volume name */
-        value = gf_strdup (snap_vol->parent_volname);
-        if (!value)
-                goto out;
-
-        snprintf (key, sizeof (key), "%s.origin-volname", keyprefix);
-        ret = dict_set_dynstr (dict, key, value);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "Failed to set parent "
-                        "volume name in dictionary: %s", key);
-                goto out;
-        }
-        value = NULL;
-
+            gf_msg(this->name, GF_LOG_ERROR, EINVAL, GD_MSG_INVALID_ENTRY,
+                   "Invalid volume status");
+            ret = -1;
+            goto out;
+    }
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+               "Failed to set volume status"
+               " in dictionary: %s",
+               key);
+        goto out;
+    }
+
+    ret = glusterd_volinfo_find(snap_vol->parent_volname, &origin_vol);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, EINVAL, GD_MSG_VOL_NOT_FOUND,
+               "failed to get the parent "
+               "volinfo for the volume %s",
+               snap_vol->volname);
+        goto out;
+    }
+
+    /* "snap-max-hard-limit" might not be set by user explicitly,
+     * in that case it's better to consider the default value.
+     * Hence not erroring out if Key is not found.
+     */
+    ret = dict_get_uint64(conf->opts, GLUSTERD_STORE_KEY_SNAP_MAX_HARD_LIMIT,
+                          &opt_hard_max);
+    if (ret) {
         ret = 0;
+        gf_msg_debug(this->name, 0,
+                     "%s is not present in "
+                     "opts dictionary",
+                     GLUSTERD_STORE_KEY_SNAP_MAX_HARD_LIMIT);
+    }
+
+    if (opt_hard_max < origin_vol->snap_max_hard_limit) {
+        snap_limit = opt_hard_max;
+        gf_msg_debug(this->name, 0,
+                     "system snap-max-hard-limit is"
+                     " lesser than volume snap-max-hard-limit, "
+                     "snap-max-hard-limit value is set to %d",
+                     snap_limit);
+    } else {
+        snap_limit = origin_vol->snap_max_hard_limit;
+        gf_msg_debug(this->name, 0,
+                     "volume snap-max-hard-limit is"
+                     " lesser than system snap-max-hard-limit, "
+                     "snap-max-hard-limit value is set to %d",
+                     snap_limit);
+    }
+
+    keylen = snprintf(key, sizeof(key), "%s.snaps-available", keyprefix);
+    if (snap_limit > origin_vol->snap_count)
+        ret = dict_set_int32n(dict, key, keylen,
+                              snap_limit - origin_vol->snap_count);
+    else
+        ret = dict_set_int32(dict, key, 0);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+               "Failed to set available snaps");
+        goto out;
+    }
+
+    keylen = snprintf(key, sizeof(key), "%s.snapcount", keyprefix);
+    ret = dict_set_int32n(dict, key, keylen, origin_vol->snap_count);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+               "Could not save snapcount");
+        goto out;
+    }
+
+    if (!detail)
+        goto out;
+
+    /* Parent volume name */
+    value = gf_strdup(snap_vol->parent_volname);
+    if (!value)
+        goto out;
+
+    keylen = snprintf(key, sizeof(key), "%s.origin-volname", keyprefix);
+    ret = dict_set_dynstrn(dict, key, keylen, value);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+               "Failed to set parent "
+               "volume name in dictionary: %s",
+               key);
+        goto out;
+    }
+    value = NULL;
+
+    ret = 0;
 out:
-        if (value)
-                GF_FREE (value);
+    if (value)
+        GF_FREE(value);
 
-        return ret;
+    return ret;
 }
 
 static int
-glusterd_snapshot_get_snap_detail (dict_t *dict, glusterd_snap_t *snap,
-                                char *keyprefix, glusterd_volinfo_t *volinfo)
+glusterd_snapshot_get_snap_detail(dict_t *dict, glusterd_snap_t *snap,
+                                  const char *keyprefix,
+                                  glusterd_volinfo_t *volinfo)
 {
-        int                 ret           = -1;
-        int                 volcount      = 0;
-        char                key[PATH_MAX] = {0,};
-        char                timestr[64]   = {0,};
-        char               *value         = NULL;
-        glusterd_volinfo_t *snap_vol      = NULL;
-        glusterd_volinfo_t *tmp_vol       = NULL;
-        xlator_t           *this          = NULL;
-
-        this = THIS;
-
-        GF_ASSERT (dict);
-        GF_ASSERT (snap);
-        GF_ASSERT (keyprefix);
-
-        /* Snap Name */
-        value = gf_strdup (snap->snapname);
-        if (!value)
-                goto out;
-
-        snprintf (key, sizeof (key), "%s.snapname", keyprefix);
-        ret = dict_set_dynstr (dict, key, value);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "Failed to set "
-                        "snap name in dictionary");
-                goto out;
-        }
-
-        /* Snap ID */
-        value = gf_strdup (uuid_utoa (snap->snap_id));
+    int ret = -1;
+    int volcount = 0;
+    char key[32] = ""; /* keyprefix is quite small, up to 16 bytes */
+    int keylen;
+    char timestr[GF_TIMESTR_SIZE] = "";
+    char *value = NULL;
+    glusterd_volinfo_t *snap_vol = NULL;
+    glusterd_volinfo_t *tmp_vol = NULL;
+    xlator_t *this = NULL;
+
+    this = THIS;
+
+    GF_ASSERT(dict);
+    GF_ASSERT(snap);
+    GF_ASSERT(keyprefix);
+
+    /* Snap Name */
+    value = gf_strdup(snap->snapname);
+    if (!value)
+        goto out;
+
+    keylen = snprintf(key, sizeof(key), "%s.snapname", keyprefix);
+    ret = dict_set_dynstrn(dict, key, keylen, value);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+               "Failed to set "
+               "snap name in dictionary");
+        goto out;
+    }
+
+    /* Snap ID */
+    value = gf_strdup(uuid_utoa(snap->snap_id));
+    if (NULL == value) {
+        ret = -1;
+        goto out;
+    }
+
+    keylen = snprintf(key, sizeof(key), "%s.snap-id", keyprefix);
+    ret = dict_set_dynstrn(dict, key, keylen, value);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+               "Failed to set "
+               "snap id in dictionary");
+        goto out;
+    }
+    value = NULL;
+
+    gf_time_fmt(timestr, sizeof timestr, snap->time_stamp, gf_timefmt_FT);
+    value = gf_strdup(timestr);
+
+    if (NULL == value) {
+        ret = -1;
+        goto out;
+    }
+
+    keylen = snprintf(key, sizeof(key), "%s.snap-time", keyprefix);
+    ret = dict_set_dynstrn(dict, key, keylen, value);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+               "Failed to set "
+               "snap time stamp in dictionary");
+        goto out;
+    }
+    value = NULL;
+
+    /* If snap description is provided then add that into dictionary */
+    if (NULL != snap->description) {
+        value = gf_strdup(snap->description);
         if (NULL == value) {
-                ret = -1;
-                goto out;
-        }
-
-        snprintf (key, sizeof (key), "%s.snap-id", keyprefix);
-        ret = dict_set_dynstr (dict, key, value);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "Failed to set "
-                                "snap id in dictionary");
-                goto out;
-        }
-        value = NULL;
-
-        gf_time_fmt (timestr, sizeof timestr, snap->time_stamp,
-                     gf_timefmt_FT);
-        value = gf_strdup (timestr);
-
-        if (NULL == value) {
-                ret = -1;
-                goto out;
+            ret = -1;
+            goto out;
         }
 
-        snprintf (key, sizeof (key), "%s.snap-time", keyprefix);
-        ret = dict_set_dynstr (dict, key, value);
+        keylen = snprintf(key, sizeof(key), "%s.snap-desc", keyprefix);
+        ret = dict_set_dynstrn(dict, key, keylen, value);
         if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "Failed to set "
-                                "snap time stamp in dictionary");
-                goto out;
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+                   "Failed to set "
+                   "snap description in dictionary");
+            goto out;
         }
         value = NULL;
+    }
 
-        /* If snap description is provided then add that into dictionary */
-        if (NULL != snap->description) {
-                value = gf_strdup (snap->description);
-                if (NULL == value) {
-                        ret = -1;
-                        goto out;
-                }
-
-                snprintf (key, sizeof (key), "%s.snap-desc", keyprefix);
-                ret = dict_set_dynstr (dict, key, value);
-                if (ret) {
-                        gf_log (this->name, GF_LOG_ERROR, "Failed to set "
-                                        "snap description in dictionary");
-                        goto out;
-                }
-                value = NULL;
-        }
-
-        snprintf (key, sizeof (key), "%s.snap-status", keyprefix);
-        switch (snap->snap_status) {
+    keylen = snprintf(key, sizeof(key), "%s.snap-status", keyprefix);
+    switch (snap->snap_status) {
         case GD_SNAP_STATUS_INIT:
-                ret = dict_set_str (dict, key, "Init");
-                break;
+            ret = dict_set_nstrn(dict, key, keylen, "Init", SLEN("Init"));
+            break;
         case GD_SNAP_STATUS_IN_USE:
-                ret = dict_set_str (dict, key, "In-use");
-                break;
+            ret = dict_set_nstrn(dict, key, keylen, "In-use", SLEN("In-use"));
+            break;
         case GD_SNAP_STATUS_DECOMMISSION:
-                ret = dict_set_str (dict, key, "Decommisioned");
-                break;
+            ret = dict_set_nstrn(dict, key, keylen, "Decommisioned",
+                                 SLEN("Decommisioned"));
+            break;
         case GD_SNAP_STATUS_RESTORED:
-                ret = dict_set_str (dict, key, "Restored");
-                break;
+            ret = dict_set_nstrn(dict, key, keylen, "Restored",
+                                 SLEN("Restored"));
+            break;
         case GD_SNAP_STATUS_NONE:
-                ret = dict_set_str (dict, key, "None");
-                break;
+            ret = dict_set_nstrn(dict, key, keylen, "None", SLEN("None"));
+            break;
         default:
-                gf_log (this->name, GF_LOG_ERROR, "Invalid snap status");
-                ret = -1;
-                goto out;
-        }
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "Failed to set snap status "
-                        "in dictionary");
-                goto out;
-        }
-
-        if (volinfo) {
-                volcount = 1;
-                snprintf (key, sizeof (key), "%s.vol%d", keyprefix, volcount);
-                ret = glusterd_snapshot_get_snapvol_detail (dict,
-                                                        volinfo, key, 0);
-                if (ret) {
-                        gf_log (this->name, GF_LOG_ERROR, "Failed to "
-                                "get volume detail %s for snap %s",
-                                snap_vol->volname, snap->snapname);
-                        goto out;
-                }
-                goto done;
-        }
-
-        cds_list_for_each_entry_safe (snap_vol, tmp_vol, &snap->volumes,
-                                      vol_list) {
-                volcount++;
-                snprintf (key, sizeof (key), "%s.vol%d", keyprefix, volcount);
-                ret = glusterd_snapshot_get_snapvol_detail (dict,
-                                                        snap_vol, key, 1);
-                if (ret) {
-                        gf_log (this->name, GF_LOG_ERROR, "Failed to "
-                                "get volume detail %s for snap %s",
-                                snap_vol->volname, snap->snapname);
-                        goto out;
-                }
-        }
+            gf_msg(this->name, GF_LOG_ERROR, EINVAL, GD_MSG_INVALID_ENTRY,
+                   "Invalid snap status");
+            ret = -1;
+            goto out;
+    }
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+               "Failed to set snap status "
+               "in dictionary");
+        goto out;
+    }
+
+    if (volinfo) {
+        volcount = 1;
+        snprintf(key, sizeof(key), "%s.vol%d", keyprefix, volcount);
+        ret = glusterd_snapshot_get_snapvol_detail(dict, volinfo, key, 0);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, EINVAL, GD_MSG_DICT_GET_FAILED,
+                   "Failed to "
+                   "get volume detail %s for snap %s",
+                   snap_vol->volname, snap->snapname);
+            goto out;
+        }
+        goto done;
+    }
+
+    cds_list_for_each_entry_safe(snap_vol, tmp_vol, &snap->volumes, vol_list)
+    {
+        volcount++;
+        snprintf(key, sizeof(key), "%s.vol%d", keyprefix, volcount);
+        ret = glusterd_snapshot_get_snapvol_detail(dict, snap_vol, key, 1);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+                   "Failed to "
+                   "get volume detail %s for snap %s",
+                   snap_vol->volname, snap->snapname);
+            goto out;
+        }
+    }
 
 done:
-        snprintf (key, sizeof (key), "%s.vol-count", keyprefix);
-        ret = dict_set_int32 (dict, key, volcount);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "Failed to set %s",
-                        key);
-                goto out;
-        }
-
-        ret = 0;
+    keylen = snprintf(key, sizeof(key), "%s.vol-count", keyprefix);
+    ret = dict_set_int32n(dict, key, keylen, volcount);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+               "Failed to set %s", key);
+        goto out;
+    }
+
+    ret = 0;
 out:
-        if (value)
-                GF_FREE (value);
+    if (value)
+        GF_FREE(value);
 
-        return ret;
+    return ret;
 }
 
 static int
-glusterd_snapshot_get_all_snap_info (dict_t *dict)
+glusterd_snapshot_get_all_snap_info(dict_t *dict)
 {
-        int                 ret           = -1;
-        int                 snapcount     = 0;
-        char                key[PATH_MAX] = {0,};
-        glusterd_snap_t    *snap          = NULL;
-        glusterd_snap_t    *tmp_snap      = NULL;
-        glusterd_conf_t    *priv          = NULL;
-        xlator_t           *this          = NULL;
-
-        this = THIS;
-        priv = this->private;
-        GF_ASSERT (priv);
-
-        /* General parameter validation */
-        GF_ASSERT (dict);
-
-        cds_list_for_each_entry_safe (snap, tmp_snap, &priv->snapshots,
-                                      snap_list) {
-                snapcount++;
-                snprintf (key, sizeof (key), "snap%d", snapcount);
-                ret = glusterd_snapshot_get_snap_detail (dict, snap, key, NULL);
-                if (ret) {
-                        gf_log (this->name, GF_LOG_ERROR, "Failed to get "
-                                "snapdetail for snap %s", snap->snapname);
-                        goto out;
-                }
-        }
-
-        ret = dict_set_int32 (dict, "snapcount", snapcount);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "Failed to set snapcount");
-                goto out;
-        }
-
-        ret = 0;
+    int ret = -1;
+    int snapcount = 0;
+    char key[16] = "";
+    glusterd_snap_t *snap = NULL;
+    glusterd_snap_t *tmp_snap = NULL;
+    glusterd_conf_t *priv = NULL;
+    xlator_t *this = NULL;
+
+    this = THIS;
+    priv = this->private;
+    GF_ASSERT(priv);
+
+    /* General parameter validation */
+    GF_ASSERT(dict);
+
+    cds_list_for_each_entry_safe(snap, tmp_snap, &priv->snapshots, snap_list)
+    {
+        snapcount++;
+        snprintf(key, sizeof(key), "snap%d", snapcount);
+        ret = glusterd_snapshot_get_snap_detail(dict, snap, key, NULL);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+                   "Failed to get "
+                   "snapdetail for snap %s",
+                   snap->snapname);
+            goto out;
+        }
+    }
+
+    ret = dict_set_int32n(dict, "snapcount", SLEN("snapcount"), snapcount);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+               "Failed to set snapcount");
+        goto out;
+    }
+
+    ret = 0;
 out:
-        return ret;
+    return ret;
 }
 
 int
-glusterd_snapshot_get_info_by_volume (dict_t *dict, char *volname,
-                                      char *err_str, size_t len)
+glusterd_snapshot_get_info_by_volume(dict_t *dict, char *volname, char *err_str,
+                                     size_t len)
 {
-        int                  ret           = -1;
-        int                  snapcount     = 0;
-        int                  snap_limit    = 0;
-        char                *value         = NULL;
-        char                 key[PATH_MAX] = "";
-        glusterd_volinfo_t  *volinfo       = NULL;
-        glusterd_volinfo_t  *snap_vol      = NULL;
-        glusterd_volinfo_t  *tmp_vol       = NULL;
-        glusterd_conf_t     *conf          = NULL;
-        xlator_t            *this          = NULL;
-        uint64_t            opt_hard_max   = GLUSTERD_SNAPS_MAX_HARD_LIMIT;
-
-        this = THIS;
-        conf = this->private;
-        GF_ASSERT (conf);
-
-        GF_ASSERT (dict);
-        GF_ASSERT (volname);
-
-        ret = glusterd_volinfo_find (volname, &volinfo);
-        if (ret) {
-                snprintf (err_str, len, "Volume (%s) does not exist", volname);
-                gf_log (this->name, GF_LOG_ERROR, "%s", err_str);
-                goto out;
-        }
-
-        /* "snap-max-hard-limit" might not be set by user explicitly,
-         * in that case it's better to consider the default value.
-         * Hence not erroring out if Key is not found.
-         */
-        ret = dict_get_uint64 (conf->opts,
-                               GLUSTERD_STORE_KEY_SNAP_MAX_HARD_LIMIT,
-                               &opt_hard_max);
-        if (ret) {
-                ret = 0;
-                gf_log (this->name, GF_LOG_DEBUG, "%s is not present in "
-                        "opts dictionary",
-                        GLUSTERD_STORE_KEY_SNAP_MAX_HARD_LIMIT);
-        }
-
-        if (opt_hard_max < volinfo->snap_max_hard_limit) {
-                snap_limit = opt_hard_max;
-                gf_log(this->name, GF_LOG_DEBUG, "system snap-max-hard-limit is"
-                       " lesser than volume snap-max-hard-limit, "
-                       "snap-max-hard-limit value is set to %d", snap_limit);
-        } else {
-                snap_limit = volinfo->snap_max_hard_limit;
-                gf_log(this->name, GF_LOG_DEBUG, "volume snap-max-hard-limit is"
-                       " lesser than system snap-max-hard-limit, "
-                       "snap-max-hard-limit value is set to %d", snap_limit);
-        }
-
-        if (snap_limit > volinfo->snap_count)
-                ret = dict_set_int32 (dict, "snaps-available",
-                        snap_limit - volinfo->snap_count);
-        else
-                ret = dict_set_int32 (dict, "snaps-available", 0);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "Failed to set available snaps");
-                goto out;
-        }
-
-        /* Origin volume name */
-        value = gf_strdup (volinfo->volname);
-        if (!value)
-                goto out;
-
-        ret = dict_set_dynstr (dict, "origin-volname", value);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "Failed to set parent "
-                        "volume name in dictionary: %s", key);
-                goto out;
-        }
-        value = NULL;
-
-        cds_list_for_each_entry_safe (snap_vol, tmp_vol, &volinfo->snap_volumes,
-                                      snapvol_list) {
-                snapcount++;
-                snprintf (key, sizeof (key), "snap%d", snapcount);
-                ret = glusterd_snapshot_get_snap_detail (dict,
-                                                         snap_vol->snapshot,
-                                                         key, snap_vol);
-                if (ret) {
-                        gf_log (this->name, GF_LOG_ERROR, "Failed to get "
-                                "snapdetail for snap %s",
-                                snap_vol->snapshot->snapname);
-                        goto out;
-                }
-        }
-        ret = dict_set_int32 (dict, "snapcount", snapcount);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "Failed to set snapcount");
-                goto out;
-        }
-
+    int ret = -1;
+    int snapcount = 0;
+    int snap_limit = 0;
+    char *value = NULL;
+    char key[16] = "";
+    glusterd_volinfo_t *volinfo = NULL;
+    glusterd_volinfo_t *snap_vol = NULL;
+    glusterd_volinfo_t *tmp_vol = NULL;
+    glusterd_conf_t *conf = NULL;
+    xlator_t *this = NULL;
+    uint64_t opt_hard_max = GLUSTERD_SNAPS_MAX_HARD_LIMIT;
+
+    this = THIS;
+    conf = this->private;
+    GF_ASSERT(conf);
+
+    GF_ASSERT(dict);
+    GF_ASSERT(volname);
+
+    ret = glusterd_volinfo_find(volname, &volinfo);
+    if (ret) {
+        snprintf(err_str, len, "Volume (%s) does not exist", volname);
+        gf_msg(this->name, GF_LOG_ERROR, EINVAL, GD_MSG_VOL_NOT_FOUND, "%s",
+               err_str);
+        goto out;
+    }
+
+    /* "snap-max-hard-limit" might not be set by user explicitly,
+     * in that case it's better to consider the default value.
+     * Hence not erroring out if Key is not found.
+     */
+    ret = dict_get_uint64(conf->opts, GLUSTERD_STORE_KEY_SNAP_MAX_HARD_LIMIT,
+                          &opt_hard_max);
+    if (ret) {
         ret = 0;
+        gf_msg_debug(this->name, 0,
+                     "%s is not present in "
+                     "opts dictionary",
+                     GLUSTERD_STORE_KEY_SNAP_MAX_HARD_LIMIT);
+    }
+
+    if (opt_hard_max < volinfo->snap_max_hard_limit) {
+        snap_limit = opt_hard_max;
+        gf_msg_debug(this->name, 0,
+                     "system snap-max-hard-limit is"
+                     " lesser than volume snap-max-hard-limit, "
+                     "snap-max-hard-limit value is set to %d",
+                     snap_limit);
+    } else {
+        snap_limit = volinfo->snap_max_hard_limit;
+        gf_msg_debug(this->name, 0,
+                     "volume snap-max-hard-limit is"
+                     " lesser than system snap-max-hard-limit, "
+                     "snap-max-hard-limit value is set to %d",
+                     snap_limit);
+    }
+
+    if (snap_limit > volinfo->snap_count)
+        ret = dict_set_int32n(dict, "snaps-available", SLEN("snaps-available"),
+                              snap_limit - volinfo->snap_count);
+    else
+        ret = dict_set_int32n(dict, "snaps-available", SLEN("snaps-available"),
+                              0);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+               "Failed to set available snaps");
+        goto out;
+    }
+
+    /* Origin volume name */
+    value = gf_strdup(volinfo->volname);
+    if (!value)
+        goto out;
+
+    ret = dict_set_dynstrn(dict, "origin-volname", SLEN("origin-volname"),
+                           value);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+               "Failed to set parent "
+               "volume name in dictionary: %s",
+               value);
+        goto out;
+    }
+    value = NULL;
+
+    cds_list_for_each_entry_safe(snap_vol, tmp_vol, &volinfo->snap_volumes,
+                                 snapvol_list)
+    {
+        snapcount++;
+        snprintf(key, sizeof(key), "snap%d", snapcount);
+        ret = glusterd_snapshot_get_snap_detail(dict, snap_vol->snapshot, key,
+                                                snap_vol);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+                   "Failed to get "
+                   "snapdetail for snap %s",
+                   snap_vol->snapshot->snapname);
+            goto out;
+        }
+    }
+    ret = dict_set_int32n(dict, "snapcount", SLEN("snapcount"), snapcount);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+               "Failed to set snapcount");
+        goto out;
+    }
+
+    ret = 0;
 out:
-        if (value)
-                GF_FREE (value);
+    if (value)
+        GF_FREE(value);
 
-        return ret;
+    return ret;
 }
 
 /* This function will be called from RPC handler routine.
@@ -3324,272 +3586,280 @@ out:
  * @return -1 on error and 0 on success
  */
 int
-glusterd_handle_snapshot_info (rpcsvc_request_t *req, glusterd_op_t op,
-                               dict_t *dict, char *err_str, size_t len)
+glusterd_handle_snapshot_info(rpcsvc_request_t *req, glusterd_op_t op,
+                              dict_t *dict, char *err_str, size_t len)
 {
-        int              ret             =       -1;
-        int8_t          snap_driven      =        1;
-        char            *volname         =      NULL;
-        char            *snapname        =      NULL;
-        glusterd_snap_t *snap            =      NULL;
-        xlator_t        *this            =      NULL;
-        int32_t         cmd              =      GF_SNAP_INFO_TYPE_ALL;
-
-        this = THIS;
-        GF_ASSERT (this);
-
-        GF_VALIDATE_OR_GOTO (this->name, req, out);
-        GF_VALIDATE_OR_GOTO (this->name, dict, out);
-
-
-        ret = dict_get_int32 (dict, "cmd", &cmd);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "Failed to get type "
-                        "of snapshot info");
-                goto out;
-        }
-
-        switch (cmd) {
-                case GF_SNAP_INFO_TYPE_ALL:
-                {
-                        ret = glusterd_snapshot_get_all_snap_info (dict);
-                        if (ret) {
-                                gf_log (this->name, GF_LOG_ERROR,
-                                        "Failed to get info of all snaps");
-                                goto out;
-                        }
-                        break;
-                }
-
-                case GF_SNAP_INFO_TYPE_SNAP:
-                {
-                        ret = dict_get_str (dict, "snapname", &snapname);
-                        if (ret) {
-                                gf_log (this->name, GF_LOG_ERROR,
-                                        "Failed to get snap name");
-                                goto out;
-                        }
-
-                        ret = dict_set_int32 (dict, "snapcount", 1);
-                        if (ret) {
-                                gf_log (this->name, GF_LOG_ERROR,
-                                        "Failed to set snapcount");
-                                goto out;
-                        }
-
-                        snap = glusterd_find_snap_by_name (snapname);
-                        if (!snap) {
-                                snprintf (err_str, len,
-                                          "Snapshot (%s) does not exist",
-                                          snapname);
-                                gf_log (this->name, GF_LOG_ERROR,
-                                        "%s", err_str);
-                                ret = -1;
-                                goto out;
-                        }
-                        ret = glusterd_snapshot_get_snap_detail (dict, snap,
-                                                                "snap1", NULL);
-                        if (ret) {
-                                gf_log (this->name, GF_LOG_ERROR,
-                                        "Failed to get snap detail of snap "
-                                        "%s", snap->snapname);
-                                goto out;
-                        }
-                        break;
-                }
-
-                case GF_SNAP_INFO_TYPE_VOL:
-                {
-                        ret = dict_get_str (dict, "volname", &volname);
-                        if (ret) {
-                                gf_log (this->name, GF_LOG_ERROR,
-                                        "Failed to get volname");
-                                goto out;
-                        }
-                        ret = glusterd_snapshot_get_info_by_volume (dict,
-                                                        volname, err_str, len);
-                        if (ret) {
-                                gf_log (this->name, GF_LOG_ERROR,
-                                        "Failed to get volume info of volume "
-                                        "%s", volname);
-                                goto out;
-                        }
-                        snap_driven = 0;
-                        break;
-                }
-        }
-
-        ret = dict_set_int8 (dict, "snap-driven", snap_driven);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "Failed to set snap-driven");
+    int ret = -1;
+    int8_t snap_driven = 1;
+    char *volname = NULL;
+    char *snapname = NULL;
+    glusterd_snap_t *snap = NULL;
+    xlator_t *this = NULL;
+    int32_t cmd = GF_SNAP_INFO_TYPE_ALL;
+
+    this = THIS;
+    GF_ASSERT(this);
+
+    GF_VALIDATE_OR_GOTO(this->name, req, out);
+    GF_VALIDATE_OR_GOTO(this->name, dict, out);
+
+    ret = dict_get_int32n(dict, "sub-cmd", SLEN("sub-cmd"), &cmd);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+               "Failed to get type "
+               "of snapshot info");
+        goto out;
+    }
+
+    switch (cmd) {
+        case GF_SNAP_INFO_TYPE_ALL: {
+            ret = glusterd_snapshot_get_all_snap_info(dict);
+            if (ret) {
+                gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+                       "Failed to get info of all snaps");
+                goto out;
+            }
+            break;
+        }
+
+        case GF_SNAP_INFO_TYPE_SNAP: {
+            ret = dict_get_strn(dict, "snapname", SLEN("snapname"), &snapname);
+            if (ret) {
+                gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+                       "Failed to get snap name");
+                goto out;
+            }
+
+            ret = dict_set_int32n(dict, "snapcount", SLEN("snapcount"), 1);
+            if (ret) {
+                gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+                       "Failed to set snapcount");
+                goto out;
+            }
+
+            snap = glusterd_find_snap_by_name(snapname);
+            if (!snap) {
+                snprintf(err_str, len, "Snapshot (%s) does not exist",
+                         snapname);
+                gf_msg(this->name, GF_LOG_ERROR, EINVAL, GD_MSG_SNAP_NOT_FOUND,
+                       "%s", err_str);
+                ret = -1;
                 goto out;
-        }
-
-        /* If everything is successful then send the response back to cli.
-         * In case of failure the caller of this function will take care
-           of the response */
-        ret = glusterd_op_send_cli_response (op, 0, 0, req, dict, err_str);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "Failed to send cli "
-                                "response");
+            }
+            ret = glusterd_snapshot_get_snap_detail(dict, snap, "snap1", NULL);
+            if (ret) {
+                gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_SNAP_NOT_FOUND,
+                       "Failed to get snap detail of snap "
+                       "%s",
+                       snap->snapname);
                 goto out;
-        }
-
-        ret = 0;
+            }
+            break;
+        }
+
+        case GF_SNAP_INFO_TYPE_VOL: {
+            ret = dict_get_strn(dict, "volname", SLEN("volname"), &volname);
+            if (ret) {
+                gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_VOL_NOT_FOUND,
+                       "Failed to get volname");
+                goto out;
+            }
+            ret = glusterd_snapshot_get_info_by_volume(dict, volname, err_str,
+                                                       len);
+            if (ret) {
+                gf_msg(this->name, GF_LOG_ERROR, EINVAL, GD_MSG_VOL_NOT_FOUND,
+                       "Failed to get volume info of volume "
+                       "%s",
+                       volname);
+                goto out;
+            }
+            snap_driven = 0;
+            break;
+        }
+    }
+
+    ret = dict_set_int8(dict, "snap-driven", snap_driven);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+               "Failed to set snap-driven");
+        goto out;
+    }
+
+    /* If everything is successful then send the response back to cli.
+     * In case of failure the caller of this function will take care
+       of the response */
+    ret = glusterd_op_send_cli_response(op, 0, 0, req, dict, err_str);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_NO_CLI_RESP,
+               "Failed to send cli "
+               "response");
+        goto out;
+    }
+
+    ret = 0;
 
 out:
-        return ret;
+    return ret;
 }
 
 /* This function sets all the snapshot names in the dictionary */
 int
-glusterd_snapshot_get_all_snapnames (dict_t *dict)
+glusterd_snapshot_get_all_snapnames(dict_t *dict)
 {
-        int              ret             = -1;
-        int              snapcount       = 0;
-        char            *snapname        = NULL;
-        char             key[PATH_MAX]   = {0,};
-        glusterd_snap_t *snap            = NULL;
-        glusterd_snap_t *tmp_snap        = NULL;
-        glusterd_conf_t *priv            = NULL;
-        xlator_t        *this            = NULL;
-
-        this = THIS;
-        priv = this->private;
-        GF_ASSERT (priv);
-        GF_ASSERT (dict);
-
-        cds_list_for_each_entry_safe (snap, tmp_snap, &priv->snapshots,
-                                      snap_list) {
-                snapcount++;
-                snapname = gf_strdup (snap->snapname);
-                if (!snapname) {
-                        gf_log (this->name, GF_LOG_ERROR, "strdup failed");
-                        ret = -1;
-                        goto out;
-                }
-                snprintf (key, sizeof (key), "snapname%d", snapcount);
-                ret = dict_set_dynstr (dict, key, snapname);
-                if (ret) {
-                        GF_FREE (snapname);
-                        gf_log (this->name, GF_LOG_ERROR, "Failed to set %s",
-                                key);
-                        goto out;
-                }
+    int ret = -1;
+    int snapcount = 0;
+    char *snapname = NULL;
+    char key[64] = "";
+    int keylen;
+    glusterd_snap_t *snap = NULL;
+    glusterd_snap_t *tmp_snap = NULL;
+    glusterd_conf_t *priv = NULL;
+    xlator_t *this = NULL;
+
+    this = THIS;
+    priv = this->private;
+    GF_ASSERT(priv);
+    GF_ASSERT(dict);
+
+    cds_list_for_each_entry_safe(snap, tmp_snap, &priv->snapshots, snap_list)
+    {
+        snapcount++;
+        snapname = gf_strdup(snap->snapname);
+        if (!snapname) {
+            gf_msg(this->name, GF_LOG_ERROR, ENOMEM, GD_MSG_NO_MEMORY,
+                   "strdup failed");
+            ret = -1;
+            goto out;
         }
-
-        ret = dict_set_int32 (dict, "snapcount", snapcount);
+        keylen = snprintf(key, sizeof(key), "snapname%d", snapcount);
+        ret = dict_set_dynstrn(dict, key, keylen, snapname);
         if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "Failed to set snapcount");
-                goto out;
+            GF_FREE(snapname);
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+                   "Failed to set %s", key);
+            goto out;
         }
+    }
 
-        ret = 0;
+    ret = dict_set_int32n(dict, "snapcount", SLEN("snapcount"), snapcount);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+               "Failed to set snapcount");
+        goto out;
+    }
+
+    ret = 0;
 out:
 
-        return ret;
+    return ret;
 }
 
 /* This function sets all the snapshot names
    under a given volume in the dictionary */
 int
-glusterd_snapshot_get_vol_snapnames (dict_t *dict, glusterd_volinfo_t *volinfo)
+glusterd_snapshot_get_vol_snapnames(dict_t *dict, glusterd_volinfo_t *volinfo)
 {
-        int                 ret             = -1;
-        int                 snapcount       = 0;
-        char               *snapname        = NULL;
-        char                key[PATH_MAX]   = {0,};
-        glusterd_volinfo_t *snap_vol        = NULL;
-        glusterd_volinfo_t *tmp_vol         = NULL;
-        xlator_t           *this            = NULL;
-
-        this = THIS;
-        GF_ASSERT (dict);
-        GF_ASSERT (volinfo);
-
-        cds_list_for_each_entry_safe (snap_vol, tmp_vol,
-                                  &volinfo->snap_volumes, snapvol_list) {
-                snapcount++;
-                snprintf (key, sizeof (key), "snapname%d", snapcount);
-
-                ret = dict_set_dynstr_with_alloc (dict, key,
-                                                  snap_vol->snapshot->snapname);
-                if (ret) {
-                        gf_log (this->name, GF_LOG_ERROR, "Failed to "
-                                "set %s", key);
-                        GF_FREE (snapname);
-                        goto out;
-                }
-        }
-
-        ret = dict_set_int32 (dict, "snapcount", snapcount);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "Failed to set snapcount");
-                goto out;
-        }
-
-        ret = 0;
+    int ret = -1;
+    int snapcount = 0;
+    char *snapname = NULL;
+    char key[32] = "";
+    glusterd_volinfo_t *snap_vol = NULL;
+    glusterd_volinfo_t *tmp_vol = NULL;
+    xlator_t *this = NULL;
+
+    this = THIS;
+    GF_ASSERT(dict);
+    GF_ASSERT(volinfo);
+
+    cds_list_for_each_entry_safe(snap_vol, tmp_vol, &volinfo->snap_volumes,
+                                 snapvol_list)
+    {
+        snapcount++;
+        snprintf(key, sizeof(key), "snapname%d", snapcount);
+
+        ret = dict_set_dynstr_with_alloc(dict, key,
+                                         snap_vol->snapshot->snapname);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+                   "Failed to "
+                   "set %s",
+                   key);
+            GF_FREE(snapname);
+            goto out;
+        }
+    }
+
+    ret = dict_set_int32n(dict, "snapcount", SLEN("snapcount"), snapcount);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+               "Failed to set snapcount");
+        goto out;
+    }
+
+    ret = 0;
 out:
 
-        return ret;
+    return ret;
 }
 
 int
-glusterd_handle_snapshot_list (rpcsvc_request_t *req, glusterd_op_t op,
-                               dict_t *dict, char *err_str, size_t len)
+glusterd_handle_snapshot_list(rpcsvc_request_t *req, glusterd_op_t op,
+                              dict_t *dict, char *err_str, size_t len,
+                              uint32_t *op_errno)
 {
-        int                 ret             = -1;
-        char               *volname         = NULL;
-        glusterd_volinfo_t *volinfo         = NULL;
-        xlator_t           *this            = NULL;
-
-        this = THIS;
+    int ret = -1;
+    char *volname = NULL;
+    glusterd_volinfo_t *volinfo = NULL;
+    xlator_t *this = NULL;
 
-        GF_VALIDATE_OR_GOTO (this->name, req, out);
-        GF_VALIDATE_OR_GOTO (this->name, dict, out);
+    this = THIS;
 
-        /* Ignore error for getting volname as it is optional */
-        ret = dict_get_str (dict, "volname", &volname);
+    GF_VALIDATE_OR_GOTO(this->name, req, out);
+    GF_VALIDATE_OR_GOTO(this->name, dict, out);
+    GF_VALIDATE_OR_GOTO(this->name, op_errno, out);
 
-        if (NULL == volname) {
-                ret = glusterd_snapshot_get_all_snapnames (dict);
-                if (ret) {
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "Failed to get snapshot list");
-                        goto out;
-                }
-        } else {
-                ret = glusterd_volinfo_find (volname, &volinfo);
-                if (ret) {
-                        snprintf (err_str, len,
-                                 "Volume (%s) does not exist", volname);
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "%s", err_str);
-                        goto out;
-                }
+    /* Ignore error for getting volname as it is optional */
+    ret = dict_get_strn(dict, "volname", SLEN("volname"), &volname);
 
-                ret = glusterd_snapshot_get_vol_snapnames (dict, volinfo);
-                if (ret) {
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "Failed to get snapshot list for volume %s",
-                                volname);
-                        goto out;
-                }
+    if (NULL == volname) {
+        ret = glusterd_snapshot_get_all_snapnames(dict);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, EINVAL, GD_MSG_SNAP_LIST_GET_FAIL,
+                   "Failed to get snapshot list");
+            goto out;
+        }
+    } else {
+        ret = glusterd_volinfo_find(volname, &volinfo);
+        if (ret) {
+            snprintf(err_str, len, "Volume (%s) does not exist", volname);
+            gf_msg(this->name, GF_LOG_ERROR, EINVAL, GD_MSG_VOL_NOT_FOUND, "%s",
+                   err_str);
+            *op_errno = EG_NOVOL;
+            goto out;
         }
 
-        /* If everything is successful then send the response back to cli.
-        In case of failure the caller of this function will take of response.*/
-        ret = glusterd_op_send_cli_response (op, 0, 0, req, dict, err_str);
+        ret = glusterd_snapshot_get_vol_snapnames(dict, volinfo);
         if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "Failed to send cli "
-                                "response");
-                goto out;
+            gf_msg(this->name, GF_LOG_ERROR, EINVAL, GD_MSG_SNAP_LIST_GET_FAIL,
+                   "Failed to get snapshot list for volume %s", volname);
+            goto out;
         }
+    }
 
-        ret = 0;
+    /* If everything is successful then send the response back to cli.
+    In case of failure the caller of this function will take of response.*/
+    ret = glusterd_op_send_cli_response(op, 0, 0, req, dict, err_str);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_NO_CLI_RESP,
+               "Failed to send cli "
+               "response");
+        goto out;
+    }
+
+    ret = 0;
 
 out:
-        return ret;
+    return ret;
 }
 
 /* This is a snapshot create handler function. This function will be
@@ -3605,169 +3875,185 @@ out:
  * @return              Negative value on Failure and 0 in success
  */
 int
-glusterd_handle_snapshot_create (rpcsvc_request_t *req, glusterd_op_t op,
-                               dict_t *dict, char *err_str, size_t len)
+glusterd_handle_snapshot_create(rpcsvc_request_t *req, glusterd_op_t op,
+                                dict_t *dict, char *err_str, size_t len)
 {
-        int           ret                                  = -1;
-        char         *volname                              = NULL;
-        char         *snapname                             = NULL;
-        int64_t       volcount                             = 0;
-        xlator_t     *this                                 = NULL;
-        char          key[PATH_MAX]                        = "";
-        char         *username                             = NULL;
-        char         *password                             = NULL;
-        uuid_t       *uuid_ptr                             = NULL;
-        uuid_t        tmp_uuid                             = {0};
-        int           i                                    = 0;
-        gf_boolean_t  timestamp                            = _gf_false;
-        char          snap_volname[GD_VOLUME_NAME_MAX]     = {0, };
-        char          new_snapname[GLUSTERD_MAX_SNAP_NAME] = {0, };
-        char          gmt_snaptime[GLUSTERD_MAX_SNAP_NAME] = {0, };
-        time_t        snap_time;
-        this = THIS;
-        GF_ASSERT (this);
-        GF_ASSERT (req);
-        GF_ASSERT (dict);
-        GF_ASSERT (err_str);
-
-        ret = dict_get_int64 (dict, "volcount", &volcount);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "failed to "
-                        "get the volume count");
-                goto out;
-        }
-        if (volcount <= 0) {
-                gf_log (this->name, GF_LOG_ERROR, "Invalid volume count %"PRId64
-                        " supplied", volcount);
-                ret = -1;
-                goto out;
-        }
-
-        ret = dict_get_str (dict, "snapname", &snapname);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "failed to get the snapname");
-                goto out;
-        }
-
-        timestamp = dict_get_str_boolean (dict, "no-timestamp", _gf_false);
-        if (ret) {
-                gf_log (this->name, GF_LOG_DEBUG, "no-timestamp flag "
-                        "is not set");
-                goto out;
-        }
-
-        ret = dict_set_int64 (dict, "snap-time", (int64_t)time(&snap_time));
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "Unable to set snap-time");
-                goto out;
-        }
-
-        if (!timestamp) {
-                strftime (gmt_snaptime, sizeof (gmt_snaptime),
-                          "_GMT-%Y.%m.%d-%H.%M.%S", gmtime(&snap_time));
-                snprintf (new_snapname, sizeof (new_snapname), "%s%s",
-                          snapname, gmt_snaptime);
-                ret = dict_set_dynstr_with_alloc (dict, "snapname",
-                                                  new_snapname);
-                if (ret) {
-                        gf_log (this->name, GF_LOG_ERROR, "Unable to update "
-                                "snap-name");
-                        goto out;
-                }
-                snapname = new_snapname;
-        }
-
-        if (strlen(snapname) >= GLUSTERD_MAX_SNAP_NAME) {
-                snprintf (err_str, len, "snapname cannot exceed 255 "
-                          "characters");
-                gf_log (this->name, GF_LOG_ERROR, "%s", err_str);
-                ret = -1;
-                goto out;
-        }
-
-        uuid_ptr = GF_CALLOC (1, sizeof(uuid_t), gf_common_mt_uuid_t);
+    int ret = -1;
+    char *volname = NULL;
+    char *snapname = NULL;
+    int64_t volcount = 0;
+    xlator_t *this = NULL;
+    char key[64] = "";
+    int keylen;
+    char *username = NULL;
+    char *password = NULL;
+    uuid_t *uuid_ptr = NULL;
+    uuid_t tmp_uuid = {0};
+    int i = 0;
+    int timestamp = 0;
+    char snap_volname[GD_VOLUME_NAME_MAX] = "";
+    char new_snapname[GLUSTERD_MAX_SNAP_NAME] = "";
+    char gmt_snaptime[GLUSTERD_MAX_SNAP_NAME] = "";
+    time_t snap_time;
+    this = THIS;
+    GF_ASSERT(this);
+    GF_ASSERT(req);
+    GF_ASSERT(dict);
+    GF_ASSERT(err_str);
+
+    ret = dict_get_int64(dict, "volcount", &volcount);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+               "failed to "
+               "get the volume count");
+        goto out;
+    }
+    if (volcount <= 0) {
+        gf_msg(this->name, GF_LOG_ERROR, EINVAL, GD_MSG_INVALID_ENTRY,
+               "Invalid volume count %" PRId64 " supplied", volcount);
+        ret = -1;
+        goto out;
+    }
+
+    ret = dict_get_strn(dict, "snapname", SLEN("snapname"), &snapname);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+               "failed to get the snapname");
+        goto out;
+    }
+
+    timestamp = dict_get_str_boolean(dict, "no-timestamp", _gf_false);
+    if (timestamp == -1) {
+        gf_log(this->name, GF_LOG_ERROR,
+               "Failed to get "
+               "no-timestamp flag ");
+        goto out;
+    }
+
+    snap_time = gf_time();
+    ret = dict_set_int64(dict, "snap-time", (int64_t)snap_time);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+               "Unable to set snap-time");
+        goto out;
+    }
+
+    if (!timestamp) {
+        strftime(gmt_snaptime, sizeof(gmt_snaptime), "_GMT-%Y.%m.%d-%H.%M.%S",
+                 gmtime(&snap_time));
+        snprintf(new_snapname, sizeof(new_snapname), "%s%s", snapname,
+                 gmt_snaptime);
+        ret = dict_set_dynstr_with_alloc(dict, "snapname", new_snapname);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+                   "Unable to update "
+                   "snap-name");
+            goto out;
+        }
+        snapname = new_snapname;
+    }
+
+    if (strlen(snapname) >= GLUSTERD_MAX_SNAP_NAME) {
+        snprintf(err_str, len,
+                 "snapname cannot exceed 255 "
+                 "characters");
+        gf_msg(this->name, GF_LOG_ERROR, EINVAL, GD_MSG_INVALID_ENTRY, "%s",
+               err_str);
+        ret = -1;
+        goto out;
+    }
+
+    uuid_ptr = GF_MALLOC(sizeof(uuid_t), gf_common_mt_uuid_t);
+    if (!uuid_ptr) {
+        gf_msg(this->name, GF_LOG_ERROR, ENOMEM, GD_MSG_NO_MEMORY,
+               "Out Of Memory");
+        ret = -1;
+        goto out;
+    }
+
+    gf_uuid_generate(*uuid_ptr);
+    ret = dict_set_bin(dict, "snap-id", uuid_ptr, sizeof(uuid_t));
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+               "Unable to set snap-id");
+        GF_FREE(uuid_ptr);
+        goto out;
+    }
+    uuid_ptr = NULL;
+
+    for (i = 1; i <= volcount; i++) {
+        keylen = snprintf(key, sizeof(key), "volname%d", i);
+        ret = dict_get_strn(dict, key, keylen, &volname);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+                   "Failed to get volume name");
+            goto out;
+        }
+
+        /* generate internal username and password  for the snap*/
+        gf_uuid_generate(tmp_uuid);
+        username = gf_strdup(uuid_utoa(tmp_uuid));
+        keylen = snprintf(key, sizeof(key), "volume%d_username", i);
+        ret = dict_set_dynstrn(dict, key, keylen, username);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+                   "Failed to set snap "
+                   "username for volume %s",
+                   volname);
+            GF_FREE(username);
+            goto out;
+        }
+
+        gf_uuid_generate(tmp_uuid);
+        password = gf_strdup(uuid_utoa(tmp_uuid));
+        keylen = snprintf(key, sizeof(key), "volume%d_password", i);
+        ret = dict_set_dynstrn(dict, key, keylen, password);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+                   "Failed to set snap "
+                   "password for volume %s",
+                   volname);
+            GF_FREE(password);
+            goto out;
+        }
+
+        uuid_ptr = GF_MALLOC(sizeof(uuid_t), gf_common_mt_uuid_t);
         if (!uuid_ptr) {
-                gf_log (this->name, GF_LOG_ERROR, "Out Of Memory");
-                ret = -1;
-                goto out;
+            gf_msg(this->name, GF_LOG_ERROR, ENOMEM, GD_MSG_NO_MEMORY,
+                   "Out Of Memory");
+            ret = -1;
+            goto out;
         }
 
-        gf_uuid_generate (*uuid_ptr);
-        ret = dict_set_bin (dict, "snap-id", uuid_ptr, sizeof(uuid_t));
+        snprintf(key, sizeof(key), "vol%d_volid", i);
+        gf_uuid_generate(*uuid_ptr);
+        ret = dict_set_bin(dict, key, uuid_ptr, sizeof(uuid_t));
         if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "Unable to set snap-id");
-                GF_FREE (uuid_ptr);
-                goto out;
-        }
-        uuid_ptr = NULL;
-
-        for (i = 1; i <= volcount; i++) {
-                snprintf (key, sizeof (key), "volname%d", i);
-                ret = dict_get_str (dict, key, &volname);
-                if (ret) {
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "Failed to get volume name");
-                        goto out;
-                }
-
-                /* generate internal username and password  for the snap*/
-                gf_uuid_generate (tmp_uuid);
-                username = gf_strdup (uuid_utoa (tmp_uuid));
-                snprintf (key, sizeof(key), "volume%d_username", i);
-                ret = dict_set_dynstr (dict, key, username);
-                if (ret) {
-                        gf_log (this->name, GF_LOG_ERROR, "Failed to set snap "
-                                "username for volume %s", volname);
-                        GF_FREE (username);
-                        goto out;
-                }
-
-                gf_uuid_generate (tmp_uuid);
-                password = gf_strdup (uuid_utoa (tmp_uuid));
-                snprintf (key, sizeof(key), "volume%d_password", i);
-                ret = dict_set_dynstr (dict, key, password);
-                if (ret) {
-                        gf_log (this->name, GF_LOG_ERROR, "Failed to set snap "
-                                "password for volume %s", volname);
-                        GF_FREE (password);
-                        goto out;
-                }
-
-                uuid_ptr = GF_CALLOC (1, sizeof(uuid_t), gf_common_mt_uuid_t);
-                if (!uuid_ptr) {
-                        gf_log (this->name, GF_LOG_ERROR, "Out Of Memory");
-                        ret = -1;
-                        goto out;
-                }
-
-                snprintf (key, sizeof(key) - 1, "vol%d_volid", i);
-                gf_uuid_generate (*uuid_ptr);
-                ret = dict_set_bin (dict, key, uuid_ptr, sizeof(uuid_t));
-                if (ret) {
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "Unable to set snap_volid");
-                        GF_FREE (uuid_ptr);
-                        goto out;
-                }
-                GLUSTERD_GET_UUID_NOHYPHEN (snap_volname, *uuid_ptr);
-                snprintf (key, sizeof (key), "snap-volname%d", i);
-                ret = dict_set_dynstr_with_alloc (dict, key, snap_volname);
-                if (ret) {
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "Unable to set snap volname");
-                        GF_FREE (uuid_ptr);
-                        goto out;
-                }
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+                   "Unable to set snap_volid");
+            GF_FREE(uuid_ptr);
+            goto out;
         }
-
-        ret = glusterd_mgmt_v3_initiate_snap_phases (req, op, dict);
+        GLUSTERD_GET_UUID_NOHYPHEN(snap_volname, *uuid_ptr);
+        snprintf(key, sizeof(key), "snap-volname%d", i);
+        ret = dict_set_dynstr_with_alloc(dict, key, snap_volname);
         if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "Failed to initiate snap "
-                                "phases");
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+                   "Unable to set snap volname");
+            GF_FREE(uuid_ptr);
+            goto out;
         }
+    }
+
+    ret = glusterd_mgmt_v3_initiate_snap_phases(req, op, dict);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_SNAP_INIT_FAIL,
+               "Failed to initiate snap "
+               "phases");
+    }
 
 out:
-        return ret;
+    return ret;
 }
 
 /* This is a snapshot status handler function. This function will be
@@ -3786,30 +4072,30 @@ out:
  *
  */
 int
-glusterd_handle_snapshot_status (rpcsvc_request_t *req, glusterd_op_t op,
-                                 dict_t *dict, char *err_str, size_t len)
+glusterd_handle_snapshot_status(rpcsvc_request_t *req, glusterd_op_t op,
+                                dict_t *dict, char *err_str, size_t len)
 {
-        int                     ret             =       -1;
-        xlator_t                *this           =       NULL;
-
-        this = THIS;
-        GF_ASSERT (this);
+    int ret = -1;
+    xlator_t *this = NULL;
 
-        GF_ASSERT (req);
-        GF_ASSERT (dict);
-        GF_ASSERT (err_str);
+    this = THIS;
+    GF_ASSERT(this);
 
+    GF_ASSERT(req);
+    GF_ASSERT(dict);
+    GF_ASSERT(err_str);
 
-        ret = glusterd_mgmt_v3_initiate_snap_phases (req, op, dict);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "Failed to initiate "
-                        "snap phases");
-                goto out;
-        }
+    ret = glusterd_mgmt_v3_initiate_snap_phases(req, op, dict);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_SNAP_INIT_FAIL,
+               "Failed to initiate "
+               "snap phases");
+        goto out;
+    }
 
-        ret = 0;
+    ret = 0;
 out:
-        return ret;
+    return ret;
 }
 
 /* This is a snapshot clone handler function. This function will be
@@ -3825,133 +4111,143 @@ out:
  * @return              Negative value on Failure and 0 in success
  */
 int
-glusterd_handle_snapshot_clone (rpcsvc_request_t *req, glusterd_op_t op,
+glusterd_handle_snapshot_clone(rpcsvc_request_t *req, glusterd_op_t op,
                                dict_t *dict, char *err_str, size_t len)
 {
-        int           ret                              = -1;
-        char         *clonename                        = NULL;
-        char         *snapname                         = NULL;
-        int64_t       volcount                         = 0;
-        xlator_t     *this                             = NULL;
-        char          key[PATH_MAX]                    = "";
-        char         *username                         = NULL;
-        char         *password                         = NULL;
-        char         *volname                          = NULL;
-        uuid_t       *uuid_ptr                         = NULL;
-        uuid_t        tmp_uuid                         = {0};
-        int           i                                = 0;
-        char          snap_volname[GD_VOLUME_NAME_MAX] = {0, };
-
-        this = THIS;
-        GF_ASSERT (this);
-        GF_ASSERT (req);
-        GF_ASSERT (dict);
-        GF_ASSERT (err_str);
-
-        ret = dict_get_str (dict, "clonename", &clonename);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "failed to "
-                        "get the clone name");
-                goto out;
-        }
-        /*We need to take a volume lock on clone name*/
-        volname = gf_strdup (clonename);
-        snprintf (key, sizeof(key), "volname1");
-        ret = dict_set_dynstr (dict, key, volname);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "Failed to set clone "
-                        "name for volume locking");
-                GF_FREE (volname);
-                goto out;
-        }
-
-        ret = dict_get_str (dict, "snapname", &snapname);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "failed to get the snapname");
-                goto out;
-        }
-
-        uuid_ptr = GF_CALLOC (1, sizeof(uuid_t), gf_common_mt_uuid_t);
-        if (!uuid_ptr) {
-                gf_log (this->name, GF_LOG_ERROR, "Out Of Memory");
-                ret = -1;
-                goto out;
-        }
-
-        gf_uuid_generate (*uuid_ptr);
-        ret = dict_set_bin (dict, "clone-id", uuid_ptr, sizeof(uuid_t));
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "Unable to set clone-id");
-                GF_FREE (uuid_ptr);
-                goto out;
-        }
-        uuid_ptr = NULL;
-
-        ret = dict_get_str (dict, "snapname", &snapname);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "Failed to get snapname name");
-                goto out;
-        }
-
-        gf_uuid_generate (tmp_uuid);
-        username = gf_strdup (uuid_utoa (tmp_uuid));
-        snprintf (key, sizeof(key), "volume1_username");
-        ret = dict_set_dynstr (dict, key, username);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "Failed to set clone "
-                        "username for volume %s", clonename);
-                GF_FREE (username);
-                goto out;
-        }
-
-        gf_uuid_generate (tmp_uuid);
-        password = gf_strdup (uuid_utoa (tmp_uuid));
-        snprintf (key, sizeof(key), "volume1_password");
-        ret = dict_set_dynstr (dict, key, password);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "Failed to set clone "
-                        "password for volume %s", clonename);
-                GF_FREE (password);
-                goto out;
-        }
-
-        uuid_ptr = GF_CALLOC (1, sizeof(uuid_t), gf_common_mt_uuid_t);
-        if (!uuid_ptr) {
-                gf_log (this->name, GF_LOG_ERROR, "Out Of Memory");
-                ret = -1;
-                goto out;
-        }
-
-        snprintf (key, sizeof(key) - 1, "vol1_volid");
-        gf_uuid_generate (*uuid_ptr);
-        ret = dict_set_bin (dict, key, uuid_ptr, sizeof(uuid_t));
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "Unable to set clone_volid");
-                GF_FREE (uuid_ptr);
-                goto out;
-        }
-        snprintf (key, sizeof (key), "clone-volname%d", i);
-        ret = dict_set_dynstr_with_alloc (dict, key, snap_volname);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "Unable to set snap volname");
-                GF_FREE (uuid_ptr);
-                goto out;
-        }
-
-        ret = glusterd_mgmt_v3_initiate_snap_phases (req, op, dict);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "Failed to initiate snap "
-                                "phases");
-        }
+    int ret = -1;
+    char *clonename = NULL;
+    char *snapname = NULL;
+    xlator_t *this = NULL;
+    char key[64] = "";
+    int keylen;
+    char *username = NULL;
+    char *password = NULL;
+    char *volname = NULL;
+    uuid_t *uuid_ptr = NULL;
+    uuid_t tmp_uuid = {0};
+    int i = 0;
+    char snap_volname[GD_VOLUME_NAME_MAX] = "";
+
+    this = THIS;
+    GF_ASSERT(this);
+    GF_ASSERT(req);
+    GF_ASSERT(dict);
+    GF_ASSERT(err_str);
+
+    ret = dict_get_strn(dict, "clonename", SLEN("clonename"), &clonename);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+               "failed to "
+               "get the clone name");
+        goto out;
+    }
+    /*We need to take a volume lock on clone name*/
+    volname = gf_strdup(clonename);
+    keylen = snprintf(key, sizeof(key), "volname1");
+    ret = dict_set_dynstrn(dict, key, keylen, volname);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+               "Failed to set clone "
+               "name for volume locking");
+        GF_FREE(volname);
+        goto out;
+    }
+
+    ret = dict_get_strn(dict, "snapname", SLEN("snapname"), &snapname);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+               "failed to get the snapname");
+        goto out;
+    }
+
+    uuid_ptr = GF_MALLOC(sizeof(uuid_t), gf_common_mt_uuid_t);
+    if (!uuid_ptr) {
+        gf_msg(this->name, GF_LOG_ERROR, ENOMEM, GD_MSG_NO_MEMORY,
+               "Out Of Memory");
+        ret = -1;
+        goto out;
+    }
+
+    gf_uuid_generate(*uuid_ptr);
+    ret = dict_set_bin(dict, "clone-id", uuid_ptr, sizeof(uuid_t));
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+               "Unable to set clone-id");
+        GF_FREE(uuid_ptr);
+        goto out;
+    }
+    uuid_ptr = NULL;
+
+    ret = dict_get_strn(dict, "snapname", SLEN("snapname"), &snapname);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+               "Failed to get snapname name");
+        goto out;
+    }
+
+    gf_uuid_generate(tmp_uuid);
+    username = gf_strdup(uuid_utoa(tmp_uuid));
+    keylen = snprintf(key, sizeof(key), "volume1_username");
+    ret = dict_set_dynstrn(dict, key, keylen, username);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+               "Failed to set clone "
+               "username for volume %s",
+               clonename);
+        GF_FREE(username);
+        goto out;
+    }
+
+    gf_uuid_generate(tmp_uuid);
+    password = gf_strdup(uuid_utoa(tmp_uuid));
+    keylen = snprintf(key, sizeof(key), "volume1_password");
+    ret = dict_set_dynstrn(dict, key, keylen, password);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+               "Failed to set clone "
+               "password for volume %s",
+               clonename);
+        GF_FREE(password);
+        goto out;
+    }
+
+    uuid_ptr = GF_MALLOC(sizeof(uuid_t), gf_common_mt_uuid_t);
+    if (!uuid_ptr) {
+        gf_msg(this->name, GF_LOG_ERROR, ENOMEM, GD_MSG_NO_MEMORY,
+               "Out Of Memory");
+        ret = -1;
+        goto out;
+    }
+
+    snprintf(key, sizeof(key), "vol1_volid");
+    gf_uuid_generate(*uuid_ptr);
+    ret = dict_set_bin(dict, key, uuid_ptr, sizeof(uuid_t));
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+               "Unable to set clone_volid");
+        GF_FREE(uuid_ptr);
+        goto out;
+    }
+    snprintf(key, sizeof(key), "clone-volname%d", i);
+    ret = dict_set_dynstr_with_alloc(dict, key, snap_volname);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+               "Unable to set snap volname");
+        GF_FREE(uuid_ptr);
+        goto out;
+    }
+
+    ret = glusterd_mgmt_v3_initiate_snap_phases(req, op, dict);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_SNAP_INIT_FAIL,
+               "Failed to initiate "
+               "snap phases");
+    }
 
 out:
-        return ret;
+    return ret;
 }
 
-
 /* This is a snapshot restore handler function. This function will be
  * executed in the originator node. This function is responsible for
  * calling mgmt_v3 framework to do the actual restore on all the bricks
@@ -3965,264 +4261,284 @@ out:
  * @return              Negative value on Failure and 0 in success
  */
 int
-glusterd_handle_snapshot_restore (rpcsvc_request_t *req, glusterd_op_t op,
-                               dict_t *dict, char *err_str, size_t len)
+glusterd_handle_snapshot_restore(rpcsvc_request_t *req, glusterd_op_t op,
+                                 dict_t *dict, char *err_str,
+                                 uint32_t *op_errno, size_t len)
 {
-        int                     ret             = -1;
-        char                    *snapname       = NULL;
-        char                    *buf            = NULL;
-        glusterd_conf_t         *conf           = NULL;
-        xlator_t                *this           = NULL;
-        glusterd_snap_t         *snap           = NULL;
-        glusterd_volinfo_t      *snap_volinfo   = NULL;
-        int32_t                 i               =    0;
-        char                    key[PATH_MAX]   =    "";
-
-        this = THIS;
-        GF_ASSERT (this);
-        conf = this->private;
-
-        GF_ASSERT (conf);
-        GF_ASSERT (req);
-        GF_ASSERT (dict);
-        GF_ASSERT (err_str);
-
-        ret = dict_get_str (dict, "snapname", &snapname);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "Failed to "
-                        "get snapname");
-                goto out;
-        }
-
-        snap = glusterd_find_snap_by_name (snapname);
-        if (!snap) {
-                snprintf (err_str, len, "Snapshot (%s) does not exist",
-                          snapname);
-                gf_log (this->name, GF_LOG_ERROR, "%s", err_str);
-                ret = -1;
-                goto out;
-        }
-
-        list_for_each_entry (snap_volinfo, &snap->volumes, vol_list) {
-                i++;
-                snprintf (key, sizeof (key), "volname%d", i);
-                buf = gf_strdup (snap_volinfo->parent_volname);
-                if (!buf) {
-                        ret = -1;
-                        goto out;
-                }
-                ret = dict_set_dynstr (dict, key, buf);
-                if (ret) {
-                        gf_log (this->name, GF_LOG_ERROR, "Could not set "
-                                "parent volume name %s in the dict",
-                                snap_volinfo->parent_volname);
-                        GF_FREE (buf);
-                        goto out;
-                }
-                buf = NULL;
-        }
-
-        ret = dict_set_int32 (dict, "volcount", i);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "Could not save volume count");
-                goto out;
-        }
-
-        ret = glusterd_mgmt_v3_initiate_snap_phases (req, op, dict);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "Failed to initiate snap "
-                                "phases");
-                goto out;
-        }
-
-        ret = 0;
+    int ret = -1;
+    char *snapname = NULL;
+    char *buf = NULL;
+    glusterd_conf_t *conf = NULL;
+    xlator_t *this = NULL;
+    glusterd_snap_t *snap = NULL;
+    glusterd_volinfo_t *snap_volinfo = NULL;
+    int32_t i = 0;
+    char key[64] = "";
+    int keylen;
+
+    this = THIS;
+    GF_ASSERT(this);
+    conf = this->private;
+
+    GF_ASSERT(conf);
+    GF_ASSERT(req);
+    GF_ASSERT(dict);
+    GF_ASSERT(err_str);
+
+    ret = dict_get_strn(dict, "snapname", SLEN("snapname"), &snapname);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+               "Failed to "
+               "get snapname");
+        goto out;
+    }
+
+    snap = glusterd_find_snap_by_name(snapname);
+    if (!snap) {
+        snprintf(err_str, len, "Snapshot (%s) does not exist", snapname);
+        *op_errno = EG_NOSNAP;
+        gf_msg(this->name, GF_LOG_ERROR, EINVAL, GD_MSG_SNAP_NOT_FOUND, "%s",
+               err_str);
+        ret = -1;
+        goto out;
+    }
+
+    list_for_each_entry(snap_volinfo, &snap->volumes, vol_list)
+    {
+        i++;
+        keylen = snprintf(key, sizeof(key), "volname%d", i);
+        buf = gf_strdup(snap_volinfo->parent_volname);
+        if (!buf) {
+            ret = -1;
+            goto out;
+        }
+        ret = dict_set_dynstrn(dict, key, keylen, buf);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+                   "Could not set "
+                   "parent volume name %s in the dict",
+                   snap_volinfo->parent_volname);
+            GF_FREE(buf);
+            goto out;
+        }
+        buf = NULL;
+    }
+
+    ret = dict_set_int32n(dict, "volcount", SLEN("volcount"), i);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+               "Could not save volume count");
+        goto out;
+    }
+
+    ret = glusterd_mgmt_v3_initiate_snap_phases(req, op, dict);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_SNAP_INIT_FAIL,
+               "Failed to initiate snap phases");
+        goto out;
+    }
+
+    ret = 0;
 
 out:
-        return ret;
+    return ret;
 }
 
-glusterd_snap_t*
-glusterd_create_snap_object (dict_t *dict, dict_t *rsp_dict)
+glusterd_snap_t *
+glusterd_create_snap_object(dict_t *dict, dict_t *rsp_dict)
 {
-        char              *snapname    = NULL;
-        uuid_t            *snap_id     = NULL;
-        char              *description = NULL;
-        glusterd_snap_t   *snap        = NULL;
-        xlator_t          *this        = NULL;
-        glusterd_conf_t   *priv        = NULL;
-        int                ret         = -1;
-        int64_t            time_stamp  = 0;
-
-        this = THIS;
-        priv = this->private;
-
-        GF_ASSERT (dict);
-        GF_ASSERT (rsp_dict);
-
-        /* Fetch snapname, description, id and time from dict */
-        ret = dict_get_str (dict, "snapname", &snapname);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "Unable to fetch snapname");
-                goto out;
-        }
-
-        /* Ignore ret value for description*/
-        ret = dict_get_str (dict, "description", &description);
-
-        ret = dict_get_bin (dict, "snap-id", (void **)&snap_id);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "Unable to fetch snap_id");
-                goto out;
-        }
-
-        ret = dict_get_int64 (dict, "snap-time", &time_stamp);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "Unable to fetch snap-time");
-                goto out;
-        }
-        if (time_stamp <= 0) {
-                ret = -1;
-                gf_log (this->name, GF_LOG_ERROR, "Invalid time-stamp: %"PRId64,
-                        time_stamp);
-                goto out;
-        }
-
-        cds_list_for_each_entry (snap, &priv->snapshots, snap_list) {
-                if (!strcmp (snap->snapname, snapname) ||
-                    !gf_uuid_compare (snap->snap_id, *snap_id)) {
-                        gf_log (THIS->name, GF_LOG_ERROR,
-                                "Found duplicate snap %s (%s)",
-                                snap->snapname, uuid_utoa (snap->snap_id));
-                        ret = -1;
-                        break;
-                }
-        }
-        if (ret) {
-                snap = NULL;
-                goto out;
-        }
-
-        snap = glusterd_new_snap_object ();
-        if (!snap) {
-                gf_log (this->name, GF_LOG_ERROR, "Could not create "
-                        "the snap object for snap %s", snapname);
-                goto out;
-        }
-
-        strcpy (snap->snapname, snapname);
-        gf_uuid_copy (snap->snap_id, *snap_id);
-        snap->time_stamp = (time_t)time_stamp;
-        /* Set the status as GD_SNAP_STATUS_INIT and once the backend snapshot
-           is taken and snap is really ready to use, set the status to
-           GD_SNAP_STATUS_IN_USE. This helps in identifying the incomplete
-           snapshots and cleaning them up.
-        */
-        snap->snap_status = GD_SNAP_STATUS_INIT;
-        if (description) {
-                snap->description = gf_strdup (description);
-                if (snap->description == NULL) {
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "Saving the Snapshot Description Failed");
-                        ret = -1;
-                        goto out;
-                 }
-        }
-
-        ret = glusterd_store_snap (snap);
-        if (ret) {
-                gf_log (this->name, GF_LOG_WARNING, "Could not store snap"
-                        "object %s", snap->snapname);
-                goto out;
-        }
-
-        glusterd_list_add_order (&snap->snap_list, &priv->snapshots,
-                                 glusterd_compare_snap_time);
-
-        gf_log (this->name, GF_LOG_TRACE, "Snapshot %s added to the list",
-                snap->snapname);
-
-        ret = 0;
+    char *snapname = NULL;
+    uuid_t *snap_id = NULL;
+    char *description = NULL;
+    glusterd_snap_t *snap = NULL;
+    xlator_t *this = NULL;
+    glusterd_conf_t *priv = NULL;
+    int ret = -1;
+    int64_t time_stamp = 0;
+
+    this = THIS;
+    priv = this->private;
+
+    GF_ASSERT(dict);
+    GF_ASSERT(rsp_dict);
+
+    /* Fetch snapname, description, id and time from dict */
+    ret = dict_get_strn(dict, "snapname", SLEN("snapname"), &snapname);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+               "Unable to fetch snapname");
+        goto out;
+    }
+
+    /* Ignore ret value for description*/
+    ret = dict_get_strn(dict, "description", SLEN("description"), &description);
+
+    ret = dict_get_bin(dict, "snap-id", (void **)&snap_id);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+               "Unable to fetch snap_id");
+        goto out;
+    }
+
+    ret = dict_get_int64(dict, "snap-time", &time_stamp);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+               "Unable to fetch snap-time");
+        goto out;
+    }
+    if (time_stamp <= 0) {
+        ret = -1;
+        gf_msg(this->name, GF_LOG_ERROR, EINVAL, GD_MSG_INVALID_ENTRY,
+               "Invalid time-stamp: %" PRId64, time_stamp);
+        goto out;
+    }
+
+    cds_list_for_each_entry(snap, &priv->snapshots, snap_list)
+    {
+        if (!strcmp(snap->snapname, snapname) ||
+            !gf_uuid_compare(snap->snap_id, *snap_id)) {
+            gf_msg(THIS->name, GF_LOG_ERROR, 0, GD_MSG_SNAP_CREATION_FAIL,
+                   "Found duplicate snap %s (%s)", snap->snapname,
+                   uuid_utoa(snap->snap_id));
+            ret = -1;
+            break;
+        }
+    }
+    if (ret) {
+        snap = NULL;
+        goto out;
+    }
+
+    snap = glusterd_new_snap_object();
+    if (!snap) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_SNAP_CREATION_FAIL,
+               "Could not create "
+               "the snap object for snap %s",
+               snapname);
+        goto out;
+    }
+
+    gf_strncpy(snap->snapname, snapname, sizeof(snap->snapname));
+    gf_uuid_copy(snap->snap_id, *snap_id);
+    snap->time_stamp = (time_t)time_stamp;
+    /* Set the status as GD_SNAP_STATUS_INIT and once the backend snapshot
+       is taken and snap is really ready to use, set the status to
+       GD_SNAP_STATUS_IN_USE. This helps in identifying the incomplete
+       snapshots and cleaning them up.
+    */
+    snap->snap_status = GD_SNAP_STATUS_INIT;
+    if (description) {
+        snap->description = gf_strdup(description);
+        if (snap->description == NULL) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_SNAP_CREATION_FAIL,
+                   "Saving the Snapshot Description Failed");
+            ret = -1;
+            goto out;
+        }
+    }
+
+    ret = glusterd_store_snap(snap);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_WARNING, 0, GD_MSG_SNAP_CREATION_FAIL,
+               "Could not store snap"
+               "object %s",
+               snap->snapname);
+        goto out;
+    }
+
+    glusterd_list_add_order(&snap->snap_list, &priv->snapshots,
+                            glusterd_compare_snap_time);
+
+    gf_msg_trace(this->name, 0, "Snapshot %s added to the list",
+                 snap->snapname);
+
+    ret = 0;
 
 out:
-        if (ret) {
-                if (snap)
-                        glusterd_snap_remove (rsp_dict, snap,
-                                              _gf_true, _gf_true);
-                snap = NULL;
-        }
+    if (ret) {
+        if (snap)
+            glusterd_snap_remove(rsp_dict, snap, _gf_true, _gf_true, _gf_false);
+        snap = NULL;
+    }
 
-        return snap;
+    return snap;
 }
 
 /* Added missed_snap_entry to rsp_dict */
 int32_t
-glusterd_add_missed_snaps_to_dict (dict_t *rsp_dict,
-                                   glusterd_volinfo_t *snap_vol,
-                                   glusterd_brickinfo_t *brickinfo,
-                                   int32_t brick_number, int32_t op)
+glusterd_add_missed_snaps_to_dict(dict_t *rsp_dict,
+                                  glusterd_volinfo_t *snap_vol,
+                                  glusterd_brickinfo_t *brickinfo,
+                                  int32_t brick_number, int32_t op)
 {
-        char                   *snap_uuid                       = NULL;
-        char                    missed_snap_entry[PATH_MAX]     = "";
-        char                    name_buf[PATH_MAX]              = "";
-        int32_t                 missed_snap_count               = -1;
-        int32_t                 ret                             = -1;
-        xlator_t               *this                            = NULL;
-
-        this = THIS;
-        GF_ASSERT (this);
-        GF_ASSERT (rsp_dict);
-        GF_ASSERT (snap_vol);
-        GF_ASSERT (brickinfo);
-
-        snap_uuid = gf_strdup (uuid_utoa (snap_vol->snapshot->snap_id));
-        if (!snap_uuid) {
-                ret = -1;
-                goto out;
-        }
-
-        snprintf (missed_snap_entry, sizeof(missed_snap_entry),
-                  "%s:%s=%s:%d:%s:%d:%d", uuid_utoa(brickinfo->uuid),
-                  snap_uuid, snap_vol->volname, brick_number, brickinfo->path,
-                  op, GD_MISSED_SNAP_PENDING);
-
-        /* Fetch the missed_snap_count from the dict */
-        ret = dict_get_int32 (rsp_dict, "missed_snap_count",
-                              &missed_snap_count);
-        if (ret) {
-                /* Initialize the missed_snap_count for the first time */
-                missed_snap_count = 0;
-        }
-
-        /* Setting the missed_snap_entry in the rsp_dict */
-        snprintf (name_buf, sizeof(name_buf), "missed_snaps_%d",
-                  missed_snap_count);
-        ret = dict_set_dynstr_with_alloc (rsp_dict, name_buf,
-                                          missed_snap_entry);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "Failed to set missed_snap_entry (%s) "
-                        "in the rsp_dict.", missed_snap_entry);
-                goto out;
-        }
-        missed_snap_count++;
-
-        /* Setting the new missed_snap_count in the dict */
-        ret = dict_set_int32 (rsp_dict, "missed_snap_count",
-                              missed_snap_count);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "Failed to set missed_snap_count for %s "
-                        "in the rsp_dict.", missed_snap_entry);
-                goto out;
-        }
+    char *snap_uuid = NULL;
+    char missed_snap_entry[PATH_MAX] = "";
+    char name_buf[PATH_MAX] = "";
+    int32_t missed_snap_count = -1;
+    int32_t ret = -1;
+    xlator_t *this = NULL;
+    int32_t len = 0;
+
+    this = THIS;
+    GF_ASSERT(this);
+    GF_ASSERT(rsp_dict);
+    GF_ASSERT(snap_vol);
+    GF_ASSERT(brickinfo);
+
+    snap_uuid = gf_strdup(uuid_utoa(snap_vol->snapshot->snap_id));
+    if (!snap_uuid) {
+        ret = -1;
+        goto out;
+    }
+
+    len = snprintf(missed_snap_entry, sizeof(missed_snap_entry),
+                   "%s:%s=%s:%d:%s:%d:%d", uuid_utoa(brickinfo->uuid),
+                   snap_uuid, snap_vol->volname, brick_number, brickinfo->path,
+                   op, GD_MISSED_SNAP_PENDING);
+    if ((len < 0) || (len >= sizeof(missed_snap_entry))) {
+        gf_smsg(this->name, GF_LOG_ERROR, 0, GD_MSG_COPY_FAIL, NULL);
+        goto out;
+    }
+
+    /* Fetch the missed_snap_count from the dict */
+    ret = dict_get_int32n(rsp_dict, "missed_snap_count",
+                          SLEN("missed_snap_count"), &missed_snap_count);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+                "Key=missed_snap_count", NULL);
+        /* Initialize the missed_snap_count for the first time */
+        missed_snap_count = 0;
+    }
+
+    /* Setting the missed_snap_entry in the rsp_dict */
+    snprintf(name_buf, sizeof(name_buf), "missed_snaps_%d", missed_snap_count);
+    ret = dict_set_dynstr_with_alloc(rsp_dict, name_buf, missed_snap_entry);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+               "Failed to set missed_snap_entry (%s) "
+               "in the rsp_dict.",
+               missed_snap_entry);
+        goto out;
+    }
+    missed_snap_count++;
+
+    /* Setting the new missed_snap_count in the dict */
+    ret = dict_set_int32n(rsp_dict, "missed_snap_count",
+                          SLEN("missed_snap_count"), missed_snap_count);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+               "Failed to set missed_snap_count for %s "
+               "in the rsp_dict.",
+               missed_snap_entry);
+        goto out;
+    }
 
 out:
-        if (snap_uuid)
-                GF_FREE (snap_uuid);
+    if (snap_uuid)
+        GF_FREE(snap_uuid);
 
-        gf_log (this->name, GF_LOG_TRACE, "Returning %d", ret);
-        return ret;
+    gf_msg_trace(this->name, 0, "Returning %d", ret);
+    return ret;
 }
 
 /* This function actually calls the command (or the API) for taking the
@@ -4231,318 +4547,384 @@ out:
    for glusterd
 */
 int32_t
-glusterd_take_lvm_snapshot (glusterd_brickinfo_t *brickinfo,
-                            char *origin_brick_path)
+glusterd_take_lvm_snapshot(glusterd_brickinfo_t *brickinfo,
+                           char *origin_brick_path)
 {
-        char             msg[NAME_MAX]    = "";
-        char             buf[PATH_MAX]    = "";
-        char            *ptr              = NULL;
-        char            *origin_device    = NULL;
-        int              ret              = -1;
-        gf_boolean_t     match            = _gf_false;
-        runner_t         runner           = {0,};
-        xlator_t        *this             = NULL;
-
-        this = THIS;
-        GF_ASSERT (this);
-        GF_ASSERT (brickinfo);
-        GF_ASSERT (origin_brick_path);
-
-        origin_device = glusterd_get_brick_mount_device (origin_brick_path);
-        if (!origin_device) {
-                gf_log (this->name, GF_LOG_ERROR, "getting device name for "
-                        "the brick %s failed", origin_brick_path);
-                goto out;
-        }
-
-        /* Figuring out if setactivationskip flag is supported or not */
-        runinit (&runner);
-        snprintf (msg, sizeof (msg), "running lvcreate help");
-        runner_add_args (&runner, LVM_CREATE, "--help", NULL);
-        runner_log (&runner, "", GF_LOG_DEBUG, msg);
-        runner_redir (&runner, STDOUT_FILENO, RUN_PIPE);
-        ret = runner_start (&runner);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "Failed to run lvcreate help");
-                runner_end (&runner);
-                goto out;
-        }
-
-        /* Looking for setactivationskip in lvcreate --help */
-        do {
-                ptr = fgets(buf, sizeof(buf),
-                            runner_chio (&runner, STDOUT_FILENO));
-                if (ptr) {
-                        if (strstr(buf, "setactivationskip")) {
-                                match = _gf_true;
-                                break;
-                        }
-                }
-        } while (ptr != NULL);
-        runner_end (&runner);
-
-        /* Taking the actual snapshot */
-        runinit (&runner);
-        snprintf (msg, sizeof (msg), "taking snapshot of the brick %s",
-                  origin_brick_path);
-        if (match == _gf_true)
-                runner_add_args (&runner, LVM_CREATE, "-s", origin_device,
-                                 "--setactivationskip", "n", "--name",
-                                 brickinfo->device_path, NULL);
-        else
-                runner_add_args (&runner, LVM_CREATE, "-s", origin_device,
-                                 "--name", brickinfo->device_path, NULL);
-        runner_log (&runner, this->name, GF_LOG_DEBUG, msg);
-        ret = runner_run (&runner);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "taking snapshot of the "
-                        "brick (%s) of device %s failed",
-                        origin_brick_path, origin_device);
-        }
+    char msg[NAME_MAX] = "";
+    char buf[PATH_MAX] = "";
+    char *ptr = NULL;
+    char *origin_device = NULL;
+    int ret = -1;
+    gf_boolean_t match = _gf_false;
+    runner_t runner = {
+        0,
+    };
+    xlator_t *this = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+    GF_ASSERT(brickinfo);
+    GF_ASSERT(origin_brick_path);
+
+    origin_device = glusterd_get_brick_mount_device(origin_brick_path);
+    if (!origin_device) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_BRICK_GET_INFO_FAIL,
+               "getting device name for "
+               "the brick %s failed",
+               origin_brick_path);
+        goto out;
+    }
+
+    /* Figuring out if setactivationskip flag is supported or not */
+    runinit(&runner);
+    snprintf(msg, sizeof(msg), "running lvcreate help");
+    runner_add_args(&runner, LVM_CREATE, "--help", NULL);
+    runner_log(&runner, "", GF_LOG_DEBUG, msg);
+    runner_redir(&runner, STDOUT_FILENO, RUN_PIPE);
+    ret = runner_start(&runner);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, errno, GD_MSG_LVCREATE_FAIL,
+               "Failed to run lvcreate help");
+        runner_end(&runner);
+        goto out;
+    }
+
+    /* Looking for setactivationskip in lvcreate --help */
+    do {
+        ptr = fgets(buf, sizeof(buf), runner_chio(&runner, STDOUT_FILENO));
+        if (ptr) {
+            if (strstr(buf, "setactivationskip")) {
+                match = _gf_true;
+                break;
+            }
+        }
+    } while (ptr != NULL);
+    runner_end(&runner);
+
+    /* Taking the actual snapshot */
+    runinit(&runner);
+    snprintf(msg, sizeof(msg), "taking snapshot of the brick %s",
+             origin_brick_path);
+    if (match == _gf_true)
+        runner_add_args(&runner, LVM_CREATE, "-s", origin_device,
+                        "--setactivationskip", "n", "--name",
+                        brickinfo->device_path, NULL);
+    else
+        runner_add_args(&runner, LVM_CREATE, "-s", origin_device, "--name",
+                        brickinfo->device_path, NULL);
+    runner_log(&runner, this->name, GF_LOG_DEBUG, msg);
+    ret = runner_run(&runner);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_SNAP_CREATION_FAIL,
+               "taking snapshot of the "
+               "brick (%s) of device %s failed",
+               origin_brick_path, origin_device);
+    }
 
 out:
-        return ret;
+    if (origin_device)
+        GF_FREE(origin_device);
+
+    return ret;
 }
 
 int32_t
-glusterd_snap_brick_create (glusterd_volinfo_t *snap_volinfo,
-                            glusterd_brickinfo_t *brickinfo,
-                            int32_t brick_count)
+glusterd_snap_brick_create(glusterd_volinfo_t *snap_volinfo,
+                           glusterd_brickinfo_t *brickinfo, int32_t brick_count,
+                           int32_t clone)
 {
-        int32_t          ret                             = -1;
-        xlator_t        *this                            = NULL;
-        glusterd_conf_t *priv                            = NULL;
-        char             snap_brick_mount_path[PATH_MAX] = "";
-        struct stat      statbuf                         = {0, };
-
-        this = THIS;
-        priv = this->private;
-
-        GF_ASSERT (snap_volinfo);
-        GF_ASSERT (brickinfo);
-
-        snprintf (snap_brick_mount_path, sizeof (snap_brick_mount_path),
-                  "%s/%s/brick%d",  snap_mount_folder, snap_volinfo->volname,
-                  brick_count + 1);
-
-        ret = mkdir_p (snap_brick_mount_path, 0777, _gf_true);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "creating the brick directory"
-                        " %s for the snapshot %s(device: %s) failed",
-                        snap_brick_mount_path, snap_volinfo->volname,
-                        brickinfo->device_path);
-                goto out;
-        }
-        /* mount the snap logical device on the directory inside
-           /run/gluster/snaps/<snapname>/@snap_brick_mount_path
-           Way to mount the snap brick via mount api is this.
-           ret = mount (device, snap_brick_mount_path, entry->mnt_type,
-                        MS_MGC_VAL, "nouuid");
-           But for now, mounting using runner apis.
-        */
-        ret = glusterd_mount_lvm_snapshot (brickinfo, snap_brick_mount_path);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "Failed to mount lvm snapshot.");
-                goto out;
-        }
-
-        ret = stat (brickinfo->path, &statbuf);
-        if (ret) {
-                gf_log (this->name, GF_LOG_WARNING, "stat of the brick %s"
-                        "(brick mount: %s) failed (%s)", brickinfo->path,
-                        snap_brick_mount_path, strerror (errno));
-                goto out;
-        }
-        ret = sys_lsetxattr (brickinfo->path,
-                             GF_XATTR_VOL_ID_KEY,
-                             snap_volinfo->volume_id, 16,
-                             XATTR_REPLACE);
-        if (ret == -1) {
-                gf_log (this->name, GF_LOG_ERROR, "Failed to set "
-                        "extended attribute %s on %s. Reason: "
-                        "%s, snap: %s", GF_XATTR_VOL_ID_KEY,
-                        brickinfo->path, strerror (errno),
-                        snap_volinfo->volname);
-                goto out;
-        }
+    int32_t ret = -1;
+    xlator_t *this = NULL;
+    char snap_brick_mount_path[PATH_MAX] = "";
+    char clone_uuid[64] = "";
+    struct stat statbuf = {
+        0,
+    };
+    int32_t len = 0;
+
+    this = THIS;
+
+    GF_ASSERT(snap_volinfo);
+    GF_ASSERT(brickinfo);
+
+    if (clone) {
+        GLUSTERD_GET_UUID_NOHYPHEN(clone_uuid, snap_volinfo->volume_id);
+        len = snprintf(snap_brick_mount_path, sizeof(snap_brick_mount_path),
+                       "%s/%s/brick%d", snap_mount_dir, clone_uuid,
+                       brick_count + 1);
+    } else {
+        len = snprintf(snap_brick_mount_path, sizeof(snap_brick_mount_path),
+                       "%s/%s/brick%d", snap_mount_dir, snap_volinfo->volname,
+                       brick_count + 1);
+    }
+    if ((len < 0) || (len >= sizeof(snap_brick_mount_path))) {
+        goto out;
+    }
+
+    ret = mkdir_p(snap_brick_mount_path, 0755, _gf_true);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, errno, GD_MSG_DIR_OP_FAILED,
+               "creating the brick directory"
+               " %s for the snapshot %s(device: %s) failed",
+               snap_brick_mount_path, snap_volinfo->volname,
+               brickinfo->device_path);
+        goto out;
+    }
+    /* mount the snap logical device on the directory inside
+       /run/gluster/snaps/<snapname>/@snap_brick_mount_path
+       Way to mount the snap brick via mount api is this.
+       ret = mount (device, snap_brick_mount_path, entry->mnt_type,
+                    MS_MGC_VAL, "nouuid");
+       But for now, mounting using runner apis.
+    */
+    ret = glusterd_mount_lvm_snapshot(brickinfo, snap_brick_mount_path);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_LVM_MOUNT_FAILED,
+               "Failed to mount lvm snapshot.");
+        goto out;
+    }
+
+    ret = sys_stat(brickinfo->path, &statbuf);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_WARNING, errno, GD_MSG_FILE_OP_FAILED,
+               "stat of the brick %s"
+               "(brick mount: %s) failed (%s)",
+               brickinfo->path, snap_brick_mount_path, strerror(errno));
+        goto out;
+    }
+    ret = sys_lsetxattr(brickinfo->path, GF_XATTR_VOL_ID_KEY,
+                        snap_volinfo->volume_id, 16, XATTR_REPLACE);
+    if (ret == -1) {
+        gf_msg(this->name, GF_LOG_ERROR, errno, GD_MSG_SET_XATTR_FAIL,
+               "Failed to set "
+               "extended attribute %s on %s. Reason: "
+               "%s, snap: %s",
+               GF_XATTR_VOL_ID_KEY, brickinfo->path, strerror(errno),
+               snap_volinfo->volname);
+        goto out;
+    }
 
 out:
-        if (ret) {
-                gf_log (this->name, GF_LOG_WARNING, "unmounting the snap brick"
-                        " mount %s", snap_brick_mount_path);
-                /*umount2 system call doesn't cleanup mtab entry after un-mount.
-                  So use external umount command*/
-                glusterd_umount (snap_brick_mount_path);
-        }
-
-        gf_log (this->name, GF_LOG_TRACE, "Returning %d", ret);
-        return ret;
+    if (ret) {
+        gf_msg(this->name, GF_LOG_WARNING, 0, GD_MSG_UMOUNTING_SNAP_BRICK,
+               "unmounting the snap brick"
+               " mount %s",
+               snap_brick_mount_path);
+        /*umount2 system call doesn't cleanup mtab entry after un-mount.
+          So use external umount command*/
+        glusterd_umount(snap_brick_mount_path);
+    }
+
+    gf_msg_trace(this->name, 0, "Returning %d", ret);
+    return ret;
 }
 
 static int32_t
-glusterd_add_brick_to_snap_volume (dict_t *dict, dict_t *rsp_dict,
-                                    glusterd_volinfo_t  *snap_vol,
-                                    glusterd_brickinfo_t *original_brickinfo,
-                                    int64_t volcount, int32_t brick_count,
-                                    int clone)
+glusterd_add_brick_to_snap_volume(dict_t *dict, dict_t *rsp_dict,
+                                  glusterd_volinfo_t *snap_vol,
+                                  glusterd_brickinfo_t *original_brickinfo,
+                                  int64_t volcount, int32_t brick_count,
+                                  int clone)
 {
-        char                    key[PATH_MAX]                   = "";
-        char                   *value                           = NULL;
-        char                   *snap_brick_dir                  = NULL;
-        char                    snap_brick_path[PATH_MAX]       = "";
-        char                   *snap_device                     = NULL;
-        glusterd_brickinfo_t   *snap_brickinfo                  = NULL;
-        gf_boolean_t            add_missed_snap                 = _gf_false;
-        int32_t                 ret                             = -1;
-        xlator_t               *this                            = NULL;
-
-        this = THIS;
-        GF_ASSERT (this);
-        GF_ASSERT (dict);
-        GF_ASSERT (rsp_dict);
-        GF_ASSERT (snap_vol);
-        GF_ASSERT (original_brickinfo);
-
-        snprintf (key, sizeof(key), "vol%"PRId64".origin_brickpath%d",
-                  volcount, brick_count);
-        ret = dict_set_dynstr_with_alloc (dict, key, original_brickinfo->path);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "Failed to set %s", key);
-                goto out;
-        }
-
-        ret = glusterd_brickinfo_new (&snap_brickinfo);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "initializing the brick for the snap "
-                        "volume failed (snapname: %s)",
-                        snap_vol->snapshot->snapname);
-                goto out;
-        }
-
-        snprintf (key, sizeof(key) - 1, "vol%"PRId64".fstype%d", volcount,
-                  brick_count);
-        ret = dict_get_str (dict, key, &value);
-        if (!ret) {
-                /* Update the fstype in original brickinfo as well */
-                strcpy (original_brickinfo->fstype, value);
-                strcpy (snap_brickinfo->fstype, value);
-        } else {
-                if (is_origin_glusterd (dict) == _gf_true)
-                        add_missed_snap = _gf_true;
-        }
-
-        snprintf (key, sizeof(key) - 1, "vol%"PRId64".mnt_opts%d", volcount,
-                  brick_count);
-        ret = dict_get_str (dict, key, &value);
-        if (!ret) {
-                /* Update the mnt_opts in original brickinfo as well */
-                strcpy (original_brickinfo->mnt_opts, value);
-                strcpy (snap_brickinfo->mnt_opts, value);
-        } else {
-                if (is_origin_glusterd (dict) == _gf_true)
-                        add_missed_snap = _gf_true;
-        }
-
-        snprintf (key, sizeof(key) - 1, "vol%"PRId64".brickdir%d", volcount,
-                  brick_count);
-        ret = dict_get_str (dict, key, &snap_brick_dir);
-        if (ret) {
-                /* Using original brickinfo here because it will be a
-                 * pending snapshot and storing the original brickinfo
-                 * will help in mapping while recreating the missed snapshot
-                 */
-                gf_log (this->name, GF_LOG_WARNING, "Unable to fetch "
-                        "snap mount path(%s). Adding to missed_snap_list", key);
-                snap_brickinfo->snap_status = -1;
-
-                snap_brick_dir = original_brickinfo->mount_dir;
-
-                /* In origiator node add snaps missed
-                 * from different nodes to the dict
-                 */
-                if (is_origin_glusterd (dict) == _gf_true)
-                        add_missed_snap = _gf_true;
-        }
-
-        if ((snap_brickinfo->snap_status != -1) &&
-            (!gf_uuid_compare (original_brickinfo->uuid, MY_UUID)) &&
-            (!glusterd_is_brick_started (original_brickinfo))) {
-                /* In case if the brick goes down after prevalidate. */
-                gf_log (this->name, GF_LOG_WARNING, "brick %s:%s is not"
-                        " started (snap: %s)",
-                        original_brickinfo->hostname,
-                        original_brickinfo->path,
-                        snap_vol->snapshot->snapname);
-
-                snap_brickinfo->snap_status = -1;
-                add_missed_snap = _gf_true;
-        }
-
-        if (add_missed_snap) {
-                ret = glusterd_add_missed_snaps_to_dict (rsp_dict,
-                                            snap_vol,
-                                            original_brickinfo,
-                                            brick_count + 1,
-                                            GF_SNAP_OPTION_TYPE_CREATE);
-                if (ret) {
-                        gf_log (this->name, GF_LOG_ERROR, "Failed to add missed"
-                                " snapshot info for %s:%s in the rsp_dict",
-                                original_brickinfo->hostname,
-                                original_brickinfo->path);
-                        goto out;
-                }
-        }
-
-        /* Create brick-path in the format /var/run/gluster/snaps/ *
-         * <snap-uuid>/<original-brick#>/snap-brick-dir *
+    char key[64] = "";
+    int keylen;
+    char *value = NULL;
+    char *snap_brick_dir = NULL;
+    char snap_brick_path[PATH_MAX] = "";
+    char clone_uuid[64] = "";
+    char *snap_device = NULL;
+    glusterd_brickinfo_t *snap_brickinfo = NULL;
+    gf_boolean_t add_missed_snap = _gf_false;
+    int32_t ret = -1;
+    xlator_t *this = NULL;
+    char abspath[PATH_MAX] = "";
+    int32_t len = 0;
+
+    this = THIS;
+    GF_ASSERT(this);
+    GF_ASSERT(dict);
+    GF_ASSERT(rsp_dict);
+    GF_ASSERT(snap_vol);
+    GF_ASSERT(original_brickinfo);
+
+    snprintf(key, sizeof(key), "vol%" PRId64 ".origin_brickpath%d", volcount,
+             brick_count);
+    ret = dict_set_dynstr_with_alloc(dict, key, original_brickinfo->path);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+               "Failed to set %s", key);
+        goto out;
+    }
+
+    ret = glusterd_brickinfo_new(&snap_brickinfo);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_BRICK_NEW_INFO_FAIL,
+               "initializing the brick for the snap "
+               "volume failed (snapname: %s)",
+               snap_vol->snapshot->snapname);
+        goto out;
+    }
+
+    keylen = snprintf(key, sizeof(key), "vol%" PRId64 ".fstype%d", volcount,
+                      brick_count);
+    ret = dict_get_strn(dict, key, keylen, &value);
+    if (!ret) {
+        /* Update the fstype in original brickinfo as well */
+        gf_strncpy(original_brickinfo->fstype, value,
+                   sizeof(original_brickinfo->fstype));
+        gf_strncpy(snap_brickinfo->fstype, value,
+                   sizeof(snap_brickinfo->fstype));
+    } else {
+        if (is_origin_glusterd(dict) == _gf_true)
+            add_missed_snap = _gf_true;
+    }
+
+    keylen = snprintf(key, sizeof(key), "vol%" PRId64 ".mnt_opts%d", volcount,
+                      brick_count);
+    ret = dict_get_strn(dict, key, keylen, &value);
+    if (!ret) {
+        /* Update the mnt_opts in original brickinfo as well */
+        gf_strncpy(original_brickinfo->mnt_opts, value,
+                   sizeof(original_brickinfo->mnt_opts));
+        gf_strncpy(snap_brickinfo->mnt_opts, value,
+                   sizeof(snap_brickinfo->mnt_opts));
+    } else {
+        if (is_origin_glusterd(dict) == _gf_true)
+            add_missed_snap = _gf_true;
+    }
+
+    keylen = snprintf(key, sizeof(key), "vol%" PRId64 ".brickdir%d", volcount,
+                      brick_count);
+    ret = dict_get_strn(dict, key, keylen, &snap_brick_dir);
+    if (ret) {
+        /* Using original brickinfo here because it will be a
+         * pending snapshot and storing the original brickinfo
+         * will help in mapping while recreating the missed snapshot
          */
-        snprintf (snap_brick_path, sizeof(snap_brick_path),
-                  "%s/%s/brick%d%s", snap_mount_folder,
-                  snap_vol->volname, brick_count+1,
-                  snap_brick_dir);
-
-        snprintf (key, sizeof(key), "vol%"PRId64".brick_snapdevice%d",
-                  volcount, brick_count);
-        ret = dict_get_str (dict, key, &snap_device);
-        if (ret) {
-                /* If the device name is empty, so will be the brick path
-                 * Hence the missed snap has already been added above
-                 */
-                gf_log (this->name, GF_LOG_ERROR, "Unable to fetch "
-                        "snap device (%s). Leaving empty", key);
-        } else
-                strcpy (snap_brickinfo->device_path, snap_device);
-
-        ret = gf_canonicalize_path (snap_brick_path);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "Failed to canonicalize path");
-                goto out;
-        }
+        gf_msg(this->name, GF_LOG_WARNING, 0, GD_MSG_SNAP_NOT_FOUND,
+               "Unable to fetch "
+               "snap mount path(%s). Adding to missed_snap_list",
+               key);
+        snap_brickinfo->snap_status = -1;
 
-        strcpy (snap_brickinfo->hostname, original_brickinfo->hostname);
-        strcpy (snap_brickinfo->path, snap_brick_path);
-        strcpy (snap_brickinfo->mount_dir, original_brickinfo->mount_dir);
-        gf_uuid_copy (snap_brickinfo->uuid, original_brickinfo->uuid);
-        /* AFR changelog names are based on brick_id and hence the snap
-         * volume's bricks must retain the same ID */
-        cds_list_add_tail (&snap_brickinfo->brick_list, &snap_vol->bricks);
+        snap_brick_dir = original_brickinfo->mount_dir;
 
-        if (clone) {
-                GLUSTERD_ASSIGN_BRICKID_TO_BRICKINFO (snap_brickinfo, snap_vol,
-                                                      brick_count);
-        } else
-                strcpy (snap_brickinfo->brick_id, original_brickinfo->brick_id);
+        /* In origiator node add snaps missed
+         * from different nodes to the dict
+         */
+        if (is_origin_glusterd(dict) == _gf_true)
+            add_missed_snap = _gf_true;
+    }
+
+    if ((snap_brickinfo->snap_status != -1) &&
+        (!gf_uuid_compare(original_brickinfo->uuid, MY_UUID)) &&
+        (!glusterd_is_brick_started(original_brickinfo))) {
+        /* In case if the brick goes down after prevalidate. */
+        gf_msg(this->name, GF_LOG_WARNING, 0, GD_MSG_BRICK_DISCONNECTED,
+               "brick %s:%s is not"
+               " started (snap: %s)",
+               original_brickinfo->hostname, original_brickinfo->path,
+               snap_vol->snapshot->snapname);
+
+        snap_brickinfo->snap_status = -1;
+        add_missed_snap = _gf_true;
+    }
+
+    if (add_missed_snap) {
+        ret = glusterd_add_missed_snaps_to_dict(
+            rsp_dict, snap_vol, original_brickinfo, brick_count + 1,
+            GF_SNAP_OPTION_TYPE_CREATE);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_MISSEDSNAP_INFO_SET_FAIL,
+                   "Failed to add missed"
+                   " snapshot info for %s:%s in the rsp_dict",
+                   original_brickinfo->hostname, original_brickinfo->path);
+            goto out;
+        }
+    }
+
+    /* Create brick-path in the format /var/run/gluster/snaps/ *
+     * <snap-uuid>/<original-brick#>/snap-brick-dir *
+     */
+    if (clone) {
+        GLUSTERD_GET_UUID_NOHYPHEN(clone_uuid, snap_vol->volume_id);
+        len = snprintf(snap_brick_path, sizeof(snap_brick_path),
+                       "%s/%s/brick%d%s", snap_mount_dir, clone_uuid,
+                       brick_count + 1, snap_brick_dir);
+    } else {
+        len = snprintf(snap_brick_path, sizeof(snap_brick_path),
+                       "%s/%s/brick%d%s", snap_mount_dir, snap_vol->volname,
+                       brick_count + 1, snap_brick_dir);
+    }
+    if ((len < 0) || (len >= sizeof(snap_brick_path))) {
+        ret = -1;
+        goto out;
+    }
+
+    keylen = snprintf(key, sizeof(key), "vol%" PRId64 ".brick_snapdevice%d",
+                      volcount, brick_count);
+    ret = dict_get_strn(dict, key, keylen, &snap_device);
+    if (ret) {
+        /* If the device name is empty, so will be the brick path
+         * Hence the missed snap has already been added above
+         */
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_SNAP_NOT_FOUND,
+               "Unable to fetch "
+               "snap device (%s). Leaving empty",
+               key);
+    } else
+        gf_strncpy(snap_brickinfo->device_path, snap_device,
+                   sizeof(snap_brickinfo->device_path));
+
+    ret = gf_canonicalize_path(snap_brick_path);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_CANONICALIZE_FAIL,
+               "Failed to canonicalize path");
+        goto out;
+    }
+
+    gf_strncpy(snap_brickinfo->hostname, original_brickinfo->hostname,
+               sizeof(snap_brickinfo->hostname));
+    gf_strncpy(snap_brickinfo->path, snap_brick_path,
+               sizeof(snap_brickinfo->path));
+
+    if (!realpath(snap_brick_path, abspath)) {
+        /* ENOENT indicates that brick path has not been created which
+         * is a valid scenario */
+        if (errno != ENOENT) {
+            gf_msg(this->name, GF_LOG_CRITICAL, errno,
+                   GD_MSG_BRICKINFO_CREATE_FAIL,
+                   "realpath () "
+                   "failed for brick %s. The underlying filesystem"
+                   " may be in bad state",
+                   snap_brick_path);
+            ret = -1;
+            goto out;
+        }
+    }
+    gf_strncpy(snap_brickinfo->real_path, abspath,
+               sizeof(snap_brickinfo->real_path));
+
+    gf_strncpy(snap_brickinfo->mount_dir, original_brickinfo->mount_dir,
+               sizeof(snap_brickinfo->mount_dir));
+    gf_uuid_copy(snap_brickinfo->uuid, original_brickinfo->uuid);
+    /* AFR changelog names are based on brick_id and hence the snap
+     * volume's bricks must retain the same ID */
+    cds_list_add_tail(&snap_brickinfo->brick_list, &snap_vol->bricks);
+
+    if (clone) {
+        GLUSTERD_ASSIGN_BRICKID_TO_BRICKINFO(snap_brickinfo, snap_vol,
+                                             brick_count);
+    } else
+        gf_strncpy(snap_brickinfo->brick_id, original_brickinfo->brick_id,
+                   sizeof(snap_brickinfo->brick_id));
 
 out:
-        if (ret && snap_brickinfo)
-                GF_FREE (snap_brickinfo);
+    if (ret && snap_brickinfo)
+        GF_FREE(snap_brickinfo);
 
-        gf_log (this->name, GF_LOG_TRACE, "Returning %d", ret);
-        return ret;
+    gf_msg_trace(this->name, 0, "Returning %d", ret);
+    return ret;
 }
 
 /* This function will update the file-system label of the
@@ -4553,328 +4935,536 @@ out:
  * @return 0 on success and -1 on failure
  */
 int
-glusterd_update_fs_label (glusterd_brickinfo_t *brickinfo)
+glusterd_update_fs_label(glusterd_brickinfo_t *brickinfo)
 {
-        int32_t         ret                     = -1;
-        char            msg [PATH_MAX]          = "";
-        char            label [NAME_MAX]        = "";
-        uuid_t          uuid                    = {0,};
-        runner_t        runner                  = {0,};
-        xlator_t       *this                    = NULL;
-
-        this = THIS;
-        GF_ASSERT (this);
-        GF_ASSERT (brickinfo);
-
-        /* Generate a new UUID */
-        gf_uuid_generate (uuid);
-
-        GLUSTERD_GET_UUID_NOHYPHEN (label, uuid);
-
-        runinit (&runner);
+    int32_t ret = -1;
+    char msg[PATH_MAX] = "";
+    char label[NAME_MAX] = "";
+    uuid_t uuid = {
+        0,
+    };
+    runner_t runner = {
+        0,
+    };
+    xlator_t *this = NULL;
+    int32_t len = 0;
+
+    this = THIS;
+    GF_ASSERT(this);
+    GF_ASSERT(brickinfo);
+
+    /* Generate a new UUID */
+    gf_uuid_generate(uuid);
+
+    GLUSTERD_GET_UUID_NOHYPHEN(label, uuid);
+
+    runinit(&runner);
+
+    /* Call the file-system specific tools to update the file-system
+     * label. Currently we are only supporting xfs and ext2/ext3/ext4
+     * file-system.
+     */
+    if (0 == strcmp(brickinfo->fstype, "xfs")) {
+        /* XFS label is of size 12. Therefore we should truncate the
+         * label to 12 bytes*/
+        label[12] = '\0';
+        len = snprintf(msg, sizeof(msg),
+                       "Changing filesystem label "
+                       "of %s brick to %s",
+                       brickinfo->path, label);
+        if (len < 0) {
+            strcpy(msg, "<error>");
+        }
+        /* Run the run xfs_admin tool to change the label
+         * of the file-system */
+        runner_add_args(&runner, "xfs_admin", "-L", label,
+                        brickinfo->device_path, NULL);
+    } else if (0 == strcmp(brickinfo->fstype, "ext4") ||
+               0 == strcmp(brickinfo->fstype, "ext3") ||
+               0 == strcmp(brickinfo->fstype, "ext2")) {
+        /* Ext2/Ext3/Ext4 label is of size 16. Therefore we should
+         * truncate the label to 16 bytes*/
+        label[16] = '\0';
+        len = snprintf(msg, sizeof(msg),
+                       "Changing filesystem label "
+                       "of %s brick to %s",
+                       brickinfo->path, label);
+        if (len < 0) {
+            strcpy(msg, "<error>");
+        }
+        /* For ext2/ext3/ext4 run tune2fs to change the
+         * file-system label */
+        runner_add_args(&runner, "tune2fs", "-L", label, brickinfo->device_path,
+                        NULL);
+    } else {
+        gf_msg(this->name, GF_LOG_WARNING, EOPNOTSUPP, GD_MSG_OP_UNSUPPORTED,
+               "Changing file-system "
+               "label of %s file-system is not supported as of now",
+               brickinfo->fstype);
+        runner_end(&runner);
+        ret = -1;
+        goto out;
+    }
+
+    runner_log(&runner, this->name, GF_LOG_DEBUG, msg);
+    ret = runner_run(&runner);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_FS_LABEL_UPDATE_FAIL,
+               "Failed to change "
+               "filesystem label of %s brick to %s",
+               brickinfo->path, label);
+        goto out;
+    }
+
+    ret = 0;
+out:
+    return ret;
+}
 
-        /* Call the file-system specific tools to update the file-system
-         * label. Currently we are only supporting xfs and ext2/ext3/ext4
+static int32_t
+glusterd_take_brick_snapshot(dict_t *dict, glusterd_volinfo_t *snap_vol,
+                             glusterd_brickinfo_t *brickinfo, int32_t volcount,
+                             int32_t brick_count, int32_t clone)
+{
+    char *origin_brick_path = NULL;
+    char key[64] = "";
+    int keylen;
+    int32_t ret = -1;
+    gf_boolean_t snap_activate = _gf_false;
+    xlator_t *this = NULL;
+    glusterd_conf_t *priv = NULL;
+
+    this = THIS;
+    priv = this->private;
+    GF_ASSERT(this);
+    GF_ASSERT(dict);
+    GF_ASSERT(snap_vol);
+    GF_ASSERT(brickinfo);
+    GF_ASSERT(priv);
+
+    if (strlen(brickinfo->device_path) == 0) {
+        gf_msg(this->name, GF_LOG_ERROR, EINVAL, GD_MSG_INVALID_ENTRY,
+               "Device path is empty "
+               "brick %s:%s",
+               brickinfo->hostname, brickinfo->path);
+        ret = -1;
+        goto out;
+    }
+
+    keylen = snprintf(key, sizeof(key), "vol%d.origin_brickpath%d", volcount,
+                      brick_count);
+    ret = dict_get_strn(dict, key, keylen, &origin_brick_path);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_WARNING, 0, GD_MSG_DICT_GET_FAILED,
+               "Unable to fetch "
+               "brick path (%s)",
+               key);
+        goto out;
+    }
+
+    ret = glusterd_take_lvm_snapshot(brickinfo, origin_brick_path);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_SNAP_CREATION_FAIL,
+               "Failed to take snapshot of "
+               "brick %s:%s",
+               brickinfo->hostname, origin_brick_path);
+        goto out;
+    }
+
+    /* After the snapshot both the origin brick (LVM brick) and
+     * the snapshot brick will have the same file-system label. This
+     * will cause lot of problems at mount time. Therefore we must
+     * generate a new label for the snapshot brick
+     */
+    ret = glusterd_update_fs_label(brickinfo);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_FS_LABEL_UPDATE_FAIL,
+               "Failed to update "
+               "file-system label for %s brick",
+               brickinfo->path);
+        /* Failing to update label should not cause snapshot failure.
+         * Currently label is updated only for XFS and ext2/ext3/ext4
          * file-system.
          */
-        if (0 == strcmp (brickinfo->fstype, "xfs")) {
-                /* XFS label is of size 12. Therefore we should truncate the
-                 * label to 12 bytes*/
-                label [12] = '\0';
-                snprintf (msg, sizeof (msg), "Changing filesystem label of "
-                          "%s brick to %s", brickinfo->path, label);
-                /* Run the run xfs_admin tool to change the label
-                 * of the file-system */
-                runner_add_args (&runner, "xfs_admin", "-L", label,
-                                 brickinfo->device_path, NULL);
-        } else if (0 == strcmp (brickinfo->fstype, "ext4") ||
-                   0 == strcmp (brickinfo->fstype, "ext3") ||
-                   0 == strcmp (brickinfo->fstype, "ext2")) {
-                /* Ext2/Ext3/Ext4 label is of size 16. Therefore we should
-                 * truncate the label to 16 bytes*/
-                label [16] = '\0';
-                snprintf (msg, sizeof (msg), "Changing filesystem label of "
-                          "%s brick to %s", brickinfo->path, label);
-                /* For ext2/ext3/ext4 run tune2fs to change the
-                 * file-system label */
-                runner_add_args (&runner, "tune2fs", "-L", label,
-                                 brickinfo->device_path, NULL);
-        } else {
-                gf_log (this->name, GF_LOG_WARNING, "Changing file-system "
-                        "label of %s file-system is not supported as of now",
-                        brickinfo->fstype);
-                runner_end (&runner);
-                ret = -1;
-                goto out;
-        }
+    }
 
-        runner_log (&runner, this->name, GF_LOG_DEBUG, msg);
-        ret = runner_run (&runner);
+    /* create the complete brick here in case of clone and
+     * activate-on-create configuration.
+     */
+    snap_activate = dict_get_str_boolean(
+        priv->opts, GLUSTERD_STORE_KEY_SNAP_ACTIVATE, _gf_false);
+    if (clone || snap_activate) {
+        ret = glusterd_snap_brick_create(snap_vol, brickinfo, brick_count,
+                                         clone);
         if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "Failed to change "
-                        "filesystem label of %s brick to %s",
-                        brickinfo->path, label);
-                goto out;
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_BRICK_CREATION_FAIL,
+                   "not able to "
+                   "create the brick for the snap %s, volume %s",
+                   snap_vol->snapshot->snapname, snap_vol->volname);
+            goto out;
         }
+    }
 
-        ret = 0;
 out:
-        return ret;
+    gf_msg_trace(this->name, 0, "Returning %d", ret);
+    return ret;
 }
 
-static int32_t
-glusterd_take_brick_snapshot (dict_t *dict, glusterd_volinfo_t *snap_vol,
-                              glusterd_brickinfo_t *brickinfo,
-                              int32_t volcount, int32_t brick_count)
+static int
+glusterd_snap_clear_unsupported_opt(
+    glusterd_volinfo_t *volinfo,
+    struct gd_snap_unsupported_opt_t *unsupported_opt)
 {
-        char                   *origin_brick_path   = NULL;
-        char                    key[PATH_MAX]       = "";
-        int32_t                 ret                 = -1;
-        xlator_t               *this                = NULL;
+    int ret = -1;
+    int i = 0;
 
-        this = THIS;
-        GF_ASSERT (this);
-        GF_ASSERT (dict);
-        GF_ASSERT (snap_vol);
-        GF_ASSERT (brickinfo);
+    GF_VALIDATE_OR_GOTO("glusterd", volinfo, out);
 
-        if (strlen(brickinfo->device_path) == 0) {
-                gf_log (this->name, GF_LOG_ERROR, "Device path is empty "
-                        "brick %s:%s", brickinfo->hostname, brickinfo->path);
+    for (i = 0; unsupported_opt[i].key; i++) {
+        glusterd_volinfo_get(volinfo, unsupported_opt[i].key,
+                             &unsupported_opt[i].value);
+
+        if (unsupported_opt[i].value) {
+            unsupported_opt[i].value = gf_strdup(unsupported_opt[i].value);
+            if (!unsupported_opt[i].value) {
                 ret = -1;
                 goto out;
+            }
+            dict_del(volinfo->dict, unsupported_opt[i].key);
         }
+    }
 
-        snprintf (key, sizeof(key) - 1, "vol%d.origin_brickpath%d", volcount,
-                  brick_count);
-        ret = dict_get_str (dict, key, &origin_brick_path);
-        if (ret) {
-                gf_log (this->name, GF_LOG_WARNING, "Unable to fetch "
-                        "brick path (%s)", key);
-                goto out;
-        }
+    ret = 0;
+out:
+    return ret;
+}
 
-        ret = glusterd_take_lvm_snapshot (brickinfo, origin_brick_path);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "Failed to take snapshot of "
-                        "brick %s:%s", brickinfo->hostname, origin_brick_path);
-                goto out;
-        }
+static int
+glusterd_snap_set_unsupported_opt(
+    glusterd_volinfo_t *volinfo,
+    struct gd_snap_unsupported_opt_t *unsupported_opt)
+{
+    int ret = -1;
+    int i = 0;
 
-        /* After the snapshot both the origin brick (LVM brick) and
-         * the snapshot brick will have the same file-system label. This
-         * will cause lot of problems at mount time. Therefore we must
-         * generate a new label for the snapshot brick
-         */
-        ret = glusterd_update_fs_label (brickinfo);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "Failed to update "
-                        "file-system label for %s brick", brickinfo->path);
-                /* Failing to update label should not cause snapshot failure.
-                 * Currently label is updated only for XFS and ext2/ext3/ext4
-                 * file-system.
-                 */
-        }
+    GF_VALIDATE_OR_GOTO("glusterd", volinfo, out);
 
-        /* create the complete brick here */
-        ret = glusterd_snap_brick_create (snap_vol, brickinfo, brick_count);
+    for (i = 0; unsupported_opt[i].key; i++) {
+        if (!unsupported_opt[i].value)
+            continue;
+
+        ret = dict_set_dynstr(volinfo->dict, unsupported_opt[i].key,
+                              unsupported_opt[i].value);
         if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "not able to"
-                        " create the brick for the snap %s"
-                        ", volume %s", snap_vol->snapshot->snapname,
-                        snap_vol->volname);
-                goto out;
+            gf_msg("glusterd", GF_LOG_ERROR, errno, GD_MSG_DICT_SET_FAILED,
+                   "dict set failed");
+            goto out;
         }
+        unsupported_opt[i].value = NULL;
+    }
 
+    ret = 0;
 out:
-        gf_log (this->name, GF_LOG_TRACE, "Returning %d", ret);
-        return ret;
+    return ret;
 }
 
 glusterd_volinfo_t *
-glusterd_do_snap_vol (glusterd_volinfo_t *origin_vol, glusterd_snap_t *snap,
-                      dict_t *dict, dict_t *rsp_dict, int64_t volcount,
-                      int clone)
+glusterd_do_snap_vol(glusterd_volinfo_t *origin_vol, glusterd_snap_t *snap,
+                     dict_t *dict, dict_t *rsp_dict, int64_t volcount,
+                     int clone)
 {
-        char                    key[PATH_MAX]                   = "";
-        char                   *username                        = NULL;
-        char                   *password                        = NULL;
-        glusterd_brickinfo_t   *brickinfo                       = NULL;
-        glusterd_conf_t        *priv                            = NULL;
-        glusterd_volinfo_t     *snap_vol                        = NULL;
-        uuid_t                 *snap_volid                      = NULL;
-        int32_t                 ret                             = -1;
-        int32_t                 brick_count                     = 0;
-        xlator_t               *this                            = NULL;
-        int64_t                 brick_order                     = 0;
-        char                    *clonename                      = NULL;
-        gf_boolean_t            conf_present                    = _gf_false;
-
-        this = THIS;
-        GF_ASSERT (this);
-
-        priv = this->private;
-        GF_ASSERT (priv);
-        GF_ASSERT (dict);
-        GF_ASSERT (origin_vol);
-        GF_ASSERT (rsp_dict);
-
-        /* fetch username, password and vol_id from dict*/
-        snprintf (key, sizeof(key), "volume%"PRId64"_username", volcount);
-        ret = dict_get_str (dict, key, &username);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "Failed to get %s for "
-                        "snap %s", key, snap->snapname);
-                goto out;
-        }
-        snprintf (key, sizeof(key), "volume%"PRId64"_password", volcount);
-        ret = dict_get_str (dict, key, &password);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "Failed to get %s for "
-                        "snap %s", key, snap->snapname);
-                goto out;
+    char key[64] = "";
+    int keylen;
+    char *username = NULL;
+    char *password = NULL;
+    glusterd_brickinfo_t *brickinfo = NULL;
+    glusterd_conf_t *priv = NULL;
+    glusterd_volinfo_t *snap_vol = NULL;
+    uuid_t *snap_volid = NULL;
+    int32_t ret = -1;
+    int32_t brick_count = 0;
+    xlator_t *this = NULL;
+    char *clonename = NULL;
+    gf_boolean_t conf_present = _gf_false;
+    int i = 0;
+
+    struct gd_snap_unsupported_opt_t unsupported_opt[] = {
+        {.key = VKEY_FEATURES_QUOTA, .value = NULL},
+        {.key = VKEY_FEATURES_INODE_QUOTA, .value = NULL},
+        {.key = "feature.deem-statfs", .value = NULL},
+        {.key = "features.quota-deem-statfs", .value = NULL},
+        {.key = NULL, .value = NULL}};
+
+    this = THIS;
+    GF_ASSERT(this);
+
+    priv = this->private;
+    GF_ASSERT(priv);
+    GF_ASSERT(dict);
+    GF_ASSERT(origin_vol);
+    GF_ASSERT(rsp_dict);
+
+    /* fetch username, password and vol_id from dict*/
+    keylen = snprintf(key, sizeof(key), "volume%" PRId64 "_username", volcount);
+    ret = dict_get_strn(dict, key, keylen, &username);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+               "Failed to get %s for "
+               "snap %s",
+               key, snap->snapname);
+        goto out;
+    }
+    keylen = snprintf(key, sizeof(key), "volume%" PRId64 "_password", volcount);
+    ret = dict_get_strn(dict, key, keylen, &password);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+               "Failed to get %s for "
+               "snap %s",
+               key, snap->snapname);
+        goto out;
+    }
+
+    snprintf(key, sizeof(key), "vol%" PRId64 "_volid", volcount);
+    ret = dict_get_bin(dict, key, (void **)&snap_volid);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+               "Unable to fetch snap_volid");
+        goto out;
+    }
+
+    /* We are not setting the username and password here as
+     * we need to set the user name and password passed in
+     * the dictionary
+     */
+    ret = glusterd_volinfo_dup(origin_vol, &snap_vol, _gf_false);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_VOL_OP_FAILED,
+               "Failed to duplicate volinfo "
+               "for the snapshot %s",
+               snap->snapname);
+        goto out;
+    }
+
+    /* uuid is used as lvm snapshot name.
+       This will avoid restrictions on snapshot names provided by user */
+    gf_uuid_copy(snap_vol->volume_id, *snap_volid);
+    snap_vol->is_snap_volume = _gf_true;
+    snap_vol->snapshot = snap;
+
+    if (clone) {
+        snap_vol->is_snap_volume = _gf_false;
+        ret = dict_get_strn(dict, "clonename", SLEN("clonename"), &clonename);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+                   "Failed to get %s "
+                   "for snap %s",
+                   key, snap->snapname);
+            goto out;
+        }
+        cds_list_add_tail(&snap_vol->vol_list, &snap->volumes);
+        gf_strncpy(snap_vol->volname, clonename, sizeof(snap_vol->volname));
+        gf_uuid_copy(snap_vol->restored_from_snap,
+                     origin_vol->snapshot->snap_id);
+
+    } else {
+        GLUSTERD_GET_UUID_NOHYPHEN(snap_vol->volname, *snap_volid);
+        gf_strncpy(snap_vol->parent_volname, origin_vol->volname,
+                   sizeof(snap_vol->parent_volname));
+        ret = glusterd_list_add_snapvol(origin_vol, snap_vol);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_SNAP_LIST_SET_FAIL,
+                   "could not add the "
+                   "snap volume %s to the list",
+                   snap_vol->volname);
+            goto out;
         }
-
-        snprintf (key, sizeof(key) - 1, "vol%"PRId64"_volid", volcount);
-        ret = dict_get_bin (dict, key, (void **)&snap_volid);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "Unable to fetch snap_volid");
-                goto out;
-        }
-
-        /* We are not setting the username and password here as
-         * we need to set the user name and password passed in
-         * the dictionary
-         */
-        ret = glusterd_volinfo_dup (origin_vol, &snap_vol, _gf_false);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "Failed to duplicate volinfo "
-                        "for the snapshot %s", snap->snapname);
-                goto out;
-        }
-
-        /* uuid is used as lvm snapshot name.
-           This will avoid restrictions on snapshot names provided by user */
-        gf_uuid_copy (snap_vol->volume_id, *snap_volid);
-        snap_vol->is_snap_volume = _gf_true;
-        snap_vol->snapshot = snap;
-
-        if (clone) {
-                snap_vol->is_snap_volume = _gf_false;
-                ret = dict_get_str (dict, "clonename", &clonename);
-                if (ret) {
-                        gf_log (this->name, GF_LOG_ERROR, "Failed to get %s "
-                                "for snap %s", key, snap->snapname);
-                        goto out;
-                }
-                cds_list_add_tail (&snap_vol->vol_list, &snap->volumes);
-                strcpy(snap_vol->volname, clonename);
-        } else {
-                GLUSTERD_GET_UUID_NOHYPHEN (snap_vol->volname, *snap_volid);
-                strcpy (snap_vol->parent_volname, origin_vol->volname);
-                ret = glusterd_list_add_snapvol (origin_vol, snap_vol);
-                if (ret) {
-                        gf_log (this->name, GF_LOG_ERROR, "could not add the "
-                                "snap volume %s to the list",
-                                snap_vol->volname);
-                        goto out;
-                }
         /* TODO : Sync before taking a snapshot */
         /* Copy the status and config files of geo-replication before
          * taking a snapshot. During restore operation these files needs
          * to be copied back in /var/lib/glusterd/georeplication/
          */
-                ret = glusterd_copy_geo_rep_files (origin_vol, snap_vol,
-                                                   rsp_dict);
-                if (ret) {
-                        gf_log (this->name, GF_LOG_ERROR, "Failed to copy "
-                                "geo-rep config and status files for volume %s",
-                                origin_vol->volname);
-                        goto out;
-                }
-        }
-        glusterd_auth_set_username (snap_vol, username);
-        glusterd_auth_set_password (snap_vol, password);
-
-        /* Adding snap brickinfos to the snap volinfo */
-        brick_count = 0;
-        cds_list_for_each_entry (brickinfo, &origin_vol->bricks, brick_list) {
-                ret = glusterd_add_brick_to_snap_volume (dict, rsp_dict,
-                                                         snap_vol, brickinfo,
-                                                         volcount, brick_count,
-                                                         clone);
-                if (ret) {
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "Failed to add the snap brick for "
-                                "%s:%s to the snap volume",
-                                brickinfo->hostname, brickinfo->path);
-                        goto out;
-                }
-                brick_count++;
-        }
-
-
-        /* During snapshot creation if I/O is in progress,
-         * then barrier value is enabled. Hence during snapshot create
-         * and in-turn snapshot restore the barrier value is set to enable.
-         * Because of this further I/O on the mount point fails.
-         * Hence remove the barrier key from newly created snap volinfo
-         * before storing and generating the brick volfiles. Also update
-         * the snap vol's version after removing the barrier key.
+        ret = glusterd_copy_geo_rep_files(origin_vol, snap_vol, rsp_dict);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_VOL_OP_FAILED,
+                   "Failed to copy "
+                   "geo-rep config and status files for volume %s",
+                   origin_vol->volname);
+            goto out;
+        }
+    }
+
+    glusterd_auth_set_username(snap_vol, username);
+    glusterd_auth_set_password(snap_vol, password);
+
+    /* Adding snap brickinfos to the snap volinfo */
+    brick_count = 0;
+    cds_list_for_each_entry(brickinfo, &origin_vol->bricks, brick_list)
+    {
+        ret = glusterd_add_brick_to_snap_volume(
+            dict, rsp_dict, snap_vol, brickinfo, volcount, brick_count, clone);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_BRICK_ADD_FAIL,
+                   "Failed to add the snap brick for "
+                   "%s:%s to the snap volume",
+                   brickinfo->hostname, brickinfo->path);
+            goto out;
+        }
+        brick_count++;
+    }
+
+    /* During snapshot creation if I/O is in progress,
+     * then barrier value is enabled. Hence during snapshot create
+     * and in-turn snapshot restore the barrier value is set to enable.
+     * Because of this further I/O on the mount point fails.
+     * Hence remove the barrier key from newly created snap volinfo
+     * before storing and generating the brick volfiles. Also update
+     * the snap vol's version after removing the barrier key.
+     */
+    dict_deln(snap_vol->dict, "features.barrier", SLEN("features.barrier"));
+    gd_update_volume_op_versions(snap_vol);
+
+    /* *
+     * Create the export file from the node where ganesha.enable "on"
+     * is executed
+     * */
+    if (glusterd_is_ganesha_cluster() &&
+        glusterd_check_ganesha_export(snap_vol)) {
+        if (is_origin_glusterd(dict)) {
+            ret = manage_export_config(clonename, "on", NULL);
+            if (ret) {
+                gf_msg(this->name, GF_LOG_ERROR, 0,
+                       GD_MSG_EXPORT_FILE_CREATE_FAIL,
+                       "Failed to create"
+                       "export file for NFS-Ganesha\n");
+                goto out;
+            }
+        }
+
+        ret = dict_set_dynstr_with_alloc(snap_vol->dict,
+                                         "features.cache-invalidation", "on");
+        ret = gd_ganesha_send_dbus(clonename, "on");
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_EXPORT_FILE_CREATE_FAIL,
+                   "Dynamic export addition/deletion failed."
+                   " Please see log file for details. Clone name = %s",
+                   clonename);
+            goto out;
+        }
+    }
+    if (!glusterd_is_ganesha_cluster() &&
+        glusterd_check_ganesha_export(snap_vol)) {
+        /* This happens when a snapshot was created when Ganesha was
+         * enabled globally. Then Ganesha disabled from the cluster.
+         * In such cases, we will have the volume level option set
+         * on dict, So we have to disable it as it doesn't make sense
+         * to keep the option.
          */
-        dict_del (snap_vol->dict, "features.barrier");
-        gd_update_volume_op_versions (snap_vol);
-
-        ret = glusterd_store_volinfo (snap_vol,
-                                      GLUSTERD_VOLINFO_VER_AC_INCREMENT);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "Failed to store snapshot "
-                        "volinfo (%s) for snap %s", snap_vol->volname,
-                        snap->snapname);
-                goto out;
-        }
 
-        ret = glusterd_copy_quota_files (origin_vol, snap_vol, &conf_present);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "Failed to copy quota "
-                        "config and cksum for volume %s", origin_vol->volname);
-                goto out;
-        }
-
-        ret = generate_brick_volfiles (snap_vol);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "generating the brick "
-                        "volfiles for the snap %s (volume: %s) failed",
-                        snap->snapname, origin_vol->volname);
-                goto out;
-        }
+        ret = dict_set_dynstr(snap_vol->dict, "ganesha.enable", "off");
+        if (ret)
+            goto out;
+    }
+
+    ret = glusterd_store_volinfo(snap_vol, GLUSTERD_VOLINFO_VER_AC_INCREMENT);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_VOLINFO_SET_FAIL,
+               "Failed to store snapshot "
+               "volinfo (%s) for snap %s",
+               snap_vol->volname, snap->snapname);
+        goto out;
+    }
+
+    ret = glusterd_copy_quota_files(origin_vol, snap_vol, &conf_present);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_SNAP_VOL_CONFIG_FAIL,
+               "Failed to copy quota "
+               "config and cksum for volume %s",
+               origin_vol->volname);
+        goto out;
+    }
+
+    if (snap_vol->is_snap_volume) {
+        ret = glusterd_snap_clear_unsupported_opt(snap_vol, unsupported_opt);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_VOL_OP_FAILED,
+                   "Failed to clear quota "
+                   "option for the snap %s (volume: %s)",
+                   snap->snapname, origin_vol->volname);
+            goto out;
+        }
+    }
+
+    ret = generate_brick_volfiles(snap_vol);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_VOLFILE_CREATE_FAIL,
+               "generating the brick "
+               "volfiles for the snap %s (volume: %s) failed",
+               snap->snapname, origin_vol->volname);
+        goto reset_option;
+    }
+
+    ret = generate_client_volfiles(snap_vol, GF_CLIENT_TRUSTED);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_VOLFILE_CREATE_FAIL,
+               "generating the trusted "
+               "client volfiles for the snap %s (volume: %s) failed",
+               snap->snapname, origin_vol->volname);
+        goto reset_option;
+    }
+
+    ret = generate_client_volfiles(snap_vol, GF_CLIENT_OTHER);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_VOLFILE_CREATE_FAIL,
+               "generating the client "
+               "volfiles for the snap %s (volume: %s) failed",
+               snap->snapname, origin_vol->volname);
+        goto reset_option;
+    }
+
+reset_option:
+    if (snap_vol->is_snap_volume) {
+        if (glusterd_snap_set_unsupported_opt(snap_vol, unsupported_opt)) {
+            ret = -1;
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_VOL_OP_FAILED,
+                   "Failed to reset quota "
+                   "option for the snap %s (volume: %s)",
+                   snap->snapname, origin_vol->volname);
+        }
+    }
+out:
+    if (ret) {
+        for (i = 0; unsupported_opt[i].key; i++)
+            GF_FREE(unsupported_opt[i].value);
 
-        ret = generate_client_volfiles (snap_vol, GF_CLIENT_TRUSTED);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "generating the trusted "
-                        "client volfiles for the snap %s (volume: %s) failed",
-                        snap->snapname, origin_vol->volname);
-                goto out;
-        }
+        if (snap_vol) {
+            if (glusterd_is_ganesha_cluster() &&
+                glusterd_check_ganesha_export(snap_vol)) {
+                if (is_origin_glusterd(dict)) {
+                    ret = manage_export_config(clonename, "on", NULL);
+                    if (ret) {
+                        gf_msg(this->name, GF_LOG_ERROR, 0,
+                               GD_MSG_EXPORT_FILE_CREATE_FAIL,
+                               "Failed to create"
+                               "export file for NFS-Ganesha\n");
+                    }
+                }
 
-        ret = generate_client_volfiles (snap_vol, GF_CLIENT_OTHER);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "generating the client "
-                        "volfiles for the snap %s (volume: %s) failed",
-                        snap->snapname, origin_vol->volname);
-                goto out;
-        }
+                ret = gd_ganesha_send_dbus(clonename, "off");
+                if (ret) {
+                    gf_msg(this->name, GF_LOG_ERROR, 0,
+                           GD_MSG_EXPORT_FILE_CREATE_FAIL,
+                           "Dynamic export addition/deletion failed."
+                           " Please see log file for details. Clone name = %s",
+                           clonename);
+                }
+            }
 
-out:
-        if (ret) {
-                if (snap_vol)
-                        glusterd_snap_volume_remove (rsp_dict, snap_vol,
-                                                     _gf_true, _gf_true);
-                snap_vol = NULL;
+            glusterd_snap_volume_remove(rsp_dict, snap_vol, _gf_true, _gf_true);
         }
+        snap_vol = NULL;
+    }
 
-        return snap_vol;
+    return snap_vol;
 }
 
 /*This is the prevalidate function for both activate and deactive of snap
@@ -4882,269 +5472,294 @@ out:
  * For Deactivate operation pass is_op_activate as _gf_false
  * */
 int
-glusterd_snapshot_activate_deactivate_prevalidate (dict_t *dict,
-                char **op_errstr, dict_t *rsp_dict, gf_boolean_t is_op_activate)
+glusterd_snapshot_activate_deactivate_prevalidate(dict_t *dict,
+                                                  char **op_errstr,
+                                                  uint32_t *op_errno,
+                                                  dict_t *rsp_dict,
+                                                  gf_boolean_t is_op_activate)
 {
-        int32_t                 ret                   = -1;
-        char                    *snapname             = NULL;
-        xlator_t                *this                 = NULL;
-        glusterd_snap_t         *snap                 = NULL;
-        glusterd_volinfo_t      *snap_volinfo         = NULL;
-        char                    err_str[PATH_MAX]     = "";
-        gf_loglevel_t           loglevel              = GF_LOG_ERROR;
-        glusterd_volume_status  volume_status         = GLUSTERD_STATUS_STOPPED;
-        int                     flags                 = 0;
-
-        this = THIS;
-
-        if (!dict || !op_errstr) {
-                gf_log (this->name, GF_LOG_ERROR, "input parameters NULL");
-                goto out;
-        }
-
-        ret = dict_get_str (dict, "snapname", &snapname);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "Getting the snap name "
-                        "failed");
-                goto out;
-        }
-
-        snap = glusterd_find_snap_by_name (snapname);
-        if (!snap) {
-                snprintf (err_str, sizeof (err_str), "Snapshot (%s) does not "
-                          "exist.", snapname);
-                ret = -1;
-                goto out;
-        }
-
-        /*If its activation of snap then fetch the flags*/
+    int32_t ret = -1;
+    char *snapname = NULL;
+    xlator_t *this = NULL;
+    glusterd_snap_t *snap = NULL;
+    glusterd_volinfo_t *snap_volinfo = NULL;
+    char err_str[PATH_MAX] = "";
+    gf_loglevel_t loglevel = GF_LOG_ERROR;
+    glusterd_volume_status volume_status = GLUSTERD_STATUS_STOPPED;
+    int flags = 0;
+
+    this = THIS;
+    GF_VALIDATE_OR_GOTO("glusterd", this, out);
+    GF_VALIDATE_OR_GOTO(this->name, op_errno, out);
+
+    if (!dict || !op_errstr) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_INVALID_ENTRY,
+               "input parameters NULL");
+        goto out;
+    }
+
+    ret = dict_get_strn(dict, "snapname", SLEN("snapname"), &snapname);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+               "Getting the snap name "
+               "failed");
+        goto out;
+    }
+
+    snap = glusterd_find_snap_by_name(snapname);
+    if (!snap) {
+        snprintf(err_str, sizeof(err_str),
+                 "Snapshot (%s) does not "
+                 "exist.",
+                 snapname);
+        gf_smsg(this->name, GF_LOG_ERROR, 0, GD_MSG_SNAP_NOT_FOUND,
+                "Snapname=%s", snapname, NULL);
+        *op_errno = EG_NOSNAP;
+        ret = -1;
+        goto out;
+    }
+
+    /*If its activation of snap then fetch the flags*/
+    if (is_op_activate) {
+        ret = dict_get_int32n(dict, "flags", SLEN("flags"), &flags);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+                   "Unable to get flags");
+            goto out;
+        }
+    }
+
+    /* TODO : As of now there is only volume in snapshot.
+     * Change this when multiple volume snapshot is introduced
+     */
+    snap_volinfo = cds_list_entry(snap->volumes.next, glusterd_volinfo_t,
+                                  vol_list);
+    if (!snap_volinfo) {
+        gf_msg(this->name, GF_LOG_ERROR, EINVAL, GD_MSG_VOLINFO_GET_FAIL,
+               "Unable to fetch snap_volinfo");
+        ret = -1;
+        goto out;
+    }
+
+    /*TODO: When multiple snapvolume are involved a cumulative
+     * logic is required to tell whether is snapshot is
+     * started/partially started/stopped*/
+    if (is_op_activate) {
+        volume_status = GLUSTERD_STATUS_STARTED;
+    }
+
+    if (snap_volinfo->status == volume_status) {
         if (is_op_activate) {
-                ret = dict_get_int32 (dict, "flags", &flags);
-                if (ret) {
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "Unable to get flags");
-                        goto out;
-                }
-        }
-
-        /* TODO : As of now there is only volume in snapshot.
-        * Change this when multiple volume snapshot is introduced
-        */
-        snap_volinfo = cds_list_entry (snap->volumes.next, glusterd_volinfo_t,
-                                       vol_list);
-        if (!snap_volinfo) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "Unable to fetch snap_volinfo");
+            /* if flag is to GF_CLI_FLAG_OP_FORCE
+             * try to start the snap volume, even
+             * if the volume_status is GLUSTERD_STATUS_STARTED.
+             * By doing so we try to bring
+             * back the brick processes that are down*/
+            if (!(flags & GF_CLI_FLAG_OP_FORCE)) {
+                snprintf(err_str, sizeof(err_str),
+                         "Snapshot %s is already activated.", snapname);
+                *op_errno = EINVAL;
                 ret = -1;
-                goto out;
-        }
-
-        /*TODO: When multiple snapvolume are involved a cummulative
-         * logic is required to tell whether is snapshot is
-         * started/partially started/stopped*/
-        if (is_op_activate) {
-                volume_status = GLUSTERD_STATUS_STARTED;
-        }
-
-        if (snap_volinfo->status == volume_status) {
-                if (is_op_activate) {
-                        /* if flag is to GF_CLI_FLAG_OP_FORCE
-                         * try to start the snap volume, even
-                         * if the volume_status is GLUSTERD_STATUS_STARTED.
-                         * By doing so we try to bring
-                         * back the brick processes that are down*/
-                        if (!(flags & GF_CLI_FLAG_OP_FORCE)) {
-                                snprintf (err_str, sizeof (err_str),
-                                          "Snapshot %s is already activated.",
-                                          snapname);
-                                ret = -1;
-                        }
-                } else {
-                        snprintf (err_str, sizeof (err_str),
-                               "Snapshot %s is already deactivated.", snapname);
-                        ret = -1;
-                }
-                goto out;
-        }
-        ret = 0;
+            }
+        } else {
+            snprintf(err_str, sizeof(err_str),
+                     "Snapshot %s is already deactivated.", snapname);
+            *op_errno = EINVAL;
+            ret = -1;
+        }
+        goto out;
+    }
+    ret = 0;
 out:
 
-        if (ret && err_str[0] != '\0') {
-                gf_log (this->name, loglevel, "%s", err_str);
-                *op_errstr = gf_strdup (err_str);
-        }
+    if (ret && err_str[0] != '\0' && op_errstr) {
+        gf_msg(this->name, loglevel, 0, GD_MSG_SNAPSHOT_OP_FAILED, "%s",
+               err_str);
+        *op_errstr = gf_strdup(err_str);
+    }
 
-        return ret;
+    return ret;
 }
 
 int32_t
-glusterd_handle_snapshot_delete_vol (dict_t *dict, char *err_str, int len)
+glusterd_handle_snapshot_delete_vol(dict_t *dict, char *err_str,
+                                    uint32_t *op_errno, int len)
 {
-        int32_t                 ret             = -1;
-        int32_t                 i               = 0;
-        glusterd_volinfo_t      *snap_volinfo   = NULL;
-        glusterd_volinfo_t      *volinfo        = NULL;
-        glusterd_volinfo_t      *temp_volinfo   = NULL;
-        char                    key[PATH_MAX]   =  "";
-        xlator_t                *this           = NULL;
-        char                    *volname        = NULL;
-
-        this = THIS;
-        GF_ASSERT (this);
-        GF_ASSERT (dict);
-
-        ret = dict_get_str (dict, "volname", &volname);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "Failed to get "
-                        "volume name");
-                goto out;
-        }
-
-        ret = glusterd_volinfo_find (volname, &volinfo);
-        if (ret) {
-                snprintf (err_str, len, "Volume (%s) does not exist", volname);
-                gf_log (this->name, GF_LOG_ERROR, "Failed to get volinfo of "
-                        "volume %s", volname);
-                goto out;
-        }
-
-        ret = glusterd_snapshot_get_vol_snapnames (dict, volinfo);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "Failed to get snapshot list for volume %s", volname);
-                goto out;
-        }
-
-        ret = 0;
+    int32_t ret = -1;
+    glusterd_volinfo_t *volinfo = NULL;
+    xlator_t *this = NULL;
+    char *volname = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+    GF_ASSERT(dict);
+    GF_VALIDATE_OR_GOTO(this->name, op_errno, out);
+
+    ret = dict_get_strn(dict, "volname", SLEN("volname"), &volname);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+               "Failed to get "
+               "volume name");
+        goto out;
+    }
+
+    ret = glusterd_volinfo_find(volname, &volinfo);
+    if (ret) {
+        snprintf(err_str, len, "Volume (%s) does not exist", volname);
+        *op_errno = EG_NOVOL;
+        gf_msg(this->name, GF_LOG_ERROR, EINVAL, GD_MSG_VOL_NOT_FOUND,
+               "Failed to get volinfo of "
+               "volume %s",
+               volname);
+        goto out;
+    }
+
+    ret = glusterd_snapshot_get_vol_snapnames(dict, volinfo);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_SNAP_LIST_GET_FAIL,
+               "Failed to get snapshot list for volume %s", volname);
+        goto out;
+    }
+
+    ret = 0;
 out:
-        return ret;
+    return ret;
 }
 
-int32_t
-glusterd_handle_snapshot_delete_all (dict_t *dict)
+static int32_t
+glusterd_handle_snapshot_delete_all(dict_t *dict)
 {
-        int32_t         ret             = -1;
-        int32_t         i               =  0;
-        char            key[PATH_MAX]   =  "";
-        glusterd_conf_t *priv           = NULL;
-        glusterd_snap_t *snap           = NULL;
-        glusterd_snap_t *tmp_snap       = NULL;
-        xlator_t        *this           = NULL;
-
-        this = THIS;
-        GF_ASSERT (this);
-        priv = this->private;
-        GF_ASSERT (priv);
-
-        GF_ASSERT (dict);
-
-        cds_list_for_each_entry_safe (snap, tmp_snap, &priv->snapshots,
-                                      snap_list) {
-                /* indexing from 1 to n, to keep it uniform with other code
-                 * paths
-                 */
-                i++;
-                ret = snprintf (key, sizeof (key), "snapname%d", i);
-                if (ret < 0) {
-                        goto out;
-                }
-
-                ret = dict_set_dynstr_with_alloc (dict, key, snap->snapname);
-                if (ret) {
-                        gf_log (this->name, GF_LOG_ERROR, "Could not save "
-                                "snap name");
-                        goto out;
-                }
+    int32_t ret = -1;
+    int32_t i = 0;
+    char key[32] = "";
+    glusterd_conf_t *priv = NULL;
+    glusterd_snap_t *snap = NULL;
+    glusterd_snap_t *tmp_snap = NULL;
+    xlator_t *this = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+    priv = this->private;
+    GF_ASSERT(priv);
+
+    GF_ASSERT(dict);
+
+    cds_list_for_each_entry_safe(snap, tmp_snap, &priv->snapshots, snap_list)
+    {
+        /* indexing from 1 to n, to keep it uniform with other code
+         * paths
+         */
+        i++;
+        ret = snprintf(key, sizeof(key), "snapname%d", i);
+        if (ret < 0) {
+            goto out;
         }
 
-        ret = dict_set_int32 (dict, "snapcount", i);
+        ret = dict_set_dynstr_with_alloc(dict, key, snap->snapname);
         if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "Could not save snapcount");
-                goto out;
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+                   "Could not save "
+                   "snap name");
+            goto out;
         }
+    }
 
-        ret = 0;
+    ret = dict_set_int32n(dict, "snapcount", SLEN("snapcount"), i);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+               "Could not save snapcount");
+        goto out;
+    }
+
+    ret = 0;
 
 out:
-        return ret;
+    return ret;
 }
 
 int32_t
-glusterd_handle_snapshot_delete_type_snap (rpcsvc_request_t *req,
-                                           glusterd_op_t op,
-                                           dict_t *dict, char *err_str,
-                                           size_t len)
+glusterd_handle_snapshot_delete_type_snap(rpcsvc_request_t *req,
+                                          glusterd_op_t op, dict_t *dict,
+                                          char *err_str, uint32_t *op_errno,
+                                          size_t len)
 {
-        int32_t                 ret             = -1;
-        int64_t                  volcount       = 0;
-        char                    *snapname       = NULL;
-        char                    *volname        = NULL;
-        char                     key[PATH_MAX]  = "";
-        glusterd_snap_t         *snap           = NULL;
-        glusterd_volinfo_t      *snap_vol       = NULL;
-        glusterd_volinfo_t      *tmp            = NULL;
-        xlator_t                *this           = NULL;
-
-        this = THIS;
-        GF_ASSERT (this);
-
-        GF_ASSERT (req);
-        GF_ASSERT (dict);
-        GF_ASSERT (err_str);
-
-        ret = dict_get_str (dict, "snapname", &snapname);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "Failed to get snapname");
-                goto out;
-        }
-
-        snap = glusterd_find_snap_by_name (snapname);
-        if (!snap) {
-                snprintf (err_str, len, "Snapshot (%s) does not exist",
-                          snapname);
-                gf_log (this->name, GF_LOG_ERROR,
-                        "%s", err_str);
-                ret = -1;
-                goto out;
-        }
-
-        /* Set volnames in the dict to get mgmt_v3 lock */
-        cds_list_for_each_entry_safe (snap_vol, tmp, &snap->volumes, vol_list) {
-                volcount++;
-                volname = gf_strdup (snap_vol->parent_volname);
-                if (!volname) {
-                        ret = -1;
-                        gf_log (this->name, GF_LOG_ERROR, "strdup failed");
-                        goto out;
-                }
-
-                snprintf (key, sizeof (key), "volname%"PRId64, volcount);
-                ret = dict_set_dynstr (dict, key, volname);
-                if (ret) {
-                        gf_log (this->name, GF_LOG_ERROR, "Failed to set "
-                                "volume name in dictionary");
-                        GF_FREE (volname);
-                        goto out;
-                }
-                volname = NULL;
-        }
-        ret = dict_set_int64 (dict, "volcount", volcount);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "Failed to set volcount");
-                goto out;
-        }
-
-        ret = glusterd_mgmt_v3_initiate_snap_phases (req, op, dict);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "Failed to initiate snap "
-                        "phases");
-                goto out;
-        }
-
-        ret = 0;
+    int32_t ret = -1;
+    int64_t volcount = 0;
+    char *snapname = NULL;
+    char *volname = NULL;
+    char key[64] = "";
+    int keylen;
+    glusterd_snap_t *snap = NULL;
+    glusterd_volinfo_t *snap_vol = NULL;
+    glusterd_volinfo_t *tmp = NULL;
+    xlator_t *this = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+
+    GF_ASSERT(req);
+    GF_ASSERT(dict);
+    GF_ASSERT(err_str);
+
+    ret = dict_get_strn(dict, "snapname", SLEN("snapname"), &snapname);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+               "Failed to get snapname");
+        goto out;
+    }
+
+    snap = glusterd_find_snap_by_name(snapname);
+    if (!snap) {
+        snprintf(err_str, len, "Snapshot (%s) does not exist", snapname);
+        *op_errno = EG_NOSNAP;
+        gf_msg(this->name, GF_LOG_ERROR, EINVAL, GD_MSG_SNAP_NOT_FOUND, "%s",
+               err_str);
+        ret = -1;
+        goto out;
+    }
+
+    /* Set volnames in the dict to get mgmt_v3 lock */
+    cds_list_for_each_entry_safe(snap_vol, tmp, &snap->volumes, vol_list)
+    {
+        volcount++;
+        volname = gf_strdup(snap_vol->parent_volname);
+        if (!volname) {
+            ret = -1;
+            gf_msg(this->name, GF_LOG_ERROR, ENOMEM, GD_MSG_NO_MEMORY,
+                   "strdup failed");
+            goto out;
+        }
+
+        keylen = snprintf(key, sizeof(key), "volname%" PRId64, volcount);
+        ret = dict_set_dynstrn(dict, key, keylen, volname);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+                   "Failed to set "
+                   "volume name in dictionary");
+            GF_FREE(volname);
+            goto out;
+        }
+        volname = NULL;
+    }
+    ret = dict_set_int64(dict, "volcount", volcount);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+               "Failed to set volcount");
+        goto out;
+    }
+
+    ret = glusterd_mgmt_v3_initiate_snap_phases(req, op, dict);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_SNAP_INIT_FAIL,
+               "Failed to initiate snap "
+               "phases");
+        goto out;
+    }
+
+    ret = 0;
 
-out :
-        return ret;
+out:
+    return ret;
 }
 
 /* This is a snapshot remove handler function. This function will be
@@ -5160,2625 +5775,3017 @@ out :
  * @return              Negative value on Failure and 0 in success
  */
 int
-glusterd_handle_snapshot_delete (rpcsvc_request_t *req, glusterd_op_t op,
-                                 dict_t *dict, char *err_str, size_t len)
+glusterd_handle_snapshot_delete(rpcsvc_request_t *req, glusterd_op_t op,
+                                dict_t *dict, char *err_str, uint32_t *op_errno,
+                                size_t len)
 {
-        int                      ret            = -1;
-        xlator_t                *this           = NULL;
-        int32_t                  delete_cmd     = -1;
+    int ret = -1;
+    xlator_t *this = NULL;
+    int32_t delete_cmd = -1;
 
-        this = THIS;
+    this = THIS;
 
-        GF_ASSERT (this);
+    GF_ASSERT(this);
 
-        GF_ASSERT (req);
-        GF_ASSERT (dict);
-        GF_ASSERT (err_str);
+    GF_ASSERT(req);
+    GF_ASSERT(dict);
+    GF_ASSERT(err_str);
+    GF_VALIDATE_OR_GOTO(this->name, op_errno, out);
 
-        ret = dict_get_int32 (dict, "delete-cmd", &delete_cmd);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "Failed to get delete-cmd");
-                goto out;
-        }
+    ret = dict_get_int32n(dict, "sub-cmd", SLEN("sub-cmd"), &delete_cmd);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_COMMAND_NOT_FOUND,
+               "Failed to get sub-cmd");
+        goto out;
+    }
 
-        switch (delete_cmd) {
+    switch (delete_cmd) {
         case GF_SNAP_DELETE_TYPE_SNAP:
-                ret = glusterd_handle_snapshot_delete_type_snap (req, op, dict,
-                                                                 err_str, len);
-                if (ret) {
-                        gf_log (this->name, GF_LOG_ERROR, "Failed to handle "
-                                "snapshot delete for type SNAP");
-                        goto out;
-                }
-                break;
+        case GF_SNAP_DELETE_TYPE_ITER:
+            ret = glusterd_handle_snapshot_delete_type_snap(
+                req, op, dict, err_str, op_errno, len);
+            if (ret) {
+                gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_SNAP_REMOVE_FAIL,
+                       "Failed to handle "
+                       "snapshot delete for type SNAP");
+                goto out;
+            }
+            break;
 
         case GF_SNAP_DELETE_TYPE_ALL:
-                ret = glusterd_handle_snapshot_delete_all (dict);
-                if (ret) {
-                        gf_log (this->name, GF_LOG_ERROR, "Failed to handle "
-                                "snapshot delete for type ALL");
-                        goto out;
-                }
-                break;
+            ret = glusterd_handle_snapshot_delete_all(dict);
+            if (ret) {
+                gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_SNAP_REMOVE_FAIL,
+                       "Failed to handle "
+                       "snapshot delete for type ALL");
+                goto out;
+            }
+            break;
 
         case GF_SNAP_DELETE_TYPE_VOL:
-                ret = glusterd_handle_snapshot_delete_vol (dict, err_str, len);
-                if (ret) {
-                        gf_log (this->name, GF_LOG_ERROR, "Failed to handle "
-                                "snapshot delete for type VOL");
-                        goto out;
-                }
-                break;
+            ret = glusterd_handle_snapshot_delete_vol(dict, err_str, op_errno,
+                                                      len);
+            if (ret) {
+                gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_SNAP_REMOVE_FAIL,
+                       "Failed to handle "
+                       "snapshot delete for type VOL");
+                goto out;
+            }
+            break;
 
         default:
-                gf_log (this->name, GF_LOG_ERROR, "Wrong snapshot delete type");
-                break;
-        }
-
-        if ( ret == 0 && (delete_cmd == GF_SNAP_DELETE_TYPE_ALL ||
-                          delete_cmd == GF_SNAP_DELETE_TYPE_VOL)) {
-                ret = glusterd_op_send_cli_response (op, 0, 0, req, dict,
-                                                     err_str);
-                if (ret) {
-                        gf_log (this->name, GF_LOG_ERROR, "Failed to send cli "
-                                "response");
-                        goto out;
-                }
-        }
-        ret = 0;
+            *op_errno = EINVAL;
+            gf_msg(this->name, GF_LOG_ERROR, EINVAL, GD_MSG_INVALID_ENTRY,
+                   "Wrong snapshot delete type");
+            break;
+    }
+
+    if (ret == 0 && (delete_cmd == GF_SNAP_DELETE_TYPE_ALL ||
+                     delete_cmd == GF_SNAP_DELETE_TYPE_VOL)) {
+        ret = glusterd_op_send_cli_response(op, 0, 0, req, dict, err_str);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_NO_CLI_RESP,
+                   "Failed to send cli "
+                   "response");
+            goto out;
+        }
+    }
+    ret = 0;
 out:
-        return ret;
+    return ret;
 }
 
 int
-glusterd_snapshot_remove_prevalidate (dict_t *dict, char **op_errstr,
-                                      dict_t *rsp_dict)
+glusterd_snapshot_remove_prevalidate(dict_t *dict, char **op_errstr,
+                                     uint32_t *op_errno, dict_t *rsp_dict)
 {
-        int32_t             ret         = -1;
-        char               *snapname    = NULL;
-        xlator_t           *this        = NULL;
-        glusterd_snap_t    *snap        = NULL;
-
-        this = THIS;
-
-        if (!dict || !op_errstr) {
-                gf_log (this->name, GF_LOG_ERROR, "input parameters NULL");
-                goto out;
-        }
-
-        ret = dict_get_str (dict, "snapname", &snapname);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "Getting the snap name "
-                        "failed");
-                goto out;
-        }
-
-        snap = glusterd_find_snap_by_name (snapname);
-        if (!snap) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "Snapshot (%s) does not exist", snapname);
-                ret = -1;
-                goto out;
-        }
-
-        ret = dict_set_dynstr_with_alloc (dict, "snapuuid",
-                                          uuid_utoa (snap->snap_id));
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "Failed to set snap "
-                        "uuid in response dictionary for %s snapshot",
-                        snap->snapname);
-                goto out;
-        }
-
-        ret = 0;
+    int32_t ret = -1;
+    char *snapname = NULL;
+    xlator_t *this = NULL;
+    glusterd_snap_t *snap = NULL;
+
+    this = THIS;
+    GF_VALIDATE_OR_GOTO("glusterd", this, out);
+    GF_VALIDATE_OR_GOTO(this->name, op_errno, out);
+
+    if (!dict || !op_errstr) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_INVALID_ENTRY,
+               "input parameters NULL");
+        goto out;
+    }
+
+    ret = dict_get_strn(dict, "snapname", SLEN("snapname"), &snapname);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+               "Getting the snap name "
+               "failed");
+        goto out;
+    }
+
+    snap = glusterd_find_snap_by_name(snapname);
+    if (!snap) {
+        gf_msg(this->name, GF_LOG_ERROR, EINVAL, GD_MSG_SNAP_NOT_FOUND,
+               "Snapshot (%s) does not exist", snapname);
+        *op_errno = EG_NOSNAP;
+        ret = -1;
+        goto out;
+    }
+
+    ret = dict_set_dynstr_with_alloc(dict, "snapuuid",
+                                     uuid_utoa(snap->snap_id));
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+               "Failed to set snap "
+               "uuid in response dictionary for %s snapshot",
+               snap->snapname);
+        goto out;
+    }
+
+    ret = 0;
 out:
-        return ret;
+    return ret;
 }
 
 int
-glusterd_snapshot_status_prevalidate (dict_t *dict, char **op_errstr,
-                                      dict_t *rsp_dict)
+glusterd_snapshot_status_prevalidate(dict_t *dict, char **op_errstr,
+                                     uint32_t *op_errno, dict_t *rsp_dict)
 {
-        int                     ret             =       -1;
-        char                    *snapname       =       NULL;
-        glusterd_conf_t         *conf           =       NULL;
-        xlator_t                *this           =       NULL;
-        int32_t                 cmd             =       -1;
-        glusterd_volinfo_t      *volinfo        =       NULL;
-        char                    *volname        =       NULL;
-
-        this = THIS;
-        GF_ASSERT (this);
-        conf = this->private;
-
-        GF_ASSERT (conf);
-        GF_ASSERT (op_errstr);
-        if (!dict) {
-                gf_log (this->name, GF_LOG_ERROR, "Input dict is NULL");
-                goto out;
-        }
-
-        ret = dict_get_int32 (dict, "status-cmd", &cmd);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "Could not fetch status cmd");
-                goto out;
-        }
-
-        switch (cmd) {
-                case GF_SNAP_STATUS_TYPE_ALL:
-                {
-                        break;
-                }
-                case GF_SNAP_STATUS_TYPE_SNAP:
-                {
-                        ret = dict_get_str (dict, "snapname", &snapname);
-                        if (ret) {
-                                gf_log (this->name, GF_LOG_ERROR,
-                                        "Could not fetch snapname");
-                                goto out;
-                        }
-
-                        if (!glusterd_find_snap_by_name (snapname)) {
-                                ret = gf_asprintf (op_errstr, "Snapshot (%s) "
-                                                  "does not exist", snapname);
-                                if (ret < 0) {
-                                        goto out;
-                                }
-                                ret = -1;
-                                gf_log (this->name, GF_LOG_ERROR,
-                                        "Snapshot (%s) does not exist",
-                                        snapname);
-                                goto out;
-                        }
-                        break;
-                }
-                case GF_SNAP_STATUS_TYPE_VOL:
-                {
-                        ret = dict_get_str (dict, "volname", &volname);
-                        if (ret) {
-                                gf_log (this->name, GF_LOG_ERROR,
-                                        "Could not fetch volname");
-                                goto out;
-                        }
-
-                        ret = glusterd_volinfo_find (volname, &volinfo);
-                        if (ret) {
-                                ret = gf_asprintf (op_errstr, "Volume (%s) "
-                                                  "does not exist", volname);
-                                if (ret < 0) {
-                                        goto out;
-                                }
-                                ret = -1;
-                                gf_log (this->name, GF_LOG_ERROR, "Volume "
-                                        "%s not present", volname);
-                                goto out;
-                        }
-                        break;
-
+    int ret = -1;
+    char *snapname = NULL;
+    glusterd_conf_t *conf = NULL;
+    xlator_t *this = NULL;
+    int32_t cmd = -1;
+    glusterd_volinfo_t *volinfo = NULL;
+    char *volname = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+    conf = this->private;
+    GF_ASSERT(conf);
+    GF_ASSERT(op_errstr);
+    GF_VALIDATE_OR_GOTO(this->name, op_errno, out);
+
+    if (!dict) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_INVALID_ENTRY,
+               "Input dict is NULL");
+        goto out;
+    }
+
+    ret = dict_get_int32n(dict, "sub-cmd", SLEN("sub-cmd"), &cmd);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+               "Could not fetch status cmd");
+        goto out;
+    }
+
+    switch (cmd) {
+        case GF_SNAP_STATUS_TYPE_ALL: {
+            break;
+        }
+        case GF_SNAP_STATUS_TYPE_ITER:
+        case GF_SNAP_STATUS_TYPE_SNAP: {
+            ret = dict_get_strn(dict, "snapname", SLEN("snapname"), &snapname);
+            if (ret) {
+                gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+                       "Could not fetch snapname");
+                goto out;
+            }
+
+            if (!glusterd_find_snap_by_name(snapname)) {
+                ret = gf_asprintf(op_errstr,
+                                  "Snapshot (%s) "
+                                  "does not exist",
+                                  snapname);
+                *op_errno = EG_NOSNAP;
+                if (ret < 0) {
+                    goto out;
                 }
-                default:
-                {
-                        gf_log (this->name, GF_LOG_ERROR, "Invalid command");
-                        break;
+                ret = -1;
+                gf_msg(this->name, GF_LOG_ERROR, EINVAL, GD_MSG_SNAP_NOT_FOUND,
+                       "Snapshot (%s) does not exist", snapname);
+                goto out;
+            }
+            break;
+        }
+        case GF_SNAP_STATUS_TYPE_VOL: {
+            ret = dict_get_strn(dict, "volname", SLEN("volname"), &volname);
+            if (ret) {
+                gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+                       "Could not fetch volname");
+                goto out;
+            }
+
+            ret = glusterd_volinfo_find(volname, &volinfo);
+            if (ret) {
+                ret = gf_asprintf(op_errstr,
+                                  "Volume (%s) "
+                                  "does not exist",
+                                  volname);
+                *op_errno = EG_NOVOL;
+                if (ret < 0) {
+                    goto out;
                 }
+                ret = -1;
+                gf_msg(this->name, GF_LOG_ERROR, EINVAL, GD_MSG_VOL_NOT_FOUND,
+                       "Volume "
+                       "%s not present",
+                       volname);
+                goto out;
+            }
+            break;
         }
-        ret = 0;
+        default: {
+            gf_msg(this->name, GF_LOG_ERROR, EINVAL, GD_MSG_COMMAND_NOT_FOUND,
+                   "Invalid command");
+            *op_errno = EINVAL;
+            break;
+        }
+    }
+    ret = 0;
 
 out:
-        return ret;
+    return ret;
 }
 
 int32_t
-glusterd_snapshot_activate_commit (dict_t *dict, char **op_errstr,
-                                 dict_t *rsp_dict)
+glusterd_snapshot_activate_commit(dict_t *dict, char **op_errstr,
+                                  dict_t *rsp_dict)
 {
-        int32_t                   ret                  = -1;
-        char                     *snapname             = NULL;
-        glusterd_snap_t          *snap                 = NULL;
-        glusterd_volinfo_t       *snap_volinfo         = NULL;
-        xlator_t                 *this                 = NULL;
-        int                      flags                 = 0;
-
-        this = THIS;
-        GF_ASSERT (this);
-        GF_ASSERT (dict);
-        GF_ASSERT (rsp_dict);
-        GF_ASSERT (op_errstr);
-
-        if (!dict || !op_errstr) {
-                gf_log (this->name, GF_LOG_ERROR, "input parameters NULL");
-                goto out;
-        }
-
-        ret = dict_get_str (dict, "snapname", &snapname);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "Getting the snap name "
-                        "failed");
-                goto out;
-        }
-
-        ret = dict_get_int32 (dict, "flags", &flags);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "Unable to get flags");
-                goto out;
-        }
-
-        snap = glusterd_find_snap_by_name (snapname);
-        if (!snap) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "Snapshot (%s) does not exist", snapname);
-                ret = -1;
-                goto out;
-        }
-
-        /* TODO : As of now there is only volume in snapshot.
-        * Change this when multiple volume snapshot is introduced
-        */
-        snap_volinfo = cds_list_entry (snap->volumes.next, glusterd_volinfo_t,
-                                       vol_list);
-        if (!snap_volinfo) {
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "Unable to fetch snap_volinfo");
-                        ret = -1;
-                        goto out;
-        }
-
-        ret = glusterd_start_volume (snap_volinfo, flags, _gf_true);
-
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "Failed to activate snap volume %s of the snap %s",
-                        snap_volinfo->volname, snap->snapname);
-                goto out;
-        }
-
-        ret = dict_set_dynstr_with_alloc (rsp_dict, "snapuuid",
-                                          uuid_utoa (snap->snap_id));
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "Failed to set snap "
-                        "uuid in response dictionary for %s snapshot",
-                        snap->snapname);
-                goto out;
-        }
-
-        ret = 0;
+    int32_t ret = -1;
+    char *snapname = NULL;
+    glusterd_snap_t *snap = NULL;
+    glusterd_volinfo_t *snap_volinfo = NULL;
+    glusterd_brickinfo_t *brickinfo = NULL;
+    xlator_t *this = NULL;
+    int flags = 0;
+    int brick_count = -1;
+
+    this = THIS;
+    GF_ASSERT(this);
+    GF_ASSERT(dict);
+    GF_ASSERT(rsp_dict);
+    GF_ASSERT(op_errstr);
+
+    if (!dict || !op_errstr) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_INVALID_ENTRY,
+               "input parameters NULL");
+        goto out;
+    }
+
+    ret = dict_get_strn(dict, "snapname", SLEN("snapname"), &snapname);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+               "Getting the snap name "
+               "failed");
+        goto out;
+    }
+
+    ret = dict_get_int32n(dict, "flags", SLEN("flags"), &flags);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+               "Unable to get flags");
+        goto out;
+    }
+
+    snap = glusterd_find_snap_by_name(snapname);
+    if (!snap) {
+        gf_msg(this->name, GF_LOG_ERROR, EINVAL, GD_MSG_SNAP_NOT_FOUND,
+               "Snapshot (%s) does not exist", snapname);
+        ret = -1;
+        goto out;
+    }
+
+    /* TODO : As of now there is only volume in snapshot.
+     * Change this when multiple volume snapshot is introduced
+     */
+    snap_volinfo = cds_list_entry(snap->volumes.next, glusterd_volinfo_t,
+                                  vol_list);
+    if (!snap_volinfo) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_VOLINFO_GET_FAIL,
+               "Unable to fetch snap_volinfo");
+        ret = -1;
+        goto out;
+    }
+
+    /* create the complete brick here */
+    cds_list_for_each_entry(brickinfo, &snap_volinfo->bricks, brick_list)
+    {
+        brick_count++;
+        if (gf_uuid_compare(brickinfo->uuid, MY_UUID))
+            continue;
+        ret = glusterd_snap_brick_create(snap_volinfo, brickinfo, brick_count,
+                                         _gf_false);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_BRICK_CREATION_FAIL,
+                   "not able to "
+                   "create the brick for the snap %s, volume %s",
+                   snap_volinfo->snapshot->snapname, snap_volinfo->volname);
+            goto out;
+        }
+    }
+
+    ret = glusterd_start_volume(snap_volinfo, flags, _gf_true);
+
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_SNAP_ACTIVATE_FAIL,
+               "Failed to activate snap volume %s of the snap %s",
+               snap_volinfo->volname, snap->snapname);
+        goto out;
+    }
+
+    ret = dict_set_dynstr_with_alloc(rsp_dict, "snapuuid",
+                                     uuid_utoa(snap->snap_id));
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+               "Failed to set snap "
+               "uuid in response dictionary for %s snapshot",
+               snap->snapname);
+        goto out;
+    }
+
+    ret = 0;
 out:
-        return ret;
+    return ret;
 }
 
 int32_t
-glusterd_snapshot_deactivate_commit (dict_t *dict, char **op_errstr,
-                                 dict_t *rsp_dict)
+glusterd_snapshot_deactivate_commit(dict_t *dict, char **op_errstr,
+                                    dict_t *rsp_dict)
 {
-        int32_t                   ret                  = -1;
-        char                     *snapname             = NULL;
-        glusterd_snap_t          *snap                 = NULL;
-        glusterd_volinfo_t       *snap_volinfo         = NULL;
-        xlator_t                 *this                 = NULL;
-
-        this = THIS;
-        GF_ASSERT (this);
-        GF_ASSERT (dict);
-        GF_ASSERT (rsp_dict);
-        GF_ASSERT (op_errstr);
-
-        if (!dict || !op_errstr) {
-                gf_log (this->name, GF_LOG_ERROR, "input parameters NULL");
-                goto out;
-        }
-
-        ret = dict_get_str (dict, "snapname", &snapname);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "Getting the snap name "
-                        "failed");
-                goto out;
-        }
-
-        snap = glusterd_find_snap_by_name (snapname);
-        if (!snap) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "Snapshot (%s) does not exist", snapname);
-                ret = -1;
-                goto out;
-        }
-
-        /* TODO : As of now there is only volume in snapshot.
-        * Change this when multiple volume snapshot is introduced
-        */
-        snap_volinfo = cds_list_entry (snap->volumes.next, glusterd_volinfo_t,
-                                       vol_list);
-        if (!snap_volinfo) {
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "Unable to fetch snap_volinfo");
-                        ret = -1;
-                        goto out;
-        }
-
-        ret = glusterd_stop_volume (snap_volinfo);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "Failed to deactivate"
-                        "snap %s", snapname);
-                goto out;
-        }
-
-        ret = dict_set_dynstr_with_alloc (rsp_dict, "snapuuid",
-                                          uuid_utoa (snap->snap_id));
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "Failed to set snap "
-                        "uuid in response dictionary for %s snapshot",
-                        snap->snapname);
-                goto out;
-        }
-
-        ret = 0;
+    int32_t ret = -1;
+    char *snapname = NULL;
+    glusterd_snap_t *snap = NULL;
+    glusterd_volinfo_t *snap_volinfo = NULL;
+    xlator_t *this = NULL;
+    char snap_path[PATH_MAX] = "";
+
+    this = THIS;
+    GF_ASSERT(this);
+    GF_ASSERT(dict);
+    GF_ASSERT(rsp_dict);
+    GF_ASSERT(op_errstr);
+
+    if (!dict || !op_errstr) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_INVALID_ENTRY,
+               "input parameters NULL");
+        goto out;
+    }
+
+    ret = dict_get_strn(dict, "snapname", SLEN("snapname"), &snapname);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+               "Getting the snap name "
+               "failed");
+        goto out;
+    }
+
+    snap = glusterd_find_snap_by_name(snapname);
+    if (!snap) {
+        gf_msg(this->name, GF_LOG_ERROR, EINVAL, GD_MSG_SNAP_NOT_FOUND,
+               "Snapshot (%s) does not exist", snapname);
+        ret = -1;
+        goto out;
+    }
+
+    /* TODO : As of now there is only volume in snapshot.
+     * Change this when multiple volume snapshot is introduced
+     */
+    snap_volinfo = cds_list_entry(snap->volumes.next, glusterd_volinfo_t,
+                                  vol_list);
+    if (!snap_volinfo) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_VOLINFO_GET_FAIL,
+               "Unable to fetch snap_volinfo");
+        ret = -1;
+        goto out;
+    }
+
+    ret = glusterd_stop_volume(snap_volinfo);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_SNAP_DEACTIVATE_FAIL,
+               "Failed to deactivate"
+               "snap %s",
+               snapname);
+        goto out;
+    }
+
+    ret = glusterd_snap_unmount(this, snap_volinfo);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_GLUSTERD_UMOUNT_FAIL,
+               "Failed to unmounts for %s", snap->snapname);
+    }
+
+    /*Remove /var/run/gluster/snaps/<snap-name> entry for deactivated snaps.
+     * This entry will be created again during snap activate.
+     */
+    snprintf(snap_path, sizeof(snap_path), "%s/%s", snap_mount_dir, snapname);
+    ret = recursive_rmdir(snap_path);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, errno, GD_MSG_DIR_OP_FAILED,
+               "Failed to remove "
+               "%s directory : error : %s",
+               snap_path, strerror(errno));
+        goto out;
+    }
+
+    ret = dict_set_dynstr_with_alloc(rsp_dict, "snapuuid",
+                                     uuid_utoa(snap->snap_id));
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+               "Failed to set snap "
+               "uuid in response dictionary for %s snapshot",
+               snap->snapname);
+        goto out;
+    }
+
+    ret = 0;
 out:
-        return ret;
+    return ret;
 }
 
 int32_t
-glusterd_snapshot_remove_commit (dict_t *dict, char **op_errstr,
-                                 dict_t *rsp_dict)
+glusterd_snapshot_remove_commit(dict_t *dict, char **op_errstr,
+                                dict_t *rsp_dict)
 {
-        int32_t                   ret                  = -1;
-        char                     *snapname             = NULL;
-        char                     *dup_snapname         = NULL;
-        glusterd_snap_t          *snap                 = NULL;
-        glusterd_conf_t          *priv                 = NULL;
-        glusterd_volinfo_t       *snap_volinfo         = NULL;
-        xlator_t                 *this                 = NULL;
-
-        this = THIS;
-        GF_ASSERT (this);
-        GF_ASSERT (dict);
-        GF_ASSERT (rsp_dict);
-        GF_ASSERT (op_errstr);
-
-        priv = this->private;
-        GF_ASSERT (priv);
-
-        if (!dict || !op_errstr) {
-                gf_log (this->name, GF_LOG_ERROR, "input parameters NULL");
-                goto out;
-        }
-
-        ret = dict_get_str (dict, "snapname", &snapname);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "Getting the snap name "
-                        "failed");
-                goto out;
-        }
-
-        snap = glusterd_find_snap_by_name (snapname);
-        if (!snap) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "Snapshot (%s) does not exist", snapname);
-                ret = -1;
-                goto out;
-        }
-
-        ret = dict_set_dynstr_with_alloc (rsp_dict, "snapuuid",
-                                          uuid_utoa (snap->snap_id));
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "Failed to set snap uuid in "
-                        "response dictionary for %s snapshot",
-                        snap->snapname);
-                goto out;
-        }
-
-        /* Save the snap status as GD_SNAP_STATUS_DECOMMISSION so
-         * that if the node goes down the snap would be removed
+    int32_t ret = -1;
+    char *snapname = NULL;
+    char *dup_snapname = NULL;
+    glusterd_snap_t *snap = NULL;
+    glusterd_conf_t *priv = NULL;
+    glusterd_volinfo_t *snap_volinfo = NULL;
+    xlator_t *this = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+    GF_ASSERT(dict);
+    GF_ASSERT(rsp_dict);
+    GF_ASSERT(op_errstr);
+
+    priv = this->private;
+    GF_ASSERT(priv);
+
+    if (!dict || !op_errstr) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_INVALID_ENTRY,
+               "input parameters NULL");
+        goto out;
+    }
+
+    ret = dict_get_strn(dict, "snapname", SLEN("snapname"), &snapname);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+               "Getting the snap name "
+               "failed");
+        goto out;
+    }
+
+    snap = glusterd_find_snap_by_name(snapname);
+    if (!snap) {
+        gf_msg(this->name, GF_LOG_ERROR, EINVAL, GD_MSG_SNAP_NOT_FOUND,
+               "Snapshot (%s) does not exist", snapname);
+        ret = -1;
+        goto out;
+    }
+
+    ret = dict_set_dynstr_with_alloc(rsp_dict, "snapuuid",
+                                     uuid_utoa(snap->snap_id));
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+               "Failed to set snap uuid in "
+               "response dictionary for %s snapshot",
+               snap->snapname);
+        goto out;
+    }
+
+    /* Save the snap status as GD_SNAP_STATUS_DECOMMISSION so
+     * that if the node goes down the snap would be removed
+     */
+    snap->snap_status = GD_SNAP_STATUS_DECOMMISSION;
+    ret = glusterd_store_snap(snap);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_SNAP_OBJECT_STORE_FAIL,
+               "Failed to "
+               "store snap object %s",
+               snap->snapname);
+        goto out;
+    } else
+        gf_msg(this->name, GF_LOG_INFO, 0, GD_MSG_OP_SUCCESS,
+               "Successfully marked "
+               "snap %s for decommission.",
+               snap->snapname);
+
+    if (is_origin_glusterd(dict) == _gf_true) {
+        /* TODO : As of now there is only volume in snapshot.
+         * Change this when multiple volume snapshot is introduced
          */
-        snap->snap_status = GD_SNAP_STATUS_DECOMMISSION;
-        ret = glusterd_store_snap (snap);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "Failed to "
-                        "store snap object %s", snap->snapname);
-                goto out;
-        } else
-                gf_log (this->name, GF_LOG_INFO, "Successfully marked "
-                        "snap %s for decommission.", snap->snapname);
-
-        if (is_origin_glusterd (dict) == _gf_true) {
-                /* TODO : As of now there is only volume in snapshot.
-                 * Change this when multiple volume snapshot is introduced
-                 */
-                snap_volinfo = cds_list_entry (snap->volumes.next,
-                                               glusterd_volinfo_t,
-                                               vol_list);
-                if (!snap_volinfo) {
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "Unable to fetch snap_volinfo");
-                        ret = -1;
-                        goto out;
-                }
-
-                /* From origin glusterd check if      *
-                 * any peers with snap bricks is down */
-                ret = glusterd_find_missed_snap (rsp_dict, snap_volinfo,
-                                                 &priv->peers,
-                                                 GF_SNAP_OPTION_TYPE_DELETE);
-                if (ret) {
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "Failed to find missed snap deletes");
-                        goto out;
-                }
-        }
-
-        ret = glusterd_snap_remove (rsp_dict, snap, _gf_true, _gf_false);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "Failed to remove snap %s",
-                        snapname);
-                goto out;
-        }
-
-        dup_snapname = gf_strdup (snapname);
-        if (!dup_snapname) {
-                gf_log (this->name, GF_LOG_ERROR, "Strdup failed");
-                ret = -1;
-                goto out;
-        }
-
-        ret = dict_set_dynstr (rsp_dict, "snapname", dup_snapname);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "Failed to set the snapname");
-                GF_FREE (dup_snapname);
-                goto out;
-        }
-
-        ret = 0;
+        snap_volinfo = cds_list_entry(snap->volumes.next, glusterd_volinfo_t,
+                                      vol_list);
+        if (!snap_volinfo) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_VOLINFO_GET_FAIL,
+                   "Unable to fetch snap_volinfo");
+            ret = -1;
+            goto out;
+        }
+
+        /* From origin glusterd check if      *
+         * any peers with snap bricks is down */
+        ret = glusterd_find_missed_snap(rsp_dict, snap_volinfo, &priv->peers,
+                                        GF_SNAP_OPTION_TYPE_DELETE);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_MISSED_SNAP_GET_FAIL,
+                   "Failed to find missed snap deletes");
+            goto out;
+        }
+    }
+
+    ret = glusterd_snap_remove(rsp_dict, snap, _gf_true, _gf_false, _gf_false);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_SNAP_REMOVE_FAIL,
+               "Failed to remove snap %s", snapname);
+        goto out;
+    }
+
+    dup_snapname = gf_strdup(snapname);
+    if (!dup_snapname) {
+        gf_msg(this->name, GF_LOG_ERROR, ENOMEM, GD_MSG_NO_MEMORY,
+               "Strdup failed");
+        ret = -1;
+        goto out;
+    }
+
+    ret = dict_set_dynstr(rsp_dict, "snapname", dup_snapname);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+               "Failed to set the snapname");
+        GF_FREE(dup_snapname);
+        goto out;
+    }
+
+    ret = 0;
 out:
-        return ret;
+    return ret;
 }
 
 int32_t
-glusterd_do_snap_cleanup (dict_t *dict, char **op_errstr, dict_t *rsp_dict)
+glusterd_do_snap_cleanup(dict_t *dict, char **op_errstr, dict_t *rsp_dict)
 {
-        int32_t                  ret                   = -1;
-        char                     *name                 = NULL;
-        xlator_t                 *this                 = NULL;
-        glusterd_conf_t          *conf                 = NULL;
-        glusterd_volinfo_t       *volinfo              = NULL;
-        glusterd_snap_t          *snap                 = NULL;
-
-        this = THIS;
-        GF_ASSERT (this);
-        conf = this->private;
-        GF_ASSERT (conf);
-
-        if (!dict || !op_errstr) {
-                gf_log (this->name, GF_LOG_ERROR, "input parameters NULL");
-                goto out;
-        }
-
-        ret = dict_get_str (dict, "snapname", &name);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "getting the snap "
-                        "name failed (volume: %s)", volinfo->volname);
-                goto out;
-        }
-
-        /*
-          If the snapname is not found that means the failure happened at
-          staging, or in commit, before the snap object is created, in which
-          case there is nothing to cleanup. So set ret to 0.
-        */
-        snap = glusterd_find_snap_by_name (name);
-        if (!snap) {
-                gf_log (this->name, GF_LOG_INFO, "Snapshot (%s) does not exist",
-                        name);
-                ret = 0;
-                goto out;
-        }
-
-        ret = glusterd_snap_remove (rsp_dict, snap, _gf_true, _gf_true);
-        if (ret) {
-                /* Ignore failure as this is a cleanup of half cooked
-                   snapshot */
-                gf_log (this->name, GF_LOG_DEBUG, "removing the snap %s failed",
-                        name);
-                ret = 0;
-        }
+    int32_t ret = -1;
+    char *name = NULL;
+    char *volname = NULL;
+    xlator_t *this = NULL;
+    glusterd_conf_t *conf = NULL;
+    glusterd_snap_t *snap = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+    conf = this->private;
+    GF_ASSERT(conf);
+
+    if (!dict || !op_errstr) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_INVALID_ENTRY,
+               "input parameters NULL");
+        goto out;
+    }
+
+    /* As of now snapshot of multiple volumes are not supported */
+    ret = dict_get_strn(dict, "volname1", SLEN("volname1"), &volname);
+    if (ret) {
+        gf_msg("glusterd", GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+               "Unable to get"
+               " volume name");
+        goto out;
+    }
+
+    ret = dict_get_strn(dict, "snapname", SLEN("snapname"), &name);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+               "getting the snap "
+               "name failed (volume: %s)",
+               volname);
+        goto out;
+    }
+
+    /*
+      If the snapname is not found that means the failure happened at
+      staging, or in commit, before the snap object is created, in which
+      case there is nothing to cleanup. So set ret to 0.
+    */
+    snap = glusterd_find_snap_by_name(name);
+    if (!snap) {
+        gf_msg(this->name, GF_LOG_INFO, EINVAL, GD_MSG_SNAP_NOT_FOUND,
+               "Snapshot (%s) does not exist", name);
+        ret = 0;
+        goto out;
+    }
+
+    ret = glusterd_snap_remove(rsp_dict, snap, _gf_true, _gf_true, _gf_false);
+    if (ret) {
+        /* Ignore failure as this is a cleanup of half cooked
+           snapshot */
+        gf_msg_debug(this->name, 0, "removing the snap %s failed", name);
+        ret = 0;
+    }
 
-        name = NULL;
+    name = NULL;
 
-        ret = 0;
+    ret = 0;
 
 out:
 
-        return ret;
+    return ret;
 }
 
 /* In case of a successful, delete or create operation, during post_validate *
  * look for missed snap operations and update the missed snap lists */
 int32_t
-glusterd_snapshot_update_snaps_post_validate (dict_t *dict, char **op_errstr,
-                                              dict_t *rsp_dict)
+glusterd_snapshot_update_snaps_post_validate(dict_t *dict, char **op_errstr,
+                                             dict_t *rsp_dict)
 {
-        int32_t                  ret                    = -1;
-        int32_t                  missed_snap_count      = -1;
-        xlator_t                *this                   = NULL;
-
-        this = THIS;
-        GF_ASSERT (this);
-        GF_ASSERT (dict);
-        GF_ASSERT (rsp_dict);
-        GF_ASSERT (op_errstr);
-
-        ret = dict_get_int32 (dict, "missed_snap_count",
-                              &missed_snap_count);
-        if (ret) {
-                gf_log (this->name, GF_LOG_DEBUG, "No missed snaps");
-                ret = 0;
-                goto out;
-        }
-
-        ret = glusterd_add_missed_snaps_to_list (dict, missed_snap_count);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "Failed to add missed snaps to list");
-                goto out;
-        }
-
-        ret = glusterd_store_update_missed_snaps ();
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "Failed to update missed_snaps_list");
-                goto out;
-        }
+    int32_t ret = -1;
+    int32_t missed_snap_count = -1;
+    xlator_t *this = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+    GF_ASSERT(dict);
+    GF_ASSERT(rsp_dict);
+    GF_ASSERT(op_errstr);
+
+    ret = dict_get_int32n(dict, "missed_snap_count", SLEN("missed_snap_count"),
+                          &missed_snap_count);
+    if (ret) {
+        gf_msg_debug(this->name, 0, "No missed snaps");
+        ret = 0;
+        goto out;
+    }
+
+    ret = glusterd_add_missed_snaps_to_list(dict, missed_snap_count);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_MISSEDSNAP_INFO_SET_FAIL,
+               "Failed to add missed snaps to list");
+        goto out;
+    }
+
+    ret = glusterd_store_update_missed_snaps();
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_MISSEDSNAP_INFO_SET_FAIL,
+               "Failed to update missed_snaps_list");
+        goto out;
+    }
 
 out:
-        gf_log (this->name, GF_LOG_TRACE, "Returning %d", ret);
-        return ret;
+    gf_msg_trace(this->name, 0, "Returning %d", ret);
+    return ret;
 }
 
 int
-glusterd_take_brick_snapshot_task (void *opaque)
+glusterd_take_brick_snapshot_task(void *opaque)
 {
-        int ret                             = 0;
-        snap_create_args_t  *snap_args      = NULL;
-        char                 key[PATH_MAX]  = "";
-
-        GF_ASSERT (opaque);
-
-        snap_args = (snap_create_args_t*) opaque;
-        THIS = snap_args->this;
-
-        ret = glusterd_take_brick_snapshot (snap_args->dict,
-                                            snap_args->snap_vol,
-                                            snap_args->brickinfo,
-                                            snap_args->volcount,
-                                            snap_args->brickorder);
-
-        if (ret) {
-                gf_log (THIS->name, GF_LOG_ERROR, "Failed to "
-                      "take backend snapshot for brick "
-                      "%s:%s volume(%s)", snap_args->brickinfo->hostname,
-                      snap_args->brickinfo->path, snap_args->snap_vol->volname);
-        }
-
-        snprintf (key, sizeof (key), "snap-vol%d.brick%d.status",
-                  snap_args->volcount, snap_args->brickorder);
-        if (dict_set_int32 (snap_args->rsp_dict, key, (ret)?0:1)) {
-                gf_log (THIS->name, GF_LOG_ERROR, "failed to "
-                        "add %s to dict", key);
-                ret = -1;
-                goto out;
-        }
+    int ret = 0;
+    int32_t clone = 0;
+    snap_create_args_t *snap_args = NULL;
+    char *clonename = NULL;
+    char key[64] = "";
+    int keylen;
+
+    GF_ASSERT(opaque);
+
+    snap_args = (snap_create_args_t *)opaque;
+    THIS = snap_args->this;
+
+    /* Try and fetch clonename. If present set status with clonename *
+     * else do so as snap-vol */
+    ret = dict_get_strn(snap_args->dict, "clonename", SLEN("clonename"),
+                        &clonename);
+    if (ret) {
+        keylen = snprintf(key, sizeof(key), "snap-vol%d.brick%d.status",
+                          snap_args->volcount, snap_args->brickorder);
+    } else {
+        keylen = snprintf(key, sizeof(key), "clone%d.brick%d.status",
+                          snap_args->volcount, snap_args->brickorder);
+        clone = 1;
+    }
+
+    ret = glusterd_take_brick_snapshot(
+        snap_args->dict, snap_args->snap_vol, snap_args->brickinfo,
+        snap_args->volcount, snap_args->brickorder, clone);
+
+    if (ret) {
+        gf_msg(THIS->name, GF_LOG_ERROR, 0, GD_MSG_SNAP_CREATION_FAIL,
+               "Failed to "
+               "take backend snapshot for brick "
+               "%s:%s volume(%s)",
+               snap_args->brickinfo->hostname, snap_args->brickinfo->path,
+               snap_args->snap_vol->volname);
+    }
+
+    if (dict_set_int32n(snap_args->rsp_dict, key, keylen, (ret) ? 0 : 1)) {
+        gf_msg(THIS->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+               "failed to "
+               "add %s to dict",
+               key);
+        ret = -1;
+        goto out;
+    }
 
 out:
-        return ret;
+    return ret;
 }
 
 int32_t
-glusterd_take_brick_snapshot_cbk (int ret, call_frame_t *frame, void *opaque)
+glusterd_take_brick_snapshot_cbk(int ret, call_frame_t *frame, void *opaque)
 {
-        snap_create_args_t  *snap_args = NULL;
-        struct syncargs     *args      = NULL;
+    snap_create_args_t *snap_args = NULL;
+    struct syncargs *args = NULL;
 
-        GF_ASSERT (opaque);
+    GF_ASSERT(opaque);
 
-        snap_args = (snap_create_args_t*) opaque;
-        args  = snap_args->args;
+    snap_args = (snap_create_args_t *)opaque;
+    args = snap_args->args;
 
-        if (ret)
-                args->op_ret = ret;
+    if (ret)
+        args->op_ret = ret;
 
-        GF_FREE (opaque);
-        synctask_barrier_wake(args);
-        return 0;
+    GF_FREE(opaque);
+    synctask_barrier_wake(args);
+    return 0;
 }
 
 int32_t
-glusterd_schedule_brick_snapshot (dict_t *dict, dict_t *rsp_dict,
-                                  glusterd_snap_t *snap)
+glusterd_schedule_brick_snapshot(dict_t *dict, dict_t *rsp_dict,
+                                 glusterd_snap_t *snap)
 {
-        int                     ret             = -1;
-        int32_t                 volcount        = 0;
-        int32_t                 brickcount      = 0;
-        int32_t                 brickorder      = 0;
-        int32_t                 taskcount       = 0;
-        char                    key[PATH_MAX]   = "";
-        xlator_t               *this            = NULL;
-        glusterd_volinfo_t     *snap_vol        = NULL;
-        glusterd_brickinfo_t   *brickinfo       = NULL;
-        struct syncargs         args            = {0};
-        snap_create_args_t     *snap_args       = NULL;
-
-        this = THIS;
-        GF_ASSERT(this);
-        GF_ASSERT(dict);
-        GF_ASSERT(snap);
-
-        synctask_barrier_init ((&args));
-        cds_list_for_each_entry (snap_vol, &snap->volumes, vol_list) {
-                volcount++;
-                brickcount = 0;
-                brickorder = 0;
-                cds_list_for_each_entry (brickinfo, &snap_vol->bricks,
-                                                            brick_list) {
-                        snprintf (key, sizeof(key) - 1,
-                                  "snap-vol%d.brick%d.order", volcount,
-                                  brickcount);
-                        ret = dict_set_int32 (rsp_dict, key, brickorder);
-                        if (ret) {
-                                gf_log (this->name, GF_LOG_ERROR,
-                                        "Failed to set %s", key);
-                                goto out;
-                        }
-
-                        if ((gf_uuid_compare (brickinfo->uuid, MY_UUID)) ||
-                            (brickinfo->snap_status == -1)) {
-                                if (!gf_uuid_compare (brickinfo->uuid, MY_UUID)) {
-                                        brickcount++;
-                                        snprintf (key, sizeof (key),
-                                                  "snap-vol%d.brick%d.status",
-                                                  volcount, brickorder);
-                                        ret = dict_set_int32 (rsp_dict, key, 0);
-                                        if (ret) {
-                                                gf_log (this->name,
-                                                        GF_LOG_ERROR,
-                                                        "failed to add %s to "
-                                                        "dict", key);
-                                                goto out;
-                                        }
-                                }
-                                brickorder++;
-                                continue;
-                        }
-
-                        snap_args = GF_CALLOC (1, sizeof (*snap_args),
-                                               gf_gld_mt_snap_create_args_t);
-                        if (!snap_args) {
-                                ret = -1;
-                                goto out;
-                        }
-
-
-                        snap_args->this = this;
-                        snap_args->dict = dict;
-                        snap_args->rsp_dict = rsp_dict;
-                        snap_args->snap_vol = snap_vol;
-                        snap_args->brickinfo = brickinfo;
-                        snap_args->volcount = volcount;
-                        snap_args->brickcount = brickcount;
-                        snap_args->brickorder = brickorder;
-                        snap_args->args = &args;
-
-                        ret = synctask_new (this->ctx->env,
-                                            glusterd_take_brick_snapshot_task,
-                                            glusterd_take_brick_snapshot_cbk,
-                                            NULL, snap_args);
-                        if (ret) {
-                                gf_log (this->name, GF_LOG_ERROR, "Failed to "
-                                        "spawn task for snapshot create");
-                                GF_FREE (snap_args);
-                                goto out;
-                        }
-                        taskcount++;
-                        brickcount++;
-                        brickorder++;
-                }
-
-                snprintf (key, sizeof (key), "snap-vol%d_brickcount", volcount);
-                ret = dict_set_int64 (rsp_dict, key, brickcount);
-                if (ret) {
-                        gf_log (this->name, GF_LOG_ERROR, "failed to "
-                                "add %s to dict", key);
-                        goto out;
-                }
-        }
-        synctask_barrier_wait ((&args), taskcount);
-        taskcount = 0;
-
-        if (args.op_ret)
-                gf_log (this->name, GF_LOG_ERROR, "Failed to create snapshot");
-
-        ret = args.op_ret;
+    int ret = -1;
+    int32_t volcount = 0;
+    int32_t brickcount = 0;
+    int32_t brickorder = 0;
+    int32_t taskcount = 0;
+    char key[64] = "";
+    int keylen;
+    xlator_t *this = NULL;
+    glusterd_volinfo_t *snap_vol = NULL;
+    glusterd_brickinfo_t *brickinfo = NULL;
+    struct syncargs args = {0};
+    snap_create_args_t *snap_args = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+    GF_ASSERT(dict);
+    GF_ASSERT(snap);
+
+    ret = synctask_barrier_init((&args));
+    if (ret)
+        goto out;
+    cds_list_for_each_entry(snap_vol, &snap->volumes, vol_list)
+    {
+        volcount++;
+        brickcount = 0;
+        brickorder = 0;
+        cds_list_for_each_entry(brickinfo, &snap_vol->bricks, brick_list)
+        {
+            keylen = snprintf(key, sizeof(key), "snap-vol%d.brick%d.order",
+                              volcount, brickcount);
+            ret = dict_set_int32n(rsp_dict, key, keylen, brickorder);
+            if (ret) {
+                gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+                       "Failed to set %s", key);
+                goto out;
+            }
+
+            if ((gf_uuid_compare(brickinfo->uuid, MY_UUID)) ||
+                (brickinfo->snap_status == -1)) {
+                if (!gf_uuid_compare(brickinfo->uuid, MY_UUID)) {
+                    brickcount++;
+                    keylen = snprintf(key, sizeof(key),
+                                      "snap-vol%d.brick%d.status", volcount,
+                                      brickorder);
+                    ret = dict_set_int32n(rsp_dict, key, keylen, 0);
+                    if (ret) {
+                        gf_msg(this->name, GF_LOG_ERROR, 0,
+                               GD_MSG_DICT_SET_FAILED,
+                               "failed to add %s to "
+                               "dict",
+                               key);
+                        goto out;
+                    }
+                }
+                brickorder++;
+                continue;
+            }
+
+            snap_args = GF_CALLOC(1, sizeof(*snap_args),
+                                  gf_gld_mt_snap_create_args_t);
+            if (!snap_args) {
+                ret = -1;
+                goto out;
+            }
+
+            snap_args->this = this;
+            snap_args->dict = dict;
+            snap_args->rsp_dict = rsp_dict;
+            snap_args->snap_vol = snap_vol;
+            snap_args->brickinfo = brickinfo;
+            snap_args->volcount = volcount;
+            snap_args->brickcount = brickcount;
+            snap_args->brickorder = brickorder;
+            snap_args->args = &args;
+
+            ret = synctask_new(
+                this->ctx->env, glusterd_take_brick_snapshot_task,
+                glusterd_take_brick_snapshot_cbk, NULL, snap_args);
+            if (ret) {
+                gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_SNAP_CREATION_FAIL,
+                       "Failed to "
+                       "spawn task for snapshot create");
+                GF_FREE(snap_args);
+                goto out;
+            }
+            taskcount++;
+            brickcount++;
+            brickorder++;
+        }
+
+        snprintf(key, sizeof(key), "snap-vol%d_brickcount", volcount);
+        ret = dict_set_int64(rsp_dict, key, brickcount);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+                   "failed to "
+                   "add %s to dict",
+                   key);
+            goto out;
+        }
+    }
+    synctask_barrier_wait((&args), taskcount);
+    taskcount = 0;
+
+    if (args.op_ret)
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_SNAP_CREATION_FAIL,
+               "Failed to create snapshot");
+
+    ret = args.op_ret;
 out:
-        if (ret && taskcount)
-                synctask_barrier_wait ((&args), taskcount);
+    if (ret && taskcount)
+        synctask_barrier_wait((&args), taskcount);
 
-        return ret;
+    return ret;
 }
 
-glusterd_snap_t*
-glusterd_create_snap_object_for_clone (dict_t *dict, dict_t *rsp_dict)
+glusterd_snap_t *
+glusterd_create_snap_object_for_clone(dict_t *dict, dict_t *rsp_dict)
 {
-        char              *snapname    = NULL;
-        uuid_t            *snap_id     = NULL;
-        glusterd_snap_t   *snap        = NULL;
-        xlator_t          *this        = NULL;
-        glusterd_conf_t   *priv        = NULL;
-        int                ret         = -1;
-        int64_t            time_stamp  = 0;
-
-        this = THIS;
-        priv = this->private;
-
-        GF_ASSERT (dict);
-        GF_ASSERT (rsp_dict);
-
-        /* Fetch snapname, description, id and time from dict */
-        ret = dict_get_str (dict, "clonename", &snapname);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "Unable to fetch clonename");
-                goto out;
-        }
-
-        ret = dict_get_bin (dict, "clone-id", (void **)&snap_id);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "Unable to fetch clone_id");
-                goto out;
-        }
-
-        snap = glusterd_new_snap_object ();
-        if (!snap) {
-                gf_log (this->name, GF_LOG_ERROR, "Could not create "
-                        "the snap object for snap %s", snapname);
-                goto out;
-        }
-
-        strcpy (snap->snapname, snapname);
-        gf_uuid_copy (snap->snap_id, *snap_id);
-
-        ret = 0;
+    char *snapname = NULL;
+    uuid_t *snap_id = NULL;
+    glusterd_snap_t *snap = NULL;
+    xlator_t *this = NULL;
+    int ret = -1;
+
+    this = THIS;
+
+    GF_ASSERT(dict);
+    GF_ASSERT(rsp_dict);
+
+    /* Fetch snapname, description, id and time from dict */
+    ret = dict_get_strn(dict, "clonename", SLEN("clonename"), &snapname);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+               "Unable to fetch clonename");
+        goto out;
+    }
+
+    ret = dict_get_bin(dict, "clone-id", (void **)&snap_id);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+               "Unable to fetch clone_id");
+        goto out;
+    }
+
+    snap = glusterd_new_snap_object();
+    if (!snap) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_SNAP_OBJ_NEW_FAIL,
+               "Could not create "
+               "the snap object for snap %s",
+               snapname);
+        goto out;
+    }
+
+    gf_strncpy(snap->snapname, snapname, sizeof(snap->snapname));
+    gf_uuid_copy(snap->snap_id, *snap_id);
+
+    ret = 0;
 
 out:
-        if (ret) {
-                if (snap)
-                        glusterd_snap_remove (rsp_dict, snap,
-                                              _gf_true, _gf_true);
-                snap = NULL;
-        }
+    if (ret) {
+        snap = NULL;
+    }
 
-        return snap;
+    return snap;
 }
 
-
 int32_t
-glusterd_snapshot_clone_commit (dict_t *dict, char **op_errstr,
-                                 dict_t *rsp_dict)
+glusterd_snapshot_clone_commit(dict_t *dict, char **op_errstr, dict_t *rsp_dict)
 {
-        int                     ret                     = -1;
-        int64_t                 i                       = 0;
-        int64_t                 volcount                = 0;
-        int32_t                 snap_activate           = 0;
-        char                    *snapname               = NULL;
-        char                    *volname                = NULL;
-        char                    *tmp_name               = NULL;
-        char                    key[PATH_MAX]           = "";
-        xlator_t                *this                   = NULL;
-        glusterd_snap_t         *snap_parent            = NULL;
-        glusterd_snap_t         *snap                   = NULL;
-        glusterd_volinfo_t      *origin_vol             = NULL;
-        glusterd_volinfo_t      *snap_vol               = NULL;
-        glusterd_brickinfo_t    *brickinfo              = NULL;
-        glusterd_conf_t         *priv                   = NULL;
-
-        this = THIS;
-        GF_ASSERT(this);
-        GF_ASSERT(dict);
-        GF_ASSERT(op_errstr);
-        GF_ASSERT(rsp_dict);
-        priv = this->private;
-        GF_ASSERT(priv);
-
-
-        ret = dict_get_str (dict, "clonename", &snapname);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "Unable to fetch clonename");
-                goto out;
-        }
-        tmp_name = gf_strdup (snapname);
-        if (!tmp_name) {
-                gf_log (this->name, GF_LOG_ERROR, "Out of memory");
-                ret = -1;
-                goto out;
-        }
-
-        ret = dict_set_dynstr (rsp_dict, "clonename", tmp_name);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "Unable to set clonename in rsp_dict");
-                GF_FREE (tmp_name);
-                goto out;
-        }
-        tmp_name = NULL;
-
-
-       ret = dict_get_str (dict, "snapname", &volname);
-       if (ret) {
-               gf_log (this->name, GF_LOG_ERROR,
-                       "failed to get snap name");
-               goto out;
-       }
-       snap_parent = glusterd_find_snap_by_name (volname);
-       /* TODO : As of now there is only one volume in snapshot.
-        * Change this when multiple volume snapshot is introduced
-        */
-       origin_vol = cds_list_entry (snap_parent->volumes.next,
-                                  glusterd_volinfo_t, vol_list);
-       if (!origin_vol) {
-               gf_log ("glusterd", GF_LOG_ERROR, "Failed to get snap "
-               "volinfo %s", snap_parent->snapname);
-               goto out;
-       }
-       snap = glusterd_create_snap_object_for_clone (dict, rsp_dict);
-       if (!snap) {
-               gf_log (this->name, GF_LOG_ERROR, "creating the"
-                       "snap object %s failed", snapname);
-               ret = -1;
-               goto out;
-       }
-
-       snap_vol = glusterd_do_snap_vol (origin_vol, snap, dict,
-                                       rsp_dict, 1, 1);
-       if (!snap_vol) {
-              ret = -1;
-              gf_log (this->name, GF_LOG_WARNING, "taking the "
-                      "snapshot of the volume %s failed", volname);
-              goto out;
-       }
-       volcount = 1;
-       ret = dict_set_int64 (rsp_dict, "volcount", volcount);
-       if (ret) {
-               gf_log (this->name, GF_LOG_ERROR, "Failed to set volcount");
-               goto out;
-       }
-
-       ret = glusterd_schedule_brick_snapshot (dict, rsp_dict, snap);
-       if (ret) {
-               gf_log (this->name, GF_LOG_ERROR, "Failed to take backend "
-                       "snapshot %s", snap->snapname);
-               goto out;
-       }
-
-       cds_list_del_init (&snap_vol->vol_list);
-       ret = dict_set_dynstr_with_alloc (rsp_dict, "snapuuid",
-                                         uuid_utoa (snap->snap_id));
-       if (ret) {
-               gf_log (this->name, GF_LOG_ERROR, "Failed to set snap "
-                       "uuid in response dictionary for %s snapshot",
-                       snap->snapname);
-               goto out;
-       }
-       ret = glusterd_snapdsvc_init (snap_vol);
-       glusterd_list_add_order (&snap_vol->vol_list, &priv->volumes,
-                       glusterd_compare_volume_name);
-
-       ret = 0;
-
+    int ret = -1;
+    int64_t volcount = 0;
+    char *snapname = NULL;
+    char *volname = NULL;
+    char *tmp_name = NULL;
+    xlator_t *this = NULL;
+    glusterd_snap_t *snap_parent = NULL;
+    glusterd_snap_t *snap = NULL;
+    glusterd_volinfo_t *origin_vol = NULL;
+    glusterd_volinfo_t *snap_vol = NULL;
+    glusterd_conf_t *priv = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+    GF_ASSERT(dict);
+    GF_ASSERT(op_errstr);
+    GF_ASSERT(rsp_dict);
+    priv = this->private;
+    GF_ASSERT(priv);
+
+    ret = dict_get_strn(dict, "clonename", SLEN("clonename"), &snapname);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+               "Unable to fetch clonename");
+        goto out;
+    }
+    tmp_name = gf_strdup(snapname);
+    if (!tmp_name) {
+        gf_msg(this->name, GF_LOG_ERROR, ENOMEM, GD_MSG_NO_MEMORY,
+               "Out of memory");
+        ret = -1;
+        goto out;
+    }
+
+    ret = dict_set_dynstr(rsp_dict, "clonename", tmp_name);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+               "Unable to set clonename in rsp_dict");
+        GF_FREE(tmp_name);
+        goto out;
+    }
+    tmp_name = NULL;
+
+    ret = dict_get_strn(dict, "snapname", SLEN("snapname"), &volname);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+               "failed to get snap name");
+        goto out;
+    }
+
+    snap_parent = glusterd_find_snap_by_name(volname);
+    if (!snap_parent) {
+        gf_msg(this->name, GF_LOG_ERROR, EINVAL, GD_MSG_SNAP_NOT_FOUND,
+               "Failed to "
+               "fetch snap %s",
+               volname);
+        goto out;
+    }
+
+    /* TODO : As of now there is only one volume in snapshot.
+     * Change this when multiple volume snapshot is introduced
+     */
+    origin_vol = cds_list_entry(snap_parent->volumes.next, glusterd_volinfo_t,
+                                vol_list);
+    if (!origin_vol) {
+        gf_msg("glusterd", GF_LOG_ERROR, 0, GD_MSG_VOLINFO_GET_FAIL,
+               "Failed to get snap "
+               "volinfo %s",
+               snap_parent->snapname);
+        goto out;
+    }
+
+    snap = glusterd_create_snap_object_for_clone(dict, rsp_dict);
+    if (!snap) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_SNAP_OBJ_NEW_FAIL,
+               "creating the"
+               "snap object %s failed",
+               snapname);
+        ret = -1;
+        goto out;
+    }
+
+    snap_vol = glusterd_do_snap_vol(origin_vol, snap, dict, rsp_dict, 1, 1);
+    if (!snap_vol) {
+        ret = -1;
+        gf_msg(this->name, GF_LOG_WARNING, 0, GD_MSG_SNAP_CREATION_FAIL,
+               "taking the "
+               "snapshot of the volume %s failed",
+               volname);
+        goto out;
+    }
+
+    volcount = 1;
+    ret = dict_set_int64(rsp_dict, "volcount", volcount);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+               "Failed to set volcount");
+        goto out;
+    }
+
+    ret = glusterd_schedule_brick_snapshot(dict, rsp_dict, snap);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_SNAP_BACKEND_MAKE_FAIL,
+               "Failed to take backend "
+               "snapshot %s",
+               snap->snapname);
+        goto out;
+    }
+
+    cds_list_del_init(&snap_vol->vol_list);
+    ret = dict_set_dynstr_with_alloc(rsp_dict, "snapuuid",
+                                     uuid_utoa(snap_vol->volume_id));
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+               "Failed to set snap "
+               "uuid in response dictionary for %s snapshot",
+               snap->snapname);
+        goto out;
+    }
+
+    glusterd_list_add_order(&snap_vol->vol_list, &priv->volumes,
+                            glusterd_compare_volume_name);
+
+    ret = 0;
 
 out:
-       if (ret) {
-               if (snap)
-                       glusterd_snap_remove (rsp_dict, snap,
-                                             _gf_true, _gf_true);
-               snap = NULL;
-       }
-
-       gf_log (this->name, GF_LOG_TRACE, "Returning %d", ret);
-       return ret;
-}
+    if (ret) {
+        if (snap)
+            glusterd_snap_remove(rsp_dict, snap, _gf_true, _gf_true, _gf_true);
+        snap = NULL;
+    }
 
+    gf_msg_trace(this->name, 0, "Returning %d", ret);
+    return ret;
+}
 
 int32_t
-glusterd_snapshot_create_commit (dict_t *dict, char **op_errstr,
-                                 dict_t *rsp_dict)
+glusterd_snapshot_create_commit(dict_t *dict, char **op_errstr,
+                                uint32_t *op_errno, dict_t *rsp_dict)
 {
-        int                     ret                     = -1;
-        int64_t                 i                       = 0;
-        int64_t                 volcount                = 0;
-        int32_t                 snap_activate           = 0;
-        char                    *snapname               = NULL;
-        char                    *volname                = NULL;
-        char                    *tmp_name               = NULL;
-        char                    key[PATH_MAX]           = "";
-        xlator_t                *this                   = NULL;
-        glusterd_snap_t         *snap                   = NULL;
-        glusterd_volinfo_t      *origin_vol             = NULL;
-        glusterd_volinfo_t      *snap_vol               = NULL;
-        glusterd_brickinfo_t    *brickinfo              = NULL;
-        glusterd_conf_t         *priv                   = NULL;
-
-        this = THIS;
-        GF_ASSERT(this);
-        GF_ASSERT(dict);
-        GF_ASSERT(op_errstr);
-        GF_ASSERT(rsp_dict);
-        priv = this->private;
-        GF_ASSERT(priv);
-
-        ret = dict_get_int64 (dict, "volcount", &volcount);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "failed to "
-                        "get the volume count");
-                goto out;
-        }
-
-        ret = dict_get_str (dict, "snapname", &snapname);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "Unable to fetch snapname");
-                goto out;
-        }
-        tmp_name = gf_strdup (snapname);
-        if (!tmp_name) {
-                gf_log (this->name, GF_LOG_ERROR, "Out of memory");
-                ret = -1;
-                goto out;
-        }
-
-        ret = dict_set_dynstr (rsp_dict, "snapname", tmp_name);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "Unable to set snapname in rsp_dict");
-                GF_FREE (tmp_name);
-                goto out;
-        }
-        tmp_name = NULL;
-
-        snap = glusterd_create_snap_object (dict, rsp_dict);
-        if (!snap) {
-                gf_log (this->name, GF_LOG_ERROR, "creating the"
-                        "snap object %s failed", snapname);
-                ret = -1;
+    int ret = -1;
+    int64_t i = 0;
+    int64_t volcount = 0;
+    int32_t snap_activate = 0;
+    int32_t flags = 0;
+    char *snapname = NULL;
+    char *volname = NULL;
+    char *tmp_name = NULL;
+    char key[64] = "";
+    int keylen;
+    xlator_t *this = NULL;
+    glusterd_snap_t *snap = NULL;
+    glusterd_volinfo_t *origin_vol = NULL;
+    glusterd_volinfo_t *snap_vol = NULL;
+    glusterd_conf_t *priv = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+    GF_ASSERT(dict);
+    GF_ASSERT(op_errstr);
+    GF_VALIDATE_OR_GOTO(this->name, op_errno, out);
+    GF_ASSERT(rsp_dict);
+    priv = this->private;
+    GF_ASSERT(priv);
+
+    ret = dict_get_int64(dict, "volcount", &volcount);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+               "failed to "
+               "get the volume count");
+        goto out;
+    }
+
+    ret = dict_get_strn(dict, "snapname", SLEN("snapname"), &snapname);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+               "Unable to fetch snapname");
+        goto out;
+    }
+    tmp_name = gf_strdup(snapname);
+    if (!tmp_name) {
+        gf_msg(this->name, GF_LOG_ERROR, ENOMEM, GD_MSG_NO_MEMORY,
+               "Out of memory");
+        ret = -1;
+        goto out;
+    }
+
+    ret = dict_set_dynstr(rsp_dict, "snapname", tmp_name);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+               "Unable to set snapname in rsp_dict");
+        GF_FREE(tmp_name);
+        goto out;
+    }
+    tmp_name = NULL;
+
+    snap = glusterd_create_snap_object(dict, rsp_dict);
+    if (!snap) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_SNAP_CREATION_FAIL,
+               "creating the"
+               "snap object %s failed",
+               snapname);
+        ret = -1;
+        goto out;
+    }
+
+    for (i = 1; i <= volcount; i++) {
+        keylen = snprintf(key, sizeof(key), "volname%" PRId64, i);
+        ret = dict_get_strn(dict, key, keylen, &volname);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+                   "failed to get volume name");
+            goto out;
+        }
+
+        ret = glusterd_volinfo_find(volname, &origin_vol);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, EINVAL, GD_MSG_VOL_NOT_FOUND,
+                   "failed to get the volinfo for "
+                   "the volume %s",
+                   volname);
+            goto out;
+        }
+
+        if (is_origin_glusterd(dict)) {
+            ret = glusterd_is_snap_soft_limit_reached(origin_vol, rsp_dict);
+            if (ret) {
+                gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_SNAPSHOT_OP_FAILED,
+                       "Failed to "
+                       "check soft limit exceeded or not, "
+                       "for volume %s ",
+                       origin_vol->volname);
+                goto out;
+            }
+        }
+
+        snap_vol = glusterd_do_snap_vol(origin_vol, snap, dict, rsp_dict, i, 0);
+        if (!snap_vol) {
+            ret = -1;
+            gf_msg(this->name, GF_LOG_WARNING, 0, GD_MSG_SNAP_CREATION_FAIL,
+                   "taking the "
+                   "snapshot of the volume %s failed",
+                   volname);
+            goto out;
+        }
+    }
+    ret = dict_set_int64(rsp_dict, "volcount", volcount);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+               "Failed to set volcount");
+        goto out;
+    }
+
+    ret = glusterd_schedule_brick_snapshot(dict, rsp_dict, snap);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_SNAP_CREATION_FAIL,
+               "Failed to take backend "
+               "snapshot %s",
+               snap->snapname);
+        goto out;
+    }
+
+    ret = dict_set_dynstr_with_alloc(rsp_dict, "snapuuid",
+                                     uuid_utoa(snap->snap_id));
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+               "Failed to set snap "
+               "uuid in response dictionary for %s snapshot",
+               snap->snapname);
+        goto out;
+    }
+
+    snap_activate = dict_get_str_boolean(
+        priv->opts, GLUSTERD_STORE_KEY_SNAP_ACTIVATE, _gf_false);
+    if (!snap_activate) {
+        cds_list_for_each_entry(snap_vol, &snap->volumes, vol_list)
+        {
+            snap_vol->status = GLUSTERD_STATUS_STOPPED;
+            ret = glusterd_store_volinfo(snap_vol,
+                                         GLUSTERD_VOLINFO_VER_AC_INCREMENT);
+            if (ret) {
+                gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_VOLINFO_SET_FAIL,
+                       "Failed to store snap volinfo %s", snap_vol->volname);
                 goto out;
+            }
         }
 
-        for (i = 1; i <= volcount; i++) {
-                snprintf (key, sizeof (key), "volname%"PRId64, i);
-                ret = dict_get_str (dict, key, &volname);
-                if (ret) {
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "failed to get volume name");
-                        goto out;
-                }
-
-                ret = glusterd_volinfo_find (volname, &origin_vol);
-                if (ret) {
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "failed to get the volinfo for "
-                                "the volume %s", volname);
-                        goto out;
-                }
+        goto out;
+    }
 
-                if (is_origin_glusterd (dict)) {
-                        ret = glusterd_is_snap_soft_limit_reached (origin_vol,
-                                                                   rsp_dict);
-                        if (ret) {
-                                gf_log (this->name, GF_LOG_ERROR, "Failed to "
-                                        "check soft limit exceeded or not, "
-                                        "for volume %s ", origin_vol->volname);
-                                goto out;
-                        }
-                }
+    /* Activate created bricks in case of activate-on-create config. */
+    ret = dict_get_int32n(dict, "flags", SLEN("flags"), &flags);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+               "Unable to get flags");
+        goto out;
+    }
 
-                snap_vol = glusterd_do_snap_vol (origin_vol, snap, dict,
-                                                 rsp_dict, i, 0);
-                if (!snap_vol) {
-                        ret = -1;
-                        gf_log (this->name, GF_LOG_WARNING, "taking the "
-                                "snapshot of the volume %s failed", volname);
-                        goto out;
-                }
-        }
-        ret = dict_set_int64 (rsp_dict, "volcount", volcount);
+    cds_list_for_each_entry(snap_vol, &snap->volumes, vol_list)
+    {
+        ret = glusterd_start_volume(snap_vol, flags, _gf_true);
         if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "Failed to set volcount");
-                goto out;
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_SNAP_ACTIVATE_FAIL,
+                   "Failed to activate snap volume %s of the "
+                   "snap %s",
+                   snap_vol->volname, snap->snapname);
+            goto out;
         }
+    }
 
-        ret = glusterd_schedule_brick_snapshot (dict, rsp_dict, snap);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "Failed to take backend "
-                        "snapshot %s", snap->snapname);
-                goto out;
-        }
-
-        ret = dict_set_dynstr_with_alloc (rsp_dict, "snapuuid",
-                                          uuid_utoa (snap->snap_id));
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "Failed to set snap "
-                        "uuid in response dictionary for %s snapshot",
-                        snap->snapname);
-                goto out;
-        }
-
-        snap_activate = dict_get_str_boolean (priv->opts,
-                                              GLUSTERD_STORE_KEY_SNAP_ACTIVATE,
-                                              _gf_false);
-        if (!snap_activate) {
-                cds_list_for_each_entry (snap_vol, &snap->volumes, vol_list) {
-                        snap_vol->status = GLUSTERD_STATUS_STOPPED;
-                        ret = glusterd_store_volinfo (snap_vol,
-                                             GLUSTERD_VOLINFO_VER_AC_INCREMENT);
-                        if (ret) {
-                                gf_log (this->name, GF_LOG_ERROR,
-                                        "Failed to store snap volinfo %s",
-                                        snap_vol->volname);
-                                goto out;
-                        }
-                }
-
-                goto out;
-        }
-
-        cds_list_for_each_entry (snap_vol, &snap->volumes, vol_list) {
-                cds_list_for_each_entry (brickinfo, &snap_vol->bricks,
-                                         brick_list) {
-                        ret = glusterd_brick_start (snap_vol, brickinfo,
-                                                    _gf_false);
-                        if (ret) {
-                                gf_log (this->name, GF_LOG_WARNING, "starting "
-                                        "the brick %s:%s for the snap %s "
-                                        "(volume: %s) failed",
-                                        brickinfo->hostname, brickinfo->path,
-                                        snap_vol->snapshot->snapname,
-                                        snap_vol->volname);
-                                goto out;
-                        }
-                }
-
-                snap_vol->status = GLUSTERD_STATUS_STARTED;
-                ret = glusterd_store_volinfo (snap_vol,
-                                             GLUSTERD_VOLINFO_VER_AC_INCREMENT);
-                if (ret) {
-                        gf_log (this->name, GF_LOG_ERROR, "Failed to store "
-                                 "snap volinfo %s", snap_vol->volname);
-                        goto out;
-                }
-        }
-
-        ret = 0;
+    ret = 0;
 
 out:
-        if (ret) {
-                if (snap)
-                        glusterd_snap_remove (rsp_dict, snap,
-                                              _gf_true, _gf_true);
-                snap = NULL;
-        }
+    if (ret) {
+        if (snap)
+            glusterd_snap_remove(rsp_dict, snap, _gf_true, _gf_true, _gf_false);
+        snap = NULL;
+    }
 
-        gf_log (this->name, GF_LOG_TRACE, "Returning %d", ret);
-        return ret;
+    gf_msg_trace(this->name, 0, "Returning %d", ret);
+    return ret;
 }
 
 int
-snap_max_hard_limit_set_commit (dict_t *dict, uint64_t value,
-                                char *volname, char **op_errstr)
+snap_max_hard_limit_set_commit(dict_t *dict, uint64_t value, char *volname,
+                               char **op_errstr)
 {
-        char                err_str[PATH_MAX]    = "";
-        glusterd_conf_t    *conf                 = NULL;
-        glusterd_volinfo_t *volinfo              = NULL;
-        int                 ret                  = -1;
-        xlator_t           *this                 = NULL;
-        char               *next_version         = NULL;
+    char err_str[PATH_MAX] = "";
+    glusterd_conf_t *conf = NULL;
+    glusterd_volinfo_t *volinfo = NULL;
+    int ret = -1;
+    xlator_t *this = NULL;
+    char *next_version = NULL;
 
-        this = THIS;
+    this = THIS;
 
-        GF_ASSERT (this);
-        GF_ASSERT (dict);
-        GF_ASSERT (op_errstr);
+    GF_ASSERT(this);
+    GF_ASSERT(dict);
+    GF_ASSERT(op_errstr);
 
-        conf = this->private;
+    conf = this->private;
 
-        GF_ASSERT (conf);
-
-        /* TODO: Initiate auto deletion when there is a limit change */
-        if (!volname) {
-                /* For system limit */
-                ret = dict_set_uint64 (conf->opts,
-                                       GLUSTERD_STORE_KEY_SNAP_MAX_HARD_LIMIT,
-                                       value);
-                if (ret) {
-                        gf_log (this->name, GF_LOG_ERROR, "Failed to store "
-                                "%s in the options",
-                                GLUSTERD_STORE_KEY_SNAP_MAX_HARD_LIMIT);
-                        goto out;
-                }
+    GF_ASSERT(conf);
 
+    /* TODO: Initiate auto deletion when there is a limit change */
+    if (!volname) {
+        /* For system limit */
+        ret = dict_set_uint64(conf->opts,
+                              GLUSTERD_STORE_KEY_SNAP_MAX_HARD_LIMIT, value);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+                   "Failed to store "
+                   "%s in the options",
+                   GLUSTERD_STORE_KEY_SNAP_MAX_HARD_LIMIT);
+            goto out;
+        }
 
-                ret = glusterd_get_next_global_opt_version_str (conf->opts,
-                                                                &next_version);
-                if (ret)
-                        goto out;
+        ret = glusterd_get_next_global_opt_version_str(conf->opts,
+                                                       &next_version);
+        if (ret)
+            goto out;
 
-                ret = dict_set_str (conf->opts, GLUSTERD_GLOBAL_OPT_VERSION,
-                                    next_version);
-                if (ret)
-                        goto out;
+        ret = dict_set_strn(conf->opts, GLUSTERD_GLOBAL_OPT_VERSION,
+                            SLEN(GLUSTERD_GLOBAL_OPT_VERSION), next_version);
+        if (ret)
+            goto out;
 
-                ret = glusterd_store_options (this, conf->opts);
-                if (ret) {
-                        gf_log (this->name, GF_LOG_ERROR, "Failed to store "
-                                "options");
-                        goto out;
-                }
-        } else {
-               /*  For one volume */
-                ret = glusterd_volinfo_find (volname, &volinfo);
-                if (ret) {
-                        snprintf (err_str, PATH_MAX, "Failed to get the"
-                                   " volinfo for volume %s", volname);
-                        goto out;
-                }
+        ret = glusterd_store_options(this, conf->opts);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_STORE_FAIL,
+                   "Failed to store "
+                   "options");
+            goto out;
+        }
+    } else {
+        /*  For one volume */
+        ret = glusterd_volinfo_find(volname, &volinfo);
+        if (ret) {
+            snprintf(err_str, PATH_MAX,
+                     "Failed to get the"
+                     " volinfo for volume %s",
+                     volname);
+            goto out;
+        }
 
-                volinfo->snap_max_hard_limit = value;
+        volinfo->snap_max_hard_limit = value;
 
-                ret = glusterd_store_volinfo (volinfo,
-                                        GLUSTERD_VOLINFO_VER_AC_INCREMENT);
-                if (ret) {
-                        snprintf (err_str, PATH_MAX, "Failed to store "
-                                 "snap-max-hard-limit for volume %s", volname);
-                        goto out;
-                }
+        ret = glusterd_store_volinfo(volinfo,
+                                     GLUSTERD_VOLINFO_VER_AC_INCREMENT);
+        if (ret) {
+            snprintf(err_str, PATH_MAX,
+                     "Failed to store "
+                     "snap-max-hard-limit for volume %s",
+                     volname);
+            goto out;
         }
+    }
 
-        ret = 0;
+    ret = 0;
 out:
-        if (ret) {
-                *op_errstr = gf_strdup (err_str);
-                gf_log (this->name, GF_LOG_ERROR, "%s", err_str);
-        }
-        return ret;
+    if (ret) {
+        *op_errstr = gf_strdup(err_str);
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_SNAPSHOT_OP_FAILED, "%s",
+               err_str);
+    }
+    return ret;
 }
 
 int
-glusterd_snapshot_config_commit (dict_t *dict, char **op_errstr,
-                                 dict_t *rsp_dict)
+glusterd_snapshot_config_commit(dict_t *dict, char **op_errstr,
+                                dict_t *rsp_dict)
 {
-        char               *volname              = NULL;
-        xlator_t           *this                 = NULL;
-        int                 ret                  = -1;
-        char                err_str[PATH_MAX]    = {0,};
-        glusterd_conf_t    *conf                 = NULL;
-        int                 config_command       = 0;
-        uint64_t            hard_limit           = 0;
-        uint64_t            soft_limit           = 0;
-        char               *next_version         = NULL;
-        char               *auto_delete          = NULL;
-        char               *snap_activate        = NULL;
-        gf_boolean_t        system_conf          = _gf_false;
-
-        this = THIS;
-
-        GF_ASSERT (this);
-        GF_ASSERT (dict);
-        GF_ASSERT (op_errstr);
-
-        conf = this->private;
-
-        GF_ASSERT (conf);
-
-        ret = dict_get_int32 (dict, "config-command", &config_command);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "failed to get config-command type");
-                goto out;
-        }
-        if (config_command != GF_SNAP_CONFIG_TYPE_SET) {
-                ret = 0;
-                goto out;
-        }
-
-        ret = dict_get_str (dict, "volname", &volname);
+    char *volname = NULL;
+    xlator_t *this = NULL;
+    int ret = -1;
+    glusterd_conf_t *conf = NULL;
+    int config_command = 0;
+    uint64_t hard_limit = 0;
+    uint64_t soft_limit = 0;
+    char *next_version = NULL;
+    char *auto_delete = NULL;
+    char *snap_activate = NULL;
+    gf_boolean_t system_conf = _gf_false;
+
+    this = THIS;
+
+    GF_ASSERT(this);
+    GF_ASSERT(dict);
+    GF_ASSERT(op_errstr);
+
+    conf = this->private;
+
+    GF_ASSERT(conf);
+
+    ret = dict_get_int32n(dict, "config-command", SLEN("config-command"),
+                          &config_command);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_COMMAND_NOT_FOUND,
+               "failed to get config-command type");
+        goto out;
+    }
+    if (config_command != GF_SNAP_CONFIG_TYPE_SET) {
+        ret = 0;
+        goto out;
+    }
 
-        /* config values snap-max-hard-limit and snap-max-soft-limit are
-         * optional and hence we are not erroring out if values are not
-         * present
-         */
-        gd_get_snap_conf_values_if_present (dict, &hard_limit,
-                                            &soft_limit);
+    ret = dict_get_strn(dict, "volname", SLEN("volname"), &volname);
 
-        if (hard_limit) {
-                /* Commit ops for snap-max-hard-limit */
-                ret = snap_max_hard_limit_set_commit (dict, hard_limit, volname,
-                                                      op_errstr);
-                if (ret) {
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "snap-max-hard-limit set commit failed.");
-                        goto out;
-                }
-        }
+    /* config values snap-max-hard-limit and snap-max-soft-limit are
+     * optional and hence we are not erroring out if values are not
+     * present
+     */
+    gd_get_snap_conf_values_if_present(dict, &hard_limit, &soft_limit);
 
-        if (soft_limit) {
-                /* For system limit */
-                system_conf = _gf_true;
-                ret = dict_set_uint64 (conf->opts,
-                                       GLUSTERD_STORE_KEY_SNAP_MAX_SOFT_LIMIT,
-                                       soft_limit);
-                if (ret) {
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "Failed to save %s in the dictionary",
-                                GLUSTERD_STORE_KEY_SNAP_MAX_SOFT_LIMIT);
-                        goto out;
-                }
+    if (hard_limit) {
+        /* Commit ops for snap-max-hard-limit */
+        ret = snap_max_hard_limit_set_commit(dict, hard_limit, volname,
+                                             op_errstr);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_HARD_LIMIT_SET_FAIL,
+                   "snap-max-hard-limit set commit failed.");
+            goto out;
         }
+    }
 
-        if (hard_limit || soft_limit) {
-                ret = 0;
-                goto done;
+    if (soft_limit) {
+        /* For system limit */
+        system_conf = _gf_true;
+        ret = dict_set_uint64(
+            conf->opts, GLUSTERD_STORE_KEY_SNAP_MAX_SOFT_LIMIT, soft_limit);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+                   "Failed to save %s in the dictionary",
+                   GLUSTERD_STORE_KEY_SNAP_MAX_SOFT_LIMIT);
+            goto out;
         }
+    }
 
-        if (!dict_get_str(dict,
-                          GLUSTERD_STORE_KEY_SNAP_AUTO_DELETE,
-                          &auto_delete)) {
-                system_conf = _gf_true;
-                ret = dict_set_dynstr_with_alloc (conf->opts,
-                                GLUSTERD_STORE_KEY_SNAP_AUTO_DELETE,
-                                auto_delete);
-                if (ret) {
-                        gf_log (this->name, GF_LOG_ERROR, "Could not "
-                                "save auto-delete value in conf->opts");
-                        goto out;
-                }
-        } else if (!dict_get_str(dict,
-                                 GLUSTERD_STORE_KEY_SNAP_ACTIVATE,
-                                 &snap_activate)) {
-                system_conf = _gf_true;
-                ret = dict_set_dynstr_with_alloc (conf->opts,
-                                GLUSTERD_STORE_KEY_SNAP_ACTIVATE,
-                                snap_activate);
-                if (ret) {
-                        gf_log (this->name, GF_LOG_ERROR, "Could not save "
-                                "snap-activate-on-create value in conf->opts");
-                        goto out;
-                }
-        } else {
-                ret = -1;
-                gf_log (this->name, GF_LOG_ERROR, "Invalid option");
-                goto out;
-        }
+    if (hard_limit || soft_limit) {
+        ret = 0;
+        goto done;
+    }
+
+    if (!dict_get_strn(dict, GLUSTERD_STORE_KEY_SNAP_AUTO_DELETE,
+                       SLEN(GLUSTERD_STORE_KEY_SNAP_AUTO_DELETE),
+                       &auto_delete)) {
+        system_conf = _gf_true;
+        ret = dict_set_dynstr_with_alloc(
+            conf->opts, GLUSTERD_STORE_KEY_SNAP_AUTO_DELETE, auto_delete);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+                   "Could not "
+                   "save auto-delete value in conf->opts");
+            goto out;
+        }
+    } else if (!dict_get_strn(dict, GLUSTERD_STORE_KEY_SNAP_ACTIVATE,
+                              SLEN(GLUSTERD_STORE_KEY_SNAP_ACTIVATE),
+                              &snap_activate)) {
+        system_conf = _gf_true;
+        ret = dict_set_dynstr_with_alloc(
+            conf->opts, GLUSTERD_STORE_KEY_SNAP_ACTIVATE, snap_activate);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+                   "Could not save "
+                   "snap-activate-on-create value in conf->opts");
+            goto out;
+        }
+    } else {
+        ret = -1;
+        gf_msg(this->name, GF_LOG_ERROR, EINVAL, GD_MSG_INVALID_ENTRY,
+               "Invalid option");
+        goto out;
+    }
 
 done:
-        if (system_conf) {
-                ret = glusterd_get_next_global_opt_version_str (conf->opts,
-                                                                &next_version);
-                if (ret) {
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "Failed to get next global opt-version");
-                                goto out;
-                }
+    if (system_conf) {
+        ret = glusterd_get_next_global_opt_version_str(conf->opts,
+                                                       &next_version);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, 0,
+                   GD_MSG_GLOBAL_OP_VERSION_GET_FAIL,
+                   "Failed to get next global opt-version");
+            goto out;
+        }
 
-                ret = dict_set_str (conf->opts, GLUSTERD_GLOBAL_OPT_VERSION,
-                                    next_version);
-                if (ret) {
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "Failed to set next global opt-version");
-                        goto out;
-                }
+        ret = dict_set_strn(conf->opts, GLUSTERD_GLOBAL_OPT_VERSION,
+                            SLEN(GLUSTERD_GLOBAL_OPT_VERSION), next_version);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, 0,
+                   GD_MSG_GLOBAL_OP_VERSION_SET_FAIL,
+                   "Failed to set next global opt-version");
+            goto out;
+        }
 
-                ret = glusterd_store_options (this, conf->opts);
-                if (ret) {
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "Failed to store options");
-                        goto out;
-                }
+        ret = glusterd_store_options(this, conf->opts);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_STORE_FAIL,
+                   "Failed to store options");
+            goto out;
         }
+    }
 
 out:
-        gf_log (this->name, GF_LOG_TRACE, "Returning %d", ret);
-        return ret;
+    gf_msg_trace(this->name, 0, "Returning %d", ret);
+    return ret;
 }
 
-int
-glusterd_get_brick_lvm_details (dict_t *rsp_dict,
+static int
+glusterd_get_brick_lvm_details(dict_t *rsp_dict,
                                glusterd_brickinfo_t *brickinfo, char *volname,
-                                char *device, char *key_prefix)
+                               char *device, const char *key_prefix)
 {
+    int ret = -1;
+    glusterd_conf_t *priv = NULL;
+    runner_t runner = {
+        0,
+    };
+    xlator_t *this = NULL;
+    char msg[PATH_MAX] = "";
+    char buf[PATH_MAX] = "";
+    char *ptr = NULL;
+    char *token = NULL;
+    char key[160] = ""; /* key_prefix is 128 bytes at most */
+    char *value = NULL;
+
+    GF_ASSERT(rsp_dict);
+    GF_ASSERT(brickinfo);
+    GF_ASSERT(volname);
+    this = THIS;
+    GF_ASSERT(this);
+    priv = this->private;
+    GF_ASSERT(priv);
+
+    device = glusterd_get_brick_mount_device(brickinfo->path);
+    if (!device) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_BRICK_GET_INFO_FAIL,
+               "Getting device name for "
+               "the brick %s:%s failed",
+               brickinfo->hostname, brickinfo->path);
+        goto out;
+    }
+    runinit(&runner);
+    snprintf(msg, sizeof(msg),
+             "running lvs command, "
+             "for getting snap status");
+    /* Using lvs command fetch the Volume Group name,
+     * Percentage of data filled and Logical Volume size
+     *
+     * "-o" argument is used to get the desired information,
+     * example : "lvs /dev/VolGroup/thin_vol -o vgname,lv_size",
+     * will get us Volume Group name and Logical Volume size.
+     *
+     * Here separator used is ":",
+     * for the above given command with separator ":",
+     * The output will be "vgname:lvsize"
+     */
+    runner_add_args(&runner, LVS, device, "--noheading", "-o",
+                    "vg_name,data_percent,lv_size", "--separator", ":", NULL);
+    runner_redir(&runner, STDOUT_FILENO, RUN_PIPE);
+    runner_log(&runner, "", GF_LOG_DEBUG, msg);
+    ret = runner_start(&runner);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, errno, GD_MSG_LVS_FAIL,
+               "Could not perform lvs action");
+        goto end;
+    }
+    do {
+        ptr = fgets(buf, sizeof(buf), runner_chio(&runner, STDOUT_FILENO));
+
+        if (ptr == NULL)
+            break;
+        token = strtok(buf, ":");
+        if (token != NULL) {
+            while (token[0] == ' ')
+                token++;
+            value = gf_strdup(token);
+            if (!value) {
+                ret = -1;
+                goto end;
+            }
+            ret = snprintf(key, sizeof(key), "%s.vgname", key_prefix);
+            if (ret < 0) {
+                goto end;
+            }
 
-        int                     ret             =       -1;
-        glusterd_conf_t         *priv           =       NULL;
-        runner_t                runner          =       {0,};
-        xlator_t                *this           =       NULL;
-        char                    msg[PATH_MAX]   =       "";
-        char                    buf[PATH_MAX]   =       "";
-        char                    *ptr            =       NULL;
-        char                    *token          =       NULL;
-        char                    key[PATH_MAX]   =       "";
-        char                    *value          =       NULL;
-
-        GF_ASSERT (rsp_dict);
-        GF_ASSERT (brickinfo);
-        GF_ASSERT (volname);
-        this = THIS;
-        GF_ASSERT (this);
-        priv = this->private;
-        GF_ASSERT (priv);
-
-        device = glusterd_get_brick_mount_device (brickinfo->path);
-        if (!device) {
-                gf_log (this->name, GF_LOG_ERROR, "Getting device name for "
-                        "the brick %s:%s failed", brickinfo->hostname,
-                         brickinfo->path);
-                goto out;
-        }
-        runinit (&runner);
-        snprintf (msg, sizeof (msg), "running lvs command, "
-                  "for getting snap status");
-        /* Using lvs command fetch the Volume Group name,
-         * Percentage of data filled and Logical Volume size
-         *
-         * "-o" argument is used to get the desired information,
-         * example : "lvs /dev/VolGroup/thin_vol -o vgname,lv_size",
-         * will get us Volume Group name and Logical Volume size.
-         *
-         * Here separator used is ":",
-         * for the above given command with separator ":",
-         * The output will be "vgname:lvsize"
-         */
-        runner_add_args (&runner, LVS, device, "--noheading", "-o",
-                         "vg_name,data_percent,lv_size",
-                         "--separator", ":", NULL);
-        runner_redir (&runner, STDOUT_FILENO, RUN_PIPE);
-        runner_log (&runner, "", GF_LOG_DEBUG, msg);
-        ret = runner_start (&runner);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "Could not perform lvs action");
+            ret = dict_set_dynstr(rsp_dict, key, value);
+            if (ret) {
+                gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+                       "Could not save vgname ");
                 goto end;
+            }
         }
-        do {
-                ptr = fgets (buf, sizeof (buf),
-                             runner_chio (&runner, STDOUT_FILENO));
-
-                if (ptr == NULL)
-                        break;
-                token = strtok (buf, ":");
-                if (token != NULL) {
-                        while (token && token[0] == ' ')
-                                token++;
-                        if (!token) {
-                                ret = -1;
-                                gf_log (this->name, GF_LOG_ERROR,
-                                        "Invalid vg entry");
-                                goto end;
-                        }
-                        value = gf_strdup (token);
-                        if (!value) {
-                                ret = -1;
-                                goto end;
-                        }
-                        ret = snprintf (key, sizeof (key), "%s.vgname",
-                                        key_prefix);
-                        if (ret < 0) {
-                                goto end;
-                        }
-
-                        ret = dict_set_dynstr (rsp_dict, key, value);
-                        if (ret) {
-                                gf_log (this->name, GF_LOG_ERROR,
-                                        "Could not save vgname ");
-                                goto end;
-                        }
-                }
 
-                token = strtok (NULL, ":");
-                if (token != NULL) {
-                        value = gf_strdup (token);
-                        if (!value) {
-                                ret = -1;
-                                goto end;
-                        }
-                        ret = snprintf (key, sizeof (key), "%s.data",
-                                        key_prefix);
-                        if (ret < 0) {
-                                goto end;
-                        }
-
-                        ret = dict_set_dynstr (rsp_dict, key, value);
-                        if (ret) {
-                                gf_log (this->name, GF_LOG_ERROR,
-                                        "Could not save data percent ");
-                                goto end;
-                        }
-                }
-                token = strtok (NULL, ":");
-                if (token != NULL) {
-                        value = gf_strdup (token);
-                        if (!value) {
-                                ret = -1;
-                                goto end;
-                        }
-                        ret = snprintf (key, sizeof (key), "%s.lvsize",
-                                        key_prefix);
-                        if (ret < 0) {
-                                goto end;
-                        }
-
-                        ret = dict_set_dynstr (rsp_dict, key, value);
-                        if (ret) {
-                                gf_log (this->name, GF_LOG_ERROR,
-                                        "Could not save meta data percent ");
-                                goto end;
-                        }
-                }
+        token = strtok(NULL, ":");
+        if (token != NULL) {
+            value = gf_strdup(token);
+            if (!value) {
+                gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_STRDUP_FAILED,
+                        "token=%s", token, NULL);
+                ret = -1;
+                goto end;
+            }
+            ret = snprintf(key, sizeof(key), "%s.data", key_prefix);
+            if (ret < 0) {
+                gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_COPY_FAIL,
+                        NULL);
+                goto end;
+            }
+
+            ret = dict_set_dynstr(rsp_dict, key, value);
+            if (ret) {
+                gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+                       "Could not save data percent ");
+                goto end;
+            }
+        }
+        token = strtok(NULL, ":");
+        if (token != NULL) {
+            value = gf_strdup(token);
+            if (!value) {
+                gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_STRDUP_FAILED,
+                        "token=%s", token, NULL);
+                ret = -1;
+                goto end;
+            }
+            ret = snprintf(key, sizeof(key), "%s.lvsize", key_prefix);
+            if (ret < 0) {
+                gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_COPY_FAIL,
+                        NULL);
+                goto end;
+            }
 
-        } while (ptr != NULL);
+            ret = dict_set_dynstr(rsp_dict, key, value);
+            if (ret) {
+                gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+                       "Could not save meta data percent ");
+                goto end;
+            }
+        }
 
-        ret = 0;
+    } while (ptr != NULL);
+
+    ret = 0;
 
 end:
-        runner_end (&runner);
+    runner_end(&runner);
 
 out:
-        if (ret && value) {
-                GF_FREE (value);
-        }
+    if (ret && value) {
+        GF_FREE(value);
+    }
+
+    if (device)
+        GF_FREE(device);
 
-        return ret;
+    return ret;
 }
 
-int
-glusterd_get_single_brick_status (char **op_errstr, dict_t *rsp_dict,
-                                 char *keyprefix, int index,
+static int
+glusterd_get_single_brick_status(char **op_errstr, dict_t *rsp_dict,
+                                 const char *keyprefix, int index,
                                  glusterd_volinfo_t *snap_volinfo,
                                  glusterd_brickinfo_t *brickinfo)
 {
-        int             ret                     = -1;
-        xlator_t        *this                   = NULL;
-        glusterd_conf_t *priv                   = NULL;
-        char            key[PATH_MAX]           = "";
-        char            *device                 = NULL;
-        char            *value                  = NULL;
-        char            brick_path[PATH_MAX]    = "";
-        char            pidfile[PATH_MAX]       = "";
-        pid_t           pid                     = -1;
-
-        this = THIS;
-        GF_ASSERT (this);
-        priv = this->private;
-        GF_ASSERT (priv);
-
-        GF_ASSERT (op_errstr);
-        GF_ASSERT (rsp_dict);
-        GF_ASSERT (keyprefix);
-        GF_ASSERT (snap_volinfo);
-        GF_ASSERT (brickinfo);
-
-        ret = snprintf (key, sizeof (key), "%s.brick%d.path", keyprefix,
-                        index);
-        if (ret < 0) {
-                goto out;
+    int ret = -1;
+    xlator_t *this = NULL;
+    glusterd_conf_t *priv = NULL;
+    char key[128] = ""; /* keyprefix is not longer than 64 bytes */
+    int keylen;
+    char *device = NULL;
+    char *value = NULL;
+    char brick_path[PATH_MAX] = "";
+    char pidfile[PATH_MAX] = "";
+    pid_t pid = -1;
+
+    this = THIS;
+    GF_ASSERT(this);
+    priv = this->private;
+    GF_ASSERT(priv);
+
+    GF_ASSERT(op_errstr);
+    GF_ASSERT(rsp_dict);
+    GF_ASSERT(keyprefix);
+    GF_ASSERT(snap_volinfo);
+    GF_ASSERT(brickinfo);
+
+    keylen = snprintf(key, sizeof(key), "%s.brick%d.path", keyprefix, index);
+    if (keylen < 0) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_COPY_FAIL, NULL);
+        ret = -1;
+        goto out;
+    }
+
+    ret = snprintf(brick_path, sizeof(brick_path), "%s:%s", brickinfo->hostname,
+                   brickinfo->path);
+    if (ret < 0) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_COPY_FAIL, NULL);
+        goto out;
+    }
+
+    value = gf_strdup(brick_path);
+    if (!value) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_STRDUP_FAILED,
+                "brick_path=%s", brick_path, NULL);
+        ret = -1;
+        goto out;
+    }
+
+    ret = dict_set_dynstrn(rsp_dict, key, keylen, value);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+               "Unable to store "
+               "brick_path %s",
+               brickinfo->path);
+        goto out;
+    }
+
+    if (brickinfo->snap_status == -1) {
+        /* Setting vgname as "Pending Snapshot" */
+        value = gf_strdup("Pending Snapshot");
+        if (!value) {
+            ret = -1;
+            goto out;
         }
 
-        ret = snprintf (brick_path, sizeof (brick_path),
-                        "%s:%s", brickinfo->hostname, brickinfo->path);
-        if (ret < 0) {
-                goto out;
+        keylen = snprintf(key, sizeof(key), "%s.brick%d.vgname", keyprefix,
+                          index);
+        ret = dict_set_dynstrn(rsp_dict, key, keylen, value);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+                   "Could not save vgname ");
+            goto out;
         }
 
-        value = gf_strdup (brick_path);
+        ret = 0;
+        goto out;
+    }
+    value = NULL;
+
+    keylen = snprintf(key, sizeof(key), "%s.brick%d.status", keyprefix, index);
+    if (keylen < 0) {
+        ret = -1;
+        goto out;
+    }
+
+    if (brickinfo->status == GF_BRICK_STOPPED) {
+        value = gf_strdup("No");
         if (!value) {
-                ret = -1;
-                goto out;
+            ret = -1;
+            goto out;
         }
-
-        ret = dict_set_dynstr (rsp_dict, key, value);
+        ret = dict_set_strn(rsp_dict, key, keylen, value);
         if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "Unable to store "
-                        "brick_path %s", brickinfo->path);
-                goto out;
-        }
-
-        if (brickinfo->snap_status == -1) {
-                /* Setting vgname as "Pending Snapshot" */
-                value = gf_strdup ("Pending Snapshot");
-                if (!value) {
-                        ret = -1;
-                        goto out;
-                }
-
-                snprintf (key, sizeof (key), "%s.brick%d.vgname",
-                          keyprefix, index);
-                ret = dict_set_dynstr (rsp_dict, key, value);
-                if (ret) {
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "Could not save vgname ");
-                        goto out;
-                }
-
-                ret = 0;
-                goto out;
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+                   "Could not save brick status");
+            goto out;
         }
         value = NULL;
-
-        ret = snprintf (key, sizeof (key), "%s.brick%d.status",
-                        keyprefix, index);
-        if (ret < 0) {
-                goto out;
-        }
-
-        if (brickinfo->status == GF_BRICK_STOPPED) {
-                value = gf_strdup ("No");
-                if (!value) {
-                        ret = -1;
-                        goto out;
-                }
-                ret = dict_set_str (rsp_dict, key, value);
-                if (ret) {
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "Could not save brick status");
-                        goto out;
-                }
-                value = NULL;
-        } else {
-                value = gf_strdup ("Yes");
-                if (!value) {
-                        ret = -1;
-                        goto out;
-                }
-                ret = dict_set_str (rsp_dict, key, value);
-                if (ret) {
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "Could not save brick status");
-                        goto out;
-                }
-                value = NULL;
-
-                GLUSTERD_GET_BRICK_PIDFILE (pidfile, snap_volinfo,
-                                            brickinfo, priv);
-                ret = gf_is_service_running (pidfile, &pid);
-
-                ret = snprintf (key, sizeof (key), "%s.brick%d.pid",
-                                keyprefix, index);
-                if (ret < 0) {
-                        goto out;
-                }
-
-                ret = dict_set_int32 (rsp_dict, key, pid);
-                if (ret) {
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "Could not save pid %d", pid);
-                        goto out;
-                }
-        }
-
-        ret = snprintf (key, sizeof (key), "%s.brick%d",
-                        keyprefix, index);
-        if (ret < 0) {
-                goto out;
+    } else {
+        value = gf_strdup("Yes");
+        if (!value) {
+            ret = -1;
+            goto out;
         }
-
-        ret = glusterd_get_brick_lvm_details (rsp_dict, brickinfo,
-                                              snap_volinfo->volname,
-                                              device, key);
+        ret = dict_set_strn(rsp_dict, key, keylen, value);
         if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "Failed to get "
-                        "brick LVM details");
-                goto out;
-        }
-out:
-        if (ret && value) {
-                GF_FREE (value);
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+                   "Could not save brick status");
+            goto out;
         }
+        value = NULL;
 
-        return ret;
-}
-
-int
-glusterd_get_single_snap_status (char **op_errstr, dict_t *rsp_dict,
-                                 char *keyprefix, glusterd_snap_t *snap)
-{
-        int                      ret                 =       -1;
-        xlator_t                *this                =       NULL;
-        char                     key[PATH_MAX]       =       "";
-        char                     brickkey[PATH_MAX]  =       "";
-        glusterd_volinfo_t      *snap_volinfo        =       NULL;
-        glusterd_volinfo_t      *tmp_volinfo         =       NULL;
-        glusterd_brickinfo_t    *brickinfo           =       NULL;
-        int                      volcount            =       0;
-        int                      brickcount          =       0;
-
-        this = THIS;
-        GF_ASSERT (this);
-
-        GF_ASSERT (op_errstr);
-        GF_ASSERT (rsp_dict);
-        GF_ASSERT (keyprefix);
-        GF_ASSERT (snap);
-
-        cds_list_for_each_entry_safe (snap_volinfo, tmp_volinfo, &snap->volumes,
-                                      vol_list) {
-                ret = snprintf (key, sizeof (key), "%s.vol%d", keyprefix,
-                                volcount);
-                if (ret < 0) {
-                        goto out;
-                }
-                cds_list_for_each_entry (brickinfo, &snap_volinfo->bricks,
-                                         brick_list) {
-                        if (!glusterd_is_local_brick (this, snap_volinfo,
-                            brickinfo)) {
-                                brickcount++;
-                                continue;
-                        }
-
-                        ret = glusterd_get_single_brick_status (op_errstr,
-                                                  rsp_dict, key, brickcount,
-                                                  snap_volinfo, brickinfo);
-
-                        if (ret) {
-                                gf_log (this->name, GF_LOG_ERROR, "Getting "
-                                        "single snap status failed");
-                                goto out;
-                        }
-                        brickcount++;
-                }
-                ret = snprintf (brickkey, sizeof (brickkey), "%s.brickcount",
-                                key);
-                if (ret < 0) {
-                        goto out;
-                }
-
-                ret = dict_set_int32 (rsp_dict, brickkey, brickcount);
-                if (ret) {
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "Could not save brick count");
-                        goto out;
-                }
-                volcount++;
-        }
+        GLUSTERD_GET_BRICK_PIDFILE(pidfile, snap_volinfo, brickinfo, priv);
 
-        ret = snprintf (key, sizeof (key), "%s.volcount", keyprefix);
-        if (ret < 0) {
-                goto out;
+        if (gf_is_service_running(pidfile, &pid)) {
+            keylen = snprintf(key, sizeof(key), "%s.brick%d.pid", keyprefix,
+                              index);
+            if (keylen < 0) {
+                ret = -1;
+                gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_COPY_FAIL,
+                        NULL);
+                goto out;
+            }
+
+            ret = dict_set_int32n(rsp_dict, key, keylen, pid);
+            if (ret) {
+                gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+                       "Could not save pid %d", pid);
+                goto out;
+            }
+        }
+    }
+
+    keylen = snprintf(key, sizeof(key), "%s.brick%d", keyprefix, index);
+    if (keylen < 0) {
+        ret = -1;
+        goto out;
+    }
+    /* While getting snap status we should show relevant information
+     * for deactivated snaps.
+     */
+    if (snap_volinfo->status == GLUSTERD_STATUS_STOPPED) {
+        /* Setting vgname as "Deactivated Snapshot" */
+        value = gf_strdup("N/A (Deactivated Snapshot)");
+        if (!value) {
+            ret = -1;
+            goto out;
         }
 
-        ret = dict_set_int32 (rsp_dict, key, volcount);
+        keylen = snprintf(key, sizeof(key), "%s.brick%d.vgname", keyprefix,
+                          index);
+        ret = dict_set_dynstrn(rsp_dict, key, keylen, value);
         if (ret) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "Could not save volcount");
-                goto out;
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+                   "Could not save vgname ");
+            goto out;
         }
 
+        ret = 0;
+        goto out;
+    }
+
+    ret = glusterd_get_brick_lvm_details(rsp_dict, brickinfo,
+                                         snap_volinfo->volname, device, key);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_BRICK_GET_INFO_FAIL,
+               "Failed to get "
+               "brick LVM details");
+        goto out;
+    }
 out:
+    if (ret && value) {
+        GF_FREE(value);
+    }
 
-        return ret;
+    return ret;
 }
 
-int
-glusterd_get_each_snap_object_status (char **op_errstr, dict_t *rsp_dict,
-                                      glusterd_snap_t *snap, char *keyprefix)
+static int
+glusterd_get_single_snap_status(char **op_errstr, dict_t *rsp_dict,
+                                const char *keyprefix, glusterd_snap_t *snap)
 {
-        int                     ret             =       -1;
-        char                    key[PATH_MAX]   =       "";
-        char                    *temp           =       NULL;
-        xlator_t                *this           =       NULL;
-
-        this = THIS;
-        GF_ASSERT (this);
-        GF_ASSERT (op_errstr);
-        GF_ASSERT (rsp_dict);
-        GF_ASSERT (snap);
-        GF_ASSERT (keyprefix);
-
-        /* TODO : Get all the snap volume info present in snap object,
-         * as of now, There will be only one snapvolinfo per snap object
-         */
-        ret = snprintf (key, sizeof (key), "%s.snapname", keyprefix);
-        if (ret < 0) {
-                goto out;
-        }
-
-        temp = gf_strdup (snap->snapname);
-        if (temp == NULL) {
-                ret = -1;
-                goto out;
-        }
-        ret = dict_set_dynstr (rsp_dict, key, temp);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "Could not save "
-                        "snap name");
-                goto out;
-        }
+    int ret = -1;
+    xlator_t *this = NULL;
+    char key[64] = ""; /* keyprefix is "status.snap0" */
+    int keylen;
+    char brickkey[PATH_MAX] = "";
+    glusterd_volinfo_t *snap_volinfo = NULL;
+    glusterd_volinfo_t *tmp_volinfo = NULL;
+    glusterd_brickinfo_t *brickinfo = NULL;
+    int volcount = 0;
+    int brickcount = 0;
+
+    this = THIS;
+    GF_ASSERT(this);
+
+    GF_ASSERT(op_errstr);
+    GF_ASSERT(rsp_dict);
+    GF_ASSERT(keyprefix);
+    GF_ASSERT(snap);
+
+    cds_list_for_each_entry_safe(snap_volinfo, tmp_volinfo, &snap->volumes,
+                                 vol_list)
+    {
+        keylen = snprintf(key, sizeof(key), "%s.vol%d", keyprefix, volcount);
+        if (keylen < 0) {
+            gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_COPY_FAIL, NULL);
+            ret = -1;
+            goto out;
+        }
+        cds_list_for_each_entry(brickinfo, &snap_volinfo->bricks, brick_list)
+        {
+            if (!glusterd_is_local_brick(this, snap_volinfo, brickinfo)) {
+                brickcount++;
+                continue;
+            }
 
-        temp = NULL;
+            ret = glusterd_get_single_brick_status(
+                op_errstr, rsp_dict, key, brickcount, snap_volinfo, brickinfo);
 
-        ret = snprintf (key, sizeof (key), "%s.uuid", keyprefix);
-        if (ret < 0) {
+            if (ret) {
+                gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_SNAP_STATUS_FAIL,
+                       "Getting "
+                       "single snap status failed");
                 goto out;
+            }
+            brickcount++;
         }
-
-        temp = gf_strdup (uuid_utoa (snap->snap_id));
-        if (temp == NULL) {
-                ret = -1;
-                goto out;
+        keylen = snprintf(brickkey, sizeof(brickkey), "%s.brickcount", key);
+        if (keylen < 0) {
+            gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_COPY_FAIL, NULL);
+            goto out;
         }
 
-        ret = dict_set_dynstr (rsp_dict, key, temp);
+        ret = dict_set_int32n(rsp_dict, brickkey, keylen, brickcount);
         if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "Could not save "
-                        "snap UUID");
-                goto out;
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+                   "Could not save brick count");
+            goto out;
         }
+        volcount++;
+    }
 
-        temp = NULL;
+    keylen = snprintf(key, sizeof(key), "%s.volcount", keyprefix);
+    if (keylen < 0) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_COPY_FAIL, NULL);
+        ret = -1;
+        goto out;
+    }
 
-        ret = glusterd_get_single_snap_status (op_errstr, rsp_dict, keyprefix,
-                                               snap);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "Could not get single snap status");
-                goto out;
-        }
+    ret = dict_set_int32n(rsp_dict, key, keylen, volcount);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+               "Could not save volcount");
+        goto out;
+    }
 
-        ret = snprintf (key, sizeof (key), "%s.volcount", keyprefix);
-        if (ret < 0) {
-                goto out;
-        }
-
-        ret = dict_set_int32 (rsp_dict, key, 1);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "Could not save volcount");
-                goto out;
-        }
 out:
-        if (ret && temp)
-                GF_FREE (temp);
 
-        return ret;
+    return ret;
 }
 
-int
-glusterd_get_snap_status_of_volume (char **op_errstr, dict_t *rsp_dict,
-                                    char *volname, char *keyprefix) {
-        int                     ret              =       -1;
-        glusterd_volinfo_t      *snap_volinfo    =       NULL;
-        glusterd_volinfo_t      *temp_volinfo    =       NULL;
-        glusterd_volinfo_t      *volinfo         =       NULL;
-        char                    key[PATH_MAX]    =        "";
-        xlator_t                *this            =       NULL;
-        glusterd_conf_t         *priv            =       NULL;
-        int                     i                =        0;
-
-        this = THIS;
-        GF_ASSERT (this);
-        priv = this->private;
-        GF_ASSERT (priv);
-
-        GF_ASSERT (op_errstr);
-        GF_ASSERT (rsp_dict);
-        GF_ASSERT (volname);
-        GF_ASSERT (keyprefix);
-
-        ret = glusterd_volinfo_find (volname, &volinfo);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "Failed to get volinfo of "
-                        "volume %s", volname);
-                goto out;
-        }
-
-        cds_list_for_each_entry_safe (snap_volinfo, temp_volinfo,
-                                      &volinfo->snap_volumes, snapvol_list) {
-                ret = snprintf (key, sizeof (key),
-                                "status.snap%d.snapname", i);
-                if (ret < 0) {
-                        goto out;
-                }
-
-                ret = dict_set_dynstr_with_alloc (rsp_dict, key,
-                                    snap_volinfo->snapshot->snapname);
-                if (ret) {
-                        gf_log (this->name, GF_LOG_ERROR, "Could not save "
-                                "snap name");
-                        goto out;
-                }
-
-                i++;
-        }
-
-        ret = dict_set_int32 (rsp_dict, "status.snapcount", i);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "Failed to save snapcount");
-                ret = -1;
-                goto out;
-        }
+static int
+glusterd_get_each_snap_object_status(char **op_errstr, dict_t *rsp_dict,
+                                     glusterd_snap_t *snap,
+                                     const char *keyprefix)
+{
+    int ret = -1;
+    char key[32] = ""; /* keyprefix is "status.snap0" */
+    int keylen;
+    char *temp = NULL;
+    xlator_t *this = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+    GF_ASSERT(op_errstr);
+    GF_ASSERT(rsp_dict);
+    GF_ASSERT(snap);
+    GF_ASSERT(keyprefix);
+
+    /* TODO : Get all the snap volume info present in snap object,
+     * as of now, There will be only one snapvolinfo per snap object
+     */
+    keylen = snprintf(key, sizeof(key), "%s.snapname", keyprefix);
+    if (keylen < 0) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_COPY_FAIL, NULL);
+        ret = -1;
+        goto out;
+    }
+
+    temp = gf_strdup(snap->snapname);
+    if (temp == NULL) {
+        ret = -1;
+        goto out;
+    }
+    ret = dict_set_dynstrn(rsp_dict, key, keylen, temp);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+               "Could not save "
+               "snap name");
+        goto out;
+    }
+
+    temp = NULL;
+
+    keylen = snprintf(key, sizeof(key), "%s.uuid", keyprefix);
+    if (keylen < 0) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_COPY_FAIL, NULL);
+        ret = -1;
+        goto out;
+    }
+
+    temp = gf_strdup(uuid_utoa(snap->snap_id));
+    if (temp == NULL) {
+        ret = -1;
+        goto out;
+    }
+
+    ret = dict_set_dynstrn(rsp_dict, key, keylen, temp);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+               "Could not save "
+               "snap UUID");
+        goto out;
+    }
+
+    temp = NULL;
+
+    ret = glusterd_get_single_snap_status(op_errstr, rsp_dict, keyprefix, snap);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_SNAP_STATUS_FAIL,
+               "Could not get single snap status");
+        goto out;
+    }
+
+    keylen = snprintf(key, sizeof(key), "%s.volcount", keyprefix);
+    if (keylen < 0) {
+        ret = keylen;
+        goto out;
+    }
+
+    ret = dict_set_int32n(rsp_dict, key, keylen, 1);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+               "Could not save volcount");
+        goto out;
+    }
 out:
-        return ret;
+    if (ret && temp)
+        GF_FREE(temp);
+
+    return ret;
 }
 
 int
-glusterd_get_all_snapshot_status (dict_t *dict, char **op_errstr,
-                              dict_t *rsp_dict)
+glusterd_get_snap_status_of_volume(char **op_errstr, dict_t *rsp_dict,
+                                   char *volname, char *keyprefix)
 {
-        int32_t                 i               =       0;
-        int                     ret             =       -1;
-        char                    key[PATH_MAX]   =       "";
-        glusterd_conf_t         *priv           =       NULL;
-        glusterd_snap_t         *snap           =       NULL;
-        glusterd_snap_t         *tmp_snap       =       NULL;
-        xlator_t                *this           =       NULL;
-
-        this = THIS;
-        GF_ASSERT (this);
-        priv = this->private;
-        GF_ASSERT (priv);
-
-        GF_ASSERT (dict);
-        GF_ASSERT (op_errstr);
-
-        cds_list_for_each_entry_safe (snap, tmp_snap, &priv->snapshots,
-                                      snap_list) {
-                ret = snprintf (key, sizeof (key),
-                                "status.snap%d.snapname", i);
-                if (ret < 0) {
-                        goto out;
-                }
-
-                ret = dict_set_dynstr_with_alloc (rsp_dict, key,
-                                                  snap->snapname);
-                if (ret) {
-                        gf_log (this->name, GF_LOG_ERROR, "Could not save "
-                                "snap name");
-                        goto out;
-                }
-
-                i++;
+    int ret = -1;
+    glusterd_volinfo_t *snap_volinfo = NULL;
+    glusterd_volinfo_t *temp_volinfo = NULL;
+    glusterd_volinfo_t *volinfo = NULL;
+    char key[64] = "";
+    xlator_t *this = NULL;
+    glusterd_conf_t *priv = NULL;
+    int i = 0;
+
+    this = THIS;
+    GF_ASSERT(this);
+    priv = this->private;
+    GF_ASSERT(priv);
+
+    GF_ASSERT(op_errstr);
+    GF_ASSERT(rsp_dict);
+    GF_ASSERT(volname);
+    GF_ASSERT(keyprefix);
+
+    ret = glusterd_volinfo_find(volname, &volinfo);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, EINVAL, GD_MSG_VOL_NOT_FOUND,
+               "Failed to get volinfo of "
+               "volume %s",
+               volname);
+        goto out;
+    }
+
+    cds_list_for_each_entry_safe(snap_volinfo, temp_volinfo,
+                                 &volinfo->snap_volumes, snapvol_list)
+    {
+        ret = snprintf(key, sizeof(key), "status.snap%d.snapname", i);
+        if (ret < 0) {
+            gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_COPY_FAIL, NULL);
+            goto out;
         }
 
-        ret = dict_set_int32 (rsp_dict, "status.snapcount", i);
+        ret = dict_set_dynstr_with_alloc(rsp_dict, key,
+                                         snap_volinfo->snapshot->snapname);
         if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "Could not save snapcount");
-                goto out;
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+                   "Could not save "
+                   "snap name");
+            goto out;
         }
 
-        ret = 0;
-out :
-        return ret;
-}
+        i++;
+    }
 
+    ret = dict_set_int32n(rsp_dict, "status.snapcount",
+                          SLEN("status.snapcount"), i);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+               "Failed to save snapcount");
+        ret = -1;
+        goto out;
+    }
+out:
+    return ret;
+}
 
 int
-glusterd_snapshot_status_commit (dict_t *dict, char **op_errstr,
+glusterd_get_all_snapshot_status(dict_t *dict, char **op_errstr,
                                  dict_t *rsp_dict)
 {
-        xlator_t        *this             =       NULL;
-        int             ret               =       -1;
-        glusterd_conf_t *conf             =       NULL;
-        char            *get_buffer       =       NULL;
-        int32_t         cmd               =       -1;
-        char            *snapname         =       NULL;
-        glusterd_snap_t *snap             =       NULL;
-        char            *volname          =       NULL;
+    int32_t i = 0;
+    int ret = -1;
+    char key[64] = "";
+    glusterd_conf_t *priv = NULL;
+    glusterd_snap_t *snap = NULL;
+    glusterd_snap_t *tmp_snap = NULL;
+    xlator_t *this = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+    priv = this->private;
+    GF_ASSERT(priv);
+
+    GF_ASSERT(dict);
+    GF_ASSERT(op_errstr);
+
+    cds_list_for_each_entry_safe(snap, tmp_snap, &priv->snapshots, snap_list)
+    {
+        ret = snprintf(key, sizeof(key), "status.snap%d.snapname", i);
+        if (ret < 0) {
+            gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_COPY_FAIL, NULL);
+            goto out;
+        }
 
-        this = THIS;
+        ret = dict_set_dynstr_with_alloc(rsp_dict, key, snap->snapname);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+                   "Could not save "
+                   "snap name");
+            goto out;
+        }
 
-        GF_ASSERT (this);
-        GF_ASSERT (dict);
-        GF_ASSERT (op_errstr);
+        i++;
+    }
 
-        conf = this->private;
+    ret = dict_set_int32n(rsp_dict, "status.snapcount",
+                          SLEN("status.snapcount"), i);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+               "Could not save snapcount");
+        goto out;
+    }
 
-        GF_ASSERT (conf);
-        ret = dict_get_int32 (dict, "status-cmd", &cmd);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "Failed to get status cmd type");
-                goto out;
-        }
+    ret = 0;
+out:
+    return ret;
+}
 
-        ret = dict_set_int32 (rsp_dict, "status-cmd", cmd);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "Could not save status cmd in rsp dictionary");
-                goto out;
-        }
-        switch (cmd) {
-                case GF_SNAP_STATUS_TYPE_ALL:
-                {
-                        ret = glusterd_get_all_snapshot_status (dict, op_errstr,
-                                                           rsp_dict);
-                        if (ret) {
-                                gf_log (this->name, GF_LOG_ERROR, "Unable to "
-                                        "get snapshot status");
-                                goto out;
-                        }
-                        break;
-                }
-                case GF_SNAP_STATUS_TYPE_SNAP:
-                {
-
-                        ret = dict_get_str (dict, "snapname", &snapname);
-                        if (ret) {
-                                gf_log (this->name, GF_LOG_ERROR, "Unable to "
-                                        "get snap name");
-                                goto out;
-                        }
-
-                        snap = glusterd_find_snap_by_name (snapname);
-                        if (!snap) {
-                                ret = gf_asprintf (op_errstr, "Snapshot (%s) "
-                                                  "does not exist", snapname);
-                                if (ret < 0) {
-                                        goto out;
-                                }
-                                ret = -1;
-                                gf_log (this->name, GF_LOG_ERROR, "Unable to "
-                                        "get snap volinfo");
-                                goto out;
-                        }
-                        ret = glusterd_get_each_snap_object_status (op_errstr,
-                                          rsp_dict, snap, "status.snap0");
-                        if (ret) {
-                                gf_log (this->name, GF_LOG_ERROR, "Unable to "
-                                        "get status of snap %s", get_buffer);
-                                goto out;
-                        }
-
-                        ret = dict_set_int32 (rsp_dict, "status.snapcount", 1);
-                        if (ret) {
-                                gf_log (this->name, GF_LOG_ERROR, "Unable to "
-                                        "set snapcount to 1");
-                                goto out;
-                        }
-                        break;
-                }
-                case GF_SNAP_STATUS_TYPE_VOL:
-                {
-                        ret = dict_get_str (dict, "volname", &volname);
-                        if (ret) {
-                                gf_log (this->name, GF_LOG_ERROR, "Unable to"
-                                        " get volume name");
-                                goto out;
-                        }
-
-                        ret = glusterd_get_snap_status_of_volume (op_errstr,
-                                           rsp_dict, volname, "status.vol0");
-                        if (ret) {
-                                gf_log (this->name, GF_LOG_ERROR, "Function :"
-                                        " glusterd_get_snap_status_of_volume "
-                                        "failed");
-                                goto out;
-                        }
+int
+glusterd_snapshot_status_commit(dict_t *dict, char **op_errstr,
+                                dict_t *rsp_dict)
+{
+    xlator_t *this = NULL;
+    int ret = -1;
+    glusterd_conf_t *conf = NULL;
+    int32_t cmd = -1;
+    char *snapname = NULL;
+    glusterd_snap_t *snap = NULL;
+    char *volname = NULL;
+
+    this = THIS;
+
+    GF_ASSERT(this);
+    GF_ASSERT(dict);
+    GF_ASSERT(op_errstr);
+
+    conf = this->private;
+
+    GF_ASSERT(conf);
+    ret = dict_get_int32n(dict, "sub-cmd", SLEN("sub-cmd"), &cmd);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+               "Failed to get status cmd type");
+        goto out;
+    }
+
+    ret = dict_set_int32n(rsp_dict, "sub-cmd", SLEN("sub-cmd"), cmd);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+               "Could not save status cmd in rsp dictionary");
+        goto out;
+    }
+    switch (cmd) {
+        case GF_SNAP_STATUS_TYPE_ALL: {
+            ret = glusterd_get_all_snapshot_status(dict, op_errstr, rsp_dict);
+            if (ret) {
+                gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_SNAP_STATUS_FAIL,
+                       "Unable to "
+                       "get snapshot status");
+                goto out;
+            }
+            break;
+        }
+        case GF_SNAP_STATUS_TYPE_ITER:
+        case GF_SNAP_STATUS_TYPE_SNAP: {
+            ret = dict_get_strn(dict, "snapname", SLEN("snapname"), &snapname);
+            if (ret) {
+                gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+                       "Unable to "
+                       "get snap name");
+                goto out;
+            }
+
+            snap = glusterd_find_snap_by_name(snapname);
+            if (!snap) {
+                ret = gf_asprintf(op_errstr,
+                                  "Snapshot (%s) "
+                                  "does not exist",
+                                  snapname);
+                if (ret < 0) {
+                    goto out;
                 }
-        }
-        ret = 0;
+                ret = -1;
+                gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_VOLINFO_GET_FAIL,
+                       "Unable to "
+                       "get snap volinfo");
+                goto out;
+            }
+            ret = glusterd_get_each_snap_object_status(op_errstr, rsp_dict,
+                                                       snap, "status.snap0");
+            if (ret) {
+                gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_SNAP_STATUS_FAIL,
+                       "Unable to "
+                       "get status of snap");
+                goto out;
+            }
+
+            ret = dict_set_int32n(rsp_dict, "status.snapcount",
+                                  SLEN("status.snapcount"), 1);
+            if (ret) {
+                gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+                       "Unable to "
+                       "set snapcount to 1");
+                goto out;
+            }
+            break;
+        }
+        case GF_SNAP_STATUS_TYPE_VOL: {
+            ret = dict_get_strn(dict, "volname", SLEN("volname"), &volname);
+            if (ret) {
+                gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+                       "Unable to"
+                       " get volume name");
+                goto out;
+            }
+
+            ret = glusterd_get_snap_status_of_volume(op_errstr, rsp_dict,
+                                                     volname, "status.vol0");
+            if (ret) {
+                gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_SNAP_STATUS_FAIL,
+                       "Function :"
+                       " glusterd_get_snap_status_of_volume "
+                       "failed");
+                goto out;
+            }
+        }
+    }
+    ret = 0;
 out:
-        return ret;
+    return ret;
 }
 
 int32_t
-glusterd_handle_snap_limit (dict_t *dict, dict_t *rsp_dict)
+glusterd_handle_snap_limit(dict_t *dict, dict_t *rsp_dict)
 {
-        int32_t             ret                 = -1;
-        xlator_t           *this                = NULL;
-        glusterd_conf_t    *priv                = NULL;
-        uint64_t            effective_max_limit = 0;
-        int64_t             volcount            = 0;
-        int                 i                   = 0;
-        char               *volname             = NULL;
-        char                key[PATH_MAX]       = {0, };
-        glusterd_volinfo_t *volinfo             = NULL;
-        uint64_t            limit               = 0;
-        int64_t             count               = 0;
-        glusterd_snap_t    *snap                = NULL;
-        glusterd_volinfo_t *tmp_volinfo         = NULL;
-        glusterd_volinfo_t *other_volinfo       = NULL;
-        uint64_t            opt_max_hard        = GLUSTERD_SNAPS_MAX_HARD_LIMIT;
-        uint64_t            opt_max_soft        = GLUSTERD_SNAPS_DEF_SOFT_LIMIT_PERCENT;
-
-        this = THIS;
-        GF_ASSERT (this);
-        GF_ASSERT (dict);
-        GF_ASSERT (rsp_dict);
-
-        priv = this->private;
-        GF_ASSERT (priv);
-
-        ret = dict_get_int64 (dict, "volcount", &volcount);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "failed to get the volcount");
-                goto out;
+    int32_t ret = -1;
+    xlator_t *this = NULL;
+    glusterd_conf_t *priv = NULL;
+    uint64_t effective_max_limit = 0;
+    int64_t volcount = 0;
+    int i = 0;
+    char *volname = NULL;
+    char key[64] = "";
+    int keylen;
+    char msg[PATH_MAX] = "";
+    glusterd_volinfo_t *volinfo = NULL;
+    uint64_t limit = 0;
+    int64_t count = 0;
+    glusterd_snap_t *snap = NULL;
+    glusterd_volinfo_t *tmp_volinfo = NULL;
+    uint64_t opt_max_hard = GLUSTERD_SNAPS_MAX_HARD_LIMIT;
+    uint64_t opt_max_soft = GLUSTERD_SNAPS_DEF_SOFT_LIMIT_PERCENT;
+
+    this = THIS;
+    GF_ASSERT(this);
+    GF_ASSERT(dict);
+    GF_ASSERT(rsp_dict);
+
+    priv = this->private;
+    GF_ASSERT(priv);
+
+    ret = dict_get_int64(dict, "volcount", &volcount);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+               "failed to get the volcount");
+        goto out;
+    }
+
+    for (i = 1; i <= volcount; i++) {
+        keylen = snprintf(key, sizeof(key), "volname%d", i);
+        ret = dict_get_strn(dict, key, keylen, &volname);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+                   "failed to get the "
+                   "volname");
+            goto out;
+        }
+
+        ret = glusterd_volinfo_find(volname, &volinfo);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, EINVAL, GD_MSG_VOL_NOT_FOUND,
+                   "volinfo for %s "
+                   "not found",
+                   volname);
+            goto out;
         }
 
-        for (i = 1; i <= volcount; i++) {
-                snprintf (key, sizeof (key), "volname%d", i);
-                ret = dict_get_str (dict, key, &volname);
-                if (ret) {
-                        gf_log (this->name, GF_LOG_ERROR, "failed to get the "
-                                "volname");
-                        goto out;
-                }
+        /* config values snap-max-hard-limit and snap-max-soft-limit are
+         * optional and hence we are not erroring out if values are not
+         * present
+         */
+        gd_get_snap_conf_values_if_present(priv->opts, &opt_max_hard,
+                                           &opt_max_soft);
 
-                ret = glusterd_volinfo_find (volname, &volinfo);
-                if (ret) {
-                        gf_log (this->name, GF_LOG_ERROR, "volinfo for %s "
-                                "not found", volname);
-                        goto out;
-                }
+        /* The minimum of the 2 limits i.e system wide limit and
+           volume wide limit will be considered
+        */
+        if (volinfo->snap_max_hard_limit < opt_max_hard)
+            effective_max_limit = volinfo->snap_max_hard_limit;
+        else
+            effective_max_limit = opt_max_hard;
 
-                /* config values snap-max-hard-limit and snap-max-soft-limit are
-                 * optional and hence we are not erroring out if values are not
-                 * present
-                 */
-                gd_get_snap_conf_values_if_present (priv->opts, &opt_max_hard,
-                                                    &opt_max_soft);
+        limit = (opt_max_soft * effective_max_limit) / 100;
 
-                /* The minimum of the 2 limits i.e system wide limit and
-                   volume wide limit will be considered
-                */
-                if (volinfo->snap_max_hard_limit < opt_max_hard)
-                        effective_max_limit = volinfo->snap_max_hard_limit;
-                else
-                        effective_max_limit = opt_max_hard;
+        count = volinfo->snap_count - limit;
+        if (count <= 0)
+            goto out;
 
-                limit = (opt_max_soft * effective_max_limit)/100;
+        tmp_volinfo = cds_list_entry(volinfo->snap_volumes.next,
+                                     glusterd_volinfo_t, snapvol_list);
+        snap = tmp_volinfo->snapshot;
+        GF_ASSERT(snap);
 
-                count = volinfo->snap_count - limit;
-                if (count <= 0)
-                        goto out;
+        gf_msg(this->name, GF_LOG_WARNING, 0, GD_MSG_SOFT_LIMIT_REACHED,
+               "Soft-limit "
+               "(value = %" PRIu64
+               ") of volume %s is reached. "
+               "Deleting snapshot %s.",
+               limit, volinfo->volname, snap->snapname);
 
-                tmp_volinfo = cds_list_entry (volinfo->snap_volumes.next,
-                                              glusterd_volinfo_t, snapvol_list);
-                snap = tmp_volinfo->snapshot;
-                GF_ASSERT (snap);
-
-                LOCK (&snap->lock);
-                {
-                        snap->snap_status = GD_SNAP_STATUS_DECOMMISSION;
-                        ret = glusterd_store_snap (snap);
-                        if (ret) {
-                                gf_log (this->name, GF_LOG_ERROR, "could "
-                                        "not store snap object %s",
-                                        snap->snapname);
-                                goto unlock;
-                        }
-
-                        ret = glusterd_snap_remove (rsp_dict, snap,
-                                                    _gf_true, _gf_true);
-                        if (ret)
-                                gf_log (this->name, GF_LOG_WARNING,
-                                        "failed to remove snap %s",
-                                        snap->snapname);
-                }
-        unlock: UNLOCK (&snap->lock);
+        snprintf(msg, sizeof(msg),
+                 "snapshot_name=%s;"
+                 "snapshot_uuid=%s",
+                 snap->snapname, uuid_utoa(snap->snap_id));
+
+        LOCK(&snap->lock);
+        {
+            snap->snap_status = GD_SNAP_STATUS_DECOMMISSION;
+            ret = glusterd_store_snap(snap);
+            if (ret) {
+                gf_msg(this->name, GF_LOG_ERROR, 0,
+                       GD_MSG_SNAP_OBJECT_STORE_FAIL,
+                       "could "
+                       "not store snap object %s",
+                       snap->snapname);
+                goto unlock;
+            }
+
+            ret = glusterd_snap_remove(rsp_dict, snap, _gf_true, _gf_true,
+                                       _gf_false);
+            if (ret)
+                gf_msg(this->name, GF_LOG_WARNING, 0, GD_MSG_SNAP_REMOVE_FAIL,
+                       "failed to remove snap %s", snap->snapname);
+        }
+    unlock:
+        UNLOCK(&snap->lock);
+        if (is_origin_glusterd(dict) == _gf_true) {
+            if (ret)
+                gf_event(EVENT_SNAPSHOT_DELETE_FAILED, "%s", msg);
+            else
+                gf_event(EVENT_SNAPSHOT_DELETED, "%s", msg);
         }
+    }
 
 out:
-        return ret;
+    return ret;
 }
 
 int32_t
-glusterd_snapshot_clone_postvalidate (dict_t *dict, int32_t op_ret,
-                                       char **op_errstr, dict_t *rsp_dict)
+glusterd_snapshot_clone_postvalidate(dict_t *dict, int32_t op_ret,
+                                     char **op_errstr, dict_t *rsp_dict)
 {
-        xlator_t              *this           = NULL;
-        glusterd_conf_t       *priv           = NULL;
-        int                    ret            = -1;
-        int32_t                cleanup        = 0;
-        glusterd_snap_t       *snap           = NULL;
-        glusterd_volinfo_t    *snap_vol       = NULL;
-        char                  *clonename      = NULL;
-        char                  *auto_delete    = NULL;
-
-        this = THIS;
-
-        GF_ASSERT (this);
-        GF_ASSERT (dict);
-        GF_ASSERT (rsp_dict);
-
-        priv = this->private;
-        GF_ASSERT (priv);
-
-        if (op_ret) {
-                ret = dict_get_int32 (dict, "cleanup", &cleanup);
-                if (!ret && cleanup) {
-                        ret = glusterd_do_snap_cleanup (dict, op_errstr,
-                                                        rsp_dict);
-                        if (ret) {
-                                gf_log (this->name, GF_LOG_WARNING, "cleanup "
-                                        "operation failed");
-                                goto out;
-                        }
-                }
-                /* Irrespective of status of cleanup its better
-                 * to return from this function. As the functions
-                 * following this block is not required to be
-                 * executed in case of failure scenario.
-                 */
-                ret = 0;
-                goto out;
-        }
-
-        ret = dict_get_str (dict, "clonename", &clonename);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "Unable to fetch "
-                        "clonename");
-                goto out;
-        }
-
-        ret = glusterd_volinfo_find (clonename, &snap_vol);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "unable to find clone "
-                        "%s volinfo", clonename);
-                goto out;
-        }
-
-        ret = glusterd_snapshot_update_snaps_post_validate (dict,
-                                                            op_errstr,
-                                                            rsp_dict);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "Failed to "
-                        "create snapshot");
-                goto out;
-        }
+    xlator_t *this = NULL;
+    glusterd_conf_t *priv = NULL;
+    int ret = -1;
+    int32_t cleanup = 0;
+    glusterd_snap_t *snap = NULL;
+    glusterd_volinfo_t *snap_vol = NULL;
+    char *clonename = NULL;
+
+    this = THIS;
+
+    GF_ASSERT(this);
+    GF_ASSERT(dict);
+    GF_ASSERT(rsp_dict);
+
+    priv = this->private;
+    GF_ASSERT(priv);
+
+    ret = dict_get_strn(dict, "clonename", SLEN("clonename"), &clonename);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+               "Unable to fetch "
+               "clonename");
+        goto out;
+    }
+
+    ret = glusterd_volinfo_find(clonename, &snap_vol);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_VOL_NOT_FOUND,
+               "unable to find clone "
+               "%s volinfo",
+               clonename);
+        goto out;
+    }
+
+    if (snap_vol)
         snap = snap_vol->snapshot;
-        ret = glusterd_snapobject_delete (snap);
-        if (ret) {
-                gf_log (this->name, GF_LOG_WARNING, "Failed to delete "
-                        "snap object %s", snap->snapname);
-                goto out;
-        }
-        snap_vol->snapshot = NULL;
-
+    else {
+        ret = -1;
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_SNAP_NOT_FOUND,
+               "Snapshot volume is null");
+        goto out;
+    }
+
+    /* Fetch snap object from snap_vol and delete it all in case of *
+     * a failure, or else, just delete the snap object as it is not *
+     * needed in case of a clone                                    *
+     */
+    if (op_ret) {
+        ret = dict_get_int32n(dict, "cleanup", SLEN("cleanup"), &cleanup);
+        if (!ret && cleanup && snap) {
+            glusterd_snap_remove(rsp_dict, snap, _gf_true, _gf_true, _gf_true);
+        }
+        /* Irrespective of status of cleanup its better
+         * to return from this function. As the functions
+         * following this block is not required to be
+         * executed in case of failure scenario.
+         */
         ret = 0;
+        goto out;
+    }
+
+    ret = glusterd_snapobject_delete(snap);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_WARNING, 0, GD_MSG_SNAP_REMOVE_FAIL,
+               "Failed to delete "
+               "snap object %s",
+               snap->snapname);
+        goto out;
+    }
+    snap_vol->snapshot = NULL;
 
 out:
-        return ret;
+    return ret;
 }
 
-
 int32_t
-glusterd_snapshot_create_postvalidate (dict_t *dict, int32_t op_ret,
-                                       char **op_errstr, dict_t *rsp_dict)
+glusterd_snapshot_create_postvalidate(dict_t *dict, int32_t op_ret,
+                                      char **op_errstr, dict_t *rsp_dict)
 {
-        xlator_t        *this           = NULL;
-        glusterd_conf_t *priv           = NULL;
-        int              ret            = -1;
-        int32_t          cleanup        = 0;
-        glusterd_snap_t *snap           = NULL;
-        char            *snapname       = NULL;
-        char            *auto_delete    = NULL;
-
-        this = THIS;
-
-        GF_ASSERT (this);
-        GF_ASSERT (dict);
-        GF_ASSERT (rsp_dict);
-
-        priv = this->private;
-        GF_ASSERT (priv);
-
-        if (op_ret) {
-                ret = dict_get_int32 (dict, "cleanup", &cleanup);
-                if (!ret && cleanup) {
-                        ret = glusterd_do_snap_cleanup (dict, op_errstr,
-                                                        rsp_dict);
-                        if (ret) {
-                                gf_log (this->name, GF_LOG_WARNING, "cleanup "
-                                        "operation failed");
-                                goto out;
-                        }
-                }
-                /* Irrespective of status of cleanup its better
-                 * to return from this function. As the functions
-                 * following this block is not required to be
-                 * executed in case of failure scenario.
-                 */
-                ret = 0;
-                goto out;
-        }
+    xlator_t *this = NULL;
+    glusterd_conf_t *priv = NULL;
+    int ret = -1;
+    int32_t cleanup = 0;
+    glusterd_snap_t *snap = NULL;
+    char *snapname = NULL;
+    char *volname = NULL;
+    glusterd_volinfo_t *volinfo = NULL;
+    uint64_t opt_hard_max = GLUSTERD_SNAPS_MAX_HARD_LIMIT;
+    uint64_t opt_max_soft = GLUSTERD_SNAPS_DEF_SOFT_LIMIT_PERCENT;
+    int64_t effective_max_limit = 0;
+    int64_t soft_limit = 0;
+    int32_t snap_activate = _gf_false;
+
+    this = THIS;
+
+    GF_ASSERT(this);
+    GF_ASSERT(dict);
+    GF_ASSERT(rsp_dict);
+
+    priv = this->private;
+    GF_ASSERT(priv);
+
+    if (op_ret) {
+        ret = dict_get_int32n(dict, "cleanup", SLEN("cleanup"), &cleanup);
+        if (!ret && cleanup) {
+            ret = glusterd_do_snap_cleanup(dict, op_errstr, rsp_dict);
+            if (ret) {
+                gf_msg(this->name, GF_LOG_WARNING, 0, GD_MSG_SNAP_CLEANUP_FAIL,
+                       "cleanup "
+                       "operation failed");
+                goto out;
+            }
+        }
+        /* Irrespective of status of cleanup its better
+         * to return from this function. As the functions
+         * following this block is not required to be
+         * executed in case of failure scenario.
+         */
+        ret = 0;
+        goto out;
+    }
+
+    ret = dict_get_strn(dict, "snapname", SLEN("snapname"), &snapname);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+               "Unable to fetch "
+               "snapname");
+        goto out;
+    }
+
+    snap = glusterd_find_snap_by_name(snapname);
+    if (!snap) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_SNAP_NOT_FOUND,
+               "unable to find snap "
+               "%s",
+               snapname);
+        goto out;
+    }
+
+    snap->snap_status = GD_SNAP_STATUS_IN_USE;
+    ret = glusterd_store_snap(snap);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_WARNING, 0, GD_MSG_SNAP_OBJECT_STORE_FAIL,
+               "Could not store snap"
+               "object %s",
+               snap->snapname);
+        goto out;
+    }
+
+    ret = glusterd_snapshot_update_snaps_post_validate(dict, op_errstr,
+                                                       rsp_dict);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_SNAP_CREATION_FAIL,
+               "Failed to "
+               "create snapshot");
+        goto out;
+    }
 
-        ret = dict_get_str (dict, "snapname", &snapname);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "Unable to fetch "
-                        "snapname");
-                goto out;
-        }
+    /*
+     * If activate_on_create was enabled, and we have reached this  *
+     * section of the code, that means, that after successfully     *
+     * creating the snapshot, we have also successfully started the *
+     * snapshot bricks on all nodes. So from originator node we can *
+     * send EVENT_SNAPSHOT_ACTIVATED event.                         *
+     *                                                              *
+     * Also check, if hard limit and soft limit is reached in case  *
+     * of successfully creating the snapshot, and generate the event *
+     */
+    if (is_origin_glusterd(dict) == _gf_true) {
+        snap_activate = dict_get_str_boolean(
+            priv->opts, GLUSTERD_STORE_KEY_SNAP_ACTIVATE, _gf_false);
 
-        snap = glusterd_find_snap_by_name (snapname);
-        if (!snap) {
-                gf_log (this->name, GF_LOG_ERROR, "unable to find snap "
-                        "%s", snapname);
-                goto out;
+        if (snap_activate == _gf_true) {
+            gf_event(EVENT_SNAPSHOT_ACTIVATED,
+                     "snapshot_name=%s;"
+                     "snapshot_uuid=%s",
+                     snap->snapname, uuid_utoa(snap->snap_id));
         }
 
-        snap->snap_status = GD_SNAP_STATUS_IN_USE;
-        ret = glusterd_store_snap (snap);
+        ret = dict_get_strn(dict, "volname1", SLEN("volname1"), &volname);
         if (ret) {
-                gf_log (this->name, GF_LOG_WARNING, "Could not store snap"
-                        "object %s", snap->snapname);
-                goto out;
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+                   "Failed to get volname.");
+            goto out;
         }
 
-        ret = glusterd_snapshot_update_snaps_post_validate (dict,
-                                                            op_errstr,
-                                                            rsp_dict);
+        ret = glusterd_volinfo_find(volname, &volinfo);
         if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "Failed to "
-                        "create snapshot");
-                goto out;
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_VOL_NOT_FOUND,
+                   "Failed to get volinfo.");
+            goto out;
         }
 
-        /* "auto-delete" might not be set by user explicitly,
-         * in that case it's better to consider the default value.
-         * Hence not erroring out if Key is not found.
+        /* config values snap-max-hard-limit and snap-max-soft-limit are
+         * optional and hence we are not erroring out if values are not
+         * present
          */
-        ret = dict_get_str_boolean (priv->opts,
-                                    GLUSTERD_STORE_KEY_SNAP_AUTO_DELETE,
-                                    _gf_false);
-        if ( _gf_true == ret ) {
-                //ignore the errors of autodelete
-                ret = glusterd_handle_snap_limit (dict, rsp_dict);
-        }
-        ret = 0;
+        gd_get_snap_conf_values_if_present(priv->opts, &opt_hard_max,
+                                           &opt_max_soft);
+
+        if (volinfo->snap_max_hard_limit < opt_hard_max)
+            effective_max_limit = volinfo->snap_max_hard_limit;
+        else
+            effective_max_limit = opt_hard_max;
+
+        /*
+         * Check for hard limit. If it is reached after taking *
+         * this snapshot, then generate event for the same. If *
+         * it is not reached, then check for the soft limit,   *
+         * and generate event accordingly.                     *
+         */
+        if (volinfo->snap_count >= effective_max_limit) {
+            gf_event(EVENT_SNAPSHOT_HARD_LIMIT_REACHED,
+                     "volume_name=%s;volume_id=%s", volname,
+                     uuid_utoa(volinfo->volume_id));
+        } else {
+            soft_limit = (opt_max_soft * effective_max_limit) / 100;
+            if (volinfo->snap_count >= soft_limit) {
+                gf_event(EVENT_SNAPSHOT_SOFT_LIMIT_REACHED,
+                         "volume_name=%s;volume_id=%s", volname,
+                         uuid_utoa(volinfo->volume_id));
+            }
+        }
+    }
+
+    /* "auto-delete" might not be set by user explicitly,
+     * in that case it's better to consider the default value.
+     * Hence not erroring out if Key is not found.
+     */
+    ret = dict_get_str_boolean(priv->opts, GLUSTERD_STORE_KEY_SNAP_AUTO_DELETE,
+                               _gf_false);
+    if (_gf_true == ret) {
+        ret = glusterd_handle_snap_limit(dict, rsp_dict);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_WARNING, 0, GD_MSG_SNAP_REMOVE_FAIL,
+                   "failed to remove snap");
+            /* ignore the errors of autodelete */
+            ret = 0;
+        }
+    }
 
 out:
-        return ret;
+    return ret;
 }
 
 int32_t
-glusterd_snapshot (dict_t *dict, char **op_errstr, dict_t *rsp_dict)
+glusterd_snapshot(dict_t *dict, char **op_errstr, uint32_t *op_errno,
+                  dict_t *rsp_dict)
 {
+    xlator_t *this = NULL;
+    glusterd_conf_t *priv = NULL;
+    int32_t snap_command = 0;
+    char *snap_name = NULL;
+    char temp[PATH_MAX] = "";
+    int ret = -1;
+
+    this = THIS;
+
+    GF_ASSERT(this);
+    GF_ASSERT(dict);
+    GF_ASSERT(rsp_dict);
+    GF_VALIDATE_OR_GOTO(this->name, op_errno, out);
+
+    priv = this->private;
+    GF_ASSERT(priv);
+
+    ret = dict_get_int32n(dict, "type", SLEN("type"), &snap_command);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_COMMAND_NOT_FOUND,
+               "unable to get the type of "
+               "the snapshot command");
+        goto out;
+    }
+
+    switch (snap_command) {
+        case (GF_SNAP_OPTION_TYPE_CREATE):
+            ret = glusterd_snapshot_create_commit(dict, op_errstr, op_errno,
+                                                  rsp_dict);
+            if (ret) {
+                gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_SNAP_CREATION_FAIL,
+                       "Failed to "
+                       "create snapshot");
+                goto out;
+            }
+            break;
 
-        xlator_t        *this           = NULL;
-        glusterd_conf_t *priv           = NULL;
-        int32_t          snap_command   = 0;
-        char            *snap_name      = NULL;
-        char             temp[PATH_MAX] = "";
-        int              ret            = -1;
-
-        this = THIS;
-
-        GF_ASSERT (this);
-        GF_ASSERT (dict);
-        GF_ASSERT (rsp_dict);
-
-        priv = this->private;
-        GF_ASSERT (priv);
-
-        ret = dict_get_int32 (dict, "type", &snap_command);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "unable to get the type of "
-                        "the snapshot command");
+        case (GF_SNAP_OPTION_TYPE_CLONE):
+            ret = glusterd_snapshot_clone_commit(dict, op_errstr, rsp_dict);
+            if (ret) {
+                gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_SNAP_CLONE_FAILED,
+                       "Failed to "
+                       "clone snapshot");
                 goto out;
-        }
+            }
+            break;
 
-        switch (snap_command) {
+        case GF_SNAP_OPTION_TYPE_CONFIG:
+            ret = glusterd_snapshot_config_commit(dict, op_errstr, rsp_dict);
+            if (ret) {
+                gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_SNAP_CONFIG_FAIL,
+                       "snapshot config failed");
+                goto out;
+            }
+            break;
 
-        case (GF_SNAP_OPTION_TYPE_CREATE):
-                ret = glusterd_snapshot_create_commit (dict, op_errstr,
-                                                       rsp_dict);
-                if (ret) {
-                        gf_log (this->name, GF_LOG_ERROR, "Failed to "
-                                "create snapshot");
-                        goto out;
+        case GF_SNAP_OPTION_TYPE_DELETE:
+            ret = glusterd_snapshot_remove_commit(dict, op_errstr, rsp_dict);
+            if (ret) {
+                gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_SNAP_REMOVE_FAIL,
+                       "Failed to "
+                       "delete snapshot");
+                if (*op_errstr) {
+                    /* If error string is already set
+                     * then goto out */
+                    goto out;
                 }
-                break;
 
-       case (GF_SNAP_OPTION_TYPE_CLONE):
-                ret = glusterd_snapshot_clone_commit (dict, op_errstr,
-                                                       rsp_dict);
+                ret = dict_get_strn(dict, "snapname", SLEN("snapname"),
+                                    &snap_name);
                 if (ret) {
-                        gf_log (this->name, GF_LOG_ERROR, "Failed to "
-                                "clone snapshot");
-                        goto out;
+                    gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+                           "Failed to get snapname");
+                    snap_name = "NA";
                 }
-                break;
 
-        case GF_SNAP_OPTION_TYPE_CONFIG:
-                ret = glusterd_snapshot_config_commit (dict, op_errstr,
-                                                       rsp_dict);
-                if (ret) {
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "snapshot config failed");
-                        goto out;
-                }
-                break;
+                snprintf(temp, sizeof(temp),
+                         "Snapshot %s might "
+                         "not be in an usable state.",
+                         snap_name);
 
-        case GF_SNAP_OPTION_TYPE_DELETE:
-                ret = glusterd_snapshot_remove_commit (dict, op_errstr,
-                                                       rsp_dict);
-                if (ret) {
-                        gf_log (this->name, GF_LOG_ERROR, "Failed to "
-                                "delete snapshot");
-                        if (*op_errstr) {
-                                /* If error string is already set
-                                 * then goto out */
-                                goto out;
-                        }
-
-                        ret = dict_get_str (dict, "snapname", &snap_name);
-                        if (ret) {
-                                gf_log (this->name, GF_LOG_ERROR,
-                                        "Failed to get snapname");
-                                snap_name = "NA";
-                        }
-
-                        snprintf (temp, sizeof (temp), "Snapshot %s might "
-                                  "not be in an usable state.", snap_name);
-
-                        *op_errstr = gf_strdup (temp);
-                        ret = -1;
-                        goto out;
-                }
-                break;
+                *op_errstr = gf_strdup(temp);
+                ret = -1;
+                goto out;
+            }
+            break;
 
         case GF_SNAP_OPTION_TYPE_RESTORE:
-                ret = glusterd_snapshot_restore (dict, op_errstr,
-                                                 rsp_dict);
-                if (ret) {
-                        gf_log (this->name, GF_LOG_WARNING, "Failed to "
-                                "restore snapshot");
-                        goto out;
-                }
+            ret = glusterd_snapshot_restore(dict, op_errstr, rsp_dict);
+            if (ret) {
+                gf_msg(this->name, GF_LOG_WARNING, 0, GD_MSG_SNAP_RESTORE_FAIL,
+                       "Failed to "
+                       "restore snapshot");
+                goto out;
+            }
 
-                break;
-        case  GF_SNAP_OPTION_TYPE_ACTIVATE:
-                ret = glusterd_snapshot_activate_commit (dict, op_errstr,
-                                                 rsp_dict);
-                if (ret) {
-                        gf_log (this->name, GF_LOG_WARNING, "Failed to "
-                                "activate snapshot");
-                        goto out;
-                }
+            break;
+        case GF_SNAP_OPTION_TYPE_ACTIVATE:
+            ret = glusterd_snapshot_activate_commit(dict, op_errstr, rsp_dict);
+            if (ret) {
+                gf_msg(this->name, GF_LOG_WARNING, 0, GD_MSG_SNAP_ACTIVATE_FAIL,
+                       "Failed to "
+                       "activate snapshot");
+                goto out;
+            }
 
-                break;
+            break;
 
         case GF_SNAP_OPTION_TYPE_DEACTIVATE:
-                ret = glusterd_snapshot_deactivate_commit (dict, op_errstr,
-                                                 rsp_dict);
-                if (ret) {
-                        gf_log (this->name, GF_LOG_WARNING, "Failed to "
-                                "deactivate snapshot");
-                        goto out;
-                }
+            ret = glusterd_snapshot_deactivate_commit(dict, op_errstr,
+                                                      rsp_dict);
+            if (ret) {
+                gf_msg(this->name, GF_LOG_WARNING, 0,
+                       GD_MSG_SNAP_DEACTIVATE_FAIL,
+                       "Failed to "
+                       "deactivate snapshot");
+                goto out;
+            }
 
-                break;
+            break;
 
         case GF_SNAP_OPTION_TYPE_STATUS:
-                ret = glusterd_snapshot_status_commit (dict, op_errstr,
-                                                       rsp_dict);
-                if (ret) {
-                        gf_log (this->name, GF_LOG_ERROR, "Failed to "
-                                "show snapshot status");
-                        goto out;
-                }
-                break;
-
+            ret = glusterd_snapshot_status_commit(dict, op_errstr, rsp_dict);
+            if (ret) {
+                gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_SNAP_STATUS_FAIL,
+                       "Failed to "
+                       "show snapshot status");
+                goto out;
+            }
+            break;
 
         default:
-                gf_log (this->name, GF_LOG_WARNING, "invalid snap command");
-                goto out;
-                break;
-        }
+            gf_msg(this->name, GF_LOG_WARNING, EINVAL, GD_MSG_INVALID_ENTRY,
+                   "invalid snap command");
+            goto out;
+            break;
+    }
 
-        ret = 0;
+    ret = 0;
 
 out:
-        return ret;
+    return ret;
 }
 
 int
-glusterd_snapshot_brickop (dict_t *dict, char **op_errstr, dict_t *rsp_dict)
+glusterd_snapshot_brickop(dict_t *dict, char **op_errstr, dict_t *rsp_dict)
 {
-        int            ret       = -1;
-        int64_t        vol_count = 0;
-        int64_t        count     = 1;
-        char           key[1024] = {0,};
-        char           *volname  = NULL;
-        int32_t        snap_command = 0;
-        xlator_t       *this     = NULL;
-        char           *op_type  = NULL;
-
-        this = THIS;
-
-        GF_ASSERT (this);
-        GF_ASSERT (dict);
-        GF_ASSERT (rsp_dict);
-
-        ret = dict_get_int32 (dict, "type", &snap_command);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "unable to get the type of "
-                        "the snapshot command");
-                goto out;
-        }
-
-        switch (snap_command) {
+    int ret = -1;
+    int64_t vol_count = 0;
+    int64_t count = 1;
+    char key[64] = "";
+    int keylen;
+    char *volname = NULL;
+    int32_t snap_command = 0;
+    xlator_t *this = NULL;
+    char *op_type = NULL;
+
+    this = THIS;
+
+    GF_ASSERT(this);
+    GF_ASSERT(dict);
+    GF_ASSERT(rsp_dict);
+
+    ret = dict_get_int32n(dict, "type", SLEN("type"), &snap_command);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_COMMAND_NOT_FOUND,
+               "unable to get the type of "
+               "the snapshot command");
+        goto out;
+    }
+
+    switch (snap_command) {
         case GF_SNAP_OPTION_TYPE_CREATE:
 
-                /* op_type with tell us whether its pre-commit operation
-                 * or post-commit
+            /* op_type with tell us whether its pre-commit operation
+             * or post-commit
+             */
+            ret = dict_get_strn(dict, "operation-type", SLEN("operation-type"),
+                                &op_type);
+            if (ret) {
+                gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+                       "Failed to fetch "
+                       "operation type");
+                goto out;
+            }
+
+            if (strcmp(op_type, "pre") == 0) {
+                /* BRICK OP PHASE for enabling barrier, Enable barrier
+                 * if its a pre-commit operation
                  */
-                ret = dict_get_str (dict, "operation-type", &op_type);
+                ret = glusterd_set_barrier_value(dict, "enable");
                 if (ret) {
-                        gf_log (this->name, GF_LOG_ERROR, "Failed to fetch "
-                                "operation type");
-                        goto out;
+                    gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+                           "Failed to "
+                           "set barrier value as enable in dict");
+                    goto out;
+                }
+            } else if (strcmp(op_type, "post") == 0) {
+                /* BRICK OP PHASE for disabling barrier, Disable barrier
+                 * if its a post-commit operation
+                 */
+                ret = glusterd_set_barrier_value(dict, "disable");
+                if (ret) {
+                    gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+                           "Failed to "
+                           "set barrier value as disable in "
+                           "dict");
+                    goto out;
                 }
+            } else {
+                ret = -1;
+                gf_msg(this->name, GF_LOG_ERROR, EINVAL, GD_MSG_INVALID_ENTRY,
+                       "Invalid op_type");
+                goto out;
+            }
 
-                if (strcmp (op_type, "pre") == 0) {
-                        /* BRICK OP PHASE for enabling barrier, Enable barrier
-                         * if its a pre-commit operation
-                         */
-                        ret = glusterd_set_barrier_value (dict, "enable");
-                        if (ret) {
-                                gf_log (this->name, GF_LOG_ERROR, "Failed to "
-                                        "set barrier value as enable in dict");
-                                goto out;
-                        }
-                } else if (strcmp (op_type, "post") == 0) {
-                        /* BRICK OP PHASE for disabling barrier, Disable barrier
-                         * if its a post-commit operation
-                         */
-                        ret = glusterd_set_barrier_value (dict, "disable");
-                        if (ret) {
-                                gf_log (this->name, GF_LOG_ERROR, "Failed to "
-                                        "set barrier value as disable in "
-                                        "dict");
-                                goto out;
-                        }
-                } else {
-                        ret = -1;
-                        gf_log (this->name, GF_LOG_ERROR, "Invalid op_type");
-                        goto out;
+            ret = dict_get_int64(dict, "volcount", &vol_count);
+            if (ret)
+                goto out;
+            while (count <= vol_count) {
+                keylen = snprintf(key, sizeof(key), "volname%" PRId64, count);
+                ret = dict_get_strn(dict, key, keylen, &volname);
+                if (ret) {
+                    gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+                           "Unable to get volname");
+                    goto out;
                 }
+                ret = dict_set_strn(dict, "volname", SLEN("volname"), volname);
+                if (ret)
+                    goto out;
 
-                ret = dict_get_int64 (dict, "volcount", &vol_count);
+                ret = gd_brick_op_phase(GD_OP_SNAP, NULL, dict, op_errstr);
                 if (ret)
-                        goto out;
-                while (count <= vol_count) {
-                        snprintf (key, 1024, "volname%"PRId64, count);
-                        ret = dict_get_str (dict, key, &volname);
-                        if (ret) {
-                                gf_log (this->name, GF_LOG_ERROR,
-                                        "Unable to get volname");
-                                goto out;
-                        }
-                        ret = dict_set_str (dict, "volname", volname);
-                        if (ret)
-                                goto out;
-
-                        ret = gd_brick_op_phase (GD_OP_SNAP, NULL, dict,
-                                        op_errstr);
-                        if (ret)
-                                goto out;
-                        volname = NULL;
-                        count++;
-                }
+                    goto out;
+                volname = NULL;
+                count++;
+            }
 
-                dict_del (dict, "volname");
-                ret = 0;
-                break;
+            dict_deln(dict, "volname", SLEN("volname"));
+            ret = 0;
+            break;
         case GF_SNAP_OPTION_TYPE_DELETE:
-                break;
+            break;
         default:
-                break;
-        }
+            break;
+    }
 
 out:
-        return ret;
+    return ret;
 }
 
 int
-glusterd_snapshot_prevalidate (dict_t *dict, char **op_errstr,
-                               dict_t *rsp_dict)
+glusterd_snapshot_prevalidate(dict_t *dict, char **op_errstr, dict_t *rsp_dict,
+                              uint32_t *op_errno)
 {
-        int                snap_command          = 0;
-        xlator_t           *this                 = NULL;
-        int                ret                   = -1;
-
-        this = THIS;
-
-        GF_ASSERT (this);
-        GF_ASSERT (dict);
-        GF_ASSERT (rsp_dict);
-
-        ret = dict_get_int32 (dict, "type", &snap_command);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "unable to get the type of "
-                        "the snapshot command");
-                goto out;
-        }
-
-        switch (snap_command) {
+    int snap_command = 0;
+    xlator_t *this = NULL;
+    int ret = -1;
+
+    this = THIS;
+
+    GF_ASSERT(this);
+    GF_ASSERT(dict);
+    GF_ASSERT(rsp_dict);
+    GF_VALIDATE_OR_GOTO(this->name, op_errno, out);
+
+    ret = dict_get_int32n(dict, "type", SLEN("type"), &snap_command);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_COMMAND_NOT_FOUND,
+               "unable to get the type of "
+               "the snapshot command");
+        goto out;
+    }
+
+    switch (snap_command) {
         case (GF_SNAP_OPTION_TYPE_CREATE):
-                ret = glusterd_snapshot_create_prevalidate (dict, op_errstr,
-                                                            rsp_dict);
-                if (ret) {
-                        gf_log (this->name, GF_LOG_WARNING, "Snapshot create "
-                                "pre-validation failed");
-                        goto out;
-                }
-                break;
+            ret = glusterd_snapshot_create_prevalidate(dict, op_errstr,
+                                                       rsp_dict, op_errno);
+            if (ret) {
+                gf_msg(this->name, GF_LOG_WARNING, 0, GD_MSG_SNAP_CREATION_FAIL,
+                       "Snapshot create "
+                       "pre-validation failed");
+                goto out;
+            }
+            break;
 
         case (GF_SNAP_OPTION_TYPE_CLONE):
-                ret = glusterd_snapshot_clone_prevalidate (dict, op_errstr,
-                                                            rsp_dict);
-                if (ret) {
-                        gf_log (this->name, GF_LOG_WARNING, "Snapshot clone "
-                                "pre-validation failed");
-                        goto out;
-                }
-                break;
-
+            ret = glusterd_snapshot_clone_prevalidate(dict, op_errstr, rsp_dict,
+                                                      op_errno);
+            if (ret) {
+                gf_msg(this->name, GF_LOG_WARNING, 0,
+                       GD_MSG_SNAP_CLONE_PREVAL_FAILED,
+                       "Snapshot clone "
+                       "pre-validation failed");
+                goto out;
+            }
+            break;
 
         case (GF_SNAP_OPTION_TYPE_CONFIG):
-                ret = glusterd_snapshot_config_prevalidate (dict, op_errstr);
-                if (ret) {
-                        gf_log (this->name, GF_LOG_WARNING, "Snapshot config "
-                                "pre-validation failed");
-                        goto out;
-                }
-                break;
+            ret = glusterd_snapshot_config_prevalidate(dict, op_errstr,
+                                                       op_errno);
+            if (ret) {
+                gf_msg(this->name, GF_LOG_WARNING, 0, GD_MSG_SNAP_CONFIG_FAIL,
+                       "Snapshot config "
+                       "pre-validation failed");
+                goto out;
+            }
+            break;
 
         case GF_SNAP_OPTION_TYPE_RESTORE:
-                ret = glusterd_snapshot_restore_prevalidate (dict, op_errstr,
-                                                             rsp_dict);
-                if (ret) {
-                        gf_log (this->name, GF_LOG_WARNING, "Snapshot restore "
-                                        "validation failed");
-                        goto out;
-                }
-                break;
+            ret = glusterd_snapshot_restore_prevalidate(dict, op_errstr,
+                                                        op_errno, rsp_dict);
+            if (ret) {
+                gf_msg(this->name, GF_LOG_WARNING, 0, GD_MSG_SNAP_RESTORE_FAIL,
+                       "Snapshot restore "
+                       "validation failed");
+                goto out;
+            }
+            break;
 
-        case  GF_SNAP_OPTION_TYPE_ACTIVATE:
-                ret = glusterd_snapshot_activate_deactivate_prevalidate (dict,
-                                                op_errstr, rsp_dict, _gf_true);
-                if (ret) {
-                        gf_log (this->name, GF_LOG_WARNING, "Snapshot activate "
-                                        "validation failed");
-                        goto out;
-                }
-                break;
+        case GF_SNAP_OPTION_TYPE_ACTIVATE:
+            ret = glusterd_snapshot_activate_deactivate_prevalidate(
+                dict, op_errstr, op_errno, rsp_dict, _gf_true);
+            if (ret) {
+                gf_msg(this->name, GF_LOG_WARNING, 0, GD_MSG_SNAP_ACTIVATE_FAIL,
+                       "Snapshot activate "
+                       "validation failed");
+                goto out;
+            }
+            break;
         case GF_SNAP_OPTION_TYPE_DEACTIVATE:
-                ret = glusterd_snapshot_activate_deactivate_prevalidate (dict,
-                                                op_errstr, rsp_dict, _gf_false);
-                if (ret) {
-                        gf_log (this->name, GF_LOG_WARNING,
-                        "Snapshot deactivate validation failed");
-                        goto out;
-                }
-                break;
+            ret = glusterd_snapshot_activate_deactivate_prevalidate(
+                dict, op_errstr, op_errno, rsp_dict, _gf_false);
+            if (ret) {
+                gf_msg(this->name, GF_LOG_WARNING, 0,
+                       GD_MSG_SNAP_DEACTIVATE_FAIL,
+                       "Snapshot deactivate validation failed");
+                goto out;
+            }
+            break;
         case GF_SNAP_OPTION_TYPE_DELETE:
-                ret = glusterd_snapshot_remove_prevalidate (dict, op_errstr,
-                                                            rsp_dict);
-                if (ret) {
-                        gf_log (this->name, GF_LOG_WARNING, "Snapshot remove "
-                                "validation failed");
-                        goto out;
-                }
-                break;
+            ret = glusterd_snapshot_remove_prevalidate(dict, op_errstr,
+                                                       op_errno, rsp_dict);
+            if (ret) {
+                gf_msg(this->name, GF_LOG_WARNING, 0, GD_MSG_SNAP_REMOVE_FAIL,
+                       "Snapshot remove "
+                       "validation failed");
+                goto out;
+            }
+            break;
 
         case GF_SNAP_OPTION_TYPE_STATUS:
-                ret = glusterd_snapshot_status_prevalidate (dict, op_errstr,
-                                                            rsp_dict);
-                if (ret) {
-                        gf_log (this->name, GF_LOG_WARNING, "Snapshot status "
-                                "validation failed");
-                        goto out;
-                }
-                break;
+            ret = glusterd_snapshot_status_prevalidate(dict, op_errstr,
+                                                       op_errno, rsp_dict);
+            if (ret) {
+                gf_msg(this->name, GF_LOG_WARNING, 0, GD_MSG_SNAP_STATUS_FAIL,
+                       "Snapshot status "
+                       "validation failed");
+                goto out;
+            }
+            break;
 
         default:
-                gf_log (this->name, GF_LOG_WARNING, "invalid snap command");
-                goto out;
-        }
+            gf_msg(this->name, GF_LOG_WARNING, EINVAL, GD_MSG_COMMAND_NOT_FOUND,
+                   "invalid snap command");
+            *op_errno = EINVAL;
+            goto out;
+    }
 
-        ret = 0;
+    ret = 0;
 out:
-        return ret;
+    return ret;
+}
+
+/* This function is called to remove the trashpath, in cases
+ * when the restore operation is successful and we don't need
+ * the backup, and incases when the restore op is failed before
+ * commit, and we don't need to revert the backup.
+ *
+ * @param volname  name of the volume which is being restored
+ *
+ * @return 0 on success or -1 on failure
+ */
+int
+glusterd_remove_trashpath(char *volname)
+{
+    int ret = -1;
+    char delete_path[PATH_MAX] = {
+        0,
+    };
+    xlator_t *this = NULL;
+    glusterd_conf_t *priv = NULL;
+    struct stat stbuf = {
+        0,
+    };
+    int32_t len = 0;
+
+    this = THIS;
+    GF_ASSERT(this);
+    priv = this->private;
+
+    GF_ASSERT(volname);
+
+    len = snprintf(delete_path, sizeof(delete_path),
+                   "%s/" GLUSTERD_TRASH "/vols-%s.deleted", priv->workdir,
+                   volname);
+    if ((len < 0) || (len >= sizeof(delete_path))) {
+        goto out;
+    }
+
+    ret = sys_lstat(delete_path, &stbuf);
+    if (ret) {
+        /* If the trash dir does not exist, return *
+         * without failure                         *
+         */
+        if (errno == ENOENT) {
+            ret = 0;
+            goto out;
+        } else {
+            gf_msg(this->name, GF_LOG_ERROR, errno, GD_MSG_DIR_OP_FAILED,
+                   "Failed to lstat "
+                   "backup dir (%s)",
+                   delete_path);
+            goto out;
+        }
+    }
+
+    /* Delete the backup copy of volume folder */
+    ret = recursive_rmdir(delete_path);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, errno, GD_MSG_DIR_OP_FAILED,
+               "Failed to remove "
+               "backup dir (%s)",
+               delete_path);
+        goto out;
+    }
+
+    ret = 0;
+out:
+    return ret;
 }
 
 /* This function is called if snapshot restore operation
@@ -7792,46 +8799,41 @@ out:
  * @return 0 on success or -1 on failure
  */
 int
-glusterd_snapshot_restore_cleanup (dict_t *rsp_dict,
-                                   char *volname,
-                                   glusterd_snap_t *snap)
+glusterd_snapshot_restore_cleanup(dict_t *rsp_dict, char *volname,
+                                  glusterd_snap_t *snap)
 {
-        int                     ret                     = -1;
-        char                    delete_path[PATH_MAX]   = {0,};
-        xlator_t               *this                    = NULL;
-        glusterd_conf_t        *priv                    = NULL;
-
-        this = THIS;
-        GF_ASSERT (this);
-        priv = this->private;
-
-        GF_ASSERT (rsp_dict);
-        GF_ASSERT (volname);
-        GF_ASSERT (snap);
-
-        snprintf (delete_path, sizeof (delete_path),
-                  "%s/"GLUSTERD_TRASH"/vols-%s.deleted", priv->workdir,
-                  volname);
-
-        /* Now delete the snap entry. */
-        ret = glusterd_snap_remove (rsp_dict, snap, _gf_false, _gf_true);
-        if (ret) {
-                gf_log (this->name, GF_LOG_WARNING, "Failed to delete "
-                        "snap %s", snap->snapname);
-                goto out;
-        }
-
-        /* Delete the backup copy of volume folder */
-        ret = recursive_rmdir (delete_path);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "Failed to remove "
-                        "backup dir (%s)", delete_path);
-                goto out;
-        }
-
-        ret = 0;
+    int ret = -1;
+    xlator_t *this = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+
+    GF_ASSERT(rsp_dict);
+    GF_ASSERT(volname);
+    GF_ASSERT(snap);
+
+    /* Now delete the snap entry. */
+    ret = glusterd_snap_remove(rsp_dict, snap, _gf_false, _gf_true, _gf_false);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_WARNING, 0, GD_MSG_SNAP_REMOVE_FAIL,
+               "Failed to delete "
+               "snap %s",
+               snap->snapname);
+        goto out;
+    }
+
+    /* Delete the backup copy of volume folder */
+    ret = glusterd_remove_trashpath(volname);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, errno, GD_MSG_DIR_OP_FAILED,
+               "Failed to remove "
+               "backup dir");
+        goto out;
+    }
+
+    ret = 0;
 out:
-        return ret;
+    return ret;
 }
 
 /* This function is called when the snapshot restore operation failed
@@ -7842,70 +8844,108 @@ out:
  * @return 0 on success and -1 on failure
  */
 int
-glusterd_snapshot_revert_partial_restored_vol (glusterd_volinfo_t *volinfo)
+glusterd_snapshot_revert_partial_restored_vol(glusterd_volinfo_t *volinfo)
 {
-        int                     ret                     = 0;
-        char                    pathname [PATH_MAX]     = {0,};
-        char                    trash_path[PATH_MAX]    = {0,};
-        glusterd_volinfo_t     *reverted_vol            = NULL;
-        glusterd_volinfo_t     *snap_vol                = NULL;
-        glusterd_volinfo_t     *tmp_vol                 = NULL;
-        glusterd_conf_t        *priv                    = NULL;
-        xlator_t               *this                    = NULL;
-
-        this = THIS;
-        GF_ASSERT (this);
-        priv = this->private;
-        GF_ASSERT (priv);
-        GF_ASSERT (volinfo);
-
-        GLUSTERD_GET_VOLUME_DIR (pathname, volinfo, priv);
-
-        snprintf (trash_path, sizeof (trash_path),
-                  "%s/"GLUSTERD_TRASH"/vols-%s.deleted", priv->workdir,
-                  volinfo->volname);
-
-        /* Since snapshot restore failed we cannot rely on the volume
-         * data stored under vols folder. Therefore delete the origin
-         * volume's backend folder.*/
-        ret = recursive_rmdir (pathname);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "Failed to remove "
-                        "%s directory", pathname);
-                goto out;
-        }
-
-        /* Now move the backup copy of the vols to its original
-         * location.*/
-        ret = rename (trash_path, pathname);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "Failed to rename folder "
-                        "from %s to %s", trash_path, pathname);
-                goto out;
-        }
-
-        /* Retrieve the volume from the store */
-        reverted_vol = glusterd_store_retrieve_volume (volinfo->volname, NULL);
-        if (NULL == reverted_vol) {
-                gf_log (this->name, GF_LOG_ERROR, "Failed to load restored "
-                        "%s volume", volinfo->volname);
-                goto out;
-        }
-
-        /* Retrieve the snap_volumes list from the older volinfo */
-        reverted_vol->snap_count = volinfo->snap_count;
-        cds_list_for_each_entry_safe (snap_vol, tmp_vol, &volinfo->snap_volumes,
-                                      snapvol_list) {
-                cds_list_add_tail (&snap_vol->snapvol_list,
-                                   &reverted_vol->snap_volumes);
-        }
-
-        /* Since we retrieved the volinfo from store now we don't
-         * want the older volinfo. Therefore delete the older volinfo */
-        glusterd_volinfo_unref (volinfo);
-        ret = 0;
+    int ret = 0;
+    char pathname[PATH_MAX] = "";
+    char trash_path[PATH_MAX] = "";
+    glusterd_brickinfo_t *brickinfo = NULL;
+    glusterd_volinfo_t *reverted_vol = NULL;
+    glusterd_volinfo_t *snap_vol = NULL;
+    glusterd_volinfo_t *tmp_vol = NULL;
+    glusterd_conf_t *priv = NULL;
+    xlator_t *this = NULL;
+    int32_t len = 0;
+
+    this = THIS;
+    GF_ASSERT(this);
+    priv = this->private;
+    GF_ASSERT(priv);
+    GF_ASSERT(volinfo);
+
+    GLUSTERD_GET_VOLUME_DIR(pathname, volinfo, priv);
+
+    len = snprintf(trash_path, sizeof(trash_path),
+                   "%s/" GLUSTERD_TRASH "/vols-%s.deleted", priv->workdir,
+                   volinfo->volname);
+    if ((len < 0) || (len >= sizeof(trash_path))) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_COPY_FAIL, NULL);
+        ret = -1;
+        goto out;
+    }
+
+    /* Since snapshot restore failed we cannot rely on the volume
+     * data stored under vols folder. Therefore delete the origin
+     * volume's backend folder.*/
+    ret = recursive_rmdir(pathname);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, errno, GD_MSG_DIR_OP_FAILED,
+               "Failed to remove "
+               "%s directory",
+               pathname);
+        goto out;
+    }
+
+    /* Now move the backup copy of the vols to its original
+     * location.*/
+    ret = sys_rename(trash_path, pathname);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, errno, GD_MSG_DIR_OP_FAILED,
+               "Failed to rename folder "
+               "from %s to %s",
+               trash_path, pathname);
+        goto out;
+    }
+
+    /* Retrieve the volume from the store */
+    reverted_vol = glusterd_store_retrieve_volume(volinfo->volname, NULL);
+    if (NULL == reverted_vol) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_VOL_OP_FAILED,
+               "Failed to load restored "
+               "%s volume",
+               volinfo->volname);
+        goto out;
+    }
+
+    /* Retrieve the snap_volumes list from the older volinfo */
+    reverted_vol->snap_count = volinfo->snap_count;
+    cds_list_for_each_entry_safe(snap_vol, tmp_vol, &volinfo->snap_volumes,
+                                 snapvol_list)
+    {
+        cds_list_add_tail(&snap_vol->snapvol_list, &reverted_vol->snap_volumes);
+
+        cds_list_for_each_entry(brickinfo, &snap_vol->bricks, brick_list)
+        {
+            /*
+             * If the brick is not of this peer, or snapshot is    *
+             * missed for the brick don't restore the xattr for it *
+             */
+            if ((!gf_uuid_compare(brickinfo->uuid, MY_UUID)) &&
+                (brickinfo->snap_status != -1)) {
+                /*
+                 * We need to restore volume id of all snap *
+                 * bricks to volume id of the snap volume.  *
+                 */
+                ret = sys_lsetxattr(brickinfo->path, GF_XATTR_VOL_ID_KEY,
+                                    snap_vol->volume_id,
+                                    sizeof(snap_vol->volume_id), XATTR_REPLACE);
+                if (ret == -1) {
+                    gf_smsg(this->name, GF_LOG_ERROR, 0, GD_MSG_SET_XATTR_FAIL,
+                            "Attribute=%s, Path=%s, Reason=%s, Snap=%s",
+                            GF_XATTR_VOL_ID_KEY, brickinfo->path,
+                            strerror(errno), snap_vol->volname, NULL);
+                    goto out;
+                }
+            }
+        }
+    }
+
+    /* Since we retrieved the volinfo from store now we don't
+     * want the older volinfo. Therefore delete the older volinfo */
+    glusterd_volinfo_unref(volinfo);
+    ret = 0;
 out:
-        return ret;
+    return ret;
 }
 
 /* This function is called when glusterd is started and we need
@@ -7916,42 +8956,46 @@ out:
  * @return 0 on success and -1 on failure
  */
 int
-glusterd_snapshot_revert_restore_from_snap (glusterd_snap_t *snap)
+glusterd_snapshot_revert_restore_from_snap(glusterd_snap_t *snap)
 {
-        int                     ret                     = -1;
-        char                    volname [PATH_MAX]      = {0,};
-        glusterd_volinfo_t     *snap_volinfo            = NULL;
-        glusterd_volinfo_t     *volinfo                 = NULL;
-        xlator_t               *this                    = NULL;
-
-        this = THIS;
-
-        GF_ASSERT (this);
-        GF_ASSERT (snap);
-
-        /* TODO : As of now there is only one volume in snapshot.
-         * Change this when multiple volume snapshot is introduced
-         */
-        snap_volinfo = cds_list_entry (snap->volumes.next, glusterd_volinfo_t,
-                                       vol_list);
-
-        strcpy (volname, snap_volinfo->parent_volname);
-
-        ret = glusterd_volinfo_find (volname, &volinfo);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "Could not get volinfo of "
-                        "%s", snap_volinfo->parent_volname);
-                goto out;
-        }
-
-        ret = glusterd_snapshot_revert_partial_restored_vol (volinfo);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "Failed to revert snapshot "
-                        "restore operation for %s volume", volname);
-                goto out;
-        }
+    int ret = -1;
+    char volname[PATH_MAX] = "";
+    glusterd_volinfo_t *snap_volinfo = NULL;
+    glusterd_volinfo_t *volinfo = NULL;
+    xlator_t *this = NULL;
+
+    this = THIS;
+
+    GF_ASSERT(this);
+    GF_ASSERT(snap);
+
+    /* TODO : As of now there is only one volume in snapshot.
+     * Change this when multiple volume snapshot is introduced
+     */
+    snap_volinfo = cds_list_entry(snap->volumes.next, glusterd_volinfo_t,
+                                  vol_list);
+
+    gf_strncpy(volname, snap_volinfo->parent_volname, sizeof(volname));
+
+    ret = glusterd_volinfo_find(volname, &volinfo);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, EINVAL, GD_MSG_VOL_NOT_FOUND,
+               "Could not get volinfo of "
+               "%s",
+               snap_volinfo->parent_volname);
+        goto out;
+    }
+
+    ret = glusterd_snapshot_revert_partial_restored_vol(volinfo);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_SNAP_RESTORE_REVERT_FAIL,
+               "Failed to revert snapshot "
+               "restore operation for %s volume",
+               volname);
+        goto out;
+    }
 out:
-        return ret;
+    return ret;
 }
 
 /* This function is called from post-validation. Based on the op_ret
@@ -7966,210 +9010,239 @@ out:
  * @return 0 on success and -1 on failure
  */
 int
-glusterd_snapshot_restore_postop (dict_t *dict, int32_t op_ret,
-                                  char **op_errstr, dict_t *rsp_dict)
+glusterd_snapshot_restore_postop(dict_t *dict, int32_t op_ret, char **op_errstr,
+                                 dict_t *rsp_dict)
 {
-        int                     ret             = -1;
-        char                   *name            = NULL;
-        char                   *volname         = NULL;
-        int                     cleanup         = 0;
-        glusterd_snap_t        *snap            = NULL;
-        glusterd_volinfo_t     *volinfo         = NULL;
-        xlator_t               *this            = NULL;
-
-        this = THIS;
-
-        GF_ASSERT (this);
-        GF_ASSERT (dict);
-        GF_ASSERT (rsp_dict);
-
-        ret = dict_get_str (dict, "snapname", &name);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "getting the snap "
-                        "name failed (volume: %s)", name);
-                goto out;
-        }
-
-        snap = glusterd_find_snap_by_name (name);
-        if (!snap) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "Snapshot (%s) does not exist", name);
-                ret = -1;
-                goto out;
-        }
-
-        /* TODO: fix this when multiple volume support will come */
-        ret = dict_get_str (dict, "volname1", &volname);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "failed to get volume name");
-                goto out;
-        }
-
-        ret = glusterd_volinfo_find (volname, &volinfo);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "Volume (%s) does not exist ", volname);
-                goto out;
+    int ret = -1;
+    char *name = NULL;
+    char *volname = NULL;
+    int cleanup = 0;
+    glusterd_snap_t *snap = NULL;
+    glusterd_volinfo_t *volinfo = NULL;
+    xlator_t *this = NULL;
+
+    this = THIS;
+
+    GF_ASSERT(this);
+    GF_ASSERT(dict);
+    GF_ASSERT(rsp_dict);
+
+    ret = dict_get_strn(dict, "snapname", SLEN("snapname"), &name);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+               "getting the snap "
+               "name failed (volume: %s)",
+               name);
+        goto out;
+    }
+
+    snap = glusterd_find_snap_by_name(name);
+    if (!snap) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_SNAP_NOT_FOUND,
+               "Snapshot (%s) does not exist", name);
+        ret = -1;
+        goto out;
+    }
+
+    /* TODO: fix this when multiple volume support will come */
+    ret = dict_get_strn(dict, "volname1", SLEN("volname1"), &volname);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+               "failed to get volume name");
+        goto out;
+    }
+
+    ret = glusterd_volinfo_find(volname, &volinfo);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_VOL_NOT_FOUND,
+               "Volume (%s) does not exist ", volname);
+        goto out;
+    }
+
+    ret = dict_get_strn(dict, "snapname", SLEN("snapname"), &name);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+               "getting the snap "
+               "name failed (volume: %s)",
+               volinfo->volname);
+        goto out;
+    }
+
+    snap = glusterd_find_snap_by_name(name);
+    if (!snap) {
+        gf_msg(this->name, GF_LOG_ERROR, EINVAL, GD_MSG_SNAP_NOT_FOUND,
+               "snap %s is not found", name);
+        ret = -1;
+        goto out;
+    }
+
+    /* On success perform the cleanup operation */
+    if (0 == op_ret) {
+        ret = glusterd_snapshot_restore_cleanup(rsp_dict, volname, snap);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_SNAP_CLEANUP_FAIL,
+                   "Failed to perform "
+                   "snapshot restore cleanup for %s volume",
+                   volname);
+            goto out;
+        }
+    } else { /* On failure revert snapshot restore */
+        ret = dict_get_int32n(dict, "cleanup", SLEN("cleanup"), &cleanup);
+        /* Perform cleanup only when required */
+        if (ret || (0 == cleanup)) {
+            /* Delete the backup copy of volume folder */
+            ret = glusterd_remove_trashpath(volinfo->volname);
+            if (ret) {
+                gf_msg(this->name, GF_LOG_ERROR, errno, GD_MSG_DIR_OP_FAILED,
+                       "Failed to remove backup dir");
+                goto out;
+            }
+            ret = 0;
+            goto out;
+        }
+
+        ret = glusterd_snapshot_revert_partial_restored_vol(volinfo);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_SNAP_RESTORE_REVERT_FAIL,
+                   "Failed to revert "
+                   "restore operation for %s volume",
+                   volname);
+            goto out;
         }
 
-        ret = dict_get_str (dict, "snapname", &name);
+        snap->snap_status = GD_SNAP_STATUS_IN_USE;
+        /* We need to save this in disk */
+        ret = glusterd_store_snap(snap);
         if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "getting the snap "
-                        "name failed (volume: %s)", volinfo->volname);
-                goto out;
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_SNAP_OBJECT_STORE_FAIL,
+                   "Could not store snap object for %s snap", snap->snapname);
+            goto out;
         }
 
-        snap = glusterd_find_snap_by_name (name);
-        if (!snap) {
-                gf_log (this->name, GF_LOG_ERROR, "snap %s is not found", name);
-                ret = -1;
-                goto out;
-        }
-
-        /* On success perform the cleanup operation */
-        if (0 == op_ret) {
-                ret = glusterd_snapshot_restore_cleanup (rsp_dict,
-                                                         volname,
-                                                         snap);
-                if (ret) {
-                        gf_log (this->name, GF_LOG_ERROR, "Failed to perform "
-                                "snapshot restore cleanup for %s volume",
-                                volname);
-                        goto out;
-                }
-        } else { /* On failure revert snapshot restore */
-                ret = dict_get_int32 (dict, "cleanup", &cleanup);
-                /* Perform cleanup only when required */
-                if (ret || (0 == cleanup)) {
-                        ret = 0;
-                        goto out;
-                }
-
-                ret = glusterd_snapshot_revert_partial_restored_vol (volinfo);
-                if (ret) {
-                        gf_log (this->name, GF_LOG_ERROR, "Failed to revert "
-                                "restore operation for %s volume", volname);
-                        goto out;
-                }
-
-                snap->snap_status = GD_SNAP_STATUS_IN_USE;
-                /* We need to save this in disk */
-                ret = glusterd_store_snap (snap);
-                if (ret) {
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "Could not store snap object for %s snap",
-                                snap->snapname);
-                        goto out;
-                }
+        /* After restore fails, we have to remove mount point for
+         * deactivated snaps which was created at start of restore op.
+         */
+        if (volinfo->status == GLUSTERD_STATUS_STOPPED) {
+            ret = glusterd_snap_unmount(this, volinfo);
+            if (ret) {
+                gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_GLUSTERD_UMOUNT_FAIL,
+                       "Failed to unmounts for %s", snap->snapname);
+            }
         }
+    }
 
-        ret = 0;
+    ret = 0;
 out:
-        return ret;
+    return ret;
 }
 
 int
-glusterd_snapshot_postvalidate (dict_t *dict, int32_t op_ret, char **op_errstr,
-                                dict_t *rsp_dict)
+glusterd_snapshot_postvalidate(dict_t *dict, int32_t op_ret, char **op_errstr,
+                               dict_t *rsp_dict)
 {
-        int                snap_command          = 0;
-        xlator_t           *this                 = NULL;
-        int                ret                   = -1;
+    int snap_command = 0;
+    xlator_t *this = NULL;
+    int ret = -1;
 
-        this = THIS;
+    this = THIS;
 
-        GF_ASSERT (this);
-        GF_ASSERT (dict);
-        GF_ASSERT (rsp_dict);
+    GF_ASSERT(this);
+    GF_ASSERT(dict);
+    GF_ASSERT(rsp_dict);
 
-        ret = dict_get_int32 (dict, "type", &snap_command);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "unable to get the type of "
-                        "the snapshot command");
-                goto out;
-        }
+    ret = dict_get_int32n(dict, "type", SLEN("type"), &snap_command);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_COMMAND_NOT_FOUND,
+               "unable to get the type of "
+               "the snapshot command");
+        goto out;
+    }
 
-        switch (snap_command) {
+    switch (snap_command) {
         case GF_SNAP_OPTION_TYPE_CREATE:
-                ret = glusterd_snapshot_create_postvalidate (dict, op_ret,
-                                                             op_errstr,
-                                                             rsp_dict);
-                if (ret) {
-                        gf_log (this->name, GF_LOG_WARNING, "Snapshot create "
-                                "post-validation failed");
-                        goto out;
-                }
-                glusterd_fetchsnap_notify (this);
-                break;
-         case GF_SNAP_OPTION_TYPE_CLONE:
-                ret = glusterd_snapshot_clone_postvalidate (dict, op_ret,
-                                                             op_errstr,
-                                                             rsp_dict);
-                if (ret) {
-                        gf_log (this->name, GF_LOG_WARNING, "Snapshot create "
-                                "post-validation failed");
-                        goto out;
-                }
-                glusterd_fetchsnap_notify (this);
-                break;
+            ret = glusterd_snapshot_create_postvalidate(dict, op_ret, op_errstr,
+                                                        rsp_dict);
+            if (ret) {
+                gf_msg(this->name, GF_LOG_WARNING, 0, GD_MSG_SNAP_CREATION_FAIL,
+                       "Snapshot create "
+                       "post-validation failed");
+                goto out;
+            }
+            glusterd_fetchsnap_notify(this);
+            break;
+        case GF_SNAP_OPTION_TYPE_CLONE:
+            ret = glusterd_snapshot_clone_postvalidate(dict, op_ret, op_errstr,
+                                                       rsp_dict);
+            if (ret) {
+                gf_msg(this->name, GF_LOG_WARNING, 0,
+                       GD_MSG_SNAP_CLONE_POSTVAL_FAILED,
+                       "Snapshot create "
+                       "post-validation failed");
+                goto out;
+            }
+            glusterd_fetchsnap_notify(this);
+            break;
         case GF_SNAP_OPTION_TYPE_DELETE:
-                if (op_ret) {
-                        gf_log (this->name, GF_LOG_DEBUG,
-                                "op_ret = %d. Not performing delete "
-                                "post_validate", op_ret);
-                        ret = 0;
-                        goto out;
-                }
-                ret = glusterd_snapshot_update_snaps_post_validate (dict,
-                                                                    op_errstr,
-                                                                    rsp_dict);
-                if (ret) {
-                        gf_log (this->name, GF_LOG_ERROR, "Failed to "
-                                "update missed snaps list");
-                        goto out;
-                }
-                glusterd_fetchsnap_notify (this);
-                break;
+            if (op_ret) {
+                gf_msg_debug(this->name, 0,
+                             "op_ret = %d. Not performing delete "
+                             "post_validate",
+                             op_ret);
+                ret = 0;
+                goto out;
+            }
+            ret = glusterd_snapshot_update_snaps_post_validate(dict, op_errstr,
+                                                               rsp_dict);
+            if (ret) {
+                gf_msg(this->name, GF_LOG_ERROR, 0,
+                       GD_MSG_MISSED_SNAP_LIST_STORE_FAIL,
+                       "Failed to "
+                       "update missed snaps list");
+                goto out;
+            }
+            glusterd_fetchsnap_notify(this);
+            break;
         case GF_SNAP_OPTION_TYPE_RESTORE:
-                ret = glusterd_snapshot_update_snaps_post_validate (dict,
-                                                                    op_errstr,
-                                                                    rsp_dict);
-                if (ret) {
-                        gf_log (this->name, GF_LOG_ERROR, "Failed to "
-                                "update missed snaps list");
-                        goto out;
-                }
+            ret = glusterd_snapshot_update_snaps_post_validate(dict, op_errstr,
+                                                               rsp_dict);
+            if (ret) {
+                gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_SNAP_RESTORE_FAIL,
+                       "Failed to "
+                       "update missed snaps list");
+                goto out;
+            }
 
-                ret = glusterd_snapshot_restore_postop (dict, op_ret,
-                                                        op_errstr, rsp_dict);
-                if (ret) {
-                        gf_log (this->name, GF_LOG_ERROR, "Failed to "
-                                "perform snapshot restore post-op");
-                        goto out;
-                }
-                glusterd_fetchsnap_notify (this);
-                break;
+            ret = glusterd_snapshot_restore_postop(dict, op_ret, op_errstr,
+                                                   rsp_dict);
+            if (ret) {
+                gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_SNAP_RESTORE_FAIL,
+                       "Failed to "
+                       "perform snapshot restore post-op");
+                goto out;
+            }
+            glusterd_fetchsnap_notify(this);
+            break;
         case GF_SNAP_OPTION_TYPE_ACTIVATE:
         case GF_SNAP_OPTION_TYPE_DEACTIVATE:
-                glusterd_fetchsnap_notify (this);
-                break;
+            glusterd_fetchsnap_notify(this);
+            break;
         case GF_SNAP_OPTION_TYPE_STATUS:
         case GF_SNAP_OPTION_TYPE_CONFIG:
         case GF_SNAP_OPTION_TYPE_INFO:
         case GF_SNAP_OPTION_TYPE_LIST:
-                 /*Nothing to be done. But want to
-                 * avoid the default case warning*/
-                ret = 0;
-                break;
+            /*Nothing to be done. But want to
+             * avoid the default case warning*/
+            ret = 0;
+            break;
         default:
-                gf_log (this->name, GF_LOG_WARNING, "invalid snap command");
-                goto out;
-        }
+            gf_msg(this->name, GF_LOG_WARNING, 0, GD_MSG_COMMAND_NOT_FOUND,
+                   "invalid snap command");
+            goto out;
+    }
 
-        ret = 0;
+    ret = 0;
 out:
-        return ret;
+    return ret;
 }
 
 /*
@@ -8177,585 +9250,588 @@ out:
 */
 
 static gf_boolean_t
-glusterd_is_lvm_cmd_available (char *lvm_cmd)
+glusterd_is_lvm_cmd_available(char *lvm_cmd)
 {
-        int32_t     ret  = 0;
-        struct stat buf  = {0,};
-
-        if (!lvm_cmd)
-                return _gf_false;
-
-        ret = stat (lvm_cmd, &buf);
-        if (ret != 0) {
-                gf_log (THIS->name, GF_LOG_ERROR,
-                        "stat fails on %s, exiting. (errno = %d (%s))",
-                        lvm_cmd, errno, strerror(errno));
-                return _gf_false;
-        }
-
-        if ((!ret) && (!S_ISREG(buf.st_mode))) {
-                gf_log (THIS->name, GF_LOG_CRITICAL,
-                        "Provided command %s is not a regular file,"
-                        "exiting", lvm_cmd);
-                return _gf_false;
-        }
-
-        if ((!ret) && (!(buf.st_mode & S_IXUSR))) {
-                gf_log (THIS->name, GF_LOG_CRITICAL,
-                        "Provided command %s has no exec permissions,"
-                        "exiting", lvm_cmd);
-                return _gf_false;
-        }
-
-        return _gf_true;
+    int32_t ret = 0;
+    struct stat buf = {
+        0,
+    };
+
+    if (!lvm_cmd)
+        return _gf_false;
+
+    ret = sys_stat(lvm_cmd, &buf);
+    if (ret != 0) {
+        gf_msg(THIS->name, GF_LOG_ERROR, errno, GD_MSG_FILE_OP_FAILED,
+               "stat fails on %s, exiting. (errno = %d (%s))", lvm_cmd, errno,
+               strerror(errno));
+        return _gf_false;
+    }
+
+    if ((!ret) && (!S_ISREG(buf.st_mode))) {
+        gf_msg(THIS->name, GF_LOG_CRITICAL, EINVAL, GD_MSG_COMMAND_NOT_FOUND,
+               "Provided command %s is not a regular file,"
+               "exiting",
+               lvm_cmd);
+        return _gf_false;
+    }
+
+    if ((!ret) && (!(buf.st_mode & S_IXUSR))) {
+        gf_msg(THIS->name, GF_LOG_CRITICAL, 0, GD_MSG_NO_EXEC_PERMS,
+               "Provided command %s has no exec permissions,"
+               "exiting",
+               lvm_cmd);
+        return _gf_false;
+    }
+
+    return _gf_true;
 }
 
 int
-glusterd_handle_snapshot_fn (rpcsvc_request_t *req)
+glusterd_handle_snapshot_fn(rpcsvc_request_t *req)
 {
-        int32_t               ret            = 0;
-        dict_t               *dict           = NULL;
-        gf_cli_req            cli_req        = {{0},};
-        glusterd_op_t         cli_op         = GD_OP_SNAP;
-        int                   type           = 0;
-        glusterd_conf_t      *conf           = NULL;
-        char                 *host_uuid      = NULL;
-        char                  err_str[2048]  = {0,};
-        xlator_t             *this           = NULL;
-
-        GF_ASSERT (req);
-
-        this = THIS;
-        GF_ASSERT (this);
-        conf = this->private;
-        GF_ASSERT (conf);
-
-        ret = xdr_to_generic (req->msg[0], &cli_req,
-                              (xdrproc_t)xdr_gf_cli_req);
-        if (ret < 0) {
-                req->rpc_err = GARBAGE_ARGS;
-                goto out;
-        }
-
-        if (cli_req.dict.dict_len > 0) {
-                dict = dict_new ();
-                if (!dict)
-                        goto out;
-
-                ret = dict_unserialize (cli_req.dict.dict_val,
-                                        cli_req.dict.dict_len,
-                                        &dict);
-                if (ret < 0) {
-                        gf_log (this->name, GF_LOG_ERROR, "failed to "
-                                "unserialize req-buffer to dictionary");
-                        snprintf (err_str, sizeof (err_str), "Unable to decode "
-                                  "the command");
-                        goto out;
-                }
-
-                dict->extra_stdfree = cli_req.dict.dict_val;
-
-                host_uuid = gf_strdup (uuid_utoa(MY_UUID));
-                if (host_uuid == NULL) {
-                        snprintf (err_str, sizeof (err_str), "Failed to get "
-                                  "the uuid of local glusterd");
-                        ret = -1;
-                        goto out;
-                }
-                ret = dict_set_dynstr (dict, "host-uuid", host_uuid);
-                if (ret) {
-                        GF_FREE (host_uuid);
-                        goto out;
-                }
-
-
-        } else {
-                gf_log (this->name, GF_LOG_ERROR, "request dict length is %d",
-                        cli_req.dict.dict_len);
-                goto out;
-        }
-
-        if (conf->op_version < GD_OP_VERSION_3_6_0) {
-                snprintf (err_str, sizeof (err_str), "Cluster operating version"
-                          " is lesser than the supported version "
-                          "for a snapshot");
-                gf_log (this->name, GF_LOG_ERROR, "%s (%d < %d)", err_str,
-                        conf->op_version, GD_OP_VERSION_3_6_0);
-                ret = -1;
-                goto out;
-        }
-
-        ret = dict_get_int32 (dict, "type", &type);
+    int32_t ret = 0;
+    dict_t *dict = NULL;
+    gf_cli_req cli_req = {
+        {0},
+    };
+    glusterd_op_t cli_op = GD_OP_SNAP;
+    int type = 0;
+    glusterd_conf_t *conf = NULL;
+    char *host_uuid = NULL;
+    char err_str[2048] = "";
+    xlator_t *this = NULL;
+    uint32_t op_errno = 0;
+
+    GF_ASSERT(req);
+
+    this = THIS;
+    GF_ASSERT(this);
+    conf = this->private;
+    GF_ASSERT(conf);
+
+    ret = xdr_to_generic(req->msg[0], &cli_req, (xdrproc_t)xdr_gf_cli_req);
+    if (ret < 0) {
+        req->rpc_err = GARBAGE_ARGS;
+        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_GARBAGE_ARGS, NULL);
+        goto out;
+    }
+
+    if (cli_req.dict.dict_len > 0) {
+        dict = dict_new();
+        if (!dict)
+            goto out;
+
+        ret = dict_unserialize(cli_req.dict.dict_val, cli_req.dict.dict_len,
+                               &dict);
         if (ret < 0) {
-                snprintf (err_str, sizeof (err_str), "Command type not found");
-                gf_log (this->name, GF_LOG_ERROR, "%s", err_str);
-                goto out;
-        }
-
-        if (!glusterd_is_lvm_cmd_available (LVM_CREATE)) {
-                snprintf (err_str, sizeof (err_str), "LVM commands not found,"
-                          " snapshot functionality is disabled");
-                gf_log (this->name, GF_LOG_ERROR, "%s", err_str);
-                ret = -1;
-                goto out;
-        }
-
-        switch (type) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_UNSERIALIZE_FAIL,
+                   "failed to "
+                   "unserialize req-buffer to dictionary");
+            snprintf(err_str, sizeof(err_str),
+                     "Unable to decode "
+                     "the command");
+            goto out;
+        }
+
+        dict->extra_stdfree = cli_req.dict.dict_val;
+
+        host_uuid = gf_strdup(uuid_utoa(MY_UUID));
+        if (host_uuid == NULL) {
+            snprintf(err_str, sizeof(err_str),
+                     "Failed to get "
+                     "the uuid of local glusterd");
+            ret = -1;
+            goto out;
+        }
+        ret = dict_set_dynstrn(dict, "host-uuid", SLEN("host-uuid"), host_uuid);
+        if (ret) {
+            GF_FREE(host_uuid);
+            goto out;
+        }
+
+    } else {
+        gf_msg(this->name, GF_LOG_ERROR, EINVAL, GD_MSG_INVALID_ENTRY,
+               "request dict length is %d", cli_req.dict.dict_len);
+        goto out;
+    }
+
+    if (conf->op_version < GD_OP_VERSION_3_6_0) {
+        snprintf(err_str, sizeof(err_str),
+                 "Cluster operating version"
+                 " is lesser than the supported version "
+                 "for a snapshot");
+        op_errno = EG_OPNOTSUP;
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_UNSUPPORTED_VERSION,
+               "%s (%d < %d)", err_str, conf->op_version, GD_OP_VERSION_3_6_0);
+        ret = -1;
+        goto out;
+    }
+
+    ret = dict_get_int32n(dict, "type", SLEN("type"), &type);
+    if (ret < 0) {
+        snprintf(err_str, sizeof(err_str), "Command type not found");
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_COMMAND_NOT_FOUND, "%s",
+               err_str);
+        goto out;
+    }
+
+    if (!glusterd_is_lvm_cmd_available(LVM_CREATE)) {
+        snprintf(err_str, sizeof(err_str),
+                 "LVM commands not found,"
+                 " snapshot functionality is disabled");
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_COMMAND_NOT_FOUND, "%s",
+               err_str);
+        ret = -1;
+        goto out;
+    }
+
+    switch (type) {
         case GF_SNAP_OPTION_TYPE_CREATE:
-                ret = glusterd_handle_snapshot_create (req, cli_op, dict,
-                                                err_str, sizeof (err_str));
-                if (ret) {
-                        gf_log (this->name, GF_LOG_WARNING, "Snapshot create "
-                                        "failed: %s", err_str);
-                }
-                break;
+            ret = glusterd_handle_snapshot_create(req, cli_op, dict, err_str,
+                                                  sizeof(err_str));
+            if (ret) {
+                gf_msg(this->name, GF_LOG_WARNING, 0, GD_MSG_SNAP_CREATION_FAIL,
+                       "Snapshot create failed: %s", err_str);
+            }
+            break;
 
         case GF_SNAP_OPTION_TYPE_CLONE:
-                ret = glusterd_handle_snapshot_clone (req, cli_op, dict,
-                                                err_str, sizeof (err_str));
-                if (ret) {
-                        gf_log (this->name, GF_LOG_WARNING, "Snapshot clone "
-                                        "failed: %s", err_str);
-                }
-                break;
+            ret = glusterd_handle_snapshot_clone(req, cli_op, dict, err_str,
+                                                 sizeof(err_str));
+            if (ret) {
+                gf_msg(this->name, GF_LOG_WARNING, 0, GD_MSG_SNAP_CLONE_FAILED,
+                       "Snapshot clone "
+                       "failed: %s",
+                       err_str);
+            }
+            break;
 
         case GF_SNAP_OPTION_TYPE_RESTORE:
-                ret = glusterd_handle_snapshot_restore (req, cli_op, dict,
-                                                err_str, sizeof (err_str));
-                if (ret) {
-                        gf_log (this->name, GF_LOG_WARNING, "Snapshot restore "
-                                        "failed: %s", err_str);
-                }
-
-                break;
+            ret = glusterd_handle_snapshot_restore(req, cli_op, dict, err_str,
+                                                   &op_errno, sizeof(err_str));
+            if (ret) {
+                gf_msg(this->name, GF_LOG_WARNING, 0, GD_MSG_SNAP_RESTORE_FAIL,
+                       "Snapshot restore failed: %s", err_str);
+            }
+
+            break;
         case GF_SNAP_OPTION_TYPE_INFO:
-                ret = glusterd_handle_snapshot_info (req, cli_op, dict,
-                                                err_str, sizeof (err_str));
-                if (ret) {
-                        gf_log (this->name, GF_LOG_WARNING, "Snapshot info "
-                                        "failed");
-                }
-                break;
+            ret = glusterd_handle_snapshot_info(req, cli_op, dict, err_str,
+                                                sizeof(err_str));
+            if (ret) {
+                gf_msg(this->name, GF_LOG_WARNING, 0, GD_MSG_SNAP_INFO_FAIL,
+                       "Snapshot info failed");
+            }
+            break;
         case GF_SNAP_OPTION_TYPE_LIST:
-                ret = glusterd_handle_snapshot_list (req, cli_op, dict,
-                                                err_str, sizeof (err_str));
-                if (ret) {
-                        gf_log (this->name, GF_LOG_WARNING, "Snapshot list "
-                                        "failed");
-                }
-                break;
+            ret = glusterd_handle_snapshot_list(req, cli_op, dict, err_str,
+                                                sizeof(err_str), &op_errno);
+            if (ret) {
+                gf_msg(this->name, GF_LOG_WARNING, 0, GD_MSG_SNAP_LIST_GET_FAIL,
+                       "Snapshot list failed");
+            }
+            break;
         case GF_SNAP_OPTION_TYPE_CONFIG:
-                ret = glusterd_handle_snapshot_config (req, cli_op, dict,
-                                                 err_str, sizeof (err_str));
-                if (ret) {
-                        gf_log (this->name, GF_LOG_WARNING, "snapshot config "
-                                "failed");
-                }
-                break;
+            ret = glusterd_handle_snapshot_config(req, cli_op, dict, err_str,
+                                                  sizeof(err_str));
+            if (ret) {
+                gf_msg(this->name, GF_LOG_WARNING, 0, GD_MSG_SNAP_CONFIG_FAIL,
+                       "snapshot config failed");
+            }
+            break;
         case GF_SNAP_OPTION_TYPE_DELETE:
-                ret = glusterd_handle_snapshot_delete (req, cli_op, dict,
-                                                       err_str,
-                                                       sizeof (err_str));
-                if (ret) {
-                        gf_log (this->name, GF_LOG_WARNING, "Snapshot delete "
-                                "failed: %s", err_str);
-                }
-                break;
-        case  GF_SNAP_OPTION_TYPE_ACTIVATE:
-                ret = glusterd_mgmt_v3_initiate_snap_phases (req, cli_op,
-                                                             dict);
-                if (ret) {
-                        gf_log (this->name, GF_LOG_WARNING,
-                                "Snapshot activate failed: %s", err_str);
-                }
-                break;
+            ret = glusterd_handle_snapshot_delete(req, cli_op, dict, err_str,
+                                                  &op_errno, sizeof(err_str));
+            if (ret) {
+                gf_msg(this->name, GF_LOG_WARNING, 0, GD_MSG_SNAP_REMOVE_FAIL,
+                       "Snapshot delete failed: %s", err_str);
+            }
+            break;
+        case GF_SNAP_OPTION_TYPE_ACTIVATE:
+            ret = glusterd_mgmt_v3_initiate_snap_phases(req, cli_op, dict);
+            if (ret) {
+                gf_msg(this->name, GF_LOG_WARNING, 0, GD_MSG_SNAP_ACTIVATE_FAIL,
+                       "Snapshot activate failed: %s", err_str);
+            }
+            break;
         case GF_SNAP_OPTION_TYPE_DEACTIVATE:
-                ret = glusterd_mgmt_v3_initiate_snap_phases (req, cli_op,
-                                                             dict);
-                if (ret) {
-                        gf_log (this->name, GF_LOG_WARNING,
-                                "Snapshot deactivate failed: %s", err_str);
-                }
-                break;
+            ret = glusterd_mgmt_v3_initiate_snap_phases(req, cli_op, dict);
+            if (ret) {
+                gf_msg(this->name, GF_LOG_WARNING, 0,
+                       GD_MSG_SNAP_DEACTIVATE_FAIL,
+                       "Snapshot deactivate failed: %s", err_str);
+            }
+            break;
         case GF_SNAP_OPTION_TYPE_STATUS:
-                ret = glusterd_handle_snapshot_status (req, cli_op, dict,
-                                                       err_str,
-                                                       sizeof (err_str));
-                if (ret) {
-                        gf_log (this->name, GF_LOG_WARNING, "Snapshot status "
-                                "failed: %s", err_str);
-                }
-                break;
+            ret = glusterd_handle_snapshot_status(req, cli_op, dict, err_str,
+                                                  sizeof(err_str));
+            if (ret) {
+                gf_msg(this->name, GF_LOG_WARNING, 0, GD_MSG_SNAP_STATUS_FAIL,
+                       "Snapshot status failed: %s", err_str);
+            }
+            break;
         default:
-                gf_log (this->name, GF_LOG_ERROR, "Unknown snapshot request "
-                        "type (%d)", type);
-                ret = -1; /* Failure */
-        }
+            gf_msg(this->name, GF_LOG_ERROR, EINVAL, GD_MSG_COMMAND_NOT_FOUND,
+                   "Unknown snapshot request "
+                   "type (%d)",
+                   type);
+            ret = -1; /* Failure */
+    }
 
 out:
-        if (ret) {
-                if (err_str[0] == '\0')
-                        snprintf (err_str, sizeof (err_str),
-                                  "Operation failed");
-                ret = glusterd_op_send_cli_response (cli_op, ret, 0, req,
-                                                     dict, err_str);
-        }
+    if (ret) {
+        if (err_str[0] == '\0')
+            snprintf(err_str, sizeof(err_str), "Operation failed");
 
-        return ret;
+        if (ret && (op_errno == 0))
+            op_errno = EG_INTRNL;
+
+        ret = glusterd_op_send_cli_response(cli_op, ret, op_errno, req, dict,
+                                            err_str);
+    }
+
+    return ret;
 }
 
 int
-glusterd_handle_snapshot (rpcsvc_request_t *req)
+glusterd_handle_snapshot(rpcsvc_request_t *req)
 {
-        return glusterd_big_locked_handler (req, glusterd_handle_snapshot_fn);
+    return glusterd_big_locked_handler(req, glusterd_handle_snapshot_fn);
 }
 
-static inline void
-glusterd_free_snap_op (glusterd_snap_op_t *snap_op)
+static void
+glusterd_free_snap_op(glusterd_snap_op_t *snap_op)
 {
-        if (snap_op) {
-                if (snap_op->brick_path)
-                        GF_FREE (snap_op->brick_path);
+    if (snap_op) {
+        if (snap_op->brick_path)
+            GF_FREE(snap_op->brick_path);
 
-                GF_FREE (snap_op);
-        }
+        GF_FREE(snap_op);
+    }
 }
 
-static inline void
-glusterd_free_missed_snapinfo (glusterd_missed_snap_info *missed_snapinfo)
+static void
+glusterd_free_missed_snapinfo(glusterd_missed_snap_info *missed_snapinfo)
 {
-        glusterd_snap_op_t *snap_opinfo = NULL;
-        glusterd_snap_op_t *tmp         = NULL;
-
-        if (missed_snapinfo) {
-                cds_list_for_each_entry_safe (snap_opinfo, tmp,
-                                              &missed_snapinfo->snap_ops,
-                                              snap_ops_list) {
-                        glusterd_free_snap_op (snap_opinfo);
-                        snap_opinfo = NULL;
-                }
+    glusterd_snap_op_t *snap_opinfo = NULL;
+    glusterd_snap_op_t *tmp = NULL;
 
-                if (missed_snapinfo->node_uuid)
-                        GF_FREE (missed_snapinfo->node_uuid);
+    if (missed_snapinfo) {
+        cds_list_for_each_entry_safe(snap_opinfo, tmp,
+                                     &missed_snapinfo->snap_ops, snap_ops_list)
+        {
+            glusterd_free_snap_op(snap_opinfo);
+            snap_opinfo = NULL;
+        }
 
-                if (missed_snapinfo->snap_uuid)
-                        GF_FREE (missed_snapinfo->snap_uuid);
+        if (missed_snapinfo->node_uuid)
+            GF_FREE(missed_snapinfo->node_uuid);
 
-                GF_FREE (missed_snapinfo);
-        }
+        if (missed_snapinfo->snap_uuid)
+            GF_FREE(missed_snapinfo->snap_uuid);
+
+        GF_FREE(missed_snapinfo);
+    }
 }
 
 /* Look for duplicates and accordingly update the list */
 int32_t
-glusterd_update_missed_snap_entry (glusterd_missed_snap_info *missed_snapinfo,
-                                   glusterd_snap_op_t *missed_snap_op)
+glusterd_update_missed_snap_entry(glusterd_missed_snap_info *missed_snapinfo,
+                                  glusterd_snap_op_t *missed_snap_op)
 {
-        int32_t                        ret                         = -1;
-        glusterd_snap_op_t            *snap_opinfo                 = NULL;
-        gf_boolean_t                   match                       = _gf_false;
-        xlator_t                      *this                        = NULL;
-
-        this = THIS;
-        GF_ASSERT(this);
-        GF_ASSERT(missed_snapinfo);
-        GF_ASSERT(missed_snap_op);
-
-        cds_list_for_each_entry (snap_opinfo, &missed_snapinfo->snap_ops,
-                                 snap_ops_list) {
-                /* If the entry is not for the same snap_vol_id
-                 * then continue
-                 */
-                if (strcmp (snap_opinfo->snap_vol_id,
-                            missed_snap_op->snap_vol_id))
-                        continue;
-
-                if ((!strcmp (snap_opinfo->brick_path,
-                              missed_snap_op->brick_path)) &&
-                    (snap_opinfo->op == missed_snap_op->op)) {
-                        /* If two entries have conflicting status
-                         * GD_MISSED_SNAP_DONE takes precedence
-                         */
-                        if ((snap_opinfo->status == GD_MISSED_SNAP_PENDING) &&
-                            (missed_snap_op->status == GD_MISSED_SNAP_DONE)) {
-                                snap_opinfo->status = GD_MISSED_SNAP_DONE;
-                                gf_log (this->name, GF_LOG_INFO,
-                                        "Updating missed snap status "
-                                        "for %s:%s=%s:%d:%s:%d as DONE",
-                                        missed_snapinfo->node_uuid,
-                                        missed_snapinfo->snap_uuid,
-                                        snap_opinfo->snap_vol_id,
-                                        snap_opinfo->brick_num,
-                                        snap_opinfo->brick_path,
-                                        snap_opinfo->op);
-                                ret = 0;
-                                glusterd_free_snap_op (missed_snap_op);
-                                goto out;
-                        }
-                        match = _gf_true;
-                        break;
-                } else if ((snap_opinfo->brick_num ==
-                            missed_snap_op->brick_num) &&
-                            (snap_opinfo->op == GF_SNAP_OPTION_TYPE_CREATE) &&
-                            ((missed_snap_op->op ==
-                              GF_SNAP_OPTION_TYPE_DELETE) ||
-                             (missed_snap_op->op ==
-                              GF_SNAP_OPTION_TYPE_RESTORE))) {
-                        /* Optimizing create and delete entries for the same
-                         * brick and same node
-                         */
-                        gf_log (this->name, GF_LOG_INFO,
-                                "Updating missed snap status "
-                                "for %s:%s=%s:%d:%s:%d as DONE",
-                                missed_snapinfo->node_uuid,
-                                missed_snapinfo->snap_uuid,
-                                snap_opinfo->snap_vol_id,
-                                snap_opinfo->brick_num,
-                                snap_opinfo->brick_path,
-                                snap_opinfo->op);
-                        snap_opinfo->status = GD_MISSED_SNAP_DONE;
-                        ret = 0;
-                        glusterd_free_snap_op (missed_snap_op);
-                        goto out;
-                }
-        }
-
-        if (match == _gf_true) {
-                gf_log (this->name, GF_LOG_INFO,
-                        "Duplicate entry. Not updating");
-                glusterd_free_snap_op (missed_snap_op);
-        } else {
-                cds_list_add_tail (&missed_snap_op->snap_ops_list,
-                                   &missed_snapinfo->snap_ops);
-        }
-
-        ret = 0;
+    int32_t ret = -1;
+    glusterd_snap_op_t *snap_opinfo = NULL;
+    gf_boolean_t match = _gf_false;
+    xlator_t *this = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+    GF_ASSERT(missed_snapinfo);
+    GF_ASSERT(missed_snap_op);
+
+    cds_list_for_each_entry(snap_opinfo, &missed_snapinfo->snap_ops,
+                            snap_ops_list)
+    {
+        /* If the entry is not for the same snap_vol_id
+         * then continue
+         */
+        if (strcmp(snap_opinfo->snap_vol_id, missed_snap_op->snap_vol_id))
+            continue;
+
+        if ((!strcmp(snap_opinfo->brick_path, missed_snap_op->brick_path)) &&
+            (snap_opinfo->op == missed_snap_op->op)) {
+            /* If two entries have conflicting status
+             * GD_MISSED_SNAP_DONE takes precedence
+             */
+            if ((snap_opinfo->status == GD_MISSED_SNAP_PENDING) &&
+                (missed_snap_op->status == GD_MISSED_SNAP_DONE)) {
+                snap_opinfo->status = GD_MISSED_SNAP_DONE;
+                gf_msg(this->name, GF_LOG_INFO, 0,
+                       GD_MSG_MISSED_SNAP_STATUS_DONE,
+                       "Updating missed snap status "
+                       "for %s:%s=%s:%d:%s:%d as DONE",
+                       missed_snapinfo->node_uuid, missed_snapinfo->snap_uuid,
+                       snap_opinfo->snap_vol_id, snap_opinfo->brick_num,
+                       snap_opinfo->brick_path, snap_opinfo->op);
+                ret = 0;
+                glusterd_free_snap_op(missed_snap_op);
+                goto out;
+            }
+            match = _gf_true;
+            break;
+        } else if ((snap_opinfo->brick_num == missed_snap_op->brick_num) &&
+                   (snap_opinfo->op == GF_SNAP_OPTION_TYPE_CREATE) &&
+                   ((missed_snap_op->op == GF_SNAP_OPTION_TYPE_DELETE) ||
+                    (missed_snap_op->op == GF_SNAP_OPTION_TYPE_RESTORE))) {
+            /* Optimizing create and delete entries for the same
+             * brick and same node
+             */
+            gf_msg(this->name, GF_LOG_INFO, 0, GD_MSG_MISSED_SNAP_STATUS_DONE,
+                   "Updating missed snap status "
+                   "for %s:%s=%s:%d:%s:%d as DONE",
+                   missed_snapinfo->node_uuid, missed_snapinfo->snap_uuid,
+                   snap_opinfo->snap_vol_id, snap_opinfo->brick_num,
+                   snap_opinfo->brick_path, snap_opinfo->op);
+            snap_opinfo->status = GD_MISSED_SNAP_DONE;
+            ret = 0;
+            glusterd_free_snap_op(missed_snap_op);
+            goto out;
+        }
+    }
+
+    if (match == _gf_true) {
+        gf_msg(this->name, GF_LOG_INFO, 0, GD_MSG_DUP_ENTRY,
+               "Duplicate entry. Not updating");
+        glusterd_free_snap_op(missed_snap_op);
+    } else {
+        cds_list_add_tail(&missed_snap_op->snap_ops_list,
+                          &missed_snapinfo->snap_ops);
+    }
+
+    ret = 0;
 out:
-        gf_log (this->name, GF_LOG_TRACE, "Returning %d", ret);
-        return ret;
+    gf_msg_trace(this->name, 0, "Returning %d", ret);
+    return ret;
 }
 
 /* Add new missed snap entry to the missed_snaps list. */
 int32_t
-glusterd_add_new_entry_to_list (char *missed_info, char *snap_vol_id,
-                                int32_t brick_num, char *brick_path,
-                                int32_t snap_op, int32_t snap_status)
+glusterd_add_new_entry_to_list(char *missed_info, char *snap_vol_id,
+                               int32_t brick_num, char *brick_path,
+                               int32_t snap_op, int32_t snap_status)
 {
-        char                          *buf                         = NULL;
-        char                          *save_ptr                    = NULL;
-        char                           node_snap_info[PATH_MAX]    = "";
-        int32_t                        ret                         = -1;
-        glusterd_missed_snap_info     *missed_snapinfo             = NULL;
-        glusterd_snap_op_t            *missed_snap_op              = NULL;
-        glusterd_conf_t               *priv                        = NULL;
-        gf_boolean_t                   match                       = _gf_false;
-        gf_boolean_t                   free_missed_snap_info       = _gf_false;
-        xlator_t                      *this                        = NULL;
-
-        this = THIS;
-        GF_ASSERT(this);
-        GF_ASSERT(missed_info);
-        GF_ASSERT(snap_vol_id);
-        GF_ASSERT(brick_path);
-
-        priv = this->private;
-        GF_ASSERT (priv);
-
-        /* Create the snap_op object consisting of the *
-         * snap id and the op */
-        ret = glusterd_missed_snap_op_new (&missed_snap_op);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "Failed to create new missed snap object.");
-                ret = -1;
-                goto out;
-        }
-
-        missed_snap_op->snap_vol_id = gf_strdup(snap_vol_id);
-        if (!missed_snap_op->snap_vol_id) {
-                ret = -1;
-                goto out;
-        }
-        missed_snap_op->brick_path = gf_strdup(brick_path);
-        if (!missed_snap_op->brick_path) {
-                ret = -1;
-                goto out;
-        }
-        missed_snap_op->brick_num = brick_num;
-        missed_snap_op->op = snap_op;
-        missed_snap_op->status = snap_status;
-
-        /* Look for other entries for the same node and same snap */
-        cds_list_for_each_entry (missed_snapinfo, &priv->missed_snaps_list,
-                                 missed_snaps) {
-                snprintf (node_snap_info, sizeof(node_snap_info),
-                          "%s:%s", missed_snapinfo->node_uuid,
-                          missed_snapinfo->snap_uuid);
-                if (!strcmp (node_snap_info, missed_info)) {
-                        /* Found missed snapshot info for *
-                         * the same node and same snap */
-                        match = _gf_true;
-                        break;
-                }
-        }
-
-        if (match == _gf_false) {
-                /* First snap op missed for the brick */
-                ret = glusterd_missed_snapinfo_new (&missed_snapinfo);
-                if (ret) {
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "Failed to create missed snapinfo");
-                        goto out;
-                }
-                free_missed_snap_info = _gf_true;
-                buf = strtok_r (missed_info, ":", &save_ptr);
-                if (!buf) {
-                        ret = -1;
-                        goto out;
-                }
-                missed_snapinfo->node_uuid = gf_strdup(buf);
-                if (!missed_snapinfo->node_uuid) {
-                        ret = -1;
-                        goto out;
-                }
-
-                buf = strtok_r (NULL, ":", &save_ptr);
-                if (!buf) {
-                        ret = -1;
-                        goto out;
-                }
-                missed_snapinfo->snap_uuid = gf_strdup(buf);
-                if (!missed_snapinfo->snap_uuid) {
-                        ret = -1;
-                        goto out;
-                }
-
-                cds_list_add_tail (&missed_snap_op->snap_ops_list,
-                                   &missed_snapinfo->snap_ops);
-                cds_list_add_tail (&missed_snapinfo->missed_snaps,
-                                   &priv->missed_snaps_list);
+    char *buf = NULL;
+    char *save_ptr = NULL;
+    char node_snap_info[PATH_MAX] = "";
+    int32_t ret = -1;
+    glusterd_missed_snap_info *missed_snapinfo = NULL;
+    glusterd_snap_op_t *missed_snap_op = NULL;
+    glusterd_conf_t *priv = NULL;
+    gf_boolean_t match = _gf_false;
+    gf_boolean_t free_missed_snap_info = _gf_false;
+    xlator_t *this = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+    GF_ASSERT(missed_info);
+    GF_ASSERT(snap_vol_id);
+    GF_ASSERT(brick_path);
+
+    priv = this->private;
+    GF_ASSERT(priv);
+
+    /* Create the snap_op object consisting of the *
+     * snap id and the op */
+    ret = glusterd_missed_snap_op_new(&missed_snap_op);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_MISSED_SNAP_CREATE_FAIL,
+               "Failed to create new missed snap object.");
+        ret = -1;
+        goto out;
+    }
+
+    missed_snap_op->snap_vol_id = gf_strdup(snap_vol_id);
+    if (!missed_snap_op->snap_vol_id) {
+        ret = -1;
+        goto out;
+    }
+    missed_snap_op->brick_path = gf_strdup(brick_path);
+    if (!missed_snap_op->brick_path) {
+        ret = -1;
+        goto out;
+    }
+    missed_snap_op->brick_num = brick_num;
+    missed_snap_op->op = snap_op;
+    missed_snap_op->status = snap_status;
+
+    /* Look for other entries for the same node and same snap */
+    cds_list_for_each_entry(missed_snapinfo, &priv->missed_snaps_list,
+                            missed_snaps)
+    {
+        snprintf(node_snap_info, sizeof(node_snap_info), "%s:%s",
+                 missed_snapinfo->node_uuid, missed_snapinfo->snap_uuid);
+        if (!strcmp(node_snap_info, missed_info)) {
+            /* Found missed snapshot info for *
+             * the same node and same snap */
+            match = _gf_true;
+            break;
+        }
+    }
+
+    if (match == _gf_false) {
+        /* First snap op missed for the brick */
+        ret = glusterd_missed_snapinfo_new(&missed_snapinfo);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_MISSED_SNAP_CREATE_FAIL,
+                   "Failed to create missed snapinfo");
+            goto out;
+        }
+        free_missed_snap_info = _gf_true;
+        buf = strtok_r(missed_info, ":", &save_ptr);
+        if (!buf) {
+            ret = -1;
+            goto out;
+        }
+        missed_snapinfo->node_uuid = gf_strdup(buf);
+        if (!missed_snapinfo->node_uuid) {
+            ret = -1;
+            goto out;
+        }
+
+        buf = strtok_r(NULL, ":", &save_ptr);
+        if (!buf) {
+            ret = -1;
+            goto out;
+        }
+        missed_snapinfo->snap_uuid = gf_strdup(buf);
+        if (!missed_snapinfo->snap_uuid) {
+            ret = -1;
+            goto out;
+        }
+
+        cds_list_add_tail(&missed_snap_op->snap_ops_list,
+                          &missed_snapinfo->snap_ops);
+        cds_list_add_tail(&missed_snapinfo->missed_snaps,
+                          &priv->missed_snaps_list);
 
-                ret = 0;
-                goto out;
-        } else {
-                ret = glusterd_update_missed_snap_entry (missed_snapinfo,
-                                                         missed_snap_op);
-                if (ret) {
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "Failed to update existing missed snap entry.");
-                        goto out;
-                }
+        ret = 0;
+        goto out;
+    } else {
+        ret = glusterd_update_missed_snap_entry(missed_snapinfo,
+                                                missed_snap_op);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_MISSED_SNAP_CREATE_FAIL,
+                   "Failed to update existing missed snap entry.");
+            goto out;
         }
+    }
 
 out:
-        if (ret) {
-                glusterd_free_snap_op (missed_snap_op);
+    if (ret) {
+        glusterd_free_snap_op(missed_snap_op);
 
-                if (missed_snapinfo &&
-                    (free_missed_snap_info == _gf_true))
-                        glusterd_free_missed_snapinfo (missed_snapinfo);
-        }
+        if (missed_snapinfo && (free_missed_snap_info == _gf_true))
+            glusterd_free_missed_snapinfo(missed_snapinfo);
+    }
 
-        gf_log (this->name, GF_LOG_TRACE, "Returning %d", ret);
-        return ret;
+    gf_msg_trace(this->name, 0, "Returning %d", ret);
+    return ret;
 }
 
 /* Add  missing snap entries to the in-memory conf->missed_snap_list */
 int32_t
-glusterd_add_missed_snaps_to_list (dict_t *dict, int32_t missed_snap_count)
+glusterd_add_missed_snaps_to_list(dict_t *dict, int32_t missed_snap_count)
 {
-        char                          *buf                         = NULL;
-        char                          *tmp                         = NULL;
-        char                          *save_ptr                    = NULL;
-        char                          *nodeid                      = NULL;
-        char                          *snap_uuid                   = NULL;
-        char                          *snap_vol_id                 = NULL;
-        char                          *brick_path                  = NULL;
-        char                           missed_info[PATH_MAX]       = "";
-        char                           name_buf[PATH_MAX]          = "";
-        int32_t                        i                           = -1;
-        int32_t                        ret                         = -1;
-        int32_t                        brick_num                   = -1;
-        int32_t                        snap_op                     = -1;
-        int32_t                        snap_status                 = -1;
-        glusterd_conf_t               *priv                        = NULL;
-        xlator_t                      *this                        = NULL;
-
-        this = THIS;
-        GF_ASSERT(this);
-        GF_ASSERT(dict);
-
-        priv = this->private;
-        GF_ASSERT (priv);
-
-        /* We can update the missed_snaps_list without acquiring *
-         * any additional locks as big lock will be held.        */
-        for (i = 0; i < missed_snap_count; i++) {
-                snprintf (name_buf, sizeof(name_buf), "missed_snaps_%d",
-                          i);
-                ret = dict_get_str (dict, name_buf, &buf);
-                if (ret) {
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "Unable to fetch %s", name_buf);
-                        goto out;
-                }
-
-                gf_log (this->name, GF_LOG_DEBUG, "missed_snap_entry = %s",
-                        buf);
-
-                /* Need to make a duplicate string coz the same dictionary *
-                 * is resent to the non-originator nodes */
-                tmp = gf_strdup (buf);
-                if (!tmp) {
-                        ret = -1;
-                        goto out;
-                }
+    char *buf = NULL;
+    char *tmp = NULL;
+    char *save_ptr = NULL;
+    char *nodeid = NULL;
+    char *snap_uuid = NULL;
+    char *snap_vol_id = NULL;
+    char *brick_path = NULL;
+    char missed_info[PATH_MAX] = "";
+    char key[64] = "";
+    int keylen;
+    int32_t i = -1;
+    int32_t ret = -1;
+    int32_t brick_num = -1;
+    int32_t snap_op = -1;
+    int32_t snap_status = -1;
+    glusterd_conf_t *priv = NULL;
+    xlator_t *this = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+    GF_ASSERT(dict);
+
+    priv = this->private;
+    GF_ASSERT(priv);
+
+    /* We can update the missed_snaps_list without acquiring *
+     * any additional locks as big lock will be held.        */
+    for (i = 0; i < missed_snap_count; i++) {
+        keylen = snprintf(key, sizeof(key), "missed_snaps_%d", i);
+        ret = dict_get_strn(dict, key, keylen, &buf);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+                   "Unable to fetch %s", key);
+            goto out;
+        }
+
+        gf_msg_debug(this->name, 0, "missed_snap_entry = %s", buf);
+
+        /* Need to make a duplicate string coz the same dictionary *
+         * is resent to the non-originator nodes */
+        tmp = gf_strdup(buf);
+        if (!tmp) {
+            ret = -1;
+            goto out;
+        }
+
+        /* Fetch the node-id, snap-id, brick_num,
+         * brick_path, snap_op and snap status
+         */
+        nodeid = strtok_r(tmp, ":", &save_ptr);
+        snap_uuid = strtok_r(NULL, "=", &save_ptr);
+        snap_vol_id = strtok_r(NULL, ":", &save_ptr);
+        brick_num = atoi(strtok_r(NULL, ":", &save_ptr));
+        brick_path = strtok_r(NULL, ":", &save_ptr);
+        snap_op = atoi(strtok_r(NULL, ":", &save_ptr));
+        snap_status = atoi(strtok_r(NULL, ":", &save_ptr));
 
-                /* Fetch the node-id, snap-id, brick_num,
-                 * brick_path, snap_op and snap status
-                 */
-                nodeid = strtok_r (tmp, ":", &save_ptr);
-                snap_uuid = strtok_r (NULL, "=", &save_ptr);
-                snap_vol_id = strtok_r (NULL, ":", &save_ptr);
-                brick_num = atoi(strtok_r (NULL, ":", &save_ptr));
-                brick_path = strtok_r (NULL, ":", &save_ptr);
-                snap_op = atoi(strtok_r (NULL, ":", &save_ptr));
-                snap_status = atoi(strtok_r (NULL, ":", &save_ptr));
-
-                if (!nodeid || !snap_uuid || !brick_path ||
-                    !snap_vol_id || brick_num < 1 || snap_op < 1 ||
-                    snap_status < 1) {
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "Invalid missed_snap_entry");
-                        ret = -1;
-                        goto out;
-                }
+        if (!nodeid || !snap_uuid || !brick_path || !snap_vol_id ||
+            brick_num < 1 || snap_op < 1 || snap_status < 1) {
+            gf_msg(this->name, GF_LOG_ERROR, 0,
+                   GD_MSG_INVALID_MISSED_SNAP_ENTRY,
+                   "Invalid missed_snap_entry");
+            ret = -1;
+            goto out;
+        }
 
-                snprintf (missed_info, sizeof(missed_info), "%s:%s",
-                          nodeid, snap_uuid);
+        snprintf(missed_info, sizeof(missed_info), "%s:%s", nodeid, snap_uuid);
 
-                ret = glusterd_add_new_entry_to_list (missed_info,
-                                                      snap_vol_id,
-                                                      brick_num,
-                                                      brick_path,
-                                                      snap_op,
-                                                      snap_status);
-                if (ret) {
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "Failed to store missed snaps_list");
-                        goto out;
-                }
-
-                GF_FREE (tmp);
-                tmp = NULL;
+        ret = glusterd_add_new_entry_to_list(missed_info, snap_vol_id,
+                                             brick_num, brick_path, snap_op,
+                                             snap_status);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, 0,
+                   GD_MSG_MISSED_SNAP_LIST_STORE_FAIL,
+                   "Failed to store missed snaps_list");
+            goto out;
         }
 
-        ret = 0;
+        GF_FREE(tmp);
+        tmp = NULL;
+    }
+
+    ret = 0;
 out:
-        if (tmp)
-                GF_FREE (tmp);
+    if (tmp)
+        GF_FREE(tmp);
 
-        gf_log (this->name, GF_LOG_TRACE, "Returning %d", ret);
-        return ret;
+    gf_msg_trace(this->name, 0, "Returning %d", ret);
+    return ret;
 }
 
 /* This function will restore origin volume to it's snap.
@@ -8770,263 +9846,242 @@ out:
  * @return 0 on success and negative value on error
  */
 int
-gd_restore_snap_volume (dict_t *dict, dict_t *rsp_dict,
-                        glusterd_volinfo_t *orig_vol,
-                        glusterd_volinfo_t *snap_vol,
-                        int32_t volcount)
+gd_restore_snap_volume(dict_t *dict, dict_t *rsp_dict,
+                       glusterd_volinfo_t *orig_vol,
+                       glusterd_volinfo_t *snap_vol, int32_t volcount)
 {
-        int                     ret             = -1;
-        glusterd_volinfo_t      *new_volinfo    = NULL;
-        glusterd_snap_t         *snap           = NULL;
-        xlator_t                *this           = NULL;
-        glusterd_conf_t         *conf           = NULL;
-        glusterd_volinfo_t      *temp_volinfo   = NULL;
-        glusterd_volinfo_t      *voliter        = NULL;
-        gf_boolean_t             conf_present   = _gf_false;
-
-        this = THIS;
-        GF_ASSERT (this);
-        GF_ASSERT (dict);
-        GF_ASSERT (rsp_dict);
-        conf = this->private;
-        GF_ASSERT (conf);
-
-        GF_VALIDATE_OR_GOTO (this->name, orig_vol, out);
-        GF_VALIDATE_OR_GOTO (this->name, snap_vol, out);
-        snap = snap_vol->snapshot;
-        GF_VALIDATE_OR_GOTO (this->name, snap, out);
-
-        /* Set the status to under restore so that if the
-         * the node goes down during restore and comes back
-         * the state of the volume can be reverted correctly
-         */
-        snap->snap_status = GD_SNAP_STATUS_UNDER_RESTORE;
-
-        /* We need to save this in disk so that if node goes
-         * down the status is in updated state.
-         */
-        ret = glusterd_store_snap (snap);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "Could not store snap "
-                        "object for %s snap of %s volume", snap_vol->volname,
-                        snap_vol->parent_volname);
-                goto out;
-        }
-
-        /* Snap volume must be stoped before performing the
-         * restore operation.
-         */
-        ret = glusterd_stop_volume (snap_vol);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "Failed to stop "
-                        "snap volume");
-                goto out;
-        }
-
-        /* Create a new volinfo for the restored volume */
-        ret = glusterd_volinfo_dup (snap_vol, &new_volinfo, _gf_true);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "Failed to create volinfo");
-                goto out;
-        }
-
-        /* Following entries need to be derived from origin volume. */
-        strcpy (new_volinfo->volname, orig_vol->volname);
-        gf_uuid_copy (new_volinfo->volume_id, orig_vol->volume_id);
-        new_volinfo->snap_count = orig_vol->snap_count;
-        new_volinfo->snap_max_hard_limit = orig_vol->snap_max_hard_limit;
-        gf_uuid_copy (new_volinfo->restored_from_snap,
-                   snap_vol->snapshot->snap_id);
-
-        /* Use the same version as the original version */
-        new_volinfo->version = orig_vol->version;
-
-        /* Initialize the snapd service */
-        ret = glusterd_snapdsvc_init (new_volinfo);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "Failed to initialize snapd "
-                        "service for volume %s", orig_vol->volname);
-                goto out;
-        }
-
-        /* Copy the snap vol info to the new_volinfo.*/
-        ret = glusterd_snap_volinfo_restore (dict, rsp_dict, new_volinfo,
-                                             snap_vol, volcount);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "Failed to restore snap");
-                goto out;
-        }
-
-        /* In case a new node is added to the peer, after a snapshot was
-         * taken, the geo-rep files are not synced to that node. This
-         * leads to the failure of snapshot restore. Hence, ignoring the
-         * missing geo-rep files in the new node, and proceeding with
-         * snapshot restore. Once the restore is successful, the missing
-         * geo-rep files can be generated with "gluster volume geo-rep
-         * <master-vol> <slave-vol> create push-pem force"
-         */
-        ret = glusterd_restore_geo_rep_files (snap_vol);
-        if (ret) {
-                gf_log (this->name, GF_LOG_WARNING, "Failed to restore "
-                        "geo-rep files for snap %s",
-                        snap_vol->snapshot->snapname);
-        }
-
-        ret = glusterd_copy_quota_files (snap_vol, orig_vol, &conf_present);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "Failed to restore "
-                        "quota files for snap %s",
-                        snap_vol->snapshot->snapname);
-                goto out;
-        }
-
-        if (conf_present) {
-                /* TO calculate checksum of quota conf we need to send
-                 * second argument as _gf_true
-                 */
-                ret = glusterd_compute_cksum (new_volinfo, _gf_true);
-                if (ret) {
-                        gf_log (this->name, GF_LOG_ERROR, "Failed to compute "
-                                "checksum for quota conf file");
-                        goto out;
-                }
-
-                ret = glusterd_store_save_quota_version_and_cksum (new_volinfo);
-                if (ret) {
-                        gf_log (this->name, GF_LOG_ERROR, "Failed to "
-                                "store quota version and cksum");
-                        goto out;
-                }
-        }
-
-        /* New volinfo always shows the status as created. Therefore
-         * set the status to the original volume's status. */
-        glusterd_set_volume_status (new_volinfo, orig_vol->status);
-
-        cds_list_add_tail (&new_volinfo->vol_list, &conf->volumes);
-
-        ret = glusterd_store_volinfo (new_volinfo,
-                                      GLUSTERD_VOLINFO_VER_AC_INCREMENT);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "Failed to store volinfo");
-                goto out;
-        }
-
-        ret = 0;
+    int ret = -1;
+    glusterd_volinfo_t *new_volinfo = NULL;
+    glusterd_snap_t *snap = NULL;
+    xlator_t *this = NULL;
+    glusterd_conf_t *conf = NULL;
+    glusterd_volinfo_t *temp_volinfo = NULL;
+    glusterd_volinfo_t *voliter = NULL;
+    gf_boolean_t conf_present = _gf_false;
+
+    this = THIS;
+    GF_ASSERT(this);
+    GF_ASSERT(dict);
+    GF_ASSERT(rsp_dict);
+    conf = this->private;
+    GF_ASSERT(conf);
+
+    GF_VALIDATE_OR_GOTO(this->name, orig_vol, out);
+    GF_VALIDATE_OR_GOTO(this->name, snap_vol, out);
+    snap = snap_vol->snapshot;
+    GF_VALIDATE_OR_GOTO(this->name, snap, out);
+
+    /* Set the status to under restore so that if the
+     * the node goes down during restore and comes back
+     * the state of the volume can be reverted correctly
+     */
+    snap->snap_status = GD_SNAP_STATUS_UNDER_RESTORE;
+
+    /* We need to save this in disk so that if node goes
+     * down the status is in updated state.
+     */
+    ret = glusterd_store_snap(snap);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_FILE_OP_FAILED,
+               "Could not store snap "
+               "object for %s snap of %s volume",
+               snap_vol->volname, snap_vol->parent_volname);
+        goto out;
+    }
+
+    /* Snap volume must be stopped before performing the
+     * restore operation.
+     */
+    ret = glusterd_stop_volume(snap_vol);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_VOL_STOP_FAILED,
+               "Failed to stop "
+               "snap volume");
+        goto out;
+    }
+
+    /* Create a new volinfo for the restored volume */
+    ret = glusterd_volinfo_dup(snap_vol, &new_volinfo, _gf_true);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_VOL_OP_FAILED,
+               "Failed to create volinfo");
+        goto out;
+    }
+
+    /* Following entries need to be derived from origin volume. */
+    gf_strncpy(new_volinfo->volname, orig_vol->volname,
+               sizeof(new_volinfo->volname));
+    gf_uuid_copy(new_volinfo->volume_id, orig_vol->volume_id);
+    new_volinfo->snap_count = orig_vol->snap_count;
+    gf_uuid_copy(new_volinfo->restored_from_snap, snap_vol->snapshot->snap_id);
+
+    /* Use the same version as the original version */
+    new_volinfo->version = orig_vol->version;
+
+    /* Copy the snap vol info to the new_volinfo.*/
+    ret = glusterd_snap_volinfo_restore(dict, rsp_dict, new_volinfo, snap_vol,
+                                        volcount);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_SNAP_RESTORE_FAIL,
+               "Failed to restore snap");
+        goto out;
+    }
+
+    /* In case a new node is added to the peer, after a snapshot was
+     * taken, the geo-rep files are not synced to that node. This
+     * leads to the failure of snapshot restore. Hence, ignoring the
+     * missing geo-rep files in the new node, and proceeding with
+     * snapshot restore. Once the restore is successful, the missing
+     * geo-rep files can be generated with "gluster volume geo-rep
+     * <master-vol> <slave-vol> create push-pem force"
+     */
+    ret = glusterd_restore_geo_rep_files(snap_vol);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_WARNING, 0, GD_MSG_SNAP_RESTORE_FAIL,
+               "Failed to restore "
+               "geo-rep files for snap %s",
+               snap_vol->snapshot->snapname);
+    }
+
+    /* Need not save cksum, as we will copy cksum file in *
+     * this function                                           *
+     */
+    ret = glusterd_copy_quota_files(snap_vol, orig_vol, &conf_present);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_SNAP_RESTORE_FAIL,
+               "Failed to restore "
+               "quota files for snap %s",
+               snap_vol->snapshot->snapname);
+        goto out;
+    }
+
+    /* New volinfo always shows the status as created. Therefore
+     * set the status to the original volume's status. */
+    glusterd_set_volume_status(new_volinfo, orig_vol->status);
+
+    cds_list_add_tail(&new_volinfo->vol_list, &conf->volumes);
+
+    ret = glusterd_store_volinfo(new_volinfo,
+                                 GLUSTERD_VOLINFO_VER_AC_INCREMENT);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_VOL_OP_FAILED,
+               "Failed to store volinfo");
+        goto out;
+    }
+
+    ret = 0;
 out:
-        if (ret) {
-                /* In case of any failure we should free new_volinfo. Doing
-                 * this will also remove the entry we added in conf->volumes
-                 * if it was added there.
-                 */
-                if (new_volinfo)
-                        (void)glusterd_volinfo_delete (new_volinfo);
-        } else {
-                cds_list_for_each_entry_safe (voliter, temp_volinfo,
-                                              &orig_vol->snap_volumes,
-                                              snapvol_list) {
-                        cds_list_add_tail (&voliter->snapvol_list,
-                                           &new_volinfo->snap_volumes);
-                }
+    if (ret) {
+        /* In case of any failure we should free new_volinfo. Doing
+         * this will also remove the entry we added in conf->volumes
+         * if it was added there.
+         */
+        if (new_volinfo)
+            (void)glusterd_volinfo_delete(new_volinfo);
+    } else {
+        cds_list_for_each_entry_safe(voliter, temp_volinfo,
+                                     &orig_vol->snap_volumes, snapvol_list)
+        {
+            cds_list_add_tail(&voliter->snapvol_list,
+                              &new_volinfo->snap_volumes);
         }
+    }
 
-        return ret;
+    return ret;
 }
 
-
-
 int
-glusterd_snapshot_get_volnames_uuids (dict_t *dict,
-                                      char *volname,
-                                      gf_getsnap_name_uuid_rsp  *snap_info_rsp)
+glusterd_snapshot_get_volnames_uuids(dict_t *dict, char *volname,
+                                     gf_getsnap_name_uuid_rsp *snap_info_rsp)
 {
-        int                 ret             = -1;
-        int                 snapcount       = 0;
-        char                key[PATH_MAX]   = {0,};
-        glusterd_volinfo_t *snap_vol        = NULL;
-        glusterd_volinfo_t *volinfo         = NULL;
-        glusterd_volinfo_t *tmp_vol         = NULL;
-        xlator_t           *this            = NULL;
-        int                 op_errno        = 0;
-
-        this = THIS;
-        GF_ASSERT (this);
-        GF_ASSERT (volname);
-        GF_VALIDATE_OR_GOTO_WITH_ERROR (this->name, dict, out,
-                                        op_errno, EINVAL);
-        GF_VALIDATE_OR_GOTO_WITH_ERROR (this->name, volname, out,
-                                        op_errno, EINVAL);
-        GF_VALIDATE_OR_GOTO_WITH_ERROR (this->name, snap_info_rsp, out,
-                                        op_errno, EINVAL);
-
-        ret = glusterd_volinfo_find (volname, &volinfo);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "Failed to get volinfo of volume %s",
-                        volname);
-                op_errno = EINVAL;
-                goto out;
-        }
-
-        cds_list_for_each_entry_safe (snap_vol, tmp_vol, &volinfo->snap_volumes,
-                                      snapvol_list) {
-
-                if (GLUSTERD_STATUS_STARTED != snap_vol->status)
-                        continue;
-
-                snapcount++;
-
-                /* Set Snap Name */
-                snprintf (key, sizeof (key), "snapname.%d", snapcount);
-                ret = dict_set_dynstr_with_alloc (dict, key,
-                                           snap_vol->snapshot->snapname);
-                if (ret) {
-                        gf_log (this->name, GF_LOG_ERROR, "Failed to set "
-                                "snap name in dictionary");
-                        goto out;
-                }
-
-                /* Set Snap ID */
-                snprintf (key, sizeof (key), "snap-id.%d", snapcount);
-                ret = dict_set_dynstr_with_alloc (dict, key,
-                                     uuid_utoa(snap_vol->snapshot->snap_id));
-                if (ret) {
-                        gf_log (this->name, GF_LOG_ERROR, "Failed to set "
-                                "snap id in dictionary");
-                        goto out;
-                }
-
-                /* Snap Volname which is used to activate the snap vol */
-                snprintf (key, sizeof (key), "snap-volname.%d", snapcount);
-                ret = dict_set_dynstr_with_alloc (dict, key, snap_vol->volname);
-                if (ret) {
-                        gf_log (this->name, GF_LOG_ERROR, "Failed to set "
-                                "snap id in dictionary");
-                        goto out;
-                }
-        }
-
-        ret = dict_set_int32 (dict, "snap-count", snapcount);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "Failed to set snapcount");
-                op_errno = -ret;
-                goto out;
-        }
-
-        ret = dict_allocate_and_serialize (dict, &snap_info_rsp->dict.dict_val,
-                                           &snap_info_rsp->dict.dict_len);
-        if (ret) {
-                op_errno = -ret;
-                ret = -1;
-                goto out;
-        }
-
-        ret = 0;
+    int ret = -1;
+    int snapcount = 0;
+    char key[32] = "";
+    glusterd_volinfo_t *snap_vol = NULL;
+    glusterd_volinfo_t *volinfo = NULL;
+    glusterd_volinfo_t *tmp_vol = NULL;
+    xlator_t *this = NULL;
+    int op_errno = 0;
+
+    this = THIS;
+    GF_ASSERT(this);
+    GF_ASSERT(volname);
+    GF_VALIDATE_OR_GOTO_WITH_ERROR(this->name, dict, out, op_errno, EINVAL);
+    GF_VALIDATE_OR_GOTO_WITH_ERROR(this->name, volname, out, op_errno, EINVAL);
+    GF_VALIDATE_OR_GOTO_WITH_ERROR(this->name, snap_info_rsp, out, op_errno,
+                                   EINVAL);
+
+    ret = glusterd_volinfo_find(volname, &volinfo);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, EINVAL, GD_MSG_VOL_NOT_FOUND,
+               "Failed to get volinfo of volume %s", volname);
+        op_errno = EINVAL;
+        goto out;
+    }
+
+    cds_list_for_each_entry_safe(snap_vol, tmp_vol, &volinfo->snap_volumes,
+                                 snapvol_list)
+    {
+        if (GLUSTERD_STATUS_STARTED != snap_vol->status)
+            continue;
+
+        snapcount++;
+
+        /* Set Snap Name */
+        snprintf(key, sizeof(key), "snapname.%d", snapcount);
+        ret = dict_set_dynstr_with_alloc(dict, key,
+                                         snap_vol->snapshot->snapname);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+                   "Failed to set "
+                   "snap name in dictionary");
+            goto out;
+        }
+
+        /* Set Snap ID */
+        snprintf(key, sizeof(key), "snap-id.%d", snapcount);
+        ret = dict_set_dynstr_with_alloc(
+            dict, key, uuid_utoa(snap_vol->snapshot->snap_id));
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+                   "Failed to set "
+                   "snap id in dictionary");
+            goto out;
+        }
+
+        /* Snap Volname which is used to activate the snap vol */
+        snprintf(key, sizeof(key), "snap-volname.%d", snapcount);
+        ret = dict_set_dynstr_with_alloc(dict, key, snap_vol->volname);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+                   "Failed to set "
+                   "snap id in dictionary");
+            goto out;
+        }
+    }
+
+    ret = dict_set_int32n(dict, "snap-count", SLEN("snap-count"), snapcount);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+               "Failed to set snapcount");
+        op_errno = -ret;
+        goto out;
+    }
+
+    ret = dict_allocate_and_serialize(dict, &snap_info_rsp->dict.dict_val,
+                                      &snap_info_rsp->dict.dict_len);
+    if (ret) {
+        op_errno = -ret;
+        ret = -1;
+        goto out;
+    }
+
+    ret = 0;
 
 out:
-        snap_info_rsp->op_ret = ret;
-        snap_info_rsp->op_errno = op_errno;
-        snap_info_rsp->op_errstr = "";
+    snap_info_rsp->op_ret = ret;
+    snap_info_rsp->op_errno = op_errno;
+    snap_info_rsp->op_errstr = "";
 
-        return ret;
+    return ret;
 }
diff --git a/xlators/mgmt/glusterd/src/glusterd-statedump.c b/xlators/mgmt/glusterd/src/glusterd-statedump.c
index 4792d45571f..225d10cc546 100644
--- a/xlators/mgmt/glusterd/src/glusterd-statedump.c
+++ b/xlators/mgmt/glusterd/src/glusterd-statedump.c
@@ -8,237 +8,236 @@
    cases as published by the Free Software Foundation.
 */
 
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
-
-#include "statedump.h"
+#include <glusterfs/statedump.h>
 #include "glusterd.h"
 #include "glusterd-shd-svc.h"
 #include "glusterd-quotad-svc.h"
-#include "glusterd-nfs-svc.h"
 #include "glusterd-locks.h"
+#include "glusterd-messages.h"
 
 static void
-glusterd_dump_peer (glusterd_peerinfo_t *peerinfo, char *input_key, int index,
-                    gf_boolean_t xpeers)
+glusterd_dump_peer(glusterd_peerinfo_t *peerinfo, char *input_key, int index,
+                   gf_boolean_t xpeers)
 {
-        char   subkey[50]               = {0,};
-        char   key[GF_DUMP_MAX_BUF_LEN] = {0,};
+    char subkey[GF_DUMP_MAX_BUF_LEN + 11] = "";
+    char key[GF_DUMP_MAX_BUF_LEN] = "";
 
-        strncpy (key, input_key, (GF_DUMP_MAX_BUF_LEN - 1));
+    strncpy(key, input_key, sizeof(key) - 1);
 
-        snprintf (subkey, sizeof (subkey), "%s%d", key, index);
+    snprintf(subkey, sizeof(subkey), "%s%d", key, index);
 
-        gf_proc_dump_build_key (key, subkey, "uuid");
-        gf_proc_dump_write (key, "%s",
-                            uuid_utoa (peerinfo->uuid));
+    gf_proc_dump_build_key(key, subkey, "uuid");
+    gf_proc_dump_write(key, "%s", uuid_utoa(peerinfo->uuid));
 
-        gf_proc_dump_build_key (key, subkey, "hostname");
-        gf_proc_dump_write (key, "%s", peerinfo->hostname);
+    gf_proc_dump_build_key(key, subkey, "hostname");
+    gf_proc_dump_write(key, "%s", peerinfo->hostname);
 
-        gf_proc_dump_build_key (key, subkey, "port");
-        gf_proc_dump_write (key, "%d", peerinfo->port);
+    gf_proc_dump_build_key(key, subkey, "port");
+    gf_proc_dump_write(key, "%d", peerinfo->port);
 
-        gf_proc_dump_build_key (key, subkey, "state");
-        gf_proc_dump_write (key, "%d", peerinfo->state.state);
+    gf_proc_dump_build_key(key, subkey, "state");
+    gf_proc_dump_write(key, "%d", peerinfo->state.state);
 
-        gf_proc_dump_build_key (key, subkey, "quorum-action");
-        gf_proc_dump_write (key, "%d", peerinfo->quorum_action);
+    gf_proc_dump_build_key(key, subkey, "quorum-action");
+    gf_proc_dump_write(key, "%d", peerinfo->quorum_action);
 
-        gf_proc_dump_build_key (key, subkey, "quorum-contrib");
-        gf_proc_dump_write (key, "%d",
-                            peerinfo->quorum_contrib);
+    gf_proc_dump_build_key(key, subkey, "quorum-contrib");
+    gf_proc_dump_write(key, "%d", peerinfo->quorum_contrib);
 
-        gf_proc_dump_build_key (key, subkey, "detaching");
-        gf_proc_dump_write (key, "%d", peerinfo->detaching);
-
-        gf_proc_dump_build_key (key, subkey, "locked");
-        gf_proc_dump_write (key, "%d", peerinfo->locked);
+    gf_proc_dump_build_key(key, subkey, "detaching");
+    gf_proc_dump_write(key, "%d", peerinfo->detaching);
 
+    gf_proc_dump_build_key(key, subkey, "locked");
+    gf_proc_dump_write(key, "%d", peerinfo->locked);
 }
 
-
 static void
-glusterd_dump_peer_rpcstat (glusterd_peerinfo_t *peerinfo, char *input_key,
-                            int index)
+glusterd_dump_peer_rpcstat(glusterd_peerinfo_t *peerinfo, char *input_key,
+                           int index)
 {
-        rpc_clnt_connection_t *conn                                = NULL;
-        int                    ret                                 = -1;
-        rpc_clnt_t            *rpc                                 = NULL;
-        char                   rpcsvc_peername[RPCSVC_PEER_STRLEN] = {0,};
-        char                   subkey[50]                          = {0,};
-        char                   key[GF_DUMP_MAX_BUF_LEN]            = {0,};
-
-        strncpy (key, input_key, (GF_DUMP_MAX_BUF_LEN - 1));
-
-        /* Dump the rpc connection statistics */
-        rpc = peerinfo->rpc;
-        if (rpc) {
-                conn = &rpc->conn;
-                snprintf (subkey, sizeof (subkey), "%s%d", key, index);
-                ret = rpcsvc_transport_peername (conn->trans,
-                                                 (char *)&rpcsvc_peername,
-                                                 sizeof (rpcsvc_peername));
-                if (!ret) {
-                        gf_proc_dump_build_key (key, subkey, "rpc.peername");
-                        gf_proc_dump_write (key, "%s", rpcsvc_peername);
-                }
-                gf_proc_dump_build_key (key, subkey, "rpc.connected");
-                gf_proc_dump_write (key, "%d", conn->connected);
-
-                gf_proc_dump_build_key (key, subkey, "rpc.total-bytes-read");
-                gf_proc_dump_write (key, "%"PRIu64,
-                                    conn->trans->total_bytes_read);
-
-                gf_proc_dump_build_key (key, subkey, "rpc.total-bytes-written");
-                gf_proc_dump_write (key, "%"PRIu64,
-                                    conn->trans->total_bytes_write);
-
-                gf_proc_dump_build_key (key, subkey, "rpc.ping_msgs_sent");
-                gf_proc_dump_write (key, "%"PRIu64, conn->pingcnt);
-
-                gf_proc_dump_build_key (key, subkey, "rpc.msgs_sent");
-                gf_proc_dump_write (key, "%"PRIu64, conn->msgcnt);
+    rpc_clnt_connection_t *conn = NULL;
+    int ret = -1;
+    rpc_clnt_t *rpc = NULL;
+    char rpcsvc_peername[RPCSVC_PEER_STRLEN] = "";
+    char subkey[GF_DUMP_MAX_BUF_LEN + 11] = "";
+    char key[GF_DUMP_MAX_BUF_LEN] = "";
+
+    strncpy(key, input_key, sizeof(key) - 1);
+
+    /* Dump the rpc connection statistics */
+    rpc = peerinfo->rpc;
+    if (rpc) {
+        conn = &rpc->conn;
+        snprintf(subkey, sizeof(subkey), "%s%d", key, index);
+        ret = rpcsvc_transport_peername(conn->trans, (char *)&rpcsvc_peername,
+                                        sizeof(rpcsvc_peername));
+        if (!ret) {
+            gf_proc_dump_build_key(key, subkey, "rpc.peername");
+            gf_proc_dump_write(key, "%s", rpcsvc_peername);
         }
+        gf_proc_dump_build_key(key, subkey, "rpc.connected");
+        gf_proc_dump_write(key, "%d", conn->connected);
 
-}
+        gf_proc_dump_build_key(key, subkey, "rpc.total-bytes-read");
+        gf_proc_dump_write(key, "%" PRIu64, conn->trans->total_bytes_read);
+
+        gf_proc_dump_build_key(key, subkey, "rpc.total-bytes-written");
+        gf_proc_dump_write(key, "%" PRIu64, conn->trans->total_bytes_write);
 
+        gf_proc_dump_build_key(key, subkey, "rpc.ping_msgs_sent");
+        gf_proc_dump_write(key, "%" PRIu64, conn->pingcnt);
+
+        gf_proc_dump_build_key(key, subkey, "rpc.msgs_sent");
+        gf_proc_dump_write(key, "%" PRIu64, conn->msgcnt);
+    }
+}
 
 static void
-glusterd_dump_client_details (glusterd_conf_t *conf)
+glusterd_dump_client_details(glusterd_conf_t *conf)
 {
-        rpc_transport_t         *xprt                     = NULL;
-        char                     key[GF_DUMP_MAX_BUF_LEN] = {0,};
-        char                     subkey[50]               = {0,};
-        int                      index                    = 1;
-
-        pthread_mutex_lock (&conf->xprt_lock);
+    rpc_transport_t *xprt = NULL;
+    char key[GF_DUMP_MAX_BUF_LEN] = "";
+    char subkey[50] = "";
+    int index = 1;
+
+    pthread_mutex_lock(&conf->xprt_lock);
+    {
+        list_for_each_entry(xprt, &conf->xprt_list, list)
         {
-                list_for_each_entry (xprt, &conf->xprt_list, list) {
-                        snprintf (subkey, sizeof (subkey), "glusterd.client%d",
-                                  index);
-
-                        gf_proc_dump_build_key (key, subkey, "identifier");
-                        gf_proc_dump_write (key, "%s",
-                                            xprt->peerinfo.identifier);
-
-                        gf_proc_dump_build_key (key, subkey, "volname");
-                        gf_proc_dump_write (key, "%s",
-                                            xprt->peerinfo.volname);
-
-                        gf_proc_dump_build_key (key, subkey, "max-op-version");
-                        gf_proc_dump_write (key, "%u",
-                                            xprt->peerinfo.max_op_version);
-
-                        gf_proc_dump_build_key (key, subkey, "min-op-version");
-                        gf_proc_dump_write (key, "%u",
-                                            xprt->peerinfo.min_op_version);
-                        index++;
-                }
+            snprintf(subkey, sizeof(subkey), "glusterd.client%d", index);
+
+            gf_proc_dump_build_key(key, subkey, "identifier");
+            gf_proc_dump_write(key, "%s", xprt->peerinfo.identifier);
+
+            gf_proc_dump_build_key(key, subkey, "volname");
+            gf_proc_dump_write(key, "%s", xprt->peerinfo.volname);
+
+            gf_proc_dump_build_key(key, subkey, "max-op-version");
+            gf_proc_dump_write(key, "%u", xprt->peerinfo.max_op_version);
+
+            gf_proc_dump_build_key(key, subkey, "min-op-version");
+            gf_proc_dump_write(key, "%u", xprt->peerinfo.min_op_version);
+            index++;
         }
-        pthread_mutex_unlock (&conf->xprt_lock);
+    }
+    pthread_mutex_unlock(&conf->xprt_lock);
 }
 
-
 /* The following function is just for dumping mgmt_v3_lock dictionary, any other
  * dict passed to this API will not work */
 
 static void
-glusterd_dict_mgmt_v3_lock_statedump (dict_t *dict)
+glusterd_dict_mgmt_v3_lock_statedump(dict_t *dict)
 {
-        int          ret                       = 0;
-        int          dumplen                   = 0;
-        data_pair_t *trav                      = NULL;
-        char         key[GF_DUMP_MAX_BUF_LEN]  = {0,};
-        char         dump[64*1024]             = {0,};
-
-        if (!dict) {
-                gf_log_callingfn ("glusterd", GF_LOG_WARNING, "dict NULL");
-                goto out;
-        }
-        for (trav = dict->members_list; trav; trav = trav->next) {
-                if (strstr (trav->key, "debug.last-success-bt") != NULL) {
-                        ret = snprintf (&dump[dumplen], sizeof(dump) - dumplen,
-                                        "\n\t%s:%s", trav->key,
-                                        trav->value->data);
-                } else {
-                        ret = snprintf (&dump[dumplen], sizeof(dump) - dumplen,
-                                        "\n\t%s:%s", trav->key,
-                                        uuid_utoa (((glusterd_mgmt_v3_lock_obj *)
-                                         (trav->value->data))->lock_owner));
-                }
-                if ((ret == -1) || !ret)
-                        return;
-                dumplen += ret;
+    int ret = 0;
+    int dumplen = 0;
+    data_pair_t *trav = NULL;
+    char key[GF_DUMP_MAX_BUF_LEN] = "";
+    char dump[64 * 1024] = "";
+
+    if (!dict) {
+        gf_msg_callingfn("glusterd", GF_LOG_WARNING, EINVAL, GD_MSG_DICT_EMPTY,
+                         "dict NULL");
+        goto out;
+    }
+    for (trav = dict->members_list; trav; trav = trav->next) {
+        if (strstr(trav->key, "debug.last-success-bt") != NULL) {
+            ret = snprintf(&dump[dumplen], sizeof(dump) - dumplen, "\n\t%s:%s",
+                           trav->key, trav->value->data);
+        } else {
+            ret = snprintf(
+                &dump[dumplen], sizeof(dump) - dumplen, "\n\t%s:%s", trav->key,
+                uuid_utoa(((glusterd_mgmt_v3_lock_obj *)(trav->value->data))
+                              ->lock_owner));
         }
+        if ((ret == -1) || !ret)
+            return;
+        dumplen += ret;
+    }
 
-        if (dumplen) {
-                gf_proc_dump_build_key (key, "glusterd", "mgmt_v3_lock");
-                gf_proc_dump_write (key, "%s", dump);
-        }
+    if (dumplen) {
+        gf_proc_dump_build_key(key, "glusterd", "mgmt_v3_lock");
+        gf_proc_dump_write(key, "%s", dump);
+    }
 
 out:
-        return;
+    return;
 }
 
-
 int
-glusterd_dump_priv (xlator_t *this)
+glusterd_dump_priv(xlator_t *this)
 {
-        int                   index                    = 1;
-        glusterd_conf_t      *priv                     = NULL;
-        char                  key[GF_DUMP_MAX_BUF_LEN] = {0,};
-        char                  subkey[50]               = {0,};
-        glusterd_peerinfo_t  *peerinfo                 = NULL;
-        glusterd_volinfo_t   *volinfo                  = NULL;
+    glusterd_conf_t *priv = NULL;
+    char key[GF_DUMP_MAX_BUF_LEN] = "";
+    int port = 0;
+    struct pmap_registry *pmap = NULL;
 
-        GF_VALIDATE_OR_GOTO ("glusterd", this, out);
+    GF_VALIDATE_OR_GOTO("glusterd", this, out);
 
-        priv = this->private;
-        if (!priv)
-                return 0;
-
-        gf_proc_dump_build_key (key, "xlator.glusterd", "priv");
-        gf_proc_dump_add_section (key);
-
-        pthread_mutex_lock (&priv->mutex);
-        {
-                gf_proc_dump_build_key (key, "glusterd", "my-uuid");
-                gf_proc_dump_write (key, "%s", uuid_utoa (priv->uuid));
-
-                gf_proc_dump_build_key (key, "glusterd", "working-directory");
-                gf_proc_dump_write (key, "%s", priv->workdir);
-
-                gf_proc_dump_build_key (key, "glusterd", "max-op-version");
-                gf_proc_dump_write (key, "%d", GD_OP_VERSION_MAX);
+    priv = this->private;
+    if (!priv)
+        return 0;
 
-                gf_proc_dump_build_key (key, "glusterd", "min-op-version");
-                gf_proc_dump_write (key, "%d", GD_OP_VERSION_MIN);
+    gf_proc_dump_build_key(key, "xlator.glusterd", "priv");
+    gf_proc_dump_add_section("%s", key);
 
-                gf_proc_dump_build_key (key, "glusterd", "current-op-version");
-                gf_proc_dump_write (key, "%d", priv->op_version);
+    pthread_mutex_lock(&priv->mutex);
+    {
+        gf_proc_dump_build_key(key, "glusterd", "my-uuid");
+        gf_proc_dump_write(key, "%s", uuid_utoa(priv->uuid));
 
-                gf_proc_dump_build_key (key, "glusterd", "ping-timeout");
-                gf_proc_dump_write (key, "%d", priv->ping_timeout);
+        gf_proc_dump_build_key(key, "glusterd", "working-directory");
+        gf_proc_dump_write(key, "%s", priv->workdir);
 
-                gf_proc_dump_build_key (key, "glusterd", "shd.online");
-                gf_proc_dump_write (key, "%d", priv->shd_svc.online);
+        gf_proc_dump_build_key(key, "glusterd", "max-op-version");
+        gf_proc_dump_write(key, "%d", GD_OP_VERSION_MAX);
 
-                gf_proc_dump_build_key (key, "glusterd", "nfs.online");
-                gf_proc_dump_write (key, "%d", priv->nfs_svc.online);
+        gf_proc_dump_build_key(key, "glusterd", "min-op-version");
+        gf_proc_dump_write(key, "%d", GD_OP_VERSION_MIN);
 
-                gf_proc_dump_build_key (key, "glusterd", "quotad.online");
-                gf_proc_dump_write (key, "%d", priv->quotad_svc.online);
+        gf_proc_dump_build_key(key, "glusterd", "current-op-version");
+        gf_proc_dump_write(key, "%d", priv->op_version);
 
-                GLUSTERD_DUMP_PEERS (&priv->peers, uuid_list, _gf_false);
-                glusterd_dump_client_details (priv);
-                glusterd_dict_mgmt_v3_lock_statedump(priv->mgmt_v3_lock);
-                dict_dump_to_statedump (priv->opts, "options", "glusterd");
+        gf_proc_dump_build_key(key, "glusterd", "ping-timeout");
+        gf_proc_dump_write(key, "%d", priv->ping_timeout);
+#ifdef BUILD_GNFS
+        gf_proc_dump_build_key(key, "glusterd", "nfs.online");
+        gf_proc_dump_write(key, "%d", priv->nfs_svc.online);
+#endif
+        gf_proc_dump_build_key(key, "glusterd", "quotad.online");
+        gf_proc_dump_write(key, "%d", priv->quotad_svc.online);
+
+        gf_proc_dump_build_key(key, "glusterd", "bitd.online");
+        gf_proc_dump_write(key, "%d", priv->bitd_svc.online);
+
+        gf_proc_dump_build_key(key, "glusterd", "scrub.online");
+        gf_proc_dump_write(key, "%d", priv->scrub_svc.online);
+
+        /* Dump peer details */
+        GLUSTERD_DUMP_PEERS(&priv->peers, uuid_list, _gf_false);
+
+        /* Dump pmap data structure from base port to last alloc */
+        pmap = priv->pmap;
+        if (pmap) {
+            for (port = pmap->base_port; port <= pmap->last_alloc; port++) {
+                gf_proc_dump_build_key(key, "glusterd", "pmap_port");
+                gf_proc_dump_write(key, "%d", port);
+                gf_proc_dump_build_key(key, "glusterd", "pmap[%d].type", port);
+                gf_proc_dump_write(key, "%d", pmap->ports[port].type);
+                gf_proc_dump_build_key(key, "glusterd", "pmap[%d].brickname",
+                                       port);
+                gf_proc_dump_write(key, "%s", pmap->ports[port].brickname);
+            }
         }
-        pthread_mutex_unlock (&priv->mutex);
+        /* Dump client details */
+        glusterd_dump_client_details(priv);
+
+        /* Dump mgmt_v3_lock from the dictionary if any */
+        glusterd_dict_mgmt_v3_lock_statedump(priv->mgmt_v3_lock);
+        dict_dump_to_statedump(priv->opts, "options", "glusterd");
+    }
+    pthread_mutex_unlock(&priv->mutex);
 
 out:
-        return 0;
+    return 0;
 }
diff --git a/xlators/mgmt/glusterd/src/glusterd-statedump.h b/xlators/mgmt/glusterd/src/glusterd-statedump.h
index 9917560b542..b5ef1f48e82 100644
--- a/xlators/mgmt/glusterd/src/glusterd-statedump.h
+++ b/xlators/mgmt/glusterd/src/glusterd-statedump.h
@@ -11,13 +11,8 @@
 #ifndef _GLUSTERD_STATEDUMP_H_
 #define _GLUSTERD_STATEDUMP_H_
 
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
-
-#include "xlator.h"
+#include <glusterfs/xlator.h>
 
 int
-glusterd_dump_priv (xlator_t *this);
+glusterd_dump_priv(xlator_t *this);
 #endif
diff --git a/xlators/mgmt/glusterd/src/glusterd-store.c b/xlators/mgmt/glusterd/src/glusterd-store.c
index 0e581b198a9..d94dceb10b7 100644
--- a/xlators/mgmt/glusterd/src/glusterd-store.c
+++ b/xlators/mgmt/glusterd/src/glusterd-store.c
@@ -8,39 +8,35 @@
    cases as published by the Free Software Foundation.
 */
 
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
-
 #include "glusterd-op-sm.h"
 #include <inttypes.h>
 
-
-#include "globals.h"
-#include "glusterfs.h"
-#include "compat.h"
-#include "dict.h"
+#include <glusterfs/glusterfs.h>
+#include <glusterfs/compat.h>
+#include <glusterfs/dict.h>
 #include "protocol-common.h"
-#include "xlator.h"
-#include "logging.h"
-#include "timer.h"
-#include "defaults.h"
-#include "compat.h"
-#include "compat-errno.h"
-#include "statedump.h"
+#include <glusterfs/xlator.h>
+#include <glusterfs/logging.h>
+#include <glusterfs/timer.h>
+#include <glusterfs/syscall.h>
+#include <glusterfs/defaults.h>
+#include <glusterfs/compat.h>
+#include <glusterfs/compat-errno.h>
+#include <glusterfs/statedump.h>
 #include "glusterd-mem-types.h"
 #include "glusterd.h"
 #include "glusterd-sm.h"
 #include "glusterd-op-sm.h"
 #include "glusterd-utils.h"
 #include "glusterd-hooks.h"
-#include "store.h"
+#include <glusterfs/store.h>
 #include "glusterd-store.h"
 #include "glusterd-snapshot-utils.h"
+#include "glusterd-messages.h"
 
 #include "rpc-clnt.h"
-#include "common-utils.h"
+#include <glusterfs/common-utils.h>
+#include <glusterfs/quota-common-utils.h>
 
 #include <sys/resource.h>
 #include <inttypes.h>
@@ -52,235 +48,269 @@
 #include "mntent_compat.h"
 #endif
 
+#define GLUSTERD_GET_BRICK_DIR(path, volinfo, priv)                            \
+    do {                                                                       \
+        int32_t _brick_len;                                                    \
+        if (volinfo->is_snap_volume) {                                         \
+            _brick_len = snprintf(path, PATH_MAX, "%s/snaps/%s/%s/%s",         \
+                                  priv->workdir, volinfo->snapshot->snapname,  \
+                                  volinfo->volname, GLUSTERD_BRICK_INFO_DIR);  \
+        } else {                                                               \
+            _brick_len = snprintf(path, PATH_MAX, "%s/%s/%s/%s",               \
+                                  priv->workdir, GLUSTERD_VOLUME_DIR_PREFIX,   \
+                                  volinfo->volname, GLUSTERD_BRICK_INFO_DIR);  \
+        }                                                                      \
+        if ((_brick_len < 0) || (_brick_len >= PATH_MAX)) {                    \
+            path[0] = 0;                                                       \
+        }                                                                      \
+    } while (0)
+
 void
-glusterd_replace_slash_with_hyphen (char *str)
+glusterd_replace_slash_with_hyphen(char *str)
 {
-        char                    *ptr = NULL;
+    char *ptr = NULL;
 
-        ptr = strchr (str, '/');
+    ptr = strchr(str, '/');
 
-        while (ptr) {
-                *ptr = '-';
-                ptr = strchr (str, '/');
-        }
+    while (ptr) {
+        *ptr = '-';
+        ptr = strchr(ptr, '/');
+    }
 }
 
 int32_t
-glusterd_store_create_brick_dir (glusterd_volinfo_t *volinfo)
+glusterd_store_create_brick_dir(glusterd_volinfo_t *volinfo)
 {
-        int32_t                 ret = -1;
-        char                    brickdirpath[PATH_MAX] = {0,};
-        glusterd_conf_t         *priv = NULL;
+    int32_t ret = -1;
+    char brickdirpath[PATH_MAX] = {
+        0,
+    };
+    glusterd_conf_t *priv = NULL;
 
-        GF_ASSERT (volinfo);
+    GF_ASSERT(volinfo);
 
-        priv = THIS->private;
-        GF_ASSERT (priv);
+    priv = THIS->private;
+    GF_ASSERT(priv);
 
-        GLUSTERD_GET_BRICK_DIR (brickdirpath, volinfo, priv);
-        ret = gf_store_mkdir (brickdirpath);
+    GLUSTERD_GET_BRICK_DIR(brickdirpath, volinfo, priv);
+    ret = gf_store_mkdir(brickdirpath);
 
-        return ret;
+    return ret;
 }
 
 static void
-glusterd_store_key_vol_brick_set (glusterd_brickinfo_t *brickinfo,
-                                char *key_vol_brick, size_t len)
+glusterd_store_key_vol_brick_set(glusterd_brickinfo_t *brickinfo,
+                                 char *key_vol_brick, size_t len)
 {
-        GF_ASSERT (brickinfo);
-        GF_ASSERT (key_vol_brick);
-        GF_ASSERT (len >= PATH_MAX);
+    GF_ASSERT(brickinfo);
+    GF_ASSERT(key_vol_brick);
+    GF_ASSERT(len >= PATH_MAX);
 
-        snprintf (key_vol_brick, len, "%s", brickinfo->path);
-        glusterd_replace_slash_with_hyphen (key_vol_brick);
+    snprintf(key_vol_brick, len, "%s", brickinfo->path);
+    glusterd_replace_slash_with_hyphen(key_vol_brick);
 }
 
 static void
-glusterd_store_brickinfofname_set (glusterd_brickinfo_t *brickinfo,
-                                char *brickfname, size_t len)
+glusterd_store_brickinfofname_set(glusterd_brickinfo_t *brickinfo,
+                                  char *brickfname, size_t len)
 {
-        char                    key_vol_brick[PATH_MAX] = {0};
+    char key_vol_brick[PATH_MAX] = {0};
 
-        GF_ASSERT (brickfname);
-        GF_ASSERT (brickinfo);
-        GF_ASSERT (len >= PATH_MAX);
+    GF_ASSERT(brickfname);
+    GF_ASSERT(brickinfo);
+    GF_ASSERT(len >= PATH_MAX);
 
-        glusterd_store_key_vol_brick_set (brickinfo, key_vol_brick,
-                                        sizeof (key_vol_brick));
-        snprintf (brickfname, len, "%s:%s", brickinfo->hostname, key_vol_brick);
+    glusterd_store_key_vol_brick_set(brickinfo, key_vol_brick,
+                                     sizeof(key_vol_brick));
+    snprintf(brickfname, len, "%s:%s", brickinfo->hostname, key_vol_brick);
 }
 
 static void
-glusterd_store_brickinfopath_set (glusterd_volinfo_t *volinfo,
-                              glusterd_brickinfo_t *brickinfo,
-                              char *brickpath, size_t len)
+glusterd_store_brickinfopath_set(glusterd_volinfo_t *volinfo,
+                                 glusterd_brickinfo_t *brickinfo,
+                                 char *brickpath, size_t len)
 {
-        char                    brickfname[PATH_MAX] = {0};
-        char                    brickdirpath[PATH_MAX] = {0,};
-        glusterd_conf_t         *priv = NULL;
-
-        GF_ASSERT (brickpath);
-        GF_ASSERT (brickinfo);
-        GF_ASSERT (len >= PATH_MAX);
-
-        priv = THIS->private;
-        GF_ASSERT (priv);
-
-        GLUSTERD_GET_BRICK_DIR (brickdirpath, volinfo, priv);
-        glusterd_store_brickinfofname_set (brickinfo, brickfname,
-                                        sizeof (brickfname));
-        snprintf (brickpath, len, "%s/%s", brickdirpath, brickfname);
+    char brickfname[PATH_MAX] = {0};
+    char brickdirpath[PATH_MAX] = {
+        0,
+    };
+    glusterd_conf_t *priv = NULL;
+
+    GF_ASSERT(brickpath);
+    GF_ASSERT(brickinfo);
+    GF_ASSERT(len >= PATH_MAX);
+
+    priv = THIS->private;
+    GF_ASSERT(priv);
+
+    GLUSTERD_GET_BRICK_DIR(brickdirpath, volinfo, priv);
+    glusterd_store_brickinfofname_set(brickinfo, brickfname,
+                                      sizeof(brickfname));
+    snprintf(brickpath, len, "%s/%s", brickdirpath, brickfname);
 }
 
 static void
-glusterd_store_snapd_path_set (glusterd_volinfo_t *volinfo,
-                               char *snapd_path, size_t len)
+glusterd_store_snapd_path_set(glusterd_volinfo_t *volinfo, char *snapd_path,
+                              size_t len)
 {
-        char                    volpath[PATH_MAX] = {0, };
-        glusterd_conf_t         *priv = NULL;
+    char volpath[PATH_MAX] = {
+        0,
+    };
+    glusterd_conf_t *priv = NULL;
 
-        GF_ASSERT (volinfo);
-        GF_ASSERT (len >= PATH_MAX);
+    GF_ASSERT(volinfo);
+    GF_ASSERT(len >= PATH_MAX);
 
-        priv = THIS->private;
-        GF_ASSERT (priv);
+    priv = THIS->private;
+    GF_ASSERT(priv);
 
-        GLUSTERD_GET_VOLUME_DIR (volpath, volinfo, priv);
+    GLUSTERD_GET_VOLUME_DIR(volpath, volinfo, priv);
 
-        snprintf (snapd_path, len, "%s/snapd.info", volpath);
+    snprintf(snapd_path, len, "%s/snapd.info", volpath);
 }
 
 gf_boolean_t
-glusterd_store_is_valid_brickpath (char *volname, char *brick)
+glusterd_store_is_valid_brickpath(char *volname, char *brick)
 {
-        glusterd_brickinfo_t    *brickinfo = NULL;
-        glusterd_volinfo_t      *volinfo = NULL;
-        int32_t                 ret = 0;
-        size_t                  volname_len = strlen (volname);
-        xlator_t                *this = NULL;
-        int                     bpath_len = 0;
-        const char              delim[2] = "/";
-        char                    *sub_dir = NULL;
-        char                    *saveptr = NULL;
-        char                    *brickpath_ptr = NULL;
-
-        this = THIS;
-        GF_ASSERT (this);
-
-        ret = glusterd_brickinfo_new_from_brick (brick, &brickinfo);
-        if (ret) {
-                gf_log (this->name, GF_LOG_WARNING, "Failed to create brick "
-                        "info for brick %s", brick);
-                ret = 0;
-                goto out;
-        }
-        ret = glusterd_volinfo_new (&volinfo);
-        if (ret) {
-                gf_log (this->name, GF_LOG_WARNING, "Failed to create volinfo");
-                ret = 0;
-                goto out;
-        }
-        if (volname_len >= sizeof (volinfo->volname)) {
-                gf_log (this->name, GF_LOG_WARNING, "volume name too long");
-                ret = 0;
-                goto out;
-        }
-        memcpy (volinfo->volname, volname, volname_len+1);
-
-        /* Check whether brickpath is less than PATH_MAX */
-        ret = 1;
-        bpath_len = strlen (brickinfo->path);
-
-        if (brickinfo->path[bpath_len - 1] != '/') {
-                if (strlen (brickinfo->path) >= PATH_MAX) {
-                        ret = 0;
-                        goto out;
-                }
-        } else {
-                /* Path has a trailing "/" which should not be considered in
-                 * length check validation
-                 */
-                if (strlen (brickinfo->path) >= PATH_MAX + 1) {
-                        ret = 0;
-                        goto out;
-                }
+    glusterd_brickinfo_t *brickinfo = NULL;
+    glusterd_volinfo_t *volinfo = NULL;
+    int32_t ret = 0;
+    size_t volname_len = strlen(volname);
+    xlator_t *this = NULL;
+    int bpath_len = 0;
+    const char delim[2] = "/";
+    char *sub_dir = NULL;
+    char *saveptr = NULL;
+    char *brickpath_ptr = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+
+    ret = glusterd_brickinfo_new_from_brick(brick, &brickinfo, _gf_false, NULL);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_WARNING, 0, GD_MSG_BRICK_CREATION_FAIL,
+               "Failed to create brick "
+               "info for brick %s",
+               brick);
+        ret = 0;
+        goto out;
+    }
+    ret = glusterd_volinfo_new(&volinfo);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_WARNING, 0, GD_MSG_VOLFILE_CREATE_FAIL,
+               "Failed to create volinfo");
+        ret = 0;
+        goto out;
+    }
+    if (volname_len >= sizeof(volinfo->volname)) {
+        gf_msg(this->name, GF_LOG_WARNING, 0, GD_MSG_NAME_TOO_LONG,
+               "volume name too long");
+        ret = 0;
+        goto out;
+    }
+    memcpy(volinfo->volname, volname, volname_len + 1);
+
+    /* Check whether brickpath is less than PATH_MAX */
+    ret = 1;
+    bpath_len = strlen(brickinfo->path);
+
+    if (brickinfo->path[bpath_len - 1] != '/') {
+        if (bpath_len >= PATH_MAX) {
+            ret = 0;
+            goto out;
+        }
+    } else {
+        /* Path has a trailing "/" which should not be considered in
+         * length check validation
+         */
+        if (bpath_len >= PATH_MAX + 1) {
+            ret = 0;
+            goto out;
         }
+    }
 
-        /* The following validation checks whether each sub directories in the
-         * brick path meets the POSIX max length validation
-         */
+    /* The following validation checks whether each sub directories in the
+     * brick path meets the POSIX max length validation
+     */
 
-        brickpath_ptr = brickinfo->path;
-        sub_dir = strtok_r (brickpath_ptr, delim, &saveptr);
+    brickpath_ptr = brickinfo->path;
+    sub_dir = strtok_r(brickpath_ptr, delim, &saveptr);
 
-        while (sub_dir != NULL) {
-                if (strlen(sub_dir) >= _POSIX_PATH_MAX) {
-                        ret = 0;
-                        goto out;
-                }
-                sub_dir = strtok_r (NULL, delim, &saveptr);
+    while (sub_dir != NULL) {
+        if (strlen(sub_dir) >= _POSIX_PATH_MAX) {
+            ret = 0;
+            goto out;
         }
+        sub_dir = strtok_r(NULL, delim, &saveptr);
+    }
 
 out:
-        if (brickinfo)
-                glusterd_brickinfo_delete (brickinfo);
-        if (volinfo)
-                glusterd_volinfo_unref (volinfo);
+    if (brickinfo)
+        glusterd_brickinfo_delete(brickinfo);
+    if (volinfo)
+        glusterd_volinfo_unref(volinfo);
 
-        return ret;
+    return ret;
 }
 
 int32_t
-glusterd_store_volinfo_brick_fname_write (int vol_fd,
+glusterd_store_volinfo_brick_fname_write(int vol_fd,
                                          glusterd_brickinfo_t *brickinfo,
-                                         int32_t brick_count)
+                                         int32_t brick_count,
+                                         int is_thin_arbiter)
 {
-        char                    key[PATH_MAX] = {0,};
-        char                    brickfname[PATH_MAX] = {0,};
-        int32_t                 ret = -1;
-
-        snprintf (key, sizeof (key), "%s-%d", GLUSTERD_STORE_KEY_VOL_BRICK,
-                  brick_count);
-        glusterd_store_brickinfofname_set (brickinfo, brickfname,
-                                        sizeof (brickfname));
-        ret = gf_store_save_value (vol_fd, key, brickfname);
-        if (ret)
-                goto out;
-
-out:
-        return ret;
+    char key[64] = {
+        0,
+    };
+    char brickfname[PATH_MAX] = {
+        0,
+    };
+    int32_t ret = -1;
+
+    if (!is_thin_arbiter) {
+        snprintf(key, sizeof(key), "%s-%d", GLUSTERD_STORE_KEY_VOL_BRICK,
+                 brick_count);
+    } else {
+        snprintf(key, sizeof(key), "%s-%d", GLUSTERD_STORE_KEY_VOL_TA_BRICK,
+                 brick_count);
+    }
+    glusterd_store_brickinfofname_set(brickinfo, brickfname,
+                                      sizeof(brickfname));
+    ret = gf_store_save_value(vol_fd, key, brickfname);
+    return ret;
 }
 
 int32_t
-glusterd_store_create_brick_shandle_on_absence (glusterd_volinfo_t *volinfo,
-                                                glusterd_brickinfo_t *brickinfo)
+glusterd_store_create_brick_shandle_on_absence(glusterd_volinfo_t *volinfo,
+                                               glusterd_brickinfo_t *brickinfo)
 {
-        char                    brickpath[PATH_MAX] = {0,};
-        int32_t                 ret = 0;
-
-        GF_ASSERT (volinfo);
-        GF_ASSERT (brickinfo);
-
-        glusterd_store_brickinfopath_set (volinfo, brickinfo, brickpath,
-                                          sizeof (brickpath));
-        ret = gf_store_handle_create_on_absence (&brickinfo->shandle,
-                                                 brickpath);
-        return ret;
+    char brickpath[PATH_MAX] = {
+        0,
+    };
+    int32_t ret = 0;
+
+    GF_ASSERT(volinfo);
+    GF_ASSERT(brickinfo);
+
+    glusterd_store_brickinfopath_set(volinfo, brickinfo, brickpath,
+                                     sizeof(brickpath));
+    ret = gf_store_handle_create_on_absence(&brickinfo->shandle, brickpath);
+    return ret;
 }
 
 int32_t
-glusterd_store_create_snapd_shandle_on_absence (glusterd_volinfo_t *volinfo)
+glusterd_store_create_snapd_shandle_on_absence(glusterd_volinfo_t *volinfo)
 {
-        char                    snapd_path[PATH_MAX] = {0,};
-        int32_t                 ret = 0;
+    char snapd_path[PATH_MAX] = {
+        0,
+    };
+    int32_t ret = 0;
 
-        GF_ASSERT (volinfo);
+    GF_ASSERT(volinfo);
 
-        glusterd_store_snapd_path_set (volinfo, snapd_path,
-                                       sizeof (snapd_path));
-        ret = gf_store_handle_create_on_absence (&volinfo->snapd.handle,
-                                                 snapd_path);
-        return ret;
+    glusterd_store_snapd_path_set(volinfo, snapd_path, sizeof(snapd_path));
+    ret = gf_store_handle_create_on_absence(&volinfo->snapd.handle, snapd_path);
+    return ret;
 }
 
 /* Store the bricks snapshot details only if required
@@ -288,465 +318,415 @@ glusterd_store_create_snapd_shandle_on_absence (glusterd_volinfo_t *volinfo)
  * The snapshot details will be stored only if the cluster op-version is
  * greater than or equal to 4
  */
-int
-gd_store_brick_snap_details_write (int fd, glusterd_brickinfo_t *brickinfo)
+static int
+gd_store_brick_snap_details_write(int fd, glusterd_brickinfo_t *brickinfo)
 {
-        int ret = -1;
-        xlator_t *this = NULL;
-        glusterd_conf_t *conf = NULL;
-        char value[256] = {0,};
-
-        this = THIS;
-        GF_ASSERT (this != NULL);
-        conf = this->private;
-        GF_VALIDATE_OR_GOTO (this->name, (conf != NULL), out);
+    int ret = -1;
+    xlator_t *this = NULL;
+    glusterd_conf_t *conf = NULL;
+    char value[5 * PATH_MAX];
+    uint total_len = 0;
 
-        GF_VALIDATE_OR_GOTO (this->name, (fd > 0), out);
-        GF_VALIDATE_OR_GOTO (this->name, (brickinfo != NULL), out);
+    this = THIS;
+    GF_ASSERT(this != NULL);
+    conf = this->private;
+    GF_VALIDATE_OR_GOTO(this->name, (conf != NULL), out);
 
-        if (conf->op_version < GD_OP_VERSION_3_6_0) {
-                ret = 0;
-                goto out;
-        }
-
-        if (strlen(brickinfo->device_path) > 0) {
-                snprintf (value, sizeof(value), "%s", brickinfo->device_path);
-                ret = gf_store_save_value (fd,
-                                GLUSTERD_STORE_KEY_BRICK_DEVICE_PATH, value);
-                if (ret)
-                        goto out;
-        }
-
-        if (strlen(brickinfo->mount_dir) > 0) {
-                memset (value, 0, sizeof (value));
-                snprintf (value, sizeof(value), "%s", brickinfo->mount_dir);
-                ret = gf_store_save_value (fd,
-                                GLUSTERD_STORE_KEY_BRICK_MOUNT_DIR, value);
-                if (ret)
-                        goto out;
-        }
-
-        if (strlen (brickinfo->fstype) > 0) {
-                snprintf (value, sizeof (value), "%s", brickinfo->fstype);
-                ret = gf_store_save_value (fd,
-                                GLUSTERD_STORE_KEY_BRICK_FSTYPE, value);
-                if (ret) {
-                        gf_log (this->name, GF_LOG_ERROR, "Failed to save "
-                                "brick fs type of brick %s", brickinfo->path);
-                        goto out;
-                }
-        }
-
-        if (strlen (brickinfo->mnt_opts) > 0) {
-                snprintf (value, sizeof (value), "%s", brickinfo->mnt_opts);
-                ret = gf_store_save_value (fd,
-                                GLUSTERD_STORE_KEY_BRICK_MNTOPTS, value);
-                if (ret) {
-                        gf_log (this->name, GF_LOG_ERROR, "Failed to save "
-                                "brick mnt opts of brick %s", brickinfo->path);
-                        goto out;
-                }
-        }
-
-        memset (value, 0, sizeof (value));
-        snprintf (value, sizeof(value), "%d", brickinfo->snap_status);
-        ret = gf_store_save_value (fd, GLUSTERD_STORE_KEY_BRICK_SNAP_STATUS,
-                                   value);
+    GF_VALIDATE_OR_GOTO(this->name, (fd > 0), out);
+    GF_VALIDATE_OR_GOTO(this->name, (brickinfo != NULL), out);
 
+    if (conf->op_version < GD_OP_VERSION_3_6_0) {
+        ret = 0;
+        goto out;
+    }
+
+    if (brickinfo->device_path[0] != '\0') {
+        ret = snprintf(value + total_len, sizeof(value) - total_len, "%s=%s\n",
+                       GLUSTERD_STORE_KEY_BRICK_DEVICE_PATH,
+                       brickinfo->device_path);
+        if (ret < 0 || ret >= sizeof(value) - total_len) {
+            ret = -1;
+            goto err;
+        }
+        total_len += ret;
+    }
+
+    if (brickinfo->mount_dir[0] != '\0') {
+        ret = snprintf(value + total_len, sizeof(value) - total_len, "%s=%s\n",
+                       GLUSTERD_STORE_KEY_BRICK_MOUNT_DIR,
+                       brickinfo->mount_dir);
+        if (ret < 0 || ret >= sizeof(value) - total_len) {
+            ret = -1;
+            goto err;
+        }
+        total_len += ret;
+    }
+
+    if (brickinfo->fstype[0] != '\0') {
+        ret = snprintf(value + total_len, sizeof(value) - total_len, "%s=%s\n",
+                       GLUSTERD_STORE_KEY_BRICK_FSTYPE, brickinfo->fstype);
+        if (ret < 0 || ret >= sizeof(value) - total_len) {
+            ret = -1;
+            goto err;
+        }
+        total_len += ret;
+    }
+
+    if (brickinfo->mnt_opts[0] != '\0') {
+        ret = snprintf(value + total_len, sizeof(value) - total_len, "%s=%s\n",
+                       GLUSTERD_STORE_KEY_BRICK_MNTOPTS, brickinfo->mnt_opts);
+        if (ret < 0 || ret >= sizeof(value) - total_len) {
+            ret = -1;
+            goto err;
+        }
+        total_len += ret;
+    }
+
+    ret = snprintf(value + total_len, sizeof(value) - total_len, "%s=%d\n",
+                   GLUSTERD_STORE_KEY_BRICK_SNAP_STATUS,
+                   brickinfo->snap_status);
+    if (ret < 0 || ret >= sizeof(value) - total_len) {
+        ret = -1;
+        goto err;
+    }
+    total_len += ret;
+
+    ret = snprintf(value + total_len, sizeof(value) - total_len,
+                   "%s=%" PRIu64 "\n", GLUSTERD_STORE_KEY_BRICK_FSID,
+                   brickinfo->statfs_fsid);
+    if (ret < 0 || ret >= sizeof(value) - total_len) {
+        ret = -1;
+        goto err;
+    }
+
+    ret = gf_store_save_items(fd, value);
+err:
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_FS_LABEL_UPDATE_FAIL,
+               "Failed to save "
+               "snap detils of brick %s",
+               brickinfo->path);
+    }
 out:
-        return ret;
+    return ret;
 }
 
-int32_t
-glusterd_store_brickinfo_write (int fd, glusterd_brickinfo_t *brickinfo)
+static int32_t
+glusterd_store_brickinfo_write(int fd, glusterd_brickinfo_t *brickinfo)
 {
-        char                    value[256] = {0,};
-        int32_t                 ret = 0;
-
-        GF_ASSERT (brickinfo);
-        GF_ASSERT (fd > 0);
-
-        ret = gf_store_save_value (fd, GLUSTERD_STORE_KEY_BRICK_HOSTNAME,
-                                   brickinfo->hostname);
-        if (ret)
-                goto out;
-
-        ret = gf_store_save_value (fd, GLUSTERD_STORE_KEY_BRICK_PATH,
-                                   brickinfo->path);
-        if (ret)
-                goto out;
-
-        snprintf (value, sizeof(value), "%d", brickinfo->port);
-        ret = gf_store_save_value (fd, GLUSTERD_STORE_KEY_BRICK_PORT, value);
-
-        snprintf (value, sizeof(value), "%d", brickinfo->rdma_port);
-        ret = gf_store_save_value (fd, GLUSTERD_STORE_KEY_BRICK_RDMA_PORT,
-                                   value);
-
-        snprintf (value, sizeof(value), "%d", brickinfo->decommissioned);
-        ret = gf_store_save_value (fd, GLUSTERD_STORE_KEY_BRICK_DECOMMISSIONED,
-                                   value);
-        if (ret)
-                goto out;
-
-        ret = gf_store_save_value (fd, GLUSTERD_STORE_KEY_BRICK_ID,
-                                   brickinfo->brick_id);
-        if (ret)
-                goto out;
-
-        ret = gd_store_brick_snap_details_write (fd, brickinfo);
-        if (ret)
-                goto out;
-
-        if (!brickinfo->vg[0])
-                goto out;
-
-        ret = gf_store_save_value (fd, GLUSTERD_STORE_KEY_BRICK_VGNAME,
-                                         brickinfo->vg);
+    char value[5 * PATH_MAX];
+    int32_t ret = -1;
+
+    GF_ASSERT(brickinfo);
+    GF_ASSERT(fd > 0);
+
+    ret = snprintf(value, sizeof(value),
+                   "%s=%s\n%s=%s\n%s=%s\n%s=%s\n%s=%d\n%s=%d\n%s=%d\n%s=%s\n",
+                   GLUSTERD_STORE_KEY_BRICK_UUID, uuid_utoa(brickinfo->uuid),
+                   GLUSTERD_STORE_KEY_BRICK_HOSTNAME, brickinfo->hostname,
+                   GLUSTERD_STORE_KEY_BRICK_PATH, brickinfo->path,
+                   GLUSTERD_STORE_KEY_BRICK_REAL_PATH, brickinfo->path,
+                   GLUSTERD_STORE_KEY_BRICK_PORT, brickinfo->port,
+                   GLUSTERD_STORE_KEY_BRICK_RDMA_PORT, brickinfo->rdma_port,
+                   GLUSTERD_STORE_KEY_BRICK_DECOMMISSIONED,
+                   brickinfo->decommissioned, GLUSTERD_STORE_KEY_BRICK_ID,
+                   brickinfo->brick_id);
+
+    if (ret < 0 || ret >= sizeof(value)) {
+        ret = -1;
+        goto out;
+    }
+
+    ret = gf_store_save_items(fd, value);
+    if (ret)
+        goto out;
+
+    ret = gd_store_brick_snap_details_write(fd, brickinfo);
+    if (ret)
+        goto out;
+
+    if (!brickinfo->vg[0])
+        goto out;
+
+    ret = gf_store_save_value(fd, GLUSTERD_STORE_KEY_BRICK_VGNAME,
+                              brickinfo->vg);
 out:
-        gf_log (THIS->name, GF_LOG_DEBUG, "Returning %d", ret);
-        return ret;
+    gf_msg_debug(THIS->name, 0, "Returning %d", ret);
+    return ret;
 }
 
 int32_t
-glusterd_store_snapd_write (int fd, glusterd_volinfo_t *volinfo)
+glusterd_store_snapd_write(int fd, glusterd_volinfo_t *volinfo)
 {
-        char                    value[256] = {0,};
-        int32_t                 ret        = 0;
-        xlator_t               *this       = NULL;
-
-        GF_ASSERT (volinfo);
-        GF_ASSERT (fd > 0);
-
-        this = THIS;
-        GF_ASSERT (this);
-
-        snprintf (value, sizeof(value), "%d", volinfo->snapd.port);
-        ret = gf_store_save_value (fd, GLUSTERD_STORE_KEY_SNAPD_PORT, value);
-        if (ret)
-                gf_log (this->name, GF_LOG_ERROR, "failed to store the snapd "
-                        "port of volume %s", volinfo->volname);
-
-
-        gf_log (this->name, GF_LOG_DEBUG, "Returning %d", ret);
-        return ret;
+    char value[64] = {
+        0,
+    };
+    int32_t ret = 0;
+    xlator_t *this = NULL;
+
+    GF_ASSERT(volinfo);
+    GF_ASSERT(fd > 0);
+
+    this = THIS;
+    GF_ASSERT(this);
+
+    snprintf(value, sizeof(value), "%d", volinfo->snapd.port);
+    ret = gf_store_save_value(fd, GLUSTERD_STORE_KEY_SNAPD_PORT, value);
+    if (ret)
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_SNAPD_PORT_STORE_FAIL,
+               "failed to store the snapd "
+               "port of volume %s",
+               volinfo->volname);
+
+    gf_msg_debug(this->name, 0, "Returning %d", ret);
+    return ret;
 }
 
-int32_t
-glusterd_store_perform_brick_store (glusterd_brickinfo_t *brickinfo)
+static int32_t
+glusterd_store_perform_brick_store(glusterd_brickinfo_t *brickinfo)
 {
-        int                         fd = -1;
-        int32_t                     ret = -1;
-        GF_ASSERT (brickinfo);
-
-        fd = gf_store_mkstemp (brickinfo->shandle);
-        if (fd <= 0) {
-                ret = -1;
-                goto out;
-        }
-
-        ret = glusterd_store_brickinfo_write (fd, brickinfo);
-        if (ret)
-                goto out;
+    int fd = -1;
+    int32_t ret = -1;
+    GF_ASSERT(brickinfo);
+
+    fd = gf_store_mkstemp(brickinfo->shandle);
+    if (fd <= 0) {
+        ret = -1;
+        goto out;
+    }
+    ret = glusterd_store_brickinfo_write(fd, brickinfo);
+    if (ret)
+        goto out;
 
 out:
-        if (ret && (fd > 0))
-                gf_store_unlink_tmppath (brickinfo->shandle);
-        gf_log (THIS->name, GF_LOG_DEBUG, "Returning %d", ret);
-        return ret;
+    if (ret && (fd > 0)) {
+        gf_store_unlink_tmppath(brickinfo->shandle);
+    }
+    gf_msg_debug(THIS->name, 0, "Returning %d", ret);
+    return ret;
 }
 
 int32_t
-glusterd_store_perform_snapd_store (glusterd_volinfo_t *volinfo)
+glusterd_store_perform_snapd_store(glusterd_volinfo_t *volinfo)
 {
-        int                         fd  = -1;
-        int32_t                     ret = -1;
-        xlator_t                  *this = NULL;
-
-        GF_ASSERT (volinfo);
-
-        this = THIS;
-        GF_ASSERT (this);
-
-        fd = gf_store_mkstemp (volinfo->snapd.handle);
-        if (fd <= 0) {
-                gf_log (this->name, GF_LOG_ERROR, "failed to create the "
-                        "temporary file for the snapd store handle of volume "
-                        "%s", volinfo->volname);
-                goto out;
-        }
-
-        ret = glusterd_store_snapd_write (fd, volinfo);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "failed to write snapd port "
-                        "info to store handle (volume: %s", volinfo->volname);
-                goto out;
-        }
-
-        ret = gf_store_rename_tmppath (volinfo->snapd.handle);
+    int fd = -1;
+    int32_t ret = -1;
+    xlator_t *this = NULL;
+
+    GF_ASSERT(volinfo);
+
+    this = THIS;
+    GF_ASSERT(this);
+
+    fd = gf_store_mkstemp(volinfo->snapd.handle);
+    if (fd <= 0) {
+        gf_msg(this->name, GF_LOG_ERROR, errno, GD_MSG_FILE_OP_FAILED,
+               "failed to create the "
+               "temporary file for the snapd store handle of volume "
+               "%s",
+               volinfo->volname);
+        goto out;
+    }
+
+    ret = glusterd_store_snapd_write(fd, volinfo);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_SNAPD_PORT_STORE_FAIL,
+               "failed to write snapd port "
+               "info to store handle (volume: %s",
+               volinfo->volname);
+        goto out;
+    }
+
+    ret = gf_store_rename_tmppath(volinfo->snapd.handle);
 
 out:
-        if (ret && (fd > 0))
-                gf_store_unlink_tmppath (volinfo->snapd.handle);
-        gf_log (THIS->name, GF_LOG_DEBUG, "Returning %d", ret);
-        return ret;
+    if (ret && (fd > 0))
+        gf_store_unlink_tmppath(volinfo->snapd.handle);
+    gf_msg_debug(THIS->name, 0, "Returning %d", ret);
+    return ret;
 }
 
-int32_t
-glusterd_store_brickinfo (glusterd_volinfo_t *volinfo,
-                          glusterd_brickinfo_t *brickinfo, int32_t brick_count,
-                          int vol_fd)
+static int32_t
+glusterd_store_brickinfo(glusterd_volinfo_t *volinfo,
+                         glusterd_brickinfo_t *brickinfo, int32_t brick_count,
+                         int vol_fd, int is_thin_arbiter)
 {
-        int32_t                 ret = -1;
+    int32_t ret = -1;
 
-        GF_ASSERT (volinfo);
-        GF_ASSERT (brickinfo);
+    GF_ASSERT(volinfo);
+    GF_ASSERT(brickinfo);
 
-        ret = glusterd_store_volinfo_brick_fname_write (vol_fd, brickinfo,
-                                                       brick_count);
-        if (ret)
-                goto out;
-
-        ret = glusterd_store_create_brick_dir (volinfo);
-        if (ret)
-                goto out;
+    ret = glusterd_store_volinfo_brick_fname_write(
+        vol_fd, brickinfo, brick_count, is_thin_arbiter);
+    if (ret)
+        goto out;
 
-        ret = glusterd_store_create_brick_shandle_on_absence (volinfo,
-                                                              brickinfo);
-        if (ret)
-                goto out;
+    ret = glusterd_store_create_brick_shandle_on_absence(volinfo, brickinfo);
+    if (ret)
+        goto out;
 
-        ret = glusterd_store_perform_brick_store (brickinfo);
+    ret = glusterd_store_perform_brick_store(brickinfo);
 out:
-        gf_log (THIS->name, GF_LOG_DEBUG, "Returning with %d", ret);
-        return ret;
+    gf_msg_debug(THIS->name, 0, "Returning with %d", ret);
+    return ret;
 }
 
 int32_t
-glusterd_store_snapd_info (glusterd_volinfo_t *volinfo)
+glusterd_store_snapd_info(glusterd_volinfo_t *volinfo)
 {
-        int32_t                 ret  = -1;
-        xlator_t               *this = NULL;
-
-        GF_ASSERT (volinfo);
-
-        this = THIS;
-        GF_ASSERT (this);
-
-        ret = glusterd_store_create_snapd_shandle_on_absence (volinfo);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "failed to create store "
-                        "handle for snapd (volume: %s)", volinfo->volname);
-                goto out;
-        }
-
-        ret = glusterd_store_perform_snapd_store (volinfo);
-        if (ret)
-                gf_log (this->name, GF_LOG_ERROR, "failed to store snapd info "
-                        "of the volume %s", volinfo->volname);
+    int32_t ret = -1;
+    xlator_t *this = NULL;
+
+    GF_ASSERT(volinfo);
+
+    this = THIS;
+    GF_ASSERT(this);
+
+    ret = glusterd_store_create_snapd_shandle_on_absence(volinfo);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_STORE_HANDLE_CREATE_FAIL,
+               "failed to create store "
+               "handle for snapd (volume: %s)",
+               volinfo->volname);
+        goto out;
+    }
+
+    ret = glusterd_store_perform_snapd_store(volinfo);
+    if (ret)
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_SNAPD_INFO_STORE_FAIL,
+               "failed to store snapd info "
+               "of the volume %s",
+               volinfo->volname);
 
 out:
-        if (ret)
-                gf_store_unlink_tmppath (volinfo->snapd.handle);
+    if (ret)
+        gf_store_unlink_tmppath(volinfo->snapd.handle);
 
-        gf_log (this->name, GF_LOG_DEBUG, "Returning with %d", ret);
-        return ret;
+    gf_msg_debug(this->name, 0, "Returning with %d", ret);
+    return ret;
 }
 
 int32_t
-glusterd_store_delete_brick (glusterd_brickinfo_t *brickinfo, char *delete_path)
+glusterd_store_delete_brick(glusterd_brickinfo_t *brickinfo, char *delete_path)
 {
-        int32_t                 ret = -1;
-        glusterd_conf_t         *priv = NULL;
-        char                    brickpath[PATH_MAX] = {0,};
-        char                    *ptr = NULL;
-        char                    *tmppath = NULL;
-        xlator_t                *this = NULL;
-
-        this = THIS;
-        GF_ASSERT (this);
-        GF_ASSERT (brickinfo);
-
-        priv = this->private;
-        GF_ASSERT (priv);
-
-        tmppath = gf_strdup (brickinfo->path);
-
-        ptr = strchr (tmppath, '/');
-
-        while (ptr) {
-                *ptr = '-';
-                ptr = strchr (tmppath, '/');
-        }
-
-        snprintf (brickpath, sizeof (brickpath),
-                  "%s/"GLUSTERD_BRICK_INFO_DIR"/%s:%s", delete_path,
-                  brickinfo->hostname, tmppath);
+    int32_t ret = -1;
+    glusterd_conf_t *priv = NULL;
+    char brickpath[PATH_MAX] = {
+        0,
+    };
+    char *ptr = NULL;
+    char *tmppath = NULL;
+    xlator_t *this = NULL;
 
-        GF_FREE (tmppath);
-
-        ret = unlink (brickpath);
-
-        if ((ret < 0) && (errno != ENOENT)) {
-                gf_log (this->name, GF_LOG_DEBUG, "Unlink failed on %s, "
-                        "reason: %s", brickpath, strerror(errno));
-                ret = -1;
-                goto out;
-        } else {
-                ret = 0;
-        }
-
-out:
-        if (brickinfo->shandle) {
-                gf_store_handle_destroy (brickinfo->shandle);
-                brickinfo->shandle = NULL;
-        }
-        gf_log (this->name, GF_LOG_DEBUG, "Returning with %d", ret);
-        return ret;
-}
+    this = THIS;
+    GF_ASSERT(this);
+    GF_ASSERT(brickinfo);
 
-int32_t
-glusterd_store_remove_bricks (glusterd_volinfo_t *volinfo, char *delete_path)
-{
-        int32_t                 ret = 0;
-        glusterd_brickinfo_t    *tmp = NULL;
-        glusterd_conf_t         *priv = NULL;
-        char                    brickdir [PATH_MAX] = {0,};
-        DIR                     *dir = NULL;
-        struct dirent           *entry = NULL;
-        char                    path[PATH_MAX] = {0,};
-        xlator_t                *this = NULL;
-
-        this = THIS;
-        GF_ASSERT (this);
-
-        GF_ASSERT (volinfo);
-
-        cds_list_for_each_entry (tmp, &volinfo->bricks, brick_list) {
-                ret = glusterd_store_delete_brick (tmp, delete_path);
-                if (ret)
-                        goto out;
-        }
+    priv = this->private;
+    GF_ASSERT(priv);
 
-        priv = this->private;
-        GF_ASSERT (priv);
+    tmppath = gf_strdup(brickinfo->path);
 
-        snprintf (brickdir, sizeof (brickdir), "%s/%s", delete_path,
-                  GLUSTERD_BRICK_INFO_DIR);
+    ptr = strchr(tmppath, '/');
 
-        dir = opendir (brickdir);
+    while (ptr) {
+        *ptr = '-';
+        ptr = strchr(tmppath, '/');
+    }
 
-        GF_FOR_EACH_ENTRY_IN_DIR (entry, dir);
+    snprintf(brickpath, sizeof(brickpath),
+             "%s/" GLUSTERD_BRICK_INFO_DIR "/%s:%s", delete_path,
+             brickinfo->hostname, tmppath);
 
-        while (entry) {
-                snprintf (path, sizeof (path), "%s/%s",
-                          brickdir, entry->d_name);
-                ret = unlink (path);
-                if (ret && errno != ENOENT) {
-                        gf_log (this->name, GF_LOG_DEBUG, "Unable to unlink %s, "
-                                "reason: %s", path, strerror(errno));
-                }
-                GF_FOR_EACH_ENTRY_IN_DIR (entry, dir);
-        }
+    GF_FREE(tmppath);
 
-        closedir (dir);
+    ret = sys_unlink(brickpath);
 
-        ret = rmdir (brickdir);
+    if ((ret < 0) && (errno != ENOENT)) {
+        gf_msg_debug(this->name, 0, "Unlink failed on %s", brickpath);
+        ret = -1;
+        goto out;
+    } else {
+        ret = 0;
+    }
 
 out:
-        gf_log (this->name, GF_LOG_DEBUG, "Returning with %d", ret);
-        return ret;
+    if (brickinfo->shandle) {
+        gf_store_handle_destroy(brickinfo->shandle);
+        brickinfo->shandle = NULL;
+    }
+    gf_msg_debug(this->name, 0, "Returning with %d", ret);
+    return ret;
 }
 
 static int
-_storeslaves (dict_t *this, char *key, data_t *value, void *data)
+_storeopts(dict_t *dict_value, char *key, data_t *value, void *data)
 {
-        int32_t                      ret = 0;
-        gf_store_handle_t           *shandle = NULL;
-        xlator_t                    *xl = NULL;
-
-        xl = THIS;
-        GF_ASSERT (xl);
+    int32_t ret = 0;
+    int32_t exists = 0;
+    int32_t option_len = 0;
+    gf_store_handle_t *shandle = NULL;
+    glusterd_volinfo_data_store_t *dict_data = NULL;
+    xlator_t *this = NULL;
 
-        shandle = (gf_store_handle_t*)data;
+    this = THIS;
+    GF_ASSERT(this);
 
-        GF_ASSERT (shandle);
-        GF_ASSERT (shandle->fd > 0);
-        GF_ASSERT (shandle->path);
-        GF_ASSERT (key);
-        GF_ASSERT (value && value->data);
+    dict_data = (glusterd_volinfo_data_store_t *)data;
+    shandle = dict_data->shandle;
 
-        if ((!shandle) || (shandle->fd <= 0) || (!shandle->path))
-                return -1;
+    GF_ASSERT(shandle);
+    GF_ASSERT(shandle->fd > 0);
+    GF_ASSERT(key);
+    GF_ASSERT(value);
+    GF_ASSERT(value->data);
 
-        if (!key)
-                return -1;
-        if (!value || !value->data)
-                return -1;
-
-        gf_log (xl->name, GF_LOG_DEBUG, "Storing in volinfo:key= %s, val=%s",
-                key, value->data);
-
-        ret = gf_store_save_value (shandle->fd, key, (char*)value->data);
-        if (ret) {
-                gf_log (xl->name, GF_LOG_ERROR, "Unable to write into store"
-                                " handle for path: %s", shandle->path);
-                return -1;
-        }
-        return 0;
-}
-
-
-int _storeopts (dict_t *this, char *key, data_t *value, void *data)
-{
-        int32_t                      ret = 0;
-        int32_t                      exists = 0;
-        gf_store_handle_t           *shandle = NULL;
-        xlator_t                    *xl = NULL;
-
-        xl = THIS;
-        GF_ASSERT (xl);
-
-        shandle = (gf_store_handle_t*)data;
-
-        GF_ASSERT (shandle);
-        GF_ASSERT (shandle->fd > 0);
-        GF_ASSERT (shandle->path);
-        GF_ASSERT (key);
-        GF_ASSERT (value && value->data);
-
-        if ((!shandle) || (shandle->fd <= 0) || (!shandle->path))
-                return -1;
-
-        if (!key)
-                return -1;
-        if (!value || !value->data)
-                return -1;
-
-        if (is_key_glusterd_hooks_friendly (key)) {
-                exists = 1;
-
-        } else {
-                exists = glusterd_check_option_exists (key, NULL);
-        }
-
-        if (1 == exists) {
-                gf_log (xl->name, GF_LOG_DEBUG, "Storing in volinfo:key= %s, "
-                        "val=%s", key, value->data);
+    if (dict_data->key_check == 1) {
+        if (is_key_glusterd_hooks_friendly(key)) {
+            exists = 1;
 
         } else {
-                gf_log (xl->name, GF_LOG_DEBUG, "Discarding:key= %s, val=%s",
-                        key, value->data);
-                return 0;
-        }
-
-        ret = gf_store_save_value (shandle->fd, key, (char*)value->data);
-        if (ret) {
-                gf_log (xl->name, GF_LOG_ERROR, "Unable to write into store"
-                        " handle for path: %s", shandle->path);
-                return -1;
-        }
+            exists = glusterd_check_option_exists(key, NULL);
+        }
+    }
+    if (exists == 1 || dict_data->key_check == 0) {
+        gf_msg_debug(this->name, 0,
+                     "Storing in buffer for volinfo:key= %s, "
+                     "val=%s",
+                     key, value->data);
+    } else {
+        gf_msg_debug(this->name, 0, "Discarding:key= %s, val=%s", key,
+                     value->data);
         return 0;
+    }
+
+    /*
+     * The option_len considers the length of the key value
+     * pair and along with that '=' and '\n', but as value->len
+     * already considers a NULL at the end of the data, adding
+     * just 1.
+     */
+    option_len = strlen(key) + value->len + 1;
+
+    if ((VOLINFO_BUFFER_SIZE - dict_data->buffer_len - 1) < option_len) {
+        ret = gf_store_save_items(shandle->fd, dict_data->buffer);
+        if (ret) {
+            gf_smsg(this->name, GF_LOG_ERROR, 0, GD_MSG_FILE_OP_FAILED, NULL);
+            return -1;
+        }
+        dict_data->buffer_len = 0;
+        dict_data->buffer[0] = '\0';
+    }
+    ret = snprintf(dict_data->buffer + dict_data->buffer_len, option_len + 1,
+                   "%s=%s\n", key, value->data);
+    if (ret < 0 || ret > option_len + 1) {
+        gf_smsg(this->name, GF_LOG_ERROR, EINVAL, GD_MSG_COPY_FAIL, NULL);
+        return -1;
+    }
+
+    dict_data->buffer_len += ret;
+
+    return 0;
 }
 
 /* Store the volumes snapshot details only if required
@@ -754,3450 +734,3963 @@ int _storeopts (dict_t *this, char *key, data_t *value, void *data)
  * The snapshot details will be stored only if the cluster op-version is
  * greater than or equal to 4
  */
-int
-glusterd_volume_write_snap_details (int fd, glusterd_volinfo_t *volinfo)
+static int
+glusterd_volume_write_snap_details(int fd, glusterd_volinfo_t *volinfo)
 {
-        int              ret           = -1;
-        xlator_t        *this          = NULL;
-        glusterd_conf_t *conf          = NULL;
-        char             buf[PATH_MAX] = {0,};
-
-        this = THIS;
-        GF_ASSERT (this != NULL);
-        conf = this->private;
-        GF_VALIDATE_OR_GOTO (this->name, (conf != NULL), out);
-
-        GF_VALIDATE_OR_GOTO (this->name, (fd > 0), out);
-        GF_VALIDATE_OR_GOTO (this->name, (volinfo != NULL), out);
-
-        if (conf->op_version < GD_OP_VERSION_3_6_0) {
-                ret = 0;
-                goto out;
-        }
-
-        snprintf (buf, sizeof (buf), "%s", volinfo->parent_volname);
-        ret = gf_store_save_value (fd, GLUSTERD_STORE_KEY_PARENT_VOLNAME, buf);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "Failed to store "
-                        GLUSTERD_STORE_KEY_PARENT_VOLNAME);
-                goto out;
-        }
-
-        ret = gf_store_save_value (fd, GLUSTERD_STORE_KEY_VOL_RESTORED_SNAP,
-                                   uuid_utoa (volinfo->restored_from_snap));
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "Unable to write restored_from_snap");
-                goto out;
-        }
-
-        memset (buf, 0, sizeof (buf));
-        snprintf (buf, sizeof (buf), "%"PRIu64, volinfo->snap_max_hard_limit);
-        ret = gf_store_save_value (fd, GLUSTERD_STORE_KEY_SNAP_MAX_HARD_LIMIT,
-                                   buf);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "Unable to write snap-max-hard-limit");
-                goto out;
-        }
-
-        ret = glusterd_store_snapd_info (volinfo);
-        if (ret)
-                gf_log (this->name, GF_LOG_ERROR, "snapd info store failed "
-                        "volume: %s", volinfo->volname);
-
+    int ret = -1;
+    xlator_t *this = NULL;
+    glusterd_conf_t *conf = NULL;
+    char buf[PATH_MAX] = {
+        0,
+    };
+
+    this = THIS;
+    GF_ASSERT(this != NULL);
+    conf = this->private;
+    GF_VALIDATE_OR_GOTO(this->name, (conf != NULL), out);
+
+    GF_VALIDATE_OR_GOTO(this->name, (fd > 0), out);
+    GF_VALIDATE_OR_GOTO(this->name, (volinfo != NULL), out);
+
+    if (conf->op_version < GD_OP_VERSION_3_6_0) {
+        ret = 0;
+        goto out;
+    }
+
+    ret = snprintf(buf, sizeof(buf), "%s=%s\n%s=%s\n%s=%" PRIu64 "\n",
+                   GLUSTERD_STORE_KEY_PARENT_VOLNAME, volinfo->parent_volname,
+                   GLUSTERD_STORE_KEY_VOL_RESTORED_SNAP,
+                   uuid_utoa(volinfo->restored_from_snap),
+                   GLUSTERD_STORE_KEY_SNAP_MAX_HARD_LIMIT,
+                   volinfo->snap_max_hard_limit);
+    if (ret < 0 || ret >= sizeof(buf)) {
+        ret = -1;
+        goto err;
+    }
+
+    ret = gf_store_save_items(fd, buf);
+    if (ret) {
+        goto err;
+    }
+    ret = glusterd_store_snapd_info(volinfo);
+err:
+    if (ret)
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_SNAPINFO_WRITE_FAIL,
+               "Failed to write snap details"
+               " for volume %s",
+               volinfo->volname);
 out:
-        if (ret)
-                gf_log (this->name, GF_LOG_ERROR, "Failed to write snap details"
-                        " for volume %s", volinfo->volname);
-        return ret;
+    return ret;
 }
 
-int32_t
-glusterd_volume_write_tier_details (int fd, glusterd_volinfo_t *volinfo)
+static int32_t
+glusterd_volume_exclude_options_write(int fd, glusterd_volinfo_t *volinfo)
 {
-        int32_t      ret            = -1;
-        char         buf[PATH_MAX]  = "";
-
-        if (volinfo->type != GF_CLUSTER_TYPE_TIER) {
-                ret = 0;
-                goto out;
-        }
+    char *str = NULL;
+    char buf[PATH_MAX];
+    uint total_len = 0;
+    int32_t ret = -1;
+    xlator_t *this = THIS;
+    glusterd_conf_t *conf = NULL;
+
+    GF_ASSERT(this);
+    GF_ASSERT(fd > 0);
+    GF_ASSERT(volinfo);
+    conf = this->private;
+    GF_VALIDATE_OR_GOTO(this->name, (conf != NULL), out);
+
+    ret = snprintf(buf + total_len, sizeof(buf) - total_len,
+                   "%s=%d\n%s=%d\n%s=%d\n%s=%d\n%s=%d\n%s=%d\n",
+                   GLUSTERD_STORE_KEY_VOL_TYPE, volinfo->type,
+                   GLUSTERD_STORE_KEY_VOL_COUNT, volinfo->brick_count,
+                   GLUSTERD_STORE_KEY_VOL_STATUS, volinfo->status,
+                   GLUSTERD_STORE_KEY_VOL_SUB_COUNT, volinfo->sub_count,
+                   GLUSTERD_STORE_KEY_VOL_STRIPE_CNT, volinfo->stripe_count,
+                   GLUSTERD_STORE_KEY_VOL_REPLICA_CNT, volinfo->replica_count);
+    if (ret < 0 || ret >= sizeof(buf) - total_len) {
+        ret = -1;
+        goto out;
+    }
+    total_len += ret;
+
+    if ((conf->op_version >= GD_OP_VERSION_3_7_6) && volinfo->arbiter_count) {
+        ret = snprintf(buf + total_len, sizeof(buf) - total_len, "%s=%d\n",
+                       GLUSTERD_STORE_KEY_VOL_ARBITER_CNT,
+                       volinfo->arbiter_count);
+        if (ret < 0 || ret >= sizeof(buf) - total_len) {
+            ret = -1;
+            goto out;
+        }
+        total_len += ret;
+    }
+
+    if (conf->op_version >= GD_OP_VERSION_3_6_0) {
+        ret = snprintf(
+            buf + total_len, sizeof(buf) - total_len, "%s=%d\n%s=%d\n",
+            GLUSTERD_STORE_KEY_VOL_DISPERSE_CNT, volinfo->disperse_count,
+            GLUSTERD_STORE_KEY_VOL_REDUNDANCY_CNT, volinfo->redundancy_count);
+        if (ret < 0 || ret >= sizeof(buf) - total_len) {
+            ret = -1;
+            goto out;
+        }
+        total_len += ret;
+    }
+
+    ret = snprintf(buf + total_len, sizeof(buf) - total_len,
+                   "%s=%d\n%s=%d\n%s=%s\n", GLUSTERD_STORE_KEY_VOL_VERSION,
+                   volinfo->version, GLUSTERD_STORE_KEY_VOL_TRANSPORT,
+                   volinfo->transport_type, GLUSTERD_STORE_KEY_VOL_ID,
+                   uuid_utoa(volinfo->volume_id));
+    if (ret < 0 || ret >= sizeof(buf) - total_len) {
+        ret = -1;
+        goto out;
+    }
+    total_len += ret;
+
+    str = glusterd_auth_get_username(volinfo);
+    if (str) {
+        ret = snprintf(buf + total_len, sizeof(buf) - total_len, "%s=%s\n",
+                       GLUSTERD_STORE_KEY_USERNAME, str);
+        if (ret < 0 || ret >= sizeof(buf) - total_len) {
+            ret = -1;
+            goto out;
+        }
+        total_len += ret;
+    }
+
+    str = glusterd_auth_get_password(volinfo);
+    if (str) {
+        ret = snprintf(buf + total_len, sizeof(buf) - total_len, "%s=%s\n",
+                       GLUSTERD_STORE_KEY_PASSWORD, str);
+        if (ret < 0 || ret >= sizeof(buf) - total_len) {
+            ret = -1;
+            goto out;
+        }
+        total_len += ret;
+    }
+
+    ret = snprintf(buf + total_len, sizeof(buf) - total_len, "%s=%d\n%s=%d\n",
+                   GLUSTERD_STORE_KEY_VOL_OP_VERSION, volinfo->op_version,
+                   GLUSTERD_STORE_KEY_VOL_CLIENT_OP_VERSION,
+                   volinfo->client_op_version);
+    if (ret < 0 || ret >= sizeof(buf) - total_len) {
+        ret = -1;
+        goto out;
+    }
+    total_len += ret;
+
+    if (conf->op_version >= GD_OP_VERSION_3_7_6) {
+        ret = snprintf(buf + total_len, sizeof(buf) - total_len, "%s=%d\n",
+                       GLUSTERD_STORE_KEY_VOL_QUOTA_VERSION,
+                       volinfo->quota_xattr_version);
+        if (ret < 0 || ret >= sizeof(buf) - total_len) {
+            ret = -1;
+            goto out;
+        }
+        total_len += ret;
+    }
+    if (conf->op_version >= GD_OP_VERSION_3_10_0) {
+        ret = snprintf(buf + total_len, sizeof(buf) - total_len, "%s=0\n",
+                       GF_TIER_ENABLED);
+        if (ret < 0 || ret >= sizeof(buf) - total_len) {
+            ret = -1;
+            goto out;
+        }
+        total_len += ret;
+    }
+
+    if ((conf->op_version >= GD_OP_VERSION_7_0) &&
+        volinfo->thin_arbiter_count) {
+        ret = snprintf(buf + total_len, sizeof(buf) - total_len, "%s=%d\n",
+                       GLUSTERD_STORE_KEY_VOL_THIN_ARBITER_CNT,
+                       volinfo->thin_arbiter_count);
+        if (ret < 0 || ret >= sizeof(buf) - total_len) {
+            ret = -1;
+            goto out;
+        }
+        total_len += ret;
+    }
+
+    ret = gf_store_save_items(fd, buf);
+    if (ret)
+        goto out;
+
+    ret = glusterd_volume_write_snap_details(fd, volinfo);
 
-        snprintf (buf, sizeof (buf), "%d", volinfo->tier_info.cold_brick_count);
-        ret = gf_store_save_value (fd, GLUSTERD_STORE_KEY_COLD_COUNT, buf);
-        if (ret)
-                goto out;
-
-        snprintf (buf, sizeof (buf), "%d",
-                  volinfo->tier_info.cold_replica_count);
-        ret = gf_store_save_value (fd,
-                                   GLUSTERD_STORE_KEY_COLD_REPLICA_COUNT,
-                                   buf);
-        if (ret)
-                goto out;
-
-        snprintf (buf, sizeof (buf), "%d", volinfo->tier_info.cold_disperse_count);
-        ret = gf_store_save_value (fd, GLUSTERD_STORE_KEY_COLD_DISPERSE_COUNT,
-                                   buf);
-        if (ret)
-                goto out;
-
-        snprintf (buf, sizeof (buf), "%d", volinfo->tier_info.hot_brick_count);
-        ret = gf_store_save_value (fd, GLUSTERD_STORE_KEY_HOT_COUNT,
-                                   buf);
-        if (ret)
-                goto out;
-
-        snprintf (buf, sizeof (buf), "%d", volinfo->tier_info.hot_replica_count);
-        ret = gf_store_save_value (fd, GLUSTERD_STORE_KEY_HOT_REPLICA_COUNT,
-                                   buf);
-        if (ret)
-                goto out;
-
-        snprintf (buf, sizeof (buf), "%d", volinfo->tier_info.hot_type);
-        ret = gf_store_save_value (fd, GLUSTERD_STORE_KEY_HOT_TYPE, buf);
-        if (ret)
-                goto out;
-
-        snprintf (buf, sizeof (buf), "%d", volinfo->tier_info.cold_type);
-        ret = gf_store_save_value (fd, GLUSTERD_STORE_KEY_COLD_TYPE, buf);
-        if (ret)
-                goto out;
-
- out:
-        return ret;
+out:
+    if (ret)
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_VOL_VALS_WRITE_FAIL,
+               "Unable to write volume "
+               "values for %s",
+               volinfo->volname);
+    return ret;
 }
 
-int32_t
-glusterd_volume_exclude_options_write (int fd, glusterd_volinfo_t *volinfo)
+static void
+glusterd_store_voldirpath_set(glusterd_volinfo_t *volinfo, char *voldirpath)
 {
-        char        *str            = NULL;
-        char         buf[PATH_MAX]  = "";
-        int32_t      ret            = -1;
-        xlator_t    *this           = NULL;
-
-        this = THIS;
-        GF_ASSERT (this);
-        GF_ASSERT (fd > 0);
-        GF_ASSERT (volinfo);
-
-        snprintf (buf, sizeof (buf), "%d", volinfo->type);
-        ret = gf_store_save_value (fd, GLUSTERD_STORE_KEY_VOL_TYPE, buf);
-        if (ret)
-                goto out;
+    glusterd_conf_t *priv = NULL;
 
-        snprintf (buf, sizeof (buf), "%d", volinfo->brick_count);
-        ret = gf_store_save_value (fd, GLUSTERD_STORE_KEY_VOL_COUNT, buf);
-        if (ret)
-                goto out;
-
-        snprintf (buf, sizeof (buf), "%d", volinfo->status);
-        ret = gf_store_save_value (fd, GLUSTERD_STORE_KEY_VOL_STATUS, buf);
-        if (ret)
-                goto out;
-
-        snprintf (buf, sizeof (buf), "%d", volinfo->sub_count);
-        ret = gf_store_save_value (fd, GLUSTERD_STORE_KEY_VOL_SUB_COUNT, buf);
-        if (ret)
-                goto out;
-
-        snprintf (buf, sizeof (buf), "%d", volinfo->stripe_count);
-        ret = gf_store_save_value (fd, GLUSTERD_STORE_KEY_VOL_STRIPE_CNT, buf);
-        if (ret)
-                goto out;
+    GF_ASSERT(volinfo);
+    priv = THIS->private;
+    GF_ASSERT(priv);
 
-        snprintf (buf, sizeof (buf), "%d", volinfo->replica_count);
-        ret = gf_store_save_value (fd, GLUSTERD_STORE_KEY_VOL_REPLICA_CNT,
-                                   buf);
-        if (ret)
-                goto out;
-
-        snprintf (buf, sizeof (buf), "%d", volinfo->disperse_count);
-        ret = gf_store_save_value (fd, GLUSTERD_STORE_KEY_VOL_DISPERSE_CNT,
-                                   buf);
-        if (ret)
-                goto out;
-
-        snprintf (buf, sizeof (buf), "%d", volinfo->redundancy_count);
-        ret = gf_store_save_value (fd, GLUSTERD_STORE_KEY_VOL_REDUNDANCY_CNT,
-                                   buf);
-        if (ret)
-                goto out;
-
-        snprintf (buf, sizeof (buf), "%d", volinfo->version);
-        ret = gf_store_save_value (fd, GLUSTERD_STORE_KEY_VOL_VERSION, buf);
-        if (ret)
-                goto out;
-
-        snprintf (buf, sizeof (buf), "%d", volinfo->transport_type);
-        ret = gf_store_save_value (fd, GLUSTERD_STORE_KEY_VOL_TRANSPORT, buf);
-        if (ret)
-                goto out;
-
-        ret = gf_store_save_value (fd, GLUSTERD_STORE_KEY_VOL_ID,
-                                   uuid_utoa (volinfo->volume_id));
-        if (ret)
-                goto out;
-
-        str = glusterd_auth_get_username (volinfo);
-        if (str) {
-                ret = gf_store_save_value (fd, GLUSTERD_STORE_KEY_USERNAME,
-                                           str);
-                if (ret)
-                goto out;
-        }
-
-        str = glusterd_auth_get_password (volinfo);
-        if (str) {
-                ret = gf_store_save_value (fd, GLUSTERD_STORE_KEY_PASSWORD,
-                                           str);
-                if (ret)
-                        goto out;
-        }
-
-        snprintf (buf, sizeof (buf), "%d", volinfo->op_version);
-        ret = gf_store_save_value (fd, GLUSTERD_STORE_KEY_VOL_OP_VERSION, buf);
-        if (ret)
-                goto out;
-
-        snprintf (buf, sizeof (buf), "%d", volinfo->client_op_version);
-        ret = gf_store_save_value (fd, GLUSTERD_STORE_KEY_VOL_CLIENT_OP_VERSION,
-                                   buf);
-        if (ret)
-                goto out;
-        if (volinfo->caps) {
-                snprintf (buf, sizeof (buf), "%d", volinfo->caps);
-                ret = gf_store_save_value (fd, GLUSTERD_STORE_KEY_VOL_CAPS,
-                                           buf);
-                if (ret)
-                        goto out;
-        }
-
-        ret = glusterd_volume_write_tier_details (fd, volinfo);
-
-        ret = glusterd_volume_write_snap_details (fd, volinfo);
-
-out:
-        if (ret)
-                gf_log (this->name, GF_LOG_ERROR, "Unable to write volume "
-                        "values for %s", volinfo->volname);
-        return ret;
+    GLUSTERD_GET_VOLUME_DIR(voldirpath, volinfo, priv);
 }
 
 static void
-glusterd_store_voldirpath_set (glusterd_volinfo_t *volinfo, char *voldirpath,
-                               size_t len)
+glusterd_store_piddirpath_set(glusterd_volinfo_t *volinfo, char *piddirpath)
 {
-        glusterd_conf_t         *priv = NULL;
+    glusterd_conf_t *priv = NULL;
 
-        GF_ASSERT (volinfo);
-        priv = THIS->private;
-        GF_ASSERT (priv);
+    GF_ASSERT(volinfo);
+    priv = THIS->private;
+    GF_ASSERT(priv);
 
-        GLUSTERD_GET_VOLUME_DIR (voldirpath, volinfo, priv);
+    GLUSTERD_GET_VOLUME_PID_DIR(piddirpath, volinfo, priv);
 }
 
 static int32_t
-glusterd_store_create_volume_dir (glusterd_volinfo_t *volinfo)
+glusterd_store_create_volume_dirs(glusterd_volinfo_t *volinfo)
 {
-        int32_t                 ret = -1;
-        char                    voldirpath[PATH_MAX] = {0,};
-
-        GF_ASSERT (volinfo);
+    int32_t ret = -1;
+    char dirpath[PATH_MAX] = {
+        0,
+    };
 
-        glusterd_store_voldirpath_set (volinfo, voldirpath,
-                                       sizeof (voldirpath));
-        ret = gf_store_mkdir (voldirpath);
+    GF_ASSERT(volinfo);
 
-        gf_log (THIS->name, GF_LOG_DEBUG, "Returning with %d", ret);
-        return ret;
-}
+    glusterd_store_voldirpath_set(volinfo, dirpath);
+    ret = gf_store_mkdir(dirpath);
+    if (ret)
+        goto out;
 
-int32_t
-glusterd_store_create_snap_dir (glusterd_snap_t *snap)
-{
-        int32_t            ret                   = -1;
-        char               snapdirpath[PATH_MAX] = {0,};
-        glusterd_conf_t   *priv                  = NULL;
+    glusterd_store_piddirpath_set(volinfo, dirpath);
+    ret = gf_store_mkdir(dirpath);
+    if (ret)
+        goto out;
 
-        priv = THIS->private;
-        GF_ASSERT (priv);
-        GF_ASSERT (snap);
-
-        GLUSTERD_GET_SNAP_DIR (snapdirpath, snap, priv);
-
-        ret = mkdir_p (snapdirpath, 0755, _gf_true);
-        if (ret) {
-                gf_log (THIS->name, GF_LOG_ERROR, "Failed to create snaps dir "
-                        "%s", snapdirpath);
-        }
-        return ret;
+out:
+    gf_msg_debug(THIS->name, 0, "Returning with %d", ret);
+    return ret;
 }
 
 int32_t
-glusterd_store_volinfo_write (int fd, glusterd_volinfo_t *volinfo)
+glusterd_store_create_snap_dir(glusterd_snap_t *snap)
 {
-        int32_t                         ret = -1;
-        gf_store_handle_t              *shandle = NULL;
-        GF_ASSERT (fd > 0);
-        GF_ASSERT (volinfo);
-        GF_ASSERT (volinfo->shandle);
-
-        shandle = volinfo->shandle;
-        ret = glusterd_volume_exclude_options_write (fd, volinfo);
-        if (ret)
-                goto out;
-
-        shandle->fd = fd;
-        dict_foreach (volinfo->dict, _storeopts, shandle);
-
-        dict_foreach (volinfo->gsync_slaves, _storeslaves, shandle);
-        shandle->fd = 0;
-out:
-        gf_log (THIS->name, GF_LOG_DEBUG, "Returning %d", ret);
-        return ret;
+    int32_t ret = -1;
+    char snapdirpath[PATH_MAX] = {
+        0,
+    };
+    glusterd_conf_t *priv = NULL;
+
+    priv = THIS->private;
+    GF_ASSERT(priv);
+    GF_ASSERT(snap);
+
+    GLUSTERD_GET_SNAP_DIR(snapdirpath, snap, priv);
+
+    ret = mkdir_p(snapdirpath, 0755, _gf_true);
+    if (ret) {
+        gf_msg(THIS->name, GF_LOG_ERROR, errno, GD_MSG_CREATE_DIR_FAILED,
+               "Failed to create snaps dir "
+               "%s",
+               snapdirpath);
+    }
+    return ret;
 }
 
-int32_t
-glusterd_store_snapinfo_write (glusterd_snap_t *snap)
+static int32_t
+glusterd_store_volinfo_write(int fd, glusterd_volinfo_t *volinfo)
 {
-        int32_t  ret            = -1;
-        int      fd             = 0;
-        char     buf[PATH_MAX]  = "";
-
-        GF_ASSERT (snap);
-
-        fd = gf_store_mkstemp (snap->shandle);
-        if (fd <= 0)
-               goto out;
-
-        ret = gf_store_save_value (fd, GLUSTERD_STORE_KEY_SNAP_ID,
-                                   uuid_utoa (snap->snap_id));
-        if (ret)
-                goto out;
-
-        snprintf (buf, sizeof (buf), "%d", snap->snap_status);
-        ret = gf_store_save_value (fd, GLUSTERD_STORE_KEY_SNAP_STATUS, buf);
-        if (ret)
-                goto out;
-
-        snprintf (buf, sizeof (buf), "%d", snap->snap_restored);
-        ret = gf_store_save_value (fd, GLUSTERD_STORE_KEY_SNAP_RESTORED, buf);
-        if (ret)
-                goto out;
-
-        if (snap->description) {
-                ret = gf_store_save_value (fd, GLUSTERD_STORE_KEY_SNAP_DESC,
-                                           snap->description);
-                if (ret)
-                        goto out;
+    int32_t ret = -1;
+    gf_store_handle_t *shandle = NULL;
+    GF_ASSERT(fd > 0);
+    GF_ASSERT(volinfo);
+    GF_ASSERT(volinfo->shandle);
+    xlator_t *this = NULL;
+    glusterd_volinfo_data_store_t *dict_data = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+
+    shandle = volinfo->shandle;
+
+    dict_data = GF_CALLOC(1, sizeof(glusterd_volinfo_data_store_t),
+                          gf_gld_mt_volinfo_dict_data_t);
+    if (dict_data == NULL) {
+        gf_smsg(this->name, GF_LOG_ERROR, 0, GD_MSG_NO_MEMORY, NULL);
+        return -1;
+    }
+
+    ret = glusterd_volume_exclude_options_write(fd, volinfo);
+    if (ret) {
+        goto out;
+    }
+
+    dict_data->shandle = shandle;
+    dict_data->key_check = 1;
+
+    shandle->fd = fd;
+    dict_foreach(volinfo->dict, _storeopts, (void *)dict_data);
+
+    dict_data->key_check = 0;
+    dict_foreach(volinfo->gsync_slaves, _storeopts, (void *)dict_data);
+
+    if (dict_data->buffer_len > 0) {
+        ret = gf_store_save_items(fd, dict_data->buffer);
+        if (ret) {
+            gf_smsg(this->name, GF_LOG_ERROR, 0, GD_MSG_FILE_OP_FAILED, NULL);
+            goto out;
         }
+    }
 
-        snprintf (buf, sizeof (buf), "%ld", snap->time_stamp);
-        ret = gf_store_save_value (fd, GLUSTERD_STORE_KEY_SNAP_TIMESTAMP, buf);
-
+    shandle->fd = 0;
 out:
-        gf_log (THIS->name, GF_LOG_DEBUG, "Returning %d", ret);
-        return ret;
+    GF_FREE(dict_data);
+    gf_msg_debug(this->name, 0, "Returning %d", ret);
+    return ret;
 }
 
-static void
-glusterd_store_rbstatepath_set (glusterd_volinfo_t *volinfo, char *rbstatepath,
-                                size_t len)
+static int32_t
+glusterd_store_snapinfo_write(glusterd_snap_t *snap)
 {
-        char    voldirpath[PATH_MAX] = {0,};
-        GF_ASSERT (volinfo);
-        GF_ASSERT (rbstatepath);
-        GF_ASSERT (len <= PATH_MAX);
-
-        glusterd_store_voldirpath_set (volinfo, voldirpath,
-                                       sizeof (voldirpath));
-        snprintf (rbstatepath, len, "%s/%s", voldirpath,
-                  GLUSTERD_VOLUME_RBSTATE_FILE);
+    int32_t ret = -1;
+    int fd = 0;
+    char buf[PATH_MAX];
+    uint total_len = 0;
+
+    GF_ASSERT(snap);
+
+    fd = gf_store_mkstemp(snap->shandle);
+    if (fd <= 0)
+        goto out;
+
+    ret = snprintf(buf + total_len, sizeof(buf) - total_len,
+                   "%s=%s\n%s=%d\n%s=%d\n", GLUSTERD_STORE_KEY_SNAP_ID,
+                   uuid_utoa(snap->snap_id), GLUSTERD_STORE_KEY_SNAP_STATUS,
+                   snap->snap_status, GLUSTERD_STORE_KEY_SNAP_RESTORED,
+                   snap->snap_restored);
+    if (ret < 0 || ret >= sizeof(buf) - total_len) {
+        ret = -1;
+        goto out;
+    }
+    total_len += ret;
+
+    if (snap->description) {
+        ret = snprintf(buf + total_len, sizeof(buf) - total_len, "%s=%s\n",
+                       GLUSTERD_STORE_KEY_SNAP_DESC, snap->description);
+        if (ret < 0 || ret >= sizeof(buf) - total_len) {
+            ret = -1;
+            goto out;
+        }
+        total_len += ret;
+    }
+
+    ret = snprintf(buf + total_len, sizeof(buf) - total_len, "%s=%ld\n",
+                   GLUSTERD_STORE_KEY_SNAP_TIMESTAMP, snap->time_stamp);
+    if (ret < 0 || ret >= sizeof(buf) - total_len) {
+        ret = -1;
+        goto out;
+    }
+    ret = gf_store_save_items(fd, buf);
+
+out:
+    gf_msg_debug(THIS->name, 0, "Returning %d", ret);
+    return ret;
 }
 
 static void
-glusterd_store_volfpath_set (glusterd_volinfo_t *volinfo, char *volfpath,
-                             size_t len)
+glusterd_store_volfpath_set(glusterd_volinfo_t *volinfo, char *volfpath,
+                            size_t len)
 {
-        char    voldirpath[PATH_MAX] = {0,};
-        GF_ASSERT (volinfo);
-        GF_ASSERT (volfpath);
-        GF_ASSERT (len <= PATH_MAX);
-
-        glusterd_store_voldirpath_set (volinfo, voldirpath,
-                                       sizeof (voldirpath));
-        snprintf (volfpath, len, "%s/%s", voldirpath, GLUSTERD_VOLUME_INFO_FILE);
+    char voldirpath[PATH_MAX] = {
+        0,
+    };
+    GF_ASSERT(volinfo);
+    GF_ASSERT(volfpath);
+    GF_ASSERT(len <= PATH_MAX);
+
+    glusterd_store_voldirpath_set(volinfo, voldirpath);
+    snprintf(volfpath, len, "%s/%s", voldirpath, GLUSTERD_VOLUME_INFO_FILE);
 }
 
 static void
-glusterd_store_node_state_path_set (glusterd_volinfo_t *volinfo,
-                                    char *node_statepath, size_t len)
+glusterd_store_node_state_path_set(glusterd_volinfo_t *volinfo,
+                                   char *node_statepath, size_t len)
 {
-        char    voldirpath[PATH_MAX] = {0,};
-        GF_ASSERT (volinfo);
-        GF_ASSERT (node_statepath);
-        GF_ASSERT (len <= PATH_MAX);
-
-        glusterd_store_voldirpath_set (volinfo, voldirpath,
-                                       sizeof (voldirpath));
-        snprintf (node_statepath, len, "%s/%s", voldirpath,
-                  GLUSTERD_NODE_STATE_FILE);
+    char voldirpath[PATH_MAX] = {
+        0,
+    };
+    GF_ASSERT(volinfo);
+    GF_ASSERT(node_statepath);
+    GF_ASSERT(len <= PATH_MAX);
+
+    glusterd_store_voldirpath_set(volinfo, voldirpath);
+    snprintf(node_statepath, len, "%s/%s", voldirpath,
+             GLUSTERD_NODE_STATE_FILE);
 }
 
 static void
-glusterd_store_quota_conf_path_set (glusterd_volinfo_t *volinfo,
-                                    char *quota_conf_path, size_t len)
+glusterd_store_quota_conf_path_set(glusterd_volinfo_t *volinfo,
+                                   char *quota_conf_path, size_t len)
 {
-        char    voldirpath[PATH_MAX] = {0,};
-        GF_ASSERT (volinfo);
-        GF_ASSERT (quota_conf_path);
-        GF_ASSERT (len <= PATH_MAX);
-
-        glusterd_store_voldirpath_set (volinfo, voldirpath,
-                                       sizeof (voldirpath));
-        snprintf (quota_conf_path, len, "%s/%s", voldirpath,
-                  GLUSTERD_VOLUME_QUOTA_CONFIG);
+    char voldirpath[PATH_MAX] = {
+        0,
+    };
+    GF_ASSERT(volinfo);
+    GF_ASSERT(quota_conf_path);
+    GF_ASSERT(len <= PATH_MAX);
+
+    glusterd_store_voldirpath_set(volinfo, voldirpath);
+    snprintf(quota_conf_path, len, "%s/%s", voldirpath,
+             GLUSTERD_VOLUME_QUOTA_CONFIG);
 }
 
 static void
-glusterd_store_missed_snaps_list_path_set (char *missed_snaps_list,
-                                           size_t len)
+glusterd_store_missed_snaps_list_path_set(char *missed_snaps_list, size_t len)
 {
-        glusterd_conf_t *priv = NULL;
+    glusterd_conf_t *priv = NULL;
 
-        priv = THIS->private;
-        GF_ASSERT (priv);
-        GF_ASSERT (missed_snaps_list);
-        GF_ASSERT (len <= PATH_MAX);
+    priv = THIS->private;
+    GF_ASSERT(priv);
+    GF_ASSERT(missed_snaps_list);
+    GF_ASSERT(len <= PATH_MAX);
 
-        snprintf (missed_snaps_list, len, "%s/snaps/"
-                  GLUSTERD_MISSED_SNAPS_LIST_FILE, priv->workdir);
+    snprintf(missed_snaps_list, len,
+             "%s/snaps/" GLUSTERD_MISSED_SNAPS_LIST_FILE, priv->workdir);
 }
 
 static void
-glusterd_store_snapfpath_set (glusterd_snap_t *snap, char *snap_fpath,
-                              size_t len)
-{
-        glusterd_conf_t *priv = NULL;
-        priv = THIS->private;
-        GF_ASSERT (priv);
-        GF_ASSERT (snap);
-        GF_ASSERT (snap_fpath);
-        GF_ASSERT (len <= PATH_MAX);
-
-        snprintf (snap_fpath, len, "%s/snaps/%s/%s", priv->workdir,
-                  snap->snapname, GLUSTERD_SNAP_INFO_FILE);
-}
-
-int32_t
-glusterd_store_create_rbstate_shandle_on_absence (glusterd_volinfo_t *volinfo)
+glusterd_store_snapfpath_set(glusterd_snap_t *snap, char *snap_fpath,
+                             size_t len)
 {
-        char            rbstatepath[PATH_MAX] = {0};
-        int32_t         ret                   = 0;
-
-        GF_ASSERT (volinfo);
-
-        glusterd_store_rbstatepath_set (volinfo, rbstatepath, sizeof (rbstatepath));
-        ret = gf_store_handle_create_on_absence (&volinfo->rb_shandle,
-                                                 rbstatepath);
-        return ret;
+    glusterd_conf_t *priv = NULL;
+    priv = THIS->private;
+    GF_ASSERT(priv);
+    GF_ASSERT(snap);
+    GF_ASSERT(snap_fpath);
+    GF_ASSERT(len <= PATH_MAX);
+
+    snprintf(snap_fpath, len, "%s/snaps/%s/%s", priv->workdir, snap->snapname,
+             GLUSTERD_SNAP_INFO_FILE);
 }
 
 int32_t
-glusterd_store_create_vol_shandle_on_absence (glusterd_volinfo_t *volinfo)
+glusterd_store_create_vol_shandle_on_absence(glusterd_volinfo_t *volinfo)
 {
-        char            volfpath[PATH_MAX] = {0};
-        int32_t         ret = 0;
+    char volfpath[PATH_MAX] = {0};
+    int32_t ret = 0;
 
-        GF_ASSERT (volinfo);
+    GF_ASSERT(volinfo);
 
-        glusterd_store_volfpath_set (volinfo, volfpath, sizeof (volfpath));
-        ret = gf_store_handle_create_on_absence (&volinfo->shandle, volfpath);
-        return ret;
+    glusterd_store_volfpath_set(volinfo, volfpath, sizeof(volfpath));
+    ret = gf_store_handle_create_on_absence(&volinfo->shandle, volfpath);
+    return ret;
 }
 
 int32_t
-glusterd_store_create_nodestate_sh_on_absence (glusterd_volinfo_t *volinfo)
+glusterd_store_create_nodestate_sh_on_absence(glusterd_volinfo_t *volinfo)
 {
-        char            node_state_path[PATH_MAX] = {0};
-        int32_t         ret                   = 0;
+    char node_state_path[PATH_MAX] = {0};
+    int32_t ret = 0;
 
-        GF_ASSERT (volinfo);
+    GF_ASSERT(volinfo);
 
-        glusterd_store_node_state_path_set (volinfo, node_state_path,
-                                            sizeof (node_state_path));
-        ret =
-          gf_store_handle_create_on_absence (&volinfo->node_state_shandle,
-                                             node_state_path);
+    glusterd_store_node_state_path_set(volinfo, node_state_path,
+                                       sizeof(node_state_path));
+    ret = gf_store_handle_create_on_absence(&volinfo->node_state_shandle,
+                                            node_state_path);
 
-        return ret;
+    return ret;
 }
 
 int32_t
-glusterd_store_create_quota_conf_sh_on_absence (glusterd_volinfo_t *volinfo)
+glusterd_store_create_quota_conf_sh_on_absence(glusterd_volinfo_t *volinfo)
 {
-        char            quota_conf_path[PATH_MAX] = {0};
-        int32_t         ret                       = 0;
+    char quota_conf_path[PATH_MAX] = {0};
+    int32_t ret = 0;
 
-        GF_ASSERT (volinfo);
+    GF_ASSERT(volinfo);
 
-        glusterd_store_quota_conf_path_set (volinfo, quota_conf_path,
-                                            sizeof (quota_conf_path));
-        ret =
-          gf_store_handle_create_on_absence (&volinfo->quota_conf_shandle,
-                                             quota_conf_path);
+    glusterd_store_quota_conf_path_set(volinfo, quota_conf_path,
+                                       sizeof(quota_conf_path));
+    ret = gf_store_handle_create_on_absence(&volinfo->quota_conf_shandle,
+                                            quota_conf_path);
 
-        return ret;
+    return ret;
 }
 
 static int32_t
-glusterd_store_create_missed_snaps_list_shandle_on_absence ()
+glusterd_store_create_missed_snaps_list_shandle_on_absence()
 {
-        char               missed_snaps_list[PATH_MAX] = "";
-        int32_t            ret                         = -1;
-        glusterd_conf_t   *priv                        = NULL;
-        xlator_t          *this                        = NULL;
-
-        this = THIS;
-        GF_ASSERT (this);
+    char missed_snaps_list[PATH_MAX] = "";
+    int32_t ret = -1;
+    glusterd_conf_t *priv = NULL;
+    xlator_t *this = NULL;
 
-        priv = this->private;
-        GF_ASSERT (priv);
+    this = THIS;
+    GF_ASSERT(this);
 
-        glusterd_store_missed_snaps_list_path_set (missed_snaps_list,
-                                                   sizeof(missed_snaps_list));
+    priv = this->private;
+    GF_ASSERT(priv);
 
-        ret = gf_store_handle_create_on_absence
-                                            (&priv->missed_snaps_list_shandle,
-                                             missed_snaps_list);
-        return ret;
-}
-
-int32_t
-glusterd_store_create_snap_shandle_on_absence (glusterd_snap_t *snap)
-{
-        char            snapfpath[PATH_MAX] = {0};
-        int32_t         ret = 0;
-
-        GF_ASSERT (snap);
+    glusterd_store_missed_snaps_list_path_set(missed_snaps_list,
+                                              sizeof(missed_snaps_list));
 
-        glusterd_store_snapfpath_set (snap, snapfpath, sizeof (snapfpath));
-        ret = gf_store_handle_create_on_absence (&snap->shandle, snapfpath);
-        return ret;
+    ret = gf_store_handle_create_on_absence(&priv->missed_snaps_list_shandle,
+                                            missed_snaps_list);
+    return ret;
 }
 
 int32_t
-glusterd_store_brickinfos (glusterd_volinfo_t *volinfo, int vol_fd)
+glusterd_store_create_snap_shandle_on_absence(glusterd_snap_t *snap)
 {
-        int32_t                 ret = 0;
-        glusterd_brickinfo_t    *brickinfo = NULL;
-        int32_t                 brick_count = 0;
+    char snapfpath[PATH_MAX] = {0};
+    int32_t ret = 0;
 
-        GF_ASSERT (volinfo);
+    GF_ASSERT(snap);
 
-        cds_list_for_each_entry (brickinfo, &volinfo->bricks, brick_list) {
-                ret = glusterd_store_brickinfo (volinfo, brickinfo,
-                                brick_count, vol_fd);
-                if (ret)
-                        goto out;
-                brick_count++;
-        }
-out:
-        gf_log (THIS->name, GF_LOG_DEBUG, "Returning %d", ret);
-        return ret;
+    glusterd_store_snapfpath_set(snap, snapfpath, sizeof(snapfpath));
+    ret = gf_store_handle_create_on_absence(&snap->shandle, snapfpath);
+    return ret;
 }
 
-int32_t
-glusterd_store_rbstate_write (int fd, glusterd_volinfo_t *volinfo)
+static int32_t
+glusterd_store_brickinfos(glusterd_volinfo_t *volinfo, int vol_fd)
 {
-        int     ret             = -1;
-        int     port            = 0;
-        char    buf[PATH_MAX]   = {0, };
-
-        GF_ASSERT (fd > 0);
-        GF_ASSERT (volinfo);
-
-        snprintf (buf, sizeof (buf), "%d", volinfo->rep_brick.rb_status);
-        ret = gf_store_save_value (fd, GLUSTERD_STORE_KEY_RB_STATUS, buf);
+    int32_t ret = 0;
+    glusterd_brickinfo_t *brickinfo = NULL;
+    glusterd_brickinfo_t *ta_brickinfo = NULL;
+    int32_t brick_count = 0;
+    int32_t ta_brick_count = 0;
+
+    GF_ASSERT(volinfo);
+
+    cds_list_for_each_entry(brickinfo, &volinfo->bricks, brick_list)
+    {
+        ret = glusterd_store_brickinfo(volinfo, brickinfo, brick_count, vol_fd,
+                                       0);
         if (ret)
-                goto out;
-
-        if (volinfo->rep_brick.rb_status > GF_RB_STATUS_NONE) {
-
-                snprintf (buf, sizeof (buf), "%s:%s",
-                          volinfo->rep_brick.src_brick->hostname,
-                          volinfo->rep_brick.src_brick->path);
-                ret = gf_store_save_value (fd, GLUSTERD_STORE_KEY_RB_SRC_BRICK,
-                                           buf);
-                if (ret)
-                        goto out;
-
-                snprintf (buf, sizeof (buf), "%s:%s",
-                          volinfo->rep_brick.dst_brick->hostname,
-                          volinfo->rep_brick.dst_brick->path);
-                ret = gf_store_save_value (fd, GLUSTERD_STORE_KEY_RB_DST_BRICK,
-                                           buf);
-                if (ret)
-                        goto out;
-
-                switch (volinfo->transport_type) {
-                        case GF_TRANSPORT_RDMA:
-                                port = volinfo->rep_brick.dst_brick->rdma_port;
-                                break;
-
-                        case GF_TRANSPORT_TCP:
-                        case GF_TRANSPORT_BOTH_TCP_RDMA:
-                                port = volinfo->rep_brick.dst_brick->port;
-                                break;
-                }
-
-                snprintf (buf, sizeof (buf), "%d", port);
-                ret = gf_store_save_value (fd, GLUSTERD_STORE_KEY_RB_DST_PORT,
-                                           buf);
-                if (ret)
-                        goto out;
-                gf_uuid_unparse (volinfo->rep_brick.rb_id, buf);
-                ret = gf_store_save_value (fd, GF_REPLACE_BRICK_TID_KEY, buf);
-        }
-
-        ret = 0;
-out:
-        gf_log (THIS->name, GF_LOG_DEBUG, "Returning %d", ret);
-        return ret;
-}
-
-int32_t
-glusterd_store_perform_rbstate_store (glusterd_volinfo_t *volinfo)
-{
-        int                         fd = -1;
-        int32_t                     ret = -1;
-        GF_ASSERT (volinfo);
-
-        fd = gf_store_mkstemp (volinfo->rb_shandle);
-        if (fd <= 0) {
-                ret = -1;
-                goto out;
-        }
-
-        ret = glusterd_store_rbstate_write (fd, volinfo);
+            goto out;
+        brick_count++;
+    }
+    if (volinfo->thin_arbiter_count == 1) {
+        ta_brickinfo = list_first_entry(&volinfo->ta_bricks,
+                                        glusterd_brickinfo_t, brick_list);
+        ret = glusterd_store_brickinfo(volinfo, ta_brickinfo, ta_brick_count,
+                                       vol_fd, 1);
         if (ret)
-                goto out;
-
-        ret = gf_store_rename_tmppath (volinfo->rb_shandle);
-        if (ret)
-                goto out;
+            goto out;
+    }
 
 out:
-        if (ret && (fd > 0))
-                gf_store_unlink_tmppath (volinfo->rb_shandle);
-        gf_log (THIS->name, GF_LOG_DEBUG, "Returning %d", ret);
-        return ret;
-}
-
-int
-_gd_store_rebalance_dict (dict_t *dict, char *key, data_t *value, void *data)
-{
-        int             ret = -1;
-        int             fd = 0;
-
-        fd = *(int *)data;
-
-        ret = gf_store_save_value (fd, key, value->data);
-
-        return ret;
+    gf_msg_debug(THIS->name, 0, "Returning %d", ret);
+    return ret;
 }
 
 int32_t
-glusterd_store_node_state_write (int fd, glusterd_volinfo_t *volinfo)
+glusterd_store_node_state_write(int fd, glusterd_volinfo_t *volinfo)
 {
-        int     ret             = -1;
-        char    buf[PATH_MAX]   = {0, };
+    int ret = -1;
+    char buf[PATH_MAX];
+    char uuid[UUID_SIZE + 1];
+    uint total_len = 0;
+    glusterd_volinfo_data_store_t *dict_data = NULL;
+    gf_store_handle_t shandle;
+    xlator_t *this = NULL;
 
-        GF_ASSERT (fd > 0);
-        GF_ASSERT (volinfo);
+    this = THIS;
+    GF_ASSERT(this);
 
-        if (volinfo->rebal.defrag_cmd == GF_DEFRAG_CMD_STATUS) {
-                ret = 0;
-                goto out;
-        }
-
-        snprintf (buf, sizeof (buf), "%d", volinfo->rebal.defrag_cmd);
-        ret = gf_store_save_value (fd, GLUSTERD_STORE_KEY_VOL_DEFRAG, buf);
-        if (ret)
-                goto out;
-
-        snprintf (buf, sizeof (buf), "%d", volinfo->rebal.defrag_status);
-        ret = gf_store_save_value (fd, GLUSTERD_STORE_KEY_VOL_DEFRAG_STATUS,
-                                   buf);
-        if (ret)
-                goto out;
+    GF_ASSERT(fd > 0);
+    GF_ASSERT(volinfo);
 
-
-        snprintf (buf, sizeof (buf), "%d", volinfo->rebal.op);
-        ret = gf_store_save_value (fd, GLUSTERD_STORE_KEY_DEFRAG_OP, buf);
-        if (ret)
-                goto out;
-
-        gf_uuid_unparse (volinfo->rebal.rebalance_id, buf);
-        ret = gf_store_save_value (fd, GF_REBALANCE_TID_KEY, buf);
-        if (ret)
-                goto out;
-
-        if (volinfo->rebal.dict) {
-                dict_foreach (volinfo->rebal.dict, _gd_store_rebalance_dict,
-                              &fd);
-        }
+    if (volinfo->rebal.defrag_cmd == GF_DEFRAG_CMD_STATUS) {
+        ret = 0;
+        goto out;
+    }
+
+    gf_uuid_unparse(volinfo->rebal.rebalance_id, uuid);
+    ret = snprintf(buf + total_len, sizeof(buf) - total_len,
+                   "%s=%d\n%s=%d\n%s=%d\n%s=%s\n",
+                   GLUSTERD_STORE_KEY_VOL_DEFRAG, volinfo->rebal.defrag_cmd,
+                   GLUSTERD_STORE_KEY_VOL_DEFRAG_STATUS,
+                   volinfo->rebal.defrag_status, GLUSTERD_STORE_KEY_DEFRAG_OP,
+                   volinfo->rebal.op, GF_REBALANCE_TID_KEY, uuid);
+    if (ret < 0 || ret >= sizeof(buf) - total_len) {
+        ret = -1;
+        goto out;
+    }
+    total_len += ret;
+
+    ret = snprintf(
+        buf + total_len, sizeof(buf) - total_len,
+        "%s=%" PRIu64 "\n%s=%" PRIu64 "\n%s=%" PRIu64 "\n%s=%" PRIu64
+        "\n%s=%" PRIu64 "\n%s=%lf\n",
+        GLUSTERD_STORE_KEY_VOL_DEFRAG_REB_FILES, volinfo->rebal.rebalance_files,
+        GLUSTERD_STORE_KEY_VOL_DEFRAG_SIZE, volinfo->rebal.rebalance_data,
+        GLUSTERD_STORE_KEY_VOL_DEFRAG_SCANNED, volinfo->rebal.lookedup_files,
+        GLUSTERD_STORE_KEY_VOL_DEFRAG_FAILURES,
+        volinfo->rebal.rebalance_failures,
+        GLUSTERD_STORE_KEY_VOL_DEFRAG_SKIPPED, volinfo->rebal.skipped_files,
+        GLUSTERD_STORE_KEY_VOL_DEFRAG_RUN_TIME, volinfo->rebal.rebalance_time);
+    if (ret < 0 || ret >= sizeof(buf) - total_len) {
+        ret = -1;
+        goto out;
+    }
+
+    ret = gf_store_save_items(fd, buf);
+    if (ret) {
+        goto out;
+    }
+
+    if (volinfo->rebal.dict) {
+        dict_data = GF_CALLOC(1, sizeof(glusterd_volinfo_data_store_t),
+                              gf_gld_mt_volinfo_dict_data_t);
+        if (dict_data == NULL) {
+            gf_smsg(this->name, GF_LOG_ERROR, 0, GD_MSG_NO_MEMORY, NULL);
+            return -1;
+        }
+        dict_data->shandle = &shandle;
+        shandle.fd = fd;
+        dict_foreach(volinfo->rebal.dict, _storeopts, (void *)dict_data);
+        if (dict_data->buffer_len > 0) {
+            ret = gf_store_save_items(fd, dict_data->buffer);
+            if (ret) {
+                gf_smsg(this->name, GF_LOG_ERROR, 0, GD_MSG_FILE_OP_FAILED,
+                        NULL);
+                goto out;
+                ;
+            }
+        }
+    }
 out:
-        gf_log (THIS->name, GF_LOG_DEBUG, "Returning %d", ret);
-        return ret;
+    GF_FREE(dict_data);
+    gf_msg_debug(this->name, 0, "Returning %d", ret);
+    return ret;
 }
 
 int32_t
-glusterd_store_perform_node_state_store (glusterd_volinfo_t *volinfo)
+glusterd_store_perform_node_state_store(glusterd_volinfo_t *volinfo)
 {
-        int                         fd = -1;
-        int32_t                     ret = -1;
-        GF_ASSERT (volinfo);
+    int fd = -1;
+    int32_t ret = -1;
+    GF_ASSERT(volinfo);
 
-        fd = gf_store_mkstemp (volinfo->node_state_shandle);
-        if (fd <= 0) {
-                ret = -1;
-                goto out;
-        }
+    fd = gf_store_mkstemp(volinfo->node_state_shandle);
+    if (fd <= 0) {
+        ret = -1;
+        goto out;
+    }
 
-        ret = glusterd_store_node_state_write (fd, volinfo);
-        if (ret)
-                goto out;
+    ret = glusterd_store_node_state_write(fd, volinfo);
+    if (ret)
+        goto out;
 
-        ret = gf_store_rename_tmppath (volinfo->node_state_shandle);
-        if (ret)
-                goto out;
+    ret = gf_store_rename_tmppath(volinfo->node_state_shandle);
+    if (ret)
+        goto out;
 
 out:
-        if (ret && (fd > 0))
-                gf_store_unlink_tmppath (volinfo->node_state_shandle);
-        gf_log (THIS->name, GF_LOG_DEBUG, "Returning %d", ret);
-        return ret;
+    if (ret && (fd > 0))
+        gf_store_unlink_tmppath(volinfo->node_state_shandle);
+    gf_msg_debug(THIS->name, 0, "Returning %d", ret);
+    return ret;
 }
 
-int32_t
-glusterd_store_perform_volume_store (glusterd_volinfo_t *volinfo)
+static int32_t
+glusterd_store_perform_volume_store(glusterd_volinfo_t *volinfo)
 {
-        int                         fd = -1;
-        int32_t                     ret = -1;
-        GF_ASSERT (volinfo);
+    int fd = -1;
+    int32_t ret = -1;
+    GF_ASSERT(volinfo);
 
-        fd = gf_store_mkstemp (volinfo->shandle);
-        if (fd <= 0) {
-                ret = -1;
-                goto out;
-        }
+    fd = gf_store_mkstemp(volinfo->shandle);
+    if (fd <= 0) {
+        ret = -1;
+        goto out;
+    }
 
-        ret = glusterd_store_volinfo_write (fd, volinfo);
-        if (ret)
-                goto out;
+    ret = glusterd_store_volinfo_write(fd, volinfo);
+    if (ret)
+        goto out;
 
-        ret = glusterd_store_brickinfos (volinfo, fd);
-        if (ret)
-                goto out;
+    ret = glusterd_store_create_brick_dir(volinfo);
+    if (ret)
+        goto out;
+
+    ret = glusterd_store_brickinfos(volinfo, fd);
+    if (ret)
+        goto out;
 
 out:
-        if (ret && (fd > 0))
-                gf_store_unlink_tmppath (volinfo->shandle);
-        gf_log (THIS->name, GF_LOG_DEBUG, "Returning %d", ret);
-        return ret;
+    if (ret && (fd > 0))
+        gf_store_unlink_tmppath(volinfo->shandle);
+    gf_msg_debug(THIS->name, 0, "Returning %d", ret);
+    return ret;
 }
 
 void
-glusterd_perform_volinfo_version_action (glusterd_volinfo_t *volinfo,
-                                         glusterd_volinfo_ver_ac_t ac)
+glusterd_perform_volinfo_version_action(glusterd_volinfo_t *volinfo,
+                                        glusterd_volinfo_ver_ac_t ac)
 {
-        GF_ASSERT (volinfo);
+    GF_ASSERT(volinfo);
 
-        switch (ac) {
+    switch (ac) {
         case GLUSTERD_VOLINFO_VER_AC_NONE:
-        break;
+            break;
         case GLUSTERD_VOLINFO_VER_AC_INCREMENT:
-                volinfo->version++;
-        break;
+            volinfo->version++;
+            break;
         case GLUSTERD_VOLINFO_VER_AC_DECREMENT:
-                volinfo->version--;
-        break;
-        }
+            volinfo->version--;
+            break;
+    }
 }
 
 void
-glusterd_store_bricks_cleanup_tmp (glusterd_volinfo_t *volinfo)
+glusterd_store_bricks_cleanup_tmp(glusterd_volinfo_t *volinfo)
 {
-        glusterd_brickinfo_t    *brickinfo           = NULL;
+    glusterd_brickinfo_t *brickinfo = NULL;
 
-        GF_ASSERT (volinfo);
+    GF_ASSERT(volinfo);
 
-        cds_list_for_each_entry (brickinfo, &volinfo->bricks, brick_list) {
-                gf_store_unlink_tmppath (brickinfo->shandle);
-        }
+    cds_list_for_each_entry(brickinfo, &volinfo->bricks, brick_list)
+    {
+        gf_store_unlink_tmppath(brickinfo->shandle);
+    }
 }
 
 void
-glusterd_store_volume_cleanup_tmp (glusterd_volinfo_t *volinfo)
+glusterd_store_volume_cleanup_tmp(glusterd_volinfo_t *volinfo)
 {
-        GF_ASSERT (volinfo);
-
-        glusterd_store_bricks_cleanup_tmp (volinfo);
+    GF_ASSERT(volinfo);
 
-        gf_store_unlink_tmppath (volinfo->shandle);
+    glusterd_store_bricks_cleanup_tmp(volinfo);
 
-        gf_store_unlink_tmppath (volinfo->rb_shandle);
+    gf_store_unlink_tmppath(volinfo->shandle);
 
-        gf_store_unlink_tmppath (volinfo->node_state_shandle);
+    gf_store_unlink_tmppath(volinfo->node_state_shandle);
 
-        gf_store_unlink_tmppath (volinfo->snapd.handle);
+    gf_store_unlink_tmppath(volinfo->snapd.handle);
 }
 
 int32_t
-glusterd_store_brickinfos_atomic_update (glusterd_volinfo_t *volinfo)
+glusterd_store_brickinfos_atomic_update(glusterd_volinfo_t *volinfo)
 {
-        int                      ret            = -1;
-        glusterd_brickinfo_t    *brickinfo      = NULL;
+    int ret = -1;
+    glusterd_brickinfo_t *brickinfo = NULL;
+    glusterd_brickinfo_t *ta_brickinfo = NULL;
 
-        GF_ASSERT (volinfo);
+    GF_ASSERT(volinfo);
+
+    cds_list_for_each_entry(brickinfo, &volinfo->bricks, brick_list)
+    {
+        ret = gf_store_rename_tmppath(brickinfo->shandle);
+        if (ret)
+            goto out;
+    }
+
+    if (volinfo->thin_arbiter_count == 1) {
+        ta_brickinfo = list_first_entry(&volinfo->ta_bricks,
+                                        glusterd_brickinfo_t, brick_list);
+        ret = gf_store_rename_tmppath(ta_brickinfo->shandle);
+        if (ret)
+            goto out;
+    }
 
-        cds_list_for_each_entry (brickinfo, &volinfo->bricks, brick_list) {
-                ret = gf_store_rename_tmppath (brickinfo->shandle);
-                if (ret)
-                        goto out;
-        }
 out:
-        return ret;
+    return ret;
 }
 
 int32_t
-glusterd_store_volinfo_atomic_update (glusterd_volinfo_t *volinfo)
+glusterd_store_volinfo_atomic_update(glusterd_volinfo_t *volinfo)
 {
-        int ret = -1;
-        GF_ASSERT (volinfo);
+    int ret = -1;
+    GF_ASSERT(volinfo);
 
-        ret = gf_store_rename_tmppath (volinfo->shandle);
-        if (ret)
-                goto out;
+    ret = gf_store_rename_tmppath(volinfo->shandle);
+    if (ret)
+        goto out;
 
 out:
-        if (ret)
-                gf_log (THIS->name, GF_LOG_ERROR, "Couldn't rename "
-                        "temporary file(s): Reason %s", strerror (errno));
-        return ret;
+    if (ret)
+        gf_msg(THIS->name, GF_LOG_ERROR, errno, GD_MSG_FILE_OP_FAILED,
+               "Couldn't rename "
+               "temporary file(s)");
+    return ret;
 }
 
 int32_t
-glusterd_store_volume_atomic_update (glusterd_volinfo_t *volinfo)
+glusterd_store_volume_atomic_update(glusterd_volinfo_t *volinfo)
 {
-        int ret = -1;
-        GF_ASSERT (volinfo);
+    int ret = -1;
+    GF_ASSERT(volinfo);
 
-        ret = glusterd_store_brickinfos_atomic_update (volinfo);
-        if (ret)
-                goto out;
+    ret = glusterd_store_brickinfos_atomic_update(volinfo);
+    if (ret)
+        goto out;
 
-        ret = glusterd_store_volinfo_atomic_update (volinfo);
+    ret = glusterd_store_volinfo_atomic_update(volinfo);
 
 out:
-        return ret;
+    return ret;
 }
 
 int32_t
-glusterd_store_snap_atomic_update (glusterd_snap_t *snap)
+glusterd_store_snap_atomic_update(glusterd_snap_t *snap)
 {
-        int ret = -1;
-        GF_ASSERT (snap);
+    int ret = -1;
+    GF_ASSERT(snap);
 
-        ret = gf_store_rename_tmppath (snap->shandle);
-        if (ret)
-                gf_log (THIS->name, GF_LOG_ERROR, "Couldn't rename "
-                        "temporary file(s): Reason %s", strerror (errno));
+    ret = gf_store_rename_tmppath(snap->shandle);
+    if (ret)
+        gf_msg(THIS->name, GF_LOG_ERROR, errno, GD_MSG_FILE_OP_FAILED,
+               "Couldn't rename "
+               "temporary file(s)");
 
-        return ret;
+    return ret;
 }
 
 int32_t
-glusterd_store_snap (glusterd_snap_t *snap)
+glusterd_store_snap(glusterd_snap_t *snap)
 {
-        int32_t                 ret = -1;
-
-        GF_ASSERT (snap);
-
-        ret = glusterd_store_create_snap_dir (snap);
-        if (ret) {
-                gf_log (THIS->name, GF_LOG_ERROR, "Failed to create snap dir");
-                goto out;
-        }
-
-        ret = glusterd_store_create_snap_shandle_on_absence (snap);
-        if (ret) {
-                gf_log (THIS->name, GF_LOG_ERROR, "Failed to create snap info "
-                        "file");
-                goto out;
-        }
-
-        ret = glusterd_store_snapinfo_write (snap);
-        if (ret) {
-                gf_log (THIS->name, GF_LOG_ERROR, "Failed to write snap info");
-                goto out;
-        }
-
-        ret = glusterd_store_snap_atomic_update (snap);
-        if (ret) {
-                gf_log (THIS->name, GF_LOG_ERROR,"Failed to do automic update");
-                goto out;
-        }
+    int32_t ret = -1;
+
+    GF_ASSERT(snap);
+
+    ret = glusterd_store_create_snap_dir(snap);
+    if (ret) {
+        gf_msg(THIS->name, GF_LOG_ERROR, 0, GD_MSG_SNAPDIR_CREATE_FAIL,
+               "Failed to create snap dir");
+        goto out;
+    }
+
+    ret = glusterd_store_create_snap_shandle_on_absence(snap);
+    if (ret) {
+        gf_msg(THIS->name, GF_LOG_ERROR, 0, GD_MSG_SNAPINFO_CREATE_FAIL,
+               "Failed to create snap info "
+               "file");
+        goto out;
+    }
+
+    ret = glusterd_store_snapinfo_write(snap);
+    if (ret) {
+        gf_msg(THIS->name, GF_LOG_ERROR, 0, GD_MSG_SNAPINFO_WRITE_FAIL,
+               "Failed to write snap info");
+        goto out;
+    }
+
+    ret = glusterd_store_snap_atomic_update(snap);
+    if (ret) {
+        gf_msg(THIS->name, GF_LOG_ERROR, 0, GD_MSG_SNAP_AUTOMIC_UPDATE_FAIL,
+               "Failed to do automic update");
+        goto out;
+    }
 
 out:
-        if (ret && snap->shandle)
-                gf_store_unlink_tmppath (snap->shandle);
+    if (ret && snap->shandle)
+        gf_store_unlink_tmppath(snap->shandle);
 
-        gf_log (THIS->name, GF_LOG_TRACE, "Returning %d", ret);
-        return ret;
+    gf_msg_trace(THIS->name, 0, "Returning %d", ret);
+    return ret;
 }
 
 int32_t
-glusterd_store_volinfo (glusterd_volinfo_t *volinfo, glusterd_volinfo_ver_ac_t ac)
+glusterd_store_volinfo(glusterd_volinfo_t *volinfo,
+                       glusterd_volinfo_ver_ac_t ac)
 {
-        int32_t                 ret = -1;
-
-        GF_ASSERT (volinfo);
-
-        glusterd_perform_volinfo_version_action (volinfo, ac);
-        ret = glusterd_store_create_volume_dir (volinfo);
+    int32_t ret = -1;
+    glusterfs_ctx_t *ctx = NULL;
+    xlator_t *this = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+    ctx = this->ctx;
+    GF_ASSERT(ctx);
+    GF_ASSERT(volinfo);
+
+    pthread_mutex_lock(&ctx->cleanup_lock);
+    pthread_mutex_lock(&volinfo->store_volinfo_lock);
+    {
+        glusterd_perform_volinfo_version_action(volinfo, ac);
+
+        ret = glusterd_store_create_volume_dirs(volinfo);
         if (ret)
-                goto out;
+            goto unlock;
 
-        ret = glusterd_store_create_vol_shandle_on_absence (volinfo);
+        ret = glusterd_store_create_vol_shandle_on_absence(volinfo);
         if (ret)
-                goto out;
+            goto unlock;
 
-        ret = glusterd_store_create_rbstate_shandle_on_absence (volinfo);
+        ret = glusterd_store_create_nodestate_sh_on_absence(volinfo);
         if (ret)
-                goto out;
+            goto unlock;
 
-        ret = glusterd_store_create_nodestate_sh_on_absence (volinfo);
+        ret = glusterd_store_perform_volume_store(volinfo);
         if (ret)
-                goto out;
+            goto unlock;
 
-        ret = glusterd_store_perform_volume_store (volinfo);
-        if (ret)
-                goto out;
-
-        ret = glusterd_store_volume_atomic_update (volinfo);
+        ret = glusterd_store_volume_atomic_update(volinfo);
         if (ret) {
-                glusterd_perform_volinfo_version_action (volinfo,
-                                                         GLUSTERD_VOLINFO_VER_AC_DECREMENT);
-                goto out;
+            glusterd_perform_volinfo_version_action(
+                volinfo, GLUSTERD_VOLINFO_VER_AC_DECREMENT);
+            goto unlock;
         }
 
-        ret = glusterd_store_perform_rbstate_store (volinfo);
+        ret = glusterd_store_perform_node_state_store(volinfo);
         if (ret)
-                goto out;
-
-        ret = glusterd_store_perform_node_state_store (volinfo);
-        if (ret)
-                goto out;
+            goto unlock;
 
         /* checksum should be computed at the end */
-        ret = glusterd_compute_cksum (volinfo, _gf_false);
+        ret = glusterd_compute_cksum(volinfo, _gf_false);
         if (ret)
-                goto out;
+            goto unlock;
+    }
+unlock:
+    pthread_mutex_unlock(&volinfo->store_volinfo_lock);
+    pthread_mutex_unlock(&ctx->cleanup_lock);
 
-out:
-        if (ret)
-                glusterd_store_volume_cleanup_tmp (volinfo);
+    if (ret)
+        glusterd_store_volume_cleanup_tmp(volinfo);
 
-        gf_log (THIS->name, GF_LOG_DEBUG, "Returning %d", ret);
+    gf_msg_debug(THIS->name, 0, "Returning %d", ret);
 
-        return ret;
+    return ret;
 }
 
 int32_t
-glusterd_store_delete_volume (glusterd_volinfo_t *volinfo)
+glusterd_store_delete_volume(glusterd_volinfo_t *volinfo)
 {
-        char             pathname[PATH_MAX]    = {0,};
-        int32_t          ret                   = 0;
-        glusterd_conf_t *priv                  = NULL;
-        char             delete_path[PATH_MAX] = {0,};
-        char             trashdir[PATH_MAX]    = {0,};
-        xlator_t        *this                  = NULL;
-        gf_boolean_t     rename_fail           = _gf_false;
-
-        this = THIS;
-        GF_ASSERT (this);
-
-        GF_ASSERT (volinfo);
-        priv = this->private;
-
-        GF_ASSERT (priv);
-
-        GLUSTERD_GET_VOLUME_DIR (pathname, volinfo, priv);
-
-        snprintf (delete_path, sizeof (delete_path),
-        "%s/"GLUSTERD_TRASH"/%s.deleted", priv->workdir,
-        uuid_utoa (volinfo->volume_id));
-
-        snprintf (trashdir, sizeof (trashdir), "%s/"GLUSTERD_TRASH,
-                  priv->workdir);
-
-        ret = mkdir (trashdir, 0777);
-        if (ret && errno != EEXIST) {
-                gf_log (this->name, GF_LOG_ERROR, "Failed to create trash "
-                        "directory, reason : %s", strerror (errno));
-                ret = -1;
-                goto out;
-        }
-
-        ret = rename (pathname, delete_path);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "Failed to rename volume "
-                        "directory for volume %s", volinfo->volname);
-                rename_fail = _gf_true;
-                goto out;
-        }
-
-        ret = recursive_rmdir (trashdir);
-        if (ret) {
-                gf_log (this->name, GF_LOG_DEBUG, "Failed to rmdir: %s, Reason:"
-                        " %s", trashdir, strerror (errno));
-        }
+    char pathname[PATH_MAX] = {
+        0,
+    };
+    int32_t ret = 0;
+    glusterd_conf_t *priv = NULL;
+    char delete_path[PATH_MAX] = {
+        0,
+    };
+    char trashdir[PATH_MAX] = {
+        0,
+    };
+    xlator_t *this = NULL;
+    gf_boolean_t rename_fail = _gf_false;
+    int32_t len = 0;
+
+    this = THIS;
+    GF_ASSERT(this);
+
+    GF_ASSERT(volinfo);
+    priv = this->private;
+
+    GF_ASSERT(priv);
+
+    GLUSTERD_GET_VOLUME_DIR(pathname, volinfo, priv);
+
+    len = snprintf(delete_path, sizeof(delete_path),
+                   "%s/" GLUSTERD_TRASH "/%s.deleted", priv->workdir,
+                   uuid_utoa(volinfo->volume_id));
+    if ((len < 0) || (len >= sizeof(delete_path))) {
+        goto out;
+    }
+
+    len = snprintf(trashdir, sizeof(trashdir), "%s/" GLUSTERD_TRASH,
+                   priv->workdir);
+    if ((len < 0) || (len >= sizeof(trashdir))) {
+        goto out;
+    }
+
+    ret = sys_mkdir(trashdir, 0755);
+    if (ret && errno != EEXIST) {
+        gf_msg(this->name, GF_LOG_ERROR, errno, GD_MSG_CREATE_DIR_FAILED,
+               "Failed to create trash "
+               "directory");
+        goto out;
+    }
+
+    ret = sys_rename(pathname, delete_path);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, errno, GD_MSG_DIR_OP_FAILED,
+               "Failed to rename volume "
+               "directory for volume %s",
+               volinfo->volname);
+        rename_fail = _gf_true;
+        goto out;
+    }
+
+    ret = recursive_rmdir(trashdir);
+    if (ret) {
+        gf_msg_debug(this->name, 0, "Failed to rmdir: %s", trashdir);
+    }
 
 out:
-        if (volinfo->shandle) {
-                gf_store_handle_destroy (volinfo->shandle);
-                volinfo->shandle = NULL;
-        }
-        ret = (rename_fail == _gf_true) ? -1: 0;
-
-        gf_log (this->name, GF_LOG_DEBUG, "Returning %d", ret);
-        return ret;
+    if (volinfo->shandle) {
+        gf_store_handle_destroy(volinfo->shandle);
+        volinfo->shandle = NULL;
+    }
+    ret = (rename_fail == _gf_true) ? -1 : 0;
+
+    gf_msg_debug(this->name, 0, "Returning %d", ret);
+    return ret;
 }
 
 /*TODO: cleanup the duplicate code and implement a generic function for
  * deleting snap/volume depending on the parameter flag */
 int32_t
-glusterd_store_delete_snap (glusterd_snap_t *snap)
+glusterd_store_delete_snap(glusterd_snap_t *snap)
 {
-        char             pathname[PATH_MAX]    = {0,};
-        int32_t          ret                   = 0;
-        glusterd_conf_t *priv                  = NULL;
-        DIR             *dir                   = NULL;
-        struct dirent   *entry                 = NULL;
-        char             path[PATH_MAX]        = {0,};
-        char             delete_path[PATH_MAX] = {0,};
-        char             trashdir[PATH_MAX]    = {0,};
-        struct stat      st                    = {0, };
-        xlator_t        *this                  = NULL;
-        gf_boolean_t     rename_fail           = _gf_false;
-
-        this = THIS;
-        priv = this->private;
-        GF_ASSERT (priv);
-
-        GF_ASSERT (snap);
-        GLUSTERD_GET_SNAP_DIR (pathname, snap, priv);
-
-        snprintf (delete_path, sizeof (delete_path),
-        "%s/"GLUSTERD_TRASH"/snap-%s.deleted", priv->workdir,
-        uuid_utoa (snap->snap_id));
-
-        snprintf (trashdir, sizeof (trashdir), "%s/"GLUSTERD_TRASH,
-                  priv->workdir);
-
-        ret = mkdir (trashdir, 0777);
-        if (ret && errno != EEXIST) {
-                gf_log (this->name, GF_LOG_ERROR, "Failed to create trash "
-                        "directory, reason : %s", strerror (errno));
-                ret = -1;
-                goto out;
-        }
-
-        ret = rename (pathname, delete_path);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "Failed to rename snap "
-                        "directory %s to %s", pathname, delete_path);
-                rename_fail = _gf_true;
-                goto out;
-        }
-
-        dir = opendir (delete_path);
-        if (!dir) {
-                gf_log (this->name, GF_LOG_DEBUG, "Failed to open directory %s."
-                        " Reason : %s", delete_path, strerror (errno));
-                ret = 0;
-                goto out;
-        }
-
-        GF_FOR_EACH_ENTRY_IN_DIR (entry, dir);
-        while (entry) {
-                snprintf (path, PATH_MAX, "%s/%s", delete_path, entry->d_name);
-                ret = stat (path, &st);
-                if (ret == -1) {
-                        gf_log (this->name, GF_LOG_DEBUG, "Failed to stat "
-                                "entry %s : %s", path, strerror (errno));
-                        goto stat_failed;
-                }
-
-                if (S_ISDIR (st.st_mode))
-                        ret = rmdir (path);
-                else
-                        ret = unlink (path);
-
-                if (ret) {
-                        gf_log (this->name, GF_LOG_DEBUG, " Failed to remove "
-                                "%s. Reason : %s", path, strerror (errno));
-                }
-
-                gf_log (this->name, GF_LOG_DEBUG, "%s %s",
-                                ret ? "Failed to remove":"Removed",
-                                entry->d_name);
-stat_failed:
-                memset (path, 0, sizeof(path));
-                GF_FOR_EACH_ENTRY_IN_DIR (entry, dir);
-        }
-
-        ret = closedir (dir);
-        if (ret) {
-                gf_log (this->name, GF_LOG_DEBUG, "Failed to close dir %s. "
-                        "Reason : %s",delete_path, strerror (errno));
-        }
+    char pathname[PATH_MAX] = {
+        0,
+    };
+    int32_t ret = 0;
+    glusterd_conf_t *priv = NULL;
+    DIR *dir = NULL;
+    struct dirent *entry = NULL;
+    struct dirent scratch[2] = {
+        {
+            0,
+        },
+    };
+    char path[PATH_MAX] = {
+        0,
+    };
+    char delete_path[PATH_MAX] = {
+        0,
+    };
+    char trashdir[PATH_MAX] = {
+        0,
+    };
+    struct stat st = {
+        0,
+    };
+    xlator_t *this = NULL;
+    gf_boolean_t rename_fail = _gf_false;
+    int32_t len = 0;
+
+    this = THIS;
+    priv = this->private;
+    GF_ASSERT(priv);
+
+    GF_ASSERT(snap);
+    GLUSTERD_GET_SNAP_DIR(pathname, snap, priv);
+
+    len = snprintf(delete_path, sizeof(delete_path),
+                   "%s/" GLUSTERD_TRASH "/snap-%s.deleted", priv->workdir,
+                   uuid_utoa(snap->snap_id));
+    if ((len < 0) || (len >= sizeof(delete_path))) {
+        goto out;
+    }
+
+    len = snprintf(trashdir, sizeof(trashdir), "%s/" GLUSTERD_TRASH,
+                   priv->workdir);
+    if ((len < 0) || (len >= sizeof(trashdir))) {
+        goto out;
+    }
+
+    ret = sys_mkdir(trashdir, 0755);
+    if (ret && errno != EEXIST) {
+        gf_msg(this->name, GF_LOG_ERROR, errno, GD_MSG_CREATE_DIR_FAILED,
+               "Failed to create trash "
+               "directory");
+        goto out;
+    }
+
+    ret = sys_rename(pathname, delete_path);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, errno, GD_MSG_DIR_OP_FAILED,
+               "Failed to rename snap "
+               "directory %s to %s",
+               pathname, delete_path);
+        rename_fail = _gf_true;
+        goto out;
+    }
+
+    dir = sys_opendir(delete_path);
+    if (!dir) {
+        gf_msg_debug(this->name, 0, "Failed to open directory %s.",
+                     delete_path);
+        goto out;
+    }
+
+    while ((entry = sys_readdir(dir, scratch))) {
+        if (gf_irrelevant_entry(entry))
+            continue;
+        len = snprintf(path, PATH_MAX, "%s/%s", delete_path, entry->d_name);
+        if ((len < 0) || (len >= PATH_MAX)) {
+            goto stat_failed;
+        }
+        ret = sys_stat(path, &st);
+        if (ret == -1) {
+            gf_msg_debug(this->name, 0,
+                         "Failed to stat "
+                         "entry %s",
+                         path);
+            goto stat_failed;
+        }
+
+        if (S_ISDIR(st.st_mode))
+            ret = sys_rmdir(path);
+        else
+            ret = sys_unlink(path);
 
-        ret = rmdir (delete_path);
         if (ret) {
-                gf_log (this->name, GF_LOG_DEBUG, "Failed to rmdir: %s,err: %s",
-                        delete_path, strerror (errno));
-        }
-        ret = rmdir (trashdir);
-        if (ret) {
-                gf_log (this->name, GF_LOG_DEBUG, "Failed to rmdir: %s, Reason:"
-                        " %s", trashdir, strerror (errno));
-        }
+            gf_msg_debug(this->name, 0,
+                         " Failed to remove "
+                         "%s",
+                         path);
+        }
+
+        gf_msg_debug(this->name, 0, "%s %s",
+                     ret ? "Failed to remove" : "Removed", entry->d_name);
+    stat_failed:
+        memset(path, 0, sizeof(path));
+    }
+
+    ret = sys_closedir(dir);
+    if (ret) {
+        gf_msg_debug(this->name, 0, "Failed to close dir %s.", delete_path);
+    }
+
+    ret = sys_rmdir(delete_path);
+    if (ret) {
+        gf_msg_debug(this->name, 0, "Failed to rmdir: %s", delete_path);
+    }
+    ret = sys_rmdir(trashdir);
+    if (ret) {
+        gf_msg_debug(this->name, 0, "Failed to rmdir: %s", trashdir);
+    }
 
 out:
-        if (snap->shandle) {
-                gf_store_handle_destroy (snap->shandle);
-                snap->shandle = NULL;
-        }
-        ret = (rename_fail == _gf_true) ? -1: 0;
-
-        gf_log (this->name, GF_LOG_DEBUG, "Returning %d", ret);
-        return ret;
+    if (snap->shandle) {
+        gf_store_handle_destroy(snap->shandle);
+        snap->shandle = NULL;
+    }
+    ret = (rename_fail == _gf_true) ? -1 : 0;
+
+    gf_msg_debug(this->name, 0, "Returning %d", ret);
+    return ret;
 }
 
 int
-glusterd_store_global_info (xlator_t *this)
+glusterd_store_global_info(xlator_t *this)
 {
-        int                     ret                     = -1;
-        glusterd_conf_t         *conf                   = NULL;
-        char                    op_version_str[15]      = {0,};
-        char                    path[PATH_MAX]          = {0,};
-        gf_store_handle_t       *handle                 = NULL;
-        char                    *uuid_str               = NULL;
-        char                     buf[256]               = {0, };
-
-        conf = this->private;
-
-        uuid_str = gf_strdup (uuid_utoa (MY_UUID));
-        if (!uuid_str)
-                goto out;
-
-        if (!conf->handle) {
-                snprintf (path, PATH_MAX, "%s/%s", conf->workdir,
-                          GLUSTERD_INFO_FILE);
-                ret = gf_store_handle_new (path, &handle);
-                if (ret) {
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "Unable to get store handle");
-                        goto out;
-                }
-
-                conf->handle = handle;
-        } else
-                handle = conf->handle;
-
-        /* These options need to be available for all users */
-        ret = chmod (handle->path, 0644);
+    int ret = -1;
+    glusterd_conf_t *conf = NULL;
+    char buf[PATH_MAX];
+    uint total_len = 0;
+    gf_store_handle_t *handle = NULL;
+    char *uuid_str = NULL;
+
+    conf = this->private;
+
+    uuid_str = gf_strdup(uuid_utoa(MY_UUID));
+    if (!uuid_str)
+        goto out;
+
+    if (!conf->handle) {
+        ret = snprintf(buf, sizeof(buf), "%s/%s", conf->workdir,
+                       GLUSTERD_INFO_FILE);
+        if ((ret < 0) || (ret >= sizeof(buf))) {
+            ret = -1;
+            goto out;
+        }
+        ret = gf_store_handle_new(buf, &handle);
         if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "chmod error for %s: %s",
-                        GLUSTERD_INFO_FILE, strerror (errno));
-                goto out;
-        }
-
-        handle->fd = gf_store_mkstemp (handle);
-        if (handle->fd <= 0) {
-                ret = -1;
-                goto out;
-        }
-
-        ret = gf_store_save_value (handle->fd, GLUSTERD_STORE_UUID_KEY,
-                                   uuid_str);
-        if (ret) {
-                gf_log (this->name, GF_LOG_CRITICAL,
-                        "Storing uuid failed ret = %d", ret);
-                goto out;
-        }
-
-        snprintf (op_version_str, 15, "%d", conf->op_version);
-        ret = gf_store_save_value (handle->fd, GD_OP_VERSION_KEY,
-                                   op_version_str);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "Storing op-version failed ret = %d", ret);
-                goto out;
-        }
-
-        ret = gf_store_rename_tmppath (handle);
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_STORE_HANDLE_GET_FAIL,
+                   "Unable to get store handle");
+            goto out;
+        }
+
+        conf->handle = handle;
+    } else
+        handle = conf->handle;
+
+    /* These options need to be available for all users */
+    ret = sys_chmod(handle->path, 0644);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, errno, GD_MSG_FILE_OP_FAILED,
+               "chmod error for %s", GLUSTERD_INFO_FILE);
+        goto out;
+    }
+
+    handle->fd = gf_store_mkstemp(handle);
+    if (handle->fd < 0) {
+        ret = -1;
+        goto out;
+    }
+
+    ret = snprintf(buf, sizeof(buf), "%s=%s\n", GLUSTERD_STORE_UUID_KEY,
+                   uuid_str);
+    if (ret < 0 || ret >= sizeof(buf)) {
+        ret = -1;
+        goto out;
+    }
+    total_len += ret;
+
+    ret = snprintf(buf + total_len, sizeof(buf) - total_len, "%s=%d\n",
+                   GD_OP_VERSION_KEY, conf->op_version);
+    if (ret < 0 || ret >= sizeof(buf) - total_len) {
+        ret = -1;
+        goto out;
+    }
+
+    ret = gf_store_save_items(handle->fd, buf);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_CRITICAL, 0, GD_MSG_OP_VERS_STORE_FAIL,
+               "Storing glusterd global-info failed ret = %d", ret);
+        goto out;
+    }
+
+    ret = gf_store_rename_tmppath(handle);
 out:
-        if (handle) {
-                if (ret && (handle->fd > 0))
-                        gf_store_unlink_tmppath (handle);
+    if (handle) {
+        if (ret && (handle->fd >= 0))
+            gf_store_unlink_tmppath(handle);
+    }
 
-                if (handle->fd > 0) {
-                        handle->fd = 0;
-                }
-        }
-
-        if (uuid_str)
-                GF_FREE (uuid_str);
+    if (uuid_str)
+        GF_FREE(uuid_str);
 
-        if (ret)
-                gf_log (this->name, GF_LOG_ERROR,
-                        "Failed to store glusterd global-info");
+    if (ret)
+        gf_msg(this->name, GF_LOG_ERROR, 0,
+               GD_MSG_GLUSTERD_GLOBAL_INFO_STORE_FAIL,
+               "Failed to store glusterd global-info");
 
-        return ret;
+    return ret;
 }
 
 int
-glusterd_retrieve_op_version (xlator_t *this, int *op_version)
+glusterd_store_max_op_version(xlator_t *this)
 {
-        char                    *op_version_str = NULL;
-        glusterd_conf_t         *priv           = NULL;
-        int                     ret             = -1;
-        int                     tmp_version     = 0;
-        char                    *tmp            = NULL;
-        char                    path[PATH_MAX]  = {0,};
-        gf_store_handle_t       *handle         = NULL;
-
-        priv = this->private;
-
-        if (!priv->handle) {
-                snprintf (path, PATH_MAX, "%s/%s", priv->workdir,
-                          GLUSTERD_INFO_FILE);
-                ret = gf_store_handle_retrieve (path, &handle);
-
-                if (ret) {
-                        gf_log ("", GF_LOG_DEBUG, "Unable to get store "
-                                "handle!");
-                        goto out;
-                }
-
-                priv->handle = handle;
-        }
-
-        ret = gf_store_retrieve_value (priv->handle, GD_OP_VERSION_KEY,
-                                       &op_version_str);
-        if (ret) {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "No previous op_version present");
-                goto out;
-        }
-
-        tmp_version = strtol (op_version_str, &tmp, 10);
-        if ((tmp_version <= 0) || (tmp && strlen (tmp) > 1)) {
-                gf_log (this->name, GF_LOG_WARNING, "invalid version number");
-                goto out;
-        }
-
-        *op_version = tmp_version;
-
-        ret = 0;
+    int ret = -1;
+    glusterd_conf_t *conf = NULL;
+    char op_version_str[15] = {
+        0,
+    };
+    char path[PATH_MAX] = {
+        0,
+    };
+    gf_store_handle_t *handle = NULL;
+    int32_t len = 0;
+
+    conf = this->private;
+
+    len = snprintf(path, PATH_MAX, "%s/%s", conf->workdir,
+                   GLUSTERD_UPGRADE_FILE);
+    if ((len < 0) || (len >= PATH_MAX)) {
+        goto out;
+    }
+    ret = gf_store_handle_new(path, &handle);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_STORE_HANDLE_GET_FAIL,
+               "Unable to get store handle");
+        goto out;
+    }
+
+    /* These options need to be available for all users */
+    ret = sys_chmod(handle->path, 0644);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, errno, GD_MSG_FILE_OP_FAILED,
+               "chmod error for %s", GLUSTERD_UPGRADE_FILE);
+        goto out;
+    }
+
+    handle->fd = gf_store_mkstemp(handle);
+    if (handle->fd < 0) {
+        ret = -1;
+        goto out;
+    }
+
+    snprintf(op_version_str, sizeof(op_version_str), "%d", GD_OP_VERSION_MAX);
+    ret = gf_store_save_value(handle->fd, GD_MAX_OP_VERSION_KEY,
+                              op_version_str);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_OP_VERS_STORE_FAIL,
+               "Storing op-version failed ret = %d", ret);
+        goto out;
+    }
+
+    ret = gf_store_rename_tmppath(handle);
 out:
-        if (op_version_str)
-                GF_FREE (op_version_str);
-
-        return ret;
+    if (handle) {
+        if (ret && (handle->fd >= 0))
+            gf_store_unlink_tmppath(handle);
+    }
+
+    if (ret)
+        gf_msg(this->name, GF_LOG_ERROR, 0,
+               GD_MSG_GLUSTERD_GLOBAL_INFO_STORE_FAIL,
+               "Failed to store max op-version");
+    if (handle)
+        gf_store_handle_destroy(handle);
+    return ret;
 }
 
 int
-glusterd_retrieve_sys_snap_max_limit (xlator_t *this, uint64_t *limit,
-                                      char *key)
+glusterd_retrieve_max_op_version(xlator_t *this, int *op_version)
 {
-        char                    *limit_str      = NULL;
-        glusterd_conf_t         *priv           = NULL;
-        int                     ret             = -1;
-        uint64_t                tmp_limit       = 0;
-        char                    *tmp            = NULL;
-        char                    path[PATH_MAX]  = {0,};
-        gf_store_handle_t       *handle         = NULL;
-
-        GF_ASSERT (this);
-        priv = this->private;
-
-        GF_ASSERT (priv);
-        GF_ASSERT (limit);
-        GF_ASSERT (key);
-
-        if (!priv->handle) {
-                snprintf (path, PATH_MAX, "%s/%s", priv->workdir,
-                          GLUSTERD_INFO_FILE);
-                ret = gf_store_handle_retrieve (path, &handle);
-
-                if (ret) {
-                        gf_log ("", GF_LOG_DEBUG, "Unable to get store "
-                                "handle!");
-                        goto out;
-                }
+    char *op_version_str = NULL;
+    glusterd_conf_t *priv = NULL;
+    int ret = -1;
+    int tmp_version = 0;
+    char *tmp = NULL;
+    char path[PATH_MAX] = {
+        0,
+    };
+    gf_store_handle_t *handle = NULL;
+    int32_t len = 0;
+
+    priv = this->private;
+
+    len = snprintf(path, PATH_MAX, "%s/%s", priv->workdir,
+                   GLUSTERD_UPGRADE_FILE);
+    if ((len < 0) || (len >= PATH_MAX)) {
+        goto out;
+    }
+    ret = gf_store_handle_retrieve(path, &handle);
+
+    if (ret) {
+        gf_msg_debug(this->name, 0,
+                     "Unable to get store "
+                     "handle!");
+        goto out;
+    }
+
+    ret = gf_store_retrieve_value(handle, GD_MAX_OP_VERSION_KEY,
+                                  &op_version_str);
+    if (ret) {
+        gf_msg_debug(this->name, 0, "No previous op_version present");
+        goto out;
+    }
+
+    tmp_version = strtol(op_version_str, &tmp, 10);
+    if ((tmp_version <= 0) || (tmp && strlen(tmp) > 1)) {
+        gf_msg(this->name, GF_LOG_WARNING, EINVAL, GD_MSG_UNSUPPORTED_VERSION,
+               "invalid version number");
+        goto out;
+    }
+
+    *op_version = tmp_version;
+
+    ret = 0;
+out:
+    if (op_version_str)
+        GF_FREE(op_version_str);
+    if (handle)
+        gf_store_handle_destroy(handle);
+    return ret;
+}
 
-                priv->handle = handle;
-        }
+int
+glusterd_retrieve_op_version(xlator_t *this, int *op_version)
+{
+    char *op_version_str = NULL;
+    glusterd_conf_t *priv = NULL;
+    int ret = -1;
+    int tmp_version = 0;
+    char *tmp = NULL;
+    char path[PATH_MAX] = {
+        0,
+    };
+    gf_store_handle_t *handle = NULL;
+    int32_t len = 0;
+
+    priv = this->private;
+
+    if (!priv->handle) {
+        len = snprintf(path, PATH_MAX, "%s/%s", priv->workdir,
+                       GLUSTERD_INFO_FILE);
+        if ((len < 0) || (len >= PATH_MAX)) {
+            goto out;
+        }
+        ret = gf_store_handle_retrieve(path, &handle);
 
-        ret = gf_store_retrieve_value (priv->handle,
-                                       key,
-                                       &limit_str);
         if (ret) {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "No previous %s present", key);
-                goto out;
+            gf_msg_debug(this->name, 0,
+                         "Unable to get store "
+                         "handle!");
+            goto out;
         }
 
-        tmp_limit = strtoul (limit_str, &tmp, 10);
-        if ((tmp_limit <= 0) || (tmp && strlen (tmp) > 1)) {
-                gf_log (this->name, GF_LOG_WARNING, "invalid version number");
-                goto out;
-        }
+        priv->handle = handle;
+    }
 
-        *limit = tmp_limit;
+    ret = gf_store_retrieve_value(priv->handle, GD_OP_VERSION_KEY,
+                                  &op_version_str);
+    if (ret) {
+        gf_msg_debug(this->name, 0, "No previous op_version present");
+        goto out;
+    }
 
-        ret = 0;
+    tmp_version = strtol(op_version_str, &tmp, 10);
+    if ((tmp_version <= 0) || (tmp && strlen(tmp) > 1)) {
+        gf_msg(this->name, GF_LOG_WARNING, EINVAL, GD_MSG_UNSUPPORTED_VERSION,
+               "invalid version number");
+        goto out;
+    }
+
+    *op_version = tmp_version;
+
+    ret = 0;
 out:
-        if (limit_str)
-                GF_FREE (limit_str);
+    if (op_version_str)
+        GF_FREE(op_version_str);
 
-        return ret;
+    return ret;
 }
 
 int
-glusterd_restore_op_version (xlator_t *this)
+glusterd_restore_op_version(xlator_t *this)
 {
-        glusterd_conf_t *conf           = NULL;
-        int              ret            = 0;
-        int              op_version     = 0;
-
-        conf = this->private;
-
-        ret = glusterd_retrieve_op_version (this, &op_version);
-        if (!ret) {
-                if ((op_version < GD_OP_VERSION_MIN) ||
-                    (op_version > GD_OP_VERSION_MAX)) {
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "wrong op-version (%d) retrieved", op_version);
-                        ret = -1;
-                        goto out;
-                }
-                conf->op_version = op_version;
-                gf_log ("glusterd", GF_LOG_INFO,
-                        "retrieved op-version: %d", conf->op_version);
-                goto out;
-        }
-
-        /* op-version can be missing from the store file in 2 cases,
-         * 1. This is a new install of glusterfs
-         * 2. This is an upgrade of glusterfs from a version without op-version
-         *    to a version with op-version (eg. 3.3 -> 3.4)
-         *
-         * Detection of a new install or an upgrade from an older install can be
-         * done by checking for the presence of the its peer-id in the store
-         * file.  If peer-id is present, the installation is an upgrade else, it
-         * is a new install.
-         *
-         * For case 1, set op-version to GD_OP_VERSION_MAX.
-         * For case 2, set op-version to GD_OP_VERSION_MIN.
-         */
-        ret = glusterd_retrieve_uuid();
-        if (ret) {
-                gf_log (this->name, GF_LOG_INFO, "Detected new install. Setting"
-                        " op-version to maximum : %d", GD_OP_VERSION_MAX);
-                conf->op_version = GD_OP_VERSION_MAX;
-        } else {
-                gf_log (this->name, GF_LOG_INFO, "Upgrade detected. Setting"
-                        " op-version to minimum : %d", GD_OP_VERSION_MIN);
-                conf->op_version = GD_OP_VERSION_MIN;
-        }
-        ret = 0;
+    glusterd_conf_t *conf = NULL;
+    int ret = 0;
+    int op_version = 0;
+
+    conf = this->private;
+
+    ret = glusterd_retrieve_op_version(this, &op_version);
+    if (!ret) {
+        if ((op_version < GD_OP_VERSION_MIN) ||
+            (op_version > GD_OP_VERSION_MAX)) {
+            gf_msg(this->name, GF_LOG_ERROR, EINVAL, GD_MSG_UNSUPPORTED_VERSION,
+                   "wrong op-version (%d) retrieved", op_version);
+            ret = -1;
+            goto out;
+        }
+        conf->op_version = op_version;
+        gf_msg("glusterd", GF_LOG_INFO, 0, GD_MSG_OP_VERS_INFO,
+               "retrieved op-version: %d", conf->op_version);
+        goto out;
+    }
+
+    /* op-version can be missing from the store file in 2 cases,
+     * 1. This is a new install of glusterfs
+     * 2. This is an upgrade of glusterfs from a version without op-version
+     *    to a version with op-version (eg. 3.3 -> 3.4)
+     *
+     * Detection of a new install or an upgrade from an older install can be
+     * done by checking for the presence of the its peer-id in the store
+     * file.  If peer-id is present, the installation is an upgrade else, it
+     * is a new install.
+     *
+     * For case 1, set op-version to GD_OP_VERSION_MAX.
+     * For case 2, set op-version to GD_OP_VERSION_MIN.
+     */
+    ret = glusterd_retrieve_uuid();
+    if (ret) {
+        gf_msg(this->name, GF_LOG_INFO, 0, GD_MSG_OP_VERS_SET_INFO,
+               "Detected new install. Setting"
+               " op-version to maximum : %d",
+               GD_OP_VERSION_MAX);
+        conf->op_version = GD_OP_VERSION_MAX;
+    } else {
+        gf_msg(this->name, GF_LOG_INFO, 0, GD_MSG_OP_VERS_SET_INFO,
+               "Upgrade detected. Setting"
+               " op-version to minimum : %d",
+               GD_OP_VERSION_MIN);
+        conf->op_version = GD_OP_VERSION_MIN;
+    }
+    ret = 0;
 out:
-        return ret;
+    return ret;
 }
 
 int32_t
-glusterd_retrieve_uuid ()
+glusterd_retrieve_uuid()
 {
-        char            *uuid_str = NULL;
-        int32_t         ret = -1;
-        gf_store_handle_t *handle = NULL;
-        glusterd_conf_t *priv = NULL;
-        char            path[PATH_MAX] = {0,};
-
-        priv = THIS->private;
-
-        if (!priv->handle) {
-                snprintf (path, PATH_MAX, "%s/%s", priv->workdir,
-                          GLUSTERD_INFO_FILE);
-                ret = gf_store_handle_retrieve (path, &handle);
-
-                if (ret) {
-                        gf_log ("", GF_LOG_DEBUG, "Unable to get store"
-                                "handle!");
-                        goto out;
-                }
-
-                priv->handle = handle;
-        }
-
-        ret = gf_store_retrieve_value (priv->handle, GLUSTERD_STORE_UUID_KEY,
-                                       &uuid_str);
+    char *uuid_str = NULL;
+    int32_t ret = -1;
+    gf_store_handle_t *handle = NULL;
+    glusterd_conf_t *priv = NULL;
+    xlator_t *this = NULL;
+    char path[PATH_MAX] = {
+        0,
+    };
+    int32_t len = 0;
+
+    this = THIS;
+    priv = this->private;
+
+    if (!priv->handle) {
+        len = snprintf(path, PATH_MAX, "%s/%s", priv->workdir,
+                       GLUSTERD_INFO_FILE);
+        if ((len < 0) || (len >= PATH_MAX)) {
+            goto out;
+        }
+        ret = gf_store_handle_retrieve(path, &handle);
 
         if (ret) {
-                gf_log ("", GF_LOG_DEBUG, "No previous uuid is present");
-                goto out;
-        }
-
-        gf_uuid_parse (uuid_str, priv->uuid);
+            gf_msg_debug(this->name, 0,
+                         "Unable to get store"
+                         "handle!");
+            goto out;
+        }
+
+        priv->handle = handle;
+    }
+    pthread_mutex_lock(&priv->mutex);
+    {
+        ret = gf_store_retrieve_value(priv->handle, GLUSTERD_STORE_UUID_KEY,
+                                      &uuid_str);
+    }
+    pthread_mutex_unlock(&priv->mutex);
+    if (ret) {
+        gf_msg_debug(this->name, 0, "No previous uuid is present");
+        goto out;
+    }
+
+    gf_uuid_parse(uuid_str, priv->uuid);
 
 out:
-        GF_FREE (uuid_str);
-        gf_log ("", GF_LOG_DEBUG, "Returning %d", ret);
-        return ret;
+    GF_FREE(uuid_str);
+    gf_msg_debug(this->name, 0, "Returning %d", ret);
+    return ret;
 }
 
 int
-glusterd_store_retrieve_snapd (glusterd_volinfo_t *volinfo)
+glusterd_store_retrieve_snapd(glusterd_volinfo_t *volinfo)
 {
-        int                     ret                     = -1;
-        int                     exists                  = 0;
-        char                    *key                    = NULL;
-        char                    *value                  = NULL;
-        char                    volpath[PATH_MAX]       = {0,};
-        char                    path[PATH_MAX]          = {0,};
-        xlator_t                *this                   = NULL;
-        glusterd_conf_t         *conf                   = NULL;
-        gf_store_iter_t         *iter                   = NULL;
-        gf_store_op_errno_t     op_errno                = GD_STORE_SUCCESS;
-
-        this = THIS;
-        GF_ASSERT (this);
-        conf = THIS->private;
-        GF_ASSERT (volinfo);
-
-        if (conf->op_version < GD_OP_VERSION_3_6_0) {
-                ret = 0;
-                goto out;
-        }
-
-        /*
-         * This is needed for upgrade situations. Say a volume is created with
-         * older version of glusterfs and upgraded to a glusterfs version equal
-         * to or greater than GD_OP_VERSION_3_6_0. The older glusterd would not
-         * have created the snapd.info file related to snapshot daemon for user
-         * serviceable snapshots. So as part of upgrade when the new glusterd
-         * starts, as part of restore (restoring the volume to be precise), it
-         * tries to snapd related info from snapd.info file. But since there was
-         * no such file till now, the restore operation fails. Thus, to prevent
-         * it from happening check whether user serviceable snapshots features
-         * is enabled before restoring snapd. If its disbaled, then simply
-         * exit by returning success (without even checking for the snapd.info).
-         */
-
-        if (!dict_get_str_boolean (volinfo->dict, "features.uss", _gf_false)) {
-                ret = 0;
-                goto out;
-        }
-
-        GLUSTERD_GET_VOLUME_DIR(volpath, volinfo, conf);
-
-        snprintf (path, sizeof (path), "%s/%s", volpath,
-                  GLUSTERD_VOLUME_SNAPD_INFO_FILE);
-
-        ret = gf_store_handle_retrieve (path, &volinfo->snapd.handle);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "volinfo handle is NULL");
-                goto out;
-        }
-
-        ret = gf_store_iter_new (volinfo->snapd.handle, &iter);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "Failed to get new store "
-                        "iter");
-                goto out;
-        }
-
-        ret = gf_store_iter_get_next (iter, &key, &value, &op_errno);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "Failed to get next store "
-                        "iter");
-                goto out;
-        }
-
-        while (!ret) {
-                if (!strncmp (key, GLUSTERD_STORE_KEY_SNAPD_PORT,
-                              strlen (GLUSTERD_STORE_KEY_SNAPD_PORT))) {
-                        volinfo->snapd.port = atoi (value);
-                }
-
-                ret = gf_store_iter_get_next (iter, &key, &value,
-                                              &op_errno);
-        }
-
-        if (op_errno != GD_STORE_EOF)
-                goto out;
-
-        ret = gf_store_iter_destroy (iter);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "Failed to destroy store "
-                        "iter");
-                goto out;
-        }
-
+    int ret = -1;
+    char *key = NULL;
+    char *value = NULL;
+    char volpath[PATH_MAX] = {
+        0,
+    };
+    char path[PATH_MAX] = {
+        0,
+    };
+    xlator_t *this = NULL;
+    glusterd_conf_t *conf = NULL;
+    gf_store_iter_t *iter = NULL;
+    gf_store_op_errno_t op_errno = GD_STORE_SUCCESS;
+    int32_t len = 0;
+
+    this = THIS;
+    GF_ASSERT(this);
+    conf = THIS->private;
+    GF_ASSERT(volinfo);
+
+    if (conf->op_version < GD_OP_VERSION_3_6_0) {
         ret = 0;
+        goto out;
+    }
+
+    /*
+     * This is needed for upgrade situations. Say a volume is created with
+     * older version of glusterfs and upgraded to a glusterfs version equal
+     * to or greater than GD_OP_VERSION_3_6_0. The older glusterd would not
+     * have created the snapd.info file related to snapshot daemon for user
+     * serviceable snapshots. So as part of upgrade when the new glusterd
+     * starts, as part of restore (restoring the volume to be precise), it
+     * tries to snapd related info from snapd.info file. But since there was
+     * no such file till now, the restore operation fails. Thus, to prevent
+     * it from happening check whether user serviceable snapshots features
+     * is enabled before restoring snapd. If its disabled, then simply
+     * exit by returning success (without even checking for the snapd.info).
+     */
+
+    if (!dict_get_str_boolean(volinfo->dict, "features.uss", _gf_false)) {
+        ret = 0;
+        goto out;
+    }
+
+    GLUSTERD_GET_VOLUME_DIR(volpath, volinfo, conf);
+
+    len = snprintf(path, sizeof(path), "%s/%s", volpath,
+                   GLUSTERD_VOLUME_SNAPD_INFO_FILE);
+    if ((len < 0) || (len >= sizeof(path))) {
+        goto out;
+    }
+
+    ret = gf_store_handle_retrieve(path, &volinfo->snapd.handle);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_HANDLE_NULL,
+               "volinfo handle is NULL");
+        goto out;
+    }
+
+    ret = gf_store_iter_new(volinfo->snapd.handle, &iter);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_STORE_ITER_GET_FAIL,
+               "Failed to get new store "
+               "iter");
+        goto out;
+    }
+
+    ret = gf_store_iter_get_next(iter, &key, &value, &op_errno);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_STORE_ITER_GET_FAIL,
+               "Failed to get next store "
+               "iter");
+        goto out;
+    }
+
+    while (!ret) {
+        if (!strncmp(key, GLUSTERD_STORE_KEY_SNAPD_PORT,
+                     SLEN(GLUSTERD_STORE_KEY_SNAPD_PORT))) {
+            volinfo->snapd.port = atoi(value);
+        }
+
+        ret = gf_store_iter_get_next(iter, &key, &value, &op_errno);
+    }
+
+    if (op_errno != GD_STORE_EOF)
+        goto out;
+
+    ret = 0;
 
 out:
-        return ret;
+    if (gf_store_iter_destroy(&iter)) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_STORE_ITER_DESTROY_FAIL,
+               "Failed to destroy store iter");
+        ret = -1;
+    }
+
+    return ret;
 }
 
 int32_t
-glusterd_store_retrieve_bricks (glusterd_volinfo_t *volinfo)
+glusterd_store_retrieve_bricks(glusterd_volinfo_t *volinfo)
 {
-        int32_t                 ret = 0;
-        glusterd_brickinfo_t    *brickinfo = NULL;
-        gf_store_iter_t         *iter = NULL;
-        char                    *key = NULL;
-        char                    *value = NULL;
-        char                    brickdir[PATH_MAX] = {0,};
-        char                    path[PATH_MAX] = {0,};
-        glusterd_conf_t         *priv = NULL;
-        int32_t                 brick_count = 0;
-        char                    tmpkey[4096] = {0,};
-        gf_store_iter_t         *tmpiter = NULL;
-        char                    *tmpvalue = NULL;
-        struct pmap_registry    *pmap = NULL;
-        int                      brickid = 0;
-        gf_store_op_errno_t     op_errno = GD_STORE_SUCCESS;
-
-        GF_ASSERT (volinfo);
-        GF_ASSERT (volinfo->volname);
-
-        priv = THIS->private;
-
-        GLUSTERD_GET_BRICK_DIR (brickdir, volinfo, priv);
-
-        ret = gf_store_iter_new (volinfo->shandle, &tmpiter);
+    int32_t ret = 0;
+    glusterd_brickinfo_t *brickinfo = NULL;
+    glusterd_brickinfo_t *ta_brickinfo = NULL;
+    gf_store_iter_t *iter = NULL;
+    char *key = NULL;
+    char *value = NULL;
+    char brickdir[PATH_MAX] = {
+        0,
+    };
+    char path[PATH_MAX] = {
+        0,
+    };
+    glusterd_conf_t *priv = NULL;
+    int32_t brick_count = 0;
+    int32_t ta_brick_count = 0;
+    char tmpkey[32] = {
+        0,
+    };
+    gf_store_iter_t *tmpiter = NULL;
+    char *tmpvalue = NULL;
+    char abspath[PATH_MAX] = {0};
+    struct pmap_registry *pmap = NULL;
+    xlator_t *this = NULL;
+    int brickid = 0;
+    /* ta_brick_id initialization with 2 since ta-brick id starts with
+     * volname-ta-2
+     */
+    int ta_brick_id = 2;
+    gf_store_op_errno_t op_errno = GD_STORE_SUCCESS;
+    int32_t len = 0;
+
+    GF_ASSERT(volinfo);
+    GF_ASSERT(volinfo->volname);
+
+    this = THIS;
+    priv = this->private;
+
+    GLUSTERD_GET_BRICK_DIR(brickdir, volinfo, priv);
+
+    ret = gf_store_iter_new(volinfo->shandle, &tmpiter);
+
+    if (ret)
+        goto out;
+
+    while (brick_count < volinfo->brick_count) {
+        ret = glusterd_brickinfo_new(&brickinfo);
 
         if (ret)
-                goto out;
-
-        while (brick_count < volinfo->brick_count) {
-                ret = glusterd_brickinfo_new (&brickinfo);
-
-                if (ret)
-                        goto out;
-                snprintf (tmpkey, sizeof (tmpkey), "%s-%d",
-                          GLUSTERD_STORE_KEY_VOL_BRICK,brick_count);
-                ret = gf_store_iter_get_matching (tmpiter, tmpkey, &tmpvalue);
-                snprintf (path, sizeof (path), "%s/%s", brickdir, tmpvalue);
-
-                GF_FREE (tmpvalue);
-
-                tmpvalue = NULL;
+            goto out;
+        snprintf(tmpkey, sizeof(tmpkey), "%s-%d", GLUSTERD_STORE_KEY_VOL_BRICK,
+                 brick_count);
+        ret = gf_store_iter_get_matching(tmpiter, tmpkey, &tmpvalue);
+        len = snprintf(path, sizeof(path), "%s/%s", brickdir, tmpvalue);
+        GF_FREE(tmpvalue);
+        tmpvalue = NULL;
+        if ((len < 0) || (len >= sizeof(path))) {
+            ret = -1;
+            goto out;
+        }
 
-                ret = gf_store_handle_retrieve (path, &brickinfo->shandle);
+        ret = gf_store_handle_retrieve(path, &brickinfo->shandle);
 
-                if (ret)
-                        goto out;
+        if (ret)
+            goto out;
 
-                ret = gf_store_iter_new (brickinfo->shandle, &iter);
+        ret = gf_store_iter_new(brickinfo->shandle, &iter);
 
-                if (ret)
-                        goto out;
+        if (ret)
+            goto out;
 
-                ret = gf_store_iter_get_next (iter, &key, &value, &op_errno);
-                if (ret) {
-                        gf_log ("glusterd", GF_LOG_ERROR, "Unable to iterate "
-                                "the store for brick: %s, reason: %s", path,
-                                gf_store_strerror (op_errno));
-                        goto out;
+        ret = gf_store_iter_get_next(iter, &key, &value, &op_errno);
+        if (ret) {
+            gf_msg("glusterd", GF_LOG_ERROR, op_errno,
+                   GD_MSG_STORE_ITER_GET_FAIL,
+                   "Unable to iterate "
+                   "the store for brick: %s",
+                   path);
+            goto out;
+        }
+        while (!ret) {
+            if (!strncmp(key, GLUSTERD_STORE_KEY_BRICK_HOSTNAME,
+                         SLEN(GLUSTERD_STORE_KEY_BRICK_HOSTNAME))) {
+                if (snprintf(brickinfo->hostname, sizeof(brickinfo->hostname),
+                             "%s", value) >= sizeof(brickinfo->hostname)) {
+                    gf_msg("glusterd", GF_LOG_ERROR, op_errno,
+                           GD_MSG_PARSE_BRICKINFO_FAIL,
+                           "brick hostname truncated: %s", brickinfo->hostname);
+                    goto out;
                 }
-                while (!ret) {
-                        if (!strncmp (key, GLUSTERD_STORE_KEY_BRICK_HOSTNAME,
-                                      strlen (GLUSTERD_STORE_KEY_BRICK_HOSTNAME))) {
-                                strncpy (brickinfo->hostname, value, 1024);
-                        } else if (!strncmp (key, GLUSTERD_STORE_KEY_BRICK_PATH,
-                                    strlen (GLUSTERD_STORE_KEY_BRICK_PATH))) {
-                                strncpy (brickinfo->path, value,
-                                         sizeof (brickinfo->path));
-                        } else if (!strncmp (key, GLUSTERD_STORE_KEY_BRICK_PORT,
-                                    strlen (GLUSTERD_STORE_KEY_BRICK_PORT))) {
-                                gf_string2int (value, &brickinfo->port);
-
-                                if (brickinfo->port < priv->base_port) {
-                                        /* This is required to adhere to the
-                                           IANA standards */
-                                        brickinfo->port = 0;
-                                } else {
-                                        /* This is required to have proper ports
-                                           assigned to bricks after restart */
-                                        pmap = pmap_registry_get (THIS);
-                                        if (pmap->last_alloc <= brickinfo->port)
-                                                pmap->last_alloc =
-                                                        brickinfo->port + 1;
-                                }
-                        } else if (!strncmp (key, GLUSTERD_STORE_KEY_BRICK_RDMA_PORT,
-                                    strlen (GLUSTERD_STORE_KEY_BRICK_RDMA_PORT))) {
-                                gf_string2int (value, &brickinfo->rdma_port);
-
-                                if (brickinfo->rdma_port < priv->base_port) {
-                                        /* This is required to adhere to the
-                                           IANA standards */
-                                        brickinfo->rdma_port = 0;
-                                } else {
-                                        /* This is required to have proper ports
-                                           assigned to bricks after restart */
-                                        pmap = pmap_registry_get (THIS);
-                                        if (pmap->last_alloc <=
-                                            brickinfo->rdma_port)
-                                                pmap->last_alloc =
-                                                        brickinfo->rdma_port +1;
-                                }
-
-                        } else if (!strncmp (key, GLUSTERD_STORE_KEY_BRICK_DECOMMISSIONED,
-                                             strlen (GLUSTERD_STORE_KEY_BRICK_DECOMMISSIONED))) {
-                                gf_string2int (value, &brickinfo->decommissioned);
-                        } else if (!strncmp (key, GLUSTERD_STORE_KEY_BRICK_DEVICE_PATH,
-                                             strlen (GLUSTERD_STORE_KEY_BRICK_DEVICE_PATH))) {
-                                strncpy (brickinfo->device_path, value,
-                                         sizeof (brickinfo->device_path));
-                        } else if (!strncmp (key, GLUSTERD_STORE_KEY_BRICK_MOUNT_DIR,
-                                             strlen (GLUSTERD_STORE_KEY_BRICK_MOUNT_DIR))) {
-                                strncpy (brickinfo->mount_dir, value,
-                                         sizeof (brickinfo->mount_dir));
-                        } else if (!strncmp (key, GLUSTERD_STORE_KEY_BRICK_SNAP_STATUS,
-                                             strlen (GLUSTERD_STORE_KEY_BRICK_SNAP_STATUS))) {
-                                gf_string2int (value, &brickinfo->snap_status);
-                        } else if (!strncmp (key, GLUSTERD_STORE_KEY_BRICK_FSTYPE,
-                                             strlen (GLUSTERD_STORE_KEY_BRICK_FSTYPE))) {
-                                strncpy (brickinfo->fstype, value,
-                                         sizeof (brickinfo->fstype));
-                        } else if (!strncmp (key, GLUSTERD_STORE_KEY_BRICK_MNTOPTS,
-                                             strlen (GLUSTERD_STORE_KEY_BRICK_MNTOPTS))) {
-                                strncpy (brickinfo->mnt_opts, value,
-                                         sizeof (brickinfo->mnt_opts));
-                        } else if (!strncmp (key,
-                                    GLUSTERD_STORE_KEY_BRICK_VGNAME,
-                                    strlen (GLUSTERD_STORE_KEY_BRICK_VGNAME))) {
-                                strncpy (brickinfo->vg, value,
-                                         sizeof (brickinfo->vg));
-                        } else if (!strcmp(key, GLUSTERD_STORE_KEY_BRICK_ID)) {
-                                strncpy (brickinfo->brick_id, value,
-                                         sizeof (brickinfo->brick_id));
-                        } else {
-                                gf_log ("", GF_LOG_ERROR, "Unknown key: %s",
-                                        key);
-                        }
-
-                        GF_FREE (key);
-                        GF_FREE (value);
-                        key = NULL;
-                        value = NULL;
-
-                        ret = gf_store_iter_get_next (iter, &key, &value,
-                                                      &op_errno);
+            } else if (!strncmp(key, GLUSTERD_STORE_KEY_BRICK_PATH,
+                                SLEN(GLUSTERD_STORE_KEY_BRICK_PATH))) {
+                if (snprintf(brickinfo->path, sizeof(brickinfo->path), "%s",
+                             value) >= sizeof(brickinfo->path)) {
+                    gf_msg("glusterd", GF_LOG_ERROR, op_errno,
+                           GD_MSG_PARSE_BRICKINFO_FAIL,
+                           "brick path truncated: %s", brickinfo->path);
+                    goto out;
                 }
-
-                if (op_errno != GD_STORE_EOF) {
-                        gf_log ("", GF_LOG_ERROR, "Error parsing brickinfo: "
-                                "op_errno=%d", op_errno);
-                        goto out;
+            } else if (!strncmp(key, GLUSTERD_STORE_KEY_BRICK_REAL_PATH,
+                                SLEN(GLUSTERD_STORE_KEY_BRICK_REAL_PATH))) {
+                if (snprintf(brickinfo->real_path, sizeof(brickinfo->real_path),
+                             "%s", value) >= sizeof(brickinfo->real_path)) {
+                    gf_msg("glusterd", GF_LOG_ERROR, op_errno,
+                           GD_MSG_PARSE_BRICKINFO_FAIL,
+                           "real_path truncated: %s", brickinfo->real_path);
+                    goto out;
                 }
-                ret = gf_store_iter_destroy (iter);
-
-                if (ret)
-                        goto out;
-
-                if (brickinfo->brick_id[0] == '\0') {
-                        /* This is an old volume upgraded to op_version 4 */
-                       GLUSTERD_ASSIGN_BRICKID_TO_BRICKINFO (brickinfo, volinfo,
-                                                             brickid++);
+            } else if (!strncmp(key, GLUSTERD_STORE_KEY_BRICK_PORT,
+                                SLEN(GLUSTERD_STORE_KEY_BRICK_PORT))) {
+                ret = gf_string2int(value, &brickinfo->port);
+                if (ret == -1) {
+                    gf_msg(this->name, GF_LOG_ERROR, EINVAL,
+                           GD_MSG_INCOMPATIBLE_VALUE,
+                           "Failed to convert "
+                           "string to integer");
                 }
 
-                cds_list_add_tail (&brickinfo->brick_list, &volinfo->bricks);
-                brick_count++;
-        }
+                if (brickinfo->port < priv->base_port) {
+                    /* This is required to adhere to the
+                       IANA standards */
+                    brickinfo->port = 0;
+                } else {
+                    /* This is required to have proper ports
+                       assigned to bricks after restart */
+                    pmap = pmap_registry_get(THIS);
+                    if (pmap->last_alloc <= brickinfo->port)
+                        pmap->last_alloc = brickinfo->port + 1;
+                }
+            } else if (!strncmp(key, GLUSTERD_STORE_KEY_BRICK_RDMA_PORT,
+                                SLEN(GLUSTERD_STORE_KEY_BRICK_RDMA_PORT))) {
+                ret = gf_string2int(value, &brickinfo->rdma_port);
+                if (ret == -1) {
+                    gf_msg(this->name, GF_LOG_ERROR, EINVAL,
+                           GD_MSG_INCOMPATIBLE_VALUE,
+                           "Failed to convert "
+                           "string to integer");
+                }
 
-        ret = gf_store_iter_destroy (tmpiter);
-        if (ret)
-                goto out;
-out:
-        gf_log ("", GF_LOG_DEBUG, "Returning with %d", ret);
+                if (brickinfo->rdma_port < priv->base_port) {
+                    /* This is required to adhere to the
+                       IANA standards */
+                    brickinfo->rdma_port = 0;
+                } else {
+                    /* This is required to have proper ports
+                       assigned to bricks after restart */
+                    pmap = pmap_registry_get(THIS);
+                    if (pmap->last_alloc <= brickinfo->rdma_port)
+                        pmap->last_alloc = brickinfo->rdma_port + 1;
+                }
 
-        return ret;
-}
+            } else if (!strncmp(
+                           key, GLUSTERD_STORE_KEY_BRICK_DECOMMISSIONED,
+                           SLEN(GLUSTERD_STORE_KEY_BRICK_DECOMMISSIONED))) {
+                ret = gf_string2int(value, &brickinfo->decommissioned);
+                if (ret == -1) {
+                    gf_msg(this->name, GF_LOG_ERROR, EINVAL,
+                           GD_MSG_INCOMPATIBLE_VALUE,
+                           "Failed to convert "
+                           "string to integer");
+                }
 
+            } else if (!strncmp(key, GLUSTERD_STORE_KEY_BRICK_DEVICE_PATH,
+                                SLEN(GLUSTERD_STORE_KEY_BRICK_DEVICE_PATH))) {
+                if (snprintf(brickinfo->device_path,
+                             sizeof(brickinfo->device_path), "%s",
+                             value) >= sizeof(brickinfo->device_path)) {
+                    gf_msg("glusterd", GF_LOG_ERROR, op_errno,
+                           GD_MSG_PARSE_BRICKINFO_FAIL,
+                           "device_path truncated: %s", brickinfo->device_path);
+                    goto out;
+                }
+            } else if (!strncmp(key, GLUSTERD_STORE_KEY_BRICK_MOUNT_DIR,
+                                SLEN(GLUSTERD_STORE_KEY_BRICK_MOUNT_DIR))) {
+                if (snprintf(brickinfo->mount_dir, sizeof(brickinfo->mount_dir),
+                             "%s", value) >= sizeof(brickinfo->mount_dir)) {
+                    gf_msg("glusterd", GF_LOG_ERROR, op_errno,
+                           GD_MSG_PARSE_BRICKINFO_FAIL,
+                           "mount_dir truncated: %s", brickinfo->mount_dir);
+                    goto out;
+                }
+            } else if (!strncmp(key, GLUSTERD_STORE_KEY_BRICK_SNAP_STATUS,
+                                SLEN(GLUSTERD_STORE_KEY_BRICK_SNAP_STATUS))) {
+                ret = gf_string2int(value, &brickinfo->snap_status);
+                if (ret == -1) {
+                    gf_msg(this->name, GF_LOG_ERROR, EINVAL,
+                           GD_MSG_INCOMPATIBLE_VALUE,
+                           "Failed to convert "
+                           "string to integer");
+                }
 
-int32_t
-glusterd_store_retrieve_rbstate (glusterd_volinfo_t *volinfo)
-{
-        int32_t                   ret                   = -1;
-        gf_store_iter_t           *iter                 = NULL;
-        char                      *key                  = NULL;
-        char                      *value                = NULL;
-        char                      volpath[PATH_MAX]     = {0,};
-        glusterd_conf_t           *priv                 = NULL;
-        char                      path[PATH_MAX]        = {0,};
-        gf_store_op_errno_t       op_errno              = GD_STORE_SUCCESS;
-        xlator_t                 *this                  = NULL;
-
-        this = THIS;
-        GF_ASSERT (this);
-        priv = this->private;
-        GF_ASSERT (priv);
-        GF_ASSERT (volinfo);
-
-        GLUSTERD_GET_VOLUME_DIR(volpath, volinfo, priv);
-        snprintf (path, sizeof (path), "%s/%s", volpath,
-                  GLUSTERD_VOLUME_RBSTATE_FILE);
-
-        ret = gf_store_handle_retrieve (path, &volinfo->rb_shandle);
+            } else if (!strncmp(key, GLUSTERD_STORE_KEY_BRICK_FSTYPE,
+                                SLEN(GLUSTERD_STORE_KEY_BRICK_FSTYPE))) {
+                if (snprintf(brickinfo->fstype, sizeof(brickinfo->fstype), "%s",
+                             value) >= sizeof(brickinfo->fstype)) {
+                    gf_msg("glusterd", GF_LOG_ERROR, op_errno,
+                           GD_MSG_PARSE_BRICKINFO_FAIL, "fstype truncated: %s",
+                           brickinfo->fstype);
+                    goto out;
+                }
+            } else if (!strncmp(key, GLUSTERD_STORE_KEY_BRICK_MNTOPTS,
+                                SLEN(GLUSTERD_STORE_KEY_BRICK_MNTOPTS))) {
+                if (snprintf(brickinfo->mnt_opts, sizeof(brickinfo->mnt_opts),
+                             "%s", value) >= sizeof(brickinfo->mnt_opts)) {
+                    gf_msg("glusterd", GF_LOG_ERROR, op_errno,
+                           GD_MSG_PARSE_BRICKINFO_FAIL,
+                           "mnt_opts truncated: %s", brickinfo->mnt_opts);
+                    goto out;
+                }
+            } else if (!strncmp(key, GLUSTERD_STORE_KEY_BRICK_VGNAME,
+                                SLEN(GLUSTERD_STORE_KEY_BRICK_VGNAME))) {
+                if (snprintf(brickinfo->vg, sizeof(brickinfo->vg), "%s",
+                             value) >= sizeof(brickinfo->vg)) {
+                    gf_msg("glusterd", GF_LOG_ERROR, op_errno,
+                           GD_MSG_PARSE_BRICKINFO_FAIL,
+                           "brickinfo->vg truncated: %s", brickinfo->vg);
+                    goto out;
+                }
+            } else if (!strcmp(key, GLUSTERD_STORE_KEY_BRICK_ID)) {
+                if (snprintf(brickinfo->brick_id, sizeof(brickinfo->brick_id),
+                             "%s", value) >= sizeof(brickinfo->brick_id)) {
+                    gf_msg("glusterd", GF_LOG_ERROR, op_errno,
+                           GD_MSG_PARSE_BRICKINFO_FAIL,
+                           "brick_id truncated: %s", brickinfo->brick_id);
+                    goto out;
+                }
+            } else if (!strncmp(key, GLUSTERD_STORE_KEY_BRICK_FSID,
+                                SLEN(GLUSTERD_STORE_KEY_BRICK_FSID))) {
+                ret = gf_string2uint64(value, &brickinfo->statfs_fsid);
+                if (ret) {
+                    gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_INVALID_ENTRY,
+                           "%s "
+                           "is not a valid uint64_t value",
+                           value);
+                }
 
-        if (ret)
-                goto out;
+            } else if (!strcmp(key, GLUSTERD_STORE_KEY_BRICK_UUID)) {
+                gf_uuid_parse(value, brickinfo->uuid);
+            } else {
+                gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_UNKNOWN_KEY,
+                       "Unknown key: %s", key);
+            }
 
-        ret = gf_store_iter_new (volinfo->rb_shandle, &iter);
+            GF_FREE(key);
+            GF_FREE(value);
+            key = NULL;
+            value = NULL;
 
-        if (ret)
-                goto out;
+            ret = gf_store_iter_get_next(iter, &key, &value, &op_errno);
+        }
 
-        ret = gf_store_iter_get_next (iter, &key, &value, &op_errno);
-        if (ret)
-                goto out;
+        if (op_errno != GD_STORE_EOF) {
+            gf_msg(this->name, GF_LOG_ERROR, op_errno,
+                   GD_MSG_PARSE_BRICKINFO_FAIL,
+                   "Error parsing brickinfo: "
+                   "op_errno=%d",
+                   op_errno);
+            goto out;
+        }
 
-        while (!ret) {
-                if (!strncmp (key, GLUSTERD_STORE_KEY_RB_STATUS,
-                              strlen (GLUSTERD_STORE_KEY_RB_STATUS))) {
-                        volinfo->rep_brick.rb_status = atoi (value);
+        if (brickinfo->brick_id[0] == '\0') {
+            /* This is an old volume upgraded to op_version 4 */
+            GLUSTERD_ASSIGN_BRICKID_TO_BRICKINFO(brickinfo, volinfo, brickid++);
+        }
+        /* Populate brickinfo->real_path for normal volumes, for
+         * snapshot or snapshot restored volume this would be done post
+         * creating the brick mounts
+         */
+        if (gf_uuid_is_null(brickinfo->uuid))
+            (void)glusterd_resolve_brick(brickinfo);
+        if (brickinfo->real_path[0] == '\0' && !volinfo->is_snap_volume &&
+            gf_uuid_is_null(volinfo->restored_from_snap)) {
+            /* By now if the brick is a local brick then it will be
+             * able to resolve which is the only thing we want now
+             * for checking  whether the brickinfo->uuid matches
+             * with MY_UUID for realpath check. Hence do not handle
+             * error
+             */
+            if (!gf_uuid_compare(brickinfo->uuid, MY_UUID)) {
+                if (!realpath(brickinfo->path, abspath)) {
+                    gf_msg(this->name, GF_LOG_CRITICAL, errno,
+                           GD_MSG_BRICKINFO_CREATE_FAIL,
+                           "realpath() failed for brick %s"
+                           ". The underlying file system "
+                           "may be in bad state",
+                           brickinfo->path);
+                    ret = -1;
+                    goto out;
                 }
-
-                if (volinfo->rep_brick.rb_status > GF_RB_STATUS_NONE) {
-                        if (!strncmp (key, GLUSTERD_STORE_KEY_RB_SRC_BRICK,
-                                      strlen (GLUSTERD_STORE_KEY_RB_SRC_BRICK))) {
-                                ret = glusterd_brickinfo_new_from_brick (value,
-                                                &volinfo->rep_brick.src_brick);
-                                if (ret)
-                                        goto out;
-                        } else if (!strncmp (key, GLUSTERD_STORE_KEY_RB_DST_BRICK,
-                                             strlen (GLUSTERD_STORE_KEY_RB_DST_BRICK))) {
-                                ret = glusterd_brickinfo_new_from_brick (value,
-                                                &volinfo->rep_brick.dst_brick);
-                                if (ret)
-                                        goto out;
-                        } else if (!strncmp (key, GLUSTERD_STORE_KEY_RB_DST_PORT,
-                                             strlen (GLUSTERD_STORE_KEY_RB_DST_PORT))) {
-                                switch (volinfo->transport_type) {
-                                case GF_TRANSPORT_RDMA:
-                                volinfo->rep_brick.dst_brick->rdma_port =
-                                                 atoi (value);
-                                        break;
-
-                                case GF_TRANSPORT_TCP:
-                                case GF_TRANSPORT_BOTH_TCP_RDMA:
-                                volinfo->rep_brick.dst_brick->port =
-                                                atoi (value);
-                                        break;
-                                }
-                        } else if (!strncmp (key, GF_REPLACE_BRICK_TID_KEY,
-                                             strlen (GF_REPLACE_BRICK_TID_KEY))) {
-                                        gf_uuid_parse (value,
-                                                    volinfo->rep_brick.rb_id);
-                        }
+                if (strlen(abspath) >= sizeof(brickinfo->real_path)) {
+                    ret = -1;
+                    goto out;
                 }
-
-                GF_FREE (key);
-                GF_FREE (value);
-                key = NULL;
-                value = NULL;
-
-                ret = gf_store_iter_get_next (iter, &key, &value, &op_errno);
-        }
-
-        if (op_errno != GD_STORE_EOF)
+                (void)strncpy(brickinfo->real_path, abspath,
+                              sizeof(brickinfo->real_path));
+            }
+        }
+
+        /* Handle upgrade case of shared_brick_count 'fsid' */
+        /* Ideally statfs_fsid should never be 0 if done right */
+        if (!gf_uuid_compare(brickinfo->uuid, MY_UUID) &&
+            brickinfo->statfs_fsid == 0) {
+            struct statvfs brickstat = {
+                0,
+            };
+            ret = sys_statvfs(brickinfo->path, &brickstat);
+            if (ret) {
+                gf_msg(this->name, GF_LOG_WARNING, errno,
+                       GD_MSG_BRICKINFO_CREATE_FAIL,
+                       "failed to get statfs() call on brick %s",
+                       brickinfo->path);
+                /* No need for treating it as an error, lets continue
+                   with just a message */
+            } else {
+                brickinfo->statfs_fsid = brickstat.f_fsid;
+            }
+        }
+
+        cds_list_add_tail(&brickinfo->brick_list, &volinfo->bricks);
+        brick_count++;
+    }
+
+    if (gf_store_iter_destroy(&tmpiter)) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_STORE_ITER_DESTROY_FAIL,
+               "Failed to destroy store iter");
+        ret = -1;
+        goto out;
+    }
+
+    ret = gf_store_iter_new(volinfo->shandle, &tmpiter);
+
+    if (ret)
+        goto out;
+
+    if (volinfo->thin_arbiter_count == 1) {
+        snprintf(tmpkey, sizeof(tmpkey), "%s-%d",
+                 GLUSTERD_STORE_KEY_VOL_TA_BRICK, 0);
+        while (ta_brick_count < volinfo->subvol_count) {
+            ret = glusterd_brickinfo_new(&ta_brickinfo);
+            if (ret)
+                goto out;
+
+            ret = gf_store_iter_get_matching(tmpiter, tmpkey, &tmpvalue);
+
+            len = snprintf(path, sizeof(path), "%s/%s", brickdir, tmpvalue);
+            GF_FREE(tmpvalue);
+            tmpvalue = NULL;
+            if ((len < 0) || (len >= sizeof(path))) {
+                ret = -1;
                 goto out;
+            }
 
-        ret = gf_store_iter_destroy (iter);
+            ret = gf_store_handle_retrieve(path, &ta_brickinfo->shandle);
 
-        if (ret)
+            if (ret)
                 goto out;
 
-out:
-        gf_log (this->name, GF_LOG_TRACE, "Returning with %d", ret);
+            ret = gf_store_iter_new(ta_brickinfo->shandle, &iter);
 
-        return ret;
-}
-
-int32_t
-glusterd_store_retrieve_node_state (glusterd_volinfo_t *volinfo)
-{
-        int32_t              ret               = -1;
-        gf_store_iter_t     *iter              = NULL;
-        char                *key               = NULL;
-        char                *value             = NULL;
-        char                *dup_value         = NULL;
-        char                 volpath[PATH_MAX] = {0,};
-        glusterd_conf_t     *priv              = NULL;
-        char                 path[PATH_MAX]    = {0,};
-        gf_store_op_errno_t  op_errno          = GD_STORE_SUCCESS;
-        dict_t              *tmp_dict          = NULL;
-        xlator_t            *this              = NULL;
-
-        this = THIS;
-        GF_ASSERT (this);
-        priv = this->private;
-        GF_ASSERT (priv);
-        GF_ASSERT (volinfo);
-
-        GLUSTERD_GET_VOLUME_DIR(volpath, volinfo, priv);
-        snprintf (path, sizeof (path), "%s/%s", volpath,
-                  GLUSTERD_NODE_STATE_FILE);
-
-        ret = gf_store_handle_retrieve (path, &volinfo->node_state_shandle);
-        if (ret)
+            if (ret)
                 goto out;
 
-        ret = gf_store_iter_new (volinfo->node_state_shandle, &iter);
-
-        if (ret)
+            ret = gf_store_iter_get_next(iter, &key, &value, &op_errno);
+            if (ret) {
+                gf_msg("glusterd", GF_LOG_ERROR, op_errno,
+                       GD_MSG_STORE_ITER_GET_FAIL,
+                       "Unable to iterate "
+                       "the store for brick: %s",
+                       path);
                 goto out;
+            }
 
-        ret = gf_store_iter_get_next (iter, &key, &value, &op_errno);
-        if (ret)
-                goto out;
+            while (!ret) {
+                if (!strncmp(key, GLUSTERD_STORE_KEY_BRICK_HOSTNAME,
+                             SLEN(GLUSTERD_STORE_KEY_BRICK_HOSTNAME))) {
+                    if (snprintf(ta_brickinfo->hostname,
+                                 sizeof(ta_brickinfo->hostname), "%s",
+                                 value) >= sizeof(ta_brickinfo->hostname)) {
+                        gf_msg("glusterd", GF_LOG_ERROR, op_errno,
+                               GD_MSG_PARSE_BRICKINFO_FAIL,
+                               "brick hostname truncated: %s",
+                               ta_brickinfo->hostname);
+                        goto out;
+                    }
+                } else if (!strncmp(key, GLUSTERD_STORE_KEY_BRICK_PATH,
+                                    SLEN(GLUSTERD_STORE_KEY_BRICK_PATH))) {
+                    if (snprintf(ta_brickinfo->path, sizeof(ta_brickinfo->path),
+                                 "%s", value) >= sizeof(ta_brickinfo->path)) {
+                        gf_msg("glusterd", GF_LOG_ERROR, op_errno,
+                               GD_MSG_PARSE_BRICKINFO_FAIL,
+                               "brick path truncated: %s", ta_brickinfo->path);
+                        goto out;
+                    }
+                } else if (!strncmp(key, GLUSTERD_STORE_KEY_BRICK_REAL_PATH,
+                                    SLEN(GLUSTERD_STORE_KEY_BRICK_REAL_PATH))) {
+                    if (snprintf(ta_brickinfo->real_path,
+                                 sizeof(ta_brickinfo->real_path), "%s",
+                                 value) >= sizeof(ta_brickinfo->real_path)) {
+                        gf_msg("glusterd", GF_LOG_ERROR, op_errno,
+                               GD_MSG_PARSE_BRICKINFO_FAIL,
+                               "real_path truncated: %s",
+                               ta_brickinfo->real_path);
+                        goto out;
+                    }
+                } else if (!strncmp(key, GLUSTERD_STORE_KEY_BRICK_PORT,
+                                    SLEN(GLUSTERD_STORE_KEY_BRICK_PORT))) {
+                    ret = gf_string2int(value, &ta_brickinfo->port);
+                    if (ret == -1) {
+                        gf_msg(this->name, GF_LOG_ERROR, EINVAL,
+                               GD_MSG_INCOMPATIBLE_VALUE,
+                               "Failed to convert "
+                               "string to integer");
+                    }
+
+                    if (ta_brickinfo->port < priv->base_port) {
+                        /* This is required to adhere to the
+                        IANA standards */
+                        ta_brickinfo->port = 0;
+                    }
+                } else if (!strncmp(key, GLUSTERD_STORE_KEY_BRICK_RDMA_PORT,
+                                    SLEN(GLUSTERD_STORE_KEY_BRICK_RDMA_PORT))) {
+                    ret = gf_string2int(value, &ta_brickinfo->rdma_port);
+                    if (ret == -1) {
+                        gf_msg(this->name, GF_LOG_ERROR, EINVAL,
+                               GD_MSG_INCOMPATIBLE_VALUE,
+                               "Failed to convert "
+                               "string to integer");
+                    }
+
+                    if (ta_brickinfo->rdma_port < priv->base_port) {
+                        /* This is required to adhere to the
+                        IANA standards */
+                        ta_brickinfo->rdma_port = 0;
+                    }
+                } else if (!strncmp(
+                               key, GLUSTERD_STORE_KEY_BRICK_DECOMMISSIONED,
+                               SLEN(GLUSTERD_STORE_KEY_BRICK_DECOMMISSIONED))) {
+                    ret = gf_string2int(value, &ta_brickinfo->decommissioned);
+                    if (ret == -1) {
+                        gf_msg(this->name, GF_LOG_ERROR, EINVAL,
+                               GD_MSG_INCOMPATIBLE_VALUE,
+                               "Failed to convert "
+                               "string to integer");
+                    }
+
+                } else if (!strcmp(key, GLUSTERD_STORE_KEY_BRICK_ID)) {
+                    if (snprintf(ta_brickinfo->brick_id,
+                                 sizeof(ta_brickinfo->brick_id), "%s",
+                                 value) >= sizeof(ta_brickinfo->brick_id)) {
+                        gf_msg("glusterd", GF_LOG_ERROR, op_errno,
+                               GD_MSG_PARSE_BRICKINFO_FAIL,
+                               "brick_id truncated: %s",
+                               ta_brickinfo->brick_id);
+                        goto out;
+                    }
+                } else if (!strncmp(key, GLUSTERD_STORE_KEY_BRICK_FSID,
+                                    SLEN(GLUSTERD_STORE_KEY_BRICK_FSID))) {
+                    ret = gf_string2uint64(value, &ta_brickinfo->statfs_fsid);
+                    if (ret) {
+                        gf_msg(this->name, GF_LOG_ERROR, 0,
+                               GD_MSG_INVALID_ENTRY,
+                               "%s "
+                               "is not a valid uint64_t value",
+                               value);
+                    }
+                } else if (!strcmp(key, GLUSTERD_STORE_KEY_BRICK_UUID)) {
+                    gf_uuid_parse(value, brickinfo->uuid);
+                } else if (!strncmp(
+                               key, GLUSTERD_STORE_KEY_BRICK_SNAP_STATUS,
+                               SLEN(GLUSTERD_STORE_KEY_BRICK_SNAP_STATUS))) {
+                    ret = gf_string2int(value, &ta_brickinfo->snap_status);
+                    if (ret == -1) {
+                        gf_msg(this->name, GF_LOG_ERROR, EINVAL,
+                               GD_MSG_INCOMPATIBLE_VALUE,
+                               "Failed to convert "
+                               "string to integer");
+                    }
 
-        while (ret == 0) {
-                if (!strncmp (key, GLUSTERD_STORE_KEY_VOL_DEFRAG,
-                              strlen (GLUSTERD_STORE_KEY_VOL_DEFRAG))) {
-                        volinfo->rebal.defrag_cmd = atoi (value);
-                } else if (!strncmp (key, GLUSTERD_STORE_KEY_VOL_DEFRAG_STATUS,
-                           strlen (GLUSTERD_STORE_KEY_VOL_DEFRAG_STATUS))) {
-                                volinfo->rebal.defrag_status = atoi (value);
-                } else if (!strncmp (key, GF_REBALANCE_TID_KEY,
-                                     strlen (GF_REBALANCE_TID_KEY))) {
-                        gf_uuid_parse (value, volinfo->rebal.rebalance_id);
-                } else if (!strncmp (key, GLUSTERD_STORE_KEY_DEFRAG_OP,
-                                        strlen (GLUSTERD_STORE_KEY_DEFRAG_OP))) {
-                        volinfo->rebal.op = atoi (value);
                 } else {
-                        if (!tmp_dict) {
-                                tmp_dict = dict_new ();
-                                if (!tmp_dict) {
-                                        ret = -1;
-                                        goto out;
-                                }
-                        }
-                        dup_value = gf_strdup (value);
-                        if (!dup_value) {
-                                ret = -1;
-                                gf_log (this->name, GF_LOG_ERROR,
-                                        "Failed to strdup value string");
-                                goto out;
-                        }
-                        ret = dict_set_str (tmp_dict, key, dup_value);
-                        if (ret) {
-                                gf_log (this->name, GF_LOG_ERROR,
-                                                "Error setting data in rebal "
-                                                "dict.");
-                                goto out;
-                        }
-                        dup_value = NULL;
+                    gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_UNKNOWN_KEY,
+                           "Unknown key: %s", key);
                 }
 
-                GF_FREE (key);
-                GF_FREE (value);
+                GF_FREE(key);
+                GF_FREE(value);
                 key = NULL;
                 value = NULL;
+                ret = gf_store_iter_get_next(iter, &key, &value, &op_errno);
+            }
 
-                ret = gf_store_iter_get_next (iter, &key, &value, &op_errno);
-        }
-        if (tmp_dict)
-                volinfo->rebal.dict = dict_ref (tmp_dict);
+            GLUSTERD_ASSIGN_BRICKID_TO_TA_BRICKINFO(ta_brickinfo, volinfo,
+                                                    ta_brick_id);
+            ta_brick_id += 3;
 
-        if (op_errno != GD_STORE_EOF) {
-                ret = -1;
-                goto out;
+            cds_list_add_tail(&ta_brickinfo->brick_list, &volinfo->ta_bricks);
+            ta_brick_count++;
         }
+    }
 
-        ret = gf_store_iter_destroy (iter);
-
-        if (ret)
-                goto out;
+    assign_brick_groups(volinfo);
+    ret = 0;
 
 out:
-        if (dup_value)
-                GF_FREE (dup_value);
-        if (ret && volinfo->rebal.dict)
-                dict_unref (volinfo->rebal.dict);
-        if (tmp_dict)
-                dict_unref (tmp_dict);
+    if (gf_store_iter_destroy(&tmpiter)) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_STORE_ITER_DESTROY_FAIL,
+               "Failed to destroy store iter");
+        ret = -1;
+    }
 
-        gf_log (this->name, GF_LOG_TRACE, "Returning with %d", ret);
+    if (gf_store_iter_destroy(&iter)) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_STORE_ITER_DESTROY_FAIL,
+               "Failed to destroy store iter");
+        ret = -1;
+    }
 
-        return ret;
-}
+    gf_msg_debug(this->name, 0, "Returning with %d", ret);
 
+    return ret;
+}
 
-int
-glusterd_store_update_volinfo (glusterd_volinfo_t *volinfo)
+int32_t
+glusterd_store_retrieve_node_state(glusterd_volinfo_t *volinfo)
 {
-        int                     ret                     = -1;
-        int                     exists                  = 0;
-        char                    *key                    = NULL;
-        char                    *value                  = NULL;
-        char                    volpath[PATH_MAX]       = {0,};
-        char                    path[PATH_MAX]          = {0,};
-        xlator_t                *this                   = NULL;
-        glusterd_conf_t         *conf                   = NULL;
-        gf_store_iter_t         *iter                   = NULL;
-        gf_store_op_errno_t     op_errno                = GD_STORE_SUCCESS;
-
-        this = THIS;
-        GF_ASSERT (this);
-        conf = THIS->private;
-        GF_ASSERT (volinfo);
-
-        GLUSTERD_GET_VOLUME_DIR(volpath, volinfo, conf);
-
-        snprintf (path, sizeof (path), "%s/%s", volpath,
-                  GLUSTERD_VOLUME_INFO_FILE);
-
-        ret = gf_store_handle_retrieve (path, &volinfo->shandle);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "volinfo handle is NULL");
-                goto out;
-        }
-
-        ret = gf_store_iter_new (volinfo->shandle, &iter);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "Failed to get new store "
-                        "iter");
+    int32_t ret = -1;
+    gf_store_iter_t *iter = NULL;
+    char *key = NULL;
+    char *value = NULL;
+    char *dup_value = NULL;
+    char volpath[PATH_MAX] = {
+        0,
+    };
+    glusterd_conf_t *priv = NULL;
+    char path[PATH_MAX] = {
+        0,
+    };
+    gf_store_op_errno_t op_errno = GD_STORE_SUCCESS;
+    dict_t *tmp_dict = NULL;
+    xlator_t *this = NULL;
+    int32_t len = 0;
+
+    this = THIS;
+    GF_ASSERT(this);
+    priv = this->private;
+    GF_ASSERT(priv);
+    GF_ASSERT(volinfo);
+
+    GLUSTERD_GET_VOLUME_DIR(volpath, volinfo, priv);
+    len = snprintf(path, sizeof(path), "%s/%s", volpath,
+                   GLUSTERD_NODE_STATE_FILE);
+    if ((len < 0) || (len >= PATH_MAX)) {
+        goto out;
+    }
+
+    ret = gf_store_handle_retrieve(path, &volinfo->node_state_shandle);
+    if (ret)
+        goto out;
+
+    ret = gf_store_iter_new(volinfo->node_state_shandle, &iter);
+
+    if (ret)
+        goto out;
+
+    ret = gf_store_iter_get_next(iter, &key, &value, &op_errno);
+
+    if (ret)
+        goto out;
+
+    while (ret == 0) {
+        if (!strncmp(key, GLUSTERD_STORE_KEY_VOL_DEFRAG,
+                     SLEN(GLUSTERD_STORE_KEY_VOL_DEFRAG))) {
+            volinfo->rebal.defrag_cmd = atoi(value);
+        } else if (!strncmp(key, GLUSTERD_STORE_KEY_VOL_DEFRAG_STATUS,
+                            SLEN(GLUSTERD_STORE_KEY_VOL_DEFRAG_STATUS))) {
+            volinfo->rebal.defrag_status = atoi(value);
+        } else if (!strncmp(key, GF_REBALANCE_TID_KEY,
+                            SLEN(GF_REBALANCE_TID_KEY))) {
+            gf_uuid_parse(value, volinfo->rebal.rebalance_id);
+        } else if (!strncmp(key, GLUSTERD_STORE_KEY_DEFRAG_OP,
+                            SLEN(GLUSTERD_STORE_KEY_DEFRAG_OP))) {
+            volinfo->rebal.op = atoi(value);
+        } else if (!strncmp(key, GLUSTERD_STORE_KEY_VOL_DEFRAG_REB_FILES,
+                            SLEN(GLUSTERD_STORE_KEY_VOL_DEFRAG_REB_FILES))) {
+            sscanf(value, "%" PRIu64, &volinfo->rebal.rebalance_files);
+        } else if (!strncmp(key, GLUSTERD_STORE_KEY_VOL_DEFRAG_SIZE,
+                            SLEN(GLUSTERD_STORE_KEY_VOL_DEFRAG_SIZE))) {
+            sscanf(value, "%" PRIu64, &volinfo->rebal.rebalance_data);
+        } else if (!strncmp(key, GLUSTERD_STORE_KEY_VOL_DEFRAG_SCANNED,
+                            SLEN(GLUSTERD_STORE_KEY_VOL_DEFRAG_SCANNED))) {
+            sscanf(value, "%" PRIu64, &volinfo->rebal.lookedup_files);
+        } else if (!strncmp(key, GLUSTERD_STORE_KEY_VOL_DEFRAG_FAILURES,
+                            SLEN(GLUSTERD_STORE_KEY_VOL_DEFRAG_FAILURES))) {
+            sscanf(value, "%" PRIu64, &volinfo->rebal.rebalance_failures);
+        } else if (!strncmp(key, GLUSTERD_STORE_KEY_VOL_DEFRAG_SKIPPED,
+                            SLEN(GLUSTERD_STORE_KEY_VOL_DEFRAG_SKIPPED))) {
+            sscanf(value, "%" PRIu64, &volinfo->rebal.skipped_files);
+        } else if (!strncmp(key, GLUSTERD_STORE_KEY_VOL_DEFRAG_RUN_TIME,
+                            SLEN(GLUSTERD_STORE_KEY_VOL_DEFRAG_RUN_TIME))) {
+            volinfo->rebal.rebalance_time = atoi(value);
+        } else {
+            if (!tmp_dict) {
+                tmp_dict = dict_new();
+                if (!tmp_dict) {
+                    ret = -1;
+                    goto out;
+                }
+            }
+            dup_value = gf_strdup(value);
+            if (!dup_value) {
+                ret = -1;
+                gf_msg(this->name, GF_LOG_ERROR, ENOMEM, GD_MSG_NO_MEMORY,
+                       "Failed to strdup value string");
                 goto out;
-        }
-
-        ret = gf_store_iter_get_next (iter, &key, &value, &op_errno);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "Failed to get next store "
-                        "iter");
+            }
+            ret = dict_set_str(tmp_dict, key, dup_value);
+            if (ret) {
+                gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+                       "Error setting data in rebal "
+                       "dict.");
                 goto out;
+            }
+            dup_value = NULL;
         }
 
-        while (!ret) {
-                gf_log ("", GF_LOG_DEBUG, "key = %s value = %s", key, value);
-                if (!strncmp (key, GLUSTERD_STORE_KEY_VOL_TYPE,
-                              strlen (GLUSTERD_STORE_KEY_VOL_TYPE))) {
-                        volinfo->type = atoi (value);
-                } else if (!strncmp (key, GLUSTERD_STORE_KEY_VOL_COUNT,
-                                     strlen (GLUSTERD_STORE_KEY_VOL_COUNT))) {
-                        volinfo->brick_count = atoi (value);
-                } else if (!strncmp (key, GLUSTERD_STORE_KEY_VOL_STATUS,
-                                     strlen (GLUSTERD_STORE_KEY_VOL_STATUS))) {
-                        volinfo->status = atoi (value);
-                } else if (!strncmp (key, GLUSTERD_STORE_KEY_VOL_VERSION,
-                                     strlen (GLUSTERD_STORE_KEY_VOL_VERSION))) {
-                        volinfo->version = atoi (value);
-                } else if (!strncmp (key, GLUSTERD_STORE_KEY_VOL_PORT,
-                                     strlen (GLUSTERD_STORE_KEY_VOL_PORT))) {
-                        volinfo->port = atoi (value);
-                } else if (!strncmp (key, GLUSTERD_STORE_KEY_VOL_SUB_COUNT,
-                                     strlen (GLUSTERD_STORE_KEY_VOL_SUB_COUNT))) {
-                        volinfo->sub_count = atoi (value);
-                } else if (!strncmp (key, GLUSTERD_STORE_KEY_VOL_STRIPE_CNT,
-                                     strlen (GLUSTERD_STORE_KEY_VOL_STRIPE_CNT))) {
-                        volinfo->stripe_count = atoi (value);
-                } else if (!strncmp (key, GLUSTERD_STORE_KEY_VOL_REPLICA_CNT,
-                                     strlen (GLUSTERD_STORE_KEY_VOL_REPLICA_CNT))) {
-                        volinfo->replica_count = atoi (value);
-                } else if (!strncmp (key, GLUSTERD_STORE_KEY_VOL_DISPERSE_CNT,
-                                     strlen (GLUSTERD_STORE_KEY_VOL_DISPERSE_CNT))) {
-                        volinfo->disperse_count = atoi (value);
-                } else if (!strncmp (key, GLUSTERD_STORE_KEY_VOL_REDUNDANCY_CNT,
-                                     strlen (GLUSTERD_STORE_KEY_VOL_REDUNDANCY_CNT))) {
-                        volinfo->redundancy_count = atoi (value);
-                } else if (!strncmp (key, GLUSTERD_STORE_KEY_VOL_TRANSPORT,
-                                     strlen (GLUSTERD_STORE_KEY_VOL_TRANSPORT))) {
-                        volinfo->transport_type = atoi (value);
-                } else if (!strncmp (key, GLUSTERD_STORE_KEY_VOL_ID,
-                                     strlen (GLUSTERD_STORE_KEY_VOL_ID))) {
-                        ret = gf_uuid_parse (value, volinfo->volume_id);
-                        if (ret)
-                                gf_log ("", GF_LOG_WARNING,
-                                        "failed to parse uuid");
-
-                } else if (!strncmp (key, GLUSTERD_STORE_KEY_USERNAME,
-                                     strlen (GLUSTERD_STORE_KEY_USERNAME))) {
-
-                        glusterd_auth_set_username (volinfo, value);
-
-                } else if (!strncmp (key, GLUSTERD_STORE_KEY_PASSWORD,
-                                     strlen (GLUSTERD_STORE_KEY_PASSWORD))) {
-
-                        glusterd_auth_set_password (volinfo, value);
-
-                } else if (strstr (key, "slave")) {
-                        ret = dict_set_dynstr (volinfo->gsync_slaves, key,
-                                                gf_strdup (value));
-                        if (ret) {
-                                gf_log ("",GF_LOG_ERROR, "Error in "
-                                                "dict_set_str");
-                                goto out;
-                        }
-                        gf_log ("", GF_LOG_DEBUG, "Parsed as "GEOREP" "
-                                " slave:key=%s,value:%s", key, value);
-
-                } else if (!strncmp (key, GLUSTERD_STORE_KEY_VOL_OP_VERSION,
-                                strlen (GLUSTERD_STORE_KEY_VOL_OP_VERSION))) {
-                        volinfo->op_version = atoi (value);
-                } else if (!strncmp (key, GLUSTERD_STORE_KEY_VOL_CLIENT_OP_VERSION,
-                                strlen (GLUSTERD_STORE_KEY_VOL_CLIENT_OP_VERSION))) {
-                        volinfo->client_op_version = atoi (value);
-                } else if (!strncmp (key, GLUSTERD_STORE_KEY_VOL_CAPS,
-                                     strlen (GLUSTERD_STORE_KEY_VOL_CAPS))) {
-                        volinfo->caps = atoi (value);
-                } else if (!strncmp (key, GLUSTERD_STORE_KEY_SNAP_MAX_HARD_LIMIT,
-                                strlen (GLUSTERD_STORE_KEY_SNAP_MAX_HARD_LIMIT))) {
-                        volinfo->snap_max_hard_limit = (uint64_t) atoll (value);
-                } else if (!strncmp (key, GLUSTERD_STORE_KEY_VOL_RESTORED_SNAP,
-                              strlen (GLUSTERD_STORE_KEY_VOL_RESTORED_SNAP))) {
-                        ret = gf_uuid_parse (value, volinfo->restored_from_snap);
-                        if (ret)
-                                gf_log (this->name, GF_LOG_WARNING,
-                                        "failed to parse restored snap's uuid");
-                } else if (!strncmp (key, GLUSTERD_STORE_KEY_PARENT_VOLNAME,
-                                strlen (GLUSTERD_STORE_KEY_PARENT_VOLNAME))) {
-                        strncpy (volinfo->parent_volname, value,
-                                 sizeof(volinfo->parent_volname) - 1);
-                } else if (!strncmp (key, GLUSTERD_STORE_KEY_COLD_COUNT,
-                                     strlen (key))) {
-                        volinfo->tier_info.cold_brick_count = atoi (value);
-                } else if (!strncmp (key, GLUSTERD_STORE_KEY_COLD_REPLICA_COUNT,
-                                     strlen (key))) {
-                        volinfo->tier_info.cold_replica_count = atoi (value);
-                } else if (!strncmp (key, GLUSTERD_STORE_KEY_COLD_DISPERSE_COUNT,
-                                     strlen (key))) {
-                        volinfo->tier_info.cold_disperse_count = atoi (value);
-                } else if (!strncmp (key, GLUSTERD_STORE_KEY_HOT_COUNT,
-                                     strlen (key))) {
-                        volinfo->tier_info.cold_brick_count = atoi (value);
-                } else if (!strncmp (key, GLUSTERD_STORE_KEY_HOT_REPLICA_COUNT,
-                                     strlen (key))) {
-                        volinfo->tier_info.cold_replica_count = atoi (value);
-                } else if (!strncmp (key, GLUSTERD_STORE_KEY_HOT_TYPE,
-                                     strlen (key))) {
-                        volinfo->tier_info.hot_type = atoi (value);
-                } else if (!strncmp (key, GLUSTERD_STORE_KEY_COLD_TYPE,
-                                     strlen (key))) {
-                        volinfo->tier_info.cold_type = atoi (value);
-                } else {
-
-                        if (is_key_glusterd_hooks_friendly (key)) {
-                                exists = 1;
-
-                        } else  {
-                                exists = glusterd_check_option_exists (key,
-                                                                       NULL);
-                        }
-
-                        switch (exists) {
-                        case -1:
-                                ret = -1;
-                                goto out;
-
-                        case 0:
-                                 /*Ignore GLUSTERD_STORE_KEY_VOL_BRICK since
-                                  glusterd_store_retrieve_bricks gets it later*/
-                                if (!strstr (key, GLUSTERD_STORE_KEY_VOL_BRICK))
-                                        gf_log ("", GF_LOG_WARNING,
-                                                "Unknown key: %s", key);
-                                break;
-
-                        case 1:
-                                /*The following strcmp check is to ensure that
-                                 * glusterd does not restore the quota limits
-                                 * into volinfo->dict post upgradation from 3.3
-                                 * to 3.4 as the same limits will now be stored
-                                 * in xattrs on the respective directories.
-                                 */
-                                if (!strcmp (key, "features.limit-usage"))
-                                        break;
-                                ret = dict_set_str(volinfo->dict, key,
-                                                   gf_strdup (value));
-                                if (ret) {
-                                        gf_log ("",GF_LOG_ERROR, "Error in "
-                                                "dict_set_str");
-                                        goto out;
-                                }
-                                gf_log ("", GF_LOG_DEBUG, "Parsed as Volume-"
-                                        "set:key=%s,value:%s", key, value);
-                                break;
-                        }
-                }
-
-                GF_FREE (key);
-                GF_FREE (value);
-                key = NULL;
-                value = NULL;
-
-                ret = gf_store_iter_get_next (iter, &key, &value, &op_errno);
-        }
-
-        /* backward compatibility */
-        {
+        GF_FREE(key);
+        GF_FREE(value);
+        key = NULL;
+        value = NULL;
 
-                switch (volinfo->type) {
+        ret = gf_store_iter_get_next(iter, &key, &value, &op_errno);
+    }
+    if (tmp_dict) {
+        volinfo->rebal.dict = dict_ref(tmp_dict);
+    }
 
-                        case GF_CLUSTER_TYPE_NONE:
-                                volinfo->stripe_count  = 1;
-                                volinfo->replica_count = 1;
-                        break;
+    if (op_errno != GD_STORE_EOF) {
+        ret = -1;
+        goto out;
+    }
 
-                        case GF_CLUSTER_TYPE_STRIPE:
-                                volinfo->stripe_count  = volinfo->sub_count;
-                                volinfo->replica_count = 1;
-                        break;
+    ret = 0;
 
-                        case GF_CLUSTER_TYPE_REPLICATE:
-                                volinfo->stripe_count  = 1;
-                                volinfo->replica_count = volinfo->sub_count;
-                        break;
-
-                        case GF_CLUSTER_TYPE_STRIPE_REPLICATE:
-                                /* Introduced in 3.3 */
-                                GF_ASSERT (volinfo->stripe_count > 0);
-                                GF_ASSERT (volinfo->replica_count > 0);
-                        break;
-
-                        case GF_CLUSTER_TYPE_DISPERSE:
-                                GF_ASSERT (volinfo->disperse_count > 0);
-                                GF_ASSERT (volinfo->redundancy_count > 0);
-                        break;
-
-                        case GF_CLUSTER_TYPE_TIER:
-                        break;
-
-                        default:
-                                GF_ASSERT (0);
-                        break;
-                }
-
-                volinfo->dist_leaf_count = glusterd_get_dist_leaf_count (volinfo);
-
-                volinfo->subvol_count = (volinfo->brick_count /
-                                         volinfo->dist_leaf_count);
-
-                /* Only calculate volume op-versions if they are not found */
-                if (!volinfo->op_version && !volinfo->client_op_version)
-                        gd_update_volume_op_versions (volinfo);
-        }
-
-        if (op_errno != GD_STORE_EOF)
-                goto out;
-
-        ret = gf_store_iter_destroy (iter);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "Failed to destroy store "
-                        "iter");
-                goto out;
-        }
-
-        ret = 0;
 out:
-        return ret;
+    if (gf_store_iter_destroy(&iter)) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_STORE_ITER_DESTROY_FAIL,
+               "Failed to destroy store iter");
+        ret = -1;
+    }
+
+    if (dup_value)
+        GF_FREE(dup_value);
+    if (ret) {
+        if (volinfo->rebal.dict)
+            dict_unref(volinfo->rebal.dict);
+    }
+    if (tmp_dict)
+        dict_unref(tmp_dict);
+
+    gf_msg_trace(this->name, 0, "Returning with %d", ret);
+
+    return ret;
 }
 
-glusterd_volinfo_t*
-glusterd_store_retrieve_volume (char *volname, glusterd_snap_t *snap)
+int
+glusterd_store_update_volinfo(glusterd_volinfo_t *volinfo)
 {
-        int32_t                    ret                  = -1;
-        glusterd_volinfo_t        *volinfo              = NULL;
-        glusterd_volinfo_t        *origin_volinfo       = NULL;
-        glusterd_conf_t           *priv                 = NULL;
-        xlator_t                  *this                 = NULL;
-
+    int ret = -1;
+    int exists = 0;
+    char *key = NULL;
+    char *value = NULL;
+    char volpath[PATH_MAX] = {
+        0,
+    };
+    char path[PATH_MAX] = {
+        0,
+    };
+    xlator_t *this = NULL;
+    glusterd_conf_t *conf = NULL;
+    gf_store_iter_t *iter = NULL;
+    gf_store_op_errno_t op_errno = GD_STORE_SUCCESS;
+    int32_t len = 0;
+
+    this = THIS;
+    GF_ASSERT(this);
+    conf = THIS->private;
+    GF_ASSERT(volinfo);
+
+    GLUSTERD_GET_VOLUME_DIR(volpath, volinfo, conf);
+
+    len = snprintf(path, sizeof(path), "%s/%s", volpath,
+                   GLUSTERD_VOLUME_INFO_FILE);
+    if ((len < 0) || (len >= sizeof(path))) {
+        goto out;
+    }
+
+    ret = gf_store_handle_retrieve(path, &volinfo->shandle);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_HANDLE_NULL,
+               "volinfo handle is NULL");
+        goto out;
+    }
+
+    ret = gf_store_iter_new(volinfo->shandle, &iter);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_STORE_ITER_GET_FAIL,
+               "Failed to get new store "
+               "iter");
+        goto out;
+    }
+
+    ret = gf_store_iter_get_next(iter, &key, &value, &op_errno);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_STORE_ITER_GET_FAIL,
+               "Failed to get next store "
+               "iter");
+        goto out;
+    }
+
+    while (!ret) {
+        gf_msg_debug(this->name, 0, "key = %s value = %s", key, value);
+        if (!strncmp(key, GLUSTERD_STORE_KEY_VOL_TYPE,
+                     SLEN(GLUSTERD_STORE_KEY_VOL_TYPE))) {
+            volinfo->type = atoi(value);
+        } else if (!strncmp(key, GLUSTERD_STORE_KEY_VOL_COUNT,
+                            SLEN(GLUSTERD_STORE_KEY_VOL_COUNT))) {
+            volinfo->brick_count = atoi(value);
+        } else if (!strncmp(key, GLUSTERD_STORE_KEY_VOL_STATUS,
+                            SLEN(GLUSTERD_STORE_KEY_VOL_STATUS))) {
+            volinfo->status = atoi(value);
+        } else if (!strncmp(key, GLUSTERD_STORE_KEY_VOL_VERSION,
+                            SLEN(GLUSTERD_STORE_KEY_VOL_VERSION))) {
+            volinfo->version = atoi(value);
+        } else if (!strncmp(key, GLUSTERD_STORE_KEY_VOL_PORT,
+                            SLEN(GLUSTERD_STORE_KEY_VOL_PORT))) {
+            volinfo->port = atoi(value);
+        } else if (!strncmp(key, GLUSTERD_STORE_KEY_VOL_SUB_COUNT,
+                            SLEN(GLUSTERD_STORE_KEY_VOL_SUB_COUNT))) {
+            volinfo->sub_count = atoi(value);
+        } else if (!strncmp(key, GLUSTERD_STORE_KEY_VOL_STRIPE_CNT,
+                            SLEN(GLUSTERD_STORE_KEY_VOL_STRIPE_CNT))) {
+            volinfo->stripe_count = atoi(value);
+        } else if (!strncmp(key, GLUSTERD_STORE_KEY_VOL_REPLICA_CNT,
+                            SLEN(GLUSTERD_STORE_KEY_VOL_REPLICA_CNT))) {
+            volinfo->replica_count = atoi(value);
+        } else if (!strcmp(key, GLUSTERD_STORE_KEY_VOL_ARBITER_CNT)) {
+            volinfo->arbiter_count = atoi(value);
+        } else if (!strcmp(key, GLUSTERD_STORE_KEY_VOL_THIN_ARBITER_CNT)) {
+            volinfo->thin_arbiter_count = atoi(value);
+        } else if (!strncmp(key, GLUSTERD_STORE_KEY_VOL_DISPERSE_CNT,
+                            SLEN(GLUSTERD_STORE_KEY_VOL_DISPERSE_CNT))) {
+            volinfo->disperse_count = atoi(value);
+        } else if (!strncmp(key, GLUSTERD_STORE_KEY_VOL_REDUNDANCY_CNT,
+                            SLEN(GLUSTERD_STORE_KEY_VOL_REDUNDANCY_CNT))) {
+            volinfo->redundancy_count = atoi(value);
+        } else if (!strncmp(key, GLUSTERD_STORE_KEY_VOL_TRANSPORT,
+                            SLEN(GLUSTERD_STORE_KEY_VOL_TRANSPORT))) {
+            volinfo->transport_type = atoi(value);
+        } else if (!strncmp(key, GLUSTERD_STORE_KEY_VOL_ID,
+                            SLEN(GLUSTERD_STORE_KEY_VOL_ID))) {
+            ret = gf_uuid_parse(value, volinfo->volume_id);
+            if (ret)
+                gf_msg(this->name, GF_LOG_WARNING, 0, GD_MSG_UUID_PARSE_FAIL,
+                       "failed to parse uuid");
+
+        } else if (!strncmp(key, GLUSTERD_STORE_KEY_USERNAME,
+                            SLEN(GLUSTERD_STORE_KEY_USERNAME))) {
+            glusterd_auth_set_username(volinfo, value);
+
+        } else if (!strncmp(key, GLUSTERD_STORE_KEY_PASSWORD,
+                            SLEN(GLUSTERD_STORE_KEY_PASSWORD))) {
+            glusterd_auth_set_password(volinfo, value);
+
+        } else if (strstr(key, "slave")) {
+            ret = dict_set_dynstr(volinfo->gsync_slaves, key, gf_strdup(value));
+            if (ret) {
+                gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+                       "Error in "
+                       "dict_set_str");
+                goto out;
+            }
+            gf_msg_debug(this->name, 0,
+                         "Parsed as " GEOREP
+                         " "
+                         " slave:key=%s,value:%s",
+                         key, value);
+
+        } else if (!strncmp(key, GLUSTERD_STORE_KEY_VOL_OP_VERSION,
+                            SLEN(GLUSTERD_STORE_KEY_VOL_OP_VERSION))) {
+            volinfo->op_version = atoi(value);
+        } else if (!strncmp(key, GLUSTERD_STORE_KEY_VOL_CLIENT_OP_VERSION,
+                            SLEN(GLUSTERD_STORE_KEY_VOL_CLIENT_OP_VERSION))) {
+            volinfo->client_op_version = atoi(value);
+        } else if (!strncmp(key, GLUSTERD_STORE_KEY_SNAP_MAX_HARD_LIMIT,
+                            SLEN(GLUSTERD_STORE_KEY_SNAP_MAX_HARD_LIMIT))) {
+            volinfo->snap_max_hard_limit = (uint64_t)atoll(value);
+        } else if (!strncmp(key, GLUSTERD_STORE_KEY_VOL_RESTORED_SNAP,
+                            SLEN(GLUSTERD_STORE_KEY_VOL_RESTORED_SNAP))) {
+            ret = gf_uuid_parse(value, volinfo->restored_from_snap);
+            if (ret)
+                gf_msg(this->name, GF_LOG_WARNING, 0, GD_MSG_UUID_PARSE_FAIL,
+                       "failed to parse restored snap's uuid");
+        } else if (!strncmp(key, GLUSTERD_STORE_KEY_PARENT_VOLNAME,
+                            SLEN(GLUSTERD_STORE_KEY_PARENT_VOLNAME))) {
+            if (snprintf(volinfo->parent_volname,
+                         sizeof(volinfo->parent_volname), "%s",
+                         value) >= sizeof(volinfo->parent_volname)) {
+                gf_msg("glusterd", GF_LOG_ERROR, op_errno,
+                       GD_MSG_PARSE_BRICKINFO_FAIL,
+                       "parent_volname truncated: %s", volinfo->parent_volname);
+                goto out;
+            }
+        } else if (!strncmp(key, GLUSTERD_STORE_KEY_VOL_QUOTA_VERSION,
+                            SLEN(GLUSTERD_STORE_KEY_VOL_QUOTA_VERSION))) {
+            volinfo->quota_xattr_version = atoi(value);
+        } else {
+            if (is_key_glusterd_hooks_friendly(key)) {
+                exists = 1;
 
-        this = THIS;
-        GF_ASSERT (this);
-        priv = this->private;
-        GF_ASSERT (priv);
-        GF_ASSERT (volname);
+            } else {
+                exists = glusterd_check_option_exists(key, NULL);
+            }
+
+            switch (exists) {
+                case -1:
+                    ret = -1;
+                    goto out;
+
+                case 0:
+                    /*Ignore GLUSTERD_STORE_KEY_VOL_BRICK since
+                     glusterd_store_retrieve_bricks gets it later.
+                     also, ignore tier-enabled key as we deprecated
+                     tier xlator*/
+                    if (!strstr(key, GLUSTERD_STORE_KEY_VOL_BRICK) ||
+                        !strstr(key, GF_TIER_ENABLED))
+                        gf_msg(this->name, GF_LOG_WARNING, 0,
+                               GD_MSG_UNKNOWN_KEY, "Unknown key: %s", key);
+                    break;
+
+                case 1:
+                    /*The following strcmp check is to ensure that
+                     * glusterd does not restore the quota limits
+                     * into volinfo->dict post upgradation from 3.3
+                     * to 3.4 as the same limits will now be stored
+                     * in xattrs on the respective directories.
+                     */
+                    if (!strcmp(key, "features.limit-usage"))
+                        break;
+                    ret = dict_set_str(volinfo->dict, key, gf_strdup(value));
+                    if (ret) {
+                        gf_msg(this->name, GF_LOG_ERROR, 0,
+                               GD_MSG_DICT_SET_FAILED,
+                               "Error in "
+                               "dict_set_str");
+                        goto out;
+                    }
+                    gf_msg_debug(this->name, 0,
+                                 "Parsed as Volume-"
+                                 "set:key=%s,value:%s",
+                                 key, value);
+                    break;
+            }
+        }
+
+        GF_FREE(key);
+        GF_FREE(value);
+        key = NULL;
+        value = NULL;
+
+        ret = gf_store_iter_get_next(iter, &key, &value, &op_errno);
+    }
+
+    /* backward compatibility */
+    {
+        switch (volinfo->type) {
+            case GF_CLUSTER_TYPE_NONE:
+                volinfo->stripe_count = 1;
+                volinfo->replica_count = 1;
+                break;
+
+            case GF_CLUSTER_TYPE_REPLICATE:
+                volinfo->stripe_count = 1;
+                volinfo->replica_count = volinfo->sub_count;
+                break;
+
+            case GF_CLUSTER_TYPE_DISPERSE:
+                GF_ASSERT(volinfo->disperse_count > 0);
+                GF_ASSERT(volinfo->redundancy_count > 0);
+                break;
+
+            case GF_CLUSTER_TYPE_STRIPE:
+            case GF_CLUSTER_TYPE_STRIPE_REPLICATE:
+                gf_msg(this->name, GF_LOG_CRITICAL, ENOTSUP,
+                       GD_MSG_VOLINFO_STORE_FAIL,
+                       "The volume type is no more supported. Please refer to "
+                       "glusterfs-6.0 release-notes for how to migrate from "
+                       "this volume type");
+                break;
+
+            default:
+                GF_ASSERT(0);
+                break;
+        }
+
+        volinfo->dist_leaf_count = glusterd_get_dist_leaf_count(volinfo);
+
+        volinfo->subvol_count = (volinfo->brick_count /
+                                 volinfo->dist_leaf_count);
+
+        /* Only calculate volume op-versions if they are not found */
+        if (!volinfo->op_version && !volinfo->client_op_version)
+            gd_update_volume_op_versions(volinfo);
+    }
+
+    if (op_errno != GD_STORE_EOF)
+        goto out;
+
+    ret = 0;
 
-        ret = glusterd_volinfo_new (&volinfo);
-        if (ret)
-                goto out;
+out:
+    if (gf_store_iter_destroy(&iter)) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_STORE_ITER_DESTROY_FAIL,
+               "Failed to destroy store iter");
+        ret = -1;
+    }
 
-        strncpy (volinfo->volname, volname, GD_VOLUME_NAME_MAX);
-        volinfo->snapshot = snap;
-        if (snap)
-                volinfo->is_snap_volume = _gf_true;
+    return ret;
+}
 
-        ret = glusterd_store_update_volinfo (volinfo);
+glusterd_volinfo_t *
+glusterd_store_retrieve_volume(char *volname, glusterd_snap_t *snap)
+{
+    int32_t ret = -1;
+    glusterd_volinfo_t *volinfo = NULL;
+    glusterd_volinfo_t *origin_volinfo = NULL;
+    glusterd_conf_t *priv = NULL;
+    xlator_t *this = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+    priv = this->private;
+    GF_ASSERT(priv);
+    GF_ASSERT(volname);
+
+    ret = glusterd_volinfo_new(&volinfo);
+    if (ret)
+        goto out;
+
+    if (snprintf(volinfo->volname, NAME_MAX + 1, "%s", volname) >= NAME_MAX + 1)
+        goto out;
+    volinfo->snapshot = snap;
+    if (snap)
+        volinfo->is_snap_volume = _gf_true;
+
+    ret = glusterd_store_update_volinfo(volinfo);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_VOLINFO_UPDATE_FAIL,
+               "Failed to update volinfo "
+               "for %s volume",
+               volname);
+        goto out;
+    }
+
+    ret = glusterd_store_retrieve_bricks(volinfo);
+    if (ret)
+        goto out;
+
+    ret = glusterd_store_retrieve_snapd(volinfo);
+    if (ret)
+        goto out;
+
+    ret = glusterd_compute_cksum(volinfo, _gf_false);
+    if (ret)
+        goto out;
+
+    ret = glusterd_store_retrieve_quota_version(volinfo);
+    if (ret)
+        goto out;
+
+    ret = glusterd_store_create_quota_conf_sh_on_absence(volinfo);
+    if (ret)
+        goto out;
+
+    ret = glusterd_compute_cksum(volinfo, _gf_true);
+    if (ret)
+        goto out;
+
+    ret = glusterd_store_save_quota_version_and_cksum(volinfo);
+    if (ret)
+        goto out;
+
+    if (!snap) {
+        glusterd_list_add_order(&volinfo->vol_list, &priv->volumes,
+                                glusterd_compare_volume_name);
+
+    } else {
+        ret = glusterd_volinfo_find(volinfo->parent_volname, &origin_volinfo);
         if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "Failed to update volinfo "
-                        "for %s volume", volname);
-                goto out;
-        }
-
-        ret = glusterd_store_retrieve_bricks (volinfo);
-        if (ret)
-                goto out;
-
-        ret = glusterd_store_retrieve_snapd (volinfo);
-        if (ret)
-                goto out;
-
-        ret = glusterd_compute_cksum (volinfo, _gf_false);
-        if (ret)
-                goto out;
-
-        ret = glusterd_store_retrieve_quota_version (volinfo);
-        if (ret)
-                goto out;
-
-        ret = glusterd_store_create_quota_conf_sh_on_absence (volinfo);
-        if (ret)
-                goto out;
-
-        ret = glusterd_compute_cksum (volinfo, _gf_true);
-        if (ret)
-                goto out;
-
-        ret = glusterd_store_save_quota_version_and_cksum (volinfo);
-        if (ret)
-                goto out;
-
-
-        if (!snap) {
-                glusterd_list_add_order (&volinfo->vol_list, &priv->volumes,
-                                         glusterd_compare_volume_name);
-
-        } else {
-                ret = glusterd_volinfo_find (volinfo->parent_volname,
-                                             &origin_volinfo);
-                if (ret) {
-                        gf_log (this->name, GF_LOG_ERROR, "Parent volinfo "
-                                "not found for %s volume", volname);
-                        goto out;
-                }
-                glusterd_list_add_snapvol (origin_volinfo, volinfo);
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_VOLINFO_GET_FAIL,
+                   "Parent volinfo "
+                   "not found for %s volume",
+                   volname);
+            goto out;
         }
+        glusterd_list_add_snapvol(origin_volinfo, volinfo);
+    }
 
 out:
-        if (ret) {
-                if (volinfo)
-                        glusterd_volinfo_unref (volinfo);
-                volinfo = NULL;
-        }
+    if (ret) {
+        if (volinfo)
+            glusterd_volinfo_unref(volinfo);
+        volinfo = NULL;
+    }
 
-        gf_log (this->name, GF_LOG_TRACE, "Returning with %d", ret);
+    gf_msg_trace(this->name, 0, "Returning with %d", ret);
 
-        return volinfo;
+    return volinfo;
 }
 
-static inline void
-glusterd_store_set_options_path (glusterd_conf_t *conf, char *path, size_t len)
+static void
+glusterd_store_set_options_path(glusterd_conf_t *conf, char *path, size_t len)
 {
-        snprintf (path, len, "%s/options", conf->workdir);
-}
-
-int
-_store_global_opts (dict_t *this, char *key, data_t *value, void *data)
-{
-        gf_store_handle_t *shandle = data;
-
-        gf_store_save_value (shandle->fd, key, (char*)value->data);
-        return 0;
+    snprintf(path, len, "%s/options", conf->workdir);
 }
 
 int32_t
-glusterd_store_options (xlator_t *this, dict_t *opts)
+glusterd_store_options(xlator_t *this, dict_t *opts)
 {
-        gf_store_handle_t               *shandle = NULL;
-        glusterd_conf_t                 *conf = NULL;
-        char                            path[PATH_MAX] = {0};
-        int                             fd = -1;
-        int32_t                         ret = -1;
-
-        conf = this->private;
-        glusterd_store_set_options_path (conf, path, sizeof (path));
-
-        ret = gf_store_handle_new (path, &shandle);
-        if (ret)
-                goto out;
-
-        fd = gf_store_mkstemp (shandle);
-        if (fd <= 0) {
-                ret = -1;
-                goto out;
+    gf_store_handle_t *shandle = NULL;
+    glusterd_conf_t *conf = NULL;
+    char path[PATH_MAX] = {0};
+    int fd = -1;
+    int32_t ret = -1;
+    glusterd_volinfo_data_store_t *dict_data = NULL;
+
+    conf = this->private;
+    glusterd_store_set_options_path(conf, path, sizeof(path));
+
+    ret = gf_store_handle_new(path, &shandle);
+    if (ret) {
+        goto out;
+    }
+
+    fd = gf_store_mkstemp(shandle);
+    if (fd <= 0) {
+        ret = -1;
+        goto out;
+    }
+
+    dict_data = GF_CALLOC(1, sizeof(glusterd_volinfo_data_store_t),
+                          gf_gld_mt_volinfo_dict_data_t);
+    if (dict_data == NULL) {
+        gf_smsg(this->name, GF_LOG_ERROR, 0, GD_MSG_NO_MEMORY, NULL);
+        return -1;
+    }
+    dict_data->shandle = shandle;
+    shandle->fd = fd;
+    dict_foreach(opts, _storeopts, (void *)dict_data);
+    if (dict_data->buffer_len > 0) {
+        ret = gf_store_save_items(fd, dict_data->buffer);
+        if (ret) {
+            gf_smsg(this->name, GF_LOG_ERROR, 0, GD_MSG_FILE_OP_FAILED, NULL);
+            goto out;
         }
+    }
 
-        shandle->fd = fd;
-        dict_foreach (opts, _store_global_opts, shandle);
-        shandle->fd = 0;
-        ret = gf_store_rename_tmppath (shandle);
-        if (ret)
-                goto out;
+    ret = gf_store_rename_tmppath(shandle);
 out:
-        if ((ret < 0) && (fd > 0))
-                gf_store_unlink_tmppath (shandle);
-        gf_store_handle_destroy (shandle);
-        return ret;
+    shandle->fd = 0;
+    GF_FREE(dict_data);
+    if ((ret < 0) && (fd > 0)) {
+        gf_store_unlink_tmppath(shandle);
+    }
+    gf_store_handle_destroy(shandle);
+    return ret;
 }
 
 int32_t
-glusterd_store_retrieve_options (xlator_t *this)
+glusterd_store_retrieve_options(xlator_t *this)
 {
-        char                            path[PATH_MAX] = {0};
-        glusterd_conf_t                 *conf = NULL;
-        gf_store_handle_t               *shandle = NULL;
-        gf_store_iter_t                 *iter = NULL;
-        char                            *key = NULL;
-        char                            *value = NULL;
-        gf_store_op_errno_t             op_errno = 0;
-        int                             ret = -1;
-
-        conf = this->private;
-        glusterd_store_set_options_path (conf, path, sizeof (path));
-
-        ret = gf_store_handle_retrieve (path, &shandle);
-        if (ret)
-                goto out;
-
-        ret = gf_store_iter_new (shandle, &iter);
-        if (ret)
-                goto out;
-
-        ret = gf_store_iter_get_next (iter, &key, &value, &op_errno);
-        while (!ret) {
-                ret = dict_set_dynstr (conf->opts, key, value);
-                if (ret) {
-                        GF_FREE (key);
-                        GF_FREE (value);
-                        goto out;
-                }
-                GF_FREE (key);
-                key = NULL;
-                value = NULL;
-
-                ret = gf_store_iter_get_next (iter, &key, &value, &op_errno);
-        }
-        if (op_errno != GD_STORE_EOF)
-                goto out;
-        ret = 0;
+    char path[PATH_MAX] = {0};
+    glusterd_conf_t *conf = NULL;
+    gf_store_handle_t *shandle = NULL;
+    gf_store_iter_t *iter = NULL;
+    char *key = NULL;
+    char *value = NULL;
+    gf_store_op_errno_t op_errno = 0;
+    int ret = -1;
+
+    conf = this->private;
+    glusterd_store_set_options_path(conf, path, sizeof(path));
+
+    ret = gf_store_handle_retrieve(path, &shandle);
+    if (ret)
+        goto out;
+
+    ret = gf_store_iter_new(shandle, &iter);
+    if (ret)
+        goto out;
+
+    ret = gf_store_iter_get_next(iter, &key, &value, &op_errno);
+    while (!ret) {
+        ret = dict_set_dynstr(conf->opts, key, value);
+        if (ret) {
+            GF_FREE(key);
+            GF_FREE(value);
+            goto out;
+        }
+        GF_FREE(key);
+        key = NULL;
+        value = NULL;
+
+        ret = gf_store_iter_get_next(iter, &key, &value, &op_errno);
+    }
+    if (op_errno != GD_STORE_EOF)
+        goto out;
+    ret = 0;
 out:
-        gf_store_iter_destroy (iter);
-        gf_store_handle_destroy (shandle);
-        return ret;
+    (void)gf_store_iter_destroy(&iter);
+    gf_store_handle_destroy(shandle);
+    return ret;
 }
 
 int32_t
-glusterd_store_retrieve_volumes (xlator_t  *this, glusterd_snap_t *snap)
+glusterd_store_retrieve_volumes(xlator_t *this, glusterd_snap_t *snap)
 {
-        int32_t                ret              = -1;
-        char                   path[PATH_MAX]   = {0,};
-        glusterd_conf_t       *priv             = NULL;
-        DIR                   *dir              = NULL;
-        struct dirent         *entry            = NULL;
-        glusterd_volinfo_t    *volinfo          = NULL;
-
-        GF_ASSERT (this);
-        priv = this->private;
-
-        GF_ASSERT (priv);
-
-        if (snap)
-                snprintf (path, PATH_MAX, "%s/snaps/%s", priv->workdir,
-                          snap->snapname);
-        else
-                snprintf (path, PATH_MAX, "%s/%s", priv->workdir,
-                          GLUSTERD_VOLUME_DIR_PREFIX);
-
-        dir = opendir (path);
-
-        if (!dir) {
-                gf_log ("", GF_LOG_ERROR, "Unable to open dir %s", path);
-                goto out;
-        }
-
-        GF_FOR_EACH_ENTRY_IN_DIR (entry, dir);
-
-        while (entry) {
-                if (snap && ((!strcmp (entry->d_name, "geo-replication")) ||
-                             (!strcmp (entry->d_name, "info"))))
-                        goto next;
-
-                volinfo = glusterd_store_retrieve_volume (entry->d_name, snap);
-                if (!volinfo) {
-                        gf_log ("", GF_LOG_ERROR, "Unable to restore "
-                                "volume: %s", entry->d_name);
-                        ret = -1;
-                        goto out;
-                }
-
-                ret = glusterd_store_retrieve_rbstate (volinfo);
-                if (ret) {
-                        /* Backward compatibility */
-                        gf_log ("", GF_LOG_INFO, "Creating a new rbstate "
-                                "for volume: %s.", entry->d_name);
-                        ret = glusterd_store_create_rbstate_shandle_on_absence (volinfo);
-                        ret = glusterd_store_perform_rbstate_store (volinfo);
-                }
-
-                ret = glusterd_store_retrieve_node_state (volinfo);
-                if (ret) {
-                        /* Backward compatibility */
-                        gf_log ("", GF_LOG_INFO, "Creating a new node_state "
-                                "for volume: %s.", entry->d_name);
-                        glusterd_store_create_nodestate_sh_on_absence (volinfo);
-                        ret = glusterd_store_perform_node_state_store (volinfo);
-
-                }
-next:
-                GF_FOR_EACH_ENTRY_IN_DIR (entry, dir);
+    int32_t ret = -1;
+    char path[PATH_MAX] = {
+        0,
+    };
+    glusterd_conf_t *priv = NULL;
+    DIR *dir = NULL;
+    struct dirent *entry = NULL;
+    struct dirent scratch[2] = {
+        {
+            0,
+        },
+    };
+    glusterd_volinfo_t *volinfo = NULL;
+    struct stat st = {
+        0,
+    };
+    char entry_path[PATH_MAX] = {
+        0,
+    };
+    int32_t len = 0;
+
+    GF_ASSERT(this);
+    priv = this->private;
+
+    GF_ASSERT(priv);
+
+    if (snap)
+        len = snprintf(path, PATH_MAX, "%s/snaps/%s", priv->workdir,
+                       snap->snapname);
+    else
+        len = snprintf(path, PATH_MAX, "%s/%s", priv->workdir,
+                       GLUSTERD_VOLUME_DIR_PREFIX);
+    if ((len < 0) || (len >= PATH_MAX)) {
+        goto out;
+    }
+
+    dir = sys_opendir(path);
+
+    if (!dir) {
+        gf_msg(this->name, GF_LOG_ERROR, errno, GD_MSG_DIR_OP_FAILED,
+               "Unable to open dir %s", path);
+        goto out;
+    }
+
+    while ((entry = sys_readdir(dir, scratch))) {
+        if (gf_irrelevant_entry(entry))
+            continue;
+        if (snap && ((!strcmp(entry->d_name, "geo-replication")) ||
+                     (!strcmp(entry->d_name, "info"))))
+            continue;
+
+        len = snprintf(entry_path, PATH_MAX, "%s/%s", path, entry->d_name);
+        if ((len < 0) || (len >= PATH_MAX))
+            continue;
+
+        ret = sys_lstat(entry_path, &st);
+        if (ret == -1) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_INVALID_ENTRY,
+                   "Failed to stat entry %s : %s", path, strerror(errno));
+            continue;
+        }
+
+        if (!S_ISDIR(st.st_mode)) {
+            gf_msg_debug(this->name, 0, "%s is not a valid volume",
+                         entry->d_name);
+            continue;
+        }
+
+        volinfo = glusterd_store_retrieve_volume(entry->d_name, snap);
+        if (!volinfo) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_VOL_RESTORE_FAIL,
+                   "Unable to restore "
+                   "volume: %s",
+                   entry->d_name);
+            ret = -1;
+            goto out;
+        }
+
+        ret = glusterd_store_retrieve_node_state(volinfo);
+        if (ret) {
+            /* Backward compatibility */
+            gf_msg(this->name, GF_LOG_INFO, 0, GD_MSG_NEW_NODE_STATE_CREATION,
+                   "Creating a new node_state "
+                   "for volume: %s.",
+                   entry->d_name);
+            glusterd_store_create_nodestate_sh_on_absence(volinfo);
+            glusterd_store_perform_node_state_store(volinfo);
         }
+    }
 
-        ret = 0;
+    ret = 0;
 
 out:
-        if (dir)
-                closedir (dir);
-        gf_log ("", GF_LOG_DEBUG, "Returning with %d", ret);
+    if (dir)
+        sys_closedir(dir);
+    gf_msg_debug(this->name, 0, "Returning with %d", ret);
 
-        return ret;
+    return ret;
 }
 
 /* Figure out the brick mount path, from the brick path */
 int32_t
-glusterd_find_brick_mount_path (char *brick_path, int32_t brick_count,
-                                char **brick_mount_path)
+glusterd_find_brick_mount_path(char *brick_path, char **brick_mount_path)
 {
-        char                     brick_num[PATH_MAX] = "";
-        char                    *ptr                 = NULL;
-        int32_t                  ret                 = -1;
-        xlator_t                *this                = NULL;
-
-        this = THIS;
-        GF_ASSERT (this);
-        GF_ASSERT (brick_path);
-        GF_ASSERT (brick_mount_path);
-
-        *brick_mount_path = gf_strdup (brick_path);
-        if (!*brick_mount_path) {
-                ret = -1;
-                goto out;
-        }
-
-        snprintf (brick_num, sizeof(brick_num), "brick%d", brick_count);
-
-        /* Finding the pointer to the end of
-         * /var/run/gluster/snaps/<snap-uuid>
+    char *ptr = NULL;
+    int32_t ret = -1;
+    xlator_t *this = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+    GF_ASSERT(brick_path);
+    GF_ASSERT(brick_mount_path);
+
+    *brick_mount_path = gf_strdup(brick_path);
+    if (!*brick_mount_path) {
+        ret = -1;
+        goto out;
+    }
+
+    /* Finding the pointer to the end of
+     * /var/run/gluster/snaps/<snap-uuid>
+     */
+    ptr = strstr(*brick_mount_path, "brick");
+    if (!ptr) {
+        /* Snapshot bricks must have brick num as part
+         * of the brickpath
          */
-        ptr = strstr (*brick_mount_path, brick_num);
-        if (!ptr) {
-                /* Snapshot bricks must have brick num as part
-                 * of the brickpath
-                 */
-                gf_log (this->name, GF_LOG_ERROR,
-                        "Invalid brick path(%s)", brick_path);
-                ret = -1;
-                goto out;
-        }
-
-        /* Moving the pointer to the end of
-         * /var/run/gluster/snaps/<snap-uuid>/<brick_num>
-         * and assigning '\0' to it.
-         */
-        ptr += strlen(brick_num);
+        gf_msg(this->name, GF_LOG_ERROR, EINVAL, GD_MSG_INVALID_ENTRY,
+               "Invalid brick path(%s)", brick_path);
+        ret = -1;
+        goto out;
+    }
+
+    /* Moving the pointer to the end of
+     * /var/run/gluster/snaps/<snap-uuid>/<brick_num>
+     * and assigning '\0' to it.
+     */
+    while ((*ptr != '\0') && (*ptr != '/'))
+        ptr++;
+
+    if (*ptr == '/') {
         *ptr = '\0';
+    }
 
-        ret = 0;
+    ret = 0;
 out:
-        if (ret && *brick_mount_path) {
-                GF_FREE (*brick_mount_path);
-                *brick_mount_path = NULL;
-        }
-        gf_log (this->name, GF_LOG_TRACE, "Returning with %d", ret);
-        return ret;
+    if (ret && *brick_mount_path) {
+        GF_FREE(*brick_mount_path);
+        *brick_mount_path = NULL;
+    }
+    gf_msg_trace(this->name, 0, "Returning with %d", ret);
+    return ret;
 }
 
 /* Check if brick_mount_path is already mounted. If not, mount the device_path
  * at the brick_mount_path
  */
 int32_t
-glusterd_mount_brick_paths (char *brick_mount_path,
-                            glusterd_brickinfo_t *brickinfo)
+glusterd_mount_brick_paths(char *brick_mount_path,
+                           glusterd_brickinfo_t *brickinfo)
 {
-        int32_t                  ret                 = -1;
-        runner_t                 runner              = {0, };
-        char                     buff [PATH_MAX]     = {0, };
-        struct mntent            save_entry          = {0, };
-        struct mntent           *entry               = NULL;
-        xlator_t                *this                = NULL;
-        glusterd_conf_t         *priv                = NULL;
-
-        this = THIS;
-        GF_ASSERT (this);
-        GF_ASSERT (brick_mount_path);
-        GF_ASSERT (brickinfo);
-
-        priv = this->private;
-        GF_ASSERT (priv);
-
-        /* Check if the brick_mount_path is already mounted */
-        entry = glusterd_get_mnt_entry_info (brick_mount_path, buff,
-                                             sizeof (buff), &save_entry);
-        if (entry) {
-                gf_log (this->name, GF_LOG_INFO,
-                        "brick_mount_path (%s) already mounted.",
-                        brick_mount_path);
-                ret = 0;
-                goto out;
-        }
-
-        /* TODO RHEL 6.5 has the logical volumes inactive by default
-         * on reboot. Hence activating the logical vol. Check behaviour
-         * on other systems
-         */
-        /* Activate the snapshot */
-        runinit (&runner);
-        runner_add_args (&runner, "lvchange", "-ay", brickinfo->device_path,
-                         NULL);
-        ret = runner_run (&runner);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "Failed to activate %s. Error: %s",
-                        brickinfo->device_path, strerror(errno));
-                goto out;
-        } else
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "Activating %s successful", brickinfo->device_path);
-
-        /* Mount the snapshot */
-        ret = glusterd_mount_lvm_snapshot (brickinfo, brick_mount_path);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "Failed to mount lvm snapshot.");
-                goto out;
-        }
+    int32_t ret = -1;
+    runner_t runner = {
+        0,
+    };
+    char buff[PATH_MAX] = {
+        0,
+    };
+    struct mntent save_entry = {
+        0,
+    };
+    struct mntent *entry = NULL;
+    xlator_t *this = NULL;
+    glusterd_conf_t *priv = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+    GF_ASSERT(brick_mount_path);
+    GF_ASSERT(brickinfo);
+
+    priv = this->private;
+    GF_ASSERT(priv);
+
+    /* Check if the brick_mount_path is already mounted */
+    entry = glusterd_get_mnt_entry_info(brick_mount_path, buff, sizeof(buff),
+                                        &save_entry);
+    if (entry) {
+        gf_msg(this->name, GF_LOG_INFO, 0, GD_MSG_ALREADY_MOUNTED,
+               "brick_mount_path (%s) already mounted.", brick_mount_path);
+        ret = 0;
+        goto out;
+    }
+
+    /* TODO RHEL 6.5 has the logical volumes inactive by default
+     * on reboot. Hence activating the logical vol. Check behaviour
+     * on other systems
+     */
+    /* Activate the snapshot */
+    runinit(&runner);
+    runner_add_args(&runner, "lvchange", "-ay", brickinfo->device_path, NULL);
+    ret = runner_run(&runner);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, errno, GD_MSG_SNAP_ACTIVATE_FAIL,
+               "Failed to activate %s.", brickinfo->device_path);
+        goto out;
+    } else
+        gf_msg_debug(this->name, 0, "Activating %s successful",
+                     brickinfo->device_path);
+
+    /* Mount the snapshot */
+    ret = glusterd_mount_lvm_snapshot(brickinfo, brick_mount_path);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_SNAP_MOUNT_FAIL,
+               "Failed to mount lvm snapshot.");
+        goto out;
+    }
 
 out:
-        gf_log (this->name, GF_LOG_TRACE, "Returning with %d", ret);
-        return ret;
+    gf_msg_trace(this->name, 0, "Returning with %d", ret);
+    return ret;
 }
 
-static int32_t
-glusterd_recreate_vol_brick_mounts (xlator_t  *this,
-                                    glusterd_volinfo_t *volinfo)
+int32_t
+glusterd_recreate_vol_brick_mounts(xlator_t *this, glusterd_volinfo_t *volinfo)
 {
-        char                    *brick_mount_path    = NULL;
-        glusterd_brickinfo_t    *brickinfo           = NULL;
-        int32_t                  ret                 = -1;
-        int32_t                  brick_count         = -1;
-        struct stat              st_buf              = {0, };
-
-        GF_ASSERT (this);
-        GF_ASSERT (volinfo);
-
-        brick_count = 0;
-        cds_list_for_each_entry (brickinfo, &volinfo->bricks, brick_list) {
-                brick_count++;
-                /* If the brick is not of this node, or its
-                 * snapshot is pending, or the brick is not
-                 * a snapshotted brick, we continue
-                */
-                if ((gf_uuid_compare (brickinfo->uuid, MY_UUID)) ||
-                    (brickinfo->snap_status == -1) ||
-                    (strlen(brickinfo->device_path) == 0))
-                        continue;
-
-                /* Fetch the brick mount path from the brickinfo->path */
-                ret = glusterd_find_brick_mount_path (brickinfo->path,
-                                                      brick_count,
-                                                      &brick_mount_path);
-                if (ret) {
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "Failed to find brick_mount_path for %s",
-                                brickinfo->path);
-                        goto out;
-                }
+    char *brick_mount_path = NULL;
+    glusterd_brickinfo_t *brickinfo = NULL;
+    int32_t ret = -1;
+    struct stat st_buf = {
+        0,
+    };
+    char abspath[PATH_MAX] = {0};
+
+    GF_ASSERT(this);
+    GF_ASSERT(volinfo);
+
+    cds_list_for_each_entry(brickinfo, &volinfo->bricks, brick_list)
+    {
+        /* If the brick is not of this node, or its
+         * snapshot is pending, or the brick is not
+         * a snapshotted brick, we continue
+         */
+        if ((gf_uuid_compare(brickinfo->uuid, MY_UUID)) ||
+            (brickinfo->snap_status == -1) ||
+            (strlen(brickinfo->device_path) == 0))
+            continue;
+
+        /* Fetch the brick mount path from the brickinfo->path */
+        ret = glusterd_find_brick_mount_path(brickinfo->path,
+                                             &brick_mount_path);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_BRK_MNTPATH_GET_FAIL,
+                   "Failed to find brick_mount_path for %s", brickinfo->path);
+            goto out;
+        }
 
-                /* Check if the brickinfo path is present.
-                 * If not create the brick_mount_path */
-                ret = lstat (brickinfo->path, &st_buf);
+        /* Check if the brickinfo path is present.
+         * If not create the brick_mount_path */
+        ret = sys_lstat(brickinfo->path, &st_buf);
+        if (ret) {
+            if (errno == ENOENT) {
+                ret = mkdir_p(brick_mount_path, 0755, _gf_true);
                 if (ret) {
-                        if (errno == ENOENT) {
-                                ret = mkdir_p (brick_mount_path, 0777,
-                                               _gf_true);
-                                if (ret) {
-                                        gf_log (this->name, GF_LOG_ERROR,
-                                                "Failed to create %s. "
-                                                "Error: %s", brick_mount_path,
-                                                strerror (errno));
-                                        goto out;
-                                }
-                        } else {
-                                gf_log (this->name, GF_LOG_ERROR,
-                                        "Brick Path(%s) not valid. "
-                                        "Error: %s", brickinfo->path,
-                                        strerror(errno));
-                                goto out;
-                        }
+                    gf_msg(this->name, GF_LOG_ERROR, errno,
+                           GD_MSG_CREATE_DIR_FAILED, "Failed to create %s. ",
+                           brick_mount_path);
+                    goto out;
                 }
+            } else {
+                gf_msg(this->name, GF_LOG_ERROR, errno, GD_MSG_FILE_OP_FAILED,
+                       "Brick Path(%s) not valid. ", brickinfo->path);
+                goto out;
+            }
+        }
 
-                /* Check if brick_mount_path is already mounted.
-                 * If not, mount the device_path at the brick_mount_path */
-                ret = glusterd_mount_brick_paths (brick_mount_path, brickinfo);
-                if (ret) {
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "Failed to mount brick_mount_path");
+        /* Check if brick_mount_path is already mounted.
+         * If not, mount the device_path at the brick_mount_path */
+        ret = glusterd_mount_brick_paths(brick_mount_path, brickinfo);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_BRK_MNTPATH_MOUNT_FAIL,
+                   "Failed to mount brick_mount_path");
+        }
+        if (!gf_uuid_compare(brickinfo->uuid, MY_UUID)) {
+            if (brickinfo->real_path[0] == '\0') {
+                if (!realpath(brickinfo->path, abspath)) {
+                    gf_msg(this->name, GF_LOG_CRITICAL, errno,
+                           GD_MSG_BRICKINFO_CREATE_FAIL,
+                           "realpath() failed for brick %s"
+                           ". The underlying file system "
+                           "may be in bad state",
+                           brickinfo->path);
+                    ret = -1;
+                    goto out;
                 }
-
-                if (brick_mount_path) {
-                        GF_FREE (brick_mount_path);
-                        brick_mount_path = NULL;
+                if (strlen(abspath) >= sizeof(brickinfo->real_path)) {
+                    ret = -1;
+                    goto out;
                 }
+                (void)strncpy(brickinfo->real_path, abspath,
+                              sizeof(brickinfo->real_path));
+            }
         }
 
-        ret = 0;
-out:
-        if (ret && brick_mount_path)
-                GF_FREE (brick_mount_path);
-
-        gf_log (this->name, GF_LOG_TRACE, "Returning with %d", ret);
-        return ret;
-}
-
-int32_t
-glusterd_resolve_snap_bricks (xlator_t  *this, glusterd_snap_t *snap)
-{
-        int32_t                 ret = -1;
-        glusterd_volinfo_t      *volinfo = NULL;
-        glusterd_brickinfo_t    *brickinfo = NULL;
-
-        GF_ASSERT (this);
-        GF_VALIDATE_OR_GOTO (this->name, snap, out);
-
-        cds_list_for_each_entry (volinfo, &snap->volumes, vol_list) {
-                cds_list_for_each_entry (brickinfo, &volinfo->bricks,
-                                         brick_list) {
-                        ret = glusterd_resolve_brick (brickinfo);
-                        if (ret) {
-                                gf_log (this->name, GF_LOG_ERROR,
-                                        "resolve brick failed in restore");
-                                goto out;
-                        }
-                }
+        if (brick_mount_path) {
+            GF_FREE(brick_mount_path);
+            brick_mount_path = NULL;
         }
+    }
 
-        ret = 0;
-
+    ret = 0;
 out:
-        gf_log (this->name, GF_LOG_TRACE, "Returning with %d", ret);
+    if (ret && brick_mount_path)
+        GF_FREE(brick_mount_path);
 
-        return ret;
+    gf_msg_trace(this->name, 0, "Returning with %d", ret);
+    return ret;
 }
 
-int
-glusterd_store_update_snap (glusterd_snap_t *snap)
+int32_t
+glusterd_resolve_snap_bricks(xlator_t *this, glusterd_snap_t *snap)
 {
-        int                     ret = -1;
-        char                    *key                    = NULL;
-        char                    *value                  = NULL;
-        char                    snappath[PATH_MAX]      = {0,};
-        char                    path[PATH_MAX]          = {0,};
-        xlator_t                *this                   = NULL;
-        glusterd_conf_t         *conf                   = NULL;
-        gf_store_iter_t         *iter                   = NULL;
-        gf_store_op_errno_t     op_errno                = GD_STORE_SUCCESS;
-
-        this = THIS;
-        conf = this->private;
-        GF_ASSERT (snap);
-
-        GLUSTERD_GET_SNAP_DIR (snappath, snap, conf);
-
-        snprintf (path, sizeof (path), "%s/%s", snappath,
-                  GLUSTERD_SNAP_INFO_FILE);
-
-        ret = gf_store_handle_retrieve (path, &snap->shandle);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "snap handle is NULL");
-                goto out;
-        }
+    int32_t ret = -1;
+    glusterd_volinfo_t *volinfo = NULL;
+    glusterd_brickinfo_t *brickinfo = NULL;
 
-        ret = gf_store_iter_new (snap->shandle, &iter);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "Failed to get new store "
-                        "iter");
-                goto out;
-        }
+    GF_ASSERT(this);
+    GF_VALIDATE_OR_GOTO(this->name, snap, out);
 
-        ret = gf_store_iter_get_next (iter, &key, &value, &op_errno);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "Failed to get next store "
-                        "iter");
+    cds_list_for_each_entry(volinfo, &snap->volumes, vol_list)
+    {
+        cds_list_for_each_entry(brickinfo, &volinfo->bricks, brick_list)
+        {
+            ret = glusterd_resolve_brick(brickinfo);
+            if (ret) {
+                gf_event(EVENT_BRICKPATH_RESOLVE_FAILED,
+                         "peer=%s;volume=%s;brick=%s", brickinfo->hostname,
+                         volinfo->volname, brickinfo->path);
+                gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_RESOLVE_BRICK_FAIL,
+                       "resolve brick failed in restore");
                 goto out;
+            }
         }
+    }
 
-        while (!ret) {
-                gf_log (this->name, GF_LOG_DEBUG, "key = %s value = %s",
-                        key, value);
-
-                if (!strncmp (key, GLUSTERD_STORE_KEY_SNAP_ID,
-                                     strlen (GLUSTERD_STORE_KEY_SNAP_ID))) {
-                        ret = gf_uuid_parse (value, snap->snap_id);
-                        if (ret)
-                                gf_log (this->name, GF_LOG_WARNING,
-                                        "Failed to parse uuid");
-                } else if (!strncmp (key, GLUSTERD_STORE_KEY_SNAP_RESTORED,
-                                   strlen (GLUSTERD_STORE_KEY_SNAP_RESTORED))) {
-                        snap->snap_restored = atoi (value);
-                } else if (!strncmp (key, GLUSTERD_STORE_KEY_SNAP_STATUS,
-                                     strlen (GLUSTERD_STORE_KEY_SNAP_STATUS))) {
-                        snap->snap_status = atoi (value);
-                } else if (!strncmp (key, GLUSTERD_STORE_KEY_SNAP_DESC,
-                                     strlen (GLUSTERD_STORE_KEY_SNAP_DESC))) {
-                        snap->description = gf_strdup (value);
-                } else if (!strncmp (key, GLUSTERD_STORE_KEY_SNAP_TIMESTAMP,
-                                  strlen (GLUSTERD_STORE_KEY_SNAP_TIMESTAMP))) {
-                        snap->time_stamp = atoi (value);
-                }
+    ret = 0;
 
-                GF_FREE (key);
-                GF_FREE (value);
-                key = NULL;
-                value = NULL;
-
-                ret = gf_store_iter_get_next (iter, &key, &value, &op_errno);
-        }
+out:
+    gf_msg_trace(this->name, 0, "Returning with %d", ret);
 
-        if (op_errno != GD_STORE_EOF)
-                goto out;
+    return ret;
+}
 
-        ret = gf_store_iter_destroy (iter);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "Failed to destroy store "
-                        "iter");
-        }
+int
+glusterd_store_update_snap(glusterd_snap_t *snap)
+{
+    int ret = -1;
+    char *key = NULL;
+    char *value = NULL;
+    char snappath[PATH_MAX] = {
+        0,
+    };
+    char path[PATH_MAX] = {
+        0,
+    };
+    xlator_t *this = NULL;
+    glusterd_conf_t *conf = NULL;
+    gf_store_iter_t *iter = NULL;
+    gf_store_op_errno_t op_errno = GD_STORE_SUCCESS;
+    int32_t len = 0;
+
+    this = THIS;
+    conf = this->private;
+    GF_ASSERT(snap);
+
+    GLUSTERD_GET_SNAP_DIR(snappath, snap, conf);
+
+    len = snprintf(path, sizeof(path), "%s/%s", snappath,
+                   GLUSTERD_SNAP_INFO_FILE);
+    if ((len < 0) || (len >= sizeof(path))) {
+        goto out;
+    }
+
+    ret = gf_store_handle_retrieve(path, &snap->shandle);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_HANDLE_NULL,
+               "snap handle is NULL");
+        goto out;
+    }
+
+    ret = gf_store_iter_new(snap->shandle, &iter);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_STORE_ITER_GET_FAIL,
+               "Failed to get new store "
+               "iter");
+        goto out;
+    }
+
+    ret = gf_store_iter_get_next(iter, &key, &value, &op_errno);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_STORE_ITER_GET_FAIL,
+               "Failed to get next store "
+               "iter");
+        goto out;
+    }
+
+    while (!ret) {
+        gf_msg_debug(this->name, 0, "key = %s value = %s", key, value);
+
+        if (!strncmp(key, GLUSTERD_STORE_KEY_SNAP_ID,
+                     SLEN(GLUSTERD_STORE_KEY_SNAP_ID))) {
+            ret = gf_uuid_parse(value, snap->snap_id);
+            if (ret)
+                gf_msg(this->name, GF_LOG_WARNING, 0, GD_MSG_UUID_PARSE_FAIL,
+                       "Failed to parse uuid");
+        } else if (!strncmp(key, GLUSTERD_STORE_KEY_SNAP_RESTORED,
+                            SLEN(GLUSTERD_STORE_KEY_SNAP_RESTORED))) {
+            snap->snap_restored = atoi(value);
+        } else if (!strncmp(key, GLUSTERD_STORE_KEY_SNAP_STATUS,
+                            SLEN(GLUSTERD_STORE_KEY_SNAP_STATUS))) {
+            snap->snap_status = atoi(value);
+        } else if (!strncmp(key, GLUSTERD_STORE_KEY_SNAP_DESC,
+                            SLEN(GLUSTERD_STORE_KEY_SNAP_DESC))) {
+            snap->description = gf_strdup(value);
+        } else if (!strncmp(key, GLUSTERD_STORE_KEY_SNAP_TIMESTAMP,
+                            SLEN(GLUSTERD_STORE_KEY_SNAP_TIMESTAMP))) {
+            snap->time_stamp = atoi(value);
+        }
+
+        GF_FREE(key);
+        GF_FREE(value);
+        key = NULL;
+        value = NULL;
+
+        ret = gf_store_iter_get_next(iter, &key, &value, &op_errno);
+    }
+
+    if (op_errno != GD_STORE_EOF)
+        goto out;
+
+    ret = 0;
 
 out:
-        return ret;
+    if (gf_store_iter_destroy(&iter)) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_STORE_ITER_DESTROY_FAIL,
+               "Failed to destroy store iter");
+        ret = -1;
+    }
+
+    return ret;
 }
 
 int32_t
-glusterd_store_retrieve_snap (char *snapname)
+glusterd_store_retrieve_snap(char *snapname)
 {
-        int32_t                ret      = -1;
-        glusterd_snap_t       *snap     = NULL;
-        glusterd_conf_t       *priv     = NULL;
-        xlator_t              *this     = NULL;
-
-        this = THIS;
-        priv = this->private;
-        GF_ASSERT (priv);
-        GF_ASSERT (snapname);
-
-        snap = glusterd_new_snap_object ();
-        if (!snap) {
-                gf_log (this->name, GF_LOG_ERROR, "Failed to create "
-                                                  " snap object");
-                goto out;
-        }
-
-        strncpy (snap->snapname, snapname, strlen(snapname));
-        ret = glusterd_store_update_snap (snap);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "Failed to update snapshot "
-                        "for %s snap", snapname);
-                goto out;
-        }
-
-        ret = glusterd_store_retrieve_volumes (this, snap);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "Failed to retrieve "
-                        "snap volumes for snap %s", snapname);
-                goto out;
-        }
-
-        /* TODO: list_add_order can do 'N-square' comparisons and
-           is not efficient. Find a better solution to store the snap
-           in order */
-        glusterd_list_add_order (&snap->snap_list, &priv->snapshots,
-                                 glusterd_compare_snap_time);
+    int32_t ret = -1;
+    glusterd_snap_t *snap = NULL;
+    glusterd_conf_t *priv = NULL;
+    xlator_t *this = NULL;
+
+    this = THIS;
+    priv = this->private;
+    GF_ASSERT(priv);
+    GF_ASSERT(snapname);
+
+    snap = glusterd_new_snap_object();
+    if (!snap) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_SNAP_OBJECT_STORE_FAIL,
+               "Failed to create "
+               " snap object");
+        goto out;
+    }
+
+    if (snprintf(snap->snapname, sizeof(snap->snapname), "%s", snapname) >=
+        sizeof(snap->snapname))
+        goto out;
+    ret = glusterd_store_update_snap(snap);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_SNAPSHOT_UPDATE_FAIL,
+               "Failed to update snapshot "
+               "for %s snap",
+               snapname);
+        goto out;
+    }
+
+    ret = glusterd_store_retrieve_volumes(this, snap);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_SNAP_VOL_RETRIEVE_FAIL,
+               "Failed to retrieve "
+               "snap volumes for snap %s",
+               snapname);
+        goto out;
+    }
+
+    /* TODO: list_add_order can do 'N-square' comparisons and
+       is not efficient. Find a better solution to store the snap
+       in order */
+    glusterd_list_add_order(&snap->snap_list, &priv->snapshots,
+                            glusterd_compare_snap_time);
 
 out:
-        gf_log (this->name, GF_LOG_TRACE, "Returning with %d", ret);
-        return ret;
+    gf_msg_trace(this->name, 0, "Returning with %d", ret);
+    return ret;
 }
 
 /* Read the missed_snap_list and update the in-memory structs */
 int32_t
-glusterd_store_retrieve_missed_snaps_list (xlator_t  *this)
+glusterd_store_retrieve_missed_snaps_list(xlator_t *this)
 {
-        char                   buf[PATH_MAX]    = "";
-        char                   path[PATH_MAX]   = "";
-        char                  *snap_vol_id      = NULL;
-        char                  *missed_node_info = NULL;
-        char                  *brick_path       = NULL;
-        char                  *value            = NULL;
-        char                  *save_ptr         = NULL;
-        FILE                  *fp               = NULL;
-        int32_t                brick_num        = -1;
-        int32_t                snap_op          = -1;
-        int32_t                snap_status      = -1;
-        int32_t                ret              = -1;
-        glusterd_conf_t       *priv             = NULL;
-        gf_store_op_errno_t    store_errno      = GD_STORE_SUCCESS;
-
-        GF_ASSERT (this);
-        priv = this->private;
-        GF_ASSERT (priv);
-
-        /* Get the path of the missed_snap_list */
-        glusterd_store_missed_snaps_list_path_set (path, sizeof(path));
-
-        fp = fopen (path, "r");
-        if (!fp) {
-                /* If errno is ENOENT then there are no missed snaps yet */
-                if (errno != ENOENT) {
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "Failed to open %s. "
-                                "Error: %s", path, strerror(errno));
-                } else {
-                        gf_log (this->name, GF_LOG_INFO,
-                                "No missed snaps list.");
-                        ret = 0;
-                }
-                goto out;
+    char path[PATH_MAX] = "";
+    char *snap_vol_id = NULL;
+    char *missed_node_info = NULL;
+    char *brick_path = NULL;
+    char *value = NULL;
+    char *save_ptr = NULL;
+    FILE *fp = NULL;
+    int32_t brick_num = -1;
+    int32_t snap_op = -1;
+    int32_t snap_status = -1;
+    int32_t ret = -1;
+    glusterd_conf_t *priv = NULL;
+    gf_store_op_errno_t store_errno = GD_STORE_SUCCESS;
+
+    GF_ASSERT(this);
+    priv = this->private;
+    GF_ASSERT(priv);
+
+    /* Get the path of the missed_snap_list */
+    glusterd_store_missed_snaps_list_path_set(path, sizeof(path));
+
+    fp = fopen(path, "r");
+    if (!fp) {
+        /* If errno is ENOENT then there are no missed snaps yet */
+        if (errno != ENOENT) {
+            gf_msg(this->name, GF_LOG_ERROR, errno, GD_MSG_FILE_OP_FAILED,
+                   "Failed to open %s. ", path);
+        } else {
+            gf_msg(this->name, GF_LOG_INFO, 0, GD_MSG_MISSED_SNAP_LIST_EMPTY,
+                   "No missed snaps list.");
+            ret = 0;
         }
+        goto out;
+    }
 
-        do {
-                ret = gf_store_read_and_tokenize (fp, buf,
-                                                  &missed_node_info, &value,
-                                                  &store_errno);
-                if (ret) {
-                        if (store_errno == GD_STORE_EOF) {
-                                gf_log (this->name,
-                                        GF_LOG_DEBUG,
-                                        "EOF for missed_snap_list");
-                                ret = 0;
-                                break;
-                        }
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "Failed to fetch data from "
-                                "missed_snaps_list. Error: %s",
-                                gf_store_strerror (store_errno));
-                        goto out;
-                }
-
-                /* Fetch the brick_num, brick_path, snap_op and snap status */
-                snap_vol_id = strtok_r (value, ":", &save_ptr);
-                brick_num = atoi(strtok_r (NULL, ":", &save_ptr));
-                brick_path = strtok_r (NULL, ":", &save_ptr);
-                snap_op = atoi(strtok_r (NULL, ":", &save_ptr));
-                snap_status = atoi(strtok_r (NULL, ":", &save_ptr));
-
-                if (!missed_node_info || !brick_path || !snap_vol_id ||
-                    brick_num < 1 || snap_op < 1 ||
-                    snap_status < 1) {
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "Invalid missed_snap_entry");
-                        ret = -1;
-                        goto out;
-                }
-
-                ret = glusterd_add_new_entry_to_list (missed_node_info,
-                                                      snap_vol_id,
-                                                      brick_num,
-                                                      brick_path,
-                                                      snap_op,
-                                                      snap_status);
-                if (ret) {
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "Failed to store missed snaps_list");
-                        goto out;
-                }
+    do {
+        ret = gf_store_read_and_tokenize(fp, &missed_node_info, &value,
+                                         &store_errno);
+        if (ret) {
+            if (store_errno == GD_STORE_EOF) {
+                gf_msg_debug(this->name, 0, "EOF for missed_snap_list");
+                ret = 0;
+                break;
+            }
+            gf_msg(this->name, GF_LOG_ERROR, store_errno,
+                   GD_MSG_MISSED_SNAP_GET_FAIL,
+                   "Failed to fetch data from "
+                   "missed_snaps_list.");
+            goto out;
+        }
+
+        /* Fetch the brick_num, brick_path, snap_op and snap status */
+        snap_vol_id = strtok_r(value, ":", &save_ptr);
+        brick_num = atoi(strtok_r(NULL, ":", &save_ptr));
+        brick_path = strtok_r(NULL, ":", &save_ptr);
+        snap_op = atoi(strtok_r(NULL, ":", &save_ptr));
+        snap_status = atoi(strtok_r(NULL, ":", &save_ptr));
+
+        if (!missed_node_info || !brick_path || !snap_vol_id || brick_num < 1 ||
+            snap_op < 1 || snap_status < 1) {
+            gf_msg(this->name, GF_LOG_ERROR, EINVAL,
+                   GD_MSG_INVALID_MISSED_SNAP_ENTRY,
+                   "Invalid missed_snap_entry");
+            ret = -1;
+            goto out;
+        }
+
+        ret = glusterd_add_new_entry_to_list(missed_node_info, snap_vol_id,
+                                             brick_num, brick_path, snap_op,
+                                             snap_status);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, 0,
+                   GD_MSG_MISSED_SNAP_LIST_STORE_FAIL,
+                   "Failed to store missed snaps_list");
+            goto out;
+        }
 
-        } while (store_errno == GD_STORE_SUCCESS);
+    } while (store_errno == GD_STORE_SUCCESS);
 
-        ret = 0;
+    ret = 0;
 out:
-        gf_log (this->name, GF_LOG_TRACE, "Returning with %d", ret);
-        return ret;
+    if (fp)
+        fclose(fp);
+
+    gf_msg_trace(this->name, 0, "Returning with %d", ret);
+    return ret;
 }
 
 int32_t
-glusterd_store_retrieve_snaps (xlator_t  *this)
+glusterd_store_retrieve_snaps(xlator_t *this)
 {
-        int32_t                ret              = 0;
-        char                   path[PATH_MAX]   = {0,};
-        glusterd_conf_t       *priv             = NULL;
-        DIR                   *dir              = NULL;
-        struct dirent         *entry            = NULL;
-
-        GF_ASSERT (this);
-        priv = this->private;
-
-        GF_ASSERT (priv);
-
-        snprintf (path, PATH_MAX, "%s/snaps", priv->workdir);
-
-        dir = opendir (path);
-
-        if (!dir) {
-                /* If snaps dir doesn't exists ignore the error for
-                   backward compatibility */
-                if (errno != ENOENT) {
-                        ret = -1;
-                        gf_log ("", GF_LOG_ERROR, "Unable to open dir %s", path);
-                }
-                goto out;
-        }
-
-        GF_FOR_EACH_ENTRY_IN_DIR (entry, dir);
-
-        while (entry) {
-                if (strcmp (entry->d_name, GLUSTERD_MISSED_SNAPS_LIST_FILE)) {
-                        ret = glusterd_store_retrieve_snap (entry->d_name);
-                        if (ret) {
-                                gf_log (this->name, GF_LOG_ERROR,
-                                        "Unable to restore snapshot: %s",
-                                        entry->d_name);
-                                goto out;
-                        }
-                }
-                GF_FOR_EACH_ENTRY_IN_DIR (entry, dir);
-        }
-
-        /* Retrieve missed_snaps_list */
-        ret = glusterd_store_retrieve_missed_snaps_list (this);
-        if (ret) {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "Failed to retrieve missed_snaps_list");
-                goto out;
-        }
+    int32_t ret = 0;
+    char path[PATH_MAX] = {
+        0,
+    };
+    glusterd_conf_t *priv = NULL;
+    DIR *dir = NULL;
+    struct dirent *entry = NULL;
+    struct dirent scratch[2] = {
+        {
+            0,
+        },
+    };
+    int32_t len = 0;
+
+    GF_ASSERT(this);
+    priv = this->private;
+
+    GF_ASSERT(priv);
+
+    len = snprintf(path, PATH_MAX, "%s/snaps", priv->workdir);
+    if ((len < 0) || (len >= PATH_MAX)) {
+        ret = -1;
+        goto out;
+    }
+
+    dir = sys_opendir(path);
+
+    if (!dir) {
+        /* If snaps dir doesn't exists ignore the error for
+           backward compatibility */
+        if (errno != ENOENT) {
+            ret = -1;
+            gf_msg(this->name, GF_LOG_ERROR, errno, GD_MSG_DIR_OP_FAILED,
+                   "Unable to open dir %s", path);
+        }
+        goto out;
+    }
+
+    while ((entry = sys_readdir(dir, scratch))) {
+        if (gf_irrelevant_entry(entry))
+            continue;
+        if (strcmp(entry->d_name, GLUSTERD_MISSED_SNAPS_LIST_FILE)) {
+            ret = glusterd_store_retrieve_snap(entry->d_name);
+            if (ret) {
+                gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_SNAP_RESTORE_FAIL,
+                       "Unable to restore snapshot: %s", entry->d_name);
+                goto out;
+            }
+        }
+    }
+
+    /* Retrieve missed_snaps_list */
+    ret = glusterd_store_retrieve_missed_snaps_list(this);
+    if (ret) {
+        gf_msg_debug(this->name, 0, "Failed to retrieve missed_snaps_list");
+        goto out;
+    }
 
 out:
-        if (dir)
-                closedir (dir);
-        gf_log ("", GF_LOG_DEBUG, "Returning with %d", ret);
+    if (dir)
+        sys_closedir(dir);
+    gf_msg_debug(this->name, 0, "Returning with %d", ret);
 
-        return ret;
+    return ret;
 }
 
 /* Writes all the contents of conf->missed_snap_list */
 int32_t
-glusterd_store_write_missed_snapinfo (int32_t fd)
+glusterd_store_write_missed_snapinfo(int32_t fd)
 {
-        char                           key[PATH_MAX]               = "";
-        char                           value[PATH_MAX]             = "";
-        int32_t                        ret                         = -1;
-        glusterd_conf_t               *priv                        = NULL;
-        glusterd_missed_snap_info     *missed_snapinfo             = NULL;
-        glusterd_snap_op_t            *snap_opinfo                 = NULL;
-        xlator_t                      *this                        = NULL;
-
-        this = THIS;
-        GF_ASSERT(this);
-
-        priv = this->private;
-        GF_ASSERT (priv);
-
-        /* Write the missed_snap_entry */
-        cds_list_for_each_entry (missed_snapinfo, &priv->missed_snaps_list,
-                                 missed_snaps) {
-                cds_list_for_each_entry (snap_opinfo,
-                                         &missed_snapinfo->snap_ops,
-                                         snap_ops_list) {
-                        snprintf (key, sizeof(key), "%s:%s",
-                                  missed_snapinfo->node_uuid,
-                                  missed_snapinfo->snap_uuid);
-                        snprintf (value, sizeof(value), "%s:%d:%s:%d:%d",
-                                  snap_opinfo->snap_vol_id,
-                                  snap_opinfo->brick_num,
-                                  snap_opinfo->brick_path,
-                                  snap_opinfo->op, snap_opinfo->status);
-                        ret = gf_store_save_value (fd, key, value);
-                        if (ret) {
-                                gf_log (this->name, GF_LOG_ERROR,
-                                        "Failed to write missed snapinfo");
-                                goto out;
-                        }
-                }
-        }
-
-        ret = 0;
+    char key[(UUID_SIZE * 2) + 2];
+    char value[PATH_MAX];
+    int32_t ret = -1;
+    glusterd_conf_t *priv = NULL;
+    glusterd_missed_snap_info *missed_snapinfo = NULL;
+    glusterd_snap_op_t *snap_opinfo = NULL;
+    xlator_t *this = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+
+    priv = this->private;
+    GF_ASSERT(priv);
+
+    /* Write the missed_snap_entry */
+    cds_list_for_each_entry(missed_snapinfo, &priv->missed_snaps_list,
+                            missed_snaps)
+    {
+        cds_list_for_each_entry(snap_opinfo, &missed_snapinfo->snap_ops,
+                                snap_ops_list)
+        {
+            snprintf(key, sizeof(key), "%s:%s", missed_snapinfo->node_uuid,
+                     missed_snapinfo->snap_uuid);
+            snprintf(value, sizeof(value), "%s:%d:%s:%d:%d",
+                     snap_opinfo->snap_vol_id, snap_opinfo->brick_num,
+                     snap_opinfo->brick_path, snap_opinfo->op,
+                     snap_opinfo->status);
+            ret = gf_store_save_value(fd, key, value);
+            if (ret) {
+                gf_msg(this->name, GF_LOG_ERROR, 0,
+                       GD_MSG_MISSEDSNAP_INFO_SET_FAIL,
+                       "Failed to write missed snapinfo");
+                goto out;
+            }
+        }
+    }
+
+    ret = 0;
 out:
-        gf_log (this->name, GF_LOG_TRACE, "Returning %d", ret);
-        return ret;
+    gf_msg_trace(this->name, 0, "Returning %d", ret);
+    return ret;
 }
 
 /* Adds the missed snap entries to the in-memory conf->missed_snap_list *
  * and writes them to disk */
 int32_t
-glusterd_store_update_missed_snaps ()
+glusterd_store_update_missed_snaps()
 {
-        int32_t                        fd                          = -1;
-        int32_t                        ret                         = -1;
-        glusterd_conf_t               *priv                        = NULL;
-        xlator_t                      *this                        = NULL;
-
-        this = THIS;
-        GF_ASSERT(this);
-
-        priv = this->private;
-        GF_ASSERT (priv);
-
-        ret = glusterd_store_create_missed_snaps_list_shandle_on_absence ();
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "Unable to obtain "
-                        "missed_snaps_list store handle.");
-                goto out;
-        }
-
-        fd = gf_store_mkstemp (priv->missed_snaps_list_shandle);
-        if (fd <= 0) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "Failed to create tmp file");
-                ret = -1;
-                goto out;
-        }
-
-        ret = glusterd_store_write_missed_snapinfo (fd);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "Failed to write missed snaps to disk");
-                goto out;
-        }
-
-        ret = gf_store_rename_tmppath (priv->missed_snaps_list_shandle);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "Failed to rename the tmp file");
-                goto out;
-        }
+    int32_t fd = -1;
+    int32_t ret = -1;
+    glusterd_conf_t *priv = NULL;
+    xlator_t *this = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+
+    priv = this->private;
+    GF_ASSERT(priv);
+
+    ret = glusterd_store_create_missed_snaps_list_shandle_on_absence();
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0,
+               GD_MSG_MISSED_SNAP_LIST_STORE_HANDLE_GET_FAIL,
+               "Unable to obtain "
+               "missed_snaps_list store handle.");
+        goto out;
+    }
+
+    fd = gf_store_mkstemp(priv->missed_snaps_list_shandle);
+    if (fd <= 0) {
+        gf_msg(this->name, GF_LOG_ERROR, errno, GD_MSG_FILE_OP_FAILED,
+               "Failed to create tmp file");
+        ret = -1;
+        goto out;
+    }
+
+    ret = glusterd_store_write_missed_snapinfo(fd);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_MISSED_SNAP_CREATE_FAIL,
+               "Failed to write missed snaps to disk");
+        goto out;
+    }
+
+    ret = gf_store_rename_tmppath(priv->missed_snaps_list_shandle);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, errno, GD_MSG_FILE_OP_FAILED,
+               "Failed to rename the tmp file");
+        goto out;
+    }
 out:
-        if (ret && (fd > 0)) {
-                ret = gf_store_unlink_tmppath (priv->missed_snaps_list_shandle);
-                if (ret) {
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "Failed to unlink the tmp file");
-                }
-                ret = -1;
+    if (ret && (fd > 0)) {
+        ret = gf_store_unlink_tmppath(priv->missed_snaps_list_shandle);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_TMP_FILE_UNLINK_FAIL,
+                   "Failed to unlink the tmp file");
         }
+        ret = -1;
+    }
 
-        gf_log (this->name, GF_LOG_TRACE, "Returning %d", ret);
-        return ret;
+    gf_msg_trace(this->name, 0, "Returning %d", ret);
+    return ret;
 }
 
 int32_t
-glusterd_store_delete_peerinfo (glusterd_peerinfo_t *peerinfo)
+glusterd_store_delete_peerinfo(glusterd_peerinfo_t *peerinfo)
 {
-        int32_t                         ret = -1;
-        glusterd_conf_t                 *priv = NULL;
-        char                            peerdir[PATH_MAX] = {0,};
-        char                            filepath[PATH_MAX] = {0,};
-        char                            hostname_path[PATH_MAX] = {0,};
-
-
-        if (!peerinfo) {
-                ret = 0;
-                goto out;
-        }
-
-        priv = THIS->private;
-
-        snprintf (peerdir, PATH_MAX, "%s/peers", priv->workdir);
+    int32_t ret = -1;
+    glusterd_conf_t *priv = NULL;
+    xlator_t *this = NULL;
+    char peerdir[PATH_MAX] = {
+        0,
+    };
+    char filepath[PATH_MAX] = {
+        0,
+    };
+    char hostname_path[PATH_MAX] = {
+        0,
+    };
+    int32_t len = 0;
+
+    if (!peerinfo) {
+        ret = 0;
+        goto out;
+    }
 
+    this = THIS;
+    priv = this->private;
 
-        if (gf_uuid_is_null (peerinfo->uuid)) {
+    len = snprintf(peerdir, PATH_MAX, "%s/peers", priv->workdir);
+    if ((len < 0) || (len >= PATH_MAX)) {
+        goto out;
+    }
 
-                if (peerinfo->hostname) {
-                        snprintf (filepath, PATH_MAX, "%s/%s", peerdir,
-                                  peerinfo->hostname);
-                } else {
-                       ret = 0;
-                       goto out;
-                }
+    if (gf_uuid_is_null(peerinfo->uuid)) {
+        if (peerinfo->hostname) {
+            len = snprintf(filepath, PATH_MAX, "%s/%s", peerdir,
+                           peerinfo->hostname);
+            if ((len < 0) || (len >= PATH_MAX)) {
+                goto out;
+            }
         } else {
+            ret = 0;
+            goto out;
+        }
+    } else {
+        len = snprintf(filepath, PATH_MAX, "%s/%s", peerdir,
+                       uuid_utoa(peerinfo->uuid));
+        if ((len < 0) || (len >= PATH_MAX)) {
+            goto out;
+        }
+        len = snprintf(hostname_path, PATH_MAX, "%s/%s", peerdir,
+                       peerinfo->hostname);
+        if ((len < 0) || (len >= PATH_MAX)) {
+            goto out;
+        }
 
-                snprintf (filepath, PATH_MAX, "%s/%s", peerdir,
-                          uuid_utoa (peerinfo->uuid));
-                snprintf (hostname_path, PATH_MAX, "%s/%s",
-                          peerdir, peerinfo->hostname);
-
-                ret = unlink (hostname_path);
+        ret = sys_unlink(hostname_path);
 
-                if (!ret)
-                        goto out;
-        }
+        if (!ret)
+            goto out;
+    }
 
-        ret = unlink (filepath);
-        if (ret && (errno == ENOENT))
-                ret = 0;
+    ret = sys_unlink(filepath);
+    if (ret && (errno == ENOENT))
+        ret = 0;
 
 out:
-        if (peerinfo->shandle) {
-                gf_store_handle_destroy (peerinfo->shandle);
-                peerinfo->shandle = NULL;
-        }
-        gf_log ("", GF_LOG_DEBUG, "Returning with %d", ret);
+    if (peerinfo && peerinfo->shandle) {
+        gf_store_handle_destroy(peerinfo->shandle);
+        peerinfo->shandle = NULL;
+    }
+    gf_msg_debug((this ? this->name : "glusterd"), 0, "Returning with %d", ret);
 
-        return ret;
+    return ret;
 }
 
 void
-glusterd_store_peerinfo_dirpath_set (char *path, size_t len)
+glusterd_store_peerinfo_dirpath_set(char *path, size_t len)
 {
-        glusterd_conf_t                 *priv = NULL;
-        GF_ASSERT (path);
-        GF_ASSERT (len >= PATH_MAX);
+    glusterd_conf_t *priv = NULL;
+    GF_ASSERT(path);
+    GF_ASSERT(len >= PATH_MAX);
 
-        priv = THIS->private;
-        snprintf (path, len, "%s/peers", priv->workdir);
+    priv = THIS->private;
+    snprintf(path, len, "%s/peers", priv->workdir);
 }
 
 int32_t
-glusterd_store_create_peer_dir ()
+glusterd_store_create_peer_dir()
 {
-        int32_t                             ret = 0;
-        char                            path[PATH_MAX];
+    int32_t ret = 0;
+    char path[PATH_MAX];
 
-        glusterd_store_peerinfo_dirpath_set (path, sizeof (path));
-        ret = gf_store_mkdir (path);
+    glusterd_store_peerinfo_dirpath_set(path, sizeof(path));
+    ret = gf_store_mkdir(path);
 
-        gf_log ("", GF_LOG_DEBUG, "Returning with %d", ret);
-        return ret;
+    gf_msg_debug("glusterd", 0, "Returning with %d", ret);
+    return ret;
 }
 
 static void
-glusterd_store_uuid_peerpath_set (glusterd_peerinfo_t *peerinfo, char *peerfpath,
-                             size_t len)
+glusterd_store_uuid_peerpath_set(glusterd_peerinfo_t *peerinfo, char *peerfpath,
+                                 size_t len)
 {
-        char                    peerdir[PATH_MAX];
-        char                    str[50] = {0};
+    char peerdir[PATH_MAX];
+    char str[50] = {0};
 
-        GF_ASSERT (peerinfo);
-        GF_ASSERT (peerfpath);
-        GF_ASSERT (len >= PATH_MAX);
+    GF_ASSERT(peerinfo);
+    GF_ASSERT(peerfpath);
+    GF_ASSERT(len >= PATH_MAX);
 
-        glusterd_store_peerinfo_dirpath_set (peerdir, sizeof (peerdir));
-        gf_uuid_unparse (peerinfo->uuid, str);
-        snprintf (peerfpath, len, "%s/%s", peerdir, str);
+    glusterd_store_peerinfo_dirpath_set(peerdir, sizeof(peerdir));
+    gf_uuid_unparse(peerinfo->uuid, str);
+    snprintf(peerfpath, len, "%s/%s", peerdir, str);
 }
 
 static void
-glusterd_store_hostname_peerpath_set (glusterd_peerinfo_t *peerinfo,
-                                       char *peerfpath, size_t len)
+glusterd_store_hostname_peerpath_set(glusterd_peerinfo_t *peerinfo,
+                                     char *peerfpath, size_t len)
 {
-        char                    peerdir[PATH_MAX];
+    char peerdir[PATH_MAX];
 
-        GF_ASSERT (peerinfo);
-        GF_ASSERT (peerfpath);
-        GF_ASSERT (len >= PATH_MAX);
+    GF_ASSERT(peerinfo);
+    GF_ASSERT(peerfpath);
+    GF_ASSERT(len >= PATH_MAX);
 
-        glusterd_store_peerinfo_dirpath_set (peerdir, sizeof (peerdir));
-        snprintf (peerfpath, len, "%s/%s", peerdir, peerinfo->hostname);
+    glusterd_store_peerinfo_dirpath_set(peerdir, sizeof(peerdir));
+    snprintf(peerfpath, len, "%s/%s", peerdir, peerinfo->hostname);
 }
 
 int32_t
-glusterd_store_peerinfo_hostname_shandle_create (glusterd_peerinfo_t *peerinfo)
+glusterd_store_peerinfo_hostname_shandle_create(glusterd_peerinfo_t *peerinfo)
 {
-        char                    peerfpath[PATH_MAX];
-        int32_t                 ret = -1;
-
-        glusterd_store_hostname_peerpath_set (peerinfo, peerfpath,
-                                              sizeof (peerfpath));
-        ret = gf_store_handle_create_on_absence (&peerinfo->shandle,
-                                                 peerfpath);
-        return ret;
+    char peerfpath[PATH_MAX];
+    int32_t ret = -1;
+
+    glusterd_store_hostname_peerpath_set(peerinfo, peerfpath,
+                                         sizeof(peerfpath));
+    ret = gf_store_handle_create_on_absence(&peerinfo->shandle, peerfpath);
+    return ret;
 }
 
 int32_t
-glusterd_store_peerinfo_uuid_shandle_create (glusterd_peerinfo_t *peerinfo)
+glusterd_store_peerinfo_uuid_shandle_create(glusterd_peerinfo_t *peerinfo)
 {
-        char                    peerfpath[PATH_MAX];
-        int32_t                 ret = -1;
-
-        glusterd_store_uuid_peerpath_set (peerinfo, peerfpath,
-                                          sizeof (peerfpath));
-        ret = gf_store_handle_create_on_absence (&peerinfo->shandle,
-                                                 peerfpath);
-        return ret;
+    char peerfpath[PATH_MAX];
+    int32_t ret = -1;
+
+    glusterd_store_uuid_peerpath_set(peerinfo, peerfpath, sizeof(peerfpath));
+    ret = gf_store_handle_create_on_absence(&peerinfo->shandle, peerfpath);
+    return ret;
 }
 
 int32_t
-glusterd_peerinfo_hostname_shandle_check_destroy (glusterd_peerinfo_t *peerinfo)
+glusterd_peerinfo_hostname_shandle_check_destroy(glusterd_peerinfo_t *peerinfo)
 {
-        char                    peerfpath[PATH_MAX];
-        int32_t                 ret = -1;
-        struct  stat            stbuf = {0,};
-
-        glusterd_store_hostname_peerpath_set (peerinfo, peerfpath,
-                                              sizeof (peerfpath));
-        ret = stat (peerfpath, &stbuf);
-        if (!ret) {
-                if (peerinfo->shandle)
-                        gf_store_handle_destroy (peerinfo->shandle);
-                peerinfo->shandle = NULL;
-                ret = unlink (peerfpath);
-        }
-        return ret;
+    char peerfpath[PATH_MAX];
+    int32_t ret = -1;
+    struct stat stbuf = {
+        0,
+    };
+
+    glusterd_store_hostname_peerpath_set(peerinfo, peerfpath,
+                                         sizeof(peerfpath));
+    ret = sys_stat(peerfpath, &stbuf);
+    if (!ret) {
+        if (peerinfo->shandle)
+            gf_store_handle_destroy(peerinfo->shandle);
+        peerinfo->shandle = NULL;
+        ret = sys_unlink(peerfpath);
+    }
+    return ret;
 }
 
 int32_t
-glusterd_store_create_peer_shandle (glusterd_peerinfo_t *peerinfo)
+glusterd_store_create_peer_shandle(glusterd_peerinfo_t *peerinfo)
 {
-        int32_t                 ret = 0;
+    int32_t ret = 0;
 
-        GF_ASSERT (peerinfo);
+    GF_ASSERT(peerinfo);
 
-        if (gf_uuid_is_null (peerinfo->uuid)) {
-                ret = glusterd_store_peerinfo_hostname_shandle_create (peerinfo);
-        } else {
-                ret = glusterd_peerinfo_hostname_shandle_check_destroy (peerinfo);
-                ret = glusterd_store_peerinfo_uuid_shandle_create (peerinfo);
-        }
-        return ret;
+    if (gf_uuid_is_null(peerinfo->uuid)) {
+        ret = glusterd_store_peerinfo_hostname_shandle_create(peerinfo);
+    } else {
+        ret = glusterd_peerinfo_hostname_shandle_check_destroy(peerinfo);
+        ret = glusterd_store_peerinfo_uuid_shandle_create(peerinfo);
+    }
+    return ret;
 }
 
-int32_t
-glusterd_store_peer_write (int fd, glusterd_peerinfo_t *peerinfo)
+static int32_t
+glusterd_store_peer_write(int fd, glusterd_peerinfo_t *peerinfo)
 {
-        char                      buf[50] = {0};
-        int32_t                   ret = 0;
-        int32_t                   i = 1;
-        glusterd_peer_hostname_t *hostname = NULL;
-        char                     *key = NULL;
-
-        ret = gf_store_save_value (fd, GLUSTERD_STORE_KEY_PEER_UUID,
-                                   uuid_utoa (peerinfo->uuid));
-        if (ret)
-                goto out;
-
-        snprintf (buf, sizeof (buf), "%d", peerinfo->state.state);
-        ret = gf_store_save_value (fd, GLUSTERD_STORE_KEY_PEER_STATE, buf);
-        if (ret)
-                goto out;
-
-        cds_list_for_each_entry (hostname, &peerinfo->hostnames,
-                                 hostname_list) {
-                ret = gf_asprintf (&key, GLUSTERD_STORE_KEY_PEER_HOSTNAME"%d",
-                                   i);
-                if (ret < 0)
-                        goto out;
-                ret = gf_store_save_value (fd, key, hostname->hostname);
-                if (ret)
-                        goto out;
-                GF_FREE (key);
-                key = NULL;
-                i++;
-        }
-
+    char buf[PATH_MAX];
+    uint total_len = 0;
+    int32_t ret = 0;
+    int32_t i = 1;
+    glusterd_peer_hostname_t *hostname = NULL;
+
+    ret = snprintf(buf + total_len, sizeof(buf) - total_len, "%s=%s\n%s=%d\n",
+                   GLUSTERD_STORE_KEY_PEER_UUID, uuid_utoa(peerinfo->uuid),
+                   GLUSTERD_STORE_KEY_PEER_STATE, peerinfo->state.state);
+    if (ret < 0 || ret >= sizeof(buf) - total_len) {
+        ret = -1;
+        goto out;
+    }
+    total_len += ret;
+
+    cds_list_for_each_entry(hostname, &peerinfo->hostnames, hostname_list)
+    {
+        ret = snprintf(buf + total_len, sizeof(buf) - total_len,
+                       GLUSTERD_STORE_KEY_PEER_HOSTNAME "%d=%s\n", i,
+                       hostname->hostname);
+        if (ret < 0 || ret >= sizeof(buf) - total_len) {
+            ret = -1;
+            goto out;
+        }
+        total_len += ret;
+        i++;
+    }
+
+    ret = gf_store_save_items(fd, buf);
 out:
-        gf_log ("", GF_LOG_DEBUG, "Returning with %d", ret);
-        return ret;
+    gf_msg_debug("glusterd", 0, "Returning with %d", ret);
+    return ret;
 }
 
 int32_t
-glusterd_store_perform_peer_store (glusterd_peerinfo_t *peerinfo)
+glusterd_store_perform_peer_store(glusterd_peerinfo_t *peerinfo)
 {
-        int                                 fd = -1;
-        int32_t                             ret = -1;
+    int fd = -1;
+    int32_t ret = -1;
 
-        GF_ASSERT (peerinfo);
+    GF_ASSERT(peerinfo);
 
-        fd = gf_store_mkstemp (peerinfo->shandle);
-        if (fd <= 0) {
-                ret = -1;
-                goto out;
-        }
+    fd = gf_store_mkstemp(peerinfo->shandle);
+    if (fd <= 0) {
+        ret = -1;
+        goto out;
+    }
 
-        ret = glusterd_store_peer_write (fd, peerinfo);
-        if (ret)
-                goto out;
+    ret = glusterd_store_peer_write(fd, peerinfo);
+    if (ret)
+        goto out;
 
-        ret = gf_store_rename_tmppath (peerinfo->shandle);
+    ret = gf_store_rename_tmppath(peerinfo->shandle);
 out:
-        if (ret && (fd > 0))
-                gf_store_unlink_tmppath (peerinfo->shandle);
-        gf_log ("", GF_LOG_DEBUG, "Returning %d", ret);
-        return ret;
+    if (ret && (fd > 0))
+        gf_store_unlink_tmppath(peerinfo->shandle);
+    gf_msg_debug("glusterd", 0, "Returning %d", ret);
+    return ret;
 }
 
 int32_t
-glusterd_store_peerinfo (glusterd_peerinfo_t *peerinfo)
+glusterd_store_peerinfo(glusterd_peerinfo_t *peerinfo)
 {
-        int32_t                         ret = -1;
+    int32_t ret = -1;
 
-        GF_ASSERT (peerinfo);
+    GF_ASSERT(peerinfo);
 
-        ret = glusterd_store_create_peer_dir ();
-        if (ret)
-                goto out;
+    ret = glusterd_store_create_peer_dir();
+    if (ret)
+        goto out;
 
-        ret = glusterd_store_create_peer_shandle (peerinfo);
-        if (ret)
-                goto out;
+    ret = glusterd_store_create_peer_shandle(peerinfo);
+    if (ret)
+        goto out;
 
-        ret = glusterd_store_perform_peer_store (peerinfo);
+    ret = glusterd_store_perform_peer_store(peerinfo);
 out:
-        gf_log ("", GF_LOG_DEBUG, "Returning with %d", ret);
-        return ret;
+    gf_msg_debug("glusterd", 0, "Returning with %d", ret);
+    return ret;
 }
 
 int32_t
-glusterd_store_retrieve_peers (xlator_t *this)
+glusterd_store_retrieve_peers(xlator_t *this)
 {
-        int32_t                   ret                = 0;
-        glusterd_conf_t          *priv               = NULL;
-        DIR                      *dir                = NULL;
-        struct dirent            *entry              = NULL;
-        char                      path[PATH_MAX]     = {0,};
-        glusterd_peerinfo_t      *peerinfo           = NULL;
-        char                     *hostname           = NULL;
-        gf_store_handle_t        *shandle            = NULL;
-        char                      filepath[PATH_MAX] = {0,};
-        gf_store_iter_t          *iter               = NULL;
-        char                     *key                = NULL;
-        char                     *value              = NULL;
-        glusterd_peerctx_args_t   args               = {0};
-        gf_store_op_errno_t       op_errno           = GD_STORE_SUCCESS;
-        glusterd_peer_hostname_t *address            = NULL;
-
-        GF_ASSERT (this);
-        priv = this->private;
-
-        GF_ASSERT (priv);
-
-        snprintf (path, PATH_MAX, "%s/%s", priv->workdir,
-                  GLUSTERD_PEER_DIR_PREFIX);
-
-        dir = opendir (path);
-
-        if (!dir) {
-                gf_log ("", GF_LOG_ERROR, "Unable to open dir %s", path);
-                ret = -1;
-                goto out;
-        }
-
-        GF_FOR_EACH_ENTRY_IN_DIR (entry, dir);
+    int32_t ret = 0;
+    glusterd_conf_t *priv = NULL;
+    DIR *dir = NULL;
+    struct dirent *entry = NULL;
+    struct dirent scratch[2] = {
+        {
+            0,
+        },
+    };
+    char path[PATH_MAX] = {
+        0,
+    };
+    glusterd_peerinfo_t *peerinfo = NULL;
+    gf_store_handle_t *shandle = NULL;
+    char filepath[PATH_MAX] = {
+        0,
+    };
+    gf_store_iter_t *iter = NULL;
+    char *key = NULL;
+    char *value = NULL;
+    glusterd_peerctx_args_t args = {0};
+    gf_store_op_errno_t op_errno = GD_STORE_SUCCESS;
+    glusterd_peer_hostname_t *address = NULL;
+    uuid_t tmp_uuid;
+    gf_boolean_t is_ok;
+    int32_t len;
+
+    GF_ASSERT(this);
+    priv = this->private;
+
+    GF_ASSERT(priv);
+
+    len = snprintf(path, PATH_MAX, "%s/%s", priv->workdir,
+                   GLUSTERD_PEER_DIR_PREFIX);
+    if ((len < 0) || (len >= PATH_MAX)) {
+        ret = -1;
+        goto out;
+    }
+
+    dir = sys_opendir(path);
+
+    if (!dir) {
+        gf_msg(this->name, GF_LOG_ERROR, errno, GD_MSG_DIR_OP_FAILED,
+               "Unable to open dir %s", path);
+        ret = -1;
+        goto out;
+    }
+
+    while ((entry = sys_readdir(dir, scratch))) {
+        if (gf_irrelevant_entry(entry))
+            continue;
+        if (gf_uuid_parse(entry->d_name, tmp_uuid) != 0) {
+            gf_log(this->name, GF_LOG_WARNING, "skipping non-peer file %s",
+                   entry->d_name);
+            continue;
+        }
+        is_ok = _gf_false;
+        len = snprintf(filepath, PATH_MAX, "%s/%s", path, entry->d_name);
+        if ((len < 0) || (len >= PATH_MAX)) {
+            goto next;
+        }
+        ret = gf_store_handle_retrieve(filepath, &shandle);
+        if (ret)
+            goto next;
 
-        while (entry) {
-                snprintf (filepath, PATH_MAX, "%s/%s", path, entry->d_name);
-                ret = gf_store_handle_retrieve (filepath, &shandle);
-                if (ret)
-                        goto out;
+        ret = gf_store_iter_new(shandle, &iter);
+        if (ret)
+            goto next;
 
-                ret = gf_store_iter_new (shandle, &iter);
-                if (ret)
-                        goto out;
+        ret = gf_store_iter_get_next(iter, &key, &value, &op_errno);
+        if (ret) {
+            goto next;
+        }
 
-                ret = gf_store_iter_get_next (iter, &key, &value, &op_errno);
-                if (ret)
-                        goto out;
+        /* Create an empty peerinfo object before reading in the
+         * details
+         */
+        peerinfo = glusterd_peerinfo_new(GD_FRIEND_STATE_DEFAULT, NULL, NULL,
+                                         0);
+        if (peerinfo == NULL) {
+            ret = -1;
+            goto next;
+        }
 
-                /* Create an empty peerinfo object before reading in the
-                 * details
-                 */
-                peerinfo = glusterd_peerinfo_new (GD_FRIEND_STATE_DEFAULT, NULL,
-                                                  NULL, 0);
-                if (peerinfo == NULL) {
-                        ret = -1;
-                        goto out;
+        while (!ret) {
+            if (!strncmp(GLUSTERD_STORE_KEY_PEER_UUID, key,
+                         SLEN(GLUSTERD_STORE_KEY_PEER_UUID))) {
+                if (value)
+                    gf_uuid_parse(value, peerinfo->uuid);
+            } else if (!strncmp(GLUSTERD_STORE_KEY_PEER_STATE, key,
+                                SLEN(GLUSTERD_STORE_KEY_PEER_STATE))) {
+                peerinfo->state.state = atoi(value);
+            } else if (!strncmp(GLUSTERD_STORE_KEY_PEER_HOSTNAME, key,
+                                SLEN(GLUSTERD_STORE_KEY_PEER_HOSTNAME))) {
+                ret = gd_add_address_to_peer(peerinfo, value);
+                if (ret) {
+                    gf_msg(this->name, GF_LOG_ERROR, 0,
+                           GD_MSG_ADD_ADDRESS_TO_PEER_FAIL,
+                           "Could not add address to peer");
                 }
+            } else {
+                gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_UNKNOWN_KEY,
+                       "Unknown key: %s", key);
+            }
 
-                while (!ret) {
-
-                        if (!strncmp (GLUSTERD_STORE_KEY_PEER_UUID, key,
-                                      strlen (GLUSTERD_STORE_KEY_PEER_UUID))) {
-                                if (value)
-                                        gf_uuid_parse (value, peerinfo->uuid);
-                        } else if (!strncmp (GLUSTERD_STORE_KEY_PEER_STATE,
-                                    key,
-                                    strlen (GLUSTERD_STORE_KEY_PEER_STATE))) {
-                                peerinfo->state.state = atoi (value);
-                        } else if (!strncmp (GLUSTERD_STORE_KEY_PEER_HOSTNAME,
-                                   key,
-                                   strlen (GLUSTERD_STORE_KEY_PEER_HOSTNAME))) {
-                                ret = gd_add_address_to_peer (peerinfo, value);
-                        } else {
-                                gf_log ("", GF_LOG_ERROR, "Unknown key: %s",
-                                        key);
-                        }
-
-                        GF_FREE (key);
-                        GF_FREE (value);
-                        key = NULL;
-                        value = NULL;
-
-                        ret = gf_store_iter_get_next (iter, &key, &value,
-                                                      &op_errno);
-                }
-                if (op_errno != GD_STORE_EOF) {
-                        GF_FREE(hostname);
-                        goto out;
-                }
+            GF_FREE(key);
+            GF_FREE(value);
+            key = NULL;
+            value = NULL;
 
-                (void) gf_store_iter_destroy (iter);
+            ret = gf_store_iter_get_next(iter, &key, &value, &op_errno);
+        }
+        if (op_errno != GD_STORE_EOF) {
+            goto next;
+        }
 
-                /* Set first hostname from peerinfo->hostnames to
-                 * peerinfo->hostname
-                 */
-                address = cds_list_entry (peerinfo->hostnames.next,
-                                          glusterd_peer_hostname_t,
-                                          hostname_list);
-                if (!address) {
-                        ret = -1;
-                        goto out;
-                }
-                peerinfo->hostname = gf_strdup (address->hostname);
+        if (gf_uuid_is_null(peerinfo->uuid)) {
+            gf_log("", GF_LOG_ERROR,
+                   "Null UUID while attempting to read peer from '%s'",
+                   filepath);
+            goto next;
+        }
 
-                ret = glusterd_friend_add_from_peerinfo (peerinfo, 1, NULL);
-                if (ret)
-                        goto out;
+        /* Set first hostname from peerinfo->hostnames to
+         * peerinfo->hostname
+         */
+        address = cds_list_entry(peerinfo->hostnames.next,
+                                 glusterd_peer_hostname_t, hostname_list);
+        peerinfo->hostname = gf_strdup(address->hostname);
 
-                peerinfo->shandle = shandle;
-                peerinfo = NULL;
-                GF_FOR_EACH_ENTRY_IN_DIR (entry, dir);
-        }
+        ret = glusterd_friend_add_from_peerinfo(peerinfo, 1, NULL);
+        if (ret)
+            goto next;
 
-        args.mode = GD_MODE_ON;
+        peerinfo->shandle = shandle;
+        is_ok = _gf_true;
 
-        rcu_read_lock ();
-        cds_list_for_each_entry_rcu (peerinfo, &priv->peers, uuid_list) {
-                ret = glusterd_friend_rpc_create (this, peerinfo, &args);
-                if (ret)
-                        break;
+    next:
+        (void)gf_store_iter_destroy(&iter);
+
+        if (!is_ok) {
+            gf_log(this->name, GF_LOG_WARNING,
+                   "skipping malformed peer file %s", entry->d_name);
+            if (peerinfo) {
+                glusterd_peerinfo_cleanup(peerinfo);
+            }
         }
-        rcu_read_unlock ();
         peerinfo = NULL;
+    }
+
+    args.mode = GD_MODE_ON;
+
+    RCU_READ_LOCK;
+    cds_list_for_each_entry_rcu(peerinfo, &priv->peers, uuid_list)
+    {
+        ret = glusterd_friend_rpc_create(this, peerinfo, &args);
+        if (ret)
+            break;
+    }
+    RCU_READ_UNLOCK;
+    peerinfo = NULL;
 
 out:
-        if (peerinfo)
-                glusterd_peerinfo_cleanup (peerinfo);
 
-        if (dir)
-                closedir (dir);
-        gf_log ("", GF_LOG_DEBUG, "Returning with %d", ret);
+    if (dir)
+        sys_closedir(dir);
+    gf_msg_debug(this->name, 0, "Returning with %d", ret);
 
-        return ret;
+    return ret;
 }
 
 /* Bricks for snap volumes are hosted at /var/run/gluster/snaps
@@ -4206,49 +4699,58 @@ out:
  * paths need to be recreated and re-mounted
  */
 int32_t
-glusterd_recreate_all_snap_brick_mounts (xlator_t  *this)
+glusterd_recreate_all_snap_brick_mounts(xlator_t *this)
 {
-        int32_t                  ret       = 0;
-        glusterd_conf_t         *priv      = NULL;
-        glusterd_volinfo_t      *volinfo   = NULL;
-        glusterd_snap_t         *snap      = NULL;
-
-        GF_ASSERT (this);
-        priv = this->private;
-        GF_ASSERT (priv);
-
-        /* Recreate bricks of volumes restored from snaps */
-        cds_list_for_each_entry (volinfo, &priv->volumes, vol_list) {
-                /* If the volume is not a restored volume then continue */
-                if (gf_uuid_is_null (volinfo->restored_from_snap))
-                        continue;
-
-                ret = glusterd_recreate_vol_brick_mounts (this, volinfo);
+    int32_t ret = 0;
+    glusterd_conf_t *priv = NULL;
+    glusterd_volinfo_t *volinfo = NULL;
+    glusterd_snap_t *snap = NULL;
+
+    GF_ASSERT(this);
+    priv = this->private;
+    GF_ASSERT(priv);
+
+    /* Recreate bricks of volumes restored from snaps */
+    cds_list_for_each_entry(volinfo, &priv->volumes, vol_list)
+    {
+        /* If the volume is not a restored volume then continue */
+        if (gf_uuid_is_null(volinfo->restored_from_snap))
+            continue;
+
+        ret = glusterd_recreate_vol_brick_mounts(this, volinfo);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_BRK_MNT_RECREATE_FAIL,
+                   "Failed to recreate brick mounts "
+                   "for %s",
+                   volinfo->volname);
+            goto out;
+        }
+    }
+
+    /* Recreate bricks of snapshot volumes
+     * We are not creating brick mounts for stopped snaps.
+     */
+    cds_list_for_each_entry(snap, &priv->snapshots, snap_list)
+    {
+        cds_list_for_each_entry(volinfo, &snap->volumes, vol_list)
+        {
+            if (volinfo->status != GLUSTERD_STATUS_STOPPED) {
+                ret = glusterd_recreate_vol_brick_mounts(this, volinfo);
                 if (ret) {
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "Failed to recreate brick mounts "
-                                "for %s", volinfo->volname);
-                        goto out;
-                }
-        }
-
-        /* Recreate bricks of snapshot volumes */
-        cds_list_for_each_entry (snap, &priv->snapshots, snap_list) {
-                cds_list_for_each_entry (volinfo, &snap->volumes, vol_list) {
-                        ret = glusterd_recreate_vol_brick_mounts (this,
-                                                                  volinfo);
-                        if (ret) {
-                                gf_log (this->name, GF_LOG_ERROR,
-                                        "Failed to recreate brick mounts "
-                                        "for %s", snap->snapname);
-                                goto out;
-                        }
+                    gf_msg(this->name, GF_LOG_ERROR, 0,
+                           GD_MSG_BRK_MNT_RECREATE_FAIL,
+                           "Failed to recreate brick "
+                           "mounts for %s",
+                           snap->snapname);
+                    goto out;
                 }
+            }
         }
+    }
 
 out:
-        gf_log (this->name, GF_LOG_TRACE, "Returning with %d", ret);
-        return ret;
+    gf_msg_trace(this->name, 0, "Returning with %d", ret);
+    return ret;
 }
 
 /* When the snapshot command from cli is received, the on disk and
@@ -4265,252 +4767,359 @@ out:
  * taken.
  */
 int32_t
-glusterd_snap_cleanup (xlator_t  *this)
+glusterd_snap_cleanup(xlator_t *this)
 {
-        dict_t               *dict = NULL;
-        int32_t               ret  = 0;
-        glusterd_conf_t      *priv = NULL;
-        glusterd_snap_t      *snap = NULL;
-        glusterd_snap_t      *tmp_snap = NULL;
-
-        GF_ASSERT (this);
-        priv = this->private;
-        GF_ASSERT (priv);
-
-        dict = dict_new();
-        if (!dict) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "Failed to create dict");
-                ret = -1;
-                goto out;
-        }
-
-        cds_list_for_each_entry_safe (snap, tmp_snap, &priv->snapshots,
-                                      snap_list) {
-                if (snap->snap_status == GD_SNAP_STATUS_RESTORED) {
-                        ret = glusterd_snapshot_revert_restore_from_snap (snap);
-                        if (ret) {
-                                gf_log (this->name, GF_LOG_WARNING, "Failed to "
-                                        "revert partially restored snapshot "
-                                        "(%s)", snap->snapname);
-                                goto out;
-                        }
-                } else if (snap->snap_status != GD_SNAP_STATUS_IN_USE) {
-                        ret = glusterd_snap_remove (dict, snap,
-                                                    _gf_true, _gf_true);
-                        if (ret) {
-                                gf_log (this->name, GF_LOG_ERROR,
-                                        "Failed to remove the snapshot %s",
-                                        snap->snapname);
-                                goto out;
-                        }
-                }
-        }
+    dict_t *dict = NULL;
+    int32_t ret = 0;
+    glusterd_conf_t *priv = NULL;
+    glusterd_snap_t *snap = NULL;
+    glusterd_snap_t *tmp_snap = NULL;
+
+    GF_ASSERT(this);
+    priv = this->private;
+    GF_ASSERT(priv);
+
+    dict = dict_new();
+    if (!dict) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_CREATE_FAIL,
+               "Failed to create dict");
+        ret = -1;
+        goto out;
+    }
+
+    cds_list_for_each_entry_safe(snap, tmp_snap, &priv->snapshots, snap_list)
+    {
+        if (snap->snap_status == GD_SNAP_STATUS_RESTORED) {
+            ret = glusterd_snapshot_revert_restore_from_snap(snap);
+            if (ret) {
+                gf_msg(this->name, GF_LOG_WARNING, 0,
+                       GD_MSG_SNAP_RESTORE_REVERT_FAIL,
+                       "Failed to "
+                       "revert partially restored snapshot "
+                       "(%s)",
+                       snap->snapname);
+                goto out;
+            }
+        } else if (snap->snap_status != GD_SNAP_STATUS_IN_USE) {
+            ret = glusterd_snap_remove(dict, snap, _gf_true, _gf_true,
+                                       _gf_false);
+            if (ret) {
+                gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_SNAP_REMOVE_FAIL,
+                       "Failed to remove the snapshot %s", snap->snapname);
+                goto out;
+            }
+        }
+    }
 out:
-        if (dict)
-                dict_unref (dict);
+    if (dict)
+        dict_unref(dict);
 
-        gf_log (this->name, GF_LOG_TRACE, "Returning with %d", ret);
-        return ret;
+    gf_msg_trace(this->name, 0, "Returning with %d", ret);
+    return ret;
 }
 
 int32_t
-glusterd_resolve_all_bricks (xlator_t  *this)
+glusterd_resolve_all_bricks(xlator_t *this)
 {
-        int32_t                 ret        = 0;
-        glusterd_conf_t         *priv      = NULL;
-        glusterd_volinfo_t      *volinfo   = NULL;
-        glusterd_brickinfo_t    *brickinfo = NULL;
-        glusterd_snap_t         *snap      = NULL;
-
-        GF_ASSERT (this);
-        priv = this->private;
-
-        GF_ASSERT (priv);
-
-        /* Resolve bricks of volumes */
-        cds_list_for_each_entry (volinfo, &priv->volumes, vol_list) {
-                cds_list_for_each_entry (brickinfo, &volinfo->bricks,
-                                         brick_list) {
-                        ret = glusterd_resolve_brick (brickinfo);
-                        if (ret) {
-                                gf_log ("glusterd", GF_LOG_ERROR,
-                                        "resolve brick failed in restore");
-                                goto out;
-                        }
-                }
-        }
+    int32_t ret = 0;
+    glusterd_conf_t *priv = NULL;
+    glusterd_volinfo_t *volinfo = NULL;
+    glusterd_brickinfo_t *brickinfo = NULL;
+    glusterd_snap_t *snap = NULL;
 
-        /* Resolve bricks of snapshot volumes */
-        cds_list_for_each_entry (snap, &priv->snapshots, snap_list) {
-                ret = glusterd_resolve_snap_bricks (this, snap);
-                if (ret) {
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "resolving the snap bricks"
-                                " failed for snap: %s",
-                                snap->snapname);
-                        goto out;
-                }
+    GF_ASSERT(this);
+    priv = this->private;
+
+    GF_ASSERT(priv);
+
+    /* Resolve bricks of volumes */
+    cds_list_for_each_entry(volinfo, &priv->volumes, vol_list)
+    {
+        cds_list_for_each_entry(brickinfo, &volinfo->bricks, brick_list)
+        {
+            ret = glusterd_resolve_brick(brickinfo);
+            if (ret) {
+                gf_event(EVENT_BRICKPATH_RESOLVE_FAILED,
+                         "peer=%s;volume=%s;brick=%s", brickinfo->hostname,
+                         volinfo->volname, brickinfo->path);
+                gf_msg("glusterd", GF_LOG_ERROR, 0, GD_MSG_RESOLVE_BRICK_FAIL,
+                       "Failed to resolve brick %s with host %s of volume %s"
+                       " in restore",
+                       brickinfo->path, brickinfo->hostname, volinfo->volname);
+                goto out;
+            }
+        }
+    }
+
+    /* Resolve bricks of snapshot volumes */
+    cds_list_for_each_entry(snap, &priv->snapshots, snap_list)
+    {
+        ret = glusterd_resolve_snap_bricks(this, snap);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_SNAP_RESOLVE_BRICK_FAIL,
+                   "resolving the snap bricks"
+                   " failed for snap: %s",
+                   snap->snapname);
+            goto out;
         }
+    }
 
 out:
-        gf_log (this->name, GF_LOG_TRACE, "Returning with %d", ret);
-        return ret;
+    gf_msg_trace(this->name, 0, "Returning with %d", ret);
+    return ret;
 }
 
 int32_t
-glusterd_restore ()
+glusterd_restore()
 {
-        int32_t         ret = -1;
-        xlator_t        *this = NULL;
+    int32_t ret = -1;
+    xlator_t *this = NULL;
+
+    this = THIS;
+
+    ret = glusterd_options_init(this);
+    if (ret < 0)
+        goto out;
+
+    ret = glusterd_store_retrieve_volumes(this, NULL);
+    if (ret)
+        goto out;
+
+    ret = glusterd_store_retrieve_peers(this);
+    if (ret)
+        goto out;
+
+    /* While retrieving snapshots, if the snapshot status
+       is not GD_SNAP_STATUS_IN_USE, then the snapshot is
+       cleaned up. To do that, the snap volume has to be
+       stopped by stopping snapshot volume's bricks. And for
+       that the snapshot bricks should be resolved. But without
+       retrieving the peers, resolving bricks will fail. So
+       do retrieving of snapshots after retrieving peers.
+    */
+    ret = glusterd_store_retrieve_snaps(this);
+    if (ret)
+        goto out;
+
+    ret = glusterd_resolve_all_bricks(this);
+    if (ret)
+        goto out;
+
+    ret = glusterd_snap_cleanup(this);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_SNAP_CLEANUP_FAIL,
+               "Failed to perform "
+               "a cleanup of the snapshots");
+        goto out;
+    }
+
+    ret = glusterd_recreate_all_snap_brick_mounts(this);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_SNAP_BRK_MNT_RECREATE_FAIL,
+               "Failed to recreate "
+               "all snap brick mounts");
+        goto out;
+    }
 
-        this = THIS;
-
-        ret = glusterd_store_retrieve_volumes (this, NULL);
-        if (ret)
-                goto out;
-
-        ret = glusterd_store_retrieve_peers (this);
-        if (ret)
-                goto out;
+out:
+    gf_msg_debug(this->name, 0, "Returning %d", ret);
+    return ret;
+}
 
-        /* While retrieving snapshots, if the snapshot status
-           is not GD_SNAP_STATUS_IN_USE, then the snapshot is
-           cleaned up. To do that, the snap volume has to be
-           stopped by stopping snapshot volume's bricks. And for
-           that the snapshot bricks should be resolved. But without
-           retrieving the peers, resolving bricks will fail. So
-           do retrieving of snapshots after retrieving peers.
-        */
-        ret = glusterd_store_retrieve_snaps (this);
-        if (ret)
-                goto out;
+int
+glusterd_store_retrieve_quota_version(glusterd_volinfo_t *volinfo)
+{
+    int ret = -1;
+    uint32_t version = 0;
+    char cksum_path[PATH_MAX] = {
+        0,
+    };
+    char path[PATH_MAX] = {
+        0,
+    };
+    char *version_str = NULL;
+    char *tmp = NULL;
+    xlator_t *this = NULL;
+    glusterd_conf_t *conf = NULL;
+    gf_store_handle_t *handle = NULL;
+    int32_t len = 0;
+
+    this = THIS;
+    GF_ASSERT(this);
+    conf = this->private;
+    GF_ASSERT(conf);
+
+    GLUSTERD_GET_VOLUME_DIR(path, volinfo, conf);
+    len = snprintf(cksum_path, sizeof(cksum_path), "%s/%s", path,
+                   GLUSTERD_VOL_QUOTA_CKSUM_FILE);
+    if ((len < 0) || (len >= sizeof(cksum_path))) {
+        goto out;
+    }
+
+    ret = gf_store_handle_new(cksum_path, &handle);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_STORE_HANDLE_GET_FAIL,
+               "Unable to get store handle "
+               "for %s",
+               cksum_path);
+        goto out;
+    }
+
+    ret = gf_store_retrieve_value(handle, "version", &version_str);
+    if (ret) {
+        gf_msg_debug(this->name, 0, "Version absent");
+        ret = 0;
+        goto out;
+    }
 
-        ret = glusterd_resolve_all_bricks (this);
-        if (ret)
-                goto out;
+    version = strtoul(version_str, &tmp, 10);
+    if ((errno == ERANGE) || (errno == EINVAL)) {
+        gf_msg_debug(this->name, 0, "Invalid version number");
+        goto out;
+    }
+    volinfo->quota_conf_version = version;
+    ret = 0;
 
-        ret = glusterd_snap_cleanup (this);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "Failed to perform "
-                        "a cleanup of the snapshots");
-                goto out;
-        }
+out:
+    if (version_str)
+        GF_FREE(version_str);
+    gf_store_handle_destroy(handle);
+    return ret;
+}
 
-        ret = glusterd_recreate_all_snap_brick_mounts (this);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "Failed to recreate "
-                        "all snap brick mounts");
-                goto out;
-        }
+int
+glusterd_store_save_quota_version_and_cksum(glusterd_volinfo_t *volinfo)
+{
+    gf_store_handle_t *shandle = NULL;
+    glusterd_conf_t *conf = NULL;
+    xlator_t *this = NULL;
+    char path[PATH_MAX] = {0};
+    char cksum_path[PATH_MAX + 32] = {
+        0,
+    };
+    char buf[64] = {0};
+    int fd = -1;
+    int32_t ret = -1;
+    int32_t len = 0;
+
+    this = THIS;
+    conf = this->private;
+
+    GLUSTERD_GET_VOLUME_DIR(path, volinfo, conf);
+    len = snprintf(cksum_path, sizeof(cksum_path), "%s/%s", path,
+                   GLUSTERD_VOL_QUOTA_CKSUM_FILE);
+    if ((len < 0) || (len >= sizeof(cksum_path))) {
+        goto out;
+    }
+
+    ret = gf_store_handle_new(cksum_path, &shandle);
+    if (ret)
+        goto out;
+
+    fd = gf_store_mkstemp(shandle);
+    if (fd <= 0) {
+        ret = -1;
+        goto out;
+    }
+
+    ret = snprintf(buf, sizeof(buf), "cksum=%u\nversion=%u\n",
+                   volinfo->quota_conf_cksum, volinfo->quota_conf_version);
+    if (ret < 0 || ret >= sizeof(buf)) {
+        ret = -1;
+        goto out;
+    }
+
+    ret = gf_store_save_items(fd, buf);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_CKSUM_STORE_FAIL,
+               "Failed to store quota cksum and version");
+        goto out;
+    }
+
+    ret = gf_store_rename_tmppath(shandle);
+    if (ret)
+        goto out;
 
 out:
-        gf_log ("", GF_LOG_DEBUG, "Returning %d", ret);
-        return ret;
+    if ((ret < 0) && (fd > 0))
+        gf_store_unlink_tmppath(shandle);
+    gf_store_handle_destroy(shandle);
+    return ret;
 }
 
-int
-glusterd_store_retrieve_quota_version (glusterd_volinfo_t *volinfo)
+int32_t
+glusterd_quota_conf_write_header(int fd)
 {
-        int                 ret                  = -1;
-        uint32_t            version              = 0;
-        char                cksum_path[PATH_MAX] = {0,};
-        char                path[PATH_MAX]       = {0,};
-        char               *version_str          = NULL;
-        char               *tmp                  = NULL;
-        xlator_t           *this                 = NULL;
-        glusterd_conf_t    *conf                 = NULL;
-        gf_store_handle_t  *handle               = NULL;
-
-        this = THIS;
-        GF_ASSERT (this);
-        conf = this->private;
-        GF_ASSERT (conf);
-
-        GLUSTERD_GET_VOLUME_DIR (path, volinfo, conf);
-        snprintf (cksum_path, sizeof (cksum_path), "%s/%s", path,
-                  GLUSTERD_VOL_QUOTA_CKSUM_FILE);
-
-        ret = gf_store_handle_new (cksum_path, &handle);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "Unable to get store handle "
-                        "for %s", cksum_path);
-                goto out;
-        }
+    int header_len = 0;
+    int ret = -1;
+    xlator_t *this = NULL;
+    glusterd_conf_t *conf = NULL;
 
-        ret = gf_store_retrieve_value (handle, "version", &version_str);
-        if (ret) {
-                gf_log (this->name, GF_LOG_DEBUG, "Version absent");
-                ret = 0;
-                goto out;
-        }
+    this = THIS;
+    GF_VALIDATE_OR_GOTO("quota", this, out);
 
-        version = strtoul (version_str, &tmp, 10);
-	if ((errno == ERANGE) || (errno == EINVAL)) {
-                 gf_log (this->name, GF_LOG_DEBUG, "Invalid version number");
-                 goto out;
-        }
-        volinfo->quota_conf_version = version;
-        ret = 0;
+    conf = this->private;
+    GF_VALIDATE_OR_GOTO(this->name, conf, out);
+
+    if (conf->op_version < GD_OP_VERSION_3_7_0) {
+        header_len = SLEN(QUOTA_CONF_HEADER_1_1);
+        ret = gf_nwrite(fd, QUOTA_CONF_HEADER_1_1, header_len);
+    } else {
+        header_len = SLEN(QUOTA_CONF_HEADER);
+        ret = gf_nwrite(fd, QUOTA_CONF_HEADER, header_len);
+    }
+
+    if (ret != header_len) {
+        ret = -1;
+        goto out;
+    }
+
+    ret = 0;
 
 out:
-        if (version_str)
-                GF_FREE (version_str);
-        gf_store_handle_destroy (handle);
-        return ret;
+    if (ret < 0)
+        gf_msg_callingfn("quota", GF_LOG_ERROR, 0, GD_MSG_QUOTA_CONF_WRITE_FAIL,
+                         "failed to write "
+                         "header to a quota conf");
+
+    return ret;
 }
 
-int
-glusterd_store_save_quota_version_and_cksum (glusterd_volinfo_t *volinfo)
+int32_t
+glusterd_quota_conf_write_gfid(int fd, void *buf, char type)
 {
-        gf_store_handle_t               *shandle = NULL;
-        glusterd_conf_t                 *conf = NULL;
-        xlator_t                        *this = NULL;
-        char                            path[PATH_MAX] = {0};
-        char                            cksum_path[PATH_MAX] = {0,};
-        char                            buf[256] = {0};
-        int                             fd = -1;
-        int32_t                         ret = -1;
-
-        this = THIS;
-        conf = this->private;
-
-        GLUSTERD_GET_VOLUME_DIR (path, volinfo, conf);
-        snprintf (cksum_path, sizeof (cksum_path), "%s/%s", path,
-                  GLUSTERD_VOL_QUOTA_CKSUM_FILE);
-
-        ret = gf_store_handle_new (cksum_path, &shandle);
-        if (ret)
-                goto out;
+    int ret = -1;
+    xlator_t *this = NULL;
+    glusterd_conf_t *conf = NULL;
 
-        fd = gf_store_mkstemp (shandle);
-        if (fd <= 0) {
-                ret = -1;
-                goto out;
-        }
+    this = THIS;
+    GF_VALIDATE_OR_GOTO("quota", this, out);
 
-        snprintf (buf, sizeof (buf)-1, "%u", volinfo->quota_conf_cksum);
-        ret = gf_store_save_value (fd, "cksum", buf);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "Failed to store cksum");
-                goto out;
-        }
+    conf = this->private;
+    GF_VALIDATE_OR_GOTO(this->name, conf, out);
 
-        memset (buf, 0, sizeof (buf));
-        snprintf (buf, sizeof (buf)-1, "%u", volinfo->quota_conf_version);
-        ret = gf_store_save_value (fd, "version", buf);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "Failed to store version");
-                goto out;
+    ret = gf_nwrite(fd, buf, 16);
+    if (ret != 16) {
+        ret = -1;
+        goto out;
+    }
+
+    if (conf->op_version >= GD_OP_VERSION_3_7_0) {
+        ret = gf_nwrite(fd, &type, 1);
+        if (ret != 1) {
+            ret = -1;
+            goto out;
         }
+    }
 
-        ret = gf_store_rename_tmppath (shandle);
-        if (ret)
-                goto out;
+    ret = 0;
 
 out:
-        if ((ret < 0) && (fd > 0))
-                gf_store_unlink_tmppath (shandle);
-        gf_store_handle_destroy (shandle);
-        return ret;
+    if (ret < 0)
+        gf_msg_callingfn("quota", GF_LOG_ERROR, 0, GD_MSG_QUOTA_CONF_WRITE_FAIL,
+                         "failed to write "
+                         "gfid %s to a quota conf",
+                         uuid_utoa(buf));
+
+    return ret;
 }
diff --git a/xlators/mgmt/glusterd/src/glusterd-store.h b/xlators/mgmt/glusterd/src/glusterd-store.h
index 72fdd851d23..83f4df0783e 100644
--- a/xlators/mgmt/glusterd/src/glusterd-store.h
+++ b/xlators/mgmt/glusterd/src/glusterd-store.h
@@ -10,173 +10,207 @@
 #ifndef _GLUSTERD_HA_H_
 #define _GLUSTERD_HA_H_
 
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
-
 #include <pthread.h>
-#include "compat-uuid.h"
-
-#include "glusterfs.h"
-#include "xlator.h"
-#include "run.h"
-#include "logging.h"
-#include "call-stub.h"
-#include "fd.h"
-#include "byte-order.h"
+#include <glusterfs/compat-uuid.h>
+
+#include <glusterfs/glusterfs.h>
+#include <glusterfs/xlator.h>
+#include <glusterfs/run.h>
+#include <glusterfs/logging.h>
+#include <glusterfs/call-stub.h>
+#include <glusterfs/byte-order.h>
 #include "glusterd.h"
 #include "rpcsvc.h"
 
-typedef enum glusterd_store_ver_ac_{
-        GLUSTERD_VOLINFO_VER_AC_NONE = 0,
-        GLUSTERD_VOLINFO_VER_AC_INCREMENT = 1,
-        GLUSTERD_VOLINFO_VER_AC_DECREMENT = 2,
+typedef enum glusterd_store_ver_ac_ {
+    GLUSTERD_VOLINFO_VER_AC_NONE = 0,
+    GLUSTERD_VOLINFO_VER_AC_INCREMENT = 1,
+    GLUSTERD_VOLINFO_VER_AC_DECREMENT = 2,
 } glusterd_volinfo_ver_ac_t;
 
-
-#define GLUSTERD_STORE_UUID_KEY                 "UUID"
-
-#define GLUSTERD_STORE_KEY_VOL_TYPE             "type"
-#define GLUSTERD_STORE_KEY_VOL_COUNT            "count"
-#define GLUSTERD_STORE_KEY_VOL_STATUS           "status"
-#define GLUSTERD_STORE_KEY_VOL_PORT             "port"
-#define GLUSTERD_STORE_KEY_VOL_SUB_COUNT        "sub_count"
-#define GLUSTERD_STORE_KEY_VOL_STRIPE_CNT       "stripe_count"
-#define GLUSTERD_STORE_KEY_VOL_REPLICA_CNT      "replica_count"
-#define GLUSTERD_STORE_KEY_VOL_DISPERSE_CNT     "disperse_count"
-#define GLUSTERD_STORE_KEY_VOL_REDUNDANCY_CNT   "redundancy_count"
-#define GLUSTERD_STORE_KEY_VOL_BRICK            "brick"
-#define GLUSTERD_STORE_KEY_VOL_VERSION          "version"
-#define GLUSTERD_STORE_KEY_VOL_TRANSPORT        "transport-type"
-#define GLUSTERD_STORE_KEY_VOL_ID               "volume-id"
-#define GLUSTERD_STORE_KEY_VOL_RESTORED_SNAP    "restored_from_snap"
-#define GLUSTERD_STORE_KEY_RB_STATUS            "rb_status"
-#define GLUSTERD_STORE_KEY_RB_SRC_BRICK         "rb_src"
-#define GLUSTERD_STORE_KEY_RB_DST_BRICK         "rb_dst"
-#define GLUSTERD_STORE_KEY_RB_DST_PORT          "rb_port"
-#define GLUSTERD_STORE_KEY_VOL_DEFRAG           "rebalance_status"
-#define GLUSTERD_STORE_KEY_VOL_DEFRAG_STATUS    "status"
-#define GLUSTERD_STORE_KEY_DEFRAG_OP            "rebalance_op"
-#define GLUSTERD_STORE_KEY_USERNAME             "username"
-#define GLUSTERD_STORE_KEY_PASSWORD             "password"
-#define GLUSTERD_STORE_KEY_PARENT_VOLNAME       "parent_volname"
-#define GLUSTERD_STORE_KEY_VOL_OP_VERSION       "op-version"
+#define UUID_SIZE 36
+#define VOLINFO_BUFFER_SIZE 4093
+#define GLUSTERD_STORE_UUID_KEY "UUID"
+
+#define GLUSTERD_STORE_KEY_VOL_TYPE "type"
+#define GLUSTERD_STORE_KEY_VOL_COUNT "count"
+#define GLUSTERD_STORE_KEY_VOL_STATUS "status"
+#define GLUSTERD_STORE_KEY_VOL_PORT "port"
+#define GLUSTERD_STORE_KEY_VOL_SUB_COUNT "sub_count"
+#define GLUSTERD_STORE_KEY_VOL_STRIPE_CNT "stripe_count"
+#define GLUSTERD_STORE_KEY_VOL_REPLICA_CNT "replica_count"
+#define GLUSTERD_STORE_KEY_VOL_DISPERSE_CNT "disperse_count"
+#define GLUSTERD_STORE_KEY_VOL_REDUNDANCY_CNT "redundancy_count"
+#define GLUSTERD_STORE_KEY_VOL_ARBITER_CNT "arbiter_count"
+#define GLUSTERD_STORE_KEY_VOL_THIN_ARBITER_CNT "thin_arbiter_count"
+#define GLUSTERD_STORE_KEY_VOL_BRICK "brick"
+#define GLUSTERD_STORE_KEY_VOL_TA_BRICK "ta-brick"
+#define GLUSTERD_STORE_KEY_VOL_VERSION "version"
+#define GLUSTERD_STORE_KEY_VOL_TRANSPORT "transport-type"
+#define GLUSTERD_STORE_KEY_VOL_ID "volume-id"
+#define GLUSTERD_STORE_KEY_VOL_RESTORED_SNAP "restored_from_snap"
+#define GLUSTERD_STORE_KEY_RB_STATUS "rb_status"
+#define GLUSTERD_STORE_KEY_RB_SRC_BRICK "rb_src"
+#define GLUSTERD_STORE_KEY_RB_DST_BRICK "rb_dst"
+#define GLUSTERD_STORE_KEY_RB_DST_PORT "rb_port"
+#define GLUSTERD_STORE_KEY_VOL_DEFRAG "rebalance_status"
+#define GLUSTERD_STORE_KEY_VOL_DEFRAG_STATUS "status"
+#define GLUSTERD_STORE_KEY_DEFRAG_OP "rebalance_op"
+#define GLUSTERD_STORE_KEY_USERNAME "username"
+#define GLUSTERD_STORE_KEY_PASSWORD "password"
+#define GLUSTERD_STORE_KEY_PARENT_VOLNAME "parent_volname"
+#define GLUSTERD_STORE_KEY_VOL_OP_VERSION "op-version"
 #define GLUSTERD_STORE_KEY_VOL_CLIENT_OP_VERSION "client-op-version"
-
-#define GLUSTERD_STORE_KEY_COLD_TYPE            "cold_type"
-#define GLUSTERD_STORE_KEY_COLD_COUNT           "cold_count"
-#define GLUSTERD_STORE_KEY_COLD_REPLICA_COUNT   "cold_replica_count"
-#define GLUSTERD_STORE_KEY_COLD_DISPERSE_COUNT  "cold_disperse_count"
-#define GLUSTERD_STORE_KEY_HOT_TYPE             "hot_type"
-#define GLUSTERD_STORE_KEY_HOT_COUNT            "hot_count"
-#define GLUSTERD_STORE_KEY_HOT_REPLICA_COUNT    "hot_replica_count"
-
-#define GLUSTERD_STORE_KEY_SNAP_NAME            "name"
-#define GLUSTERD_STORE_KEY_SNAP_ID              "snap-id"
-#define GLUSTERD_STORE_KEY_SNAP_DESC            "desc"
-#define GLUSTERD_STORE_KEY_SNAP_TIMESTAMP       "time-stamp"
-#define GLUSTERD_STORE_KEY_SNAP_STATUS          "status"
-#define GLUSTERD_STORE_KEY_SNAP_RESTORED        "snap-restored"
-#define GLUSTERD_STORE_KEY_SNAP_MAX_HARD_LIMIT  "snap-max-hard-limit"
-#define GLUSTERD_STORE_KEY_SNAP_AUTO_DELETE     "auto-delete"
-#define GLUSTERD_STORE_KEY_SNAP_MAX_SOFT_LIMIT  "snap-max-soft-limit"
-#define GLUSTERD_STORE_KEY_SNAPD_PORT           "snapd-port"
-#define GLUSTERD_STORE_KEY_SNAP_ACTIVATE        "snap-activate-on-create"
-
-#define GLUSTERD_STORE_KEY_BRICK_HOSTNAME       "hostname"
-#define GLUSTERD_STORE_KEY_BRICK_PATH           "path"
-#define GLUSTERD_STORE_KEY_BRICK_PORT           "listen-port"
-#define GLUSTERD_STORE_KEY_BRICK_RDMA_PORT      "rdma.listen-port"
+#define GLUSTERD_STORE_KEY_VOL_QUOTA_VERSION "quota-version"
+
+#define GLUSTERD_STORE_KEY_SNAP_NAME "name"
+#define GLUSTERD_STORE_KEY_SNAP_ID "snap-id"
+#define GLUSTERD_STORE_KEY_SNAP_DESC "desc"
+#define GLUSTERD_STORE_KEY_SNAP_TIMESTAMP "time-stamp"
+#define GLUSTERD_STORE_KEY_SNAP_STATUS "status"
+#define GLUSTERD_STORE_KEY_SNAP_RESTORED "snap-restored"
+#define GLUSTERD_STORE_KEY_SNAP_MAX_HARD_LIMIT "snap-max-hard-limit"
+#define GLUSTERD_STORE_KEY_SNAP_AUTO_DELETE "auto-delete"
+#define GLUSTERD_STORE_KEY_SNAP_MAX_SOFT_LIMIT "snap-max-soft-limit"
+#define GLUSTERD_STORE_KEY_SNAPD_PORT "snapd-port"
+#define GLUSTERD_STORE_KEY_SNAP_ACTIVATE "snap-activate-on-create"
+
+#define GLUSTERD_STORE_KEY_BRICK_HOSTNAME "hostname"
+#define GLUSTERD_STORE_KEY_BRICK_PATH "path"
+#define GLUSTERD_STORE_KEY_BRICK_REAL_PATH "real_path"
+#define GLUSTERD_STORE_KEY_BRICK_PORT "listen-port"
+#define GLUSTERD_STORE_KEY_BRICK_RDMA_PORT "rdma.listen-port"
 #define GLUSTERD_STORE_KEY_BRICK_DECOMMISSIONED "decommissioned"
-#define GLUSTERD_STORE_KEY_BRICK_VGNAME         "vg"
-#define GLUSTERD_STORE_KEY_BRICK_DEVICE_PATH    "device_path"
-#define GLUSTERD_STORE_KEY_BRICK_MOUNT_DIR      "mount_dir"
-#define GLUSTERD_STORE_KEY_BRICK_SNAP_STATUS    "snap-status"
-#define GLUSTERD_STORE_KEY_BRICK_FSTYPE         "fs-type"
-#define GLUSTERD_STORE_KEY_BRICK_MNTOPTS        "mnt-opts"
-#define GLUSTERD_STORE_KEY_BRICK_ID             "brick-id"
-
-#define GLUSTERD_STORE_KEY_PEER_UUID            "uuid"
-#define GLUSTERD_STORE_KEY_PEER_HOSTNAME        "hostname"
-#define GLUSTERD_STORE_KEY_PEER_STATE           "state"
+#define GLUSTERD_STORE_KEY_BRICK_VGNAME "vg"
+#define GLUSTERD_STORE_KEY_BRICK_DEVICE_PATH "device_path"
+#define GLUSTERD_STORE_KEY_BRICK_MOUNT_DIR "mount_dir"
+#define GLUSTERD_STORE_KEY_BRICK_SNAP_STATUS "snap-status"
+#define GLUSTERD_STORE_KEY_BRICK_FSTYPE "fs-type"
+#define GLUSTERD_STORE_KEY_BRICK_MNTOPTS "mnt-opts"
+#define GLUSTERD_STORE_KEY_BRICK_ID "brick-id"
+#define GLUSTERD_STORE_KEY_BRICK_FSID "brick-fsid"
+#define GLUSTERD_STORE_KEY_BRICK_UUID "uuid"
+
+#define GLUSTERD_STORE_KEY_PEER_UUID "uuid"
+#define GLUSTERD_STORE_KEY_PEER_HOSTNAME "hostname"
+#define GLUSTERD_STORE_KEY_PEER_STATE "state"
+#define GLUSTERD_STORE_KEY_VOL_CAPS "caps" /* left just for backward compat */
+
+#define GLUSTERD_STORE_KEY_VOL_DEFRAG_REB_FILES "rebalanced-files"
+#define GLUSTERD_STORE_KEY_VOL_DEFRAG_SIZE "size"
+#define GLUSTERD_STORE_KEY_VOL_DEFRAG_SCANNED "scanned"
+#define GLUSTERD_STORE_KEY_VOL_DEFRAG_FAILURES "failures"
+#define GLUSTERD_STORE_KEY_VOL_DEFRAG_SKIPPED "skipped"
+#define GLUSTERD_STORE_KEY_VOL_DEFRAG_RUN_TIME "run-time"
+
+#define GLUSTERD_STORE_KEY_VOL_MIGRATED_FILES "migrated-files"
+#define GLUSTERD_STORE_KEY_VOL_MIGRATED_SIZE "migration-size"
+#define GLUSTERD_STORE_KEY_VOL_MIGRATIONS_SCANNED "migration-scanned"
+#define GLUSTERD_STORE_KEY_VOL_MIGRATIONS_FAILURES "migration-failures"
+#define GLUSTERD_STORE_KEY_VOL_MIGRATIONS_SKIPPED "migration-skipped"
+#define GLUSTERD_STORE_KEY_VOL_MIGRATION_RUN_TIME "migration-run-time"
+
+#define GLUSTERD_STORE_KEY_GANESHA_GLOBAL "nfs-ganesha"
 
-#define GLUSTERD_STORE_KEY_VOL_CAPS             "caps"
+/*
+ * The structure is responsible for handling the parameter for writes into
+ * the buffer before it is finally written to the file. The writes will be
+ * of the form of key-value pairs.
+ */
+struct glusterd_volinfo_data_store_ {
+    gf_store_handle_t *shandle; /*Contains fd and path of the file */
+    int16_t buffer_len;
+    char key_check; /* flag to check if key is to be validated before write*/
+    char buffer[VOLINFO_BUFFER_SIZE];
+};
+typedef struct glusterd_volinfo_data_store_ glusterd_volinfo_data_store_t;
 
 int32_t
-glusterd_store_volinfo (glusterd_volinfo_t *volinfo, glusterd_volinfo_ver_ac_t ac);
+glusterd_store_volinfo(glusterd_volinfo_t *volinfo,
+                       glusterd_volinfo_ver_ac_t ac);
 
 int32_t
-glusterd_store_delete_volume (glusterd_volinfo_t *volinfo);
+glusterd_store_delete_volume(glusterd_volinfo_t *volinfo);
 
 int32_t
-glusterd_store_delete_snap (glusterd_snap_t *snap);
+glusterd_store_delete_snap(glusterd_snap_t *snap);
 
 int32_t
-glusterd_retrieve_uuid ();
+glusterd_retrieve_uuid();
 
 int32_t
-glusterd_store_peerinfo (glusterd_peerinfo_t *peerinfo);
+glusterd_store_peerinfo(glusterd_peerinfo_t *peerinfo);
 
 int32_t
-glusterd_store_delete_peerinfo (glusterd_peerinfo_t *peerinfo);
+glusterd_store_delete_peerinfo(glusterd_peerinfo_t *peerinfo);
 
 int32_t
-glusterd_store_delete_brick (glusterd_brickinfo_t *brickinfo,
-                             char *delete_path);
+glusterd_store_delete_brick(glusterd_brickinfo_t *brickinfo, char *delete_path);
 
 int32_t
-glusterd_restore ();
+glusterd_restore();
 
 void
-glusterd_perform_volinfo_version_action (glusterd_volinfo_t *volinfo,
-                                         glusterd_volinfo_ver_ac_t ac);
+glusterd_perform_volinfo_version_action(glusterd_volinfo_t *volinfo,
+                                        glusterd_volinfo_ver_ac_t ac);
 gf_boolean_t
-glusterd_store_is_valid_brickpath (char *volname, char *brick);
+glusterd_store_is_valid_brickpath(char *volname, char *brick);
 
 int32_t
-glusterd_store_perform_node_state_store (glusterd_volinfo_t *volinfo);
+glusterd_store_perform_node_state_store(glusterd_volinfo_t *volinfo);
+
+int
+glusterd_retrieve_op_version(xlator_t *this, int *op_version);
+
+int
+glusterd_retrieve_max_op_version(xlator_t *this, int *op_version);
 
 int
-glusterd_retrieve_op_version (xlator_t *this, int *op_version);
+glusterd_store_max_op_version(xlator_t *this);
 
 int
-glusterd_store_global_info (xlator_t *this);
+glusterd_store_global_info(xlator_t *this);
 
 int32_t
-glusterd_store_retrieve_options (xlator_t *this);
+glusterd_store_retrieve_options(xlator_t *this);
 
 int32_t
-glusterd_store_retrieve_bricks (glusterd_volinfo_t *volinfo);
+glusterd_store_retrieve_bricks(glusterd_volinfo_t *volinfo);
 
 int32_t
-glusterd_store_options (xlator_t *this, dict_t *opts);
+glusterd_store_options(xlator_t *this, dict_t *opts);
 
 void
-glusterd_replace_slash_with_hyphen (char *str);
-
-int32_t
-glusterd_store_perform_volume_store (glusterd_volinfo_t *volinfo);
+glusterd_replace_slash_with_hyphen(char *str);
 
 int32_t
-glusterd_store_create_quota_conf_sh_on_absence (glusterd_volinfo_t *volinfo);
+glusterd_store_create_quota_conf_sh_on_absence(glusterd_volinfo_t *volinfo);
 
 int
-glusterd_store_retrieve_quota_version (glusterd_volinfo_t *volinfo);
+glusterd_store_retrieve_quota_version(glusterd_volinfo_t *volinfo);
 
 int
-glusterd_store_save_quota_version_and_cksum (glusterd_volinfo_t *volinfo);
+glusterd_store_save_quota_version_and_cksum(glusterd_volinfo_t *volinfo);
 
 int32_t
-glusterd_store_snap (glusterd_snap_t *snap);
+glusterd_store_snap(glusterd_snap_t *snap);
 
 int32_t
-glusterd_store_update_missed_snaps ();
+glusterd_store_update_missed_snaps();
 
-glusterd_volinfo_t*
-glusterd_store_retrieve_volume (char *volname, glusterd_snap_t *snap);
+glusterd_volinfo_t *
+glusterd_store_retrieve_volume(char *volname, glusterd_snap_t *snap);
 
 int
-glusterd_restore_op_version (xlator_t *this);
+glusterd_restore_op_version(xlator_t *this);
+
+int32_t
+glusterd_quota_conf_write_header(int fd);
+
+int32_t
+glusterd_quota_conf_write_gfid(int fd, void *buf, char type);
+
+int32_t
+glusterd_recreate_vol_brick_mounts(xlator_t *this, glusterd_volinfo_t *volinfo);
 
 #endif
diff --git a/xlators/mgmt/glusterd/src/glusterd-svc-helper.c b/xlators/mgmt/glusterd/src/glusterd-svc-helper.c
index b4280dcb9ba..ca845903c4f 100644
--- a/xlators/mgmt/glusterd/src/glusterd-svc-helper.c
+++ b/xlators/mgmt/glusterd/src/glusterd-svc-helper.c
@@ -7,149 +7,1041 @@
    later), or the GNU General Public License, version 2 (GPLv2), in all
    cases as published by the Free Software Foundation.
 */
+#include <signal.h>
 
-#include "globals.h"
-#include "run.h"
+#include <glusterfs/globals.h>
+#include <glusterfs/run.h>
 #include "glusterd.h"
-#include "glusterfs.h"
+#include <glusterfs/glusterfs.h>
 #include "glusterd-utils.h"
 #include "glusterd-svc-mgmt.h"
 #include "glusterd-shd-svc.h"
 #include "glusterd-quotad-svc.h"
+#ifdef BUILD_GNFS
 #include "glusterd-nfs-svc.h"
+#endif
 #include "glusterd-bitd-svc.h"
+#include "glusterd-shd-svc-helper.h"
 #include "glusterd-scrub-svc.h"
+#include "glusterd-svc-helper.h"
+#include <glusterfs/syscall.h>
+#include "glusterd-snapshot-utils.h"
 
 int
-glusterd_svcs_reconfigure (glusterd_volinfo_t *volinfo)
+glusterd_svcs_reconfigure(glusterd_volinfo_t *volinfo)
 {
-        int              ret  = 0;
-        xlator_t        *this = THIS;
-        glusterd_conf_t *conf = NULL;
+    int ret = 0;
+    xlator_t *this = THIS;
+    glusterd_conf_t *conf = NULL;
+    char *svc_name = NULL;
 
-        GF_ASSERT (this);
+    GF_ASSERT(this);
 
-        conf = this->private;
-        GF_ASSERT (conf);
+    conf = this->private;
+    GF_ASSERT(conf);
 
-        ret = glusterd_nfssvc_reconfigure ();
+#ifdef BUILD_GNFS
+    svc_name = "nfs";
+    ret = glusterd_nfssvc_reconfigure();
+    if (ret)
+        goto out;
+#endif
+    svc_name = "self-heald";
+    if (volinfo) {
+        ret = glusterd_shdsvc_reconfigure(volinfo);
         if (ret)
-                goto out;
+            goto out;
+    }
 
-        if (volinfo && !glusterd_is_shd_compatible_volume (volinfo)) {
-                ; /* Do nothing */
-        } else {
-                ret = glusterd_shdsvc_reconfigure ();
-                if (ret)
-                        goto out;
-        }
-        if (conf->op_version == GD_OP_VERSION_MIN)
-                goto out;
+    if (conf->op_version == GD_OP_VERSION_MIN)
+        goto out;
 
-        if (volinfo && !glusterd_is_volume_quota_enabled (volinfo))
-                goto out;
+    svc_name = "quotad";
+    ret = glusterd_quotadsvc_reconfigure();
+    if (ret)
+        goto out;
 
-        ret = glusterd_quotadsvc_reconfigure ();
-        if (ret)
-                goto out;
+    svc_name = "bitd";
+    ret = glusterd_bitdsvc_reconfigure();
+    if (ret)
+        goto out;
 
-        ret = glusterd_bitdsvc_reconfigure ();
-        if (ret)
-                goto out;
+    svc_name = "scrubber";
+    ret = glusterd_scrubsvc_reconfigure();
+out:
+    if (ret && svc_name)
+        gf_event(EVENT_SVC_RECONFIGURE_FAILED, "svc_name=%s", svc_name);
+    return ret;
+}
 
-        ret = glusterd_scrubsvc_reconfigure ();
+int
+glusterd_svcs_stop(glusterd_volinfo_t *volinfo)
+{
+    int ret = 0;
+    xlator_t *this = NULL;
+    glusterd_conf_t *priv = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+
+    priv = this->private;
+    GF_ASSERT(priv);
+
+#ifdef BUILD_GNFS
+    ret = priv->nfs_svc.stop(&(priv->nfs_svc), SIGKILL);
+    if (ret)
+        goto out;
+#endif
+    ret = priv->quotad_svc.stop(&(priv->quotad_svc), SIGTERM);
+    if (ret)
+        goto out;
+
+    if (volinfo) {
+        ret = volinfo->shd.svc.stop(&(volinfo->shd.svc), SIGTERM);
         if (ret)
-                goto out;
+            goto out;
+    }
+
+    ret = priv->bitd_svc.stop(&(priv->bitd_svc), SIGTERM);
+    if (ret)
+        goto out;
+
+    ret = priv->scrub_svc.stop(&(priv->scrub_svc), SIGTERM);
+
 out:
-        return ret;
+    return ret;
 }
 
 int
-glusterd_svcs_stop ()
+glusterd_svcs_manager(glusterd_volinfo_t *volinfo)
 {
-        int              ret  = 0;
-        xlator_t        *this = NULL;
-        glusterd_conf_t *priv = NULL;
+    int ret = 0;
+    xlator_t *this = THIS;
+    glusterd_conf_t *conf = NULL;
 
-        this = THIS;
-        GF_ASSERT (this);
+    GF_ASSERT(this);
 
-        priv = this->private;
-        GF_ASSERT (priv);
+    conf = this->private;
+    GF_ASSERT(conf);
 
-        ret = glusterd_svc_stop (&(priv->nfs_svc), SIGKILL);
-        if (ret)
-                goto out;
+    if (volinfo && volinfo->is_snap_volume)
+        return 0;
 
-        ret = glusterd_svc_stop (&(priv->shd_svc), SIGTERM);
-        if (ret)
-                goto out;
+#if BUILD_GNFS
+    ret = conf->nfs_svc.manager(&(conf->nfs_svc), NULL, PROC_START_NO_WAIT);
+    if (ret)
+        goto out;
+#endif
+    if (conf->op_version == GD_OP_VERSION_MIN)
+        goto out;
 
-        ret = glusterd_svc_stop (&(priv->quotad_svc), SIGTERM);
-        if (ret)
-                goto out;
+    ret = conf->quotad_svc.manager(&(conf->quotad_svc), volinfo,
+                                   PROC_START_NO_WAIT);
+    if (ret == -EINVAL)
+        ret = 0;
+    if (ret)
+        goto out;
+
+    ret = conf->bitd_svc.manager(&(conf->bitd_svc), NULL, PROC_START_NO_WAIT);
+    if (ret == -EINVAL)
+        ret = 0;
+    if (ret)
+        goto out;
 
-        ret = glusterd_svc_stop (&(priv->bitd_svc), SIGTERM);
+    if (volinfo) {
+        ret = volinfo->shd.svc.manager(&(volinfo->shd.svc), volinfo,
+                                       PROC_START_NO_WAIT);
+        if (ret == -EINVAL)
+            ret = 0;
         if (ret)
-                goto out;
+            goto out;
+    }
+
+    ret = conf->scrub_svc.manager(&(conf->scrub_svc), NULL, PROC_START_NO_WAIT);
+    if (ret == -EINVAL)
+        ret = 0;
+out:
+    return ret;
+}
+
+int
+glusterd_svc_check_volfile_identical(char *svc_name,
+                                     glusterd_graph_builder_t builder,
+                                     gf_boolean_t *identical)
+{
+    char orgvol[PATH_MAX] = {
+        0,
+    };
+    char *tmpvol = NULL;
+    glusterd_conf_t *conf = NULL;
+    xlator_t *this = NULL;
+    int ret = -1;
+    int need_unlink = 0;
+    int tmp_fd = -1;
 
-        ret = glusterd_svc_stop (&(priv->scrub_svc), SIGTERM);
+    this = THIS;
 
+    GF_ASSERT(this);
+    GF_ASSERT(identical);
+    conf = this->private;
+
+    glusterd_svc_build_volfile_path(svc_name, conf->workdir, orgvol,
+                                    sizeof(orgvol));
+
+    ret = gf_asprintf(&tmpvol, "/tmp/g%s-XXXXXX", svc_name);
+    if (ret < 0) {
+        goto out;
+    }
+
+    /* coverity[SECURE_TEMP] mkstemp uses 0600 as the mode and is safe */
+    tmp_fd = mkstemp(tmpvol);
+    if (tmp_fd < 0) {
+        gf_msg(this->name, GF_LOG_WARNING, errno, GD_MSG_FILE_OP_FAILED,
+               "Unable to create temp file"
+               " %s:(%s)",
+               tmpvol, strerror(errno));
+        ret = -1;
+        goto out;
+    }
+
+    need_unlink = 1;
+
+    ret = glusterd_create_global_volfile(builder, tmpvol, NULL);
+    if (ret)
+        goto out;
+
+    ret = glusterd_check_files_identical(orgvol, tmpvol, identical);
 out:
-        return ret;
+    if (need_unlink)
+        sys_unlink(tmpvol);
+
+    if (tmpvol != NULL)
+        GF_FREE(tmpvol);
+
+    if (tmp_fd >= 0)
+        sys_close(tmp_fd);
+
+    return ret;
 }
 
 int
-glusterd_svcs_manager (glusterd_volinfo_t *volinfo)
+glusterd_svc_check_topology_identical(char *svc_name,
+                                      glusterd_graph_builder_t builder,
+                                      gf_boolean_t *identical)
 {
-        int              ret  = 0;
-        xlator_t        *this = THIS;
-        glusterd_conf_t *conf = NULL;
+    char orgvol[PATH_MAX] = {
+        0,
+    };
+    char *tmpvol = NULL;
+    glusterd_conf_t *conf = NULL;
+    xlator_t *this = THIS;
+    int ret = -1;
+    int tmpclean = 0;
+    int tmpfd = -1;
 
-        GF_ASSERT (this);
+    if ((!identical) || (!this) || (!this->private)) {
+        gf_smsg(THIS->name, GF_LOG_ERROR, errno, GD_MSG_INVALID_ARGUMENT, NULL);
+        goto out;
+    }
 
-        conf = this->private;
-        GF_ASSERT (conf);
+    conf = this->private;
+    GF_VALIDATE_OR_GOTO(this->name, conf, out);
 
-        if (volinfo && volinfo->is_snap_volume)
-                return 0;
+    /* Fetch the original volfile */
+    glusterd_svc_build_volfile_path(svc_name, conf->workdir, orgvol,
+                                    sizeof(orgvol));
 
-        ret = conf->nfs_svc.manager (&(conf->nfs_svc), NULL,
-                                     PROC_START_NO_WAIT);
-        if (ret)
-                goto out;
+    /* Create the temporary volfile */
+    ret = gf_asprintf(&tmpvol, "/tmp/g%s-XXXXXX", svc_name);
+    if (ret < 0) {
+        goto out;
+    }
 
-        ret = conf->shd_svc.manager (&(conf->shd_svc), volinfo,
-                                     PROC_START_NO_WAIT);
-        if (ret == -EINVAL)
-                ret = 0;
-        if (ret)
-                goto out;
+    /* coverity[SECURE_TEMP] mkstemp uses 0600 as the mode and is safe */
+    tmpfd = mkstemp(tmpvol);
+    if (tmpfd < 0) {
+        gf_msg(this->name, GF_LOG_WARNING, errno, GD_MSG_FILE_OP_FAILED,
+               "Unable to create temp file"
+               " %s:(%s)",
+               tmpvol, strerror(errno));
+        ret = -1;
+        goto out;
+    }
 
-        if (conf->op_version == GD_OP_VERSION_MIN)
-                goto out;
+    tmpclean = 1; /* SET the flag to unlink() tmpfile */
 
-        ret = conf->quotad_svc.manager (&(conf->quotad_svc), volinfo,
-                                        PROC_START_NO_WAIT);
-        if (ret == -EINVAL)
-                ret = 0;
-        if (ret)
+    ret = glusterd_create_global_volfile(builder, tmpvol, NULL);
+    if (ret)
+        goto out;
+
+    /* Compare the topology of volfiles */
+    ret = glusterd_check_topology_identical(orgvol, tmpvol, identical);
+out:
+    if (tmpfd >= 0)
+        sys_close(tmpfd);
+    if (tmpclean)
+        sys_unlink(tmpvol);
+    if (tmpvol != NULL)
+        GF_FREE(tmpvol);
+    return ret;
+}
+
+int
+glusterd_volume_svc_check_volfile_identical(
+    char *svc_name, dict_t *mode_dict, glusterd_volinfo_t *volinfo,
+    glusterd_vol_graph_builder_t builder, gf_boolean_t *identical)
+{
+    char orgvol[PATH_MAX] = {
+        0,
+    };
+    char *tmpvol = NULL;
+    xlator_t *this = NULL;
+    int ret = -1;
+    int need_unlink = 0;
+    int tmp_fd = -1;
+
+    this = THIS;
+
+    GF_VALIDATE_OR_GOTO("glusterd", this, out);
+    GF_VALIDATE_OR_GOTO(this->name, identical, out);
+
+    /* This builds volfile for volume level dameons */
+    glusterd_volume_svc_build_volfile_path(svc_name, volinfo, orgvol,
+                                           sizeof(orgvol));
+
+    ret = gf_asprintf(&tmpvol, "/tmp/g%s-XXXXXX", svc_name);
+    if (ret < 0) {
+        goto out;
+    }
+
+    /* coverity[SECURE_TEMP] mkstemp uses 0600 as the mode and is safe */
+    tmp_fd = mkstemp(tmpvol);
+    if (tmp_fd < 0) {
+        gf_msg(this->name, GF_LOG_WARNING, errno, GD_MSG_FILE_OP_FAILED,
+               "Unable to create temp file"
+               " %s:(%s)",
+               tmpvol, strerror(errno));
+        ret = -1;
+        goto out;
+    }
+
+    need_unlink = 1;
+
+    ret = builder(volinfo, tmpvol, mode_dict);
+    if (ret)
+        goto out;
+
+    ret = glusterd_check_files_identical(orgvol, tmpvol, identical);
+out:
+    if (need_unlink)
+        sys_unlink(tmpvol);
+
+    if (tmpvol != NULL)
+        GF_FREE(tmpvol);
+
+    if (tmp_fd >= 0)
+        sys_close(tmp_fd);
+
+    return ret;
+}
+
+int
+glusterd_volume_svc_check_topology_identical(
+    char *svc_name, dict_t *mode_dict, glusterd_volinfo_t *volinfo,
+    glusterd_vol_graph_builder_t builder, gf_boolean_t *identical)
+{
+    char orgvol[PATH_MAX] = {
+        0,
+    };
+    char *tmpvol = NULL;
+    glusterd_conf_t *conf = NULL;
+    xlator_t *this = THIS;
+    int ret = -1;
+    int tmpclean = 0;
+    int tmpfd = -1;
+
+    if ((!identical) || (!this) || (!this->private)) {
+        gf_smsg(THIS->name, GF_LOG_ERROR, errno, GD_MSG_INVALID_ARGUMENT, NULL);
+        goto out;
+    }
+
+    conf = this->private;
+    GF_VALIDATE_OR_GOTO(this->name, conf, out);
+
+    /* This builds volfile for volume level dameons */
+    glusterd_volume_svc_build_volfile_path(svc_name, volinfo, orgvol,
+                                           sizeof(orgvol));
+    /* Create the temporary volfile */
+    ret = gf_asprintf(&tmpvol, "/tmp/g%s-XXXXXX", svc_name);
+    if (ret < 0) {
+        goto out;
+    }
+
+    /* coverity[SECURE_TEMP] mkstemp uses 0600 as the mode and is safe */
+    tmpfd = mkstemp(tmpvol);
+    if (tmpfd < 0) {
+        gf_msg(this->name, GF_LOG_WARNING, errno, GD_MSG_FILE_OP_FAILED,
+               "Unable to create temp file"
+               " %s:(%s)",
+               tmpvol, strerror(errno));
+        ret = -1;
+        goto out;
+    }
+
+    tmpclean = 1; /* SET the flag to unlink() tmpfile */
+
+    ret = builder(volinfo, tmpvol, mode_dict);
+    if (ret)
+        goto out;
+
+    /* Compare the topology of volfiles */
+    ret = glusterd_check_topology_identical(orgvol, tmpvol, identical);
+out:
+    if (tmpfd >= 0)
+        sys_close(tmpfd);
+    if (tmpclean)
+        sys_unlink(tmpvol);
+    if (tmpvol != NULL)
+        GF_FREE(tmpvol);
+    return ret;
+}
+
+gf_boolean_t
+glusterd_is_svcproc_attachable(glusterd_svc_proc_t *svc_proc)
+{
+    int pid = -1;
+    glusterd_svc_t *parent_svc = NULL;
+
+    if (!svc_proc)
+        return _gf_false;
+
+    if (svc_proc->status == GF_SVC_STARTING)
+        return _gf_true;
+
+    if (svc_proc->status == GF_SVC_STARTED ||
+        svc_proc->status == GF_SVC_DISCONNECTED) {
+        parent_svc = cds_list_entry(svc_proc->svcs.next, glusterd_svc_t,
+                                    mux_svc);
+        if (parent_svc && gf_is_service_running(parent_svc->proc.pidfile, &pid))
+            return _gf_true;
+    }
+
+    if (svc_proc->status == GF_SVC_DIED || svc_proc->status == GF_SVC_STOPPING)
+        return _gf_false;
+
+    return _gf_false;
+}
+
+void *
+__gf_find_compatible_svc(gd_node_type daemon)
+{
+    glusterd_svc_proc_t *svc_proc = NULL;
+    struct cds_list_head *svc_procs = NULL;
+    glusterd_conf_t *conf = NULL;
+
+    conf = THIS->private;
+    GF_VALIDATE_OR_GOTO("glusterd", conf, out);
+
+    switch (daemon) {
+        case GD_NODE_SHD: {
+            svc_procs = &conf->shd_procs;
+            if (!svc_procs)
                 goto out;
+        } break;
+        default:
+            /* Add support for other client daemons here */
+            goto out;
+    }
 
-        ret = conf->bitd_svc.manager (&(conf->bitd_svc), volinfo,
-                                      PROC_START_NO_WAIT);
-        if (ret == -EINVAL)
-                ret = 0;
-        if (ret)
+    cds_list_for_each_entry(svc_proc, svc_procs, svc_proc_list)
+    {
+        if (glusterd_is_svcproc_attachable(svc_proc))
+            return (void *)svc_proc;
+        /*
+         * Logic to select one process goes here. Currently there is only one
+         * shd_proc. So selecting the first one;
+         */
+    }
+out:
+    return NULL;
+}
+
+glusterd_svc_proc_t *
+glusterd_svcprocess_new()
+{
+    glusterd_svc_proc_t *new_svcprocess = NULL;
+
+    new_svcprocess = GF_CALLOC(1, sizeof(*new_svcprocess),
+                               gf_gld_mt_glusterd_svc_proc_t);
+
+    if (!new_svcprocess)
+        return NULL;
+
+    CDS_INIT_LIST_HEAD(&new_svcprocess->svc_proc_list);
+    CDS_INIT_LIST_HEAD(&new_svcprocess->svcs);
+    new_svcprocess->notify = glusterd_muxsvc_common_rpc_notify;
+    new_svcprocess->status = GF_SVC_STARTING;
+    return new_svcprocess;
+}
+
+int
+glusterd_shd_svc_mux_init(glusterd_volinfo_t *volinfo, glusterd_svc_t *svc)
+{
+    int ret = -1;
+    glusterd_svc_proc_t *mux_proc = NULL;
+    glusterd_conn_t *mux_conn = NULL;
+    glusterd_conf_t *conf = NULL;
+    glusterd_svc_t *parent_svc = NULL;
+    int pid = -1;
+    gf_boolean_t stop_daemon = _gf_false;
+    char pidfile[PATH_MAX] = {
+        0,
+    };
+
+    GF_VALIDATE_OR_GOTO("glusterd", svc, out);
+    GF_VALIDATE_OR_GOTO("glusterd", volinfo, out);
+    conf = THIS->private;
+    GF_VALIDATE_OR_GOTO("glusterd", conf, out);
+    GF_VALIDATE_OR_GOTO("glusterd", svc, out);
+
+    pthread_mutex_lock(&conf->attach_lock);
+    {
+        if (svc->inited && !glusterd_proc_is_running(&(svc->proc))) {
+            /* This is the case when shd process was abnormally killed */
+            pthread_mutex_unlock(&conf->attach_lock);
+            glusterd_shd_svcproc_cleanup(&volinfo->shd);
+            pthread_mutex_lock(&conf->attach_lock);
+        }
+
+        if (!svc->inited) {
+            glusterd_svc_build_shd_pidfile(volinfo, pidfile, sizeof(pidfile));
+            ret = snprintf(svc->proc.name, sizeof(svc->proc.name), "%s",
+                           "glustershd");
+            if (ret < 0)
+                goto unlock;
+
+            ret = snprintf(svc->proc.pidfile, sizeof(svc->proc.pidfile), "%s",
+                           pidfile);
+            if (ret < 0)
+                goto unlock;
+
+            if (gf_is_service_running(pidfile, &pid)) {
+                /* Just connect is required, but we don't know what happens
+                 * during the disconnect. So better to reattach.
+                 */
+                mux_proc = __gf_find_compatible_svc_from_pid(GD_NODE_SHD, pid);
+            }
+
+            if (!mux_proc) {
+                if (pid != -1 && sys_access(pidfile, R_OK) == 0) {
+                    /* stale pid file, stop and unlink it. This has to be
+                     * done outside the attach_lock.
+                     */
+                    stop_daemon = _gf_true;
+                }
+                mux_proc = __gf_find_compatible_svc(GD_NODE_SHD);
+            }
+            if (mux_proc) {
+                /* Take first entry from the process */
+                parent_svc = cds_list_entry(mux_proc->svcs.next, glusterd_svc_t,
+                                            mux_svc);
+                mux_conn = &parent_svc->conn;
+                if (volinfo)
+                    volinfo->shd.attached = _gf_true;
+            } else {
+                mux_proc = glusterd_svcprocess_new();
+                if (!mux_proc) {
+                    ret = -1;
+                    goto unlock;
+                }
+                cds_list_add_tail(&mux_proc->svc_proc_list, &conf->shd_procs);
+            }
+            svc->svc_proc = mux_proc;
+            cds_list_del_init(&svc->mux_svc);
+            cds_list_add_tail(&svc->mux_svc, &mux_proc->svcs);
+            ret = glusterd_shdsvc_init(volinfo, mux_conn, mux_proc);
+            if (ret) {
+                pthread_mutex_unlock(&conf->attach_lock);
+                gf_msg(THIS->name, GF_LOG_ERROR, 0, GD_MSG_FAILED_INIT_SHDSVC,
+                       "Failed to init shd "
+                       "service");
                 goto out;
+            }
+            gf_msg_debug(THIS->name, 0, "shd service initialized");
+            svc->inited = _gf_true;
+        }
+        ret = 0;
+    }
+unlock:
+    pthread_mutex_unlock(&conf->attach_lock);
+out:
+    if (stop_daemon) {
+        glusterd_proc_stop(&svc->proc, SIGTERM, PROC_STOP_FORCE);
+        glusterd_unlink_file(pidfile);
+    }
+    return ret;
+}
 
-        ret = conf->scrub_svc.manager (&(conf->scrub_svc), NULL,
-                                       PROC_START_NO_WAIT);
-        if (ret == -EINVAL)
-                ret = 0;
+void *
+__gf_find_compatible_svc_from_pid(gd_node_type daemon, pid_t pid)
+{
+    glusterd_svc_proc_t *svc_proc = NULL;
+    struct cds_list_head *svc_procs = NULL;
+    glusterd_svc_t *svc = NULL;
+    pid_t mux_pid = -1;
+    glusterd_conf_t *conf = NULL;
+
+    conf = THIS->private;
+    if (!conf)
+        return NULL;
+
+    switch (daemon) {
+        case GD_NODE_SHD: {
+            svc_procs = &conf->shd_procs;
+            if (!svc_procs)
+                return NULL;
+        } break;
+        default:
+            /* Add support for other client daemons here */
+            return NULL;
+    }
+
+    cds_list_for_each_entry(svc_proc, svc_procs, svc_proc_list)
+    {
+        cds_list_for_each_entry(svc, &svc_proc->svcs, mux_svc)
+        {
+            if (gf_is_service_running(svc->proc.pidfile, &mux_pid)) {
+                if (mux_pid == pid &&
+                    glusterd_is_svcproc_attachable(svc_proc)) {
+                    /*TODO
+                     * inefficient loop, but at the moment, there is only
+                     * one shd.
+                     */
+                    return svc_proc;
+                }
+            }
+        }
+    }
+    return NULL;
+}
+
+static int32_t
+my_callback(struct rpc_req *req, struct iovec *iov, int count, void *v_frame)
+{
+    call_frame_t *frame = v_frame;
+    xlator_t *this = NULL;
+    glusterd_conf_t *conf = NULL;
+
+    GF_VALIDATE_OR_GOTO("glusterd", frame, out);
+    this = frame->this;
+    GF_VALIDATE_OR_GOTO("glusterd", this, out);
+    conf = this->private;
+    GF_VALIDATE_OR_GOTO(this->name, conf, out);
+
+    if (GF_ATOMIC_DEC(conf->blockers) == 0) {
+        synccond_broadcast(&conf->cond_blockers);
+    }
+
+    STACK_DESTROY(frame->root);
+out:
+    return 0;
+}
+
+static int32_t
+glusterd_svc_attach_cbk(struct rpc_req *req, struct iovec *iov, int count,
+                        void *v_frame)
+{
+    call_frame_t *frame = v_frame;
+    glusterd_volinfo_t *volinfo = NULL;
+    glusterd_shdsvc_t *shd = NULL;
+    glusterd_svc_t *svc = frame->cookie;
+    glusterd_conf_t *conf = NULL;
+    int *flag = (int *)frame->local;
+    xlator_t *this = THIS;
+    int ret = -1;
+    gf_getspec_rsp rsp = {
+        0,
+    };
+
+    GF_VALIDATE_OR_GOTO("glusterd", this, out);
+    conf = this->private;
+    GF_VALIDATE_OR_GOTO("glusterd", conf, out);
+    GF_VALIDATE_OR_GOTO("glusterd", frame, out);
+    GF_VALIDATE_OR_GOTO("glusterd", svc, out);
+
+    frame->local = NULL;
+    frame->cookie = NULL;
+
+    if (!strcmp(svc->name, "glustershd")) {
+        /* Get volinfo->shd from svc object */
+        shd = cds_list_entry(svc, glusterd_shdsvc_t, svc);
+        if (!shd) {
+            gf_msg("glusterd", GF_LOG_ERROR, 0, GD_MSG_SHD_OBJ_GET_FAIL,
+                   "Failed to get shd object "
+                   "from shd service");
+            goto out;
+        }
+
+        /* Get volinfo from shd */
+        volinfo = cds_list_entry(shd, glusterd_volinfo_t, shd);
+        if (!volinfo) {
+            gf_msg("glusterd", GF_LOG_ERROR, 0, GD_MSG_VOLINFO_GET_FAIL,
+                   "Failed to get volinfo from "
+                   "from shd");
+            goto out;
+        }
+    }
+
+    if (!iov) {
+        gf_msg(frame->this->name, GF_LOG_ERROR, 0, GD_MSG_REQ_DECODE_FAIL,
+               "iov is NULL");
+        ret = -1;
+        goto out;
+    }
+
+    ret = xdr_to_generic(*iov, &rsp, (xdrproc_t)xdr_gf_getspec_rsp);
+    if (ret < 0) {
+        gf_msg(frame->this->name, GF_LOG_ERROR, 0, GD_MSG_REQ_DECODE_FAIL,
+               "XDR decoding error");
+        ret = -1;
+        goto out;
+    }
+
+    if (rsp.op_ret == 0) {
+        svc->online = _gf_true;
+        gf_msg(this->name, GF_LOG_INFO, 0, GD_MSG_SVC_ATTACH_FAIL,
+               "svc %s of volume %s attached successfully to pid %d", svc->name,
+               volinfo->volname, glusterd_proc_get_pid(&svc->proc));
+    } else {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_SVC_ATTACH_FAIL,
+               "svc %s of volume %s failed to attach to pid %d", svc->name,
+               volinfo->volname, glusterd_proc_get_pid(&svc->proc));
+        if (!strcmp(svc->name, "glustershd")) {
+            glusterd_shd_svcproc_cleanup(&volinfo->shd);
+        }
+    }
+out:
+    if (flag) {
+        GF_FREE(flag);
+    }
+
+    if (volinfo)
+        glusterd_volinfo_unref(volinfo);
+
+    if (GF_ATOMIC_DEC(conf->blockers) == 0) {
+        synccond_broadcast(&conf->cond_blockers);
+    }
+    STACK_DESTROY(frame->root);
+    return 0;
+}
+
+extern size_t
+build_volfile_path(char *volume_id, char *path, size_t path_len,
+                   char *trusted_str, dict_t *dict);
+
+int
+__glusterd_send_svc_configure_req(glusterd_svc_t *svc, int flags,
+                                  struct rpc_clnt *rpc, char *volfile_id,
+                                  int op)
+{
+    int ret = -1;
+    struct iobuf *iobuf = NULL;
+    struct iobref *iobref = NULL;
+    struct iovec iov = {
+        0,
+    };
+    char path[PATH_MAX] = {
+        '\0',
+    };
+    struct stat stbuf = {
+        0,
+    };
+    int32_t spec_fd = -1;
+    size_t file_len = -1;
+    char *volfile_content = NULL;
+    ssize_t req_size = 0;
+    call_frame_t *frame = NULL;
+    gd1_mgmt_brick_op_req brick_req;
+    dict_t *dict = NULL;
+    void *req = &brick_req;
+    void *errlbl = &&err;
+    struct rpc_clnt_connection *conn;
+    xlator_t *this = THIS;
+    glusterd_conf_t *conf = THIS->private;
+    extern struct rpc_clnt_program gd_brick_prog;
+    fop_cbk_fn_t cbkfn = my_callback;
+
+    if (!rpc) {
+        gf_msg("glusterd", GF_LOG_ERROR, 0, GD_MSG_PARAM_NULL,
+               "called with null rpc");
+        return -1;
+    }
+
+    conn = &rpc->conn;
+    if (!conn->connected || conn->disconnected) {
+        gf_msg(this->name, GF_LOG_INFO, 0, GD_MSG_CONNECT_RETURNED,
+               "not connected yet");
+        return -1;
+    }
+
+    brick_req.op = op;
+    brick_req.name = volfile_id;
+    brick_req.input.input_val = NULL;
+    brick_req.input.input_len = 0;
+    brick_req.dict.dict_val = NULL;
+    brick_req.dict.dict_len = 0;
+
+    frame = create_frame(this, this->ctx->pool);
+    if (!frame) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_FRAME_CREATE_FAIL,
+                NULL);
+        goto *errlbl;
+    }
+
+    if (op == GLUSTERD_SVC_ATTACH) {
+        dict = dict_new();
+        if (!dict) {
+            gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_CREATE_FAIL,
+                    NULL);
+            ret = -ENOMEM;
+            goto *errlbl;
+        }
+
+        (void)build_volfile_path(volfile_id, path, sizeof(path), NULL, dict);
+
+        ret = sys_stat(path, &stbuf);
+        if (ret < 0) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_SVC_ATTACH_FAIL,
+                   "Unable to stat %s (%s)", path, strerror(errno));
+            ret = -EINVAL;
+            goto *errlbl;
+        }
+
+        file_len = stbuf.st_size;
+        volfile_content = GF_MALLOC(file_len + 1, gf_common_mt_char);
+        if (!volfile_content) {
+            gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_NO_MEMORY, NULL);
+            ret = -ENOMEM;
+            goto *errlbl;
+        }
+        spec_fd = open(path, O_RDONLY);
+        if (spec_fd < 0) {
+            gf_msg(THIS->name, GF_LOG_WARNING, 0, GD_MSG_SVC_ATTACH_FAIL,
+                   "failed to read volfile %s", path);
+            ret = -EIO;
+            goto *errlbl;
+        }
+        ret = sys_read(spec_fd, volfile_content, file_len);
+        if (ret == file_len) {
+            brick_req.input.input_val = volfile_content;
+            brick_req.input.input_len = file_len;
+        } else {
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_SVC_ATTACH_FAIL,
+                   "read failed on path %s. File size=%" GF_PRI_SIZET
+                   "read size=%d",
+                   path, file_len, ret);
+            ret = -EIO;
+            goto *errlbl;
+        }
+        if (dict->count > 0) {
+            ret = dict_allocate_and_serialize(dict, &brick_req.dict.dict_val,
+                                              &brick_req.dict.dict_len);
+            if (ret) {
+                gf_smsg(this->name, GF_LOG_ERROR, errno,
+                        GD_MSG_DICT_ALLOC_AND_SERL_LENGTH_GET_FAIL, NULL);
+                goto *errlbl;
+            }
+        }
+
+        frame->cookie = svc;
+        frame->local = GF_CALLOC(1, sizeof(int), gf_gld_mt_int);
+        *((int *)frame->local) = flags;
+        cbkfn = glusterd_svc_attach_cbk;
+    }
+
+    req_size = xdr_sizeof((xdrproc_t)xdr_gd1_mgmt_brick_op_req, req);
+    iobuf = iobuf_get2(rpc->ctx->iobuf_pool, req_size);
+    if (!iobuf) {
+        goto *errlbl;
+    }
+    errlbl = &&maybe_free_iobuf;
 
+    iov.iov_base = iobuf->ptr;
+    iov.iov_len = iobuf_pagesize(iobuf);
+
+    iobref = iobref_new();
+    if (!iobref) {
+        goto *errlbl;
+    }
+    errlbl = &&free_iobref;
+
+    iobref_add(iobref, iobuf);
+    /*
+     * Drop our reference to the iobuf.  The iobref should already have
+     * one after iobref_add, so when we unref that we'll free the iobuf as
+     * well.  This allows us to pass just the iobref as frame->local.
+     */
+    iobuf_unref(iobuf);
+    /* Set the pointer to null so we don't free it on a later error. */
+    iobuf = NULL;
+
+    /* Create the xdr payload */
+    ret = xdr_serialize_generic(iov, req, (xdrproc_t)xdr_gd1_mgmt_brick_op_req);
+    if (ret == -1) {
+        goto *errlbl;
+    }
+    iov.iov_len = ret;
+
+    /* Send the msg */
+    GF_ATOMIC_INC(conf->blockers);
+    ret = rpc_clnt_submit(rpc, &gd_brick_prog, op, cbkfn, &iov, 1, NULL, 0,
+                          iobref, frame, NULL, 0, NULL, 0, NULL);
+    if (dict)
+        dict_unref(dict);
+    GF_FREE(volfile_content);
+    if (spec_fd >= 0)
+        sys_close(spec_fd);
+    return ret;
+
+free_iobref:
+    iobref_unref(iobref);
+maybe_free_iobuf:
+    if (iobuf) {
+        iobuf_unref(iobuf);
+    }
+err:
+    if (dict)
+        dict_unref(dict);
+    if (brick_req.dict.dict_val)
+        GF_FREE(brick_req.dict.dict_val);
+
+    GF_FREE(volfile_content);
+    if (spec_fd >= 0)
+        sys_close(spec_fd);
+    if (frame)
+        STACK_DESTROY(frame->root);
+    return -1;
+}
+
+int
+glusterd_attach_svc(glusterd_svc_t *svc, glusterd_volinfo_t *volinfo, int flags)
+{
+    glusterd_conf_t *conf = THIS->private;
+    int ret = -1;
+    int tries;
+    rpc_clnt_t *rpc = NULL;
+
+    GF_VALIDATE_OR_GOTO("glusterd", conf, out);
+    GF_VALIDATE_OR_GOTO("glusterd", svc, out);
+    GF_VALIDATE_OR_GOTO("glusterd", volinfo, out);
+
+    gf_msg("glusterd", GF_LOG_INFO, 0, GD_MSG_ATTACH_INFO,
+           "adding svc %s (volume=%s) to existing "
+           "process with pid %d",
+           svc->name, volinfo->volname, glusterd_proc_get_pid(&svc->proc));
+
+    rpc = rpc_clnt_ref(svc->conn.rpc);
+    for (tries = 15; tries > 0; --tries) {
+        /* There might be a case that the volume for which we're attempting to
+         * attach a shd svc might become stale and in the process of deletion.
+         * Given that the volinfo object is being already passed here before
+         * that sequence of operation has happened we might be operating on a
+         * stale volume. At every sync task switch we should check for existance
+         * of the volume now
+         */
+        if (!glusterd_volume_exists(volinfo->volname)) {
+            gf_msg(THIS->name, GF_LOG_INFO, 0, GD_MSG_SVC_ATTACH_FAIL,
+                   "Volume %s "
+                   " is marked as stale, not attempting further shd svc attach "
+                   "attempts",
+                   volinfo->volname);
+            ret = 0;
+            goto out;
+        }
+        if (rpc) {
+            pthread_mutex_lock(&conf->attach_lock);
+            {
+                ret = __glusterd_send_svc_configure_req(
+                    svc, flags, rpc, svc->proc.volfileid, GLUSTERD_SVC_ATTACH);
+            }
+            pthread_mutex_unlock(&conf->attach_lock);
+            if (!ret) {
+                volinfo->shd.attached = _gf_true;
+                goto out;
+            }
+        }
+        /*
+         * It might not actually be safe to manipulate the lock
+         * like this, but if we don't then the connection can
+         * never actually complete and retries are useless.
+         * Unfortunately, all of the alternatives (e.g. doing
+         * all of this in a separate thread) are much more
+         * complicated and risky.
+         * TBD: see if there's a better way
+         */
+        synclock_unlock(&conf->big_lock);
+        synctask_sleep(1);
+        synclock_lock(&conf->big_lock);
+    }
+    ret = -1;
+    gf_msg("glusterd", GF_LOG_WARNING, 0, GD_MSG_SVC_ATTACH_FAIL,
+           "attach failed for %s(volume=%s)", svc->name, volinfo->volname);
 out:
-        return ret;
+    if (rpc)
+        rpc_clnt_unref(rpc);
+    return ret;
 }
 
+int
+glusterd_detach_svc(glusterd_svc_t *svc, glusterd_volinfo_t *volinfo, int sig)
+{
+    glusterd_conf_t *conf = THIS->private;
+    int ret = -1;
+    int tries;
+    rpc_clnt_t *rpc = NULL;
+
+    GF_VALIDATE_OR_GOTO(THIS->name, conf, out);
+    GF_VALIDATE_OR_GOTO(THIS->name, svc, out);
+    GF_VALIDATE_OR_GOTO(THIS->name, volinfo, out);
+
+    gf_msg(THIS->name, GF_LOG_INFO, 0, GD_MSG_DETACH_INFO,
+           "removing svc %s (volume=%s) from existing "
+           "process with pid %d",
+           svc->name, volinfo->volname, glusterd_proc_get_pid(&svc->proc));
 
+    rpc = rpc_clnt_ref(svc->conn.rpc);
+    for (tries = 15; tries > 0; --tries) {
+        if (rpc) {
+            /*For detach there is no flags, and we are not using sig.*/
+            pthread_mutex_lock(&conf->attach_lock);
+            {
+                ret = __glusterd_send_svc_configure_req(svc, 0, svc->conn.rpc,
+                                                        svc->proc.volfileid,
+                                                        GLUSTERD_SVC_DETACH);
+            }
+            pthread_mutex_unlock(&conf->attach_lock);
+            if (!ret) {
+                goto out;
+            }
+        }
+        /*
+         * It might not actually be safe to manipulate the lock
+         * like this, but if we don't then the connection can
+         * never actually complete and retries are useless.
+         * Unfortunately, all of the alternatives (e.g. doing
+         * all of this in a separate thread) are much more
+         * complicated and risky.
+         * TBD: see if there's a better way
+         */
+        synclock_unlock(&conf->big_lock);
+        synctask_sleep(1);
+        synclock_lock(&conf->big_lock);
+    }
+    ret = -1;
+    gf_msg("glusterd", GF_LOG_WARNING, 0, GD_MSG_SVC_DETACH_FAIL,
+           "detach failed for %s(volume=%s)", svc->name, volinfo->volname);
+out:
+    if (rpc)
+        rpc_clnt_unref(rpc);
+    return ret;
+}
diff --git a/xlators/mgmt/glusterd/src/glusterd-svc-helper.h b/xlators/mgmt/glusterd/src/glusterd-svc-helper.h
index 2af75bc5c9a..12717dc58ac 100644
--- a/xlators/mgmt/glusterd/src/glusterd-svc-helper.h
+++ b/xlators/mgmt/glusterd/src/glusterd-svc-helper.h
@@ -11,20 +11,62 @@
 #ifndef _GLUSTERD_SVC_HELPER_H_
 #define _GLUSTERD_SVC_HELPER_H_
 
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
-
 #include "glusterd.h"
 #include "glusterd-svc-mgmt.h"
+#include "glusterd-volgen.h"
+
+int
+glusterd_svcs_reconfigure(glusterd_volinfo_t *volinfo);
+
+int
+glusterd_svcs_stop(glusterd_volinfo_t *vol);
+
+int
+glusterd_svcs_manager(glusterd_volinfo_t *volinfo);
+
+int
+glusterd_svc_check_volfile_identical(char *svc_name,
+                                     glusterd_graph_builder_t builder,
+                                     gf_boolean_t *identical);
+int
+glusterd_svc_check_topology_identical(char *svc_name,
+                                      glusterd_graph_builder_t builder,
+                                      gf_boolean_t *identical);
+int
+glusterd_volume_svc_check_volfile_identical(char *svc_name, dict_t *mode_dict,
+                                            glusterd_volinfo_t *volinfo,
+                                            glusterd_vol_graph_builder_t,
+                                            gf_boolean_t *identical);
+int
+glusterd_volume_svc_check_topology_identical(char *svc_name, dict_t *mode_dict,
+                                             glusterd_volinfo_t *volinfo,
+                                             glusterd_vol_graph_builder_t,
+                                             gf_boolean_t *identical);
+void
+glusterd_volume_svc_build_volfile_path(char *server, glusterd_volinfo_t *vol,
+                                       char *volfile, size_t len);
+void *
+__gf_find_compatible_svc(gd_node_type daemon);
+
+glusterd_svc_proc_t *
+glusterd_svcprocess_new();
 
 int
-glusterd_svcs_reconfigure (glusterd_volinfo_t *volinfo);
+glusterd_shd_svc_mux_init(glusterd_volinfo_t *volinfo, glusterd_svc_t *svc);
+
+void *
+__gf_find_compatible_svc_from_pid(gd_node_type daemon, pid_t pid);
+
+int
+glusterd_attach_svc(glusterd_svc_t *svc, glusterd_volinfo_t *volinfo,
+                    int flags);
 
 int
-glusterd_svcs_stop ();
+glusterd_detach_svc(glusterd_svc_t *svc, glusterd_volinfo_t *volinfo, int sig);
 
 int
-glusterd_svcs_manager (glusterd_volinfo_t *volinfo);
+__glusterd_send_svc_configure_req(glusterd_svc_t *svc, int flag,
+                                  struct rpc_clnt *rpc, char *volfile_id,
+                                  int op);
+
 #endif
diff --git a/xlators/mgmt/glusterd/src/glusterd-svc-mgmt.c b/xlators/mgmt/glusterd/src/glusterd-svc-mgmt.c
index af4272231d2..18b3fb13630 100644
--- a/xlators/mgmt/glusterd/src/glusterd-svc-mgmt.c
+++ b/xlators/mgmt/glusterd/src/glusterd-svc-mgmt.c
@@ -8,332 +8,529 @@
    cases as published by the Free Software Foundation.
 */
 
-#include "globals.h"
-#include "run.h"
+#include <glusterfs/globals.h>
+#include <glusterfs/run.h>
 #include "glusterd.h"
-#include "glusterfs.h"
+#include <glusterfs/glusterfs.h>
 #include "glusterd-utils.h"
 #include "glusterd-svc-mgmt.h"
 #include "glusterd-proc-mgmt.h"
 #include "glusterd-conn-mgmt.h"
 #include "glusterd-messages.h"
+#include <glusterfs/syscall.h>
+#include "glusterd-shd-svc-helper.h"
 
 int
-glusterd_svc_create_rundir (char *rundir)
+glusterd_svc_create_rundir(char *rundir)
 {
-        int ret      = -1;
-
-        ret = mkdir (rundir, 0777);
-        if ((ret == -1) && (EEXIST != errno)) {
-                gf_log (THIS->name, GF_LOG_ERROR, "Unable to create rundir %s",
-                        rundir);
-        }
-        return ret;
+    int ret = -1;
+
+    ret = mkdir_p(rundir, 0755, _gf_true);
+    if ((ret == -1) && (EEXIST != errno)) {
+        gf_msg(THIS->name, GF_LOG_ERROR, errno, GD_MSG_CREATE_DIR_FAILED,
+               "Unable to create rundir %s", rundir);
+    }
+    return ret;
 }
 
-static void
-glusterd_svc_build_logfile_path (char *server, char *logdir, char *logfile,
-                                 size_t len)
+void
+glusterd_svc_build_logfile_path(char *server, char *logdir, char *logfile,
+                                size_t len)
 {
-        snprintf (logfile, len, "%s/%s.log", logdir, server);
+    snprintf(logfile, len, "%s/%s.log", logdir, server);
 }
 
-static void
-glusterd_svc_build_volfileid_path (char *server, char *volfileid, size_t len)
+void
+glusterd_svc_build_volfileid_path(char *server, char *volfileid, size_t len)
 {
-        snprintf (volfileid, len, "gluster/%s", server);
+    snprintf(volfileid, len, "gluster/%s", server);
 }
 
 static int
-glusterd_svc_init_common (glusterd_svc_t *svc,
-                          char *svc_name, char *workdir,
-                          char *rundir, char *logdir,
-                          glusterd_svc_manager_t manager,
-                          glusterd_svc_start_t start,
-                          glusterd_svc_stop_t stop,
-                          glusterd_conn_notify_t notify)
+glusterd_svc_init_common(glusterd_svc_t *svc, char *svc_name, char *workdir,
+                         char *rundir, char *logdir,
+                         glusterd_conn_notify_t notify)
 {
-        int                     ret                        = -1;
-        glusterd_conf_t        *priv                       = NULL;
-        xlator_t               *this                       = NULL;
-        char                    pidfile[PATH_MAX]          = {0,};
-        char                    logfile[PATH_MAX]          = {0,};
-        char                    volfile[PATH_MAX]          = {0,};
-        char                    sockfpath[PATH_MAX]        = {0,};
-        char                    volfileid[256]             = {0};
-        char                    *volfileserver             = NULL;
-
-        this = THIS;
-        GF_ASSERT (this);
-
-        priv = this->private;
-        GF_ASSERT (priv);
-
-        ret = snprintf (svc->name, sizeof (svc->name), "%s", svc_name);
-        if (ret < 0)
-                goto out;
-
-        svc->manager = manager;
-        svc->start = start;
-        svc->stop = stop;
-
-        if (!notify)
-                notify = glusterd_svc_common_rpc_notify;
-
-        glusterd_svc_create_rundir (rundir);
-
-        /* Initialize the connection mgmt */
-        glusterd_conn_build_socket_filepath (rundir, MY_UUID,
-                                             sockfpath, sizeof (sockfpath));
-
-        ret = glusterd_conn_init (&(svc->conn), sockfpath, 600, notify);
-        if (ret)
-                goto out;
-
-        /* Initialize the process mgmt */
-        glusterd_svc_build_pidfile_path (svc_name, workdir, pidfile,
-                                         sizeof(pidfile));
-        glusterd_svc_build_volfile_path (svc_name, workdir, volfile,
-                                         sizeof (volfile));
-
-        glusterd_svc_build_logfile_path (svc_name, logdir, logfile,
-                                         sizeof (logfile));
-        glusterd_svc_build_volfileid_path (svc_name, volfileid,
-                                           sizeof(volfileid));
-
-        if (dict_get_str (this->options, "transport.socket.bind-address",
-                          &volfileserver) != 0) {
-                volfileserver = "localhost";
-        }
-
-        ret = glusterd_proc_init (&(svc->proc), svc_name, pidfile, logdir,
-                                  logfile, volfile, volfileid, volfileserver);
-        if (ret)
-                goto out;
+    int ret = -1;
+    glusterd_conf_t *priv = NULL;
+    xlator_t *this = NULL;
+    char pidfile[PATH_MAX] = {
+        0,
+    };
+    char logfile[PATH_MAX] = {
+        0,
+    };
+    char volfile[PATH_MAX] = {
+        0,
+    };
+    char sockfpath[PATH_MAX] = {
+        0,
+    };
+    char volfileid[256] = {0};
+    char *volfileserver = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+
+    priv = this->private;
+    GF_ASSERT(priv);
+
+    ret = snprintf(svc->name, sizeof(svc->name), "%s", svc_name);
+    if (ret < 0)
+        goto out;
+
+    if (!notify)
+        notify = glusterd_svc_common_rpc_notify;
+
+    glusterd_svc_create_rundir(rundir);
+
+    /* Initialize the connection mgmt */
+    glusterd_conn_build_socket_filepath(rundir, MY_UUID, sockfpath,
+                                        sizeof(sockfpath));
+
+    ret = glusterd_conn_init(&(svc->conn), sockfpath, 600, notify);
+    if (ret)
+        goto out;
+
+    /* Initialize the process mgmt */
+    glusterd_svc_build_pidfile_path(svc_name, priv->rundir, pidfile,
+                                    sizeof(pidfile));
+
+    glusterd_svc_build_volfile_path(svc_name, workdir, volfile,
+                                    sizeof(volfile));
+
+    glusterd_svc_build_logfile_path(svc_name, logdir, logfile, sizeof(logfile));
+    glusterd_svc_build_volfileid_path(svc_name, volfileid, sizeof(volfileid));
+
+    if (dict_get_strn(this->options, "transport.socket.bind-address",
+                      SLEN("transport.socket.bind-address"),
+                      &volfileserver) != 0) {
+        volfileserver = "localhost";
+    }
+
+    ret = glusterd_proc_init(&(svc->proc), svc_name, pidfile, logdir, logfile,
+                             volfile, volfileid, volfileserver);
+    if (ret)
+        goto out;
 
 out:
-        gf_log (this->name, GF_LOG_DEBUG, "Returning %d", ret);
-        return ret;
+    gf_msg_debug(this->name, 0, "Returning %d", ret);
+    return ret;
 }
 
-
 static int
-svc_add_args (dict_t *cmdline, char *arg, data_t *value, void *data)
+svc_add_args(dict_t *cmdline, char *arg, data_t *value, void *data)
 {
-        runner_t *runner = data;
-        runner_add_arg (runner, value->data);
-        return 0;
+    runner_t *runner = data;
+    runner_add_arg(runner, value->data);
+    return 0;
 }
 
-int glusterd_svc_init (glusterd_svc_t *svc, char *svc_name,
-                       glusterd_svc_manager_t manager,
-                       glusterd_svc_start_t start,
-                       glusterd_svc_stop_t stop)
+int
+glusterd_svc_init(glusterd_svc_t *svc, char *svc_name)
 {
-        int              ret               = -1;
-        char             rundir[PATH_MAX]  = {0,};
-        glusterd_conf_t *priv              = NULL;
-        xlator_t        *this              = NULL;
+    int ret = -1;
+    char rundir[PATH_MAX] = {
+        0,
+    };
+    glusterd_conf_t *priv = NULL;
+    xlator_t *this = NULL;
 
-        this = THIS;
-        GF_ASSERT (this);
+    this = THIS;
+    GF_ASSERT(this);
 
-        priv = this->private;
-        GF_ASSERT (priv);
+    priv = this->private;
+    GF_ASSERT(priv);
 
-        glusterd_svc_build_rundir (svc_name, priv->workdir, rundir,
-                                   sizeof (rundir));
-        ret = glusterd_svc_init_common (svc, svc_name, priv->workdir, rundir,
-                                        DEFAULT_LOG_FILE_DIRECTORY, manager,
-                                        start, stop, NULL);
+    glusterd_svc_build_rundir(svc_name, priv->rundir, rundir, sizeof(rundir));
+    ret = glusterd_svc_init_common(svc, svc_name, priv->workdir, rundir,
+                                   priv->logdir, NULL);
 
-        return ret;
+    return ret;
 }
 
 int
-glusterd_svc_start (glusterd_svc_t *svc, int flags, dict_t *cmdline)
+glusterd_svc_start(glusterd_svc_t *svc, int flags, dict_t *cmdline)
 {
-        int                  ret                        = -1;
-        runner_t             runner                     = {0,};
-        glusterd_conf_t     *priv                       = NULL;
-        xlator_t            *this                       = NULL;
-        char                 valgrind_logfile[PATH_MAX] = {0};
-        char                 glusterd_uuid_option[1024] = {0};
-
-        this = THIS;
-        GF_ASSERT (this);
-
-        priv = this->private;
-        GF_ASSERT (priv);
-
-        if (glusterd_proc_is_running (&(svc->proc))) {
-                ret = 0;
-                goto out;
+    int ret = -1;
+    runner_t runner = {
+        0,
+    };
+    glusterd_conf_t *priv = NULL;
+    xlator_t *this = NULL;
+    char valgrind_logfile[PATH_MAX] = {0};
+    char *localtime_logging = NULL;
+    char *log_level = NULL;
+    char daemon_log_level[30] = {0};
+    char msg[1024] = {
+        0,
+    };
+    int32_t len = 0;
+
+    this = THIS;
+    GF_ASSERT(this);
+
+    priv = this->private;
+    GF_VALIDATE_OR_GOTO("glusterd", priv, out);
+    GF_VALIDATE_OR_GOTO("glusterd", svc, out);
+
+    pthread_mutex_lock(&priv->attach_lock);
+    {
+        if (glusterd_proc_is_running(&(svc->proc))) {
+            ret = 0;
+            goto unlock;
         }
 
-        ret = access (svc->proc.volfile, F_OK);
+        ret = sys_access(svc->proc.volfile, F_OK);
         if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "Volfile %s is not present",
-                        svc->proc.volfile);
-                goto out;
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_VOLFILE_NOT_FOUND,
+                   "Volfile %s is not present", svc->proc.volfile);
+            goto unlock;
         }
 
-        runinit (&runner);
+        runinit(&runner);
+
+        if (this->ctx->cmd_args.vgtool != _gf_none) {
+            len = snprintf(valgrind_logfile, PATH_MAX, "%s/valgrind-%s.log",
+                           svc->proc.logdir, svc->name);
+            if ((len < 0) || (len >= PATH_MAX)) {
+                ret = -1;
+                goto unlock;
+            }
 
-        if (priv->valgrind) {
-                snprintf (valgrind_logfile, PATH_MAX, "%s/valgrind-%s.log",
-                          svc->proc.logfile, svc->name);
+            if (this->ctx->cmd_args.vgtool == _gf_memcheck)
+                runner_add_args(&runner, "valgrind", "--leak-check=full",
+                                "--trace-children=yes", "--track-origins=yes",
+                                NULL);
+            else
+                runner_add_args(&runner, "valgrind", "--tool=drd", NULL);
 
-                runner_add_args (&runner, "valgrind", "--leak-check=full",
-                                 "--trace-children=yes", "--track-origins=yes",
-                                 NULL);
-                runner_argprintf (&runner, "--log-file=%s", valgrind_logfile);
+            runner_argprintf(&runner, "--log-file=%s", valgrind_logfile);
         }
 
-        runner_add_args (&runner, SBIN_DIR"/glusterfs",
-                         "-s", svc->proc.volfileserver,
-                         "--volfile-id", svc->proc.volfileid,
-                         "-p", svc->proc.pidfile,
-                         "-l", svc->proc.logfile,
-                         "-S", svc->conn.sockpath,
-                         NULL);
+        runner_add_args(&runner, SBIN_DIR "/glusterfs", "-s",
+                        svc->proc.volfileserver, "--volfile-id",
+                        svc->proc.volfileid, "-p", svc->proc.pidfile, "-l",
+                        svc->proc.logfile, "-S", svc->conn.sockpath, NULL);
+
+        if (dict_get_strn(priv->opts, GLUSTERD_LOCALTIME_LOGGING_KEY,
+                          SLEN(GLUSTERD_LOCALTIME_LOGGING_KEY),
+                          &localtime_logging) == 0) {
+            if (strcmp(localtime_logging, "enable") == 0)
+                runner_add_arg(&runner, "--localtime-logging");
+        }
+        if (dict_get_strn(priv->opts, GLUSTERD_DAEMON_LOG_LEVEL_KEY,
+                          SLEN(GLUSTERD_DAEMON_LOG_LEVEL_KEY),
+                          &log_level) == 0) {
+            snprintf(daemon_log_level, 30, "--log-level=%s", log_level);
+            runner_add_arg(&runner, daemon_log_level);
+        }
+
+        if (this->ctx->cmd_args.global_threading) {
+            runner_add_arg(&runner, "--global-threading");
+        }
 
         if (cmdline)
-                dict_foreach (cmdline, svc_add_args, (void *) &runner);
+            dict_foreach(cmdline, svc_add_args, (void *)&runner);
 
-        gf_log (this->name, GF_LOG_DEBUG, "Starting %s service", svc->name);
+        snprintf(msg, sizeof(msg), "Starting %s service", svc->name);
+        runner_log(&runner, this->name, GF_LOG_DEBUG, msg);
 
         if (flags == PROC_START_NO_WAIT) {
-                ret = runner_run_nowait (&runner);
+            ret = runner_run_nowait(&runner);
         } else {
-                synclock_unlock (&priv->big_lock);
-                {
-                        ret = runner_run (&runner);
-                }
-                synclock_lock (&priv->big_lock);
+            synclock_unlock(&priv->big_lock);
+            {
+                ret = runner_run(&runner);
+            }
+            synclock_lock(&priv->big_lock);
         }
-
+    }
+unlock:
+    pthread_mutex_unlock(&priv->attach_lock);
 out:
-        gf_log (this->name, GF_LOG_DEBUG, "Returning %d", ret);
+    gf_msg_debug(this->name, 0, "Returning %d", ret);
 
-        return ret;
+    return ret;
 }
 
-int glusterd_svc_stop (glusterd_svc_t *svc, int sig)
+int
+glusterd_svc_stop(glusterd_svc_t *svc, int sig)
 {
-        int ret = -1;
-
-        ret = glusterd_proc_stop (&(svc->proc), sig, PROC_STOP_FORCE);
-        if (ret)
-                goto out;
-        glusterd_conn_disconnect (&(svc->conn));
-
-        if (ret == 0) {
-                svc->online =  _gf_false;
-                (void) glusterd_unlink_file ((char *)svc->conn.sockpath);
-        }
+    int ret = -1;
+
+    ret = glusterd_proc_stop(&(svc->proc), sig, PROC_STOP_FORCE);
+    if (ret)
+        goto out;
+    glusterd_conn_disconnect(&(svc->conn));
+
+    if (ret == 0) {
+        svc->online = _gf_false;
+        (void)glusterd_unlink_file((char *)svc->conn.sockpath);
+    }
+    gf_msg(THIS->name, GF_LOG_INFO, 0, GD_MSG_SVC_STOP_SUCCESS,
+           "%s service is stopped", svc->name);
 out:
-        gf_log (THIS->name, GF_LOG_DEBUG, "Returning %d", ret);
+    gf_msg_debug(THIS->name, 0, "Returning %d", ret);
 
-        return ret;
+    return ret;
 }
 
 void
-glusterd_svc_build_pidfile_path (char *server, char *workdir, char *path,
-                                 size_t len)
+glusterd_svc_build_pidfile_path(char *server, char *workdir, char *path,
+                                size_t len)
 {
-        char    dir[PATH_MAX] = {0};
+    char dir[PATH_MAX] = {0};
 
-        GF_ASSERT (len == PATH_MAX);
+    GF_ASSERT(len == PATH_MAX);
 
-        glusterd_svc_build_rundir (server, workdir, dir, sizeof (dir));
-        snprintf (path, len, "%s/%s.pid", dir, server);
+    glusterd_svc_build_rundir(server, workdir, dir, sizeof(dir));
+    snprintf(path, len, "%s/%s.pid", dir, server);
 }
 
 void
-glusterd_svc_build_volfile_path (char *server, char *workdir, char *volfile,
-                                 size_t len)
+glusterd_svc_build_volfile_path(char *server, char *workdir, char *volfile,
+                                size_t len)
 {
-        char  dir[PATH_MAX] = {0,};
+    char dir[PATH_MAX] = {
+        0,
+    };
+
+    GF_ASSERT(len == PATH_MAX);
 
-        GF_ASSERT (len == PATH_MAX);
+    glusterd_svc_build_svcdir(server, workdir, dir, sizeof(dir));
 
-        glusterd_svc_build_svcdir (server, workdir, dir, sizeof (dir));
-        snprintf (volfile, len, "%s/%s-server.vol", dir, server);
+    if (!strcmp(server, "quotad"))
+        /*quotad has different volfile name*/
+        snprintf(volfile, len, "%s/%s.vol", dir, server);
+    else
+        snprintf(volfile, len, "%s/%s-server.vol", dir, server);
 }
 
 void
-glusterd_svc_build_svcdir (char *server, char *workdir, char *path, size_t len)
+glusterd_svc_build_svcdir(char *server, char *workdir, char *path, size_t len)
 {
-        GF_ASSERT (len == PATH_MAX);
+    GF_ASSERT(len == PATH_MAX);
 
-        snprintf (path, len, "%s/%s", workdir, server);
+    snprintf(path, len, "%s/%s", workdir, server);
 }
 
 void
-glusterd_svc_build_rundir (char *server, char *workdir, char *path, size_t len)
+glusterd_svc_build_rundir(char *server, char *workdir, char *path, size_t len)
 {
-        char    dir[PATH_MAX] = {0};
+    char dir[PATH_MAX] = {0};
 
-        GF_ASSERT (len == PATH_MAX);
+    GF_ASSERT(len == PATH_MAX);
 
-        glusterd_svc_build_svcdir (server, workdir, dir, sizeof (dir));
-        snprintf (path, len, "%s/run", dir);
+    glusterd_svc_build_svcdir(server, workdir, dir, sizeof(dir));
+    snprintf(path, len, "%s", dir);
 }
 
 int
-glusterd_svc_reconfigure (int (*create_volfile) ())
+glusterd_svc_reconfigure(int (*create_volfile)())
 {
-        int ret = -1;
+    int ret = -1;
 
-        ret = create_volfile ();
-        if (ret)
-                goto out;
+    ret = create_volfile();
+    if (ret)
+        goto out;
 
-        ret = glusterd_fetchspec_notify (THIS);
+    ret = glusterd_fetchspec_notify(THIS);
 out:
-        return ret;
+    return ret;
 }
 
 int
-glusterd_svc_common_rpc_notify (glusterd_conn_t *conn,
-                                rpc_clnt_event_t event)
+glusterd_svc_common_rpc_notify(glusterd_conn_t *conn, rpc_clnt_event_t event)
 {
-        int                  ret  = 0;
-        glusterd_svc_t      *svc  = NULL;
-        xlator_t            *this = NULL;
-
-        this = THIS;
-        GF_ASSERT (this);
-
-        /* Get the parent onject i.e. svc using list_entry macro */
-        svc = cds_list_entry (conn, glusterd_svc_t, conn);
-        if (!svc) {
-                gf_log (this->name, GF_LOG_ERROR, "Failed to get the service");
-                return -1;
-        }
+    int ret = 0;
+    glusterd_svc_t *svc = NULL;
+    xlator_t *this = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+
+    /* Get the parent onject i.e. svc using list_entry macro */
+    svc = cds_list_entry(conn, glusterd_svc_t, conn);
+    if (!svc) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_SVC_GET_FAIL,
+               "Failed to get the service");
+        return -1;
+    }
+
+    switch (event) {
+        case RPC_CLNT_CONNECT:
+            gf_msg_debug(this->name, 0,
+                         "%s has connected with "
+                         "glusterd.",
+                         svc->name);
+            gf_event(EVENT_SVC_CONNECTED, "svc_name=%s", svc->name);
+            svc->online = _gf_true;
+            break;
 
-        switch (event) {
+        case RPC_CLNT_DISCONNECT:
+            if (svc->online) {
+                gf_msg(this->name, GF_LOG_INFO, 0, GD_MSG_NODE_DISCONNECTED,
+                       "%s has disconnected "
+                       "from glusterd.",
+                       svc->name);
+                gf_event(EVENT_SVC_DISCONNECTED, "svc_name=%s", svc->name);
+                svc->online = _gf_false;
+            }
+            break;
+
+        default:
+            gf_msg_trace(this->name, 0, "got some other RPC event %d", event);
+            break;
+    }
+
+    return ret;
+}
+
+void
+glusterd_volume_svc_build_volfile_path(char *server, glusterd_volinfo_t *vol,
+                                       char *volfile, size_t len)
+{
+    GF_ASSERT(len == PATH_MAX);
+
+    if (!strcmp(server, "glustershd")) {
+        glusterd_svc_build_shd_volfile_path(vol, volfile, len);
+    }
+}
+
+int
+glusterd_muxsvc_common_rpc_notify(glusterd_svc_proc_t *mux_proc,
+                                  rpc_clnt_event_t event)
+{
+    int ret = 0;
+    glusterd_svc_t *svc = NULL;
+    glusterd_svc_t *tmp = NULL;
+    xlator_t *this = NULL;
+    gf_boolean_t need_logging = _gf_false;
+
+    this = THIS;
+    GF_ASSERT(this);
+
+    if (!mux_proc) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_SVC_GET_FAIL,
+               "Failed to get the svc proc data");
+        return -1;
+    }
+
+    /* Currently this function was used for shd svc, if this function is
+     * using for another svc, change ths glustershd reference. We can get
+     * the svc name from any of the attached svc's
+     */
+    switch (event) {
         case RPC_CLNT_CONNECT:
-                gf_log (this->name, GF_LOG_DEBUG, "%s has connected with "
-                        "glusterd.", svc->name);
-                svc->online =  _gf_true;
-                break;
+            gf_msg_debug(this->name, 0,
+                         "glustershd has connected with glusterd.");
+            gf_event(EVENT_SVC_CONNECTED, "svc_name=glustershd");
+            cds_list_for_each_entry_safe(svc, tmp, &mux_proc->svcs, mux_svc)
+            {
+                if (svc->online)
+                    continue;
+                svc->online = _gf_true;
+            }
+            if (mux_proc->status != GF_SVC_STARTED)
+                mux_proc->status = GF_SVC_STARTED;
+
+            break;
 
         case RPC_CLNT_DISCONNECT:
+            cds_list_for_each_entry_safe(svc, tmp, &mux_proc->svcs, mux_svc)
+            {
                 if (svc->online) {
-                        gf_msg (this->name, GF_LOG_INFO, 0,
-                                GD_MSG_NODE_DISCONNECTED, "%s has disconnected "
-                                "from glusterd.", svc->name);
-                        svc->online =  _gf_false;
+                    if (!need_logging)
+                        need_logging = _gf_true;
+                    svc->online = _gf_false;
+                }
+            }
+            if (mux_proc->status != GF_SVC_DIED) {
+                svc = cds_list_entry(mux_proc->svcs.next, glusterd_svc_t,
+                                     mux_svc);
+                if (svc && !glusterd_proc_is_running(&svc->proc)) {
+                    mux_proc->status = GF_SVC_DIED;
+                } else {
+                    mux_proc->status = GF_SVC_DISCONNECTED;
                 }
-                break;
+            }
+
+            if (need_logging) {
+                gf_msg(this->name, GF_LOG_INFO, 0, GD_MSG_NODE_DISCONNECTED,
+                       "glustershd has disconnected from glusterd.");
+                gf_event(EVENT_SVC_DISCONNECTED, "svc_name=glustershd");
+            }
+            break;
 
         default:
-                gf_log (this->name, GF_LOG_TRACE,
-                        "got some other RPC event %d", event);
-                break;
-        }
+            gf_msg_trace(this->name, 0, "got some other RPC event %d", event);
+            break;
+    }
 
-        return ret;
+    return ret;
+}
+
+int
+glusterd_muxsvc_conn_init(glusterd_conn_t *conn, glusterd_svc_proc_t *mux_proc,
+                          char *sockpath, int frame_timeout,
+                          glusterd_muxsvc_conn_notify_t notify)
+{
+    int ret = -1;
+    dict_t *options = NULL;
+    struct rpc_clnt *rpc = NULL;
+    xlator_t *this = THIS;
+    glusterd_svc_t *svc = NULL;
+
+    options = dict_new();
+    if (!this || !options)
+        goto out;
+
+    svc = cds_list_entry(conn, glusterd_svc_t, conn);
+    if (!svc) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_SVC_GET_FAIL,
+               "Failed to get the service");
+        goto out;
+    }
+
+    ret = rpc_transport_unix_options_build(options, sockpath, frame_timeout);
+    if (ret)
+        goto out;
+
+    ret = dict_set_int32n(options, "transport.socket.ignore-enoent",
+                          SLEN("transport.socket.ignore-enoent"), 1);
+    if (ret)
+        goto out;
+
+    /* @options is free'd by rpc_transport when destroyed */
+    rpc = rpc_clnt_new(options, this, (char *)svc->name, 16);
+    if (!rpc) {
+        ret = -1;
+        goto out;
+    }
+
+    ret = rpc_clnt_register_notify(rpc, glusterd_muxsvc_conn_common_notify,
+                                   mux_proc);
+    if (ret)
+        goto out;
+
+    ret = snprintf(conn->sockpath, sizeof(conn->sockpath), "%s", sockpath);
+    if (ret < 0)
+        goto out;
+    else
+        ret = 0;
+
+    conn->frame_timeout = frame_timeout;
+    conn->rpc = rpc;
+    mux_proc->notify = notify;
+out:
+    if (options)
+        dict_unref(options);
+    if (ret) {
+        if (rpc) {
+            rpc_clnt_unref(rpc);
+            rpc = NULL;
+        }
+    }
+    return ret;
 }
diff --git a/xlators/mgmt/glusterd/src/glusterd-svc-mgmt.h b/xlators/mgmt/glusterd/src/glusterd-svc-mgmt.h
index bb4f6f18fc8..5daee993833 100644
--- a/xlators/mgmt/glusterd/src/glusterd-svc-mgmt.h
+++ b/xlators/mgmt/glusterd/src/glusterd-svc-mgmt.h
@@ -11,68 +11,102 @@
 #ifndef _GLUSTERD_SVC_MGMT_H_
 #define _GLUSTERD_SVC_MGMT_H_
 
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
-
 #include "glusterd-proc-mgmt.h"
 #include "glusterd-conn-mgmt.h"
+#include "glusterd-rcu.h"
 
 struct glusterd_svc_;
-typedef struct glusterd_svc_ glusterd_svc_t;
 
-typedef int (*glusterd_svc_manager_t) (glusterd_svc_t *svc,
-                                       void *data, int flags);
-typedef int (*glusterd_svc_start_t) (glusterd_svc_t *svc, int flags);
-typedef int (*glusterd_svc_stop_t) (glusterd_svc_t *svc, int sig);
+typedef struct glusterd_svc_ glusterd_svc_t;
+typedef struct glusterd_svc_proc_ glusterd_svc_proc_t;
+
+typedef void (*glusterd_svc_build_t)(glusterd_svc_t *svc);
+
+typedef int (*glusterd_svc_manager_t)(glusterd_svc_t *svc, void *data,
+                                      int flags);
+typedef int (*glusterd_svc_start_t)(glusterd_svc_t *svc, int flags);
+typedef int (*glusterd_svc_stop_t)(glusterd_svc_t *svc, int sig);
+typedef int (*glusterd_svc_reconfigure_t)(void *data);
+
+typedef int (*glusterd_muxsvc_conn_notify_t)(glusterd_svc_proc_t *mux_proc,
+                                             rpc_clnt_event_t event);
+
+typedef enum gf_svc_status {
+    GF_SVC_STARTING,
+    GF_SVC_STARTED,
+    GF_SVC_STOPPING,
+    GF_SVC_DISCONNECTED,
+    GF_SVC_DIED,
+} gf_svc_status_t;
+
+struct glusterd_svc_proc_ {
+    struct cds_list_head svc_proc_list;
+    struct cds_list_head svcs;
+    glusterd_muxsvc_conn_notify_t notify;
+    rpc_clnt_t *rpc;
+    void *data;
+    gf_svc_status_t status;
+};
 
 struct glusterd_svc_ {
-        char                      name[PATH_MAX];
-        glusterd_conn_t           conn;
-        glusterd_proc_t           proc;
-        glusterd_svc_manager_t    manager;
-        glusterd_svc_start_t      start;
-        glusterd_svc_stop_t       stop;
-        gf_boolean_t              online;
+    glusterd_conn_t conn;
+    glusterd_svc_manager_t manager;
+    glusterd_svc_start_t start;
+    glusterd_svc_stop_t stop;
+    glusterd_svc_reconfigure_t reconfigure;
+    glusterd_svc_proc_t *svc_proc;
+    struct cds_list_head mux_svc;
+    glusterd_proc_t proc;
+    char name[NAME_MAX];
+    gf_boolean_t online;
+    gf_boolean_t inited;
 };
 
 int
-glusterd_svc_create_rundir (char *rundir);
+glusterd_svc_create_rundir(char *rundir);
 
 int
-glusterd_svc_init (glusterd_svc_t *svc, char *svc_name,
-                   glusterd_svc_manager_t manager,
-                   glusterd_svc_start_t start,
-                   glusterd_svc_stop_t stop);
+glusterd_svc_init(glusterd_svc_t *svc, char *svc_name);
 
 int
-glusterd_svc_start (glusterd_svc_t *svc, int flags, dict_t *cmdline);
+glusterd_svc_start(glusterd_svc_t *svc, int flags, dict_t *cmdline);
 
 int
-glusterd_svc_stop (glusterd_svc_t *svc, int sig);
+glusterd_svc_stop(glusterd_svc_t *svc, int sig);
+
+void
+glusterd_svc_build_pidfile_path(char *server, char *workdir, char *path,
+                                size_t len);
 
 void
-glusterd_svc_build_pidfile_path (char *server, char *workdir,
-                                 char *path, size_t len);
+glusterd_svc_build_volfile_path(char *server, char *workdir, char *volfile,
+                                size_t len);
 
 void
-glusterd_svc_build_volfile_path (char *server, char *workdir,
-                                 char *volfile, size_t len);
+glusterd_svc_build_logfile_path(char *server, char *logdir, char *logfile,
+                                size_t len);
 
 void
-glusterd_svc_build_svcdir (char *server, char *workdir,
-                           char *path, size_t len);
+glusterd_svc_build_svcdir(char *server, char *workdir, char *path, size_t len);
 
 void
-glusterd_svc_build_rundir (char *server, char *workdir,
-                           char *path, size_t len);
+glusterd_svc_build_rundir(char *server, char *workdir, char *path, size_t len);
+
+int
+glusterd_svc_reconfigure(int (*create_volfile)());
 
 int
-glusterd_svc_reconfigure (int (*create_volfile) ());
+glusterd_svc_common_rpc_notify(glusterd_conn_t *conn, rpc_clnt_event_t event);
 
 int
-glusterd_svc_common_rpc_notify (glusterd_conn_t *conn,
-                                rpc_clnt_event_t event);
+glusterd_muxsvc_common_rpc_notify(glusterd_svc_proc_t *conn,
+                                  rpc_clnt_event_t event);
 
+int
+glusterd_proc_get_pid(glusterd_proc_t *proc);
+
+int
+glusterd_muxsvc_conn_init(glusterd_conn_t *conn, glusterd_svc_proc_t *mux_proc,
+                          char *sockpath, int frame_timeout,
+                          glusterd_muxsvc_conn_notify_t notify);
 #endif
diff --git a/xlators/mgmt/glusterd/src/glusterd-syncop.c b/xlators/mgmt/glusterd/src/glusterd-syncop.c
index a00438e6d96..b73d37ad08e 100644
--- a/xlators/mgmt/glusterd/src/glusterd-syncop.c
+++ b/xlators/mgmt/glusterd/src/glusterd-syncop.c
@@ -22,193 +22,194 @@
 #include "glusterd-locks.h"
 #include "glusterd-snapshot-utils.h"
 #include "glusterd-messages.h"
+#include "glusterd-errno.h"
 
 extern glusterd_op_info_t opinfo;
 
 void
-gd_synctask_barrier_wait (struct syncargs *args, int count)
+gd_synctask_barrier_wait(struct syncargs *args, int count)
 {
-        glusterd_conf_t *conf = THIS->private;
+    glusterd_conf_t *conf = THIS->private;
 
-        synclock_unlock (&conf->big_lock);
-        synctask_barrier_wait (args, count);
-        synclock_lock (&conf->big_lock);
+    synclock_unlock(&conf->big_lock);
+    synctask_barrier_wait(args, count);
+    synclock_lock(&conf->big_lock);
 
-	syncbarrier_destroy (&args->barrier);
+    syncbarrier_destroy(&args->barrier);
 }
 
 static void
-gd_collate_errors (struct syncargs *args, int op_ret, int op_errno,
-                   char *op_errstr, int op_code,
-                   glusterd_peerinfo_t *peerinfo, u_char *uuid)
+gd_collate_errors(struct syncargs *args, int op_ret, int op_errno,
+                  char *op_errstr, int op_code, uuid_t peerid, u_char *uuid)
 {
-        char     err_str[PATH_MAX] = "Please check log file for details.";
-        char     op_err[PATH_MAX] = "";
-        int      len               = -1;
-        char    *peer_str      = NULL;
-
-        if (op_ret) {
-                args->op_ret = op_ret;
-                args->op_errno = op_errno;
-
-                if (peerinfo)
-                        peer_str = peerinfo->hostname;
-                else
-                        peer_str = uuid_utoa (uuid);
-
-                if (op_errstr && strcmp (op_errstr, "")) {
-                        len = snprintf (err_str, sizeof(err_str) - 1,
-                                        "Error: %s", op_errstr);
-                        err_str[len] = '\0';
-                }
+    char err_str[PATH_MAX] = "Please check log file for details.";
+    char op_err[PATH_MAX] = "";
+    int len = -1;
+    char *peer_str = NULL;
+    glusterd_peerinfo_t *peerinfo = NULL;
+
+    if (op_ret) {
+        args->op_ret = op_ret;
+        args->op_errno = op_errno;
+
+        RCU_READ_LOCK;
+        peerinfo = glusterd_peerinfo_find(peerid, NULL);
+        if (peerinfo)
+            peer_str = gf_strdup(peerinfo->hostname);
+        else
+            peer_str = gf_strdup(uuid_utoa(uuid));
+        RCU_READ_UNLOCK;
 
-                switch (op_code){
-                        case GLUSTERD_MGMT_CLUSTER_LOCK :
-                        {
-                                len = snprintf (op_err, sizeof(op_err) - 1,
-                                                "Locking failed on %s. %s",
-                                                peer_str, err_str);
-                                break;
-                        }
-                        case GLUSTERD_MGMT_CLUSTER_UNLOCK :
-                        {
-                                len = snprintf (op_err, sizeof(op_err) - 1,
-                                                "Unlocking failed on %s. %s",
-                                                peer_str, err_str);
-                                break;
-                        }
-                        case GLUSTERD_MGMT_STAGE_OP :
-                        {
-                                len = snprintf (op_err, sizeof(op_err) - 1,
-                                                "Staging failed on %s. %s",
-                                                peer_str, err_str);
-                                break;
-                        }
-                        case GLUSTERD_MGMT_COMMIT_OP :
-                        {
-                                len = snprintf (op_err, sizeof(op_err) - 1,
-                                                "Commit failed on %s. %s",
-                                                peer_str, err_str);
-                                break;
-                        }
-                }
-                op_err[len] = '\0';
-
-                if (args->errstr) {
-                        len = snprintf (err_str, sizeof(err_str) - 1,
-                                        "%s\n%s", args->errstr,
-                                        op_err);
-                        GF_FREE (args->errstr);
-                        args->errstr = NULL;
-                } else
-                        len = snprintf (err_str, sizeof(err_str) - 1,
-                                        "%s", op_err);
-                err_str[len] = '\0';
-
-                gf_log ("", GF_LOG_ERROR, "%s", op_err);
-                args->errstr = gf_strdup (err_str);
+        if (op_errstr && strcmp(op_errstr, "")) {
+            len = snprintf(err_str, sizeof(err_str) - 1, "Error: %s",
+                           op_errstr);
+            err_str[len] = '\0';
         }
 
-        return;
+        switch (op_code) {
+            case GLUSTERD_MGMT_CLUSTER_LOCK: {
+                len = snprintf(op_err, sizeof(op_err) - 1,
+                               "Locking failed on %s. %s", peer_str, err_str);
+                break;
+            }
+            case GLUSTERD_MGMT_CLUSTER_UNLOCK: {
+                len = snprintf(op_err, sizeof(op_err) - 1,
+                               "Unlocking failed on %s. %s", peer_str, err_str);
+                break;
+            }
+            case GLUSTERD_MGMT_STAGE_OP: {
+                len = snprintf(op_err, sizeof(op_err) - 1,
+                               "Staging failed on %s. %s", peer_str, err_str);
+                break;
+            }
+            case GLUSTERD_MGMT_COMMIT_OP: {
+                len = snprintf(op_err, sizeof(op_err) - 1,
+                               "Commit failed on %s. %s", peer_str, err_str);
+                break;
+            }
+        }
+
+        if (len > 0)
+            op_err[len] = '\0';
+
+        if (args->errstr) {
+            len = snprintf(err_str, sizeof(err_str) - 1, "%s\n%s", args->errstr,
+                           op_err);
+            GF_FREE(args->errstr);
+            args->errstr = NULL;
+        } else
+            len = snprintf(err_str, sizeof(err_str) - 1, "%s", op_err);
+        err_str[len] = '\0';
+
+        gf_msg("glusterd", GF_LOG_ERROR, 0, GD_MSG_MGMT_OP_FAIL, "%s", op_err);
+        args->errstr = gf_strdup(err_str);
+    }
+
+    GF_FREE(peer_str);
+
+    return;
 }
 
 void
-gd_syncargs_init (struct syncargs *args, dict_t *op_ctx)
+gd_syncargs_init(struct syncargs *args, dict_t *op_ctx)
 {
-        args->dict = op_ctx;
-        pthread_mutex_init (&args->lock_dict, NULL);
+    args->dict = op_ctx;
+    pthread_mutex_init(&args->lock_dict, NULL);
 }
 
 static void
-gd_stage_op_req_free (gd1_mgmt_stage_op_req *req)
+gd_stage_op_req_free(gd1_mgmt_stage_op_req *req)
 {
-        if (!req)
-                return;
+    if (!req)
+        return;
 
-        GF_FREE (req->buf.buf_val);
-        GF_FREE (req);
+    GF_FREE(req->buf.buf_val);
+    GF_FREE(req);
 }
 
 static void
-gd_commit_op_req_free (gd1_mgmt_commit_op_req *req)
+gd_commit_op_req_free(gd1_mgmt_commit_op_req *req)
 {
-        if (!req)
-                return;
+    if (!req)
+        return;
 
-        GF_FREE (req->buf.buf_val);
-        GF_FREE (req);
+    GF_FREE(req->buf.buf_val);
+    GF_FREE(req);
 }
 
 static void
-gd_brick_op_req_free (gd1_mgmt_brick_op_req *req)
+gd_brick_op_req_free(gd1_mgmt_brick_op_req *req)
 {
-        if (!req)
-                return;
+    if (!req)
+        return;
 
-        if (strcmp (req->name, "") != 0)
-                GF_FREE (req->name);
-        GF_FREE (req->input.input_val);
-        GF_FREE (req);
+    if (req->dict.dict_val)
+        GF_FREE(req->dict.dict_val);
+    GF_FREE(req->input.input_val);
+    GF_FREE(req);
 }
 
 int
-gd_syncop_submit_request (struct rpc_clnt *rpc, void *req, void *local,
-                          void *cookie, rpc_clnt_prog_t *prog, int procnum,
-                          fop_cbk_fn_t cbkfn, xdrproc_t xdrproc)
+gd_syncop_submit_request(struct rpc_clnt *rpc, void *req, void *local,
+                         void *cookie, rpc_clnt_prog_t *prog, int procnum,
+                         fop_cbk_fn_t cbkfn, xdrproc_t xdrproc)
 {
-        int            ret      = -1;
-        struct iobuf  *iobuf    = NULL;
-        struct iobref *iobref   = NULL;
-        int            count    = 0;
-        struct iovec   iov      = {0, };
-        ssize_t        req_size = 0;
-        call_frame_t  *frame    = NULL;
-
-        GF_ASSERT (rpc);
-        if (!req)
-                goto out;
+    int ret = -1;
+    struct iobuf *iobuf = NULL;
+    struct iobref *iobref = NULL;
+    int count = 0;
+    struct iovec iov = {
+        0,
+    };
+    ssize_t req_size = 0;
+    call_frame_t *frame = NULL;
 
-        req_size = xdr_sizeof (xdrproc, req);
-        iobuf = iobuf_get2 (rpc->ctx->iobuf_pool, req_size);
-        if (!iobuf)
-                goto out;
+    GF_ASSERT(rpc);
+    if (!req)
+        goto out;
 
-        iobref = iobref_new ();
-        if (!iobref)
-                goto out;
+    req_size = xdr_sizeof(xdrproc, req);
+    iobuf = iobuf_get2(rpc->ctx->iobuf_pool, req_size);
+    if (!iobuf)
+        goto out;
 
-        frame = create_frame (THIS, THIS->ctx->pool);
-        if (!frame)
-                goto out;
+    iobref = iobref_new();
+    if (!iobref)
+        goto out;
 
-        iobref_add (iobref, iobuf);
+    frame = create_frame(THIS, THIS->ctx->pool);
+    if (!frame)
+        goto out;
 
-        iov.iov_base = iobuf->ptr;
-        iov.iov_len  = iobuf_pagesize (iobuf);
+    iobref_add(iobref, iobuf);
 
-        /* Create the xdr payload */
-        ret = xdr_serialize_generic (iov, req, xdrproc);
-        if (ret == -1)
-                goto out;
+    iov.iov_base = iobuf->ptr;
+    iov.iov_len = iobuf_pagesize(iobuf);
 
-        iov.iov_len = ret;
-        count = 1;
+    /* Create the xdr payload */
+    ret = xdr_serialize_generic(iov, req, xdrproc);
+    if (ret == -1)
+        goto out;
 
-        frame->local = local;
-        frame->cookie = cookie;
+    iov.iov_len = ret;
+    count = 1;
 
-        /* Send the msg */
-        ret = rpc_clnt_submit (rpc, prog, procnum, cbkfn,
-                               &iov, count, NULL, 0, iobref,
-                               frame, NULL, 0, NULL, 0, NULL);
+    frame->local = local;
+    frame->cookie = cookie;
 
-        /* TODO: do we need to start ping also? */
+    /* Send the msg */
+    ret = rpc_clnt_submit(rpc, prog, procnum, cbkfn, &iov, count, NULL, 0,
+                          iobref, frame, NULL, 0, NULL, 0, NULL);
+
+    /* TODO: do we need to start ping also? */
 
 out:
-        iobref_unref (iobref);
-        iobuf_unref (iobuf);
+    iobref_unref(iobref);
+    iobuf_unref(iobuf);
 
-        return ret;
+    if (ret && frame)
+        STACK_DESTROY(frame->root);
+    return ret;
 }
 
 /* Defined in glusterd-rpc-ops.c */
@@ -217,1572 +218,1826 @@ extern struct rpc_clnt_program gd_brick_prog;
 extern struct rpc_clnt_program gd_mgmt_v3_prog;
 
 int
-glusterd_syncop_aggr_rsp_dict (glusterd_op_t op, dict_t *aggr, dict_t *rsp)
+glusterd_syncop_aggr_rsp_dict(glusterd_op_t op, dict_t *aggr, dict_t *rsp)
 {
-        int        ret  = 0;
-        xlator_t  *this = NULL;
+    int ret = 0;
+    xlator_t *this = NULL;
 
-        this = THIS;
-        GF_ASSERT (this);
+    this = THIS;
+    GF_ASSERT(this);
 
-        switch (op) {
+    switch (op) {
         case GD_OP_CREATE_VOLUME:
         case GD_OP_ADD_BRICK:
         case GD_OP_START_VOLUME:
-                ret = glusterd_aggr_brick_mount_dirs (aggr, rsp);
-                if (ret) {
-                        gf_log (this->name, GF_LOG_ERROR, "Failed to "
-                                "aggregate brick mount dirs");
-                        goto out;
-                }
-        break;
+            ret = glusterd_aggr_brick_mount_dirs(aggr, rsp);
+            if (ret) {
+                gf_msg(this->name, GF_LOG_ERROR, 0,
+                       GD_MSG_BRICK_MOUNDIRS_AGGR_FAIL,
+                       "Failed to "
+                       "aggregate brick mount dirs");
+                goto out;
+            }
+            break;
 
         case GD_OP_REPLACE_BRICK:
-                ret = glusterd_rb_use_rsp_dict (aggr, rsp);
-                if (ret)
-                        goto out;
-        break;
+        case GD_OP_RESET_BRICK:
+            ret = glusterd_rb_use_rsp_dict(aggr, rsp);
+            if (ret)
+                goto out;
+            break;
 
         case GD_OP_SYNC_VOLUME:
-                ret = glusterd_sync_use_rsp_dict (aggr, rsp);
-                if (ret)
-                        goto out;
-        break;
-
-        case GD_OP_PROFILE_VOLUME:
-                ret = glusterd_profile_volume_use_rsp_dict (aggr, rsp);
-                if (ret)
-                        goto out;
-        break;
+            ret = glusterd_sync_use_rsp_dict(aggr, rsp);
+            if (ret)
+                goto out;
+            break;
 
         case GD_OP_GSYNC_CREATE:
-        break;
+            break;
 
         case GD_OP_GSYNC_SET:
-                ret = glusterd_gsync_use_rsp_dict (aggr, rsp, NULL);
-                if (ret)
-                        goto out;
-        break;
+            ret = glusterd_gsync_use_rsp_dict(aggr, rsp, NULL);
+            if (ret)
+                goto out;
+            break;
 
         case GD_OP_STATUS_VOLUME:
-                ret = glusterd_volume_status_copy_to_op_ctx_dict (aggr, rsp);
-                if (ret)
-                        goto out;
-        break;
-
-        case GD_OP_REBALANCE:
-        case GD_OP_DEFRAG_BRICK_VOLUME:
-                ret = glusterd_volume_rebalance_use_rsp_dict (aggr, rsp);
-                if (ret)
-                        goto out;
-        break;
+            ret = glusterd_volume_status_copy_to_op_ctx_dict(aggr, rsp);
+            if (ret)
+                goto out;
+            break;
 
         case GD_OP_HEAL_VOLUME:
-                ret = glusterd_volume_heal_use_rsp_dict (aggr, rsp);
-                if (ret)
-                        goto out;
+            ret = glusterd_volume_heal_use_rsp_dict(aggr, rsp);
+            if (ret)
+                goto out;
 
-        break;
+            break;
 
         case GD_OP_CLEARLOCKS_VOLUME:
-                ret = glusterd_use_rsp_dict (aggr, rsp);
-                if (ret)
-                        goto out;
-        break;
+            ret = glusterd_use_rsp_dict(aggr, rsp);
+            if (ret)
+                goto out;
+            break;
 
         case GD_OP_QUOTA:
-                ret = glusterd_volume_quota_copy_to_op_ctx_dict (aggr, rsp);
-                if (ret)
-                        goto out;
-                break;
+            ret = glusterd_volume_quota_copy_to_op_ctx_dict(aggr, rsp);
+            if (ret)
+                goto out;
+            break;
 
         case GD_OP_SYS_EXEC:
-                ret = glusterd_sys_exec_output_rsp_dict (aggr, rsp);
-                if (ret)
-                        goto out;
-        break;
+            ret = glusterd_sys_exec_output_rsp_dict(aggr, rsp);
+            if (ret)
+                goto out;
+            break;
 
         case GD_OP_SNAP:
-                ret = glusterd_snap_use_rsp_dict (aggr, rsp);
-                if (ret)
-                        goto out;
-        break;
+            ret = glusterd_snap_use_rsp_dict(aggr, rsp);
+            if (ret)
+                goto out;
+            break;
+
+        case GD_OP_SCRUB_STATUS:
+            ret = glusterd_volume_bitrot_scrub_use_rsp_dict(aggr, rsp);
+            break;
+
+        case GD_OP_SCRUB_ONDEMAND:
+            break;
+
+        case GD_OP_MAX_OPVERSION:
+            ret = glusterd_max_opversion_use_rsp_dict(aggr, rsp);
+            break;
+
+        case GD_OP_PROFILE_VOLUME:
+            ret = glusterd_profile_volume_use_rsp_dict(aggr, rsp);
+            break;
+
+        case GD_OP_REBALANCE:
+        case GD_OP_DEFRAG_BRICK_VOLUME:
+            ret = glusterd_volume_rebalance_use_rsp_dict(aggr, rsp);
+            break;
 
         default:
-        break;
-        }
+            break;
+    }
 out:
-        return ret;
+    return ret;
 }
 
 int32_t
-gd_syncop_mgmt_v3_lock_cbk_fn (struct rpc_req *req, struct iovec *iov,
-                                 int count, void *myframe)
+gd_syncop_mgmt_v3_lock_cbk_fn(struct rpc_req *req, struct iovec *iov, int count,
+                              void *myframe)
 {
-        int                         ret           = -1;
-        struct syncargs            *args          = NULL;
-        glusterd_peerinfo_t        *peerinfo      = NULL;
-        gd1_mgmt_v3_lock_rsp        rsp           = {{0},};
-        call_frame_t               *frame         = NULL;
-        int                         op_ret        = -1;
-        int                         op_errno      = -1;
-        xlator_t                   *this          = NULL;
-
-        this = THIS;
-        GF_ASSERT (this);
-        GF_ASSERT(req);
-        GF_ASSERT(myframe);
-
-        frame  = myframe;
-        args   = frame->local;
-        peerinfo = frame->cookie;
-        frame->local = NULL;
-        frame->cookie = NULL;
-
-        if (-1 == req->rpc_status) {
-                op_errno = ENOTCONN;
-                goto out;
-        }
-
-        GF_VALIDATE_OR_GOTO_WITH_ERROR (this->name, iov, out, op_errno,
-                                        EINVAL);
-
-        ret = xdr_to_generic (*iov, &rsp,
-                              (xdrproc_t)xdr_gd1_mgmt_v3_lock_rsp);
-        if (ret < 0)
-                goto out;
-
-        gf_uuid_copy (args->uuid, rsp.uuid);
-
-        op_ret = rsp.op_ret;
-        op_errno = rsp.op_errno;
+    int ret = -1;
+    struct syncargs *args = NULL;
+    gd1_mgmt_v3_lock_rsp rsp = {
+        {0},
+    };
+    call_frame_t *frame = NULL;
+    int op_ret = -1;
+    int op_errno = -1;
+    xlator_t *this = NULL;
+    uuid_t *peerid = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+    GF_ASSERT(req);
+    GF_ASSERT(myframe);
+
+    frame = myframe;
+    args = frame->local;
+    peerid = frame->cookie;
+    frame->local = NULL;
+    frame->cookie = NULL;
+
+    if (-1 == req->rpc_status) {
+        op_errno = ENOTCONN;
+        goto out;
+    }
+
+    GF_VALIDATE_OR_GOTO_WITH_ERROR(this->name, iov, out, op_errno, EINVAL);
+
+    ret = xdr_to_generic(*iov, &rsp, (xdrproc_t)xdr_gd1_mgmt_v3_lock_rsp);
+    if (ret < 0)
+        goto out;
+
+    gf_uuid_copy(args->uuid, rsp.uuid);
+
+    op_ret = rsp.op_ret;
+    op_errno = rsp.op_errno;
 out:
-        gd_mgmt_v3_collate_errors (args, op_ret, op_errno, NULL,
-                                   GLUSTERD_MGMT_V3_LOCK,
-                                   peerinfo, rsp.uuid);
-        STACK_DESTROY (frame->root);
-        synctask_barrier_wake(args);
-        return 0;
+    gd_mgmt_v3_collate_errors(args, op_ret, op_errno, NULL,
+                              GLUSTERD_MGMT_V3_LOCK, *peerid, rsp.uuid);
+
+    GF_FREE(peerid);
+    /* req->rpc_status set to -1 means, STACK_DESTROY will be called from
+     * the caller function.
+     */
+    if (req->rpc_status != -1)
+        STACK_DESTROY(frame->root);
+    synctask_barrier_wake(args);
+    return 0;
 }
 
 int32_t
-gd_syncop_mgmt_v3_lock_cbk (struct rpc_req *req, struct iovec *iov,
-                            int count, void *myframe)
+gd_syncop_mgmt_v3_lock_cbk(struct rpc_req *req, struct iovec *iov, int count,
+                           void *myframe)
 {
-        return glusterd_big_locked_cbk (req, iov, count, myframe,
-                                        gd_syncop_mgmt_v3_lock_cbk_fn);
+    return glusterd_big_locked_cbk(req, iov, count, myframe,
+                                   gd_syncop_mgmt_v3_lock_cbk_fn);
 }
 
 int
-gd_syncop_mgmt_v3_lock (glusterd_op_t op, dict_t *op_ctx,
-                        glusterd_peerinfo_t *peerinfo,
-                        struct syncargs *args, uuid_t my_uuid,
-                        uuid_t recv_uuid, uuid_t txn_id)
+gd_syncop_mgmt_v3_lock(glusterd_op_t op, dict_t *op_ctx,
+                       glusterd_peerinfo_t *peerinfo, struct syncargs *args,
+                       uuid_t my_uuid, uuid_t recv_uuid, uuid_t txn_id)
 {
-        int                      ret  = -1;
-        gd1_mgmt_v3_lock_req     req  = {{0},};
-        glusterd_conf_t         *conf = THIS->private;
-
-        GF_ASSERT(op_ctx);
-        GF_ASSERT(peerinfo);
-        GF_ASSERT(args);
-
-        ret = dict_allocate_and_serialize (op_ctx,
-                                           &req.dict.dict_val,
-                                           &req.dict.dict_len);
-        if (ret)
-                goto out;
-
-        gf_uuid_copy (req.uuid, my_uuid);
-        gf_uuid_copy (req.txn_id, txn_id);
-        req.op = op;
-        synclock_unlock (&conf->big_lock);
-        ret = gd_syncop_submit_request (peerinfo->rpc, &req, args, peerinfo,
-                                        &gd_mgmt_v3_prog,
-                                        GLUSTERD_MGMT_V3_LOCK,
-                                        gd_syncop_mgmt_v3_lock_cbk,
-                                        (xdrproc_t)
-                                        xdr_gd1_mgmt_v3_lock_req);
-        synclock_lock (&conf->big_lock);
+    int ret = -1;
+    gd1_mgmt_v3_lock_req req = {
+        {0},
+    };
+    uuid_t *peerid = NULL;
+
+    GF_ASSERT(op_ctx);
+    GF_ASSERT(peerinfo);
+    GF_ASSERT(args);
+
+    ret = dict_allocate_and_serialize(op_ctx, &req.dict.dict_val,
+                                      &req.dict.dict_len);
+    if (ret) {
+        gf_smsg("glusterd", GF_LOG_ERROR, errno,
+                GD_MSG_DICT_ALLOC_AND_SERL_LENGTH_GET_FAIL, NULL);
+        goto out;
+    }
+
+    gf_uuid_copy(req.uuid, my_uuid);
+    gf_uuid_copy(req.txn_id, txn_id);
+    req.op = op;
+
+    GD_ALLOC_COPY_UUID(peerid, peerinfo->uuid, ret);
+    if (ret)
+        goto out;
+
+    ret = gd_syncop_submit_request(peerinfo->rpc, &req, args, peerid,
+                                   &gd_mgmt_v3_prog, GLUSTERD_MGMT_V3_LOCK,
+                                   gd_syncop_mgmt_v3_lock_cbk,
+                                   (xdrproc_t)xdr_gd1_mgmt_v3_lock_req);
 out:
-        gf_log ("", GF_LOG_DEBUG, "Returning %d", ret);
-        return ret;
+    GF_FREE(req.dict.dict_val);
+    gf_msg_debug("glusterd", 0, "Returning %d", ret);
+    return ret;
 }
 
 int32_t
-gd_syncop_mgmt_v3_unlock_cbk_fn (struct rpc_req *req, struct iovec *iov,
-                                 int count, void *myframe)
+gd_syncop_mgmt_v3_unlock_cbk_fn(struct rpc_req *req, struct iovec *iov,
+                                int count, void *myframe)
 {
-        int                          ret           = -1;
-        struct syncargs             *args          = NULL;
-        glusterd_peerinfo_t         *peerinfo      = NULL;
-        gd1_mgmt_v3_unlock_rsp       rsp           = {{0},};
-        call_frame_t                *frame         = NULL;
-        int                          op_ret        = -1;
-        int                          op_errno      = -1;
-        xlator_t                    *this          = NULL;
-
-        this = THIS;
-        GF_ASSERT (this);
-        GF_ASSERT(req);
-        GF_ASSERT(myframe);
-
-        frame  = myframe;
-        args   = frame->local;
-        peerinfo = frame->cookie;
-        frame->local = NULL;
-        frame->cookie = NULL;
-
-        if (-1 == req->rpc_status) {
-                op_errno = ENOTCONN;
-                goto out;
-        }
-
-        GF_VALIDATE_OR_GOTO_WITH_ERROR (this->name, iov, out, op_errno,
-                                        EINVAL);
-
-        ret = xdr_to_generic (*iov, &rsp,
-                              (xdrproc_t)xdr_gd1_mgmt_v3_unlock_rsp);
-        if (ret < 0)
-                goto out;
-
-        gf_uuid_copy (args->uuid, rsp.uuid);
-
-        /* Set peer as locked, so we unlock only the locked peers */
-        if (rsp.op_ret == 0)
-                peerinfo->locked = _gf_true;
-        op_ret = rsp.op_ret;
-        op_errno = rsp.op_errno;
+    int ret = -1;
+    struct syncargs *args = NULL;
+    gd1_mgmt_v3_unlock_rsp rsp = {
+        {0},
+    };
+    call_frame_t *frame = NULL;
+    int op_ret = -1;
+    int op_errno = -1;
+    xlator_t *this = NULL;
+    uuid_t *peerid = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+    GF_ASSERT(req);
+    GF_ASSERT(myframe);
+
+    frame = myframe;
+    args = frame->local;
+    peerid = frame->cookie;
+    frame->local = NULL;
+    frame->cookie = NULL;
+
+    if (-1 == req->rpc_status) {
+        op_errno = ENOTCONN;
+        goto out;
+    }
+
+    GF_VALIDATE_OR_GOTO_WITH_ERROR(this->name, iov, out, op_errno, EINVAL);
+
+    ret = xdr_to_generic(*iov, &rsp, (xdrproc_t)xdr_gd1_mgmt_v3_unlock_rsp);
+    if (ret < 0)
+        goto out;
+
+    gf_uuid_copy(args->uuid, rsp.uuid);
+
+    op_ret = rsp.op_ret;
+    op_errno = rsp.op_errno;
 out:
-        gd_mgmt_v3_collate_errors (args, op_ret, op_errno, NULL,
-                                   GLUSTERD_MGMT_V3_UNLOCK,
-                                   peerinfo, rsp.uuid);
-        STACK_DESTROY (frame->root);
-        synctask_barrier_wake(args);
-        return 0;
+    gd_mgmt_v3_collate_errors(args, op_ret, op_errno, NULL,
+                              GLUSTERD_MGMT_V3_UNLOCK, *peerid, rsp.uuid);
+
+    GF_FREE(peerid);
+    /* req->rpc_status set to -1 means, STACK_DESTROY will be called from
+     * the caller function.
+     */
+    if (req->rpc_status != -1)
+        STACK_DESTROY(frame->root);
+    synctask_barrier_wake(args);
+    return 0;
 }
 
 int32_t
-gd_syncop_mgmt_v3_unlock_cbk (struct rpc_req *req, struct iovec *iov,
-                              int count, void *myframe)
+gd_syncop_mgmt_v3_unlock_cbk(struct rpc_req *req, struct iovec *iov, int count,
+                             void *myframe)
 {
-        return glusterd_big_locked_cbk (req, iov, count, myframe,
-                                        gd_syncop_mgmt_v3_unlock_cbk_fn);
+    return glusterd_big_locked_cbk(req, iov, count, myframe,
+                                   gd_syncop_mgmt_v3_unlock_cbk_fn);
 }
 
 int
-gd_syncop_mgmt_v3_unlock (dict_t *op_ctx, glusterd_peerinfo_t *peerinfo,
-                          struct syncargs *args, uuid_t my_uuid,
-                          uuid_t recv_uuid, uuid_t txn_id)
+gd_syncop_mgmt_v3_unlock(dict_t *op_ctx, glusterd_peerinfo_t *peerinfo,
+                         struct syncargs *args, uuid_t my_uuid,
+                         uuid_t recv_uuid, uuid_t txn_id)
 {
-        int                          ret  = -1;
-        gd1_mgmt_v3_unlock_req       req  = {{0},};
-        glusterd_conf_t             *conf = THIS->private;
-
-        GF_ASSERT(op_ctx);
-        GF_ASSERT(peerinfo);
-        GF_ASSERT(args);
-
-        ret = dict_allocate_and_serialize (op_ctx,
-                                           &req.dict.dict_val,
-                                           &req.dict.dict_len);
-        if (ret)
-                goto out;
-
-        gf_uuid_copy (req.uuid, my_uuid);
-        gf_uuid_copy (req.txn_id, txn_id);
-        synclock_unlock (&conf->big_lock);
-        ret = gd_syncop_submit_request (peerinfo->rpc, &req, args, peerinfo,
-                                        &gd_mgmt_v3_prog,
-                                        GLUSTERD_MGMT_V3_UNLOCK,
-                                        gd_syncop_mgmt_v3_unlock_cbk,
-                                        (xdrproc_t)
-                                        xdr_gd1_mgmt_v3_unlock_req);
-        synclock_lock (&conf->big_lock);
+    int ret = -1;
+    gd1_mgmt_v3_unlock_req req = {
+        {0},
+    };
+    uuid_t *peerid = NULL;
+
+    GF_ASSERT(op_ctx);
+    GF_ASSERT(peerinfo);
+    GF_ASSERT(args);
+
+    ret = dict_allocate_and_serialize(op_ctx, &req.dict.dict_val,
+                                      &req.dict.dict_len);
+    if (ret) {
+        gf_smsg("glusterd", GF_LOG_ERROR, errno,
+                GD_MSG_DICT_ALLOC_AND_SERL_LENGTH_GET_FAIL, NULL);
+        goto out;
+    }
+
+    gf_uuid_copy(req.uuid, my_uuid);
+    gf_uuid_copy(req.txn_id, txn_id);
+
+    GD_ALLOC_COPY_UUID(peerid, peerinfo->uuid, ret);
+    if (ret)
+        goto out;
+
+    ret = gd_syncop_submit_request(peerinfo->rpc, &req, args, peerid,
+                                   &gd_mgmt_v3_prog, GLUSTERD_MGMT_V3_UNLOCK,
+                                   gd_syncop_mgmt_v3_unlock_cbk,
+                                   (xdrproc_t)xdr_gd1_mgmt_v3_unlock_req);
 out:
-        gf_log ("", GF_LOG_DEBUG, "Returning %d", ret);
-        return ret;
+    GF_FREE(req.dict.dict_val);
+    gf_msg_debug("glusterd", 0, "Returning %d", ret);
+    return ret;
 }
 
 int32_t
-_gd_syncop_mgmt_lock_cbk (struct rpc_req *req, struct iovec *iov,
-                          int count, void *myframe)
+_gd_syncop_mgmt_lock_cbk(struct rpc_req *req, struct iovec *iov, int count,
+                         void *myframe)
 {
-        int                         ret             = -1;
-        struct syncargs             *args           = NULL;
-        glusterd_peerinfo_t         *peerinfo       = NULL;
-        gd1_mgmt_cluster_lock_rsp   rsp             = {{0},};
-        call_frame_t                *frame          = NULL;
-        int                         op_ret          = -1;
-        int                         op_errno        = -1;
-        xlator_t                    *this           = NULL;
-
-        this = THIS;
-        GF_ASSERT (this);
-
-        frame  = myframe;
-        args   = frame->local;
-        peerinfo = frame->cookie;
-        frame->local = NULL;
-        frame->cookie = NULL;
-
-        if (-1 == req->rpc_status) {
-                op_errno = ENOTCONN;
-                goto out;
-        }
-
-        GF_VALIDATE_OR_GOTO_WITH_ERROR (this->name, iov, out, op_errno,
-                                        EINVAL);
-
-        ret = xdr_to_generic (*iov, &rsp,
-                              (xdrproc_t)xdr_gd1_mgmt_cluster_lock_rsp);
-        if (ret < 0)
-                goto out;
-
-        gf_uuid_copy (args->uuid, rsp.uuid);
-
+    int ret = -1;
+    struct syncargs *args = NULL;
+    glusterd_peerinfo_t *peerinfo = NULL;
+    gd1_mgmt_cluster_lock_rsp rsp = {
+        {0},
+    };
+    call_frame_t *frame = NULL;
+    int op_ret = -1;
+    int op_errno = -1;
+    xlator_t *this = NULL;
+    uuid_t *peerid = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+
+    frame = myframe;
+    args = frame->local;
+    peerid = frame->cookie;
+    frame->local = NULL;
+    frame->cookie = NULL;
+
+    if (-1 == req->rpc_status) {
+        op_errno = ENOTCONN;
+        goto out;
+    }
+
+    GF_VALIDATE_OR_GOTO_WITH_ERROR(this->name, iov, out, op_errno, EINVAL);
+
+    ret = xdr_to_generic(*iov, &rsp, (xdrproc_t)xdr_gd1_mgmt_cluster_lock_rsp);
+    if (ret < 0)
+        goto out;
+
+    gf_uuid_copy(args->uuid, rsp.uuid);
+
+    RCU_READ_LOCK;
+    peerinfo = glusterd_peerinfo_find(*peerid, NULL);
+    if (peerinfo) {
         /* Set peer as locked, so we unlock only the locked peers */
         if (rsp.op_ret == 0)
-                peerinfo->locked = _gf_true;
-        op_ret = rsp.op_ret;
-        op_errno = rsp.op_errno;
+            peerinfo->locked = _gf_true;
+        RCU_READ_UNLOCK;
+    } else {
+        RCU_READ_UNLOCK;
+        rsp.op_ret = -1;
+        gf_msg(this->name, GF_LOG_ERROR, EINVAL, GD_MSG_PEER_NOT_FOUND,
+               "Could not find peer with "
+               "ID %s",
+               uuid_utoa(*peerid));
+    }
+
+    op_ret = rsp.op_ret;
+    op_errno = rsp.op_errno;
 out:
-        gd_collate_errors (args, op_ret, op_errno, NULL,
-                           GLUSTERD_MGMT_CLUSTER_LOCK, peerinfo, rsp.uuid);
-        STACK_DESTROY (frame->root);
-        synctask_barrier_wake(args);
-        return 0;
+    gd_collate_errors(args, op_ret, op_errno, NULL, GLUSTERD_MGMT_CLUSTER_LOCK,
+                      *peerid, rsp.uuid);
+
+    GF_FREE(peerid);
+    /* req->rpc_status set to -1 means, STACK_DESTROY will be called from
+     * the caller function.
+     */
+    if (req->rpc_status != -1)
+        STACK_DESTROY(frame->root);
+    synctask_barrier_wake(args);
+    return 0;
 }
 
 int32_t
-gd_syncop_mgmt_lock_cbk (struct rpc_req *req, struct iovec *iov, int count,
-                         void *myframe)
+gd_syncop_mgmt_lock_cbk(struct rpc_req *req, struct iovec *iov, int count,
+                        void *myframe)
 {
-        return glusterd_big_locked_cbk (req, iov, count, myframe,
-                                        _gd_syncop_mgmt_lock_cbk);
+    return glusterd_big_locked_cbk(req, iov, count, myframe,
+                                   _gd_syncop_mgmt_lock_cbk);
 }
 
 int
-gd_syncop_mgmt_lock (glusterd_peerinfo_t *peerinfo, struct syncargs *args,
-                     uuid_t my_uuid, uuid_t recv_uuid)
+gd_syncop_mgmt_lock(glusterd_peerinfo_t *peerinfo, struct syncargs *args,
+                    uuid_t my_uuid, uuid_t recv_uuid)
 {
-        int                       ret = -1;
-        gd1_mgmt_cluster_lock_req req  = {{0},};
-        glusterd_conf_t           *conf = THIS->private;
-
-        gf_uuid_copy (req.uuid, my_uuid);
-        synclock_unlock (&conf->big_lock);
-        ret = gd_syncop_submit_request (peerinfo->rpc, &req, args, peerinfo,
-                                        &gd_mgmt_prog,
-                                        GLUSTERD_MGMT_CLUSTER_LOCK,
-                                        gd_syncop_mgmt_lock_cbk,
-                                        (xdrproc_t) xdr_gd1_mgmt_cluster_lock_req);
-        synclock_lock (&conf->big_lock);
-        return ret;
+    int ret = -1;
+    gd1_mgmt_cluster_lock_req req = {
+        {0},
+    };
+    uuid_t *peerid = NULL;
+
+    gf_uuid_copy(req.uuid, my_uuid);
+    GD_ALLOC_COPY_UUID(peerid, peerinfo->uuid, ret);
+    if (ret)
+        goto out;
+
+    ret = gd_syncop_submit_request(peerinfo->rpc, &req, args, peerid,
+                                   &gd_mgmt_prog, GLUSTERD_MGMT_CLUSTER_LOCK,
+                                   gd_syncop_mgmt_lock_cbk,
+                                   (xdrproc_t)xdr_gd1_mgmt_cluster_lock_req);
+out:
+    return ret;
 }
 
 int32_t
-_gd_syncop_mgmt_unlock_cbk (struct rpc_req *req, struct iovec *iov,
-                            int count, void *myframe)
+_gd_syncop_mgmt_unlock_cbk(struct rpc_req *req, struct iovec *iov, int count,
+                           void *myframe)
 {
-        int                          ret           = -1;
-        struct syncargs             *args          = NULL;
-        glusterd_peerinfo_t         *peerinfo      = NULL;
-        gd1_mgmt_cluster_unlock_rsp  rsp           = {{0},};
-        call_frame_t                *frame         = NULL;
-        int                          op_ret        = -1;
-        int                          op_errno      = -1;
-        xlator_t                    *this          = NULL;
-
-        this = THIS;
-        GF_ASSERT (this);
-
-        frame = myframe;
-        args  = frame->local;
-        peerinfo = frame->cookie;
-        frame->local = NULL;
-
-        if (-1 == req->rpc_status) {
-                op_errno = ENOTCONN;
-                goto out;
-        }
-
-        GF_VALIDATE_OR_GOTO_WITH_ERROR (this->name, iov, out, op_errno,
-                                        EINVAL);
-
-        ret = xdr_to_generic (*iov, &rsp,
-                              (xdrproc_t)xdr_gd1_mgmt_cluster_unlock_rsp);
-        if (ret < 0)
-                goto out;
-
-        gf_uuid_copy (args->uuid, rsp.uuid);
-
+    int ret = -1;
+    struct syncargs *args = NULL;
+    glusterd_peerinfo_t *peerinfo = NULL;
+    gd1_mgmt_cluster_unlock_rsp rsp = {
+        {0},
+    };
+    call_frame_t *frame = NULL;
+    int op_ret = -1;
+    int op_errno = -1;
+    xlator_t *this = NULL;
+    uuid_t *peerid = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+
+    frame = myframe;
+    args = frame->local;
+    peerid = frame->cookie;
+    frame->local = NULL;
+    frame->cookie = NULL;
+
+    if (-1 == req->rpc_status) {
+        op_errno = ENOTCONN;
+        goto out;
+    }
+
+    GF_VALIDATE_OR_GOTO_WITH_ERROR(this->name, iov, out, op_errno, EINVAL);
+
+    ret = xdr_to_generic(*iov, &rsp,
+                         (xdrproc_t)xdr_gd1_mgmt_cluster_unlock_rsp);
+    if (ret < 0)
+        goto out;
+
+    gf_uuid_copy(args->uuid, rsp.uuid);
+
+    RCU_READ_LOCK;
+    peerinfo = glusterd_peerinfo_find(*peerid, NULL);
+    if (peerinfo) {
         peerinfo->locked = _gf_false;
-        op_ret = rsp.op_ret;
-        op_errno = rsp.op_errno;
+        RCU_READ_UNLOCK;
+    } else {
+        RCU_READ_UNLOCK;
+        rsp.op_ret = -1;
+        gf_msg(this->name, GF_LOG_ERROR, EINVAL, GD_MSG_PEER_NOT_FOUND,
+               "Could not find peer with "
+               "ID %s",
+               uuid_utoa(*peerid));
+    }
+
+    op_ret = rsp.op_ret;
+    op_errno = rsp.op_errno;
 out:
-        gd_collate_errors (args, op_ret, op_errno, NULL,
-                           GLUSTERD_MGMT_CLUSTER_UNLOCK, peerinfo, rsp.uuid);
-        STACK_DESTROY (frame->root);
-        synctask_barrier_wake(args);
-        return 0;
+    gd_collate_errors(args, op_ret, op_errno, NULL,
+                      GLUSTERD_MGMT_CLUSTER_UNLOCK, *peerid, rsp.uuid);
+
+    GF_FREE(peerid);
+    /* req->rpc_status set to -1 means, STACK_DESTROY will be called from
+     * the caller function.
+     */
+    if (req->rpc_status != -1)
+        STACK_DESTROY(frame->root);
+    synctask_barrier_wake(args);
+    return 0;
 }
 
 int32_t
-gd_syncop_mgmt_unlock_cbk (struct rpc_req *req, struct iovec *iov,
-                           int count, void *myframe)
+gd_syncop_mgmt_unlock_cbk(struct rpc_req *req, struct iovec *iov, int count,
+                          void *myframe)
 {
-        return glusterd_big_locked_cbk (req, iov, count, myframe,
-                                        _gd_syncop_mgmt_unlock_cbk);
+    return glusterd_big_locked_cbk(req, iov, count, myframe,
+                                   _gd_syncop_mgmt_unlock_cbk);
 }
 
-
 int
-gd_syncop_mgmt_unlock (glusterd_peerinfo_t *peerinfo, struct syncargs *args,
-                       uuid_t my_uuid, uuid_t recv_uuid)
+gd_syncop_mgmt_unlock(glusterd_peerinfo_t *peerinfo, struct syncargs *args,
+                      uuid_t my_uuid, uuid_t recv_uuid)
 {
-        int                         ret     = -1;
-        gd1_mgmt_cluster_unlock_req req     = {{0},};
-        glusterd_conf_t             *conf   = THIS->private;
-
-        gf_uuid_copy (req.uuid, my_uuid);
-        synclock_unlock (&conf->big_lock);
-        ret = gd_syncop_submit_request (peerinfo->rpc, &req, args, peerinfo,
-                                        &gd_mgmt_prog,
-                                        GLUSTERD_MGMT_CLUSTER_UNLOCK,
-                                        gd_syncop_mgmt_unlock_cbk,
-                                        (xdrproc_t) xdr_gd1_mgmt_cluster_lock_req);
-        synclock_lock (&conf->big_lock);
-        return ret;
+    int ret = -1;
+    gd1_mgmt_cluster_unlock_req req = {
+        {0},
+    };
+    uuid_t *peerid = NULL;
+
+    gf_uuid_copy(req.uuid, my_uuid);
+    GD_ALLOC_COPY_UUID(peerid, peerinfo->uuid, ret);
+    if (ret)
+        goto out;
+
+    ret = gd_syncop_submit_request(peerinfo->rpc, &req, args, peerid,
+                                   &gd_mgmt_prog, GLUSTERD_MGMT_CLUSTER_UNLOCK,
+                                   gd_syncop_mgmt_unlock_cbk,
+                                   (xdrproc_t)xdr_gd1_mgmt_cluster_lock_req);
+out:
+    return ret;
 }
 
 int32_t
-_gd_syncop_stage_op_cbk (struct rpc_req *req, struct iovec *iov,
-                         int count, void *myframe)
+_gd_syncop_stage_op_cbk(struct rpc_req *req, struct iovec *iov, int count,
+                        void *myframe)
 {
-        int                         ret           = -1;
-        gd1_mgmt_stage_op_rsp       rsp           = {{0},};
-        struct syncargs             *args         = NULL;
-        xlator_t                    *this         = NULL;
-        dict_t                      *rsp_dict     = NULL;
-        call_frame_t                *frame        = NULL;
-        glusterd_peerinfo_t         *peerinfo     = NULL;
-        int                         op_ret        = -1;
-        int                         op_errno      = -1;
-
-        this  = THIS;
-        GF_ASSERT (this);
-
-        frame = myframe;
-        args  = frame->local;
-        peerinfo = frame->cookie;
-        frame->local = NULL;
-
-        if (-1 == req->rpc_status) {
-                op_errno = ENOTCONN;
-                goto out;
-        }
-
-        GF_VALIDATE_OR_GOTO_WITH_ERROR (this->name, iov, out, op_errno,
-                                        EINVAL);
-
-        ret = xdr_to_generic (*iov, &rsp,
-                              (xdrproc_t)xdr_gd1_mgmt_stage_op_rsp);
-        if (ret < 0)
-                goto out;
-
-        if (rsp.dict.dict_len) {
-                /* Unserialize the dictionary */
-                rsp_dict  = dict_new ();
-
-                ret = dict_unserialize (rsp.dict.dict_val,
-                                        rsp.dict.dict_len,
-                                        &rsp_dict);
-                if (ret < 0) {
-                        GF_FREE (rsp.dict.dict_val);
-                        goto out;
-                } else {
-                        rsp_dict->extra_stdfree = rsp.dict.dict_val;
-                }
-        }
-
-        peerinfo = glusterd_peerinfo_find (rsp.uuid, NULL);
-        if (peerinfo == NULL) {
-                ret = -1;
-                gf_log (this->name, GF_LOG_CRITICAL, "Staging response "
-                        "for 'Volume %s' received from unknown "
-                        "peer: %s", gd_op_list[rsp.op],
-                        uuid_utoa (rsp.uuid));
-                goto out;
+    int ret = -1;
+    gd1_mgmt_stage_op_rsp rsp = {
+        {0},
+    };
+    struct syncargs *args = NULL;
+    xlator_t *this = NULL;
+    dict_t *rsp_dict = NULL;
+    call_frame_t *frame = NULL;
+    int op_ret = -1;
+    int op_errno = -1;
+    uuid_t *peerid = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+
+    frame = myframe;
+    args = frame->local;
+    peerid = frame->cookie;
+    frame->local = NULL;
+    frame->cookie = NULL;
+
+    if (-1 == req->rpc_status) {
+        op_errno = ENOTCONN;
+        goto out;
+    }
+
+    GF_VALIDATE_OR_GOTO_WITH_ERROR(this->name, iov, out, op_errno, EINVAL);
+
+    ret = xdr_to_generic(*iov, &rsp, (xdrproc_t)xdr_gd1_mgmt_stage_op_rsp);
+    if (ret < 0)
+        goto out;
+
+    if (rsp.dict.dict_len) {
+        /* Unserialize the dictionary */
+        rsp_dict = dict_new();
+
+        ret = dict_unserialize(rsp.dict.dict_val, rsp.dict.dict_len, &rsp_dict);
+        if (ret < 0) {
+            GF_FREE(rsp.dict.dict_val);
+            goto out;
+        } else {
+            rsp_dict->extra_stdfree = rsp.dict.dict_val;
         }
-
-        gf_uuid_copy (args->uuid, rsp.uuid);
-        if (rsp.op == GD_OP_REPLACE_BRICK || rsp.op == GD_OP_QUOTA ||
-            rsp.op == GD_OP_CREATE_VOLUME || rsp.op == GD_OP_ADD_BRICK ||
-            rsp.op == GD_OP_START_VOLUME) {
-                pthread_mutex_lock (&args->lock_dict);
-                {
-                        ret = glusterd_syncop_aggr_rsp_dict (rsp.op, args->dict,
-                                                             rsp_dict);
-                        if (ret)
-                                gf_log (this->name, GF_LOG_ERROR, "%s",
-                                        "Failed to aggregate response from "
-                                        " node/brick");
-                }
-                pthread_mutex_unlock (&args->lock_dict);
+    }
+
+    RCU_READ_LOCK;
+    ret = (glusterd_peerinfo_find(rsp.uuid, NULL) == NULL);
+    RCU_READ_UNLOCK;
+    if (ret) {
+        ret = -1;
+        gf_msg(this->name, GF_LOG_CRITICAL, 0, GD_MSG_RESP_FROM_UNKNOWN_PEER,
+               "Staging response "
+               "for 'Volume %s' received from unknown "
+               "peer: %s",
+               gd_op_list[rsp.op], uuid_utoa(rsp.uuid));
+        goto out;
+    }
+
+    gf_uuid_copy(args->uuid, rsp.uuid);
+    if (rsp.op == GD_OP_REPLACE_BRICK || rsp.op == GD_OP_QUOTA ||
+        rsp.op == GD_OP_CREATE_VOLUME || rsp.op == GD_OP_ADD_BRICK ||
+        rsp.op == GD_OP_START_VOLUME) {
+        pthread_mutex_lock(&args->lock_dict);
+        {
+            ret = glusterd_syncop_aggr_rsp_dict(rsp.op, args->dict, rsp_dict);
+            if (ret)
+                gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_RESP_AGGR_FAIL, "%s",
+                       "Failed to aggregate response from "
+                       " node/brick");
         }
+        pthread_mutex_unlock(&args->lock_dict);
+    }
 
-        op_ret = rsp.op_ret;
-        op_errno = rsp.op_errno;
+    op_ret = rsp.op_ret;
+    op_errno = rsp.op_errno;
 
 out:
-        gd_collate_errors (args, op_ret, op_errno, rsp.op_errstr,
-                           GLUSTERD_MGMT_STAGE_OP, peerinfo, rsp.uuid);
-
-        if (rsp_dict)
-                dict_unref (rsp_dict);
-
-        STACK_DESTROY (frame->root);
-        synctask_barrier_wake(args);
-        return 0;
+    gd_collate_errors(args, op_ret, op_errno, rsp.op_errstr,
+                      GLUSTERD_MGMT_STAGE_OP, *peerid, rsp.uuid);
+
+    if (rsp_dict)
+        dict_unref(rsp_dict);
+    GF_FREE(peerid);
+    /* req->rpc_status set to -1 means, STACK_DESTROY will be called from
+     * the caller function.
+     */
+    if (req->rpc_status != -1)
+        STACK_DESTROY(frame->root);
+    synctask_barrier_wake(args);
+    return 0;
 }
 
 int32_t
-gd_syncop_stage_op_cbk (struct rpc_req *req, struct iovec *iov,
-                        int count, void *myframe)
+gd_syncop_stage_op_cbk(struct rpc_req *req, struct iovec *iov, int count,
+                       void *myframe)
 {
-        return glusterd_big_locked_cbk (req, iov, count, myframe,
-                                        _gd_syncop_stage_op_cbk);
+    return glusterd_big_locked_cbk(req, iov, count, myframe,
+                                   _gd_syncop_stage_op_cbk);
 }
 
-
 int
-gd_syncop_mgmt_stage_op (glusterd_peerinfo_t *peerinfo, struct syncargs *args,
-                         uuid_t my_uuid, uuid_t recv_uuid, int op,
-                         dict_t *dict_out, dict_t *op_ctx)
+gd_syncop_mgmt_stage_op(glusterd_peerinfo_t *peerinfo, struct syncargs *args,
+                        uuid_t my_uuid, uuid_t recv_uuid, int op,
+                        dict_t *dict_out, dict_t *op_ctx)
 {
-        gd1_mgmt_stage_op_req *req  = NULL;
-        glusterd_conf_t       *conf = THIS->private;
-        int                   ret   = -1;
-
-        req = GF_CALLOC (1, sizeof (*req), gf_gld_mt_mop_stage_req_t);
-        if (!req)
-                goto out;
-
-        gf_uuid_copy (req->uuid, my_uuid);
-        req->op = op;
-
-        ret = dict_allocate_and_serialize (dict_out,
-                                           &req->buf.buf_val, &req->buf.buf_len);
-        if (ret)
-                goto out;
-
-        synclock_unlock (&conf->big_lock);
-        ret = gd_syncop_submit_request (peerinfo->rpc, req, args, peerinfo,
-                                        &gd_mgmt_prog, GLUSTERD_MGMT_STAGE_OP,
-                                        gd_syncop_stage_op_cbk,
-                                        (xdrproc_t) xdr_gd1_mgmt_stage_op_req);
-        synclock_lock (&conf->big_lock);
+    gd1_mgmt_stage_op_req *req = NULL;
+    int ret = -1;
+    uuid_t *peerid = NULL;
+
+    req = GF_CALLOC(1, sizeof(*req), gf_gld_mt_mop_stage_req_t);
+    if (!req) {
+        gf_smsg("glusterd", GF_LOG_ERROR, errno, GD_MSG_NO_MEMORY, NULL);
+        goto out;
+    }
+
+    gf_uuid_copy(req->uuid, my_uuid);
+    req->op = op;
+
+    ret = dict_allocate_and_serialize(dict_out, &req->buf.buf_val,
+                                      &req->buf.buf_len);
+    if (ret) {
+        gf_smsg("glusterd", GF_LOG_ERROR, errno,
+                GD_MSG_DICT_ALLOC_AND_SERL_LENGTH_GET_FAIL, NULL);
+        goto out;
+    }
+
+    GD_ALLOC_COPY_UUID(peerid, peerinfo->uuid, ret);
+    if (ret)
+        goto out;
+
+    ret = gd_syncop_submit_request(
+        peerinfo->rpc, req, args, peerid, &gd_mgmt_prog, GLUSTERD_MGMT_STAGE_OP,
+        gd_syncop_stage_op_cbk, (xdrproc_t)xdr_gd1_mgmt_stage_op_req);
 out:
-        gd_stage_op_req_free (req);
-        return ret;
-
+    gd_stage_op_req_free(req);
+    return ret;
 }
 
 int32_t
-_gd_syncop_brick_op_cbk (struct rpc_req *req, struct iovec *iov,
-                        int count, void *myframe)
+_gd_syncop_brick_op_cbk(struct rpc_req *req, struct iovec *iov, int count,
+                        void *myframe)
 {
-        struct syncargs        *args  = NULL;
-        gd1_mgmt_brick_op_rsp  rsp   = {0,};
-        int                    ret   = -1;
-        call_frame_t           *frame = NULL;
-        xlator_t               *this = NULL;
-
-        this = THIS;
-        GF_ASSERT (this);
-
-        frame = myframe;
-        args = frame->local;
-        frame->local = NULL;
-
-        /* initialize */
-        args->op_ret   = -1;
-        args->op_errno = EINVAL;
-
-        if (-1 == req->rpc_status) {
-                args->op_errno = ENOTCONN;
-                goto out;
+    struct syncargs *args = NULL;
+    gd1_mgmt_brick_op_rsp rsp = {
+        0,
+    };
+    int ret = -1;
+    call_frame_t *frame = NULL;
+    xlator_t *this = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+
+    frame = myframe;
+    args = frame->local;
+    frame->local = NULL;
+
+    /* initialize */
+    args->op_ret = -1;
+    args->op_errno = EINVAL;
+
+    if (-1 == req->rpc_status) {
+        args->op_errno = ENOTCONN;
+        goto out;
+    }
+
+    GF_VALIDATE_OR_GOTO_WITH_ERROR(this->name, iov, out, args->op_errno,
+                                   EINVAL);
+
+    ret = xdr_to_generic(*iov, &rsp, (xdrproc_t)xdr_gd1_mgmt_brick_op_rsp);
+    if (ret < 0)
+        goto out;
+
+    if (rsp.output.output_len) {
+        args->dict = dict_new();
+        if (!args->dict) {
+            gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_CREATE_FAIL,
+                    NULL);
+            ret = -1;
+            args->op_errno = ENOMEM;
+            goto out;
         }
 
-        GF_VALIDATE_OR_GOTO_WITH_ERROR (this->name, iov, out, args->op_errno,
-                                        EINVAL);
-
-        ret = xdr_to_generic (*iov, &rsp,
-                              (xdrproc_t)xdr_gd1_mgmt_brick_op_rsp);
-        if (ret < 0)
-                goto out;
-
-        if (rsp.output.output_len) {
-                args->dict  = dict_new ();
-                if (!args->dict) {
-                        ret = -1;
-                        args->op_errno = ENOMEM;
-                        goto out;
-                }
-
-                ret = dict_unserialize (rsp.output.output_val,
-                                        rsp.output.output_len,
-                                        &args->dict);
-                if (ret < 0)
-                        goto out;
+        ret = dict_unserialize(rsp.output.output_val, rsp.output.output_len,
+                               &args->dict);
+        if (ret < 0) {
+            gf_smsg(this->name, GF_LOG_ERROR, errno,
+                    GD_MSG_DICT_UNSERIALIZE_FAIL, NULL);
+            goto out;
         }
+    }
 
-        args->op_ret = rsp.op_ret;
-        args->op_errno = rsp.op_errno;
-        args->errstr = gf_strdup (rsp.op_errstr);
+    args->op_ret = rsp.op_ret;
+    args->op_errno = rsp.op_errno;
+    args->errstr = gf_strdup(rsp.op_errstr);
 
 out:
-        if ((rsp.op_errstr) && (strcmp (rsp.op_errstr, "") != 0))
-                free (rsp.op_errstr);
-        free (rsp.output.output_val);
-
-        STACK_DESTROY (frame->root);
-        __wake (args);
-
-        return 0;
+    if ((rsp.op_errstr) && (strcmp(rsp.op_errstr, "") != 0))
+        free(rsp.op_errstr);
+    free(rsp.output.output_val);
+
+    /* req->rpc_status set to -1 means, STACK_DESTROY will be called from
+     * the caller function.
+     */
+    if (req->rpc_status != -1)
+        STACK_DESTROY(frame->root);
+    __wake(args);
+
+    return 0;
 }
 
 int32_t
-gd_syncop_brick_op_cbk (struct rpc_req *req, struct iovec *iov,
-                        int count, void *myframe)
+gd_syncop_brick_op_cbk(struct rpc_req *req, struct iovec *iov, int count,
+                       void *myframe)
 {
-        return glusterd_big_locked_cbk (req, iov, count, myframe,
-                                        _gd_syncop_brick_op_cbk);
+    return glusterd_big_locked_cbk(req, iov, count, myframe,
+                                   _gd_syncop_brick_op_cbk);
 }
 
 int
-gd_syncop_mgmt_brick_op (struct rpc_clnt *rpc, glusterd_pending_node_t *pnode,
-                         int op, dict_t *dict_out, dict_t *op_ctx,
-                         char **errstr)
+gd_syncop_mgmt_brick_op(struct rpc_clnt *rpc, glusterd_pending_node_t *pnode,
+                        int op, dict_t *dict_out, dict_t *op_ctx, char **errstr)
 {
-        struct syncargs        args = {0, };
-        gd1_mgmt_brick_op_req  *req  = NULL;
-        int                    ret  = 0;
-        xlator_t               *this = NULL;
-
-        this = THIS;
-        args.op_ret = -1;
-        args.op_errno = ENOTCONN;
-
-        if ((pnode->type == GD_NODE_NFS) ||
-            (pnode->type == GD_NODE_QUOTAD) ||
-            ((pnode->type == GD_NODE_SHD) && (op == GD_OP_STATUS_VOLUME))) {
-                ret = glusterd_node_op_build_payload (op, &req, dict_out);
-
-        } else {
-                ret = glusterd_brick_op_build_payload (op, pnode->node, &req,
-                                                       dict_out);
+    struct syncargs args = {
+        0,
+    };
+    gd1_mgmt_brick_op_req *req = NULL;
+    int ret = 0;
+    xlator_t *this = NULL;
+
+    this = THIS;
+    args.op_ret = -1;
+    args.op_errno = ENOTCONN;
+
+    if ((pnode->type == GD_NODE_NFS) || (pnode->type == GD_NODE_QUOTAD) ||
+        (pnode->type == GD_NODE_SCRUB) ||
+        ((pnode->type == GD_NODE_SHD) && (op == GD_OP_STATUS_VOLUME))) {
+        ret = glusterd_node_op_build_payload(op, &req, dict_out);
+
+    } else {
+        ret = glusterd_brick_op_build_payload(op, pnode->node, &req, dict_out);
+    }
+
+    if (ret)
+        goto out;
+
+    GD_SYNCOP(rpc, (&args), NULL, gd_syncop_brick_op_cbk, req, &gd_brick_prog,
+              req->op, xdr_gd1_mgmt_brick_op_req);
+
+    if (args.errstr) {
+        if ((strlen(args.errstr) > 0) && errstr)
+            *errstr = args.errstr;
+        else
+            GF_FREE(args.errstr);
+    }
 
+    if (GD_OP_STATUS_VOLUME == op) {
+        ret = dict_set_int32(args.dict, "index", pnode->index);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+                   "Error setting index on brick status"
+                   " rsp dict");
+            args.op_ret = -1;
+            goto out;
         }
-
-        if (ret)
-                goto out;
-
-        GD_SYNCOP (rpc, (&args), NULL, gd_syncop_brick_op_cbk, req,
-                   &gd_brick_prog, req->op, xdr_gd1_mgmt_brick_op_req);
-
-        if (args.errstr) {
-                if ((strlen(args.errstr) > 0) && errstr)
-                        *errstr = args.errstr;
-                else
-                        GF_FREE (args.errstr);
+    }
+
+    if (req->op == GLUSTERD_BRICK_TERMINATE) {
+        if (args.op_ret && (args.op_errno == ENOTCONN)) {
+            /*
+             * This is actually OK.  It happens when the target
+             * brick process exits and we saw the closed connection
+             * before we read the response.  If we didn't read the
+             * response quickly enough that's kind of our own
+             * fault, and the fact that the process exited means
+             * that our goal of terminating the brick was achieved.
+             */
+            args.op_ret = 0;
         }
+    }
 
-        if (GD_OP_STATUS_VOLUME == op) {
-                ret = dict_set_int32 (args.dict, "index", pnode->index);
-                if (ret) {
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "Error setting index on brick status"
-                                " rsp dict");
-                        args.op_ret = -1;
-                        goto out;
-                }
-        }
-        if (args.op_ret == 0)
-                glusterd_handle_node_rsp (dict_out, pnode->node, op,
-                                          args.dict, op_ctx, errstr,
-                                          pnode->type);
+    if (args.op_ret == 0)
+        glusterd_handle_node_rsp(dict_out, pnode->node, op, args.dict, op_ctx,
+                                 errstr, pnode->type);
 
 out:
-        errno = args.op_errno;
-        if (args.dict)
-                dict_unref (args.dict);
-        gd_brick_op_req_free (req);
-        return args.op_ret;
-
+    errno = args.op_errno;
+    if (args.dict)
+        dict_unref(args.dict);
+    if (args.op_ret && errstr && (*errstr == NULL)) {
+        if (op == GD_OP_HEAL_VOLUME) {
+            gf_asprintf(errstr,
+                        "Glusterd Syncop Mgmt brick op '%s' failed."
+                        " Please check glustershd log file for details.",
+                        gd_op_list[op]);
+        } else {
+            gf_asprintf(errstr,
+                        "Glusterd Syncop Mgmt brick op '%s' failed."
+                        " Please check brick log file for details.",
+                        gd_op_list[op]);
+        }
+    }
+    gd_brick_op_req_free(req);
+    return args.op_ret;
 }
 
 int32_t
-_gd_syncop_commit_op_cbk (struct rpc_req *req, struct iovec *iov,
-                          int count, void *myframe)
+_gd_syncop_commit_op_cbk(struct rpc_req *req, struct iovec *iov, int count,
+                         void *myframe)
 {
-        int                     ret               = -1;
-        gd1_mgmt_commit_op_rsp  rsp               = {{0},};
-        struct syncargs        *args              = NULL;
-        xlator_t               *this              = NULL;
-        dict_t                 *rsp_dict          = NULL;
-        call_frame_t           *frame             = NULL;
-        glusterd_peerinfo_t    *peerinfo          = NULL;
-        int                     op_ret            = -1;
-        int                     op_errno          = -1;
-        int                     type              = GF_QUOTA_OPTION_TYPE_NONE;
-
-        this  = THIS;
-        GF_ASSERT (this);
-
-        frame = myframe;
-        args  = frame->local;
-        peerinfo = frame->cookie;
-        frame->local = NULL;
-
-        if (-1 == req->rpc_status) {
-                op_errno = ENOTCONN;
-                goto out;
-        }
-
-        GF_VALIDATE_OR_GOTO_WITH_ERROR (this->name, iov, out, op_errno,
-                                        EINVAL);
-
-        ret = xdr_to_generic (*iov, &rsp,
-                              (xdrproc_t)xdr_gd1_mgmt_commit_op_rsp);
+    int ret = -1;
+    gd1_mgmt_commit_op_rsp rsp = {
+        {0},
+    };
+    struct syncargs *args = NULL;
+    xlator_t *this = NULL;
+    dict_t *rsp_dict = NULL;
+    call_frame_t *frame = NULL;
+    int op_ret = -1;
+    int op_errno = -1;
+    int type = GF_QUOTA_OPTION_TYPE_NONE;
+    uuid_t *peerid = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+
+    frame = myframe;
+    args = frame->local;
+    peerid = frame->cookie;
+    frame->local = NULL;
+    frame->cookie = NULL;
+
+    if (-1 == req->rpc_status) {
+        op_errno = ENOTCONN;
+        goto out;
+    }
+
+    GF_VALIDATE_OR_GOTO_WITH_ERROR(this->name, iov, out, op_errno, EINVAL);
+
+    ret = xdr_to_generic(*iov, &rsp, (xdrproc_t)xdr_gd1_mgmt_commit_op_rsp);
+    if (ret < 0) {
+        goto out;
+    }
+
+    if (rsp.dict.dict_len) {
+        /* Unserialize the dictionary */
+        rsp_dict = dict_new();
+
+        ret = dict_unserialize(rsp.dict.dict_val, rsp.dict.dict_len, &rsp_dict);
         if (ret < 0) {
-                goto out;
-        }
-
-        if (rsp.dict.dict_len) {
-                /* Unserialize the dictionary */
-                rsp_dict  = dict_new ();
-
-                ret = dict_unserialize (rsp.dict.dict_val,
-                                        rsp.dict.dict_len,
-                                        &rsp_dict);
-                if (ret < 0) {
-                        GF_FREE (rsp.dict.dict_val);
-                        goto out;
-                } else {
-                        rsp_dict->extra_stdfree = rsp.dict.dict_val;
-                }
-        }
-
-        peerinfo = glusterd_peerinfo_find (rsp.uuid, NULL);
-
-        if (peerinfo == NULL) {
-                ret = -1;
-                gf_log (this->name, GF_LOG_CRITICAL, "Commit response "
-                        "for 'Volume %s' received from unknown "
-                        "peer: %s", gd_op_list[rsp.op],
-                        uuid_utoa (rsp.uuid));
-                goto out;
+            GF_FREE(rsp.dict.dict_val);
+            goto out;
+        } else {
+            rsp_dict->extra_stdfree = rsp.dict.dict_val;
         }
-
-        gf_uuid_copy (args->uuid, rsp.uuid);
-        if (rsp.op == GD_OP_QUOTA) {
-                ret = dict_get_int32 (args->dict, "type", &type);
-                if (ret) {
-                        gf_log (this->name, GF_LOG_ERROR, "Failed to get "
-                                "opcode");
-                        goto out;
-                }
+    }
+
+    RCU_READ_LOCK;
+    ret = (glusterd_peerinfo_find(rsp.uuid, NULL) == 0);
+    RCU_READ_UNLOCK;
+    if (ret) {
+        ret = -1;
+        gf_msg(this->name, GF_LOG_CRITICAL, 0, GD_MSG_RESP_FROM_UNKNOWN_PEER,
+               "Commit response "
+               "for 'Volume %s' received from unknown "
+               "peer: %s",
+               gd_op_list[rsp.op], uuid_utoa(rsp.uuid));
+        goto out;
+    }
+
+    gf_uuid_copy(args->uuid, rsp.uuid);
+    if (rsp.op == GD_OP_QUOTA) {
+        ret = dict_get_int32(args->dict, "type", &type);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+                   "Failed to get "
+                   "opcode");
+            goto out;
         }
-
-        if ((rsp.op != GD_OP_QUOTA) || (type == GF_QUOTA_OPTION_TYPE_LIST)) {
-                pthread_mutex_lock (&args->lock_dict);
-                {
-                        ret = glusterd_syncop_aggr_rsp_dict (rsp.op, args->dict,
-                                                             rsp_dict);
-                        if (ret)
-                                gf_log (this->name, GF_LOG_ERROR, "%s",
-                                        "Failed to aggregate response from "
-                                        " node/brick");
-                }
-                pthread_mutex_unlock (&args->lock_dict);
+    }
+
+    if ((rsp.op != GD_OP_QUOTA) || (type == GF_QUOTA_OPTION_TYPE_LIST)) {
+        pthread_mutex_lock(&args->lock_dict);
+        {
+            ret = glusterd_syncop_aggr_rsp_dict(rsp.op, args->dict, rsp_dict);
+            if (ret)
+                gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_RESP_AGGR_FAIL, "%s",
+                       "Failed to aggregate response from "
+                       " node/brick");
         }
+        pthread_mutex_unlock(&args->lock_dict);
+    }
 
-        op_ret = rsp.op_ret;
-        op_errno = rsp.op_errno;
+    op_ret = rsp.op_ret;
+    op_errno = rsp.op_errno;
 
 out:
-        gd_collate_errors (args, op_ret, op_errno, rsp.op_errstr,
-                           GLUSTERD_MGMT_COMMIT_OP, peerinfo, rsp.uuid);
-        if (rsp_dict)
-                dict_unref (rsp_dict);
-
-        STACK_DESTROY (frame->root);
-        synctask_barrier_wake(args);
-
-        return 0;
+    gd_collate_errors(args, op_ret, op_errno, rsp.op_errstr,
+                      GLUSTERD_MGMT_COMMIT_OP, *peerid, rsp.uuid);
+    if (rsp_dict)
+        dict_unref(rsp_dict);
+    GF_FREE(peerid);
+    /* req->rpc_status set to -1 means, STACK_DESTROY will be called from
+     * the caller function.
+     */
+    if (req->rpc_status != -1)
+        STACK_DESTROY(frame->root);
+    synctask_barrier_wake(args);
+
+    return 0;
 }
 
 int32_t
-gd_syncop_commit_op_cbk (struct rpc_req *req, struct iovec *iov,
-                         int count, void *myframe)
+gd_syncop_commit_op_cbk(struct rpc_req *req, struct iovec *iov, int count,
+                        void *myframe)
 {
-        return glusterd_big_locked_cbk (req, iov, count, myframe,
-                                        _gd_syncop_commit_op_cbk);
+    return glusterd_big_locked_cbk(req, iov, count, myframe,
+                                   _gd_syncop_commit_op_cbk);
 }
 
-
 int
-gd_syncop_mgmt_commit_op (glusterd_peerinfo_t *peerinfo, struct syncargs *args,
-                          uuid_t my_uuid, uuid_t recv_uuid,
-                          int op, dict_t *dict_out, dict_t *op_ctx)
+gd_syncop_mgmt_commit_op(glusterd_peerinfo_t *peerinfo, struct syncargs *args,
+                         uuid_t my_uuid, uuid_t recv_uuid, int op,
+                         dict_t *dict_out, dict_t *op_ctx)
 {
-        glusterd_conf_t        *conf = THIS->private;
-        gd1_mgmt_commit_op_req *req  = NULL;
-        int                    ret  = -1;
-
-        req = GF_CALLOC (1, sizeof (*req), gf_gld_mt_mop_commit_req_t);
-        if (!req)
-                goto out;
-
-        gf_uuid_copy (req->uuid, my_uuid);
-        req->op = op;
-
-        ret = dict_allocate_and_serialize (dict_out,
-                                           &req->buf.buf_val, &req->buf.buf_len);
-        if (ret)
-                goto out;
-
-        synclock_unlock (&conf->big_lock);
-        ret = gd_syncop_submit_request (peerinfo->rpc, req, args, peerinfo,
-                                        &gd_mgmt_prog, GLUSTERD_MGMT_COMMIT_OP,
-                                        gd_syncop_commit_op_cbk,
-                                        (xdrproc_t) xdr_gd1_mgmt_commit_op_req);
-        synclock_lock (&conf->big_lock);
+    gd1_mgmt_commit_op_req *req = NULL;
+    int ret = -1;
+    uuid_t *peerid = NULL;
+
+    req = GF_CALLOC(1, sizeof(*req), gf_gld_mt_mop_commit_req_t);
+    if (!req) {
+        gf_smsg("glusterd", GF_LOG_ERROR, errno, GD_MSG_NO_MEMORY, NULL);
+        goto out;
+    }
+
+    gf_uuid_copy(req->uuid, my_uuid);
+    req->op = op;
+
+    ret = dict_allocate_and_serialize(dict_out, &req->buf.buf_val,
+                                      &req->buf.buf_len);
+    if (ret) {
+        gf_smsg("glusterd", GF_LOG_ERROR, errno,
+                GD_MSG_DICT_ALLOC_AND_SERL_LENGTH_GET_FAIL, NULL);
+        goto out;
+    }
+
+    GD_ALLOC_COPY_UUID(peerid, peerinfo->uuid, ret);
+    if (ret)
+        goto out;
+
+    ret = gd_syncop_submit_request(peerinfo->rpc, req, args, peerid,
+                                   &gd_mgmt_prog, GLUSTERD_MGMT_COMMIT_OP,
+                                   gd_syncop_commit_op_cbk,
+                                   (xdrproc_t)xdr_gd1_mgmt_commit_op_req);
 out:
-        gd_commit_op_req_free (req);
-        return ret;
+    gd_commit_op_req_free(req);
+    return ret;
 }
 
-
 int
-gd_build_peers_list (struct cds_list_head *peers,
-                     struct cds_list_head *xact_peers, glusterd_op_t op)
+gd_lock_op_phase(glusterd_conf_t *conf, glusterd_op_t op, dict_t *op_ctx,
+                 char **op_errstr, uuid_t txn_id,
+                 glusterd_op_info_t *txn_opinfo, gf_boolean_t cluster_lock)
 {
-        glusterd_peerinfo_t *peerinfo = NULL;
-        int                 npeers      = 0;
-
-        GF_ASSERT (peers);
-        GF_ASSERT (xact_peers);
-
-        rcu_read_lock ();
-        cds_list_for_each_entry_rcu (peerinfo, peers, uuid_list) {
-                if (!peerinfo->connected)
-                        continue;
-                if (op != GD_OP_SYNC_VOLUME &&
-                    peerinfo->state.state != GD_FRIEND_STATE_BEFRIENDED)
-                        continue;
-
-                cds_list_add_tail (&peerinfo->op_peers_list, xact_peers);
-                npeers++;
-        }
-        rcu_read_unlock ();
-
-        return npeers;
-}
-
-int
-gd_build_local_xaction_peers_list (struct cds_list_head *peers,
-                                   struct cds_list_head *xact_peers,
-                                   glusterd_op_t op)
-{
-        glusterd_peerinfo_t    *peerinfo    = NULL;
-        glusterd_local_peers_t *local_peers = NULL;
-        int                     npeers      = 0;
-
-        GF_ASSERT (peers);
-        GF_ASSERT (xact_peers);
-
-        rcu_read_lock ();
-        cds_list_for_each_entry_rcu (peerinfo, peers, uuid_list) {
-                if (!peerinfo->connected)
-                        continue;
-                if (op != GD_OP_SYNC_VOLUME &&
-                    peerinfo->state.state != GD_FRIEND_STATE_BEFRIENDED)
-                        continue;
-
-                local_peers = GF_CALLOC (1, sizeof (*local_peers),
-                                         gf_gld_mt_local_peers_t);
-                if (!local_peers) {
-                        npeers = -1;
-                        goto unlock;
-                }
-                CDS_INIT_LIST_HEAD (&local_peers->op_peers_list);
-                local_peers->peerinfo = peerinfo;
-                cds_list_add_tail (&local_peers->op_peers_list, xact_peers);
-                npeers++;
-        }
-unlock:
-        rcu_read_unlock ();
-
-        return npeers;
-}
-
-int
-gd_lock_op_phase (glusterd_conf_t  *conf, glusterd_op_t op, dict_t *op_ctx,
-                  char **op_errstr, int npeers, uuid_t txn_id,
-                  struct cds_list_head *peers)
-{
-        int                     ret         = -1;
-        int                     peer_cnt    = 0;
-        uuid_t                  peer_uuid   = {0};
-        xlator_t               *this        = NULL;
-        glusterd_peerinfo_t    *peerinfo    = NULL;
-        struct syncargs         args        = {0};
-
-        if (!npeers) {
-                ret = 0;
-                goto out;
-        }
-
-        this = THIS;
-        synctask_barrier_init((&args));
-        peer_cnt = 0;
-        list_for_each_local_xaction_peers (peerinfo, peers) {
-                if (conf->op_version < GD_OP_VERSION_3_6_0) {
-                        /* Reset lock status */
-                        peerinfo->locked = _gf_false;
-                        gd_syncop_mgmt_lock (peerinfo, &args,
-                                             MY_UUID, peer_uuid);
-                } else
-                        gd_syncop_mgmt_v3_lock (op, op_ctx, peerinfo, &args,
-                                                MY_UUID, peer_uuid, txn_id);
-                peer_cnt++;
-        }
-        gd_synctask_barrier_wait((&args), peer_cnt);
-
-        if (args.op_ret) {
-                if (args.errstr)
-                         *op_errstr = gf_strdup (args.errstr);
-                else {
-                        ret = gf_asprintf (op_errstr, "Another transaction "
-                                           "could be in progress. Please try "
-                                           "again after sometime.");
-                        if (ret == -1)
-                                *op_errstr = NULL;
+    int ret = -1;
+    int peer_cnt = 0;
+    uuid_t peer_uuid = {0};
+    xlator_t *this = NULL;
+    glusterd_peerinfo_t *peerinfo = NULL;
+    struct syncargs args = {0};
+
+    this = THIS;
+    GF_VALIDATE_OR_GOTO("glusterd", this, out);
+
+    ret = synctask_barrier_init((&args));
+    if (ret)
+        goto out;
+
+    peer_cnt = 0;
+
+    RCU_READ_LOCK;
+    cds_list_for_each_entry_rcu(peerinfo, &conf->peers, uuid_list)
+    {
+        /* Only send requests to peers who were available before the
+         * transaction started
+         */
+        if (peerinfo->generation > txn_opinfo->txn_generation)
+            continue;
+
+        if (!peerinfo->connected)
+            continue;
+        if (op != GD_OP_SYNC_VOLUME &&
+            peerinfo->state.state != GD_FRIEND_STATE_BEFRIENDED)
+            continue;
+
+        if (cluster_lock) {
+            /* Reset lock status */
+            peerinfo->locked = _gf_false;
+            gd_syncop_mgmt_lock(peerinfo, &args, MY_UUID, peer_uuid);
+        } else
+            gd_syncop_mgmt_v3_lock(op, op_ctx, peerinfo, &args, MY_UUID,
+                                   peer_uuid, txn_id);
+        peer_cnt++;
+    }
+    RCU_READ_UNLOCK;
+
+    if (0 == peer_cnt) {
+        ret = 0;
+        goto out;
+    }
 
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "Failed to acquire lock");
+    gd_synctask_barrier_wait((&args), peer_cnt);
 
-                }
+    if (args.op_ret) {
+        if (args.errstr)
+            *op_errstr = gf_strdup(args.errstr);
+        else {
+            ret = gf_asprintf(op_errstr,
+                              "Another transaction "
+                              "could be in progress. Please try "
+                              "again after some time.");
+            if (ret == -1)
+                *op_errstr = NULL;
+
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_PEER_LOCK_FAIL,
+                   "Failed to acquire lock");
         }
+    }
 
-        ret = args.op_ret;
+    ret = args.op_ret;
 
-        gf_log (this->name, GF_LOG_DEBUG, "Sent lock op req for 'Volume %s' "
-                "to %d peers. Returning %d", gd_op_list[op], peer_cnt, ret);
+    gf_msg_debug(this->name, 0,
+                 "Sent lock op req for 'Volume %s' "
+                 "to %d peers. Returning %d",
+                 gd_op_list[op], peer_cnt, ret);
 out:
-        return ret;
+    return ret;
 }
 
 int
-gd_stage_op_phase (struct cds_list_head *peers, glusterd_op_t op,
-                   dict_t *op_ctx, dict_t *req_dict, char **op_errstr,
-                   int npeers)
+gd_stage_op_phase(glusterd_op_t op, dict_t *op_ctx, dict_t *req_dict,
+                  char **op_errstr, glusterd_op_info_t *txn_opinfo)
 {
-        int                     ret             = -1;
-        int                     peer_cnt        = 0;
-        dict_t                 *rsp_dict        = NULL;
-        char                   *hostname        = NULL;
-        xlator_t               *this            = NULL;
-        glusterd_peerinfo_t    *peerinfo        = NULL;
-        uuid_t                  tmp_uuid        = {0};
-        char                   *errstr          = NULL;
-        struct syncargs         args            = {0};
-        dict_t                 *aggr_dict       = NULL;
-
-        this = THIS;
-        GF_ASSERT (this);
-
-        rsp_dict = dict_new ();
-        if (!rsp_dict)
-                goto out;
-
-        if ((op == GD_OP_CREATE_VOLUME) || (op == GD_OP_ADD_BRICK) ||
-            (op == GD_OP_START_VOLUME))
-                aggr_dict = req_dict;
-        else
-                aggr_dict = op_ctx;
-
-        ret = glusterd_validate_quorum (this, op, req_dict, op_errstr);
-        if (ret) {
-                gf_msg (this->name, GF_LOG_CRITICAL, 0,
-                        GD_MSG_SERVER_QUORUM_NOT_MET,
-                        "Server quorum not met. Rejecting operation.");
-                goto out;
-        }
-
-        ret = glusterd_op_stage_validate (op, req_dict, op_errstr, rsp_dict);
+    int ret = -1;
+    int peer_cnt = 0;
+    dict_t *rsp_dict = NULL;
+    char *hostname = NULL;
+    xlator_t *this = NULL;
+    glusterd_conf_t *conf = NULL;
+    glusterd_peerinfo_t *peerinfo = NULL;
+    uuid_t tmp_uuid = {0};
+    char *errstr = NULL;
+    struct syncargs args = {0};
+    dict_t *aggr_dict = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+    conf = this->private;
+    GF_ASSERT(conf);
+
+    rsp_dict = dict_new();
+    if (!rsp_dict) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_CREATE_FAIL, NULL);
+        goto out;
+    }
+
+    if ((op == GD_OP_CREATE_VOLUME) || (op == GD_OP_ADD_BRICK) ||
+        (op == GD_OP_START_VOLUME))
+        aggr_dict = req_dict;
+    else
+        aggr_dict = op_ctx;
+
+    ret = glusterd_validate_quorum(this, op, req_dict, op_errstr);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_CRITICAL, 0, GD_MSG_SERVER_QUORUM_NOT_MET,
+               "Server quorum not met. Rejecting operation.");
+        goto out;
+    }
+
+    ret = glusterd_op_stage_validate(op, req_dict, op_errstr, rsp_dict);
+    if (ret) {
+        hostname = "localhost";
+        goto stage_done;
+    }
+
+    if ((op == GD_OP_REPLACE_BRICK || op == GD_OP_QUOTA ||
+         op == GD_OP_CREATE_VOLUME || op == GD_OP_ADD_BRICK ||
+         op == GD_OP_START_VOLUME)) {
+        ret = glusterd_syncop_aggr_rsp_dict(op, aggr_dict, rsp_dict);
         if (ret) {
-                hostname = "localhost";
-                goto stage_done;
-        }
-
-        if ((op == GD_OP_REPLACE_BRICK || op == GD_OP_QUOTA ||
-             op == GD_OP_CREATE_VOLUME || op == GD_OP_ADD_BRICK ||
-             op == GD_OP_START_VOLUME)) {
-                ret = glusterd_syncop_aggr_rsp_dict (op, aggr_dict, rsp_dict);
-                if (ret) {
-                        gf_log (this->name, GF_LOG_ERROR, "%s",
-                                "Failed to aggregate response from node/brick");
-                        goto out;
-                }
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_RESP_AGGR_FAIL, "%s",
+                   "Failed to aggregate response from node/brick");
+            goto out;
         }
-        dict_unref (rsp_dict);
-        rsp_dict = NULL;
+    }
+    dict_unref(rsp_dict);
+    rsp_dict = NULL;
 
 stage_done:
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, LOGSTR_STAGE_FAIL,
-                        gd_op_list[op], hostname, (*op_errstr) ? ":" : " ",
-                        (*op_errstr) ? *op_errstr : " ");
-                if (*op_errstr == NULL)
-                        gf_asprintf (op_errstr, OPERRSTR_STAGE_FAIL, hostname);
-                goto out;
-        }
-
-        if (!npeers) {
-                ret = 0;
-                goto out;
-        }
-
-        gd_syncargs_init (&args, aggr_dict);
-        synctask_barrier_init((&args));
-        peer_cnt = 0;
-
-        list_for_each_local_xaction_peers (peerinfo, peers) {
-                ret = gd_syncop_mgmt_stage_op (peerinfo, &args,
-                                               MY_UUID, tmp_uuid,
-                                               op, req_dict, op_ctx);
-                peer_cnt++;
-        }
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_VALIDATE_FAILED,
+               LOGSTR_STAGE_FAIL, gd_op_list[op], hostname,
+               (*op_errstr) ? ":" : " ", (*op_errstr) ? *op_errstr : " ");
+        if (*op_errstr == NULL)
+            gf_asprintf(op_errstr, OPERRSTR_STAGE_FAIL, hostname);
+        goto out;
+    }
+
+    gd_syncargs_init(&args, aggr_dict);
+    ret = synctask_barrier_init((&args));
+    if (ret)
+        goto out;
+
+    peer_cnt = 0;
+
+    RCU_READ_LOCK;
+    cds_list_for_each_entry_rcu(peerinfo, &conf->peers, uuid_list)
+    {
+        /* Only send requests to peers who were available before the
+         * transaction started
+         */
+        if (peerinfo->generation > txn_opinfo->txn_generation)
+            continue;
+
+        if (!peerinfo->connected)
+            continue;
+        if (op != GD_OP_SYNC_VOLUME &&
+            peerinfo->state.state != GD_FRIEND_STATE_BEFRIENDED)
+            continue;
+
+        (void)gd_syncop_mgmt_stage_op(peerinfo, &args, MY_UUID, tmp_uuid, op,
+                                      req_dict, op_ctx);
+        peer_cnt++;
+    }
+    RCU_READ_UNLOCK;
+
+    if (0 == peer_cnt) {
+        ret = 0;
+        goto out;
+    }
 
-        gf_log (this->name, GF_LOG_DEBUG, "Sent stage op req for 'Volume %s' "
-                "to %d peers", gd_op_list[op], peer_cnt);
+    gf_msg_debug(this->name, 0,
+                 "Sent stage op req for 'Volume %s' "
+                 "to %d peers",
+                 gd_op_list[op], peer_cnt);
 
-        gd_synctask_barrier_wait((&args), peer_cnt);
+    gd_synctask_barrier_wait((&args), peer_cnt);
 
-        if (args.errstr)
-                 *op_errstr = gf_strdup (args.errstr);
-        else if (dict_get_str (aggr_dict, "errstr", &errstr) == 0)
-                *op_errstr = gf_strdup (errstr);
+    if (args.errstr)
+        *op_errstr = gf_strdup(args.errstr);
+    else if (dict_get_str(aggr_dict, "errstr", &errstr) == 0)
+        *op_errstr = gf_strdup(errstr);
 
-        ret = args.op_ret;
+    ret = args.op_ret;
 
 out:
-        if ((ret == 0) && (op == GD_OP_QUOTA)) {
-                ret = glusterd_validate_and_set_gfid (op_ctx, req_dict,
-                                                      op_errstr);
-                if (ret)
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "Failed to validate and set gfid");
-        }
+    if ((ret == 0) && (op == GD_OP_QUOTA)) {
+        ret = glusterd_validate_and_set_gfid(op_ctx, req_dict, op_errstr);
+        if (ret)
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_GFID_VALIDATE_SET_FAIL,
+                   "Failed to validate and set gfid");
+    }
 
-        if (rsp_dict)
-                dict_unref (rsp_dict);
-        return ret;
+    if (rsp_dict)
+        dict_unref(rsp_dict);
+    return ret;
 }
 
 int
-gd_commit_op_phase (struct cds_list_head *peers, glusterd_op_t op,
-                    dict_t *op_ctx, dict_t *req_dict, char **op_errstr,
-                    int npeers)
+gd_commit_op_phase(glusterd_op_t op, dict_t *op_ctx, dict_t *req_dict,
+                   char **op_errstr, glusterd_op_info_t *txn_opinfo)
 {
-        dict_t                 *rsp_dict      = NULL;
-        int                     peer_cnt      = -1;
-        int                     ret           = -1;
-        char                   *hostname      = NULL;
-        glusterd_peerinfo_t    *peerinfo      = NULL;
-        xlator_t               *this          = NULL;
-        uuid_t                  tmp_uuid      = {0};
-        char                   *errstr        = NULL;
-        struct syncargs         args          = {0};
-        int                     type          = GF_QUOTA_OPTION_TYPE_NONE;
-
-        this = THIS;
-        rsp_dict = dict_new ();
-        if (!rsp_dict) {
-                ret = -1;
-                goto out;
-        }
-
-        ret = glusterd_op_commit_perform (op, req_dict, op_errstr, rsp_dict);
+    dict_t *rsp_dict = NULL;
+    int peer_cnt = -1;
+    int ret = -1;
+    char *hostname = NULL;
+    glusterd_peerinfo_t *peerinfo = NULL;
+    xlator_t *this = NULL;
+    glusterd_conf_t *conf = NULL;
+    uuid_t tmp_uuid = {0};
+    char *errstr = NULL;
+    struct syncargs args = {0};
+    int type = GF_QUOTA_OPTION_TYPE_NONE;
+    uint32_t cmd = 0;
+    gf_boolean_t origin_glusterd = _gf_false;
+
+    this = THIS;
+    GF_ASSERT(this);
+    conf = this->private;
+    GF_ASSERT(conf);
+
+    rsp_dict = dict_new();
+    if (!rsp_dict) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_CREATE_FAIL, NULL);
+        ret = -1;
+        goto out;
+    }
+
+    ret = glusterd_op_commit_perform(op, req_dict, op_errstr, rsp_dict);
+    if (ret) {
+        hostname = "localhost";
+        goto commit_done;
+    }
+
+    if (op == GD_OP_QUOTA) {
+        ret = dict_get_int32(op_ctx, "type", &type);
         if (ret) {
-                hostname = "localhost";
-                goto commit_done;
-        }
-
-        if (op == GD_OP_QUOTA) {
-                ret = dict_get_int32 (op_ctx, "type", &type);
-                if (ret) {
-                        gf_log (this->name, GF_LOG_ERROR, "Failed to get "
-                                "opcode");
-                        goto out;
-                }
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+                   "Failed to get "
+                   "opcode");
+            goto out;
         }
+    }
 
-        if (((op == GD_OP_QUOTA) && ((type == GF_QUOTA_OPTION_TYPE_LIST) ||
-             (type == GF_QUOTA_OPTION_TYPE_LIST_OBJECTS))) ||
-            ((op != GD_OP_SYNC_VOLUME) && (op != GD_OP_QUOTA))) {
-
-                ret =  glusterd_syncop_aggr_rsp_dict (op, op_ctx,
-                                                      rsp_dict);
-                if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "%s", "Failed to aggregate "
-                        "response from node/brick");
-                goto out;
-                }
+    if (((op == GD_OP_QUOTA) &&
+         ((type == GF_QUOTA_OPTION_TYPE_LIST) ||
+          (type == GF_QUOTA_OPTION_TYPE_LIST_OBJECTS))) ||
+        ((op != GD_OP_SYNC_VOLUME) && (op != GD_OP_QUOTA))) {
+        ret = glusterd_syncop_aggr_rsp_dict(op, op_ctx, rsp_dict);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_RESP_AGGR_FAIL, "%s",
+                   "Failed to aggregate "
+                   "response from node/brick");
+            goto out;
         }
+    }
 
-        dict_unref (rsp_dict);
-        rsp_dict = NULL;
+    dict_unref(rsp_dict);
+    rsp_dict = NULL;
 
 commit_done:
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_COMMIT_OP_FAIL,
+               LOGSTR_COMMIT_FAIL, gd_op_list[op], hostname,
+               (*op_errstr) ? ":" : " ", (*op_errstr) ? *op_errstr : " ");
+        if (*op_errstr == NULL)
+            gf_asprintf(op_errstr, OPERRSTR_COMMIT_FAIL, hostname);
+        goto out;
+    }
+
+    gd_syncargs_init(&args, op_ctx);
+    ret = synctask_barrier_init((&args));
+    if (ret)
+        goto out;
+
+    peer_cnt = 0;
+    origin_glusterd = is_origin_glusterd(req_dict);
+
+    if (op == GD_OP_STATUS_VOLUME) {
+        ret = dict_get_uint32(req_dict, "cmd", &cmd);
         if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, LOGSTR_COMMIT_FAIL,
-                        gd_op_list[op], hostname, (*op_errstr) ? ":" : " ",
-                        (*op_errstr) ? *op_errstr : " ");
-                if (*op_errstr == NULL)
-                        gf_asprintf (op_errstr, OPERRSTR_COMMIT_FAIL,
-                                     hostname);
-                goto out;
+            gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_GET_FAILED,
+                    "Key=cmd", NULL);
+            goto out;
         }
 
-        if (!npeers) {
+        if (origin_glusterd) {
+            if ((cmd & GF_CLI_STATUS_ALL)) {
                 ret = 0;
                 goto out;
+            }
         }
+    }
 
-        gd_syncargs_init (&args, op_ctx);
-        synctask_barrier_init((&args));
-        peer_cnt = 0;
-
-        list_for_each_local_xaction_peers (peerinfo, peers) {
-                ret = gd_syncop_mgmt_commit_op (peerinfo, &args,
-                                                MY_UUID, tmp_uuid,
-                                                op, req_dict, op_ctx);
-                peer_cnt++;
-        }
-        gd_synctask_barrier_wait((&args), peer_cnt);
-        ret = args.op_ret;
-        if (args.errstr)
-                 *op_errstr = gf_strdup (args.errstr);
-        else if (dict_get_str (op_ctx, "errstr", &errstr) == 0)
-                *op_errstr = gf_strdup (errstr);
-
-        gf_log (this->name, GF_LOG_DEBUG, "Sent commit op req for 'Volume %s' "
-                "to %d peers", gd_op_list[op], peer_cnt);
+    RCU_READ_LOCK;
+    cds_list_for_each_entry_rcu(peerinfo, &conf->peers, uuid_list)
+    {
+        /* Only send requests to peers who were available before the
+         * transaction started
+         */
+        if (peerinfo->generation > txn_opinfo->txn_generation)
+            continue;
+
+        if (!peerinfo->connected)
+            continue;
+        if (op != GD_OP_SYNC_VOLUME &&
+            peerinfo->state.state != GD_FRIEND_STATE_BEFRIENDED)
+            continue;
+
+        (void)gd_syncop_mgmt_commit_op(peerinfo, &args, MY_UUID, tmp_uuid, op,
+                                       req_dict, op_ctx);
+        peer_cnt++;
+    }
+    RCU_READ_UNLOCK;
+
+    if (0 == peer_cnt) {
+        ret = 0;
+        goto out;
+    }
+
+    gd_synctask_barrier_wait((&args), peer_cnt);
+    ret = args.op_ret;
+    if (args.errstr)
+        *op_errstr = gf_strdup(args.errstr);
+    else if (dict_get_str(op_ctx, "errstr", &errstr) == 0)
+        *op_errstr = gf_strdup(errstr);
+
+    gf_msg_debug(this->name, 0,
+                 "Sent commit op req for 'Volume %s' "
+                 "to %d peers",
+                 gd_op_list[op], peer_cnt);
 out:
-        if (!ret)
-                glusterd_op_modify_op_ctx (op, op_ctx);
+    if (!ret)
+        glusterd_op_modify_op_ctx(op, op_ctx);
 
-        if (rsp_dict)
-                dict_unref (rsp_dict);
+    if (rsp_dict)
+        dict_unref(rsp_dict);
 
-        GF_FREE (args.errstr);
-        args.errstr = NULL;
+    GF_FREE(args.errstr);
+    args.errstr = NULL;
 
-        return ret;
+    return ret;
 }
 
 int
-gd_unlock_op_phase (glusterd_conf_t  *conf, glusterd_op_t op, int *op_ret,
-                    rpcsvc_request_t *req, dict_t *op_ctx, char *op_errstr,
-                    int npeers, char *volname, gf_boolean_t is_acquired,
-                    uuid_t txn_id, struct cds_list_head *peers)
+gd_unlock_op_phase(glusterd_conf_t *conf, glusterd_op_t op, int *op_ret,
+                   rpcsvc_request_t *req, dict_t *op_ctx, char *op_errstr,
+                   char *volname, gf_boolean_t is_acquired, uuid_t txn_id,
+                   glusterd_op_info_t *txn_opinfo, gf_boolean_t cluster_lock)
 {
-        glusterd_peerinfo_t    *peerinfo    = NULL;
-        uuid_t                  tmp_uuid    = {0};
-        int                     peer_cnt    = 0;
-        int                     ret         = -1;
-        xlator_t               *this        = NULL;
-        struct syncargs         args        = {0};
+    glusterd_peerinfo_t *peerinfo = NULL;
+    uuid_t tmp_uuid = {0};
+    int peer_cnt = 0;
+    int ret = -1;
+    xlator_t *this = NULL;
+    struct syncargs args = {0};
+    int32_t global = 0;
+    char *type = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+
+    /* If the lock has not been held during this
+     * transaction, do not send unlock requests */
+    if (!is_acquired) {
+        ret = 0;
+        goto out;
+    }
+
+    ret = synctask_barrier_init((&args));
+    if (ret)
+        goto out;
+
+    peer_cnt = 0;
+
+    if (cluster_lock) {
+        RCU_READ_LOCK;
+        cds_list_for_each_entry_rcu(peerinfo, &conf->peers, uuid_list)
+        {
+            /* Only send requests to peers who were available before
+             * the transaction started
+             */
+            if (peerinfo->generation > txn_opinfo->txn_generation)
+                continue;
+
+            if (!peerinfo->connected)
+                continue;
+            if (op != GD_OP_SYNC_VOLUME &&
+                peerinfo->state.state != GD_FRIEND_STATE_BEFRIENDED)
+                continue;
+
+            /* Only unlock peers that were locked */
+            if (peerinfo->locked) {
+                gd_syncop_mgmt_unlock(peerinfo, &args, MY_UUID, tmp_uuid);
+                peer_cnt++;
+            }
+        }
+        RCU_READ_UNLOCK;
+    } else {
+        ret = dict_get_int32(op_ctx, "hold_global_locks", &global);
+        if (!ret && global)
+            type = "global";
+        else
+            type = "vol";
+        if (volname || global) {
+            RCU_READ_LOCK;
+            cds_list_for_each_entry_rcu(peerinfo, &conf->peers, uuid_list)
+            {
+                /* Only send requests to peers who were
+                 * available before the transaction started
+                 */
+                if (peerinfo->generation > txn_opinfo->txn_generation)
+                    continue;
 
-        this = THIS;
-        GF_ASSERT (this);
+                if (!peerinfo->connected)
+                    continue;
+                if (op != GD_OP_SYNC_VOLUME &&
+                    peerinfo->state.state != GD_FRIEND_STATE_BEFRIENDED)
+                    continue;
 
-        if (!npeers) {
-                ret = 0;
-                goto out;
+                gd_syncop_mgmt_v3_unlock(op_ctx, peerinfo, &args, MY_UUID,
+                                         tmp_uuid, txn_id);
+                peer_cnt++;
+            }
+            RCU_READ_UNLOCK;
         }
+    }
 
-        /* If the lock has not been held during this
-         * transaction, do not send unlock requests */
-        if (!is_acquired) {
-                ret = 0;
-                goto out;
-        }
+    if (0 == peer_cnt) {
+        ret = 0;
+        goto out;
+    }
 
-        synctask_barrier_init((&args));
-        peer_cnt = 0;
-
-        if (conf->op_version < GD_OP_VERSION_3_6_0) {
-                list_for_each_local_xaction_peers (peerinfo, peers) {
-                        /* Only unlock peers that were locked */
-                        if (peerinfo->locked) {
-                                gd_syncop_mgmt_unlock (peerinfo, &args,
-                                                       MY_UUID, tmp_uuid);
-                                peer_cnt++;
-                        }
-                }
-        } else {
-                if (volname) {
-                        list_for_each_local_xaction_peers (peerinfo, peers) {
-                                gd_syncop_mgmt_v3_unlock (op_ctx, peerinfo,
-                                                          &args, MY_UUID,
-                                                          tmp_uuid, txn_id);
-                                peer_cnt++;
-                        }
-                }
-        }
-        gd_synctask_barrier_wait((&args), peer_cnt);
+    gd_synctask_barrier_wait((&args), peer_cnt);
 
-        ret = args.op_ret;
+    ret = args.op_ret;
 
-        gf_log (this->name, GF_LOG_DEBUG, "Sent unlock op req for 'Volume %s' "
-                "to %d peers. Returning %d", gd_op_list[op], peer_cnt, ret);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "Failed to unlock "
-                        "on some peer(s)");
-        }
+    gf_msg_debug(this->name, 0,
+                 "Sent unlock op req for 'Volume %s' "
+                 "to %d peers. Returning %d",
+                 gd_op_list[op], peer_cnt, ret);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_PEER_UNLOCK_FAIL,
+               "Failed to unlock "
+               "on some peer(s)");
+    }
 
 out:
-        /* If unlock failed, and op_ret was previously set
-         * priority is given to the op_ret. If op_ret was
-         * not set, and unlock failed, then set op_ret */
-        if (!*op_ret)
-                *op_ret = ret;
-
-        if (is_acquired) {
-                /* Based on the op-version,
-                 * we release the cluster or mgmt_v3 lock
-                 * and clear the op */
-
-                glusterd_op_clear_op (op);
-                if (conf->op_version < GD_OP_VERSION_3_6_0)
-                        glusterd_unlock (MY_UUID);
-                else {
-                        if (volname) {
-                                ret = glusterd_mgmt_v3_unlock (volname, MY_UUID,
-                                                               "vol");
-                                if (ret)
-                                        gf_log (this->name, GF_LOG_ERROR,
-                                                "Unable to release lock for %s",
-                                                volname);
-                        }
-                }
+    /* If unlock failed, and op_ret was previously set
+     * priority is given to the op_ret. If op_ret was
+     * not set, and unlock failed, then set op_ret */
+    if (!*op_ret)
+        *op_ret = ret;
+
+    if (is_acquired) {
+        /* Based on the op-version,
+         * we release the cluster or mgmt_v3 lock
+         * and clear the op */
+
+        glusterd_op_clear_op(op);
+        if (cluster_lock)
+            glusterd_unlock(MY_UUID);
+        else {
+            if (type) {
+                ret = glusterd_mgmt_v3_unlock(volname, MY_UUID, type);
+                if (ret)
+                    gf_msg(this->name, GF_LOG_ERROR, 0,
+                           GD_MSG_MGMTV3_UNLOCK_FAIL,
+                           "Unable to release lock for %s", volname);
+            }
         }
+    }
 
-        if (!*op_ret)
-                *op_ret = ret;
+    if (!*op_ret)
+        *op_ret = ret;
 
-        /*
-         * If there are any quorum events while the OP is in progress, process
-         * them.
-         */
-        if (conf->pending_quorum_action)
-                glusterd_do_quorum_action ();
+    /*
+     * If there are any quorum events while the OP is in progress, process
+     * them.
+     */
+    if (conf->pending_quorum_action)
+        glusterd_do_quorum_action();
 
-        return 0;
+    return 0;
 }
 
 int
-gd_get_brick_count (struct cds_list_head *bricks)
+gd_get_brick_count(struct cds_list_head *bricks)
 {
-        glusterd_pending_node_t *pending_node = NULL;
-        int                     npeers        = 0;
-        cds_list_for_each_entry (pending_node, bricks, list) {
-                npeers++;
-        }
-        return npeers;
+    glusterd_pending_node_t *pending_node = NULL;
+    int npeers = 0;
+    cds_list_for_each_entry(pending_node, bricks, list) { npeers++; }
+    return npeers;
 }
 
 int
-gd_brick_op_phase (glusterd_op_t op, dict_t *op_ctx, dict_t *req_dict,
-                   char **op_errstr)
+gd_brick_op_phase(glusterd_op_t op, dict_t *op_ctx, dict_t *req_dict,
+                  char **op_errstr)
 {
-        glusterd_pending_node_t *pending_node = NULL;
-        struct cds_list_head    selected = {0,};
-        xlator_t                *this = NULL;
-        int                     brick_count = 0;
-        int                     ret = -1;
-        rpc_clnt_t              *rpc = NULL;
-        dict_t                  *rsp_dict = NULL;
-        glusterd_conf_t         *conf = NULL;
-
-        this = THIS;
-        conf = this->private;
-        rsp_dict = dict_new ();
-        if (!rsp_dict) {
+    glusterd_pending_node_t *pending_node = NULL;
+    glusterd_pending_node_t *tmp = NULL;
+    struct cds_list_head selected = {
+        0,
+    };
+    xlator_t *this = NULL;
+    int brick_count = 0;
+    int ret = -1;
+    rpc_clnt_t *rpc = NULL;
+    dict_t *rsp_dict = NULL;
+    int32_t cmd = GF_OP_CMD_NONE;
+    glusterd_volinfo_t *volinfo = NULL;
+
+    this = THIS;
+    rsp_dict = dict_new();
+    if (!rsp_dict) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_CREATE_FAIL, NULL);
+        ret = -1;
+        goto out;
+    }
+
+    CDS_INIT_LIST_HEAD(&selected);
+    ret = glusterd_op_bricks_select(op, req_dict, op_errstr, &selected,
+                                    rsp_dict);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_BRICK_OP_FAIL, "%s",
+               (*op_errstr) ? *op_errstr
+                            : "Brick op failed. Check "
+                              "glusterd log file for more details.");
+        goto out;
+    }
+
+    if (op == GD_OP_HEAL_VOLUME) {
+        ret = glusterd_syncop_aggr_rsp_dict(op, op_ctx, rsp_dict);
+        if (ret)
+            goto out;
+    }
+    dict_unref(rsp_dict);
+    rsp_dict = NULL;
+
+    brick_count = 0;
+    cds_list_for_each_entry_safe(pending_node, tmp, &selected, list)
+    {
+        rpc = glusterd_pending_node_get_rpc(pending_node);
+        /* In the case of rebalance if the rpc object is null, we try to
+         * create the rpc object. if the rebalance daemon is down, it returns
+         * -1. otherwise, rpc object will be created and referenced.
+         */
+        if (!rpc) {
+            if (pending_node->type == GD_NODE_REBALANCE && pending_node->node) {
+                volinfo = pending_node->node;
+                ret = glusterd_rebalance_rpc_create(volinfo);
+                if (ret) {
+                    ret = 0;
+                    glusterd_defrag_volume_node_rsp(req_dict, NULL, op_ctx);
+                    goto out;
+                } else {
+                    rpc = glusterd_defrag_rpc_get(volinfo->rebal.defrag);
+                }
+            } else {
                 ret = -1;
+                gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_RPC_FAILURE,
+                       "Brick Op failed "
+                       "due to rpc failure.");
                 goto out;
+            }
         }
 
-        CDS_INIT_LIST_HEAD (&selected);
-        ret = glusterd_op_bricks_select (op, req_dict, op_errstr, &selected,
-                                         rsp_dict);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "%s",
-                       (*op_errstr)? *op_errstr: "Brick op failed. Check "
-                       "glusterd log file for more details.");
-                goto out;
-        }
-
-        if (op == GD_OP_HEAL_VOLUME) {
-                ret = glusterd_syncop_aggr_rsp_dict (op, op_ctx, rsp_dict);
-                if (ret)
-                        goto out;
+        ret = gd_syncop_mgmt_brick_op(rpc, pending_node, op, req_dict, op_ctx,
+                                      op_errstr);
+        if (op == GD_OP_STATUS_VOLUME) {
+            /* for client-list its enough to quit the loop
+             * once we get the value from one brick
+             * */
+            ret = dict_get_int32(req_dict, "cmd", &cmd);
+            if (!ret && (cmd & GF_CLI_STATUS_CLIENT_LIST)) {
+                if (dict_get(op_ctx, "client-count"))
+                    break;
+            }
         }
-        dict_unref (rsp_dict);
-        rsp_dict = NULL;
-
-        brick_count = 0;
-        cds_list_for_each_entry (pending_node, &selected, list) {
-                rpc = glusterd_pending_node_get_rpc (pending_node);
-                if (!rpc) {
-                        if (pending_node->type == GD_NODE_REBALANCE) {
-                                ret = 0;
-                                glusterd_defrag_volume_node_rsp (req_dict,
-                                                                 NULL, op_ctx);
-                                goto out;
-                        }
-
-                        ret = -1;
-                        gf_log (this->name, GF_LOG_ERROR, "Brick Op failed "
-                                "due to rpc failure.");
-                        goto out;
-                }
-                ret = gd_syncop_mgmt_brick_op (rpc, pending_node, op, req_dict,
-                                               op_ctx, op_errstr);
-                if (ret)
-                        goto out;
+        if (ret)
+            goto out;
 
-                brick_count++;
-        }
+        brick_count++;
+        glusterd_pending_node_put_rpc(pending_node);
+        GF_FREE(pending_node);
+    }
 
-        ret = 0;
+    pending_node = NULL;
+    ret = 0;
 out:
-        if (rsp_dict)
-                dict_unref (rsp_dict);
-        gf_log (this->name, GF_LOG_DEBUG, "Sent op req to %d bricks",
-                brick_count);
-        return ret;
+    if (pending_node && pending_node->node)
+        glusterd_pending_node_put_rpc(pending_node);
+
+    if (rsp_dict)
+        dict_unref(rsp_dict);
+    gf_msg_debug(this->name, 0, "Sent op req to %d bricks", brick_count);
+    return ret;
 }
 
 void
-gd_sync_task_begin (dict_t *op_ctx, rpcsvc_request_t * req)
+gd_sync_task_begin(dict_t *op_ctx, rpcsvc_request_t *req)
 {
-        int                         ret              = -1;
-        int                         op_ret           = -1;
-        int                         npeers           = 0;
-        dict_t                      *req_dict        = NULL;
-        glusterd_conf_t             *conf            = NULL;
-        glusterd_op_t               op               = 0;
-        int32_t                     tmp_op           = 0;
-        char                        *op_errstr       = NULL;
-        char                        *tmp             = NULL;
-        char                        *volname         = NULL;
-        xlator_t                    *this            = NULL;
-        gf_boolean_t                is_acquired      = _gf_false;
-        uuid_t                      *txn_id          = NULL;
-        struct cds_list_head        xaction_peers    = {0,};
-        glusterd_op_info_t          txn_opinfo       = {{0},};
-
-        this = THIS;
-        GF_ASSERT (this);
-        conf = this->private;
-        GF_ASSERT (conf);
-
-        CDS_INIT_LIST_HEAD (&xaction_peers);
-
-        ret = dict_get_int32 (op_ctx, GD_SYNC_OPCODE_KEY, &tmp_op);
+    int ret = -1;
+    int op_ret = -1;
+    dict_t *req_dict = NULL;
+    glusterd_conf_t *conf = NULL;
+    glusterd_op_t op = GD_OP_NONE;
+    int32_t tmp_op = 0;
+    char *op_errstr = NULL;
+    char *tmp = NULL;
+    char *global = NULL;
+    char *volname = NULL;
+    xlator_t *this = NULL;
+    gf_boolean_t is_acquired = _gf_false;
+    gf_boolean_t is_global = _gf_false;
+    uuid_t *txn_id = NULL;
+    glusterd_op_info_t txn_opinfo = {
+        {0},
+    };
+    uint32_t op_errno = 0;
+    gf_boolean_t cluster_lock = _gf_false;
+    uint32_t timeout = 0;
+
+    this = THIS;
+    GF_ASSERT(this);
+    conf = this->private;
+    GF_ASSERT(conf);
+
+    ret = dict_get_int32(op_ctx, GD_SYNC_OPCODE_KEY, &tmp_op);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+               "Failed to get volume "
+               "operation");
+        goto out;
+    }
+    op = tmp_op;
+
+    /* Generate a transaction-id for this operation and
+     * save it in the dict */
+    ret = glusterd_generate_txn_id(op_ctx, &txn_id);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_TRANS_IDGEN_FAIL,
+               "Failed to generate transaction id");
+        goto out;
+    }
+
+    /* Save opinfo for this transaction with the transaction id */
+    glusterd_txn_opinfo_init(&txn_opinfo, NULL, &op, NULL, NULL);
+    ret = glusterd_set_txn_opinfo(txn_id, &txn_opinfo);
+    if (ret)
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_TRANS_OPINFO_SET_FAIL,
+               "Unable to set transaction's opinfo");
+
+    gf_msg_debug(this->name, 0, "Transaction ID : %s", uuid_utoa(*txn_id));
+
+    /* Save the MY_UUID as the originator_uuid */
+    ret = glusterd_set_originator_uuid(op_ctx);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_UUID_SET_FAIL,
+               "Failed to set originator_uuid.");
+        goto out;
+    }
+
+    if (conf->op_version < GD_OP_VERSION_3_6_0)
+        cluster_lock = _gf_true;
+
+    /* Based on the op_version, acquire a cluster or mgmt_v3 lock */
+    if (cluster_lock) {
+        ret = glusterd_lock(MY_UUID);
         if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "Failed to get volume "
-                        "operation");
-                goto out;
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_GLUSTERD_LOCK_FAIL,
+                   "Unable to acquire lock");
+            gf_asprintf(&op_errstr,
+                        "Another transaction is in progress. "
+                        "Please try again after some time.");
+            goto out;
         }
-        op = tmp_op;
+    } else {
+        /* Cli will add timeout key to dict if the default timeout is
+         * other than 2 minutes. Here we use this value to check whether
+         * mgmt_v3_lock_timeout should be set to default value or we
+         * need to change the value according to timeout value
+         * i.e, timeout + 120 seconds. */
+        ret = dict_get_uint32(op_ctx, "timeout", &timeout);
+        if (!ret)
+            conf->mgmt_v3_lock_timeout = timeout + 120;
 
-        /* Generate a transaction-id for this operation and
-         * save it in the dict */
-        ret = glusterd_generate_txn_id (op_ctx, &txn_id);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "Failed to generate transaction id");
-                goto out;
+        ret = dict_get_str(op_ctx, "globalname", &global);
+        if (!ret) {
+            is_global = _gf_true;
+            goto global;
         }
 
-        /* Save opinfo for this transaction with the transaction id */
-        glusterd_txn_opinfo_init (&txn_opinfo, NULL, &op, NULL, NULL);
-        ret = glusterd_set_txn_opinfo (txn_id, &txn_opinfo);
-        if (ret)
-                gf_log (this->name, GF_LOG_ERROR,
-                        "Unable to set transaction's opinfo");
-
-        gf_log (this->name, GF_LOG_DEBUG,
-                "Transaction ID : %s", uuid_utoa (*txn_id));
-
-        /* Save the MY_UUID as the originator_uuid */
-        ret = glusterd_set_originator_uuid (op_ctx);
+        /* If no volname is given as a part of the command, locks will
+         * not be held */
+        ret = dict_get_str(op_ctx, "volname", &tmp);
         if (ret) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "Failed to set originator_uuid.");
+            gf_msg_debug("glusterd", 0,
+                         "Failed to get volume "
+                         "name");
+            goto local_locking_done;
+        } else {
+            /* Use a copy of volname, as cli response will be
+             * sent before the unlock, and the volname in the
+             * dict, might be removed */
+            volname = gf_strdup(tmp);
+            if (!volname)
                 goto out;
         }
 
-        /* Based on the op_version, acquire a cluster or mgmt_v3 lock */
-        if (conf->op_version < GD_OP_VERSION_3_6_0) {
-                ret = glusterd_lock (MY_UUID);
-                if (ret) {
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "Unable to acquire lock");
-                        gf_asprintf (&op_errstr,
-                                     "Another transaction is in progress. "
-                                     "Please try again after sometime.");
-                        goto out;
-                }
-        } else {
-
-                /* If no volname is given as a part of the command, locks will
-                 * not be held */
-                ret = dict_get_str (op_ctx, "volname", &tmp);
-                if (ret) {
-                        gf_log ("", GF_LOG_DEBUG, "Failed to get volume "
-                                "name");
-                        goto local_locking_done;
-                } else {
-                        /* Use a copy of volname, as cli response will be
-                         * sent before the unlock, and the volname in the
-                         * dict, might be removed */
-                        volname = gf_strdup (tmp);
-                        if (!volname)
-                                goto out;
-                }
+        ret = glusterd_mgmt_v3_lock(volname, MY_UUID, &op_errno, "vol");
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_MGMTV3_LOCK_GET_FAIL,
+                   "Unable to acquire lock for %s", volname);
+            gf_asprintf(&op_errstr,
+                        "Another transaction is in progress "
+                        "for %s. Please try again after some time.",
+                        volname);
+            goto out;
+        }
+    }
 
-                ret = glusterd_mgmt_v3_lock (volname, MY_UUID, "vol");
-                if (ret) {
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "Unable to acquire lock for %s", volname);
-                        gf_asprintf (&op_errstr,
-                                     "Another transaction is in progress "
-                                     "for %s. Please try again after sometime.",
-                                     volname);
-                        goto out;
-                }
+global:
+    if (is_global) {
+        ret = glusterd_mgmt_v3_lock(global, MY_UUID, &op_errno, "global");
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_MGMTV3_LOCK_GET_FAIL,
+                   "Unable to acquire lock for %s", global);
+            gf_asprintf(&op_errstr,
+                        "Another transaction is in progress "
+                        "for %s. Please try again after some time.",
+                        global);
+            is_global = _gf_false;
+            goto out;
         }
+    }
 
-        is_acquired = _gf_true;
+    is_acquired = _gf_true;
 
 local_locking_done:
 
-        /* Maintain xaction_peers on per transaction basis */
-        npeers = gd_build_local_xaction_peers_list (&conf->peers,
-                                                    &xaction_peers, op);
-        if (npeers == -1) {
-                gf_log (this->name, GF_LOG_ERROR, "building local peers list "
-                        "failed");
-                goto out;
-        }
-
-        /* If no volname is given as a part of the command, locks will
-         * not be held */
-        if (volname || (conf->op_version < GD_OP_VERSION_3_6_0)) {
-                ret = gd_lock_op_phase (conf, op, op_ctx, &op_errstr,
-                                        npeers, *txn_id, &xaction_peers);
-                if (ret) {
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "Locking Peers Failed.");
-                        goto out;
-                }
-        }
-
-        ret = glusterd_op_build_payload (&req_dict, &op_errstr, op_ctx);
+    /* If no volname is given as a part of the command, locks will
+     * not be held */
+    if (volname || cluster_lock || is_global) {
+        ret = gd_lock_op_phase(conf, op, op_ctx, &op_errstr, *txn_id,
+                               &txn_opinfo, cluster_lock);
         if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, LOGSTR_BUILD_PAYLOAD,
-                        gd_op_list[op]);
-                if (op_errstr == NULL)
-                        gf_asprintf (&op_errstr, OPERRSTR_BUILD_PAYLOAD);
-                goto out;
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_PEER_LOCK_FAIL,
+                   "Locking Peers Failed.");
+            goto out;
         }
+    }
+
+    ret = glusterd_op_build_payload(&req_dict, &op_errstr, op_ctx);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_BRICK_OP_PAYLOAD_BUILD_FAIL,
+               LOGSTR_BUILD_PAYLOAD, gd_op_list[op]);
+        if (op_errstr == NULL)
+            gf_asprintf(&op_errstr, OPERRSTR_BUILD_PAYLOAD);
+        goto out;
+    }
+
+    ret = gd_stage_op_phase(op, op_ctx, req_dict, &op_errstr, &txn_opinfo);
+    if (ret)
+        goto out;
+
+    ret = gd_brick_op_phase(op, op_ctx, req_dict, &op_errstr);
+    if (ret)
+        goto out;
+
+    ret = gd_commit_op_phase(op, op_ctx, req_dict, &op_errstr, &txn_opinfo);
+    if (ret)
+        goto out;
+
+    ret = 0;
+out:
+    op_ret = ret;
+    if (txn_id) {
+        if (global)
+            (void)gd_unlock_op_phase(conf, op, &op_ret, req, op_ctx, op_errstr,
+                                     global, is_acquired, *txn_id, &txn_opinfo,
+                                     cluster_lock);
+        else
+            (void)gd_unlock_op_phase(conf, op, &op_ret, req, op_ctx, op_errstr,
+                                     volname, is_acquired, *txn_id, &txn_opinfo,
+                                     cluster_lock);
 
-        ret = gd_stage_op_phase (&xaction_peers, op, op_ctx, req_dict,
-                                 &op_errstr, npeers);
-        if (ret)
-                goto out;
-
-        ret = gd_brick_op_phase (op, op_ctx, req_dict, &op_errstr);
-        if (ret)
-                goto out;
-
-        ret = gd_commit_op_phase (&xaction_peers, op, op_ctx, req_dict,
-                                  &op_errstr, npeers);
+        /* Clearing the transaction opinfo */
+        ret = glusterd_clear_txn_opinfo(txn_id);
         if (ret)
-                goto out;
-
-        ret = 0;
-out:
-        op_ret = ret;
-        if (txn_id) {
-                (void) gd_unlock_op_phase (conf, op, &op_ret, req,
-                                           op_ctx, op_errstr,
-                                           npeers, volname,
-                                           is_acquired, *txn_id,
-                                           &xaction_peers);
-
-                /* Clearing the transaction opinfo */
-                ret = glusterd_clear_txn_opinfo (txn_id);
-                if (ret)
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "Unable to clear transaction's "
-                                "opinfo for transaction ID : %s",
-                                uuid_utoa (*txn_id));
-        }
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_TRANS_OPINFO_CLEAR_FAIL,
+                   "Unable to clear transaction's "
+                   "opinfo for transaction ID : %s",
+                   uuid_utoa(*txn_id));
+    }
 
-        glusterd_op_send_cli_response (op, op_ret, 0, req, op_ctx, op_errstr);
+    if (op_ret && (op_errno == 0))
+        op_errno = EG_INTRNL;
 
-        gd_cleanup_local_xaction_peers_list (&xaction_peers);
+    glusterd_op_send_cli_response(op, op_ret, op_errno, req, op_ctx, op_errstr);
 
-        if (volname)
-                GF_FREE (volname);
+    if (volname)
+        GF_FREE(volname);
 
-        if (req_dict)
-                dict_unref (req_dict);
+    if (req_dict)
+        dict_unref(req_dict);
 
-        if (op_errstr) {
-                GF_FREE (op_errstr);
-                op_errstr = NULL;
-        }
+    if (op_errstr) {
+        GF_FREE(op_errstr);
+        op_errstr = NULL;
+    }
 
-        return;
+    return;
 }
 
 int32_t
-glusterd_op_begin_synctask (rpcsvc_request_t *req, glusterd_op_t op,
-                            void *dict)
+glusterd_op_begin_synctask(rpcsvc_request_t *req, glusterd_op_t op, void *dict)
 {
-        int              ret = 0;
+    int ret = 0;
 
-        ret = dict_set_int32 (dict, GD_SYNC_OPCODE_KEY, op);
-        if (ret) {
-                gf_log (THIS->name, GF_LOG_ERROR,
-                        "dict set failed for setting operations");
-                goto out;
-        }
+    ret = dict_set_int32(dict, GD_SYNC_OPCODE_KEY, op);
+    if (ret) {
+        gf_msg(THIS->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+               "dict set failed for setting operations");
+        goto out;
+    }
 
-        gd_sync_task_begin (dict, req);
-        ret = 0;
+    gd_sync_task_begin(dict, req);
+    ret = 0;
 out:
 
-        return ret;
+    return ret;
 }
diff --git a/xlators/mgmt/glusterd/src/glusterd-syncop.h b/xlators/mgmt/glusterd/src/glusterd-syncop.h
index d86a5ba2131..a265f2135c6 100644
--- a/xlators/mgmt/glusterd/src/glusterd-syncop.h
+++ b/xlators/mgmt/glusterd/src/glusterd-syncop.h
@@ -10,70 +10,84 @@
 #ifndef __RPC_SYNCOP_H
 #define __RPC_SYNCOP_H
 
-#include "syncop.h"
+#include <glusterfs/syncop.h>
 #include "glusterd-sm.h"
 #include "glusterd.h"
 
 #define GD_SYNC_OPCODE_KEY "sync-mgmt-operation"
 
 /* gd_syncop_* */
-#define GD_SYNCOP(rpc, stb, cookie, cbk, req, prog, procnum, xdrproc) do {     \
-                int ret = 0;                                                   \
-                struct  synctask        *task = NULL;                          \
-                glusterd_conf_t         *conf= THIS->private;                  \
+#define GD_SYNCOP(rpc, stb, cookie, cbk, req, prog, procnum, xdrproc)          \
+    do {                                                                       \
+        int ret = 0;                                                           \
+        struct synctask *task = NULL;                                          \
+        glusterd_conf_t *conf = THIS->private;                                 \
                                                                                \
-                task = synctask_get ();                                        \
-                stb->task = task;                                              \
+        task = synctask_get();                                                 \
+        stb->task = task;                                                      \
                                                                                \
-                /*This is to ensure that the brick_op_cbk is able to           \
-                 * take the big lock*/                                         \
-                synclock_unlock (&conf->big_lock);                             \
-                ret = gd_syncop_submit_request (rpc, req, stb, cookie,         \
-                                                prog, procnum, cbk,            \
-                                                (xdrproc_t)xdrproc);           \
-                if (!ret)                                                      \
-                        synctask_yield (stb->task);                            \
-                synclock_lock (&conf->big_lock);                               \
-        } while (0)
+        /*This is to ensure that the brick_op_cbk is able to                   \
+         * take the big lock*/                                                 \
+        synclock_unlock(&conf->big_lock);                                      \
+        ret = gd_syncop_submit_request(rpc, req, stb, cookie, prog, procnum,   \
+                                       cbk, (xdrproc_t)xdrproc);               \
+        if (!ret)                                                              \
+            synctask_yield(stb->task, NULL);                                   \
+        else                                                                   \
+            gf_asprintf(&stb->errstr,                                          \
+                        "%s failed. Check log file"                            \
+                        " for more details",                                   \
+                        (prog)->progname);                                     \
+        synclock_lock(&conf->big_lock);                                        \
+    } while (0)
 
-int gd_syncop_submit_request (struct rpc_clnt *rpc, void *req, void *local,
-                              void *cookie, rpc_clnt_prog_t *prog, int procnum,
-                              fop_cbk_fn_t cbkfn, xdrproc_t xdrproc);
-int gd_syncop_mgmt_lock (glusterd_peerinfo_t *peerinfo, struct syncargs *arg,
-                         uuid_t my_uuid, uuid_t recv_uuid);
+#define GD_ALLOC_COPY_UUID(dst_ptr, uuid, ret)                                 \
+    do {                                                                       \
+        dst_ptr = GF_MALLOC(sizeof(*dst_ptr), gf_common_mt_uuid_t);            \
+        if (dst_ptr) {                                                         \
+            gf_uuid_copy(*dst_ptr, uuid);                                      \
+            ret = 0;                                                           \
+        } else {                                                               \
+            ret = -1;                                                          \
+        }                                                                      \
+    } while (0)
 
-int gd_syncop_mgmt_unlock (glusterd_peerinfo_t *peerinfo, struct syncargs *arg,
-                           uuid_t my_uuid, uuid_t recv_uuid);
+int32_t
+gd_syncop_brick_op_cbk(struct rpc_req *req, struct iovec *iov, int count,
+                       void *myframe);
 
-int gd_syncop_mgmt_stage_op (glusterd_peerinfo_t *peerinfo,
-                             struct syncargs *arg, uuid_t my_uuid,
-                             uuid_t recv_uuid, int op, dict_t *dict_out,
-                             dict_t *op_ctx);
-
-int gd_syncop_mgmt_commit_op (glusterd_peerinfo_t *peerinfo,
-                              struct syncargs *arg, uuid_t my_uuid,
-                              uuid_t recv_uuid, int op, dict_t *dict_out,
-                              dict_t *op_ctx);
+int
+gd_syncop_submit_request(struct rpc_clnt *rpc, void *req, void *local,
+                         void *cookie, rpc_clnt_prog_t *prog, int procnum,
+                         fop_cbk_fn_t cbkfn, xdrproc_t xdrproc);
+int
+gd_syncop_mgmt_lock(glusterd_peerinfo_t *peerinfo, struct syncargs *arg,
+                    uuid_t my_uuid, uuid_t recv_uuid);
 
-void
-gd_synctask_barrier_wait (struct syncargs *args, int count);
+int
+gd_syncop_mgmt_unlock(glusterd_peerinfo_t *peerinfo, struct syncargs *arg,
+                      uuid_t my_uuid, uuid_t recv_uuid);
 
 int
-gd_build_peers_list (struct cds_list_head *peers,
-                     struct cds_list_head *xact_peers, glusterd_op_t op);
+gd_syncop_mgmt_stage_op(glusterd_peerinfo_t *peerinfo, struct syncargs *arg,
+                        uuid_t my_uuid, uuid_t recv_uuid, int op,
+                        dict_t *dict_out, dict_t *op_ctx);
 
 int
-gd_build_local_xaction_peers_list (struct cds_list_head *peers,
-                                   struct cds_list_head *xact_peers,
-                                   glusterd_op_t op);
+gd_syncop_mgmt_commit_op(glusterd_peerinfo_t *peerinfo, struct syncargs *arg,
+                         uuid_t my_uuid, uuid_t recv_uuid, int op,
+                         dict_t *dict_out, dict_t *op_ctx);
+
+void
+gd_synctask_barrier_wait(struct syncargs *args, int count);
 
 int
-gd_brick_op_phase (glusterd_op_t op, dict_t *op_ctx, dict_t *req_dict,
-                   char **op_errstr);
+gd_brick_op_phase(glusterd_op_t op, dict_t *op_ctx, dict_t *req_dict,
+                  char **op_errstr);
 
 int
-glusterd_syncop_aggr_rsp_dict (glusterd_op_t op, dict_t *aggr, dict_t *rsp);
+glusterd_syncop_aggr_rsp_dict(glusterd_op_t op, dict_t *aggr, dict_t *rsp);
 
 void
-gd_syncargs_init (struct syncargs *args, dict_t *op_ctx);
+gd_syncargs_init(struct syncargs *args, dict_t *op_ctx);
 #endif /* __RPC_SYNCOP_H */
diff --git a/xlators/mgmt/glusterd/src/glusterd-tierd-svc-helper.c b/xlators/mgmt/glusterd/src/glusterd-tierd-svc-helper.c
new file mode 100644
index 00000000000..035795b3deb
--- /dev/null
+++ b/xlators/mgmt/glusterd/src/glusterd-tierd-svc-helper.c
@@ -0,0 +1,207 @@
+/*
+   Copyright (c) 2016 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
+
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
+
+#include "glusterd.h"
+#include "glusterd-utils.h"
+#include "glusterd-tierd-svc-helper.h"
+#include "glusterd-messages.h"
+#include <glusterfs/syscall.h>
+#include "glusterd-volgen.h"
+
+void
+glusterd_svc_build_tierd_rundir(glusterd_volinfo_t *volinfo, char *path,
+                                int path_len)
+{
+    char workdir[PATH_MAX] = {
+        0,
+    };
+    glusterd_conf_t *priv = THIS->private;
+
+    GLUSTERD_GET_TIER_DIR(workdir, volinfo, priv);
+
+    snprintf(path, path_len, "%s/run", workdir);
+}
+
+void
+glusterd_svc_build_tierd_socket_filepath(glusterd_volinfo_t *volinfo,
+                                         char *path, int path_len)
+{
+    char sockfilepath[PATH_MAX] = {
+        0,
+    };
+    char rundir[PATH_MAX] = {
+        0,
+    };
+    int32_t len = 0;
+
+    glusterd_svc_build_tierd_rundir(volinfo, rundir, sizeof(rundir));
+    len = snprintf(sockfilepath, sizeof(sockfilepath), "%s/run-%s", rundir,
+                   uuid_utoa(MY_UUID));
+    if ((len < 0) || (len >= sizeof(sockfilepath))) {
+        sockfilepath[0] = 0;
+    }
+
+    glusterd_set_socket_filepath(sockfilepath, path, path_len);
+}
+
+void
+glusterd_svc_build_tierd_pidfile(glusterd_volinfo_t *volinfo, char *path,
+                                 int path_len)
+{
+    char rundir[PATH_MAX] = {
+        0,
+    };
+
+    glusterd_svc_build_tierd_rundir(volinfo, rundir, sizeof(rundir));
+
+    snprintf(path, path_len, "%s/%s-tierd.pid", rundir, volinfo->volname);
+}
+
+void
+glusterd_svc_build_tierd_volfile_path(glusterd_volinfo_t *volinfo, char *path,
+                                      int path_len)
+{
+    char workdir[PATH_MAX] = {
+        0,
+    };
+    glusterd_conf_t *priv = THIS->private;
+
+    GLUSTERD_GET_VOLUME_DIR(workdir, volinfo, priv);
+
+    snprintf(path, path_len, "%s/%s-tierd.vol", workdir, volinfo->volname);
+}
+
+void
+glusterd_svc_build_tierd_logdir(char *logdir, char *volname, size_t len)
+{
+    glusterd_conf_t *conf = THIS->private;
+    snprintf(logdir, len, "%s/tier/%s", priv->logdir, volname);
+}
+
+void
+glusterd_svc_build_tierd_logfile(char *logfile, char *logdir, size_t len)
+{
+    snprintf(logfile, len, "%s/tierd.log", logdir);
+}
+
+int
+glusterd_svc_check_tier_volfile_identical(char *svc_name,
+                                          glusterd_volinfo_t *volinfo,
+                                          gf_boolean_t *identical)
+{
+    char orgvol[PATH_MAX] = {
+        0,
+    };
+    char *tmpvol = NULL;
+    xlator_t *this = NULL;
+    int ret = -1;
+    int need_unlink = 0;
+    int tmp_fd = -1;
+
+    this = THIS;
+
+    GF_VALIDATE_OR_GOTO(THIS->name, this, out);
+    GF_VALIDATE_OR_GOTO(this->name, identical, out);
+
+    glusterd_svc_build_tierd_volfile_path(volinfo, orgvol, sizeof(orgvol));
+
+    ret = gf_asprintf(&tmpvol, "/tmp/g%s-XXXXXX", svc_name);
+    if (ret < 0) {
+        goto out;
+    }
+
+    /* coverity[SECURE_TEMP] mkstemp uses 0600 as the mode and is safe */
+    tmp_fd = mkstemp(tmpvol);
+    if (tmp_fd < 0) {
+        gf_msg(this->name, GF_LOG_WARNING, errno, GD_MSG_FILE_OP_FAILED,
+               "Unable to create temp file"
+               " %s:(%s)",
+               tmpvol, strerror(errno));
+        ret = -1;
+        goto out;
+    }
+
+    need_unlink = 1;
+    ret = build_rebalance_volfile(volinfo, tmpvol, NULL);
+    if (ret)
+        goto out;
+
+    ret = glusterd_check_files_identical(orgvol, tmpvol, identical);
+    if (ret)
+        goto out;
+
+out:
+    if (need_unlink)
+        sys_unlink(tmpvol);
+
+    if (tmpvol != NULL)
+        GF_FREE(tmpvol);
+
+    if (tmp_fd >= 0)
+        sys_close(tmp_fd);
+
+    return ret;
+}
+
+int
+glusterd_svc_check_tier_topology_identical(char *svc_name,
+                                           glusterd_volinfo_t *volinfo,
+                                           gf_boolean_t *identical)
+{
+    char orgvol[PATH_MAX] = {
+        0,
+    };
+    char *tmpvol = NULL;
+    glusterd_conf_t *conf = NULL;
+    xlator_t *this = THIS;
+    int ret = -1;
+    int tmpclean = 0;
+    int tmpfd = -1;
+
+    if ((!identical) || (!this) || (!this->private))
+        goto out;
+
+    conf = this->private;
+    GF_VALIDATE_OR_GOTO(this->name, conf, out);
+
+    glusterd_svc_build_tierd_volfile_path(volinfo, orgvol, sizeof(orgvol));
+
+    ret = gf_asprintf(&tmpvol, "/tmp/g%s-XXXXXX", svc_name);
+    if (ret < 0) {
+        goto out;
+    }
+
+    /* coverity[SECURE_TEMP] mkstemp uses 0600 as the mode and is safe */
+    tmpfd = mkstemp(tmpvol);
+    if (tmpfd < 0) {
+        gf_msg(this->name, GF_LOG_WARNING, errno, GD_MSG_FILE_OP_FAILED,
+               "Unable to create temp file"
+               " %s:(%s)",
+               tmpvol, strerror(errno));
+        ret = -1;
+        goto out;
+    }
+
+    tmpclean = 1; /* SET the flag to unlink() tmpfile */
+    ret = build_rebalance_volfile(volinfo, tmpvol, NULL);
+    if (ret)
+        goto out;
+
+    /* Compare the topology of volfiles */
+    ret = glusterd_check_topology_identical(orgvol, tmpvol, identical);
+out:
+    if (tmpfd >= 0)
+        sys_close(tmpfd);
+    if (tmpclean)
+        sys_unlink(tmpvol);
+    if (tmpvol != NULL)
+        GF_FREE(tmpvol);
+    return ret;
+}
diff --git a/xlators/mgmt/glusterd/src/glusterd-utils.c b/xlators/mgmt/glusterd/src/glusterd-utils.c
index aa27ebb18d3..90ef2cf4c9c 100644
--- a/xlators/mgmt/glusterd/src/glusterd-utils.c
+++ b/xlators/mgmt/glusterd/src/glusterd-utils.c
@@ -7,10 +7,6 @@
    later), or the GNU General Public License, version 2 (GPLv2), in all
    cases as published by the Free Software Foundation.
 */
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
 #include <inttypes.h>
 
 #if defined(GF_LINUX_HOST_OS)
@@ -24,21 +20,20 @@
 #include <libxml/xmlwriter.h>
 #endif
 
-#include "globals.h"
-#include "glusterfs.h"
-#include "compat.h"
-#include "dict.h"
-#include "xlator.h"
-#include "logging.h"
+#include <glusterfs/glusterfs.h>
+#include <glusterfs/compat.h>
+#include <glusterfs/dict.h>
+#include <glusterfs/xlator.h>
+#include <glusterfs/logging.h>
 #include "glusterd-messages.h"
-#include "timer.h"
-#include "defaults.h"
-#include "compat.h"
-#include "syncop.h"
-#include "run.h"
-#include "compat-errno.h"
-#include "statedump.h"
-#include "syscall.h"
+#include <glusterfs/timer.h>
+#include <glusterfs/defaults.h>
+#include <glusterfs/compat.h>
+#include <glusterfs/syncop.h>
+#include <glusterfs/run.h>
+#include <glusterfs/compat-errno.h>
+#include <glusterfs/statedump.h>
+#include <glusterfs/syscall.h>
 #include "glusterd-mem-types.h"
 #include "glusterd.h"
 #include "glusterd-op-sm.h"
@@ -48,8 +43,9 @@
 #include "glusterd-store.h"
 #include "glusterd-volgen.h"
 #include "glusterd-pmap.h"
-#include "glusterfs-acl.h"
+#include <glusterfs/glusterfs-acl.h>
 #include "glusterd-syncop.h"
+#include "glusterd-mgmt.h"
 #include "glusterd-locks.h"
 #include "glusterd-messages.h"
 #include "glusterd-volgen.h"
@@ -57,10 +53,14 @@
 #include "glusterd-svc-mgmt.h"
 #include "glusterd-svc-helper.h"
 #include "glusterd-shd-svc.h"
-#include "glusterd-nfs-svc.h"
 #include "glusterd-quotad-svc.h"
 #include "glusterd-snapd-svc.h"
 #include "glusterd-bitd-svc.h"
+#include "glusterd-gfproxyd-svc.h"
+#include "glusterd-server-quorum.h"
+#include <glusterfs/quota-common-utils.h>
+#include <glusterfs/common-utils.h>
+#include "glusterd-shd-svc-helper.h"
 
 #include "xdr-generic.h"
 #include <sys/resource.h>
@@ -74,449 +74,630 @@
 #include <fnmatch.h>
 #include <sys/statvfs.h>
 #include <ifaddrs.h>
-#ifdef HAVE_BD_XLATOR
-#include <lvm2app.h>
-#endif
 
 #ifdef GF_SOLARIS_HOST_OS
 #include <sys/sockio.h>
 #endif
 
-#define NFS_PROGRAM         100003
-#define NFSV3_VERSION       3
+#ifdef __FreeBSD__
+#include <sys/sysctl.h>
+#include <sys/param.h>
+#include <sys/queue.h>
+#include <libprocstat.h>
+#include <libutil.h>
+#endif
+
+#define NFS_PROGRAM 100003
+#define NFSV3_VERSION 3
+
+#define MOUNT_PROGRAM 100005
+#define MOUNTV3_VERSION 3
+#define MOUNTV1_VERSION 1
+
+#define NLM_PROGRAM 100021
+#define NLMV4_VERSION 4
+#define NLMV1_VERSION 1
+
+#ifdef BUILD_GNFS
+#define GLUSTERD_GET_NFS_PIDFILE(pidfile, priv)                                \
+    do {                                                                       \
+        int32_t _nfs_pid_len;                                                  \
+        _nfs_pid_len = snprintf(pidfile, PATH_MAX, "%s/nfs/nfs.pid",           \
+                                priv->rundir);                                 \
+        if ((_nfs_pid_len < 0) || (_nfs_pid_len >= PATH_MAX)) {                \
+            pidfile[0] = 0;                                                    \
+        }                                                                      \
+    } while (0)
+#endif
+
+#define GLUSTERD_GET_QUOTAD_PIDFILE(pidfile, priv)                             \
+    do {                                                                       \
+        int32_t _quotad_pid_len;                                               \
+        _quotad_pid_len = snprintf(pidfile, PATH_MAX, "%s/quotad/quotad.pid",  \
+                                   priv->rundir);                              \
+        if ((_quotad_pid_len < 0) || (_quotad_pid_len >= PATH_MAX)) {          \
+            pidfile[0] = 0;                                                    \
+        }                                                                      \
+    } while (0)
+
+gf_boolean_t
+is_brick_mx_enabled(void)
+{
+    char *value = NULL;
+    int ret = 0;
+    gf_boolean_t enabled = _gf_false;
+    xlator_t *this = NULL;
+    glusterd_conf_t *priv = NULL;
+
+    this = THIS;
+
+    priv = this->private;
+
+    ret = dict_get_strn(priv->opts, GLUSTERD_BRICK_MULTIPLEX_KEY,
+                        SLEN(GLUSTERD_BRICK_MULTIPLEX_KEY), &value);
+
+    if (!ret)
+        ret = gf_string2boolean(value, &enabled);
+
+    return ret ? _gf_false : enabled;
+}
+
+int
+get_mux_limit_per_process(int *mux_limit)
+{
+    char *value = NULL;
+    int ret = -1;
+    int max_bricks_per_proc = 0;
+    xlator_t *this = NULL;
+    glusterd_conf_t *priv = NULL;
+
+    this = THIS;
+    GF_VALIDATE_OR_GOTO("glusterd", this, out);
+
+    priv = this->private;
+    GF_VALIDATE_OR_GOTO(this->name, priv, out);
+
+    if (!is_brick_mx_enabled()) {
+        max_bricks_per_proc = 1;
+        ret = 0;
+        goto out;
+    }
+
+    ret = dict_get_strn(priv->opts, GLUSTERD_BRICKMUX_LIMIT_KEY,
+                        SLEN(GLUSTERD_BRICKMUX_LIMIT_KEY), &value);
+    if (ret) {
+        value = GLUSTERD_BRICKMUX_LIMIT_DFLT_VALUE;
+    }
+    ret = gf_string2int(value, &max_bricks_per_proc);
+    if (ret)
+        goto out;
+
+out:
+    *mux_limit = max_bricks_per_proc;
+
+    gf_msg_debug("glusterd", 0, "Mux limit set to %d bricks per process",
+                 *mux_limit);
+
+    return ret;
+}
+
+int
+get_gd_vol_thread_limit(int *thread_limit)
+{
+    char *value = NULL;
+    int ret = -1;
+    int vol_per_thread_limit = 0;
+    xlator_t *this = NULL;
+    glusterd_conf_t *priv = NULL;
 
-#define MOUNT_PROGRAM       100005
-#define MOUNTV3_VERSION     3
-#define MOUNTV1_VERSION     1
+    this = THIS;
+    GF_VALIDATE_OR_GOTO("glusterd", this, out);
 
-#define NLM_PROGRAM         100021
-#define NLMV4_VERSION       4
-#define NLMV1_VERSION       1
+    priv = this->private;
+    GF_VALIDATE_OR_GOTO(this->name, priv, out);
+
+    if (!is_brick_mx_enabled()) {
+        vol_per_thread_limit = 1;
+        ret = 0;
+        goto out;
+    }
+
+    ret = dict_get_strn(priv->opts, GLUSTERD_VOL_CNT_PER_THRD,
+                        SLEN(GLUSTERD_VOL_CNT_PER_THRD), &value);
+    if (ret) {
+        value = GLUSTERD_VOL_CNT_PER_THRD_DEFAULT_VALUE;
+    }
+    ret = gf_string2int(value, &vol_per_thread_limit);
+    if (ret)
+        goto out;
+
+out:
+    *thread_limit = vol_per_thread_limit;
+
+    gf_msg_debug("glusterd", 0,
+                 "Per Thread volume limit set to %d glusterd to populate dict "
+                 "data parallel",
+                 *thread_limit);
+
+    return ret;
+}
 
 extern struct volopt_map_entry glusterd_volopt_map[];
+extern glusterd_all_vol_opts valid_all_vol_opts[];
 
 static glusterd_lock_t lock;
 
+static int
+_brick_for_each(glusterd_volinfo_t *volinfo, dict_t *mod_dict, void *data,
+                int (*fn)(glusterd_volinfo_t *, glusterd_brickinfo_t *,
+                          dict_t *mod_dict, void *))
+{
+    int ret = 0;
+    glusterd_brickinfo_t *brickinfo = NULL;
+    xlator_t *this = THIS;
+
+    cds_list_for_each_entry(brickinfo, &volinfo->bricks, brick_list)
+    {
+        gf_msg_debug(this->name, 0, "Found a brick - %s:%s",
+                     brickinfo->hostname, brickinfo->path);
+        ret = fn(volinfo, brickinfo, mod_dict, data);
+        if (ret)
+            goto out;
+    }
+out:
+    return ret;
+}
+
+/* This is going to be a O(n^2) operation as we have to pick a brick,
+   make sure it belong to this machine, and compare another brick belonging
+   to this machine (if exists), is sharing the backend */
+static void
+gd_set_shared_brick_count(glusterd_volinfo_t *volinfo)
+{
+    glusterd_brickinfo_t *brickinfo = NULL;
+    glusterd_brickinfo_t *trav = NULL;
+
+    cds_list_for_each_entry(brickinfo, &volinfo->bricks, brick_list)
+    {
+        if (gf_uuid_compare(brickinfo->uuid, MY_UUID))
+            continue;
+        brickinfo->fs_share_count = 0;
+        cds_list_for_each_entry(trav, &volinfo->bricks, brick_list)
+        {
+            if (!gf_uuid_compare(trav->uuid, MY_UUID) &&
+                (trav->statfs_fsid == brickinfo->statfs_fsid)) {
+                brickinfo->fs_share_count++;
+            }
+        }
+    }
+
+    return;
+}
+
+int
+glusterd_volume_brick_for_each(glusterd_volinfo_t *volinfo, void *data,
+                               int (*fn)(glusterd_volinfo_t *,
+                                         glusterd_brickinfo_t *,
+                                         dict_t *mod_dict, void *))
+{
+    gd_set_shared_brick_count(volinfo);
+
+    return _brick_for_each(volinfo, NULL, data, fn);
+}
 
 int32_t
-glusterd_get_lock_owner (uuid_t *uuid)
+glusterd_get_lock_owner(uuid_t *uuid)
 {
-        gf_uuid_copy (*uuid, lock.owner) ;
-        return 0;
+    gf_uuid_copy(*uuid, lock.owner);
+    return 0;
 }
 
 static int32_t
-glusterd_set_lock_owner (uuid_t owner)
+glusterd_set_lock_owner(uuid_t owner)
 {
-        gf_uuid_copy (lock.owner, owner);
-        //TODO: set timestamp
-        return 0;
+    gf_uuid_copy(lock.owner, owner);
+    // TODO: set timestamp
+    return 0;
 }
 
 static int32_t
-glusterd_unset_lock_owner (uuid_t owner)
+glusterd_unset_lock_owner(uuid_t owner)
 {
-        gf_uuid_clear (lock.owner);
-        //TODO: set timestamp
-        return 0;
+    gf_uuid_clear(lock.owner);
+    // TODO: set timestamp
+    return 0;
 }
 
 gf_boolean_t
-glusterd_is_fuse_available ()
+glusterd_is_fuse_available()
 {
-
-        int     fd = 0;
+    int fd = 0;
 
 #ifdef __NetBSD__
-	fd = open ("/dev/puffs", O_RDWR);
+    fd = open("/dev/puffs", O_RDWR);
 #else
-        fd = open ("/dev/fuse", O_RDWR);
+    fd = open("/dev/fuse", O_RDWR);
 #endif
 
-        if (fd > -1 && !close (fd))
-                return _gf_true;
-        else
-                return _gf_false;
+    if (fd > -1 && !sys_close(fd))
+        return _gf_true;
+    else
+        return _gf_false;
 }
 
 int32_t
-glusterd_lock (uuid_t   uuid)
+glusterd_lock(uuid_t uuid)
 {
+    uuid_t owner;
+    char new_owner_str[50] = "";
+    char owner_str[50] = "";
+    int ret = -1;
+    xlator_t *this = NULL;
 
-        uuid_t  owner;
-        char    new_owner_str[50];
-        char    owner_str[50];
-        int     ret = -1;
-        xlator_t *this = NULL;
-
-        this = THIS;
-        GF_ASSERT (this);
+    this = THIS;
+    GF_ASSERT(this);
 
-        GF_ASSERT (uuid);
+    GF_ASSERT(uuid);
 
-        glusterd_get_lock_owner (&owner);
+    glusterd_get_lock_owner(&owner);
 
-        if (!gf_uuid_is_null (owner)) {
-                gf_log (this->name, GF_LOG_ERROR, "Unable to get lock"
-                        " for uuid: %s, lock held by: %s",
-                        uuid_utoa_r (uuid, new_owner_str),
-                        uuid_utoa_r (owner, owner_str));
-                goto out;
-        }
+    if (!gf_uuid_is_null(owner)) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_GLUSTERD_LOCK_FAIL,
+               "Unable to get lock"
+               " for uuid: %s, lock held by: %s",
+               uuid_utoa_r(uuid, new_owner_str), uuid_utoa_r(owner, owner_str));
+        goto out;
+    }
 
-        ret = glusterd_set_lock_owner (uuid);
+    ret = glusterd_set_lock_owner(uuid);
 
-        if (!ret) {
-                gf_log (this->name, GF_LOG_DEBUG, "Cluster lock held by"
-                         " %s", uuid_utoa (uuid));
-        }
+    if (!ret) {
+        gf_msg_debug(this->name, 0,
+                     "Cluster lock held by"
+                     " %s",
+                     uuid_utoa(uuid));
+    }
 
 out:
-        return ret;
+    return ret;
 }
 
-
 int32_t
-glusterd_unlock (uuid_t uuid)
+glusterd_unlock(uuid_t uuid)
 {
-        uuid_t  owner;
-        char    new_owner_str[50];
-        char    owner_str[50];
-        int32_t ret = -1;
-        xlator_t *this = NULL;
+    uuid_t owner;
+    char new_owner_str[50] = "";
+    char owner_str[50] = "";
+    int32_t ret = -1;
+    xlator_t *this = NULL;
 
-        this = THIS;
-        GF_ASSERT (this);
+    this = THIS;
+    GF_ASSERT(this);
 
-        GF_ASSERT (uuid);
+    GF_ASSERT(uuid);
 
-        glusterd_get_lock_owner (&owner);
+    glusterd_get_lock_owner(&owner);
 
-        if (gf_uuid_is_null (owner)) {
-                gf_log (this->name, GF_LOG_ERROR, "Cluster lock not held!");
-                goto out;
-        }
+    if (gf_uuid_is_null(owner)) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_GLUSTERD_LOCK_FAIL,
+               "Cluster lock not held!");
+        goto out;
+    }
 
-        ret = gf_uuid_compare (uuid, owner);
+    ret = gf_uuid_compare(uuid, owner);
 
-        if (ret) {
-               gf_log (this->name, GF_LOG_ERROR, "Cluster lock held by %s ,"
-                       "unlock req from %s!", uuid_utoa_r (owner ,owner_str)
-                        , uuid_utoa_r (uuid, new_owner_str));
-               goto out;
-        }
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_GLUSTERD_LOCK_FAIL,
+               "Cluster lock held by %s ,"
+               "unlock req from %s!",
+               uuid_utoa_r(owner, owner_str), uuid_utoa_r(uuid, new_owner_str));
+        goto out;
+    }
 
-        ret = glusterd_unset_lock_owner (uuid);
+    ret = glusterd_unset_lock_owner(uuid);
 
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "Unable to clear cluster "
-                        "lock");
-                goto out;
-        }
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_GLUSTERD_UNLOCK_FAIL,
+               "Unable to clear cluster "
+               "lock");
+        goto out;
+    }
 
-        ret = 0;
+    ret = 0;
 
 out:
-        return ret;
+    return ret;
 }
 
-
 int
-glusterd_get_uuid (uuid_t *uuid)
+glusterd_get_uuid(uuid_t *uuid)
 {
-        glusterd_conf_t         *priv = NULL;
+    glusterd_conf_t *priv = NULL;
 
-        priv = THIS->private;
+    priv = THIS->private;
 
-        GF_ASSERT (priv);
+    GF_ASSERT(priv);
 
-        gf_uuid_copy (*uuid, MY_UUID);
+    gf_uuid_copy(*uuid, MY_UUID);
 
-        return 0;
+    return 0;
 }
 
 int
-glusterd_submit_request_unlocked (struct rpc_clnt *rpc, void *req,
-                                  call_frame_t *frame, rpc_clnt_prog_t *prog,
-                                  int procnum, struct iobref *iobref,
-                                  xlator_t *this, fop_cbk_fn_t cbkfn,
-                                  xdrproc_t xdrproc)
-{
-        char                    new_iobref  = 0;
-        int                     ret         = -1;
-        int                     count       = 0;
-        ssize_t                 req_size    = 0;
-        struct iobuf           *iobuf       = NULL;
-        struct iovec            iov         = {0, };
-
-        GF_ASSERT (rpc);
-        GF_ASSERT (this);
-
-        if (req) {
-                req_size = xdr_sizeof (xdrproc, req);
-                iobuf = iobuf_get2 (this->ctx->iobuf_pool, req_size);
-                if (!iobuf) {
-                        goto out;
-                };
+glusterd_submit_request(struct rpc_clnt *rpc, void *req, call_frame_t *frame,
+                        rpc_clnt_prog_t *prog, int procnum,
+                        struct iobref *iobref, xlator_t *this,
+                        fop_cbk_fn_t cbkfn, xdrproc_t xdrproc)
+{
+    char new_iobref = 0;
+    int ret = -1;
+    int count = 0;
+    ssize_t req_size = 0;
+    struct iobuf *iobuf = NULL;
+    struct iovec iov = {
+        0,
+    };
+
+    GF_ASSERT(rpc);
+    GF_ASSERT(this);
+
+    if (req) {
+        req_size = xdr_sizeof(xdrproc, req);
+        iobuf = iobuf_get2(this->ctx->iobuf_pool, req_size);
+        if (!iobuf) {
+            goto out;
+        };
 
-                if (!iobref) {
-                        iobref = iobref_new ();
-                        if (!iobref) {
-                                goto out;
-                        }
+        if (!iobref) {
+            iobref = iobref_new();
+            if (!iobref) {
+                gf_smsg("glusterd", GF_LOG_ERROR, ENOMEM, GD_MSG_NO_MEMORY,
+                        NULL);
+                goto out;
+            }
 
-                        new_iobref = 1;
-                }
+            new_iobref = 1;
+        }
 
-                iobref_add (iobref, iobuf);
+        iobref_add(iobref, iobuf);
 
-                iov.iov_base = iobuf->ptr;
-                iov.iov_len  = iobuf_pagesize (iobuf);
+        iov.iov_base = iobuf->ptr;
+        iov.iov_len = iobuf_pagesize(iobuf);
 
-                /* Create the xdr payload */
-                ret = xdr_serialize_generic (iov, req, xdrproc);
-                if (ret == -1) {
-                        goto out;
-                }
-                iov.iov_len = ret;
-                count = 1;
+        /* Create the xdr payload */
+        ret = xdr_serialize_generic(iov, req, xdrproc);
+        if (ret == -1) {
+            goto out;
         }
-
-        /* Send the msg */
-        rpc_clnt_submit (rpc, prog, procnum, cbkfn, &iov, count, NULL, 0,
-                         iobref, frame, NULL, 0, NULL, 0, NULL);
-
-        /* Unconditionally set ret to 0 here. This is to guard against a double
-         * STACK_DESTROY in case of a failure in rpc_clnt_submit AFTER the
-         * request is sent over the wire: once in the callback function of the
-         * request and once in the error codepath of some of the callers of
-         * glusterd_submit_request().
-         */
-        ret = 0;
+        iov.iov_len = ret;
+        count = 1;
+    }
+
+    /* Send the msg */
+    rpc_clnt_submit(rpc, prog, procnum, cbkfn, &iov, count, NULL, 0, iobref,
+                    frame, NULL, 0, NULL, 0, NULL);
+
+    /* Unconditionally set ret to 0 here. This is to guard against a double
+     * STACK_DESTROY in case of a failure in rpc_clnt_submit AFTER the
+     * request is sent over the wire: once in the callback function of the
+     * request and once in the error codepath of some of the callers of
+     * glusterd_submit_request().
+     */
+    ret = 0;
 out:
-        if (new_iobref) {
-                iobref_unref (iobref);
-        }
+    if (new_iobref) {
+        iobref_unref(iobref);
+    }
 
-        iobuf_unref (iobuf);
+    iobuf_unref(iobuf);
 
-        return ret;
+    return ret;
 }
 
-
-int
-glusterd_submit_request (struct rpc_clnt *rpc, void *req,
-                         call_frame_t *frame, rpc_clnt_prog_t *prog,
-                         int procnum, struct iobref *iobref,
-                         xlator_t *this, fop_cbk_fn_t cbkfn, xdrproc_t xdrproc)
-{
-        glusterd_conf_t         *priv = THIS->private;
-        int ret = -1;
-
-        synclock_unlock (&priv->big_lock);
-        {
-                ret = glusterd_submit_request_unlocked (rpc, req, frame, prog,
-                                                        procnum, iobref, this,
-                                                        cbkfn, xdrproc);
-        }
-        synclock_lock (&priv->big_lock);
-
-        return ret;
-}
-
-
 struct iobuf *
-glusterd_serialize_reply (rpcsvc_request_t *req, void *arg,
-                          struct iovec *outmsg, xdrproc_t xdrproc)
-{
-        struct iobuf            *iob = NULL;
-        ssize_t                  retlen = -1;
-        ssize_t                  rsp_size = 0;
-
-        /* First, get the io buffer into which the reply in arg will
-         * be serialized.
-         */
-        rsp_size = xdr_sizeof (xdrproc, arg);
-        iob = iobuf_get2 (req->svc->ctx->iobuf_pool, rsp_size);
-        if (!iob) {
-                gf_log ("", GF_LOG_ERROR, "Failed to get iobuf");
-                goto ret;
-        }
-
-        iobuf_to_iovec (iob, outmsg);
-        /* Use the given serializer to translate the give C structure in arg
-         * to XDR format which will be written into the buffer in outmsg.
-         */
-        /* retlen is used to received the error since size_t is unsigned and we
-         * need -1 for error notification during encoding.
-         */
-        retlen = xdr_serialize_generic (*outmsg, arg, xdrproc);
-        if (retlen == -1) {
-                gf_log ("", GF_LOG_ERROR, "Failed to encode message");
-                goto ret;
-        }
-
-        outmsg->iov_len = retlen;
+glusterd_serialize_reply(rpcsvc_request_t *req, void *arg, struct iovec *outmsg,
+                         xdrproc_t xdrproc)
+{
+    struct iobuf *iob = NULL;
+    ssize_t retlen = -1;
+    ssize_t rsp_size = 0;
+
+    /* First, get the io buffer into which the reply in arg will
+     * be serialized.
+     */
+    rsp_size = xdr_sizeof(xdrproc, arg);
+    iob = iobuf_get2(req->svc->ctx->iobuf_pool, rsp_size);
+    if (!iob) {
+        gf_msg("glusterd", GF_LOG_ERROR, ENOMEM, GD_MSG_NO_MEMORY,
+               "Failed to get iobuf");
+        goto ret;
+    }
+
+    iobuf_to_iovec(iob, outmsg);
+    /* Use the given serializer to translate the give C structure in arg
+     * to XDR format which will be written into the buffer in outmsg.
+     */
+    /* retlen is used to received the error since size_t is unsigned and we
+     * need -1 for error notification during encoding.
+     */
+    retlen = xdr_serialize_generic(*outmsg, arg, xdrproc);
+    if (retlen == -1) {
+        gf_msg("glusterd", GF_LOG_ERROR, 0, GD_MSG_ENCODE_FAIL,
+               "Failed to encode message");
+        goto ret;
+    }
+
+    outmsg->iov_len = retlen;
 ret:
-        if (retlen == -1) {
-                iobuf_unref (iob);
-                iob = NULL;
-        }
+    if (retlen == -1) {
+        iobuf_unref(iob);
+        iob = NULL;
+    }
 
-        return iob;
+    return iob;
 }
 
 int
-glusterd_submit_reply (rpcsvc_request_t *req, void *arg,
-                       struct iovec *payload, int payloadcount,
-                       struct iobref *iobref, xdrproc_t xdrproc)
-{
-        struct iobuf           *iob        = NULL;
-        int                     ret        = -1;
-        struct iovec            rsp        = {0,};
-        char                    new_iobref = 0;
-
-        if (!req) {
-                GF_ASSERT (req);
-                goto out;
-        }
-
+glusterd_submit_reply(rpcsvc_request_t *req, void *arg, struct iovec *payload,
+                      int payloadcount, struct iobref *iobref,
+                      xdrproc_t xdrproc)
+{
+    struct iobuf *iob = NULL;
+    int ret = -1;
+    struct iovec rsp = {
+        0,
+    };
+    char new_iobref = 0;
+
+    if (!req) {
+        GF_ASSERT(req);
+        goto out;
+    }
+
+    if (!iobref) {
+        iobref = iobref_new();
         if (!iobref) {
-                iobref = iobref_new ();
-                if (!iobref) {
-                        gf_log ("", GF_LOG_ERROR, "out of memory");
-                        goto out;
-                }
-
-                new_iobref = 1;
+            gf_msg("glusterd", GF_LOG_ERROR, ENOMEM, GD_MSG_NO_MEMORY,
+                   "out of memory");
+            goto out;
         }
 
-        iob = glusterd_serialize_reply (req, arg, &rsp, xdrproc);
-        if (!iob) {
-                gf_log ("", GF_LOG_ERROR, "Failed to serialize reply");
-        } else {
-                iobref_add (iobref, iob);
-        }
+        new_iobref = 1;
+    }
 
-        ret = rpcsvc_submit_generic (req, &rsp, 1, payload, payloadcount,
-                                     iobref);
+    iob = glusterd_serialize_reply(req, arg, &rsp, xdrproc);
+    if (!iob) {
+        gf_msg("glusterd", GF_LOG_ERROR, 0, GD_MSG_SERIALIZE_MSG_FAIL,
+               "Failed to serialize reply");
+    } else {
+        iobref_add(iobref, iob);
+    }
 
-        /* Now that we've done our job of handing the message to the RPC layer
-         * we can safely unref the iob in the hope that RPC layer must have
-         * ref'ed the iob on receiving into the txlist.
-         */
-        if (ret == -1) {
-                gf_log ("", GF_LOG_ERROR, "Reply submission failed");
-                goto out;
-        }
+    ret = rpcsvc_submit_generic(req, &rsp, 1, payload, payloadcount, iobref);
 
-        ret = 0;
-out:
-
-        if (new_iobref) {
-                iobref_unref (iobref);
-        }
+    /* Now that we've done our job of handing the message to the RPC layer
+     * we can safely unref the iob in the hope that RPC layer must have
+     * ref'ed the iob on receiving into the txlist.
+     */
+    if (ret == -1) {
+        gf_msg("glusterd", GF_LOG_ERROR, 0, GD_MSG_REPLY_SUBMIT_FAIL,
+               "Reply submission failed");
+        goto out;
+    }
 
-        if (iob)
-                iobuf_unref (iob);
-        return ret;
-}
+    ret = 0;
+out:
 
-gf_boolean_t
-glusterd_check_volume_exists (char *volname)
-{
-        glusterd_volinfo_t *volinfo = NULL;
+    if (new_iobref) {
+        iobref_unref(iobref);
+    }
 
-        return (glusterd_volinfo_find (volname, &volinfo) == 0);
+    if (iob)
+        iobuf_unref(iob);
+    return ret;
 }
 
 glusterd_volinfo_t *
-glusterd_volinfo_unref (glusterd_volinfo_t *volinfo)
+glusterd_volinfo_unref(glusterd_volinfo_t *volinfo)
 {
-        int refcnt = -1;
+    int refcnt = -1;
+    glusterd_conf_t *conf = THIS->private;
 
-        pthread_mutex_lock (&volinfo->reflock);
+    pthread_mutex_lock(&conf->volume_lock);
+    {
+        pthread_mutex_lock(&volinfo->reflock);
         {
-                refcnt = --volinfo->refcnt;
-        }
-        pthread_mutex_unlock (&volinfo->reflock);
-
-        if (!refcnt) {
-                glusterd_volinfo_delete (volinfo);
-                return NULL;
+            refcnt = --volinfo->refcnt;
         }
+        pthread_mutex_unlock(&volinfo->reflock);
+    }
+    pthread_mutex_unlock(&conf->volume_lock);
+    if (!refcnt) {
+        glusterd_volinfo_delete(volinfo);
+        return NULL;
+    }
 
-        return volinfo;
+    return volinfo;
 }
 
 glusterd_volinfo_t *
-glusterd_volinfo_ref (glusterd_volinfo_t *volinfo)
+glusterd_volinfo_ref(glusterd_volinfo_t *volinfo)
 {
-        pthread_mutex_lock (&volinfo->reflock);
-        {
-                ++volinfo->refcnt;
-        }
-        pthread_mutex_unlock (&volinfo->reflock);
+    pthread_mutex_lock(&volinfo->reflock);
+    {
+        ++volinfo->refcnt;
+    }
+    pthread_mutex_unlock(&volinfo->reflock);
 
-        return volinfo;
+    return volinfo;
 }
 
 int32_t
-glusterd_volinfo_new (glusterd_volinfo_t **volinfo)
+glusterd_volinfo_new(glusterd_volinfo_t **volinfo)
 {
-        glusterd_volinfo_t      *new_volinfo = NULL;
-        int32_t                 ret = -1;
+    glusterd_volinfo_t *new_volinfo = NULL;
+    int32_t ret = -1;
 
-        GF_ASSERT (volinfo);
+    GF_ASSERT(volinfo);
 
-        new_volinfo = GF_CALLOC (1, sizeof(*new_volinfo),
-                                 gf_gld_mt_glusterd_volinfo_t);
+    new_volinfo = GF_CALLOC(1, sizeof(*new_volinfo),
+                            gf_gld_mt_glusterd_volinfo_t);
 
-        if (!new_volinfo)
-                goto out;
+    if (!new_volinfo)
+        goto out;
 
-        LOCK_INIT (&new_volinfo->lock);
-        CDS_INIT_LIST_HEAD (&new_volinfo->vol_list);
-        CDS_INIT_LIST_HEAD (&new_volinfo->snapvol_list);
-        CDS_INIT_LIST_HEAD (&new_volinfo->bricks);
-        CDS_INIT_LIST_HEAD (&new_volinfo->snap_volumes);
+    LOCK_INIT(&new_volinfo->lock);
+    CDS_INIT_LIST_HEAD(&new_volinfo->vol_list);
+    CDS_INIT_LIST_HEAD(&new_volinfo->snapvol_list);
+    CDS_INIT_LIST_HEAD(&new_volinfo->bricks);
+    CDS_INIT_LIST_HEAD(&new_volinfo->ta_bricks);
+    CDS_INIT_LIST_HEAD(&new_volinfo->snap_volumes);
 
-        new_volinfo->dict = dict_new ();
-        if (!new_volinfo->dict) {
-                GF_FREE (new_volinfo);
+    new_volinfo->dict = dict_new();
+    if (!new_volinfo->dict) {
+        gf_smsg("glusterd", GF_LOG_ERROR, errno, GD_MSG_DICT_CREATE_FAIL, NULL);
+        GF_FREE(new_volinfo);
 
-                goto out;
-        }
+        goto out;
+    }
 
-        new_volinfo->gsync_slaves = dict_new ();
-        if (!new_volinfo->gsync_slaves) {
-                dict_unref (new_volinfo->dict);
-                GF_FREE (new_volinfo);
-                goto out;
-        }
+    new_volinfo->gsync_slaves = dict_new();
+    if (!new_volinfo->gsync_slaves) {
+        gf_smsg("glusterd", GF_LOG_ERROR, errno, GD_MSG_DICT_CREATE_FAIL, NULL);
+        dict_unref(new_volinfo->dict);
+        GF_FREE(new_volinfo);
+        goto out;
+    }
 
-        new_volinfo->gsync_active_slaves = dict_new ();
-        if (!new_volinfo->gsync_active_slaves) {
-                dict_unref (new_volinfo->dict);
-                dict_unref (new_volinfo->gsync_slaves);
-                GF_FREE (new_volinfo);
-                goto out;
-        }
+    new_volinfo->gsync_active_slaves = dict_new();
+    if (!new_volinfo->gsync_active_slaves) {
+        gf_smsg("glusterd", GF_LOG_ERROR, errno, GD_MSG_DICT_CREATE_FAIL, NULL);
+        dict_unref(new_volinfo->dict);
+        dict_unref(new_volinfo->gsync_slaves);
+        GF_FREE(new_volinfo);
+        goto out;
+    }
 
-        snprintf (new_volinfo->parent_volname, GD_VOLUME_NAME_MAX, "N/A");
+    snprintf(new_volinfo->parent_volname, GD_VOLUME_NAME_MAX, "N/A");
 
-        new_volinfo->snap_max_hard_limit = GLUSTERD_SNAPS_MAX_HARD_LIMIT;
+    new_volinfo->snap_max_hard_limit = GLUSTERD_SNAPS_MAX_HARD_LIMIT;
 
-        new_volinfo->xl = THIS;
+    new_volinfo->xl = THIS;
 
-        pthread_mutex_init (&new_volinfo->reflock, NULL);
-        *volinfo = glusterd_volinfo_ref (new_volinfo);
+    glusterd_snapdsvc_build(&new_volinfo->snapd.svc);
+    glusterd_gfproxydsvc_build(&new_volinfo->gfproxyd.svc);
+    glusterd_shdsvc_build(&new_volinfo->shd.svc);
 
-        ret = 0;
+    pthread_mutex_init(&new_volinfo->store_volinfo_lock, NULL);
+    pthread_mutex_init(&new_volinfo->reflock, NULL);
+
+    *volinfo = glusterd_volinfo_ref(new_volinfo);
+
+    ret = 0;
 
 out:
-        gf_log (THIS->name, GF_LOG_DEBUG, "Returning %d", ret);
-        return ret;
+    gf_msg_debug(THIS->name, 0, "Returning %d", ret);
+    return ret;
 }
 
 /* This function will create a new volinfo and then
@@ -529,59 +710,61 @@ out:
  * @return 0 on success else -1
  */
 int32_t
-glusterd_volinfo_dup (glusterd_volinfo_t *volinfo,
-                      glusterd_volinfo_t **dup_volinfo,
-                      gf_boolean_t set_userauth)
-{
-        int32_t                 ret             = -1;
-        xlator_t                *this           = NULL;
-        glusterd_volinfo_t      *new_volinfo    = NULL;
-
-        this = THIS;
-        GF_ASSERT (this);
-        GF_VALIDATE_OR_GOTO (this->name, volinfo, out);
-        GF_VALIDATE_OR_GOTO (this->name, dup_volinfo, out);
-
-        ret = glusterd_volinfo_new (&new_volinfo);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "not able to create the "
-                        "duplicate volinfo for the volume %s",
-                        volinfo->volname);
-                goto out;
-        }
-
-        new_volinfo->type = volinfo->type;
-        new_volinfo->replica_count = volinfo->replica_count;
-        new_volinfo->stripe_count = volinfo->stripe_count;
-        new_volinfo->disperse_count = volinfo->disperse_count;
-        new_volinfo->redundancy_count = volinfo->redundancy_count;
-        new_volinfo->dist_leaf_count = volinfo->dist_leaf_count;
-        new_volinfo->sub_count = volinfo->sub_count;
-        new_volinfo->transport_type = volinfo->transport_type;
-        new_volinfo->brick_count = volinfo->brick_count;
-        new_volinfo->tier_info = volinfo->tier_info;
-        new_volinfo->quota_conf_version = volinfo->quota_conf_version;
-
-        dict_copy (volinfo->dict, new_volinfo->dict);
-        dict_copy (volinfo->gsync_slaves, new_volinfo->gsync_slaves);
-        dict_copy (volinfo->gsync_active_slaves,
-                   new_volinfo->gsync_active_slaves);
-        gd_update_volume_op_versions (new_volinfo);
-
-        if (set_userauth) {
-                glusterd_auth_set_username (new_volinfo,
-                                            volinfo->auth.username);
-                glusterd_auth_set_password (new_volinfo,
-                                            volinfo->auth.password);
-        }
-
-        *dup_volinfo = new_volinfo;
-        ret = 0;
+glusterd_volinfo_dup(glusterd_volinfo_t *volinfo,
+                     glusterd_volinfo_t **dup_volinfo,
+                     gf_boolean_t set_userauth)
+{
+    int32_t ret = -1;
+    xlator_t *this = NULL;
+    glusterd_volinfo_t *new_volinfo = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+    GF_VALIDATE_OR_GOTO(this->name, volinfo, out);
+    GF_VALIDATE_OR_GOTO(this->name, dup_volinfo, out);
+
+    ret = glusterd_volinfo_new(&new_volinfo);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_VOLINFO_SET_FAIL,
+               "not able to create the "
+               "duplicate volinfo for the volume %s",
+               volinfo->volname);
+        goto out;
+    }
+
+    new_volinfo->type = volinfo->type;
+    new_volinfo->replica_count = volinfo->replica_count;
+    new_volinfo->arbiter_count = volinfo->arbiter_count;
+    new_volinfo->stripe_count = volinfo->stripe_count;
+    new_volinfo->disperse_count = volinfo->disperse_count;
+    new_volinfo->redundancy_count = volinfo->redundancy_count;
+    new_volinfo->dist_leaf_count = volinfo->dist_leaf_count;
+    new_volinfo->sub_count = volinfo->sub_count;
+    new_volinfo->subvol_count = volinfo->subvol_count;
+    new_volinfo->transport_type = volinfo->transport_type;
+    new_volinfo->brick_count = volinfo->brick_count;
+    new_volinfo->quota_conf_version = volinfo->quota_conf_version;
+    new_volinfo->quota_xattr_version = volinfo->quota_xattr_version;
+    new_volinfo->snap_max_hard_limit = volinfo->snap_max_hard_limit;
+    new_volinfo->quota_conf_cksum = volinfo->quota_conf_cksum;
+
+    dict_copy(volinfo->dict, new_volinfo->dict);
+    dict_copy(volinfo->gsync_slaves, new_volinfo->gsync_slaves);
+    dict_copy(volinfo->gsync_active_slaves, new_volinfo->gsync_active_slaves);
+    gd_update_volume_op_versions(new_volinfo);
+
+    if (set_userauth) {
+        glusterd_auth_set_username(new_volinfo, volinfo->auth.username);
+        glusterd_auth_set_password(new_volinfo, volinfo->auth.password);
+    }
+
+    *dup_volinfo = new_volinfo;
+    ret = 0;
 out:
-        if (ret && (NULL != new_volinfo)) {
-                (void) glusterd_volinfo_delete (new_volinfo);
-        }
-        return ret;
+    if (ret && (NULL != new_volinfo)) {
+        (void)glusterd_volinfo_delete(new_volinfo);
+    }
+    return ret;
 }
 
 /* This function will duplicate brickinfo
@@ -592,47 +775,50 @@ out:
  * @return 0 on success else -1
  */
 int32_t
-glusterd_brickinfo_dup (glusterd_brickinfo_t *brickinfo,
-                        glusterd_brickinfo_t *dup_brickinfo)
-{
-        int32_t         ret     = -1;
-        xlator_t        *this   = NULL;
-
-        this = THIS;
-        GF_ASSERT (this);
-
-        GF_VALIDATE_OR_GOTO (this->name, brickinfo, out);
-        GF_VALIDATE_OR_GOTO (this->name, dup_brickinfo, out);
-
-        strcpy (dup_brickinfo->hostname, brickinfo->hostname);
-        strcpy (dup_brickinfo->path, brickinfo->path);
-        strcpy (dup_brickinfo->device_path, brickinfo->device_path);
-        strcpy (dup_brickinfo->fstype, brickinfo->fstype);
-        strcpy (dup_brickinfo->mnt_opts, brickinfo->mnt_opts);
-        ret = gf_canonicalize_path (dup_brickinfo->path);
-        if (ret) {
-                gf_log (THIS->name, GF_LOG_ERROR, "Failed to canonicalize "
-                        "brick path");
-                goto out;
-        }
-        gf_uuid_copy (dup_brickinfo->uuid, brickinfo->uuid);
-
-        dup_brickinfo->port = brickinfo->port;
-        dup_brickinfo->rdma_port = brickinfo->rdma_port;
-        if (NULL != brickinfo->logfile) {
-                dup_brickinfo->logfile = gf_strdup (brickinfo->logfile);
-                if (NULL == dup_brickinfo->logfile) {
-                        ret = -1;
-                        goto out;
-                }
+glusterd_brickinfo_dup(glusterd_brickinfo_t *brickinfo,
+                       glusterd_brickinfo_t *dup_brickinfo)
+{
+    int32_t ret = -1;
+    xlator_t *this = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+
+    GF_VALIDATE_OR_GOTO(this->name, brickinfo, out);
+    GF_VALIDATE_OR_GOTO(this->name, dup_brickinfo, out);
+
+    strcpy(dup_brickinfo->hostname, brickinfo->hostname);
+    strcpy(dup_brickinfo->path, brickinfo->path);
+    strcpy(dup_brickinfo->real_path, brickinfo->real_path);
+    strcpy(dup_brickinfo->device_path, brickinfo->device_path);
+    strcpy(dup_brickinfo->fstype, brickinfo->fstype);
+    strcpy(dup_brickinfo->mnt_opts, brickinfo->mnt_opts);
+    ret = gf_canonicalize_path(dup_brickinfo->path);
+    if (ret) {
+        gf_msg(THIS->name, GF_LOG_ERROR, 0, GD_MSG_CANONICALIZE_FAIL,
+               "Failed to canonicalize "
+               "brick path");
+        goto out;
+    }
+    gf_uuid_copy(dup_brickinfo->uuid, brickinfo->uuid);
+
+    dup_brickinfo->port = brickinfo->port;
+    dup_brickinfo->rdma_port = brickinfo->rdma_port;
+    if (NULL != brickinfo->logfile) {
+        dup_brickinfo->logfile = gf_strdup(brickinfo->logfile);
+        if (NULL == dup_brickinfo->logfile) {
+            ret = -1;
+            goto out;
         }
-        strcpy (dup_brickinfo->brick_id, brickinfo->brick_id);
-        strcpy (dup_brickinfo->mount_dir, brickinfo->mount_dir);
-        dup_brickinfo->status = brickinfo->status;
-        dup_brickinfo->snap_status = brickinfo->snap_status;
+    }
+    strcpy(dup_brickinfo->brick_id, brickinfo->brick_id);
+    strcpy(dup_brickinfo->mount_dir, brickinfo->mount_dir);
+    dup_brickinfo->status = brickinfo->status;
+    dup_brickinfo->snap_status = brickinfo->snap_status;
 out:
-        return ret;
+    return ret;
 }
+
 /*
  * gd_vol_is_geo_rep_active:
  *      This function checks for any running geo-rep session for
@@ -644,1424 +830,2127 @@ out:
  */
 
 gf_boolean_t
-gd_vol_is_geo_rep_active (glusterd_volinfo_t *volinfo)
+gd_vol_is_geo_rep_active(glusterd_volinfo_t *volinfo)
 {
-        gf_boolean_t     active = _gf_false;
+    gf_boolean_t active = _gf_false;
 
-        GF_ASSERT (volinfo);
+    GF_ASSERT(volinfo);
 
-        if (volinfo->gsync_active_slaves &&
-            volinfo->gsync_active_slaves->count > 0)
-                active = _gf_true;
+    if (volinfo->gsync_active_slaves && volinfo->gsync_active_slaves->count > 0)
+        active = _gf_true;
 
-        return active;
+    return active;
 }
 
 void
-glusterd_auth_cleanup (glusterd_volinfo_t *volinfo) {
-
-        GF_ASSERT (volinfo);
+glusterd_auth_cleanup(glusterd_volinfo_t *volinfo)
+{
+    GF_ASSERT(volinfo);
 
-        GF_FREE (volinfo->auth.username);
+    GF_FREE(volinfo->auth.username);
 
-        GF_FREE (volinfo->auth.password);
+    GF_FREE(volinfo->auth.password);
 }
 
 char *
-glusterd_auth_get_username (glusterd_volinfo_t *volinfo) {
-
-        GF_ASSERT (volinfo);
+glusterd_auth_get_username(glusterd_volinfo_t *volinfo)
+{
+    GF_ASSERT(volinfo);
 
-        return volinfo->auth.username;
+    return volinfo->auth.username;
 }
 
 char *
-glusterd_auth_get_password (glusterd_volinfo_t *volinfo) {
-
-        GF_ASSERT (volinfo);
+glusterd_auth_get_password(glusterd_volinfo_t *volinfo)
+{
+    GF_ASSERT(volinfo);
 
-        return volinfo->auth.password;
+    return volinfo->auth.password;
 }
 
 int32_t
-glusterd_auth_set_username (glusterd_volinfo_t *volinfo, char *username) {
-
-        GF_ASSERT (volinfo);
-        GF_ASSERT (username);
+glusterd_auth_set_username(glusterd_volinfo_t *volinfo, char *username)
+{
+    GF_ASSERT(volinfo);
+    GF_ASSERT(username);
 
-        volinfo->auth.username = gf_strdup (username);
-        return 0;
+    volinfo->auth.username = gf_strdup(username);
+    return 0;
 }
 
 int32_t
-glusterd_auth_set_password (glusterd_volinfo_t *volinfo, char *password) {
-
-        GF_ASSERT (volinfo);
-        GF_ASSERT (password);
+glusterd_auth_set_password(glusterd_volinfo_t *volinfo, char *password)
+{
+    GF_ASSERT(volinfo);
+    GF_ASSERT(password);
 
-        volinfo->auth.password = gf_strdup (password);
-        return 0;
+    volinfo->auth.password = gf_strdup(password);
+    return 0;
 }
 
 int32_t
-glusterd_brickinfo_delete (glusterd_brickinfo_t *brickinfo)
+glusterd_brickinfo_delete(glusterd_brickinfo_t *brickinfo)
 {
-        int32_t         ret = -1;
+    int32_t ret = -1;
 
-        GF_ASSERT (brickinfo);
+    GF_ASSERT(brickinfo);
 
-        cds_list_del_init (&brickinfo->brick_list);
+    cds_list_del_init(&brickinfo->brick_list);
 
-        GF_FREE (brickinfo->logfile);
-        GF_FREE (brickinfo);
+    (void)gf_store_handle_destroy(brickinfo->shandle);
 
-        ret = 0;
+    GF_FREE(brickinfo->logfile);
+    GF_FREE(brickinfo);
 
-        return ret;
+    ret = 0;
+
+    return ret;
 }
 
 int32_t
-glusterd_volume_brickinfos_delete (glusterd_volinfo_t *volinfo)
+glusterd_volume_brickinfos_delete(glusterd_volinfo_t *volinfo)
 {
-        glusterd_brickinfo_t    *brickinfo = NULL;
-        glusterd_brickinfo_t    *tmp = NULL;
-        int32_t                 ret = 0;
+    glusterd_brickinfo_t *brickinfo = NULL;
+    glusterd_brickinfo_t *tmp = NULL;
+    int32_t ret = 0;
 
-        GF_ASSERT (volinfo);
+    GF_ASSERT(volinfo);
 
-        cds_list_for_each_entry_safe (brickinfo, tmp, &volinfo->bricks,
-                                      brick_list) {
-                ret = glusterd_brickinfo_delete (brickinfo);
-                if (ret)
-                        goto out;
-        }
+    cds_list_for_each_entry_safe(brickinfo, tmp, &volinfo->bricks, brick_list)
+    {
+        ret = glusterd_brickinfo_delete(brickinfo);
+        if (ret)
+            goto out;
+    }
 
 out:
-        gf_log (THIS->name, GF_LOG_DEBUG, "Returning %d", ret);
-        return ret;
+    gf_msg_debug(THIS->name, 0, "Returning %d", ret);
+    return ret;
 }
 
 int
-glusterd_volinfo_remove (glusterd_volinfo_t *volinfo)
+glusterd_volinfo_remove(glusterd_volinfo_t *volinfo)
 {
-        cds_list_del_init (&volinfo->vol_list);
-        glusterd_volinfo_unref (volinfo);
-        return 0;
+    cds_list_del_init(&volinfo->vol_list);
+    glusterd_volinfo_unref(volinfo);
+    return 0;
 }
 
 int32_t
-glusterd_volinfo_delete (glusterd_volinfo_t *volinfo)
+glusterd_volinfo_delete(glusterd_volinfo_t *volinfo)
+{
+    int32_t ret = -1;
+
+    GF_ASSERT(volinfo);
+
+    cds_list_del_init(&volinfo->vol_list);
+    cds_list_del_init(&volinfo->snapvol_list);
+
+    ret = glusterd_volume_brickinfos_delete(volinfo);
+    if (ret)
+        goto out;
+    if (volinfo->dict)
+        dict_unref(volinfo->dict);
+    if (volinfo->gsync_slaves)
+        dict_unref(volinfo->gsync_slaves);
+    if (volinfo->gsync_active_slaves)
+        dict_unref(volinfo->gsync_active_slaves);
+    GF_FREE(volinfo->logdir);
+    if (volinfo->rebal.dict)
+        dict_unref(volinfo->rebal.dict);
+
+    /* Destroy the connection object for per volume svc daemons */
+    glusterd_conn_term(&volinfo->snapd.svc.conn);
+    glusterd_conn_term(&volinfo->gfproxyd.svc.conn);
+
+    gf_store_handle_destroy(volinfo->quota_conf_shandle);
+    gf_store_handle_destroy(volinfo->shandle);
+    gf_store_handle_destroy(volinfo->node_state_shandle);
+    gf_store_handle_destroy(volinfo->snapd.handle);
+
+    glusterd_auth_cleanup(volinfo);
+    glusterd_shd_svcproc_cleanup(&volinfo->shd);
+
+    pthread_mutex_destroy(&volinfo->store_volinfo_lock);
+    pthread_mutex_destroy(&volinfo->reflock);
+    LOCK_DESTROY(&volinfo->lock);
+
+    GF_FREE(volinfo);
+    ret = 0;
+out:
+    gf_msg_debug(THIS->name, 0, "Returning %d", ret);
+    return ret;
+}
+
+int32_t
+glusterd_brickprocess_new(glusterd_brick_proc_t **brickprocess)
 {
-        int32_t                 ret = -1;
+    glusterd_brick_proc_t *new_brickprocess = NULL;
+    int32_t ret = -1;
 
-        GF_ASSERT (volinfo);
+    GF_VALIDATE_OR_GOTO(THIS->name, brickprocess, out);
 
-        cds_list_del_init (&volinfo->vol_list);
-        cds_list_del_init (&volinfo->snapvol_list);
+    new_brickprocess = GF_CALLOC(1, sizeof(*new_brickprocess),
+                                 gf_gld_mt_glusterd_brick_proc_t);
 
-        ret = glusterd_volume_brickinfos_delete (volinfo);
-        if (ret)
-                goto out;
-        if (volinfo->dict)
-                dict_unref (volinfo->dict);
-        if (volinfo->gsync_slaves)
-                dict_unref (volinfo->gsync_slaves);
-        if (volinfo->gsync_active_slaves)
-                dict_unref (volinfo->gsync_active_slaves);
-        GF_FREE (volinfo->logdir);
-        if (volinfo->rebal.dict)
-                dict_unref (volinfo->rebal.dict);
-
-        gf_store_handle_destroy (volinfo->quota_conf_shandle);
-
-        glusterd_auth_cleanup (volinfo);
-
-        pthread_mutex_destroy (&volinfo->reflock);
-        GF_FREE (volinfo);
-        ret = 0;
+    if (!new_brickprocess)
+        goto out;
+
+    CDS_INIT_LIST_HEAD(&new_brickprocess->bricks);
+    CDS_INIT_LIST_HEAD(&new_brickprocess->brick_proc_list);
+
+    new_brickprocess->brick_count = 0;
+    *brickprocess = new_brickprocess;
+
+    ret = 0;
 
 out:
-        gf_log (THIS->name, GF_LOG_DEBUG, "Returning %d", ret);
-        return ret;
+    gf_msg_debug(THIS->name, 0, "Returning %d", ret);
+    return ret;
 }
 
 int32_t
-glusterd_brickinfo_new (glusterd_brickinfo_t **brickinfo)
+glusterd_brickinfo_new(glusterd_brickinfo_t **brickinfo)
 {
-        glusterd_brickinfo_t      *new_brickinfo = NULL;
-        int32_t                   ret = -1;
+    glusterd_brickinfo_t *new_brickinfo = NULL;
+    int32_t ret = -1;
 
-        GF_ASSERT (brickinfo);
+    GF_ASSERT(brickinfo);
 
-        new_brickinfo = GF_CALLOC (1, sizeof(*new_brickinfo),
-                                   gf_gld_mt_glusterd_brickinfo_t);
+    new_brickinfo = GF_CALLOC(1, sizeof(*new_brickinfo),
+                              gf_gld_mt_glusterd_brickinfo_t);
 
-        if (!new_brickinfo)
-                goto out;
-
-        CDS_INIT_LIST_HEAD (&new_brickinfo->brick_list);
+    if (!new_brickinfo)
+        goto out;
 
-        *brickinfo = new_brickinfo;
+    CDS_INIT_LIST_HEAD(&new_brickinfo->brick_list);
+    CDS_INIT_LIST_HEAD(&new_brickinfo->mux_bricks);
+    pthread_mutex_init(&new_brickinfo->restart_mutex, NULL);
+    *brickinfo = new_brickinfo;
 
-        ret = 0;
+    ret = 0;
 
 out:
-        gf_log (THIS->name, GF_LOG_DEBUG, "Returning %d", ret);
-        return ret;
+    gf_msg_debug(THIS->name, 0, "Returning %d", ret);
+    return ret;
 }
 
 int
-glusterd_get_next_available_brickid (glusterd_volinfo_t *volinfo)
-{
-        glusterd_brickinfo_t *brickinfo    = NULL;
-        char                 *token        = NULL;
-        int                   brickid      = 0;
-        int                   max_brickid  = -1;
-        int                   ret          = -1;
-
-        cds_list_for_each_entry (brickinfo, &volinfo->bricks, brick_list) {
-                token = strrchr (brickinfo->brick_id, '-');
-                ret = gf_string2int32 (++token, &brickid);
-                if (ret < 0) {
-                        gf_log (THIS->name, GF_LOG_ERROR,
-                                "Unable to generate brick ID");
-                        return ret;
-                }
-                if (brickid > max_brickid)
-                        max_brickid = brickid;
+glusterd_get_next_available_brickid(glusterd_volinfo_t *volinfo)
+{
+    glusterd_brickinfo_t *brickinfo = NULL;
+    char *token = NULL;
+    int brickid = 0;
+    int max_brickid = -1;
+    int ret = -1;
+
+    cds_list_for_each_entry(brickinfo, &volinfo->bricks, brick_list)
+    {
+        token = strrchr(brickinfo->brick_id, '-');
+        ret = gf_string2int32(++token, &brickid);
+        if (ret < 0) {
+            gf_msg(THIS->name, GF_LOG_ERROR, 0, GD_MSG_BRICK_ID_GEN_FAILED,
+                   "Unable to generate brick ID");
+            return ret;
         }
+        if (brickid > max_brickid)
+            max_brickid = brickid;
+    }
 
-        return max_brickid + 1 ;
+    return max_brickid + 1;
 }
 
 int32_t
-glusterd_resolve_brick (glusterd_brickinfo_t *brickinfo)
+glusterd_resolve_brick(glusterd_brickinfo_t *brickinfo)
 {
-        int32_t                 ret = -1;
-        xlator_t                *this = NULL;
+    int32_t ret = -1;
+    xlator_t *this = NULL;
 
-        this = THIS;
-        GF_ASSERT (this);
+    this = THIS;
+    GF_ASSERT(this);
 
-        GF_ASSERT (brickinfo);
+    GF_ASSERT(brickinfo);
+    if (!gf_uuid_compare(brickinfo->uuid, MY_UUID) ||
+        (glusterd_peerinfo_find_by_uuid(brickinfo->uuid) != NULL)) {
+        ret = 0;
+        goto out;
+    }
 
-        ret = glusterd_hostname_to_uuid (brickinfo->hostname, brickinfo->uuid);
-        gf_log (this->name, GF_LOG_DEBUG, "Returning %d", ret);
-        return ret;
+    ret = glusterd_hostname_to_uuid(brickinfo->hostname, brickinfo->uuid);
+
+out:
+    gf_msg_debug(this->name, 0, "Returning %d", ret);
+    return ret;
 }
 
 int32_t
-glusterd_get_brick_mount_dir (char *brickpath, char *hostname, char *mount_dir)
-{
-        char                   *mnt_pt        = NULL;
-        char                   *brick_dir     = NULL;
-        int32_t                 ret           = -1;
-        uuid_t                  brick_uuid    = {0, };
-        xlator_t               *this          = NULL;
-
-        this = THIS;
-        GF_ASSERT (this);
-        GF_ASSERT (brickpath);
-        GF_ASSERT (hostname);
-        GF_ASSERT (mount_dir);
-
-        ret = glusterd_hostname_to_uuid (hostname, brick_uuid);
+glusterd_get_brick_mount_dir(char *brickpath, char *hostname, char *mount_dir)
+{
+    char *mnt_pt = NULL;
+    char *brick_dir = NULL;
+    int32_t ret = -1;
+    uuid_t brick_uuid = {
+        0,
+    };
+    xlator_t *this = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+    GF_ASSERT(brickpath);
+    GF_ASSERT(hostname);
+    GF_ASSERT(mount_dir);
+
+    ret = glusterd_hostname_to_uuid(hostname, brick_uuid);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_HOSTNAME_TO_UUID_FAIL,
+               "Failed to convert hostname %s to uuid", hostname);
+        goto out;
+    }
+
+    if (!gf_uuid_compare(brick_uuid, MY_UUID)) {
+        ret = glusterd_get_brick_root(brickpath, &mnt_pt);
         if (ret) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "Failed to convert hostname %s to uuid",
-                        hostname);
-                goto out;
+            gf_msg(this->name, GF_LOG_WARNING, 0,
+                   GD_MSG_BRICKPATH_ROOT_GET_FAIL,
+                   "Could not get the root of the brick path %s", brickpath);
+            goto out;
         }
 
-        if (!gf_uuid_compare (brick_uuid, MY_UUID)) {
-                ret = glusterd_get_brick_root (brickpath, &mnt_pt);
-                if (ret) {
-                        gf_log (this->name, GF_LOG_WARNING,
-                                "Could not get the root of the brick path %s",
-                                brickpath);
-                        goto out;
-                }
-
-                if (strncmp (brickpath, mnt_pt, strlen(mnt_pt))) {
-                        gf_log (this->name, GF_LOG_WARNING,
-                                "brick: %s brick mount: %s",
-                                brickpath, mnt_pt);
-                        ret = -1;
-                        goto out;
-                }
+        if (strncmp(brickpath, mnt_pt, strlen(mnt_pt))) {
+            gf_msg(this->name, GF_LOG_WARNING, 0,
+                   GD_MSG_BRKPATH_MNTPNT_MISMATCH, "brick: %s brick mount: %s",
+                   brickpath, mnt_pt);
+            ret = -1;
+            goto out;
+        }
 
-                brick_dir = &brickpath[strlen (mnt_pt)];
-                brick_dir++;
+        brick_dir = &brickpath[strlen(mnt_pt)];
+        if (brick_dir[0] == '/')
+            brick_dir++;
 
-                snprintf (mount_dir, PATH_MAX, "/%s", brick_dir);
-        }
+        snprintf(mount_dir, VALID_GLUSTERD_PATHMAX, "/%s", brick_dir);
+    }
 
 out:
-        gf_log (this->name, GF_LOG_TRACE, "Returning %d", ret);
-        return ret;
+    if (mnt_pt)
+        GF_FREE(mnt_pt);
+
+    gf_msg_trace(this->name, 0, "Returning %d", ret);
+    return ret;
 }
 
 int32_t
-glusterd_brickinfo_new_from_brick (char *brick,
-                                   glusterd_brickinfo_t **brickinfo)
-{
-        char                   *hostname      = NULL;
-        char                   *path          = NULL;
-        char                   *tmp_host      = NULL;
-        char                   *tmp_path      = NULL;
-        char                   *vg            = NULL;
-        int32_t                 ret           = -1;
-        glusterd_brickinfo_t   *new_brickinfo = NULL;
-        xlator_t               *this          = NULL;
-
-        this = THIS;
-        GF_ASSERT (this);
-        GF_ASSERT (brick);
-        GF_ASSERT (brickinfo);
-
-        tmp_host = gf_strdup (brick);
-        if (tmp_host && !get_host_name (tmp_host, &hostname))
-                goto out;
-        tmp_path = gf_strdup (brick);
-        if (tmp_path && !get_path_name (tmp_path, &path))
-                goto out;
-
-        GF_ASSERT (hostname);
-        GF_ASSERT (path);
-
-        ret = glusterd_brickinfo_new (&new_brickinfo);
-        if (ret)
-                goto out;
+glusterd_brickinfo_new_from_brick(char *brick, glusterd_brickinfo_t **brickinfo,
+                                  gf_boolean_t construct_real_path,
+                                  char **op_errstr)
+{
+    char *hostname = NULL;
+    char *path = NULL;
+    char *tmp_host = NULL;
+    char *tmp_path = NULL;
+    int32_t ret = -1;
+    glusterd_brickinfo_t *new_brickinfo = NULL;
+    xlator_t *this = NULL;
+    char abspath[PATH_MAX] = "";
+
+    this = THIS;
+    GF_ASSERT(this);
+    GF_ASSERT(brick);
+    GF_ASSERT(brickinfo);
+
+    tmp_host = gf_strdup(brick);
+    if (tmp_host && !get_host_name(tmp_host, &hostname))
+        goto out;
+    tmp_path = gf_strdup(brick);
+    if (tmp_path && !get_path_name(tmp_path, &path))
+        goto out;
+
+    GF_ASSERT(hostname);
+    GF_ASSERT(path);
+
+    ret = glusterd_brickinfo_new(&new_brickinfo);
+    if (ret)
+        goto out;
+
+    ret = gf_canonicalize_path(path);
+    if (ret)
+        goto out;
+    ret = snprintf(new_brickinfo->hostname, sizeof(new_brickinfo->hostname),
+                   "%s", hostname);
+    if (ret < 0 || ret >= sizeof(new_brickinfo->hostname)) {
+        ret = -1;
+        goto out;
+    }
+    ret = snprintf(new_brickinfo->path, sizeof(new_brickinfo->path), "%s",
+                   path);
+    if (ret < 0 || ret >= sizeof(new_brickinfo->path)) {
+        ret = -1;
+        goto out;
+    }
 
-#ifdef HAVE_BD_XLATOR
-        vg = strchr (path, '?');
-        /* ? is used as a delimiter for vg */
-        if (vg) {
-                strncpy (new_brickinfo->vg, vg + 1, PATH_MAX - 1);
-                *vg = '\0';
+    if (construct_real_path) {
+        ret = glusterd_hostname_to_uuid(new_brickinfo->hostname,
+                                        new_brickinfo->uuid);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_HOSTNAME_TO_UUID_FAIL,
+                   "Failed to convert hostname %s to uuid", hostname);
+            if (op_errstr)
+                gf_asprintf(op_errstr,
+                            "Host %s is not in "
+                            "\'Peer in Cluster\' state",
+                            new_brickinfo->hostname);
+            goto out;
         }
-        new_brickinfo->caps = CAPS_BD;
-#else
-        vg = NULL; /* Avoid compiler warnings when BD not enabled */
-#endif
-        ret = gf_canonicalize_path (path);
-        if (ret)
+    }
+
+    if (construct_real_path && !gf_uuid_compare(new_brickinfo->uuid, MY_UUID) &&
+        new_brickinfo->real_path[0] == '\0') {
+        if (!realpath(new_brickinfo->path, abspath)) {
+            /* ENOENT indicates that brick path has not been created
+             * which is a valid scenario */
+            if (errno != ENOENT) {
+                gf_msg(this->name, GF_LOG_CRITICAL, errno,
+                       GD_MSG_BRICKINFO_CREATE_FAIL,
+                       "realpath"
+                       " () failed for brick %s. The "
+                       "underlying filesystem may be in bad "
+                       "state. Error - %s",
+                       new_brickinfo->path, strerror(errno));
+                ret = -1;
                 goto out;
+            }
+        }
+        if (strlen(abspath) >= sizeof(new_brickinfo->real_path)) {
+            ret = -1;
+            goto out;
+        }
+        (void)strncpy(new_brickinfo->real_path, abspath,
+                      sizeof(new_brickinfo->real_path));
+    }
 
-        strncpy (new_brickinfo->hostname, hostname, 1024);
-        strncpy (new_brickinfo->path, path, 1024);
-
-        *brickinfo = new_brickinfo;
+    *brickinfo = new_brickinfo;
 
-        ret = 0;
+    ret = 0;
 out:
-        GF_FREE (tmp_host);
-        if (tmp_host)
-                GF_FREE (tmp_path);
+    GF_FREE(tmp_host);
+    if (tmp_host)
+        GF_FREE(tmp_path);
 
-        gf_log (this->name, GF_LOG_DEBUG, "Returning %d", ret);
-        return ret;
+    gf_msg_debug(this->name, 0, "Returning %d", ret);
+    return ret;
 }
 
 static gf_boolean_t
-_is_prefix (char *str1, char *str2)
-{
-        GF_ASSERT (str1);
-        GF_ASSERT (str2);
-
-        int             i = 0;
-        int             len1 = 0;
-        int             len2 = 0;
-        int             small_len = 0;
-        char            *bigger = NULL;
-        gf_boolean_t    prefix = _gf_true;
-
-        len1 = strlen (str1);
-        len2 = strlen (str2);
-        small_len = min (len1, len2);
-        for (i = 0; i < small_len; i++) {
-                if (str1[i] != str2[i]) {
-                        prefix = _gf_false;
-                        break;
-                }
+_is_prefix(char *str1, char *str2)
+{
+    GF_ASSERT(str1);
+    GF_ASSERT(str2);
+
+    int i = 0;
+    int len1 = 0;
+    int len2 = 0;
+    int small_len = 0;
+    char *bigger = NULL;
+    gf_boolean_t prefix = _gf_true;
+
+    len1 = strlen(str1);
+    len2 = strlen(str2);
+    small_len = min(len1, len2);
+
+    /*
+     * If either one (not both) of the strings are 0-length, they are not
+     * prefixes of each other.
+     */
+    if ((small_len == 0) && (len1 != len2)) {
+        return _gf_false;
+    }
+
+    for (i = 0; i < small_len; i++) {
+        if (str1[i] != str2[i]) {
+            prefix = _gf_false;
+            break;
         }
+    }
 
-        if (len1 < len2)
-            bigger = str2;
+    if (len1 < len2)
+        bigger = str2;
 
-        else if (len1 > len2)
-            bigger = str1;
+    else if (len1 > len2)
+        bigger = str1;
 
-        else
-            return prefix;
+    else
+        return prefix;
 
-        if (bigger[small_len] != '/')
-            prefix = _gf_false;
+    if (bigger[small_len] != '/')
+        prefix = _gf_false;
 
-        return prefix;
+    return prefix;
 }
 
 /* Checks if @path is available in the peer identified by @uuid
  * 'availability' is determined by querying current state of volumes
  * in the cluster. */
 gf_boolean_t
-glusterd_is_brickpath_available (uuid_t uuid, char *path)
-{
-        glusterd_brickinfo_t    *brickinfo = NULL;
-        glusterd_volinfo_t      *volinfo   = NULL;
-        glusterd_conf_t         *priv      = NULL;
-        gf_boolean_t            available  = _gf_false;
-        char                    tmp_path[PATH_MAX+1] = {0};
-        char                    tmp_brickpath[PATH_MAX+1] = {0};
-
-        priv = THIS->private;
-
-        strncpy (tmp_path, path, PATH_MAX);
-        /* path may not yet exist */
-        if (!realpath (path, tmp_path)) {
-                if (errno != ENOENT) {
-                        goto out;
-                }
-                /* When realpath(3) fails, tmp_path is undefined. */
-                strncpy(tmp_path,path,PATH_MAX);
-        }
-
-        cds_list_for_each_entry (volinfo, &priv->volumes, vol_list) {
-                cds_list_for_each_entry (brickinfo, &volinfo->bricks,
-                                         brick_list) {
-                        if (gf_uuid_compare (uuid, brickinfo->uuid))
-                                continue;
-
-                        if (!realpath (brickinfo->path, tmp_brickpath)) {
-                            if (errno == ENOENT)
-                                strncpy (tmp_brickpath, brickinfo->path,
-                                         PATH_MAX);
-                            else
-                                goto out;
-                        }
-
-                        if (_is_prefix (tmp_brickpath, tmp_path))
-                                goto out;
-                }
+glusterd_is_brickpath_available(uuid_t uuid, char *path)
+{
+    glusterd_brickinfo_t *brickinfo = NULL;
+    glusterd_volinfo_t *volinfo = NULL;
+    glusterd_conf_t *priv = NULL;
+    gf_boolean_t available = _gf_false;
+    char tmp_path[PATH_MAX] = "";
+
+    priv = THIS->private;
+
+    if (snprintf(tmp_path, PATH_MAX, "%s", path) >= PATH_MAX)
+        goto out;
+    /* path may not yet exist */
+    if (!realpath(path, tmp_path)) {
+        if (errno != ENOENT) {
+            gf_msg(THIS->name, GF_LOG_CRITICAL, errno,
+                   GD_MSG_BRICKINFO_CREATE_FAIL,
+                   "realpath"
+                   " () failed for brick %s. The "
+                   "underlying filesystem may be in bad "
+                   "state. Error - %s",
+                   path, strerror(errno));
+            goto out;
         }
-        available = _gf_true;
+        /* When realpath(3) fails, tmp_path is undefined. */
+        (void)snprintf(tmp_path, sizeof(tmp_path), "%s", path);
+    }
+
+    cds_list_for_each_entry(volinfo, &priv->volumes, vol_list)
+    {
+        cds_list_for_each_entry(brickinfo, &volinfo->bricks, brick_list)
+        {
+            if (gf_uuid_compare(uuid, brickinfo->uuid))
+                continue;
+            if (_is_prefix(brickinfo->real_path, tmp_path)) {
+                gf_msg(THIS->name, GF_LOG_CRITICAL, 0,
+                       GD_MSG_BRICKINFO_CREATE_FAIL,
+                       "_is_prefix call failed for brick %s "
+                       "against brick %s",
+                       tmp_path, brickinfo->real_path);
+                goto out;
+            }
+        }
+    }
+    available = _gf_true;
 out:
-        return available;
+    return available;
 }
 
-#ifdef HAVE_BD_XLATOR
-/*
- * Sets the tag of the format "trusted.glusterfs.volume-id:<uuid>" in
- * the brick VG. It is used to avoid using same VG for another brick.
- * @volume-id - gfid, @brick - brick info, @msg - Error message returned
- * to the caller
- */
 int
-glusterd_bd_set_vg_tag (unsigned char *volume_id, glusterd_brickinfo_t *brick,
-                        char *msg, int msg_size)
-{
-        lvm_t        handle    = NULL;
-        vg_t         vg        = NULL;
-        char        *uuid      = NULL;
-        int          ret       = -1;
-
-        gf_asprintf (&uuid, "%s:%s", GF_XATTR_VOL_ID_KEY,
-                     uuid_utoa (volume_id));
-        if (!uuid) {
-                snprintf (msg, sizeof(*msg), "Could not allocate memory "
-                          "for tag");
-                return -1;
+glusterd_validate_and_create_brickpath(glusterd_brickinfo_t *brickinfo,
+                                       uuid_t volume_id, char *volname,
+                                       char **op_errstr, gf_boolean_t is_force,
+                                       gf_boolean_t ignore_partition)
+{
+    int ret = -1;
+    char parentdir[PATH_MAX] = "";
+    struct stat parent_st = {
+        0,
+    };
+    struct stat brick_st = {
+        0,
+    };
+    struct stat root_st = {
+        0,
+    };
+    char msg[2048] = "";
+    gf_boolean_t is_created = _gf_false;
+    char glusterfs_dir_path[PATH_MAX] = "";
+    int32_t len = 0;
+
+    ret = sys_mkdir(brickinfo->path, 0755);
+    if (ret) {
+        if (errno != EEXIST) {
+            len = snprintf(msg, sizeof(msg),
+                           "Failed to create "
+                           "brick directory for brick %s:%s. "
+                           "Reason : %s ",
+                           brickinfo->hostname, brickinfo->path,
+                           strerror(errno));
+            gf_smsg(
+                "glusterd", GF_LOG_ERROR, errno, GD_MSG_CREATE_BRICK_DIR_FAILED,
+                "Brick_hostname=%s, Brick_path=%s, Reason=%s",
+                brickinfo->hostname, brickinfo->path, strerror(errno), NULL);
+            goto out;
         }
+    } else {
+        is_created = _gf_true;
+    }
+
+    ret = sys_lstat(brickinfo->path, &brick_st);
+    if (ret) {
+        len = snprintf(msg, sizeof(msg),
+                       "lstat failed on %s. "
+                       "Reason : %s",
+                       brickinfo->path, strerror(errno));
+        gf_smsg("glusterd", GF_LOG_ERROR, errno, GD_MSG_LSTAT_FAIL,
+                "Failed on Brick_path=%s, Reason=%s", brickinfo->path,
+                strerror(errno), NULL);
+        goto out;
+    }
+
+    if ((!is_created) && (!S_ISDIR(brick_st.st_mode))) {
+        len = snprintf(msg, sizeof(msg),
+                       "The provided path %s "
+                       "which is already present, is not a directory",
+                       brickinfo->path);
+        gf_smsg("glusterd", GF_LOG_ERROR, errno, GD_MSG_DIR_OP_FAILED,
+                "Brick_path=%s", brickinfo->path, NULL);
+        ret = -1;
+        goto out;
+    }
 
-        handle = lvm_init (NULL);
-        if (!handle) {
-                snprintf (msg, sizeof(*msg), "lvm_init failed");
+    len = snprintf(parentdir, sizeof(parentdir), "%s/..", brickinfo->path);
+    if ((len < 0) || (len >= sizeof(parentdir))) {
+        ret = -1;
+        goto out;
+    }
+
+    ret = sys_lstat("/", &root_st);
+    if (ret) {
+        len = snprintf(msg, sizeof(msg),
+                       "lstat failed on /. "
+                       "Reason : %s",
+                       strerror(errno));
+        gf_smsg("glusterd", GF_LOG_ERROR, errno, GD_MSG_LSTAT_FAIL,
+                "Failed on /, Reason=%s", strerror(errno), NULL);
+        goto out;
+    }
+
+    ret = sys_lstat(parentdir, &parent_st);
+    if (ret) {
+        len = snprintf(msg, sizeof(msg),
+                       "lstat failed on %s. "
+                       "Reason : %s",
+                       parentdir, strerror(errno));
+        gf_smsg("glusterd", GF_LOG_ERROR, errno, GD_MSG_LSTAT_FAIL,
+                "Failed on parentdir=%s, Reason=%s", parentdir, strerror(errno),
+                NULL);
+        goto out;
+    }
+    if (strncmp(volname, GLUSTER_SHARED_STORAGE,
+                SLEN(GLUSTER_SHARED_STORAGE)) &&
+        sizeof(GLUSTERD_DEFAULT_WORKDIR) <= (strlen(brickinfo->path) + 1) &&
+        !strncmp(brickinfo->path, GLUSTERD_DEFAULT_WORKDIR,
+                 (sizeof(GLUSTERD_DEFAULT_WORKDIR) - 1))) {
+        len = snprintf(msg, sizeof(msg),
+                       "Brick isn't allowed to be "
+                       "created inside glusterd's working directory.");
+        gf_smsg("glusterd", GF_LOG_ERROR, errno, GD_MSG_BRICK_CREATION_FAIL,
+                NULL);
+        ret = -1;
+        goto out;
+    }
+
+    if (!is_force) {
+        if (brick_st.st_dev != parent_st.st_dev) {
+            len = snprintf(msg, sizeof(msg),
+                           "The brick %s:%s "
+                           "is a mount point. Please create a "
+                           "sub-directory under the mount point "
+                           "and use that as the brick directory. "
+                           "Or use 'force' at the end of the "
+                           "command if you want to override this "
+                           "behavior.",
+                           brickinfo->hostname, brickinfo->path);
+            gf_smsg("glusterd", GF_LOG_ERROR, errno, GD_MSG_BRICK_CREATE_MNTPNT,
+                    "Use 'force' at the end of the command if you want to "
+                    "override this behavior, Brick_hostname=%s, Brick_path=%s",
+                    brickinfo->hostname, brickinfo->path, NULL);
+            ret = -1;
+            goto out;
+        } else if (parent_st.st_dev == root_st.st_dev) {
+            len = snprintf(msg, sizeof(msg),
+                           "The brick %s:%s "
+                           "is being created in the root "
+                           "partition. It is recommended that "
+                           "you don't use the system's root "
+                           "partition for storage backend. Or "
+                           "use 'force' at the end of the "
+                           "command if you want to override this "
+                           "behavior.",
+                           brickinfo->hostname, brickinfo->path);
+            gf_smsg("glusterd", GF_LOG_ERROR, errno, GD_MSG_BRICK_CREATE_ROOT,
+                    "Use 'force' at the end of the command if you want to "
+                    "override this behavior, Brick_hostname=%s, Brick_path=%s",
+                    brickinfo->hostname, brickinfo->path, NULL);
+
+            /* If --wignore-partition flag is used, ignore warnings
+             * related to bricks being on root partition when 'force'
+             * is not used */
+            if ((len < 0) || (len >= sizeof(msg)) || !ignore_partition) {
+                ret = -1;
                 goto out;
+            }
         }
+    }
 
-        vg = lvm_vg_open (handle, brick->vg, "w", 0);
-        if (!vg) {
-                snprintf (msg, sizeof(*msg), "Could not open VG %s",
-                          brick->vg);
-                goto out;
-        }
+    ret = glusterd_check_and_set_brick_xattr(
+        brickinfo->hostname, brickinfo->path, volume_id, op_errstr, is_force);
+    if (ret)
+        goto out;
 
-        if (lvm_vg_add_tag (vg, uuid) < 0) {
-                snprintf (msg, sizeof(*msg), "Could not set tag %s for "
-                          "VG %s", uuid, brick->vg);
-                goto out;
-        }
-        lvm_vg_write (vg);
-        ret = 0;
-out:
-        GF_FREE (uuid);
+    /* create .glusterfs directory */
+    len = snprintf(glusterfs_dir_path, sizeof(glusterfs_dir_path), "%s/%s",
+                   brickinfo->path, ".glusterfs");
+    if ((len < 0) || (len >= sizeof(glusterfs_dir_path))) {
+        ret = -1;
+        goto out;
+    }
+
+    ret = sys_mkdir(glusterfs_dir_path, 0600);
+    if (ret && (errno != EEXIST)) {
+        len = snprintf(msg, sizeof(msg),
+                       "Failed to create "
+                       ".glusterfs directory for brick %s:%s. "
+                       "Reason : %s ",
+                       brickinfo->hostname, brickinfo->path, strerror(errno));
+        gf_smsg("glusterd", GF_LOG_ERROR, errno,
+                GD_MSG_CREATE_GLUSTER_DIR_FAILED,
+                "Brick_hostname=%s, Brick_path=%s, Reason=%s",
+                brickinfo->hostname, brickinfo->path, strerror(errno), NULL);
+        goto out;
+    }
+
+    ret = 0;
 
-        if (vg)
-                lvm_vg_close (vg);
-        if (handle)
-                lvm_quit (handle);
+out:
+    if (len < 0) {
+        ret = -1;
+    }
+    if (ret && is_created) {
+        (void)recursive_rmdir(brickinfo->path);
+    }
+    if (ret && !*op_errstr && msg[0] != '\0')
+        *op_errstr = gf_strdup(msg);
 
-        return ret;
+    return ret;
 }
-#endif
-
 
-int
-glusterd_validate_and_create_brickpath (glusterd_brickinfo_t *brickinfo,
-                                        uuid_t volume_id, char **op_errstr,
-                                        gf_boolean_t is_force)
+int32_t
+glusterd_volume_brickinfo_get(uuid_t uuid, char *hostname, char *path,
+                              glusterd_volinfo_t *volinfo,
+                              glusterd_brickinfo_t **brickinfo)
 {
-        int          ret                 = -1;
-        char         parentdir[PATH_MAX] = {0,};
-        struct stat  parent_st           = {0,};
-        struct stat  brick_st            = {0,};
-        struct stat  root_st             = {0,};
-        char         msg[2048]           = {0,};
-        gf_boolean_t is_created          = _gf_false;
-
-        ret = mkdir (brickinfo->path, 0777);
-        if (ret) {
-                if (errno != EEXIST) {
-                        snprintf (msg, sizeof (msg), "Failed to create brick "
-                                  "directory for brick %s:%s. Reason : %s ",
-                                  brickinfo->hostname, brickinfo->path,
-                                  strerror (errno));
-                        goto out;
-                }
-        } else {
-                is_created = _gf_true;
-        }
-
-        ret = lstat (brickinfo->path, &brick_st);
-        if (ret) {
-                snprintf (msg, sizeof (msg), "lstat failed on %s. Reason : %s",
-                          brickinfo->path, strerror (errno));
-                goto out;
-        }
-
-        if ((!is_created) && (!S_ISDIR (brick_st.st_mode))) {
-                snprintf (msg, sizeof (msg), "The provided path %s which is "
-                          "already present, is not a directory",
-                          brickinfo->path);
-                ret = -1;
-                goto out;
-        }
-
-        snprintf (parentdir, sizeof (parentdir), "%s/..", brickinfo->path);
-
-        ret = lstat ("/", &root_st);
-        if (ret) {
-                snprintf (msg, sizeof (msg), "lstat failed on /. Reason : %s",
-                          strerror (errno));
-                goto out;
-        }
+    glusterd_brickinfo_t *brickiter = NULL;
+    uuid_t peer_uuid = {0};
+    int32_t ret = -1;
+    xlator_t *this = NULL;
 
-        ret = lstat (parentdir, &parent_st);
-        if (ret) {
-                snprintf (msg, sizeof (msg), "lstat failed on %s. Reason : %s",
-                          parentdir, strerror (errno));
-                goto out;
-        }
-
-        if (!is_force) {
-                if (brick_st.st_dev != parent_st.st_dev) {
-                        snprintf (msg, sizeof (msg), "The brick %s:%s is a "
-                                  "mount point. Please create a sub-directory "
-                                  "under the mount point and use that as the "
-                                  "brick directory. Or use 'force' at the end "
-                                  "of the command if you want to override this "
-                                  "behavior.", brickinfo->hostname,
-                                  brickinfo->path);
-                        ret = -1;
-                        goto out;
-                }
-                else if (parent_st.st_dev == root_st.st_dev) {
-                        snprintf (msg, sizeof (msg), "The brick %s:%s "
-                                  "is being created in the root partition. It "
-                                  "is recommended that you don't use the "
-                                  "system's root partition for storage backend."
-                                  " Or use 'force' at the end of the command if"
-                                  " you want to override this behavior.",
-                                  brickinfo->hostname, brickinfo->path);
-                        ret = -1;
-                        goto out;
-                }
-        }
+    this = THIS;
 
-#ifdef HAVE_BD_XLATOR
-        if (brickinfo->vg[0]) {
-                ret = glusterd_bd_set_vg_tag (volume_id, brickinfo, msg,
-                                              sizeof(msg));
-                if (ret)
-                        goto out;
-        }
-#endif
-        ret = glusterd_check_and_set_brick_xattr (brickinfo->hostname,
-                                                  brickinfo->path, volume_id,
-                                                  op_errstr, is_force);
+    if (uuid) {
+        gf_uuid_copy(peer_uuid, uuid);
+    } else {
+        ret = glusterd_hostname_to_uuid(hostname, peer_uuid);
         if (ret)
-                goto out;
+            goto out;
+    }
+    ret = -1;
+    cds_list_for_each_entry(brickiter, &volinfo->bricks, brick_list)
+    {
+        if ((gf_uuid_is_null(brickiter->uuid)) &&
+            (glusterd_resolve_brick(brickiter) != 0))
+            goto out;
+        if (gf_uuid_compare(peer_uuid, brickiter->uuid))
+            continue;
 
-        ret = 0;
+        if (strcmp(brickiter->path, path) == 0) {
+            gf_msg_debug(this->name, 0, LOGSTR_FOUND_BRICK, brickiter->hostname,
+                         brickiter->path, volinfo->volname);
+            ret = 0;
+            if (brickinfo)
+                *brickinfo = brickiter;
+            break;
+        }
+    }
 
 out:
-        if (ret && is_created)
-                rmdir (brickinfo->path);
-        if (ret && !*op_errstr && msg[0] != '\0')
-                *op_errstr = gf_strdup (msg);
-
-        return ret;
+    gf_msg_debug(this->name, 0, "Returning %d", ret);
+    return ret;
 }
 
 int32_t
-glusterd_volume_brickinfo_get (uuid_t uuid, char *hostname, char *path,
-                               glusterd_volinfo_t *volinfo,
-                               glusterd_brickinfo_t **brickinfo)
+glusterd_volume_ta_brickinfo_get(uuid_t uuid, char *hostname, char *path,
+                                 glusterd_volinfo_t *volinfo,
+                                 glusterd_brickinfo_t **ta_brickinfo)
 {
-        glusterd_brickinfo_t    *brickiter = NULL;
-        uuid_t                  peer_uuid = {0};
-        int32_t                 ret = -1;
-        xlator_t                *this = NULL;
+    glusterd_brickinfo_t *ta_brickiter = NULL;
+    int32_t ret = -1;
+    xlator_t *this = NULL;
 
-        this = THIS;
+    this = THIS;
 
-        if (uuid) {
-                gf_uuid_copy (peer_uuid, uuid);
-        } else {
-                ret = glusterd_hostname_to_uuid (hostname, peer_uuid);
-                if (ret)
-                        goto out;
-        }
-        ret = -1;
-        cds_list_for_each_entry (brickiter, &volinfo->bricks, brick_list) {
-
-                if ((gf_uuid_is_null (brickiter->uuid)) &&
-                    (glusterd_resolve_brick (brickiter) != 0))
-                        goto out;
-                if (gf_uuid_compare (peer_uuid, brickiter->uuid))
-                        continue;
+    ret = -1;
 
-                if (strcmp (brickiter->path, path) == 0) {
-                        gf_log (this->name, GF_LOG_DEBUG, LOGSTR_FOUND_BRICK,
-                                brickiter->hostname, brickiter->path,
-                                volinfo->volname);
-                        ret = 0;
-                        if (brickinfo)
-                                *brickinfo = brickiter;
-                        break;
-                }
+    cds_list_for_each_entry(ta_brickiter, &volinfo->ta_bricks, brick_list)
+    {
+        if (strcmp(ta_brickiter->path, path) == 0 &&
+            strcmp(ta_brickiter->hostname, hostname) == 0) {
+            gf_msg_debug(this->name, 0, LOGSTR_FOUND_BRICK,
+                         ta_brickiter->hostname, ta_brickiter->path,
+                         volinfo->volname);
+            ret = 0;
+            if (ta_brickinfo)
+                *ta_brickinfo = ta_brickiter;
+            break;
         }
+    }
 
-out:
-        gf_log (this->name, GF_LOG_DEBUG, "Returning %d", ret);
-        return ret;
+    gf_msg_debug(this->name, 0, "Returning %d", ret);
+    return ret;
 }
 
 int32_t
-glusterd_volume_brickinfo_get_by_brick (char *brick,
-                                        glusterd_volinfo_t *volinfo,
-                                        glusterd_brickinfo_t **brickinfo)
+glusterd_volume_brickinfo_get_by_brick(char *brick, glusterd_volinfo_t *volinfo,
+                                       glusterd_brickinfo_t **brickinfo,
+                                       gf_boolean_t construct_real_path)
 {
-        int32_t                 ret = -1;
-        glusterd_brickinfo_t    *tmp_brickinfo = NULL;
+    int32_t ret = -1;
+    glusterd_brickinfo_t *tmp_brickinfo = NULL;
 
-        GF_ASSERT (brick);
-        GF_ASSERT (volinfo);
+    GF_ASSERT(brick);
+    GF_ASSERT(volinfo);
 
-        ret = glusterd_brickinfo_new_from_brick (brick, &tmp_brickinfo);
-        if (ret)
-                goto out;
+    ret = glusterd_brickinfo_new_from_brick(brick, &tmp_brickinfo,
+                                            construct_real_path, NULL);
+    if (ret)
+        goto out;
 
-        ret = glusterd_volume_brickinfo_get (NULL, tmp_brickinfo->hostname,
-                                             tmp_brickinfo->path, volinfo,
-                                             brickinfo);
-        (void) glusterd_brickinfo_delete (tmp_brickinfo);
+    ret = glusterd_volume_brickinfo_get(
+        NULL, tmp_brickinfo->hostname, tmp_brickinfo->path, volinfo, brickinfo);
+    (void)glusterd_brickinfo_delete(tmp_brickinfo);
 out:
-        gf_log ("", GF_LOG_DEBUG, "Returning %d", ret);
-        return ret;
+    gf_msg_debug("glusterd", 0, "Returning %d", ret);
+    return ret;
 }
 
 gf_boolean_t
-glusterd_is_brick_decommissioned (glusterd_volinfo_t *volinfo, char *hostname,
-                                  char *path)
-{
-        gf_boolean_t            decommissioned = _gf_false;
-        glusterd_brickinfo_t    *brickinfo = NULL;
-        int                     ret = -1;
-
-        ret = glusterd_volume_brickinfo_get (NULL, hostname, path, volinfo,
-                                             &brickinfo);
-        if (ret)
-                goto out;
-        decommissioned = brickinfo->decommissioned;
+glusterd_is_brick_decommissioned(glusterd_volinfo_t *volinfo, char *hostname,
+                                 char *path)
+{
+    gf_boolean_t decommissioned = _gf_false;
+    glusterd_brickinfo_t *brickinfo = NULL;
+    int ret = -1;
+
+    ret = glusterd_volume_brickinfo_get(NULL, hostname, path, volinfo,
+                                        &brickinfo);
+    if (ret)
+        goto out;
+    decommissioned = brickinfo->decommissioned;
 out:
-        return decommissioned;
+    return decommissioned;
 }
 
 int
-glusterd_volinfo_find_by_volume_id (uuid_t volume_id, glusterd_volinfo_t **volinfo)
+glusterd_volinfo_find_by_volume_id(uuid_t volume_id,
+                                   glusterd_volinfo_t **volinfo)
 {
-        int32_t                 ret = -1;
-        xlator_t                *this = NULL;
-        glusterd_volinfo_t      *voliter = NULL;
-        glusterd_conf_t         *priv = NULL;
+    int32_t ret = -1;
+    xlator_t *this = NULL;
+    glusterd_volinfo_t *voliter = NULL;
+    glusterd_conf_t *priv = NULL;
 
-        if (!volume_id)
-                return -1;
+    if (!volume_id) {
+        gf_smsg("glusterd", GF_LOG_ERROR, errno, GD_MSG_INVALID_ARGUMENT, NULL);
+        return -1;
+    }
 
-        this = THIS;
-        priv = this->private;
+    this = THIS;
+    priv = this->private;
 
-        cds_list_for_each_entry (voliter, &priv->volumes, vol_list) {
-                if (gf_uuid_compare (volume_id, voliter->volume_id))
-                        continue;
-                *volinfo = voliter;
-                ret = 0;
-                gf_log (this->name, GF_LOG_DEBUG, "Volume %s found",
-                        voliter->volname);
-                break;
-        }
-        return ret;
+    cds_list_for_each_entry(voliter, &priv->volumes, vol_list)
+    {
+        if (gf_uuid_compare(volume_id, voliter->volume_id))
+            continue;
+        *volinfo = voliter;
+        ret = 0;
+        gf_msg_debug(this->name, 0, "Volume %s found", voliter->volname);
+        break;
+    }
+    return ret;
 }
 
 int32_t
-glusterd_volinfo_find (char *volname, glusterd_volinfo_t **volinfo)
-{
-        glusterd_volinfo_t      *tmp_volinfo = NULL;
-        int32_t                 ret = -1;
-        xlator_t                *this = NULL;
-        glusterd_conf_t         *priv = NULL;
-
-        GF_ASSERT (volname);
-        this = THIS;
-        GF_ASSERT (this);
-
-        priv = this->private;
-        GF_ASSERT (priv);
-
-        cds_list_for_each_entry (tmp_volinfo, &priv->volumes, vol_list) {
-                if (!strcmp (tmp_volinfo->volname, volname)) {
-                        gf_log (this->name, GF_LOG_DEBUG, "Volume %s found",
-                                volname);
-                        ret = 0;
-                        *volinfo = tmp_volinfo;
-                        break;
-                }
+glusterd_volinfo_find(const char *volname, glusterd_volinfo_t **volinfo)
+{
+    glusterd_volinfo_t *tmp_volinfo = NULL;
+    int32_t ret = -1;
+    xlator_t *this = NULL;
+    glusterd_conf_t *priv = NULL;
+
+    GF_ASSERT(volname);
+    this = THIS;
+    GF_ASSERT(this);
+
+    priv = this->private;
+    GF_ASSERT(priv);
+
+    cds_list_for_each_entry(tmp_volinfo, &priv->volumes, vol_list)
+    {
+        if (!strcmp(tmp_volinfo->volname, volname)) {
+            gf_msg_debug(this->name, 0, "Volume %s found", volname);
+            ret = 0;
+            *volinfo = tmp_volinfo;
+            break;
         }
+    }
 
-        gf_log (this->name, GF_LOG_DEBUG, "Returning %d", ret);
-        return ret;
+    gf_msg_debug(this->name, 0, "Returning %d", ret);
+    return ret;
+}
+
+gf_boolean_t
+glusterd_volume_exists(const char *volname)
+{
+    glusterd_volinfo_t *tmp_volinfo = NULL;
+    gf_boolean_t volume_found = _gf_false;
+    xlator_t *this = NULL;
+    glusterd_conf_t *priv = NULL;
+
+    GF_ASSERT(volname);
+    this = THIS;
+    GF_ASSERT(this);
+
+    priv = this->private;
+    GF_ASSERT(priv);
+
+    cds_list_for_each_entry(tmp_volinfo, &priv->volumes, vol_list)
+    {
+        if (!strcmp(tmp_volinfo->volname, volname)) {
+            gf_msg_debug(this->name, 0, "Volume %s found", volname);
+            volume_found = _gf_true;
+            break;
+        }
+    }
+
+    return volume_found;
 }
 
 int32_t
-glusterd_service_stop (const char *service, char *pidfile, int sig,
-                       gf_boolean_t force_kill)
+glusterd_service_stop(const char *service, char *pidfile, int sig,
+                      gf_boolean_t force_kill)
 {
-        int32_t  ret = -1;
-        pid_t    pid = -1;
-        xlator_t *this = NULL;
+    int32_t ret = -1;
+    pid_t pid = -1;
+    xlator_t *this = NULL;
 
-        this = THIS;
-        GF_ASSERT (this);
-        if (!gf_is_service_running (pidfile, &pid)) {
+    this = THIS;
+    GF_ASSERT(this);
+    if (!gf_is_service_running(pidfile, &pid)) {
+        ret = 0;
+        gf_msg(this->name, GF_LOG_INFO, 0, GD_MSG_ALREADY_STOPPED,
+               "%s already stopped", service);
+        goto out;
+    }
+    gf_msg_debug(this->name, 0,
+                 "Stopping gluster %s running in pid: "
+                 "%d",
+                 service, pid);
+
+    ret = kill(pid, sig);
+    if (ret) {
+        switch (errno) {
+            case ESRCH:
+                gf_msg_debug(this->name, 0, "%s is already stopped", service);
                 ret = 0;
-                gf_log (this->name, GF_LOG_INFO, "%s already stopped", service);
                 goto out;
+            default:
+                gf_msg(this->name, GF_LOG_ERROR, errno, GD_MSG_SVC_KILL_FAIL,
+                       "Unable to kill %s "
+                       "service, reason:%s",
+                       service, strerror(errno));
         }
-        gf_log (this->name, GF_LOG_DEBUG, "Stopping gluster %s running in pid: "
-                "%d", service, pid);
+    }
+    if (!force_kill)
+        goto out;
 
-        ret = kill (pid, sig);
+    sleep(1);
+    if (gf_is_service_running(pidfile, &pid)) {
+        ret = kill(pid, SIGKILL);
         if (ret) {
-                switch (errno) {
-                case ESRCH:
-                        gf_log (this->name, GF_LOG_DEBUG, "%s is already stopped",
-                                service);
-                        ret = 0;
-                        goto out;
-                default:
-                        gf_msg (this->name, GF_LOG_ERROR, errno,
-                                GD_MSG_SVC_KILL_FAIL, "Unable to kill %s "
-                                "service, reason:%s", service,
-                                strerror (errno));
-                }
-        }
-        if (!force_kill)
-                goto out;
-
-        sleep (1);
-        if (gf_is_service_running (pidfile, NULL)) {
-                ret = kill (pid, SIGKILL);
-                if (ret) {
-                        gf_msg (this->name, GF_LOG_ERROR, errno,
-                                GD_MSG_PID_KILL_FAIL, "Unable to kill pid:%d, "
-                                "reason:%s", pid, strerror(errno));
-                        goto out;
-                }
+            gf_msg(this->name, GF_LOG_ERROR, errno, GD_MSG_PID_KILL_FAIL,
+                   "Unable to kill pid:%d, "
+                   "reason:%s",
+                   pid, strerror(errno));
+            goto out;
         }
+    }
 
-        ret = 0;
+    ret = 0;
 out:
-        return ret;
-}
-
-void
-glusterd_set_socket_filepath (char *sock_filepath, char *sockpath, size_t len)
-{
-        char md5_sum[MD5_DIGEST_LENGTH*2+1] = {0,};
-
-        md5_wrapper ((unsigned char *) sock_filepath, strlen(sock_filepath), md5_sum);
-        snprintf (sockpath, len, "%s/%s.socket", GLUSTERD_SOCK_DIR, md5_sum);
+    return ret;
 }
 
-void
-glusterd_set_brick_socket_filepath (glusterd_volinfo_t *volinfo,
-                                    glusterd_brickinfo_t *brickinfo,
-                                    char *sockpath, size_t len)
+int32_t
+glusterd_service_stop_nolock(const char *service, char *pidfile, int sig,
+                             gf_boolean_t force_kill)
 {
-        char                    export_path[PATH_MAX] = {0,};
-        char                    sock_filepath[PATH_MAX] = {0,};
-        char                    volume_dir[PATH_MAX] = {0,};
-        xlator_t                *this = NULL;
-        glusterd_conf_t         *priv = NULL;
-        int                     expected_file_len = 0;
-
-        expected_file_len = strlen (GLUSTERD_SOCK_DIR) + strlen ("/") +
-                            MD5_DIGEST_LENGTH*2 + strlen (".socket") + 1;
-        GF_ASSERT (len >= expected_file_len);
-        this = THIS;
-        GF_ASSERT (this);
+    int32_t ret = -1;
+    pid_t pid = -1;
+    xlator_t *this = NULL;
+    FILE *file = NULL;
 
-        priv = this->private;
+    this = THIS;
+    GF_ASSERT(this);
 
-        GLUSTERD_GET_VOLUME_DIR (volume_dir, volinfo, priv);
-        GLUSTERD_REMOVE_SLASH_FROM_PATH (brickinfo->path, export_path);
-        snprintf (sock_filepath, PATH_MAX, "%s/run/%s-%s",
-                  volume_dir, brickinfo->hostname, export_path);
+    file = fopen(pidfile, "r+");
+    if (file) {
+        ret = fscanf(file, "%d", &pid);
+        if (ret <= 0) {
+            gf_msg_debug(this->name, 0, "Unable to read pidfile: %s", pidfile);
+            goto out;
+        }
+    }
 
-        glusterd_set_socket_filepath (sock_filepath, sockpath, len);
-}
+    if (kill(pid, 0) < 0) {
+        ret = 0;
+        gf_msg_debug(this->name, 0, "%s process not running: (%d) %s", service,
+                     pid, strerror(errno));
+        goto out;
+    }
+    gf_msg_debug(this->name, 0,
+                 "Stopping gluster %s service running with "
+                 "pid: %d",
+                 service, pid);
+
+    ret = kill(pid, sig);
+    if (ret) {
+        switch (errno) {
+            case ESRCH:
+                gf_msg_debug(this->name, 0, "%s is already stopped", service);
+                ret = 0;
+                goto out;
+            default:
+                gf_msg(this->name, GF_LOG_ERROR, errno, GD_MSG_SVC_KILL_FAIL,
+                       "Unable to kill %s "
+                       "service, reason:%s",
+                       service, strerror(errno));
+        }
+    }
+    if (!force_kill)
+        goto out;
 
-/* connection happens only if it is not aleady connected,
- * reconnections are taken care by rpc-layer
- */
-int32_t
-glusterd_brick_connect (glusterd_volinfo_t  *volinfo,
-                        glusterd_brickinfo_t  *brickinfo, char *socketpath)
-{
-        int                     ret = 0;
-        char                    volume_id_str[64];
-        char                    *brickid = NULL;
-        dict_t                  *options = NULL;
-        struct rpc_clnt         *rpc = NULL;
-        glusterd_conf_t         *priv = THIS->private;
-
-        GF_ASSERT (volinfo);
-        GF_ASSERT (brickinfo);
-        GF_ASSERT (socketpath);
-
-        if (brickinfo->rpc == NULL) {
-                /* Setting frame-timeout to 10mins (600seconds).
-                 * Unix domain sockets ensures that the connection is reliable.
-                 * The default timeout of 30mins used for unreliable network
-                 * connections is too long for unix domain socket connections.
-                 */
-                ret = rpc_transport_unix_options_build (&options, socketpath,
-                                                        600);
-                if (ret)
-                        goto out;
+    sleep(1);
+    if (kill(pid, 0) == 0) {
+        ret = kill(pid, SIGKILL);
+        if (ret) {
+            /* Process is already dead, don't fail */
+            if (errno == ESRCH) {
+                gf_msg(this->name, GF_LOG_ERROR, errno, GD_MSG_PID_KILL_FAIL,
+                       "Unable to find pid:%d, "
+                       "must be dead already. Ignoring.",
+                       pid);
+            } else {
+                gf_msg(this->name, GF_LOG_ERROR, errno, GD_MSG_PID_KILL_FAIL,
+                       "Unable to kill pid:%d, "
+                       "reason:%s",
+                       pid, strerror(errno));
+                goto out;
+            }
+        }
+    }
 
-                uuid_utoa_r (volinfo->volume_id, volume_id_str);
-                ret = gf_asprintf (&brickid, "%s:%s:%s", volume_id_str,
-                                   brickinfo->hostname, brickinfo->path);
-                if (ret < 0)
-                        goto out;
+    ret = 0;
 
-                ret = glusterd_rpc_create (&rpc, options,
-                                           glusterd_brick_rpc_notify,
-                                           brickid);
-                if (ret) {
-                        GF_FREE (brickid);
-                        goto out;
-                }
-                brickinfo->rpc = rpc;
-        }
 out:
+    if (file)
+        fclose(file);
 
-        gf_log ("", GF_LOG_DEBUG, "Returning %d", ret);
-        return ret;
+    return ret;
 }
+void
+glusterd_set_socket_filepath(char *sock_filepath, char *sockpath, size_t len)
+{
+    char xxh64[GF_XXH64_DIGEST_LENGTH * 2 + 1] = {
+        0,
+    };
 
-static int
-_mk_rundir_p (glusterd_volinfo_t *volinfo)
-{
-        char voldir[PATH_MAX]   = {0,};
-        char rundir[PATH_MAX]   = {0,};
-        glusterd_conf_t *priv   = NULL;
-        xlator_t        *this   = NULL;
-        int             ret     = -1;
-
-        this = THIS;
-        priv = this->private;
-        GLUSTERD_GET_VOLUME_DIR (voldir, volinfo, priv);
-        snprintf (rundir, sizeof (rundir)-1, "%s/run", voldir);
-        ret =  mkdir_p (rundir, 0777, _gf_true);
-        if (ret)
-                gf_log (this->name, GF_LOG_ERROR, "Failed to create rundir");
-        return ret;
+    gf_xxh64_wrapper((unsigned char *)sock_filepath, strlen(sock_filepath),
+                     GF_XXHSUM64_DEFAULT_SEED, xxh64);
+    snprintf(sockpath, len, "%s/%s.socket", GLUSTERD_SOCK_DIR, xxh64);
 }
 
+void
+glusterd_set_brick_socket_filepath(glusterd_volinfo_t *volinfo,
+                                   glusterd_brickinfo_t *brickinfo,
+                                   char *sockpath, size_t len)
+{
+    char volume_dir[PATH_MAX] = "";
+    xlator_t *this = NULL;
+    glusterd_conf_t *priv = NULL;
+    int expected_file_len = 0;
+    char export_path[PATH_MAX] = "";
+    char sock_filepath[PATH_MAX] = "";
+    int32_t slen = 0;
+
+    expected_file_len = SLEN(GLUSTERD_SOCK_DIR) + SLEN("/") +
+                        SHA256_DIGEST_LENGTH * 2 + SLEN(".socket") + 1;
+    GF_ASSERT(len >= expected_file_len);
+    this = THIS;
+    GF_ASSERT(this);
+
+    priv = this->private;
+
+    GLUSTERD_GET_VOLUME_PID_DIR(volume_dir, volinfo, priv);
+    GLUSTERD_REMOVE_SLASH_FROM_PATH(brickinfo->path, export_path);
+    slen = snprintf(sock_filepath, PATH_MAX, "%s/run/%s-%s", volume_dir,
+                    brickinfo->hostname, export_path);
+    if (slen < 0) {
+        sock_filepath[0] = 0;
+    }
+    glusterd_set_socket_filepath(sock_filepath, sockpath, len);
+}
+
+/* connection happens only if it is not already connected,
+ * reconnections are taken care by rpc-layer
+ */
 int32_t
-glusterd_volume_start_glusterfs (glusterd_volinfo_t  *volinfo,
-                                 glusterd_brickinfo_t  *brickinfo,
-                                 gf_boolean_t wait)
-{
-        int32_t                 ret = -1;
-        xlator_t                *this = NULL;
-        glusterd_conf_t         *priv = NULL;
-        char                    pidfile[PATH_MAX+1] = {0,};
-        char                    volfile[PATH_MAX] = {0,};
-        runner_t                runner = {0,};
-        char                    exp_path[PATH_MAX] = {0,};
-        char                    logfile[PATH_MAX] = {0,};
-        int                     port = 0;
-        int                     rdma_port = 0;
-        char                    *bind_address = NULL;
-        char                    socketpath[PATH_MAX] = {0};
-        char                    glusterd_uuid[1024] = {0,};
-        char                    valgrind_logfile[PATH_MAX] = {0};
-
-        GF_ASSERT (volinfo);
-        GF_ASSERT (brickinfo);
-
-        this = THIS;
-        GF_ASSERT (this);
-
-        priv = this->private;
-        GF_ASSERT (priv);
-
-        if (brickinfo->snap_status == -1) {
-                gf_log (this->name, GF_LOG_INFO,
-                        "Snapshot is pending on %s:%s. "
-                        "Hence not starting the brick",
-                        brickinfo->hostname,
-                        brickinfo->path);
-                ret = 0;
-                goto out;
+glusterd_brick_connect(glusterd_volinfo_t *volinfo,
+                       glusterd_brickinfo_t *brickinfo, char *socketpath)
+{
+    int ret = 0;
+    char volume_id_str[64] = "";
+    char *brickid = NULL;
+    dict_t *options = NULL;
+    struct rpc_clnt *rpc = NULL;
+
+    GF_ASSERT(volinfo);
+    GF_ASSERT(brickinfo);
+    GF_ASSERT(socketpath);
+
+    if (brickinfo->rpc == NULL) {
+        /* Setting frame-timeout to 10mins (600seconds).
+         * Unix domain sockets ensures that the connection is reliable.
+         * The default timeout of 30mins used for unreliable network
+         * connections is too long for unix domain socket connections.
+         */
+        options = dict_new();
+        if (!options) {
+            gf_smsg("glusterd", GF_LOG_ERROR, errno, GD_MSG_DICT_CREATE_FAIL,
+                    NULL);
+            goto out;
         }
 
-        ret = _mk_rundir_p (volinfo);
+        ret = rpc_transport_unix_options_build(options, socketpath, 600);
         if (ret)
-                goto out;
-
-        glusterd_set_brick_socket_filepath (volinfo, brickinfo, socketpath,
-                                            sizeof (socketpath));
-
-        GLUSTERD_GET_BRICK_PIDFILE (pidfile, volinfo, brickinfo, priv);
-        if (gf_is_service_running (pidfile, NULL))
-                goto connect;
-
-        port = brickinfo->port;
-        if (!port)
-                port = pmap_registry_alloc (THIS);
+            goto out;
 
-        /* Build the exp_path, before starting the glusterfsd even in
-           valgrind mode. Otherwise all the glusterfsd processes start
-           writing the valgrind log to the same file.
-        */
-        GLUSTERD_REMOVE_SLASH_FROM_PATH (brickinfo->path, exp_path);
-        runinit (&runner);
-
-        if (priv->valgrind) {
-                /* Run bricks with valgrind */
-                if (volinfo->logdir) {
-                        snprintf (valgrind_logfile, PATH_MAX,
-                                  "%s/valgrind-%s-%s.log",
-                                  volinfo->logdir,
-                                  volinfo->volname, exp_path);
-                } else {
-                        snprintf (valgrind_logfile, PATH_MAX,
-                                  "%s/bricks/valgrind-%s-%s.log",
-                                  DEFAULT_LOG_FILE_DIRECTORY,
-                                  volinfo->volname, exp_path);
-                }
+        uuid_utoa_r(volinfo->volume_id, volume_id_str);
+        ret = gf_asprintf(&brickid, "%s:%s:%s", volume_id_str,
+                          brickinfo->hostname, brickinfo->path);
+        if (ret < 0)
+            goto out;
 
-                runner_add_args (&runner, "valgrind", "--leak-check=full",
-                                 "--trace-children=yes", "--track-origins=yes",
-                                 NULL);
-                runner_argprintf (&runner, "--log-file=%s", valgrind_logfile);
+        ret = glusterd_rpc_create(&rpc, options, glusterd_brick_rpc_notify,
+                                  brickid, _gf_false);
+        if (ret) {
+            GF_FREE(brickid);
+            goto out;
         }
+        brickinfo->rpc = rpc;
+    }
+out:
+    if (options)
+        dict_unref(options);
+    gf_msg_debug("glusterd", 0, "Returning %d", ret);
+    return ret;
+}
 
-        if (volinfo->is_snap_volume) {
-                snprintf (volfile, PATH_MAX,"/%s/%s/%s.%s.%s",
-                          GLUSTERD_VOL_SNAP_DIR_PREFIX,
-                          volinfo->snapshot->snapname, volinfo->volname,
-                          brickinfo->hostname, exp_path);
-        } else {
-                snprintf (volfile, PATH_MAX, "%s.%s.%s", volinfo->volname,
-                          brickinfo->hostname, exp_path);
-        }
+static int
+_mk_rundir_p(glusterd_volinfo_t *volinfo)
+{
+    char rundir[PATH_MAX] = "";
+    glusterd_conf_t *priv = NULL;
+    xlator_t *this = NULL;
+    int ret = -1;
 
+    this = THIS;
+    priv = this->private;
+    GLUSTERD_GET_VOLUME_PID_DIR(rundir, volinfo, priv);
+    ret = mkdir_p(rundir, 0755, _gf_true);
+    if (ret)
+        gf_msg(this->name, GF_LOG_ERROR, errno, GD_MSG_CREATE_DIR_FAILED,
+               "Failed to create rundir");
+    return ret;
+}
+
+int32_t
+glusterd_volume_start_glusterfs(glusterd_volinfo_t *volinfo,
+                                glusterd_brickinfo_t *brickinfo,
+                                gf_boolean_t wait)
+{
+    int32_t ret = -1;
+    xlator_t *this = NULL;
+    glusterd_conf_t *priv = NULL;
+    char pidfile[PATH_MAX + 1] = "";
+    char volfile[PATH_MAX] = "";
+    runner_t runner = {
+        0,
+    };
+    char exp_path[PATH_MAX] = "";
+    char logfile[PATH_MAX] = "";
+    int port = 0;
+    int rdma_port = 0;
+    char *bind_address = NULL;
+    char *localtime_logging = NULL;
+    char socketpath[PATH_MAX] = "";
+    char glusterd_uuid[1024] = "";
+    char valgrind_logfile[PATH_MAX] = "";
+    char rdma_brick_path[PATH_MAX] = "";
+    struct rpc_clnt *rpc = NULL;
+    rpc_clnt_connection_t *conn = NULL;
+    int pid = -1;
+    int32_t len = 0;
+    glusterd_brick_proc_t *brick_proc = NULL;
+    char *inet_family = NULL;
+    char *global_threading = NULL;
+    bool threading = false;
+
+    GF_ASSERT(volinfo);
+    GF_ASSERT(brickinfo);
+
+    this = THIS;
+    GF_ASSERT(this);
+
+    priv = this->private;
+    GF_ASSERT(priv);
+
+    if (brickinfo->snap_status == -1) {
+        gf_msg(this->name, GF_LOG_INFO, 0, GD_MSG_SNAPSHOT_PENDING,
+               "Snapshot is pending on %s:%s. "
+               "Hence not starting the brick",
+               brickinfo->hostname, brickinfo->path);
+        ret = 0;
+        goto out;
+    }
+
+    GLUSTERD_GET_BRICK_PIDFILE(pidfile, volinfo, brickinfo, priv);
+    if (gf_is_service_running(pidfile, &pid)) {
+        goto connect;
+    }
+
+    /*
+     * There are all sorts of races in the start/stop code that could leave
+     * a UNIX-domain socket or RPC-client object associated with a
+     * long-dead incarnation of this brick, while the new incarnation is
+     * listening on a new socket at the same path and wondering why we
+     * haven't shown up.  To avoid the whole mess and be on the safe side,
+     * we just blow away anything that might have been left over, and start
+     * over again.
+     */
+    glusterd_set_brick_socket_filepath(volinfo, brickinfo, socketpath,
+                                       sizeof(socketpath));
+    (void)glusterd_unlink_file(socketpath);
+    rpc = brickinfo->rpc;
+    if (rpc) {
+        brickinfo->rpc = NULL;
+        conn = &rpc->conn;
+        pthread_mutex_lock(&conn->lock);
+        if (conn->reconnect) {
+            (void)gf_timer_call_cancel(rpc->ctx, conn->reconnect);
+            conn->reconnect = NULL;
+        }
+        pthread_mutex_unlock(&conn->lock);
+        rpc_clnt_unref(rpc);
+    }
+
+    port = pmap_assign_port(THIS, brickinfo->port, brickinfo->path);
+    if (!port) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_PORTS_EXHAUSTED,
+               "All the ports in the range are exhausted, can't start "
+               "brick %s for volume %s",
+               brickinfo->path, volinfo->volname);
+        ret = -1;
+        goto out;
+    }
+    /* Build the exp_path, before starting the glusterfsd even in
+       valgrind mode. Otherwise all the glusterfsd processes start
+       writing the valgrind log to the same file.
+    */
+    GLUSTERD_REMOVE_SLASH_FROM_PATH(brickinfo->path, exp_path);
+
+retry:
+    runinit(&runner);
+
+    if (this->ctx->cmd_args.vgtool != _gf_none) {
+        /* Run bricks with valgrind. */
         if (volinfo->logdir) {
-                snprintf (logfile, PATH_MAX, "%s/%s.log",
-                          volinfo->logdir, exp_path);
+            len = snprintf(valgrind_logfile, PATH_MAX, "%s/valgrind-%s-%s.log",
+                           volinfo->logdir, volinfo->volname, exp_path);
         } else {
-                snprintf (logfile, PATH_MAX, "%s/bricks/%s.log",
-                          DEFAULT_LOG_FILE_DIRECTORY, exp_path);
-        }
-        if (!brickinfo->logfile)
-                brickinfo->logfile = gf_strdup (logfile);
-
-        (void) snprintf (glusterd_uuid, 1024, "*-posix.glusterd-uuid=%s",
-                         uuid_utoa (MY_UUID));
-        runner_add_args (&runner, SBIN_DIR"/glusterfsd",
-                         "-s", brickinfo->hostname, "--volfile-id", volfile,
-                         "-p", pidfile, "-S", socketpath,
-                         "--brick-name", brickinfo->path,
-                         "-l", brickinfo->logfile,
-                         "--xlator-option", glusterd_uuid,
-                         NULL);
-
-        runner_add_arg (&runner, "--brick-port");
-        if (volinfo->transport_type != GF_TRANSPORT_BOTH_TCP_RDMA) {
-                runner_argprintf (&runner, "%d", port);
-        } else {
-                rdma_port = brickinfo->rdma_port;
-                if (!rdma_port)
-                        rdma_port = pmap_registry_alloc (THIS);
-                runner_argprintf (&runner, "%d,%d", port, rdma_port);
-                runner_add_arg (&runner, "--xlator-option");
-                runner_argprintf (&runner, "%s-server.transport.rdma.listen-port=%d",
-                                  volinfo->volname, rdma_port);
+            len = snprintf(valgrind_logfile, PATH_MAX,
+                           "%s/bricks/valgrind-%s-%s.log", priv->logdir,
+                           volinfo->volname, exp_path);
         }
-
-        runner_add_arg (&runner, "--xlator-option");
-        runner_argprintf (&runner, "%s-server.listen-port=%d",
-                          volinfo->volname, port);
-
-        if (dict_get_str (this->options, "transport.socket.bind-address",
-                          &bind_address) == 0) {
-                runner_add_arg (&runner, "--xlator-option");
-                runner_argprintf (&runner, "transport.socket.bind-address=%s",
-                                  bind_address);
+        if ((len < 0) || (len >= PATH_MAX)) {
+            ret = -1;
+            goto out;
         }
 
-        if (volinfo->transport_type == GF_TRANSPORT_RDMA)
-                runner_argprintf (&runner, "--volfile-server-transport=rdma");
-        else if (volinfo->transport_type == GF_TRANSPORT_BOTH_TCP_RDMA)
-                runner_argprintf (&runner,
-                                  "--volfile-server-transport=socket,rdma");
-
-        if (volinfo->memory_accounting)
-                runner_add_arg (&runner, "--mem-accounting");
-
-        runner_log (&runner, "", GF_LOG_DEBUG, "Starting GlusterFS");
-        if (wait) {
-                synclock_unlock (&priv->big_lock);
-                ret = runner_run (&runner);
-                synclock_lock (&priv->big_lock);
-
-        } else {
-                ret = runner_run_nowait (&runner);
+        if (this->ctx->cmd_args.vgtool == _gf_memcheck)
+            runner_add_args(&runner, "valgrind", "--leak-check=full",
+                            "--trace-children=yes", "--track-origins=yes",
+                            NULL);
+        else
+            runner_add_args(&runner, "valgrind", "--tool=drd", NULL);
+
+        runner_argprintf(&runner, "--log-file=%s", valgrind_logfile);
+    }
+
+    if (volinfo->is_snap_volume) {
+        len = snprintf(volfile, PATH_MAX, "/%s/%s/%s/%s.%s.%s",
+                       GLUSTERD_VOL_SNAP_DIR_PREFIX,
+                       volinfo->snapshot->snapname, volinfo->volname,
+                       volinfo->volname, brickinfo->hostname, exp_path);
+    } else {
+        len = snprintf(volfile, PATH_MAX, "%s.%s.%s", volinfo->volname,
+                       brickinfo->hostname, exp_path);
+    }
+    if ((len < 0) || (len >= PATH_MAX)) {
+        ret = -1;
+        goto out;
+    }
+
+    if (volinfo->logdir) {
+        len = snprintf(logfile, PATH_MAX, "%s/%s.log", volinfo->logdir,
+                       exp_path);
+    } else {
+        len = snprintf(logfile, PATH_MAX, "%s/bricks/%s.log", priv->logdir,
+                       exp_path);
+    }
+    if ((len < 0) || (len >= PATH_MAX)) {
+        ret = -1;
+        goto out;
+    }
+
+    if (!brickinfo->logfile)
+        brickinfo->logfile = gf_strdup(logfile);
+
+    (void)snprintf(glusterd_uuid, 1024, "*-posix.glusterd-uuid=%s",
+                   uuid_utoa(MY_UUID));
+    runner_add_args(&runner, SBIN_DIR "/glusterfsd", "-s", brickinfo->hostname,
+                    "--volfile-id", volfile, "-p", pidfile, "-S", socketpath,
+                    "--brick-name", brickinfo->path, "-l", brickinfo->logfile,
+                    "--xlator-option", glusterd_uuid, "--process-name", "brick",
+                    NULL);
+
+    if (dict_get_strn(priv->opts, GLUSTERD_LOCALTIME_LOGGING_KEY,
+                      SLEN(GLUSTERD_LOCALTIME_LOGGING_KEY),
+                      &localtime_logging) == 0) {
+        if (strcmp(localtime_logging, "enable") == 0)
+            runner_add_arg(&runner, "--localtime-logging");
+    }
+
+    runner_add_arg(&runner, "--brick-port");
+    if (volinfo->transport_type != GF_TRANSPORT_BOTH_TCP_RDMA) {
+        runner_argprintf(&runner, "%d", port);
+    } else {
+        len = snprintf(rdma_brick_path, sizeof(rdma_brick_path), "%s.rdma",
+                       brickinfo->path);
+        if ((len < 0) || (len >= sizeof(rdma_brick_path))) {
+            ret = -1;
+            goto out;
         }
-
-        if (ret)
+        rdma_port = pmap_assign_port(THIS, brickinfo->rdma_port,
+                                     rdma_brick_path);
+        if (!rdma_port) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_PORTS_EXHAUSTED,
+                   "All rdma ports in the "
+                   "range are exhausted, can't start brick %s for "
+                   "volume %s",
+                   rdma_brick_path, volinfo->volname);
+            ret = -1;
+            goto out;
+        }
+        runner_argprintf(&runner, "%d,%d", port, rdma_port);
+        runner_add_arg(&runner, "--xlator-option");
+        runner_argprintf(&runner, "%s-server.transport.rdma.listen-port=%d",
+                         volinfo->volname, rdma_port);
+    }
+
+    if (dict_get_strn(volinfo->dict, VKEY_CONFIG_GLOBAL_THREADING,
+                      SLEN(VKEY_CONFIG_GLOBAL_THREADING),
+                      &global_threading) == 0) {
+        if ((gf_string2boolean(global_threading, &threading) == 0) &&
+            threading) {
+            runner_add_arg(&runner, "--global-threading");
+        }
+    }
+
+    runner_add_arg(&runner, "--xlator-option");
+    runner_argprintf(&runner, "%s-server.listen-port=%d", volinfo->volname,
+                     port);
+
+    if (dict_get_strn(this->options, "transport.socket.bind-address",
+                      SLEN("transport.socket.bind-address"),
+                      &bind_address) == 0) {
+        runner_add_arg(&runner, "--xlator-option");
+        runner_argprintf(&runner, "transport.socket.bind-address=%s",
+                         bind_address);
+    }
+
+    if (volinfo->transport_type == GF_TRANSPORT_RDMA)
+        runner_argprintf(&runner, "--volfile-server-transport=rdma");
+    else if (volinfo->transport_type == GF_TRANSPORT_BOTH_TCP_RDMA)
+        runner_argprintf(&runner, "--volfile-server-transport=socket,rdma");
+
+    ret = dict_get_str(this->options, "transport.address-family", &inet_family);
+    if (!ret) {
+        runner_add_arg(&runner, "--xlator-option");
+        runner_argprintf(&runner, "transport.address-family=%s", inet_family);
+    }
+
+    if (volinfo->memory_accounting)
+        runner_add_arg(&runner, "--mem-accounting");
+
+    if (is_brick_mx_enabled())
+        runner_add_arg(&runner, "--brick-mux");
+
+    runner_log(&runner, "", GF_LOG_DEBUG, "Starting GlusterFS");
+
+    brickinfo->port = port;
+    brickinfo->rdma_port = rdma_port;
+    brickinfo->status = GF_BRICK_STARTING;
+    brickinfo->port_registered = _gf_false;
+
+    if (wait) {
+        synclock_unlock(&priv->big_lock);
+        errno = 0;
+        ret = runner_run(&runner);
+        if (errno != 0)
+            ret = errno;
+        synclock_lock(&priv->big_lock);
+
+        if (ret == EADDRINUSE) {
+            /* retry after getting a new port */
+            gf_msg(this->name, GF_LOG_WARNING, -ret,
+                   GD_MSG_SRC_BRICK_PORT_UNAVAIL,
+                   "Port %d is used by other process", port);
+
+            port = pmap_registry_alloc(this);
+            if (!port) {
+                gf_msg(this->name, GF_LOG_CRITICAL, 0, GD_MSG_NO_FREE_PORTS,
+                       "Couldn't allocate a port");
+                ret = -1;
                 goto out;
-
-        brickinfo->port = port;
-        brickinfo->rdma_port = rdma_port;
+            }
+            gf_msg(this->name, GF_LOG_NOTICE, 0, GD_MSG_RETRY_WITH_NEW_PORT,
+                   "Retrying to start brick %s with new port %d",
+                   brickinfo->path, port);
+            goto retry;
+        }
+    } else {
+        ret = runner_run_nowait(&runner);
+    }
+
+    if (ret) {
+        brickinfo->port = 0;
+        brickinfo->rdma_port = 0;
+        goto out;
+    }
+
+    ret = glusterd_brickprocess_new(&brick_proc);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_BRICKPROC_NEW_FAILED,
+               "Failed to create "
+               "new brick process instance");
+        goto out;
+    }
+
+    brick_proc->port = brickinfo->port;
+    cds_list_add_tail(&brick_proc->brick_proc_list, &priv->brick_procs);
+    brickinfo->brick_proc = brick_proc;
+    cds_list_add_tail(&brickinfo->mux_bricks, &brick_proc->bricks);
+    brickinfo->brick_proc = brick_proc;
+    brick_proc->brick_count++;
 
 connect:
-        ret = glusterd_brick_connect (volinfo, brickinfo, socketpath);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "Failed to connect to brick %s:%s on %s",
-                        brickinfo->hostname, brickinfo->path, socketpath);
-                goto out;
-        }
+    ret = glusterd_brick_connect(volinfo, brickinfo, socketpath);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_BRICK_DISCONNECTED,
+               "Failed to connect to brick %s:%s on %s", brickinfo->hostname,
+               brickinfo->path, socketpath);
+        goto out;
+    }
+
 out:
-        return ret;
+    if (ret)
+        brickinfo->status = GF_BRICK_STOPPED;
+    return ret;
 }
 
 int32_t
-glusterd_brick_unlink_socket_file (glusterd_volinfo_t *volinfo,
-                                   glusterd_brickinfo_t *brickinfo)
+glusterd_brick_unlink_socket_file(glusterd_volinfo_t *volinfo,
+                                  glusterd_brickinfo_t *brickinfo)
 {
-        char                    path[PATH_MAX] = {0,};
-        char                    socketpath[PATH_MAX] = {0};
-        xlator_t                *this = NULL;
-        glusterd_conf_t         *priv = NULL;
+    char path[PATH_MAX] = "";
+    char socketpath[PATH_MAX] = "";
+    xlator_t *this = NULL;
+    glusterd_conf_t *priv = NULL;
 
-        GF_ASSERT (volinfo);
-        GF_ASSERT (brickinfo);
+    GF_ASSERT(volinfo);
+    GF_ASSERT(brickinfo);
 
-        this = THIS;
-        GF_ASSERT (this);
+    this = THIS;
+    GF_ASSERT(this);
 
-        priv = this->private;
-        GLUSTERD_GET_VOLUME_DIR (path, volinfo, priv);
-        glusterd_set_brick_socket_filepath (volinfo, brickinfo, socketpath,
-                                            sizeof (socketpath));
+    priv = this->private;
+    GLUSTERD_GET_VOLUME_DIR(path, volinfo, priv);
+    glusterd_set_brick_socket_filepath(volinfo, brickinfo, socketpath,
+                                       sizeof(socketpath));
 
-        return glusterd_unlink_file (socketpath);
+    return glusterd_unlink_file(socketpath);
 }
 
 int32_t
-glusterd_brick_disconnect (glusterd_brickinfo_t *brickinfo)
+glusterd_brick_disconnect(glusterd_brickinfo_t *brickinfo)
 {
-        rpc_clnt_t              *rpc = NULL;
-        glusterd_conf_t         *priv = THIS->private;
+    rpc_clnt_t *rpc = NULL;
+    glusterd_conf_t *priv = THIS->private;
 
-        GF_ASSERT (brickinfo);
+    GF_ASSERT(brickinfo);
 
-        if (!brickinfo) {
-                gf_log_callingfn ("glusterd", GF_LOG_WARNING, "!brickinfo");
-                return -1;
-        }
+    if (!brickinfo) {
+        gf_msg_callingfn("glusterd", GF_LOG_WARNING, EINVAL,
+                         GD_MSG_BRICK_NOT_FOUND, "!brickinfo");
+        return -1;
+    }
 
-        rpc            = brickinfo->rpc;
-        brickinfo->rpc = NULL;
+    rpc = brickinfo->rpc;
+    brickinfo->rpc = NULL;
 
-        if (rpc) {
-                glusterd_rpc_clnt_unref (priv, rpc);
-        }
+    if (rpc) {
+        glusterd_rpc_clnt_unref(priv, rpc);
+    }
 
-        return 0;
+    return 0;
 }
 
-int32_t
-glusterd_volume_stop_glusterfs (glusterd_volinfo_t  *volinfo,
-                                glusterd_brickinfo_t   *brickinfo,
-                                gf_boolean_t del_brick)
-{
-        xlator_t        *this                   = NULL;
-        glusterd_conf_t *priv                   = NULL;
-        char            pidfile[PATH_MAX]       = {0,};
-        int             ret                     = 0;
+static gf_boolean_t
+unsafe_option(dict_t *this, char *key, data_t *value, void *arg)
+{
+    /*
+     * Certain options are safe because they're already being handled other
+     * ways, such as being copied down to the bricks (all auth options) or
+     * being made irrelevant (event-threads).  All others are suspect and
+     * must be checked in the next function.
+     */
+    if (fnmatch("*auth*", key, 0) == 0) {
+        return _gf_false;
+    }
 
-        GF_ASSERT (volinfo);
-        GF_ASSERT (brickinfo);
+    if (fnmatch("*event-threads", key, 0) == 0) {
+        return _gf_false;
+    }
 
-        this = THIS;
-        GF_ASSERT (this);
+    if (fnmatch("*diagnostics.brick-log*", key, 0) == 0) {
+        return _gf_false;
+    }
 
-        priv = this->private;
-        if (del_brick)
-                cds_list_del_init (&brickinfo->brick_list);
-
-        if (GLUSTERD_STATUS_STARTED == volinfo->status) {
-                (void) glusterd_brick_disconnect (brickinfo);
-                GLUSTERD_GET_BRICK_PIDFILE (pidfile, volinfo, brickinfo, priv);
-                ret = glusterd_service_stop ("brick", pidfile, SIGTERM, _gf_false);
-                if (ret == 0) {
-                        glusterd_set_brick_status (brickinfo, GF_BRICK_STOPPED);
-                        (void) glusterd_brick_unlink_socket_file (volinfo, brickinfo);
-                }
-        }
+    if (fnmatch("*diagnostics.client-log*", key, 0) == 0) {
+        return _gf_false;
+    }
+    if (fnmatch("user.*", key, 0) == 0) {
+        return _gf_false;
+    }
 
-        if (del_brick)
-                glusterd_delete_brick (volinfo, brickinfo);
+    return _gf_true;
+}
 
-        return ret;
+static int
+opts_mismatch(dict_t *dict1, char *key, data_t *value1, void *dict2)
+{
+    data_t *value2 = dict_get(dict2, key);
+    int32_t min_len;
+
+    /*
+     * If the option is only present on one, we can either look at the
+     * default or assume a mismatch.  Looking at the default is pretty
+     * hard, because that's part of a structure within each translator and
+     * there's no dlopen interface to get at it, so we assume a mismatch.
+     * If the user really wants them to match (and for their bricks to be
+     * multiplexed, they can always reset the option).
+     */
+    if (!value2) {
+        gf_log(THIS->name, GF_LOG_DEBUG, "missing option %s", key);
+        return -1;
+    }
+
+    min_len = MIN(value1->len, value2->len);
+    if (strncmp(value1->data, value2->data, min_len) != 0) {
+        gf_log(THIS->name, GF_LOG_DEBUG, "option mismatch, %s, %s != %s", key,
+               value1->data, value2->data);
+        return -1;
+    }
+
+    return 0;
 }
 
-/* Free LINE[0..N-1] and then the LINE buffer.  */
-static void
-free_lines (char **line, size_t n)
+int
+glusterd_brickprocess_delete(glusterd_brick_proc_t *brick_proc)
 {
-  size_t i;
-  for (i = 0; i < n; i++)
-    GF_FREE (line[i]);
-  GF_FREE (line);
+    cds_list_del_init(&brick_proc->brick_proc_list);
+    cds_list_del_init(&brick_proc->bricks);
+
+    GF_FREE(brick_proc);
+
+    return 0;
 }
 
-char **
-glusterd_readin_file (const char *filepath, int *line_count)
+int
+glusterd_brick_process_remove_brick(glusterd_brickinfo_t *brickinfo,
+                                    int *last_brick)
 {
-        int         ret                    = -1;
-        int         n                      = 8;
-        int         counter                = 0;
-        char        buffer[PATH_MAX + 256] = {0};
-        char      **lines                  = NULL;
-        FILE       *fp                     = NULL;
-        void       *p;
-
-        fp = fopen (filepath, "r");
-        if (!fp)
-                goto out;
+    int ret = -1;
+    xlator_t *this = NULL;
+    glusterd_conf_t *priv = NULL;
+    glusterd_brick_proc_t *brick_proc = NULL;
 
-        lines = GF_CALLOC (1, n * sizeof (*lines), gf_gld_mt_charptr);
-        if (!lines)
-                goto out;
+    this = THIS;
+    GF_VALIDATE_OR_GOTO("glusterd", this, out);
 
-        for (counter = 0; fgets (buffer, sizeof (buffer), fp); counter++) {
-
-                if (counter == n-1) {
-                        n *= 2;
-                        p = GF_REALLOC (lines, n * sizeof (char *));
-                        if (!p) {
-                                free_lines (lines, n/2);
-                                lines = NULL;
-                                goto out;
-                        }
-                        lines = p;
-                }
+    priv = this->private;
+    GF_VALIDATE_OR_GOTO(this->name, priv, out);
+    GF_VALIDATE_OR_GOTO(this->name, brickinfo, out);
 
-                lines[counter] = gf_strdup (buffer);
+    brick_proc = brickinfo->brick_proc;
+    if (!brick_proc) {
+        if (brickinfo->status != GF_BRICK_STARTED) {
+            /* this function will be called from gluster_pmap_signout and
+             * glusterd_volume_stop_glusterfs. So it is possible to have
+             * brick_proc set as null.
+             */
+            ret = 0;
         }
+        goto out;
+    }
 
-        lines[counter] = NULL;
-        /* Reduce allocation to minimal size.  */
-        p = GF_REALLOC (lines, (counter + 1) * sizeof (char *));
-        if (!p) {
-                free_lines (lines, counter);
-                lines = NULL;
-                goto out;
-        }
-        lines = p;
+    GF_VALIDATE_OR_GOTO(this->name, (brick_proc->brick_count > 0), out);
 
-        *line_count = counter;
-        ret = 0;
+    cds_list_del_init(&brickinfo->mux_bricks);
+    brick_proc->brick_count--;
 
- out:
+    /* If all bricks have been removed, delete the brick process */
+    if (brick_proc->brick_count == 0) {
+        if (last_brick != NULL)
+            *last_brick = 1;
+        ret = glusterd_brickprocess_delete(brick_proc);
         if (ret)
-                gf_log (THIS->name, GF_LOG_ERROR, "%s", strerror (errno));
-        if (fp)
-                fclose (fp);
-
-        return lines;
-}
-
-int
-glusterd_compare_lines (const void *a, const void *b) {
-
-        return strcmp(* (char * const *) a, * (char * const *) b);
+            goto out;
+    }
+    brickinfo->brick_proc = NULL;
+    ret = 0;
+out:
+    return ret;
 }
 
 int
-glusterd_sort_and_redirect (const char *src_filepath, int dest_fd)
+glusterd_brick_process_add_brick(glusterd_brickinfo_t *brickinfo,
+                                 glusterd_brickinfo_t *parent_brickinfo)
 {
-        int            ret          = -1;
-        int            line_count   = 0;
-        int            counter      = 0;
-        char         **lines        = NULL;
+    int ret = -1;
+    xlator_t *this = NULL;
+    glusterd_conf_t *priv = NULL;
+    glusterd_brick_proc_t *brick_proc = NULL;
 
+    this = THIS;
+    GF_VALIDATE_OR_GOTO("glusterd", this, out);
 
-        if (!src_filepath || dest_fd < 0)
-                goto out;
+    priv = this->private;
+    GF_VALIDATE_OR_GOTO(this->name, priv, out);
+    GF_VALIDATE_OR_GOTO(this->name, brickinfo, out);
 
-        lines = glusterd_readin_file (src_filepath, &line_count);
-        if (!lines)
+    if (!parent_brickinfo) {
+        ret = glusterd_brick_proc_for_port(brickinfo->port, &brick_proc);
+        if (ret) {
+            ret = glusterd_brickprocess_new(&brick_proc);
+            if (ret) {
+                gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_BRICKPROC_NEW_FAILED,
+                       "Failed to create "
+                       "new brick process instance");
                 goto out;
+            }
 
-        qsort (lines, line_count, sizeof (*lines), glusterd_compare_lines);
-
-        for (counter = 0; lines[counter]; counter++) {
+            brick_proc->port = brickinfo->port;
 
-                ret = write (dest_fd, lines[counter],
-                             strlen (lines[counter]));
-                if (ret < 0)
-                        goto out;
-
-                GF_FREE (lines[counter]);
+            cds_list_add_tail(&brick_proc->brick_proc_list, &priv->brick_procs);
         }
-
+    } else {
         ret = 0;
- out:
-        GF_FREE (lines);
+        brick_proc = parent_brickinfo->brick_proc;
+    }
 
-        return ret;
+    cds_list_add_tail(&brickinfo->mux_bricks, &brick_proc->bricks);
+    brickinfo->brick_proc = brick_proc;
+    brick_proc->brick_count++;
+out:
+    return ret;
 }
 
+/* ret = 0 only when you get a brick process associated with the port
+ * ret = -1 otherwise
+ */
 int
-glusterd_volume_compute_cksum (glusterd_volinfo_t  *volinfo, char *cksum_path,
-                               char *filepath, gf_boolean_t is_quota_conf,
-                               uint32_t *cs)
+glusterd_brick_proc_for_port(int port, glusterd_brick_proc_t **brickprocess)
 {
-        int32_t                 ret                     = -1;
-        uint32_t                cksum                   = 0;
-        int                     fd                      = -1;
-        int                     sort_fd                 = 0;
-        char                    sort_filepath[PATH_MAX] = {0};
-        char                   *cksum_path_final        = NULL;
-        char                    buf[4096]               = {0,};
-        gf_boolean_t            unlink_sortfile         = _gf_false;
-        glusterd_conf_t        *priv                    = NULL;
-        xlator_t               *this                    = NULL;
+    int ret = -1;
+    xlator_t *this = NULL;
+    glusterd_conf_t *priv = NULL;
+    glusterd_brick_proc_t *brick_proc = NULL;
 
-        GF_ASSERT (volinfo);
-        this = THIS;
-        priv = THIS->private;
-        GF_ASSERT (priv);
+    this = THIS;
+    GF_VALIDATE_OR_GOTO("glusterd", this, out);
 
-        fd = open (cksum_path, O_RDWR | O_APPEND | O_CREAT| O_TRUNC, 0600);
+    priv = this->private;
+    GF_VALIDATE_OR_GOTO(this->name, priv, out);
 
-        if (-1 == fd) {
-                gf_log (this->name, GF_LOG_ERROR, "Unable to open %s,"
-                        " errno: %d", cksum_path, errno);
-                ret = -1;
-                goto out;
+    cds_list_for_each_entry(brick_proc, &priv->brick_procs, brick_proc_list)
+    {
+        if (brick_proc->port == port) {
+            *brickprocess = brick_proc;
+            ret = 0;
+            break;
         }
+    }
+out:
+    return ret;
+}
 
-        if (!is_quota_conf) {
-                snprintf (sort_filepath, sizeof (sort_filepath),
-                          "/tmp/%s.XXXXXX", volinfo->volname);
-
-                sort_fd = mkstemp (sort_filepath);
-                if (sort_fd < 0) {
-                        gf_log (this->name, GF_LOG_ERROR, "Could not generate "
-                                "temp file, reason: %s for volume: %s",
-                                strerror (errno), volinfo->volname);
-                        goto out;
-                } else {
-                        unlink_sortfile = _gf_true;
-                }
+int32_t
+glusterd_volume_stop_glusterfs(glusterd_volinfo_t *volinfo,
+                               glusterd_brickinfo_t *brickinfo,
+                               gf_boolean_t del_brick)
+{
+    xlator_t *this = NULL;
+    glusterd_conf_t *conf = NULL;
+    int ret = -1;
+    char *op_errstr = NULL;
+    char pidfile[PATH_MAX] = "";
+    int last_brick = -1;
+
+    GF_ASSERT(volinfo);
+    GF_ASSERT(brickinfo);
+
+    this = THIS;
+    GF_ASSERT(this);
+
+    conf = this->private;
+    GF_VALIDATE_OR_GOTO(this->name, conf, out);
+
+    ret = glusterd_brick_process_remove_brick(brickinfo, &last_brick);
+    if (ret) {
+        gf_msg_debug(this->name, 0,
+                     "Couldn't remove brick from"
+                     " brick process");
+        goto out;
+    }
+
+    if (del_brick)
+        cds_list_del_init(&brickinfo->brick_list);
+
+    if (GLUSTERD_STATUS_STARTED == volinfo->status) {
+        /*
+         * In a post-multiplexing world, even if we're not actually
+         * doing any multiplexing, just dropping the RPC connection
+         * isn't enough.  There might be many such connections during
+         * the brick daemon's lifetime, even if we only consider the
+         * management RPC port (because tests etc. might be manually
+         * attaching and detaching bricks).  Therefore, we have to send
+         * an actual signal instead.
+         */
+        if (is_brick_mx_enabled() && last_brick != 1) {
+            ret = send_attach_req(this, brickinfo->rpc, brickinfo->path, NULL,
+                                  NULL, GLUSTERD_BRICK_TERMINATE);
+            if (ret && brickinfo->status == GF_BRICK_STARTED) {
+                gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_BRICK_STOP_FAIL,
+                       "Failed to send"
+                       " detach request for brick %s",
+                       brickinfo->path);
+                goto out;
+            }
+            gf_log(this->name, GF_LOG_INFO,
+                   "Detach request for "
+                   "brick %s:%s is sent successfully",
+                   brickinfo->hostname, brickinfo->path);
 
-                /* sort the info file, result in sort_filepath */
+        } else {
+            gf_msg_debug(this->name, 0,
+                         "About to stop glusterfsd"
+                         " for brick %s:%s",
+                         brickinfo->hostname, brickinfo->path);
+            ret = glusterd_brick_terminate(volinfo, brickinfo, NULL, 0,
+                                           &op_errstr);
+            if (ret && brickinfo->status == GF_BRICK_STARTED) {
+                gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_BRICK_STOP_FAIL,
+                       "Failed to kill"
+                       " the brick %s",
+                       brickinfo->path);
+                goto out;
+            }
+
+            if (op_errstr) {
+                GF_FREE(op_errstr);
+            }
+            if (is_brick_mx_enabled()) {
+                /* In case of brick multiplexing we need to make
+                 * sure the port is cleaned up from here as the
+                 * RPC connection may not have been originated
+                 * for the same brick instance
+                 */
+                pmap_registry_remove(THIS, brickinfo->port, brickinfo->path,
+                                     GF_PMAP_PORT_BRICKSERVER, NULL, _gf_true);
+            }
+        }
 
-                ret = glusterd_sort_and_redirect (filepath, sort_fd);
-                if (ret) {
-                        gf_log (this->name, GF_LOG_ERROR, "sorting info file "
-                                "failed");
-                        goto out;
-                }
+        (void)glusterd_brick_disconnect(brickinfo);
+        ret = 0;
+    }
 
-                ret = close (sort_fd);
-                if (ret)
-                        goto out;
-        }
+    GLUSTERD_GET_BRICK_PIDFILE(pidfile, volinfo, brickinfo, conf);
+    gf_msg_debug(this->name, 0, "Unlinking pidfile %s", pidfile);
+    (void)sys_unlink(pidfile);
 
-        cksum_path_final = is_quota_conf ? filepath : sort_filepath;
+    brickinfo->status = GF_BRICK_STOPPED;
+    brickinfo->start_triggered = _gf_false;
+    brickinfo->brick_proc = NULL;
+    if (del_brick)
+        glusterd_delete_brick(volinfo, brickinfo);
+out:
+    return ret;
+}
 
-        ret = get_checksum_for_path (cksum_path_final, &cksum);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "unable to get "
-                        "checksum for path: %s", cksum_path_final);
+/* Free LINE[0..N-1] and then the LINE buffer.  */
+static void
+free_lines(char **line, size_t n)
+{
+    size_t i;
+    for (i = 0; i < n; i++)
+        GF_FREE(line[i]);
+    GF_FREE(line);
+}
+
+static char **
+glusterd_readin_file(const char *filepath, int *line_count)
+{
+    int ret = -1;
+    int n = 8;
+    int counter = 0;
+    char buffer[PATH_MAX + 256] = "";
+    char **lines = NULL;
+    FILE *fp = NULL;
+    void *p;
+
+    fp = fopen(filepath, "r");
+    if (!fp)
+        goto out;
+
+    lines = GF_CALLOC(1, n * sizeof(*lines), gf_gld_mt_charptr);
+    if (!lines)
+        goto out;
+
+    for (counter = 0; fgets(buffer, sizeof(buffer), fp); counter++) {
+        if (counter == n - 1) {
+            n *= 2;
+            p = GF_REALLOC(lines, n * sizeof(char *));
+            if (!p) {
+                free_lines(lines, n / 2);
+                lines = NULL;
                 goto out;
-        }
-        if (!is_quota_conf) {
-                snprintf (buf, sizeof (buf), "%s=%u\n", "info", cksum);
-                ret = write (fd, buf, strlen (buf));
-                if (ret <= 0) {
-                        ret = -1;
-                        goto out;
-                }
+            }
+            lines = p;
         }
 
-        ret = get_checksum_for_file (fd, &cksum);
-        if (ret)
-                goto out;
+        lines[counter] = gf_strdup(buffer);
+    }
+
+    lines[counter] = NULL;
+    /* Reduce allocation to minimal size.  */
+    p = GF_REALLOC(lines, (counter + 1) * sizeof(char *));
+    if (!p) {
+        /* coverity[TAINTED_SCALAR] */
+        free_lines(lines, counter);
+        lines = NULL;
+        goto out;
+    }
+    lines = p;
 
-        *cs = cksum;
+    *line_count = counter;
+    ret = 0;
 
 out:
-        if (fd > 0)
-               close (fd);
-        if (unlink_sortfile)
-               unlink (sort_filepath);
-        gf_log (this->name, GF_LOG_DEBUG, "Returning with %d", ret);
+    if (ret)
+        gf_msg(THIS->name, GF_LOG_ERROR, errno, GD_MSG_READIN_FILE_FAILED, "%s",
+               strerror(errno));
+    if (fp)
+        fclose(fp);
 
-        return ret;
+    return lines;
 }
 
-int glusterd_compute_cksum (glusterd_volinfo_t *volinfo,
-                            gf_boolean_t is_quota_conf)
+int
+glusterd_compare_lines(const void *a, const void *b)
 {
-        int               ret                  = -1;
-        uint32_t          cs                   = 0;
-        char              cksum_path[PATH_MAX] = {0,};
-        char              path[PATH_MAX]       = {0,};
-        char              filepath[PATH_MAX]   = {0,};
-        glusterd_conf_t  *conf                 = NULL;
-        xlator_t         *this                 = NULL;
+    return strcmp(*(char *const *)a, *(char *const *)b);
+}
 
-        this = THIS;
-        GF_ASSERT (this);
-        conf = this->private;
-        GF_ASSERT (conf);
+static int
+glusterd_sort_and_redirect(const char *src_filepath, int dest_fd)
+{
+    int ret = -1;
+    int line_count = 0;
+    int counter = 0;
+    char **lines = NULL;
+
+    if (!src_filepath || dest_fd < 0)
+        goto out;
 
-        GLUSTERD_GET_VOLUME_DIR (path, volinfo, conf);
+    lines = glusterd_readin_file(src_filepath, &line_count);
+    if (!lines)
+        goto out;
+
+    qsort(lines, line_count, sizeof(*lines), glusterd_compare_lines);
+
+    for (counter = 0; lines[counter]; counter++) {
+        ret = sys_write(dest_fd, lines[counter], strlen(lines[counter]));
+        if (ret < 0)
+            goto out;
 
-        if (is_quota_conf) {
-                snprintf (cksum_path, sizeof (cksum_path), "%s/%s", path,
-                          GLUSTERD_VOL_QUOTA_CKSUM_FILE);
-                snprintf (filepath, sizeof (filepath), "%s/%s", path,
-                          GLUSTERD_VOLUME_QUOTA_CONFIG);
+        GF_FREE(lines[counter]);
+    }
+
+    ret = 0;
+out:
+    GF_FREE(lines);
+
+    return ret;
+}
+
+static int
+glusterd_volume_compute_cksum(glusterd_volinfo_t *volinfo, char *cksum_path,
+                              char *filepath, gf_boolean_t is_quota_conf,
+                              uint32_t *cs)
+{
+    int32_t ret = -1;
+    uint32_t cksum = 0;
+    int fd = -1;
+    int sort_fd = 0;
+    char sort_filepath[PATH_MAX] = "";
+    char buf[32];
+    gf_boolean_t unlink_sortfile = _gf_false;
+    glusterd_conf_t *priv = THIS->private;
+    xlator_t *this = THIS;
+    mode_t orig_umask = 0;
+
+    GF_ASSERT(volinfo);
+    GF_ASSERT(priv);
+
+    fd = open(cksum_path, O_RDWR | O_APPEND | O_CREAT | O_TRUNC, 0600);
+    if (-1 == fd) {
+        gf_msg(this->name, GF_LOG_ERROR, errno, GD_MSG_FILE_OP_FAILED,
+               "Unable to open %s,"
+               " errno: %d",
+               cksum_path, errno);
+        ret = -1;
+        goto out;
+    }
+
+    if (!is_quota_conf) {
+        snprintf(sort_filepath, sizeof(sort_filepath), "/tmp/%s.XXXXXX",
+                 volinfo->volname);
+
+        orig_umask = umask(S_IRWXG | S_IRWXO);
+        sort_fd = mkstemp(sort_filepath);
+        umask(orig_umask);
+        if (-1 == sort_fd) {
+            gf_msg(this->name, GF_LOG_ERROR, errno, GD_MSG_FILE_OP_FAILED,
+                   "Could not generate "
+                   "temp file, reason: %s for volume: %s",
+                   strerror(errno), volinfo->volname);
+            goto out;
         } else {
-                snprintf (cksum_path, sizeof (cksum_path), "%s/%s", path,
-                          GLUSTERD_CKSUM_FILE);
-                snprintf (filepath, sizeof (filepath), "%s/%s", path,
-                          GLUSTERD_VOLUME_INFO_FILE);
+            unlink_sortfile = _gf_true;
         }
 
-        ret = glusterd_volume_compute_cksum (volinfo, cksum_path, filepath,
-                                             is_quota_conf, &cs);
+        /* sort the info file, result in sort_filepath */
+
+        ret = glusterd_sort_and_redirect(filepath, sort_fd);
         if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "Failed to compute checksum "
-                        "for volume %s", volinfo->volname);
-                goto out;
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_FILE_OP_FAILED,
+                   "sorting info file "
+                   "failed");
+            goto out;
         }
 
-        if (is_quota_conf)
-                volinfo->quota_conf_cksum = cs;
-        else
-                volinfo->cksum = cs;
+        ret = sys_close(sort_fd);
+        if (ret)
+            goto out;
+
+        ret = get_checksum_for_path(sort_filepath, &cksum, priv->op_version);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_CKSUM_GET_FAIL,
+                   "unable to get "
+                   "checksum for path: %s",
+                   sort_filepath);
+            goto out;
+        }
+
+        ret = snprintf(buf, sizeof(buf), "info=%u\n", cksum);
+        ret = sys_write(fd, buf, ret);
+        if (ret <= 0) {
+            ret = -1;
+            goto out;
+        }
+    } else if (priv->op_version < GD_OP_VERSION_7_0) {
+        ret = get_checksum_for_path(filepath, &cksum, priv->op_version);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_CKSUM_GET_FAIL,
+                   "unable to get "
+                   "checksum for path: %s",
+                   filepath);
+            goto out;
+        }
+    }
+
+    ret = get_checksum_for_file(fd, &cksum, priv->op_version);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_CKSUM_GET_FAIL,
+               "unable to get checksum for path: %s", filepath);
+        goto out;
+    }
+
+    *cs = cksum;
 
-        ret = 0;
 out:
-        return ret;
+    if (fd != -1)
+        sys_close(fd);
+    if (unlink_sortfile)
+        sys_unlink(sort_filepath);
+    gf_msg_debug(this->name, 0, "Returning with %d", ret);
+
+    return ret;
 }
 
 int
-_add_dict_to_prdict (dict_t *this, char *key, data_t *value, void *data)
-{
-        glusterd_dict_ctx_t     *ctx = NULL;
-        char                    optkey[512] = {0,};
-        int                     ret = -1;
-
-        ctx = data;
-        snprintf (optkey, sizeof (optkey), "%s.%s%d", ctx->prefix,
-                  ctx->key_name, ctx->opt_count);
-        ret = dict_set_str (ctx->dict, optkey, key);
-        if (ret)
-                gf_log ("", GF_LOG_ERROR, "option add for %s%d %s",
-                        ctx->key_name, ctx->opt_count, key);
-        snprintf (optkey, sizeof (optkey), "%s.%s%d", ctx->prefix,
-                  ctx->val_name, ctx->opt_count);
-        ret = dict_set_str (ctx->dict, optkey, value->data);
-        if (ret)
-                gf_log ("", GF_LOG_ERROR, "option add for %s%d %s",
-                        ctx->val_name, ctx->opt_count, value->data);
-        ctx->opt_count++;
-
-        return ret;
+glusterd_compute_cksum(glusterd_volinfo_t *volinfo, gf_boolean_t is_quota_conf)
+{
+    int ret = -1;
+    uint32_t cs = 0;
+    char cksum_path[PATH_MAX] = "";
+    char path[PATH_MAX] = "";
+    char filepath[PATH_MAX] = "";
+    glusterd_conf_t *conf = NULL;
+    xlator_t *this = NULL;
+    int32_t len1 = 0;
+    int32_t len2 = 0;
+
+    this = THIS;
+    GF_ASSERT(this);
+    conf = this->private;
+    GF_ASSERT(conf);
+
+    GLUSTERD_GET_VOLUME_DIR(path, volinfo, conf);
+
+    if (is_quota_conf) {
+        len1 = snprintf(cksum_path, sizeof(cksum_path), "%s/%s", path,
+                        GLUSTERD_VOL_QUOTA_CKSUM_FILE);
+        len2 = snprintf(filepath, sizeof(filepath), "%s/%s", path,
+                        GLUSTERD_VOLUME_QUOTA_CONFIG);
+    } else {
+        len1 = snprintf(cksum_path, sizeof(cksum_path), "%s/%s", path,
+                        GLUSTERD_CKSUM_FILE);
+        len2 = snprintf(filepath, sizeof(filepath), "%s/%s", path,
+                        GLUSTERD_VOLUME_INFO_FILE);
+    }
+    if ((len1 < 0) || (len2 < 0) || (len1 >= sizeof(cksum_path)) ||
+        (len2 >= sizeof(filepath))) {
+        goto out;
+    }
+
+    ret = glusterd_volume_compute_cksum(volinfo, cksum_path, filepath,
+                                        is_quota_conf, &cs);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_CKSUM_COMPUTE_FAIL,
+               "Failed to compute checksum "
+               "for volume %s",
+               volinfo->volname);
+        goto out;
+    }
+
+    if (is_quota_conf)
+        volinfo->quota_conf_cksum = cs;
+    else
+        volinfo->cksum = cs;
+
+    ret = 0;
+out:
+    return ret;
 }
 
-int32_t
-glusterd_add_bricks_hname_path_to_dict (dict_t *dict,
-                                        glusterd_volinfo_t *volinfo)
+static int
+_add_dict_to_prdict(dict_t *this, char *key, data_t *value, void *data)
 {
-        glusterd_brickinfo_t    *brickinfo = NULL;
-        int                     ret = 0;
-        char                    key[256] = {0};
-        int                     index = 0;
+    glusterd_dict_ctx_t *ctx = data;
+    char optkey[64]; /* optkey are usually quite small */
+    int ret = -1;
 
+    ret = snprintf(optkey, sizeof(optkey), "%s.%s%d", ctx->prefix,
+                   ctx->key_name, ctx->opt_count);
+    if (ret < 0 || ret >= sizeof(optkey))
+        return -1;
+    ret = dict_set_strn(ctx->dict, optkey, ret, key);
+    if (ret)
+        gf_msg("glusterd", GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+               "option add for %s%d %s", ctx->key_name, ctx->opt_count, key);
+    ret = snprintf(optkey, sizeof(optkey), "%s.%s%d", ctx->prefix,
+                   ctx->val_name, ctx->opt_count);
+    if (ret < 0 || ret >= sizeof(optkey))
+        return -1;
+    ret = dict_set_strn(ctx->dict, optkey, ret, value->data);
+    if (ret)
+        gf_msg("glusterd", GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+               "option add for %s%d %s", ctx->val_name, ctx->opt_count,
+               value->data);
+    ctx->opt_count++;
 
-        cds_list_for_each_entry (brickinfo, &volinfo->bricks, brick_list) {
-                snprintf (key, sizeof (key), "%d-hostname", index);
-                ret = dict_set_str (dict, key, brickinfo->hostname);
-                if (ret)
-                        goto out;
+    return ret;
+}
 
-                snprintf (key, sizeof (key), "%d-path", index);
-                ret = dict_set_str (dict, key, brickinfo->path);
-                if (ret)
-                        goto out;
+int32_t
+glusterd_add_bricks_hname_path_to_dict(dict_t *dict,
+                                       glusterd_volinfo_t *volinfo)
+{
+    glusterd_brickinfo_t *brickinfo = NULL;
+    int ret = 0;
+    char key[64] = "";
+    int index = 0;
+
+    cds_list_for_each_entry(brickinfo, &volinfo->bricks, brick_list)
+    {
+        ret = snprintf(key, sizeof(key), "%d-hostname", index);
+        ret = dict_set_strn(dict, key, ret, brickinfo->hostname);
+        if (ret) {
+            gf_smsg("glusterd", GF_LOG_ERROR, errno, GD_MSG_DICT_SET_FAILED,
+                    "Key=%s", key, NULL);
+            goto out;
+        }
 
-                index++;
+        ret = snprintf(key, sizeof(key), "%d-path", index);
+        ret = dict_set_strn(dict, key, ret, brickinfo->path);
+        if (ret) {
+            gf_smsg("glusterd", GF_LOG_ERROR, errno, GD_MSG_DICT_SET_FAILED,
+                    "Key=%s", key, NULL);
+            goto out;
         }
+
+        index++;
+    }
 out:
-        return ret;
+    return ret;
 }
 
 /* The prefix represents the type of volume to be added.
@@ -2069,860 +2958,1258 @@ out:
  * snap1, snap2, for snapshot volumes
  */
 int32_t
-glusterd_add_volume_to_dict (glusterd_volinfo_t *volinfo,
-                             dict_t  *dict, int32_t count,
-                             char *prefix)
-{
-        int32_t                 ret               = -1;
-        char                    pfx[512]          = {0,};
-        char                    key[512]          = {0,};
-        glusterd_brickinfo_t    *brickinfo        = NULL;
-        int32_t                 i                 = 1;
-        char                    *volume_id_str    = NULL;
-        char                    *src_brick        = NULL;
-        char                    *dst_brick        = NULL;
-        char                    *str              = NULL;
-        glusterd_dict_ctx_t     ctx               = {0};
-        char                    *rebalance_id_str = NULL;
-        char                    *rb_id_str        = NULL;
-        xlator_t                *this             = NULL;
-
-        this = THIS;
-        GF_ASSERT (this);
-        GF_ASSERT (dict);
-        GF_ASSERT (volinfo);
-        GF_ASSERT (prefix);
-
-        snprintf (key, sizeof (key), "%s%d.name", prefix, count);
-        ret = dict_set_str (dict, key, volinfo->volname);
-        if (ret)
-                goto out;
-
-        memset (key, 0, sizeof (key));
-        snprintf (key, sizeof (key), "%s%d.type", prefix, count);
-        ret = dict_set_int32 (dict, key, volinfo->type);
+glusterd_add_volume_to_dict(glusterd_volinfo_t *volinfo, dict_t *dict,
+                            int32_t count, char *prefix)
+{
+    int32_t ret = -1;
+    char pfx[32] = ""; /* prefix should be quite small */
+    char key[64] = "";
+    int keylen;
+    glusterd_brickinfo_t *brickinfo = NULL;
+    glusterd_brickinfo_t *ta_brickinfo = NULL;
+    int32_t i = 1;
+    char *volume_id_str = NULL;
+    char *str = NULL;
+    glusterd_dict_ctx_t ctx = {0};
+    char *rebalance_id_str = NULL;
+    char *rb_id_str = NULL;
+    xlator_t *this = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+    GF_ASSERT(dict);
+    GF_ASSERT(volinfo);
+    GF_ASSERT(prefix);
+
+    ret = snprintf(pfx, sizeof(pfx), "%s%d", prefix, count);
+    if (ret < 0 || ret >= sizeof(pfx)) {
+        ret = -1;
+        goto out;
+    }
+
+    keylen = snprintf(key, sizeof(key), "%s.name", pfx);
+    ret = dict_set_strn(dict, key, keylen, volinfo->volname);
+    if (ret)
+        goto out;
+
+    keylen = snprintf(key, sizeof(key), "%s.type", pfx);
+    ret = dict_set_int32n(dict, key, keylen, volinfo->type);
+    if (ret)
+        goto out;
+
+    keylen = snprintf(key, sizeof(key), "%s.brick_count", pfx);
+    ret = dict_set_int32n(dict, key, keylen, volinfo->brick_count);
+    if (ret)
+        goto out;
+
+    keylen = snprintf(key, sizeof(key), "%s.version", pfx);
+    ret = dict_set_int32n(dict, key, keylen, volinfo->version);
+    if (ret)
+        goto out;
+
+    keylen = snprintf(key, sizeof(key), "%s.status", pfx);
+    ret = dict_set_int32n(dict, key, keylen, volinfo->status);
+    if (ret)
+        goto out;
+
+    keylen = snprintf(key, sizeof(key), "%s.sub_count", pfx);
+    ret = dict_set_int32n(dict, key, keylen, volinfo->sub_count);
+    if (ret)
+        goto out;
+
+    keylen = snprintf(key, sizeof(key), "%s.subvol_count", pfx);
+    ret = dict_set_int32n(dict, key, keylen, volinfo->subvol_count);
+    if (ret)
+        goto out;
+
+    keylen = snprintf(key, sizeof(key), "%s.stripe_count", pfx);
+    ret = dict_set_int32n(dict, key, keylen, volinfo->stripe_count);
+    if (ret)
+        goto out;
+
+    keylen = snprintf(key, sizeof(key), "%s.replica_count", pfx);
+    ret = dict_set_int32n(dict, key, keylen, volinfo->replica_count);
+    if (ret)
+        goto out;
+
+    keylen = snprintf(key, sizeof(key), "%s.arbiter_count", pfx);
+    ret = dict_set_int32n(dict, key, keylen, volinfo->arbiter_count);
+    if (ret)
+        goto out;
+
+    keylen = snprintf(key, sizeof(key), "%s.thin_arbiter_count", pfx);
+    ret = dict_set_int32n(dict, key, keylen, volinfo->thin_arbiter_count);
+    if (ret)
+        goto out;
+
+    keylen = snprintf(key, sizeof(key), "%s.disperse_count", pfx);
+    ret = dict_set_int32n(dict, key, keylen, volinfo->disperse_count);
+    if (ret)
+        goto out;
+
+    keylen = snprintf(key, sizeof(key), "%s.redundancy_count", pfx);
+    ret = dict_set_int32n(dict, key, keylen, volinfo->redundancy_count);
+    if (ret)
+        goto out;
+
+    keylen = snprintf(key, sizeof(key), "%s.dist_count", pfx);
+    ret = dict_set_int32n(dict, key, keylen, volinfo->dist_leaf_count);
+    if (ret)
+        goto out;
+
+    snprintf(key, sizeof(key), "%s.ckusm", pfx);
+    ret = dict_set_int64(dict, key, volinfo->cksum);
+    if (ret)
+        goto out;
+
+    snprintf(key, sizeof(key), "%s.transport_type", pfx);
+    ret = dict_set_uint32(dict, key, volinfo->transport_type);
+    if (ret)
+        goto out;
+
+    snprintf(key, sizeof(key), "%s.stage_deleted", pfx);
+    ret = dict_set_uint32(dict, key, (uint32_t)volinfo->stage_deleted);
+    if (ret)
+        goto out;
+
+    ret = gd_add_vol_snap_details_to_dict(dict, pfx, volinfo);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_SET_FAILED,
+                "vol snap details", NULL);
+        goto out;
+    }
+
+    volume_id_str = gf_strdup(uuid_utoa(volinfo->volume_id));
+    if (!volume_id_str) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_STRDUP_FAILED,
+                "volume id=%s", volinfo->volume_id, NULL);
+        ret = -1;
+        goto out;
+    }
+    keylen = snprintf(key, sizeof(key), "%s.volume_id", pfx);
+    ret = dict_set_dynstrn(dict, key, keylen, volume_id_str);
+    if (ret)
+        goto out;
+    volume_id_str = NULL;
+
+    keylen = snprintf(key, sizeof(key), "%s.username", pfx);
+    str = glusterd_auth_get_username(volinfo);
+    if (str) {
+        ret = dict_set_dynstrn(dict, key, keylen, gf_strdup(str));
         if (ret)
-                goto out;
+            goto out;
+    }
 
-        memset (key, 0, sizeof (key));
-        snprintf (key, sizeof (key), "%s%d.brick_count", prefix, count);
-        ret = dict_set_int32 (dict, key, volinfo->brick_count);
+    keylen = snprintf(key, sizeof(key), "%s.password", pfx);
+    str = glusterd_auth_get_password(volinfo);
+    if (str) {
+        ret = dict_set_dynstrn(dict, key, keylen, gf_strdup(str));
         if (ret)
-                goto out;
+            goto out;
+    }
 
-        memset (key, 0, sizeof (key));
-        snprintf (key, sizeof (key), "%s%d.version", prefix, count);
-        ret = dict_set_int32 (dict, key, volinfo->version);
-        if (ret)
-                goto out;
+    keylen = snprintf(key, sizeof(key), "%s.rebalance", pfx);
+    ret = dict_set_int32n(dict, key, keylen, volinfo->rebal.defrag_cmd);
+    if (ret)
+        goto out;
 
-        memset (key, 0, sizeof (key));
-        snprintf (key, sizeof (key), "%s%d.status", prefix, count);
-        ret = dict_set_int32 (dict, key, volinfo->status);
-        if (ret)
-                goto out;
+    rebalance_id_str = gf_strdup(uuid_utoa(volinfo->rebal.rebalance_id));
+    if (!rebalance_id_str) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_STRDUP_FAILED,
+                "rebalance_id=%s", volinfo->rebal.rebalance_id, NULL);
+        ret = -1;
+        goto out;
+    }
+    keylen = snprintf(key, sizeof(key), "%s.rebalance-id", pfx);
+    ret = dict_set_dynstrn(dict, key, keylen, rebalance_id_str);
+    if (ret)
+        goto out;
+    rebalance_id_str = NULL;
+
+    snprintf(key, sizeof(key), "%s.rebalance-op", pfx);
+    ret = dict_set_uint32(dict, key, volinfo->rebal.op);
+    if (ret)
+        goto out;
+
+    if (volinfo->rebal.dict) {
+        ctx.dict = dict;
+        ctx.prefix = pfx;
+        ctx.opt_count = 1;
+        ctx.key_name = "rebal-dict-key";
+        ctx.val_name = "rebal-dict-value";
 
-        memset (key, 0, sizeof (key));
-        snprintf (key, sizeof (key), "%s%d.sub_count", prefix, count);
-        ret = dict_set_int32 (dict, key, volinfo->sub_count);
+        dict_foreach(volinfo->rebal.dict, _add_dict_to_prdict, &ctx);
+        ctx.opt_count--;
+        keylen = snprintf(key, sizeof(key), "volume%d.rebal-dict-count", count);
+        ret = dict_set_int32n(dict, key, keylen, ctx.opt_count);
         if (ret)
-                goto out;
-
-        memset (key, 0, sizeof (key));
-        snprintf (key, sizeof (key), "%s%d.stripe_count", prefix, count);
-        ret = dict_set_int32 (dict, key, volinfo->stripe_count);
+            goto out;
+    }
+
+    ctx.dict = dict;
+    ctx.prefix = pfx;
+    ctx.opt_count = 1;
+    ctx.key_name = "key";
+    ctx.val_name = "value";
+    GF_ASSERT(volinfo->dict);
+
+    dict_foreach(volinfo->dict, _add_dict_to_prdict, &ctx);
+    ctx.opt_count--;
+    keylen = snprintf(key, sizeof(key), "%s.opt-count", pfx);
+    ret = dict_set_int32n(dict, key, keylen, ctx.opt_count);
+    if (ret)
+        goto out;
+
+    ctx.dict = dict;
+    ctx.prefix = pfx;
+    ctx.opt_count = 1;
+    ctx.key_name = "slave-num";
+    ctx.val_name = "slave-val";
+    GF_ASSERT(volinfo->gsync_slaves);
+
+    dict_foreach(volinfo->gsync_slaves, _add_dict_to_prdict, &ctx);
+    ctx.opt_count--;
+
+    keylen = snprintf(key, sizeof(key), "%s.gsync-count", pfx);
+    ret = dict_set_int32n(dict, key, keylen, ctx.opt_count);
+    if (ret)
+        goto out;
+
+    cds_list_for_each_entry(brickinfo, &volinfo->bricks, brick_list)
+    {
+        keylen = snprintf(key, sizeof(key), "%s.brick%d.hostname", pfx, i);
+        ret = dict_set_strn(dict, key, keylen, brickinfo->hostname);
         if (ret)
-                goto out;
+            goto out;
 
-        memset (key, 0, sizeof (key));
-        snprintf (key, sizeof (key), "%s%d.replica_count", prefix, count);
-        ret = dict_set_int32 (dict, key, volinfo->replica_count);
+        keylen = snprintf(key, sizeof(key), "%s.brick%d.path", pfx, i);
+        ret = dict_set_strn(dict, key, keylen, brickinfo->path);
         if (ret)
-                goto out;
+            goto out;
 
-        memset (key, 0, sizeof (key));
-        snprintf (key, sizeof (key), "%s%d.disperse_count", prefix, count);
-        ret = dict_set_int32 (dict, key, volinfo->disperse_count);
+        keylen = snprintf(key, sizeof(key), "%s.brick%d.decommissioned", pfx,
+                          i);
+        ret = dict_set_int32n(dict, key, keylen, brickinfo->decommissioned);
         if (ret)
-                goto out;
+            goto out;
 
-        memset (key, 0, sizeof (key));
-        snprintf (key, sizeof (key), "%s%d.redundancy_count", prefix, count);
-        ret = dict_set_int32 (dict, key, volinfo->redundancy_count);
+        keylen = snprintf(key, sizeof(key), "%s.brick%d.brick_id", pfx, i);
+        ret = dict_set_strn(dict, key, keylen, brickinfo->brick_id);
         if (ret)
-                goto out;
+            goto out;
 
-        memset (key, 0, sizeof (key));
-        snprintf (key, sizeof (key), "%s%d.dist_count", prefix, count);
-        ret = dict_set_int32 (dict, key, volinfo->dist_leaf_count);
+        snprintf(key, sizeof(key), "%s.brick%d.uuid", pfx, i);
+        ret = dict_set_dynstr_with_alloc(dict, key, uuid_utoa(brickinfo->uuid));
         if (ret)
-                goto out;
+            goto out;
 
-        memset (key, 0, sizeof (key));
-        snprintf (key, sizeof (key), "%s%d.ckusm", prefix, count);
-        ret = dict_set_int64 (dict, key, volinfo->cksum);
+        snprintf(key, sizeof(key), "%s.brick%d", pfx, i);
+        ret = gd_add_brick_snap_details_to_dict(dict, key, brickinfo);
         if (ret)
-                goto out;
+            goto out;
 
-        memset (key, 0, sizeof (key));
-        snprintf (key, sizeof (key), "%s%d.transport_type", prefix, count);
-        ret = dict_set_uint32 (dict, key, volinfo->transport_type);
-        if (ret)
-                goto out;
+        i++;
+    }
 
-        memset (key, 0, sizeof (key));
-        snprintf (key, sizeof (key), "%s%d", prefix, count);
-        ret = gd_add_vol_snap_details_to_dict (dict, key, volinfo);
-        if (ret)
+    i = 1;
+    if (volinfo->thin_arbiter_count == 1) {
+        cds_list_for_each_entry(ta_brickinfo, &volinfo->ta_bricks, brick_list)
+        {
+            keylen = snprintf(key, sizeof(key), "%s.ta-brick%d.hostname", pfx,
+                              i);
+            ret = dict_set_strn(dict, key, keylen, ta_brickinfo->hostname);
+            if (ret)
                 goto out;
 
-        volume_id_str = gf_strdup (uuid_utoa (volinfo->volume_id));
-        if (!volume_id_str) {
-                ret = -1;
+            keylen = snprintf(key, sizeof(key), "%s.ta-brick%d.path", pfx, i);
+            ret = dict_set_strn(dict, key, keylen, ta_brickinfo->path);
+            if (ret)
                 goto out;
-        }
-        memset (key, 0, sizeof (key));
-        snprintf (key, sizeof (key), "%s%d.volume_id", prefix, count);
-        ret = dict_set_dynstr (dict, key, volume_id_str);
-        if (ret)
-                goto out;
-        volume_id_str = NULL;
-
-        memset (key, 0, sizeof (key));
-        snprintf (key, sizeof (key), "%s%d.username", prefix, count);
-        str = glusterd_auth_get_username (volinfo);
-        if (str) {
-                ret = dict_set_dynstr (dict, key, gf_strdup (str));
-                if (ret)
-                        goto out;
-        }
-
-        memset (key, 0, sizeof (key));
-        snprintf (key, sizeof (key), "%s%d.password", prefix, count);
-        str = glusterd_auth_get_password (volinfo);
-        if (str) {
-                ret = dict_set_dynstr (dict, key, gf_strdup (str));
-                if (ret)
-                        goto out;
-        }
 
-        memset (key, 0, sizeof (key));
-        snprintf (key, 256, "%s%d.rebalance", prefix, count);
-        ret = dict_set_int32 (dict, key, volinfo->rebal.defrag_cmd);
-        if (ret)
+            keylen = snprintf(key, sizeof(key), "%s.ta-brick%d.decommissioned",
+                              pfx, i);
+            ret = dict_set_int32n(dict, key, keylen,
+                                  ta_brickinfo->decommissioned);
+            if (ret)
                 goto out;
 
-        rebalance_id_str = gf_strdup (uuid_utoa
-                        (volinfo->rebal.rebalance_id));
-        if (!rebalance_id_str) {
-                ret = -1;
+            keylen = snprintf(key, sizeof(key), "%s.ta-brick%d.brick_id", pfx,
+                              i);
+            ret = dict_set_strn(dict, key, keylen, ta_brickinfo->brick_id);
+            if (ret)
                 goto out;
-        }
-        memset (key, 0, sizeof (key));
-        snprintf (key, 256, "%s%d.rebalance-id", prefix, count);
-        ret = dict_set_dynstr (dict, key, rebalance_id_str);
-        if (ret)
-                goto out;
-        rebalance_id_str = NULL;
 
-        memset (key, 0, sizeof (key));
-        snprintf (key, sizeof (key), "%s%d.rebalance-op", prefix, count);
-        ret = dict_set_uint32 (dict, key, volinfo->rebal.op);
-        if (ret)
+            snprintf(key, sizeof(key), "%s.ta-brick%d.uuid", pfx, i);
+            ret = dict_set_dynstr_with_alloc(dict, key,
+                                             uuid_utoa(ta_brickinfo->uuid));
+            if (ret)
                 goto out;
 
-        if (volinfo->rebal.dict) {
-                snprintf (pfx, sizeof (pfx), "%s%d", prefix, count);
-                ctx.dict = dict;
-                ctx.prefix = pfx;
-                ctx.opt_count = 1;
-                ctx.key_name = "rebal-dict-key";
-                ctx.val_name = "rebal-dict-value";
-
-                dict_foreach (volinfo->rebal.dict, _add_dict_to_prdict, &ctx);
-                ctx.opt_count--;
-                memset (key, 0, sizeof (key));
-                snprintf (key, sizeof (key), "volume%d.rebal-dict-count", count);
-                ret = dict_set_int32 (dict, key, ctx.opt_count);
-                if (ret)
-                        goto out;
+            i++;
         }
+    }
 
-        memset (key, 0, sizeof (key));
-        snprintf (key, 256, "%s%d."GLUSTERD_STORE_KEY_RB_STATUS, prefix, count);
-        ret = dict_set_int32 (dict, key, volinfo->rep_brick.rb_status);
-        if (ret)
-                goto out;
+    /* Add volume op-versions to dict. This prevents volume inconsistencies
+     * in the cluster
+     */
+    keylen = snprintf(key, sizeof(key), "%s.op-version", pfx);
+    ret = dict_set_int32n(dict, key, keylen, volinfo->op_version);
+    if (ret)
+        goto out;
+    keylen = snprintf(key, sizeof(key), "%s.client-op-version", pfx);
+    ret = dict_set_int32n(dict, key, keylen, volinfo->client_op_version);
+    if (ret)
+        goto out;
 
-        if (volinfo->rep_brick.rb_status > GF_RB_STATUS_NONE) {
+    keylen = snprintf(key, sizeof(key), "%s.quota-xattr-version", pfx);
+    ret = dict_set_int32n(dict, key, keylen, volinfo->quota_xattr_version);
+out:
+    GF_FREE(volume_id_str);
+    GF_FREE(rebalance_id_str);
+    GF_FREE(rb_id_str);
 
-                memset (key, 0, sizeof (key));
-                snprintf (key, 256, "%s%d."GLUSTERD_STORE_KEY_RB_SRC_BRICK,
-                          prefix, count);
-                gf_asprintf (&src_brick, "%s:%s",
-                             volinfo->rep_brick.src_brick->hostname,
-                             volinfo->rep_brick.src_brick->path);
-                ret = dict_set_dynstr (dict, key, src_brick);
-                if (ret)
-                        goto out;
+    if (key[0] != '\0' && ret != 0)
+        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_SET_FAILED,
+                "Key=%s", key, NULL);
+    gf_msg_debug(this->name, 0, "Returning with %d", ret);
+    return ret;
+}
 
-                memset (key, 0, sizeof (key));
-                snprintf (key, 256, "%s%d."GLUSTERD_STORE_KEY_RB_DST_BRICK,
-                          prefix, count);
-                gf_asprintf (&dst_brick, "%s:%s",
-                             volinfo->rep_brick.dst_brick->hostname,
-                             volinfo->rep_brick.dst_brick->path);
-                ret = dict_set_dynstr (dict, key, dst_brick);
-                if (ret)
-                        goto out;
+/* The prefix represents the type of volume to be added.
+ * It will be "volume" for normal volumes, and snap# like
+ * snap1, snap2, for snapshot volumes
+ */
+int
+glusterd_vol_add_quota_conf_to_dict(glusterd_volinfo_t *volinfo, dict_t *load,
+                                    int vol_idx, char *prefix)
+{
+    int fd = -1;
+    unsigned char buf[16] = "";
+    char key[64];
+    char key_prefix[32];
+    int gfid_idx = 0;
+    int ret = -1;
+    xlator_t *this = NULL;
+    char type = 0;
+    float version = 0.0f;
+
+    this = THIS;
+    GF_ASSERT(this);
+    GF_ASSERT(prefix);
+
+    ret = glusterd_store_create_quota_conf_sh_on_absence(volinfo);
+    if (ret)
+        goto out;
+
+    fd = open(volinfo->quota_conf_shandle->path, O_RDONLY);
+    if (fd == -1) {
+        ret = -1;
+        goto out;
+    }
 
-                rb_id_str = gf_strdup (uuid_utoa (volinfo->rep_brick.rb_id));
-                if (!rb_id_str) {
-                        ret = -1;
-                        goto out;
-                }
+    ret = quota_conf_read_version(fd, &version);
+    if (ret)
+        goto out;
 
-                memset (key, 0, sizeof (key));
-                snprintf (key, sizeof (key), "%s%d.rb_id", prefix, count);
-                ret = dict_set_dynstr (dict, key, rb_id_str);
-                if (ret)
-                        goto out;
-                rb_id_str = NULL;
+    ret = snprintf(key_prefix, sizeof(key_prefix), "%s%d", prefix, vol_idx);
+    if (ret < 0 || ret >= sizeof(key_prefix)) {
+        ret = -1;
+        goto out;
+    }
+    for (gfid_idx = 0;; gfid_idx++) {
+        ret = quota_conf_read_gfid(fd, buf, &type, version);
+        if (ret == 0) {
+            break;
+        } else if (ret < 0) {
+            gf_msg(this->name, GF_LOG_CRITICAL, 0, GD_MSG_QUOTA_CONF_CORRUPT,
+                   "Quota "
+                   "configuration store may be corrupt.");
+            goto out;
         }
 
-        snprintf (pfx, sizeof (pfx), "%s%d", prefix, count);
-        ctx.dict = dict;
-        ctx.prefix = pfx;
-        ctx.opt_count = 1;
-        ctx.key_name = "key";
-        ctx.val_name = "value";
-        GF_ASSERT (volinfo->dict);
-
-        dict_foreach (volinfo->dict, _add_dict_to_prdict, &ctx);
-        ctx.opt_count--;
-        memset (key, 0, sizeof (key));
-        snprintf (key, sizeof (key), "%s%d.opt-count", prefix, count);
-        ret = dict_set_int32 (dict, key, ctx.opt_count);
-        if (ret)
-                goto out;
-
-        ctx.dict = dict;
-        ctx.prefix = pfx;
-        ctx.opt_count = 1;
-        ctx.key_name = "slave-num";
-        ctx.val_name = "slave-val";
-        GF_ASSERT (volinfo->gsync_slaves);
-
-        dict_foreach (volinfo->gsync_slaves, _add_dict_to_prdict, &ctx);
-        ctx.opt_count--;
-
-        memset (key, 0, sizeof (key));
-        snprintf (key, sizeof (key), "%s%d.gsync-count", prefix, count);
-        ret = dict_set_int32 (dict, key, ctx.opt_count);
-        if (ret)
-                goto out;
-
-        cds_list_for_each_entry (brickinfo, &volinfo->bricks, brick_list) {
-                memset (key, 0, sizeof (key));
-                snprintf (key, sizeof (key), "%s%d.brick%d.hostname",
-                          prefix, count, i);
-                ret = dict_set_str (dict, key, brickinfo->hostname);
-                if (ret)
-                        goto out;
-
-                memset (key, 0, sizeof (key));
-                snprintf (key, sizeof (key), "%s%d.brick%d.path",
-                          prefix, count, i);
-                ret = dict_set_str (dict, key, brickinfo->path);
-                if (ret)
-                        goto out;
-
-                memset (key, 0, sizeof (key));
-                snprintf (key, sizeof (key), "%s%d.brick%d.decommissioned",
-                          prefix, count, i);
-                ret = dict_set_int32 (dict, key, brickinfo->decommissioned);
-                if (ret)
-                        goto out;
-
-                memset (key, 0, sizeof (key));
-                snprintf (key, sizeof (key), "%s%d.brick%d.brick_id",
-                          prefix, count, i);
-                ret = dict_set_str (dict, key, brickinfo->brick_id);
-                if (ret)
-                        goto out;
-
-                memset (key, 0, sizeof (key));
-                snprintf (key, sizeof (key), "%s%d.brick%d", prefix, count, i);
-                ret = gd_add_brick_snap_details_to_dict (dict, key, brickinfo);
-                if (ret)
-                        goto out;
+        snprintf(key, sizeof(key) - 1, "%s.gfid%d", key_prefix, gfid_idx);
+        ret = dict_set_dynstr_with_alloc(load, key, uuid_utoa(buf));
+        if (ret) {
+            gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_SET_FAILED,
+                    "Key=%s", key, NULL);
+            goto out;
+        }
 
-                i++;
+        snprintf(key, sizeof(key) - 1, "%s.gfid-type%d", key_prefix, gfid_idx);
+        ret = dict_set_int8(load, key, type);
+        if (ret) {
+            gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_SET_FAILED,
+                    "Key=%s", key, NULL);
+            goto out;
         }
+    }
+
+    ret = snprintf(key, sizeof(key), "%s.gfid-count", key_prefix);
+    ret = dict_set_int32n(load, key, ret, gfid_idx);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_SET_FAILED,
+                "Key=%s", key, NULL);
+        goto out;
+    }
+
+    snprintf(key, sizeof(key), "%s.quota-cksum", key_prefix);
+    ret = dict_set_uint32(load, key, volinfo->quota_conf_cksum);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_SET_FAILED,
+                "Key=%s", key, NULL);
+        goto out;
+    }
+
+    snprintf(key, sizeof(key), "%s.quota-version", key_prefix);
+    ret = dict_set_uint32(load, key, volinfo->quota_conf_version);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_SET_FAILED,
+                "Key=%s", key, NULL);
+        goto out;
+    }
+
+    ret = 0;
+out:
+    if (fd != -1)
+        sys_close(fd);
+    return ret;
+}
+
+void *
+glusterd_add_bulk_volumes_create_thread(void *data)
+{
+    int32_t ret = -1;
+    glusterd_conf_t *priv = NULL;
+    glusterd_volinfo_t *volinfo = NULL;
+    int32_t count = 0;
+    xlator_t *this = NULL;
+    glusterd_add_dict_args_t *arg = NULL;
+    dict_t *dict = NULL;
+    int start = 0;
+    int end = 0;
+
+    GF_ASSERT(data);
+
+    arg = data;
+    dict = arg->voldict;
+    start = arg->start;
+    end = arg->end;
+    this = arg->this;
+    THIS = arg->this;
+    priv = this->private;
+    GF_ASSERT(priv);
+
+    cds_list_for_each_entry(volinfo, &priv->volumes, vol_list)
+    {
+        count++;
+
+        /* Skip volumes if index count is less than start
+           index to handle volume for specific thread
+        */
+        if (count < start)
+            continue;
 
-        /* Add volume op-versions to dict. This prevents volume inconsistencies
-         * in the cluster
-         */
-        memset (key, 0, sizeof (key));
-        snprintf (key, sizeof (key), "%s%d.op-version", prefix, count);
-        ret = dict_set_int32 (dict, key, volinfo->op_version);
+        /* No need to process volume if index count is greater
+           than end index
+        */
+        if (count > end)
+            break;
+
+        ret = glusterd_add_volume_to_dict(volinfo, dict, count, "volume");
         if (ret)
-                goto out;
-        memset (key, 0, sizeof (key));
-        snprintf (key, sizeof (key), "%s%d.client-op-version", prefix, count);
-        ret = dict_set_int32 (dict, key, volinfo->client_op_version);
+            goto out;
+        if (!dict_get_sizen(volinfo->dict, VKEY_FEATURES_QUOTA))
+            continue;
+        ret = glusterd_vol_add_quota_conf_to_dict(volinfo, dict, count,
+                                                  "volume");
         if (ret)
-                goto out;
-
-        /*Add volume Capability (BD Xlator) to dict*/
-        memset (key, 0 ,sizeof (key));
-        snprintf (key, sizeof (key), "%s%d.caps", prefix, count);
-        ret = dict_set_int32 (dict, key, volinfo->caps);
+            goto out;
+    }
 
 out:
-        GF_FREE (volume_id_str);
-        GF_FREE (rebalance_id_str);
-        GF_FREE (rb_id_str);
-
-        gf_log (this->name, GF_LOG_DEBUG, "Returning with %d", ret);
-        return ret;
+    GF_ATOMIC_DEC(priv->thread_count);
+    free(arg);
+    return NULL;
 }
 
-/* The prefix represents the type of volume to be added.
- * It will be "volume" for normal volumes, and snap# like
- * snap1, snap2, for snapshot volumes
- */
 int
-glusterd_vol_add_quota_conf_to_dict (glusterd_volinfo_t *volinfo, dict_t* load,
-                                     int vol_idx, char *prefix)
+glusterd_dict_searialize(dict_t *dict_arr[], int count, int totcount, char *buf)
 {
-        int   fd                    = -1;
-        char  *gfid_str             = NULL;
-        unsigned char  buf[16]      = {0};
-        char  key[PATH_MAX]         = {0};
-        int   gfid_idx              = 0;
-        int   ret                   = -1;
-        xlator_t *this              = NULL;
-
-        this = THIS;
-        GF_ASSERT (this);
-        GF_ASSERT (prefix);
-
-        ret = glusterd_store_create_quota_conf_sh_on_absence (volinfo);
-        if (ret)
-                goto out;
-
-        fd = open (volinfo->quota_conf_shandle->path, O_RDONLY);
-        if (fd == -1) {
-                ret = -1;
-                goto out;
-        }
+    int i = 0;
+    int32_t keylen = 0;
+    int64_t netword = 0;
+    data_pair_t *pair = NULL;
+    int dict_count = 0;
+    int ret = 0;
 
-        ret = glusterd_store_quota_conf_skip_header (this, fd);
-        if (ret)
-                goto out;
+    netword = hton32(totcount);
+    memcpy(buf, &netword, sizeof(netword));
+    buf += DICT_HDR_LEN;
 
-        for (gfid_idx=0; ; gfid_idx++) {
-
-                ret = read (fd, (void*)&buf, 16) ;
-                if (ret <= 0) {
-                        //Finished reading all entries in the conf file
-                        break;
-                }
-                if (ret != 16) {
-                        //This should never happen. We must have a multiple of
-                        //entry_sz bytes in our configuration file.
-                        gf_log (this->name, GF_LOG_CRITICAL, "Quota "
-                                "configuration store may be corrupt.");
-                        goto out;
+    for (i = 0; i < count; i++) {
+        if (dict_arr[i]) {
+            dict_count = dict_arr[i]->count;
+            pair = dict_arr[i]->members_list;
+            while (dict_count) {
+                if (!pair) {
+                    gf_msg("glusterd", GF_LOG_ERROR, 0,
+                           LG_MSG_PAIRS_LESS_THAN_COUNT,
+                           "less than count data pairs found!");
+                    ret = -1;
+                    goto out;
                 }
 
-                gfid_str = gf_strdup (uuid_utoa (buf));
-                if (!gfid_str) {
-                        ret = -1;
-                        goto out;
+                if (!pair->key) {
+                    gf_msg("glusterd", GF_LOG_ERROR, 0, LG_MSG_NULL_PTR,
+                           "pair->key is null!");
+                    ret = -1;
+                    goto out;
                 }
 
-                snprintf (key, sizeof(key)-1, "%s%d.gfid%d", prefix,
-                          vol_idx, gfid_idx);
-                key[sizeof(key)-1] = '\0';
-                ret = dict_set_dynstr (load, key, gfid_str);
-                if (ret) {
-                        goto out;
+                keylen = strlen(pair->key);
+                netword = hton32(keylen);
+                memcpy(buf, &netword, sizeof(netword));
+                buf += DICT_DATA_HDR_KEY_LEN;
+                if (!pair->value) {
+                    gf_msg("glusterd", GF_LOG_ERROR, 0, LG_MSG_NULL_PTR,
+                           "pair->value is null!");
+                    ret = -1;
+                    goto out;
                 }
 
-                gfid_str = NULL;
-        }
+                netword = hton32(pair->value->len);
+                memcpy(buf, &netword, sizeof(netword));
+                buf += DICT_DATA_HDR_VAL_LEN;
 
-        snprintf (key, sizeof(key)-1, "%s%d.gfid-count", prefix, vol_idx);
-        key[sizeof(key)-1] = '\0';
-        ret = dict_set_int32 (load, key, gfid_idx);
-        if (ret)
-                goto out;
+                memcpy(buf, pair->key, keylen);
+                buf += keylen;
+                *buf++ = '\0';
 
-        snprintf (key, sizeof(key)-1, "%s%d.quota-cksum", prefix, vol_idx);
-        key[sizeof(key)-1] = '\0';
-        ret = dict_set_uint32 (load, key, volinfo->quota_conf_cksum);
-        if (ret)
-                goto out;
+                if (pair->value->data) {
+                    memcpy(buf, pair->value->data, pair->value->len);
+                    buf += pair->value->len;
+                }
 
-        snprintf (key, sizeof(key)-1, "%s%d.quota-version", prefix, vol_idx);
-        key[sizeof(key)-1] = '\0';
-        ret = dict_set_uint32 (load, key, volinfo->quota_conf_version);
-        if (ret)
-                goto out;
+                pair = pair->next;
+                dict_count--;
+            }
+        }
+    }
 
-        ret = 0;
 out:
-        if (fd != -1)
-                close (fd);
-        GF_FREE (gfid_str);
-        return ret;
+    for (i = 0; i < count; i++) {
+        if (dict_arr[i])
+            dict_unref(dict_arr[i]);
+    }
+    return ret;
 }
 
-int32_t
-glusterd_add_volumes_to_export_dict (dict_t **peer_data)
-{
-        int32_t                 ret = -1;
-        dict_t                  *dict = NULL;
-        glusterd_conf_t         *priv = NULL;
-        glusterd_volinfo_t      *volinfo = NULL;
-        int32_t                 count = 0;
-        glusterd_dict_ctx_t     ctx            = {0};
-        xlator_t               *this = NULL;
-
-        this = THIS;
-        GF_ASSERT (this);
-        priv = this->private;
-        GF_ASSERT (priv);
-
-        dict = dict_new ();
-        if (!dict)
-                goto out;
+int
+glusterd_dict_arr_serialize(dict_t *dict_arr[], int count, char **buf,
+                            u_int *length)
+{
+    ssize_t len = 0;
+    int i = 0;
+    int totcount = 0;
+    int ret = 0;
 
-        cds_list_for_each_entry (volinfo, &priv->volumes, vol_list) {
-                count++;
-                ret = glusterd_add_volume_to_dict (volinfo, dict, count,
-                                                   "volume");
-                if (ret)
-                        goto out;
-                if (!glusterd_is_volume_quota_enabled (volinfo))
-                        continue;
-                ret = glusterd_vol_add_quota_conf_to_dict (volinfo, dict,
-                                                           count, "volume");
-                if (ret)
-                        goto out;
+    for (i = 0; i < count; i++) {
+        if (dict_arr[i]) {
+            len += dict_serialized_length_lk(dict_arr[i]);
+            totcount += dict_arr[i]->count;
         }
+    }
 
-        ret = dict_set_int32 (dict, "count", count);
-        if (ret)
-                goto out;
+    // Subtract HDR_LEN except one dictionary
+    len = len - ((count - 1) * DICT_HDR_LEN);
 
-        ctx.dict = dict;
-        ctx.prefix = "global";
-        ctx.opt_count = 1;
-        ctx.key_name = "key";
-        ctx.val_name = "val";
-        dict_foreach (priv->opts, _add_dict_to_prdict, &ctx);
-        ctx.opt_count--;
-        ret = dict_set_int32 (dict, "global-opt-count", ctx.opt_count);
-        if (ret)
-                goto out;
+    *buf = GF_MALLOC(len, gf_common_mt_char);
+    if (*buf == NULL) {
+        ret = -ENOMEM;
+        goto out;
+    }
 
-        *peer_data = dict;
-out:
-        if (ret)
-                dict_unref (dict);
+    if (length != NULL) {
+        *length = len;
+    }
 
-        gf_log (this->name, GF_LOG_TRACE, "Returning %d", ret);
-        return ret;
+    ret = glusterd_dict_searialize(dict_arr, count, totcount, *buf);
+
+out:
+    return ret;
 }
 
 int32_t
-glusterd_compare_friend_volume (dict_t *peer_data, int32_t count,
-                                int32_t *status, char *hostname)
-{
-
-        int32_t                 ret = -1;
-        char                    key[512] = {0,};
-        glusterd_volinfo_t      *volinfo = NULL;
-        char                    *volname = NULL;
-        uint32_t                cksum = 0;
-        uint32_t                quota_cksum = 0;
-        uint32_t                quota_version = 0;
-        int32_t                 version = 0;
-        xlator_t                *this = NULL;
-
-        GF_ASSERT (peer_data);
-        GF_ASSERT (status);
-
-        this = THIS;
-        GF_ASSERT (this);
+glusterd_add_volumes_to_export_dict(dict_t *peer_data, char **buf,
+                                    u_int *length)
+{
+    int32_t ret = -1;
+    dict_t *dict_arr[128] = {
+        0,
+    };
+    glusterd_conf_t *priv = NULL;
+    glusterd_volinfo_t *volinfo = NULL;
+    int32_t count = 0;
+    glusterd_dict_ctx_t ctx = {0};
+    xlator_t *this = NULL;
+    int totthread = 0;
+    int volcnt = 0;
+    int start = 1;
+    int endindex = 0;
+    int vol_per_thread_limit = 0;
+    glusterd_add_dict_args_t *arg = NULL;
+    pthread_t th_id = {
+        0,
+    };
+    int th_ret = 0;
+    int i = 0;
+
+    this = THIS;
+    GF_ASSERT(this);
+    priv = this->private;
+    GF_ASSERT(priv);
+
+    /* Count the total number of volumes */
+    cds_list_for_each_entry(volinfo, &priv->volumes, vol_list) volcnt++;
+
+    get_gd_vol_thread_limit(&vol_per_thread_limit);
+
+    if ((vol_per_thread_limit == 1) || (vol_per_thread_limit == 0) ||
+        (vol_per_thread_limit > 100)) {
+        totthread = 0;
+    } else {
+        totthread = volcnt / vol_per_thread_limit;
+        if (totthread) {
+            endindex = volcnt % vol_per_thread_limit;
+            if (endindex)
+                totthread++;
+        }
+    }
+
+    if (totthread == 0) {
+        cds_list_for_each_entry(volinfo, &priv->volumes, vol_list)
+        {
+            count++;
+            ret = glusterd_add_volume_to_dict(volinfo, peer_data, count,
+                                              "volume");
+            if (ret)
+                goto out;
+
+            if (!dict_get_sizen(volinfo->dict, VKEY_FEATURES_QUOTA))
+                continue;
+
+            ret = glusterd_vol_add_quota_conf_to_dict(volinfo, peer_data, count,
+                                                      "volume");
+            if (ret)
+                goto out;
+        }
+    } else {
+        for (i = 0; i < totthread; i++) {
+            arg = calloc(1, sizeof(*arg));
+            dict_arr[i] = dict_new();
+            arg->this = this;
+            arg->voldict = dict_arr[i];
+            arg->start = start;
+            if ((i + 1) != totthread) {
+                arg->end = ((i + 1) * vol_per_thread_limit);
+            } else {
+                arg->end = (((i + 1) * vol_per_thread_limit) + endindex);
+            }
+            th_ret = gf_thread_create_detached(
+                &th_id, glusterd_add_bulk_volumes_create_thread, arg,
+                "bulkvoldict");
+            if (th_ret) {
+                gf_log(this->name, GF_LOG_ERROR,
+                       "glusterd_add_bulk_volume %s"
+                       " thread creation failed",
+                       "bulkvoldict");
+                free(arg);
+                goto out;
+            }
+
+            start = start + vol_per_thread_limit;
+            GF_ATOMIC_INC(priv->thread_count);
+            gf_log(this->name, GF_LOG_INFO,
+                   "Create thread %d to populate dict data for volume"
+                   " start index is %d end index is %d",
+                   (i + 1), arg->start, arg->end);
+        }
+        while (GF_ATOMIC_GET(priv->thread_count)) {
+            sleep(1);
+        }
+
+        gf_log(this->name, GF_LOG_INFO,
+               "Finished dictionary population in all threads");
+    }
+
+    ret = dict_set_int32n(peer_data, "count", SLEN("count"), volcnt);
+    if (ret)
+        goto out;
+
+    ctx.dict = peer_data;
+    ctx.prefix = "global";
+    ctx.opt_count = 1;
+    ctx.key_name = "key";
+    ctx.val_name = "val";
+    dict_foreach(priv->opts, _add_dict_to_prdict, &ctx);
+    ctx.opt_count--;
+    ret = dict_set_int32n(peer_data, "global-opt-count",
+                          SLEN("global-opt-count"), ctx.opt_count);
+    if (ret)
+        goto out;
+
+    if (totthread) {
+        gf_log(this->name, GF_LOG_INFO,
+               "Merged multiple dictionaries into a single one");
+        dict_arr[totthread++] = dict_ref(peer_data);
+        ret = glusterd_dict_arr_serialize(dict_arr, totthread, buf, length);
+        gf_log(this->name, GF_LOG_INFO, "Serialize dictionary data returned %d",
+               ret);
+    }
 
-        snprintf (key, sizeof (key), "volume%d.name", count);
-        ret = dict_get_str (peer_data, key, &volname);
-        if (ret)
-                goto out;
+out:
 
-        ret = glusterd_volinfo_find (volname, &volinfo);
+    gf_msg_trace(this->name, 0, "Returning %d", ret);
+    return ret;
+}
 
-        if (ret) {
-                *status = GLUSTERD_VOL_COMP_UPDATE_REQ;
-                ret = 0;
-                goto out;
+static int32_t
+glusterd_compare_friend_volume(dict_t *peer_data, int32_t count,
+                               int32_t *status, char *hostname)
+{
+    int32_t ret = -1;
+    char key[64] = "";
+    char key_prefix[32];
+    int keylen;
+    glusterd_volinfo_t *volinfo = NULL;
+    char *volname = NULL;
+    uint32_t cksum = 0;
+    uint32_t quota_cksum = 0;
+    uint32_t quota_version = 0;
+    uint32_t stage_deleted = 0;
+    int32_t version = 0;
+    xlator_t *this = NULL;
+
+    GF_ASSERT(peer_data);
+    GF_ASSERT(status);
+
+    this = THIS;
+    GF_ASSERT(this);
+
+    snprintf(key_prefix, sizeof(key_prefix), "volume%d", count);
+    keylen = snprintf(key, sizeof(key), "%s.name", key_prefix);
+    ret = dict_get_strn(peer_data, key, keylen, &volname);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_GET_FAILED,
+                "Key=%s", key, NULL);
+        goto out;
+    }
+
+    ret = glusterd_volinfo_find(volname, &volinfo);
+    if (ret) {
+        snprintf(key, sizeof(key), "%s.stage_deleted", key_prefix);
+        ret = dict_get_uint32(peer_data, key, &stage_deleted);
+        /* stage_deleted = 1 means the volume is still in the process of
+         * deleting a volume, so we shouldn't be trying to create a
+         * fresh volume here which would lead to a stale entry
+         */
+        if (!ret && stage_deleted == 0)
+            *status = GLUSTERD_VOL_COMP_UPDATE_REQ;
+        goto out;
+    }
+
+    keylen = snprintf(key, sizeof(key), "%s.version", key_prefix);
+    ret = dict_get_int32n(peer_data, key, keylen, &version);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_GET_FAILED,
+                "Key=%s", key, NULL);
+        goto out;
+    }
+
+    if (version > volinfo->version) {
+        // Mismatch detected
+        gf_msg(this->name, GF_LOG_INFO, 0, GD_MSG_VOL_VERS_MISMATCH,
+               "Version of volume %s differ. local version = %d, "
+               "remote version = %d on peer %s",
+               volinfo->volname, volinfo->version, version, hostname);
+        GF_ATOMIC_INIT(volinfo->volpeerupdate, 1);
+        *status = GLUSTERD_VOL_COMP_UPDATE_REQ;
+        goto out;
+    } else if (version < volinfo->version) {
+        *status = GLUSTERD_VOL_COMP_SCS;
+        goto out;
+    }
+
+    // Now, versions are same, compare cksums.
+    //
+    snprintf(key, sizeof(key), "%s.ckusm", key_prefix);
+    ret = dict_get_uint32(peer_data, key, &cksum);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_GET_FAILED,
+                "Key=%s", key, NULL);
+        goto out;
+    }
+
+    if (cksum != volinfo->cksum) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_CKSUM_VERS_MISMATCH,
+               "Version of Cksums %s differ. local cksum = %u, remote "
+               "cksum = %u on peer %s",
+               volinfo->volname, volinfo->cksum, cksum, hostname);
+        *status = GLUSTERD_VOL_COMP_RJT;
+        goto out;
+    }
+
+    if (!dict_get_sizen(volinfo->dict, VKEY_FEATURES_QUOTA))
+        goto skip_quota;
+
+    snprintf(key, sizeof(key), "%s.quota-version", key_prefix);
+    ret = dict_get_uint32(peer_data, key, &quota_version);
+    if (ret) {
+        gf_msg_debug(this->name, 0,
+                     "quota-version key absent for"
+                     " volume %s in peer %s's response",
+                     volinfo->volname, hostname);
+    } else {
+        if (quota_version > volinfo->quota_conf_version) {
+            // Mismatch detected
+            gf_msg(this->name, GF_LOG_INFO, 0,
+                   GD_MSG_QUOTA_CONFIG_VERS_MISMATCH,
+                   "Quota configuration versions of volume %s "
+                   "differ. local version = %d, remote version = "
+                   "%d on peer %s",
+                   volinfo->volname, volinfo->quota_conf_version, quota_version,
+                   hostname);
+            *status = GLUSTERD_VOL_COMP_UPDATE_REQ;
+            goto out;
+        } else if (quota_version < volinfo->quota_conf_version) {
+            *status = GLUSTERD_VOL_COMP_SCS;
+            goto out;
         }
-
-        memset (key, 0, sizeof (key));
-        snprintf (key, sizeof (key), "volume%d.version", count);
-        ret = dict_get_int32 (peer_data, key, &version);
-        if (ret)
-                goto out;
-
-        if (version > volinfo->version) {
-                //Mismatch detected
-                ret = 0;
-                gf_msg (this->name, GF_LOG_INFO, 0, GD_MSG_VOL_VERS_MISMATCH,
-                        "Version of volume %s differ. local version = %d, "
-                        "remote version = %d on peer %s", volinfo->volname,
-                        volinfo->version, version, hostname);
-                *status = GLUSTERD_VOL_COMP_UPDATE_REQ;
-                goto out;
-        } else if (version < volinfo->version) {
-                *status = GLUSTERD_VOL_COMP_SCS;
-                goto out;
+    }
+
+    // Now, versions are same, compare cksums.
+    //
+    snprintf(key, sizeof(key), "%s.quota-cksum", key_prefix);
+    ret = dict_get_uint32(peer_data, key, &quota_cksum);
+    if (ret) {
+        gf_msg_debug(this->name, 0,
+                     "quota checksum absent for "
+                     "volume %s in peer %s's response",
+                     volinfo->volname, hostname);
+    } else {
+        if (quota_cksum != volinfo->quota_conf_cksum) {
+            gf_msg(this->name, GF_LOG_ERROR, 0,
+                   GD_MSG_QUOTA_CONFIG_CKSUM_MISMATCH,
+                   "Cksums of "
+                   "quota configuration of volume %s differ. local"
+                   " cksum = %u, remote  cksum = %u on peer %s",
+                   volinfo->volname, volinfo->quota_conf_cksum, quota_cksum,
+                   hostname);
+            *status = GLUSTERD_VOL_COMP_RJT;
+            goto out;
         }
+    }
 
-        //Now, versions are same, compare cksums.
-        //
-        memset (key, 0, sizeof (key));
-        snprintf (key, sizeof (key), "volume%d.ckusm", count);
-        ret = dict_get_uint32 (peer_data, key, &cksum);
-        if (ret)
-                goto out;
+skip_quota:
+    *status = GLUSTERD_VOL_COMP_SCS;
 
-        if (cksum != volinfo->cksum) {
-                ret = 0;
-                gf_msg (this->name, GF_LOG_ERROR, 0, GD_MSG_CKSUM_VERS_MISMATCH,
-                        "Version of Cksums %s differ. local cksum = %u, remote "
-                        "cksum = %u on peer %s", volinfo->volname,
-                        volinfo->cksum, cksum, hostname);
-                *status = GLUSTERD_VOL_COMP_RJT;
-                goto out;
-        }
+out:
+    keylen = snprintf(key, sizeof(key), "%s.update", key_prefix);
+
+    if (*status == GLUSTERD_VOL_COMP_UPDATE_REQ) {
+        ret = dict_set_int32n(peer_data, key, keylen, 1);
+    } else {
+        ret = dict_set_int32n(peer_data, key, keylen, 0);
+    }
+    if (*status == GLUSTERD_VOL_COMP_RJT) {
+        gf_event(EVENT_COMPARE_FRIEND_VOLUME_FAILED, "volume=%s",
+                 volinfo->volname);
+    }
+    gf_msg_debug(this->name, 0, "Returning with ret: %d, status: %d", ret,
+                 *status);
+    return ret;
+}
 
-        memset (key, 0, sizeof (key));
-        snprintf (key, sizeof (key), "volume%d.quota-version", count);
-        ret = dict_get_uint32 (peer_data, key, &quota_version);
+static int32_t
+import_prdict_dict(dict_t *peer_data, dict_t *dst_dict, char *key_prefix,
+                   char *value_prefix, int opt_count, char *prefix)
+{
+    char key[512] = "";
+    int keylen;
+    int32_t ret = 0;
+    int i = 1;
+    char *opt_key = NULL;
+    char *opt_val = NULL;
+    char *dup_opt_val = NULL;
+    char msg[2048] = "";
+
+    while (i <= opt_count) {
+        keylen = snprintf(key, sizeof(key), "%s.%s%d", prefix, key_prefix, i);
+        ret = dict_get_strn(peer_data, key, keylen, &opt_key);
         if (ret) {
-                gf_log (this->name, GF_LOG_DEBUG, "quota-version key absent for"
-                        " volume %s in peer %s's response", volinfo->volname,
-                        hostname);
-                ret = 0;
-        } else {
-                if (quota_version > volinfo->quota_conf_version) {
-                        //Mismatch detected
-                        ret = 0;
-                        gf_msg (this->name, GF_LOG_INFO, 0,
-                                GD_MSG_QUOTA_CONFIG_VERS_MISMATCH,
-                                "Quota configuration versions of volume %s "
-                                "differ. local version = %d, remote version = "
-                                "%d on peer %s", volinfo->volname,
-                                volinfo->quota_conf_version,
-                                quota_version, hostname);
-                        *status = GLUSTERD_VOL_COMP_UPDATE_REQ;
-                        goto out;
-                } else if (quota_version < volinfo->quota_conf_version) {
-                        *status = GLUSTERD_VOL_COMP_SCS;
-                        goto out;
-                }
+            snprintf(msg, sizeof(msg),
+                     "Volume dict key not "
+                     "specified");
+            goto out;
         }
 
-        //Now, versions are same, compare cksums.
-        //
-        memset (key, 0, sizeof (key));
-        snprintf (key, sizeof (key), "volume%d.quota-cksum", count);
-        ret = dict_get_uint32 (peer_data, key, &quota_cksum);
+        keylen = snprintf(key, sizeof(key), "%s.%s%d", prefix, value_prefix, i);
+        ret = dict_get_strn(peer_data, key, keylen, &opt_val);
         if (ret) {
-                gf_log (this->name, GF_LOG_DEBUG, "quota checksum absent for "
-                        "volume %s in peer %s's response", volinfo->volname,
-                        hostname);
-                ret = 0;
-        } else {
-                if (quota_cksum != volinfo->quota_conf_cksum) {
-                        ret = 0;
-                        gf_msg (this->name, GF_LOG_ERROR, 0,
-                                GD_MSG_QUOTA_CONFIG_CKSUM_MISMATCH, "Cksums of "
-                                "quota configuration of volume %s differ. local"
-                                " cksum = %u, remote  cksum = %u on peer %s",
-                                volinfo->volname, volinfo->quota_conf_cksum,
-                                quota_cksum, hostname);
-                        *status = GLUSTERD_VOL_COMP_RJT;
-                        goto out;
-                }
+            snprintf(msg, sizeof(msg),
+                     "Volume dict value not "
+                     "specified");
+            goto out;
         }
-        *status = GLUSTERD_VOL_COMP_SCS;
-
-out:
-        gf_log (this->name, GF_LOG_DEBUG, "Returning with ret: %d, status: %d",
-                ret, *status);
-        return ret;
-}
-
-static int32_t
-import_prdict_dict (dict_t *peer_data, dict_t  *dst_dict, char *key_prefix,
-                    char *value_prefix, int opt_count, char *prefix)
-{
-        char                    key[512] = {0,};
-        int32_t                 ret = 0;
-        int                     i = 1;
-        char                    *opt_key = NULL;
-        char                    *opt_val = NULL;
-        char                    *dup_opt_val = NULL;
-        char                    msg[2048] = {0};
-
-        while (i <= opt_count) {
-                memset (key, 0, sizeof (key));
-                snprintf (key, sizeof (key), "%s.%s%d",
-                          prefix, key_prefix, i);
-                ret = dict_get_str (peer_data, key, &opt_key);
-                if (ret) {
-                        snprintf (msg, sizeof (msg), "Volume dict key not "
-                                  "specified");
-                        goto out;
-                }
-
-                memset (key, 0, sizeof (key));
-                snprintf (key, sizeof (key), "%s.%s%d",
-                          prefix, value_prefix, i);
-                ret = dict_get_str (peer_data, key, &opt_val);
-                if (ret) {
-                        snprintf (msg, sizeof (msg), "Volume dict value not "
-                                  "specified");
-                        goto out;
-                }
-                dup_opt_val = gf_strdup (opt_val);
-                if (!dup_opt_val) {
-                        ret = -1;
-                        goto out;
-                }
-                ret = dict_set_dynstr (dst_dict, opt_key, dup_opt_val);
-                if (ret) {
-                        snprintf (msg, sizeof (msg), "Volume set %s %s "
-                                  "unsuccessful", opt_key, dup_opt_val);
-                        goto out;
-                }
-                i++;
+        dup_opt_val = gf_strdup(opt_val);
+        if (!dup_opt_val) {
+            ret = -1;
+            goto out;
+        }
+        ret = dict_set_dynstr(dst_dict, opt_key, dup_opt_val);
+        if (ret) {
+            snprintf(msg, sizeof(msg),
+                     "Volume set %s %s "
+                     "unsuccessful",
+                     opt_key, dup_opt_val);
+            goto out;
         }
+        i++;
+    }
 
 out:
-        if (msg[0])
-                gf_log ("glusterd", GF_LOG_ERROR, "%s", msg);
-        gf_log ("", GF_LOG_DEBUG, "Returning with %d", ret);
-        return ret;
-
+    if (msg[0])
+        gf_msg("glusterd", GF_LOG_ERROR, 0, GD_MSG_IMPORT_PRDICT_DICT, "%s",
+               msg);
+    gf_msg_debug("glusterd", 0, "Returning with %d", ret);
+    return ret;
 }
 
-
 int
-glusterd_spawn_daemons (void *opaque)
+glusterd_spawn_daemons(void *opaque)
 {
-        glusterd_conf_t *conf = THIS->private;
-        gf_boolean_t    start_bricks = !conf->restart_done;
-        int             ret             = -1;
-
-        if (start_bricks) {
-                glusterd_restart_bricks (conf);
-                conf->restart_done = _gf_true;
-        }
-        glusterd_restart_gsyncds (conf);
-        glusterd_restart_rebalance (conf);
-        ret = glusterd_snapdsvc_restart ();
+    glusterd_conf_t *conf = THIS->private;
+    int ret = -1;
 
-        return ret;
+    /* glusterd_restart_brick() will take the sync_lock. */
+    glusterd_restart_bricks(NULL);
+    glusterd_restart_gsyncds(conf);
+    glusterd_restart_rebalance(conf);
+    ret = glusterd_snapdsvc_restart();
+    ret = glusterd_gfproxydsvc_restart();
+    ret = glusterd_shdsvc_restart();
+    return ret;
 }
 
+static int32_t
+glusterd_import_friend_volume_opts(dict_t *peer_data, int count,
+                                   glusterd_volinfo_t *volinfo, char *prefix)
+{
+    char key[64];
+    int keylen;
+    int32_t ret = -1;
+    int opt_count = 0;
+    char msg[2048] = "";
+    char volume_prefix[32];
+
+    GF_ASSERT(peer_data);
+    GF_ASSERT(volinfo);
+
+    snprintf(volume_prefix, sizeof(volume_prefix), "%s%d", prefix, count);
+
+    keylen = snprintf(key, sizeof(key), "%s.opt-count", volume_prefix);
+    ret = dict_get_int32n(peer_data, key, keylen, &opt_count);
+    if (ret) {
+        snprintf(msg, sizeof(msg),
+                 "Volume option count not "
+                 "specified for %s",
+                 volinfo->volname);
+        goto out;
+    }
+
+    ret = import_prdict_dict(peer_data, volinfo->dict, "key", "value",
+                             opt_count, volume_prefix);
+    if (ret) {
+        snprintf(msg, sizeof(msg),
+                 "Unable to import options dict "
+                 "specified for %s",
+                 volinfo->volname);
+        goto out;
+    }
+
+    keylen = snprintf(key, sizeof(key), "%s.gsync-count", volume_prefix);
+    ret = dict_get_int32n(peer_data, key, keylen, &opt_count);
+    if (ret) {
+        snprintf(msg, sizeof(msg),
+                 "Gsync count not "
+                 "specified for %s",
+                 volinfo->volname);
+        goto out;
+    }
+
+    ret = import_prdict_dict(peer_data, volinfo->gsync_slaves, "slave-num",
+                             "slave-val", opt_count, volume_prefix);
+    if (ret) {
+        snprintf(msg, sizeof(msg),
+                 "Unable to import gsync sessions "
+                 "specified for %s",
+                 volinfo->volname);
+        goto out;
+    }
 
-int32_t
-glusterd_import_friend_volume_opts (dict_t *peer_data, int count,
-                                    glusterd_volinfo_t *volinfo,
-                                    char *prefix)
-{
-        char                    key[512] = {0,};
-        int32_t                 ret = -1;
-        int                     opt_count = 0;
-        char                    msg[2048] = {0};
-        char                    volume_prefix[1024] = {0};
-
-        GF_ASSERT (peer_data);
-        GF_ASSERT (volinfo);
-
-        memset (key, 0, sizeof (key));
-        snprintf (key, sizeof (key), "%s%d.opt-count", prefix, count);
-        ret = dict_get_int32 (peer_data, key, &opt_count);
-        if (ret) {
-                snprintf (msg, sizeof (msg), "Volume option count not "
-                          "specified for %s", volinfo->volname);
-                goto out;
-        }
+out:
+    if (msg[0])
+        gf_msg("glusterd", GF_LOG_ERROR, 0, GD_MSG_VOL_OPTS_IMPORT_FAIL, "%s",
+               msg);
+    gf_msg_debug("glusterd", 0, "Returning with %d", ret);
+    return ret;
+}
 
-        snprintf (volume_prefix, sizeof (volume_prefix), "%s%d", prefix, count);
-        ret = import_prdict_dict (peer_data, volinfo->dict, "key", "value",
-                                  opt_count, volume_prefix);
-        if (ret) {
-                snprintf (msg, sizeof (msg), "Unable to import options dict "
-                          "specified for %s", volinfo->volname);
-                goto out;
-        }
+static int32_t
+glusterd_import_new_ta_brick(dict_t *peer_data, int32_t vol_count,
+                             int32_t brick_count,
+                             glusterd_brickinfo_t **ta_brickinfo, char *prefix)
+{
+    char key[128];
+    char key_prefix[64];
+    int keylen;
+    int ret = -1;
+    char *hostname = NULL;
+    char *path = NULL;
+    char *brick_id = NULL;
+    int decommissioned = 0;
+    glusterd_brickinfo_t *new_ta_brickinfo = NULL;
+    char msg[256] = "";
+    char *brick_uuid_str = NULL;
+
+    GF_ASSERT(peer_data);
+    GF_ASSERT(vol_count >= 0);
+    GF_ASSERT(ta_brickinfo);
+    GF_ASSERT(prefix);
+
+    ret = snprintf(key_prefix, sizeof(key_prefix), "%s%d.ta-brick%d", prefix,
+                   vol_count, brick_count);
+
+    if (ret < 0 || ret >= sizeof(key_prefix)) {
+        ret = -1;
+        snprintf(msg, sizeof(msg), "key_prefix too long");
+        goto out;
+    }
+
+    keylen = snprintf(key, sizeof(key), "%s.hostname", key_prefix);
+    ret = dict_get_strn(peer_data, key, keylen, &hostname);
+    if (ret) {
+        snprintf(msg, sizeof(msg), "%s missing in payload", key);
+        goto out;
+    }
+
+    keylen = snprintf(key, sizeof(key), "%s.path", key_prefix);
+    ret = dict_get_strn(peer_data, key, keylen, &path);
+    if (ret) {
+        snprintf(msg, sizeof(msg), "%s missing in payload", key);
+        goto out;
+    }
+
+    keylen = snprintf(key, sizeof(key), "%s.brick_id", key_prefix);
+    ret = dict_get_strn(peer_data, key, keylen, &brick_id);
+
+    keylen = snprintf(key, sizeof(key), "%s.decommissioned", key_prefix);
+    ret = dict_get_int32n(peer_data, key, keylen, &decommissioned);
+    if (ret) {
+        /* For backward compatibility */
+        ret = 0;
+    }
 
-        memset (key, 0, sizeof (key));
-        snprintf (key, sizeof (key), "%s%d.gsync-count", prefix, count);
-        ret = dict_get_int32 (peer_data, key, &opt_count);
-        if (ret) {
-                snprintf (msg, sizeof (msg), "Gsync count not "
-                          "specified for %s", volinfo->volname);
-                goto out;
-        }
+    ret = glusterd_brickinfo_new(&new_ta_brickinfo);
+    if (ret)
+        goto out;
 
-        ret = import_prdict_dict (peer_data, volinfo->gsync_slaves, "slave-num",
-                                  "slave-val", opt_count, volume_prefix);
-        if (ret) {
-                snprintf (msg, sizeof (msg), "Unable to import gsync sessions "
-                          "specified for %s", volinfo->volname);
-                goto out;
-        }
+    ret = snprintf(new_ta_brickinfo->path, sizeof(new_ta_brickinfo->path), "%s",
+                   path);
+    if (ret < 0 || ret >= sizeof(new_ta_brickinfo->path)) {
+        ret = -1;
+        goto out;
+    }
+    ret = snprintf(new_ta_brickinfo->hostname,
+                   sizeof(new_ta_brickinfo->hostname), "%s", hostname);
+    if (ret < 0 || ret >= sizeof(new_ta_brickinfo->hostname)) {
+        ret = -1;
+        goto out;
+    }
+    new_ta_brickinfo->decommissioned = decommissioned;
+    if (brick_id)
+        (void)snprintf(new_ta_brickinfo->brick_id,
+                       sizeof(new_ta_brickinfo->brick_id), "%s", brick_id);
+    keylen = snprintf(key, sizeof(key), "%s.uuid", key_prefix);
+    ret = dict_get_strn(peer_data, key, keylen, &brick_uuid_str);
+    if (ret)
+        goto out;
+    gf_uuid_parse(brick_uuid_str, new_ta_brickinfo->uuid);
+
+    *ta_brickinfo = new_ta_brickinfo;
 
 out:
-        if (msg[0])
-                gf_log ("glusterd", GF_LOG_ERROR, "%s", msg);
-        gf_log ("", GF_LOG_DEBUG, "Returning with %d", ret);
-        return ret;
+    if (msg[0]) {
+        gf_msg("glusterd", GF_LOG_ERROR, 0, GD_MSG_BRICK_IMPORT_FAIL, "%s",
+               msg);
+        gf_event(EVENT_IMPORT_BRICK_FAILED, "peer=%s;ta-brick=%s",
+                 new_ta_brickinfo->hostname, new_ta_brickinfo->path);
+    }
+    gf_msg_debug("glusterd", 0, "Returning with %d", ret);
+    return ret;
 }
 
 /* The prefix represents the type of volume to be added.
  * It will be "volume" for normal volumes, and snap# like
  * snap1, snap2, for snapshot volumes
  */
-int32_t
-glusterd_import_new_brick (dict_t *peer_data, int32_t vol_count,
-                           int32_t brick_count,
-                           glusterd_brickinfo_t **brickinfo,
-                           char *prefix)
-{
-        char                    key[512] = {0,};
-        int                     ret = -1;
-        char                    *hostname = NULL;
-        char                    *path = NULL;
-        char                    *brick_id = NULL;
-        int                     decommissioned = 0;
-        glusterd_brickinfo_t    *new_brickinfo = NULL;
-        char                    msg[2048] = {0};
-        xlator_t                *this     = NULL;
-
-        this = THIS;
-        GF_ASSERT (this);
-        GF_ASSERT (peer_data);
-        GF_ASSERT (vol_count >= 0);
-        GF_ASSERT (brickinfo);
-        GF_ASSERT (prefix);
-
-        memset (key, 0, sizeof (key));
-        snprintf (key, sizeof (key), "%s%d.brick%d.hostname",
-                  prefix, vol_count, brick_count);
-        ret = dict_get_str (peer_data, key, &hostname);
-        if (ret) {
-                snprintf (msg, sizeof (msg), "%s missing in payload", key);
-                goto out;
-        }
-
-        memset (key, 0, sizeof (key));
-        snprintf (key, sizeof (key), "%s%d.brick%d.path",
-                  prefix, vol_count, brick_count);
-        ret = dict_get_str (peer_data, key, &path);
-        if (ret) {
-                snprintf (msg, sizeof (msg), "%s missing in payload", key);
-                goto out;
-        }
-
-        memset (key, 0, sizeof (key));
-        snprintf (key, sizeof (key), "%s%d.brick%d.brick_id",
-                  prefix, vol_count, brick_count);
-        ret = dict_get_str (peer_data, key, &brick_id);
-
-        memset (key, 0, sizeof (key));
-        snprintf (key, sizeof (key), "%s%d.brick%d.decommissioned",
-                  prefix, vol_count, brick_count);
-        ret = dict_get_int32 (peer_data, key, &decommissioned);
-        if (ret) {
-                /* For backward compatibility */
-                ret = 0;
-        }
-
-        ret = glusterd_brickinfo_new (&new_brickinfo);
-        if (ret)
-                goto out;
-
-        strcpy (new_brickinfo->path, path);
-        strcpy (new_brickinfo->hostname, hostname);
-        new_brickinfo->decommissioned = decommissioned;
-        if (brick_id)
-                strcpy (new_brickinfo->brick_id, brick_id);
+static int32_t
+glusterd_import_new_brick(dict_t *peer_data, int32_t vol_count,
+                          int32_t brick_count, glusterd_brickinfo_t **brickinfo,
+                          char *prefix)
+{
+    char key[128];
+    char key_prefix[64];
+    int keylen;
+    int ret = -1;
+    char *hostname = NULL;
+    char *path = NULL;
+    char *brick_id = NULL;
+    int decommissioned = 0;
+    glusterd_brickinfo_t *new_brickinfo = NULL;
+    char msg[256] = "";
+    char *brick_uuid_str = NULL;
+
+    GF_ASSERT(peer_data);
+    GF_ASSERT(vol_count >= 0);
+    GF_ASSERT(brickinfo);
+    GF_ASSERT(prefix);
+
+    ret = snprintf(key_prefix, sizeof(key_prefix), "%s%d.brick%d", prefix,
+                   vol_count, brick_count);
+    if (ret < 0 || ret >= sizeof(key_prefix)) {
+        ret = -1;
+        snprintf(msg, sizeof(msg), "key_prefix too long");
+        goto out;
+    }
+    keylen = snprintf(key, sizeof(key), "%s.hostname", key_prefix);
+    ret = dict_get_strn(peer_data, key, keylen, &hostname);
+    if (ret) {
+        snprintf(msg, sizeof(msg), "%s missing in payload", key);
+        goto out;
+    }
+
+    keylen = snprintf(key, sizeof(key), "%s.path", key_prefix);
+    ret = dict_get_strn(peer_data, key, keylen, &path);
+    if (ret) {
+        snprintf(msg, sizeof(msg), "%s missing in payload", key);
+        goto out;
+    }
+
+    keylen = snprintf(key, sizeof(key), "%s.brick_id", key_prefix);
+    ret = dict_get_strn(peer_data, key, keylen, &brick_id);
+
+    keylen = snprintf(key, sizeof(key), "%s.decommissioned", key_prefix);
+    ret = dict_get_int32n(peer_data, key, keylen, &decommissioned);
+    if (ret) {
+        /* For backward compatibility */
+        ret = 0;
+    }
 
-        memset (key, 0, sizeof (key));
-        snprintf (key, sizeof (key), "%s%d.brick%d", prefix, vol_count,
-                  brick_count);
-        ret = gd_import_new_brick_snap_details (peer_data, key, new_brickinfo);
-        if (ret)
-                goto out;
+    ret = glusterd_brickinfo_new(&new_brickinfo);
+    if (ret)
+        goto out;
 
-        //peerinfo might not be added yet
-        (void) glusterd_resolve_brick (new_brickinfo);
-        ret = 0;
-        *brickinfo = new_brickinfo;
+    ret = snprintf(new_brickinfo->path, sizeof(new_brickinfo->path), "%s",
+                   path);
+    if (ret < 0 || ret >= sizeof(new_brickinfo->path)) {
+        ret = -1;
+        goto out;
+    }
+    ret = snprintf(new_brickinfo->hostname, sizeof(new_brickinfo->hostname),
+                   "%s", hostname);
+    if (ret < 0 || ret >= sizeof(new_brickinfo->hostname)) {
+        ret = -1;
+        goto out;
+    }
+    new_brickinfo->decommissioned = decommissioned;
+    if (brick_id)
+        (void)snprintf(new_brickinfo->brick_id, sizeof(new_brickinfo->brick_id),
+                       "%s", brick_id);
+
+    ret = gd_import_new_brick_snap_details(peer_data, key_prefix,
+                                           new_brickinfo);
+    if (ret)
+        goto out;
+
+    keylen = snprintf(key, sizeof(key), "%s.uuid", key_prefix);
+    ret = dict_get_strn(peer_data, key, keylen, &brick_uuid_str);
+    if (ret)
+        goto out;
+    gf_uuid_parse(brick_uuid_str, new_brickinfo->uuid);
+
+    *brickinfo = new_brickinfo;
 out:
-        if (msg[0])
-                gf_log ("glusterd", GF_LOG_ERROR, "%s", msg);
-        gf_log ("", GF_LOG_DEBUG, "Returning with %d", ret);
-        return ret;
+    if (msg[0]) {
+        gf_msg("glusterd", GF_LOG_ERROR, 0, GD_MSG_BRICK_IMPORT_FAIL, "%s",
+               msg);
+        if (new_brickinfo)
+            gf_event(EVENT_IMPORT_BRICK_FAILED, "peer=%s;brick=%s",
+                     new_brickinfo->hostname, new_brickinfo->path);
+    }
+    gf_msg_debug("glusterd", 0, "Returning with %d", ret);
+    return ret;
 }
 
 /* The prefix represents the type of volume to be added.
  * It will be "volume" for normal volumes, and snap# like
  * snap1, snap2, for snapshot volumes
  */
-int32_t
-glusterd_import_bricks (dict_t *peer_data, int32_t vol_count,
-                        glusterd_volinfo_t *new_volinfo, char *prefix)
-{
-        int                     ret = -1;
-        int                     brick_count = 1;
-        int                     brickid = 0;
-        glusterd_brickinfo_t     *new_brickinfo = NULL;
-
-        GF_ASSERT (peer_data);
-        GF_ASSERT (vol_count >= 0);
-        GF_ASSERT (new_volinfo);
-        GF_ASSERT (prefix);
-        while (brick_count <= new_volinfo->brick_count) {
-
-                ret = glusterd_import_new_brick (peer_data, vol_count,
-                                                 brick_count,
-                                                 &new_brickinfo, prefix);
-                if (ret)
-                        goto out;
-                if (new_brickinfo->brick_id[0] == '\0')
-                        /*We were probed from a peer having op-version
-                         less than GD_OP_VER_PERSISTENT_AFR_XATTRS*/
-                        GLUSTERD_ASSIGN_BRICKID_TO_BRICKINFO (new_brickinfo,
-                                                              new_volinfo,
-                                                              brickid++);
-                cds_list_add_tail (&new_brickinfo->brick_list,
-                                   &new_volinfo->bricks);
-                brick_count++;
-        }
-        ret = 0;
+static int32_t
+glusterd_import_bricks(dict_t *peer_data, int32_t vol_count,
+                       glusterd_volinfo_t *new_volinfo, char *prefix)
+{
+    int ret = -1;
+    int brick_count = 1;
+    int ta_brick_count = 1;
+    int brickid = 0;
+    glusterd_brickinfo_t *new_brickinfo = NULL;
+    glusterd_brickinfo_t *new_ta_brickinfo = NULL;
+
+    GF_ASSERT(peer_data);
+    GF_ASSERT(vol_count >= 0);
+    GF_ASSERT(new_volinfo);
+    GF_ASSERT(prefix);
+    while (brick_count <= new_volinfo->brick_count) {
+        ret = glusterd_import_new_brick(peer_data, vol_count, brick_count,
+                                        &new_brickinfo, prefix);
+        if (ret)
+            goto out;
+        if (new_brickinfo->brick_id[0] == '\0')
+            /*We were probed from a peer having op-version
+             less than GD_OP_VER_PERSISTENT_AFR_XATTRS*/
+            GLUSTERD_ASSIGN_BRICKID_TO_BRICKINFO(new_brickinfo, new_volinfo,
+                                                 brickid++);
+        cds_list_add_tail(&new_brickinfo->brick_list, &new_volinfo->bricks);
+        brick_count++;
+    }
+
+    if (new_volinfo->thin_arbiter_count == 1) {
+        while (ta_brick_count <= new_volinfo->subvol_count) {
+            ret = glusterd_import_new_ta_brick(peer_data, vol_count,
+                                               ta_brick_count,
+                                               &new_ta_brickinfo, prefix);
+            if (ret)
+                goto out;
+            cds_list_add_tail(&new_ta_brickinfo->brick_list,
+                              &new_volinfo->ta_bricks);
+            ta_brick_count++;
+        }
+    }
+    ret = 0;
 out:
-        gf_log ("", GF_LOG_DEBUG, "Returning with %d", ret);
-        return ret;
+    gf_msg_debug("glusterd", 0, "Returning with %d", ret);
+    return ret;
 }
 
 /* The prefix represents the type of volume to be added.
@@ -2930,153 +4217,170 @@ out:
  * snap1, snap2, for snapshot volumes
  */
 int
-glusterd_import_quota_conf (dict_t *peer_data, int vol_idx,
-                            glusterd_volinfo_t *new_volinfo,
-                            char *prefix)
-{
-        int     gfid_idx         = 0;
-        int     gfid_count       = 0;
-        int     ret              = -1;
-        int     fd               = -1;
-        char    key[PATH_MAX]    = {0};
-        char    *gfid_str        = NULL;
-        uuid_t   gfid            = {0,};
-        xlator_t *this           = NULL;
-
-        this = THIS;
-        GF_ASSERT (this);
-        GF_ASSERT (peer_data);
-        GF_ASSERT (prefix);
-
-        if (!glusterd_is_volume_quota_enabled (new_volinfo)) {
-                (void) glusterd_clean_up_quota_store (new_volinfo);
-                return 0;
-        }
-
-        ret = glusterd_store_create_quota_conf_sh_on_absence (new_volinfo);
-        if (ret)
-                goto out;
-
-        fd = gf_store_mkstemp (new_volinfo->quota_conf_shandle);
-        if (fd < 0) {
-                ret = -1;
-                goto out;
-        }
-
-        snprintf (key, sizeof (key)-1, "%s%d.quota-cksum", prefix, vol_idx);
-        key[sizeof(key)-1] = '\0';
-        ret = dict_get_uint32 (peer_data, key, &new_volinfo->quota_conf_cksum);
-        if (ret)
-                gf_log (this->name, GF_LOG_DEBUG, "Failed to get quota cksum");
+glusterd_import_quota_conf(dict_t *peer_data, int vol_idx,
+                           glusterd_volinfo_t *new_volinfo, char *prefix)
+{
+    int gfid_idx = 0;
+    int gfid_count = 0;
+    int ret = -1;
+    int fd = -1;
+    char key[128];
+    char key_prefix[64];
+    int keylen;
+    char *gfid_str = NULL;
+    uuid_t gfid = {
+        0,
+    };
+    xlator_t *this = NULL;
+    int8_t gfid_type = 0;
+
+    this = THIS;
+    GF_ASSERT(this);
+    GF_ASSERT(peer_data);
+    GF_ASSERT(prefix);
+
+    if (!glusterd_is_volume_quota_enabled(new_volinfo)) {
+        (void)glusterd_clean_up_quota_store(new_volinfo);
+        return 0;
+    }
 
-        snprintf (key, sizeof (key)-1, "%s%d.quota-version", prefix, vol_idx);
-        key[sizeof(key)-1] = '\0';
-        ret = dict_get_uint32 (peer_data, key,
-                               &new_volinfo->quota_conf_version);
-        if (ret)
-                gf_log (this->name, GF_LOG_DEBUG, "Failed to get quota "
-                                                  "version");
+    ret = glusterd_store_create_quota_conf_sh_on_absence(new_volinfo);
+    if (ret)
+        goto out;
 
-        snprintf (key, sizeof (key)-1, "%s%d.gfid-count", prefix, vol_idx);
-        key[sizeof(key)-1] = '\0';
-        ret = dict_get_int32 (peer_data, key, &gfid_count);
-        if (ret)
-                goto out;
+    fd = gf_store_mkstemp(new_volinfo->quota_conf_shandle);
+    if (fd < 0) {
+        ret = -1;
+        goto out;
+    }
 
-        ret = glusterd_store_quota_conf_stamp_header (this, fd);
+    ret = snprintf(key_prefix, sizeof(key_prefix), "%s%d", prefix, vol_idx);
+    if (ret < 0 || ret >= sizeof(key_prefix)) {
+        ret = -1;
+        gf_msg_debug(this->name, 0, "Failed to set key_prefix for quota conf");
+        goto out;
+    }
+    snprintf(key, sizeof(key), "%s.quota-cksum", key_prefix);
+    ret = dict_get_uint32(peer_data, key, &new_volinfo->quota_conf_cksum);
+    if (ret)
+        gf_msg_debug(this->name, 0, "Failed to get quota cksum");
+
+    snprintf(key, sizeof(key), "%s.quota-version", key_prefix);
+    ret = dict_get_uint32(peer_data, key, &new_volinfo->quota_conf_version);
+    if (ret)
+        gf_msg_debug(this->name, 0,
+                     "Failed to get quota "
+                     "version");
+
+    keylen = snprintf(key, sizeof(key), "%s.gfid-count", key_prefix);
+    ret = dict_get_int32n(peer_data, key, keylen, &gfid_count);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_GET_FAILED,
+                "Key=%s", key, NULL);
+        goto out;
+    }
+
+    ret = glusterd_quota_conf_write_header(fd);
+    if (ret)
+        goto out;
+
+    for (gfid_idx = 0; gfid_idx < gfid_count; gfid_idx++) {
+        keylen = snprintf(key, sizeof(key) - 1, "%s.gfid%d", key_prefix,
+                          gfid_idx);
+        ret = dict_get_strn(peer_data, key, keylen, &gfid_str);
         if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "Failed to add header to tmp "
-                        "file");
-                goto out;
+            gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_GET_FAILED,
+                    "Key=%s", key, NULL);
+            goto out;
         }
 
-        gfid_idx = 0;
-        for (gfid_idx = 0; gfid_idx < gfid_count; gfid_idx++) {
-
-                snprintf (key, sizeof (key)-1, "%s%d.gfid%d",
-                          prefix, vol_idx, gfid_idx);
-                key[sizeof(key)-1] = '\0';
-                ret = dict_get_str (peer_data, key, &gfid_str);
-                if (ret)
-                        goto out;
-
-                gf_uuid_parse (gfid_str, gfid);
-                ret = write (fd, (void*)gfid, 16);
-                if (ret != 16) {
-                        gf_log (this->name, GF_LOG_CRITICAL, "Unable to write "
-                                "gfid %s into quota.conf for %s", gfid_str,
-                                new_volinfo->volname);
-                        ret = -1;
-                        goto out;
-                }
+        snprintf(key, sizeof(key) - 1, "%s.gfid-type%d", key_prefix, gfid_idx);
+        ret = dict_get_int8(peer_data, key, &gfid_type);
+        if (ret)
+            gfid_type = GF_QUOTA_CONF_TYPE_USAGE;
 
+        gf_uuid_parse(gfid_str, gfid);
+        ret = glusterd_quota_conf_write_gfid(fd, gfid, (char)gfid_type);
+        if (ret < 0) {
+            gf_msg(this->name, GF_LOG_CRITICAL, errno,
+                   GD_MSG_QUOTA_CONF_WRITE_FAIL,
+                   "Unable to write "
+                   "gfid %s into quota.conf for %s",
+                   gfid_str, new_volinfo->volname);
+            ret = -1;
+            goto out;
         }
+    }
 
-        ret = gf_store_rename_tmppath (new_volinfo->quota_conf_shandle);
+    ret = gf_store_rename_tmppath(new_volinfo->quota_conf_shandle);
 
-        ret = 0;
+    ret = 0;
 
 out:
-        if (!ret) {
-                ret = glusterd_compute_cksum (new_volinfo, _gf_true);
-                if (ret) {
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "Failed to compute checksum");
-                        goto clear_quota_conf;
-                }
-
-                ret = glusterd_store_save_quota_version_and_cksum (new_volinfo);
-                if (ret)
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "Failed to save quota version and checksum");
+    if (!ret) {
+        ret = glusterd_compute_cksum(new_volinfo, _gf_true);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_CKSUM_COMPUTE_FAIL,
+                   "Failed to compute checksum");
+            goto clear_quota_conf;
         }
 
+        ret = glusterd_store_save_quota_version_and_cksum(new_volinfo);
+        if (ret)
+            gf_msg(this->name, GF_LOG_ERROR, 0,
+                   GD_MSG_QUOTA_CKSUM_VER_STORE_FAIL,
+                   "Failed to save quota version and checksum");
+    }
+
 clear_quota_conf:
-        if (ret && (fd > 0)) {
-                gf_store_unlink_tmppath (new_volinfo->quota_conf_shandle);
-                (void) gf_store_handle_destroy
-                                              (new_volinfo->quota_conf_shandle);
-                new_volinfo->quota_conf_shandle = NULL;
-        }
+    if (ret && (fd > 0)) {
+        gf_store_unlink_tmppath(new_volinfo->quota_conf_shandle);
+        (void)gf_store_handle_destroy(new_volinfo->quota_conf_shandle);
+        new_volinfo->quota_conf_shandle = NULL;
+    }
 
-        return ret;
+    return ret;
 }
 
 int
-gd_import_friend_volume_rebal_dict (dict_t *dict, int count,
-                                    glusterd_volinfo_t *volinfo)
-{
-        int  ret        = -1;
-        char key[256]   = {0,};
-        int  dict_count  = 0;
-        char prefix[64] = {0};
-
-        GF_ASSERT (dict);
-        GF_ASSERT (volinfo);
-
-        memset (key, 0, sizeof (key));
-        snprintf (key, sizeof (key), "volume%d.rebal-dict-count", count);
-        ret = dict_get_int32 (dict, key, &dict_count);
-        if (ret) {
-                /* Older peers will not have this dict */
-                ret = 0;
-                goto out;
-        }
+gd_import_friend_volume_rebal_dict(dict_t *dict, int count,
+                                   glusterd_volinfo_t *volinfo)
+{
+    int ret = -1;
+    char key[64] = "";
+    int dict_count = 0;
+    char key_prefix[32];
+
+    GF_ASSERT(dict);
+    GF_ASSERT(volinfo);
+    xlator_t *this = THIS;
+    GF_ASSERT(this);
+
+    snprintf(key_prefix, sizeof(key_prefix), "volume%d", count);
+    ret = snprintf(key, sizeof(key), "%s.rebal-dict-count", key_prefix);
+    ret = dict_get_int32n(dict, key, ret, &dict_count);
+    if (ret) {
+        /* Older peers will not have this dict */
+        gf_smsg(this->name, GF_LOG_INFO, errno, GD_MSG_DICT_GET_FAILED,
+                "Key=%s", key, NULL);
+        ret = 0;
+        goto out;
+    }
 
-        volinfo->rebal.dict = dict_new ();
-        if(!volinfo->rebal.dict) {
-                ret = -1;
-                goto out;
-        }
+    volinfo->rebal.dict = dict_new();
+    if (!volinfo->rebal.dict) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_CREATE_FAIL, NULL);
+        ret = -1;
+        goto out;
+    }
 
-        snprintf (prefix, sizeof (prefix), "volume%d", count);
-        ret = import_prdict_dict (dict, volinfo->rebal.dict, "rebal-dict-key",
-                                  "rebal-dict-value", dict_count, prefix);
+    ret = import_prdict_dict(dict, volinfo->rebal.dict, "rebal-dict-key",
+                             "rebal-dict-value", dict_count, key_prefix);
 out:
-        if (ret && volinfo->rebal.dict)
-                dict_unref (volinfo->rebal.dict);
-        gf_log (THIS->name, GF_LOG_DEBUG, "Returning with %d", ret);
-        return ret;
+    if (ret && volinfo->rebal.dict)
+        dict_unref(volinfo->rebal.dict);
+    gf_msg_debug(this->name, 0, "Returning with %d", ret);
+    return ret;
 }
 
 /* The prefix represents the type of volume to be added.
@@ -3084,1459 +4388,2835 @@ out:
  * snap1, snap2, for snapshot volumes
  */
 int32_t
-glusterd_import_volinfo (dict_t *peer_data, int count,
-                         glusterd_volinfo_t **volinfo,
-                         char *prefix)
-{
-        int                ret               = -1;
-        char               key[256]          = {0};
-        char               *parent_volname   = NULL;
-        char               *volname          = NULL;
-        glusterd_volinfo_t *new_volinfo      = NULL;
-        char               *volume_id_str    = NULL;
-        char               *restored_snap    = NULL;
-        char               msg[2048]         = {0};
-        char               *src_brick        = NULL;
-        char               *dst_brick        = NULL;
-        char               *str              = NULL;
-        int                rb_status         = 0;
-        char               *rebalance_id_str = NULL;
-        char               *rb_id_str        = NULL;
-        int                op_version        = 0;
-        int                client_op_version = 0;
-
-        GF_ASSERT (peer_data);
-        GF_ASSERT (volinfo);
-        GF_ASSERT (prefix);
-
-        snprintf (key, sizeof (key), "%s%d.name", prefix, count);
-        ret = dict_get_str (peer_data, key, &volname);
-        if (ret) {
-                snprintf (msg, sizeof (msg), "%s missing in payload", key);
-                goto out;
-        }
-
-        ret = glusterd_volinfo_new (&new_volinfo);
-        if (ret)
-                goto out;
-        strncpy (new_volinfo->volname, volname, sizeof (new_volinfo->volname));
-
-        memset (key, 0, sizeof (key));
-        snprintf (key, sizeof (key), "%s%d.type", prefix, count);
-        ret = dict_get_int32 (peer_data, key, &new_volinfo->type);
-        if (ret) {
-                snprintf (msg, sizeof (msg), "%s missing in payload for %s",
-                          key, volname);
-                goto out;
-        }
-
-        snprintf (key, sizeof (key), "%s%d.parent_volname", prefix, count);
-        ret = dict_get_str (peer_data, key, &parent_volname);
-        if (!ret)
-                strncpy (new_volinfo->parent_volname, parent_volname,
-                         sizeof(new_volinfo->parent_volname));
-
-        memset (key, 0, sizeof (key));
-        snprintf (key, sizeof (key), "%s%d.brick_count", prefix, count);
-        ret = dict_get_int32 (peer_data, key, &new_volinfo->brick_count);
-        if (ret) {
-                snprintf (msg, sizeof (msg), "%s missing in payload for %s",
-                          key, volname);
-                goto out;
-        }
-
-        memset (key, 0, sizeof (key));
-        snprintf (key, sizeof (key), "%s%d.version", prefix, count);
-        ret = dict_get_int32 (peer_data, key, &new_volinfo->version);
-        if (ret) {
-                snprintf (msg, sizeof (msg), "%s missing in payload for %s",
-                          key, volname);
-                goto out;
-        }
-
-        memset (key, 0, sizeof (key));
-        snprintf (key, sizeof (key), "%s%d.status", prefix, count);
-        ret = dict_get_int32 (peer_data, key, (int32_t *)&new_volinfo->status);
-        if (ret) {
-                snprintf (msg, sizeof (msg), "%s missing in payload for %s",
-                          key, volname);
-                goto out;
-        }
-
-        memset (key, 0, sizeof (key));
-        snprintf (key, sizeof (key), "%s%d.sub_count", prefix, count);
-        ret = dict_get_int32 (peer_data, key, &new_volinfo->sub_count);
-        if (ret) {
-                snprintf (msg, sizeof (msg), "%s missing in payload for %s",
-                          key, volname);
-                goto out;
-        }
-
-        /* not having a 'stripe_count' key is not a error
-           (as peer may be of old version) */
-        memset (key, 0, sizeof (key));
-        snprintf (key, sizeof (key), "%s%d.stripe_count", prefix, count);
-        ret = dict_get_int32 (peer_data, key, &new_volinfo->stripe_count);
-        if (ret)
-                gf_log (THIS->name, GF_LOG_INFO,
-                        "peer is possibly old version");
-
-        /* not having a 'replica_count' key is not a error
-           (as peer may be of old version) */
-        memset (key, 0, sizeof (key));
-        snprintf (key, sizeof (key), "%s%d.replica_count", prefix, count);
-        ret = dict_get_int32 (peer_data, key, &new_volinfo->replica_count);
-        if (ret)
-                gf_log (THIS->name, GF_LOG_INFO,
-                        "peer is possibly old version");
-
-        /* not having a 'disperse_count' key is not a error
-           (as peer may be of old version) */
-        memset (key, 0, sizeof (key));
-        snprintf (key, sizeof (key), "%s%d.disperse_count", prefix, count);
-        ret = dict_get_int32 (peer_data, key, &new_volinfo->disperse_count);
-        if (ret)
-                gf_log (THIS->name, GF_LOG_INFO,
-                        "peer is possibly old version");
-
-        /* not having a 'redundancy_count' key is not a error
-           (as peer may be of old version) */
-        memset (key, 0, sizeof (key));
-        snprintf (key, sizeof (key), "%s%d.redundancy_count", prefix, count);
-        ret = dict_get_int32 (peer_data, key, &new_volinfo->redundancy_count);
-        if (ret)
-                gf_log (THIS->name, GF_LOG_INFO,
-                        "peer is possibly old version");
-
-        /* not having a 'dist_count' key is not a error
-           (as peer may be of old version) */
-        memset (key, 0, sizeof (key));
-        snprintf (key, sizeof (key), "%s%d.dist_count", prefix, count);
-        ret = dict_get_int32 (peer_data, key, &new_volinfo->dist_leaf_count);
-        if (ret)
-                gf_log (THIS->name, GF_LOG_INFO,
-                        "peer is possibly old version");
-        new_volinfo->subvol_count = new_volinfo->brick_count/
-                                    glusterd_get_dist_leaf_count (new_volinfo);
-        memset (key, 0, sizeof (key));
-        snprintf (key, sizeof (key), "%s%d.ckusm", prefix, count);
-        ret = dict_get_uint32 (peer_data, key, &new_volinfo->cksum);
-        if (ret) {
-                snprintf (msg, sizeof (msg), "%s missing in payload for %s",
-                          key, volname);
-                goto out;
-        }
-
-        memset (key, 0, sizeof (key));
-        snprintf (key, sizeof (key), "%s%d.volume_id", prefix, count);
-        ret = dict_get_str (peer_data, key, &volume_id_str);
-        if (ret) {
-                snprintf (msg, sizeof (msg), "%s missing in payload for %s",
-                          key, volname);
-                goto out;
-        }
-
-        gf_uuid_parse (volume_id_str, new_volinfo->volume_id);
-
-        memset (key, 0, sizeof (key));
-        snprintf (key, sizeof (key), "%s%d.username", prefix, count);
-        ret = dict_get_str (peer_data, key, &str);
-        if (!ret) {
-                ret = glusterd_auth_set_username (new_volinfo, str);
-                if (ret)
-                        goto out;
-        }
-
-        memset (key, 0, sizeof (key));
-        snprintf (key, sizeof (key), "%s%d.password", prefix, count);
-        ret = dict_get_str (peer_data, key, &str);
-        if (!ret) {
-                ret = glusterd_auth_set_password (new_volinfo, str);
-                if (ret)
-                        goto out;
-        }
-
-        memset (key, 0, sizeof (key));
-        snprintf (key, sizeof (key), "%s%d.transport_type", prefix, count);
-        ret = dict_get_uint32 (peer_data, key, &new_volinfo->transport_type);
-        if (ret) {
-                snprintf (msg, sizeof (msg), "%s missing in payload for %s",
-                          key, volname);
-                goto out;
-        }
-
-        memset (key, 0, sizeof (key));
-        snprintf (key, sizeof (key), "%s%d.rebalance", prefix, count);
-        ret = dict_get_uint32 (peer_data, key, &new_volinfo->rebal.defrag_cmd);
-        if (ret) {
-                snprintf (msg, sizeof (msg), "%s missing in payload for %s",
-                          key, volname);
-                goto out;
-        }
-
-        memset (key, 0, sizeof (key));
-        snprintf (key, sizeof (key), "%s%d.rebalance-id", prefix, count);
-        ret = dict_get_str (peer_data, key, &rebalance_id_str);
-        if (ret) {
-                /* This is not present in older glusterfs versions,
-                 * so don't error out
-                 */
-                ret = 0;
-        } else {
-                gf_uuid_parse (rebalance_id_str, new_volinfo->rebal.rebalance_id);
-        }
-
-        memset (key, 0, sizeof (key));
-        snprintf (key, sizeof (key), "%s%d.rebalance-op", prefix, count);
-        ret = dict_get_uint32 (peer_data, key,
-                               (uint32_t *) &new_volinfo->rebal.op);
-        if (ret) {
-                /* This is not present in older glusterfs versions,
-                 * so don't error out
-                 */
-                ret = 0;
-        }
-        ret = gd_import_friend_volume_rebal_dict (peer_data, count,
-                                                  new_volinfo);
-        if (ret) {
-                snprintf (msg, sizeof (msg), "Failed to import rebalance dict "
-                          "for volume.");
-                goto out;
+glusterd_import_volinfo(dict_t *peer_data, int count,
+                        glusterd_volinfo_t **volinfo, char *prefix)
+{
+    int ret = -1;
+    char key[64] = "";
+    char key_prefix[32];
+    int keylen;
+    char *parent_volname = NULL;
+    char *volname = NULL;
+    glusterd_volinfo_t *new_volinfo = NULL;
+    char *volume_id_str = NULL;
+    char msg[2048] = "";
+    char *str = NULL;
+    char *rebalance_id_str = NULL;
+    int op_version = 0;
+    int client_op_version = 0;
+    uint32_t stage_deleted = 0;
+
+    GF_ASSERT(peer_data);
+    GF_ASSERT(volinfo);
+    GF_ASSERT(prefix);
+
+    ret = snprintf(key_prefix, sizeof(key_prefix), "%s%d", prefix, count);
+    if (ret < 0 || ret >= sizeof(key_prefix)) {
+        ret = -1;
+        snprintf(msg, sizeof(msg), "key_prefix too big");
+        goto out;
+    }
+
+    keylen = snprintf(key, sizeof(key), "%s.name", key_prefix);
+    ret = dict_get_strn(peer_data, key, keylen, &volname);
+    if (ret) {
+        snprintf(msg, sizeof(msg), "%s missing in payload", key);
+        goto out;
+    }
+
+    snprintf(key, sizeof(key), "%s.stage_deleted", key_prefix);
+    ret = dict_get_uint32(peer_data, key, &stage_deleted);
+    /* stage_deleted = 1 means the volume is still in the process of
+     * deleting a volume, so we shouldn't be trying to create a
+     * fresh volume here which would lead to a stale entry
+     */
+    if (stage_deleted) {
+        goto out;
+    }
+
+    ret = glusterd_volinfo_new(&new_volinfo);
+    if (ret)
+        goto out;
+    ret = snprintf(new_volinfo->volname, sizeof(new_volinfo->volname), "%s",
+                   volname);
+    if (ret < 0 || ret >= sizeof(new_volinfo->volname)) {
+        ret = -1;
+        goto out;
+    }
+    keylen = snprintf(key, sizeof(key), "%s.type", key_prefix);
+    ret = dict_get_int32n(peer_data, key, keylen, &new_volinfo->type);
+    if (ret) {
+        snprintf(msg, sizeof(msg), "%s missing in payload for %s", key,
+                 volname);
+        goto out;
+    }
+
+    keylen = snprintf(key, sizeof(key), "%s.parent_volname", key_prefix);
+    ret = dict_get_strn(peer_data, key, keylen, &parent_volname);
+    if (!ret) {
+        ret = snprintf(new_volinfo->parent_volname,
+                       sizeof(new_volinfo->parent_volname), "%s",
+                       parent_volname);
+        if (ret < 0 || ret >= sizeof(new_volinfo->volname)) {
+            ret = -1;
+            goto out;
         }
-
-        memset (key, 0, sizeof (key));
-        snprintf (key, 256, "%s%d."GLUSTERD_STORE_KEY_RB_STATUS, prefix, count);
-        ret = dict_get_int32 (peer_data, key, &rb_status);
+    }
+    keylen = snprintf(key, sizeof(key), "%s.brick_count", key_prefix);
+    ret = dict_get_int32n(peer_data, key, keylen, &new_volinfo->brick_count);
+    if (ret) {
+        snprintf(msg, sizeof(msg), "%s missing in payload for %s", key,
+                 volname);
+        goto out;
+    }
+
+    keylen = snprintf(key, sizeof(key), "%s.version", key_prefix);
+    ret = dict_get_int32n(peer_data, key, keylen, &new_volinfo->version);
+    if (ret) {
+        snprintf(msg, sizeof(msg), "%s missing in payload for %s", key,
+                 volname);
+        goto out;
+    }
+
+    keylen = snprintf(key, sizeof(key), "%s.status", key_prefix);
+    ret = dict_get_int32n(peer_data, key, keylen,
+                          (int32_t *)&new_volinfo->status);
+    if (ret) {
+        snprintf(msg, sizeof(msg), "%s missing in payload for %s", key,
+                 volname);
+        goto out;
+    }
+
+    keylen = snprintf(key, sizeof(key), "%s.sub_count", key_prefix);
+    ret = dict_get_int32n(peer_data, key, keylen, &new_volinfo->sub_count);
+    if (ret) {
+        snprintf(msg, sizeof(msg), "%s missing in payload for %s", key,
+                 volname);
+        goto out;
+    }
+
+    keylen = snprintf(key, sizeof(key), "%s.subvol_count", key_prefix);
+    ret = dict_get_int32n(peer_data, key, keylen, &new_volinfo->subvol_count);
+    if (ret) {
+        snprintf(msg, sizeof(msg), "%s missing in payload for %s", key,
+                 volname);
+        goto out;
+    }
+
+    /* not having a 'stripe_count' key is not a error
+       (as peer may be of old version) */
+    keylen = snprintf(key, sizeof(key), "%s.stripe_count", key_prefix);
+    ret = dict_get_int32n(peer_data, key, keylen, &new_volinfo->stripe_count);
+    if (ret)
+        gf_msg(THIS->name, GF_LOG_INFO, 0, GD_MSG_DICT_GET_FAILED,
+               "peer is possibly old version");
+
+    /* not having a 'replica_count' key is not a error
+       (as peer may be of old version) */
+    keylen = snprintf(key, sizeof(key), "%s.replica_count", key_prefix);
+    ret = dict_get_int32n(peer_data, key, keylen, &new_volinfo->replica_count);
+    if (ret)
+        gf_msg(THIS->name, GF_LOG_INFO, 0, GD_MSG_DICT_GET_FAILED,
+               "peer is possibly old version");
+
+    /* not having a 'arbiter_count' key is not a error
+       (as peer may be of old version) */
+    keylen = snprintf(key, sizeof(key), "%s.arbiter_count", key_prefix);
+    ret = dict_get_int32n(peer_data, key, keylen, &new_volinfo->arbiter_count);
+    if (ret)
+        gf_msg(THIS->name, GF_LOG_INFO, 0, GD_MSG_DICT_GET_FAILED,
+               "peer is possibly old version");
+
+    /* not having a 'thin_arbiter_count' key is not a error
+       (as peer may be of old version) */
+    keylen = snprintf(key, sizeof(key), "%s.thin_arbiter_count", key_prefix);
+    ret = dict_get_int32n(peer_data, key, keylen,
+                          &new_volinfo->thin_arbiter_count);
+    if (ret)
+        gf_msg(THIS->name, GF_LOG_INFO, 0, GD_MSG_DICT_GET_FAILED,
+               "peer is possibly old version");
+
+    /* not having a 'disperse_count' key is not a error
+       (as peer may be of old version) */
+    keylen = snprintf(key, sizeof(key), "%s.disperse_count", key_prefix);
+    ret = dict_get_int32n(peer_data, key, keylen, &new_volinfo->disperse_count);
+    if (ret)
+        gf_msg(THIS->name, GF_LOG_INFO, 0, GD_MSG_DICT_GET_FAILED,
+               "peer is possibly old version");
+
+    /* not having a 'redundancy_count' key is not a error
+       (as peer may be of old version) */
+    keylen = snprintf(key, sizeof(key), "%s.redundancy_count", key_prefix);
+    ret = dict_get_int32n(peer_data, key, keylen,
+                          &new_volinfo->redundancy_count);
+    if (ret)
+        gf_msg(THIS->name, GF_LOG_INFO, 0, GD_MSG_DICT_GET_FAILED,
+               "peer is possibly old version");
+
+    /* not having a 'dist_count' key is not a error
+       (as peer may be of old version) */
+    keylen = snprintf(key, sizeof(key), "%s.dist_count", key_prefix);
+    ret = dict_get_int32n(peer_data, key, keylen,
+                          &new_volinfo->dist_leaf_count);
+    if (ret)
+        gf_msg(THIS->name, GF_LOG_INFO, 0, GD_MSG_DICT_GET_FAILED,
+               "peer is possibly old version");
+
+    new_volinfo->subvol_count = new_volinfo->brick_count /
+                                glusterd_get_dist_leaf_count(new_volinfo);
+    snprintf(key, sizeof(key), "%s.ckusm", key_prefix);
+    ret = dict_get_uint32(peer_data, key, &new_volinfo->cksum);
+    if (ret) {
+        snprintf(msg, sizeof(msg), "%s missing in payload for %s", key,
+                 volname);
+        goto out;
+    }
+
+    keylen = snprintf(key, sizeof(key), "%s.volume_id", key_prefix);
+    ret = dict_get_strn(peer_data, key, keylen, &volume_id_str);
+    if (ret) {
+        snprintf(msg, sizeof(msg), "%s missing in payload for %s", key,
+                 volname);
+        goto out;
+    }
+
+    gf_uuid_parse(volume_id_str, new_volinfo->volume_id);
+
+    keylen = snprintf(key, sizeof(key), "%s.username", key_prefix);
+    ret = dict_get_strn(peer_data, key, keylen, &str);
+    if (!ret) {
+        ret = glusterd_auth_set_username(new_volinfo, str);
         if (ret)
-                goto out;
-        new_volinfo->rep_brick.rb_status = rb_status;
-
-        if (new_volinfo->rep_brick.rb_status > GF_RB_STATUS_NONE) {
-
-                memset (key, 0, sizeof (key));
-                snprintf (key, 256, "%s%d."GLUSTERD_STORE_KEY_RB_SRC_BRICK,
-                          prefix, count);
-                ret = dict_get_str (peer_data, key, &src_brick);
-                if (ret)
-                        goto out;
-
-                ret = glusterd_brickinfo_new_from_brick (src_brick,
-                                        &new_volinfo->rep_brick.src_brick);
-                if (ret) {
-                        gf_log ("", GF_LOG_ERROR, "Unable to create"
-                                " src brickinfo");
-                        goto out;
-                }
-
-                memset (key, 0, sizeof (key));
-                snprintf (key, 256, "%s%d."GLUSTERD_STORE_KEY_RB_DST_BRICK,
-                          prefix, count);
-                ret = dict_get_str (peer_data, key, &dst_brick);
-                if (ret)
-                        goto out;
-
-                ret = glusterd_brickinfo_new_from_brick (dst_brick,
-                                     &new_volinfo->rep_brick.dst_brick);
-                if (ret) {
-                        gf_log ("", GF_LOG_ERROR, "Unable to create"
-                                " dst brickinfo");
-                        goto out;
-                }
-
-                memset (key, 0, sizeof (key));
-                snprintf (key, sizeof (key), "%s%d.rb_id", prefix, count);
-                ret = dict_get_str (peer_data, key, &rb_id_str);
-                if (ret) {
-                        /* This is not present in older glusterfs versions,
-                         * so don't error out
-                         */
-                        ret = 0;
-                } else {
-                        gf_uuid_parse (rb_id_str, new_volinfo->rep_brick.rb_id);
-                }
-        }
-
-        memset (key, 0, sizeof (key));
-        snprintf (key, sizeof (key), "%s%d", prefix, count);
-        ret = gd_import_volume_snap_details (peer_data, new_volinfo, key,
-                                             volname);
-        if (ret) {
-                gf_log ("glusterd", GF_LOG_ERROR, "Failed to import snapshot "
-                        "details for volume %s", volname);
-                goto out;
-        }
+            goto out;
+    }
 
-        ret = glusterd_import_friend_volume_opts (peer_data, count,
-                                                  new_volinfo, prefix);
+    keylen = snprintf(key, sizeof(key), "%s.password", key_prefix);
+    ret = dict_get_strn(peer_data, key, keylen, &str);
+    if (!ret) {
+        ret = glusterd_auth_set_password(new_volinfo, str);
         if (ret)
-                goto out;
-
-        /* Import the volume's op-versions if available else set it to 1.
-         * Not having op-versions implies this informtation was obtained from a
-         * op-version 1 friend (gluster-3.3), ergo the cluster is at op-version
-         * 1 and all volumes are at op-versions 1.
-         *
-         * Either both the volume op-versions should be absent or both should be
-         * present. Only one being present is a failure
+            goto out;
+    }
+
+    snprintf(key, sizeof(key), "%s.transport_type", key_prefix);
+    ret = dict_get_uint32(peer_data, key, &new_volinfo->transport_type);
+    if (ret) {
+        snprintf(msg, sizeof(msg), "%s missing in payload for %s", key,
+                 volname);
+        goto out;
+    }
+
+    snprintf(key, sizeof(key), "%s.rebalance", key_prefix);
+    ret = dict_get_uint32(peer_data, key, &new_volinfo->rebal.defrag_cmd);
+    if (ret) {
+        snprintf(msg, sizeof(msg), "%s missing in payload for %s", key,
+                 volname);
+        goto out;
+    }
+
+    keylen = snprintf(key, sizeof(key), "%s.rebalance-id", key_prefix);
+    ret = dict_get_strn(peer_data, key, keylen, &rebalance_id_str);
+    if (ret) {
+        /* This is not present in older glusterfs versions,
+         * so don't error out
          */
-        memset (key, 0, sizeof (key));
-        snprintf (key, sizeof (key), "%s%d.op-version", prefix, count);
-        ret = dict_get_int32 (peer_data, key, &op_version);
-        if (ret)
-                ret = 0;
-        memset (key, 0, sizeof (key));
-        snprintf (key, sizeof (key), "%s%d.client-op-version", prefix, count);
-        ret = dict_get_int32 (peer_data, key, &client_op_version);
-        if (ret)
-                ret = 0;
-
-        if (op_version && client_op_version) {
-                new_volinfo->op_version = op_version;
-                new_volinfo->client_op_version = client_op_version;
-        } else if (((op_version == 0) && (client_op_version != 0)) ||
-                   ((op_version != 0) && (client_op_version == 0))) {
-                ret = -1;
-                gf_log ("glusterd", GF_LOG_ERROR,
-                        "Only one volume op-version found");
-                goto out;
-        } else {
-                new_volinfo->op_version = 1;
-                new_volinfo->client_op_version = 1;
-        }
-
-        memset (key, 0 ,sizeof (key));
-        snprintf (key, sizeof (key), "%s%d.caps", prefix, count);
-        /*This is not present in older glusterfs versions, so ignore ret value*/
-        ret = dict_get_int32 (peer_data, key, &new_volinfo->caps);
-
-        ret = glusterd_import_bricks (peer_data, count, new_volinfo, prefix);
-        if (ret)
-                goto out;
+        ret = 0;
+    } else {
+        gf_uuid_parse(rebalance_id_str, new_volinfo->rebal.rebalance_id);
+    }
+
+    snprintf(key, sizeof(key), "%s.rebalance-op", key_prefix);
+    /* This is not present in older glusterfs versions,
+     * so don't error out
+     */
+    ret = dict_get_uint32(peer_data, key, (uint32_t *)&new_volinfo->rebal.op);
+
+    ret = gd_import_friend_volume_rebal_dict(peer_data, count, new_volinfo);
+    if (ret) {
+        snprintf(msg, sizeof(msg),
+                 "Failed to import rebalance dict "
+                 "for volume.");
+        goto out;
+    }
+
+    ret = gd_import_volume_snap_details(peer_data, new_volinfo, key_prefix,
+                                        volname);
+    if (ret) {
+        gf_msg("glusterd", GF_LOG_ERROR, 0, GD_MSG_SNAP_DETAILS_IMPORT_FAIL,
+               "Failed to import snapshot "
+               "details for volume %s",
+               volname);
+        goto out;
+    }
+
+    ret = glusterd_import_friend_volume_opts(peer_data, count, new_volinfo,
+                                             prefix);
+    if (ret)
+        goto out;
+
+    /* Import the volume's op-versions if available else set it to 1.
+     * Not having op-versions implies this informtation was obtained from a
+     * op-version 1 friend (gluster-3.3), ergo the cluster is at op-version
+     * 1 and all volumes are at op-versions 1.
+     *
+     * Either both the volume op-versions should be absent or both should be
+     * present. Only one being present is a failure
+     */
+    keylen = snprintf(key, sizeof(key), "%s.op-version", key_prefix);
+    ret = dict_get_int32n(peer_data, key, keylen, &op_version);
+    if (ret)
+        ret = 0;
+    keylen = snprintf(key, sizeof(key), "%s.client-op-version", key_prefix);
+    ret = dict_get_int32n(peer_data, key, keylen, &client_op_version);
+    if (ret)
+        ret = 0;
 
-        *volinfo = new_volinfo;
+    if (op_version && client_op_version) {
+        new_volinfo->op_version = op_version;
+        new_volinfo->client_op_version = client_op_version;
+    } else if (((op_version == 0) && (client_op_version != 0)) ||
+               ((op_version != 0) && (client_op_version == 0))) {
+        ret = -1;
+        gf_msg("glusterd", GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+               "Only one volume op-version found");
+        goto out;
+    } else {
+        new_volinfo->op_version = 1;
+        new_volinfo->client_op_version = 1;
+    }
+
+    keylen = snprintf(key, sizeof(key), "%s.quota-xattr-version", key_prefix);
+    /*This is not present in older glusterfs versions, so ignore ret value*/
+    ret = dict_get_int32n(peer_data, key, keylen,
+                          &new_volinfo->quota_xattr_version);
+
+    ret = glusterd_import_bricks(peer_data, count, new_volinfo, prefix);
+    if (ret)
+        goto out;
+
+    *volinfo = new_volinfo;
 out:
-        if (msg[0])
-                gf_log ("glusterd", GF_LOG_ERROR, "%s", msg);
-        gf_log ("", GF_LOG_DEBUG, "Returning with %d", ret);
-        return ret;
+    if (msg[0]) {
+        gf_msg("glusterd", GF_LOG_ERROR, 0, GD_MSG_VOLINFO_IMPORT_FAIL, "%s",
+               msg);
+        gf_event(EVENT_IMPORT_VOLUME_FAILED, "volume=%s", new_volinfo->volname);
+    }
+    gf_msg_debug("glusterd", 0, "Returning with %d", ret);
+    return ret;
 }
 
 int32_t
-glusterd_volume_disconnect_all_bricks (glusterd_volinfo_t *volinfo)
-{
-        int                  ret = 0;
-        glusterd_brickinfo_t *brickinfo = NULL;
-        GF_ASSERT (volinfo);
-
-        cds_list_for_each_entry (brickinfo, &volinfo->bricks, brick_list) {
-                if (glusterd_is_brick_started (brickinfo)) {
-                        ret = glusterd_brick_disconnect (brickinfo);
-                        if (ret) {
-                                gf_log ("glusterd", GF_LOG_ERROR, "Failed to "
-                                        "disconnect %s:%s", brickinfo->hostname,
-                                        brickinfo->path);
-                                break;
-                        }
+glusterd_volume_disconnect_all_bricks(glusterd_volinfo_t *volinfo)
+{
+    int ret = 0;
+    glusterd_brickinfo_t *brickinfo = NULL;
+    glusterd_brick_proc_t *brick_proc = NULL;
+    int brick_count = 0;
+
+    GF_ASSERT(volinfo);
+
+    cds_list_for_each_entry(brickinfo, &volinfo->bricks, brick_list)
+    {
+        if (glusterd_is_brick_started(brickinfo)) {
+            /* If brick multiplexing is enabled then we can't
+             * blindly set brickinfo->rpc to NULL as it might impact
+             * the other attached bricks.
+             */
+            ret = glusterd_brick_proc_for_port(brickinfo->port, &brick_proc);
+            if (!ret) {
+                brick_count = brick_proc->brick_count;
+            }
+            if (!is_brick_mx_enabled() || brick_count == 0) {
+                ret = glusterd_brick_disconnect(brickinfo);
+                if (ret) {
+                    gf_msg("glusterd", GF_LOG_ERROR, 0,
+                           GD_MSD_BRICK_DISCONNECT_FAIL,
+                           "Failed to "
+                           "disconnect %s:%s",
+                           brickinfo->hostname, brickinfo->path);
+                    break;
                 }
+            }
         }
+    }
 
-        return ret;
+    return ret;
 }
 
 int32_t
-glusterd_volinfo_copy_brick_portinfo (glusterd_volinfo_t *new_volinfo,
-                                      glusterd_volinfo_t *old_volinfo)
-{
-        char                    pidfile[PATH_MAX+1] = {0,};
-        glusterd_brickinfo_t   *new_brickinfo       = NULL;
-        glusterd_brickinfo_t   *old_brickinfo       = NULL;
-        glusterd_conf_t        *priv                = NULL;
-        int                     ret                 = 0;
-        xlator_t               *this                = NULL;
-
-        GF_ASSERT (new_volinfo);
-        GF_ASSERT (old_volinfo);
-        this = THIS;
-        GF_ASSERT (this);
-        priv = this->private;
-        GF_ASSERT (priv);
-
-        if (_gf_false == glusterd_is_volume_started (new_volinfo))
-                goto out;
-
-        cds_list_for_each_entry (new_brickinfo, &new_volinfo->bricks,
-                                 brick_list) {
-                ret = glusterd_volume_brickinfo_get (new_brickinfo->uuid,
-                                                     new_brickinfo->hostname,
-                                                     new_brickinfo->path,
-                                                     old_volinfo,
-                                                     &old_brickinfo);
-                if (ret == 0) {
-                        GLUSTERD_GET_BRICK_PIDFILE (pidfile, old_volinfo,
-                                                    old_brickinfo, priv);
-                        if (gf_is_service_running (pidfile, NULL))
-                                new_brickinfo->port = old_brickinfo->port;
-
+glusterd_volinfo_copy_brickinfo(glusterd_volinfo_t *old_volinfo,
+                                glusterd_volinfo_t *new_volinfo)
+{
+    glusterd_brickinfo_t *new_brickinfo = NULL;
+    glusterd_brickinfo_t *old_brickinfo = NULL;
+    glusterd_brickinfo_t *new_ta_brickinfo = NULL;
+    glusterd_brickinfo_t *old_ta_brickinfo = NULL;
+    glusterd_conf_t *priv = NULL;
+    int ret = 0;
+    xlator_t *this = NULL;
+    char abspath[PATH_MAX] = "";
+
+    GF_ASSERT(new_volinfo);
+    GF_ASSERT(old_volinfo);
+    this = THIS;
+    GF_ASSERT(this);
+    priv = this->private;
+    GF_ASSERT(priv);
+
+    cds_list_for_each_entry(new_brickinfo, &new_volinfo->bricks, brick_list)
+    {
+        ret = glusterd_volume_brickinfo_get(
+            new_brickinfo->uuid, new_brickinfo->hostname, new_brickinfo->path,
+            old_volinfo, &old_brickinfo);
+        if (ret == 0) {
+            new_brickinfo->port = old_brickinfo->port;
+
+            if (old_brickinfo->real_path[0] == '\0') {
+                if (!realpath(new_brickinfo->path, abspath)) {
+                    /* Here an ENOENT should also be a
+                     * failure as the brick is expected to
+                     * be in existence
+                     */
+                    gf_msg(this->name, GF_LOG_CRITICAL, errno,
+                           GD_MSG_BRICKINFO_CREATE_FAIL,
+                           "realpath () failed for brick "
+                           "%s. The underlying filesystem "
+                           "may be in bad state",
+                           new_brickinfo->path);
+                    ret = -1;
+                    goto out;
+                }
+                if (strlen(abspath) >= sizeof(new_brickinfo->real_path)) {
+                    ret = -1;
+                    goto out;
+                }
+                (void)strncpy(new_brickinfo->real_path, abspath,
+                              sizeof(new_brickinfo->real_path));
+            } else {
+                (void)strncpy(new_brickinfo->real_path,
+                              old_brickinfo->real_path,
+                              sizeof(new_brickinfo->real_path));
+            }
+        }
+    }
+    if (new_volinfo->thin_arbiter_count == 1) {
+        cds_list_for_each_entry(new_ta_brickinfo, &new_volinfo->ta_bricks,
+                                brick_list)
+        {
+            ret = glusterd_volume_ta_brickinfo_get(
+                new_ta_brickinfo->uuid, new_ta_brickinfo->hostname,
+                new_ta_brickinfo->path, old_volinfo, &old_ta_brickinfo);
+            if (ret == 0) {
+                new_ta_brickinfo->port = old_ta_brickinfo->port;
+
+                if (old_ta_brickinfo->real_path[0] == '\0') {
+                    if (!realpath(new_ta_brickinfo->path, abspath)) {
+                        /* Here an ENOENT should also be a
+                         * failure as the brick is expected to
+                         * be in existence
+                         */
+                        gf_msg(this->name, GF_LOG_CRITICAL, errno,
+                               GD_MSG_BRICKINFO_CREATE_FAIL,
+                               "realpath () failed for brick "
+                               "%s. The underlying filesystem "
+                               "may be in bad state",
+                               new_brickinfo->path);
+                        ret = -1;
+                        goto out;
+                    }
+                    if (strlen(abspath) >=
+                        sizeof(new_ta_brickinfo->real_path)) {
+                        ret = -1;
+                        goto out;
+                    }
+                    (void)strncpy(new_ta_brickinfo->real_path, abspath,
+                                  sizeof(new_ta_brickinfo->real_path));
+                } else {
+                    (void)strncpy(new_ta_brickinfo->real_path,
+                                  old_ta_brickinfo->real_path,
+                                  sizeof(new_ta_brickinfo->real_path));
                 }
+            }
         }
+    }
+    ret = 0;
+
 out:
-        ret = 0;
-        return ret;
+    return ret;
 }
 
 int32_t
-glusterd_volinfo_stop_stale_bricks (glusterd_volinfo_t *new_volinfo,
-                                    glusterd_volinfo_t *old_volinfo)
-{
-        glusterd_brickinfo_t *new_brickinfo = NULL;
-        glusterd_brickinfo_t *old_brickinfo = NULL;
-
-        int             ret = 0;
-        GF_ASSERT (new_volinfo);
-        GF_ASSERT (old_volinfo);
-        if (_gf_false == glusterd_is_volume_started (old_volinfo))
-                goto out;
-        cds_list_for_each_entry (old_brickinfo, &old_volinfo->bricks,
-                                 brick_list) {
-                ret = glusterd_volume_brickinfo_get (old_brickinfo->uuid,
-                                                     old_brickinfo->hostname,
-                                                     old_brickinfo->path,
-                                                     new_volinfo,
-                                                     &new_brickinfo);
-                /* If the brick is stale, i.e it's not a part of the new volume
-                 * or if it's part of the new volume and is pending a snap,
-                 * then stop the brick process
-                 */
-                if (ret || (new_brickinfo->snap_status == -1)) {
-                        /*TODO: may need to switch to 'atomic' flavour of
-                         * brick_stop, once we make peer rpc program also
-                         * synctask enabled*/
-                        ret = glusterd_brick_stop (old_volinfo, old_brickinfo,
-                                                   _gf_false);
-                        if (ret)
-                                gf_msg ("glusterd", GF_LOG_ERROR, 0,
-                                        GD_MSG_BRICK_STOP_FAIL, "Failed to stop"
-                                        " brick %s:%s", old_brickinfo->hostname,
-                                        old_brickinfo->path);
-                }
-        }
-        ret = 0;
+glusterd_volinfo_stop_stale_bricks(glusterd_volinfo_t *new_volinfo,
+                                   glusterd_volinfo_t *old_volinfo)
+{
+    glusterd_brickinfo_t *new_brickinfo = NULL;
+    glusterd_brickinfo_t *old_brickinfo = NULL;
+
+    int ret = 0;
+    GF_ASSERT(new_volinfo);
+    GF_ASSERT(old_volinfo);
+    if (_gf_false == glusterd_is_volume_started(old_volinfo))
+        goto out;
+    cds_list_for_each_entry(old_brickinfo, &old_volinfo->bricks, brick_list)
+    {
+        ret = glusterd_volume_brickinfo_get(
+            old_brickinfo->uuid, old_brickinfo->hostname, old_brickinfo->path,
+            new_volinfo, &new_brickinfo);
+        /* If the brick is stale, i.e it's not a part of the new volume
+         * or if it's part of the new volume and is pending a snap or if it's
+         * brick multiplexing enabled, then stop the brick process
+         */
+        if (ret || (new_brickinfo->snap_status == -1) ||
+            GF_ATOMIC_GET(old_volinfo->volpeerupdate)) {
+            /*TODO: may need to switch to 'atomic' flavour of
+             * brick_stop, once we make peer rpc program also
+             * synctask enabled*/
+            ret = glusterd_brick_stop(old_volinfo, old_brickinfo, _gf_false);
+            if (ret)
+                gf_msg("glusterd", GF_LOG_ERROR, 0, GD_MSG_BRICK_STOP_FAIL,
+                       "Failed to stop"
+                       " brick %s:%s",
+                       old_brickinfo->hostname, old_brickinfo->path);
+        }
+    }
+    ret = 0;
 out:
-        gf_log ("", GF_LOG_DEBUG, "Returning with %d", ret);
-        return ret;
+    gf_msg_debug("glusterd", 0, "Returning with %d", ret);
+    return ret;
 }
 
 int32_t
-glusterd_delete_stale_volume (glusterd_volinfo_t *stale_volinfo,
-                              glusterd_volinfo_t *valid_volinfo)
-{
-        int32_t                  ret            = -1;
-        glusterd_volinfo_t      *temp_volinfo   = NULL;
-        glusterd_volinfo_t      *voliter        = NULL;
-        xlator_t                *this           = NULL;
-        glusterd_svc_t          *svc            = NULL;
-
-        GF_ASSERT (stale_volinfo);
-        GF_ASSERT (valid_volinfo);
-        this = THIS;
-        GF_ASSERT (this);
-
-        /* Copy snap_volumes list from stale_volinfo to valid_volinfo */
-        valid_volinfo->snap_count = 0;
-        cds_list_for_each_entry_safe (voliter, temp_volinfo,
-                                      &stale_volinfo->snap_volumes,
-                                      snapvol_list) {
-                cds_list_add_tail (&voliter->snapvol_list,
-                                   &valid_volinfo->snap_volumes);
-                valid_volinfo->snap_count++;
-        }
-
-        if ((!gf_uuid_is_null (stale_volinfo->restored_from_snap)) &&
-            (gf_uuid_compare (stale_volinfo->restored_from_snap,
-                           valid_volinfo->restored_from_snap))) {
-                ret = glusterd_lvm_snapshot_remove (NULL, stale_volinfo);
-                if (ret) {
-                        gf_log(this->name, GF_LOG_WARNING,
-                               "Failed to remove lvm snapshot for "
-                               "restored volume %s", stale_volinfo->volname);
-                }
+glusterd_delete_stale_volume(glusterd_volinfo_t *stale_volinfo,
+                             glusterd_volinfo_t *valid_volinfo)
+{
+    int32_t ret = -1;
+    glusterd_volinfo_t *temp_volinfo = NULL;
+    glusterd_volinfo_t *voliter = NULL;
+    xlator_t *this = NULL;
+    glusterd_svc_t *svc = NULL;
+
+    GF_ASSERT(stale_volinfo);
+    GF_ASSERT(valid_volinfo);
+    this = THIS;
+    GF_ASSERT(this);
+
+    /* Copy snap_volumes list from stale_volinfo to valid_volinfo */
+    valid_volinfo->snap_count = 0;
+    cds_list_for_each_entry_safe(voliter, temp_volinfo,
+                                 &stale_volinfo->snap_volumes, snapvol_list)
+    {
+        cds_list_add_tail(&voliter->snapvol_list, &valid_volinfo->snap_volumes);
+        valid_volinfo->snap_count++;
+    }
+
+    if ((!gf_uuid_is_null(stale_volinfo->restored_from_snap)) &&
+        (gf_uuid_compare(stale_volinfo->restored_from_snap,
+                         valid_volinfo->restored_from_snap))) {
+        ret = glusterd_lvm_snapshot_remove(NULL, stale_volinfo);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_WARNING, 0, GD_MSG_SNAP_REMOVE_FAIL,
+                   "Failed to remove lvm snapshot for "
+                   "restored volume %s",
+                   stale_volinfo->volname);
+        }
+    }
+
+    /* If stale volume is in started state, stop the stale bricks if the new
+     * volume is started else, stop all bricks.
+     * We don't want brick_rpc_notify to access already deleted brickinfo,
+     * so disconnect all bricks from stale_volinfo (unconditionally), since
+     * they are being deleted subsequently.
+     */
+    if (glusterd_is_volume_started(stale_volinfo)) {
+        if (glusterd_is_volume_started(valid_volinfo)) {
+            (void)glusterd_volinfo_stop_stale_bricks(valid_volinfo,
+                                                     stale_volinfo);
+
+        } else {
+            (void)glusterd_stop_bricks(stale_volinfo);
         }
 
-        /* If stale volume is in started state, copy the port numbers of the
-         * local bricks if they exist in the valid volume information.
-         * stop stale bricks. Stale volume information is going to be deleted.
-         * Which deletes the valid brick information inside stale volinfo.
-         * We dont want brick_rpc_notify to access already deleted brickinfo.
-         * Disconnect all bricks from stale_volinfo (unconditionally), since
-         * they are being deleted subsequently.
-         */
-        if (glusterd_is_volume_started (stale_volinfo)) {
-                if (glusterd_is_volume_started (valid_volinfo)) {
-                        (void) glusterd_volinfo_stop_stale_bricks (valid_volinfo,
-                                                                   stale_volinfo);
-                        //Only valid bricks will be running now.
-                        (void) glusterd_volinfo_copy_brick_portinfo (valid_volinfo,
-                                                                     stale_volinfo);
+        (void)glusterd_volume_disconnect_all_bricks(stale_volinfo);
+    }
+    /* Delete all the bricks and stores and vol files. They will be created
+     * again by the valid_volinfo. Volume store delete should not be
+     * performed because some of the bricks could still be running,
+     * keeping pid files under run directory
+     */
+    (void)glusterd_delete_all_bricks(stale_volinfo);
+    if (stale_volinfo->shandle) {
+        sys_unlink(stale_volinfo->shandle->path);
+        (void)gf_store_handle_destroy(stale_volinfo->shandle);
+        stale_volinfo->shandle = NULL;
+    }
 
-                } else {
-                        (void) glusterd_stop_bricks (stale_volinfo);
-                }
+    /* Marking volume as stopped, so that svc manager stops snapd
+     * and we are deleting the volume.
+     */
+    stale_volinfo->status = GLUSTERD_STATUS_STOPPED;
 
-                (void) glusterd_volume_disconnect_all_bricks (stale_volinfo);
-        }
-        /* Delete all the bricks and stores and vol files. They will be created
-         * again by the valid_volinfo. Volume store delete should not be
-         * performed because some of the bricks could still be running,
-         * keeping pid files under run directory
-         */
-        (void) glusterd_delete_all_bricks (stale_volinfo);
-        if (stale_volinfo->shandle) {
-                unlink (stale_volinfo->shandle->path);
-                (void) gf_store_handle_destroy (stale_volinfo->shandle);
-                stale_volinfo->shandle = NULL;
-        }
-        if (!stale_volinfo->is_snap_volume) {
-                svc = &(stale_volinfo->snapd.svc);
-                (void) svc->manager (svc, stale_volinfo, PROC_START_NO_WAIT);
-        }
-        (void) glusterd_volinfo_remove (stale_volinfo);
+    if (!stale_volinfo->is_snap_volume) {
+        svc = &(stale_volinfo->snapd.svc);
+        (void)svc->manager(svc, stale_volinfo, PROC_START_NO_WAIT);
+    }
+    svc = &(stale_volinfo->shd.svc);
+    (void)svc->manager(svc, stale_volinfo, PROC_START_NO_WAIT);
 
-        return 0;
+    (void)glusterd_volinfo_remove(stale_volinfo);
+
+    return 0;
 }
 
 /* This function updates the rebalance information of the new volinfo using the
  * information from the old volinfo.
  */
 int
-gd_check_and_update_rebalance_info (glusterd_volinfo_t *old_volinfo,
-                                    glusterd_volinfo_t *new_volinfo)
-{
-        int                  ret  = -1;
-        glusterd_rebalance_t *old = NULL;
-        glusterd_rebalance_t *new = NULL;
-
-        GF_ASSERT (old_volinfo);
-        GF_ASSERT (new_volinfo);
-
-        old = &(old_volinfo->rebal);
-        new = &(new_volinfo->rebal);
-
-        //Disconnect from rebalance process
-        if (old->defrag && old->defrag->rpc) {
-                rpc_transport_disconnect (old->defrag->rpc->conn.trans);
-        }
+gd_check_and_update_rebalance_info(glusterd_volinfo_t *old_volinfo,
+                                   glusterd_volinfo_t *new_volinfo)
+{
+    int ret = -1;
+    glusterd_rebalance_t *old = NULL;
+    glusterd_rebalance_t *new = NULL;
+
+    GF_ASSERT(old_volinfo);
+    GF_ASSERT(new_volinfo);
+
+    old = &(old_volinfo->rebal);
+    new = &(new_volinfo->rebal);
+
+    // Disconnect from rebalance process
+    if (glusterd_defrag_rpc_get(old->defrag)) {
+        rpc_transport_disconnect(old->defrag->rpc->conn.trans, _gf_false);
+        glusterd_defrag_rpc_put(old->defrag);
+    }
+
+    if (!gf_uuid_is_null(old->rebalance_id) &&
+        gf_uuid_compare(old->rebalance_id, new->rebalance_id)) {
+        (void)gd_stop_rebalance_process(old_volinfo);
+        goto out;
+    }
+
+    /* If the tasks match, copy the status and other information of the
+     * rebalance process from old_volinfo to new_volinfo
+     */
+    new->defrag_status = old->defrag_status;
+    new->rebalance_files = old->rebalance_files;
+    new->rebalance_data = old->rebalance_data;
+    new->lookedup_files = old->lookedup_files;
+    new->skipped_files = old->skipped_files;
+    new->rebalance_failures = old->rebalance_failures;
+    new->rebalance_time = old->rebalance_time;
+
+    /* glusterd_rebalance_t.{op, id, defrag_cmd} are copied during volume
+     * import a new defrag object should come to life with rebalance being
+     * restarted
+     */
+out:
+    return ret;
+}
 
-        if (!gf_uuid_is_null (old->rebalance_id) &&
-            gf_uuid_compare (old->rebalance_id, new->rebalance_id)) {
-                (void)gd_stop_rebalance_process (old_volinfo);
-                goto out;
+static int32_t
+glusterd_import_friend_volume(dict_t *peer_data, int count)
+{
+    int32_t ret = -1;
+    glusterd_conf_t *priv = NULL;
+    xlator_t *this = NULL;
+    glusterd_volinfo_t *old_volinfo = NULL;
+    glusterd_volinfo_t *new_volinfo = NULL;
+    glusterd_svc_t *svc = NULL;
+    int32_t update = 0;
+    char key[64] = "";
+
+    GF_ASSERT(peer_data);
+
+    this = THIS;
+    GF_ASSERT(this);
+    priv = this->private;
+    GF_ASSERT(priv);
+
+    ret = snprintf(key, sizeof(key), "volume%d.update", count);
+    ret = dict_get_int32n(peer_data, key, ret, &update);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_GET_FAILED,
+                "Key=%s", key, NULL);
+        goto out;
+    }
+
+    if (!update) {
+        /* if update is 0 that means the volume is not imported */
+        gf_smsg(this->name, GF_LOG_INFO, 0, GD_MSG_VOLUME_NOT_IMPORTED, NULL);
+        goto out;
+    }
+
+    ret = glusterd_import_volinfo(peer_data, count, &new_volinfo, "volume");
+    if (ret)
+        goto out;
+
+    if (!new_volinfo) {
+        gf_msg_debug(this->name, 0, "Not importing snap volume");
+        goto out;
+    }
+
+    ret = glusterd_volinfo_find(new_volinfo->volname, &old_volinfo);
+    if (0 == ret) {
+        if (new_volinfo->version <= old_volinfo->version) {
+            /* When this condition is true, it already means that
+             * the other synctask thread of import volume has
+             * already up to date volume, so just ignore this volume
+             * now
+             */
+            goto out;
         }
-
-        /* If the tasks match, copy the status and other information of the
-         * rebalance process from old_volinfo to new_volinfo
+        /* Ref count the old_volinfo such that deleting it doesn't crash
+         * if its been already in use by other thread
          */
-        new->defrag_status      = old->defrag_status;
-        new->rebalance_files    = old->rebalance_files;
-        new->rebalance_data     = old->rebalance_data;
-        new->lookedup_files     = old->lookedup_files;
-        new->skipped_files      = old->skipped_files;
-        new->rebalance_failures = old->rebalance_failures;
-        new->rebalance_time     = old->rebalance_time;
-        new->dict               = (old->dict ? dict_ref (old->dict) : NULL);
-
-        /* glusterd_rebalance_t.{op, id, defrag_cmd} are copied during volume
-         * import
-         * a new defrag object should come to life with rebalance being restarted
+        glusterd_volinfo_ref(old_volinfo);
+        (void)gd_check_and_update_rebalance_info(old_volinfo, new_volinfo);
+
+        /* Copy brick ports & real_path from the old volinfo always.
+         * The old_volinfo will be cleaned up and this information
+         * could be lost
          */
+        (void)glusterd_volinfo_copy_brickinfo(old_volinfo, new_volinfo);
+
+        (void)glusterd_delete_stale_volume(old_volinfo, new_volinfo);
+        glusterd_volinfo_unref(old_volinfo);
+    }
+
+    ret = glusterd_store_volinfo(new_volinfo, GLUSTERD_VOLINFO_VER_AC_NONE);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_VOLINFO_STORE_FAIL,
+               "Failed to store "
+               "volinfo for volume %s",
+               new_volinfo->volname);
+        goto out;
+    }
+
+    ret = glusterd_create_volfiles(new_volinfo);
+    if (ret)
+        goto out;
+
+    glusterd_list_add_order(&new_volinfo->vol_list, &priv->volumes,
+                            glusterd_compare_volume_name);
+
+    if (glusterd_is_volume_started(new_volinfo)) {
+        (void)glusterd_start_bricks(new_volinfo);
+        if (glusterd_is_snapd_enabled(new_volinfo)) {
+            svc = &(new_volinfo->snapd.svc);
+            if (svc->manager(svc, new_volinfo, PROC_START_NO_WAIT)) {
+                gf_event(EVENT_SVC_MANAGER_FAILED, "svc_name=%s", svc->name);
+            }
+        }
+        svc = &(new_volinfo->shd.svc);
+        if (svc->manager(svc, new_volinfo, PROC_START_NO_WAIT)) {
+            gf_event(EVENT_SVC_MANAGER_FAILED, "svc_name=%s", svc->name);
+        }
+    }
+
+    ret = glusterd_import_quota_conf(peer_data, count, new_volinfo, "volume");
+    if (ret) {
+        gf_event(EVENT_IMPORT_QUOTA_CONF_FAILED, "volume=%s",
+                 new_volinfo->volname);
+        goto out;
+    }
+
+    ret = glusterd_fetchspec_notify(this);
 out:
-        return ret;
+    gf_msg_debug("glusterd", 0, "Returning with ret: %d", ret);
+    return ret;
 }
 
 int32_t
-glusterd_import_friend_volume (dict_t *peer_data, size_t count)
-{
-
-        int32_t                 ret = -1;
-        glusterd_conf_t         *priv = NULL;
-        xlator_t                *this = NULL;
-        glusterd_volinfo_t      *old_volinfo = NULL;
-        glusterd_volinfo_t      *new_volinfo = NULL;
-        glusterd_svc_t          *svc         = NULL;
-
-        GF_ASSERT (peer_data);
-
-        this = THIS;
-        GF_ASSERT (this);
-        priv = this->private;
-        GF_ASSERT (priv);
-        ret = glusterd_import_volinfo (peer_data, count,
-                                       &new_volinfo, "volume");
-        if (ret)
-                goto out;
-
-        if (!new_volinfo) {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "Not importing snap volume");
-                goto out;
-        }
-
-        ret = glusterd_snapdsvc_init (new_volinfo);
+glusterd_import_friend_volumes_synctask(void *opaque)
+{
+    int32_t ret = -1;
+    int32_t count = 0;
+    int i = 1;
+    xlator_t *this = NULL;
+    glusterd_conf_t *conf = NULL;
+    dict_t *peer_data = NULL;
+    glusterd_friend_synctask_args_t *arg = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+
+    conf = this->private;
+    GF_ASSERT(conf);
+
+    arg = opaque;
+    if (!arg)
+        goto out;
+
+    peer_data = dict_new();
+    if (!peer_data) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_CREATE_FAIL, NULL);
+        goto out;
+    }
+
+    ret = dict_unserialize(arg->dict_buf, arg->dictlen, &peer_data);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_UNSERIALIZE_FAIL,
+                NULL);
+        errno = ENOMEM;
+        goto out;
+    }
+
+    ret = dict_get_int32n(peer_data, "count", SLEN("count"), &count);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_GET_FAILED,
+                "Key=count", NULL);
+        goto out;
+    }
+
+    synclock_lock(&conf->big_lock);
+
+    /* We need to ensure that importing a volume shouldn't race with an
+     * other thread where as part of restarting glusterd, bricks are
+     * restarted (refer glusterd_restart_bricks ())
+     */
+    while (conf->restart_bricks) {
+        synccond_wait(&conf->cond_restart_bricks, &conf->big_lock);
+    }
+    conf->restart_bricks = _gf_true;
+
+    while (i <= count) {
+        ret = glusterd_import_friend_volume(peer_data, i);
         if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "Failed to initialize "
-                        "snapdsvc for volume %s", new_volinfo->volname);
-                goto out;
+            break;
         }
-
-        ret = glusterd_volinfo_find (new_volinfo->volname, &old_volinfo);
-        if (0 == ret) {
-                /* snapdsvc initialization of old_volinfo is also required here
-                 * as glusterd_delete_stale_volume () invokes snapdsvc manager
-                 */
-                ret = glusterd_snapdsvc_init (old_volinfo);
-                if (ret) {
-                        gf_log (this->name, GF_LOG_ERROR, "Failed to initialize"
-                                " snapdsvc for old volume %s",
-                                old_volinfo->volname);
-                        goto out;
-                }
-                (void) gd_check_and_update_rebalance_info (old_volinfo,
-                                                           new_volinfo);
-                (void) glusterd_delete_stale_volume (old_volinfo, new_volinfo);
-        }
-
-        if (glusterd_is_volume_started (new_volinfo)) {
-                (void) glusterd_start_bricks (new_volinfo);
-                if (glusterd_is_snapd_enabled (new_volinfo)) {
-                        svc = &(new_volinfo->snapd.svc);
-                        (void) svc->start (svc, PROC_START_NO_WAIT);
-                }
-        }
-
-        ret = glusterd_store_volinfo (new_volinfo, GLUSTERD_VOLINFO_VER_AC_NONE);
-        ret = glusterd_create_volfiles_and_notify_services (new_volinfo);
-        if (ret)
-                goto out;
-
-        ret = glusterd_import_quota_conf (peer_data, count,
-                                          new_volinfo, "volume");
-        if (ret)
-                goto out;
-
-        glusterd_list_add_order (&new_volinfo->vol_list, &priv->volumes,
-                                 glusterd_compare_volume_name);
-
+        i++;
+    }
+    if (i > count) {
+        glusterd_svcs_manager(NULL);
+    }
+    conf->restart_bricks = _gf_false;
+    synccond_broadcast(&conf->cond_restart_bricks);
 out:
-        gf_log ("", GF_LOG_DEBUG, "Returning with ret: %d", ret);
-        return ret;
+    if (peer_data)
+        dict_unref(peer_data);
+    if (arg) {
+        if (arg->dict_buf)
+            GF_FREE(arg->dict_buf);
+        GF_FREE(arg);
+    }
+
+    gf_msg_debug("glusterd", 0, "Returning with %d", ret);
+    return ret;
 }
 
 int32_t
-glusterd_import_friend_volumes (dict_t *peer_data)
+glusterd_import_friend_volumes(dict_t *peer_data)
 {
-        int32_t                 ret = -1;
-        int32_t                 count = 0;
-        int                     i = 1;
+    int32_t ret = -1;
+    int32_t count = 0;
+    int i = 1;
 
-        GF_ASSERT (peer_data);
+    GF_ASSERT(peer_data);
 
-        ret = dict_get_int32 (peer_data, "count", &count);
-        if (ret)
-                goto out;
+    ret = dict_get_int32n(peer_data, "count", SLEN("count"), &count);
+    if (ret) {
+        gf_smsg("glusterd", GF_LOG_ERROR, errno, GD_MSG_DICT_GET_FAILED,
+                "Key=count", NULL);
+        goto out;
+    }
 
-        while (i <= count) {
-                ret = glusterd_import_friend_volume (peer_data, i);
-                if (ret)
-                        goto out;
-                i++;
-        }
+    while (i <= count) {
+        ret = glusterd_import_friend_volume(peer_data, i);
+        if (ret)
+            goto out;
+        i++;
+    }
 
 out:
-        gf_log ("", GF_LOG_DEBUG, "Returning with %d", ret);
-        return ret;
+    gf_msg_debug("glusterd", 0, "Returning with %d", ret);
+    return ret;
 }
 
 int
-glusterd_get_global_opt_version (dict_t *opts, uint32_t *version)
-{
-        int     ret = -1;
-        char    *version_str = NULL;
-
-        ret = dict_get_str (opts, GLUSTERD_GLOBAL_OPT_VERSION, &version_str);
-        if (ret)
-                goto out;
-
-        ret = gf_string2uint (version_str, version);
-        if (ret)
-                goto out;
-        ret = 0;
+glusterd_get_global_server_quorum_ratio(dict_t *opts, double *quorum)
+{
+    int ret = -1;
+    char *quorum_str = NULL;
+    xlator_t *this = THIS;
+    GF_ASSERT(this);
+
+    ret = dict_get_strn(opts, GLUSTERD_QUORUM_RATIO_KEY,
+                        SLEN(GLUSTERD_QUORUM_RATIO_KEY), &quorum_str);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_GET_FAILED,
+                "Key=%s", GLUSTERD_QUORUM_RATIO_KEY, NULL);
+        goto out;
+    }
+
+    ret = gf_string2percent(quorum_str, quorum);
+    if (ret)
+        goto out;
+    ret = 0;
 out:
-        return ret;
+    return ret;
 }
 
 int
-glusterd_get_next_global_opt_version_str (dict_t *opts, char **version_str)
-{
-        int             ret = -1;
-        char            version_string[64] = {0};
-        uint32_t        version = 0;
+glusterd_get_global_opt_version(dict_t *opts, uint32_t *version)
+{
+    int ret = -1;
+    char *version_str = NULL;
+    xlator_t *this = THIS;
+    GF_ASSERT(this);
+
+    ret = dict_get_strn(opts, GLUSTERD_GLOBAL_OPT_VERSION,
+                        SLEN(GLUSTERD_GLOBAL_OPT_VERSION), &version_str);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_GET_FAILED,
+                "Key=%s", GLUSTERD_GLOBAL_OPT_VERSION, NULL);
+        goto out;
+    }
+
+    ret = gf_string2uint(version_str, version);
+    if (ret)
+        goto out;
+    ret = 0;
+out:
+    return ret;
+}
 
-        ret = glusterd_get_global_opt_version (opts, &version);
-        if (ret)
-                goto out;
-        version++;
-        snprintf (version_string, sizeof (version_string), "%"PRIu32, version);
-        *version_str = gf_strdup (version_string);
-        if (*version_str)
-                ret = 0;
+int
+glusterd_get_next_global_opt_version_str(dict_t *opts, char **version_str)
+{
+    int ret = -1;
+    char version_string[64] = "";
+    uint32_t version = 0;
+
+    ret = glusterd_get_global_opt_version(opts, &version);
+    if (ret)
+        goto out;
+    version++;
+    snprintf(version_string, sizeof(version_string), "%" PRIu32, version);
+    *version_str = gf_strdup(version_string);
+    if (*version_str)
+        ret = 0;
 out:
-        return ret;
+    return ret;
 }
 
 int32_t
-glusterd_import_global_opts (dict_t *friend_data)
-{
-        xlator_t        *this = NULL;
-        glusterd_conf_t *conf = NULL;
-        int             ret = -1;
-        dict_t          *import_options = NULL;
-        int             count = 0;
-        uint32_t        local_version = 0;
-        uint32_t        remote_version = 0;
-
-        this = THIS;
-        conf = this->private;
-
-        ret = dict_get_int32 (friend_data, "global-opt-count", &count);
-        if (ret) {
-                //old version peer
-                ret = 0;
-                goto out;
-        }
+glusterd_import_global_opts(dict_t *friend_data)
+{
+    xlator_t *this = NULL;
+    glusterd_conf_t *conf = NULL;
+    int ret = -1;
+    dict_t *import_options = NULL;
+    int count = 0;
+    uint32_t local_version = 0;
+    uint32_t remote_version = 0;
+    double old_quorum = 0.0;
+    double new_quorum = 0.0;
+
+    this = THIS;
+    conf = this->private;
+
+    ret = dict_get_int32n(friend_data, "global-opt-count",
+                          SLEN("global-opt-count"), &count);
+    if (ret) {
+        // old version peer
+        gf_smsg(this->name, GF_LOG_INFO, errno, GD_MSG_DICT_GET_FAILED,
+                "Key=global-opt-count", NULL);
+        ret = 0;
+        goto out;
+    }
+
+    import_options = dict_new();
+    if (!import_options) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_CREATE_FAIL, NULL);
+        goto out;
+    }
+    ret = import_prdict_dict(friend_data, import_options, "key", "val", count,
+                             "global");
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_GLOBAL_OPT_IMPORT_FAIL,
+               "Failed to import"
+               " global options");
+        goto out;
+    }
+
+    /* Not handling ret since server-quorum-ratio might not yet be set */
+    ret = glusterd_get_global_server_quorum_ratio(conf->opts, &old_quorum);
+    ret = glusterd_get_global_server_quorum_ratio(import_options, &new_quorum);
+
+    ret = glusterd_get_global_opt_version(conf->opts, &local_version);
+    if (ret)
+        goto out;
+    ret = glusterd_get_global_opt_version(import_options, &remote_version);
+    if (ret)
+        goto out;
+
+    if (remote_version > local_version) {
+        ret = glusterd_store_options(this, import_options);
+        if (ret)
+            goto out;
+        dict_unref(conf->opts);
+        conf->opts = dict_ref(import_options);
 
-        import_options = dict_new ();
-        if (!import_options)
-                goto out;
-        ret = import_prdict_dict (friend_data, import_options, "key", "val",
-                                  count, "global");
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "Failed to import"
-                        " global options");
-                goto out;
+        /* If server quorum ratio has changed, restart bricks to
+         * recompute if quorum is met. If quorum is not met bricks are
+         * not started and those already running are stopped
+         */
+        if (old_quorum != new_quorum) {
+            glusterd_launch_synctask(glusterd_restart_bricks, NULL);
         }
+    }
 
-        ret = glusterd_get_global_opt_version (conf->opts, &local_version);
-        if (ret)
-                goto out;
-        ret = glusterd_get_global_opt_version (import_options, &remote_version);
-        if (ret)
-                goto out;
-        if (remote_version > local_version) {
-                ret = glusterd_store_options (this, import_options);
-                if (ret)
-                        goto out;
-                dict_unref (conf->opts);
-                conf->opts = dict_ref (import_options);
-        }
-        ret = 0;
+    ret = 0;
 out:
-        if (import_options)
-                dict_unref (import_options);
-        return ret;
+    if (import_options)
+        dict_unref(import_options);
+    return ret;
 }
 
 int32_t
-glusterd_compare_friend_data (dict_t *peer_data, int32_t *status,
-                              char *hostname)
-{
-        int32_t          ret       = -1;
-        int32_t          count     = 0;
-        int              i         = 1;
-        gf_boolean_t     update    = _gf_false;
-        gf_boolean_t     stale_nfs = _gf_false;
-        gf_boolean_t     stale_shd = _gf_false;
-        gf_boolean_t     stale_qd  = _gf_false;
-        xlator_t        *this      = NULL;
-        glusterd_conf_t *priv      = NULL;
-
-        this = THIS;
-        GF_ASSERT (this);
-        GF_ASSERT (peer_data);
-        GF_ASSERT (status);
-
-        priv = this->private;
-        GF_ASSERT (priv);
-
-        ret = glusterd_import_global_opts (peer_data);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "Importing global "
-                        "options failed");
-                goto out;
-        }
-
-        ret = dict_get_int32 (peer_data, "count", &count);
+glusterd_compare_friend_data(dict_t *peer_data, int32_t *status, char *hostname)
+{
+    int32_t ret = -1;
+    int32_t count = 0;
+    int i = 1;
+    gf_boolean_t update = _gf_false;
+    xlator_t *this = NULL;
+    glusterd_conf_t *priv = NULL;
+    glusterd_friend_synctask_args_t *arg = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+    GF_ASSERT(peer_data);
+    GF_ASSERT(status);
+
+    priv = this->private;
+    GF_ASSERT(priv);
+    ret = glusterd_import_global_opts(peer_data);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_GLOBAL_OPT_IMPORT_FAIL,
+               "Importing global "
+               "options failed");
+        goto out;
+    }
+
+    ret = dict_get_int32n(peer_data, "count", SLEN("count"), &count);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_GET_FAILED,
+                "Key=count", NULL);
+        goto out;
+    }
+
+    while (i <= count) {
+        ret = glusterd_compare_friend_volume(peer_data, i, status, hostname);
         if (ret)
-                goto out;
-
-        while (i <= count) {
-                ret = glusterd_compare_friend_volume (peer_data, i, status,
-                                                      hostname);
-                if (ret)
-                        goto out;
-
-                if (GLUSTERD_VOL_COMP_RJT == *status) {
-                        ret = 0;
-                        goto out;
-                }
-                if (GLUSTERD_VOL_COMP_UPDATE_REQ == *status)
-                        update = _gf_true;
+            goto out;
 
-                i++;
+        if (GLUSTERD_VOL_COMP_RJT == *status) {
+            ret = 0;
+            goto out;
         }
-
-        if (update) {
-                if (glusterd_proc_is_running (&(priv->nfs_svc.proc)))
-                        stale_nfs = _gf_true;
-                if (glusterd_proc_is_running (&(priv->shd_svc.proc)))
-                        stale_shd = _gf_true;
-                if (glusterd_proc_is_running (&(priv->quotad_svc.proc)))
-                        stale_qd  = _gf_true;
-                ret = glusterd_import_friend_volumes (peer_data);
-                if (ret)
-                        goto out;
-                if (_gf_false == glusterd_are_all_volumes_stopped ()) {
-                        ret = glusterd_svcs_manager (NULL);
-                } else {
-                        if (stale_nfs)
-                                priv->nfs_svc.stop (&(priv->nfs_svc), SIGKILL);
-                        if (stale_shd)
-                                priv->shd_svc.stop (&(priv->shd_svc), SIGTERM);
-                        if (stale_qd)
-                                priv->quotad_svc.stop (&(priv->quotad_svc),
-                                                       SIGTERM);
-                }
+        if (GLUSTERD_VOL_COMP_UPDATE_REQ == *status) {
+            update = _gf_true;
         }
+        i++;
+    }
 
-out:
-        gf_log (this->name, GF_LOG_DEBUG,
-                "Returning with ret: %d, status: %d", ret, *status);
-        return ret;
-}
-
-struct rpc_clnt*
-glusterd_pending_node_get_rpc (glusterd_pending_node_t *pending_node)
-{
-        struct rpc_clnt *rpc = NULL;
-        glusterd_brickinfo_t    *brickinfo = NULL;
-        glusterd_volinfo_t      *volinfo   = NULL;
-        glusterd_svc_t          *svc       = NULL;
-
-        GF_VALIDATE_OR_GOTO (THIS->name, pending_node, out);
-        GF_VALIDATE_OR_GOTO (THIS->name, pending_node->node, out);
-
-        if (pending_node->type == GD_NODE_BRICK) {
-                brickinfo = pending_node->node;
-                rpc       = brickinfo->rpc;
-
-        } else if (pending_node->type == GD_NODE_SHD ||
-                   pending_node->type == GD_NODE_NFS ||
-                   pending_node->type == GD_NODE_QUOTAD) {
-                svc = pending_node->node;
-                rpc = svc->conn.rpc;
-        } else if (pending_node->type == GD_NODE_REBALANCE) {
-                volinfo = pending_node->node;
-                if (volinfo->rebal.defrag)
-                        rpc = volinfo->rebal.defrag->rpc;
-        } else if (pending_node->type == GD_NODE_SNAPD) {
-                volinfo = pending_node->node;
-                rpc = volinfo->snapd.svc.conn.rpc;
-        } else {
-                GF_ASSERT (0);
+    if (update) {
+        /* Launch the import friend volume as a separate synctask as it
+         * has to trigger start bricks where we may need to wait for the
+         * first brick to come up before attaching the subsequent bricks
+         * in case brick multiplexing is enabled
+         */
+        arg = GF_CALLOC(1, sizeof(*arg), gf_common_mt_char);
+        ret = dict_allocate_and_serialize(peer_data, &arg->dict_buf,
+                                          &arg->dictlen);
+        if (ret < 0) {
+            gf_log(this->name, GF_LOG_ERROR,
+                   "dict_serialize failed while handling "
+                   " import friend volume request");
+            goto out;
         }
 
+        glusterd_launch_synctask(glusterd_import_friend_volumes_synctask, arg);
+    }
+
 out:
-        return rpc;
+    if (ret && arg) {
+        GF_FREE(arg);
+    }
+    gf_msg_debug(this->name, 0, "Returning with ret: %d, status: %d", ret,
+                 *status);
+    return ret;
 }
 
-int32_t
-glusterd_unlink_file (char *sockfpath)
+struct rpc_clnt *
+glusterd_defrag_rpc_get(glusterd_defrag_info_t *defrag)
 {
-        int             ret = 0;
+    struct rpc_clnt *rpc = NULL;
 
-        ret = unlink (sockfpath);
-        if (ret) {
-                if (ENOENT == errno)
-                        ret = 0;
-                else
-                        gf_log (THIS->name, GF_LOG_ERROR, "Failed to remove %s"
-                                " error: %s", sockfpath, strerror (errno));
-        }
+    if (!defrag)
+        return NULL;
 
-        return ret;
+    LOCK(&defrag->lock);
+    {
+        rpc = rpc_clnt_ref(defrag->rpc);
+    }
+    UNLOCK(&defrag->lock);
+    return rpc;
 }
 
-void
-glusterd_nfs_pmap_deregister ()
+struct rpc_clnt *
+glusterd_defrag_rpc_put(glusterd_defrag_info_t *defrag)
 {
-        if (pmap_unset (MOUNT_PROGRAM, MOUNTV3_VERSION))
-                gf_log ("", GF_LOG_INFO, "De-registered MOUNTV3 successfully");
-        else
-                gf_log ("", GF_LOG_ERROR, "De-register MOUNTV3 is unsuccessful");
+    struct rpc_clnt *rpc = NULL;
 
-        if (pmap_unset (MOUNT_PROGRAM, MOUNTV1_VERSION))
-                gf_log ("", GF_LOG_INFO, "De-registered MOUNTV1 successfully");
-        else
-                gf_log ("", GF_LOG_ERROR, "De-register MOUNTV1 is unsuccessful");
-
-        if (pmap_unset (NFS_PROGRAM, NFSV3_VERSION))
-                gf_log ("", GF_LOG_INFO, "De-registered NFSV3 successfully");
-        else
-                gf_log ("", GF_LOG_ERROR, "De-register NFSV3 is unsuccessful");
-
-        if (pmap_unset (NLM_PROGRAM, NLMV4_VERSION))
-                gf_log ("", GF_LOG_INFO, "De-registered NLM v4 successfully");
-        else
-                gf_log ("", GF_LOG_ERROR, "De-registration of NLM v4 failed");
+    if (!defrag)
+        return NULL;
 
-        if (pmap_unset (NLM_PROGRAM, NLMV1_VERSION))
-                gf_log ("", GF_LOG_INFO, "De-registered NLM v1 successfully");
-        else
-                gf_log ("", GF_LOG_ERROR, "De-registration of NLM v1 failed");
+    LOCK(&defrag->lock);
+    {
+        rpc = rpc_clnt_unref(defrag->rpc);
+        defrag->rpc = rpc;
+    }
+    UNLOCK(&defrag->lock);
+    return rpc;
+}
+
+struct rpc_clnt *
+glusterd_pending_node_get_rpc(glusterd_pending_node_t *pending_node)
+{
+    struct rpc_clnt *rpc = NULL;
+    glusterd_brickinfo_t *brickinfo = NULL;
+    glusterd_volinfo_t *volinfo = NULL;
+    glusterd_svc_t *svc = NULL;
+
+    GF_VALIDATE_OR_GOTO(THIS->name, pending_node, out);
+    GF_VALIDATE_OR_GOTO(THIS->name, pending_node->node, out);
+
+    if (pending_node->type == GD_NODE_BRICK) {
+        brickinfo = pending_node->node;
+        rpc = brickinfo->rpc;
+
+    } else if (pending_node->type == GD_NODE_SHD ||
+               pending_node->type == GD_NODE_NFS ||
+               pending_node->type == GD_NODE_QUOTAD ||
+               pending_node->type == GD_NODE_SCRUB) {
+        svc = pending_node->node;
+        rpc = svc->conn.rpc;
+    } else if (pending_node->type == GD_NODE_REBALANCE) {
+        volinfo = pending_node->node;
+        rpc = glusterd_defrag_rpc_get(volinfo->rebal.defrag);
+
+    } else if (pending_node->type == GD_NODE_SNAPD) {
+        volinfo = pending_node->node;
+        rpc = volinfo->snapd.svc.conn.rpc;
+    } else {
+        GF_ASSERT(0);
+    }
 
-        if (pmap_unset (ACL_PROGRAM, ACLV3_VERSION))
-                gf_log ("", GF_LOG_INFO, "De-registered ACL v3 successfully");
-        else
-                gf_log ("", GF_LOG_ERROR, "De-registration of ACL v3 failed");
+out:
+    return rpc;
 }
 
-int
-glusterd_add_node_to_dict (char *server, dict_t *dict, int count,
-                           dict_t *vol_opts)
+void
+glusterd_pending_node_put_rpc(glusterd_pending_node_t *pending_node)
 {
-        int                     ret               = -1;
-        char                    pidfile[PATH_MAX] = {0,};
-        gf_boolean_t            running           = _gf_false;
-        int                     pid               = -1;
-        int                     port              = 0;
-        glusterd_svc_t         *svc               = NULL;
-        char                    key[1024]         = {0,};
-        xlator_t               *this              = NULL;
-        glusterd_conf_t        *priv              = NULL;
-
-        this = THIS;
-        GF_ASSERT (this);
-
-        priv = this->private;
-        GF_ASSERT (priv);
+    glusterd_volinfo_t *volinfo = NULL;
 
-        glusterd_svc_build_pidfile_path (server, priv->workdir, pidfile,
-                                         sizeof (pidfile));
+    switch (pending_node->type) {
+        case GD_NODE_REBALANCE:
+            volinfo = pending_node->node;
+            glusterd_defrag_rpc_put(volinfo->rebal.defrag);
+            break;
 
-        if (strcmp(server, priv->shd_svc.name) == 0)
-                svc = &(priv->shd_svc);
-        else if (strcmp(server, priv->nfs_svc.name) == 0)
-                svc = &(priv->nfs_svc);
-        else if (strcmp(server, priv->quotad_svc.name) == 0)
-                svc = &(priv->quotad_svc);
+        default:
+            break;
+    }
+}
 
-        //Consider service to be running only when glusterd sees it Online
-        if (svc->online)
-                running = gf_is_service_running (pidfile, &pid);
+int32_t
+glusterd_unlink_file(char *sockfpath)
+{
+    int ret = 0;
 
-        /* For nfs-servers/self-heal-daemon setting
-         * brick<n>.hostname = "NFS Server" / "Self-heal Daemon"
-         * brick<n>.path = uuid
-         * brick<n>.port = 0
-         *
-         * This might be confusing, but cli displays the name of
-         * the brick as hostname+path, so this will make more sense
-         * when output.
-         */
-        snprintf (key, sizeof (key), "brick%d.hostname", count);
-        if (!strcmp (server, priv->nfs_svc.name))
-                ret = dict_set_str (dict, key, "NFS Server");
-        else if (!strcmp (server, priv->shd_svc.name))
-                ret = dict_set_str (dict, key, "Self-heal Daemon");
-        else if (!strcmp (server, priv->quotad_svc.name))
-                ret = dict_set_str (dict, key, "Quota Daemon");
-        if (ret)
-                goto out;
+    ret = sys_unlink(sockfpath);
+    if (ret) {
+        if (ENOENT == errno)
+            ret = 0;
+        else
+            gf_msg(THIS->name, GF_LOG_ERROR, errno, GD_MSG_FILE_OP_FAILED,
+                   "Failed to remove %s"
+                   " error: %s",
+                   sockfpath, strerror(errno));
+    }
 
-        memset (key, 0, sizeof (key));
-        snprintf (key, sizeof (key), "brick%d.path", count);
-        ret = dict_set_dynstr (dict, key, gf_strdup (uuid_utoa (MY_UUID)));
-        if (ret)
-                goto out;
+    return ret;
+}
 
-        memset (key, 0, sizeof (key));
-        snprintf (key, sizeof (key), "brick%d.port", count);
-        /* Port is available only for the NFS server.
-         * Self-heal daemon doesn't provide any port for access
-         * by entities other than gluster.
-         */
-        if (!strcmp (server, priv->nfs_svc.name)) {
-                if (dict_get (vol_opts, "nfs.port")) {
-                        ret = dict_get_int32 (vol_opts, "nfs.port", &port);
-                        if (ret)
-                                goto out;
-                } else
-                        port = GF_NFS3_PORT;
-        }
-        ret = dict_set_int32 (dict, key, port);
-        if (ret)
-                goto out;
+void
+glusterd_nfs_pmap_deregister()
+{
+    if (pmap_unset(MOUNT_PROGRAM, MOUNTV3_VERSION))
+        gf_msg("glusterd", GF_LOG_INFO, 0, GD_MSG_DEREGISTER_SUCCESS,
+               "De-registered MOUNTV3 successfully");
+    else
+        gf_msg("glusterd", GF_LOG_ERROR, 0, GD_MSG_PMAP_UNSET_FAIL,
+               "De-register MOUNTV3 is unsuccessful");
+
+    if (pmap_unset(MOUNT_PROGRAM, MOUNTV1_VERSION))
+        gf_msg("glusterd", GF_LOG_INFO, 0, GD_MSG_DEREGISTER_SUCCESS,
+               "De-registered MOUNTV1 successfully");
+    else
+        gf_msg("glusterd", GF_LOG_ERROR, 0, GD_MSG_PMAP_UNSET_FAIL,
+               "De-register MOUNTV1 is unsuccessful");
+
+    if (pmap_unset(NFS_PROGRAM, NFSV3_VERSION))
+        gf_msg("glusterd", GF_LOG_INFO, 0, GD_MSG_DEREGISTER_SUCCESS,
+               "De-registered NFSV3 successfully");
+    else
+        gf_msg("glusterd", GF_LOG_ERROR, 0, GD_MSG_PMAP_UNSET_FAIL,
+               "De-register NFSV3 is unsuccessful");
+
+    if (pmap_unset(NLM_PROGRAM, NLMV4_VERSION))
+        gf_msg("glusterd", GF_LOG_INFO, 0, GD_MSG_DEREGISTER_SUCCESS,
+               "De-registered NLM v4 successfully");
+    else
+        gf_msg("glusterd", GF_LOG_ERROR, 0, GD_MSG_PMAP_UNSET_FAIL,
+               "De-registration of NLM v4 failed");
+
+    if (pmap_unset(NLM_PROGRAM, NLMV1_VERSION))
+        gf_msg("glusterd", GF_LOG_INFO, 0, GD_MSG_DEREGISTER_SUCCESS,
+               "De-registered NLM v1 successfully");
+    else
+        gf_msg("glusterd", GF_LOG_ERROR, 0, GD_MSG_PMAP_UNSET_FAIL,
+               "De-registration of NLM v1 failed");
+
+    if (pmap_unset(ACL_PROGRAM, ACLV3_VERSION))
+        gf_msg("glusterd", GF_LOG_INFO, 0, GD_MSG_DEREGISTER_SUCCESS,
+               "De-registered ACL v3 successfully");
+    else
+        gf_msg("glusterd", GF_LOG_ERROR, 0, GD_MSG_PMAP_UNSET_FAIL,
+               "De-registration of ACL v3 failed");
+}
 
-        memset (key, 0, sizeof (key));
-        snprintf (key, sizeof (key), "brick%d.pid", count);
-        ret = dict_set_int32 (dict, key, pid);
-        if (ret)
-                goto out;
+int
+glusterd_add_node_to_dict(char *server, dict_t *dict, int count,
+                          dict_t *vol_opts)
+{
+    int ret = -1;
+    char pidfile[PATH_MAX] = "";
+    gf_boolean_t running = _gf_false;
+    int pid = -1;
+    int port = 0;
+    glusterd_svc_t *svc = NULL;
+    char key[64] = "";
+    int keylen;
+    xlator_t *this = NULL;
+    glusterd_conf_t *priv = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+
+    priv = this->private;
+    GF_ASSERT(priv);
+
+    if (!strcmp(server, "")) {
+        ret = 0;
+        goto out;
+    }
 
-        memset (key, 0, sizeof (key));
-        snprintf (key, sizeof (key), "brick%d.status", count);
-        ret = dict_set_int32 (dict, key, running);
-        if (ret)
-                goto out;
+    glusterd_svc_build_pidfile_path(server, priv->rundir, pidfile,
+                                    sizeof(pidfile));
 
+    if (strcmp(server, priv->quotad_svc.name) == 0)
+        svc = &(priv->quotad_svc);
+#ifdef BUILD_GNFS
+    else if (strcmp(server, priv->nfs_svc.name) == 0)
+        svc = &(priv->nfs_svc);
+#endif
+    else if (strcmp(server, priv->bitd_svc.name) == 0)
+        svc = &(priv->bitd_svc);
+    else if (strcmp(server, priv->scrub_svc.name) == 0)
+        svc = &(priv->scrub_svc);
+    else {
+        ret = 0;
+        goto out;
+    }
+
+    // Consider service to be running only when glusterd sees it Online
+    if (svc->online)
+        running = gf_is_service_running(pidfile, &pid);
+
+    /* For nfs-servers/self-heal-daemon setting
+     * brick<n>.hostname = "NFS Server" / "Self-heal Daemon"
+     * brick<n>.path = uuid
+     * brick<n>.port = 0
+     *
+     * This might be confusing, but cli displays the name of
+     * the brick as hostname+path, so this will make more sense
+     * when output.
+     */
+
+    keylen = snprintf(key, sizeof(key), "brick%d.hostname", count);
+    if (!strcmp(server, priv->quotad_svc.name))
+        ret = dict_set_nstrn(dict, key, keylen, "Quota Daemon",
+                             SLEN("Quota Daemon"));
+#ifdef BUILD_GNFS
+    else if (!strcmp(server, priv->nfs_svc.name))
+        ret = dict_set_nstrn(dict, key, keylen, "NFS Server",
+                             SLEN("NFS Server"));
+#endif
+    else if (!strcmp(server, priv->bitd_svc.name))
+        ret = dict_set_nstrn(dict, key, keylen, "Bitrot Daemon",
+                             SLEN("Bitrot Daemon"));
+    else if (!strcmp(server, priv->scrub_svc.name))
+        ret = dict_set_nstrn(dict, key, keylen, "Scrubber Daemon",
+                             SLEN("Scrubber Daemon"));
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_SET_FAILED,
+                "Key=%s", key, NULL);
+        goto out;
+    }
+
+    keylen = snprintf(key, sizeof(key), "brick%d.path", count);
+    ret = dict_set_dynstrn(dict, key, keylen, gf_strdup(uuid_utoa(MY_UUID)));
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_SET_FAILED,
+                "Key=%s", key, NULL);
+        goto out;
+    }
+
+#ifdef BUILD_GNFS
+    /* Port is available only for the NFS server.
+     * Self-heal daemon doesn't provide any port for access
+     * by entities other than gluster.
+     */
+    if (!strcmp(server, priv->nfs_svc.name)) {
+        if (dict_getn(vol_opts, "nfs.port", SLEN("nfs.port"))) {
+            ret = dict_get_int32n(vol_opts, "nfs.port", SLEN("nfs.port"),
+                                  &port);
+            if (ret) {
+                gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_GET_FAILED,
+                        "Key=nfs.port", NULL);
+                goto out;
+            }
+        } else
+            port = GF_NFS3_PORT;
+    }
+#endif
+    keylen = snprintf(key, sizeof(key), "brick%d.port", count);
+    ret = dict_set_int32n(dict, key, keylen, port);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_SET_FAILED,
+                "Key=%s", key, NULL);
+        goto out;
+    }
+
+    keylen = snprintf(key, sizeof(key), "brick%d.pid", count);
+    ret = dict_set_int32n(dict, key, keylen, pid);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_SET_FAILED,
+                "Key=%s", key, NULL);
+        goto out;
+    }
+
+    keylen = snprintf(key, sizeof(key), "brick%d.status", count);
+    ret = dict_set_int32n(dict, key, keylen, running);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_SET_FAILED,
+                "Key=%s", key, NULL);
+        goto out;
+    }
 
 out:
-        gf_log (THIS->name, GF_LOG_DEBUG, "Returning %d", ret);
-        return ret;
+    gf_msg_debug(THIS->name, 0, "Returning %d", ret);
+    return ret;
 }
 
 int
-glusterd_remote_hostname_get (rpcsvc_request_t *req, char *remote_host, int len)
-{
-        GF_ASSERT (req);
-        GF_ASSERT (remote_host);
-        GF_ASSERT (req->trans);
+glusterd_remote_hostname_get(rpcsvc_request_t *req, char *remote_host, int len)
+{
+    GF_ASSERT(req);
+    GF_ASSERT(remote_host);
+    GF_ASSERT(req->trans);
+
+    char *name = NULL;
+    char *hostname = NULL;
+    char *tmp_host = NULL;
+    char *canon = NULL;
+    int ret = 0;
+
+    name = req->trans->peerinfo.identifier;
+    tmp_host = gf_strdup(name);
+    if (tmp_host)
+        get_host_name(tmp_host, &hostname);
+
+    GF_ASSERT(hostname);
+    if (!hostname) {
+        memset(remote_host, 0, len);
+        ret = -1;
+        goto out;
+    }
 
-        char *name = NULL;
-        char *hostname = NULL;
-        char *tmp_host = NULL;
-        int  ret = 0;
+    if ((gf_get_hostname_from_ip(hostname, &canon) == 0) && canon) {
+        GF_FREE(tmp_host);
+        tmp_host = hostname = canon;
+    }
 
-        name = req->trans->peerinfo.identifier;
-        tmp_host = gf_strdup (name);
-        if (tmp_host)
-                get_host_name (tmp_host, &hostname);
+    (void)snprintf(remote_host, len, "%s", hostname);
 
-        GF_ASSERT (hostname);
-        if (!hostname) {
-                memset (remote_host, 0, len);
-                ret = -1;
-                goto out;
-        }
+out:
+    GF_FREE(tmp_host);
+    return ret;
+}
 
-        strncpy (remote_host, hostname, strlen (hostname));
+gf_boolean_t
+glusterd_are_all_volumes_stopped()
+{
+    glusterd_conf_t *priv = NULL;
+    xlator_t *this = NULL;
+    glusterd_volinfo_t *voliter = NULL;
 
+    this = THIS;
+    GF_ASSERT(this);
+    priv = this->private;
+    GF_ASSERT(priv);
 
-out:
-        GF_FREE (tmp_host);
-        return ret;
+    cds_list_for_each_entry(voliter, &priv->volumes, vol_list)
+    {
+        if (voliter->status == GLUSTERD_STATUS_STARTED)
+            return _gf_false;
+    }
+
+    return _gf_true;
 }
 
 gf_boolean_t
-glusterd_are_all_volumes_stopped ()
+glusterd_all_shd_compatible_volumes_stopped()
 {
-        glusterd_conf_t                         *priv = NULL;
-        xlator_t                                *this = NULL;
-        glusterd_volinfo_t                      *voliter = NULL;
-
-        this = THIS;
-        GF_ASSERT (this);
-        priv = this->private;
-        GF_ASSERT (priv);
+    glusterd_conf_t *priv = NULL;
+    xlator_t *this = NULL;
+    glusterd_volinfo_t *voliter = NULL;
 
-        cds_list_for_each_entry (voliter, &priv->volumes, vol_list) {
-                if (voliter->status == GLUSTERD_STATUS_STARTED)
-                        return _gf_false;
-        }
+    this = THIS;
+    GF_ASSERT(this);
+    priv = this->private;
+    GF_ASSERT(priv);
 
-        return _gf_true;
+    cds_list_for_each_entry(voliter, &priv->volumes, vol_list)
+    {
+        if (!glusterd_is_shd_compatible_volume(voliter))
+            continue;
+        if (voliter->status == GLUSTERD_STATUS_STARTED)
+            return _gf_false;
+    }
 
+    return _gf_true;
 }
 
 gf_boolean_t
-glusterd_all_shd_compatible_volumes_stopped ()
+glusterd_all_volumes_with_quota_stopped()
 {
-        glusterd_conf_t                         *priv = NULL;
-        xlator_t                                *this = NULL;
-        glusterd_volinfo_t                      *voliter = NULL;
+    glusterd_conf_t *priv = NULL;
+    xlator_t *this = NULL;
+    glusterd_volinfo_t *voliter = NULL;
 
-        this = THIS;
-        GF_ASSERT (this);
-        priv = this->private;
-        GF_ASSERT (priv);
+    this = THIS;
+    GF_ASSERT(this);
+    priv = this->private;
+    GF_ASSERT(priv);
 
-        cds_list_for_each_entry (voliter, &priv->volumes, vol_list) {
-                if (!glusterd_is_shd_compatible_volume (voliter))
-                        continue;
-                if (voliter->status == GLUSTERD_STATUS_STARTED)
-                        return _gf_false;
-        }
+    cds_list_for_each_entry(voliter, &priv->volumes, vol_list)
+    {
+        if (!glusterd_is_volume_quota_enabled(voliter))
+            continue;
+        if (voliter->status == GLUSTERD_STATUS_STARTED)
+            return _gf_false;
+    }
 
-        return _gf_true;
+    return _gf_true;
 }
 
 gf_boolean_t
-glusterd_all_volumes_with_quota_stopped ()
+glusterd_have_volumes()
 {
-        glusterd_conf_t                   *priv     = NULL;
-        xlator_t                          *this     = NULL;
-        glusterd_volinfo_t                *voliter  = NULL;
+    xlator_t *this = NULL;
+    glusterd_conf_t *priv = NULL;
+    gf_boolean_t volumes_exist = _gf_false;
 
-        this = THIS;
-        GF_ASSERT (this);
-        priv = this->private;
-        GF_ASSERT (priv);
+    this = THIS;
+    GF_VALIDATE_OR_GOTO("glusterd", (this != NULL), out);
 
-        cds_list_for_each_entry (voliter, &priv->volumes, vol_list) {
-                if (!glusterd_is_volume_quota_enabled (voliter))
-                        continue;
-                if (voliter->status == GLUSTERD_STATUS_STARTED)
-                        return _gf_false;
-        }
+    priv = this->private;
+    GF_VALIDATE_OR_GOTO(this->name, (priv != NULL), out);
 
-        return _gf_true;
+    volumes_exist = !cds_list_empty(&priv->volumes);
+out:
+    return volumes_exist;
 }
 
 int
-glusterd_volume_count_get (void)
+glusterd_volume_count_get(void)
 {
-        glusterd_volinfo_t      *tmp_volinfo = NULL;
-        int32_t                 ret = 0;
-        xlator_t                *this = NULL;
-        glusterd_conf_t         *priv = NULL;
+    glusterd_volinfo_t *tmp_volinfo = NULL;
+    int32_t ret = 0;
+    xlator_t *this = NULL;
+    glusterd_conf_t *priv = NULL;
 
-        this = THIS;
-        GF_ASSERT (this);
+    this = THIS;
+    GF_ASSERT(this);
 
-        priv = this->private;
+    priv = this->private;
 
-        cds_list_for_each_entry (tmp_volinfo, &priv->volumes, vol_list) {
-                ret++;
-        }
-
-
-        gf_log ("", GF_LOG_DEBUG, "Returning %d", ret);
-        return ret;
+    cds_list_for_each_entry(tmp_volinfo, &priv->volumes, vol_list) { ret++; }
 
+    gf_msg_debug("glusterd", 0, "Returning %d", ret);
+    return ret;
 }
 
 int
-glusterd_brickinfo_get (uuid_t uuid, char *hostname, char *path,
-                        glusterd_brickinfo_t **brickinfo)
+glusterd_brickinfo_get(uuid_t uuid, char *hostname, char *path,
+                       glusterd_brickinfo_t **brickinfo)
 {
-        glusterd_volinfo_t              *volinfo     = NULL;
-        glusterd_conf_t                 *priv = NULL;
-        xlator_t                        *this = NULL;
-        int                             ret = -1;
+    glusterd_volinfo_t *volinfo = NULL;
+    glusterd_conf_t *priv = NULL;
+    xlator_t *this = NULL;
+    int ret = -1;
 
-        GF_ASSERT (path);
+    GF_ASSERT(path);
 
-        this = THIS;
-        GF_ASSERT (this);
+    this = THIS;
+    GF_ASSERT(this);
 
-        priv = this->private;
+    priv = this->private;
 
-        cds_list_for_each_entry (volinfo, &priv->volumes, vol_list) {
-
-                ret = glusterd_volume_brickinfo_get (uuid, hostname, path,
-                                                     volinfo, brickinfo);
-                if (ret == 0)
-                        /*Found*/
-                        goto out;
-        }
+    cds_list_for_each_entry(volinfo, &priv->volumes, vol_list)
+    {
+        ret = glusterd_volume_brickinfo_get(uuid, hostname, path, volinfo,
+                                            brickinfo);
+        if (ret == 0)
+            /*Found*/
+            goto out;
+    }
 out:
-        return ret;
+    return ret;
 }
 
-int
-glusterd_brick_start (glusterd_volinfo_t *volinfo,
-                      glusterd_brickinfo_t *brickinfo,
-                      gf_boolean_t wait)
+static int32_t
+my_callback(struct rpc_req *req, struct iovec *iov, int count, void *v_frame)
 {
-        int                                     ret   = -1;
-        xlator_t                                *this = NULL;
+    call_frame_t *frame = v_frame;
+    glusterd_conf_t *conf = frame->this->private;
 
-        this = THIS;
-        GF_ASSERT (this);
+    if (GF_ATOMIC_DEC(conf->blockers) == 0) {
+        synccond_broadcast(&conf->cond_blockers);
+    }
 
-        if ((!brickinfo) || (!volinfo))
-                goto out;
+    STACK_DESTROY(frame->root);
+    return 0;
+}
 
-        if (gf_uuid_is_null (brickinfo->uuid)) {
-                ret = glusterd_resolve_brick (brickinfo);
-                if (ret) {
-                        gf_log (this->name, GF_LOG_ERROR, FMTSTR_RESOLVE_BRICK,
-                                brickinfo->hostname, brickinfo->path);
-                        goto out;
-                }
-        }
+static int32_t
+attach_brick_callback(struct rpc_req *req, struct iovec *iov, int count,
+                      void *v_frame)
+{
+    call_frame_t *frame = v_frame;
+    glusterd_conf_t *conf = frame->this->private;
+    glusterd_brickinfo_t *brickinfo = frame->local;
+    glusterd_brickinfo_t *other_brick = frame->cookie;
+    glusterd_volinfo_t *volinfo = NULL;
+    xlator_t *this = THIS;
+    int ret = -1;
+    char pidfile1[PATH_MAX] = "";
+    char pidfile2[PATH_MAX] = "";
+    gf_getspec_rsp rsp = {
+        0,
+    };
+    int last_brick = -1;
+
+    frame->local = NULL;
+    frame->cookie = NULL;
+
+    if (!iov) {
+        gf_log(frame->this->name, GF_LOG_ERROR, "iov is NULL");
+        ret = -1;
+        goto out;
+    }
 
-        if (gf_uuid_compare (brickinfo->uuid, MY_UUID)) {
-                ret = 0;
-                goto out;
-        }
-        ret = glusterd_volume_start_glusterfs (volinfo, brickinfo, wait);
+    ret = xdr_to_generic(*iov, &rsp, (xdrproc_t)xdr_gf_getspec_rsp);
+    if (ret < 0) {
+        gf_log(frame->this->name, GF_LOG_ERROR, "XDR decoding error");
+        ret = -1;
+        goto out;
+    }
+
+    ret = glusterd_get_volinfo_from_brick(other_brick->path, &volinfo);
+    if (ret) {
+        gf_msg(THIS->name, GF_LOG_ERROR, 0, GD_MSG_VOLINFO_GET_FAIL,
+               "Failed to get volinfo"
+               " from brick(%s) so  pidfile copying/unlink will fail",
+               other_brick->path);
+        goto out;
+    }
+    GLUSTERD_GET_BRICK_PIDFILE(pidfile1, volinfo, other_brick, conf);
+    volinfo = NULL;
+
+    ret = glusterd_get_volinfo_from_brick(brickinfo->path, &volinfo);
+    if (ret) {
+        gf_msg(THIS->name, GF_LOG_ERROR, 0, GD_MSG_VOLINFO_GET_FAIL,
+               "Failed to get volinfo"
+               " from brick(%s) so  pidfile copying/unlink will fail",
+               brickinfo->path);
+        goto out;
+    }
+    GLUSTERD_GET_BRICK_PIDFILE(pidfile2, volinfo, brickinfo, conf);
+
+    if (rsp.op_ret == 0) {
+        brickinfo->port_registered = _gf_true;
+
+        /* PID file is copied once brick has attached
+           successfully
+        */
+        ret = glusterd_copy_file(pidfile1, pidfile2);
         if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "Unable to start brick %s:%s",
-                        brickinfo->hostname, brickinfo->path);
-                goto out;
+            gf_msg(this->name, GF_LOG_ERROR, ENOMEM, GD_MSG_NO_MEMORY,
+                   "Could not copy file %s to %s", pidfile1, pidfile2);
+            goto out;
         }
 
+        brickinfo->status = GF_BRICK_STARTED;
+        brickinfo->rpc = rpc_clnt_ref(other_brick->rpc);
+        gf_log(THIS->name, GF_LOG_INFO, "brick %s is attached successfully",
+               brickinfo->path);
+    } else {
+        gf_log(THIS->name, GF_LOG_INFO,
+               "attach_brick failed pidfile"
+               " is %s for brick_path %s",
+               pidfile2, brickinfo->path);
+        brickinfo->port = 0;
+        brickinfo->status = GF_BRICK_STOPPED;
+        ret = glusterd_brick_process_remove_brick(brickinfo, &last_brick);
+        if (ret)
+            gf_msg_debug(this->name, 0,
+                         "Couldn't remove brick from"
+                         " brick process");
+        LOCK(&volinfo->lock);
+        ret = glusterd_store_volinfo(volinfo, GLUSTERD_VOLINFO_VER_AC_NONE);
+        UNLOCK(&volinfo->lock);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_VOLINFO_SET_FAIL,
+                   "Failed to store volinfo of "
+                   "%s volume",
+                   volinfo->volname);
+            goto out;
+        }
+    }
 out:
-        gf_log (this->name, GF_LOG_DEBUG, "returning %d ", ret);
-        return ret;
+    if (GF_ATOMIC_DEC(conf->blockers) == 0) {
+        synccond_broadcast(&conf->cond_blockers);
+    }
+    STACK_DESTROY(frame->root);
+    return 0;
 }
 
 int
-glusterd_restart_bricks (glusterd_conf_t *conf)
-{
-        int                   ret            = 0;
-        glusterd_volinfo_t   *volinfo        = NULL;
-        glusterd_brickinfo_t *brickinfo      = NULL;
-        glusterd_snap_t      *snap           = NULL;
-        gf_boolean_t          start_svcs     = _gf_false;
-        xlator_t             *this           = NULL;
+send_attach_req(xlator_t *this, struct rpc_clnt *rpc, char *path,
+                glusterd_brickinfo_t *brickinfo,
+                glusterd_brickinfo_t *other_brick, int op)
+{
+    int ret = -1;
+    struct iobuf *iobuf = NULL;
+    struct iobref *iobref = NULL;
+    struct iovec iov = {
+        0,
+    };
+    ssize_t req_size = 0;
+    call_frame_t *frame = NULL;
+    gd1_mgmt_brick_op_req brick_req;
+    void *req = &brick_req;
+    void *errlbl = &&err;
+    struct rpc_clnt_connection *conn;
+    glusterd_conf_t *conf = this->private;
+    extern struct rpc_clnt_program gd_brick_prog;
+    fop_cbk_fn_t cbkfn = my_callback;
+
+    if (!rpc) {
+        gf_log(this->name, GF_LOG_ERROR, "called with null rpc");
+        return -1;
+    }
 
-        this = THIS;
-        GF_ASSERT (this);
+    conn = &rpc->conn;
+    if (!conn->connected || conn->disconnected) {
+        gf_log(this->name, GF_LOG_INFO, "not connected yet");
+        return -1;
+    }
+
+    brick_req.op = op;
+    brick_req.name = path;
+    brick_req.input.input_val = NULL;
+    brick_req.input.input_len = 0;
+    brick_req.dict.dict_val = NULL;
+    brick_req.dict.dict_len = 0;
+
+    req_size = xdr_sizeof((xdrproc_t)xdr_gd1_mgmt_brick_op_req, req);
+    iobuf = iobuf_get2(rpc->ctx->iobuf_pool, req_size);
+    if (!iobuf) {
+        goto *errlbl;
+    }
+    errlbl = &&maybe_free_iobuf;
+
+    iov.iov_base = iobuf->ptr;
+    iov.iov_len = iobuf_pagesize(iobuf);
+
+    iobref = iobref_new();
+    if (!iobref) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_NO_MEMORY, NULL);
+        goto *errlbl;
+    }
+    errlbl = &&free_iobref;
+
+    frame = create_frame(this, this->ctx->pool);
+    if (!frame) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_FRAME_CREATE_FAIL,
+                NULL);
+        goto *errlbl;
+    }
+
+    iobref_add(iobref, iobuf);
+    /*
+     * Drop our reference to the iobuf.  The iobref should already have
+     * one after iobref_add, so when we unref that we'll free the iobuf as
+     * well.  This allows us to pass just the iobref as frame->local.
+     */
+    iobuf_unref(iobuf);
+    /* Set the pointer to null so we don't free it on a later error. */
+    iobuf = NULL;
+
+    /* Create the xdr payload */
+    ret = xdr_serialize_generic(iov, req, (xdrproc_t)xdr_gd1_mgmt_brick_op_req);
+    if (ret == -1) {
+        goto *errlbl;
+    }
+
+    iov.iov_len = ret;
+
+    if (op == GLUSTERD_BRICK_ATTACH) {
+        frame->local = brickinfo;
+        frame->cookie = other_brick;
+        cbkfn = attach_brick_callback;
+    }
+    /* Send the msg */
+    GF_ATOMIC_INC(conf->blockers);
+    ret = rpc_clnt_submit(rpc, &gd_brick_prog, op, cbkfn, &iov, 1, NULL, 0,
+                          iobref, frame, NULL, 0, NULL, 0, NULL);
+
+free_iobref:
+    iobref_unref(iobref);
+maybe_free_iobuf:
+    if (iobuf) {
+        iobuf_unref(iobuf);
+    }
+err:
+    return ret;
+}
+
+extern size_t
+build_volfile_path(char *volume_id, char *path, size_t path_len,
+                   char *trusted_str, dict_t *dict);
 
-        cds_list_for_each_entry (volinfo, &conf->volumes, vol_list) {
-                if (volinfo->status != GLUSTERD_STATUS_STARTED)
-                        continue;
-                if (start_svcs == _gf_false) {
-                        start_svcs = _gf_true;
-                        glusterd_svcs_manager (volinfo);
-                }
-                gf_log (this->name, GF_LOG_DEBUG, "starting the volume %s",
-                        volinfo->volname);
-                cds_list_for_each_entry (brickinfo, &volinfo->bricks,
-                                         brick_list) {
-                        glusterd_brick_start (volinfo, brickinfo, _gf_false);
+static int
+attach_brick(xlator_t *this, glusterd_brickinfo_t *brickinfo,
+             glusterd_brickinfo_t *other_brick, glusterd_volinfo_t *volinfo,
+             glusterd_volinfo_t *other_vol)
+{
+    glusterd_conf_t *conf = this->private;
+    char unslashed[PATH_MAX] = {
+        '\0',
+    };
+    char full_id[PATH_MAX] = {
+        '\0',
+    };
+    char path[PATH_MAX] = {
+        '\0',
+    };
+    int ret = -1;
+    int tries;
+    rpc_clnt_t *rpc;
+    int32_t len;
+
+    gf_log(this->name, GF_LOG_INFO, "add brick %s to existing process for %s",
+           brickinfo->path, other_brick->path);
+
+    GLUSTERD_REMOVE_SLASH_FROM_PATH(brickinfo->path, unslashed);
+
+    if (volinfo->is_snap_volume) {
+        len = snprintf(full_id, sizeof(full_id), "/%s/%s/%s/%s.%s.%s",
+                       GLUSTERD_VOL_SNAP_DIR_PREFIX,
+                       volinfo->snapshot->snapname, volinfo->volname,
+                       volinfo->volname, brickinfo->hostname, unslashed);
+    } else {
+        len = snprintf(full_id, sizeof(full_id), "%s.%s.%s", volinfo->volname,
+                       brickinfo->hostname, unslashed);
+    }
+    if ((len < 0) || (len >= sizeof(full_id))) {
+        goto out;
+    }
+
+    (void)build_volfile_path(full_id, path, sizeof(path), NULL, NULL);
+
+    for (tries = 15; tries > 0; --tries) {
+        rpc = rpc_clnt_ref(other_brick->rpc);
+        if (rpc) {
+            ret = send_attach_req(this, rpc, path, brickinfo, other_brick,
+                                  GLUSTERD_BRICK_ATTACH);
+            rpc_clnt_unref(rpc);
+            if (!ret) {
+                ret = pmap_registry_extend(this, other_brick->port,
+                                           brickinfo->path);
+                if (ret != 0) {
+                    gf_log(this->name, GF_LOG_ERROR,
+                           "adding brick to process failed");
+                    goto out;
+                }
+                brickinfo->port = other_brick->port;
+                ret = glusterd_brick_process_add_brick(brickinfo, other_brick);
+                if (ret) {
+                    gf_msg(this->name, GF_LOG_ERROR, 0,
+                           GD_MSG_BRICKPROC_ADD_BRICK_FAILED,
+                           "Adding brick %s:%s to brick "
+                           "process failed",
+                           brickinfo->hostname, brickinfo->path);
+                    return ret;
                 }
+                return 0;
+            }
+        }
+        /*
+         * It might not actually be safe to manipulate the lock
+         * like this, but if we don't then the connection can
+         * never actually complete and retries are useless.
+         * Unfortunately, all of the alternatives (e.g. doing
+         * all of this in a separate thread) are much more
+         * complicated and risky.
+         * TBD: see if there's a better way
+         */
+        synclock_unlock(&conf->big_lock);
+        synctask_sleep(1);
+        synclock_lock(&conf->big_lock);
+    }
+
+out:
+    gf_log(this->name, GF_LOG_WARNING, "attach failed for %s", brickinfo->path);
+    return ret;
+}
+
+/* This name was just getting too long, hence the abbreviations. */
+static glusterd_brickinfo_t *
+find_compat_brick_in_vol(glusterd_conf_t *conf,
+                         glusterd_volinfo_t *srch_vol, /* volume to search */
+                         glusterd_volinfo_t *comp_vol, /* volume to compare */
+                         glusterd_brickinfo_t *brickinfo)
+{
+    xlator_t *this = THIS;
+    glusterd_brickinfo_t *other_brick = NULL;
+    glusterd_brick_proc_t *brick_proc = NULL;
+    char pidfile2[PATH_MAX] = "";
+    int32_t pid2 = -1;
+    int16_t retries = 15;
+    int mux_limit = -1;
+    int ret = -1;
+    gf_boolean_t brick_status = _gf_false;
+    gf_boolean_t is_shared_storage = _gf_false;
+
+    /*
+     * If comp_vol is provided, we have to check *volume* compatibility
+     * before we can check *brick* compatibility.
+     */
+    if (comp_vol) {
+        /*
+         * We should not attach bricks of a normal volume to bricks
+         * of shared storage volume.
+         */
+        if (!strcmp(srch_vol->volname, GLUSTER_SHARED_STORAGE))
+            is_shared_storage = _gf_true;
+
+        if (!strcmp(comp_vol->volname, GLUSTER_SHARED_STORAGE)) {
+            if (!is_shared_storage)
+                return NULL;
+        } else if (is_shared_storage)
+            return NULL;
+
+        /*
+         * It's kind of a shame that we have to do this check in both
+         * directions, but an option might only exist on one of the two
+         * dictionaries and dict_foreach_match will only find that one.
+         */
+
+        gf_log(THIS->name, GF_LOG_DEBUG, "comparing options for %s and %s",
+               comp_vol->volname, srch_vol->volname);
+
+        if (dict_foreach_match(comp_vol->dict, unsafe_option, NULL,
+                               opts_mismatch, srch_vol->dict) < 0) {
+            gf_log(THIS->name, GF_LOG_DEBUG, "failure forward");
+            return NULL;
+        }
+
+        if (dict_foreach_match(srch_vol->dict, unsafe_option, NULL,
+                               opts_mismatch, comp_vol->dict) < 0) {
+            gf_log(THIS->name, GF_LOG_DEBUG, "failure backward");
+            return NULL;
         }
 
-        cds_list_for_each_entry (snap, &conf->snapshots, snap_list) {
-                cds_list_for_each_entry (volinfo, &snap->volumes, vol_list) {
-                        if (volinfo->status != GLUSTERD_STATUS_STARTED)
-                                continue;
-                        if (start_svcs == _gf_false) {
-                                start_svcs = _gf_true;
-                                glusterd_svcs_manager (volinfo);
-                        }
-                        start_svcs = _gf_true;
-                        gf_log (this->name, GF_LOG_DEBUG, "starting the snap "
-                                "volume %s", volinfo->volname);
-                        cds_list_for_each_entry (brickinfo, &volinfo->bricks,
-                                                 brick_list) {
-                                glusterd_brick_start (volinfo, brickinfo,
-                                                      _gf_false);
-                        }
+        gf_log(THIS->name, GF_LOG_DEBUG, "all options match");
+    }
+
+    ret = get_mux_limit_per_process(&mux_limit);
+    if (ret) {
+        gf_msg_debug(THIS->name, 0,
+                     "Retrieving brick mux "
+                     "limit failed. Returning NULL");
+        return NULL;
+    }
+
+    cds_list_for_each_entry(other_brick, &srch_vol->bricks, brick_list)
+    {
+        if (other_brick == brickinfo) {
+            continue;
+        }
+        if (gf_uuid_compare(brickinfo->uuid, other_brick->uuid)) {
+            continue;
+        }
+        if (other_brick->status != GF_BRICK_STARTED &&
+            other_brick->status != GF_BRICK_STARTING) {
+            continue;
+        }
+
+        ret = glusterd_brick_proc_for_port(other_brick->port, &brick_proc);
+        if (ret) {
+            gf_msg_debug(THIS->name, 0,
+                         "Couldn't get brick "
+                         "process corresponding to brick %s:%s",
+                         other_brick->hostname, other_brick->path);
+            continue;
+        }
+
+        if (mux_limit != 0) {
+            if (brick_proc->brick_count >= mux_limit)
+                continue;
+        } else {
+            /* This means that the "cluster.max-bricks-per-process"
+             * options hasn't yet been explicitly set. Continue
+             * as if there's no limit set
+             */
+            gf_msg(THIS->name, GF_LOG_WARNING, 0, GD_MSG_NO_MUX_LIMIT,
+                   "cluster.max-bricks-per-process options isn't "
+                   "set. Continuing with no limit set for "
+                   "brick multiplexing.");
+        }
+        /* The first brick process might take some time to finish its
+         * handshake with glusterd and prepare the graph. We can't
+         * afford to send attach_req for other bricks till that time.
+         * brick process sends PMAP_SIGNIN event after processing the
+         * volfile and hence it's safe to assume that if glusterd has
+         * received a pmap signin request for the same brick, we are
+         * good for subsequent attach requests.
+         */
+        retries = 15;
+        while (retries > 0) {
+            if (other_brick->port_registered) {
+                GLUSTERD_GET_BRICK_PIDFILE(pidfile2, srch_vol, other_brick,
+                                           conf);
+                if (sys_access(pidfile2, F_OK) == 0 &&
+                    gf_is_service_running(pidfile2, &pid2)) {
+                    gf_msg_debug(this->name, 0,
+                                 "brick %s is running as a pid %d ",
+                                 other_brick->path, pid2);
+                    brick_status = _gf_true;
+                    break;
+                }
+            }
+
+            synclock_unlock(&conf->big_lock);
+            gf_msg_debug(this->name, 0,
+                         "brick %s is still"
+                         " starting, waiting for 2 seconds ",
+                         other_brick->path);
+            synctask_sleep(2);
+            synclock_lock(&conf->big_lock);
+            retries--;
+        }
+
+        if (!brick_status) {
+            gf_log(this->name, GF_LOG_INFO,
+                   "brick has not come up so cleaning up dead brick %s:%s",
+                   other_brick->hostname, other_brick->path);
+            other_brick->status = GF_BRICK_STOPPED;
+            if (pidfile2[0])
+                sys_unlink(pidfile2);
+            continue;
+        }
+        return other_brick;
+    }
+
+    return NULL;
+}
+
+static glusterd_brickinfo_t *
+find_compatible_brick(glusterd_conf_t *conf, glusterd_volinfo_t *volinfo,
+                      glusterd_brickinfo_t *brickinfo,
+                      glusterd_volinfo_t **other_vol_p)
+{
+    glusterd_brickinfo_t *other_brick = NULL;
+    glusterd_volinfo_t *other_vol = NULL;
+    glusterd_snap_t *snap = NULL;
+
+    /* Just return NULL here if multiplexing is disabled. */
+    if (!is_brick_mx_enabled()) {
+        return NULL;
+    }
+
+    other_brick = find_compat_brick_in_vol(conf, volinfo, NULL, brickinfo);
+    if (other_brick) {
+        *other_vol_p = volinfo;
+        return other_brick;
+    }
+
+    /*
+     * This check is necessary because changes to a volume's
+     * transport options aren't propagated to snapshots.  Such a
+     * change might break compatibility between the two, but we
+     * have no way to "evict" a brick from the process it's
+     * currently in.  If we keep it separate from the start, we
+     * avoid the problem.  Note that snapshot bricks can still be
+     * colocated with one another, even if they're for different
+     * volumes, because the only thing likely to differ is their
+     * auth options and those are not a factor in determining
+     * compatibility.
+     *
+     * The very same immutability of snapshot bricks' transport
+     * options, which can make them incompatible with their parent
+     * volumes, ensures that once-compatible snapshot bricks will
+     * remain compatible.  However, the same is not true for bricks
+     * belonging to two non-snapshot volumes.  In that case, a
+     * change to one might break compatibility and require them to
+     * be separated, which is not yet done.
+     *
+     * TBD: address the option-change issue for non-snapshot bricks
+     */
+    if (!volinfo->is_snap_volume) {
+        cds_list_for_each_entry(other_vol, &conf->volumes, vol_list)
+        {
+            if (other_vol == volinfo) {
+                continue;
+            }
+            other_brick = find_compat_brick_in_vol(conf, other_vol, volinfo,
+                                                   brickinfo);
+            if (other_brick) {
+                *other_vol_p = other_vol;
+                return other_brick;
+            }
+        }
+    } else {
+        cds_list_for_each_entry(snap, &conf->snapshots, snap_list)
+        {
+            cds_list_for_each_entry(other_vol, &snap->volumes, vol_list)
+            {
+                if (other_vol == volinfo) {
+                    continue;
+                }
+                other_brick = find_compat_brick_in_vol(conf, other_vol, volinfo,
+                                                       brickinfo);
+                if (other_brick) {
+                    *other_vol_p = other_vol;
+                    return other_brick;
                 }
+            }
         }
+    }
 
-        return ret;
+    return NULL;
 }
 
+/* Below function is use to populate sockpath based on passed pid
+   value as a argument after check the value from proc and also
+   check if passed pid is match with running  glusterfs process
+*/
+
 int
-_local_gsyncd_start (dict_t *this, char *key, data_t *value, void *data)
-{
-        char               *path_list                   = NULL;
-        char               *slave                       = NULL;
-        char               *slave_url                   = NULL;
-        char               *slave_vol                   = NULL;
-        char               *slave_host                  = NULL;
-        char               *statefile                   = NULL;
-        char                buf[1024]                   = "faulty";
-        int                 uuid_len                    = 0;
-        int                 ret                         = 0;
-        int                 op_ret                      = 0;
-        int                 ret_status                  = 0;
-        char                uuid_str[64]                = {0};
-        glusterd_volinfo_t *volinfo                     = NULL;
-        char                confpath[PATH_MAX]          = "";
-        char               *op_errstr                   = NULL;
-        glusterd_conf_t    *priv                        = NULL;
-        gf_boolean_t        is_template_in_use          = _gf_false;
-        gf_boolean_t        is_paused                   = _gf_false;
-        char               *key1                        = NULL;
-        xlator_t           *this1                       = NULL;
-
-        this1 = THIS;
-        GF_ASSERT (this1);
-        priv = this1->private;
-        GF_ASSERT (priv);
-        GF_ASSERT (data);
-
-        volinfo = data;
-        slave = strchr(value->data, ':');
-        if (slave)
-                slave ++;
-        else
-                return 0;
-        uuid_len = (slave - value->data - 1);
+glusterd_get_sock_from_brick_pid(int pid, char *sockpath, size_t len)
+{
+    char buf[1024] = "";
+    char cmdline[2048] = "";
+    xlator_t *this = NULL;
+    int fd = -1;
+    int i = 0, j = 0;
+    char *ptr = NULL;
+    char *brptr = NULL;
+    char tmpsockpath[PATH_MAX] = "";
+    size_t blen = 0;
+    int ret = -1;
+
+    this = THIS;
+    GF_ASSERT(this);
+
+#ifdef __FreeBSD__
+    blen = sizeof(buf);
+    int mib[4];
+
+    mib[0] = CTL_KERN;
+    mib[1] = KERN_PROC;
+    mib[2] = KERN_PROC_ARGS;
+    mib[3] = pid;
+
+    if (sys_sysctl(mib, 4, buf, &blen, NULL, blen) != 0) {
+        gf_log(this->name, GF_LOG_ERROR, "brick process %d is not running",
+               pid);
+        return ret;
+    }
+#else
+    char fname[128] = "";
+    snprintf(fname, sizeof(fname), "/proc/%d/cmdline", pid);
 
-        strncpy (uuid_str, (char*)value->data, uuid_len);
+    if (sys_access(fname, R_OK) != 0) {
+        gf_log(this->name, GF_LOG_ERROR, "brick process %d is not running",
+               pid);
+        return ret;
+    }
 
-        /* Getting Local Brickpaths */
-        ret = glusterd_get_local_brickpaths (volinfo, &path_list);
+    fd = open(fname, O_RDONLY);
+    if (fd != -1) {
+        blen = (int)sys_read(fd, buf, 1024);
+    } else {
+        gf_log(this->name, GF_LOG_ERROR, "open failed %s to open a file %s",
+               strerror(errno), fname);
+        return ret;
+    }
+#endif
 
-        /*Generating the conf file path needed by gsyncd */
-        ret = glusterd_get_slave_info (slave, &slave_url, &slave_host,
-                                       &slave_vol, &op_errstr);
-        if (ret) {
-                gf_log (this1->name, GF_LOG_ERROR,
-                        "Unable to fetch slave details.");
-                ret = -1;
-                goto out;
+    /* convert cmdline to single string */
+    for (i = 0, j = 0; i < blen; i++) {
+        if (buf[i] == '\0')
+            cmdline[j++] = ' ';
+        else if (buf[i] < 32 || buf[i] > 126) /* remove control char */
+            continue;
+        else if (buf[i] == '"' || buf[i] == '\\') {
+            cmdline[j++] = '\\';
+            cmdline[j++] = buf[i];
+        } else {
+            cmdline[j++] = buf[i];
         }
+    }
+    cmdline[j] = '\0';
+    if (fd)
+        sys_close(fd);
+    if (!strstr(cmdline, "glusterfs"))
+        return ret;
 
-        ret = snprintf (confpath, sizeof(confpath) - 1,
-                        "%s/"GEOREP"/%s_%s_%s/gsyncd.conf",
-                        priv->workdir, volinfo->volname,
-                        slave_host, slave_vol);
-        confpath[ret] = '\0';
+    ptr = strstr(cmdline, "-S ");
+    if (!ptr)
+        return ret;
+    ptr = strchr(ptr, '/');
+    if (!ptr)
+        return ret;
+    brptr = strstr(ptr, "--brick-name");
+    if (!brptr)
+        return ret;
+    i = 0;
 
-        /* Fetching the last status of the node */
-        ret = glusterd_get_statefile_name (volinfo, slave,
-                                           confpath, &statefile,
-                                           &is_template_in_use);
-        if (ret) {
-                if (!strstr(slave, "::"))
-                        gf_log (this1->name, GF_LOG_INFO,
-                                "%s is not a valid slave url.", slave);
-                else
-                        gf_log (this1->name, GF_LOG_INFO, "Unable to get"
-                                " statefile's name");
-                goto out;
-        }
+    while (ptr < brptr) {
+        if (*ptr != 32)
+            tmpsockpath[i++] = *ptr;
+        ptr++;
+    }
 
-        /* If state-file entry is missing from the config file,
-         * do not start gsyncd on restart */
-        if (is_template_in_use) {
-                gf_log (this1->name, GF_LOG_INFO,
-                        "state-file entry is missing in config file."
-                        "Not Restarting");
-                goto out;
-        }
+    if (tmpsockpath[0]) {
+        strncpy(sockpath, tmpsockpath, i);
+        ret = 0;
+    }
 
-        is_template_in_use = _gf_false;
+    return ret;
+}
 
-        ret = gsync_status (volinfo->volname, slave, confpath,
-                            &ret_status, &is_template_in_use);
-        if (ret == -1) {
-                gf_log (this1->name, GF_LOG_INFO,
-                        GEOREP" start option validation failed ");
-                ret = 0;
-                goto out;
-        }
+char *
+search_brick_path_from_proc(pid_t brick_pid, char *brickpath)
+{
+    char *brick_path = NULL;
+#ifdef __FreeBSD__
+    struct filestat *fst;
+    struct procstat *ps;
+    struct kinfo_proc *kp;
+    struct filestat_list *head;
 
-        if (is_template_in_use == _gf_true) {
-                gf_log (this1->name, GF_LOG_INFO,
-                        "pid-file entry is missing in config file."
-                        "Not Restarting");
-                ret = 0;
-                goto out;
-        }
+    ps = procstat_open_sysctl();
+    if (ps == NULL)
+        goto out;
 
-        ret = glusterd_gsync_read_frm_status (statefile, buf, sizeof (buf));
-        if (ret < 0) {
-                gf_log (this1->name, GF_LOG_ERROR, "Unable to read the status");
-                goto out;
+    kp = kinfo_getproc(brick_pid);
+    if (kp == NULL)
+        goto out;
+
+    head = procstat_getfiles(ps, (void *)kp, 0);
+    if (head == NULL)
+        goto out;
+
+    STAILQ_FOREACH(fst, head, next)
+    {
+        if (fst->fs_fd < 0)
+            continue;
+
+        if (!strcmp(fst->fs_path, brickpath)) {
+            brick_path = gf_strdup(fst->fs_path);
+            break;
         }
+    }
 
-        /* Move the pointer two characters ahead to surpass '//' */
-        if ((key1 = strchr (slave, '/')))
-                key1 = key1 + 2;
+out:
+    if (head != NULL)
+        procstat_freefiles(ps, head);
+    if (kp != NULL)
+        free(kp);
+    procstat_close(ps);
+#else
+    struct dirent *dp = NULL;
+    DIR *dirp = NULL;
+    size_t len = 0;
+    int fd = -1;
+    char path[PATH_MAX] = "";
+    struct dirent scratch[2] = {
+        {
+            0,
+        },
+    };
+
+    if (!brickpath)
+        goto out;
+
+    len = sprintf(path, "/proc/%d/fd/", brick_pid);
+    if (len >= (sizeof(path) - 2))
+        goto out;
+
+    dirp = sys_opendir(path);
+    if (!dirp)
+        goto out;
+
+    fd = dirfd(dirp);
+    if (fd < 0)
+        goto out;
+
+    while ((dp = sys_readdir(dirp, scratch))) {
+        if (!strcmp(dp->d_name, ".") || !strcmp(dp->d_name, ".."))
+            continue;
+
+        /* check for non numerical descriptors */
+        if (!strtol(dp->d_name, (char **)NULL, 10))
+            continue;
+
+        len = readlinkat(fd, dp->d_name, path, sizeof(path) - 1);
+        /* TODO: handle len == -1 -> error condition in readlinkat */
+        if (len > 1) {
+            path[len] = '\0';
+            if (!strcmp(path, brickpath)) {
+                brick_path = gf_strdup(path);
+                break;
+            }
+        }
+    }
+out:
+    if (dirp)
+        sys_closedir(dirp);
+#endif
+    return brick_path;
+}
 
-        /* Looks for the last status, to find if the sessiom was running
-         * when the node went down. If the session was not started or
-         * not started, do not restart the geo-rep session */
-        if ((!strcmp (buf, "Not Started")) ||
-            (!strcmp (buf, "Stopped"))) {
-                gf_log (this1->name, GF_LOG_INFO,
-                        "Geo-Rep Session was not started between "
-                        "%s and %s::%s. Not Restarting", volinfo->volname,
-                        slave_url, slave_vol);
-                goto out;
-        } else if (strstr(buf, "Paused")) {
-                is_paused = _gf_true;
-        } else if ((!strcmp (buf, "Config Corrupted"))) {
-                gf_log (this1->name, GF_LOG_INFO,
-                        "Recovering from a corrupted config. "
-                        "Not Restarting. Use start (force) to "
-                        "start the session between %s and %s::%s.",
-                        volinfo->volname,
-                        slave_url, slave_vol);
-                goto out;
+int
+glusterd_brick_start(glusterd_volinfo_t *volinfo,
+                     glusterd_brickinfo_t *brickinfo, gf_boolean_t wait,
+                     gf_boolean_t only_connect)
+{
+    int ret = -1;
+    xlator_t *this = NULL;
+    glusterd_brickinfo_t *other_brick;
+    glusterd_conf_t *conf = NULL;
+    int32_t pid = -1;
+    char pidfile[PATH_MAX] = "";
+    char socketpath[PATH_MAX] = "";
+    char *brickpath = NULL;
+    glusterd_volinfo_t *other_vol;
+    gf_boolean_t is_service_running = _gf_false;
+    uuid_t volid = {
+        0,
+    };
+    ssize_t size = -1;
+
+    this = THIS;
+    GF_ASSERT(this);
+    conf = this->private;
+
+    if ((!brickinfo) || (!volinfo)) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_INVALID_ARGUMENT, NULL);
+        goto out;
+    }
+
+    if (gf_uuid_is_null(brickinfo->uuid)) {
+        ret = glusterd_resolve_brick(brickinfo);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_RESOLVE_BRICK_FAIL,
+                   FMTSTR_RESOLVE_BRICK, brickinfo->hostname, brickinfo->path);
+            gf_event(EVENT_BRICKPATH_RESOLVE_FAILED,
+                     "peer=%s;volume=%s;brick=%s", brickinfo->hostname,
+                     volinfo->volname, brickinfo->path);
+            goto out;
         }
+    }
 
-        if (is_paused) {
-                glusterd_start_gsync (volinfo, slave, path_list, confpath,
-                                      uuid_str, NULL, _gf_true);
+    if (gf_uuid_compare(brickinfo->uuid, MY_UUID)) {
+        ret = 0;
+        goto out;
+    }
+
+    /* If a trigger to start the brick is already initiated then no need for
+     * a reattempt as it's an overkill. With glusterd_brick_start ()
+     * function being used in multiple places, when glusterd restarts we see
+     * three different triggers for an attempt to start the brick process
+     * due to the quorum handling code in glusterd_friend_sm.
+     */
+    if (brickinfo->status == GF_BRICK_STARTING || brickinfo->start_triggered ||
+        GF_ATOMIC_GET(volinfo->volpeerupdate)) {
+        gf_msg_debug(this->name, 0,
+                     "brick %s is already in starting "
+                     "phase",
+                     brickinfo->path);
+        ret = 0;
+        goto out;
+    }
+    if (!only_connect)
+        brickinfo->start_triggered = _gf_true;
+
+    GLUSTERD_GET_BRICK_PIDFILE(pidfile, volinfo, brickinfo, conf);
+
+    /* Compare volume-id xattr is helpful to ensure the existence of a
+       brick_root path before the start/attach a brick
+    */
+    size = sys_lgetxattr(brickinfo->path, GF_XATTR_VOL_ID_KEY, volid, 16);
+    if (size != 16) {
+        gf_log(this->name, GF_LOG_ERROR,
+               "Missing %s extended attribute on brick root (%s),"
+               " brick is deemed not to be a part of the volume (%s) ",
+               GF_XATTR_VOL_ID_KEY, brickinfo->path, volinfo->volname);
+        goto out;
+    }
+
+    if (strncmp(uuid_utoa(volinfo->volume_id), uuid_utoa(volid),
+                GF_UUID_BUF_SIZE)) {
+        gf_log(this->name, GF_LOG_ERROR,
+               "Mismatching %s extended attribute on brick root (%s),"
+               " brick is deemed not to be a part of the volume (%s)",
+               GF_XATTR_VOL_ID_KEY, brickinfo->path, volinfo->volname);
+        goto out;
+    }
+    is_service_running = gf_is_service_running(pidfile, &pid);
+    if (is_service_running) {
+        if (is_brick_mx_enabled()) {
+            brickpath = search_brick_path_from_proc(pid, brickinfo->path);
+            if (!brickpath) {
+                if (only_connect)
+                    return 0;
+                gf_log(this->name, GF_LOG_INFO,
+                       "Either pid %d is not running or brick"
+                       " path %s is not consumed so cleanup pidfile",
+                       pid, brickinfo->path);
+                /* brick isn't running,so unlink stale pidfile
+                 * if any.
+                 */
+                if (sys_access(pidfile, R_OK) == 0) {
+                    sys_unlink(pidfile);
+                }
+                goto run;
+            }
+            GF_FREE(brickpath);
+            ret = glusterd_get_sock_from_brick_pid(pid, socketpath,
+                                                   sizeof(socketpath));
+            if (ret) {
+                if (only_connect)
+                    return 0;
+                gf_log(this->name, GF_LOG_INFO,
+                       "Either pid %d is not running or does "
+                       "not match with any running brick "
+                       "processes",
+                       pid);
+                /* Fetch unix socket is failed so unlink pidfile */
+                if (sys_access(pidfile, R_OK) == 0) {
+                    sys_unlink(pidfile);
+                }
+                goto run;
+            }
+        }
+        if (brickinfo->status != GF_BRICK_STARTING &&
+            brickinfo->status != GF_BRICK_STARTED) {
+            gf_log(this->name, GF_LOG_INFO,
+                   "discovered already-running brick %s", brickinfo->path);
+            (void)pmap_registry_bind(this, brickinfo->port, brickinfo->path,
+                                     GF_PMAP_PORT_BRICKSERVER, NULL);
+            brickinfo->port_registered = _gf_true;
+            /*
+             * This will unfortunately result in a separate RPC
+             * connection per brick, even though they're all in
+             * the same process.  It works, but it would be nicer
+             * if we could find a pre-existing connection to that
+             * same port (on another brick) and re-use that.
+             * TBD: re-use RPC connection across bricks
+             */
+            if (!is_brick_mx_enabled()) {
+                glusterd_set_brick_socket_filepath(
+                    volinfo, brickinfo, socketpath, sizeof(socketpath));
+            }
+            gf_log(this->name, GF_LOG_DEBUG,
+                   "Using %s as sockfile for brick %s of volume %s ",
+                   socketpath, brickinfo->path, volinfo->volname);
+
+            (void)glusterd_brick_connect(volinfo, brickinfo, socketpath);
+
+            ret = glusterd_brick_process_add_brick(brickinfo, NULL);
+            if (ret) {
+                gf_msg(this->name, GF_LOG_ERROR, 0,
+                       GD_MSG_BRICKPROC_ADD_BRICK_FAILED,
+                       "Adding brick %s:%s to brick process "
+                       "failed.",
+                       brickinfo->hostname, brickinfo->path);
+                goto out;
+            }
+            /* We need to set the status back to STARTING so that
+             * while the other (re)start brick requests come in for
+             * other bricks, this brick can be considered as
+             * compatible.
+             */
+            brickinfo->status = GF_BRICK_STARTING;
         }
-        else {
-               /* Add slave to the dict indicating geo-rep session is running*/
-               ret = dict_set_dynstr_with_alloc (volinfo->gsync_active_slaves,
-                                                 key1, "running");
-               if (ret) {
-                       gf_log (this1->name, GF_LOG_ERROR, "Unable to set key:%s"
-                               " value:running in the dict", key1);
-                       goto out;
-               }
-               ret = glusterd_start_gsync (volinfo, slave, path_list, confpath,
-                                      uuid_str, NULL, _gf_false);
-               if (ret)
-                       dict_del (volinfo->gsync_active_slaves, key1);
+        return 0;
+    }
+    if (only_connect)
+        return 0;
+
+run:
+    ret = _mk_rundir_p(volinfo);
+    if (ret)
+        goto out;
+
+    other_brick = find_compatible_brick(conf, volinfo, brickinfo, &other_vol);
+    if (other_brick) {
+        /* mark the brick to starting as send_attach_req might take few
+         * iterations to successfully attach the brick and we don't want
+         * to get into a state where another needless trigger to start
+         * the brick is processed
+         */
+        brickinfo->status = GF_BRICK_STARTING;
+        ret = attach_brick(this, brickinfo, other_brick, volinfo, other_vol);
+        if (ret == 0) {
+            goto out;
         }
+        /* Attach_brick is failed so unlink pidfile */
+        if (sys_access(pidfile, R_OK) == 0) {
+            sys_unlink(pidfile);
+        }
+    }
+
+    /*
+     * This hack is necessary because our brick-process management is a
+     * total nightmare.  We expect a brick process's socket and pid files
+     * to be ready *immediately* after we start it.  Ditto for it calling
+     * back to bind its port.  Unfortunately, none of that is realistic.
+     * Any process takes non-zero time to start up.  This has *always* been
+     * racy and unsafe; it just became more visible with multiplexing.
+     *
+     * The right fix would be to do all of this setup *in the parent*,
+     * which would include (among other things) getting the PID back from
+     * the "runner" code.  That's all prohibitively difficult and risky.
+     * To work around the more immediate problems, we create a stub pidfile
+     * here to let gf_is_service_running know that we expect the process to
+     * be there shortly, and then it gets filled in with a real PID when
+     * the process does finish starting up.
+     *
+     * TBD: pray for GlusterD 2 to be ready soon.
+     */
+    gf_log(this->name, GF_LOG_INFO,
+           "starting a fresh brick process for "
+           "brick %s",
+           brickinfo->path);
+    ret = glusterd_volume_start_glusterfs(volinfo, brickinfo, wait);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_BRICK_DISCONNECTED,
+               "Unable to start brick %s:%s", brickinfo->hostname,
+               brickinfo->path);
+        gf_event(EVENT_BRICK_START_FAILED, "peer=%s;volume=%s;brick=%s",
+                 brickinfo->hostname, volinfo->volname, brickinfo->path);
+        goto out;
+    }
+
+out:
+    if (ret && brickinfo) {
+        brickinfo->start_triggered = _gf_false;
+    }
+    gf_msg_debug(this->name, 0, "returning %d ", ret);
+    return ret;
+}
+
+int
+glusterd_restart_bricks(void *opaque)
+{
+    int ret = 0;
+    glusterd_volinfo_t *volinfo = NULL;
+    glusterd_brickinfo_t *brickinfo = NULL;
+    glusterd_snap_t *snap = NULL;
+    gf_boolean_t start_svcs = _gf_false;
+    xlator_t *this = NULL;
+    glusterd_conf_t *conf = NULL;
+    int active_count = 0;
+    int quorum_count = 0;
+    gf_boolean_t node_quorum = _gf_false;
+
+    this = THIS;
+    GF_VALIDATE_OR_GOTO("glusterd", this, return_block);
+
+    conf = this->private;
+    GF_VALIDATE_OR_GOTO(this->name, conf, return_block);
+
+    synclock_lock(&conf->big_lock);
+
+    /* We need to ensure that restarting the bricks during glusterd restart
+     * shouldn't race with the import volume thread (refer
+     * glusterd_compare_friend_data ())
+     */
+    while (conf->restart_bricks) {
+        synccond_wait(&conf->cond_restart_bricks, &conf->big_lock);
+    }
+    conf->restart_bricks = _gf_true;
+
+    GF_ATOMIC_INC(conf->blockers);
+    ret = glusterd_get_quorum_cluster_counts(this, &active_count,
+                                             &quorum_count);
+    if (ret)
+        goto out;
+
+    if (does_quorum_meet(active_count, quorum_count))
+        node_quorum = _gf_true;
+
+    cds_list_for_each_entry(volinfo, &conf->volumes, vol_list)
+    {
+        if (volinfo->status != GLUSTERD_STATUS_STARTED) {
+            continue;
+        }
+        gf_msg_debug(this->name, 0, "starting the volume %s", volinfo->volname);
+
+        /* Check the quorum, if quorum is not met, don't start the
+           bricks. Stop bricks in case they are running.
+        */
+        ret = check_quorum_for_brick_start(volinfo, node_quorum);
+        if (ret == 0) {
+            gf_msg(this->name, GF_LOG_INFO, 0, GD_MSG_SERVER_QUORUM_NOT_MET,
+                   "Skipping brick "
+                   "restart for volume %s as quorum is not met",
+                   volinfo->volname);
+            (void)glusterd_stop_bricks(volinfo);
+            continue;
+        } else if (ret == 2 && conf->restart_done == _gf_true) {
+            /* If glusterd has been restarted and quorum is not
+             * applicable then do not restart the bricks as this
+             * might start bricks brought down purposely, say for
+             * maintenance
+             */
+            continue;
+        } else {
+            start_svcs = _gf_true;
+            cds_list_for_each_entry(brickinfo, &volinfo->bricks, brick_list)
+            {
+                if (!brickinfo->start_triggered) {
+                    pthread_mutex_lock(&brickinfo->restart_mutex);
+                    {
+                        glusterd_brick_start(volinfo, brickinfo, _gf_false,
+                                             _gf_false);
+                    }
+                    pthread_mutex_unlock(&brickinfo->restart_mutex);
+                }
+            }
+            ret = glusterd_store_volinfo(volinfo, GLUSTERD_VOLINFO_VER_AC_NONE);
+            if (ret) {
+                gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_VOLINFO_STORE_FAIL,
+                       "Failed to "
+                       "write volinfo for volume %s",
+                       volinfo->volname);
+                goto out;
+            }
+        }
+    }
+
+    cds_list_for_each_entry(snap, &conf->snapshots, snap_list)
+    {
+        cds_list_for_each_entry(volinfo, &snap->volumes, vol_list)
+        {
+            if (volinfo->status != GLUSTERD_STATUS_STARTED)
+                continue;
+            /* Check the quorum, if quorum is not met, don't start
+             * the bricks
+             */
+            ret = check_quorum_for_brick_start(volinfo, node_quorum);
+            if (ret == 0) {
+                gf_msg(this->name, GF_LOG_INFO, 0, GD_MSG_SERVER_QUORUM_NOT_MET,
+                       "Skipping"
+                       " brick restart for volume %s as "
+                       "quorum is not met",
+                       volinfo->volname);
+                continue;
+            }
+            start_svcs = _gf_true;
+            gf_msg_debug(this->name, 0,
+                         "starting the snap "
+                         "volume %s",
+                         volinfo->volname);
+            cds_list_for_each_entry(brickinfo, &volinfo->bricks, brick_list)
+            {
+                if (!brickinfo->start_triggered) {
+                    pthread_mutex_lock(&brickinfo->restart_mutex);
+                    {
+                        /* coverity[SLEEP] */
+                        glusterd_brick_start(volinfo, brickinfo, _gf_false,
+                                             _gf_false);
+                    }
+                    pthread_mutex_unlock(&brickinfo->restart_mutex);
+                }
+            }
+            ret = glusterd_store_volinfo(volinfo, GLUSTERD_VOLINFO_VER_AC_NONE);
+            if (ret) {
+                gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_VOLINFO_STORE_FAIL,
+                       "Failed to "
+                       "write volinfo for volume %s",
+                       volinfo->volname);
+                goto out;
+            }
+        }
+    }
+    if (start_svcs == _gf_true) {
+        glusterd_svcs_manager(NULL);
+    }
+
+    ret = 0;
 
 out:
-        if (statefile)
-                GF_FREE (statefile);
+    conf->restart_done = _gf_true;
+    conf->restart_bricks = _gf_false;
+    if (GF_ATOMIC_DEC(conf->blockers) == 0) {
+        synccond_broadcast(&conf->cond_blockers);
+    }
+    synccond_broadcast(&conf->cond_restart_bricks);
 
-        if (is_template_in_use) {
-               op_ret = glusterd_create_status_file (volinfo->volname, slave,
-                                                     slave_host, slave_vol,
-                                                     "Config Corrupted");
-               if (op_ret) {
-                        gf_log (this1->name, GF_LOG_ERROR,
-                                "Unable to create status file"
-                                ". Error : %s", strerror (errno));
-                        ret = op_ret;
-               }
-        }
+return_block:
+    return ret;
+}
 
-        GF_FREE (path_list);
-        GF_FREE (op_errstr);
+int
+_local_gsyncd_start(dict_t *this, char *key, data_t *value, void *data)
+{
+    char *path_list = NULL;
+    char *slave = NULL;
+    char *slave_url = NULL;
+    char *slave_vol = NULL;
+    char *slave_host = NULL;
+    char *statefile = NULL;
+    char buf[1024] = "faulty";
+    int ret = 0;
+    int op_ret = 0;
+    int ret_status = 0;
+    char uuid_str[64] = "";
+    glusterd_volinfo_t *volinfo = NULL;
+    char confpath[PATH_MAX] = "";
+    char *op_errstr = NULL;
+    glusterd_conf_t *priv = NULL;
+    gf_boolean_t is_template_in_use = _gf_false;
+    gf_boolean_t is_paused = _gf_false;
+    char key1[1024] = "";
+    xlator_t *this1 = NULL;
+
+    this1 = THIS;
+    GF_ASSERT(this1);
+    priv = this1->private;
+    GF_ASSERT(priv);
+    GF_ASSERT(data);
+
+    volinfo = data;
+    slave = strchr(value->data, ':');
+    if (slave)
+        slave++;
+    else
+        return 0;
 
-        return ret;
+    (void)snprintf(uuid_str, sizeof(uuid_str), "%s", (char *)value->data);
+
+    /* Getting Local Brickpaths */
+    ret = glusterd_get_local_brickpaths(volinfo, &path_list);
+
+    /*Generating the conf file path needed by gsyncd */
+    ret = glusterd_get_slave_info(slave, &slave_url, &slave_host, &slave_vol,
+                                  &op_errstr);
+    if (ret) {
+        gf_msg(this1->name, GF_LOG_ERROR, 0, GD_MSG_SLAVEINFO_FETCH_ERROR,
+               "Unable to fetch slave details.");
+        ret = -1;
+        goto out;
+    }
+
+    ret = snprintf(confpath, sizeof(confpath) - 1,
+                   "%s/" GEOREP "/%s_%s_%s/gsyncd.conf", priv->workdir,
+                   volinfo->volname, slave_host, slave_vol);
+    confpath[ret] = '\0';
+
+    /* Fetching the last status of the node */
+    ret = glusterd_get_statefile_name(volinfo, slave, confpath, &statefile,
+                                      &is_template_in_use);
+    if (ret) {
+        if (!strstr(slave, "::"))
+            gf_msg(this1->name, GF_LOG_INFO, 0, GD_MSG_SLAVE_URL_INVALID,
+                   "%s is not a valid slave url.", slave);
+        else
+            gf_msg(this1->name, GF_LOG_INFO, 0,
+                   GD_MSG_GET_STATEFILE_NAME_FAILED,
+                   "Unable to get"
+                   " statefile's name");
+        goto out;
+    }
+
+    /* If state-file entry is missing from the config file,
+     * do not start gsyncd on restart */
+    if (is_template_in_use) {
+        gf_msg(this1->name, GF_LOG_INFO, 0, GD_MSG_NO_STATEFILE_ENTRY,
+               "state-file entry is missing in config file."
+               "Not Restarting");
+        goto out;
+    }
+
+    is_template_in_use = _gf_false;
+
+    ret = gsync_status(volinfo->volname, slave, confpath, &ret_status,
+                       &is_template_in_use);
+    if (ret == -1) {
+        gf_msg(this1->name, GF_LOG_INFO, 0, GD_MSG_GSYNC_VALIDATION_FAIL,
+               GEOREP " start option validation failed ");
+        ret = 0;
+        goto out;
+    }
+
+    if (is_template_in_use == _gf_true) {
+        gf_msg(this1->name, GF_LOG_INFO, 0, GD_MSG_PIDFILE_NOT_FOUND,
+               "pid-file entry is missing in config file."
+               "Not Restarting");
+        ret = 0;
+        goto out;
+    }
+
+    ret = glusterd_gsync_read_frm_status(statefile, buf, sizeof(buf));
+    if (ret <= 0) {
+        gf_msg(this1->name, GF_LOG_ERROR, 0, GD_MSG_STAT_FILE_READ_FAILED,
+               "Unable to read the status");
+        goto out;
+    }
+
+    /* Form key1 which is "<user@><slave_host>::<slavevol>" */
+    snprintf(key1, sizeof(key1), "%s::%s", slave_url, slave_vol);
+
+    /* Looks for the last status, to find if the session was running
+     * when the node went down. If the session was just created or
+     * stopped, do not restart the geo-rep session */
+    if ((!strcmp(buf, "Created")) || (!strcmp(buf, "Stopped"))) {
+        gf_msg(this1->name, GF_LOG_INFO, 0, GD_MSG_GEO_REP_START_FAILED,
+               "Geo-Rep Session was not started between "
+               "%s and %s::%s. Not Restarting",
+               volinfo->volname, slave_url, slave_vol);
+        goto out;
+    } else if (strstr(buf, "Paused")) {
+        is_paused = _gf_true;
+    } else if ((!strcmp(buf, "Config Corrupted"))) {
+        gf_msg(this1->name, GF_LOG_INFO, 0, GD_MSG_RECOVERING_CORRUPT_CONF,
+               "Recovering from a corrupted config. "
+               "Not Restarting. Use start (force) to "
+               "start the session between %s and %s::%s.",
+               volinfo->volname, slave_url, slave_vol);
+        goto out;
+    }
+
+    if (is_paused) {
+        glusterd_start_gsync(volinfo, slave, path_list, confpath, uuid_str,
+                             NULL, _gf_true);
+    } else {
+        /* Add slave to the dict indicating geo-rep session is running*/
+        ret = dict_set_dynstr_with_alloc(volinfo->gsync_active_slaves, key1,
+                                         "running");
+        if (ret) {
+            gf_msg(this1->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+                   "Unable to set key:%s"
+                   " value:running in the dict",
+                   key1);
+            goto out;
+        }
+        ret = glusterd_start_gsync(volinfo, slave, path_list, confpath,
+                                   uuid_str, NULL, _gf_false);
+        if (ret)
+            dict_del(volinfo->gsync_active_slaves, key1);
+    }
+
+out:
+    if (statefile)
+        GF_FREE(statefile);
+    if (slave_url)
+        GF_FREE(slave_url);
+
+    if (is_template_in_use) {
+        op_ret = glusterd_create_status_file(
+            volinfo->volname, slave, slave_host, slave_vol, "Config Corrupted");
+        if (op_ret) {
+            gf_msg(this1->name, GF_LOG_ERROR, 0,
+                   GD_MSG_STATUSFILE_CREATE_FAILED,
+                   "Unable to create status file"
+                   ". Error : %s",
+                   strerror(errno));
+            ret = op_ret;
+        }
+    }
+    if (slave_vol)
+        GF_FREE(slave_vol);
+    GF_FREE(path_list);
+    GF_FREE(op_errstr);
+
+    return ret;
 }
 
 int
-glusterd_volume_restart_gsyncds (glusterd_volinfo_t *volinfo)
+glusterd_volume_restart_gsyncds(glusterd_volinfo_t *volinfo)
 {
-        GF_ASSERT (volinfo);
+    GF_ASSERT(volinfo);
 
-        dict_foreach (volinfo->gsync_slaves, _local_gsyncd_start, volinfo);
-        return 0;
+    dict_foreach(volinfo->gsync_slaves, _local_gsyncd_start, volinfo);
+    return 0;
 }
 
 int
-glusterd_restart_gsyncds (glusterd_conf_t *conf)
+glusterd_restart_gsyncds(glusterd_conf_t *conf)
 {
-        glusterd_volinfo_t       *volinfo = NULL;
-        int                      ret = 0;
+    glusterd_volinfo_t *volinfo = NULL;
+    int ret = 0;
 
-        cds_list_for_each_entry (volinfo, &conf->volumes, vol_list) {
-                glusterd_volume_restart_gsyncds (volinfo);
-        }
-        return ret;
+    cds_list_for_each_entry(volinfo, &conf->volumes, vol_list)
+    {
+        glusterd_volume_restart_gsyncds(volinfo);
+    }
+    return ret;
 }
 
-inline int
-glusterd_get_dist_leaf_count (glusterd_volinfo_t *volinfo)
+int
+glusterd_calc_dist_leaf_count(int rcount, int scount)
+{
+    return (rcount ? rcount : 1) * (scount ? scount : 1);
+}
+
+int
+glusterd_get_dist_leaf_count(glusterd_volinfo_t *volinfo)
 {
     int rcount = volinfo->replica_count;
     int scount = volinfo->stripe_count;
@@ -4544,1862 +7224,2044 @@ glusterd_get_dist_leaf_count (glusterd_volinfo_t *volinfo)
     if (volinfo->type == GF_CLUSTER_TYPE_DISPERSE)
         return volinfo->disperse_count;
 
-    return (rcount ? rcount : 1) * (scount ? scount : 1);
+    return glusterd_calc_dist_leaf_count(rcount, scount);
 }
 
 int
-glusterd_get_brickinfo (xlator_t *this, const char *brickname, int port,
-                        gf_boolean_t localhost, glusterd_brickinfo_t **brickinfo)
-{
-        glusterd_conf_t         *priv = NULL;
-        glusterd_volinfo_t      *volinfo = NULL;
-        glusterd_brickinfo_t    *tmpbrkinfo = NULL;
-        int                     ret = -1;
-
-        GF_ASSERT (brickname);
-        GF_ASSERT (this);
-
-        priv = this->private;
-        cds_list_for_each_entry (volinfo, &priv->volumes, vol_list) {
-                cds_list_for_each_entry (tmpbrkinfo, &volinfo->bricks,
-                                         brick_list) {
-                        if (localhost && !gf_is_local_addr (tmpbrkinfo->hostname))
-                                continue;
-                        if (!strcmp(tmpbrkinfo->path, brickname) &&
-                            (tmpbrkinfo->port == port)) {
-                                *brickinfo = tmpbrkinfo;
-                                return 0;
-                        }
+glusterd_get_brickinfo(xlator_t *this, const char *brickname, int port,
+                       glusterd_brickinfo_t **brickinfo)
+{
+    glusterd_conf_t *priv = NULL;
+    glusterd_volinfo_t *volinfo = NULL;
+    glusterd_brickinfo_t *tmpbrkinfo = NULL;
+    glusterd_snap_t *snap = NULL;
+    int ret = -1;
+
+    GF_ASSERT(brickname);
+    GF_ASSERT(this);
+
+    priv = this->private;
+    cds_list_for_each_entry(volinfo, &priv->volumes, vol_list)
+    {
+        cds_list_for_each_entry(tmpbrkinfo, &volinfo->bricks, brick_list)
+        {
+            if (gf_uuid_compare(tmpbrkinfo->uuid, MY_UUID))
+                continue;
+            if (!strcmp(tmpbrkinfo->path, brickname) &&
+                (tmpbrkinfo->port == port)) {
+                *brickinfo = tmpbrkinfo;
+                return 0;
+            }
+        }
+    }
+    /* In case normal volume is not found, check for snapshot volumes */
+    cds_list_for_each_entry(snap, &priv->snapshots, snap_list)
+    {
+        cds_list_for_each_entry(volinfo, &snap->volumes, vol_list)
+        {
+            cds_list_for_each_entry(tmpbrkinfo, &volinfo->bricks, brick_list)
+            {
+                if (gf_uuid_compare(tmpbrkinfo->uuid, MY_UUID))
+                    continue;
+                if (!strcmp(tmpbrkinfo->path, brickname)) {
+                    *brickinfo = tmpbrkinfo;
+                    return 0;
                 }
+            }
         }
-        return ret;
+    }
+
+    return ret;
 }
 
-glusterd_brickinfo_t*
-glusterd_get_brickinfo_by_position (glusterd_volinfo_t *volinfo, uint32_t pos)
+glusterd_brickinfo_t *
+glusterd_get_brickinfo_by_position(glusterd_volinfo_t *volinfo, uint32_t pos)
 {
-        glusterd_brickinfo_t    *tmpbrkinfo = NULL;
+    glusterd_brickinfo_t *tmpbrkinfo = NULL;
 
-        cds_list_for_each_entry (tmpbrkinfo, &volinfo->bricks, brick_list) {
-                if (pos == 0)
-                        return tmpbrkinfo;
-                pos--;
-        }
-        return NULL;
+    cds_list_for_each_entry(tmpbrkinfo, &volinfo->bricks, brick_list)
+    {
+        if (pos == 0)
+            return tmpbrkinfo;
+        pos--;
+    }
+    return NULL;
 }
 
 void
-glusterd_set_brick_status (glusterd_brickinfo_t  *brickinfo,
-                           gf_brick_status_t status)
-{
-        GF_ASSERT (brickinfo);
-        brickinfo->status = status;
-        if (GF_BRICK_STARTED == status) {
-                gf_log ("glusterd", GF_LOG_DEBUG, "Setting brick %s:%s status "
-                        "to started", brickinfo->hostname, brickinfo->path);
-        } else {
-                gf_log ("glusterd", GF_LOG_DEBUG, "Setting brick %s:%s status "
-                        "to stopped", brickinfo->hostname, brickinfo->path);
-        }
+glusterd_set_brick_status(glusterd_brickinfo_t *brickinfo,
+                          gf_brick_status_t status)
+{
+    GF_ASSERT(brickinfo);
+    brickinfo->status = status;
+    if (GF_BRICK_STARTED == status) {
+        gf_msg_debug("glusterd", 0,
+                     "Setting brick %s:%s status "
+                     "to started",
+                     brickinfo->hostname, brickinfo->path);
+    } else {
+        gf_msg_debug("glusterd", 0,
+                     "Setting brick %s:%s status "
+                     "to stopped",
+                     brickinfo->hostname, brickinfo->path);
+    }
 }
 
 gf_boolean_t
-glusterd_is_brick_started (glusterd_brickinfo_t  *brickinfo)
+glusterd_is_brick_started(glusterd_brickinfo_t *brickinfo)
 {
-        GF_ASSERT (brickinfo);
-        return (brickinfo->status == GF_BRICK_STARTED);
+    GF_ASSERT(brickinfo);
+    return (brickinfo->status == GF_BRICK_STARTED);
 }
 
 int
-glusterd_friend_brick_belongs (glusterd_volinfo_t *volinfo,
-                               glusterd_brickinfo_t *brickinfo, void* uuid)
+glusterd_friend_brick_belongs(glusterd_volinfo_t *volinfo,
+                              glusterd_brickinfo_t *brickinfo, void *uuid)
 {
-        int             ret = -1;
+    int ret = -1;
 
-        GF_ASSERT (volinfo);
-        GF_ASSERT (brickinfo);
-        GF_ASSERT (uuid);
+    GF_ASSERT(volinfo);
+    GF_ASSERT(brickinfo);
+    GF_ASSERT(uuid);
 
-        if (gf_uuid_is_null (brickinfo->uuid)) {
-                ret = glusterd_resolve_brick (brickinfo);
-                if (ret) {
-                        GF_ASSERT (0);
-                        goto out;
-                }
+    if (gf_uuid_is_null(brickinfo->uuid)) {
+        ret = glusterd_resolve_brick(brickinfo);
+        if (ret) {
+            GF_ASSERT(0);
+            goto out;
         }
-        if (!gf_uuid_compare (brickinfo->uuid, *((uuid_t *)uuid)))
-                return 0;
+    }
+    if (!gf_uuid_compare(brickinfo->uuid, *((uuid_t *)uuid)))
+        return 0;
 out:
-        return -1;
+    return -1;
 }
 
 int
-glusterd_get_brick_root (char *path, char **mount_point)
+glusterd_get_brick_root(char *path, char **mount_point)
 {
-        char           *ptr            = NULL;
-        char           *mnt_pt         = NULL;
-        struct stat     brickstat      = {0};
-        struct stat     buf            = {0};
-
-        if (!path)
-                goto err;
-        mnt_pt = gf_strdup (path);
-        if (!mnt_pt)
-                goto err;
-        if (stat (mnt_pt, &brickstat))
-                goto err;
+    char *ptr = NULL;
+    char *mnt_pt = NULL;
+    struct stat brickstat = {0};
+    struct stat buf = {0};
+    xlator_t *this = THIS;
+    GF_ASSERT(this);
 
-        while ((ptr = strrchr (mnt_pt, '/')) &&
-               ptr != mnt_pt) {
+    if (!path) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_INVALID_ARGUMENT, NULL);
+        goto err;
+    }
+    mnt_pt = gf_strdup(path);
+    if (!mnt_pt) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_INVALID_ARGUMENT, NULL);
+        goto err;
+    }
+    if (sys_stat(mnt_pt, &brickstat))
+        goto err;
 
-                *ptr = '\0';
-                if (stat (mnt_pt, &buf)) {
-                        gf_log (THIS->name, GF_LOG_ERROR, "error in "
-                                "stat: %s", strerror (errno));
-                        goto err;
-                }
+    while ((ptr = strrchr(mnt_pt, '/')) && ptr != mnt_pt) {
+        *ptr = '\0';
+        if (sys_stat(mnt_pt, &buf)) {
+            gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_FILE_OP_FAILED,
+                    "Error in stat=%s", strerror(errno), NULL);
+            goto err;
+        }
 
-                if (brickstat.st_dev != buf.st_dev) {
-                        *ptr = '/';
-                        break;
-                }
+        if (brickstat.st_dev != buf.st_dev) {
+            *ptr = '/';
+            break;
         }
+    }
 
-        if (ptr == mnt_pt) {
-                if (stat ("/", &buf)) {
-                        gf_log (THIS->name, GF_LOG_ERROR, "error in "
-                                "stat: %s", strerror (errno));
-                        goto err;
-                }
-                if (brickstat.st_dev == buf.st_dev)
-                        strcpy (mnt_pt, "/");
+    if (ptr == mnt_pt) {
+        if (sys_stat("/", &buf)) {
+            gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_FILE_OP_FAILED,
+                    "Error in stat=%s", strerror(errno), NULL);
+            goto err;
         }
+        if (brickstat.st_dev == buf.st_dev)
+            strcpy(mnt_pt, "/");
+    }
 
-        *mount_point = mnt_pt;
-        return 0;
+    *mount_point = mnt_pt;
+    return 0;
 
- err:
-        GF_FREE (mnt_pt);
-        return -1;
+err:
+    GF_FREE(mnt_pt);
+    return -1;
 }
 
-static char*
-glusterd_parse_inode_size (char *stream, char *pattern)
+static char *
+glusterd_parse_inode_size(char *stream, char *pattern)
 {
-        char *needle = NULL;
-        char *trail  = NULL;
+    char *needle = NULL;
+    char *trail = NULL;
 
-        needle = strstr (stream, pattern);
-        if (!needle)
-                goto out;
+    needle = strstr(stream, pattern);
+    if (!needle)
+        goto out;
 
-        needle = nwstrtail (needle, pattern);
+    needle = nwstrtail(needle, pattern);
 
-        trail = needle;
-        while (trail && isdigit (*trail)) trail++;
-        if (trail)
-                *trail = '\0';
+    trail = needle;
+    while (trail && isdigit(*trail))
+        trail++;
+    if (trail)
+        *trail = '\0';
 
 out:
-        return needle;
+    return needle;
 }
 
 static struct fs_info {
-        char *fs_type_name;
-        char *fs_tool_name;
-        char *fs_tool_arg;
-        char *fs_tool_pattern;
-        char *fs_tool_pkg;
-} glusterd_fs[] = {
-        /* some linux have these in /usr/sbin/and others in /sbin/? */
-        { "xfs", "xfs_info", NULL, "isize=", "xfsprogs" },
-        { "ext3", "tune2fs", "-l", "Inode size:", "e2fsprogs" },
-        { "ext4", "tune2fs", "-l", "Inode size:", "e2fsprogs" },
-        { "btrfs", NULL, NULL, NULL, NULL },
-        { NULL, NULL, NULL, NULL, NULL}
-};
+    char *fs_type_name;
+    char *fs_tool_name;
+    char *fs_tool_arg;
+    char *fs_tool_pattern;
+    char *fs_tool_pkg;
+} glusterd_fs[] = {{"xfs", "xfs_info", NULL, "isize=", "xfsprogs"},
+                   {"ext3", "tune2fs", "-l", "Inode size:", "e2fsprogs"},
+                   {"ext4", "tune2fs", "-l", "Inode size:", "e2fsprogs"},
+                   {"btrfs", NULL, NULL, NULL, NULL},
+                   {"zfs", NULL, NULL, NULL, NULL},
+                   {NULL, NULL, NULL, NULL, NULL}};
 
 static int
-glusterd_add_inode_size_to_dict (dict_t *dict, int count)
-{
-        int             ret               = -1;
-        char            key[1024]         = {0};
-        char            buffer[4096]      = {0};
-        char           *inode_size        = NULL;
-        char           *device            = NULL;
-        char           *fs_name           = NULL;
-        char           *cur_word          = NULL;
-        char           *trail             = NULL;
-        runner_t        runner            = {0, };
-        struct fs_info *fs                = NULL;
-        char            fs_tool_name[256] = {0, };
-        static dict_t  *cached_fs         = NULL;
-
-        memset (key, 0, sizeof (key));
-        snprintf (key, sizeof (key), "brick%d.device", count);
-        ret = dict_get_str (dict, key, &device);
-        if (ret)
-                goto out;
-
-        if (cached_fs) {
-                if (dict_get_str (cached_fs, device, &cur_word) == 0) {
-                        goto cached;
-                }
-        } else {
-                cached_fs = dict_new ();
-        }
-
-        memset (key, 0, sizeof (key));
-        snprintf (key, sizeof (key), "brick%d.fs_name", count);
-        ret = dict_get_str (dict, key, &fs_name);
-        if (ret)
-                goto out;
-
-        runinit (&runner);
-        runner_redir (&runner, STDOUT_FILENO, RUN_PIPE);
-
-        for (fs = glusterd_fs ; fs->fs_type_name; fs++) {
-                if (strcmp (fs_name, fs->fs_type_name) == 0) {
-                        snprintf (fs_tool_name, sizeof (fs_tool_name),
-                                  "/usr/sbin/%s", fs->fs_tool_name);
-                        if (access (fs_tool_name, R_OK|X_OK) == 0)
-                                runner_add_arg (&runner, fs_tool_name);
-                        else {
-                                snprintf (fs_tool_name, sizeof (fs_tool_name),
-                                          "/sbin/%s", fs->fs_tool_name);
-                                if (access (fs_tool_name, R_OK|X_OK) == 0)
-                                        runner_add_arg (&runner, fs_tool_name);
-                        }
-                        break;
-                }
-        }
-
-        if (runner.argv[0]) {
-                if (fs->fs_tool_arg)
-                        runner_add_arg (&runner, fs->fs_tool_arg);
-                runner_add_arg (&runner, device);
-        } else {
-                gf_log (THIS->name, GF_LOG_ERROR, "could not find %s to get"
-                        "inode size for %s (%s): %s package missing?",
-                        fs->fs_tool_name, device, fs_name, fs->fs_tool_pkg);
-                goto out;
-        }
-
-        ret = runner_start (&runner);
-        if (ret) {
-                gf_log (THIS->name, GF_LOG_ERROR, "failed to execute "
-                        "\"%s\": %s", fs->fs_tool_name, strerror (errno));
-                /*
-                 * Runner_start might return an error after the child has
-                 * been forked, e.g. if the program isn't there.  In that
-                 * case, we still need to call runner_end to reap the
-                 * child and free resources.  Fortunately, that seems to
-                 * be harmless for other kinds of failures.
-                 */
-                (void) runner_end (&runner);
-                goto out;
-        }
+glusterd_add_inode_size_to_dict(dict_t *dict, int count)
+{
+    int ret = -1;
+    char key[64];
+    char buffer[4096] = "";
+    char *device = NULL;
+    char *fs_name = NULL;
+    char *cur_word = NULL;
+    char *trail = NULL;
+    runner_t runner = {
+        0,
+    };
+    struct fs_info *fs = NULL;
+    static dict_t *cached_fs = NULL;
+    xlator_t *this = THIS;
+    GF_ASSERT(this);
+
+    ret = snprintf(key, sizeof(key), "brick%d.device", count);
+    ret = dict_get_strn(dict, key, ret, &device);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_GET_FAILED,
+                "Key=%s", key, NULL);
+        goto out;
+    }
+
+    if (cached_fs) {
+        if (dict_get_str(cached_fs, device, &cur_word) == 0) {
+            goto cached;
+        }
+    } else {
+        cached_fs = dict_new();
+    }
+
+    ret = snprintf(key, sizeof(key), "brick%d.fs_name", count);
+    ret = dict_get_strn(dict, key, ret, &fs_name);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_GET_FAILED,
+                "Key=%s", key, NULL);
+        goto out;
+    }
+
+    runinit(&runner);
+    runner_redir(&runner, STDOUT_FILENO, RUN_PIPE);
+
+    for (fs = glusterd_fs; fs->fs_type_name; fs++) {
+        if (strcmp(fs_name, fs->fs_type_name) == 0) {
+            if (!fs->fs_tool_name) {
+                /* dynamic inodes */
+                gf_smsg(this->name, GF_LOG_INFO, 0, GD_MSG_INODE_SIZE_GET_FAIL,
+                        "The brick on device uses dynamic inode sizes",
+                        "Device=%s (%s)", device, fs_name, NULL);
+                cur_word = "N/A";
+                goto cached;
+            }
+            runner_add_arg(&runner, fs->fs_tool_name);
+            break;
+        }
+    }
+
+    if (runner.argv[0]) {
+        if (fs->fs_tool_arg)
+            runner_add_arg(&runner, fs->fs_tool_arg);
+        runner_add_arg(&runner, device);
+    } else {
+        gf_smsg(this->name, GF_LOG_ERROR, 0, GD_MSG_INODE_SIZE_GET_FAIL,
+                "Could not find tool to get inode size for device", "Tool=%s",
+                fs->fs_tool_name, "Device=%s (%s)", device, fs_name,
+                "Missing package=%s ?", fs->fs_tool_pkg, NULL);
+        goto out;
+    }
+
+    ret = runner_start(&runner);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_CMD_EXEC_FAIL,
+                "Failed to execute \"%s\"", fs->fs_tool_name, NULL);
+        /*
+         * Runner_start might return an error after the child has
+         * been forked, e.g. if the program isn't there.  In that
+         * case, we still need to call runner_end to reap the
+         * child and free resources.  Fortunately, that seems to
+         * be harmless for other kinds of failures.
+         */
+        (void)runner_end(&runner);
+        goto out;
+    }
+
+    for (;;) {
+        if (fgets(buffer, sizeof(buffer),
+                  runner_chio(&runner, STDOUT_FILENO)) == NULL)
+            break;
+        trail = strrchr(buffer, '\n');
+        if (trail)
+            *trail = '\0';
 
-        for (;;) {
-                if (fgets (buffer, sizeof (buffer),
-                    runner_chio (&runner, STDOUT_FILENO)) == NULL)
-                        break;
-                trail = strrchr (buffer, '\n');
-                if (trail)
-                        *trail = '\0';
+        cur_word = glusterd_parse_inode_size(buffer, fs->fs_tool_pattern);
 
-                cur_word =
-                        glusterd_parse_inode_size (buffer, fs->fs_tool_pattern);
+        if (cur_word)
+            break;
+    }
 
-                if (cur_word)
-                        break;
-        }
+    ret = runner_end(&runner);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_CMD_EXEC_FAIL,
+                "Tool exited with non-zero exit status", "Tool=%s",
+                fs->fs_tool_name, NULL);
 
-        ret = runner_end (&runner);
-        if (ret) {
-                gf_log (THIS->name, GF_LOG_ERROR,
-                        "%s exited with non-zero exit status",
-                        fs->fs_tool_name);
-
-                goto out;
-        }
-        if (!cur_word) {
-                ret = -1;
-                gf_log (THIS->name, GF_LOG_ERROR,
-                        "Unable to retrieve inode size using %s",
-                        fs->fs_tool_name);
-                goto out;
-        }
+        goto out;
+    }
+    if (!cur_word) {
+        ret = -1;
+        gf_smsg(this->name, GF_LOG_ERROR, 0, GD_MSG_INODE_SIZE_GET_FAIL,
+                "Using Tool=%s", fs->fs_tool_name, NULL);
+        goto out;
+    }
 
-        if (dict_set_dynstr_with_alloc (cached_fs, device, cur_word)) {
-                /* not fatal if not entered into the cache */
-                gf_log (THIS->name, GF_LOG_DEBUG,
-                        "failed to cache fs inode size for %s", device);
-        }
+    if (dict_set_dynstr_with_alloc(cached_fs, device, cur_word)) {
+        /* not fatal if not entered into the cache */
+        gf_msg_debug(this->name, 0, "failed to cache fs inode size for %s",
+                     device);
+    }
 
 cached:
-        memset (key, 0, sizeof (key));
-        snprintf (key, sizeof (key), "brick%d.inode_size", count);
+    snprintf(key, sizeof(key), "brick%d.inode_size", count);
 
-        ret = dict_set_dynstr_with_alloc (dict, key, cur_word);
+    ret = dict_set_dynstr_with_alloc(dict, key, cur_word);
 
 out:
-        if (ret)
-                gf_log (THIS->name, GF_LOG_ERROR, "failed to get inode size");
-        return ret;
+    if (ret)
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_INODE_SIZE_GET_FAIL, NULL);
+    return ret;
 }
 
 struct mntent *
-glusterd_get_mnt_entry_info (char *mnt_pt, char *buff, int buflen,
-                             struct mntent *entry_ptr)
+glusterd_get_mnt_entry_info(char *mnt_pt, char *buff, int buflen,
+                            struct mntent *entry_ptr)
 {
-        struct mntent  *entry   = NULL;
-        FILE           *mtab    = NULL;
+    struct mntent *entry = NULL;
+    FILE *mtab = NULL;
+    char abspath[PATH_MAX] = "";
 
-        GF_ASSERT (mnt_pt);
-        GF_ASSERT (buff);
-        GF_ASSERT (entry_ptr);
+    GF_ASSERT(mnt_pt);
+    GF_ASSERT(buff);
+    GF_ASSERT(entry_ptr);
 
-        mtab = setmntent (_PATH_MOUNTED, "r");
-        if (!mtab)
-                goto out;
+    mtab = setmntent(_PATH_MOUNTED, "r");
+    if (!mtab)
+        goto out;
 
-        entry = getmntent_r (mtab, entry_ptr, buff, buflen);
+    if (!realpath(mnt_pt, abspath)) {
+        gf_msg(THIS->name, GF_LOG_ERROR, 0, GD_MSG_MNTENTRY_GET_FAIL,
+               "realpath () failed for path %s", mnt_pt);
+        goto out;
+    }
 
-        while (1) {
-                if (!entry)
-                        goto out;
+    entry = getmntent_r(mtab, entry_ptr, buff, buflen);
 
-                if (!strcmp (entry->mnt_dir, mnt_pt) &&
-                    strcmp (entry->mnt_type, "rootfs"))
-                        break;
-                entry = getmntent_r (mtab, entry_ptr, buff, buflen);
-        }
+    while (1) {
+        if (!entry)
+            goto out;
+
+        if (!strcmp(entry->mnt_dir, abspath) &&
+            strcmp(entry->mnt_type, "rootfs"))
+            break;
+        entry = getmntent_r(mtab, entry_ptr, buff, buflen);
+    }
 
 out:
-        if (NULL != mtab) {
-                endmntent (mtab);
-        }
-        return entry;
+    if (NULL != mtab) {
+        endmntent(mtab);
+    }
+    return entry;
 }
 
 static int
-glusterd_add_brick_mount_details (glusterd_brickinfo_t *brickinfo,
-                                  dict_t *dict, int count)
-{
-        int             ret                  = -1;
-        char            key[1024]            = {0};
-        char            buff [PATH_MAX]      = {0};
-        char            base_key[1024]       = {0};
-        struct mntent   save_entry           = {0};
-        char           *mnt_pt               = NULL;
-        struct mntent  *entry                = NULL;
-
-        snprintf (base_key, sizeof (base_key), "brick%d", count);
-
-        ret = glusterd_get_brick_root (brickinfo->path, &mnt_pt);
-        if (ret)
-                goto out;
-
-        entry = glusterd_get_mnt_entry_info (mnt_pt, buff, sizeof (buff),
-                                             &save_entry);
-        if (!entry) {
-                ret = -1;
-                goto out;
-        }
+glusterd_add_brick_mount_details(glusterd_brickinfo_t *brickinfo, dict_t *dict,
+                                 int count)
+{
+    int ret = -1;
+    char key[64] = "";
+    char buff[PATH_MAX] = "";
+    char base_key[32] = "";
+    struct mntent save_entry = {0};
+    char *mnt_pt = NULL;
+    struct mntent *entry = NULL;
+    xlator_t *this = THIS;
+    GF_ASSERT(this);
+
+    snprintf(base_key, sizeof(base_key), "brick%d", count);
+
+    ret = glusterd_get_brick_root(brickinfo->path, &mnt_pt);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_BRICKPATH_ROOT_GET_FAIL,
+                NULL);
+        goto out;
+    }
+
+    entry = glusterd_get_mnt_entry_info(mnt_pt, buff, sizeof(buff),
+                                        &save_entry);
+    if (!entry) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_GET_MNT_ENTRY_INFO_FAIL,
+                NULL);
+        ret = -1;
+        goto out;
+    }
 
-        /* get device file */
-        memset (key, 0, sizeof (key));
-        snprintf (key, sizeof (key), "%s.device", base_key);
+    /* get device file */
+    snprintf(key, sizeof(key), "%s.device", base_key);
 
-        ret = dict_set_dynstr_with_alloc (dict, key, entry->mnt_fsname);
-        if (ret)
-                goto out;
+    ret = dict_set_dynstr_with_alloc(dict, key, entry->mnt_fsname);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_SET_FAILED,
+                "Key=%s", key, NULL);
+        goto out;
+    }
 
-        /* fs type */
-        memset (key, 0, sizeof (key));
-        snprintf (key, sizeof (key), "%s.fs_name", base_key);
+    /* fs type */
+    snprintf(key, sizeof(key), "%s.fs_name", base_key);
 
-        ret = dict_set_dynstr_with_alloc (dict, key, entry->mnt_type);
-        if (ret)
-                goto out;
+    ret = dict_set_dynstr_with_alloc(dict, key, entry->mnt_type);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_SET_FAILED,
+                "Key=%s", key, NULL);
+        goto out;
+    }
 
-        /* mount options */
-        memset (key, 0, sizeof (key));
-        snprintf (key, sizeof (key), "%s.mnt_options", base_key);
+    /* mount options */
+    snprintf(key, sizeof(key), "%s.mnt_options", base_key);
 
-        ret = dict_set_dynstr_with_alloc (dict, key, entry->mnt_opts);
+    ret = dict_set_dynstr_with_alloc(dict, key, entry->mnt_opts);
 
- out:
-        GF_FREE (mnt_pt);
+out:
+    if (mnt_pt)
+        GF_FREE(mnt_pt);
 
-        return ret;
+    return ret;
 }
 
-char*
-glusterd_get_brick_mount_device (char *brick_path)
-{
-        int             ret                  = -1;
-        char           *mnt_pt               = NULL;
-        char           *device               = NULL;
-        char            buff [PATH_MAX]      = "";
-        struct mntent  *entry                = NULL;
-        struct mntent   save_entry           = {0,};
-        xlator_t       *this                 = NULL;
-
-        this = THIS;
-        GF_ASSERT (this);
-        GF_ASSERT (brick_path);
-
-        ret = glusterd_get_brick_root (brick_path, &mnt_pt);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "Failed to get mount point "
-                        "for %s brick", brick_path);
-                goto out;
-        }
-
-        entry = glusterd_get_mnt_entry_info (mnt_pt, buff, sizeof (buff),
-                                             &save_entry);
-        if (NULL == entry) {
-                gf_log (this->name, GF_LOG_ERROR, "Failed to get mnt entry "
-                        "for %s mount path", mnt_pt);
-                goto out;
-        }
-
-        /* get the fs_name/device */
-        device = gf_strdup (entry->mnt_fsname);
+char *
+glusterd_get_brick_mount_device(char *brick_path)
+{
+    int ret = -1;
+    char *mnt_pt = NULL;
+    char *device = NULL;
+    char buff[PATH_MAX] = "";
+    struct mntent *entry = NULL;
+    struct mntent save_entry = {
+        0,
+    };
+    xlator_t *this = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+    GF_ASSERT(brick_path);
+
+    ret = glusterd_get_brick_root(brick_path, &mnt_pt);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_BRICKPATH_ROOT_GET_FAIL,
+               "Failed to get mount point "
+               "for %s brick",
+               brick_path);
+        goto out;
+    }
+
+    entry = glusterd_get_mnt_entry_info(mnt_pt, buff, sizeof(buff),
+                                        &save_entry);
+    if (NULL == entry) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_MNTENTRY_GET_FAIL,
+               "Failed to get mnt entry "
+               "for %s mount path",
+               mnt_pt);
+        goto out;
+    }
+
+    /* get the fs_name/device */
+    device = gf_strdup(entry->mnt_fsname);
 
 out:
-        return device;
+    if (mnt_pt)
+        GF_FREE(mnt_pt);
+
+    return device;
 }
 
 int
-glusterd_add_brick_detail_to_dict (glusterd_volinfo_t *volinfo,
-                                   glusterd_brickinfo_t *brickinfo,
-                                   dict_t *dict, int count)
-{
-        int             ret               = -1;
-        uint64_t        memtotal          = 0;
-        uint64_t        memfree           = 0;
-        uint64_t        inodes_total      = 0;
-        uint64_t        inodes_free       = 0;
-        uint64_t        block_size        = 0;
-        char            key[1024]         = {0};
-        char            base_key[1024]    = {0};
-        struct statvfs  brickstat         = {0};
-        xlator_t       *this              = NULL;
-
-        this = THIS;
-        GF_ASSERT (volinfo);
-        GF_ASSERT (brickinfo);
-        GF_ASSERT (dict);
-
-        snprintf (base_key, sizeof (base_key), "brick%d", count);
-
-        ret = statvfs (brickinfo->path, &brickstat);
+glusterd_add_brick_detail_to_dict(glusterd_volinfo_t *volinfo,
+                                  glusterd_brickinfo_t *brickinfo, dict_t *dict,
+                                  int count)
+{
+    int ret = -1;
+    uint64_t memtotal = 0;
+    uint64_t memfree = 0;
+    uint64_t inodes_total = 0;
+    uint64_t inodes_free = 0;
+    uint64_t block_size = 0;
+    char key[64];
+    char base_key[32];
+    struct statvfs brickstat = {0};
+    xlator_t *this = NULL;
+
+    this = THIS;
+    GF_ASSERT(volinfo);
+    GF_ASSERT(brickinfo);
+    GF_ASSERT(dict);
+
+    snprintf(base_key, sizeof(base_key), "brick%d", count);
+
+    ret = sys_statvfs(brickinfo->path, &brickstat);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, errno, GD_MSG_FILE_OP_FAILED,
+               "statfs error: %s ", strerror(errno));
+        goto out;
+    }
+
+    /* file system block size */
+    block_size = brickstat.f_bsize;
+    snprintf(key, sizeof(key), "%s.block_size", base_key);
+    ret = dict_set_uint64(dict, key, block_size);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_SET_FAILED,
+                "Key=%s", key, NULL);
+        goto out;
+    }
+
+    /* free space in brick */
+    memfree = brickstat.f_bfree * brickstat.f_bsize;
+    snprintf(key, sizeof(key), "%s.free", base_key);
+    ret = dict_set_uint64(dict, key, memfree);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_SET_FAILED,
+                "Key=%s", key, NULL);
+        goto out;
+    }
+
+    /* total space of brick */
+    memtotal = brickstat.f_blocks * brickstat.f_bsize;
+    snprintf(key, sizeof(key), "%s.total", base_key);
+    ret = dict_set_uint64(dict, key, memtotal);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_SET_FAILED,
+                "Key=%s", key, NULL);
+        goto out;
+    }
+
+    /* inodes: total and free counts only for ext2/3/4 and xfs */
+    inodes_total = brickstat.f_files;
+    if (inodes_total) {
+        snprintf(key, sizeof(key), "%s.total_inodes", base_key);
+        ret = dict_set_uint64(dict, key, inodes_total);
         if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "statfs error: %s ",
-                        strerror (errno));
-                goto out;
-        }
-
-        /* file system block size */
-        block_size = brickstat.f_bsize;
-        memset (key, 0, sizeof (key));
-        snprintf (key, sizeof (key), "%s.block_size", base_key);
-        ret = dict_set_uint64 (dict, key, block_size);
-        if (ret)
-                goto out;
-
-        /* free space in brick */
-        memfree = brickstat.f_bfree * brickstat.f_bsize;
-        memset (key, 0, sizeof (key));
-        snprintf (key, sizeof (key), "%s.free", base_key);
-        ret = dict_set_uint64 (dict, key, memfree);
-        if (ret)
-                goto out;
-
-        /* total space of brick */
-        memtotal = brickstat.f_blocks * brickstat.f_bsize;
-        memset (key, 0, sizeof (key));
-        snprintf (key, sizeof (key), "%s.total", base_key);
-        ret = dict_set_uint64 (dict, key, memtotal);
-        if (ret)
-                goto out;
-
-        /* inodes: total and free counts only for ext2/3/4 and xfs */
-        inodes_total = brickstat.f_files;
-        if (inodes_total) {
-                memset (key, 0, sizeof (key));
-                snprintf (key, sizeof (key), "%s.total_inodes", base_key);
-                ret = dict_set_uint64 (dict, key, inodes_total);
-                if (ret)
-                        goto out;
+            gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_SET_FAILED,
+                    "Key=%s", key, NULL);
+            goto out;
         }
+    }
 
-        inodes_free = brickstat.f_ffree;
-        if (inodes_free) {
-                memset (key, 0, sizeof (key));
-                snprintf (key, sizeof (key), "%s.free_inodes", base_key);
-                ret = dict_set_uint64 (dict, key, inodes_free);
-                if (ret)
-                        goto out;
+    inodes_free = brickstat.f_ffree;
+    if (inodes_free) {
+        snprintf(key, sizeof(key), "%s.free_inodes", base_key);
+        ret = dict_set_uint64(dict, key, inodes_free);
+        if (ret) {
+            gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_SET_FAILED,
+                    "Key=%s", key, NULL);
+            goto out;
         }
+    }
 
-        ret = glusterd_add_brick_mount_details (brickinfo, dict, count);
-        if (ret)
-                goto out;
+    ret = glusterd_add_brick_mount_details(brickinfo, dict, count);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_ADD_BRICK_MNT_INFO_FAIL,
+                NULL);
+        goto out;
+    }
 
-        ret = glusterd_add_inode_size_to_dict (dict, count);
- out:
-        if (ret)
-                gf_log (this->name, GF_LOG_DEBUG, "Error adding brick"
-                        " detail to dict: %s", strerror (errno));
-        return ret;
+    ret = glusterd_add_inode_size_to_dict(dict, count);
+out:
+    if (ret)
+        gf_msg_debug(this->name, 0,
+                     "Error adding brick"
+                     " detail to dict: %s",
+                     strerror(errno));
+    return ret;
 }
 
 int32_t
-glusterd_add_brick_to_dict (glusterd_volinfo_t *volinfo,
-                            glusterd_brickinfo_t *brickinfo,
-                            dict_t  *dict, int32_t count)
-{
-
-        int             ret                   = -1;
-        int32_t         pid                   = -1;
-        int32_t         brick_online          = -1;
-        char            key[1024]             = {0};
-        char            base_key[1024]        = {0};
-        char            pidfile[PATH_MAX]     = {0};
-        xlator_t        *this                 = NULL;
-        glusterd_conf_t *priv                 = NULL;
-
-        GF_ASSERT (volinfo);
-        GF_ASSERT (brickinfo);
-        GF_ASSERT (dict);
-
-        this = THIS;
-        GF_ASSERT (this);
-
-        priv = this->private;
-
-        snprintf (base_key, sizeof (base_key), "brick%d", count);
-        snprintf (key, sizeof (key), "%s.hostname", base_key);
-
-        ret = dict_set_str (dict, key, brickinfo->hostname);
-        if (ret)
-                goto out;
-
-        memset (key, 0, sizeof (key));
-        snprintf (key, sizeof (key), "%s.path", base_key);
-        ret = dict_set_str (dict, key, brickinfo->path);
-        if (ret)
-                goto out;
-
-        /* add peer uuid */
-        memset (key, 0, sizeof (key));
-        snprintf (key, sizeof (key), "%s.peerid", base_key);
-        ret = dict_set_dynstr_with_alloc (dict, key,
-                                          uuid_utoa (brickinfo->uuid));
-        if (ret) {
-                goto out;
+glusterd_add_brick_to_dict(glusterd_volinfo_t *volinfo,
+                           glusterd_brickinfo_t *brickinfo, dict_t *dict,
+                           int32_t count)
+{
+    int ret = -1;
+    int32_t pid = -1;
+    char key[64];
+    int keylen;
+    char base_key[32];
+    char pidfile[PATH_MAX] = "";
+    xlator_t *this = NULL;
+    glusterd_conf_t *priv = NULL;
+    gf_boolean_t brick_online = _gf_false;
+    char *brickpath = NULL;
+
+    GF_ASSERT(volinfo);
+    GF_ASSERT(brickinfo);
+    GF_ASSERT(dict);
+
+    this = THIS;
+    GF_ASSERT(this);
+
+    priv = this->private;
+
+    snprintf(base_key, sizeof(base_key), "brick%d", count);
+    keylen = snprintf(key, sizeof(key), "%s.hostname", base_key);
+
+    ret = dict_set_strn(dict, key, keylen, brickinfo->hostname);
+    if (ret)
+        goto out;
+
+    keylen = snprintf(key, sizeof(key), "%s.path", base_key);
+    ret = dict_set_strn(dict, key, keylen, brickinfo->path);
+    if (ret)
+        goto out;
+
+    /* add peer uuid */
+    snprintf(key, sizeof(key), "%s.peerid", base_key);
+    ret = dict_set_dynstr_with_alloc(dict, key, uuid_utoa(brickinfo->uuid));
+    if (ret) {
+        goto out;
+    }
+
+    keylen = snprintf(key, sizeof(key), "%s.port", base_key);
+    ret = dict_set_int32n(
+        dict, key, keylen,
+        (volinfo->transport_type == GF_TRANSPORT_RDMA) ? 0 : brickinfo->port);
+    if (ret)
+        goto out;
+
+    keylen = snprintf(key, sizeof(key), "%s.rdma_port", base_key);
+    if (volinfo->transport_type == GF_TRANSPORT_RDMA) {
+        ret = dict_set_int32n(dict, key, keylen, brickinfo->port);
+    } else if (volinfo->transport_type == GF_TRANSPORT_BOTH_TCP_RDMA) {
+        ret = dict_set_int32n(dict, key, keylen, brickinfo->rdma_port);
+    } else
+        ret = dict_set_int32n(dict, key, keylen, 0);
+
+    if (ret)
+        goto out;
+
+    GLUSTERD_GET_BRICK_PIDFILE(pidfile, volinfo, brickinfo, priv);
+
+    if (glusterd_is_brick_started(brickinfo)) {
+        if (gf_is_service_running(pidfile, &pid) &&
+            brickinfo->port_registered) {
+            if (!is_brick_mx_enabled()) {
+                brick_online = _gf_true;
+            } else {
+                brickpath = search_brick_path_from_proc(pid, brickinfo->path);
+                if (!brickpath) {
+                    gf_log(this->name, GF_LOG_INFO,
+                           "brick path %s is not consumed", brickinfo->path);
+                    brick_online = _gf_false;
+                } else {
+                    brick_online = _gf_true;
+                    GF_FREE(brickpath);
+                }
+            }
+        } else {
+            pid = -1;
         }
+    }
 
-        memset (key, 0, sizeof (key));
-        snprintf (key, sizeof (key), "%s.port", base_key);
-        ret = dict_set_int32 (dict, key, (volinfo->transport_type ==
-                              GF_TRANSPORT_RDMA) ? 0 : brickinfo->port);
-        if (ret)
-                goto out;
-
-        memset (key, 0, sizeof (key));
-        snprintf (key, sizeof (key), "%s.rdma_port", base_key);
-        if (volinfo->transport_type == GF_TRANSPORT_RDMA) {
-                ret = dict_set_int32 (dict, key, brickinfo->port);
-        } else if (volinfo->transport_type == GF_TRANSPORT_BOTH_TCP_RDMA) {
-                ret = dict_set_int32 (dict, key, brickinfo->rdma_port);
-        } else
-                ret = dict_set_int32 (dict, key, 0);
-
-        if (ret)
-                goto out;
-
-
-        GLUSTERD_GET_BRICK_PIDFILE (pidfile, volinfo, brickinfo, priv);
+    keylen = snprintf(key, sizeof(key), "%s.pid", base_key);
+    ret = dict_set_int32n(dict, key, keylen, pid);
+    if (ret)
+        goto out;
 
-        brick_online = gf_is_service_running (pidfile, &pid);
-
-        memset (key, 0, sizeof (key));
-        snprintf (key, sizeof (key), "%s.pid", base_key);
-        ret = dict_set_int32 (dict, key, pid);
-        if (ret)
-                goto out;
-
-        memset (key, 0, sizeof (key));
-        snprintf (key, sizeof (key), "%s.status", base_key);
-        ret = dict_set_int32 (dict, key, brick_online);
+    keylen = snprintf(key, sizeof(key), "%s.status", base_key);
+    ret = dict_set_int32n(dict, key, keylen, brick_online);
 
 out:
-        if (ret)
-                gf_log (this->name, GF_LOG_DEBUG, "Returning %d", ret);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_SET_FAILED,
+                "Key=%s", key, NULL);
+        gf_msg_debug(this->name, 0, "Returning %d", ret);
+    }
 
-        return ret;
+    return ret;
 }
 
 int32_t
-glusterd_get_all_volnames (dict_t *dict)
+glusterd_get_all_volnames(dict_t *dict)
 {
-        int                    ret        = -1;
-        int32_t                vol_count  = 0;
-        char                   key[256]   = {0};
-        glusterd_volinfo_t    *entry      = NULL;
-        glusterd_conf_t       *priv       = NULL;
+    int ret = -1;
+    int32_t vol_count = 0;
+    char key[64] = "";
+    int keylen;
+    glusterd_volinfo_t *entry = NULL;
+    glusterd_conf_t *priv = NULL;
 
-        priv = THIS->private;
-        GF_ASSERT (priv);
+    priv = THIS->private;
+    GF_ASSERT(priv);
 
-        cds_list_for_each_entry (entry, &priv->volumes, vol_list) {
-                memset (key, 0, sizeof (key));
-                snprintf (key, sizeof (key), "vol%d", vol_count);
-                ret = dict_set_str (dict, key, entry->volname);
-                if (ret)
-                        goto out;
+    cds_list_for_each_entry(entry, &priv->volumes, vol_list)
+    {
+        keylen = snprintf(key, sizeof(key), "vol%d", vol_count);
+        ret = dict_set_strn(dict, key, keylen, entry->volname);
+        if (ret)
+            goto out;
 
-                vol_count++;
-        }
+        vol_count++;
+    }
 
-        ret = dict_set_int32 (dict, "vol_count", vol_count);
+    ret = dict_set_int32n(dict, "vol_count", SLEN("vol_count"), vol_count);
 
- out:
-        if (ret)
-                gf_log (THIS->name, GF_LOG_ERROR, "failed to get all "
-                        "volume names for status");
-        return ret;
+out:
+    if (ret)
+        gf_msg(THIS->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+               "failed to get all "
+               "volume names for status");
+    return ret;
 }
 
 int
-glusterd_all_volume_cond_check (glusterd_condition_func func, int status,
-                                void *ctx)
+glusterd_all_volume_cond_check(glusterd_condition_func func, int status,
+                               void *ctx)
 {
-        glusterd_conf_t         *priv = NULL;
-        glusterd_volinfo_t      *volinfo = NULL;
-        glusterd_brickinfo_t    *brickinfo = NULL;
-        int                     ret = -1;
-        xlator_t                *this = NULL;
+    glusterd_conf_t *priv = NULL;
+    glusterd_volinfo_t *volinfo = NULL;
+    glusterd_brickinfo_t *brickinfo = NULL;
+    int ret = -1;
+    xlator_t *this = NULL;
 
-        this = THIS;
-        priv = this->private;
+    this = THIS;
+    priv = this->private;
 
-        cds_list_for_each_entry (volinfo, &priv->volumes, vol_list) {
-                cds_list_for_each_entry (brickinfo, &volinfo->bricks,
-                                         brick_list) {
-                        ret = func (volinfo, brickinfo, ctx);
-                        if (ret != status) {
-                                ret = -1;
-                                goto out;
-                        }
-                }
+    cds_list_for_each_entry(volinfo, &priv->volumes, vol_list)
+    {
+        cds_list_for_each_entry(brickinfo, &volinfo->bricks, brick_list)
+        {
+            ret = func(volinfo, brickinfo, ctx);
+            if (ret != status) {
+                ret = -1;
+                goto out;
+            }
         }
-        ret = 0;
+    }
+    ret = 0;
 out:
-        gf_log ("", GF_LOG_DEBUG, "returning %d", ret);
-        return ret;
+    gf_msg_debug("glusterd", 0, "returning %d", ret);
+    return ret;
 }
 
-
 int
-glusterd_brick_stop (glusterd_volinfo_t *volinfo,
-                     glusterd_brickinfo_t *brickinfo,
-                     gf_boolean_t del_brick)
+glusterd_brick_stop(glusterd_volinfo_t *volinfo,
+                    glusterd_brickinfo_t *brickinfo, gf_boolean_t del_brick)
 {
-        int                                     ret   = -1;
-        xlator_t                                *this = NULL;
-        glusterd_conf_t                         *conf = NULL;
+    int ret = -1;
+    xlator_t *this = NULL;
+    glusterd_conf_t *conf = NULL;
 
-        this = THIS;
-        GF_ASSERT (this);
-        conf = this->private;
-        GF_ASSERT (conf);
+    this = THIS;
+    GF_ASSERT(this);
+    conf = this->private;
+    GF_ASSERT(conf);
 
-        if ((!brickinfo) || (!volinfo))
-                goto out;
+    if ((!brickinfo) || (!volinfo)) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_INVALID_ARGUMENT, NULL);
+        goto out;
+    }
 
-        if (gf_uuid_is_null (brickinfo->uuid)) {
-                ret = glusterd_resolve_brick (brickinfo);
-                if (ret) {
-                        gf_log (this->name, GF_LOG_ERROR, FMTSTR_RESOLVE_BRICK,
-                                brickinfo->hostname, brickinfo->path);
-                        goto out;
-                }
-        }
-
-        if (gf_uuid_compare (brickinfo->uuid, MY_UUID)) {
-                ret = 0;
-                if (del_brick)
-                        glusterd_delete_brick (volinfo, brickinfo);
-                goto out;
-        }
-
-        gf_log (this->name, GF_LOG_DEBUG, "About to stop glusterfs"
-                " for brick %s:%s", brickinfo->hostname,
-                brickinfo->path);
-        ret = glusterd_volume_stop_glusterfs (volinfo, brickinfo, del_brick);
+    if (gf_uuid_is_null(brickinfo->uuid)) {
+        ret = glusterd_resolve_brick(brickinfo);
         if (ret) {
-                gf_log (this->name, GF_LOG_CRITICAL, "Unable to stop"
-                        " brick: %s:%s", brickinfo->hostname,
-                        brickinfo->path);
-                goto out;
+            gf_event(EVENT_BRICKPATH_RESOLVE_FAILED,
+                     "peer=%s;volume=%s;brick=%s", brickinfo->hostname,
+                     volinfo->volname, brickinfo->path);
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_RESOLVE_BRICK_FAIL,
+                   FMTSTR_RESOLVE_BRICK, brickinfo->hostname, brickinfo->path);
+            goto out;
         }
+    }
+
+    if (gf_uuid_compare(brickinfo->uuid, MY_UUID)) {
+        ret = 0;
+        if (del_brick)
+            glusterd_delete_brick(volinfo, brickinfo);
+        goto out;
+    }
+
+    ret = glusterd_volume_stop_glusterfs(volinfo, brickinfo, del_brick);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_CRITICAL, 0, GD_MSG_BRICK_STOP_FAIL,
+               "Unable to stop"
+               " brick: %s:%s",
+               brickinfo->hostname, brickinfo->path);
+        goto out;
+    }
 
 out:
-        gf_log (this->name, GF_LOG_DEBUG, "returning %d ", ret);
-        return ret;
+    gf_msg_debug(this->name, 0, "returning %d ", ret);
+    return ret;
 }
 
 int
-glusterd_is_defrag_on (glusterd_volinfo_t *volinfo)
+glusterd_is_defrag_on(glusterd_volinfo_t *volinfo)
 {
-        return (volinfo->rebal.defrag != NULL);
-}
-
-gf_boolean_t
-glusterd_is_rb_ongoing (glusterd_volinfo_t *volinfo)
-{
-        gf_boolean_t     ret = _gf_false;
-
-        GF_ASSERT (volinfo);
-
-        if (glusterd_is_rb_started (volinfo) ||
-            glusterd_is_rb_paused (volinfo))
-                ret = _gf_true;
-
-        return ret;
+    return (volinfo->rebal.defrag != NULL);
 }
 
 int
-glusterd_new_brick_validate (char *brick, glusterd_brickinfo_t *brickinfo,
-                             char *op_errstr, size_t len)
+glusterd_new_brick_validate(char *brick, glusterd_brickinfo_t *brickinfo,
+                            char *op_errstr, size_t len, char *op)
 {
-        glusterd_brickinfo_t    *newbrickinfo = NULL;
-        int                     ret = -1;
-        gf_boolean_t            is_allocated = _gf_false;
-        glusterd_peerinfo_t     *peerinfo = NULL;
-        glusterd_conf_t         *priv = NULL;
-        xlator_t                *this = NULL;
+    glusterd_brickinfo_t *newbrickinfo = NULL;
+    int ret = -1;
+    gf_boolean_t is_allocated = _gf_false;
+    glusterd_peerinfo_t *peerinfo = NULL;
+    glusterd_conf_t *priv = NULL;
+    xlator_t *this = NULL;
 
-        this = THIS;
-        GF_ASSERT (this);
-        priv = this->private;
-        GF_ASSERT (priv);
+    this = THIS;
+    GF_ASSERT(this);
+    priv = this->private;
+    GF_ASSERT(priv);
 
+    GF_ASSERT(brick);
+    GF_ASSERT(op_errstr);
 
-        GF_ASSERT (brick);
-        GF_ASSERT (op_errstr);
-
-        if (!brickinfo) {
-                ret = glusterd_brickinfo_new_from_brick (brick, &newbrickinfo);
-                if (ret)
-                        goto out;
-                is_allocated = _gf_true;
-        } else {
-                newbrickinfo = brickinfo;
+    if (!brickinfo) {
+        ret = glusterd_brickinfo_new_from_brick(brick, &newbrickinfo, _gf_true,
+                                                NULL);
+        if (ret)
+            goto out;
+        is_allocated = _gf_true;
+    } else {
+        newbrickinfo = brickinfo;
+    }
+
+    ret = glusterd_resolve_brick(newbrickinfo);
+    if (ret) {
+        snprintf(op_errstr, len,
+                 "Host %s is not in \'Peer "
+                 "in Cluster\' state",
+                 newbrickinfo->hostname);
+        goto out;
+    }
+
+    if (!gf_uuid_compare(MY_UUID, newbrickinfo->uuid)) {
+        /* brick is local */
+        if (!glusterd_is_brickpath_available(newbrickinfo->uuid,
+                                             newbrickinfo->path)) {
+            snprintf(op_errstr, len,
+                     "Brick: %s not available."
+                     " Brick may be containing or be contained "
+                     "by an existing brick.",
+                     brick);
+            if (op && (!strcmp(op, "GF_RESET_OP_COMMIT") ||
+                       !strcmp(op, "GF_RESET_OP_COMMIT_FORCE")))
+                ret = 1;
+            else
+                ret = -1;
+            goto out;
         }
 
-        ret = glusterd_resolve_brick (newbrickinfo);
-        if (ret) {
-                snprintf(op_errstr, len, "Host %s is not in \'Peer "
-                         "in Cluster\' state", newbrickinfo->hostname);
-                goto out;
+    } else {
+        peerinfo = glusterd_peerinfo_find_by_uuid(newbrickinfo->uuid);
+        if (peerinfo == NULL) {
+            ret = -1;
+            snprintf(op_errstr, len, "Failed to find host %s",
+                     newbrickinfo->hostname);
+            goto out;
         }
 
-        if (!gf_uuid_compare (MY_UUID, newbrickinfo->uuid)) {
-                /* brick is local */
-                if (!glusterd_is_brickpath_available (newbrickinfo->uuid,
-                                                      newbrickinfo->path)) {
-                        snprintf(op_errstr, len, "Brick: %s not available."
-                                 " Brick may be containing or be contained "
-                                 "by an existing brick", brick);
-                        ret = -1;
-                        goto out;
-                }
-
-        } else {
-                peerinfo = glusterd_peerinfo_find_by_uuid (newbrickinfo->uuid);
-                if (peerinfo == NULL) {
-                        ret = -1;
-                        snprintf (op_errstr, len, "Failed to find host %s",
-                                  newbrickinfo->hostname);
-                        goto out;
-                }
-
-                if ((!peerinfo->connected)) {
-                        snprintf(op_errstr, len, "Host %s not connected",
-                                 newbrickinfo->hostname);
-                        ret = -1;
-                        goto out;
-                }
+        if ((!peerinfo->connected)) {
+            snprintf(op_errstr, len, "Host %s not connected",
+                     newbrickinfo->hostname);
+            ret = -1;
+            goto out;
+        }
 
-                if (peerinfo->state.state != GD_FRIEND_STATE_BEFRIENDED) {
-                        snprintf(op_errstr, len, "Host %s is not in \'Peer "
-                                 "in Cluster\' state",
-                                 newbrickinfo->hostname);
-                        ret = -1;
-                        goto out;
-                }
+        if (peerinfo->state.state != GD_FRIEND_STATE_BEFRIENDED) {
+            snprintf(op_errstr, len,
+                     "Host %s is not in \'Peer "
+                     "in Cluster\' state",
+                     newbrickinfo->hostname);
+            ret = -1;
+            goto out;
         }
+    }
 
-        ret = 0;
+    ret = 0;
 out:
-        if (is_allocated)
-                glusterd_brickinfo_delete (newbrickinfo);
-        if (op_errstr[0] != '\0')
-                gf_log (this->name, GF_LOG_ERROR, "%s", op_errstr);
-        gf_log (this->name, GF_LOG_DEBUG, "returning %d ", ret);
-        return ret;
+    if (is_allocated)
+        glusterd_brickinfo_delete(newbrickinfo);
+    if (op_errstr[0] != '\0')
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_BRICK_VALIDATE_FAIL, "%s",
+               op_errstr);
+    gf_msg_debug(this->name, 0, "returning %d ", ret);
+    return ret;
 }
 
 int
-glusterd_is_rb_started(glusterd_volinfo_t *volinfo)
-{
-        gf_log ("", GF_LOG_DEBUG,
-                "is_rb_started:status=%d", volinfo->rep_brick.rb_status);
-        return (volinfo->rep_brick.rb_status == GF_RB_STATUS_STARTED);
-
-}
-
-int
-glusterd_is_rb_paused ( glusterd_volinfo_t *volinfo)
-{
-        gf_log ("", GF_LOG_DEBUG,
-                "is_rb_paused:status=%d", volinfo->rep_brick.rb_status);
-
-        return (volinfo->rep_brick.rb_status == GF_RB_STATUS_PAUSED);
-}
-
-inline int
-glusterd_set_rb_status (glusterd_volinfo_t *volinfo, gf_rb_status_t status)
+glusterd_rb_check_bricks(glusterd_volinfo_t *volinfo, glusterd_brickinfo_t *src,
+                         glusterd_brickinfo_t *dst)
 {
-        gf_log ("", GF_LOG_DEBUG,
-                "setting status from %d to %d",
-                volinfo->rep_brick.rb_status,
-                status);
+    glusterd_replace_brick_t *rb = NULL;
 
-        volinfo->rep_brick.rb_status = status;
-        return 0;
-}
+    GF_ASSERT(volinfo);
 
-inline int
-glusterd_rb_check_bricks (glusterd_volinfo_t *volinfo,
-                          glusterd_brickinfo_t *src, glusterd_brickinfo_t *dst)
-{
-        glusterd_replace_brick_t        *rb = NULL;
+    rb = &volinfo->rep_brick;
 
-        GF_ASSERT (volinfo);
-
-        rb = &volinfo->rep_brick;
-
-        if (!rb->src_brick || !rb->dst_brick)
-                return -1;
+    if (!rb->src_brick || !rb->dst_brick) {
+        gf_smsg("glusterd", GF_LOG_ERROR, errno, GD_MSG_INVALID_ARGUMENT, NULL);
+        return -1;
+    }
 
-        if (strcmp (rb->src_brick->hostname, src->hostname) ||
-            strcmp (rb->src_brick->path, src->path)) {
-                gf_log("", GF_LOG_ERROR, "Replace brick src bricks differ");
-                return -1;
-        }
+    if (strcmp(rb->src_brick->hostname, src->hostname) ||
+        strcmp(rb->src_brick->path, src->path)) {
+        gf_msg("glusterd", GF_LOG_ERROR, 0, GD_MSG_RB_SRC_BRICKS_MISMATCH,
+               "Replace brick src bricks differ");
+        return -1;
+    }
 
-        if (strcmp (rb->dst_brick->hostname, dst->hostname) ||
-            strcmp (rb->dst_brick->path, dst->path)) {
-                gf_log ("", GF_LOG_ERROR, "Replace brick dst bricks differ");
-                return -1;
-        }
+    if (strcmp(rb->dst_brick->hostname, dst->hostname) ||
+        strcmp(rb->dst_brick->path, dst->path)) {
+        gf_msg("glusterd", GF_LOG_ERROR, 0, GD_MSG_RB_DST_BRICKS_MISMATCH,
+               "Replace brick dst bricks differ");
+        return -1;
+    }
 
-        return 0;
+    return 0;
 }
 
 /*path needs to be absolute; works only on gfid, volume-id*/
 static int
-glusterd_is_uuid_present (char *path, char *xattr, gf_boolean_t *present)
+glusterd_is_uuid_present(char *path, char *xattr, gf_boolean_t *present)
 {
-        GF_ASSERT (path);
-        GF_ASSERT (xattr);
-        GF_ASSERT (present);
+    GF_ASSERT(path);
+    GF_ASSERT(xattr);
+    GF_ASSERT(present);
 
-        int     ret      = -1;
-        uuid_t  uid     = {0,};
+    int ret = -1;
+    uuid_t uid = {
+        0,
+    };
 
-        if (!path || !xattr || !present)
-                goto out;
+    if (!path || !xattr || !present)
+        goto out;
 
-        ret = sys_lgetxattr (path, xattr, &uid, 16);
+    ret = sys_lgetxattr(path, xattr, &uid, 16);
 
-        if (ret >= 0) {
-                *present = _gf_true;
-                ret = 0;
-                goto out;
-        }
+    if (ret >= 0) {
+        *present = _gf_true;
+        ret = 0;
+        goto out;
+    }
 
-        switch (errno) {
+    switch (errno) {
 #if defined(ENODATA)
-                case ENODATA: /* FALLTHROUGH */
+        case ENODATA: /* FALLTHROUGH */
 #endif
 #if defined(ENOATTR) && (ENOATTR != ENODATA)
-                case ENOATTR: /* FALLTHROUGH */
+        case ENOATTR: /* FALLTHROUGH */
 #endif
-                case ENOTSUP:
-                        *present = _gf_false;
-                        ret = 0;
-                        break;
-                default:
-                        break;
-        }
+        case ENOTSUP:
+            *present = _gf_false;
+            ret = 0;
+            break;
+        default:
+            break;
+    }
 out:
-        return ret;
+    return ret;
 }
 
 /*path needs to be absolute*/
 static int
-glusterd_is_path_in_use (char *path, gf_boolean_t *in_use, char **op_errstr)
-{
-        int             i               = 0;
-        int             ret             = -1;
-        gf_boolean_t    used            = _gf_false;
-        char            dir[PATH_MAX]   = {0,};
-        char            *curdir         = NULL;
-        char            msg[2048]       = {0};
-        char            *keys[3]         = {GFID_XATTR_KEY,
-                                            GF_XATTR_VOL_ID_KEY,
-                                            NULL};
-
-        GF_ASSERT (path);
-        if (!path)
-                goto out;
+glusterd_is_path_in_use(char *path, gf_boolean_t *in_use, char **op_errstr)
+{
+    int i = 0;
+    int ret = -1;
+    gf_boolean_t used = _gf_false;
+    char dir[PATH_MAX] = "";
+    char *curdir = NULL;
+    char msg[2048] = "";
+    char *keys[3] = {GFID_XATTR_KEY, GF_XATTR_VOL_ID_KEY, NULL};
 
-        strcpy (dir, path);
-        curdir = dir;
-        do {
-                for (i = 0; !used && keys[i]; i++) {
-                        ret = glusterd_is_uuid_present (curdir, keys[i], &used);
-                        if (ret)
-                                goto out;
-                }
+    GF_ASSERT(path);
+    if (!path)
+        goto out;
 
-                if (used)
-                        break;
+    if (snprintf(dir, PATH_MAX, "%s", path) >= PATH_MAX)
+        goto out;
 
-                curdir = dirname (curdir);
-                if (!strcmp (curdir, "."))
-                        goto out;
+    curdir = dir;
+    do {
+        for (i = 0; !used && keys[i]; i++) {
+            ret = glusterd_is_uuid_present(curdir, keys[i], &used);
+            if (ret)
+                goto out;
+        }
 
+        if (used)
+            break;
 
-        } while (strcmp (curdir, "/"));
+        curdir = dirname(curdir);
+        if (!strcmp(curdir, "."))
+            goto out;
 
-        if (!strcmp (curdir, "/")) {
-                for (i = 0; !used && keys[i]; i++) {
-                        ret = glusterd_is_uuid_present (curdir, keys[i], &used);
-                        if (ret)
-                                goto out;
-                }
-        }
+    } while (strcmp(curdir, "/"));
 
-        ret = 0;
-        *in_use = used;
-out:
-        if (ret) {
-                snprintf (msg, sizeof (msg), "Failed to get extended "
-                          "attribute %s, reason: %s", keys[i],
-                          strerror (errno));
+    if (!strcmp(curdir, "/")) {
+        for (i = 0; !used && keys[i]; i++) {
+            ret = glusterd_is_uuid_present(curdir, keys[i], &used);
+            if (ret)
+                goto out;
         }
+    }
 
-        if (*in_use) {
-                if (!strcmp (path, curdir)) {
-                        snprintf (msg, sizeof (msg), "%s is already part of a "
-                          "volume", path);
-                } else {
-                        snprintf (msg, sizeof (msg), "parent directory %s is "
-                          "already part of a volume", curdir);
-                }
+    ret = 0;
+    *in_use = used;
+out:
+    if (ret) {
+        snprintf(msg, sizeof(msg),
+                 "Failed to get extended "
+                 "attribute %s, reason: %s",
+                 keys[i], strerror(errno));
+    }
+
+    if (*in_use) {
+        if (path && curdir && !strcmp(path, curdir)) {
+            snprintf(msg, sizeof(msg),
+                     "%s is already part of a "
+                     "volume",
+                     path);
+        } else {
+            snprintf(msg, sizeof(msg),
+                     "parent directory %s is "
+                     "already part of a volume",
+                     curdir);
         }
+    }
 
-        if (strlen (msg)) {
-                gf_log (THIS->name, GF_LOG_ERROR, "%s", msg);
-                *op_errstr = gf_strdup (msg);
-        }
+    if (strlen(msg)) {
+        gf_msg(THIS->name, GF_LOG_ERROR, errno, GD_MSG_PATH_ALREADY_PART_OF_VOL,
+               "%s", msg);
+        *op_errstr = gf_strdup(msg);
+    }
 
-        return ret;
+    return ret;
 }
 
 int
-glusterd_check_and_set_brick_xattr (char *host, char *path, uuid_t uuid,
-                                    char **op_errstr, gf_boolean_t is_force)
-{
-        int             ret             = -1;
-        char            msg[2048]       = {0,};
-        gf_boolean_t    in_use          = _gf_false;
-        int             flags           = 0;
-
-        /* Check for xattr support in backend fs */
-        ret = sys_lsetxattr (path, "trusted.glusterfs.test",
-                             "working", 8, 0);
-        if (ret == -1) {
-                snprintf (msg, sizeof (msg), "Glusterfs is not"
-                          " supported on brick: %s:%s.\nSetting"
-                          " extended attributes failed, reason:"
-                          " %s.", host, path, strerror(errno));
-                goto out;
-
-        } else {
-                sys_lremovexattr (path, "trusted.glusterfs.test");
-        }
-
-        ret = glusterd_is_path_in_use (path, &in_use, op_errstr);
-        if (ret)
-                goto out;
-
-        if (in_use && !is_force) {
-                ret = -1;
-                goto out;
+glusterd_check_and_set_brick_xattr(char *host, char *path, uuid_t uuid,
+                                   char **op_errstr, gf_boolean_t is_force)
+{
+    int ret = -1;
+    char msg[2048] = "";
+    gf_boolean_t in_use = _gf_false;
+    int flags = 0;
+    xlator_t *this = THIS;
+    GF_ASSERT(this);
+
+    /* Check for xattr support in backend fs */
+    ret = sys_lsetxattr(path, "trusted.glusterfs.test", "working", 8, 0);
+    if (ret == -1) {
+        snprintf(msg, sizeof(msg),
+                 "Glusterfs is not"
+                 " supported on brick: %s:%s.\nSetting"
+                 " extended attributes failed, reason:"
+                 " %s.",
+                 host, path, strerror(errno));
+        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_SET_XATTR_BRICK_FAIL,
+                "Host=%s, Path=%s", host, path, NULL);
+        goto out;
+
+    } else {
+        ret = sys_lremovexattr(path, "trusted.glusterfs.test");
+        if (ret) {
+            snprintf(msg, sizeof(msg),
+                     "Removing test extended"
+                     " attribute failed, reason: %s",
+                     strerror(errno));
+            gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_REMOVE_XATTR_FAIL,
+                    NULL);
+            goto out;
         }
+    }
 
+    ret = glusterd_is_path_in_use(path, &in_use, op_errstr);
+    if (ret)
+        goto out;
 
-        if (!is_force)
-                flags = XATTR_CREATE;
-
-        ret = sys_lsetxattr (path, GF_XATTR_VOL_ID_KEY, uuid, 16,
-                             flags);
-        if (ret == -1) {
-                snprintf (msg, sizeof (msg), "Failed to set extended "
-                          "attributes %s, reason: %s",
-                          GF_XATTR_VOL_ID_KEY, strerror (errno));
-                goto out;
-        }
-
-        ret = 0;
+    if (in_use && !is_force) {
+        ret = -1;
+        goto out;
+    }
+
+    if (!is_force)
+        flags = XATTR_CREATE;
+
+    ret = sys_lsetxattr(path, GF_XATTR_VOL_ID_KEY, uuid, 16, flags);
+    if (ret == -1) {
+        snprintf(msg, sizeof(msg),
+                 "Failed to set extended "
+                 "attributes %s, reason: %s",
+                 GF_XATTR_VOL_ID_KEY, strerror(errno));
+        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_SET_XATTR_FAIL,
+                "Attriutes=%s", GF_XATTR_VOL_ID_KEY, NULL);
+        goto out;
+    }
+
+    ret = 0;
 out:
-        if (strlen (msg))
-                *op_errstr = gf_strdup (msg);
+    if (strlen(msg))
+        *op_errstr = gf_strdup(msg);
 
-        return ret;
+    return ret;
 }
 
-int
-glusterd_sm_tr_log_transition_add_to_dict (dict_t *dict,
-                                           glusterd_sm_tr_log_t *log, int i,
-                                           int count)
-{
-        int     ret = -1;
-        char    key[512] = {0};
-        char    timestr[64] = {0,};
-        char    *str = NULL;
-
-        GF_ASSERT (dict);
-        GF_ASSERT (log);
-
-        memset (key, 0, sizeof (key));
-        snprintf (key, sizeof (key), "log%d-old-state", count);
-        str = log->state_name_get (log->transitions[i].old_state);
-        ret = dict_set_str (dict, key, str);
-        if (ret)
-                goto out;
-
-        memset (key, 0, sizeof (key));
-        snprintf (key, sizeof (key), "log%d-event", count);
-        str = log->event_name_get (log->transitions[i].event);
-        ret = dict_set_str (dict, key, str);
-        if (ret)
-                goto out;
-
-        memset (key, 0, sizeof (key));
-        snprintf (key, sizeof (key), "log%d-new-state", count);
-        str = log->state_name_get (log->transitions[i].new_state);
-        ret = dict_set_str (dict, key, str);
-        if (ret)
-                goto out;
-
-
-        memset (key, 0, sizeof (key));
-        snprintf (key, sizeof (key), "log%d-time", count);
-        gf_time_fmt (timestr, sizeof timestr, log->transitions[i].time,
-                     gf_timefmt_FT);
-        ret = dict_set_dynstr_with_alloc (dict, key, timestr);
-        if (ret)
-                goto out;
+static int
+glusterd_sm_tr_log_transition_add_to_dict(dict_t *dict,
+                                          glusterd_sm_tr_log_t *log, int i,
+                                          int count)
+{
+    int ret = -1;
+    char key[64] = "";
+    int keylen;
+    char timestr[GF_TIMESTR_SIZE] = "";
+    char *str = NULL;
+
+    GF_ASSERT(dict);
+    GF_ASSERT(log);
+
+    keylen = snprintf(key, sizeof(key), "log%d-old-state", count);
+    str = log->state_name_get(log->transitions[i].old_state);
+    ret = dict_set_strn(dict, key, keylen, str);
+    if (ret)
+        goto out;
+
+    keylen = snprintf(key, sizeof(key), "log%d-event", count);
+    str = log->event_name_get(log->transitions[i].event);
+    ret = dict_set_strn(dict, key, keylen, str);
+    if (ret)
+        goto out;
+
+    keylen = snprintf(key, sizeof(key), "log%d-new-state", count);
+    str = log->state_name_get(log->transitions[i].new_state);
+    ret = dict_set_strn(dict, key, keylen, str);
+    if (ret)
+        goto out;
+
+    snprintf(key, sizeof(key), "log%d-time", count);
+    gf_time_fmt(timestr, sizeof timestr, log->transitions[i].time,
+                gf_timefmt_FT);
+    ret = dict_set_dynstr_with_alloc(dict, key, timestr);
+    if (ret)
+        goto out;
 
 out:
-        gf_log ("", GF_LOG_DEBUG, "returning %d", ret);
-        return ret;
+    if (key[0] != '\0' && ret != 0)
+        gf_smsg("glusterd", GF_LOG_ERROR, errno, GD_MSG_DICT_SET_FAILED,
+                "Key=%s", key, NULL);
+    gf_msg_debug("glusterd", 0, "returning %d", ret);
+    return ret;
 }
 
 int
-glusterd_sm_tr_log_add_to_dict (dict_t *dict,
-                                glusterd_sm_tr_log_t *circular_log)
-{
-        int     ret = -1;
-        int     i = 0;
-        int     start = 0;
-        int     end     = 0;
-        int     index = 0;
-        char    key[256] = {0};
-        glusterd_sm_tr_log_t *log = NULL;
-        int     count = 0;
-
-        GF_ASSERT (dict);
-        GF_ASSERT (circular_log);
-
-        log = circular_log;
-        if (!log->count)
-                return 0;
+glusterd_sm_tr_log_add_to_dict(dict_t *dict, glusterd_sm_tr_log_t *circular_log)
+{
+    int ret = -1;
+    int i = 0;
+    int start = 0;
+    int end = 0;
+    int index = 0;
+    char key[16] = {0};
+    glusterd_sm_tr_log_t *log = NULL;
+    int count = 0;
+
+    GF_ASSERT(dict);
+    GF_ASSERT(circular_log);
+
+    log = circular_log;
+    if (!log->count)
+        return 0;
 
-        if (log->count == log->size)
-                start = log->current + 1;
+    if (log->count == log->size)
+        start = log->current + 1;
 
-        end = start + log->count;
-        for (i = start; i < end; i++, count++) {
-                index = i % log->count;
-                ret = glusterd_sm_tr_log_transition_add_to_dict (dict, log, index,
-                                                                 count);
-                if (ret)
-                        goto out;
-        }
+    end = start + log->count;
+    for (i = start; i < end; i++, count++) {
+        index = i % log->count;
+        ret = glusterd_sm_tr_log_transition_add_to_dict(dict, log, index,
+                                                        count);
+        if (ret)
+            goto out;
+    }
 
-        memset (key, 0, sizeof (key));
-        snprintf (key, sizeof (key), "count");
-        ret = dict_set_int32 (dict, key, log->count);
+    ret = snprintf(key, sizeof(key), "count");
+    ret = dict_set_int32n(dict, key, ret, log->count);
 
 out:
-        gf_log ("", GF_LOG_DEBUG, "returning %d", ret);
-        return ret;
+    gf_msg_debug("glusterd", 0, "returning %d", ret);
+    return ret;
 }
 
 int
-glusterd_sm_tr_log_init (glusterd_sm_tr_log_t *log,
-                         char * (*state_name_get) (int),
-                         char * (*event_name_get) (int),
-                         size_t  size)
+glusterd_sm_tr_log_init(glusterd_sm_tr_log_t *log, char *(*state_name_get)(int),
+                        char *(*event_name_get)(int), size_t size)
 {
-        glusterd_sm_transition_t *transitions = NULL;
-        int                      ret = -1;
+    glusterd_sm_transition_t *transitions = NULL;
+    int ret = -1;
 
-        GF_ASSERT (size > 0);
-        GF_ASSERT (log && state_name_get && event_name_get);
+    GF_ASSERT(size > 0);
+    GF_ASSERT(log && state_name_get && event_name_get);
 
-        if (!log || !state_name_get || !event_name_get || (size <= 0))
-                goto out;
+    if (!log || !state_name_get || !event_name_get || (size <= 0))
+        goto out;
 
-        transitions = GF_CALLOC (size, sizeof (*transitions),
-                                 gf_gld_mt_sm_tr_log_t);
-        if (!transitions)
-                goto out;
+    transitions = GF_CALLOC(size, sizeof(*transitions), gf_gld_mt_sm_tr_log_t);
+    if (!transitions)
+        goto out;
 
-        log->transitions = transitions;
-        log->size        = size;
-        log->state_name_get = state_name_get;
-        log->event_name_get = event_name_get;
-        ret = 0;
+    log->transitions = transitions;
+    log->size = size;
+    log->state_name_get = state_name_get;
+    log->event_name_get = event_name_get;
+    ret = 0;
 
 out:
-        gf_log ("", GF_LOG_DEBUG, "returning %d", ret);
-        return ret;
+    gf_msg_debug("glusterd", 0, "returning %d", ret);
+    return ret;
 }
 
 void
-glusterd_sm_tr_log_delete (glusterd_sm_tr_log_t *log)
+glusterd_sm_tr_log_delete(glusterd_sm_tr_log_t *log)
 {
-        if (!log)
-                return;
-        GF_FREE (log->transitions);
+    if (!log)
         return;
+    GF_FREE(log->transitions);
+    return;
 }
 
 int
-glusterd_sm_tr_log_transition_add (glusterd_sm_tr_log_t *log,
-                                   int old_state, int new_state,
-                                   int event)
-{
-        glusterd_sm_transition_t *transitions = NULL;
-        int                      ret = -1;
-        int                      next = 0;
-        xlator_t                 *this = NULL;
-
-        this = THIS;
-        GF_ASSERT (this);
-
-        GF_ASSERT (log);
-        if (!log)
-                goto out;
-
-        transitions = log->transitions;
-        if (!transitions)
-                goto out;
-
-        if (log->count)
-                next = (log->current + 1) % log->size;
-        else
-                next = 0;
-
-        transitions[next].old_state = old_state;
-        transitions[next].new_state = new_state;
-        transitions[next].event     = event;
-        time (&transitions[next].time);
-        log->current = next;
-        if (log->count < log->size)
-                log->count++;
-        ret = 0;
-        gf_log (this->name, GF_LOG_DEBUG, "Transitioning from '%s' to '%s' "
-                "due to event '%s'", log->state_name_get (old_state),
-                log->state_name_get (new_state), log->event_name_get (event));
+glusterd_sm_tr_log_transition_add(glusterd_sm_tr_log_t *log, int old_state,
+                                  int new_state, int event)
+{
+    glusterd_sm_transition_t *transitions = NULL;
+    int ret = -1;
+    int next = 0;
+    xlator_t *this = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+
+    GF_ASSERT(log);
+    if (!log)
+        goto out;
+
+    transitions = log->transitions;
+    if (!transitions)
+        goto out;
+
+    if (log->count)
+        next = (log->current + 1) % log->size;
+    else
+        next = 0;
+
+    transitions[next].old_state = old_state;
+    transitions[next].new_state = new_state;
+    transitions[next].event = event;
+    transitions[next].time = gf_time();
+
+    log->current = next;
+    if (log->count < log->size)
+        log->count++;
+    ret = 0;
+    gf_msg_debug(this->name, 0,
+                 "Transitioning from '%s' to '%s' "
+                 "due to event '%s'",
+                 log->state_name_get(old_state), log->state_name_get(new_state),
+                 log->event_name_get(event));
 out:
-        gf_log (this->name, GF_LOG_DEBUG, "returning %d", ret);
-        return ret;
+    gf_msg_debug(this->name, 0, "returning %d", ret);
+    return ret;
 }
 
 int
-glusterd_remove_pending_entry (struct cds_list_head *list, void *elem)
-{
-        glusterd_pending_node_t *pending_node = NULL;
-        glusterd_pending_node_t *tmp = NULL;
-        int                     ret = 0;
-
-        cds_list_for_each_entry_safe (pending_node, tmp, list, list) {
-                if (elem == pending_node->node) {
-                        cds_list_del_init (&pending_node->list);
-                        GF_FREE (pending_node);
-                        ret = 0;
-                        goto out;
-                }
+glusterd_remove_pending_entry(struct cds_list_head *list, void *elem)
+{
+    glusterd_pending_node_t *pending_node = NULL;
+    glusterd_pending_node_t *tmp = NULL;
+    int ret = 0;
+
+    cds_list_for_each_entry_safe(pending_node, tmp, list, list)
+    {
+        if (elem == pending_node->node) {
+            cds_list_del_init(&pending_node->list);
+            GF_FREE(pending_node);
+            ret = 0;
+            goto out;
         }
+    }
 out:
-        gf_log (THIS->name, GF_LOG_DEBUG, "returning %d", ret);
-        return ret;
-
+    gf_msg_debug(THIS->name, 0, "returning %d", ret);
+    return ret;
 }
 
 int
-glusterd_clear_pending_nodes (struct cds_list_head *list)
+glusterd_clear_pending_nodes(struct cds_list_head *list)
 {
-        glusterd_pending_node_t *pending_node = NULL;
-        glusterd_pending_node_t *tmp = NULL;
+    glusterd_pending_node_t *pending_node = NULL;
+    glusterd_pending_node_t *tmp = NULL;
 
-        cds_list_for_each_entry_safe (pending_node, tmp, list, list) {
-                cds_list_del_init (&pending_node->list);
-                GF_FREE (pending_node);
-        }
+    cds_list_for_each_entry_safe(pending_node, tmp, list, list)
+    {
+        cds_list_del_init(&pending_node->list);
+        GF_FREE(pending_node);
+    }
 
-        return 0;
+    return 0;
 }
 
 int32_t
-glusterd_delete_volume (glusterd_volinfo_t *volinfo)
+glusterd_delete_volume(glusterd_volinfo_t *volinfo)
 {
-        int             ret = -1;
-        GF_ASSERT (volinfo);
+    int ret = -1;
+    GF_ASSERT(volinfo);
 
-        ret = glusterd_store_delete_volume (volinfo);
+    ret = glusterd_store_delete_volume(volinfo);
 
-        if (ret)
-                goto out;
+    if (ret)
+        goto out;
 
-        glusterd_volinfo_remove (volinfo);
+    glusterd_volinfo_remove(volinfo);
 out:
-        gf_log (THIS->name, GF_LOG_DEBUG, "returning %d", ret);
-        return ret;
+    gf_msg_debug(THIS->name, 0, "returning %d", ret);
+    return ret;
 }
 
 int32_t
-glusterd_delete_brick (glusterd_volinfo_t* volinfo,
-                       glusterd_brickinfo_t *brickinfo)
+glusterd_delete_brick(glusterd_volinfo_t *volinfo,
+                      glusterd_brickinfo_t *brickinfo)
 {
-        int             ret = 0;
-        char      voldir[PATH_MAX] = {0,};
-        glusterd_conf_t *priv = THIS->private;
-        GF_ASSERT (volinfo);
-        GF_ASSERT (brickinfo);
+    int ret = 0;
+    char voldir[PATH_MAX] = "";
+    glusterd_conf_t *priv = THIS->private;
+    GF_ASSERT(volinfo);
+    GF_ASSERT(brickinfo);
 
-        GLUSTERD_GET_VOLUME_DIR(voldir, volinfo, priv);
+    GLUSTERD_GET_VOLUME_DIR(voldir, volinfo, priv);
 
-        glusterd_delete_volfile (volinfo, brickinfo);
-        glusterd_store_delete_brick (brickinfo, voldir);
-        glusterd_brickinfo_delete (brickinfo);
-        volinfo->brick_count--;
-        return ret;
+    glusterd_delete_volfile(volinfo, brickinfo);
+    glusterd_store_delete_brick(brickinfo, voldir);
+    glusterd_brickinfo_delete(brickinfo);
+    volinfo->brick_count--;
+    return ret;
 }
 
 int32_t
-glusterd_delete_all_bricks (glusterd_volinfo_t* volinfo)
+glusterd_delete_all_bricks(glusterd_volinfo_t *volinfo)
 {
-        int             ret = 0;
-        glusterd_brickinfo_t *brickinfo = NULL;
-        glusterd_brickinfo_t *tmp = NULL;
+    int ret = 0;
+    glusterd_brickinfo_t *brickinfo = NULL;
+    glusterd_brickinfo_t *tmp = NULL;
 
-        GF_ASSERT (volinfo);
+    GF_ASSERT(volinfo);
 
-        cds_list_for_each_entry_safe (brickinfo, tmp, &volinfo->bricks,
-                                      brick_list) {
-                ret = glusterd_delete_brick (volinfo, brickinfo);
-        }
-        return ret;
+    cds_list_for_each_entry_safe(brickinfo, tmp, &volinfo->bricks, brick_list)
+    {
+        ret = glusterd_delete_brick(volinfo, brickinfo);
+    }
+    return ret;
 }
 
 int
-glusterd_get_local_brickpaths (glusterd_volinfo_t *volinfo, char **pathlist)
-{
-        char                 **path_tokens  = NULL;
-        char                  *tmp_path_list = NULL;
-        char                   path[PATH_MAX] = "";
-        int32_t                count          = 0;
-        int32_t                pathlen        = 0;
-        int32_t                total_len      = 0;
-        int32_t                ret            = 0;
-        int                    i              = 0;
-        glusterd_brickinfo_t  *brickinfo      = NULL;
-
-        if ((!volinfo) || (!pathlist))
-            goto out;
-
-        path_tokens = GF_CALLOC (sizeof(char*), volinfo->brick_count,
-                                 gf_gld_mt_charptr);
-        if (!path_tokens) {
-                gf_log ("", GF_LOG_DEBUG, "Could not allocate memory.");
-                ret = -1;
-                goto out;
-        }
+glusterd_get_local_brickpaths(glusterd_volinfo_t *volinfo, char **pathlist)
+{
+    char **path_tokens = NULL;
+    char *tmp_path_list = NULL;
+    char path[PATH_MAX] = "";
+    int32_t count = 0;
+    int32_t pathlen = 0;
+    int32_t total_len = 0;
+    int32_t ret = 0;
+    int i = 0;
+    glusterd_brickinfo_t *brickinfo = NULL;
+
+    if ((!volinfo) || (!pathlist)) {
+        gf_smsg("glusterd", GF_LOG_ERROR, errno, GD_MSG_INVALID_ARGUMENT, NULL);
+        goto out;
+    }
+
+    path_tokens = GF_CALLOC(sizeof(char *), volinfo->brick_count,
+                            gf_gld_mt_charptr);
+    if (!path_tokens) {
+        gf_msg("glusterd", GF_LOG_ERROR, ENOMEM, GD_MSG_NO_MEMORY,
+               "Could not allocate memory.");
+        ret = -1;
+        goto out;
+    }
 
-        cds_list_for_each_entry (brickinfo, &volinfo->bricks, brick_list) {
-                if (gf_uuid_compare (brickinfo->uuid, MY_UUID))
-                        continue;
+    cds_list_for_each_entry(brickinfo, &volinfo->bricks, brick_list)
+    {
+        if (gf_uuid_compare(brickinfo->uuid, MY_UUID))
+            continue;
 
-                 pathlen = snprintf (path, sizeof(path),
-                                     "--path=%s ", brickinfo->path);
-                 if (pathlen < sizeof(path))
-                        path[pathlen] = '\0';
-                 else
-                        path[sizeof(path)-1] = '\0';
-                 path_tokens[count] = gf_strdup (path);
-                 if (!path_tokens[count]) {
-                        gf_log ("", GF_LOG_DEBUG,
-                                "Could not allocate memory.");
-                        ret = -1;
-                        goto out;
-                 }
-                 count++;
-                 total_len += pathlen;
+        pathlen = snprintf(path, sizeof(path), "--path=%s ", brickinfo->path);
+        if (pathlen < sizeof(path))
+            path[pathlen] = '\0';
+        else
+            path[sizeof(path) - 1] = '\0';
+        path_tokens[count] = gf_strdup(path);
+        if (!path_tokens[count]) {
+            gf_msg("glusterd", GF_LOG_ERROR, ENOMEM, GD_MSG_NO_MEMORY,
+                   "Could not allocate memory.");
+            ret = -1;
+            goto out;
         }
+        count++;
+        total_len += pathlen;
+    }
 
-        tmp_path_list = GF_CALLOC (sizeof(char), total_len + 1,
-                                   gf_gld_mt_char);
-        if (!tmp_path_list) {
-                gf_log ("", GF_LOG_DEBUG, "Could not allocate memory.");
-                ret = -1;
-                goto out;
-        }
+    tmp_path_list = GF_CALLOC(sizeof(char), total_len + 1, gf_gld_mt_char);
+    if (!tmp_path_list) {
+        gf_msg("glusterd", GF_LOG_ERROR, ENOMEM, GD_MSG_NO_MEMORY,
+               "Could not allocate memory.");
+        ret = -1;
+        goto out;
+    }
 
-        for (i = 0; i < count; i++)
-                strcat (tmp_path_list, path_tokens[i]);
+    for (i = 0; i < count; i++)
+        strcat(tmp_path_list, path_tokens[i]);
 
-        if (count)
-                *pathlist = tmp_path_list;
+    if (count)
+        *pathlist = tmp_path_list;
 
-        ret = count;
+    ret = count;
 out:
-        if (path_tokens) {
-                for (i = 0; i < count; i++) {
-                        GF_FREE (path_tokens[i]);
-                }
+    if (path_tokens) {
+        for (i = 0; i < count; i++) {
+            GF_FREE(path_tokens[i]);
         }
+    }
 
-        GF_FREE (path_tokens);
-        path_tokens = NULL;
+    GF_FREE(path_tokens);
+    path_tokens = NULL;
 
-        if (ret == 0) {
-                gf_log ("", GF_LOG_DEBUG, "No Local Bricks Present.");
-                GF_FREE (tmp_path_list);
-                tmp_path_list = NULL;
-        }
+    if (ret == 0) {
+        gf_msg_debug("glusterd", 0, "No Local Bricks Present.");
+        GF_FREE(tmp_path_list);
+        tmp_path_list = NULL;
+    }
 
-        gf_log ("", GF_LOG_DEBUG, "Returning %d", ret);
-        return ret;
+    gf_msg_debug("glusterd", 0, "Returning %d", ret);
+    return ret;
 }
 
 int
-glusterd_start_gsync (glusterd_volinfo_t *master_vol, char *slave,
-                      char *path_list, char *conf_path,
-                      char *glusterd_uuid_str,
-                      char **op_errstr, gf_boolean_t is_pause)
-{
-        int32_t         ret     = 0;
-        int32_t         status  = 0;
-        char            uuid_str [64] = {0};
-        runner_t        runner = {0,};
-        xlator_t        *this = NULL;
-        glusterd_conf_t *priv = NULL;
-        int             errcode = 0;
-        gf_boolean_t    is_template_in_use = _gf_false;
-
-        this = THIS;
-        GF_ASSERT (this);
-        priv = this->private;
-        GF_ASSERT (priv);
-
-        uuid_utoa_r (MY_UUID, uuid_str);
-
-        if (!path_list) {
-                ret = 0;
-                gf_log ("", GF_LOG_DEBUG, "No Bricks in this node."
-                        " Not starting gsyncd.");
-                goto out;
-        }
-
-        ret = gsync_status (master_vol->volname, slave, conf_path,
-                            &status, &is_template_in_use);
-        if (status == 0)
-                goto out;
-
-        if (is_template_in_use == _gf_true) {
-                gf_asprintf (op_errstr, GEOREP" start failed for %s %s : "
-                             "pid-file entry missing in config file",
-                             master_vol->volname, slave);
-                ret = -1;
-                goto out;
-        }
-
-        uuid_utoa_r (master_vol->volume_id, uuid_str);
-        runinit (&runner);
-        runner_add_args  (&runner, GSYNCD_PREFIX"/gsyncd",
-                          path_list, "-c", NULL);
-        runner_argprintf (&runner, "%s", conf_path);
-        runner_argprintf (&runner, ":%s", master_vol->volname);
-        runner_add_args  (&runner, slave, "--config-set", "session-owner",
-                          uuid_str, NULL);
-        synclock_unlock (&priv->big_lock);
-        ret = runner_run (&runner);
-        synclock_lock (&priv->big_lock);
-        if (ret == -1) {
-                errcode = -1;
-                goto out;
-        }
-
-        runinit (&runner);
-        runner_add_args  (&runner, GSYNCD_PREFIX"/gsyncd",
-                          path_list, "--monitor", "-c", NULL);
-        runner_argprintf (&runner, "%s", conf_path);
-        runner_argprintf (&runner, "--iprefix=%s", DATADIR);
-        runner_argprintf (&runner, ":%s", master_vol->volname);
-        runner_argprintf (&runner, "--glusterd-uuid=%s",
-                          uuid_utoa (priv->uuid));
-        runner_add_arg   (&runner, slave);
-        if (is_pause)
-                runner_add_arg   (&runner, "--pause-on-start");
-        synclock_unlock (&priv->big_lock);
-        ret = runner_run (&runner);
-        synclock_lock (&priv->big_lock);
-        if (ret == -1) {
-                gf_asprintf (op_errstr, GEOREP" start failed for %s %s",
-                             master_vol->volname, slave);
-                goto out;
-        }
-
+glusterd_start_gsync(glusterd_volinfo_t *master_vol, char *slave,
+                     char *path_list, char *conf_path, char *glusterd_uuid_str,
+                     char **op_errstr, gf_boolean_t is_pause)
+{
+    int32_t ret = 0;
+    int32_t status = 0;
+    char uuid_str[64] = "";
+    runner_t runner = {
+        0,
+    };
+    xlator_t *this = NULL;
+    glusterd_conf_t *priv = NULL;
+    int errcode = 0;
+    gf_boolean_t is_template_in_use = _gf_false;
+
+    this = THIS;
+    GF_ASSERT(this);
+    priv = this->private;
+    GF_ASSERT(priv);
+
+    uuid_utoa_r(MY_UUID, uuid_str);
+
+    if (!path_list) {
         ret = 0;
+        gf_msg_debug("glusterd", 0,
+                     "No Bricks in this node."
+                     " Not starting gsyncd.");
+        goto out;
+    }
+
+    ret = gsync_status(master_vol->volname, slave, conf_path, &status,
+                       &is_template_in_use);
+    if (status == 0)
+        goto out;
+
+    if (is_template_in_use == _gf_true) {
+        gf_asprintf(op_errstr,
+                    GEOREP
+                    " start failed for %s %s : "
+                    "pid-file entry missing in config file",
+                    master_vol->volname, slave);
+        ret = -1;
+        goto out;
+    }
+
+    uuid_utoa_r(master_vol->volume_id, uuid_str);
+    runinit(&runner);
+    runner_add_args(&runner, GSYNCD_PREFIX "/gsyncd", path_list, "-c", NULL);
+    runner_argprintf(&runner, "%s", conf_path);
+    runner_argprintf(&runner, ":%s", master_vol->volname);
+    runner_add_args(&runner, slave, "--config-set", "session-owner", NULL);
+    runner_argprintf(&runner, "--value=%s", uuid_str);
+    synclock_unlock(&priv->big_lock);
+    ret = runner_run(&runner);
+    synclock_lock(&priv->big_lock);
+    if (ret == -1) {
+        errcode = -1;
+        goto out;
+    }
+
+    runinit(&runner);
+    runner_add_args(&runner, GSYNCD_PREFIX "/gsyncd", path_list, "--monitor",
+                    "-c", NULL);
+    runner_argprintf(&runner, "%s", conf_path);
+    runner_argprintf(&runner, "--iprefix=%s", DATADIR);
+    runner_argprintf(&runner, ":%s", master_vol->volname);
+    runner_argprintf(&runner, "--glusterd-uuid=%s", uuid_utoa(priv->uuid));
+    runner_add_arg(&runner, slave);
+    if (is_pause)
+        runner_add_arg(&runner, "--pause-on-start");
+    synclock_unlock(&priv->big_lock);
+    ret = runner_run(&runner);
+    synclock_lock(&priv->big_lock);
+    if (ret == -1) {
+        gf_asprintf(op_errstr, GEOREP " start failed for %s %s",
+                    master_vol->volname, slave);
+        goto out;
+    }
+
+    ret = 0;
 
 out:
-        if ((ret != 0) && errcode == -1) {
-                if (op_errstr)
-                        *op_errstr = gf_strdup ("internal error, cannot start "
-                                                "the " GEOREP " session");
-        }
+    if ((ret != 0) && errcode == -1) {
+        if (op_errstr)
+            *op_errstr = gf_strdup(
+                "internal error, cannot start "
+                "the " GEOREP " session");
+    }
 
-        gf_log ("", GF_LOG_DEBUG, "Returning %d", ret);
-        return ret;
+    gf_msg_debug("glusterd", 0, "Returning %d", ret);
+    return ret;
 }
 
 int32_t
-glusterd_recreate_volfiles (glusterd_conf_t *conf)
+glusterd_recreate_volfiles(glusterd_conf_t *conf)
 {
+    glusterd_volinfo_t *volinfo = NULL;
+    int ret = 0;
+    int op_ret = 0;
 
-        glusterd_volinfo_t      *volinfo = NULL;
-        int                      ret = 0;
-        int                      op_ret = 0;
+    GF_ASSERT(conf);
 
-        GF_ASSERT (conf);
-        cds_list_for_each_entry (volinfo, &conf->volumes, vol_list) {
-                ret = generate_brick_volfiles (volinfo);
-                if (ret) {
-                        gf_log ("glusterd", GF_LOG_ERROR, "Failed to "
-                                "regenerate brick volfiles for %s",
-                                volinfo->volname);
-                        op_ret = ret;
-                }
-                ret = generate_client_volfiles (volinfo, GF_CLIENT_TRUSTED);
-                if (ret) {
-                        gf_log ("glusterd", GF_LOG_ERROR, "Failed to "
-                                "regenerate trusted client volfiles for %s",
-                                volinfo->volname);
-                        op_ret = ret;
-                }
-                ret = generate_client_volfiles (volinfo, GF_CLIENT_OTHER);
-                if (ret) {
-                        gf_log ("glusterd", GF_LOG_ERROR, "Failed to "
-                                "regenerate client volfiles for %s",
-                                volinfo->volname);
-                        op_ret = ret;
-                }
+    cds_list_for_each_entry(volinfo, &conf->volumes, vol_list)
+    {
+        ret = generate_brick_volfiles(volinfo);
+        if (ret) {
+            gf_msg("glusterd", GF_LOG_ERROR, 0, GD_MSG_VOLFILE_CREATE_FAIL,
+                   "Failed to "
+                   "regenerate brick volfiles for %s",
+                   volinfo->volname);
+            op_ret = ret;
         }
-        return op_ret;
+        ret = generate_client_volfiles(volinfo, GF_CLIENT_TRUSTED);
+        if (ret) {
+            gf_msg("glusterd", GF_LOG_ERROR, 0, GD_MSG_VOLFILE_CREATE_FAIL,
+                   "Failed to "
+                   "regenerate trusted client volfiles for %s",
+                   volinfo->volname);
+            op_ret = ret;
+        }
+        ret = generate_client_volfiles(volinfo, GF_CLIENT_OTHER);
+        if (ret) {
+            gf_msg("glusterd", GF_LOG_ERROR, 0, GD_MSG_VOLFILE_CREATE_FAIL,
+                   "Failed to "
+                   "regenerate client volfiles for %s",
+                   volinfo->volname);
+            op_ret = ret;
+        }
+    }
+    return op_ret;
 }
 
 int32_t
-glusterd_handle_upgrade_downgrade (dict_t *options, glusterd_conf_t *conf)
+glusterd_handle_upgrade_downgrade(dict_t *options, glusterd_conf_t *conf,
+                                  gf_boolean_t upgrade, gf_boolean_t downgrade)
 {
-        int              ret                            = 0;
-        char            *type                           = NULL;
-        gf_boolean_t     upgrade                        = _gf_false;
-        gf_boolean_t     downgrade                      = _gf_false;
-        gf_boolean_t     regenerate_volfiles            = _gf_false;
-        gf_boolean_t     terminate                      = _gf_false;
+    int ret = 0;
+    gf_boolean_t regenerate_volfiles = _gf_false;
+    gf_boolean_t terminate = _gf_false;
 
-        ret = dict_get_str (options, "upgrade", &type);
-        if (!ret) {
-                ret = gf_string2boolean (type, &upgrade);
-                if (ret) {
-                        gf_log ("glusterd", GF_LOG_ERROR, "upgrade option "
-                                "%s is not a valid boolean type", type);
-                        ret = -1;
-                        goto out;
-                }
-                if (_gf_true == upgrade)
-                        regenerate_volfiles = _gf_true;
-        }
+    if (_gf_true == upgrade)
+        regenerate_volfiles = _gf_true;
 
-        ret = dict_get_str (options, "downgrade", &type);
-        if (!ret) {
-                ret = gf_string2boolean (type, &downgrade);
-                if (ret) {
-                        gf_log ("glusterd", GF_LOG_ERROR, "downgrade option "
-                                "%s is not a valid boolean type", type);
-                        ret = -1;
-                        goto out;
-                }
-        }
-
-        if (upgrade && downgrade) {
-                gf_log ("glusterd", GF_LOG_ERROR, "Both upgrade and downgrade"
-                        " options are set. Only one should be on");
-                ret = -1;
-                goto out;
-        }
+    if (upgrade && downgrade) {
+        gf_msg("glusterd", GF_LOG_ERROR, 0, GD_MSG_WRONG_OPTS_SETTING,
+               "Both upgrade and downgrade"
+               " options are set. Only one should be on");
+        ret = -1;
+        goto out;
+    }
 
-        if (!upgrade && !downgrade)
-                ret = 0;
-        else
-                terminate = _gf_true;
-        if (regenerate_volfiles) {
-                ret = glusterd_recreate_volfiles (conf);
-        }
+    if (!upgrade && !downgrade)
+        ret = 0;
+    else
+        terminate = _gf_true;
+    if (regenerate_volfiles) {
+        ret = glusterd_recreate_volfiles(conf);
+    }
 out:
-        if (terminate && (ret == 0))
-                kill (getpid(), SIGTERM);
-        return ret;
+    if (terminate && (ret == 0))
+        kill(getpid(), SIGTERM);
+    return ret;
 }
 
+static inline int
+glusterd_is_replica_volume(int type)
+{
+    if (type == GF_CLUSTER_TYPE_REPLICATE)
+        return 1;
+    return 0;
+}
 gf_boolean_t
-glusterd_is_volume_replicate (glusterd_volinfo_t *volinfo)
+glusterd_is_volume_replicate(glusterd_volinfo_t *volinfo)
 {
-        gf_boolean_t    replicates = _gf_false;
-        if (volinfo && ((volinfo->type == GF_CLUSTER_TYPE_REPLICATE) ||
-            (volinfo->type == GF_CLUSTER_TYPE_STRIPE_REPLICATE)))
-                replicates = _gf_true;
-        return replicates;
+    return glusterd_is_replica_volume((volinfo->type));
 }
 
 gf_boolean_t
-glusterd_is_shd_compatible_volume (glusterd_volinfo_t *volinfo)
+glusterd_is_shd_compatible_type(int type)
 {
-        switch (volinfo->type) {
+    switch (type) {
         case GF_CLUSTER_TYPE_REPLICATE:
-        case GF_CLUSTER_TYPE_STRIPE_REPLICATE:
         case GF_CLUSTER_TYPE_DISPERSE:
-                return _gf_true;
+            return _gf_true;
+    }
+    return _gf_false;
+}
 
-        }
-        return _gf_false;
+gf_boolean_t
+glusterd_is_shd_compatible_volume(glusterd_volinfo_t *volinfo)
+{
+    return glusterd_is_shd_compatible_type(volinfo->type);
 }
 
 int
-glusterd_set_dump_options (char *dumpoptions_path, char *options,
-                           int option_cnt)
+glusterd_set_dump_options(char *dumpoptions_path, char *options, int option_cnt)
 {
-        int              ret         = 0;
-        char            *dup_options = NULL;
-        char            *option      = NULL;
-        char            *tmpptr      = NULL;
-        FILE            *fp          = NULL;
-        int              nfs_cnt     = 0;
-        xlator_t        *this        = NULL;
-        glusterd_conf_t *priv        = NULL;
+    int ret = 0;
+    char *dup_options = NULL;
+    char *option = NULL;
+    char *tmpptr = NULL;
+    FILE *fp = NULL;
+    int nfs_cnt = 0;
+    xlator_t *this = NULL;
+    glusterd_conf_t *priv = NULL;
 
-        this = THIS;
-        GF_ASSERT (this);
+    this = THIS;
+    GF_ASSERT(this);
 
-        priv = this->private;
-        GF_ASSERT (priv);
+    priv = this->private;
+    GF_ASSERT(priv);
 
-        if (0 == option_cnt ||
-            (option_cnt == 1 && (!strcmp (options, "nfs ")))) {
+    if (0 == option_cnt || (option_cnt == 1 && (!strcmp(options, "nfs ")))) {
+        ret = 0;
+        goto out;
+    }
+
+    fp = fopen(dumpoptions_path, "w");
+    if (!fp) {
+        ret = -1;
+        goto out;
+    }
+    dup_options = gf_strdup(options);
+
+    if (!dup_options) {
+        goto out;
+    }
+    gf_msg("glusterd", GF_LOG_INFO, 0, GD_MSG_STATEDUMP_OPTS_RCVD,
+           "Received following statedump options: %s", dup_options);
+    option = strtok_r(dup_options, " ", &tmpptr);
+    while (option) {
+        if (!strcmp(option, priv->nfs_svc.name)) {
+            if (nfs_cnt > 0) {
+                sys_unlink(dumpoptions_path);
                 ret = 0;
                 goto out;
+            }
+            nfs_cnt++;
+            option = strtok_r(NULL, " ", &tmpptr);
+            continue;
         }
-
-        fp = fopen (dumpoptions_path, "w");
-        if (!fp) {
-                ret = -1;
-                goto out;
-        }
-        dup_options = gf_strdup (options);
-        gf_log ("", GF_LOG_INFO, "Received following statedump options: %s",
-                dup_options);
-        option = strtok_r (dup_options, " ", &tmpptr);
-        while (option) {
-                if (!strcmp (option, priv->nfs_svc.name)) {
-                        if (nfs_cnt > 0) {
-                                unlink (dumpoptions_path);
-                                ret = 0;
-                                goto out;
-                        }
-                        nfs_cnt++;
-                        option = strtok_r (NULL, " ", &tmpptr);
-                        continue;
-                }
-                fprintf (fp, "%s=yes\n", option);
-                option = strtok_r (NULL, " ", &tmpptr);
-        }
+        fprintf(fp, "%s=yes\n", option);
+        option = strtok_r(NULL, " ", &tmpptr);
+    }
 
 out:
-        if (fp)
-                fclose (fp);
-        GF_FREE (dup_options);
-        return ret;
+    if (fp)
+        fclose(fp);
+    GF_FREE(dup_options);
+    return ret;
 }
 
-int
-glusterd_brick_statedump (glusterd_volinfo_t *volinfo,
-                          glusterd_brickinfo_t *brickinfo,
-                          char *options, int option_cnt, char **op_errstr)
-{
-        int                     ret = -1;
-        xlator_t                *this = NULL;
-        glusterd_conf_t         *conf = NULL;
-        char                    pidfile_path[PATH_MAX] = {0,};
-        char                    dumpoptions_path[PATH_MAX] = {0,};
-        FILE                    *pidfile = NULL;
-        pid_t                   pid = -1;
-
-        this = THIS;
-        GF_ASSERT (this);
-        conf = this->private;
-        GF_ASSERT (conf);
-
-        if (gf_uuid_is_null (brickinfo->uuid)) {
-                ret = glusterd_resolve_brick (brickinfo);
-                if (ret) {
-                        gf_log ("glusterd", GF_LOG_ERROR,
-                                "Cannot resolve brick %s:%s",
-                                brickinfo->hostname, brickinfo->path);
-                        goto out;
-                }
+static int
+glusterd_brick_signal(glusterd_volinfo_t *volinfo,
+                      glusterd_brickinfo_t *brickinfo, char *options,
+                      int option_cnt, char **op_errstr, int sig)
+{
+    int ret = -1;
+    xlator_t *this = NULL;
+    glusterd_conf_t *conf = NULL;
+    char pidfile_path[PATH_MAX] = "";
+    char dumpoptions_path[PATH_MAX] = "";
+    FILE *pidfile = NULL;
+    pid_t pid = -1;
+
+    this = THIS;
+    GF_ASSERT(this);
+    conf = this->private;
+    GF_ASSERT(conf);
+
+    if (gf_uuid_is_null(brickinfo->uuid)) {
+        ret = glusterd_resolve_brick(brickinfo);
+        if (ret) {
+            gf_msg("glusterd", GF_LOG_ERROR, 0, GD_MSG_RESOLVE_BRICK_FAIL,
+                   "Cannot resolve brick %s:%s", brickinfo->hostname,
+                   brickinfo->path);
+            goto out;
         }
+    }
 
-        if (gf_uuid_compare (brickinfo->uuid, MY_UUID)) {
-                ret = 0;
-                goto out;
-        }
+    if (gf_uuid_compare(brickinfo->uuid, MY_UUID)) {
+        ret = 0;
+        goto out;
+    }
 
-        GLUSTERD_GET_BRICK_PIDFILE (pidfile_path, volinfo, brickinfo, conf);
+    GLUSTERD_GET_BRICK_PIDFILE(pidfile_path, volinfo, brickinfo, conf);
 
-        pidfile = fopen (pidfile_path, "r");
-        if (!pidfile) {
-                gf_log ("", GF_LOG_ERROR, "Unable to open pidfile: %s",
-                        pidfile_path);
-                ret = -1;
-                goto out;
-        }
-
-        ret = fscanf (pidfile, "%d", &pid);
-        if (ret <= 0) {
-                gf_log ("", GF_LOG_ERROR, "Unable to get pid of brick process");
-                ret = -1;
-                goto out;
-        }
+    /* TBD: use gf_is_service_running instead of almost-identical code? */
+    pidfile = fopen(pidfile_path, "r");
+    if (!pidfile) {
+        gf_msg("glusterd", GF_LOG_ERROR, errno, GD_MSG_FILE_OP_FAILED,
+               "Unable to open pidfile: %s", pidfile_path);
+        ret = -1;
+        goto out;
+    }
 
-        snprintf (dumpoptions_path, sizeof (dumpoptions_path),
-                  DEFAULT_VAR_RUN_DIRECTORY"/glusterdump.%d.options", pid);
-        ret = glusterd_set_dump_options (dumpoptions_path, options, option_cnt);
+    ret = fscanf(pidfile, "%d", &pid);
+    if (ret <= 0) {
+        gf_msg("glusterd", GF_LOG_ERROR, errno, GD_MSG_FILE_OP_FAILED,
+               "Unable to get pid of brick process");
+        ret = -1;
+        goto out;
+    }
+
+    if (pid == 0) {
+        gf_msg("glusterd", GF_LOG_WARNING, 0, GD_MSG_NO_SIG_TO_PID_ZERO,
+               "refusing to send signal %d to pid zero", sig);
+        goto out;
+    }
+
+    if (sig == SIGUSR1) {
+        snprintf(dumpoptions_path, sizeof(dumpoptions_path),
+                 DEFAULT_VAR_RUN_DIRECTORY "/glusterdump.%d.options", pid);
+        ret = glusterd_set_dump_options(dumpoptions_path, options, option_cnt);
         if (ret < 0) {
-                gf_log ("", GF_LOG_ERROR, "error while parsing the statedump "
-                        "options");
-                ret = -1;
-                goto out;
+            gf_msg("glusterd", GF_LOG_ERROR, 0, GD_MSG_BRK_STATEDUMP_FAIL,
+                   "error while parsing the statedump "
+                   "options");
+            ret = -1;
+            goto out;
         }
+    }
 
-        gf_log ("", GF_LOG_INFO, "Performing statedump on brick with pid %d",
-                pid);
+    gf_msg("glusterd", GF_LOG_INFO, 0, GD_MSG_STATEDUMP_INFO,
+           "sending signal %d to brick with pid %d", sig, pid);
 
-        kill (pid, SIGUSR1);
+    kill(pid, sig);
 
-        sleep (1);
-        ret = 0;
+    sleep(1);
+    sys_unlink(dumpoptions_path);
+    ret = 0;
 out:
-        unlink (dumpoptions_path);
-        if (pidfile)
-                fclose (pidfile);
-        return ret;
+    if (pidfile)
+        fclose(pidfile);
+    return ret;
 }
 
 int
-glusterd_nfs_statedump (char *options, int option_cnt, char **op_errstr)
-{
-        int                     ret = -1;
-        xlator_t                *this = NULL;
-        glusterd_conf_t         *conf = NULL;
-        char                    pidfile_path[PATH_MAX] = {0,};
-        char                    path[PATH_MAX] = {0,};
-        FILE                    *pidfile = NULL;
-        pid_t                   pid = -1;
-        char                    dumpoptions_path[PATH_MAX] = {0,};
-        char                    *option = NULL;
-        char                    *tmpptr = NULL;
-        char                    *dup_options = NULL;
-        char                    msg[256] = {0,};
-
-        this = THIS;
-        GF_ASSERT (this);
-        conf = this->private;
-        GF_ASSERT (conf);
-
-        dup_options = gf_strdup (options);
-        option = strtok_r (dup_options, " ", &tmpptr);
-        if (strcmp (option, conf->nfs_svc.name)) {
-                snprintf (msg, sizeof (msg), "for nfs statedump, options should"
-                          " be after the key nfs");
-                *op_errstr = gf_strdup (msg);
-                ret = -1;
-                goto out;
-        }
+glusterd_brick_statedump(glusterd_volinfo_t *volinfo,
+                         glusterd_brickinfo_t *brickinfo, char *options,
+                         int option_cnt, char **op_errstr)
+{
+    return glusterd_brick_signal(volinfo, brickinfo, options, option_cnt,
+                                 op_errstr, SIGUSR1);
+}
 
-        GLUSTERD_GET_NFS_DIR (path, conf);
-        GLUSTERD_GET_NFS_PIDFILE (pidfile_path, path);
+int
+glusterd_brick_terminate(glusterd_volinfo_t *volinfo,
+                         glusterd_brickinfo_t *brickinfo, char *options,
+                         int option_cnt, char **op_errstr)
+{
+    return glusterd_brick_signal(volinfo, brickinfo, options, option_cnt,
+                                 op_errstr, SIGTERM);
+}
 
-        pidfile = fopen (pidfile_path, "r");
-        if (!pidfile) {
-                gf_log ("", GF_LOG_ERROR, "Unable to open pidfile: %s",
-                        pidfile_path);
-                ret = -1;
-                goto out;
-        }
+#ifdef BUILD_GNFS
+int
+glusterd_nfs_statedump(char *options, int option_cnt, char **op_errstr)
+{
+    int ret = -1;
+    xlator_t *this = NULL;
+    glusterd_conf_t *conf = NULL;
+    char pidfile_path[PATH_MAX] = "";
+    FILE *pidfile = NULL;
+    pid_t pid = -1;
+    char dumpoptions_path[PATH_MAX] = "";
+    char *option = NULL;
+    char *tmpptr = NULL;
+    char *dup_options = NULL;
+    char msg[256] = "";
+
+    this = THIS;
+    GF_ASSERT(this);
+    conf = this->private;
+    GF_ASSERT(conf);
+
+    dup_options = gf_strdup(options);
+
+    if (!dup_options) {
+        goto out;
+    }
+    option = strtok_r(dup_options, " ", &tmpptr);
+    if (strcmp(option, conf->nfs_svc.name)) {
+        snprintf(msg, sizeof(msg),
+                 "for nfs statedump, options should"
+                 " be after the key nfs");
+        gf_smsg("glusterd", GF_LOG_ERROR, errno, GD_MSG_INVALID_ENTRY,
+                "Options misplaced", NULL);
+        *op_errstr = gf_strdup(msg);
+        ret = -1;
+        goto out;
+    }
 
-        ret = fscanf (pidfile, "%d", &pid);
-        if (ret <= 0) {
-                gf_log ("", GF_LOG_ERROR, "Unable to get pid of brick process");
-                ret = -1;
-                goto out;
-        }
+    GLUSTERD_GET_NFS_PIDFILE(pidfile_path, conf);
 
-        snprintf (dumpoptions_path, sizeof (dumpoptions_path),
-                  DEFAULT_VAR_RUN_DIRECTORY"/glusterdump.%d.options", pid);
-        ret = glusterd_set_dump_options (dumpoptions_path, options, option_cnt);
-        if (ret < 0) {
-                gf_log ("", GF_LOG_ERROR, "error while parsing the statedump "
-                        "options");
-                ret = -1;
-                goto out;
-        }
+    pidfile = fopen(pidfile_path, "r");
+    if (!pidfile) {
+        gf_msg("glusterd", GF_LOG_ERROR, errno, GD_MSG_FILE_OP_FAILED,
+               "Unable to open pidfile: %s", pidfile_path);
+        ret = -1;
+        goto out;
+    }
 
-        gf_log ("", GF_LOG_INFO, "Performing statedump on nfs server with "
-                "pid %d", pid);
+    ret = fscanf(pidfile, "%d", &pid);
+    if (ret <= 0) {
+        gf_msg("glusterd", GF_LOG_ERROR, errno, GD_MSG_FILE_OP_FAILED,
+               "Unable to get pid of brick process");
+        ret = -1;
+        goto out;
+    }
+
+    snprintf(dumpoptions_path, sizeof(dumpoptions_path),
+             DEFAULT_VAR_RUN_DIRECTORY "/glusterdump.%d.options", pid);
+    ret = glusterd_set_dump_options(dumpoptions_path, options, option_cnt);
+    if (ret < 0) {
+        gf_msg("glusterd", GF_LOG_ERROR, 0, GD_MSG_BRK_STATEDUMP_FAIL,
+               "error while parsing the statedump "
+               "options");
+        ret = -1;
+        goto out;
+    }
 
-        kill (pid, SIGUSR1);
+    gf_msg("glusterd", GF_LOG_INFO, 0, GD_MSG_STATEDUMP_INFO,
+           "Performing statedump on nfs server with "
+           "pid %d",
+           pid);
 
-        sleep (1);
+    kill(pid, SIGUSR1);
 
-        ret = 0;
+    sleep(1);
+    /* coverity[TAINTED_STRING] */
+    sys_unlink(dumpoptions_path);
+    ret = 0;
 out:
-        if (pidfile)
-                fclose (pidfile);
-        unlink (dumpoptions_path);
-        GF_FREE (dup_options);
-        return ret;
+    if (pidfile)
+        fclose(pidfile);
+    GF_FREE(dup_options);
+    return ret;
 }
+#endif
 
 int
-glusterd_quotad_statedump (char *options, int option_cnt, char **op_errstr)
-{
-        int                     ret = -1;
-        xlator_t                *this = NULL;
-        glusterd_conf_t         *conf = NULL;
-        char                    pidfile_path[PATH_MAX] = {0,};
-        char                    path[PATH_MAX] = {0,};
-        FILE                    *pidfile = NULL;
-        pid_t                   pid = -1;
-        char                    dumpoptions_path[PATH_MAX] = {0,};
-        char                    *option = NULL;
-        char                    *tmpptr = NULL;
-        char                    *dup_options = NULL;
-        char                    msg[256] = {0,};
-
-        this = THIS;
-        GF_ASSERT (this);
-        conf = this->private;
-        GF_ASSERT (conf);
-
-        dup_options = gf_strdup (options);
-        option = strtok_r (dup_options, " ", &tmpptr);
-        if (strcmp (option, conf->quotad_svc.name)) {
-                snprintf (msg, sizeof (msg), "for quotad statedump, options "
-                          "should be after the key 'quotad'");
-                *op_errstr = gf_strdup (msg);
-                ret = -1;
-                goto out;
-        }
+glusterd_client_statedump(char *volname, char *options, int option_cnt,
+                          char **op_errstr)
+{
+    int ret = 0;
+    char *dup_options = NULL;
+    char *option = NULL;
+    char *tmpptr = NULL;
+    char msg[256] = "";
+    char *target_ip = NULL;
+    char *pid = NULL;
+
+    dup_options = gf_strdup(options);
+    if (!dup_options) {
+        gf_smsg("glusterd", GF_LOG_ERROR, errno, GD_MSG_STRDUP_FAILED,
+                "options=%s", options, NULL);
+        goto out;
+    }
+    option = strtok_r(dup_options, " ", &tmpptr);
+    if (strcmp(option, "client")) {
+        snprintf(msg, sizeof(msg),
+                 "for gluster client statedump, options "
+                 "should be after the key 'client'");
+        gf_smsg("glusterd", GF_LOG_ERROR, errno, GD_MSG_INVALID_ENTRY,
+                "Options misplaced", NULL);
+        *op_errstr = gf_strdup(msg);
+        ret = -1;
+        goto out;
+    }
+    target_ip = strtok_r(NULL, " ", &tmpptr);
+    if (target_ip == NULL) {
+        snprintf(msg, sizeof(msg), "ip address not specified");
+        gf_smsg("glusterd", GF_LOG_ERROR, errno, GD_MSG_INVALID_ENTRY, msg,
+                NULL);
+        *op_errstr = gf_strdup(msg);
+        ret = -1;
+        goto out;
+    }
+
+    pid = strtok_r(NULL, " ", &tmpptr);
+    if (pid == NULL) {
+        snprintf(msg, sizeof(msg), "pid not specified");
+        gf_smsg("glusterd", GF_LOG_ERROR, errno, GD_MSG_INVALID_ENTRY, msg,
+                NULL);
+        *op_errstr = gf_strdup(msg);
+        ret = -1;
+        goto out;
+    }
 
-        GLUSTERD_GET_QUOTAD_DIR (path, conf);
-        GLUSTERD_GET_QUOTAD_PIDFILE (pidfile_path, path);
+    ret = glusterd_client_statedump_submit_req(volname, target_ip, pid);
+out:
+    GF_FREE(dup_options);
+    return ret;
+}
 
-        pidfile = fopen (pidfile_path, "r");
-        if (!pidfile) {
-                gf_log (this->name, GF_LOG_ERROR, "Unable to open pidfile: %s",
-                        pidfile_path);
-                ret = -1;
-                goto out;
-        }
+int
+glusterd_quotad_statedump(char *options, int option_cnt, char **op_errstr)
+{
+    int ret = -1;
+    xlator_t *this = NULL;
+    glusterd_conf_t *conf = NULL;
+    char pidfile_path[PATH_MAX] = "";
+    FILE *pidfile = NULL;
+    pid_t pid = -1;
+    char dumpoptions_path[PATH_MAX] = "";
+    char *option = NULL;
+    char *tmpptr = NULL;
+    char *dup_options = NULL;
+    char msg[256] = "";
+
+    this = THIS;
+    GF_ASSERT(this);
+    conf = this->private;
+    GF_ASSERT(conf);
+
+    dup_options = gf_strdup(options);
+    if (!dup_options) {
+        goto out;
+    }
+    option = strtok_r(dup_options, " ", &tmpptr);
+    if (strcmp(option, conf->quotad_svc.name)) {
+        snprintf(msg, sizeof(msg),
+                 "for quotad statedump, options "
+                 "should be after the key 'quotad'");
+        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_INVALID_ENTRY,
+                "Options misplaced", NULL);
+        *op_errstr = gf_strdup(msg);
+        ret = -1;
+        goto out;
+    }
 
-        ret = fscanf (pidfile, "%d", &pid);
-        if (ret <= 0) {
-                gf_log (this->name, GF_LOG_ERROR, "Unable to get pid of quotad "
-                        "process");
-                ret = -1;
-                goto out;
-        }
+    GLUSTERD_GET_QUOTAD_PIDFILE(pidfile_path, conf);
 
-        snprintf (dumpoptions_path, sizeof (dumpoptions_path),
-                  DEFAULT_VAR_RUN_DIRECTORY"/glusterdump.%d.options", pid);
-        ret = glusterd_set_dump_options (dumpoptions_path, options, option_cnt);
-        if (ret < 0) {
-                gf_log (this->name, GF_LOG_ERROR, "error while parsing "
-                        "statedump options");
-                ret = -1;
-                goto out;
-        }
+    pidfile = fopen(pidfile_path, "r");
+    if (!pidfile) {
+        gf_msg(this->name, GF_LOG_ERROR, errno, GD_MSG_FILE_OP_FAILED,
+               "Unable to open pidfile: %s", pidfile_path);
+        ret = -1;
+        goto out;
+    }
+
+    ret = fscanf(pidfile, "%d", &pid);
+    if (ret <= 0) {
+        gf_msg(this->name, GF_LOG_ERROR, errno, GD_MSG_FILE_OP_FAILED,
+               "Unable to get pid of quotad "
+               "process");
+        ret = -1;
+        goto out;
+    }
+
+    snprintf(dumpoptions_path, sizeof(dumpoptions_path),
+             DEFAULT_VAR_RUN_DIRECTORY "/glusterdump.%d.options", pid);
+    ret = glusterd_set_dump_options(dumpoptions_path, options, option_cnt);
+    if (ret < 0) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_BRK_STATEDUMP_FAIL,
+               "error while parsing "
+               "statedump options");
+        ret = -1;
+        goto out;
+    }
 
-        gf_log (this->name, GF_LOG_INFO, "Performing statedump on quotad with "
-                "pid %d", pid);
+    gf_msg(this->name, GF_LOG_INFO, 0, GD_MSG_STATEDUMP_INFO,
+           "Performing statedump on quotad with "
+           "pid %d",
+           pid);
 
-        kill (pid, SIGUSR1);
+    kill(pid, SIGUSR1);
 
-        sleep (1);
+    sleep(1);
 
-        ret = 0;
+    /* coverity[TAINTED_STRING] */
+    sys_unlink(dumpoptions_path);
+    ret = 0;
 out:
-        if (pidfile)
-                fclose (pidfile);
-        unlink (dumpoptions_path);
-        GF_FREE (dup_options);
-        return ret;
+    if (pidfile)
+        fclose(pidfile);
+    GF_FREE(dup_options);
+    return ret;
 }
 
 /* Checks if the given peer contains bricks belonging to the given volume.
@@ -6409,29 +9271,67 @@ out:
  *   0 - if peer contains no bricks
  */
 int
-glusterd_friend_contains_vol_bricks (glusterd_volinfo_t *volinfo,
-                                     uuid_t friend_uuid)
+glusterd_friend_contains_vol_bricks(glusterd_volinfo_t *volinfo,
+                                    uuid_t friend_uuid)
 {
-        int                     ret = 0;
-        glusterd_brickinfo_t    *brickinfo = NULL;
-        int                     count = 0;
+    int ret = 0;
+    glusterd_brickinfo_t *brickinfo = NULL;
+    int count = 0;
 
-        GF_ASSERT (volinfo);
+    GF_ASSERT(volinfo);
 
-        cds_list_for_each_entry (brickinfo, &volinfo->bricks, brick_list) {
-                if (!gf_uuid_compare (brickinfo->uuid, friend_uuid)) {
-                        count++;
-                }
+    cds_list_for_each_entry(brickinfo, &volinfo->bricks, brick_list)
+    {
+        if (!gf_uuid_compare(brickinfo->uuid, friend_uuid)) {
+            count++;
         }
+    }
+
+    if (count) {
+        if (count == volinfo->brick_count)
+            ret = 2;
+        else
+            ret = 1;
+    }
+    gf_msg_debug(THIS->name, 0, "Returning %d", ret);
+    return ret;
+}
+
+/* Checks if the given peer contains bricks belonging to the given volume.
+ * Returns,
+ *   2 - if peer contains all the bricks
+ *   1 - if peer contains at least 1 brick
+ *   0 - if peer contains no bricks
+ */
+int
+glusterd_friend_contains_snap_bricks(glusterd_snap_t *snapinfo,
+                                     uuid_t friend_uuid)
+{
+    int ret = -1;
+    glusterd_volinfo_t *volinfo = NULL;
+    glusterd_brickinfo_t *brickinfo = NULL;
+    int count = 0;
+
+    GF_VALIDATE_OR_GOTO("glusterd", snapinfo, out);
 
-        if (count) {
-                if (count == volinfo->brick_count)
-                        ret = 2;
-                else
-                        ret = 1;
+    cds_list_for_each_entry(volinfo, &snapinfo->volumes, vol_list)
+    {
+        cds_list_for_each_entry(brickinfo, &volinfo->bricks, brick_list)
+        {
+            if (!gf_uuid_compare(brickinfo->uuid, friend_uuid)) {
+                count++;
+            }
         }
-        gf_log (THIS->name, GF_LOG_DEBUG, "Returning %d", ret);
-        return ret;
+    }
+
+    if (count > 0)
+        ret = 1;
+    else
+        ret = 0;
+
+out:
+    gf_msg_debug(THIS->name, 0, "Returning %d", ret);
+    return ret;
 }
 
 /* Cleanup the stale volumes left behind in the cluster. The volumes which are
@@ -6439,2288 +9339,3295 @@ glusterd_friend_contains_vol_bricks (glusterd_volinfo_t *volinfo,
  * cluster.
  */
 int
-glusterd_friend_remove_cleanup_vols (uuid_t uuid)
-{
-        int                     ret = -1;
-        glusterd_conf_t         *priv = NULL;
-        glusterd_volinfo_t      *volinfo = NULL;
-        glusterd_volinfo_t      *tmp_volinfo = NULL;
-
-        priv = THIS->private;
-        GF_ASSERT (priv);
-
-        cds_list_for_each_entry_safe (volinfo, tmp_volinfo, &priv->volumes,
-                                      vol_list) {
-                if (glusterd_friend_contains_vol_bricks (volinfo, uuid) == 2) {
-                        gf_log (THIS->name, GF_LOG_INFO,
-                                "Deleting stale volume %s", volinfo->volname);
-                        ret = glusterd_delete_volume (volinfo);
-                        if (ret) {
-                                gf_log (THIS->name, GF_LOG_ERROR,
-                                        "Error deleting stale volume");
-                                goto out;
-                        }
+glusterd_friend_remove_cleanup_vols(uuid_t uuid)
+{
+    int ret = -1;
+    glusterd_conf_t *priv = NULL;
+    glusterd_svc_t *svc = NULL;
+    glusterd_volinfo_t *volinfo = NULL;
+    glusterd_volinfo_t *tmp_volinfo = NULL;
+
+    priv = THIS->private;
+    GF_ASSERT(priv);
+
+    cds_list_for_each_entry_safe(volinfo, tmp_volinfo, &priv->volumes, vol_list)
+    {
+        if (!glusterd_friend_contains_vol_bricks(volinfo, MY_UUID)) {
+            /*Stop snapd daemon service if snapd daemon is running*/
+            if (!volinfo->is_snap_volume) {
+                svc = &(volinfo->snapd.svc);
+                ret = svc->stop(svc, SIGTERM);
+                if (ret) {
+                    gf_msg(THIS->name, GF_LOG_ERROR, 0, GD_MSG_SVC_STOP_FAIL,
+                           "Failed "
+                           "to stop snapd daemon service");
                 }
+            }
+
+            if (glusterd_is_shd_compatible_volume(volinfo)) {
+                /*
+                 * Sending stop request for all volumes. So it is fine
+                 * to send stop for mux shd
+                 */
+                svc = &(volinfo->shd.svc);
+                ret = svc->stop(svc, SIGTERM);
+                if (ret) {
+                    gf_msg(THIS->name, GF_LOG_ERROR, 0, GD_MSG_SVC_STOP_FAIL,
+                           "Failed "
+                           "to stop shd daemon service");
+                }
+            }
         }
-        ret = 0;
+
+        if (glusterd_friend_contains_vol_bricks(volinfo, uuid) == 2) {
+            gf_msg(THIS->name, GF_LOG_INFO, 0, GD_MSG_STALE_VOL_DELETE_INFO,
+                   "Deleting stale volume %s", volinfo->volname);
+            ret = glusterd_delete_volume(volinfo);
+            if (ret) {
+                gf_msg(THIS->name, GF_LOG_ERROR, 0,
+                       GD_MSG_STALE_VOL_REMOVE_FAIL,
+                       "Error deleting stale volume");
+                goto out;
+            }
+        }
+    }
+
+    /* Reconfigure all daemon services upon peer detach */
+    ret = glusterd_svcs_reconfigure(NULL);
+    if (ret) {
+        gf_msg(THIS->name, GF_LOG_ERROR, 0, GD_MSG_SVC_STOP_FAIL,
+               "Failed to reconfigure all daemon services.");
+    }
+    ret = 0;
 out:
-        gf_log (THIS->name, GF_LOG_DEBUG, "Returning %d", ret);
-        return ret;
+    gf_msg_debug(THIS->name, 0, "Returning %d", ret);
+    return ret;
 }
 
 int
-glusterd_get_bitd_filepath (char *filepath, glusterd_volinfo_t *volinfo)
+glusterd_get_bitd_filepath(char *filepath, glusterd_volinfo_t *volinfo)
 {
-        int   ret             = 0;
-        char  path[PATH_MAX]  = {0,};
-        glusterd_conf_t *priv = NULL;
+    int ret = 0;
+    char path[PATH_MAX] = "";
+    glusterd_conf_t *priv = NULL;
+    int32_t len = 0;
 
-        priv = THIS->private;
+    priv = THIS->private;
 
-        GLUSTERD_GET_VOLUME_DIR (path, volinfo, priv);
+    GLUSTERD_GET_VOLUME_DIR(path, volinfo, priv);
 
-        snprintf (filepath, PATH_MAX,
-                  "%s/%s-bitd.vol", path, volinfo->volname);
+    len = snprintf(filepath, PATH_MAX, "%s/%s-bitd.vol", path,
+                   volinfo->volname);
+    if ((len < 0) || (len >= PATH_MAX)) {
+        ret = -1;
+    }
 
-        return ret;
+    return ret;
 }
 
 int
-glusterd_get_client_filepath (char *filepath, glusterd_volinfo_t *volinfo,
-                              gf_transport_type type)
+glusterd_get_client_filepath(char *filepath, glusterd_volinfo_t *volinfo,
+                             gf_transport_type type)
 {
-        int   ret             = 0;
-        char  path[PATH_MAX]  = {0,};
-        glusterd_conf_t *priv = NULL;
+    int ret = 0;
+    char path[PATH_MAX] = "";
+    glusterd_conf_t *priv = NULL;
+    int32_t len = 0;
 
-        priv = THIS->private;
+    priv = THIS->private;
 
-        GLUSTERD_GET_VOLUME_DIR (path, volinfo, priv);
+    GLUSTERD_GET_VOLUME_DIR(path, volinfo, priv);
 
-        switch (type) {
+    switch (type) {
         case GF_TRANSPORT_TCP:
-                snprintf (filepath, PATH_MAX,
-                          "%s/%s.tcp-fuse.vol", path, volinfo->volname);
-                break;
+            len = snprintf(filepath, PATH_MAX, "%s/%s.tcp-fuse.vol", path,
+                           volinfo->volname);
+            break;
 
         case GF_TRANSPORT_RDMA:
-                snprintf (filepath, PATH_MAX,
-                          "%s/%s.rdma-fuse.vol", path, volinfo->volname);
-                break;
+            len = snprintf(filepath, PATH_MAX, "%s/%s.rdma-fuse.vol", path,
+                           volinfo->volname);
+            break;
         default:
-                ret = -1;
-                break;
-        }
+            ret = -1;
+            break;
+    }
+    if ((len < 0) || (len >= PATH_MAX)) {
+        ret = -1;
+    }
 
-        return ret;
+    return ret;
 }
 
 int
-glusterd_get_trusted_client_filepath (char *filepath,
-                                      glusterd_volinfo_t *volinfo,
-                                      gf_transport_type type)
+glusterd_get_trusted_client_filepath(char *filepath,
+                                     glusterd_volinfo_t *volinfo,
+                                     gf_transport_type type)
 {
-        int   ret             = 0;
-        char  path[PATH_MAX]  = {0,};
-        glusterd_conf_t *priv = NULL;
+    int ret = 0;
+    char path[PATH_MAX] = "";
+    glusterd_conf_t *priv = NULL;
+    int32_t len = 0;
 
-        priv = THIS->private;
+    priv = THIS->private;
 
-        GLUSTERD_GET_VOLUME_DIR (path, volinfo, priv);
+    GLUSTERD_GET_VOLUME_DIR(path, volinfo, priv);
 
-        switch (type) {
+    switch (type) {
         case GF_TRANSPORT_TCP:
-                snprintf (filepath, PATH_MAX, "%s/trusted-%s.tcp-fuse.vol",
-                          path, volinfo->volname);
-                break;
+            len = snprintf(filepath, PATH_MAX, "%s/trusted-%s.tcp-fuse.vol",
+                           path, volinfo->volname);
+            break;
 
         case GF_TRANSPORT_RDMA:
-                snprintf (filepath, PATH_MAX, "%s/trusted-%s.rdma-fuse.vol",
-                          path, volinfo->volname);
-                break;
+            len = snprintf(filepath, PATH_MAX, "%s/trusted-%s.rdma-fuse.vol",
+                           path, volinfo->volname);
+            break;
         default:
-                ret = -1;
-                break;
-        }
+            ret = -1;
+            break;
+    }
+    if ((len < 0) || (len >= PATH_MAX)) {
+        ret = -1;
+    }
 
-        return ret;
+    return ret;
 }
 
 int
-glusterd_volume_defrag_restart (glusterd_volinfo_t *volinfo, char *op_errstr,
-                              size_t len, int cmd, defrag_cbk_fn_t cbk)
+glusterd_get_dummy_client_filepath(char *filepath, glusterd_volinfo_t *volinfo,
+                                   gf_transport_type type)
 {
-        xlator_t        *this             = NULL;
-        glusterd_conf_t *priv             = NULL;
-        char            pidfile[PATH_MAX] = {0,};
-        int             ret               = -1;
-        pid_t           pid               = 0;
+    int ret = 0;
 
-        this = THIS;
-        GF_ASSERT (this);
-
-        priv = this->private;
-        if (!priv)
-                return ret;
+    switch (type) {
+        case GF_TRANSPORT_TCP:
+        case GF_TRANSPORT_BOTH_TCP_RDMA:
+            snprintf(filepath, PATH_MAX, "/tmp/%s.tcp-fuse.vol",
+                     volinfo->volname);
+            break;
 
-        /* Don't start the rebalance process if the stautus is already
-         * completed, stopped or failed. If the status is started, check if
-         * there is an existing process already and connect to it. If not, then
-         * start the rebalance process
-         */
-        switch (volinfo->rebal.defrag_status) {
-        case GF_DEFRAG_STATUS_COMPLETE:
-        case GF_DEFRAG_STATUS_STOPPED:
-        case GF_DEFRAG_STATUS_FAILED:
-                break;
-        case GF_DEFRAG_STATUS_STARTED:
-                GLUSTERD_GET_DEFRAG_PID_FILE(pidfile, volinfo, priv);
-                if (gf_is_service_running (pidfile, &pid)) {
-                        glusterd_rebalance_rpc_create (volinfo, _gf_true);
-                        break;
-                }
-        case GF_DEFRAG_STATUS_NOT_STARTED:
-                glusterd_handle_defrag_start (volinfo, op_errstr, len, cmd,
-                                              cbk, volinfo->rebal.op);
-                break;
+        case GF_TRANSPORT_RDMA:
+            snprintf(filepath, PATH_MAX, "/tmp/%s.rdma-fuse.vol",
+                     volinfo->volname);
+            break;
         default:
-                gf_log (this->name, GF_LOG_ERROR, "Unknown defrag status (%d)."
-                        "Not starting rebalance process for %s.",
-                        volinfo->rebal.defrag_status, volinfo->volname);
-                break;
-        }
+            ret = -1;
+            break;
+    }
 
-        return ret;
+    return ret;
 }
 
 int
-glusterd_restart_rebalance (glusterd_conf_t *conf)
+glusterd_volume_defrag_restart(glusterd_volinfo_t *volinfo, char *op_errstr,
+                               size_t len, int cmd, defrag_cbk_fn_t cbk)
 {
-        glusterd_volinfo_t       *volinfo = NULL;
-        int                      ret = 0;
-        char                     op_errstr[256];
+    xlator_t *this = NULL;
+    glusterd_conf_t *priv = NULL;
+    char pidfile[PATH_MAX] = "";
+    int ret = -1;
+    pid_t pid = 0;
 
-        cds_list_for_each_entry (volinfo, &conf->volumes, vol_list) {
-                if (!volinfo->rebal.defrag_cmd)
-                        continue;
-                if (!gd_should_i_start_rebalance (volinfo))
-                        continue;
-                glusterd_volume_defrag_restart (volinfo, op_errstr, 256,
-                                        volinfo->rebal.defrag_cmd, NULL);
-        }
+    this = THIS;
+    GF_ASSERT(this);
+
+    priv = this->private;
+    if (!priv)
         return ret;
+
+    /* Don't start the rebalance process if the stautus is already
+     * completed, stopped or failed. If the status is started, check if
+     * there is an existing process already and connect to it. If not, then
+     * start the rebalance process
+     */
+
+    switch (volinfo->rebal.defrag_status) {
+        case GF_DEFRAG_STATUS_COMPLETE:
+        case GF_DEFRAG_STATUS_STOPPED:
+        case GF_DEFRAG_STATUS_FAILED:
+            break;
+        case GF_DEFRAG_STATUS_STARTED:
+            GLUSTERD_GET_DEFRAG_PID_FILE(pidfile, volinfo, priv);
+            if (gf_is_service_running(pidfile, &pid)) {
+                ret = glusterd_rebalance_defrag_init(volinfo, cbk);
+                if (ret) {
+                    gf_msg(this->name, GF_LOG_ERROR, 0,
+                           GD_MSG_REBALANCE_START_FAIL,
+                           "Failed to initialize  defrag."
+                           "Not starting rebalance process for "
+                           "%s.",
+                           volinfo->volname);
+                    gf_event(EVENT_REBALANCE_START_FAILED, "volume=%s",
+                             volinfo->volname);
+                    goto out;
+                }
+                ret = glusterd_rebalance_rpc_create(volinfo);
+                break;
+            }
+        case GF_DEFRAG_STATUS_NOT_STARTED:
+            ret = glusterd_handle_defrag_start(volinfo, op_errstr, len, cmd,
+                                               cbk, volinfo->rebal.op);
+            if (ret) {
+                volinfo->rebal.defrag_status = GF_DEFRAG_STATUS_FAILED;
+                gf_event(EVENT_REBALANCE_START_FAILED, "volume=%s",
+                         volinfo->volname);
+            }
+            break;
+        default:
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_REBALANCE_START_FAIL,
+                   "Unknown defrag status (%d)."
+                   "Not starting rebalance process for %s.",
+                   volinfo->rebal.defrag_status, volinfo->volname);
+            break;
+    }
+out:
+    return ret;
 }
 
 void
-glusterd_volinfo_reset_defrag_stats (glusterd_volinfo_t *volinfo)
+glusterd_defrag_info_set(glusterd_volinfo_t *volinfo, dict_t *dict, int cmd,
+                         int status, int op)
 {
-        glusterd_rebalance_t *rebal = NULL;
-        GF_ASSERT (volinfo);
+    xlator_t *this = NULL;
+    int ret = -1;
+    char *task_id_str = NULL;
+    glusterd_rebalance_t *rebal = NULL;
 
-        rebal = &volinfo->rebal;
-        rebal->rebalance_files = 0;
-        rebal->rebalance_data = 0;
-        rebal->lookedup_files = 0;
-        rebal->rebalance_failures = 0;
-        rebal->rebalance_time = 0;
-        rebal->skipped_files = 0;
+    this = THIS;
+    rebal = &volinfo->rebal;
 
-}
+    rebal->defrag_cmd = cmd;
+    rebal->defrag_status = status;
+    rebal->op = op;
 
-gf_boolean_t
-glusterd_is_local_brick (xlator_t *this, glusterd_volinfo_t *volinfo,
-                         glusterd_brickinfo_t *brickinfo)
-{
-        gf_boolean_t    local = _gf_false;
-        int             ret = 0;
-        glusterd_conf_t *conf = NULL;
+    if (gf_uuid_is_null(rebal->rebalance_id))
+        return;
 
-        if (gf_uuid_is_null (brickinfo->uuid)) {
-                ret = glusterd_resolve_brick (brickinfo);
-                if (ret)
-                        goto out;
+    if (is_origin_glusterd(dict)) {
+        ret = glusterd_generate_and_set_task_id(dict, GF_REBALANCE_TID_KEY,
+                                                SLEN(GF_REBALANCE_TID_KEY));
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_TASKID_GEN_FAIL,
+                   "Failed to generate task-id");
+            goto out;
         }
-        conf = this->private;
-        local = !gf_uuid_compare (brickinfo->uuid, MY_UUID);
+    }
+    ret = dict_get_strn(dict, GF_REBALANCE_TID_KEY, SLEN(GF_REBALANCE_TID_KEY),
+                        &task_id_str);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_WARNING, 0, GD_MSG_REBALANCE_ID_MISSING,
+               "Missing rebalance-id");
+        ret = 0;
+        goto out;
+    }
+
+    gf_uuid_parse(task_id_str, rebal->rebalance_id);
 out:
-        return local;
+
+    if (ret) {
+        gf_msg_debug(this->name, 0, "Rebalance start validate failed");
+    }
+    return;
 }
+
 int
-glusterd_validate_volume_id (dict_t *op_dict, glusterd_volinfo_t *volinfo)
+glusterd_restart_rebalance_for_volume(glusterd_volinfo_t *volinfo)
 {
-        int     ret             = -1;
-        char    *volid_str      = NULL;
-        uuid_t  vol_uid         = {0, };
-        xlator_t *this          = NULL;
-
-        this = THIS;
-        GF_ASSERT (this);
-
-        ret = dict_get_str (op_dict, "vol-id", &volid_str);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "Failed to get volume id for "
-                        "volume %s", volinfo->volname);
-                goto out;
-        }
-        ret = gf_uuid_parse (volid_str, vol_uid);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "Failed to parse volume id "
-                        "for volume %s", volinfo->volname);
-                goto out;
-        }
+    int ret = -1;
+    char op_errstr[PATH_MAX] = "";
 
-        if (gf_uuid_compare (vol_uid, volinfo->volume_id)) {
-                gf_log (this->name, GF_LOG_ERROR, "Volume ids of volume %s - %s"
-                        " and %s - are different. Possibly a split brain among "
-                        "peers.", volinfo->volname, volid_str,
-                        uuid_utoa (volinfo->volume_id));
-                ret = -1;
-                goto out;
+    if (!gd_should_i_start_rebalance(volinfo)) {
+        /* Store the rebalance-id and rebalance command even if
+         * the peer isn't starting a rebalance process. On peers
+         * where a rebalance process is started,
+         * glusterd_handle_defrag_start performs the storing.
+         *
+         * Storing this is needed for having 'volume status'
+         * work correctly.
+         */
+        volinfo->rebal.defrag_status = GF_DEFRAG_STATUS_NOT_STARTED;
+        return 0;
+    }
+    if (!volinfo->rebal.defrag_cmd) {
+        volinfo->rebal.defrag_status = GF_DEFRAG_STATUS_FAILED;
+        return -1;
+    }
+
+    ret = glusterd_volume_defrag_restart(volinfo, op_errstr, PATH_MAX,
+                                         volinfo->rebal.defrag_cmd,
+                                         volinfo->rebal.op == GD_OP_REMOVE_BRICK
+                                             ? glusterd_remove_brick_migrate_cbk
+                                             : NULL);
+    if (!ret) {
+        /* If remove brick is started then ensure that on a glusterd
+         * restart decommission_is_in_progress is set to avoid remove
+         * brick commit to happen when rebalance is not completed.
+         */
+        if (volinfo->rebal.op == GD_OP_REMOVE_BRICK &&
+            volinfo->rebal.defrag_status == GF_DEFRAG_STATUS_STARTED) {
+            volinfo->decommission_in_progress = 1;
         }
-
-out:
-        return ret;
+    }
+    return ret;
 }
-
 int
-glusterd_defrag_volume_status_update (glusterd_volinfo_t *volinfo,
-                                      dict_t *rsp_dict)
+glusterd_restart_rebalance(glusterd_conf_t *conf)
 {
-        int                             ret = 0;
-        uint64_t                        files = 0;
-        uint64_t                        size = 0;
-        uint64_t                        lookup = 0;
-        gf_defrag_status_t              status = GF_DEFRAG_STATUS_NOT_STARTED;
-        uint64_t                        failures = 0;
-        uint64_t                        skipped = 0;
-        xlator_t                       *this = NULL;
-        double                          run_time = 0;
-
-        this = THIS;
+    glusterd_volinfo_t *volinfo = NULL;
+    int ret = 0;
 
-        ret = dict_get_uint64 (rsp_dict, "files", &files);
-        if (ret)
-                gf_log (this->name, GF_LOG_TRACE,
-                        "failed to get file count");
-
-        ret = dict_get_uint64 (rsp_dict, "size", &size);
-        if (ret)
-                gf_log (this->name, GF_LOG_TRACE,
-                        "failed to get size of xfer");
-
-        ret = dict_get_uint64 (rsp_dict, "lookups", &lookup);
-        if (ret)
-                gf_log (this->name, GF_LOG_TRACE,
-                        "failed to get lookedup file count");
+    cds_list_for_each_entry(volinfo, &conf->volumes, vol_list)
+    {
+        glusterd_restart_rebalance_for_volume(volinfo);
+    }
+    return ret;
+}
 
-        ret = dict_get_int32 (rsp_dict, "status", (int32_t *)&status);
-        if (ret)
-                gf_log (this->name, GF_LOG_TRACE,
-                        "failed to get status");
+void
+glusterd_volinfo_reset_defrag_stats(glusterd_volinfo_t *volinfo)
+{
+    glusterd_rebalance_t *rebal = NULL;
+    GF_ASSERT(volinfo);
 
-        ret = dict_get_uint64 (rsp_dict, "failures", &failures);
-        if (ret)
-                gf_log (this->name, GF_LOG_TRACE,
-                        "failed to get failure count");
+    rebal = &volinfo->rebal;
+    rebal->rebalance_files = 0;
+    rebal->rebalance_data = 0;
+    rebal->lookedup_files = 0;
+    rebal->rebalance_failures = 0;
+    rebal->rebalance_time = 0;
+    rebal->skipped_files = 0;
+}
 
-        ret = dict_get_uint64 (rsp_dict, "skipped", &skipped);
-        if (ret)
-                gf_log (this->name, GF_LOG_TRACE,
-                        "failed to get skipped count");
+gf_boolean_t
+glusterd_is_local_brick(xlator_t *this, glusterd_volinfo_t *volinfo,
+                        glusterd_brickinfo_t *brickinfo)
+{
+    gf_boolean_t local = _gf_false;
+    int ret = 0;
 
-        ret = dict_get_double (rsp_dict, "run-time", &run_time);
+    if (gf_uuid_is_null(brickinfo->uuid)) {
+        ret = glusterd_resolve_brick(brickinfo);
         if (ret)
-                gf_log (this->name, GF_LOG_TRACE,
-                        "failed to get run-time");
-
-        if (files)
-                volinfo->rebal.rebalance_files = files;
-        if (size)
-                volinfo->rebal.rebalance_data = size;
-        if (lookup)
-                volinfo->rebal.lookedup_files = lookup;
-        if (status)
-                volinfo->rebal.defrag_status = status;
-        if (failures)
-                volinfo->rebal.rebalance_failures = failures;
-        if (skipped)
-                volinfo->rebal.skipped_files = skipped;
-        if (run_time)
-                volinfo->rebal.rebalance_time = run_time;
-
-        return ret;
+            goto out;
+    }
+    local = !gf_uuid_compare(brickinfo->uuid, MY_UUID);
+out:
+    return local;
 }
-
 int
-glusterd_check_topology_identical (const char   *filename1,
-                                   const char   *filename2,
-                                   gf_boolean_t *identical)
-{
-        int                     ret    = -1; /* FAILURE */
-        xlator_t                *this  = THIS;
-        FILE                    *fp1   = NULL;
-        FILE                    *fp2   = NULL;
-        glusterfs_graph_t       *grph1 = NULL;
-        glusterfs_graph_t       *grph2 = NULL;
-
-        /* Invalid xlator, Nothing to do */
-        if (!this)
-                return (-1);
-
-        /* Sanitize the inputs */
-        GF_VALIDATE_OR_GOTO (this->name, filename1, out);
-        GF_VALIDATE_OR_GOTO (this->name, filename2, out);
-        GF_VALIDATE_OR_GOTO (this->name, identical, out);
-
-        /* fopen() the volfile1 to create the graph */
-        fp1 = fopen (filename1, "r");
-        if (fp1 == NULL) {
-                gf_log (this->name, GF_LOG_ERROR, "fopen() on file: %s failed "
-                        "(%s)", filename1, strerror (errno));
-                goto out;
-        }
-
-        /* fopen() the volfile2 to create the graph */
-        fp2 = fopen (filename2, "r");
-        if (fp2 == NULL) {
-                gf_log (this->name, GF_LOG_ERROR, "fopen() on file: %s failed "
-                        "(%s)", filename2, strerror (errno));
-                goto out;
-        }
-
-        /* create the graph for filename1 */
-        grph1 = glusterfs_graph_construct(fp1);
-        if (grph1 == NULL)
-                goto out;
-
-        /* create the graph for filename2 */
-        grph2 = glusterfs_graph_construct(fp2);
-        if (grph2 == NULL)
-                goto out;
+glusterd_validate_volume_id(dict_t *op_dict, glusterd_volinfo_t *volinfo)
+{
+    int ret = -1;
+    char *volid_str = NULL;
+    uuid_t vol_uid = {
+        0,
+    };
+    xlator_t *this = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+
+    ret = dict_get_strn(op_dict, "vol-id", SLEN("vol-id"), &volid_str);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+               "Failed to get volume id for "
+               "volume %s",
+               volinfo->volname);
+        goto out;
+    }
+    ret = gf_uuid_parse(volid_str, vol_uid);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_UUID_PARSE_FAIL,
+               "Failed to parse volume id "
+               "for volume %s",
+               volinfo->volname);
+        goto out;
+    }
+
+    if (gf_uuid_compare(vol_uid, volinfo->volume_id)) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_VOL_ID_MISMATCH,
+               "Volume ids of volume %s - %s"
+               " and %s - are different. Possibly a split brain among "
+               "peers.",
+               volinfo->volname, volid_str, uuid_utoa(volinfo->volume_id));
+        ret = -1;
+        goto out;
+    }
 
-        /* compare the graph topology */
-        *identical = is_graph_topology_equal(grph1, grph2);
-        ret = 0; /* SUCCESS */
 out:
-        if (fp1)
-                fclose(fp1);
-        if (fp2)
-                fclose(fp2);
-        if (grph1)
-                glusterfs_graph_destroy(grph1);
-        if (grph2)
-                glusterfs_graph_destroy(grph2);
-
-        gf_log (this->name, GF_LOG_DEBUG, "Returning with %d", ret);
-        return ret;
+    return ret;
 }
 
 int
-glusterd_check_files_identical (char *filename1, char *filename2,
-                                gf_boolean_t *identical)
-{
-        int                     ret = -1;
-        struct stat             buf1 = {0,};
-        struct stat             buf2 = {0,};
-        uint32_t                cksum1 = 0;
-        uint32_t                cksum2 = 0;
-        xlator_t                *this = NULL;
-
-        GF_ASSERT (filename1);
-        GF_ASSERT (filename2);
-        GF_ASSERT (identical);
-
-        this = THIS;
-
-        ret = stat (filename1, &buf1);
-
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "stat on file: %s failed "
-                        "(%s)", filename1, strerror (errno));
-                goto out;
-        }
-
-        ret = stat (filename2, &buf2);
-
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "stat on file: %s failed "
-                        "(%s)", filename2, strerror (errno));
-                goto out;
-        }
-
-        if (buf1.st_size != buf2.st_size) {
-                *identical = _gf_false;
-                goto out;
-        }
-
-        ret = get_checksum_for_path (filename1, &cksum1);
-        if (ret)
-                goto out;
+glusterd_defrag_volume_status_update(glusterd_volinfo_t *volinfo,
+                                     dict_t *rsp_dict, int32_t cmd)
+{
+    int ret = 0;
+    int ret2 = 0;
+    uint64_t files = 0;
+    uint64_t size = 0;
+    uint64_t lookup = 0;
+    gf_defrag_status_t status = GF_DEFRAG_STATUS_NOT_STARTED;
+    uint64_t failures = 0;
+    uint64_t skipped = 0;
+    xlator_t *this = NULL;
+    double run_time = 0;
+    uint64_t promoted = 0;
+    uint64_t demoted = 0;
+    uint64_t time_left = 0;
+
+    this = THIS;
+
+    ret = dict_get_uint64(rsp_dict, "files", &files);
+    if (ret)
+        gf_msg_trace(this->name, 0, "failed to get file count");
+
+    ret = dict_get_uint64(rsp_dict, "size", &size);
+    if (ret)
+        gf_msg_trace(this->name, 0, "failed to get size of xfer");
+
+    ret = dict_get_uint64(rsp_dict, "lookups", &lookup);
+    if (ret)
+        gf_msg_trace(this->name, 0, "failed to get lookedup file count");
+
+    ret = dict_get_int32n(rsp_dict, "status", SLEN("status"),
+                          (int32_t *)&status);
+    if (ret)
+        gf_msg_trace(this->name, 0, "failed to get status");
+
+    ret = dict_get_uint64(rsp_dict, "failures", &failures);
+    if (ret)
+        gf_msg_trace(this->name, 0, "failed to get failure count");
+
+    ret = dict_get_uint64(rsp_dict, "skipped", &skipped);
+    if (ret)
+        gf_msg_trace(this->name, 0, "failed to get skipped count");
+
+    ret = dict_get_uint64(rsp_dict, "promoted", &promoted);
+    if (ret)
+        gf_msg_trace(this->name, 0, "failed to get promoted count");
+
+    ret = dict_get_uint64(rsp_dict, "demoted", &demoted);
+    if (ret)
+        gf_msg_trace(this->name, 0, "failed to get demoted count");
+
+    ret = dict_get_double(rsp_dict, "run-time", &run_time);
+    if (ret)
+        gf_msg_trace(this->name, 0, "failed to get run-time");
+
+    ret2 = dict_get_uint64(rsp_dict, "time-left", &time_left);
+    if (ret2)
+        gf_msg_trace(this->name, 0, "failed to get time left");
+
+    if (files)
+        volinfo->rebal.rebalance_files = files;
+    if (size)
+        volinfo->rebal.rebalance_data = size;
+    if (lookup)
+        volinfo->rebal.lookedup_files = lookup;
+    if (status)
+        volinfo->rebal.defrag_status = status;
+    if (failures)
+        volinfo->rebal.rebalance_failures = failures;
+    if (skipped)
+        volinfo->rebal.skipped_files = skipped;
+    if (run_time)
+        volinfo->rebal.rebalance_time = run_time;
+    if (!ret2)
+        volinfo->rebal.time_left = time_left;
+
+    return ret;
+}
 
+int
+glusterd_check_topology_identical(const char *filename1, const char *filename2,
+                                  gf_boolean_t *identical)
+{
+    int ret = -1; /* FAILURE */
+    xlator_t *this = THIS;
+    FILE *fp1 = NULL;
+    FILE *fp2 = NULL;
+    glusterfs_graph_t *grph1 = NULL;
+    glusterfs_graph_t *grph2 = NULL;
+
+    /* Invalid xlator, Nothing to do */
+    if (!this)
+        return (-1);
+
+    /* Sanitize the inputs */
+    GF_VALIDATE_OR_GOTO(this->name, filename1, out);
+    GF_VALIDATE_OR_GOTO(this->name, filename2, out);
+    GF_VALIDATE_OR_GOTO(this->name, identical, out);
+
+    /* fopen() the volfile1 to create the graph */
+    fp1 = fopen(filename1, "r");
+    if (fp1 == NULL) {
+        gf_msg(this->name, GF_LOG_ERROR, errno, GD_MSG_FILE_OP_FAILED,
+               "fopen() on file: %s failed "
+               "(%s)",
+               filename1, strerror(errno));
+        goto out;
+    }
+
+    /* fopen() the volfile2 to create the graph */
+    fp2 = fopen(filename2, "r");
+    if (fp2 == NULL) {
+        gf_msg(this->name, GF_LOG_ERROR, errno, GD_MSG_FILE_OP_FAILED,
+               "fopen() on file: %s failed "
+               "(%s)",
+               filename2, strerror(errno));
+        goto out;
+    }
+
+    /* create the graph for filename1 */
+    grph1 = glusterfs_graph_construct(fp1);
+    if (grph1 == NULL)
+        goto out;
+
+    /* create the graph for filename2 */
+    grph2 = glusterfs_graph_construct(fp2);
+    if (grph2 == NULL)
+        goto out;
+
+    /* compare the graph topology */
+    *identical = is_graph_topology_equal(grph1, grph2);
+    ret = 0; /* SUCCESS */
+out:
+    if (fp1)
+        fclose(fp1);
+    if (fp2)
+        fclose(fp2);
+    if (grph1)
+        glusterfs_graph_destroy(grph1);
+    if (grph2)
+        glusterfs_graph_destroy(grph2);
 
-        ret = get_checksum_for_path (filename2, &cksum2);
-        if (ret)
-                goto out;
+    gf_msg_debug(this->name, 0, "Returning with %d", ret);
+    return ret;
+}
 
-        if (cksum1 != cksum2)
-                *identical = _gf_false;
-        else
-                *identical = _gf_true;
+int
+glusterd_check_files_identical(char *filename1, char *filename2,
+                               gf_boolean_t *identical)
+{
+    int ret = -1;
+    struct stat buf1 = {
+        0,
+    };
+    struct stat buf2 = {
+        0,
+    };
+    uint32_t cksum1 = 0;
+    uint32_t cksum2 = 0;
+    xlator_t *this = NULL;
+    glusterd_conf_t *priv = NULL;
+
+    GF_ASSERT(filename1);
+    GF_ASSERT(filename2);
+    GF_ASSERT(identical);
+
+    this = THIS;
+    GF_VALIDATE_OR_GOTO("glusterd", this, out);
+    priv = this->private;
+    GF_VALIDATE_OR_GOTO(this->name, priv, out);
+
+    ret = sys_stat(filename1, &buf1);
+
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, errno, GD_MSG_FILE_OP_FAILED,
+               "stat on file: %s failed "
+               "(%s)",
+               filename1, strerror(errno));
+        goto out;
+    }
+
+    ret = sys_stat(filename2, &buf2);
+
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, errno, GD_MSG_FILE_OP_FAILED,
+               "stat on file: %s failed "
+               "(%s)",
+               filename2, strerror(errno));
+        goto out;
+    }
+
+    if (buf1.st_size != buf2.st_size) {
+        *identical = _gf_false;
+        goto out;
+    }
+
+    ret = get_checksum_for_path(filename1, &cksum1, priv->op_version);
+    if (ret)
+        goto out;
+
+    ret = get_checksum_for_path(filename2, &cksum2, priv->op_version);
+    if (ret)
+        goto out;
+
+    if (cksum1 != cksum2)
+        *identical = _gf_false;
+    else
+        *identical = _gf_true;
 
 out:
-        gf_log (this->name, GF_LOG_DEBUG, "Returning with %d", ret);
-        return ret;
+    gf_msg_debug(this->name, 0, "Returning with %d", ret);
+    return ret;
 }
 
 int
-glusterd_volset_help (dict_t *dict, char **op_errstr)
+glusterd_volset_help(dict_t *dict, char **op_errstr)
 {
-        int                     ret = -1;
-        gf_boolean_t            xml_out = _gf_false;
-        xlator_t                *this = NULL;
+    int ret = -1;
+    gf_boolean_t xml_out = _gf_false;
+#if (!HAVE_LIB_XML)
+    xlator_t *this = NULL;
 
-        this = THIS;
+    this = THIS;
+#endif
 
-        if (!dict) {
-                if (!(dict = glusterd_op_get_ctx ())) {
-                        ret = 0;
-                        goto out;
-                }
+    if (!dict) {
+        if (!(dict = glusterd_op_get_ctx())) {
+            ret = 0;
+            goto out;
         }
+    }
 
-        if (dict_get (dict, "help" )) {
-                xml_out = _gf_false;
+    if (dict_getn(dict, "help", SLEN("help"))) {
+        xml_out = _gf_false;
 
-        } else if (dict_get (dict, "help-xml" )) {
-                xml_out = _gf_true;
+    } else if (dict_getn(dict, "help-xml", SLEN("help-xml"))) {
+        xml_out = _gf_true;
 #if (HAVE_LIB_XML)
-                ret = 0;
+        ret = 0;
 #else
-                gf_log (this->name, GF_LOG_ERROR,
-                        "libxml not present in the system");
-                if (op_errstr)
-                        *op_errstr = gf_strdup ("Error: xml libraries not "
-                                                "present to produce "
-                                                "xml-output");
-                goto out;
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_MODULE_NOT_INSTALLED,
+               "libxml not present in the system");
+        if (op_errstr)
+            *op_errstr = gf_strdup(
+                "Error: xml libraries not "
+                "present to produce "
+                "xml-output");
+        goto out;
 #endif
 
-        } else {
-                goto out;
-        }
+    } else {
+        goto out;
+    }
 
-        ret = glusterd_get_volopt_content (dict, xml_out);
-        if (ret && op_errstr)
-                *op_errstr = gf_strdup ("Failed to get volume options help");
- out:
+    ret = glusterd_get_volopt_content(dict, xml_out);
+    if (ret && op_errstr)
+        *op_errstr = gf_strdup("Failed to get volume options help");
+out:
 
-        gf_log ("glusterd", GF_LOG_DEBUG, "Returning %d", ret);
-        return ret;
+    gf_msg_debug("glusterd", 0, "Returning %d", ret);
+    return ret;
 }
 
 int
-glusterd_to_cli (rpcsvc_request_t *req, gf_cli_rsp *arg, struct iovec *payload,
-                 int payloadcount, struct iobref *iobref, xdrproc_t xdrproc,
-                 dict_t *dict)
-{
-        int                ret = -1;
-        char               *cmd = NULL;
-        int                op_ret = 0;
-        char               *op_errstr = NULL;
-        int                op_errno = 0;
-        xlator_t           *this = NULL;
-
-        this = THIS;
-        GF_ASSERT (this);
-
-        op_ret = arg->op_ret;
-        op_errstr = arg->op_errstr;
-        op_errno = arg->op_errno;
-
-        ret = dict_get_str (dict, "cmd-str", &cmd);
-        if (ret)
-                gf_log (this->name, GF_LOG_ERROR, "Failed to get command "
-                        "string");
-
-        if (cmd) {
-                if (op_ret)
-                        gf_cmd_log ("", "%s : FAILED %s %s", cmd,
-                                       (op_errstr)? ":" : " ",
-                                       (op_errstr)? op_errstr : " ");
-                else
-                        gf_cmd_log ("", "%s : SUCCESS", cmd);
-        }
+glusterd_to_cli(rpcsvc_request_t *req, gf_cli_rsp *arg, struct iovec *payload,
+                int payloadcount, struct iobref *iobref, xdrproc_t xdrproc,
+                dict_t *dict)
+{
+    int ret = -1;
+    char *cmd = NULL;
+    int op_ret = 0;
+    char *op_errstr = NULL;
+    xlator_t *this = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+
+    op_ret = arg->op_ret;
+    op_errstr = arg->op_errstr;
+
+    ret = dict_get_strn(dict, "cmd-str", SLEN("cmd-str"), &cmd);
+    if (ret)
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+               "Failed to get command "
+               "string");
+
+    if (cmd) {
+        if (op_ret)
+            gf_cmd_log("", "%s : FAILED %s %s", cmd, (op_errstr) ? ":" : " ",
+                       (op_errstr) ? op_errstr : " ");
+        else
+            gf_cmd_log("", "%s : SUCCESS", cmd);
+    }
 
-        glusterd_submit_reply (req, arg, payload, payloadcount, iobref,
-                               (xdrproc_t) xdrproc);
-        if (dict)
-                dict_unref (dict);
+    glusterd_submit_reply(req, arg, payload, payloadcount, iobref,
+                          (xdrproc_t)xdrproc);
 
-        return ret;
+    if (dict) {
+        dict_unref(dict);
+    }
+    return ret;
 }
 
 static int32_t
-glusterd_append_gsync_status (dict_t *dst, dict_t *src)
+glusterd_append_gsync_status(dict_t *dst, dict_t *src)
 {
-        int                ret = 0;
-        char               *stop_msg = NULL;
-
-        ret = dict_get_str (src, "gsync-status", &stop_msg);
-        if (ret) {
-                ret = 0;
-                goto out;
-        }
-
-        ret = dict_set_dynstr_with_alloc (dst, "gsync-status", stop_msg);
-        if (ret) {
-                gf_log ("glusterd", GF_LOG_WARNING, "Unable to set the stop"
-                        "message in the ctx dictionary");
-                goto out;
-        }
+    int ret = 0;
+    char *stop_msg = NULL;
 
+    ret = dict_get_strn(src, "gsync-status", SLEN("gsync-status"), &stop_msg);
+    if (ret) {
+        gf_smsg("glusterd", GF_LOG_ERROR, errno, GD_MSG_DICT_GET_FAILED,
+                "Key=gsync-status", NULL);
         ret = 0;
- out:
-        gf_log ("glusterd", GF_LOG_DEBUG, "Returning %d", ret);
-        return ret;
-
+        goto out;
+    }
+
+    ret = dict_set_dynstr_with_alloc(dst, "gsync-status", stop_msg);
+    if (ret) {
+        gf_msg("glusterd", GF_LOG_WARNING, 0, GD_MSG_DICT_SET_FAILED,
+               "Unable to set the stop"
+               "message in the ctx dictionary");
+        goto out;
+    }
+
+    ret = 0;
+out:
+    gf_msg_debug("glusterd", 0, "Returning %d", ret);
+    return ret;
 }
 
 int32_t
-glusterd_append_status_dicts (dict_t *dst, dict_t *src)
+glusterd_append_status_dicts(dict_t *dst, dict_t *src)
 {
-        char                sts_val_name[PATH_MAX] = {0, };
-        int                 dst_count              = 0;
-        int                 src_count              = 0;
-        int                 i                      = 0;
-        int                 ret                    = 0;
-        gf_gsync_status_t  *sts_val                = NULL;
-        gf_gsync_status_t  *dst_sts_val            = NULL;
+    char sts_val_name[PATH_MAX] = "";
+    int dst_count = 0;
+    int src_count = 0;
+    int i = 0;
+    int ret = 0;
+    gf_gsync_status_t *sts_val = NULL;
+    gf_gsync_status_t *dst_sts_val = NULL;
 
-        GF_ASSERT (dst);
+    GF_ASSERT(dst);
 
-        if (src == NULL)
-                goto out;
+    if (src == NULL)
+        goto out;
 
-        ret = dict_get_int32 (dst, "gsync-count", &dst_count);
-        if (ret)
-                dst_count = 0;
+    ret = dict_get_int32n(dst, "gsync-count", SLEN("gsync-count"), &dst_count);
+    if (ret)
+        dst_count = 0;
 
-        ret = dict_get_int32 (src, "gsync-count", &src_count);
-        if (ret || !src_count) {
-                gf_log ("", GF_LOG_DEBUG, "Source brick empty");
-                ret = 0;
-                goto out;
-        }
+    ret = dict_get_int32n(src, "gsync-count", SLEN("gsync-count"), &src_count);
+    if (ret || !src_count) {
+        gf_msg_debug("glusterd", 0, "Source brick empty");
+        ret = 0;
+        goto out;
+    }
 
-        for (i = 0; i < src_count; i++) {
-                memset (sts_val_name, '\0', sizeof(sts_val_name));
-                snprintf (sts_val_name, sizeof(sts_val_name), "status_value%d", i);
+    for (i = 0; i < src_count; i++) {
+        snprintf(sts_val_name, sizeof(sts_val_name), "status_value%d", i);
 
-                ret = dict_get_bin (src, sts_val_name, (void **) &sts_val);
-                if (ret)
-                        goto out;
+        ret = dict_get_bin(src, sts_val_name, (void **)&sts_val);
+        if (ret)
+            goto out;
 
-                dst_sts_val = GF_CALLOC (1, sizeof(gf_gsync_status_t),
-                                         gf_common_mt_gsync_status_t);
-                if (!dst_sts_val) {
-                        gf_log ("", GF_LOG_ERROR, "Out Of Memory");
-                        goto out;
-                }
+        dst_sts_val = GF_MALLOC(sizeof(gf_gsync_status_t),
+                                gf_common_mt_gsync_status_t);
+        if (!dst_sts_val) {
+            gf_msg("glusterd", GF_LOG_ERROR, ENOMEM, GD_MSG_NO_MEMORY,
+                   "Out Of Memory");
+            goto out;
+        }
 
-                memcpy (dst_sts_val, sts_val, sizeof(gf_gsync_status_t));
+        memcpy(dst_sts_val, sts_val, sizeof(gf_gsync_status_t));
 
-                memset (sts_val_name, '\0', sizeof(sts_val_name));
-                snprintf (sts_val_name, sizeof(sts_val_name), "status_value%d", i + dst_count);
+        snprintf(sts_val_name, sizeof(sts_val_name), "status_value%d",
+                 i + dst_count);
 
-                ret = dict_set_bin (dst, sts_val_name, dst_sts_val, sizeof(gf_gsync_status_t));
-                if (ret)
-                        goto out;
+        ret = dict_set_bin(dst, sts_val_name, dst_sts_val,
+                           sizeof(gf_gsync_status_t));
+        if (ret) {
+            GF_FREE(dst_sts_val);
+            goto out;
         }
+    }
 
-        ret = dict_set_int32 (dst, "gsync-count", dst_count+src_count);
-
- out:
-        gf_log ("", GF_LOG_DEBUG, "Returning %d", ret);
-        return ret;
+    ret = dict_set_int32n(dst, "gsync-count", SLEN("gsync-count"),
+                          dst_count + src_count);
 
+out:
+    gf_msg_debug("glusterd", 0, "Returning %d", ret);
+    return ret;
 }
 
 int32_t
-glusterd_aggr_brick_mount_dirs (dict_t *aggr, dict_t *rsp_dict)
-{
-        char                   key[PATH_MAX]   = "";
-        char                  *brick_mount_dir = NULL;
-        int32_t                brick_count     = -1;
-        int32_t                ret             = -1;
-        int32_t                i               = -1;
-        xlator_t              *this            = NULL;
-
-        this = THIS;
-        GF_ASSERT (this);
-        GF_ASSERT (aggr);
-        GF_ASSERT (rsp_dict);
+glusterd_aggr_brick_mount_dirs(dict_t *aggr, dict_t *rsp_dict)
+{
+    char key[64] = "";
+    int keylen;
+    char *brick_mount_dir = NULL;
+    int32_t brick_count = -1;
+    int32_t ret = -1;
+    int32_t i = -1;
+    xlator_t *this = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+    GF_ASSERT(aggr);
+    GF_ASSERT(rsp_dict);
+
+    ret = dict_get_int32n(rsp_dict, "brick_count", SLEN("brick_count"),
+                          &brick_count);
+    if (ret) {
+        gf_msg_debug(this->name, 0, "No brick_count present");
+        ret = 0;
+        goto out;
+    }
 
-        ret = dict_get_int32 (rsp_dict, "brick_count", &brick_count);
+    for (i = 1; i <= brick_count; i++) {
+        brick_mount_dir = NULL;
+        keylen = snprintf(key, sizeof(key), "brick%d.mount_dir", i);
+        ret = dict_get_strn(rsp_dict, key, keylen, &brick_mount_dir);
         if (ret) {
-                gf_log (this->name, GF_LOG_DEBUG, "No brick_count present");
-                ret = 0;
-                goto out;
+            /* Coz the info will come from a different node */
+            gf_msg_debug(this->name, 0, "%s not present", key);
+            continue;
         }
 
-        for (i = 1; i <= brick_count; i++) {
-                brick_mount_dir = NULL;
-                snprintf (key, sizeof(key), "brick%d.mount_dir", i);
-                ret = dict_get_str (rsp_dict, key, &brick_mount_dir);
-                if (ret) {
-                        /* Coz the info will come from a different node */
-                        gf_log (this->name, GF_LOG_DEBUG,
-                                "%s not present", key);
-                        continue;
-                }
-
-                ret = dict_set_dynstr_with_alloc (aggr, key,
-                                                  brick_mount_dir);
-                if (ret) {
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "Failed to set %s", key);
-                        goto out;
-                }
+        ret = dict_set_dynstr_with_alloc(aggr, key, brick_mount_dir);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+                   "Failed to set %s", key);
+            goto out;
         }
+    }
 
-        ret = 0;
+    ret = 0;
 out:
-        gf_log (this->name, GF_LOG_TRACE, "Returning %d ", ret);
-        return ret;
+    gf_msg_trace(this->name, 0, "Returning %d ", ret);
+    return ret;
 }
 
 int32_t
-glusterd_gsync_use_rsp_dict (dict_t *aggr, dict_t *rsp_dict, char *op_errstr)
+glusterd_gsync_use_rsp_dict(dict_t *aggr, dict_t *rsp_dict, char *op_errstr)
 {
-        dict_t             *ctx = NULL;
-        int                ret = 0;
-        char               *conf_path = NULL;
+    dict_t *ctx = NULL;
+    int ret = 0;
+    char *conf_path = NULL;
 
-        if (aggr) {
-                ctx = aggr;
+    if (aggr) {
+        ctx = aggr;
 
-        } else {
-                ctx = glusterd_op_get_ctx ();
-                if (!ctx) {
-                        gf_log ("", GF_LOG_ERROR,
-                                "Operation Context is not present");
-                        GF_ASSERT (0);
-                }
+    } else {
+        ctx = glusterd_op_get_ctx();
+        if (!ctx) {
+            gf_msg("glusterd", GF_LOG_ERROR, 0, GD_MSG_OPCTX_GET_FAIL,
+                   "Operation Context is not present");
+            GF_ASSERT(0);
         }
+    }
 
-        if (rsp_dict) {
-                ret = glusterd_append_status_dicts (ctx, rsp_dict);
-                if (ret)
-                        goto out;
+    if (rsp_dict) {
+        ret = glusterd_append_status_dicts(ctx, rsp_dict);
+        if (ret)
+            goto out;
 
-                ret = glusterd_append_gsync_status (ctx, rsp_dict);
-                if (ret)
-                        goto out;
+        ret = glusterd_append_gsync_status(ctx, rsp_dict);
+        if (ret)
+            goto out;
 
-                ret = dict_get_str (rsp_dict, "conf_path", &conf_path);
-                if (!ret && conf_path) {
-                        ret = dict_set_dynstr_with_alloc (ctx, "conf_path",
-                                                          conf_path);
-                        if (ret) {
-                                gf_log ("", GF_LOG_ERROR,
-                                        "Unable to store conf path.");
-                                goto out;
-                        }
-                }
-        }
-        if ((op_errstr) && (strcmp ("", op_errstr))) {
-                ret = dict_set_dynstr_with_alloc (ctx, "errstr",
-                                                  op_errstr);
-                if (ret)
-                        goto out;
+        ret = dict_get_strn(rsp_dict, "conf_path", SLEN("conf_path"),
+                            &conf_path);
+        if (!ret && conf_path) {
+            ret = dict_set_dynstr_with_alloc(ctx, "conf_path", conf_path);
+            if (ret) {
+                gf_msg("glusterd", GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+                       "Unable to store conf path.");
+                goto out;
+            }
         }
+    }
+    if ((op_errstr) && (strcmp("", op_errstr))) {
+        ret = dict_set_dynstr_with_alloc(ctx, "errstr", op_errstr);
+        if (ret)
+            goto out;
+    }
 
-        ret = 0;
- out:
-        gf_log ("", GF_LOG_DEBUG, "Returning %d ", ret);
-        return ret;
+    ret = 0;
+out:
+    gf_msg_debug("glusterd", 0, "Returning %d ", ret);
+    return ret;
 }
 
 int32_t
-glusterd_rb_use_rsp_dict (dict_t *aggr, dict_t *rsp_dict)
+glusterd_rb_use_rsp_dict(dict_t *aggr, dict_t *rsp_dict)
 {
-        int32_t  src_port = 0;
-        int32_t  dst_port = 0;
-        int      ret      = 0;
-        dict_t  *ctx      = NULL;
-        xlator_t *this    = NULL;
+    int32_t src_port = 0;
+    int32_t dst_port = 0;
+    int ret = 0;
+    dict_t *ctx = NULL;
+    xlator_t *this = NULL;
 
-        this = THIS;
-        GF_ASSERT (this);
+    this = THIS;
+    GF_ASSERT(this);
 
-        if (aggr) {
-                ctx = aggr;
+    if (aggr) {
+        ctx = aggr;
 
-        } else {
-                ctx = glusterd_op_get_ctx ();
-                if (!ctx) {
-                        gf_log ("", GF_LOG_ERROR,
-                                "Operation Context is not present");
-                        GF_ASSERT (0);
-                }
+    } else {
+        ctx = glusterd_op_get_ctx();
+        if (!ctx) {
+            gf_msg("glusterd", GF_LOG_ERROR, 0, GD_MSG_OPCTX_GET_FAIL,
+                   "Operation Context is not present");
+            GF_ASSERT(0);
         }
+    }
 
-        if (rsp_dict) {
-                ret = dict_get_int32 (rsp_dict, "src-brick-port", &src_port);
-                if (ret == 0) {
-                        gf_log ("", GF_LOG_DEBUG,
-                                "src-brick-port=%d found", src_port);
-                }
-
-                ret = dict_get_int32 (rsp_dict, "dst-brick-port", &dst_port);
-                if (ret == 0) {
-                        gf_log ("", GF_LOG_DEBUG,
-                                "dst-brick-port=%d found", dst_port);
-                }
+    if (rsp_dict) {
+        ret = dict_get_int32n(rsp_dict, "src-brick-port",
+                              SLEN("src-brick-port"), &src_port);
+        if (ret == 0) {
+            gf_msg_debug("glusterd", 0, "src-brick-port=%d found", src_port);
+        }
 
-                ret = glusterd_aggr_brick_mount_dirs (ctx, rsp_dict);
-                if (ret) {
-                        gf_log (this->name, GF_LOG_ERROR, "Failed to "
-                                "aggregate brick mount dirs");
-                        goto out;
-                }
+        ret = dict_get_int32n(rsp_dict, "dst-brick-port",
+                              SLEN("dst-brick-port"), &dst_port);
+        if (ret == 0) {
+            gf_msg_debug("glusterd", 0, "dst-brick-port=%d found", dst_port);
         }
 
-        if (src_port) {
-                ret = dict_set_int32 (ctx, "src-brick-port",
-                                      src_port);
-                if (ret) {
-                        gf_log ("", GF_LOG_DEBUG,
-                                "Could not set src-brick");
-                        goto out;
-                }
+        ret = glusterd_aggr_brick_mount_dirs(ctx, rsp_dict);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_BRICK_MOUNDIRS_AGGR_FAIL,
+                   "Failed to "
+                   "aggregate brick mount dirs");
+            goto out;
         }
+    }
 
-        if (dst_port) {
-                ret = dict_set_int32 (ctx, "dst-brick-port",
-                                      dst_port);
-                if (ret) {
-                        gf_log ("", GF_LOG_DEBUG,
-                                "Could not set dst-brick");
-                        goto out;
-                }
+    if (src_port) {
+        ret = dict_set_int32n(ctx, "src-brick-port", SLEN("src-brick-port"),
+                              src_port);
+        if (ret) {
+            gf_msg_debug("glusterd", 0, "Could not set src-brick");
+            goto out;
+        }
+    }
 
+    if (dst_port) {
+        ret = dict_set_int32n(ctx, "dst-brick-port", SLEN("dst-brick-port"),
+                              dst_port);
+        if (ret) {
+            gf_msg_debug("glusterd", 0, "Could not set dst-brick");
+            goto out;
         }
+    }
 
 out:
-        return ret;
-
+    return ret;
 }
 
 int32_t
-glusterd_sync_use_rsp_dict (dict_t *aggr, dict_t *rsp_dict)
+glusterd_sync_use_rsp_dict(dict_t *aggr, dict_t *rsp_dict)
 {
-        int      ret      = 0;
+    int ret = 0;
 
-        GF_ASSERT (rsp_dict);
+    GF_ASSERT(rsp_dict);
+    xlator_t *this = THIS;
+    GF_ASSERT(this);
 
-        if (!rsp_dict) {
-                goto out;
-        }
+    if (!rsp_dict) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_CREATE_FAIL, NULL);
+        goto out;
+    }
 
-        ret = glusterd_import_friend_volumes (rsp_dict);
+    ret = glusterd_import_friend_volumes(rsp_dict);
 out:
-        return ret;
-
+    return ret;
 }
 
 static int
-_profile_volume_add_friend_rsp (dict_t *this, char *key, data_t *value,
+_profile_volume_add_friend_rsp(dict_t *this, char *key, data_t *value,
                                void *data)
 {
-        char    new_key[256] = {0};
-        glusterd_pr_brick_rsp_conv_t *rsp_ctx = NULL;
-        data_t  *new_value = NULL;
-        int     brick_count = 0;
-        char    brick_key[256];
+    char new_key[264] = "";
+    int new_key_len;
+    glusterd_pr_brick_rsp_conv_t *rsp_ctx = NULL;
+    data_t *new_value = NULL;
+    int brick_count = 0;
+    char brick_key[256] = "";
 
-        if (strcmp (key, "count") == 0)
-                return 0;
-        sscanf (key, "%d%s", &brick_count, brick_key);
-        rsp_ctx = data;
-        new_value = data_copy (value);
-        GF_ASSERT (new_value);
-        snprintf (new_key, sizeof (new_key), "%d%s",
-                  rsp_ctx->count + brick_count, brick_key);
-        dict_set (rsp_ctx->dict, new_key, new_value);
+    if (strcmp(key, "count") == 0)
         return 0;
+    sscanf(key, "%d%s", &brick_count, brick_key);
+    rsp_ctx = data;
+    new_value = data_copy(value);
+    GF_ASSERT(new_value);
+    new_key_len = snprintf(new_key, sizeof(new_key), "%d%s",
+                           rsp_ctx->count + brick_count, brick_key);
+    dict_setn(rsp_ctx->dict, new_key, new_key_len, new_value);
+    return 0;
 }
 
 int
-glusterd_profile_volume_use_rsp_dict (dict_t *aggr, dict_t *rsp_dict)
-{
-        int     ret = 0;
-        glusterd_pr_brick_rsp_conv_t rsp_ctx = {0};
-        int32_t brick_count = 0;
-        int32_t count = 0;
-        dict_t  *ctx_dict = NULL;
-        glusterd_op_t   op = GD_OP_NONE;
-
-        GF_ASSERT (rsp_dict);
-
-        ret = dict_get_int32 (rsp_dict, "count", &brick_count);
-        if (ret) {
-                ret = 0; //no bricks in the rsp
-                goto out;
-        }
-
-        op = glusterd_op_get_op ();
-        GF_ASSERT (GD_OP_PROFILE_VOLUME == op);
-        if (aggr) {
-                ctx_dict = aggr;
-
-        } else {
-                ctx_dict = glusterd_op_get_ctx ();
-        }
-
-        ret = dict_get_int32 (ctx_dict, "count", &count);
-        rsp_ctx.count = count;
-        rsp_ctx.dict = ctx_dict;
-        dict_foreach (rsp_dict, _profile_volume_add_friend_rsp, &rsp_ctx);
-        dict_del (ctx_dict, "count");
-        ret = dict_set_int32 (ctx_dict, "count", count + brick_count);
+glusterd_profile_volume_use_rsp_dict(dict_t *aggr, dict_t *rsp_dict)
+{
+    int ret = 0;
+    glusterd_pr_brick_rsp_conv_t rsp_ctx = {0};
+    int32_t brick_count = 0;
+    int32_t count = 0;
+    dict_t *ctx_dict = NULL;
+    xlator_t *this = NULL;
+
+    GF_ASSERT(rsp_dict);
+    this = THIS;
+    GF_ASSERT(this);
+
+    ret = dict_get_int32n(rsp_dict, "count", SLEN("count"), &brick_count);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_GET_FAILED,
+                "Key=count", NULL);
+        ret = 0;  // no bricks in the rsp
+        goto out;
+    }
+    if (aggr) {
+        ctx_dict = aggr;
+
+    } else {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_OPCTX_GET_FAIL,
+               "Operation Context is not present");
+        ret = -1;
+        goto out;
+    }
+
+    ret = dict_get_int32n(ctx_dict, "count", SLEN("count"), &count);
+    rsp_ctx.count = count;
+    rsp_ctx.dict = ctx_dict;
+    dict_foreach(rsp_dict, _profile_volume_add_friend_rsp, &rsp_ctx);
+    ret = dict_set_int32n(ctx_dict, "count", SLEN("count"),
+                          count + brick_count);
 out:
-        return ret;
+    return ret;
 }
 
 static int
-glusterd_volume_status_add_peer_rsp (dict_t *this, char *key, data_t *value,
-                                     void *data)
-{
-        glusterd_status_rsp_conv_t      *rsp_ctx = NULL;
-        data_t                          *new_value = NULL;
-        char                            brick_key[1024] = {0,};
-        char                            new_key[1024] = {0,};
-        int32_t                         index = 0;
-        int32_t                         ret = 0;
-
-        /* Skip the following keys, they are already present in the ctx_dict */
-        /* Also, skip all the task related pairs. They will be added to the
-         * ctx_dict later
-         */
-        if (!strcmp (key, "count") || !strcmp (key, "cmd") ||
-            !strcmp (key, "brick-index-max") || !strcmp (key, "other-count") ||
-            !strncmp (key, "task", 4))
-                return 0;
+glusterd_volume_status_add_peer_rsp(dict_t *this, char *key, data_t *value,
+                                    void *data)
+{
+    glusterd_status_rsp_conv_t *rsp_ctx = NULL;
+    data_t *new_value = NULL;
+    char brick_key[1024] = "";
+    char new_key[1024] = "";
+    int32_t index = 0;
+    int32_t ret = -1;
+    int32_t len = 0;
+
+    /* Skip the following keys, they are already present in the ctx_dict */
+    /* Also, skip all the task related pairs. They will be added to the
+     * ctx_dict later
+     */
+    if (!strcmp(key, "count") || !strcmp(key, "cmd") ||
+        !strcmp(key, "brick-index-max") || !strcmp(key, "other-count") ||
+        !strncmp(key, "task", 4))
+        return 0;
 
-        rsp_ctx = data;
-        new_value = data_copy (value);
-        GF_ASSERT (new_value);
+    rsp_ctx = data;
+    new_value = data_copy(value);
+    GF_ASSERT(new_value);
 
-        sscanf (key, "brick%d.%s", &index, brick_key);
+    sscanf(key, "brick%d.%s", &index, brick_key);
 
-        if (index > rsp_ctx->brick_index_max) {
-                snprintf (new_key, sizeof (new_key), "brick%d.%s",
-                          index + rsp_ctx->other_count, brick_key);
-        } else {
-                strncpy (new_key, key, sizeof (new_key));
-                new_key[sizeof (new_key) - 1] = 0;
-        }
+    if (index > rsp_ctx->brick_index_max) {
+        len = snprintf(new_key, sizeof(new_key), "brick%d.%s",
+                       index + rsp_ctx->other_count, brick_key);
+    } else {
+        len = snprintf(new_key, sizeof(new_key), "%s", key);
+    }
+    if (len < 0 || len >= sizeof(new_key))
+        goto out;
 
-        ret = dict_set (rsp_ctx->dict, new_key, new_value);
-        if (ret)
-                gf_log ("", GF_LOG_ERROR, "Unable to set key: %s in dict",
-                        key);
+    ret = dict_setn(rsp_ctx->dict, new_key, len, new_value);
+out:
+    if (ret) {
+        data_unref(new_value);
+        gf_msg("glusterd", GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+               "Unable to set key: %s in dict", key);
+    }
 
-        return 0;
+    return 0;
 }
 
 static int
-glusterd_volume_status_copy_tasks_to_ctx_dict (dict_t *this, char *key,
-                                               data_t *value, void *data)
+glusterd_volume_status_copy_tasks_to_ctx_dict(dict_t *this, char *key,
+                                              data_t *value, void *data)
 {
-        int     ret = 0;
-        dict_t  *ctx_dict = NULL;
-        data_t  *new_value = NULL;
+    int ret = 0;
+    dict_t *ctx_dict = NULL;
+    data_t *new_value = NULL;
 
-        if (strncmp (key, "task", 4))
-                return 0;
+    if (strncmp(key, "task", 4))
+        return 0;
 
-        ctx_dict = data;
-        GF_ASSERT (ctx_dict);
+    ctx_dict = data;
+    GF_ASSERT(ctx_dict);
 
-        new_value = data_copy (value);
-        GF_ASSERT (new_value);
+    new_value = data_copy(value);
+    GF_ASSERT(new_value);
 
-        ret = dict_set (ctx_dict, key, new_value);
+    ret = dict_set(ctx_dict, key, new_value);
 
-        return ret;
+    return ret;
 }
 
 int
-glusterd_volume_status_aggregate_tasks_status (dict_t *ctx_dict,
-                                               dict_t *rsp_dict)
-{
-        int             ret             = -1;
-        xlator_t        *this           = NULL;
-        int             local_count     = 0;
-        int             remote_count    = 0;
-        int             i               = 0;
-        int             j               = 0;
-        char            key[128]        = {0,};
-        char            *task_type      = NULL;
-        int             local_status    = 0;
-        int             remote_status   = 0;
-        char            *local_task_id  = NULL;
-        char            *remote_task_id = NULL;
-
-        GF_ASSERT (ctx_dict);
-        GF_ASSERT (rsp_dict);
-
-        this = THIS;
-        GF_ASSERT (this);
-
-        ret = dict_get_int32 (rsp_dict, "tasks", &remote_count);
+glusterd_volume_status_aggregate_tasks_status(dict_t *ctx_dict,
+                                              dict_t *rsp_dict)
+{
+    int ret = -1;
+    xlator_t *this = NULL;
+    int local_count = 0;
+    int remote_count = 0;
+    int i = 0;
+    int j = 0;
+    char key[128] = "";
+    int keylen;
+    char *task_type = NULL;
+    int local_status = 0;
+    int remote_status = 0;
+    char *local_task_id = NULL;
+    char *remote_task_id = NULL;
+
+    GF_ASSERT(ctx_dict);
+    GF_ASSERT(rsp_dict);
+
+    this = THIS;
+    GF_ASSERT(this);
+
+    ret = dict_get_int32n(rsp_dict, "tasks", SLEN("tasks"), &remote_count);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+               "Failed to get remote task count");
+        goto out;
+    }
+    /* Local count will not be present when this is called for the first
+     * time with the origins rsp_dict
+     */
+    ret = dict_get_int32n(ctx_dict, "tasks", SLEN("tasks"), &local_count);
+    if (ret) {
+        ret = dict_foreach(
+            rsp_dict, glusterd_volume_status_copy_tasks_to_ctx_dict, ctx_dict);
+        if (ret)
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+                   "Failed to copy tasks"
+                   "to ctx_dict.");
+        goto out;
+    }
+
+    if (local_count != remote_count) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_TASKS_COUNT_MISMATCH,
+               "Local tasks count (%d) and "
+               "remote tasks count (%d) do not match. Not aggregating "
+               "tasks status.",
+               local_count, remote_count);
+        ret = -1;
+        goto out;
+    }
+
+    /* Update the tasks statuses. For every remote tasks, search for the
+     * local task, and update the local task status based on the remote
+     * status.
+     */
+    for (i = 0; i < remote_count; i++) {
+        keylen = snprintf(key, sizeof(key), "task%d.type", i);
+        ret = dict_get_strn(rsp_dict, key, keylen, &task_type);
         if (ret) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "Failed to get remote task count");
-                goto out;
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+                   "Failed to get task typpe from rsp dict");
+            goto out;
         }
-        /* Local count will not be present when this is called for the first
-         * time with the origins rsp_dict
+
+        /* Skip replace-brick status as it is going to be the same on
+         * all peers. rb_status is set by the replace brick commit
+         * function on all peers based on the replace brick command.
+         * We return the value of rb_status as the status for a
+         * replace-brick task in a 'volume status' command.
          */
-        ret = dict_get_int32 (ctx_dict, "tasks", &local_count);
+        if (!strcmp(task_type, "Replace brick"))
+            continue;
+
+        keylen = snprintf(key, sizeof(key), "task%d.status", i);
+        ret = dict_get_int32n(rsp_dict, key, keylen, &remote_status);
         if (ret) {
-                ret = dict_foreach (rsp_dict,
-                                glusterd_volume_status_copy_tasks_to_ctx_dict,
-                                ctx_dict);
-                if (ret)
-                        gf_log (this->name, GF_LOG_ERROR, "Failed to copy tasks"
-                                "to ctx_dict.");
-                goto out;
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+                   "Failed to get task status from rsp dict");
+            goto out;
         }
-
-        if (local_count != remote_count) {
-                gf_log (this->name, GF_LOG_ERROR, "Local tasks count (%d) and "
-                        "remote tasks count (%d) do not match. Not aggregating "
-                        "tasks status.", local_count, remote_count);
-                ret = -1;
-                goto out;
+        keylen = snprintf(key, sizeof(key), "task%d.id", i);
+        ret = dict_get_strn(rsp_dict, key, keylen, &remote_task_id);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+                   "Failed to get task id from rsp dict");
+            goto out;
         }
+        for (j = 0; j < local_count; j++) {
+            keylen = snprintf(key, sizeof(key), "task%d.id", j);
+            ret = dict_get_strn(ctx_dict, key, keylen, &local_task_id);
+            if (ret) {
+                gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+                       "Failed to get local task-id");
+                goto out;
+            }
+
+            if (strncmp(remote_task_id, local_task_id,
+                        strlen(remote_task_id))) {
+                /* Quit if a matching local task is not found */
+                if (j == (local_count - 1)) {
+                    gf_msg(this->name, GF_LOG_ERROR, 0,
+                           GD_MSG_TASKS_COUNT_MISMATCH,
+                           "Could not find matching local "
+                           "task for task %s",
+                           remote_task_id);
+                    goto out;
+                }
+                continue;
+            }
+
+            keylen = snprintf(key, sizeof(key), "task%d.status", j);
+            ret = dict_get_int32n(ctx_dict, key, keylen, &local_status);
+            if (ret) {
+                gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+                       "Failed to get local task status");
+                goto out;
+            }
+
+            /* Rebalance has 5 states,
+             * NOT_STARTED, STARTED, STOPPED, COMPLETE, FAILED
+             * The precedence used to determine the aggregate status
+             * is as below,
+             * STARTED > FAILED > STOPPED > COMPLETE > NOT_STARTED
+             */
+            /* TODO: Move this to a common place utilities that both
+             * CLI and glusterd need.
+             * Till then if the below algorithm is changed, change
+             * it in cli_xml_output_vol_rebalance_status in
+             * cli-xml-output.c
+             */
+            ret = 0;
+            int rank[] = {[GF_DEFRAG_STATUS_STARTED] = 1,
+                          [GF_DEFRAG_STATUS_FAILED] = 2,
+                          [GF_DEFRAG_STATUS_STOPPED] = 3,
+                          [GF_DEFRAG_STATUS_COMPLETE] = 4,
+                          [GF_DEFRAG_STATUS_NOT_STARTED] = 5};
+            if (rank[remote_status] <= rank[local_status])
+                ret = dict_set_int32n(ctx_dict, key, keylen, remote_status);
+            if (ret) {
+                gf_msg(this->name, GF_LOG_ERROR, 0,
+                       GD_MSG_TASK_STATUS_UPDATE_FAIL,
+                       "Failed to "
+                       "update task status");
+                goto out;
+            }
+            break;
+        }
+    }
 
-        /* Update the tasks statuses. For every remote tasks, search for the
-         * local task, and update the local task status based on the remote
-         * status.
-         */
-        for (i = 0; i < remote_count; i++) {
+out:
+    return ret;
+}
 
-                memset (key, 0, sizeof (key));
-                snprintf (key, sizeof (key), "task%d.type", i);
-                ret = dict_get_str (rsp_dict, key, &task_type);
-                if (ret) {
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "Failed to get task typpe from rsp dict");
-                        goto out;
-                }
+gf_boolean_t
+glusterd_status_has_tasks(int cmd)
+{
+    if (((cmd & GF_CLI_STATUS_MASK) == GF_CLI_STATUS_NONE) &&
+        (cmd & GF_CLI_STATUS_VOL))
+        return _gf_true;
+    return _gf_false;
+}
 
-                /* Skip replace-brick status as it is going to be the same on
-                 * all peers. rb_status is set by the replace brick commit
-                 * function on all peers based on the replace brick command.
-                 * We return the value of rb_status as the status for a
-                 * replace-brick task in a 'volume status' command.
-                 */
-                if (!strcmp (task_type, "Replace brick"))
-                        continue;
+int
+glusterd_volume_status_copy_to_op_ctx_dict(dict_t *aggr, dict_t *rsp_dict)
+{
+    int ret = 0;
+    glusterd_status_rsp_conv_t rsp_ctx = {0};
+    int32_t cmd = GF_CLI_STATUS_NONE;
+    int32_t node_count = 0;
+    int32_t other_count = 0;
+    int32_t brick_index_max = -1;
+    int32_t hot_brick_count = -1;
+    int32_t type = -1;
+    int32_t rsp_node_count = 0;
+    int32_t rsp_other_count = 0;
+    int vol_count = -1;
+    int i = 0;
+    dict_t *ctx_dict = NULL;
+    char key[64] = "";
+    int keylen;
+    char *volname = NULL;
+    glusterd_volinfo_t *volinfo = NULL;
+
+    GF_ASSERT(rsp_dict);
+    xlator_t *this = THIS;
+    GF_ASSERT(this);
+
+    if (aggr) {
+        ctx_dict = aggr;
+
+    } else {
+        ctx_dict = glusterd_op_get_ctx(GD_OP_STATUS_VOLUME);
+    }
+
+    ret = dict_get_int32n(ctx_dict, "cmd", SLEN("cmd"), &cmd);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED, "Key=cmd",
+                NULL);
+        goto out;
+    }
+
+    if (cmd & GF_CLI_STATUS_ALL && is_origin_glusterd(ctx_dict)) {
+        ret = dict_get_int32n(rsp_dict, "vol_count", SLEN("vol_count"),
+                              &vol_count);
+        if (ret == 0) {
+            ret = dict_set_int32n(ctx_dict, "vol_count", SLEN("vol_count"),
+                                  vol_count);
+            if (ret) {
+                gf_smsg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+                        "Key=vol_count", NULL);
+                goto out;
+            }
 
-                memset (key, 0, sizeof (key));
-                snprintf (key, sizeof (key), "task%d.status", i);
-                ret = dict_get_int32 (rsp_dict, key, &remote_status);
+            for (i = 0; i < vol_count; i++) {
+                keylen = snprintf(key, sizeof(key), "vol%d", i);
+                ret = dict_get_strn(rsp_dict, key, keylen, &volname);
                 if (ret) {
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "Failed to get task status from rsp dict");
-                        goto out;
+                    gf_smsg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+                            "Key=%s", key, NULL);
+                    goto out;
                 }
-                snprintf (key, sizeof (key), "task%d.id", i);
-                ret = dict_get_str (rsp_dict, key, &remote_task_id);
+
+                ret = dict_set_strn(ctx_dict, key, keylen, volname);
                 if (ret) {
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "Failed to get task id from rsp dict");
-                        goto out;
-                }
-                for (j = 0; j < local_count; j++) {
-                        memset (key, 0, sizeof (key));
-                        snprintf (key, sizeof (key), "task%d.id", j);
-                        ret = dict_get_str (ctx_dict, key, &local_task_id);
-                        if (ret) {
-                                gf_log (this->name, GF_LOG_ERROR,
-                                        "Failed to get local task-id");
-                                goto out;
-                        }
-
-                        if (strncmp (remote_task_id, local_task_id,
-                                     strlen (remote_task_id))) {
-                                /* Quit if a matching local task is not found */
-                                if (j == (local_count - 1)) {
-                                        gf_log (this->name, GF_LOG_ERROR,
-                                                "Could not find matching local "
-                                                "task for task %s",
-                                                remote_task_id);
-                                        goto out;
-                                }
-                                continue;
-                        }
-
-                        memset (key, 0, sizeof (key));
-                        snprintf (key, sizeof (key), "task%d.status", j);
-                        ret = dict_get_int32 (ctx_dict, key, &local_status);
-                        if (ret) {
-                                gf_log (this->name, GF_LOG_ERROR,
-                                        "Failed to get local task status");
-                                goto out;
-                        }
-
-                        /* Rebalance has 5 states,
-                         * NOT_STARTED, STARTED, STOPPED, COMPLETE, FAILED
-                         * The precedence used to determine the aggregate status
-                         * is as below,
-                         * STARTED > FAILED > STOPPED > COMPLETE > NOT_STARTED
-                         */
-                        /* TODO: Move this to a common place utilities that both
-                         * CLI and glusterd need.
-                         * Till then if the below algorithm is changed, change
-                         * it in cli_xml_output_vol_rebalance_status in
-                         * cli-xml-output.c
-                         */
-                        ret = 0;
-                        int rank[] = {
-                                [GF_DEFRAG_STATUS_STARTED] = 1,
-                                [GF_DEFRAG_STATUS_FAILED] = 2,
-                                [GF_DEFRAG_STATUS_STOPPED] = 3,
-                                [GF_DEFRAG_STATUS_COMPLETE] = 4,
-                                [GF_DEFRAG_STATUS_NOT_STARTED] = 5
-                        };
-                        if (rank[remote_status] <= rank[local_status])
-                                        ret = dict_set_int32 (ctx_dict, key,
-                                                              remote_status);
-                        if (ret) {
-                                gf_log (this->name, GF_LOG_ERROR, "Failed to "
-                                        "update task status");
-                                goto out;
-                        }
-                        break;
+                    gf_smsg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+                            "Key=%s", key, NULL);
+                    goto out;
                 }
+            }
+        } else {
+            /* Ignore the error as still the aggregation applies in
+             * case its a task sub command */
+            ret = 0;
+        }
+    }
+
+    if ((cmd & GF_CLI_STATUS_TASKS) != 0)
+        goto aggregate_tasks;
+
+    ret = dict_get_int32n(rsp_dict, "count", SLEN("count"), &rsp_node_count);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_INFO, 0, GD_MSG_DICT_GET_FAILED, "Key=count",
+                NULL);
+        ret = 0;  // no bricks in the rsp
+        goto out;
+    }
+
+    ret = dict_get_int32n(rsp_dict, "other-count", SLEN("other-count"),
+                          &rsp_other_count);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+                "Key=other-count", NULL);
+        goto out;
+    }
+
+    ret = dict_get_int32n(ctx_dict, "count", SLEN("count"), &node_count);
+    ret = dict_get_int32n(ctx_dict, "other-count", SLEN("other-count"),
+                          &other_count);
+    if (!dict_getn(ctx_dict, "brick-index-max", SLEN("brick-index-max"))) {
+        ret = dict_get_int32n(rsp_dict, "brick-index-max",
+                              SLEN("brick-index-max"), &brick_index_max);
+        if (ret) {
+            gf_smsg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+                    "Key=brick-index-max", NULL);
+            goto out;
+        }
+        ret = dict_set_int32n(ctx_dict, "brick-index-max",
+                              SLEN("brick-index-max"), brick_index_max);
+        if (ret) {
+            gf_smsg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+                    "Key=brick-index-max", NULL);
+            goto out;
+        }
+
+    } else {
+        ret = dict_get_int32n(ctx_dict, "brick-index-max",
+                              SLEN("brick-index-max"), &brick_index_max);
+        if (ret) {
+            gf_smsg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+                    "Key=brick-index-max", NULL);
+            goto out;
         }
+    }
+
+    rsp_ctx.count = node_count;
+    rsp_ctx.brick_index_max = brick_index_max;
+    rsp_ctx.other_count = other_count;
+    rsp_ctx.dict = ctx_dict;
+
+    dict_foreach(rsp_dict, glusterd_volume_status_add_peer_rsp, &rsp_ctx);
+
+    ret = dict_set_int32n(ctx_dict, "count", SLEN("count"),
+                          node_count + rsp_node_count);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+                "Key=count", NULL);
+        goto out;
+    }
+
+    ret = dict_set_int32n(ctx_dict, "other-count", SLEN("other-count"),
+                          (other_count + rsp_other_count));
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+                "Key=other-count", NULL);
+        goto out;
+    }
+
+    ret = dict_get_strn(ctx_dict, "volname", SLEN("volname"), &volname);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+                "Key=volname", NULL);
+        goto out;
+    }
+
+    ret = glusterd_volinfo_find(volname, &volinfo);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_ERROR, 0, GD_MSG_VOLINFO_GET_FAIL,
+                "Volume=%s", volname, NULL);
+        goto out;
+    }
+
+    ret = dict_set_int32n(ctx_dict, "hot_brick_count", SLEN("hot_brick_count"),
+                          hot_brick_count);
+    if (ret) {
+        gf_smsg(THIS->name, GF_LOG_ERROR, errno, GD_MSG_DICT_SET_FAILED,
+                "Key=hot_brick_count", NULL);
+        goto out;
+    }
+
+    ret = dict_set_int32n(ctx_dict, "type", SLEN("type"), type);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_SET_FAILED,
+                "Key=type", NULL);
+        goto out;
+    }
+
+aggregate_tasks:
+    /* Tasks are only present for a normal status command for a volume or
+     * for an explicit tasks status command for a volume
+     */
+    if (!(cmd & GF_CLI_STATUS_ALL) &&
+        (((cmd & GF_CLI_STATUS_TASKS) != 0) || glusterd_status_has_tasks(cmd)))
+        ret = glusterd_volume_status_aggregate_tasks_status(ctx_dict, rsp_dict);
 
 out:
-        return ret;
+    return ret;
 }
 
-gf_boolean_t
-glusterd_status_has_tasks (int cmd) {
-        if (((cmd & GF_CLI_STATUS_MASK) == GF_CLI_STATUS_NONE) &&
-             (cmd & GF_CLI_STATUS_VOL))
-                return _gf_true;
-        return _gf_false;
+int
+glusterd_max_opversion_use_rsp_dict(dict_t *dst, dict_t *src)
+{
+    int ret = -1;
+    int src_max_opversion = -1;
+    int max_opversion = -1;
+
+    GF_VALIDATE_OR_GOTO(THIS->name, dst, out);
+    GF_VALIDATE_OR_GOTO(THIS->name, src, out);
+
+    ret = dict_get_int32n(dst, "max-opversion", SLEN("max-opversion"),
+                          &max_opversion);
+    if (ret)
+        gf_msg(THIS->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+               "Maximum supported op-version not set in destination "
+               "dictionary");
+
+    ret = dict_get_int32n(src, "max-opversion", SLEN("max-opversion"),
+                          &src_max_opversion);
+    if (ret) {
+        gf_msg(THIS->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+               "Failed to get maximum supported op-version from source");
+        goto out;
+    }
+
+    if (max_opversion == -1 || src_max_opversion < max_opversion)
+        max_opversion = src_max_opversion;
+
+    ret = dict_set_int32n(dst, "max-opversion", SLEN("max-opversion"),
+                          max_opversion);
+    if (ret) {
+        gf_msg(THIS->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+               "Failed to set max op-version");
+        goto out;
+    }
+out:
+    return ret;
 }
 
 int
-glusterd_volume_status_copy_to_op_ctx_dict (dict_t *aggr, dict_t *rsp_dict)
-{
-        int                             ret = 0;
-        glusterd_status_rsp_conv_t      rsp_ctx = {0};
-        int32_t                         cmd = GF_CLI_STATUS_NONE;
-        int32_t                         node_count = 0;
-        int32_t                         other_count = 0;
-        int32_t                         brick_index_max = -1;
-        int32_t                         rsp_node_count = 0;
-        int32_t                         rsp_other_count = 0;
-        int                             vol_count = -1;
-        int                             i = 0;
-        dict_t                          *ctx_dict = NULL;
-        char                            key[PATH_MAX] = {0,};
-        char                            *volname = NULL;
-
-        GF_ASSERT (rsp_dict);
+glusterd_volume_bitrot_scrub_use_rsp_dict(dict_t *aggr, dict_t *rsp_dict)
+{
+    int ret = -1;
+    int j = 0;
+    uint64_t value = 0;
+    char key[64] = "";
+    int keylen;
+    char *last_scrub_time = NULL;
+    char *scrub_time = NULL;
+    char *volname = NULL;
+    char *node_uuid = NULL;
+    char *node_uuid_str = NULL;
+    char *bitd_log = NULL;
+    char *scrub_log = NULL;
+    char *scrub_freq = NULL;
+    char *scrub_state = NULL;
+    char *scrub_impact = NULL;
+    char *bad_gfid_str = NULL;
+    xlator_t *this = NULL;
+    glusterd_conf_t *priv = NULL;
+    glusterd_volinfo_t *volinfo = NULL;
+    int src_count = 0;
+    int dst_count = 0;
+    int8_t scrub_running = 0;
+
+    this = THIS;
+    GF_ASSERT(this);
+
+    priv = this->private;
+    GF_ASSERT(priv);
+
+    ret = dict_get_strn(aggr, "volname", SLEN("volname"), &volname);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+               "Unable to get volume name");
+        goto out;
+    }
+
+    ret = glusterd_volinfo_find(volname, &volinfo);
+    if (ret) {
+        gf_msg(THIS->name, GF_LOG_ERROR, 0, GD_MSG_VOL_NOT_FOUND,
+               "Unable to find volinfo for volume: %s", volname);
+        goto out;
+    }
+
+    ret = dict_get_int32n(aggr, "count", SLEN("count"), &dst_count);
+
+    ret = dict_get_int32n(rsp_dict, "count", SLEN("count"), &src_count);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+               "failed to get count value");
+        ret = 0;
+        goto out;
+    }
+
+    ret = dict_set_int32n(aggr, "count", SLEN("count"), src_count + dst_count);
+    if (ret)
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+               "Failed to set count in dictonary");
+
+    keylen = snprintf(key, sizeof(key), "node-uuid-%d", src_count);
+    ret = dict_get_strn(rsp_dict, key, keylen, &node_uuid);
+    if (!ret) {
+        node_uuid_str = gf_strdup(node_uuid);
+        keylen = snprintf(key, sizeof(key), "node-uuid-%d",
+                          src_count + dst_count);
+        ret = dict_set_dynstrn(aggr, key, keylen, node_uuid_str);
+        if (ret) {
+            gf_msg_debug(this->name, 0, "failed to set node-uuid");
+        }
+    }
 
-        if (aggr) {
-                ctx_dict = aggr;
+    snprintf(key, sizeof(key), "scrub-running-%d", src_count);
+    ret = dict_get_int8(rsp_dict, key, &scrub_running);
+    if (!ret) {
+        snprintf(key, sizeof(key), "scrub-running-%d", src_count + dst_count);
+        ret = dict_set_int8(aggr, key, scrub_running);
+        if (ret) {
+            gf_msg_debug(this->name, 0,
+                         "Failed to set "
+                         "scrub-running value");
+        }
+    }
 
-        } else {
-                ctx_dict = glusterd_op_get_ctx (GD_OP_STATUS_VOLUME);
+    snprintf(key, sizeof(key), "scrubbed-files-%d", src_count);
+    ret = dict_get_uint64(rsp_dict, key, &value);
+    if (!ret) {
+        snprintf(key, sizeof(key), "scrubbed-files-%d", src_count + dst_count);
+        ret = dict_set_uint64(aggr, key, value);
+        if (ret) {
+            gf_msg_debug(this->name, 0,
+                         "Failed to set "
+                         "scrubbed-file value");
+        }
+    }
 
+    snprintf(key, sizeof(key), "unsigned-files-%d", src_count);
+    ret = dict_get_uint64(rsp_dict, key, &value);
+    if (!ret) {
+        snprintf(key, sizeof(key), "unsigned-files-%d", src_count + dst_count);
+        ret = dict_set_uint64(aggr, key, value);
+        if (ret) {
+            gf_msg_debug(this->name, 0,
+                         "Failed to set "
+                         "unsigned-file value");
+        }
+    }
+
+    keylen = snprintf(key, sizeof(key), "last-scrub-time-%d", src_count);
+    ret = dict_get_strn(rsp_dict, key, keylen, &last_scrub_time);
+    if (!ret) {
+        scrub_time = gf_strdup(last_scrub_time);
+        keylen = snprintf(key, sizeof(key), "last-scrub-time-%d",
+                          src_count + dst_count);
+        ret = dict_set_dynstrn(aggr, key, keylen, scrub_time);
+        if (ret) {
+            gf_msg_debug(this->name, 0,
+                         "Failed to set "
+                         "last scrub time value");
         }
+    }
 
-        ret = dict_get_int32 (ctx_dict, "cmd", &cmd);
-        if (ret)
-                goto out;
+    snprintf(key, sizeof(key), "scrub-duration-%d", src_count);
+    ret = dict_get_uint64(rsp_dict, key, &value);
+    if (!ret) {
+        snprintf(key, sizeof(key), "scrub-duration-%d", src_count + dst_count);
+        ret = dict_set_uint64(aggr, key, value);
+        if (ret) {
+            gf_msg_debug(this->name, 0,
+                         "Failed to set "
+                         "scrubbed-duration value");
+        }
+    }
 
-        if (cmd & GF_CLI_STATUS_ALL && is_origin_glusterd (ctx_dict)) {
-                ret = dict_get_int32 (rsp_dict, "vol_count", &vol_count);
-                if (ret == 0) {
-                        ret = dict_set_int32 (ctx_dict, "vol_count",
-                                              vol_count);
-                        if (ret)
-                                goto out;
-
-                        for (i = 0; i < vol_count; i++) {
-                                memset (key, 0, sizeof (key));
-                                snprintf (key, sizeof (key), "vol%d", i);
-                                ret = dict_get_str (rsp_dict, key, &volname);
-                                if (ret)
-                                        goto out;
-
-                                ret = dict_set_str (ctx_dict, key, volname);
-                                if (ret)
-                                        goto out;
-                        }
+    snprintf(key, sizeof(key), "error-count-%d", src_count);
+    ret = dict_get_uint64(rsp_dict, key, &value);
+    if (!ret) {
+        snprintf(key, sizeof(key), "error-count-%d", src_count + dst_count);
+        ret = dict_set_uint64(aggr, key, value);
+        if (ret) {
+            gf_msg_debug(this->name, 0,
+                         "Failed to set error "
+                         "count value");
+        }
+
+        /* Storing all the bad files in the dictionary */
+        for (j = 0; j < value; j++) {
+            keylen = snprintf(key, sizeof(key), "quarantine-%d-%d", j,
+                              src_count);
+            ret = dict_get_strn(rsp_dict, key, keylen, &bad_gfid_str);
+            if (!ret) {
+                snprintf(key, sizeof(key), "quarantine-%d-%d", j,
+                         src_count + dst_count);
+                ret = dict_set_dynstr_with_alloc(aggr, key, bad_gfid_str);
+                if (ret) {
+                    gf_msg_debug(this->name, 0,
+                                 "Failed to"
+                                 "bad file gfid ");
                 }
+            }
         }
+    }
 
-        if ((cmd & GF_CLI_STATUS_TASKS) != 0)
-                goto aggregate_tasks;
-
-        ret = dict_get_int32 (rsp_dict, "count", &rsp_node_count);
+    ret = dict_get_strn(rsp_dict, "bitrot_log_file", SLEN("bitrot_log_file"),
+                        &bitd_log);
+    if (!ret) {
+        ret = dict_set_dynstr_with_alloc(aggr, "bitrot_log_file", bitd_log);
         if (ret) {
-                ret = 0; //no bricks in the rsp
-                goto out;
+            gf_msg_debug(this->name, 0,
+                         "Failed to set "
+                         "bitrot log file location");
+            goto out;
         }
+    }
 
-        ret = dict_get_int32 (rsp_dict, "other-count", &rsp_other_count);
+    ret = dict_get_strn(rsp_dict, "scrub_log_file", SLEN("scrub_log_file"),
+                        &scrub_log);
+    if (!ret) {
+        ret = dict_set_dynstr_with_alloc(aggr, "scrub_log_file", scrub_log);
         if (ret) {
-                gf_log (THIS->name, GF_LOG_ERROR,
-                        "Failed to get other count from rsp_dict");
-                goto out;
+            gf_msg_debug(this->name, 0,
+                         "Failed to set "
+                         "scrubber log file location");
+            goto out;
         }
+    }
 
-        ret = dict_get_int32 (ctx_dict, "count", &node_count);
-        ret = dict_get_int32 (ctx_dict, "other-count", &other_count);
-        if (!dict_get (ctx_dict, "brick-index-max")) {
-                ret = dict_get_int32 (rsp_dict, "brick-index-max", &brick_index_max);
-                if (ret)
-                        goto out;
-                ret = dict_set_int32 (ctx_dict, "brick-index-max", brick_index_max);
-                if (ret)
-                        goto out;
-
-        } else {
-                ret = dict_get_int32 (ctx_dict, "brick-index-max", &brick_index_max);
+    ret = dict_get_strn(rsp_dict, "features.scrub-freq",
+                        SLEN("features.scrub-freq"), &scrub_freq);
+    if (!ret) {
+        ret = dict_set_dynstr_with_alloc(aggr, "features.scrub-freq",
+                                         scrub_freq);
+        if (ret) {
+            gf_msg_debug(this->name, 0,
+                         "Failed to set "
+                         "scrub-frequency value to dictionary");
+            goto out;
         }
+    }
 
-        rsp_ctx.count = node_count;
-        rsp_ctx.brick_index_max = brick_index_max;
-        rsp_ctx.other_count = other_count;
-        rsp_ctx.dict = ctx_dict;
-
-        dict_foreach (rsp_dict, glusterd_volume_status_add_peer_rsp, &rsp_ctx);
-
-        ret = dict_set_int32 (ctx_dict, "count", node_count + rsp_node_count);
+    ret = dict_get_strn(rsp_dict, "features.scrub-throttle",
+                        SLEN("features.scrub-throttle"), &scrub_impact);
+    if (!ret) {
+        ret = dict_set_dynstr_with_alloc(aggr, "features.scrub-throttle",
+                                         scrub_impact);
         if (ret) {
-                gf_log (THIS->name, GF_LOG_ERROR,
-                        "Failed to update node count");
-                goto out;
+            gf_msg_debug(this->name, 0,
+                         "Failed to set "
+                         "scrub-throttle value to dictionary");
+            goto out;
         }
+    }
 
-        ret = dict_set_int32 (ctx_dict, "other-count",
-                              (other_count + rsp_other_count));
+    ret = dict_get_strn(rsp_dict, "features.scrub", SLEN("features.scrub"),
+                        &scrub_state);
+    if (!ret) {
+        ret = dict_set_dynstr_with_alloc(aggr, "features.scrub", scrub_state);
         if (ret) {
-                gf_log (THIS->name, GF_LOG_ERROR,
-                        "Failed to update other-count");
-                goto out;
+            gf_msg_debug(this->name, 0,
+                         "Failed to set "
+                         "scrub state value to dictionary");
+            goto out;
         }
+    }
 
-aggregate_tasks:
-        /* Tasks are only present for a normal status command for a volume or
-         * for an explicit tasks status command for a volume
-         */
-        if (!(cmd & GF_CLI_STATUS_ALL) &&
-            (((cmd & GF_CLI_STATUS_TASKS) != 0) ||
-             glusterd_status_has_tasks (cmd)))
-                ret = glusterd_volume_status_aggregate_tasks_status (ctx_dict,
-                                                                     rsp_dict);
-
+    ret = 0;
 out:
-        return ret;
+    return ret;
 }
 
 int
-glusterd_volume_rebalance_use_rsp_dict (dict_t *aggr, dict_t *rsp_dict)
-{
-        char                 key[256]      = {0,};
-        char                *node_uuid     = NULL;
-        char                *node_uuid_str = NULL;
-        char                *volname       = NULL;
-        dict_t              *ctx_dict      = NULL;
-        double               elapsed_time  = 0;
-        glusterd_conf_t     *conf          = NULL;
-        glusterd_op_t        op            = GD_OP_NONE;
-        glusterd_peerinfo_t *peerinfo      = NULL;
-        glusterd_volinfo_t  *volinfo       = NULL;
-        int                  ret           = 0;
-        int32_t              index         = 0;
-        int32_t              count         = 0;
-        int32_t              current_index = 2;
-        int32_t              value32       = 0;
-        uint64_t             value         = 0;
-        char                *peer_uuid_str = NULL;
+glusterd_bitrot_volume_node_rsp(dict_t *aggr, dict_t *rsp_dict)
+{
+    int ret = -1;
+    uint64_t value = 0;
+    char key[64] = "";
+    int keylen;
+    char buf[1024] = "";
+    int32_t i = 0;
+    int32_t j = 0;
+    char *last_scrub_time = NULL;
+    char *scrub_time = NULL;
+    char *volname = NULL;
+    char *scrub_freq = NULL;
+    char *scrub_state = NULL;
+    char *scrub_impact = NULL;
+    char *bad_gfid_str = NULL;
+    xlator_t *this = NULL;
+    glusterd_conf_t *priv = NULL;
+    glusterd_volinfo_t *volinfo = NULL;
+    int8_t scrub_running = 0;
+
+    this = THIS;
+    GF_ASSERT(this);
+
+    priv = this->private;
+    GF_ASSERT(priv);
+
+    ret = dict_set_strn(aggr, "bitrot_log_file", SLEN("bitrot_log_file"),
+                        priv->bitd_svc.proc.logfile);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+               "Failed to set bitrot log file location");
+        goto out;
+    }
+
+    ret = dict_set_strn(aggr, "scrub_log_file", SLEN("scrub_log_file"),
+                        priv->scrub_svc.proc.logfile);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+               "Failed to set scrubber log file location");
+        goto out;
+    }
+
+    ret = dict_get_strn(aggr, "volname", SLEN("volname"), &volname);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+               "Unable to get volume name");
+        goto out;
+    }
+
+    ret = glusterd_volinfo_find(volname, &volinfo);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_VOL_NOT_FOUND,
+               "Unable to find volinfo for volume: %s", volname);
+        goto out;
+    }
+
+    ret = dict_get_int32n(aggr, "count", SLEN("count"), &i);
+    i++;
+
+    ret = dict_set_int32n(aggr, "count", SLEN("count"), i);
+    if (ret)
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+               "Failed to set count");
+
+    snprintf(buf, sizeof(buf), "%s", uuid_utoa(MY_UUID));
+
+    snprintf(key, sizeof(key), "node-uuid-%d", i);
+    ret = dict_set_dynstr_with_alloc(aggr, key, buf);
+    if (ret)
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+               "failed to set node-uuid");
+
+    ret = dict_get_strn(volinfo->dict, "features.scrub-freq",
+                        SLEN("features.scrub-freq"), &scrub_freq);
+    if (!ret) {
+        ret = dict_set_strn(aggr, "features.scrub-freq",
+                            SLEN("features.scrub-freq"), scrub_freq);
+        if (ret) {
+            gf_msg_debug(this->name, 0,
+                         "Failed to set "
+                         "scrub-frequency value to dictionary");
+        }
+    } else {
+        /* By Default scrub-frequency is bi-weekly. So when user
+         * enable bitrot then scrub-frequency value will not be
+         * present in volinfo->dict. Setting by-default value of
+         * scrub-frequency explicitly for presenting it to scrub
+         * status.
+         */
+        ret = dict_set_dynstr_with_alloc(aggr, "features.scrub-freq",
+                                         "biweekly");
+        if (ret) {
+            gf_msg_debug(this->name, 0,
+                         "Failed to set "
+                         "scrub-frequency value to dictionary");
+        }
+    }
 
-        GF_ASSERT (rsp_dict);
-        conf = THIS->private;
+    ret = dict_get_strn(volinfo->dict, "features.scrub-throttle",
+                        SLEN("features.scrub-throttle"), &scrub_impact);
+    if (!ret) {
+        ret = dict_set_strn(aggr, "features.scrub-throttle",
+                            SLEN("features.scrub-throttle"), scrub_impact);
+        if (ret) {
+            gf_msg_debug(this->name, 0,
+                         "Failed to set "
+                         "scrub-throttle value to dictionary");
+        }
+    } else {
+        /* By Default scrub-throttle is lazy. So when user
+         * enable bitrot then scrub-throttle value will not be
+         * present in volinfo->dict. Setting by-default value of
+         * scrub-throttle explicitly for presenting it to
+         * scrub status.
+         */
+        ret = dict_set_dynstr_with_alloc(aggr, "features.scrub-throttle",
+                                         "lazy");
+        if (ret) {
+            gf_msg_debug(this->name, 0,
+                         "Failed to set "
+                         "scrub-throttle value to dictionary");
+        }
+    }
 
-        op = glusterd_op_get_op ();
-        GF_ASSERT ((GD_OP_REBALANCE == op) ||
-                   (GD_OP_DEFRAG_BRICK_VOLUME == op));
+    ret = dict_get_strn(volinfo->dict, "features.scrub", SLEN("features.scrub"),
+                        &scrub_state);
+    if (!ret) {
+        ret = dict_set_strn(aggr, "features.scrub", SLEN("features.scrub"),
+                            scrub_state);
+        if (ret) {
+            gf_msg_debug(this->name, 0,
+                         "Failed to set "
+                         "scrub state value to dictionary");
+        }
+    }
 
-        if (aggr) {
-                ctx_dict = aggr;
+    ret = dict_get_int8(rsp_dict, "scrub-running", &scrub_running);
+    if (!ret) {
+        snprintf(key, sizeof(key), "scrub-running-%d", i);
+        ret = dict_set_uint64(aggr, key, scrub_running);
+        if (ret) {
+            gf_msg_debug(this->name, 0,
+                         "Failed to set "
+                         "scrub-running value");
+        }
+    }
 
-        } else {
-                ctx_dict = glusterd_op_get_ctx (op);
+    ret = dict_get_uint64(rsp_dict, "scrubbed-files", &value);
+    if (!ret) {
+        snprintf(key, sizeof(key), "scrubbed-files-%d", i);
+        ret = dict_set_uint64(aggr, key, value);
+        if (ret) {
+            gf_msg_debug(this->name, 0,
+                         "Failed to set "
+                         "scrubbed-file value");
+        }
+    }
 
+    ret = dict_get_uint64(rsp_dict, "unsigned-files", &value);
+    if (!ret) {
+        snprintf(key, sizeof(key), "unsigned-files-%d", i);
+        ret = dict_set_uint64(aggr, key, value);
+        if (ret) {
+            gf_msg_debug(this->name, 0,
+                         "Failed to set "
+                         "unsigned-file value");
         }
+    }
 
-        if (!ctx_dict)
-                goto out;
+    ret = dict_get_strn(rsp_dict, "last-scrub-time", SLEN("last-scrub-time"),
+                        &last_scrub_time);
+    if (!ret) {
+        keylen = snprintf(key, sizeof(key), "last-scrub-time-%d", i);
 
-        ret = dict_get_str (ctx_dict, "volname", &volname);
+        scrub_time = gf_strdup(last_scrub_time);
+        ret = dict_set_dynstrn(aggr, key, keylen, scrub_time);
         if (ret) {
-                gf_log ("", GF_LOG_ERROR, "Unable to get volume name");
-                goto out;
+            gf_msg_debug(this->name, 0,
+                         "Failed to set "
+                         "last scrub time value");
         }
+    }
 
-        ret  = glusterd_volinfo_find (volname, &volinfo);
+    ret = dict_get_uint64(rsp_dict, "scrub-duration", &value);
+    if (!ret) {
+        snprintf(key, sizeof(key), "scrub-duration-%d", i);
+        ret = dict_set_uint64(aggr, key, value);
+        if (ret) {
+            gf_msg_debug(this->name, 0,
+                         "Failed to set "
+                         "scrubbed-duration value");
+        }
+    }
 
-        if (ret)
-                goto out;
+    ret = dict_get_uint64(rsp_dict, "total-count", &value);
+    if (!ret) {
+        snprintf(key, sizeof(key), "error-count-%d", i);
+        ret = dict_set_uint64(aggr, key, value);
+        if (ret) {
+            gf_msg_debug(this->name, 0,
+                         "Failed to set error "
+                         "count value");
+        }
+
+        /* Storing all the bad files in the dictionary */
+        for (j = 0; j < value; j++) {
+            keylen = snprintf(key, sizeof(key), "quarantine-%d", j);
+            ret = dict_get_strn(rsp_dict, key, keylen, &bad_gfid_str);
+            if (!ret) {
+                snprintf(key, sizeof(key), "quarantine-%d-%d", j, i);
+                ret = dict_set_dynstr_with_alloc(aggr, key, bad_gfid_str);
+                if (ret) {
+                    gf_msg_debug(this->name, 0,
+                                 "Failed to"
+                                 "bad file gfid ");
+                }
+            }
+        }
+    }
 
-        ret = dict_get_int32 (rsp_dict, "count", &index);
-        if (ret)
-                gf_log ("", GF_LOG_ERROR, "failed to get index");
+    ret = 0;
+out:
+    return ret;
+}
 
-        memset (key, 0, 256);
-        snprintf (key, 256, "node-uuid-%d", index);
-        ret = dict_get_str (rsp_dict, key, &node_uuid);
-        if (!ret) {
-                node_uuid_str = gf_strdup (node_uuid);
+int
+glusterd_volume_rebalance_use_rsp_dict(dict_t *aggr, dict_t *rsp_dict)
+{
+    char key[64] = "";
+    int keylen;
+    char *node_uuid = NULL;
+    char *node_uuid_str = NULL;
+    char *volname = NULL;
+    dict_t *ctx_dict = NULL;
+    double elapsed_time = 0;
+    glusterd_conf_t *conf = NULL;
+    glusterd_peerinfo_t *peerinfo = NULL;
+    glusterd_volinfo_t *volinfo = NULL;
+    int ret = 0;
+    int32_t index = 0;
+    int32_t count = 0;
+    int32_t current_index = 1;
+    int32_t value32 = 0;
+    uint64_t value = 0;
+    char *peer_uuid_str = NULL;
+    xlator_t *this = NULL;
+
+    GF_ASSERT(rsp_dict);
+    this = THIS;
+    GF_ASSERT(this);
+    conf = this->private;
+
+    if (conf->op_version < GD_OP_VERSION_6_0)
+        current_index = 2;
+    if (aggr) {
+        ctx_dict = aggr;
+
+    } else {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_OPCTX_GET_FAIL,
+               "Operation Context is not present");
+        goto out;
+    }
+
+    ret = dict_get_strn(ctx_dict, "volname", SLEN("volname"), &volname);
+    if (ret) {
+        gf_msg("glusterd", GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+               "Unable to get volume name");
+        goto out;
+    }
+
+    ret = glusterd_volinfo_find(volname, &volinfo);
+
+    if (ret)
+        goto out;
+
+    ret = dict_get_int32n(rsp_dict, "count", SLEN("count"), &index);
+    if (ret)
+        gf_msg("glusterd", GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+               "failed to get index from rsp dict");
+
+    keylen = snprintf(key, sizeof(key), "node-uuid-%d", index);
+    ret = dict_get_strn(rsp_dict, key, keylen, &node_uuid);
+    if (!ret) {
+        node_uuid_str = gf_strdup(node_uuid);
+
+        /* Finding the index of the node-uuid in the peer-list */
+        RCU_READ_LOCK;
+        cds_list_for_each_entry_rcu(peerinfo, &conf->peers, uuid_list)
+        {
+            peer_uuid_str = gd_peer_uuid_str(peerinfo);
+            if (strcmp(peer_uuid_str, node_uuid_str) == 0)
+                break;
 
-                /* Finding the index of the node-uuid in the peer-list */
-                rcu_read_lock ();
-                cds_list_for_each_entry_rcu (peerinfo, &conf->peers,
-                                             uuid_list) {
-                        peer_uuid_str = gd_peer_uuid_str (peerinfo);
-                        if (strcmp (peer_uuid_str, node_uuid_str) == 0)
-                                break;
+            current_index++;
+        }
+        RCU_READ_UNLOCK;
 
-                        current_index++;
-                }
-                rcu_read_unlock ();
-
-                /* Setting the largest index value as the total count. */
-                ret = dict_get_int32 (ctx_dict, "count", &count);
-                if (count < current_index) {
-                    ret = dict_set_int32 (ctx_dict, "count", current_index);
-                    if (ret)
-                            gf_log ("", GF_LOG_ERROR, "Failed to set count");
-                }
+        /* Setting the largest index value as the total count. */
+        ret = dict_get_int32n(ctx_dict, "count", SLEN("count"), &count);
+        if (count < current_index) {
+            ret = dict_set_int32n(ctx_dict, "count", SLEN("count"),
+                                  current_index);
+            if (ret)
+                gf_msg("glusterd", GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+                       "Failed to set count");
+        }
 
-                /* Setting the same index for the node, as is in the peerlist.*/
-                memset (key, 0, 256);
-                snprintf (key, 256, "node-uuid-%d", current_index);
-                ret = dict_set_dynstr (ctx_dict, key, node_uuid_str);
-                if (ret) {
-                        gf_log (THIS->name, GF_LOG_DEBUG,
-                                "failed to set node-uuid");
-                }
+        /* Setting the same index for the node, as is in the peerlist.*/
+        keylen = snprintf(key, sizeof(key), "node-uuid-%d", current_index);
+        ret = dict_set_dynstrn(ctx_dict, key, keylen, node_uuid_str);
+        if (ret) {
+            gf_msg_debug(THIS->name, 0, "failed to set node-uuid");
         }
+    }
 
-        snprintf (key, 256, "files-%d", index);
-        ret = dict_get_uint64 (rsp_dict, key, &value);
-        if (!ret) {
-                memset (key, 0, 256);
-                snprintf (key, 256, "files-%d", current_index);
-                ret = dict_set_uint64 (ctx_dict, key, value);
-                if (ret) {
-                        gf_log (THIS->name, GF_LOG_DEBUG,
-                                "failed to set the file count");
-                }
+    snprintf(key, sizeof(key), "files-%d", index);
+    ret = dict_get_uint64(rsp_dict, key, &value);
+    if (!ret) {
+        snprintf(key, sizeof(key), "files-%d", current_index);
+        ret = dict_set_uint64(ctx_dict, key, value);
+        if (ret) {
+            gf_msg_debug(THIS->name, 0, "failed to set the file count");
         }
+    }
 
-        memset (key, 0, 256);
-        snprintf (key, 256, "size-%d", index);
-        ret = dict_get_uint64 (rsp_dict, key, &value);
-        if (!ret) {
-                memset (key, 0, 256);
-                snprintf (key, 256, "size-%d", current_index);
-                ret = dict_set_uint64 (ctx_dict, key, value);
-                if (ret) {
-                        gf_log (THIS->name, GF_LOG_DEBUG,
-                                "failed to set the size of migration");
-                }
+    snprintf(key, sizeof(key), "size-%d", index);
+    ret = dict_get_uint64(rsp_dict, key, &value);
+    if (!ret) {
+        snprintf(key, sizeof(key), "size-%d", current_index);
+        ret = dict_set_uint64(ctx_dict, key, value);
+        if (ret) {
+            gf_msg_debug(THIS->name, 0, "failed to set the size of migration");
         }
+    }
 
-        memset (key, 0, 256);
-        snprintf (key, 256, "lookups-%d", index);
-        ret = dict_get_uint64 (rsp_dict, key, &value);
-        if (!ret) {
-                memset (key, 0, 256);
-                snprintf (key, 256, "lookups-%d", current_index);
-                ret = dict_set_uint64 (ctx_dict, key, value);
-                if (ret) {
-                        gf_log (THIS->name, GF_LOG_DEBUG,
-                                "failed to set lookuped file count");
-                }
+    snprintf(key, sizeof(key), "lookups-%d", index);
+    ret = dict_get_uint64(rsp_dict, key, &value);
+    if (!ret) {
+        snprintf(key, sizeof(key), "lookups-%d", current_index);
+        ret = dict_set_uint64(ctx_dict, key, value);
+        if (ret) {
+            gf_msg_debug(THIS->name, 0, "failed to set looked up file count");
         }
+    }
 
-        memset (key, 0, 256);
-        snprintf (key, 256, "status-%d", index);
-        ret = dict_get_int32 (rsp_dict, key, &value32);
-        if (!ret) {
-                memset (key, 0, 256);
-                snprintf (key, 256, "status-%d", current_index);
-                ret = dict_set_int32 (ctx_dict, key, value32);
-                if (ret) {
-                        gf_log (THIS->name, GF_LOG_DEBUG,
-                                "failed to set status");
-                }
+    keylen = snprintf(key, sizeof(key), "status-%d", index);
+    ret = dict_get_int32n(rsp_dict, key, keylen, &value32);
+    if (!ret) {
+        keylen = snprintf(key, sizeof(key), "status-%d", current_index);
+        ret = dict_set_int32n(ctx_dict, key, keylen, value32);
+        if (ret) {
+            gf_msg_debug(THIS->name, 0, "failed to set status");
         }
+    }
 
-        memset (key, 0, 256);
-        snprintf (key, 256, "failures-%d", index);
-        ret = dict_get_uint64 (rsp_dict, key, &value);
-        if (!ret) {
-                memset (key, 0, 256);
-                snprintf (key, 256, "failures-%d", current_index);
-                ret = dict_set_uint64 (ctx_dict, key, value);
-                if (ret) {
-                        gf_log (THIS->name, GF_LOG_DEBUG,
-                                "failed to set failure count");
-                }
+    snprintf(key, sizeof(key), "failures-%d", index);
+    ret = dict_get_uint64(rsp_dict, key, &value);
+    if (!ret) {
+        snprintf(key, sizeof(key), "failures-%d", current_index);
+        ret = dict_set_uint64(ctx_dict, key, value);
+        if (ret) {
+            gf_msg_debug(THIS->name, 0, "failed to set failure count");
         }
+    }
 
-        memset (key, 0, 256);
-        snprintf (key, 256, "skipped-%d", index);
-        ret = dict_get_uint64 (rsp_dict, key, &value);
-        if (!ret) {
-                memset (key, 0, 256);
-                snprintf (key, 256, "skipped-%d", current_index);
-                ret = dict_set_uint64 (ctx_dict, key, value);
-                if (ret) {
-                        gf_log (THIS->name, GF_LOG_DEBUG,
-                                "failed to set skipped count");
-                }
+    snprintf(key, sizeof(key), "skipped-%d", index);
+    ret = dict_get_uint64(rsp_dict, key, &value);
+    if (!ret) {
+        snprintf(key, sizeof(key), "skipped-%d", current_index);
+        ret = dict_set_uint64(ctx_dict, key, value);
+        if (ret) {
+            gf_msg_debug(THIS->name, 0, "failed to set skipped count");
+        }
+    }
+    snprintf(key, sizeof(key), "run-time-%d", index);
+    ret = dict_get_double(rsp_dict, key, &elapsed_time);
+    if (!ret) {
+        snprintf(key, sizeof(key), "run-time-%d", current_index);
+        ret = dict_set_double(ctx_dict, key, elapsed_time);
+        if (ret) {
+            gf_msg_debug(THIS->name, 0, "failed to set run-time");
         }
-        memset (key, 0, 256);
-        snprintf (key, 256, "run-time-%d", index);
-        ret = dict_get_double (rsp_dict, key, &elapsed_time);
-        if (!ret) {
-                memset (key, 0, 256);
-                snprintf (key, 256, "run-time-%d", current_index);
-                ret = dict_set_double (ctx_dict, key, elapsed_time);
-                if (ret) {
-                        gf_log (THIS->name, GF_LOG_DEBUG,
-                                "failed to set run-time");
-                }
+    }
+
+    snprintf(key, sizeof(key), "time-left-%d", index);
+    ret = dict_get_uint64(rsp_dict, key, &value);
+    if (!ret) {
+        snprintf(key, sizeof(key), "time-left-%d", current_index);
+        ret = dict_set_uint64(ctx_dict, key, value);
+        if (ret) {
+            gf_msg_debug(THIS->name, 0, "failed to set time-left");
+        }
+    }
+    snprintf(key, sizeof(key), "demoted-%d", index);
+    ret = dict_get_uint64(rsp_dict, key, &value);
+    if (!ret) {
+        snprintf(key, sizeof(key), "demoted-%d", current_index);
+        ret = dict_set_uint64(ctx_dict, key, value);
+        if (ret) {
+            gf_msg_debug(THIS->name, 0, "failed to set demoted count");
+        }
+    }
+    snprintf(key, sizeof(key), "promoted-%d", index);
+    ret = dict_get_uint64(rsp_dict, key, &value);
+    if (!ret) {
+        snprintf(key, sizeof(key), "promoted-%d", current_index);
+        ret = dict_set_uint64(ctx_dict, key, value);
+        if (ret) {
+            gf_msg_debug(THIS->name, 0, "failed to set promoted count");
         }
+    }
 
-        ret = 0;
+    ret = 0;
 
 out:
-        return ret;
+    return ret;
 }
 
 int
-glusterd_sys_exec_output_rsp_dict (dict_t *dst, dict_t *src)
-{
-        char           output_name[PATH_MAX] = "";
-        char          *output = NULL;
-        int            ret      = 0;
-        int            i      = 0;
-        int            len    = 0;
-        int            src_output_count      = 0;
-        int            dst_output_count      = 0;
+glusterd_sys_exec_output_rsp_dict(dict_t *dst, dict_t *src)
+{
+    char output_name[64] = "";
+    char *output = NULL;
+    int ret = 0;
+    int i = 0;
+    int keylen;
+    int src_output_count = 0;
+    int dst_output_count = 0;
+
+    if (!dst || !src) {
+        gf_msg("glusterd", GF_LOG_ERROR, 0, GD_MSG_DICT_EMPTY,
+               "Source or Destination "
+               "dict is empty.");
+        goto out;
+    }
+
+    ret = dict_get_int32n(dst, "output_count", SLEN("output_count"),
+                          &dst_output_count);
+
+    ret = dict_get_int32n(src, "output_count", SLEN("output_count"),
+                          &src_output_count);
+    if (ret) {
+        gf_msg_debug("glusterd", 0, "No output from source");
+        ret = 0;
+        goto out;
+    }
 
-        if (!dst || !src) {
-                gf_log ("", GF_LOG_ERROR, "Source or Destination "
-                        "dict is empty.");
-                goto out;
+    for (i = 1; i <= src_output_count; i++) {
+        keylen = snprintf(output_name, sizeof(output_name), "output_%d", i);
+        if (keylen <= 0 || keylen >= sizeof(output_name)) {
+            ret = -1;
+            goto out;
         }
-
-        ret = dict_get_int32 (dst, "output_count", &dst_output_count);
-
-        ret = dict_get_int32 (src, "output_count", &src_output_count);
+        ret = dict_get_strn(src, output_name, keylen, &output);
         if (ret) {
-                gf_log ("", GF_LOG_DEBUG, "No output from source");
-                ret = 0;
-                goto out;
+            gf_msg("glusterd", GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+                   "Unable to fetch %s", output_name);
+            goto out;
         }
 
-        for (i = 1; i <= src_output_count; i++) {
-                len = snprintf (output_name, sizeof(output_name) - 1,
-                                "output_%d", i);
-                output_name[len] = '\0';
-                ret = dict_get_str (src, output_name, &output);
-                if (ret) {
-                        gf_log ("", GF_LOG_ERROR, "Unable to fetch %s",
-                                output_name);
-                        goto out;
-                }
+        keylen = snprintf(output_name, sizeof(output_name), "output_%d",
+                          i + dst_output_count);
+        if (keylen <= 0 || keylen >= sizeof(output_name)) {
+            ret = -1;
+            goto out;
+        }
 
-                len = snprintf (output_name, sizeof(output_name) - 1,
-                                "output_%d", i+dst_output_count);
-                output_name[len] = '\0';
-                ret = dict_set_dynstr (dst, output_name, gf_strdup (output));
-                if (ret) {
-                        gf_log ("", GF_LOG_ERROR, "Unable to set %s",
-                                output_name);
-                        goto out;
-                }
+        ret = dict_set_dynstrn(dst, output_name, keylen, gf_strdup(output));
+        if (ret) {
+            gf_msg("glusterd", GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+                   "Unable to set %s", output_name);
+            goto out;
         }
+    }
 
-        ret = dict_set_int32 (dst, "output_count",
-                              dst_output_count+src_output_count);
+    ret = dict_set_int32n(dst, "output_count", SLEN("output_count"),
+                          dst_output_count + src_output_count);
 out:
-        gf_log ("", GF_LOG_DEBUG, "Returning %d", ret);
-        return ret;
+    gf_msg_debug("glusterd", 0, "Returning %d", ret);
+    return ret;
 }
 
 int
-glusterd_use_rsp_dict (dict_t *aggr, dict_t *rsp_dict)
+glusterd_use_rsp_dict(dict_t *aggr, dict_t *rsp_dict)
 {
-        int            ret      = 0;
-        glusterd_op_t  op       = GD_OP_NONE;
+    int ret = 0;
 
-        op = glusterd_op_get_op ();
-        GF_ASSERT (aggr);
-        GF_ASSERT (rsp_dict);
+    GF_ASSERT(aggr);
+    GF_ASSERT(rsp_dict);
 
-        if (!aggr)
-                goto out;
-        dict_copy (rsp_dict, aggr);
+    if (!aggr)
+        goto out;
+    dict_copy(rsp_dict, aggr);
 out:
-        return ret;
+    return ret;
 }
 
 int
-glusterd_volume_heal_use_rsp_dict (dict_t *aggr, dict_t *rsp_dict)
-{
-        int                 ret         = 0;
-        dict_t             *ctx_dict    = NULL;
-        uuid_t             *txn_id      = NULL;
-        glusterd_op_info_t  txn_op_info = {{0},};
-        glusterd_op_t       op          = GD_OP_NONE;
-
-        GF_ASSERT (rsp_dict);
-
-        ret = dict_get_bin (aggr, "transaction_id", (void **)&txn_id);
-        if (ret)
-                goto out;
-        gf_log (THIS->name, GF_LOG_DEBUG, "transaction ID = %s",
-                uuid_utoa (*txn_id));
-
-        ret = glusterd_get_txn_opinfo (txn_id, &txn_op_info);
-        if (ret) {
-                gf_log (THIS->name, GF_LOG_ERROR, "Failed to get txn_op_info "
-                        "for txn_id = %s", uuid_utoa (*txn_id));
-                goto out;
-        }
-
-        op = txn_op_info.op;
-        GF_ASSERT (GD_OP_HEAL_VOLUME == op);
-
-        if (aggr) {
-                ctx_dict = aggr;
-
-        } else {
-                ctx_dict = txn_op_info.op_ctx;
-        }
-
-        if (!ctx_dict)
-                goto out;
-        dict_copy (rsp_dict, ctx_dict);
+glusterd_volume_heal_use_rsp_dict(dict_t *aggr, dict_t *rsp_dict)
+{
+    int ret = 0;
+    dict_t *ctx_dict = NULL;
+    uuid_t *txn_id = NULL;
+    glusterd_op_info_t txn_op_info = {
+        {0},
+    };
+    glusterd_op_t op = GD_OP_NONE;
+
+    GF_ASSERT(rsp_dict);
+
+    ret = dict_get_bin(aggr, "transaction_id", (void **)&txn_id);
+    if (ret)
+        goto out;
+    gf_msg_debug(THIS->name, 0, "transaction ID = %s", uuid_utoa(*txn_id));
+
+    ret = glusterd_get_txn_opinfo(txn_id, &txn_op_info);
+    if (ret) {
+        gf_msg_callingfn(THIS->name, GF_LOG_ERROR, 0,
+                         GD_MSG_TRANS_OPINFO_GET_FAIL,
+                         "Unable to get transaction opinfo "
+                         "for transaction ID : %s",
+                         uuid_utoa(*txn_id));
+        goto out;
+    }
+
+    op = txn_op_info.op;
+    GF_ASSERT(GD_OP_HEAL_VOLUME == op);
+
+    if (aggr) {
+        ctx_dict = aggr;
+
+    } else {
+        ctx_dict = txn_op_info.op_ctx;
+    }
+
+    if (!ctx_dict)
+        goto out;
+    dict_copy(rsp_dict, ctx_dict);
 out:
-        return ret;
+    return ret;
 }
 
 int
-_profile_volume_add_brick_rsp (dict_t *this, char *key, data_t *value,
-                             void *data)
+_profile_volume_add_brick_rsp(dict_t *this, char *key, data_t *value,
+                              void *data)
 {
-        char    new_key[256] = {0};
-        glusterd_pr_brick_rsp_conv_t *rsp_ctx = NULL;
-        data_t  *new_value = NULL;
+    char new_key[256] = "";
+    int keylen;
+    glusterd_pr_brick_rsp_conv_t *rsp_ctx = NULL;
+    data_t *new_value = NULL;
 
-        rsp_ctx = data;
-        new_value = data_copy (value);
-        GF_ASSERT (new_value);
-        snprintf (new_key, sizeof (new_key), "%d-%s", rsp_ctx->count, key);
-        dict_set (rsp_ctx->dict, new_key, new_value);
-        return 0;
+    rsp_ctx = data;
+    new_value = data_copy(value);
+    GF_ASSERT(new_value);
+    keylen = snprintf(new_key, sizeof(new_key), "%d-%s", rsp_ctx->count, key);
+    dict_setn(rsp_ctx->dict, new_key, keylen, new_value);
+    return 0;
 }
 
 int
-glusterd_volume_quota_copy_to_op_ctx_dict (dict_t *dict, dict_t *rsp_dict)
-{
-        int        ret            = -1;
-        int        i              = 0;
-        int        count          = 0;
-        int        rsp_dict_count = 0;
-        char      *uuid_str       = NULL;
-        char      *uuid_str_dup   = NULL;
-        char       key[256]       = {0,};
-        xlator_t  *this           = NULL;
-        int        type           = GF_QUOTA_OPTION_TYPE_NONE;
-
-        this = THIS;
-        GF_ASSERT (this);
-
-        ret = dict_get_int32 (dict, "type", &type);
+glusterd_volume_quota_copy_to_op_ctx_dict(dict_t *dict, dict_t *rsp_dict)
+{
+    int ret = -1;
+    int i = 0;
+    int count = 0;
+    int rsp_dict_count = 0;
+    char *uuid_str = NULL;
+    char *uuid_str_dup = NULL;
+    char key[64] = "";
+    int keylen;
+    xlator_t *this = NULL;
+    int type = GF_QUOTA_OPTION_TYPE_NONE;
+
+    this = THIS;
+    GF_ASSERT(this);
+
+    ret = dict_get_int32n(dict, "type", SLEN("type"), &type);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+               "Failed to get quota opcode");
+        goto out;
+    }
+
+    if ((type != GF_QUOTA_OPTION_TYPE_LIMIT_USAGE) &&
+        (type != GF_QUOTA_OPTION_TYPE_LIMIT_OBJECTS) &&
+        (type != GF_QUOTA_OPTION_TYPE_REMOVE) &&
+        (type != GF_QUOTA_OPTION_TYPE_REMOVE_OBJECTS)) {
+        dict_copy(rsp_dict, dict);
+        ret = 0;
+        goto out;
+    }
+
+    ret = dict_get_int32n(rsp_dict, "count", SLEN("count"), &rsp_dict_count);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+               "Failed to get the count of "
+               "gfids from the rsp dict");
+        goto out;
+    }
+
+    ret = dict_get_int32n(dict, "count", SLEN("count"), &count);
+    if (ret)
+        /* The key "count" is absent in op_ctx when this function is
+         * called after self-staging on the originator. This must not
+         * be treated as error.
+         */
+        gf_msg_debug(this->name, 0,
+                     "Failed to get count of gfids"
+                     " from req dict. This could be because count is not yet"
+                     " copied from rsp_dict into op_ctx");
+
+    for (i = 0; i < rsp_dict_count; i++) {
+        keylen = snprintf(key, sizeof(key), "gfid%d", i);
+        ret = dict_get_strn(rsp_dict, key, keylen, &uuid_str);
         if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "Failed to get quota opcode");
-                goto out;
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+                   "Failed to get gfid "
+                   "from rsp dict");
+            goto out;
         }
 
-        if ((type != GF_QUOTA_OPTION_TYPE_LIMIT_USAGE) &&
-            (type != GF_QUOTA_OPTION_TYPE_LIMIT_OBJECTS) &&
-            (type != GF_QUOTA_OPTION_TYPE_REMOVE) &&
-            (type != GF_QUOTA_OPTION_TYPE_REMOVE_OBJECTS)) {
-                dict_copy (rsp_dict, dict);
-                ret = 0;
-                goto out;
+        uuid_str_dup = gf_strdup(uuid_str);
+        if (!uuid_str_dup) {
+            ret = -1;
+            goto out;
         }
 
-        ret = dict_get_int32 (rsp_dict, "count", &rsp_dict_count);
+        keylen = snprintf(key, sizeof(key), "gfid%d", i + count);
+        ret = dict_set_dynstrn(dict, key, keylen, uuid_str_dup);
         if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "Failed to get the count of "
-                        "gfids from the rsp dict");
-                goto out;
-        }
-
-        ret = dict_get_int32 (dict, "count", &count);
-        if (ret)
-                /* The key "count" is absent in op_ctx when this function is
-                 * called after self-staging on the originator. This must not
-                 * be treated as error.
-                 */
-                gf_log (this->name, GF_LOG_DEBUG, "Failed to get count of gfids"
-                        " from req dict. This could be because count is not yet"
-                        " copied from rsp_dict into op_ctx");
-
-        for (i = 0; i < rsp_dict_count; i++) {
-                snprintf (key, sizeof(key)-1, "gfid%d", i);
-
-                ret = dict_get_str (rsp_dict, key, &uuid_str);
-                if (ret) {
-                        gf_log (this->name, GF_LOG_ERROR, "Failed to get gfid "
-                                "from rsp dict");
-                        goto out;
-                }
-
-                snprintf (key, sizeof (key)-1, "gfid%d", i + count);
-
-                uuid_str_dup = gf_strdup (uuid_str);
-                if (!uuid_str_dup) {
-                        ret = -1;
-                        goto out;
-                }
-
-                ret = dict_set_dynstr (dict, key, uuid_str_dup);
-                if (ret) {
-                        gf_log (this->name, GF_LOG_ERROR, "Failed to set gfid "
-                                "from rsp dict into req dict");
-                        GF_FREE (uuid_str_dup);
-                        goto out;
-                }
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+                   "Failed to set gfid "
+                   "from rsp dict into req dict");
+            GF_FREE(uuid_str_dup);
+            goto out;
         }
+    }
 
-        ret = dict_set_int32 (dict, "count", rsp_dict_count + count);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR,  "Failed to set aggregated "
-                        "count in req dict");
-                goto out;
-        }
+    ret = dict_set_int32n(dict, "count", SLEN("count"), rsp_dict_count + count);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+               "Failed to set aggregated "
+               "count in req dict");
+        goto out;
+    }
 
 out:
-        return ret;
+    return ret;
 }
 
 int
-glusterd_profile_volume_brick_rsp (void *pending_entry,
-                                   dict_t *rsp_dict, dict_t *op_ctx,
-                                   char **op_errstr, gd_node_type type)
-{
-        int                             ret = 0;
-        glusterd_pr_brick_rsp_conv_t    rsp_ctx = {0};
-        int32_t                         count = 0;
-        char                            brick[PATH_MAX+1024] = {0};
-        char                            key[256] = {0};
-        char                            *full_brick = NULL;
-        glusterd_brickinfo_t            *brickinfo = NULL;
-        xlator_t                        *this = NULL;
-        glusterd_conf_t                 *priv = NULL;
-
-        GF_ASSERT (rsp_dict);
-        GF_ASSERT (op_ctx);
-        GF_ASSERT (op_errstr);
-        GF_ASSERT (pending_entry);
-
-        this = THIS;
-        GF_ASSERT (this);
-        priv = this->private;
-        GF_ASSERT (priv);
-
-        ret = dict_get_int32 (op_ctx, "count", &count);
-        if (ret) {
-                count = 1;
-        } else {
-                count++;
-        }
-        snprintf (key, sizeof (key), "%d-brick", count);
-        if (type == GD_NODE_BRICK) {
-                brickinfo = pending_entry;
-                snprintf (brick, sizeof (brick), "%s:%s", brickinfo->hostname,
-                          brickinfo->path);
-        } else if (type == GD_NODE_NFS) {
-                snprintf (brick, sizeof (brick), "%s", uuid_utoa (MY_UUID));
-        }
-        full_brick = gf_strdup (brick);
-        GF_ASSERT (full_brick);
-        ret = dict_set_dynstr (op_ctx, key, full_brick);
-
-        rsp_ctx.count = count;
-        rsp_ctx.dict = op_ctx;
-        dict_foreach (rsp_dict, _profile_volume_add_brick_rsp, &rsp_ctx);
-        dict_del (op_ctx, "count");
-        ret = dict_set_int32 (op_ctx, "count", count);
-        return ret;
-}
-
-//input-key: <replica-id>:<child-id>-*
-//output-key: <brick-id>-*
-int
-_heal_volume_add_shd_rsp (dict_t *this, char *key, data_t *value, void *data)
-{
-        char                            new_key[256] = {0,};
-        char                            int_str[16] = {0};
-        data_t                          *new_value = NULL;
-        char                            *rxl_end = NULL;
-        char                            *rxl_child_end = NULL;
-        glusterd_volinfo_t              *volinfo = NULL;
-        int                             rxl_id = 0;
-        int                             rxl_child_id = 0;
-        int                             brick_id = 0;
-        int                             int_len = 0;
-        int                             ret = 0;
-        glusterd_heal_rsp_conv_t        *rsp_ctx = NULL;
-        glusterd_brickinfo_t            *brickinfo = NULL;
-
-        rsp_ctx = data;
-        rxl_end = strchr (key, '-');
-        if (!rxl_end)
-                goto out;
-
-        int_len = strlen (key) - strlen (rxl_end);
-        strncpy (int_str, key, int_len);
-        int_str[int_len] = '\0';
-        ret = gf_string2int (int_str, &rxl_id);
-        if (ret)
-                goto out;
-
-        rxl_child_end = strchr (rxl_end + 1, '-');
-        if (!rxl_child_end)
-                goto out;
-
-        int_len = strlen (rxl_end) - strlen (rxl_child_end) - 1;
-        strncpy (int_str, rxl_end + 1, int_len);
-        int_str[int_len] = '\0';
-        ret = gf_string2int (int_str, &rxl_child_id);
-        if (ret)
-                goto out;
-
-        volinfo = rsp_ctx->volinfo;
-        brick_id = rxl_id * volinfo->replica_count + rxl_child_id;
-
-        if (!strcmp (rxl_child_end, "-status")) {
-                brickinfo = glusterd_get_brickinfo_by_position (volinfo,
-                                                                brick_id);
-                if (!brickinfo)
-                        goto out;
-                if (!glusterd_is_local_brick (rsp_ctx->this, volinfo,
-                                              brickinfo))
-                        goto out;
-        }
-        new_value = data_copy (value);
-        snprintf (new_key, sizeof (new_key), "%d%s", brick_id, rxl_child_end);
-        dict_set (rsp_ctx->dict, new_key, new_value);
+glusterd_profile_volume_brick_rsp(void *pending_entry, dict_t *rsp_dict,
+                                  dict_t *op_ctx, char **op_errstr,
+                                  gd_node_type type)
+{
+    int ret = 0;
+    glusterd_pr_brick_rsp_conv_t rsp_ctx = {0};
+    int32_t count = 0;
+    char brick[PATH_MAX + 1024] = "";
+    char key[64] = "";
+    int keylen;
+    char *full_brick = NULL;
+    glusterd_brickinfo_t *brickinfo = NULL;
+    xlator_t *this = NULL;
+    glusterd_conf_t *priv = NULL;
+
+    GF_ASSERT(rsp_dict);
+    GF_ASSERT(op_ctx);
+    GF_ASSERT(op_errstr);
+    GF_ASSERT(pending_entry);
+
+    this = THIS;
+    GF_ASSERT(this);
+    priv = this->private;
+    GF_ASSERT(priv);
+
+    ret = dict_get_int32n(op_ctx, "count", SLEN("count"), &count);
+    if (ret) {
+        count = 1;
+    } else {
+        count++;
+    }
+    if (type == GD_NODE_BRICK) {
+        brickinfo = pending_entry;
+        snprintf(brick, sizeof(brick), "%s:%s", brickinfo->hostname,
+                 brickinfo->path);
+    } else if (type == GD_NODE_NFS) {
+        snprintf(brick, sizeof(brick), "%s", uuid_utoa(MY_UUID));
+    }
+    full_brick = gf_strdup(brick);
+    GF_ASSERT(full_brick);
+    keylen = snprintf(key, sizeof(key), "%d-brick", count);
+    ret = dict_set_dynstrn(op_ctx, key, keylen, full_brick);
+
+    rsp_ctx.count = count;
+    rsp_ctx.dict = op_ctx;
+    dict_foreach(rsp_dict, _profile_volume_add_brick_rsp, &rsp_ctx);
+    ret = dict_set_int32n(op_ctx, "count", SLEN("count"), count);
+    return ret;
+}
+
+// input-key: <replica-id>:<child-id>-*
+// output-key: <brick-id>-*
+int
+_heal_volume_add_shd_rsp(dict_t *this, char *key, data_t *value, void *data)
+{
+    char new_key[256] = "";
+    char int_str[16] = "";
+    data_t *new_value = NULL;
+    char *rxl_end = NULL;
+    int rxl_end_len;
+    char *rxl_child_end = NULL;
+    glusterd_volinfo_t *volinfo = NULL;
+    int rxl_id = 0;
+    int rxl_child_id = 0;
+    int brick_id = 0;
+    int int_len = 0;
+    int ret = 0;
+    glusterd_heal_rsp_conv_t *rsp_ctx = NULL;
+    glusterd_brickinfo_t *brickinfo = NULL;
+
+    rsp_ctx = data;
+    rxl_end = strchr(key, '-');
+    if (!rxl_end)
+        goto out;
+
+    rxl_child_end = strchr(rxl_end + 1, '-');
+    if (!rxl_child_end)
+        goto out;
+
+    rxl_end_len = strlen(rxl_end);
+    int_len = strlen(key) - rxl_end_len;
+    (void)memcpy(int_str, key, int_len);
+    int_str[int_len] = '\0';
+
+    ret = gf_string2int(int_str, &rxl_id);
+    if (ret)
+        goto out;
+
+    int_len = rxl_end_len - strlen(rxl_child_end) - 1;
+    (void)memcpy(int_str, rxl_end + 1, int_len);
+    int_str[int_len] = '\0';
+
+    ret = gf_string2int(int_str, &rxl_child_id);
+    if (ret)
+        goto out;
+
+    volinfo = rsp_ctx->volinfo;
+    brick_id = rxl_id * volinfo->replica_count + rxl_child_id;
+
+    if (!strcmp(rxl_child_end, "-status")) {
+        brickinfo = glusterd_get_brickinfo_by_position(volinfo, brick_id);
+        if (!brickinfo)
+            goto out;
+        if (!glusterd_is_local_brick(rsp_ctx->this, volinfo, brickinfo))
+            goto out;
+    }
+    new_value = data_copy(value);
+    int_len = snprintf(new_key, sizeof(new_key), "%d%s", brick_id,
+                       rxl_child_end);
+    dict_setn(rsp_ctx->dict, new_key, int_len, new_value);
 
 out:
-        return 0;
+    return 0;
 }
 
 int
-_heal_volume_add_shd_rsp_of_statistics (dict_t *this, char *key, data_t
-                                             *value, void *data)
-{
-        char                            new_key[256] = {0,};
-        char                            int_str[16] = {0,};
-        char                            key_begin_string[128] = {0,};
-        data_t                          *new_value = NULL;
-        char                            *rxl_end = NULL;
-        char                            *rxl_child_end = NULL;
-        glusterd_volinfo_t              *volinfo = NULL;
-        char                            *key_begin_str = NULL;
-        int                             rxl_id = 0;
-        int                             rxl_child_id = 0;
-        int                             brick_id = 0;
-        int                             int_len = 0;
-        int                             ret = 0;
-        glusterd_heal_rsp_conv_t        *rsp_ctx = NULL;
-        glusterd_brickinfo_t            *brickinfo = NULL;
-
-        rsp_ctx = data;
-        key_begin_str = strchr (key, '-');
-        if (!key_begin_str)
-                goto out;
-
-        int_len = strlen (key) - strlen (key_begin_str);
-        strncpy (key_begin_string, key, int_len);
-        key_begin_string[int_len] = '\0';
-
-        rxl_end = strchr (key_begin_str + 1, '-');
-        if (!rxl_end)
-                goto out;
-
-        int_len = strlen (key_begin_str) - strlen (rxl_end) - 1;
-        strncpy (int_str, key_begin_str + 1, int_len);
-        int_str[int_len] = '\0';
-        ret = gf_string2int (int_str, &rxl_id);
-        if (ret)
-                goto out;
-
-
-        rxl_child_end = strchr (rxl_end + 1, '-');
-        if (!rxl_child_end)
-                goto out;
-
-        int_len = strlen (rxl_end) - strlen (rxl_child_end) - 1;
-        strncpy (int_str, rxl_end + 1, int_len);
-        int_str[int_len] = '\0';
-        ret = gf_string2int (int_str, &rxl_child_id);
-        if (ret)
-                goto out;
-
-        volinfo = rsp_ctx->volinfo;
-        brick_id = rxl_id * volinfo->replica_count + rxl_child_id;
-
-        brickinfo = glusterd_get_brickinfo_by_position (volinfo, brick_id);
-        if (!brickinfo)
-                goto out;
-        if (!glusterd_is_local_brick (rsp_ctx->this, volinfo, brickinfo))
-                goto out;
-
-        new_value = data_copy (value);
-        snprintf (new_key, sizeof (new_key), "%s-%d%s", key_begin_string,
-                  brick_id, rxl_child_end);
-        dict_set (rsp_ctx->dict, new_key, new_value);
+_heal_volume_add_shd_rsp_of_statistics(dict_t *this, char *key, data_t *value,
+                                       void *data)
+{
+    char new_key[256] = "";
+    char int_str[16] = "";
+    char key_begin_string[128] = "";
+    data_t *new_value = NULL;
+    char *rxl_end = NULL;
+    int rxl_end_len;
+    char *rxl_child_end = NULL;
+    glusterd_volinfo_t *volinfo = NULL;
+    char *key_begin_str = NULL;
+    int key_begin_strlen;
+    int rxl_id = 0;
+    int rxl_child_id = 0;
+    int brick_id = 0;
+    int int_len = 0;
+    int ret = 0;
+    glusterd_heal_rsp_conv_t *rsp_ctx = NULL;
+    glusterd_brickinfo_t *brickinfo = NULL;
+
+    rsp_ctx = data;
+    key_begin_str = strchr(key, '-');
+    if (!key_begin_str)
+        goto out;
+
+    rxl_end = strchr(key_begin_str + 1, '-');
+    if (!rxl_end)
+        goto out;
+
+    rxl_child_end = strchr(rxl_end + 1, '-');
+    if (!rxl_child_end)
+        goto out;
+
+    key_begin_strlen = strlen(key_begin_str);
+    int_len = strlen(key) - key_begin_strlen;
+
+    (void)memcpy(key_begin_string, key, int_len);
+    key_begin_string[int_len] = '\0';
+
+    rxl_end_len = strlen(rxl_end);
+    int_len = key_begin_strlen - rxl_end_len - 1;
+    (void)memcpy(int_str, key_begin_str + 1, int_len);
+    int_str[int_len] = '\0';
+    ret = gf_string2int(int_str, &rxl_id);
+    if (ret)
+        goto out;
+
+    int_len = rxl_end_len - strlen(rxl_child_end) - 1;
+    (void)memcpy(int_str, rxl_end + 1, int_len);
+    int_str[int_len] = '\0';
+    ret = gf_string2int(int_str, &rxl_child_id);
+    if (ret)
+        goto out;
+
+    volinfo = rsp_ctx->volinfo;
+    brick_id = rxl_id * volinfo->replica_count + rxl_child_id;
+
+    brickinfo = glusterd_get_brickinfo_by_position(volinfo, brick_id);
+    if (!brickinfo)
+        goto out;
+    if (!glusterd_is_local_brick(rsp_ctx->this, volinfo, brickinfo))
+        goto out;
+
+    new_value = data_copy(value);
+    int_len = snprintf(new_key, sizeof(new_key), "%s-%d%s", key_begin_string,
+                       brick_id, rxl_child_end);
+    dict_setn(rsp_ctx->dict, new_key, int_len, new_value);
 
 out:
-        return 0;
-
+    return 0;
 }
 
 int
-glusterd_heal_volume_brick_rsp (dict_t *req_dict, dict_t *rsp_dict,
-                                dict_t *op_ctx, char **op_errstr)
-{
-        int                             ret = 0;
-        glusterd_heal_rsp_conv_t        rsp_ctx = {0};
-        char                            *volname = NULL;
-        glusterd_volinfo_t              *volinfo = NULL;
-        int                             heal_op = -1;
-
-        GF_ASSERT (rsp_dict);
-        GF_ASSERT (op_ctx);
-        GF_ASSERT (op_errstr);
-
-        ret = dict_get_str (req_dict, "volname", &volname);
-        if (ret) {
-                gf_log ("", GF_LOG_ERROR, "Unable to get volume name");
-                goto out;
-        }
-
-        ret = dict_get_int32 (req_dict, "heal-op", &heal_op);
-        if (ret) {
-                gf_log ("", GF_LOG_ERROR, "Unable to get heal_op");
-                goto out;
-        }
-
-
-        ret  = glusterd_volinfo_find (volname, &volinfo);
-
-        if (ret)
-                goto out;
-
-        rsp_ctx.dict = op_ctx;
-        rsp_ctx.volinfo = volinfo;
-        rsp_ctx.this = THIS;
-        if (heal_op == GF_SHD_OP_STATISTICS)
-                dict_foreach (rsp_dict, _heal_volume_add_shd_rsp_of_statistics,
-                              &rsp_ctx);
-        else
-                dict_foreach (rsp_dict, _heal_volume_add_shd_rsp, &rsp_ctx);
-
+glusterd_heal_volume_brick_rsp(dict_t *req_dict, dict_t *rsp_dict,
+                               dict_t *op_ctx, char **op_errstr)
+{
+    int ret = 0;
+    glusterd_heal_rsp_conv_t rsp_ctx = {0};
+    char *volname = NULL;
+    glusterd_volinfo_t *volinfo = NULL;
+    int heal_op = -1;
+
+    GF_ASSERT(rsp_dict);
+    GF_ASSERT(op_ctx);
+    GF_ASSERT(op_errstr);
+
+    ret = dict_get_strn(req_dict, "volname", SLEN("volname"), &volname);
+    if (ret) {
+        gf_msg("glusterd", GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+               "Unable to get volume name");
+        goto out;
+    }
+
+    ret = dict_get_int32n(req_dict, "heal-op", SLEN("heal-op"), &heal_op);
+    if (ret) {
+        gf_msg("glusterd", GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+               "Unable to get heal_op");
+        goto out;
+    }
+
+    ret = glusterd_volinfo_find(volname, &volinfo);
+
+    if (ret)
+        goto out;
+
+    rsp_ctx.dict = op_ctx;
+    rsp_ctx.volinfo = volinfo;
+    rsp_ctx.this = THIS;
+    if (heal_op == GF_SHD_OP_STATISTICS)
+        dict_foreach(rsp_dict, _heal_volume_add_shd_rsp_of_statistics,
+                     &rsp_ctx);
+    else
+        dict_foreach(rsp_dict, _heal_volume_add_shd_rsp, &rsp_ctx);
 
 out:
-        return ret;
+    return ret;
 }
 
 int
-_status_volume_add_brick_rsp (dict_t *this, char *key, data_t *value,
-                              void *data)
+_status_volume_add_brick_rsp(dict_t *this, char *key, data_t *value, void *data)
 {
-        char                            new_key[256] = {0,};
-        data_t                          *new_value = 0;
-        glusterd_pr_brick_rsp_conv_t    *rsp_ctx = NULL;
+    char new_key[256] = "";
+    int keylen;
+    data_t *new_value = 0;
+    glusterd_pr_brick_rsp_conv_t *rsp_ctx = NULL;
 
-        rsp_ctx = data;
-        new_value = data_copy (value);
-        snprintf (new_key, sizeof (new_key), "brick%d.%s", rsp_ctx->count, key);
-        dict_set (rsp_ctx->dict, new_key, new_value);
+    rsp_ctx = data;
+    new_value = data_copy(value);
+    keylen = snprintf(new_key, sizeof(new_key), "brick%d.%s", rsp_ctx->count,
+                      key);
+    dict_setn(rsp_ctx->dict, new_key, keylen, new_value);
 
-        return 0;
+    return 0;
 }
 
 int
-glusterd_status_volume_brick_rsp (dict_t *rsp_dict, dict_t *op_ctx,
-                                  char **op_errstr)
-{
-        int                             ret = 0;
-        glusterd_pr_brick_rsp_conv_t    rsp_ctx = {0};
-        int32_t                         count = 0;
-        int                             index = 0;
+glusterd_status_volume_brick_rsp(dict_t *rsp_dict, dict_t *op_ctx,
+                                 char **op_errstr)
+{
+    int ret = 0;
+    glusterd_pr_brick_rsp_conv_t rsp_ctx = {0};
+    int32_t count = 0;
+    int index = 0;
+
+    GF_ASSERT(rsp_dict);
+    GF_ASSERT(op_ctx);
+    GF_ASSERT(op_errstr);
+
+    ret = dict_get_int32n(op_ctx, "count", SLEN("count"), &count);
+    if (ret) {
+        count = 0;
+    } else {
+        count++;
+    }
+    ret = dict_get_int32n(rsp_dict, "index", SLEN("index"), &index);
+    if (ret) {
+        gf_msg(THIS->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+               "Couldn't get node index");
+        goto out;
+    }
+    dict_deln(rsp_dict, "index", SLEN("index"));
+
+    rsp_ctx.count = index;
+    rsp_ctx.dict = op_ctx;
+    dict_foreach(rsp_dict, _status_volume_add_brick_rsp, &rsp_ctx);
+    ret = dict_set_int32n(op_ctx, "count", SLEN("count"), count);
 
-        GF_ASSERT (rsp_dict);
-        GF_ASSERT (op_ctx);
-        GF_ASSERT (op_errstr);
+out:
+    return ret;
+}
 
-        ret = dict_get_int32 (op_ctx, "count", &count);
+int
+glusterd_status_volume_client_list(dict_t *rsp_dict, dict_t *op_ctx,
+                                   char **op_errstr)
+{
+    int ret = 0;
+    char *process = 0;
+    int32_t count = 0;
+    int32_t fuse_count = 0;
+    int32_t gfapi_count = 0;
+    int32_t rebalance_count = 0;
+    int32_t glustershd_count = 0;
+    int32_t quotad_count = 0;
+    int32_t snapd_count = 0;
+    int32_t client_count = 0;
+    int i = 0;
+    char key[64] = "";
+
+    GF_ASSERT(rsp_dict);
+    GF_ASSERT(op_ctx);
+    GF_ASSERT(op_errstr);
+
+    ret = dict_get_int32n(rsp_dict, "clientcount", SLEN("clientcount"),
+                          &client_count);
+    if (ret) {
+        gf_msg(THIS->name, GF_LOG_INFO, 0, GD_MSG_DICT_GET_FAILED,
+               "Couldn't get node index");
+    }
+    ret = dict_set_int32n(op_ctx, "client-count", SLEN("client-count"),
+                          client_count);
+    if (ret) {
+        gf_msg(THIS->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+               "Couldn't get node index");
+        goto out;
+    }
+    for (i = 0; i < client_count; i++) {
+        count = 0;
+        ret = snprintf(key, sizeof(key), "client%d.name", i);
+        ret = dict_get_strn(rsp_dict, key, ret, &process);
         if (ret) {
-                count = 0;
-        } else {
-                count++;
+            gf_msg(THIS->name, GF_LOG_INFO, 0, GD_MSG_DICT_GET_FAILED,
+                   "Couldn't get client name");
+            goto out;
         }
-        ret = dict_get_int32 (rsp_dict, "index", &index);
+        ret = dict_add_dynstr_with_alloc(op_ctx, key, process);
         if (ret) {
-                gf_log (THIS->name, GF_LOG_ERROR, "Couldn't get node index");
-                goto out;
+            gf_msg(THIS->name, GF_LOG_INFO, 0, GD_MSG_DICT_SET_FAILED,
+                   "Couldn't set client name");
+        }
+        if (!strncmp(process, "fuse", 4)) {
+            ret = dict_get_int32n(op_ctx, "fuse-count", SLEN("fuse-count"),
+                                  &count);
+            if (ret) {
+                gf_msg(THIS->name, GF_LOG_INFO, 0, GD_MSG_DICT_GET_FAILED,
+                       "Couldn't get fuse-count");
+            }
+            fuse_count++;
+            continue;
+        } else if (!strncmp(process, "gfapi", 5)) {
+            ret = dict_get_int32n(op_ctx, "gfapi-count", SLEN("gfapi-count"),
+                                  &count);
+            if (ret) {
+                gf_msg(THIS->name, GF_LOG_INFO, 0, GD_MSG_DICT_GET_FAILED,
+                       "Couldn't get gfapi-count");
+            }
+            gfapi_count++;
+            continue;
+
+        } else if (!strcmp(process, "rebalance")) {
+            ret = dict_get_int32n(op_ctx, "rebalance-count",
+                                  SLEN("rebalance-count"), &count);
+            if (ret) {
+                gf_msg(THIS->name, GF_LOG_INFO, 0, GD_MSG_DICT_GET_FAILED,
+                       "Couldn't get rebalance-count");
+            }
+            rebalance_count++;
+            continue;
+        } else if (!strcmp(process, "glustershd")) {
+            ret = dict_get_int32n(op_ctx, "glustershd-count",
+                                  SLEN("glustershd-count"), &count);
+            if (ret) {
+                gf_msg(THIS->name, GF_LOG_INFO, 0, GD_MSG_DICT_GET_FAILED,
+                       "Couldn't get glustershd-count");
+            }
+            glustershd_count++;
+            continue;
+        } else if (!strcmp(process, "quotad")) {
+            ret = dict_get_int32n(op_ctx, "quotad-count", SLEN("quotad-count"),
+                                  &count);
+            if (ret) {
+                gf_msg(THIS->name, GF_LOG_INFO, 0, GD_MSG_DICT_GET_FAILED,
+                       "Couldn't get quotad-count");
+            }
+            quotad_count++;
+            continue;
+        } else if (!strcmp(process, "snapd")) {
+            ret = dict_get_int32n(op_ctx, "snapd-count", SLEN("snapd-count"),
+                                  &count);
+            if (ret) {
+                gf_msg(THIS->name, GF_LOG_INFO, 0, GD_MSG_DICT_GET_FAILED,
+                       "Couldn't get snapd-count");
+            }
+            snapd_count++;
+        }
+    }
+
+    if (fuse_count) {
+        ret = dict_set_int32n(op_ctx, "fuse-count", SLEN("fuse-count"),
+                              fuse_count);
+        if (ret) {
+            gf_msg(THIS->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+                   "Couldn't set fuse-count");
+            goto out;
         }
-        dict_del (rsp_dict, "index");
-
-        rsp_ctx.count = index;
-        rsp_ctx.dict = op_ctx;
-        dict_foreach (rsp_dict, _status_volume_add_brick_rsp, &rsp_ctx);
-        ret = dict_set_int32 (op_ctx, "count", count);
-
-out:
-        return ret;
-}
-
-int
-glusterd_defrag_volume_node_rsp (dict_t *req_dict, dict_t *rsp_dict,
-                                 dict_t *op_ctx)
-{
-        int                             ret = 0;
-        char                            *volname = NULL;
-        glusterd_volinfo_t              *volinfo = NULL;
-        char                            key[256] = {0,};
-        int32_t                         i = 0;
-        char                            buf[1024] = {0,};
-        char                            *node_str = NULL;
-        glusterd_conf_t                 *priv = NULL;
-
-        priv = THIS->private;
-        GF_ASSERT (req_dict);
-
-        ret = dict_get_str (req_dict, "volname", &volname);
+    }
+    if (gfapi_count) {
+        ret = dict_set_int32n(op_ctx, "gfapi-count", SLEN("gfapi-count"),
+                              gfapi_count);
         if (ret) {
-                gf_log ("", GF_LOG_ERROR, "Unable to get volume name");
-                goto out;
+            gf_msg(THIS->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+                   "Couldn't set gfapi-count");
+            goto out;
         }
-
-        ret  = glusterd_volinfo_find (volname, &volinfo);
-
-        if (ret)
-                goto out;
-
-        if (rsp_dict) {
-                ret = glusterd_defrag_volume_status_update (volinfo,
-                                                            rsp_dict);
+    }
+    if (rebalance_count) {
+        ret = dict_set_int32n(op_ctx, "rebalance-count",
+                              SLEN("rebalance-count"), rebalance_count);
+        if (ret) {
+            gf_msg(THIS->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+                   "Couldn't set rebalance-count");
+            goto out;
         }
-
-        if (!op_ctx) {
-                dict_copy (rsp_dict, op_ctx);
-                goto out;
+    }
+    if (glustershd_count) {
+        ret = dict_set_int32n(op_ctx, "glustershd-count",
+                              SLEN("glustershd-count"), glustershd_count);
+        if (ret) {
+            gf_msg(THIS->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+                   "Couldn't set glustershd-count");
+            goto out;
         }
+    }
+    if (quotad_count) {
+        ret = dict_set_int32n(op_ctx, "quotad-count", SLEN("quotad-count"),
+                              quotad_count);
+        if (ret) {
+            gf_msg(THIS->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+                   "Couldn't set quotad-count");
+            goto out;
+        }
+    }
+    if (snapd_count) {
+        ret = dict_set_int32n(op_ctx, "snapd-count", SLEN("snapd-count"),
+                              snapd_count);
+        if (ret) {
+            gf_msg(THIS->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+                   "Couldn't set snapd-count");
+            goto out;
+        }
+    }
 
-        ret = dict_get_int32 (op_ctx, "count", &i);
-        i++;
-
-        ret = dict_set_int32 (op_ctx, "count", i);
-        if (ret)
-                gf_log (THIS->name, GF_LOG_ERROR, "Failed to set count");
-
-        snprintf (buf, 1024, "%s", uuid_utoa (MY_UUID));
-        node_str = gf_strdup (buf);
-
-        snprintf (key, 256, "node-uuid-%d",i);
-        ret = dict_set_dynstr (op_ctx, key, node_str);
-        if (ret)
-                gf_log (THIS->name, GF_LOG_ERROR,
-                        "failed to set node-uuid");
-
-        memset (key, 0 , 256);
-        snprintf (key, 256, "files-%d", i);
-        ret = dict_set_uint64 (op_ctx, key, volinfo->rebal.rebalance_files);
-        if (ret)
-                gf_log (THIS->name, GF_LOG_ERROR,
-                        "failed to set file count");
-
-        memset (key, 0 , 256);
-        snprintf (key, 256, "size-%d", i);
-        ret = dict_set_uint64 (op_ctx, key, volinfo->rebal.rebalance_data);
-        if (ret)
-                gf_log (THIS->name, GF_LOG_ERROR,
-                        "failed to set size of xfer");
-
-        memset (key, 0 , 256);
-        snprintf (key, 256, "lookups-%d", i);
-        ret = dict_set_uint64 (op_ctx, key, volinfo->rebal.lookedup_files);
-        if (ret)
-                gf_log (THIS->name, GF_LOG_ERROR,
-                        "failed to set lookedup file count");
-
-        memset (key, 0 , 256);
-        snprintf (key, 256, "status-%d", i);
-        ret = dict_set_int32 (op_ctx, key, volinfo->rebal.defrag_status);
-        if (ret)
-                gf_log (THIS->name, GF_LOG_ERROR,
-                        "failed to set status");
-
-        memset (key, 0 , 256);
-        snprintf (key, 256, "failures-%d", i);
-        ret = dict_set_uint64 (op_ctx, key, volinfo->rebal.rebalance_failures);
-        if (ret)
-                gf_log (THIS->name, GF_LOG_ERROR,
-                        "failed to set failure count");
+out:
+    return ret;
+}
 
-        memset (key, 0 , 256);
-        snprintf (key, 256, "skipped-%d", i);
-        ret = dict_set_uint64 (op_ctx, key, volinfo->rebal.skipped_files);
-        if (ret)
-                gf_log (THIS->name, GF_LOG_ERROR,
-                        "failed to set skipped count");
+int
+glusterd_rebalance_rsp(dict_t *op_ctx, glusterd_rebalance_t *index, int32_t i)
+{
+    int ret = 0;
+    char key[64] = "";
+    int keylen;
+
+    snprintf(key, sizeof(key), "files-%d", i);
+    ret = dict_set_uint64(op_ctx, key, index->rebalance_files);
+    if (ret)
+        gf_msg(THIS->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+               "failed to set file count");
+
+    snprintf(key, sizeof(key), "size-%d", i);
+    ret = dict_set_uint64(op_ctx, key, index->rebalance_data);
+    if (ret)
+        gf_msg(THIS->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+               "failed to set size of xfer");
+
+    snprintf(key, sizeof(key), "lookups-%d", i);
+    ret = dict_set_uint64(op_ctx, key, index->lookedup_files);
+    if (ret)
+        gf_msg(THIS->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+               "failed to set lookedup file count");
+
+    keylen = snprintf(key, sizeof(key), "status-%d", i);
+    ret = dict_set_int32n(op_ctx, key, keylen, index->defrag_status);
+    if (ret)
+        gf_msg(THIS->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+               "failed to set status");
+
+    snprintf(key, sizeof(key), "failures-%d", i);
+    ret = dict_set_uint64(op_ctx, key, index->rebalance_failures);
+    if (ret)
+        gf_msg(THIS->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+               "failed to set failure count");
+
+    snprintf(key, sizeof(key), "skipped-%d", i);
+    ret = dict_set_uint64(op_ctx, key, index->skipped_files);
+    if (ret)
+        gf_msg(THIS->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+               "failed to set skipped count");
+
+    snprintf(key, sizeof(key), "run-time-%d", i);
+    ret = dict_set_double(op_ctx, key, index->rebalance_time);
+    if (ret)
+        gf_msg(THIS->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+               "failed to set run-time");
+
+    return ret;
+}
 
-        memset (key, 0, 256);
-        snprintf (key, 256, "run-time-%d", i);
-        ret = dict_set_double (op_ctx, key, volinfo->rebal.rebalance_time);
-        if (ret)
-                gf_log (THIS->name, GF_LOG_ERROR,
-                        "failed to set run-time");
+int
+glusterd_defrag_volume_node_rsp(dict_t *req_dict, dict_t *rsp_dict,
+                                dict_t *op_ctx)
+{
+    int ret = 0;
+    char *volname = NULL;
+    glusterd_volinfo_t *volinfo = NULL;
+    char key[64] = "";
+    int keylen;
+    int32_t i = 0;
+    char buf[64] = "";
+    char *node_str = NULL;
+    int32_t cmd = 0;
+
+    GF_ASSERT(req_dict);
+
+    ret = dict_get_strn(req_dict, "volname", SLEN("volname"), &volname);
+    if (ret) {
+        gf_msg("glusterd", GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+               "Unable to get volume name");
+        goto out;
+    }
+
+    ret = glusterd_volinfo_find(volname, &volinfo);
+
+    ret = dict_get_int32n(req_dict, "rebalance-command",
+                          SLEN("rebalance-command"), &cmd);
+    if (ret) {
+        gf_msg(THIS->name, GF_LOG_ERROR, errno, GD_MSG_DICT_GET_FAILED,
+               "Unable to get the cmd");
+        goto out;
+    }
+
+    if (rsp_dict) {
+        ret = glusterd_defrag_volume_status_update(volinfo, rsp_dict, cmd);
+    }
+
+    if (!op_ctx) {
+        dict_copy(rsp_dict, op_ctx);
+        goto out;
+    }
+
+    ret = dict_get_int32n(op_ctx, "count", SLEN("count"), &i);
+    i++;
+
+    ret = dict_set_int32n(op_ctx, "count", SLEN("count"), i);
+    if (ret)
+        gf_msg(THIS->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+               "Failed to set count");
+
+    snprintf(buf, sizeof(buf), "%s", uuid_utoa(MY_UUID));
+    node_str = gf_strdup(buf);
+
+    keylen = snprintf(key, sizeof(key), "node-uuid-%d", i);
+    ret = dict_set_dynstrn(op_ctx, key, keylen, node_str);
+    if (ret)
+        gf_msg(THIS->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+               "failed to set node-uuid");
+
+    glusterd_rebalance_rsp(op_ctx, &volinfo->rebal, i);
+
+    snprintf(key, sizeof(key), "time-left-%d", i);
+    ret = dict_set_uint64(op_ctx, key, volinfo->rebal.time_left);
+    if (ret)
+        gf_msg(THIS->name, GF_LOG_ERROR, errno, GD_MSG_DICT_SET_FAILED,
+               "failed to set time left");
 
 out:
-        return ret;
+    return ret;
 }
 int32_t
-glusterd_handle_node_rsp (dict_t *req_dict, void *pending_entry,
-                          glusterd_op_t op, dict_t *rsp_dict, dict_t *op_ctx,
-                          char **op_errstr, gd_node_type type)
+glusterd_handle_node_rsp(dict_t *req_dict, void *pending_entry,
+                         glusterd_op_t op, dict_t *rsp_dict, dict_t *op_ctx,
+                         char **op_errstr, gd_node_type type)
 {
-        int                     ret = 0;
+    int ret = 0;
+    int32_t cmd = GF_OP_CMD_NONE;
 
-        GF_ASSERT (op_errstr);
+    GF_ASSERT(op_errstr);
 
-        switch (op) {
+    switch (op) {
         case GD_OP_PROFILE_VOLUME:
-                ret = glusterd_profile_volume_brick_rsp (pending_entry,
-                                                         rsp_dict, op_ctx,
-                                                         op_errstr, type);
-                break;
+            ret = glusterd_profile_volume_brick_rsp(pending_entry, rsp_dict,
+                                                    op_ctx, op_errstr, type);
+            break;
         case GD_OP_STATUS_VOLUME:
-                ret = glusterd_status_volume_brick_rsp (rsp_dict, op_ctx,
-                                                        op_errstr);
-                break;
-
+            ret = dict_get_int32n(req_dict, "cmd", SLEN("cmd"), &cmd);
+            if (!ret && (cmd & GF_CLI_STATUS_CLIENT_LIST)) {
+                ret = glusterd_status_volume_client_list(rsp_dict, op_ctx,
+                                                         op_errstr);
+            } else
+                ret = glusterd_status_volume_brick_rsp(rsp_dict, op_ctx,
+                                                       op_errstr);
+            break;
         case GD_OP_DEFRAG_BRICK_VOLUME:
-                glusterd_defrag_volume_node_rsp (req_dict,
-                                                 rsp_dict, op_ctx);
-                break;
+            glusterd_defrag_volume_node_rsp(req_dict, rsp_dict, op_ctx);
+            break;
 
         case GD_OP_HEAL_VOLUME:
-                ret = glusterd_heal_volume_brick_rsp (req_dict, rsp_dict,
-                                                      op_ctx, op_errstr);
-                break;
+            ret = glusterd_heal_volume_brick_rsp(req_dict, rsp_dict, op_ctx,
+                                                 op_errstr);
+            break;
+        case GD_OP_SCRUB_STATUS:
+            ret = glusterd_bitrot_volume_node_rsp(op_ctx, rsp_dict);
+
+            break;
         default:
-                break;
-        }
+            break;
+    }
 
-        gf_log ("", GF_LOG_DEBUG, "Returning %d", ret);
-        return ret;
+    gf_msg_debug("glusterd", 0, "Returning %d", ret);
+    return ret;
 }
 
 int32_t
-glusterd_set_originator_uuid (dict_t *dict)
+glusterd_set_originator_uuid(dict_t *dict)
 {
-        int          ret              = -1;
-        uuid_t      *originator_uuid  = NULL;
-
-        GF_ASSERT (dict);
+    int ret = -1;
+    uuid_t *originator_uuid = NULL;
 
-        originator_uuid = GF_CALLOC (1, sizeof(uuid_t),
-                                     gf_common_mt_uuid_t);
-        if (!originator_uuid) {
-                ret = -1;
-                goto out;
-        }
+    GF_ASSERT(dict);
 
-        gf_uuid_copy (*originator_uuid, MY_UUID);
-        ret = dict_set_bin (dict, "originator_uuid",
-                            originator_uuid, sizeof (uuid_t));
-        if (ret) {
-                gf_log ("", GF_LOG_ERROR,
-                        "Failed to set originator_uuid.");
-                goto out;
-        }
+    originator_uuid = GF_MALLOC(sizeof(uuid_t), gf_common_mt_uuid_t);
+    if (!originator_uuid) {
+        ret = -1;
+        goto out;
+    }
+
+    gf_uuid_copy(*originator_uuid, MY_UUID);
+    ret = dict_set_bin(dict, "originator_uuid", originator_uuid,
+                       sizeof(uuid_t));
+    if (ret) {
+        gf_msg("glusterd", GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+               "Failed to set originator_uuid.");
+        goto out;
+    }
 
 out:
-        if (ret && originator_uuid)
-                GF_FREE (originator_uuid);
+    if (ret && originator_uuid)
+        GF_FREE(originator_uuid);
 
-        return ret;
+    return ret;
 }
 
 /* Should be used only when an operation is in progress, as that is the only
  * time a lock_owner is set
  */
 gf_boolean_t
-is_origin_glusterd (dict_t *dict)
-{
-        gf_boolean_t  ret              = _gf_false;
-        uuid_t        lock_owner       = {0,};
-        uuid_t        *originator_uuid = NULL;
-
-        GF_ASSERT (dict);
-
-        ret = dict_get_bin (dict, "originator_uuid",
-                            (void **) &originator_uuid);
+is_origin_glusterd(dict_t *dict)
+{
+    gf_boolean_t ret = _gf_false;
+    uuid_t lock_owner = {
+        0,
+    };
+    uuid_t *originator_uuid = NULL;
+
+    GF_ASSERT(dict);
+
+    ret = dict_get_bin(dict, "originator_uuid", (void **)&originator_uuid);
+    if (ret) {
+        /* If not originator_uuid has been set, then the command
+         * has been originated from a glusterd running on older version
+         * Hence fetching the lock owner */
+        ret = glusterd_get_lock_owner(&lock_owner);
         if (ret) {
-                /* If not originator_uuid has been set, then the command
-                 * has been originated from a glusterd running on older version
-                 * Hence fetching the lock owner */
-                ret = glusterd_get_lock_owner (&lock_owner);
-                if (ret) {
-                        ret = _gf_false;
-                        goto out;
-                }
-                ret = !gf_uuid_compare (MY_UUID, lock_owner);
-        } else
-                ret = !gf_uuid_compare (MY_UUID, *originator_uuid);
+            ret = _gf_false;
+            goto out;
+        }
+        ret = !gf_uuid_compare(MY_UUID, lock_owner);
+    } else
+        ret = !gf_uuid_compare(MY_UUID, *originator_uuid);
 
 out:
-        return ret;
+    return ret;
 }
 
 int
-glusterd_generate_and_set_task_id (dict_t *dict, char *key)
+glusterd_generate_and_set_task_id(dict_t *dict, char *key, const int keylen)
 {
-        int             ret = -1;
-        uuid_t          task_id = {0,};
-        char            *uuid_str = NULL;
-        xlator_t        *this = NULL;
+    int ret = -1;
+    uuid_t task_id = {
+        0,
+    };
+    char *uuid_str = NULL;
+    xlator_t *this = NULL;
 
-        GF_ASSERT (dict);
+    GF_ASSERT(dict);
 
-        this = THIS;
-        GF_ASSERT (this);
+    this = THIS;
+    GF_ASSERT(this);
 
-        gf_uuid_generate (task_id);
-        uuid_str = gf_strdup (uuid_utoa (task_id));
-        if (!uuid_str) {
-                ret = -1;
-                goto out;
-        }
-
-        ret = dict_set_dynstr (dict, key, uuid_str);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "Failed to set %s in dict",
-                        key);
-                goto out;
-        }
-        gf_log (this->name, GF_LOG_INFO, "Generated task-id %s for key %s",
-                uuid_str, key);
+    gf_uuid_generate(task_id);
+    uuid_str = gf_strdup(uuid_utoa(task_id));
+    if (!uuid_str) {
+        ret = -1;
+        goto out;
+    }
+
+    ret = dict_set_dynstrn(dict, key, keylen, uuid_str);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+               "Failed to set %s in dict", key);
+        goto out;
+    }
+    gf_msg(this->name, GF_LOG_INFO, 0, GD_MSG_TASK_ID_INFO,
+           "Generated task-id %s for key %s", uuid_str, key);
 
 out:
-        if (ret)
-                GF_FREE (uuid_str);
-        return ret;
+    if (ret)
+        GF_FREE(uuid_str);
+    return ret;
 }
 
 int
-glusterd_copy_uuid_to_dict (uuid_t uuid, dict_t *dict, char *key)
+glusterd_copy_uuid_to_dict(uuid_t uuid, dict_t *dict, char *key,
+                           const int keylen)
 {
-        int             ret = -1;
-        char            tmp_str[40] = {0,};
-        char            *task_id_str = NULL;
+    int ret = -1;
+    char tmp_str[40] = "";
+    char *task_id_str = NULL;
 
-        GF_ASSERT (dict);
-        GF_ASSERT (key);
+    GF_ASSERT(dict);
+    GF_ASSERT(key);
 
-        gf_uuid_unparse (uuid, tmp_str);
-        task_id_str = gf_strdup (tmp_str);
-        if (!task_id_str)
-                return -1;
+    gf_uuid_unparse(uuid, tmp_str);
+    task_id_str = gf_strdup(tmp_str);
+    if (!task_id_str)
+        return -1;
 
-        ret = dict_set_dynstr (dict, key, task_id_str);
-        if (ret) {
-                GF_FREE (task_id_str);
-                gf_log (THIS->name, GF_LOG_ERROR,
-                        "Error setting uuid in dict with key %s", key);
-        }
+    ret = dict_set_dynstrn(dict, key, keylen, task_id_str);
+    if (ret) {
+        GF_FREE(task_id_str);
+        gf_msg(THIS->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+               "Error setting uuid in dict with key %s", key);
+    }
 
-        return 0;
+    return 0;
 }
 
-int
-_update_volume_op_versions (dict_t *this, char *key, data_t *value, void *data)
+static int
+_update_volume_op_versions(dict_t *this, char *key, data_t *value, void *data)
 {
-        int                op_version = 0;
-        glusterd_volinfo_t *ctx       = NULL;
-        gf_boolean_t       enabled    = _gf_true;
-        int                ret        = -1;
+    int op_version = 0;
+    glusterd_volinfo_t *ctx = NULL;
+    gf_boolean_t enabled = _gf_true;
+    int ret = -1;
+    struct volopt_map_entry *vmep = NULL;
 
-        GF_ASSERT (data);
-        ctx = data;
+    GF_ASSERT(data);
+    ctx = data;
 
-        op_version = glusterd_get_op_version_for_key (key);
+    vmep = gd_get_vmep(key);
+    op_version = glusterd_get_op_version_from_vmep(vmep);
 
-        if (gd_is_xlator_option (key) || gd_is_boolean_option (key)) {
-                ret = gf_string2boolean (value->data, &enabled);
-                if (ret)
-                        return 0;
+    if (gd_is_xlator_option(vmep) || gd_is_boolean_option(vmep)) {
+        ret = gf_string2boolean(value->data, &enabled);
+        if (ret)
+            return 0;
 
-                if (!enabled)
-                        return 0;
-        }
+        if (!enabled)
+            return 0;
+    }
 
-        if (op_version > ctx->op_version)
-                ctx->op_version = op_version;
+    if (op_version > ctx->op_version)
+        ctx->op_version = op_version;
 
-        if (gd_is_client_option (key) &&
-            (op_version > ctx->client_op_version))
-                ctx->client_op_version = op_version;
+    if (gd_is_client_option(vmep) && (op_version > ctx->client_op_version))
+        ctx->client_op_version = op_version;
 
-        return 0;
+    return 0;
 }
 
 void
-gd_update_volume_op_versions (glusterd_volinfo_t *volinfo)
+gd_update_volume_op_versions(glusterd_volinfo_t *volinfo)
 {
-        glusterd_conf_t *conf = NULL;
-        gf_boolean_t    ob_enabled = _gf_false;
+    glusterd_conf_t *conf = NULL;
+    gf_boolean_t ob_enabled = _gf_false;
 
-        GF_ASSERT (volinfo);
+    GF_ASSERT(volinfo);
 
-        conf = THIS->private;
-        GF_ASSERT (conf);
+    conf = THIS->private;
+    GF_ASSERT(conf);
 
-        /* Reset op-versions to minimum */
-        volinfo->op_version = 1;
-        volinfo->client_op_version = 1;
+    /* Reset op-versions to minimum */
+    volinfo->op_version = 1;
+    volinfo->client_op_version = 1;
 
-        dict_foreach (volinfo->dict, _update_volume_op_versions, volinfo);
+    dict_foreach(volinfo->dict, _update_volume_op_versions, volinfo);
 
-        /* Special case for open-behind
-         * If cluster op-version >= 2 and open-behind hasn't been explicitly
-         * disabled, volume op-versions must be updated to account for it
-         */
+    /* Special case for open-behind
+     * If cluster op-version >= 2 and open-behind hasn't been explicitly
+     * disabled, volume op-versions must be updated to account for it
+     */
 
-        /* TODO: Remove once we have a general way to update automatically
-         * enabled features
-         */
-        if (conf->op_version >= 2) {
-                ob_enabled = dict_get_str_boolean (volinfo->dict,
-                                                   "performance.open-behind",
-                                                   _gf_true);
-                if (ob_enabled) {
-
-                        if (volinfo->op_version < 2)
-                                volinfo->op_version = 2;
-                        if (volinfo->client_op_version < 2)
-                                volinfo->client_op_version = 2;
-                }
+    /* TODO: Remove once we have a general way to update automatically
+     * enabled features
+     */
+    if (conf->op_version >= 2) {
+        ob_enabled = dict_get_str_boolean(volinfo->dict,
+                                          "performance.open-behind", _gf_true);
+        if (ob_enabled) {
+            if (volinfo->op_version < 2)
+                volinfo->op_version = 2;
+            if (volinfo->client_op_version < 2)
+                volinfo->client_op_version = 2;
         }
+    }
 
-        if (volinfo->type == GF_CLUSTER_TYPE_DISPERSE) {
-                if (volinfo->op_version < GD_OP_VERSION_3_6_0)
-                        volinfo->op_version = GD_OP_VERSION_3_6_0;
-                if (volinfo->client_op_version < GD_OP_VERSION_3_6_0)
-                        volinfo->client_op_version = GD_OP_VERSION_3_6_0;
-        }
+    if (volinfo->type == GF_CLUSTER_TYPE_DISPERSE) {
+        if (volinfo->op_version < GD_OP_VERSION_3_6_0)
+            volinfo->op_version = GD_OP_VERSION_3_6_0;
+        if (volinfo->client_op_version < GD_OP_VERSION_3_6_0)
+            volinfo->client_op_version = GD_OP_VERSION_3_6_0;
+    }
 
-        return;
+    return;
 }
 
 int
-op_version_check (xlator_t *this, int min_op_version, char *msg, int msglen)
-{
-       int              ret  = 0;
-       glusterd_conf_t *priv = NULL;
-
-       GF_ASSERT (this);
-       GF_ASSERT (msg);
-
-       priv = this->private;
-       if (priv->op_version < min_op_version) {
-                snprintf (msg, msglen, "One or more nodes do not support "
-                          "the required op-version. Cluster op-version must "
-                          "atleast be %d.", min_op_version);
-                gf_log (this->name, GF_LOG_ERROR, "%s", msg);
-                ret = -1;
-      }
-      return ret;
+op_version_check(xlator_t *this, int min_op_version, char *msg, int msglen)
+{
+    int ret = 0;
+    glusterd_conf_t *priv = NULL;
+
+    GF_ASSERT(this);
+    GF_ASSERT(msg);
+
+    priv = this->private;
+    if (priv->op_version < min_op_version) {
+        snprintf(msg, msglen,
+                 "One or more nodes do not support "
+                 "the required op-version. Cluster op-version must "
+                 "at least be %d.",
+                 min_op_version);
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_UNSUPPORTED_VERSION, "%s",
+               msg);
+        ret = -1;
+    }
+    return ret;
 }
 
-
 /* A task is committed/completed once the task-id for it is cleared */
 gf_boolean_t
-gd_is_remove_brick_committed (glusterd_volinfo_t *volinfo)
+gd_is_remove_brick_committed(glusterd_volinfo_t *volinfo)
 {
-        GF_ASSERT (volinfo);
+    GF_ASSERT(volinfo);
 
-        if ((GD_OP_REMOVE_BRICK == volinfo->rebal.op) &&
-            !gf_uuid_is_null (volinfo->rebal.rebalance_id))
-                        return _gf_false;
+    if ((GD_OP_REMOVE_BRICK == volinfo->rebal.op) &&
+        !gf_uuid_is_null(volinfo->rebal.rebalance_id))
+        return _gf_false;
 
-        return _gf_true;
+    return _gf_true;
 }
 
 gf_boolean_t
-glusterd_is_status_tasks_op (glusterd_op_t op, dict_t *dict)
+glusterd_is_status_tasks_op(glusterd_op_t op, dict_t *dict)
 {
-        int           ret             = -1;
-        uint32_t      cmd             = GF_CLI_STATUS_NONE;
-        gf_boolean_t  is_status_tasks = _gf_false;
+    int ret = -1;
+    uint32_t cmd = GF_CLI_STATUS_NONE;
+    gf_boolean_t is_status_tasks = _gf_false;
 
-        if (op != GD_OP_STATUS_VOLUME)
-                goto out;
+    if (op != GD_OP_STATUS_VOLUME)
+        goto out;
 
-        ret = dict_get_uint32 (dict, "cmd", &cmd);
-        if (ret) {
-                gf_log (THIS->name, GF_LOG_ERROR, "Failed to get opcode");
-                goto out;
-        }
+    ret = dict_get_uint32(dict, "cmd", &cmd);
+    if (ret) {
+        gf_msg(THIS->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+               "Failed to get opcode");
+        goto out;
+    }
 
-        if (cmd & GF_CLI_STATUS_TASKS)
-                is_status_tasks = _gf_true;
+    if (cmd & GF_CLI_STATUS_TASKS)
+        is_status_tasks = _gf_true;
 
 out:
-        return is_status_tasks;
+    return is_status_tasks;
 }
 
 /* Tells if rebalance needs to be started for the given volume on the peer
@@ -8736,361 +12643,347 @@ out:
  * peer
  */
 gf_boolean_t
-gd_should_i_start_rebalance  (glusterd_volinfo_t *volinfo) {
-        gf_boolean_t         retval     = _gf_false;
-        int                  ret        = -1;
-        glusterd_brickinfo_t *brick     = NULL;
-        int                  count      = 0;
-        int                  i          = 0;
-        char                 key[1023]  = {0,};
-        char                 *brickname = NULL;
-
-
-        switch (volinfo->rebal.op) {
+gd_should_i_start_rebalance(glusterd_volinfo_t *volinfo)
+{
+    gf_boolean_t retval = _gf_false;
+    int ret = -1;
+    glusterd_brickinfo_t *brick = NULL;
+    int count = 0;
+    int i = 0;
+    char key[64] = "";
+    int keylen;
+    char *brickname = NULL;
+
+    switch (volinfo->rebal.op) {
         case GD_OP_REBALANCE:
-                cds_list_for_each_entry (brick, &volinfo->bricks, brick_list) {
-                        if (gf_uuid_compare (MY_UUID, brick->uuid) == 0) {
-                                retval = _gf_true;
-                                break;
-                        }
-                }
-                break;
+            cds_list_for_each_entry(brick, &volinfo->bricks, brick_list)
+            {
+                if (gf_uuid_compare(MY_UUID, brick->uuid) == 0) {
+                    retval = _gf_true;
+                    break;
+                }
+            }
+            break;
         case GD_OP_REMOVE_BRICK:
-                ret = dict_get_int32 (volinfo->rebal.dict, "count", &count);
-                if (ret) {
-                        goto out;
-                }
-                for (i = 1; i <= count; i++) {
-                        memset (key, 0, sizeof (key));
-                        snprintf (key, sizeof (key), "brick%d", i);
-                        ret = dict_get_str (volinfo->rebal.dict, key,
-                                            &brickname);
-                        if (ret)
-                                goto out;
-                        ret = glusterd_volume_brickinfo_get_by_brick (brickname,
-                                                                      volinfo,
-                                                                      &brick);
-                        if (ret)
-                                goto out;
-                        if (gf_uuid_compare (MY_UUID, brick->uuid) == 0) {
-                                retval = _gf_true;
-                                break;
-                        }
+            ret = dict_get_int32n(volinfo->rebal.dict, "count", SLEN("count"),
+                                  &count);
+            if (ret) {
+                goto out;
+            }
+            for (i = 1; i <= count; i++) {
+                keylen = snprintf(key, sizeof(key), "brick%d", i);
+                ret = dict_get_strn(volinfo->rebal.dict, key, keylen,
+                                    &brickname);
+                if (ret)
+                    goto out;
+                ret = glusterd_volume_brickinfo_get_by_brick(brickname, volinfo,
+                                                             &brick, _gf_false);
+                if (ret)
+                    goto out;
+                if (gf_uuid_compare(MY_UUID, brick->uuid) == 0) {
+                    retval = _gf_true;
+                    break;
                 }
-                break;
+            }
+            break;
         default:
-                break;
-        }
+            break;
+    }
 
 out:
-        return retval;
+    return retval;
 }
 
 int
-glusterd_is_volume_quota_enabled (glusterd_volinfo_t *volinfo)
+glusterd_is_volume_quota_enabled(glusterd_volinfo_t *volinfo)
 {
-        return (glusterd_volinfo_get_boolean (volinfo, VKEY_FEATURES_QUOTA));
+    return (glusterd_volinfo_get_boolean(volinfo, VKEY_FEATURES_QUOTA));
 }
 
 int
-glusterd_is_bitrot_enabled (glusterd_volinfo_t *volinfo)
+glusterd_is_volume_inode_quota_enabled(glusterd_volinfo_t *volinfo)
 {
-        return glusterd_volinfo_get_boolean (volinfo, VKEY_FEATURES_BITROT);
+    return (glusterd_volinfo_get_boolean(volinfo, VKEY_FEATURES_INODE_QUOTA));
 }
 
 int
-glusterd_validate_and_set_gfid (dict_t *op_ctx, dict_t *req_dict,
-                                char **op_errstr)
+glusterd_is_bitrot_enabled(glusterd_volinfo_t *volinfo)
 {
-        int        ret           = -1;
-        int        count         = 0;
-        int        i             = 0;
-        int        op_code       = GF_QUOTA_OPTION_TYPE_NONE;
-        uuid_t     uuid1         = {0};
-        uuid_t     uuid2         = {0,};
-        char      *path          = NULL;
-        char       key[256]      = {0,};
-        char      *uuid1_str     = NULL;
-        char      *uuid1_str_dup = NULL;
-        char      *uuid2_str     = NULL;
-        xlator_t  *this          = NULL;
-
-        this = THIS;
-        GF_ASSERT (this);
+    return glusterd_volinfo_get_boolean(volinfo, VKEY_FEATURES_BITROT);
+}
 
-        ret = dict_get_int32 (op_ctx, "type", &op_code);
+int
+glusterd_validate_and_set_gfid(dict_t *op_ctx, dict_t *req_dict,
+                               char **op_errstr)
+{
+    int ret = -1;
+    int count = 0;
+    int i = 0;
+    int op_code = GF_QUOTA_OPTION_TYPE_NONE;
+    uuid_t uuid1 = {0};
+    uuid_t uuid2 = {
+        0,
+    };
+    char *path = NULL;
+    char key[64] = "";
+    int keylen;
+    char *uuid1_str = NULL;
+    char *uuid1_str_dup = NULL;
+    char *uuid2_str = NULL;
+    xlator_t *this = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+
+    ret = dict_get_int32n(op_ctx, "type", SLEN("type"), &op_code);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+               "Failed to get quota opcode");
+        goto out;
+    }
+
+    if ((op_code != GF_QUOTA_OPTION_TYPE_LIMIT_USAGE) &&
+        (op_code != GF_QUOTA_OPTION_TYPE_LIMIT_OBJECTS) &&
+        (op_code != GF_QUOTA_OPTION_TYPE_REMOVE) &&
+        (op_code != GF_QUOTA_OPTION_TYPE_REMOVE_OBJECTS)) {
+        ret = 0;
+        goto out;
+    }
+
+    ret = dict_get_strn(op_ctx, "path", SLEN("path"), &path);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+               "Failed to get path");
+        goto out;
+    }
+
+    ret = dict_get_int32n(op_ctx, "count", SLEN("count"), &count);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+               "Failed to get count");
+        goto out;
+    }
+
+    /* If count is 0, fail the command with ENOENT.
+     *
+     * If count is 1, treat gfid0 as the gfid on which the operation
+     * is to be performed and resume the command.
+     *
+     * if count > 1, get the 0th gfid from the op_ctx and,
+     * compare it with the remaining 'count -1' gfids.
+     * If they are found to be the same, set gfid0 in the op_ctx and
+     * resume the operation, else error out.
+     */
+
+    if (count == 0) {
+        gf_asprintf(op_errstr,
+                    "Failed to get trusted.gfid attribute "
+                    "on path %s. Reason : %s",
+                    path, strerror(ENOENT));
+        ret = -ENOENT;
+        goto out;
+    }
+
+    keylen = snprintf(key, sizeof(key), "gfid%d", 0);
+
+    ret = dict_get_strn(op_ctx, key, keylen, &uuid1_str);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+               "Failed to get key '%s'", key);
+        goto out;
+    }
+
+    gf_uuid_parse(uuid1_str, uuid1);
+
+    for (i = 1; i < count; i++) {
+        keylen = snprintf(key, sizeof(key), "gfid%d", i);
+
+        ret = dict_get_strn(op_ctx, key, keylen, &uuid2_str);
         if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "Failed to get quota opcode");
-                goto out;
-        }
-
-        if ((op_code != GF_QUOTA_OPTION_TYPE_LIMIT_USAGE) &&
-            (op_code != GF_QUOTA_OPTION_TYPE_LIMIT_OBJECTS) &&
-            (op_code != GF_QUOTA_OPTION_TYPE_REMOVE) &&
-            (op_code != GF_QUOTA_OPTION_TYPE_REMOVE_OBJECTS)) {
-                ret = 0;
-                goto out;
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+                   "Failed to get key "
+                   "'%s'",
+                   key);
+            goto out;
         }
 
-        ret = dict_get_str (op_ctx, "path", &path);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "Failed to get path");
-                goto out;
-        }
+        gf_uuid_parse(uuid2_str, uuid2);
 
-        ret = dict_get_int32 (op_ctx, "count", &count);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "Failed to get count");
-                goto out;
+        if (gf_uuid_compare(uuid1, uuid2)) {
+            gf_asprintf(op_errstr,
+                        "gfid mismatch between %s and "
+                        "%s for path %s",
+                        uuid1_str, uuid2_str, path);
+            ret = -1;
+            goto out;
         }
+    }
 
-        /* If count is 0, fail the command with ENOENT.
-         *
-         * If count is 1, treat gfid0 as the gfid on which the operation
-         * is to be performed and resume the command.
-         *
-         * if count > 1, get the 0th gfid from the op_ctx and,
-         * compare it with the remaining 'count -1' gfids.
-         * If they are found to be the same, set gfid0 in the op_ctx and
-         * resume the operation, else error out.
-         */
-
-        if (count == 0) {
-                gf_asprintf (op_errstr, "Failed to get trusted.gfid attribute "
-                             "on path %s. Reason : %s", path,
-                             strerror (ENOENT));
-                ret = -ENOENT;
-                goto out;
+    if (i == count) {
+        uuid1_str_dup = gf_strdup(uuid1_str);
+        if (!uuid1_str_dup) {
+            ret = -1;
+            goto out;
         }
 
-        snprintf (key, sizeof (key) - 1, "gfid%d", 0);
-
-        ret = dict_get_str (op_ctx, key, &uuid1_str);
+        ret = dict_set_dynstrn(req_dict, "gfid", SLEN("gfid"), uuid1_str_dup);
         if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "Failed to get key '%s'",
-                        key);
-                goto out;
-        }
-
-        gf_uuid_parse (uuid1_str, uuid1);
-
-        for (i = 1; i < count; i++) {
-                snprintf (key, sizeof (key)-1, "gfid%d", i);
-
-                ret = dict_get_str (op_ctx, key, &uuid2_str);
-                if (ret) {
-                        gf_log (this->name, GF_LOG_ERROR, "Failed to get key "
-                                "'%s'", key);
-                        goto out;
-                }
-
-                gf_uuid_parse (uuid2_str, uuid2);
-
-                if (gf_uuid_compare (uuid1, uuid2)) {
-                        gf_asprintf (op_errstr, "gfid mismatch between %s and "
-                                     "%s for path %s", uuid1_str, uuid2_str,
-                                     path);
-                        ret = -1;
-                        goto out;
-                }
-        }
-
-        if (i == count) {
-                uuid1_str_dup = gf_strdup (uuid1_str);
-                if (!uuid1_str_dup) {
-                        ret = -1;
-                        goto out;
-                }
-
-                ret = dict_set_dynstr (req_dict, "gfid", uuid1_str_dup);
-                if (ret) {
-                        gf_log (this->name, GF_LOG_ERROR, "Failed to set gfid");
-                        GF_FREE (uuid1_str_dup);
-                        goto out;
-                }
-        } else {
-                gf_log (this->name, GF_LOG_ERROR, "Failed to iterate through %d"
-                        " entries in the req dict", count);
-                ret = -1;
-                goto out;
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+                   "Failed to set gfid");
+            GF_FREE(uuid1_str_dup);
+            goto out;
         }
+    } else {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_ITER_FAIL,
+               "Failed to iterate through %d"
+               " entries in the req dict",
+               count);
+        ret = -1;
+        goto out;
+    }
 
-        ret = 0;
+    ret = 0;
 out:
-        return ret;
+    return ret;
 }
 
 void
-glusterd_clean_up_quota_store (glusterd_volinfo_t *volinfo)
+glusterd_clean_up_quota_store(glusterd_volinfo_t *volinfo)
 {
-        char      voldir[PATH_MAX]         = {0,};
-        char      quota_confpath[PATH_MAX] = {0,};
-        char      cksum_path[PATH_MAX]     = {0,};
-        xlator_t  *this                    = NULL;
-        glusterd_conf_t *conf              = NULL;
+    char voldir[PATH_MAX] = "";
+    char quota_confpath[PATH_MAX] = "";
+    char cksum_path[PATH_MAX] = "";
+    xlator_t *this = NULL;
+    glusterd_conf_t *conf = NULL;
+    int32_t len = 0;
 
-        this = THIS;
-        GF_ASSERT (this);
-        conf = this->private;
-        GF_ASSERT (conf);
+    this = THIS;
+    GF_ASSERT(this);
+    conf = this->private;
+    GF_ASSERT(conf);
 
-        GLUSTERD_GET_VOLUME_DIR (voldir, volinfo, conf);
+    GLUSTERD_GET_VOLUME_DIR(voldir, volinfo, conf);
 
-        snprintf (quota_confpath, sizeof (quota_confpath), "%s/%s", voldir,
-                  GLUSTERD_VOLUME_QUOTA_CONFIG);
-        snprintf (cksum_path, sizeof (cksum_path), "%s/%s", voldir,
-                  GLUSTERD_VOL_QUOTA_CKSUM_FILE);
+    len = snprintf(quota_confpath, sizeof(quota_confpath), "%s/%s", voldir,
+                   GLUSTERD_VOLUME_QUOTA_CONFIG);
+    if ((len < 0) || (len >= sizeof(quota_confpath))) {
+        quota_confpath[0] = 0;
+    }
+    len = snprintf(cksum_path, sizeof(cksum_path), "%s/%s", voldir,
+                   GLUSTERD_VOL_QUOTA_CKSUM_FILE);
+    if ((len < 0) || (len >= sizeof(cksum_path))) {
+        cksum_path[0] = 0;
+    }
 
-        unlink (quota_confpath);
-        unlink (cksum_path);
+    sys_unlink(quota_confpath);
+    sys_unlink(cksum_path);
 
-        gf_store_handle_destroy (volinfo->quota_conf_shandle);
-        volinfo->quota_conf_shandle = NULL;
-        volinfo->quota_conf_version = 0;
-
-}
-
-#define QUOTA_CONF_HEADER                                                \
-        "GlusterFS Quota conf | version: v%d.%d\n"
-
-int
-glusterd_store_quota_conf_skip_header (xlator_t *this, int fd)
-{
-        char buf[PATH_MAX] = {0,};
-
-        snprintf (buf, sizeof(buf)-1, QUOTA_CONF_HEADER, 1, 1);
-        return gf_skip_header_section (fd, strlen (buf));
-}
-
-int
-glusterd_store_quota_conf_stamp_header (xlator_t *this, int fd)
-{
-        char buf[PATH_MAX]  = {0,};
-        int  buf_len        = 0;
-        ssize_t  ret        = -1;
-        ssize_t  written    = 0;
-
-        snprintf (buf, sizeof(buf)-1, QUOTA_CONF_HEADER, 1, 1);
-        buf_len = strlen (buf);
-        for (written = 0; written != buf_len; written += ret) {
-                ret = write (fd, buf + written, buf_len - written);
-                if (ret == -1) {
-                        goto out;
-                }
-        }
-
-        ret = 0;
-out:
-        return ret;
+    gf_store_handle_destroy(volinfo->quota_conf_shandle);
+    volinfo->quota_conf_shandle = NULL;
+    volinfo->quota_conf_version = 0;
 }
 
 int
-glusterd_remove_auxiliary_mount (char *volname)
+glusterd_remove_auxiliary_mount(char *volname)
 {
-        int       ret                = -1;
-        char      mountdir[PATH_MAX] = {0,};
-        char      pidfile[PATH_MAX]  = {0,};
-        xlator_t *this               = NULL;
+    int ret = -1;
+    char mountdir[PATH_MAX] = "";
+    xlator_t *this = NULL;
 
-        this = THIS;
-        GF_ASSERT (this);
+    this = THIS;
+    GF_ASSERT(this);
 
-        GLUSTERFS_GET_AUX_MOUNT_PIDFILE (pidfile, volname);
+    GLUSTERD_GET_QUOTA_LIMIT_MOUNT_PATH(mountdir, volname, "/");
+    ret = gf_umount_lazy(this->name, mountdir, 1);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, errno, GD_MSG_LAZY_UMOUNT_FAIL,
+               "umount on %s failed, "
+               "reason : %s",
+               mountdir, strerror(errno));
 
-        if (!gf_is_service_running (pidfile, NULL)) {
-                gf_log (this->name, GF_LOG_DEBUG, "Aux mount of volume %s "
-                        "absent, hence returning", volname);
-                return 0;
-        }
-
-        GLUSTERD_GET_QUOTA_AUX_MOUNT_PATH (mountdir, volname, "/");
-        ret = gf_umount_lazy (this->name, mountdir, 1);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "umount on %s failed, "
-                        "reason : %s", mountdir, strerror (errno));
+        /* Hide EBADF as it means the mount is already gone */
+        if (errno == EBADF)
+            ret = 0;
+    }
 
-                /* Hide EBADF as it means the mount is already gone */
-                if (errno == EBADF)
-                       ret = 0;
-        }
-
-        return ret;
+    return ret;
 }
 
 /* Stops the rebalance process of the given volume
  */
 int
-gd_stop_rebalance_process (glusterd_volinfo_t *volinfo)
+gd_stop_rebalance_process(glusterd_volinfo_t *volinfo)
 {
-        int              ret               = -1;
-        xlator_t        *this              = NULL;
-        glusterd_conf_t *conf              = NULL;
-        char             pidfile[PATH_MAX] = {0,};
+    int ret = -1;
+    xlator_t *this = NULL;
+    glusterd_conf_t *conf = NULL;
+    char pidfile[PATH_MAX] = "";
 
-        GF_ASSERT (volinfo);
+    GF_ASSERT(volinfo);
 
-        this = THIS;
-        GF_ASSERT (this);
+    this = THIS;
+    GF_ASSERT(this);
 
-        conf = this->private;
-        GF_ASSERT (conf);
+    conf = this->private;
+    GF_ASSERT(conf);
 
-        GLUSTERD_GET_DEFRAG_PID_FILE (pidfile, volinfo, conf);
-        ret = glusterd_service_stop ("rebalance", pidfile, SIGTERM, _gf_true);
+    GLUSTERD_GET_DEFRAG_PID_FILE(pidfile, volinfo, conf);
+    ret = glusterd_service_stop("rebalance", pidfile, SIGTERM, _gf_true);
 
-        return ret;
+    return ret;
 }
 
 rpc_clnt_t *
-glusterd_rpc_clnt_unref (glusterd_conf_t *conf, rpc_clnt_t *rpc)
+glusterd_rpc_clnt_unref(glusterd_conf_t *conf, rpc_clnt_t *rpc)
 {
-        rpc_clnt_t *ret = NULL;
+    rpc_clnt_t *ret = NULL;
 
-        GF_ASSERT (conf);
-        GF_ASSERT (rpc);
-        synclock_unlock (&conf->big_lock);
-        ret = rpc_clnt_unref (rpc);
-        synclock_lock (&conf->big_lock);
+    GF_ASSERT(conf);
+    GF_ASSERT(rpc);
+    synclock_unlock(&conf->big_lock);
+    (void)rpc_clnt_reconnect_cleanup(&rpc->conn);
+    ret = rpc_clnt_unref(rpc);
+    synclock_lock(&conf->big_lock);
 
-        return ret;
+    return ret;
 }
 
 int32_t
 glusterd_compare_volume_name(struct cds_list_head *list1,
                              struct cds_list_head *list2)
 {
-        glusterd_volinfo_t *volinfo1 = NULL;
-        glusterd_volinfo_t *volinfo2 = NULL;
+    glusterd_volinfo_t *volinfo1 = NULL;
+    glusterd_volinfo_t *volinfo2 = NULL;
 
-        volinfo1 = cds_list_entry (list1, glusterd_volinfo_t, vol_list);
-        volinfo2 = cds_list_entry (list2, glusterd_volinfo_t, vol_list);
-        return strcmp(volinfo1->volname, volinfo2->volname);
+    volinfo1 = cds_list_entry(list1, glusterd_volinfo_t, vol_list);
+    volinfo2 = cds_list_entry(list2, glusterd_volinfo_t, vol_list);
+    return strcmp(volinfo1->volname, volinfo2->volname);
 }
 
 static int
-gd_default_synctask_cbk (int ret, call_frame_t *frame, void *opaque)
+gd_default_synctask_cbk(int ret, call_frame_t *frame, void *opaque)
 {
-        glusterd_conf_t     *priv = THIS->private;
-        synclock_unlock (&priv->big_lock);
-        return ret;
+    glusterd_conf_t *priv = THIS->private;
+    synclock_unlock(&priv->big_lock);
+    return ret;
 }
 
 void
-glusterd_launch_synctask (synctask_fn_t fn, void *opaque)
+glusterd_launch_synctask(synctask_fn_t fn, void *opaque)
 {
-        xlator_t        *this = NULL;
-        glusterd_conf_t *priv = NULL;
-        int             ret   = -1;
+    xlator_t *this = NULL;
+    int ret = -1;
 
-        this = THIS;
-        priv = this->private;
+    this = THIS;
 
-        synclock_lock (&priv->big_lock);
-        ret = synctask_new (this->ctx->env, fn, gd_default_synctask_cbk, NULL,
-                            opaque);
-        if (ret)
-                gf_log (this->name, GF_LOG_CRITICAL, "Failed to spawn bricks"
-                        " and other volume related services");
+    /* synclock_lock must be called from within synctask, @fn must call it
+     * before it starts with its work*/
+    ret = synctask_new(this->ctx->env, fn, gd_default_synctask_cbk, NULL,
+                       opaque);
+    if (ret)
+        gf_msg(this->name, GF_LOG_CRITICAL, 0, GD_MSG_SPAWN_SVCS_FAIL,
+               "Failed to spawn bricks"
+               " and other volume related services");
 }
 
 /*
@@ -9102,8 +12995,8 @@ glusterd_launch_synctask (synctask_fn_t fn, void *opaque)
  * @option  - option to be set to default. If NULL, all possible options will be
  *            set to default
  *
- * Returns 0 on sucess and -1 on failure. If @option is given, but doesn't match
- * any of the options that could be set, it is a success.
+ * Returns 0 on success and -1 on failure. If @option is given, but doesn't
+ * match any of the options that could be set, it is a success.
  */
 /*
  * TODO: Make this able to parse the volume-set table to set options
@@ -9114,45 +13007,145 @@ glusterd_launch_synctask (synctask_fn_t fn, void *opaque)
  * default values
  */
 int
-glusterd_enable_default_options (glusterd_volinfo_t *volinfo, char *option)
+glusterd_enable_default_options(glusterd_volinfo_t *volinfo, char *option)
 {
-        int ret = 0;
-        xlator_t *this = NULL;
-        glusterd_conf_t *conf = NULL;
+    int ret = 0;
+    xlator_t *this = NULL;
+    glusterd_conf_t *conf = NULL;
+#ifdef IPV6_DEFAULT
+    char *addr_family = "inet6";
+#else
+    char *addr_family = "inet";
+#endif
+
+    this = THIS;
+    GF_ASSERT(this);
 
-        this = THIS;
-        GF_ASSERT (this);
+    GF_VALIDATE_OR_GOTO(this->name, volinfo, out);
 
-        GF_VALIDATE_OR_GOTO (this->name, volinfo, out);
+    conf = this->private;
+    GF_ASSERT(conf);
+
+#ifdef GD_OP_VERSION_3_8_0
+    if (conf->op_version >= GD_OP_VERSION_3_8_0) {
+        /* nfs.disable needs to be enabled for new volumes with
+         * >= gluster version 3.7 (for now) 3.8 later
+         */
+        if (!option || !strcmp(NFS_DISABLE_MAP_KEY, option)) {
+            ret = dict_set_dynstr_with_alloc(volinfo->dict, NFS_DISABLE_MAP_KEY,
+                                             "on");
+            if (ret) {
+                gf_msg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_SET_FAILED,
+                       "Failed to set option '" NFS_DISABLE_MAP_KEY
+                       "' on volume "
+                       "%s",
+                       volinfo->volname);
+                goto out;
+            }
+        }
+    }
+#endif
+
+    if (conf->op_version >= GD_OP_VERSION_3_7_0) {
+        /* Set needed volume options in volinfo->dict
+         * For ex.,
+         *
+         * if (!option || !strcmp("someoption", option) {
+         *      ret = dict_set_str(volinfo->dict, "someoption", "on");
+         *      ...
+         * }
+         * */
+
+        /* Option 'features.quota-deem-statfs' should not be turned off
+         * with 'gluster volume reset <VOLNAME>', since quota features
+         * can be reset only with 'gluster volume quota <VOLNAME>
+         * disable'.
+         */
 
-        conf = this->private;
-        GF_ASSERT (conf);
+        if (!option || !strcmp("features.quota-deem-statfs", option)) {
+            if (glusterd_is_volume_quota_enabled(volinfo)) {
+                ret = dict_set_dynstr_with_alloc(
+                    volinfo->dict, "features.quota-deem-statfs", "on");
+                if (ret) {
+                    gf_msg(this->name, GF_LOG_ERROR, errno,
+                           GD_MSG_DICT_SET_FAILED,
+                           "Failed to set option "
+                           "'features.quota-deem-statfs' "
+                           "on volume %s",
+                           volinfo->volname);
+                    goto out;
+                }
+            }
+        }
+    }
+
+    if (conf->op_version >= GD_OP_VERSION_3_9_0) {
+        if (!option || !strcmp("transport.address-family", option)) {
+            if (volinfo->transport_type == GF_TRANSPORT_TCP) {
+                ret = dict_set_dynstr_with_alloc(
+                    volinfo->dict, "transport.address-family", addr_family);
+                if (ret) {
+                    gf_msg(this->name, GF_LOG_ERROR, errno,
+                           GD_MSG_DICT_SET_FAILED,
+                           "failed to set transport."
+                           "address-family on %s",
+                           volinfo->volname);
+                    goto out;
+                }
+            }
+        }
+    }
 
-        if (conf->op_version >= GD_OP_VERSION_3_6_0) {
-                /* Set needed volume options in volinfo->dict
-                 * For ex.,
-                 *
-                 * if (!option || !strcmp("someoption", option) {
-                 *      ret = dict_set_str(volinfo->dict, "someoption", "on");
-                 *      ...
-                 * }
-                 * */
+    if (conf->op_version >= GD_OP_VERSION_7_0) {
+        ret = dict_set_dynstr_with_alloc(volinfo->dict,
+                                         "storage.fips-mode-rchecksum", "on");
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_SET_FAILED,
+                   "Failed to set option 'storage.fips-mode-rchecksum' "
+                   "on volume %s",
+                   volinfo->volname);
+            goto out;
         }
+    }
 out:
-        return ret;
+    return ret;
+}
+
+void
+glusterd_get_gfproxy_client_volfile(glusterd_volinfo_t *volinfo, char *path,
+                                    int path_len)
+{
+    char workdir[PATH_MAX] = "";
+    glusterd_conf_t *priv = THIS->private;
+
+    GLUSTERD_GET_VOLUME_DIR(workdir, volinfo, priv);
+
+    switch (volinfo->transport_type) {
+        case GF_TRANSPORT_TCP:
+        case GF_TRANSPORT_BOTH_TCP_RDMA:
+            snprintf(path, path_len, "%s/trusted-%s.tcp-gfproxy-fuse.vol",
+                     workdir, volinfo->volname);
+            break;
+
+        case GF_TRANSPORT_RDMA:
+            snprintf(path, path_len, "%s/trusted-%s.rdma-gfproxy-fuse.vol",
+                     workdir, volinfo->volname);
+            break;
+        default:
+            break;
+    }
 }
 
 void
-glusterd_get_rebalance_volfile (glusterd_volinfo_t *volinfo,
-                                char *path, int path_len)
+glusterd_get_rebalance_volfile(glusterd_volinfo_t *volinfo, char *path,
+                               int path_len)
 {
-        char                    workdir[PATH_MAX]      = {0,};
-        glusterd_conf_t        *priv                   = THIS->private;
+    char workdir[PATH_MAX] = "";
+    glusterd_conf_t *priv = THIS->private;
 
-        GLUSTERD_GET_VOLUME_DIR (workdir, volinfo, priv);
+    GLUSTERD_GET_VOLUME_DIR(workdir, volinfo, priv);
 
-        snprintf (path, path_len, "%s/%s-rebalance.vol", workdir,
-                  volinfo->volname);
+    snprintf(path, path_len, "%s/%s-rebalance.vol", workdir, volinfo->volname);
 }
 
 /* This function will update the backend file-system
@@ -9165,376 +13158,844 @@ glusterd_get_rebalance_volfile (glusterd_volinfo_t *volinfo,
  * @return 0 on success and -1 on failure
  */
 int
-glusterd_update_mntopts (char *brick_path, glusterd_brickinfo_t *brickinfo)
-{
-        int32_t               ret               = -1;
-        char                 *mnt_pt            = NULL;
-        char                  buff[PATH_MAX]    = "";
-        char                  msg[PATH_MAX]     = "";
-        char                 *cmd               = NULL;
-        struct mntent        *entry             = NULL;
-        struct mntent         save_entry        = {0,};
-        runner_t              runner            = {0,};
-        xlator_t             *this              = NULL;
+glusterd_update_mntopts(char *brick_path, glusterd_brickinfo_t *brickinfo)
+{
+    int32_t ret = -1;
+    char *mnt_pt = NULL;
+    char buff[PATH_MAX] = "";
+    struct mntent *entry = NULL;
+    struct mntent save_entry = {
+        0,
+    };
+    xlator_t *this = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+    GF_ASSERT(brick_path);
+    GF_ASSERT(brickinfo);
+
+    ret = glusterd_get_brick_root(brick_path, &mnt_pt);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_BRICKPATH_ROOT_GET_FAIL,
+               "getting the root "
+               "of the brick (%s) failed ",
+               brick_path);
+        goto out;
+    }
+
+    entry = glusterd_get_mnt_entry_info(mnt_pt, buff, sizeof(buff),
+                                        &save_entry);
+    if (!entry) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_MNTENTRY_GET_FAIL,
+               "getting the mount entry for "
+               "the brick (%s) failed",
+               brick_path);
+        ret = -1;
+        goto out;
+    }
 
-        this = THIS;
-        GF_ASSERT (this);
-        GF_ASSERT (brick_path);
-        GF_ASSERT (brickinfo);
+    if (snprintf(brickinfo->fstype, sizeof(brickinfo->fstype), "%s",
+                 entry->mnt_type) >= sizeof(brickinfo->fstype)) {
+        ret = -1;
+        goto out;
+    }
+    (void)snprintf(brickinfo->mnt_opts, sizeof(brickinfo->mnt_opts), "%s",
+                   entry->mnt_opts);
 
-        ret = glusterd_get_brick_root (brick_path, &mnt_pt);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "getting the root "
-                        "of the brick (%s) failed ", brick_path);
-                goto out;
-        }
+    gf_strncpy(brickinfo->mnt_opts, entry->mnt_opts,
+               sizeof(brickinfo->mnt_opts));
 
-        entry = glusterd_get_mnt_entry_info (mnt_pt, buff, sizeof (buff),
-                                             &save_entry);
-        if (!entry) {
-                gf_log (this->name, GF_LOG_ERROR, "getting the mount entry for "
-                        "the brick (%s) failed", brick_path);
-                ret = -1;
-                goto out;
-        }
+    ret = 0;
+out:
+    if (mnt_pt)
+        GF_FREE(mnt_pt);
+    return ret;
+}
 
-        strcpy (brickinfo->fstype, entry->mnt_type);
-        strcpy (brickinfo->mnt_opts, entry->mnt_opts);
+int
+glusterd_get_value_for_vme_entry(struct volopt_map_entry *vme, char **def_val)
+{
+    int ret = -1;
+    char *key = NULL;
+    xlator_t *this = NULL;
+    char *descr = NULL;
+    char *local_def_val = NULL;
+    void *dl_handle = NULL;
+    volume_opt_list_t vol_opt_handle = {
+        {0},
+    };
+
+    this = THIS;
+    GF_ASSERT(this);
+
+    CDS_INIT_LIST_HEAD(&vol_opt_handle.list);
+
+    if (_get_xlator_opt_key_from_vme(vme, &key)) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_GET_KEY_FAILED,
+               "Failed to get %s key from "
+               "volume option entry",
+               vme->key);
+        goto out;
+    }
+
+    ret = xlator_volopt_dynload(vme->voltype, &dl_handle, &vol_opt_handle);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_XLATOR_VOLOPT_DYNLOAD_ERROR,
+               "xlator_volopt_dynload error "
+               "(%d)",
+               ret);
+        ret = -2;
+        goto cont;
+    }
+
+    ret = xlator_option_info_list(&vol_opt_handle, key, &local_def_val, &descr);
+    if (ret) {
+        /*Swallow Error if option not found*/
+        gf_msg(this->name, GF_LOG_WARNING, 0, GD_MSG_GET_KEY_FAILED,
+               "Failed to get option for %s "
+               "key",
+               key);
+        ret = -2;
+        goto cont;
+    }
+    if (!local_def_val)
+        local_def_val = "(null)";
+
+    *def_val = gf_strdup(local_def_val);
+
+cont:
+    if (dl_handle) {
+        dlclose(dl_handle);
+        dl_handle = NULL;
+        vol_opt_handle.given_opt = NULL;
+    }
+    if (key) {
+        _free_xlator_opt_key(key);
+        key = NULL;
+    }
+
+    if (ret)
+        goto out;
 
-        ret = 0;
 out:
-        GF_FREE (mnt_pt);
-        return ret;
+    gf_msg_debug(this->name, 0, "Returning %d", ret);
+    return ret;
 }
 
 int
-glusterd_get_value_for_vme_entry (struct volopt_map_entry *vme, char **def_val)
-{
-        int                      ret = -1;
-        char                    *key = NULL;
-        xlator_t                *this = NULL;
-        char                    *descr = NULL;
-        char                    *local_def_val = NULL;
-        void                    *dl_handle = NULL;
-        volume_opt_list_t        vol_opt_handle = {{0},};
+glusterd_get_global_max_op_version(rpcsvc_request_t *req, dict_t *ctx,
+                                   int count)
+{
+    int ret = -1;
+    char *def_val = NULL;
+    char dict_key[50] = "";
+    int keylen;
+
+    ret = glusterd_mgmt_v3_initiate_all_phases(req, GD_OP_MAX_OPVERSION, ctx);
+
+    ret = dict_get_strn(ctx, "max-opversion", SLEN("max-opversion"), &def_val);
+    if (ret) {
+        gf_msg(THIS->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+               "Failed to get max-opversion value from"
+               " dictionary");
+        goto out;
+    }
+
+    keylen = sprintf(dict_key, "key%d", count);
+    ret = dict_set_nstrn(ctx, dict_key, keylen, GLUSTERD_MAX_OP_VERSION_KEY,
+                         SLEN(GLUSTERD_MAX_OP_VERSION_KEY));
+    if (ret) {
+        gf_msg(THIS->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+               "Failed to set %s in "
+               "dictionary",
+               GLUSTERD_MAX_OP_VERSION_KEY);
+        goto out;
+    }
+
+    sprintf(dict_key, "value%d", count);
+    ret = dict_set_dynstr_with_alloc(ctx, dict_key, def_val);
+    if (ret) {
+        gf_msg(THIS->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+               "Failed to set %s for key %s in dictionary", def_val,
+               GLUSTERD_MAX_OP_VERSION_KEY);
+        goto out;
+    }
 
-        this = THIS;
-        GF_ASSERT (this);
+out:
+    return ret;
+}
 
-        CDS_INIT_LIST_HEAD (&vol_opt_handle.list);
+int
+glusterd_get_global_options_for_all_vols(rpcsvc_request_t *req, dict_t *ctx,
+                                         char **op_errstr)
+{
+    int ret = -1;
+    int count = 0;
+    gf_boolean_t all_opts = _gf_false;
+    gf_boolean_t key_found = _gf_false;
+    glusterd_conf_t *priv = NULL;
+    xlator_t *this = NULL;
+    char *key = NULL;
+    char *key_fixed = NULL;
+    char dict_key[50] = "";
+    char *def_val = NULL;
+    char err_str[PATH_MAX] = "";
+    char *allvolopt = NULL;
+    int32_t i = 0;
+    gf_boolean_t exists = _gf_false;
+    gf_boolean_t need_free = _gf_false;
+
+    this = THIS;
+    GF_VALIDATE_OR_GOTO(THIS->name, this, out);
+
+    priv = this->private;
+    GF_VALIDATE_OR_GOTO(this->name, priv, out);
+
+    GF_VALIDATE_OR_GOTO(this->name, ctx, out);
+
+    ret = dict_get_strn(ctx, "key", SLEN("key"), &key);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+               "Failed to get option key from dictionary");
+        goto out;
+    }
+
+    if (strcasecmp(key, "all") == 0)
+        all_opts = _gf_true;
+    else {
+        exists = glusterd_check_option_exists(key, &key_fixed);
+        if (!exists) {
+            snprintf(err_str, sizeof(err_str),
+                     "Option "
+                     "with name: %s does not exist",
+                     key);
+            gf_msg(this->name, GF_LOG_ERROR, EINVAL, GD_MSG_UNKNOWN_KEY, "%s",
+                   err_str);
+            if (key_fixed)
+                snprintf(err_str, sizeof(err_str), "Did you mean %s?",
+                         key_fixed);
+            ret = -1;
+            goto out;
+        }
+        if (key_fixed)
+            key = key_fixed;
+    }
+    /* coverity[CONSTANT_EXPRESSION_RESULT] */
+    ALL_VOLUME_OPTION_CHECK("all", _gf_true, key, ret, op_errstr, out);
 
-        if (_get_xlator_opt_key_from_vme (vme, &key)) {
-                gf_log (this->name, GF_LOG_ERROR, "Failed to get %s key from "
-                        "volume option entry", vme->key);
+    for (i = 0; valid_all_vol_opts[i].option; i++) {
+        allvolopt = valid_all_vol_opts[i].option;
+
+        if (!all_opts && strcmp(key, allvolopt) != 0)
+            continue;
+
+        /* Found global option */
+        if (strcmp(allvolopt, GLUSTERD_MAX_OP_VERSION_KEY) == 0) {
+            count++;
+            ret = glusterd_get_global_max_op_version(req, ctx, count);
+            if (ret)
                 goto out;
+            else
+                continue;
         }
 
-        ret = xlator_volopt_dynload (vme->voltype, &dl_handle, &vol_opt_handle);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "xlator_volopt_dynload error "
-                        "(%d)", ret);
-                ret = -2;
-                goto cont;
+        ret = dict_get_str(priv->opts, allvolopt, &def_val);
+
+        /* If global option isn't set explicitly */
+
+        if (!def_val) {
+            if (!strcmp(allvolopt, GLUSTERD_GLOBAL_OP_VERSION_KEY)) {
+                gf_asprintf(&def_val, "%d", priv->op_version);
+                need_free = _gf_true;
+            } else {
+                gf_asprintf(&def_val, "%s (DEFAULT)",
+                            valid_all_vol_opts[i].dflt_val);
+                need_free = _gf_true;
+            }
         }
 
-        ret = xlator_option_info_list (&vol_opt_handle,key,
-                                       &local_def_val, &descr);
+        count++;
+        ret = sprintf(dict_key, "key%d", count);
+        ret = dict_set_strn(ctx, dict_key, ret, allvolopt);
         if (ret) {
-                /*Swallow Error if option not found*/
-                gf_log (this->name, GF_LOG_ERROR, "Failed to get option for %s "
-                        "key", key);
-                ret = -2;
-                goto cont;
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+                   "Failed to set %s in dictionary", allvolopt);
+            goto out;
         }
-        if (!local_def_val)
-                local_def_val = "(null)";
-
-        *def_val = gf_strdup (local_def_val);
 
-cont:
-        if (dl_handle) {
-                dlclose (dl_handle);
-                dl_handle = NULL;
-                vol_opt_handle.given_opt = NULL;
+        sprintf(dict_key, "value%d", count);
+        ret = dict_set_dynstr_with_alloc(ctx, dict_key, def_val);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+                   "Failed to set %s for key %s in dictionary", def_val,
+                   allvolopt);
+            goto out;
         }
-        if (key) {
-                _free_xlator_opt_key (key);
-                key = NULL;
+
+        if (need_free) {
+            GF_FREE(def_val);
+            need_free = _gf_false;
         }
+        def_val = NULL;
+        allvolopt = NULL;
 
-        if (ret)
-                goto out;
+        if (!all_opts)
+            break;
+    }
+
+    ret = dict_set_int32n(ctx, "count", SLEN("count"), count);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+               "Failed to set count in dictionary");
+    }
 
 out:
-        gf_log (this->name, GF_LOG_DEBUG, "Returning %d", ret);
-        return ret;
+    if (ret && !all_opts && !key_found) {
+        if (err_str[0] == 0)
+            snprintf(err_str, sizeof(err_str), "option %s does not exist", key);
+        if (*op_errstr == NULL)
+            *op_errstr = gf_strdup(err_str);
+    }
+
+    if (ret && need_free) {
+        GF_FREE(def_val);
+    }
+    GF_FREE(key_fixed);
+    gf_msg_debug(THIS->name, 0, "Returning %d", ret);
+
+    return ret;
 }
 
-int
-glusterd_get_default_val_for_volopt (dict_t *ctx, gf_boolean_t all_opts,
-                                     char *input_key, char *orig_key,
-                                     dict_t *vol_dict, char **op_errstr)
+char *
+glusterd_get_option_value(glusterd_volinfo_t *volinfo, char *key)
 {
-        struct volopt_map_entry *vme = NULL;
-        int                      ret = -1;
-        int                      count = 0;
-        char                     err_str[PATH_MAX] = "";
-        xlator_t                *this = NULL;
-        char                    *def_val = NULL;
-        char                     dict_key[50] = {0,};
-        gf_boolean_t             key_found = _gf_false;
+    char *value = NULL;
 
-        this = THIS;
-        GF_ASSERT (this);
+    if (!glusterd_is_volume_replicate(volinfo))
+        goto ret;
 
-        GF_VALIDATE_OR_GOTO (this->name, vol_dict, out);
-
-        /* Check whether key is passed for a single option */
-        if (!all_opts && !input_key) {
-                gf_log (this->name, GF_LOG_ERROR, "Key is NULL");
-                goto out;
+    if (!strcmp(key, "performance.client-io-threads")) {
+        value = "off";
+    } else if (!strcmp(key, "cluster.quorum-type")) {
+        if (volinfo->replica_count % 2) {
+            value = "auto";
         }
+    }
+ret:
+    return value;
+}
 
-        for (vme = &glusterd_volopt_map[0]; vme->key; vme++) {
-                if (!all_opts && strcmp (vme->key, input_key))
-                        continue;
-
-                key_found = _gf_true;
-                /* First look for the key in the vol_dict, if its not
-                 * present then look for translator default value */
-                ret = dict_get_str (vol_dict, vme->key, &def_val);
-                if (!def_val) {
-                        if (vme->value) {
-                                def_val = vme->value;
-                        } else {
-                                ret = glusterd_get_value_for_vme_entry
-                                         (vme, &def_val);
-                                if (!all_opts && ret)
-                                        goto out;
-                                else if (ret == -2)
-                                        continue;
-                        }
-                }
-                count++;
-                sprintf (dict_key, "key%d", count);
-                ret = dict_set_str(ctx, dict_key, vme->key);
-                if (ret) {
-                        gf_log (this->name, GF_LOG_ERROR, "Failed to "
-                                "set %s in dictionary", vme->key);
-                        goto out;
-                }
-                sprintf (dict_key, "value%d", count);
-                ret = dict_set_dynstr_with_alloc (ctx, dict_key, def_val);
-                if (ret) {
-                        gf_log (this->name, GF_LOG_ERROR, "Failed to "
-                                "set %s for key %s in dictionary", def_val,
-                                vme->key);
+int
+glusterd_get_default_val_for_volopt(dict_t *ctx, gf_boolean_t all_opts,
+                                    char *input_key, char *orig_key,
+                                    glusterd_volinfo_t *volinfo,
+                                    char **op_errstr)
+{
+    struct volopt_map_entry *vme = NULL;
+    int ret = -1;
+    int count = 0;
+    xlator_t *this = NULL;
+    char *def_val = NULL;
+    char *def_val_str = NULL;
+    char dict_key[50] = "";
+    int keylen;
+    gf_boolean_t key_found = _gf_false;
+    gf_boolean_t get_value_vme = _gf_false;
+    glusterd_conf_t *priv = NULL;
+    dict_t *vol_dict = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+
+    priv = this->private;
+    GF_VALIDATE_OR_GOTO(this->name, priv, out);
+
+    vol_dict = volinfo->dict;
+    GF_VALIDATE_OR_GOTO(this->name, vol_dict, out);
+
+    /* Check whether key is passed for a single option */
+    if (!all_opts && !input_key) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_KEY_NULL, "Key is NULL");
+        goto out;
+    }
+
+    for (vme = &glusterd_volopt_map[0]; vme->key; vme++) {
+        if (!all_opts && strcmp(vme->key, input_key))
+            continue;
+        key_found = _gf_true;
+        get_value_vme = _gf_false;
+        /* First look for the key in the priv->opts for global option
+         * and then into vol_dict, if its not present then look for
+         * translator default value */
+        keylen = strlen(vme->key);
+        ret = dict_get_strn(priv->opts, vme->key, keylen, &def_val);
+        if (!def_val) {
+            ret = dict_get_strn(vol_dict, vme->key, keylen, &def_val);
+            if (ret == -ENOENT)
+                def_val = glusterd_get_option_value(volinfo, vme->key);
+            if (!def_val) {
+                if (vme->value) {
+                    def_val = vme->value;
+                } else {
+                    ret = glusterd_get_value_for_vme_entry(vme, &def_val);
+                    get_value_vme = _gf_true;
+                    if (!all_opts && ret)
                         goto out;
+                    else if (ret == -2)
+                        continue;
                 }
-                def_val = NULL;
-                if (!all_opts)
-                        break;
-
+            }
         }
-        if (!all_opts && !key_found)
-                goto out;
-
-        ret = dict_set_int32 (ctx, "count", count);
+        count++;
+        keylen = sprintf(dict_key, "key%d", count);
+        ret = dict_set_strn(ctx, dict_key, keylen, vme->key);
         if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "Failed to set count "
-                        "in dictionary");
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+                   "Failed to "
+                   "set %s in dictionary",
+                   vme->key);
+            goto out;
         }
+        sprintf(dict_key, "value%d", count);
+        if (get_value_vme) {  // the value was never changed  - DEFAULT is used
+            gf_asprintf(&def_val_str, "%s (DEFAULT)", def_val);
+            ret = dict_set_dynstr_with_alloc(ctx, dict_key, def_val_str);
+            GF_FREE(def_val_str);
+            def_val_str = NULL;
+        } else
+            ret = dict_set_dynstr_with_alloc(ctx, dict_key, def_val);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+                   "Failed to "
+                   "set %s for key %s in dictionary",
+                   def_val, vme->key);
+            goto out;
+        }
+        if (get_value_vme)
+            GF_FREE(def_val);
+
+        def_val = NULL;
+        if (!all_opts)
+            break;
+    }
+    if (!all_opts && !key_found)
+        goto out;
+
+    ret = dict_set_int32n(ctx, "count", SLEN("count"), count);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+               "Failed to set count "
+               "in dictionary");
+    }
 
 out:
-        if (ret && !all_opts && !key_found) {
-                snprintf (err_str, sizeof (err_str),
-                          "option %s does not exist", orig_key);
-                *op_errstr = gf_strdup (err_str);
-        }
-        gf_log (this->name, GF_LOG_DEBUG, "Returning %d", ret);
-        return ret;
+    if (ret && !all_opts && !key_found) {
+        char err_str[PATH_MAX];
+        snprintf(err_str, sizeof(err_str), "option %s does not exist",
+                 orig_key);
+        *op_errstr = gf_strdup(err_str);
+    }
+    if (def_val)
+        GF_FREE(def_val);
+    gf_msg_debug(this->name, 0, "Returning %d", ret);
+    return ret;
 }
 
 int
-glusterd_get_volopt_content (dict_t * ctx, gf_boolean_t xml_out)
-{
-        void                    *dl_handle = NULL;
-        volume_opt_list_t        vol_opt_handle = {{0},};
-        char                    *key = NULL;
-        struct volopt_map_entry *vme = NULL;
-        int                      ret = -1;
-        char                    *def_val = NULL;
-        char                    *descr = NULL;
-        char                     output_string[51200] = {0, };
-        char                    *output = NULL;
-        char                     tmp_str[2048] = {0, };
+glusterd_get_volopt_content(dict_t *ctx, gf_boolean_t xml_out)
+{
+    void *dl_handle = NULL;
+    volume_opt_list_t vol_opt_handle = {
+        {0},
+    };
+    char *key = NULL;
+    struct volopt_map_entry *vme = NULL;
+    int ret = -1;
+    char *def_val = NULL;
+    char *descr = NULL;
+    char *output = NULL;
+    size_t size = 0;
+    size_t used = 0;
 #if (HAVE_LIB_XML)
-        xmlTextWriterPtr         writer = NULL;
-        xmlBufferPtr             buf = NULL;
+    xmlTextWriterPtr writer = NULL;
+    xmlBufferPtr buf = NULL;
 
-        if (xml_out) {
-                ret = init_sethelp_xml_doc (&writer, &buf);
-                if (ret) /*logging done in init_xml_lib*/
-                        goto out;
-        }
+    if (xml_out) {
+        ret = init_sethelp_xml_doc(&writer, &buf);
+        if (ret) /*logging done in init_xml_lib*/
+            goto out;
+    }
 #endif
 
-        CDS_INIT_LIST_HEAD (&vol_opt_handle.list);
+    if (!xml_out) {
+        size = 65536;
+        output = GF_MALLOC(size, gf_common_mt_char);
+        if (output == NULL) {
+            ret = -1;
+            goto out;
+        }
+    }
 
-        for (vme = &glusterd_volopt_map[0]; vme->key; vme++) {
+    CDS_INIT_LIST_HEAD(&vol_opt_handle.list);
 
-                if ((vme->type == NO_DOC) || (vme->type == GLOBAL_NO_DOC))
-                        continue;
+    for (vme = &glusterd_volopt_map[0]; vme->key; vme++) {
+        if ((vme->type == NO_DOC) || (vme->type == GLOBAL_NO_DOC))
+            continue;
 
-                if (vme->description) {
-                        descr = vme->description;
-                        def_val = vme->value;
-                } else {
-                        if (_get_xlator_opt_key_from_vme (vme, &key)) {
-                                gf_log ("glusterd", GF_LOG_DEBUG, "Failed to "
-                                        "get %s key from volume option entry",
-                                        vme->key);
-                                goto out; /*Some error while geting key*/
-                        }
-
-                        ret = xlator_volopt_dynload (vme->voltype,
-                                                     &dl_handle,
-                                                     &vol_opt_handle);
-
-                        if (ret) {
-                                gf_log ("glusterd", GF_LOG_DEBUG,
-                                        "xlator_volopt_dynload error(%d)", ret);
-                                ret = 0;
-                                goto cont;
-                        }
-
-                        ret = xlator_option_info_list (&vol_opt_handle, key,
-                                                       &def_val, &descr);
-                        if (ret) { /*Swallow Error i.e if option not found*/
-                                gf_log ("glusterd", GF_LOG_DEBUG,
-                                        "Failed to get option for %s key", key);
-                                ret = 0;
-                                goto cont;
-                        }
-                }
+        if (vme->description) {
+            descr = vme->description;
+            def_val = vme->value;
+        } else {
+            if (_get_xlator_opt_key_from_vme(vme, &key)) {
+                gf_msg_debug("glusterd", 0,
+                             "Failed to "
+                             "get %s key from volume option entry",
+                             vme->key);
+                goto out; /*Some error while getting key*/
+            }
+
+            ret = xlator_volopt_dynload(vme->voltype, &dl_handle,
+                                        &vol_opt_handle);
+
+            if (ret) {
+                gf_msg_debug("glusterd", 0, "xlator_volopt_dynload error(%d)",
+                             ret);
+                ret = 0;
+                goto cont;
+            }
+
+            ret = xlator_option_info_list(&vol_opt_handle, key, &def_val,
+                                          &descr);
+            if (ret) { /*Swallow Error i.e if option not found*/
+                gf_msg_debug("glusterd", 0, "Failed to get option for %s key",
+                             key);
+                ret = 0;
+                goto cont;
+            }
+        }
 
-                if (xml_out) {
+        if (xml_out) {
 #if (HAVE_LIB_XML)
-                        if (xml_add_volset_element (writer,vme->key,
-                                                    def_val, descr)) {
-                                ret = -1;
-                                goto cont;
-                        }
+            if (xml_add_volset_element(writer, vme->key, def_val, descr)) {
+                ret = -1;
+                goto cont;
+            }
 #else
-                        gf_log ("glusterd", GF_LOG_ERROR, "Libxml not present");
+            gf_msg("glusterd", GF_LOG_ERROR, 0, GD_MSG_MODULE_NOT_INSTALLED,
+                   "Libxml not present");
 #endif
-                } else {
-                        snprintf (tmp_str, sizeof (tmp_str), "Option: %s\nDefault "
-                                        "Value: %s\nDescription: %s\n\n",
-                                        vme->key, def_val, descr);
-                        strcat (output_string, tmp_str);
+        } else {
+            void *tmp;
+            int len;
+
+            do {
+                len = snprintf(output + used, size - used,
+                               "Option: %s\nDefault Value: %s\n"
+                               "Description: %s\n\n",
+                               vme->key, def_val, descr);
+                if (len < 0) {
+                    ret = -1;
+                    goto cont;
                 }
-cont:
-                if (dl_handle) {
-                        dlclose (dl_handle);
-                        dl_handle = NULL;
-                        vol_opt_handle.given_opt = NULL;
+                if (used + len < size) {
+                    used += len;
+                    break;
                 }
-                if (key) {
-                        _free_xlator_opt_key (key);
-                        key = NULL;
+
+                size += (len + 65536) & ~65535;
+                tmp = GF_REALLOC(output, size);
+                if (tmp == NULL) {
+                    ret = -1;
+                    goto cont;
                 }
-                if (ret)
-                        goto out;
+                output = tmp;
+            } while (1);
         }
+    cont:
+        if (dl_handle) {
+            dlclose(dl_handle);
+            dl_handle = NULL;
+            vol_opt_handle.given_opt = NULL;
+        }
+        if (key) {
+            _free_xlator_opt_key(key);
+            key = NULL;
+        }
+        if (ret)
+            goto out;
+    }
 
 #if (HAVE_LIB_XML)
-        if ((xml_out) &&
-            (ret = end_sethelp_xml_doc (writer)))
-                goto out;
+    if ((xml_out) && (ret = end_sethelp_xml_doc(writer)))
+        goto out;
 #else
-        if (xml_out)
-                gf_log ("glusterd", GF_LOG_ERROR, "Libxml not present");
+    if (xml_out)
+        gf_msg("glusterd", GF_LOG_ERROR, 0, GD_MSG_MODULE_NOT_INSTALLED,
+               "Libxml not present");
 #endif
 
-        if (!xml_out)
-                output = gf_strdup (output_string);
-        else
+    if (xml_out) {
 #if (HAVE_LIB_XML)
-                output = gf_strdup ((char *)buf->content);
-#else
-                gf_log ("glusterd", GF_LOG_ERROR, "Libxml not present");
-#endif
-
+        output = gf_strdup((char *)buf->content);
         if (NULL == output) {
-                ret = -1;
-                goto out;
+            ret = -1;
+            goto out;
         }
+#else
+        gf_msg("glusterd", GF_LOG_ERROR, 0, GD_MSG_MODULE_NOT_INSTALLED,
+               "Libxml not present");
+#endif
+    }
 
-        ret = dict_set_dynstr (ctx, "help-str", output);
+    ret = dict_set_dynstrn(ctx, "help-str", SLEN("help-str"), output);
+    if (ret >= 0) {
+        output = NULL;
+    }
 out:
-        gf_log ("glusterd", GF_LOG_DEBUG, "Returning %d", ret);
-        return ret;
+    GF_FREE(output);
+    gf_msg_debug("glusterd", 0, "Returning %d", ret);
+    return ret;
 }
 
 int
-glusterd_check_client_op_version_support (char *volname, uint32_t op_version,
-                                          char **op_errstr)
-{
-        int                     ret = 0;
-        xlator_t                *this = NULL;
-        glusterd_conf_t         *priv = NULL;
-        rpc_transport_t         *xprt = NULL;
+glusterd_check_client_op_version_support(char *volname, uint32_t op_version,
+                                         char **op_errstr)
+{
+    int ret = 0;
+    xlator_t *this = NULL;
+    glusterd_conf_t *priv = NULL;
+    rpc_transport_t *xprt = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+    priv = this->private;
+    GF_ASSERT(priv);
+
+    pthread_mutex_lock(&priv->xprt_lock);
+    list_for_each_entry(xprt, &priv->xprt_list, list)
+    {
+        if ((!strcmp(volname, xprt->peerinfo.volname)) &&
+            ((op_version > xprt->peerinfo.max_op_version) ||
+             (op_version < xprt->peerinfo.min_op_version))) {
+            ret = -1;
+            break;
+        }
+    }
+    pthread_mutex_unlock(&priv->xprt_lock);
+
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_UNSUPPORTED_VERSION,
+               "Client %s is running with min_op_version as %d and "
+               "max_op_version as %d and don't support the required "
+               "op-version %d",
+               xprt->peerinfo.identifier, xprt->peerinfo.min_op_version,
+               xprt->peerinfo.max_op_version, op_version);
+        if (op_errstr)
+            ret = gf_asprintf(op_errstr,
+                              "One of the client %s is "
+                              "running with op-version %d and "
+                              "doesn't support the required "
+                              "op-version %d. This client needs to"
+                              " be upgraded or disconnected "
+                              "before running this command again",
+                              xprt->peerinfo.identifier,
+                              xprt->peerinfo.max_op_version, op_version);
 
-        this = THIS;
-        GF_ASSERT(this);
-        priv = this->private;
-        GF_ASSERT(priv);
+        return -1;
+    }
+    return 0;
+}
 
-        pthread_mutex_lock (&priv->xprt_lock);
-        list_for_each_entry (xprt, &priv->xprt_list, list) {
-                if ((!strcmp(volname, xprt->peerinfo.volname)) &&
-                    ((op_version > xprt->peerinfo.max_op_version) ||
-                     (op_version < xprt->peerinfo.min_op_version))) {
-                        ret = -1;
-                        break;
-                }
-        }
-        pthread_mutex_unlock (&priv->xprt_lock);
+gf_boolean_t
+glusterd_have_peers()
+{
+    xlator_t *this = NULL;
+    glusterd_conf_t *conf = NULL;
 
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "One or more clients "
-                        "don't support the required op-version");
-                if (op_errstr)
-                        ret = gf_asprintf (op_errstr, "One or more connected "
-                                           "clients cannot support the feature "
-                                           "being set. These clients need to be"
-                                           " upgraded or disconnected before "
-                                           "running this command again");
-                return -1;
-        }
-        return 0;
+    this = THIS;
+    GF_ASSERT(this);
+    conf = this->private;
+    GF_ASSERT(conf);
+
+    return !cds_list_empty(&conf->peers);
 }
 
 gf_boolean_t
-glusterd_have_peers ()
+glusterd_is_volume_started(glusterd_volinfo_t *volinfo)
 {
-        xlator_t        *this = NULL;
-        glusterd_conf_t *conf = NULL;
+    GF_ASSERT(volinfo);
+    return (volinfo->status == GLUSTERD_STATUS_STARTED);
+}
 
-        this = THIS;
-        GF_ASSERT (this);
-        conf = this->private;
-        GF_ASSERT (conf);
+int
+glusterd_volume_get_type_str(glusterd_volinfo_t *volinfo, char **voltype_str)
+{
+    int ret = -1;
+    int type = 0;
 
-        return !cds_list_empty (&conf->peers);
+    GF_VALIDATE_OR_GOTO(THIS->name, volinfo, out);
+
+    type = get_vol_type(volinfo->type, volinfo->dist_leaf_count,
+                        volinfo->brick_count);
+
+    *voltype_str = vol_type_str[type];
+
+    ret = 0;
+out:
+    return ret;
 }
 
-gf_boolean_t
-glusterd_is_volume_started (glusterd_volinfo_t  *volinfo)
+int
+glusterd_volume_get_status_str(glusterd_volinfo_t *volinfo, char *status_str)
+{
+    int ret = -1;
+
+    GF_VALIDATE_OR_GOTO(THIS->name, volinfo, out);
+    GF_VALIDATE_OR_GOTO(THIS->name, status_str, out);
+
+    switch (volinfo->status) {
+        case GLUSTERD_STATUS_NONE:
+            sprintf(status_str, "%s", "Created");
+            break;
+        case GLUSTERD_STATUS_STARTED:
+            sprintf(status_str, "%s", "Started");
+            break;
+        case GLUSTERD_STATUS_STOPPED:
+            sprintf(status_str, "%s", "Stopped");
+            break;
+        default:
+            goto out;
+    }
+    ret = 0;
+out:
+    return ret;
+}
+
+void
+glusterd_brick_get_status_str(glusterd_brickinfo_t *brickinfo, char *status_str)
+{
+    GF_VALIDATE_OR_GOTO(THIS->name, brickinfo, out);
+    GF_VALIDATE_OR_GOTO(THIS->name, status_str, out);
+
+    switch (brickinfo->status) {
+        case GF_BRICK_STOPPED:
+            sprintf(status_str, "%s", "Stopped");
+            break;
+        case GF_BRICK_STARTED:
+            sprintf(status_str, "%s", "Started");
+            break;
+        case GF_BRICK_STARTING:
+            sprintf(status_str, "%s", "Starting");
+            break;
+        case GF_BRICK_STOPPING:
+            sprintf(status_str, "%s", "Stopping");
+            break;
+        default:
+            sprintf(status_str, "%s", "None");
+            break;
+    }
+
+out:
+    return;
+}
+
+int
+glusterd_volume_get_transport_type_str(glusterd_volinfo_t *volinfo,
+                                       char *transport_type_str)
+{
+    int ret = -1;
+
+    GF_VALIDATE_OR_GOTO(THIS->name, volinfo, out);
+    GF_VALIDATE_OR_GOTO(THIS->name, transport_type_str, out);
+
+    switch (volinfo->transport_type) {
+        case GF_TRANSPORT_TCP:
+            sprintf(transport_type_str, "%s", "tcp");
+            break;
+        case GF_TRANSPORT_RDMA:
+            sprintf(transport_type_str, "%s", "rdma");
+            break;
+        case GF_TRANSPORT_BOTH_TCP_RDMA:
+            sprintf(transport_type_str, "%s", "tcp_rdma_both");
+            break;
+        default:
+            goto out;
+    }
+    ret = 0;
+out:
+    return ret;
+}
+
+int
+glusterd_volume_get_quorum_status_str(glusterd_volinfo_t *volinfo,
+                                      char *quorum_status_str)
+{
+    int ret = -1;
+
+    GF_VALIDATE_OR_GOTO(THIS->name, volinfo, out);
+    GF_VALIDATE_OR_GOTO(THIS->name, quorum_status_str, out);
+
+    switch (volinfo->quorum_status) {
+        case NOT_APPLICABLE_QUORUM:
+            sprintf(quorum_status_str, "%s", "not_applicable");
+            break;
+        case MEETS_QUORUM:
+            sprintf(quorum_status_str, "%s", "meets");
+            break;
+        case DOESNT_MEET_QUORUM:
+            sprintf(quorum_status_str, "%s", "does_not_meet");
+            break;
+        default:
+            goto out;
+    }
+    ret = 0;
+out:
+    return ret;
+}
+
+int
+glusterd_volume_get_rebalance_status_str(glusterd_volinfo_t *volinfo,
+                                         char *rebal_status_str)
 {
-        GF_ASSERT (volinfo);
-        return (volinfo->status == GLUSTERD_STATUS_STARTED);
+    int ret = -1;
+
+    GF_VALIDATE_OR_GOTO(THIS->name, volinfo, out);
+    GF_VALIDATE_OR_GOTO(THIS->name, rebal_status_str, out);
+
+    switch (volinfo->rebal.defrag_status) {
+        case GF_DEFRAG_STATUS_NOT_STARTED:
+            sprintf(rebal_status_str, "%s", "not_started");
+            break;
+        case GF_DEFRAG_STATUS_STARTED:
+            sprintf(rebal_status_str, "%s", "started");
+            break;
+        case GF_DEFRAG_STATUS_STOPPED:
+            sprintf(rebal_status_str, "%s", "stopped");
+            break;
+        case GF_DEFRAG_STATUS_COMPLETE:
+            sprintf(rebal_status_str, "%s", "completed");
+            break;
+        case GF_DEFRAG_STATUS_FAILED:
+            sprintf(rebal_status_str, "%s", "failed");
+            break;
+        case GF_DEFRAG_STATUS_LAYOUT_FIX_STARTED:
+            sprintf(rebal_status_str, "%s", "layout_fix_started");
+            break;
+        case GF_DEFRAG_STATUS_LAYOUT_FIX_STOPPED:
+            sprintf(rebal_status_str, "%s", "layout_fix_stopped");
+            break;
+        case GF_DEFRAG_STATUS_LAYOUT_FIX_COMPLETE:
+            sprintf(rebal_status_str, "%s", "layout_fix_complete");
+            break;
+        case GF_DEFRAG_STATUS_LAYOUT_FIX_FAILED:
+            sprintf(rebal_status_str, "%s", "layout_fix_failed");
+            break;
+        default:
+            goto out;
+    }
+    ret = 0;
+out:
+    return ret;
 }
 
 /* This function will insert the element to the list in a order.
@@ -9544,35 +14005,1042 @@ glusterd_is_volume_started (glusterd_volinfo_t  *volinfo)
    >0: if first argument is greater than second argument
    <0: if first argument is less than second argument */
 void
-glusterd_list_add_order (struct cds_list_head *new, struct cds_list_head *head,
+glusterd_list_add_order(struct cds_list_head *new, struct cds_list_head *head,
                         int (*compare)(struct cds_list_head *,
                                        struct cds_list_head *))
 {
-        struct cds_list_head *pos = NULL;
+    struct cds_list_head *pos = NULL;
 
-        cds_list_for_each_rcu (pos, head) {
-                if (compare (new, pos) <= 0)
-                        break;
+    cds_list_for_each_rcu(pos, head)
+    {
+        if (compare(new, pos) <= 0)
+            break;
+    }
+
+    cds_list_add_rcu(new, rcu_dereference(pos->prev));
+}
+
+int32_t
+glusterd_count_connected_peers(int32_t *count)
+{
+    glusterd_peerinfo_t *peerinfo = NULL;
+    glusterd_conf_t *conf = NULL;
+    int32_t ret = -1;
+    xlator_t *this = NULL;
+
+    this = THIS;
+    GF_VALIDATE_OR_GOTO("glusterd", this, out);
+    conf = this->private;
+    GF_VALIDATE_OR_GOTO(this->name, conf, out);
+    GF_VALIDATE_OR_GOTO(this->name, count, out);
+
+    *count = 1;
+
+    RCU_READ_LOCK;
+    cds_list_for_each_entry_rcu(peerinfo, &conf->peers, uuid_list)
+    {
+        /* Find peer who is connected and is a friend */
+        if ((peerinfo->connected) &&
+            (peerinfo->state.state == GD_FRIEND_STATE_BEFRIENDED)) {
+            (*count)++;
         }
+    }
+    RCU_READ_UNLOCK;
+
+    ret = 0;
+out:
+    return ret;
+}
+
+char *
+gd_get_shd_key(int type)
+{
+    char *key = NULL;
 
-        cds_list_add_rcu (new, rcu_dereference (pos->prev));
+    switch (type) {
+        case GF_CLUSTER_TYPE_REPLICATE:
+            key = "cluster.self-heal-daemon";
+            break;
+        case GF_CLUSTER_TYPE_DISPERSE:
+            key = "cluster.disperse-self-heal-daemon";
+            break;
+        default:
+            key = NULL;
+            break;
+    }
+    return key;
+}
+
+int
+glusterd_handle_replicate_brick_ops(glusterd_volinfo_t *volinfo,
+                                    glusterd_brickinfo_t *brickinfo,
+                                    glusterd_op_t op)
+{
+    int32_t ret = -1;
+    char tmpmount[] = "/tmp/mntXXXXXX";
+    char logfile[PATH_MAX] = "";
+    int dirty[3] = {
+        0,
+    };
+    runner_t runner = {0};
+    glusterd_conf_t *priv = NULL;
+    char *pid = NULL;
+    char vpath[PATH_MAX] = "";
+    char *volfileserver = NULL;
+
+    xlator_t *this = THIS;
+    GF_ASSERT(this);
+    priv = this->private;
+    GF_VALIDATE_OR_GOTO(this->name, priv, out);
+
+    dirty[2] = hton32(1);
+
+    ret = sys_lsetxattr(brickinfo->path, GF_AFR_DIRTY, dirty, sizeof(dirty), 0);
+    if (ret == -1) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_SET_XATTR_FAIL,
+                "Attribute=%s", GF_AFR_DIRTY, "Reason=%s", strerror(errno),
+                NULL);
+        goto out;
+    }
+
+    if (mkdtemp(tmpmount) == NULL) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_CREATE_DIR_FAILED,
+                NULL);
+        ret = -1;
+        goto out;
+    }
+
+    ret = gf_asprintf(&pid, "%d", GF_CLIENT_PID_ADD_REPLICA_MOUNT);
+    if (ret < 0)
+        goto out;
+
+    switch (op) {
+        case GD_OP_REPLACE_BRICK:
+            if (dict_get_strn(this->options, "transport.socket.bind-address",
+                              SLEN("transport.socket.bind-address"),
+                              &volfileserver) != 0)
+                volfileserver = "localhost";
+
+            snprintf(logfile, sizeof(logfile), "%s/%s-replace-brick-mount.log",
+                     priv->logdir, volinfo->volname);
+            if (!*logfile) {
+                ret = -1;
+                goto out;
+            }
+            runinit(&runner);
+            runner_add_args(&runner, SBIN_DIR "/glusterfs", "-s", volfileserver,
+                            "--volfile-id", volinfo->volname, "--client-pid",
+                            pid, "-l", logfile, tmpmount, NULL);
+            break;
+
+        case GD_OP_ADD_BRICK:
+            snprintf(logfile, sizeof(logfile), "%s/%s-add-brick-mount.log",
+                     priv->logdir, volinfo->volname);
+            if (!*logfile) {
+                ret = -1;
+                goto out;
+            }
+            ret = glusterd_get_dummy_client_filepath(vpath, volinfo,
+                                                     volinfo->transport_type);
+            if (ret) {
+                gf_log("", GF_LOG_ERROR,
+                       "Failed to get "
+                       "volfile path");
+                goto out;
+            }
+            runinit(&runner);
+            runner_add_args(&runner, SBIN_DIR "/glusterfs", "--volfile", vpath,
+                            "--client-pid", pid, "-l", logfile, tmpmount, NULL);
+            break;
+        default:
+            break;
+    }
+    synclock_unlock(&priv->big_lock);
+    ret = runner_run(&runner);
+
+    if (ret) {
+        gf_log(this->name, GF_LOG_ERROR,
+               "mount command"
+               " failed.");
+        goto lock;
+    }
+    ret = sys_lsetxattr(
+        tmpmount,
+        (op == GD_OP_REPLACE_BRICK) ? GF_AFR_REPLACE_BRICK : GF_AFR_ADD_BRICK,
+        brickinfo->brick_id, sizeof(brickinfo->brick_id), 0);
+    if (ret == -1)
+        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_SET_XATTR_FAIL,
+                "Attribute=%s, Reason=%s",
+                (op == GD_OP_REPLACE_BRICK) ? GF_AFR_REPLACE_BRICK
+                                            : GF_AFR_ADD_BRICK,
+                strerror(errno), NULL);
+    gf_umount_lazy(this->name, tmpmount, 1);
+lock:
+    synclock_lock(&priv->big_lock);
+out:
+    if (pid)
+        GF_FREE(pid);
+    gf_msg_debug(this->name, 0, "Returning with ret");
+    return ret;
 }
 
 void
-gd_cleanup_local_xaction_peers_list (struct cds_list_head *xact_peers)
+assign_brick_groups(glusterd_volinfo_t *volinfo)
+{
+    glusterd_brickinfo_t *brickinfo = NULL;
+    uint16_t group_num = 0;
+    int in_group = 0;
+
+    list_for_each_entry(brickinfo, &volinfo->bricks, brick_list)
+    {
+        brickinfo->group = group_num;
+        if (++in_group >= volinfo->replica_count) {
+            in_group = 0;
+            ++group_num;
+        }
+    }
+}
+
+glusterd_brickinfo_t *
+get_last_brick_of_brick_group(glusterd_volinfo_t *volinfo,
+                              glusterd_brickinfo_t *brickinfo)
+{
+    glusterd_brickinfo_t *next = NULL;
+    glusterd_brickinfo_t *last = NULL;
+
+    last = brickinfo;
+    for (;;) {
+        next = list_next(last, &volinfo->bricks, glusterd_brickinfo_t,
+                         brick_list);
+        if (!next || (next->group != brickinfo->group)) {
+            break;
+        }
+        last = next;
+    }
+
+    return last;
+}
+
+int
+glusterd_get_rb_dst_brickinfo(glusterd_volinfo_t *volinfo,
+                              glusterd_brickinfo_t **brickinfo)
+{
+    int32_t ret = -1;
+
+    if (!volinfo || !brickinfo)
+        goto out;
+
+    *brickinfo = volinfo->rep_brick.dst_brick;
+
+    ret = 0;
+
+out:
+    return ret;
+}
+
+int
+rb_update_dstbrick_port(glusterd_brickinfo_t *dst_brickinfo, dict_t *rsp_dict,
+                        dict_t *req_dict)
+{
+    int ret = 0;
+    int dict_ret = 0;
+    int dst_port = 0;
+
+    dict_ret = dict_get_int32n(req_dict, "dst-brick-port",
+                               SLEN("dst-brick-port"), &dst_port);
+    if (!dict_ret)
+        dst_brickinfo->port = dst_port;
+
+    if (gf_is_local_addr(dst_brickinfo->hostname)) {
+        gf_msg("glusterd", GF_LOG_INFO, 0, GD_MSG_BRK_PORT_NO_ADD_INDO,
+               "adding dst-brick port no %d", dst_port);
+
+        if (rsp_dict) {
+            ret = dict_set_int32n(rsp_dict, "dst-brick-port",
+                                  SLEN("dst-brick-port"), dst_brickinfo->port);
+            if (ret) {
+                gf_msg_debug("glusterd", 0,
+                             "Could not set dst-brick port no in rsp dict");
+                goto out;
+            }
+        }
+
+        if (req_dict && !dict_ret) {
+            ret = dict_set_int32n(req_dict, "dst-brick-port",
+                                  SLEN("dst-brick-port"), dst_brickinfo->port);
+            if (ret) {
+                gf_msg_debug("glusterd", 0, "Could not set dst-brick port no");
+                goto out;
+            }
+        }
+    }
+out:
+    return ret;
+}
+
+int
+glusterd_brick_op_prerequisites(dict_t *dict, char **op, glusterd_op_t *gd_op,
+                                char **volname, glusterd_volinfo_t **volinfo,
+                                char **src_brick,
+                                glusterd_brickinfo_t **src_brickinfo,
+                                char *pidfile, char **op_errstr,
+                                dict_t *rsp_dict)
+{
+    int ret = 0;
+    char msg[2048] = "";
+    gsync_status_param_t param = {
+        0,
+    };
+    xlator_t *this = NULL;
+    glusterd_conf_t *priv = NULL;
+    glusterd_volinfo_t *v = NULL;
+    glusterd_brickinfo_t *b = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+
+    priv = this->private;
+    GF_ASSERT(priv);
+
+    ret = dict_get_strn(dict, "operation", SLEN("operation"), op);
+    if (ret) {
+        gf_msg_debug(this->name, 0, "dict get on operation type failed");
+        goto out;
+    }
+
+    *gd_op = gd_cli_to_gd_op(*op);
+    if (*gd_op < 0)
+        goto out;
+
+    ret = dict_get_strn(dict, "volname", SLEN("volname"), volname);
+
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+               "Unable to get volume name");
+        goto out;
+    }
+
+    ret = glusterd_volinfo_find(*volname, volinfo);
+    if (ret) {
+        snprintf(msg, sizeof(msg), "volume: %s does not exist", *volname);
+        *op_errstr = gf_strdup(msg);
+        goto out;
+    }
+
+    if (GLUSTERD_STATUS_STARTED != (*volinfo)->status) {
+        ret = -1;
+        snprintf(msg, sizeof(msg), "volume: %s is not started", *volname);
+        *op_errstr = gf_strdup(msg);
+        goto out;
+    }
+
+    /* If geo-rep is configured, for this volume, it should be stopped. */
+    param.volinfo = *volinfo;
+    ret = glusterd_check_geo_rep_running(&param, op_errstr);
+    if (ret || param.is_active) {
+        ret = -1;
+        goto out;
+    }
+
+    if (glusterd_is_defrag_on(*volinfo)) {
+        snprintf(msg, sizeof(msg),
+                 "Volume name %s rebalance is in "
+                 "progress. Please retry after completion",
+                 *volname);
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_OIP_RETRY_LATER, "%s", msg);
+        *op_errstr = gf_strdup(msg);
+        ret = -1;
+        goto out;
+    }
+
+    if (dict) {
+        if (!glusterd_is_fuse_available()) {
+            gf_msg(this->name, GF_LOG_ERROR, 0,
+                   (*gd_op == GD_OP_REPLACE_BRICK)
+                       ? GD_MSG_RB_CMD_FAIL
+                       : GD_MSG_RESET_BRICK_CMD_FAIL,
+                   "Unable to open /dev/"
+                   "fuse (%s), %s command failed",
+                   strerror(errno), gd_rb_op_to_str(*op));
+            snprintf(msg, sizeof(msg),
+                     "Fuse unavailable\n "
+                     "%s failed",
+                     gd_rb_op_to_str(*op));
+            *op_errstr = gf_strdup(msg);
+            ret = -1;
+            goto out;
+        }
+    }
+
+    ret = dict_get_strn(dict, "src-brick", SLEN("src-brick"), src_brick);
+
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+               "Unable to get src brick");
+        goto out;
+    }
+
+    gf_msg_debug(this->name, 0, "src brick=%s", *src_brick);
+
+    ret = glusterd_volume_brickinfo_get_by_brick(*src_brick, *volinfo,
+                                                 src_brickinfo, _gf_false);
+    if (ret) {
+        snprintf(msg, sizeof(msg),
+                 "brick: %s does not exist in "
+                 "volume: %s",
+                 *src_brick, *volname);
+        gf_smsg(this->name, GF_LOG_ERROR, 0, GD_MSG_BRICK_NOT_FOUND,
+                "Brick=%s, Volume=%s", *src_brick, *volname, NULL);
+        *op_errstr = gf_strdup(msg);
+        goto out;
+    }
+
+    if (gf_is_local_addr((*src_brickinfo)->hostname)) {
+        gf_msg_debug(this->name, 0, "I AM THE SOURCE HOST");
+        if ((*src_brickinfo)->port && rsp_dict) {
+            ret = dict_set_int32n(rsp_dict, "src-brick-port",
+                                  SLEN("src-brick-port"),
+                                  (*src_brickinfo)->port);
+            if (ret) {
+                gf_msg_debug(this->name, 0, "Could not set src-brick-port=%d",
+                             (*src_brickinfo)->port);
+            }
+        }
+
+        v = *volinfo;
+        b = *src_brickinfo;
+        GLUSTERD_GET_BRICK_PIDFILE(pidfile, v, b, priv);
+    }
+
+    ret = 0;
+out:
+    return ret;
+}
+
+int
+glusterd_get_dst_brick_info(char **dst_brick, char *volname, char **op_errstr,
+                            glusterd_brickinfo_t **dst_brickinfo, char **host,
+                            dict_t *dict, char **dup_dstbrick)
+{
+    char *path = NULL;
+    char *c = NULL;
+    char msg[2048] = "";
+    xlator_t *this = NULL;
+    glusterd_conf_t *priv = NULL;
+    int ret = 0;
+
+    this = THIS;
+    GF_ASSERT(this);
+
+    priv = this->private;
+    GF_ASSERT(priv);
+
+    ret = dict_get_strn(dict, "dst-brick", SLEN("dst-brick"), dst_brick);
+
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+               "Unable to get dest brick.");
+        goto out;
+    }
+
+    gf_msg_debug(this->name, 0, "dst brick=%s", *dst_brick);
+
+    if (!glusterd_store_is_valid_brickpath(volname, *dst_brick) ||
+        !glusterd_is_valid_volfpath(volname, *dst_brick)) {
+        snprintf(msg, sizeof(msg),
+                 "brick path %s is too "
+                 "long.",
+                 *dst_brick);
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_BRKPATH_TOO_LONG, "%s", msg);
+        *op_errstr = gf_strdup(msg);
+
+        ret = -1;
+        goto out;
+    }
+
+    *dup_dstbrick = gf_strdup(*dst_brick);
+    if (!*dup_dstbrick) {
+        ret = -1;
+        goto out;
+    }
+
+    /*
+     * IPv4 address contains '.' and ipv6 addresses contains ':'
+     * So finding the last occurrence of ':' to
+     * mark the start of brick path
+     */
+    c = strrchr(*dup_dstbrick, ':');
+    if (c != NULL) {
+        c[0] = '\0';
+        *host = *dup_dstbrick;
+        path = c++;
+    }
+
+    if (!host || !path) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_BAD_FORMAT,
+               "dst brick %s is not of "
+               "form <HOSTNAME>:<export-dir>",
+               *dst_brick);
+        ret = -1;
+        goto out;
+    }
+
+    ret = glusterd_brickinfo_new_from_brick(*dst_brick, dst_brickinfo, _gf_true,
+                                            NULL);
+    if (ret)
+        goto out;
+
+    ret = 0;
+out:
+    return ret;
+}
+
+int
+glusterd_get_volinfo_from_brick(char *brick, glusterd_volinfo_t **volinfo)
+{
+    int ret = -1;
+    xlator_t *this = NULL;
+    glusterd_conf_t *conf = NULL;
+    glusterd_volinfo_t *voliter = NULL;
+    glusterd_brickinfo_t *brickiter = NULL;
+    glusterd_snap_t *snap = NULL;
+
+    this = THIS;
+    GF_VALIDATE_OR_GOTO("glusterd", this, out);
+    conf = this->private;
+    GF_VALIDATE_OR_GOTO(this->name, conf, out);
+
+    /* First check for normal volumes */
+    cds_list_for_each_entry(voliter, &conf->volumes, vol_list)
+    {
+        cds_list_for_each_entry(brickiter, &voliter->bricks, brick_list)
+        {
+            if (gf_uuid_compare(brickiter->uuid, MY_UUID))
+                continue;
+            if (!strcmp(brickiter->path, brick)) {
+                *volinfo = voliter;
+                return 0;
+            }
+        }
+    }
+    /* In case normal volume is not found, check for snapshot volumes */
+    cds_list_for_each_entry(snap, &conf->snapshots, snap_list)
+    {
+        cds_list_for_each_entry(voliter, &snap->volumes, vol_list)
+        {
+            cds_list_for_each_entry(brickiter, &voliter->bricks, brick_list)
+            {
+                if (gf_uuid_compare(brickiter->uuid, MY_UUID))
+                    continue;
+                if (!strcmp(brickiter->path, brick)) {
+                    *volinfo = voliter;
+                    return 0;
+                }
+            }
+        }
+    }
+
+out:
+    return ret;
+}
+
+glusterd_op_t
+gd_cli_to_gd_op(char *cli_op)
 {
-        glusterd_local_peers_t *local_peers = NULL;
-        glusterd_local_peers_t *tmp         = NULL;
+    if (!strcmp(cli_op, "GF_RESET_OP_START") ||
+        !strcmp(cli_op, "GF_RESET_OP_COMMIT") ||
+        !strcmp(cli_op, "GF_RESET_OP_COMMIT_FORCE")) {
+        return GD_OP_RESET_BRICK;
+    }
+
+    if (!strcmp(cli_op, "GF_REPLACE_OP_COMMIT_FORCE"))
+        return GD_OP_REPLACE_BRICK;
+
+    return -1;
+}
 
-        GF_ASSERT (xact_peers);
+char *
+gd_rb_op_to_str(char *op)
+{
+    if (!strcmp(op, "GF_RESET_OP_START"))
+        return "reset-brick start";
+    if (!strcmp(op, "GF_RESET_OP_COMMIT"))
+        return "reset-brick commit";
+    if (!strcmp(op, "GF_RESET_OP_COMMIT_FORCE"))
+        return "reset-brick commit force";
+    if (!strcmp(op, "GF_REPLACE_OP_COMMIT_FORCE"))
+        return "replace-brick commit force";
+    return NULL;
+}
+
+gf_boolean_t
+glusterd_is_profile_on(glusterd_volinfo_t *volinfo)
+{
+    int ret = -1;
+    gf_boolean_t is_latency_on = _gf_false;
+    gf_boolean_t is_fd_stats_on = _gf_false;
 
-        if (cds_list_empty (xact_peers))
-                return;
+    GF_ASSERT(volinfo);
 
-        cds_list_for_each_entry_safe (local_peers, tmp, xact_peers,
-                                      op_peers_list) {
-                GF_FREE (local_peers);
-                /*  local_peers->peerinfo need not be freed because it does not
-                 *  ownership of peerinfo, but merely refer it */
+    ret = glusterd_volinfo_get_boolean(volinfo, VKEY_DIAG_CNT_FOP_HITS);
+    if (ret != -1)
+        is_fd_stats_on = ret;
+    ret = glusterd_volinfo_get_boolean(volinfo, VKEY_DIAG_LAT_MEASUREMENT);
+    if (ret != -1)
+        is_latency_on = ret;
+    if ((_gf_true == is_latency_on) && (_gf_true == is_fd_stats_on))
+        return _gf_true;
+    return _gf_false;
+}
+
+int32_t
+glusterd_add_shd_to_dict(glusterd_volinfo_t *volinfo, dict_t *dict,
+                         int32_t count)
+{
+    int ret = -1;
+    int32_t pid = -1;
+    int32_t brick_online = -1;
+    char key[64] = {0};
+    int keylen;
+    char *pidfile = NULL;
+    xlator_t *this = NULL;
+    char *uuid_str = NULL;
+
+    this = THIS;
+    GF_VALIDATE_OR_GOTO(THIS->name, this, out);
+
+    GF_VALIDATE_OR_GOTO(this->name, volinfo, out);
+    GF_VALIDATE_OR_GOTO(this->name, dict, out);
+
+    keylen = snprintf(key, sizeof(key), "brick%d.hostname", count);
+    ret = dict_set_nstrn(dict, key, keylen, "Self-heal Daemon",
+                         SLEN("Self-heal Daemon"));
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED, "Key=%s",
+                key, NULL);
+        goto out;
+    }
+
+    keylen = snprintf(key, sizeof(key), "brick%d.path", count);
+    uuid_str = gf_strdup(uuid_utoa(MY_UUID));
+    if (!uuid_str) {
+        ret = -1;
+        goto out;
+    }
+    ret = dict_set_dynstrn(dict, key, keylen, uuid_str);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED, "Key=%s",
+                key, NULL);
+        goto out;
+    }
+    uuid_str = NULL;
+
+    /* shd doesn't have a port. but the cli needs a port key with
+     * a zero value to parse.
+     * */
+
+    keylen = snprintf(key, sizeof(key), "brick%d.port", count);
+    ret = dict_set_int32n(dict, key, keylen, 0);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED, "Key=%s",
+                key, NULL);
+        goto out;
+    }
+
+    pidfile = volinfo->shd.svc.proc.pidfile;
+
+    brick_online = gf_is_service_running(pidfile, &pid);
+
+    /* If shd is not running, then don't print the pid */
+    if (!brick_online)
+        pid = -1;
+    keylen = snprintf(key, sizeof(key), "brick%d.pid", count);
+    ret = dict_set_int32n(dict, key, keylen, pid);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED, "Key=%s",
+                key, NULL);
+        goto out;
+    }
+
+    keylen = snprintf(key, sizeof(key), "brick%d.status", count);
+    ret = dict_set_int32n(dict, key, keylen, brick_online);
+
+out:
+    if (uuid_str)
+        GF_FREE(uuid_str);
+    if (ret)
+        gf_msg(this ? this->name : "glusterd", GF_LOG_ERROR, 0,
+               GD_MSG_DICT_SET_FAILED,
+               "Returning %d. adding values to dict failed", ret);
+
+    return ret;
+}
+
+static gf_ai_compare_t
+glusterd_compare_addrinfo(struct addrinfo *first, struct addrinfo *next)
+{
+    int ret = -1;
+    struct addrinfo *tmp1 = NULL;
+    struct addrinfo *tmp2 = NULL;
+    char firstip[NI_MAXHOST] = {0.};
+    char nextip[NI_MAXHOST] = {
+        0,
+    };
+
+    for (tmp1 = first; tmp1 != NULL; tmp1 = tmp1->ai_next) {
+        ret = getnameinfo(tmp1->ai_addr, tmp1->ai_addrlen, firstip, NI_MAXHOST,
+                          NULL, 0, NI_NUMERICHOST);
+        if (ret)
+            return GF_AI_COMPARE_ERROR;
+        for (tmp2 = next; tmp2 != NULL; tmp2 = tmp2->ai_next) {
+            ret = getnameinfo(tmp2->ai_addr, tmp2->ai_addrlen, nextip,
+                              NI_MAXHOST, NULL, 0, NI_NUMERICHOST);
+            if (ret)
+                return GF_AI_COMPARE_ERROR;
+            if (!strcmp(firstip, nextip)) {
+                return GF_AI_COMPARE_MATCH;
+            }
+        }
+    }
+    return GF_AI_COMPARE_NO_MATCH;
+}
+
+/* Check for non optimal brick order for Replicate/Disperse :
+ * Checks if bricks belonging to a replicate or disperse
+ * volume are present on the same server
+ */
+int32_t
+glusterd_check_brick_order(dict_t *dict, char *err_str, int32_t type,
+                           char **volname, char **brick_list,
+                           int32_t *brick_count, int32_t sub_count)
+{
+    int ret = -1;
+    int i = 0;
+    int j = 0;
+    int k = 0;
+    xlator_t *this = NULL;
+    addrinfo_list_t *ai_list = NULL;
+    addrinfo_list_t *ai_list_tmp1 = NULL;
+    addrinfo_list_t *ai_list_tmp2 = NULL;
+    char *brick = NULL;
+    char *brick_list_dup = NULL;
+    char *brick_list_ptr = NULL;
+    char *tmpptr = NULL;
+    struct addrinfo *ai_info = NULL;
+    char brick_addr[128] = {
+        0,
+    };
+    int addrlen = 0;
+
+    const char failed_string[2048] =
+        "Failed to perform brick order "
+        "check. Use 'force' at the end of the command"
+        " if you want to override this behavior. ";
+    const char found_string[2048] =
+        "Multiple bricks of a %s "
+        "volume are present on the same server. This "
+        "setup is not optimal. Bricks should be on "
+        "different nodes to have best fault tolerant "
+        "configuration. Use 'force' at the end of the "
+        "command if you want to override this "
+        "behavior. ";
+
+    this = THIS;
+
+    GF_ASSERT(this);
+
+    ai_list = MALLOC(sizeof(addrinfo_list_t));
+    ai_list->info = NULL;
+    CDS_INIT_LIST_HEAD(&ai_list->list);
+
+    if (!(*volname)) {
+        ret = dict_get_strn(dict, "volname", SLEN("volname"), &(*volname));
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+                   "Unable to get volume name");
+            goto out;
+        }
+    }
+
+    if (!(*brick_list)) {
+        ret = dict_get_strn(dict, "bricks", SLEN("bricks"), &(*brick_list));
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+                   "Bricks check : Could not "
+                   "retrieve bricks list");
+            goto out;
         }
+    }
+
+    if (!(*brick_count)) {
+        ret = dict_get_int32n(dict, "count", SLEN("count"), &(*brick_count));
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+                   "Bricks check : Could not "
+                   "retrieve brick count");
+            goto out;
+        }
+    }
+
+    brick_list_dup = brick_list_ptr = gf_strdup(*brick_list);
+    /* Resolve hostnames and get addrinfo */
+    while (i < *brick_count) {
+        ++i;
+        brick = strtok_r(brick_list_dup, " \n", &tmpptr);
+        brick_list_dup = tmpptr;
+        if (brick == NULL)
+            goto check_failed;
+        tmpptr = strrchr(brick, ':');
+        if (tmpptr == NULL)
+            goto check_failed;
+        addrlen = strlen(brick) - strlen(tmpptr);
+        strncpy(brick_addr, brick, addrlen);
+        brick_addr[addrlen] = '\0';
+        ret = getaddrinfo(brick_addr, NULL, NULL, &ai_info);
+        if (ret != 0) {
+            ret = 0;
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_HOSTNAME_RESOLVE_FAIL,
+                   "unable to resolve host name for addr %s", brick_addr);
+            goto out;
+        }
+        ai_list_tmp1 = MALLOC(sizeof(addrinfo_list_t));
+        if (ai_list_tmp1 == NULL) {
+            ret = 0;
+            gf_msg(this->name, GF_LOG_ERROR, ENOMEM, GD_MSG_NO_MEMORY,
+                   "failed to allocate "
+                   "memory");
+            freeaddrinfo(ai_info);
+            goto out;
+        }
+        ai_list_tmp1->info = ai_info;
+        cds_list_add_tail(&ai_list_tmp1->list, &ai_list->list);
+        ai_list_tmp1 = NULL;
+    }
+
+    i = 0;
+    ai_list_tmp1 = cds_list_entry(ai_list->list.next, addrinfo_list_t, list);
+
+    if (*brick_count < sub_count) {
+        sub_count = *brick_count;
+    }
+
+    /* Check for bad brick order */
+    while (i < *brick_count) {
+        ++i;
+        ai_info = ai_list_tmp1->info;
+        ai_list_tmp1 = cds_list_entry(ai_list_tmp1->list.next, addrinfo_list_t,
+                                      list);
+        if (0 == i % sub_count) {
+            j = 0;
+            continue;
+        }
+        ai_list_tmp2 = ai_list_tmp1;
+        k = j;
+        while (k < sub_count - 1) {
+            ++k;
+            ret = glusterd_compare_addrinfo(ai_info, ai_list_tmp2->info);
+            if (GF_AI_COMPARE_ERROR == ret)
+                goto check_failed;
+            if (GF_AI_COMPARE_MATCH == ret)
+                goto found_bad_brick_order;
+            ai_list_tmp2 = cds_list_entry(ai_list_tmp2->list.next,
+                                          addrinfo_list_t, list);
+        }
+        ++j;
+    }
+    gf_msg_debug(this->name, 0, "Brick order okay");
+    ret = 0;
+    goto out;
+
+check_failed:
+    gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_BAD_BRKORDER_CHECK_FAIL,
+           "Failed bad brick order check");
+    snprintf(err_str, sizeof(failed_string), failed_string);
+    ret = -1;
+    goto out;
+
+found_bad_brick_order:
+    gf_msg(this->name, GF_LOG_INFO, 0, GD_MSG_BAD_BRKORDER,
+           "Bad brick order found");
+    if (type == GF_CLUSTER_TYPE_DISPERSE) {
+        snprintf(err_str, sizeof(found_string), found_string, "disperse");
+    } else {
+        snprintf(err_str, sizeof(found_string), found_string, "replicate");
+    }
+
+    ret = -1;
+out:
+    ai_list_tmp2 = NULL;
+    GF_FREE(brick_list_ptr);
+    cds_list_for_each_entry(ai_list_tmp1, &ai_list->list, list)
+    {
+        if (ai_list_tmp1->info)
+            freeaddrinfo(ai_list_tmp1->info);
+        free(ai_list_tmp2);
+        ai_list_tmp2 = ai_list_tmp1;
+    }
+    free(ai_list);
+    free(ai_list_tmp2);
+    gf_msg_debug("glusterd", 0, "Returning %d", ret);
+    return ret;
+}
+
+static gf_boolean_t
+search_peer_in_auth_list(char *peer_hostname, char *auth_allow_list)
+{
+    if (strstr(auth_allow_list, peer_hostname)) {
+        return _gf_true;
+    }
+
+    return _gf_false;
+}
+
+/* glusterd_add_peers_to_auth_list() adds peers into auth.allow list
+ * if auth.allow list is not empty. This is called for add-brick and
+ * replica brick operations to avoid failing the temporary mount. New
+ * volfiles will be generated and clients are notified reg new volfiles.
+ */
+void
+glusterd_add_peers_to_auth_list(char *volname)
+{
+    int ret = 0;
+    glusterd_volinfo_t *volinfo = NULL;
+    glusterd_peerinfo_t *peerinfo = NULL;
+    xlator_t *this = NULL;
+    glusterd_conf_t *conf = NULL;
+    int32_t len = 0;
+    char *auth_allow_list = NULL;
+    char *new_auth_allow_list = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+    conf = this->private;
+    GF_ASSERT(conf);
+
+    GF_VALIDATE_OR_GOTO(this->name, volname, out);
+
+    ret = glusterd_volinfo_find(volname, &volinfo);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_VOL_NOT_FOUND,
+               "Unable to find volume: %s", volname);
+        goto out;
+    }
+
+    ret = dict_get_str_sizen(volinfo->dict, "auth.allow", &auth_allow_list);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_INFO, errno, GD_MSG_DICT_GET_FAILED,
+               "auth allow list is not set");
+        goto out;
+    }
+    cds_list_for_each_entry_rcu(peerinfo, &conf->peers, uuid_list)
+    {
+        len += strlen(peerinfo->hostname);
+    }
+    len += strlen(auth_allow_list) + 1;
+
+    new_auth_allow_list = GF_CALLOC(1, len, gf_common_mt_char);
+
+    new_auth_allow_list = strncat(new_auth_allow_list, auth_allow_list,
+                                  strlen(auth_allow_list));
+    cds_list_for_each_entry_rcu(peerinfo, &conf->peers, uuid_list)
+    {
+        ret = search_peer_in_auth_list(peerinfo->hostname, new_auth_allow_list);
+        if (!ret) {
+            gf_log(this->name, GF_LOG_DEBUG,
+                   "peer %s not found in auth.allow list", peerinfo->hostname);
+            new_auth_allow_list = strcat(new_auth_allow_list, ",");
+            new_auth_allow_list = strncat(new_auth_allow_list,
+                                          peerinfo->hostname,
+                                          strlen(peerinfo->hostname));
+        }
+    }
+    if (strcmp(new_auth_allow_list, auth_allow_list) != 0) {
+        /* In case, new_auth_allow_list is not same as auth_allow_list,
+         * we need to update the volinfo->dict with new_auth_allow_list.
+         * we delete the auth_allow_list and replace it with
+         * new_auth_allow_list. for reverting the changes in post commit, we
+         * keep the copy of auth_allow_list as old_auth_allow_list in
+         * volinfo->dict.
+         */
+        dict_del_sizen(volinfo->dict, "auth.allow");
+        ret = dict_set_strn(volinfo->dict, "auth.allow", SLEN("auth.allow"),
+                            new_auth_allow_list);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_SET_FAILED,
+                   "Unable to set new auth.allow list");
+            goto out;
+        }
+        ret = dict_set_strn(volinfo->dict, "old.auth.allow",
+                            SLEN("old.auth.allow"), auth_allow_list);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_SET_FAILED,
+                   "Unable to set old auth.allow list");
+            goto out;
+        }
+        ret = glusterd_create_volfiles_and_notify_services(volinfo);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_WARNING, 0, GD_MSG_VOLFILE_CREATE_FAIL,
+                   "failed to create volfiles");
+            goto out;
+        }
+    }
+out:
+    GF_FREE(new_auth_allow_list);
+    return;
+}
+
+int
+glusterd_replace_old_auth_allow_list(char *volname)
+{
+    int ret = 0;
+    glusterd_volinfo_t *volinfo = NULL;
+    xlator_t *this = NULL;
+    char *old_auth_allow_list = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+
+    GF_VALIDATE_OR_GOTO(this->name, volname, out);
+
+    ret = glusterd_volinfo_find(volname, &volinfo);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_VOL_NOT_FOUND,
+               "Unable to find volume: %s", volname);
+        goto out;
+    }
+
+    ret = dict_get_str_sizen(volinfo->dict, "old.auth.allow",
+                             &old_auth_allow_list);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_INFO, errno, GD_MSG_DICT_GET_FAILED,
+               "old auth allow list is not set, no need to replace the list");
+        ret = 0;
+        goto out;
+    }
+
+    dict_del_sizen(volinfo->dict, "auth.allow");
+    ret = dict_set_strn(volinfo->dict, "auth.allow", SLEN("auth.allow"),
+                        old_auth_allow_list);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_SET_FAILED,
+               "Unable to replace auth.allow list");
+        goto out;
+    }
+
+    dict_del_sizen(volinfo->dict, "old.auth.allow");
+
+    ret = glusterd_create_volfiles_and_notify_services(volinfo);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_WARNING, 0, GD_MSG_VOLFILE_CREATE_FAIL,
+               "failed to create volfiles");
+        goto out;
+    }
+    ret = glusterd_store_volinfo(volinfo, GLUSTERD_VOLINFO_VER_AC_INCREMENT);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_WARNING, 0, GD_MSG_VOLINFO_STORE_FAIL,
+               "failed to store volinfo");
+        goto out;
+    }
+out:
+    return ret;
 }
diff --git a/xlators/mgmt/glusterd/src/glusterd-utils.h b/xlators/mgmt/glusterd/src/glusterd-utils.h
index 80c7c19d508..bf6ac295e26 100644
--- a/xlators/mgmt/glusterd/src/glusterd-utils.h
+++ b/xlators/mgmt/glusterd/src/glusterd-utils.h
@@ -10,20 +10,14 @@
 #ifndef _GLUSTERD_UTILS_H
 #define _GLUSTERD_UTILS_H
 
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
-
 #include <pthread.h>
-#include "compat-uuid.h"
-
-#include "glusterfs.h"
-#include "xlator.h"
-#include "logging.h"
-#include "call-stub.h"
-#include "fd.h"
-#include "byte-order.h"
+#include <glusterfs/compat-uuid.h>
+
+#include <glusterfs/glusterfs.h>
+#include <glusterfs/xlator.h>
+#include <glusterfs/logging.h>
+#include <glusterfs/call-stub.h>
+#include <glusterfs/byte-order.h>
 #include "glusterd.h"
 #include "rpc-clnt.h"
 #include "protocol-common.h"
@@ -32,646 +26,840 @@
 #include "glusterd-peer-utils.h"
 
 #define GLUSTERD_SOCK_DIR "/var/run/gluster"
-#define GLUSTERD_ASSIGN_BRICKID_TO_BRICKINFO(brickinfo, volinfo, brickid) do {\
-        sprintf (brickinfo->brick_id, "%s-client-%d",\
-                 volinfo->volname, brickid);\
-} while (0)
-
-#define list_for_each_local_xaction_peers(xact_peer, xact_peers_head)      \
-        glusterd_local_peers_t *pos = NULL;                                \
-        for (pos = cds_list_entry ((xact_peers_head)->next,                \
-             glusterd_local_peers_t, op_peers_list),                       \
-             xact_peer = pos->peerinfo;                                    \
-             &pos->op_peers_list != (xact_peers_head);                     \
-             pos = cds_list_entry(pos->op_peers_list.next,                 \
-                                  glusterd_local_peers_t,  op_peers_list), \
-             xact_peer = pos->peerinfo)
+#define GLUSTERD_ASSIGN_BRICKID_TO_BRICKINFO(brickinfo, volinfo, brickid)      \
+    do {                                                                       \
+        sprintf(brickinfo->brick_id, "%s-client-%d", volinfo->volname,         \
+                brickid);                                                      \
+    } while (0)
+
+#define GLUSTERD_ASSIGN_BRICKID_TO_TA_BRICKINFO(ta_brickinfo, volinfo,         \
+                                                brickid)                       \
+    do {                                                                       \
+        sprintf(ta_brickinfo->brick_id, "%s-ta-%d", volinfo->volname,          \
+                brickid);                                                      \
+    } while (0)
+
+#define ALL_VOLUME_OPTION_CHECK(volname, get_opt, key, ret, op_errstr, label)  \
+    do {                                                                       \
+        gf_boolean_t _all = !strcmp("all", volname);                           \
+        gf_boolean_t _key_all = !strcmp(key, "all");                           \
+        gf_boolean_t _is_valid_opt = _gf_false;                                \
+        int32_t i = 0;                                                         \
+                                                                               \
+        if (!get_opt &&                                                        \
+            (_key_all || !strcmp(key, GLUSTERD_MAX_OP_VERSION_KEY))) {         \
+            ret = -1;                                                          \
+            *op_errstr = gf_strdup("Not a valid option to set");               \
+            goto out;                                                          \
+        }                                                                      \
+        if (_key_all) {                                                        \
+            _is_valid_opt = _gf_true;                                          \
+        } else {                                                               \
+            for (i = 0; valid_all_vol_opts[i].option; i++) {                   \
+                if (!strcmp(key, valid_all_vol_opts[i].option)) {              \
+                    _is_valid_opt = _gf_true;                                  \
+                    break;                                                     \
+                }                                                              \
+            }                                                                  \
+        }                                                                      \
+        if (_all && !_is_valid_opt) {                                          \
+            ret = -1;                                                          \
+            *op_errstr = gf_strdup("Not a valid option for all volumes");      \
+            goto label;                                                        \
+        } else if (!_all && _is_valid_opt) {                                   \
+            ret = -1;                                                          \
+            *op_errstr = gf_strdup("Not a valid option for single volume");    \
+            goto label;                                                        \
+        }                                                                      \
+    } while (0)
 
 struct glusterd_lock_ {
-        uuid_t  owner;
-        time_t  timestamp;
+    uuid_t owner;
+    time_t timestamp;
 };
 
 typedef struct glusterd_dict_ctx_ {
-        dict_t  *dict;
-        int     opt_count;
-        char    *key_name;
-        char    *val_name;
-        char    *prefix;
+    dict_t *dict;
+    int opt_count;
+    char *key_name;
+    char *val_name;
+    char *prefix;
 } glusterd_dict_ctx_t;
 
+gf_boolean_t
+is_brick_mx_enabled(void);
+
 int
-glusterd_compare_lines (const void *a, const void *b);
+glusterd_compare_lines(const void *a, const void *b);
 
-typedef int (*glusterd_condition_func) (glusterd_volinfo_t *volinfo,
-                                        glusterd_brickinfo_t *brickinfo,
-                                        void *ctx);
+typedef int (*glusterd_condition_func)(glusterd_volinfo_t *volinfo,
+                                       glusterd_brickinfo_t *brickinfo,
+                                       void *ctx);
 typedef struct glusterd_lock_ glusterd_lock_t;
 
 int32_t
-glusterd_get_lock_owner (uuid_t *cur_owner);
+glusterd_get_lock_owner(uuid_t *cur_owner);
 
 int32_t
-glusterd_lock (uuid_t new_owner);
+glusterd_lock(uuid_t new_owner);
 
 int32_t
-glusterd_unlock (uuid_t owner);
+glusterd_unlock(uuid_t owner);
 
 int32_t
-glusterd_get_uuid (uuid_t *uuid);
+glusterd_get_uuid(uuid_t *uuid);
+
+char *
+gd_get_shd_key(int type);
 
 int
-glusterd_submit_reply (rpcsvc_request_t *req, void *arg,
-                       struct iovec *payload, int payloadcount,
-                       struct iobref *iobref, xdrproc_t xdrproc);
+glusterd_submit_reply(rpcsvc_request_t *req, void *arg, struct iovec *payload,
+                      int payloadcount, struct iobref *iobref,
+                      xdrproc_t xdrproc);
 
 int
-glusterd_to_cli (rpcsvc_request_t *req, gf_cli_rsp *arg, struct iovec *payload,
-                 int payloadcount, struct iobref *iobref, xdrproc_t xdrproc,
-                 dict_t *dict);
+glusterd_to_cli(rpcsvc_request_t *req, gf_cli_rsp *arg, struct iovec *payload,
+                int payloadcount, struct iobref *iobref, xdrproc_t xdrproc,
+                dict_t *dict);
 
 int
-glusterd_submit_request (struct rpc_clnt *rpc, void *req,
-                         call_frame_t *frame, rpc_clnt_prog_t *prog,
-                         int procnum, struct iobref *iobref,
-                         xlator_t *this, fop_cbk_fn_t cbkfn, xdrproc_t xdrproc);
+glusterd_submit_request(struct rpc_clnt *rpc, void *req, call_frame_t *frame,
+                        rpc_clnt_prog_t *prog, int procnum,
+                        struct iobref *iobref, xlator_t *this,
+                        fop_cbk_fn_t cbkfn, xdrproc_t xdrproc);
 int32_t
-glusterd_volinfo_new (glusterd_volinfo_t **volinfo);
+glusterd_volinfo_new(glusterd_volinfo_t **volinfo);
 
 int32_t
-glusterd_volinfo_dup (glusterd_volinfo_t *volinfo,
-                      glusterd_volinfo_t **dup_volinfo,
-                      gf_boolean_t set_userauth);
+glusterd_volinfo_dup(glusterd_volinfo_t *volinfo,
+                     glusterd_volinfo_t **dup_volinfo,
+                     gf_boolean_t set_userauth);
 
 char *
-glusterd_auth_get_username (glusterd_volinfo_t *volinfo);
+glusterd_auth_get_username(glusterd_volinfo_t *volinfo);
 
 char *
-glusterd_auth_get_password (glusterd_volinfo_t *volinfo);
+glusterd_auth_get_password(glusterd_volinfo_t *volinfo);
 
 int32_t
-glusterd_auth_set_username (glusterd_volinfo_t *volinfo, char *username);
+glusterd_auth_set_username(glusterd_volinfo_t *volinfo, char *username);
 
 int32_t
-glusterd_auth_set_password (glusterd_volinfo_t *volinfo, char *password);
+glusterd_auth_set_password(glusterd_volinfo_t *volinfo, char *password);
 
 void
-glusterd_auth_cleanup (glusterd_volinfo_t *volinfo);
+glusterd_auth_cleanup(glusterd_volinfo_t *volinfo);
 
-gf_boolean_t
-glusterd_check_volume_exists (char *volname);
+int32_t
+glusterd_brickprocess_new(glusterd_brick_proc_t **brickprocess);
 
 int32_t
-glusterd_brickinfo_new (glusterd_brickinfo_t **brickinfo);
+glusterd_brickinfo_new(glusterd_brickinfo_t **brickinfo);
 
 int32_t
-glusterd_brickinfo_new_from_brick (char *brick, glusterd_brickinfo_t **brickinfo);
+glusterd_brickinfo_new_from_brick(char *brick, glusterd_brickinfo_t **brickinfo,
+                                  gf_boolean_t construct_real_path,
+                                  char **op_errstr);
 
 int32_t
-glusterd_volinfo_find (char *volname, glusterd_volinfo_t **volinfo);
+glusterd_volinfo_find(const char *volname, glusterd_volinfo_t **volinfo);
+
+gf_boolean_t
+glusterd_volume_exists(const char *volname);
 
 int
-glusterd_volinfo_find_by_volume_id (uuid_t volume_id, glusterd_volinfo_t **volinfo);
+glusterd_volinfo_find_by_volume_id(uuid_t volume_id,
+                                   glusterd_volinfo_t **volinfo);
 
 int32_t
 glusterd_service_stop(const char *service, char *pidfile, int sig,
                       gf_boolean_t force_kill);
 
+int32_t
+glusterd_service_stop_nolock(const char *service, char *pidfile, int sig,
+                             gf_boolean_t force_kill);
+
 int
-glusterd_get_next_available_brickid (glusterd_volinfo_t *volinfo);
+glusterd_get_next_available_brickid(glusterd_volinfo_t *volinfo);
 
 int32_t
-glusterd_resolve_brick (glusterd_brickinfo_t *brickinfo);
+glusterd_resolve_brick(glusterd_brickinfo_t *brickinfo);
+
+int
+glusterd_brick_process_add_brick(glusterd_brickinfo_t *brickinfo,
+                                 glusterd_brickinfo_t *parent_brickinfo);
+
+int
+glusterd_brick_process_remove_brick(glusterd_brickinfo_t *brickinfo,
+                                    int *last_brick);
+
+int
+glusterd_brick_proc_for_port(int port, glusterd_brick_proc_t **brickprocess);
 
 int32_t
-glusterd_volume_start_glusterfs (glusterd_volinfo_t  *volinfo,
-                                 glusterd_brickinfo_t   *brickinfo,
-                                 gf_boolean_t wait);
+glusterd_volume_start_glusterfs(glusterd_volinfo_t *volinfo,
+                                glusterd_brickinfo_t *brickinfo,
+                                gf_boolean_t wait);
 
 int32_t
-glusterd_volume_stop_glusterfs (glusterd_volinfo_t  *volinfo,
-                                glusterd_brickinfo_t   *brickinfo,
-                                gf_boolean_t del_brick);
+glusterd_volume_stop_glusterfs(glusterd_volinfo_t *volinfo,
+                               glusterd_brickinfo_t *brickinfo,
+                               gf_boolean_t del_brick);
+
+int
+send_attach_req(xlator_t *this, struct rpc_clnt *rpc, char *path,
+                glusterd_brickinfo_t *brick, glusterd_brickinfo_t *other_brick,
+                int op);
 
 glusterd_volinfo_t *
-glusterd_volinfo_ref (glusterd_volinfo_t *volinfo);
+glusterd_volinfo_ref(glusterd_volinfo_t *volinfo);
 
 glusterd_volinfo_t *
-glusterd_volinfo_unref (glusterd_volinfo_t *volinfo);
+glusterd_volinfo_unref(glusterd_volinfo_t *volinfo);
 
 int32_t
-glusterd_volinfo_delete (glusterd_volinfo_t *volinfo);
+glusterd_volinfo_delete(glusterd_volinfo_t *volinfo);
 
 int32_t
-glusterd_brickinfo_delete (glusterd_brickinfo_t *brickinfo);
+glusterd_brickinfo_delete(glusterd_brickinfo_t *brickinfo);
 
 gf_boolean_t
-glusterd_is_cli_op_req (int32_t op);
+glusterd_is_cli_op_req(int32_t op);
 
 int32_t
-glusterd_volume_brickinfo_get_by_brick (char *brick,
-                                        glusterd_volinfo_t *volinfo,
-                                        glusterd_brickinfo_t **brickinfo);
+glusterd_volume_brickinfo_get_by_brick(char *brick, glusterd_volinfo_t *volinfo,
+                                       glusterd_brickinfo_t **brickinfo,
+                                       gf_boolean_t construct_real_path);
 
 int32_t
-glusterd_add_volumes_to_export_dict (dict_t **peer_data);
+glusterd_add_volumes_to_export_dict(dict_t *peer_data, char **buf,
+                                    u_int *length);
 
 int32_t
-glusterd_compare_friend_data (dict_t *peer_data, int32_t *status,
-                              char *hostname);
+glusterd_compare_friend_data(dict_t *peer_data, int32_t *status,
+                             char *hostname);
 
 int
-glusterd_compute_cksum (glusterd_volinfo_t  *volinfo,
-                        gf_boolean_t is_quota_conf);
+glusterd_compute_cksum(glusterd_volinfo_t *volinfo, gf_boolean_t is_quota_conf);
 
 void
-glusterd_set_socket_filepath (char *sock_filepath, char *sockpath, size_t len);
+glusterd_set_socket_filepath(char *sock_filepath, char *sockpath, size_t len);
+
+struct rpc_clnt *
+glusterd_pending_node_get_rpc(glusterd_pending_node_t *pending_node);
 
-struct rpc_clnt*
-glusterd_pending_node_get_rpc (glusterd_pending_node_t *pending_node);
+void
+glusterd_pending_node_put_rpc(glusterd_pending_node_t *pending_node);
 
 int
-glusterd_remote_hostname_get (rpcsvc_request_t *req,
-                              char *remote_host, int len);
+glusterd_remote_hostname_get(rpcsvc_request_t *req, char *remote_host, int len);
+
+int32_t
+glusterd_import_friend_volumes_synctask(void *opaque);
+
 int32_t
-glusterd_import_friend_volumes (dict_t *peer_data);
+glusterd_import_friend_volumes(dict_t *peer_data);
 void
-glusterd_set_volume_status (glusterd_volinfo_t  *volinfo,
-                            glusterd_volume_status status);
+glusterd_set_volume_status(glusterd_volinfo_t *volinfo,
+                           glusterd_volume_status status);
 
 int32_t
-glusterd_volume_count_get (void);
+glusterd_volume_count_get(void);
 int32_t
-glusterd_add_volume_to_dict (glusterd_volinfo_t *volinfo,
-                             dict_t  *dict, int32_t count,
-                             char *prefix);
+glusterd_add_volume_to_dict(glusterd_volinfo_t *volinfo, dict_t *dict,
+                            int32_t count, char *prefix);
 int
-glusterd_get_brickinfo (xlator_t *this, const char *brickname,
-                        int port, gf_boolean_t localhost,
-                        glusterd_brickinfo_t **brickinfo);
+glusterd_get_brickinfo(xlator_t *this, const char *brickname, int port,
+                       glusterd_brickinfo_t **brickinfo);
 
 void
-glusterd_set_brick_status (glusterd_brickinfo_t  *brickinfo,
-                            gf_brick_status_t status);
+glusterd_set_brick_status(glusterd_brickinfo_t *brickinfo,
+                          gf_brick_status_t status);
 
 gf_boolean_t
-glusterd_is_brick_started (glusterd_brickinfo_t  *brickinfo);
+glusterd_is_brick_started(glusterd_brickinfo_t *brickinfo);
 
 int
-glusterd_friend_brick_belongs (glusterd_volinfo_t *volinfo,
-                               glusterd_brickinfo_t *brickinfo, void *uuid);
+glusterd_friend_brick_belongs(glusterd_volinfo_t *volinfo,
+                              glusterd_brickinfo_t *brickinfo, void *uuid);
 int
-glusterd_all_volume_cond_check (glusterd_condition_func func, int status,
-                                void *ctx);
+glusterd_all_volume_cond_check(glusterd_condition_func func, int status,
+                               void *ctx);
 int
-glusterd_brick_start (glusterd_volinfo_t *volinfo,
-                      glusterd_brickinfo_t *brickinfo,
-                      gf_boolean_t wait);
+glusterd_brick_start(glusterd_volinfo_t *volinfo,
+                     glusterd_brickinfo_t *brickinfo, gf_boolean_t wait,
+                     gf_boolean_t only_connect);
 int
-glusterd_brick_stop (glusterd_volinfo_t *volinfo,
-                     glusterd_brickinfo_t *brickinfo,
-                     gf_boolean_t del_brick);
+glusterd_brick_stop(glusterd_volinfo_t *volinfo,
+                    glusterd_brickinfo_t *brickinfo, gf_boolean_t del_brick);
 
 int
-glusterd_is_defrag_on (glusterd_volinfo_t *volinfo);
+glusterd_is_defrag_on(glusterd_volinfo_t *volinfo);
 
 int32_t
-glusterd_volinfo_bricks_delete (glusterd_volinfo_t *volinfo);
+glusterd_volinfo_bricks_delete(glusterd_volinfo_t *volinfo);
 
 int
-glusterd_new_brick_validate (char *brick, glusterd_brickinfo_t *brickinfo,
-                             char *op_errstr, size_t len);
+glusterd_new_brick_validate(char *brick, glusterd_brickinfo_t *brickinfo,
+                            char *op_errstr, size_t len, char *op);
 int32_t
-glusterd_volume_brickinfos_delete (glusterd_volinfo_t *volinfo);
+glusterd_volume_brickinfos_delete(glusterd_volinfo_t *volinfo);
 
 int32_t
-glusterd_volume_brickinfo_get (uuid_t uuid, char *hostname, char *path,
-                               glusterd_volinfo_t *volinfo,
-                               glusterd_brickinfo_t **brickinfo);
+glusterd_volume_brickinfo_get(uuid_t uuid, char *hostname, char *path,
+                              glusterd_volinfo_t *volinfo,
+                              glusterd_brickinfo_t **brickinfo);
 
 int
-glusterd_brickinfo_get (uuid_t uuid, char *hostname, char *path,
+glusterd_brickinfo_get(uuid_t uuid, char *hostname, char *path,
                        glusterd_brickinfo_t **brickinfo);
-int
-glusterd_is_rb_started (glusterd_volinfo_t *volinfo);
 
 int
-glusterd_is_rb_paused (glusterd_volinfo_t *volinfo);
+glusterd_rb_check_bricks(glusterd_volinfo_t *volinfo,
+                         glusterd_brickinfo_t *src_brick,
+                         glusterd_brickinfo_t *dst_brick);
 
 int
-glusterd_set_rb_status (glusterd_volinfo_t *volinfo, gf_rb_status_t status);
-
-gf_boolean_t
-glusterd_is_rb_ongoing (glusterd_volinfo_t *volinfo);
+glusterd_check_and_set_brick_xattr(char *host, char *path, uuid_t uuid,
+                                   char **op_errstr, gf_boolean_t is_force);
 
 int
-glusterd_rb_check_bricks (glusterd_volinfo_t *volinfo,
-                          glusterd_brickinfo_t *src_brick,
-                          glusterd_brickinfo_t *dst_brick);
-
+glusterd_validate_and_create_brickpath(glusterd_brickinfo_t *brickinfo,
+                                       uuid_t volume_id, char *volname,
+                                       char **op_errstr, gf_boolean_t is_force,
+                                       gf_boolean_t ignore_partition);
 int
-glusterd_check_and_set_brick_xattr (char *host, char *path, uuid_t uuid,
-                                    char **op_errstr, gf_boolean_t is_force);
-
+glusterd_sm_tr_log_transition_add(glusterd_sm_tr_log_t *log, int old_state,
+                                  int new_state, int event);
 int
-glusterd_validate_and_create_brickpath (glusterd_brickinfo_t *brickinfo,
-                                        uuid_t volume_id, char **op_errstr,
-                                        gf_boolean_t is_force);
-int
-glusterd_sm_tr_log_transition_add (glusterd_sm_tr_log_t *log,
-                                           int old_state, int new_state,
-                                           int event);
-int
-glusterd_sm_tr_log_init (glusterd_sm_tr_log_t *log,
-                         char * (*state_name_get) (int),
-                         char * (*event_name_get) (int),
-                         size_t  size);
+glusterd_sm_tr_log_init(glusterd_sm_tr_log_t *log, char *(*state_name_get)(int),
+                        char *(*event_name_get)(int), size_t size);
 void
-glusterd_sm_tr_log_delete (glusterd_sm_tr_log_t *log);
+glusterd_sm_tr_log_delete(glusterd_sm_tr_log_t *log);
 
 int
-glusterd_sm_tr_log_add_to_dict (dict_t *dict,
-                                glusterd_sm_tr_log_t *circular_log);
+glusterd_sm_tr_log_add_to_dict(dict_t *dict,
+                               glusterd_sm_tr_log_t *circular_log);
 int
-glusterd_remove_pending_entry (struct cds_list_head *list, void *elem);
+glusterd_remove_pending_entry(struct cds_list_head *list, void *elem);
 int
-glusterd_clear_pending_nodes (struct cds_list_head *list);
+glusterd_clear_pending_nodes(struct cds_list_head *list);
 int32_t
-glusterd_brick_connect (glusterd_volinfo_t  *volinfo,
-                        glusterd_brickinfo_t  *brickinfo, char *socketpath);
+glusterd_brick_connect(glusterd_volinfo_t *volinfo,
+                       glusterd_brickinfo_t *brickinfo, char *socketpath);
 int32_t
-glusterd_brick_disconnect (glusterd_brickinfo_t *brickinfo);
+glusterd_brick_disconnect(glusterd_brickinfo_t *brickinfo);
 int32_t
-glusterd_delete_volume (glusterd_volinfo_t *volinfo);
+glusterd_delete_volume(glusterd_volinfo_t *volinfo);
 int32_t
-glusterd_delete_brick (glusterd_volinfo_t* volinfo,
-                       glusterd_brickinfo_t *brickinfo);
+glusterd_delete_brick(glusterd_volinfo_t *volinfo,
+                      glusterd_brickinfo_t *brickinfo);
 
 int32_t
-glusterd_delete_all_bricks (glusterd_volinfo_t* volinfo);
+glusterd_delete_all_bricks(glusterd_volinfo_t *volinfo);
 
 int
-glusterd_spawn_daemons (void *opaque);
+glusterd_spawn_daemons(void *opaque);
 
 int
-glusterd_restart_gsyncds (glusterd_conf_t *conf);
+glusterd_restart_gsyncds(glusterd_conf_t *conf);
 
 int
-glusterd_start_gsync (glusterd_volinfo_t *master_vol, char *slave,
-                      char *path_list, char *conf_path,
-                      char *glusterd_uuid_str,
-                      char **op_errstr, gf_boolean_t is_pause);
+glusterd_start_gsync(glusterd_volinfo_t *master_vol, char *slave,
+                     char *path_list, char *conf_path, char *glusterd_uuid_str,
+                     char **op_errstr, gf_boolean_t is_pause);
 int
-glusterd_get_local_brickpaths (glusterd_volinfo_t *volinfo,
-                               char **pathlist);
+glusterd_get_local_brickpaths(glusterd_volinfo_t *volinfo, char **pathlist);
 
 int32_t
-glusterd_recreate_bricks (glusterd_conf_t *conf);
+glusterd_recreate_bricks(glusterd_conf_t *conf);
 int32_t
-glusterd_handle_upgrade_downgrade (dict_t *options, glusterd_conf_t *conf);
+glusterd_handle_upgrade_downgrade(dict_t *options, glusterd_conf_t *conf,
+                                  gf_boolean_t upgrade, gf_boolean_t downgrade);
 
 int
-glusterd_add_brick_detail_to_dict (glusterd_volinfo_t *volinfo,
-                                   glusterd_brickinfo_t *brickinfo,
-                                   dict_t  *dict, int32_t count);
+glusterd_add_brick_detail_to_dict(glusterd_volinfo_t *volinfo,
+                                  glusterd_brickinfo_t *brickinfo, dict_t *dict,
+                                  int32_t count);
 
 int32_t
-glusterd_add_brick_to_dict (glusterd_volinfo_t *volinfo,
-                            glusterd_brickinfo_t *brickinfo,
-                            dict_t  *dict, int32_t count);
+glusterd_add_brick_to_dict(glusterd_volinfo_t *volinfo,
+                           glusterd_brickinfo_t *brickinfo, dict_t *dict,
+                           int32_t count);
 
 int32_t
-glusterd_get_all_volnames (dict_t *dict);
+glusterd_get_all_volnames(dict_t *dict);
 
 gf_boolean_t
-glusterd_is_fuse_available ();
+glusterd_is_fuse_available();
+
+int
+glusterd_brick_statedump(glusterd_volinfo_t *volinfo,
+                         glusterd_brickinfo_t *brickinfo, char *options,
+                         int option_cnt, char **op_errstr);
 
 int
-glusterd_brick_statedump (glusterd_volinfo_t *volinfo,
-                          glusterd_brickinfo_t *brickinfo,
-                          char *options, int option_cnt, char **op_errstr);
+glusterd_brick_terminate(glusterd_volinfo_t *volinfo,
+                         glusterd_brickinfo_t *brickinfo, char *options,
+                         int option_cnt, char **op_errstr);
+
+#ifdef BUILD_GNFS
 int
-glusterd_nfs_statedump (char *options, int option_cnt, char **op_errstr);
+glusterd_nfs_statedump(char *options, int option_cnt, char **op_errstr);
+#endif
 
 int
-glusterd_quotad_statedump (char *options, int option_cnt, char **op_errstr);
+glusterd_client_statedump(char *volname, char *options, int option_cnt,
+                          char **op_errstr);
+
+int
+glusterd_quotad_statedump(char *options, int option_cnt, char **op_errstr);
 
 gf_boolean_t
-glusterd_is_volume_replicate (glusterd_volinfo_t *volinfo);
+glusterd_is_volume_replicate(glusterd_volinfo_t *volinfo);
 
 gf_boolean_t
-glusterd_is_brick_decommissioned (glusterd_volinfo_t *volinfo, char *hostname,
-                                  char *path);
+glusterd_is_brick_decommissioned(glusterd_volinfo_t *volinfo, char *hostname,
+                                 char *path);
+int
+glusterd_friend_contains_vol_bricks(glusterd_volinfo_t *volinfo,
+                                    uuid_t friend_uuid);
+
 int
-glusterd_friend_contains_vol_bricks (glusterd_volinfo_t *volinfo,
+glusterd_friend_contains_snap_bricks(glusterd_snap_t *snapinfo,
                                      uuid_t friend_uuid);
 int
-glusterd_friend_remove_cleanup_vols (uuid_t uuid);
+glusterd_friend_remove_cleanup_vols(uuid_t uuid);
 
 int
-glusterd_get_client_filepath (char *filepath,
-                              glusterd_volinfo_t *volinfo,
-                              gf_transport_type type);
+glusterd_get_client_filepath(char *filepath, glusterd_volinfo_t *volinfo,
+                             gf_transport_type type);
+int
+glusterd_get_trusted_client_filepath(char *filepath,
+                                     glusterd_volinfo_t *volinfo,
+                                     gf_transport_type type);
 int
-glusterd_get_trusted_client_filepath (char *filepath,
-                                      glusterd_volinfo_t *volinfo,
-                                      gf_transport_type type);
+glusterd_restart_rebalance(glusterd_conf_t *conf);
+
 int
-glusterd_restart_rebalance (glusterd_conf_t *conf);
+glusterd_restart_rebalance_for_volume(glusterd_volinfo_t *volinfo);
+
+void
+glusterd_defrag_info_set(glusterd_volinfo_t *volinfo, dict_t *dict, int cmd,
+                         int status, int op);
 
 int32_t
-glusterd_add_bricks_hname_path_to_dict (dict_t *dict,
-                                        glusterd_volinfo_t *volinfo);
+glusterd_add_bricks_hname_path_to_dict(dict_t *dict,
+                                       glusterd_volinfo_t *volinfo);
+
+int
+glusterd_add_node_to_dict(char *server, dict_t *dict, int count,
+                          dict_t *vol_opts);
 
 int
-glusterd_add_node_to_dict (char *server, dict_t *dict, int count,
-                           dict_t *vol_opts);
+glusterd_calc_dist_leaf_count(int rcount, int scount);
 
 int
-glusterd_get_dist_leaf_count (glusterd_volinfo_t *volinfo);
+glusterd_get_dist_leaf_count(glusterd_volinfo_t *volinfo);
 
-glusterd_brickinfo_t*
-glusterd_get_brickinfo_by_position (glusterd_volinfo_t *volinfo, uint32_t pos);
+glusterd_brickinfo_t *
+glusterd_get_brickinfo_by_position(glusterd_volinfo_t *volinfo, uint32_t pos);
 
 gf_boolean_t
-glusterd_is_local_brick (xlator_t *this, glusterd_volinfo_t *volinfo,
-                         glusterd_brickinfo_t *brickinfo);
+glusterd_is_local_brick(xlator_t *this, glusterd_volinfo_t *volinfo,
+                        glusterd_brickinfo_t *brickinfo);
 int
-glusterd_validate_volume_id (dict_t *op_dict, glusterd_volinfo_t *volinfo);
+glusterd_validate_volume_id(dict_t *op_dict, glusterd_volinfo_t *volinfo);
 
 int
-glusterd_defrag_volume_status_update (glusterd_volinfo_t *volinfo,
-                                      dict_t *rsp_dict);
+glusterd_defrag_volume_status_update(glusterd_volinfo_t *volinfo,
+                                     dict_t *rsp_dict, int32_t cmd);
 
 int
-glusterd_check_files_identical (char *filename1, char *filename2,
-                                gf_boolean_t *identical);
+glusterd_check_files_identical(char *filename1, char *filename2,
+                               gf_boolean_t *identical);
 
 int
-glusterd_check_topology_identical (const char *filename1,
-                                   const char *filename2,
-                                   gf_boolean_t *identical);
+glusterd_check_topology_identical(const char *filename1, const char *filename2,
+                                  gf_boolean_t *identical);
 
 void
-glusterd_volinfo_reset_defrag_stats (glusterd_volinfo_t *volinfo);
+glusterd_volinfo_reset_defrag_stats(glusterd_volinfo_t *volinfo);
 int
-glusterd_volset_help (dict_t *dict, char **op_errstr);
+glusterd_volset_help(dict_t *dict, char **op_errstr);
 
 int32_t
-glusterd_sync_use_rsp_dict (dict_t *aggr, dict_t *rsp_dict);
+glusterd_sync_use_rsp_dict(dict_t *aggr, dict_t *rsp_dict);
 int32_t
-glusterd_gsync_use_rsp_dict (dict_t *aggr, dict_t *rsp_dict, char *op_errstr);
+glusterd_gsync_use_rsp_dict(dict_t *aggr, dict_t *rsp_dict, char *op_errstr);
 int32_t
-glusterd_rb_use_rsp_dict (dict_t *aggr, dict_t *rsp_dict);
+glusterd_rb_use_rsp_dict(dict_t *aggr, dict_t *rsp_dict);
 int
-glusterd_profile_volume_use_rsp_dict (dict_t *aggr, dict_t *rsp_dict);
+glusterd_profile_volume_use_rsp_dict(dict_t *aggr, dict_t *rsp_dict);
 int
-glusterd_volume_status_copy_to_op_ctx_dict (dict_t *aggr, dict_t *rsp_dict);
+glusterd_volume_status_copy_to_op_ctx_dict(dict_t *aggr, dict_t *rsp_dict);
 int
-glusterd_volume_rebalance_use_rsp_dict (dict_t *aggr, dict_t *rsp_dict);
+glusterd_volume_rebalance_use_rsp_dict(dict_t *aggr, dict_t *rsp_dict);
 int
-glusterd_volume_heal_use_rsp_dict (dict_t *aggr, dict_t *rsp_dict);
+glusterd_volume_heal_use_rsp_dict(dict_t *aggr, dict_t *rsp_dict);
 int
-glusterd_use_rsp_dict (dict_t *aggr, dict_t *rsp_dict);
+glusterd_use_rsp_dict(dict_t *aggr, dict_t *rsp_dict);
 int
-glusterd_sys_exec_output_rsp_dict (dict_t *aggr, dict_t *rsp_dict);
+glusterd_sys_exec_output_rsp_dict(dict_t *aggr, dict_t *rsp_dict);
 int32_t
-glusterd_handle_node_rsp (dict_t *req_ctx, void *pending_entry,
-                          glusterd_op_t op, dict_t *rsp_dict, dict_t *op_ctx,
-                          char **op_errstr, gd_node_type type);
+glusterd_handle_node_rsp(dict_t *req_ctx, void *pending_entry, glusterd_op_t op,
+                         dict_t *rsp_dict, dict_t *op_ctx, char **op_errstr,
+                         gd_node_type type);
 int
-glusterd_volume_rebalance_use_rsp_dict (dict_t *aggr, dict_t *rsp_dict);
+glusterd_max_opversion_use_rsp_dict(dict_t *dst, dict_t *src);
+
+int
+glusterd_volume_bitrot_scrub_use_rsp_dict(dict_t *aggr, dict_t *rsp_dict);
+
 int
-glusterd_volume_heal_use_rsp_dict (dict_t *aggr, dict_t *rsp_dict);
+glusterd_volume_heal_use_rsp_dict(dict_t *aggr, dict_t *rsp_dict);
 
 int32_t
-glusterd_check_if_quota_trans_enabled (glusterd_volinfo_t *volinfo);
+glusterd_check_if_quota_trans_enabled(glusterd_volinfo_t *volinfo);
 
 int
-glusterd_volume_quota_copy_to_op_ctx_dict (dict_t *aggr, dict_t *rsp);
+glusterd_volume_quota_copy_to_op_ctx_dict(dict_t *aggr, dict_t *rsp);
 int
-_profile_volume_add_brick_rsp (dict_t *this, char *key, data_t *value,
-                             void *data);
+_profile_volume_add_brick_rsp(dict_t *this, char *key, data_t *value,
+                              void *data);
 int
-glusterd_profile_volume_brick_rsp (void *pending_entry,
-                                   dict_t *rsp_dict, dict_t *op_ctx,
-                                   char **op_errstr, gd_node_type type);
+glusterd_profile_volume_brick_rsp(void *pending_entry, dict_t *rsp_dict,
+                                  dict_t *op_ctx, char **op_errstr,
+                                  gd_node_type type);
 
 int32_t
-glusterd_set_originator_uuid (dict_t *dict);
+glusterd_set_originator_uuid(dict_t *dict);
 
 /* Should be used only when an operation is in progress, as that is the only
  * time a lock_owner is set
  */
 gf_boolean_t
-is_origin_glusterd (dict_t *dict);
+is_origin_glusterd(dict_t *dict);
 
 int
-glusterd_get_next_global_opt_version_str (dict_t *opts, char **version_str);
+glusterd_get_next_global_opt_version_str(dict_t *opts, char **version_str);
 
 int
-glusterd_generate_and_set_task_id (dict_t *dict, char *key);
+glusterd_generate_and_set_task_id(dict_t *dict, char *key, const int keylen);
 
 int
-glusterd_validate_and_set_gfid (dict_t *op_ctx, dict_t *req_dict,
-                                char **op_errstr);
+glusterd_validate_and_set_gfid(dict_t *op_ctx, dict_t *req_dict,
+                               char **op_errstr);
 
 int
-glusterd_copy_uuid_to_dict (uuid_t uuid, dict_t *dict, char *key);
+glusterd_copy_uuid_to_dict(uuid_t uuid, dict_t *dict, char *key,
+                           const int keylen);
 
 gf_boolean_t
-glusterd_is_same_address (char *name1, char *name2);
+glusterd_is_same_address(char *name1, char *name2);
 
 void
-gd_update_volume_op_versions (glusterd_volinfo_t *volinfo);
+gd_update_volume_op_versions(glusterd_volinfo_t *volinfo);
 
 int
-op_version_check (xlator_t *this, int min_op_version, char *msg, int msglen);
+op_version_check(xlator_t *this, int min_op_version, char *msg, int msglen);
 
 gf_boolean_t
-gd_is_remove_brick_committed (glusterd_volinfo_t *volinfo);
+gd_is_remove_brick_committed(glusterd_volinfo_t *volinfo);
 
 int
-glusterd_get_slave_details_confpath (glusterd_volinfo_t *volinfo,
-                                     dict_t *dict, char **slave_url,
-                                     char **slave_host, char **slave_vol,
-                                     char **conf_path, char **op_errstr);
+glusterd_remove_brick_validate_bricks(gf1_op_commands cmd, int32_t brick_count,
+                                      dict_t *dict, glusterd_volinfo_t *volinfo,
+                                      char **errstr, gf_cli_defrag_type);
+int
+glusterd_get_slave_details_confpath(glusterd_volinfo_t *volinfo, dict_t *dict,
+                                    char **slave_url, char **slave_host,
+                                    char **slave_vol, char **conf_path,
+                                    char **op_errstr);
 
 int
-glusterd_get_slave_info (char *slave,
-                         char **slave_url, char **hostname,
-                         char **slave_vol, char **op_errstr);
+glusterd_get_slave_info(char *slave, char **slave_url, char **hostname,
+                        char **slave_vol, char **op_errstr);
 
 int
-glusterd_get_statefile_name (glusterd_volinfo_t *volinfo, char *slave,
-                             char *conf_path, char **statefile,
-                             gf_boolean_t *is_template_in_use);
+glusterd_get_statefile_name(glusterd_volinfo_t *volinfo, char *slave,
+                            char *conf_path, char **statefile,
+                            gf_boolean_t *is_template_in_use);
 
 int
-glusterd_gsync_read_frm_status (char *path, char *buf, size_t blen);
+glusterd_gsync_read_frm_status(char *path, char *buf, size_t blen);
 
 int
-glusterd_create_status_file (char *master, char *slave, char *slave_url,
-                             char *slave_vol, char *status);
+glusterd_create_status_file(char *master, char *slave, char *slave_url,
+                            char *slave_vol, char *status);
 
 int
-glusterd_check_restart_gsync_session (glusterd_volinfo_t *volinfo, char *slave,
-                                      dict_t *resp_dict, char *path_list,
-                                      char *conf_path, gf_boolean_t is_force);
+glusterd_check_restart_gsync_session(glusterd_volinfo_t *volinfo, char *slave,
+                                     dict_t *resp_dict, char *path_list,
+                                     char *conf_path, gf_boolean_t is_force);
 
 int
-glusterd_check_gsync_running_local (char *master, char *slave,
-                                    char *conf_path,
-                                    gf_boolean_t *is_run);
+glusterd_check_gsync_running_local(char *master, char *slave, char *conf_path,
+                                   gf_boolean_t *is_run);
 
 gf_boolean_t
-glusterd_is_status_tasks_op (glusterd_op_t op, dict_t *dict);
+glusterd_is_status_tasks_op(glusterd_op_t op, dict_t *dict);
 
 gf_boolean_t
-gd_should_i_start_rebalance  (glusterd_volinfo_t *volinfo);
+gd_should_i_start_rebalance(glusterd_volinfo_t *volinfo);
 
 int
-glusterd_is_volume_quota_enabled (glusterd_volinfo_t *volinfo);
+glusterd_is_volume_quota_enabled(glusterd_volinfo_t *volinfo);
 
 int
-glusterd_is_bitrot_enabled (glusterd_volinfo_t *volinfo);
+glusterd_is_volume_inode_quota_enabled(glusterd_volinfo_t *volinfo);
+
+int
+glusterd_is_bitrot_enabled(glusterd_volinfo_t *volinfo);
 
 gf_boolean_t
-glusterd_all_volumes_with_quota_stopped ();
+glusterd_all_volumes_with_quota_stopped();
 
 void
-glusterd_clean_up_quota_store (glusterd_volinfo_t *volinfo);
+glusterd_clean_up_quota_store(glusterd_volinfo_t *volinfo);
 
 int
-glusterd_store_quota_conf_skip_header (xlator_t *this, int fd);
-
-int
-glusterd_store_quota_conf_stamp_header (xlator_t *this, int fd);
-
-int
-glusterd_remove_auxiliary_mount (char *volname);
+glusterd_remove_auxiliary_mount(char *volname);
 
 gf_boolean_t
-glusterd_status_has_tasks (int cmd);
+glusterd_status_has_tasks(int cmd);
 
 int
-gd_stop_rebalance_process (glusterd_volinfo_t *volinfo);
+gd_stop_rebalance_process(glusterd_volinfo_t *volinfo);
 
 rpc_clnt_t *
-glusterd_rpc_clnt_unref (glusterd_conf_t *conf, rpc_clnt_t *rpc);
+glusterd_rpc_clnt_unref(glusterd_conf_t *conf, rpc_clnt_t *rpc);
 
 int32_t
 glusterd_compare_volume_name(struct cds_list_head *, struct cds_list_head *);
 
-char*
-glusterd_get_brick_mount_device (char *brick_path);
+char *
+glusterd_get_brick_mount_device(char *brick_path);
 
 struct mntent *
-glusterd_get_mnt_entry_info (char *mnt_pt, char *buff, int buflen,
-                             struct mntent *entry_ptr);
+glusterd_get_mnt_entry_info(char *mnt_pt, char *buff, int buflen,
+                            struct mntent *entry_ptr);
 
 int
-glusterd_get_brick_root (char *path, char **mount_point);
+glusterd_get_brick_root(char *path, char **mount_point);
 
 int32_t
-glusterd_lvm_snapshot_remove (dict_t *rsp_dict, glusterd_volinfo_t *snap_vol);
+glusterd_lvm_snapshot_remove(dict_t *rsp_dict, glusterd_volinfo_t *snap_vol);
 
 gf_boolean_t
-gd_vol_is_geo_rep_active (glusterd_volinfo_t *volinfo);
+gd_vol_is_geo_rep_active(glusterd_volinfo_t *volinfo);
 
 int32_t
-glusterd_get_brick_mount_dir (char *brickpath, char *hostname, char *mount_dir);
+glusterd_get_brick_mount_dir(char *brickpath, char *hostname, char *mount_dir);
 
 int32_t
-glusterd_aggr_brick_mount_dirs (dict_t *aggr, dict_t *rsp_dict);
+glusterd_aggr_brick_mount_dirs(dict_t *aggr, dict_t *rsp_dict);
 
 int32_t
-glusterd_take_lvm_snapshot (glusterd_brickinfo_t *brickinfo,
-                            char *origin_brick_path);
+glusterd_take_lvm_snapshot(glusterd_brickinfo_t *brickinfo,
+                           char *origin_brick_path);
 
 void
-glusterd_launch_synctask (synctask_fn_t fn, void *opaque);
+glusterd_launch_synctask(synctask_fn_t fn, void *opaque);
 
 int
-glusterd_enable_default_options (glusterd_volinfo_t *volinfo, char *option);
+glusterd_enable_default_options(glusterd_volinfo_t *volinfo, char *option);
 
 int
-glusterd_unlink_file (char *sock_file_path);
+glusterd_unlink_file(char *sock_file_path);
 
 int32_t
-glusterd_find_brick_mount_path (char *brick_path, int32_t brick_count,
-                                char **brick_mount_path);
+glusterd_find_brick_mount_path(char *brick_path, char **brick_mount_path);
+
 /*
  * Function to retrieve list of snap volnames and their uuids
  */
-int glusterd_snapshot_get_volnames_uuids (dict_t *dict,
-           char *volname, gf_getsnap_name_uuid_rsp *snap_info_rsp);
+int
+glusterd_snapshot_get_volnames_uuids(dict_t *dict, char *volname,
+                                     gf_getsnap_name_uuid_rsp *snap_info_rsp);
+
+int
+glusterd_update_mntopts(char *brick_path, glusterd_brickinfo_t *brickinfo);
+
+int
+glusterd_update_fs_label(glusterd_brickinfo_t *brickinfo);
 
 int
-glusterd_update_mntopts (char *brick_path, glusterd_brickinfo_t *brickinfo);
+glusterd_get_volopt_content(dict_t *dict, gf_boolean_t xml_out);
 
 int
-glusterd_update_fs_label (glusterd_brickinfo_t *brickinfo);
+glusterd_get_global_max_op_version(rpcsvc_request_t *req, dict_t *ctx,
+                                   int count);
 
 int
-glusterd_get_volopt_content (dict_t *dict, gf_boolean_t xml_out);
+glusterd_get_global_options_for_all_vols(rpcsvc_request_t *req, dict_t *dict,
+                                         char **op_errstr);
 
 int
-glusterd_get_default_val_for_volopt (dict_t *dict, gf_boolean_t all_opts,
-                                     char *key, char *orig_key,
-                                     dict_t *vol_dict, char **err_str);
+glusterd_get_default_val_for_volopt(dict_t *dict, gf_boolean_t all_opts,
+                                    char *key, char *orig_key,
+                                    glusterd_volinfo_t *volinfo,
+                                    char **err_str);
 
 int
-glusterd_check_client_op_version_support (char *volname, uint32_t op_version,
-                                          char **op_errstr);
+glusterd_check_client_op_version_support(char *volname, uint32_t op_version,
+                                         char **op_errstr);
 
 gf_boolean_t
-glusterd_have_peers ();
+glusterd_have_peers();
+
+gf_boolean_t
+glusterd_have_volumes();
+
+void
+glusterd_get_rebalance_volfile(glusterd_volinfo_t *volinfo, char *path,
+                               int path_len);
 
 void
-glusterd_get_rebalance_volfile (glusterd_volinfo_t *volinfo,
-                                char *path, int path_len);
+glusterd_get_gfproxy_client_volfile(glusterd_volinfo_t *volinfo, char *path,
+                                    int path_len);
 
 int32_t
-glusterd_brickinfo_dup (glusterd_brickinfo_t *brickinfo,
-                        glusterd_brickinfo_t *dup_brickinfo);
+glusterd_brickinfo_dup(glusterd_brickinfo_t *brickinfo,
+                       glusterd_brickinfo_t *dup_brickinfo);
 
 int
-glusterd_vol_add_quota_conf_to_dict (glusterd_volinfo_t *volinfo, dict_t *load,
-                                     int vol_idx, char *prefix);
+glusterd_vol_add_quota_conf_to_dict(glusterd_volinfo_t *volinfo, dict_t *load,
+                                    int vol_idx, char *prefix);
 
 int32_t
-glusterd_import_volinfo (dict_t *peer_data, int count,
-                         glusterd_volinfo_t **volinfo,
-                         char *prefix);
+glusterd_import_volinfo(dict_t *peer_data, int count,
+                        glusterd_volinfo_t **volinfo, char *prefix);
 
 int
-glusterd_import_quota_conf (dict_t *peer_data, int vol_idx,
-                            glusterd_volinfo_t *new_volinfo,
-                            char *prefix);
+glusterd_import_quota_conf(dict_t *peer_data, int vol_idx,
+                           glusterd_volinfo_t *new_volinfo, char *prefix);
 
 gf_boolean_t
-glusterd_is_shd_compatible_volume (glusterd_volinfo_t *volinfo);
+glusterd_is_shd_compatible_volume(glusterd_volinfo_t *volinfo);
 
+gf_boolean_t
+glusterd_is_shd_compatible_type(int type);
 
 gf_boolean_t
-glusterd_are_all_volumes_stopped ();
+glusterd_are_all_volumes_stopped();
 
 gf_boolean_t
-glusterd_all_shd_compatible_volumes_stopped ();
+glusterd_all_shd_compatible_volumes_stopped();
 
 void
-glusterd_nfs_pmap_deregister ();
+glusterd_nfs_pmap_deregister();
 
 gf_boolean_t
-glusterd_is_volume_started (glusterd_volinfo_t  *volinfo);
+glusterd_is_volume_started(glusterd_volinfo_t *volinfo);
+
+int
+glusterd_volume_get_type_str(glusterd_volinfo_t *volinfo, char **vol_type_str);
+
+int
+glusterd_volume_get_status_str(glusterd_volinfo_t *volinfo, char *status_str);
+
+void
+glusterd_brick_get_status_str(glusterd_brickinfo_t *brickinfo,
+                              char *status_str);
+
+int
+glusterd_volume_get_transport_type_str(glusterd_volinfo_t *volinfo,
+                                       char *transport_type_str);
+
+int
+glusterd_volume_get_quorum_status_str(glusterd_volinfo_t *volinfo,
+                                      char *quorum_status_str);
+
+int
+glusterd_volume_get_rebalance_status_str(glusterd_volinfo_t *volinfo,
+                                         char *rebal_status_str);
 
 void
-glusterd_list_add_order (struct cds_list_head *new, struct cds_list_head *head,
+glusterd_list_add_order(struct cds_list_head *new, struct cds_list_head *head,
                         int (*compare)(struct cds_list_head *,
                                        struct cds_list_head *));
 
+struct rpc_clnt *
+glusterd_defrag_rpc_get(glusterd_defrag_info_t *defrag);
+
+struct rpc_clnt *
+glusterd_defrag_rpc_put(glusterd_defrag_info_t *defrag);
+
+int32_t
+glusterd_count_connected_peers(int32_t *count);
+
+int
+glusterd_volume_brick_for_each(glusterd_volinfo_t *volinfo, void *data,
+                               int (*fn)(glusterd_volinfo_t *,
+                                         glusterd_brickinfo_t *,
+                                         dict_t *mod_dict, void *));
+
+int
+glusterd_get_dummy_client_filepath(char *filepath, glusterd_volinfo_t *volinfo,
+                                   gf_transport_type type);
+
+int
+glusterd_handle_replicate_brick_ops(glusterd_volinfo_t *volinfo,
+                                    glusterd_brickinfo_t *brickinfo,
+                                    glusterd_op_t op);
 void
-gd_cleanup_local_xaction_peers_list (struct cds_list_head *peers);
+assign_brick_groups(glusterd_volinfo_t *volinfo);
+
+glusterd_brickinfo_t *
+get_last_brick_of_brick_group(glusterd_volinfo_t *volinfo,
+                              glusterd_brickinfo_t *brickinfo);
+int
+glusterd_get_rb_dst_brickinfo(glusterd_volinfo_t *volinfo,
+                              glusterd_brickinfo_t **brickinfo);
+int
+rb_update_dstbrick_port(glusterd_brickinfo_t *dst_brickinfo, dict_t *rsp_dict,
+                        dict_t *req_dict);
+int
+glusterd_op_perform_replace_brick(glusterd_volinfo_t *volinfo, char *old_brick,
+                                  char *new_brick, dict_t *dict);
+int32_t
+glusterd_brick_unlink_socket_file(glusterd_volinfo_t *volinfo,
+                                  glusterd_brickinfo_t *brickinfo);
+char *
+gd_rb_op_to_str(char *op);
+
+glusterd_op_t
+gd_cli_to_gd_op(char *cli_op);
+
+int
+glusterd_get_dst_brick_info(char **dst_brick, char *volname, char **op_errstr,
+                            glusterd_brickinfo_t **dst_brickinfo, char **host,
+                            dict_t *dict, char **dup_dstbrick);
+
+int
+glusterd_brick_op_prerequisites(dict_t *dict, char **op, glusterd_op_t *gd_op,
+                                char **volname, glusterd_volinfo_t **volinfo,
+                                char **src_brick,
+                                glusterd_brickinfo_t **src_brickinfo,
+                                char *pidfile, char **op_errstr,
+                                dict_t *rsp_dict);
+
+int
+glusterd_get_volinfo_from_brick(char *brick, glusterd_volinfo_t **volinfo);
+
+gf_boolean_t
+glusterd_is_profile_on(glusterd_volinfo_t *volinfo);
+
+char *
+search_brick_path_from_proc(pid_t brick_pid, char *brickpath);
+
+int32_t
+glusterd_add_shd_to_dict(glusterd_volinfo_t *volinfo, dict_t *dict,
+                         int32_t count);
+int32_t
+glusterd_check_brick_order(dict_t *dict, char *err_str, int32_t type,
+                           char **volname, char **bricks, int32_t *brick_count,
+                           int32_t sub_count);
 
 #endif
diff --git a/xlators/mgmt/glusterd/src/glusterd-volgen.c b/xlators/mgmt/glusterd/src/glusterd-volgen.c
index bbac2a2283b..8d6fb5e0fac 100644
--- a/xlators/mgmt/glusterd/src/glusterd-volgen.c
+++ b/xlators/mgmt/glusterd/src/glusterd-volgen.c
@@ -8,39 +8,67 @@
    cases as published by the Free Software Foundation.
 */
 
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
-
 #include <fnmatch.h>
 #include <sys/wait.h>
 #include <dlfcn.h>
 #include <utime.h>
 
-#include "xlator.h"
+#include <glusterfs/xlator.h>
 #include "glusterd.h"
-#include "defaults.h"
-#include "logging.h"
-#include "dict.h"
-#include "graph-utils.h"
+#include <glusterfs/defaults.h>
+#include <glusterfs/syscall.h>
+#include <glusterfs/logging.h>
+#include <glusterfs/dict.h>
+#include <glusterfs/graph-utils.h>
+#include <glusterfs/common-utils.h>
 #include "glusterd-store.h"
 #include "glusterd-hooks.h"
-#include "trie.h"
+#include <glusterfs/trie.h>
 #include "glusterd-mem-types.h"
 #include "cli1-xdr.h"
 #include "glusterd-volgen.h"
 #include "glusterd-geo-rep.h"
 #include "glusterd-utils.h"
-#include "run.h"
-#include "options.h"
+#include "glusterd-messages.h"
+#include <glusterfs/run.h>
+#include <glusterfs/options.h>
 #include "glusterd-snapshot-utils.h"
 #include "glusterd-svc-mgmt.h"
 #include "glusterd-svc-helper.h"
 #include "glusterd-snapd-svc-helper.h"
+#include "glusterd-shd-svc-helper.h"
+#include "glusterd-gfproxyd-svc-helper.h"
+
+struct gd_validate_reconf_opts {
+    dict_t *options;
+    char **op_errstr;
+};
 
 extern struct volopt_map_entry glusterd_volopt_map[];
 
+#define RPC_SET_OPT(XL, CLI_OPT, XLATOR_OPT, ERROR_CMD)                        \
+    do {                                                                       \
+        char *_value = NULL;                                                   \
+                                                                               \
+        if (dict_get_str_sizen(set_dict, CLI_OPT, &_value) == 0) {             \
+            if (xlator_set_fixed_option(XL, "transport.socket." XLATOR_OPT,    \
+                                        _value) != 0) {                        \
+                gf_msg("glusterd", GF_LOG_WARNING, errno,                      \
+                       GD_MSG_XLATOR_SET_OPT_FAIL,                             \
+                       "failed to set " XLATOR_OPT);                           \
+                ERROR_CMD;                                                     \
+            }                                                                  \
+        }                                                                      \
+    } while (0 /* CONSTCOND */)
+
+static int
+volgen_graph_build_clients(volgen_graph_t *graph, glusterd_volinfo_t *volinfo,
+                           dict_t *set_dict, void *param);
+
+static int
+build_client_graph(volgen_graph_t *graph, glusterd_volinfo_t *volinfo,
+                   dict_t *mod_dict);
+
 /*********************************************
  *
  * xlator generation / graph manipulation API
@@ -48,1061 +76,1083 @@ extern struct volopt_map_entry glusterd_volopt_map[];
  *********************************************/
 
 static void
-set_graph_errstr (volgen_graph_t *graph, const char *str)
+set_graph_errstr(volgen_graph_t *graph, const char *str)
 {
-        if (!graph->errstr)
-                return;
-
-        *graph->errstr = gf_strdup (str);
-}
-
-static xlator_t *
-xlator_instantiate_va (const char *type, const char *format, va_list arg)
-{
-        xlator_t *xl = NULL;
-        char *volname = NULL;
-        int ret = 0;
-
-        ret = gf_vasprintf (&volname, format, arg);
-        if (ret < 0) {
-                volname = NULL;
-
-                goto error;
-        }
-
-        xl = GF_CALLOC (1, sizeof (*xl), gf_common_mt_xlator_t);
-        if (!xl)
-                goto error;
-        ret = xlator_set_type_virtual (xl, type);
-        if (ret)
-                goto error;
-        xl->options = get_new_dict();
-        if (!xl->options)
-                goto error;
-        xl->name = volname;
-        CDS_INIT_LIST_HEAD (&xl->volume_options);
-
-        xl->ctx = THIS->ctx;
-
-        return xl;
-
- error:
-        gf_log ("", GF_LOG_ERROR, "creating xlator of type %s failed",
-                type);
-        GF_FREE (volname);
-        if (xl)
-                xlator_destroy (xl);
+    if (!graph->errstr)
+        return;
 
-        return NULL;
+    *graph->errstr = gf_strdup(str);
 }
 
-#ifdef __not_used_as_of_now_
 static xlator_t *
-xlator_instantiate (const char *type, const char *format, ...)
-{
-        va_list arg;
-        xlator_t *xl;
-
-        va_start (arg, format);
-        xl = xlator_instantiate_va (type, format, arg);
-        va_end (arg);
-
-        return xl;
+xlator_instantiate_va(const char *type, const char *format, va_list arg)
+{
+    xlator_t *xl = NULL;
+    char *volname = NULL;
+    int ret = 0;
+    xlator_t *this = THIS;
+    GF_ASSERT(this);
+
+    ret = gf_vasprintf(&volname, format, arg);
+    if (ret < 0) {
+        volname = NULL;
+
+        goto error;
+    }
+
+    xl = GF_CALLOC(1, sizeof(*xl), gf_common_mt_xlator_t);
+    if (!xl) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_NO_MEMORY, NULL);
+        goto error;
+    }
+    ret = xlator_set_type_virtual(xl, type);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_XLATOR_SET_OPT_FAIL,
+                NULL);
+        goto error;
+    }
+    xl->options = dict_new();
+    if (!xl->options) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_CREATE_FAIL, NULL);
+        goto error;
+    }
+    xl->name = volname;
+    CDS_INIT_LIST_HEAD(&xl->volume_options);
+
+    xl->ctx = THIS->ctx;
+
+    return xl;
+
+error:
+    gf_smsg(this->name, GF_LOG_ERROR, 0, GD_MSG_XLATOR_CREATE_FAIL, "Type=%s",
+            type, NULL);
+    GF_FREE(volname);
+    if (xl)
+        xlator_destroy(xl);
+
+    return NULL;
 }
-#endif
 
 static int
-volgen_xlator_link (xlator_t *pxl, xlator_t *cxl)
+volgen_xlator_link(xlator_t *pxl, xlator_t *cxl)
 {
-        int ret = 0;
+    int ret = 0;
 
-        ret = glusterfs_xlator_link (pxl, cxl);
-        if (ret == -1) {
-                gf_log ("", GF_LOG_ERROR,
-                        "Out of memory, cannot link xlators %s <- %s",
-                        pxl->name, cxl->name);
-        }
+    ret = glusterfs_xlator_link(pxl, cxl);
+    if (ret == -1) {
+        gf_msg("glusterd", GF_LOG_ERROR, ENOMEM, GD_MSG_NO_MEMORY,
+               "Out of memory, cannot link xlators %s <- %s", pxl->name,
+               cxl->name);
+    }
 
-        return ret;
+    return ret;
 }
 
 static int
-volgen_graph_link (volgen_graph_t *graph, xlator_t *xl)
+volgen_graph_link(volgen_graph_t *graph, xlator_t *xl)
 {
-        int ret = 0;
+    int ret = 0;
 
-        /* no need to care about graph->top here */
-        if (graph->graph.first)
-                ret = volgen_xlator_link (xl, graph->graph.first);
-        if (ret == -1) {
-                gf_log ("", GF_LOG_ERROR, "failed to add graph entry %s",
-                        xl->name);
+    /* no need to care about graph->top here */
+    if (graph->graph.first)
+        ret = volgen_xlator_link(xl, graph->graph.first);
+    if (ret == -1) {
+        gf_msg("glusterd", GF_LOG_ERROR, 0, GD_MSG_GRAPH_ENTRY_ADD_FAIL,
+               "failed to add graph entry %s", xl->name);
 
-                return -1;
-        }
+        return -1;
+    }
 
-        return 0;
+    return 0;
 }
 
 static xlator_t *
-volgen_graph_add_as (volgen_graph_t *graph, const char *type,
-                     const char *format, ...)
+volgen_graph_add_as(volgen_graph_t *graph, const char *type, const char *format,
+                    ...)
 {
-        va_list arg;
-        xlator_t *xl = NULL;
+    va_list arg;
+    xlator_t *xl = NULL;
 
-        va_start (arg, format);
-        xl = xlator_instantiate_va (type, format, arg);
-        va_end (arg);
+    va_start(arg, format);
+    xl = xlator_instantiate_va(type, format, arg);
+    va_end(arg);
 
-        if (!xl)
-                return NULL;
+    if (!xl)
+        return NULL;
 
-        if (volgen_graph_link (graph, xl)) {
-                xlator_destroy (xl);
+    if (volgen_graph_link(graph, xl)) {
+        xlator_destroy(xl);
 
-                return NULL;
-        } else
-                glusterfs_graph_set_first (&graph->graph, xl);
+        return NULL;
+    } else
+        glusterfs_graph_set_first(&graph->graph, xl);
 
-        return xl;
+    return xl;
 }
 
 static xlator_t *
-volgen_graph_add_nolink (volgen_graph_t *graph, const char *type,
-                         const char *format, ...)
+volgen_graph_add_nolink(volgen_graph_t *graph, const char *type,
+                        const char *format, ...)
 {
-        va_list arg;
-        xlator_t *xl = NULL;
+    va_list arg;
+    xlator_t *xl = NULL;
 
-        va_start (arg, format);
-        xl = xlator_instantiate_va (type, format, arg);
-        va_end (arg);
+    va_start(arg, format);
+    xl = xlator_instantiate_va(type, format, arg);
+    va_end(arg);
 
-        if (!xl)
-                return NULL;
+    if (!xl)
+        return NULL;
 
-        glusterfs_graph_set_first (&graph->graph, xl);
+    glusterfs_graph_set_first(&graph->graph, xl);
 
-        return xl;
+    return xl;
 }
 
 static xlator_t *
-volgen_graph_add (volgen_graph_t *graph, char *type, char *volname)
+volgen_graph_add(volgen_graph_t *graph, char *type, char *volname)
 {
-        char *shorttype = NULL;
+    char *shorttype = NULL;
 
-        shorttype = strrchr (type, '/');
-        GF_ASSERT (shorttype);
-        shorttype++;
-        GF_ASSERT (*shorttype);
+    shorttype = strrchr(type, '/');
+    GF_ASSERT(shorttype);
+    shorttype++;
+    GF_ASSERT(*shorttype);
 
-        return volgen_graph_add_as (graph, type, "%s-%s", volname, shorttype);
+    return volgen_graph_add_as(graph, type, "%s-%s", volname, shorttype);
 }
 
+#define xlator_set_fixed_option(xl, key, value)                                \
+    xlator_set_option(xl, key, SLEN(key), value)
+
 /* XXX Seems there is no such generic routine?
  * Maybe should put to xlator.c ??
  */
 static int
-xlator_set_option (xlator_t *xl, char *key, char *value)
+xlator_set_option(xlator_t *xl, char *key, const int keylen, char *value)
 {
-        char *dval     = NULL;
+    char *dval = gf_strdup(value);
 
-        dval = gf_strdup (value);
-        if (!dval) {
-                gf_log ("", GF_LOG_ERROR,
-                        "failed to set xlator opt: %s[%s] = %s",
-                        xl->name, key, value);
+    if (!dval) {
+        gf_msg("glusterd", GF_LOG_ERROR, errno, GD_MSG_NO_MEMORY,
+               "failed to set xlator opt: %s[%s] = %s", xl->name, key, value);
 
-                return -1;
-        }
+        return -1;
+    }
 
-        return dict_set_dynstr (xl->options, key, dval);
+    return dict_set_dynstrn(xl->options, key, keylen, dval);
 }
 
+#define xlator_get_fixed_option(xl, key, value)                                \
+    xlator_get_option(xl, key, SLEN(key), value)
+
 static int
-xlator_get_option (xlator_t *xl, char *key, char **value)
+xlator_get_option(xlator_t *xl, char *key, const int keylen, char **value)
 {
-        GF_ASSERT (xl);
-        return dict_get_str (xl->options, key, value);
+    GF_ASSERT(xl);
+    return dict_get_strn(xl->options, key, keylen, value);
 }
 
-static inline xlator_t *
-first_of (volgen_graph_t *graph)
+static xlator_t *
+first_of(volgen_graph_t *graph)
 {
-        return (xlator_t *)graph->graph.first;
+    return (xlator_t *)graph->graph.first;
 }
 
-
-
-
 /**************************
  *
  * Trie glue
  *
  *************************/
 
-
 static int
-volopt_selector (int lvl, char **patt, void *param,
-                     int (*optcbk)(char *word, void *param))
-{
-        struct volopt_map_entry *vme = NULL;
-        char *w = NULL;
-        int i = 0;
-        int len = 0;
-        int ret = 0;
-        char *dot = NULL;
-
-        for (vme = glusterd_volopt_map; vme->key; vme++) {
-                w = vme->key;
-
-                for (i = 0; i < lvl; i++) {
-                        if (patt[i]) {
-                                w = strtail (w, patt[i]);
-                                GF_ASSERT (!w || *w);
-                                if (!w || *w != '.')
-                                        goto next;
-                        } else {
-                                w = strchr (w, '.');
-                                GF_ASSERT (w);
-                        }
-                        w++;
-                }
-
-                dot = strchr (w, '.');
-                if (dot) {
-                        len = dot - w;
-                        w = gf_strdup (w);
-                        if (!w)
-                                return -1;
-                        w[len] = '\0';
-                }
-                ret = optcbk (w, param);
-                if (dot)
-                        GF_FREE (w);
-                if (ret)
-                        return -1;
- next:
-                continue;
+volopt_selector(int lvl, char **patt, void *param,
+                int (*optcbk)(char *word, void *param))
+{
+    struct volopt_map_entry *vme = NULL;
+    char *w = NULL;
+    int i = 0;
+    int len = 0;
+    int ret = 0;
+    char *dot = NULL;
+
+    for (vme = glusterd_volopt_map; vme->key; vme++) {
+        w = vme->key;
+
+        for (i = 0; i < lvl; i++) {
+            if (patt[i]) {
+                w = strtail(w, patt[i]);
+                GF_ASSERT(!w || *w);
+                if (!w || *w != '.')
+                    goto next;
+            } else {
+                w = strchr(w, '.');
+                GF_ASSERT(w);
+            }
+            w++;
+        }
+
+        dot = strchr(w, '.');
+        if (dot) {
+            len = dot - w;
+            w = gf_strdup(w);
+            if (!w)
+                return -1;
+            w[len] = '\0';
         }
+        ret = optcbk(w, param);
+        if (dot)
+            GF_FREE(w);
+        if (ret)
+            return -1;
+    next:
+        continue;
+    }
 
-        return 0;
+    return 0;
 }
 
 static int
-volopt_trie_cbk (char *word, void *param)
+volopt_trie_cbk(char *word, void *param)
 {
-        return trie_add ((trie_t *)param, word);
+    return trie_add((trie_t *)param, word);
 }
 
 static int
-process_nodevec (struct trienodevec *nodevec, char **hint)
+process_nodevec(struct trienodevec *nodevec, char **outputhint, char *inputhint)
 {
-        int ret = 0;
-        char *hint1 = NULL;
-        char *hint2 = NULL;
-        char *hintinfx = "";
-        trienode_t **nodes = nodevec->nodes;
+    int ret = 0;
+    char *hint1 = NULL;
+    char *hint2 = NULL;
+    char *hintinfx = "";
+    trienode_t **nodes = nodevec->nodes;
 
-        if (!nodes[0]) {
-                *hint = NULL;
-                return 0;
-        }
+    if (!nodes[0]) {
+        *outputhint = NULL;
+        return 0;
+    }
 
 #if 0
         /* Limit as in git */
         if (trienode_get_dist (nodes[0]) >= 6) {
-                *hint = NULL;
+                *outputhint = NULL;
                 return 0;
         }
 #endif
 
-        if (trienode_get_word (nodes[0], &hint1))
-                return -1;
+    if (trienode_get_word(nodes[0], &hint1))
+        return -1;
 
-        if (nodevec->cnt < 2 || !nodes[1]) {
-                *hint = hint1;
-                return 0;
-        }
+    if (nodevec->cnt < 2 || !nodes[1]) {
+        *outputhint = hint1;
+        return 0;
+    }
 
-        if (trienode_get_word (nodes[1], &hint2))
-                return -1;
+    if (trienode_get_word(nodes[1], &hint2)) {
+        GF_FREE(hint1);
+        return -1;
+    }
 
-        if (*hint)
-                hintinfx = *hint;
-        ret = gf_asprintf (hint, "%s or %s%s", hint1, hintinfx, hint2);
-        if (ret > 0)
-                ret = 0;
-        return ret;
+    if (inputhint)
+        hintinfx = inputhint;
+    ret = gf_asprintf(outputhint, "%s or %s%s", hint1, hintinfx, hint2);
+    if (ret > 0)
+        ret = 0;
+    if (hint1)
+        GF_FREE(hint1);
+    if (hint2)
+        GF_FREE(hint2);
+    return ret;
 }
 
 static int
-volopt_trie_section (int lvl, char **patt, char *word, char **hint, int hints)
+volopt_trie_section(int lvl, char **patt, char *word, char **outputhint,
+                    char *inputhint, int hints)
 {
-        trienode_t *nodes[] = { NULL, NULL };
-        struct trienodevec nodevec = { nodes, 2};
-        trie_t *trie = NULL;
-        int ret = 0;
+    trienode_t *nodes[] = {NULL, NULL};
+    struct trienodevec nodevec = {nodes, 2};
+    trie_t *trie = NULL;
+    int ret = 0;
 
-        trie = trie_new ();
-        if (!trie)
-                return -1;
+    trie = trie_new();
+    if (!trie)
+        return -1;
 
-        if (volopt_selector (lvl, patt, trie, &volopt_trie_cbk)) {
-                trie_destroy (trie);
+    if (volopt_selector(lvl, patt, trie, &volopt_trie_cbk)) {
+        trie_destroy(trie);
 
-                return -1;
-        }
+        return -1;
+    }
 
-        GF_ASSERT (hints <= 2);
-        nodevec.cnt = hints;
-        ret = trie_measure_vec (trie, word, &nodevec);
-        if (!ret && nodevec.nodes[0])
-                ret = process_nodevec (&nodevec, hint);
+    GF_ASSERT(hints <= 2);
+    nodevec.cnt = hints;
+    ret = trie_measure_vec(trie, word, &nodevec);
+    if (!ret && nodevec.nodes[0])
+        ret = process_nodevec(&nodevec, outputhint, inputhint);
 
-        trie_destroy (trie);
+    trie_destroy(trie);
 
-        return ret;
+    return ret;
 }
 
 static int
-volopt_trie (char *key, char **hint)
+volopt_trie(char *key, char **hint)
 {
-        char *patt[] = { NULL };
-        char *fullhint = NULL;
-        char *dot = NULL;
-        char *dom = NULL;
-        int   len = 0;
-        int   ret = 0;
-
-        *hint = NULL;
+    char *patt[] = {NULL};
+    char *fullhint = NULL;
+    char *inputhint = NULL;
+    char *dot = NULL;
+    char *dom = NULL;
+    int len = 0;
+    int ret = 0;
 
-        dot = strchr (key, '.');
-        if (!dot)
-                return volopt_trie_section (1, patt, key, hint, 2);
+    *hint = NULL;
 
-        len = dot - key;
-        dom = gf_strdup (key);
-        if (!dom)
-                return -1;
-        dom[len] = '\0';
-
-        ret = volopt_trie_section (0, NULL, dom, patt, 1);
-        GF_FREE (dom);
-        if (ret) {
-                patt[0] = NULL;
-                goto out;
-        }
-        if (!patt[0])
-                goto out;
+    dot = strchr(key, '.');
+    if (!dot)
+        return volopt_trie_section(1, patt, key, hint, inputhint, 2);
 
-        *hint = "...";
-        ret = volopt_trie_section (1, patt, dot + 1, hint, 2);
-        if (ret)
-                goto out;
-        if (*hint) {
-                ret = gf_asprintf (&fullhint, "%s.%s", patt[0], *hint);
-                GF_FREE (*hint);
-                if (ret >= 0) {
-                        ret = 0;
-                        *hint = fullhint;
-                }
-        }
+    len = dot - key;
+    dom = gf_strdup(key);
+    if (!dom)
+        return -1;
+    dom[len] = '\0';
+
+    ret = volopt_trie_section(0, NULL, dom, patt, inputhint, 1);
+    GF_FREE(dom);
+    if (ret) {
+        patt[0] = NULL;
+        goto out;
+    }
+    if (!patt[0])
+        goto out;
+
+    inputhint = "...";
+    ret = volopt_trie_section(1, patt, dot + 1, hint, inputhint, 2);
+    if (ret)
+        goto out;
+    if (*hint) {
+        ret = gf_asprintf(&fullhint, "%s.%s", patt[0], *hint);
+        GF_FREE(*hint);
+        if (ret >= 0) {
+            ret = 0;
+            *hint = fullhint;
+        }
+    }
 
- out:
-        GF_FREE (patt[0]);
-        if (ret)
-                *hint = NULL;
+out:
+    GF_FREE(patt[0]);
+    if (ret)
+        *hint = NULL;
 
-        return ret;
+    return ret;
 }
 
-
-
-
 /**************************
  *
  * Volume generation engine
  *
  **************************/
 
-
-typedef int (*volgen_opthandler_t) (volgen_graph_t *graph,
-                                    struct volopt_map_entry *vme,
-                                    void *param);
+typedef int (*volgen_opthandler_t)(volgen_graph_t *graph,
+                                   struct volopt_map_entry *vme, void *param);
 
 struct opthandler_data {
-        volgen_graph_t *graph;
-        volgen_opthandler_t handler;
-        struct volopt_map_entry *vme;
-        gf_boolean_t found;
-        gf_boolean_t data_t_fake;
-        int rv;
-        char *volname;
-        void *param;
+    volgen_graph_t *graph;
+    volgen_opthandler_t handler;
+    struct volopt_map_entry *vme;
+    gf_boolean_t found;
+    gf_boolean_t data_t_fake;
+    int rv;
+    char *volname;
+    void *param;
 };
 
-static int
-process_option (char *key, data_t *value, void *param)
+static void
+process_option(char *key, data_t *value, void *param)
 {
-        struct opthandler_data *odt = param;
-        struct volopt_map_entry vme = {0,};
+    struct opthandler_data *odt = param;
+    struct volopt_map_entry vme = {
+        0,
+    };
 
-        if (odt->rv)
-                return 0;
-        odt->found = _gf_true;
-
-        vme.key = key;
-        vme.voltype = odt->vme->voltype;
-        vme.option = odt->vme->option;
-        vme.op_version = odt->vme->op_version;
-
-        if (!vme.option) {
-                vme.option = strrchr (key, '.');
-                if (vme.option)
-                        vme.option++;
-                else
-                        vme.option = key;
-        }
-        if (odt->data_t_fake)
-                vme.value = (char *)value;
+    if (odt->rv)
+        return;
+    odt->found = _gf_true;
+
+    vme.key = key;
+    vme.voltype = odt->vme->voltype;
+    vme.option = odt->vme->option;
+    vme.op_version = odt->vme->op_version;
+
+    if (!vme.option) {
+        vme.option = strrchr(key, '.');
+        if (vme.option)
+            vme.option++;
         else
-                vme.value = value->data;
+            vme.option = key;
+    }
+    if (odt->data_t_fake)
+        vme.value = (char *)value;
+    else
+        vme.value = value->data;
 
-        odt->rv = odt->handler (odt->graph, &vme, odt->param);
-        return 0;
+    odt->rv = odt->handler(odt->graph, &vme, odt->param);
+    return;
 }
 
 static int
-volgen_graph_set_options_generic (volgen_graph_t *graph, dict_t *dict,
-                                  void *param, volgen_opthandler_t handler)
-{
-        struct volopt_map_entry *vme = NULL;
-        struct opthandler_data odt = {0,};
-        data_t *data = NULL;
-
-        odt.graph = graph;
-        odt.handler = handler;
-        odt.param = param;
-        (void)data;
-
-        for (vme = glusterd_volopt_map; vme->key; vme++) {
-                odt.vme = vme;
-                odt.found = _gf_false;
-                odt.data_t_fake = _gf_false;
-
-                data = dict_get (dict, vme->key);
-
-                if (data)
-                        process_option (vme->key, data, &odt);
-                if (odt.rv)
-                        return odt.rv;
-
-                if (odt.found)
-                        continue;
-
-                /* check for default value */
-
-                if (vme->value) {
-                        /* stupid hack to be able to reuse dict iterator
-                         * in this context
-                         */
-                        odt.data_t_fake = _gf_true;
-                        process_option (vme->key, (data_t *)vme->value, &odt);
-                        if (odt.rv)
-                                return odt.rv;
-                }
+volgen_graph_set_options_generic(volgen_graph_t *graph, dict_t *dict,
+                                 void *param, volgen_opthandler_t handler)
+{
+    struct volopt_map_entry *vme = NULL;
+    struct opthandler_data odt = {
+        0,
+    };
+    data_t *data = NULL;
+    int keylen;
+
+    odt.graph = graph;
+    odt.handler = handler;
+    odt.param = param;
+    (void)data;
+
+    for (vme = glusterd_volopt_map; vme->key; vme++) {
+        keylen = strlen(vme->key);
+        if (keylen == SLEN("performance.client-io-threads") &&
+            !strcmp(vme->key, "performance.client-io-threads") &&
+            dict_get_str_boolean(dict, "skip-CLIOT", _gf_false) == _gf_true) {
+            continue;
         }
 
-        return 0;
-}
+        odt.vme = vme;
+        odt.found = _gf_false;
+        odt.data_t_fake = _gf_false;
+        data = dict_getn(dict, vme->key, keylen);
+        if (data)
+            process_option(vme->key, data, &odt);
+        if (odt.rv)
+            return odt.rv;
 
-static int
-no_filter_option_handler (volgen_graph_t *graph, struct volopt_map_entry *vme,
-                          void *param)
-{
-        xlator_t *trav;
-        int ret = 0;
+        if (odt.found)
+            continue;
 
-        for (trav = first_of (graph); trav; trav = trav->next) {
-                if (strcmp (trav->type, vme->voltype) != 0)
-                        continue;
+        /* check for default value */
 
-                ret = xlator_set_option (trav, vme->option, vme->value);
-                if (ret)
-                        break;
+        if (vme->value) {
+            /* stupid hack to be able to reuse dict iterator
+             * in this context
+             */
+            odt.data_t_fake = _gf_true;
+            process_option(vme->key, (data_t *)vme->value, &odt);
+            if (odt.rv)
+                return odt.rv;
         }
-        return ret;
+    }
+
+    return 0;
 }
 
 static int
-basic_option_handler (volgen_graph_t *graph, struct volopt_map_entry *vme,
-                      void *param)
+no_filter_option_handler(volgen_graph_t *graph, struct volopt_map_entry *vme,
+                         void *param)
+{
+    xlator_t *trav;
+    int ret = 0;
+
+    for (trav = first_of(graph); trav; trav = trav->next) {
+        if (strcmp(trav->type, vme->voltype) != 0)
+            continue;
+        if (strcmp(vme->option, "ta-remote-port") == 0) {
+            if (strstr(trav->name, "-ta-") != NULL) {
+                ret = xlator_set_option(trav, "remote-port",
+                                        strlen(vme->option), vme->value);
+            }
+            continue;
+        }
+        ret = xlator_set_option(trav, vme->option, strlen(vme->option),
+                                vme->value);
+        if (ret)
+            break;
+    }
+    return ret;
+}
+
+static int
+basic_option_handler(volgen_graph_t *graph, struct volopt_map_entry *vme,
+                     void *param)
 {
-        int     ret = 0;
+    int ret = 0;
 
-        if (vme->option[0] == '!')
-                goto out;
+    if (vme->option[0] == '!')
+        goto out;
 
-        ret = no_filter_option_handler (graph, vme, param);
+    ret = no_filter_option_handler(graph, vme, param);
 out:
-        return ret;
+    return ret;
 }
 
 static int
-volgen_graph_set_options (volgen_graph_t *graph, dict_t *dict)
+volgen_graph_set_options(volgen_graph_t *graph, dict_t *dict)
 {
-        return volgen_graph_set_options_generic (graph, dict, NULL,
-                                                 &basic_option_handler);
+    return volgen_graph_set_options_generic(graph, dict, NULL,
+                                            &basic_option_handler);
 }
 
 static int
-optget_option_handler (volgen_graph_t *graph, struct volopt_map_entry *vme,
-                       void *param)
+optget_option_handler(volgen_graph_t *graph, struct volopt_map_entry *vme,
+                      void *param)
 {
-        struct volopt_map_entry *vme2 = param;
+    struct volopt_map_entry *vme2 = param;
 
-        if (strcmp (vme->key, vme2->key) == 0)
-                vme2->value = vme->value;
+    if (strcmp(vme->key, vme2->key) == 0)
+        vme2->value = vme->value;
 
-        return 0;
+    return 0;
 }
 
 /* This getter considers defaults also. */
 static int
-volgen_dict_get (dict_t *dict, char *key, char **value)
+volgen_dict_get(dict_t *dict, char *key, char **value)
 {
-        struct volopt_map_entry vme = {0,};
-        int ret = 0;
+    struct volopt_map_entry vme = {
+        0,
+    };
+    int ret = 0;
 
-        vme.key = key;
+    vme.key = key;
 
-        ret = volgen_graph_set_options_generic (NULL, dict, &vme,
-                                                &optget_option_handler);
-        if (ret) {
-                gf_log ("", GF_LOG_ERROR, "Out of memory");
+    ret = volgen_graph_set_options_generic(NULL, dict, &vme,
+                                           &optget_option_handler);
+    if (ret) {
+        gf_msg("glusterd", GF_LOG_ERROR, ENOMEM, GD_MSG_NO_MEMORY,
+               "Out of memory");
 
-                return -1;
-        }
+        return -1;
+    }
 
-        *value = vme.value;
+    *value = vme.value;
 
-        return 0;
+    return 0;
 }
 
 static int
-option_complete (char *key, char **completion)
+option_complete(char *key, char **completion)
 {
-        struct volopt_map_entry *vme = NULL;
+    struct volopt_map_entry *vme = NULL;
 
-        *completion = NULL;
-        for (vme = glusterd_volopt_map; vme->key; vme++) {
-                if (strcmp (strchr (vme->key, '.') + 1, key) != 0)
-                        continue;
+    *completion = NULL;
+    for (vme = glusterd_volopt_map; vme->key; vme++) {
+        if (strcmp(strchr(vme->key, '.') + 1, key) != 0)
+            continue;
 
-                if (*completion && strcmp (*completion, vme->key) != 0) {
-                        /* cancel on non-unique match */
-                        *completion = NULL;
+        if (*completion && strcmp(*completion, vme->key) != 0) {
+            /* cancel on non-unique match */
+            *completion = NULL;
 
-                        return 0;
-                } else
-                        *completion = vme->key;
-        }
+            return 0;
+        } else
+            *completion = vme->key;
+    }
 
-        if (*completion) {
-                /* For sake of unified API we want
-                 * have the completion to be a to-be-freed
-                 * string.
-                 */
-                *completion = gf_strdup (*completion);
-                return -!*completion;
-        }
+    if (*completion) {
+        /* For sake of unified API we want
+         * have the completion to be a to-be-freed
+         * string.
+         */
+        *completion = gf_strdup(*completion);
+        return -!*completion;
+    }
 
-        return 0;
+    return 0;
 }
 
 int
-glusterd_volinfo_get (glusterd_volinfo_t *volinfo, char *key, char **value)
+glusterd_volinfo_get(glusterd_volinfo_t *volinfo, char *key, char **value)
 {
-        return volgen_dict_get (volinfo->dict, key, value);
+    return volgen_dict_get(volinfo->dict, key, value);
 }
 
 int
-glusterd_volinfo_get_boolean (glusterd_volinfo_t *volinfo, char *key)
+glusterd_volinfo_get_boolean(glusterd_volinfo_t *volinfo, char *key)
 {
-        char *val = NULL;
-        gf_boolean_t  enabled = _gf_false;
-        int ret = 0;
+    char *val = NULL;
+    gf_boolean_t enabled = _gf_false;
+    int ret = 0;
 
-        ret = glusterd_volinfo_get (volinfo, key, &val);
-        if (ret)
-                return -1;
+    ret = glusterd_volinfo_get(volinfo, key, &val);
+    if (ret)
+        return -1;
 
-        if (val)
-                ret = gf_string2boolean (val, &enabled);
-        if (ret) {
-                gf_log ("", GF_LOG_ERROR, "value for %s option is not valid", key);
+    if (val)
+        ret = gf_string2boolean(val, &enabled);
+    if (ret) {
+        gf_msg("glusterd", GF_LOG_ERROR, EINVAL, GD_MSG_INVALID_ENTRY,
+               "value for %s option is not valid", key);
 
-                return -1;
-        }
+        return -1;
+    }
 
-        return enabled;
+    return enabled;
 }
 
 gf_boolean_t
-glusterd_check_voloption_flags (char *key, int32_t flags)
-{
-        char *completion = NULL;
-        struct volopt_map_entry *vmep = NULL;
-        int   ret = 0;
-
-        COMPLETE_OPTION(key, completion, ret);
-        for (vmep = glusterd_volopt_map; vmep->key; vmep++) {
-                if (strcmp (vmep->key, key) == 0) {
-                        if (vmep->flags & flags)
-                                return _gf_true;
-                        else
-                                return _gf_false;
-                }
+glusterd_check_voloption_flags(char *key, int32_t flags)
+{
+    char *completion = NULL;
+    struct volopt_map_entry *vmep = NULL;
+    int ret = 0;
+
+    COMPLETE_OPTION(key, completion, ret);
+    for (vmep = glusterd_volopt_map; vmep->key; vmep++) {
+        if (strcmp(vmep->key, key) == 0) {
+            if (vmep->flags & flags)
+                return _gf_true;
+            else
+                return _gf_false;
         }
+    }
 
-        return _gf_false;
+    return _gf_false;
 }
 
 gf_boolean_t
-glusterd_check_globaloption (char *key)
-{
-        char *completion = NULL;
-        struct volopt_map_entry *vmep = NULL;
-        int   ret = 0;
-
-        COMPLETE_OPTION(key, completion, ret);
-        for (vmep = glusterd_volopt_map; vmep->key; vmep++) {
-                if (strcmp (vmep->key, key) == 0) {
-                        if ((vmep->type == GLOBAL_DOC) ||
-                            (vmep->type == GLOBAL_NO_DOC))
-                                return _gf_true;
-                        else
-                                return _gf_false;
-                }
+glusterd_check_globaloption(char *key)
+{
+    char *completion = NULL;
+    struct volopt_map_entry *vmep = NULL;
+    int ret = 0;
+
+    COMPLETE_OPTION(key, completion, ret);
+    for (vmep = glusterd_volopt_map; vmep->key; vmep++) {
+        if (strcmp(vmep->key, key) == 0) {
+            if ((vmep->type == GLOBAL_DOC) || (vmep->type == GLOBAL_NO_DOC))
+                return _gf_true;
+            else
+                return _gf_false;
         }
+    }
 
-        return _gf_false;
+    return _gf_false;
 }
 
 gf_boolean_t
-glusterd_check_localoption (char *key)
-{
-        char *completion = NULL;
-        struct volopt_map_entry *vmep = NULL;
-        int   ret = 0;
-
-        COMPLETE_OPTION(key, completion, ret);
-        for (vmep = glusterd_volopt_map; vmep->key; vmep++) {
-                if (strcmp (vmep->key, key) == 0) {
-                        if ((vmep->type == DOC) ||
-                            (vmep->type == NO_DOC))
-                                return _gf_true;
-                        else
-                                return _gf_false;
-                }
-        }
-
-        return _gf_false;
-}
+glusterd_check_localoption(char *key)
+{
+    char *completion = NULL;
+    struct volopt_map_entry *vmep = NULL;
+    int ret = 0;
 
-int
-glusterd_check_voloption (char *key)
-{
-        char *completion = NULL;
-        struct volopt_map_entry *vmep = NULL;
-        int ret = 0;
-
-        COMPLETE_OPTION(key, completion, ret);
-        for (vmep = glusterd_volopt_map; vmep->key; vmep++) {
-                if (strcmp (vmep->key, key) == 0) {
-                        if ((vmep->type == DOC) ||
-                            (vmep->type == DOC))
-                                return _gf_true;
-                        else
-                                return _gf_false;
-                }
+    COMPLETE_OPTION(key, completion, ret);
+    for (vmep = glusterd_volopt_map; vmep->key; vmep++) {
+        if (strcmp(vmep->key, key) == 0) {
+            if ((vmep->type == DOC) || (vmep->type == NO_DOC))
+                return _gf_true;
+            else
+                return _gf_false;
         }
+    }
 
-        return _gf_false;
-
+    return _gf_false;
 }
 
 int
-glusterd_check_option_exists (char *key, char **completion)
-{
-        struct volopt_map_entry vme = {0,};
-        struct volopt_map_entry *vmep = NULL;
-        int ret = 0;
-        xlator_t *this = THIS;
-
-        (void)vme;
-        (void)vmep;
-
-        if (!strchr (key, '.')) {
-                if (completion) {
-                        ret = option_complete (key, completion);
-                        if (ret) {
-                                gf_log (this->name, GF_LOG_ERROR,
-                                        "Out of memory");
-                                return -1;
-                        }
-
-                        ret = !!*completion;
-                        if (ret)
-                                return ret;
-                        else
-                                goto trie;
-                } else
-                        return 0;
-        }
-
-        for (vmep = glusterd_volopt_map; vmep->key; vmep++) {
-                if (strcmp (vmep->key, key) == 0) {
-                        ret = 1;
-                        break;
-                }
-        }
+glusterd_check_option_exists(char *key, char **completion)
+{
+    struct volopt_map_entry vme = {
+        0,
+    };
+    struct volopt_map_entry *vmep = NULL;
+    int ret = 0;
+    xlator_t *this = THIS;
+
+    (void)vme;
+    (void)vmep;
+
+    if (!strchr(key, '.')) {
+        if (completion) {
+            ret = option_complete(key, completion);
+            if (ret) {
+                gf_msg(this->name, GF_LOG_ERROR, ENOMEM, GD_MSG_NO_MEMORY,
+                       "Out of memory");
+                return -1;
+            }
 
-        if (ret || !completion)
+            ret = !!*completion;
+            if (ret)
                 return ret;
+            else
+                goto trie;
+        } else
+            return 0;
+    }
 
- trie:
-        ret = volopt_trie (key, completion);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "Some error occurred during keyword hinting");
+    for (vmep = glusterd_volopt_map; vmep->key; vmep++) {
+        if (strcmp(vmep->key, key) == 0) {
+            ret = 1;
+            break;
         }
+    }
 
+    if (ret || !completion)
         return ret;
+
+trie:
+    ret = volopt_trie(key, completion);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, errno, GD_MSG_ERROR_ENCOUNTERED,
+               "Some error occurred during keyword hinting");
+    }
+
+    return ret;
 }
 
 int
-glusterd_volopt_validate (glusterd_volinfo_t *volinfo, dict_t *dict, char *key,
-                          char *value, char **op_errstr)
-{
-        struct volopt_map_entry *vme     = NULL;
-        char                    *volname = NULL;
-        int                     ret = 0;
-        xlator_t                *this = THIS;
-
-        if (!dict || !key || !value) {
-                gf_log_callingfn (this->name, GF_LOG_WARNING, "Invalid "
-                                  "Arguments (dict=%p, key=%s, value=%s)", dict,
-                                  key, value);
-                return -1;
-        }
-
-        for (vme = &glusterd_volopt_map[0]; vme->key; vme++) {
-                if ((vme->validate_fn) &&
-                    ((!strcmp (key, vme->key)) ||
-                     (!strcmp (key, strchr (vme->key, '.') + 1)))) {
-                        ret = vme->validate_fn (volinfo, dict, key, value,
-                                                op_errstr);
-                        if (ret)
-                                goto out;
-                        break;
-                }
+glusterd_volopt_validate(glusterd_volinfo_t *volinfo, dict_t *dict, char *key,
+                         char *value, char **op_errstr)
+{
+    struct volopt_map_entry *vme = NULL;
+    int ret = 0;
+    xlator_t *this = THIS;
+
+    if (!dict || !key || !value) {
+        gf_msg_callingfn(this->name, GF_LOG_WARNING, EINVAL,
+                         GD_MSG_INVALID_ENTRY,
+                         "Invalid "
+                         "Arguments (dict=%p, key=%s, value=%s)",
+                         dict, key, value);
+        return -1;
+    }
+
+    for (vme = &glusterd_volopt_map[0]; vme->key; vme++) {
+        if ((vme->validate_fn) && ((!strcmp(key, vme->key)) ||
+                                   (!strcmp(key, strchr(vme->key, '.') + 1)))) {
+            if ((vme->type != GLOBAL_DOC && vme->type != GLOBAL_NO_DOC) &&
+                !volinfo) {
+                gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_INVALID_ENTRY,
+                       "%s is not"
+                       " a global option",
+                       vme->key);
+                ret = -1;
+                goto out;
+            }
+            ret = vme->validate_fn(volinfo, dict, key, value, op_errstr);
+            if (ret)
+                goto out;
+            break;
         }
+    }
 out:
-        return ret;
+    return ret;
 }
 
-char*
-glusterd_get_trans_type_rb (gf_transport_type ttype)
+char *
+glusterd_get_trans_type_rb(gf_transport_type ttype)
 {
-        char *trans_type = NULL;
+    char *trans_type = NULL;
 
-        switch (ttype) {
+    switch (ttype) {
         case GF_TRANSPORT_RDMA:
-                gf_asprintf (&trans_type, "rdma");
-                break;
+            gf_asprintf(&trans_type, "rdma");
+            break;
         case GF_TRANSPORT_TCP:
         case GF_TRANSPORT_BOTH_TCP_RDMA:
-                gf_asprintf (&trans_type, "tcp");
-                break;
+            gf_asprintf(&trans_type, "tcp");
+            break;
         default:
-                gf_log (THIS->name, GF_LOG_ERROR, "Unknown "
-                        "transport type");
-        }
+            gf_msg(THIS->name, GF_LOG_ERROR, EINVAL, GD_MSG_INVALID_ENTRY,
+                   "Unknown "
+                   "transport type");
+    }
 
-        return trans_type;
+    return trans_type;
 }
 
 static int
-_xl_link_children (xlator_t *parent, xlator_t *children, size_t child_count)
-{
-        xlator_t        *trav = NULL;
-        size_t          seek = 0;
-        int             ret = -1;
-
-        if (child_count == 0)
-                goto out;
-        seek = child_count;
-        for (trav = children; --seek; trav = trav->next);
-        for (; child_count--; trav = trav->prev) {
-                ret = volgen_xlator_link (parent, trav);
-                gf_log (THIS->name, GF_LOG_DEBUG, "%s:%s", parent->name,
-                        trav->name);
-                if (ret)
-                        goto out;
+_xl_link_children(xlator_t *parent, xlator_t *children, size_t child_count)
+{
+    xlator_t *trav = NULL;
+    size_t seek = 0;
+    int ret = -1;
+    xlator_t *this = THIS;
+    GF_ASSERT(this);
+
+    if (child_count == 0)
+        goto out;
+    seek = child_count;
+    for (trav = children; --seek; trav = trav->next)
+        ;
+    for (; child_count--; trav = trav->prev) {
+        ret = volgen_xlator_link(parent, trav);
+        gf_msg_debug(this->name, 0, "%s:%s", parent->name, trav->name);
+        if (ret) {
+            gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_XLATOR_LINK_FAIL,
+                    NULL);
+            goto out;
         }
-        ret = 0;
+    }
+    ret = 0;
 out:
-        return ret;
+    return ret;
 }
 
 static int
-volgen_graph_merge_sub (volgen_graph_t *dgraph, volgen_graph_t *sgraph,
-                        size_t child_count)
+volgen_graph_merge_sub(volgen_graph_t *dgraph, volgen_graph_t *sgraph,
+                       size_t child_count)
 {
-        xlator_t *trav = NULL;
-        int      ret   = 0;
+    xlator_t *trav = NULL;
+    int ret = 0;
 
-        GF_ASSERT (dgraph->graph.first);
+    GF_ASSERT(dgraph->graph.first);
 
-        ret = _xl_link_children (first_of (dgraph), first_of (sgraph),
-                                 child_count);
-        if (ret)
-                goto out;
+    ret = _xl_link_children(first_of(dgraph), first_of(sgraph), child_count);
+    if (ret)
+        goto out;
 
-        for (trav = first_of (dgraph); trav->next; trav = trav->next);
+    for (trav = first_of(dgraph); trav->next; trav = trav->next)
+        ;
 
-        trav->next = first_of (sgraph);
-        trav->next->prev = trav;
-        dgraph->graph.xl_count += sgraph->graph.xl_count;
+    trav->next = first_of(sgraph);
+    trav->next->prev = trav;
+    dgraph->graph.xl_count += sgraph->graph.xl_count;
 
 out:
-        return ret;
+    return ret;
 }
 
 static void
-volgen_apply_filters (char *orig_volfile)
-{
-        DIR           *filterdir = NULL;
-        struct dirent  entry = {0,};
-        struct dirent *next = NULL;
-        char          *filterpath = NULL;
-        struct stat    statbuf = {0,};
+volgen_apply_filters(char *orig_volfile)
+{
+    DIR *filterdir = NULL;
+    struct dirent *entry = NULL;
+    struct dirent scratch[2] = {
+        {
+            0,
+        },
+    };
+    struct stat statbuf = {
+        0,
+    };
+    char filterpath[PATH_MAX] = {
+        0,
+    };
+
+    filterdir = sys_opendir(FILTERDIR);
+
+    if (!filterdir)
+        return;
+
+    for (;;) {
+        errno = 0;
+
+        entry = sys_readdir(filterdir, scratch);
 
-        filterdir = opendir(FILTERDIR);
-        if (!filterdir) {
-                return;
+        if (!entry || errno != 0) {
+            gf_smsg("glusterd", GF_LOG_ERROR, errno, GD_MSG_READ_ERROR, NULL);
+            break;
         }
 
-        while ((readdir_r(filterdir,&entry,&next) == 0) && next) {
-                if (!strncmp(entry.d_name,".",sizeof(entry.d_name))) {
-                        continue;
-                }
-                if (!strncmp(entry.d_name,"..",sizeof(entry.d_name))) {
-                        continue;
-                }
-                /*
-                 * d_type isn't guaranteed to be present/valid on all systems,
-                 * so do an explicit stat instead.
-                 */
-                if (gf_asprintf(&filterpath,"%s/%.*s",FILTERDIR,
-                                sizeof(entry.d_name), entry.d_name) == (-1)) {
-                        continue;
-                }
-                /* Deliberately use stat instead of lstat to allow symlinks. */
-                if (stat(filterpath,&statbuf) == (-1)) {
-                        goto free_fp;
-                }
-                if (!S_ISREG(statbuf.st_mode)) {
-                        goto free_fp;
-                }
-                /*
-                 * We could check the mode in statbuf directly, or just skip
-                 * this entirely and check for EPERM after exec fails, but this
-                 * is cleaner.
-                 */
-                if (access(filterpath,X_OK) != 0) {
-                        goto free_fp;
-                }
-                if (runcmd(filterpath,orig_volfile,NULL)) {
-                        gf_log("",GF_LOG_ERROR,"failed to run filter %.*s",
-                               (int)sizeof(entry.d_name), entry.d_name);
-                }
-free_fp:
-                GF_FREE(filterpath);
+        if (strcmp(entry->d_name, ".") == 0 || strcmp(entry->d_name, "..") == 0)
+            continue;
+        /*
+         * d_type isn't guaranteed to be present/valid on all systems,
+         * so do an explicit stat instead.
+         */
+        (void)snprintf(filterpath, sizeof(filterpath), "%s/%s", FILTERDIR,
+                       entry->d_name);
+
+        /* Deliberately use stat instead of lstat to allow symlinks. */
+        if (sys_stat(filterpath, &statbuf) == -1)
+            continue;
+
+        if (!S_ISREG(statbuf.st_mode))
+            continue;
+        /*
+         * We could check the mode in statbuf directly, or just skip
+         * this entirely and check for EPERM after exec fails, but this
+         * is cleaner.
+         */
+        if (sys_access(filterpath, X_OK) != 0)
+            continue;
+
+        if (runcmd(filterpath, orig_volfile, NULL)) {
+            gf_msg("glusterd", GF_LOG_ERROR, 0, GD_MSG_FILTER_RUN_FAILED,
+                   "failed to run filter %s", entry->d_name);
         }
+    }
 
-        closedir (filterdir);
+    (void)sys_closedir(filterdir);
 }
 
 static int
-volgen_write_volfile (volgen_graph_t *graph, char *filename)
+volgen_write_volfile(volgen_graph_t *graph, char *filename)
 {
-        char        *ftmp = NULL;
-        FILE        *f = NULL;
-        int          fd   = 0;
-        xlator_t    *this = NULL;
+    char *ftmp = NULL;
+    FILE *f = NULL;
+    int fd = 0;
+    xlator_t *this = NULL;
 
-        this = THIS;
+    this = THIS;
 
-        if (gf_asprintf (&ftmp, "%s.tmp", filename) == -1) {
-                ftmp = NULL;
+    if (gf_asprintf(&ftmp, "%s.tmp", filename) == -1) {
+        ftmp = NULL;
+        goto error;
+    }
 
-                goto error;
-        }
+    fd = sys_creat(ftmp, S_IRUSR | S_IWUSR);
+    if (fd < 0) {
+        gf_msg(this->name, GF_LOG_ERROR, errno, GD_MSG_FILE_OP_FAILED,
+               "file creation failed");
+        goto error;
+    }
 
-        fd = creat (ftmp, S_IRUSR | S_IWUSR);
-        if (fd < 0) {
-                gf_log (this->name, GF_LOG_ERROR, "%s",
-                        strerror (errno));
-                goto error;
-        }
+    sys_close(fd);
 
-        close (fd);
-
-        f = fopen (ftmp, "w");
-        if (!f)
-                goto error;
-
-        if (glusterfs_graph_print_file (f, &graph->graph) == -1)
-                goto error;
-
-        if (fclose (f) != 0) {
-                gf_log (THIS->name, GF_LOG_ERROR, "fclose on the file %s "
-                        "failed (%s)", ftmp, strerror (errno));
-                /*
-                 * Even though fclose has failed here, we have to set f to NULL.
-                 * Otherwise when the code path goes to error, there again we
-                 * try to close it which might cause undefined behavior such as
-                 * process crash.
-                 */
-                f = NULL;
-                goto error;
-        }
+    f = fopen(ftmp, "w");
+    if (!f)
+        goto error;
+
+    if (glusterfs_graph_print_file(f, &graph->graph) == -1)
+        goto error;
 
+    if (fclose(f) != 0) {
+        gf_msg(THIS->name, GF_LOG_ERROR, errno, GD_MSG_FILE_OP_FAILED,
+               "fclose on the file %s "
+               "failed",
+               ftmp);
+        /*
+         * Even though fclose has failed here, we have to set f to NULL.
+         * Otherwise when the code path goes to error, there again we
+         * try to close it which might cause undefined behavior such as
+         * process crash.
+         */
         f = NULL;
+        goto error;
+    }
 
-        if (rename (ftmp, filename) == -1)
-                goto error;
+    f = NULL;
 
-        GF_FREE (ftmp);
+    if (sys_rename(ftmp, filename) == -1)
+        goto error;
 
-        volgen_apply_filters(filename);
+    GF_FREE(ftmp);
 
-        return 0;
+    volgen_apply_filters(filename);
 
- error:
+    return 0;
 
-        GF_FREE (ftmp);
-        if (f)
-                fclose (f);
+error:
 
-        gf_log (this->name, GF_LOG_ERROR,
-                "failed to create volfile %s", filename);
+    GF_FREE(ftmp);
+    if (f)
+        fclose(f);
 
-        return -1;
+    gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_VOLFILE_CREATE_FAIL,
+           "failed to create volfile %s", filename);
+
+    return -1;
 }
 
 static void
-volgen_graph_free (volgen_graph_t *graph)
+volgen_graph_free(volgen_graph_t *graph)
 {
-        xlator_t *trav = NULL;
-        xlator_t *trav_old = NULL;
+    xlator_t *trav = NULL;
+    xlator_t *trav_old = NULL;
 
-        for (trav = first_of (graph) ;; trav = trav->next) {
-                if (trav_old)
-                        xlator_destroy (trav_old);
+    for (trav = first_of(graph);; trav = trav->next) {
+        if (trav_old)
+            xlator_destroy(trav_old);
 
-                trav_old = trav;
+        trav_old = trav;
 
-                if (!trav)
-                        break;
-        }
+        if (!trav)
+            break;
+    }
 }
 
 static int
-build_graph_generic (volgen_graph_t *graph, glusterd_volinfo_t *volinfo,
-                     dict_t *mod_dict, void *param,
-                     int (*builder) (volgen_graph_t *graph,
-                                     glusterd_volinfo_t *volinfo,
-                                     dict_t *set_dict, void *param))
+build_graph_generic(volgen_graph_t *graph, glusterd_volinfo_t *volinfo,
+                    dict_t *mod_dict, void *param,
+                    int (*builder)(volgen_graph_t *graph,
+                                   glusterd_volinfo_t *volinfo,
+                                   dict_t *set_dict, void *param))
 {
-        dict_t *set_dict = NULL;
-        int ret = 0;
+    dict_t *set_dict = NULL;
+    int ret = 0;
 
-        if (mod_dict) {
-                set_dict = dict_copy (volinfo->dict, NULL);
-                if (!set_dict)
-                        return -1;
-                 dict_copy (mod_dict, set_dict);
-                 /* XXX dict_copy swallows errors */
-        } else {
-                set_dict = volinfo->dict;
-        }
+    if (mod_dict) {
+        set_dict = dict_copy_with_ref(volinfo->dict, NULL);
+        if (!set_dict)
+            return -1;
+        dict_copy(mod_dict, set_dict);
+        /* XXX dict_copy swallows errors */
+    } else {
+        set_dict = volinfo->dict;
+    }
 
-        ret = builder (graph, volinfo, set_dict, param);
-        if (!ret)
-                ret = volgen_graph_set_options (graph, set_dict);
+    ret = builder(graph, volinfo, set_dict, param);
+    if (!ret)
+        ret = volgen_graph_set_options(graph, set_dict);
 
-        if (mod_dict)
-                dict_destroy (set_dict);
+    if (mod_dict)
+        dict_unref(set_dict);
 
-        return ret;
+    return ret;
 }
 
 static gf_transport_type
-transport_str_to_type (char *tt)
+transport_str_to_type(char *tt)
 {
-        gf_transport_type type = GF_TRANSPORT_TCP;
+    gf_transport_type type = GF_TRANSPORT_TCP;
 
-        if (!strcmp ("tcp", tt))
-                type = GF_TRANSPORT_TCP;
-        else if (!strcmp ("rdma", tt))
-                type = GF_TRANSPORT_RDMA;
-        else if (!strcmp ("tcp,rdma", tt))
-                type = GF_TRANSPORT_BOTH_TCP_RDMA;
-        return type;
+    if (!strcmp("tcp", tt))
+        type = GF_TRANSPORT_TCP;
+    else if (!strcmp("rdma", tt))
+        type = GF_TRANSPORT_RDMA;
+    else if (!strcmp("tcp,rdma", tt))
+        type = GF_TRANSPORT_BOTH_TCP_RDMA;
+    return type;
 }
 
 static void
-transport_type_to_str (gf_transport_type type, char *tt)
+transport_type_to_str(gf_transport_type type, char *tt)
 {
-        switch (type) {
+    switch (type) {
         case GF_TRANSPORT_RDMA:
-                strcpy (tt, "rdma");
-                break;
+            strcpy(tt, "rdma");
+            break;
         case GF_TRANSPORT_TCP:
-                strcpy (tt, "tcp");
-                break;
+            strcpy(tt, "tcp");
+            break;
         case GF_TRANSPORT_BOTH_TCP_RDMA:
-                strcpy (tt, "tcp,rdma");
-                break;
-        }
+            strcpy(tt, "tcp,rdma");
+            break;
+    }
 }
 
 static void
-get_vol_transport_type (glusterd_volinfo_t *volinfo, char *tt)
+get_vol_transport_type(glusterd_volinfo_t *volinfo, char *tt)
 {
-        transport_type_to_str (volinfo->transport_type, tt);
+    transport_type_to_str(volinfo->transport_type, tt);
 }
 
-/* If no value has specfied for tcp,rdma volume from cli
+#ifdef BUILD_GNFS
+/* If no value has specified for tcp,rdma volume from cli
  * use tcp as default value.Otherwise, use transport type
  * mentioned in volinfo
  */
 static void
-get_vol_nfs_transport_type (glusterd_volinfo_t *volinfo, char *tt)
+get_vol_nfs_transport_type(glusterd_volinfo_t *volinfo, char *tt)
 {
-        if (volinfo->transport_type == GF_TRANSPORT_BOTH_TCP_RDMA) {
-                strcpy (tt, "tcp");
-                gf_log ("glusterd", GF_LOG_INFO,
-                        "The default transport type for tcp,rdma volume "
-                        "is tcp if option is not defined by the user ");
-        } else
-                transport_type_to_str (volinfo->transport_type, tt);
+    if (volinfo->transport_type == GF_TRANSPORT_BOTH_TCP_RDMA) {
+        strcpy(tt, "tcp");
+        gf_msg("glusterd", GF_LOG_INFO, 0, GD_MSG_DEFAULT_OPT_INFO,
+               "The default transport type for tcp,rdma volume "
+               "is tcp if option is not defined by the user ");
+    } else
+        transport_type_to_str(volinfo->transport_type, tt);
 }
+#endif
 
 /*  gets the volinfo, dict, a character array for filling in
  *  the transport type and a boolean option which says whether
@@ -1111,1718 +1161,2271 @@ get_vol_nfs_transport_type (glusterd_volinfo_t *volinfo, char *tt)
  *  and client transport type is filled in the character array
  */
 static void
-get_transport_type (glusterd_volinfo_t *volinfo, dict_t *set_dict,
-                    char *transt, gf_boolean_t is_nfs)
-{
-        int    ret   = -1;
-        char  *tt    = NULL;
-        char  *key   = NULL;
-        typedef void (*transport_type) (glusterd_volinfo_t *volinfo, char *tt);
-        transport_type get_transport;
-
-        if (is_nfs == _gf_false) {
-                key = "client-transport-type";
-                get_transport = get_vol_transport_type;
-        } else {
-                key = "nfs.transport-type";
-                get_transport = get_vol_nfs_transport_type;
-        }
+get_transport_type(glusterd_volinfo_t *volinfo, dict_t *set_dict, char *transt,
+                   gf_boolean_t is_nfs)
+{
+    int ret = -1;
+    char *tt = NULL;
 
-        ret = dict_get_str (set_dict, key, &tt);
+    if (is_nfs == _gf_false) {
+        ret = dict_get_str_sizen(set_dict, "client-transport-type", &tt);
+        if (ret)
+            get_vol_transport_type(volinfo, transt);
+    } else {
+#ifdef BUILD_GNFS
+        ret = dict_get_str_sizen(set_dict, "nfs.transport-type", &tt);
         if (ret)
-                get_transport (volinfo, transt);
-        if (!ret)
-                strcpy (transt, tt);
+            get_vol_nfs_transport_type(volinfo, transt);
+#endif
+    }
+
+    if (!ret)
+        strcpy(transt, tt);
 }
 
 static int
-server_auth_option_handler (volgen_graph_t *graph,
-                            struct volopt_map_entry *vme, void *param)
+server_auth_option_handler(volgen_graph_t *graph, struct volopt_map_entry *vme,
+                           void *param)
 {
-        xlator_t *xl = NULL;
-        xlator_list_t *trav = NULL;
-        char *aa = NULL;
-        int   ret   = 0;
-        char *key = NULL;
+    xlator_t *xl = NULL;
+    char *aa = NULL;
+    int ret = 0;
+    char *key = NULL;
+    char *auth_path = NULL;
 
-        if (strcmp (vme->option, "!server-auth") != 0)
-                return 0;
+    if (strcmp(vme->option, "!server-auth") != 0)
+        return 0;
 
-        xl = first_of (graph);
+    xl = first_of(graph);
 
-        /* from 'auth.allow' -> 'allow', and 'auth.reject' -> 'reject' */
-        key = strchr (vme->key, '.') + 1;
+    /* from 'auth.allow' -> 'allow', and 'auth.reject' -> 'reject' */
+    key = strchr(vme->key, '.') + 1;
 
-        for (trav = xl->children; trav; trav = trav->next) {
-                ret = gf_asprintf (&aa, "auth.addr.%s.%s", trav->xlator->name,
-                                   key);
-                if (ret != -1) {
-                        ret = xlator_set_option (xl, aa, vme->value);
-                        GF_FREE (aa);
-                }
-                if (ret)
-                        return -1;
-        }
+    ret = xlator_get_fixed_option(xl, "auth-path", &auth_path);
+    if (ret) {
+        gf_msg("glusterd", GF_LOG_ERROR, 0, GD_MSG_DEFAULT_OPT_INFO,
+               "Failed to get auth-path from server graph");
+        return -1;
+    }
+    ret = gf_asprintf(&aa, "auth.addr.%s.%s", auth_path, key);
+    if (ret != -1) {
+        ret = xlator_set_option(xl, aa, ret, vme->value);
+        GF_FREE(aa);
+    }
+    if (ret)
+        return -1;
 
-        return 0;
+    return 0;
 }
 
 static int
-loglevel_option_handler (volgen_graph_t *graph,
-                         struct volopt_map_entry *vme, void *param)
+loglevel_option_handler(volgen_graph_t *graph, struct volopt_map_entry *vme,
+                        void *param)
 {
-        char *role = param;
-        struct volopt_map_entry vme2 = {0,};
+    char *role = param;
+    struct volopt_map_entry vme2 = {
+        0,
+    };
 
-        if ( (strcmp (vme->option, "!client-log-level") != 0 &&
-               strcmp (vme->option, "!brick-log-level") != 0)
-            || !strstr (vme->key, role))
-                return 0;
+    if ((strcmp(vme->option, "!client-log-level") != 0 &&
+         strcmp(vme->option, "!brick-log-level") != 0) ||
+        !strstr(vme->key, role))
+        return 0;
 
-        memcpy (&vme2, vme, sizeof (vme2));
-        vme2.option = "log-level";
+    memcpy(&vme2, vme, sizeof(vme2));
+    vme2.option = "log-level";
 
-        return basic_option_handler (graph, &vme2, NULL);
+    return basic_option_handler(graph, &vme2, NULL);
 }
 
 static int
-server_check_marker_off (volgen_graph_t *graph, struct volopt_map_entry *vme,
-                         glusterd_volinfo_t *volinfo)
+threads_option_handler(volgen_graph_t *graph, struct volopt_map_entry *vme,
+                       void *param)
 {
-        gf_boolean_t enabled = _gf_false;
-        int ret = 0;
-
-        GF_ASSERT (volinfo);
-        GF_ASSERT (vme);
-
-        if (strcmp (vme->option, "!xtime") != 0)
-                return 0;
-
-        ret = gf_string2boolean (vme->value, &enabled);
-        if (ret || enabled)
-                goto out;
-
-        ret = glusterd_volinfo_get_boolean (volinfo, VKEY_MARKER_XTIME);
-        if (ret < 0) {
-                gf_log ("", GF_LOG_WARNING, "failed to get the marker status");
-                ret = -1;
-                goto out;
-        }
+    char *role = param;
+    struct volopt_map_entry vme2 = {
+        0,
+    };
 
-        if (ret) {
-                enabled = _gf_false;
-                glusterd_check_geo_rep_configured (volinfo, &enabled);
-
-                if (enabled) {
-                        gf_log ("", GF_LOG_WARNING, GEOREP" sessions active"
-                                "for the volume %s, cannot disable marker "
-                                ,volinfo->volname);
-                        set_graph_errstr (graph,
-                                          VKEY_MARKER_XTIME" cannot be disabled "
-                                          "while "GEOREP" sessions exist");
-                        ret = -1;
-                        goto out;
-                }
-        }
+    if ((strcmp(vme->option, "!client-threads") != 0 &&
+         strcmp(vme->option, "!brick-threads") != 0) ||
+        !strstr(vme->key, role))
+        return 0;
 
-        ret = 0;
- out:
-        gf_log ("", GF_LOG_DEBUG, "Returning %d", ret);
-        return ret;
+    memcpy(&vme2, vme, sizeof(vme2));
+    vme2.option = "threads";
 
+    return basic_option_handler(graph, &vme2, NULL);
 }
 
 static int
-sys_loglevel_option_handler (volgen_graph_t *graph,
-                             struct volopt_map_entry *vme,
-                             void *param)
+server_check_changelog_off(volgen_graph_t *graph, struct volopt_map_entry *vme,
+                           glusterd_volinfo_t *volinfo)
 {
-        char  *role = NULL;
-        struct volopt_map_entry vme2 = {0,};
+    gf_boolean_t enabled = _gf_false;
+    int ret = 0;
 
-        role = (char *) param;
+    GF_ASSERT(volinfo);
+    GF_ASSERT(vme);
 
-        if (strcmp (vme->option, "!sys-log-level") != 0 ||
-            !strstr (vme->key, role))
-                return 0;
+    if (strcmp(vme->option, "changelog") != 0)
+        return 0;
 
-        memcpy (&vme2, vme, sizeof (vme2));
-        vme2.option = "sys-log-level";
+    ret = gf_string2boolean(vme->value, &enabled);
+    if (ret || enabled)
+        goto out;
 
-        return basic_option_handler (graph, &vme2, NULL);
+    ret = glusterd_volinfo_get_boolean(volinfo, VKEY_CHANGELOG);
+    if (ret < 0) {
+        gf_msg("glusterd", GF_LOG_WARNING, 0, GD_MSG_CHANGELOG_GET_FAIL,
+               "failed to get the changelog status");
+        ret = -1;
+        goto out;
+    }
+
+    if (ret) {
+        enabled = _gf_false;
+        glusterd_check_geo_rep_configured(volinfo, &enabled);
+
+        if (enabled) {
+            gf_msg("glusterd", GF_LOG_WARNING, 0, GD_MSG_XLATOR_SET_OPT_FAIL,
+                   GEOREP
+                   " sessions active"
+                   "for the volume %s, cannot disable changelog ",
+                   volinfo->volname);
+            set_graph_errstr(graph, VKEY_CHANGELOG
+                             " cannot be disabled "
+                             "while " GEOREP " sessions exist");
+            ret = -1;
+            goto out;
+        }
+    }
+
+    ret = 0;
+out:
+    gf_msg_debug("glusterd", 0, "Returning %d", ret);
+    return ret;
 }
 
 static int
-logger_option_handler (volgen_graph_t *graph, struct volopt_map_entry *vme,
-                       void *param)
+server_check_marker_off(volgen_graph_t *graph, struct volopt_map_entry *vme,
+                        glusterd_volinfo_t *volinfo)
 {
-        char  *role = NULL;
-        struct volopt_map_entry vme2 = {0,};
+    gf_boolean_t enabled = _gf_false;
+    int ret = 0;
 
-        role = (char *) param;
+    GF_ASSERT(volinfo);
+    GF_ASSERT(vme);
 
-        if (strcmp (vme->option, "!logger") != 0 ||
-            !strstr (vme->key, role))
-                return 0;
+    if (strcmp(vme->option, "!xtime") != 0)
+        return 0;
 
-        memcpy (&vme2, vme, sizeof (vme2));
-        vme2.option = "logger";
+    ret = gf_string2boolean(vme->value, &enabled);
+    if (ret || enabled)
+        goto out;
 
-        return basic_option_handler (graph, &vme2, NULL);
+    ret = glusterd_volinfo_get_boolean(volinfo, VKEY_MARKER_XTIME);
+    if (ret < 0) {
+        gf_msg("glusterd", GF_LOG_WARNING, 0, GD_MSG_MARKER_STATUS_GET_FAIL,
+               "failed to get the marker status");
+        ret = -1;
+        goto out;
+    }
+
+    if (ret) {
+        enabled = _gf_false;
+        glusterd_check_geo_rep_configured(volinfo, &enabled);
+
+        if (enabled) {
+            gf_msg("glusterd", GF_LOG_WARNING, 0, GD_MSG_MARKER_DISABLE_FAIL,
+                   GEOREP
+                   " sessions active"
+                   "for the volume %s, cannot disable marker ",
+                   volinfo->volname);
+            set_graph_errstr(graph, VKEY_MARKER_XTIME
+                             " cannot be disabled "
+                             "while " GEOREP " sessions exist");
+            ret = -1;
+            goto out;
+        }
+    }
+
+    ret = 0;
+out:
+    gf_msg_debug("glusterd", 0, "Returning %d", ret);
+    return ret;
 }
 
 static int
-log_format_option_handler (volgen_graph_t *graph, struct volopt_map_entry *vme,
-                           void *param)
+sys_loglevel_option_handler(volgen_graph_t *graph, struct volopt_map_entry *vme,
+                            void *param)
 {
-        char  *role = NULL;
-        struct volopt_map_entry vme2 = {0,};
+    char *role = NULL;
+    struct volopt_map_entry vme2 = {
+        0,
+    };
 
-        role = (char *) param;
+    role = (char *)param;
 
-        if (strcmp (vme->option, "!log-format") != 0 ||
-            !strstr (vme->key, role))
-                return 0;
+    if (strcmp(vme->option, "!sys-log-level") != 0 || !strstr(vme->key, role))
+        return 0;
 
-        memcpy (&vme2, vme, sizeof (vme2));
-        vme2.option = "log-format";
+    memcpy(&vme2, vme, sizeof(vme2));
+    vme2.option = "sys-log-level";
 
-        return basic_option_handler (graph, &vme2, NULL);
+    return basic_option_handler(graph, &vme2, NULL);
 }
 
 static int
-log_buf_size_option_handler (volgen_graph_t *graph,
-                             struct volopt_map_entry *vme,
-                             void *param)
+logger_option_handler(volgen_graph_t *graph, struct volopt_map_entry *vme,
+                      void *param)
 {
-        char  *role = NULL;
-        struct volopt_map_entry vme2 = {0,};
+    char *role = NULL;
+    struct volopt_map_entry vme2 = {
+        0,
+    };
 
-        role = (char *) param;
+    role = (char *)param;
 
-        if (strcmp (vme->option, "!log-buf-size") != 0 ||
-            !strstr (vme->key, role))
-                return 0;
+    if (strcmp(vme->option, "!logger") != 0 || !strstr(vme->key, role))
+        return 0;
 
-        memcpy (&vme2, vme, sizeof (vme2));
-        vme2.option = "log-buf-size";
+    memcpy(&vme2, vme, sizeof(vme2));
+    vme2.option = "logger";
 
-        return basic_option_handler (graph, &vme2, NULL);
+    return basic_option_handler(graph, &vme2, NULL);
 }
 
 static int
-log_flush_timeout_option_handler (volgen_graph_t *graph,
-                                  struct volopt_map_entry *vme,
-                                  void *param)
+log_format_option_handler(volgen_graph_t *graph, struct volopt_map_entry *vme,
+                          void *param)
 {
-        char  *role = NULL;
-        struct volopt_map_entry vme2 = {0,};
+    char *role = NULL;
+    struct volopt_map_entry vme2 = {
+        0,
+    };
 
-        role = (char *) param;
+    role = (char *)param;
 
-        if (strcmp (vme->option, "!log-flush-timeout") != 0 ||
-            !strstr (vme->key, role))
-                return 0;
+    if (strcmp(vme->option, "!log-format") != 0 || !strstr(vme->key, role))
+        return 0;
 
-        memcpy (&vme2, vme, sizeof (vme2));
-        vme2.option = "log-flush-timeout";
+    memcpy(&vme2, vme, sizeof(vme2));
+    vme2.option = "log-format";
 
-        return basic_option_handler (graph, &vme2, NULL);
+    return basic_option_handler(graph, &vme2, NULL);
 }
 
 static int
-volgen_graph_set_xl_options (volgen_graph_t *graph, dict_t *dict)
+log_localtime_logging_option_handler(volgen_graph_t *graph,
+                                     struct volopt_map_entry *vme, void *param)
 {
-        int32_t   ret               = -1;
-        char     *xlator            = NULL;
-        char     xlator_match[1024] = {0,}; /* for posix* -> *posix* */
-        char     *loglevel          = NULL;
-        xlator_t *trav              = NULL;
+    char *role = NULL;
+    struct volopt_map_entry vme2 = {
+        0,
+    };
 
-        ret = dict_get_str (dict, "xlator", &xlator);
-        if (ret)
-                goto out;
+    role = (char *)param;
 
-        ret = dict_get_str (dict, "loglevel", &loglevel);
-        if (ret)
-                goto out;
-
-        snprintf (xlator_match, 1024, "*%s", xlator);
+    if (strcmp(vme->option, "!cluster.localtime-logging") != 0 ||
+        !strstr(vme->key, role))
+        return 0;
 
-        for (trav = first_of (graph); trav; trav = trav->next) {
-                if (fnmatch(xlator_match, trav->type, FNM_NOESCAPE) == 0) {
-                        gf_log ("glusterd", GF_LOG_DEBUG, "Setting log level for xlator: %s",
-                                trav->type);
-                        ret = xlator_set_option (trav, "log-level", loglevel);
-                        if (ret)
-                                break;
-                }
-        }
+    memcpy(&vme2, vme, sizeof(vme2));
+    vme2.option = GLUSTERD_LOCALTIME_LOGGING_KEY;
 
- out:
-        return ret;
+    return basic_option_handler(graph, &vme2, NULL);
 }
 
 static int
-server_spec_option_handler (volgen_graph_t *graph,
-                            struct volopt_map_entry *vme, void *param)
+log_buf_size_option_handler(volgen_graph_t *graph, struct volopt_map_entry *vme,
+                            void *param)
 {
-        int                     ret = 0;
-        glusterd_volinfo_t      *volinfo = NULL;
+    char *role = NULL;
+    struct volopt_map_entry vme2 = {
+        0,
+    };
 
-        volinfo = param;
+    role = (char *)param;
 
-        ret = server_auth_option_handler (graph, vme, NULL);
-        if (!ret)
-                ret = server_check_marker_off (graph, vme, volinfo);
+    if (strcmp(vme->option, "!log-buf-size") != 0 || !strstr(vme->key, role))
+        return 0;
 
-        if (!ret)
-                ret = loglevel_option_handler (graph, vme, "brick");
+    memcpy(&vme2, vme, sizeof(vme2));
+    vme2.option = "log-buf-size";
 
-        if (!ret)
-                ret = sys_loglevel_option_handler (graph, vme, "brick");
+    return basic_option_handler(graph, &vme2, NULL);
+}
 
-        if (!ret)
-                ret = logger_option_handler (graph, vme, "brick");
+static int
+log_flush_timeout_option_handler(volgen_graph_t *graph,
+                                 struct volopt_map_entry *vme, void *param)
+{
+    char *role = NULL;
+    struct volopt_map_entry vme2 = {
+        0,
+    };
 
-        if (!ret)
-                ret = log_format_option_handler (graph, vme, "brick");
+    role = (char *)param;
 
-        if (!ret)
-                ret = log_buf_size_option_handler (graph, vme, "brick");
+    if (strcmp(vme->option, "!log-flush-timeout") != 0 ||
+        !strstr(vme->key, role))
+        return 0;
 
-        if (!ret)
-                ret = log_flush_timeout_option_handler (graph, vme, "brick");
+    memcpy(&vme2, vme, sizeof(vme2));
+    vme2.option = "log-flush-timeout";
 
-        return ret;
+    return basic_option_handler(graph, &vme2, NULL);
 }
 
 static int
-server_spec_extended_option_handler (volgen_graph_t *graph,
-                                     struct volopt_map_entry *vme, void *param)
-{
-        int     ret           = 0;
-        dict_t *dict          = NULL;
-
-        GF_ASSERT (param);
-        dict = (dict_t *)param;
-
-        ret = server_auth_option_handler (graph, vme, NULL);
-        if (!ret)
-                ret = volgen_graph_set_xl_options (graph, dict);
+volgen_graph_set_xl_options(volgen_graph_t *graph, dict_t *dict)
+{
+    int32_t ret = -1;
+    char *xlator = NULL;
+    char xlator_match[1024] = {
+        0,
+    }; /* for posix* -> *posix* */
+    char *loglevel = NULL;
+    xlator_t *trav = NULL;
+    xlator_t *this = THIS;
+    GF_ASSERT(this);
+
+    ret = dict_get_str_sizen(dict, "xlator", &xlator);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_GET_FAILED,
+                "Key=xlator", NULL);
+        goto out;
+    }
+
+    ret = dict_get_str_sizen(dict, "loglevel", &loglevel);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_GET_FAILED,
+                "Key=loglevel", NULL);
+        goto out;
+    }
+
+    snprintf(xlator_match, 1024, "*%s", xlator);
+
+    for (trav = first_of(graph); trav; trav = trav->next) {
+        if (fnmatch(xlator_match, trav->type, FNM_NOESCAPE) == 0) {
+            gf_msg_debug("glusterd", 0, "Setting log level for xlator: %s",
+                         trav->type);
+            ret = xlator_set_fixed_option(trav, "log-level", loglevel);
+            if (ret)
+                break;
+        }
+    }
 
-        return ret;
+out:
+    return ret;
 }
 
-static void get_vol_tstamp_file (char *filename, glusterd_volinfo_t *volinfo);
-
 static int
-brick_graph_add_posix (volgen_graph_t *graph, glusterd_volinfo_t *volinfo,
-                        dict_t *set_dict, glusterd_brickinfo_t *brickinfo)
+server_spec_option_handler(volgen_graph_t *graph, struct volopt_map_entry *vme,
+                           void *param)
 {
-        int             ret = -1;
-        gf_boolean_t    quota_enabled = _gf_true;
-        gf_boolean_t    trash_enabled = _gf_false;
-        gf_boolean_t    pgfid_feat    = _gf_false;
-        char            *value = NULL;
-        xlator_t        *xl = NULL;
+    int ret = 0;
+    glusterd_volinfo_t *volinfo = NULL;
 
-        if (!graph || !volinfo || !set_dict || !brickinfo)
-                goto out;
+    volinfo = param;
 
-        ret = glusterd_volinfo_get (volinfo, VKEY_FEATURES_QUOTA, &value);
-        if (value) {
-                ret = gf_string2boolean (value, &quota_enabled);
-                if (ret)
-                        goto out;
-        }
+    ret = server_auth_option_handler(graph, vme, NULL);
+    if (!ret)
+        ret = server_check_marker_off(graph, vme, volinfo);
 
-        ret = glusterd_volinfo_get (volinfo, VKEY_FEATURES_TRASH, &value);
-        if (value) {
-                ret = gf_string2boolean (value, &trash_enabled);
-                if (ret)
-                        goto out;
-        }
+    if (!ret)
+        ret = server_check_changelog_off(graph, vme, volinfo);
 
-        ret = glusterd_volinfo_get (volinfo,
-                                    "update-link-count-parent",
-                                    &value);
-        if (value) {
-                ret = gf_string2boolean (value, &pgfid_feat);
-                if (ret)
-                        goto out;
-        }
+    if (!ret)
+        ret = loglevel_option_handler(graph, vme, "brick");
 
-        ret = -1;
+    if (!ret)
+        ret = sys_loglevel_option_handler(graph, vme, "brick");
 
-        xl = volgen_graph_add (graph, "storage/posix", volinfo->volname);
-        if (!xl)
-                goto out;
+    if (!ret)
+        ret = logger_option_handler(graph, vme, "brick");
 
-        ret = xlator_set_option (xl, "directory", brickinfo->path);
-        if (ret)
-                goto out;
+    if (!ret)
+        ret = log_format_option_handler(graph, vme, "brick");
 
-        ret = xlator_set_option (xl, "volume-id",
-                                 uuid_utoa (volinfo->volume_id));
-        if (ret)
-                goto out;
+    if (!ret)
+        ret = log_buf_size_option_handler(graph, vme, "brick");
 
-        if (quota_enabled || pgfid_feat || trash_enabled)
-                xlator_set_option (xl, "update-link-count-parent",
-                                   "on");
-out:
-        return ret;
+    if (!ret)
+        ret = log_flush_timeout_option_handler(graph, vme, "brick");
+
+    if (!ret)
+        ret = log_localtime_logging_option_handler(graph, vme, "brick");
+
+    if (!ret)
+        ret = threads_option_handler(graph, vme, "brick");
+
+    return ret;
 }
+
 static int
-brick_graph_add_trash (volgen_graph_t *graph, glusterd_volinfo_t *volinfo,
-                        dict_t *set_dict, glusterd_brickinfo_t *brickinfo)
+server_spec_extended_option_handler(volgen_graph_t *graph,
+                                    struct volopt_map_entry *vme, void *param)
 {
-        int ret = -1;
-        xlator_t *xl = NULL;
+    int ret = 0;
+    dict_t *dict = NULL;
 
-        xl = volgen_graph_add (graph, "features/trash", volinfo->volname);
-        if (!xl)
-                goto out;
-        ret = xlator_set_option (xl, "trash-dir", ".trashcan");
-        if (ret)
-                goto out;
-        ret = xlator_set_option (xl, "brick-path", brickinfo->path);
-        if (ret)
-                goto out;
-        ret = xlator_set_option (xl, "trash-internal-op", "off");
-        if (ret)
-                goto out;
-out:
-        return ret;
+    GF_ASSERT(param);
+    dict = (dict_t *)param;
+
+    ret = server_auth_option_handler(graph, vme, NULL);
+    if (!ret)
+        ret = volgen_graph_set_xl_options(graph, dict);
+
+    return ret;
 }
 
+static void
+get_vol_tstamp_file(char *filename, glusterd_volinfo_t *volinfo);
+
 static int
-brick_graph_add_arbiter (volgen_graph_t *graph, glusterd_volinfo_t *volinfo,
-                         dict_t *set_dict, glusterd_brickinfo_t *brickinfo)
-{
-        xlator_t *xl = NULL;
-        int ret = -1;
+gfproxy_server_graph_builder(volgen_graph_t *graph, glusterd_volinfo_t *volinfo,
+                             dict_t *set_dict, void *param)
+{
+    xlator_t *xl = NULL;
+    /*char            *value          = NULL;*/
+    char transt[16] = {
+        0,
+    };
+    char key[1024] = {
+        0,
+    };
+    int keylen;
+    /*char            port_str[7]     = {0, };*/
+    int ret = 0;
+    char *username = NULL;
+    char *password = NULL;
+    /*int             rclusters       = 0;*/
+
+    xlator_t *this = THIS;
+    GF_ASSERT(this);
+    /* We are a trusted client */
+    ret = dict_set_uint32(set_dict, "trusted-client", GF_CLIENT_TRUSTED);
+    if (ret != 0) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_SET_FAILED,
+                "Key=trusted-client", NULL);
+        goto out;
+    }
+
+    ret = dict_set_int32_sizen(set_dict, "gfproxy-server", 1);
+    if (ret != 0) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_SET_FAILED,
+                "Key=gfproxy-server", NULL);
+        goto out;
+    }
+
+    /* Build the client section of the graph first */
+    build_client_graph(graph, volinfo, set_dict);
+
+    /* Clear this setting so that future users of set_dict do not end up
+     * thinking they are a gfproxy server */
+    dict_del_sizen(set_dict, "gfproxy-server");
+    dict_del_sizen(set_dict, "trusted-client");
+
+    /* Then add the server to it */
+    get_vol_transport_type(volinfo, transt);
+    xl = volgen_graph_add(graph, "protocol/server", volinfo->volname);
+    if (!xl)
+        goto out;
+
+    ret = xlator_set_fixed_option(xl, "transport-type", transt);
+    if (ret != 0)
+        goto out;
+
+    /* Set username and password */
+    username = glusterd_auth_get_username(volinfo);
+    password = glusterd_auth_get_password(volinfo);
+    if (username) {
+        keylen = snprintf(key, sizeof(key), "auth.login.gfproxyd-%s.allow",
+                          volinfo->volname);
+        ret = xlator_set_option(xl, key, keylen, username);
+        if (ret)
+            return -1;
+    }
+
+    if (password) {
+        keylen = snprintf(key, sizeof(key), "auth.login.%s.password", username);
+        ret = xlator_set_option(xl, key, keylen, password);
+        if (ret != 0)
+            goto out;
+    }
+
+    snprintf(key, sizeof(key), "gfproxyd-%s", volinfo->volname);
+    ret = xlator_set_fixed_option(xl, "auth-path", key);
 
-        if (volinfo->arbiter_count != 1)
-                return 0;
-        /*TODO: Parse brickinfo and add the arbiter xlator only if brick is the
-         * last brick (i.e. 3rd brick) of the replcia pair.*/
-        xl = volgen_graph_add (graph, "features/arbiter", volinfo->volname);
-        if (!xl)
-                goto out;
-        ret = 0;
 out:
-        return ret;
+    return ret;
 }
 
 static int
-brick_graph_add_bd (volgen_graph_t *graph, glusterd_volinfo_t *volinfo,
-                     dict_t *set_dict, glusterd_brickinfo_t *brickinfo)
+brick_graph_add_posix(volgen_graph_t *graph, glusterd_volinfo_t *volinfo,
+                      dict_t *set_dict, glusterd_brickinfo_t *brickinfo)
 {
-        int ret = -1;
+    char tmpstr[10] = {
+        0,
+    };
+    int ret = -1;
+    gf_boolean_t quota_enabled = _gf_true;
+    gf_boolean_t trash_enabled = _gf_false;
+    gf_boolean_t pgfid_feat = _gf_false;
+    char *value = NULL;
+    xlator_t *xl = NULL;
+    xlator_t *this = NULL;
+    glusterd_conf_t *priv = NULL;
+
+    this = THIS;
+
+    if (!graph || !volinfo || !set_dict || !brickinfo) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_INVALID_ARGUMENT, NULL);
+        goto out;
+    }
+
+    priv = this->private;
+    GF_VALIDATE_OR_GOTO("glusterd", priv, out);
+
+    ret = glusterd_volinfo_get(volinfo, VKEY_FEATURES_QUOTA, &value);
+    if (value) {
+        ret = gf_string2boolean(value, &quota_enabled);
+        if (ret)
+            goto out;
+    }
 
-        if (!graph || !volinfo || !set_dict || !brickinfo)
-                goto out;
+    ret = glusterd_volinfo_get(volinfo, VKEY_FEATURES_TRASH, &value);
+    if (value) {
+        ret = gf_string2boolean(value, &trash_enabled);
+        if (ret)
+            goto out;
+    }
 
-        ret = 0;
+    ret = glusterd_volinfo_get(volinfo, "update-link-count-parent", &value);
+    if (value) {
+        ret = gf_string2boolean(value, &pgfid_feat);
+        if (ret)
+            goto out;
+    }
 
-#ifdef HAVE_BD_XLATOR
-        if (*brickinfo->vg != '\0') {
-                xlator_t *xl = NULL;
-                /* Now add BD v2 xlator if volume is BD type */
-                xl = volgen_graph_add (graph, "storage/bd", volinfo->volname);
-                if (!xl) {
-                        ret = -1;
-                        goto out;
-                }
+    ret = -1;
 
-                ret = xlator_set_option (xl, "device", "vg");
-                if (ret)
-                        goto out;
+    xl = volgen_graph_add(graph, "storage/posix", volinfo->volname);
+    if (!xl)
+        goto out;
 
-                ret = xlator_set_option (xl, "export", brickinfo->vg);
-                if (ret)
-                        goto out;
+    ret = xlator_set_fixed_option(xl, "directory", brickinfo->path);
+    if (ret)
+        goto out;
+
+    ret = xlator_set_fixed_option(xl, "volume-id",
+                                  uuid_utoa(volinfo->volume_id));
+    if (ret)
+        goto out;
+
+    if (quota_enabled || pgfid_feat || trash_enabled) {
+        ret = xlator_set_fixed_option(xl, "update-link-count-parent", "on");
+        if (ret) {
+            goto out;
         }
-#endif
+    }
 
+    if (priv->op_version >= GD_OP_VERSION_7_0) {
+        ret = xlator_set_fixed_option(xl, "fips-mode-rchecksum", "on");
+        if (ret) {
+            goto out;
+        }
+    }
+    snprintf(tmpstr, sizeof(tmpstr), "%d", brickinfo->fs_share_count);
+    ret = xlator_set_fixed_option(xl, "shared-brick-count", tmpstr);
 out:
-        return ret;
+    return ret;
 }
 
 static int
-brick_graph_add_bitrot_stub (volgen_graph_t *graph, glusterd_volinfo_t *volinfo,
-                            dict_t *set_dict, glusterd_brickinfo_t *brickinfo)
+brick_graph_add_selinux(volgen_graph_t *graph, glusterd_volinfo_t *volinfo,
+                        dict_t *set_dict, glusterd_brickinfo_t *brickinfo)
 {
-        xlator_t        *xl  = NULL;
-        int		 ret = -1;
+    xlator_t *xl = NULL;
+    int ret = -1;
+    xlator_t *this = THIS;
+    GF_ASSERT(this);
 
-        if (!graph || !volinfo || !set_dict || !brickinfo)
-                goto out;
-
-        xl = volgen_graph_add (graph, "features/bitrot-stub", volinfo->volname);
-        if (!xl)
-                goto out;
+    if (!graph || !volinfo) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_INVALID_ARGUMENT, NULL);
+        goto out;
+    }
 
-        ret = xlator_set_option (xl, "export", brickinfo->path);
+    xl = volgen_graph_add(graph, "features/selinux", volinfo->volname);
+    if (!xl)
+        goto out;
 
+    ret = 0;
 out:
-        return ret;
+    return ret;
 }
 
 static int
-brick_graph_add_changelog (volgen_graph_t *graph, glusterd_volinfo_t *volinfo,
-                            dict_t *set_dict, glusterd_brickinfo_t *brickinfo)
+brick_graph_add_trash(volgen_graph_t *graph, glusterd_volinfo_t *volinfo,
+                      dict_t *set_dict, glusterd_brickinfo_t *brickinfo)
 {
-        xlator_t        *xl = NULL;
-        char            changelog_basepath[PATH_MAX]    = {0,};
-        int             ret = -1;
-
-        if (!graph || !volinfo || !set_dict || !brickinfo)
-                goto out;
-
-        xl = volgen_graph_add (graph, "features/changelog", volinfo->volname);
-        if (!xl)
-                goto out;
-
-        ret = xlator_set_option (xl, "changelog-brick", brickinfo->path);
-        if (ret)
-                goto out;
-
-        snprintf (changelog_basepath, sizeof (changelog_basepath),
-                  "%s/%s", brickinfo->path, ".glusterfs/changelogs");
-        ret = xlator_set_option (xl, "changelog-dir", changelog_basepath);
-        if (ret)
-                goto out;
+    int ret = -1;
+    xlator_t *xl = NULL;
+
+    xl = volgen_graph_add(graph, "features/trash", volinfo->volname);
+    if (!xl)
+        goto out;
+    ret = xlator_set_fixed_option(xl, "trash-dir", ".trashcan");
+    if (ret)
+        goto out;
+    ret = xlator_set_fixed_option(xl, "brick-path", brickinfo->path);
+    if (ret)
+        goto out;
+    ret = xlator_set_fixed_option(xl, "trash-internal-op", "off");
+    if (ret)
+        goto out;
 out:
-        return ret;
+    return ret;
 }
 
 static int
-brick_graph_add_changetimerecorder (volgen_graph_t *graph,
-                                    glusterd_volinfo_t *volinfo,
-                                    dict_t *set_dict,
-                                    glusterd_brickinfo_t *brickinfo)
-{
-        xlator_t        *xl = NULL;
-        int             ret = -1;
-        glusterd_brickinfo_t *brickiter     = NULL;
-        glusterd_brickinfo_t *tmp           = NULL;
-        char                 *brickname     = NULL;
-        char                 *path          = NULL;
-        char                 *volname       = NULL;
-        int                   bricknum      = 0;
-        char     index_basepath[PATH_MAX]   = {0};
-
-        if (!graph || !volinfo || !set_dict || !brickinfo)
-                goto out;
-
-        path      = brickinfo->path;
-
-        xl = volgen_graph_add (graph, "features/changetimerecorder",
-                               volinfo->volname);
+brick_graph_add_arbiter(volgen_graph_t *graph, glusterd_volinfo_t *volinfo,
+                        dict_t *set_dict, glusterd_brickinfo_t *brickinfo)
+{
+    xlator_t *xl = NULL;
+    glusterd_brickinfo_t *last = NULL;
+    int ret = -1;
 
+    if (volinfo->arbiter_count != 1)
+        return 0;
 
-        ret = xlator_set_option (xl, "db-type", "sqlite3");
-        if (ret)
-                goto out;
+    /* Add arbiter only if it is the last (i.e. 3rd) brick. */
+    last = get_last_brick_of_brick_group(volinfo, brickinfo);
+    if (last != brickinfo)
+        return 0;
 
-        bricknum = 0;
-        list_for_each_entry_safe (brickiter, tmp, &volinfo->bricks,
-                                  brick_list) {
-                if (brickiter == brickinfo)
-                        break;
-                bricknum++;
-        }
-        if (bricknum < volinfo->tier_info.hot_brick_count) {
-                ret = xlator_set_option (xl, "hot-brick", "on");
-        } else {
-                ret = xlator_set_option (xl, "hot-brick", "off");
-        }
+    xl = volgen_graph_add(graph, "features/arbiter", volinfo->volname);
+    if (!xl)
+        goto out;
+    ret = 0;
+out:
+    return ret;
+}
 
-        brickname = strrchr(path, '/') + 1;
-        snprintf (index_basepath, sizeof (index_basepath), "%s.db",
-                  brickname);
-        ret = xlator_set_option (xl, "db-name", index_basepath);
-        if (ret)
-                goto out;
+static int
+brick_graph_add_bitrot_stub(volgen_graph_t *graph, glusterd_volinfo_t *volinfo,
+                            dict_t *set_dict, glusterd_brickinfo_t *brickinfo)
+{
+    xlator_t *xl = NULL;
+    int ret = -1;
+    char *value = NULL;
+    xlator_t *this = THIS;
+
+    if (!graph || !volinfo || !set_dict || !brickinfo) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_INVALID_ARGUMENT, NULL);
+        goto out;
+    }
+
+    xl = volgen_graph_add(graph, "features/bitrot-stub", volinfo->volname);
+    if (!xl)
+        goto out;
+
+    ret = xlator_set_fixed_option(xl, "export", brickinfo->path);
+    if (ret) {
+        gf_log(this->name, GF_LOG_WARNING,
+               "failed to set the export "
+               "option in bit-rot-stub");
+        goto out;
+    }
+
+    ret = glusterd_volinfo_get(volinfo, VKEY_FEATURES_BITROT, &value);
+    ret = xlator_set_fixed_option(xl, "bitrot", value);
+    if (ret)
+        gf_log(this->name, GF_LOG_WARNING,
+               "failed to set bitrot "
+               "enable option in bit-rot-stub");
 
-        snprintf (index_basepath, sizeof (index_basepath), "%s/%s",
-                  path, ".glusterfs/");
-        ret = xlator_set_option (xl, "db-path", index_basepath);
-        if (ret)
-                goto out;
+out:
+    return ret;
+}
 
-        ret = xlator_set_option (xl, "record-exit", "off");
+static int
+brick_graph_add_changelog(volgen_graph_t *graph, glusterd_volinfo_t *volinfo,
+                          dict_t *set_dict, glusterd_brickinfo_t *brickinfo)
+{
+    xlator_t *xl = NULL;
+    char changelog_basepath[PATH_MAX] = {
+        0,
+    };
+    int ret = -1;
+    int32_t len = 0;
+    xlator_t *this = THIS;
+    GF_ASSERT(this);
+
+    if (!graph || !volinfo || !set_dict || !brickinfo) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_INVALID_ARGUMENT, NULL);
+        goto out;
+    }
+
+    xl = volgen_graph_add(graph, "features/changelog", volinfo->volname);
+    if (!xl)
+        goto out;
+
+    ret = xlator_set_fixed_option(xl, "changelog-brick", brickinfo->path);
+    if (ret)
+        goto out;
+
+    len = snprintf(changelog_basepath, sizeof(changelog_basepath), "%s/%s",
+                   brickinfo->path, ".glusterfs/changelogs");
+    if ((len < 0) || (len >= sizeof(changelog_basepath))) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_COPY_FAIL, NULL);
+        ret = -1;
+        goto out;
+    }
+    ret = xlator_set_fixed_option(xl, "changelog-dir", changelog_basepath);
+    if (ret)
+        goto out;
+
+    ret = glusterd_is_bitrot_enabled(volinfo);
+    if (ret == -1) {
+        goto out;
+    } else if (ret) {
+        ret = xlator_set_fixed_option(xl, "changelog-notification", "on");
         if (ret)
-                goto out;
-
-        ret = xlator_set_option (xl, "record-entry", "on");
+            goto out;
+    } else {
+        ret = xlator_set_fixed_option(xl, "changelog-notification", "off");
         if (ret)
-                goto out;
+            goto out;
+    }
+out:
+    return ret;
+}
 
+static int
+brick_graph_add_acl(volgen_graph_t *graph, glusterd_volinfo_t *volinfo,
+                    dict_t *set_dict, glusterd_brickinfo_t *brickinfo)
+{
+    xlator_t *xl = NULL;
+    int ret = -1;
+    xlator_t *this = THIS;
+    GF_ASSERT(this);
+
+    if (!graph || !volinfo || !set_dict) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_INVALID_ARGUMENT, NULL);
+        goto out;
+    }
+
+    ret = dict_get_str_boolean(set_dict, "features.acl", 1);
+    if (!ret) {
+        /* Skip creating this volume if option is disabled */
+        /* By default, this is 'true' */
+        goto out;
+    } else if (ret < 0) {
+        /* lets not treat this as error, as this option is not critical,
+           and implemented for debug help */
+        gf_log(THIS->name, GF_LOG_INFO,
+               "failed to get 'features.acl' flag from dict");
+    }
+
+    xl = volgen_graph_add(graph, "features/access-control", volinfo->volname);
+    if (!xl) {
+        ret = -1;
+        goto out;
+    }
+    ret = 0;
 out:
-        return ret;
+    return ret;
 }
 
 static int
-brick_graph_add_acl (volgen_graph_t *graph, glusterd_volinfo_t *volinfo,
+brick_graph_add_locks(volgen_graph_t *graph, glusterd_volinfo_t *volinfo,
                       dict_t *set_dict, glusterd_brickinfo_t *brickinfo)
 {
-        xlator_t        *xl = NULL;
-        int             ret = -1;
+    xlator_t *xl = NULL;
+    int ret = -1;
+    xlator_t *this = THIS;
+    GF_ASSERT(this);
 
-        if (!graph || !volinfo || !set_dict)
-                goto out;
+    if (!graph || !volinfo || !set_dict) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_INVALID_ARGUMENT, NULL);
+        goto out;
+    }
 
-        xl = volgen_graph_add (graph, "features/access-control",
-                                volinfo->volname);
-        if (!xl)
-                goto out;
+    xl = volgen_graph_add(graph, "features/locks", volinfo->volname);
+    if (!xl)
+        goto out;
 
-        ret = 0;
+    ret = 0;
 out:
-        return ret;
+    return ret;
 }
 
 static int
-brick_graph_add_locks (volgen_graph_t *graph, glusterd_volinfo_t *volinfo,
+brick_graph_add_iot(volgen_graph_t *graph, glusterd_volinfo_t *volinfo,
+                    dict_t *set_dict, glusterd_brickinfo_t *brickinfo)
+{
+    xlator_t *xl = NULL;
+    int ret = -1;
+    xlator_t *this = THIS;
+    GF_ASSERT(this);
+
+    if (!graph || !volinfo || !set_dict) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_INVALID_ARGUMENT, NULL);
+        goto out;
+    }
+
+    xl = volgen_graph_add(graph, "performance/io-threads", volinfo->volname);
+    if (!xl)
+        goto out;
+    ret = 0;
+out:
+    return ret;
+}
+
+static int
+brick_graph_add_barrier(volgen_graph_t *graph, glusterd_volinfo_t *volinfo,
                         dict_t *set_dict, glusterd_brickinfo_t *brickinfo)
 {
+    xlator_t *xl = NULL;
+    int ret = -1;
+    xlator_t *this = THIS;
 
-        xlator_t        *xl = NULL;
-        int             ret = -1;
-
-        if (!graph || !volinfo || !set_dict)
-                goto out;
+    if (!graph || !volinfo) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_INVALID_ARGUMENT, NULL);
+        goto out;
+    }
 
-        xl = volgen_graph_add (graph, "features/locks",
-                                volinfo->volname);
-        if (!xl)
-                goto out;
+    xl = volgen_graph_add(graph, "features/barrier", volinfo->volname);
+    if (!xl)
+        goto out;
 
-        ret = 0;
+    ret = 0;
 out:
-        return ret;
+    return ret;
 }
 
 static int
-brick_graph_add_iot (volgen_graph_t *graph, glusterd_volinfo_t *volinfo,
-                      dict_t *set_dict, glusterd_brickinfo_t *brickinfo)
+brick_graph_add_sdfs(volgen_graph_t *graph, glusterd_volinfo_t *volinfo,
+                     dict_t *set_dict, glusterd_brickinfo_t *brickinfo)
 {
+    xlator_t *xl = NULL;
+    int ret = -1;
+    xlator_t *this = THIS;
+    GF_ASSERT(this);
 
-        xlator_t        *xl = NULL;
-        int             ret = -1;
+    if (!graph || !volinfo) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_INVALID_ARGUMENT, NULL);
+        goto out;
+    }
 
-        if (!graph || !volinfo || !set_dict)
-                goto out;
-
-        xl = volgen_graph_add (graph, "performance/io-threads",
-                                volinfo->volname);
-        if (!xl)
-                goto out;
+    if (!dict_get_str_boolean(set_dict, "features.sdfs", 0)) {
+        /* update only if option is enabled */
         ret = 0;
+        goto out;
+    }
+
+    xl = volgen_graph_add(graph, "features/sdfs", volinfo->volname);
+    if (!xl)
+        goto out;
+    /* If we don't set this option here, the translator by default marks
+       it 'pass-through' */
+    ret = xlator_set_fixed_option(xl, "pass-through", "false");
+    if (ret)
+        goto out;
+
+    ret = 0;
 out:
-        return ret;
+    return ret;
 }
 
 static int
-brick_graph_add_barrier (volgen_graph_t *graph, glusterd_volinfo_t *volinfo,
-                         dict_t *set_dict, glusterd_brickinfo_t *brickinfo)
+brick_graph_add_namespace(volgen_graph_t *graph, glusterd_volinfo_t *volinfo,
+                          dict_t *set_dict, glusterd_brickinfo_t *brickinfo)
 {
+    xlator_t *xl = NULL;
+    int ret = -1;
+    xlator_t *this = THIS;
+    GF_ASSERT(this);
 
-        xlator_t        *xl = NULL;
-        int             ret = -1;
+    if (!graph || !volinfo || !set_dict) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_INVALID_ARGUMENT, NULL);
+        goto out;
+    }
 
-        if (!graph || !volinfo)
-                goto out;
+    ret = dict_get_str_boolean(set_dict, "features.tag-namespaces", 0);
+    if (ret == -1)
+        goto out;
 
-        xl = volgen_graph_add (graph, "features/barrier", volinfo->volname);
+    if (ret) {
+        xl = volgen_graph_add(graph, "features/namespace", volinfo->volname);
         if (!xl)
-                goto out;
+            goto out;
+    }
 
-        ret = 0;
+    ret = 0;
 out:
-        return ret;
+    return ret;
 }
 
-static int
-brick_graph_add_index (volgen_graph_t *graph, glusterd_volinfo_t *volinfo,
-                        dict_t *set_dict, glusterd_brickinfo_t *brickinfo)
+xlator_t *
+add_one_peer(volgen_graph_t *graph, glusterd_brickinfo_t *peer, char *volname,
+             uint16_t index)
 {
-        xlator_t        *xl = NULL;
-        char            index_basepath[PATH_MAX]   = {0};
-        int             ret = -1;
+    xlator_t *kid;
 
-        if (!graph || !volinfo || !brickinfo || !set_dict)
-                goto out;
+    kid = volgen_graph_add_nolink(graph, "protocol/client", "%s-client-%u",
+                                  volname, index++);
+    if (!kid) {
+        return NULL;
+    }
 
-        xl = volgen_graph_add (graph, "features/index", volinfo->volname);
-        if (!xl)
-                goto out;
+    /* TBD: figure out where to get the proper transport list */
+    if (xlator_set_fixed_option(kid, "transport-type", "socket")) {
+        return NULL;
+    }
+    if (xlator_set_fixed_option(kid, "remote-host", peer->hostname)) {
+        return NULL;
+    }
+    if (xlator_set_fixed_option(kid, "remote-subvolume", peer->path)) {
+        return NULL;
+    }
+    /* TBD: deal with RDMA, SSL */
 
-        snprintf (index_basepath, sizeof (index_basepath), "%s/%s",
-                  brickinfo->path, ".glusterfs/indices");
-        ret = xlator_set_option (xl, "index-base", index_basepath);
-        if (ret)
-                goto out;
-        if (volinfo->type == GF_CLUSTER_TYPE_DISPERSE) {
-                ret = xlator_set_option (xl, "xattrop64-watchlist",
-                                         "trusted.ec.dirty");
-                if (ret)
-                        goto out;
-        }
-out:
-        return ret;
+    return kid;
 }
 
 static int
-brick_graph_add_marker (volgen_graph_t *graph, glusterd_volinfo_t *volinfo,
-                         dict_t *set_dict, glusterd_brickinfo_t *brickinfo)
+brick_graph_add_index(volgen_graph_t *graph, glusterd_volinfo_t *volinfo,
+                      dict_t *set_dict, glusterd_brickinfo_t *brickinfo)
 {
-        int             ret = -1;
-        xlator_t        *xl = NULL;
-        char            tstamp_file[PATH_MAX] = {0,};
-        char            volume_id[64] = {0,};
-
-        if (!graph || !volinfo || !set_dict)
-                goto out;
-
-        xl = volgen_graph_add (graph, "features/marker", volinfo->volname);
-        if (!xl)
-                goto out;
-
-        gf_uuid_unparse (volinfo->volume_id, volume_id);
-        ret = xlator_set_option (xl, "volume-uuid", volume_id);
+    xlator_t *xl = NULL;
+    char *pending_xattr = NULL;
+    char index_basepath[PATH_MAX] = {0};
+    int ret = -1;
+    int32_t len = 0;
+    xlator_t *this = THIS;
+    GF_ASSERT(this);
+
+    if (!graph || !volinfo || !brickinfo || !set_dict) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_INVALID_ARGUMENT, NULL);
+        goto out;
+    }
+
+    xl = volgen_graph_add(graph, "features/index", volinfo->volname);
+    if (!xl)
+        goto out;
+
+    len = snprintf(index_basepath, sizeof(index_basepath), "%s/%s",
+                   brickinfo->path, ".glusterfs/indices");
+    if ((len < 0) || (len >= sizeof(index_basepath))) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_COPY_FAIL, NULL);
+        goto out;
+    }
+
+    ret = xlator_set_fixed_option(xl, "index-base", index_basepath);
+    if (ret)
+        goto out;
+    if (volinfo->type == GF_CLUSTER_TYPE_DISPERSE) {
+        ret = xlator_set_fixed_option(xl, "xattrop64-watchlist",
+                                      "trusted.ec.dirty");
         if (ret)
-                goto out;
-        get_vol_tstamp_file (tstamp_file, volinfo);
-        ret = xlator_set_option (xl, "timestamp-file", tstamp_file);
+            goto out;
+    }
+    if ((volinfo->type == GF_CLUSTER_TYPE_REPLICATE ||
+         volinfo->type == GF_CLUSTER_TYPE_NONE)) {
+        ret = xlator_set_fixed_option(xl, "xattrop-dirty-watchlist",
+                                      "trusted.afr.dirty");
         if (ret)
-                goto out;
-
+            goto out;
+        ret = gf_asprintf(&pending_xattr, "trusted.afr.%s-", volinfo->volname);
+        if (ret < 0)
+            goto out;
+        ret = xlator_set_fixed_option(xl, "xattrop-pending-watchlist",
+                                      pending_xattr);
+        if (ret)
+            goto out;
+    }
 out:
-        return ret;
+    GF_FREE(pending_xattr);
+    return ret;
 }
 
 static int
-brick_graph_add_quota (volgen_graph_t *graph, glusterd_volinfo_t *volinfo,
-                         dict_t *set_dict, glusterd_brickinfo_t *brickinfo)
+brick_graph_add_marker(volgen_graph_t *graph, glusterd_volinfo_t *volinfo,
+                       dict_t *set_dict, glusterd_brickinfo_t *brickinfo)
 {
-        int             ret = -1;
-        xlator_t        *xl = NULL;
-        char            *value = NULL;
-
-        if (!graph || !volinfo || !set_dict)
-                goto out;
-
-        xl = volgen_graph_add (graph, "features/quota", volinfo->volname);
-        if (!xl)
-                goto out;
-
-        ret = xlator_set_option (xl, "volume-uuid", volinfo->volname);
-        if (ret)
-                goto out;
+    int ret = -1;
+    xlator_t *xl = NULL;
+    char tstamp_file[PATH_MAX] = {
+        0,
+    };
+    char volume_id[64] = {
+        0,
+    };
+    char buf[32] = {
+        0,
+    };
+    xlator_t *this = THIS;
+    GF_ASSERT(this);
+
+    if (!graph || !volinfo || !set_dict) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_INVALID_ARGUMENT, NULL);
+        goto out;
+    }
+
+    xl = volgen_graph_add(graph, "features/marker", volinfo->volname);
+    if (!xl)
+        goto out;
+
+    gf_uuid_unparse(volinfo->volume_id, volume_id);
+    ret = xlator_set_fixed_option(xl, "volume-uuid", volume_id);
+    if (ret)
+        goto out;
+    get_vol_tstamp_file(tstamp_file, volinfo);
+    ret = xlator_set_fixed_option(xl, "timestamp-file", tstamp_file);
+    if (ret)
+        goto out;
+
+    snprintf(buf, sizeof(buf), "%d", volinfo->quota_xattr_version);
+    ret = xlator_set_fixed_option(xl, "quota-version", buf);
+    if (ret)
+        goto out;
 
-        ret = glusterd_volinfo_get (volinfo, VKEY_FEATURES_QUOTA, &value);
-        if (value) {
-                ret = xlator_set_option (xl, "server-quota", value);
-                if (ret)
-                        goto out;
-        }
 out:
-        return ret;
+    return ret;
 }
 
 static int
-brick_graph_add_ro (volgen_graph_t *graph, glusterd_volinfo_t *volinfo,
-                     dict_t *set_dict, glusterd_brickinfo_t *brickinfo)
+brick_graph_add_quota(volgen_graph_t *graph, glusterd_volinfo_t *volinfo,
+                      dict_t *set_dict, glusterd_brickinfo_t *brickinfo)
 {
-        int             ret = -1;
-        xlator_t        *xl = NULL;
-
-        if (!graph || !volinfo || !set_dict)
-                goto out;
+    int ret = -1;
+    xlator_t *xl = NULL;
+    char *value = NULL;
+    xlator_t *this = THIS;
+    GF_ASSERT(this);
 
-        if (dict_get_str_boolean (set_dict, "features.read-only", 0) &&
-            dict_get_str_boolean (set_dict, "features.worm", 0)) {
-                gf_log (THIS->name, GF_LOG_ERROR,
-                        "read-only and worm cannot be set together");
-                ret = -1;
-                goto out;
-        }
+    if (!graph || !volinfo || !set_dict) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_INVALID_ARGUMENT, NULL);
+        goto out;
+    }
 
-        xl = volgen_graph_add (graph, "features/read-only", volinfo->volname);
-        if (!xl)
-                return -1;
-        ret = xlator_set_option (xl, "read-only", "off");
-                if (ret)
-                        return -1;
+    xl = volgen_graph_add(graph, "features/quota", volinfo->volname);
+    if (!xl)
+        goto out;
 
-        ret = 0;
+    ret = xlator_set_fixed_option(xl, "volume-uuid", volinfo->volname);
+    if (ret)
+        goto out;
 
+    ret = glusterd_volinfo_get(volinfo, VKEY_FEATURES_QUOTA, &value);
+    if (value) {
+        ret = xlator_set_fixed_option(xl, "server-quota", value);
+        if (ret)
+            goto out;
+    }
 out:
-        return ret;
+    return ret;
 }
 
 static int
-brick_graph_add_worm (volgen_graph_t *graph, glusterd_volinfo_t *volinfo,
-                       dict_t *set_dict, glusterd_brickinfo_t *brickinfo)
-{
-        int             ret = -1;
-        xlator_t        *xl = NULL;
-
-        if (!graph || !volinfo || !set_dict)
-                goto out;
-
-        if (dict_get_str_boolean (set_dict, "features.read-only", 0) &&
-            dict_get_str_boolean (set_dict, "features.worm", 0)) {
-                gf_log (THIS->name, GF_LOG_ERROR,
-                        "read-only and worm cannot be set together");
-                ret = -1;
-                goto out;
-        }
+brick_graph_add_ro(volgen_graph_t *graph, glusterd_volinfo_t *volinfo,
+                   dict_t *set_dict, glusterd_brickinfo_t *brickinfo)
+{
+    int ret = -1;
+    xlator_t *xl = NULL;
+    xlator_t *this = THIS;
+    GF_ASSERT(this);
+
+    if (!graph || !volinfo || !set_dict) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_INVALID_ARGUMENT, NULL);
+        goto out;
+    }
+
+    if (dict_get_str_boolean(set_dict, "features.read-only", 0) &&
+        (dict_get_str_boolean(set_dict, "features.worm", 0) ||
+         dict_get_str_boolean(set_dict, "features.worm-file-level", 0))) {
+        gf_msg(THIS->name, GF_LOG_ERROR, errno, GD_MSG_DICT_GET_FAILED,
+               "read-only and worm cannot be set together");
+        ret = -1;
+        goto out;
+    }
 
-        xl = volgen_graph_add (graph, "features/worm", volinfo->volname);
-        if (!xl)
-                return -1;
+    xl = volgen_graph_add(graph, "features/read-only", volinfo->volname);
+    if (!xl)
+        return -1;
+    ret = xlator_set_fixed_option(xl, "read-only", "off");
+    if (ret)
+        return -1;
 
-        ret = 0;
+    ret = 0;
 
 out:
-        return ret;
+    return ret;
 }
 
 static int
-brick_graph_add_cdc (volgen_graph_t *graph, glusterd_volinfo_t *volinfo,
-                      dict_t *set_dict, glusterd_brickinfo_t *brickinfo)
+brick_graph_add_worm(volgen_graph_t *graph, glusterd_volinfo_t *volinfo,
+                     dict_t *set_dict, glusterd_brickinfo_t *brickinfo)
 {
-        int             ret = -1;
-        xlator_t        *xl = NULL;
+    int ret = -1;
+    xlator_t *xl = NULL;
+    xlator_t *this = THIS;
+    GF_ASSERT(this);
+
+    if (!graph || !volinfo || !set_dict) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_INVALID_ARGUMENT, NULL);
+        goto out;
+    }
+
+    if (dict_get_str_boolean(set_dict, "features.read-only", 0) &&
+        (dict_get_str_boolean(set_dict, "features.worm", 0) ||
+         dict_get_str_boolean(set_dict, "features.worm-file-level", 0))) {
+        gf_msg(THIS->name, GF_LOG_ERROR, 0, GD_MSG_INCOMPATIBLE_VALUE,
+               "read-only and worm cannot be set together");
+        ret = -1;
+        goto out;
+    }
 
-        if (!graph || !volinfo || !set_dict)
-                goto out;
+    xl = volgen_graph_add(graph, "features/worm", volinfo->volname);
+    if (!xl)
+        return -1;
 
-        /* Check for compress volume option, and add it to the graph on
-         * server side */
-        ret = dict_get_str_boolean (set_dict, "network.compression", 0);
-        if (ret == -1)
-                goto out;
-        if (ret) {
-                xl = volgen_graph_add (graph, "features/cdc",
-                                       volinfo->volname);
-                if (!xl) {
-                        ret = -1;
-                        goto out;
-                }
-                ret = xlator_set_option (xl, "mode", "server");
-                if (ret)
-                        goto out;
+    ret = 0;
+
+out:
+    return ret;
+}
+
+static int
+brick_graph_add_cdc(volgen_graph_t *graph, glusterd_volinfo_t *volinfo,
+                    dict_t *set_dict, glusterd_brickinfo_t *brickinfo)
+{
+    int ret = -1;
+    xlator_t *xl = NULL;
+    xlator_t *this = THIS;
+    GF_ASSERT(this);
+
+    if (!graph || !volinfo || !set_dict) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_INVALID_ARGUMENT, NULL);
+        goto out;
+    }
+
+    /* Check for compress volume option, and add it to the graph on
+     * server side */
+    ret = dict_get_str_boolean(set_dict, "network.compression", 0);
+    if (ret == -1)
+        goto out;
+    if (ret) {
+        xl = volgen_graph_add(graph, "features/cdc", volinfo->volname);
+        if (!xl) {
+            ret = -1;
+            goto out;
         }
+        ret = xlator_set_fixed_option(xl, "mode", "server");
+        if (ret)
+            goto out;
+    }
 out:
-        return ret;
+    return ret;
 }
 
 static int
-brick_graph_add_io_stats (volgen_graph_t *graph, glusterd_volinfo_t *volinfo,
-                           dict_t *set_dict, glusterd_brickinfo_t *brickinfo)
+brick_graph_add_io_stats(volgen_graph_t *graph, glusterd_volinfo_t *volinfo,
+                         dict_t *set_dict, glusterd_brickinfo_t *brickinfo)
 {
-        int             ret = -1;
-        xlator_t        *xl = NULL;
+    int ret = -1;
+    xlator_t *xl = NULL;
+    xlator_t *this = THIS;
+    glusterd_conf_t *priv = this->private;
 
-        if (!graph || !volinfo || !set_dict || !brickinfo)
-                goto out;
+    if (!graph || !set_dict || !brickinfo) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_INVALID_ARGUMENT, NULL);
+        goto out;
+    }
 
-        xl = volgen_graph_add_as (graph, "debug/io-stats", brickinfo->path);
-        if (!xl)
-                goto out;
+    xl = volgen_graph_add_as(graph, "debug/io-stats", brickinfo->path);
+    if (!xl)
+        goto out;
 
-        ret = 0;
+    ret = xlator_set_fixed_option(xl, "unique-id", brickinfo->path);
+    if (ret)
+        goto out;
+
+    if (priv->op_version >= GD_OP_VERSION_7_1) {
+        ret = xlator_set_fixed_option(xl, "volume-id",
+                                      uuid_utoa(volinfo->volume_id));
+        if (ret)
+            goto out;
+    }
+
+    ret = 0;
 out:
-        return ret;
+    return ret;
 }
 
 static int
-brick_graph_add_upcall (volgen_graph_t *graph, glusterd_volinfo_t *volinfo,
-                         dict_t *set_dict, glusterd_brickinfo_t *brickinfo)
+brick_graph_add_upcall(volgen_graph_t *graph, glusterd_volinfo_t *volinfo,
+                       dict_t *set_dict, glusterd_brickinfo_t *brickinfo)
 {
+    xlator_t *xl = NULL;
+    int ret = -1;
+    xlator_t *this = THIS;
+    GF_ASSERT(this);
 
-        xlator_t        *xl = NULL;
-        int             ret = -1;
-
-        if (!graph || !volinfo || !set_dict)
-                goto out;
+    if (!graph || !volinfo || !set_dict) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_INVALID_ARGUMENT, NULL);
+        goto out;
+    }
 
-        xl = volgen_graph_add (graph, "features/upcall", volinfo->volname);
-        if (!xl) {
-                gf_log ("glusterd", GF_LOG_WARNING,
-                        "failed to add features/upcall to graph");
-                goto out;
-        }
+    xl = volgen_graph_add(graph, "features/upcall", volinfo->volname);
+    if (!xl) {
+        gf_msg("glusterd", GF_LOG_WARNING, 0, GD_MSG_GRAPH_FEATURE_ADD_FAIL,
+               "failed to add features/upcall to graph");
+        goto out;
+    }
 
-        ret = 0;
+    ret = 0;
 out:
-        return ret;
+    return ret;
 }
 
 static int
-brick_graph_add_server (volgen_graph_t *graph, glusterd_volinfo_t *volinfo,
-                         dict_t *set_dict, glusterd_brickinfo_t *brickinfo)
+brick_graph_add_leases(volgen_graph_t *graph, glusterd_volinfo_t *volinfo,
+                       dict_t *set_dict, glusterd_brickinfo_t *brickinfo)
 {
-        int             ret = -1;
-        xlator_t        *xl = NULL;
-        char            transt[16] = {0,};
-        char            *username = NULL;
-        char            *password = NULL;
-        char            key[1024] = {0};
-        char            *ssl_user = NULL;
-        char            *value = NULL;
+    xlator_t *xl = NULL;
+    int ret = -1;
+    xlator_t *this = THIS;
+    GF_ASSERT(this);
 
-        if (!graph || !volinfo || !set_dict || !brickinfo)
-                goto out;
-
-        get_vol_transport_type (volinfo, transt);
+    if (!graph || !volinfo || !set_dict) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_INVALID_ARGUMENT, NULL);
+        goto out;
+    }
 
-        username = glusterd_auth_get_username (volinfo);
-        password = glusterd_auth_get_password (volinfo);
+    xl = volgen_graph_add(graph, "features/leases", volinfo->volname);
+    if (!xl) {
+        gf_msg("glusterd", GF_LOG_WARNING, 0, GD_MSG_GRAPH_FEATURE_ADD_FAIL,
+               "failed to add features/leases to graph");
+        goto out;
+    }
 
-        xl = volgen_graph_add (graph, "protocol/server", volinfo->volname);
-        if (!xl)
-                goto out;
+    ret = 0;
+out:
+    return ret;
+}
 
-        ret = xlator_set_option (xl, "transport-type", transt);
+static int
+brick_graph_add_server(volgen_graph_t *graph, glusterd_volinfo_t *volinfo,
+                       dict_t *set_dict, glusterd_brickinfo_t *brickinfo)
+{
+    int ret = -1;
+    xlator_t *xl = NULL;
+    char transt[16] = {
+        0,
+    };
+    char *username = NULL;
+    char *password = NULL;
+    char key[1024] = {0};
+    char *ssl_user = NULL;
+    char *volname = NULL;
+    char *address_family_data = NULL;
+    int32_t len = 0;
+    xlator_t *this = THIS;
+    GF_ASSERT(this);
+
+    if (!graph || !volinfo || !set_dict || !brickinfo) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_INVALID_ARGUMENT, NULL);
+        goto out;
+    }
+
+    get_vol_transport_type(volinfo, transt);
+
+    username = glusterd_auth_get_username(volinfo);
+    password = glusterd_auth_get_password(volinfo);
+
+    xl = volgen_graph_add(graph, "protocol/server", volinfo->volname);
+    if (!xl)
+        goto out;
+
+    ret = xlator_set_fixed_option(xl, "transport-type", transt);
+    if (ret)
+        goto out;
+
+    /*In the case of running multiple glusterds on a single machine,
+     * we should ensure that bricks don't listen on all IPs on that
+     * machine and break the IP based separation being brought about.*/
+    if (dict_get_sizen(THIS->options, "transport.socket.bind-address")) {
+        ret = xlator_set_fixed_option(xl, "transport.socket.bind-address",
+                                      brickinfo->hostname);
         if (ret)
-                goto out;
-
-        /*In the case of running multiple glusterds on a single machine,
-         * we should ensure that bricks don't listen on all IPs on that
-         * machine and break the IP based separation being brought about.*/
-        if (dict_get (THIS->options, "transport.socket.bind-address")) {
-                ret = xlator_set_option (xl, "transport.socket.bind-address",
-                                         brickinfo->hostname);
-                if (ret)
-                        return -1;
-        }
-
-        if (dict_get_str (set_dict, SSL_CERT_DEPTH_OPT, &value) == 0) {
-                ret = xlator_set_option (xl, "ssl-cert-depth", value);
-                if (ret) {
-                        gf_log ("glusterd", GF_LOG_WARNING,
-                                "failed to set ssl-cert-depth");
-                        return -1;
-                }
+            return -1;
+    }
+
+    RPC_SET_OPT(xl, SSL_OWN_CERT_OPT, "ssl-own-cert", return -1);
+    RPC_SET_OPT(xl, SSL_PRIVATE_KEY_OPT, "ssl-private-key", return -1);
+    RPC_SET_OPT(xl, SSL_CA_LIST_OPT, "ssl-ca-list", return -1);
+    RPC_SET_OPT(xl, SSL_CRL_PATH_OPT, "ssl-crl-path", return -1);
+    RPC_SET_OPT(xl, SSL_CERT_DEPTH_OPT, "ssl-cert-depth", return -1);
+    RPC_SET_OPT(xl, SSL_CIPHER_LIST_OPT, "ssl-cipher-list", return -1);
+    RPC_SET_OPT(xl, SSL_DH_PARAM_OPT, "ssl-dh-param", return -1);
+    RPC_SET_OPT(xl, SSL_EC_CURVE_OPT, "ssl-ec-curve", return -1);
+
+    if (dict_get_str_sizen(volinfo->dict, "transport.address-family",
+                           &address_family_data) == 0) {
+        ret = xlator_set_fixed_option(xl, "transport.address-family",
+                                      address_family_data);
+        if (ret) {
+            gf_log("glusterd", GF_LOG_WARNING,
+                   "failed to set transport.address-family");
+            return -1;
         }
+    }
 
-        if (dict_get_str (set_dict, SSL_CIPHER_LIST_OPT, &value) == 0) {
-                ret = xlator_set_option (xl, "ssl-cipher-list", value);
-                if (ret) {
-                        gf_log ("glusterd", GF_LOG_WARNING,
-                                "failed to set ssl-cipher-list");
-                        return -1;
-                }
+    if (username) {
+        len = snprintf(key, sizeof(key), "auth.login.%s.allow",
+                       brickinfo->path);
+        if ((len < 0) || (len >= sizeof(key))) {
+            return -1;
         }
 
-        if (username) {
-                memset (key, 0, sizeof (key));
-                snprintf (key, sizeof (key), "auth.login.%s.allow",
-                                        brickinfo->path);
+        ret = xlator_set_option(xl, key, len, username);
+        if (ret)
+            return -1;
+    }
 
-                ret = xlator_set_option (xl, key, username);
-                if (ret)
-                        return -1;
+    if (password) {
+        len = snprintf(key, sizeof(key), "auth.login.%s.password", username);
+        if ((len < 0) || (len >= sizeof(key))) {
+            return -1;
         }
+        ret = xlator_set_option(xl, key, len, password);
+        if (ret)
+            return -1;
+    }
 
-        if (password) {
-                memset (key, 0, sizeof (key));
-                snprintf (key, sizeof (key), "auth.login.%s.password",
-                          username);
+    ret = xlator_set_fixed_option(xl, "auth-path", brickinfo->path);
+    if (ret)
+        return -1;
 
-                ret = xlator_set_option (xl, key, password);
-                if (ret)
-                        return -1;
-        }
+    volname = volinfo->is_snap_volume ? volinfo->parent_volname
+                                      : volinfo->volname;
 
-        if (dict_get_str (volinfo->dict, "auth.ssl-allow", &ssl_user) == 0) {
-                memset (key, 0, sizeof (key));
-                snprintf (key, sizeof (key), "auth.login.%s.ssl-allow",
-                                       brickinfo->path);
+    if (volname && !strcmp(volname, GLUSTER_SHARED_STORAGE)) {
+        ret = xlator_set_fixed_option(xl, "strict-auth-accept", "true");
+        if (ret)
+            return -1;
+    }
 
-                ret = xlator_set_option (xl, key, ssl_user);
-                if (ret)
-                        return -1;
+    if (dict_get_str_sizen(volinfo->dict, "auth.ssl-allow", &ssl_user) == 0) {
+        len = snprintf(key, sizeof(key), "auth.login.%s.ssl-allow",
+                       brickinfo->path);
+        if ((len < 0) || (len >= sizeof(key))) {
+            return -1;
         }
 
+        ret = xlator_set_option(xl, key, len, ssl_user);
+        if (ret)
+            return -1;
+    }
+
 out:
-        return ret;
+    return ret;
 }
 
 static int
-brick_graph_add_pump (volgen_graph_t *graph, glusterd_volinfo_t *volinfo,
-                       dict_t *set_dict, glusterd_brickinfo_t *brickinfo)
+brick_graph_add_pump(volgen_graph_t *graph, glusterd_volinfo_t *volinfo,
+                     dict_t *set_dict, glusterd_brickinfo_t *brickinfo)
 {
-        int             ret = -1;
-        int             pump          = 0;
-        xlator_t        *xl            = NULL;
-        xlator_t        *txl           = NULL;
-        xlator_t        *rbxl          = NULL;
-        char            *username      = NULL;
-        char            *password      = NULL;
-        char            *ptranst       = NULL;
-        char            *value         = NULL;
-
+    int ret = -1;
+    int pump = 0;
+    xlator_t *xl = NULL;
+    xlator_t *txl = NULL;
+    xlator_t *rbxl = NULL;
+    char *username = NULL;
+    char *password = NULL;
+    char *ptranst = NULL;
+    char *address_family_data = NULL;
+    xlator_t *this = THIS;
+    GF_ASSERT(this);
+
+    if (!graph || !volinfo || !set_dict) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_INVALID_ARGUMENT, NULL);
+        goto out;
+    }
+
+    ret = dict_get_int32(volinfo->dict, "enable-pump", &pump);
+    if (ret == -ENOENT) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_GET_FAILED,
+                "Key=enable-pump", NULL);
+        ret = pump = 0;
+    }
+    if (ret)
+        return -1;
 
-        if (!graph || !volinfo || !set_dict)
-                goto out;
+    username = glusterd_auth_get_username(volinfo);
+    password = glusterd_auth_get_password(volinfo);
 
-        ret = dict_get_int32 (volinfo->dict, "enable-pump", &pump);
-        if (ret == -ENOENT)
-                ret = pump = 0;
-        if (ret)
-                return -1;
+    if (pump) {
+        txl = first_of(graph);
 
-        username = glusterd_auth_get_username (volinfo);
-        password = glusterd_auth_get_password (volinfo);
-
-        if (pump) {
-                txl = first_of (graph);
-
-                rbxl = volgen_graph_add_nolink (graph, "protocol/client",
-                                                "%s-replace-brick",
-                                                volinfo->volname);
-                if (!rbxl)
-                        return -1;
-
-                ptranst = glusterd_get_trans_type_rb (volinfo->transport_type);
-                if (NULL == ptranst)
-                        return -1;
-
-                if (dict_get_str (set_dict, SSL_CERT_DEPTH_OPT, &value) == 0) {
-                        ret = xlator_set_option (rbxl, "ssl-cert-depth", value);
-                        if (ret) {
-                                gf_log ("glusterd", GF_LOG_WARNING,
-                                        "failed to set ssl-cert-depth");
-                                return -1;
-                        }
-                }
+        rbxl = volgen_graph_add_nolink(graph, "protocol/client",
+                                       "%s-replace-brick", volinfo->volname);
+        if (!rbxl)
+            return -1;
 
-                if (dict_get_str (set_dict, SSL_CIPHER_LIST_OPT, &value) == 0) {
-                        ret = xlator_set_option (rbxl, "ssl-cipher-list",
-                                                 value);
-                        if (ret) {
-                                gf_log ("glusterd", GF_LOG_WARNING,
-                                        "failed to set ssl-cipher-list");
-                                return -1;
-                        }
-                }
+        ptranst = glusterd_get_trans_type_rb(volinfo->transport_type);
+        if (NULL == ptranst)
+            return -1;
 
-                if (username) {
-                        ret = xlator_set_option (rbxl, "username", username);
-                        if (ret)
-                                return -1;
-                }
+        RPC_SET_OPT(rbxl, SSL_OWN_CERT_OPT, "ssl-own-cert", return -1);
+        RPC_SET_OPT(rbxl, SSL_PRIVATE_KEY_OPT, "ssl-private-key", return -1);
+        RPC_SET_OPT(rbxl, SSL_CA_LIST_OPT, "ssl-ca-list", return -1);
+        RPC_SET_OPT(rbxl, SSL_CRL_PATH_OPT, "ssl-crl-path", return -1);
+        RPC_SET_OPT(rbxl, SSL_CERT_DEPTH_OPT, "ssl-cert-depth", return -1);
+        RPC_SET_OPT(rbxl, SSL_CIPHER_LIST_OPT, "ssl-cipher-list", return -1);
+        RPC_SET_OPT(rbxl, SSL_DH_PARAM_OPT, "ssl-dh-param", return -1);
+        RPC_SET_OPT(rbxl, SSL_EC_CURVE_OPT, "ssl-ec-curve", return -1);
 
-                if (password) {
-                        ret = xlator_set_option (rbxl, "password", password);
-                        if (ret)
-                                return -1;
-                }
+        if (username) {
+            ret = xlator_set_fixed_option(rbxl, "username", username);
+            if (ret)
+                return -1;
+        }
 
-                ret = xlator_set_option (rbxl, "transport-type", ptranst);
-                GF_FREE (ptranst);
-                if (ret)
-                        return -1;
+        if (password) {
+            ret = xlator_set_fixed_option(rbxl, "password", password);
+            if (ret)
+                return -1;
+        }
 
-                xl = volgen_graph_add_nolink (graph, "cluster/pump", "%s-pump",
-                                              volinfo->volname);
-                if (!xl)
-                        return -1;
-                ret = volgen_xlator_link (xl, txl);
-                if (ret)
-                        return -1;
-                ret = volgen_xlator_link (xl, rbxl);
-                if (ret)
-                        return -1;
+        ret = xlator_set_fixed_option(rbxl, "transport-type", ptranst);
+        GF_FREE(ptranst);
+        if (ret)
+            return -1;
+
+        if (dict_get_str_sizen(volinfo->dict, "transport.address-family",
+                               &address_family_data) == 0) {
+            ret = xlator_set_fixed_option(rbxl, "transport.address-family",
+                                          address_family_data);
+            if (ret) {
+                gf_log("glusterd", GF_LOG_WARNING,
+                       "failed to set transport.address-family");
+                return -1;
+            }
         }
 
+        xl = volgen_graph_add_nolink(graph, "cluster/pump", "%s-pump",
+                                     volinfo->volname);
+        if (!xl)
+            return -1;
+        ret = volgen_xlator_link(xl, txl);
+        if (ret)
+            return -1;
+        ret = volgen_xlator_link(xl, rbxl);
+        if (ret)
+            return -1;
+    }
+
 out:
-        return ret;
+    return ret;
 }
 
-
 /* The order of xlator definition here determines
  * the topology of the brick graph */
 static volgen_brick_xlator_t server_graph_table[] = {
-        {brick_graph_add_server, NULL},
-        {brick_graph_add_io_stats, NULL},
-        {brick_graph_add_cdc, NULL},
-        {brick_graph_add_ro, NULL},
-        {brick_graph_add_worm, NULL},
-        {brick_graph_add_quota, "quota"},
-        {brick_graph_add_marker, "marker"},
-        {brick_graph_add_index, "index"},
-        {brick_graph_add_barrier, NULL},
-        {brick_graph_add_iot, "io-threads"},
-        {brick_graph_add_upcall, "upcall"},
-        {brick_graph_add_pump, NULL},
-        {brick_graph_add_locks, "locks"},
-        {brick_graph_add_acl, "acl"},
-	{brick_graph_add_bitrot_stub, "bitrot-stub"},
-        {brick_graph_add_changelog, "changelog"},
-        {brick_graph_add_changetimerecorder, "changetimerecorder"},
-        {brick_graph_add_bd, "bd"},
-        {brick_graph_add_trash, "trash"},
-        {brick_graph_add_arbiter, "arbiter"},
-        {brick_graph_add_posix, "posix"},
+    {brick_graph_add_server, NULL},
+    {brick_graph_add_io_stats, "NULL"},
+    {brick_graph_add_sdfs, "sdfs"},
+    {brick_graph_add_namespace, "namespace"},
+    {brick_graph_add_cdc, NULL},
+    {brick_graph_add_quota, "quota"},
+    {brick_graph_add_index, "index"},
+    {brick_graph_add_barrier, NULL},
+    {brick_graph_add_marker, "marker"},
+    {brick_graph_add_selinux, "selinux"},
+    {brick_graph_add_iot, "io-threads"},
+    {brick_graph_add_upcall, "upcall"},
+    {brick_graph_add_leases, "leases"},
+    {brick_graph_add_pump, NULL},
+    {brick_graph_add_ro, NULL},
+    {brick_graph_add_worm, NULL},
+    {brick_graph_add_locks, "locks"},
+    {brick_graph_add_acl, "acl"},
+    {brick_graph_add_bitrot_stub, "bitrot-stub"},
+    {brick_graph_add_changelog, "changelog"},
+    {brick_graph_add_trash, "trash"},
+    {brick_graph_add_arbiter, "arbiter"},
+    {brick_graph_add_posix, "posix"},
 };
 
 static glusterd_server_xlator_t
-get_server_xlator (char *xlator)
+get_server_xlator(char *xlator)
 {
-        int i = 0;
-        int size = sizeof (server_graph_table)/sizeof (server_graph_table[0]);
+    int i = 0;
+    int size = sizeof(server_graph_table) / sizeof(server_graph_table[0]);
 
-        for (i = 0; i < size; i++) {
-                if (!server_graph_table[i].dbg_key)
-                        continue;
-                if (strcmp (xlator, server_graph_table[i].dbg_key))
-                        return GF_XLATOR_SERVER;
-        }
+    for (i = 0; i < size; i++) {
+        if (!server_graph_table[i].dbg_key)
+            continue;
+        if (strcmp(xlator, server_graph_table[i].dbg_key))
+            return GF_XLATOR_SERVER;
+    }
 
-        return GF_XLATOR_NONE;
+    return GF_XLATOR_NONE;
 }
 
 static glusterd_client_xlator_t
-get_client_xlator (char *xlator)
+get_client_xlator(char *xlator)
 {
-        glusterd_client_xlator_t subvol = GF_CLNT_XLATOR_NONE;
+    glusterd_client_xlator_t subvol = GF_CLNT_XLATOR_NONE;
 
-        if (strcmp (xlator, "client") == 0)
-                subvol = GF_CLNT_XLATOR_FUSE;
+    if (strcmp(xlator, "client") == 0)
+        subvol = GF_CLNT_XLATOR_FUSE;
 
-        return subvol;
+    return subvol;
 }
 
 static int
-debugxl_option_handler (volgen_graph_t *graph, struct volopt_map_entry *vme,
+debugxl_option_handler(volgen_graph_t *graph, struct volopt_map_entry *vme,
                        void *param)
 {
-        char *volname = NULL;
-        gf_boolean_t enabled = _gf_false;
+    char *volname = NULL;
+    gf_boolean_t enabled = _gf_false;
 
-        volname = param;
-
-        if (strcmp (vme->option, "!debug") != 0)
-                return 0;
+    volname = param;
 
-        if (!strcmp (vme->key , "debug.trace") ||
-            !strcmp (vme->key, "debug.error-gen")) {
-                if (get_server_xlator (vme->value) == GF_XLATOR_NONE &&
-                    get_client_xlator (vme->value) == GF_CLNT_XLATOR_NONE)
-                        return 0;
-                else
-                        goto add_graph;
-        }
+    if (strcmp(vme->option, "!debug") != 0)
+        return 0;
 
-        if (gf_string2boolean (vme->value, &enabled) == -1)
-                return -1;
-        if (!enabled)
-                return 0;
+    if (!strcmp(vme->key, "debug.trace") ||
+        !strcmp(vme->key, "debug.error-gen") ||
+        !strcmp(vme->key, "debug.delay-gen")) {
+        if (get_server_xlator(vme->value) == GF_XLATOR_NONE &&
+            get_client_xlator(vme->value) == GF_CLNT_XLATOR_NONE)
+            return 0;
+    }
+
+    if (gf_string2boolean(vme->value, &enabled) == -1)
+        goto add_graph;
+    if (!enabled)
+        return 0;
 
 add_graph:
-        if (volgen_graph_add (graph, vme->voltype, volname))
-                return 0;
-        else
-                return -1;
+    if (strcmp(vme->value, "off") == 0)
+        return 0;
+    if (volgen_graph_add(graph, vme->voltype, volname))
+        return 0;
+    else
+        return -1;
 }
 
 int
-check_and_add_debug_xl (volgen_graph_t *graph, dict_t *set_dict, char *volname,
-                        char *xlname)
+check_and_add_debug_xl(volgen_graph_t *graph, dict_t *set_dict, char *volname,
+                       char *xlname)
 {
-        int       ret       = 0;
-        char     *value_str = NULL;
+    int i = 0;
+    int ret = 0;
+    char *value_str = NULL;
+    static char *xls[] = {"debug.trace", "debug.error-gen", "debug.delay-gen",
+                          NULL};
 
-        if (!xlname)
-                goto out;
+    if (!xlname)
+        goto out;
 
-        ret = dict_get_str (set_dict, "debug.trace", &value_str);
+    while (xls[i]) {
+        ret = dict_get_str(set_dict, xls[i], &value_str);
         if (!ret) {
-                if (strcmp (xlname, value_str) == 0) {
-                        ret = volgen_graph_set_options_generic (graph,
-                                                set_dict, volname,
-                                                &debugxl_option_handler);
-                        if (ret)
-                                goto out;
-                }
-        }
-
-        ret = dict_get_str (set_dict, "debug.error-gen", &value_str);
-        if (!ret) {
-                if (strcmp (xlname, value_str) == 0) {
-                        ret = volgen_graph_set_options_generic (graph,
-                                                set_dict, volname,
-                                                &debugxl_option_handler);
-                        if (ret)
-                        goto out;
-                }
+            if (strcmp(xlname, value_str) == 0) {
+                ret = volgen_graph_set_options_generic(graph, set_dict, volname,
+                                                       &debugxl_option_handler);
+                if (ret)
+                    goto out;
+            }
         }
-
-        ret = 0;
+        i++;
+    }
+    ret = 0;
 
 out:
-        return ret;
+    return ret;
 }
 
 static int
-server_graph_builder (volgen_graph_t *graph, glusterd_volinfo_t *volinfo,
-                      dict_t *set_dict, void *param)
+server_graph_builder(volgen_graph_t *graph, glusterd_volinfo_t *volinfo,
+                     dict_t *set_dict, void *param)
 {
-        int                   ret           = 0;
-        char                 *xlator        = NULL;
-        char                 *loglevel      = NULL;
-        int                   i             = 0;
-
-
-        i = sizeof (server_graph_table)/sizeof (server_graph_table[0]) - 1;
+    int ret = 0;
+    char *xlator = NULL;
+    char *loglevel = NULL;
+    int i = 0;
 
-        while (i >= 0) {
-                ret = server_graph_table[i].builder (graph, volinfo, set_dict,
-                                                     param);
-                if (ret) {
-                        gf_log ("glusterd", GF_LOG_ERROR, "Builing graph "
-                                "failed for server graph table entry: %d", i);
-                        goto out;
-                }
+    i = sizeof(server_graph_table) / sizeof(server_graph_table[0]) - 1;
 
-                ret = check_and_add_debug_xl (graph, set_dict,
-                                              volinfo->volname,
-                                              server_graph_table[i].dbg_key);
-                if (ret)
-                        goto out;
-
-                i--;
+    while (i >= 0) {
+        ret = server_graph_table[i].builder(graph, volinfo, set_dict, param);
+        if (ret) {
+            gf_msg("glusterd", GF_LOG_ERROR, 0, GD_MSG_BUILD_GRAPH_FAILED,
+                   "Builing graph "
+                   "failed for server graph table entry: %d",
+                   i);
+            goto out;
         }
 
+        ret = check_and_add_debug_xl(graph, set_dict, volinfo->volname,
+                                     server_graph_table[i].dbg_key);
+        if (ret)
+            goto out;
 
-        ret = dict_get_str (set_dict, "xlator", &xlator);
+        i--;
+    }
 
-        /* got a cli log level request */
-        if (!ret) {
-                ret = dict_get_str (set_dict, "loglevel", &loglevel);
-                if (ret) {
-                        gf_log ("glusterd", GF_LOG_ERROR, "could not get both"
-                                " translator name and loglevel for log level request");
-                        goto out;
-                }
+    ret = dict_get_str_sizen(set_dict, "xlator", &xlator);
+
+    /* got a cli log level request */
+    if (!ret) {
+        ret = dict_get_str_sizen(set_dict, "loglevel", &loglevel);
+        if (ret) {
+            gf_msg("glusterd", GF_LOG_ERROR, errno, GD_MSG_DICT_GET_FAILED,
+                   "could not get both"
+                   " translator name and loglevel for log level request");
+            goto out;
         }
+    }
 
-        ret = volgen_graph_set_options_generic (graph, set_dict,
-                                                (xlator && loglevel) ? (void *)set_dict : volinfo,
-                                                (xlator && loglevel) ?  &server_spec_extended_option_handler :
-                                                 &server_spec_option_handler);
+    ret = volgen_graph_set_options_generic(
+        graph, set_dict, (xlator && loglevel) ? (void *)set_dict : volinfo,
+        (xlator && loglevel) ? &server_spec_extended_option_handler
+                             : &server_spec_option_handler);
 
- out:
-        return ret;
+out:
+    return ret;
 }
 
-
 /* builds a graph for server role , with option overrides in mod_dict */
 static int
-build_server_graph (volgen_graph_t *graph, glusterd_volinfo_t *volinfo,
-                    dict_t *mod_dict, glusterd_brickinfo_t *brickinfo)
+build_server_graph(volgen_graph_t *graph, glusterd_volinfo_t *volinfo,
+                   dict_t *mod_dict, glusterd_brickinfo_t *brickinfo)
 {
-        return build_graph_generic (graph, volinfo, mod_dict, brickinfo,
-                                    &server_graph_builder);
+    return build_graph_generic(graph, volinfo, mod_dict, brickinfo,
+                               &server_graph_builder);
 }
 
 static int
-perfxl_option_handler (volgen_graph_t *graph, struct volopt_map_entry *vme,
-                       void *param)
+perfxl_option_handler(volgen_graph_t *graph, struct volopt_map_entry *vme,
+                      void *param)
 {
-        gf_boolean_t enabled = _gf_false;
-        glusterd_volinfo_t *volinfo = NULL;
+    gf_boolean_t enabled = _gf_false;
+    glusterd_volinfo_t *volinfo = NULL;
+    xlator_t *this = NULL;
+    glusterd_conf_t *priv = NULL;
 
-        GF_ASSERT (param);
-        volinfo = param;
+    GF_VALIDATE_OR_GOTO("glusterd", param, out);
+    volinfo = param;
+    this = THIS;
+    GF_VALIDATE_OR_GOTO("glusterd", this, out);
+    priv = this->private;
+    GF_VALIDATE_OR_GOTO("glusterd", priv, out);
 
-        if (strcmp (vme->option, "!perf") != 0)
-                return 0;
+    if (strcmp(vme->option, "!perf") != 0)
+        return 0;
 
-        if (gf_string2boolean (vme->value, &enabled) == -1)
-                return -1;
-        if (!enabled)
-                return 0;
+    if (gf_string2boolean(vme->value, &enabled) == -1)
+        return -1;
+    if (!enabled)
+        return 0;
 
-        /* Check op-version before adding the 'open-behind' xlator in the graph
+    /* Check op-version before adding the 'open-behind' xlator in the graph
+     */
+    if (!strcmp(vme->key, "performance.open-behind") &&
+        (vme->op_version > volinfo->client_op_version))
+        return 0;
+
+    if (priv->op_version < GD_OP_VERSION_3_12_2) {
+        /* For replicate volumes do not load io-threads as it affects
+         * performance
          */
-        if (!strcmp (vme->key, "performance.open-behind") &&
-            (vme->op_version > volinfo->client_op_version))
-                return 0;
+        if (!strcmp(vme->key, "performance.client-io-threads") &&
+            (GF_CLUSTER_TYPE_REPLICATE == volinfo->type))
+            return 0;
+    }
+
+    /* if VKEY_READDIR_AHEAD is enabled and parallel readdir is
+     * not enabled then load readdir-ahead here else it will be
+     * loaded as a child of dht */
+    if (!strcmp(vme->key, VKEY_READDIR_AHEAD) &&
+        glusterd_volinfo_get_boolean(volinfo, VKEY_PARALLEL_READDIR))
+        return 0;
 
-        if (volgen_graph_add (graph, vme->voltype, volinfo->volname))
-                return 0;
-        else
-                return -1;
+    if (volgen_graph_add(graph, vme->voltype, volinfo->volname))
+        return 0;
+out:
+    return -1;
 }
 
 static int
-nfsperfxl_option_handler (volgen_graph_t *graph, struct volopt_map_entry *vme,
-                       void *param)
+gfproxy_server_perfxl_option_handler(volgen_graph_t *graph,
+                                     struct volopt_map_entry *vme, void *param)
 {
-        char *volname = NULL;
-        gf_boolean_t enabled = _gf_false;
+    GF_ASSERT(param);
 
-        volname = param;
+    /* write-behind is the *not* allowed for gfproxy-servers */
+    if (strstr(vme->key, "write-behind")) {
+        return 0;
+    }
 
-        if (strcmp (vme->option, "!nfsperf") != 0)
-                return 0;
+    perfxl_option_handler(graph, vme, param);
 
-        if (gf_string2boolean (vme->value, &enabled) == -1)
-                return -1;
-        if (!enabled)
-                return 0;
-
-        if (volgen_graph_add (graph, vme->voltype, volname))
-                return 0;
-        else
-                return -1;
+    return 0;
 }
 
-#if (HAVE_LIB_XML)
-int
-end_sethelp_xml_doc (xmlTextWriterPtr writer)
+static int
+gfproxy_client_perfxl_option_handler(volgen_graph_t *graph,
+                                     struct volopt_map_entry *vme, void *param)
 {
-        int             ret = -1;
+    GF_ASSERT(param);
 
-        ret = xmlTextWriterEndElement(writer);
-        if (ret < 0) {
-                gf_log ("glusterd", GF_LOG_ERROR, "Could not end an "
-                        "xmlElemetnt");
-                ret = -1;
-                goto out;
-        }
-        ret = xmlTextWriterEndDocument (writer);
-        if (ret < 0) {
-                gf_log ("glusterd", GF_LOG_ERROR, "Could not end an "
-                        "xmlDocument");
-                ret = -1;
-                goto out;
-        }
-        ret = 0;
- out:
-        gf_log ("glusterd", GF_LOG_DEBUG, "Returning %d", ret);
-        return ret;
+    /* write-behind is the only allowed "perf" for gfproxy-clients */
+    if (!strstr(vme->key, "write-behind"))
+        return 0;
+
+    perfxl_option_handler(graph, vme, param);
 
+    return 0;
 }
 
-int
-init_sethelp_xml_doc (xmlTextWriterPtr *writer, xmlBufferPtr  *buf)
+#ifdef BUILD_GNFS
+static int
+nfsperfxl_option_handler(volgen_graph_t *graph, struct volopt_map_entry *vme,
+                         void *param)
 {
-        int ret;
-
-        *buf = xmlBufferCreateSize (8192);
-        if (buf == NULL) {
-                gf_log ("glusterd", GF_LOG_ERROR, "Error creating the xml "
-                          "buffer");
-                ret = -1;
-                goto out;
-        }
-
-        xmlBufferSetAllocationScheme (*buf,XML_BUFFER_ALLOC_DOUBLEIT);
+    char *volname = NULL;
+    gf_boolean_t enabled = _gf_false;
 
-        *writer = xmlNewTextWriterMemory(*buf, 0);
-        if (writer == NULL) {
-                gf_log ("glusterd", GF_LOG_ERROR, " Error creating the xml "
-                         "writer");
-                ret = -1;
-                goto out;
-        }
-
-        ret = xmlTextWriterStartDocument(*writer, "1.0", "UTF-8", "yes");
-        if (ret < 0) {
-                gf_log ("glusterd", GF_LOG_ERROR, "Error While starting the "
-                         "xmlDoc");
-                goto out;
-        }
+    volname = param;
 
-        ret = xmlTextWriterStartElement(*writer, (xmlChar *)"options");
-        if (ret < 0) {
-                gf_log ("glusterd", GF_LOG_ERROR, "Could not create an "
-                        "xmlElemetnt");
-                ret = -1;
-                goto out;
-        }
+    if (strcmp(vme->option, "!nfsperf") != 0)
+        return 0;
 
+    if (gf_string2boolean(vme->value, &enabled) == -1)
+        return -1;
+    if (!enabled)
+        return 0;
 
-        ret = 0;
+    if (volgen_graph_add(graph, vme->voltype, volname))
+        return 0;
+    else
+        return -1;
+}
+#endif
 
- out:
-        gf_log ("glusterd", GF_LOG_DEBUG, "Returning %d", ret);
-        return ret;
+#if (HAVE_LIB_XML)
+int
+end_sethelp_xml_doc(xmlTextWriterPtr writer)
+{
+    int ret = -1;
 
+    ret = xmlTextWriterEndElement(writer);
+    if (ret < 0) {
+        gf_msg("glusterd", GF_LOG_ERROR, 0, GD_MSG_XML_TEXT_WRITE_FAIL,
+               "Could not end an "
+               "xmlElement");
+        ret = -1;
+        goto out;
+    }
+    ret = xmlTextWriterEndDocument(writer);
+    if (ret < 0) {
+        gf_msg("glusterd", GF_LOG_ERROR, 0, GD_MSG_XML_TEXT_WRITE_FAIL,
+               "Could not end an "
+               "xmlDocument");
+        ret = -1;
+        goto out;
+    }
+    ret = 0;
+out:
+    gf_msg_debug("glusterd", 0, "Returning %d", ret);
+    return ret;
 }
 
 int
-xml_add_volset_element (xmlTextWriterPtr writer, const char *name,
-                                 const char *def_val, const char *dscrpt)
+init_sethelp_xml_doc(xmlTextWriterPtr *writer, xmlBufferPtr *buf)
 {
+    int ret = -1;
 
-        int                     ret = -1;
+    if (!writer || !buf)
+        goto out;
 
-        GF_ASSERT (name);
+    *buf = xmlBufferCreateSize(8192);
+    if (buf == NULL) {
+        gf_msg("glusterd", GF_LOG_ERROR, ENOMEM, GD_MSG_NO_MEMORY,
+               "Error creating the xml "
+               "buffer");
+        ret = -1;
+        goto out;
+    }
 
-        ret = xmlTextWriterStartElement(writer, (xmlChar *) "option");
-        if (ret < 0) {
-                gf_log ("glusterd", GF_LOG_ERROR, "Could not create an "
-                        "xmlElemetnt");
-                ret = -1;
-                goto out;
-        }
+    xmlBufferSetAllocationScheme(*buf, XML_BUFFER_ALLOC_DOUBLEIT);
 
-        ret = xmlTextWriterWriteFormatElement(writer, (xmlChar*)"defaultValue",
-                                              "%s", def_val);
-        if (ret < 0) {
-                gf_log ("glusterd", GF_LOG_ERROR, "Could not create an "
-                        "xmlElemetnt");
-                ret = -1;
-                goto out;
-        }
+    *writer = xmlNewTextWriterMemory(*buf, 0);
+    if (writer == NULL) {
+        gf_msg("glusterd", GF_LOG_ERROR, ENOMEM, GD_MSG_NO_MEMORY,
+               " Error creating the xml "
+               "writer");
+        ret = -1;
+        goto out;
+    }
+
+    ret = xmlTextWriterStartDocument(*writer, "1.0", "UTF-8", "yes");
+    if (ret < 0) {
+        gf_msg("glusterd", GF_LOG_ERROR, 0, GD_MSG_XML_DOC_START_FAIL,
+               "Error While starting the "
+               "xmlDoc");
+        goto out;
+    }
+
+    ret = xmlTextWriterStartElement(*writer, (xmlChar *)"options");
+    if (ret < 0) {
+        gf_msg("glusterd", GF_LOG_ERROR, 0, GD_MSG_XML_ELE_CREATE_FAIL,
+               "Could not create an "
+               "xmlElement");
+        ret = -1;
+        goto out;
+    }
 
-        ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"description",
-                                              "%s",  dscrpt );
-        if (ret < 0) {
-                gf_log ("glusterd", GF_LOG_ERROR, "Could not create an "
-                        "xmlElemetnt");
-                ret = -1;
-                goto out;
-        }
+    ret = 0;
 
-        ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *) "name", "%s",
-                                               name);
-        if (ret < 0) {
-                gf_log ("glusterd", GF_LOG_ERROR, "Could not create an "
-                        "xmlElemetnt");
-                ret = -1;
-                goto out;
-        }
+out:
+    gf_msg_debug("glusterd", 0, "Returning %d", ret);
+    return ret;
+}
 
-        ret = xmlTextWriterEndElement(writer);
-        if (ret < 0) {
-                gf_log ("glusterd", GF_LOG_ERROR, "Could not end an "
-                        "xmlElemetnt");
-                ret = -1;
-                goto out;
-        }
+int
+xml_add_volset_element(xmlTextWriterPtr writer, const char *name,
+                       const char *def_val, const char *dscrpt)
+{
+    int ret = -1;
 
-        ret = 0;
- out:
-        gf_log ("glusterd", GF_LOG_DEBUG, "Returning %d", ret);
-        return ret;
+    GF_ASSERT(name);
+
+    ret = xmlTextWriterStartElement(writer, (xmlChar *)"option");
+    if (ret < 0) {
+        gf_msg("glusterd", GF_LOG_ERROR, 0, GD_MSG_XML_ELE_CREATE_FAIL,
+               "Could not create an "
+               "xmlElemetnt");
+        ret = -1;
+        goto out;
+    }
+
+    ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"defaultValue",
+                                          "%s", def_val);
+    if (ret < 0) {
+        gf_msg("glusterd", GF_LOG_ERROR, 0, GD_MSG_XML_ELE_CREATE_FAIL,
+               "Could not create an "
+               "xmlElemetnt");
+        ret = -1;
+        goto out;
+    }
+
+    ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"description",
+                                          "%s", dscrpt);
+    if (ret < 0) {
+        gf_msg("glusterd", GF_LOG_ERROR, 0, GD_MSG_XML_ELE_CREATE_FAIL,
+               "Could not create an "
+               "xmlElemetnt");
+        ret = -1;
+        goto out;
+    }
+
+    ret = xmlTextWriterWriteFormatElement(writer, (xmlChar *)"name", "%s",
+                                          name);
+    if (ret < 0) {
+        gf_msg("glusterd", GF_LOG_ERROR, 0, GD_MSG_XML_ELE_CREATE_FAIL,
+               "Could not create an "
+               "xmlElemetnt");
+        ret = -1;
+        goto out;
+    }
+
+    ret = xmlTextWriterEndElement(writer);
+    if (ret < 0) {
+        gf_msg("glusterd", GF_LOG_ERROR, 0, GD_MSG_XML_ELE_CREATE_FAIL,
+               "Could not end an "
+               "xmlElemetnt");
+        ret = -1;
+        goto out;
+    }
 
+    ret = 0;
+out:
+    gf_msg_debug("glusterd", 0, "Returning %d", ret);
+    return ret;
 }
 
 #endif
 
 int
-_get_xlator_opt_key_from_vme ( struct volopt_map_entry *vme, char **key)
-{
-        int ret = 0;
-
-        GF_ASSERT (vme);
-        GF_ASSERT (key);
-
-
-        if (!strcmp (vme->key, AUTH_ALLOW_MAP_KEY))
-                *key = gf_strdup (AUTH_ALLOW_OPT_KEY);
-        else if (!strcmp (vme->key, AUTH_REJECT_MAP_KEY))
-                *key = gf_strdup (AUTH_REJECT_OPT_KEY);
-        else if (!strcmp (vme->key, NFS_DISABLE_MAP_KEY))
-                *key = gf_strdup (NFS_DISABLE_OPT_KEY);
-        else {
-                if (vme->option) {
-                        if  (vme->option[0] == '!') {
-                                *key = vme->option + 1;
-                                if (!*key[0])
-                                        ret = -1;
-                        } else {
-                                *key = vme->option;
-                        }
-                } else {
-                        *key = strchr (vme->key, '.');
-                        if (*key) {
-                                (*key) ++;
-                                if (!*key[0])
-                                        ret = -1;
-                        } else {
-                                ret = -1;
-                        }
-                }
+_get_xlator_opt_key_from_vme(struct volopt_map_entry *vme, char **key)
+{
+    int ret = 0;
+
+    GF_ASSERT(vme);
+    GF_ASSERT(key);
+
+    if (!strcmp(vme->key, AUTH_ALLOW_MAP_KEY))
+        *key = gf_strdup(AUTH_ALLOW_OPT_KEY);
+    else if (!strcmp(vme->key, AUTH_REJECT_MAP_KEY))
+        *key = gf_strdup(AUTH_REJECT_OPT_KEY);
+#ifdef BUILD_GNFS
+    else if (!strcmp(vme->key, NFS_DISABLE_MAP_KEY))
+        *key = gf_strdup(NFS_DISABLE_OPT_KEY);
+#endif
+    else {
+        if (vme->option) {
+            if (vme->option[0] == '!') {
+                *key = vme->option + 1;
+                if (!*key[0])
+                    ret = -1;
+            } else {
+                *key = vme->option;
+            }
+        } else {
+            *key = strchr(vme->key, '.');
+            if (*key) {
+                (*key)++;
+                if (!*key[0])
+                    ret = -1;
+            } else {
+                ret = -1;
+            }
         }
-        if (ret)
-                gf_log ("glusterd", GF_LOG_ERROR, "Wrong entry found in  "
-                        "glusterd_volopt_map entry %s", vme->key);
-        else
-                gf_log ("glusterd", GF_LOG_DEBUG, "Returning %d", ret);
+    }
+    if (ret)
+        gf_msg("glusterd", GF_LOG_ERROR, EINVAL, GD_MSG_INVALID_ENTRY,
+               "Wrong entry found in  "
+               "glusterd_volopt_map entry %s",
+               vme->key);
+    else
+        gf_msg_debug("glusterd", 0, "Returning %d", ret);
 
-        return ret;
+    return ret;
 }
 
 void
-_free_xlator_opt_key (char *key)
+_free_xlator_opt_key(char *key)
 {
-        GF_ASSERT (key);
+    GF_ASSERT(key);
 
-        if (!strcmp (key, AUTH_ALLOW_OPT_KEY) ||
-            !strcmp (key, AUTH_REJECT_OPT_KEY) ||
-            !strcmp (key, NFS_DISABLE_OPT_KEY))
-                GF_FREE (key);
+    if (!strcmp(key, AUTH_ALLOW_OPT_KEY) || !strcmp(key, AUTH_REJECT_OPT_KEY) ||
+        !strcmp(key, NFS_DISABLE_OPT_KEY))
+        GF_FREE(key);
 
-        return;
+    return;
 }
 
 static xlator_t *
-volgen_graph_build_client (volgen_graph_t *graph, glusterd_volinfo_t *volinfo,
-                           char *hostname, char *subvol, char *xl_id,
-                           char *transt, dict_t *set_dict)
-{
-        xlator_t                *xl                 = NULL;
-        int                      ret                = -2;
-        uint32_t                 client_type        = GF_CLIENT_OTHER;
-        char                    *str                = NULL;
-        char                    *ssl_str            = NULL;
-        gf_boolean_t             ssl_bool           = _gf_false;
-        char                    *value              = NULL;
-
-        GF_ASSERT (graph);
-        GF_ASSERT (subvol);
-        GF_ASSERT (xl_id);
-        GF_ASSERT (transt);
-
-        xl = volgen_graph_add_nolink (graph, "protocol/client",
-                                      "%s", xl_id);
-        if (!xl)
-                goto err;
-
-        ret = xlator_set_option (xl, "ping-timeout", "42");
+volgen_graph_build_client(volgen_graph_t *graph, glusterd_volinfo_t *volinfo,
+                          char *hostname, char *port, char *subvol, char *xl_id,
+                          char *transt, dict_t *set_dict)
+{
+    xlator_t *xl = NULL;
+    int ret = -2;
+    uint32_t client_type = GF_CLIENT_OTHER;
+    char *str = NULL;
+    char *ssl_str = NULL;
+    gf_boolean_t ssl_bool = _gf_false;
+    char *address_family_data = NULL;
+
+    GF_ASSERT(graph);
+    GF_ASSERT(subvol);
+    GF_ASSERT(xl_id);
+    GF_ASSERT(transt);
+
+    xl = volgen_graph_add_nolink(graph, "protocol/client", "%s", xl_id);
+    if (!xl)
+        goto err;
+
+    ret = xlator_set_fixed_option(xl, "ping-timeout", "42");
+    if (ret)
+        goto err;
+
+    if (hostname) {
+        ret = xlator_set_fixed_option(xl, "remote-host", hostname);
         if (ret)
-                goto err;
+            goto err;
+    }
 
-        if (hostname) {
-                ret = xlator_set_option (xl, "remote-host", hostname);
-                if (ret)
-                        goto err;
-        }
-
-        ret = xlator_set_option (xl, "remote-subvolume", subvol);
+    if (port) {
+        ret = xlator_set_fixed_option(xl, "remote-port", port);
         if (ret)
-                goto err;
+            goto err;
+    }
 
-        ret = xlator_set_option (xl, "transport-type", transt);
-        if (ret)
-                goto err;
+    ret = xlator_set_fixed_option(xl, "remote-subvolume", subvol);
+    if (ret)
+        goto err;
 
-        ret = dict_get_uint32 (set_dict, "trusted-client",
-                               &client_type);
-
-        if (!ret && client_type == GF_CLIENT_TRUSTED) {
-                str = NULL;
-                str = glusterd_auth_get_username (volinfo);
-                if (str) {
-                        ret = xlator_set_option (xl, "username",
-                                                 str);
-                        if (ret)
-                                goto err;
-                }
+    ret = xlator_set_fixed_option(xl, "transport-type", transt);
+    if (ret)
+        goto err;
 
-                str = glusterd_auth_get_password (volinfo);
-                if (str) {
-                        ret = xlator_set_option (xl, "password",
-                                                 str);
-                        if (ret)
-                                goto err;
-                }
+    if (dict_get_str_sizen(volinfo->dict, "transport.address-family",
+                           &address_family_data) == 0) {
+        ret = xlator_set_fixed_option(xl, "transport.address-family",
+                                      address_family_data);
+        if (ret) {
+            gf_log("glusterd", GF_LOG_WARNING,
+                   "failed to set transport.address-family");
+            goto err;
         }
+    }
 
-        if (dict_get_str(set_dict,"client.ssl",&ssl_str) == 0) {
-                if (gf_string2boolean(ssl_str,&ssl_bool) == 0) {
-                        if (ssl_bool) {
-                                ret = xlator_set_option(xl,
-                                        "transport.socket.ssl-enabled",
-                                        "true");
-                                if (ret) {
-                                        goto err;
-                                }
-                        }
-                }
+    ret = dict_get_uint32(set_dict, "trusted-client", &client_type);
+
+    if (!ret && (client_type == GF_CLIENT_TRUSTED ||
+                 client_type == GF_CLIENT_TRUSTED_PROXY)) {
+        str = NULL;
+        str = glusterd_auth_get_username(volinfo);
+        if (str) {
+            ret = xlator_set_fixed_option(xl, "username", str);
+            if (ret)
+                goto err;
         }
 
-        if (dict_get_str (set_dict, SSL_CERT_DEPTH_OPT, &value) == 0) {
-                ret = xlator_set_option (xl, "ssl-cert-depth", value);
-                if (ret) {
-                        gf_log ("glusterd", GF_LOG_WARNING,
-                                "failed to set ssl-cert-depth");
-                        goto err;
-                }
+        str = glusterd_auth_get_password(volinfo);
+        if (str) {
+            ret = xlator_set_fixed_option(xl, "password", str);
+            if (ret)
+                goto err;
         }
+    }
 
-        if (dict_get_str (set_dict, SSL_CIPHER_LIST_OPT, &value) == 0) {
-                ret = xlator_set_option (xl, "ssl-cipher-list", value);
+    if (dict_get_str_sizen(set_dict, "client.ssl", &ssl_str) == 0) {
+        if (gf_string2boolean(ssl_str, &ssl_bool) == 0) {
+            if (ssl_bool) {
+                ret = xlator_set_fixed_option(
+                    xl, "transport.socket.ssl-enabled", "true");
                 if (ret) {
-                        gf_log ("glusterd", GF_LOG_WARNING,
-                                "failed to set ssl-cipher-list");
-                        goto err;
+                    goto err;
                 }
+            }
         }
+    }
 
-        return xl;
+    RPC_SET_OPT(xl, SSL_OWN_CERT_OPT, "ssl-own-cert", goto err);
+    RPC_SET_OPT(xl, SSL_PRIVATE_KEY_OPT, "ssl-private-key", goto err);
+    RPC_SET_OPT(xl, SSL_CA_LIST_OPT, "ssl-ca-list", goto err);
+    RPC_SET_OPT(xl, SSL_CRL_PATH_OPT, "ssl-crl-path", goto err);
+    RPC_SET_OPT(xl, SSL_CERT_DEPTH_OPT, "ssl-cert-depth", goto err);
+    RPC_SET_OPT(xl, SSL_CIPHER_LIST_OPT, "ssl-cipher-list", goto err);
+    RPC_SET_OPT(xl, SSL_DH_PARAM_OPT, "ssl-dh-param", goto err);
+    RPC_SET_OPT(xl, SSL_EC_CURVE_OPT, "ssl-ec-curve", goto err);
+
+    return xl;
 err:
-        return NULL;
+    return NULL;
 }
 
 static int
-volgen_graph_build_clients (volgen_graph_t *graph, glusterd_volinfo_t *volinfo,
-                            dict_t *set_dict, void *param)
-{
-        int                      i                  = 0;
-        int                      ret                = -1;
-        char                     transt[16]         = {0,};
-        char                    *volname            = NULL;
-        glusterd_brickinfo_t    *brick              = NULL;
-        xlator_t                *xl                 = NULL;
-
-        volname = volinfo->volname;
-
-        if (volinfo->brick_count == 0) {
-                gf_log ("", GF_LOG_ERROR,
-                        "volume inconsistency: brick count is 0");
-                goto out;
-        }
-
-        if ((volinfo->dist_leaf_count < volinfo->brick_count) &&
-            ((volinfo->brick_count % volinfo->dist_leaf_count) != 0)) {
-                gf_log ("", GF_LOG_ERROR,
-                        "volume inconsistency: "
-                        "total number of bricks (%d) is not divisible with "
-                        "number of bricks per cluster (%d) in a multi-cluster "
-                        "setup",
-                        volinfo->brick_count, volinfo->dist_leaf_count);
-                goto out;
-        }
-
-        get_transport_type (volinfo, set_dict, transt, _gf_false);
-
-        if (!strcmp (transt, "tcp,rdma"))
-                strcpy (transt, "tcp");
-
-        i = 0;
-        cds_list_for_each_entry (brick, &volinfo->bricks, brick_list) {
-                xl = volgen_graph_build_client (graph, volinfo,
-                                                brick->hostname, brick->path,
-                                                brick->brick_id,
-                                                transt, set_dict);
-                if (!xl) {
+volgen_graph_build_clients(volgen_graph_t *graph, glusterd_volinfo_t *volinfo,
+                           dict_t *set_dict, void *param)
+{
+    int i = 0;
+    int ret = -1;
+    char transt[16] = {
+        0,
+    };
+    glusterd_brickinfo_t *brick = NULL;
+    glusterd_brickinfo_t *ta_brick = NULL;
+    xlator_t *xl = NULL;
+    int subvol_index = 0;
+    int thin_arbiter_index = 0;
+
+    if (volinfo->brick_count == 0) {
+        gf_msg("glusterd", GF_LOG_ERROR, 0, GD_MSG_VOLUME_INCONSISTENCY,
+               "volume inconsistency: brick count is 0");
+        goto out;
+    }
+
+    if ((volinfo->dist_leaf_count < volinfo->brick_count) &&
+        ((volinfo->brick_count % volinfo->dist_leaf_count) != 0)) {
+        gf_msg("glusterd", GF_LOG_ERROR, 0, GD_MSG_VOLUME_INCONSISTENCY,
+               "volume inconsistency: "
+               "total number of bricks (%d) is not divisible with "
+               "number of bricks per cluster (%d) in a multi-cluster "
+               "setup",
+               volinfo->brick_count, volinfo->dist_leaf_count);
+        goto out;
+    }
+
+    get_transport_type(volinfo, set_dict, transt, _gf_false);
+
+    if (!strcmp(transt, "tcp,rdma"))
+        strcpy(transt, "tcp");
+
+    i = 0;
+    cds_list_for_each_entry(brick, &volinfo->bricks, brick_list)
+    {
+        /* insert ta client xlator entry.
+         * eg - If subvol count is > 1, then after every two client xlator
+         * entries there should be a ta client xlator entry in the volfile. ta
+         * client xlator indexes are - 2, 5, 8 etc depending on the index of
+         * subvol.
+         */
+        if (volinfo->thin_arbiter_count &&
+            (i + 1) % (volinfo->replica_count + 1) == 0) {
+            thin_arbiter_index = 0;
+            cds_list_for_each_entry(ta_brick, &volinfo->ta_bricks, brick_list)
+            {
+                if (thin_arbiter_index == subvol_index) {
+                    xl = volgen_graph_build_client(
+                        graph, volinfo, ta_brick->hostname, NULL,
+                        ta_brick->path, ta_brick->brick_id, transt, set_dict);
+                    if (!xl) {
                         ret = -1;
                         goto out;
+                    }
+                }
+                thin_arbiter_index++;
+            }
+            subvol_index++;
+        }
+        xl = volgen_graph_build_client(graph, volinfo, brick->hostname, NULL,
+                                       brick->path, brick->brick_id, transt,
+                                       set_dict);
+        if (!xl) {
+            ret = -1;
+            goto out;
+        }
+
+        i++;
+    }
+
+    /* Add ta client xlator entry for last subvol
+     * Above loop will miss out on making the ta client
+     * xlator entry for the last subvolume in the volfile
+     */
+    if (volinfo->thin_arbiter_count) {
+        thin_arbiter_index = 0;
+        cds_list_for_each_entry(ta_brick, &volinfo->ta_bricks, brick_list)
+        {
+            if (thin_arbiter_index == subvol_index) {
+                xl = volgen_graph_build_client(
+                    graph, volinfo, ta_brick->hostname, NULL, ta_brick->path,
+                    ta_brick->brick_id, transt, set_dict);
+                if (!xl) {
+                    ret = -1;
+                    goto out;
                 }
+            }
 
-                i++;
+            thin_arbiter_index++;
         }
+    }
 
-        if (i != volinfo->brick_count) {
-                gf_log ("", GF_LOG_ERROR,
-                        "volume inconsistency: actual number of bricks (%d) "
-                        "differs from brick count (%d)", i,
-                        volinfo->brick_count);
+    if (i != volinfo->brick_count) {
+        gf_msg("glusterd", GF_LOG_ERROR, 0, GD_MSG_VOLUME_INCONSISTENCY,
+               "volume inconsistency: actual number of bricks (%d) "
+               "differs from brick count (%d)",
+               i, volinfo->brick_count);
 
-                ret = -1;
-                goto out;
-        }
-        ret = 0;
+        ret = -1;
+        goto out;
+    }
+    ret = 0;
 out:
-        return ret;
+    return ret;
 }
 
 static int
-volgen_link_bricks (volgen_graph_t *graph,
-                    glusterd_volinfo_t *volinfo, char *xl_type,
-                    char *xl_namefmt, size_t child_count,
-                    size_t sub_count,
-                    xlator_t *trav)
-{
-        int             i = 0;
-        int             j = 0;
-        xlator_t        *xl  = NULL;
-        char            *volname = NULL;
-        int             ret     = -1;
-
-        if (child_count == 0)
-                goto out;
-        volname = volinfo->volname;
+volgen_link_bricks(volgen_graph_t *graph, glusterd_volinfo_t *volinfo,
+                   char *xl_type, char *xl_namefmt, size_t child_count,
+                   size_t sub_count, size_t start_count, xlator_t *trav)
+{
+    int i = 0;
+    int j = start_count;
+    xlator_t *xl = NULL;
+    char *volname = NULL;
+    int ret = -1;
 
-        for (;; trav = trav->prev) {
-                if ((i % sub_count) == 0) {
-                        xl = volgen_graph_add_nolink (graph, xl_type,
-                                                      xl_namefmt, volname, j);
-                        if (!xl) {
-                                ret = -1;
-                                goto out;
-                        }
-                        j++;
-                }
+    if (child_count == 0)
+        goto out;
+    volname = volinfo->volname;
 
-                ret = volgen_xlator_link (xl, trav);
-                if (ret)
-                        goto out;
+    for (;; trav = trav->prev) {
+        if ((i % sub_count) == 0) {
+            xl = volgen_graph_add_nolink(graph, xl_type, xl_namefmt, volname,
+                                         j);
+            j++;
+        }
 
-                i++;
-                if (i == child_count)
-                        break;
+        if (!xl) {
+            ret = -1;
+            goto out;
+        }
+
+        if (strncmp(xl_type, "performance/readdir-ahead",
+                    SLEN("performance/readdir-ahead")) == 0) {
+            ret = xlator_set_fixed_option(xl, "performance.readdir-ahead",
+                                          "on");
+            if (ret)
+                goto out;
         }
 
-        ret = j;
+        ret = volgen_xlator_link(xl, trav);
+        if (ret)
+            goto out;
+
+        i++;
+        if (i == child_count)
+            break;
+    }
+
+    ret = j - start_count;
 out:
-        return ret;
+    return ret;
 }
 
 static int
-volgen_link_bricks_from_list_tail (volgen_graph_t *graph,
-                             glusterd_volinfo_t *volinfo,
-                             char *xl_type,
-                             char *xl_namefmt, size_t child_count,
-                             size_t sub_count)
+volgen_link_bricks_from_list_tail_start(volgen_graph_t *graph,
+                                        glusterd_volinfo_t *volinfo,
+                                        char *xl_type, char *xl_namefmt,
+                                        size_t child_count, size_t sub_count,
+                                        size_t start_count)
 {
-        xlator_t *trav = NULL;
-        size_t     cnt = child_count;
+    xlator_t *trav = NULL;
+    size_t cnt = child_count;
 
-        for (trav = first_of(graph); --cnt; trav = trav->next)
-                ;
+    if (!cnt)
+        return -1;
+
+    for (trav = first_of(graph); --cnt; trav = trav->next)
+        ;
 
-        return volgen_link_bricks (graph,  volinfo,
-                                   xl_type,
-                                   xl_namefmt,
-                                   child_count,
-                                   sub_count,
-                                   trav);
+    return volgen_link_bricks(graph, volinfo, xl_type, xl_namefmt, child_count,
+                              sub_count, start_count, trav);
 }
 
 static int
-volgen_link_bricks_from_list_head (volgen_graph_t *graph,
-                                   glusterd_volinfo_t *volinfo, char *xl_type,
-                                   char *xl_namefmt, size_t child_count,
-                                   size_t sub_count)
+volgen_link_bricks_from_list_tail(volgen_graph_t *graph,
+                                  glusterd_volinfo_t *volinfo, char *xl_type,
+                                  char *xl_namefmt, size_t child_count,
+                                  size_t sub_count)
 {
-        xlator_t *trav = NULL;
+    xlator_t *trav = NULL;
+    size_t cnt = child_count;
+
+    if (!cnt)
+        return -1;
 
-        for (trav = first_of(graph); trav->next; trav = trav->next)
-                ;
+    for (trav = first_of(graph); --cnt; trav = trav->next)
+        ;
 
-        return volgen_link_bricks (graph,  volinfo,
-                                   xl_type,
-                                   xl_namefmt,
-                                   child_count,
-                                   sub_count,
-                                   trav);
+    return volgen_link_bricks(graph, volinfo, xl_type, xl_namefmt, child_count,
+                              sub_count, 0, trav);
 }
 
 /**
@@ -2830,2588 +3433,3322 @@ volgen_link_bricks_from_list_head (volgen_graph_t *graph,
  * Generates  snapview-client
  */
 static int
-volgen_graph_build_snapview_client (volgen_graph_t *graph,
-                                    glusterd_volinfo_t *volinfo,
-                                    char *volname, dict_t *set_dict)
-{
-        int                ret                  = 0;
-        xlator_t          *prev_top             = NULL;
-        xlator_t          *prot_clnt            = NULL;
-        xlator_t          *svc                  = NULL;
-        char               transt [16]          = {0,};
-        char              *svc_args[]           = {"features/snapview-client",
-                                                   "%s-snapview-client"};
-        char               subvol [1024]        = {0,};
-        char               xl_id [1024]         = {0,};
-
-        prev_top = (xlator_t *)(graph->graph.first);
-
-        snprintf (subvol, sizeof (subvol), "snapd-%s", volinfo->volname);
-        snprintf (xl_id, sizeof (xl_id), "%s-snapd-client", volinfo->volname);
-
-        get_transport_type (volinfo, set_dict, transt, _gf_false);
-
-        prot_clnt = volgen_graph_build_client (graph, volinfo, NULL, subvol,
-                                               xl_id, transt, set_dict);
-        if (!prot_clnt) {
-                ret = -1;
-                goto out;
-        }
-
-        svc = volgen_graph_add_nolink (graph, svc_args[0], svc_args[1],
-                                       volname);
-        if (!svc) {
-                ret = -1;
-                goto out;
-        }
-
-        /**
-         * Ordering the below two traslators (cur_top & prot_clnt) is important
-         * as snapview client implementation is built on the policy that
-         * normal volume path goes to FIRST_CHILD and snap world operations
-         * goes to SECOND_CHILD
-         **/
-        ret = volgen_xlator_link (graph->graph.first, prev_top);
-        if (ret) {
-                gf_log (THIS->name, GF_LOG_ERROR, "failed to link the "
-                        "snapview-client to distribute");
-                goto out;
-        }
+volgen_graph_build_snapview_client(volgen_graph_t *graph,
+                                   glusterd_volinfo_t *volinfo, char *volname,
+                                   dict_t *set_dict)
+{
+    int ret = 0;
+    xlator_t *prev_top = NULL;
+    xlator_t *prot_clnt = NULL;
+    xlator_t *svc = NULL;
+    char transt[16] = {
+        0,
+    };
+    char *svc_args[] = {"features/snapview-client", "%s-snapview-client"};
+    char subvol[1024] = {
+        0,
+    };
+    char xl_id[1024] = {
+        0,
+    };
+
+    prev_top = (xlator_t *)(graph->graph.first);
+
+    snprintf(subvol, sizeof(subvol), "snapd-%s", volinfo->volname);
+    snprintf(xl_id, sizeof(xl_id), "%s-snapd-client", volinfo->volname);
+
+    get_transport_type(volinfo, set_dict, transt, _gf_false);
+
+    prot_clnt = volgen_graph_build_client(graph, volinfo, NULL, NULL, subvol,
+                                          xl_id, transt, set_dict);
+    if (!prot_clnt) {
+        ret = -1;
+        goto out;
+    }
 
-        ret = volgen_xlator_link (graph->graph.first, prot_clnt);
-        if (ret) {
-                gf_log (THIS->name, GF_LOG_ERROR, "failed to link the "
-                        "snapview-client to snapview-server");
-                goto out;
-        }
+    svc = volgen_graph_add_nolink(graph, svc_args[0], svc_args[1], volname);
+    if (!svc) {
+        ret = -1;
+        goto out;
+    }
+
+    /**
+     * Ordering the below two traslators (cur_top & prot_clnt) is important
+     * as snapview client implementation is built on the policy that
+     * normal volume path goes to FIRST_CHILD and snap world operations
+     * goes to SECOND_CHILD
+     **/
+    ret = volgen_xlator_link(graph->graph.first, prev_top);
+    if (ret) {
+        gf_msg(THIS->name, GF_LOG_ERROR, 0, GD_MSG_XLATOR_LINK_FAIL,
+               "failed to link the "
+               "snapview-client to distribute");
+        goto out;
+    }
+
+    ret = volgen_xlator_link(graph->graph.first, prot_clnt);
+    if (ret) {
+        gf_msg(THIS->name, GF_LOG_ERROR, 0, GD_MSG_XLATOR_LINK_FAIL,
+               "failed to link the "
+               "snapview-client to snapview-server");
+        goto out;
+    }
 
 out:
-        return ret;
+    return ret;
 }
 
 gf_boolean_t
-_xl_is_client_decommissioned (xlator_t *xl, glusterd_volinfo_t *volinfo)
-{
-        int             ret = 0;
-        gf_boolean_t    decommissioned = _gf_false;
-        char            *hostname = NULL;
-        char            *path = NULL;
+_xl_is_client_decommissioned(xlator_t *xl, glusterd_volinfo_t *volinfo)
+{
+    int ret = 0;
+    gf_boolean_t decommissioned = _gf_false;
+    char *hostname = NULL;
+    char *path = NULL;
+
+    GF_ASSERT(!strcmp(xl->type, "protocol/client"));
+    ret = xlator_get_fixed_option(xl, "remote-host", &hostname);
+    if (ret) {
+        GF_ASSERT(0);
+        gf_msg("glusterd", GF_LOG_ERROR, 0, GD_MSG_REMOTE_HOST_GET_FAIL,
+               "Failed to get remote-host "
+               "from client %s",
+               xl->name);
+        goto out;
+    }
+    ret = xlator_get_fixed_option(xl, "remote-subvolume", &path);
+    if (ret) {
+        GF_ASSERT(0);
+        gf_msg("glusterd", GF_LOG_ERROR, 0, GD_MSG_REMOTE_HOST_GET_FAIL,
+               "Failed to get remote-host "
+               "from client %s",
+               xl->name);
+        goto out;
+    }
+
+    decommissioned = glusterd_is_brick_decommissioned(volinfo, hostname, path);
+out:
+    return decommissioned;
+}
 
-        GF_ASSERT (!strcmp (xl->type, "protocol/client"));
-        ret = xlator_get_option (xl, "remote-host", &hostname);
-        if (ret) {
-                GF_ASSERT (0);
-                gf_log ("glusterd", GF_LOG_ERROR, "Failed to get remote-host "
-                        "from client %s", xl->name);
-                goto out;
-        }
-        ret = xlator_get_option (xl, "remote-subvolume", &path);
-        if (ret) {
-                GF_ASSERT (0);
-                gf_log ("glusterd", GF_LOG_ERROR, "Failed to get remote-host "
-                        "from client %s", xl->name);
-                goto out;
-        }
+gf_boolean_t
+_xl_has_decommissioned_clients(xlator_t *xl, glusterd_volinfo_t *volinfo)
+{
+    xlator_list_t *xl_child = NULL;
+    gf_boolean_t decommissioned = _gf_false;
+    xlator_t *cxl = NULL;
+
+    if (!xl)
+        goto out;
+
+    if (!strcmp(xl->type, "protocol/client")) {
+        decommissioned = _xl_is_client_decommissioned(xl, volinfo);
+        goto out;
+    }
+
+    xl_child = xl->children;
+    while (xl_child) {
+        cxl = xl_child->xlator;
+        /* this can go into 2 depths if the volume type
+           is stripe-replicate */
+        decommissioned = _xl_has_decommissioned_clients(cxl, volinfo);
+        if (decommissioned)
+            break;
+
+        xl_child = xl_child->next;
+    }
+out:
+    return decommissioned;
+}
 
-        decommissioned = glusterd_is_brick_decommissioned (volinfo, hostname,
-                                                           path);
+static int
+_graph_get_decommissioned_children(xlator_t *dht, glusterd_volinfo_t *volinfo,
+                                   char **children)
+{
+    int ret = -1;
+    xlator_list_t *xl_child = NULL;
+    xlator_t *cxl = NULL;
+    gf_boolean_t comma = _gf_false;
+
+    *children = NULL;
+    xl_child = dht->children;
+    while (xl_child) {
+        cxl = xl_child->xlator;
+        if (_xl_has_decommissioned_clients(cxl, volinfo)) {
+            if (!*children) {
+                *children = GF_CALLOC(16 * GF_UNIT_KB, 1, gf_common_mt_char);
+                if (!*children)
+                    goto out;
+            }
+
+            if (comma)
+                strcat(*children, ",");
+            strcat(*children, cxl->name);
+            comma = _gf_true;
+        }
+
+        xl_child = xl_child->next;
+    }
+    ret = 0;
 out:
-        return decommissioned;
+    return ret;
 }
 
-gf_boolean_t
-_xl_has_decommissioned_clients (xlator_t *xl, glusterd_volinfo_t *volinfo)
+static int
+volgen_graph_build_readdir_ahead(volgen_graph_t *graph,
+                                 glusterd_volinfo_t *volinfo,
+                                 size_t child_count)
 {
-        xlator_list_t   *xl_child = NULL;
-        gf_boolean_t    decommissioned = _gf_false;
-        xlator_t        *cxl = NULL;
-
-        if (!xl)
-                goto out;
+    int32_t clusters = 0;
 
-        if (!strcmp (xl->type, "protocol/client")) {
-                decommissioned = _xl_is_client_decommissioned (xl, volinfo);
-                goto out;
-        }
+    if (graph->type == GF_QUOTAD || graph->type == GF_SNAPD ||
+        !glusterd_volinfo_get_boolean(volinfo, VKEY_PARALLEL_READDIR))
+        goto out;
 
-        xl_child = xl->children;
-        while (xl_child) {
-                cxl = xl_child->xlator;
-                /* this can go into 2 depths if the volume type
-                   is stripe-replicate */
-                decommissioned = _xl_has_decommissioned_clients (cxl, volinfo);
-                if (decommissioned)
-                        break;
+    clusters = volgen_link_bricks_from_list_tail(
+        graph, volinfo, "performance/readdir-ahead", "%s-readdir-ahead-%d",
+        child_count, 1);
 
-                xl_child = xl_child->next;
-        }
 out:
-        return decommissioned;
+    return clusters;
 }
 
 static int
-_graph_get_decommissioned_children (xlator_t *dht, glusterd_volinfo_t *volinfo,
-                                    char **children)
-{
-        int             ret = -1;
-        xlator_list_t   *xl_child = NULL;
-        xlator_t        *cxl = NULL;
-        gf_boolean_t    comma = _gf_false;
-
-        *children = NULL;
-        xl_child = dht->children;
-        while (xl_child) {
-                cxl = xl_child->xlator;
-                if (_xl_has_decommissioned_clients (cxl, volinfo)) {
-                        if (!*children) {
-                                *children = GF_CALLOC (16 * GF_UNIT_KB, 1,
-                                                       gf_common_mt_char);
-                                if (!*children)
-                                        goto out;
-                        }
-
-                        if (comma)
-                                strcat (*children, ",");
-                        strcat (*children, cxl->name);
-                        comma = _gf_true;
-                }
+volgen_graph_build_dht_cluster(volgen_graph_t *graph,
+                               glusterd_volinfo_t *volinfo, size_t child_count,
+                               gf_boolean_t is_quotad)
+{
+    int32_t clusters = 0;
+    int ret = -1;
+    char *decommissioned_children = NULL;
+    xlator_t *dht = NULL;
+    char *voltype = "cluster/distribute";
+    char *name_fmt = NULL;
+
+    /* NUFA and Switch section */
+    if (dict_get_str_boolean(volinfo->dict, "cluster.nufa", 0) &&
+        dict_get_str_boolean(volinfo->dict, "cluster.switch", 0)) {
+        gf_msg(THIS->name, GF_LOG_ERROR, errno, GD_MSG_DICT_GET_FAILED,
+               "nufa and switch cannot be set together");
+        ret = -1;
+        goto out;
+    }
+
+    /* Check for NUFA volume option, and change the voltype */
+    if (dict_get_str_boolean(volinfo->dict, "cluster.nufa", 0))
+        voltype = "cluster/nufa";
+
+    /* Check for switch volume option, and change the voltype */
+    if (dict_get_str_boolean(volinfo->dict, "cluster.switch", 0))
+        voltype = "cluster/switch";
+
+    if (is_quotad)
+        name_fmt = "%s";
+    else
+        name_fmt = "%s-dht";
+
+    clusters = volgen_link_bricks_from_list_tail(
+        graph, volinfo, voltype, name_fmt, child_count, child_count);
+    if (clusters < 0)
+        goto out;
+
+    dht = first_of(graph);
+    ret = _graph_get_decommissioned_children(dht, volinfo,
+                                             &decommissioned_children);
+    if (ret)
+        goto out;
+    if (decommissioned_children) {
+        ret = xlator_set_fixed_option(dht, "decommissioned-bricks",
+                                      decommissioned_children);
+        if (ret)
+            goto out;
+    }
+    ret = 0;
+out:
+    GF_FREE(decommissioned_children);
+    return ret;
+}
 
-                xl_child = xl_child->next;
+static int
+volgen_graph_build_ec_clusters(volgen_graph_t *graph,
+                               glusterd_volinfo_t *volinfo)
+{
+    int i = 0;
+    int ret = 0;
+    int clusters = 0;
+    char *disperse_args[] = {"cluster/disperse", "%s-disperse-%d"};
+    xlator_t *ec = NULL;
+    char option[32] = {0};
+    int start_count = 0;
+
+    clusters = volgen_link_bricks_from_list_tail_start(
+        graph, volinfo, disperse_args[0], disperse_args[1],
+        volinfo->brick_count, volinfo->disperse_count, start_count);
+    if (clusters < 0)
+        goto out;
+
+    sprintf(option, "%d", volinfo->redundancy_count);
+    ec = first_of(graph);
+    for (i = 0; i < clusters; i++) {
+        ret = xlator_set_fixed_option(ec, "redundancy", option);
+        if (ret) {
+            clusters = -1;
+            goto out;
         }
-        ret = 0;
+
+        ec = ec->next;
+    }
 out:
-        return ret;
+    return clusters;
 }
 
 static int
-volgen_graph_build_dht_cluster (volgen_graph_t *graph,
-                                glusterd_volinfo_t *volinfo, size_t child_count,
-                                gf_boolean_t is_quotad)
-{
-        int32_t                 clusters                 = 0;
-        int                     ret                      = -1;
-        char                    *decommissioned_children = NULL;
-        xlator_t                *dht                     = NULL;
-        char                    *voltype                 = "cluster/distribute";
-        char                    *name_fmt                = NULL;
-
-        /* NUFA and Switch section */
-        if (dict_get_str_boolean (volinfo->dict, "cluster.nufa", 0) &&
-            dict_get_str_boolean (volinfo->dict, "cluster.switch", 0)) {
-                gf_log (THIS->name, GF_LOG_ERROR,
-                        "nufa and switch cannot be set together");
-                ret = -1;
+set_afr_pending_xattrs_option(volgen_graph_t *graph,
+                              glusterd_volinfo_t *volinfo, int clusters)
+{
+    xlator_t *xlator = NULL;
+    xlator_t **afr_xlators_list = NULL;
+    xlator_t *this = NULL;
+    glusterd_conf_t *conf = NULL;
+    glusterd_brickinfo_t *brick = NULL;
+    glusterd_brickinfo_t *ta_brick = NULL;
+    char *ptr = NULL;
+    int i = 0;
+    int index = -1;
+    int ret = 0;
+    char *afr_xattrs_list = NULL;
+    int list_size = -1;
+    int ta_brick_index = 0;
+    int subvol_index = 0;
+
+    this = THIS;
+    GF_VALIDATE_OR_GOTO("glusterd", this, out);
+    conf = this->private;
+    GF_VALIDATE_OR_GOTO(this->name, conf, out);
+
+    if (conf->op_version < GD_OP_VERSION_3_9_0)
+        return ret;
+
+    /* (brick_id x rep.count) + (rep.count-1 commas) + NULL*/
+    list_size = (1024 * volinfo->replica_count) + (volinfo->replica_count - 1) +
+                1;
+    afr_xattrs_list = GF_CALLOC(1, list_size, gf_common_mt_char);
+    if (!afr_xattrs_list)
+        goto out;
+
+    ptr = afr_xattrs_list;
+    afr_xlators_list = GF_CALLOC(clusters, sizeof(xlator_t *),
+                                 gf_common_mt_xlator_t);
+    if (!afr_xlators_list)
+        goto out;
+
+    xlator = first_of(graph);
+
+    for (i = 0, index = clusters - 1; i < clusters; i++) {
+        afr_xlators_list[index--] = xlator;
+        xlator = xlator->next;
+    }
+
+    i = 1;
+    index = 0;
+
+    cds_list_for_each_entry(brick, &volinfo->bricks, brick_list)
+    {
+        if (index == clusters)
+            break;
+        strncat(ptr, brick->brick_id, strlen(brick->brick_id));
+        if (i == volinfo->replica_count) {
+            /* add ta client xlator in afr-pending-xattrs before making entries
+             * for client xlators in volfile.
+             * ta client xlator indexes are - 2, 5, 8 depending on the index of
+             * subvol. e.g- For first subvol ta client xlator id is volname-ta-2
+             * For pending-xattr, ta name would be
+             * 'volname-ta-2.{{volume-uuid}}' from GD_OP_VERSION_7_3.
+             */
+            ta_brick_index = 0;
+            if (volinfo->thin_arbiter_count == 1) {
+                ptr[strlen(brick->brick_id)] = ',';
+                cds_list_for_each_entry(ta_brick, &volinfo->ta_bricks,
+                                        brick_list)
+                {
+                    if (ta_brick_index == subvol_index) {
+                        break;
+                    }
+                    ta_brick_index++;
+                }
+                if (conf->op_version < GD_OP_VERSION_7_3) {
+                    strncat(ptr, ta_brick->brick_id,
+                            strlen(ta_brick->brick_id));
+                } else {
+                    char ta_volname[PATH_MAX] = "";
+                    int len = snprintf(ta_volname, PATH_MAX, "%s.%s",
+                                       ta_brick->brick_id,
+                                       uuid_utoa(volinfo->volume_id));
+                    strncat(ptr, ta_volname, len);
+                }
+            }
+
+            ret = xlator_set_fixed_option(afr_xlators_list[index++],
+                                          "afr-pending-xattr", afr_xattrs_list);
+            if (ret)
                 goto out;
+            memset(afr_xattrs_list, 0, list_size);
+            ptr = afr_xattrs_list;
+            i = 1;
+            subvol_index++;
+            continue;
         }
+        ptr[strlen(brick->brick_id)] = ',';
+        ptr += strlen(brick->brick_id) + 1;
+        i++;
+    }
 
-        /* Check for NUFA volume option, and change the voltype */
-        if (dict_get_str_boolean (volinfo->dict, "cluster.nufa", 0))
-                voltype = "cluster/nufa";
+out:
+    GF_FREE(afr_xattrs_list);
+    GF_FREE(afr_xlators_list);
+    return ret;
+}
 
-        /* Check for switch volume option, and change the voltype */
-        if (dict_get_str_boolean (volinfo->dict, "cluster.switch", 0))
-                voltype = "cluster/switch";
+static int
+set_volfile_id_option(volgen_graph_t *graph, glusterd_volinfo_t *volinfo,
+                      int clusters)
+{
+    xlator_t *xlator = NULL;
+    int i = 0;
+    int ret = -1;
+    glusterd_conf_t *conf = NULL;
+    xlator_t *this = NULL;
 
-        if (is_quotad)
-                name_fmt = "%s";
-        else
-                name_fmt = "%s-dht";
-
-        clusters = volgen_link_bricks_from_list_tail (graph,  volinfo,
-                                                voltype,
-                                                name_fmt,
-                                                child_count,
-                                                child_count);
-        if (clusters < 0)
-                goto out;
+    this = THIS;
+    GF_VALIDATE_OR_GOTO("glusterd", this, out);
+    conf = this->private;
+    GF_VALIDATE_OR_GOTO(this->name, conf, out);
+
+    if (conf->op_version < GD_OP_VERSION_9_0)
+        return 0;
+    xlator = first_of(graph);
 
-        dht = first_of (graph);
-        ret = _graph_get_decommissioned_children (dht, volinfo,
-                                                  &decommissioned_children);
+    for (i = 0; i < clusters; i++) {
+        ret = xlator_set_fixed_option(xlator, "volume-id",
+                                      uuid_utoa(volinfo->volume_id));
         if (ret)
-                goto out;
-        if (decommissioned_children) {
-                ret = xlator_set_option (dht, "decommissioned-bricks",
-                                         decommissioned_children);
-                if (ret)
-                        goto out;
-        }
-        ret = 0;
+            goto out;
+
+        xlator = xlator->next;
+    }
+
 out:
-        GF_FREE (decommissioned_children);
-        return ret;
+    return ret;
 }
 
 static int
-volgen_graph_build_ec_clusters (volgen_graph_t *graph,
+volgen_graph_build_afr_clusters(volgen_graph_t *graph,
                                 glusterd_volinfo_t *volinfo)
 {
-        int                     i = 0;
-        int                     ret = 0;
-        int                     clusters            = 0;
-        char                    *disperse_args[]    = {"cluster/disperse",
-                                                       "%s-disperse-%d"};
-        xlator_t                *ec                 = NULL;
-        char                    option[32]          = {0};
-
-        clusters = volgen_link_bricks_from_list_tail (graph, volinfo,
-                                                disperse_args[0],
-                                                disperse_args[1],
-                                                volinfo->brick_count,
-                                                volinfo->disperse_count);
-        if (clusters < 0)
+    int i = 0;
+    int ret = 0;
+    int clusters = 0;
+    char *replicate_type = "cluster/replicate";
+    char *replicate_name = "%s-replicate-%d";
+    xlator_t *afr = NULL;
+    char option[32] = {0};
+    glusterd_brickinfo_t *ta_brick = NULL;
+    int ta_brick_index = 0;
+    int ta_replica_offset = 0;
+    int ta_brick_offset = 0;
+    char ta_option[4096] = {
+        0,
+    };
+
+    /* In thin-arbiter case brick count and replica count remain same
+     * but due to additional entries of ta client xlators in the volfile,
+     * GD1 is manipulated to include these client xlators while linking them to
+     * afr/cluster entry in the volfile.
+     */
+    if (volinfo->thin_arbiter_count == 1) {
+        ta_replica_offset = 1;
+        ta_brick_offset = volinfo->subvol_count;
+    }
+
+    clusters = volgen_link_bricks_from_list_tail(
+        graph, volinfo, replicate_type, replicate_name,
+        volinfo->brick_count + ta_brick_offset,
+        volinfo->replica_count + ta_replica_offset);
+
+    if (clusters < 0)
+        goto out;
+
+    ret = set_afr_pending_xattrs_option(graph, volinfo, clusters);
+    if (ret) {
+        clusters = -1;
+        goto out;
+    }
+
+    ret = set_volfile_id_option(graph, volinfo, clusters);
+    if (ret) {
+        clusters = -1;
+        goto out;
+    }
+
+    if (!volinfo->arbiter_count && !volinfo->thin_arbiter_count)
+        goto out;
+
+    afr = first_of(graph);
+
+    if (volinfo->arbiter_count) {
+        sprintf(option, "%d", volinfo->arbiter_count);
+        for (i = 0; i < clusters; i++) {
+            ret = xlator_set_fixed_option(afr, "arbiter-count", option);
+            if (ret) {
+                clusters = -1;
                 goto out;
+            }
+
+            afr = afr->next;
+        }
+    }
 
-        sprintf(option, "%d", volinfo->redundancy_count);
-        ec = first_of (graph);
+    if (volinfo->thin_arbiter_count == 1) {
         for (i = 0; i < clusters; i++) {
-                ret = xlator_set_option (ec, "redundancy", option);
-                if (ret) {
-                        clusters = -1;
-                        goto out;
+            ta_brick_index = 0;
+            cds_list_for_each_entry(ta_brick, &volinfo->ta_bricks, brick_list)
+            {
+                if (ta_brick_index == i) {
+                    break;
                 }
-
-                ec = ec->next;
-        }
+                ta_brick_index++;
+            }
+            snprintf(ta_option, sizeof(ta_option), "%s:%s", ta_brick->hostname,
+                     ta_brick->path);
+            ret = xlator_set_fixed_option(afr, "thin-arbiter", ta_option);
+            if (ret) {
+                clusters = -1;
+                goto out;
+            }
+            afr = afr->next;
+        }
+    }
 out:
-        return clusters;
+    return clusters;
 }
 
 static int
-volume_volgen_graph_build_clusters (volgen_graph_t *graph,
-                                    glusterd_volinfo_t *volinfo,
-                                    gf_boolean_t is_quotad)
-{
-        char                    *replicate_args[]   = {"cluster/replicate",
-                                                       "%s-replicate-%d"};
-        char                    *tier_args[]        = {"cluster/tier",
-                                                       "%s-tier-%d"};
-        char                    *stripe_args[]      = {"cluster/stripe",
-                                                       "%s-stripe-%d"};
-        char                    *disperse_args[]    = {"cluster/disperse",
-                                                       "%s-disperse-%d"};
-        char                    option[32]          = "";
-        int                     rclusters           = 0;
-        int                     clusters            = 0;
-        int                     dist_count          = 0;
-        int                     ret                 = -1;
-        xlator_t               *ec                  = NULL;
-        xlator_t               *client              = NULL;
-
-        if (!volinfo->dist_leaf_count)
-                goto out;
-
-        if (volinfo->dist_leaf_count == 1)
-                goto build_distribute;
+volume_volgen_graph_build_clusters(volgen_graph_t *graph,
+                                   glusterd_volinfo_t *volinfo,
+                                   gf_boolean_t is_quotad)
+{
+    int clusters = 0;
+    int dist_count = 0;
+    int ret = -1;
 
-        /* All other cases, it will have one or the other cluster type */
-        switch (volinfo->type) {
-        case GF_CLUSTER_TYPE_REPLICATE:
-                clusters = volgen_link_bricks_from_list_tail
-                        (graph, volinfo,
-                         replicate_args[0],
-                         replicate_args[1],
-                         volinfo->brick_count,
-                         volinfo->replica_count);
-
-                if (clusters < 0)
-                        goto out;
-                break;
-        case GF_CLUSTER_TYPE_STRIPE:
-                clusters = volgen_link_bricks_from_list_tail (graph, volinfo,
-                                                        stripe_args[0],
-                                                        stripe_args[1],
-                                                        volinfo->brick_count,
-                                                        volinfo->stripe_count);
-                if (clusters < 0)
-                        goto out;
-                break;
-        case GF_CLUSTER_TYPE_TIER:
-                ret = volgen_link_bricks_from_list_head (graph, volinfo,
-                                                         tier_args[0],
-                                                         tier_args[1],
-                                                         volinfo->brick_count,
-                                                         volinfo->replica_count);
-                break;
-        case GF_CLUSTER_TYPE_STRIPE_REPLICATE:
-                /* Replicate after the clients, then stripe */
-                if (volinfo->replica_count == 0)
-                        goto out;
-                clusters = volgen_link_bricks_from_list_tail (graph, volinfo,
-                                                        replicate_args[0],
-                                                        replicate_args[1],
-                                                        volinfo->brick_count,
-                                                        volinfo->replica_count);
-                if (clusters < 0)
-                        goto out;
+    if (!volinfo->dist_leaf_count)
+        goto out;
 
-                rclusters = volinfo->brick_count / volinfo->replica_count;
-                GF_ASSERT (rclusters == clusters);
-                clusters = volgen_link_bricks_from_list_tail (graph, volinfo,
-                                                        stripe_args[0],
-                                                        stripe_args[1],
-                                                        rclusters,
-                                                        volinfo->stripe_count);
-                if (clusters < 0)
-                        goto out;
-                break;
+    if (volinfo->dist_leaf_count == 1)
+        goto build_distribute;
 
+    /* All other cases, it will have one or the other cluster type */
+    switch (volinfo->type) {
+        case GF_CLUSTER_TYPE_REPLICATE:
+            clusters = volgen_graph_build_afr_clusters(graph, volinfo);
+            if (clusters < 0)
+                goto out;
+            break;
         case GF_CLUSTER_TYPE_DISPERSE:
-                clusters = volgen_graph_build_ec_clusters (graph, volinfo);
-                if (clusters < 0)
-                        goto out;
+            clusters = volgen_graph_build_ec_clusters(graph, volinfo);
+            if (clusters < 0)
+                goto out;
 
-                break;
+            break;
         default:
-                gf_log ("", GF_LOG_ERROR, "volume inconsistency: "
-                        "unrecognized clustering type");
-                goto out;
-        }
+            gf_msg("glusterd", GF_LOG_ERROR, 0, GD_MSG_VOLUME_INCONSISTENCY,
+                   "volume inconsistency: "
+                   "unrecognized clustering type");
+            goto out;
+    }
 
 build_distribute:
-        dist_count = volinfo->brick_count / volinfo->dist_leaf_count;
-        if (!dist_count) {
-                ret = -1;
-                goto out;
-        }
+    dist_count = volinfo->brick_count / volinfo->dist_leaf_count;
+    if (!dist_count) {
+        ret = -1;
+        goto out;
+    }
+    clusters = volgen_graph_build_readdir_ahead(graph, volinfo, dist_count);
+    if (clusters < 0)
+        goto out;
 
-        ret = volgen_graph_build_dht_cluster (graph, volinfo,
-                                              dist_count, is_quotad);
-        if (ret)
-                goto out;
+    ret = volgen_graph_build_dht_cluster(graph, volinfo, dist_count, is_quotad);
+    if (ret)
+        goto out;
 
-        ret = 0;
+    ret = 0;
 out:
-        return ret;
+    return ret;
 }
 
-static int client_graph_set_perf_options(volgen_graph_t *graph,
-                                         glusterd_volinfo_t *volinfo,
-                                         dict_t *set_dict)
-{
-        data_t *tmp_data = NULL;
-        char *volname = NULL;
+static int
+client_graph_set_rda_options(volgen_graph_t *graph, glusterd_volinfo_t *volinfo,
+                             dict_t *set_dict)
+{
+    char *rda_cache_s = NULL;
+    int32_t ret = 0;
+    uint64_t rda_cache_size = 0;
+    char *rda_req_s = NULL;
+    uint64_t rda_req_size = 0;
+    uint64_t new_cache_size = 0;
+    char new_cache_size_str[50] = {
+        0,
+    };
+    char new_req_size_str[50] = {
+        0,
+    };
+    int dist_count = 0;
+
+    dist_count = volinfo->brick_count / volinfo->dist_leaf_count;
+    if (dist_count <= 1)
+        goto out;
+
+    if (graph->type == GF_QUOTAD || graph->type == GF_SNAPD ||
+        !glusterd_volinfo_get_boolean(volinfo, VKEY_PARALLEL_READDIR) ||
+        !glusterd_volinfo_get_boolean(volinfo, VKEY_READDIR_AHEAD))
+        goto out;
+
+    /* glusterd_volinfo_get() will get the default value if nothing set
+     * explicitly. Hence it is important to check set_dict before checking
+     * glusterd_volinfo_get, so that we consider key value of the in
+     * progress volume set option.
+     */
+    ret = dict_get_str_sizen(set_dict, VKEY_RDA_CACHE_LIMIT, &rda_cache_s);
+    if (ret < 0) {
+        ret = glusterd_volinfo_get(volinfo, VKEY_RDA_CACHE_LIMIT, &rda_cache_s);
+        if (ret < 0)
+            goto out;
+    }
+    ret = gf_string2bytesize_uint64(rda_cache_s, &rda_cache_size);
+    if (ret < 0) {
+        set_graph_errstr(
+            graph, "invalid number format in option " VKEY_RDA_CACHE_LIMIT);
+        goto out;
+    }
+
+    ret = dict_get_str_sizen(set_dict, VKEY_RDA_REQUEST_SIZE, &rda_req_s);
+    if (ret < 0) {
+        ret = glusterd_volinfo_get(volinfo, VKEY_RDA_REQUEST_SIZE, &rda_req_s);
+        if (ret < 0)
+            goto out;
+    }
+    ret = gf_string2bytesize_uint64(rda_req_s, &rda_req_size);
+    if (ret < 0) {
+        set_graph_errstr(
+            graph, "invalid number format in option " VKEY_RDA_REQUEST_SIZE);
+        goto out;
+    }
+
+    if (rda_cache_size == 0 || rda_req_size == 0) {
+        set_graph_errstr(graph, "Value cannot be 0");
+        ret = -1;
+        goto out;
+    }
 
-        /*
-         * Logic to make sure NFS doesn't have performance translators by
-         * default for a volume
-         */
-        volname = volinfo->volname;
-        tmp_data = dict_get (set_dict, "nfs-volume-file");
-        if (!tmp_data)
-                return volgen_graph_set_options_generic(graph, set_dict,
-                                                        volinfo,
-                                                        &perfxl_option_handler);
+    new_cache_size = rda_cache_size / dist_count;
+    if (new_cache_size < rda_req_size) {
+        if (new_cache_size < 4 * 1024)
+            new_cache_size = rda_req_size = 4 * 1024;
         else
-                return volgen_graph_set_options_generic(graph, set_dict,
-                                                        volname,
-                                                        &nfsperfxl_option_handler);
+            rda_req_size = new_cache_size;
+
+        snprintf(new_req_size_str, sizeof(new_req_size_str), "%" PRId64 "%s",
+                 rda_req_size, "B");
+        ret = dict_set_dynstr_with_alloc(set_dict, VKEY_RDA_REQUEST_SIZE,
+                                         new_req_size_str);
+        if (ret < 0)
+            goto out;
+    }
+
+    snprintf(new_cache_size_str, sizeof(new_cache_size_str), "%" PRId64 "%s",
+             new_cache_size, "B");
+    ret = dict_set_dynstr_with_alloc(set_dict, VKEY_RDA_CACHE_LIMIT,
+                                     new_cache_size_str);
+    if (ret < 0)
+        goto out;
+
+out:
+    return ret;
 }
 
 static int
-graph_set_generic_options (xlator_t *this, volgen_graph_t *graph,
-                           dict_t *set_dict, char *identifier)
-{
-        int     ret = 0;
-
-        ret = volgen_graph_set_options_generic (graph, set_dict, "client",
-                                                &loglevel_option_handler);
-
-        if (ret)
-                gf_log (this->name, GF_LOG_WARNING, "changing %s log level"
-                        " failed", identifier);
-
-        ret = volgen_graph_set_options_generic (graph, set_dict, "client",
-                                                &sys_loglevel_option_handler);
-        if (ret)
-                gf_log (this->name, GF_LOG_WARNING, "changing %s syslog "
-                        "level failed", identifier);
-
-        ret = volgen_graph_set_options_generic (graph, set_dict, "client",
-                                                &logger_option_handler);
-
-        if (ret)
-                gf_log (this->name, GF_LOG_WARNING, "changing %s logger"
-                        " failed", identifier);
-
-        ret = volgen_graph_set_options_generic (graph, set_dict, "client",
-                                                &log_format_option_handler);
-        if (ret)
-                gf_log (this->name, GF_LOG_WARNING, "changing %s log format"
-                        " failed", identifier);
+client_graph_set_perf_options(volgen_graph_t *graph,
+                              glusterd_volinfo_t *volinfo, dict_t *set_dict)
+{
+    int ret = 0;
+
+    /*
+     * Logic to make sure gfproxy-client gets custom performance translators
+     */
+    ret = dict_get_str_boolean(set_dict, "gfproxy-client", 0);
+    if (ret == 1) {
+        return volgen_graph_set_options_generic(
+            graph, set_dict, volinfo, &gfproxy_client_perfxl_option_handler);
+    }
+
+    /*
+     * Logic to make sure gfproxy-server gets custom performance translators
+     */
+    ret = dict_get_str_boolean(set_dict, "gfproxy-server", 0);
+    if (ret == 1) {
+        return volgen_graph_set_options_generic(
+            graph, set_dict, volinfo, &gfproxy_server_perfxl_option_handler);
+    }
+
+    /*
+     * Logic to make sure NFS doesn't have performance translators by
+     * default for a volume
+     */
+    ret = client_graph_set_rda_options(graph, volinfo, set_dict);
+    if (ret < 0)
+        return ret;
 
-        ret = volgen_graph_set_options_generic (graph, set_dict, "client",
-                                                &log_buf_size_option_handler);
-        if (ret)
-                gf_log (this->name, GF_LOG_WARNING, "Failed to change "
-                        "log-buf-size option");
+#ifdef BUILD_GNFS
+    data_t *tmp_data = NULL;
+    char *volname = NULL;
 
-        ret = volgen_graph_set_options_generic (graph, set_dict, "client",
-                                                &log_flush_timeout_option_handler);
-        if (ret)
-                gf_log (this->name, GF_LOG_WARNING, "Failed to change "
-                        "log-flush-timeout option");
-        return 0;
+    tmp_data = dict_get_sizen(set_dict, "nfs-volume-file");
+    if (tmp_data) {
+        volname = volinfo->volname;
+        return volgen_graph_set_options_generic(graph, set_dict, volname,
+                                                &nfsperfxl_option_handler);
+    } else
+#endif
+        return volgen_graph_set_options_generic(graph, set_dict, volinfo,
+                                                &perfxl_option_handler);
 }
 
 static int
-volume_volgen_graph_build_clusters_tier (volgen_graph_t *graph,
-                                         glusterd_volinfo_t *volinfo,
-                                         gf_boolean_t is_quotad)
-{
-        int                ret = -1;
-        xlator_t          *root;
-        xlator_t          *xl, *hxl, *cxl;
-        glusterd_brickinfo_t    *brick              = NULL;
-        char              *rule;
-        int                st_brick_count = 0;
-        int                st_replica_count = 0;
-        int                st_disperse_count = 0;
-        int                st_dist_leaf_count = 0;
-        int                st_type = 0;
-        char               st_volname[GD_VOLUME_NAME_MAX];
-        int                dist_count = 0;
-
-        st_brick_count     = volinfo->brick_count;
-        st_replica_count   = volinfo->replica_count;
-        st_disperse_count  = volinfo->disperse_count;
-        st_type            = volinfo->type;
-        st_dist_leaf_count = volinfo->dist_leaf_count;
-        strcpy(st_volname, volinfo->volname);
-
-        volinfo->dist_leaf_count = volinfo->tier_info.cold_dist_leaf_count;
-        volinfo->brick_count    = volinfo->tier_info.cold_brick_count;
-        volinfo->replica_count  = volinfo->tier_info.cold_replica_count;
-        volinfo->disperse_count = volinfo->tier_info.cold_disperse_count;
-        volinfo->type           = volinfo->tier_info.cold_type;
-        sprintf (volinfo->volname, "%s-cold", st_volname);
-
-        ret = volume_volgen_graph_build_clusters (graph, volinfo, is_quotad);
-        if (ret)
-                goto out;
-        cxl = first_of(graph);
-
-        volinfo->type           = volinfo->tier_info.hot_type;
-        volinfo->brick_count    = volinfo->tier_info.hot_brick_count;
-        volinfo->replica_count  = volinfo->tier_info.hot_replica_count;
-        volinfo->dist_leaf_count = glusterd_get_dist_leaf_count(volinfo);
-        volinfo->disperse_count = 0;
-
-        sprintf (volinfo->volname, "%s-hot", st_volname);
-
-        dist_count = volinfo->brick_count / volinfo->dist_leaf_count;
-
-        if (volinfo->dist_leaf_count != 1) {
-                ret = volgen_link_bricks_from_list_head
-                        (graph, volinfo,
-                         "cluster/replicate",
-                         "%s-replicate-%d",
-                         volinfo->brick_count,
-                         volinfo->replica_count);
-                if (ret != -1)
-                        volgen_link_bricks_from_list_tail (graph,  volinfo,
-                                                           "cluster/distribute",
-                                                           "%s-dht",
-                                                           dist_count,
-                                                           dist_count);
-        } else {
-                ret = volgen_link_bricks_from_list_head (graph,  volinfo,
-                                                 "cluster/distribute",
-                                                 "%s-dht",
-                                                 dist_count,
-                                                 dist_count);
-        }
-        if (ret == -1)
-                goto out;
-
-        hxl = first_of(graph);
-
-        volinfo->type           = GF_CLUSTER_TYPE_TIER;
-        xl = volgen_graph_add_nolink (graph, "cluster/tier", "%s",
-                                      "tier-dht", 0);
-        gf_asprintf(&rule, "%s-hot-dht", st_volname);
-        xlator_set_option(xl, "rule", rule);
-        xlator_set_option(xl, "xattr-name", "trusted.tier-gfid");
-
-        ret = volgen_xlator_link (xl, cxl);
-        ret = volgen_xlator_link (xl, hxl);
-
-        st_type = GF_CLUSTER_TYPE_TIER;
-
- out:
-        volinfo->brick_count     = st_brick_count;
-        volinfo->replica_count   = st_replica_count;
-        volinfo->disperse_count  = st_disperse_count;
-        volinfo->type            = st_type;
-        volinfo->dist_leaf_count = st_dist_leaf_count;
-        strcpy(volinfo->volname, st_volname);
-
-        return ret;
+graph_set_generic_options(xlator_t *this, volgen_graph_t *graph,
+                          dict_t *set_dict, char *identifier)
+{
+    int ret = 0;
+
+    ret = volgen_graph_set_options_generic(graph, set_dict, "client",
+                                           &loglevel_option_handler);
+
+    if (ret)
+        gf_msg(this->name, GF_LOG_WARNING, 0, GD_MSG_GRAPH_SET_OPT_FAIL,
+               "changing %s log level"
+               " failed",
+               identifier);
+
+    ret = volgen_graph_set_options_generic(graph, set_dict, "client",
+                                           &sys_loglevel_option_handler);
+    if (ret)
+        gf_msg(this->name, GF_LOG_WARNING, 0, GD_MSG_GRAPH_SET_OPT_FAIL,
+               "changing %s syslog "
+               "level failed",
+               identifier);
+
+    ret = volgen_graph_set_options_generic(graph, set_dict, "client",
+                                           &logger_option_handler);
+
+    if (ret)
+        gf_msg(this->name, GF_LOG_WARNING, 0, GD_MSG_GRAPH_SET_OPT_FAIL,
+               "changing %s logger"
+               " failed",
+               identifier);
+
+    ret = volgen_graph_set_options_generic(graph, set_dict, "client",
+                                           &log_format_option_handler);
+    if (ret)
+        gf_msg(this->name, GF_LOG_WARNING, 0, GD_MSG_GRAPH_SET_OPT_FAIL,
+               "changing %s log format"
+               " failed",
+               identifier);
+
+    ret = volgen_graph_set_options_generic(graph, set_dict, "client",
+                                           &log_buf_size_option_handler);
+    if (ret)
+        gf_msg(this->name, GF_LOG_WARNING, 0, GD_MSG_GRAPH_SET_OPT_FAIL,
+               "Failed to change "
+               "log-buf-size option");
+
+    ret = volgen_graph_set_options_generic(graph, set_dict, "client",
+                                           &log_flush_timeout_option_handler);
+    if (ret)
+        gf_msg(this->name, GF_LOG_WARNING, 0, GD_MSG_GRAPH_SET_OPT_FAIL,
+               "Failed to change "
+               "log-flush-timeout option");
+
+    ret = volgen_graph_set_options_generic(
+        graph, set_dict, "client", &log_localtime_logging_option_handler);
+    if (ret)
+        gf_msg(this->name, GF_LOG_WARNING, 0, GD_MSG_GRAPH_SET_OPT_FAIL,
+               "Failed to change "
+               "log-localtime-logging option");
+
+    ret = volgen_graph_set_options_generic(graph, set_dict, "client",
+                                           &threads_option_handler);
+
+    if (ret)
+        gf_msg(this->name, GF_LOG_WARNING, 0, GD_MSG_GRAPH_SET_OPT_FAIL,
+               "changing %s threads failed", identifier);
+
+    return 0;
 }
 
 static int
-client_graph_builder (volgen_graph_t *graph, glusterd_volinfo_t *volinfo,
-                      dict_t *set_dict, void *param)
-{
-        int              ret           = 0;
-        xlator_t        *xl            = NULL;
-        char            *volname       = NULL;
-        glusterd_conf_t *conf          = THIS->private;
-        char            *tmp           = NULL;
-        gf_boolean_t     var           = _gf_false;
-        gf_boolean_t     ob            = _gf_false;
-        gf_boolean_t     uss_enabled   = _gf_false;
-        gf_boolean_t     rebal_volfile = _gf_false;
-        xlator_t        *this          = THIS;
-
-        GF_ASSERT (this);
-        GF_ASSERT (conf);
-
-        volname = volinfo->volname;
-        ret = volgen_graph_build_clients (graph, volinfo, set_dict,
-                                          param);
+client_graph_builder(volgen_graph_t *graph, glusterd_volinfo_t *volinfo,
+                     dict_t *set_dict, void *param)
+{
+    int ret = 0;
+    xlator_t *xl = NULL;
+    char *volname = NULL;
+    glusterd_conf_t *conf = THIS->private;
+    char *tmp = NULL;
+    gf_boolean_t var = _gf_false;
+    gf_boolean_t ob = _gf_false;
+    int uss_enabled = -1;
+    xlator_t *this = THIS;
+    char *subvol = NULL;
+    size_t namelen = 0;
+    char *xl_id = NULL;
+    gf_boolean_t gfproxy_clnt = _gf_false;
+
+    GF_ASSERT(this);
+    GF_ASSERT(conf);
+
+    ret = dict_get_str_boolean(set_dict, "gfproxy-client", 0);
+    if (ret == -1)
+        goto out;
+
+    volname = volinfo->volname;
+    if (ret == 0) {
+        ret = volgen_graph_build_clients(graph, volinfo, set_dict, param);
         if (ret)
-                goto out;
+            goto out;
 
-        if (volinfo->type == GF_CLUSTER_TYPE_TIER)
-                ret = volume_volgen_graph_build_clusters_tier (graph, volinfo, _gf_false);
         else
-                ret = volume_volgen_graph_build_clusters (graph, volinfo, _gf_false);
+            ret = volume_volgen_graph_build_clusters(graph, volinfo, _gf_false);
 
         if (ret == -1)
-                goto out;
-
-        ret = dict_get_str_boolean (set_dict, "features.shard", _gf_false);
-        if (ret == -1)
-                goto out;
-
-        if (ret) {
-                xl = volgen_graph_add (graph, "features/shard", volname);
-                if (!xl) {
-                        ret = -1;
-                        goto out;
-                }
+            goto out;
+    } else {
+        gfproxy_clnt = _gf_true;
+        namelen = strlen(volinfo->volname) + SLEN("gfproxyd-") + 1;
+        subvol = alloca(namelen);
+        snprintf(subvol, namelen, "gfproxyd-%s", volinfo->volname);
+
+        namelen = strlen(volinfo->volname) + SLEN("-gfproxy-client") + 1;
+        xl_id = alloca(namelen);
+        snprintf(xl_id, namelen, "%s-gfproxy-client", volinfo->volname);
+        volgen_graph_build_client(graph, volinfo, NULL, NULL, subvol, xl_id,
+                                  "tcp", set_dict);
+    }
+
+    ret = dict_get_str_boolean(set_dict, "features.cloudsync", _gf_false);
+    if (ret == -1)
+        goto out;
+
+    if (ret) {
+        xl = volgen_graph_add(graph, "features/cloudsync", volname);
+        if (!xl) {
+            ret = -1;
+            goto out;
         }
+    }
 
-        /* As of now snapshot volume is read-only. Read-only xlator is loaded
-         * in client graph so that AFR & DHT healing can be done in server.
-         */
-        if (volinfo->is_snap_volume) {
-                xl = volgen_graph_add (graph, "features/read-only", volname);
-                if (!xl) {
-                        gf_log (this->name, GF_LOG_ERROR, "Failed to add "
-                                "read-only feature to the graph of %s "
-                                "snapshot with %s origin volume",
-                                volname, volinfo->parent_volname);
-                        ret = -1;
-                        goto out;
-                }
-                ret = xlator_set_option (xl, "read-only", "on");
-                if (ret)
-                        goto out;
-
-        }
+    ret = dict_get_str_boolean(set_dict, "features.shard", _gf_false);
+    if (ret == -1)
+        goto out;
 
-        /* Check for compress volume option, and add it to the graph on client side */
-        ret = dict_get_str_boolean (set_dict, "network.compression", 0);
-        if (ret == -1)
-                goto out;
-        if (ret) {
-                xl = volgen_graph_add (graph, "features/cdc", volname);
-                if (!xl) {
-                        ret = -1;
-                        goto out;
-                }
-                ret = xlator_set_option (xl, "mode", "client");
-                if (ret)
-                        goto out;
+    if (ret) {
+        xl = volgen_graph_add(graph, "features/shard", volname);
+        if (!xl) {
+            ret = -1;
+            goto out;
+        }
+    }
+    /* a. ret will be -1 if features.ctime is not set in the volinfo->dict which
+     * means ctime should be loaded into the graph.
+     * b. ret will be 1 if features.ctime is explicitly turned on through
+     * volume set and in that case ctime should be loaded into the graph.
+     * c. ret will be 0 if features.ctime is explicitly turned off and in that
+     * case ctime shouldn't be loaded into the graph.
+     */
+    ret = dict_get_str_boolean(set_dict, "features.ctime", -1);
+    if (conf->op_version >= GD_OP_VERSION_5_0 && ret) {
+        xl = volgen_graph_add(graph, "features/utime", volname);
+        if (!xl) {
+            ret = -1;
+            goto out;
         }
+    }
 
-        ret = dict_get_str_boolean (set_dict, "features.encryption", _gf_false);
-        if (ret == -1)
-                goto out;
-        if (ret) {
-                xl = volgen_graph_add (graph, "encryption/crypt", volname);
-                if (!xl) {
-                        ret = -1;
-                        goto out;
-                }
+    /* As of now snapshot volume is read-only. Read-only xlator is loaded
+     * in client graph so that AFR & DHT healing can be done in server.
+     */
+    if (volinfo->is_snap_volume) {
+        xl = volgen_graph_add(graph, "features/read-only", volname);
+        if (!xl) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_GRAPH_FEATURE_ADD_FAIL,
+                   "Failed to add "
+                   "read-only feature to the graph of %s "
+                   "snapshot with %s origin volume",
+                   volname, volinfo->parent_volname);
+            ret = -1;
+            goto out;
+        }
+        ret = xlator_set_fixed_option(xl, "read-only", "on");
+        if (ret)
+            goto out;
+    }
+
+    /* Check for compress volume option, and add it to the graph on client side
+     */
+    ret = dict_get_str_boolean(set_dict, "network.compression", 0);
+    if (ret == -1)
+        goto out;
+    if (ret) {
+        xl = volgen_graph_add(graph, "features/cdc", volname);
+        if (!xl) {
+            ret = -1;
+            goto out;
         }
+        ret = xlator_set_fixed_option(xl, "mode", "client");
+        if (ret)
+            goto out;
+    }
 
-        if (conf->op_version == GD_OP_VERSION_MIN) {
-                ret = glusterd_volinfo_get_boolean (volinfo,
-                                                    VKEY_FEATURES_QUOTA);
-                if (ret == -1)
-                        goto out;
-                if (ret) {
-                        xl = volgen_graph_add (graph, "features/quota",
-                                              volname);
-                        if (!xl) {
-                                ret = -1;
-                                goto out;
-                        }
-                }
+    /* gfproxy needs the quiesce translator */
+    if (gfproxy_clnt) {
+        xl = volgen_graph_add(graph, "features/quiesce", volname);
+        if (!xl) {
+            ret = -1;
+            goto out;
         }
+    }
 
-
-        ret = dict_get_str_boolean (set_dict, "features.file-snapshot",
-                                    _gf_false);
+    if (conf->op_version == GD_OP_VERSION_MIN) {
+        ret = glusterd_volinfo_get_boolean(volinfo, VKEY_FEATURES_QUOTA);
         if (ret == -1)
-                goto out;
+            goto out;
         if (ret) {
-                xl = volgen_graph_add (graph, "features/qemu-block", volname);
-
-                if (!xl) {
-                        ret = -1;
-                        goto out;
-                }
-        }
-
-        /* Do not allow changing read-after-open option if root-squash is
-           enabled.
-        */
-        ret = dict_get_str (set_dict, "performance.read-after-open", &tmp);
-        if (!ret) {
-                ret = dict_get_str (volinfo->dict, "server.root-squash", &tmp);
-                if (!ret) {
-                        ob = _gf_false;
-                        ret = gf_string2boolean (tmp, &ob);
-                        if (!ret && ob) {
-                                gf_log (this->name, GF_LOG_WARNING,
-                                        "root-squash is enabled. Please turn it"
-                                        " off to change read-after-open "
-                                        "option");
-                                ret = -1;
-                                goto out;
-                        }
-                }
+            xl = volgen_graph_add(graph, "features/quota", volname);
+            if (!xl) {
+                ret = -1;
+                goto out;
+            }
         }
+    }
 
-        /* open behind causes problems when root-squash is enabled
-           (by allowing reads to happen even though the squashed user
-           does not have permissions to do so) as it fakes open to be
-           successful and later sends reads on anonymous fds. So when
-           root-squash is enabled, open-behind's option to read after
-           open is done is also enabled.
-        */
-        ret = dict_get_str (set_dict, "server.root-squash", &tmp);
+    /* Do not allow changing read-after-open option if root-squash is
+       enabled.
+    */
+    ret = dict_get_str_sizen(set_dict, "performance.read-after-open", &tmp);
+    if (!ret) {
+        ret = dict_get_str_sizen(volinfo->dict, "server.root-squash", &tmp);
         if (!ret) {
-                ret = gf_string2boolean (tmp, &var);
-                if (ret)
-                        goto out;
-
-                if (var) {
-                        ret = dict_get_str (volinfo->dict,
-                                            "performance.read-after-open",
-                                            &tmp);
-                        if (!ret) {
-                                ret = gf_string2boolean (tmp, &ob);
-                                /* go ahead with turning read-after-open on
-                                   even if string2boolean conversion fails,
-                                   OR if read-after-open option is turned off
-                                */
-                                if (ret || !ob)
-                                        ret = dict_set_str (set_dict,
-                                                 "performance.read-after-open",
-                                                            "yes");
-                        } else {
-                                ret = dict_set_str (set_dict,
-                                                    "performance.read-after-open",
-                                                    "yes");
-                        }
-                } else {
-                        /* When root-squash has to be turned off, open-behind's
-                           read-after-open option should be reset to what was
-                           there before root-squash was turned on. If the option
-                           cannot be found in volinfo's dict, it means that
-                           option was not set before turning on root-squash.
-                        */
-                        ob = _gf_false;
-                        ret = dict_get_str (volinfo->dict,
-                                            "performance.read-after-open",
-                                            &tmp);
-                        if (!ret) {
-                                ret = gf_string2boolean (tmp, &ob);
-
-                                if (!ret && ob) {
-                                        ret = dict_set_str (set_dict,
-                                                 "performance.read-after-open",
-                                                            "yes");
-                                }
-                        }
-                        /* consider operation is failure only if read-after-open
-                           option is enabled and could not set into set_dict
-                        */
-                        if (!ob)
-                                ret = 0;
-                }
-                if (ret) {
-                        gf_log (this->name, GF_LOG_WARNING, "setting "
-                                "open behind option as part of root "
-                                "squash failed");
-                        goto out;
-                }
-        }
-
-        ret = dict_get_str_boolean (set_dict, "server.manage-gids", _gf_false);
-        if (ret != -1) {
-               ret = dict_set_str (set_dict, "client.send-gids",
-                                   ret ? "false" : "true");
-               if (ret)
-                       gf_log (THIS->name, GF_LOG_WARNING, "changing client"
-                               " protocol option failed");
+            ob = _gf_false;
+            ret = gf_string2boolean(tmp, &ob);
+            if (!ret && ob) {
+                gf_msg(this->name, GF_LOG_WARNING, 0,
+                       GD_MSG_ROOT_SQUASH_ENABLED,
+                       "root-squash is enabled. Please turn it"
+                       " off to change read-after-open "
+                       "option");
+                ret = -1;
+                goto out;
+            }
         }
+    }
 
-        ret = client_graph_set_perf_options(graph, volinfo, set_dict);
+    /* open behind causes problems when root-squash is enabled
+       (by allowing reads to happen even though the squashed user
+       does not have permissions to do so) as it fakes open to be
+       successful and later sends reads on anonymous fds. So when
+       root-squash is enabled, open-behind's option to read after
+       open is done is also enabled.
+    */
+    ret = dict_get_str_sizen(set_dict, "server.root-squash", &tmp);
+    if (!ret) {
+        ret = gf_string2boolean(tmp, &var);
         if (ret)
-                goto out;
-
-        uss_enabled = dict_get_str_boolean (set_dict, "features.uss",
-                                            _gf_false);
-        if (uss_enabled == -1)
-                goto out;
-        if (uss_enabled && !volinfo->is_snap_volume) {
-                rebal_volfile = dict_get_str_boolean (set_dict,
-                                                   "rebalance-volfile-creation",
-                                                   _gf_false);
-                if (rebal_volfile == -1)
-                        goto out;
-
-                if (!rebal_volfile) {
-                        ret = volgen_graph_build_snapview_client
-                                                   (graph, volinfo,
-                                                    volname, set_dict);
-                        if (ret == -1)
-                                goto out;
+            goto out;
+
+        if (var) {
+            ret = dict_get_str_sizen(volinfo->dict,
+                                     "performance.read-after-open", &tmp);
+            if (!ret) {
+                ret = gf_string2boolean(tmp, &ob);
+                /* go ahead with turning read-after-open on
+                   even if string2boolean conversion fails,
+                   OR if read-after-open option is turned off
+                */
+                if (ret || !ob)
+                    ret = dict_set_sizen_str_sizen(
+                        set_dict, "performance.read-after-open", "yes");
+            } else {
+                ret = dict_set_sizen_str_sizen(
+                    set_dict, "performance.read-after-open", "yes");
+            }
+        } else {
+            /* When root-squash has to be turned off, open-behind's
+               read-after-open option should be reset to what was
+               there before root-squash was turned on. If the option
+               cannot be found in volinfo's dict, it means that
+               option was not set before turning on root-squash.
+            */
+            ob = _gf_false;
+            ret = dict_get_str_sizen(volinfo->dict,
+                                     "performance.read-after-open", &tmp);
+            if (!ret) {
+                ret = gf_string2boolean(tmp, &ob);
+
+                if (!ret && ob) {
+                    ret = dict_set_sizen_str_sizen(
+                        set_dict, "performance.read-after-open", "yes");
                 }
+            }
+            /* consider operation is failure only if read-after-open
+               option is enabled and could not set into set_dict
+            */
+            if (!ob)
+                ret = 0;
         }
-
-        ret = dict_get_str_boolean (set_dict, "ganesha.enable", _gf_false);
-
-        if (ret == -1) {
-                gf_log (this->name, GF_LOG_WARNING, "setting ganesha.enable"
-                        "option failed.");
-                goto out;
-        }
-
         if (ret) {
-                xl = volgen_graph_add (graph, "features/ganesha", volname);
-
-                if (!xl) {
-                        gf_log (this->name, GF_LOG_ERROR, "failed to add"
-                               "add features/ganesha to graph");
-                        ret = -1;
-                        goto out;
-                }
-        }
-
-        /* add debug translators depending on the options */
-        ret = check_and_add_debug_xl (graph, set_dict, volname,
-                                      "client");
+            gf_msg(this->name, GF_LOG_WARNING, 0, GD_MSG_ROOT_SQUASH_FAILED,
+                   "setting "
+                   "open behind option as part of root "
+                   "squash failed");
+            goto out;
+        }
+    }
+
+    ret = dict_get_str_boolean(set_dict, "server.manage-gids", _gf_false);
+    if (ret != -1) {
+        ret = dict_set_str_sizen(set_dict, "client.send-gids",
+                                 ret ? "false" : "true");
         if (ret)
-                return -1;
+            gf_msg(THIS->name, GF_LOG_WARNING, errno, GD_MSG_DICT_SET_FAILED,
+                   "changing client"
+                   " protocol option failed");
+    }
+
+    ret = client_graph_set_perf_options(graph, volinfo, set_dict);
+    if (ret)
+        goto out;
+
+    uss_enabled = dict_get_str_boolean(set_dict, "features.uss", _gf_false);
+    if (uss_enabled == -1)
+        goto out;
+    if (uss_enabled && !volinfo->is_snap_volume) {
+        ret = volgen_graph_build_snapview_client(graph, volinfo, volname,
+                                                 set_dict);
+        if (ret == -1)
+            goto out;
+    }
 
-        ret = -1;
-        xl = volgen_graph_add_as (graph, "debug/io-stats", volname);
-        if (!xl)
-                goto out;
+    /* add debug translators depending on the options */
+    ret = check_and_add_debug_xl(graph, set_dict, volname, "client");
+    if (ret)
+        return -1;
 
-        ret = graph_set_generic_options (this, graph, set_dict, "client");
+    /* if the client is part of 'gfproxyd' server, then we need to keep the
+       volume name as 'gfproxyd-<volname>', for better portmapper options */
+    subvol = volname;
+    ret = dict_get_str_boolean(set_dict, "gfproxy-server", 0);
+    if (ret > 0) {
+        namelen = strlen(volinfo->volname) + SLEN("gfproxyd-") + 1;
+        subvol = alloca(namelen);
+        snprintf(subvol, namelen, "gfproxyd-%s", volname);
+    }
+
+    ret = -1;
+    xl = volgen_graph_add_as(graph, "debug/io-stats", subvol);
+    if (!xl) {
+        goto out;
+    }
+
+    ret = graph_set_generic_options(this, graph, set_dict, "client");
 out:
-        return ret;
+    return ret;
 }
 
-
 /* builds a graph for client role , with option overrides in mod_dict */
 static int
-build_client_graph (volgen_graph_t *graph, glusterd_volinfo_t *volinfo,
-                    dict_t *mod_dict)
+build_client_graph(volgen_graph_t *graph, glusterd_volinfo_t *volinfo,
+                   dict_t *mod_dict)
 {
-        return build_graph_generic (graph, volinfo, mod_dict, NULL,
-                                    &client_graph_builder);
+    return build_graph_generic(graph, volinfo, mod_dict, NULL,
+                               &client_graph_builder);
 }
 
-char *gd_shd_options[] = {
-        "!self-heal-daemon",
-        "!heal-timeout",
-        NULL
-};
+char *gd_shd_options[] = {"!self-heal-daemon", "!heal-timeout", NULL};
 
-char*
-gd_get_matching_option (char **options, char *option)
+char *
+gd_get_matching_option(char **options, char *option)
 {
-        while (*options && strcmp (*options, option))
-                options++;
-        return *options;
+    while (*options && strcmp(*options, option))
+        options++;
+    return *options;
 }
 
 static int
-shd_option_handler (volgen_graph_t *graph, struct volopt_map_entry *vme,
-                    void *param)
+bitrot_option_handler(volgen_graph_t *graph, struct volopt_map_entry *vme,
+                      void *param)
 {
-        int                     ret = 0;
-        struct volopt_map_entry new_vme = {0};
-        char                    *shd_option = NULL;
+    xlator_t *xl = NULL;
+    int ret = 0;
 
-        shd_option = gd_get_matching_option (gd_shd_options, vme->option);
-        if ((vme->option[0] == '!') && !shd_option)
-                goto out;
-        new_vme = *vme;
-        if (shd_option) {
-                new_vme.option = shd_option + 1;//option with out '!'
-        }
+    xl = first_of(graph);
 
-        ret = no_filter_option_handler (graph, &new_vme, param);
-out:
-        return ret;
+    if (!strcmp(vme->option, "expiry-time")) {
+        ret = xlator_set_fixed_option(xl, "expiry-time", vme->value);
+        if (ret)
+            return -1;
+    }
+
+    if (!strcmp(vme->option, "signer-threads")) {
+        ret = xlator_set_fixed_option(xl, "signer-threads", vme->value);
+        if (ret)
+            return -1;
+    }
+
+    return ret;
 }
 
 static int
-nfs_option_handler (volgen_graph_t *graph,
-                    struct volopt_map_entry *vme, void *param)
+scrubber_option_handler(volgen_graph_t *graph, struct volopt_map_entry *vme,
+                        void *param)
 {
-        xlator_t *xl = NULL;
-        char *aa = NULL;
-        int   ret   = 0;
-        glusterd_volinfo_t *volinfo = NULL;
-
-        volinfo = param;
-
-        xl = first_of (graph);
-
-/*        if (vme->type  == GLOBAL_DOC || vme->type == GLOBAL_NO_DOC) {
-
-                ret = xlator_set_option (xl, vme->key, vme->value);
-        }*/
-        if (!volinfo || (volinfo->volname[0] == '\0'))
-                return 0;
-
-        if (! strcmp (vme->option, "!rpc-auth.addr.*.allow")) {
-                ret = gf_asprintf (&aa, "rpc-auth.addr.%s.allow",
-                                        volinfo->volname);
-
-                if (ret != -1) {
-                        ret = xlator_set_option (xl, aa, vme->value);
-                        GF_FREE (aa);
-                }
-
-                if (ret)
-                        return -1;
-        }
-
-        if (! strcmp (vme->option, "!rpc-auth.addr.*.reject")) {
-                ret = gf_asprintf (&aa, "rpc-auth.addr.%s.reject",
-                                        volinfo->volname);
-
-                if (ret != -1) {
-                        ret = xlator_set_option (xl, aa, vme->value);
-                        GF_FREE (aa);
-                }
-
-                if (ret)
-                        return -1;
-        }
-
-        if (! strcmp (vme->option, "!rpc-auth.auth-unix.*")) {
-                ret = gf_asprintf (&aa, "rpc-auth.auth-unix.%s",
-                                        volinfo->volname);
-
-                if (ret != -1) {
-                        ret = xlator_set_option (xl, aa, vme->value);
-                        GF_FREE (aa);
-                }
-
-                if (ret)
-                        return -1;
-        }
-        if (! strcmp (vme->option, "!rpc-auth.auth-null.*")) {
-                ret = gf_asprintf (&aa, "rpc-auth.auth-null.%s",
-                                        volinfo->volname);
+    xlator_t *xl = NULL;
+    int ret = 0;
 
-                if (ret != -1) {
-                        ret = xlator_set_option (xl, aa, vme->value);
-                        GF_FREE (aa);
-                }
+    xl = first_of(graph);
 
-                if (ret)
-                        return -1;
-        }
-
-        if (! strcmp (vme->option, "!nfs3.*.trusted-sync")) {
-                ret = gf_asprintf (&aa, "nfs3.%s.trusted-sync",
-                                        volinfo->volname);
+    if (!strcmp(vme->option, "scrub-throttle")) {
+        ret = xlator_set_fixed_option(xl, "scrub-throttle", vme->value);
+        if (ret)
+            return -1;
+    }
 
-                if (ret != -1) {
-                        ret = xlator_set_option (xl, aa, vme->value);
-                        GF_FREE (aa);
-                }
+    if (!strcmp(vme->option, "scrub-frequency")) {
+        ret = xlator_set_fixed_option(xl, "scrub-freq", vme->value);
+        if (ret)
+            return -1;
+    }
 
-                if (ret)
-                        return -1;
+    if (!strcmp(vme->option, "scrubber")) {
+        if (!strcmp(vme->value, "pause")) {
+            ret = xlator_set_fixed_option(xl, "scrub-state", vme->value);
+            if (ret)
+                return -1;
         }
+    }
 
-        if (! strcmp (vme->option, "!nfs3.*.trusted-write")) {
-                ret = gf_asprintf (&aa, "nfs3.%s.trusted-write",
-                                        volinfo->volname);
+    return ret;
+}
 
-                if (ret != -1) {
-                        ret = xlator_set_option (xl, aa, vme->value);
-                        GF_FREE (aa);
-                }
+static int
+shd_option_handler(volgen_graph_t *graph, struct volopt_map_entry *vme,
+                   void *param)
+{
+    int ret = 0;
+    struct volopt_map_entry new_vme = {0};
+    char *shd_option = NULL;
+
+    shd_option = gd_get_matching_option(gd_shd_options, vme->option);
+    if ((vme->option[0] == '!') && !shd_option)
+        goto out;
+    new_vme = *vme;
+    if (shd_option) {
+        new_vme.option = shd_option + 1;  // option with out '!'
+    }
+
+    ret = no_filter_option_handler(graph, &new_vme, param);
+out:
+    return ret;
+}
 
-                if (ret)
-                        return -1;
-        }
+#ifdef BUILD_GNFS
+static int
+nfs_option_handler(volgen_graph_t *graph, struct volopt_map_entry *vme,
+                   void *param)
+{
+    static struct nfs_opt nfs_opts[] = {
+        /* {pattern, printf_pattern} */
+        {"!rpc-auth.addr.*.allow", "rpc-auth.addr.%s.allow"},
+        {"!rpc-auth.addr.*.reject", "rpc-auth.addr.%s.reject"},
+        {"!rpc-auth.auth-unix.*", "rpc-auth.auth-unix.%s"},
+        {"!rpc-auth.auth-null.*", "rpc-auth.auth-null.%s"},
+        {"!nfs3.*.trusted-sync", "nfs3.%s.trusted-sync"},
+        {"!nfs3.*.trusted-write", "nfs3.%s.trusted-write"},
+        {"!nfs3.*.volume-access", "nfs3.%s.volume-access"},
+        {"!rpc-auth.ports.*.insecure", "rpc-auth.ports.%s.insecure"},
+        {"!nfs-disable", "nfs.%s.disable"},
+        {NULL, NULL}};
+    xlator_t *xl = NULL;
+    char *aa = NULL;
+    int ret = 0;
+    glusterd_volinfo_t *volinfo = NULL;
+    int keylen;
+    struct nfs_opt *opt = NULL;
+
+    volinfo = param;
+
+    if (!volinfo || (volinfo->volname[0] == '\0')) {
+        gf_smsg("glusterd", GF_LOG_ERROR, errno, GD_MSG_INVALID_ARGUMENT, NULL);
+        return 0;
+    }
 
-        if (! strcmp (vme->option, "!nfs3.*.volume-access")) {
-                ret = gf_asprintf (&aa, "nfs3.%s.volume-access",
-                                        volinfo->volname);
+    if (!vme || !(vme->option)) {
+        gf_smsg("glusterd", GF_LOG_ERROR, errno, GD_MSG_INVALID_ARGUMENT, NULL);
+        return 0;
+    }
 
-                if (ret != -1) {
-                        ret = xlator_set_option (xl, aa, vme->value);
-                        GF_FREE (aa);
-                }
+    xl = first_of(graph);
 
-                if (ret)
-                        return -1;
-        }
+    for (opt = nfs_opts; opt->pattern; opt++) {
+        if (!strcmp(vme->option, opt->pattern)) {
+            keylen = gf_asprintf(&aa, opt->printf_pattern, volinfo->volname);
 
-        if (! strcmp (vme->option, "!nfs3.*.export-dir")) {
-                ret = gf_asprintf (&aa, "nfs3.%s.export-dir",
-                                        volinfo->volname);
+            if (keylen == -1) {
+                return -1;
+            }
 
-                if (ret != -1) {
-                        ret = gf_canonicalize_path (vme->value);
-                        if (ret)
-                                return -1;
+            ret = xlator_set_option(xl, aa, keylen, vme->value);
+            GF_FREE(aa);
 
-                        ret = xlator_set_option (xl, aa, vme->value);
-                        GF_FREE (aa);
-                }
+            if (ret)
+                return -1;
 
-                if (ret)
-                        return -1;
+            goto out;
         }
+    }
 
+    if (!strcmp(vme->option, "!nfs3.*.export-dir")) {
+        keylen = gf_asprintf(&aa, "nfs3.%s.export-dir", volinfo->volname);
 
-
-        if (! strcmp (vme->option, "!rpc-auth.ports.*.insecure")) {
-                ret = gf_asprintf (&aa, "rpc-auth.ports.%s.insecure",
-                                        volinfo->volname);
-
-                if (ret != -1) {
-                        ret = xlator_set_option (xl, aa, vme->value);
-                        GF_FREE (aa);
-                }
-
-                if (ret)
-                        return -1;
+        if (keylen == -1) {
+            return -1;
         }
 
-
-        if (! strcmp (vme->option, "!nfs-disable")) {
-                ret = gf_asprintf (&aa, "nfs.%s.disable",
-                                        volinfo->volname);
-
-                if (ret != -1) {
-                        ret = xlator_set_option (xl, aa, vme->value);
-                        GF_FREE (aa);
-                }
-
-                if (ret)
-                        return -1;
+        ret = gf_canonicalize_path(vme->value);
+        if (ret) {
+            GF_FREE(aa);
+            return -1;
         }
+        ret = xlator_set_option(xl, aa, keylen, vme->value);
+        GF_FREE(aa);
 
-        if ((strcmp (vme->voltype, "nfs/server") == 0) &&
-             (vme->option && vme->option[0]!='!') ) {
-               ret = xlator_set_option (xl, vme->option, vme->value);
-                if (ret)
-                        return -1;
-        }
+        if (ret)
+            return -1;
+    } else if ((strcmp(vme->voltype, "nfs/server") == 0) &&
+               (vme->option[0] != '!')) {
+        ret = xlator_set_option(xl, vme->option, strlen(vme->option),
+                                vme->value);
+        if (ret)
+            return -1;
+    }
 
+out:
+    return 0;
+}
 
-        /*key = strchr (vme->key, '.') + 1;
+#endif
+char *
+volgen_get_shd_key(int type)
+{
+    char *key = NULL;
 
-        for (trav = xl->children; trav; trav = trav->next) {
-                ret = gf_asprintf (&aa, "auth.addr.%s.%s", trav->xlator->name,
-                                   key);
-                if (ret != -1) {
-                        ret = xlator_set_option (xl, aa, vme->value);
-                        GF_FREE (aa);
-                }
-                if (ret)
-                        return -1;
-        }*/
+    switch (type) {
+        case GF_CLUSTER_TYPE_REPLICATE:
+            key = "cluster.self-heal-daemon";
+            break;
+        case GF_CLUSTER_TYPE_DISPERSE:
+            key = "cluster.disperse-self-heal-daemon";
+            break;
+        default:
+            key = NULL;
+            break;
+    }
 
-        return 0;
+    return key;
 }
 
-char*
-volgen_get_shd_key (glusterd_volinfo_t *volinfo)
+static int
+volgen_set_shd_key_enable(dict_t *set_dict, const int type)
 {
-        char    *key = NULL;
+    int ret = 0;
 
-        switch (volinfo->type) {
+    switch (type) {
         case GF_CLUSTER_TYPE_REPLICATE:
-        case GF_CLUSTER_TYPE_STRIPE_REPLICATE:
-                key = "cluster.self-heal-daemon";
-                break;
+            ret = dict_set_sizen_str_sizen(set_dict, "cluster.self-heal-daemon",
+                                           "enable");
+            break;
         case GF_CLUSTER_TYPE_DISPERSE:
-                key = "cluster.disperse-self-heal-daemon";
-                break;
+            ret = dict_set_sizen_str_sizen(
+                set_dict, "cluster.disperse-self-heal-daemon", "enable");
+            break;
         default:
-                key = NULL;
-                break;
-        }
+            break;
+    }
 
-        return key;
+    return ret;
 }
 
 static gf_boolean_t
-volgen_is_shd_compatible_xl (char *xl_type)
+volgen_is_shd_compatible_xl(char *xl_type)
 {
-        char            *shd_xls[] = {"cluster/replicate", "cluster/disperse",
-                                      NULL};
-        if (gf_get_index_by_elem (shd_xls, xl_type) != -1)
-                return _gf_true;
+    char *shd_xls[] = {"cluster/replicate", "cluster/disperse", NULL};
+    if (gf_get_index_by_elem(shd_xls, xl_type) != -1)
+        return _gf_true;
 
-        return _gf_false;
+    return _gf_false;
 }
 
 static int
-volgen_graph_set_iam_shd (volgen_graph_t *graph)
+volgen_graph_set_iam_shd(volgen_graph_t *graph)
 {
-        xlator_t        *trav;
-        int             ret = 0;
+    xlator_t *trav;
+    int ret = 0;
 
-        for (trav = first_of (graph); trav; trav = trav->next) {
-                if (!volgen_is_shd_compatible_xl (trav->type))
-                        continue;
+    for (trav = first_of(graph); trav; trav = trav->next) {
+        if (!volgen_is_shd_compatible_xl(trav->type))
+            continue;
 
-                ret = xlator_set_option (trav, "iam-self-heal-daemon", "yes");
-                if (ret)
-                        break;
-        }
-        return ret;
+        ret = xlator_set_fixed_option(trav, "iam-self-heal-daemon", "yes");
+        if (ret)
+            break;
+    }
+    return ret;
 }
 
 static int
-prepare_shd_volume_options (glusterd_volinfo_t *volinfo,
-                            dict_t *mod_dict, dict_t *set_dict)
+prepare_shd_volume_options(glusterd_volinfo_t *volinfo, dict_t *mod_dict,
+                           dict_t *set_dict)
 {
-        char    *key = NULL;
-        int     ret = 0;
+    int ret = 0;
 
-        key = volgen_get_shd_key (volinfo);
-        if (!key) {
-                ret = -1;
-                goto out;
-        }
-
-        ret = dict_set_str (set_dict, key, "enable");
-        if (ret)
-                goto out;
+    ret = volgen_set_shd_key_enable(set_dict, volinfo->type);
+    if (ret)
+        goto out;
 
-        ret = dict_set_uint32 (set_dict, "trusted-client", GF_CLIENT_TRUSTED);
-        if (ret)
-                goto out;
+    ret = dict_set_uint32(set_dict, "trusted-client", GF_CLIENT_TRUSTED);
+    if (ret) {
+        gf_smsg("glusterd", GF_LOG_ERROR, errno, GD_MSG_DICT_SET_FAILED,
+                "Key=trusted-client", NULL);
+        goto out;
+    }
 
-        dict_copy (volinfo->dict, set_dict);
-        if (mod_dict)
-                dict_copy (mod_dict, set_dict);
+    dict_copy(volinfo->dict, set_dict);
+    if (mod_dict)
+        dict_copy(mod_dict, set_dict);
 out:
-        return ret;
+    return ret;
 }
 
 static int
-volgen_graph_build_replicate_clusters (volgen_graph_t *graph,
-                                       glusterd_volinfo_t *volinfo)
+build_afr_ec_clusters(volgen_graph_t *graph, glusterd_volinfo_t *volinfo)
 {
-        char               *replicate_args[]   = {"cluster/replicate",
-                                                  "%s-replicate-%d"};
+    int clusters = -1;
+    switch (volinfo->type) {
+        case GF_CLUSTER_TYPE_REPLICATE:
+            clusters = volgen_graph_build_afr_clusters(graph, volinfo);
+            break;
 
-        return volgen_link_bricks_from_list_tail (graph, volinfo, "cluster/replicate",
-                                            "%s-replicate-%d",
-                                            volinfo->brick_count,
-                                            volinfo->replica_count);
+        case GF_CLUSTER_TYPE_DISPERSE:
+            clusters = volgen_graph_build_ec_clusters(graph, volinfo);
+            break;
+    }
+    return clusters;
 }
 
 static int
-build_shd_clusters (volgen_graph_t *graph, glusterd_volinfo_t *volinfo,
-                    dict_t *set_dict)
+build_shd_clusters(volgen_graph_t *graph, glusterd_volinfo_t *volinfo,
+                   dict_t *set_dict)
 {
-        int     ret = 0;
-        int     clusters = -1;
+    int ret = 0;
+    int clusters = -1;
 
-        ret = volgen_graph_build_clients (graph, volinfo, set_dict, NULL);
-        if (ret)
-                goto out;
+    ret = volgen_graph_build_clients(graph, volinfo, set_dict, NULL);
+    if (ret)
+        goto out;
+    clusters = build_afr_ec_clusters(graph, volinfo);
 
-        switch (volinfo->type) {
-        case GF_CLUSTER_TYPE_REPLICATE:
-        case GF_CLUSTER_TYPE_STRIPE_REPLICATE:
-                clusters = volgen_graph_build_replicate_clusters (graph,
-                                                                  volinfo);
-                break;
+out:
+    return clusters;
+}
+
+gf_boolean_t
+gd_is_self_heal_enabled(glusterd_volinfo_t *volinfo, dict_t *dict)
+{
+    char *shd_key = NULL;
+    gf_boolean_t shd_enabled = _gf_false;
+
+    GF_VALIDATE_OR_GOTO("glusterd", volinfo, out);
 
+    switch (volinfo->type) {
+        case GF_CLUSTER_TYPE_REPLICATE:
         case GF_CLUSTER_TYPE_DISPERSE:
-                clusters = volgen_graph_build_ec_clusters (graph, volinfo);
-                break;
-        }
+            shd_key = volgen_get_shd_key(volinfo->type);
+            shd_enabled = dict_get_str_boolean(dict, shd_key, _gf_true);
+            break;
+        default:
+            break;
+    }
 out:
-        return clusters;
+    return shd_enabled;
 }
 
-static int
-build_shd_volume_graph (xlator_t *this, volgen_graph_t *graph,
-                        glusterd_volinfo_t *volinfo,
-                        dict_t *mod_dict, dict_t *set_dict,
-                        gf_boolean_t graph_check, gf_boolean_t *valid_config)
+int
+build_rebalance_volfile(glusterd_volinfo_t *volinfo, char *filepath,
+                        dict_t *mod_dict)
 {
-        volgen_graph_t     cgraph         = {0};
-        int     ret = 0;
-        int     clusters = -1;
+    volgen_graph_t graph = {
+        0,
+    };
+    xlator_t *xl = NULL;
+    int ret = -1;
+    xlator_t *this = NULL;
+    dict_t *set_dict = NULL;
 
-        if (!graph_check && (volinfo->status != GLUSTERD_STATUS_STARTED))
-                goto out;
+    this = THIS;
 
-        if (!glusterd_is_shd_compatible_volume (volinfo))
-                goto out;
+    graph.type = GF_REBALANCED;
 
-        /* Shd graph is valid only when there is at least one
-         * replica/disperse volume is present
+    if (volinfo->brick_count <= volinfo->dist_leaf_count) {
+        /*
+         * Volume is not a distribute volume or
+         * contains only 1 brick, no need to create
+         * the volfiles.
          */
-        *valid_config = _gf_true;
+        return 0;
+    }
 
-        ret = prepare_shd_volume_options (volinfo, mod_dict, set_dict);
-        if (ret)
-                goto out;
+    set_dict = dict_copy_with_ref(volinfo->dict, NULL);
+    if (!set_dict)
+        return -1;
 
-        clusters = build_shd_clusters (&cgraph, volinfo, set_dict);
-        if (clusters < 0) {
-                ret = -1;
-                goto out;
-        }
+    if (mod_dict) {
+        dict_copy(mod_dict, set_dict);
+        /* XXX dict_copy swallows errors */
+    }
 
-        ret = volgen_graph_set_options_generic (&cgraph, set_dict,
-                                                volinfo, shd_option_handler);
-        if (ret)
-                goto out;
+    /* Rebalance is always a trusted client*/
+    ret = dict_set_uint32(set_dict, "trusted-client", GF_CLIENT_TRUSTED);
+    if (ret)
+        return -1;
 
-        ret = volgen_graph_set_iam_shd (&cgraph);
-        if (ret)
-                goto out;
+    ret = volgen_graph_build_clients(&graph, volinfo, set_dict, NULL);
+    if (ret)
+        goto out;
 
-        ret = volgen_graph_merge_sub (graph, &cgraph, clusters);
-        if (ret)
-                goto out;
+    ret = volume_volgen_graph_build_clusters(&graph, volinfo, _gf_false);
+    if (ret)
+        goto out;
+
+    xl = volgen_graph_add_as(&graph, "debug/io-stats", volinfo->volname);
+    if (!xl) {
+        ret = -1;
+        goto out;
+    }
+
+    ret = graph_set_generic_options(this, &graph, set_dict, "rebalance-daemon");
+    if (ret)
+        goto out;
+
+    ret = volgen_graph_set_options_generic(&graph, set_dict, volinfo,
+                                           basic_option_handler);
+
+    if (!ret)
+        ret = volgen_write_volfile(&graph, filepath);
 
-        ret = graph_set_generic_options (this, graph, set_dict,
-                                         "self-heal daemon");
 out:
-        return ret;
+    volgen_graph_free(&graph);
+
+    dict_unref(set_dict);
+
+    return ret;
 }
 
-int
-build_shd_graph (volgen_graph_t *graph, dict_t *mod_dict)
-{
-        glusterd_volinfo_t *voliter       = NULL;
-        xlator_t           *this          = NULL;
-        glusterd_conf_t    *priv          = NULL;
-        dict_t             *set_dict      = NULL;
-        int                ret            = 0;
-        gf_boolean_t       valid_config   = _gf_false;
-        xlator_t           *iostxl        = NULL;
-        int                clusters      = 0;
-        gf_boolean_t       graph_check    = _gf_false;
-
-        this = THIS;
-        priv = this->private;
-
-        set_dict = dict_new ();
-        if (!set_dict) {
-                ret = -ENOMEM;
-                goto out;
-        }
+static int
+build_shd_volume_graph(xlator_t *this, volgen_graph_t *graph,
+                       glusterd_volinfo_t *volinfo, dict_t *mod_dict,
+                       dict_t *set_dict, gf_boolean_t graph_check)
+{
+    volgen_graph_t cgraph = {0};
+    int ret = 0;
+    int clusters = -1;
 
-        graph_check = dict_get_str_boolean (mod_dict, "graph-check", 0);
-        iostxl = volgen_graph_add_as (graph, "debug/io-stats", "glustershd");
-        if (!iostxl) {
-                ret = -1;
-                goto out;
-        }
+    if (!glusterd_is_shd_compatible_volume(volinfo))
+        goto out;
 
-        cds_list_for_each_entry (voliter, &priv->volumes, vol_list) {
-                ret = build_shd_volume_graph (this, graph, voliter, mod_dict,
-                                              set_dict, graph_check,
-                                              &valid_config);
+    ret = prepare_shd_volume_options(volinfo, mod_dict, set_dict);
+    if (ret)
+        goto out;
 
-                ret = dict_reset (set_dict);
-                if (ret)
-                        goto out;
-        }
+    clusters = build_shd_clusters(&cgraph, volinfo, set_dict);
+    if (clusters < 0) {
+        ret = -1;
+        goto out;
+    }
+
+    ret = volgen_graph_set_options_generic(&cgraph, set_dict, volinfo,
+                                           shd_option_handler);
+    if (ret)
+        goto out;
+
+    ret = volgen_graph_set_iam_shd(&cgraph);
+    if (ret)
+        goto out;
+
+    ret = volgen_graph_merge_sub(graph, &cgraph, clusters);
+    if (ret)
+        goto out;
+
+    ret = graph_set_generic_options(this, graph, set_dict, "self-heal daemon");
 out:
-        if (set_dict)
-                dict_unref (set_dict);
-        if (!valid_config)
-                ret = -EINVAL;
-        return ret;
+    return ret;
 }
 
-/* builds a graph for nfs server role, with option overrides in mod_dict */
 int
-build_nfs_graph (volgen_graph_t *graph, dict_t *mod_dict)
-{
-        volgen_graph_t      cgraph        = {0,};
-        glusterd_volinfo_t *voliter       = NULL;
-        xlator_t           *this          = NULL;
-        glusterd_conf_t    *priv          = NULL;
-        dict_t             *set_dict      = NULL;
-        xlator_t           *nfsxl         = NULL;
-        char               *skey          = NULL;
-        int                 ret           = 0;
-        char               nfs_xprt[16]   = {0,};
-        char               *volname       = NULL;
-        data_t             *data          = NULL;
-
-        this = THIS;
-        GF_ASSERT (this);
-        priv = this->private;
-        GF_ASSERT (priv);
-
-        set_dict = dict_new ();
-        if (!set_dict) {
-                gf_log ("", GF_LOG_ERROR, "Out of memory");
-                return -1;
-        }
+build_shd_graph(glusterd_volinfo_t *volinfo, volgen_graph_t *graph,
+                dict_t *mod_dict)
+{
+    xlator_t *this = NULL;
+    dict_t *set_dict = NULL;
+    int ret = 0;
+    xlator_t *iostxl = NULL;
+    gf_boolean_t graph_check = _gf_false;
+
+    this = THIS;
+
+    set_dict = dict_new();
+    if (!set_dict) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_CREATE_FAIL, NULL);
+        ret = -ENOMEM;
+        goto out;
+    }
+
+    if (mod_dict)
+        graph_check = dict_get_str_boolean(mod_dict, "graph-check", 0);
+    iostxl = volgen_graph_add_as(graph, "debug/io-stats", volinfo->volname);
+    if (!iostxl) {
+        ret = -1;
+        goto out;
+    }
 
-        nfsxl = volgen_graph_add_as (graph, "nfs/server", "nfs-server");
-        if (!nfsxl) {
-                ret = -1;
-                goto out;
-        }
-        ret = xlator_set_option (nfsxl, "nfs.dynamic-volumes", "on");
-        if (ret)
-                goto out;
+    ret = build_shd_volume_graph(this, graph, volinfo, mod_dict, set_dict,
+                                 graph_check);
 
-        ret = xlator_set_option (nfsxl, "nfs.nlm", "on");
-        if (ret)
-                goto out;
+out:
+    if (set_dict)
+        dict_unref(set_dict);
+    return ret;
+}
 
-        ret = xlator_set_option (nfsxl, "nfs.drc", "off");
+#ifdef BUILD_GNFS
+
+static int
+volgen_graph_set_iam_nfsd(const volgen_graph_t *graph)
+{
+    xlator_t *trav;
+    int ret = 0;
+
+    for (trav = first_of((volgen_graph_t *)graph); trav; trav = trav->next) {
+        if (strcmp(trav->type, "cluster/replicate") != 0)
+            continue;
+
+        ret = xlator_set_fixed_option(trav, "iam-nfs-daemon", "yes");
         if (ret)
-                goto out;
+            break;
+    }
+    return ret;
+}
 
-        cds_list_for_each_entry (voliter, &priv->volumes, vol_list) {
-                if (voliter->status != GLUSTERD_STATUS_STARTED)
-                        continue;
+/* builds a graph for nfs server role, with option overrides in mod_dict */
+int
+build_nfs_graph(volgen_graph_t *graph, dict_t *mod_dict)
+{
+    volgen_graph_t cgraph = {
+        0,
+    };
+    glusterd_volinfo_t *voliter = NULL;
+    xlator_t *this = NULL;
+    glusterd_conf_t *priv = NULL;
+    dict_t *set_dict = NULL;
+    xlator_t *nfsxl = NULL;
+    char *skey = NULL;
+    int ret = 0;
+    char nfs_xprt[16] = {
+        0,
+    };
+    char *volname = NULL;
+    data_t *data = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+    priv = this->private;
+    GF_ASSERT(priv);
+
+    set_dict = dict_new();
+    if (!set_dict) {
+        gf_msg("glusterd", GF_LOG_ERROR, ENOMEM, GD_MSG_NO_MEMORY,
+               "Out of memory");
+        return -1;
+    }
 
-                if (dict_get_str_boolean (voliter->dict, "nfs.disable", 0))
-                        continue;
+    nfsxl = volgen_graph_add_as(graph, "nfs/server", "nfs-server");
+    if (!nfsxl) {
+        ret = -1;
+        goto out;
+    }
+    ret = xlator_set_fixed_option(nfsxl, "nfs.dynamic-volumes", "on");
+    if (ret)
+        goto out;
 
-                ret = gf_asprintf (&skey, "rpc-auth.addr.%s.allow",
-                                   voliter->volname);
-                if (ret == -1) {
-                        gf_log ("", GF_LOG_ERROR, "Out of memory");
-                        goto out;
-                }
-                ret = xlator_set_option (nfsxl, skey, "*");
-                GF_FREE (skey);
-                if (ret)
-                        goto out;
+    ret = xlator_set_fixed_option(nfsxl, "nfs.nlm", "on");
+    if (ret)
+        goto out;
 
-                ret = gf_asprintf (&skey, "nfs3.%s.volume-id",
-                                   voliter->volname);
-                if (ret == -1) {
-                        gf_log ("", GF_LOG_ERROR, "Out of memory");
-                        goto out;
-                }
-                ret = xlator_set_option (nfsxl, skey, uuid_utoa (voliter->volume_id));
-                GF_FREE (skey);
-                if (ret)
-                        goto out;
+    ret = xlator_set_fixed_option(nfsxl, "nfs.drc", "off");
+    if (ret)
+        goto out;
 
-                /* If both RDMA and TCP are the transport_type, use TCP for NFS
-                 * client protocols, because tcp,rdma volume can be created in
-                 * servers which does not have rdma supported hardware
-                 * The transport type specified here is client transport type
-                 * which is used for communication between gluster-nfs and brick
-                 * processes.
-                 * User can specify client transport for tcp,rdma volume using
-                 * nfs.transport-type, if it is not set by user default
-                 * one will be tcp.
-                 */
-                memset (&cgraph, 0, sizeof (cgraph));
-                if (mod_dict)
-                        get_transport_type (voliter, mod_dict, nfs_xprt, _gf_true);
-                else
-                        get_transport_type (voliter, voliter->dict, nfs_xprt, _gf_true);
-
-                ret = dict_set_str (set_dict, "performance.stat-prefetch", "off");
-                if (ret)
-                        goto out;
+    cds_list_for_each_entry(voliter, &priv->volumes, vol_list)
+    {
+        if (voliter->status != GLUSTERD_STATUS_STARTED)
+            continue;
 
-                ret = dict_set_str (set_dict, "performance.client-io-threads",
-                                    "off");
-                if (ret)
-                        goto out;
+        if (dict_get_str_boolean(voliter->dict, NFS_DISABLE_MAP_KEY, 0))
+            continue;
 
-                ret = dict_set_str (set_dict, "client-transport-type",
-                                    nfs_xprt);
-                if (ret)
-                        goto out;
+        ret = gf_asprintf(&skey, "rpc-auth.addr.%s.allow", voliter->volname);
+        if (ret == -1) {
+            gf_msg("glusterd", GF_LOG_ERROR, ENOMEM, GD_MSG_NO_MEMORY,
+                   "Out of memory");
+            goto out;
+        }
+        ret = xlator_set_option(nfsxl, skey, ret, "*");
+        GF_FREE(skey);
+        if (ret)
+            goto out;
 
-                ret = dict_set_uint32 (set_dict, "trusted-client",
-                                       GF_CLIENT_TRUSTED);
-                if (ret)
-                        goto out;
+        ret = gf_asprintf(&skey, "nfs3.%s.volume-id", voliter->volname);
+        if (ret == -1) {
+            gf_msg("glusterd", GF_LOG_ERROR, 0, GD_MSG_NO_MEMORY,
+                   "Out of memory");
+            goto out;
+        }
+        ret = xlator_set_option(nfsxl, skey, ret,
+                                uuid_utoa(voliter->volume_id));
+        GF_FREE(skey);
+        if (ret)
+            goto out;
+
+        /* If both RDMA and TCP are the transport_type, use TCP for NFS
+         * client protocols, because tcp,rdma volume can be created in
+         * servers which does not have rdma supported hardware
+         * The transport type specified here is client transport type
+         * which is used for communication between gluster-nfs and brick
+         * processes.
+         * User can specify client transport for tcp,rdma volume using
+         * nfs.transport-type, if it is not set by user default
+         * one will be tcp.
+         */
+        memset(&cgraph, 0, sizeof(cgraph));
+        if (mod_dict)
+            get_transport_type(voliter, mod_dict, nfs_xprt, _gf_true);
+        else
+            get_transport_type(voliter, voliter->dict, nfs_xprt, _gf_true);
 
-                ret = dict_set_str (set_dict, "nfs-volume-file", "yes");
-                if (ret)
-                        goto out;
+        ret = dict_set_sizen_str_sizen(set_dict, "performance.stat-prefetch",
+                                       "off");
+        if (ret) {
+            gf_smsg("glusterd", GF_LOG_ERROR, errno, GD_MSG_DICT_SET_FAILED,
+                    "Key=performance.stat-prefetch", NULL);
+            goto out;
+        }
 
-                if (mod_dict && (data = dict_get (mod_dict, "volume-name"))) {
-                        volname = data->data;
-                        if (strcmp (volname, voliter->volname) == 0)
-                                dict_copy (mod_dict, set_dict);
-                }
+        ret = dict_set_sizen_str_sizen(set_dict,
+                                       "performance.client-io-threads", "off");
+        if (ret) {
+            gf_smsg("glusterd", GF_LOG_ERROR, errno, GD_MSG_DICT_SET_FAILED,
+                    "Key=performance.client-io-threads", NULL);
+            goto out;
+        }
 
-                ret = build_client_graph (&cgraph, voliter, set_dict);
-                if (ret)
-                        goto out;
+        ret = dict_set_str_sizen(set_dict, "client-transport-type", nfs_xprt);
+        if (ret) {
+            gf_smsg("glusterd", GF_LOG_ERROR, errno, GD_MSG_DICT_SET_FAILED,
+                    "Key=client-transport-type", NULL);
+            goto out;
+        }
 
-                if (mod_dict) {
-                        dict_copy (mod_dict, set_dict);
-                        ret = volgen_graph_set_options_generic (&cgraph, set_dict, voliter,
-                                                                basic_option_handler);
-                } else {
-                        ret = volgen_graph_set_options_generic (&cgraph, voliter->dict, voliter,
-                                                                basic_option_handler);
-                }
+        ret = dict_set_uint32(set_dict, "trusted-client", GF_CLIENT_TRUSTED);
+        if (ret) {
+            gf_smsg("glusterd", GF_LOG_ERROR, errno, GD_MSG_DICT_SET_FAILED,
+                    "Key=trusted-client", NULL);
+            goto out;
+        }
 
-                if (ret)
-                        goto out;
+        ret = dict_set_sizen_str_sizen(set_dict, "nfs-volume-file", "yes");
+        if (ret) {
+            gf_smsg("glusterd", GF_LOG_ERROR, errno, GD_MSG_DICT_SET_FAILED,
+                    "Key=nfs-volume-file", NULL);
+            goto out;
+        }
 
-                ret = volgen_graph_merge_sub (graph, &cgraph, 1);
-                if (ret)
-                        goto out;
-                ret = dict_reset (set_dict);
-                if (ret)
-                        goto out;
+        if (mod_dict && (data = dict_get_sizen(mod_dict, "volume-name"))) {
+            volname = data->data;
+            if (strcmp(volname, voliter->volname) == 0)
+                dict_copy(mod_dict, set_dict);
         }
 
-        cds_list_for_each_entry (voliter, &priv->volumes, vol_list) {
+        ret = build_client_graph(&cgraph, voliter, set_dict);
+        if (ret)
+            goto out;
 
-                if (mod_dict) {
-                        ret = volgen_graph_set_options_generic (graph, mod_dict, voliter,
-                                                                nfs_option_handler);
-                } else {
-                        ret = volgen_graph_set_options_generic (graph, voliter->dict, voliter,
-                                                                nfs_option_handler);
-                }
+        if (mod_dict) {
+            dict_copy(mod_dict, set_dict);
+            ret = volgen_graph_set_options_generic(&cgraph, set_dict, voliter,
+                                                   basic_option_handler);
+        } else {
+            ret = volgen_graph_set_options_generic(
+                &cgraph, voliter->dict, voliter, basic_option_handler);
+        }
 
-                if (ret)
-                        gf_log ("glusterd", GF_LOG_WARNING, "Could not set "
-                                 "vol-options for the volume %s", voliter->volname);
+        if (ret)
+            goto out;
+
+        ret = volgen_graph_set_iam_nfsd(&cgraph);
+        if (ret)
+            goto out;
+
+        ret = volgen_graph_merge_sub(graph, &cgraph, 1);
+        if (ret)
+            goto out;
+        ret = dict_reset(set_dict);
+        if (ret)
+            goto out;
+    }
+
+    cds_list_for_each_entry(voliter, &priv->volumes, vol_list)
+    {
+        if (mod_dict) {
+            ret = volgen_graph_set_options_generic(graph, mod_dict, voliter,
+                                                   nfs_option_handler);
+        } else {
+            ret = volgen_graph_set_options_generic(graph, voliter->dict,
+                                                   voliter, nfs_option_handler);
         }
 
- out:
-        gf_log ("glusterd", GF_LOG_DEBUG, "Returning %d", ret);
-        dict_destroy (set_dict);
+        if (ret)
+            gf_msg("glusterd", GF_LOG_WARNING, 0, GD_MSG_GRAPH_SET_OPT_FAIL,
+                   "Could not set "
+                   "vol-options for the volume %s",
+                   voliter->volname);
+    }
 
-        return ret;
-}
+out:
+    gf_msg_debug("glusterd", 0, "Returning %d", ret);
+    dict_unref(set_dict);
 
+    return ret;
+}
+#endif
 /****************************
  *
  * Volume generation interface
  *
  ****************************/
 
-
 static void
-get_brick_filepath (char *filename, glusterd_volinfo_t *volinfo,
-                    glusterd_brickinfo_t *brickinfo)
+get_brick_filepath(char *filename, glusterd_volinfo_t *volinfo,
+                   glusterd_brickinfo_t *brickinfo, char *prefix)
+{
+    char path[PATH_MAX] = {
+        0,
+    };
+    char brick[PATH_MAX] = {
+        0,
+    };
+    glusterd_conf_t *priv = NULL;
+    int32_t len = 0;
+
+    priv = THIS->private;
+
+    GLUSTERD_REMOVE_SLASH_FROM_PATH(brickinfo->path, brick);
+    GLUSTERD_GET_VOLUME_DIR(path, volinfo, priv);
+
+    if (prefix)
+        len = snprintf(filename, PATH_MAX, "%s/%s.%s.%s.%s.vol", path,
+                       volinfo->volname, prefix, brickinfo->hostname, brick);
+    else
+        len = snprintf(filename, PATH_MAX, "%s/%s.%s.%s.vol", path,
+                       volinfo->volname, brickinfo->hostname, brick);
+    if ((len < 0) || (len >= PATH_MAX)) {
+        filename[0] = 0;
+    }
+}
+
+gf_boolean_t
+glusterd_is_valid_volfpath(char *volname, char *brick)
+{
+    char volfpath[PATH_MAX] = {
+        0,
+    };
+    glusterd_brickinfo_t *brickinfo = NULL;
+    glusterd_volinfo_t *volinfo = NULL;
+    int32_t ret = 0;
+    xlator_t *this = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+
+    ret = glusterd_brickinfo_new_from_brick(brick, &brickinfo, _gf_false, NULL);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_WARNING, 0, GD_MSG_BRICKINFO_CREATE_FAIL,
+               "Failed to create brickinfo"
+               " for brick %s",
+               brick);
+        ret = 0;
+        goto out;
+    }
+    ret = glusterd_volinfo_new(&volinfo);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_WARNING, 0, GD_MSG_VOLINFO_STORE_FAIL,
+               "Failed to create volinfo");
+        ret = 0;
+        goto out;
+    }
+    (void)snprintf(volinfo->volname, sizeof(volinfo->volname), "%s", volname);
+    get_brick_filepath(volfpath, volinfo, brickinfo, NULL);
+
+    ret = ((strlen(volfpath) < PATH_MAX) &&
+           strlen(strrchr(volfpath, '/')) < _POSIX_PATH_MAX);
+
+out:
+    if (brickinfo)
+        glusterd_brickinfo_delete(brickinfo);
+    if (volinfo)
+        glusterd_volinfo_unref(volinfo);
+    return ret;
+}
+
+int
+glusterd_build_gfproxyd_volfile(glusterd_volinfo_t *volinfo, char *filename)
 {
-        char  path[PATH_MAX]   = {0,};
-        char  brick[PATH_MAX]  = {0,};
-        glusterd_conf_t *priv  = NULL;
+    volgen_graph_t graph = {
+        0,
+    };
+    int ret = -1;
 
-        priv = THIS->private;
+    ret = build_graph_generic(&graph, volinfo, NULL, NULL,
+                              &gfproxy_server_graph_builder);
+    if (ret == 0)
+        ret = volgen_write_volfile(&graph, filename);
 
-        GLUSTERD_REMOVE_SLASH_FROM_PATH (brickinfo->path, brick);
-        GLUSTERD_GET_VOLUME_DIR (path, volinfo, priv);
+    volgen_graph_free(&graph);
 
-        snprintf (filename, PATH_MAX, "%s/%s.%s.%s.vol",
-                  path, volinfo->volname,
-                  brickinfo->hostname,
-                  brick);
+    return ret;
 }
 
-gf_boolean_t
-glusterd_is_valid_volfpath (char *volname, char *brick)
+int
+glusterd_generate_gfproxyd_volfile(glusterd_volinfo_t *volinfo)
 {
-        char                    volfpath[PATH_MAX] = {0,};
-        glusterd_brickinfo_t    *brickinfo = NULL;
-        glusterd_volinfo_t      *volinfo = NULL;
-        int32_t                 ret = 0;
-        xlator_t                *this = NULL;
+    char filename[PATH_MAX] = {
+        0,
+    };
+    int ret = -1;
 
-        this = THIS;
-        GF_ASSERT (this);
+    GF_ASSERT(volinfo);
 
-        ret = glusterd_brickinfo_new_from_brick (brick, &brickinfo);
-        if (ret) {
-                gf_log (this->name, GF_LOG_WARNING, "Failed to create brickinfo"
-                        " for brick %s", brick );
-                ret = 0;
-                goto out;
-        }
-        ret = glusterd_volinfo_new (&volinfo);
-        if (ret) {
-                gf_log (this->name, GF_LOG_WARNING, "Failed to create volinfo");
-                ret = 0;
-                goto out;
-        }
-        strncpy (volinfo->volname, volname, sizeof (volinfo->volname));
-        get_brick_filepath (volfpath, volinfo, brickinfo);
+    glusterd_svc_build_gfproxyd_volfile_path(volinfo, filename, PATH_MAX - 1);
 
-        ret = ((strlen(volfpath) < PATH_MAX) &&
-                strlen (strrchr(volfpath, '/')) < _POSIX_PATH_MAX);
+    ret = glusterd_build_gfproxyd_volfile(volinfo, filename);
 
-out:
-        if (brickinfo)
-                glusterd_brickinfo_delete (brickinfo);
-        if (volinfo)
-                glusterd_volinfo_unref (volinfo);
-        return ret;
+    return ret;
 }
 
 static int
-glusterd_generate_brick_volfile (glusterd_volinfo_t *volinfo,
-                                 glusterd_brickinfo_t *brickinfo)
+glusterd_generate_brick_volfile(glusterd_volinfo_t *volinfo,
+                                glusterd_brickinfo_t *brickinfo,
+                                dict_t *mod_dict, void *data)
 {
-        volgen_graph_t graph = {0,};
-        char    filename[PATH_MAX] = {0,};
-        int     ret = -1;
+    volgen_graph_t graph = {
+        0,
+    };
+    char filename[PATH_MAX] = {
+        0,
+    };
+    int ret = -1;
 
-        GF_ASSERT (volinfo);
-        GF_ASSERT (brickinfo);
+    GF_ASSERT(volinfo);
+    GF_ASSERT(brickinfo);
 
-        get_brick_filepath (filename, volinfo, brickinfo);
+    get_brick_filepath(filename, volinfo, brickinfo, NULL);
 
-        ret = build_server_graph (&graph, volinfo, NULL, brickinfo);
-        if (!ret)
-                ret = volgen_write_volfile (&graph, filename);
+    ret = build_server_graph(&graph, volinfo, mod_dict, brickinfo);
+    if (!ret)
+        ret = volgen_write_volfile(&graph, filename);
 
-        volgen_graph_free (&graph);
+    volgen_graph_free(&graph);
 
-        return ret;
+    return ret;
 }
 
 int
-build_quotad_graph (volgen_graph_t *graph, dict_t *mod_dict)
-{
-        volgen_graph_t     cgraph         = {0};
-        glusterd_volinfo_t *voliter       = NULL;
-        xlator_t           *this          = NULL;
-        glusterd_conf_t    *priv          = NULL;
-        dict_t             *set_dict      = NULL;
-        int                ret            = 0;
-        xlator_t           *quotad_xl     = NULL;
-        char               *skey          = NULL;
-
-        this = THIS;
-        GF_ASSERT (this);
-
-        priv = this->private;
-        GF_ASSERT (priv);
-
-        set_dict = dict_new ();
-        if (!set_dict) {
-                ret = -ENOMEM;
-                goto out;
-        }
+build_quotad_graph(volgen_graph_t *graph, dict_t *mod_dict)
+{
+    volgen_graph_t cgraph = {0};
+    glusterd_volinfo_t *voliter = NULL;
+    xlator_t *this = NULL;
+    glusterd_conf_t *priv = NULL;
+    dict_t *set_dict = NULL;
+    int ret = 0;
+    xlator_t *quotad_xl = NULL;
+    char *skey = NULL;
 
-        quotad_xl = volgen_graph_add_as (graph, "features/quotad", "quotad");
-        if (!quotad_xl) {
-                ret = -1;
-                goto out;
-        }
+    this = THIS;
+    GF_ASSERT(this);
 
-        cds_list_for_each_entry (voliter, &priv->volumes, vol_list) {
-                if (voliter->status != GLUSTERD_STATUS_STARTED)
-                        continue;
+    priv = this->private;
+    GF_ASSERT(priv);
 
-                if (1 != glusterd_is_volume_quota_enabled (voliter))
-                        continue;
+    graph->type = GF_QUOTAD;
 
-                ret = dict_set_uint32 (set_dict, "trusted-client",
-                                       GF_CLIENT_TRUSTED);
-                if (ret)
-                        goto out;
+    set_dict = dict_new();
+    if (!set_dict) {
+        ret = -ENOMEM;
+        goto out;
+    }
 
-                dict_copy (voliter->dict, set_dict);
-                if (mod_dict)
-                        dict_copy (mod_dict, set_dict);
+    quotad_xl = volgen_graph_add_as(graph, "features/quotad", "quotad");
+    if (!quotad_xl) {
+        ret = -1;
+        goto out;
+    }
 
-                ret = gf_asprintf(&skey, "%s.volume-id", voliter->volname);
-                if (ret == -1) {
-                        gf_log("", GF_LOG_ERROR, "Out of memory");
-                        goto out;
-                }
-                ret = xlator_set_option(quotad_xl, skey, voliter->volname);
-                GF_FREE(skey);
-                if (ret)
-                        goto out;
+    cds_list_for_each_entry(voliter, &priv->volumes, vol_list)
+    {
+        if (voliter->status != GLUSTERD_STATUS_STARTED)
+            continue;
 
-                memset (&cgraph, 0, sizeof (cgraph));
-                ret = volgen_graph_build_clients (&cgraph, voliter, set_dict,
-                                                  NULL);
-                if (ret)
-                        goto out;
+        if (1 != glusterd_is_volume_quota_enabled(voliter))
+            continue;
 
-                ret = volume_volgen_graph_build_clusters (&cgraph, voliter,
-                                                          _gf_true);
-                if (ret) {
-                        ret = -1;
-                        goto out;
-                }
+        ret = dict_set_uint32(set_dict, "trusted-client", GF_CLIENT_TRUSTED);
+        if (ret) {
+            gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_SET_FAILED,
+                    "Key=trusted-client", NULL);
+            goto out;
+        }
 
-                if (mod_dict) {
-                        dict_copy (mod_dict, set_dict);
-                        ret = volgen_graph_set_options_generic (&cgraph, set_dict,
-                                                        voliter,
-                                                        basic_option_handler);
-                } else {
-                        ret = volgen_graph_set_options_generic (&cgraph,
-                                                                voliter->dict,
-                                                                voliter,
-                                                                basic_option_handler);
-                }
-                if (ret)
-                        goto out;
+        dict_copy(voliter->dict, set_dict);
+        if (mod_dict)
+            dict_copy(mod_dict, set_dict);
 
-                ret = volgen_graph_merge_sub (graph, &cgraph, 1);
-                if (ret)
-                        goto out;
+        ret = gf_asprintf(&skey, "%s.volume-id", voliter->volname);
+        if (ret == -1) {
+            gf_msg("glusterd", GF_LOG_ERROR, ENOMEM, GD_MSG_NO_MEMORY,
+                   "Out of memory");
+            goto out;
+        }
+        ret = xlator_set_option(quotad_xl, skey, ret, voliter->volname);
+        GF_FREE(skey);
+        if (ret)
+            goto out;
 
-                ret = dict_reset (set_dict);
-                if (ret)
-                        goto out;
+        memset(&cgraph, 0, sizeof(cgraph));
+        ret = volgen_graph_build_clients(&cgraph, voliter, set_dict, NULL);
+        if (ret)
+            goto out;
+
+        ret = volume_volgen_graph_build_clusters(&cgraph, voliter, _gf_true);
+        if (ret) {
+            ret = -1;
+            goto out;
         }
 
+        if (mod_dict) {
+            dict_copy(mod_dict, set_dict);
+            ret = volgen_graph_set_options_generic(&cgraph, set_dict, voliter,
+                                                   basic_option_handler);
+        } else {
+            ret = volgen_graph_set_options_generic(
+                &cgraph, voliter->dict, voliter, basic_option_handler);
+        }
+        if (ret)
+            goto out;
+
+        ret = volgen_graph_merge_sub(graph, &cgraph, 1);
+        if (ret)
+            goto out;
+
+        ret = dict_reset(set_dict);
+        if (ret)
+            goto out;
+    }
+
 out:
-        if (set_dict)
-                dict_unref (set_dict);
-        return ret;
+    if (set_dict)
+        dict_unref(set_dict);
+    return ret;
 }
 
 static void
-get_vol_tstamp_file (char *filename, glusterd_volinfo_t *volinfo)
+get_vol_tstamp_file(char *filename, glusterd_volinfo_t *volinfo)
 {
-        glusterd_conf_t *priv  = NULL;
+    glusterd_conf_t *priv = NULL;
 
-        priv = THIS->private;
+    priv = THIS->private;
 
-        GLUSTERD_GET_VOLUME_DIR (filename, volinfo, priv);
-        strncat (filename, "/marker.tstamp",
-                 PATH_MAX - strlen(filename) - 1);
+    GLUSTERD_GET_VOLUME_DIR(filename, volinfo, priv);
+    strncat(filename, "/marker.tstamp", PATH_MAX - strlen(filename) - 1);
 }
 
 static void
-get_parent_vol_tstamp_file (char *filename, glusterd_volinfo_t *volinfo)
+get_parent_vol_tstamp_file(char *filename, glusterd_volinfo_t *volinfo)
 {
-        glusterd_conf_t *priv  = NULL;
-        xlator_t        *this  = NULL;
+    glusterd_conf_t *priv = NULL;
+    xlator_t *this = NULL;
+    int32_t len = 0;
 
-        this = THIS;
-        GF_ASSERT (this);
-        priv = this->private;
-        GF_ASSERT (priv);
+    this = THIS;
+    GF_ASSERT(this);
+    priv = this->private;
+    GF_ASSERT(priv);
 
-        snprintf (filename, PATH_MAX, "%s/vols/%s", priv->workdir,
-                  volinfo->parent_volname);
-        strncat (filename, "/marker.tstamp",
-                 PATH_MAX - strlen(filename) - 1);
+    len = snprintf(filename, PATH_MAX, "%s/vols/%s/marker.tstamp",
+                   priv->workdir, volinfo->parent_volname);
+    if ((len < 0) || (len >= PATH_MAX)) {
+        filename[0] = 0;
+    }
 }
 
 int
-generate_brick_volfiles (glusterd_volinfo_t *volinfo)
+generate_brick_volfiles(glusterd_volinfo_t *volinfo)
+{
+    char tstamp_file[PATH_MAX] = {
+        0,
+    };
+    char parent_tstamp_file[PATH_MAX] = {
+        0,
+    };
+    int ret = -1;
+    xlator_t *this = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+
+    ret = glusterd_volinfo_get_boolean(volinfo, VKEY_MARKER_XTIME);
+    if (ret == -1)
+        return -1;
+
+    assign_brick_groups(volinfo);
+    get_vol_tstamp_file(tstamp_file, volinfo);
+
+    if (ret) {
+        ret = open(tstamp_file, O_WRONLY | O_CREAT | O_EXCL, 0600);
+        if (ret == -1 && errno == EEXIST) {
+            gf_msg_debug(this->name, 0, "timestamp file exist");
+            ret = -2;
+        }
+        if (ret == -1) {
+            gf_msg(this->name, GF_LOG_ERROR, errno, GD_MSG_FILE_OP_FAILED,
+                   "failed to create "
+                   "%s",
+                   tstamp_file);
+            return -1;
+        }
+        if (ret >= 0) {
+            sys_close(ret);
+            /* If snap_volume, retain timestamp for marker.tstamp
+             * from parent. Geo-replication depends on mtime of
+             * 'marker.tstamp' to decide the volume-mark, i.e.,
+             * geo-rep start time just after session is created.
+             */
+            if (volinfo->is_snap_volume) {
+                get_parent_vol_tstamp_file(parent_tstamp_file, volinfo);
+                ret = gf_set_timestamp(parent_tstamp_file, tstamp_file);
+                if (ret) {
+                    gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_TSTAMP_SET_FAIL,
+                           "Unable to set atime and mtime"
+                           " of %s as of %s",
+                           tstamp_file, parent_tstamp_file);
+                    goto out;
+                }
+            }
+        }
+    } else {
+        ret = sys_unlink(tstamp_file);
+        if (ret == -1 && errno == ENOENT)
+            ret = 0;
+        if (ret == -1) {
+            gf_msg(this->name, GF_LOG_ERROR, errno, GD_MSG_FILE_OP_FAILED,
+                   "failed to unlink "
+                   "%s",
+                   tstamp_file);
+            return -1;
+        }
+    }
+
+    ret = glusterd_volume_brick_for_each(volinfo, NULL,
+                                         glusterd_generate_brick_volfile);
+    if (ret)
+        goto out;
+
+    ret = 0;
+
+out:
+    gf_msg_debug(this->name, 0, "Returning %d", ret);
+    return ret;
+}
+
+static int
+generate_single_transport_client_volfile(glusterd_volinfo_t *volinfo,
+                                         char *filepath, dict_t *dict)
 {
-        glusterd_brickinfo_t    *brickinfo                    = NULL;
-        char                     tstamp_file[PATH_MAX]        = {0,};
-        char                     parent_tstamp_file[PATH_MAX] = {0,};
-        int                      ret                          = -1;
-        xlator_t                *this                         = NULL;
+    volgen_graph_t graph = {
+        0,
+    };
+    int ret = -1;
 
-        this = THIS;
-        GF_ASSERT (this);
+    ret = build_client_graph(&graph, volinfo, dict);
+    if (!ret)
+        ret = volgen_write_volfile(&graph, filepath);
 
-        ret = glusterd_volinfo_get_boolean (volinfo, VKEY_MARKER_XTIME);
-        if (ret == -1)
-                return -1;
+    volgen_graph_free(&graph);
 
-        get_vol_tstamp_file (tstamp_file, volinfo);
+    return ret;
+}
 
-        if (ret) {
-                ret = open (tstamp_file, O_WRONLY|O_CREAT|O_EXCL, 0600);
-                if (ret == -1 && errno == EEXIST) {
-                        gf_log (this->name, GF_LOG_DEBUG,
-                                "timestamp file exist");
-                        ret = -2;
-                }
-                if (ret == -1) {
-                        gf_log (this->name, GF_LOG_ERROR, "failed to create "
-                                "%s (%s)", tstamp_file, strerror (errno));
-                        return -1;
-                }
-                if (ret >= 0) {
-                        close (ret);
-                        /* If snap_volume, retain timestamp for marker.tstamp
-                         * from parent. Geo-replication depends on mtime of
-                         * 'marker.tstamp' to decide the volume-mark, i.e.,
-                         * geo-rep start time just after session is created.
-                         */
-                        if (volinfo->is_snap_volume) {
-                                get_parent_vol_tstamp_file (parent_tstamp_file,
-                                                            volinfo);
-                                ret = gf_set_timestamp (parent_tstamp_file,
-                                                        tstamp_file);
-                                if (ret) {
-                                        gf_log (this->name, GF_LOG_ERROR,
-                                                "Unable to set atime and mtime"
-                                                " of %s as of %s", tstamp_file,
-                                                parent_tstamp_file);
-                                        goto out;
-                                }
-                        }
+int
+glusterd_generate_client_per_brick_volfile(glusterd_volinfo_t *volinfo)
+{
+    char filepath[PATH_MAX] = {
+        0,
+    };
+    glusterd_brickinfo_t *brick = NULL;
+    volgen_graph_t graph = {
+        0,
+    };
+    dict_t *dict = NULL;
+    xlator_t *xl = NULL;
+    int ret = -1;
+    char *ssl_str = NULL;
+    gf_boolean_t ssl_bool = _gf_false;
+    xlator_t *this = THIS;
+    GF_ASSERT(this);
+
+    dict = dict_new();
+    if (!dict) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_CREATE_FAIL, NULL);
+        goto out;
+    }
+
+    ret = dict_set_uint32(dict, "trusted-client", GF_CLIENT_TRUSTED);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_SET_FAILED,
+                "Key=trusted-client", NULL);
+        goto free_dict;
+    }
+
+    if (dict_get_str_sizen(volinfo->dict, "client.ssl", &ssl_str) == 0) {
+        if (gf_string2boolean(ssl_str, &ssl_bool) == 0) {
+            if (ssl_bool) {
+                if (dict_set_dynstr_with_alloc(dict, "client.ssl", "on") != 0) {
+                    ret = -1;
+                    goto free_dict;
                 }
+            }
         } else {
-                ret = unlink (tstamp_file);
-                if (ret == -1 && errno == ENOENT)
-                        ret = 0;
-                if (ret == -1) {
-                        gf_log (this->name, GF_LOG_ERROR, "failed to unlink "
-                                "%s (%s)", tstamp_file, strerror (errno));
-                        return -1;
-                }
+            ret = -1;
+            goto free_dict;
         }
+    }
 
-        cds_list_for_each_entry (brickinfo, &volinfo->bricks, brick_list) {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "Found a brick - %s:%s", brickinfo->hostname,
-                        brickinfo->path);
-
-                ret = glusterd_generate_brick_volfile (volinfo, brickinfo);
-                if (ret)
-                        goto out;
-
+    cds_list_for_each_entry(brick, &volinfo->bricks, brick_list)
+    {
+        xl = volgen_graph_build_client(&graph, volinfo, brick->hostname, NULL,
+                                       brick->path, brick->brick_id, "tcp",
+                                       dict);
+        if (!xl) {
+            ret = -1;
+            goto out;
         }
 
-        ret = 0;
+        get_brick_filepath(filepath, volinfo, brick, "client");
+        ret = volgen_write_volfile(&graph, filepath);
+        if (ret < 0)
+            goto out;
 
-out:
-        gf_log (this->name, GF_LOG_DEBUG, "Returning %d", ret);
-        return ret;
-}
+        volgen_graph_free(&graph);
+        memset(&graph, 0, sizeof(graph));
+    }
 
-static int
-generate_single_transport_client_volfile (glusterd_volinfo_t *volinfo,
-                                          char *filepath, dict_t *dict)
-{
-        volgen_graph_t graph = {0,};
-        int     ret = -1;
+    ret = 0;
+out:
+    if (ret)
+        volgen_graph_free(&graph);
 
-        ret = build_client_graph (&graph, volinfo, dict);
-        if (!ret)
-                ret = volgen_write_volfile (&graph, filepath);
+free_dict:
 
-        volgen_graph_free (&graph);
+    if (dict)
+        dict_unref(dict);
 
-        return ret;
+    return ret;
 }
 
 static void
-enumerate_transport_reqs (gf_transport_type type, char **types)
+enumerate_transport_reqs(gf_transport_type type, char **types)
 {
-        switch (type) {
+    switch (type) {
         case GF_TRANSPORT_TCP:
-                types[0] = "tcp";
-                break;
+            types[0] = "tcp";
+            break;
         case GF_TRANSPORT_RDMA:
-                types[0] = "rdma";
-                break;
+            types[0] = "rdma";
+            break;
         case GF_TRANSPORT_BOTH_TCP_RDMA:
-                types[0] = "tcp";
-                types[1] = "rdma";
-                break;
-        }
+            types[0] = "tcp";
+            types[1] = "rdma";
+            break;
+    }
 }
 
 int
-generate_client_volfiles (glusterd_volinfo_t *volinfo,
-                          glusterd_client_type_t client_type)
-{
-        int                i                  = 0;
-        int                ret                = -1;
-        char               filepath[PATH_MAX] = {0,};
-        char               *types[]           = {NULL, NULL, NULL};
-        dict_t             *dict              = NULL;
-        xlator_t           *this              = NULL;
-        gf_transport_type  type               = GF_TRANSPORT_TCP;
-
-        this = THIS;
-
-        enumerate_transport_reqs (volinfo->transport_type, types);
-        dict = dict_new ();
-        if (!dict)
-                goto out;
-        for (i = 0; types[i]; i++) {
-                memset (filepath, 0, sizeof (filepath));
-                ret = dict_set_str (dict, "client-transport-type", types[i]);
-                if (ret)
-                        goto out;
-                type = transport_str_to_type (types[i]);
-
-                ret = dict_set_uint32 (dict, "trusted-client", client_type);
-                if (ret)
-                        goto out;
+generate_dummy_client_volfiles(glusterd_volinfo_t *volinfo)
+{
+    int i = 0;
+    int ret = -1;
+    char filepath[PATH_MAX] = {
+        0,
+    };
+    char *types[] = {NULL, NULL, NULL};
+    dict_t *dict = NULL;
+    xlator_t *this = NULL;
+    gf_transport_type type = GF_TRANSPORT_TCP;
+
+    this = THIS;
+
+    enumerate_transport_reqs(volinfo->transport_type, types);
+    dict = dict_new();
+    if (!dict) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_CREATE_FAIL, NULL);
+        goto out;
+    }
+    for (i = 0; types[i]; i++) {
+        ret = dict_set_str(dict, "client-transport-type", types[i]);
+        if (ret) {
+            gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_SET_FAILED,
+                    "Key=client-transport-type", NULL);
+            goto out;
+        }
+        type = transport_str_to_type(types[i]);
 
-                if (client_type == GF_CLIENT_TRUSTED) {
-                        ret = glusterd_get_trusted_client_filepath (filepath,
-                                                                    volinfo,
-                                                                    type);
-                } else {
-                        ret = glusterd_get_client_filepath (filepath,
-                                                            volinfo,
-                                                            type);
-                }
-                if (ret) {
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "Received invalid transport-type");
-                        goto out;
-                }
+        ret = dict_set_uint32(dict, "trusted-client", GF_CLIENT_OTHER);
+        if (ret) {
+            gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_SET_FAILED,
+                    "Key=trusted-client", NULL);
+            goto out;
+        }
 
-                ret = generate_single_transport_client_volfile (volinfo,
-                                                                filepath,
-                                                                dict);
-                if (ret)
-                        goto out;
+        ret = glusterd_get_dummy_client_filepath(filepath, volinfo, type);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, EINVAL, GD_MSG_INVALID_ENTRY,
+                   "Received invalid transport-type.");
+            goto out;
         }
 
-        /* Generate volfile for rebalance process */
-        ret = dict_set_int32 (dict, "rebalance-volfile-creation", _gf_true);
+        ret = generate_single_transport_client_volfile(volinfo, filepath, dict);
+        if (ret)
+            goto out;
+    }
+
+out:
+    if (dict)
+        dict_unref(dict);
+
+    gf_msg_trace("glusterd", 0, "Returning %d", ret);
+    return ret;
+}
+
+int
+generate_client_volfiles(glusterd_volinfo_t *volinfo,
+                         glusterd_client_type_t client_type)
+{
+    int i = 0;
+    int ret = -1;
+    char filepath[PATH_MAX] = {
+        0,
+    };
+    char *volname = NULL;
+    char *types[] = {NULL, NULL, NULL};
+    dict_t *dict = NULL;
+    xlator_t *this = NULL;
+    gf_transport_type type = GF_TRANSPORT_TCP;
+
+    this = THIS;
+
+    volname = volinfo->is_snap_volume ? volinfo->parent_volname
+                                      : volinfo->volname;
+
+    if (volname && !strcmp(volname, GLUSTER_SHARED_STORAGE) &&
+        client_type != GF_CLIENT_TRUSTED) {
+        /*
+         * shared storage volume cannot be mounted from non trusted
+         * nodes. So we are not creating volfiles for non-trusted
+         * clients for shared volumes as well as snapshot of shared
+         * volumes.
+         */
+
+        ret = 0;
+        gf_msg_debug("glusterd", 0,
+                     "Skipping the non-trusted volfile"
+                     "creation for shared storage volume. Volume %s",
+                     volname);
+        goto out;
+    }
+
+    enumerate_transport_reqs(volinfo->transport_type, types);
+    dict = dict_new();
+    if (!dict) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_CREATE_FAIL, NULL);
+        goto out;
+    }
+    for (i = 0; types[i]; i++) {
+        ret = dict_set_str(dict, "client-transport-type", types[i]);
         if (ret) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "Failed to set rebalance-volfile-creation");
-                goto out;
+            gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_SET_FAILED,
+                    "Key=client-transport-type", NULL);
+            goto out;
         }
+        type = transport_str_to_type(types[i]);
 
-        glusterd_get_rebalance_volfile (volinfo, filepath, PATH_MAX);
+        ret = dict_set_uint32(dict, "trusted-client", client_type);
+        if (ret) {
+            gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_SET_FAILED,
+                    "Key=trusted-client", NULL);
+            goto out;
+        }
 
-        ret = generate_single_transport_client_volfile (volinfo,
-                                                        filepath,
-                                                        dict);
+        if (client_type == GF_CLIENT_TRUSTED) {
+            ret = glusterd_get_trusted_client_filepath(filepath, volinfo, type);
+        } else if (client_type == GF_CLIENT_TRUSTED_PROXY) {
+            glusterd_get_gfproxy_client_volfile(volinfo, filepath, PATH_MAX);
+            ret = dict_set_int32_sizen(dict, "gfproxy-client", 1);
+        } else {
+            ret = glusterd_get_client_filepath(filepath, volinfo, type);
+        }
         if (ret) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "Failed to create rebalance volfile for %s",
-                        volinfo->volname);
-                goto out;
+            gf_msg(this->name, GF_LOG_ERROR, EINVAL, GD_MSG_INVALID_ENTRY,
+                   "Received invalid transport-type");
+            goto out;
         }
 
+        ret = generate_single_transport_client_volfile(volinfo, filepath, dict);
+        if (ret)
+            goto out;
+    }
+
+    /* Generate volfile for rebalance process */
+    glusterd_get_rebalance_volfile(volinfo, filepath, PATH_MAX);
+    ret = build_rebalance_volfile(volinfo, filepath, dict);
+
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_VOLFILE_CREATE_FAIL,
+               "Failed to create rebalance volfile for %s", volinfo->volname);
+        goto out;
+    }
+
 out:
-        if (dict)
-                dict_unref (dict);
+    if (dict)
+        dict_unref(dict);
 
-        gf_log ("", GF_LOG_TRACE, "Returning %d", ret);
-        return ret;
+    gf_msg_trace("glusterd", 0, "Returning %d", ret);
+    return ret;
 }
 
 int
-glusterd_snapdsvc_generate_volfile (volgen_graph_t *graph,
-                                    glusterd_volinfo_t *volinfo)
-{
-        xlator_t       *xl              = NULL;
-        char           *username        = NULL;
-        char           *passwd          = NULL;
-        int             ret             = 0;
-        char            key [PATH_MAX]  = {0, };
-        dict_t         *set_dict        = NULL;
-        char           *loglevel        = NULL;
-        char           *xlator          = NULL;
-        char           *value           = NULL;
-
-        set_dict = dict_copy (volinfo->dict, NULL);
-        if (!set_dict)
-                return -1;
+glusterd_snapdsvc_generate_volfile(volgen_graph_t *graph,
+                                   glusterd_volinfo_t *volinfo)
+{
+    xlator_t *xl = NULL;
+    char *username = NULL;
+    char *passwd = NULL;
+    int ret = 0;
+    char key[PATH_MAX] = {
+        0,
+    };
+    dict_t *set_dict = NULL;
+    char *loglevel = NULL;
+    char *xlator = NULL;
+    char *ssl_str = NULL;
+    gf_boolean_t ssl_bool = _gf_false;
+
+    set_dict = dict_copy(volinfo->dict, NULL);
+    if (!set_dict)
+        return -1;
 
-        ret = dict_get_str (set_dict, "xlator", &xlator);
-        if (!ret) {
-                ret = dict_get_str (set_dict, "loglevel", &loglevel);
-                if (ret) {
-                        gf_log ("glusterd", GF_LOG_ERROR, "could not get both"
-                                " translator name and loglevel for log level "
-                                "request");
-                        return -1;
-                }
+    ret = dict_get_str_sizen(set_dict, "xlator", &xlator);
+    if (!ret) {
+        ret = dict_get_str_sizen(set_dict, "loglevel", &loglevel);
+        if (ret) {
+            gf_msg("glusterd", GF_LOG_ERROR, errno, GD_MSG_DICT_GET_FAILED,
+                   "could not get both"
+                   " translator name and loglevel for log level "
+                   "request");
+            return -1;
         }
+    }
 
-        xl = volgen_graph_add (graph, "features/snapview-server",
-                               volinfo->volname);
-        if (!xl)
-                return -1;
+    xl = volgen_graph_add(graph, "features/snapview-server", volinfo->volname);
+    if (!xl)
+        return -1;
 
-        ret = xlator_set_option (xl, "volname", volinfo->volname);
-        if (ret)
-                return -1;
+    ret = xlator_set_fixed_option(xl, "volname", volinfo->volname);
+    if (ret)
+        return -1;
 
-        xl = volgen_graph_add (graph, "performance/io-threads",
-                               volinfo->volname);
-        if (!xl)
-                return -1;
+    xl = volgen_graph_add(graph, "performance/io-threads", volinfo->volname);
+    if (!xl)
+        return -1;
 
-        snprintf (key, sizeof (key), "snapd-%s", volinfo->volname);
-        xl = volgen_graph_add_as (graph, "debug/io-stats", key);
-        if (!xl)
-                return -1;
+    snprintf(key, sizeof(key), "snapd-%s", volinfo->volname);
+    xl = volgen_graph_add_as(graph, "debug/io-stats", key);
+    if (!xl)
+        return -1;
 
-        xl = volgen_graph_add (graph, "protocol/server", volinfo->volname);
-        if (!xl)
-                return -1;
+    xl = volgen_graph_add(graph, "protocol/server", volinfo->volname);
+    if (!xl)
+        return -1;
 
-        ret = xlator_set_option (xl, "transport-type", "tcp");
-        if (ret)
-                return -1;
+    ret = xlator_set_fixed_option(xl, "transport-type", "tcp");
+    if (ret)
+        return -1;
 
-        if (dict_get_str (set_dict, SSL_CERT_DEPTH_OPT, &value) == 0) {
-                ret = xlator_set_option (xl, "ssl-cert-depth", value);
+    if (dict_get_str_sizen(set_dict, "server.ssl", &ssl_str) == 0) {
+        if (gf_string2boolean(ssl_str, &ssl_bool) == 0) {
+            if (ssl_bool) {
+                ret = xlator_set_fixed_option(
+                    xl, "transport.socket.ssl-enabled", "true");
                 if (ret) {
-                        gf_log ("glusterd", GF_LOG_WARNING,
-                                "failed to set ssl-cert-depth");
-                        return -1;
+                    return -1;
                 }
-        }
+            }
+        }
+    }
+
+    RPC_SET_OPT(xl, SSL_OWN_CERT_OPT, "ssl-own-cert", return -1);
+    RPC_SET_OPT(xl, SSL_PRIVATE_KEY_OPT, "ssl-private-key", return -1);
+    RPC_SET_OPT(xl, SSL_CA_LIST_OPT, "ssl-ca-list", return -1);
+    RPC_SET_OPT(xl, SSL_CRL_PATH_OPT, "ssl-crl-path", return -1);
+    RPC_SET_OPT(xl, SSL_CERT_DEPTH_OPT, "ssl-cert-depth", return -1);
+    RPC_SET_OPT(xl, SSL_CIPHER_LIST_OPT, "ssl-cipher-list", return -1);
+    RPC_SET_OPT(xl, SSL_DH_PARAM_OPT, "ssl-dh-param", return -1);
+    RPC_SET_OPT(xl, SSL_EC_CURVE_OPT, "ssl-ec-curve", return -1);
+
+    username = glusterd_auth_get_username(volinfo);
+    passwd = glusterd_auth_get_password(volinfo);
+
+    ret = snprintf(key, sizeof(key), "auth.login.snapd-%s.allow",
+                   volinfo->volname);
+    ret = xlator_set_option(xl, key, ret, username);
+    if (ret)
+        return -1;
 
-        if (dict_get_str (set_dict, SSL_CIPHER_LIST_OPT, &value) == 0) {
-                ret = xlator_set_option (xl, "ssl-cipher-list", value);
-                if (ret) {
-                        gf_log ("glusterd", GF_LOG_WARNING,
-                                "failed to set ssl-cipher-list");
-                        return -1;
-                }
-        }
+    ret = snprintf(key, sizeof(key), "auth.login.%s.password", username);
+    ret = xlator_set_option(xl, key, ret, passwd);
+    if (ret)
+        return -1;
+
+    snprintf(key, sizeof(key), "snapd-%s", volinfo->volname);
+    ret = xlator_set_fixed_option(xl, "auth-path", key);
+    if (ret)
+        return -1;
 
-        username = glusterd_auth_get_username (volinfo);
-        passwd = glusterd_auth_get_password (volinfo);
+    ret = volgen_graph_set_options_generic(
+        graph, set_dict, (xlator && loglevel) ? (void *)set_dict : volinfo,
+        (xlator && loglevel) ? &server_spec_extended_option_handler
+                             : &server_spec_option_handler);
 
-        snprintf (key, sizeof (key), "auth.login.snapd-%s.allow",
-                                                        volinfo->volname);
-        ret = xlator_set_option (xl, key, username);
-        if (ret)
-                return -1;
+    return ret;
+}
 
-        snprintf (key, sizeof (key), "auth.login.%s.password", username);
-        ret = xlator_set_option (xl, key, passwd);
-        if (ret)
-                return -1;
+static int
+prepare_bitrot_scrub_volume_options(glusterd_volinfo_t *volinfo,
+                                    dict_t *mod_dict, dict_t *set_dict)
+{
+    int ret = 0;
+    xlator_t *this = THIS;
+    GF_ASSERT(this);
 
-        ret = volgen_graph_set_options_generic
-                (graph, set_dict,
-                 (xlator && loglevel)? (void *)set_dict: volinfo,
-                 (xlator && loglevel) ?
-                 &server_spec_extended_option_handler:
-                 &server_spec_option_handler);
+    ret = dict_set_uint32(set_dict, "trusted-client", GF_CLIENT_TRUSTED);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_SET_FAILED,
+                "Key=trusted-client", NULL);
+        goto out;
+    }
 
-        return ret;
+    dict_copy(volinfo->dict, set_dict);
+    if (mod_dict)
+        dict_copy(mod_dict, set_dict);
+
+out:
+    return ret;
 }
 
-int
-build_bitd_graph (volgen_graph_t *graph, dict_t *mod_dict)
-{
-        volgen_graph_t        cgraph        = {0};
-        glusterd_volinfo_t   *voliter       = NULL;
-        xlator_t             *this          = NULL;
-        glusterd_conf_t      *priv          = NULL;
-        dict_t               *set_dict      = NULL;
-        int                   ret           = 0;
-        xlator_t             *bitd_xl       = NULL;
-        xlator_t             *xl            = NULL;
-        xlator_t             *trav          = NULL;
-        xlator_t             *txl           = NULL;
-        char                 *skey          = NULL;
-        char                  transt[16]    = {0,};
-        glusterd_brickinfo_t *brickinfo     = NULL;
-        char                 *br_args[]     = {"features/bit-rot",
-                                               "bit-rot"};
-        int32_t               count         = 0;
-
-        this = THIS;
-        GF_ASSERT (this);
-
-        priv = this->private;
-        GF_ASSERT (priv);
-
-        set_dict = dict_new ();
-        if (!set_dict) {
-                ret = -ENOMEM;
-                goto out;
-        }
+static int
+build_bitd_clusters(volgen_graph_t *graph, glusterd_volinfo_t *volinfo,
+                    dict_t *set_dict, int brick_count, unsigned int numbricks)
+{
+    int ret = -1;
+    int clusters = 0;
+    xlator_t *xl = NULL;
+    char *brick_hint = NULL;
+    char *bitrot_args[] = {"features/bit-rot", "%s-bit-rot-%d"};
 
-        if (mod_dict)
-                dict_copy (mod_dict, set_dict);
+    ret = volgen_link_bricks_from_list_tail(graph, volinfo, bitrot_args[0],
+                                            bitrot_args[1], brick_count,
+                                            brick_count);
+    clusters = ret;
 
-        list_for_each_entry (voliter, &priv->volumes, vol_list) {
-                if (voliter->status != GLUSTERD_STATUS_STARTED)
-                        continue;
+    xl = first_of(graph);
 
-                if (!glusterd_is_bitrot_enabled (voliter))
-                        continue;
+    ret = gf_asprintf(&brick_hint, "%d", numbricks);
+    if (ret < 0)
+        goto out;
 
-                memset (transt, '\0', 16);
+    ret = xlator_set_fixed_option(xl, "brick-count", brick_hint);
+    if (ret)
+        goto out;
 
-                get_transport_type (voliter, set_dict, transt, _gf_false);
-                if (!strcmp (transt, "tcp,rdma"))
-                        strcpy (transt, "tcp");
+    ret = clusters;
 
+out:
+    GF_FREE(brick_hint);
+    brick_hint = NULL;
+    return ret;
+}
 
-                list_for_each_entry (brickinfo, &voliter->bricks, brick_list) {
-                        if (!glusterd_is_local_brick (this, voliter, brickinfo))
-                                continue;
-                        xl  = volgen_graph_build_client (graph, voliter,
-                                                         brickinfo->hostname,
-                                                         brickinfo->path,
-                                                         brickinfo->brick_id, transt,
-                                                         set_dict);
-                        if (!xl) {
-                                ret = -1;
-                                goto out;
-                        }
+static int
+build_bitd_volume_graph(volgen_graph_t *graph, glusterd_volinfo_t *volinfo,
+                        dict_t *mod_dict, unsigned int numbricks)
+{
+    volgen_graph_t cgraph = {0};
+    xlator_t *this = NULL;
+    xlator_t *xl = NULL;
+    dict_t *set_dict = NULL;
+    glusterd_conf_t *priv = NULL;
+    int ret = 0;
+    int clusters = -1;
+    glusterd_brickinfo_t *brickinfo = NULL;
+    int brick_count = 0;
+    char transt[16] = {
+        0,
+    };
+
+    this = THIS;
+    GF_ASSERT(this);
+
+    priv = this->private;
+    GF_ASSERT(priv);
+
+    set_dict = dict_new();
+    if (!set_dict) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_CREATE_FAIL, NULL);
+        ret = -1;
+        goto out;
+    }
 
-                        count++;
-                }
-        }
+    ret = prepare_bitrot_scrub_volume_options(volinfo, mod_dict, set_dict);
+    if (ret)
+        goto out;
 
-        bitd_xl = volgen_graph_add_nolink (graph, br_args[0], br_args[1]);
-        if (!bitd_xl) {
-                ret = -1;
-                goto out;
-        }
+    get_transport_type(volinfo, set_dict, transt, _gf_false);
+    if (!strncmp(transt, "tcp,rdma", SLEN("tcp,rdma")))
+        (void)snprintf(transt, sizeof(transt), "%s", "tcp");
 
-        txl = first_of (graph);
-        for (trav = txl; count; trav = trav->next)
-                count--;
+    cds_list_for_each_entry(brickinfo, &volinfo->bricks, brick_list)
+    {
+        if (!glusterd_is_local_brick(this, volinfo, brickinfo))
+            continue;
 
-        for (; trav != txl; trav = trav->prev) {
-                ret = volgen_xlator_link (bitd_xl, trav);
-                if (ret)
-                        goto out;
+        xl = volgen_graph_build_client(&cgraph, volinfo, brickinfo->hostname,
+                                       NULL, brickinfo->path,
+                                       brickinfo->brick_id, transt, set_dict);
+        if (!xl) {
+            ret = -1;
+            goto out;
         }
+        brick_count++;
+    }
 
+    if (brick_count == 0) {
         ret = 0;
+        goto out;
+    }
 
-out:
-        if (set_dict)
-                dict_unref (set_dict);
+    clusters = build_bitd_clusters(&cgraph, volinfo, set_dict, brick_count,
+                                   numbricks);
+    if (clusters < 0) {
+        ret = -1;
+        goto out;
+    }
 
-        gf_log(this->name, GF_LOG_DEBUG, "Returning %d", ret);
+    ret = volgen_graph_set_options_generic(&cgraph, set_dict, volinfo,
+                                           bitrot_option_handler);
+    if (ret)
+        goto out;
 
-        return ret;
+    ret = volgen_graph_merge_sub(graph, &cgraph, clusters);
+    if (ret)
+        goto out;
+
+    ret = graph_set_generic_options(this, graph, set_dict, "Bitrot");
+
+out:
+    if (set_dict)
+        dict_unref(set_dict);
+
+    return ret;
 }
 
 int
-build_scrub_graph (volgen_graph_t *graph, dict_t *mod_dict)
-{
-        volgen_graph_t        cgraph        = {0};
-        glusterd_volinfo_t   *voliter       = NULL;
-        xlator_t             *this          = NULL;
-        glusterd_conf_t      *priv          = NULL;
-        dict_t               *set_dict      = NULL;
-        int                   ret           = 0;
-        xlator_t             *bitd_xl       = NULL;
-        xlator_t             *xl            = NULL;
-        xlator_t             *trav          = NULL;
-        xlator_t             *txl           = NULL;
-        char                 *skey          = NULL;
-        char                  transt[16]    = {0,};
-        glusterd_brickinfo_t *brickinfo     = NULL;
-        char                 *br_args[]     = {"features/bit-rot",
-                                               "bit-rot"};
-        int32_t               count         = 0;
-
-        this = THIS;
-        GF_ASSERT (this);
-
-        priv = this->private;
-        GF_ASSERT (priv);
-
-        set_dict = dict_new ();
-        if (!set_dict) {
-                ret = -ENOMEM;
-                goto out;
+build_bitd_graph(volgen_graph_t *graph, dict_t *mod_dict)
+{
+    glusterd_volinfo_t *voliter = NULL;
+    xlator_t *this = NULL;
+    glusterd_conf_t *priv = NULL;
+    int ret = 0;
+    xlator_t *iostxl = NULL;
+    glusterd_brickinfo_t *brickinfo = NULL;
+    unsigned int numbricks = 0;
+
+    this = THIS;
+    GF_ASSERT(this);
+    priv = this->private;
+    GF_ASSERT(priv);
+
+    iostxl = volgen_graph_add_as(graph, "debug/io-stats", "bitd");
+    if (!iostxl) {
+        ret = -1;
+        goto out;
+    }
+
+    /* TODO: do way with this extra loop _if possible_ */
+    cds_list_for_each_entry(voliter, &priv->volumes, vol_list)
+    {
+        if (voliter->status != GLUSTERD_STATUS_STARTED)
+            continue;
+        if (!glusterd_is_bitrot_enabled(voliter))
+            continue;
+
+        cds_list_for_each_entry(brickinfo, &voliter->bricks, brick_list)
+        {
+            if (!glusterd_is_local_brick(this, voliter, brickinfo))
+                continue;
+            numbricks++;
         }
+    }
 
-        if (mod_dict)
-                dict_copy (mod_dict, set_dict);
-
-        list_for_each_entry (voliter, &priv->volumes, vol_list) {
-                if (voliter->status != GLUSTERD_STATUS_STARTED)
-                        continue;
-
-                memset (transt, '\0', 16);
-
-                get_transport_type (voliter, set_dict, transt, _gf_false);
-                if (!strcmp (transt, "tcp,rdma"))
-                        strcpy (transt, "tcp");
-
-
-                list_for_each_entry (brickinfo, &voliter->bricks, brick_list) {
-                        if (!glusterd_is_local_brick (this, voliter, brickinfo))
-                                continue;
-                        /*To do: check whether bitd is enable or not if "
-                         * "not then continue;
-                         * Since bitd is a service running within the "
-                         * trusted storage pool, it is treated as a trusted
-                         * client.
-                         */
-                        xl  = volgen_graph_build_client (graph, voliter,
-                                                         brickinfo->hostname,
-                                                         brickinfo->path,
-                                                         brickinfo->brick_id, transt,
-                                                         set_dict);
-                        if (!xl) {
-                                ret = -1;
-                                goto out;
-                        }
-
-                        count++;
-                }
-        }
+    cds_list_for_each_entry(voliter, &priv->volumes, vol_list)
+    {
+        if (voliter->status != GLUSTERD_STATUS_STARTED)
+            continue;
 
-        bitd_xl = volgen_graph_add_nolink (graph, br_args[0], br_args[1]);
-        if (!bitd_xl) {
-                ret = -1;
-                goto out;
-        }
+        if (!glusterd_is_bitrot_enabled(voliter))
+            continue;
 
-        ret = xlator_set_option (bitd_xl, "scrubber", "true");
-        if (ret)
-                goto out;
+        ret = build_bitd_volume_graph(graph, voliter, mod_dict, numbricks);
+    }
+out:
+    return ret;
+}
 
-        txl = first_of (graph);
-        for (trav = txl; count; trav = trav->next)
-                count--;
+static int
+build_scrub_clusters(volgen_graph_t *graph, glusterd_volinfo_t *volinfo,
+                     dict_t *set_dict, int brick_count)
+{
+    int ret = -1;
+    int clusters = 0;
+    xlator_t *xl = NULL;
+    char *scrub_args[] = {"features/bit-rot", "%s-bit-rot-%d"};
 
-        for (; trav != txl; trav = trav->prev) {
-                ret = volgen_xlator_link (bitd_xl, trav);
-                if (ret)
-                        goto out;
-        }
+    ret = volgen_link_bricks_from_list_tail(
+        graph, volinfo, scrub_args[0], scrub_args[1], brick_count, brick_count);
+    clusters = ret;
 
-        ret = 0;
+    xl = first_of(graph);
 
-out:
-        if (set_dict)
-                dict_unref (set_dict);
+    ret = xlator_set_fixed_option(xl, "scrubber", "true");
+    if (ret)
+        goto out;
 
-        gf_log(this->name, GF_LOG_DEBUG, "Returning %d", ret);
+    ret = clusters;
 
-        return ret;
+out:
+    return ret;
 }
 
-int
-glusterd_snapdsvc_create_volfile (glusterd_volinfo_t *volinfo)
-{
-        volgen_graph_t  graph                   = {0,};
-        int             ret                     = -1;
-        char            filename [PATH_MAX]     = {0,};
+static int
+build_scrub_volume_graph(volgen_graph_t *graph, glusterd_volinfo_t *volinfo,
+                         dict_t *mod_dict)
+{
+    volgen_graph_t cgraph = {0};
+    dict_t *set_dict = NULL;
+    xlator_t *this = NULL;
+    xlator_t *xl = NULL;
+    glusterd_conf_t *priv = NULL;
+    int ret = 0;
+    int clusters = -1;
+    int brick_count = 0;
+    char transt[16] = {
+        0,
+    };
+    glusterd_brickinfo_t *brickinfo = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+
+    priv = this->private;
+    GF_ASSERT(priv);
+
+    set_dict = dict_new();
+    if (!set_dict) {
+        ret = -1;
+        goto out;
+    }
 
-        glusterd_svc_build_snapd_volfile (volinfo, filename, PATH_MAX);
+    ret = prepare_bitrot_scrub_volume_options(volinfo, mod_dict, set_dict);
+    if (ret)
+        goto out;
 
-        ret = glusterd_snapdsvc_generate_volfile (&graph, volinfo);
-        if (!ret)
-                ret = volgen_write_volfile (&graph, filename);
+    get_transport_type(volinfo, set_dict, transt, _gf_false);
+    if (!strncmp(transt, "tcp,rdma", SLEN("tcp,rdma")))
+        (void)snprintf(transt, sizeof(transt), "%s", "tcp");
 
-        volgen_graph_free (&graph);
+    cds_list_for_each_entry(brickinfo, &volinfo->bricks, brick_list)
+    {
+        if (!glusterd_is_local_brick(this, volinfo, brickinfo))
+            continue;
 
-        return ret;
-}
+        xl = volgen_graph_build_client(&cgraph, volinfo, brickinfo->hostname,
+                                       NULL, brickinfo->path,
+                                       brickinfo->brick_id, transt, set_dict);
+        if (!xl) {
+            ret = -1;
+            goto out;
+        }
+        brick_count++;
+    }
 
-int
-glusterd_create_rb_volfiles (glusterd_volinfo_t *volinfo,
-                             glusterd_brickinfo_t *brickinfo)
-{
-        int ret = -1;
+    if (brick_count == 0) {
+        ret = 0;
+        goto out;
+    }
+
+    clusters = build_scrub_clusters(&cgraph, volinfo, set_dict, brick_count);
+    if (clusters < 0) {
+        ret = -1;
+        goto out;
+    }
 
-        ret = glusterd_generate_brick_volfile (volinfo, brickinfo);
-        if (!ret)
-                ret = generate_client_volfiles (volinfo, GF_CLIENT_TRUSTED);
-        if (!ret)
-                ret = glusterd_fetchspec_notify (THIS);
+    ret = volgen_graph_set_options_generic(&cgraph, set_dict, volinfo,
+                                           scrubber_option_handler);
+    if (ret)
+        goto out;
 
-        return ret;
+    ret = volgen_graph_merge_sub(graph, &cgraph, clusters);
+    if (ret)
+        goto out;
+
+    ret = graph_set_generic_options(this, graph, set_dict, "Scrubber");
+out:
+    if (set_dict)
+        dict_unref(set_dict);
+
+    return ret;
 }
 
 int
-glusterd_create_volfiles (glusterd_volinfo_t *volinfo)
+build_scrub_graph(volgen_graph_t *graph, dict_t *mod_dict)
 {
-        int        ret  = -1;
-        xlator_t  *this = NULL;
+    glusterd_volinfo_t *voliter = NULL;
+    xlator_t *this = NULL;
+    glusterd_conf_t *priv = NULL;
+    int ret = 0;
+    xlator_t *iostxl = NULL;
 
-        this = THIS;
+    this = THIS;
+    GF_ASSERT(this);
+    priv = this->private;
+    GF_ASSERT(priv);
 
-        ret = generate_brick_volfiles (volinfo);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "Could not generate volfiles for bricks");
-                goto out;
-        }
+    iostxl = volgen_graph_add_as(graph, "debug/io-stats", "scrub");
+    if (!iostxl) {
+        ret = -1;
+        goto out;
+    }
 
-        ret = generate_client_volfiles (volinfo, GF_CLIENT_TRUSTED);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "Could not generate trusted client volfiles");
-                goto out;
-        }
+    cds_list_for_each_entry(voliter, &priv->volumes, vol_list)
+    {
+        if (voliter->status != GLUSTERD_STATUS_STARTED)
+            continue;
 
-        ret = generate_client_volfiles (volinfo, GF_CLIENT_OTHER);
-        if (ret)
-                gf_log (this->name, GF_LOG_ERROR,
-                        "Could not generate client volfiles");
+        if (!glusterd_is_bitrot_enabled(voliter))
+            continue;
 
+        ret = build_scrub_volume_graph(graph, voliter, mod_dict);
+    }
 out:
-        return ret;
+    return ret;
 }
 
 int
-glusterd_create_volfiles_and_notify_services (glusterd_volinfo_t *volinfo)
+glusterd_snapdsvc_create_volfile(glusterd_volinfo_t *volinfo)
 {
-        int        ret  = -1;
-        xlator_t  *this = NULL;
+    volgen_graph_t graph = {
+        0,
+    };
+    int ret = -1;
+    char filename[PATH_MAX] = {
+        0,
+    };
 
-        this = THIS;
+    graph.type = GF_SNAPD;
+    glusterd_svc_build_snapd_volfile(volinfo, filename, PATH_MAX);
 
-        ret = glusterd_create_volfiles (volinfo);
-        if (ret)
-                goto out;
+    ret = glusterd_snapdsvc_generate_volfile(&graph, volinfo);
+    if (!ret)
+        ret = volgen_write_volfile(&graph, filename);
 
-        ret = glusterd_fetchspec_notify (this);
+    volgen_graph_free(&graph);
 
-out:
-        return ret;
+    return ret;
 }
 
 int
-glusterd_create_global_volfile (int (*builder) (volgen_graph_t *graph,
-                                                dict_t *set_dict),
-                                char *filepath, dict_t  *mod_dict)
+glusterd_create_rb_volfiles(glusterd_volinfo_t *volinfo,
+                            glusterd_brickinfo_t *brickinfo)
 {
-        volgen_graph_t graph = {0,};
-        int     ret = -1;
+    int ret = -1;
 
-        ret = builder (&graph, mod_dict);
-        if (!ret)
-                ret = volgen_write_volfile (&graph, filepath);
+    ret = glusterd_generate_brick_volfile(volinfo, brickinfo, NULL, NULL);
+    if (!ret)
+        ret = generate_client_volfiles(volinfo, GF_CLIENT_TRUSTED);
+    if (!ret)
+        ret = glusterd_fetchspec_notify(THIS);
 
-        volgen_graph_free (&graph);
-
-        return ret;
+    return ret;
 }
 
 int
-glusterd_delete_volfile (glusterd_volinfo_t *volinfo,
-                         glusterd_brickinfo_t *brickinfo)
+glusterd_create_volfiles(glusterd_volinfo_t *volinfo)
 {
-        int  ret = 0;
-        char filename[PATH_MAX] = {0,};
+    int ret = -1;
+    xlator_t *this = NULL;
 
-        GF_ASSERT (volinfo);
-        GF_ASSERT (brickinfo);
+    this = THIS;
 
-        get_brick_filepath (filename, volinfo, brickinfo);
-        ret = unlink (filename);
-        if (ret)
-                gf_log ("glusterd", GF_LOG_ERROR, "failed to delete file: %s, "
-                        "reason: %s", filename, strerror (errno));
-        return ret;
+    ret = generate_brick_volfiles(volinfo);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_VOLFILE_CREATE_FAIL,
+               "Could not generate volfiles for bricks");
+        goto out;
+    }
+
+    ret = generate_client_volfiles(volinfo, GF_CLIENT_TRUSTED);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_VOLFILE_CREATE_FAIL,
+               "Could not generate trusted client volfiles");
+        goto out;
+    }
+
+    ret = generate_client_volfiles(volinfo, GF_CLIENT_TRUSTED_PROXY);
+    if (ret) {
+        gf_log(this->name, GF_LOG_ERROR,
+               "Could not generate gfproxy client volfiles");
+        goto out;
+    }
+
+    ret = generate_client_volfiles(volinfo, GF_CLIENT_OTHER);
+    if (ret)
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_VOLFILE_CREATE_FAIL,
+               "Could not generate client volfiles");
+
+    ret = glusterd_generate_gfproxyd_volfile(volinfo);
+    if (ret)
+        gf_log(this->name, GF_LOG_ERROR, "Could not generate gfproxy volfiles");
+
+    ret = glusterd_shdsvc_create_volfile(volinfo);
+    if (ret)
+        gf_log(this->name, GF_LOG_ERROR, "Could not generate shd volfiles");
+
+    dict_del_sizen(volinfo->dict, "skip-CLIOT");
+
+out:
+    return ret;
 }
 
 int
-validate_shdopts (glusterd_volinfo_t *volinfo,
-                  dict_t *val_dict,
-                  char **op_errstr)
+glusterd_create_volfiles_and_notify_services(glusterd_volinfo_t *volinfo)
 {
-        volgen_graph_t graph = {0,};
-        int     ret = -1;
+    int ret = -1;
+    xlator_t *this = NULL;
 
-        graph.errstr = op_errstr;
+    this = THIS;
 
-        if (!glusterd_is_shd_compatible_volume (volinfo)) {
-                ret = 0;
-                goto out;
-        }
-        ret = dict_set_str (val_dict, "graph-check", "on");
-        if (ret)
-                goto out;
-        ret = build_shd_graph (&graph, val_dict);
-        if (!ret)
-                ret = graph_reconf_validateopt (&graph.graph, op_errstr);
+    ret = glusterd_create_volfiles(volinfo);
+    if (ret)
+        goto out;
 
-        volgen_graph_free (&graph);
+    ret = glusterd_fetchspec_notify(this);
 
-        gf_log ("glusterd", GF_LOG_DEBUG, "Returning %d", ret);
 out:
-        dict_del (val_dict, "graph-check");
-        return ret;
+    return ret;
 }
 
 int
-validate_nfsopts (glusterd_volinfo_t *volinfo,
-                    dict_t *val_dict,
-                    char **op_errstr)
+glusterd_create_global_volfile(glusterd_graph_builder_t builder, char *filepath,
+                               dict_t *mod_dict)
 {
-        volgen_graph_t graph = {0,};
-        int     ret = -1;
-        char    transport_type[16] = {0,};
-        char    *tt                = NULL;
-        char    err_str[4096]      = {0,};
-        xlator_t *this             = THIS;
+    volgen_graph_t graph = {
+        0,
+    };
+    int ret = -1;
 
-        GF_ASSERT (this);
+    ret = builder(&graph, mod_dict);
+    if (!ret)
+        ret = volgen_write_volfile(&graph, filepath);
 
-        graph.errstr = op_errstr;
-
-        get_vol_transport_type (volinfo, transport_type);
-        ret = dict_get_str (val_dict, "nfs.transport-type", &tt);
-        if (!ret) {
-                if (volinfo->transport_type != GF_TRANSPORT_BOTH_TCP_RDMA) {
-                        snprintf (err_str, sizeof (err_str), "Changing nfs "
-                                  "transport type is allowed only for volumes "
-                                  "of transport type tcp,rdma");
-                        gf_log (this->name, GF_LOG_ERROR, "%s", err_str);
-                        *op_errstr = gf_strdup (err_str);
-                        ret = -1;
-                        goto out;
-                }
-                if (strcmp (tt,"tcp") && strcmp (tt,"rdma")) {
-                        snprintf (err_str, sizeof (err_str), "wrong transport "
-                                  "type %s", tt);
-                        *op_errstr = gf_strdup (err_str);
-                        ret = -1;
-                        goto out;
-                }
-        }
+    volgen_graph_free(&graph);
 
-        ret = dict_set_str (val_dict, "volume-name", volinfo->volname);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "Failed to set volume name");
-                goto out;
-        }
+    return ret;
+}
 
-        ret = build_nfs_graph (&graph, val_dict);
-        if (!ret)
-                ret = graph_reconf_validateopt (&graph.graph, op_errstr);
+int
+glusterd_delete_volfile(glusterd_volinfo_t *volinfo,
+                        glusterd_brickinfo_t *brickinfo)
+{
+    int ret = 0;
+    char filename[PATH_MAX] = {
+        0,
+    };
 
-        volgen_graph_free (&graph);
+    GF_ASSERT(volinfo);
+    GF_ASSERT(brickinfo);
 
-out:
-        if (dict_get (val_dict, "volume-name"))
-                dict_del (val_dict, "volume-name");
-        gf_log (this->name, GF_LOG_DEBUG, "Returning %d", ret);
-        return ret;
+    get_brick_filepath(filename, volinfo, brickinfo, NULL);
+    ret = sys_unlink(filename);
+    if (ret)
+        gf_msg("glusterd", GF_LOG_ERROR, errno, GD_MSG_FILE_OP_FAILED,
+               "failed to delete file: %s", filename);
+    return ret;
 }
 
-
 int
-validate_clientopts (glusterd_volinfo_t *volinfo,
-                     dict_t *val_dict,
-                     char **op_errstr)
+validate_shdopts(glusterd_volinfo_t *volinfo, dict_t *val_dict,
+                 char **op_errstr)
 {
-        volgen_graph_t graph = {0,};
-        int            ret   = -1;
-
-        GF_ASSERT (volinfo);
+    volgen_graph_t graph = {
+        0,
+    };
+    int ret = -1;
 
-        graph.errstr = op_errstr;
+    graph.errstr = op_errstr;
 
-        ret = build_client_graph (&graph, volinfo, val_dict);
-        if (!ret)
-                ret = graph_reconf_validateopt (&graph.graph, op_errstr);
+    if (!glusterd_is_shd_compatible_volume(volinfo)) {
+        ret = 0;
+        goto out;
+    }
+    ret = dict_set_int32_sizen(val_dict, "graph-check", 1);
+    if (ret) {
+        gf_smsg("glusterd", GF_LOG_ERROR, errno, GD_MSG_DICT_SET_FAILED,
+                "Key=graph-check", NULL);
+        goto out;
+    }
+    ret = build_shd_graph(volinfo, &graph, val_dict);
+    if (!ret)
+        ret = graph_reconf_validateopt(&graph.graph, op_errstr);
+
+    volgen_graph_free(&graph);
+
+    gf_msg_debug("glusterd", 0, "Returning %d", ret);
+out:
+    dict_del_sizen(val_dict, "graph-check");
+    return ret;
+}
 
-        volgen_graph_free (&graph);
+#ifdef BUILD_GNFS
+static int
+validate_nfsopts(glusterd_volinfo_t *volinfo, dict_t *val_dict,
+                 char **op_errstr)
+{
+    volgen_graph_t graph = {
+        0,
+    };
+    int ret = -1;
+    char transport_type[16] = {
+        0,
+    };
+    char *tt = NULL;
+    char err_str[128] = {
+        0,
+    };
+    xlator_t *this = THIS;
+
+    GF_ASSERT(this);
+
+    graph.errstr = op_errstr;
+
+    get_vol_transport_type(volinfo, transport_type);
+    ret = dict_get_str_sizen(val_dict, "nfs.transport-type", &tt);
+    if (!ret) {
+        if (volinfo->transport_type != GF_TRANSPORT_BOTH_TCP_RDMA) {
+            snprintf(err_str, sizeof(err_str),
+                     "Changing nfs "
+                     "transport type is allowed only for volumes "
+                     "of transport type tcp,rdma");
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_OP_UNSUPPORTED, "%s",
+                   err_str);
+            *op_errstr = gf_strdup(err_str);
+            ret = -1;
+            goto out;
+        }
+        if (strcmp(tt, "tcp") && strcmp(tt, "rdma")) {
+            snprintf(err_str, sizeof(err_str),
+                     "wrong transport "
+                     "type %s",
+                     tt);
+            gf_smsg(this->name, GF_LOG_ERROR, 0, GD_MSG_INCOMPATIBLE_VALUE,
+                    "Type=%s", tt, NULL);
+            *op_errstr = gf_strdup(err_str);
+            ret = -1;
+            goto out;
+        }
+    }
+
+    ret = dict_set_str_sizen(val_dict, "volume-name", volinfo->volname);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_SET_FAILED,
+               "Failed to set volume name");
+        goto out;
+    }
+
+    ret = build_nfs_graph(&graph, val_dict);
+    if (!ret)
+        ret = graph_reconf_validateopt(&graph.graph, op_errstr);
+
+    volgen_graph_free(&graph);
 
-        gf_log ("", GF_LOG_DEBUG, "Returning %d", ret);
-        return ret;
+out:
+    if (dict_get_sizen(val_dict, "volume-name"))
+        dict_del_sizen(val_dict, "volume-name");
+    gf_msg_debug(this->name, 0, "Returning %d", ret);
+    return ret;
 }
+#endif
 
 int
-validate_brickopts (glusterd_volinfo_t *volinfo,
-                    glusterd_brickinfo_t *brickinfo,
-                    dict_t *val_dict,
+validate_clientopts(glusterd_volinfo_t *volinfo, dict_t *val_dict,
                     char **op_errstr)
 {
-        volgen_graph_t graph = {0,};
-        int            ret   = -1;
+    volgen_graph_t graph = {
+        0,
+    };
+    int ret = -1;
 
-        GF_ASSERT (volinfo);
+    GF_ASSERT(volinfo);
 
-        graph.errstr = op_errstr;
+    graph.errstr = op_errstr;
 
-        ret = build_server_graph (&graph, volinfo, val_dict, brickinfo);
-        if (!ret)
-                ret = graph_reconf_validateopt (&graph.graph, op_errstr);
+    ret = build_client_graph(&graph, volinfo, val_dict);
+    if (!ret)
+        ret = graph_reconf_validateopt(&graph.graph, op_errstr);
 
-        volgen_graph_free (&graph);
+    volgen_graph_free(&graph);
 
-        gf_log ("", GF_LOG_DEBUG, "Returning %d", ret);
-        return ret;
+    gf_msg_debug("glusterd", 0, "Returning %d", ret);
+    return ret;
 }
 
 int
-glusterd_validate_brickreconf (glusterd_volinfo_t *volinfo,
-                               dict_t *val_dict,
-                               char **op_errstr)
-{
-        glusterd_brickinfo_t *brickinfo = NULL;
-        int                   ret = -1;
+validate_brickopts(glusterd_volinfo_t *volinfo, glusterd_brickinfo_t *brickinfo,
+                   dict_t *mod_dict, void *reconf)
+{
+    volgen_graph_t graph = {
+        0,
+    };
+    int ret = -1;
+    struct gd_validate_reconf_opts *brickreconf = reconf;
+    dict_t *val_dict = brickreconf->options;
+    char **op_errstr = brickreconf->op_errstr;
+    dict_t *full_dict = NULL;
+
+    GF_ASSERT(volinfo);
+
+    graph.errstr = op_errstr;
+    full_dict = dict_new();
+    if (!full_dict) {
+        gf_smsg("glusterd", GF_LOG_ERROR, errno, GD_MSG_DICT_CREATE_FAIL, NULL);
+        ret = -1;
+        goto out;
+    }
 
-        cds_list_for_each_entry (brickinfo, &volinfo->bricks, brick_list) {
-                gf_log ("", GF_LOG_DEBUG,
-                        "Validating %s", brickinfo->hostname);
+    if (mod_dict)
+        dict_copy(mod_dict, full_dict);
 
-                ret = validate_brickopts (volinfo, brickinfo, val_dict,
-                                          op_errstr);
-                if (ret)
-                        goto out;
-        }
+    if (val_dict)
+        dict_copy(val_dict, full_dict);
 
-        ret = 0;
+    ret = build_server_graph(&graph, volinfo, full_dict, brickinfo);
+    if (!ret)
+        ret = graph_reconf_validateopt(&graph.graph, op_errstr);
+
+    volgen_graph_free(&graph);
 
 out:
-        return ret;
+    if (full_dict)
+        dict_unref(full_dict);
+
+    gf_msg_debug("glusterd", 0, "Returning %d", ret);
+    return ret;
 }
 
-static int
-_check_globalopt (dict_t *this, char *key, data_t *value, void *ret_val)
+int
+glusterd_validate_brickreconf(glusterd_volinfo_t *volinfo, dict_t *val_dict,
+                              char **op_errstr)
 {
-        int *ret = NULL;
-
-        ret = ret_val;
-        if (*ret)
-                return 0;
-        if (!glusterd_check_globaloption (key))
-                *ret = 1;
+    int ret = -1;
+    struct gd_validate_reconf_opts brickreconf = {0};
 
-        return 0;
+    brickreconf.options = val_dict;
+    brickreconf.op_errstr = op_errstr;
+    ret = glusterd_volume_brick_for_each(volinfo, &brickreconf,
+                                         validate_brickopts);
+    return ret;
 }
 
-int
-glusterd_validate_globalopts (glusterd_volinfo_t *volinfo,
-                              dict_t *val_dict, char **op_errstr)
+static int
+_check_globalopt(dict_t *this, char *key, data_t *value, void *ret_val)
 {
-        int ret = 0;
-
-        dict_foreach (val_dict, _check_globalopt, &ret);
-        if (ret) {
-                *op_errstr = gf_strdup ( "option specified is not a global option");
-                return -1;
-        }
-        ret = glusterd_validate_brickreconf (volinfo, val_dict, op_errstr);
+    int *ret = NULL;
 
-        if (ret) {
-                gf_log ("", GF_LOG_DEBUG,
-                        "Could not Validate  bricks");
-                goto out;
-        }
+    ret = ret_val;
+    if (*ret)
+        return 0;
+    if (!glusterd_check_globaloption(key))
+        *ret = 1;
 
-        ret = validate_clientopts (volinfo, val_dict, op_errstr);
-        if (ret) {
-                gf_log ("", GF_LOG_DEBUG,
-                        "Could not Validate client");
-                goto out;
-        }
+    return 0;
+}
 
-        ret = validate_nfsopts (volinfo, val_dict, op_errstr);
-        if (ret) {
-                gf_log ("", GF_LOG_DEBUG, "Could not Validate nfs");
-                goto out;
-        }
+int
+glusterd_validate_globalopts(glusterd_volinfo_t *volinfo, dict_t *val_dict,
+                             char **op_errstr)
+{
+    int ret = 0;
 
-        ret = validate_shdopts (volinfo, val_dict, op_errstr);
-        if (ret) {
-                gf_log ("", GF_LOG_DEBUG, "Could not Validate self-heald");
-                goto out;
-        }
+    dict_foreach(val_dict, _check_globalopt, &ret);
+    if (ret) {
+        *op_errstr = gf_strdup("option specified is not a global option");
+        return -1;
+    }
+    ret = glusterd_validate_brickreconf(volinfo, val_dict, op_errstr);
+
+    if (ret) {
+        gf_msg_debug("glusterd", 0, "Could not Validate  bricks");
+        goto out;
+    }
+
+    ret = validate_clientopts(volinfo, val_dict, op_errstr);
+    if (ret) {
+        gf_msg_debug("glusterd", 0, "Could not Validate client");
+        goto out;
+    }
+#ifdef BUILD_GNFS
+    ret = validate_nfsopts(volinfo, val_dict, op_errstr);
+    if (ret) {
+        gf_msg_debug("glusterd", 0, "Could not Validate nfs");
+        goto out;
+    }
+#endif
+    ret = validate_shdopts(volinfo, val_dict, op_errstr);
+    if (ret) {
+        gf_msg_debug("glusterd", 0, "Could not Validate self-heald");
+        goto out;
+    }
 
 out:
-        gf_log ("", GF_LOG_DEBUG, "Returning %d", ret);
-        return ret;
+    gf_msg_debug("glusterd", 0, "Returning %d", ret);
+    return ret;
 }
 
 static int
-_check_localopt (dict_t *this, char *key, data_t *value, void *ret_val)
+_check_localopt(dict_t *this, char *key, data_t *value, void *ret_val)
 {
-        int *ret = NULL;
-
-        ret = ret_val;
-        if (*ret)
-                return 0;
-        if (!glusterd_check_localoption (key))
-                *ret = 1;
+    int *ret = NULL;
 
+    ret = ret_val;
+    if (*ret)
         return 0;
+    if (!glusterd_check_localoption(key))
+        *ret = 1;
+
+    return 0;
 }
 
 int
-glusterd_validate_reconfopts (glusterd_volinfo_t *volinfo, dict_t *val_dict,
-                              char **op_errstr)
+glusterd_validate_reconfopts(glusterd_volinfo_t *volinfo, dict_t *val_dict,
+                             char **op_errstr)
 {
-        int ret = 0;
-
-        dict_foreach (val_dict, _check_localopt, &ret);
-        if (ret) {
-                *op_errstr = gf_strdup ( "option specified is not a local option");
-                return -1;
-        }
-        ret = glusterd_validate_brickreconf (volinfo, val_dict, op_errstr);
-
-        if (ret) {
-                gf_log ("", GF_LOG_DEBUG,
-                        "Could not Validate  bricks");
-                goto out;
-        }
-
-        ret = validate_clientopts (volinfo, val_dict, op_errstr);
-        if (ret) {
-                gf_log ("", GF_LOG_DEBUG,
-                        "Could not Validate client");
-                goto out;
-        }
-
-        ret = validate_nfsopts (volinfo, val_dict, op_errstr);
-        if (ret) {
-                gf_log ("", GF_LOG_DEBUG, "Could not Validate nfs");
-                goto out;
-        }
-
-
-        ret = validate_shdopts (volinfo, val_dict, op_errstr);
-        if (ret) {
-                gf_log ("", GF_LOG_DEBUG, "Could not Validate self-heald");
-                goto out;
-        }
+    int ret = 0;
 
+    dict_foreach(val_dict, _check_localopt, &ret);
+    if (ret) {
+        *op_errstr = gf_strdup("option specified is not a local option");
+        return -1;
+    }
+    ret = glusterd_validate_brickreconf(volinfo, val_dict, op_errstr);
+
+    if (ret) {
+        gf_msg_debug("glusterd", 0, "Could not Validate  bricks");
+        goto out;
+    }
+
+    ret = validate_clientopts(volinfo, val_dict, op_errstr);
+    if (ret) {
+        gf_msg_debug("glusterd", 0, "Could not Validate client");
+        goto out;
+    }
+
+#ifdef BUILD_GNFS
+    ret = validate_nfsopts(volinfo, val_dict, op_errstr);
+    if (ret) {
+        gf_msg_debug("glusterd", 0, "Could not Validate nfs");
+        goto out;
+    }
+#endif
+    ret = validate_shdopts(volinfo, val_dict, op_errstr);
+    if (ret) {
+        gf_msg_debug("glusterd", 0, "Could not Validate self-heald");
+        goto out;
+    }
 
 out:
-        gf_log ("", GF_LOG_DEBUG, "Returning %d", ret);
-        return ret;
+    gf_msg_debug("glusterd", 0, "Returning %d", ret);
+    return ret;
 }
 
-static struct volopt_map_entry *
-_gd_get_vmep (char *key) {
-        char *completion = NULL;
-        struct volopt_map_entry *vmep = NULL;
-        int ret = 0;
-
-        COMPLETE_OPTION ((char *)key, completion, ret);
-        for (vmep = glusterd_volopt_map; vmep->key; vmep++) {
-                if (strcmp (vmep->key, key) == 0)
-                        return vmep;
-        }
+struct volopt_map_entry *
+gd_get_vmep(const char *key)
+{
+    char *completion = NULL;
+    struct volopt_map_entry *vmep = NULL;
+    int ret = 0;
 
+    if (!key)
         return NULL;
+
+    COMPLETE_OPTION((char *)key, completion, ret);
+    for (vmep = glusterd_volopt_map; vmep->key; vmep++) {
+        if (strcmp(vmep->key, key) == 0)
+            return vmep;
+    }
+
+    return NULL;
 }
 
 uint32_t
-glusterd_get_op_version_for_key (char *key)
+glusterd_get_op_version_from_vmep(struct volopt_map_entry *vmep)
 {
-        struct volopt_map_entry *vmep = NULL;
+    if (vmep)
+        return vmep->op_version;
 
-        GF_ASSERT (key);
-
-        vmep = _gd_get_vmep (key);
-        if (vmep)
-                return vmep->op_version;
-
-        return 0;
+    return 0;
 }
 
 gf_boolean_t
-gd_is_client_option (char *key)
+gd_is_client_option(struct volopt_map_entry *vmep)
 {
-        struct volopt_map_entry *vmep = NULL;
+    if (vmep && (vmep->flags & VOLOPT_FLAG_CLIENT_OPT))
+        return _gf_true;
 
-        GF_ASSERT (key);
-
-        vmep = _gd_get_vmep (key);
-        if (vmep && (vmep->flags & OPT_FLAG_CLIENT_OPT))
-                return _gf_true;
-
-        return _gf_false;
+    return _gf_false;
 }
 
 gf_boolean_t
-gd_is_xlator_option (char *key)
+gd_is_xlator_option(struct volopt_map_entry *vmep)
 {
-        struct volopt_map_entry *vmep = NULL;
-
-        GF_ASSERT (key);
+    if (vmep && (vmep->flags & VOLOPT_FLAG_XLATOR_OPT))
+        return _gf_true;
 
-        vmep = _gd_get_vmep (key);
-        if (vmep && (vmep->flags & OPT_FLAG_XLATOR_OPT))
-                return _gf_true;
-
-        return _gf_false;
+    return _gf_false;
 }
 
-volume_option_type_t
-_gd_get_option_type (char *key)
+static volume_option_type_t
+_gd_get_option_type(struct volopt_map_entry *vmep)
 {
-        struct volopt_map_entry *vmep = NULL;
-        void                    *dl_handle = NULL;
-        volume_opt_list_t       vol_opt_list = {{0},};
-        int                     ret = -1;
-        volume_option_t         *opt = NULL;
-        char                    *xlopt_key = NULL;
-        volume_option_type_t    opt_type = GF_OPTION_TYPE_MAX;
-
-        GF_ASSERT (key);
+    void *dl_handle = NULL;
+    volume_opt_list_t vol_opt_list = {
+        {0},
+    };
+    int ret = -1;
+    volume_option_t *opt = NULL;
+    char *xlopt_key = NULL;
+    volume_option_type_t opt_type = GF_OPTION_TYPE_MAX;
 
-        vmep = _gd_get_vmep (key);
+    if (vmep) {
+        CDS_INIT_LIST_HEAD(&vol_opt_list.list);
+        ret = xlator_volopt_dynload(vmep->voltype, &dl_handle, &vol_opt_list);
+        if (ret)
+            goto out;
 
-        if (vmep) {
-                CDS_INIT_LIST_HEAD (&vol_opt_list.list);
-                ret = xlator_volopt_dynload (vmep->voltype, &dl_handle,
-                                             &vol_opt_list);
-                if (ret)
-                        goto out;
+        if (_get_xlator_opt_key_from_vme(vmep, &xlopt_key))
+            goto out;
 
-                if (_get_xlator_opt_key_from_vme (vmep, &xlopt_key))
-                        goto out;
+        opt = xlator_volume_option_get_list(&vol_opt_list, xlopt_key);
+        _free_xlator_opt_key(xlopt_key);
 
-                opt = xlator_volume_option_get_list (&vol_opt_list, xlopt_key);
-                _free_xlator_opt_key (xlopt_key);
-
-                if (opt)
-                        opt_type = opt->type;
-        }
+        if (opt)
+            opt_type = opt->type;
+    }
 
 out:
-        if (dl_handle) {
-                dlclose (dl_handle);
-                dl_handle = NULL;
-        }
+    if (dl_handle) {
+        dlclose(dl_handle);
+        dl_handle = NULL;
+    }
 
-        return opt_type;
+    return opt_type;
 }
 
 gf_boolean_t
-gd_is_boolean_option (char *key)
+gd_is_boolean_option(struct volopt_map_entry *vmep)
 {
-        GF_ASSERT (key);
+    if (GF_OPTION_TYPE_BOOL == _gd_get_option_type(vmep))
+        return _gf_true;
 
-        if (GF_OPTION_TYPE_BOOL == _gd_get_option_type (key))
-                return _gf_true;
+    return _gf_false;
+}
+
+int
+glusterd_shdsvc_generate_volfile(glusterd_volinfo_t *volinfo, char *filename,
+                                 dict_t *mode_dict)
+{
+    int ret = -1;
+    volgen_graph_t graph = {
+        0,
+    };
+
+    graph.type = GF_SHD;
+    ret = build_shd_graph(volinfo, &graph, mode_dict);
+    if (!ret)
+        ret = volgen_write_volfile(&graph, filename);
+
+    volgen_graph_free(&graph);
 
-        return _gf_false;
+    return ret;
 }
diff --git a/xlators/mgmt/glusterd/src/glusterd-volgen.h b/xlators/mgmt/glusterd/src/glusterd-volgen.h
index 02f8df0cf7d..cd4d0c7d0cc 100644
--- a/xlators/mgmt/glusterd/src/glusterd-volgen.h
+++ b/xlators/mgmt/glusterd/src/glusterd-volgen.h
@@ -10,31 +10,37 @@
 #ifndef _GLUSTERD_VOLGEN_H_
 #define _GLUSTERD_VOLGEN_H_
 
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
-
 #if (HAVE_LIB_XML)
 #include <libxml/encoding.h>
 #include <libxml/xmlwriter.h>
 #endif
 
 #include "glusterd.h"
+#include "glusterd-messages.h"
 
 /* volopt map key name definitions */
 
-#define VKEY_DIAG_CNT_FOP_HITS    "diagnostics.count-fop-hits"
+#define VKEY_DIAG_CNT_FOP_HITS "diagnostics.count-fop-hits"
 #define VKEY_DIAG_LAT_MEASUREMENT "diagnostics.latency-measurement"
 #define VKEY_FEATURES_LIMIT_USAGE "features.limit-usage"
-#define VKEY_FEATURES_SOFT_LIMIT  "features.soft-limit"
-#define VKEY_MARKER_XTIME         GEOREP".indexing"
-#define VKEY_MARKER_XTIME_FORCE   GEOREP".ignore-pid-check"
-#define VKEY_CHANGELOG            "changelog.changelog"
-#define VKEY_FEATURES_QUOTA       "features.quota"
-#define VKEY_FEATURES_TRASH       "features.trash"
-#define VKEY_FEATURES_BITROT      "features.bitrot"
-#define VKEY_FEATURES_SCRUB       "features.scrub"
+#define VKEY_FEATURES_SOFT_LIMIT "features.soft-limit"
+#define VKEY_MARKER_XTIME GEOREP ".indexing"
+#define VKEY_MARKER_XTIME_FORCE GEOREP ".ignore-pid-check"
+#define VKEY_CHANGELOG "changelog.changelog"
+#define VKEY_FEATURES_QUOTA "features.quota"
+#define VKEY_FEATURES_INODE_QUOTA "features.inode-quota"
+#define VKEY_FEATURES_TRASH "features.trash"
+#define VKEY_FEATURES_BITROT "features.bitrot"
+#define VKEY_FEATURES_SCRUB "features.scrub"
+#define VKEY_FEATURES_SELINUX "features.selinux"
+#define VKEY_PARALLEL_READDIR "performance.parallel-readdir"
+#define VKEY_READDIR_AHEAD "performance.readdir-ahead"
+#define VKEY_RDA_CACHE_LIMIT "performance.rda-cache-limit"
+#define VKEY_RDA_REQUEST_SIZE "performance.rda-request-size"
+#define VKEY_CONFIG_GFPROXY "config.gfproxyd"
+#define VKEY_CONFIG_GLOBAL_THREADING "config.global-threading"
+#define VKEY_CONFIG_CLIENT_THREADS "config.client-threads"
+#define VKEY_CONFIG_BRICK_THREADS "config.brick-threads"
 
 #define AUTH_ALLOW_MAP_KEY "auth.allow"
 #define AUTH_REJECT_MAP_KEY "auth.reject"
@@ -43,60 +49,85 @@
 #define AUTH_REJECT_OPT_KEY "auth.addr.*.reject"
 #define NFS_DISABLE_OPT_KEY "nfs.*.disable"
 
-#define SSL_CERT_DEPTH_OPT  "ssl.certificate-depth"
+#define SSL_OWN_CERT_OPT "ssl.own-cert"
+#define SSL_PRIVATE_KEY_OPT "ssl.private-key"
+#define SSL_CA_LIST_OPT "ssl.ca-list"
+#define SSL_CRL_PATH_OPT "ssl.crl-path"
+#define SSL_CERT_DEPTH_OPT "ssl.certificate-depth"
 #define SSL_CIPHER_LIST_OPT "ssl.cipher-list"
-
+#define SSL_DH_PARAM_OPT "ssl.dh-param"
+#define SSL_EC_CURVE_OPT "ssl.ec-curve"
 
 typedef enum {
-        GF_CLIENT_TRUSTED,
-        GF_CLIENT_OTHER
+    GF_CLIENT_TRUSTED,
+    GF_CLIENT_OTHER,
+    GF_CLIENT_TRUSTED_PROXY,
 } glusterd_client_type_t;
 
+/* It indicates the type of volfile that the graph is built for */
+typedef enum {
+    GF_REBALANCED = 1,
+    GF_QUOTAD,
+    GF_SNAPD,
+    GF_SHD,
+} glusterd_graph_type_t;
+
 struct volgen_graph {
-        char **errstr;
-        glusterfs_graph_t graph;
+    char **errstr;
+    glusterfs_graph_t graph;
+    glusterd_graph_type_t type;
 };
 typedef struct volgen_graph volgen_graph_t;
 
-#define COMPLETE_OPTION(key, completion, ret)                           \
-        do {                                                            \
-                if (!strchr (key, '.')) {                               \
-                        ret = option_complete (key, &completion);       \
-                        if (ret) {                                      \
-                                gf_log ("", GF_LOG_ERROR, "Out of memory"); \
-                                return _gf_false;                       \
-                        }                                               \
-                                                                        \
-                        if (!completion) {                              \
-                                gf_log ("", GF_LOG_ERROR, "option %s does not" \
-                                        "exist", key);                  \
-                                return _gf_false;                       \
-                        }                                               \
-                }                                                       \
-                                                                        \
-                if (completion)                                         \
-                        GF_FREE (completion);                           \
-        } while (0);
+typedef int (*glusterd_graph_builder_t)(volgen_graph_t *graph,
+                                        dict_t *mod_dict);
+typedef int (*glusterd_vol_graph_builder_t)(glusterd_volinfo_t *,
+                                            char *filename, dict_t *mod_dict);
+
+#define COMPLETE_OPTION(key, completion, ret)                                  \
+    do {                                                                       \
+        if (!strchr(key, '.')) {                                               \
+            ret = option_complete(key, &completion);                           \
+            if (ret) {                                                         \
+                gf_msg("", GF_LOG_ERROR, ENOMEM, GD_MSG_NO_MEMORY,             \
+                       "Out of memory");                                       \
+                return _gf_false;                                              \
+            }                                                                  \
+                                                                               \
+            if (!completion) {                                                 \
+                gf_msg("", GF_LOG_ERROR, 0, GD_MSG_INVALID_ENTRY,              \
+                       "option %s does not"                                    \
+                       "exist",                                                \
+                       key);                                                   \
+                return _gf_false;                                              \
+            }                                                                  \
+        }                                                                      \
+                                                                               \
+        if (completion)                                                        \
+            GF_FREE(completion);                                               \
+    } while (0);
 
 typedef enum gd_volopt_flags_ {
-        OPT_FLAG_NONE,
-        OPT_FLAG_FORCE = 0x01,      // option needs force to be reset
-        OPT_FLAG_XLATOR_OPT = 0x02, // option enables/disables xlators
-        OPT_FLAG_CLIENT_OPT = 0x04, // option affects clients
+    VOLOPT_FLAG_NONE,
+    VOLOPT_FLAG_FORCE = 0x01,       /* option needs force to be reset */
+    VOLOPT_FLAG_XLATOR_OPT = 0x02,  /* option enables/disables xlators */
+    VOLOPT_FLAG_CLIENT_OPT = 0x04,  /* option affects clients */
+    VOLOPT_FLAG_NEVER_RESET = 0x08, /* option which should not be reset */
 } gd_volopt_flags_t;
 
 typedef enum {
-        GF_XLATOR_POSIX = 0,
-        GF_XLATOR_ACL,
-        GF_XLATOR_LOCKS,
-        GF_XLATOR_UPCALL,
-        GF_XLATOR_IOT,
-        GF_XLATOR_INDEX,
-        GF_XLATOR_MARKER,
-        GF_XLATOR_IO_STATS,
-        GF_XLATOR_BD,
-        GF_XLATOR_SERVER,
-        GF_XLATOR_NONE,
+    GF_XLATOR_POSIX = 0,
+    GF_XLATOR_ACL,
+    GF_XLATOR_LOCKS,
+    GF_XLATOR_LEASES,
+    GF_XLATOR_UPCALL,
+    GF_XLATOR_IOT,
+    GF_XLATOR_INDEX,
+    GF_XLATOR_MARKER,
+    GF_XLATOR_IO_STATS,
+    GF_XLATOR_BD,
+    GF_XLATOR_SERVER,
+    GF_XLATOR_NONE,
 } glusterd_server_xlator_t;
 
 /* As of now debug xlators can be loaded only below fuse in the client
@@ -105,180 +136,203 @@ typedef enum {
  * available.
  */
 typedef enum {
-        GF_CLNT_XLATOR_FUSE = 0,
-        GF_CLNT_XLATOR_NONE,
+    GF_CLNT_XLATOR_FUSE = 0,
+    GF_CLNT_XLATOR_NONE,
 } glusterd_client_xlator_t;
 
-typedef enum  { DOC, NO_DOC, GLOBAL_DOC, GLOBAL_NO_DOC } option_type_t;
+typedef enum { DOC, NO_DOC, GLOBAL_DOC, GLOBAL_NO_DOC } option_type_t;
 
-typedef int (*vme_option_validation) (glusterd_volinfo_t *volinfo, dict_t *dict,
-                                      char *key, char *value, char **op_errstr);
+typedef int (*vme_option_validation)(glusterd_volinfo_t *volinfo, dict_t *dict,
+                                     char *key, char *value, char **op_errstr);
 
 struct volopt_map_entry {
-        char *key;
-        char *voltype;
-        char *option;
-        char *value;
-        option_type_t type;
-        uint32_t flags;
-        uint32_t op_version;
-        char *description;
-        vme_option_validation validate_fn;
-        /* If client_option is true, the option affects clients.
-         * this is used to calculate client-op-version of volumes
-         */
-        //gf_boolean_t client_option;
+    char *key;
+    char *voltype;
+    char *option;
+    char *value;
+    option_type_t type;
+    uint32_t flags;
+    uint32_t op_version;
+    char *description;
+    vme_option_validation validate_fn;
+    /* If client_option is true, the option affects clients.
+     * this is used to calculate client-op-version of volumes
+     */
+    // gf_boolean_t client_option;
 };
 
-typedef
-int (*brick_xlator_builder) (volgen_graph_t *graph,
-                             glusterd_volinfo_t *volinfo, dict_t *set_dict,
-                             glusterd_brickinfo_t *brickinfo);
+typedef int (*brick_xlator_builder)(volgen_graph_t *graph,
+                                    glusterd_volinfo_t *volinfo,
+                                    dict_t *set_dict,
+                                    glusterd_brickinfo_t *brickinfo);
 
 struct volgen_brick_xlator {
-        /* function that builds a xlator */
-        brick_xlator_builder builder;
-        /* debug key for a xlator that
-         * gets used for adding debug translators like trace, error-gen
-         * before this xlator */
-        char *dbg_key;
+    /* function that builds a xlator */
+    brick_xlator_builder builder;
+    /* debug key for a xlator that
+     * gets used for adding debug translators like trace, error-gen,
+     * delay-gen before this xlator */
+    char *dbg_key;
+};
+
+struct nfs_opt {
+    const char *pattern;
+    const char *printf_pattern;
 };
+
 typedef struct volgen_brick_xlator volgen_brick_xlator_t;
 
 int
-glusterd_snapdsvc_create_volfile (glusterd_volinfo_t *volinfo);
+glusterd_snapdsvc_create_volfile(glusterd_volinfo_t *volinfo);
+
+int
+glusterd_snapdsvc_generate_volfile(volgen_graph_t *graph,
+                                   glusterd_volinfo_t *volinfo);
 
 int
-glusterd_snapdsvc_generate_volfile (volgen_graph_t *graph,
-                                    glusterd_volinfo_t *volinfo);
+glusterd_create_global_volfile(glusterd_graph_builder_t builder, char *filepath,
+                               dict_t *mod_dict);
 
 int
-glusterd_create_global_volfile (int (*builder) (volgen_graph_t *graph,
-                                                dict_t *set_dict),
-                                char *filepath, dict_t  *mod_dict);
+glusterd_create_rb_volfiles(glusterd_volinfo_t *volinfo,
+                            glusterd_brickinfo_t *brickinfo);
 
 int
-glusterd_create_rb_volfiles (glusterd_volinfo_t *volinfo,
-                                 glusterd_brickinfo_t *brickinfo);
+glusterd_create_volfiles(glusterd_volinfo_t *volinfo);
 
 int
-glusterd_create_volfiles (glusterd_volinfo_t *volinfo);
+glusterd_create_volfiles_and_notify_services(glusterd_volinfo_t *volinfo);
 
 int
-glusterd_create_volfiles_and_notify_services (glusterd_volinfo_t *volinfo);
+glusterd_generate_client_per_brick_volfile(glusterd_volinfo_t *volinfo);
 
 void
-glusterd_get_nfs_filepath (char *filename);
+glusterd_get_nfs_filepath(char *filename);
 
 void
-glusterd_get_shd_filepath (char *filename);
+glusterd_get_shd_filepath(char *filename);
 
 int
-build_shd_graph (volgen_graph_t *graph, dict_t *mod_dict);
+build_shd_graph(glusterd_volinfo_t *volinfo, volgen_graph_t *graph,
+                dict_t *mod_dict);
 
+#ifdef BUILD_GNFS
 int
-build_nfs_graph (volgen_graph_t *graph, dict_t *mod_dict);
+build_nfs_graph(volgen_graph_t *graph, dict_t *mod_dict);
+#endif
+int
+build_quotad_graph(volgen_graph_t *graph, dict_t *mod_dict);
 
 int
-build_quotad_graph (volgen_graph_t *graph, dict_t *mod_dict);
+build_rebalance_volfile(glusterd_volinfo_t *volinfo, char *filepath,
+                        dict_t *mod_dict);
 
 int
-build_bitd_graph (volgen_graph_t *graph, dict_t *mod_dict);
+build_bitd_graph(volgen_graph_t *graph, dict_t *mod_dict);
 
 int
-build_scrub_graph (volgen_graph_t *graph, dict_t *mod_dict);
+build_scrub_graph(volgen_graph_t *graph, dict_t *mod_dict);
 
 int
-glusterd_delete_volfile (glusterd_volinfo_t *volinfo,
-                             glusterd_brickinfo_t *brickinfo);
+glusterd_delete_volfile(glusterd_volinfo_t *volinfo,
+                        glusterd_brickinfo_t *brickinfo);
 int
-glusterd_delete_snap_volfile (glusterd_volinfo_t *volinfo,
-                              glusterd_volinfo_t *snap_volinfo,
-                              glusterd_brickinfo_t *brickinfo);
+glusterd_delete_snap_volfile(glusterd_volinfo_t *volinfo,
+                             glusterd_volinfo_t *snap_volinfo,
+                             glusterd_brickinfo_t *brickinfo);
 
 int
-glusterd_volinfo_get (glusterd_volinfo_t *volinfo, char *key, char **value);
+glusterd_volinfo_get(glusterd_volinfo_t *volinfo, char *key, char **value);
 
 int
-glusterd_volinfo_get_boolean (glusterd_volinfo_t *volinfo, char *key);
+glusterd_volinfo_get_boolean(glusterd_volinfo_t *volinfo, char *key);
 
 int
-glusterd_validate_globalopts (glusterd_volinfo_t *volinfo, dict_t *val_dict,
-                              char **op_errstr);
+glusterd_validate_globalopts(glusterd_volinfo_t *volinfo, dict_t *val_dict,
+                             char **op_errstr);
 
 int
-glusterd_validate_localopts (dict_t *val_dict, char **op_errstr);
+glusterd_validate_localopts(dict_t *val_dict, char **op_errstr);
 
 gf_boolean_t
-glusterd_check_globaloption (char *key);
+glusterd_check_globaloption(char *key);
 
 gf_boolean_t
-glusterd_check_voloption_flags (char *key, int32_t flags);
+glusterd_check_voloption_flags(char *key, int32_t flags);
 
 gf_boolean_t
-glusterd_is_valid_volfpath (char *volname, char *brick);
+glusterd_is_valid_volfpath(char *volname, char *brick);
 
 int
-generate_brick_volfiles (glusterd_volinfo_t *volinfo);
+generate_brick_volfiles(glusterd_volinfo_t *volinfo);
 
 int
-generate_snap_brick_volfiles (glusterd_volinfo_t *volinfo,
-                                  glusterd_volinfo_t *snap_volinfo);
+generate_snap_brick_volfiles(glusterd_volinfo_t *volinfo,
+                             glusterd_volinfo_t *snap_volinfo);
 int
-generate_client_volfiles (glusterd_volinfo_t *volinfo,
-                              glusterd_client_type_t client_type);
+generate_client_volfiles(glusterd_volinfo_t *volinfo,
+                         glusterd_client_type_t client_type);
 int
-generate_snap_client_volfiles (glusterd_volinfo_t *actual_volinfo,
-                               glusterd_volinfo_t *snap_volinfo,
-                               glusterd_client_type_t client_type,
-                               gf_boolean_t vol_restore);
+generate_snap_client_volfiles(glusterd_volinfo_t *actual_volinfo,
+                              glusterd_volinfo_t *snap_volinfo,
+                              glusterd_client_type_t client_type,
+                              gf_boolean_t vol_restore);
 
 int
-_get_xlator_opt_key_from_vme ( struct volopt_map_entry *vme, char **key);
+_get_xlator_opt_key_from_vme(struct volopt_map_entry *vme, char **key);
 
 void
-_free_xlator_opt_key (char *key);
-
+_free_xlator_opt_key(char *key);
 
 #if (HAVE_LIB_XML)
 int
-init_sethelp_xml_doc (xmlTextWriterPtr *writer, xmlBufferPtr  *buf);
+init_sethelp_xml_doc(xmlTextWriterPtr *writer, xmlBufferPtr *buf);
 
 int
-xml_add_volset_element (xmlTextWriterPtr writer, const char *name,
-                        const char *def_val, const char *dscrpt);
+xml_add_volset_element(xmlTextWriterPtr writer, const char *name,
+                       const char *def_val, const char *dscrpt);
 int
-end_sethelp_xml_doc (xmlTextWriterPtr writer);
+end_sethelp_xml_doc(xmlTextWriterPtr writer);
 #endif /* HAVE_LIB_XML */
 
-char*
-glusterd_get_trans_type_rb (gf_transport_type ttype);
+char *
+glusterd_get_trans_type_rb(gf_transport_type ttype);
 
-int
-glusterd_check_nfs_volfile_identical (gf_boolean_t *identical);
-
-int
-glusterd_check_nfs_topology_identical (gf_boolean_t *identical);
+struct volopt_map_entry *
+gd_get_vmep(const char *key);
 
 uint32_t
-glusterd_get_op_version_for_key (char *key);
+glusterd_get_op_version_from_vmep(struct volopt_map_entry *vmep);
 
 gf_boolean_t
-gd_is_client_option (char *key);
+gd_is_client_option(struct volopt_map_entry *vmep);
 
 gf_boolean_t
-gd_is_xlator_option (char *key);
+gd_is_xlator_option(struct volopt_map_entry *vmep);
 
 gf_boolean_t
-gd_is_boolean_option (char *key);
+gd_is_boolean_option(struct volopt_map_entry *vmep);
 
+char *
+volgen_get_shd_key(int type);
 
-char*
-volgen_get_shd_key (glusterd_volinfo_t *volinfo);
+int
+glusterd_volopt_validate(glusterd_volinfo_t *volinfo, dict_t *dict, char *key,
+                         char *value, char **op_errstr);
+gf_boolean_t
+gd_is_self_heal_enabled(glusterd_volinfo_t *volinfo, dict_t *dict);
+
+int
+generate_dummy_client_volfiles(glusterd_volinfo_t *volinfo);
 
 int
-glusterd_volopt_validate (glusterd_volinfo_t *volinfo, dict_t *dict, char *key,
-                          char *value, char **op_errstr);
+glusterd_generate_gfproxyd_volfile(glusterd_volinfo_t *volinfo);
 
+int
+glusterd_build_gfproxyd_volfile(glusterd_volinfo_t *volinfo, char *filename);
+
+int
+glusterd_shdsvc_generate_volfile(glusterd_volinfo_t *volinfo, char *filename,
+                                 dict_t *mode_dict);
 
 #endif
diff --git a/xlators/mgmt/glusterd/src/glusterd-volume-ops.c b/xlators/mgmt/glusterd/src/glusterd-volume-ops.c
index a42f08c1600..814ab14fb27 100644
--- a/xlators/mgmt/glusterd/src/glusterd-volume-ops.c
+++ b/xlators/mgmt/glusterd/src/glusterd-volume-ops.c
@@ -7,17 +7,8 @@
    later), or the GNU General Public License, version 2 (GPLv2), in all
    cases as published by the Free Software Foundation.
 */
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
-
-#ifdef HAVE_BD_XLATOR
-#include <lvm2app.h>
-#endif
-
-#include "common-utils.h"
-#include "syscall.h"
+#include <glusterfs/common-utils.h>
+#include <glusterfs/syscall.h>
 #include "cli1-xdr.h"
 #include "xdr-generic.h"
 #include "glusterd.h"
@@ -27,12 +18,14 @@
 #include "glusterd-utils.h"
 #include "glusterd-volgen.h"
 #include "glusterd-messages.h"
-#include "run.h"
+#include <glusterfs/run.h>
 #include "glusterd-snapshot-utils.h"
 #include "glusterd-svc-mgmt.h"
 #include "glusterd-svc-helper.h"
 #include "glusterd-shd-svc.h"
 #include "glusterd-snapd-svc.h"
+#include "glusterd-mgmt.h"
+#include "glusterd-server-quorum.h"
 
 #include <stdint.h>
 #include <sys/socket.h>
@@ -41,2838 +34,3000 @@
 #include <netinet/in.h>
 #include <stdlib.h>
 
-#define glusterd_op_start_volume_args_get(dict, volname, flags) \
-        glusterd_op_stop_volume_args_get (dict, volname, flags)
+#define glusterd_op_start_volume_args_get(dict, volname, flags)                \
+    glusterd_op_stop_volume_args_get(dict, volname, flags)
 
-gf_ai_compare_t
-glusterd_compare_addrinfo (struct addrinfo *first, struct addrinfo *next)
-{
-        int             ret = -1;
-        struct addrinfo *tmp1 = NULL;
-        struct addrinfo *tmp2 = NULL;
-        char            firstip[NI_MAXHOST] = {0.};
-        char            nextip[NI_MAXHOST] = {0,};
-
-        for (tmp1 = first; tmp1 != NULL; tmp1 = tmp1->ai_next) {
-                ret = getnameinfo (tmp1->ai_addr, tmp1->ai_addrlen, firstip,
-                                   NI_MAXHOST, NULL, 0, NI_NUMERICHOST);
-                if (ret)
-                        return GF_AI_COMPARE_ERROR;
-                for (tmp2 = next; tmp2 != NULL; tmp2 = tmp2->ai_next) {
-                      ret = getnameinfo (tmp2->ai_addr, tmp2->ai_addrlen,
-                                         nextip, NI_MAXHOST, NULL, 0,
-                                         NI_NUMERICHOST);
-                      if (ret)
-                              return GF_AI_COMPARE_ERROR;
-                      if (!strcmp (firstip, nextip)) {
-                              return GF_AI_COMPARE_MATCH;
-                      }
-                }
-        }
-        return GF_AI_COMPARE_NO_MATCH;
-}
-
-/* Check for non optimal brick order for replicate :
- * Checks if bricks belonging to a replicate volume
- * are present on the same server
- */
-int32_t
-glusterd_check_brick_order(dict_t *dict, char *err_str)
+int
+__glusterd_handle_create_volume(rpcsvc_request_t *req)
 {
-        int             ret             = -1;
-        int             i               = 0;
-        int             j               = 0;
-        int             k               = 0;
-        xlator_t        *this           = NULL;
-        addrinfo_list_t *ai_list        = NULL;
-        addrinfo_list_t *ai_list_tmp1   = NULL;
-        addrinfo_list_t *ai_list_tmp2   = NULL;
-        char            *brick          = NULL;
-        char            *brick_list     = NULL;
-        char            *brick_list_dup = NULL;
-        char            *brick_list_ptr = NULL;
-        char            *tmpptr         = NULL;
-        char            *volname        = NULL;
-        int32_t         brick_count     = 0;
-        int32_t         type            = GF_CLUSTER_TYPE_NONE;
-        int32_t         sub_count       = 0;
-        struct addrinfo *ai_info        = NULL;
-
-        const char      failed_string[2048] = "Failed to perform brick order "
-                                "check. Use 'force' at the end of the command"
-                                " if you want to override this behavior. ";
-        const char      found_string[2048]  = "Multiple bricks of a replicate "
-                                "volume are present on the same server. This "
-                                "setup is not optimal. Use 'force' at the "
-                                "end of the command if you want to override "
-                                "this behavior. ";
-
-        this = THIS;
-
-        GF_ASSERT(this);
-
-        ai_list = malloc (sizeof (addrinfo_list_t));
-        ai_list->info = NULL;
-        CDS_INIT_LIST_HEAD (&ai_list->list);
-
-        ret = dict_get_str (dict, "volname", &volname);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "Unable to get volume name");
-                goto out;
-        }
-
-        ret = dict_get_int32 (dict, "type", &type);
-        if (ret) {
-                snprintf (err_str, 512, "Unable to get type of volume %s",
-                          volname);
-                gf_log (this->name, GF_LOG_WARNING, "%s", err_str);
-                goto out;
-        }
-
-        ret = dict_get_str (dict, "bricks", &brick_list);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "Bricks check : Could not "
-                        "retrieve bricks list");
-                goto out;
-        }
-
-        ret = dict_get_int32 (dict, "count", &brick_count);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "Bricks check : Could not "
-                        "retrieve brick count");
-                goto out;
-        }
+    int32_t ret = -1;
+    gf_cli_req cli_req = {{
+        0,
+    }};
+    dict_t *dict = NULL;
+    char *bricks = NULL;
+    char *volname = NULL;
+    int brick_count = 0;
+    int thin_arbiter_count = 0;
+    void *cli_rsp = NULL;
+    char err_str[2048] = {
+        0,
+    };
+    gf_cli_rsp rsp = {
+        0,
+    };
+    xlator_t *this = NULL;
+    glusterd_conf_t *conf = NULL;
+    char *free_ptr = NULL;
+    char *trans_type = NULL;
+    char *address_family_str = NULL;
+    uuid_t volume_id = {
+        0,
+    };
+    uuid_t tmp_uuid = {0};
+    int32_t type = 0;
+    char *username = NULL;
+    char *password = NULL;
+#ifdef IPV6_DEFAULT
+    char *addr_family = "inet6";
+#else
+    char *addr_family = "inet";
+#endif
+    glusterd_volinfo_t *volinfo = NULL;
 
-        if (type != GF_CLUSTER_TYPE_DISPERSE) {
-                ret = dict_get_int32 (dict, "replica-count", &sub_count);
-                if (ret) {
-                        gf_log (this->name, GF_LOG_ERROR, "Bricks check : Could"
-                                " not retrieve replica count");
-                        goto out;
-                }
-                gf_log (this->name, GF_LOG_DEBUG, "Replicate cluster type "
-                        "found. Checking brick order.");
-        } else {
-                ret = dict_get_int32 (dict, "disperse-count", &sub_count);
-                if (ret) {
-                        gf_log (this->name, GF_LOG_ERROR, "Bricks check : Could"
-                                " not retrieve disperse count");
-                        goto out;
-                }
-                gf_log (this->name, GF_LOG_INFO, "Disperse cluster type"
-                        " found. Checking brick order.");
-        }
+    GF_ASSERT(req);
 
-        brick_list_dup = brick_list_ptr = gf_strdup(brick_list);
-        /* Resolve hostnames and get addrinfo */
-        while (i < brick_count) {
-                ++i;
-                brick = strtok_r (brick_list_dup, " \n", &tmpptr);
-                brick_list_dup = tmpptr;
-                if (brick == NULL)
-                        goto check_failed;
-                brick = strtok_r (brick, ":", &tmpptr);
-                if (brick == NULL)
-                        goto check_failed;
-                ret = getaddrinfo (brick, NULL, NULL, &ai_info);
-                if (ret != 0) {
-                        ret = 0;
-                        gf_log (this->name, GF_LOG_ERROR, "unable to resolve "
-                                "host name");
-                        goto out;
-                }
-                ai_list_tmp1 = malloc (sizeof (addrinfo_list_t));
-                if (ai_list_tmp1 == NULL) {
-                        ret = 0;
-                        gf_log (this->name, GF_LOG_ERROR, "failed to allocate "
-                                "memory");
-                        goto out;
-                }
-                ai_list_tmp1->info = ai_info;
-                cds_list_add_tail (&ai_list_tmp1->list, &ai_list->list);
-                ai_list_tmp1 = NULL;
-        }
+    this = THIS;
+    GF_ASSERT(this);
 
-        i = 0;
-        ai_list_tmp1 = cds_list_entry (ai_list->list.next,
-                                       addrinfo_list_t, list);
-
-        /* Check for bad brick order */
-        while (i < brick_count) {
-                ++i;
-                ai_info = ai_list_tmp1->info;
-                ai_list_tmp1 = cds_list_entry (ai_list_tmp1->list.next,
-                                               addrinfo_list_t, list);
-                if (0 == i % sub_count) {
-                        j = 0;
-                        continue;
-                }
-                ai_list_tmp2 = ai_list_tmp1;
-                k = j;
-                while (k < sub_count - 1) {
-                        ++k;
-                        ret = glusterd_compare_addrinfo (ai_info,
-                                                         ai_list_tmp2->info);
-                        if (GF_AI_COMPARE_ERROR == ret)
-                                goto check_failed;
-                        if (GF_AI_COMPARE_MATCH == ret)
-                                goto found_bad_brick_order;
-                        ai_list_tmp2 = cds_list_entry (ai_list_tmp2->list.next,
-                                                       addrinfo_list_t, list);
-                }
-                ++j;
-        }
-        gf_log (this->name, GF_LOG_DEBUG, "Brick order okay");
-        ret = 0;
-        goto out;
+    conf = this->private;
+    GF_VALIDATE_OR_GOTO(this->name, conf, out);
 
-check_failed:
-        gf_log (this->name, GF_LOG_ERROR, "Failed bad brick order check");
-        snprintf (err_str, sizeof (failed_string), failed_string);
-        ret = -1;
+    ret = -1;
+    ret = xdr_to_generic(req->msg[0], &cli_req, (xdrproc_t)xdr_gf_cli_req);
+    if (ret < 0) {
+        req->rpc_err = GARBAGE_ARGS;
+        snprintf(err_str, sizeof(err_str),
+                 "Failed to decode request "
+                 "received from cli");
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_REQ_DECODE_FAIL, "%s",
+               err_str);
         goto out;
+    }
 
-found_bad_brick_order:
-        gf_log (this->name, GF_LOG_INFO, "Bad brick order found");
-        snprintf (err_str, sizeof (found_string), found_string);
-        ret = -1;
-out:
-        ai_list_tmp2 = NULL;
-        GF_FREE (brick_list_ptr);
-        cds_list_for_each_entry (ai_list_tmp1, &ai_list->list, list) {
-                if (ai_list_tmp1->info)
-                          freeaddrinfo (ai_list_tmp1->info);
-                free (ai_list_tmp2);
-                ai_list_tmp2 = ai_list_tmp1;
-        }
-        free (ai_list_tmp2);
-        return ret;
-}
+    gf_msg_debug(this->name, 0, "Received create volume req");
 
-int
-__glusterd_handle_create_volume (rpcsvc_request_t *req)
-{
-        int32_t                 ret         = -1;
-        gf_cli_req              cli_req     = {{0,}};
-        dict_t                 *dict        = NULL;
-        char                   *bricks      = NULL;
-        char                   *volname     = NULL;
-        int                    brick_count  = 0;
-        void                   *cli_rsp     = NULL;
-        char                    err_str[2048] = {0,};
-        gf_cli_rsp              rsp         = {0,};
-        xlator_t               *this        = NULL;
-        char                   *free_ptr    = NULL;
-        char                   *trans_type  = NULL;
-        uuid_t                  volume_id   = {0,};
-        uuid_t                  tmp_uuid    = {0};
-        int32_t                 type        = 0;
-        char                   *username    = NULL;
-        char                   *password    = NULL;
-
-        GF_ASSERT (req);
-
-        this = THIS;
-        GF_ASSERT(this);
+    if (cli_req.dict.dict_len) {
+        /* Unserialize the dictionary */
+        dict = dict_new();
 
-        ret = -1;
-        ret = xdr_to_generic (req->msg[0], &cli_req, (xdrproc_t)xdr_gf_cli_req);
+        ret = dict_unserialize(cli_req.dict.dict_val, cli_req.dict.dict_len,
+                               &dict);
         if (ret < 0) {
-                req->rpc_err = GARBAGE_ARGS;
-                snprintf (err_str, sizeof (err_str), "Failed to decode request "
-                          "received from cli");
-                gf_log (this->name, GF_LOG_ERROR, "%s", err_str);
-                goto out;
-        }
-
-        gf_log (this->name, GF_LOG_DEBUG, "Received create volume req");
-
-        if (cli_req.dict.dict_len) {
-                /* Unserialize the dictionary */
-                dict  = dict_new ();
-
-                ret = dict_unserialize (cli_req.dict.dict_val,
-                                        cli_req.dict.dict_len,
-                                        &dict);
-                if (ret < 0) {
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "failed to "
-                                "unserialize req-buffer to dictionary");
-                        snprintf (err_str, sizeof (err_str), "Unable to decode "
-                                  "the command");
-                        goto out;
-                } else {
-                        dict->extra_stdfree = cli_req.dict.dict_val;
-                }
-        }
-
-        ret = dict_get_str (dict, "volname", &volname);
-
-        if (ret) {
-                snprintf (err_str, sizeof (err_str), "Unable to get volume "
-                          "name");
-                gf_log (this->name, GF_LOG_ERROR, "%s", err_str);
-                goto out;
-        }
-
-        if ((ret = glusterd_check_volume_exists (volname))) {
-                snprintf (err_str, sizeof (err_str), "Volume %s already exists",
-                          volname);
-                gf_log (this->name, GF_LOG_ERROR, "%s", err_str);
-                goto out;
-        }
-
-        ret = dict_get_int32 (dict, "count", &brick_count);
-        if (ret) {
-                snprintf (err_str, sizeof (err_str), "Unable to get brick count"
-                          " for volume %s", volname);
-                gf_log (this->name, GF_LOG_ERROR, "%s", err_str);
-                goto out;
-        }
-
-        ret = dict_get_int32 (dict, "type", &type);
-        if (ret) {
-                snprintf (err_str, sizeof (err_str), "Unable to get type of "
-                          "volume %s", volname);
-                gf_log (this->name, GF_LOG_ERROR, "%s", err_str);
-                goto out;
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_UNSERIALIZE_FAIL,
+                   "failed to "
+                   "unserialize req-buffer to dictionary");
+            snprintf(err_str, sizeof(err_str),
+                     "Unable to decode "
+                     "the command");
+            goto out;
+        } else {
+            dict->extra_stdfree = cli_req.dict.dict_val;
         }
+    }
 
-        ret = dict_get_str (dict, "transport", &trans_type);
-        if (ret) {
-                snprintf (err_str, sizeof (err_str), "Unable to get "
-                          "transport-type of volume %s", volname);
-                gf_log (this->name, GF_LOG_ERROR, "%s", err_str);
-                goto out;
-        }
-        ret = dict_get_str (dict, "bricks", &bricks);
-        if (ret) {
-                snprintf (err_str, sizeof (err_str), "Unable to get bricks for "
-                          "volume %s", volname);
-                gf_log (this->name, GF_LOG_ERROR, "%s", err_str);
-                goto out;
-        }
+    ret = dict_get_strn(dict, "volname", SLEN("volname"), &volname);
 
-        if (!dict_get (dict, "force")) {
-                gf_log (this->name, GF_LOG_ERROR, "Failed to get 'force' flag");
-                goto out;
-        }
+    if (ret) {
+        snprintf(err_str, sizeof(err_str),
+                 "Unable to get volume "
+                 "name");
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED, "%s",
+               err_str);
+        goto out;
+    }
 
-        gf_uuid_generate (volume_id);
-        free_ptr = gf_strdup (uuid_utoa (volume_id));
-        ret = dict_set_dynstr (dict, "volume-id", free_ptr);
-        if (ret) {
-                snprintf (err_str, sizeof (err_str), "Unable to set volume "
-                          "id of volume %s", volname);
-                gf_log (this->name, GF_LOG_ERROR, "%s", err_str);
-                goto out;
-        }
-        free_ptr = NULL;
+    ret = glusterd_volinfo_find(volname, &volinfo);
+    if (!ret) {
+        ret = -1;
+        snprintf(err_str, sizeof(err_str), "Volume %s already exists", volname);
+        gf_msg(this->name, GF_LOG_ERROR, EEXIST, GD_MSG_VOL_ALREADY_EXIST, "%s",
+               err_str);
+        goto out;
+    }
+
+    ret = dict_get_int32n(dict, "count", SLEN("count"), &brick_count);
+    if (ret) {
+        snprintf(err_str, sizeof(err_str),
+                 "Unable to get brick count"
+                 " for volume %s",
+                 volname);
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED, "%s",
+               err_str);
+        goto out;
+    }
+
+    ret = dict_get_int32n(dict, "type", SLEN("type"), &type);
+    if (ret) {
+        snprintf(err_str, sizeof(err_str),
+                 "Unable to get type of "
+                 "volume %s",
+                 volname);
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED, "%s",
+               err_str);
+        goto out;
+    }
+
+    ret = dict_get_strn(dict, "transport", SLEN("transport"), &trans_type);
+    if (ret) {
+        snprintf(err_str, sizeof(err_str),
+                 "Unable to get "
+                 "transport-type of volume %s",
+                 volname);
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED, "%s",
+               err_str);
+        goto out;
+    }
 
-        /* generate internal username and password */
+    ret = dict_get_strn(this->options, "transport.address-family",
+                        SLEN("transport.address-family"), &address_family_str);
 
-        gf_uuid_generate (tmp_uuid);
-        username = gf_strdup (uuid_utoa (tmp_uuid));
-        ret = dict_set_dynstr (dict, "internal-username", username);
+    if (!ret) {
+        ret = dict_set_dynstr_with_alloc(dict, "transport.address-family",
+                                         address_family_str);
         if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "Failed to set username for "
-                        "volume %s", volname);
-                goto out;
+            gf_log(this->name, GF_LOG_ERROR,
+                   "failed to set transport.address-family");
+            goto out;
         }
+    } else if (!strcmp(trans_type, "tcp")) {
+        /* Setting default as inet for trans_type tcp if the op-version
+         * is >= 3.8.0
+         */
+        if (conf->op_version >= GD_OP_VERSION_3_8_0) {
+            ret = dict_set_dynstr_with_alloc(dict, "transport.address-family",
+                                             addr_family);
+            if (ret) {
+                gf_log(this->name, GF_LOG_ERROR,
+                       "failed to set "
+                       "transport.address-family "
+                       "to %s",
+                       addr_family);
+                goto out;
+            }
+        }
+    }
+    ret = dict_get_strn(dict, "bricks", SLEN("bricks"), &bricks);
+    if (ret) {
+        snprintf(err_str, sizeof(err_str),
+                 "Unable to get bricks for "
+                 "volume %s",
+                 volname);
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED, "%s",
+               err_str);
+        goto out;
+    }
+
+    ret = dict_get_int32n(dict, "thin-arbiter-count",
+                          SLEN("thin-arbiter-count"), &thin_arbiter_count);
+    if (thin_arbiter_count && conf->op_version < GD_OP_VERSION_7_0) {
+        snprintf(err_str, sizeof(err_str),
+                 "Cannot execute command. "
+                 "The cluster is operating at version %d. "
+                 "Thin-arbiter volume creation is unavailable in "
+                 "this version",
+                 conf->op_version);
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_GLUSTERD_OP_FAILED, "%s",
+               err_str);
+        ret = -1;
+        goto out;
+    }
 
-        gf_uuid_generate (tmp_uuid);
-        password = gf_strdup (uuid_utoa (tmp_uuid));
-        ret = dict_set_dynstr (dict, "internal-password", password);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "Failed to set password for "
-                        "volume %s", volname);
-                goto out;
-        }
+    if (!dict_getn(dict, "force", SLEN("force"))) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+               "Failed to get 'force' flag");
+        goto out;
+    }
+
+    gf_uuid_generate(volume_id);
+    free_ptr = gf_strdup(uuid_utoa(volume_id));
+    ret = dict_set_dynstrn(dict, "volume-id", SLEN("volume-id"), free_ptr);
+    if (ret) {
+        snprintf(err_str, sizeof(err_str),
+                 "Unable to set volume "
+                 "id of volume %s",
+                 volname);
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED, "%s",
+               err_str);
+        goto out;
+    }
+    free_ptr = NULL;
+
+    /* generate internal username and password */
+
+    gf_uuid_generate(tmp_uuid);
+    username = gf_strdup(uuid_utoa(tmp_uuid));
+    ret = dict_set_dynstrn(dict, "internal-username", SLEN("internal-username"),
+                           username);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+               "Failed to set username for "
+               "volume %s",
+               volname);
+        goto out;
+    }
+
+    gf_uuid_generate(tmp_uuid);
+    password = gf_strdup(uuid_utoa(tmp_uuid));
+    ret = dict_set_dynstrn(dict, "internal-password", SLEN("internal-password"),
+                           password);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+               "Failed to set password for "
+               "volume %s",
+               volname);
+        goto out;
+    }
 
-        ret = glusterd_op_begin_synctask (req, GD_OP_CREATE_VOLUME, dict);
+    ret = glusterd_op_begin_synctask(req, GD_OP_CREATE_VOLUME, dict);
 
 out:
-        if (ret) {
-                rsp.op_ret = -1;
-                rsp.op_errno = 0;
-                if (err_str[0] == '\0')
-                        snprintf (err_str, sizeof (err_str),
-                                  "Operation failed");
-                rsp.op_errstr = err_str;
-                cli_rsp = &rsp;
-                glusterd_to_cli (req, cli_rsp, NULL, 0, NULL,
-                                 (xdrproc_t)xdr_gf_cli_rsp, dict);
-                ret = 0; //Client response sent, prevent second response
-        }
-
-        GF_FREE(free_ptr);
-
-        return ret;
+    if (ret) {
+        rsp.op_ret = -1;
+        rsp.op_errno = 0;
+        if (err_str[0] == '\0')
+            snprintf(err_str, sizeof(err_str), "Operation failed");
+        rsp.op_errstr = err_str;
+        cli_rsp = &rsp;
+        glusterd_to_cli(req, cli_rsp, NULL, 0, NULL, (xdrproc_t)xdr_gf_cli_rsp,
+                        dict);
+        ret = 0;  // Client response sent, prevent second response
+    }
+
+    GF_FREE(free_ptr);
+
+    return ret;
 }
 
 int
-glusterd_handle_create_volume (rpcsvc_request_t *req)
+glusterd_handle_create_volume(rpcsvc_request_t *req)
 {
-        return glusterd_big_locked_handler (req,
-                                            __glusterd_handle_create_volume);
+    return glusterd_big_locked_handler(req, __glusterd_handle_create_volume);
 }
 
 int
-__glusterd_handle_cli_start_volume (rpcsvc_request_t *req)
+__glusterd_handle_cli_start_volume(rpcsvc_request_t *req)
 {
-        int32_t                         ret = -1;
-        gf_cli_req                      cli_req = {{0,}};
-        char                            *volname = NULL;
-        dict_t                          *dict = NULL;
-        glusterd_op_t                   cli_op = GD_OP_START_VOLUME;
-        char                            errstr[2048] = {0,};
-        xlator_t                        *this = NULL;
-
-        this = THIS;
-        GF_ASSERT (this);
-        GF_ASSERT (req);
-
-        ret = xdr_to_generic (req->msg[0], &cli_req, (xdrproc_t)xdr_gf_cli_req);
-        if (ret < 0) {
-                snprintf (errstr, sizeof (errstr), "Failed to decode message "
-                        "received from cli");
-                req->rpc_err = GARBAGE_ARGS;
-                gf_log (this->name, sizeof (errstr), "%s", errstr);
-                goto out;
-        }
-
-        if (cli_req.dict.dict_len) {
-                /* Unserialize the dictionary */
-                dict  = dict_new ();
-
-                ret = dict_unserialize (cli_req.dict.dict_val,
-                                        cli_req.dict.dict_len,
-                                        &dict);
-                if (ret < 0) {
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "failed to "
-                                "unserialize req-buffer to dictionary");
-                        snprintf (errstr, sizeof (errstr), "Unable to decode "
-                                  "the command");
-                        goto out;
-                }
-        }
-
-        ret = dict_get_str (dict, "volname", &volname);
-        if (ret) {
-                snprintf (errstr, sizeof (errstr), "Unable to get volume name");
-                gf_log (this->name, GF_LOG_ERROR, "%s", errstr);
-                goto out;
-        }
-
-        gf_log (this->name, GF_LOG_DEBUG, "Received start vol req"
-                " for volume %s", volname);
+    int32_t ret = -1;
+    gf_cli_req cli_req = {{
+        0,
+    }};
+    char *volname = NULL;
+    dict_t *dict = NULL;
+    glusterd_op_t cli_op = GD_OP_START_VOLUME;
+    char errstr[2048] = {
+        0,
+    };
+    xlator_t *this = NULL;
+    glusterd_conf_t *conf = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+    GF_ASSERT(req);
+
+    conf = this->private;
+    GF_ASSERT(conf);
+    ret = xdr_to_generic(req->msg[0], &cli_req, (xdrproc_t)xdr_gf_cli_req);
+    if (ret < 0) {
+        snprintf(errstr, sizeof(errstr),
+                 "Failed to decode message "
+                 "received from cli");
+        req->rpc_err = GARBAGE_ARGS;
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_REQ_DECODE_FAIL, "%s",
+               errstr);
+        goto out;
+    }
 
-        ret = glusterd_op_begin_synctask (req, GD_OP_START_VOLUME, dict);
+    if (cli_req.dict.dict_len) {
+        /* Unserialize the dictionary */
+        dict = dict_new();
 
+        ret = dict_unserialize(cli_req.dict.dict_val, cli_req.dict.dict_len,
+                               &dict);
+        if (ret < 0) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_UNSERIALIZE_FAIL,
+                   "failed to "
+                   "unserialize req-buffer to dictionary");
+            snprintf(errstr, sizeof(errstr),
+                     "Unable to decode "
+                     "the command");
+            goto out;
+        }
+    }
+
+    ret = dict_get_strn(dict, "volname", SLEN("volname"), &volname);
+    if (ret) {
+        snprintf(errstr, sizeof(errstr), "Unable to get volume name");
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED, "%s",
+               errstr);
+        goto out;
+    }
+
+    gf_msg_debug(this->name, 0,
+                 "Received start vol req"
+                 " for volume %s",
+                 volname);
+
+    if (conf->op_version <= GD_OP_VERSION_3_7_6) {
+        gf_msg_debug(this->name, 0,
+                     "The cluster is operating at "
+                     "version less than or equal to %d. Volume start "
+                     "falling back to syncop framework.",
+                     GD_OP_VERSION_3_7_6);
+        ret = glusterd_op_begin_synctask(req, GD_OP_START_VOLUME, dict);
+    } else {
+        ret = glusterd_mgmt_v3_initiate_all_phases(req, GD_OP_START_VOLUME,
+                                                   dict);
+    }
 out:
-        free (cli_req.dict.dict_val); //its malloced by xdr
+    free(cli_req.dict.dict_val);  // its malloced by xdr
 
-        if (ret) {
-                if(errstr[0] == '\0')
-                        snprintf (errstr, sizeof (errstr), "Operation failed");
-                ret = glusterd_op_send_cli_response (cli_op, ret, 0, req,
-                                                     dict, errstr);
-        }
+    if (ret) {
+        if (errstr[0] == '\0')
+            snprintf(errstr, sizeof(errstr), "Operation failed");
+        ret = glusterd_op_send_cli_response(cli_op, ret, 0, req, dict, errstr);
+    }
 
-        return ret;
+    return ret;
 }
 
 int
-glusterd_handle_cli_start_volume (rpcsvc_request_t *req)
+glusterd_handle_cli_start_volume(rpcsvc_request_t *req)
 {
-        return glusterd_big_locked_handler (req,
-                                            __glusterd_handle_cli_start_volume);
+    return glusterd_big_locked_handler(req, __glusterd_handle_cli_start_volume);
 }
 
 int
-__glusterd_handle_cli_stop_volume (rpcsvc_request_t *req)
+__glusterd_handle_cli_stop_volume(rpcsvc_request_t *req)
 {
-        int32_t                         ret = -1;
-        gf_cli_req                      cli_req = {{0,}};
-        char                            *dup_volname = NULL;
-        dict_t                          *dict = NULL;
-        glusterd_op_t                   cli_op = GD_OP_STOP_VOLUME;
-        xlator_t                        *this = NULL;
-        char                            err_str[2048] = {0,};
-
-        this = THIS;
-        GF_ASSERT (this);
-        GF_ASSERT (req);
-
-        ret = xdr_to_generic (req->msg[0], &cli_req, (xdrproc_t)xdr_gf_cli_req);
-        if (ret < 0) {
-                snprintf (err_str, sizeof (err_str), "Failed to decode message "
-                          "received from cli");
-                req->rpc_err = GARBAGE_ARGS;
-                gf_log (this->name, GF_LOG_ERROR, "%s", err_str);
-                goto out;
-        }
-        if (cli_req.dict.dict_len) {
-                /* Unserialize the dictionary */
-                dict  = dict_new ();
-
-                ret = dict_unserialize (cli_req.dict.dict_val,
-                                        cli_req.dict.dict_len,
-                                        &dict);
-                if (ret < 0) {
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "failed to "
-                                "unserialize req-buffer to dictionary");
-                        snprintf (err_str, sizeof (err_str), "Unable to decode "
-                                  "the command");
-                        goto out;
-                }
-        }
-
-        ret = dict_get_str (dict, "volname", &dup_volname);
-
-        if (ret) {
-                snprintf (err_str, sizeof (err_str), "Failed to get volume "
-                          "name");
-                gf_log (this->name, GF_LOG_ERROR, "%s", err_str);
-                goto out;
-        }
-
-        gf_log (this->name, GF_LOG_DEBUG, "Received stop vol req "
-                "for volume %s", dup_volname);
+    int32_t ret = -1;
+    gf_cli_req cli_req = {{
+        0,
+    }};
+    char *dup_volname = NULL;
+    dict_t *dict = NULL;
+    glusterd_op_t cli_op = GD_OP_STOP_VOLUME;
+    xlator_t *this = NULL;
+    char err_str[64] = {
+        0,
+    };
+    glusterd_conf_t *conf = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+    GF_ASSERT(req);
+    conf = this->private;
+    GF_ASSERT(conf);
+
+    ret = xdr_to_generic(req->msg[0], &cli_req, (xdrproc_t)xdr_gf_cli_req);
+    if (ret < 0) {
+        snprintf(err_str, sizeof(err_str),
+                 "Failed to decode message "
+                 "received from cli");
+        req->rpc_err = GARBAGE_ARGS;
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_REQ_DECODE_FAIL, "%s",
+               err_str);
+        goto out;
+    }
+    if (cli_req.dict.dict_len) {
+        /* Unserialize the dictionary */
+        dict = dict_new();
 
-        ret = glusterd_op_begin_synctask (req, GD_OP_STOP_VOLUME, dict);
+        ret = dict_unserialize(cli_req.dict.dict_val, cli_req.dict.dict_len,
+                               &dict);
+        if (ret < 0) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_UNSERIALIZE_FAIL,
+                   "failed to "
+                   "unserialize req-buffer to dictionary");
+            snprintf(err_str, sizeof(err_str),
+                     "Unable to decode "
+                     "the command");
+            goto out;
+        }
+    }
+
+    ret = dict_get_strn(dict, "volname", SLEN("volname"), &dup_volname);
+
+    if (ret) {
+        snprintf(err_str, sizeof(err_str),
+                 "Failed to get volume "
+                 "name");
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED, "%s",
+               err_str);
+        goto out;
+    }
+
+    gf_msg_debug(this->name, 0,
+                 "Received stop vol req "
+                 "for volume %s",
+                 dup_volname);
+
+    if (conf->op_version < GD_OP_VERSION_4_1_0) {
+        gf_msg_debug(this->name, 0,
+                     "The cluster is operating at "
+                     "version less than %d. Volume start "
+                     "falling back to syncop framework.",
+                     GD_OP_VERSION_4_1_0);
+        ret = glusterd_op_begin_synctask(req, GD_OP_STOP_VOLUME, dict);
+    } else {
+        ret = glusterd_mgmt_v3_initiate_all_phases(req, GD_OP_STOP_VOLUME,
+                                                   dict);
+    }
 
 out:
-        free (cli_req.dict.dict_val); //its malloced by xdr
+    free(cli_req.dict.dict_val);  // its malloced by xdr
 
-        if (ret) {
-                if (err_str[0] == '\0')
-                        snprintf (err_str, sizeof (err_str),
-                                  "Operation failed");
-                ret = glusterd_op_send_cli_response (cli_op, ret, 0, req,
-                                                     dict, err_str);
-        }
+    if (ret) {
+        if (err_str[0] == '\0')
+            snprintf(err_str, sizeof(err_str), "Operation failed");
+        ret = glusterd_op_send_cli_response(cli_op, ret, 0, req, dict, err_str);
+    }
 
-        return ret;
+    return ret;
 }
 
 int
-glusterd_handle_cli_stop_volume (rpcsvc_request_t *req)
+glusterd_handle_cli_stop_volume(rpcsvc_request_t *req)
 {
-        return glusterd_big_locked_handler (req,
-                                            __glusterd_handle_cli_stop_volume);
+    return glusterd_big_locked_handler(req, __glusterd_handle_cli_stop_volume);
 }
 
 int
-__glusterd_handle_cli_delete_volume (rpcsvc_request_t *req)
+__glusterd_handle_cli_delete_volume(rpcsvc_request_t *req)
 {
-        int32_t        ret         = -1;
-        gf_cli_req     cli_req     = {{0,},};
-        glusterd_op_t  cli_op      = GD_OP_DELETE_VOLUME;
-        dict_t        *dict        = NULL;
-        char          *volname     = NULL;
-        char          err_str[2048]= {0,};
-        xlator_t      *this        = NULL;
-
-        this = THIS;
-        GF_ASSERT (this);
+    int32_t ret = -1;
+    gf_cli_req cli_req = {
+        {
+            0,
+        },
+    };
+    glusterd_op_t cli_op = GD_OP_DELETE_VOLUME;
+    dict_t *dict = NULL;
+    char *volname = NULL;
+    char err_str[64] = {
+        0,
+    };
+    xlator_t *this = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+
+    GF_ASSERT(req);
+
+    ret = xdr_to_generic(req->msg[0], &cli_req, (xdrproc_t)xdr_gf_cli_req);
+    if (ret < 0) {
+        snprintf(err_str, sizeof(err_str),
+                 "Failed to decode request "
+                 "received from cli");
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_REQ_DECODE_FAIL, "%s",
+               err_str);
+        req->rpc_err = GARBAGE_ARGS;
+        goto out;
+    }
 
-        GF_ASSERT (req);
+    if (cli_req.dict.dict_len) {
+        /* Unserialize the dictionary */
+        dict = dict_new();
 
-        ret = xdr_to_generic (req->msg[0], &cli_req, (xdrproc_t)xdr_gf_cli_req);
+        ret = dict_unserialize(cli_req.dict.dict_val, cli_req.dict.dict_len,
+                               &dict);
         if (ret < 0) {
-                snprintf (err_str, sizeof (err_str), "Failed to decode request "
-                          "received from cli");
-                gf_log (this->name, GF_LOG_ERROR, "%s", err_str);
-                req->rpc_err = GARBAGE_ARGS;
-                goto out;
-        }
-
-        if (cli_req.dict.dict_len) {
-                /* Unserialize the dictionary */
-                dict  = dict_new ();
-
-                ret = dict_unserialize (cli_req.dict.dict_val,
-                                        cli_req.dict.dict_len,
-                                        &dict);
-                if (ret < 0) {
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "failed to "
-                                "unserialize req-buffer to dictionary");
-                        snprintf (err_str, sizeof (err_str), "Unable to decode "
-                                  "the command");
-                        goto out;
-                }
-        }
-
-        ret = dict_get_str (dict, "volname", &volname);
-        if (ret) {
-                snprintf (err_str, sizeof (err_str), "Failed to get volume "
-                          "name");
-                gf_log (this->name, GF_LOG_ERROR, "%s", err_str);
-                req->rpc_err = GARBAGE_ARGS;
-                goto out;
-        }
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_UNSERIALIZE_FAIL,
+                   "failed to "
+                   "unserialize req-buffer to dictionary");
+            snprintf(err_str, sizeof(err_str),
+                     "Unable to decode "
+                     "the command");
+            goto out;
+        }
+    }
+
+    ret = dict_get_strn(dict, "volname", SLEN("volname"), &volname);
+    if (ret) {
+        snprintf(err_str, sizeof(err_str),
+                 "Failed to get volume "
+                 "name");
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED, "%s",
+               err_str);
+        req->rpc_err = GARBAGE_ARGS;
+        goto out;
+    }
 
-        gf_log (this->name, GF_LOG_DEBUG, "Received delete vol req"
-                "for volume %s", volname);
+    gf_msg_debug(this->name, 0,
+                 "Received delete vol req"
+                 "for volume %s",
+                 volname);
 
-        ret = glusterd_op_begin_synctask (req, GD_OP_DELETE_VOLUME, dict);
+    ret = glusterd_op_begin_synctask(req, GD_OP_DELETE_VOLUME, dict);
 
 out:
-        free (cli_req.dict.dict_val); //its malloced by xdr
+    free(cli_req.dict.dict_val);  // its malloced by xdr
 
-        if (ret) {
-                if (err_str[0] == '\0')
-                        snprintf (err_str, sizeof (err_str),
-                                  "Operation failed");
-                ret = glusterd_op_send_cli_response (cli_op, ret, 0, req,
-                                                     dict, err_str);
-        }
+    if (ret) {
+        if (err_str[0] == '\0')
+            snprintf(err_str, sizeof(err_str), "Operation failed");
+        ret = glusterd_op_send_cli_response(cli_op, ret, 0, req, dict, err_str);
+    }
 
-        return ret;
+    return ret;
 }
-
 int
-glusterd_handle_cli_delete_volume (rpcsvc_request_t *req)
+glusterd_handle_cli_delete_volume(rpcsvc_request_t *req)
 {
-        return glusterd_big_locked_handler (req,
-                                            __glusterd_handle_cli_delete_volume);
+    return glusterd_big_locked_handler(req,
+                                       __glusterd_handle_cli_delete_volume);
 }
-
 static int
-glusterd_handle_heal_enable_disable (rpcsvc_request_t *req, dict_t *dict,
-                                     glusterd_volinfo_t *volinfo)
+glusterd_handle_heal_options_enable_disable(rpcsvc_request_t *req, dict_t *dict,
+                                            glusterd_volinfo_t *volinfo)
 {
-        gf_xl_afr_op_t                  heal_op = GF_SHD_OP_INVALID;
-        int                             ret = 0;
-        xlator_t                        *this = THIS;
-        char                            *key = NULL;
-        char                            *value = NULL;
-
-        ret = dict_get_int32 (dict, "heal-op", (int32_t *)&heal_op);
-        if (ret || (heal_op == GF_SHD_OP_INVALID)) {
-                ret = -1;
-                goto out;
-        }
+    gf_xl_afr_op_t heal_op = GF_SHD_OP_INVALID;
+    int ret = 0;
+    char *key = NULL;
+    char *value = NULL;
+    xlator_t *this = THIS;
+    GF_ASSERT(this);
+
+    ret = dict_get_int32n(dict, "heal-op", SLEN("heal-op"),
+                          (int32_t *)&heal_op);
+    if (ret || (heal_op == GF_SHD_OP_INVALID)) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_GET_FAILED,
+                "Key=heal-op", NULL);
+        ret = -1;
+        goto out;
+    }
 
-        if ((heal_op != GF_SHD_OP_HEAL_ENABLE) &&
-            (heal_op != GF_SHD_OP_HEAL_DISABLE)) {
-                ret = -EINVAL;
-                goto out;
-        }
+    if ((heal_op != GF_SHD_OP_HEAL_ENABLE) &&
+        (heal_op != GF_SHD_OP_HEAL_DISABLE) &&
+        (heal_op != GF_SHD_OP_GRANULAR_ENTRY_HEAL_ENABLE) &&
+        (heal_op != GF_SHD_OP_GRANULAR_ENTRY_HEAL_DISABLE)) {
+        ret = -EINVAL;
+        goto out;
+    }
 
-        key = volgen_get_shd_key (volinfo);
+    if (((heal_op == GF_SHD_OP_GRANULAR_ENTRY_HEAL_ENABLE) ||
+         (heal_op == GF_SHD_OP_GRANULAR_ENTRY_HEAL_DISABLE)) &&
+        (volinfo->type == GF_CLUSTER_TYPE_DISPERSE)) {
+        ret = -1;
+        goto out;
+    }
+
+    if ((heal_op == GF_SHD_OP_HEAL_ENABLE) ||
+        (heal_op == GF_SHD_OP_GRANULAR_ENTRY_HEAL_ENABLE)) {
+        value = "enable";
+    } else if ((heal_op == GF_SHD_OP_HEAL_DISABLE) ||
+               (heal_op == GF_SHD_OP_GRANULAR_ENTRY_HEAL_DISABLE)) {
+        value = "disable";
+    }
+
+    if ((heal_op == GF_SHD_OP_HEAL_ENABLE) ||
+        (heal_op == GF_SHD_OP_HEAL_DISABLE)) {
+        key = volgen_get_shd_key(volinfo->type);
         if (!key) {
-                ret = -1;
-                goto out;
+            ret = -1;
+            goto out;
         }
-
-        /* Convert this command to volume-set command based on volume type */
-        ret = dict_set_str (dict, "key1", key);
-        if (ret)
-                goto out;
-
-        if (heal_op == GF_SHD_OP_HEAL_ENABLE) {
-                value = "enable";
-        } else if (heal_op == GF_SHD_OP_HEAL_DISABLE) {
-                value = "disable";
+    } else {
+        key = "cluster.granular-entry-heal";
+        ret = dict_set_int8(dict, "is-special-key", 1);
+        if (ret) {
+            gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_SET_FAILED,
+                    "Key=is-special-key", NULL);
+            goto out;
         }
+    }
 
-        ret = dict_set_str (dict, "value1", value);
-        if (ret)
-                goto out;
+    ret = dict_set_strn(dict, "key1", SLEN("key1"), key);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_SET_FAILED,
+                "Key=key1", NULL);
+        goto out;
+    }
 
-        ret = dict_set_int32 (dict, "count", 1);
-        if (ret)
-                goto out;
+    ret = dict_set_strn(dict, "value1", SLEN("value1"), value);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_SET_FAILED,
+                "Key=value1", NULL);
+        goto out;
+    }
 
-        ret = glusterd_op_begin_synctask (req, GD_OP_SET_VOLUME, dict);
+    ret = dict_set_int32n(dict, "count", SLEN("count"), 1);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_SET_FAILED,
+                "Key=count", NULL);
+        goto out;
+    }
+
+    ret = glusterd_op_begin_synctask(req, GD_OP_SET_VOLUME, dict);
 
 out:
-        return ret;
+    return ret;
 }
 
 int
-__glusterd_handle_cli_heal_volume (rpcsvc_request_t *req)
+__glusterd_handle_cli_heal_volume(rpcsvc_request_t *req)
 {
-        int32_t                         ret = -1;
-        gf_cli_req                      cli_req = {{0,}};
-        dict_t                          *dict = NULL;
-        glusterd_op_t                   cli_op = GD_OP_HEAL_VOLUME;
-        char                            *volname = NULL;
-        glusterd_volinfo_t              *volinfo = NULL;
-        xlator_t                        *this = NULL;
-        char                            op_errstr[2048] = {0,};
-
-        GF_ASSERT (req);
-
-        ret = xdr_to_generic (req->msg[0], &cli_req, (xdrproc_t)xdr_gf_cli_req);
-        if (ret < 0) {
-                //failed to decode msg;
-                req->rpc_err = GARBAGE_ARGS;
-                goto out;
-        }
+    int32_t ret = -1;
+    gf_cli_req cli_req = {{
+        0,
+    }};
+    dict_t *dict = NULL;
+    glusterd_op_t cli_op = GD_OP_HEAL_VOLUME;
+    char *volname = NULL;
+    glusterd_volinfo_t *volinfo = NULL;
+    xlator_t *this = NULL;
+    char op_errstr[2048] = {
+        0,
+    };
+
+    this = THIS;
+    GF_ASSERT(this);
+
+    GF_ASSERT(req);
+
+    ret = xdr_to_generic(req->msg[0], &cli_req, (xdrproc_t)xdr_gf_cli_req);
+    if (ret < 0) {
+        // failed to decode msg;
+        req->rpc_err = GARBAGE_ARGS;
+        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_GARBAGE_ARGS, NULL);
+        goto out;
+    }
 
-        this = THIS;
-        GF_ASSERT (this);
-
-        if (cli_req.dict.dict_len) {
-                /* Unserialize the dictionary */
-                dict  = dict_new ();
-
-                ret = dict_unserialize (cli_req.dict.dict_val,
-                                        cli_req.dict.dict_len,
-                                        &dict);
-                if (ret < 0) {
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "failed to "
-                                "unserialize req-buffer to dictionary");
-                        snprintf (op_errstr, sizeof (op_errstr),
-                        "Unable to decode the command");
-                        goto out;
-                } else {
-                        dict->extra_stdfree = cli_req.dict.dict_val;
-                }
-        }
+    if (cli_req.dict.dict_len) {
+        /* Unserialize the dictionary */
+        dict = dict_new();
 
-        ret = dict_get_str (dict, "volname", &volname);
-        if (ret) {
-                snprintf (op_errstr, sizeof (op_errstr), "Unable to find "
-                          "volume name");
-                gf_log (this->name, GF_LOG_ERROR, "%s", op_errstr);
-                goto out;
+        ret = dict_unserialize(cli_req.dict.dict_val, cli_req.dict.dict_len,
+                               &dict);
+        if (ret < 0) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_UNSERIALIZE_FAIL,
+                   "failed to "
+                   "unserialize req-buffer to dictionary");
+            snprintf(op_errstr, sizeof(op_errstr),
+                     "Unable to decode the command");
+            goto out;
+        } else {
+            dict->extra_stdfree = cli_req.dict.dict_val;
         }
+    }
 
-        gf_log (this->name, GF_LOG_INFO, "Received heal vol req "
-                "for volume %s", volname);
+    ret = dict_get_strn(dict, "volname", SLEN("volname"), &volname);
+    if (ret) {
+        snprintf(op_errstr, sizeof(op_errstr),
+                 "Unable to find "
+                 "volume name");
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED, "%s",
+               op_errstr);
+        goto out;
+    }
 
-        ret = glusterd_volinfo_find (volname, &volinfo);
-        if (ret) {
-                snprintf (op_errstr, sizeof (op_errstr),
-                          "Volume %s does not exist", volname);
-                goto out;
-        }
+    gf_msg(this->name, GF_LOG_INFO, 0, GD_MSG_HEAL_VOL_REQ_RCVD,
+           "Received heal vol req "
+           "for volume %s",
+           volname);
 
-        ret = glusterd_handle_heal_enable_disable (req, dict, volinfo);
-        if (ret == -EINVAL) {
-                ret = 0;
-        } else {
-                /*
-                 * If the return value is -ve but not -EINVAL then the command
-                 * failed. If the return value is 0 then the synctask for the
-                 * op has begun, so in both cases just 'goto out'. If there was
-                 * a failure it will respond with an error, otherwise the
-                 * synctask will take the responsibility of sending the
-                 * response.
-                 */
-                goto out;
-        }
+    ret = glusterd_volinfo_find(volname, &volinfo);
+    if (ret) {
+        snprintf(op_errstr, sizeof(op_errstr), "Volume %s does not exist",
+                 volname);
+        goto out;
+    }
 
-        ret = glusterd_add_bricks_hname_path_to_dict (dict, volinfo);
-        if (ret)
-                goto out;
+    ret = glusterd_handle_heal_options_enable_disable(req, dict, volinfo);
+    if (ret == -EINVAL) {
+        ret = 0;
+    } else {
+        /*
+         * If the return value is -ve but not -EINVAL then the command
+         * failed. If the return value is 0 then the synctask for the
+         * op has begun, so in both cases just 'goto out'. If there was
+         * a failure it will respond with an error, otherwise the
+         * synctask will take the responsibility of sending the
+         * response.
+         */
+        goto out;
+    }
 
-        ret = dict_set_int32 (dict, "count", volinfo->brick_count);
-        if (ret)
-                goto out;
+    ret = glusterd_add_bricks_hname_path_to_dict(dict, volinfo);
+    if (ret)
+        goto out;
 
-        ret = glusterd_op_begin_synctask (req, GD_OP_HEAL_VOLUME, dict);
+    ret = dict_set_int32n(dict, "count", SLEN("count"), volinfo->brick_count);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_SET_FAILED,
+                "Key=count", NULL);
+        goto out;
+    }
 
-out:
-        if (ret) {
-                if (op_errstr[0] == '\0')
-                        snprintf (op_errstr, sizeof (op_errstr),
-                                  "operation failed");
-                gf_log (this->name, GF_LOG_ERROR, "%s", op_errstr);
-                ret = glusterd_op_send_cli_response (cli_op, ret, 0, req,
-                                                     dict, op_errstr);
-        }
+    ret = glusterd_op_begin_synctask(req, GD_OP_HEAL_VOLUME, dict);
 
-        return ret;
+out:
+    if (ret) {
+        if (op_errstr[0] == '\0')
+            snprintf(op_errstr, sizeof(op_errstr), "operation failed");
+        gf_msg((this ? this->name : "glusterd"), GF_LOG_ERROR, 0,
+               GD_MSG_GLUSTERD_OP_FAILED, "%s", op_errstr);
+        ret = glusterd_op_send_cli_response(cli_op, ret, 0, req, dict,
+                                            op_errstr);
+    }
+
+    return ret;
 }
 
 int
-glusterd_handle_cli_heal_volume (rpcsvc_request_t *req)
+glusterd_handle_cli_heal_volume(rpcsvc_request_t *req)
 {
-        return glusterd_big_locked_handler (req,
-                                            __glusterd_handle_cli_heal_volume);
+    return glusterd_big_locked_handler(req, __glusterd_handle_cli_heal_volume);
 }
 
 int
-__glusterd_handle_cli_statedump_volume (rpcsvc_request_t *req)
+__glusterd_handle_cli_statedump_volume(rpcsvc_request_t *req)
 {
-        int32_t                         ret = -1;
-        gf_cli_req                      cli_req = {{0,}};
-        char                            *volname = NULL;
-        char                            *options = NULL;
-        dict_t                          *dict = NULL;
-        int32_t                         option_cnt = 0;
-        glusterd_op_t                   cli_op = GD_OP_STATEDUMP_VOLUME;
-        char                            err_str[2048] = {0,};
-        xlator_t                        *this = NULL;
-        glusterd_conf_t                 *priv = NULL;
-
-        this = THIS;
-        GF_ASSERT (this);
-        priv = this->private;
-        GF_ASSERT (priv);
-
-        GF_ASSERT (req);
+    int32_t ret = -1;
+    gf_cli_req cli_req = {{
+        0,
+    }};
+    char *volname = NULL;
+    char *options = NULL;
+    dict_t *dict = NULL;
+    int32_t option_cnt = 0;
+    glusterd_op_t cli_op = GD_OP_STATEDUMP_VOLUME;
+    char err_str[128] = {
+        0,
+    };
+    xlator_t *this = NULL;
+    glusterd_conf_t *priv = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+    priv = this->private;
+    GF_ASSERT(priv);
+
+    GF_ASSERT(req);
+
+    ret = -1;
+    ret = xdr_to_generic(req->msg[0], &cli_req, (xdrproc_t)xdr_gf_cli_req);
+    if (ret < 0) {
+        req->rpc_err = GARBAGE_ARGS;
+        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_GARBAGE_ARGS, NULL);
+        goto out;
+    }
+    if (cli_req.dict.dict_len) {
+        /* Unserialize the dictionary */
+        dict = dict_new();
 
-        ret = -1;
-        ret = xdr_to_generic (req->msg[0], &cli_req, (xdrproc_t)xdr_gf_cli_req);
+        ret = dict_unserialize(cli_req.dict.dict_val, cli_req.dict.dict_len,
+                               &dict);
         if (ret < 0) {
-                req->rpc_err = GARBAGE_ARGS;
-                goto out;
-        }
-        if (cli_req.dict.dict_len) {
-                /* Unserialize the dictionary */
-                dict  = dict_new ();
-
-                ret = dict_unserialize (cli_req.dict.dict_val,
-                                        cli_req.dict.dict_len,
-                                        &dict);
-                if (ret < 0) {
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "failed to "
-                                "unserialize req-buffer to dictionary");
-                        snprintf (err_str, sizeof (err_str), "Unable to "
-                                  "decode the command");
-                        goto out;
-                }
-        }
-        ret = dict_get_str (dict, "volname", &volname);
-        if (ret) {
-                snprintf (err_str, sizeof (err_str), "Unable to get the volume "
-                          "name");
-                gf_log (this->name, GF_LOG_ERROR, "%s", err_str);
-                goto out;
-        }
-
-        ret = dict_get_str (dict, "options", &options);
-        if (ret) {
-                snprintf (err_str, sizeof (err_str), "Unable to get options");
-                gf_log (this->name, GF_LOG_ERROR, "%s", err_str);
-                goto out;
-        }
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_UNSERIALIZE_FAIL,
+                   "failed to "
+                   "unserialize req-buffer to dictionary");
+            snprintf(err_str, sizeof(err_str),
+                     "Unable to "
+                     "decode the command");
+            goto out;
+        }
+    }
+    ret = dict_get_strn(dict, "volname", SLEN("volname"), &volname);
+    if (ret) {
+        snprintf(err_str, sizeof(err_str), "Unable to get the volume name");
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED, "%s",
+               err_str);
+        goto out;
+    }
 
-        ret = dict_get_int32 (dict, "option_cnt", &option_cnt);
-        if (ret) {
-                snprintf (err_str , sizeof (err_str), "Unable to get option "
-                          "count");
-                gf_log (this->name, GF_LOG_ERROR, "%s", err_str);
-                goto out;
-        }
+    ret = dict_get_strn(dict, "options", SLEN("options"), &options);
+    if (ret) {
+        snprintf(err_str, sizeof(err_str), "Unable to get options");
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED, "%s",
+               err_str);
+        goto out;
+    }
+
+    ret = dict_get_int32n(dict, "option_cnt", SLEN("option_cnt"), &option_cnt);
+    if (ret) {
+        snprintf(err_str, sizeof(err_str),
+                 "Unable to get option "
+                 "count");
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED, "%s",
+               err_str);
+        goto out;
+    }
 
-        if (priv->op_version == GD_OP_VERSION_MIN &&
-            strstr (options, "quotad")) {
-                snprintf (err_str, sizeof (err_str), "The cluster is operating "
-                          "at op-version 1. Taking quotad's statedump is "
-                          "disallowed in this state");
-                ret = -1;
-                goto out;
-        }
+    if (priv->op_version == GD_OP_VERSION_MIN && strstr(options, "quotad")) {
+        snprintf(err_str, sizeof(err_str),
+                 "The cluster is operating "
+                 "at op-version 1. Taking quotad's statedump is "
+                 "disallowed in this state");
+        ret = -1;
+        goto out;
+    }
 
-        gf_log (this->name, GF_LOG_INFO, "Received statedump request for "
-                "volume %s with options %s", volname, options);
+    gf_msg(this->name, GF_LOG_INFO, 0, GD_MSG_STATEDUMP_VOL_REQ_RCVD,
+           "Received statedump request for "
+           "volume %s with options %s",
+           volname, options);
 
-        ret = glusterd_op_begin_synctask (req, GD_OP_STATEDUMP_VOLUME, dict);
+    ret = glusterd_op_begin_synctask(req, GD_OP_STATEDUMP_VOLUME, dict);
 
 out:
-        if (ret) {
-                if (err_str[0] == '\0')
-                        snprintf (err_str, sizeof (err_str),
-                                  "Operation failed");
-                ret = glusterd_op_send_cli_response (cli_op, ret, 0, req,
-                                                     dict, err_str);
-        }
-        free (cli_req.dict.dict_val);
-
-        return ret;
+    if (ret) {
+        if (err_str[0] == '\0')
+            snprintf(err_str, sizeof(err_str), "Operation failed");
+        ret = glusterd_op_send_cli_response(cli_op, ret, 0, req, dict, err_str);
+    }
+    free(cli_req.dict.dict_val);
+
+    return ret;
 }
 
 int
-glusterd_handle_cli_statedump_volume (rpcsvc_request_t *req)
-{
-        return glusterd_big_locked_handler (req,
-                                            __glusterd_handle_cli_statedump_volume);
-}
-
-#ifdef HAVE_BD_XLATOR
-/*
- * Validates if given VG in the brick exists or not. Also checks if VG has
- * GF_XATTR_VOL_ID_KEY tag set to avoid using same VG for multiple bricks.
- * Tag is checked only during glusterd_op_stage_create_volume. Tag is set during
- * glusterd_validate_and_create_brickpath().
- * @brick - brick info, @check_tag - check for VG tag or not
- * @msg - Error message to return to caller
- */
-int
-glusterd_is_valid_vg (glusterd_brickinfo_t *brick, int check_tag, char *msg)
+glusterd_handle_cli_statedump_volume(rpcsvc_request_t *req)
 {
-        lvm_t                     handle     = NULL;
-        vg_t                      vg         = NULL;
-        char                     *vg_name    = NULL;
-        int                       retval     = 0;
-        char                     *p          = NULL;
-        char                     *ptr        = NULL;
-        struct dm_list           *dm_lvlist = NULL;
-        struct dm_list           *dm_seglist = NULL;
-        struct lvm_lv_list       *lv_list    = NULL;
-        struct lvm_property_value prop       = {0, };
-        struct lvm_lvseg_list    *seglist    = NULL;
-        struct dm_list           *taglist    = NULL;
-        struct lvm_str_list      *strl       = NULL;
-
-        handle = lvm_init (NULL);
-        if (!handle) {
-                sprintf (msg, "lvm_init failed, could not validate vg");
-                return -1;
-        }
-        if (*brick->vg == '\0') { /* BD xlator has vg in brick->path */
-                p = gf_strdup (brick->path);
-                vg_name = strtok_r (p, "/", &ptr);
-        } else
-                vg_name = brick->vg;
-
-        vg = lvm_vg_open (handle, vg_name, "r", 0);
-        if (!vg) {
-                sprintf (msg, "no such vg: %s", vg_name);
-                retval = -1;
-                goto out;
-        }
-        if (!check_tag)
-                goto next;
-
-        taglist = lvm_vg_get_tags (vg);
-        if (!taglist)
-                goto next;
-
-        dm_list_iterate_items (strl, taglist) {
-                if (!strncmp(strl->str, GF_XATTR_VOL_ID_KEY,
-                             strlen (GF_XATTR_VOL_ID_KEY))) {
-                            sprintf (msg, "VG %s is already part of"
-                                    " a brick", vg_name);
-                            retval = -1;
-                            goto out;
-                    }
-        }
-next:
-
-        brick->caps = CAPS_BD | CAPS_OFFLOAD_COPY | CAPS_OFFLOAD_SNAPSHOT;
-
-        dm_lvlist = lvm_vg_list_lvs (vg);
-        if (!dm_lvlist)
-                goto out;
-
-        dm_list_iterate_items (lv_list, dm_lvlist) {
-                dm_seglist = lvm_lv_list_lvsegs (lv_list->lv);
-                dm_list_iterate_items (seglist, dm_seglist) {
-                        prop = lvm_lvseg_get_property (seglist->lvseg,
-                                                       "segtype");
-                        if (!prop.is_valid || !prop.value.string)
-                                continue;
-                        if (!strcmp (prop.value.string, "thin-pool")) {
-                                brick->caps |= CAPS_THIN;
-                                gf_log (THIS->name, GF_LOG_INFO, "Thin Pool "
-                                        "\"%s\" will be used for thin LVs",
-                                        lvm_lv_get_name (lv_list->lv));
-                                break;
-                        }
-                }
-        }
-
-        retval = 0;
-out:
-        if (vg)
-                lvm_vg_close (vg);
-        lvm_quit (handle);
-        if (p)
-                GF_FREE (p);
-        return retval;
+    return glusterd_big_locked_handler(req,
+                                       __glusterd_handle_cli_statedump_volume);
 }
-#endif
 
 /* op-sm */
 int
-glusterd_op_stage_create_volume (dict_t *dict, char **op_errstr,
-                                 dict_t *rsp_dict)
+glusterd_op_stage_create_volume(dict_t *dict, char **op_errstr,
+                                dict_t *rsp_dict)
 {
-        int                                     ret = 0;
-        char                                    *volname = NULL;
-        gf_boolean_t                            exists = _gf_false;
-        char                                    *bricks = NULL;
-        char                                    *brick_list = NULL;
-        char                                    *free_ptr = NULL;
-        char                                    key[PATH_MAX] = "";
-        glusterd_brickinfo_t                    *brick_info = NULL;
-        int32_t                                 brick_count = 0;
-        int32_t                                 local_brick_count = 0;
-        int32_t                                 i = 0;
-        int32_t                                 type = 0;
-        char                                    *brick = NULL;
-        char                                    *tmpptr = NULL;
-        xlator_t                                *this = NULL;
-        glusterd_conf_t                         *priv = NULL;
-        char                                    msg[2048] = {0};
-        uuid_t                                  volume_uuid;
-        char                                    *volume_uuid_str;
-        gf_boolean_t                             is_force = _gf_false;
-
-        this = THIS;
-        GF_ASSERT (this);
-        priv = this->private;
-        GF_ASSERT (priv);
-        GF_ASSERT (rsp_dict);
-
-        ret = dict_get_str (dict, "volname", &volname);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "Unable to get volume name");
-                goto out;
-        }
-
-        exists = glusterd_check_volume_exists (volname);
-        if (exists) {
-                snprintf (msg, sizeof (msg), "Volume %s already exists",
-                          volname);
-                ret = -1;
-                goto out;
-        } else {
-                ret = 0;
-        }
+    int ret = 0;
+    char *volname = NULL;
+    char *bricks = NULL;
+    char *brick_list = NULL;
+    char *free_ptr = NULL;
+    char key[64] = "";
+    glusterd_brickinfo_t *brick_info = NULL;
+    int32_t brick_count = 0;
+    int32_t local_brick_count = 0;
+    int32_t i = 0;
+    int32_t type = 0;
+    int32_t replica_count = 0;
+    int32_t disperse_count = 0;
+    char *brick = NULL;
+    char *tmpptr = NULL;
+    xlator_t *this = NULL;
+    glusterd_conf_t *priv = NULL;
+    char msg[2048] = {0};
+    uuid_t volume_uuid;
+    char *volume_uuid_str;
+    gf_boolean_t is_force = _gf_false;
+    glusterd_volinfo_t *volinfo = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+    priv = this->private;
+    GF_ASSERT(priv);
+    GF_ASSERT(rsp_dict);
+
+    ret = dict_get_strn(dict, "volname", SLEN("volname"), &volname);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+               "Unable to get volume name");
+        goto out;
+    }
 
-        ret = dict_get_int32 (dict, "count", &brick_count);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "Unable to get brick count "
-                        "for volume %s", volname);
-                goto out;
-        }
+    ret = glusterd_volinfo_find(volname, &volinfo);
+    if (!ret) {
+        snprintf(msg, sizeof(msg), "Volume %s already exists", volname);
+        ret = -1;
+        goto out;
+    }
+
+    ret = dict_get_int32n(dict, "count", SLEN("count"), &brick_count);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+               "Unable to get brick count "
+               "for volume %s",
+               volname);
+        goto out;
+    }
+
+    ret = dict_get_strn(dict, "volume-id", SLEN("volume-id"), &volume_uuid_str);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+               "Unable to get volume id of "
+               "volume %s",
+               volname);
+        goto out;
+    }
+
+    ret = gf_uuid_parse(volume_uuid_str, volume_uuid);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_UUID_PARSE_FAIL,
+               "Unable to parse volume id of"
+               " volume %s",
+               volname);
+        goto out;
+    }
+
+    ret = dict_get_strn(dict, "bricks", SLEN("bricks"), &bricks);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+               "Unable to get bricks for "
+               "volume %s",
+               volname);
+        goto out;
+    }
 
-        ret = dict_get_str (dict, "volume-id", &volume_uuid_str);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "Unable to get volume id of "
-                        "volume %s", volname);
-                goto out;
-        }
+    is_force = dict_get_str_boolean(dict, "force", _gf_false);
 
-        ret = gf_uuid_parse (volume_uuid_str, volume_uuid);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "Unable to parse volume id of"
-                        " volume %s", volname);
-                goto out;
+    if (bricks) {
+        brick_list = gf_strdup(bricks);
+        if (!brick_list) {
+            ret = -1;
+            goto out;
+        } else {
+            free_ptr = brick_list;
         }
+    }
 
-        ret = dict_get_str (dict, "bricks", &bricks);
+    /*Check brick order if the volume type is replicate or disperse. If
+     * force at the end of command not given then check brick order.
+     */
+    if (is_origin_glusterd(dict)) {
+        ret = dict_get_int32n(dict, "type", SLEN("type"), &type);
         if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "Unable to get bricks for "
-                        "volume %s", volname);
-                goto out;
+            snprintf(msg, sizeof(msg),
+                     "Unable to get type of "
+                     "volume %s",
+                     volname);
+            gf_msg(this->name, GF_LOG_WARNING, 0, GD_MSG_DICT_GET_FAILED, "%s",
+                   msg);
+            goto out;
         }
 
-        is_force = dict_get_str_boolean (dict, "force", _gf_false);
-
-        if (bricks) {
-                brick_list = gf_strdup (bricks);
-                if (!brick_list) {
-                        ret = -1;
-                        goto out;
-                } else {
-                        free_ptr = brick_list;
+        if (!is_force) {
+            if (type == GF_CLUSTER_TYPE_REPLICATE) {
+                ret = dict_get_int32n(dict, "replica-count",
+                                      SLEN("replica-count"), &replica_count);
+                if (ret) {
+                    gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+                           "Bricks check : Could"
+                           " not retrieve replica count");
+                    goto out;
                 }
-        }
-
-        while ( i < brick_count) {
-                i++;
-                brick= strtok_r (brick_list, " \n", &tmpptr);
-                brick_list = tmpptr;
-
-                if (!glusterd_store_is_valid_brickpath (volname, brick)) {
-                        snprintf (msg, sizeof (msg), "brick path %s is too "
-                                  "long.", brick);
-                        ret = -1;
-                        goto out;
+                gf_msg_debug(this->name, 0,
+                             "Replicate cluster type "
+                             "found. Checking brick order.");
+                ret = glusterd_check_brick_order(dict, msg, type, &volname,
+                                                 &bricks, &brick_count,
+                                                 replica_count);
+            } else if (type == GF_CLUSTER_TYPE_DISPERSE) {
+                ret = dict_get_int32n(dict, "disperse-count",
+                                      SLEN("disperse-count"), &disperse_count);
+                if (ret) {
+                    gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+                           "Bricks check : Could"
+                           " not retrieve disperse count");
+                    goto out;
                 }
+                gf_msg_debug(this->name, 0,
+                             "Disperse cluster type"
+                             " found. Checking brick order.");
+                ret = glusterd_check_brick_order(dict, msg, type, &volname,
+                                                 &bricks, &brick_count,
+                                                 disperse_count);
+            }
+            if (ret) {
+                gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_BAD_BRKORDER,
+                       "Not creating the volume because of "
+                       "bad brick order. %s",
+                       msg);
+                *op_errstr = gf_strdup(msg);
+                goto out;
+            }
+        }
+    }
+
+    while (i < brick_count) {
+        i++;
+        brick = strtok_r(brick_list, " \n", &tmpptr);
+        brick_list = tmpptr;
+
+        if (!glusterd_store_is_valid_brickpath(volname, brick)) {
+            snprintf(msg, sizeof(msg),
+                     "brick path %s is too "
+                     "long.",
+                     brick);
+            ret = -1;
+            goto out;
+        }
+
+        if (!glusterd_is_valid_volfpath(volname, brick)) {
+            snprintf(msg, sizeof(msg),
+                     "Volume file path for "
+                     "volume %s and brick path %s is too long.",
+                     volname, brick);
+            ret = -1;
+            goto out;
+        }
+
+        ret = glusterd_brickinfo_new_from_brick(brick, &brick_info, _gf_true,
+                                                op_errstr);
+        if (ret)
+            goto out;
 
-                if (!glusterd_is_valid_volfpath (volname, brick)) {
-                        snprintf (msg, sizeof (msg), "Volume file path for "
-                                  "volume %s and brick path %s is too long.",
-                                   volname, brick);
-                        ret = -1;
-                        goto out;
-                }
+        ret = glusterd_new_brick_validate(brick, brick_info, msg, sizeof(msg),
+                                          NULL);
+        if (ret)
+            goto out;
 
-                ret = glusterd_brickinfo_new_from_brick (brick, &brick_info);
-                if (ret)
-                        goto out;
+        ret = glusterd_resolve_brick(brick_info);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_RESOLVE_BRICK_FAIL,
+                   FMTSTR_RESOLVE_BRICK, brick_info->hostname,
+                   brick_info->path);
+            goto out;
+        }
 
-                ret = glusterd_new_brick_validate (brick, brick_info, msg,
-                                                   sizeof (msg));
-                if (ret)
-                        goto out;
+        if (!gf_uuid_compare(brick_info->uuid, MY_UUID)) {
+            ret = glusterd_validate_and_create_brickpath(
+                brick_info, volume_uuid, volname, op_errstr, is_force,
+                _gf_false);
+            if (ret)
+                goto out;
 
-                ret = glusterd_resolve_brick (brick_info);
+            /* A bricks mount dir is required only by snapshots which were
+             * introduced in gluster-3.6.0
+             */
+            if (priv->op_version >= GD_OP_VERSION_3_6_0) {
+                ret = glusterd_get_brick_mount_dir(brick_info->path,
+                                                   brick_info->hostname,
+                                                   brick_info->mount_dir);
                 if (ret) {
-                        gf_log (this->name, GF_LOG_ERROR, FMTSTR_RESOLVE_BRICK,
-                                brick_info->hostname, brick_info->path);
-                        goto out;
-                }
-
-                if (!gf_uuid_compare (brick_info->uuid, MY_UUID)) {
-#ifdef HAVE_BD_XLATOR
-                        if (brick_info->vg[0]) {
-                                ret = glusterd_is_valid_vg (brick_info, 1, msg);
-                                if (ret)
-                                        goto out;
-                        }
-#endif
-                        ret = glusterd_validate_and_create_brickpath (brick_info,
-                                                          volume_uuid, op_errstr,
-                                                          is_force);
-                        if (ret)
-                                goto out;
-
-                        /* A bricks mount dir is required only by snapshots which were
-                         * introduced in gluster-3.6.0
-                         */
-                        if (priv->op_version >= GD_OP_VERSION_3_6_0) {
-                                ret = glusterd_get_brick_mount_dir
-                                        (brick_info->path, brick_info->hostname,
-                                         brick_info->mount_dir);
-                                if (ret) {
-                                        gf_log (this->name, GF_LOG_ERROR,
-                                                "Failed to get brick mount_dir");
-                                        goto out;
-                                }
-
-                                snprintf (key, sizeof(key), "brick%d.mount_dir",
-                                          i);
-                                ret = dict_set_dynstr_with_alloc
-                                        (rsp_dict, key, brick_info->mount_dir);
-                                if (ret) {
-                                        gf_log (this->name, GF_LOG_ERROR,
-                                                "Failed to set %s", key);
-                                        goto out;
-                                }
-                        }
-                        local_brick_count = i;
-
-                        brick_list = tmpptr;
+                    gf_msg(this->name, GF_LOG_ERROR, 0,
+                           GD_MSG_BRICK_MOUNTDIR_GET_FAIL,
+                           "Failed to get brick mount_dir");
+                    goto out;
                 }
-                glusterd_brickinfo_delete (brick_info);
-                brick_info = NULL;
-        }
 
-        /*Check brick order if the volume type is replicate or disperse. If
-         * force at the end of command not given then check brick order.
-         */
-        if (is_origin_glusterd (dict)) {
-                ret = dict_get_int32 (dict, "type", &type);
+                snprintf(key, sizeof(key), "brick%d.mount_dir", i);
+                ret = dict_set_dynstr_with_alloc(rsp_dict, key,
+                                                 brick_info->mount_dir);
                 if (ret) {
-                        snprintf (msg, sizeof (msg), "Unable to get type of "
-                                  "volume %s", volname);
-                        gf_log (this->name, GF_LOG_WARNING, "%s", msg);
-                        goto out;
+                    gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+                           "Failed to set %s", key);
+                    goto out;
                 }
+            }
+            local_brick_count = i;
 
-                if (!is_force) {
-                        if ((type == GF_CLUSTER_TYPE_REPLICATE) ||
-                            (type == GF_CLUSTER_TYPE_STRIPE_REPLICATE) ||
-                            (type == GF_CLUSTER_TYPE_DISPERSE)) {
-                                ret = glusterd_check_brick_order(dict, msg);
-                                if (ret) {
-                                        gf_log(this->name, GF_LOG_ERROR, "Not "
-                                               "creating volume because of bad "
-                                               "brick order");
-                                        goto out;
-                                }
-                        }
-                }
+            brick_list = tmpptr;
         }
+        glusterd_brickinfo_delete(brick_info);
+        brick_info = NULL;
+    }
 
-        ret = dict_set_int32 (rsp_dict, "brick_count", local_brick_count);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "Failed to set local_brick_count");
-                goto out;
-        }
+    ret = dict_set_int32n(rsp_dict, "brick_count", SLEN("brick_count"),
+                          local_brick_count);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+               "Failed to set local_brick_count");
+        goto out;
+    }
 out:
-        GF_FREE (free_ptr);
-        if (brick_info)
-                glusterd_brickinfo_delete (brick_info);
-
-        if (msg[0] != '\0') {
-                gf_log (this->name, GF_LOG_ERROR, "%s", msg);
-                *op_errstr = gf_strdup (msg);
-        }
-        gf_log (this->name, GF_LOG_DEBUG, "Returning %d", ret);
-
-        return ret;
+    GF_FREE(free_ptr);
+    if (brick_info)
+        glusterd_brickinfo_delete(brick_info);
+
+    if (msg[0] != '\0') {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_OP_STAGE_CREATE_VOL_FAIL,
+               "%s", msg);
+        *op_errstr = gf_strdup(msg);
+    }
+    gf_msg_debug(this->name, 0, "Returning %d", ret);
+
+    return ret;
 }
 
 int
-glusterd_op_stop_volume_args_get (dict_t *dict, char** volname, int *flags)
+glusterd_op_stop_volume_args_get(dict_t *dict, char **volname, int *flags)
 {
-        int ret = -1;
-        xlator_t *this = NULL;
+    int ret = -1;
+    xlator_t *this = NULL;
 
-        this = THIS;
-        GF_ASSERT (this);
+    this = THIS;
+    GF_ASSERT(this);
 
-        if (!dict || !volname || !flags)
-                goto out;
+    if (!dict) {
+        gf_smsg(this->name, GF_LOG_ERROR, 0, GD_MSG_INVALID_ARGUMENT, NULL);
+        goto out;
+    }
 
-        ret = dict_get_str (dict, "volname", volname);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "Unable to get volume name");
-                goto out;
-        }
+    if (!volname) {
+        gf_smsg(this->name, GF_LOG_ERROR, 0, GD_MSG_INVALID_ARGUMENT, NULL);
+        goto out;
+    }
 
-        ret = dict_get_int32 (dict, "flags", flags);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "Unable to get flags");
-                goto out;
-        }
+    if (!flags) {
+        gf_smsg(this->name, GF_LOG_ERROR, 0, GD_MSG_INVALID_ARGUMENT, NULL);
+        goto out;
+    }
+
+    ret = dict_get_strn(dict, "volname", SLEN("volname"), volname);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+                "Key=volname", NULL);
+        goto out;
+    }
+
+    ret = dict_get_int32n(dict, "flags", SLEN("flags"), flags);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+                "Key=flags", NULL);
+        goto out;
+    }
 out:
-        return ret;
+    return ret;
 }
 
 int
-glusterd_op_statedump_volume_args_get (dict_t *dict, char **volname,
-                                       char **options, int *option_cnt)
+glusterd_op_statedump_volume_args_get(dict_t *dict, char **volname,
+                                      char **options, int *option_cnt)
 {
-        int ret = -1;
+    int ret = -1;
 
-        if (!dict || !volname || !options || !option_cnt)
-                goto out;
+    if (!dict || !volname || !options || !option_cnt) {
+        gf_smsg("glusterd", GF_LOG_ERROR, errno, GD_MSG_INVALID_ARGUMENT, NULL);
+        goto out;
+    }
 
-        ret = dict_get_str (dict, "volname", volname);
-        if (ret) {
-                gf_log ("", GF_LOG_ERROR, "Unable to get volname");
-                goto out;
-        }
+    ret = dict_get_strn(dict, "volname", SLEN("volname"), volname);
+    if (ret) {
+        gf_smsg("glusterd", GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+                "Key=volname", NULL);
+        goto out;
+    }
 
-        ret = dict_get_str (dict, "options", options);
-        if (ret) {
-                gf_log ("", GF_LOG_ERROR, "Unable to get options");
-                goto out;
-        }
+    ret = dict_get_strn(dict, "options", SLEN("options"), options);
+    if (ret) {
+        gf_smsg("glusterd", GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+                "Key=options", NULL);
+        goto out;
+    }
 
-        ret = dict_get_int32 (dict, "option_cnt", option_cnt);
-        if (ret) {
-                gf_log ("", GF_LOG_ERROR, "Unable to get option count");
-                goto out;
-        }
+    ret = dict_get_int32n(dict, "option_cnt", SLEN("option_cnt"), option_cnt);
+    if (ret) {
+        gf_smsg("glusterd", GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+                "Key=option_cnt", NULL);
+        goto out;
+    }
 
 out:
-        return ret;
+    return ret;
 }
 
 int
-glusterd_op_stage_start_volume (dict_t *dict, char **op_errstr,
-                                dict_t *rsp_dict)
+glusterd_op_stage_start_volume(dict_t *dict, char **op_errstr, dict_t *rsp_dict)
 {
-        int                                     ret = 0;
-        char                                    *volname = NULL;
-        char                                    key[PATH_MAX] = "";
-        int                                     flags = 0;
-        int32_t                                 brick_count = 0;
-        int32_t                                 local_brick_count = 0;
-        gf_boolean_t                            exists = _gf_false;
-        glusterd_volinfo_t                      *volinfo = NULL;
-        glusterd_brickinfo_t                    *brickinfo = NULL;
-        char                                    msg[2048] = {0,};
-        glusterd_conf_t                         *priv = NULL;
-        xlator_t                                *this = NULL;
-        uuid_t                                  volume_id = {0,};
-        char                                    volid[50] = {0,};
-        char                                    xattr_volid[50] = {0,};
-        int                                     caps = 0;
-
-        this = THIS;
-        GF_ASSERT (this);
-        priv = this->private;
-        GF_ASSERT (priv);
-        GF_ASSERT (rsp_dict);
-
-        ret = glusterd_op_start_volume_args_get (dict, &volname, &flags);
-        if (ret)
-                goto out;
+    int ret = 0;
+    char *volname = NULL;
+    char key[64] = "";
+    int flags = 0;
+    int32_t brick_count = 0;
+    int32_t local_brick_count = 0;
+    glusterd_volinfo_t *volinfo = NULL;
+    glusterd_brickinfo_t *brickinfo = NULL;
+    char msg[2048] = {
+        0,
+    };
+    glusterd_conf_t *priv = NULL;
+    xlator_t *this = NULL;
+    uuid_t volume_id = {
+        0,
+    };
+    char volid[50] = {
+        0,
+    };
+    char xattr_volid[50] = {
+        0,
+    };
+    int32_t len = 0;
+
+    this = THIS;
+    GF_ASSERT(this);
+    priv = this->private;
+    GF_ASSERT(priv);
+    GF_ASSERT(rsp_dict);
+
+    ret = glusterd_op_start_volume_args_get(dict, &volname, &flags);
+    if (ret)
+        goto out;
 
-        exists = glusterd_check_volume_exists (volname);
+    ret = glusterd_volinfo_find(volname, &volinfo);
+    if (ret) {
+        snprintf(msg, sizeof(msg), FMTSTR_CHECK_VOL_EXISTS, volname);
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_VOLINFO_GET_FAIL,
+               FMTSTR_CHECK_VOL_EXISTS, volname);
+        goto out;
+    }
 
-        if (!exists) {
-                snprintf (msg, sizeof (msg), FMTSTR_CHECK_VOL_EXISTS, volname);
-                ret = -1;
-                goto out;
-        }
+    /* This is an incremental approach to have all the volinfo objects ref
+     * count. The first attempt is made in volume start transaction to
+     * ensure it doesn't race with import volume where stale volume is
+     * deleted. There are multiple instances of GlusterD crashing in
+     * bug-948686.t because of this. Once this approach is full proof, all
+     * other volinfo objects will be refcounted.
+     */
+    glusterd_volinfo_ref(volinfo);
 
-        ret  = glusterd_volinfo_find (volname, &volinfo);
+    if (priv->op_version > GD_OP_VERSION_3_7_5) {
+        ret = glusterd_validate_quorum(this, GD_OP_START_VOLUME, dict,
+                                       op_errstr);
         if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, FMTSTR_CHECK_VOL_EXISTS,
-                        volname);
-                goto out;
+            gf_msg(this->name, GF_LOG_CRITICAL, 0, GD_MSG_SERVER_QUORUM_NOT_MET,
+                   "Server quorum not met. Rejecting operation.");
+            goto out;
         }
+    }
 
-        ret = glusterd_validate_volume_id (dict, volinfo);
-        if (ret)
-                goto out;
+    ret = glusterd_validate_volume_id(dict, volinfo);
+    if (ret)
+        goto out;
 
-        if (!(flags & GF_CLI_FLAG_OP_FORCE)) {
-                if (glusterd_is_volume_started (volinfo)) {
-                        snprintf (msg, sizeof (msg), "Volume %s already "
-                                  "started", volname);
-                        ret = -1;
-                        goto out;
+    if (!(flags & GF_CLI_FLAG_OP_FORCE)) {
+        if (glusterd_is_volume_started(volinfo)) {
+            snprintf(msg, sizeof(msg),
+                     "Volume %s already "
+                     "started",
+                     volname);
+            ret = -1;
+            goto out;
+        }
+    }
+
+    cds_list_for_each_entry(brickinfo, &volinfo->bricks, brick_list)
+    {
+        brick_count++;
+        ret = glusterd_resolve_brick(brickinfo);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_RESOLVE_BRICK_FAIL,
+                   FMTSTR_RESOLVE_BRICK, brickinfo->hostname, brickinfo->path);
+            goto out;
+        }
+
+        if ((gf_uuid_compare(brickinfo->uuid, MY_UUID)) ||
+            (brickinfo->snap_status == -1))
+            continue;
+
+        ret = gf_lstat_dir(brickinfo->path, NULL);
+        if (ret && (flags & GF_CLI_FLAG_OP_FORCE)) {
+            continue;
+        } else if (ret) {
+            len = snprintf(msg, sizeof(msg),
+                           "Failed to find "
+                           "brick directory %s for volume %s. "
+                           "Reason : %s",
+                           brickinfo->path, volname, strerror(errno));
+            if (len < 0) {
+                strcpy(msg, "<error>");
+            }
+            goto out;
+        }
+        ret = sys_lgetxattr(brickinfo->path, GF_XATTR_VOL_ID_KEY, volume_id,
+                            16);
+        if (ret < 0 && (!(flags & GF_CLI_FLAG_OP_FORCE))) {
+            len = snprintf(msg, sizeof(msg),
+                           "Failed to get "
+                           "extended attribute %s for brick dir "
+                           "%s. Reason : %s",
+                           GF_XATTR_VOL_ID_KEY, brickinfo->path,
+                           strerror(errno));
+            if (len < 0) {
+                strcpy(msg, "<error>");
+            }
+            ret = -1;
+            goto out;
+        } else if (ret < 0) {
+            ret = sys_lsetxattr(brickinfo->path, GF_XATTR_VOL_ID_KEY,
+                                volinfo->volume_id, 16, XATTR_CREATE);
+            if (ret == -1) {
+                len = snprintf(msg, sizeof(msg),
+                               "Failed to "
+                               "set extended attribute %s on "
+                               "%s. Reason: %s",
+                               GF_XATTR_VOL_ID_KEY, brickinfo->path,
+                               strerror(errno));
+                if (len < 0) {
+                    strcpy(msg, "<error>");
                 }
+                goto out;
+            } else {
+                continue;
+            }
+        }
+        if (gf_uuid_compare(volinfo->volume_id, volume_id)) {
+            len = snprintf(msg, sizeof(msg),
+                           "Volume id "
+                           "mismatch for brick %s:%s. Expected "
+                           "volume id %s, volume id %s found",
+                           brickinfo->hostname, brickinfo->path,
+                           uuid_utoa_r(volinfo->volume_id, volid),
+                           uuid_utoa_r(volume_id, xattr_volid));
+            if (len < 0) {
+                strcpy(msg, "<error>");
+            }
+            ret = -1;
+            goto out;
         }
 
-        cds_list_for_each_entry (brickinfo, &volinfo->bricks, brick_list) {
-                brick_count++;
-                ret = glusterd_resolve_brick (brickinfo);
+        /* A bricks mount dir is required only by snapshots which were
+         * introduced in gluster-3.6.0
+         */
+        if (priv->op_version >= GD_OP_VERSION_3_6_0) {
+            if (strlen(brickinfo->mount_dir) < 1) {
+                ret = glusterd_get_brick_mount_dir(
+                    brickinfo->path, brickinfo->hostname, brickinfo->mount_dir);
                 if (ret) {
-                        gf_log (this->name, GF_LOG_ERROR, FMTSTR_RESOLVE_BRICK,
-                                brickinfo->hostname, brickinfo->path);
-                        goto out;
-                }
-
-                if ((gf_uuid_compare (brickinfo->uuid, MY_UUID)) ||
-                    (brickinfo->snap_status == -1))
-                        continue;
-
-                ret = gf_lstat_dir (brickinfo->path, NULL);
-                if (ret && (flags & GF_CLI_FLAG_OP_FORCE)) {
-                        continue;
-                } else if (ret) {
-                        snprintf (msg, sizeof (msg), "Failed to find "
-                                          "brick directory %s for volume %s. "
-                                          "Reason : %s", brickinfo->path,
-                                          volname, strerror (errno));
-                        goto out;
-                }
-                ret = sys_lgetxattr (brickinfo->path, GF_XATTR_VOL_ID_KEY,
-                                     volume_id, 16);
-                if (ret < 0 && (!(flags & GF_CLI_FLAG_OP_FORCE))) {
-                        snprintf (msg, sizeof (msg), "Failed to get "
-                                  "extended attribute %s for brick dir %s. "
-                                  "Reason : %s", GF_XATTR_VOL_ID_KEY,
-                                  brickinfo->path, strerror (errno));
-                        ret = -1;
-                        goto out;
-                } else if (ret < 0) {
-                        ret = sys_lsetxattr (brickinfo->path,
-                                             GF_XATTR_VOL_ID_KEY,
-                                             volinfo->volume_id, 16,
-                                             XATTR_CREATE);
-                        if (ret == -1) {
-                                snprintf (msg, sizeof (msg), "Failed to set "
-                                        "extended attribute %s on %s. Reason: "
-                                        "%s", GF_XATTR_VOL_ID_KEY,
-                                        brickinfo->path, strerror (errno));
-                                goto out;
-                        } else {
-                                continue;
-                        }
-                }
-                if (gf_uuid_compare (volinfo->volume_id, volume_id)) {
-                        snprintf (msg, sizeof (msg), "Volume id mismatch for "
-                                  "brick %s:%s. Expected volume id %s, "
-                                  "volume id %s found", brickinfo->hostname,
-                                  brickinfo->path,
-                                  uuid_utoa_r (volinfo->volume_id, volid),
-                                  uuid_utoa_r (volume_id, xattr_volid));
-                        ret = -1;
-                        goto out;
+                    gf_msg(this->name, GF_LOG_ERROR, 0,
+                           GD_MSG_BRICK_MOUNTDIR_GET_FAIL,
+                           "Failed to get brick mount_dir");
+                    goto out;
                 }
 
-                /* A bricks mount dir is required only by snapshots which were
-                 * introduced in gluster-3.6.0
-                 */
-                if (priv->op_version >= GD_OP_VERSION_3_6_0) {
-                        if (strlen(brickinfo->mount_dir) < 1) {
-                                ret = glusterd_get_brick_mount_dir
-                                        (brickinfo->path, brickinfo->hostname,
-                                         brickinfo->mount_dir);
-                                if (ret) {
-                                        gf_log (this->name, GF_LOG_ERROR,
-                                                "Failed to get brick mount_dir");
-                                        goto out;
-                                }
-
-                                snprintf (key, sizeof(key), "brick%d.mount_dir",
-                                          brick_count);
-                                ret = dict_set_dynstr_with_alloc
-                                        (rsp_dict, key, brickinfo->mount_dir);
-                                if (ret) {
-                                        gf_log (this->name, GF_LOG_ERROR,
-                                                "Failed to set %s", key);
-                                        goto out;
-                                }
-                                local_brick_count = brick_count;
-                        }
+                snprintf(key, sizeof(key), "brick%d.mount_dir", brick_count);
+                ret = dict_set_dynstr_with_alloc(rsp_dict, key,
+                                                 brickinfo->mount_dir);
+                if (ret) {
+                    gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+                           "Failed to set %s", key);
+                    goto out;
                 }
-
-#ifdef HAVE_BD_XLATOR
-                if (brickinfo->vg[0])
-                        caps = CAPS_BD | CAPS_THIN |
-                                CAPS_OFFLOAD_COPY | CAPS_OFFLOAD_SNAPSHOT;
-                /* Check for VG/thin pool if its BD volume */
-                if (brickinfo->vg[0]) {
-                        ret = glusterd_is_valid_vg (brickinfo, 0, msg);
-                        if (ret)
-                                goto out;
-                        /* if anyone of the brick does not have thin support,
-                           disable it for entire volume */
-                        caps &= brickinfo->caps;
-                } else
-                        caps = 0;
-#endif
+                local_brick_count = brick_count;
+            }
         }
+    }
 
-        ret = dict_set_int32 (rsp_dict, "brick_count", local_brick_count);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "Failed to set local_brick_count");
-                goto out;
-        }
+    ret = dict_set_int32n(rsp_dict, "brick_count", SLEN("brick_count"),
+                          local_brick_count);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+               "Failed to set local_brick_count");
+        goto out;
+    }
 
-        volinfo->caps = caps;
-        ret = 0;
+    ret = 0;
 out:
-        if (ret && (msg[0] != '\0')) {
-                gf_log (this->name, GF_LOG_ERROR, "%s", msg);
-                *op_errstr = gf_strdup (msg);
-        }
-        return ret;
+    if (volinfo)
+        glusterd_volinfo_unref(volinfo);
+
+    if (ret && (msg[0] != '\0')) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_OP_STAGE_START_VOL_FAIL,
+               "%s", msg);
+        *op_errstr = gf_strdup(msg);
+    }
+    return ret;
 }
 
 int
-glusterd_op_stage_stop_volume (dict_t *dict, char **op_errstr)
+glusterd_op_stage_stop_volume(dict_t *dict, char **op_errstr)
 {
-        int                                     ret = -1;
-        char                                    *volname = NULL;
-        int                                     flags = 0;
-        gf_boolean_t                            exists = _gf_false;
-        glusterd_volinfo_t                      *volinfo = NULL;
-        char                                    msg[2048] = {0};
-        xlator_t                                *this = NULL;
-        gsync_status_param_t                    param = {0,};
-
-        this = THIS;
-        GF_ASSERT (this);
-
-        ret = glusterd_op_stop_volume_args_get (dict, &volname, &flags);
-        if (ret)
-                goto out;
-
-        exists = glusterd_check_volume_exists (volname);
-
-        if (!exists) {
-                snprintf (msg, sizeof (msg), FMTSTR_CHECK_VOL_EXISTS, volname);
-                gf_log (this->name, GF_LOG_ERROR, "%s", msg);
-                ret = -1;
-                goto out;
-        }
-
-        ret  = glusterd_volinfo_find (volname, &volinfo);
-        if (ret) {
-                snprintf (msg, sizeof (msg), FMTSTR_CHECK_VOL_EXISTS, volname);
-                gf_log (this->name, GF_LOG_ERROR, "%s", msg);
-                goto out;
-        }
-
-        ret = glusterd_validate_volume_id (dict, volinfo);
-        if (ret)
-                goto out;
+    int ret = -1;
+    char *volname = NULL;
+    int flags = 0;
+    glusterd_volinfo_t *volinfo = NULL;
+    char msg[2048] = {0};
+    xlator_t *this = NULL;
+    gsync_status_param_t param = {
+        0,
+    };
+
+    this = THIS;
+    GF_ASSERT(this);
+
+    ret = glusterd_op_stop_volume_args_get(dict, &volname, &flags);
+    if (ret) {
+        snprintf(msg, sizeof(msg), "Failed to get details of volume %s",
+                 volname);
+        gf_smsg(this->name, GF_LOG_ERROR, 0, GD_MSG_VOL_STOP_ARGS_GET_FAILED,
+                "Volume name=%s", volname, NULL);
+        goto out;
+    }
 
-        /* If 'force' flag is given, no check is required */
-        if (flags & GF_CLI_FLAG_OP_FORCE)
-                goto out;
+    ret = glusterd_volinfo_find(volname, &volinfo);
+    if (ret) {
+        snprintf(msg, sizeof(msg), FMTSTR_CHECK_VOL_EXISTS, volname);
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_VOLINFO_GET_FAIL, "%s", msg);
+        goto out;
+    }
 
-        if (_gf_false == glusterd_is_volume_started (volinfo)) {
-                snprintf (msg, sizeof(msg), "Volume %s "
-                          "is not in the started state", volname);
-                gf_log (this->name, GF_LOG_ERROR, "%s", msg);
-                ret = -1;
-                goto out;
-        }
+    ret = glusterd_validate_volume_id(dict, volinfo);
+    if (ret)
+        goto out;
 
-        /* If geo-rep is configured, for this volume, it should be stopped. */
-        param.volinfo = volinfo;
-        ret = glusterd_check_geo_rep_running (&param, op_errstr);
-        if (ret || param.is_active) {
-                ret = -1;
-                goto out;
-        }
+    /* If 'force' flag is given, no check is required */
+    if (flags & GF_CLI_FLAG_OP_FORCE)
+        goto out;
 
-        if (glusterd_is_rb_ongoing (volinfo)) {
-                snprintf (msg, sizeof (msg), "Replace brick is in progress on "
-                          "volume %s. Please retry after replace-brick "
-                          "operation is committed or aborted", volname);
-                gf_log (this->name, GF_LOG_WARNING, "replace-brick in progress "
-                        "on volume %s", volname);
-                ret = -1;
-                goto out;
-        }
+    if (_gf_false == glusterd_is_volume_started(volinfo)) {
+        snprintf(msg, sizeof(msg),
+                 "Volume %s "
+                 "is not in the started state",
+                 volname);
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_VOL_NOT_STARTED, "%s", msg);
+        ret = -1;
+        goto out;
+    }
 
-        if (glusterd_is_defrag_on (volinfo)) {
-                snprintf (msg, sizeof(msg), "rebalance session is "
-                          "in progress for the volume '%s'", volname);
-                gf_log (this->name, GF_LOG_WARNING, "%s", msg);
-                ret = -1;
-                goto out;
-        }
-        if (volinfo->rep_brick.rb_status != GF_RB_STATUS_NONE) {
-                snprintf (msg, sizeof(msg), "replace-brick session is "
-                          "in progress for the volume '%s'", volname);
-                gf_log (this->name, GF_LOG_WARNING, "%s", msg);
-                ret = -1;
-                goto out;
-        }
+    /* If geo-rep is configured, for this volume, it should be stopped. */
+    param.volinfo = volinfo;
+    ret = glusterd_check_geo_rep_running(&param, op_errstr);
+    if (ret || param.is_active) {
+        ret = -1;
+        goto out;
+    }
+
+    ret = glusterd_check_ganesha_export(volinfo);
+    if (ret) {
+        ret = ganesha_manage_export(dict, "off", _gf_false, op_errstr);
+        if (ret) {
+            gf_msg(THIS->name, GF_LOG_WARNING, 0,
+                   GD_MSG_NFS_GNS_UNEXPRT_VOL_FAIL,
+                   "Could not "
+                   "unexport volume via NFS-Ganesha");
+            ret = 0;
+        }
+    }
+
+    if (glusterd_is_defrag_on(volinfo)) {
+        snprintf(msg, sizeof(msg),
+                 "rebalance session is "
+                 "in progress for the volume '%s'",
+                 volname);
+        gf_msg(this->name, GF_LOG_WARNING, 0, GD_MSG_OIP, "%s", msg);
+        ret = -1;
+        goto out;
+    }
 
 out:
-        if (msg[0] != 0)
-                *op_errstr = gf_strdup (msg);
-        gf_log (this->name, GF_LOG_DEBUG, "Returning %d", ret);
+    if (msg[0] != 0)
+        *op_errstr = gf_strdup(msg);
+    gf_msg_debug(this->name, 0, "Returning %d", ret);
 
-        return ret;
+    return ret;
 }
 
 int
-glusterd_op_stage_delete_volume (dict_t *dict, char **op_errstr)
+glusterd_op_stage_delete_volume(dict_t *dict, char **op_errstr)
 {
-        int                                     ret = 0;
-        char                                    *volname = NULL;
-        gf_boolean_t                            exists = _gf_false;
-        glusterd_volinfo_t                      *volinfo = NULL;
-        char                                    msg[2048] = {0};
-        xlator_t                                *this = NULL;
-
-        this = THIS;
-        GF_ASSERT (this);
-
-        ret = dict_get_str (dict, "volname", &volname);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "Unable to get volume name");
-                goto out;
-        }
-
-        exists = glusterd_check_volume_exists (volname);
-        if (!exists) {
-                snprintf (msg, sizeof (msg), FMTSTR_CHECK_VOL_EXISTS, volname);
-                ret = -1;
-                goto out;
-        } else {
-                ret = 0;
-        }
-
-        ret  = glusterd_volinfo_find (volname, &volinfo);
-        if (ret) {
-                snprintf (msg, sizeof (msg), FMTSTR_CHECK_VOL_EXISTS, volname);
-                goto out;
-        }
+    int ret = 0;
+    char *volname = NULL;
+    glusterd_volinfo_t *volinfo = NULL;
+    char msg[2048] = {0};
+    xlator_t *this = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+
+    ret = dict_get_strn(dict, "volname", SLEN("volname"), &volname);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+               "Unable to get volume name");
+        goto out;
+    }
 
-        ret = glusterd_validate_volume_id (dict, volinfo);
-        if (ret)
-                goto out;
+    ret = glusterd_volinfo_find(volname, &volinfo);
+    if (ret) {
+        snprintf(msg, sizeof(msg), FMTSTR_CHECK_VOL_EXISTS, volname);
+        goto out;
+    }
 
-        if (glusterd_is_volume_started (volinfo)) {
-                snprintf (msg, sizeof (msg), "Volume %s has been started."
-                          "Volume needs to be stopped before deletion.",
-                          volname);
-                ret = -1;
-                goto out;
-        }
+    ret = glusterd_validate_volume_id(dict, volinfo);
+    if (ret)
+        goto out;
 
-        if (volinfo->snap_count > 0 ||
-            !cds_list_empty (&volinfo->snap_volumes)) {
-                snprintf (msg, sizeof (msg), "Cannot delete Volume %s ,"
-                        "as it has %"PRIu64" snapshots. "
-                        "To delete the volume, "
-                        "first delete all the snapshots under it.",
-                          volname, volinfo->snap_count);
-                ret = -1;
-                goto out;
-        }
+    if (glusterd_is_volume_started(volinfo)) {
+        snprintf(msg, sizeof(msg),
+                 "Volume %s has been started."
+                 "Volume needs to be stopped before deletion.",
+                 volname);
+        ret = -1;
+        goto out;
+    }
+
+    if (volinfo->snap_count > 0 || !cds_list_empty(&volinfo->snap_volumes)) {
+        snprintf(msg, sizeof(msg),
+                 "Cannot delete Volume %s ,"
+                 "as it has %" PRIu64
+                 " snapshots. "
+                 "To delete the volume, "
+                 "first delete all the snapshots under it.",
+                 volname, volinfo->snap_count);
+        ret = -1;
+        goto out;
+    }
 
-        ret = 0;
+    if (!glusterd_are_all_peers_up()) {
+        ret = -1;
+        snprintf(msg, sizeof(msg), "Some of the peers are down");
+        goto out;
+    }
+    volinfo->stage_deleted = _gf_true;
+    gf_log(this->name, GF_LOG_INFO,
+           "Setting stage deleted flag to true for "
+           "volume %s",
+           volinfo->volname);
+    ret = 0;
 
 out:
-        if (msg[0] != '\0') {
-                gf_log (this->name, GF_LOG_ERROR, "%s", msg);
-                *op_errstr = gf_strdup (msg);
-        }
-        gf_log (this->name, GF_LOG_DEBUG, "Returning %d", ret);
-
-        return ret;
+    if (msg[0] != '\0') {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_OP_STAGE_DELETE_VOL_FAIL,
+               "%s", msg);
+        *op_errstr = gf_strdup(msg);
+    }
+    gf_msg_debug(this->name, 0, "Returning %d", ret);
+
+    return ret;
 }
 
 static int
-glusterd_handle_heal_cmd (xlator_t *this, glusterd_volinfo_t *volinfo,
-                          dict_t *dict, char **op_errstr)
+glusterd_handle_heal_cmd(xlator_t *this, glusterd_volinfo_t *volinfo,
+                         dict_t *dict, char **op_errstr)
 {
-        glusterd_conf_t          *priv        = NULL;
-        gf_xl_afr_op_t           heal_op      = GF_SHD_OP_INVALID;
-        int                      ret          = 0;
-        char                     msg[2408]    = {0,};
-        char                     *offline_msg = "Self-heal daemon is not running. "
-                                      "Check self-heal daemon log file.";
-
-        priv = this->private;
-        ret = dict_get_int32 (dict, "heal-op", (int32_t*)&heal_op);
-        if (ret) {
-                ret = -1;
-                *op_errstr = gf_strdup("Heal operation not specified");
-                goto out;
-        }
+    glusterd_svc_t *svc = NULL;
+    gf_xl_afr_op_t heal_op = GF_SHD_OP_INVALID;
+    int ret = 0;
+    char msg[2408] = {
+        0,
+    };
+    char *offline_msg =
+        "Self-heal daemon is not running. "
+        "Check self-heal daemon log file.";
+
+    ret = dict_get_int32n(dict, "heal-op", SLEN("heal-op"),
+                          (int32_t *)&heal_op);
+    if (ret) {
+        ret = -1;
+        *op_errstr = gf_strdup("Heal operation not specified");
+        goto out;
+    }
 
-        switch (heal_op) {
+    svc = &(volinfo->shd.svc);
+    switch (heal_op) {
         case GF_SHD_OP_INVALID:
         case GF_SHD_OP_HEAL_ENABLE: /* This op should be handled in volume-set*/
-        case GF_SHD_OP_HEAL_DISABLE:/* This op should be handled in volume-set*/
-        case GF_SHD_OP_SBRAIN_HEAL_FROM_BIGGER_FILE:/*glfsheal cmd*/
-        case GF_SHD_OP_SBRAIN_HEAL_FROM_BRICK:/*glfsheal cmd*/
-                ret = -1;
-                *op_errstr = gf_strdup("Invalid heal-op");
-                goto out;
+        case GF_SHD_OP_HEAL_DISABLE: /* This op should be handled in
+                                        volume-set*/
+        case GF_SHD_OP_GRANULAR_ENTRY_HEAL_ENABLE:  /* This op should be handled
+                                                       in volume-set */
+        case GF_SHD_OP_GRANULAR_ENTRY_HEAL_DISABLE: /* This op should be handled
+                                                       in volume-set */
+        case GF_SHD_OP_HEAL_SUMMARY:                /*glfsheal cmd*/
+        case GF_SHD_OP_SBRAIN_HEAL_FROM_BIGGER_FILE:  /*glfsheal cmd*/
+        case GF_SHD_OP_SBRAIN_HEAL_FROM_LATEST_MTIME: /*glfsheal cmd*/
+        case GF_SHD_OP_SBRAIN_HEAL_FROM_BRICK:        /*glfsheal cmd*/
+            ret = -1;
+            *op_errstr = gf_strdup("Invalid heal-op");
+            goto out;
 
         case GF_SHD_OP_HEAL_INDEX:
         case GF_SHD_OP_HEAL_FULL:
-                if (!glusterd_is_shd_compatible_volume (volinfo)) {
-                        ret = -1;
-                        snprintf (msg, sizeof (msg), "Volume %s is not of type "
-                                  "replicate or disperse", volinfo->volname);
-                        *op_errstr = gf_strdup (msg);
-                        goto out;
-                }
+            if (!glusterd_is_shd_compatible_volume(volinfo)) {
+                ret = -1;
+                snprintf(msg, sizeof(msg),
+                         "Volume %s is not of type "
+                         "replicate or disperse",
+                         volinfo->volname);
+                *op_errstr = gf_strdup(msg);
+                goto out;
+            }
 
-                if (!priv->shd_svc.online) {
-                        ret = -1;
-                        *op_errstr = gf_strdup (offline_msg);
-                        goto out;
-                }
-                break;
+            if (!svc->online) {
+                ret = -1;
+                *op_errstr = gf_strdup(offline_msg);
+                goto out;
+            }
+            break;
         case GF_SHD_OP_INDEX_SUMMARY:
         case GF_SHD_OP_SPLIT_BRAIN_FILES:
         case GF_SHD_OP_STATISTICS:
         case GF_SHD_OP_STATISTICS_HEAL_COUNT:
         case GF_SHD_OP_STATISTICS_HEAL_COUNT_PER_REPLICA:
-                if (!glusterd_is_volume_replicate (volinfo)) {
-                        ret = -1;
-                        snprintf (msg, sizeof (msg), "Volume %s is not of type "
-                                  "replicate", volinfo->volname);
-                        *op_errstr = gf_strdup (msg);
-                        goto out;
-                }
-
-                if (!priv->shd_svc.online) {
-                        ret = -1;
-                        *op_errstr = gf_strdup (offline_msg);
-                        goto out;
-                }
-                break;
-        case GF_SHD_OP_HEALED_FILES:
-        case GF_SHD_OP_HEAL_FAILED_FILES:
+            if (!glusterd_is_volume_replicate(volinfo)) {
                 ret = -1;
-                snprintf (msg, sizeof (msg), "Command not supported. "
-                          "Please use \"gluster volume heal %s info\" "
-                          "and logs to find the heal information.",
-                          volinfo->volname);
-                *op_errstr = gf_strdup (msg);
+                snprintf(msg, sizeof(msg),
+                         "This command is supported "
+                         "for only volume of replicated "
+                         "type. Volume %s is not of type "
+                         "replicate",
+                         volinfo->volname);
+                *op_errstr = gf_strdup(msg);
                 goto out;
+            }
 
-        }
+            if (!svc->online) {
+                ret = -1;
+                *op_errstr = gf_strdup(offline_msg);
+                goto out;
+            }
+            break;
+        case GF_SHD_OP_HEALED_FILES:
+        case GF_SHD_OP_HEAL_FAILED_FILES:
+            ret = -1;
+            snprintf(msg, sizeof(msg),
+                     "Command not supported. "
+                     "Please use \"gluster volume heal %s info\" "
+                     "and logs to find the heal information.",
+                     volinfo->volname);
+            *op_errstr = gf_strdup(msg);
+            goto out;
+    }
 out:
-        if (ret)
-                gf_log (this->name, GF_LOG_WARNING, "%s", *op_errstr);
-        return ret;
+    if (ret)
+        gf_msg(this->name, GF_LOG_WARNING, 0, GD_MSG_HANDLE_HEAL_CMD_FAIL, "%s",
+               *op_errstr);
+    return ret;
 }
 
 int
-glusterd_op_stage_heal_volume (dict_t *dict, char **op_errstr)
+glusterd_op_stage_heal_volume(dict_t *dict, char **op_errstr)
 {
-        int                                     ret = 0;
-        char                                    *volname = NULL;
-        gf_boolean_t                            enabled = _gf_false;
-        glusterd_volinfo_t                      *volinfo = NULL;
-        char                                    msg[2048];
-        glusterd_conf_t                         *priv = NULL;
-        dict_t                                  *opt_dict = NULL;
-        xlator_t                                *this = NULL;
-
-        this = THIS;
-        GF_ASSERT (this);
-
-        priv = this->private;
-        if (!priv) {
-                ret = -1;
-                gf_log (this->name, GF_LOG_ERROR,
-                        "priv is NULL");
-                goto out;
-        }
-
-        ret = dict_get_str (dict, "volname", &volname);
-        if (ret) {
-                gf_log ("", GF_LOG_ERROR, "Unable to get volume name");
-                goto out;
-        }
+    int ret = 0;
+    char *volname = NULL;
+    gf_boolean_t enabled = _gf_false;
+    glusterd_volinfo_t *volinfo = NULL;
+    char msg[2048];
+    glusterd_conf_t *priv = NULL;
+    dict_t *opt_dict = NULL;
+    xlator_t *this = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+
+    priv = this->private;
+    if (!priv) {
+        ret = -1;
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_PRIV_NULL, "priv is NULL");
+        goto out;
+    }
 
-        ret  = glusterd_volinfo_find (volname, &volinfo);
-        if (ret) {
-                ret = -1;
-                snprintf (msg, sizeof (msg), "Volume %s does not exist", volname);
-                gf_log (this->name, GF_LOG_ERROR, "%s", msg);
-                *op_errstr = gf_strdup (msg);
-                goto out;
-        }
+    ret = dict_get_strn(dict, "volname", SLEN("volname"), &volname);
+    if (ret) {
+        gf_msg("glusterd", GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+               "Unable to get volume name");
+        goto out;
+    }
 
-        ret = glusterd_validate_volume_id (dict, volinfo);
-        if (ret)
-                goto out;
+    ret = glusterd_volinfo_find(volname, &volinfo);
+    if (ret) {
+        ret = -1;
+        snprintf(msg, sizeof(msg), "Volume %s does not exist", volname);
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_VOL_NOT_FOUND, "%s", msg);
+        *op_errstr = gf_strdup(msg);
+        goto out;
+    }
 
-        if (!glusterd_is_volume_started (volinfo)) {
-                ret = -1;
-                snprintf (msg, sizeof (msg), "Volume %s is not started.",
-                          volname);
-                gf_log (THIS->name, GF_LOG_WARNING, "%s", msg);
-                *op_errstr = gf_strdup (msg);
-                goto out;
-        }
+    ret = glusterd_validate_volume_id(dict, volinfo);
+    if (ret)
+        goto out;
 
-        opt_dict = volinfo->dict;
-        if (!opt_dict) {
-                ret = 0;
-                goto out;
-        }
+    if (!glusterd_is_volume_started(volinfo)) {
+        ret = -1;
+        snprintf(msg, sizeof(msg), "Volume %s is not started.", volname);
+        gf_smsg(this->name, GF_LOG_WARNING, 0, GD_MSG_VOL_NOT_STARTED,
+                "Volume=%s", volname, NULL);
+        *op_errstr = gf_strdup(msg);
+        goto out;
+    }
 
-        enabled = dict_get_str_boolean (opt_dict, volgen_get_shd_key (volinfo),
-                                        1);
-        if (!enabled) {
-                ret = -1;
-                snprintf (msg, sizeof (msg), "Self-heal-daemon is "
-                          "disabled. Heal will not be triggered on volume %s",
-                          volname);
-                gf_log (this->name, GF_LOG_WARNING, "%s", msg);
-                *op_errstr = gf_strdup (msg);
-                goto out;
-        }
+    opt_dict = volinfo->dict;
+    if (!opt_dict) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_GET_FAILED, NULL);
+        ret = 0;
+        goto out;
+    }
+    enabled = gd_is_self_heal_enabled(volinfo, opt_dict);
+    if (!enabled) {
+        ret = -1;
+        snprintf(msg, sizeof(msg),
+                 "Self-heal-daemon is "
+                 "disabled. Heal will not be triggered on volume %s",
+                 volname);
+        gf_msg(this->name, GF_LOG_WARNING, 0, GD_MSG_SELF_HEALD_DISABLED, "%s",
+               msg);
+        *op_errstr = gf_strdup(msg);
+        goto out;
+    }
 
-        ret = glusterd_handle_heal_cmd (this, volinfo, dict, op_errstr);
-        if (ret)
-                goto out;
+    ret = glusterd_handle_heal_cmd(this, volinfo, dict, op_errstr);
+    if (ret)
+        goto out;
 
-        ret = 0;
+    ret = 0;
 out:
-        gf_log ("", GF_LOG_DEBUG, "Returning %d", ret);
+    gf_msg_debug("glusterd", 0, "Returning %d", ret);
 
-        return ret;
+    return ret;
 }
 
 int
-glusterd_op_stage_statedump_volume (dict_t *dict, char **op_errstr)
+glusterd_op_stage_statedump_volume(dict_t *dict, char **op_errstr)
 {
-        int                     ret = -1;
-        char                    *volname = NULL;
-        char                    *options = NULL;
-        int                     option_cnt = 0;
-        gf_boolean_t            is_running = _gf_false;
-        glusterd_volinfo_t      *volinfo = NULL;
-        char                    msg[2408] = {0,};
-        xlator_t                *this     = NULL;
-        glusterd_conf_t         *priv     = NULL;
-
-        this = THIS;
-        GF_ASSERT (this);
-        priv = this->private;
-        GF_ASSERT (priv);
-
-        ret = glusterd_op_statedump_volume_args_get (dict, &volname, &options,
-                                                     &option_cnt);
-        if (ret)
-                goto out;
+    int ret = -1;
+    char *volname = NULL;
+    char *options = NULL;
+    int option_cnt = 0;
+    gf_boolean_t is_running = _gf_false;
+    glusterd_volinfo_t *volinfo = NULL;
+    char msg[2408] = {
+        0,
+    };
+    xlator_t *this = NULL;
+    glusterd_conf_t *priv = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+    priv = this->private;
+    GF_ASSERT(priv);
+
+    ret = glusterd_op_statedump_volume_args_get(dict, &volname, &options,
+                                                &option_cnt);
+    if (ret)
+        goto out;
 
-        ret = glusterd_volinfo_find (volname, &volinfo);
-        if (ret) {
-                snprintf (msg, sizeof(msg), FMTSTR_CHECK_VOL_EXISTS, volname);
-                goto out;
-        }
+    ret = glusterd_volinfo_find(volname, &volinfo);
+    if (ret) {
+        snprintf(msg, sizeof(msg), FMTSTR_CHECK_VOL_EXISTS, volname);
+        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_VOLINFO_GET_FAIL,
+                "Volume=%s", volname, NULL);
+        goto out;
+    }
 
-        ret = glusterd_validate_volume_id (dict, volinfo);
-        if (ret)
-                goto out;
+    ret = glusterd_validate_volume_id(dict, volinfo);
+    if (ret)
+        goto out;
 
-        is_running = glusterd_is_volume_started (volinfo);
-        if (!is_running) {
-                snprintf (msg, sizeof(msg), "Volume %s is not in the started"
-                          " state", volname);
-                ret = -1;
-                goto out;
-        }
+    is_running = glusterd_is_volume_started(volinfo);
+    if (!is_running) {
+        snprintf(msg, sizeof(msg),
+                 "Volume %s is not in the started"
+                 " state",
+                 volname);
+        ret = -1;
+        goto out;
+    }
 
-        if (priv->op_version == GD_OP_VERSION_MIN &&
-            strstr (options, "quotad")) {
-                snprintf (msg, sizeof (msg), "The cluster is operating "
-                          "at op-version 1. Taking quotad's statedump is "
-                          "disallowed in this state");
-                ret = -1;
-                goto out;
-        }
-        if ((strstr (options, "quotad")) &&
-            (!glusterd_is_volume_quota_enabled (volinfo))) {
-                    snprintf (msg, sizeof (msg), "Quota is not enabled on "
-                              "volume %s", volname);
-                    ret = -1;
-                    goto out;
-        }
+    if (priv->op_version == GD_OP_VERSION_MIN && strstr(options, "quotad")) {
+        snprintf(msg, sizeof(msg),
+                 "The cluster is operating "
+                 "at op-version 1. Taking quotad's statedump is "
+                 "disallowed in this state");
+        ret = -1;
+        goto out;
+    }
+    if ((strstr(options, "quotad")) &&
+        (!glusterd_is_volume_quota_enabled(volinfo))) {
+        snprintf(msg, sizeof(msg),
+                 "Quota is not enabled on "
+                 "volume %s",
+                 volname);
+        ret = -1;
+        goto out;
+    }
 out:
-        if (ret && msg[0] != '\0')
-                *op_errstr = gf_strdup (msg);
-        gf_log (this->name, GF_LOG_DEBUG, "Returning %d", ret);
-        return ret;
+    if (ret && msg[0] != '\0')
+        *op_errstr = gf_strdup(msg);
+    gf_msg_debug(this->name, 0, "Returning %d", ret);
+    return ret;
 }
 
 int
-glusterd_op_stage_clearlocks_volume (dict_t *dict, char **op_errstr)
+glusterd_op_stage_clearlocks_volume(dict_t *dict, char **op_errstr)
 {
-        int                     ret = -1;
-        char                    *volname = NULL;
-        char                    *path    = NULL;
-        char                    *type    = NULL;
-        char                    *kind    = NULL;
-        glusterd_volinfo_t      *volinfo = NULL;
-        char                    msg[2048] = {0,};
-
-        ret = dict_get_str (dict, "volname", &volname);
-        if (ret) {
-                snprintf (msg, sizeof(msg), "Failed to get volume name");
-                gf_log (THIS->name, GF_LOG_ERROR, "%s", msg);
-                *op_errstr = gf_strdup (msg);
-                goto out;
-        }
+    int ret = -1;
+    char *volname = NULL;
+    char *path = NULL;
+    char *type = NULL;
+    char *kind = NULL;
+    glusterd_volinfo_t *volinfo = NULL;
+    char msg[2048] = {
+        0,
+    };
+
+    ret = dict_get_strn(dict, "volname", SLEN("volname"), &volname);
+    if (ret) {
+        snprintf(msg, sizeof(msg), "Failed to get volume name");
+        gf_msg(THIS->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED, "%s", msg);
+        *op_errstr = gf_strdup(msg);
+        goto out;
+    }
 
-        ret = dict_get_str (dict, "path", &path);
-        if (ret) {
-                snprintf (msg, sizeof(msg), "Failed to get path");
-                gf_log (THIS->name, GF_LOG_ERROR, "%s", msg);
-                *op_errstr = gf_strdup (msg);
-                goto out;
-        }
+    ret = dict_get_strn(dict, "path", SLEN("path"), &path);
+    if (ret) {
+        snprintf(msg, sizeof(msg), "Failed to get path");
+        gf_msg(THIS->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED, "%s", msg);
+        *op_errstr = gf_strdup(msg);
+        goto out;
+    }
 
-        ret = dict_get_str (dict, "kind", &kind);
-        if (ret) {
-                snprintf (msg, sizeof(msg), "Failed to get kind");
-                gf_log ("", GF_LOG_ERROR, "%s", msg);
-                *op_errstr = gf_strdup (msg);
-                goto out;
-        }
+    ret = dict_get_strn(dict, "kind", SLEN("kind"), &kind);
+    if (ret) {
+        snprintf(msg, sizeof(msg), "Failed to get kind");
+        gf_msg("glusterd", GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED, "%s", msg);
+        *op_errstr = gf_strdup(msg);
+        goto out;
+    }
 
-        ret = dict_get_str (dict, "type", &type);
-        if (ret) {
-                snprintf (msg, sizeof(msg), "Failed to get type");
-                gf_log ("", GF_LOG_ERROR, "%s", msg);
-                *op_errstr = gf_strdup (msg);
-                goto out;
-        }
+    ret = dict_get_strn(dict, "type", SLEN("type"), &type);
+    if (ret) {
+        snprintf(msg, sizeof(msg), "Failed to get type");
+        gf_msg("glusterd", GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED, "%s", msg);
+        *op_errstr = gf_strdup(msg);
+        goto out;
+    }
 
-        ret = glusterd_volinfo_find (volname, &volinfo);
-        if (ret) {
-                snprintf (msg, sizeof(msg), "Volume %s does not exist",
-                          volname);
-                gf_log ("", GF_LOG_ERROR, "%s", msg);
-                *op_errstr = gf_strdup (msg);
-                goto out;
-        }
+    ret = glusterd_volinfo_find(volname, &volinfo);
+    if (ret) {
+        snprintf(msg, sizeof(msg), "Volume %s does not exist", volname);
+        gf_msg("glusterd", GF_LOG_ERROR, 0, GD_MSG_VOL_NOT_FOUND, "%s", msg);
+        *op_errstr = gf_strdup(msg);
+        goto out;
+    }
 
-        ret = glusterd_validate_volume_id (dict, volinfo);
-        if (ret)
-                goto out;
+    ret = glusterd_validate_volume_id(dict, volinfo);
+    if (ret)
+        goto out;
 
-        if (!glusterd_is_volume_started (volinfo)) {
-                snprintf (msg, sizeof(msg), "Volume %s is not started",
-                          volname);
-                gf_log ("", GF_LOG_ERROR, "%s", msg);
-                *op_errstr = gf_strdup (msg);
-                goto out;
-        }
+    if (!glusterd_is_volume_started(volinfo)) {
+        snprintf(msg, sizeof(msg), "Volume %s is not started", volname);
+        gf_msg("glusterd", GF_LOG_ERROR, 0, GD_MSG_VOL_NOT_STARTED, "%s", msg);
+        *op_errstr = gf_strdup(msg);
+        goto out;
+    }
 
-        ret = 0;
+    ret = 0;
 out:
-        gf_log ("", GF_LOG_DEBUG, "Returning %d", ret);
-        return ret;
+    gf_msg_debug("glusterd", 0, "Returning %d", ret);
+    return ret;
 }
 
 int
-glusterd_op_create_volume (dict_t *dict, char **op_errstr)
+glusterd_op_create_volume(dict_t *dict, char **op_errstr)
 {
-        int                   ret        = 0;
-        char                 *volname    = NULL;
-        glusterd_conf_t      *priv       = NULL;
-        glusterd_volinfo_t   *volinfo    = NULL;
-        gf_boolean_t          vol_added = _gf_false;
-        glusterd_brickinfo_t *brickinfo  = NULL;
-        xlator_t             *this       = NULL;
-        char                 *brick      = NULL;
-        int32_t               count      = 0;
-        int32_t               i          = 1;
-        char                 *bricks     = NULL;
-        char                 *brick_list = NULL;
-        char                 *free_ptr   = NULL;
-        char                 *saveptr    = NULL;
-        char                 *trans_type = NULL;
-        char                 *str        = NULL;
-        char                 *username   = NULL;
-        char                 *password   = NULL;
-        int                   caps       = 0;
-        int                   brickid    = 0;
-        char                  msg[1024] __attribute__((unused)) = {0, };
-        char                 *brick_mount_dir = NULL;
-        char                  key[PATH_MAX]   = "";
-
-        this = THIS;
-        GF_ASSERT (this);
-
-        priv = this->private;
-        GF_ASSERT (priv);
-
-        ret = glusterd_volinfo_new (&volinfo);
+    int ret = 0;
+    char *volname = NULL;
+    glusterd_conf_t *priv = NULL;
+    glusterd_volinfo_t *volinfo = NULL;
+    gf_boolean_t vol_added = _gf_false;
+    glusterd_brickinfo_t *brickinfo = NULL;
+    glusterd_brickinfo_t *ta_brickinfo = NULL;
+    xlator_t *this = NULL;
+    char *brick = NULL;
+    char *ta_brick = NULL;
+    int32_t count = 0;
+    int32_t i = 1;
+    char *bricks = NULL;
+    char *ta_bricks = NULL;
+    char *brick_list = NULL;
+    char *ta_brick_list = NULL;
+    char *free_ptr = NULL;
+    char *ta_free_ptr = NULL;
+    char *saveptr = NULL;
+    char *ta_saveptr = NULL;
+    char *trans_type = NULL;
+    char *str = NULL;
+    char *username = NULL;
+    char *password = NULL;
+    int brickid = 0;
+    char msg[1024] __attribute__((unused)) = {
+        0,
+    };
+    char *brick_mount_dir = NULL;
+    char key[64] = "";
+    char *address_family_str = NULL;
+    struct statvfs brickstat = {
+        0,
+    };
+
+    this = THIS;
+    GF_ASSERT(this);
+
+    priv = this->private;
+    GF_ASSERT(priv);
+
+    ret = glusterd_volinfo_new(&volinfo);
+
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, ENOMEM, GD_MSG_NO_MEMORY,
+               "Unable to allocate memory for volinfo");
+        goto out;
+    }
 
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "Unable to allocate memory for volinfo");
-                goto out;
-        }
+    ret = dict_get_strn(dict, "volname", SLEN("volname"), &volname);
 
-        ret = dict_get_str (dict, "volname", &volname);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+               "Unable to get volume name");
+        goto out;
+    }
 
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "Unable to get volume name");
-                goto out;
-        }
+    if (snprintf(volinfo->volname, sizeof(volinfo->volname), "%s", volname) >=
+        sizeof(volinfo->volname)) {
+        ret = -1;
+        goto out;
+    }
 
-        strncpy (volinfo->volname, volname, sizeof(volinfo->volname));
-        GF_ASSERT (volinfo->volname);
+    GF_ASSERT(volinfo->volname);
 
-        ret = dict_get_int32 (dict, "type", &volinfo->type);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "Unable to get type of volume"
-                        " %s", volname);
-                goto out;
-        }
-
-        ret = dict_get_int32 (dict, "count", &volinfo->brick_count);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "Unable to get brick count of"
-                        " volume %s", volname);
-                goto out;
-        }
+    ret = dict_get_int32n(dict, "type", SLEN("type"), &volinfo->type);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+               "Unable to get type of volume"
+               " %s",
+               volname);
+        goto out;
+    }
+
+    ret = dict_get_int32n(dict, "count", SLEN("count"), &volinfo->brick_count);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+               "Unable to get brick count of"
+               " volume %s",
+               volname);
+        goto out;
+    }
 
-        ret = dict_get_int32 (dict, "port", &volinfo->port);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "Unable to get port");
-                goto out;
-        }
+    ret = dict_get_int32n(dict, "port", SLEN("port"), &volinfo->port);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+               "Unable to get port");
+        goto out;
+    }
+
+    ret = dict_get_strn(dict, "bricks", SLEN("bricks"), &bricks);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+               "Unable to get bricks for "
+               "volume %s",
+               volname);
+        goto out;
+    }
+
+    /* replica-count 1 means, no replication, file is in one brick only */
+    volinfo->replica_count = 1;
+    /* stripe-count 1 means, no striping, file is present as a whole */
+    volinfo->stripe_count = 1;
+
+    if (GF_CLUSTER_TYPE_REPLICATE == volinfo->type) {
+        /* performance.client-io-threads is turned on to default,
+         * however this has adverse effects on replicate volumes due to
+         * replication design issues, till that get addressed
+         * performance.client-io-threads option is turned off for all
+         * replicate volumes
+         */
+        if (priv->op_version >= GD_OP_VERSION_3_12_2) {
+            ret = dict_set_nstrn(volinfo->dict, "performance.client-io-threads",
+                                 SLEN("performance.client-io-threads"), "off",
+                                 SLEN("off"));
+            if (ret) {
+                gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+                       "Failed to set "
+                       "performance.client-io-threads to off");
+                goto out;
+            }
+        }
+        ret = dict_get_int32n(dict, "replica-count", SLEN("replica-count"),
+                              &volinfo->replica_count);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+                   "Failed to get "
+                   "replica count for volume %s",
+                   volname);
+            goto out;
+        }
+
+        /* coverity[unused_value] arbiter count is optional */
+        ret = dict_get_int32n(dict, "arbiter-count", SLEN("arbiter-count"),
+                              &volinfo->arbiter_count);
+        ret = dict_get_int32n(dict, "thin-arbiter-count",
+                              SLEN("thin-arbiter-count"),
+                              &volinfo->thin_arbiter_count);
+        if (volinfo->thin_arbiter_count) {
+            ret = dict_get_strn(dict, "ta-brick", SLEN("ta-brick"), &ta_bricks);
+            if (ret) {
+                gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+                       "Unable to get thin arbiter brick for "
+                       "volume %s",
+                       volname);
+                goto out;
+            }
+        }
+
+    } else if (GF_CLUSTER_TYPE_DISPERSE == volinfo->type) {
+        ret = dict_get_int32n(dict, "disperse-count", SLEN("disperse-count"),
+                              &volinfo->disperse_count);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+                   "Failed to get "
+                   "disperse count for volume %s",
+                   volname);
+            goto out;
+        }
+        ret = dict_get_int32n(dict, "redundancy-count",
+                              SLEN("redundancy-count"),
+                              &volinfo->redundancy_count);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+                   "Failed to get "
+                   "redundancy count for volume %s",
+                   volname);
+            goto out;
+        }
+        if (priv->op_version < GD_OP_VERSION_3_6_0) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_UNSUPPORTED_VERSION,
+                   "Disperse volume "
+                   "needs op-version 3.6.0 or higher");
+            ret = -1;
+            goto out;
+        }
+    }
+
+    /* dist-leaf-count is the count of brick nodes for a given
+       subvolume of distribute */
+    volinfo->dist_leaf_count = glusterd_get_dist_leaf_count(volinfo);
+
+    /* subvol_count is the count of number of subvolumes present
+       for a given distribute volume */
+    volinfo->subvol_count = (volinfo->brick_count / volinfo->dist_leaf_count);
+
+    /* Keep sub-count same as earlier, for the sake of backward
+       compatibility */
+    if (volinfo->dist_leaf_count > 1)
+        volinfo->sub_count = volinfo->dist_leaf_count;
+
+    ret = dict_get_strn(dict, "transport", SLEN("transport"), &trans_type);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+               "Unable to get transport type of volume %s", volname);
+        goto out;
+    }
 
-        count = volinfo->brick_count;
+    ret = dict_get_strn(dict, "volume-id", SLEN("volume-id"), &str);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+               "Unable to get volume-id of volume %s", volname);
+        goto out;
+    }
+    ret = gf_uuid_parse(str, volinfo->volume_id);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_UUID_PARSE_FAIL,
+               "unable to parse uuid %s of volume %s", str, volname);
+        goto out;
+    }
 
-        ret = dict_get_str (dict, "bricks", &bricks);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "Unable to get bricks for "
-                        "volume %s", volname);
+    ret = dict_get_strn(dict, "internal-username", SLEN("internal-username"),
+                        &username);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+               "unable to get internal username of volume %s", volname);
+        goto out;
+    }
+    glusterd_auth_set_username(volinfo, username);
+
+    ret = dict_get_strn(dict, "internal-password", SLEN("internal-password"),
+                        &password);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+               "unable to get internal password of volume %s", volname);
+        goto out;
+    }
+    glusterd_auth_set_password(volinfo, password);
+
+    if (strcasecmp(trans_type, "rdma") == 0) {
+        volinfo->transport_type = GF_TRANSPORT_RDMA;
+    } else if (strcasecmp(trans_type, "tcp") == 0) {
+        volinfo->transport_type = GF_TRANSPORT_TCP;
+    } else {
+        volinfo->transport_type = GF_TRANSPORT_BOTH_TCP_RDMA;
+    }
+
+    if (ta_bricks) {
+        ta_brick_list = gf_strdup(ta_bricks);
+        ta_free_ptr = ta_brick_list;
+    }
+
+    if (volinfo->thin_arbiter_count) {
+        ta_brick = strtok_r(ta_brick_list + 1, " \n", &ta_saveptr);
+
+        count = 1;
+        brickid = volinfo->replica_count;
+        /* assign brickid to ta_bricks
+         * Following loop runs for number of subvols times. Although
+         * there is only one ta-brick for a volume but the volume fuse volfile
+         * requires an entry of ta-brick for each subvolume. Also, the ta-brick
+         * id needs to be adjusted according to the subvol count.
+         * For eg- For first subvolume ta-brick id is volname-ta-2, for second
+         * subvol ta-brick id is volname-ta-5.
+         */
+        while (count <= volinfo->subvol_count) {
+            ret = glusterd_brickinfo_new_from_brick(ta_brick, &ta_brickinfo,
+                                                    _gf_false, op_errstr);
+            if (ret)
                 goto out;
-        }
-
-        /* replica-count 1 means, no replication, file is in one brick only */
-        volinfo->replica_count = 1;
-        /* stripe-count 1 means, no striping, file is present as a whole */
-        volinfo->stripe_count = 1;
 
-        if (GF_CLUSTER_TYPE_REPLICATE == volinfo->type) {
-                ret = dict_get_int32 (dict, "replica-count",
-                                      &volinfo->replica_count);
-                if (ret) {
-                        gf_log (this->name, GF_LOG_ERROR, "Failed to get "
-                                 "replica count for volume %s", volname);
-                        goto out;
-                }
-        } else if (GF_CLUSTER_TYPE_STRIPE == volinfo->type) {
-                ret = dict_get_int32 (dict, "stripe-count",
-                                      &volinfo->stripe_count);
-                if (ret) {
-                        gf_log (this->name, GF_LOG_ERROR, "Failed to get stripe"
-                                " count for volume %s", volname);
-                        goto out;
-                }
-        } else if (GF_CLUSTER_TYPE_STRIPE_REPLICATE == volinfo->type) {
-                ret = dict_get_int32 (dict, "stripe-count",
-                                      &volinfo->stripe_count);
-                if (ret) {
-                        gf_log (this->name, GF_LOG_ERROR, "Failed to get stripe"
-                                " count for volume %s", volname);
-                        goto out;
-                }
-                ret = dict_get_int32 (dict, "replica-count",
-                                      &volinfo->replica_count);
-                if (ret) {
-                        gf_log (this->name, GF_LOG_ERROR, "Failed to get "
-                                "replica count for volume %s", volname);
-                        goto out;
-                }
-        } else if (GF_CLUSTER_TYPE_DISPERSE == volinfo->type) {
-                ret = dict_get_int32 (dict, "disperse-count",
-                                      &volinfo->disperse_count);
-                if (ret) {
-                        gf_log (this->name, GF_LOG_ERROR, "Failed to get "
-                                 "disperse count for volume %s", volname);
-                        goto out;
-                }
-                ret = dict_get_int32 (dict, "redundancy-count",
-                                      &volinfo->redundancy_count);
-                if (ret) {
-                        gf_log (this->name, GF_LOG_ERROR, "Failed to get "
-                                 "redundancy count for volume %s", volname);
-                        goto out;
-                }
-                if (priv->op_version < GD_OP_VERSION_3_6_0) {
-                        gf_log (this->name, GF_LOG_ERROR, "Disperse volume "
-                                "needs op-version 3.6.0 or higher");
-                        ret = -1;
-                        goto out;
-                }
+            GLUSTERD_ASSIGN_BRICKID_TO_TA_BRICKINFO(ta_brickinfo, volinfo,
+                                                    brickid);
+            cds_list_add_tail(&ta_brickinfo->brick_list, &volinfo->ta_bricks);
+            count++;
+            brickid += volinfo->replica_count + 1;
         }
+    }
 
-        /* dist-leaf-count is the count of brick nodes for a given
-           subvolume of distribute */
-        volinfo->dist_leaf_count = glusterd_get_dist_leaf_count (volinfo);
+    if (bricks) {
+        brick_list = gf_strdup(bricks);
+        free_ptr = brick_list;
+    }
 
-        /* subvol_count is the count of number of subvolumes present
-           for a given distribute volume */
-        volinfo->subvol_count = (volinfo->brick_count /
-                                 volinfo->dist_leaf_count);
+    count = volinfo->brick_count;
 
-        /* Keep sub-count same as earlier, for the sake of backward
-           compatibility */
-        if (volinfo->dist_leaf_count > 1)
-                volinfo->sub_count = volinfo->dist_leaf_count;
+    if (count)
+        brick = strtok_r(brick_list + 1, " \n", &saveptr);
 
-        ret = dict_get_str (dict, "transport", &trans_type);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "Unable to get transport type of volume %s", volname);
-                goto out;
-        }
-
-        ret = dict_get_str (dict, "volume-id", &str);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "Unable to get volume-id of volume %s", volname);
-                goto out;
-        }
-        ret = gf_uuid_parse (str, volinfo->volume_id);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "unable to parse uuid %s of volume %s", str, volname);
-                goto out;
+    brickid = glusterd_get_next_available_brickid(volinfo);
+    if (brickid < 0)
+        goto out;
+    while (i <= count) {
+        ret = glusterd_brickinfo_new_from_brick(brick, &brickinfo, _gf_true,
+                                                op_errstr);
+        if (ret)
+            goto out;
+        if (volinfo->thin_arbiter_count == 1 &&
+            (brickid + 1) % (volinfo->replica_count + 1) == 0) {
+            brickid = brickid + 1;
         }
+        GLUSTERD_ASSIGN_BRICKID_TO_BRICKINFO(brickinfo, volinfo, brickid++);
 
-        ret = dict_get_str (dict, "internal-username", &username);
+        ret = glusterd_resolve_brick(brickinfo);
         if (ret) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "unable to get internal username of volume %s",
-                        volname);
-                goto out;
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_RESOLVE_BRICK_FAIL,
+                   FMTSTR_RESOLVE_BRICK, brickinfo->hostname, brickinfo->path);
+            goto out;
         }
-        glusterd_auth_set_username (volinfo, username);
 
-        ret = dict_get_str (dict, "internal-password", &password);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "unable to get internal password of volume %s",
-                        volname);
-                goto out;
-        }
-        glusterd_auth_set_password (volinfo, password);
-
-        if (strcasecmp (trans_type, "rdma") == 0) {
-                volinfo->transport_type = GF_TRANSPORT_RDMA;
-        } else if (strcasecmp (trans_type, "tcp") == 0) {
-                volinfo->transport_type = GF_TRANSPORT_TCP;
-        } else {
-                volinfo->transport_type = GF_TRANSPORT_BOTH_TCP_RDMA;
-        }
-
-        if (bricks) {
-                brick_list = gf_strdup (bricks);
-                free_ptr = brick_list;
-        }
-
-        if (count)
-                brick = strtok_r (brick_list+1, " \n", &saveptr);
-        caps = CAPS_BD | CAPS_THIN | CAPS_OFFLOAD_COPY | CAPS_OFFLOAD_SNAPSHOT;
-
-        brickid = glusterd_get_next_available_brickid (volinfo);
-        if (brickid < 0)
-                goto out;
-        while ( i <= count) {
-                ret = glusterd_brickinfo_new_from_brick (brick, &brickinfo);
-                if (ret)
-                        goto out;
-
-                GLUSTERD_ASSIGN_BRICKID_TO_BRICKINFO (brickinfo, volinfo,
-                                                      brickid++);
-
-                ret = glusterd_resolve_brick (brickinfo);
-                if (ret) {
-                        gf_log (this->name, GF_LOG_ERROR, FMTSTR_RESOLVE_BRICK,
-                                brickinfo->hostname, brickinfo->path);
-                        goto out;
-                }
-
-                /* A bricks mount dir is required only by snapshots which were
-                 * introduced in gluster-3.6.0
-                 */
-                if (priv->op_version >= GD_OP_VERSION_3_6_0) {
-                        brick_mount_dir = NULL;
-                        snprintf (key, sizeof(key), "brick%d.mount_dir", i);
-                        ret = dict_get_str (dict, key, &brick_mount_dir);
-                        if (ret) {
-                                gf_log (this->name, GF_LOG_ERROR,
-                                        "%s not present", key);
-                                goto out;
-                        }
-                        strncpy (brickinfo->mount_dir, brick_mount_dir,
-                                 sizeof(brickinfo->mount_dir));
-                }
-
-#ifdef HAVE_BD_XLATOR
-                if (!gf_uuid_compare (brickinfo->uuid, MY_UUID)
-                    && brickinfo->vg[0]) {
-                        ret = glusterd_is_valid_vg (brickinfo, 0, msg);
-                        if (ret) {
-                                gf_log (this->name, GF_LOG_ERROR, "%s", msg);
-                                goto out;
-                        }
-
-                        /* if anyone of the brick does not have thin
-                           support, disable it for entire volume */
-                        caps &= brickinfo->caps;
-                } else {
-                                caps = 0;
-                }
-
-#endif
+        /* A bricks mount dir is required only by snapshots which were
+         * introduced in gluster-3.6.0
+         */
+        if (priv->op_version >= GD_OP_VERSION_3_6_0) {
+            brick_mount_dir = NULL;
+            ret = snprintf(key, sizeof(key), "brick%d.mount_dir", i);
+            ret = dict_get_strn(dict, key, ret, &brick_mount_dir);
+            if (ret) {
+                gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+                       "%s not present", key);
+                goto out;
+            }
+            snprintf(brickinfo->mount_dir, sizeof(brickinfo->mount_dir), "%s",
+                     brick_mount_dir);
+        }
+
+        if (!gf_uuid_compare(brickinfo->uuid, MY_UUID)) {
+            ret = sys_statvfs(brickinfo->path, &brickstat);
+            if (ret) {
+                gf_log("brick-op", GF_LOG_ERROR,
+                       "Failed to fetch disk"
+                       " utilization from the brick (%s:%s). Please "
+                       "check health of the brick. Error code was %s",
+                       brickinfo->hostname, brickinfo->path, strerror(errno));
+                goto out;
+            }
+            brickinfo->statfs_fsid = brickstat.f_fsid;
+        }
+
+        cds_list_add_tail(&brickinfo->brick_list, &volinfo->bricks);
+        brick = strtok_r(NULL, " \n", &saveptr);
+        i++;
+    }
+
+    ret = glusterd_enable_default_options(volinfo, NULL);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_FAIL_DEFAULT_OPT_SET,
+               "Failed to set default "
+               "options on create for volume %s",
+               volinfo->volname);
+        goto out;
+    }
 
-                cds_list_add_tail (&brickinfo->brick_list, &volinfo->bricks);
-                brick = strtok_r (NULL, " \n", &saveptr);
-                i++;
-        }
+    ret = dict_get_strn(dict, "transport.address-family",
+                        SLEN("transport.address-family"), &address_family_str);
 
-        ret = glusterd_enable_default_options (volinfo, NULL);
+    if (!ret) {
+        ret = dict_set_dynstr_with_alloc(
+            volinfo->dict, "transport.address-family", address_family_str);
         if (ret) {
-                gf_msg (this->name, GF_LOG_ERROR, 0,
-                        GD_MSG_FAIL_DEFAULT_OPT_SET, "Failed to set default "
-                        "options on create for volume %s", volinfo->volname);
-                goto out;
+            gf_log(this->name, GF_LOG_ERROR,
+                   "Failed to set transport.address-family for %s",
+                   volinfo->volname);
+            goto out;
         }
+    }
 
-        gd_update_volume_op_versions (volinfo);
-
-        volinfo->caps = caps;
-
-        ret = glusterd_snapdsvc_init (volinfo);
-        if (ret) {
-                *op_errstr = gf_strdup ("Failed to initialize snapd service");
-                goto out;
-        }
+    gd_update_volume_op_versions(volinfo);
 
-        ret = glusterd_store_volinfo (volinfo, GLUSTERD_VOLINFO_VER_AC_INCREMENT);
-        if (ret) {
-                glusterd_store_delete_volume (volinfo);
-                *op_errstr = gf_strdup ("Failed to store the Volume information");
-                goto out;
-        }
+    ret = glusterd_store_volinfo(volinfo, GLUSTERD_VOLINFO_VER_AC_INCREMENT);
+    if (ret) {
+        glusterd_store_delete_volume(volinfo);
+        *op_errstr = gf_strdup(
+            "Failed to store the "
+            "Volume information");
+        goto out;
+    }
 
-        ret = glusterd_create_volfiles_and_notify_services (volinfo);
-        if (ret) {
-                *op_errstr = gf_strdup ("Failed to create volume files");
-                goto out;
-        }
+    ret = glusterd_create_volfiles_and_notify_services(volinfo);
+    if (ret) {
+        *op_errstr = gf_strdup("Failed to create volume files");
+        goto out;
+    }
 
-        volinfo->rebal.defrag_status = 0;
-        glusterd_list_add_order (&volinfo->vol_list, &priv->volumes,
-                                 glusterd_compare_volume_name);
-        vol_added = _gf_true;
+    volinfo->rebal.defrag_status = 0;
+    glusterd_list_add_order(&volinfo->vol_list, &priv->volumes,
+                            glusterd_compare_volume_name);
+    vol_added = _gf_true;
 
 out:
-        GF_FREE(free_ptr);
-        if (!vol_added && volinfo)
-                glusterd_volinfo_unref (volinfo);
-        return ret;
+    GF_FREE(free_ptr);
+    GF_FREE(ta_free_ptr);
+    if (!vol_added && volinfo)
+        glusterd_volinfo_unref(volinfo);
+    return ret;
 }
 
 int
-glusterd_start_volume (glusterd_volinfo_t *volinfo, int flags,
-                       gf_boolean_t wait)
+glusterd_start_volume(glusterd_volinfo_t *volinfo, int flags, gf_boolean_t wait)
 
 {
-        int                             ret             = 0;
-        glusterd_brickinfo_t           *brickinfo       = NULL;
-        xlator_t                       *this            = NULL;
-        glusterd_volinfo_ver_ac_t       verincrement    = 0;
-
-        this = THIS;
-        GF_ASSERT (this);
-        GF_ASSERT (volinfo);
-
-        cds_list_for_each_entry (brickinfo, &volinfo->bricks, brick_list) {
-                ret = glusterd_brick_start (volinfo, brickinfo, wait);
-                /* If 'force' try to start all bricks regardless of success or
-                 * failure
-                 */
-                if (!(flags & GF_CLI_FLAG_OP_FORCE) && ret)
-                        goto out;
-        }
-
-        /* Increment the volinfo version only if there is a
-         * change in status. Force option can be used to start
-         * dead bricks even if the volume is in started state.
-         * In such case volume status will be GLUSTERD_STATUS_STARTED.
-         * Therefore we should not increment the volinfo version.*/
-        if (GLUSTERD_STATUS_STARTED != volinfo->status) {
-                verincrement = GLUSTERD_VOLINFO_VER_AC_INCREMENT;
-        } else {
-                verincrement = GLUSTERD_VOLINFO_VER_AC_NONE;
-        }
-
-        glusterd_set_volume_status (volinfo, GLUSTERD_STATUS_STARTED);
-
-        ret = glusterd_store_volinfo (volinfo, verincrement);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "Failed to store volinfo of "
-                        "%s volume", volinfo->volname);
-                goto out;
+    int ret = 0;
+    glusterd_brickinfo_t *brickinfo = NULL;
+    xlator_t *this = NULL;
+    glusterd_volinfo_ver_ac_t verincrement = 0;
+
+    this = THIS;
+    GF_ASSERT(this);
+    GF_ASSERT(volinfo);
+
+    cds_list_for_each_entry(brickinfo, &volinfo->bricks, brick_list)
+    {
+        /* Mark start_triggered to false so that in case if this brick
+         * was brought down through gf_attach utility, the
+         * brickinfo->start_triggered wouldn't have been updated to
+         * _gf_false
+         */
+        if (flags & GF_CLI_FLAG_OP_FORCE) {
+            brickinfo->start_triggered = _gf_false;
         }
+        ret = glusterd_brick_start(volinfo, brickinfo, wait, _gf_false);
+        /* If 'force' try to start all bricks regardless of success or
+         * failure
+         */
+        if (!(flags & GF_CLI_FLAG_OP_FORCE) && ret)
+            goto out;
+    }
+
+    /* Increment the volinfo version only if there is a
+     * change in status. Force option can be used to start
+     * dead bricks even if the volume is in started state.
+     * In such case volume status will be GLUSTERD_STATUS_STARTED.
+     * Therefore we should not increment the volinfo version.*/
+    if (GLUSTERD_STATUS_STARTED != volinfo->status) {
+        verincrement = GLUSTERD_VOLINFO_VER_AC_INCREMENT;
+    } else {
+        verincrement = GLUSTERD_VOLINFO_VER_AC_NONE;
+    }
+
+    glusterd_set_volume_status(volinfo, GLUSTERD_STATUS_STARTED);
+    /* Update volinfo on disk in critical section because
+       attach_brick_callback can also call store_volinfo for same
+       volume to update volinfo on disk
+    */
+    /* coverity[ORDER_REVERSAL] */
+    LOCK(&volinfo->lock);
+    ret = glusterd_store_volinfo(volinfo, verincrement);
+    UNLOCK(&volinfo->lock);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_VOLINFO_SET_FAIL,
+               "Failed to store volinfo of "
+               "%s volume",
+               volinfo->volname);
+        goto out;
+    }
 out:
-        gf_log (this->name, GF_LOG_TRACE, "returning %d ", ret);
-        return ret;
+    gf_msg_trace(this->name, 0, "returning %d ", ret);
+    return ret;
 }
 
 int
-glusterd_op_start_volume (dict_t *dict, char **op_errstr)
+glusterd_op_start_volume(dict_t *dict, char **op_errstr)
 {
-        int                         ret             = 0;
-        int32_t                     brick_count     = 0;
-        char                       *brick_mount_dir = NULL;
-        char                        key[PATH_MAX]   = "";
-        char                       *volname         = NULL;
-        int                         flags           = 0;
-        glusterd_volinfo_t         *volinfo         = NULL;
-        glusterd_brickinfo_t       *brickinfo       = NULL;
-        xlator_t                   *this            = NULL;
-        glusterd_conf_t            *conf            = NULL;
-        glusterd_svc_t             *svc             = NULL;
-
-        this = THIS;
-        GF_ASSERT (this);
-        conf = this->private;
-        GF_ASSERT (conf);
-
-        ret = glusterd_op_start_volume_args_get (dict, &volname, &flags);
-        if (ret)
-                goto out;
-
-        ret  = glusterd_volinfo_find (volname, &volinfo);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, FMTSTR_CHECK_VOL_EXISTS,
-                        volname);
-                goto out;
-        }
+    int ret = 0;
+    int32_t brick_count = 0;
+    char *brick_mount_dir = NULL;
+    char key[64] = "";
+    char *volname = NULL;
+    int flags = 0;
+    glusterd_volinfo_t *volinfo = NULL;
+    glusterd_brickinfo_t *brickinfo = NULL;
+    xlator_t *this = NULL;
+    glusterd_conf_t *conf = NULL;
+    glusterd_svc_t *svc = NULL;
+    char *str = NULL;
+    gf_boolean_t option = _gf_false;
+
+    this = THIS;
+    GF_ASSERT(this);
+    conf = this->private;
+    GF_ASSERT(conf);
+
+    ret = glusterd_op_start_volume_args_get(dict, &volname, &flags);
+    if (ret)
+        goto out;
 
-        /* A bricks mount dir is required only by snapshots which were
-         * introduced in gluster-3.6.0
-         */
-        if (conf->op_version >= GD_OP_VERSION_3_6_0) {
-                cds_list_for_each_entry (brickinfo, &volinfo->bricks,
-                                         brick_list) {
-                        brick_count++;
-                        /* Don't check bricks that are not owned by you
-                         */
-                        if (gf_uuid_compare (brickinfo->uuid, MY_UUID))
-                                continue;
-                        if (strlen(brickinfo->mount_dir) < 1) {
-                                brick_mount_dir = NULL;
-                                snprintf (key, sizeof(key), "brick%d.mount_dir",
-                                          brick_count);
-                                ret = dict_get_str (dict, key,
-                                                    &brick_mount_dir);
-                                if (ret) {
-                                        gf_log (this->name, GF_LOG_ERROR,
-                                                "%s not present", key);
-                                        goto out;
-                                }
-                                strncpy (brickinfo->mount_dir, brick_mount_dir,
-                                         sizeof(brickinfo->mount_dir));
-                        }
+    ret = glusterd_volinfo_find(volname, &volinfo);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_VOL_NOT_FOUND,
+               FMTSTR_CHECK_VOL_EXISTS, volname);
+        goto out;
+    }
+
+    /* This is an incremental approach to have all the volinfo objects ref
+     * count. The first attempt is made in volume start transaction to
+     * ensure it doesn't race with import volume where stale volume is
+     * deleted. There are multiple instances of GlusterD crashing in
+     * bug-948686.t because of this. Once this approach is full proof, all
+     * other volinfo objects will be refcounted.
+     */
+    glusterd_volinfo_ref(volinfo);
+
+    /* A bricks mount dir is required only by snapshots which were
+     * introduced in gluster-3.6.0
+     */
+    if (conf->op_version >= GD_OP_VERSION_3_6_0) {
+        cds_list_for_each_entry(brickinfo, &volinfo->bricks, brick_list)
+        {
+            brick_count++;
+            /* Don't check bricks that are not owned by you
+             */
+            if (gf_uuid_compare(brickinfo->uuid, MY_UUID))
+                continue;
+            if (strlen(brickinfo->mount_dir) < 1) {
+                brick_mount_dir = NULL;
+                ret = snprintf(key, sizeof(key), "brick%d.mount_dir",
+                               brick_count);
+                ret = dict_get_strn(dict, key, ret, &brick_mount_dir);
+                if (ret) {
+                    gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+                           "%s not present", key);
+                    goto out;
+                }
+                if (snprintf(brickinfo->mount_dir, sizeof(brickinfo->mount_dir),
+                             "%s",
+                             brick_mount_dir) >= sizeof(brickinfo->mount_dir)) {
+                    ret = -1;
+                    goto out;
                 }
+            }
         }
+    }
 
-        ret = glusterd_start_volume (volinfo, flags, _gf_true);
-        if (ret)
-                goto out;
+    ret = dict_get_str(conf->opts, GLUSTERD_STORE_KEY_GANESHA_GLOBAL, &str);
+    if (ret != 0) {
+        gf_msg(this->name, GF_LOG_INFO, 0, GD_MSG_DICT_GET_FAILED,
+               "Global dict not present.");
+        ret = 0;
 
-        if (!volinfo->is_snap_volume) {
-                svc = &(volinfo->snapd.svc);
-                ret = svc->manager (svc, volinfo, PROC_START_NO_WAIT);
-                if (ret)
-                        goto out;
-        }
+    } else {
+        ret = gf_string2boolean(str, &option);
+        /* Check if the feature is enabled and set nfs-disable to true */
+        if (option) {
+            gf_msg_debug(this->name, 0, "NFS-Ganesha is enabled");
+            /* Gluster-nfs should not start when NFS-Ganesha is enabled*/
+            ret = dict_set_str(volinfo->dict, NFS_DISABLE_MAP_KEY, "on");
+            if (ret) {
+                gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+                       "Failed to set nfs.disable for"
+                       "volume %s",
+                       volname);
+                goto out;
+            }
+        }
+    }
+
+    ret = glusterd_start_volume(volinfo, flags, _gf_true);
+    if (ret)
+        goto out;
+
+    if (!volinfo->is_snap_volume) {
+        svc = &(volinfo->snapd.svc);
+        ret = svc->manager(svc, volinfo, PROC_START_NO_WAIT);
+        if (ret)
+            goto out;
+    }
 
-        ret = glusterd_svcs_manager (volinfo);
+    svc = &(volinfo->gfproxyd.svc);
+    ret = svc->manager(svc, volinfo, PROC_START_NO_WAIT);
+    ret = glusterd_svcs_manager(volinfo);
 
 out:
-        gf_log (this->name, GF_LOG_TRACE, "returning %d ", ret);
-        return ret;
+    if (volinfo)
+        glusterd_volinfo_unref(volinfo);
+
+    gf_msg_trace(this->name, 0, "returning %d ", ret);
+    return ret;
 }
 
 int
-glusterd_stop_volume (glusterd_volinfo_t *volinfo)
+glusterd_stop_volume(glusterd_volinfo_t *volinfo)
 {
-        int                     ret                     = -1;
-        glusterd_brickinfo_t    *brickinfo              = NULL;
-        char                    mountdir[PATH_MAX]      = {0,};
-        char                    pidfile[PATH_MAX]       = {0,};
-        xlator_t                *this                   = NULL;
-        glusterd_svc_t          *svc                    = NULL;
-
-        this = THIS;
-        GF_ASSERT (this);
-
-        GF_VALIDATE_OR_GOTO (this->name, volinfo, out);
+    int ret = -1;
+    glusterd_brickinfo_t *brickinfo = NULL;
+    xlator_t *this = NULL;
+    glusterd_svc_t *svc = NULL;
 
-        cds_list_for_each_entry (brickinfo, &volinfo->bricks, brick_list) {
-                ret = glusterd_brick_stop (volinfo, brickinfo, _gf_false);
-                if (ret) {
-                        gf_log (this->name, GF_LOG_ERROR, "Failed to stop "
-                                "brick (%s)", brickinfo->path);
-                        goto out;
-                }
-        }
+    this = THIS;
+    GF_ASSERT(this);
 
-        glusterd_set_volume_status (volinfo, GLUSTERD_STATUS_STOPPED);
+    GF_VALIDATE_OR_GOTO(this->name, volinfo, out);
 
-        ret = glusterd_store_volinfo (volinfo, GLUSTERD_VOLINFO_VER_AC_INCREMENT);
+    cds_list_for_each_entry(brickinfo, &volinfo->bricks, brick_list)
+    {
+        ret = glusterd_brick_stop(volinfo, brickinfo, _gf_false);
         if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "Failed to store volinfo of "
-                        "%s volume", volinfo->volname);
-                goto out;
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_BRICK_STOP_FAIL,
+                   "Failed to stop "
+                   "brick (%s)",
+                   brickinfo->path);
+            goto out;
         }
+    }
 
-        /* If quota auxiliary mount is present, unmount it */
-        GLUSTERFS_GET_AUX_MOUNT_PIDFILE (pidfile, volinfo->volname);
+    glusterd_set_volume_status(volinfo, GLUSTERD_STATUS_STOPPED);
 
-        if (!gf_is_service_running (pidfile, NULL)) {
-                gf_log (this->name, GF_LOG_DEBUG, "Aux mount of volume %s "
-                        "absent", volinfo->volname);
-        } else {
-                GLUSTERD_GET_QUOTA_AUX_MOUNT_PATH (mountdir, volinfo->volname,
-                                                   "/");
-
-                ret = gf_umount_lazy (this->name, mountdir, 0);
-                if (ret)
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "umount on %s failed, reason : %s",
-                                mountdir, strerror (errno));
-        }
+    ret = glusterd_store_volinfo(volinfo, GLUSTERD_VOLINFO_VER_AC_INCREMENT);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_VOLINFO_SET_FAIL,
+               "Failed to store volinfo of "
+               "%s volume",
+               volinfo->volname);
+        goto out;
+    }
 
-        if (!volinfo->is_snap_volume) {
-                svc = &(volinfo->snapd.svc);
-                ret = svc->manager (svc, volinfo, PROC_START_NO_WAIT);
-                if (ret)
-                        goto out;
-        }
+    if (!volinfo->is_snap_volume) {
+        svc = &(volinfo->snapd.svc);
+        ret = svc->manager(svc, volinfo, PROC_START_NO_WAIT);
+        if (ret)
+            goto out;
+    }
 
-        ret = glusterd_svcs_manager (volinfo);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "Failed to notify graph "
-                        "change for %s volume", volinfo->volname);
+    ret = glusterd_svcs_manager(volinfo);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_VOL_GRAPH_CHANGE_NOTIFY_FAIL,
+               "Failed to notify graph "
+               "change for %s volume",
+               volinfo->volname);
 
-                goto out;
-        }
+        goto out;
+    }
 
 out:
-        return ret;
+    return ret;
 }
 
 int
-glusterd_op_stop_volume (dict_t *dict)
+glusterd_op_stop_volume(dict_t *dict)
 {
-        int                                     ret = 0;
-        int                                     flags = 0;
-        char                                    *volname = NULL;
-        glusterd_volinfo_t                      *volinfo = NULL;
-        xlator_t                                *this = NULL;
+    int ret = 0;
+    int flags = 0;
+    char *volname = NULL;
+    glusterd_volinfo_t *volinfo = NULL;
+    xlator_t *this = NULL;
 
-        this = THIS;
-        GF_ASSERT (this);
+    this = THIS;
+    GF_ASSERT(this);
 
-        ret = glusterd_op_stop_volume_args_get (dict, &volname, &flags);
-        if (ret)
-                goto out;
+    ret = glusterd_op_stop_volume_args_get(dict, &volname, &flags);
+    if (ret)
+        goto out;
 
-        ret  = glusterd_volinfo_find (volname, &volinfo);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, FMTSTR_CHECK_VOL_EXISTS,
-                        volname);
-                goto out;
-        }
+    ret = glusterd_volinfo_find(volname, &volinfo);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_VOL_NOT_FOUND,
+               FMTSTR_CHECK_VOL_EXISTS, volname);
+        goto out;
+    }
 
-        ret = glusterd_stop_volume (volinfo);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "Failed to stop %s volume",
-                        volname);
-                goto out;
-        }
+    ret = glusterd_stop_volume(volinfo);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_VOL_STOP_FAILED,
+               "Failed to stop %s volume", volname);
+        goto out;
+    }
 out:
-        return ret;
+    gf_msg_trace(this->name, 0, "returning %d ", ret);
+    return ret;
 }
 
 int
-glusterd_op_delete_volume (dict_t *dict)
+glusterd_op_delete_volume(dict_t *dict)
 {
-        int                                     ret = 0;
-        char                                    *volname = NULL;
-        glusterd_conf_t                         *priv = NULL;
-        glusterd_volinfo_t                      *volinfo = NULL;
-        xlator_t                                *this = NULL;
-
-        this = THIS;
-        GF_ASSERT (this);
-        priv = this->private;
-        GF_ASSERT (priv);
-
-        ret = dict_get_str (dict, "volname", &volname);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "Unable to get volume name");
-                goto out;
-        }
+    int ret = 0;
+    char *volname = NULL;
+    glusterd_volinfo_t *volinfo = NULL;
+    xlator_t *this = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+
+    ret = dict_get_strn(dict, "volname", SLEN("volname"), &volname);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+               "Unable to get volume name");
+        goto out;
+    }
 
-        ret  = glusterd_volinfo_find (volname, &volinfo);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, FMTSTR_CHECK_VOL_EXISTS,
-                        volname);
-                goto out;
-        }
+    ret = glusterd_volinfo_find(volname, &volinfo);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_VOL_NOT_FOUND,
+               FMTSTR_CHECK_VOL_EXISTS, volname);
+        goto out;
+    }
 
-        ret = glusterd_remove_auxiliary_mount (volname);
+    if (glusterd_check_ganesha_export(volinfo) && is_origin_glusterd(dict)) {
+        ret = manage_export_config(volname, "off", NULL);
         if (ret)
-                goto out;
+            gf_msg(this->name, GF_LOG_WARNING, 0, 0,
+                   "Could not delete ganesha export conf file "
+                   "for %s",
+                   volname);
+    }
 
-        ret = glusterd_delete_volume (volinfo);
+    ret = glusterd_delete_volume(volinfo);
 out:
-        gf_log (this->name, GF_LOG_DEBUG, "returning %d", ret);
-        return ret;
+    gf_msg_debug(this->name, 0, "returning %d", ret);
+    return ret;
 }
 
 int
-glusterd_op_heal_volume (dict_t *dict, char **op_errstr)
+glusterd_op_heal_volume(dict_t *dict, char **op_errstr)
 {
-        int                                     ret = 0;
-        /* Necessary subtasks of heal are completed in brick op */
+    int ret = 0;
+    /* Necessary subtasks of heal are completed in brick op */
 
-        return ret;
+    return ret;
 }
 
 int
-glusterd_op_statedump_volume (dict_t *dict, char **op_errstr)
+glusterd_op_statedump_volume(dict_t *dict, char **op_errstr)
 {
-        int                     ret = 0;
-        char                    *volname = NULL;
-        char                    *options = NULL;
-        int                     option_cnt = 0;
-        glusterd_volinfo_t      *volinfo = NULL;
-        glusterd_brickinfo_t    *brickinfo = NULL;
-
-        ret = glusterd_op_statedump_volume_args_get (dict, &volname, &options,
-                                                     &option_cnt);
-        if (ret)
-                goto out;
+    int ret = 0;
+    char *volname = NULL;
+    char *options = NULL;
+    int option_cnt = 0;
+    glusterd_volinfo_t *volinfo = NULL;
+    glusterd_brickinfo_t *brickinfo = NULL;
+
+    ret = glusterd_op_statedump_volume_args_get(dict, &volname, &options,
+                                                &option_cnt);
+    if (ret)
+        goto out;
 
-        ret = glusterd_volinfo_find (volname, &volinfo);
+    ret = glusterd_volinfo_find(volname, &volinfo);
+    if (ret)
+        goto out;
+    gf_msg_debug("glusterd", 0, "Performing statedump on volume %s", volname);
+    if (strstr(options, "quotad")) {
+        ret = glusterd_quotad_statedump(options, option_cnt, op_errstr);
         if (ret)
-                goto out;
-        gf_log ("", GF_LOG_DEBUG, "Performing statedump on volume %s", volname);
-        if (strstr (options, "nfs") != NULL) {
-                ret = glusterd_nfs_statedump (options, option_cnt, op_errstr);
-                if (ret)
-                        goto out;
-
-        } else if (strstr (options, "quotad")) {
-                ret = glusterd_quotad_statedump (options, option_cnt,
-                                                 op_errstr);
-                if (ret)
-                        goto out;
-        } else {
-                cds_list_for_each_entry (brickinfo, &volinfo->bricks,
-                                         brick_list) {
-                        ret = glusterd_brick_statedump (volinfo, brickinfo,
-                                                        options, option_cnt,
-                                                        op_errstr);
-                        /* Let us take the statedump of other bricks instead of
-                         * exiting, if statedump of this brick fails.
-                         */
-                        if (ret)
-                                gf_log (THIS->name, GF_LOG_WARNING, "could not "
-                                        "take the statedump of the brick %s:%s."
-                                        " Proceeding to other bricks",
-                                        brickinfo->hostname, brickinfo->path);
-                }
-        }
+            goto out;
+#ifdef BUILD_GNFS
+    } else if (strstr(options, "nfs") != NULL) {
+        ret = glusterd_nfs_statedump(options, option_cnt, op_errstr);
+        if (ret)
+            goto out;
+#endif
+    } else if (strstr(options, "client")) {
+        ret = glusterd_client_statedump(volname, options, option_cnt,
+                                        op_errstr);
+        if (ret)
+            goto out;
+
+    } else {
+        cds_list_for_each_entry(brickinfo, &volinfo->bricks, brick_list)
+        {
+            ret = glusterd_brick_statedump(volinfo, brickinfo, options,
+                                           option_cnt, op_errstr);
+            /* Let us take the statedump of other bricks instead of
+             * exiting, if statedump of this brick fails.
+             */
+            if (ret)
+                gf_msg(THIS->name, GF_LOG_WARNING, 0, GD_MSG_BRK_STATEDUMP_FAIL,
+                       "could not "
+                       "take the statedump of the brick %s:%s."
+                       " Proceeding to other bricks",
+                       brickinfo->hostname, brickinfo->path);
+        }
+    }
 
 out:
-        return ret;
+    return ret;
 }
 
 int
-glusterd_clearlocks_send_cmd (glusterd_volinfo_t *volinfo, char *cmd,
-                              char *path, char *result, char *errstr,
-                              int err_len, char *mntpt)
+glusterd_clearlocks_send_cmd(glusterd_volinfo_t *volinfo, char *cmd, char *path,
+                             char *result, char *errstr, int err_len,
+                             char *mntpt)
 {
-        int               ret                   = -1;
-        glusterd_conf_t  *priv                  = NULL;
-        char             abspath[PATH_MAX]      = {0, };
-
-        priv = THIS->private;
-
-        snprintf (abspath, sizeof (abspath), "%s/%s", mntpt, path);
-        ret = sys_lgetxattr (abspath, cmd, result, PATH_MAX);
-        if (ret < 0) {
-                snprintf (errstr, err_len, "clear-locks getxattr command "
-                          "failed. Reason: %s", strerror (errno));
-                gf_log (THIS->name, GF_LOG_DEBUG, "%s", errstr);
-                goto out;
-        }
+    int ret = -1;
+    char abspath[PATH_MAX] = {
+        0,
+    };
+
+    snprintf(abspath, sizeof(abspath), "%s/%s", mntpt, path);
+    ret = sys_lgetxattr(abspath, cmd, result, PATH_MAX);
+    if (ret < 0) {
+        snprintf(errstr, err_len,
+                 "clear-locks getxattr command "
+                 "failed. Reason: %s",
+                 strerror(errno));
+        gf_msg_debug(THIS->name, 0, "%s", errstr);
+        goto out;
+    }
 
-        ret = 0;
+    ret = 0;
 out:
-        return ret;
+    return ret;
 }
 
 int
-glusterd_clearlocks_rmdir_mount (glusterd_volinfo_t *volinfo, char *mntpt)
+glusterd_clearlocks_rmdir_mount(glusterd_volinfo_t *volinfo, char *mntpt)
 {
-        int              ret               = -1;
-        glusterd_conf_t *priv              = NULL;
+    int ret = -1;
 
-        priv = THIS->private;
-
-        ret = rmdir (mntpt);
-        if (ret) {
-                gf_log (THIS->name, GF_LOG_DEBUG, "rmdir failed");
-                goto out;
-        }
+    ret = sys_rmdir(mntpt);
+    if (ret) {
+        gf_msg_debug(THIS->name, 0, "rmdir failed");
+        goto out;
+    }
 
-        ret = 0;
+    ret = 0;
 out:
-        return ret;
+    return ret;
 }
 
 void
-glusterd_clearlocks_unmount (glusterd_volinfo_t *volinfo, char *mntpt)
+glusterd_clearlocks_unmount(glusterd_volinfo_t *volinfo, char *mntpt)
 {
-        glusterd_conf_t  *priv               = NULL;
-        runner_t          runner             = {0,};
-        int               ret                = 0;
-
-        priv = THIS->private;
-
-        /*umount failures are ignored. Using stat we could have avoided
-         * attempting to unmount a non-existent filesystem. But a failure of
-         * stat() on mount can be due to network failures.*/
-
-        runinit (&runner);
-        runner_add_args (&runner, _PATH_UMOUNT, "-f", NULL);
-        runner_argprintf (&runner, "%s", mntpt);
-
-        synclock_unlock (&priv->big_lock);
-        ret = runner_run (&runner);
-        synclock_lock (&priv->big_lock);
-        if (ret) {
-                ret = 0;
-                gf_log ("", GF_LOG_DEBUG,
-                        "umount failed on maintenance client");
-        }
+    glusterd_conf_t *priv = NULL;
+    runner_t runner = {
+        0,
+    };
+    int ret = 0;
+
+    priv = THIS->private;
+
+    /*umount failures are ignored. Using stat we could have avoided
+     * attempting to unmount a non-existent filesystem. But a failure of
+     * stat() on mount can be due to network failures.*/
+
+    runinit(&runner);
+    runner_add_args(&runner, _PATH_UMOUNT, "-f", NULL);
+    runner_argprintf(&runner, "%s", mntpt);
+
+    synclock_unlock(&priv->big_lock);
+    ret = runner_run(&runner);
+    synclock_lock(&priv->big_lock);
+    if (ret) {
+        ret = 0;
+        gf_msg_debug("glusterd", 0, "umount failed on maintenance client");
+    }
 
-        return;
+    return;
 }
 
 int
-glusterd_clearlocks_create_mount (glusterd_volinfo_t *volinfo, char **mntpt)
+glusterd_clearlocks_create_mount(glusterd_volinfo_t *volinfo, char **mntpt)
 {
-        int              ret                    = -1;
-        glusterd_conf_t *priv                   = NULL;
-        char             template[PATH_MAX]     = {0,};
-        char            *tmpl                   = NULL;
-
-        priv = THIS->private;
-
-        snprintf (template, sizeof (template), "/tmp/%s.XXXXXX",
-                  volinfo->volname);
-        tmpl = mkdtemp (template);
-        if (!tmpl) {
-                gf_log (THIS->name, GF_LOG_DEBUG, "Couldn't create temporary "
-                        "mount directory. Reason %s", strerror (errno));
-                goto out;
-        }
+    int ret = -1;
+    char template[PATH_MAX] = {
+        0,
+    };
+    char *tmpl = NULL;
+
+    snprintf(template, sizeof(template), "/tmp/%s.XXXXXX", volinfo->volname);
+    tmpl = mkdtemp(template);
+    if (!tmpl) {
+        gf_msg_debug(THIS->name, 0,
+                     "Couldn't create temporary "
+                     "mount directory. Reason %s",
+                     strerror(errno));
+        goto out;
+    }
 
-        *mntpt = gf_strdup (tmpl);
-        ret = 0;
+    *mntpt = gf_strdup(tmpl);
+    ret = 0;
 out:
-        return ret;
+    return ret;
 }
 
 int
-glusterd_clearlocks_mount (glusterd_volinfo_t *volinfo, char **xl_opts,
-                           char *mntpt)
+glusterd_clearlocks_mount(glusterd_volinfo_t *volinfo, char **xl_opts,
+                          char *mntpt)
 {
-        int             ret                             = -1;
-        int             i                               = 0;
-        glusterd_conf_t *priv                           = NULL;
-        runner_t        runner                          = {0,};
-        char            client_volfpath[PATH_MAX]       = {0,};
-        char            self_heal_opts[3][1024]      = {"*replicate*.data-self-heal=off",
-                                                        "*replicate*.metadata-self-heal=off",
-                                                        "*replicate*.entry-self-heal=off"};
-
-        priv = THIS->private;
-
-        runinit (&runner);
-        glusterd_get_trusted_client_filepath (client_volfpath, volinfo,
-                                      volinfo->transport_type);
-        runner_add_args (&runner, SBIN_DIR"/glusterfs", "-f", NULL);
-        runner_argprintf (&runner, "%s", client_volfpath);
-        runner_add_arg (&runner, "-l");
-        runner_argprintf (&runner, DEFAULT_LOG_FILE_DIRECTORY
-                          "/%s-clearlocks-mnt.log", volinfo->volname);
-        if (volinfo->memory_accounting)
-                runner_add_arg (&runner, "--mem-accounting");
-
-        for (i = 0; i < volinfo->brick_count && xl_opts[i]; i++) {
-                runner_add_arg (&runner, "--xlator-option");
-                runner_argprintf (&runner, "%s", xl_opts[i]);
-        }
-
-        for (i = 0; i < 3; i++) {
-                runner_add_args (&runner, "--xlator-option",
-                                 self_heal_opts[i], NULL);
-        }
-
-        runner_argprintf (&runner, "%s", mntpt);
-        synclock_unlock (&priv->big_lock);
-        ret = runner_run (&runner);
-        synclock_lock (&priv->big_lock);
-        if (ret) {
-                gf_log (THIS->name, GF_LOG_DEBUG,
-                        "Could not start glusterfs");
-                goto out;
-        }
-        gf_log (THIS->name, GF_LOG_DEBUG,
-                "Started glusterfs successfully");
+    int ret = -1;
+    int i = 0;
+    glusterd_conf_t *priv = NULL;
+    runner_t runner = {
+        0,
+    };
+    char client_volfpath[PATH_MAX] = {
+        0,
+    };
+    char self_heal_opts[3][1024] = {"*replicate*.data-self-heal=off",
+                                    "*replicate*.metadata-self-heal=off",
+                                    "*replicate*.entry-self-heal=off"};
+
+    priv = THIS->private;
+
+    runinit(&runner);
+    glusterd_get_trusted_client_filepath(client_volfpath, volinfo,
+                                         volinfo->transport_type);
+    runner_add_args(&runner, SBIN_DIR "/glusterfs", "-f", NULL);
+    runner_argprintf(&runner, "%s", client_volfpath);
+    runner_add_arg(&runner, "-l");
+    runner_argprintf(&runner, "%s/%s-clearlocks-mnt.log", priv->logdir,
+                     volinfo->volname);
+    if (volinfo->memory_accounting)
+        runner_add_arg(&runner, "--mem-accounting");
+
+    for (i = 0; i < volinfo->brick_count && xl_opts[i]; i++) {
+        runner_add_arg(&runner, "--xlator-option");
+        runner_argprintf(&runner, "%s", xl_opts[i]);
+    }
+
+    for (i = 0; i < 3; i++) {
+        runner_add_args(&runner, "--xlator-option", self_heal_opts[i], NULL);
+    }
+
+    runner_argprintf(&runner, "%s", mntpt);
+    synclock_unlock(&priv->big_lock);
+    ret = runner_run(&runner);
+    synclock_lock(&priv->big_lock);
+    if (ret) {
+        gf_msg_debug(THIS->name, 0, "Could not start glusterfs");
+        goto out;
+    }
+    gf_msg_debug(THIS->name, 0, "Started glusterfs successfully");
 
 out:
-        return ret;
+    return ret;
 }
 
 int
-glusterd_clearlocks_get_local_client_ports (glusterd_volinfo_t *volinfo,
-                                            char **xl_opts)
+glusterd_clearlocks_get_local_client_ports(glusterd_volinfo_t *volinfo,
+                                           char **xl_opts)
 {
-        glusterd_brickinfo_t    *brickinfo          = NULL;
-        glusterd_conf_t         *priv               = NULL;
-        char                    brickname[PATH_MAX] = {0,};
-        int                     index               = 0;
-        int                     ret                 = -1;
-        int                     i                   = 0;
-        int                     port                = 0;
-
-        GF_ASSERT (xl_opts);
-        if (!xl_opts) {
-                gf_log (THIS->name, GF_LOG_DEBUG, "Should pass non-NULL "
-                        "xl_opts");
-                goto out;
+    glusterd_brickinfo_t *brickinfo = NULL;
+    char brickname[PATH_MAX] = {
+        0,
+    };
+    int index = 0;
+    int ret = -1;
+    int i = 0;
+    int port = 0;
+    int32_t len = 0;
+
+    GF_ASSERT(xl_opts);
+    if (!xl_opts) {
+        gf_msg_debug(THIS->name, 0,
+                     "Should pass non-NULL "
+                     "xl_opts");
+        goto out;
+    }
+
+    index = -1;
+    cds_list_for_each_entry(brickinfo, &volinfo->bricks, brick_list)
+    {
+        index++;
+        if (gf_uuid_compare(brickinfo->uuid, MY_UUID))
+            continue;
+
+        if (volinfo->transport_type == GF_TRANSPORT_RDMA) {
+            len = snprintf(brickname, sizeof(brickname), "%s.rdma",
+                           brickinfo->path);
+        } else
+            len = snprintf(brickname, sizeof(brickname), "%s", brickinfo->path);
+        if ((len < 0) || (len >= sizeof(brickname))) {
+            ret = -1;
+            goto out;
         }
 
-        priv = THIS->private;
-
-        index = -1;
-        cds_list_for_each_entry (brickinfo, &volinfo->bricks, brick_list) {
-                index++;
-                if (gf_uuid_compare (brickinfo->uuid, MY_UUID))
-                        continue;
-
-                if (volinfo->transport_type == GF_TRANSPORT_RDMA) {
-                        snprintf (brickname, sizeof(brickname), "%s.rdma",
-                                  brickinfo->path);
-                } else
-                        snprintf (brickname, sizeof(brickname), "%s",
-                                  brickinfo->path);
-
-                port = pmap_registry_search (THIS, brickname,
-                                             GF_PMAP_PORT_BRICKSERVER);
-                if (!port) {
-                        ret = -1;
-                        gf_log (THIS->name, GF_LOG_DEBUG, "Couldn't get port "
-                                " for brick %s:%s", brickinfo->hostname,
-                                brickinfo->path);
-                        goto out;
-                }
+        port = pmap_registry_search(THIS, brickname, GF_PMAP_PORT_BRICKSERVER,
+                                    _gf_false);
+        if (!port) {
+            ret = -1;
+            gf_msg_debug(THIS->name, 0,
+                         "Couldn't get port "
+                         " for brick %s:%s",
+                         brickinfo->hostname, brickinfo->path);
+            goto out;
+        }
 
-                ret = gf_asprintf (&xl_opts[i], "%s-client-%d.remote-port=%d",
-                                   volinfo->volname, index, port);
-                if (ret == -1) {
-                        xl_opts[i] = NULL;
-                        goto out;
-                }
-                i++;
+        ret = gf_asprintf(&xl_opts[i], "%s-client-%d.remote-port=%d",
+                          volinfo->volname, index, port);
+        if (ret == -1) {
+            xl_opts[i] = NULL;
+            goto out;
         }
+        i++;
+    }
 
-        ret = 0;
+    ret = 0;
 out:
-        return ret;
+    return ret;
 }
 
 int
-glusterd_op_clearlocks_volume (dict_t *dict, char **op_errstr, dict_t *rsp_dict)
+glusterd_op_clearlocks_volume(dict_t *dict, char **op_errstr, dict_t *rsp_dict)
 {
-        int32_t                         ret                 = -1;
-        int                             i                   = 0;
-        char                            *volname            = NULL;
-        char                            *path               = NULL;
-        char                            *kind               = NULL;
-        char                            *type               = NULL;
-        char                            *opts               = NULL;
-        char                            *cmd_str            = NULL;
-        char                            *free_ptr           = NULL;
-        char                            msg[PATH_MAX]       = {0,};
-        char                            result[PATH_MAX]    = {0,};
-        char                            *mntpt              = NULL;
-        char                            **xl_opts           = NULL;
-        glusterd_volinfo_t              *volinfo            = NULL;
-
-        ret = dict_get_str (dict, "volname", &volname);
-        if (ret) {
-                gf_log (THIS->name, GF_LOG_ERROR, "Failed to get volume name");
-                goto out;
-        }
-        gf_log ("", GF_LOG_DEBUG, "Performing clearlocks on volume %s", volname);
-
-        ret = dict_get_str (dict, "path", &path);
-        if (ret) {
-                gf_log (THIS->name, GF_LOG_ERROR, "Failed to get path");
-                goto out;
-        }
-
-        ret = dict_get_str (dict, "kind", &kind);
-        if (ret) {
-                gf_log (THIS->name, GF_LOG_ERROR, "Failed to get kind");
-                goto out;
-        }
-
-        ret = dict_get_str (dict, "type", &type);
-        if (ret) {
-                gf_log (THIS->name, GF_LOG_ERROR, "Failed to get type");
-                goto out;
-        }
+    int32_t ret = -1;
+    int i = 0;
+    char *volname = NULL;
+    char *path = NULL;
+    char *kind = NULL;
+    char *type = NULL;
+    char *opts = NULL;
+    char *cmd_str = NULL;
+    char *free_ptr = NULL;
+    char msg[PATH_MAX] = {
+        0,
+    };
+    char result[PATH_MAX] = {
+        0,
+    };
+    char *mntpt = NULL;
+    char **xl_opts = NULL;
+    glusterd_volinfo_t *volinfo = NULL;
+    xlator_t *this = THIS;
+    GF_ASSERT(this);
+
+    ret = dict_get_strn(dict, "volname", SLEN("volname"), &volname);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+                "Key=volname", NULL);
+        goto out;
+    }
+    gf_msg_debug("glusterd", 0, "Performing clearlocks on volume %s", volname);
 
-        ret = dict_get_str (dict, "opts", &opts);
-        if (ret)
-                ret = 0;
-
-        gf_log (THIS->name, GF_LOG_INFO, "Received clear-locks request for "
-                "volume %s with kind %s type %s and options %s", volname,
-                kind, type, opts);
-
-        if (opts)
-                ret = gf_asprintf (&cmd_str, GF_XATTR_CLRLK_CMD".t%s.k%s.%s",
-                                   type, kind, opts);
-        else
-                ret = gf_asprintf (&cmd_str, GF_XATTR_CLRLK_CMD".t%s.k%s",
-                                   type, kind);
-        if (ret == -1)
-                goto out;
+    ret = dict_get_strn(dict, "path", SLEN("path"), &path);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED, "Key=path",
+                NULL);
+        goto out;
+    }
 
-        ret = glusterd_volinfo_find (volname, &volinfo);
-        if (ret) {
-                snprintf (msg, sizeof (msg), "Volume %s doesn't exist.",
-                          volname);
-                gf_log (THIS->name, GF_LOG_ERROR, "%s", msg);
-                goto out;
-        }
+    ret = dict_get_strn(dict, "kind", SLEN("kind"), &kind);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED, "Key=kind",
+                NULL);
+        goto out;
+    }
 
-        xl_opts = GF_CALLOC (volinfo->brick_count+1, sizeof (char*),
-                             gf_gld_mt_charptr);
-        if (!xl_opts)
-                goto out;
+    ret = dict_get_strn(dict, "type", SLEN("type"), &type);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED, "Key=type",
+                NULL);
+        goto out;
+    }
 
-        ret = glusterd_clearlocks_get_local_client_ports (volinfo, xl_opts);
-        if (ret) {
-                snprintf (msg, sizeof (msg), "Couldn't get port numbers of "
-                          "local bricks");
-                gf_log (THIS->name, GF_LOG_ERROR, "%s", msg);
-                goto out;
-        }
+    ret = dict_get_strn(dict, "opts", SLEN("opts"), &opts);
+    if (ret)
+        ret = 0;
 
-        ret = glusterd_clearlocks_create_mount (volinfo, &mntpt);
-        if (ret) {
-                snprintf (msg, sizeof (msg), "Creating mount directory "
-                          "for clear-locks failed.");
-                gf_log (THIS->name, GF_LOG_ERROR, "%s", msg);
-                goto out;
-        }
+    gf_smsg(this->name, GF_LOG_INFO, 0, GD_MSG_CLRCLK_VOL_REQ_RCVD,
+            "Volume=%s, Kind=%s, Type=%s, Options=%s", volname, kind, type,
+            opts, NULL);
 
-        ret = glusterd_clearlocks_mount (volinfo, xl_opts, mntpt);
-        if (ret) {
-                snprintf (msg, sizeof (msg), "Failed to mount clear-locks "
-                          "maintenance client.");
-                gf_log (THIS->name, GF_LOG_ERROR, "%s", msg);
-                goto out;
-        }
+    if (opts)
+        ret = gf_asprintf(&cmd_str, GF_XATTR_CLRLK_CMD ".t%s.k%s.%s", type,
+                          kind, opts);
+    else
+        ret = gf_asprintf(&cmd_str, GF_XATTR_CLRLK_CMD ".t%s.k%s", type, kind);
+    if (ret == -1)
+        goto out;
 
-        ret = glusterd_clearlocks_send_cmd (volinfo, cmd_str, path, result,
-                                            msg, sizeof (msg), mntpt);
-        if (ret) {
-                gf_log (THIS->name, GF_LOG_ERROR, "%s", msg);
-                goto umount;
-        }
+    ret = glusterd_volinfo_find(volname, &volinfo);
+    if (ret) {
+        snprintf(msg, sizeof(msg), "Volume %s doesn't exist.", volname);
+        gf_smsg(this->name, GF_LOG_ERROR, 0, GD_MSG_VOL_NOT_FOUND, "Volume=%s",
+                volname, NULL);
+        goto out;
+    }
 
-        free_ptr = gf_strdup(result);
-        if (dict_set_dynstr (rsp_dict, "lk-summary", free_ptr)) {
-                GF_FREE (free_ptr);
-                snprintf (msg, sizeof (msg), "Failed to set clear-locks "
-                          "result");
-                gf_log (THIS->name, GF_LOG_ERROR, "%s", msg);
-        }
+    xl_opts = GF_CALLOC(volinfo->brick_count + 1, sizeof(char *),
+                        gf_gld_mt_charptr);
+    if (!xl_opts) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_NO_MEMORY, NULL);
+        goto out;
+    }
+
+    ret = glusterd_clearlocks_get_local_client_ports(volinfo, xl_opts);
+    if (ret) {
+        snprintf(msg, sizeof(msg),
+                 "Couldn't get port numbers of "
+                 "local bricks");
+        gf_smsg(this->name, GF_LOG_ERROR, 0, GD_MSG_BRK_PORT_NUM_GET_FAIL,
+                NULL);
+        goto out;
+    }
+
+    ret = glusterd_clearlocks_create_mount(volinfo, &mntpt);
+    if (ret) {
+        snprintf(msg, sizeof(msg),
+                 "Creating mount directory "
+                 "for clear-locks failed.");
+        gf_smsg(this->name, GF_LOG_ERROR, 0,
+                GD_MSG_CLRLOCKS_MOUNTDIR_CREATE_FAIL, NULL);
+        goto out;
+    }
+
+    ret = glusterd_clearlocks_mount(volinfo, xl_opts, mntpt);
+    if (ret) {
+        snprintf(msg, sizeof(msg),
+                 "Failed to mount clear-locks "
+                 "maintenance client.");
+        gf_smsg(this->name, GF_LOG_ERROR, 0, GD_MSG_CLRLOCKS_CLNT_MOUNT_FAIL,
+                NULL);
+        goto out;
+    }
+
+    ret = glusterd_clearlocks_send_cmd(volinfo, cmd_str, path, result, msg,
+                                       sizeof(msg), mntpt);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_ERROR, 0, GD_MSG_CLRCLK_SND_CMD_FAIL, NULL);
+        goto umount;
+    }
+
+    free_ptr = gf_strdup(result);
+    if (dict_set_dynstrn(rsp_dict, "lk-summary", SLEN("lk-summary"),
+                         free_ptr)) {
+        GF_FREE(free_ptr);
+        snprintf(msg, sizeof(msg),
+                 "Failed to set clear-locks "
+                 "result");
+        gf_smsg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+                "Key=lk-summary", NULL);
+    }
 
 umount:
-        glusterd_clearlocks_unmount (volinfo, mntpt);
+    glusterd_clearlocks_unmount(volinfo, mntpt);
 
-        if (glusterd_clearlocks_rmdir_mount (volinfo, mntpt))
-                gf_log (THIS->name, GF_LOG_WARNING, "Couldn't unmount "
-                        "clear-locks mount point");
+    if (glusterd_clearlocks_rmdir_mount(volinfo, mntpt))
+        gf_smsg(this->name, GF_LOG_WARNING, 0, GD_MSG_CLRLOCKS_CLNT_UMOUNT_FAIL,
+                NULL);
 
 out:
-        if (ret)
-                *op_errstr = gf_strdup (msg);
+    if (ret)
+        *op_errstr = gf_strdup(msg);
 
-        if (xl_opts) {
-                for (i = 0; i < volinfo->brick_count && xl_opts[i]; i++)
-                        GF_FREE (xl_opts[i]);
-                GF_FREE (xl_opts);
-        }
+    if (xl_opts) {
+        for (i = 0; i < volinfo->brick_count && xl_opts[i]; i++)
+            GF_FREE(xl_opts[i]);
+        GF_FREE(xl_opts);
+    }
 
-        GF_FREE (cmd_str);
+    GF_FREE(cmd_str);
 
-        GF_FREE (mntpt);
+    GF_FREE(mntpt);
 
-        return ret;
+    return ret;
 }
diff --git a/xlators/mgmt/glusterd/src/glusterd-volume-set.c b/xlators/mgmt/glusterd/src/glusterd-volume-set.c
index 22cac46e876..398b4d76f52 100644
--- a/xlators/mgmt/glusterd/src/glusterd-volume-set.c
+++ b/xlators/mgmt/glusterd/src/glusterd-volume-set.c
@@ -1,237 +1,817 @@
 /*
-   Copyright (c) 2013 Red Hat, Inc. <http://www.redhat.com>
-   This file is part of GlusterFS.
+Copyright (c) 2013 Red Hat, Inc. <http://www.redhat.com>
+This file is part of GlusterFS.
 
-   This file is licensed to you under your choice of the GNU Lesser
-   General Public License, version 3 or any later version (LGPLv3 or
-   later), or the GNU General Public License, version 2 (GPLv2), in all
-   cases as published by the Free Software Foundation.
+This file is licensed to you under your choice of the GNU Lesser
+General Public License, version 3 or any later version (LGPLv3 or
+later), or the GNU General Public License, version 2 (GPLv2), in all
+cases as published by the Free Software Foundation.
 */
 
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
-
+#include <glusterfs/syscall.h>
 #include "glusterd-volgen.h"
 #include "glusterd-utils.h"
 
 static int
-validate_cache_max_min_size (glusterd_volinfo_t *volinfo, dict_t *dict,
-                             char *key, char *value, char **op_errstr)
+validate_cache_max_min_size(glusterd_volinfo_t *volinfo, dict_t *dict,
+                            char *key, char *value, char **op_errstr)
 {
-        char                *current_max_value = NULL;
-        char                *current_min_value = NULL;
-        char                 errstr[2048]  = "";
-        glusterd_conf_t     *priv          = NULL;
-        int                  ret           = 0;
-        uint64_t             max_value     = 0;
-        uint64_t             min_value     = 0;
-        xlator_t            *this          = NULL;
-
-        this = THIS;
-        GF_ASSERT (this);
-        priv = this->private;
-        GF_ASSERT (priv);
-
-        if ((!strcmp (key, "performance.cache-min-file-size")) ||
-            (!strcmp (key, "cache-min-file-size"))) {
-                glusterd_volinfo_get (volinfo,
-                                      "performance.cache-max-file-size",
-                                      &current_max_value);
-                if (current_max_value) {
-                        gf_string2bytesize_uint64 (current_max_value, &max_value);
-                        gf_string2bytesize_uint64 (value, &min_value);
-                        current_min_value = value;
-                }
-        } else  if ((!strcmp (key, "performance.cache-max-file-size")) ||
-                    (!strcmp (key, "cache-max-file-size"))) {
-                glusterd_volinfo_get (volinfo,
-                                      "performance.cache-min-file-size",
-                                      &current_min_value);
-                if (current_min_value) {
-                        gf_string2bytesize_uint64 (current_min_value, &min_value);
-                        gf_string2bytesize_uint64 (value, &max_value);
-                        current_max_value = value;
-                }
+    char *current_max_value = NULL;
+    char *current_min_value = NULL;
+    char errstr[2048] = "";
+    glusterd_conf_t *priv = NULL;
+    int ret = 0;
+    uint64_t max_value = 0;
+    uint64_t min_value = 0;
+    xlator_t *this = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+    priv = this->private;
+    GF_ASSERT(priv);
+
+    if ((!strcmp(key, "performance.cache-min-file-size")) ||
+        (!strcmp(key, "cache-min-file-size"))) {
+        glusterd_volinfo_get(volinfo, "performance.cache-max-file-size",
+                             &current_max_value);
+        if (current_max_value) {
+            gf_string2bytesize_uint64(current_max_value, &max_value);
+            gf_string2bytesize_uint64(value, &min_value);
+            current_min_value = value;
         }
-
-        if (min_value > max_value) {
-                snprintf (errstr, sizeof (errstr),
-                          "cache-min-file-size (%s) is greater than "
-                          "cache-max-file-size (%s)",
-                          current_min_value, current_max_value);
-                gf_log (this->name, GF_LOG_ERROR, "%s", errstr);
-                *op_errstr = gf_strdup (errstr);
-                ret = -1;
-                goto out;
+    } else if ((!strcmp(key, "performance.cache-max-file-size")) ||
+               (!strcmp(key, "cache-max-file-size"))) {
+        glusterd_volinfo_get(volinfo, "performance.cache-min-file-size",
+                             &current_min_value);
+        if (current_min_value) {
+            gf_string2bytesize_uint64(current_min_value, &min_value);
+            gf_string2bytesize_uint64(value, &max_value);
+            current_max_value = value;
         }
+    }
+
+    if (min_value > max_value) {
+        snprintf(errstr, sizeof(errstr),
+                 "cache-min-file-size (%s) is greater than "
+                 "cache-max-file-size (%s)",
+                 current_min_value, current_max_value);
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_CACHE_MINMAX_SIZE_INVALID,
+               "%s", errstr);
+        *op_errstr = gf_strdup(errstr);
+        ret = -1;
+        goto out;
+    }
 
 out:
-        gf_log (this->name, GF_LOG_DEBUG, "Returning %d", ret);
+    gf_msg_debug(this->name, 0, "Returning %d", ret);
 
-        return ret;
+    return ret;
 }
 
 static int
-validate_quota (glusterd_volinfo_t *volinfo, dict_t *dict, char *key,
-                char *value, char **op_errstr)
+validate_defrag_throttle_option(glusterd_volinfo_t *volinfo, dict_t *dict,
+                                char *key, char *value, char **op_errstr)
 {
-        char                 errstr[2048] = "";
-        glusterd_conf_t     *priv         = NULL;
-        int                  ret          = 0;
-        xlator_t            *this         = NULL;
-
-        this = THIS;
-        GF_ASSERT (this);
-        priv = this->private;
-        GF_ASSERT (priv);
-
-        ret = glusterd_volinfo_get_boolean (volinfo, VKEY_FEATURES_QUOTA);
-        if (ret == -1) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "failed to get the quota status");
-                goto out;
-        }
+    char errstr[2048] = "";
+    int ret = 0;
+    xlator_t *this = NULL;
+    int thread_count = 0;
+    long int cores_available = 0;
 
-        if (ret == _gf_false) {
-                snprintf (errstr, sizeof (errstr),
-                          "Cannot set %s. Enable quota first.", key);
-                gf_log (this->name, GF_LOG_ERROR, "%s", errstr);
-                *op_errstr = gf_strdup (errstr);
-                ret = -1;
-                goto out;
-        }
+    this = THIS;
+    GF_ASSERT(this);
+
+    cores_available = sysconf(_SC_NPROCESSORS_ONLN);
+
+    /* Throttle option should be one of lazy|normal|aggressive or a number
+     * configured by user max up to the number of cores in the machine */
 
+    if (!strcasecmp(value, "lazy") || !strcasecmp(value, "normal") ||
+        !strcasecmp(value, "aggressive")) {
         ret = 0;
+    } else if ((gf_string2int(value, &thread_count) == 0)) {
+        if ((thread_count > 0) && (thread_count <= cores_available)) {
+            ret = 0;
+        } else {
+            ret = -1;
+            snprintf(errstr, sizeof(errstr),
+                     "%s should be within"
+                     " range of 0 and maximum number of cores "
+                     "available (cores available - %ld)",
+                     key, cores_available);
+
+            gf_msg(this->name, GF_LOG_ERROR, EINVAL, GD_MSG_INVALID_ENTRY, "%s",
+                   errstr);
+
+            *op_errstr = gf_strdup(errstr);
+        }
+    } else {
+        ret = -1;
+        snprintf(errstr, sizeof(errstr),
+                 "%s should be "
+                 "{lazy|normal|aggressive} or a number up to number of"
+                 " cores available (cores available - %ld)",
+                 key, cores_available);
+        gf_msg(this->name, GF_LOG_ERROR, EINVAL, GD_MSG_INVALID_ENTRY, "%s",
+               errstr);
+        *op_errstr = gf_strdup(errstr);
+    }
+
+    gf_msg_debug(this->name, 0, "Returning %d", ret);
+
+    return ret;
+}
+
+static int
+validate_quota(glusterd_volinfo_t *volinfo, dict_t *dict, char *key,
+               char *value, char **op_errstr)
+{
+    char errstr[2048] = "";
+    glusterd_conf_t *priv = NULL;
+    int ret = 0;
+    xlator_t *this = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+    priv = this->private;
+    GF_ASSERT(priv);
+
+    ret = glusterd_volinfo_get_boolean(volinfo, VKEY_FEATURES_QUOTA);
+    if (ret == -1) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_QUOTA_GET_STAT_FAIL,
+               "failed to get the quota status");
+        goto out;
+    }
+
+    if (ret == _gf_false) {
+        snprintf(errstr, sizeof(errstr), "Cannot set %s. Enable quota first.",
+                 key);
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_QUOTA_DISABLED, "%s",
+               errstr);
+        *op_errstr = gf_strdup(errstr);
+        ret = -1;
+        goto out;
+    }
+
+    ret = 0;
 out:
-        gf_log (this->name, GF_LOG_DEBUG, "Returning %d", ret);
+    gf_msg_debug(this->name, 0, "Returning %d", ret);
 
-        return ret;
+    return ret;
 }
 
 static int
-validate_uss (glusterd_volinfo_t *volinfo, dict_t *dict, char *key,
-              char *value, char **op_errstr)
+validate_uss(glusterd_volinfo_t *volinfo, dict_t *dict, char *key, char *value,
+             char **op_errstr)
 {
-        char                 errstr[2048]  = "";
-        int                  ret           = 0;
-        xlator_t            *this          = NULL;
-        gf_boolean_t         b             = _gf_false;
-
-        this = THIS;
-        GF_ASSERT (this);
-
-        ret = gf_string2boolean (value, &b);
-        if (ret) {
-                snprintf (errstr, sizeof (errstr), "%s is not a valid boolean "
-                          "value. %s expects a valid boolean value.", value,
-                          key);
-                gf_log (this->name, GF_LOG_ERROR, "%s", errstr);
-                *op_errstr = gf_strdup (errstr);
-                goto out;
-        }
+    char errstr[2048] = "";
+    int ret = 0;
+    xlator_t *this = NULL;
+    gf_boolean_t b = _gf_false;
+
+    this = THIS;
+    GF_ASSERT(this);
+
+    ret = gf_string2boolean(value, &b);
+    if (ret) {
+        snprintf(errstr, sizeof(errstr),
+                 "%s is not a valid boolean "
+                 "value. %s expects a valid boolean value.",
+                 value, key);
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_INVALID_ENTRY, "%s", errstr);
+        *op_errstr = gf_strdup(errstr);
+        goto out;
+    }
 out:
-        gf_log (this->name, GF_LOG_DEBUG, "Returning %d", ret);
+    gf_msg_debug(this->name, 0, "Returning %d", ret);
 
-        return ret;
+    return ret;
 }
 
 static int
-validate_stripe (glusterd_volinfo_t *volinfo, dict_t *dict, char *key,
+validate_uss_dir(glusterd_volinfo_t *volinfo, dict_t *dict, char *key,
                  char *value, char **op_errstr)
 {
-        char                 errstr[2048]  = "";
-        glusterd_conf_t     *priv          = NULL;
-        int                  ret           = 0;
-        xlator_t            *this          = NULL;
-
-        this = THIS;
-        GF_ASSERT (this);
-        priv = this->private;
-        GF_ASSERT (priv);
-
-        if (volinfo->stripe_count == 1) {
-                snprintf (errstr, sizeof (errstr),
-                          "Cannot set %s for a non-stripe volume.", key);
-                gf_log (this->name, GF_LOG_ERROR, "%s", errstr);
-                *op_errstr = gf_strdup (errstr);
-                ret = -1;
-               goto out;
-        }
+    char errstr[2048] = "";
+    int ret = -1;
+    int i = 0;
+    xlator_t *this = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+
+    i = strlen(value);
+    if (i > NAME_MAX) {
+        snprintf(errstr, sizeof(errstr),
+                 "value of %s exceedes %d "
+                 "characters",
+                 key, NAME_MAX);
+        goto out;
+    } else if (i < 2) {
+        snprintf(errstr, sizeof(errstr),
+                 "value of %s too short, "
+                 "expects at least two characters",
+                 key);
+        goto out;
+    }
+
+    if (value[0] != '.') {
+        snprintf(errstr, sizeof(errstr),
+                 "%s expects value starting "
+                 "with '.' ",
+                 key);
+        goto out;
+    }
+
+    for (i = 1; value[i]; i++) {
+        if (isalnum(value[i]) || value[i] == '_' || value[i] == '-')
+            continue;
+
+        snprintf(errstr, sizeof(errstr),
+                 "%s expects value to"
+                 " contain only '0-9a-z-_'",
+                 key);
+        goto out;
+    }
+
+    ret = 0;
+out:
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, EINVAL, GD_MSG_INVALID_ENTRY, "%s",
+               errstr);
+        *op_errstr = gf_strdup(errstr);
+    }
 
+    gf_msg_debug(this->name, 0, "Returning %d", ret);
+
+    return ret;
+}
+
+static int
+validate_server_options(glusterd_volinfo_t *volinfo, dict_t *dict, char *key,
+                        char *value, char **op_errstr)
+{
+    char errstr[2048] = "";
+    xlator_t *this = NULL;
+    int ret = -1;
+    int origin_val = 0;
+
+    this = THIS;
+    GF_ASSERT(this);
+
+    if (volinfo->status == GLUSTERD_STATUS_STARTED) {
+        gf_msg(this->name, GF_LOG_INFO, 0, GD_MSG_VOL_SET_VALIDATION_INFO,
+               "Please note that "
+               "volume %s is started. This option will only get "
+               "effected after a brick restart.",
+               volinfo->volname);
+    }
+
+    ret = gf_string2int(value, &origin_val);
+    if (ret) {
+        snprintf(errstr, sizeof(errstr),
+                 "%s is not a compatible "
+                 "value. %s expects an integer value.",
+                 value, key);
+        ret = -1;
+        goto out;
+    }
+
+    if (origin_val < 0) {
+        snprintf(errstr, sizeof(errstr),
+                 "%s is not a "
+                 "compatible value. %s expects a positive"
+                 "integer value.",
+                 value, key);
+        ret = -1;
+        goto out;
+    }
+
+    ret = 0;
 out:
-        gf_log (this->name, GF_LOG_DEBUG, "Returning %d", ret);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, EINVAL, GD_MSG_INCOMPATIBLE_VALUE,
+               "%s", errstr);
+        *op_errstr = gf_strdup(errstr);
+    }
 
-        return ret;
+    return ret;
 }
 
 static int
-validate_subvols_per_directory (glusterd_volinfo_t *volinfo, dict_t *dict,
-                                char *key, char *value, char **op_errstr)
+validate_disperse(glusterd_volinfo_t *volinfo, dict_t *dict, char *key,
+                  char *value, char **op_errstr)
 {
-        char                 errstr[2048]  = "";
-        glusterd_conf_t     *priv          = NULL;
-        int                  ret           = 0;
-        int                  subvols       = 0;
-        xlator_t            *this          = NULL;
-
-        this = THIS;
-        GF_ASSERT (this);
-        priv = this->private;
-        GF_ASSERT (priv);
-
-        subvols = atoi(value);
-
-        /* Checking if the subvols-per-directory exceed the total
-           number of subvolumes. */
-        if (subvols > volinfo->subvol_count) {
-                snprintf (errstr, sizeof(errstr),
-                          "subvols-per-directory(%d) is greater "
-                          "than the number of subvolumes(%d).",
-                          subvols, volinfo->subvol_count);
-                gf_log (this->name, GF_LOG_ERROR,
-                        "%s.", errstr);
-                *op_errstr = gf_strdup (errstr);
-                ret = -1;
-                goto out;
-        }
+    char errstr[2048] = "";
+    int ret = -1;
+    xlator_t *this = NULL;
+
+    this = THIS;
+    GF_VALIDATE_OR_GOTO("glusterd", this, out);
+
+    if (volinfo->type != GF_CLUSTER_TYPE_DISPERSE) {
+        snprintf(errstr, sizeof(errstr),
+                 "Cannot set %s for a non-disperse volume.", key);
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_VOL_NOT_DISPERSE, "%s",
+               errstr);
+        *op_errstr = gf_strdup(errstr);
+        ret = -1;
+        goto out;
+    }
+    ret = 0;
+
+out:
+    gf_msg_debug(ret == 0 ? THIS->name : "glusterd", 0, "Returning %d", ret);
+
+    return ret;
+}
+
+static int
+validate_replica(glusterd_volinfo_t *volinfo, dict_t *dict, char *key,
+                 char *value, char **op_errstr)
+{
+    char errstr[2048] = "";
+    int ret = 0;
+    xlator_t *this = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+
+    if (volinfo->replica_count == 1) {
+        snprintf(errstr, sizeof(errstr),
+                 "Cannot set %s for a non-replicate volume.", key);
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_VOL_NOT_REPLICA, "%s",
+               errstr);
+        *op_errstr = gf_strdup(errstr);
+        ret = -1;
+        goto out;
+    }
 
 out:
-        gf_log (this->name, GF_LOG_DEBUG, "Returning %d", ret);
+    gf_msg_debug(this->name, 0, "Returning %d", ret);
 
-        return ret;
+    return ret;
 }
 
 static int
-validate_replica_heal_enable_disable (glusterd_volinfo_t *volinfo, dict_t *dict,
+validate_quorum_count(glusterd_volinfo_t *volinfo, dict_t *dict, char *key,
+                      char *value, char **op_errstr)
+{
+    int ret = 0;
+    xlator_t *this = NULL;
+    int q_count = 0;
+
+    this = THIS;
+    GF_ASSERT(this);
+
+    ret = gf_string2int(value, &q_count);
+    if (ret) {
+        gf_asprintf(op_errstr,
+                    "%s is not an integer. %s expects a "
+                    "valid integer value.",
+                    value, key);
+        goto out;
+    }
+
+    if (q_count < 1 || q_count > volinfo->replica_count) {
+        gf_asprintf(op_errstr, "%d in %s %d is out of range [1 - %d]", q_count,
+                    key, q_count, volinfo->replica_count);
+        ret = -1;
+    }
+
+out:
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_INVALID_ENTRY, "%s",
+               *op_errstr);
+    }
+    gf_msg_debug(this->name, 0, "Returning %d", ret);
+
+    return ret;
+}
+
+static int
+validate_subvols_per_directory(glusterd_volinfo_t *volinfo, dict_t *dict,
+                               char *key, char *value, char **op_errstr)
+{
+    char errstr[2048] = "";
+    glusterd_conf_t *priv = NULL;
+    int ret = 0;
+    int subvols = 0;
+    xlator_t *this = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+    priv = this->private;
+    GF_ASSERT(priv);
+
+    subvols = atoi(value);
+
+    /* Checking if the subvols-per-directory exceed the total
+       number of subvolumes. */
+    if (subvols > volinfo->subvol_count) {
+        snprintf(errstr, sizeof(errstr),
+                 "subvols-per-directory(%d) is greater "
+                 "than the number of subvolumes(%d).",
+                 subvols, volinfo->subvol_count);
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_SUBVOLUMES_EXCEED, "%s.",
+               errstr);
+        *op_errstr = gf_strdup(errstr);
+        ret = -1;
+        goto out;
+    }
+
+out:
+    gf_msg_debug(this->name, 0, "Returning %d", ret);
+
+    return ret;
+}
+
+static int
+validate_replica_heal_enable_disable(glusterd_volinfo_t *volinfo, dict_t *dict,
+                                     char *key, char *value, char **op_errstr)
+{
+    int ret = 0;
+
+    if (!glusterd_is_volume_replicate(volinfo)) {
+        gf_asprintf(op_errstr, "Volume %s is not of replicate type",
+                    volinfo->volname);
+        ret = -1;
+    }
+
+    return ret;
+}
+
+static int
+validate_mandatory_locking(glusterd_volinfo_t *volinfo, dict_t *dict, char *key,
+                           char *value, char **op_errstr)
+{
+    char errstr[2048] = "";
+    int ret = 0;
+    xlator_t *this = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+
+    if (strcmp(value, "off") != 0 && strcmp(value, "file") != 0 &&
+        strcmp(value, "forced") != 0 && strcmp(value, "optimal") != 0) {
+        snprintf(errstr, sizeof(errstr),
+                 "Invalid option value '%s':"
+                 " Available options are 'off', 'file', "
+                 "'forced' or 'optimal'",
+                 value);
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_INVALID_ENTRY, "%s", errstr);
+        *op_errstr = gf_strdup(errstr);
+        ret = -1;
+        goto out;
+    }
+out:
+    gf_msg_debug(this->name, 0, "Returning %d", ret);
+
+    return ret;
+}
+
+static int
+validate_disperse_heal_enable_disable(glusterd_volinfo_t *volinfo, dict_t *dict,
                                       char *key, char *value, char **op_errstr)
 {
-        int                  ret = 0;
+    int ret = 0;
 
-        if (!glusterd_is_volume_replicate (volinfo)) {
-                gf_asprintf (op_errstr, "Volume %s is not of replicate type",
-                             volinfo->volname);
-                ret = -1;
-        }
+    if (volinfo->type != GF_CLUSTER_TYPE_DISPERSE) {
+        gf_asprintf(op_errstr, "Volume %s is not of disperse type",
+                    volinfo->volname);
+        ret = -1;
+    }
 
-        return ret;
+    return ret;
 }
 
 static int
-validate_disperse_heal_enable_disable (glusterd_volinfo_t *volinfo,
-                                       dict_t *dict, char *key, char *value,
-                                       char **op_errstr)
+validate_lock_migration_option(glusterd_volinfo_t *volinfo, dict_t *dict,
+                               char *key, char *value, char **op_errstr)
 {
-        int                  ret = 0;
+    char errstr[2048] = "";
+    int ret = 0;
+    xlator_t *this = NULL;
+    gf_boolean_t b = _gf_false;
 
-        if (volinfo->type != GF_CLUSTER_TYPE_DISPERSE) {
-                gf_asprintf (op_errstr, "Volume %s is not of disperse type",
-                             volinfo->volname);
-                ret = -1;
-        }
+    this = THIS;
+    GF_ASSERT(this);
+
+    if (volinfo->replica_count > 1 || volinfo->disperse_count) {
+        snprintf(errstr, sizeof(errstr),
+                 "Lock migration is "
+                 "a experimental feature. Currently works with"
+                 " pure distribute volume only");
+        ret = -1;
+
+        gf_msg(this->name, GF_LOG_ERROR, EINVAL, GD_MSG_INVALID_ENTRY, "%s",
+               errstr);
+
+        *op_errstr = gf_strdup(errstr);
+        goto out;
+    }
 
-        return ret;
+    ret = gf_string2boolean(value, &b);
+    if (ret) {
+        snprintf(errstr, sizeof(errstr),
+                 "Invalid value"
+                 " for volume set command. Use on/off only.");
+        ret = -1;
+
+        gf_msg(this->name, GF_LOG_ERROR, EINVAL, GD_MSG_INVALID_ENTRY, "%s",
+               errstr);
+
+        *op_errstr = gf_strdup(errstr);
+
+        goto out;
+    }
+
+    gf_msg_debug(this->name, 0, "Returning %d", ret);
+
+out:
+    return ret;
+}
+
+static int
+validate_mux_limit(glusterd_volinfo_t *volinfo, dict_t *dict, char *key,
+                   char *value, char **op_errstr)
+{
+    xlator_t *this = NULL;
+    uint val = 0;
+    int ret = -1;
+
+    this = THIS;
+    GF_VALIDATE_OR_GOTO("glusterd", this, out);
+
+    if (!is_brick_mx_enabled()) {
+        gf_asprintf(op_errstr,
+                    "Brick-multiplexing is not enabled. "
+                    "Please enable brick multiplexing before trying "
+                    "to set this option.");
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_WRONG_OPTS_SETTING, "%s",
+               *op_errstr);
+        goto out;
+    }
+
+    ret = gf_string2uint(value, &val);
+    if (ret) {
+        gf_asprintf(op_errstr,
+                    "%s is not a valid count. "
+                    "%s expects an unsigned integer.",
+                    value, key);
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_INVALID_ENTRY, "%s",
+               *op_errstr);
+    }
+
+    if (val == 1) {
+        gf_asprintf(op_errstr,
+                    "Brick-multiplexing is enabled. "
+                    "Please set this option to a value other than 1 "
+                    "to make use of the brick-multiplexing feature.");
+        ret = -1;
+        goto out;
+    }
+out:
+    gf_msg_debug("glusterd", 0, "Returning %d", ret);
+
+    return ret;
+}
+
+static int
+validate_volume_per_thread_limit(glusterd_volinfo_t *volinfo, dict_t *dict,
+                                 char *key, char *value, char **op_errstr)
+{
+    xlator_t *this = NULL;
+    uint val = 0;
+    int ret = -1;
+
+    this = THIS;
+    GF_VALIDATE_OR_GOTO("glusterd", this, out);
+
+    if (!is_brick_mx_enabled()) {
+        gf_asprintf(op_errstr,
+                    "Brick-multiplexing is not enabled. "
+                    "Please enable brick multiplexing before trying "
+                    "to set this option.");
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_WRONG_OPTS_SETTING, "%s",
+               *op_errstr);
+        goto out;
+    }
+
+    ret = gf_string2uint(value, &val);
+    if (ret) {
+        gf_asprintf(op_errstr,
+                    "%s is not a valid count. "
+                    "%s expects an unsigned integer.",
+                    value, key);
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_INVALID_ENTRY, "%s",
+               *op_errstr);
+    }
+
+    if ((val < 5) || (val > 200)) {
+        gf_asprintf(
+            op_errstr,
+            "Please set this option to a value between 5 and 200 to"
+            "optimize processing large numbers of volumes in parallel.");
+        ret = -1;
+        goto out;
+    }
+out:
+    gf_msg_debug("glusterd", 0, "Returning %d", ret);
+
+    return ret;
+}
+
+static int
+validate_boolean(glusterd_volinfo_t *volinfo, dict_t *dict, char *key,
+                 char *value, char **op_errstr)
+{
+    xlator_t *this = NULL;
+    gf_boolean_t b = _gf_false;
+    int ret = -1;
+
+    this = THIS;
+    GF_VALIDATE_OR_GOTO("glusterd", this, out);
+    ret = gf_string2boolean(value, &b);
+    if (ret) {
+        gf_asprintf(op_errstr,
+                    "%s is not a valid boolean value. %s "
+                    "expects a valid boolean value.",
+                    value, key);
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_INVALID_ENTRY, "%s",
+               *op_errstr);
+    }
+out:
+    gf_msg_debug("glusterd", 0, "Returning %d", ret);
+
+    return ret;
+}
+
+static int
+validate_disperse_quorum_count(glusterd_volinfo_t *volinfo, dict_t *dict,
+                               char *key, char *value, char **op_errstr)
+{
+    int ret = -1;
+    int quorum_count = 0;
+    int data_count = 0;
+
+    ret = gf_string2int(value, &quorum_count);
+    if (ret) {
+        gf_asprintf(op_errstr,
+                    "%s is not an integer. %s expects a "
+                    "valid integer value.",
+                    value, key);
+        goto out;
+    }
+
+    if (volinfo->type != GF_CLUSTER_TYPE_DISPERSE) {
+        gf_asprintf(op_errstr, "Cannot set %s for a non-disperse volume.", key);
+        ret = -1;
+        goto out;
+    }
+
+    data_count = volinfo->disperse_count - volinfo->redundancy_count;
+    if (quorum_count < data_count || quorum_count > volinfo->disperse_count) {
+        gf_asprintf(op_errstr, "%d for %s is out of range [%d - %d]",
+                    quorum_count, key, data_count, volinfo->disperse_count);
+        ret = -1;
+        goto out;
+    }
+
+    ret = 0;
+out:
+    return ret;
+}
+
+static int
+validate_parallel_readdir(glusterd_volinfo_t *volinfo, dict_t *dict, char *key,
+                          char *value, char **op_errstr)
+{
+    int ret = -1;
+
+    ret = validate_boolean(volinfo, dict, key, value, op_errstr);
+    if (ret)
+        goto out;
+
+    ret = glusterd_is_defrag_on(volinfo);
+    if (ret) {
+        gf_asprintf(op_errstr,
+                    "%s option should be set "
+                    "after rebalance is complete",
+                    key);
+        gf_msg("glusterd", GF_LOG_ERROR, 0, GD_MSG_INVALID_ENTRY, "%s",
+               *op_errstr);
+    }
+out:
+    gf_msg_debug("glusterd", 0, "Returning %d", ret);
+
+    return ret;
+}
+
+static int
+validate_rda_cache_limit(glusterd_volinfo_t *volinfo, dict_t *dict, char *key,
+                         char *value, char **op_errstr)
+{
+    int ret = 0;
+    uint64_t rda_cache_size = 0;
+
+    ret = gf_string2bytesize_uint64(value, &rda_cache_size);
+    if (ret < 0)
+        goto out;
+
+    if (rda_cache_size <= (1 * GF_UNIT_GB))
+        goto out;
+
+    /* With release 3.11 the max value of rda_cache_limit is changed from
+     * 1GB to INFINITY. If there are clients older than 3.11 and the value
+     * of rda-cache-limit is set to > 1GB, the older clients will stop
+     * working. Hence if a user is setting rda-cache-limit to > 1GB
+     * ensure that all the clients are 3.11 or greater.
+     */
+    ret = glusterd_check_client_op_version_support(
+        volinfo->volname, GD_OP_VERSION_3_11_0, op_errstr);
+out:
+    gf_msg_debug("glusterd", 0, "Returning %d", ret);
+
+    return ret;
+}
+
+static int
+validate_worm_period(glusterd_volinfo_t *volinfo, dict_t *dict, char *key,
+                     char *value, char **op_errstr)
+{
+    xlator_t *this = NULL;
+    uint64_t period = -1;
+    int ret = -1;
+
+    this = THIS;
+    GF_VALIDATE_OR_GOTO("glusterd", this, out);
+    ret = gf_string2uint64(value, &period);
+    if (ret) {
+        gf_asprintf(op_errstr,
+                    "%s is not a valid uint64_t value."
+                    " %s expects a valid uint64_t value.",
+                    value, key);
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_INVALID_ENTRY, "%s",
+               *op_errstr);
+    }
+out:
+    gf_msg_debug("glusterd", 0, "Returning %d", ret);
+
+    return ret;
+}
+
+static int
+validate_reten_mode(glusterd_volinfo_t *volinfo, dict_t *dict, char *key,
+                    char *value, char **op_errstr)
+{
+    xlator_t *this = NULL;
+    int ret = -1;
+
+    this = THIS;
+    GF_VALIDATE_OR_GOTO("glusterd", this, out);
+    if ((strcmp(value, "relax") && strcmp(value, "enterprise"))) {
+        gf_asprintf(op_errstr,
+                    "The value of retention mode should be "
+                    "either relax or enterprise. But the value"
+                    " of %s is %s",
+                    key, value);
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_INVALID_ENTRY, "%s",
+               *op_errstr);
+        ret = -1;
+        goto out;
+    }
+    ret = 0;
+out:
+    gf_msg_debug("glusterd", 0, "Returning %d", ret);
+
+    return ret;
+}
+static int
+is_directory(const char *path)
+{
+    struct stat statbuf;
+    if (sys_stat(path, &statbuf) != 0)
+        return 0;
+    return S_ISDIR(statbuf.st_mode);
+}
+static int
+validate_statedump_path(glusterd_volinfo_t *volinfo, dict_t *dict, char *key,
+                        char *value, char **op_errstr)
+{
+    xlator_t *this = NULL;
+    this = THIS;
+    GF_ASSERT(this);
+
+    int ret = 0;
+    if (!is_directory(value)) {
+        gf_asprintf(op_errstr, "Failed: %s is not a directory", value);
+        ret = -1;
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_INVALID_ENTRY, "%s",
+               *op_errstr);
+    }
+
+    return ret;
 }
 
 /* dispatch table for VOLUME SET
@@ -254,7 +834,7 @@ validate_disperse_heal_enable_disable (glusterd_volinfo_t *volinfo,
  * Fifth field is <doctype>, which decides if the option is public and available
  * in "set help" or not. "NO_DOC" entries are not part of the public interface
  * and are subject to change at any time. This also decides if an option is
- * global (apllies to all volumes) or normal (applies to only specified volume).
+ * global (applies to all volumes) or normal (applies to only specified volume).
  *
  * Sixth field is <flags>.
  *
@@ -263,7 +843,7 @@ validate_disperse_heal_enable_disable (glusterd_volinfo_t *volinfo,
  * Eight field is description of option: If NULL, tried to fetch from
  * translator code's xlator_options table.
  *
- * Nineth field is validation function: If NULL, xlator's option specific
+ * Ninth field is validation function: If NULL, xlator's option specific
  * validation will be tried, otherwise tried at glusterd code itself.
  *
  * There are two type of entries: basic and special.
@@ -298,1500 +878,2269 @@ validate_disperse_heal_enable_disable (glusterd_volinfo_t *volinfo,
  */
 
 struct volopt_map_entry glusterd_volopt_map[] = {
-        /* DHT xlator options */
-        { .key        = "cluster.lookup-unhashed",
-          .voltype    = "cluster/distribute",
-          .op_version = 1,
-          .flags      = OPT_FLAG_CLIENT_OPT
-        },
-        { .key        = "cluster.min-free-disk",
-          .voltype    = "cluster/distribute",
-          .op_version = 1,
-          .flags      = OPT_FLAG_CLIENT_OPT
-        },
-        { .key        = "cluster.min-free-inodes",
-          .voltype    = "cluster/distribute",
-          .op_version = 1,
-          .flags      = OPT_FLAG_CLIENT_OPT
-        },
-        { .key        = "cluster.rebalance-stats",
-          .voltype    = "cluster/distribute",
-          .op_version = 2,
-          .flags      = OPT_FLAG_CLIENT_OPT
-        },
-        { .key         = "cluster.subvols-per-directory",
-          .voltype     = "cluster/distribute",
-          .option      = "directory-layout-spread",
-          .op_version  = 2,
-          .validate_fn = validate_subvols_per_directory,
-          .flags       = OPT_FLAG_CLIENT_OPT
-        },
-        { .key        = "cluster.readdir-optimize",
-          .voltype    = "cluster/distribute",
-          .op_version = 1,
-          .flags      = OPT_FLAG_CLIENT_OPT
-        },
-        { .key        = "cluster.rsync-hash-regex",
-          .voltype    = "cluster/distribute",
-          .type       = NO_DOC,
-          .op_version = 3,
-          .flags      = OPT_FLAG_CLIENT_OPT
-        },
-        { .key        = "cluster.extra-hash-regex",
-          .voltype    = "cluster/distribute",
-          .type       = NO_DOC,
-          .op_version = 3,
-          .flags      = OPT_FLAG_CLIENT_OPT
-        },
-        { .key        = "cluster.dht-xattr-name",
-          .voltype    = "cluster/distribute",
-          .option     = "xattr-name",
-          .type       = NO_DOC,
-          .op_version = 3,
-          .flags      = OPT_FLAG_CLIENT_OPT
-        },
-        { .key        = "cluster.randomize-hash-range-by-gfid",
-          .voltype    = "cluster/distribute",
-          .option     = "randomize-hash-range-by-gfid",
-          .type       = NO_DOC,
-          .op_version = GD_OP_VERSION_3_6_0,
-          .flags      = OPT_FLAG_CLIENT_OPT,
-        },
-        /* NUFA xlator options (Distribute special case) */
-        { .key        = "cluster.nufa",
-          .voltype    = "cluster/distribute",
-          .option     = "!nufa",
-          .type       = NO_DOC,
-          .op_version = 2,
-          .flags      = OPT_FLAG_CLIENT_OPT
-        },
-        { .key        = "cluster.local-volume-name",
-          .voltype    = "cluster/nufa",
-          .option     = "local-volume-name",
-          .type       = NO_DOC,
-          .op_version = 3,
-          .flags      = OPT_FLAG_CLIENT_OPT
-        },
-        { .key        = "cluster.weighted-rebalance",
-          .voltype    = "cluster/distribute",
-          .op_version = GD_OP_VERSION_3_6_0,
-        },
-
-        /* Switch xlator options (Distribute special case) */
-        { .key        = "cluster.switch",
-          .voltype    = "cluster/distribute",
-          .option     = "!switch",
-          .type       = NO_DOC,
-          .op_version = 3,
-          .flags      = OPT_FLAG_CLIENT_OPT
-        },
-        { .key        = "cluster.switch-pattern",
-          .voltype    = "cluster/switch",
-          .option     = "pattern.switch.case",
-          .type       = NO_DOC,
-          .op_version = 3,
-          .flags      = OPT_FLAG_CLIENT_OPT
-        },
-
-        /* AFR xlator options */
-        { .key        = "cluster.entry-change-log",
-          .voltype    = "cluster/replicate",
-          .op_version = 1,
-          .flags      = OPT_FLAG_CLIENT_OPT
-        },
-        { .key        = "cluster.read-subvolume",
-          .voltype    = "cluster/replicate",
-          .op_version = 1,
-          .flags     = OPT_FLAG_CLIENT_OPT
-        },
-        { .key        = "cluster.read-subvolume-index",
-          .voltype    = "cluster/replicate",
-          .op_version = 2,
-          .flags      = OPT_FLAG_CLIENT_OPT
-        },
-        { .key        = "cluster.read-hash-mode",
-          .voltype    = "cluster/replicate",
-          .op_version = 2,
-          .flags      = OPT_FLAG_CLIENT_OPT
-        },
-        { .key        = "cluster.background-self-heal-count",
-          .voltype    = "cluster/replicate",
-          .op_version = 1,
-          .flags      = OPT_FLAG_CLIENT_OPT
-        },
-        { .key        = "cluster.metadata-self-heal",
-          .voltype    = "cluster/replicate",
-          .op_version = 1,
-          .flags      = OPT_FLAG_CLIENT_OPT
-        },
-        { .key        = "cluster.data-self-heal",
-          .voltype    = "cluster/replicate",
-          .op_version = 1,
-          .flags      = OPT_FLAG_CLIENT_OPT
-        },
-        { .key        = "cluster.entry-self-heal",
-          .voltype    = "cluster/replicate",
-          .op_version = 1,
-          .flags      = OPT_FLAG_CLIENT_OPT
-        },
-        { .key           = "cluster.self-heal-daemon",
-          .voltype       = "cluster/replicate",
-          .option        = "!self-heal-daemon",
-          .op_version    = 1,
-          .validate_fn   = validate_replica_heal_enable_disable
-        },
-        { .key        = "cluster.heal-timeout",
-          .voltype    = "cluster/replicate",
-          .option     = "!heal-timeout",
-          .op_version = 2,
-          .flags      = OPT_FLAG_CLIENT_OPT
-        },
-        { .key        = "cluster.strict-readdir",
-          .voltype    = "cluster/replicate",
-          .type       = NO_DOC,
-          .op_version = 1,
-          .flags      = OPT_FLAG_CLIENT_OPT
-        },
-        { .key        = "cluster.self-heal-window-size",
-          .voltype    = "cluster/replicate",
-          .option     = "data-self-heal-window-size",
-          .op_version = 1,
-          .flags      = OPT_FLAG_CLIENT_OPT
-        },
-        { .key        = "cluster.data-change-log",
-          .voltype    = "cluster/replicate",
-          .op_version = 1,
-          .flags      = OPT_FLAG_CLIENT_OPT
-        },
-        { .key        = "cluster.metadata-change-log",
-          .voltype    = "cluster/replicate",
-          .op_version = 1,
-          .flags      = OPT_FLAG_CLIENT_OPT
-        },
-        { .key        = "cluster.data-self-heal-algorithm",
-          .voltype    = "cluster/replicate",
-          .option     = "data-self-heal-algorithm",
-          .op_version = 1,
-          .flags      = OPT_FLAG_CLIENT_OPT
-        },
-        { .key        = "cluster.eager-lock",
-          .voltype    = "cluster/replicate",
-          .op_version = 1,
-          .flags      = OPT_FLAG_CLIENT_OPT
-        },
-        { .key        = "cluster.quorum-type",
-          .voltype    = "cluster/replicate",
-          .option     = "quorum-type",
-          .op_version = 1,
-          .flags      = OPT_FLAG_CLIENT_OPT
-        },
-        { .key        = "cluster.quorum-count",
-          .voltype    = "cluster/replicate",
-          .option     = "quorum-count",
-          .op_version = 1,
-          .flags      = OPT_FLAG_CLIENT_OPT
-        },
-        { .key        = "cluster.choose-local",
-          .voltype    = "cluster/replicate",
-          .op_version = 2,
-          .flags      = OPT_FLAG_CLIENT_OPT
-        },
-        { .key        = "cluster.self-heal-readdir-size",
-          .voltype    = "cluster/replicate",
-          .op_version = 2,
-          .flags      = OPT_FLAG_CLIENT_OPT
-        },
-        { .key        = "cluster.post-op-delay-secs",
-          .voltype    = "cluster/replicate",
-          .type       = NO_DOC,
-          .op_version = 2,
-          .flags      = OPT_FLAG_CLIENT_OPT
-        },
-        { .key        = "cluster.readdir-failover",
-          .voltype    = "cluster/replicate",
-          .op_version = 2,
-          .flags      = OPT_FLAG_CLIENT_OPT
-        },
-        { .key        = "cluster.ensure-durability",
-          .voltype    = "cluster/replicate",
-          .op_version = 3,
-          .flags      = OPT_FLAG_CLIENT_OPT
-        },
-        { .key        = "cluster.consistent-metadata",
-          .voltype    = "cluster/replicate",
-          .type       = DOC,
-          .op_version = GD_OP_VERSION_3_7_0,
-          .flags      = OPT_FLAG_CLIENT_OPT
-        },
-        { .key         = "cluster.stripe-block-size",
-          .voltype     = "cluster/stripe",
-          .option      = "block-size",
-          .op_version  = 1,
-          .validate_fn = validate_stripe,
-          .flags       = OPT_FLAG_CLIENT_OPT
-        },
-        { .key        = "cluster.stripe-coalesce",
-          .voltype    = "cluster/stripe",
-          .option     = "coalesce",
-          .op_version = 1,
-          .flags      = OPT_FLAG_CLIENT_OPT
-        },
-
-        /* IO-stats xlator options */
-        { .key         = VKEY_DIAG_LAT_MEASUREMENT,
-          .voltype     = "debug/io-stats",
-          .option      = "latency-measurement",
-          .value       = "off",
-          .op_version  = 1
-        },
-        { .key         = "diagnostics.dump-fd-stats",
-          .voltype     = "debug/io-stats",
-          .op_version  = 1
-        },
-        { .key         = VKEY_DIAG_CNT_FOP_HITS,
-          .voltype     = "debug/io-stats",
-          .option      = "count-fop-hits",
-          .value       = "off",
-          .type        = NO_DOC,
-          .op_version  = 1
-        },
-        { .key         = "diagnostics.brick-log-level",
-          .voltype     = "debug/io-stats",
-          .option      = "!brick-log-level",
-          .op_version  = 1
-        },
-        { .key        = "diagnostics.client-log-level",
-          .voltype    = "debug/io-stats",
-          .option     = "!client-log-level",
-          .op_version = 1,
-          .flags      = OPT_FLAG_CLIENT_OPT
-        },
-        { .key         = "diagnostics.brick-sys-log-level",
-          .voltype     = "debug/io-stats",
-          .option      = "!sys-log-level",
-          .op_version  = 1
-        },
-        { .key        = "diagnostics.client-sys-log-level",
-          .voltype    = "debug/io-stats",
-          .option     = "!sys-log-level",
-          .op_version = 1,
-          .flags      = OPT_FLAG_CLIENT_OPT
-        },
-        { .key         = "diagnostics.brick-logger",
-          .voltype     = "debug/io-stats",
-          .option      = "!logger",
-          .op_version  = GD_OP_VERSION_3_6_0,
-        },
-        { .key        = "diagnostics.client-logger",
-          .voltype    = "debug/io-stats",
-          .option     = "!logger",
-          .op_version = GD_OP_VERSION_3_6_0,
-          .flags      = OPT_FLAG_CLIENT_OPT
-        },
-        { .key         = "diagnostics.brick-log-format",
-          .voltype     = "debug/io-stats",
-          .option      = "!log-format",
-          .op_version  = GD_OP_VERSION_3_6_0,
-        },
-        { .key        = "diagnostics.client-log-format",
-          .voltype    = "debug/io-stats",
-          .option     = "!log-format",
-          .op_version = GD_OP_VERSION_3_6_0,
-          .flags      = OPT_FLAG_CLIENT_OPT
-        },
-        { .key         = "diagnostics.brick-log-buf-size",
-          .voltype     = "debug/io-stats",
-          .option      = "!log-buf-size",
-          .op_version  = GD_OP_VERSION_3_6_0,
-        },
-        { .key        = "diagnostics.client-log-buf-size",
-          .voltype    = "debug/io-stats",
-          .option     = "!log-buf-size",
-          .op_version = GD_OP_VERSION_3_6_0,
-          .flags      = OPT_FLAG_CLIENT_OPT
-        },
-        { .key         = "diagnostics.brick-log-flush-timeout",
-          .voltype     = "debug/io-stats",
-          .option      = "!log-flush-timeout",
-          .op_version  = GD_OP_VERSION_3_6_0,
-        },
-        { .key        = "diagnostics.client-log-flush-timeout",
-          .voltype    = "debug/io-stats",
-          .option     = "!log-flush-timeout",
-          .op_version = GD_OP_VERSION_3_6_0,
-          .flags      = OPT_FLAG_CLIENT_OPT
-        },
-
-        /* IO-cache xlator options */
-        { .key         = "performance.cache-max-file-size",
-          .voltype     = "performance/io-cache",
-          .option      = "max-file-size",
-          .op_version  = 1,
-          .validate_fn = validate_cache_max_min_size,
-          .flags       = OPT_FLAG_CLIENT_OPT
-        },
-        { .key         = "performance.cache-min-file-size",
-          .voltype     = "performance/io-cache",
-          .option      = "min-file-size",
-          .op_version  = 1,
-          .validate_fn = validate_cache_max_min_size,
-          .flags       = OPT_FLAG_CLIENT_OPT
-        },
-        { .key        = "performance.cache-refresh-timeout",
-          .voltype    = "performance/io-cache",
-          .option     = "cache-timeout",
-          .op_version = 1,
-          .flags      = OPT_FLAG_CLIENT_OPT
-        },
-        { .key        = "performance.cache-priority",
-          .voltype    = "performance/io-cache",
-          .option     = "priority",
-          .op_version = 1,
-          .flags      = OPT_FLAG_CLIENT_OPT
-        },
-        { .key        = "performance.cache-size",
-          .voltype    = "performance/io-cache",
-          .op_version = 1,
-          .flags      = OPT_FLAG_CLIENT_OPT
-        },
-
-        /* IO-threads xlator options */
-        { .key         = "performance.io-thread-count",
-          .voltype     = "performance/io-threads",
-          .option      = "thread-count",
-          .op_version  = 1
-        },
-        { .key         = "performance.high-prio-threads",
-          .voltype     = "performance/io-threads",
-          .op_version  = 1
-        },
-        { .key         = "performance.normal-prio-threads",
-          .voltype     = "performance/io-threads",
-          .op_version  = 1
-        },
-        { .key         = "performance.low-prio-threads",
-          .voltype     = "performance/io-threads",
-          .op_version  = 1
-        },
-        { .key         = "performance.least-prio-threads",
-          .voltype     = "performance/io-threads",
-          .op_version  = 1
-        },
-        { .key         = "performance.enable-least-priority",
-          .voltype     = "performance/io-threads",
-          .op_version  = 1
-        },
-        { .key         = "performance.least-rate-limit",
-          .voltype     = "performance/io-threads",
-          .op_version  = 2
-        },
-
-        /* Other perf xlators' options */
-        { .key        = "performance.cache-size",
-          .voltype    = "performance/quick-read",
-          .type       = NO_DOC,
-          .op_version = 1,
-          .flags      = OPT_FLAG_CLIENT_OPT
-        },
-        { .key        = "performance.flush-behind",
-          .voltype    = "performance/write-behind",
-          .option     = "flush-behind",
-          .op_version = 1,
-          .flags      = OPT_FLAG_CLIENT_OPT
-        },
-        { .key        = "performance.nfs.flush-behind",
-          .voltype    = "performance/write-behind",
-          .option     = "flush-behind",
-          .op_version = 1,
-          .flags      = OPT_FLAG_CLIENT_OPT
-        },
-        { .key        = "performance.write-behind-window-size",
-          .voltype    = "performance/write-behind",
-          .option     = "cache-size",
-          .op_version = 1,
-          .flags      = OPT_FLAG_CLIENT_OPT
-        },
-        { .key        = "performance.nfs.write-behind-window-size",
-          .voltype    = "performance/write-behind",
-          .option     = "cache-size",
-          .op_version = 1,
-          .flags      = OPT_FLAG_CLIENT_OPT
-        },
-        { .key        = "performance.strict-o-direct",
-          .voltype    = "performance/write-behind",
-          .option     = "strict-O_DIRECT",
-          .op_version = 2,
-          .flags      = OPT_FLAG_CLIENT_OPT
-        },
-        { .key        = "performance.nfs.strict-o-direct",
-          .voltype    = "performance/write-behind",
-          .option     = "strict-O_DIRECT",
-          .op_version = 2,
-          .flags      = OPT_FLAG_CLIENT_OPT
-        },
-        { .key        = "performance.strict-write-ordering",
-          .voltype    = "performance/write-behind",
-          .option     = "strict-write-ordering",
-          .op_version = 2,
-          .flags      = OPT_FLAG_CLIENT_OPT
-        },
-        { .key        = "performance.nfs.strict-write-ordering",
-          .voltype    = "performance/write-behind",
-          .option     = "strict-write-ordering",
-          .op_version = 2,
-          .flags      = OPT_FLAG_CLIENT_OPT
-        },
-        { .key        = "performance.lazy-open",
-          .voltype    = "performance/open-behind",
-          .option     = "lazy-open",
-          .op_version = 3,
-          .flags      = OPT_FLAG_CLIENT_OPT
-        },
-        { .key        = "performance.read-after-open",
-          .voltype    = "performance/open-behind",
-          .option     = "read-after-open",
-          .op_version = 3,
-          .flags      = OPT_FLAG_CLIENT_OPT
-        },
-        { .key        = "performance.read-ahead-page-count",
-          .voltype    = "performance/read-ahead",
-          .option     = "page-count",
-          .op_version = 1,
-          .flags      = OPT_FLAG_CLIENT_OPT
-        },
-        { .key        = "performance.md-cache-timeout",
-          .voltype    = "performance/md-cache",
-          .option     = "md-cache-timeout",
-          .op_version = 2,
-          .flags      = OPT_FLAG_CLIENT_OPT
-        },
-
- 	/* Crypt xlator options */
-
-	{ .key         = "features.encryption",
-	  .voltype     = "encryption/crypt",
-	  .option      = "!feat",
-	  .value       = "off",
-	  .op_version  = 3,
-	  .description = "enable/disable client-side encryption for "
-                         "the volume.",
-	  .flags       = OPT_FLAG_CLIENT_OPT | OPT_FLAG_XLATOR_OPT
-	},
-
-        { .key         = "encryption.master-key",
-          .voltype     = "encryption/crypt",
-          .op_version  = 3,
-          .flags      = OPT_FLAG_CLIENT_OPT
-        },
-        { .key         = "encryption.data-key-size",
-          .voltype     = "encryption/crypt",
-          .op_version  = 3,
-          .flags      = OPT_FLAG_CLIENT_OPT
-        },
-        { .key         = "encryption.block-size",
-          .voltype     = "encryption/crypt",
-          .op_version  = 3,
-          .flags      = OPT_FLAG_CLIENT_OPT
-        },
-
-        /* Client xlator options */
-        { .key        = "network.frame-timeout",
-          .voltype    = "protocol/client",
-          .op_version = 1,
-          .flags      = OPT_FLAG_CLIENT_OPT
-        },
-        { .key        = "network.ping-timeout",
-          .voltype    = "protocol/client",
-          .op_version = 1,
-          .flags      = OPT_FLAG_CLIENT_OPT
-        },
-        { .key        = "network.tcp-window-size",
-          .voltype    = "protocol/client",
-          .op_version = 1,
-          .flags      = OPT_FLAG_CLIENT_OPT
-        },
-        { .key        = "features.lock-heal",
-          .voltype    = "protocol/client",
-          .option     = "lk-heal",
-          .op_version = 1,
-          .flags      = OPT_FLAG_CLIENT_OPT
-        },
-        { .key        = "features.grace-timeout",
-          .voltype    = "protocol/client",
-          .option     = "grace-timeout",
-          .op_version = 1,
-          .flags      = OPT_FLAG_CLIENT_OPT
-        },
-        { .key        = "client.ssl",
-          .voltype    = "protocol/client",
-          .option     = "transport.socket.ssl-enabled",
-          .type       = NO_DOC,
-          .op_version = 2,
-          .flags      = OPT_FLAG_CLIENT_OPT
-        },
-        { .key        = "network.remote-dio",
-          .voltype    = "protocol/client",
-          .option     = "filter-O_DIRECT",
-          .op_version = 2,
-          .flags      = OPT_FLAG_CLIENT_OPT
-        },
-        { .key         = "client.own-thread",
-          .voltype     = "protocol/client",
-          .option      = "transport.socket.own-thread",
-          .type        = NO_DOC,
-          .op_version  = GD_OP_VERSION_3_7_0,
-        },
-        { .key         = "client.event-threads",
-          .voltype     = "protocol/client",
-          .op_version  = GD_OP_VERSION_3_7_0,
-        },
-
-        /* Server xlator options */
-        { .key         = "network.ping-timeout",
-          .voltype     = "protocol/server",
-          .option      = "transport.tcp-user-timeout",
-          .op_version  = GD_OP_VERSION_3_7_0,
-        },
-        { .key         = "network.tcp-window-size",
-          .voltype     = "protocol/server",
-          .type        = NO_DOC,
-          .op_version  = 1
-        },
-        { .key         = "network.inode-lru-limit",
-          .voltype     = "protocol/server",
-          .op_version  = 1
-        },
-        { .key         = AUTH_ALLOW_MAP_KEY,
-          .voltype     = "protocol/server",
-          .option      = "!server-auth",
-          .value       = "*",
-          .op_version  = 1
-        },
-        { .key         = AUTH_REJECT_MAP_KEY,
-          .voltype     = "protocol/server",
-          .option      = "!server-auth",
-          .op_version  = 1
-        },
-        { .key         = "transport.keepalive",
-          .voltype     = "protocol/server",
-          .option      = "transport.socket.keepalive",
-          .type        = NO_DOC,
-          .op_version  = 1
-        },
-        { .key         = "server.allow-insecure",
-          .voltype     = "protocol/server",
-          .option      = "rpc-auth-allow-insecure",
-          .type        = NO_DOC,
-          .op_version  = 1
-        },
-        { .key         = "server.root-squash",
-          .voltype     = "protocol/server",
-          .option      = "root-squash",
-          .op_version  = 2
-        },
-        { .key         = "server.anonuid",
-          .voltype     = "protocol/server",
-          .option      = "anonuid",
-          .op_version  = 3
-        },
-        { .key         = "server.anongid",
-          .voltype     = "protocol/server",
-          .option      = "anongid",
-          .op_version  = 3
-        },
-        { .key         = "server.statedump-path",
-          .voltype     = "protocol/server",
-          .option      = "statedump-path",
-          .op_version  = 1
-        },
-        { .key         = "server.outstanding-rpc-limit",
-          .voltype     = "protocol/server",
-          .option      = "rpc.outstanding-rpc-limit",
-          .type        = GLOBAL_DOC,
-          .op_version  = 3
-        },
-        { .key         = "features.lock-heal",
-          .voltype     = "protocol/server",
-          .option      = "lk-heal",
-          .type        = NO_DOC,
-          .op_version  = 1
-        },
-        { .key         = "features.grace-timeout",
-          .voltype     = "protocol/server",
-          .option      = "grace-timeout",
-          .type        = NO_DOC,
-          .op_version  = 1
-        },
-        { .key         = "server.ssl",
-          .voltype     = "protocol/server",
-          .option      = "transport.socket.ssl-enabled",
-          .type        = NO_DOC,
-          .op_version  = 2
-        },
-        { .key         = "auth.ssl-allow",
-          .voltype     = "protocol/server",
-          .option      = "!ssl-allow",
-          .value       = "*",
-          .type        = NO_DOC,
-          .op_version  = GD_OP_VERSION_3_6_0,
-        },
-        { .key         = "server.manage-gids",
-          .voltype     = "protocol/server",
-          .op_version  = GD_OP_VERSION_3_6_0,
-        },
-        { .key         = "client.send-gids",
-          .voltype     = "protocol/client",
-          .type        = NO_DOC,
-          .op_version  = GD_OP_VERSION_3_6_0,
-        },
-        { .key         = "server.gid-timeout",
-          .voltype     = "protocol/server",
-          .op_version  = GD_OP_VERSION_3_6_0,
-        },
-        { .key         = "server.own-thread",
-          .voltype     = "protocol/server",
-          .option      = "transport.socket.own-thread",
-          .type        = NO_DOC,
-          .op_version  = GD_OP_VERSION_3_7_0,
-        },
-        { .key         = "server.event-threads",
-          .voltype     = "protocol/server",
-          .op_version  = GD_OP_VERSION_3_7_0,
-        },
-
-        /* Generic transport options */
-        { .key         = SSL_CERT_DEPTH_OPT,
-          .voltype     = "rpc-transport/socket",
-          .option      = "!ssl-cert-depth",
-          .op_version  = GD_OP_VERSION_3_6_0,
-        },
-        { .key         = SSL_CIPHER_LIST_OPT,
-          .voltype     = "rpc-transport/socket",
-          .option      = "!ssl-cipher-list",
-          .op_version  = GD_OP_VERSION_3_6_0,
-        },
-
-        /* Performance xlators enable/disbable options */
-        { .key         = "performance.write-behind",
-          .voltype     = "performance/write-behind",
-          .option      = "!perf",
-          .value       = "on",
-          .op_version  = 1,
-          .description = "enable/disable write-behind translator in the "
-                         "volume.",
-          .flags       = OPT_FLAG_CLIENT_OPT | OPT_FLAG_XLATOR_OPT
-        },
-        { .key         = "performance.read-ahead",
-          .voltype     = "performance/read-ahead",
-          .option      = "!perf",
-          .value       = "on",
-          .op_version  = 1,
-          .description = "enable/disable read-ahead translator in the volume.",
-          .flags       = OPT_FLAG_CLIENT_OPT | OPT_FLAG_XLATOR_OPT
-        },
-        { .key         = "performance.readdir-ahead",
-          .voltype     = "performance/readdir-ahead",
-          .option      = "!perf",
-          .value       = "off",
-          .op_version  = 3,
-          .description = "enable/disable readdir-ahead translator in the volume.",
-          .flags       = OPT_FLAG_CLIENT_OPT | OPT_FLAG_XLATOR_OPT
-        },
-
-        { .key         = "performance.io-cache",
-          .voltype     = "performance/io-cache",
-          .option      = "!perf",
-          .value       = "on",
-          .op_version  = 1,
-          .description = "enable/disable io-cache translator in the volume.",
-          .flags       = OPT_FLAG_CLIENT_OPT
-        },
-        { .key         = "performance.quick-read",
-          .voltype     = "performance/quick-read",
-          .option      = "!perf",
-          .value       = "on",
-          .op_version  = 1,
-          .description = "enable/disable quick-read translator in the volume.",
-          .flags       = OPT_FLAG_CLIENT_OPT | OPT_FLAG_XLATOR_OPT
-
-        },
-        { .key         = "performance.open-behind",
-          .voltype     = "performance/open-behind",
-          .option      = "!perf",
-          .value       = "on",
-          .op_version  = 2,
-          .description = "enable/disable open-behind translator in the volume.",
-          .flags       = OPT_FLAG_CLIENT_OPT | OPT_FLAG_XLATOR_OPT
-
-        },
-        { .key         = "performance.stat-prefetch",
-          .voltype     = "performance/md-cache",
-          .option      = "!perf",
-          .value       = "on",
-          .op_version  = 1,
-          .description = "enable/disable meta-data caching translator in the "
-                         "volume.",
-          .flags       = OPT_FLAG_CLIENT_OPT | OPT_FLAG_XLATOR_OPT
-        },
-        { .key         = "performance.client-io-threads",
-          .voltype     = "performance/io-threads",
-          .option      = "!perf",
-          .value       = "off",
-          .op_version  = 1,
-          .description = "enable/disable io-threads translator in the client "
-                         "graph of volume.",
-          .flags       = OPT_FLAG_CLIENT_OPT | OPT_FLAG_XLATOR_OPT
-        },
-        { .key         = "performance.nfs.write-behind",
-          .voltype     = "performance/write-behind",
-          .option      = "!nfsperf",
-          .value       = "on",
-          .op_version  = 1,
-          .description = "enable/disable write-behind translator in the volume",
-          .flags       = OPT_FLAG_XLATOR_OPT
-        },
-        { .key        = "performance.nfs.read-ahead",
-          .voltype    = "performance/read-ahead",
-          .option     = "!nfsperf",
-          .value      = "off",
-          .type       = NO_DOC,
-          .op_version = 1,
-          .flags      = OPT_FLAG_XLATOR_OPT
-        },
-        { .key        = "performance.nfs.io-cache",
-          .voltype    = "performance/io-cache",
-          .option     = "!nfsperf",
-          .value      = "off",
-          .type       = NO_DOC,
-          .op_version = 1,
-          .flags      = OPT_FLAG_XLATOR_OPT
-        },
-        { .key        = "performance.nfs.quick-read",
-          .voltype    = "performance/quick-read",
-          .option     = "!nfsperf",
-          .value      = "off",
-          .type       = NO_DOC,
-          .op_version = 1,
-          .flags      = OPT_FLAG_XLATOR_OPT
-        },
-        { .key        = "performance.nfs.stat-prefetch",
-          .voltype    = "performance/md-cache",
-          .option     = "!nfsperf",
-          .value      = "off",
-          .type       = NO_DOC,
-          .op_version = 1,
-          .flags      = OPT_FLAG_XLATOR_OPT
-        },
-        { .key        = "performance.nfs.io-threads",
-          .voltype    = "performance/io-threads",
-          .option     = "!nfsperf",
-          .value      = "off",
-          .type       = NO_DOC,
-          .op_version = 1,
-          .flags      = OPT_FLAG_XLATOR_OPT
-        },
-        { .key        = "performance.force-readdirp",
-          .voltype    = "performance/md-cache",
-          .option     = "force-readdirp",
-          .op_version = 2,
-          .flags      = OPT_FLAG_CLIENT_OPT
-        },
-
-	/* Feature translators */
-        { .key         = "features.file-snapshot",
-          .voltype     = "features/qemu-block",
-          .option      = "!feat",
-          .value       = "off",
-          .op_version  = 3,
-          .description = "enable/disable file-snapshot feature in the "
-                         "volume.",
-          .flags       = OPT_FLAG_CLIENT_OPT | OPT_FLAG_XLATOR_OPT
-        },
-
-        { .key         = "features.uss",
-          .voltype     = "features/snapview-server",
-          .op_version  = GD_OP_VERSION_3_6_0,
-          .value       = "off",
-          .flags       = OPT_FLAG_CLIENT_OPT | OPT_FLAG_XLATOR_OPT,
-          .validate_fn = validate_uss,
-          .description = "enable/disable User Serviceable Snapshots on the "
-                         "volume."
-        },
-
-        { .key         = "features.snapshot-directory",
-          .voltype     = "features/snapview-client",
-          .op_version  = GD_OP_VERSION_3_6_0,
-          .value       = ".snaps",
-          .flags       = OPT_FLAG_CLIENT_OPT | OPT_FLAG_XLATOR_OPT,
-          .description = "Entry point directory for entering snapshot world"
-        },
-
-        { .key         = "features.show-snapshot-directory",
-          .voltype     = "features/snapview-client",
-          .op_version  = GD_OP_VERSION_3_6_0,
-          .value       = "off",
-          .flags       = OPT_FLAG_CLIENT_OPT | OPT_FLAG_XLATOR_OPT,
-          .description = "show entry point in readdir output of "
-                         "snapdir-entry-path which is set by samba"
-        },
+    /* DHT xlator options */
+    {.key = "cluster.lookup-unhashed",
+     .voltype = "cluster/distribute",
+     .op_version = 1,
+     .flags = VOLOPT_FLAG_CLIENT_OPT},
+    {.key = "cluster.lookup-optimize",
+     .voltype = "cluster/distribute",
+     .op_version = GD_OP_VERSION_3_7_2,
+     .flags = VOLOPT_FLAG_CLIENT_OPT},
+    {.key = "cluster.min-free-disk",
+     .voltype = "cluster/distribute",
+     .op_version = 1,
+     .flags = VOLOPT_FLAG_CLIENT_OPT},
+    {.key = "cluster.min-free-inodes",
+     .voltype = "cluster/distribute",
+     .op_version = 1,
+     .flags = VOLOPT_FLAG_CLIENT_OPT},
+    {.key = "cluster.rebalance-stats",
+     .voltype = "cluster/distribute",
+     .op_version = 2,
+     .flags = VOLOPT_FLAG_CLIENT_OPT},
+    {.key = "cluster.subvols-per-directory",
+     .voltype = "cluster/distribute",
+     .option = "directory-layout-spread",
+     .op_version = 2,
+     .validate_fn = validate_subvols_per_directory,
+     .flags = VOLOPT_FLAG_CLIENT_OPT},
+    {.key = "cluster.readdir-optimize",
+     .voltype = "cluster/distribute",
+     .op_version = 1,
+     .flags = VOLOPT_FLAG_CLIENT_OPT},
+    {.key = "cluster.rsync-hash-regex",
+     .voltype = "cluster/distribute",
+     .type = NO_DOC,
+     .op_version = 3,
+     .flags = VOLOPT_FLAG_CLIENT_OPT},
+    {.key = "cluster.extra-hash-regex",
+     .voltype = "cluster/distribute",
+     .type = NO_DOC,
+     .op_version = 3,
+     .flags = VOLOPT_FLAG_CLIENT_OPT},
+    {.key = "cluster.dht-xattr-name",
+     .voltype = "cluster/distribute",
+     .option = "xattr-name",
+     .type = NO_DOC,
+     .op_version = 3,
+     .flags = VOLOPT_FLAG_CLIENT_OPT},
+    {
+        .key = "cluster.randomize-hash-range-by-gfid",
+        .voltype = "cluster/distribute",
+        .option = "randomize-hash-range-by-gfid",
+        .type = NO_DOC,
+        .op_version = GD_OP_VERSION_3_6_0,
+        .flags = VOLOPT_FLAG_CLIENT_OPT,
+    },
+    {
+        .key = "cluster.rebal-throttle",
+        .voltype = "cluster/distribute",
+        .option = "rebal-throttle",
+        .op_version = GD_OP_VERSION_3_7_0,
+        .validate_fn = validate_defrag_throttle_option,
+        .flags = VOLOPT_FLAG_CLIENT_OPT,
+    },
+
+    {
+        .key = "cluster.lock-migration",
+        .voltype = "cluster/distribute",
+        .option = "lock-migration",
+        .value = "off",
+        .op_version = GD_OP_VERSION_3_8_0,
+        .validate_fn = validate_lock_migration_option,
+        .flags = VOLOPT_FLAG_CLIENT_OPT,
+    },
+
+    {
+        .key = "cluster.force-migration",
+        .voltype = "cluster/distribute",
+        .option = "force-migration",
+        .value = "off",
+        .op_version = GD_OP_VERSION_4_0_0,
+        .flags = VOLOPT_FLAG_CLIENT_OPT,
+    },
+
+    /* NUFA xlator options (Distribute special case) */
+    {.key = "cluster.nufa",
+     .voltype = "cluster/distribute",
+     .option = "!nufa",
+     .type = NO_DOC,
+     .op_version = 2,
+     .flags = VOLOPT_FLAG_CLIENT_OPT},
+    {.key = "cluster.local-volume-name",
+     .voltype = "cluster/nufa",
+     .option = "local-volume-name",
+     .type = NO_DOC,
+     .op_version = 3,
+     .flags = VOLOPT_FLAG_CLIENT_OPT},
+    {
+        .key = "cluster.weighted-rebalance",
+        .voltype = "cluster/distribute",
+        .op_version = GD_OP_VERSION_3_6_0,
+    },
+
+    /* Switch xlator options (Distribute special case) */
+    {.key = "cluster.switch",
+     .voltype = "cluster/distribute",
+     .option = "!switch",
+     .type = NO_DOC,
+     .op_version = 3,
+     .flags = VOLOPT_FLAG_CLIENT_OPT},
+    {.key = "cluster.switch-pattern",
+     .voltype = "cluster/switch",
+     .option = "pattern.switch.case",
+     .type = NO_DOC,
+     .op_version = 3,
+     .flags = VOLOPT_FLAG_CLIENT_OPT},
+
+    /* AFR xlator options */
+    {.key = "cluster.entry-change-log",
+     .voltype = "cluster/replicate",
+     .op_version = 1,
+     .flags = VOLOPT_FLAG_CLIENT_OPT},
+    {.key = "cluster.read-subvolume",
+     .voltype = "cluster/replicate",
+     .op_version = 1,
+     .flags = VOLOPT_FLAG_CLIENT_OPT},
+    {.key = "cluster.read-subvolume-index",
+     .voltype = "cluster/replicate",
+     .op_version = 2,
+     .flags = VOLOPT_FLAG_CLIENT_OPT},
+    {.key = "cluster.read-hash-mode",
+     .voltype = "cluster/replicate",
+     .op_version = 2,
+     .flags = VOLOPT_FLAG_CLIENT_OPT},
+    {.key = "cluster.background-self-heal-count",
+     .voltype = "cluster/replicate",
+     .op_version = 1,
+     .flags = VOLOPT_FLAG_CLIENT_OPT},
+    {.key = "cluster.metadata-self-heal",
+     .voltype = "cluster/replicate",
+     .op_version = 1,
+     .validate_fn = validate_replica,
+     .flags = VOLOPT_FLAG_CLIENT_OPT},
+    {.key = "cluster.data-self-heal",
+     .voltype = "cluster/replicate",
+     .op_version = 1,
+     .validate_fn = validate_replica,
+     .flags = VOLOPT_FLAG_CLIENT_OPT},
+    {.key = "cluster.entry-self-heal",
+     .voltype = "cluster/replicate",
+     .op_version = 1,
+     .validate_fn = validate_replica,
+     .flags = VOLOPT_FLAG_CLIENT_OPT},
+    {.key = "cluster.self-heal-daemon",
+     .voltype = "cluster/replicate",
+     .option = "!self-heal-daemon",
+     .op_version = 1,
+     .validate_fn = validate_replica_heal_enable_disable},
+    {.key = "cluster.heal-timeout",
+     .voltype = "cluster/replicate",
+     .option = "!heal-timeout",
+     .op_version = 2,
+     .flags = VOLOPT_FLAG_CLIENT_OPT},
+    {.key = "cluster.strict-readdir",
+     .voltype = "cluster/replicate",
+     .type = NO_DOC,
+     .op_version = 1,
+     .flags = VOLOPT_FLAG_CLIENT_OPT},
+    {.key = "cluster.self-heal-window-size",
+     .voltype = "cluster/replicate",
+     .option = "data-self-heal-window-size",
+     .op_version = 1,
+     .flags = VOLOPT_FLAG_CLIENT_OPT},
+    {.key = "cluster.data-change-log",
+     .voltype = "cluster/replicate",
+     .op_version = 1,
+     .flags = VOLOPT_FLAG_CLIENT_OPT},
+    {.key = "cluster.metadata-change-log",
+     .voltype = "cluster/replicate",
+     .op_version = 1,
+     .flags = VOLOPT_FLAG_CLIENT_OPT},
+    {.key = "cluster.data-self-heal-algorithm",
+     .voltype = "cluster/replicate",
+     .option = "data-self-heal-algorithm",
+     .op_version = 1,
+     .flags = VOLOPT_FLAG_CLIENT_OPT},
+    {.key = "cluster.eager-lock",
+     .voltype = "cluster/replicate",
+     .op_version = 1,
+     .flags = VOLOPT_FLAG_CLIENT_OPT},
+    {.key = "disperse.eager-lock",
+     .voltype = "cluster/disperse",
+     .op_version = GD_OP_VERSION_3_7_10,
+     .flags = VOLOPT_FLAG_CLIENT_OPT},
+    {.key = "disperse.other-eager-lock",
+     .voltype = "cluster/disperse",
+     .op_version = GD_OP_VERSION_3_13_0,
+     .flags = VOLOPT_FLAG_CLIENT_OPT},
+    {.key = "disperse.eager-lock-timeout",
+     .voltype = "cluster/disperse",
+     .op_version = GD_OP_VERSION_4_0_0,
+     .flags = VOLOPT_FLAG_CLIENT_OPT},
+    {.key = "disperse.other-eager-lock-timeout",
+     .voltype = "cluster/disperse",
+     .op_version = GD_OP_VERSION_4_0_0,
+     .flags = VOLOPT_FLAG_CLIENT_OPT},
+    {.key = "cluster.quorum-type",
+     .voltype = "cluster/replicate",
+     .option = "quorum-type",
+     .op_version = 1,
+     .flags = VOLOPT_FLAG_CLIENT_OPT},
+    {.key = "cluster.quorum-count",
+     .voltype = "cluster/replicate",
+     .option = "quorum-count",
+     .op_version = 1,
+     .validate_fn = validate_quorum_count,
+     .flags = VOLOPT_FLAG_CLIENT_OPT},
+    {.key = "cluster.choose-local",
+     .voltype = "cluster/replicate",
+     .op_version = 2,
+     .flags = VOLOPT_FLAG_CLIENT_OPT},
+    {.key = "cluster.self-heal-readdir-size",
+     .voltype = "cluster/replicate",
+     .op_version = 2,
+     .flags = VOLOPT_FLAG_CLIENT_OPT},
+    {.key = "cluster.post-op-delay-secs",
+     .voltype = "cluster/replicate",
+     .type = NO_DOC,
+     .op_version = 2,
+     .flags = VOLOPT_FLAG_CLIENT_OPT},
+    {.key = "cluster.readdir-failover",
+     .voltype = "cluster/replicate",
+     .op_version = 2,
+     .flags = VOLOPT_FLAG_CLIENT_OPT},
+    {.key = "cluster.ensure-durability",
+     .voltype = "cluster/replicate",
+     .op_version = 3,
+     .flags = VOLOPT_FLAG_CLIENT_OPT},
+    {.key = "cluster.consistent-metadata",
+     .voltype = "cluster/replicate",
+     .type = DOC,
+     .op_version = GD_OP_VERSION_3_7_0,
+     .flags = VOLOPT_FLAG_CLIENT_OPT},
+    {.key = "cluster.heal-wait-queue-length",
+     .voltype = "cluster/replicate",
+     .type = DOC,
+     .op_version = GD_OP_VERSION_3_7_10,
+     .flags = VOLOPT_FLAG_CLIENT_OPT},
+    {.key = "cluster.favorite-child-policy",
+     .voltype = "cluster/replicate",
+     .type = DOC,
+     .op_version = GD_OP_VERSION_3_7_12,
+     .flags = VOLOPT_FLAG_CLIENT_OPT},
+    {.key = "cluster.full-lock",
+     .voltype = "cluster/replicate",
+     .type = NO_DOC,
+     .op_version = GD_OP_VERSION_3_13_2,
+     .flags = VOLOPT_FLAG_CLIENT_OPT},
+    {.key = "cluster.optimistic-change-log",
+     .voltype = "cluster/replicate",
+     .type = NO_DOC,
+     .op_version = GD_OP_VERSION_7_2,
+     .flags = VOLOPT_FLAG_CLIENT_OPT},
+
+    /* IO-stats xlator options */
+    {.key = VKEY_DIAG_LAT_MEASUREMENT,
+     .voltype = "debug/io-stats",
+     .option = "latency-measurement",
+     .value = "off",
+     .op_version = 1},
+    {.key = "diagnostics.dump-fd-stats",
+     .voltype = "debug/io-stats",
+     .op_version = 1},
+    {.key = VKEY_DIAG_CNT_FOP_HITS,
+     .voltype = "debug/io-stats",
+     .option = "count-fop-hits",
+     .value = "off",
+     .type = NO_DOC,
+     .op_version = 1},
+    {.key = "diagnostics.brick-log-level",
+     .voltype = "debug/io-stats",
+     .value = "INFO",
+     .option = "!brick-log-level",
+     .op_version = 1},
+    {.key = "diagnostics.client-log-level",
+     .voltype = "debug/io-stats",
+     .value = "INFO",
+     .option = "!client-log-level",
+     .op_version = 1,
+     .flags = VOLOPT_FLAG_CLIENT_OPT},
+    {.key = "diagnostics.brick-sys-log-level",
+     .voltype = "debug/io-stats",
+     .option = "!sys-log-level",
+     .op_version = 1},
+    {.key = "diagnostics.client-sys-log-level",
+     .voltype = "debug/io-stats",
+     .option = "!sys-log-level",
+     .op_version = 1,
+     .flags = VOLOPT_FLAG_CLIENT_OPT},
+    {
+        .key = "diagnostics.brick-logger",
+        .voltype = "debug/io-stats",
+        .option = "!logger",
+        .op_version = GD_OP_VERSION_3_6_0,
+    },
+    {.key = "diagnostics.client-logger",
+     .voltype = "debug/io-stats",
+     .option = "!logger",
+     .op_version = GD_OP_VERSION_3_6_0,
+     .flags = VOLOPT_FLAG_CLIENT_OPT},
+    {
+        .key = "diagnostics.brick-log-format",
+        .voltype = "debug/io-stats",
+        .option = "!log-format",
+        .op_version = GD_OP_VERSION_3_6_0,
+    },
+    {.key = "diagnostics.client-log-format",
+     .voltype = "debug/io-stats",
+     .option = "!log-format",
+     .op_version = GD_OP_VERSION_3_6_0,
+     .flags = VOLOPT_FLAG_CLIENT_OPT},
+    {
+        .key = "diagnostics.brick-log-buf-size",
+        .voltype = "debug/io-stats",
+        .option = "!log-buf-size",
+        .op_version = GD_OP_VERSION_3_6_0,
+    },
+    {.key = "diagnostics.client-log-buf-size",
+     .voltype = "debug/io-stats",
+     .option = "!log-buf-size",
+     .op_version = GD_OP_VERSION_3_6_0,
+     .flags = VOLOPT_FLAG_CLIENT_OPT},
+    {
+        .key = "diagnostics.brick-log-flush-timeout",
+        .voltype = "debug/io-stats",
+        .option = "!log-flush-timeout",
+        .op_version = GD_OP_VERSION_3_6_0,
+    },
+    {.key = "diagnostics.client-log-flush-timeout",
+     .voltype = "debug/io-stats",
+     .option = "!log-flush-timeout",
+     .op_version = GD_OP_VERSION_3_6_0,
+     .flags = VOLOPT_FLAG_CLIENT_OPT},
+    {.key = "diagnostics.stats-dump-interval",
+     .voltype = "debug/io-stats",
+     .option = "ios-dump-interval",
+     .op_version = 1},
+    {.key = "diagnostics.fop-sample-interval",
+     .voltype = "debug/io-stats",
+     .option = "ios-sample-interval",
+     .op_version = 1},
+    {
+        .key = "diagnostics.stats-dump-format",
+        .voltype = "debug/io-stats",
+        .option = "ios-dump-format",
+        .op_version = GD_OP_VERSION_3_12_0,
+    },
+    {.key = "diagnostics.fop-sample-buf-size",
+     .voltype = "debug/io-stats",
+     .option = "ios-sample-buf-size",
+     .op_version = 1},
+    {.key = "diagnostics.stats-dnscache-ttl-sec",
+     .voltype = "debug/io-stats",
+     .option = "ios-dnscache-ttl-sec",
+     .op_version = 1},
+
+    /* IO-cache xlator options */
+    {.key = "performance.cache-max-file-size",
+     .voltype = "performance/io-cache",
+     .option = "max-file-size",
+     .op_version = 1,
+     .validate_fn = validate_cache_max_min_size,
+     .flags = VOLOPT_FLAG_CLIENT_OPT},
+    {.key = "performance.cache-min-file-size",
+     .voltype = "performance/io-cache",
+     .option = "min-file-size",
+     .op_version = 1,
+     .validate_fn = validate_cache_max_min_size,
+     .flags = VOLOPT_FLAG_CLIENT_OPT},
+    {.key = "performance.cache-refresh-timeout",
+     .voltype = "performance/io-cache",
+     .option = "cache-timeout",
+     .op_version = 1,
+     .flags = VOLOPT_FLAG_CLIENT_OPT},
+    {.key = "performance.cache-priority",
+     .voltype = "performance/io-cache",
+     .option = "priority",
+     .op_version = 1,
+     .flags = VOLOPT_FLAG_CLIENT_OPT},
+    {.key = "performance.io-cache-size",
+     .voltype = "performance/io-cache",
+     .option = "cache-size",
+     .op_version = GD_OP_VERSION_8_0,
+     .flags = VOLOPT_FLAG_CLIENT_OPT},
+    {
+        .key = "performance.cache-size",
+        .voltype = "performance/io-cache",
+        .op_version = 1,
+        .flags = VOLOPT_FLAG_CLIENT_OPT,
+        .description = "Deprecated option. Use performance.io-cache-size "
+                       "to adjust the cache size of the io-cache translator, "
+                       "and use performance.quick-read-cache-size to adjust "
+                       "the cache size of the quick-read translator.",
+    },
+
+    /* IO-threads xlator options */
+    {.key = "performance.io-thread-count",
+     .voltype = "performance/io-threads",
+     .option = "thread-count",
+     .op_version = 1},
+    {.key = "performance.high-prio-threads",
+     .voltype = "performance/io-threads",
+     .op_version = 1},
+    {.key = "performance.normal-prio-threads",
+     .voltype = "performance/io-threads",
+     .op_version = 1},
+    {.key = "performance.low-prio-threads",
+     .voltype = "performance/io-threads",
+     .op_version = 1},
+    {.key = "performance.least-prio-threads",
+     .voltype = "performance/io-threads",
+     .op_version = 1},
+    {.key = "performance.enable-least-priority",
+     .voltype = "performance/io-threads",
+     .op_version = 1},
+    {.key = "performance.iot-watchdog-secs",
+     .voltype = "performance/io-threads",
+     .option = "watchdog-secs",
+     .op_version = GD_OP_VERSION_4_1_0},
+    {.key = "performance.iot-cleanup-disconnected-reqs",
+     .voltype = "performance/io-threads",
+     .option = "cleanup-disconnected-reqs",
+     .op_version = GD_OP_VERSION_4_1_0},
+    {.key = "performance.iot-pass-through",
+     .voltype = "performance/io-threads",
+     .option = "pass-through",
+     .op_version = GD_OP_VERSION_4_1_0},
+
+    /* Other perf xlators' options */
+    {.key = "performance.io-cache-pass-through",
+     .voltype = "performance/io-cache",
+     .option = "pass-through",
+     .op_version = GD_OP_VERSION_4_1_0},
+    {.key = "performance.quick-read-cache-size",
+     .voltype = "performance/quick-read",
+     .option = "cache-size",
+     .op_version = GD_OP_VERSION_8_0,
+     .flags = VOLOPT_FLAG_CLIENT_OPT},
+    {.key = "performance.cache-size",
+     .voltype = "performance/quick-read",
+     .type = NO_DOC,
+     .op_version = 1,
+     .flags = VOLOPT_FLAG_CLIENT_OPT},
+    {.key = "performance.quick-read-cache-timeout",
+     .voltype = "performance/quick-read",
+     .option = "cache-timeout",
+     .op_version = GD_OP_VERSION_8_0,
+     .flags = VOLOPT_FLAG_CLIENT_OPT},
+    {.key = "performance.qr-cache-timeout",
+     .voltype = "performance/quick-read",
+     .option = "cache-timeout",
+     .op_version = 1,
+     .flags = VOLOPT_FLAG_CLIENT_OPT,
+     .description =
+         "Deprecated option. Use performance.quick-read-cache-timeout "
+         "instead."},
+    {.key = "performance.quick-read-cache-invalidation",
+     .voltype = "performance/quick-read",
+     .option = "quick-read-cache-invalidation",
+     .op_version = GD_OP_VERSION_4_0_0,
+     .flags = VOLOPT_FLAG_CLIENT_OPT},
+    {.key = "performance.ctime-invalidation",
+     .voltype = "performance/quick-read",
+     .option = "ctime-invalidation",
+     .op_version = GD_OP_VERSION_5_0,
+     .flags = VOLOPT_FLAG_CLIENT_OPT},
+    {.key = "performance.flush-behind",
+     .voltype = "performance/write-behind",
+     .option = "flush-behind",
+     .op_version = 1,
+     .flags = VOLOPT_FLAG_CLIENT_OPT},
+    {.key = "performance.nfs.flush-behind",
+     .voltype = "performance/write-behind",
+     .option = "flush-behind",
+     .op_version = 1,
+     .flags = VOLOPT_FLAG_CLIENT_OPT},
+    {.key = "performance.write-behind-window-size",
+     .voltype = "performance/write-behind",
+     .option = "cache-size",
+     .op_version = 1,
+     .flags = VOLOPT_FLAG_CLIENT_OPT},
+    {
+        .key = "performance.resync-failed-syncs-after-fsync",
+        .voltype = "performance/write-behind",
+        .option = "resync-failed-syncs-after-fsync",
+        .op_version = GD_OP_VERSION_3_7_7,
+        .flags = VOLOPT_FLAG_CLIENT_OPT,
+        .description = "If sync of \"cached-writes issued before fsync\" "
+                       "(to backend) fails, this option configures whether "
+                       "to retry syncing them after fsync or forget them. "
+                       "If set to on, cached-writes are retried "
+                       "till a \"flush\" fop (or a successful sync) on sync "
+                       "failures. "
+                       "fsync itself is failed irrespective of the value of "
+                       "this option. ",
+    },
+    {.key = "performance.nfs.write-behind-window-size",
+     .voltype = "performance/write-behind",
+     .option = "cache-size",
+     .op_version = 1,
+     .flags = VOLOPT_FLAG_CLIENT_OPT},
+    {.key = "performance.strict-o-direct",
+     .voltype = "performance/write-behind",
+     .option = "strict-O_DIRECT",
+     .op_version = 2,
+     .flags = VOLOPT_FLAG_CLIENT_OPT},
+    {.key = "performance.nfs.strict-o-direct",
+     .voltype = "performance/write-behind",
+     .option = "strict-O_DIRECT",
+     .op_version = 2,
+     .flags = VOLOPT_FLAG_CLIENT_OPT},
+    {.key = "performance.strict-write-ordering",
+     .voltype = "performance/write-behind",
+     .option = "strict-write-ordering",
+     .op_version = 2,
+     .flags = VOLOPT_FLAG_CLIENT_OPT},
+    {.key = "performance.nfs.strict-write-ordering",
+     .voltype = "performance/write-behind",
+     .option = "strict-write-ordering",
+     .op_version = 2,
+     .flags = VOLOPT_FLAG_CLIENT_OPT},
+    {.key = "performance.write-behind-trickling-writes",
+     .voltype = "performance/write-behind",
+     .option = "trickling-writes",
+     .op_version = GD_OP_VERSION_3_13_1,
+     .flags = VOLOPT_FLAG_CLIENT_OPT},
+    {.key = "performance.aggregate-size",
+     .voltype = "performance/write-behind",
+     .option = "aggregate-size",
+     .op_version = GD_OP_VERSION_4_1_0,
+     .flags = OPT_FLAG_CLIENT_OPT},
+    {.key = "performance.nfs.write-behind-trickling-writes",
+     .voltype = "performance/write-behind",
+     .option = "trickling-writes",
+     .op_version = GD_OP_VERSION_3_13_1,
+     .flags = VOLOPT_FLAG_CLIENT_OPT},
+    {.key = "performance.lazy-open",
+     .voltype = "performance/open-behind",
+     .option = "lazy-open",
+     .op_version = 3,
+     .flags = VOLOPT_FLAG_CLIENT_OPT},
+    {.key = "performance.read-after-open",
+     .voltype = "performance/open-behind",
+     .option = "read-after-open",
+     .op_version = 3,
+     .flags = VOLOPT_FLAG_CLIENT_OPT},
+    {
+        .key = "performance.open-behind-pass-through",
+        .voltype = "performance/open-behind",
+        .option = "pass-through",
+        .op_version = GD_OP_VERSION_4_1_0,
+    },
+    {.key = "performance.read-ahead-page-count",
+     .voltype = "performance/read-ahead",
+     .option = "page-count",
+     .op_version = 1,
+     .flags = VOLOPT_FLAG_CLIENT_OPT},
+    {
+        .key = "performance.read-ahead-pass-through",
+        .voltype = "performance/read-ahead",
+        .option = "pass-through",
+        .op_version = GD_OP_VERSION_4_1_0,
+    },
+    {
+        .key = "performance.readdir-ahead-pass-through",
+        .voltype = "performance/readdir-ahead",
+        .option = "pass-through",
+        .op_version = GD_OP_VERSION_4_1_0,
+    },
+    {.key = "performance.md-cache-pass-through",
+     .voltype = "performance/md-cache",
+     .option = "pass-through",
+     .op_version = GD_OP_VERSION_4_1_0},
+    {.key = "performance.md-cache-timeout",
+     .voltype = "performance/md-cache",
+     .option = "md-cache-timeout",
+     .op_version = 2,
+     .flags = VOLOPT_FLAG_CLIENT_OPT},
+    {.key = "performance.cache-swift-metadata",
+     .voltype = "performance/md-cache",
+     .option = "cache-swift-metadata",
+     .op_version = GD_OP_VERSION_3_7_10,
+     .description = "Cache swift metadata (user.swift.metadata xattr)",
+     .flags = VOLOPT_FLAG_CLIENT_OPT},
+    {.key = "performance.cache-samba-metadata",
+     .voltype = "performance/md-cache",
+     .option = "cache-samba-metadata",
+     .op_version = GD_OP_VERSION_3_9_0,
+     .description = "Cache samba metadata (user.DOSATTRIB, security.NTACL"
+                    " xattr)",
+     .flags = VOLOPT_FLAG_CLIENT_OPT},
+    {.key = "performance.cache-capability-xattrs",
+     .voltype = "performance/md-cache",
+     .option = "cache-capability-xattrs",
+     .op_version = GD_OP_VERSION_3_10_0,
+     .description = "Cache xattrs required for capability based security",
+     .flags = VOLOPT_FLAG_CLIENT_OPT},
+    {.key = "performance.cache-ima-xattrs",
+     .voltype = "performance/md-cache",
+     .option = "cache-ima-xattrs",
+     .op_version = GD_OP_VERSION_3_10_0,
+     .description = "Cache xattrs required for IMA "
+                    "(Integrity Measurement Architecture)",
+     .flags = VOLOPT_FLAG_CLIENT_OPT},
+    {.key = "performance.md-cache-statfs",
+     .voltype = "performance/md-cache",
+     .option = "md-cache-statfs",
+     .op_version = GD_OP_VERSION_4_0_0,
+     .flags = VOLOPT_FLAG_CLIENT_OPT},
+    {.key = "performance.xattr-cache-list",
+     .voltype = "performance/md-cache",
+     .option = "xattr-cache-list",
+     .op_version = GD_OP_VERSION_4_0_0,
+     .flags = VOLOPT_FLAG_CLIENT_OPT,
+     .description = "A comma separated list of xattrs that shall be "
+                    "cached by md-cache. The only wildcard allowed is '*'"},
+    {.key = "performance.nl-cache-pass-through",
+     .voltype = "performance/nl-cache",
+     .option = "pass-through",
+     .op_version = GD_OP_VERSION_4_1_0},
+
+    /* Client xlator options */
+    {.key = "network.frame-timeout",
+     .voltype = "protocol/client",
+     .op_version = 1,
+     .flags = VOLOPT_FLAG_CLIENT_OPT},
+    {.key = "network.ping-timeout",
+     .voltype = "protocol/client",
+     .op_version = 1,
+     .flags = VOLOPT_FLAG_CLIENT_OPT},
+    {.key = "network.tcp-window-size",
+     .voltype = "protocol/client",
+     .op_version = 1,
+     .flags = VOLOPT_FLAG_CLIENT_OPT},
+    {.key = "client.ssl",
+     .voltype = "protocol/client",
+     .option = "transport.socket.ssl-enabled",
+     .value = "off",
+     .op_version = 2,
+     .description = "enable/disable client.ssl flag in the "
+                    "volume.",
+     .flags = VOLOPT_FLAG_CLIENT_OPT},
+    {.key = "network.remote-dio",
+     .voltype = "protocol/client",
+     .option = "filter-O_DIRECT",
+     .op_version = 2,
+     .flags = VOLOPT_FLAG_CLIENT_OPT},
+    {
+        .key = "client.own-thread",
+        .voltype = "protocol/client",
+        .option = "transport.socket.own-thread",
+        .type = NO_DOC,
+        .op_version = GD_OP_VERSION_3_7_0,
+    },
+    {
+        .key = "client.event-threads",
+        .voltype = "protocol/client",
+        .op_version = GD_OP_VERSION_3_7_0,
+    },
+    {.key = "client.tcp-user-timeout",
+     .voltype = "protocol/client",
+     .option = "transport.tcp-user-timeout",
+     .op_version = GD_OP_VERSION_3_10_2,
+     .value = "0", /* 0 - implies "use system default" */
+     .flags = VOLOPT_FLAG_CLIENT_OPT},
+    {.key = "client.keepalive-time",
+     .voltype = "protocol/client",
+     .option = "transport.socket.keepalive-time",
+     .op_version = GD_OP_VERSION_3_10_2,
+     .value = "20",
+     .flags = VOLOPT_FLAG_CLIENT_OPT},
+    {.key = "client.keepalive-interval",
+     .voltype = "protocol/client",
+     .option = "transport.socket.keepalive-interval",
+     .op_version = GD_OP_VERSION_3_10_2,
+     .value = "2",
+     .flags = VOLOPT_FLAG_CLIENT_OPT},
+    {.key = "client.keepalive-count",
+     .voltype = "protocol/client",
+     .option = "transport.socket.keepalive-count",
+     .op_version = GD_OP_VERSION_3_10_2,
+     .value = "9",
+     .flags = VOLOPT_FLAG_CLIENT_OPT},
+    {.key = "client.strict-locks",
+     .voltype = "protocol/client",
+     .option = "strict-locks",
+     .value = "off",
+     .op_version = GD_OP_VERSION_8_0,
+     .validate_fn = validate_boolean,
+     .type = GLOBAL_DOC,
+     .description = "When set, doesn't reopen saved fds after reconnect "
+                    "if POSIX locks are held on them. Hence subsequent "
+                    "operations on these fds will fail. This is "
+                    "necessary for stricter lock complaince as bricks "
+                    "cleanup any granted locks when a client "
+                    "disconnects."},
+
+    /* Although the following option is named ta-remote-port but it will be
+     * added as remote-port in client volfile for ta-bricks only.
+     */
+    {.key = "client.ta-brick-port",
+     .voltype = "protocol/client",
+     .option = "ta-remote-port",
+     .op_version = GD_OP_VERSION_7_0},
+
+    /* Server xlator options */
+    {.key = "network.tcp-window-size",
+     .voltype = "protocol/server",
+     .type = NO_DOC,
+     .op_version = 1},
+    {.key = "network.inode-lru-limit",
+     .voltype = "protocol/server",
+     .op_version = 1},
+    {.key = AUTH_ALLOW_MAP_KEY,
+     .voltype = "protocol/server",
+     .option = "!server-auth",
+     .value = "*",
+     .op_version = 1},
+    {.key = AUTH_REJECT_MAP_KEY,
+     .voltype = "protocol/server",
+     .option = "!server-auth",
+     .op_version = 1},
+    {.key = "transport.keepalive",
+     .voltype = "protocol/server",
+     .option = "transport.socket.keepalive",
+     .type = NO_DOC,
+     .value = "1",
+     .op_version = 1},
+    {.key = "server.allow-insecure",
+     .voltype = "protocol/server",
+     .option = "rpc-auth-allow-insecure",
+     .type = DOC,
+     .op_version = 1},
+    {.key = "server.root-squash",
+     .voltype = "protocol/server",
+     .option = "root-squash",
+     .op_version = 2},
+    {.key = "server.all-squash",
+     .voltype = "protocol/server",
+     .option = "all-squash",
+     .op_version = GD_OP_VERSION_6_0},
+    {.key = "server.anonuid",
+     .voltype = "protocol/server",
+     .option = "anonuid",
+     .op_version = 3},
+    {.key = "server.anongid",
+     .voltype = "protocol/server",
+     .option = "anongid",
+     .op_version = 3},
+    {.key = "server.statedump-path",
+     .voltype = "protocol/server",
+     .option = "statedump-path",
+     .op_version = 1,
+     .validate_fn = validate_statedump_path},
+    {.key = "server.outstanding-rpc-limit",
+     .voltype = "protocol/server",
+     .option = "rpc.outstanding-rpc-limit",
+     .type = GLOBAL_DOC,
+     .op_version = 3},
+    {.key = "server.ssl",
+     .voltype = "protocol/server",
+     .value = "off",
+     .option = "transport.socket.ssl-enabled",
+     .description = "enable/disable server.ssl flag in the "
+                    "volume.",
+     .op_version = 2},
+    {
+        .key = "auth.ssl-allow",
+        .voltype = "protocol/server",
+        .option = "!ssl-allow",
+        .value = "*",
+        .type = DOC,
+        .description = "Allow a comma separated list of common names (CN) of "
+                       "the clients that are allowed to access the server."
+                       "By default, all TLS authenticated clients are "
+                       "allowed to access the server.",
+        .op_version = GD_OP_VERSION_3_6_0,
+    },
+    {
+        .key = "server.manage-gids",
+        .voltype = "protocol/server",
+        .op_version = GD_OP_VERSION_3_6_0,
+    },
+    {
+        .key = "server.dynamic-auth",
+        .voltype = "protocol/server",
+        .op_version = GD_OP_VERSION_3_7_5,
+    },
+    {
+        .key = "client.send-gids",
+        .voltype = "protocol/client",
+        .type = NO_DOC,
+        .op_version = GD_OP_VERSION_3_6_0,
+    },
+    {
+        .key = "server.gid-timeout",
+        .voltype = "protocol/server",
+        .op_version = GD_OP_VERSION_3_6_0,
+    },
+    {
+        .key = "server.own-thread",
+        .voltype = "protocol/server",
+        .option = "transport.socket.own-thread",
+        .type = NO_DOC,
+        .op_version = GD_OP_VERSION_3_7_0,
+    },
+    {
+        .key = "server.event-threads",
+        .voltype = "protocol/server",
+        .op_version = GD_OP_VERSION_3_7_0,
+    },
+    {
+        .key = "server.tcp-user-timeout",
+        .voltype = "protocol/server",
+        .option = "transport.tcp-user-timeout",
+        .op_version = GD_OP_VERSION_3_10_2,
+    },
+    {
+        .key = "server.keepalive-time",
+        .voltype = "protocol/server",
+        .option = "transport.socket.keepalive-time",
+        .op_version = GD_OP_VERSION_3_10_2,
+        .value = "20",
+    },
+    {
+        .key = "server.keepalive-interval",
+        .voltype = "protocol/server",
+        .option = "transport.socket.keepalive-interval",
+        .op_version = GD_OP_VERSION_3_10_2,
+        .value = "2",
+    },
+    {
+        .key = "server.keepalive-count",
+        .voltype = "protocol/server",
+        .option = "transport.socket.keepalive-count",
+        .op_version = GD_OP_VERSION_3_10_2,
+        .value = "9",
+    },
+    {
+        .key = "transport.listen-backlog",
+        .voltype = "protocol/server",
+        .option = "transport.listen-backlog",
+        .op_version = GD_OP_VERSION_3_11_1,
+        .validate_fn = validate_server_options,
+        .description = "This option uses the value of backlog argument that "
+                       "defines the maximum length to which the queue of "
+                       "pending connections for socket fd may grow.",
+        .value = "1024",
+    },
+
+    /* Generic transport options */
+    {
+        .key = SSL_OWN_CERT_OPT,
+        .voltype = "rpc-transport/socket",
+        .option = "!ssl-own-cert",
+        .op_version = GD_OP_VERSION_3_7_4,
+    },
+    {
+        .key = SSL_PRIVATE_KEY_OPT,
+        .voltype = "rpc-transport/socket",
+        .option = "!ssl-private-key",
+        .op_version = GD_OP_VERSION_3_7_4,
+    },
+    {
+        .key = SSL_CA_LIST_OPT,
+        .voltype = "rpc-transport/socket",
+        .option = "!ssl-ca-list",
+        .op_version = GD_OP_VERSION_3_7_4,
+    },
+    {
+        .key = SSL_CRL_PATH_OPT,
+        .voltype = "rpc-transport/socket",
+        .option = "!ssl-crl-path",
+        .op_version = GD_OP_VERSION_3_7_4,
+    },
+    {
+        .key = SSL_CERT_DEPTH_OPT,
+        .voltype = "rpc-transport/socket",
+        .option = "!ssl-cert-depth",
+        .op_version = GD_OP_VERSION_3_6_0,
+    },
+    {
+        .key = SSL_CIPHER_LIST_OPT,
+        .voltype = "rpc-transport/socket",
+        .option = "!ssl-cipher-list",
+        .op_version = GD_OP_VERSION_3_6_0,
+    },
+    {
+        .key = SSL_DH_PARAM_OPT,
+        .voltype = "rpc-transport/socket",
+        .option = "!ssl-dh-param",
+        .op_version = GD_OP_VERSION_3_7_4,
+    },
+    {
+        .key = SSL_EC_CURVE_OPT,
+        .voltype = "rpc-transport/socket",
+        .option = "!ssl-ec-curve",
+        .op_version = GD_OP_VERSION_3_7_4,
+    },
+    {
+        .key = "transport.address-family",
+        .voltype = "protocol/server",
+        .option = "!address-family",
+        .op_version = GD_OP_VERSION_3_7_4,
+        .type = NO_DOC,
+    },
+
+    /* Performance xlators enable/disbable options */
+    {.key = "performance.write-behind",
+     .voltype = "performance/write-behind",
+     .option = "!perf",
+     .value = "on",
+     .op_version = 1,
+     .description = "enable/disable write-behind translator in the "
+                    "volume.",
+     .flags = VOLOPT_FLAG_CLIENT_OPT | VOLOPT_FLAG_XLATOR_OPT},
+    {.key = "performance.read-ahead",
+     .voltype = "performance/read-ahead",
+     .option = "!perf",
+     .value = "off",
+     .op_version = 1,
+     .description = "enable/disable read-ahead translator in the volume.",
+     .flags = VOLOPT_FLAG_CLIENT_OPT | VOLOPT_FLAG_XLATOR_OPT},
+    {.key = "performance.readdir-ahead",
+     .voltype = "performance/readdir-ahead",
+     .option = "!perf",
+     .value = "off",
+     .op_version = 3,
+     .description = "enable/disable readdir-ahead translator in the volume.",
+     .flags = VOLOPT_FLAG_CLIENT_OPT | VOLOPT_FLAG_XLATOR_OPT},
+    {.key = "performance.io-cache",
+     .voltype = "performance/io-cache",
+     .option = "!perf",
+     .value = "off",
+     .op_version = 1,
+     .description = "enable/disable io-cache translator in the volume.",
+     .flags = VOLOPT_FLAG_CLIENT_OPT},
+    {.key = "performance.open-behind",
+     .voltype = "performance/open-behind",
+     .option = "!perf",
+     .value = "on",
+     .op_version = 2,
+     .description = "enable/disable open-behind translator in the volume.",
+     .flags = VOLOPT_FLAG_CLIENT_OPT | VOLOPT_FLAG_XLATOR_OPT
+
+    },
+    {.key = "performance.quick-read",
+     .voltype = "performance/quick-read",
+     .option = "!perf",
+     .value = "on",
+     .op_version = 1,
+     .description = "enable/disable quick-read translator in the volume.",
+     .flags = VOLOPT_FLAG_CLIENT_OPT | VOLOPT_FLAG_XLATOR_OPT},
+    {.key = "performance.nl-cache",
+     .voltype = "performance/nl-cache",
+     .option = "!perf",
+     .value = "off",
+     .op_version = GD_OP_VERSION_3_11_0,
+     .description = "enable/disable negative entry caching translator in "
+                    "the volume. Enabling this option improves performance"
+                    " of 'create file/directory' workload",
+     .flags = VOLOPT_FLAG_CLIENT_OPT | VOLOPT_FLAG_XLATOR_OPT},
+    {.key = "performance.stat-prefetch",
+     .voltype = "performance/md-cache",
+     .option = "!perf",
+     .value = "on",
+     .op_version = 1,
+     .description = "enable/disable meta-data caching translator in the "
+                    "volume.",
+     .flags = VOLOPT_FLAG_CLIENT_OPT | VOLOPT_FLAG_XLATOR_OPT},
+    {.key = "performance.client-io-threads",
+     .voltype = "performance/io-threads",
+     .option = "!perf",
+     .value = "on",
+     .op_version = 1,
+     .description = "enable/disable io-threads translator in the client "
+                    "graph of volume.",
+     .flags = VOLOPT_FLAG_CLIENT_OPT | VOLOPT_FLAG_XLATOR_OPT},
+    {.key = "performance.nfs.write-behind",
+     .voltype = "performance/write-behind",
+     .option = "!nfsperf",
+     .value = "on",
+     .op_version = 1,
+     .description = "enable/disable write-behind translator in the volume",
+     .flags = VOLOPT_FLAG_XLATOR_OPT},
+    {.key = "performance.nfs.read-ahead",
+     .voltype = "performance/read-ahead",
+     .option = "!nfsperf",
+     .value = "off",
+     .type = NO_DOC,
+     .op_version = 1,
+     .flags = VOLOPT_FLAG_XLATOR_OPT},
+    {.key = "performance.nfs.io-cache",
+     .voltype = "performance/io-cache",
+     .option = "!nfsperf",
+     .value = "off",
+     .type = NO_DOC,
+     .op_version = 1,
+     .flags = VOLOPT_FLAG_XLATOR_OPT},
+    {.key = "performance.nfs.quick-read",
+     .voltype = "performance/quick-read",
+     .option = "!nfsperf",
+     .value = "off",
+     .type = NO_DOC,
+     .op_version = 1,
+     .flags = VOLOPT_FLAG_XLATOR_OPT},
+    {.key = "performance.nfs.stat-prefetch",
+     .voltype = "performance/md-cache",
+     .option = "!nfsperf",
+     .value = "off",
+     .type = NO_DOC,
+     .op_version = 1,
+     .flags = VOLOPT_FLAG_XLATOR_OPT},
+    {.key = "performance.nfs.io-threads",
+     .voltype = "performance/io-threads",
+     .option = "!nfsperf",
+     .value = "off",
+     .type = NO_DOC,
+     .op_version = 1,
+     .flags = VOLOPT_FLAG_XLATOR_OPT},
+    {.key = "performance.force-readdirp",
+     .voltype = "performance/md-cache",
+     .option = "force-readdirp",
+     .op_version = 2,
+     .flags = VOLOPT_FLAG_CLIENT_OPT},
+    {.key = "performance.cache-invalidation",
+     .voltype = "performance/md-cache",
+     .option = "cache-invalidation",
+     .op_version = GD_OP_VERSION_3_9_0,
+     .flags = VOLOPT_FLAG_CLIENT_OPT},
+
+    {.key = "performance.global-cache-invalidation",
+     .voltype = "performance/md-cache",
+     .option = "global-cache-invalidation",
+     .op_version = GD_OP_VERSION_6_0,
+     .flags = VOLOPT_FLAG_CLIENT_OPT},
+
+    /* Feature translators */
+    {.key = "features.uss",
+     .voltype = "features/snapview-server",
+     .op_version = GD_OP_VERSION_3_6_0,
+     .value = "off",
+     .flags = VOLOPT_FLAG_CLIENT_OPT | VOLOPT_FLAG_XLATOR_OPT,
+     .validate_fn = validate_uss,
+     .description = "enable/disable User Serviceable Snapshots on the "
+                    "volume."},
+
+    {.key = "features.snapshot-directory",
+     .voltype = "features/snapview-client",
+     .op_version = GD_OP_VERSION_3_6_0,
+     .value = ".snaps",
+     .flags = VOLOPT_FLAG_CLIENT_OPT | VOLOPT_FLAG_XLATOR_OPT,
+     .validate_fn = validate_uss_dir,
+     .description = "Entry point directory for entering snapshot world. "
+                    "Value can have only [0-9a-z-_] and starts with "
+                    "dot (.) and cannot exceed 255 character"},
+
+    {.key = "features.show-snapshot-directory",
+     .voltype = "features/snapview-client",
+     .op_version = GD_OP_VERSION_3_6_0,
+     .value = "off",
+     .flags = VOLOPT_FLAG_CLIENT_OPT | VOLOPT_FLAG_XLATOR_OPT,
+     .description = "show entry point in readdir output of "
+                    "snapdir-entry-path which is set by samba"},
+
+    {.key = "features.tag-namespaces",
+     .voltype = "features/namespace",
+     .op_version = GD_OP_VERSION_4_1_0,
+     .option = "tag-namespaces",
+     .value = "off",
+     .flags = OPT_FLAG_CLIENT_OPT,
+     .description = "This option enables this translator's functionality "
+                    "that tags every fop with a namespace hash for later "
+                    "throttling, stats collection, logging, etc."},
 
 #ifdef HAVE_LIB_Z
-        /* Compressor-decompressor xlator options
-         * defaults used from xlator/features/compress/src/cdc.h
-         */
-        { .key         = "network.compression",
-          .voltype     = "features/cdc",
-          .option      = "!feat",
-          .value       = "off",
-          .op_version  = 3,
-          .description = "enable/disable network compression translator",
-          .flags       = OPT_FLAG_XLATOR_OPT
-        },
-        { .key         = "network.compression.window-size",
-          .voltype     = "features/cdc",
-          .option      = "window-size",
-          .op_version  = 3
-        },
-        { .key         = "network.compression.mem-level",
-          .voltype     = "features/cdc",
-          .option      = "mem-level",
-          .op_version  = 3
-        },
-        { .key         = "network.compression.min-size",
-          .voltype     = "features/cdc",
-          .option      = "min-size",
-          .op_version  = 3
-        },
-        { .key         = "network.compression.compression-level",
-          .voltype     = "features/cdc",
-          .option      = "compression-level",
-          .op_version  = 3
-        },
-        { .key         = "network.compression.debug",
-          .voltype     = "features/cdc",
-          .option      = "debug",
-          .type        = NO_DOC,
-          .op_version  = 3
-        },
+    /* Compressor-decompressor xlator options
+     * defaults used from xlator/features/compress/src/cdc.h
+     */
+    {.key = "network.compression",
+     .voltype = "features/cdc",
+     .option = "!feat",
+     .value = "off",
+     .op_version = 3,
+     .description = "enable/disable network compression translator",
+     .flags = VOLOPT_FLAG_XLATOR_OPT},
+    {.key = "network.compression.window-size",
+     .voltype = "features/cdc",
+     .option = "window-size",
+     .op_version = 3},
+    {.key = "network.compression.mem-level",
+     .voltype = "features/cdc",
+     .option = "mem-level",
+     .op_version = 3},
+    {.key = "network.compression.min-size",
+     .voltype = "features/cdc",
+     .option = "min-size",
+     .op_version = 3},
+    {.key = "network.compression.compression-level",
+     .voltype = "features/cdc",
+     .option = "compression-level",
+     .op_version = 3},
+    {.key = "network.compression.debug",
+     .voltype = "features/cdc",
+     .option = "debug",
+     .type = NO_DOC,
+     .op_version = 3},
 #endif
 
-        /* Quota xlator options */
-        { .key           = VKEY_FEATURES_LIMIT_USAGE,
-          .voltype       = "features/quota",
-          .option        = "limit-set",
-          .type          = NO_DOC,
-          .op_version    = 1,
-        },
-        {
-          .key           = "features.quota-timeout",
-          .voltype       = "features/quota",
-          .option        = "timeout",
-          .value         = "0",
-          .op_version    = 1,
-          .validate_fn   = validate_quota,
-        },
-        { .key           = "features.default-soft-limit",
-          .voltype       = "features/quota",
-          .option        = "default-soft-limit",
-          .type          = NO_DOC,
-          .op_version    = 3,
-        },
-        { .key           = "features.soft-timeout",
-          .voltype       = "features/quota",
-          .option        = "soft-timeout",
-          .type          = NO_DOC,
-          .op_version    = 3,
-        },
-        { .key           = "features.hard-timeout",
-          .voltype       = "features/quota",
-          .option        = "hard-timeout",
-          .type          = NO_DOC,
-          .op_version    = 3,
-        },
-        { .key           = "features.alert-time",
-          .voltype       = "features/quota",
-          .option        = "alert-time",
-          .type          = NO_DOC,
-          .op_version    = 3,
-        },
-        { .key           = "features.quota-deem-statfs",
-          .voltype       = "features/quota",
-          .option        = "deem-statfs",
-          .value         = "off",
-          .type          = DOC,
-          .op_version    = 2,
-          .validate_fn   = validate_quota,
-        },
-
-        /* Marker xlator options */
-        { .key         = VKEY_MARKER_XTIME,
-          .voltype     = "features/marker",
-          .option      = "xtime",
-          .value       = "off",
-          .type        = NO_DOC,
-          .flags       = OPT_FLAG_FORCE,
-          .op_version  = 1
-        },
-        { .key         = VKEY_MARKER_XTIME,
-          .voltype     = "features/marker",
-          .option      = "!xtime",
-          .value       = "off",
-          .type        = NO_DOC,
-          .flags       = OPT_FLAG_FORCE,
-          .op_version  = 1
-        },
-        { .key         = VKEY_MARKER_XTIME_FORCE,
-          .voltype     = "features/marker",
-          .option      = "gsync-force-xtime",
-          .value       = "off",
-          .type        = NO_DOC,
-          .flags       = OPT_FLAG_FORCE,
-          .op_version  = 2
-        },
-        { .key         = VKEY_MARKER_XTIME_FORCE,
-          .voltype     = "features/marker",
-          .option      = "!gsync-force-xtime",
-          .value       = "off",
-          .type        = NO_DOC,
-          .flags       = OPT_FLAG_FORCE,
-          .op_version  = 2
-        },
-        { .key         = VKEY_FEATURES_QUOTA,
-          .voltype     = "features/marker",
-          .option      = "quota",
-          .value       = "off",
-          .type        = NO_DOC,
-          .flags       = OPT_FLAG_FORCE,
-          .op_version  = 1
-        },
-
-        { .key         = VKEY_FEATURES_BITROT,
-          .voltype     = "features/bitrot",
-          .option      = "bitrot",
-          .value       = "off",
-          .type        = NO_DOC,
-          .flags       = OPT_FLAG_FORCE,
-          .op_version  = GD_OP_VERSION_3_7_0
-        },
-
-        /* Debug xlators options */
-        { .key        = "debug.trace",
-          .voltype    = "debug/trace",
-          .option     = "!debug",
-          .value      = "off",
-          .type       = NO_DOC,
-          .op_version = 1,
-          .flags      = OPT_FLAG_XLATOR_OPT
-        },
-        { .key         = "debug.log-history",
-          .voltype     = "debug/trace",
-          .option      = "log-history",
-          .type        = NO_DOC,
-          .op_version  = 2
-        },
-        { .key         = "debug.log-file",
-          .voltype     = "debug/trace",
-          .option      = "log-file",
-          .type        = NO_DOC,
-          .op_version  = 2
-        },
-        { .key         = "debug.exclude-ops",
-          .voltype     = "debug/trace",
-          .option      = "exclude-ops",
-          .type        = NO_DOC,
-          .op_version  = 2
-        },
-        { .key         = "debug.include-ops",
-          .voltype     = "debug/trace",
-          .option      = "include-ops",
-          .type        = NO_DOC,
-          .op_version  = 2
-        },
-        { .key        = "debug.error-gen",
-          .voltype    = "debug/error-gen",
-          .option     = "!debug",
-          .value      = "off",
-          .type       = NO_DOC,
-          .op_version = 1,
-          .flags      = OPT_FLAG_XLATOR_OPT
-        },
-        { .key         = "debug.error-failure",
-          .voltype     = "debug/error-gen",
-          .option      = "failure",
-          .type        = NO_DOC,
-          .op_version  = 3
-        },
-        { .key         = "debug.error-number",
-          .voltype     = "debug/error-gen",
-          .option      = "error-no",
-          .type        = NO_DOC,
-          .op_version  = 3
-        },
-        { .key         = "debug.random-failure",
-          .voltype     = "debug/error-gen",
-          .option      = "random-failure",
-          .type        = NO_DOC,
-          .op_version  = 3
-        },
-        { .key         = "debug.error-fops",
-          .voltype     = "debug/error-gen",
-          .option      = "enable",
-          .type        = NO_DOC,
-          .op_version  = 3
-        },
-
-
-        /* NFS xlator options */
-        { .key         = "nfs.enable-ino32",
-          .voltype     = "nfs/server",
-          .option      = "nfs.enable-ino32",
-          .type        = GLOBAL_DOC,
-          .op_version  = 1
-        },
-        { .key         = "nfs.mem-factor",
-          .voltype     = "nfs/server",
-          .option      = "nfs.mem-factor",
-          .type        = GLOBAL_DOC,
-          .op_version  = 1
-        },
-        { .key         = "nfs.export-dirs",
-          .voltype     = "nfs/server",
-          .option      = "nfs3.export-dirs",
-          .type        = GLOBAL_DOC,
-          .op_version  = 1
-        },
-        { .key         = "nfs.export-volumes",
-          .voltype     = "nfs/server",
-          .option      = "nfs3.export-volumes",
-          .type        = GLOBAL_DOC,
-          .op_version  = 1
-        },
-        { .key         = "nfs.addr-namelookup",
-          .voltype     = "nfs/server",
-          .option      = "rpc-auth.addr.namelookup",
-          .type        = GLOBAL_DOC,
-          .op_version  = 1
-        },
-        { .key         = "nfs.dynamic-volumes",
-          .voltype     = "nfs/server",
-          .option      = "nfs.dynamic-volumes",
-          .type        = GLOBAL_DOC,
-          .op_version  = 1
-        },
-        { .key         = "nfs.register-with-portmap",
-          .voltype     = "nfs/server",
-          .option      = "rpc.register-with-portmap",
-          .type        = GLOBAL_DOC,
-          .op_version  = 1
-        },
-        { .key         = "nfs.outstanding-rpc-limit",
-          .voltype     = "nfs/server",
-          .option      = "rpc.outstanding-rpc-limit",
-          .type        = GLOBAL_DOC,
-          .op_version  = 3
-        },
-        { .key         = "nfs.port",
-          .voltype     = "nfs/server",
-          .option      = "nfs.port",
-          .type        = GLOBAL_DOC,
-          .op_version  = 1
-        },
-        { .key         = "nfs.rpc-auth-unix",
-          .voltype     = "nfs/server",
-          .option      = "!rpc-auth.auth-unix.*",
-          .op_version  = 1
-        },
-        { .key         = "nfs.rpc-auth-null",
-          .voltype     = "nfs/server",
-          .option      = "!rpc-auth.auth-null.*",
-          .op_version  = 1
-        },
-        { .key         = "nfs.rpc-auth-allow",
-          .voltype     = "nfs/server",
-          .option      = "!rpc-auth.addr.*.allow",
-          .op_version  = 1
-        },
-        { .key         = "nfs.rpc-auth-reject",
-          .voltype     = "nfs/server",
-          .option      = "!rpc-auth.addr.*.reject",
-          .op_version  = 1
-        },
-        { .key         = "nfs.ports-insecure",
-          .voltype     = "nfs/server",
-          .option      = "!rpc-auth.ports.*.insecure",
-          .op_version  = 1
-        },
-        { .key         = "nfs.transport-type",
-          .voltype     = "nfs/server",
-          .option      = "!nfs.transport-type",
-          .op_version  = 1,
-          .description = "Specifies the nfs transport type. Valid "
-                         "transport types are 'tcp' and 'rdma'."
-        },
-        { .key         = "nfs.trusted-sync",
-          .voltype     = "nfs/server",
-          .option      = "!nfs3.*.trusted-sync",
-          .op_version  = 1
-        },
-        { .key         = "nfs.trusted-write",
-          .voltype     = "nfs/server",
-          .option      = "!nfs3.*.trusted-write",
-          .op_version  = 1
-        },
-        { .key         = "nfs.volume-access",
-          .voltype     = "nfs/server",
-          .option      = "!nfs3.*.volume-access",
-          .op_version  = 1
-        },
-        { .key         = "nfs.export-dir",
-          .voltype     = "nfs/server",
-          .option      = "!nfs3.*.export-dir",
-          .op_version  = 1
-        },
-        { .key         = NFS_DISABLE_MAP_KEY,
-          .voltype     = "nfs/server",
-          .option      = "!nfs-disable",
-          .op_version  = 1
-        },
-        { .key         = "nfs.nlm",
-          .voltype     = "nfs/server",
-          .option      = "nfs.nlm",
-          .type        = GLOBAL_DOC,
-          .op_version  = 1
-        },
-        { .key         = "nfs.acl",
-          .voltype     = "nfs/server",
-          .option      = "nfs.acl",
-          .type        = GLOBAL_DOC,
-          .op_version  = 3
-        },
-        { .key         = "nfs.mount-udp",
-          .voltype     = "nfs/server",
-          .option      = "nfs.mount-udp",
-          .type        = GLOBAL_DOC,
-          .op_version  = 1
-        },
-        { .key         = "nfs.mount-rmtab",
-          .voltype     = "nfs/server",
-          .option      = "nfs.mount-rmtab",
-          .type        = GLOBAL_DOC,
-          .op_version  = 1
-        },
-        { .key         = "nfs.rpc-statd",
-          .voltype     = "nfs/server",
-          .option      = "nfs.rpc-statd",
-          .type        = NO_DOC,
-          .op_version  = GD_OP_VERSION_3_6_0,
-        },
-        { .key         = "nfs.log-level",
-          .voltype     = "nfs/server",
-          .option      = "nfs.log-level",
-          .type        = NO_DOC,
-          .op_version  = GD_OP_VERSION_3_6_0,
-        },
-        { .key         = "nfs.server-aux-gids",
-          .voltype     = "nfs/server",
-          .option      = "nfs.server-aux-gids",
-          .type        = NO_DOC,
-          .op_version  = 2
-        },
-        { .key         = "nfs.drc",
-          .voltype     = "nfs/server",
-          .option      = "nfs.drc",
-          .type        = GLOBAL_DOC,
-          .op_version  = 3
-        },
-        { .key         = "nfs.drc-size",
-          .voltype     = "nfs/server",
-          .option      = "nfs.drc-size",
-          .type        = GLOBAL_DOC,
-          .op_version  = 3
-        },
-        { .key         = "nfs.read-size",
-          .voltype     = "nfs/server",
-          .option      = "nfs3.read-size",
-          .type        = GLOBAL_DOC,
-          .op_version  = 3
-        },
-        { .key         = "nfs.write-size",
-          .voltype     = "nfs/server",
-          .option      = "nfs3.write-size",
-          .type        = GLOBAL_DOC,
-          .op_version  = 3
-        },
-        { .key         = "nfs.readdir-size",
-          .voltype     = "nfs/server",
-          .option      = "nfs3.readdir-size",
-          .type        = GLOBAL_DOC,
-          .op_version  = 3
-        },
-
-        /* Cli options for Export authentication on nfs mount */
-        { .key         = "nfs.exports-auth-enable",
-          .voltype     = "nfs/server",
-          .option      = "nfs.exports-auth-enable",
-          .type        = GLOBAL_DOC,
-          .op_version  = GD_OP_VERSION_3_7_0
-        },
-        { .key         = "nfs.auth-refresh-interval-sec",
-          .voltype     = "nfs/server",
-          .option      = "nfs.auth-refresh-interval-sec",
-          .type        = GLOBAL_DOC,
-          .op_version  = GD_OP_VERSION_3_7_0
-        },
-        { .key         = "nfs.auth-cache-ttl-sec",
-          .voltype     = "nfs/server",
-          .option      = "nfs.auth-cache-ttl-sec",
-          .type        = GLOBAL_DOC,
-          .op_version  = GD_OP_VERSION_3_7_0
-        },
-
-        /* Other options which don't fit any place above */
-        { .key        = "features.read-only",
-          .voltype    = "features/read-only",
-          .option     = "read-only",
-          .op_version = 1,
-          .flags      = OPT_FLAG_CLIENT_OPT | OPT_FLAG_XLATOR_OPT
-        },
-        { .key        = "features.worm",
-          .voltype    = "features/worm",
-          .option     = "worm",
-          .value      = "off",
-          .op_version = 2,
-          .flags      = OPT_FLAG_CLIENT_OPT | OPT_FLAG_XLATOR_OPT
-        },
-        { .key         = "storage.linux-aio",
-          .voltype     = "storage/posix",
-          .op_version  = 1
-        },
-        { .key         = "storage.batch-fsync-mode",
-          .voltype     = "storage/posix",
-          .op_version  = 3
-        },
-        { .key         = "storage.batch-fsync-delay-usec",
-          .voltype     = "storage/posix",
-          .op_version  = 3
-        },
-        { .key         = "storage.xattr-user-namespace-mode",
-          .voltype     = "storage/posix",
-          .op_version  = GD_OP_VERSION_3_6_0,
-        },
-        { .key         = "storage.owner-uid",
-          .voltype     = "storage/posix",
-          .option      = "brick-uid",
-          .op_version  = 1
-        },
-        { .key         = "storage.owner-gid",
-          .voltype     = "storage/posix",
-          .option      = "brick-gid",
-          .op_version  = 1
-        },
-        { .key         = "storage.node-uuid-pathinfo",
-          .voltype     = "storage/posix",
-          .op_version  = 3
-        },
-        { .key         = "storage.health-check-interval",
-          .voltype     = "storage/posix",
-          .op_version  = 3
-        },
-        { .option      = "update-link-count-parent",
-          .key         = "storage.build-pgfid",
-          .voltype     = "storage/posix",
-          .op_version  = GD_OP_VERSION_3_6_0,
-        },
-        { .key         = "storage.bd-aio",
-          .voltype     = "storage/bd",
-          .op_version  = 3
-        },
-        { .key        = "config.memory-accounting",
-          .voltype    = "configuration",
-          .option     = "!config",
-          .op_version = 2,
-          .flags      = OPT_FLAG_CLIENT_OPT
-        },
-        { .key         = "config.transport",
-          .voltype     = "configuration",
-          .option      = "!config",
-          .op_version  = 2
-        },
-        { .key         = GLUSTERD_QUORUM_TYPE_KEY,
-          .voltype     = "mgmt/glusterd",
-          .value       = "off",
-          .op_version  = 2
-        },
-        { .key         = GLUSTERD_QUORUM_RATIO_KEY,
-          .voltype     = "mgmt/glusterd",
-          .value       = "0",
-          .op_version  = 2
-        },
-        /* changelog translator - global tunables */
-        { .key         = "changelog.changelog",
-          .voltype     = "features/changelog",
-          .type        = NO_DOC,
-          .op_version  = 3
-        },
-        { .key         = "changelog.changelog-dir",
-          .voltype     = "features/changelog",
-          .type        = NO_DOC,
-          .op_version  = 3
-        },
-        { .key         = "changelog.encoding",
-          .voltype     = "features/changelog",
-          .type        = NO_DOC,
-          .op_version  = 3
-        },
-        { .key         = "changelog.rollover-time",
-          .voltype     = "features/changelog",
-          .type        = NO_DOC,
-          .op_version  = 3
-        },
-        { .key         = "changelog.fsync-interval",
-          .voltype     = "features/changelog",
-          .type        = NO_DOC,
-          .op_version  = 3
-        },
-        { .key         = "changelog.changelog-barrier-timeout",
-          .voltype     = "features/changelog",
-          .value       = BARRIER_TIMEOUT,
-          .op_version  = GD_OP_VERSION_3_6_0,
-        },
-        { .key         = "features.barrier",
-          .voltype     = "features/barrier",
-          .value       = "disable",
-          .type        = NO_DOC,
-          .op_version  = GD_OP_VERSION_3_6_0,
-        },
-        { .key         = "features.barrier-timeout",
-          .voltype     = "features/barrier",
-          .value       = BARRIER_TIMEOUT,
-          .op_version  = GD_OP_VERSION_3_6_0,
-        },
-        { .key         = "cluster.op-version",
-          .voltype     = "mgmt/glusterd",
-          .op_version  = GD_OP_VERSION_3_6_0,
-        },
-        /*Trash translator options */
-        { .key         = "features.trash",
-          .voltype     = "features/trash",
-          .op_version  = GD_OP_VERSION_3_7_0,
-        },
-        { .key         = "features.trash-dir",
-          .voltype     = "features/trash",
-          .op_version  = GD_OP_VERSION_3_7_0,
-        },
-        { .key         = "features.trash-eliminate-path",
-          .voltype     = "features/trash",
-          .op_version  = GD_OP_VERSION_3_7_0,
-        },
-        { .key         = "features.trash-max-filesize",
-          .voltype     = "features/trash",
-          .op_version  = GD_OP_VERSION_3_7_0,
-        },
-        { .key         = "features.trash-internal-op",
-          .voltype     = "features/trash",
-          .op_version  = GD_OP_VERSION_3_7_0,
-        },
-
-        /* tier translator - global tunables */
-        { .key         = "cluster.write-freq-threshold",
-          .voltype     = "cluster/tier",
-          .option      = "write-freq-threshold",
-          .op_version  = GD_OP_VERSION_3_7_0,
-          .flags       = OPT_FLAG_CLIENT_OPT
-        },
-        { .key         = "cluster.read-freq-threshold",
-          .voltype     = "cluster/tier",
-          .option      = "read-freq-threshold",
-          .op_version  = GD_OP_VERSION_3_7_0,
-          .flags       = OPT_FLAG_CLIENT_OPT
-        },
-        { .key         = "cluster.tier-promote-frequency",
-          .voltype     = "cluster/tier",
-          .option      = "tier-promote-frequency",
-          .op_version  = GD_OP_VERSION_3_7_0,
-          .flags       = OPT_FLAG_CLIENT_OPT
-        },
-        { .key         = "cluster.tier-demote-frequency",
-          .voltype     = "cluster/tier",
-          .option      = "tier-demote-frequency",
-          .op_version  = GD_OP_VERSION_3_7_0,
-          .flags       = OPT_FLAG_CLIENT_OPT
-        },
-        { .key         = "features.ctr-enabled",
-          .voltype     = "features/changetimerecorder",
-          .value       = "off",
-          .option      = "ctr-enabled",
-          .op_version  = GD_OP_VERSION_3_7_0
-        },
-        { .key         = "features.record-counters",
-          .voltype     = "features/changetimerecorder",
-          .value       = "off",
-          .option      = "record-counters",
-          .op_version  = GD_OP_VERSION_3_7_0
-        },
-        { .key         = "locks.trace",
-          .voltype     = "features/locks",
-          .type        = NO_DOC,
-          .op_version  = GD_OP_VERSION_3_7_0,
-        },
-        { .key           = "cluster.disperse-self-heal-daemon",
-          .voltype       = "cluster/disperse",
-          .type          = NO_DOC,
-          .option        = "self-heal-daemon",
-          .op_version    = GD_OP_VERSION_3_7_0,
-          .validate_fn   = validate_disperse_heal_enable_disable
-        },
-        { .key           = "cluster.quorum-reads",
-          .voltype       = "cluster/replicate",
-          .op_version    = GD_OP_VERSION_3_7_0,
-          .flags      = OPT_FLAG_CLIENT_OPT
-        },
-        { .key        = "client.bind-insecure",
-          .voltype    = "protocol/client",
-          .option     = "client-bind-insecure",
-          .type       = NO_DOC,
-          .op_version = GD_OP_VERSION_3_7_0,
-          .flags      = OPT_FLAG_CLIENT_OPT
-        },
-        { .key         = "features.ganesha",
-          .voltype     = "features/ganesha",
-          .option      = "!ganesha",
-          .type         = GLOBAL_NO_DOC,
-          .op_version   = GD_OP_VERSION_3_7_0,
-        },
-        { .key         = "ganesha.enable",
-          .voltype     = "features/ganesha",
-          .type        = NO_DOC,
-          .op_version  = GD_OP_VERSION_3_7_0,
-        },
-        { .key        = "features.shard",
-          .voltype    = "features/shard",
-          .value      = "off",
-          .option     = "!shard",
-          .op_version = GD_OP_VERSION_3_7_0,
-          .flags      = OPT_FLAG_CLIENT_OPT
-        },
-        { .key        = "features.shard-block-size",
-          .voltype    = "features/shard",
-          .op_version = GD_OP_VERSION_3_7_0,
-          .flags      = OPT_FLAG_CLIENT_OPT
-        },
-        { .key        = "features.bitrot",
-          .voltype    = "features/bitrot-stub",
-          .value      = "disable",
-          .op_version = GD_OP_VERSION_3_7_0,
-          .type       = NO_DOC,
-        },
-        /* Upcall translator options */
-        { .key         = "features.cache-invalidation",
-          .voltype     = "features/upcall",
-          .value      = "off",
-          .op_version  = GD_OP_VERSION_3_7_0,
-        },
-        { .key         = "features.cache-invalidation-timeout",
-          .voltype     = "features/upcall",
-          .op_version  = GD_OP_VERSION_3_7_0,
-        },
-        { .key         = NULL
-        }
-};
+    /* Quota xlator options */
+    {
+        .key = VKEY_FEATURES_LIMIT_USAGE,
+        .voltype = "features/quota",
+        .option = "limit-set",
+        .type = NO_DOC,
+        .op_version = 1,
+    },
+    {
+        .key = "features.default-soft-limit",
+        .voltype = "features/quota",
+        .option = "default-soft-limit",
+        .type = NO_DOC,
+        .op_version = 3,
+    },
+    {
+        .key = "features.soft-timeout",
+        .voltype = "features/quota",
+        .option = "soft-timeout",
+        .type = NO_DOC,
+        .op_version = 3,
+    },
+    {
+        .key = "features.hard-timeout",
+        .voltype = "features/quota",
+        .option = "hard-timeout",
+        .type = NO_DOC,
+        .op_version = 3,
+    },
+    {
+        .key = "features.alert-time",
+        .voltype = "features/quota",
+        .option = "alert-time",
+        .type = NO_DOC,
+        .op_version = 3,
+    },
+    {
+        .key = "features.quota-deem-statfs",
+        .voltype = "features/quota",
+        .option = "deem-statfs",
+        .value = "off",
+        .type = DOC,
+        .op_version = 2,
+        .validate_fn = validate_quota,
+    },
+
+    /* Marker xlator options */
+    {.key = VKEY_MARKER_XTIME,
+     .voltype = "features/marker",
+     .option = "xtime",
+     .value = "off",
+     .type = NO_DOC,
+     .flags = VOLOPT_FLAG_FORCE,
+     .op_version = 1},
+    {.key = VKEY_MARKER_XTIME,
+     .voltype = "features/marker",
+     .option = "!xtime",
+     .value = "off",
+     .type = NO_DOC,
+     .flags = VOLOPT_FLAG_FORCE,
+     .op_version = 1},
+    {.key = VKEY_MARKER_XTIME_FORCE,
+     .voltype = "features/marker",
+     .option = "gsync-force-xtime",
+     .value = "off",
+     .type = NO_DOC,
+     .flags = VOLOPT_FLAG_FORCE,
+     .op_version = 2},
+    {.key = VKEY_MARKER_XTIME_FORCE,
+     .voltype = "features/marker",
+     .option = "!gsync-force-xtime",
+     .value = "off",
+     .type = NO_DOC,
+     .flags = VOLOPT_FLAG_FORCE,
+     .op_version = 2},
+    {.key = VKEY_FEATURES_QUOTA,
+     .voltype = "features/marker",
+     .option = "quota",
+     .value = "off",
+     .type = NO_DOC,
+     .flags = VOLOPT_FLAG_NEVER_RESET,
+     .op_version = 1},
+    {.key = VKEY_FEATURES_INODE_QUOTA,
+     .voltype = "features/marker",
+     .option = "inode-quota",
+     .value = "off",
+     .type = NO_DOC,
+     .flags = VOLOPT_FLAG_NEVER_RESET,
+     .op_version = 1},
+    {.key = VKEY_FEATURES_BITROT,
+     .voltype = "features/bit-rot",
+     .option = "bitrot",
+     .value = "disable",
+     .type = NO_DOC,
+     .flags = VOLOPT_FLAG_FORCE,
+     .op_version = GD_OP_VERSION_3_7_0},
+
+    /* Debug xlators options */
+    {.key = "debug.trace",
+     .voltype = "debug/trace",
+     .option = "!debug",
+     .value = "off",
+     .type = NO_DOC,
+     .op_version = 1,
+     .flags = VOLOPT_FLAG_XLATOR_OPT},
+    {.key = "debug.log-history",
+     .voltype = "debug/trace",
+     .option = "log-history",
+     .type = NO_DOC,
+     .op_version = 2},
+    {.key = "debug.log-file",
+     .voltype = "debug/trace",
+     .option = "log-file",
+     .type = NO_DOC,
+     .op_version = 2},
+    {.key = "debug.exclude-ops",
+     .voltype = "debug/trace",
+     .option = "exclude-ops",
+     .type = NO_DOC,
+     .op_version = 2},
+    {.key = "debug.include-ops",
+     .voltype = "debug/trace",
+     .option = "include-ops",
+     .type = NO_DOC,
+     .op_version = 2},
+    {.key = "debug.error-gen",
+     .voltype = "debug/error-gen",
+     .option = "!debug",
+     .value = "off",
+     .type = NO_DOC,
+     .op_version = 1,
+     .flags = VOLOPT_FLAG_XLATOR_OPT},
+    {.key = "debug.error-failure",
+     .voltype = "debug/error-gen",
+     .option = "failure",
+     .type = NO_DOC,
+     .op_version = 3},
+    {.key = "debug.error-number",
+     .voltype = "debug/error-gen",
+     .option = "error-no",
+     .type = NO_DOC,
+     .op_version = 3},
+    {.key = "debug.random-failure",
+     .voltype = "debug/error-gen",
+     .option = "random-failure",
+     .type = NO_DOC,
+     .op_version = 3},
+    {.key = "debug.error-fops",
+     .voltype = "debug/error-gen",
+     .option = "enable",
+     .type = NO_DOC,
+     .op_version = 3},
+
+    /* NFS xlator options */
+    {.key = "nfs.enable-ino32",
+     .voltype = "nfs/server",
+     .option = "nfs.enable-ino32",
+     .type = GLOBAL_DOC,
+     .op_version = 1},
+    {.key = "nfs.mem-factor",
+     .voltype = "nfs/server",
+     .option = "nfs.mem-factor",
+     .type = GLOBAL_DOC,
+     .op_version = 1},
+    {.key = "nfs.export-dirs",
+     .voltype = "nfs/server",
+     .option = "nfs3.export-dirs",
+     .type = GLOBAL_DOC,
+     .op_version = 1},
+    {.key = "nfs.export-volumes",
+     .voltype = "nfs/server",
+     .option = "nfs3.export-volumes",
+     .type = GLOBAL_DOC,
+     .op_version = 1},
+    {.key = "nfs.addr-namelookup",
+     .voltype = "nfs/server",
+     .option = "rpc-auth.addr.namelookup",
+     .type = GLOBAL_DOC,
+     .op_version = 1},
+    {.key = "nfs.dynamic-volumes",
+     .voltype = "nfs/server",
+     .option = "nfs.dynamic-volumes",
+     .type = GLOBAL_DOC,
+     .op_version = 1},
+    {.key = "nfs.register-with-portmap",
+     .voltype = "nfs/server",
+     .option = "rpc.register-with-portmap",
+     .type = GLOBAL_DOC,
+     .op_version = 1},
+    {.key = "nfs.outstanding-rpc-limit",
+     .voltype = "nfs/server",
+     .option = "rpc.outstanding-rpc-limit",
+     .type = GLOBAL_DOC,
+     .op_version = 3},
+    {.key = "nfs.port",
+     .voltype = "nfs/server",
+     .option = "nfs.port",
+     .type = GLOBAL_DOC,
+     .op_version = 1},
+    {.key = "nfs.rpc-auth-unix",
+     .voltype = "nfs/server",
+     .option = "!rpc-auth.auth-unix.*",
+     .op_version = 1},
+    {.key = "nfs.rpc-auth-null",
+     .voltype = "nfs/server",
+     .option = "!rpc-auth.auth-null.*",
+     .op_version = 1},
+    {.key = "nfs.rpc-auth-allow",
+     .voltype = "nfs/server",
+     .option = "!rpc-auth.addr.*.allow",
+     .op_version = 1},
+    {.key = "nfs.rpc-auth-reject",
+     .voltype = "nfs/server",
+     .option = "!rpc-auth.addr.*.reject",
+     .op_version = 1},
+    {.key = "nfs.ports-insecure",
+     .voltype = "nfs/server",
+     .option = "!rpc-auth.ports.*.insecure",
+     .op_version = 1},
+    {.key = "nfs.transport-type",
+     .voltype = "nfs/server",
+     .option = "!nfs.transport-type",
+     .op_version = 1,
+     .description = "Specifies the nfs transport type. Valid "
+                    "transport types are 'tcp' and 'rdma'."},
+    {.key = "nfs.trusted-sync",
+     .voltype = "nfs/server",
+     .option = "!nfs3.*.trusted-sync",
+     .op_version = 1},
+    {.key = "nfs.trusted-write",
+     .voltype = "nfs/server",
+     .option = "!nfs3.*.trusted-write",
+     .op_version = 1},
+    {.key = "nfs.volume-access",
+     .voltype = "nfs/server",
+     .option = "!nfs3.*.volume-access",
+     .op_version = 1},
+    {.key = "nfs.export-dir",
+     .voltype = "nfs/server",
+     .option = "!nfs3.*.export-dir",
+     .op_version = 1},
+    {.key = NFS_DISABLE_MAP_KEY,
+     .voltype = "nfs/server",
+     .option = "!nfs-disable",
+     .value = SITE_H_NFS_DISABLE,
+     .op_version = 1},
+    {.key = "nfs.nlm",
+     .voltype = "nfs/server",
+     .option = "nfs.nlm",
+     .type = GLOBAL_DOC,
+     .op_version = 1},
+    {.key = "nfs.acl",
+     .voltype = "nfs/server",
+     .option = "nfs.acl",
+     .type = GLOBAL_DOC,
+     .op_version = 3},
+    {.key = "nfs.mount-udp",
+     .voltype = "nfs/server",
+     .option = "nfs.mount-udp",
+     .type = GLOBAL_DOC,
+     .op_version = 1},
+    {.key = "nfs.mount-rmtab",
+     .voltype = "nfs/server",
+     .option = "nfs.mount-rmtab",
+     .type = GLOBAL_DOC,
+     .op_version = 1},
+    {
+        .key = "nfs.rpc-statd",
+        .voltype = "nfs/server",
+        .option = "nfs.rpc-statd",
+        .type = NO_DOC,
+        .op_version = GD_OP_VERSION_3_6_0,
+    },
+    {
+        .key = "nfs.log-level",
+        .voltype = "nfs/server",
+        .option = "nfs.log-level",
+        .type = NO_DOC,
+        .op_version = GD_OP_VERSION_3_6_0,
+    },
+    {.key = "nfs.server-aux-gids",
+     .voltype = "nfs/server",
+     .option = "nfs.server-aux-gids",
+     .type = NO_DOC,
+     .op_version = 2},
+    {.key = "nfs.drc",
+     .voltype = "nfs/server",
+     .option = "nfs.drc",
+     .type = GLOBAL_DOC,
+     .op_version = 3},
+    {.key = "nfs.drc-size",
+     .voltype = "nfs/server",
+     .option = "nfs.drc-size",
+     .type = GLOBAL_DOC,
+     .op_version = 3},
+    {.key = "nfs.read-size",
+     .voltype = "nfs/server",
+     .option = "nfs3.read-size",
+     .type = GLOBAL_DOC,
+     .op_version = 3},
+    {.key = "nfs.write-size",
+     .voltype = "nfs/server",
+     .option = "nfs3.write-size",
+     .type = GLOBAL_DOC,
+     .op_version = 3},
+    {.key = "nfs.readdir-size",
+     .voltype = "nfs/server",
+     .option = "nfs3.readdir-size",
+     .type = GLOBAL_DOC,
+     .op_version = 3},
+    {.key = "nfs.rdirplus",
+     .voltype = "nfs/server",
+     .option = "nfs.rdirplus",
+     .type = GLOBAL_DOC,
+     .op_version = GD_OP_VERSION_3_7_12,
+     .description = "When this option is set to off NFS falls back to "
+                    "standard readdir instead of readdirp"},
+    {
+        .key = "nfs.event-threads",
+        .voltype = "nfs/server",
+        .option = "nfs.event-threads",
+        .type = NO_DOC,
+        .op_version = GD_OP_VERSION_4_0_0,
+    },
+
+    /* Cli options for Export authentication on nfs mount */
+    {.key = "nfs.exports-auth-enable",
+     .voltype = "nfs/server",
+     .option = "nfs.exports-auth-enable",
+     .type = GLOBAL_DOC,
+     .op_version = GD_OP_VERSION_3_7_0},
+    {.key = "nfs.auth-refresh-interval-sec",
+     .voltype = "nfs/server",
+     .option = "nfs.auth-refresh-interval-sec",
+     .type = GLOBAL_DOC,
+     .op_version = GD_OP_VERSION_3_7_0},
+    {.key = "nfs.auth-cache-ttl-sec",
+     .voltype = "nfs/server",
+     .option = "nfs.auth-cache-ttl-sec",
+     .type = GLOBAL_DOC,
+     .op_version = GD_OP_VERSION_3_7_0},
+
+    /* Other options which don't fit any place above */
+    {.key = "features.read-only",
+     .voltype = "features/read-only",
+     .option = "read-only",
+     .op_version = 1,
+     .flags = VOLOPT_FLAG_CLIENT_OPT | VOLOPT_FLAG_XLATOR_OPT},
+    {.key = "features.worm",
+     .voltype = "features/worm",
+     .option = "worm",
+     .value = "off",
+     .validate_fn = validate_boolean,
+     .op_version = 2,
+     .flags = VOLOPT_FLAG_CLIENT_OPT | VOLOPT_FLAG_XLATOR_OPT},
+    {.key = "features.worm-file-level",
+     .voltype = "features/worm",
+     .option = "worm-file-level",
+     .value = "off",
+     .validate_fn = validate_boolean,
+     .op_version = GD_OP_VERSION_3_8_0,
+     .flags = VOLOPT_FLAG_CLIENT_OPT | VOLOPT_FLAG_XLATOR_OPT},
+    {.key = "features.worm-files-deletable",
+     .voltype = "features/worm",
+     .option = "worm-files-deletable",
+     .value = "on",
+     .validate_fn = validate_boolean,
+     .op_version = GD_OP_VERSION_3_13_0,
+     .flags = VOLOPT_FLAG_CLIENT_OPT | VOLOPT_FLAG_XLATOR_OPT},
+    {
+        .key = "features.default-retention-period",
+        .voltype = "features/worm",
+        .option = "default-retention-period",
+        .validate_fn = validate_worm_period,
+        .op_version = GD_OP_VERSION_3_8_0,
+    },
+    {
+        .key = "features.retention-mode",
+        .voltype = "features/worm",
+        .option = "retention-mode",
+        .validate_fn = validate_reten_mode,
+        .op_version = GD_OP_VERSION_3_8_0,
+    },
+    {
+        .key = "features.auto-commit-period",
+        .voltype = "features/worm",
+        .option = "auto-commit-period",
+        .validate_fn = validate_worm_period,
+        .op_version = GD_OP_VERSION_3_8_0,
+    },
+    {.key = "storage.linux-aio", .voltype = "storage/posix", .op_version = 1},
+    {.key = "storage.batch-fsync-mode",
+     .voltype = "storage/posix",
+     .op_version = 3},
+    {.key = "storage.batch-fsync-delay-usec",
+     .voltype = "storage/posix",
+     .op_version = 3},
+    {
+        .key = "storage.xattr-user-namespace-mode",
+        .voltype = "storage/posix",
+        .op_version = GD_OP_VERSION_3_6_0,
+    },
+    {.key = "storage.owner-uid",
+     .voltype = "storage/posix",
+     .option = "brick-uid",
+     .op_version = 1},
+    {.key = "storage.owner-gid",
+     .voltype = "storage/posix",
+     .option = "brick-gid",
+     .op_version = 1},
+    {.key = "storage.node-uuid-pathinfo",
+     .voltype = "storage/posix",
+     .op_version = 3},
+    {.key = "storage.health-check-interval",
+     .voltype = "storage/posix",
+     .op_version = 3},
+    {
+        .option = "update-link-count-parent",
+        .key = "storage.build-pgfid",
+        .voltype = "storage/posix",
+        .op_version = GD_OP_VERSION_3_6_0,
+    },
+    {
+        .option = "gfid2path",
+        .key = "storage.gfid2path",
+        .type = NO_DOC,
+        .voltype = "storage/posix",
+        .op_version = GD_OP_VERSION_3_12_0,
+    },
+    {
+        .option = "gfid2path-separator",
+        .key = "storage.gfid2path-separator",
+        .voltype = "storage/posix",
+        .op_version = GD_OP_VERSION_3_12_0,
+    },
+    {
+        .key = "storage.reserve",
+        .voltype = "storage/posix",
+        .op_version = GD_OP_VERSION_3_13_0,
+    },
+    {
+        .option = "health-check-timeout",
+        .key = "storage.health-check-timeout",
+        .type = NO_DOC,
+        .voltype = "storage/posix",
+        .op_version = GD_OP_VERSION_4_0_0,
+    },
+    {
+        .option = "fips-mode-rchecksum",
+        .key = "storage.fips-mode-rchecksum",
+        .type = NO_DOC,
+        .voltype = "storage/posix",
+        .op_version = GD_OP_VERSION_4_0_0,
+    },
+    {
+        .option = "force-create-mode",
+        .key = "storage.force-create-mode",
+        .voltype = "storage/posix",
+        .op_version = GD_OP_VERSION_4_0_0,
+    },
+    {
+        .option = "force-directory-mode",
+        .key = "storage.force-directory-mode",
+        .voltype = "storage/posix",
+        .op_version = GD_OP_VERSION_4_0_0,
+    },
+    {
+        .option = "create-mask",
+        .key = "storage.create-mask",
+        .voltype = "storage/posix",
+        .op_version = GD_OP_VERSION_4_0_0,
+    },
+    {
+        .option = "create-directory-mask",
+        .key = "storage.create-directory-mask",
+        .voltype = "storage/posix",
+        .op_version = GD_OP_VERSION_4_0_0,
+    },
+    {
+        .option = "max-hardlinks",
+        .key = "storage.max-hardlinks",
+        .voltype = "storage/posix",
+        .op_version = GD_OP_VERSION_4_0_0,
+    },
+    {
+        .option = "ctime",
+        .key = "features.ctime",
+        .voltype = "storage/posix",
+        .op_version = GD_OP_VERSION_4_1_0,
+    },
+    {.key = "config.memory-accounting",
+     .voltype = "mgmt/glusterd",
+     .option = "!config",
+     .op_version = 2,
+     .flags = VOLOPT_FLAG_CLIENT_OPT},
+    {.key = "config.transport",
+     .voltype = "mgmt/glusterd",
+     .option = "!config",
+     .op_version = 2},
+    {.key = VKEY_CONFIG_GFPROXY,
+     .voltype = "configuration",
+     .option = "gfproxyd",
+     .value = "off",
+     .type = DOC,
+     .op_version = GD_OP_VERSION_3_13_0,
+     .description = "If this option is enabled, the proxy client daemon "
+                    "called gfproxyd will be started on all the trusted "
+                    "storage pool nodes"},
+    {.key = GLUSTERD_QUORUM_TYPE_KEY,
+     .voltype = "mgmt/glusterd",
+     .value = "off",
+     .op_version = 2},
+    {.key = GLUSTERD_QUORUM_RATIO_KEY,
+     .voltype = "mgmt/glusterd",
+     .value = "51",
+     .op_version = 2},
+    /* changelog translator - global tunables */
+    {.key = "changelog.changelog",
+     .voltype = "features/changelog",
+     .type = NO_DOC,
+     .op_version = 3},
+    {.key = "changelog.changelog-dir",
+     .voltype = "features/changelog",
+     .type = NO_DOC,
+     .op_version = 3},
+    {.key = "changelog.encoding",
+     .voltype = "features/changelog",
+     .type = NO_DOC,
+     .op_version = 3},
+    {.key = "changelog.rollover-time",
+     .voltype = "features/changelog",
+     .type = NO_DOC,
+     .op_version = 3},
+    {.key = "changelog.fsync-interval",
+     .voltype = "features/changelog",
+     .type = NO_DOC,
+     .op_version = 3},
+    {
+        .key = "changelog.changelog-barrier-timeout",
+        .voltype = "features/changelog",
+        .value = BARRIER_TIMEOUT,
+        .op_version = GD_OP_VERSION_3_6_0,
+    },
+    {.key = "changelog.capture-del-path",
+     .voltype = "features/changelog",
+     .type = NO_DOC,
+     .op_version = 3},
+    {
+        .key = "features.barrier",
+        .voltype = "features/barrier",
+        .value = "disable",
+        .type = NO_DOC,
+        .op_version = GD_OP_VERSION_3_7_0,
+    },
+    {
+        .key = "features.barrier-timeout",
+        .voltype = "features/barrier",
+        .value = BARRIER_TIMEOUT,
+        .op_version = GD_OP_VERSION_3_6_0,
+    },
+    {
+        .key = GLUSTERD_GLOBAL_OP_VERSION_KEY,
+        .voltype = "mgmt/glusterd",
+        .op_version = GD_OP_VERSION_3_6_0,
+    },
+    {
+        .key = GLUSTERD_MAX_OP_VERSION_KEY,
+        .voltype = "mgmt/glusterd",
+        .op_version = GD_OP_VERSION_3_10_0,
+    },
+    /*Trash translator options */
+    {
+        .key = "features.trash",
+        .voltype = "features/trash",
+        .op_version = GD_OP_VERSION_3_7_0,
+    },
+    {
+        .key = "features.trash-dir",
+        .voltype = "features/trash",
+        .op_version = GD_OP_VERSION_3_7_0,
+    },
+    {
+        .key = "features.trash-eliminate-path",
+        .voltype = "features/trash",
+        .op_version = GD_OP_VERSION_3_7_0,
+    },
+    {
+        .key = "features.trash-max-filesize",
+        .voltype = "features/trash",
+        .op_version = GD_OP_VERSION_3_7_0,
+    },
+    {
+        .key = "features.trash-internal-op",
+        .voltype = "features/trash",
+        .op_version = GD_OP_VERSION_3_7_0,
+    },
+    {.key = GLUSTERD_SHARED_STORAGE_KEY,
+     .voltype = "mgmt/glusterd",
+     .value = "disable",
+     .type = GLOBAL_DOC,
+     .op_version = GD_OP_VERSION_3_7_1,
+     .description = "Create and mount the shared storage volume"
+                    "(gluster_shared_storage) at "
+                    "/var/run/gluster/shared_storage on enabling this "
+                    "option. Unmount and delete the shared storage volume "
+                    " on disabling this option."},
+    {
+        .key = "locks.trace",
+        .voltype = "features/locks",
+        .op_version = GD_OP_VERSION_3_7_0,
+    },
+    {
+        .key = "locks.mandatory-locking",
+        .voltype = "features/locks",
+        .op_version = GD_OP_VERSION_3_8_0,
+        .validate_fn = validate_mandatory_locking,
+    },
+    {.key = "cluster.disperse-self-heal-daemon",
+     .voltype = "cluster/disperse",
+     .type = NO_DOC,
+     .option = "self-heal-daemon",
+     .op_version = GD_OP_VERSION_3_7_0,
+     .validate_fn = validate_disperse_heal_enable_disable},
+    {.key = "cluster.quorum-reads",
+     .voltype = "cluster/replicate",
+     .op_version = GD_OP_VERSION_3_7_0,
+     .flags = VOLOPT_FLAG_CLIENT_OPT},
+    {.key = "client.bind-insecure",
+     .voltype = "protocol/client",
+     .option = "client-bind-insecure",
+     .type = NO_DOC,
+     .op_version = GD_OP_VERSION_3_7_0,
+     .flags = VOLOPT_FLAG_CLIENT_OPT},
+    {.key = "features.timeout",
+     .voltype = "features/quiesce",
+     .option = "timeout",
+     .op_version = GD_OP_VERSION_4_0_0,
+     .flags = VOLOPT_FLAG_CLIENT_OPT,
+     .description = "Specifies the number of seconds the "
+                    "quiesce translator will wait "
+                    "for a CHILD_UP event before "
+                    "force-unwinding the frames it has "
+                    "currently stored for retry."},
+    {.key = "features.failover-hosts",
+     .voltype = "features/quiesce",
+     .option = "failover-hosts",
+     .op_version = GD_OP_VERSION_4_0_0,
+     .flags = VOLOPT_FLAG_CLIENT_OPT,
+     .description = "It is a comma separated list of hostname/IP "
+                    "addresses. It Specifies the list of hosts where "
+                    "the gfproxy daemons are running, to which the "
+                    "the thin clients can failover to."},
+    {.key = "features.shard",
+     .voltype = "features/shard",
+     .value = "off",
+     .option = "!shard",
+     .op_version = GD_OP_VERSION_3_7_0,
+     .description = "enable/disable sharding translator on the volume.",
+     .flags = VOLOPT_FLAG_CLIENT_OPT | VOLOPT_FLAG_XLATOR_OPT},
+    {.key = "features.shard-block-size",
+     .voltype = "features/shard",
+     .op_version = GD_OP_VERSION_3_7_0,
+     .flags = VOLOPT_FLAG_CLIENT_OPT},
+    {
+        .key = "features.shard-lru-limit",
+        .voltype = "features/shard",
+        .op_version = GD_OP_VERSION_5_0,
+        .flags = VOLOPT_FLAG_CLIENT_OPT,
+        .type = NO_DOC,
+    },
+    {.key = "features.shard-deletion-rate",
+     .voltype = "features/shard",
+     .op_version = GD_OP_VERSION_5_0,
+     .flags = VOLOPT_FLAG_CLIENT_OPT},
+    {
+        .key = "features.scrub-throttle",
+        .voltype = "features/bit-rot",
+        .value = "lazy",
+        .option = "scrub-throttle",
+        .op_version = GD_OP_VERSION_3_7_0,
+        .type = NO_DOC,
+    },
+    {
+        .key = "features.scrub-freq",
+        .voltype = "features/bit-rot",
+        .value = "biweekly",
+        .option = "scrub-frequency",
+        .op_version = GD_OP_VERSION_3_7_0,
+        .type = NO_DOC,
+    },
+    {
+        .key = "features.scrub",
+        .voltype = "features/bit-rot",
+        .option = "scrubber",
+        .op_version = GD_OP_VERSION_3_7_0,
+        .flags = VOLOPT_FLAG_FORCE,
+        .type = NO_DOC,
+    },
+    {
+        .key = "features.expiry-time",
+        .voltype = "features/bit-rot",
+        .value = SIGNING_TIMEOUT,
+        .option = "expiry-time",
+        .op_version = GD_OP_VERSION_3_7_0,
+        .type = NO_DOC,
+    },
+    {
+        .key = "features.signer-threads",
+        .voltype = "features/bit-rot",
+        .value = BR_WORKERS,
+        .option = "signer-threads",
+        .op_version = GD_OP_VERSION_8_0,
+        .type = NO_DOC,
+    },
+    /* Upcall translator options */
+    /* Upcall translator options */
+    {
+        .key = "features.cache-invalidation",
+        .voltype = "features/upcall",
+        .value = "off",
+        .op_version = GD_OP_VERSION_3_7_0,
+    },
+    {
+        .key = "features.cache-invalidation-timeout",
+        .voltype = "features/upcall",
+        .op_version = GD_OP_VERSION_3_7_0,
+    },
+    {
+        .key = "ganesha.enable",
+        .voltype = "mgmt/ganesha",
+        .value = "off",
+        .option = "ganesha.enable",
+        .op_version = GD_OP_VERSION_7_0,
+    },
+    /* Lease translator options */
+    {
+        .key = "features.leases",
+        .voltype = "features/leases",
+        .value = "off",
+        .op_version = GD_OP_VERSION_3_8_0,
+    },
+    {
+        .key = "features.lease-lock-recall-timeout",
+        .voltype = "features/leases",
+        .op_version = GD_OP_VERSION_3_8_0,
+    },
+    {.key = "disperse.background-heals",
+     .voltype = "cluster/disperse",
+     .op_version = GD_OP_VERSION_3_7_3,
+     .flags = VOLOPT_FLAG_CLIENT_OPT},
+    {.key = "disperse.heal-wait-qlength",
+     .voltype = "cluster/disperse",
+     .op_version = GD_OP_VERSION_3_7_3,
+     .flags = VOLOPT_FLAG_CLIENT_OPT},
+    {
+        .key = "cluster.heal-timeout",
+        .voltype = "cluster/disperse",
+        .option = "!heal-timeout",
+        .op_version = GD_OP_VERSION_3_7_3,
+        .type = NO_DOC,
+    },
+    {.key = "dht.force-readdirp",
+     .voltype = "cluster/distribute",
+     .option = "use-readdirp",
+     .op_version = GD_OP_VERSION_3_7_5,
+     .flags = VOLOPT_FLAG_CLIENT_OPT},
+    {.key = "disperse.read-policy",
+     .voltype = "cluster/disperse",
+     .op_version = GD_OP_VERSION_3_7_6,
+     .flags = VOLOPT_FLAG_CLIENT_OPT},
+    {.key = "cluster.shd-max-threads",
+     .voltype = "cluster/replicate",
+     .op_version = GD_OP_VERSION_3_7_12,
+     .flags = VOLOPT_FLAG_CLIENT_OPT,
+     .validate_fn = validate_replica},
+    {.key = "cluster.shd-wait-qlength",
+     .voltype = "cluster/replicate",
+     .op_version = GD_OP_VERSION_3_7_12,
+     .flags = VOLOPT_FLAG_CLIENT_OPT},
+    {.key = "cluster.locking-scheme",
+     .voltype = "cluster/replicate",
+     .type = DOC,
+     .op_version = GD_OP_VERSION_3_7_12,
+     .flags = VOLOPT_FLAG_CLIENT_OPT},
+    {.key = "cluster.granular-entry-heal",
+     .voltype = "cluster/replicate",
+     .type = DOC,
+     .op_version = GD_OP_VERSION_3_8_0,
+     .flags = VOLOPT_FLAG_CLIENT_OPT},
+    {
+        .option = "revocation-secs",
+        .key = "features.locks-revocation-secs",
+        .voltype = "features/locks",
+        .op_version = GD_OP_VERSION_3_9_0,
+    },
+    {
+        .option = "revocation-clear-all",
+        .key = "features.locks-revocation-clear-all",
+        .voltype = "features/locks",
+        .op_version = GD_OP_VERSION_3_9_0,
+    },
+    {
+        .option = "revocation-max-blocked",
+        .key = "features.locks-revocation-max-blocked",
+        .voltype = "features/locks",
+        .op_version = GD_OP_VERSION_3_9_0,
+    },
+    {
+        .option = "monkey-unlocking",
+        .key = "features.locks-monkey-unlocking",
+        .voltype = "features/locks",
+        .op_version = GD_OP_VERSION_3_9_0,
+        .type = NO_DOC,
+    },
+    {
+        .option = "notify-contention",
+        .key = "features.locks-notify-contention",
+        .voltype = "features/locks",
+        .op_version = GD_OP_VERSION_4_0_0,
+    },
+    {
+        .option = "notify-contention-delay",
+        .key = "features.locks-notify-contention-delay",
+        .voltype = "features/locks",
+        .op_version = GD_OP_VERSION_4_0_0,
+    },
+    {.key = "disperse.shd-max-threads",
+     .voltype = "cluster/disperse",
+     .op_version = GD_OP_VERSION_3_9_0,
+     .flags = VOLOPT_FLAG_CLIENT_OPT,
+     .validate_fn = validate_disperse},
+    {.key = "disperse.shd-wait-qlength",
+     .voltype = "cluster/disperse",
+     .op_version = GD_OP_VERSION_3_9_0,
+     .flags = VOLOPT_FLAG_CLIENT_OPT},
+    {.key = "disperse.cpu-extensions",
+     .voltype = "cluster/disperse",
+     .op_version = GD_OP_VERSION_3_9_0,
+     .flags = VOLOPT_FLAG_CLIENT_OPT},
+    {.key = "disperse.self-heal-window-size",
+     .voltype = "cluster/disperse",
+     .op_version = GD_OP_VERSION_3_11_0,
+     .flags = VOLOPT_FLAG_CLIENT_OPT},
+    {.key = "cluster.use-compound-fops",
+     .voltype = "cluster/replicate",
+     .value = "off",
+     .type = DOC,
+     .op_version = GD_OP_VERSION_3_8_4,
+     .flags = VOLOPT_FLAG_CLIENT_OPT},
+    {.key = "performance.parallel-readdir",
+     .voltype = "performance/readdir-ahead",
+     .option = "parallel-readdir",
+     .value = "off",
+     .type = DOC,
+     .op_version = GD_OP_VERSION_3_10_0,
+     .validate_fn = validate_parallel_readdir,
+     .description = "If this option is enabled, the readdir operation "
+                    "is performed in parallel on all the bricks, thus "
+                    "improving the performance of readdir. Note that "
+                    "the performance improvement is higher in large "
+                    "clusters"},
+    {
+        .key = "performance.rda-request-size",
+        .voltype = "performance/readdir-ahead",
+        .option = "rda-request-size",
+        .value = "131072",
+        .flags = VOLOPT_FLAG_CLIENT_OPT,
+        .type = DOC,
+        .op_version = GD_OP_VERSION_3_9_1,
+    },
+    {
+        .key = "performance.rda-low-wmark",
+        .voltype = "performance/readdir-ahead",
+        .option = "rda-low-wmark",
+        .type = NO_DOC,
+        .flags = VOLOPT_FLAG_CLIENT_OPT,
+        .op_version = GD_OP_VERSION_3_9_1,
+    },
+    {
+        .key = "performance.rda-high-wmark",
+        .voltype = "performance/readdir-ahead",
+        .type = NO_DOC,
+        .flags = VOLOPT_FLAG_CLIENT_OPT,
+        .op_version = GD_OP_VERSION_3_9_1,
+    },
+    {.key = "performance.rda-cache-limit",
+     .voltype = "performance/readdir-ahead",
+     .value = "10MB",
+     .type = DOC,
+     .flags = VOLOPT_FLAG_CLIENT_OPT,
+     .op_version = GD_OP_VERSION_3_9_1,
+     .validate_fn = validate_rda_cache_limit},
+    {
+        .key = "performance.nl-cache-positive-entry",
+        .voltype = "performance/nl-cache",
+        .type = DOC,
+        .flags = VOLOPT_FLAG_CLIENT_OPT,
+        .op_version = GD_OP_VERSION_3_11_0,
+        .description = "enable/disable storing of entries that were lookedup"
+                       " and found to be present in the volume, thus lookup"
+                       " on non existent file is served from the cache",
+    },
+    {
+        .key = "performance.nl-cache-limit",
+        .voltype = "performance/nl-cache",
+        .value = "10MB",
+        .flags = VOLOPT_FLAG_CLIENT_OPT,
+        .op_version = GD_OP_VERSION_3_11_0,
+    },
+    {
+        .key = "performance.nl-cache-timeout",
+        .voltype = "performance/nl-cache",
+        .flags = VOLOPT_FLAG_CLIENT_OPT,
+        .op_version = GD_OP_VERSION_3_11_0,
+    },
+
+    /* Brick multiplexing options */
+    {.key = GLUSTERD_BRICK_MULTIPLEX_KEY,
+     .voltype = "mgmt/glusterd",
+     .value = "disable",
+     .op_version = GD_OP_VERSION_3_10_0,
+     .validate_fn = validate_boolean,
+     .type = GLOBAL_DOC,
+     .description = "This global option can be used to enable/disable "
+                    "brick multiplexing. Brick multiplexing ensures that "
+                    "compatible brick instances can share one single "
+                    "brick process."},
+    {.key = GLUSTERD_VOL_CNT_PER_THRD,
+     .voltype = "mgmt/glusterd",
+     .value = GLUSTERD_VOL_CNT_PER_THRD_DEFAULT_VALUE,
+     .op_version = GD_OP_VERSION_7_0,
+     .validate_fn = validate_volume_per_thread_limit,
+     .type = GLOBAL_NO_DOC,
+     .description =
+         "This option can be used to limit the number of volumes "
+         "handled per thread to populate peer data.The option accepts "
+         "values in the range of 5 to 200"},
+    {.key = GLUSTERD_BRICKMUX_LIMIT_KEY,
+     .voltype = "mgmt/glusterd",
+     .value = GLUSTERD_BRICKMUX_LIMIT_DFLT_VALUE,
+     .op_version = GD_OP_VERSION_3_12_0,
+     .validate_fn = validate_mux_limit,
+     .type = GLOBAL_DOC,
+     .description = "This option can be used to limit the number of brick "
+                    "instances per brick process when brick-multiplexing "
+                    "is enabled. If not explicitly set, this tunable is "
+                    "set to 0 which denotes that brick-multiplexing can "
+                    "happen without any limit on the number of bricks per "
+                    "process. Also this option can't be set when the "
+                    "brick-multiplexing feature is disabled."},
+    {.key = "disperse.optimistic-change-log",
+     .voltype = "cluster/disperse",
+     .type = NO_DOC,
+     .op_version = GD_OP_VERSION_3_10_1,
+     .flags = VOLOPT_FLAG_CLIENT_OPT},
+    {.key = "disperse.stripe-cache",
+     .voltype = "cluster/disperse",
+     .type = NO_DOC,
+     .op_version = GD_OP_VERSION_4_0_0,
+     .flags = VOLOPT_FLAG_CLIENT_OPT},
+
+    /* Halo replication options */
+    {.key = "cluster.halo-enabled",
+     .voltype = "cluster/replicate",
+     .op_version = GD_OP_VERSION_3_11_0,
+     .flags = VOLOPT_FLAG_CLIENT_OPT},
+    {.key = "cluster.halo-shd-max-latency",
+     .voltype = "cluster/replicate",
+     .op_version = GD_OP_VERSION_3_11_0,
+     .flags = VOLOPT_FLAG_CLIENT_OPT},
+    {.key = "cluster.halo-nfsd-max-latency",
+     .voltype = "cluster/replicate",
+     .op_version = GD_OP_VERSION_3_11_0,
+     .flags = VOLOPT_FLAG_CLIENT_OPT},
+    {.key = "cluster.halo-max-latency",
+     .voltype = "cluster/replicate",
+     .op_version = GD_OP_VERSION_3_11_0,
+     .flags = VOLOPT_FLAG_CLIENT_OPT},
+    {.key = "cluster.halo-max-replicas",
+     .voltype = "cluster/replicate",
+     .op_version = GD_OP_VERSION_3_11_0,
+     .flags = VOLOPT_FLAG_CLIENT_OPT},
+    {.key = "cluster.halo-min-replicas",
+     .voltype = "cluster/replicate",
+     .op_version = GD_OP_VERSION_3_11_0,
+     .flags = VOLOPT_FLAG_CLIENT_OPT},
+    {.key = VKEY_FEATURES_SELINUX,
+     .voltype = "features/selinux",
+     .type = NO_DOC,
+     .value = "on",
+     .op_version = GD_OP_VERSION_3_11_0,
+     .description = "Convert security.selinux xattrs to "
+                    "trusted.gluster.selinux on the bricks. Recommended "
+                    "to have enabled when clients and/or bricks support "
+                    "SELinux."},
+    {.key = GLUSTERD_LOCALTIME_LOGGING_KEY,
+     .voltype = "mgmt/glusterd",
+     .type = GLOBAL_DOC,
+     .op_version = GD_OP_VERSION_3_12_0,
+     .validate_fn = validate_boolean},
+    {.key = GLUSTERD_DAEMON_LOG_LEVEL_KEY,
+     .voltype = "mgmt/glusterd",
+     .type = GLOBAL_NO_DOC,
+     .value = "INFO",
+     .op_version = GD_OP_VERSION_5_0},
+    {.key = "debug.delay-gen",
+     .voltype = "debug/delay-gen",
+     .option = "!debug",
+     .value = "off",
+     .type = NO_DOC,
+     .op_version = GD_OP_VERSION_3_13_0,
+     .flags = VOLOPT_FLAG_XLATOR_OPT},
+    {
+        .key = "delay-gen.delay-percentage",
+        .voltype = "debug/delay-gen",
+        .type = NO_DOC,
+        .op_version = GD_OP_VERSION_3_13_0,
+    },
+    {
+        .key = "delay-gen.delay-duration",
+        .voltype = "debug/delay-gen",
+        .type = NO_DOC,
+        .op_version = GD_OP_VERSION_3_13_0,
+    },
+    {
+        .key = "delay-gen.enable",
+        .voltype = "debug/delay-gen",
+        .type = NO_DOC,
+        .op_version = GD_OP_VERSION_3_13_0,
+    },
+    {.key = "disperse.parallel-writes",
+     .voltype = "cluster/disperse",
+     .type = NO_DOC,
+     .op_version = GD_OP_VERSION_3_13_0,
+     .flags = VOLOPT_FLAG_CLIENT_OPT},
+    {.key = "disperse.quorum-count",
+     .voltype = "cluster/disperse",
+     .type = NO_DOC,
+     .op_version = GD_OP_VERSION_8_0,
+     .validate_fn = validate_disperse_quorum_count,
+     .description = "This option can be used to define how many successes on"
+                    "the bricks constitute a success to the application. This"
+                    " count should be in the range"
+                    "[disperse-data-count,  disperse-count] (inclusive)",
+     .flags = VOLOPT_FLAG_CLIENT_OPT},
+    {
+        .key = "features.sdfs",
+        .voltype = "features/sdfs",
+        .value = "off",
+        .option = "!features",
+        .op_version = GD_OP_VERSION_4_0_0,
+        .description = "enable/disable dentry serialization xlator in volume",
+        .type = NO_DOC,
+    },
+    {.key = "features.cloudsync",
+     .voltype = "features/cloudsync",
+     .value = "off",
+     .op_version = GD_OP_VERSION_4_1_0,
+     .flags = VOLOPT_FLAG_CLIENT_OPT},
+    {.key = "features.ctime",
+     .voltype = "features/utime",
+     .validate_fn = validate_boolean,
+     .value = "on",
+     .option = "!utime",
+     .op_version = GD_OP_VERSION_4_1_0,
+     .description = "enable/disable utime translator on the volume.",
+     .flags = VOLOPT_FLAG_CLIENT_OPT | VOLOPT_FLAG_XLATOR_OPT},
+    {.key = "ctime.noatime",
+     .voltype = "features/utime",
+     .validate_fn = validate_boolean,
+     .value = "on",
+     .option = "noatime",
+     .op_version = GD_OP_VERSION_5_0,
+     .description = "enable/disable noatime option with ctime enabled.",
+     .flags = VOLOPT_FLAG_CLIENT_OPT | VOLOPT_FLAG_XLATOR_OPT},
+    {.key = "features.cloudsync-storetype",
+     .voltype = "features/cloudsync",
+     .op_version = GD_OP_VERSION_5_0,
+     .flags = VOLOPT_FLAG_CLIENT_OPT},
+    {.key = "features.s3plugin-seckey",
+     .voltype = "features/cloudsync",
+     .op_version = GD_OP_VERSION_5_0,
+     .flags = VOLOPT_FLAG_CLIENT_OPT},
+    {.key = "features.s3plugin-keyid",
+     .voltype = "features/cloudsync",
+     .op_version = GD_OP_VERSION_5_0,
+     .flags = VOLOPT_FLAG_CLIENT_OPT},
+    {.key = "features.s3plugin-bucketid",
+     .voltype = "features/cloudsync",
+     .op_version = GD_OP_VERSION_5_0,
+     .flags = VOLOPT_FLAG_CLIENT_OPT},
+    {.key = "features.s3plugin-hostname",
+     .voltype = "features/cloudsync",
+     .op_version = GD_OP_VERSION_5_0,
+     .flags = VOLOPT_FLAG_CLIENT_OPT},
+    {.key = "features.enforce-mandatory-lock",
+     .voltype = "features/locks",
+     .value = "off",
+     .type = NO_DOC,
+     .op_version = GD_OP_VERSION_6_0,
+     .validate_fn = validate_boolean,
+     .description = "option to enforce mandatory lock on a file",
+     .flags = VOLOPT_FLAG_XLATOR_OPT},
+    {.key = VKEY_CONFIG_GLOBAL_THREADING,
+     .voltype = "debug/io-stats",
+     .option = "global-threading",
+     .value = "off",
+     .op_version = GD_OP_VERSION_6_0},
+    {.key = VKEY_CONFIG_CLIENT_THREADS,
+     .voltype = "debug/io-stats",
+     .option = "!client-threads",
+     .value = "16",
+     .op_version = GD_OP_VERSION_6_0},
+    {.key = VKEY_CONFIG_BRICK_THREADS,
+     .voltype = "debug/io-stats",
+     .option = "!brick-threads",
+     .value = "16",
+     .op_version = GD_OP_VERSION_6_0},
+    {.key = "features.cloudsync-remote-read",
+     .voltype = "features/cloudsync",
+     .value = "off",
+     .op_version = GD_OP_VERSION_7_0,
+     .flags = VOLOPT_FLAG_CLIENT_OPT},
+    {.key = "features.cloudsync-store-id",
+     .voltype = "features/cloudsync",
+     .op_version = GD_OP_VERSION_7_0,
+     .flags = VOLOPT_FLAG_CLIENT_OPT},
+    {.key = "features.cloudsync-product-id",
+     .voltype = "features/cloudsync",
+     .op_version = GD_OP_VERSION_7_0,
+     .flags = VOLOPT_FLAG_CLIENT_OPT},
+    {
+        .key = "features.acl",
+        .voltype = "features/access-control",
+        .value = "enable",
+        .option = "!features",
+        .op_version = GD_OP_VERSION_8_0,
+        .description = "(WARNING: for debug purpose only) enable/disable "
+                       "access-control xlator in volume",
+        .type = NO_DOC,
+    },
+
+    {.key = "cluster.use-anonymous-inode",
+     .voltype = "cluster/replicate",
+     .op_version = GD_OP_VERSION_9_0,
+     .value = "yes",
+     .flags = VOLOPT_FLAG_CLIENT_OPT},
+    {.key = NULL}};
diff --git a/xlators/mgmt/glusterd/src/glusterd.c b/xlators/mgmt/glusterd/src/glusterd.c
index 6ffde511af4..7a86c2997b1 100644
--- a/xlators/mgmt/glusterd/src/glusterd.c
+++ b/xlators/mgmt/glusterd/src/glusterd.c
@@ -8,29 +8,26 @@
    cases as published by the Free Software Foundation.
 */
 
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
 #include <time.h>
 #include <grp.h>
 #include <sys/uio.h>
 #include <sys/resource.h>
 
 #include <libgen.h>
-#include "compat-uuid.h"
+#include <glusterfs/compat-uuid.h>
 
 #include "glusterd.h"
 #include "rpcsvc.h"
 #include "fnmatch.h"
-#include "xlator.h"
-#include "call-stub.h"
-#include "defaults.h"
-#include "list.h"
-#include "dict.h"
-#include "compat.h"
-#include "compat-errno.h"
-#include "syscall.h"
+#include <glusterfs/xlator.h>
+#include <glusterfs/call-stub.h>
+#include <glusterfs/defaults.h>
+#include <glusterfs/list.h>
+#include <glusterfs/dict.h>
+#include <glusterfs/options.h>
+#include <glusterfs/compat.h>
+#include <glusterfs/compat-errno.h>
+#include <glusterfs/syscall.h>
 #include "glusterd-statedump.h"
 #include "glusterd-sm.h"
 #include "glusterd-op-sm.h"
@@ -40,17 +37,21 @@
 #include "glusterd-locks.h"
 #include "glusterd-svc-mgmt.h"
 #include "glusterd-shd-svc.h"
+#ifdef BUILD_GNFS
 #include "glusterd-nfs-svc.h"
+#endif
 #include "glusterd-bitd-svc.h"
 #include "glusterd-scrub-svc.h"
 #include "glusterd-quotad-svc.h"
 #include "glusterd-snapd-svc.h"
-#include "common-utils.h"
+#include "glusterd-messages.h"
+#include <glusterfs/common-utils.h>
 #include "glusterd-geo-rep.h"
-#include "run.h"
+#include <glusterfs/run.h>
 #include "rpc-clnt-ping.h"
+#include "rpc-common-xdr.h"
 
-#include "syncop.h"
+#include <glusterfs/syncop.h>
 
 #include "glusterd-mountbroker.h"
 
@@ -66,1179 +67,1325 @@ extern struct rpcsvc_program gd_svc_cli_trusted_progs;
 extern struct rpc_clnt_program gd_brick_prog;
 extern struct rpcsvc_program glusterd_mgmt_hndsk_prog;
 
-extern char snap_mount_folder[PATH_MAX];
+extern char snap_mount_dir[VALID_GLUSTERD_PATHMAX];
 
 rpcsvc_cbk_program_t glusterd_cbk_prog = {
-        .progname  = "Gluster Callback",
-        .prognum   = GLUSTER_CBK_PROGRAM,
-        .progver   = GLUSTER_CBK_VERSION,
+    .progname = "Gluster Callback",
+    .prognum = GLUSTER_CBK_PROGRAM,
+    .progver = GLUSTER_CBK_VERSION,
 };
 
 struct rpcsvc_program *gd_inet_programs[] = {
-        &gd_svc_peer_prog,
-        &gd_svc_cli_trusted_progs, /* Must be index 1 for secure_mgmt! */
-        &gd_svc_mgmt_prog,
-        &gd_svc_mgmt_v3_prog,
-        &gluster_pmap_prog,
-        &gluster_handshake_prog,
-        &glusterd_mgmt_hndsk_prog,
+    &gd_svc_peer_prog,       &gd_svc_cli_trusted_progs, /* Must be index 1 for
+                                                           secure_mgmt! */
+    &gd_svc_mgmt_prog,       &gd_svc_mgmt_v3_prog,      &gluster_pmap_prog,
+    &gluster_handshake_prog, &glusterd_mgmt_hndsk_prog,
 };
-int gd_inet_programs_count = (sizeof (gd_inet_programs) /
-                              sizeof (gd_inet_programs[0]));
+int gd_inet_programs_count = (sizeof(gd_inet_programs) /
+                              sizeof(gd_inet_programs[0]));
 
 struct rpcsvc_program *gd_uds_programs[] = {
-        &gd_svc_cli_prog,
-        &gluster_cli_getspec_prog,
+    &gd_svc_cli_prog,
+    &gluster_cli_getspec_prog,
 };
-int gd_uds_programs_count = (sizeof (gd_uds_programs) /
-                             sizeof (gd_uds_programs[0]));
+int gd_uds_programs_count = (sizeof(gd_uds_programs) /
+                             sizeof(gd_uds_programs[0]));
 
 const char *gd_op_list[GD_OP_MAX + 1] = {
-        [GD_OP_NONE]                    = "Invalid op",
-        [GD_OP_CREATE_VOLUME]           = "Create",
-        [GD_OP_START_BRICK]             = "Start Brick",
-        [GD_OP_STOP_BRICK]              = "Stop Brick",
-        [GD_OP_DELETE_VOLUME]           = "Delete",
-        [GD_OP_START_VOLUME]            = "Start",
-        [GD_OP_STOP_VOLUME]             = "Stop",
-        [GD_OP_DEFRAG_VOLUME]           = "Rebalance",
-        [GD_OP_ADD_BRICK]               = "Add brick",
-        [GD_OP_REMOVE_BRICK]            = "Remove brick",
-        [GD_OP_REPLACE_BRICK]           = "Replace brick",
-        [GD_OP_SET_VOLUME]              = "Set",
-        [GD_OP_RESET_VOLUME]            = "Reset",
-        [GD_OP_SYNC_VOLUME]             = "Sync",
-        [GD_OP_LOG_ROTATE]              = "Log rotate",
-        [GD_OP_GSYNC_SET]               = "Geo-replication",
-        [GD_OP_PROFILE_VOLUME]          = "Profile",
-        [GD_OP_QUOTA]                   = "Quota",
-        [GD_OP_STATUS_VOLUME]           = "Status",
-        [GD_OP_REBALANCE]               = "Rebalance",
-        [GD_OP_HEAL_VOLUME]             = "Heal",
-        [GD_OP_STATEDUMP_VOLUME]        = "Statedump",
-        [GD_OP_LIST_VOLUME]             = "Lists",
-        [GD_OP_CLEARLOCKS_VOLUME]       = "Clear locks",
-        [GD_OP_DEFRAG_BRICK_VOLUME]     = "Rebalance",
-        [GD_OP_COPY_FILE]               = "Copy File",
-        [GD_OP_SYS_EXEC]                = "Execute system commands",
-        [GD_OP_GSYNC_CREATE]            = "Geo-replication Create",
-        [GD_OP_SNAP]                    = "Snapshot",
-        [GD_OP_MAX]                     = "Invalid op"
-};
+    [GD_OP_NONE] = "Invalid op",
+    [GD_OP_CREATE_VOLUME] = "Create",
+    [GD_OP_START_BRICK] = "Start Brick",
+    [GD_OP_STOP_BRICK] = "Stop Brick",
+    [GD_OP_DELETE_VOLUME] = "Delete",
+    [GD_OP_START_VOLUME] = "Start",
+    [GD_OP_STOP_VOLUME] = "Stop",
+    [GD_OP_DEFRAG_VOLUME] = "Rebalance",
+    [GD_OP_ADD_BRICK] = "Add brick",
+    [GD_OP_DETACH_TIER] = "Detach tier",
+    [GD_OP_TIER_MIGRATE] = "Tier migration",
+    [GD_OP_REMOVE_BRICK] = "Remove brick",
+    [GD_OP_REPLACE_BRICK] = "Replace brick",
+    [GD_OP_SET_VOLUME] = "Set",
+    [GD_OP_RESET_VOLUME] = "Reset",
+    [GD_OP_SYNC_VOLUME] = "Sync",
+    [GD_OP_LOG_ROTATE] = "Log rotate",
+    [GD_OP_GSYNC_SET] = "Geo-replication",
+    [GD_OP_PROFILE_VOLUME] = "Profile",
+    [GD_OP_QUOTA] = "Quota",
+    [GD_OP_STATUS_VOLUME] = "Status",
+    [GD_OP_REBALANCE] = "Rebalance",
+    [GD_OP_HEAL_VOLUME] = "Heal",
+    [GD_OP_STATEDUMP_VOLUME] = "Statedump",
+    [GD_OP_LIST_VOLUME] = "Lists",
+    [GD_OP_CLEARLOCKS_VOLUME] = "Clear locks",
+    [GD_OP_DEFRAG_BRICK_VOLUME] = "Rebalance",
+    [GD_OP_COPY_FILE] = "Copy File",
+    [GD_OP_SYS_EXEC] = "Execute system commands",
+    [GD_OP_GSYNC_CREATE] = "Geo-replication Create",
+    [GD_OP_SNAP] = "Snapshot",
+    [GD_OP_RESET_BRICK] = "Reset Brick",
+    [GD_OP_MAX_OPVERSION] = "Maximum supported op-version",
+    [GD_OP_MAX] = "Invalid op"};
 
 static int
-glusterd_opinfo_init ()
+glusterd_opinfo_init()
 {
-        int32_t ret = -1;
+    int32_t ret = -1;
 
-        opinfo.op = GD_OP_NONE;
+    opinfo.op = GD_OP_NONE;
 
-        return ret;
+    return ret;
 }
 
-
 int
-glusterd_uuid_init ()
+glusterd_uuid_init()
 {
-        int             ret = -1;
-        xlator_t        *this = NULL;
-        glusterd_conf_t *priv = NULL;
-
-        this = THIS;
-        GF_ASSERT (this);
-        priv = this->private;
-
-        ret = glusterd_retrieve_uuid ();
-        if (ret == 0) {
-                gf_log (this->name, GF_LOG_INFO,
-                        "retrieved UUID: %s", uuid_utoa (priv->uuid));
-                return 0;
-        }
+    int ret = -1;
+    xlator_t *this = NULL;
+    glusterd_conf_t *priv = NULL;
+
+    this = THIS;
+    GF_ASSERT(this);
+    priv = this->private;
+
+    ret = glusterd_retrieve_uuid();
+    if (ret == 0) {
+        gf_msg(this->name, GF_LOG_INFO, 0, GD_MSG_RETRIEVED_UUID,
+               "retrieved UUID: %s", uuid_utoa(priv->uuid));
+        return 0;
+    }
 
-        ret = glusterd_uuid_generate_save ();
+    ret = glusterd_uuid_generate_save();
 
-        if (ret) {
-                gf_log ("glusterd", GF_LOG_ERROR,
-                          "Unable to generate and save new UUID");
-                return ret;
-        }
+    if (ret) {
+        gf_msg("glusterd", GF_LOG_ERROR, 0, GD_MSG_UUID_GEN_STORE_FAIL,
+               "Unable to generate and save new UUID");
+        return ret;
+    }
 
-        return 0;
+    return 0;
 }
 
 int
-glusterd_uuid_generate_save ()
+glusterd_uuid_generate_save()
 {
-        int               ret = -1;
-        glusterd_conf_t   *priv = NULL;
-        xlator_t          *this = NULL;
+    int ret = -1;
+    glusterd_conf_t *priv = NULL;
+    xlator_t *this = NULL;
 
-        this = THIS;
-        GF_ASSERT (this);
-        priv = this->private;
-        GF_ASSERT (priv);
+    this = THIS;
+    GF_ASSERT(this);
+    priv = this->private;
+    GF_ASSERT(priv);
 
-        gf_uuid_generate (priv->uuid);
+    gf_uuid_generate(priv->uuid);
 
-        gf_log (this->name, GF_LOG_INFO, "generated UUID: %s",
-                uuid_utoa (priv->uuid));
+    gf_msg(this->name, GF_LOG_INFO, 0, GD_MSG_GENERATED_UUID,
+           "generated UUID: %s", uuid_utoa(priv->uuid));
 
-        ret = glusterd_store_global_info (this);
+    ret = glusterd_store_global_info(this);
 
-        if (ret)
-                gf_log (this->name, GF_LOG_ERROR,
-                        "Unable to store the generated uuid %s",
-                        uuid_utoa (priv->uuid));
+    if (ret)
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_UUID_STORE_FAIL,
+               "Unable to store the generated uuid %s", uuid_utoa(priv->uuid));
 
-        return ret;
+    return ret;
 }
 
 int
-glusterd_options_init (xlator_t *this)
+glusterd_options_init(xlator_t *this)
 {
-        int             ret = -1;
-        glusterd_conf_t *priv = NULL;
-        char            *initial_version = "0";
-
-        priv = this->private;
-
-        priv->opts = dict_new ();
-        if (!priv->opts)
-                goto out;
-
-        ret = glusterd_store_retrieve_options (this);
-        if (ret == 0) {
-                goto out;
-        }
-
-        ret = dict_set_str (priv->opts, GLUSTERD_GLOBAL_OPT_VERSION,
-                            initial_version);
-        if (ret)
-                goto out;
+    int ret = -1;
+    glusterd_conf_t *priv = NULL;
+    char *initial_version = "0";
+
+    priv = this->private;
+
+    priv->opts = dict_new();
+    if (!priv->opts) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_CREATE_FAIL, NULL);
+        goto out;
+    }
+
+    ret = glusterd_store_retrieve_options(this);
+    if (ret == 0) {
+        goto out;
+    }
+
+    ret = dict_set_str(priv->opts, GLUSTERD_GLOBAL_OPT_VERSION,
+                       initial_version);
+    if (ret)
+        goto out;
+
+    ret = glusterd_store_options(this, priv->opts);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_VERS_STORE_FAIL,
+               "Unable to store version");
+        return ret;
+    }
+out:
+    return 0;
+}
 
-        ret = glusterd_store_options (this, priv->opts);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "Unable to store version");
-                return ret;
-        }
+int
+glusterd_client_statedump_submit_req(char *volname, char *target_ip, char *pid)
+{
+    gf_statedump statedump_req = {
+        0,
+    };
+    glusterd_conf_t *conf = NULL;
+    int ret = 0;
+    char *end_ptr = NULL;
+    rpc_transport_t *trans = NULL;
+    char *ip_addr = NULL;
+    xlator_t *this = NULL;
+    char tmp[UNIX_PATH_MAX] = {
+        0,
+    };
+
+    this = THIS;
+    GF_ASSERT(this);
+    conf = this->private;
+    GF_ASSERT(conf);
+
+    if (target_ip == NULL || pid == NULL) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_INVALID_ARGUMENT, NULL);
+        ret = -1;
+        goto out;
+    }
+
+    statedump_req.pid = strtol(pid, &end_ptr, 10);
+
+    gf_msg_debug(this->name, 0,
+                 "Performing statedump on volume %s "
+                 "client with pid:%d host:%s",
+                 volname, statedump_req.pid, target_ip);
+
+    pthread_mutex_lock(&conf->xprt_lock);
+    {
+        list_for_each_entry(trans, &conf->xprt_list, list)
+        {
+            /* check if this connection matches "all" or the
+             * volname */
+            if (strncmp(volname, "all", NAME_MAX) &&
+                strncmp(trans->peerinfo.volname, volname, NAME_MAX)) {
+                /* no match, try next trans */
+                continue;
+            }
+
+            strcpy(tmp, trans->peerinfo.identifier);
+            ip_addr = strtok(tmp, ":");
+            if (gf_is_same_address(ip_addr, target_ip)) {
+                /* Every gluster client would have
+                 * connected to glusterd(volfile server). This
+                 * connection is used to send the statedump
+                 * request rpc to the application.
+                 */
+                gf_msg_trace(this->name, 0,
+                             "Submitting "
+                             "statedump rpc request for %s",
+                             trans->peerinfo.identifier);
+                rpcsvc_request_submit(conf->rpc, trans, &glusterd_cbk_prog,
+                                      GF_CBK_STATEDUMP, &statedump_req,
+                                      this->ctx, (xdrproc_t)xdr_gf_statedump);
+            }
+        }
+    }
+    pthread_mutex_unlock(&conf->xprt_lock);
 out:
-        return 0;
+    return ret;
 }
 
 int
-glusterd_fetchspec_notify (xlator_t *this)
+glusterd_fetchspec_notify(xlator_t *this)
 {
-        int              ret   = -1;
-        glusterd_conf_t *priv  = NULL;
-        rpc_transport_t *trans = NULL;
+    int ret = -1;
+    glusterd_conf_t *priv = NULL;
+    rpc_transport_t *trans = NULL;
 
-        priv = this->private;
+    priv = this->private;
 
-        pthread_mutex_lock (&priv->xprt_lock);
+    pthread_mutex_lock(&priv->xprt_lock);
+    {
+        list_for_each_entry(trans, &priv->xprt_list, list)
         {
-                list_for_each_entry (trans, &priv->xprt_list, list) {
-                        rpcsvc_callback_submit (priv->rpc, trans,
-                                                &glusterd_cbk_prog,
-                                                GF_CBK_FETCHSPEC, NULL, 0);
-                }
+            rpcsvc_callback_submit(priv->rpc, trans, &glusterd_cbk_prog,
+                                   GF_CBK_FETCHSPEC, NULL, 0, NULL);
         }
-        pthread_mutex_unlock (&priv->xprt_lock);
+    }
+    pthread_mutex_unlock(&priv->xprt_lock);
 
-        ret = 0;
+    ret = 0;
 
-        return ret;
+    return ret;
 }
 
 int
-glusterd_fetchsnap_notify (xlator_t *this)
+glusterd_fetchsnap_notify(xlator_t *this)
 {
-        int              ret   = -1;
-        glusterd_conf_t *priv  = NULL;
-        rpc_transport_t *trans = NULL;
-
-        priv = this->private;
-
-        /*
-         * TODO: As of now, the identification of the rpc clients in the
-         * handshake protocol is not there. So among so many glusterfs processes
-         * registered with glusterd, it is hard to identify one particular
-         * process (in this particular case, the snap daemon). So the callback
-         * notification is sent to all the transports from the transport list.
-         * Only those processes which have a rpc client registered for this
-         * callback will respond to the notification. Once the identification
-         * of the rpc clients becomes possible, the below section can be changed
-         * to send callback notification to only those rpc clients, which have
-         * registered.
-         */
-        pthread_mutex_lock (&priv->xprt_lock);
+    int ret = -1;
+    glusterd_conf_t *priv = NULL;
+    rpc_transport_t *trans = NULL;
+
+    priv = this->private;
+
+    /*
+     * TODO: As of now, the identification of the rpc clients in the
+     * handshake protocol is not there. So among so many glusterfs processes
+     * registered with glusterd, it is hard to identify one particular
+     * process (in this particular case, the snap daemon). So the callback
+     * notification is sent to all the transports from the transport list.
+     * Only those processes which have a rpc client registered for this
+     * callback will respond to the notification. Once the identification
+     * of the rpc clients becomes possible, the below section can be changed
+     * to send callback notification to only those rpc clients, which have
+     * registered.
+     */
+    pthread_mutex_lock(&priv->xprt_lock);
+    {
+        list_for_each_entry(trans, &priv->xprt_list, list)
         {
-                list_for_each_entry (trans, &priv->xprt_list, list) {
-                        rpcsvc_callback_submit (priv->rpc, trans,
-                                                &glusterd_cbk_prog,
-                                                GF_CBK_GET_SNAPS, NULL, 0);
-                }
+            rpcsvc_callback_submit(priv->rpc, trans, &glusterd_cbk_prog,
+                                   GF_CBK_GET_SNAPS, NULL, 0, NULL);
         }
-        pthread_mutex_unlock (&priv->xprt_lock);
+    }
+    pthread_mutex_unlock(&priv->xprt_lock);
 
-        ret = 0;
+    ret = 0;
 
-        return ret;
+    return ret;
 }
 
-
 int32_t
-mem_acct_init (xlator_t *this)
+mem_acct_init(xlator_t *this)
 {
-        int     ret = -1;
+    int ret = -1;
 
-        if (!this)
-                return ret;
+    if (!this)
+        return ret;
 
-        ret = xlator_mem_acct_init (this, gf_gld_mt_end + 1);
-
-        if (ret != 0) {
-                gf_log (this->name, GF_LOG_ERROR, "Memory accounting init"
-                        " failed");
-                return ret;
-        }
+    ret = xlator_mem_acct_init(this, gf_gld_mt_end + 1);
 
+    if (ret != 0) {
+        gf_msg(this->name, GF_LOG_ERROR, ENOMEM, GD_MSG_NO_MEMORY,
+               "Memory accounting init"
+               " failed");
         return ret;
+    }
+
+    return ret;
 }
 
 int
-glusterd_rpcsvc_notify (rpcsvc_t *rpc, void *xl, rpcsvc_event_t event,
-                     void *data)
+glusterd_rpcsvc_notify(rpcsvc_t *rpc, void *xl, rpcsvc_event_t event,
+                       void *data)
 {
-        xlator_t            *this = NULL;
-        rpc_transport_t     *xprt = NULL;
-        glusterd_conf_t     *priv = NULL;
-
-        if (!xl || !data) {
-                gf_log ("glusterd", GF_LOG_WARNING,
-                        "Calling rpc_notify without initializing");
-                goto out;
-        }
-
-        this = xl;
-        xprt = data;
-
-        priv = this->private;
-
-        switch (event) {
-        case RPCSVC_EVENT_ACCEPT:
-        {
-
-                pthread_mutex_lock (&priv->xprt_lock);
-                list_add_tail (&xprt->list, &priv->xprt_list);
-                pthread_mutex_unlock (&priv->xprt_lock);
-                break;
-        }
-        case RPCSVC_EVENT_DISCONNECT:
-        {
-                pthread_mutex_lock (&priv->xprt_lock);
-                list_del (&xprt->list);
-                pthread_mutex_unlock (&priv->xprt_lock);
-                pmap_registry_remove (this, 0, NULL, GF_PMAP_PORT_NONE, xprt);
+    xlator_t *this = NULL;
+    rpc_transport_t *xprt = NULL;
+    glusterd_conf_t *priv = NULL;
+
+    if (!xl || !data) {
+        gf_msg("glusterd", GF_LOG_WARNING, 0, GD_MSG_NO_INIT,
+               "Calling rpc_notify without initializing");
+        goto out;
+    }
+
+    this = xl;
+    xprt = data;
+
+    priv = this->private;
+
+    switch (event) {
+        case RPCSVC_EVENT_ACCEPT: {
+            pthread_mutex_lock(&priv->xprt_lock);
+            list_add_tail(&xprt->list, &priv->xprt_list);
+            pthread_mutex_unlock(&priv->xprt_lock);
+            break;
+        }
+        case RPCSVC_EVENT_DISCONNECT: {
+            /* A DISCONNECT event could come without an ACCEPT event
+             * happening for this transport. This happens when the server is
+             * expecting encrypted connections by the client tries to
+             * connect unecnrypted
+             */
+            if (list_empty(&xprt->list))
                 break;
+
+            pthread_mutex_lock(&priv->xprt_lock);
+            list_del(&xprt->list);
+            pthread_mutex_unlock(&priv->xprt_lock);
+            pmap_registry_remove(this, 0, NULL, GF_PMAP_PORT_ANY, xprt,
+                                 _gf_false);
+            break;
         }
 
         default:
-                break;
-        }
+            break;
+    }
 
 out:
-        return 0;
+    return 0;
 }
 
-
-static inline int32_t
-glusterd_program_register (xlator_t *this, rpcsvc_t *svc,
-                           rpcsvc_program_t *prog)
+static int32_t
+glusterd_program_register(xlator_t *this, rpcsvc_t *svc, rpcsvc_program_t *prog)
 {
-        int32_t ret = -1;
+    int32_t ret = -1;
 
-        ret = rpcsvc_program_register (svc, prog);
-        if (ret) {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "cannot register program (name: %s, prognum:%d, "
-                        "progver:%d)", prog->progname, prog->prognum,
-                        prog->progver);
-                goto out;
-        }
+    ret = rpcsvc_program_register(svc, prog, _gf_false);
+    if (ret) {
+        gf_msg_debug(this->name, 0,
+                     "cannot register program (name: %s, prognum:%d, "
+                     "progver:%d)",
+                     prog->progname, prog->prognum, prog->progver);
+        goto out;
+    }
 
 out:
-        return ret;
+    return ret;
 }
 
 int
-glusterd_rpcsvc_options_build (dict_t *options)
+glusterd_rpcsvc_options_build(dict_t *options)
 {
-        int             ret = 0;
-        uint32_t        backlog = 0;
+    int ret = 0;
+    uint32_t backlog = 0;
+    xlator_t *this = THIS;
+    GF_ASSERT(this);
 
-        ret = dict_get_uint32 (options, "transport.socket.listen-backlog",
-                               &backlog);
+    ret = dict_get_uint32(options, "transport.listen-backlog", &backlog);
 
+    if (ret) {
+        backlog = GLUSTERFS_SOCKET_LISTEN_BACKLOG;
+        ret = dict_set_uint32(options, "transport.listen-backlog", backlog);
         if (ret) {
-                backlog = GLUSTERD_SOCKET_LISTEN_BACKLOG;
-                ret = dict_set_uint32 (options,
-                                      "transport.socket.listen-backlog",
-                                      backlog);
-                if (ret)
-                        goto out;
+            gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_SET_FAILED,
+                    "Key=transport.listen-backlog", NULL);
+            goto out;
         }
+    }
 
-        gf_log ("", GF_LOG_DEBUG, "listen-backlog value: %d", backlog);
+    gf_msg_debug("glusterd", 0, "listen-backlog value: %d", backlog);
 
 out:
-        return ret;
+    return ret;
 }
 
 #if SYNCDAEMON_COMPILE
 static int
-glusterd_check_gsync_present (int *valid_state)
+glusterd_check_gsync_present(int *valid_state)
 {
-        char                buff[PATH_MAX] = {0, };
-        runner_t            runner = {0,};
-        char               *ptr = NULL;
-        int                 ret = 0;
-
-        runinit (&runner);
-        runner_add_args (&runner, GSYNCD_PREFIX"/gsyncd", "--version", NULL);
-        runner_redir (&runner, STDOUT_FILENO, RUN_PIPE);
-        ret = runner_start (&runner);
-        if (ret == -1) {
-                if (errno == ENOENT) {
-                        gf_log ("glusterd", GF_LOG_INFO, GEOREP
-                                 " module not installed in the system");
-                        *valid_state = 0;
-                }
-                else {
-                        gf_log ("glusterd", GF_LOG_ERROR, GEOREP
-                                  " module not working as desired");
-                        *valid_state = -1;
-                }
-                goto out;
-        }
-
-        ptr = fgets(buff, sizeof(buff), runner_chio (&runner, STDOUT_FILENO));
-        if (ptr) {
-                if (!strstr (buff, "gsyncd")) {
-                        ret = -1;
-                        gf_log ("glusterd", GF_LOG_ERROR, GEOREP" module not "
-                                 "working as desired");
-                        *valid_state = -1;
-                        goto out;
-                }
+    char buff[PATH_MAX] = {
+        0,
+    };
+    runner_t runner = {
+        0,
+    };
+    char *ptr = NULL;
+    int ret = 0;
+
+    runinit(&runner);
+    runner_add_args(&runner, GSYNCD_PREFIX "/gsyncd", "--version", NULL);
+    runner_redir(&runner, STDOUT_FILENO, RUN_PIPE);
+    ret = runner_start(&runner);
+    if (ret == -1) {
+        if (errno == ENOENT) {
+            gf_msg("glusterd", GF_LOG_INFO, errno, GD_MSG_MODULE_NOT_INSTALLED,
+                   GEOREP " module not installed in the system");
+            *valid_state = 0;
         } else {
-                ret = -1;
-                gf_log ("glusterd", GF_LOG_ERROR, GEOREP" module not "
-                         "working as desired");
-                *valid_state = -1;
-                goto out;
-        }
-
-        ret = 0;
- out:
-
-        runner_end (&runner);
+            gf_msg("glusterd", GF_LOG_ERROR, errno, GD_MSG_MODULE_NOT_WORKING,
+                   GEOREP " module not working as desired");
+            *valid_state = -1;
+        }
+        goto out;
+    }
+
+    ptr = fgets(buff, sizeof(buff), runner_chio(&runner, STDOUT_FILENO));
+    if (ptr) {
+        if (!strstr(buff, "gsyncd")) {
+            ret = -1;
+            gf_msg("glusterd", GF_LOG_ERROR, 0, GD_MSG_MODULE_NOT_WORKING,
+                   GEOREP
+                   " module not "
+                   "working as desired");
+            *valid_state = -1;
+            goto out;
+        }
+    } else {
+        ret = -1;
+        gf_msg("glusterd", GF_LOG_ERROR, 0, GD_MSG_MODULE_NOT_WORKING,
+               GEOREP
+               " module not "
+               "working as desired");
+        *valid_state = -1;
+        goto out;
+    }
+
+    ret = 0;
+out:
 
-        gf_log ("glusterd", GF_LOG_DEBUG, "Returning %d", ret);
-        return ret;
+    runner_end(&runner);
 
+    gf_msg_debug("glusterd", 0, "Returning %d", ret);
+    return ret;
 }
 
 static int
-group_write_allow (char *path, gid_t gid)
+group_write_allow(char *path, gid_t gid)
 {
-        struct stat st = {0,};
-        int ret        = 0;
+    struct stat st = {
+        0,
+    };
+    int ret = 0;
 
-        ret = stat (path, &st);
-        if (ret == -1)
-                goto out;
-        GF_ASSERT (S_ISDIR (st.st_mode));
+    ret = sys_stat(path, &st);
+    if (ret == -1)
+        goto out;
+    GF_ASSERT(S_ISDIR(st.st_mode));
 
-        ret = chown (path, -1, gid);
-        if (ret == -1)
-                goto out;
+    ret = sys_chown(path, -1, gid);
+    if (ret == -1)
+        goto out;
 
-        ret = chmod (path, (st.st_mode & ~S_IFMT) | S_IWGRP|S_IXGRP|S_ISVTX);
+    ret = sys_chmod(path, (st.st_mode & ~S_IFMT) | S_IWGRP | S_IXGRP | S_ISVTX);
 
- out:
-        if (ret == -1)
-                gf_log ("", GF_LOG_CRITICAL,
-                        "failed to set up write access to %s for group %d (%s)",
-                        path, gid, strerror (errno));
-        return ret;
+out:
+    if (ret == -1)
+        gf_msg("glusterd", GF_LOG_CRITICAL, errno,
+               GD_MSG_WRITE_ACCESS_GRANT_FAIL,
+               "failed to set up write access to %s for group %d (%s)", path,
+               gid, strerror(errno));
+    return ret;
 }
 
 static int
-glusterd_crt_georep_folders (char *georepdir, glusterd_conf_t *conf)
+glusterd_crt_georep_folders(char *georepdir, glusterd_conf_t *conf)
 {
-        char *greplg_s   = NULL;
-        struct group *gr = NULL;
-        int ret          = 0;
-
-        GF_ASSERT (georepdir);
-        GF_ASSERT (conf);
-
-        if (strlen (conf->workdir)+2 > PATH_MAX-strlen(GEOREP)) {
-                ret = -1;
-                gf_log ("glusterd", GF_LOG_CRITICAL,
-                        "directory path %s/"GEOREP" is longer than PATH_MAX",
-                        conf->workdir);
-                goto out;
-        }
-
-        snprintf (georepdir, PATH_MAX, "%s/"GEOREP, conf->workdir);
-        ret = mkdir_p (georepdir, 0777, _gf_true);
-        if (-1 == ret) {
-                gf_log ("glusterd", GF_LOG_CRITICAL,
-                        "Unable to create "GEOREP" directory %s",
-                        georepdir);
-                goto out;
-        }
-
-        if (strlen (DEFAULT_LOG_FILE_DIRECTORY"/"GEOREP) >= PATH_MAX) {
-                ret = -1;
-                gf_log ("glusterd", GF_LOG_CRITICAL,
-                        "directory path "DEFAULT_LOG_FILE_DIRECTORY"/"
-                        GEOREP" is longer than PATH_MAX");
-                goto out;
-        }
-        ret = mkdir_p (DEFAULT_LOG_FILE_DIRECTORY"/"GEOREP, 0777, _gf_true);
-        if (-1 == ret) {
-                gf_log ("glusterd", GF_LOG_CRITICAL,
-                        "Unable to create "GEOREP" log directory");
-                goto out;
-        }
-
-        /* Slave log file directory */
-        if (strlen(DEFAULT_LOG_FILE_DIRECTORY"/"GEOREP"-slaves") >= PATH_MAX) {
-                ret = -1;
-                gf_log ("glusterd", GF_LOG_CRITICAL,
-                        "directory path "DEFAULT_LOG_FILE_DIRECTORY"/"
-                        GEOREP"-slaves"" is longer than PATH_MAX");
-                goto out;
-        }
-        ret = mkdir_p (DEFAULT_LOG_FILE_DIRECTORY"/"GEOREP"-slaves", 0777,
-                      _gf_true);
-        if (-1 == ret) {
-                gf_log ("glusterd", GF_LOG_CRITICAL,
-                        "Unable to create "GEOREP" slave log directory");
-                goto out;
-        }
-
-        /* MountBroker log file directory */
-        if (strlen(DEFAULT_LOG_FILE_DIRECTORY"/"GEOREP"-slaves/mbr") >= PATH_MAX) {
-                ret = -1;
-                gf_log ("glusterd", GF_LOG_CRITICAL,
-                        "directory path "DEFAULT_LOG_FILE_DIRECTORY"/"GEOREP
-                        "-slaves/mbr"" is longer than PATH_MAX");
-                goto out;
-        }
-        ret = mkdir_p (DEFAULT_LOG_FILE_DIRECTORY"/"GEOREP"-slaves/mbr", 0777,
-                      _gf_true);
-        if (-1 == ret) {
-                gf_log ("glusterd", GF_LOG_CRITICAL,
-                        "Unable to create "GEOREP" mountbroker slave log directory");
-                goto out;
-        }
-
-        ret = dict_get_str (THIS->options, GEOREP"-log-group", &greplg_s);
-        if (ret)
-                ret = 0;
-        else {
-                gr = getgrnam (greplg_s);
-                if (!gr) {
-                        gf_log ("glusterd", GF_LOG_CRITICAL,
-                                "group "GEOREP"-log-group %s does not exist", greplg_s);
-                        ret = -1;
-                        goto out;
-                }
-
-                ret = group_write_allow (DEFAULT_LOG_FILE_DIRECTORY"/"GEOREP,
-                                         gr->gr_gid);
-                if (ret == 0)
-                        ret = group_write_allow (DEFAULT_LOG_FILE_DIRECTORY"/"
-                                                 GEOREP"-slaves", gr->gr_gid);
-                if (ret == 0)
-                        ret = group_write_allow (DEFAULT_LOG_FILE_DIRECTORY"/"
-                                                 GEOREP"-slaves/mbr", gr->gr_gid);
-        }
-
- out:
-        gf_log("", GF_LOG_DEBUG, "Returning %d", ret);
-        return ret;
+    char *greplg_s = NULL;
+    struct group *gr = NULL;
+    int ret = 0;
+    int gr_ret = 0;
+    int32_t len = 0;
+    char logdir[PATH_MAX] = {0};
+
+    GF_ASSERT(georepdir);
+    GF_ASSERT(conf);
+
+    if (strlen(conf->workdir) + 2 > PATH_MAX - SLEN(GEOREP)) {
+        ret = -1;
+        gf_msg("glusterd", GF_LOG_CRITICAL, 0, GD_MSG_DIRPATH_TOO_LONG,
+               "directory path %s/" GEOREP " is longer than PATH_MAX",
+               conf->workdir);
+        goto out;
+    }
+
+    len = snprintf(georepdir, PATH_MAX, "%s/" GEOREP, conf->workdir);
+    if ((len < 0) || (len >= PATH_MAX)) {
+        gf_smsg("glusterd", GF_LOG_ERROR, errno, GD_MSG_COPY_FAIL, NULL);
+        ret = -1;
+        goto out;
+    }
+    ret = mkdir_p(georepdir, 0755, _gf_true);
+    if (-1 == ret) {
+        gf_msg("glusterd", GF_LOG_CRITICAL, errno, GD_MSG_CREATE_DIR_FAILED,
+               "Unable to create " GEOREP " directory %s", georepdir);
+        goto out;
+    }
+
+    ret = dict_get_str(THIS->options, GEOREP "-log-group", &greplg_s);
+    if (ret) {
+        gf_smsg("glusterd", GF_LOG_ERROR, errno, GD_MSG_DICT_GET_FAILED,
+                "Key=log-group", NULL);
+        ret = 0;
+    } else {
+        gr = getgrnam(greplg_s);
+        if (!gr) {
+            gf_msg("glusterd", GF_LOG_CRITICAL, 0, GD_MSG_LOGGROUP_INVALID,
+                   "group " GEOREP "-log-group %s does not exist", greplg_s);
+            gr_ret = -1;
+        }
+    }
+    if ((strlen(conf->logdir) + 2 + SLEN(GEOREP)) >= PATH_MAX) {
+        ret = -1;
+        gf_msg("glusterd", GF_LOG_CRITICAL, 0, GD_MSG_DIRPATH_TOO_LONG,
+               "directory path %s/" GEOREP " is longer than PATH_MAX",
+               conf->logdir);
+        goto out;
+    }
+    len = snprintf(logdir, PATH_MAX, "%s/" GEOREP, conf->logdir);
+    if ((len < 0) || (len >= PATH_MAX)) {
+        ret = -1;
+        goto out;
+    }
+    ret = mkdir_p(logdir, 0755, _gf_true);
+    if (-1 == ret) {
+        gf_msg("glusterd", GF_LOG_CRITICAL, errno, GD_MSG_CREATE_DIR_FAILED,
+               "Unable to create " GEOREP " log directory");
+        goto out;
+    }
+    if (gr) {
+        gr_ret = group_write_allow(logdir, gr->gr_gid);
+    }
+
+    if ((strlen(conf->logdir) + 2 + SLEN(GEOREP "-slaves")) >= PATH_MAX) {
+        ret = -1;
+        gf_msg("glusterd", GF_LOG_CRITICAL, 0, GD_MSG_DIRPATH_TOO_LONG,
+               "directory path  %s/" GEOREP
+               "-slaves"
+               " is longer than PATH_MAX",
+               conf->logdir);
+        goto out;
+    }
+    len = snprintf(logdir, PATH_MAX, "%s/" GEOREP "-slaves", conf->logdir);
+    if ((len < 0) || (len >= PATH_MAX)) {
+        gf_smsg("glusterd", GF_LOG_ERROR, errno, GD_MSG_COPY_FAIL, NULL);
+        ret = -1;
+        goto out;
+    }
+    ret = mkdir_p(logdir, 0755, _gf_true);
+    if (-1 == ret) {
+        gf_msg("glusterd", GF_LOG_CRITICAL, errno, GD_MSG_CREATE_DIR_FAILED,
+               "Unable to create " GEOREP " slave log directory");
+        goto out;
+    }
+    if (gr && !gr_ret) {
+        gr_ret = group_write_allow(logdir, gr->gr_gid);
+    }
+
+    /* MountBroker log file directory */
+    if ((strlen(conf->logdir) + 2 + SLEN(GEOREP "-slaves/mbr")) >= PATH_MAX) {
+        ret = -1;
+        gf_msg("glusterd", GF_LOG_CRITICAL, 0, GD_MSG_DIRPATH_TOO_LONG,
+               "directory path  %s/" GEOREP
+               "-slaves/mbr"
+               " is longer than PATH_MAX",
+               conf->logdir);
+        goto out;
+    }
+
+    len = snprintf(logdir, PATH_MAX, "%s/" GEOREP "-slaves/mbr", conf->logdir);
+    if ((len < 0) || (len >= PATH_MAX)) {
+        gf_smsg("glusterd", GF_LOG_ERROR, errno, GD_MSG_COPY_FAIL, NULL);
+        ret = -1;
+        goto out;
+    }
+
+    ret = mkdir_p(logdir, 0755, _gf_true);
+    if (-1 == ret) {
+        gf_msg("glusterd", GF_LOG_CRITICAL, errno, GD_MSG_CREATE_DIR_FAILED,
+               "Unable to create " GEOREP " mountbroker slave log directory");
+        goto out;
+    }
+    if (gr && !gr_ret) {
+        gr_ret = group_write_allow(logdir, gr->gr_gid);
+    }
+    if (gr_ret)
+        ret = gr_ret;
+out:
+    gf_msg_debug("glusterd", 0, "Returning %d", ret);
+    return ret;
 }
 
 static void
-runinit_gsyncd_setrx (runner_t *runner, glusterd_conf_t *conf)
+runinit_gsyncd_setrx(runner_t *runner, glusterd_conf_t *conf)
 {
-        runinit (runner);
-        runner_add_args (runner, GSYNCD_PREFIX"/gsyncd", "-c", NULL);
-        runner_argprintf (runner, "%s/"GSYNC_CONF_TEMPLATE, conf->workdir);
-        runner_add_arg (runner, "--config-set-rx");
+    runinit(runner);
+    runner_add_args(runner, GSYNCD_PREFIX "/gsyncd", "-c", NULL);
+    runner_argprintf(runner, "%s/" GSYNC_CONF_TEMPLATE, conf->workdir);
+    runner_add_arg(runner, "--config-set-rx");
 }
 
 static int
-configure_syncdaemon (glusterd_conf_t *conf)
-#define RUN_GSYNCD_CMD do {                                                          \
-        ret = runner_run_reuse (&runner);                                            \
-        if (ret == -1) {                                                             \
-                runner_log (&runner, "glusterd", GF_LOG_ERROR, "command failed");    \
-                runner_end (&runner);                                                \
-                goto out;                                                            \
-        }                                                                            \
-        runner_end (&runner);                                                        \
-} while (0)
+configure_syncdaemon(glusterd_conf_t *conf)
+#define RUN_GSYNCD_CMD                                                         \
+    do {                                                                       \
+        ret = runner_run_reuse(&runner);                                       \
+        if (ret == -1) {                                                       \
+            runner_log(&runner, "glusterd", GF_LOG_ERROR, "command failed");   \
+            runner_end(&runner);                                               \
+            goto out;                                                          \
+        }                                                                      \
+        runner_end(&runner);                                                   \
+    } while (0)
 {
-        int ret = 0;
-        runner_t runner = {0,};
-        char georepdir[PATH_MAX] = {0,};
-        int valid_state = 0;
-
-        ret = setenv ("_GLUSTERD_CALLED_", "1", 1);
-        if (ret < 0) {
-                ret = 0;
-                goto out;
-        }
-        valid_state = -1;
-        ret = glusterd_check_gsync_present (&valid_state);
-        if (-1 == ret) {
-                ret = valid_state;
-                goto out;
-        }
-
-        glusterd_crt_georep_folders (georepdir, conf);
-        if (ret) {
-                ret = 0;
-                goto out;
-        }
+    int ret = 0;
+    runner_t runner = {
+        0,
+    };
+    char georepdir[PATH_MAX] = {
+        0,
+    };
+    int valid_state = 0;
+
+    ret = setenv("_GLUSTERD_CALLED_", "1", 1);
+    if (ret < 0) {
+        ret = 0;
+        goto out;
+    }
+    valid_state = -1;
+    ret = glusterd_check_gsync_present(&valid_state);
+    if (-1 == ret) {
+        ret = valid_state;
+        goto out;
+    }
+
+    glusterd_crt_georep_folders(georepdir, conf);
+    if (ret) {
+        ret = 0;
+        goto out;
+    }
+
+    /************
+     * master pre-configuration
+     ************/
+
+    /* remote-gsyncd */
+    runinit_gsyncd_setrx(&runner, conf);
+    runner_add_args(&runner, "remote-gsyncd", GSYNCD_PREFIX "/gsyncd", ".", ".",
+                    NULL);
+    RUN_GSYNCD_CMD;
+
+    runinit_gsyncd_setrx(&runner, conf);
+    runner_add_args(&runner, "remote-gsyncd", "/nonexistent/gsyncd", ".",
+                    "^ssh:", NULL);
+    RUN_GSYNCD_CMD;
+
+    /* gluster-command-dir */
+    runinit_gsyncd_setrx(&runner, conf);
+    runner_add_args(&runner, "gluster-command-dir", SBIN_DIR "/", ".", ".",
+                    NULL);
+    RUN_GSYNCD_CMD;
+
+    /* gluster-params */
+    runinit_gsyncd_setrx(&runner, conf);
+    runner_add_args(&runner, "gluster-params", "aux-gfid-mount acl", ".", ".",
+                    NULL);
+    RUN_GSYNCD_CMD;
+
+    /* ssh-command */
+    runinit_gsyncd_setrx(&runner, conf);
+    runner_add_arg(&runner, "ssh-command");
+    runner_argprintf(&runner,
+                     "ssh -oPasswordAuthentication=no "
+                     "-oStrictHostKeyChecking=no "
+                     "-i %s/secret.pem",
+                     georepdir);
+    runner_add_args(&runner, ".", ".", NULL);
+    RUN_GSYNCD_CMD;
+
+    /* ssh-command tar */
+    runinit_gsyncd_setrx(&runner, conf);
+    runner_add_arg(&runner, "ssh-command-tar");
+    runner_argprintf(&runner,
+                     "ssh -oPasswordAuthentication=no "
+                     "-oStrictHostKeyChecking=no "
+                     "-i %s/tar_ssh.pem",
+                     georepdir);
+    runner_add_args(&runner, ".", ".", NULL);
+    RUN_GSYNCD_CMD;
+
+    /* pid-file */
+    runinit_gsyncd_setrx(&runner, conf);
+    runner_add_arg(&runner, "pid-file");
+    runner_argprintf(&runner,
+                     "%s/${mastervol}_${remotehost}_${slavevol}/monitor.pid",
+                     georepdir);
+    runner_add_args(&runner, ".", ".", NULL);
+    RUN_GSYNCD_CMD;
+
+    /* geo-rep working dir */
+    runinit_gsyncd_setrx(&runner, conf);
+    runner_add_arg(&runner, "georep-session-working-dir");
+    runner_argprintf(&runner, "%s/${mastervol}_${remotehost}_${slavevol}/",
+                     georepdir);
+    runner_add_args(&runner, ".", ".", NULL);
+    RUN_GSYNCD_CMD;
+
+    /* state-file */
+    runinit_gsyncd_setrx(&runner, conf);
+    runner_add_arg(&runner, "state-file");
+    runner_argprintf(&runner,
+                     "%s/${mastervol}_${remotehost}_${slavevol}/monitor.status",
+                     georepdir);
+    runner_add_args(&runner, ".", ".", NULL);
+    RUN_GSYNCD_CMD;
+
+    /* state-detail-file */
+    runinit_gsyncd_setrx(&runner, conf);
+    runner_add_arg(&runner, "state-detail-file");
+    runner_argprintf(
+        &runner,
+        "%s/${mastervol}_${remotehost}_${slavevol}/${eSlave}-detail.status",
+        georepdir);
+    runner_add_args(&runner, ".", ".", NULL);
+    RUN_GSYNCD_CMD;
+
+    /* state-detail-file */
+    runinit_gsyncd_setrx(&runner, conf);
+    runner_add_arg(&runner, "state-detail-file");
+    runner_argprintf(
+        &runner,
+        "%s/${mastervol}_${remotehost}_${slavevol}/${eSlave}-detail.status",
+        georepdir);
+    runner_add_args(&runner, ".", ".", NULL);
+    RUN_GSYNCD_CMD;
+
+    /* state-socket */
+    runinit_gsyncd_setrx(&runner, conf);
+    runner_add_arg(&runner, "state-socket-unencoded");
+    runner_argprintf(&runner, "%s/${mastervol}/${eSlave}.socket", georepdir);
+    runner_add_args(&runner, ".", ".", NULL);
+    RUN_GSYNCD_CMD;
+
+    /* socketdir */
+    runinit_gsyncd_setrx(&runner, conf);
+    runner_add_args(&runner, "socketdir", GLUSTERD_SOCK_DIR, ".", ".", NULL);
+    RUN_GSYNCD_CMD;
+
+    /* log-file */
+    runinit_gsyncd_setrx(&runner, conf);
+    runner_add_arg(&runner, "log-file");
+    runner_argprintf(&runner, "%s/" GEOREP "/${mastervol}/${eSlave}.log",
+                     conf->logdir);
+    runner_add_args(&runner, ".", ".", NULL);
+    RUN_GSYNCD_CMD;
+
+    /* gluster-log-file */
+    runinit_gsyncd_setrx(&runner, conf);
+    runner_add_arg(&runner, "gluster-log-file");
+    runner_argprintf(
+        &runner, "%s/" GEOREP "/${mastervol}/${eSlave}${local_id}.gluster.log",
+        conf->logdir);
+    runner_add_args(&runner, ".", ".", NULL);
+    RUN_GSYNCD_CMD;
+
+    /* ignore-deletes */
+    runinit_gsyncd_setrx(&runner, conf);
+    runner_add_args(&runner, "ignore-deletes", "true", ".", ".", NULL);
+    RUN_GSYNCD_CMD;
+
+    /* special-sync-mode */
+    runinit_gsyncd_setrx(&runner, conf);
+    runner_add_args(&runner, "special-sync-mode", "partial", ".", ".", NULL);
+    RUN_GSYNCD_CMD;
+
+    /* change-detector == changelog */
+    runinit_gsyncd_setrx(&runner, conf);
+    runner_add_args(&runner, "change-detector", "changelog", ".", ".", NULL);
+    RUN_GSYNCD_CMD;
+
+    runinit_gsyncd_setrx(&runner, conf);
+    runner_add_arg(&runner, "working-dir");
+    runner_argprintf(&runner, "%s/${mastervol}/${eSlave}",
+                     DEFAULT_VAR_RUN_DIRECTORY);
+    runner_add_args(&runner, ".", ".", NULL);
+    RUN_GSYNCD_CMD;
+
+    /************
+     * slave pre-configuration
+     ************/
+
+    /* slave-gluster-command-dir */
+    runinit_gsyncd_setrx(&runner, conf);
+    runner_add_args(&runner, "slave-gluster-command-dir", SBIN_DIR "/", ".",
+                    NULL);
+    RUN_GSYNCD_CMD;
+
+    /* gluster-params */
+    runinit_gsyncd_setrx(&runner, conf);
+    runner_add_args(&runner, "gluster-params", "aux-gfid-mount acl", ".", NULL);
+    RUN_GSYNCD_CMD;
+
+    /* log-file */
+    runinit_gsyncd_setrx(&runner, conf);
+    runner_add_arg(&runner, "log-file");
+    runner_argprintf(
+        &runner,
+        "%s/" GEOREP
+        "-slaves/${session_owner}:${local_node}${local_id}.${slavevol}.log",
+        conf->logdir);
+    runner_add_args(&runner, ".", ".", NULL);
+    RUN_GSYNCD_CMD;
+
+    /* MountBroker log-file */
+    runinit_gsyncd_setrx(&runner, conf);
+    runner_add_arg(&runner, "log-file-mbr");
+    runner_argprintf(
+        &runner,
+        "%s/" GEOREP
+        "-slaves/mbr/${session_owner}:${local_node}${local_id}.${slavevol}.log",
+        conf->logdir);
+    runner_add_args(&runner, ".", ".", NULL);
+    RUN_GSYNCD_CMD;
+
+    /* gluster-log-file */
+    runinit_gsyncd_setrx(&runner, conf);
+    runner_add_arg(&runner, "gluster-log-file");
+    runner_argprintf(
+        &runner,
+        "%s/" GEOREP
+        "-slaves/"
+        "${session_owner}:${local_node}${local_id}.${slavevol}.gluster.log",
+        conf->logdir);
+    runner_add_args(&runner, ".", ".", NULL);
+    RUN_GSYNCD_CMD;
 
-        /************
-         * master pre-configuration
-         ************/
-
-        /* remote-gsyncd */
-        runinit_gsyncd_setrx (&runner, conf);
-        runner_add_args (&runner, "remote-gsyncd", GSYNCD_PREFIX"/gsyncd", ".", ".", NULL);
-        RUN_GSYNCD_CMD;
-
-        runinit_gsyncd_setrx (&runner, conf);
-        runner_add_args (&runner, "remote-gsyncd", "/nonexistent/gsyncd",
-                         ".", "^ssh:", NULL);
-        RUN_GSYNCD_CMD;
-
-        /* gluster-command-dir */
-        runinit_gsyncd_setrx (&runner, conf);
-        runner_add_args (&runner, "gluster-command-dir", SBIN_DIR"/",
-                         ".", ".", NULL);
-        RUN_GSYNCD_CMD;
-
-        /* gluster-params */
-        runinit_gsyncd_setrx (&runner, conf);
-        runner_add_args (&runner, "gluster-params",
-                         "aux-gfid-mount acl",
-                         ".", ".", NULL);
-        RUN_GSYNCD_CMD;
-
-        /* ssh-command */
-        runinit_gsyncd_setrx (&runner, conf);
-        runner_add_arg (&runner, "ssh-command");
-        runner_argprintf (&runner,
-                          "ssh -oPasswordAuthentication=no "
-                           "-oStrictHostKeyChecking=no "
-                           "-i %s/secret.pem", georepdir);
-        runner_add_args (&runner, ".", ".", NULL);
-        RUN_GSYNCD_CMD;
-
-        /* ssh-command tar */
-        runinit_gsyncd_setrx (&runner, conf);
-        runner_add_arg (&runner, "ssh-command-tar");
-        runner_argprintf (&runner,
-                          "ssh -oPasswordAuthentication=no "
-                           "-oStrictHostKeyChecking=no "
-                           "-i %s/tar_ssh.pem", georepdir);
-        runner_add_args (&runner, ".", ".", NULL);
-        RUN_GSYNCD_CMD;
-
-        /* pid-file */
-        runinit_gsyncd_setrx (&runner, conf);
-        runner_add_arg (&runner, "pid-file");
-        runner_argprintf (&runner, "%s/${mastervol}_${remotehost}_${slavevol}/${eSlave}.pid", georepdir);
-        runner_add_args (&runner, ".", ".", NULL);
-        RUN_GSYNCD_CMD;
-
-        /* geo-rep working dir */
-        runinit_gsyncd_setrx (&runner, conf);
-        runner_add_arg (&runner, "georep-session-working-dir");
-        runner_argprintf (&runner, "%s/${mastervol}_${remotehost}_${slavevol}/", georepdir);
-        runner_add_args (&runner, ".", ".", NULL);
-        RUN_GSYNCD_CMD;
-
-        /* state-file */
-        runinit_gsyncd_setrx (&runner, conf);
-        runner_add_arg (&runner, "state-file");
-        runner_argprintf (&runner, "%s/${mastervol}_${remotehost}_${slavevol}/${eSlave}.status", georepdir);
-        runner_add_args (&runner, ".", ".", NULL);
-        RUN_GSYNCD_CMD;
-
-        /* state-detail-file */
-        runinit_gsyncd_setrx (&runner, conf);
-        runner_add_arg (&runner, "state-detail-file");
-        runner_argprintf (&runner, "%s/${mastervol}_${remotehost}_${slavevol}/${eSlave}-detail.status",
-                          georepdir);
-        runner_add_args (&runner, ".", ".", NULL);
-        RUN_GSYNCD_CMD;
-
-        /* state-detail-file */
-        runinit_gsyncd_setrx (&runner, conf);
-        runner_add_arg (&runner, "state-detail-file");
-        runner_argprintf (&runner, "%s/${mastervol}_${remotehost}_${slavevol}/${eSlave}-detail.status",
-                          georepdir);
-        runner_add_args (&runner, ".", ".", NULL);
-        RUN_GSYNCD_CMD;
-
-        /* state-socket */
-        runinit_gsyncd_setrx (&runner, conf);
-        runner_add_arg (&runner, "state-socket-unencoded");
-        runner_argprintf (&runner, "%s/${mastervol}/${eSlave}.socket", georepdir);
-        runner_add_args (&runner, ".", ".", NULL);
-        RUN_GSYNCD_CMD;
-
-        /* socketdir */
-        runinit_gsyncd_setrx (&runner, conf);
-        runner_add_args (&runner, "socketdir", GLUSTERD_SOCK_DIR, ".", ".", NULL);
-        RUN_GSYNCD_CMD;
-
-        /* log-file */
-        runinit_gsyncd_setrx (&runner, conf);
-        runner_add_args (&runner,
-                         "log-file",
-                         DEFAULT_LOG_FILE_DIRECTORY"/"GEOREP"/${mastervol}/${eSlave}.log",
-                         ".", ".", NULL);
-        RUN_GSYNCD_CMD;
-
-        /* gluster-log-file */
-        runinit_gsyncd_setrx (&runner, conf);
-        runner_add_args (&runner,
-                         "gluster-log-file",
-                         DEFAULT_LOG_FILE_DIRECTORY"/"GEOREP"/${mastervol}/${eSlave}${local_id}.gluster.log",
-                         ".", ".", NULL);
-        RUN_GSYNCD_CMD;
-
-        /* ignore-deletes */
-        runinit_gsyncd_setrx (&runner, conf);
-        runner_add_args (&runner, "ignore-deletes", "true", ".", ".", NULL);
-        RUN_GSYNCD_CMD;
-
-        /* special-sync-mode */
-        runinit_gsyncd_setrx (&runner, conf);
-        runner_add_args (&runner, "special-sync-mode", "partial", ".", ".", NULL);
-        RUN_GSYNCD_CMD;
-
-        /* change-detector == changelog */
-        runinit_gsyncd_setrx (&runner, conf);
-        runner_add_args(&runner, "change-detector", "changelog", ".", ".", NULL);
-        RUN_GSYNCD_CMD;
-
-        runinit_gsyncd_setrx (&runner, conf);
-        runner_add_arg(&runner, "working-dir");
-        runner_argprintf(&runner, "%s/${mastervol}/${eSlave}",
-                         DEFAULT_VAR_RUN_DIRECTORY);
-        runner_add_args (&runner, ".", ".", NULL);
-        RUN_GSYNCD_CMD;
-
-        /************
-         * slave pre-configuration
-         ************/
-
-        /* gluster-command-dir */
-        runinit_gsyncd_setrx (&runner, conf);
-        runner_add_args (&runner, "gluster-command-dir", SBIN_DIR"/",
-                         ".", NULL);
-        RUN_GSYNCD_CMD;
-
-        /* gluster-params */
-        runinit_gsyncd_setrx (&runner, conf);
-        runner_add_args (&runner, "gluster-params",
-                         "aux-gfid-mount acl",
-                         ".", NULL);
-        RUN_GSYNCD_CMD;
-
-        /* log-file */
-        runinit_gsyncd_setrx (&runner, conf);
-        runner_add_args (&runner,
-                         "log-file",
-                         DEFAULT_LOG_FILE_DIRECTORY"/"GEOREP"-slaves/${session_owner}:${eSlave}.log",
-                         ".", NULL);
-        RUN_GSYNCD_CMD;
-
-        /* MountBroker log-file */
-        runinit_gsyncd_setrx (&runner, conf);
-        runner_add_args (&runner,
-                         "log-file-mbr",
-                         DEFAULT_LOG_FILE_DIRECTORY"/"GEOREP"-slaves/mbr/${session_owner}:${eSlave}.log",
-                         ".", NULL);
-        RUN_GSYNCD_CMD;
-
-        /* gluster-log-file */
-        runinit_gsyncd_setrx (&runner, conf);
-        runner_add_args (&runner,
-                         "gluster-log-file",
-                         DEFAULT_LOG_FILE_DIRECTORY"/"GEOREP"-slaves/${session_owner}:${eSlave}.gluster.log",
-                         ".", NULL);
-        RUN_GSYNCD_CMD;
-
- out:
-        return ret ? -1 : 0;
+out:
+    return ret ? -1 : 0;
 }
 #undef RUN_GSYNCD_CMD
-#else /* SYNCDAEMON_COMPILE */
+#else  /* SYNCDAEMON_COMPILE */
 static int
-configure_syncdaemon (glusterd_conf_t *conf)
+configure_syncdaemon(glusterd_conf_t *conf)
 {
-        return 0;
+    return 0;
 }
 #endif /* !SYNCDAEMON_COMPILE */
 
-
 static int
-check_prepare_mountbroker_root (char *mountbroker_root)
+check_prepare_mountbroker_root(char *mountbroker_root)
 {
-        int dfd0        = -1;
-        int dfd         = -1;
-        int dfd2        = -1;
-        struct stat st  = {0,};
-        struct stat st2 = {0,};
-        int ret         = 0;
-
-        ret = open (mountbroker_root, O_RDONLY);
+    int dfd0 = -1;
+    int dfd = -1;
+    int dfd2 = -1;
+    struct stat st = {
+        0,
+    };
+    struct stat st2 = {
+        0,
+    };
+    int ret = 0;
+
+    ret = open(mountbroker_root, O_RDONLY);
+    if (ret != -1) {
+        dfd = ret;
+        ret = sys_fstat(dfd, &st);
+    }
+    if (ret == -1 || !S_ISDIR(st.st_mode)) {
+        gf_msg("glusterd", GF_LOG_ERROR, errno, GD_MSG_DIR_OP_FAILED,
+               "cannot access mountbroker-root directory %s", mountbroker_root);
+        ret = -1;
+        goto out;
+    }
+    if (st.st_uid != 0 || (st.st_mode & (S_IWGRP | S_IWOTH))) {
+        gf_msg("glusterd", GF_LOG_ERROR, 0, GD_MSG_DIR_PERM_LIBERAL,
+               "permissions on mountbroker-root directory %s are "
+               "too liberal",
+               mountbroker_root);
+        ret = -1;
+        goto out;
+    }
+    if (!(st.st_mode & (S_IXGRP | S_IXOTH))) {
+        gf_msg("glusterd", GF_LOG_WARNING, 0, GD_MSG_DIR_PERM_STRICT,
+               "permissions on mountbroker-root directory %s are "
+               "probably too strict",
+               mountbroker_root);
+    }
+
+    dfd0 = dup(dfd);
+
+    for (;;) {
+        ret = sys_openat(dfd, "..", O_RDONLY, 0);
         if (ret != -1) {
-                dfd = ret;
-                ret = fstat (dfd, &st);
-        }
-        if (ret == -1 || !S_ISDIR (st.st_mode)) {
-                gf_log ("", GF_LOG_ERROR,
-                        "cannot access mountbroker-root directory %s",
-                        mountbroker_root);
-                ret = -1;
-                goto out;
-        }
-        if (st.st_uid != 0 ||
-            (st.st_mode & (S_IWGRP|S_IWOTH))) {
-                gf_log ("", GF_LOG_ERROR,
-                        "permissions on mountbroker-root directory %s are "
-                        "too liberal", mountbroker_root);
-                ret = -1;
-                goto out;
+            dfd2 = ret;
+            ret = sys_fstat(dfd2, &st2);
         }
-        if (!(st.st_mode & (S_IXGRP|S_IXOTH))) {
-                gf_log ("", GF_LOG_WARNING,
-                        "permissions on mountbroker-root directory %s are "
-                        "probably too strict", mountbroker_root);
+        if (ret == -1) {
+            gf_msg("glusterd", GF_LOG_ERROR, errno, GD_MSG_DIR_OP_FAILED,
+                   "error while checking mountbroker-root ancestors "
+                   "%d (%s)",
+                   errno, strerror(errno));
+            goto out;
         }
 
-        dfd0 = dup (dfd);
-
-        for (;;) {
-                ret = sys_openat (dfd, "..", O_RDONLY);
-                if (ret != -1) {
-                        dfd2 = ret;
-                        ret = fstat (dfd2, &st2);
-                }
-                if (ret == -1) {
-                        gf_log ("", GF_LOG_ERROR,
-                                "error while checking mountbroker-root ancestors "
-                                "%d (%s)", errno, strerror (errno));
-                        goto out;
-                }
-
-                if (st2.st_ino == st.st_ino)
-                        break; /* arrived to root */
-
-                if (st2.st_uid != 0 ||
-                    ((st2.st_mode & (S_IWGRP|S_IWOTH)) &&
-                     !(st2.st_mode & S_ISVTX))) {
-                        gf_log ("", GF_LOG_ERROR,
-                                "permissions on ancestors of mountbroker-root "
-                                "directory are too liberal");
-                        ret = -1;
-                        goto out;
-                }
-                if (!(st.st_mode & (S_IXGRP|S_IXOTH))) {
-                        gf_log ("", GF_LOG_WARNING,
-                                "permissions on ancestors of mountbroker-root "
-                                "directory are probably too strict");
-                }
-
-                close (dfd);
-                dfd = dfd2;
-                st = st2;
-        }
+        if (st2.st_ino == st.st_ino)
+            break; /* arrived to root */
 
-        ret = sys_mkdirat (dfd0, MB_HIVE, 0711);
-        if (ret == -1 && errno == EEXIST)
-                ret = 0;
-        if (ret != -1)
-                ret = sys_fstatat (dfd0, MB_HIVE, &st, AT_SYMLINK_NOFOLLOW);
-        if (ret == -1 || st.st_mode != (S_IFDIR|0711)) {
-                gf_log ("", GF_LOG_ERROR,
-                        "failed to set up mountbroker-root directory %s",
-                        mountbroker_root);
-                ret = -1;
-                goto out;
+        if (st2.st_uid != 0 ||
+            ((st2.st_mode & (S_IWGRP | S_IWOTH)) && !(st2.st_mode & S_ISVTX))) {
+            gf_msg("glusterd", GF_LOG_ERROR, 0, GD_MSG_DIR_PERM_LIBERAL,
+                   "permissions on ancestors of mountbroker-root "
+                   "directory are too liberal");
+            ret = -1;
+            goto out;
+        }
+        if (!(st.st_mode & (S_IXGRP | S_IXOTH))) {
+            gf_msg("glusterd", GF_LOG_WARNING, 0, GD_MSG_DIR_PERM_STRICT,
+                   "permissions on ancestors of mountbroker-root "
+                   "directory are probably too strict");
         }
 
-        ret = 0;
+        sys_close(dfd);
+        dfd = dfd2;
+        st = st2;
+    }
 
- out:
-        if (dfd0 != -1)
-                close (dfd0);
-        if (dfd != -1)
-                close (dfd);
-        if (dfd2 != -1)
-                close (dfd2);
+    ret = sys_mkdirat(dfd0, MB_HIVE, 0711);
+    if (ret == -1 && errno == EEXIST)
+        ret = 0;
+    if (ret != -1)
+        ret = sys_fstatat(dfd0, MB_HIVE, &st, AT_SYMLINK_NOFOLLOW);
+    if (ret == -1 || st.st_mode != (S_IFDIR | 0711)) {
+        gf_msg("glusterd", GF_LOG_ERROR, errno, GD_MSG_CREATE_DIR_FAILED,
+               "failed to set up mountbroker-root directory %s",
+               mountbroker_root);
+        ret = -1;
+        goto out;
+    }
+
+    ret = 0;
 
-        return ret;
+out:
+    if (dfd0 != -1)
+        sys_close(dfd0);
+    if (dfd != -1)
+        sys_close(dfd);
+    if (dfd2 != -1 && dfd != dfd2)
+        sys_close(dfd2);
+
+    return ret;
 }
 
 static int
-_install_mount_spec (dict_t *opts, char *key, data_t *value, void *data)
+_install_mount_spec(dict_t *opts, char *key, data_t *value, void *data)
 {
-        glusterd_conf_t *priv           = THIS->private;
-        char            *label          = NULL;
-        gf_boolean_t     georep         = _gf_false;
-        gf_boolean_t     ghadoop        = _gf_false;
-        char            *pdesc          = value->data;
-        char            *volname        = NULL;
-        int              rv             = 0;
-        gf_mount_spec_t *mspec          = NULL;
-        char            *user           = NULL;
-        char            *volfile_server = NULL;
-
-        label = strtail (key, "mountbroker.");
-
-        /* check for presence of geo-rep/hadoop label */
-        if (!label) {
-                label = strtail (key, "mountbroker-"GEOREP".");
-                if (label)
-                        georep = _gf_true;
-                else {
-                        label = strtail (key, "mountbroker-"GHADOOP".");
-                        if (label)
-                                ghadoop = _gf_true;
-                }
-        }
-
-        if (!label)
-                return 0;
-
-        mspec = GF_CALLOC (1, sizeof (*mspec), gf_gld_mt_mount_spec);
-        if (!mspec)
-                goto err;
-        mspec->label = label;
-
-        if (georep || ghadoop) {
-                volname = gf_strdup (pdesc);
-                if (!volname)
-                        goto err;
-                user = strchr (volname, ':');
-                if (user) {
-                        *user = '\0';
-                        user++;
-                } else
-                        user = label;
-
-                if (georep)
-                        rv = make_georep_mountspec (mspec, volname, user);
-
-                if (ghadoop) {
-                        volfile_server = strchr (user, ':');
-                        if (volfile_server)
-                                *volfile_server++ = '\0';
-                        else
-                                volfile_server = "localhost";
-
-                        rv = make_ghadoop_mountspec (mspec, volname, user, volfile_server);
-                }
-
-                GF_FREE (volname);
-                if (rv != 0)
-                        goto err;
-        } else if (parse_mount_pattern_desc (mspec, pdesc) != 0)
-                goto err;
-
-        cds_list_add_tail (&mspec->speclist, &priv->mount_specs);
-
+    glusterd_conf_t *priv = THIS->private;
+    char *label = NULL;
+    gf_boolean_t georep = _gf_false;
+    char *pdesc = value->data;
+    char *volname = NULL;
+    int rv = 0;
+    gf_mount_spec_t *mspec = NULL;
+    char *user = NULL;
+    xlator_t *this = THIS;
+    GF_ASSERT(this);
+
+    label = strtail(key, "mountbroker.");
+
+    /* check for presence of geo-rep label */
+    if (!label) {
+        label = strtail(key, "mountbroker-" GEOREP ".");
+        if (label)
+            georep = _gf_true;
+    }
+
+    if (!label)
         return 0;
- err:
-
-        gf_log ("", GF_LOG_ERROR,
-                "adding %smount spec failed: label: %s desc: %s",
-                georep ? GEOREP" " : "", label, pdesc);
-
-        if (mspec) {
-                if (mspec->patterns) {
-                        GF_FREE (mspec->patterns->components);
-                        GF_FREE (mspec->patterns);
-                }
-                GF_FREE (mspec);
-        }
 
-        return -1;
-}
-
-
-int
-glusterd_uds_rpcsvc_notify (rpcsvc_t *rpc, void *xl, rpcsvc_event_t event,
-                            void *data)
-{
-        /* glusterd_rpcsvc_notify() does stuff that calls coming in from the
-         * unix domain socket don't need. This is just an empty function to be
-         * used for the uds listener. This will be used later if required.
-         */
-        return 0;
+    mspec = GF_CALLOC(1, sizeof(*mspec), gf_gld_mt_mount_spec);
+    if (!mspec) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_NO_MEMORY, NULL);
+        goto err;
+    }
+    mspec->label = label;
+
+    if (georep) {
+        volname = gf_strdup(pdesc);
+        if (!volname)
+            goto err;
+        user = strchr(volname, ':');
+        if (user) {
+            *user = '\0';
+            user++;
+        } else
+            user = label;
+
+        rv = make_georep_mountspec(mspec, volname, user, priv->logdir);
+
+        GF_FREE(volname);
+        if (rv != 0)
+            goto err;
+    } else if (parse_mount_pattern_desc(mspec, pdesc) != 0)
+        goto err;
+
+    cds_list_add_tail(&mspec->speclist, &priv->mount_specs);
+
+    return 0;
+err:
+
+    gf_msg("glusterd", GF_LOG_ERROR, 0, GD_MSG_MOUNT_SPEC_INSTALL_FAIL,
+           "adding %smount spec failed: label: %s desc: %s",
+           georep ? GEOREP " " : "", label, pdesc ? pdesc : "");
+
+    if (mspec) {
+        if (mspec->patterns) {
+            GF_FREE(mspec->patterns->components);
+            GF_FREE(mspec->patterns);
+        }
+        GF_FREE(mspec);
+    }
+
+    return -1;
 }
 
 /* The glusterd unix domain socket listener only listens for cli */
 rpcsvc_t *
-glusterd_init_uds_listener (xlator_t *this)
+glusterd_init_uds_listener(xlator_t *this)
 {
-        int             ret = -1;
-        dict_t          *options = NULL;
-        rpcsvc_t        *rpc = NULL;
-        data_t          *sock_data = NULL;
-        char            sockfile[UNIX_PATH_MAX+1] = {0,};
-        int             i = 0;
-
-
-        GF_ASSERT (this);
-
-        sock_data = dict_get (this->options, "glusterd-sockfile");
-        if (!sock_data) {
-                strncpy (sockfile, DEFAULT_GLUSTERD_SOCKFILE, UNIX_PATH_MAX);
-        } else {
-                strncpy (sockfile, sock_data->data, UNIX_PATH_MAX);
-        }
-
-        options = dict_new ();
-        if (!options)
-                goto out;
-
-        ret = rpcsvc_transport_unix_options_build (&options, sockfile);
-        if (ret)
-                goto out;
-
-        rpc = rpcsvc_init (this, this->ctx, options, 8);
-        if (rpc == NULL) {
-                ret = -1;
-                goto out;
-        }
-
-        ret = rpcsvc_register_notify (rpc, glusterd_uds_rpcsvc_notify,
-                                      this);
+    int ret = -1;
+    dict_t *options = NULL;
+    rpcsvc_t *rpc = NULL;
+    data_t *sock_data = NULL;
+    char sockfile[UNIX_PATH_MAX] = {0};
+    int i = 0;
+
+    GF_ASSERT(this);
+
+    options = dict_new();
+    if (!options) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_CREATE_FAIL, NULL);
+        goto out;
+    }
+
+    sock_data = dict_get(this->options, "glusterd-sockfile");
+    (void)snprintf(sockfile, sizeof(sockfile), "%s",
+                   sock_data ? sock_data->data : DEFAULT_GLUSTERD_SOCKFILE);
+
+    ret = rpcsvc_transport_unix_options_build(options, sockfile);
+    if (ret)
+        goto out;
+
+    rpc = rpcsvc_init(this, this->ctx, options, 8);
+    if (rpc == NULL) {
+        ret = -1;
+        goto out;
+    }
+
+    ret = rpcsvc_register_notify(rpc, glusterd_rpcsvc_notify, this);
+    if (ret) {
+        gf_msg_debug(this->name, 0, "Failed to register notify function");
+        goto out;
+    }
+
+    ret = rpcsvc_create_listeners(rpc, options, this->name);
+    if (ret != 1) {
+        gf_msg_debug(this->name, 0, "Failed to create listener");
+        goto out;
+    }
+    ret = 0;
+
+    for (i = 0; i < gd_uds_programs_count; i++) {
+        ret = glusterd_program_register(this, rpc, gd_uds_programs[i]);
         if (ret) {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "Failed to register notify function");
-                goto out;
-        }
-
-        ret = rpcsvc_create_listeners (rpc, options, this->name);
-        if (ret != 1) {
-                gf_log (this->name, GF_LOG_DEBUG, "Failed to create listener");
-                goto out;
-        }
-        ret = 0;
-
-        for (i = 0; i < gd_uds_programs_count; i++) {
-                ret = glusterd_program_register (this, rpc, gd_uds_programs[i]);
-                if (ret) {
-                        i--;
-                        for (; i >= 0; i--)
-                                rpcsvc_program_unregister (rpc,
-                                                           gd_uds_programs[i]);
+            i--;
+            for (; i >= 0; i--)
+                rpcsvc_program_unregister(rpc, gd_uds_programs[i]);
 
-                        goto out;
-                }
+            goto out;
         }
+    }
 
 out:
-        if (options)
-                dict_unref (options);
-
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "Failed to start glusterd "
-                        "unix domain socket listener.");
-                if (rpc) {
-                        GF_FREE (rpc);
-                        rpc = NULL;
-                }
-        }
-        return rpc;
+    if (options)
+        dict_unref(options);
+
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0,
+               GD_MSG_GLUSTERD_SOCK_LISTENER_START_FAIL,
+               "Failed to start glusterd "
+               "unix domain socket listener.");
+        if (rpc) {
+            GF_FREE(rpc);
+            rpc = NULL;
+        }
+    }
+    return rpc;
 }
 
 void
-glusterd_stop_uds_listener (xlator_t *this)
+glusterd_stop_uds_listener(xlator_t *this)
 {
-        glusterd_conf_t         *conf = NULL;
-        rpcsvc_listener_t       *listener = NULL;
-        rpcsvc_listener_t       *next = NULL;
-        data_t                  *sock_data = NULL;
-        char                     sockfile[UNIX_PATH_MAX+1] = {0,};
+    glusterd_conf_t *conf = NULL;
+    rpcsvc_listener_t *listener = NULL;
+    rpcsvc_listener_t *next = NULL;
+    data_t *sock_data = NULL;
+    char sockfile[UNIX_PATH_MAX] = {0};
 
-        GF_ASSERT (this);
-        conf = this->private;
+    GF_ASSERT(this);
+    conf = this->private;
 
-        (void) rpcsvc_program_unregister (conf->uds_rpc, &gd_svc_cli_prog);
-        (void) rpcsvc_program_unregister (conf->uds_rpc, &gluster_handshake_prog);
+    (void)rpcsvc_program_unregister(conf->uds_rpc, &gd_svc_cli_prog);
+    (void)rpcsvc_program_unregister(conf->uds_rpc, &gluster_handshake_prog);
 
-        list_for_each_entry_safe (listener, next, &conf->uds_rpc->listeners,
-                                  list) {
-                rpcsvc_listener_destroy (listener);
-        }
+    list_for_each_entry_safe(listener, next, &conf->uds_rpc->listeners, list)
+    {
+        rpcsvc_listener_destroy(listener);
+    }
 
-        (void) rpcsvc_unregister_notify (conf->uds_rpc,
-                                         glusterd_uds_rpcsvc_notify,
-                                         this);
+    (void)rpcsvc_unregister_notify(conf->uds_rpc, glusterd_rpcsvc_notify, this);
 
-        sock_data = dict_get (this->options, "glusterd-sockfile");
-        if (!sock_data) {
-                strncpy (sockfile, DEFAULT_GLUSTERD_SOCKFILE, UNIX_PATH_MAX);
-        } else {
-                strncpy (sockfile, sock_data->data, UNIX_PATH_MAX);
-        }
-        unlink (sockfile);
-
-        GF_FREE (conf->uds_rpc);
-        conf->uds_rpc = NULL;
+    sock_data = dict_get(this->options, "glusterd-sockfile");
+    (void)snprintf(sockfile, sizeof(sockfile), "%s",
+                   sock_data ? sock_data->data : DEFAULT_GLUSTERD_SOCKFILE);
+    sys_unlink(sockfile);
 
-        return;
+    return;
 }
 
-static int
-glusterd_init_snap_folder (xlator_t *this)
+void
+glusterd_stop_listener(xlator_t *this)
 {
-        int             ret = -1;
-        struct stat     buf = {0,};
-
-        GF_ASSERT (this);
+    glusterd_conf_t *conf = NULL;
+    rpcsvc_listener_t *listener = NULL;
+    rpcsvc_listener_t *next = NULL;
+    int i = 0;
 
-        /* Snapshot volumes are mounted under /var/run/gluster/snaps folder.
-         * But /var/run is normally a symbolic link to /run folder, which
-         * creates problems as the entry point in the mtab for the mount point
-         * and glusterd maintained entry point will be different. Therefore
-         * identify the correct run folder and use it for snap volume mounting.
-         */
-        ret = lstat (GLUSTERD_VAR_RUN_DIR, &buf);
-        if (ret != 0) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "stat fails on %s, exiting. (errno = %d)",
-                        GLUSTERD_VAR_RUN_DIR, errno);
-                goto out;
-        }
+    GF_VALIDATE_OR_GOTO("glusterd", this, out);
+    conf = this->private;
+    GF_VALIDATE_OR_GOTO(this->name, conf, out);
 
-        /* If /var/run is symlink then use /run folder */
-        if (S_ISLNK (buf.st_mode)) {
-                strcpy (snap_mount_folder, GLUSTERD_RUN_DIR);
-        } else {
-                strcpy (snap_mount_folder, GLUSTERD_VAR_RUN_DIR);
-        }
+    gf_msg_debug(this->name, 0, "%s function called ", __func__);
 
-        strcat (snap_mount_folder, GLUSTERD_DEFAULT_SNAPS_BRICK_DIR);
+    for (i = 0; i < gd_inet_programs_count; i++) {
+        rpcsvc_program_unregister(conf->rpc, gd_inet_programs[i]);
+    }
 
-        ret = stat (snap_mount_folder, &buf);
-        if ((ret != 0) && (ENOENT != errno)) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "stat fails on %s, exiting. (errno = %d)",
-                        snap_mount_folder, errno);
-                ret = -1;
-                goto out;
-        }
+    list_for_each_entry_safe(listener, next, &conf->rpc->listeners, list)
+    {
+        rpcsvc_listener_destroy(listener);
+    }
 
-        if ((!ret) && (!S_ISDIR(buf.st_mode))) {
-                gf_log (this->name, GF_LOG_CRITICAL,
-                        "Provided snap path %s is not a directory,"
-                        "exiting", snap_mount_folder);
-                ret = -1;
-                goto out;
-        }
+    (void)rpcsvc_unregister_notify(conf->rpc, glusterd_rpcsvc_notify, this);
 
-        if ((-1 == ret) && (ENOENT == errno)) {
-                /* Create missing folders */
-                ret = mkdir_p (snap_mount_folder, 0777, _gf_true);
+out:
 
-                if (-1 == ret) {
-                        gf_log (this->name, GF_LOG_CRITICAL,
-                                "Unable to create directory %s"
-                                " ,errno = %d", snap_mount_folder, errno);
-                        goto out;
-                }
-        }
+    return;
+}
 
+static int
+glusterd_find_correct_var_run_dir(xlator_t *this, char *var_run_dir)
+{
+    int ret = -1;
+    struct stat buf = {
+        0,
+    };
+
+    GF_VALIDATE_OR_GOTO("glusterd", this, out);
+    GF_VALIDATE_OR_GOTO(this->name, var_run_dir, out);
+
+    /* /var/run is normally a symbolic link to /run dir, which
+     * creates problems as the entry point in the mtab for the mount point
+     * and glusterd maintained entry point will be different. Therefore
+     * identify the correct run dir and use it
+     */
+    ret = sys_lstat(GLUSTERD_VAR_RUN_DIR, &buf);
+    if (ret != 0) {
+        gf_msg(this->name, GF_LOG_ERROR, errno, GD_MSG_FILE_OP_FAILED,
+               "stat fails on %s, exiting. (errno = %d)", GLUSTERD_VAR_RUN_DIR,
+               errno);
+        goto out;
+    }
+
+    /* If /var/run is symlink then use /run dir */
+    if (S_ISLNK(buf.st_mode)) {
+        strcpy(var_run_dir, GLUSTERD_RUN_DIR);
+    } else {
+        strcpy(var_run_dir, GLUSTERD_VAR_RUN_DIR);
+    }
+
+    ret = 0;
 out:
-        return ret;
+    return ret;
 }
 
 static int
-glusterd_svc_init_all ()
+glusterd_init_var_run_dirs(xlator_t *this, char *var_run_dir,
+                           char *dir_to_be_created)
 {
-        int                 ret     = -1;
-        xlator_t           *this    = NULL;
-        glusterd_conf_t    *priv    = NULL;
+    int ret = -1;
+    struct stat buf = {
+        0,
+    };
+    char abs_path[PATH_MAX] = {
+        0,
+    };
+
+    GF_VALIDATE_OR_GOTO("glusterd", this, out);
+    GF_VALIDATE_OR_GOTO(this->name, var_run_dir, out);
+    GF_VALIDATE_OR_GOTO(this->name, dir_to_be_created, out);
+
+    snprintf(abs_path, sizeof(abs_path), "%s%s", var_run_dir,
+             dir_to_be_created);
+
+    ret = sys_stat(abs_path, &buf);
+    if ((ret != 0) && (ENOENT != errno)) {
+        gf_msg(this->name, GF_LOG_ERROR, errno, GD_MSG_FILE_OP_FAILED,
+               "stat fails on %s, exiting. (errno = %d)", abs_path, errno);
+        ret = -1;
+        goto out;
+    }
+
+    if ((!ret) && (!S_ISDIR(buf.st_mode))) {
+        gf_msg(this->name, GF_LOG_CRITICAL, ENOENT, GD_MSG_DIR_NOT_FOUND,
+               "Provided snap path %s is not a directory,"
+               "exiting",
+               abs_path);
+        ret = -1;
+        goto out;
+    }
+
+    if ((-1 == ret) && (ENOENT == errno)) {
+        /* Create missing dirs */
+        ret = mkdir_p(abs_path, 0755, _gf_true);
 
-        this = THIS;
-        GF_ASSERT (this);
-
-        priv = this->private;
-        GF_ASSERT (priv);
-
-        /* Init SHD svc */
-        ret = glusterd_shdsvc_init (&(priv->shd_svc));
-        if (ret) {
-                gf_log (THIS->name, GF_LOG_ERROR, "Failed to init shd service");
-                goto out;
+        if (-1 == ret) {
+            gf_msg(this->name, GF_LOG_CRITICAL, errno, GD_MSG_CREATE_DIR_FAILED,
+                   "Unable to create directory %s"
+                   " ,errno = %d",
+                   abs_path, errno);
+            goto out;
         }
-        gf_log (THIS->name, GF_LOG_DEBUG, "shd service initialized");
+    }
 
-        /* Init NFS svc */
-        ret = glusterd_nfssvc_init (&(priv->nfs_svc));
-        if (ret) {
-                gf_log (THIS->name, GF_LOG_ERROR, "Failed to init nfs service");
-                goto out;
-        }
-        gf_log (THIS->name, GF_LOG_DEBUG, "nfs service initialized");
+out:
+    return ret;
+}
 
-        /* Init QuotaD svc */
-        ret = glusterd_quotadsvc_init (&(priv->quotad_svc));
-        if (ret) {
-                gf_log (THIS->name, GF_LOG_ERROR, "Failed to init quotad "
-                        "service");
-                goto out;
-        }
-        gf_log (THIS->name, GF_LOG_DEBUG, "quotad service initialized");
+static int
+is_upgrade(dict_t *options, gf_boolean_t *upgrade)
+{
+    int ret = 0;
+    char *type = NULL;
 
-        /* Init BitD svc */
-        ret = glusterd_bitdsvc_init (&(priv->bitd_svc));
+    ret = dict_get_str(options, "upgrade", &type);
+    if (!ret) {
+        ret = gf_string2boolean(type, upgrade);
         if (ret) {
-                gf_log (THIS->name, GF_LOG_ERROR, "Failed to initialized BitD "
-                        "service");
-                goto out;
-        }
-        gf_log (THIS->name, GF_LOG_DEBUG, "BitD service initialized");
+            gf_msg("glusterd", GF_LOG_ERROR, 0, GD_MSG_STR_TO_BOOL_FAIL,
+                   "upgrade option "
+                   "%s is not a valid boolean type",
+                   type);
+            ret = -1;
+            goto out;
+        }
+    }
+    ret = 0;
+out:
+    return ret;
+}
 
-        ret = glusterd_scrubsvc_init (&(priv->scrub_svc));
-        if (ret) {
-                gf_log (THIS->name, GF_LOG_ERROR, "Failed to initialized scrub "
-                        "service");
-                goto out;
-        }
-        gf_log (THIS->name, GF_LOG_DEBUG, "scrub service initialized");
+static int
+is_downgrade(dict_t *options, gf_boolean_t *downgrade)
+{
+    int ret = 0;
+    char *type = NULL;
 
+    ret = dict_get_str(options, "downgrade", &type);
+    if (!ret) {
+        ret = gf_string2boolean(type, downgrade);
+        if (ret) {
+            gf_msg("glusterd", GF_LOG_ERROR, 0, GD_MSG_STR_TO_BOOL_FAIL,
+                   "downgrade option "
+                   "%s is not a valid boolean type",
+                   type);
+            ret = -1;
+            goto out;
+        }
+    }
+    ret = 0;
 out:
-        return ret;
+    return ret;
 }
 
 /*
@@ -1248,439 +1395,711 @@ out:
  *
  */
 int
-init (xlator_t *this)
+init(xlator_t *this)
 {
-        int32_t            ret               = -1;
-        rpcsvc_t          *rpc               = NULL;
-        rpcsvc_t          *uds_rpc           = NULL;
-        glusterd_conf_t   *conf              = NULL;
-        data_t            *dir_data          = NULL;
-        struct stat        buf               = {0,};
-        char               storedir [PATH_MAX] = {0,};
-        char               workdir [PATH_MAX] = {0,};
-        char               cmd_log_filename [PATH_MAX] = {0,};
-        int                first_time        = 0;
-        char              *mountbroker_root  = NULL;
-        int                i                 = 0;
-        int                total_transport   = 0;
-        char              *valgrind_str      = NULL;
-        char              *transport_type    = NULL;
-
-#ifndef GF_DARWIN_HOST_OS
-        {
-                struct rlimit lim;
-                lim.rlim_cur = 65536;
-                lim.rlim_max = 65536;
-
-                if (setrlimit (RLIMIT_NOFILE, &lim) == -1) {
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "Failed to set 'ulimit -n "
-                                " 65536': %s", strerror(errno));
-                } else {
-                        gf_log (this->name, GF_LOG_INFO,
-                                "Maximum allowed open file descriptors "
-                                "set to 65536");
-                }
-        }
+    int32_t ret = -1;
+    rpcsvc_t *rpc = NULL;
+    rpcsvc_t *uds_rpc = NULL;
+    glusterd_conf_t *conf = NULL;
+    data_t *dir_data = NULL;
+    struct stat buf = {
+        0,
+    };
+    char storedir[PATH_MAX] = {
+        0,
+    };
+    char workdir[PATH_MAX] = {
+        0,
+    };
+    char rundir[PATH_MAX] = {
+        0,
+    };
+    char logdir[VALID_GLUSTERD_PATHMAX] = {
+        0,
+    };
+    char cmd_log_filename[PATH_MAX] = {
+        0,
+    };
+    char *mountbroker_root = NULL;
+    int i = 0;
+    int total_transport = 0;
+    gf_valgrind_tool vgtool;
+    char *valgrind_str = NULL;
+    char *transport_type = NULL;
+    char var_run_dir[PATH_MAX] = {
+        0,
+    };
+    int32_t workers = 0;
+    gf_boolean_t upgrade = _gf_false;
+    gf_boolean_t downgrade = _gf_false;
+    char *localtime_logging = NULL;
+    int32_t len = 0;
+    int op_version = 0;
+
+#if defined(RUN_WITH_MEMCHECK)
+    vgtool = _gf_memcheck;
+#elif defined(RUN_WITH_DRD)
+    vgtool = _gf_drd;
+#else
+    vgtool = _gf_none;
 #endif
 
-        dir_data = dict_get (this->options, "working-directory");
-
-        if (!dir_data) {
-                //Use default working dir
-                strncpy (workdir, GLUSTERD_DEFAULT_WORKDIR, PATH_MAX);
+#ifndef GF_DARWIN_HOST_OS
+    {
+        struct rlimit lim;
+        lim.rlim_cur = 65536;
+        lim.rlim_max = 65536;
+
+        if (setrlimit(RLIMIT_NOFILE, &lim) == -1) {
+            gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_SET_XATTR_FAIL,
+                    "Failed to set 'ulimit -n 65536'", NULL);
         } else {
-                strncpy (workdir, dir_data->data, PATH_MAX);
-        }
-
-        ret = stat (workdir, &buf);
-        if ((ret != 0) && (ENOENT != errno)) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "stat fails on %s, exiting. (errno = %d)",
-                        workdir, errno);
-                exit (1);
-        }
-
-        if ((!ret) && (!S_ISDIR(buf.st_mode))) {
-                gf_log (this->name, GF_LOG_CRITICAL,
-                        "Provided working area %s is not a directory,"
-                        "exiting", workdir);
-                exit (1);
-        }
-
-
-        if ((-1 == ret) && (ENOENT == errno)) {
-                ret = mkdir_p (workdir, 0777, _gf_true);
-
-                if (-1 == ret) {
-                        gf_log (this->name, GF_LOG_CRITICAL,
-                                "Unable to create directory %s"
-                                " ,errno = %d", workdir, errno);
-                        exit (1);
-                }
-
-                first_time = 1;
-        }
-
-        setenv ("GLUSTERD_WORKDIR", workdir, 1);
-        gf_log (this->name, GF_LOG_INFO, "Using %s as working directory",
-                workdir);
-
-        ret = glusterd_init_snap_folder (this);
-
-        if (ret) {
-                gf_log (this->name, GF_LOG_CRITICAL, "Unable to create "
-                        "snap backend folder");
-                exit (1);
-        }
-
-        snprintf (cmd_log_filename, PATH_MAX, "%s/cmd_history.log",
-                  DEFAULT_LOG_FILE_DIRECTORY);
-        ret = gf_cmd_log_init (cmd_log_filename);
-
-        if (ret == -1) {
-                gf_log ("this->name", GF_LOG_CRITICAL,
-                        "Unable to create cmd log file %s", cmd_log_filename);
-                exit (1);
-        }
-
-        snprintf (storedir, PATH_MAX, "%s/vols", workdir);
-
-        ret = mkdir (storedir, 0777);
-
-        if ((-1 == ret) && (errno != EEXIST)) {
-                gf_log (this->name, GF_LOG_CRITICAL,
-                        "Unable to create volume directory %s"
-                        " ,errno = %d", storedir, errno);
-                exit (1);
-        }
-
-        snprintf (storedir, PATH_MAX, "%s/snaps", workdir);
-
-        ret = mkdir (storedir, 0777);
-
-        if ((-1 == ret) && (errno != EEXIST)) {
-                gf_log (this->name, GF_LOG_CRITICAL,
-                        "Unable to create snaps directory %s"
-                        " ,errno = %d", storedir, errno);
-                exit (1);
-        }
-
-        snprintf (storedir, PATH_MAX, "%s/peers", workdir);
-
-        ret = mkdir (storedir, 0777);
-
-        if ((-1 == ret) && (errno != EEXIST)) {
-                gf_log (this->name, GF_LOG_CRITICAL,
-                        "Unable to create peers directory %s"
-                        " ,errno = %d", storedir, errno);
-                exit (1);
-        }
-
-        snprintf (storedir, PATH_MAX, "%s/bricks", DEFAULT_LOG_FILE_DIRECTORY);
-        ret = mkdir (storedir, 0777);
-        if ((-1 == ret) && (errno != EEXIST)) {
-                gf_log (this->name, GF_LOG_CRITICAL,
-                        "Unable to create logs directory %s"
-                        " ,errno = %d", storedir, errno);
-                exit (1);
-        }
-
-        snprintf (storedir, PATH_MAX, "%s/nfs", workdir);
-        ret = mkdir (storedir, 0777);
-        if ((-1 == ret) && (errno != EEXIST)) {
-                gf_log (this->name, GF_LOG_CRITICAL,
-                        "Unable to create nfs directory %s"
-                        " ,errno = %d", storedir, errno);
-                exit (1);
-        }
-
-        snprintf (storedir, PATH_MAX, "%s/bitd", workdir);
-        ret = mkdir (storedir, 0777);
-        if ((-1 == ret) && (errno != EEXIST)) {
-                gf_log (this->name, GF_LOG_CRITICAL,
-                        "Unable to create bitrot directory %s"
-                        " ,errno = %d", storedir, errno);
-                exit (1);
-        }
-
-        snprintf (storedir, PATH_MAX, "%s/scrub", workdir);
-        ret = mkdir (storedir, 0777);
-        if ((-1 == ret) && (errno != EEXIST)) {
-                gf_log (this->name, GF_LOG_CRITICAL,
-                        "Unable to create scrub directory %s"
-                        " ,errno = %d", storedir, errno);
-                exit (1);
-        }
-
-        snprintf (storedir, PATH_MAX, "%s/glustershd", workdir);
-        ret = mkdir (storedir, 0777);
-        if ((-1 == ret) && (errno != EEXIST)) {
-                gf_log (this->name, GF_LOG_CRITICAL,
-                        "Unable to create glustershd directory %s"
-                        " ,errno = %d", storedir, errno);
-                exit (1);
-        }
-
-        snprintf (storedir, PATH_MAX, "%s/quotad", workdir);
-        ret = mkdir (storedir, 0777);
-        if ((-1 == ret) && (errno != EEXIST)) {
-                gf_log (this->name, GF_LOG_CRITICAL,
-                        "Unable to create quotad directory %s"
-                        " ,errno = %d", storedir, errno);
-                exit (1);
-        }
-
-        snprintf (storedir, PATH_MAX, "%s/groups", workdir);
-        ret = mkdir (storedir, 0777);
-        if ((-1 == ret) && (errno != EEXIST)) {
-                gf_log (this->name, GF_LOG_CRITICAL,
-                        "Unable to create glustershd directory %s"
-                        " ,errno = %d", storedir, errno);
-                exit (1);
+            gf_msg(this->name, GF_LOG_INFO, 0, GD_MSG_FILE_DESC_LIMIT_SET,
+                   "Maximum allowed open file descriptors "
+                   "set to 65536");
         }
+    }
+#endif
 
-        ret = glusterd_rpcsvc_options_build (this->options);
-        if (ret)
-                goto out;
-        rpc = rpcsvc_init (this, this->ctx, this->options, 64);
-        if (rpc == NULL) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "failed to init rpc");
-                goto out;
-        }
+    dir_data = dict_get(this->options, "run-directory");
+
+    if (!dir_data) {
+        /* Use default working dir */
+        len = snprintf(rundir, PATH_MAX, "%s", DEFAULT_VAR_RUN_DIRECTORY);
+    } else {
+        len = snprintf(rundir, PATH_MAX, "%s", dir_data->data);
+    }
+    if (len < 0 || len >= PATH_MAX)
+        exit(2);
+
+    dir_data = dict_get(this->options, "cluster-test-mode");
+    if (!dir_data) {
+        /* Use default working dir */
+        len = snprintf(logdir, VALID_GLUSTERD_PATHMAX, "%s",
+                       DEFAULT_LOG_FILE_DIRECTORY);
+    } else {
+        len = snprintf(logdir, VALID_GLUSTERD_PATHMAX, "%s", dir_data->data);
+        gf_msg(this->name, GF_LOG_INFO, 0, GD_MSG_CLUSTER_RC_ENABLE,
+               "cluster-test-mode is enabled logdir is %s", dir_data->data);
+    }
+    if (len < 0 || len >= PATH_MAX)
+        exit(2);
+
+    ret = mkdir_p(logdir, 0777, _gf_true);
+    if ((ret == -1) && (EEXIST != errno)) {
+        gf_msg(THIS->name, GF_LOG_ERROR, errno, GD_MSG_CREATE_DIR_FAILED,
+               "Unable to create log dir %s", logdir);
+        exit(1);
+    }
+
+    dir_data = dict_get(this->options, "working-directory");
+
+    if (!dir_data) {
+        // Use default working dir
+        len = snprintf(workdir, PATH_MAX, "%s", GLUSTERD_DEFAULT_WORKDIR);
+    } else {
+        len = snprintf(workdir, PATH_MAX, "%s", dir_data->data);
+    }
+    if (len < 0 || len >= PATH_MAX)
+        exit(2);
+
+    ret = sys_stat(workdir, &buf);
+    if ((ret != 0) && (ENOENT != errno)) {
+        gf_msg(this->name, GF_LOG_ERROR, errno, GD_MSG_DIR_OP_FAILED,
+               "stat fails on %s, exiting. (errno = %d)", workdir, errno);
+        exit(1);
+    }
+
+    if ((!ret) && (!S_ISDIR(buf.st_mode))) {
+        gf_msg(this->name, GF_LOG_CRITICAL, ENOENT, GD_MSG_DIR_NOT_FOUND,
+               "Provided working area %s is not a directory,"
+               "exiting",
+               workdir);
+        exit(1);
+    }
+
+    if ((-1 == ret) && (ENOENT == errno)) {
+        ret = mkdir_p(workdir, 0755, _gf_true);
 
-        ret = rpcsvc_register_notify (rpc, glusterd_rpcsvc_notify, this);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "rpcsvc_register_notify returned %d", ret);
-                goto out;
-        }
+        if (-1 == ret) {
+            gf_msg(this->name, GF_LOG_CRITICAL, errno, GD_MSG_CREATE_DIR_FAILED,
+                   "Unable to create directory %s"
+                   " ,errno = %d",
+                   workdir, errno);
+            exit(1);
+        }
+    }
+
+    setenv("GLUSTERD_WORKDIR", workdir, 1);
+    gf_msg(this->name, GF_LOG_INFO, 0, GD_MSG_CURR_WORK_DIR_INFO,
+           "Using %s as working directory", workdir);
+
+    setenv("DEFAULT_VAR_RUN_DIRECTORY", rundir, 1);
+    gf_msg(this->name, GF_LOG_INFO, 0, GD_MSG_CURR_WORK_DIR_INFO,
+           "Using %s as pid file working "
+           "directory",
+           rundir);
+
+    ret = glusterd_find_correct_var_run_dir(this, var_run_dir);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_CRITICAL, 0, GD_MSG_VAR_RUN_DIR_FIND_FAIL,
+               "Unable to find "
+               "the correct var run dir");
+        exit(1);
+    }
+
+    ret = glusterd_init_var_run_dirs(this, var_run_dir,
+                                     GLUSTERD_DEFAULT_SNAPS_BRICK_DIR);
+
+    if (ret) {
+        gf_msg(this->name, GF_LOG_CRITICAL, 0, GD_MSG_CREATE_DIR_FAILED,
+               "Unable to create "
+               "snap backend folder");
+        exit(1);
+    }
+
+    len = snprintf(snap_mount_dir, sizeof(snap_mount_dir), "%s%s", var_run_dir,
+                   GLUSTERD_DEFAULT_SNAPS_BRICK_DIR);
+    if ((len < 0) || (len >= sizeof(snap_mount_dir))) {
+        gf_msg(this->name, GF_LOG_CRITICAL, 0, GD_MSG_DIR_OP_FAILED,
+               "Snap mount dir too long");
+        exit(1);
+    }
+
+    ret = mkdir_p(GLUSTER_SHARED_STORAGE_BRICK_DIR, 0755, _gf_true);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_CRITICAL, 0, GD_MSG_DIR_OP_FAILED,
+               "Unable to create "
+               "shared storage brick");
+        exit(1);
+    }
+
+    ret = glusterd_init_var_run_dirs(this, rundir, GLUSTERD_BITD_RUN_DIR);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_CRITICAL, 0, GD_MSG_CREATE_DIR_FAILED,
+               "Unable to create "
+               "bitd running directory");
+        exit(1);
+    }
+
+    ret = glusterd_init_var_run_dirs(this, rundir, GLUSTERD_SCRUB_RUN_DIR);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_CRITICAL, 0, GD_MSG_CREATE_DIR_FAILED,
+               "Unable to create "
+               "scrub running directory");
+        exit(1);
+    }
+
+#ifdef BUILD_GNFS
+    ret = glusterd_init_var_run_dirs(this, rundir, GLUSTERD_NFS_RUN_DIR);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_CRITICAL, 0, GD_MSG_CREATE_DIR_FAILED,
+               "Unable to create "
+               "nfs running directory");
+        exit(1);
+    }
+#endif
 
-        if (this->ctx->secure_mgmt) {
-                /*
-                 * The socket code will turn on SSL based on the same check,
-                 * but that will by default turn on own-thread as well and
-                 * we're not multi-threaded enough to handle that.  Thus, we
-                 * override the value here.
-                 */
-                ret = dict_set_str (this->options,
-                                    "transport.socket.own-thread", "off");
-                if (ret != 0) {
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "failed to clear own-thread");
-                        goto out;
-                }
-                /*
-                 * With strong authentication, we can afford to allow
-                 * privileged operations over TCP.
-                 */
-                gd_inet_programs[1] = &gd_svc_cli_prog;
-                /*
-                 * This is the only place where we want secure_srvr to reflect
-                 * the management-plane setting.
-                 */
-                this->ctx->secure_srvr = MGMT_SSL_ALWAYS;
+    ret = glusterd_init_var_run_dirs(this, rundir, GLUSTERD_QUOTAD_RUN_DIR);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_CRITICAL, 0, GD_MSG_CREATE_DIR_FAILED,
+               "Unable to create "
+               "quota running directory");
+        exit(1);
+    }
+
+    snprintf(cmd_log_filename, PATH_MAX, "%s/cmd_history.log", logdir);
+    ret = gf_cmd_log_init(cmd_log_filename);
+
+    if (ret == -1) {
+        gf_msg("this->name", GF_LOG_CRITICAL, errno, GD_MSG_FILE_OP_FAILED,
+               "Unable to create cmd log file %s", cmd_log_filename);
+        exit(1);
+    }
+
+    len = snprintf(storedir, sizeof(storedir), "%s/vols", workdir);
+    if ((len < 0) || (len >= sizeof(storedir))) {
+        exit(1);
+    }
+
+    ret = sys_mkdir(storedir, 0755);
+
+    if ((-1 == ret) && (errno != EEXIST)) {
+        gf_msg(this->name, GF_LOG_CRITICAL, errno, GD_MSG_CREATE_DIR_FAILED,
+               "Unable to create volume directory %s"
+               " ,errno = %d",
+               storedir, errno);
+        exit(1);
+    }
+
+    /*keeping individual volume pid file information in /var/run/gluster* */
+    len = snprintf(storedir, sizeof(storedir), "%s/vols", rundir);
+    if ((len < 0) || (len >= sizeof(storedir))) {
+        exit(1);
+    }
+
+    ret = sys_mkdir(storedir, 0755);
+
+    if ((-1 == ret) && (errno != EEXIST)) {
+        gf_msg(this->name, GF_LOG_CRITICAL, errno, GD_MSG_CREATE_DIR_FAILED,
+               "Unable to create volume directory %s"
+               " ,errno = %d",
+               storedir, errno);
+        exit(1);
+    }
+
+    len = snprintf(storedir, sizeof(storedir), "%s/snaps", workdir);
+    if ((len < 0) || (len >= sizeof(storedir))) {
+        exit(1);
+    }
+
+    ret = sys_mkdir(storedir, 0755);
+
+    if ((-1 == ret) && (errno != EEXIST)) {
+        gf_msg(this->name, GF_LOG_CRITICAL, errno, GD_MSG_CREATE_DIR_FAILED,
+               "Unable to create snaps directory %s"
+               " ,errno = %d",
+               storedir, errno);
+        exit(1);
+    }
+
+    len = snprintf(storedir, sizeof(storedir), "%s/peers", workdir);
+    if ((len < 0) || (len >= sizeof(storedir))) {
+        exit(1);
+    }
+
+    ret = sys_mkdir(storedir, 0755);
+
+    if ((-1 == ret) && (errno != EEXIST)) {
+        gf_msg(this->name, GF_LOG_CRITICAL, errno, GD_MSG_CREATE_DIR_FAILED,
+               "Unable to create peers directory %s"
+               " ,errno = %d",
+               storedir, errno);
+        exit(1);
+    }
+
+    len = snprintf(storedir, sizeof(storedir), "%s/bricks", logdir);
+    if ((len < 0) || (len >= sizeof(storedir))) {
+        exit(1);
+    }
+
+    ret = sys_mkdir(storedir, 0755);
+    if ((-1 == ret) && (errno != EEXIST)) {
+        gf_msg(this->name, GF_LOG_CRITICAL, errno, GD_MSG_CREATE_DIR_FAILED,
+               "Unable to create logs directory %s"
+               " ,errno = %d",
+               storedir, errno);
+        exit(1);
+    }
+
+#ifdef BUILD_GNFS
+    len = snprintf(storedir, sizeof(storedir), "%s/nfs", workdir);
+    if ((len < 0) || (len >= sizeof(storedir))) {
+        exit(1);
+    }
+    ret = sys_mkdir(storedir, 0755);
+    if ((-1 == ret) && (errno != EEXIST)) {
+        gf_msg(this->name, GF_LOG_CRITICAL, errno, GD_MSG_CREATE_DIR_FAILED,
+               "Unable to create nfs directory %s"
+               " ,errno = %d",
+               storedir, errno);
+        exit(1);
+    }
+#endif
+    len = snprintf(storedir, sizeof(storedir), "%s/bitd", workdir);
+    if ((len < 0) || (len >= sizeof(storedir))) {
+        exit(1);
+    }
+    ret = sys_mkdir(storedir, 0755);
+    if ((-1 == ret) && (errno != EEXIST)) {
+        gf_msg(this->name, GF_LOG_CRITICAL, errno, GD_MSG_CREATE_DIR_FAILED,
+               "Unable to create bitrot directory %s", storedir);
+        exit(1);
+    }
+
+    len = snprintf(storedir, sizeof(storedir), "%s/scrub", workdir);
+    if ((len < 0) || (len >= sizeof(storedir))) {
+        exit(1);
+    }
+    ret = sys_mkdir(storedir, 0755);
+    if ((-1 == ret) && (errno != EEXIST)) {
+        gf_msg(this->name, GF_LOG_CRITICAL, errno, GD_MSG_CREATE_DIR_FAILED,
+               "Unable to create scrub directory %s", storedir);
+        exit(1);
+    }
+
+    len = snprintf(storedir, sizeof(storedir), "%s/glustershd", workdir);
+    if ((len < 0) || (len >= sizeof(storedir))) {
+        exit(1);
+    }
+    ret = sys_mkdir(storedir, 0755);
+    if ((-1 == ret) && (errno != EEXIST)) {
+        gf_msg(this->name, GF_LOG_CRITICAL, errno, GD_MSG_CREATE_DIR_FAILED,
+               "Unable to create glustershd directory %s"
+               " ,errno = %d",
+               storedir, errno);
+        exit(1);
+    }
+
+    len = snprintf(storedir, sizeof(storedir), "%s/quotad", workdir);
+    if ((len < 0) || (len >= sizeof(storedir))) {
+        exit(1);
+    }
+    ret = sys_mkdir(storedir, 0755);
+    if ((-1 == ret) && (errno != EEXIST)) {
+        gf_msg(this->name, GF_LOG_CRITICAL, errno, GD_MSG_CREATE_DIR_FAILED,
+               "Unable to create quotad directory %s"
+               " ,errno = %d",
+               storedir, errno);
+        exit(1);
+    }
+
+    len = snprintf(storedir, sizeof(storedir), "%s/groups", workdir);
+    if ((len < 0) || (len >= sizeof(storedir))) {
+        exit(1);
+    }
+    ret = sys_mkdir(storedir, 0755);
+    if ((-1 == ret) && (errno != EEXIST)) {
+        gf_msg(this->name, GF_LOG_CRITICAL, errno, GD_MSG_CREATE_DIR_FAILED,
+               "Unable to create glustershd directory %s"
+               " ,errno = %d",
+               storedir, errno);
+        exit(1);
+    }
+
+    ret = glusterd_rpcsvc_options_build(this->options);
+    if (ret)
+        goto out;
+    rpc = rpcsvc_init(this, this->ctx, this->options, 64);
+    if (rpc == NULL) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_RPC_INIT_FAIL,
+               "failed to init rpc");
+        goto out;
+    }
+
+    ret = rpcsvc_register_notify(rpc, glusterd_rpcsvc_notify, this);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_RPCSVC_REG_NOTIFY_RETURNED,
+               "rpcsvc_register_notify returned %d", ret);
+        goto out;
+    }
+
+    /* Enable encryption for the TCP listener is management encryption is
+     * enabled
+     */
+    if (this->ctx->secure_mgmt) {
+        ret = dict_set_str(this->options, "transport.socket.ssl-enabled", "on");
+        if (ret != 0) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
+                   "failed to set ssl-enabled in dict");
+            goto out;
         }
-
         /*
-         * only one (at most a pair - rdma and socket) listener for
-         * glusterd1_mop_prog, gluster_pmap_prog and gluster_handshake_prog.
+         * This is the only place where we want secure_srvr to reflect
+         * the management-plane setting.
          */
-
-        ret = dict_get_str (this->options, "transport-type", &transport_type);
+        this->ctx->secure_srvr = MGMT_SSL_ALWAYS;
+    }
+
+    /*
+     * only one (at most a pair - rdma and socket) listener for
+     * glusterd1_mop_prog, gluster_pmap_prog and gluster_handshake_prog.
+     */
+
+    ret = dict_get_str(this->options, "transport-type", &transport_type);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
+               "Failed to get transport type");
+        ret = -1;
+        goto out;
+    }
+
+    total_transport = rpc_transport_count(transport_type);
+    if (total_transport <= 0) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_RPC_TRANSPORT_COUNT_GET_FAIL,
+               "failed to get total number of available tranpsorts");
+        ret = -1;
+        goto out;
+    }
+
+    ret = rpcsvc_create_listeners(rpc, this->options, this->name);
+    if (ret < 1) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_RPC_LISTENER_CREATE_FAIL,
+               "creation of listener failed");
+        ret = -1;
+        goto out;
+    } else if (ret < total_transport) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_RPC_LISTENER_CREATE_FAIL,
+               "creation of %d listeners failed, continuing with "
+               "succeeded transport",
+               (total_transport - ret));
+    }
+
+    for (i = 0; i < gd_inet_programs_count; i++) {
+        ret = glusterd_program_register(this, rpc, gd_inet_programs[i]);
         if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "Failed to get transport type");
-                ret = -1;
-                goto out;
-        }
-
-        total_transport = rpc_transport_count (transport_type);
-        if (total_transport <= 0) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "failed to get total number of available tranpsorts");
-                ret = -1;
-                goto out;
-        }
-
-        ret = rpcsvc_create_listeners (rpc, this->options, this->name);
-        if (ret < 1) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "creation of listener failed");
-                ret = -1;
-                goto out;
-        } else if (ret < total_transport) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "creation of %d listeners failed, continuing with "
-                        "succeeded transport", (total_transport - ret));
-        }
-
-        for (i = 0; i < gd_inet_programs_count; i++) {
-                ret = glusterd_program_register (this, rpc,
-                                                 gd_inet_programs[i]);
-                if (ret) {
-                        i--;
-                        for (; i >= 0; i--)
-                                rpcsvc_program_unregister (rpc,
-                                                           gd_inet_programs[i]);
-
-                        goto out;
-                }
-        }
-
-        /*
-         * Start a unix domain socket listener just for cli commands This
-         * should prevent ports from being wasted by being in TIMED_WAIT when
-         * cli commands are done continuously
-         */
-        uds_rpc = glusterd_init_uds_listener (this);
-        if (uds_rpc == NULL) {
-                ret = -1;
-                goto out;
-        }
-
-        conf = GF_CALLOC (1, sizeof (glusterd_conf_t),
-                          gf_gld_mt_glusterd_conf_t);
-        GF_VALIDATE_OR_GOTO(this->name, conf, out);
-
-        CDS_INIT_LIST_HEAD (&conf->peers);
-        CDS_INIT_LIST_HEAD (&conf->volumes);
-        CDS_INIT_LIST_HEAD (&conf->snapshots);
-        CDS_INIT_LIST_HEAD (&conf->missed_snaps_list);
-
-        pthread_mutex_init (&conf->mutex, NULL);
-        conf->rpc = rpc;
-        conf->uds_rpc = uds_rpc;
-        conf->gfs_mgmt = &gd_brick_prog;
-        strncpy (conf->workdir, workdir, PATH_MAX);
-
-        synclock_init (&conf->big_lock);
-        pthread_mutex_init (&conf->xprt_lock, NULL);
-        INIT_LIST_HEAD (&conf->xprt_list);
-
-        glusterd_friend_sm_init ();
-        glusterd_op_sm_init ();
-        glusterd_opinfo_init ();
-        ret = glusterd_sm_tr_log_init (&conf->op_sm_log,
-                                       glusterd_op_sm_state_name_get,
-                                       glusterd_op_sm_event_name_get,
-                                       GLUSTERD_TR_LOG_SIZE);
-        if (ret)
-                goto out;
-
-         conf->base_port = GF_IANA_PRIV_PORTS_START;
-         if (dict_get_uint32(this->options, "base-port", &conf->base_port) == 0) {
-                 gf_log (this->name, GF_LOG_INFO,
-                         "base-port override: %d", conf->base_port);
-         }
-
-        /* Set option to run bricks on valgrind if enabled in glusterd.vol */
-        conf->valgrind = _gf_false;
-        ret = dict_get_str (this->options, "run-with-valgrind", &valgrind_str);
-        if (ret < 0) {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "cannot get run-with-valgrind value");
-        }
-        if (valgrind_str) {
-                if (gf_string2boolean (valgrind_str, &(conf->valgrind))) {
-                        gf_log (this->name, GF_LOG_WARNING,
-                                "run-with-valgrind value not a boolean string");
-                }
-        }
-
-        this->private = conf;
-        glusterd_mgmt_v3_lock_init ();
-        glusterd_txn_opinfo_dict_init ();
-
-        /* Make install copies few of the hook-scripts by creating hooks
-         * directory. Hence purposefully not doing the check for the presence of
-         * hooks directory. Doing so avoids creation of complete hooks directory
-         * tree.
-         */
-        ret = glusterd_hooks_create_hooks_directory (conf->workdir);
-        if (-1 == ret) {
-                gf_log (this->name, GF_LOG_CRITICAL,
-                        "Unable to create hooks directory ");
-                exit (1);
-        }
-
-        CDS_INIT_LIST_HEAD (&conf->mount_specs);
-
-        ret = dict_foreach (this->options, _install_mount_spec, NULL);
-        if (ret)
-                goto out;
-        ret = dict_get_str (this->options, "mountbroker-root",
-                            &mountbroker_root);
-        if (ret)
-                ret = 0;
+            i--;
+            for (; i >= 0; i--)
+                rpcsvc_program_unregister(rpc, gd_inet_programs[i]);
+
+            goto out;
+        }
+    }
+
+    /*
+     * Start a unix domain socket listener just for cli commands This
+     * should prevent ports from being wasted by being in TIMED_WAIT when
+     * cli commands are done continuously
+     */
+    uds_rpc = glusterd_init_uds_listener(this);
+    if (uds_rpc == NULL) {
+        ret = -1;
+        goto out;
+    }
+
+    conf = GF_CALLOC(1, sizeof(glusterd_conf_t), gf_gld_mt_glusterd_conf_t);
+    GF_VALIDATE_OR_GOTO(this->name, conf, out);
+
+    CDS_INIT_LIST_HEAD(&conf->peers);
+    CDS_INIT_LIST_HEAD(&conf->volumes);
+    CDS_INIT_LIST_HEAD(&conf->snapshots);
+    CDS_INIT_LIST_HEAD(&conf->missed_snaps_list);
+    CDS_INIT_LIST_HEAD(&conf->brick_procs);
+    CDS_INIT_LIST_HEAD(&conf->shd_procs);
+    pthread_mutex_init(&conf->attach_lock, NULL);
+    pthread_mutex_init(&conf->volume_lock, NULL);
+
+    pthread_mutex_init(&conf->mutex, NULL);
+    conf->rpc = rpc;
+    conf->uds_rpc = uds_rpc;
+    conf->gfs_mgmt = &gd_brick_prog;
+    conf->restart_shd = _gf_false;
+    this->private = conf;
+    /* conf->workdir and conf->rundir are smaller than PATH_MAX; gcc's
+     * snprintf checking will throw an error here if sprintf is used.
+     * Dueling gcc-8 and coverity, now coverity isn't smart enough to
+     * detect that these strncpy calls are safe. And for extra fun,
+     * the annotations don't do anything. */
+    if (strlen(workdir) >= sizeof(conf->workdir)) {
+        ret = -1;
+        goto out;
+    }
+    /* coverity[BUFFER_SIZE_WARNING] */
+    (void)strncpy(conf->workdir, workdir, sizeof(conf->workdir));
+    /* separate tests because combined tests confuses gcc */
+    if (strlen(rundir) >= sizeof(conf->rundir)) {
+        ret = -1;
+        goto out;
+    }
+    /* coverity[BUFFER_SIZE_WARNING] */
+    (void)strncpy(conf->rundir, rundir, sizeof(conf->rundir));
+
+    /* coverity[BUFFER_SIZE_WARNING] */
+    (void)strncpy(conf->logdir, logdir, sizeof(conf->logdir));
+
+    synclock_init(&conf->big_lock, SYNC_LOCK_RECURSIVE);
+    synccond_init(&conf->cond_restart_bricks);
+    synccond_init(&conf->cond_restart_shd);
+    synccond_init(&conf->cond_blockers);
+    pthread_mutex_init(&conf->xprt_lock, NULL);
+    INIT_LIST_HEAD(&conf->xprt_list);
+    pthread_mutex_init(&conf->import_volumes, NULL);
+
+    glusterd_friend_sm_init();
+    glusterd_op_sm_init();
+    glusterd_opinfo_init();
+    ret = glusterd_sm_tr_log_init(
+        &conf->op_sm_log, glusterd_op_sm_state_name_get,
+        glusterd_op_sm_event_name_get, GLUSTERD_TR_LOG_SIZE);
+    if (ret)
+        goto out;
+
+    conf->base_port = GF_IANA_PRIV_PORTS_START;
+    if (dict_get_uint32(this->options, "base-port", &conf->base_port) == 0) {
+        gf_msg(this->name, GF_LOG_INFO, 0, GD_MSG_DICT_SET_FAILED,
+               "base-port override: %d", conf->base_port);
+    }
+    conf->max_port = GF_PORT_MAX;
+    if (dict_get_uint32(this->options, "max-port", &conf->max_port) == 0) {
+        gf_msg(this->name, GF_LOG_INFO, 0, GD_MSG_DICT_SET_FAILED,
+               "max-port override: %d", conf->max_port);
+    }
+
+    conf->mgmt_v3_lock_timeout = GF_LOCK_TIMER;
+    if (dict_get_uint32(this->options, "lock-timer",
+                        &conf->mgmt_v3_lock_timeout) == 0) {
+        gf_msg(this->name, GF_LOG_INFO, 0, GD_MSG_DICT_SET_FAILED,
+               "lock-timer override: %d", conf->mgmt_v3_lock_timeout);
+    }
+
+    /* Set option to run bricks on valgrind if enabled in glusterd.vol */
+    this->ctx->cmd_args.vgtool = vgtool;
+    ret = dict_get_str(this->options, "run-with-valgrind", &valgrind_str);
+    if (ret < 0) {
+        gf_msg_debug(this->name, 0, "cannot get run-with-valgrind value");
+    }
+    if (valgrind_str) {
+        gf_boolean_t vg = _gf_false;
+
+        if (!strcmp(valgrind_str, "memcheck"))
+            this->ctx->cmd_args.vgtool = _gf_memcheck;
+        else if (!strcmp(valgrind_str, "drd"))
+            this->ctx->cmd_args.vgtool = _gf_drd;
+        else if (!gf_string2boolean(valgrind_str, &vg))
+            this->ctx->cmd_args.vgtool = (vg ? _gf_memcheck : _gf_none);
         else
-                ret = check_prepare_mountbroker_root (mountbroker_root);
-        if (ret)
-                goto out;
+            gf_msg(this->name, GF_LOG_WARNING, EINVAL, GD_MSG_INVALID_ENTRY,
+                   "run-with-valgrind is neither boolean"
+                   " nor one of 'memcheck' or 'drd'");
+    }
 
-        ret = configure_syncdaemon (conf);
-        if (ret)
-                goto out;
-
-        /* Restoring op-version needs to be done before initializing the
-         * services as glusterd_svc_init_common () invokes
-         * glusterd_conn_build_socket_filepath () which uses MY_UUID macro.
-         * MY_UUID generates a new uuid if its not been generated and writes it
-         * in the info file, Since the op-version is not read yet
-         * the default value i.e. 0 will be written for op-version and restore
-         * will fail. This is why restoring op-version needs to happen before
-         * service initialization
-         * */
-        ret = glusterd_restore_op_version (this);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "Failed to restore op_version");
-                goto out;
-        }
+    /* Store ping-timeout in conf */
+    ret = dict_get_int32(this->options, "ping-timeout", &conf->ping_timeout);
+    /* Not failing here since ping-timeout can be optional as well */
 
-        ret = glusterd_svc_init_all ();
-        if (ret)
-                goto out;
+    glusterd_mgmt_v3_lock_init();
+    glusterd_mgmt_v3_lock_timer_init();
+    glusterd_txn_opinfo_dict_init();
 
-        ret = glusterd_restore ();
-        if (ret < 0)
-                goto out;
+#ifdef BUILD_GNFS
+    glusterd_nfssvc_build(&conf->nfs_svc);
+#endif
+    glusterd_quotadsvc_build(&conf->quotad_svc);
+    glusterd_bitdsvc_build(&conf->bitd_svc);
+    glusterd_scrubsvc_build(&conf->scrub_svc);
+
+    /* Make install copies few of the hook-scripts by creating hooks
+     * directory. Hence purposefully not doing the check for the presence of
+     * hooks directory. Doing so avoids creation of complete hooks directory
+     * tree.
+     */
+    ret = glusterd_hooks_create_hooks_directory(conf->workdir);
+    if (-1 == ret) {
+        gf_msg(this->name, GF_LOG_CRITICAL, errno, GD_MSG_DIR_OP_FAILED,
+               "Unable to create hooks directory ");
+        exit(1);
+    }
+
+    CDS_INIT_LIST_HEAD(&conf->mount_specs);
+
+    ret = dict_foreach(this->options, _install_mount_spec, NULL);
+    if (ret)
+        goto out;
+    ret = dict_get_str(this->options, "mountbroker-root", &mountbroker_root);
+    if (ret)
+        ret = 0;
+    else
+        ret = check_prepare_mountbroker_root(mountbroker_root);
+    if (ret)
+        goto out;
 
-        /* If there are no 'friends', this would be the best time to
-         * spawn process/bricks that may need (re)starting since last
-         * time (this) glusterd was up.*/
+    ret = is_upgrade(this->options, &upgrade);
+    if (ret)
+        goto out;
 
-        if (cds_list_empty (&conf->peers)) {
-                glusterd_launch_synctask (glusterd_spawn_daemons, NULL);
-        }
-        ret = glusterd_options_init (this);
-        if (ret < 0)
-                goto out;
+    ret = is_downgrade(this->options, &downgrade);
+    if (ret)
+        goto out;
 
-        ret = glusterd_handle_upgrade_downgrade (this->options, conf);
+    if (!upgrade && !downgrade) {
+        ret = configure_syncdaemon(conf);
         if (ret)
-                goto out;
-
-        ret = glusterd_hooks_spawn_worker (this);
+            goto out;
+    }
+
+    /* Restoring op-version needs to be done before initializing the
+     * services as glusterd_svc_init_common () invokes
+     * glusterd_conn_build_socket_filepath () which uses MY_UUID macro.
+     * MY_UUID generates a new uuid if its not been generated and writes it
+     * in the info file, Since the op-version is not read yet
+     * the default value i.e. 0 will be written for op-version and restore
+     * will fail. This is why restoring op-version needs to happen before
+     * service initialization
+     * */
+    ret = glusterd_restore_op_version(this);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_OP_VERS_RESTORE_FAIL,
+               "Failed to restore op_version");
+        goto out;
+    }
+
+    ret = glusterd_restore();
+    if (ret < 0)
+        goto out;
+
+    if (dict_get_str(conf->opts, GLUSTERD_LOCALTIME_LOGGING_KEY,
+                     &localtime_logging) == 0) {
+        int already_enabled = gf_log_get_localtime();
+
+        if (strcmp(localtime_logging, "enable") == 0) {
+            gf_log_set_localtime(1);
+            if (!already_enabled)
+                gf_msg(this->name, GF_LOG_INFO, 0,
+                       GD_MSG_LOCALTIME_LOGGING_ENABLE,
+                       "localtime logging enable");
+        } else if (strcmp(localtime_logging, "disable") == 0) {
+            gf_log_set_localtime(0);
+            if (already_enabled)
+                gf_msg(this->name, GF_LOG_INFO, 0,
+                       GD_MSG_LOCALTIME_LOGGING_DISABLE,
+                       "localtime logging disable");
+        }
+    }
+
+    GF_ATOMIC_INIT(conf->blockers, 0);
+    ret = glusterd_handle_upgrade_downgrade(this->options, conf, upgrade,
+                                            downgrade);
+    if (ret)
+        goto out;
+
+    ret = glusterd_retrieve_max_op_version(this, &op_version);
+    /* first condition indicates file isn't present which means this code
+     * change is hitting for the first time or someone has deleted it from the
+     * backend.second condition is when max op_version differs, in both cases
+     * volfiles should be regenerated
+     */
+    if (op_version == 0 || op_version != GD_OP_VERSION_MAX) {
+        gf_log(this->name, GF_LOG_INFO,
+               "Regenerating volfiles due to a max op-version mismatch or "
+               "glusterd.upgrade file not being present, op_version retrieved:"
+               "%d, max op_version: %d",
+               op_version, GD_OP_VERSION_MAX);
+        glusterd_recreate_volfiles(conf);
+        ret = glusterd_store_max_op_version(this);
+        if (ret)
+            gf_log(this->name, GF_LOG_ERROR, "Failed to store max op-version");
+    }
+
+    /* If the peer count is less than 2 then this would be the best time to
+     * spawn process/bricks that may need (re)starting since last time
+     * (this) glusterd was up. */
+    if (glusterd_get_peers_count() < 2)
+        glusterd_launch_synctask(glusterd_spawn_daemons, NULL);
+
+    ret = glusterd_hooks_spawn_worker(this);
+    if (ret)
+        goto out;
+
+    GF_OPTION_INIT("event-threads", workers, int32, out);
+    if (workers > 0 && workers != conf->workers) {
+        conf->workers = workers;
+        ret = gf_event_reconfigure_threads(this->ctx->event_pool, workers);
         if (ret)
-                goto out;
+            goto out;
+    }
 
-        ret = 0;
+    ret = 0;
 out:
-        if (ret < 0) {
-                if (this->private != NULL) {
-                        GF_FREE (this->private);
-                        this->private = NULL;
-                }
-
+    if (ret < 0) {
+        if (this->private != NULL) {
+            GF_FREE(this->private);
+            this->private = NULL;
         }
+    }
 
-        return ret;
+    return ret;
 }
 
-
-
-
 /*
  * fini - finish function for glusterd, called before
  *        unloading gluster.
@@ -1689,27 +2108,37 @@ out:
  *
  */
 void
-fini (xlator_t *this)
+fini(xlator_t *this)
 {
-        glusterd_conf_t *conf = NULL;
-        if (!this || !this->private)
-                goto out;
+    if (!this || !this->private)
+        goto out;
 
-        conf = this->private;
+    glusterd_stop_uds_listener(this); /*stop unix socket rpc*/
+    glusterd_stop_listener(this);     /*stop tcp/ip socket rpc*/
 
-        glusterd_stop_uds_listener (this);
+#if 0
+       /* Running threads might be using these resourses, we have to cancel/stop
+        * running threads before deallocating the memory, but we don't have
+        * control over the running threads to do pthread_cancel().
+        * So memory freeing handover to kernel.
+        */
+        /*TODO: cancel/stop the running threads*/
 
+        GF_FREE (conf->uds_rpc);
+        GF_FREE (conf->rpc);
         FREE (conf->pmap);
         if (conf->handle)
                 gf_store_handle_destroy (conf->handle);
         glusterd_sm_tr_log_delete (&conf->op_sm_log);
         glusterd_mgmt_v3_lock_fini ();
+        glusterd_mgmt_v3_lock_timer_fini ();
         glusterd_txn_opinfo_dict_fini ();
         GF_FREE (conf);
 
         this->private = NULL;
+#endif
 out:
-        return;
+    return;
 }
 
 /*
@@ -1720,116 +2149,157 @@ out:
  *
  */
 int
-notify (xlator_t *this, int32_t event, void *data, ...)
+notify(xlator_t *this, int32_t event, void *data, ...)
 {
-        int          ret = 0;
-
-        switch (event) {
-                case GF_EVENT_POLLIN:
-                        break;
+    int ret = 0;
 
-                case GF_EVENT_POLLERR:
-                        break;
+    switch (event) {
+        case GF_EVENT_POLLIN:
+            break;
 
-                case GF_EVENT_TRANSPORT_CLEANUP:
-                        break;
+        case GF_EVENT_POLLERR:
+            break;
 
-                default:
-                        default_notify (this, event, data);
-                        break;
+        case GF_EVENT_CLEANUP:
+            break;
 
-        }
+        default:
+            default_notify(this, event, data);
+            break;
+    }
 
-        return ret;
+    return ret;
 }
 
-
 struct xlator_fops fops;
 
 struct xlator_cbks cbks;
 
 struct xlator_dumpops dumpops = {
-        .priv  = glusterd_dump_priv,
+    .priv = glusterd_dump_priv,
 };
 
-
 struct volume_options options[] = {
-        { .key   = {"working-directory"},
-          .type  = GF_OPTION_TYPE_PATH,
-        },
-        { .key   = {"transport-type"},
-          .type  = GF_OPTION_TYPE_ANY,
-        },
-        { .key   = {"transport.*"},
-          .type  = GF_OPTION_TYPE_ANY,
-        },
-        { .key   = {"rpc-auth.*"},
-          .type  = GF_OPTION_TYPE_ANY,
-        },
-        { .key   = {"rpc-auth-allow-insecure"},
-          .type  = GF_OPTION_TYPE_BOOL,
-        },
-        { .key  = {"upgrade"},
-          .type = GF_OPTION_TYPE_BOOL,
-        },
-        { .key  = {"downgrade"},
-          .type = GF_OPTION_TYPE_BOOL,
-        },
-        { .key = {"bind-insecure"},
-          .type = GF_OPTION_TYPE_BOOL,
-        },
-        { .key  = {"mountbroker-root"},
-          .type = GF_OPTION_TYPE_PATH,
-        },
-        { .key  = {"mountbroker.*"},
-          .type = GF_OPTION_TYPE_ANY,
-        },
-        { .key  = {"mountbroker-"GEOREP".*"},
-          .type = GF_OPTION_TYPE_ANY,
-        },
-        { .key  = {"mountbroker-"GHADOOP".*"},
-          .type = GF_OPTION_TYPE_ANY,
-        },
-        { .key = {GEOREP"-log-group"},
-          .type = GF_OPTION_TYPE_ANY,
-        },
-        { .key = {"run-with-valgrind"},
-          .type = GF_OPTION_TYPE_BOOL,
-        },
-        { .key = {"server-quorum-type"},
-          .type = GF_OPTION_TYPE_STR,
-          .value = { "none", "server"},
-          .description = "This feature is on the server-side i.e. in glusterd."
-                         " Whenever the glusterd on a machine observes that "
-                         "the quorum is not met, it brings down the bricks to "
-                         "prevent data split-brains. When the network "
-                         "connections are brought back up and the quorum is "
-                         "restored the bricks in the volume are brought back "
-                         "up."
-        },
-        { .key = {"server-quorum-ratio"},
-          .type = GF_OPTION_TYPE_PERCENT,
-          .description = "Sets the quorum percentage for the trusted "
-          "storage pool."
-        },
-        { .key = {"glusterd-sockfile"},
-          .type = GF_OPTION_TYPE_PATH,
-          .description = "The socket file on which glusterd should listen for "
-                        "cli requests. Default is "DEFAULT_GLUSTERD_SOCKFILE "."
-        },
-        { .key = {"base-port"},
-          .type = GF_OPTION_TYPE_INT,
-          .description = "Sets the base port for portmap query"
-        },
-        { .key = {"snap-brick-path"},
-          .type = GF_OPTION_TYPE_STR,
-          .description = "directory where the bricks for the snapshots will be created"
-        },
-        { .key  = {"ping-timeout"},
-          .type = GF_OPTION_TYPE_TIME,
-          .min  = 0,
-          .max  = 300,
-          .default_value = TOSTRING(RPC_DEFAULT_PING_TIMEOUT),
-        },
-        { .key   = {NULL} },
+    {
+        .key = {"working-directory"},
+        .type = GF_OPTION_TYPE_PATH,
+    },
+    {
+        .key = {"transport-type"},
+        .type = GF_OPTION_TYPE_ANY,
+    },
+    {
+        .key = {"transport.*"},
+        .type = GF_OPTION_TYPE_ANY,
+    },
+    {
+        .key = {"rpc-auth.*"},
+        .type = GF_OPTION_TYPE_ANY,
+    },
+    {
+        .key = {"rpc-auth-allow-insecure"},
+        .type = GF_OPTION_TYPE_BOOL,
+    },
+    {
+        .key = {"upgrade"},
+        .type = GF_OPTION_TYPE_BOOL,
+    },
+    {
+        .key = {"downgrade"},
+        .type = GF_OPTION_TYPE_BOOL,
+    },
+    {
+        .key = {"bind-insecure"},
+        .type = GF_OPTION_TYPE_BOOL,
+    },
+    {
+        .key = {"mountbroker-root"},
+        .type = GF_OPTION_TYPE_PATH,
+    },
+    {
+        .key = {"mountbroker.*"},
+        .type = GF_OPTION_TYPE_ANY,
+    },
+    {
+        .key = {"mountbroker-" GEOREP ".*"},
+        .type = GF_OPTION_TYPE_ANY,
+    },
+    {
+        .key = {GEOREP "-log-group"},
+        .type = GF_OPTION_TYPE_ANY,
+    },
+    {
+        .key = {"run-with-valgrind"},
+        .type = GF_OPTION_TYPE_BOOL,
+    },
+    {.key = {"server-quorum-type"},
+     .type = GF_OPTION_TYPE_STR,
+     .value = {"none", "server"},
+     .default_value = "none",
+     .description = "It can be set to none or server. When set to server, "
+                    "this option enables the specified volume to "
+                    "participate in the server-side quorum. "
+                    "This feature is on the server-side i.e. in glusterd. "
+                    "Whenever the glusterd on a machine observes that "
+                    "the quorum is not met, it brings down the bricks to "
+                    "prevent data split-brains. When the network "
+                    "connections are brought back up and the quorum is "
+                    "restored the bricks in   "
+                    "the volume are brought back up."},
+    {.key = {"server-quorum-ratio"},
+     .type = GF_OPTION_TYPE_PERCENT,
+     .description = "Sets the quorum percentage for the trusted "
+                    "storage pool."},
+    {.key = {"glusterd-sockfile"},
+     .type = GF_OPTION_TYPE_PATH,
+     .description = "The socket file on which glusterd should listen for "
+                    "cli requests. Default is " DEFAULT_GLUSTERD_SOCKFILE "."},
+    {.key = {"base-port"},
+     .type = GF_OPTION_TYPE_INT,
+     .description = "Sets the base port for portmap query"},
+    {.key = {"max-port"},
+     .type = GF_OPTION_TYPE_INT,
+     .max = GF_PORT_MAX,
+     .description = "Sets the max port for portmap query"},
+    {.key = {"mgmt-v3-lock-timeout"},
+     .type = GF_OPTION_TYPE_INT,
+     .max = 600,
+     .description = "Sets the mgmt-v3-lock-timeout for transactions."
+                    "Specifes the default timeout value after which "
+                    "lock acquired while performing transaction will "
+                    "be released."},
+    {.key = {"snap-brick-path"},
+     .type = GF_OPTION_TYPE_STR,
+     .description =
+         "directory where the bricks for the snapshots will be created"},
+    {
+        .key = {"ping-timeout"},
+        .type = GF_OPTION_TYPE_TIME,
+        .min = 0,
+        .max = 300,
+        .default_value = TOSTRING(RPC_DEFAULT_PING_TIMEOUT),
+    },
+    {.key = {"event-threads"},
+     .type = GF_OPTION_TYPE_INT,
+     .min = 1,
+     .max = 32,
+     .default_value = "2",
+     .description = "Specifies the number of event threads to execute "
+                    "in parallel. Larger values would help process"
+                    " responses faster, depending on available processing"
+                    " power. Range 1-32 threads."},
+    {.key = {NULL}},
+};
+
+xlator_api_t xlator_api = {
+    .init = init,
+    .fini = fini,
+    .mem_acct_init = mem_acct_init,
+    .op_version = {1}, /* Present from the initial version */
+    .dumpops = &dumpops,
+    .fops = &fops,
+    .cbks = &cbks,
+    .options = options,
+    .identifier = "glusterd",
+    .category = GF_MAINTAINED,
 };
diff --git a/xlators/mgmt/glusterd/src/glusterd.h b/xlators/mgmt/glusterd/src/glusterd.h
index 480cfc66269..cc4f98ecf47 100644
--- a/xlators/mgmt/glusterd/src/glusterd.h
+++ b/xlators/mgmt/glusterd/src/glusterd.h
@@ -10,479 +10,613 @@
 #ifndef _GLUSTERD_H_
 #define _GLUSTERD_H_
 
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
-
 #include <sys/types.h>
 #include <dirent.h>
 #include <pthread.h>
 #include <libgen.h>
 
-#include "compat-uuid.h"
+#include <glusterfs/compat-uuid.h>
 
 #include "rpc-clnt.h"
-#include "glusterfs.h"
-#include "xlator.h"
-#include "logging.h"
-#include "call-stub.h"
-#include "fd.h"
-#include "byte-order.h"
+#include <glusterfs/glusterfs.h>
+#include <glusterfs/xlator.h>
+#include <glusterfs/logging.h>
+#include <glusterfs/call-stub.h>
+#include <glusterfs/byte-order.h>
 #include "glusterd-mem-types.h"
 #include "rpcsvc.h"
 #include "glusterd-sm.h"
 #include "glusterd-snapd-svc.h"
+#include "glusterd-shd-svc.h"
 #include "glusterd-bitd-svc.h"
 #include "glusterd1-xdr.h"
 #include "protocol-common.h"
 #include "glusterd-pmap.h"
 #include "cli1-xdr.h"
-#include "syncop.h"
-#include "store.h"
+#include <glusterfs/syncop.h>
+#include <glusterfs/store.h>
 #include "glusterd-rcu.h"
+#include <glusterfs/events.h>
+#include "glusterd-gfproxyd-svc.h"
+
+#include "gd-common-utils.h"
+
+#define GLUSTERD_TR_LOG_SIZE 50
+#define GLUSTERD_QUORUM_TYPE_KEY "cluster.server-quorum-type"
+#define GLUSTERD_QUORUM_RATIO_KEY "cluster.server-quorum-ratio"
+#define GLUSTERD_GLOBAL_OPT_VERSION "global-option-version"
+#define GLUSTERD_GLOBAL_OP_VERSION_KEY "cluster.op-version"
+#define GLUSTERD_MAX_OP_VERSION_KEY "cluster.max-op-version"
+#define GLUSTERD_COMMON_PEM_PUB_FILE "/geo-replication/common_secret.pem.pub"
+#define GEO_CONF_MAX_OPT_VALS 6
+#define GLUSTERD_CREATE_HOOK_SCRIPT                                            \
+    "/hooks/1/gsync-create/post/"                                              \
+    "S56glusterd-geo-rep-create-post.sh"
+#define GLUSTERD_SHRD_STRG_HOOK_SCRIPT                                         \
+    "/hooks/1/set/post/"                                                       \
+    "S32gluster_enable_shared_storage.sh"
+#define GLUSTER_SHARED_STORAGE "gluster_shared_storage"
+#define GLUSTERD_SHARED_STORAGE_KEY "cluster.enable-shared-storage"
+#define GLUSTERD_BRICK_MULTIPLEX_KEY "cluster.brick-multiplex"
+#define GLUSTERD_VOL_CNT_PER_THRD "glusterd.vol_count_per_thread"
+#define GLUSTERD_BRICKMUX_LIMIT_KEY "cluster.max-bricks-per-process"
+#define GLUSTERD_BRICKMUX_LIMIT_DFLT_VALUE "250"
+#define GLUSTERD_VOL_CNT_PER_THRD_DEFAULT_VALUE "100"
+#define GLUSTERD_LOCALTIME_LOGGING_KEY "cluster.localtime-logging"
+#define GLUSTERD_DAEMON_LOG_LEVEL_KEY "cluster.daemon-log-level"
+
+#define GANESHA_HA_CONF CONFDIR "/ganesha-ha.conf"
+#define GANESHA_EXPORT_DIRECTORY CONFDIR "/exports"
 
-#define GLUSTERD_TR_LOG_SIZE            50
-#define GLUSTERD_NAME                   "glusterd"
-#define GLUSTERD_SOCKET_LISTEN_BACKLOG  128
-#define GLUSTERD_QUORUM_TYPE_KEY        "cluster.server-quorum-type"
-#define GLUSTERD_QUORUM_RATIO_KEY       "cluster.server-quorum-ratio"
-#define GLUSTERD_GLOBAL_OPT_VERSION     "global-option-version"
-#define GLUSTERD_COMMON_PEM_PUB_FILE    "/geo-replication/common_secret.pem.pub"
-#define GEO_CONF_MAX_OPT_VALS           6
-#define GLUSTERD_CREATE_HOOK_SCRIPT     "/hooks/1/gsync-create/post/" \
-                                        "S56glusterd-geo-rep-create-post.sh"
-
-#define GANESHA_HA_CONF  CONFDIR "/ganesha-ha.conf"
 #define GLUSTERD_SNAPS_MAX_HARD_LIMIT 256
 #define GLUSTERD_SNAPS_DEF_SOFT_LIMIT_PERCENT 90
 #define GLUSTERD_SNAPS_MAX_SOFT_LIMIT_PERCENT 100
 #define GLUSTERD_SERVER_QUORUM "server"
+#define STATUS_STRLEN 128
 
 #define FMTSTR_CHECK_VOL_EXISTS "Volume %s does not exist"
 #define FMTSTR_RESOLVE_BRICK "Could not find peer on which brick %s:%s resides"
 
-#define LOGSTR_FOUND_BRICK  "Found brick %s:%s in volume %s"
+#define LOGSTR_FOUND_BRICK "Found brick %s:%s in volume %s"
 #define LOGSTR_BUILD_PAYLOAD "Failed to build payload for operation 'Volume %s'"
 #define LOGSTR_STAGE_FAIL "Staging of operation 'Volume %s' failed on %s %s %s"
 #define LOGSTR_COMMIT_FAIL "Commit of operation 'Volume %s' failed on %s %s %s"
 
-#define OPERRSTR_BUILD_PAYLOAD "Failed to build payload. Please check the log "\
-                               "file for more details."
-#define OPERRSTR_STAGE_FAIL "Staging failed on %s. Please check the log file " \
-                            "for more details."
-#define OPERRSTR_COMMIT_FAIL "Commit failed on %s. Please check the log file "\
-                             "for more details."
-
+#define OPERRSTR_BUILD_PAYLOAD                                                 \
+    "Failed to build payload. Please check the log "                           \
+    "file for more details."
+#define OPERRSTR_STAGE_FAIL                                                    \
+    "Staging failed on %s. Please check the log file "                         \
+    "for more details."
+#define OPERRSTR_COMMIT_FAIL                                                   \
+    "Commit failed on %s. Please check the log file "                          \
+    "for more details."
 struct glusterd_volinfo_;
 typedef struct glusterd_volinfo_ glusterd_volinfo_t;
 
 struct glusterd_snap_;
 typedef struct glusterd_snap_ glusterd_snap_t;
 
+/* For every new feature please add respective enum of new feature
+ * at the end of latest enum (just before the GD_OP_MAX enum)
+ */
 typedef enum glusterd_op_ {
-        GD_OP_NONE = 0,
-        GD_OP_CREATE_VOLUME,
-        GD_OP_START_BRICK,
-        GD_OP_STOP_BRICK,
-        GD_OP_DELETE_VOLUME,
-        GD_OP_START_VOLUME,
-        GD_OP_STOP_VOLUME,
-        GD_OP_DEFRAG_VOLUME,
-        GD_OP_ADD_BRICK,
-        GD_OP_REMOVE_BRICK,
-        GD_OP_REPLACE_BRICK,
-        GD_OP_SET_VOLUME,
-        GD_OP_RESET_VOLUME,
-        GD_OP_SYNC_VOLUME,
-        GD_OP_LOG_ROTATE,
-        GD_OP_GSYNC_SET,
-        GD_OP_PROFILE_VOLUME,
-        GD_OP_QUOTA,
-        GD_OP_STATUS_VOLUME,
-        GD_OP_REBALANCE,
-        GD_OP_HEAL_VOLUME,
-        GD_OP_STATEDUMP_VOLUME,
-        GD_OP_LIST_VOLUME,
-        GD_OP_CLEARLOCKS_VOLUME,
-        GD_OP_DEFRAG_BRICK_VOLUME,
-        GD_OP_COPY_FILE,
-        GD_OP_SYS_EXEC,
-        GD_OP_GSYNC_CREATE,
-        GD_OP_SNAP,
-        GD_OP_BARRIER,
-        GD_OP_GANESHA,
-        GD_OP_BITROT,
-        GD_OP_MAX,
+    GD_OP_NONE = 0,
+    GD_OP_CREATE_VOLUME,
+    GD_OP_START_BRICK,
+    GD_OP_STOP_BRICK,
+    GD_OP_DELETE_VOLUME,
+    GD_OP_START_VOLUME,
+    GD_OP_STOP_VOLUME,
+    GD_OP_DEFRAG_VOLUME,
+    GD_OP_ADD_BRICK,
+    GD_OP_REMOVE_BRICK,
+    GD_OP_REPLACE_BRICK,
+    GD_OP_SET_VOLUME,
+    GD_OP_RESET_VOLUME,
+    GD_OP_SYNC_VOLUME,
+    GD_OP_LOG_ROTATE,
+    GD_OP_GSYNC_SET,
+    GD_OP_PROFILE_VOLUME,
+    GD_OP_QUOTA,
+    GD_OP_STATUS_VOLUME,
+    GD_OP_REBALANCE,
+    GD_OP_HEAL_VOLUME,
+    GD_OP_STATEDUMP_VOLUME,
+    GD_OP_LIST_VOLUME,
+    GD_OP_CLEARLOCKS_VOLUME,
+    GD_OP_DEFRAG_BRICK_VOLUME,
+    GD_OP_COPY_FILE,
+    GD_OP_SYS_EXEC,
+    GD_OP_GSYNC_CREATE,
+    GD_OP_SNAP,
+    GD_OP_BARRIER,
+    GD_OP_GANESHA, /* obsolete */
+    GD_OP_BITROT,
+    GD_OP_DETACH_TIER,
+    GD_OP_TIER_MIGRATE,
+    GD_OP_SCRUB_STATUS,
+    GD_OP_SCRUB_ONDEMAND,
+    GD_OP_RESET_BRICK,
+    GD_OP_MAX_OPVERSION,
+    GD_OP_TIER_START_STOP,
+    GD_OP_TIER_STATUS,
+    GD_OP_DETACH_TIER_STATUS,
+    GD_OP_DETACH_NOT_STARTED,
+    GD_OP_REMOVE_TIER_BRICK,
+    GD_OP_ADD_TIER_BRICK,
+    GD_OP_MAX,
 } glusterd_op_t;
 
-extern const char * gd_op_list[];
+extern const char *gd_op_list[];
 
 struct glusterd_volgen {
-        dict_t *dict;
+    dict_t *dict;
 };
 
+/* Keeping all the paths required in glusterd would
+   cause many buffer overflow errors, as we append
+   more defined paths to the brick path, workdir etc etc.
+   It is better to keep limit on this as lesser value,
+   so we get an option to continue with all functionalities.
+   For example, snapname max would be appended on brick-path and
+   would be stored on workdir... all of these being PATH_MAX, is
+   not an ideal situation for success. */
+#define VALID_GLUSTERD_PATHMAX (PATH_MAX - (256 + 64))
+
 typedef struct {
-        struct _volfile_ctx     *volfile;
-        pthread_mutex_t          mutex;
-        struct cds_list_head     peers;
-        gf_boolean_t             verify_volfile_checksum;
-        gf_boolean_t             trace;
-        uuid_t                   uuid;
-        char                     workdir[PATH_MAX];
-        rpcsvc_t                *rpc;
-        glusterd_svc_t           shd_svc;
-        glusterd_svc_t           nfs_svc;
-        glusterd_svc_t           bitd_svc;
-        glusterd_svc_t           scrub_svc;
-        glusterd_svc_t           quotad_svc;
-        struct pmap_registry    *pmap;
-        struct cds_list_head     volumes;
-        struct cds_list_head     snapshots; /*List of snap volumes */
-        pthread_mutex_t          xprt_lock;
-        struct list_head         xprt_list;
-        gf_store_handle_t       *handle;
-        gf_timer_t              *timer;
-        glusterd_sm_tr_log_t     op_sm_log;
-        struct rpc_clnt_program *gfs_mgmt;
-        dict_t                  *mgmt_v3_lock; /* Dict for saving
-                                                * mgmt_v3 locks */
-        dict_t                  *glusterd_txn_opinfo; /* Dict for saving
-                                                       * transaction opinfos */
-        uuid_t                   global_txn_id; /* To be used in
-                                                 * heterogeneous
-                                                 * cluster with no
-                                                 * transaction ids */
-
-        struct cds_list_head       mount_specs;
-        gf_boolean_t               valgrind;
-        pthread_t                  brick_thread;
-        void                      *hooks_priv;
-
-        /* need for proper handshake_t */
-        int                        op_version; /* Starts with 1 for 3.3.0 */
-        xlator_t                  *xl;  /* Should be set to 'THIS' before creating thread */
-        gf_boolean_t               pending_quorum_action;
-        dict_t                    *opts;
-        synclock_t                 big_lock;
-        gf_boolean_t               restart_done;
-        rpcsvc_t                  *uds_rpc; /* RPCSVC for the unix domain socket */
-        uint32_t                   base_port;
-        char                      *snap_bricks_directory;
-        gf_store_handle_t         *missed_snaps_list_shandle;
-        struct cds_list_head       missed_snaps_list;
-        int           ping_timeout;
+    struct _volfile_ctx *volfile;
+    pthread_mutex_t mutex;
+    struct cds_list_head peers;
+    uuid_t uuid;
+    rpcsvc_t *rpc;
+    glusterd_svc_t nfs_svc;
+    glusterd_svc_t bitd_svc;
+    glusterd_svc_t scrub_svc;
+    glusterd_svc_t quotad_svc;
+    struct pmap_registry *pmap;
+    struct cds_list_head volumes;
+    struct cds_list_head snapshots;   /*List of snap volumes */
+    struct cds_list_head brick_procs; /* List of brick processes */
+    struct cds_list_head shd_procs;   /* List of shd processes */
+    pthread_mutex_t xprt_lock;
+    struct list_head xprt_list;
+    pthread_mutex_t import_volumes;
+    gf_store_handle_t *handle;
+    gf_timer_t *timer;
+    glusterd_sm_tr_log_t op_sm_log;
+    struct rpc_clnt_program *gfs_mgmt;
+    dict_t *mgmt_v3_lock;        /* Dict for saving
+                                  * mgmt_v3 locks */
+    dict_t *glusterd_txn_opinfo; /* Dict for saving
+                                  * transaction opinfos */
+    uuid_t global_txn_id;        /* To be used in
+                                  * heterogeneous
+                                  * cluster with no
+                                  * transaction ids */
+
+    dict_t *mgmt_v3_lock_timer;
+    struct cds_list_head mount_specs;
+    pthread_t brick_thread;
+    void *hooks_priv;
+
+    xlator_t *xl; /* Should be set to 'THIS' before creating thread */
+    /* need for proper handshake_t */
+    int op_version; /* Starts with 1 for 3.3.0 */
+    gf_boolean_t pending_quorum_action;
+    gf_boolean_t verify_volfile_checksum;
+    gf_boolean_t trace;
+    gf_boolean_t restart_done;
+    dict_t *opts;
+    synclock_t big_lock;
+    synccond_t cond_restart_bricks;
+    synccond_t cond_restart_shd;
+    synccond_t cond_blockers;
+    rpcsvc_t *uds_rpc; /* RPCSVC for the unix domain socket */
+    uint32_t base_port;
+    uint32_t max_port;
+    char *snap_bricks_directory;
+    gf_store_handle_t *missed_snaps_list_shandle;
+    struct cds_list_head missed_snaps_list;
+    int ping_timeout;
+    uint32_t generation;
+    int32_t workers;
+    uint32_t mgmt_v3_lock_timeout;
+    gf_atomic_t blockers;
+    pthread_mutex_t attach_lock; /* Lock can be per process or a common one */
+    pthread_mutex_t volume_lock; /* We release the big_lock from lot of places
+                                    which might lead the modification of volinfo
+                                    list.
+                                 */
+    gf_atomic_t thread_count;
+    gf_boolean_t restart_bricks;
+    gf_boolean_t restart_shd; /* This flag prevents running two shd manager
+                                 simultaneously
+                              */
+    char workdir[VALID_GLUSTERD_PATHMAX];
+    char rundir[VALID_GLUSTERD_PATHMAX];
+    char logdir[VALID_GLUSTERD_PATHMAX];
 } glusterd_conf_t;
 
+typedef struct glusterd_add_dict_args {
+    xlator_t *this;
+    dict_t *voldict;
+    int start;
+    int end;
+} glusterd_add_dict_args_t;
+
+typedef struct glusterd_friend_synctask_args {
+    char *dict_buf;
+    u_int dictlen;
+} glusterd_friend_synctask_args_t;
 
 typedef enum gf_brick_status {
-        GF_BRICK_STOPPED,
-        GF_BRICK_STARTED,
+    GF_BRICK_STOPPED,
+    GF_BRICK_STARTED,
+    GF_BRICK_STOPPING,
+    GF_BRICK_STARTING
 } gf_brick_status_t;
 
+typedef struct glusterd_brickinfo glusterd_brickinfo_t;
+
+struct glusterd_brick_proc {
+    int port;
+    uint32_t brick_count;
+    struct cds_list_head brick_proc_list;
+    struct cds_list_head bricks;
+};
+
+typedef struct glusterd_brick_proc glusterd_brick_proc_t;
+
 struct glusterd_brickinfo {
-        char               hostname[1024];
-        char               path[PATH_MAX];
-        char               device_path[PATH_MAX];
-        char               mount_dir[PATH_MAX];
-        char               brick_id[1024];/*Client xlator name, AFR changelog name*/
-        char               fstype [NAME_MAX]; /* Brick file-system type */
-        char               mnt_opts [1024]; /* Brick mount options */
-        struct cds_list_head   brick_list;
-        uuid_t             uuid;
-        int                port;
-        int                rdma_port;
-        char              *logfile;
-        gf_boolean_t       signed_in;
-        gf_store_handle_t *shandle;
-        gf_brick_status_t  status;
-        struct rpc_clnt   *rpc;
-        int                decommissioned;
-        char vg[PATH_MAX]; /* FIXME: Use max size for length of vg */
-        int     caps; /* Capability */
-        int32_t            snap_status;
+    struct cds_list_head brick_list;
+    uuid_t uuid;
+    int port;
+    int rdma_port;
+    char *logfile;
+    gf_store_handle_t *shandle;
+    struct rpc_clnt *rpc;
+    int decommissioned;
+    gf_brick_status_t status;
+    int32_t snap_status;
+    /*
+     * The group is used to identify which bricks are part of the same
+     * replica set during brick-volfile generation, so that JBR volfiles
+     * can "cross-connect" the bricks to one another. It is also used by
+     * AFR to load the arbiter xlator in the appropriate brick in case of
+     * a replica 3 volume with arbiter enabled.
+     */
+    uint16_t group;
+    gf_boolean_t port_registered;
+    gf_boolean_t start_triggered;
+
+    /* Below are used for handling the case of multiple bricks sharing
+       the backend filesystem */
+    uint64_t statfs_fsid;
+    pthread_mutex_t restart_mutex;
+    glusterd_brick_proc_t *brick_proc; /* Information regarding mux bricks */
+    struct cds_list_head mux_bricks; /* List to store the bricks in brick_proc*/
+    uint32_t fs_share_count;
+    char hostname[NAME_MAX];
+    char path[VALID_GLUSTERD_PATHMAX];
+    char real_path[VALID_GLUSTERD_PATHMAX];
+    char device_path[VALID_GLUSTERD_PATHMAX];
+    char mount_dir[VALID_GLUSTERD_PATHMAX];
+    char brick_id[1024];   /*Client xlator name, AFR changelog name*/
+    char fstype[NAME_MAX]; /* Brick file-system type */
+    char mnt_opts[1024];   /* Brick mount options */
+    char vg[PATH_MAX];     /* FIXME: Use max size for length of vg */
 };
 
-typedef struct glusterd_brickinfo glusterd_brickinfo_t;
+struct glusterd_gfproxyd_info {
+    char *logfile;
+    short port;
+};
 
 struct gf_defrag_brickinfo_ {
-        char *name;
-        int   files;
-        int   size;
+    char *name;
+    int files;
+    int size;
 };
 
-typedef int (*defrag_cbk_fn_t) (glusterd_volinfo_t *volinfo,
-                                gf_defrag_status_t status);
+typedef int (*defrag_cbk_fn_t)(glusterd_volinfo_t *volinfo,
+                               gf_defrag_status_t status);
 
 struct glusterd_defrag_info_ {
-        uint64_t                     total_files;
-        uint64_t                     total_data;
-        uint64_t                     num_files_lookedup;
-        uint64_t                     total_failures;
-        gf_lock_t                    lock;
-        int                          cmd;
-        pthread_t                    th;
-        gf_defrag_status_t           defrag_status;
-        struct rpc_clnt             *rpc;
-        uint32_t                     connected;
-        char                         mount[1024];
-        char                         databuf[131072];
-        struct gf_defrag_brickinfo_ *bricks; /* volinfo->brick_count */
-
-        defrag_cbk_fn_t              cbk_fn;
+    uint64_t total_files;
+    uint64_t total_data;
+    uint64_t num_files_lookedup;
+    uint64_t total_failures;
+    gf_lock_t lock;
+    int cmd;
+    uint32_t connected;
+    pthread_t th;
+    struct rpc_clnt *rpc;
+    struct gf_defrag_brickinfo_ *bricks; /* volinfo->brick_count */
+    defrag_cbk_fn_t cbk_fn;
+    gf_defrag_status_t defrag_status;
+    char mount[1024];
 };
 
-
 typedef struct glusterd_defrag_info_ glusterd_defrag_info_t;
 
 typedef enum gf_transport_type_ {
-        GF_TRANSPORT_TCP,       //DEFAULT
-        GF_TRANSPORT_RDMA,
-        GF_TRANSPORT_BOTH_TCP_RDMA,
+    GF_TRANSPORT_TCP,  // DEFAULT
+    GF_TRANSPORT_RDMA,
+    GF_TRANSPORT_BOTH_TCP_RDMA,
 } gf_transport_type;
 
-
 typedef enum gf_rb_status_ {
-        GF_RB_STATUS_NONE,
-        GF_RB_STATUS_STARTED,
-        GF_RB_STATUS_PAUSED,
+    GF_RB_STATUS_NONE,
+    GF_RB_STATUS_STARTED,
+    GF_RB_STATUS_PAUSED,
 } gf_rb_status_t;
 
 struct _auth {
-        char       *username;
-        char       *password;
+    char *username;
+    char *password;
 };
 
 typedef struct _auth auth_t;
 
 /* Capabilities of xlator */
-#define CAPS_BD               0x00000001
-#define CAPS_THIN             0x00000002
-#define CAPS_OFFLOAD_COPY     0x00000004
+#define CAPS_BD 0x00000001
+#define CAPS_THIN 0x00000002
+#define CAPS_OFFLOAD_COPY 0x00000004
 #define CAPS_OFFLOAD_SNAPSHOT 0x00000008
-#define CAPS_OFFLOAD_ZERO     0x00000020
+#define CAPS_OFFLOAD_ZERO 0x00000020
+
+struct glusterd_bitrot_scrub_ {
+    char *scrub_state;
+    char *scrub_impact;
+    char *scrub_freq;
+    uint64_t scrubbed_files;
+    uint64_t unsigned_files;
+    uint64_t last_scrub_time;
+    uint64_t scrub_duration;
+    uint64_t error_count;
+};
+
+typedef struct glusterd_bitrot_scrub_ glusterd_bitrot_scrub_t;
 
 struct glusterd_rebalance_ {
-        gf_defrag_status_t       defrag_status;
-        uint64_t                 rebalance_files;
-        uint64_t                 rebalance_data;
-        uint64_t                 lookedup_files;
-        uint64_t                 skipped_files;
-        glusterd_defrag_info_t  *defrag;
-        gf_cli_defrag_type       defrag_cmd;
-        uint64_t                 rebalance_failures;
-        uuid_t                   rebalance_id;
-        double                   rebalance_time;
-        glusterd_op_t            op;
-        dict_t                  *dict; /* Dict to store misc information
-                                        * like list of bricks being removed */
+    uint64_t rebalance_files;
+    uint64_t rebalance_data;
+    uint64_t lookedup_files;
+    uint64_t skipped_files;
+    uint64_t rebalance_failures;
+    glusterd_defrag_info_t *defrag;
+    gf_cli_defrag_type defrag_cmd;
+    gf_defrag_status_t defrag_status;
+    uuid_t rebalance_id;
+    double rebalance_time;
+    uint64_t time_left;
+    dict_t *dict; /* Dict to store misc information
+                   * like list of bricks being removed */
+    glusterd_op_t op;
+    uint32_t commit_hash;
 };
 
 typedef struct glusterd_rebalance_ glusterd_rebalance_t;
 
 struct glusterd_replace_brick_ {
-        gf_rb_status_t          rb_status;
-        glusterd_brickinfo_t   *src_brick;
-        glusterd_brickinfo_t   *dst_brick;
-        uuid_t                  rb_id;
+    glusterd_brickinfo_t *src_brick;
+    glusterd_brickinfo_t *dst_brick;
 };
 
 typedef struct glusterd_replace_brick_ glusterd_replace_brick_t;
 
 typedef enum gd_quorum_status_ {
-        NOT_APPLICABLE_QUORUM, //Does not follow quorum
-        MEETS_QUORUM, //Follows quorum and meets.
-        DOESNT_MEET_QUORUM, //Follows quorum and does not meet.
+    NOT_APPLICABLE_QUORUM,  // Does not follow quorum
+    MEETS_QUORUM,           // Follows quorum and meets.
+    DOESNT_MEET_QUORUM,     // Follows quorum and does not meet.
 } gd_quorum_status_t;
 
-typedef struct tier_info_ {
-        int                       cold_type;
-        int                       cold_brick_count;
-        int                       cold_replica_count;
-        int                       cold_disperse_count;
-        int                       cold_dist_leaf_count;
-        int                       hot_type;
-        int                       hot_brick_count;
-        int                       hot_replica_count;
-} gd_tier_info_t;
-
 struct glusterd_volinfo_ {
-        gf_lock_t                 lock;
-        gf_boolean_t              is_snap_volume;
-        glusterd_snap_t          *snapshot;
-        uuid_t                    restored_from_snap;
-        gd_tier_info_t            tier_info;
-        char                      parent_volname[GD_VOLUME_NAME_MAX];
-                                         /* In case of a snap volume
-                                            i.e (is_snap_volume == TRUE) this
-                                            field will contain the name of
-                                            the volume which is snapped. In
-                                            case of a non-snap volume, this
-                                            field will be initialized as N/A */
-        char                      volname[GD_VOLUME_NAME_MAX];
-        int                       type;
-        int                       brick_count;
-        uint64_t                  snap_count;
-        uint64_t                  snap_max_hard_limit;
-        struct cds_list_head      vol_list;
-                                      /* In case of a snap volume
-                                         i.e (is_snap_volume == TRUE) this
-                                         is linked to glusterd_snap_t->volumes.
-                                         In case of a non-snap volume, this is
-                                         linked to glusterd_conf_t->volumes */
-        struct cds_list_head      snapvol_list;
-                                      /* This is a current pointer for
-                                         glusterd_volinfo_t->snap_volumes */
-        struct cds_list_head      bricks;
-        struct cds_list_head      snap_volumes;
-                                      /* TODO : Need to remove this, as this
-                                       * is already part of snapshot object.
-                                       */
-        glusterd_volume_status    status;
-        int                       sub_count;  /* backward compatibility */
-        int                       stripe_count;
-        int                       replica_count;
-        int                       arbiter_count;
-        int                       disperse_count;
-        int                       redundancy_count;
-        int                       subvol_count; /* Number of subvolumes in a
-                                                 distribute volume */
-        int                       dist_leaf_count; /* Number of bricks in one
-                                                    distribute subvolume */
-        int                       port;
-        gf_store_handle_t        *shandle;
-        gf_store_handle_t        *rb_shandle;
-        gf_store_handle_t        *node_state_shandle;
-        gf_store_handle_t        *quota_conf_shandle;
-
-        /* Defrag/rebalance related */
-        glusterd_rebalance_t      rebal;
-
-        /* Replace brick status */
-        glusterd_replace_brick_t  rep_brick;
-
-        int                       version;
-        uint32_t                  quota_conf_version;
-        uint32_t                  cksum;
-        uint32_t                  quota_conf_cksum;
-        gf_transport_type         transport_type;
-
-        dict_t                   *dict;
-
-        uuid_t                    volume_id;
-        auth_t                    auth;
-        char                     *logdir;
-
-        dict_t                   *gsync_slaves;
-        dict_t                   *gsync_active_slaves;
-
-        int                       decommission_in_progress;
-        xlator_t                 *xl;
-
-        gf_boolean_t              memory_accounting;
-        int                      caps; /* Capability */
-
-        int                       op_version;
-        int                       client_op_version;
-        pthread_mutex_t           reflock;
-        int                       refcnt;
-        gd_quorum_status_t        quorum_status;
-
-        glusterd_snapdsvc_t       snapd;
+    gf_lock_t lock;
+    glusterd_snap_t *snapshot;
+    uuid_t restored_from_snap;
+    int type;
+    int brick_count;
+    uint64_t snap_count;
+    uint64_t snap_max_hard_limit;
+    struct cds_list_head vol_list;
+    /* In case of a snap volume
+       i.e (is_snap_volume == TRUE) this
+       is linked to glusterd_snap_t->volumes.
+       In case of a non-snap volume, this is
+       linked to glusterd_conf_t->volumes */
+    struct cds_list_head snapvol_list;
+    /* This is a current pointer for
+       glusterd_volinfo_t->snap_volumes */
+    struct cds_list_head bricks;
+    struct cds_list_head ta_bricks;
+    struct cds_list_head snap_volumes;
+    /* TODO : Need to remove this, as this
+     * is already part of snapshot object.
+     */
+    glusterd_volume_status status;
+    int sub_count; /* backward compatibility */
+    int stripe_count;
+    int replica_count;
+    int arbiter_count;
+    int thin_arbiter_count;
+    int disperse_count;
+    int redundancy_count;
+    int subvol_count;    /* Number of subvolumes in a
+                          distribute volume */
+    int dist_leaf_count; /* Number of bricks in one
+                          distribute subvolume */
+    int port;
+    gf_store_handle_t *shandle;
+    gf_store_handle_t *node_state_shandle;
+    gf_store_handle_t *quota_conf_shandle;
+
+    /* Defrag/rebalance related */
+    glusterd_rebalance_t rebal;
+
+    /* Replace brick status */
+    glusterd_replace_brick_t rep_brick;
+
+    /* Bitrot scrub status*/
+    glusterd_bitrot_scrub_t bitrot_scrub;
+
+    int version;
+    uint32_t quota_conf_version;
+    uint32_t cksum;
+    uint32_t quota_conf_cksum;
+
+    dict_t *dict;
+
+    uuid_t volume_id;
+    auth_t auth;
+    char *logdir;
+
+    dict_t *gsync_slaves;
+    dict_t *gsync_active_slaves;
+
+    xlator_t *xl;
+    int decommission_in_progress;
+
+    int op_version;
+    int client_op_version;
+    int32_t quota_xattr_version;
+    pthread_mutex_t reflock;
+    int refcnt;
+    gd_quorum_status_t quorum_status;
+
+    glusterd_snapdsvc_t snapd;
+    glusterd_shdsvc_t shd;
+    glusterd_gfproxydsvc_t gfproxyd;
+    pthread_mutex_t store_volinfo_lock; /* acquire lock for
+                                         * updating the volinfo
+                                         */
+    gf_transport_type transport_type;
+    gf_boolean_t is_snap_volume;
+    gf_boolean_t memory_accounting;
+    gf_boolean_t stage_deleted; /* volume has passed staging
+                                 * for delete operation
+                                 */
+    char parent_volname[GD_VOLUME_NAME_MAX];
+    /* In case of a snap volume
+       i.e (is_snap_volume == TRUE) this
+       field will contain the name of
+       the volume which is snapped. In
+       case of a non-snap volume, this
+       field will be initialized as N/A */
+    char volname[NAME_MAX + 1];
+    /* NAME_MAX + 1 will be equal to
+     * GD_VOLUME_NAME_MAX + 5.(also to
+     * GD_VOLUME_NAME_MAX_TIER). An extra 5
+     * bytes are added to GD_VOLUME_NAME_MAX
+     * because, as part of the tiering
+     * volfile generation code, we are
+     * temporarily appending either "-hot"
+     * or "-cold" */
+    gf_atomic_t volpeerupdate;
+    /* Flag to check about volume has received updates
+       from peer
+    */
 };
 
 typedef enum gd_snap_status_ {
-        GD_SNAP_STATUS_NONE,
-        GD_SNAP_STATUS_INIT,
-        GD_SNAP_STATUS_IN_USE,
-        GD_SNAP_STATUS_DECOMMISSION,
-        GD_SNAP_STATUS_UNDER_RESTORE,
-        GD_SNAP_STATUS_RESTORED,
+    GD_SNAP_STATUS_NONE,
+    GD_SNAP_STATUS_INIT,
+    GD_SNAP_STATUS_IN_USE,
+    GD_SNAP_STATUS_DECOMMISSION,
+    GD_SNAP_STATUS_UNDER_RESTORE,
+    GD_SNAP_STATUS_RESTORED,
 } gd_snap_status_t;
 
 struct glusterd_snap_ {
-        gf_lock_t                lock;
-        struct  cds_list_head    volumes;
-        struct  cds_list_head    snap_list;
-        char                     snapname[GLUSTERD_MAX_SNAP_NAME];
-        uuid_t                   snap_id;
-        char                    *description;
-        time_t                   time_stamp;
-        gf_boolean_t             snap_restored;
-        gd_snap_status_t         snap_status;
-        gf_store_handle_t       *shandle;
+    gf_lock_t lock;
+    struct cds_list_head volumes;
+    struct cds_list_head snap_list;
+    char *description;
+    uuid_t snap_id;
+    time_t time_stamp;
+    gf_store_handle_t *shandle;
+    gd_snap_status_t snap_status;
+    gf_boolean_t snap_restored;
+    char snapname[GLUSTERD_MAX_SNAP_NAME];
 };
 
 typedef struct glusterd_snap_op_ {
-        char                  *snap_vol_id;
-        int32_t                brick_num;
-        char                  *brick_path;
-        int32_t                op;
-        int32_t                status;
-        struct cds_list_head   snap_ops_list;
+    char *snap_vol_id;
+    char *brick_path;
+    struct cds_list_head snap_ops_list;
+    int32_t brick_num;
+    int32_t op;
+    int32_t status;
 } glusterd_snap_op_t;
 
 typedef struct glusterd_missed_snap_ {
-        char                   *node_uuid;
-        char                   *snap_uuid;
-        struct cds_list_head    missed_snaps;
-        struct cds_list_head    snap_ops;
+    char *node_uuid;
+    char *snap_uuid;
+    struct cds_list_head missed_snaps;
+    struct cds_list_head snap_ops;
 } glusterd_missed_snap_info;
 
 typedef enum gd_node_type_ {
-        GD_NODE_NONE,
-        GD_NODE_BRICK,
-        GD_NODE_SHD,
-        GD_NODE_REBALANCE,
-        GD_NODE_NFS,
-        GD_NODE_QUOTAD,
-        GD_NODE_SNAPD,
+    GD_NODE_NONE,
+    GD_NODE_BRICK,
+    GD_NODE_SHD,
+    GD_NODE_REBALANCE,
+    GD_NODE_NFS,
+    GD_NODE_QUOTAD,
+    GD_NODE_SNAPD,
+    GD_NODE_BITD,
+    GD_NODE_SCRUB,
+    GD_NODE_TIERD
 } gd_node_type;
 
 typedef enum missed_snap_stat {
-        GD_MISSED_SNAP_NONE,
-        GD_MISSED_SNAP_PENDING,
-        GD_MISSED_SNAP_DONE,
+    GD_MISSED_SNAP_NONE,
+    GD_MISSED_SNAP_PENDING,
+    GD_MISSED_SNAP_DONE,
 } missed_snap_stat;
 
 typedef struct glusterd_pending_node_ {
-        struct cds_list_head list;
-        void   *node;
-        gd_node_type type;
-        int32_t index;
+    struct cds_list_head list;
+    void *node;
+    gd_node_type type;
+    int32_t index;
 } glusterd_pending_node_t;
 
 struct gsync_config_opt_vals_ {
-        char           *op_name;
-        int             no_of_pos_vals;
-        gf_boolean_t    case_sensitive;
-        char           *values[GEO_CONF_MAX_OPT_VALS];
+    char *op_name;
+    char *values[GEO_CONF_MAX_OPT_VALS];
+    int no_of_pos_vals;
+    gf_boolean_t case_sensitive;
 };
 
 enum glusterd_op_ret {
-        GLUSTERD_CONNECTION_AWAITED = 100,
+    GLUSTERD_CONNECTION_AWAITED = 100,
 };
 
 enum glusterd_vol_comp_status_ {
-        GLUSTERD_VOL_COMP_NONE = 0,
-        GLUSTERD_VOL_COMP_SCS = 1,
-        GLUSTERD_VOL_COMP_UPDATE_REQ,
-        GLUSTERD_VOL_COMP_RJT,
+    GLUSTERD_VOL_COMP_NONE = 0,
+    GLUSTERD_VOL_COMP_SCS = 1,
+    GLUSTERD_VOL_COMP_UPDATE_REQ,
+    GLUSTERD_VOL_COMP_RJT,
 };
 
 typedef struct addrinfo_list {
-        struct cds_list_head list;
-        struct addrinfo *info;
+    struct cds_list_head list;
+    struct addrinfo *info;
 } addrinfo_list_t;
 
 typedef enum {
-        GF_AI_COMPARE_NO_MATCH     = 0,
-        GF_AI_COMPARE_MATCH        = 1,
-        GF_AI_COMPARE_ERROR        = 2
+    GF_AI_COMPARE_NO_MATCH = 0,
+    GF_AI_COMPARE_MATCH = 1,
+    GF_AI_COMPARE_ERROR = 2
 } gf_ai_compare_t;
 
-#define GLUSTERD_DEFAULT_PORT    GF_DEFAULT_BASE_PORT
-#define GLUSTERD_INFO_FILE      "glusterd.info"
+#define GLUSTERD_DEFAULT_PORT GF_DEFAULT_BASE_PORT
+#define GLUSTERD_INFO_FILE "glusterd.info"
+#define GLUSTERD_UPGRADE_FILE                                                  \
+    "glusterd.upgrade" /* zero byte file to detect a need for regenerating     \
+                          volfiles in container mode */
 #define GLUSTERD_VOLUME_QUOTA_CONFIG "quota.conf"
 #define GLUSTERD_VOLUME_DIR_PREFIX "vols"
 #define GLUSTERD_PEER_DIR_PREFIX "peers"
@@ -498,608 +632,744 @@ typedef enum {
 #define GLUSTERD_MISSED_SNAPS_LIST_FILE "missed_snaps_list"
 #define GLUSTERD_VOL_SNAP_DIR_PREFIX "snaps"
 
-#define GLUSTERD_DEFAULT_SNAPS_BRICK_DIR        "/gluster/snaps"
-#define GLUSTERD_VAR_RUN_DIR                    "/var/run"
-#define GLUSTERD_RUN_DIR                        "/run"
+#define GLUSTERD_DEFAULT_SNAPS_BRICK_DIR "/gluster/snaps"
+#define GLUSTERD_BITD_RUN_DIR "/bitd"
+#define GLUSTERD_SCRUB_RUN_DIR "/scrub"
+#define GLUSTERD_NFS_RUN_DIR "/nfs"
+#define GLUSTERD_QUOTAD_RUN_DIR "/quotad"
+#define GLUSTER_SHARED_STORAGE_BRICK_DIR GLUSTERD_DEFAULT_WORKDIR "/ss_brick"
+#define GLUSTERD_VAR_RUN_DIR "/var/run"
+#define GLUSTERD_RUN_DIR "/run"
 
 /* definitions related to replace brick */
-#define RB_CLIENT_MOUNTPOINT    "rb_mount"
-#define RB_CLIENTVOL_FILENAME   "rb_client.vol"
-#define RB_DSTBRICK_PIDFILE     "rb_dst_brick.pid"
+#define RB_CLIENT_MOUNTPOINT "rb_mount"
+#define RB_CLIENTVOL_FILENAME "rb_client.vol"
+#define RB_DSTBRICK_PIDFILE "rb_dst_brick.pid"
 #define RB_DSTBRICKVOL_FILENAME "rb_dst_brick.vol"
-#define RB_PUMP_DEF_ARG         "default"
+#define RB_PUMP_DEF_ARG "default"
 
 #define GLUSTERD_UUID_LEN 50
 
-typedef ssize_t (*gd_serialize_t) (struct iovec outmsg, void *args);
-
-#define GLUSTERD_GET_VOLUME_DIR(path, volinfo, priv)                       \
-        if (volinfo->is_snap_volume) {                                     \
-                snprintf (path, PATH_MAX, "%s/snaps/%s/%s", priv->workdir, \
-                          volinfo->snapshot->snapname, volinfo->volname);  \
-        } else {                                                           \
-                snprintf (path, PATH_MAX, "%s/vols/%s", priv->workdir,     \
-                          volinfo->volname);                               \
-        }
-
-#define GLUSTERD_GET_SNAP_DIR(path, snap, priv)                           \
-                snprintf (path, PATH_MAX, "%s/snaps/%s", priv->workdir,   \
-                          snap->snapname);
-
-#define GLUSTERD_GET_SNAP_GEO_REP_DIR(path, snap, priv)                      \
-                snprintf (path, PATH_MAX, "%s/snaps/%s/%s", priv->workdir,   \
-                          snap->snapname, GEOREP);
-
-#define GLUSTERD_GET_BRICK_DIR(path, volinfo, priv)                           \
-        if (volinfo->is_snap_volume) {                                        \
-                snprintf (path, PATH_MAX, "%s/snaps/%s/%s/%s", priv->workdir, \
-                          volinfo->snapshot->snapname, volinfo->volname,      \
-                          GLUSTERD_BRICK_INFO_DIR);                           \
-        } else {                                                              \
-                snprintf (path, PATH_MAX, "%s/%s/%s/%s", priv->workdir,       \
-                          GLUSTERD_VOLUME_DIR_PREFIX, volinfo->volname,       \
-                          GLUSTERD_BRICK_INFO_DIR);                           \
-        }
-
-#define GLUSTERD_GET_NFS_DIR(path, priv) \
-        snprintf (path, PATH_MAX, "%s/nfs", priv->workdir);
-
-#define GLUSTERD_GET_QUOTAD_DIR(path, priv) \
-        snprintf (path, PATH_MAX, "%s/quotad", priv->workdir);
-
-#define GLUSTERD_GET_QUOTA_AUX_MOUNT_PATH(abspath, volname, path)      \
-        snprintf (abspath, sizeof (abspath)-1,                          \
-                  DEFAULT_VAR_RUN_DIRECTORY"/%s%s", volname, path);
-
-#define GLUSTERD_REMOVE_SLASH_FROM_PATH(path,string) do {                  \
-                int i = 0;                                                 \
-                for (i = 1; i < strlen (path); i++) {                      \
-                        string[i-1] = path[i];                             \
-                        if (string[i-1] == '/' && (i != strlen(path) - 1)) \
-                                string[i-1] = '-';                         \
-                }                                                          \
-        } while (0)
-
-#define GLUSTERD_GET_BRICK_PIDFILE(pidfile,volinfo,brickinfo, priv) do {      \
-                char exp_path[PATH_MAX] = {0,};                               \
-                char volpath[PATH_MAX]  = {0,};                               \
-                GLUSTERD_GET_VOLUME_DIR (volpath, volinfo, priv);             \
-                GLUSTERD_REMOVE_SLASH_FROM_PATH (brickinfo->path, exp_path);  \
-                snprintf (pidfile, PATH_MAX, "%s/run/%s-%s.pid",              \
-                          volpath, brickinfo->hostname, exp_path);      \
-        } while (0)
-
-#define GLUSTERD_GET_NFS_PIDFILE(pidfile,nfspath) {                     \
-                snprintf (pidfile, PATH_MAX, "%s/run/nfs.pid",          \
-                          nfspath);                                     \
-        }
-
-#define GLUSTERD_GET_QUOTAD_PIDFILE(pidfile,quotadpath) {                     \
-                snprintf (pidfile, PATH_MAX, "%s/run/quotad.pid",          \
-                          quotadpath);                                     \
-        }
-
-#define GLUSTERD_STACK_DESTROY(frame) do {\
-                frame->local = NULL;                                    \
-                STACK_DESTROY (frame->root);                            \
-        } while (0)
-
-#define GLUSTERD_GET_DEFRAG_DIR(path, volinfo, priv) do {               \
-                char vol_path[PATH_MAX];                                \
-                GLUSTERD_GET_VOLUME_DIR(vol_path, volinfo, priv);       \
-                snprintf (path, PATH_MAX, "%s/rebalance",vol_path);     \
-        } while (0)
-
-#define GLUSTERD_GET_DEFRAG_SOCK_FILE_OLD(path, volinfo, priv) do {     \
-                char defrag_path[PATH_MAX];                             \
-                GLUSTERD_GET_DEFRAG_DIR(defrag_path, volinfo, priv);    \
-                snprintf (path, PATH_MAX, "%s/%s.sock", defrag_path,    \
-                           uuid_utoa(MY_UUID));                         \
-        } while (0)
-
-#define GLUSTERD_GET_DEFRAG_SOCK_FILE(path, volinfo) do {                   \
-                snprintf (path, UNIX_PATH_MAX, DEFAULT_VAR_RUN_DIRECTORY    \
-                          "/gluster-rebalance-%s.sock",                     \
-                           uuid_utoa(volinfo->volume_id));                  \
-        } while (0)
-
-#define GLUSTERD_GET_DEFRAG_PID_FILE(path, volinfo, priv) do {          \
-                char defrag_path[PATH_MAX];                             \
-                GLUSTERD_GET_DEFRAG_DIR(defrag_path, volinfo, priv);    \
-                snprintf (path, PATH_MAX, "%s/%s.pid", defrag_path,     \
-                           uuid_utoa(MY_UUID));                         \
-        } while (0)
-
-#define GLUSTERFS_GET_AUX_MOUNT_PIDFILE(pidfile, volname) {               \
-                snprintf (pidfile, PATH_MAX-1,                            \
-                          DEFAULT_VAR_RUN_DIRECTORY"/%s.pid", volname);   \
-        }
-
-#define GLUSTERD_GET_UUID_NOHYPHEN(ret_string, uuid) do {               \
-                char *snap_volname_ptr = ret_string;                    \
-                char *snap_volid_ptr = uuid_utoa(uuid);                 \
-                while (*snap_volid_ptr) {                               \
-                        if (*snap_volid_ptr == '-') {                   \
-                                snap_volid_ptr++;                       \
-                        } else {                                        \
-                                (*snap_volname_ptr++) =                 \
-                                (*snap_volid_ptr++);                    \
-                        }                                               \
-                }                                                       \
-                *snap_volname_ptr = '\0';                               \
-        } while (0)
-
-#define GLUSTERD_DUMP_PEERS(head, member, xpeers) do {                       \
-                glusterd_peerinfo_t  *_peerinfo                = NULL;       \
-                char                  subkey[50]               = {0,};       \
-                int                   index                    = 1;          \
-                char                  key[GF_DUMP_MAX_BUF_LEN] = {0,};       \
-                                                                             \
-                if (!xpeers)                                                 \
-                        snprintf (key, sizeof (key), "glusterd.peer");       \
-                else                                                         \
-                        snprintf (key, sizeof (key),                         \
-                                  "glusterd.xaction_peer");                  \
-                                                                             \
-                rcu_read_lock ();                                            \
-                cds_list_for_each_entry_rcu (_peerinfo, head, member) {      \
-                        glusterd_dump_peer (_peerinfo, key, index, xpeers);  \
-                        if (!xpeers)                                         \
-                                glusterd_dump_peer_rpcstat (_peerinfo, key,  \
-                                                            index);          \
-                        index++;                                             \
-                }                                                            \
-                rcu_read_unlock ();                                          \
-                                                                             \
-        } while (0)
-
-int glusterd_uuid_init();
-
-int glusterd_uuid_generate_save ();
+typedef ssize_t (*gd_serialize_t)(struct iovec outmsg, void *args);
+
+#define GLUSTERD_GET_VOLUME_DIR(path, volinfo, priv)                           \
+    do {                                                                       \
+        int32_t _vol_dir_len;                                                  \
+        if (volinfo->is_snap_volume) {                                         \
+            _vol_dir_len = snprintf(                                           \
+                path, PATH_MAX, "%s/snaps/%s/%s", priv->workdir,               \
+                volinfo->snapshot->snapname, volinfo->volname);                \
+        } else {                                                               \
+            _vol_dir_len = snprintf(path, PATH_MAX, "%s/vols/%s",              \
+                                    priv->workdir, volinfo->volname);          \
+        }                                                                      \
+        if ((_vol_dir_len < 0) || (_vol_dir_len >= PATH_MAX)) {                \
+            path[0] = 0;                                                       \
+        }                                                                      \
+    } while (0)
+
+#define GLUSTERD_GET_DEFRAG_DIR(path, volinfo, priv)                           \
+    do {                                                                       \
+        char vol_path[PATH_MAX];                                               \
+        int32_t _defrag_dir_len;                                               \
+        GLUSTERD_GET_VOLUME_DIR(vol_path, volinfo, priv);                      \
+        _defrag_dir_len = snprintf(path, PATH_MAX, "%s/%s", vol_path,          \
+                                   "rebalance");                               \
+        if ((_defrag_dir_len < 0) || (_defrag_dir_len >= PATH_MAX)) {          \
+            path[0] = 0;                                                       \
+        }                                                                      \
+    } while (0)
+
+#define GLUSTERD_GET_DEFRAG_PID_FILE(path, volinfo, priv)                      \
+    do {                                                                       \
+        char defrag_path[PATH_MAX];                                            \
+        int32_t _defrag_pidfile_len;                                           \
+        GLUSTERD_GET_DEFRAG_DIR(defrag_path, volinfo, priv);                   \
+        _defrag_pidfile_len = snprintf(path, PATH_MAX, "%s/%s.pid",            \
+                                       defrag_path, uuid_utoa(MY_UUID));       \
+        if ((_defrag_pidfile_len < 0) || (_defrag_pidfile_len >= PATH_MAX)) {  \
+            path[0] = 0;                                                       \
+        }                                                                      \
+    } while (0)
+
+#define GLUSTERD_GET_SHD_RUNDIR(path, volinfo, priv)                           \
+    do {                                                                       \
+        int32_t _shd_dir_len;                                                  \
+        _shd_dir_len = snprintf(path, PATH_MAX, "%s/shd/%s", priv->rundir,     \
+                                volinfo->volname);                             \
+        if ((_shd_dir_len < 0) || (_shd_dir_len >= PATH_MAX)) {                \
+            path[0] = 0;                                                       \
+        }                                                                      \
+    } while (0)
+
+#define GLUSTERD_GET_VOLUME_PID_DIR(path, volinfo, priv)                       \
+    do {                                                                       \
+        int32_t _vol_pid_len;                                                  \
+        if (volinfo->is_snap_volume) {                                         \
+            _vol_pid_len = snprintf(path, PATH_MAX, "%s/snaps/%s/%s",          \
+                                    priv->rundir, volinfo->snapshot->snapname, \
+                                    volinfo->volname);                         \
+        } else {                                                               \
+            _vol_pid_len = snprintf(path, PATH_MAX, "%s/vols/%s",              \
+                                    priv->rundir, volinfo->volname);           \
+        }                                                                      \
+        if ((_vol_pid_len < 0) || (_vol_pid_len >= PATH_MAX)) {                \
+            path[0] = 0;                                                       \
+        }                                                                      \
+    } while (0)
+
+#define GLUSTERD_GET_SNAP_GEO_REP_DIR(path, snap, priv)                        \
+    do {                                                                       \
+        int32_t _snap_geo_len;                                                 \
+        _snap_geo_len = snprintf(path, PATH_MAX, "%s/snaps/%s/%s",             \
+                                 priv->workdir, snap->snapname, GEOREP);       \
+        if ((_snap_geo_len < 0) || (_snap_geo_len >= PATH_MAX)) {              \
+            path[0] = 0;                                                       \
+        }                                                                      \
+    } while (0)
+
+#define GLUSTERD_GET_QUOTA_LIMIT_MOUNT_PATH(abspath, volname, path)            \
+    do {                                                                       \
+        snprintf(abspath, sizeof(abspath) - 1,                                 \
+                 DEFAULT_VAR_RUN_DIRECTORY "/%s_quota_limit%s", volname,       \
+                 path);                                                        \
+    } while (0)
+
+#define GLUSTERD_REMOVE_SLASH_FROM_PATH(path, string)                          \
+    do {                                                                       \
+        int i = 0;                                                             \
+        for (i = 1; i < strlen(path); i++) {                                   \
+            string[i - 1] = path[i];                                           \
+            if (string[i - 1] == '/' && (i != strlen(path) - 1))               \
+                string[i - 1] = '-';                                           \
+        }                                                                      \
+    } while (0)
+
+#define GLUSTERD_GET_BRICK_PIDFILE(pidfile, volinfo, brickinfo, priv)          \
+    do {                                                                       \
+        char exp_path[PATH_MAX] = {                                            \
+            0,                                                                 \
+        };                                                                     \
+        char volpath[PATH_MAX] = {                                             \
+            0,                                                                 \
+        };                                                                     \
+        int32_t _brick_pid_len = 0;                                            \
+        GLUSTERD_GET_VOLUME_PID_DIR(volpath, volinfo, priv);                   \
+        GLUSTERD_REMOVE_SLASH_FROM_PATH(brickinfo->path, exp_path);            \
+        _brick_pid_len = snprintf(pidfile, PATH_MAX, "%s/%s-%s.pid", volpath,  \
+                                  brickinfo->hostname, exp_path);              \
+        if ((_brick_pid_len < 0) || (_brick_pid_len >= PATH_MAX)) {            \
+            pidfile[0] = 0;                                                    \
+        }                                                                      \
+    } while (0)
+
+#define RCU_READ_LOCK                                                          \
+    pthread_mutex_lock(&(THIS->ctx)->cleanup_lock);                            \
+    {                                                                          \
+        rcu_read_lock();                                                       \
+    }                                                                          \
+    pthread_mutex_unlock(&(THIS->ctx)->cleanup_lock);
+
+#define RCU_READ_UNLOCK                                                        \
+    pthread_mutex_lock(&(THIS->ctx)->cleanup_lock);                            \
+    {                                                                          \
+        rcu_read_unlock();                                                     \
+    }                                                                          \
+    pthread_mutex_unlock(&(THIS->ctx)->cleanup_lock);
+
+#define GLUSTERD_DUMP_PEERS(head, member, xpeers)                              \
+    do {                                                                       \
+        glusterd_peerinfo_t *_peerinfo = NULL;                                 \
+        int index = 1;                                                         \
+        char *key = NULL;                                                      \
+                                                                               \
+        key = xpeers ? "glusterd.xaction_peer" : "glusterd.peer";              \
+                                                                               \
+        RCU_READ_LOCK;                                                         \
+        cds_list_for_each_entry_rcu(_peerinfo, head, member)                   \
+        {                                                                      \
+            glusterd_dump_peer(_peerinfo, key, index, xpeers);                 \
+            if (!xpeers)                                                       \
+                glusterd_dump_peer_rpcstat(_peerinfo, key, index);             \
+            index++;                                                           \
+        }                                                                      \
+        RCU_READ_UNLOCK;                                                       \
+                                                                               \
+    } while (0)
+
+int
+glusterd_uuid_init();
+
+int
+glusterd_uuid_generate_save();
 
 #define MY_UUID (__glusterd_uuid())
 
 static inline unsigned char *
 __glusterd_uuid()
 {
-        glusterd_conf_t *priv = THIS->private;
+    glusterd_conf_t *priv = THIS->private;
 
-        if (gf_uuid_is_null (priv->uuid))
-                glusterd_uuid_init();
-        return &priv->uuid[0];
+    if (gf_uuid_is_null(priv->uuid))
+        glusterd_uuid_init();
+    return &priv->uuid[0];
 }
 
-int glusterd_big_locked_notify (struct rpc_clnt *rpc, void *mydata,
-                                rpc_clnt_event_t event,
-                                void *data, rpc_clnt_notify_t notify_fn);
+int
+glusterd_big_locked_notify(struct rpc_clnt *rpc, void *mydata,
+                           rpc_clnt_event_t event, void *data,
+                           rpc_clnt_notify_t notify_fn);
 
 int
-glusterd_big_locked_cbk (struct rpc_req *req, struct iovec *iov,
-                         int count, void *myframe, fop_cbk_fn_t fn);
+glusterd_big_locked_cbk(struct rpc_req *req, struct iovec *iov, int count,
+                        void *myframe, fop_cbk_fn_t fn);
 
-int glusterd_big_locked_handler (rpcsvc_request_t *req, rpcsvc_actor actor_fn);
+int
+glusterd_big_locked_handler(rpcsvc_request_t *req, rpcsvc_actor actor_fn);
 
 int32_t
-glusterd_brick_from_brickinfo (glusterd_brickinfo_t *brickinfo,
-                               char **new_brick);
+glusterd_brick_from_brickinfo(glusterd_brickinfo_t *brickinfo,
+                              char **new_brick);
 int
-glusterd_probe_begin (rpcsvc_request_t *req, const char *hoststr, int port,
-                      dict_t *dict, int *op_errno);
+glusterd_probe_begin(rpcsvc_request_t *req, const char *hoststr, int port,
+                     dict_t *dict, int *op_errno);
 
 int
-glusterd_xfer_friend_add_resp (rpcsvc_request_t *req, char *myhostname,
-                               char *remote_hostname, int port, int32_t op_ret,
-                               int32_t op_errno);
+glusterd_xfer_friend_add_resp(rpcsvc_request_t *req, char *myhostname,
+                              char *remote_hostname, int port, int32_t op_ret,
+                              int32_t op_errno);
 
 int
-glusterd_friend_add (const char *hoststr, int port,
-                     glusterd_friend_sm_state_t state,
-                     uuid_t *uuid, glusterd_peerinfo_t **friend,
-                     gf_boolean_t restore, glusterd_peerctx_args_t *args);
+glusterd_friend_add(const char *hoststr, int port,
+                    glusterd_friend_sm_state_t state, uuid_t *uuid,
+                    glusterd_peerinfo_t **friend, gf_boolean_t restore,
+                    glusterd_peerctx_args_t *args);
 
 int
-glusterd_friend_add_from_peerinfo (glusterd_peerinfo_t *friend,
-                                   gf_boolean_t restore,
-                                   glusterd_peerctx_args_t *args);
+glusterd_friend_add_from_peerinfo(glusterd_peerinfo_t *friend,
+                                  gf_boolean_t restore,
+                                  glusterd_peerctx_args_t *args);
 int
-glusterd_friend_rpc_create (xlator_t *this, glusterd_peerinfo_t *peerinfo,
-                            glusterd_peerctx_args_t *args);
+glusterd_friend_rpc_create(xlator_t *this, glusterd_peerinfo_t *peerinfo,
+                           glusterd_peerctx_args_t *args);
 int
-glusterd_friend_remove (uuid_t uuid, char *hostname);
+glusterd_friend_remove(uuid_t uuid, char *hostname);
 
 int
-glusterd_op_lock_send_resp (rpcsvc_request_t *req, int32_t status);
+glusterd_op_lock_send_resp(rpcsvc_request_t *req, int32_t status);
 
 int
-glusterd_op_unlock_send_resp (rpcsvc_request_t *req, int32_t status);
+glusterd_op_unlock_send_resp(rpcsvc_request_t *req, int32_t status);
 
 int
-glusterd_op_mgmt_v3_lock_send_resp (rpcsvc_request_t *req,
-                                    uuid_t *txn_id, int32_t status);
+glusterd_op_mgmt_v3_lock_send_resp(rpcsvc_request_t *req, uuid_t *txn_id,
+                                   int32_t status);
 
 int
-glusterd_op_mgmt_v3_unlock_send_resp (rpcsvc_request_t *req,
-                                      uuid_t *txn_id, int32_t status);
+glusterd_op_mgmt_v3_unlock_send_resp(rpcsvc_request_t *req, uuid_t *txn_id,
+                                     int32_t status);
 
 int
-glusterd_op_stage_send_resp (rpcsvc_request_t *req,
-                             int32_t op, int32_t status,
-                             char *op_errstr, dict_t *rsp_dict);
+glusterd_op_stage_send_resp(rpcsvc_request_t *req, int32_t op, int32_t status,
+                            char *op_errstr, dict_t *rsp_dict);
 
 int
-glusterd_op_commmit_send_resp (rpcsvc_request_t *req,
-                               int32_t op, int32_t status);
+glusterd_op_commmit_send_resp(rpcsvc_request_t *req, int32_t op,
+                              int32_t status);
 
 int32_t
-glusterd_create_volume (rpcsvc_request_t *req, dict_t *dict);
+glusterd_create_volume(rpcsvc_request_t *req, dict_t *dict);
 
 int
-glusterd_handle_incoming_friend_req (rpcsvc_request_t *req);
+glusterd_handle_incoming_friend_req(rpcsvc_request_t *req);
 
 int
-glusterd_handle_probe_query (rpcsvc_request_t *req);
+glusterd_handle_probe_query(rpcsvc_request_t *req);
 
 int
-glusterd_handle_cluster_lock (rpcsvc_request_t *req);
+glusterd_handle_cluster_lock(rpcsvc_request_t *req);
 
 int
-glusterd_handle_cluster_unlock (rpcsvc_request_t *req);
+glusterd_handle_cluster_unlock(rpcsvc_request_t *req);
 
 int
-glusterd_handle_stage_op (rpcsvc_request_t *req);
+glusterd_handle_stage_op(rpcsvc_request_t *req);
 
 int
-glusterd_handle_commit_op (rpcsvc_request_t *req);
+glusterd_handle_commit_op(rpcsvc_request_t *req);
 
 int
-glusterd_handle_cli_probe (rpcsvc_request_t *req);
+glusterd_handle_cli_probe(rpcsvc_request_t *req);
 
 int
-glusterd_handle_create_volume (rpcsvc_request_t *req);
+glusterd_handle_create_volume(rpcsvc_request_t *req);
 
 int
-glusterd_handle_defrag_volume (rpcsvc_request_t *req);
+glusterd_handle_defrag_volume(rpcsvc_request_t *req);
 
 int
-glusterd_handle_defrag_volume_v2 (rpcsvc_request_t *req);
+glusterd_handle_defrag_volume_v2(rpcsvc_request_t *req);
 
 int
-glusterd_xfer_cli_probe_resp (rpcsvc_request_t *req, int32_t op_ret,
-                              int32_t op_errno, char *op_errstr, char *hostname,
-                              int port, dict_t *dict);
+glusterd_xfer_cli_probe_resp(rpcsvc_request_t *req, int32_t op_ret,
+                             int32_t op_errno, char *op_errstr, char *hostname,
+                             int port, dict_t *dict);
 
 int
-glusterd_op_commit_send_resp (rpcsvc_request_t *req,
-                              int32_t op, int32_t status, char *op_errstr,
-                              dict_t *rsp_dict);
+glusterd_op_commit_send_resp(rpcsvc_request_t *req, int32_t op, int32_t status,
+                             char *op_errstr, dict_t *rsp_dict);
 
 int
-glusterd_xfer_friend_remove_resp (rpcsvc_request_t *req, char *hostname, int port);
+glusterd_xfer_friend_remove_resp(rpcsvc_request_t *req, char *hostname,
+                                 int port);
 
 int
-glusterd_deprobe_begin (rpcsvc_request_t *req, const char *hoststr, int port,
-                        uuid_t uuid, dict_t *dict, int *op_errno);
+glusterd_deprobe_begin(rpcsvc_request_t *req, const char *hoststr, int port,
+                       uuid_t uuid, dict_t *dict, int *op_errno);
 
 int
-glusterd_handle_cli_deprobe (rpcsvc_request_t *req);
+glusterd_handle_cli_deprobe(rpcsvc_request_t *req);
 
 int
-glusterd_handle_incoming_unfriend_req (rpcsvc_request_t *req);
+glusterd_handle_incoming_unfriend_req(rpcsvc_request_t *req);
 
 int32_t
-glusterd_list_friends (rpcsvc_request_t *req, dict_t *dict, int32_t flags);
+glusterd_list_friends(rpcsvc_request_t *req, dict_t *dict, int32_t flags);
 
 int
-glusterd_handle_cli_list_friends (rpcsvc_request_t *req);
+glusterd_handle_cli_list_friends(rpcsvc_request_t *req);
 
 int
-glusterd_handle_cli_start_volume (rpcsvc_request_t *req);
+glusterd_handle_cli_start_volume(rpcsvc_request_t *req);
 
 int
-glusterd_handle_friend_update (rpcsvc_request_t *req);
+glusterd_handle_friend_update(rpcsvc_request_t *req);
 
 int
-glusterd_handle_cli_stop_volume (rpcsvc_request_t *req);
+glusterd_handle_cli_stop_volume(rpcsvc_request_t *req);
 
 int
-glusterd_handle_cli_delete_volume (rpcsvc_request_t *req);
+glusterd_handle_cli_delete_volume(rpcsvc_request_t *req);
 
 int
-glusterd_handle_cli_get_volume (rpcsvc_request_t *req);
+glusterd_handle_cli_get_volume(rpcsvc_request_t *req);
 
 int32_t
-glusterd_get_volumes (rpcsvc_request_t *req, dict_t *dict, int32_t flags);
+glusterd_get_volumes(rpcsvc_request_t *req, dict_t *dict, int32_t flags);
 
 int
-glusterd_handle_add_brick (rpcsvc_request_t *req);
+glusterd_handle_add_brick(rpcsvc_request_t *req);
 
 int
-glusterd_handle_attach_tier (rpcsvc_request_t *req);
+glusterd_handle_tier(rpcsvc_request_t *req);
 
 int
-glusterd_handle_detach_tier (rpcsvc_request_t *req);
+glusterd_handle_attach_tier(rpcsvc_request_t *req);
 
 int
-glusterd_handle_replace_brick (rpcsvc_request_t *req);
+glusterd_handle_detach_tier(rpcsvc_request_t *req);
 
 int
-glusterd_handle_remove_brick (rpcsvc_request_t *req);
+glusterd_handle_add_tier_brick(rpcsvc_request_t *req);
 
 int
-glusterd_handle_log_rotate (rpcsvc_request_t *req);
+glusterd_handle_replace_brick(rpcsvc_request_t *req);
 
 int
-glusterd_handle_sync_volume (rpcsvc_request_t *req);
+glusterd_handle_remove_brick(rpcsvc_request_t *req);
+
+int
+glusterd_handle_log_rotate(rpcsvc_request_t *req);
+
+int
+glusterd_handle_sync_volume(rpcsvc_request_t *req);
+
+int
+glusterd_defrag_start_validate(glusterd_volinfo_t *volinfo, char *op_errstr,
+                               size_t len, glusterd_op_t op);
+
+int
+glusterd_rebalance_cmd_validate(int cmd, char *volname,
+                                glusterd_volinfo_t **volinfo, char *op_errstr,
+                                size_t len);
 
 int32_t
-glusterd_log_filename (rpcsvc_request_t *req, dict_t *dict);
+glusterd_log_filename(rpcsvc_request_t *req, dict_t *dict);
 
 int32_t
-glusterd_log_rotate (rpcsvc_request_t *req, dict_t *dict);
+glusterd_log_rotate(rpcsvc_request_t *req, dict_t *dict);
 
 int32_t
-glusterd_remove_brick (rpcsvc_request_t *req, dict_t *dict);
+glusterd_remove_brick(rpcsvc_request_t *req, dict_t *dict);
 
 int32_t
-glusterd_set_volume (rpcsvc_request_t *req, dict_t *dict);
+glusterd_set_volume(rpcsvc_request_t *req, dict_t *dict);
 
 int32_t
-glusterd_reset_volume (rpcsvc_request_t *req, dict_t *dict);
+glusterd_reset_volume(rpcsvc_request_t *req, dict_t *dict);
 
 int32_t
-glusterd_gsync_set (rpcsvc_request_t *req, dict_t *dict);
+glusterd_gsync_set(rpcsvc_request_t *req, dict_t *dict);
 
 int32_t
-glusterd_quota (rpcsvc_request_t *req, dict_t *dict);
+glusterd_quota(rpcsvc_request_t *req, dict_t *dict);
+
+int
+glusterd_handle_set_volume(rpcsvc_request_t *req);
+
+int
+glusterd_handle_reset_volume(rpcsvc_request_t *req);
 
 int
-glusterd_handle_set_volume (rpcsvc_request_t *req);
+glusterd_handle_copy_file(rpcsvc_request_t *req);
 
 int
-glusterd_handle_reset_volume (rpcsvc_request_t *req);
+glusterd_handle_sys_exec(rpcsvc_request_t *req);
 
 int
-glusterd_handle_copy_file (rpcsvc_request_t *req);
+glusterd_handle_gsync_set(rpcsvc_request_t *req);
 
 int
-glusterd_handle_sys_exec (rpcsvc_request_t *req);
+glusterd_handle_quota(rpcsvc_request_t *req);
 
 int
-glusterd_handle_gsync_set (rpcsvc_request_t *req);
+glusterd_handle_bitrot(rpcsvc_request_t *req);
 
 int
-glusterd_handle_quota (rpcsvc_request_t *req);
+glusterd_handle_fsm_log(rpcsvc_request_t *req);
 
 int
-glusterd_handle_bitrot (rpcsvc_request_t *req);
+glusterd_handle_reset_brick(rpcsvc_request_t *req);
 
 int
-glusterd_handle_fsm_log (rpcsvc_request_t *req);
+glusterd_xfer_cli_deprobe_resp(rpcsvc_request_t *req, int32_t op_ret,
+                               int32_t op_errno, char *op_errstr,
+                               char *hostname, dict_t *dict);
 
 int
-glusterd_xfer_cli_deprobe_resp (rpcsvc_request_t *req, int32_t op_ret,
-                                int32_t op_errno, char *op_errstr,
-                                char *hostname, dict_t *dict);
+glusterd_client_statedump_submit_req(char *volname, char *target_ip, char *pid);
 
 int
-glusterd_fetchspec_notify (xlator_t *this);
+glusterd_fetchspec_notify(xlator_t *this);
 
 int
-glusterd_fetchsnap_notify (xlator_t *this);
+glusterd_fetchsnap_notify(xlator_t *this);
 
 int
-glusterd_add_volume_detail_to_dict (glusterd_volinfo_t *volinfo,
-                                    dict_t  *volumes, int   count);
+glusterd_add_volume_detail_to_dict(glusterd_volinfo_t *volinfo, dict_t *volumes,
+                                   int count);
 
 int
-glusterd_restart_bricks (glusterd_conf_t *conf);
+glusterd_restart_bricks(void *opaque);
 
 int32_t
-glusterd_volume_txn (rpcsvc_request_t *req, char *volname, int flags,
-                     glusterd_op_t op);
+glusterd_volume_txn(rpcsvc_request_t *req, char *volname, int flags,
+                    glusterd_op_t op);
 
 int
-glusterd_peer_dump_version (xlator_t *this, struct rpc_clnt *rpc,
-                            glusterd_peerctx_t *peerctx);
+glusterd_peer_dump_version(xlator_t *this, struct rpc_clnt *rpc,
+                           glusterd_peerctx_t *peerctx);
 
 int
-glusterd_validate_reconfopts (glusterd_volinfo_t *volinfo, dict_t *val_dict, char **op_errstr);
+glusterd_validate_reconfopts(glusterd_volinfo_t *volinfo, dict_t *val_dict,
+                             char **op_errstr);
 int
-glusterd_handle_cli_profile_volume (rpcsvc_request_t *req);
+glusterd_handle_cli_profile_volume(rpcsvc_request_t *req);
 
 int
-glusterd_handle_getwd (rpcsvc_request_t *req);
+glusterd_handle_getwd(rpcsvc_request_t *req);
 
 int32_t
-glusterd_set_volume (rpcsvc_request_t *req, dict_t *dict);
+glusterd_set_volume(rpcsvc_request_t *req, dict_t *dict);
 int
-glusterd_peer_rpc_notify (struct rpc_clnt *rpc, void *mydata,
-                          rpc_clnt_event_t event,
-                          void *data);
+glusterd_peer_rpc_notify(struct rpc_clnt *rpc, void *mydata,
+                         rpc_clnt_event_t event, void *data);
 int
-glusterd_brick_rpc_notify (struct rpc_clnt *rpc, void *mydata,
+glusterd_brick_rpc_notify(struct rpc_clnt *rpc, void *mydata,
                           rpc_clnt_event_t event, void *data);
 
 int
-glusterd_rpc_create (struct rpc_clnt **rpc, dict_t *options,
-                     rpc_clnt_notify_t notify_fn, void *notify_data);
-
+glusterd_rpc_create(struct rpc_clnt **rpc, dict_t *options,
+                    rpc_clnt_notify_t notify_fn, void *notify_data,
+                    gf_boolean_t force);
 
 /* handler functions */
-int32_t glusterd_op_begin (rpcsvc_request_t *req, glusterd_op_t op, void *ctx,
-                           char *err_str, size_t size);
+int32_t
+glusterd_op_begin(rpcsvc_request_t *req, glusterd_op_t op, void *ctx,
+                  char *err_str, size_t size);
 
 /* removed other definitions as they have been defined elsewhere in this file*/
 
-int glusterd_handle_cli_statedump_volume (rpcsvc_request_t *req);
-int glusterd_handle_cli_clearlocks_volume (rpcsvc_request_t *req);
+int
+glusterd_handle_cli_statedump_volume(rpcsvc_request_t *req);
+int
+glusterd_handle_cli_clearlocks_volume(rpcsvc_request_t *req);
 
-int glusterd_handle_defrag_start (glusterd_volinfo_t *volinfo, char *op_errstr,
-                                  size_t len, int cmd, defrag_cbk_fn_t cbk,
-                                  glusterd_op_t op);
 int
-glusterd_rebalance_rpc_create (glusterd_volinfo_t *volinfo,
-                               gf_boolean_t reconnect);
+glusterd_handle_defrag_start(glusterd_volinfo_t *volinfo, char *op_errstr,
+                             size_t len, int cmd, defrag_cbk_fn_t cbk,
+                             glusterd_op_t op);
+int
+glusterd_rebalance_rpc_create(glusterd_volinfo_t *volinfo);
 
-int glusterd_handle_cli_heal_volume (rpcsvc_request_t *req);
+int
+glusterd_rebalance_defrag_init(glusterd_volinfo_t *volinfo,
+                               defrag_cbk_fn_t cbk);
 
-int glusterd_handle_cli_list_volume (rpcsvc_request_t *req);
+int
+glusterd_handle_cli_heal_volume(rpcsvc_request_t *req);
+
+int
+glusterd_handle_cli_list_volume(rpcsvc_request_t *req);
 
 int
-glusterd_handle_snapshot (rpcsvc_request_t *req);
+glusterd_handle_snapshot(rpcsvc_request_t *req);
 
 /* op-sm functions */
-int glusterd_op_stage_heal_volume (dict_t *dict, char **op_errstr);
-int glusterd_op_heal_volume (dict_t *dict, char **op_errstr);
-int glusterd_op_stage_gsync_set (dict_t *dict, char **op_errstr);
-int glusterd_op_gsync_set (dict_t *dict, char **op_errstr, dict_t *rsp_dict);
-int glusterd_op_stage_copy_file (dict_t *dict, char **op_errstr);
-int glusterd_op_copy_file (dict_t *dict, char **op_errstr);
-int glusterd_op_stage_sys_exec (dict_t *dict, char **op_errstr);
-int glusterd_op_sys_exec (dict_t *dict, char **op_errstr, dict_t *rsp_dict);
-int glusterd_op_stage_gsync_create (dict_t *dict, char **op_errstr);
-int glusterd_op_gsync_create (dict_t *dict, char **op_errstr, dict_t *rsp_dict);
-int glusterd_op_quota (dict_t *dict, char **op_errstr, dict_t *rsp_dict);
-
-int glusterd_op_bitrot (dict_t *dict, char **op_errstr, dict_t *rsp_dict);
-
-int glusterd_op_stage_quota (dict_t *dict, char **op_errstr, dict_t *rsp_dict);
-
-int glusterd_op_stage_bitrot (dict_t *dict, char **op_errstr, dict_t *rsp_dict);
-
-int glusterd_op_stage_replace_brick (dict_t *dict, char **op_errstr,
-                                     dict_t *rsp_dict);
-int glusterd_op_replace_brick (dict_t *dict, dict_t *rsp_dict);
-int glusterd_op_log_rotate (dict_t *dict);
-int glusterd_op_stage_log_rotate (dict_t *dict, char **op_errstr);
-int glusterd_op_stage_create_volume (dict_t *dict, char **op_errstr,
-                                     dict_t *rsp_dict);
-int glusterd_op_stage_start_volume (dict_t *dict, char **op_errstr,
-                                    dict_t *rsp_dict);
-int glusterd_op_stage_stop_volume (dict_t *dict, char **op_errstr);
-int glusterd_op_stage_delete_volume (dict_t *dict, char **op_errstr);
-int glusterd_op_create_volume (dict_t *dict, char **op_errstr);
-int glusterd_op_start_volume (dict_t *dict, char **op_errstr);
-int glusterd_op_stop_volume (dict_t *dict);
-int glusterd_op_delete_volume (dict_t *dict);
-int glusterd_handle_ganesha_op (dict_t *dict, char **op_errstr,
-                               char *key, char *value);
-int glusterd_check_ganesha_cmd (char *key, char *value,
-                                char **errstr, dict_t *dict);
-int glusterd_op_stage_set_ganesha (dict_t *dict, char **op_errstr);
-int glusterd_op_set_ganesha (dict_t *dict, char **errstr);
-int glusterd_op_add_brick (dict_t *dict, char **op_errstr);
-int glusterd_op_remove_brick (dict_t *dict, char **op_errstr);
-int glusterd_op_stage_add_brick (dict_t *dict, char **op_errstr,
-                                 dict_t *rsp_dict);
-int glusterd_op_stage_remove_brick (dict_t *dict, char **op_errstr);
-
-int glusterd_op_stage_rebalance (dict_t *dict, char **op_errstr);
-int glusterd_op_rebalance (dict_t *dict, char **op_errstr, dict_t *rsp_dict);
-
-int glusterd_op_stage_statedump_volume (dict_t *dict, char **op_errstr);
-int glusterd_op_statedump_volume (dict_t *dict, char **op_errstr);
-
-int glusterd_op_stage_clearlocks_volume (dict_t *dict, char **op_errstr);
-int glusterd_op_clearlocks_volume (dict_t *dict, char **op_errstr,
-                                   dict_t *rsp_dict);
-
-
-int glusterd_op_stage_barrier (dict_t *dict, char **op_errstr);
-int glusterd_op_barrier (dict_t *dict, char **op_errstr);
+int
+glusterd_op_stage_heal_volume(dict_t *dict, char **op_errstr);
+int
+glusterd_op_heal_volume(dict_t *dict, char **op_errstr);
+int
+glusterd_op_stage_gsync_set(dict_t *dict, char **op_errstr);
+int
+glusterd_op_gsync_set(dict_t *dict, char **op_errstr, dict_t *rsp_dict);
+int
+glusterd_op_stage_copy_file(dict_t *dict, char **op_errstr);
+int
+glusterd_op_copy_file(dict_t *dict, char **op_errstr);
+int
+glusterd_op_stage_sys_exec(dict_t *dict, char **op_errstr);
+int
+glusterd_op_sys_exec(dict_t *dict, char **op_errstr, dict_t *rsp_dict);
+int
+glusterd_op_stage_gsync_create(dict_t *dict, char **op_errstr);
+int
+glusterd_op_gsync_create(dict_t *dict, char **op_errstr, dict_t *rsp_dict);
+int
+glusterd_op_quota(dict_t *dict, char **op_errstr, dict_t *rsp_dict);
+
+int
+glusterd_op_bitrot(dict_t *dict, char **op_errstr, dict_t *rsp_dict);
+
+int
+glusterd_op_stage_quota(dict_t *dict, char **op_errstr, dict_t *rsp_dict);
+
+int
+glusterd_op_stage_bitrot(dict_t *dict, char **op_errstr, dict_t *rsp_dict);
+
+int
+glusterd_op_stage_replace_brick(dict_t *dict, char **op_errstr,
+                                dict_t *rsp_dict);
+int
+glusterd_op_replace_brick(dict_t *dict, dict_t *rsp_dict);
+int
+glusterd_op_log_rotate(dict_t *dict);
+int
+glusterd_op_stage_log_rotate(dict_t *dict, char **op_errstr);
+int
+glusterd_op_stage_create_volume(dict_t *dict, char **op_errstr,
+                                dict_t *rsp_dict);
+int
+glusterd_op_stage_start_volume(dict_t *dict, char **op_errstr,
+                               dict_t *rsp_dict);
+int
+glusterd_op_stage_stop_volume(dict_t *dict, char **op_errstr);
+int
+glusterd_op_stage_delete_volume(dict_t *dict, char **op_errstr);
+int
+glusterd_op_create_volume(dict_t *dict, char **op_errstr);
+int
+glusterd_op_start_volume(dict_t *dict, char **op_errstr);
+int
+glusterd_op_stop_volume(dict_t *dict);
+int
+glusterd_op_delete_volume(dict_t *dict);
+int
+glusterd_handle_ganesha_op(dict_t *dict, char **op_errstr, char *key,
+                           char *value);
+int
+glusterd_check_ganesha_cmd(char *key, char *value, char **errstr, dict_t *dict);
+int
+glusterd_op_stage_set_ganesha(dict_t *dict, char **op_errstr);
+int
+glusterd_op_set_ganesha(dict_t *dict, char **errstr);
+int
+ganesha_manage_export(dict_t *dict, char *value,
+                      gf_boolean_t update_cache_invalidation, char **op_errstr);
+int
+gd_ganesha_send_dbus(char *volname, char *value);
+gf_boolean_t
+glusterd_is_ganesha_cluster();
+gf_boolean_t
+glusterd_check_ganesha_export(glusterd_volinfo_t *volinfo);
+int
+stop_ganesha(char **op_errstr);
+int
+tear_down_cluster(gf_boolean_t run_teardown);
+int
+manage_export_config(char *volname, char *value, char **op_errstr);
+
+int
+glusterd_op_add_brick(dict_t *dict, char **op_errstr);
+int
+glusterd_op_remove_brick(dict_t *dict, char **op_errstr);
+int
+glusterd_op_stage_add_brick(dict_t *dict, char **op_errstr, dict_t *rsp_dict);
+int
+glusterd_op_stage_remove_brick(dict_t *dict, char **op_errstr);
+
+int
+glusterd_set_rebalance_id_for_remove_brick(dict_t *req_dict, dict_t *rsp_dict);
+
+int
+glusterd_set_rebalance_id_in_rsp_dict(dict_t *req_dict, dict_t *rsp_dict);
+
+int
+glusterd_mgmt_v3_op_stage_rebalance(dict_t *dict, char **op_errstr);
+
+int
+glusterd_mgmt_v3_op_rebalance(dict_t *dict, char **op_errstr, dict_t *rsp_dict);
+
+int
+glusterd_op_stage_rebalance(dict_t *dict, char **op_errstr);
+int
+glusterd_op_rebalance(dict_t *dict, char **op_errstr, dict_t *rsp_dict);
+
+int
+glusterd_op_stage_statedump_volume(dict_t *dict, char **op_errstr);
+int
+glusterd_op_statedump_volume(dict_t *dict, char **op_errstr);
+
+int
+glusterd_op_stage_clearlocks_volume(dict_t *dict, char **op_errstr);
+int
+glusterd_op_clearlocks_volume(dict_t *dict, char **op_errstr, dict_t *rsp_dict);
+
+int
+glusterd_op_stage_barrier(dict_t *dict, char **op_errstr);
+int
+glusterd_op_barrier(dict_t *dict, char **op_errstr);
 
 /* misc */
-void glusterd_do_replace_brick (void *data);
-int glusterd_op_perform_remove_brick (glusterd_volinfo_t  *volinfo, char *brick,
-                                      int force, int *need_migrate);
-int glusterd_op_stop_volume_args_get (dict_t *dict, char** volname, int *flags);
-int glusterd_op_statedump_volume_args_get (dict_t *dict, char **volname,
-                                           char **options, int *option_cnt);
+int
+glusterd_op_perform_remove_brick(glusterd_volinfo_t *volinfo, char *brick,
+                                 int force, int *need_migrate);
+int
+glusterd_op_stop_volume_args_get(dict_t *dict, char **volname, int *flags);
+int
+glusterd_op_statedump_volume_args_get(dict_t *dict, char **volname,
+                                      char **options, int *option_cnt);
+
+int
+glusterd_op_gsync_args_get(dict_t *dict, char **op_errstr, char **master,
+                           char **slave, char **host_uuid);
 
-int glusterd_op_gsync_args_get (dict_t *dict, char **op_errstr,
-                                char **master, char **slave, char **host_uuid);
+int
+glusterd_op_get_max_opversion(char **op_errstr, dict_t *rsp_dict);
 
-int glusterd_start_volume (glusterd_volinfo_t *volinfo, int flags,
-                           gf_boolean_t wait);
+int
+glusterd_start_volume(glusterd_volinfo_t *volinfo, int flags,
+                      gf_boolean_t wait);
 
-int glusterd_stop_volume (glusterd_volinfo_t *volinfo);
+int
+glusterd_stop_volume(glusterd_volinfo_t *volinfo);
 
 /* Synctask part */
-int32_t glusterd_op_begin_synctask (rpcsvc_request_t *req, glusterd_op_t op,
-                                    void *dict);
 int32_t
-glusterd_defrag_event_notify_handle (dict_t *dict);
+glusterd_op_begin_synctask(rpcsvc_request_t *req, glusterd_op_t op, void *dict);
+int32_t
+glusterd_defrag_event_notify_handle(dict_t *dict);
 
 int32_t
-glusterd_txn_opinfo_dict_init ();
+glusterd_txn_opinfo_dict_init();
 
 void
-glusterd_txn_opinfo_dict_fini ();
+glusterd_txn_opinfo_dict_fini();
 
 void
-glusterd_txn_opinfo_init ();
+glusterd_txn_opinfo_init();
 
 /* snapshot */
-glusterd_snap_t*
+glusterd_snap_t *
 glusterd_new_snap_object();
 
 int32_t
-glusterd_list_add_snapvol (glusterd_volinfo_t *origin_vol,
-                           glusterd_volinfo_t *snap_vol);
+glusterd_list_add_snapvol(glusterd_volinfo_t *origin_vol,
+                          glusterd_volinfo_t *snap_vol);
 
-glusterd_snap_t*
-glusterd_remove_snap_by_id (uuid_t snap_id);
+glusterd_snap_t *
+glusterd_remove_snap_by_id(uuid_t snap_id);
 
-glusterd_snap_t*
-glusterd_remove_snap_by_name (char *snap_name);
+glusterd_snap_t *
+glusterd_remove_snap_by_name(char *snap_name);
 
-glusterd_snap_t*
-glusterd_find_snap_by_name (char *snap_name);
+glusterd_snap_t *
+glusterd_find_snap_by_name(char *snap_name);
 
-glusterd_snap_t*
-glusterd_find_snap_by_id (uuid_t snap_id);
+glusterd_snap_t *
+glusterd_find_snap_by_id(uuid_t snap_id);
 
 int
-glusterd_snapshot_prevalidate (dict_t *dict, char **op_errstr,
-                               dict_t *rsp_dict);
+glusterd_snapshot_prevalidate(dict_t *dict, char **op_errstr, dict_t *rsp_dict,
+                              uint32_t *op_errno);
 int
-glusterd_snapshot_brickop (dict_t *dict, char **op_errstr, dict_t *rsp_dict);
+glusterd_snapshot_brickop(dict_t *dict, char **op_errstr, dict_t *rsp_dict);
 int
-glusterd_snapshot (dict_t *dict, char **op_errstr, dict_t *rsp_dict);
+glusterd_snapshot(dict_t *dict, char **op_errstr, uint32_t *op_errno,
+                  dict_t *rsp_dict);
 int
-glusterd_snapshot_postvalidate (dict_t *dict, int32_t op_ret, char **op_errstr,
-                                dict_t *rsp_dict);
+glusterd_snapshot_postvalidate(dict_t *dict, int32_t op_ret, char **op_errstr,
+                               dict_t *rsp_dict);
 char *
-glusterd_build_snap_device_path (char *device, char *snapname,
-                                 int32_t brick_count);
+glusterd_build_snap_device_path(char *device, char *snapname,
+                                int32_t brick_count);
 
 int32_t
-glusterd_snap_remove (dict_t *rsp_dict, glusterd_snap_t *snap,
-                      gf_boolean_t remove_lvm, gf_boolean_t force);
+glusterd_snap_remove(dict_t *rsp_dict, glusterd_snap_t *snap,
+                     gf_boolean_t remove_lvm, gf_boolean_t force,
+                     gf_boolean_t is_clone);
 int32_t
-glusterd_snapshot_cleanup (dict_t *dict, char **op_errstr, dict_t *rsp_dict);
+glusterd_snapshot_cleanup(dict_t *dict, char **op_errstr, dict_t *rsp_dict);
 
 int32_t
-glusterd_add_missed_snaps_to_list (dict_t *dict, int32_t missed_snap_count);
+glusterd_add_missed_snaps_to_list(dict_t *dict, int32_t missed_snap_count);
 
 int32_t
-glusterd_add_new_entry_to_list (char *missed_info, char *snap_vol_id,
-                                int32_t brick_num, char *brick_path,
-                                int32_t snap_op, int32_t snap_status);
+glusterd_add_new_entry_to_list(char *missed_info, char *snap_vol_id,
+                               int32_t brick_num, char *brick_path,
+                               int32_t snap_op, int32_t snap_status);
 
 int
-glusterd_snapshot_revert_restore_from_snap (glusterd_snap_t *snap);
-
+glusterd_snapshot_revert_restore_from_snap(glusterd_snap_t *snap);
 
 int
-glusterd_add_brick_status_to_dict (dict_t *dict, glusterd_volinfo_t *volinfo,
-                                   glusterd_brickinfo_t *brickinfo,
-                                   char *key_prefix);
+glusterd_add_brick_status_to_dict(dict_t *dict, glusterd_volinfo_t *volinfo,
+                                  glusterd_brickinfo_t *brickinfo,
+                                  char *key_prefix);
 
 int32_t
-glusterd_handle_snap_limit (dict_t *dict, dict_t *rsp_dict);
+glusterd_handle_snap_limit(dict_t *dict, dict_t *rsp_dict);
 
 gf_boolean_t
-glusterd_all_volumes_with_bitrot_stopped ();
+glusterd_should_i_stop_bitd();
+
+int
+glusterd_remove_brick_migrate_cbk(glusterd_volinfo_t *volinfo,
+                                  gf_defrag_status_t status);
+int
+__glusterd_handle_reset_brick(rpcsvc_request_t *req);
+
+int
+glusterd_options_init(xlator_t *this);
+
+int32_t
+glusterd_recreate_volfiles(glusterd_conf_t *conf);
+
+void
+glusterd_add_peers_to_auth_list(char *volname);
+
+int
+glusterd_replace_old_auth_allow_list(char *volname);
 
 #endif
diff --git a/xlators/mount/fuse/src/Makefile.am b/xlators/mount/fuse/src/Makefile.am
index 749fb3bbbb5..7018cad37f6 100644
--- a/xlators/mount/fuse/src/Makefile.am
+++ b/xlators/mount/fuse/src/Makefile.am
@@ -26,11 +26,12 @@ endif
 fuse_la_SOURCES = fuse-helpers.c fuse-resolve.c fuse-bridge.c \
 	$(CONTRIBDIR)/fuse-lib/misc.c $(mount_source)
 
-fuse_la_LDFLAGS = -module -avoid-version
-fuse_la_LIBADD = $(GF_LDADD) @GF_FUSE_LDADD@
+fuse_la_LDFLAGS = -module $(GF_XLATOR_DEFAULT_LDFLAGS)
+fuse_la_LIBADD = $(top_builddir)/libglusterfs/src/libglusterfs.la $(GF_LDADD) @GF_FUSE_LDADD@
 
-AM_CPPFLAGS = $(GF_CPPFLAGS) \
-	-I$(top_srcdir)/libglusterfs/src -I$(CONTRIBDIR)/fuse-include \
+AM_CPPFLAGS = $(GF_CPPFLAGS) -I$(top_srcdir)/libglusterfs/src \
+	-I$(top_srcdir)/rpc/xdr/src -I$(top_builddir)/rpc/xdr/src \
+	-I$(CONTRIBDIR)/fuse-include \
 	-I$(CONTRIBDIR)/fuse-lib $(GF_FUSE_CFLAGS)
 
 AM_CFLAGS = -Wall $(GF_CFLAGS)
diff --git a/xlators/mount/fuse/src/fuse-bridge.c b/xlators/mount/fuse/src/fuse-bridge.c
index d8e7012bb6c..0e22fe411ee 100644
--- a/xlators/mount/fuse/src/fuse-bridge.c
+++ b/xlators/mount/fuse/src/fuse-bridge.c
@@ -8,24 +8,35 @@
   cases as published by the Free Software Foundation.
 */
 
+#include <config.h>
+
 #include <sys/wait.h>
 #include "fuse-bridge.h"
-#include "mount-gluster-compat.h"
-#include "glusterfs.h"
-#include "byte-order.h"
-#include "compat-errno.h"
-#include "glusterfs-acl.h"
+#include <glusterfs/glusterfs.h>
+#include <glusterfs/byte-order.h>
+#include <glusterfs/compat-errno.h>
+#include <glusterfs/glusterfs-acl.h>
+#include <glusterfs/syscall.h>
+#include <glusterfs/timespec.h>
+#include <glusterfs/async.h>
 
 #ifdef __NetBSD__
 #undef open /* in perfuse.h, pulled from mount-gluster-compat.h */
 #endif
+typedef struct _fuse_async {
+    struct iobuf *iobuf;
+    fuse_in_header_t *finh;
+    void *msg;
+    gf_async_t async;
+} fuse_async_t;
 
-static int gf_fuse_conn_err_log;
 static int gf_fuse_xattr_enotsup_log;
 
-void fini (xlator_t *this_xl);
+void
+fini(xlator_t *this_xl);
 
-static void fuse_invalidate_inode(xlator_t *this, uint64_t fuse_ino);
+static int32_t
+fuse_invalidate_inode(xlator_t *this, uint64_t fuse_ino);
 
 /*
  * Send an invalidate notification up to fuse to purge the file from local
@@ -35,132 +46,303 @@ static void fuse_invalidate_inode(xlator_t *this, uint64_t fuse_ino);
 static int32_t
 fuse_invalidate(xlator_t *this, inode_t *inode)
 {
-        fuse_private_t *priv = this->private;
-        uint64_t nodeid;
+    fuse_private_t *priv = this->private;
+    uint64_t nodeid;
 
-        /*
-         * NOTE: We only invalidate at the moment if fopen_keep_cache is
-         * enabled because otherwise this is a departure from default
-         * behavior. Specifically, the performance/write-behind xlator
-         * causes unconditional invalidations on write requests.
-         */
-        if (!priv->fopen_keep_cache)
-                return 0;
+    /*
+     * NOTE: We only invalidate at the moment if fopen_keep_cache is
+     * enabled because otherwise this is a departure from default
+     * behavior. Specifically, the performance/write-behind xlator
+     * causes unconditional invalidations on write requests.
+     */
+    if (!priv->fopen_keep_cache)
+        return 0;
 
-        nodeid = inode_to_fuse_nodeid(inode);
-        gf_log(this->name, GF_LOG_DEBUG, "Invalidate inode id %"GF_PRI_INODE"." , nodeid);
-        fuse_log_eh (this, "Sending invalidate inode id: %"GF_PRI_INODE" gfid: %s", nodeid,
-                     uuid_utoa (inode->gfid));
-        fuse_invalidate_inode(this, nodeid);
+    nodeid = inode_to_fuse_nodeid(inode);
+    gf_log(this->name, GF_LOG_DEBUG, "Invalidate inode id %" GF_PRI_INODE ".",
+           nodeid);
+    fuse_log_eh(this, "Sending invalidate inode id: %" GF_PRI_INODE " gfid: %s",
+                nodeid, uuid_utoa(inode->gfid));
+    fuse_invalidate_inode(this, nodeid);
 
-        return 0;
+    return 0;
 }
 
 static int32_t
-fuse_forget_cbk (xlator_t *this, inode_t *inode)
+fuse_forget_cbk(xlator_t *this, inode_t *inode)
 {
-        //Nothing to free in inode ctx, hence return.
-        return 0;
+    // Nothing to free in inode ctx, hence return.
+    return 0;
 }
 
-void
-fuse_inode_set_need_lookup (inode_t *inode, xlator_t *this)
+fuse_fd_ctx_t *
+__fuse_fd_ctx_check_n_create(xlator_t *this, fd_t *fd)
 {
-	uint64_t          need_lookup = 1;
+    uint64_t val = 0;
+    int32_t ret = 0;
+    fuse_fd_ctx_t *fd_ctx = NULL;
 
-	if (!inode || !this)
-		return;
+    ret = __fd_ctx_get(fd, this, &val);
 
-	inode_ctx_set (inode, this, &need_lookup);
+    fd_ctx = (fuse_fd_ctx_t *)(unsigned long)val;
 
-	return;
+    if (fd_ctx == NULL) {
+        fd_ctx = GF_CALLOC(1, sizeof(*fd_ctx), gf_fuse_mt_fd_ctx_t);
+        if (!fd_ctx) {
+            goto out;
+        }
+        ret = __fd_ctx_set(fd, this, (uint64_t)(unsigned long)fd_ctx);
+        if (ret < 0) {
+            gf_log("glusterfs-fuse", GF_LOG_DEBUG, "fd-ctx-set failed");
+            GF_FREE(fd_ctx);
+            fd_ctx = NULL;
+        }
+    }
+out:
+    return fd_ctx;
 }
 
-
-gf_boolean_t
-fuse_inode_needs_lookup (inode_t *inode, xlator_t *this)
+fuse_fd_ctx_t *
+fuse_fd_ctx_check_n_create(xlator_t *this, fd_t *fd)
 {
-	uint64_t          need_lookup = 0;
-	gf_boolean_t      ret = _gf_false;
+    fuse_fd_ctx_t *fd_ctx = NULL;
 
-	if (!inode || !this)
-		return ret;
+    if ((fd == NULL) || (this == NULL)) {
+        goto out;
+    }
 
-	inode_ctx_get (inode, this, &need_lookup);
-	if (need_lookup)
-		ret = _gf_true;
-	need_lookup = 0;
-	inode_ctx_set (inode, this, &need_lookup);
+    LOCK(&fd->lock);
+    {
+        fd_ctx = __fuse_fd_ctx_check_n_create(this, fd);
+    }
+    UNLOCK(&fd->lock);
 
-	return ret;
+out:
+    return fd_ctx;
 }
 
+static void
+fuse_fd_ctx_destroy(xlator_t *this, fd_t *fd)
+{
+    fd_t *activefd = NULL;
+    uint64_t val = 0;
+    int ret = 0;
+    fuse_fd_ctx_t *fdctx = NULL;
+
+    ret = fd_ctx_del(fd, this, &val);
+    if (!ret) {
+        fdctx = (fuse_fd_ctx_t *)(unsigned long)val;
+        if (fdctx) {
+            activefd = fdctx->activefd;
+            if (activefd) {
+                fd_unref(activefd);
+            }
+
+            GF_FREE(fdctx);
+        }
+    }
+}
 
 fuse_fd_ctx_t *
-__fuse_fd_ctx_check_n_create (xlator_t *this, fd_t *fd)
+fuse_fd_ctx_get(xlator_t *this, fd_t *fd)
 {
-        uint64_t       val    = 0;
-        int32_t        ret    = 0;
-        fuse_fd_ctx_t *fd_ctx = NULL;
+    fuse_fd_ctx_t *fdctx = NULL;
+    uint64_t value = 0;
+    int ret = 0;
 
-        ret = __fd_ctx_get (fd, this, &val);
+    ret = fd_ctx_get(fd, this, &value);
+    if (ret < 0) {
+        goto out;
+    }
 
-        fd_ctx = (fuse_fd_ctx_t *)(unsigned long) val;
+    fdctx = (fuse_fd_ctx_t *)(unsigned long)value;
 
-        if (fd_ctx == NULL) {
-                fd_ctx = GF_CALLOC (1, sizeof (*fd_ctx),
-                                    gf_fuse_mt_fd_ctx_t);
-                if (!fd_ctx) {
-                    goto out;
-                }
-                ret = __fd_ctx_set (fd, this,
-                                    (uint64_t)(unsigned long)fd_ctx);
-                if (ret < 0) {
-                        gf_log ("glusterfs-fuse", GF_LOG_DEBUG,
-                                "fd-ctx-set failed");
-                        GF_FREE (fd_ctx);
-                        fd_ctx = NULL;
-                }
-        }
 out:
-        return fd_ctx;
+    return fdctx;
 }
 
-fuse_fd_ctx_t *
-fuse_fd_ctx_check_n_create (xlator_t *this, fd_t *fd)
+struct fusedump_timespec {
+    uint32_t len;
+    uint64_t sec;
+    uint32_t nsec;
+} __attribute__((packed));
+
+struct fusedump_signature {
+    uint32_t len;
+    char sig[8];
+} __attribute__((packed));
+
+static void
+fusedump_gettime(struct fusedump_timespec *fts)
 {
-        fuse_fd_ctx_t *fd_ctx = NULL;
+    struct timespec ts = {
+        0,
+    };
 
-        if ((fd == NULL) || (this == NULL)) {
-                goto out;
-        }
+    timespec_now_realtime(&ts);
 
-        LOCK (&fd->lock);
-        {
-                fd_ctx = __fuse_fd_ctx_check_n_create (this, fd);
-        }
-        UNLOCK (&fd->lock);
+    fts->sec = ts.tv_sec;
+    fts->nsec = ts.tv_nsec;
+}
 
-out:
-        return fd_ctx;
+static void
+fusedump_setup_meta(struct iovec *iovs, char *dir,
+                    uint32_t *fusedump_item_count,
+                    struct fusedump_timespec *fts,
+                    struct fusedump_signature *fsig)
+{
+    char glustersig[8] = {'G', 'L', 'U', 'S', 'T', 'E', 'R', 0xF5};
+
+    *fusedump_item_count = 3;
+    fts->len = sizeof(*fts);
+    fusedump_gettime(fts);
+    fsig->len = sizeof(*fsig);
+    memcpy(fsig->sig, glustersig, 8);
+
+    iovs[0] = (struct iovec){dir, sizeof(*dir)};
+    iovs[1] = (struct iovec){fusedump_item_count, sizeof(*fusedump_item_count)};
+    iovs[2] = (struct iovec){fts, fts->len};
+    iovs[3] = (struct iovec){fsig, fsig->len};
 }
 
-fuse_fd_ctx_t *
-fuse_fd_ctx_get (xlator_t *this, fd_t *fd)
+static int
+check_and_dump_fuse_W(fuse_private_t *priv, struct iovec *iov_out, int count,
+                      ssize_t res, errnomask_t errnomask)
 {
-        fuse_fd_ctx_t *fdctx = NULL;
-        uint64_t       value = 0;
-        int            ret   = 0;
+    char w = 'W';
+    struct iovec diov[4] = {
+        {
+            0,
+        },
+    };
+    uint32_t fusedump_item_count = 3;
+    struct fusedump_timespec fts = {
+        0,
+    };
+    struct fusedump_signature fsig = {
+        0,
+    };
+    struct fuse_out_header *fouh = NULL;
+
+    if (res == -1) {
+        const char *errdesc = NULL;
+        gf_loglevel_t loglevel = GF_LOG_ERROR;
+        gf_boolean_t errno_degraded = _gf_false;
+        gf_boolean_t errno_promoted = _gf_false;
+
+#define ACCOUNT_ERRNO(eno)                                                     \
+    do {                                                                       \
+        if (errno_degraded) {                                                  \
+            pthread_mutex_lock(&priv->fusedev_errno_cnt_mutex);                \
+            {                                                                  \
+                if (!++priv->fusedev_errno_cnt[FUSEDEV_##eno])                 \
+                    errno_promoted = _gf_true;                                 \
+            }                                                                  \
+            pthread_mutex_unlock(&priv->fusedev_errno_cnt_mutex);              \
+        }                                                                      \
+    } while (0)
+
+        /* If caller masked the errno, then it
+         * does not indicate an error at the application
+         * level, so we degrade the log severity to DEBUG.
+         */
+        if (errnomask && errno < ERRNOMASK_MAX &&
+            GET_ERRNO_MASK(errnomask, errno)) {
+            loglevel = GF_LOG_DEBUG;
+            errno_degraded = _gf_true;
+        }
+
+        switch (errno) {
+            /* The listed errnos are FUSE status indicators,
+             * not legit values according to POSIX (see write(3p)),
+             * so resolving them according to the standard
+             * POSIX interpretation would be misleading.
+             */
+            case ENOENT:
+                errdesc = "ENOENT";
+                ACCOUNT_ERRNO(ENOENT);
+                break;
+            case ENOTDIR:
+                errdesc = "ENOTDIR";
+                ACCOUNT_ERRNO(ENOTDIR);
+                break;
+            case ENODEV:
+                errdesc = "ENODEV";
+                ACCOUNT_ERRNO(ENODEV);
+                break;
+            case EPERM:
+                errdesc = "EPERM";
+                ACCOUNT_ERRNO(EPERM);
+                break;
+            case ENOMEM:
+                errdesc = "ENOMEM";
+                ACCOUNT_ERRNO(ENOMEM);
+                break;
+            case ENOTCONN:
+                errdesc = "ENOTCONN";
+                ACCOUNT_ERRNO(ENOTCONN);
+                break;
+            case ECONNREFUSED:
+                errdesc = "ECONNREFUSED";
+                ACCOUNT_ERRNO(ECONNREFUSED);
+                break;
+            case EOVERFLOW:
+                errdesc = "EOVERFLOW";
+                ACCOUNT_ERRNO(EOVERFLOW);
+                break;
+            case EBUSY:
+                errdesc = "EBUSY";
+                ACCOUNT_ERRNO(EBUSY);
+                break;
+            case ENOTEMPTY:
+                errdesc = "ENOTEMPTY";
+                ACCOUNT_ERRNO(ENOTEMPTY);
+                break;
+            default:
+                errdesc = strerror(errno);
+        }
+
+        gf_log_callingfn("glusterfs-fuse", loglevel,
+                         "writing to fuse device failed: %s", errdesc);
+        if (errno_promoted)
+            gf_log("glusterfs-fuse", GF_LOG_WARNING,
+                   "writing to fuse device yielded %s %d times", errdesc,
+                   UINT8_MAX + 1);
+        return errno;
+
+#undef ACCOUNT_ERRNO
+    }
+
+    fouh = iov_out[0].iov_base;
+    if (res != fouh->len) {
+        gf_log("glusterfs-fuse", GF_LOG_ERROR,
+               "inconsistent write to fuse device: "
+               "written %zd, expectd %d",
+               res, fouh->len);
+        return EINVAL;
+    }
+
+    if (priv->fuse_dump_fd == -1)
+        return 0;
 
-        ret = fd_ctx_get (fd, this, &value);
-        if (ret < 0) {
-                goto out;
-        }
+    fusedump_setup_meta(diov, &w, &fusedump_item_count, &fts, &fsig);
 
-        fdctx = (fuse_fd_ctx_t *) (unsigned long)value;
+    pthread_mutex_lock(&priv->fuse_dump_mutex);
+    res = sys_writev(priv->fuse_dump_fd, diov, sizeof(diov) / sizeof(diov[0]));
+    if (res != -1)
+        res = sys_writev(priv->fuse_dump_fd, iov_out, count);
+    pthread_mutex_unlock(&priv->fuse_dump_mutex);
 
-out:
-        return fdctx;
+    if (res == -1)
+        gf_log("glusterfs-fuse", GF_LOG_ERROR,
+               "failed to dump fuse message (W): %s", strerror(errno));
+
+    /*
+     * Return value reflects check on write to /dev/fuse,
+     * so ignore issues with dumping.
+     */
+
+    return 0;
 }
 
 /*
@@ -168,2674 +350,3503 @@ out:
  * The error value of this header is sent to kernel.
  */
 static int
-send_fuse_iov (xlator_t *this, fuse_in_header_t *finh, struct iovec *iov_out,
-               int count)
+send_fuse_iov(xlator_t *this, fuse_in_header_t *finh, struct iovec *iov_out,
+              int count)
 {
-        fuse_private_t *priv = NULL;
-        struct fuse_out_header *fouh = NULL;
-        int res, i;
+    fuse_private_t *priv = NULL;
+    struct fuse_out_header *fouh = NULL;
+    int res, i;
 
-        if (!this || !finh || !iov_out) {
-                gf_log ("send_fuse_iov", GF_LOG_ERROR,"Invalid arguments");
-                return EINVAL;
-        }
-        priv = this->private;
-
-        fouh = iov_out[0].iov_base;
-        iov_out[0].iov_len = sizeof (*fouh);
-        fouh->len = 0;
-        for (i = 0; i < count; i++)
-                fouh->len += iov_out[i].iov_len;
-        fouh->unique = finh->unique;
-
-        res = writev (priv->fd, iov_out, count);
-        gf_log ("glusterfs-fuse", GF_LOG_TRACE, "writev() result %d/%d %s",
-                res, fouh->len, res == -1 ? strerror (errno) : "");
-
-        if (res == -1)
-                return errno;
-        if (res != fouh->len)
-                return EINVAL;
-
-        if (priv->fuse_dump_fd != -1) {
-                char w = 'W';
-
-                pthread_mutex_lock (&priv->fuse_dump_mutex);
-                res = write (priv->fuse_dump_fd, &w, 1);
-                if (res != -1)
-                        res = writev (priv->fuse_dump_fd, iov_out, count);
-                pthread_mutex_unlock (&priv->fuse_dump_mutex);
-
-                if (res == -1)
-                        gf_log ("glusterfs-fuse", GF_LOG_ERROR,
-                                "failed to dump fuse message (W): %s",
-                                strerror (errno));
-        }
+    if (!this || !finh || !iov_out) {
+        gf_log("send_fuse_iov", GF_LOG_ERROR, "Invalid arguments");
+        return EINVAL;
+    }
+    priv = this->private;
 
-        return 0;
+    fouh = iov_out[0].iov_base;
+    iov_out[0].iov_len = sizeof(*fouh);
+    fouh->len = 0;
+    for (i = 0; i < count; i++)
+        fouh->len += iov_out[i].iov_len;
+    fouh->unique = finh->unique;
+
+    res = sys_writev(priv->fd, iov_out, count);
+    gf_log("glusterfs-fuse", GF_LOG_TRACE, "writev() result %d/%d %s", res,
+           fouh->len, res == -1 ? strerror(errno) : "");
+
+    return check_and_dump_fuse_W(priv, iov_out, count, res, NULL);
 }
 
 static int
-send_fuse_data (xlator_t *this, fuse_in_header_t *finh, void *data, size_t size)
+send_fuse_data(xlator_t *this, fuse_in_header_t *finh, void *data, size_t size)
 {
-        struct fuse_out_header fouh = {0, };
-        struct iovec iov_out[2];
-        int ret = 0;
+    struct fuse_out_header fouh = {
+        0,
+    };
+    struct iovec iov_out[2];
+    int ret = 0;
 
-        fouh.error = 0;
-        iov_out[0].iov_base = &fouh;
-        iov_out[1].iov_base = data;
-        iov_out[1].iov_len = size;
+    fouh.error = 0;
+    iov_out[0].iov_base = &fouh;
+    iov_out[1].iov_base = data;
+    iov_out[1].iov_len = size;
 
-        ret = send_fuse_iov (this, finh, iov_out, 2);
-        if (ret != 0)
-                gf_log ("glusterfs-fuse", GF_LOG_ERROR, "send_fuse_iov() "
-                        "failed: %s", strerror (ret));
+    ret = send_fuse_iov(this, finh, iov_out, 2);
+    if (ret != 0)
+        gf_log("glusterfs-fuse", GF_LOG_ERROR,
+               "send_fuse_iov() "
+               "failed: %s",
+               strerror(ret));
 
-        return ret;
+    return ret;
 }
 
-#define send_fuse_obj(this, finh, obj) \
-        send_fuse_data (this, finh, obj, sizeof (*(obj)))
-
+#define send_fuse_obj(this, finh, obj)                                         \
+    send_fuse_data(this, finh, obj, sizeof(*(obj)))
 
-#if FUSE_KERNEL_MINOR_VERSION >= 11
-static void
-fuse_invalidate_entry (xlator_t *this, uint64_t fuse_ino)
-{
-        struct fuse_out_header             *fouh   = NULL;
-        struct fuse_notify_inval_entry_out *fnieo  = NULL;
-        fuse_private_t                     *priv   = NULL;
-        dentry_t                           *dentry = NULL;
-        inode_t                            *inode  = NULL;
-        size_t                              nlen   = 0;
-        int                                 rv     = 0;
-        char inval_buf[INVAL_BUF_SIZE]             = {0,};
-
-        fouh  = (struct fuse_out_header *)inval_buf;
-        fnieo = (struct fuse_notify_inval_entry_out *)(fouh + 1);
-
-        priv = this->private;
-        if (priv->revchan_out == -1)
-                return;
+static int32_t
+fuse_invalidate_entry(xlator_t *this, uint64_t fuse_ino)
+{
+#if (FUSE_KERNEL_MINOR_VERSION >= 11 && defined(HAVE_FUSE_NOTIFICATIONS))
+    struct fuse_out_header *fouh = NULL;
+    struct fuse_notify_inval_entry_out *fnieo = NULL;
+    fuse_private_t *priv = NULL;
+    dentry_t *dentry = NULL;
+    dentry_t *tmp = NULL;
+    inode_t *inode = NULL;
+    size_t nlen = 0;
+    fuse_invalidate_node_t *node = NULL;
+    char gfid_str[UUID_CANONICAL_FORM_LEN + 1];
+
+    priv = this->private;
+    if (!priv->reverse_fuse_thread_started)
+        return -1;
 
-        fouh->unique = 0;
-        fouh->error = FUSE_NOTIFY_INVAL_ENTRY;
+    if (priv->invalidate_limit &&
+        (priv->invalidate_count >= priv->invalidate_limit)) {
+        return -1;
+    }
 
-        inode = fuse_ino_to_inode (fuse_ino, this);
+    inode = (inode_t *)(unsigned long)fuse_ino;
+    if (inode == NULL)
+        return -1;
 
-        list_for_each_entry (dentry, &inode->dentry_list, inode_list) {
-                nlen = strlen (dentry->name);
-                fouh->len = sizeof (*fouh) + sizeof (*fnieo) + nlen + 1;
-                fnieo->parent = inode_to_fuse_nodeid (dentry->parent);
+    list_for_each_entry_safe(dentry, tmp, &inode->dentry_list, inode_list)
+    {
+        node = GF_CALLOC(1, sizeof(*node), gf_fuse_mt_invalidate_node_t);
+        if (node == NULL)
+            return -1;
 
-                fnieo->namelen = nlen;
-                strcpy (inval_buf + sizeof (*fouh) + sizeof (*fnieo), dentry->name);
+        INIT_LIST_HEAD(&node->next);
 
-                rv = write (priv->revchan_out, inval_buf, fouh->len);
-                if (rv != fouh->len) {
-                        gf_log ("glusterfs-fuse", GF_LOG_ERROR,
-                                "kernel notification daemon defunct");
+        fouh = (struct fuse_out_header *)node->inval_buf;
+        fnieo = (struct fuse_notify_inval_entry_out *)(fouh + 1);
 
-                        close (priv->fd);
-                        break;
-                }
+        fouh->unique = 0;
+        fouh->error = FUSE_NOTIFY_INVAL_ENTRY;
 
-                gf_log ("glusterfs-fuse", GF_LOG_TRACE, "INVALIDATE entry: "
-                        "%"PRIu64"/%s", fnieo->parent, dentry->name);
+        if (ENOENT < ERRNOMASK_MAX)
+            MASK_ERRNO(node->errnomask, ENOENT);
+        if (ENOTDIR < ERRNOMASK_MAX)
+            MASK_ERRNO(node->errnomask, ENOTDIR);
+        if (EBUSY < ERRNOMASK_MAX)
+            MASK_ERRNO(node->errnomask, EBUSY);
+        if (ENOTEMPTY < ERRNOMASK_MAX)
+            MASK_ERRNO(node->errnomask, ENOTEMPTY);
+
+        if (dentry->name) {
+            nlen = strlen(dentry->name);
+            fouh->len = sizeof(*fouh) + sizeof(*fnieo) + nlen + 1;
+            fnieo->parent = inode_to_fuse_nodeid(dentry->parent);
+
+            fnieo->namelen = nlen;
+            strcpy((node->inval_buf + sizeof(*fouh) + sizeof(*fnieo)),
+                   dentry->name);
+        }
+
+        gf_log("glusterfs-fuse", GF_LOG_TRACE,
+               "INVALIDATE entry: %" PRIu64 "/%s (gfid:%s)", fnieo->parent,
+               dentry->name, uuid_utoa(inode->gfid));
+
+        if (dentry->parent) {
+            fuse_log_eh(this, "Invalidated entry %s (parent: %s) gfid:%s",
+                        dentry->name, uuid_utoa(dentry->parent->gfid),
+                        uuid_utoa_r(inode->gfid, gfid_str));
+        } else {
+            fuse_log_eh(this,
+                        "Invalidated entry %s(nodeid: %" PRIu64 ") gfid:%s",
+                        dentry->name, fnieo->parent, uuid_utoa(inode->gfid));
+        }
 
-                if (dentry->parent) {
-                        fuse_log_eh (this, "Invalidated entry %s (parent: %s)",
-                                     dentry->name,
-                                     uuid_utoa (dentry->parent->gfid));
-                } else {
-                        fuse_log_eh (this, "Invalidated entry %s(nodeid: %" PRIu64 ")",
-                                     dentry->name, fnieo->parent);
-                }
+        pthread_mutex_lock(&priv->invalidate_mutex);
+        {
+            list_add_tail(&node->next, &priv->invalidate_list);
+            priv->invalidate_count++;
+            pthread_cond_signal(&priv->invalidate_cond);
         }
+        pthread_mutex_unlock(&priv->invalidate_mutex);
+    }
 
-        if (inode)
-                inode_unref (inode);
-}
 #endif
+    return 0;
+}
 
 /*
  * Send an inval inode notification to fuse. This causes an invalidation of the
  * entire page cache mapping on the inode.
  */
-static void
+static int32_t
 fuse_invalidate_inode(xlator_t *this, uint64_t fuse_ino)
 {
-#if FUSE_KERNEL_MINOR_VERSION >= 11
-        struct fuse_out_header *fouh = NULL;
-        struct fuse_notify_inval_inode_out *fniio = NULL;
-        fuse_private_t *priv = NULL;
-        int rv = 0;
-        char inval_buf[INVAL_BUF_SIZE] = {0};
-        inode_t    *inode = NULL;
+#if (FUSE_KERNEL_MINOR_VERSION >= 11 && defined(HAVE_FUSE_NOTIFICATIONS))
+    struct fuse_out_header *fouh = NULL;
+    struct fuse_notify_inval_inode_out *fniio = NULL;
+    fuse_private_t *priv = NULL;
+    fuse_invalidate_node_t *node = NULL;
+    inode_t *inode = NULL;
+
+    priv = this->private;
 
-        fouh = (struct fuse_out_header *) inval_buf;
-        fniio = (struct fuse_notify_inval_inode_out *) (fouh + 1);
+    if (!priv->reverse_fuse_thread_started)
+        return -1;
 
-        priv = this->private;
+    if (priv->invalidate_limit &&
+        (priv->invalidate_count >= priv->invalidate_limit)) {
+        return -1;
+    }
 
-        if (priv->revchan_out < 0)
-                return;
+    inode = (inode_t *)(unsigned long)fuse_ino;
+    if (inode == NULL)
+        return -1;
 
-        fouh->unique = 0;
-        fouh->error = FUSE_NOTIFY_INVAL_INODE;
-        fouh->len = sizeof(struct fuse_out_header) +
-                sizeof(struct fuse_notify_inval_inode_out);
+    node = GF_CALLOC(1, sizeof(*node), gf_fuse_mt_invalidate_node_t);
+    if (node == NULL)
+        return -1;
 
-        /* inval the entire mapping until we learn how to be more granular */
-        fniio->ino = fuse_ino;
-        fniio->off = 0;
-        fniio->len = -1;
+    INIT_LIST_HEAD(&node->next);
 
-        inode = fuse_ino_to_inode (fuse_ino, this);
+    fouh = (struct fuse_out_header *)node->inval_buf;
+    fniio = (struct fuse_notify_inval_inode_out *)(fouh + 1);
 
-        rv = write(priv->revchan_out, inval_buf, fouh->len);
-        if (rv != fouh->len) {
-                gf_log("glusterfs-fuse", GF_LOG_ERROR, "kernel notification "
-                        "daemon defunct");
-                close(priv->fd);
-        }
+    fouh->unique = 0;
+    fouh->error = FUSE_NOTIFY_INVAL_INODE;
+    fouh->len = sizeof(struct fuse_out_header) +
+                sizeof(struct fuse_notify_inval_inode_out);
 
-        gf_log ("glusterfs-fuse", GF_LOG_TRACE, "INVALIDATE inode: %" PRIu64,
-                fuse_ino);
+    /* inval the entire mapping until we learn how to be more granular */
+    fniio->ino = fuse_ino;
+    fniio->off = 0;
+    fniio->len = -1;
 
-        if (inode) {
-                fuse_log_eh (this, "Invalidated inode %" PRIu64 " (gfid: %s)",
-                             fuse_ino, uuid_utoa (inode->gfid));
-        } else {
-                fuse_log_eh (this, "Invalidated inode %" PRIu64, fuse_ino);
-        }
+    if (ENOENT < ERRNOMASK_MAX)
+        MASK_ERRNO(node->errnomask, ENOENT);
+
+    fuse_log_eh(this, "Invalidated inode %" PRIu64 " (gfid: %s)", fuse_ino,
+                uuid_utoa(inode->gfid));
+    gf_log("glusterfs-fuse", GF_LOG_TRACE,
+           "INVALIDATE inode: %" PRIu64 "(gfid:%s)", fuse_ino,
+           uuid_utoa(inode->gfid));
+
+    pthread_mutex_lock(&priv->invalidate_mutex);
+    {
+        list_add_tail(&node->next, &priv->invalidate_list);
+        priv->invalidate_count++;
+        pthread_cond_signal(&priv->invalidate_cond);
+    }
+    pthread_mutex_unlock(&priv->invalidate_mutex);
 
-        if (inode)
-                inode_unref (inode);
 #else
-	gf_log ("glusterfs-fuse", GF_LOG_WARNING,
-		"fuse_invalidate_inode not implemented on OS X due to missing FUSE notification");
+    gf_log("glusterfs-fuse", GF_LOG_WARNING,
+           "fuse_invalidate_inode not implemented on this system");
 #endif
+    return 0;
 }
 
+#if FUSE_KERNEL_MINOR_VERSION >= 11
+/* Need this function for the signature (inode_t *, instead of uint64_t) */
+static int32_t
+fuse_inode_invalidate_fn(xlator_t *this, inode_t *inode)
+{
+    int32_t ret = 0;
+    ret = fuse_invalidate_entry(this, (uint64_t)(uintptr_t)inode);
+    return ret;
+}
+#endif
 
-int
-send_fuse_err (xlator_t *this, fuse_in_header_t *finh, int error)
+static fuse_timed_message_t *
+fuse_timed_message_new(void)
 {
-        struct fuse_out_header fouh = {0, };
-        struct iovec iov_out;
-        inode_t  *inode = NULL;
+    fuse_timed_message_t *dmsg = NULL;
 
-        fouh.error = -error;
-        iov_out.iov_base = &fouh;
+    dmsg = GF_MALLOC(sizeof(*dmsg), gf_fuse_mt_timed_message_t);
+    if (!dmsg) {
+        return NULL;
+    }
 
-        inode = fuse_ino_to_inode (finh->nodeid, this);
+    /* should be NULL if not set */
+    dmsg->fuse_message_body = NULL;
+    INIT_LIST_HEAD(&dmsg->next);
+    memset(dmsg->errnomask, 0, sizeof(dmsg->errnomask));
 
-        // filter out ENOENT
-        if (error != ENOENT) {
-                if (inode) {
-                        fuse_log_eh (this,"Sending %s for operation %d on "
-                                     "inode %s", strerror (error), finh->opcode,
-                                     uuid_utoa (inode->gfid));
-                } else {
-                        fuse_log_eh (this, "Sending %s for operation %d on "
-                                     "inode %" GF_PRI_INODE,  strerror (error),
-                                     finh->opcode, finh->nodeid);
+    return dmsg;
+}
+
+static void
+fuse_timed_message_free(fuse_timed_message_t *dmsg)
+{
+    GF_FREE(dmsg->fuse_message_body);
+    GF_FREE(dmsg);
+}
+
+static void
+send_fuse_timed(xlator_t *this, fuse_timed_message_t *dmsg)
+{
+    fuse_private_t *priv = NULL;
+
+    priv = this->private;
+
+    if (!priv->timed_response_fuse_thread_started) {
+        return;
+    }
+
+    pthread_mutex_lock(&priv->timed_mutex);
+    {
+        list_add_tail(&dmsg->next, &priv->timed_list);
+        pthread_cond_signal(&priv->timed_cond);
+    }
+    pthread_mutex_unlock(&priv->timed_mutex);
+}
+
+fuse_interrupt_record_t *
+fuse_interrupt_record_new(fuse_in_header_t *finh,
+                          fuse_interrupt_handler_t handler)
+{
+    fuse_interrupt_record_t *fir = NULL;
+
+    fir = GF_MALLOC(sizeof(*fir), gf_fuse_mt_interrupt_record_t);
+    if (!fir) {
+        return NULL;
+    }
+
+    fir->hit = _gf_false;
+    fir->interrupt_state = INTERRUPT_NONE;
+    fir->data = NULL;
+
+    fir->interrupt_handler = handler;
+    memcpy(&fir->fuse_in_header, finh, sizeof(*finh));
+    pthread_cond_init(&fir->handler_cond, NULL);
+    pthread_mutex_init(&fir->handler_mutex, NULL);
+    INIT_LIST_HEAD(&fir->next);
+
+    return fir;
+}
+
+static void
+fuse_interrupt_record_free(fuse_interrupt_record_t *fir, void **datap)
+{
+    /*
+     * If caller wishes, we give back the private data to let them deal with it
+     * however they want; otherwise we take care of freeing it.
+     */
+    if (datap) {
+        *datap = fir->data;
+    } else {
+        GF_FREE(fir->data);
+    }
+
+    GF_FREE(fir);
+}
+
+void
+fuse_interrupt_record_insert(xlator_t *this, fuse_interrupt_record_t *fir)
+{
+    fuse_private_t *priv = NULL;
+
+    priv = this->private;
+    pthread_mutex_lock(&priv->interrupt_mutex);
+    {
+        list_add_tail(&fir->next, &priv->interrupt_list);
+    }
+    pthread_mutex_unlock(&priv->interrupt_mutex);
+}
+
+static fuse_interrupt_record_t *
+fuse_interrupt_record_fetch(xlator_t *this, uint64_t unique, gf_boolean_t reap)
+{
+    fuse_interrupt_record_t *fir = NULL;
+    gf_boolean_t found = _gf_false;
+    fuse_private_t *priv = NULL;
+
+    priv = this->private;
+    pthread_mutex_lock(&priv->interrupt_mutex);
+    {
+        list_for_each_entry(fir, &priv->interrupt_list, next)
+        {
+            if (fir->fuse_in_header.unique == unique) {
+                /*
+                 * If we are to reap, we do it regardless the
+                 * hit flag; otherwise we take the record only
+                 * hasn't yet flagged hit.
+                 */
+                if (reap || !fir->hit) {
+                    found = _gf_true;
+                }
+                /*
+                 * If we are not reaping (coming from handler
+                 * context), we set the hit flag.
+                 */
+                if (!reap) {
+                    fir->hit = _gf_true;
                 }
+                break;
+            }
         }
+        if (found && reap) {
+            list_del(&fir->next);
+        }
+    }
+    pthread_mutex_unlock(&priv->interrupt_mutex);
 
-        if (inode)
-                inode_unref (inode);
+    if (found) {
+        return fir;
+    }
+    return NULL;
+}
 
-        return send_fuse_iov (this, finh, &iov_out, 1);
+static fuse_interrupt_record_t *
+fuse_interrupt_record_get(xlator_t *this, uint64_t unique)
+{
+    return fuse_interrupt_record_fetch(this, unique, _gf_false);
 }
 
-static int
-fuse_entry_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                int32_t op_ret, int32_t op_errno,
-                inode_t *inode, struct iatt *buf, dict_t *xdata)
-{
-        fuse_state_t          *state        = NULL;
-        fuse_in_header_t      *finh         = NULL;
-        struct fuse_entry_out  feo          = {0, };
-        fuse_private_t        *priv         = NULL;
-        inode_t               *linked_inode = NULL;
-
-        priv = this->private;
-        state = frame->root->state;
-        finh = state->finh;
-
-        if (op_ret == 0) {
-                if (__is_root_gfid (state->loc.inode->gfid))
-                        buf->ia_ino = 1;
-                if (gf_uuid_is_null (buf->ia_gfid)) {
-                        /* With a NULL gfid inode linking is
-                           not possible. Let's not pretend this
-                           call was a "success".
-                        */
-                        gf_log ("glusterfs-fuse", GF_LOG_WARNING,
-                                "Received NULL gfid for %s. Forcing EIO",
-                                state->loc.path);
-                        op_ret = -1;
-                        op_errno = EIO;
-                }
+static fuse_interrupt_record_t *
+fuse_interrupt_record_reap(xlator_t *this, uint64_t unique)
+{
+    return fuse_interrupt_record_fetch(this, unique, _gf_true);
+}
+
+static void
+fuse_interrupt(xlator_t *this, fuse_in_header_t *finh, void *msg,
+               struct iobuf *iobuf)
+{
+    struct fuse_interrupt_in *fii = msg;
+    fuse_interrupt_record_t *fir = NULL;
+
+    gf_log("glusterfs-fuse", GF_LOG_TRACE,
+           "unique %" PRIu64 " INTERRUPT for %" PRIu64, finh->unique,
+           fii->unique);
+
+    fir = fuse_interrupt_record_get(this, fii->unique);
+    if (fir) {
+        gf_log("glusterfs-fuse", GF_LOG_DEBUG,
+               "unique %" PRIu64 " INTERRUPT for %" PRIu64
+               ": handler triggered",
+               finh->unique, fii->unique);
+
+        fir->interrupt_handler(this, fir);
+    } else {
+        fuse_timed_message_t *dmsg = NULL;
+
+        /*
+         * No record found for this interrupt request.
+         *
+         * It's either because the handler for the interrupted message
+         * does not want to handle interrupt, or this interrupt
+         * message beat the interrupted which hasn't yet added a record
+         * to the interrupt queue. Either case we reply with error
+         * EAGAIN with some (0.01 sec) delay. That will have this
+         * interrupt request resent, unless the interrupted message
+         * has been already answered.
+         *
+         * So effectively we are looping in between kernel and
+         * userspace, which will be exited either when the interrupted
+         * message handler has added an interrupt record, or has
+         * replied to kernel. See
+         *
+         * https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/
+         * linux.git/tree/Documentation/filesystems/fuse.txt?h=v4.18#n148
+         */
+
+        gf_log("glusterfs-fuse", GF_LOG_DEBUG,
+               "unique %" PRIu64 " INTERRUPT for %" PRIu64 ": no handler found",
+               finh->unique, fii->unique);
+
+        dmsg = fuse_timed_message_new();
+        if (!dmsg) {
+            gf_log("glusterfs-fuse", GF_LOG_ERROR,
+                   "unique %" PRIu64 " INTERRUPT for %" PRIu64
+                   ":"
+                   " failed to allocate timed message",
+                   finh->unique, fii->unique);
+
+            goto out;
         }
 
-        /* log into the event-history after the null uuid check is done, since
-         * the op_ret and op_errno are being changed if the gfid is NULL.
+        dmsg->fuse_out_header.unique = finh->unique;
+        dmsg->fuse_out_header.len = sizeof(dmsg->fuse_out_header);
+        dmsg->fuse_out_header.error = -EAGAIN;
+        if (ENOENT < ERRNOMASK_MAX)
+            MASK_ERRNO(dmsg->errnomask, ENOENT);
+        timespec_now(&dmsg->scheduled_ts);
+        timespec_adjust_delta(&dmsg->scheduled_ts,
+                              (struct timespec){0, 10000000});
+
+        send_fuse_timed(this, dmsg);
+    }
+
+out:
+    GF_FREE(finh);
+}
+
+/*
+ * Function to be called in fop cbk context (if the fop engages
+ * with interrupt handling).
+ */
+gf_boolean_t
+fuse_interrupt_finish_fop(call_frame_t *frame, xlator_t *this,
+                          gf_boolean_t sync, void **datap)
+{
+    fuse_interrupt_record_t *fir = NULL;
+    fuse_state_t *state = frame->root->state;
+    fuse_in_header_t *finh = state->finh;
+    gf_boolean_t hit = _gf_false;
+    gf_boolean_t handled = _gf_false;
+    fuse_interrupt_state_t intstat_orig = INTERRUPT_NONE;
+
+    fir = fuse_interrupt_record_reap(this, finh->unique);
+    if (!fir) {
+        /*
+         * No interrupt record was inserted (however, caller would usually know
+         * about that and there is no point then in calling this function).
          */
-        fuse_log_eh (this, "op_ret: %d op_errno: %d "
-                     "%"PRIu64": %s() %s => %s", op_ret, op_errno,
-                     frame->root->unique, gf_fop_list[frame->root->op],
-                     state->loc.path, (op_ret == 0)?
-                     uuid_utoa(buf->ia_gfid):uuid_utoa(state->loc.gfid));
-
-        if (op_ret == 0) {
-                gf_log ("glusterfs-fuse", GF_LOG_TRACE,
-                        "%"PRIu64": %s() %s => %"PRIu64,
-                        frame->root->unique, gf_fop_list[frame->root->op],
-                        state->loc.path, buf->ia_ino);
-
-                buf->ia_blksize = this->ctx->page_size;
-                gf_fuse_stat2attr (buf, &feo.attr, priv->enable_ino32);
-
-                if (!buf->ia_ino) {
-                        gf_log ("glusterfs-fuse", GF_LOG_WARNING,
-                                "%"PRIu64": %s() %s returning inode 0",
-                                frame->root->unique,
-                                gf_fop_list[frame->root->op], state->loc.path);
-                }
+        return _gf_false;
+    }
+
+    /*
+     * The interrupt handler (if finds the record) modifies fir->hit; however,
+     * that could have occurred only before fuse_interrupt_record_reap(), so
+     * we are safe here with a lock-free access.
+     */
+    hit = fir->hit;
+    if (hit) {
+        pthread_mutex_lock(&fir->handler_mutex);
+        {
+            intstat_orig = fir->interrupt_state;
+            if (fir->interrupt_state == INTERRUPT_NONE) {
+                if (sync) {
+                    fir->interrupt_state = INTERRUPT_WAITING_HANDLER;
+                    while (fir->interrupt_state != INTERRUPT_SQUELCHED) {
+                        pthread_cond_wait(&fir->handler_cond,
+                                          &fir->handler_mutex);
+                    }
+                } else
+                    fir->interrupt_state = INTERRUPT_SQUELCHED;
+            }
+        }
+        pthread_mutex_unlock(&fir->handler_mutex);
+    }
+
+    GF_ASSERT(intstat_orig == INTERRUPT_NONE ||
+              intstat_orig == INTERRUPT_HANDLED ||
+              intstat_orig == INTERRUPT_SQUELCHED);
+    gf_log("glusterfs-fuse", GF_LOG_DEBUG, "intstat_orig=%d", intstat_orig);
+
+    /*
+     * From this on fir can only be referred under the conditions that imply
+     * we are to free it (otherwise interrupt handler might have already freed
+     * it).
+     */
+
+    if (/* there was no interrupt */
+        !hit ||
+        /* lost the race against interrupt handler */
+        intstat_orig != INTERRUPT_NONE ||
+        /* we took cleaning up on us */
+        sync) {
+        /* cleaning up */
+        fuse_interrupt_record_free(fir, datap);
+    } else if (datap) {
+        *datap = NULL;
+    }
+
+    handled = (intstat_orig == INTERRUPT_HANDLED);
+    if (handled) {
+        /*
+         * Fuse request was answered already from interrupt context, we can do
+         * away with the stack.
+         */
+        free_fuse_state(state);
+        STACK_DESTROY(frame->root);
+    }
 
-                linked_inode = inode_link (inode, state->loc.parent,
-                                           state->loc.name, buf);
+    /*
+     * Let caller know if they have to answer the fuse request.
+     */
+    return handled;
+}
 
-                if (linked_inode != inode) {
-                }
+/*
+ * Function to be called in interrupt handler context.
+ */
+void
+fuse_interrupt_finish_interrupt(xlator_t *this, fuse_interrupt_record_t *fir,
+                                fuse_interrupt_state_t intstat,
+                                gf_boolean_t sync, void **datap)
+{
+    fuse_in_header_t finh = {
+        0,
+    };
+    fuse_interrupt_state_t intstat_orig = INTERRUPT_NONE;
+
+    GF_ASSERT(intstat == INTERRUPT_HANDLED || intstat == INTERRUPT_SQUELCHED);
+
+    pthread_mutex_lock(&fir->handler_mutex);
+    {
+        intstat_orig = fir->interrupt_state;
+        switch (intstat_orig) {
+            case INTERRUPT_NONE:
+                fir->interrupt_state = intstat;
+                break;
+            case INTERRUPT_WAITING_HANDLER:
+                fir->interrupt_state = INTERRUPT_SQUELCHED;
+                pthread_cond_signal(&fir->handler_cond);
+                break;
+            default:
+                break;
+        }
+        finh = fir->fuse_in_header;
+    }
+    pthread_mutex_unlock(&fir->handler_mutex);
+
+    GF_ASSERT(intstat_orig == INTERRUPT_NONE ||
+              (sync && intstat_orig == INTERRUPT_WAITING_HANDLER) ||
+              (!sync && intstat_orig == INTERRUPT_SQUELCHED));
+    gf_log("glusterfs-fuse", GF_LOG_DEBUG, "intstat_orig=%d", intstat_orig);
+
+    /*
+     * From this on fir can only be referred under the conditions that imply
+     * we are to free it (otherwise fop handler might have already freed it).
+     */
+
+    if (/* we won the race, response is up to us */
+        intstat_orig == INTERRUPT_NONE &&
+        /* interrupt handling was successful, let the kernel know */
+        intstat == INTERRUPT_HANDLED) {
+        send_fuse_err(this, &finh, EINTR);
+    }
+
+    if (/* lost the race ... */
+        intstat_orig != INTERRUPT_NONE &&
+        /*
+         * ... and there is no contract with fop handler that it does the
+         * cleanup ...
+         */
+        !sync) {
+        /* ... so we do! */
+        fuse_interrupt_record_free(fir, datap);
+    } else if (datap) {
+        *datap = NULL;
+    }
+}
+
+int
+send_fuse_err(xlator_t *this, fuse_in_header_t *finh, int error)
+{
+    struct fuse_out_header fouh = {
+        0,
+    };
+    struct iovec iov_out;
+    inode_t *inode = NULL;
+
+    fouh.error = -error;
+    iov_out.iov_base = &fouh;
 
-                inode_lookup (linked_inode);
+    inode = fuse_ino_to_inode(finh->nodeid, this);
 
-                feo.nodeid = inode_to_fuse_nodeid (linked_inode);
+    // filter out ENOENT
+    if (error != ENOENT) {
+        if (inode) {
+            fuse_log_eh(this,
+                        "Sending %s for operation %d on "
+                        "inode %s",
+                        strerror(error), finh->opcode, uuid_utoa(inode->gfid));
+        } else {
+            fuse_log_eh(this,
+                        "Sending %s for operation %d on "
+                        "inode %" GF_PRI_INODE,
+                        strerror(error), finh->opcode, finh->nodeid);
+        }
+    }
 
-                inode_unref (linked_inode);
+    if (inode)
+        inode_unref(inode);
 
-                feo.entry_valid =
-                        calc_timeout_sec (priv->entry_timeout);
-                feo.entry_valid_nsec =
-                        calc_timeout_nsec (priv->entry_timeout);
-                feo.attr_valid =
-                        calc_timeout_sec (priv->attribute_timeout);
-                feo.attr_valid_nsec =
-                        calc_timeout_nsec (priv->attribute_timeout);
+    return send_fuse_iov(this, finh, &iov_out, 1);
+}
+
+static int
+fuse_entry_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+               int32_t op_ret, int32_t op_errno, inode_t *inode,
+               struct iatt *buf, dict_t *xdata)
+{
+    fuse_state_t *state = NULL;
+    fuse_in_header_t *finh = NULL;
+    struct fuse_entry_out feo = {
+        0,
+    };
+    fuse_private_t *priv = NULL;
+    inode_t *linked_inode = NULL;
+    uint64_t ctx_value = LOOKUP_NOT_NEEDED;
+
+    priv = this->private;
+    state = frame->root->state;
+    finh = state->finh;
+
+    if (op_ret == 0) {
+        if (__is_root_gfid(state->loc.inode->gfid))
+            buf->ia_ino = 1;
+        if (gf_uuid_is_null(buf->ia_gfid)) {
+            /* With a NULL gfid inode linking is
+               not possible. Let's not pretend this
+               call was a "success".
+            */
+            gf_log("glusterfs-fuse", GF_LOG_WARNING,
+                   "Received NULL gfid for %s. Forcing EIO", state->loc.path);
+            op_ret = -1;
+            op_errno = EIO;
+        }
+    }
+
+    /* log into the event-history after the null uuid check is done, since
+     * the op_ret and op_errno are being changed if the gfid is NULL.
+     */
+    fuse_log_eh(
+        this,
+        "op_ret: %d op_errno: %d "
+        "%" PRIu64 ": %s() %s => %s",
+        op_ret, op_errno, frame->root->unique, gf_fop_list[frame->root->op],
+        state->loc.path,
+        (op_ret == 0) ? uuid_utoa(buf->ia_gfid) : uuid_utoa(state->loc.gfid));
+
+    if (op_ret == 0) {
+        gf_log("glusterfs-fuse", GF_LOG_TRACE,
+               "%" PRIu64 ": %s() %s => %" PRIu64, frame->root->unique,
+               gf_fop_list[frame->root->op], state->loc.path, buf->ia_ino);
+
+        buf->ia_blksize = this->ctx->page_size;
+        gf_fuse_stat2attr(buf, &feo.attr, priv->enable_ino32);
+
+        if (!buf->ia_ino) {
+            gf_log("glusterfs-fuse", GF_LOG_WARNING,
+                   "%" PRIu64 ": %s() %s returning inode 0",
+                   frame->root->unique, gf_fop_list[frame->root->op],
+                   state->loc.path);
+        }
+
+        linked_inode = inode_link(inode, state->loc.parent, state->loc.name,
+                                  buf);
+
+        if (linked_inode == inode) {
+            inode_ctx_set(linked_inode, this, &ctx_value);
+        }
+
+        inode_lookup(linked_inode);
+
+        feo.nodeid = inode_to_fuse_nodeid(linked_inode);
+
+        inode_unref(linked_inode);
+
+        feo.entry_valid = calc_timeout_sec(priv->entry_timeout);
+        feo.entry_valid_nsec = calc_timeout_nsec(priv->entry_timeout);
+        feo.attr_valid = calc_timeout_sec(priv->attribute_timeout);
+        feo.attr_valid_nsec = calc_timeout_nsec(priv->attribute_timeout);
 
 #if FUSE_KERNEL_MINOR_VERSION >= 9
-                priv->proto_minor >= 9 ?
-                send_fuse_obj (this, finh, &feo) :
-                send_fuse_data (this, finh, &feo,
-                                FUSE_COMPAT_ENTRY_OUT_SIZE);
+        priv->proto_minor >= 9
+            ? send_fuse_obj(this, finh, &feo)
+            : send_fuse_data(this, finh, &feo, FUSE_COMPAT_ENTRY_OUT_SIZE);
 #else
-                send_fuse_obj (this, finh, &feo);
+        send_fuse_obj(this, finh, &feo);
 #endif
+    } else {
+        gf_log("glusterfs-fuse",
+               (op_errno == ENOENT ? GF_LOG_TRACE : GF_LOG_WARNING),
+               "%" PRIu64 ": %s() %s => -1 (%s)", frame->root->unique,
+               gf_fop_list[frame->root->op], state->loc.path,
+               strerror(op_errno));
+
+        if ((op_errno == ENOENT) && (priv->negative_timeout != 0)) {
+            feo.entry_valid = calc_timeout_sec(priv->negative_timeout);
+            feo.entry_valid_nsec = calc_timeout_nsec(priv->negative_timeout);
+            send_fuse_obj(this, finh, &feo);
         } else {
-                gf_log ("glusterfs-fuse",
-                        (op_errno == ENOENT ? GF_LOG_TRACE : GF_LOG_WARNING),
-                        "%"PRIu64": %s() %s => -1 (%s)", frame->root->unique,
-                        gf_fop_list[frame->root->op], state->loc.path,
-                        strerror (op_errno));
-
-		if ((op_errno == ENOENT) && (priv->negative_timeout != 0)) {
-			feo.entry_valid =
-				calc_timeout_sec (priv->negative_timeout);
-			feo.entry_valid_nsec =
-				calc_timeout_nsec (priv->negative_timeout);
-			send_fuse_obj (this, finh, &feo);
-		} else {
-			send_fuse_err (this, state->finh, op_errno);
-                }
+            send_fuse_err(this, state->finh, op_errno);
         }
+    }
 
-        free_fuse_state (state);
-        STACK_DESTROY (frame->root);
-        return 0;
+    free_fuse_state(state);
+    STACK_DESTROY(frame->root);
+    return 0;
 }
 
 static int
-fuse_newentry_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                   int32_t op_ret, int32_t op_errno,
-                   inode_t *inode, struct iatt *buf, struct iatt *preparent,
-                   struct iatt *postparent, dict_t *xdata)
+fuse_newentry_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                  int32_t op_ret, int32_t op_errno, inode_t *inode,
+                  struct iatt *buf, struct iatt *preparent,
+                  struct iatt *postparent, dict_t *xdata)
 {
-        fuse_entry_cbk (frame, cookie, this, op_ret, op_errno, inode, buf,
-                        xdata);
-        return 0;
+    /* facilitate retry of link from VFS */
+    if (op_errno == ENOENT)
+        op_errno = ESTALE;
+
+    fuse_entry_cbk(frame, cookie, this, op_ret, op_errno, inode, buf, xdata);
+    return 0;
 }
 
 static int
-fuse_lookup_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                 int32_t op_ret, int32_t op_errno,
-                 inode_t *inode, struct iatt *stat, dict_t *dict,
-                 struct iatt *postparent)
-{
-        fuse_state_t            *state = NULL;
-        call_frame_t            *prev = NULL;
-        inode_table_t           *itable = NULL;
-
-        state = frame->root->state;
-        prev  = cookie;
-
-        if (op_ret == -1 && state->is_revalidate == 1) {
-                itable = state->itable;
-		/*
-		 * A stale mapping might exist for a dentry/inode that has been
-		 * removed from another client.
-		 */
-		if (op_errno == ENOENT)
-			inode_unlink(state->loc.inode, state->loc.parent,
-				     state->loc.name);
-                inode_unref (state->loc.inode);
-                state->loc.inode = inode_new (itable);
-                state->is_revalidate = 2;
-                if (gf_uuid_is_null (state->gfid))
-                        gf_uuid_generate (state->gfid);
-                fuse_gfid_set (state);
-
-                STACK_WIND (frame, fuse_lookup_cbk,
-                            prev->this, prev->this->fops->lookup,
-                            &state->loc, state->xdata);
-                return 0;
-        }
+fuse_lookup_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                int32_t op_ret, int32_t op_errno, inode_t *inode,
+                struct iatt *stat, dict_t *dict, struct iatt *postparent)
+{
+    fuse_state_t *state = NULL;
+    call_frame_t *prev = NULL;
+    inode_table_t *itable = NULL;
 
-        fuse_entry_cbk (frame, cookie, this, op_ret, op_errno, inode, stat,
-                        dict);
+    state = frame->root->state;
+    prev = cookie;
+
+    if (op_ret == -1 && state->is_revalidate == 1) {
+        itable = state->itable;
+        /*
+         * A stale mapping might exist for a dentry/inode that has been
+         * removed from another client.
+         */
+        if (op_errno == ENOENT)
+            inode_unlink(state->loc.inode, state->loc.parent, state->loc.name);
+        inode_unref(state->loc.inode);
+        state->loc.inode = inode_new(itable);
+        state->is_revalidate = 2;
+        if (gf_uuid_is_null(state->gfid))
+            gf_uuid_generate(state->gfid);
+        fuse_gfid_set(state);
+
+        STACK_WIND(frame, fuse_lookup_cbk, prev->this, prev->this->fops->lookup,
+                   &state->loc, state->xdata);
         return 0;
+    }
+
+    fuse_entry_cbk(frame, cookie, this, op_ret, op_errno, inode, stat, dict);
+    return 0;
 }
 
 void
-fuse_fop_resume (fuse_state_t *state)
+fuse_fop_resume(fuse_state_t *state)
 {
-        fuse_resume_fn_t fn = NULL;
+    fuse_resume_fn_t fn = NULL;
 
-        /*
-         * Fail fd resolution failures right away.
-         */
-        if (state->resolve.fd && state->resolve.op_ret < 0) {
-                send_fuse_err (state->this, state->finh,
-                               state->resolve.op_errno);
-                free_fuse_state (state);
-                return;
-        }
+    /*
+     * Fail fd resolution failures right away.
+     */
+    if (state->resolve.fd && state->resolve.op_ret < 0) {
+        send_fuse_err(state->this, state->finh, state->resolve.op_errno);
+        free_fuse_state(state);
+        return;
+    }
 
-        fn = state->resume_fn;
-        fn (state);
+    fn = state->resume_fn;
+    fn(state);
 }
 
 void
-fuse_lookup_resume (fuse_state_t *state)
-{
-        if (!state->loc.parent && !state->loc.inode) {
-                gf_log ("fuse", GF_LOG_ERROR, "failed to resolve path %s",
-                        state->loc.path);
-                send_fuse_err (state->this, state->finh,
-                               state->resolve.op_errno);
-                free_fuse_state (state);
-                return;
-        }
-
-        /* parent was resolved, entry could not, may be a missing gfid?
-         * Hence try to do a regular lookup
-         */
-        if ((state->resolve.op_ret == -1)
-            && (state->resolve.op_errno == ENODATA)) {
-                state->resolve.op_ret = 0;
-        }
-
-        if (state->loc.inode) {
-                gf_log ("glusterfs-fuse", GF_LOG_TRACE,
-                        "%"PRIu64": LOOKUP %s(%s)", state->finh->unique,
-                        state->loc.path, uuid_utoa (state->loc.inode->gfid));
-                state->is_revalidate = 1;
-        } else {
-                gf_log ("glusterfs-fuse", GF_LOG_TRACE,
-                        "%"PRIu64": LOOKUP %s", state->finh->unique,
-                        state->loc.path);
-                state->loc.inode = inode_new (state->loc.parent->table);
-                if (gf_uuid_is_null (state->gfid))
-                        gf_uuid_generate (state->gfid);
-                fuse_gfid_set (state);
-        }
+fuse_lookup_resume(fuse_state_t *state)
+{
+    if (!state->loc.parent && !state->loc.inode) {
+        gf_log("fuse", GF_LOG_ERROR, "failed to resolve path %s",
+               state->loc.path);
+        send_fuse_err(state->this, state->finh, state->resolve.op_errno);
+        free_fuse_state(state);
+        return;
+    }
 
-        FUSE_FOP (state, fuse_lookup_cbk, GF_FOP_LOOKUP,
-                  lookup, &state->loc, state->xdata);
+    /* parent was resolved, entry could not, may be a missing gfid?
+     * Hence try to do a regular lookup
+     */
+    if ((state->resolve.op_ret == -1) && (state->resolve.op_errno == ENODATA)) {
+        state->resolve.op_ret = 0;
+    }
+
+    if (state->loc.inode) {
+        gf_log("glusterfs-fuse", GF_LOG_TRACE, "%" PRIu64 ": LOOKUP %s(%s)",
+               state->finh->unique, state->loc.path,
+               uuid_utoa(state->loc.inode->gfid));
+        state->is_revalidate = 1;
+    } else {
+        gf_log("glusterfs-fuse", GF_LOG_TRACE, "%" PRIu64 ": LOOKUP %s",
+               state->finh->unique, state->loc.path);
+        state->loc.inode = inode_new(state->loc.parent->table);
+        if (gf_uuid_is_null(state->gfid))
+            gf_uuid_generate(state->gfid);
+        fuse_gfid_set(state);
+    }
+
+    FUSE_FOP(state, fuse_lookup_cbk, GF_FOP_LOOKUP, lookup, &state->loc,
+             state->xdata);
 }
 
 static void
-fuse_lookup (xlator_t *this, fuse_in_header_t *finh, void *msg)
+fuse_lookup(xlator_t *this, fuse_in_header_t *finh, void *msg,
+            struct iobuf *iobuf)
 {
-        char           *name     = msg;
-        fuse_state_t   *state    = NULL;
+    char *name = msg;
+    fuse_state_t *state = NULL;
 
-        GET_STATE (this, finh, state);
+    GET_STATE(this, finh, state);
 
-        (void) fuse_resolve_entry_init (state, &state->resolve,
-                                        finh->nodeid, name);
+    (void)fuse_resolve_entry_init(state, &state->resolve, finh->nodeid, name);
 
-        fuse_resolve_and_resume (state, fuse_lookup_resume);
+    fuse_resolve_and_resume(state, fuse_lookup_resume);
 
-        return;
+    return;
 }
 
-static inline void
+static void
 do_forget(xlator_t *this, uint64_t unique, uint64_t nodeid, uint64_t nlookup)
 {
-	inode_t *fuse_inode = fuse_ino_to_inode(nodeid, this);
+    inode_t *fuse_inode = fuse_ino_to_inode(nodeid, this);
+
+    gf_log("fuse", GF_LOG_TRACE,
+           "%" PRIu64 ": FORGET %" PRIu64 "/%" PRIu64 " gfid: (%s)", unique,
+           nodeid, nlookup, uuid_utoa(fuse_inode->gfid));
 
-	fuse_log_eh(this, "%"PRIu64": FORGET %"PRIu64"/%"PRIu64" gfid: (%s)",
-		    unique, nodeid, nlookup, uuid_utoa(fuse_inode->gfid));
+    fuse_log_eh(this, "%" PRIu64 ": FORGET %" PRIu64 "/%" PRIu64 " gfid: (%s)",
+                unique, nodeid, nlookup, uuid_utoa(fuse_inode->gfid));
 
-	inode_forget(fuse_inode, nlookup);
-	inode_unref(fuse_inode);
+    inode_forget_with_unref(fuse_inode, nlookup);
 }
 
 static void
-fuse_forget (xlator_t *this, fuse_in_header_t *finh, void *msg)
+fuse_forget(xlator_t *this, fuse_in_header_t *finh, void *msg,
+            struct iobuf *iobuf)
 
 {
-        struct fuse_forget_in *ffi        = msg;
+    struct fuse_forget_in *ffi = msg;
 
-        if (finh->nodeid == 1) {
-                GF_FREE (finh);
-                return;
-        }
-
-        gf_log ("glusterfs-fuse", GF_LOG_TRACE,
-                "%"PRIu64": FORGET %"PRIu64"/%"PRIu64,
-                finh->unique, finh->nodeid, ffi->nlookup);
+    if (finh->nodeid == 1) {
+        GF_FREE(finh);
+        return;
+    }
 
-	do_forget(this, finh->unique, finh->nodeid, ffi->nlookup);
+    do_forget(this, finh->unique, finh->nodeid, ffi->nlookup);
 
-        GF_FREE (finh);
+    GF_FREE(finh);
 }
 
 #if FUSE_KERNEL_MINOR_VERSION >= 16
 static void
-fuse_batch_forget(xlator_t *this, fuse_in_header_t *finh, void *msg)
+fuse_batch_forget(xlator_t *this, fuse_in_header_t *finh, void *msg,
+                  struct iobuf *iobuf)
 {
-	struct fuse_batch_forget_in *fbfi = msg;
-	struct fuse_forget_one *ffo = (struct fuse_forget_one *) (fbfi + 1);
-	int i;
+    struct fuse_batch_forget_in *fbfi = msg;
+    struct fuse_forget_one *ffo = (struct fuse_forget_one *)(fbfi + 1);
+    int i;
 
-	gf_log("glusterfs-fuse", GF_LOG_TRACE,
-		"%"PRIu64": BATCH_FORGET %"PRIu64"/%"PRIu32,
-		finh->unique, finh->nodeid, fbfi->count);
+    gf_log("glusterfs-fuse", GF_LOG_TRACE,
+           "%" PRIu64 ": BATCH_FORGET %" PRIu64 "/%" PRIu32, finh->unique,
+           finh->nodeid, fbfi->count);
 
-	for (i = 0; i < fbfi->count; i++) {
-                if (ffo[i].nodeid == 1)
-                        continue;
-		do_forget(this, finh->unique, ffo[i].nodeid, ffo[i].nlookup);
-        }
-	GF_FREE(finh);
+    for (i = 0; i < fbfi->count; i++) {
+        if (ffo[i].nodeid == 1)
+            continue;
+        do_forget(this, finh->unique, ffo[i].nodeid, ffo[i].nlookup);
+    }
+    GF_FREE(finh);
 }
 #endif
 
 static int
-fuse_truncate_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                   int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
-                   struct iatt *postbuf, dict_t *xdata)
+fuse_truncate_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                  int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
+                  struct iatt *postbuf, dict_t *xdata)
 {
-        fuse_state_t     *state;
-        fuse_in_header_t *finh;
-        fuse_private_t   *priv = NULL;
-        struct fuse_attr_out fao;
+    fuse_state_t *state;
+    fuse_in_header_t *finh;
+    fuse_private_t *priv = NULL;
+    struct fuse_attr_out fao;
 
-        priv  = this->private;
-        state = frame->root->state;
-        finh  = state->finh;
+    priv = this->private;
+    state = frame->root->state;
+    finh = state->finh;
 
-        fuse_log_eh_fop(this, state, frame, op_ret, op_errno);
+    fuse_log_eh_fop(this, state, frame, op_ret, op_errno);
 
-        if (op_ret == 0) {
-                gf_log ("glusterfs-fuse", GF_LOG_TRACE,
-                        "%"PRIu64": %s() %s => %"PRIu64, frame->root->unique,
-                        gf_fop_list[frame->root->op],
-                        state->loc.path ? state->loc.path : "ERR",
-                        prebuf->ia_ino);
+    if (op_ret == 0) {
+        gf_log("glusterfs-fuse", GF_LOG_TRACE,
+               "%" PRIu64 ": %s() %s => %" PRIu64, frame->root->unique,
+               gf_fop_list[frame->root->op],
+               state->loc.path ? state->loc.path : "ERR", prebuf->ia_ino);
 
-                postbuf->ia_blksize = this->ctx->page_size;
-                gf_fuse_stat2attr (postbuf, &fao.attr, priv->enable_ino32);
+        postbuf->ia_blksize = this->ctx->page_size;
+        gf_fuse_stat2attr(postbuf, &fao.attr, priv->enable_ino32);
 
-                fao.attr_valid = calc_timeout_sec (priv->attribute_timeout);
-                fao.attr_valid_nsec =
-                  calc_timeout_nsec (priv->attribute_timeout);
+        fao.attr_valid = calc_timeout_sec(priv->attribute_timeout);
+        fao.attr_valid_nsec = calc_timeout_nsec(priv->attribute_timeout);
 
 #if FUSE_KERNEL_MINOR_VERSION >= 9
-                priv->proto_minor >= 9 ?
-                send_fuse_obj (this, finh, &fao) :
-                send_fuse_data (this, finh, &fao,
-                                FUSE_COMPAT_ATTR_OUT_SIZE);
+        priv->proto_minor >= 9
+            ? send_fuse_obj(this, finh, &fao)
+            : send_fuse_data(this, finh, &fao, FUSE_COMPAT_ATTR_OUT_SIZE);
 #else
-                send_fuse_obj (this, finh, &fao);
+        send_fuse_obj(this, finh, &fao);
 #endif
-        } else {
-                gf_log ("glusterfs-fuse", GF_LOG_WARNING,
-                        "%"PRIu64": %s() %s => -1 (%s)", frame->root->unique,
-                        gf_fop_list[frame->root->op],
-                        state->loc.path ? state->loc.path : "ERR",
-                        strerror (op_errno));
+    } else {
+        gf_log("glusterfs-fuse", GF_LOG_WARNING,
+               "%" PRIu64 ": %s() %s => -1 (%s)", frame->root->unique,
+               gf_fop_list[frame->root->op],
+               state->loc.path ? state->loc.path : "ERR", strerror(op_errno));
 
-                send_fuse_err (this, finh, op_errno);
-        }
+        /* facilitate retry from VFS */
+        if ((state->fd == NULL) && (op_errno == ENOENT))
+            op_errno = ESTALE;
 
-        free_fuse_state (state);
-        STACK_DESTROY (frame->root);
+        send_fuse_err(this, finh, op_errno);
+    }
 
-        return 0;
+    free_fuse_state(state);
+    STACK_DESTROY(frame->root);
+
+    return 0;
 }
 
 static int
-fuse_attr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-               int32_t op_ret, int32_t op_errno, struct iatt *buf, dict_t *xdata)
-{
-        fuse_state_t     *state;
-        fuse_in_header_t *finh;
-        fuse_private_t   *priv = NULL;
-        struct fuse_attr_out fao;
-
-        priv  = this->private;
-        state = frame->root->state;
-        finh  = state->finh;
-
-        fuse_log_eh (this, "op_ret: %d, op_errno: %d, %"PRIu64": %s() %s => "
-                   "gfid: %s", op_ret, op_errno, frame->root->unique,
-                   gf_fop_list[frame->root->op], state->loc.path,
-                   state->loc.inode ? uuid_utoa (state->loc.inode->gfid) : "");
-        if (op_ret == 0) {
-                gf_log ("glusterfs-fuse", GF_LOG_TRACE,
-                        "%"PRIu64": %s() %s => %"PRIu64, frame->root->unique,
-                        gf_fop_list[frame->root->op],
-                        state->loc.path ? state->loc.path : "ERR",
-                        buf->ia_ino);
-
-                buf->ia_blksize = this->ctx->page_size;
-                gf_fuse_stat2attr (buf, &fao.attr, priv->enable_ino32);
+fuse_root_lookup_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                     int32_t op_ret, int32_t op_errno, inode_t *inode,
+                     struct iatt *stat, dict_t *dict, struct iatt *postparent);
 
-                fao.attr_valid = calc_timeout_sec (priv->attribute_timeout);
-                fao.attr_valid_nsec =
-                  calc_timeout_nsec (priv->attribute_timeout);
+static int
+fuse_attr_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int32_t op_ret,
+              int32_t op_errno, struct iatt *buf, dict_t *xdata)
+{
+    int32_t ret = 0;
+    fuse_state_t *state;
+    fuse_in_header_t *finh;
+    fuse_private_t *priv = NULL;
+    struct fuse_attr_out fao;
+
+    priv = this->private;
+    state = frame->root->state;
+    finh = state->finh;
+
+    fuse_log_eh(this,
+                "op_ret: %d, op_errno: %d, %" PRIu64
+                ": %s() %s => "
+                "gfid: %s",
+                op_ret, op_errno, frame->root->unique,
+                gf_fop_list[frame->root->op], state->loc.path,
+                state->loc.inode ? uuid_utoa(state->loc.inode->gfid) : "");
+    if (op_ret == 0) {
+        gf_log("glusterfs-fuse", GF_LOG_TRACE,
+               "%" PRIu64 ": %s() %s => %" PRIu64, frame->root->unique,
+               gf_fop_list[frame->root->op],
+               state->loc.path ? state->loc.path : "ERR", buf->ia_ino);
+
+        buf->ia_blksize = this->ctx->page_size;
+        gf_fuse_stat2attr(buf, &fao.attr, priv->enable_ino32);
+
+        fao.attr_valid = calc_timeout_sec(priv->attribute_timeout);
+        fao.attr_valid_nsec = calc_timeout_nsec(priv->attribute_timeout);
 
 #if FUSE_KERNEL_MINOR_VERSION >= 9
-                priv->proto_minor >= 9 ?
-                send_fuse_obj (this, finh, &fao) :
-                send_fuse_data (this, finh, &fao,
-                                FUSE_COMPAT_ATTR_OUT_SIZE);
+        priv->proto_minor >= 9
+            ? send_fuse_obj(this, finh, &fao)
+            : send_fuse_data(this, finh, &fao, FUSE_COMPAT_ATTR_OUT_SIZE);
 #else
-                send_fuse_obj (this, finh, &fao);
+        send_fuse_obj(this, finh, &fao);
 #endif
-        } else {
-                GF_LOG_OCCASIONALLY ( gf_fuse_conn_err_log, "glusterfs-fuse",
-                                      GF_LOG_WARNING,
-                                      "%"PRIu64": %s() %s => -1 (%s)",
-                                      frame->root->unique,
-                                      gf_fop_list[frame->root->op],
-                                      state->loc.path ? state->loc.path : "ERR",
-                                      strerror (op_errno));
-
-                send_fuse_err (this, finh, op_errno);
+    } else {
+        /* This is moved here from fuse_getattr(). It makes sense as
+           in few cases, like the self-heal processes, some
+           translators expect a lookup() to come on root inode
+           (inode number 1). This will make sure we don't fail in any
+           case, but the positive path will get better performance,
+           by following common path for all the cases */
+        if ((finh->nodeid == 1) && (state->gfid[15] != 1)) {
+            /* The 'state->gfid[15]' check is added to prevent the
+               infinite recursions */
+            state->gfid[15] = 1;
+
+            ret = fuse_loc_fill(&state->loc, state, finh->nodeid, 0, NULL);
+            if (ret < 0) {
+                gf_log("glusterfs-fuse", GF_LOG_WARNING,
+                       "%" PRIu64 ": loc_fill() on / failed", finh->unique);
+                send_fuse_err(this, finh, ENOENT);
+                free_fuse_state(state);
+                return 0;
+            }
+
+            fuse_gfid_set(state);
+
+            FUSE_FOP(state, fuse_root_lookup_cbk, GF_FOP_LOOKUP, lookup,
+                     &state->loc, state->xdata);
+
+            return 0;
         }
 
-        free_fuse_state (state);
-        STACK_DESTROY (frame->root);
+        /* facilitate retry from VFS */
+        if ((state->fd == NULL) && (op_errno == ENOENT))
+            op_errno = ESTALE;
 
-        return 0;
+        gf_log("glusterfs-fuse", GF_LOG_WARNING,
+               "%" PRIu64
+               ": %s() "
+               "%s => -1 (%s)",
+               frame->root->unique, gf_fop_list[frame->root->op],
+               state->loc.path ? state->loc.path : "ERR", strerror(op_errno));
+
+        send_fuse_err(this, finh, op_errno);
+    }
+
+    free_fuse_state(state);
+    STACK_DESTROY(frame->root);
+
+    return 0;
 }
 
 static int
-fuse_root_lookup_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                      int32_t op_ret, int32_t op_errno,
-                      inode_t *inode, struct iatt *stat, dict_t *dict,
-                      struct iatt *postparent)
+fuse_root_lookup_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                     int32_t op_ret, int32_t op_errno, inode_t *inode,
+                     struct iatt *stat, dict_t *dict, struct iatt *postparent)
 {
-        fuse_attr_cbk (frame, cookie, this, op_ret, op_errno, stat, dict);
+    fuse_attr_cbk(frame, cookie, this, op_ret, op_errno, stat, dict);
 
-        return 0;
+    return 0;
 }
 
 void
-fuse_getattr_resume (fuse_state_t *state)
-{
-        if (!state->loc.inode) {
-                gf_log ("glusterfs-fuse", GF_LOG_ERROR,
-                        "%"PRIu64": GETATTR %"PRIu64" (%s) resolution failed",
-                        state->finh->unique, state->finh->nodeid,
-                        uuid_utoa (state->resolve.gfid));
-                send_fuse_err (state->this, state->finh,
-                               state->resolve.op_errno);
-                free_fuse_state (state);
-                return;
-        }
+fuse_getattr_resume(fuse_state_t *state)
+{
+    if (!state->loc.inode && !(state->fd && state->fd->inode)) {
+        gf_log("glusterfs-fuse", GF_LOG_ERROR,
+               "%" PRIu64 ": GETATTR %" PRIu64 " (%s) resolution failed",
+               state->finh->unique, state->finh->nodeid,
+               uuid_utoa(state->resolve.gfid));
 
-        if (!IA_ISDIR (state->loc.inode->ia_type)) {
-                state->fd = fd_lookup (state->loc.inode, state->finh->pid);
-        }
+        /* facilitate retry from VFS */
+        if ((state->fd == NULL) && (state->resolve.op_errno == ENOENT))
+            state->resolve.op_errno = ESTALE;
 
-        if (!state->fd) {
-                gf_log ("glusterfs-fuse", GF_LOG_TRACE,
-                        "%"PRIu64": GETATTR %"PRIu64" (%s)",
-                        state->finh->unique, state->finh->nodeid,
-                        state->loc.path);
+        send_fuse_err(state->this, state->finh, state->resolve.op_errno);
+        free_fuse_state(state);
+        return;
+    }
 
-                FUSE_FOP (state, fuse_attr_cbk, GF_FOP_STAT,
-                          stat, &state->loc, state->xdata);
-        } else {
+    if (state->fd == NULL && !IA_ISDIR(state->loc.inode->ia_type)) {
+        state->fd = fd_lookup(state->loc.inode, state->finh->pid);
 
-                gf_log ("glusterfs-fuse", GF_LOG_TRACE,
-                        "%"PRIu64": FGETATTR %"PRIu64" (%s/%p)",
-                        state->finh->unique, state->finh->nodeid,
-                        state->loc.path, state->fd);
+        if (state->fd == NULL)
+            state->fd = fd_lookup(state->loc.inode, 0);
+    }
 
-                FUSE_FOP (state, fuse_attr_cbk, GF_FOP_FSTAT,
-                          fstat, state->fd, state->xdata);
-        }
+    if (!state->fd) {
+        gf_log("glusterfs-fuse", GF_LOG_TRACE,
+               "%" PRIu64 ": GETATTR %" PRIu64 " (%s)", state->finh->unique,
+               state->finh->nodeid, state->loc.path);
+
+        FUSE_FOP(state, fuse_attr_cbk, GF_FOP_STAT, stat, &state->loc,
+                 state->xdata);
+    } else {
+        gf_log("glusterfs-fuse", GF_LOG_TRACE,
+               "%" PRIu64 ": FGETATTR %" PRIu64 " (%s/%p)", state->finh->unique,
+               state->finh->nodeid, state->loc.path, state->fd);
+
+        FUSE_FOP(state, fuse_attr_cbk, GF_FOP_FSTAT, fstat, state->fd,
+                 state->xdata);
+    }
 }
 
 static void
-fuse_getattr (xlator_t *this, fuse_in_header_t *finh, void *msg)
+fuse_getattr(xlator_t *this, fuse_in_header_t *finh, void *msg,
+             struct iobuf *iobuf)
 {
-        fuse_state_t *state;
-        int32_t       ret = -1;
-
-        GET_STATE (this, finh, state);
+#if FUSE_KERNEL_MINOR_VERSION >= 9
+    struct fuse_getattr_in *fgi = msg;
+    fuse_private_t *priv = NULL;
+#endif
+    fuse_state_t *state;
+    int ret = -1;
 
-        if (finh->nodeid == 1) {
-                state->gfid[15] = 1;
+    GET_STATE(this, finh, state);
+#if FUSE_KERNEL_MINOR_VERSION >= 9
+    priv = this->private;
+    if (priv->proto_minor >= 9 && fgi->getattr_flags & FUSE_GETATTR_FH)
+        state->fd = fd_ref((fd_t *)(uintptr_t)fgi->fh);
+#endif
+    if (finh->nodeid == 1) {
+        state->gfid[15] = 1;
 
-                ret = fuse_loc_fill (&state->loc, state, finh->nodeid, 0, NULL);
-                if (ret < 0) {
-                        gf_log ("glusterfs-fuse", GF_LOG_WARNING,
-                                "%"PRIu64": GETATTR on / (fuse_loc_fill() failed)",
-                                finh->unique);
-                        send_fuse_err (this, finh, ENOENT);
-                        free_fuse_state (state);
-                        return;
-                }
+        ret = fuse_loc_fill(&state->loc, state, finh->nodeid, 0, NULL);
+        if (ret < 0) {
+            gf_log("glusterfs-fuse", GF_LOG_WARNING,
+                   "%" PRIu64 ": GETATTR on / (fuse_loc_fill() failed)",
+                   finh->unique);
+            send_fuse_err(this, finh, ESTALE);
+            free_fuse_state(state);
+            return;
+        }
 
-                fuse_gfid_set (state);
+        fuse_gfid_set(state);
 
-                FUSE_FOP (state, fuse_root_lookup_cbk, GF_FOP_LOOKUP,
-                          lookup, &state->loc, state->xdata);
-                return;
-        }
+        FUSE_FOP(state, fuse_root_lookup_cbk, GF_FOP_LOOKUP, lookup,
+                 &state->loc, state->xdata);
+        return;
+    }
 
-        fuse_resolve_inode_init (state, &state->resolve, state->finh->nodeid);
+    if (state->fd)
+        fuse_resolve_fd_init(state, &state->resolve, state->fd);
+    else
+        fuse_resolve_inode_init(state, &state->resolve, state->finh->nodeid);
 
-        fuse_resolve_and_resume (state, fuse_getattr_resume);
+    fuse_resolve_and_resume(state, fuse_getattr_resume);
 }
 
 static int32_t
-fuse_fd_inherit_directio (xlator_t *this, fd_t *fd, struct fuse_open_out *foo)
-{
-        int32_t        ret    = 0;
-        fuse_fd_ctx_t *fdctx  = NULL, *tmp_fdctx = NULL;
-        fd_t          *tmp_fd = NULL;
-
-        GF_VALIDATE_OR_GOTO_WITH_ERROR ("glusterfs-fuse", this, out, ret,
-                                        -EINVAL);
-        GF_VALIDATE_OR_GOTO_WITH_ERROR ("glusterfs-fuse", fd, out, ret,
-                                        -EINVAL);
-        GF_VALIDATE_OR_GOTO_WITH_ERROR ("glusterfs-fuse", foo, out, ret,
-                                        -EINVAL);
-
-        fdctx = fuse_fd_ctx_get (this, fd);
-        if (!fdctx) {
-                ret = -ENOMEM;
-                goto out;
-        }
+fuse_fd_inherit_directio(xlator_t *this, fd_t *fd, struct fuse_open_out *foo)
+{
+    int32_t ret = 0;
+    fuse_fd_ctx_t *fdctx = NULL, *tmp_fdctx = NULL;
+    fd_t *tmp_fd = NULL;
 
-        tmp_fd = fd_lookup (fd->inode, 0);
-        if (tmp_fd) {
-                tmp_fdctx = fuse_fd_ctx_get (this, tmp_fd);
-                if (tmp_fdctx) {
-                        foo->open_flags &= ~FOPEN_DIRECT_IO;
-                        foo->open_flags |= (tmp_fdctx->open_flags
-                                            & FOPEN_DIRECT_IO);
-                }
-        }
+    GF_VALIDATE_OR_GOTO_WITH_ERROR("glusterfs-fuse", this, out, ret, -EINVAL);
+    GF_VALIDATE_OR_GOTO_WITH_ERROR("glusterfs-fuse", fd, out, ret, -EINVAL);
+    GF_VALIDATE_OR_GOTO_WITH_ERROR("glusterfs-fuse", foo, out, ret, -EINVAL);
 
-        fdctx->open_flags |= (foo->open_flags & FOPEN_DIRECT_IO);
+    fdctx = fuse_fd_ctx_get(this, fd);
+    if (!fdctx) {
+        ret = -ENOMEM;
+        goto out;
+    }
 
-        if (tmp_fd != NULL) {
-                fd_unref (tmp_fd);
+    tmp_fd = fd_lookup(fd->inode, 0);
+    if (tmp_fd) {
+        tmp_fdctx = fuse_fd_ctx_get(this, tmp_fd);
+        if (tmp_fdctx) {
+            foo->open_flags &= ~FOPEN_DIRECT_IO;
+            foo->open_flags |= (tmp_fdctx->open_flags & FOPEN_DIRECT_IO);
         }
+    }
 
-        ret = 0;
+    fdctx->open_flags |= (foo->open_flags & FOPEN_DIRECT_IO);
+
+    if (tmp_fd != NULL) {
+        fd_unref(tmp_fd);
+    }
+
+    ret = 0;
 out:
-        return ret;
+    return ret;
 }
 
-
 gf_boolean_t
-direct_io_mode (dict_t *xdata)
+direct_io_mode(dict_t *xdata)
 {
-	if (xdata && dict_get (xdata, "direct-io-mode"))
-		return _gf_true;
-	return _gf_false;
+    if (xdata && dict_get(xdata, "direct-io-mode"))
+        return _gf_true;
+    return _gf_false;
 }
 
-
 static int
-fuse_fd_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-             int32_t op_ret, int32_t op_errno, fd_t *fd, dict_t *xdata)
-{
-        fuse_state_t         *state  = NULL;
-        fuse_in_header_t     *finh   = NULL;
-        fuse_private_t       *priv   = NULL;
-        int32_t               ret    = 0;
-        struct fuse_open_out  foo    = {0, };
-
-        priv = this->private;
-        state = frame->root->state;
-        finh = state->finh;
-
-        fuse_log_eh_fop(this, state, frame, op_ret, op_errno);
-
-        if (op_ret >= 0) {
-                foo.fh = (uintptr_t) fd;
-                foo.open_flags = 0;
-
-                if (!IA_ISDIR (fd->inode->ia_type)) {
-                        if (((priv->direct_io_mode == 2)
-                             && ((state->flags & O_ACCMODE) != O_RDONLY))
-                            || (priv->direct_io_mode == 1)
-			    || (direct_io_mode (xdata)))
-                                foo.open_flags |= FOPEN_DIRECT_IO;
+fuse_fd_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int32_t op_ret,
+            int32_t op_errno, fd_t *fd, dict_t *xdata)
+{
+    fuse_state_t *state = NULL;
+    fuse_in_header_t *finh = NULL;
+    fuse_private_t *priv = NULL;
+    int32_t ret = 0;
+    struct fuse_open_out foo = {
+        0,
+    };
+
+    priv = this->private;
+    state = frame->root->state;
+    finh = state->finh;
+
+    fuse_log_eh_fop(this, state, frame, op_ret, op_errno);
+
+    if (op_ret >= 0) {
+        foo.fh = (uintptr_t)fd;
+        foo.open_flags = 0;
+
+        if (!IA_ISDIR(fd->inode->ia_type)) {
+            if (((priv->direct_io_mode == 2) &&
+                 ((state->flags & O_ACCMODE) != O_RDONLY)) ||
+                (priv->direct_io_mode == 1) || (direct_io_mode(xdata)))
+                foo.open_flags |= FOPEN_DIRECT_IO;
 #ifdef GF_DARWIN_HOST_OS
-                        /* In Linux: by default, buffer cache
-                         * is purged upon open, setting
-                         * FOPEN_KEEP_CACHE implies no-purge
-                         *
-                         * In MacFUSE: by default, buffer cache
-                         * is left intact upon open, setting
-                         * FOPEN_PURGE_UBC implies purge
-                         *
-                         * [[Interesting...]]
-                         */
-                        if (!priv->fopen_keep_cache)
-                                foo.open_flags |= FOPEN_PURGE_UBC;
+            /* In Linux: by default, buffer cache
+             * is purged upon open, setting
+             * FOPEN_KEEP_CACHE implies no-purge
+             *
+             * In MacFUSE: by default, buffer cache
+             * is left intact upon open, setting
+             * FOPEN_PURGE_UBC implies purge
+             *
+             * [[Interesting...]]
+             */
+            if (!priv->fopen_keep_cache)
+                foo.open_flags |= FOPEN_PURGE_UBC;
 #else
-                        /*
-                         * If fopen-keep-cache is enabled, we set the associated
-                         * flag here such that files are not invalidated on open.
-                         * File invalidations occur either in fuse or explicitly
-                         * when the cache is set invalid on the inode.
-                         */
-                        if (priv->fopen_keep_cache)
-                                foo.open_flags |= FOPEN_KEEP_CACHE;
+            /*
+             * If fopen-keep-cache is enabled, we set the associated
+             * flag here such that files are not invalidated on open.
+             * File invalidations occur either in fuse or explicitly
+             * when the cache is set invalid on the inode.
+             */
+            if (priv->fopen_keep_cache)
+                foo.open_flags |= FOPEN_KEEP_CACHE;
 #endif
-                }
+        }
 
-                gf_log ("glusterfs-fuse", GF_LOG_TRACE,
-                        "%"PRIu64": %s() %s => %p", frame->root->unique,
-                        gf_fop_list[frame->root->op], state->loc.path, fd);
-
-                ret = fuse_fd_inherit_directio (this, fd, &foo);
-                if (ret < 0) {
-                        op_errno = -ret;
-                        gf_log ("glusterfs-fuse", GF_LOG_WARNING,
-                                "cannot inherit direct-io values for fd "
-                                "(ptr:%p inode-gfid:%s) from fds already "
-                                "opened", fd, uuid_utoa (fd->inode->gfid));
-                        goto err;
-                }
+        gf_log("glusterfs-fuse", GF_LOG_TRACE, "%" PRIu64 ": %s() %s => %p",
+               frame->root->unique, gf_fop_list[frame->root->op],
+               state->loc.path, fd);
 
-                if (send_fuse_obj (this, finh, &foo) == ENOENT) {
-                        gf_log ("glusterfs-fuse", GF_LOG_DEBUG,
-                                "open(%s) got EINTR", state->loc.path);
-                        gf_fd_put (priv->fdtable, state->fd_no);
-                        goto out;
-                }
+        ret = fuse_fd_inherit_directio(this, fd, &foo);
+        if (ret < 0) {
+            op_errno = -ret;
+            gf_log("glusterfs-fuse", GF_LOG_WARNING,
+                   "cannot inherit direct-io values for fd "
+                   "(ptr:%p inode-gfid:%s) from fds already "
+                   "opened",
+                   fd, uuid_utoa(fd->inode->gfid));
+            goto err;
+        }
+
+        if (send_fuse_obj(this, finh, &foo) == ENOENT) {
+            gf_log("glusterfs-fuse", GF_LOG_DEBUG, "open(%s) got EINTR",
+                   state->loc.path);
+            gf_fd_put(priv->fdtable, state->fd_no);
+            goto out;
+        }
+
+        fd_bind(fd);
+    } else {
+    err:
+        /* OPEN(DIR) being an operation on inode should never fail with
+         * ENOENT. If gfid is not present, the appropriate error is
+         * ESTALE.
+         */
+        if (op_errno == ENOENT)
+            op_errno = ESTALE;
 
-                fd_bind (fd);
-        } else {
-        err:
-                gf_log ("glusterfs-fuse", GF_LOG_WARNING,
-                        "%"PRIu64": %s() %s => -1 (%s)", frame->root->unique,
-                        gf_fop_list[frame->root->op], state->loc.path,
-                        strerror (op_errno));
-
-                send_fuse_err (this, finh, op_errno);
-                gf_fd_put (priv->fdtable, state->fd_no);
-        }
+        gf_log("glusterfs-fuse", GF_LOG_WARNING,
+               "%" PRIu64 ": %s() %s => -1 (%s)", frame->root->unique,
+               gf_fop_list[frame->root->op], state->loc.path,
+               strerror(op_errno));
+
+        send_fuse_err(this, finh, op_errno);
+        gf_fd_put(priv->fdtable, state->fd_no);
+    }
 out:
-        free_fuse_state (state);
-        STACK_DESTROY (frame->root);
-        return 0;
+    free_fuse_state(state);
+    STACK_DESTROY(frame->root);
+    return 0;
 }
 
 static void
-fuse_do_truncate (fuse_state_t *state)
+fuse_do_truncate(fuse_state_t *state)
 {
-        if (state->fd) {
-                FUSE_FOP (state, fuse_truncate_cbk, GF_FOP_FTRUNCATE,
-                          ftruncate, state->fd, state->off, state->xdata);
-        } else {
-                FUSE_FOP (state, fuse_truncate_cbk, GF_FOP_TRUNCATE,
-                          truncate, &state->loc, state->off, state->xdata);
-        }
+    if (state->fd) {
+        FUSE_FOP(state, fuse_truncate_cbk, GF_FOP_FTRUNCATE, ftruncate,
+                 state->fd, state->off, state->xdata);
+    } else {
+        FUSE_FOP(state, fuse_truncate_cbk, GF_FOP_TRUNCATE, truncate,
+                 &state->loc, state->off, state->xdata);
+    }
 
-        return;
+    return;
 }
 
 static int
-fuse_setattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                  int32_t op_ret, int32_t op_errno,
-                  struct iatt *statpre, struct iatt *statpost, dict_t *xdata)
-{
-        fuse_state_t     *state;
-        fuse_in_header_t *finh;
-        fuse_private_t   *priv = NULL;
-        struct fuse_attr_out fao;
-
-        int op_done = 0;
-
-        priv  = this->private;
-        state = frame->root->state;
-        finh  = state->finh;
-
-        fuse_log_eh(this, "op_ret: %d, op_errno: %d, %"PRIu64", %s() %s => "
-                    "gfid: %s", op_ret, op_errno, frame->root->unique,
-                    gf_fop_list[frame->root->op], state->loc.path,
-                    state->loc.inode ? uuid_utoa (state->loc.inode->gfid) : "");
-
-        if (op_ret == 0) {
-                gf_log ("glusterfs-fuse", GF_LOG_TRACE,
-                        "%"PRIu64": %s() %s => %"PRIu64, frame->root->unique,
-                        gf_fop_list[frame->root->op],
-                        state->loc.path ? state->loc.path : "ERR",
-                        statpost->ia_ino);
-
-                statpost->ia_blksize = this->ctx->page_size;
-                gf_fuse_stat2attr (statpost, &fao.attr, priv->enable_ino32);
-
-                fao.attr_valid = calc_timeout_sec (priv->attribute_timeout);
-                fao.attr_valid_nsec =
-                        calc_timeout_nsec (priv->attribute_timeout);
-
-                if (state->truncate_needed) {
-                        fuse_do_truncate (state);
-                } else {
+fuse_setattr_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                 int32_t op_ret, int32_t op_errno, struct iatt *statpre,
+                 struct iatt *statpost, dict_t *xdata)
+{
+    fuse_state_t *state;
+    fuse_in_header_t *finh;
+    fuse_private_t *priv = NULL;
+    struct fuse_attr_out fao;
+
+    int op_done = 0;
+
+    priv = this->private;
+    state = frame->root->state;
+    finh = state->finh;
+
+    fuse_log_eh(this,
+                "op_ret: %d, op_errno: %d, %" PRIu64
+                ", %s() %s => "
+                "gfid: %s",
+                op_ret, op_errno, frame->root->unique,
+                gf_fop_list[frame->root->op], state->loc.path,
+                state->loc.inode ? uuid_utoa(state->loc.inode->gfid) : "");
+
+    if (op_ret == 0) {
+        gf_log("glusterfs-fuse", GF_LOG_TRACE,
+               "%" PRIu64 ": %s() %s => %" PRIu64, frame->root->unique,
+               gf_fop_list[frame->root->op],
+               state->loc.path ? state->loc.path : "ERR", statpost->ia_ino);
+
+        statpost->ia_blksize = this->ctx->page_size;
+        gf_fuse_stat2attr(statpost, &fao.attr, priv->enable_ino32);
+
+        fao.attr_valid = calc_timeout_sec(priv->attribute_timeout);
+        fao.attr_valid_nsec = calc_timeout_nsec(priv->attribute_timeout);
+
+        if (state->truncate_needed) {
+            fuse_do_truncate(state);
+        } else {
 #if FUSE_KERNEL_MINOR_VERSION >= 9
-                        priv->proto_minor >= 9 ?
-                        send_fuse_obj (this, finh, &fao) :
-                        send_fuse_data (this, finh, &fao,
-                                        FUSE_COMPAT_ATTR_OUT_SIZE);
+            priv->proto_minor >= 9
+                ? send_fuse_obj(this, finh, &fao)
+                : send_fuse_data(this, finh, &fao, FUSE_COMPAT_ATTR_OUT_SIZE);
 #else
-                        send_fuse_obj (this, finh, &fao);
+            send_fuse_obj(this, finh, &fao);
 #endif
-                        op_done = 1;
-                }
-        } else {
-                gf_log ("glusterfs-fuse", GF_LOG_WARNING,
-                        "%"PRIu64": %s() %s => -1 (%s)", frame->root->unique,
-                        gf_fop_list[frame->root->op],
-                        state->loc.path ? state->loc.path : "ERR",
-                        strerror (op_errno));
-
-                send_fuse_err (this, finh, op_errno);
-                op_done = 1;
+            op_done = 1;
         }
+    } else {
+        gf_log("glusterfs-fuse", GF_LOG_WARNING,
+               "%" PRIu64 ": %s() %s => -1 (%s)", frame->root->unique,
+               gf_fop_list[frame->root->op],
+               state->loc.path ? state->loc.path : "ERR", strerror(op_errno));
 
-        if (op_done) {
-                free_fuse_state (state);
-        }
+        /* facilitate retry from VFS */
+        if ((state->fd == NULL) && (op_errno == ENOENT))
+            op_errno = ESTALE;
 
-        STACK_DESTROY (frame->root);
+        send_fuse_err(this, finh, op_errno);
+        op_done = 1;
+    }
 
-        return 0;
+    if (op_done) {
+        free_fuse_state(state);
+    }
+
+    STACK_DESTROY(frame->root);
+
+    return 0;
 }
 
 static int32_t
-fattr_to_gf_set_attr (int32_t valid)
+fattr_to_gf_set_attr(int32_t valid)
 {
-        int32_t gf_valid = 0;
+    int32_t gf_valid = 0;
+
+    if (valid & FATTR_MODE)
+        gf_valid |= GF_SET_ATTR_MODE;
+
+    if (valid & FATTR_UID)
+        gf_valid |= GF_SET_ATTR_UID;
 
-        if (valid & FATTR_MODE)
-                gf_valid |= GF_SET_ATTR_MODE;
+    if (valid & FATTR_GID)
+        gf_valid |= GF_SET_ATTR_GID;
 
-        if (valid & FATTR_UID)
-                gf_valid |= GF_SET_ATTR_UID;
+    if (valid & FATTR_ATIME)
+        gf_valid |= GF_SET_ATTR_ATIME;
 
-        if (valid & FATTR_GID)
-                gf_valid |= GF_SET_ATTR_GID;
+    if (valid & FATTR_MTIME)
+        gf_valid |= GF_SET_ATTR_MTIME;
 
-        if (valid & FATTR_ATIME)
-                gf_valid |= GF_SET_ATTR_ATIME;
+#if FUSE_KERNEL_MINOR_VERSION >= 23
+    if (valid & FATTR_CTIME)
+        gf_valid |= GF_SET_ATTR_CTIME;
+#endif
 
-        if (valid & FATTR_MTIME)
-                gf_valid |= GF_SET_ATTR_MTIME;
+#if FUSE_KERNEL_MINOR_VERSION >= 9
+    if (valid & FATTR_ATIME_NOW)
+        gf_valid |= GF_ATTR_ATIME_NOW;
 
-        if (valid & FATTR_SIZE)
-                gf_valid |= GF_SET_ATTR_SIZE;
+    if (valid & FATTR_MTIME_NOW)
+        gf_valid |= GF_ATTR_MTIME_NOW;
+#endif
 
-        return gf_valid;
+    if (valid & FATTR_SIZE)
+        gf_valid |= GF_SET_ATTR_SIZE;
+
+    return gf_valid;
 }
 
-#define FATTR_MASK   (FATTR_SIZE                        \
-                      | FATTR_UID | FATTR_GID           \
-                      | FATTR_ATIME | FATTR_MTIME       \
-                      | FATTR_MODE)
+#define FATTR_MASK                                                             \
+    (FATTR_SIZE | FATTR_UID | FATTR_GID | FATTR_ATIME | FATTR_MTIME |          \
+     FATTR_MODE)
 
 void
-fuse_setattr_resume (fuse_state_t *state)
-{
-        if (!state->fd && !state->loc.inode) {
-                gf_log ("glusterfs-fuse", GF_LOG_ERROR,
-                        "%"PRIu64": SETATTR %"PRIu64" (%s) resolution failed",
-                        state->finh->unique, state->finh->nodeid,
-                        uuid_utoa (state->resolve.gfid));
-                send_fuse_err (state->this, state->finh,
-                               state->resolve.op_errno);
-                free_fuse_state (state);
-                return;
-        }
+fuse_setattr_resume(fuse_state_t *state)
+{
+    if (!state->fd && !state->loc.inode) {
+        gf_log("glusterfs-fuse", GF_LOG_ERROR,
+               "%" PRIu64 ": SETATTR %" PRIu64 " (%s) resolution failed",
+               state->finh->unique, state->finh->nodeid,
+               uuid_utoa(state->resolve.gfid));
+
+        /* facilitate retry from VFS */
+        if ((state->fd == NULL) && (state->resolve.op_errno == ENOENT))
+            state->resolve.op_errno = ESTALE;
 
-        gf_log ("glusterfs-fuse", GF_LOG_TRACE,
-                "%"PRIu64": SETATTR (%"PRIu64")%s", state->finh->unique,
-                state->finh->nodeid, state->loc.path);
+        send_fuse_err(state->this, state->finh, state->resolve.op_errno);
+        free_fuse_state(state);
+        return;
+    }
+
+    gf_log("glusterfs-fuse", GF_LOG_TRACE,
+           "%" PRIu64 ": SETATTR (%" PRIu64 ")%s", state->finh->unique,
+           state->finh->nodeid, state->loc.path);
 
 #ifdef GF_TEST_FFOP
-        /* this is for calls like 'fchmod()' */
-        if (!state->fd)
-                state->fd = fd_lookup (state->loc.inode, state->finh->pid);
+    /* this is for calls like 'fchmod()' */
+    if (!state->fd)
+        state->fd = fd_lookup(state->loc.inode, state->finh->pid);
 #endif /* GF_TEST_FFOP */
 
-        if ((state->valid & (FATTR_MASK)) != FATTR_SIZE) {
-                if (state->fd &&
-                    !((state->valid & FATTR_ATIME) ||
-                      (state->valid & FATTR_MTIME))) {
-                        /*
-                            there is no "futimes" call, so don't send
-                            fsetattr if ATIME or MTIME is set
-                         */
-
-                        FUSE_FOP (state, fuse_setattr_cbk, GF_FOP_FSETATTR,
-                                  fsetattr, state->fd, &state->attr,
-                                  fattr_to_gf_set_attr (state->valid),
-                                  state->xdata);
-                } else {
-                        FUSE_FOP (state, fuse_setattr_cbk, GF_FOP_SETATTR,
-                                  setattr, &state->loc, &state->attr,
-                                  fattr_to_gf_set_attr (state->valid),
-                                  state->xdata);
-                }
+    if ((state->valid & (FATTR_MASK)) != FATTR_SIZE) {
+        if (state->fd &&
+            !((state->valid & FATTR_ATIME) || (state->valid & FATTR_MTIME)
+#if FUSE_KERNEL_MINOR_VERSION >= 23
+              || (state->valid & FATTR_CTIME)
+#endif
+                  )) {
+            /*
+                there is no "futimes" call, so don't send
+                fsetattr if ATIME or MTIME is set
+             */
+
+            FUSE_FOP(state, fuse_setattr_cbk, GF_FOP_FSETATTR, fsetattr,
+                     state->fd, &state->attr,
+                     fattr_to_gf_set_attr(state->valid), state->xdata);
         } else {
-                fuse_do_truncate (state);
+            FUSE_FOP(state, fuse_setattr_cbk, GF_FOP_SETATTR, setattr,
+                     &state->loc, &state->attr,
+                     fattr_to_gf_set_attr(state->valid), state->xdata);
         }
-
+    } else {
+        fuse_do_truncate(state);
+    }
 }
 
 static void
-fuse_setattr (xlator_t *this, fuse_in_header_t *finh, void *msg)
+fuse_setattr(xlator_t *this, fuse_in_header_t *finh, void *msg,
+             struct iobuf *iobuf)
 {
-        struct fuse_setattr_in *fsi = msg;
+    struct fuse_setattr_in *fsi = msg;
 
 #if FUSE_KERNEL_MINOR_VERSION >= 9
-        fuse_private_t  *priv = NULL;
+    fuse_private_t *priv = NULL;
+#endif
+    fuse_state_t *state = NULL;
+
+    GET_STATE(this, finh, state);
+
+    if (fsi->valid & FATTR_FH && !(fsi->valid & (FATTR_ATIME | FATTR_MTIME))) {
+        /* We need no loc if kernel sent us an fd and
+         * we are not fiddling with times */
+        state->fd = FH_TO_FD(fsi->fh);
+        fuse_resolve_fd_init(state, &state->resolve, state->fd);
+    } else {
+        fuse_resolve_inode_init(state, &state->resolve, finh->nodeid);
+    }
+
+    /*
+     * This is just stub code demonstrating how to retrieve
+     * lock_owner in setattr, according to the FUSE proto.
+     * We do not make use of ATM. Its purpose is supporting
+     * mandatory locking, but getting that right is further
+     * down the road. Cf.
+     *
+     * http://thread.gmane.org/gmane.comp.file-systems.fuse.devel/
+     * 4962/focus=4982
+     *
+     * http://git.kernel.org/?p=linux/kernel/git/torvalds/
+     * linux-2.6.git;a=commit;h=v2.6.23-5896-gf333211
+     */
+#if FUSE_KERNEL_MINOR_VERSION >= 9
+    priv = this->private;
+    if (priv->proto_minor >= 9 && fsi->valid & FATTR_LOCKOWNER)
+        state->lk_owner = fsi->lock_owner;
 #endif
-        fuse_state_t *state = NULL;
 
-        GET_STATE (this, finh, state);
+    state->valid = fsi->valid;
 
-        if (fsi->valid & FATTR_FH &&
-            !(fsi->valid & (FATTR_ATIME|FATTR_MTIME))) {
-                /* We need no loc if kernel sent us an fd and
-                 * we are not fiddling with times */
-                state->fd = FH_TO_FD (fsi->fh);
-                fuse_resolve_fd_init (state, &state->resolve, state->fd);
-        } else {
-                fuse_resolve_inode_init (state, &state->resolve, finh->nodeid);
+    if ((fsi->valid & (FATTR_MASK)) != FATTR_SIZE) {
+        if (fsi->valid & FATTR_SIZE) {
+            state->off = fsi->size;
+            state->truncate_needed = _gf_true;
         }
 
-        /*
-         * This is just stub code demonstrating how to retrieve
-         * lock_owner in setattr, according to the FUSE proto.
-         * We do not make use of ATM. Its purpose is supporting
-         * mandatory locking, but getting that right is further
-         * down the road. Cf.
-         *
-         * http://thread.gmane.org/gmane.comp.file-systems.fuse.devel/
-         * 4962/focus=4982
-         *
-         * http://git.kernel.org/?p=linux/kernel/git/torvalds/
-         * linux-2.6.git;a=commit;h=v2.6.23-5896-gf333211
-         */
-#if FUSE_KERNEL_MINOR_VERSION >= 9
-        priv = this->private;
-        if (priv->proto_minor >= 9 && fsi->valid & FATTR_LOCKOWNER)
-                state->lk_owner = fsi->lock_owner;
+        state->attr.ia_size = fsi->size;
+        state->attr.ia_atime = fsi->atime;
+        state->attr.ia_mtime = fsi->mtime;
+#if FUSE_KERNEL_MINOR_VERSION >= 23
+        state->attr.ia_ctime = fsi->ctime;
+#endif
+        state->attr.ia_atime_nsec = fsi->atimensec;
+        state->attr.ia_mtime_nsec = fsi->mtimensec;
+#if FUSE_KERNEL_MINOR_VERSION >= 23
+        state->attr.ia_ctime_nsec = fsi->ctimensec;
 #endif
 
-        state->valid = fsi->valid;
+        state->attr.ia_prot = ia_prot_from_st_mode(fsi->mode);
+        state->attr.ia_uid = fsi->uid;
+        state->attr.ia_gid = fsi->gid;
+    } else {
+        state->off = fsi->size;
+    }
 
-        if ((fsi->valid & (FATTR_MASK)) != FATTR_SIZE) {
-                if (fsi->valid & FATTR_SIZE) {
-                        state->off             = fsi->size;
-                        state->truncate_needed = _gf_true;
-                }
+    fuse_resolve_and_resume(state, fuse_setattr_resume);
+}
 
-                state->attr.ia_size  = fsi->size;
-                state->attr.ia_atime = fsi->atime;
-                state->attr.ia_mtime = fsi->mtime;
-                state->attr.ia_atime_nsec = fsi->atimensec;
-                state->attr.ia_mtime_nsec = fsi->mtimensec;
+static int
+fuse_removexattr_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                     int32_t op_ret, int32_t op_errno, dict_t *xdata)
+{
+    fuse_state_t *state = NULL;
+    fuse_in_header_t *finh = NULL;
 
-                state->attr.ia_prot = ia_prot_from_st_mode (fsi->mode);
-                state->attr.ia_uid  = fsi->uid;
-                state->attr.ia_gid  = fsi->gid;
-        } else {
-                state->off = fsi->size;
-        }
+    GF_ASSERT(frame);
+    GF_ASSERT(frame->root);
+
+    state = frame->root->state;
+    finh = state->finh;
+
+    fuse_log_eh_fop(this, state, frame, op_ret, op_errno);
+
+    if (op_ret == 0) {
+        gf_log("glusterfs-fuse", GF_LOG_TRACE, "%" PRIu64 ": %s() %s => 0",
+               frame->root->unique, gf_fop_list[frame->root->op],
+               state->loc.path ? state->loc.path : "ERR");
+
+        send_fuse_err(this, finh, 0);
+    } else {
+        gf_log("glusterfs-fuse",
+               (ENODATA == op_errno) ? GF_LOG_DEBUG : GF_LOG_WARNING,
+               "%" PRIu64 ": %s() of %s on %s => -1 (%s)", frame->root->unique,
+               gf_fop_list[frame->root->op], state->name ? state->name : "",
+               state->loc.path ? state->loc.path : "ERR", strerror(op_errno));
+
+        /* facilitate retry from VFS */
+        if ((state->fd == NULL) && (op_errno == ENOENT))
+            op_errno = ESTALE;
+
+        send_fuse_err(this, finh, op_errno);
+    }
+
+    free_fuse_state(state);
+    STACK_DESTROY(frame->root);
 
-        fuse_resolve_and_resume (state, fuse_setattr_resume);
+    return 0;
 }
 
 static int
-fuse_err_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-              int32_t op_ret, int32_t op_errno, dict_t *xdata)
+fuse_err_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int32_t op_ret,
+             int32_t op_errno, dict_t *xdata)
 {
-        fuse_state_t *state = frame->root->state;
-        fuse_in_header_t *finh = state->finh;
+    fuse_state_t *state = frame->root->state;
+    fuse_in_header_t *finh = state->finh;
 
-        fuse_log_eh_fop(this, state, frame, op_ret, op_errno);
+    fuse_log_eh_fop(this, state, frame, op_ret, op_errno);
 
-        if (op_ret == 0) {
-                gf_log ("glusterfs-fuse", GF_LOG_TRACE,
-                        "%"PRIu64": %s() %s => 0", frame->root->unique,
-                        gf_fop_list[frame->root->op],
-                        state->loc.path ? state->loc.path : "ERR");
+    if (op_ret == 0) {
+        gf_log("glusterfs-fuse", GF_LOG_TRACE, "%" PRIu64 ": %s() %s => 0",
+               frame->root->unique, gf_fop_list[frame->root->op],
+               state->loc.path ? state->loc.path : "ERR");
 
-                send_fuse_err (this, finh, 0);
-        } else {
-                gf_log ("glusterfs-fuse", GF_LOG_WARNING,
-                        "%"PRIu64": %s() %s => -1 (%s)",
-                        frame->root->unique,
-                        gf_fop_list[frame->root->op],
-                        state->loc.path ? state->loc.path : "ERR",
-                        strerror (op_errno));
-
-                send_fuse_err (this, finh, op_errno);
+        send_fuse_err(this, finh, 0);
+    } else {
+        if (GF_IGNORE_IF_GSYNCD_SAFE_ERROR(frame, op_errno)) {
+            gf_log("glusterfs-fuse", GF_LOG_WARNING,
+                   "%" PRIu64 ": %s() %s => -1 (%s)", frame->root->unique,
+                   gf_fop_list[frame->root->op],
+                   state->loc.path ? state->loc.path : "ERR",
+                   strerror(op_errno));
         }
 
-        free_fuse_state (state);
-        STACK_DESTROY (frame->root);
+        /* facilitate retry from VFS */
+        if ((state->fd == NULL) && (op_errno == ENOENT))
+            op_errno = ESTALE;
 
-        return 0;
+        send_fuse_err(this, finh, op_errno);
+    }
+
+    free_fuse_state(state);
+    STACK_DESTROY(frame->root);
+
+    return 0;
 }
 
 static int
-fuse_fsync_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
-                struct iatt *postbuf, dict_t *xdata)
+fuse_flush_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+               int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
-        return fuse_err_cbk (frame, cookie, this, op_ret, op_errno, xdata);
+    fuse_private_t *priv = this->private;
+
+    if (priv->flush_handle_interrupt) {
+        if (fuse_interrupt_finish_fop(frame, this, _gf_false, NULL)) {
+            return 0;
+        }
+    }
+
+    return fuse_err_cbk(frame, cookie, this, op_ret, op_errno, xdata);
 }
 
 static int
-fuse_setxattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                   int32_t op_ret, int32_t op_errno, dict_t *xdata)
+fuse_fsync_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+               int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
+               struct iatt *postbuf, dict_t *xdata)
 {
-        if (op_ret == -1 && op_errno == ENOTSUP)
-                GF_LOG_OCCASIONALLY (gf_fuse_xattr_enotsup_log,
-                                     "glusterfs-fuse", GF_LOG_CRITICAL,
-                                     "extended attribute not supported "
-                                     "by the backend storage");
-
-        return fuse_err_cbk (frame, cookie, this, op_ret, op_errno, xdata);
+    return fuse_err_cbk(frame, cookie, this, op_ret, op_errno, xdata);
 }
 
 static int
-fuse_unlink_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                 int32_t op_ret, int32_t op_errno, struct iatt *preparent,
-                 struct iatt *postparent, dict_t *xdata)
+fuse_setxattr_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                  int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
-        fuse_state_t     *state = NULL;
-        fuse_in_header_t *finh = NULL;
-
-        state = frame->root->state;
-        finh = state->finh;
+    if (op_ret == -1 && op_errno == ENOTSUP)
+        GF_LOG_OCCASIONALLY(gf_fuse_xattr_enotsup_log, "glusterfs-fuse",
+                            GF_LOG_CRITICAL,
+                            "extended attribute not supported "
+                            "by the backend storage");
 
-        fuse_log_eh (this, "op_ret: %d, op_errno: %d, %"PRIu64": %s() %s => "
-                     "gfid: %s", op_ret, op_errno, frame->root->unique,
-                     gf_fop_list[frame->root->op], state->loc.path,
-                     state->loc.inode ? uuid_utoa (state->loc.inode->gfid) : "");
-
-        if (op_ret == 0) {
-                inode_unlink (state->loc.inode, state->loc.parent,
-                              state->loc.name);
-                gf_log ("glusterfs-fuse", GF_LOG_TRACE,
-                        "%"PRIu64": %s() %s => 0", frame->root->unique,
-                        gf_fop_list[frame->root->op], state->loc.path);
-
-                send_fuse_err (this, finh, 0);
-        } else {
-                gf_log ("glusterfs-fuse",
-                        op_errno == ENOTEMPTY ? GF_LOG_DEBUG : GF_LOG_WARNING,
-                        "%"PRIu64": %s() %s => -1 (%s)", frame->root->unique,
-                        gf_fop_list[frame->root->op], state->loc.path,
-                        strerror (op_errno));
+    return fuse_err_cbk(frame, cookie, this, op_ret, op_errno, xdata);
+}
 
-                send_fuse_err (this, finh, op_errno);
+static int
+fuse_unlink_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                int32_t op_ret, int32_t op_errno, struct iatt *preparent,
+                struct iatt *postparent, dict_t *xdata)
+{
+    fuse_state_t *state = NULL;
+    fuse_in_header_t *finh = NULL;
+
+    state = frame->root->state;
+    finh = state->finh;
+
+    fuse_log_eh(this,
+                "op_ret: %d, op_errno: %d, %" PRIu64
+                ": %s() %s => "
+                "gfid: %s",
+                op_ret, op_errno, frame->root->unique,
+                gf_fop_list[frame->root->op], state->loc.path,
+                state->loc.inode ? uuid_utoa(state->loc.inode->gfid) : "");
+
+    if (op_ret == 0) {
+        inode_unlink(state->loc.inode, state->loc.parent, state->loc.name);
+        gf_log("glusterfs-fuse", GF_LOG_TRACE, "%" PRIu64 ": %s() %s => 0",
+               frame->root->unique, gf_fop_list[frame->root->op],
+               state->loc.path);
+
+        send_fuse_err(this, finh, 0);
+    } else {
+        if (GF_IGNORE_IF_GSYNCD_SAFE_ERROR(frame, op_errno)) {
+            gf_log("glusterfs-fuse",
+                   op_errno == ENOTEMPTY ? GF_LOG_DEBUG : GF_LOG_WARNING,
+                   "%" PRIu64 ": %s() %s => -1 (%s)", frame->root->unique,
+                   gf_fop_list[frame->root->op], state->loc.path,
+                   strerror(op_errno));
         }
+        send_fuse_err(this, finh, op_errno);
+    }
 
-        free_fuse_state (state);
-        STACK_DESTROY (frame->root);
+    free_fuse_state(state);
+    STACK_DESTROY(frame->root);
 
-        return 0;
+    return 0;
 }
 
 void
-fuse_access_resume (fuse_state_t *state)
-{
-        if (!state->loc.inode) {
-                gf_log ("glusterfs-fuse", GF_LOG_ERROR,
-                        "%"PRIu64": ACCESS %"PRIu64" (%s) resolution failed",
-                        state->finh->unique, state->finh->nodeid,
-                        uuid_utoa (state->resolve.gfid));
-                send_fuse_err (state->this, state->finh,
-                               state->resolve.op_errno);
-                free_fuse_state (state);
-                return;
-        }
+fuse_access_resume(fuse_state_t *state)
+{
+    if (!state->loc.inode) {
+        gf_log("glusterfs-fuse", GF_LOG_ERROR,
+               "%" PRIu64 ": ACCESS %" PRIu64 " (%s) resolution failed",
+               state->finh->unique, state->finh->nodeid,
+               uuid_utoa(state->resolve.gfid));
 
-        gf_log ("glusterfs-fuse", GF_LOG_TRACE,
-                "%"PRIu64" ACCESS %s/%"PRIu64" mask=%d",
-                state->finh->unique, state->loc.path,
-                state->finh->nodeid, state->mask);
+        /* facilitate retry from VFS */
+        if (state->resolve.op_errno == ENOENT)
+            state->resolve.op_errno = ESTALE;
 
-        FUSE_FOP (state, fuse_err_cbk, GF_FOP_ACCESS, access,
-                  &state->loc, state->mask, state->xdata);
+        send_fuse_err(state->this, state->finh, state->resolve.op_errno);
+        free_fuse_state(state);
+        return;
+    }
+
+    gf_log("glusterfs-fuse", GF_LOG_TRACE,
+           "%" PRIu64 " ACCESS %s/%" PRIu64 " mask=%d", state->finh->unique,
+           state->loc.path, state->finh->nodeid, state->mask);
+
+    FUSE_FOP(state, fuse_err_cbk, GF_FOP_ACCESS, access, &state->loc,
+             state->mask, state->xdata);
 }
 
 static void
-fuse_access (xlator_t *this, fuse_in_header_t *finh, void *msg)
+fuse_access(xlator_t *this, fuse_in_header_t *finh, void *msg,
+            struct iobuf *iobuf)
 {
-        struct fuse_access_in *fai = msg;
-        fuse_state_t *state = NULL;
+    struct fuse_access_in *fai = msg;
+    fuse_state_t *state = NULL;
 
-        GET_STATE (this, finh, state);
+    GET_STATE(this, finh, state);
 
-        fuse_resolve_inode_init (state, &state->resolve, finh->nodeid);
+    fuse_resolve_inode_init(state, &state->resolve, finh->nodeid);
 
-        state->mask = fai->mask;
+    state->mask = fai->mask;
 
-        fuse_resolve_and_resume (state, fuse_access_resume);
+    fuse_resolve_and_resume(state, fuse_access_resume);
 
-        return;
+    return;
 }
 
 static int
-fuse_readlink_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                   int32_t op_ret, int32_t op_errno, const char *linkname,
-                   struct iatt *buf, dict_t *xdata)
-{
-        fuse_state_t     *state = NULL;
-        fuse_in_header_t *finh = NULL;
-
-        state = frame->root->state;
-        finh = state->finh;
-
-        fuse_log_eh (this, "op_ret: %d, op_errno: %d %"PRIu64": %s() => %s"
-                     " linkname: %s, gfid: %s", op_ret, op_errno,
-                     frame->root->unique, gf_fop_list[frame->root->op],
-                     state->loc.gfid, linkname,
-                     uuid_utoa (state->loc.gfid));
-
-        if (op_ret > 0) {
-                gf_log ("glusterfs-fuse", GF_LOG_TRACE,
-                        "%"PRIu64": %s => %s (size:%d)", frame->root->unique,
-                        state->loc.path, linkname, op_ret);
-                send_fuse_data (this, finh, (void *)linkname, op_ret);
-        } else {
-                gf_log ("glusterfs-fuse", GF_LOG_WARNING,
-                        "%"PRIu64": %s => -1 (%s)", frame->root->unique,
-                        state->loc.path, strerror (op_errno));
+fuse_readlink_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                  int32_t op_ret, int32_t op_errno, const char *linkname,
+                  struct iatt *buf, dict_t *xdata)
+{
+    fuse_state_t *state = NULL;
+    fuse_in_header_t *finh = NULL;
 
-                send_fuse_err (this, finh, op_errno);
-        }
+    state = frame->root->state;
+    finh = state->finh;
 
-        free_fuse_state (state);
-        STACK_DESTROY (frame->root);
+    fuse_log_eh(this,
+                "op_ret: %d, op_errno: %d %" PRIu64
+                ": %s() => %s"
+                " linkname: %s, gfid: %s",
+                op_ret, op_errno, frame->root->unique,
+                gf_fop_list[frame->root->op], state->loc.gfid, linkname,
+                uuid_utoa(state->loc.gfid));
 
-        return 0;
+    if (op_ret > 0) {
+        gf_log("glusterfs-fuse", GF_LOG_TRACE,
+               "%" PRIu64 ": %s => %s (size:%d)", frame->root->unique,
+               state->loc.path, linkname, op_ret);
+        send_fuse_data(this, finh, (void *)linkname, op_ret);
+    } else {
+        /* facilitate retry from VFS */
+        if (op_errno == ENOENT)
+            op_errno = ESTALE;
+
+        gf_log("glusterfs-fuse", GF_LOG_WARNING, "%" PRIu64 ": %s => -1 (%s)",
+               frame->root->unique, state->loc.path, strerror(op_errno));
+
+        send_fuse_err(this, finh, op_errno);
+    }
+
+    free_fuse_state(state);
+    STACK_DESTROY(frame->root);
+
+    return 0;
 }
 
 void
-fuse_readlink_resume (fuse_state_t *state)
-{
-        if (!state->loc.inode) {
-                gf_log ("glusterfs-fuse", GF_LOG_ERROR,
-                        "READLINK %"PRIu64" (%s) resolution failed",
-                        state->finh->unique, uuid_utoa (state->resolve.gfid));
-                send_fuse_err (state->this, state->finh,
-                               state->resolve.op_errno);
-                free_fuse_state (state);
-                return;
-        }
+fuse_readlink_resume(fuse_state_t *state)
+{
+    if (!state->loc.inode) {
+        gf_log("glusterfs-fuse", GF_LOG_ERROR,
+               "READLINK %" PRIu64 " (%s) resolution failed",
+               state->finh->unique, uuid_utoa(state->resolve.gfid));
+
+        /* facilitate retry from VFS */
+        if (state->resolve.op_errno == ENOENT)
+            state->resolve.op_errno = ESTALE;
 
-        gf_log ("glusterfs-fuse", GF_LOG_TRACE,
-                "%"PRIu64" READLINK %s/%s", state->finh->unique,
-                state->loc.path, uuid_utoa (state->loc.inode->gfid));
+        send_fuse_err(state->this, state->finh, state->resolve.op_errno);
+        free_fuse_state(state);
+        return;
+    }
+
+    gf_log("glusterfs-fuse", GF_LOG_TRACE, "%" PRIu64 " READLINK %s/%s",
+           state->finh->unique, state->loc.path,
+           uuid_utoa(state->loc.inode->gfid));
 
-        FUSE_FOP (state, fuse_readlink_cbk, GF_FOP_READLINK,
-                  readlink, &state->loc, 4096, state->xdata);
+    FUSE_FOP(state, fuse_readlink_cbk, GF_FOP_READLINK, readlink, &state->loc,
+             4096, state->xdata);
 }
 
 static void
-fuse_readlink (xlator_t *this, fuse_in_header_t *finh, void *msg)
+fuse_readlink(xlator_t *this, fuse_in_header_t *finh, void *msg,
+              struct iobuf *iobuf)
 {
-        fuse_state_t *state = NULL;
+    fuse_state_t *state = NULL;
 
-        GET_STATE (this, finh, state);
+    GET_STATE(this, finh, state);
 
-        fuse_resolve_inode_init (state, &state->resolve, finh->nodeid);
+    fuse_resolve_inode_init(state, &state->resolve, finh->nodeid);
 
-        fuse_resolve_and_resume (state, fuse_readlink_resume);
+    fuse_resolve_and_resume(state, fuse_readlink_resume);
 
-        return;
+    return;
 }
 
 void
-fuse_mknod_resume (fuse_state_t *state)
-{
-        if (!state->loc.parent) {
-                gf_log ("glusterfs-fuse", GF_LOG_ERROR,
-                        "MKNOD %"PRIu64"/%s (%s/%s) resolution failed",
-                        state->finh->nodeid, state->resolve.bname,
-                        uuid_utoa (state->resolve.gfid), state->resolve.bname);
-                send_fuse_err (state->this, state->finh,
-                               state->resolve.op_errno);
-                free_fuse_state (state);
-                return;
-        }
+fuse_mknod_resume(fuse_state_t *state)
+{
+    if (!state->loc.parent) {
+        gf_log("glusterfs-fuse", GF_LOG_ERROR,
+               "MKNOD %" PRIu64 "/%s (%s/%s) resolution failed",
+               state->finh->nodeid, state->resolve.bname,
+               uuid_utoa(state->resolve.gfid), state->resolve.bname);
 
-        if (state->resolve.op_errno == ENOENT) {
-                state->resolve.op_ret = 0;
-                state->resolve.op_errno = 0;
-        }
+        /* facilitate retry from VFS */
+        if (state->resolve.op_errno == ENOENT)
+            state->resolve.op_errno = ESTALE;
 
-        if (state->loc.inode) {
-                gf_log (state->this->name, GF_LOG_DEBUG, "inode already present");
-                inode_unref (state->loc.inode);
-                state->loc.inode = NULL;
-        }
+        send_fuse_err(state->this, state->finh, state->resolve.op_errno);
+        free_fuse_state(state);
+        return;
+    }
+
+    if (state->resolve.op_errno == ENOENT) {
+        state->resolve.op_ret = 0;
+        state->resolve.op_errno = 0;
+    }
 
-        state->loc.inode = inode_new (state->loc.parent->table);
+    if (state->loc.inode) {
+        gf_log(state->this->name, GF_LOG_DEBUG, "inode already present");
+        inode_unref(state->loc.inode);
+        state->loc.inode = NULL;
+    }
 
-        gf_log ("glusterfs-fuse", GF_LOG_TRACE,
-                "%"PRIu64": MKNOD %s", state->finh->unique,
-                state->loc.path);
+    state->loc.inode = inode_new(state->loc.parent->table);
 
-        FUSE_FOP (state, fuse_newentry_cbk, GF_FOP_MKNOD,
-                  mknod, &state->loc, state->mode, state->rdev, state->umask,
-                  state->xdata);
+    gf_log("glusterfs-fuse", GF_LOG_TRACE, "%" PRIu64 ": MKNOD %s",
+           state->finh->unique, state->loc.path);
+
+    FUSE_FOP(state, fuse_newentry_cbk, GF_FOP_MKNOD, mknod, &state->loc,
+             state->mode, state->rdev, state->umask, state->xdata);
 }
 
 static void
-fuse_mknod (xlator_t *this, fuse_in_header_t *finh, void *msg)
+fuse_mknod(xlator_t *this, fuse_in_header_t *finh, void *msg,
+           struct iobuf *iobuf)
 {
-        struct fuse_mknod_in *fmi = msg;
-        char         *name = (char *)(fmi + 1);
+    struct fuse_mknod_in *fmi = msg;
+    char *name = (char *)(fmi + 1);
 
-        fuse_state_t   *state = NULL;
+    fuse_state_t *state = NULL;
 #if FUSE_KERNEL_MINOR_VERSION >= 12
-        fuse_private_t *priv = NULL;
-        int32_t         ret  = -1;
+    fuse_private_t *priv = NULL;
 
-        priv = this->private;
-        if (priv->proto_minor < 12)
-                name = (char *)msg + FUSE_COMPAT_MKNOD_IN_SIZE;
+    priv = this->private;
+    if (priv->proto_minor < 12)
+        name = (char *)msg + FUSE_COMPAT_MKNOD_IN_SIZE;
 #endif
 
-        GET_STATE (this, finh, state);
+    GET_STATE(this, finh, state);
 
-        gf_uuid_generate (state->gfid);
+    gf_uuid_generate(state->gfid);
 
-        fuse_resolve_entry_init (state, &state->resolve, finh->nodeid, name);
+    fuse_resolve_entry_init(state, &state->resolve, finh->nodeid, name);
 
-        state->mode = fmi->mode;
-        state->rdev = fmi->rdev;
+    state->mode = fmi->mode;
+    state->rdev = fmi->rdev;
 
-#if FUSE_KERNEL_MINOR_VERSION >=12
-        priv = this->private;
-        FUSE_ENTRY_CREATE(this, priv, finh, state, fmi, "MKNOD");
+#if FUSE_KERNEL_MINOR_VERSION >= 12
+    priv = this->private;
+    FUSE_ENTRY_CREATE(this, priv, finh, state, fmi, "MKNOD");
 #endif
 
-        fuse_resolve_and_resume (state, fuse_mknod_resume);
+    fuse_resolve_and_resume(state, fuse_mknod_resume);
 
-        return;
+    return;
 }
 
 void
-fuse_mkdir_resume (fuse_state_t *state)
-{
-        if (!state->loc.parent) {
-                gf_log ("glusterfs-fuse", GF_LOG_ERROR,
-                        "MKDIR %"PRIu64" (%s/%s) resolution failed",
-                        state->finh->nodeid, uuid_utoa (state->resolve.gfid),
-                        state->resolve.bname);
-                send_fuse_err (state->this, state->finh,
-                               state->resolve.op_errno);
-                free_fuse_state (state);
-                return;
-        }
+fuse_mkdir_resume(fuse_state_t *state)
+{
+    if (!state->loc.parent) {
+        gf_log("glusterfs-fuse", GF_LOG_ERROR,
+               "MKDIR %" PRIu64 " (%s/%s) resolution failed",
+               state->finh->nodeid, uuid_utoa(state->resolve.gfid),
+               state->resolve.bname);
 
-        if (state->resolve.op_errno == ENOENT) {
-                state->resolve.op_ret = 0;
-                state->resolve.op_errno = 0;
-        }
+        /* facilitate retry from VFS */
+        if (state->resolve.op_errno == ENOENT)
+            state->resolve.op_errno = ESTALE;
 
-        if (state->loc.inode) {
-                gf_log (state->this->name, GF_LOG_DEBUG, "inode already present");
-                inode_unref (state->loc.inode);
-                state->loc.inode = NULL;
-        }
+        send_fuse_err(state->this, state->finh, state->resolve.op_errno);
+        free_fuse_state(state);
+        return;
+    }
+
+    if (state->resolve.op_errno == ENOENT) {
+        state->resolve.op_ret = 0;
+        state->resolve.op_errno = 0;
+    }
 
-        state->loc.inode = inode_new (state->loc.parent->table);
+    if (state->loc.inode) {
+        gf_log(state->this->name, GF_LOG_DEBUG, "inode already present");
+        inode_unref(state->loc.inode);
+        state->loc.inode = NULL;
+    }
 
-        gf_log ("glusterfs-fuse", GF_LOG_TRACE,
-                "%"PRIu64": MKDIR %s", state->finh->unique,
-                state->loc.path);
+    state->loc.inode = inode_new(state->loc.parent->table);
 
-        FUSE_FOP (state, fuse_newentry_cbk, GF_FOP_MKDIR,
-                  mkdir, &state->loc, state->mode, state->umask, state->xdata);
+    gf_log("glusterfs-fuse", GF_LOG_TRACE, "%" PRIu64 ": MKDIR %s",
+           state->finh->unique, state->loc.path);
+
+    FUSE_FOP(state, fuse_newentry_cbk, GF_FOP_MKDIR, mkdir, &state->loc,
+             state->mode, state->umask, state->xdata);
 }
 
 static void
-fuse_mkdir (xlator_t *this, fuse_in_header_t *finh, void *msg)
+fuse_mkdir(xlator_t *this, fuse_in_header_t *finh, void *msg,
+           struct iobuf *iobuf)
 {
-        struct fuse_mkdir_in *fmi = msg;
-        char *name = (char *)(fmi + 1);
-#if FUSE_KERNEL_MINOR_VERSION >=12
-        fuse_private_t *priv = NULL;
-        int32_t         ret = -1;
+    struct fuse_mkdir_in *fmi = msg;
+    char *name = (char *)(fmi + 1);
+#if FUSE_KERNEL_MINOR_VERSION >= 12
+    fuse_private_t *priv = NULL;
 #endif
 
-        fuse_state_t *state;
+    fuse_state_t *state;
 
-        GET_STATE (this, finh, state);
+    GET_STATE(this, finh, state);
 
-        gf_uuid_generate (state->gfid);
+    gf_uuid_generate(state->gfid);
 
-        fuse_resolve_entry_init (state, &state->resolve, finh->nodeid, name);
+    fuse_resolve_entry_init(state, &state->resolve, finh->nodeid, name);
 
-        state->mode = fmi->mode;
+    state->mode = fmi->mode;
 
-#if FUSE_KERNEL_MINOR_VERSION >=12
-        priv = this->private;
-        FUSE_ENTRY_CREATE(this, priv, finh, state, fmi, "MKDIR");
+#if FUSE_KERNEL_MINOR_VERSION >= 12
+    priv = this->private;
+    FUSE_ENTRY_CREATE(this, priv, finh, state, fmi, "MKDIR");
 #endif
 
-        fuse_resolve_and_resume (state, fuse_mkdir_resume);
+    fuse_resolve_and_resume(state, fuse_mkdir_resume);
 
-        return;
+    return;
 }
 
 void
-fuse_unlink_resume (fuse_state_t *state)
-{
-        if (!state->loc.parent || !state->loc.inode) {
-                gf_log ("glusterfs-fuse", GF_LOG_ERROR,
-                        "UNLINK %"PRIu64" (%s/%s) resolution failed",
-                        state->finh->nodeid, uuid_utoa (state->resolve.gfid),
-                        state->resolve.bname);
-                send_fuse_err (state->this, state->finh,
-                               state->resolve.op_errno);
-                free_fuse_state (state);
-                return;
-        }
+fuse_unlink_resume(fuse_state_t *state)
+{
+    if (!state->loc.parent || !state->loc.inode) {
+        gf_log("glusterfs-fuse", GF_LOG_ERROR,
+               "UNLINK %" PRIu64 " (%s/%s) resolution failed",
+               state->finh->nodeid, uuid_utoa(state->resolve.gfid),
+               state->resolve.bname);
+        send_fuse_err(state->this, state->finh, state->resolve.op_errno);
+        free_fuse_state(state);
+        return;
+    }
 
-        gf_log ("glusterfs-fuse", GF_LOG_TRACE,
-                "%"PRIu64": UNLINK %s", state->finh->unique,
-                state->loc.path);
+    gf_log("glusterfs-fuse", GF_LOG_TRACE, "%" PRIu64 ": UNLINK %s",
+           state->finh->unique, state->loc.path);
 
-        FUSE_FOP (state, fuse_unlink_cbk, GF_FOP_UNLINK,
-                  unlink, &state->loc, 0, state->xdata);
+    FUSE_FOP(state, fuse_unlink_cbk, GF_FOP_UNLINK, unlink, &state->loc, 0,
+             state->xdata);
 }
 
 static void
-fuse_unlink (xlator_t *this, fuse_in_header_t *finh, void *msg)
+fuse_unlink(xlator_t *this, fuse_in_header_t *finh, void *msg,
+            struct iobuf *iobuf)
 {
-        char         *name = msg;
-        fuse_state_t *state = NULL;
+    char *name = msg;
+    fuse_state_t *state = NULL;
 
-        GET_STATE (this, finh, state);
+    GET_STATE(this, finh, state);
 
-        fuse_resolve_entry_init (state, &state->resolve, finh->nodeid, name);
+    fuse_resolve_entry_init(state, &state->resolve, finh->nodeid, name);
 
-        fuse_resolve_and_resume (state, fuse_unlink_resume);
+    fuse_resolve_and_resume(state, fuse_unlink_resume);
 
-        return;
+    return;
 }
 
 void
-fuse_rmdir_resume (fuse_state_t *state)
-{
-        if (!state->loc.parent || !state->loc.inode) {
-                gf_log ("glusterfs-fuse", GF_LOG_ERROR,
-                        "RMDIR %"PRIu64" (%s/%s) resolution failed",
-                        state->finh->nodeid, uuid_utoa (state->resolve.gfid),
-                        state->resolve.bname);
-                send_fuse_err (state->this, state->finh,
-                               state->resolve.op_errno);
-                free_fuse_state (state);
-                return;
-        }
+fuse_rmdir_resume(fuse_state_t *state)
+{
+    if (!state->loc.parent || !state->loc.inode) {
+        gf_log("glusterfs-fuse", GF_LOG_ERROR,
+               "RMDIR %" PRIu64 " (%s/%s) resolution failed",
+               state->finh->nodeid, uuid_utoa(state->resolve.gfid),
+               state->resolve.bname);
+        send_fuse_err(state->this, state->finh, state->resolve.op_errno);
+        free_fuse_state(state);
+        return;
+    }
 
-        gf_log ("glusterfs-fuse", GF_LOG_TRACE,
-                "%"PRIu64": RMDIR %s", state->finh->unique,
-                state->loc.path);
+    gf_log("glusterfs-fuse", GF_LOG_TRACE, "%" PRIu64 ": RMDIR %s",
+           state->finh->unique, state->loc.path);
 
-        FUSE_FOP (state, fuse_unlink_cbk, GF_FOP_RMDIR,
-                  rmdir, &state->loc, 0, state->xdata);
+    FUSE_FOP(state, fuse_unlink_cbk, GF_FOP_RMDIR, rmdir, &state->loc, 0,
+             state->xdata);
 }
 
 static void
-fuse_rmdir (xlator_t *this, fuse_in_header_t *finh, void *msg)
+fuse_rmdir(xlator_t *this, fuse_in_header_t *finh, void *msg,
+           struct iobuf *iobuf)
 {
-        char         *name = msg;
-        fuse_state_t *state = NULL;
+    char *name = msg;
+    fuse_state_t *state = NULL;
 
-        GET_STATE (this, finh, state);
+    GET_STATE(this, finh, state);
 
-        fuse_resolve_entry_init (state, &state->resolve, finh->nodeid, name);
+    fuse_resolve_entry_init(state, &state->resolve, finh->nodeid, name);
 
-        fuse_resolve_and_resume (state, fuse_rmdir_resume);
+    fuse_resolve_and_resume(state, fuse_rmdir_resume);
 
-        return;
+    return;
 }
 
 void
-fuse_symlink_resume (fuse_state_t *state)
-{
-        if (!state->loc.parent) {
-                gf_log ("glusterfs-fuse", GF_LOG_ERROR,
-                        "SYMLINK %"PRIu64" (%s/%s) -> %s resolution failed",
-                        state->finh->nodeid, uuid_utoa (state->resolve.gfid),
-                        state->resolve.bname, state->name);
-                send_fuse_err (state->this, state->finh,
-                               state->resolve.op_errno);
-                free_fuse_state (state);
-                return;
-        }
+fuse_symlink_resume(fuse_state_t *state)
+{
+    if (!state->loc.parent) {
+        gf_log("glusterfs-fuse", GF_LOG_ERROR,
+               "SYMLINK %" PRIu64 " (%s/%s) -> %s resolution failed",
+               state->finh->nodeid, uuid_utoa(state->resolve.gfid),
+               state->resolve.bname, state->name);
 
-        if (state->resolve.op_errno == ENOENT) {
-                state->resolve.op_ret = 0;
-                state->resolve.op_errno = 0;
-        }
+        /* facilitate retry from VFS */
+        if (state->resolve.op_errno == ENOENT)
+            state->resolve.op_errno = ESTALE;
 
-        if (state->loc.inode) {
-                gf_log (state->this->name, GF_LOG_DEBUG, "inode already present");
-                inode_unref (state->loc.inode);
-                state->loc.inode = NULL;
-        }
+        send_fuse_err(state->this, state->finh, state->resolve.op_errno);
+        free_fuse_state(state);
+        return;
+    }
 
-        state->loc.inode = inode_new (state->loc.parent->table);
+    if (state->resolve.op_errno == ENOENT) {
+        state->resolve.op_ret = 0;
+        state->resolve.op_errno = 0;
+    }
+
+    if (state->loc.inode) {
+        gf_log(state->this->name, GF_LOG_DEBUG, "inode already present");
+        inode_unref(state->loc.inode);
+        state->loc.inode = NULL;
+    }
+
+    state->loc.inode = inode_new(state->loc.parent->table);
 
-        gf_log ("glusterfs-fuse", GF_LOG_TRACE,
-                "%"PRIu64": SYMLINK %s -> %s", state->finh->unique,
-                state->loc.path, state->name);
+    gf_log("glusterfs-fuse", GF_LOG_TRACE, "%" PRIu64 ": SYMLINK %s -> %s",
+           state->finh->unique, state->loc.path, state->name);
 
-        FUSE_FOP (state, fuse_newentry_cbk, GF_FOP_SYMLINK,
-                  symlink, state->name, &state->loc, state->umask, state->xdata);
+    FUSE_FOP(state, fuse_newentry_cbk, GF_FOP_SYMLINK, symlink, state->name,
+             &state->loc, state->umask, state->xdata);
 }
 
 static void
-fuse_symlink (xlator_t *this, fuse_in_header_t *finh, void *msg)
+fuse_symlink(xlator_t *this, fuse_in_header_t *finh, void *msg,
+             struct iobuf *iobuf)
 {
-        char         *name = msg;
-        char         *linkname = name + strlen (name) + 1;
-        fuse_state_t *state = NULL;
+    char *name = msg;
+    char *linkname = name + strlen(name) + 1;
+    fuse_state_t *state = NULL;
 
-        GET_STATE (this, finh, state);
+    GET_STATE(this, finh, state);
 
-        gf_uuid_generate (state->gfid);
+    gf_uuid_generate(state->gfid);
 
-        fuse_resolve_entry_init (state, &state->resolve, finh->nodeid, name);
+    fuse_resolve_entry_init(state, &state->resolve, finh->nodeid, name);
 
-        state->name = gf_strdup (linkname);
+    state->name = gf_strdup(linkname);
 
-        fuse_resolve_and_resume (state, fuse_symlink_resume);
+    fuse_resolve_and_resume(state, fuse_symlink_resume);
 
-        return;
+    return;
 }
 
 int
-fuse_rename_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                 int32_t op_ret, int32_t op_errno, struct iatt *buf,
-                 struct iatt *preoldparent, struct iatt *postoldparent,
-                 struct iatt *prenewparent, struct iatt *postnewparent,
-                 dict_t *xdata)
+fuse_rename_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                int32_t op_ret, int32_t op_errno, struct iatt *buf,
+                struct iatt *preoldparent, struct iatt *postoldparent,
+                struct iatt *prenewparent, struct iatt *postnewparent,
+                dict_t *xdata)
 {
-        fuse_state_t     *state = NULL;
-        fuse_in_header_t *finh = NULL;
-
-        state = frame->root->state;
-        finh  = state->finh;
-
-        fuse_log_eh (this, "op_ret: %d, op_errno: %d, %"PRIu64": %s() "
-                     "path: %s parent: %s ==> path: %s parent: %s"
-                     "gfid: %s", op_ret, op_errno, frame->root->unique,
-                     gf_fop_list[frame->root->op], state->loc.path,
-                     state->loc.parent?uuid_utoa (state->loc.parent->gfid):"",
-                     state->loc2.path,
-                     state->loc2.parent?uuid_utoa (state->loc2.parent->gfid):"",
-                     state->loc.inode?uuid_utoa (state->loc.inode->gfid):"");
+    fuse_state_t *state = NULL;
+    fuse_in_header_t *finh = NULL;
+    char loc_uuid_str[64] = {0}, loc2_uuid_str[64] = {0};
+
+    state = frame->root->state;
+    finh = state->finh;
+
+    fuse_log_eh(
+        this,
+        "op_ret: %d, op_errno: %d, %" PRIu64
+        ": %s() "
+        "path: %s parent: %s ==> path: %s parent: %s"
+        "gfid: %s",
+        op_ret, op_errno, frame->root->unique, gf_fop_list[frame->root->op],
+        state->loc.path,
+        (state->loc.parent ? uuid_utoa_r(state->loc.parent->gfid, loc_uuid_str)
+                           : ""),
+        state->loc2.path,
+        (state->loc2.parent
+             ? uuid_utoa_r(state->loc2.parent->gfid, loc2_uuid_str)
+             : ""),
+        state->loc.inode ? uuid_utoa(state->loc.inode->gfid) : "");
+
+    /* need to check for loc->parent to keep clang-scan happy.
+       It gets dereferenced below, and is checked for NULL above. */
+    if ((op_ret == 0) && (state->loc.parent) && (state->loc.inode)) {
+        gf_log("glusterfs-fuse", GF_LOG_TRACE,
+               "%" PRIu64 ": %s -> %s => 0 (buf->ia_ino=%" PRIu64 ")",
+               frame->root->unique, state->loc.path, state->loc2.path,
+               buf->ia_ino);
 
-        if (op_ret == 0) {
-                gf_log ("glusterfs-fuse", GF_LOG_TRACE,
-                        "%"PRIu64": %s -> %s => 0 (buf->ia_ino=%"PRIu64")",
-                        frame->root->unique, state->loc.path, state->loc2.path,
-                        buf->ia_ino);
-
-                {
-                        /* ugly ugly - to stay blind to situation where
-                           rename happens on a new inode
-                        */
-                        buf->ia_type = state->loc.inode->ia_type;
-                }
-                buf->ia_blksize = this->ctx->page_size;
+        {
+            /* ugly ugly - to stay blind to situation where
+               rename happens on a new inode
+            */
+            buf->ia_type = state->loc.inode->ia_type;
+        }
+        buf->ia_blksize = this->ctx->page_size;
 
-                inode_rename (state->loc.parent->table,
-                              state->loc.parent, state->loc.name,
-                              state->loc2.parent, state->loc2.name,
-                              state->loc.inode, buf);
+        inode_rename(state->loc.parent->table, state->loc.parent,
+                     state->loc.name, state->loc2.parent, state->loc2.name,
+                     state->loc.inode, buf);
 
-                send_fuse_err (this, finh, 0);
-        } else {
-                gf_log ("glusterfs-fuse", GF_LOG_WARNING,
-                        "%"PRIu64": %s -> %s => -1 (%s)", frame->root->unique,
-                        state->loc.path, state->loc2.path,
-                        strerror (op_errno));
-                send_fuse_err (this, finh, op_errno);
-        }
+        send_fuse_err(this, finh, 0);
+    } else {
+        gf_log("glusterfs-fuse", GF_LOG_WARNING,
+               "%" PRIu64 ": %s -> %s => -1 (%s)", frame->root->unique,
+               state->loc.path, state->loc2.path, strerror(op_errno));
+        send_fuse_err(this, finh, op_errno);
+    }
 
-        free_fuse_state (state);
-        STACK_DESTROY (frame->root);
-        return 0;
+    free_fuse_state(state);
+    STACK_DESTROY(frame->root);
+    return 0;
 }
 
 void
-fuse_rename_resume (fuse_state_t *state)
-{
-        char loc_uuid[64]  = {0,};
-        char loc2_uuid[64] = {0,};
-
-        if (!state->loc.parent || !state->loc.inode) {
-                gf_log ("glusterfs-fuse", GF_LOG_ERROR,
-                        "RENAME %"PRIu64" %s/%s -> %s/%s src resolution failed",
-                        state->finh->unique,
-                        uuid_utoa_r (state->resolve.gfid, loc_uuid),
-                        state->resolve.bname,
-                        uuid_utoa_r (state->resolve2.gfid, loc2_uuid),
-                        state->resolve2.bname);
-
-                send_fuse_err (state->this, state->finh,
-                               state->resolve.op_errno);
-                free_fuse_state (state);
-                return;
-        }
-
-        if (!state->loc2.parent) {
-                gf_log ("glusterfs-fuse", GF_LOG_ERROR,
-                        "RENAME %"PRIu64" %s/%s -> %s/%s dst resolution failed",
-                        state->finh->unique,
-                        uuid_utoa_r (state->resolve.gfid, loc_uuid),
-                        state->resolve.bname,
-                        uuid_utoa_r (state->resolve2.gfid, loc2_uuid),
-                        state->resolve2.bname);
-
-                send_fuse_err (state->this, state->finh, ENOENT);
-                free_fuse_state (state);
-                return;
-        }
+fuse_rename_resume(fuse_state_t *state)
+{
+    char loc_uuid[64] = {
+        0,
+    };
+    char loc2_uuid[64] = {
+        0,
+    };
+
+    if (!state->loc.parent || !state->loc.inode) {
+        gf_log("glusterfs-fuse", GF_LOG_ERROR,
+               "RENAME %" PRIu64 " %s/%s -> %s/%s src resolution failed",
+               state->finh->unique, uuid_utoa_r(state->resolve.gfid, loc_uuid),
+               state->resolve.bname,
+               uuid_utoa_r(state->resolve2.gfid, loc2_uuid),
+               state->resolve2.bname);
+
+        /* facilitate retry from VFS */
+        if ((!state->loc.inode) && (state->resolve.op_errno == ENOENT))
+            state->resolve.op_errno = ESTALE;
+
+        send_fuse_err(state->this, state->finh, state->resolve.op_errno);
+        free_fuse_state(state);
+        return;
+    }
+
+    if (!state->loc2.parent) {
+        gf_log("glusterfs-fuse", GF_LOG_ERROR,
+               "RENAME %" PRIu64 " %s/%s -> %s/%s dst resolution failed",
+               state->finh->unique, uuid_utoa_r(state->resolve.gfid, loc_uuid),
+               state->resolve.bname,
+               uuid_utoa_r(state->resolve2.gfid, loc2_uuid),
+               state->resolve2.bname);
+
+        send_fuse_err(state->this, state->finh, ESTALE);
+        free_fuse_state(state);
+        return;
+    }
 
-        state->resolve.op_ret = 0;
-        state->resolve2.op_ret = 0;
+    state->resolve.op_ret = 0;
+    state->resolve2.op_ret = 0;
 
-        gf_log ("glusterfs-fuse", GF_LOG_TRACE,
-                "%"PRIu64": RENAME `%s (%s)' -> `%s (%s)'",
-                state->finh->unique, state->loc.path, loc_uuid,
-                state->loc2.path, loc2_uuid);
+    gf_log("glusterfs-fuse", GF_LOG_TRACE,
+           "%" PRIu64 ": RENAME `%s (%s)' -> `%s (%s)'", state->finh->unique,
+           state->loc.path, loc_uuid, state->loc2.path, loc2_uuid);
 
-        FUSE_FOP (state, fuse_rename_cbk, GF_FOP_RENAME,
-                  rename, &state->loc, &state->loc2, state->xdata);
+    FUSE_FOP(state, fuse_rename_cbk, GF_FOP_RENAME, rename, &state->loc,
+             &state->loc2, state->xdata);
 }
 
 static void
-fuse_rename (xlator_t *this, fuse_in_header_t *finh, void *msg)
+fuse_rename(xlator_t *this, fuse_in_header_t *finh, void *msg,
+            struct iobuf *iobuf)
 {
-        struct fuse_rename_in  *fri = msg;
-        char *oldname = (char *)(fri + 1);
-        char *newname = oldname + strlen (oldname) + 1;
-        fuse_state_t *state = NULL;
+    struct fuse_rename_in *fri = msg;
+    char *oldname = (char *)(fri + 1);
+    char *newname = oldname + strlen(oldname) + 1;
+    fuse_state_t *state = NULL;
 
-        GET_STATE (this, finh, state);
+    GET_STATE(this, finh, state);
 
-        fuse_resolve_entry_init (state, &state->resolve, finh->nodeid, oldname);
+    fuse_resolve_entry_init(state, &state->resolve, finh->nodeid, oldname);
 
-        fuse_resolve_entry_init (state, &state->resolve2, fri->newdir, newname);
+    fuse_resolve_entry_init(state, &state->resolve2, fri->newdir, newname);
 
-        fuse_resolve_and_resume (state, fuse_rename_resume);
+    fuse_resolve_and_resume(state, fuse_rename_resume);
 
-        return;
+    return;
 }
 
 void
-fuse_link_resume (fuse_state_t *state)
-{
-        if (!state->loc2.inode || !state->loc.parent) {
-                gf_log ("glusterfs-fuse", GF_LOG_WARNING,
-                        "fuse_loc_fill() failed %"PRIu64": LINK %s %s",
-                        state->finh->unique, state->loc2.path, state->loc.path);
-                send_fuse_err (state->this, state->finh,
-                               state->resolve.op_errno);
-                free_fuse_state (state);
-                return;
-        }
+fuse_link_resume(fuse_state_t *state)
+{
+    if (!state->loc2.inode || !state->loc.parent) {
+        gf_log("glusterfs-fuse", GF_LOG_WARNING,
+               "fuse_loc_fill() failed %" PRIu64 ": LINK %s %s",
+               state->finh->unique, state->loc2.path, state->loc.path);
 
-        state->resolve.op_ret = 0;
-        state->resolve2.op_ret = 0;
+        /* facilitate retry from VFS */
+        if (!state->loc2.inode && (state->resolve.op_errno == ENOENT))
+            state->resolve.op_errno = ESTALE;
 
-        if (state->loc.inode) {
-                inode_unref (state->loc.inode);
-                state->loc.inode = NULL;
-        }
-        state->loc.inode = inode_ref (state->loc2.inode);
+        send_fuse_err(state->this, state->finh, state->resolve.op_errno);
+        free_fuse_state(state);
+        return;
+    }
+
+    state->resolve.op_ret = 0;
+    state->resolve2.op_ret = 0;
+
+    if (state->loc.inode) {
+        inode_unref(state->loc.inode);
+        state->loc.inode = NULL;
+    }
+    state->loc.inode = inode_ref(state->loc2.inode);
 
-        gf_log ("glusterfs-fuse", GF_LOG_TRACE,
-                "%"PRIu64": LINK() %s -> %s",
-                state->finh->unique, state->loc2.path,
-                state->loc.path);
+    gf_log("glusterfs-fuse", GF_LOG_TRACE, "%" PRIu64 ": LINK() %s -> %s",
+           state->finh->unique, state->loc2.path, state->loc.path);
 
-        FUSE_FOP (state, fuse_newentry_cbk, GF_FOP_LINK,
-                  link, &state->loc2, &state->loc, state->xdata);
+    FUSE_FOP(state, fuse_newentry_cbk, GF_FOP_LINK, link, &state->loc2,
+             &state->loc, state->xdata);
 }
 
 static void
-fuse_link (xlator_t *this, fuse_in_header_t *finh, void *msg)
+fuse_link(xlator_t *this, fuse_in_header_t *finh, void *msg,
+          struct iobuf *iobuf)
 {
-        struct fuse_link_in *fli = msg;
-        char         *name = (char *)(fli + 1);
-        fuse_state_t *state = NULL;
+    struct fuse_link_in *fli = msg;
+    char *name = (char *)(fli + 1);
+    fuse_state_t *state = NULL;
 
-        GET_STATE (this, finh, state);
+    GET_STATE(this, finh, state);
 
-        fuse_resolve_inode_init (state, &state->resolve2, fli->oldnodeid);
+    fuse_resolve_inode_init(state, &state->resolve2, fli->oldnodeid);
 
-        fuse_resolve_entry_init (state, &state->resolve, finh->nodeid, name);
+    fuse_resolve_entry_init(state, &state->resolve, finh->nodeid, name);
 
-        fuse_resolve_and_resume (state, fuse_link_resume);
+    fuse_resolve_and_resume(state, fuse_link_resume);
 
-        return;
+    return;
 }
 
 static int
-fuse_create_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                 int32_t op_ret, int32_t op_errno,
-                 fd_t *fd, inode_t *inode, struct iatt *buf,
-                 struct iatt *preparent, struct iatt *postparent, dict_t *xdata)
-{
-        fuse_state_t           *state        = NULL;
-        fuse_in_header_t       *finh         = NULL;
-        fuse_private_t         *priv         = NULL;
-        struct fuse_out_header  fouh         = {0, };
-        struct fuse_entry_out   feo          = {0, };
-        struct fuse_open_out    foo          = {0, };
-        struct iovec            iov_out[3];
-        inode_t                *linked_inode = NULL;
-
-        state    = frame->root->state;
-        priv     = this->private;
-        finh     = state->finh;
-        foo.open_flags = 0;
-
-        fuse_log_eh_fop(this, state, frame, op_ret, op_errno);
-
-        if (op_ret >= 0) {
-                foo.fh = (uintptr_t) fd;
-
-                if (((priv->direct_io_mode == 2)
-                     && ((state->flags & O_ACCMODE) != O_RDONLY))
-                    || (priv->direct_io_mode == 1)
-		    || direct_io_mode (xdata))
-                        foo.open_flags |= FOPEN_DIRECT_IO;
-
-                gf_log ("glusterfs-fuse", GF_LOG_TRACE,
-                        "%"PRIu64": %s() %s => %p (ino=%"PRIu64")",
-                        frame->root->unique, gf_fop_list[frame->root->op],
-                        state->loc.path, fd, buf->ia_ino);
-
-                buf->ia_blksize = this->ctx->page_size;
-                gf_fuse_stat2attr (buf, &feo.attr, priv->enable_ino32);
-
-                linked_inode = inode_link (inode, state->loc.parent,
-                                           state->loc.name, buf);
-
-                if (linked_inode != inode) {
-                        /*
-                           VERY racy code (if used anywhere else)
-                           -- don't do this without understanding
-                        */
-                        inode_unref (fd->inode);
-                        fd->inode = inode_ref (linked_inode);
-                }
+fuse_create_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                int32_t op_ret, int32_t op_errno, fd_t *fd, inode_t *inode,
+                struct iatt *buf, struct iatt *preparent,
+                struct iatt *postparent, dict_t *xdata)
+{
+    fuse_state_t *state = NULL;
+    fuse_in_header_t *finh = NULL;
+    fuse_private_t *priv = NULL;
+    struct fuse_out_header fouh = {
+        0,
+    };
+    struct fuse_entry_out feo = {
+        0,
+    };
+    struct fuse_open_out foo = {
+        0,
+    };
+    struct iovec iov_out[3];
+    inode_t *linked_inode = NULL;
+    uint64_t ctx_value = LOOKUP_NOT_NEEDED;
+
+    state = frame->root->state;
+    priv = this->private;
+    finh = state->finh;
+    foo.open_flags = 0;
+
+    fuse_log_eh_fop(this, state, frame, op_ret, op_errno);
+
+    if (op_ret >= 0) {
+        foo.fh = (uintptr_t)fd;
+
+        if (((priv->direct_io_mode == 2) &&
+             ((state->flags & O_ACCMODE) != O_RDONLY)) ||
+            (priv->direct_io_mode == 1) || direct_io_mode(xdata))
+            foo.open_flags |= FOPEN_DIRECT_IO;
+
+        gf_log("glusterfs-fuse", GF_LOG_TRACE,
+               "%" PRIu64 ": %s() %s => %p (ino=%" PRIu64 ")",
+               frame->root->unique, gf_fop_list[frame->root->op],
+               state->loc.path, fd, buf->ia_ino);
+
+        buf->ia_blksize = this->ctx->page_size;
+        gf_fuse_stat2attr(buf, &feo.attr, priv->enable_ino32);
+
+        linked_inode = inode_link(inode, state->loc.parent, state->loc.name,
+                                  buf);
+
+        if (linked_inode != inode) {
+            /*
+               VERY racy code (if used anywhere else)
+               -- don't do this without understanding
+            */
+            inode_unref(fd->inode);
+            fd->inode = inode_ref(linked_inode);
+        } else {
+            inode_ctx_set(linked_inode, this, &ctx_value);
+        }
 
-                inode_lookup (linked_inode);
+        inode_lookup(linked_inode);
 
-                inode_unref (linked_inode);
+        inode_unref(linked_inode);
 
-                feo.nodeid = inode_to_fuse_nodeid (linked_inode);
+        feo.nodeid = inode_to_fuse_nodeid(linked_inode);
 
-                feo.entry_valid = calc_timeout_sec (priv->entry_timeout);
-                feo.entry_valid_nsec = calc_timeout_nsec (priv->entry_timeout);
-                feo.attr_valid = calc_timeout_sec (priv->attribute_timeout);
-                feo.attr_valid_nsec =
-                  calc_timeout_nsec (priv->attribute_timeout);
+        feo.entry_valid = calc_timeout_sec(priv->entry_timeout);
+        feo.entry_valid_nsec = calc_timeout_nsec(priv->entry_timeout);
+        feo.attr_valid = calc_timeout_sec(priv->attribute_timeout);
+        feo.attr_valid_nsec = calc_timeout_nsec(priv->attribute_timeout);
 
-                fouh.error = 0;
-                iov_out[0].iov_base = &fouh;
-                iov_out[1].iov_base = &feo;
+        fouh.error = 0;
+        iov_out[0].iov_base = &fouh;
+        iov_out[1].iov_base = &feo;
 #if FUSE_KERNEL_MINOR_VERSION >= 9
-                iov_out[1].iov_len = priv->proto_minor >= 9 ?
-                                     sizeof (feo) :
-                                     FUSE_COMPAT_ENTRY_OUT_SIZE;
+        iov_out[1].iov_len = priv->proto_minor >= 9
+                                 ? sizeof(feo)
+                                 : FUSE_COMPAT_ENTRY_OUT_SIZE;
 #else
-                iov_out[1].iov_len = sizeof (feo);
+        iov_out[1].iov_len = sizeof(feo);
 #endif
-                iov_out[2].iov_base = &foo;
-                iov_out[2].iov_len = sizeof (foo);
-
-                if (send_fuse_iov (this, finh, iov_out, 3) == ENOENT) {
-                        gf_log ("glusterfs-fuse", GF_LOG_DEBUG,
-                                "create(%s) got EINTR", state->loc.path);
-                        inode_forget (inode, 1);
-                        gf_fd_put (priv->fdtable, state->fd_no);
-                        goto out;
-                }
+        iov_out[2].iov_base = &foo;
+        iov_out[2].iov_len = sizeof(foo);
 
-                fd_bind (fd);
-        } else {
-                gf_log ("glusterfs-fuse", GF_LOG_WARNING,
-                        "%"PRIu64": %s => -1 (%s)", finh->unique,
-                        state->loc.path, strerror (op_errno));
-                send_fuse_err (this, finh, op_errno);
-                gf_fd_put (priv->fdtable, state->fd_no);
+        if (send_fuse_iov(this, finh, iov_out, 3) == ENOENT) {
+            gf_log("glusterfs-fuse", GF_LOG_DEBUG, "create(%s) got EINTR",
+                   state->loc.path);
+            inode_forget(inode, 1);
+            gf_fd_put(priv->fdtable, state->fd_no);
+            goto out;
         }
+
+        fd_bind(fd);
+    } else {
+        /* facilitate retry from VFS */
+        if (op_errno == ENOENT)
+            op_errno = ESTALE;
+
+        gf_log("glusterfs-fuse", GF_LOG_WARNING, "%" PRIu64 ": %s => -1 (%s)",
+               finh->unique, state->loc.path, strerror(op_errno));
+
+        send_fuse_err(this, finh, op_errno);
+        gf_fd_put(priv->fdtable, state->fd_no);
+    }
 out:
-        free_fuse_state (state);
-        STACK_DESTROY (frame->root);
+    free_fuse_state(state);
+    STACK_DESTROY(frame->root);
 
-        return 0;
+    return 0;
 }
 
 void
-fuse_create_resume (fuse_state_t *state)
-{
-        fd_t           *fd    = NULL;
-        fuse_private_t *priv  = NULL;
-        fuse_fd_ctx_t  *fdctx = NULL;
-
-        if (!state->loc.parent) {
-                gf_log ("glusterfs-fuse", GF_LOG_WARNING,
-                        "%"PRIu64" CREATE %s/%s resolution failed",
-                        state->finh->unique, uuid_utoa (state->resolve.gfid),
-                        state->resolve.bname);
-                send_fuse_err (state->this, state->finh,
-                               state->resolve.op_errno);
-                free_fuse_state (state);
-                return;
-        }
-
-        if (state->resolve.op_errno == ENOENT) {
-                state->resolve.op_ret = 0;
-                state->resolve.op_errno = 0;
-        }
-
-        if (state->loc.inode) {
-                gf_log (state->this->name, GF_LOG_DEBUG,
-                        "inode already present");
-                inode_unref (state->loc.inode);
-        }
+fuse_create_resume(fuse_state_t *state)
+{
+    fd_t *fd = NULL;
+    fuse_private_t *priv = NULL;
+    fuse_fd_ctx_t *fdctx = NULL;
 
-        state->loc.inode = inode_new (state->loc.parent->table);
+    if (!state->loc.parent) {
+        gf_log("glusterfs-fuse", GF_LOG_WARNING,
+               "%" PRIu64 " CREATE %s/%s resolution failed",
+               state->finh->unique, uuid_utoa(state->resolve.gfid),
+               state->resolve.bname);
 
-        fd = fd_create (state->loc.inode, state->finh->pid);
-        if (fd == NULL) {
-                gf_log ("glusterfs-fuse", GF_LOG_WARNING,
-                        "%"PRIu64" CREATE cannot create a new fd",
-                        state->finh->unique);
-                send_fuse_err (state->this, state->finh, ENOMEM);
-                free_fuse_state (state);
-                return;
-        }
+        /* facilitate retry from VFS */
+        if (state->resolve.op_errno == ENOENT)
+            state->resolve.op_errno = ESTALE;
 
-        fdctx = fuse_fd_ctx_check_n_create (state->this, fd);
-        if (fdctx == NULL) {
-                gf_log ("glusterfs-fuse", GF_LOG_WARNING,
-                        "%"PRIu64" CREATE creation of fdctx failed",
-                        state->finh->unique);
-                fd_unref (fd);
-                send_fuse_err (state->this, state->finh, ENOMEM);
-                free_fuse_state (state);
-                return;
-        }
+        send_fuse_err(state->this, state->finh, state->resolve.op_errno);
+        free_fuse_state(state);
+        return;
+    }
 
-        priv = state->this->private;
+    if (state->resolve.op_errno == ENOENT) {
+        state->resolve.op_ret = 0;
+        state->resolve.op_errno = 0;
+    }
+
+    if (state->loc.inode) {
+        gf_log(state->this->name, GF_LOG_DEBUG, "inode already present");
+        inode_unref(state->loc.inode);
+    }
+
+    state->loc.inode = inode_new(state->loc.parent->table);
+
+    fd = fd_create(state->loc.inode, state->finh->pid);
+    if (fd == NULL) {
+        gf_log("glusterfs-fuse", GF_LOG_WARNING,
+               "%" PRIu64 " CREATE cannot create a new fd",
+               state->finh->unique);
+        send_fuse_err(state->this, state->finh, ENOMEM);
+        free_fuse_state(state);
+        return;
+    }
+
+    fdctx = fuse_fd_ctx_check_n_create(state->this, fd);
+    if (fdctx == NULL) {
+        gf_log("glusterfs-fuse", GF_LOG_WARNING,
+               "%" PRIu64 " CREATE creation of fdctx failed",
+               state->finh->unique);
+        fd_unref(fd);
+        send_fuse_err(state->this, state->finh, ENOMEM);
+        free_fuse_state(state);
+        return;
+    }
 
-        state->fd_no = gf_fd_unused_get (priv->fdtable, fd);
+    priv = state->this->private;
 
-        state->fd = fd_ref (fd);
-        fd->flags = state->flags;
+    state->fd_no = gf_fd_unused_get(priv->fdtable, fd);
 
-        gf_log ("glusterfs-fuse", GF_LOG_TRACE,
-                "%"PRIu64": CREATE %s", state->finh->unique,
-                state->loc.path);
+    state->fd = fd_ref(fd);
+    fd->flags = state->flags;
 
-        FUSE_FOP (state, fuse_create_cbk, GF_FOP_CREATE,
-                  create, &state->loc, state->flags, state->mode,
-                  state->umask, fd, state->xdata);
+    gf_log("glusterfs-fuse", GF_LOG_TRACE, "%" PRIu64 ": CREATE %s",
+           state->finh->unique, state->loc.path);
 
+    FUSE_FOP(state, fuse_create_cbk, GF_FOP_CREATE, create, &state->loc,
+             state->flags, state->mode, state->umask, fd, state->xdata);
 }
 
 static void
-fuse_create (xlator_t *this, fuse_in_header_t *finh, void *msg)
+fuse_create(xlator_t *this, fuse_in_header_t *finh, void *msg,
+            struct iobuf *iobuf)
 {
 #if FUSE_KERNEL_MINOR_VERSION >= 12
-        struct fuse_create_in *fci = msg;
-        fuse_private_t        *priv = NULL;
-	int32_t                ret = -1;
+    struct fuse_create_in *fci = msg;
+    fuse_private_t *priv = NULL;
 #else
-        struct fuse_open_in *fci = msg;
+    struct fuse_open_in *fci = msg;
 #endif
-        char         *name = (char *)(fci + 1);
+    char *name = (char *)(fci + 1);
 
-        fuse_state_t *state = NULL;
+    fuse_state_t *state = NULL;
 
 #if FUSE_KERNEL_MINOR_VERSION >= 12
-        priv = this->private;
-        if (priv->proto_minor < 12)
-                name = (char *)((struct fuse_open_in *)msg + 1);
+    priv = this->private;
+    if (priv->proto_minor < 12)
+        name = (char *)((struct fuse_open_in *)msg + 1);
 #endif
 
-        GET_STATE (this, finh, state);
+    GET_STATE(this, finh, state);
 
-        gf_uuid_generate (state->gfid);
+    gf_uuid_generate(state->gfid);
 
-        fuse_resolve_entry_init (state, &state->resolve, finh->nodeid, name);
+    fuse_resolve_entry_init(state, &state->resolve, finh->nodeid, name);
 
-        state->mode = fci->mode;
-        state->flags = fci->flags;
+    state->mode = fci->mode;
+    state->flags = fci->flags;
 
-#if FUSE_KERNEL_MINOR_VERSION >=12
-        priv = this->private;
-        FUSE_ENTRY_CREATE(this, priv, finh, state, fci, "CREATE");
+#if FUSE_KERNEL_MINOR_VERSION >= 12
+    priv = this->private;
+    FUSE_ENTRY_CREATE(this, priv, finh, state, fci, "CREATE");
 #endif
-        fuse_resolve_and_resume (state, fuse_create_resume);
+    fuse_resolve_and_resume(state, fuse_create_resume);
 
-        return;
+    return;
 }
 
 void
-fuse_open_resume (fuse_state_t *state)
+fuse_open_resume(fuse_state_t *state)
 {
-        fd_t           *fd    = NULL;
-        fuse_private_t *priv  = NULL;
-        fuse_fd_ctx_t  *fdctx = NULL;
+    fd_t *fd = NULL;
+    fuse_private_t *priv = NULL;
+    fuse_fd_ctx_t *fdctx = NULL;
 
-        if (!state->loc.inode) {
-                gf_log ("glusterfs-fuse", GF_LOG_ERROR,
-                        "%"PRIu64": OPEN %s resolution failed",
-                        state->finh->unique, uuid_utoa (state->resolve.gfid));
+    if (!state->loc.inode) {
+        gf_log("glusterfs-fuse", GF_LOG_ERROR,
+               "%" PRIu64 ": OPEN %s resolution failed", state->finh->unique,
+               uuid_utoa(state->resolve.gfid));
 
-                send_fuse_err (state->this, state->finh,
-                               state->resolve.op_errno);
-                free_fuse_state (state);
-                return;
-        }
+        /* facilitate retry from VFS */
+        if (state->resolve.op_errno == ENOENT)
+            state->resolve.op_errno = ESTALE;
 
-        fd = fd_create (state->loc.inode, state->finh->pid);
-        if (!fd) {
-                gf_log ("fuse", GF_LOG_ERROR,
-                        "fd is NULL");
-                send_fuse_err (state->this, state->finh, ENOENT);
-                free_fuse_state (state);
-                return;
-        }
+        send_fuse_err(state->this, state->finh, state->resolve.op_errno);
+        free_fuse_state(state);
+        return;
+    }
 
-        fdctx = fuse_fd_ctx_check_n_create (state->this, fd);
-        if (fdctx == NULL) {
-                gf_log ("glusterfs-fuse", GF_LOG_WARNING,
-                        "%"PRIu64": OPEN creation of fdctx failed",
-                        state->finh->unique);
-                fd_unref (fd);
-                send_fuse_err (state->this, state->finh, ENOMEM);
-                free_fuse_state (state);
-                return;
-        }
+    fd = fd_create(state->loc.inode, state->finh->pid);
+    if (!fd) {
+        gf_log("fuse", GF_LOG_ERROR, "fd is NULL");
+        send_fuse_err(state->this, state->finh, ENOENT);
+        free_fuse_state(state);
+        return;
+    }
+
+    fdctx = fuse_fd_ctx_check_n_create(state->this, fd);
+    if (fdctx == NULL) {
+        gf_log("glusterfs-fuse", GF_LOG_WARNING,
+               "%" PRIu64 ": OPEN creation of fdctx failed",
+               state->finh->unique);
+        fd_unref(fd);
+        send_fuse_err(state->this, state->finh, ENOMEM);
+        free_fuse_state(state);
+        return;
+    }
 
-        priv = state->this->private;
+    priv = state->this->private;
 
-        state->fd_no = gf_fd_unused_get (priv->fdtable, fd);
-        state->fd = fd_ref (fd);
-        fd->flags = state->flags;
+    state->fd_no = gf_fd_unused_get(priv->fdtable, fd);
+    state->fd = fd_ref(fd);
+    fd->flags = state->flags;
 
-        gf_log ("glusterfs-fuse", GF_LOG_TRACE,
-                "%"PRIu64": OPEN %s", state->finh->unique,
-                state->loc.path);
+    gf_log("glusterfs-fuse", GF_LOG_TRACE, "%" PRIu64 ": OPEN %s",
+           state->finh->unique, state->loc.path);
 
-        FUSE_FOP (state, fuse_fd_cbk, GF_FOP_OPEN,
-                  open, &state->loc, state->flags, fd, state->xdata);
+    FUSE_FOP(state, fuse_fd_cbk, GF_FOP_OPEN, open, &state->loc, state->flags,
+             fd, state->xdata);
 }
 
 static void
-fuse_open (xlator_t *this, fuse_in_header_t *finh, void *msg)
+fuse_open(xlator_t *this, fuse_in_header_t *finh, void *msg,
+          struct iobuf *iobuf)
 {
-        struct fuse_open_in *foi = msg;
-        fuse_state_t *state = NULL;
+    struct fuse_open_in *foi = msg;
+    fuse_state_t *state = NULL;
 
-        GET_STATE (this, finh, state);
+    GET_STATE(this, finh, state);
 
-        fuse_resolve_inode_init (state, &state->resolve, finh->nodeid);
+    fuse_resolve_inode_init(state, &state->resolve, finh->nodeid);
 
-        state->flags = foi->flags;
+    state->flags = foi->flags;
 
-        fuse_resolve_and_resume (state, fuse_open_resume);
+    fuse_resolve_and_resume(state, fuse_open_resume);
 
-        return;
+    return;
 }
 
 static int
-fuse_readv_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                int32_t op_ret, int32_t op_errno,
-                struct iovec *vector, int32_t count,
-                struct iatt *stbuf, struct iobref *iobref, dict_t *xdata)
-{
-        fuse_state_t *state = NULL;
-        fuse_in_header_t *finh = NULL;
-        struct fuse_out_header fouh = {0, };
-        struct iovec *iov_out = NULL;
-
-        state = frame->root->state;
-        finh = state->finh;
-
-        fuse_log_eh_fop(this, state, frame, op_ret, op_errno);
-
-        if (op_ret >= 0) {
-                gf_log ("glusterfs-fuse", GF_LOG_TRACE,
-                        "%"PRIu64": READ => %d/%"GF_PRI_SIZET",%"PRId64"/%"PRIu64,
-                        frame->root->unique,
-                        op_ret, state->size, state->off, stbuf->ia_size);
-
-                iov_out = GF_CALLOC (count + 1, sizeof (*iov_out),
-                                     gf_fuse_mt_iovec);
-                if (iov_out) {
-                        fouh.error = 0;
-                        iov_out[0].iov_base = &fouh;
-                        memcpy (iov_out + 1, vector, count * sizeof (*iov_out));
-                        send_fuse_iov (this, finh, iov_out, count + 1);
-                        GF_FREE (iov_out);
-                } else
-                        send_fuse_err (this, finh, ENOMEM);
-        } else {
-                gf_log ("glusterfs-fuse", GF_LOG_WARNING,
-                        "%"PRIu64": READ => %d (%s)", frame->root->unique,
-                        op_ret, strerror (op_errno));
-
-                send_fuse_err (this, finh, op_errno);
-        }
-
-        free_fuse_state (state);
-        STACK_DESTROY (frame->root);
-
-        return 0;
+fuse_readv_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+               int32_t op_ret, int32_t op_errno, struct iovec *vector,
+               int32_t count, struct iatt *stbuf, struct iobref *iobref,
+               dict_t *xdata)
+{
+    fuse_state_t *state = NULL;
+    fuse_in_header_t *finh = NULL;
+    struct fuse_out_header fouh = {
+        0,
+    };
+    struct iovec *iov_out = NULL;
+
+    state = frame->root->state;
+    finh = state->finh;
+
+    fuse_log_eh_fop(this, state, frame, op_ret, op_errno);
+
+    if (op_ret >= 0) {
+        gf_log("glusterfs-fuse", GF_LOG_TRACE,
+               "%" PRIu64 ": READ => %d/%" GF_PRI_SIZET ",%" PRId64 "/%" PRIu64,
+               frame->root->unique, op_ret, state->size, state->off,
+               stbuf->ia_size);
+
+        iov_out = GF_CALLOC(count + 1, sizeof(*iov_out), gf_fuse_mt_iovec);
+        if (iov_out) {
+            fouh.error = 0;
+            iov_out[0].iov_base = &fouh;
+            memcpy(iov_out + 1, vector, count * sizeof(*iov_out));
+            send_fuse_iov(this, finh, iov_out, count + 1);
+            GF_FREE(iov_out);
+        } else
+            send_fuse_err(this, finh, ENOMEM);
+    } else {
+        gf_log("glusterfs-fuse", GF_LOG_WARNING,
+               "%" PRIu64 ": READ => %d gfid=%s fd=%p (%s)",
+               frame->root->unique, op_ret,
+               (state->fd && state->fd->inode)
+                   ? uuid_utoa(state->fd->inode->gfid)
+                   : "nil",
+               state->fd, strerror(op_errno));
+
+        send_fuse_err(this, finh, op_errno);
+    }
+
+    free_fuse_state(state);
+    STACK_DESTROY(frame->root);
+
+    return 0;
 }
 
 void
-fuse_readv_resume (fuse_state_t *state)
+fuse_readv_resume(fuse_state_t *state)
 {
-        gf_log ("glusterfs-fuse", GF_LOG_TRACE,
-                "%"PRIu64": READ (%p, size=%zu, offset=%"PRIu64")",
-                state->finh->unique, state->fd, state->size, state->off);
+    gf_log("glusterfs-fuse", GF_LOG_TRACE,
+           "%" PRIu64 ": READ (%p, size=%zu, offset=%" PRIu64 ")",
+           state->finh->unique, state->fd, state->size, state->off);
 
-        FUSE_FOP (state, fuse_readv_cbk, GF_FOP_READ, readv, state->fd,
-                  state->size, state->off, state->io_flags, state->xdata);
+    FUSE_FOP(state, fuse_readv_cbk, GF_FOP_READ, readv, state->fd, state->size,
+             state->off, state->io_flags, state->xdata);
 }
 
 static void
-fuse_readv (xlator_t *this, fuse_in_header_t *finh, void *msg)
+fuse_readv(xlator_t *this, fuse_in_header_t *finh, void *msg,
+           struct iobuf *iobuf)
 {
-        struct fuse_read_in *fri = msg;
+    struct fuse_read_in *fri = msg;
 
 #if FUSE_KERNEL_MINOR_VERSION >= 9
-        fuse_private_t  *priv = NULL;
+    fuse_private_t *priv = NULL;
 #endif
-        fuse_state_t *state = NULL;
-        fd_t         *fd = NULL;
+    fuse_state_t *state = NULL;
+    fd_t *fd = NULL;
 
-        GET_STATE (this, finh, state);
+    GET_STATE(this, finh, state);
 
-        fd = FH_TO_FD (fri->fh);
-        state->fd = fd;
+    fd = FH_TO_FD(fri->fh);
+    state->fd = fd;
 
-        fuse_resolve_fd_init (state, &state->resolve, fd);
+    fuse_resolve_fd_init(state, &state->resolve, fd);
 
-        /* See comment by similar code in fuse_settatr */
+    /* See comment by similar code in fuse_settatr */
 #if FUSE_KERNEL_MINOR_VERSION >= 9
-        priv = this->private;
-        if (priv->proto_minor >= 9 && fri->read_flags & FUSE_READ_LOCKOWNER)
-                state->lk_owner = fri->lock_owner;
+    priv = this->private;
+    if (priv->proto_minor >= 9 && fri->read_flags & FUSE_READ_LOCKOWNER)
+        state->lk_owner = fri->lock_owner;
 #endif
 
-        state->size = fri->size;
-        state->off = fri->offset;
-        /* lets ignore 'fri->read_flags', but just consider 'fri->flags' */
+    state->size = fri->size;
+    state->off = fri->offset;
+    /* lets ignore 'fri->read_flags', but just consider 'fri->flags' */
 #if FUSE_KERNEL_MINOR_VERSION >= 9
-        state->io_flags = fri->flags;
+    state->io_flags = fri->flags;
 #endif
-        fuse_resolve_and_resume (state, fuse_readv_resume);
+    fuse_resolve_and_resume(state, fuse_readv_resume);
 }
 
 static int
-fuse_writev_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                 int32_t op_ret, int32_t op_errno,
-                 struct iatt *stbuf, struct iatt *postbuf, dict_t *xdata)
+fuse_writev_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                int32_t op_ret, int32_t op_errno, struct iatt *stbuf,
+                struct iatt *postbuf, dict_t *xdata)
 {
-        fuse_state_t *state = NULL;
-        fuse_in_header_t *finh = NULL;
-        struct fuse_write_out fwo = {0, };
+    fuse_state_t *state = NULL;
+    fuse_in_header_t *finh = NULL;
+    struct fuse_write_out fwo = {
+        0,
+    };
 
-        state = frame->root->state;
-        finh = state->finh;
+    state = frame->root->state;
+    finh = state->finh;
 
-        fuse_log_eh_fop(this, state, frame, op_ret, op_errno);
+    fuse_log_eh_fop(this, state, frame, op_ret, op_errno);
 
-        if (op_ret >= 0) {
-                gf_log ("glusterfs-fuse", GF_LOG_TRACE,
-                        "%"PRIu64": WRITE => %d/%"GF_PRI_SIZET",%"PRId64"/%"PRIu64,
-                        frame->root->unique,
-                        op_ret, state->size, state->off, stbuf->ia_size);
+    if (op_ret >= 0) {
+        gf_log("glusterfs-fuse", GF_LOG_TRACE,
+               "%" PRIu64 ": WRITE => %d/%" GF_PRI_SIZET ",%" PRId64
+               "/%" PRIu64,
+               frame->root->unique, op_ret, state->size, state->off,
+               stbuf->ia_size);
 
-                fwo.size = op_ret;
-                send_fuse_obj (this, finh, &fwo);
-        } else {
-                gf_log ("glusterfs-fuse", GF_LOG_WARNING,
-                        "%"PRIu64": WRITE => -1 (%s)", frame->root->unique,
-                        strerror (op_errno));
+        fwo.size = op_ret;
+        send_fuse_obj(this, finh, &fwo);
+    } else {
+        gf_log(
+            "glusterfs-fuse", GF_LOG_WARNING,
+            "%" PRIu64 ": WRITE => -1 gfid=%s fd=%p (%s)", frame->root->unique,
+            (state->fd && state->fd->inode) ? uuid_utoa(state->fd->inode->gfid)
+                                            : "nil",
+            state->fd, strerror(op_errno));
 
-                send_fuse_err (this, finh, op_errno);
-        }
+        send_fuse_err(this, finh, op_errno);
+    }
 
-        free_fuse_state (state);
-        STACK_DESTROY (frame->root);
+    free_fuse_state(state);
+    STACK_DESTROY(frame->root);
 
-        return 0;
+    return 0;
 }
 
 void
-fuse_write_resume (fuse_state_t *state)
+fuse_write_resume(fuse_state_t *state)
 {
-        struct iobref *iobref = NULL;
-        struct iobuf  *iobuf = NULL;
+    struct iobref *iobref = NULL;
 
+    iobref = iobref_new();
+    if (!iobref) {
+        gf_log("glusterfs-fuse", GF_LOG_ERROR,
+               "%" PRIu64 ": WRITE iobref allocation failed",
+               state->finh->unique);
+        send_fuse_err(state->this, state->finh, ENOMEM);
 
-        iobref = iobref_new ();
-        if (!iobref) {
-                gf_log ("glusterfs-fuse", GF_LOG_ERROR,
-                        "%"PRIu64": WRITE iobref allocation failed",
-                        state->finh->unique);
-                send_fuse_err (state->this, state->finh, ENOMEM);
-
-                free_fuse_state (state);
-                return;
-        }
+        free_fuse_state(state);
+        return;
+    }
 
-        iobuf = ((fuse_private_t *) (state->this->private))->iobuf;
-        iobref_add (iobref, iobuf);
+    iobref_add(iobref, state->iobuf);
 
-        gf_log ("glusterfs-fuse", GF_LOG_TRACE,
-                "%"PRIu64": WRITE (%p, size=%"GF_PRI_SIZET", offset=%"PRId64")",
-                state->finh->unique, state->fd, state->size, state->off);
+    gf_log("glusterfs-fuse", GF_LOG_TRACE,
+           "%" PRIu64 ": WRITE (%p, size=%" GF_PRI_SIZET ", offset=%" PRId64
+           ")",
+           state->finh->unique, state->fd, state->size, state->off);
 
-        FUSE_FOP (state, fuse_writev_cbk, GF_FOP_WRITE, writev, state->fd,
-                  &state->vector, 1, state->off, state->io_flags, iobref,
-                  state->xdata);
+    FUSE_FOP(state, fuse_writev_cbk, GF_FOP_WRITE, writev, state->fd,
+             &state->vector, 1, state->off, state->io_flags, iobref,
+             state->xdata);
 
-        iobref_unref (iobref);
+    iobref_unref(iobref);
 }
 
 static void
-fuse_write (xlator_t *this, fuse_in_header_t *finh, void *msg)
+fuse_write(xlator_t *this, fuse_in_header_t *finh, void *msg,
+           struct iobuf *iobuf)
 {
-        /* WRITE is special, metadata is attached to in_header,
-         * and msg is the payload as-is.
-         */
-        struct fuse_write_in *fwi = (struct fuse_write_in *)
-                                      (finh + 1);
+    /* WRITE is special, metadata is attached to in_header,
+     * and msg is the payload as-is.
+     */
+    struct fuse_write_in *fwi = (struct fuse_write_in *)(finh + 1);
 
-        fuse_state_t    *state = NULL;
-        fd_t            *fd = NULL;
+    fuse_state_t *state = NULL;
+    fd_t *fd = NULL;
 #if FUSE_KERNEL_MINOR_VERSION >= 9
-        fuse_private_t  *priv = NULL;
-        priv = this->private;
+    fuse_private_t *priv = NULL;
+    priv = this->private;
 #endif
 
-        GET_STATE (this, finh, state);
-        fd          = FH_TO_FD (fwi->fh);
-        state->fd   = fd;
-        state->size = fwi->size;
-        state->off  = fwi->offset;
+    GET_STATE(this, finh, state);
+    fd = FH_TO_FD(fwi->fh);
+    state->fd = fd;
+    state->size = fwi->size;
+    state->off = fwi->offset;
 
-        /* lets ignore 'fwi->write_flags', but just consider 'fwi->flags' */
+    /* lets ignore 'fwi->write_flags', but just consider 'fwi->flags' */
 #if FUSE_KERNEL_MINOR_VERSION >= 9
-        state->io_flags = fwi->flags;
+    state->io_flags = fwi->flags;
 #else
-	state->io_flags = fwi->write_flags;
+    state->io_flags = fwi->write_flags;
 #endif
-        /* TODO: may need to handle below flag
-           (fwi->write_flags & FUSE_WRITE_CACHE);
-        */
+    /* TODO: may need to handle below flag
+       (fwi->write_flags & FUSE_WRITE_CACHE);
+    */
 
+    fuse_resolve_fd_init(state, &state->resolve, fd);
 
-        fuse_resolve_fd_init (state, &state->resolve, fd);
-
-        /* See comment by similar code in fuse_settatr */
+    /* See comment by similar code in fuse_settatr */
 #if FUSE_KERNEL_MINOR_VERSION >= 9
-        priv = this->private;
-        if (priv->proto_minor >= 9 && fwi->write_flags & FUSE_WRITE_LOCKOWNER)
-                state->lk_owner = fwi->lock_owner;
+    priv = this->private;
+    if (priv->proto_minor >= 9 && fwi->write_flags & FUSE_WRITE_LOCKOWNER)
+        state->lk_owner = fwi->lock_owner;
 #endif
 
-        state->vector.iov_base = msg;
-        state->vector.iov_len  = fwi->size;
+    state->vector.iov_base = msg;
+    state->vector.iov_len = fwi->size;
+    state->iobuf = iobuf;
 
-        fuse_resolve_and_resume (state, fuse_write_resume);
+    fuse_resolve_and_resume(state, fuse_write_resume);
 
-        return;
+    return;
+}
+
+#if FUSE_KERNEL_MINOR_VERSION >= 28
+static int
+fuse_copy_file_range_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                         int32_t op_ret, int32_t op_errno, struct iatt *stbuf,
+                         struct iatt *prebuf_dst, struct iatt *postbuf_dst,
+                         dict_t *xdata)
+{
+    fuse_state_t *state = NULL;
+    fuse_in_header_t *finh = NULL;
+    /*
+     * Fuse kernel module uses fuse_write_out itself as the
+     * output collector. In fact, fuse_kernel.h in the upstream
+     * kernel just defines the input structure fuse_copy_file_range_in
+     * for the fop. So, just use the fuse_write_out to send the
+     * response back to the kernel.
+     */
+    struct fuse_write_out fcfro = {
+        0,
+    };
+
+    char src_gfid[GF_UUID_BUF_SIZE] = {0};
+    char dst_gfid[GF_UUID_BUF_SIZE] = {0};
+
+    state = frame->root->state;
+    finh = state->finh;
+
+    fuse_log_eh_fop(this, state, frame, op_ret, op_errno);
+
+    if (op_ret >= 0) {
+        gf_log("glusterfs-fuse", GF_LOG_TRACE,
+               "%" PRIu64 ": WRITE => %d/%" GF_PRI_SIZET ",%" PRIu64
+               " , %" PRIu64 " ,%" PRIu64 ",%" PRIu64,
+               frame->root->unique, op_ret, state->size, state->off_in,
+               state->off_out, stbuf->ia_size, postbuf_dst->ia_size);
+
+        fcfro.size = op_ret;
+        send_fuse_obj(this, finh, &fcfro);
+    } else {
+        if (state->fd && state->fd->inode)
+            uuid_utoa_r(state->fd->inode->gfid, src_gfid);
+        else
+            snprintf(src_gfid, sizeof(src_gfid), "nil");
+
+        if (state->fd_dst && state->fd_dst->inode)
+            uuid_utoa_r(state->fd_dst->inode->gfid, dst_gfid);
+        else
+            snprintf(dst_gfid, sizeof(dst_gfid), "nil");
+
+        gf_log("glusterfs-fuse", GF_LOG_WARNING,
+               "%" PRIu64
+               ": COPY_FILE_RANGE => -1 gfid_in=%s fd_in=%p "
+               "gfid_out=%s fd_out=%p (%s)",
+               frame->root->unique, src_gfid, state->fd, dst_gfid,
+               state->fd_dst, strerror(op_errno));
+
+        send_fuse_err(this, finh, op_errno);
+    }
+
+    free_fuse_state(state);
+    STACK_DESTROY(frame->root);
+
+    return 0;
 }
 
 void
-fuse_flush_resume (fuse_state_t *state)
+fuse_copy_file_range_resume(fuse_state_t *state)
 {
-        FUSE_FOP (state, fuse_err_cbk, GF_FOP_FLUSH,
-                  flush, state->fd, state->xdata);
+    char fd_uuid_str[64] = {0}, fd_dst_uuid_str[64] = {0};
+
+    gf_log("glusterfs-fuse", GF_LOG_TRACE,
+           "%" PRIu64
+           ": COPY_FILE_RANGE "
+           "(input fd: %p (gfid: %s), "
+           "output fd: %p (gfid: %s) size=%zu, "
+           "offset_in=%" PRIu64 ", offset_out=%" PRIu64 ")",
+           state->finh->unique, state->fd,
+           uuid_utoa_r(state->fd->inode->gfid, fd_uuid_str), state->fd_dst,
+           uuid_utoa_r(state->fd_dst->inode->gfid, fd_dst_uuid_str),
+           state->size, state->off_in, state->off_out);
+
+    FUSE_FOP(state, fuse_copy_file_range_cbk, GF_FOP_COPY_FILE_RANGE,
+             copy_file_range, state->fd, state->off_in, state->fd_dst,
+             state->off_out, state->size, state->io_flags, state->xdata);
 }
 
 static void
-fuse_flush (xlator_t *this, fuse_in_header_t *finh, void *msg)
+fuse_copy_file_range(xlator_t *this, fuse_in_header_t *finh, void *msg,
+                     struct iobuf *iobuf)
 {
-        struct fuse_flush_in *ffi = msg;
+    struct fuse_copy_file_range_in *fcfri = msg;
+    fuse_state_t *state = NULL;
+    fd_t *fd_in = NULL;
+    fd_t *fd_out = NULL;
 
-        fuse_state_t *state = NULL;
-        fd_t         *fd = NULL;
+    GET_STATE(this, finh, state);
 
-        GET_STATE (this, finh, state);
-        fd = FH_TO_FD (ffi->fh);
-        state->fd = fd;
+    fd_in = FH_TO_FD(fcfri->fh_in);
+    fd_out = FH_TO_FD(fcfri->fh_out);
+    state->fd = fd_in;
+    state->fd_dst = fd_out;
 
-        fuse_resolve_fd_init (state, &state->resolve, fd);
+    fuse_resolve_fd_init(state, &state->resolve, fd_in);
+    fuse_resolve_fd_init(state, &state->resolve2, fd_out);
 
-        state->lk_owner = ffi->lock_owner;
+    state->size = fcfri->len;
+    state->off_in = fcfri->off_in;
+    state->off_out = fcfri->off_out;
+    state->io_flags = fcfri->flags;
 
-        gf_log ("glusterfs-fuse", GF_LOG_TRACE,
-                "%"PRIu64": FLUSH %p", finh->unique, fd);
+    fuse_resolve_and_resume(state, fuse_copy_file_range_resume);
+}
+#endif /* FUSE_KERNEL_MINOR_VERSION >= 28 */
 
-        fuse_resolve_and_resume (state, fuse_flush_resume);
+#if FUSE_KERNEL_MINOR_VERSION >= 24 && HAVE_SEEK_HOLE
+static int
+fuse_lseek_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+               int32_t op_ret, int32_t op_errno, off_t offset, dict_t *xdata)
+{
+    fuse_state_t *state = frame->root->state;
+    fuse_in_header_t *finh = state->finh;
+    struct fuse_lseek_out flo = {
+        0,
+    };
 
-        return;
+    fuse_log_eh_fop(this, state, frame, op_ret, op_errno);
+
+    if (op_ret >= 0) {
+        flo.offset = offset;
+        send_fuse_obj(this, finh, &flo);
+    } else {
+        send_fuse_err(this, finh, op_errno);
+    }
+
+    free_fuse_state(state);
+    STACK_DESTROY(frame->root);
+
+    return 0;
 }
 
-int
-fuse_internal_release (xlator_t *this, fd_t *fd)
+static void
+fuse_lseek_resume(fuse_state_t *state)
 {
-        //This is a place holder function to prevent "xlator does not implement
-        //release_cbk" Warning log.
-        //Actual release happens as part of fuse_release which gets executed
-        //when kernel fuse sends it.
-        return 0;
+    FUSE_FOP(state, fuse_lseek_cbk, GF_FOP_SEEK, seek, state->fd, state->off,
+             state->whence, state->xdata);
 }
 
 static void
-fuse_release (xlator_t *this, fuse_in_header_t *finh, void *msg)
-{
-        struct fuse_release_in *fri       = msg;
-        fd_t                   *activefd = NULL;
-        fd_t                   *fd        = NULL;
-        uint64_t                val       = 0;
-        int                     ret       = 0;
-        fuse_state_t           *state     = NULL;
-        fuse_fd_ctx_t          *fdctx     = NULL;
-        fuse_private_t         *priv      = NULL;
-
-        GET_STATE (this, finh, state);
-        fd = FH_TO_FD (fri->fh);
-        state->fd = fd;
-
-        priv = this->private;
-
-        fuse_log_eh (this, "RELEASE(): %"PRIu64":, fd: %p, gfid: %s",
-                     finh->unique, fd, uuid_utoa (fd->inode->gfid));
-
-        gf_log ("glusterfs-fuse", GF_LOG_TRACE,
-                "%"PRIu64": RELEASE %p", finh->unique, state->fd);
-
-        ret = fd_ctx_del (fd, this, &val);
-        if (!ret) {
-                fdctx = (fuse_fd_ctx_t *)(unsigned long)val;
-                if (fdctx) {
-                        activefd = fdctx->activefd;
-                        if (activefd) {
-                                fd_unref (activefd);
-                        }
+fuse_lseek(xlator_t *this, fuse_in_header_t *finh, void *msg,
+           struct iobuf *iobuf)
+{
+    struct fuse_lseek_in *ffi = msg;
+    fuse_state_t *state = NULL;
+
+    GET_STATE(this, finh, state);
+    state->fd = FH_TO_FD(ffi->fh);
+    state->off = ffi->offset;
+
+    switch (ffi->whence) {
+        case SEEK_DATA:
+            state->whence = GF_SEEK_DATA;
+            break;
+        case SEEK_HOLE:
+            state->whence = GF_SEEK_HOLE;
+            break;
+        default:
+            /* fuse should handle other whence internally */
+            send_fuse_err(this, finh, EINVAL);
+            free_fuse_state(state);
+            return;
+    }
 
-                        GF_FREE (fdctx);
-                }
+    fuse_resolve_fd_init(state, &state->resolve, state->fd);
+    fuse_resolve_and_resume(state, fuse_lseek_resume);
+}
+#endif /* FUSE_KERNEL_MINOR_VERSION >= 24 && HAVE_SEEK_HOLE */
+
+void
+fuse_flush_resume(fuse_state_t *state)
+{
+    FUSE_FOP(state, fuse_flush_cbk, GF_FOP_FLUSH, flush, state->fd,
+             state->xdata);
+}
+
+static void
+fuse_flush_interrupt_handler(xlator_t *this, fuse_interrupt_record_t *fir)
+{
+    gf_log("glusterfs-fuse", GF_LOG_DEBUG,
+           "FLUSH unique %" PRIu64 ": interrupt handler triggered",
+           fir->fuse_in_header.unique);
+
+    fuse_interrupt_finish_interrupt(this, fir, INTERRUPT_HANDLED, _gf_false,
+                                    NULL);
+}
+
+static void
+fuse_flush(xlator_t *this, fuse_in_header_t *finh, void *msg,
+           struct iobuf *iobuf)
+{
+    struct fuse_flush_in *ffi = msg;
+    fuse_private_t *priv = NULL;
+
+    fuse_state_t *state = NULL;
+    fd_t *fd = NULL;
+
+    GET_STATE(this, finh, state);
+    fd = FH_TO_FD(ffi->fh);
+    state->fd = fd;
+
+    priv = this->private;
+    if (priv->flush_handle_interrupt) {
+        fuse_interrupt_record_t *fir = NULL;
+
+        fir = fuse_interrupt_record_new(finh, fuse_flush_interrupt_handler);
+        if (!fir) {
+            send_fuse_err(this, finh, ENOMEM);
+
+            gf_log("glusterfs-fuse", GF_LOG_ERROR,
+                   "FLUSH unique %" PRIu64
+                   ":"
+                   " interrupt record allocation failed",
+                   finh->unique);
+            free_fuse_state(state);
+
+            return;
         }
-        fd_unref (fd);
+        fuse_interrupt_record_insert(this, fir);
+    }
 
-        state->fd = NULL;
+    fuse_resolve_fd_init(state, &state->resolve, fd);
 
-        gf_fdptr_put (priv->fdtable, fd);
+    state->lk_owner = ffi->lock_owner;
 
-        send_fuse_err (this, finh, 0);
+    gf_log("glusterfs-fuse", GF_LOG_TRACE, "%" PRIu64 ": FLUSH %p",
+           finh->unique, fd);
 
-        free_fuse_state (state);
-        return;
+    fuse_resolve_and_resume(state, fuse_flush_resume);
+
+    return;
 }
 
-void
-fuse_fsync_resume (fuse_state_t *state)
+int
+fuse_internal_release(xlator_t *this, fd_t *fd)
 {
-        gf_log ("glusterfs-fuse", GF_LOG_TRACE,
-                "%"PRIu64": FSYNC %p", state->finh->unique,
-                state->fd);
+    /* This is important we cleanup our context here to avoid a leak
+       in case an error occurs and we get cleanup up by
+       call_unwind_error->...->args_wipe instead of the normal path.
+    */
+    fuse_fd_ctx_destroy(this, fd);
 
-        /* fsync_flags: 1 means "datasync" (no defines for this) */
-        FUSE_FOP (state, fuse_fsync_cbk, GF_FOP_FSYNC,
-                  fsync, state->fd, (state->flags & 1), state->xdata);
+    return 0;
 }
 
 static void
-fuse_fsync (xlator_t *this, fuse_in_header_t *finh, void *msg)
+fuse_release(xlator_t *this, fuse_in_header_t *finh, void *msg,
+             struct iobuf *iobuf)
 {
-        struct fuse_fsync_in *fsi = msg;
+    struct fuse_release_in *fri = msg;
+    fd_t *fd = NULL;
+    fuse_state_t *state = NULL;
+    fuse_private_t *priv = NULL;
 
-        fuse_state_t *state = NULL;
-        fd_t         *fd = NULL;
+    GET_STATE(this, finh, state);
+    fd = FH_TO_FD(fri->fh);
+    if (!fd)
+        goto out;
 
-        GET_STATE (this, finh, state);
-        fd = FH_TO_FD (fsi->fh);
-        state->fd = fd;
+    state->fd = fd;
 
-        fuse_resolve_fd_init (state, &state->resolve, fd);
+    priv = this->private;
 
-        state->flags = fsi->fsync_flags;
-        fuse_resolve_and_resume (state, fuse_fsync_resume);
-        return;
-}
+    fuse_log_eh(this, "RELEASE(): finh->unique: %" PRIu64 ":, fd: %p, gfid: %s",
+                finh->unique, fd, uuid_utoa(fd->inode->gfid));
 
-void
-fuse_opendir_resume (fuse_state_t *state)
-{
-        fd_t           *fd    = NULL;
-        fuse_private_t *priv  = NULL;
-        fuse_fd_ctx_t  *fdctx = NULL;
-
-        priv = state->this->private;
-
-        if (!state->loc.inode) {
-                gf_log ("glusterfs-fuse", GF_LOG_WARNING,
-                        "%"PRIu64": OPENDIR (%s) resolution failed",
-                        state->finh->unique, uuid_utoa (state->resolve.gfid));
-                send_fuse_err (state->this, state->finh,
-                               state->resolve.op_errno);
-                free_fuse_state (state);
-                return;
-        }
+    gf_log("glusterfs-fuse", GF_LOG_TRACE,
+           "finh->unique: %" PRIu64 ": RELEASE %p", finh->unique, state->fd);
 
-        fd = fd_create (state->loc.inode, state->finh->pid);
-        if (fd == NULL) {
-                gf_log ("glusterfs-fuse", GF_LOG_WARNING,
-                        "%"PRIu64": OPENDIR fd creation failed",
-                        state->finh->unique);
-                send_fuse_err (state->this, state->finh, ENOMEM);
-                free_fuse_state (state);
-                return;
-        }
+    fd_close(state->fd);
 
-        fdctx = fuse_fd_ctx_check_n_create (state->this, fd);
-        if (fdctx == NULL) {
-                gf_log ("glusterfs-fuse", GF_LOG_WARNING,
-                        "%"PRIu64": OPENDIR creation of fdctx failed",
-                        state->finh->unique);
-                fd_unref (fd);
-                send_fuse_err (state->this, state->finh, ENOMEM);
-                free_fuse_state (state);
-                return;
-        }
+    fuse_fd_ctx_destroy(this, state->fd);
+    fd_unref(fd);
+
+    gf_fdptr_put(priv->fdtable, fd);
+
+    state->fd = NULL;
 
-        state->fd = fd_ref (fd);
-        state->fd_no = gf_fd_unused_get (priv->fdtable, fd);
+out:
+    send_fuse_err(this, finh, 0);
+
+    free_fuse_state(state);
+    return;
+}
 
-        gf_log ("glusterfs-fuse", GF_LOG_TRACE,
-                "%"PRIu64": OPENDIR %s", state->finh->unique,
-                state->loc.path);
+void
+fuse_fsync_resume(fuse_state_t *state)
+{
+    gf_log("glusterfs-fuse", GF_LOG_TRACE, "%" PRIu64 ": FSYNC %p",
+           state->finh->unique, state->fd);
 
-        FUSE_FOP (state, fuse_fd_cbk, GF_FOP_OPENDIR,
-                  opendir, &state->loc, fd, state->xdata);
+    /* fsync_flags: 1 means "datasync" (no defines for this) */
+    FUSE_FOP(state, fuse_fsync_cbk, GF_FOP_FSYNC, fsync, state->fd,
+             (state->flags & 1), state->xdata);
 }
 
 static void
-fuse_opendir (xlator_t *this, fuse_in_header_t *finh, void *msg)
+fuse_fsync(xlator_t *this, fuse_in_header_t *finh, void *msg,
+           struct iobuf *iobuf)
 {
-        /*
-        struct fuse_open_in *foi = msg;
-         */
+    struct fuse_fsync_in *fsi = msg;
 
-        fuse_state_t *state = NULL;
+    fuse_state_t *state = NULL;
+    fd_t *fd = NULL;
 
-        GET_STATE (this, finh, state);
+    GET_STATE(this, finh, state);
+    fd = FH_TO_FD(fsi->fh);
+    state->fd = fd;
 
-        fuse_resolve_inode_init (state, &state->resolve, finh->nodeid);
+    fuse_resolve_fd_init(state, &state->resolve, fd);
 
-        fuse_resolve_and_resume (state, fuse_opendir_resume);
+    state->flags = fsi->fsync_flags;
+    fuse_resolve_and_resume(state, fuse_fsync_resume);
+    return;
 }
 
-unsigned char
-d_type_from_stat (struct iatt *buf)
+void
+fuse_opendir_resume(fuse_state_t *state)
 {
-        unsigned char d_type;
-
-        if (IA_ISLNK (buf->ia_type)) {
-                d_type = DT_LNK;
+    fd_t *fd = NULL;
+    fuse_private_t *priv = NULL;
+    fuse_fd_ctx_t *fdctx = NULL;
 
-        } else if (IA_ISDIR (buf->ia_type)) {
-                d_type = DT_DIR;
+    priv = state->this->private;
 
-        } else if (IA_ISFIFO (buf->ia_type)) {
-                d_type = DT_FIFO;
+    if (!state->loc.inode) {
+        gf_log("glusterfs-fuse", GF_LOG_WARNING,
+               "%" PRIu64 ": OPENDIR (%s) resolution failed",
+               state->finh->unique, uuid_utoa(state->resolve.gfid));
 
-        } else if (IA_ISSOCK (buf->ia_type)) {
-                d_type = DT_SOCK;
+        /* facilitate retry from VFS */
+        if (state->resolve.op_errno == ENOENT)
+            state->resolve.op_errno = ESTALE;
 
-        } else if (IA_ISCHR (buf->ia_type)) {
-                d_type = DT_CHR;
-
-        } else if (IA_ISBLK (buf->ia_type)) {
-                d_type = DT_BLK;
+        send_fuse_err(state->this, state->finh, state->resolve.op_errno);
+        free_fuse_state(state);
+        return;
+    }
+
+    fd = fd_create(state->loc.inode, state->finh->pid);
+    if (fd == NULL) {
+        gf_log("glusterfs-fuse", GF_LOG_WARNING,
+               "%" PRIu64 ": OPENDIR fd creation failed", state->finh->unique);
+        send_fuse_err(state->this, state->finh, ENOMEM);
+        free_fuse_state(state);
+        return;
+    }
+
+    fdctx = fuse_fd_ctx_check_n_create(state->this, fd);
+    if (fdctx == NULL) {
+        gf_log("glusterfs-fuse", GF_LOG_WARNING,
+               "%" PRIu64 ": OPENDIR creation of fdctx failed",
+               state->finh->unique);
+        fd_unref(fd);
+        send_fuse_err(state->this, state->finh, ENOMEM);
+        free_fuse_state(state);
+        return;
+    }
 
-        } else if (IA_ISREG (buf->ia_type)) {
-                d_type = DT_REG;
+    state->fd = fd_ref(fd);
+    state->fd_no = gf_fd_unused_get(priv->fdtable, fd);
 
-        } else {
-                d_type = DT_UNKNOWN;
-        }
+    gf_log("glusterfs-fuse", GF_LOG_TRACE, "%" PRIu64 ": OPENDIR %s",
+           state->finh->unique, state->loc.path);
 
-        return d_type;
+    FUSE_FOP(state, fuse_fd_cbk, GF_FOP_OPENDIR, opendir, &state->loc, fd,
+             state->xdata);
 }
 
-static int
-fuse_readdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                  int32_t op_ret, int32_t op_errno, gf_dirent_t *entries,
-                  dict_t *xdata)
+static void
+fuse_opendir(xlator_t *this, fuse_in_header_t *finh, void *msg,
+             struct iobuf *iobuf)
 {
-        fuse_state_t *state = NULL;
-        fuse_in_header_t *finh = NULL;
-        size_t        size = 0;
-        size_t        max_size = 0;
-        char         *buf = NULL;
-        gf_dirent_t  *entry = NULL;
-        struct fuse_dirent *fde = NULL;
-        fuse_private_t *priv = NULL;
+    /*
+    struct fuse_open_in *foi = msg;
+     */
 
-        state = frame->root->state;
-        finh  = state->finh;
-        priv = state->this->private;
+    fuse_state_t *state = NULL;
 
-        fuse_log_eh_fop(this, state, frame, op_ret, op_errno);
+    GET_STATE(this, finh, state);
 
-        if (op_ret < 0) {
-                gf_log ("glusterfs-fuse", GF_LOG_WARNING,
-                        "%"PRIu64": READDIR => -1 (%s)", frame->root->unique,
-                        strerror (op_errno));
+    fuse_resolve_inode_init(state, &state->resolve, finh->nodeid);
 
-                send_fuse_err (this, finh, op_errno);
-                goto out;
-        }
+    fuse_resolve_and_resume(state, fuse_opendir_resume);
+}
 
-        gf_log ("glusterfs-fuse", GF_LOG_TRACE,
-                "%"PRIu64": READDIR => %d/%"GF_PRI_SIZET",%"PRId64,
-                frame->root->unique, op_ret, state->size, state->off);
+unsigned char
+d_type_from_stat(struct iatt *buf)
+{
+    unsigned char d_type;
 
-        list_for_each_entry (entry, &entries->list, list) {
-                size_t fde_size = FUSE_DIRENT_ALIGN (FUSE_NAME_OFFSET +
-                                                     strlen (entry->d_name));
-                max_size += fde_size;
+    if (IA_ISLNK(buf->ia_type)) {
+        d_type = DT_LNK;
 
-                if (max_size > state->size) {
-                        /* we received too many entries to fit in the reply */
-                        max_size -= fde_size;
-                        break;
-                }
-        }
+    } else if (IA_ISDIR(buf->ia_type)) {
+        d_type = DT_DIR;
 
-        if (max_size == 0) {
-                send_fuse_data (this, finh, 0, 0);
-                goto out;
-        }
+    } else if (IA_ISFIFO(buf->ia_type)) {
+        d_type = DT_FIFO;
 
-        buf = GF_CALLOC (1, max_size, gf_fuse_mt_char);
-        if (!buf) {
-                gf_log ("glusterfs-fuse", GF_LOG_DEBUG,
-                        "%"PRIu64": READDIR => -1 (%s)", frame->root->unique,
-                        strerror (ENOMEM));
-                send_fuse_err (this, finh, ENOMEM);
-                goto out;
-        }
+    } else if (IA_ISSOCK(buf->ia_type)) {
+        d_type = DT_SOCK;
 
-        size = 0;
-        list_for_each_entry (entry, &entries->list, list) {
-                fde = (struct fuse_dirent *)(buf + size);
-                gf_fuse_fill_dirent (entry, fde, priv->enable_ino32);
-                size += FUSE_DIRENT_SIZE (fde);
+    } else if (IA_ISCHR(buf->ia_type)) {
+        d_type = DT_CHR;
 
-                if (size == max_size)
-                        break;
-        }
+    } else if (IA_ISBLK(buf->ia_type)) {
+        d_type = DT_BLK;
 
-        send_fuse_data (this, finh, buf, size);
+    } else if (IA_ISREG(buf->ia_type)) {
+        d_type = DT_REG;
 
-        /* TODO: */
-        /* gf_link_inodes_from_dirent (this, state->fd->inode, entries); */
+    } else {
+        d_type = DT_UNKNOWN;
+    }
 
-out:
-        free_fuse_state (state);
-        STACK_DESTROY (frame->root);
-        GF_FREE (buf);
-        return 0;
+    return d_type;
+}
 
+static int
+fuse_readdir_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                 int32_t op_ret, int32_t op_errno, gf_dirent_t *entries,
+                 dict_t *xdata)
+{
+    fuse_state_t *state = NULL;
+    fuse_in_header_t *finh = NULL;
+    size_t size = 0;
+    size_t max_size = 0;
+    char *buf = NULL;
+    gf_dirent_t *entry = NULL;
+    struct fuse_dirent *fde = NULL;
+    fuse_private_t *priv = NULL;
+
+    state = frame->root->state;
+    finh = state->finh;
+    priv = state->this->private;
+
+    fuse_log_eh_fop(this, state, frame, op_ret, op_errno);
+
+    if (op_ret < 0) {
+        gf_log("glusterfs-fuse", GF_LOG_WARNING,
+               "%" PRIu64 ": READDIR => -1 (%s)", frame->root->unique,
+               strerror(op_errno));
+
+        send_fuse_err(this, finh, op_errno);
+        goto out;
+    }
+
+    gf_log("glusterfs-fuse", GF_LOG_TRACE,
+           "%" PRIu64 ": READDIR => %d/%" GF_PRI_SIZET ",%" PRId64,
+           frame->root->unique, op_ret, state->size, state->off);
+
+    list_for_each_entry(entry, &entries->list, list)
+    {
+        size_t fde_size = FUSE_DIRENT_ALIGN(FUSE_NAME_OFFSET +
+                                            strlen(entry->d_name));
+        max_size += fde_size;
+
+        if (max_size > state->size) {
+            /* we received too many entries to fit in the reply */
+            max_size -= fde_size;
+            break;
+        }
+    }
+
+    if (max_size == 0) {
+        send_fuse_data(this, finh, 0, 0);
+        goto out;
+    }
+
+    buf = GF_CALLOC(1, max_size, gf_fuse_mt_char);
+    if (!buf) {
+        gf_log("glusterfs-fuse", GF_LOG_DEBUG,
+               "%" PRIu64 ": READDIR => -1 (%s)", frame->root->unique,
+               strerror(ENOMEM));
+        send_fuse_err(this, finh, ENOMEM);
+        goto out;
+    }
+
+    size = 0;
+    list_for_each_entry(entry, &entries->list, list)
+    {
+        fde = (struct fuse_dirent *)(buf + size);
+        gf_fuse_fill_dirent(entry, fde, priv->enable_ino32);
+        size += FUSE_DIRENT_SIZE(fde);
+
+        if (size == max_size)
+            break;
+    }
+
+    send_fuse_data(this, finh, buf, size);
+
+    /* TODO: */
+    /* gf_link_inodes_from_dirent (this, state->fd->inode, entries); */
+
+out:
+    free_fuse_state(state);
+    STACK_DESTROY(frame->root);
+    GF_FREE(buf);
+    return 0;
 }
 
 void
-fuse_readdir_resume (fuse_state_t *state)
+fuse_readdir_resume(fuse_state_t *state)
 {
-        gf_log ("glusterfs-fuse", GF_LOG_TRACE,
-                "%"PRIu64": READDIR (%p, size=%"GF_PRI_SIZET", offset=%"PRId64")",
-                state->finh->unique, state->fd, state->size, state->off);
+    gf_log("glusterfs-fuse", GF_LOG_TRACE,
+           "%" PRIu64 ": READDIR (%p, size=%" GF_PRI_SIZET ", offset=%" PRId64
+           ")",
+           state->finh->unique, state->fd, state->size, state->off);
 
-        FUSE_FOP (state, fuse_readdir_cbk, GF_FOP_READDIR,
-                  readdir, state->fd, state->size, state->off, state->xdata);
+    FUSE_FOP(state, fuse_readdir_cbk, GF_FOP_READDIR, readdir, state->fd,
+             state->size, state->off, state->xdata);
 }
 
 static void
-fuse_readdir (xlator_t *this, fuse_in_header_t *finh, void *msg)
+fuse_readdir(xlator_t *this, fuse_in_header_t *finh, void *msg,
+             struct iobuf *iobuf)
 {
-        struct fuse_read_in *fri = msg;
+    struct fuse_read_in *fri = msg;
 
-        fuse_state_t *state = NULL;
-        fd_t         *fd = NULL;
+    fuse_state_t *state = NULL;
+    fd_t *fd = NULL;
 
-        GET_STATE (this, finh, state);
-        state->size = fri->size;
-        state->off = fri->offset;
-        fd = FH_TO_FD (fri->fh);
-        state->fd = fd;
+    GET_STATE(this, finh, state);
+    state->size = fri->size;
+    state->off = fri->offset;
+    fd = FH_TO_FD(fri->fh);
+    state->fd = fd;
 
-        fuse_resolve_fd_init (state, &state->resolve, fd);
+    fuse_resolve_fd_init(state, &state->resolve, fd);
 
-        fuse_resolve_and_resume (state, fuse_readdir_resume);
+    fuse_resolve_and_resume(state, fuse_readdir_resume);
 }
 
 #if FUSE_KERNEL_MINOR_VERSION >= 20
 static int
-fuse_readdirp_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-		   int32_t op_ret, int32_t op_errno, gf_dirent_t *entries,
-		   dict_t *xdata)
-{
-	fuse_state_t *state = NULL;
-	fuse_in_header_t *finh = NULL;
-        size_t        max_size = 0;
-        size_t        size = 0;
-	char         *buf = NULL;
-	gf_dirent_t  *entry = NULL;
-	struct fuse_direntplus *fde = NULL;
-	struct fuse_entry_out *feo = NULL;
-	fuse_private_t         *priv   = NULL;
-
-	state = frame->root->state;
-	finh  = state->finh;
-	priv = this->private;
-
-	if (op_ret < 0) {
-		gf_log ("glusterfs-fuse", GF_LOG_WARNING,
-			"%"PRIu64": READDIRP => -1 (%s)", frame->root->unique,
-			strerror (op_errno));
-
-		send_fuse_err (this, finh, op_errno);
-		goto out;
-	}
-
-	gf_log ("glusterfs-fuse", GF_LOG_TRACE,
-		"%"PRIu64": READDIRP => %d/%"GF_PRI_SIZET",%"PRId64,
-		frame->root->unique, op_ret, state->size, state->off);
-
-	list_for_each_entry (entry, &entries->list, list) {
-                size_t fdes = FUSE_DIRENT_ALIGN (FUSE_NAME_OFFSET_DIRENTPLUS +
-                                                 strlen (entry->d_name));
-                max_size += fdes;
-
-                if (max_size > state->size) {
-                        /* we received too many entries to fit in the reply */
-                        max_size -= fdes;
-                        break;
-                }
-	}
-
-	if (max_size == 0) {
-		send_fuse_data (this, finh, 0, 0);
-		goto out;
-	}
-
-	buf = GF_CALLOC (1, max_size, gf_fuse_mt_char);
-	if (!buf) {
-		gf_log ("glusterfs-fuse", GF_LOG_DEBUG,
-			"%"PRIu64": READDIRP => -1 (%s)", frame->root->unique,
-			strerror (ENOMEM));
-		send_fuse_err (this, finh, ENOMEM);
-		goto out;
-	}
-
-	size = 0;
-	list_for_each_entry (entry, &entries->list, list) {
-		inode_t *linked_inode;
-
-		fde = (struct fuse_direntplus *)(buf + size);
-		feo = &fde->entry_out;
-		fde->dirent.ino = entry->d_ino;
-		fde->dirent.off = entry->d_off;
-		fde->dirent.type = entry->d_type;
-		fde->dirent.namelen = strlen (entry->d_name);
-		strncpy (fde->dirent.name, entry->d_name, fde->dirent.namelen);
-		size += FUSE_DIRENTPLUS_SIZE (fde);
-
-		if (!entry->inode)
-			goto next_entry;
-
-		entry->d_stat.ia_blksize = this->ctx->page_size;
-		gf_fuse_stat2attr (&entry->d_stat, &feo->attr, priv->enable_ino32);
-
-		linked_inode = inode_link (entry->inode, state->fd->inode,
-					   entry->d_name, &entry->d_stat);
-		if (!linked_inode)
-			goto next_entry;
-
-		inode_lookup (linked_inode);
-
-		feo->nodeid = inode_to_fuse_nodeid (linked_inode);
-
-		fuse_inode_set_need_lookup (linked_inode, this);
-
-		inode_unref (linked_inode);
-
-		feo->entry_valid =
-			calc_timeout_sec (priv->entry_timeout);
-		feo->entry_valid_nsec =
-			calc_timeout_nsec (priv->entry_timeout);
-		feo->attr_valid =
-			calc_timeout_sec (priv->attribute_timeout);
-		feo->attr_valid_nsec =
-			calc_timeout_nsec (priv->attribute_timeout);
-
-next_entry:
-                if (size == max_size)
-                        break;
-	}
+fuse_readdirp_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                  int32_t op_ret, int32_t op_errno, gf_dirent_t *entries,
+                  dict_t *xdata)
+{
+    fuse_state_t *state = NULL;
+    fuse_in_header_t *finh = NULL;
+    size_t max_size = 0;
+    size_t size = 0;
+    char *buf = NULL;
+    gf_dirent_t *entry = NULL;
+    struct fuse_direntplus *fde = NULL;
+    struct fuse_entry_out *feo = NULL;
+    fuse_private_t *priv = NULL;
+
+    state = frame->root->state;
+    finh = state->finh;
+    priv = this->private;
+
+    if (op_ret < 0) {
+        gf_log("glusterfs-fuse", GF_LOG_WARNING,
+               "%" PRIu64 ": READDIRP => -1 (%s)", frame->root->unique,
+               strerror(op_errno));
+
+        send_fuse_err(this, finh, op_errno);
+        goto out;
+    }
+
+    gf_log("glusterfs-fuse", GF_LOG_TRACE,
+           "%" PRIu64 ": READDIRP => %d/%" GF_PRI_SIZET ",%" PRId64,
+           frame->root->unique, op_ret, state->size, state->off);
+
+    list_for_each_entry(entry, &entries->list, list)
+    {
+        size_t fdes = FUSE_DIRENT_ALIGN(FUSE_NAME_OFFSET_DIRENTPLUS +
+                                        strlen(entry->d_name));
+        max_size += fdes;
+
+        if (max_size > state->size) {
+            /* we received too many entries to fit in the reply */
+            max_size -= fdes;
+            break;
+        }
+    }
+
+    if (max_size == 0) {
+        send_fuse_data(this, finh, 0, 0);
+        goto out;
+    }
+
+    buf = GF_CALLOC(1, max_size, gf_fuse_mt_char);
+    if (!buf) {
+        gf_log("glusterfs-fuse", GF_LOG_DEBUG,
+               "%" PRIu64 ": READDIRP => -1 (%s)", frame->root->unique,
+               strerror(ENOMEM));
+        send_fuse_err(this, finh, ENOMEM);
+        goto out;
+    }
+
+    size = 0;
+    list_for_each_entry(entry, &entries->list, list)
+    {
+        inode_t *linked_inode;
+
+        fde = (struct fuse_direntplus *)(buf + size);
+        feo = &fde->entry_out;
+
+        if (priv->enable_ino32)
+            fde->dirent.ino = GF_FUSE_SQUASH_INO(entry->d_ino);
+        else
+            fde->dirent.ino = entry->d_ino;
+
+        fde->dirent.off = entry->d_off;
+        fde->dirent.type = entry->d_type;
+        fde->dirent.namelen = strlen(entry->d_name);
+        (void)memcpy(fde->dirent.name, entry->d_name, fde->dirent.namelen);
+        size += FUSE_DIRENTPLUS_SIZE(fde);
+
+        if (!entry->inode)
+            goto next_entry;
+
+        entry->d_stat.ia_blksize = this->ctx->page_size;
+        gf_fuse_stat2attr(&entry->d_stat, &feo->attr, priv->enable_ino32);
+
+        linked_inode = inode_link(entry->inode, state->fd->inode, entry->d_name,
+                                  &entry->d_stat);
+        if (!linked_inode)
+            goto next_entry;
+
+        if (entry->inode != linked_inode) {
+            memset(&entry->d_stat, 0, sizeof(entry->d_stat));
+        }
+
+        feo->nodeid = inode_to_fuse_nodeid(linked_inode);
+
+        if (!((strcmp(entry->d_name, ".") == 0) ||
+              (strcmp(entry->d_name, "..") == 0))) {
+            inode_lookup(linked_inode);
+        }
+
+        inode_unref(linked_inode);
+
+        feo->entry_valid = calc_timeout_sec(priv->entry_timeout);
+        feo->entry_valid_nsec = calc_timeout_nsec(priv->entry_timeout);
+
+        if (entry->d_stat.ia_ctime) {
+            feo->attr_valid = calc_timeout_sec(priv->attribute_timeout);
+            feo->attr_valid_nsec = calc_timeout_nsec(priv->attribute_timeout);
+        } else {
+            feo->attr_valid = feo->attr_valid_nsec = 0;
+        }
 
-	send_fuse_data (this, finh, buf, size);
-out:
-	free_fuse_state (state);
-	STACK_DESTROY (frame->root);
-	GF_FREE (buf);
-	return 0;
+    next_entry:
+        if (size == max_size)
+            break;
+    }
 
+    send_fuse_data(this, finh, buf, size);
+out:
+    free_fuse_state(state);
+    STACK_DESTROY(frame->root);
+    GF_FREE(buf);
+    return 0;
 }
 
 void
-fuse_readdirp_resume (fuse_state_t *state)
+fuse_readdirp_resume(fuse_state_t *state)
 {
-	gf_log ("glusterfs-fuse", GF_LOG_TRACE,
-		"%"PRIu64": READDIRP (%p, size=%"GF_PRI_SIZET", offset=%"PRId64")",
-		state->finh->unique, state->fd, state->size, state->off);
+    gf_log("glusterfs-fuse", GF_LOG_TRACE,
+           "%" PRIu64 ": READDIRP (%p, size=%" GF_PRI_SIZET ", offset=%" PRId64
+           ")",
+           state->finh->unique, state->fd, state->size, state->off);
 
-	FUSE_FOP (state, fuse_readdirp_cbk, GF_FOP_READDIRP,
-		  readdirp, state->fd, state->size, state->off, state->xdata);
+    FUSE_FOP(state, fuse_readdirp_cbk, GF_FOP_READDIRP, readdirp, state->fd,
+             state->size, state->off, state->xdata);
 }
 
-
 static void
-fuse_readdirp (xlator_t *this, fuse_in_header_t *finh, void *msg)
+fuse_readdirp(xlator_t *this, fuse_in_header_t *finh, void *msg,
+              struct iobuf *iobuf)
 {
-	struct fuse_read_in *fri = msg;
+    struct fuse_read_in *fri = msg;
 
-	fuse_state_t *state = NULL;
-	fd_t         *fd = NULL;
+    fuse_state_t *state = NULL;
+    fd_t *fd = NULL;
 
-	GET_STATE (this, finh, state);
-	state->size = fri->size;
-	state->off = fri->offset;
-	fd = FH_TO_FD (fri->fh);
-	state->fd = fd;
+    GET_STATE(this, finh, state);
+    state->size = fri->size;
+    state->off = fri->offset;
+    fd = FH_TO_FD(fri->fh);
+    state->fd = fd;
 
-	fuse_resolve_fd_init (state, &state->resolve, fd);
+    fuse_resolve_fd_init(state, &state->resolve, fd);
 
-	fuse_resolve_and_resume (state, fuse_readdirp_resume);
+    fuse_resolve_and_resume(state, fuse_readdirp_resume);
 }
 #endif
 
@@ -2843,393 +3854,394 @@ fuse_readdirp (xlator_t *this, fuse_in_header_t *finh, void *msg)
 #ifdef FALLOC_FL_KEEP_SIZE
 static int
 fuse_fallocate_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
-		   int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
-		   struct iatt *postbuf, dict_t *xdata)
+                   int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
+                   struct iatt *postbuf, dict_t *xdata)
 {
-	return fuse_err_cbk(frame, cookie, this, op_ret, op_errno, xdata);
+    return fuse_err_cbk(frame, cookie, this, op_ret, op_errno, xdata);
 }
 
 static void
 fuse_fallocate_resume(fuse_state_t *state)
 {
-	gf_log("glusterfs-fuse", GF_LOG_TRACE,
-	       "%"PRIu64": FALLOCATE (%p, flags=%d, size=%zu, offset=%"PRId64")",
-	       state->finh->unique, state->fd, state->flags, state->size,
-	       state->off);
+    gf_log(
+        "glusterfs-fuse", GF_LOG_TRACE,
+        "%" PRIu64 ": FALLOCATE (%p, flags=%d, size=%zu, offset=%" PRId64 ")",
+        state->finh->unique, state->fd, state->flags, state->size, state->off);
 
-	if (state->flags & FALLOC_FL_PUNCH_HOLE)
-		FUSE_FOP(state, fuse_fallocate_cbk, GF_FOP_DISCARD, discard,
-			 state->fd, state->off, state->size, state->xdata);
-	else
-		FUSE_FOP(state, fuse_fallocate_cbk, GF_FOP_FALLOCATE, fallocate,
-			 state->fd, (state->flags & FALLOC_FL_KEEP_SIZE),
-			 state->off, state->size, state->xdata);
+    if (state->flags & FALLOC_FL_PUNCH_HOLE)
+        FUSE_FOP(state, fuse_fallocate_cbk, GF_FOP_DISCARD, discard, state->fd,
+                 state->off, state->size, state->xdata);
+    else
+        FUSE_FOP(state, fuse_fallocate_cbk, GF_FOP_FALLOCATE, fallocate,
+                 state->fd, (state->flags & FALLOC_FL_KEEP_SIZE), state->off,
+                 state->size, state->xdata);
 }
 
 static void
-fuse_fallocate(xlator_t *this, fuse_in_header_t *finh, void *msg)
+fuse_fallocate(xlator_t *this, fuse_in_header_t *finh, void *msg,
+               struct iobuf *iobuf)
 {
-	struct fuse_fallocate_in *ffi = msg;
-	fuse_state_t *state = NULL;
+    struct fuse_fallocate_in *ffi = msg;
+    fuse_state_t *state = NULL;
 
-	GET_STATE(this, finh, state);
-	state->off = ffi->offset;
-	state->size = ffi->length;
-	state->flags = ffi->mode;
-	state->fd = FH_TO_FD(ffi->fh);
+    GET_STATE(this, finh, state);
+    state->off = ffi->offset;
+    state->size = ffi->length;
+    state->flags = ffi->mode;
+    state->fd = FH_TO_FD(ffi->fh);
 
-	fuse_resolve_fd_init(state, &state->resolve, state->fd);
-	fuse_resolve_and_resume(state, fuse_fallocate_resume);
+    fuse_resolve_fd_init(state, &state->resolve, state->fd);
+    fuse_resolve_and_resume(state, fuse_fallocate_resume);
 }
 #endif /* FALLOC_FL_KEEP_SIZE */
 #endif /* FUSE minor version >= 19 */
 
 static void
-fuse_releasedir (xlator_t *this, fuse_in_header_t *finh, void *msg)
+fuse_releasedir(xlator_t *this, fuse_in_header_t *finh, void *msg,
+                struct iobuf *iobuf)
 {
-        struct fuse_release_in *fri       = msg;
-        fd_t                   *activefd = NULL;
-        uint64_t                val       = 0;
-        int                     ret       = 0;
-        fuse_state_t           *state     = NULL;
-        fuse_fd_ctx_t          *fdctx     = NULL;
-        fuse_private_t         *priv      = NULL;
+    struct fuse_release_in *fri = msg;
+    fuse_state_t *state = NULL;
+    fuse_private_t *priv = NULL;
 
-        GET_STATE (this, finh, state);
-        state->fd = FH_TO_FD (fri->fh);
+    GET_STATE(this, finh, state);
+    state->fd = FH_TO_FD(fri->fh);
+    if (!state->fd)
+        goto out;
 
-        priv = this->private;
+    priv = this->private;
 
-        fuse_log_eh (this, "RELEASEDIR (): %"PRIu64": fd: %p, gfid: %s",
-                     finh->unique, state->fd,
-                     uuid_utoa (state->fd->inode->gfid));
+    fuse_log_eh(this,
+                "RELEASEDIR (): finh->unique: %" PRIu64 ": fd: %p, gfid: %s",
+                finh->unique, state->fd, uuid_utoa(state->fd->inode->gfid));
 
-        gf_log ("glusterfs-fuse", GF_LOG_TRACE,
-                "%"PRIu64": RELEASEDIR %p", finh->unique, state->fd);
+    gf_log("glusterfs-fuse", GF_LOG_TRACE,
+           "finh->unique: %" PRIu64 ": RELEASEDIR %p", finh->unique, state->fd);
 
-        ret = fd_ctx_del (state->fd, this, &val);
+    fuse_fd_ctx_destroy(this, state->fd);
+    fd_unref(state->fd);
 
-        if (!ret) {
-                fdctx = (fuse_fd_ctx_t *)(unsigned long)val;
-                if (fdctx) {
-                        activefd = fdctx->activefd;
-                        if (activefd) {
-                                fd_unref (activefd);
-                        }
-
-                        GF_FREE (fdctx);
-                }
-        }
+    gf_fdptr_put(priv->fdtable, state->fd);
 
-        fd_unref (state->fd);
+    state->fd = NULL;
 
-        gf_fdptr_put (priv->fdtable, state->fd);
-
-        state->fd = NULL;
-
-        send_fuse_err (this, finh, 0);
+out:
+    send_fuse_err(this, finh, 0);
 
-        free_fuse_state (state);
+    free_fuse_state(state);
 
-        return;
+    return;
 }
 
 void
-fuse_fsyncdir_resume (fuse_state_t *state)
+fuse_fsyncdir_resume(fuse_state_t *state)
 {
-        FUSE_FOP (state, fuse_err_cbk, GF_FOP_FSYNCDIR,
-                  fsyncdir, state->fd, (state->flags & 1), state->xdata);
-
+    FUSE_FOP(state, fuse_err_cbk, GF_FOP_FSYNCDIR, fsyncdir, state->fd,
+             (state->flags & 1), state->xdata);
 }
 
 static void
-fuse_fsyncdir (xlator_t *this, fuse_in_header_t *finh, void *msg)
+fuse_fsyncdir(xlator_t *this, fuse_in_header_t *finh, void *msg,
+              struct iobuf *iobuf)
 {
-        struct fuse_fsync_in *fsi = msg;
+    struct fuse_fsync_in *fsi = msg;
 
-        fuse_state_t *state = NULL;
-        fd_t         *fd = NULL;
+    fuse_state_t *state = NULL;
+    fd_t *fd = NULL;
 
-        fd = FH_TO_FD (fsi->fh);
+    fd = FH_TO_FD(fsi->fh);
 
-        GET_STATE (this, finh, state);
-        state->fd = fd;
+    GET_STATE(this, finh, state);
+    state->fd = fd;
 
-        fuse_resolve_fd_init (state, &state->resolve, fd);
+    fuse_resolve_fd_init(state, &state->resolve, fd);
 
-        state->flags = fsi->fsync_flags;
-        fuse_resolve_and_resume (state, fuse_fsyncdir_resume);
+    state->flags = fsi->fsync_flags;
+    fuse_resolve_and_resume(state, fuse_fsyncdir_resume);
 
-        return;
+    return;
 }
 
 static int
-fuse_statfs_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                 int32_t op_ret, int32_t op_errno, struct statvfs *buf,
-                 dict_t *xdata)
+fuse_statfs_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                int32_t op_ret, int32_t op_errno, struct statvfs *buf,
+                dict_t *xdata)
 {
-        fuse_state_t *state = NULL;
-        fuse_in_header_t *finh = NULL;
-        fuse_private_t   *priv = NULL;
-        struct fuse_statfs_out fso = {{0, }, };
-
-        state = frame->root->state;
-        priv  = this->private;
-        finh  = state->finh;
-
-        fuse_log_eh (this, "op_ret: %d, op_errno: %d, %"PRIu64": %s()",
-                     op_ret, op_errno, frame->root->unique,
-                     gf_fop_list[frame->root->op]);
-
-        if (op_ret == 0) {
-#ifndef GF_DARWIN_HOST_OS
-                /* MacFUSE doesn't respect anyof these tweaks */
-                buf->f_blocks *= buf->f_frsize;
-                buf->f_blocks /= this->ctx->page_size;
-
-                buf->f_bavail *= buf->f_frsize;
-                buf->f_bavail /= this->ctx->page_size;
-
-                buf->f_bfree *= buf->f_frsize;
-                buf->f_bfree /= this->ctx->page_size;
-
-                buf->f_frsize = buf->f_bsize =this->ctx->page_size;
-#endif /* GF_DARWIN_HOST_OS */
-                fso.st.bsize   = buf->f_bsize;
-                fso.st.frsize  = buf->f_frsize;
-                fso.st.blocks  = buf->f_blocks;
-                fso.st.bfree   = buf->f_bfree;
-                fso.st.bavail  = buf->f_bavail;
-                fso.st.files   = buf->f_files;
-                fso.st.ffree   = buf->f_ffree;
-                fso.st.namelen = buf->f_namemax;
-
-                priv->proto_minor >= 4 ?
-                send_fuse_obj (this, finh, &fso) :
-                send_fuse_data (this, finh, &fso, FUSE_COMPAT_STATFS_SIZE);
-        } else {
-                gf_log ("glusterfs-fuse", GF_LOG_WARNING,
-                        "%"PRIu64": ERR => -1 (%s)", frame->root->unique,
-                        strerror (op_errno));
-                send_fuse_err (this, finh, op_errno);
-        }
+    fuse_state_t *state = NULL;
+    fuse_in_header_t *finh = NULL;
+    fuse_private_t *priv = NULL;
+    struct fuse_statfs_out fso = {
+        {
+            0,
+        },
+    };
 
-        free_fuse_state (state);
-        STACK_DESTROY (frame->root);
+    state = frame->root->state;
+    priv = this->private;
+    finh = state->finh;
 
-        return 0;
+    fuse_log_eh(this, "op_ret: %d, op_errno: %d, %" PRIu64 ": %s()", op_ret,
+                op_errno, frame->root->unique, gf_fop_list[frame->root->op]);
+
+    if (op_ret == 0) {
+        fso.st.bsize = buf->f_bsize;
+        fso.st.frsize = buf->f_frsize;
+        fso.st.blocks = buf->f_blocks;
+        fso.st.bfree = buf->f_bfree;
+        fso.st.bavail = buf->f_bavail;
+        fso.st.files = buf->f_files;
+        fso.st.ffree = buf->f_ffree;
+        fso.st.namelen = buf->f_namemax;
+
+        priv->proto_minor >= 4
+            ? send_fuse_obj(this, finh, &fso)
+            : send_fuse_data(this, finh, &fso, FUSE_COMPAT_STATFS_SIZE);
+    } else {
+        /* facilitate retry from VFS */
+        if (op_errno == ENOENT)
+            op_errno = ESTALE;
+
+        gf_log("glusterfs-fuse", GF_LOG_WARNING, "%" PRIu64 ": ERR => -1 (%s)",
+               frame->root->unique, strerror(op_errno));
+
+        send_fuse_err(this, finh, op_errno);
+    }
+
+    free_fuse_state(state);
+    STACK_DESTROY(frame->root);
+
+    return 0;
 }
 
 void
-fuse_statfs_resume (fuse_state_t *state)
+fuse_statfs_resume(fuse_state_t *state)
 {
-        if (!state->loc.inode) {
-                gf_log ("glusterfs-fuse", GF_LOG_WARNING,
-                        "%"PRIu64": STATFS (%s) resolution fail",
-                        state->finh->unique, uuid_utoa (state->resolve.gfid));
+    if (!state->loc.inode) {
+        gf_log("glusterfs-fuse", GF_LOG_WARNING,
+               "%" PRIu64 ": STATFS (%s) resolution fail", state->finh->unique,
+               uuid_utoa(state->resolve.gfid));
 
-                send_fuse_err (state->this, state->finh,
-                               state->resolve.op_errno);
-                free_fuse_state (state);
-                return;
-        }
+        /* facilitate retry from VFS */
+        if (state->resolve.op_errno == ENOENT)
+            state->resolve.op_errno = ESTALE;
 
-        gf_log ("glusterfs-fuse", GF_LOG_TRACE,
-                "%"PRIu64": STATFS", state->finh->unique);
+        send_fuse_err(state->this, state->finh, state->resolve.op_errno);
+        free_fuse_state(state);
+        return;
+    }
 
-        FUSE_FOP (state, fuse_statfs_cbk, GF_FOP_STATFS,
-                  statfs, &state->loc, state->xdata);
-}
+    gf_log("glusterfs-fuse", GF_LOG_TRACE, "%" PRIu64 ": STATFS",
+           state->finh->unique);
 
+    FUSE_FOP(state, fuse_statfs_cbk, GF_FOP_STATFS, statfs, &state->loc,
+             state->xdata);
+}
 
 static void
-fuse_statfs (xlator_t *this, fuse_in_header_t *finh, void *msg)
+fuse_statfs(xlator_t *this, fuse_in_header_t *finh, void *msg,
+            struct iobuf *iobuf)
 {
-        fuse_state_t *state = NULL;
+    fuse_state_t *state = NULL;
 
-        GET_STATE (this, finh, state);
+    GET_STATE(this, finh, state);
 
-        fuse_resolve_inode_init (state, &state->resolve, finh->nodeid);
+    fuse_resolve_inode_init(state, &state->resolve, finh->nodeid);
 
-        fuse_resolve_and_resume (state, fuse_statfs_resume);
+    fuse_resolve_and_resume(state, fuse_statfs_resume);
 }
 
-
 void
-fuse_setxattr_resume (fuse_state_t *state)
-{
-        if (!state->loc.inode) {
-                gf_log ("glusterfs-fuse", GF_LOG_WARNING,
-                        "%"PRIu64": SETXATTR %s/%"PRIu64" (%s) "
-                        "resolution failed",
-                        state->finh->unique, uuid_utoa (state->resolve.gfid),
-                        state->finh->nodeid, state->name);
-                send_fuse_err (state->this, state->finh,
-                               state->resolve.op_errno);
-                free_fuse_state (state);
-                return;
-        }
+fuse_setxattr_resume(fuse_state_t *state)
+{
+    if (!state->loc.inode) {
+        gf_log("glusterfs-fuse", GF_LOG_WARNING,
+               "%" PRIu64 ": SETXATTR %s/%" PRIu64
+               " (%s) "
+               "resolution failed",
+               state->finh->unique, uuid_utoa(state->resolve.gfid),
+               state->finh->nodeid, state->name);
+
+        /* facilitate retry from VFS */
+        if (state->resolve.op_errno == ENOENT)
+            state->resolve.op_errno = ESTALE;
+
+        send_fuse_err(state->this, state->finh, state->resolve.op_errno);
+        free_fuse_state(state);
+        return;
+    }
 
 #ifdef GF_TEST_FFOP
-        state->fd = fd_lookup (state->loc.inode, state->finh->pid);
+    state->fd = fd_lookup(state->loc.inode, state->finh->pid);
 #endif /* GF_TEST_FFOP */
 
-        if (state->fd) {
-                gf_log ("glusterfs-fuse", GF_LOG_TRACE,
-                        "%"PRIu64": SETXATTR %p/%"PRIu64" (%s)", state->finh->unique,
-                        state->fd, state->finh->nodeid, state->name);
+    if (state->fd) {
+        gf_log("glusterfs-fuse", GF_LOG_TRACE,
+               "%" PRIu64 ": SETXATTR %p/%" PRIu64 " (%s)", state->finh->unique,
+               state->fd, state->finh->nodeid, state->name);
 
-                FUSE_FOP (state, fuse_setxattr_cbk, GF_FOP_FSETXATTR,
-                          fsetxattr, state->fd, state->xattr, state->flags,
-                          state->xdata);
-        } else {
-                gf_log ("glusterfs-fuse", GF_LOG_TRACE,
-                        "%"PRIu64": SETXATTR %s/%"PRIu64" (%s)", state->finh->unique,
-                        state->loc.path, state->finh->nodeid, state->name);
+        FUSE_FOP(state, fuse_setxattr_cbk, GF_FOP_FSETXATTR, fsetxattr,
+                 state->fd, state->xattr, state->flags, state->xdata);
+    } else {
+        gf_log("glusterfs-fuse", GF_LOG_TRACE,
+               "%" PRIu64 ": SETXATTR %s/%" PRIu64 " (%s)", state->finh->unique,
+               state->loc.path, state->finh->nodeid, state->name);
 
-                FUSE_FOP (state, fuse_setxattr_cbk, GF_FOP_SETXATTR,
-                          setxattr, &state->loc, state->xattr, state->flags,
-                          state->xdata);
-        }
+        FUSE_FOP(state, fuse_setxattr_cbk, GF_FOP_SETXATTR, setxattr,
+                 &state->loc, state->xattr, state->flags, state->xdata);
+    }
 }
 
-
 static void
-fuse_setxattr (xlator_t *this, fuse_in_header_t *finh, void *msg)
+fuse_setxattr(xlator_t *this, fuse_in_header_t *finh, void *msg,
+              struct iobuf *iobuf)
 {
-        struct fuse_setxattr_in *fsi = msg;
-        char         *name = (char *)(fsi + 1);
-        char         *value = name + strlen (name) + 1;
-        struct fuse_private *priv = NULL;
+    struct fuse_setxattr_in *fsi = msg;
+    char *name = (char *)(fsi + 1);
+    char *value = name + strlen(name) + 1;
+    struct fuse_private *priv = NULL;
 
-        fuse_state_t *state = NULL;
-        char         *dict_value = NULL;
-        int32_t       ret = -1;
-        char *newkey = NULL;
+    fuse_state_t *state = NULL;
+    char *dict_value = NULL;
+    int32_t ret = -1;
+    int32_t op_errno = 0;
+    char *newkey = NULL;
 
-        priv = this->private;
+    priv = this->private;
 
-        GET_STATE (this, finh, state);
+    GET_STATE(this, finh, state);
 
 #ifdef GF_DARWIN_HOST_OS
-        if (fsi->position) {
-                gf_log ("glusterfs-fuse", GF_LOG_WARNING,
-                        "%"PRIu64": SETXATTR %s/%"PRIu64" (%s):"
-                        "refusing positioned setxattr",
-                        finh->unique, state->loc.path, finh->nodeid, name);
-                send_fuse_err (this, finh, EINVAL);
-                FREE (finh);
-                return;
-        }
+    if (fsi->position) {
+        gf_log("glusterfs-fuse", GF_LOG_WARNING,
+               "%" PRIu64 ": SETXATTR %s/%" PRIu64
+               " (%s):"
+               "refusing positioned setxattr",
+               finh->unique, state->loc.path, finh->nodeid, name);
+        op_errno = EINVAL;
+        goto done;
+    }
 #endif
 
-        if (fuse_ignore_xattr_set (priv, name)) {
-                (void) send_fuse_err (this, finh, 0);
-                return;
-        }
-
-        if (!priv->acl) {
-                if ((strcmp (name, POSIX_ACL_ACCESS_XATTR) == 0) ||
-                    (strcmp (name, POSIX_ACL_DEFAULT_XATTR) == 0)) {
-                        send_fuse_err (this, finh, EOPNOTSUPP);
-                        GF_FREE (finh);
-                        return;
-                }
-        }
-
-        if (!priv->selinux) {
-                if (strncmp (name, "security.", 9) == 0) {
-                        send_fuse_err (this, finh, EOPNOTSUPP);
-                        GF_FREE (finh);
-                        return;
-                }
-        }
-
-        /* Check if the command is for changing the log
-           level of process or specific xlator */
-        ret = is_gf_log_command (this, name, value);
-        if (ret >= 0) {
-                send_fuse_err (this, finh, ret);
-                GF_FREE (finh);
-                return;
-        }
-
-        if (!strcmp ("inode-invalidate", name)) {
-                gf_log ("fuse", GF_LOG_TRACE,
-                        "got request to invalidate %"PRIu64, finh->nodeid);
-                send_fuse_err (this, finh, 0);
+    if (fuse_ignore_xattr_set(priv, name)) {
+        goto done;
+    }
+
+    if (!priv->acl) {
+        if ((strcmp(name, POSIX_ACL_ACCESS_XATTR) == 0) ||
+            (strcmp(name, POSIX_ACL_DEFAULT_XATTR) == 0)) {
+            op_errno = EOPNOTSUPP;
+            goto done;
+        }
+    }
+
+    ret = fuse_check_selinux_cap_xattr(priv, name);
+    if (ret) {
+        op_errno = EOPNOTSUPP;
+        goto done;
+    }
+
+    /* Check if the command is for changing the log
+       level of process or specific xlator */
+    ret = is_gf_log_command(this, name, value, fsi->size);
+    if (ret >= 0) {
+        op_errno = ret;
+        goto done;
+    }
+
+    if (!strcmp("inode-invalidate", name)) {
+        gf_log("fuse", GF_LOG_TRACE, "got request to invalidate %" PRIu64,
+               finh->nodeid);
 #if FUSE_KERNEL_MINOR_VERSION >= 11
-                fuse_invalidate_entry (this, finh->nodeid);
+        ret = fuse_invalidate_entry(this, finh->nodeid);
+        if (ret)
+            op_errno = EBUSY;
 #endif
-                GF_FREE (finh);
-                return;
-        }
-
-        if (!strcmp (GFID_XATTR_KEY, name) || !strcmp (GF_XATTR_VOL_ID_KEY, name)) {
-                send_fuse_err (this, finh, EPERM);
-                GF_FREE (finh);
-                return;
-        }
-
-        state->size = fsi->size;
+        goto done;
+    }
 
-        fuse_resolve_inode_init (state, &state->resolve, finh->nodeid);
+    if (!strcmp(GFID_XATTR_KEY, name) || !strcmp(GF_XATTR_VOL_ID_KEY, name)) {
+        op_errno = EPERM;
+        goto done;
+    }
 
-        state->xattr = get_new_dict ();
-        if (!state->xattr) {
-                gf_log ("glusterfs-fuse", GF_LOG_ERROR,
-                        "%"PRIu64": SETXATTR dict allocation failed",
-                        finh->unique);
+    state->size = fsi->size;
 
-                send_fuse_err (this, finh, ENOMEM);
-                free_fuse_state (state);
-                return;
-        }
-
-        ret = fuse_flip_xattr_ns (priv, name, &newkey);
-        if (ret) {
-                send_fuse_err (this, finh, ENOMEM);
-                free_fuse_state (state);
-                return;
-        }
-
-        if (fsi->size > 0) {
-                dict_value = memdup (value, fsi->size);
-        }
-        dict_set (state->xattr, newkey,
-                  data_from_dynptr ((void *)dict_value, fsi->size));
-        dict_ref (state->xattr);
+    fuse_resolve_inode_init(state, &state->resolve, finh->nodeid);
 
-        state->flags = fsi->flags;
-        state->name = newkey;
+    state->xattr = dict_new();
+    if (!state->xattr) {
+        gf_log("glusterfs-fuse", GF_LOG_ERROR,
+               "%" PRIu64 ": SETXATTR dict allocation failed", finh->unique);
+        op_errno = ENOMEM;
+        goto done;
+    }
 
-        fuse_resolve_and_resume (state, fuse_setxattr_resume);
+    ret = fuse_flip_xattr_ns(priv, name, &newkey);
+    if (ret) {
+        op_errno = ENOMEM;
+        goto done;
+    }
 
-        return;
+    if (fsi->size > 0) {
+        /*
+         * Many translators expect setxattr values to be strings, but
+         * neither dict_get_str nor data_to_str do any checking or
+         * fixups to make sure that's the case.  To avoid nasty
+         * surprises, allocate an extra byte and add a NUL here.
+         */
+        dict_value = GF_MALLOC(fsi->size + 1, gf_common_mt_char);
+        if (dict_value == NULL) {
+            gf_log("glusterfs-fuse", GF_LOG_ERROR,
+                   "%" PRIu64 ": SETXATTR value allocation failed",
+                   finh->unique);
+            op_errno = ENOMEM;
+            GF_FREE(newkey);
+            goto done;
+        }
+        memcpy(dict_value, value, fsi->size);
+        dict_value[fsi->size] = '\0';
+    }
+    ret = dict_set_dynptr(state->xattr, newkey, dict_value, fsi->size);
+    if (ret < 0) {
+        op_errno = -ret;
+        GF_FREE(dict_value);
+        GF_FREE(newkey);
+        goto done;
+    }
+
+    state->flags = fsi->flags;
+    state->name = newkey;
+
+    fuse_resolve_and_resume(state, fuse_setxattr_resume);
+
+    return;
+
+done:
+    send_fuse_err(this, finh, op_errno);
+    free_fuse_state(state);
 }
 
-
 static void
-send_fuse_xattr (xlator_t *this, fuse_in_header_t *finh, const char *value,
-                 size_t size, size_t expected)
-{
-        struct fuse_getxattr_out fgxo;
-
-        /* linux kernel limits the size of xattr value to 64k */
-        if (size > GLUSTERFS_XATTR_LEN_MAX)
-                send_fuse_err (this, finh, E2BIG);
-        else if (expected) {
-                /* if callback for getxattr and asks for value */
-                if (size > expected)
-                        /* reply would be bigger than
-                         * what was asked by kernel */
-                        send_fuse_err (this, finh, ERANGE);
-                else
-                        send_fuse_data (this, finh, (void *)value, size);
-        } else {
-                fgxo.size = size;
-                send_fuse_obj (this, finh, &fgxo);
-        }
+send_fuse_xattr(xlator_t *this, fuse_in_header_t *finh, const char *value,
+                size_t size, size_t expected)
+{
+    struct fuse_getxattr_out fgxo;
+
+    /* linux kernel limits the size of xattr value to 64k */
+    if (size > GLUSTERFS_XATTR_LEN_MAX)
+        send_fuse_err(this, finh, E2BIG);
+    else if (expected) {
+        /* if callback for getxattr and asks for value */
+        if (size > expected)
+            /* reply would be bigger than
+             * what was asked by kernel */
+            send_fuse_err(this, finh, ERANGE);
+        else
+            send_fuse_data(this, finh, (void *)value, size);
+    } else {
+        fgxo.size = size;
+        send_fuse_obj(this, finh, &fgxo);
+    }
 }
 
 /* filter out xattrs that need not be visible on the
@@ -3240,2493 +4252,2997 @@ send_fuse_xattr (xlator_t *this, fuse_in_header_t *finh, const char *value,
 static int
 fuse_filter_xattr(char *key)
 {
-        int need_filter = 0;
-        struct fuse_private *priv = THIS->private;
+    int need_filter = 0;
+    struct fuse_private *priv = THIS->private;
 
-        if ((priv->client_pid == GF_CLIENT_PID_GSYNCD)
-            && fnmatch ("*.selinux*", key, FNM_PERIOD) == 0)
-                need_filter = 1;
+    if ((priv->client_pid == GF_CLIENT_PID_GSYNCD) &&
+        fnmatch("*.selinux*", key, FNM_PERIOD) == 0)
+        need_filter = 1;
 
-        return need_filter;
+    return need_filter;
 }
 
-
 static int
-fuse_xattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                int32_t op_ret, int32_t op_errno, dict_t *dict, dict_t *xdata)
-{
-        char           *value = "";
-        fuse_state_t   *state = NULL;
-        fuse_in_header_t *finh = NULL;
-        data_t         *value_data = NULL;
-        int             ret = -1;
-        int32_t         len = 0;
-        int32_t         len_next = 0;
-
-        state = frame->root->state;
-        finh  = state->finh;
-
-        fuse_log_eh_fop(this, state, frame, op_ret, op_errno);
-
-        if (op_ret >= 0) {
-                gf_log ("glusterfs-fuse", GF_LOG_TRACE,
-                        "%"PRIu64": %s() %s => %d", frame->root->unique,
-                        gf_fop_list[frame->root->op], state->loc.path, op_ret);
-
-                /* if successful */
-                if (state->name) {
-                        /* if callback for getxattr */
-                        value_data = dict_get (dict, state->name);
-                        if (value_data) {
-
-                                ret = value_data->len; /* Don't return the value for '\0' */
-                                value = value_data->data;
-
-                                send_fuse_xattr (this, finh, value, ret, state->size);
-                                /* if(ret >...)...else if...else */
-                        } else {
-                                send_fuse_err (this, finh, ENODATA);
-                        } /* if(value_data)...else */
-                } else {
-                        /* if callback for listxattr */
-                        /* we need to invoke fuse_filter_xattr() twice. Once
-                         * while counting size and then while filling buffer
-                         */
-                        len = dict_keys_join (NULL, 0, dict, fuse_filter_xattr);
-                        if (len < 0)
-                                goto out;
-
-                        value = alloca (len + 1);
-                        if (!value)
-                                goto out;
-
-                        len_next = dict_keys_join (value, len, dict,
-                                                   fuse_filter_xattr);
-                        if (len_next != len)
-                                gf_log (THIS->name, GF_LOG_ERROR,
-                                        "sizes not equal %d != %d",
-                                        len, len_next);
-
-                        send_fuse_xattr (this, finh, value, len, state->size);
-                } /* if(state->name)...else */
+fuse_xattr_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+               int32_t op_ret, int32_t op_errno, dict_t *dict, dict_t *xdata)
+{
+    char *value = "";
+    fuse_state_t *state = NULL;
+    fuse_in_header_t *finh = NULL;
+    data_t *value_data = NULL;
+    int ret = -1;
+    int32_t len = 0;
+    int32_t len_next = 0;
+
+    state = frame->root->state;
+    finh = state->finh;
+
+    fuse_log_eh_fop(this, state, frame, op_ret, op_errno);
+
+    if (op_ret >= 0) {
+        gf_log("glusterfs-fuse", GF_LOG_TRACE, "%" PRIu64 ": %s() %s => %d",
+               frame->root->unique, gf_fop_list[frame->root->op],
+               state->loc.path, op_ret);
+
+        /* if successful */
+        if (state->name) {
+            /* if callback for getxattr */
+            value_data = dict_get(dict, state->name);
+            if (value_data) {
+                ret = value_data->len; /* Don't return the value for '\0' */
+                value = value_data->data;
+
+                send_fuse_xattr(this, finh, value, ret, state->size);
+                /* if(ret >...)...else if...else */
+            } else {
+                send_fuse_err(this, finh, ENODATA);
+            } /* if(value_data)...else */
         } else {
-                /* if failure - no need to check if listxattr or getxattr */
-                if (op_errno != ENODATA && op_errno != ENOATTR) {
-                        if (op_errno == ENOTSUP) {
-                                GF_LOG_OCCASIONALLY (gf_fuse_xattr_enotsup_log,
-                                                     "glusterfs-fuse",
-                                                     GF_LOG_ERROR,
-                                                     "extended attribute not "
-                                                     "supported by the backend "
-                                                     "storage");
-                        } else {
-                                gf_log ("glusterfs-fuse", GF_LOG_WARNING,
-                                        "%"PRIu64": %s(%s) %s => -1 (%s)",
-                                        frame->root->unique,
-                                        gf_fop_list[frame->root->op], state->name,
-                                        state->loc.path, strerror (op_errno));
-                        }
-                } else {
-                        gf_log ("glusterfs-fuse", GF_LOG_DEBUG,
-                                "%"PRIu64": %s(%s) %s => -1 (%s)",
-                                frame->root->unique,
-                                gf_fop_list[frame->root->op], state->name,
-                                state->loc.path, strerror (op_errno));
-                } /* if(op_errno!= ENODATA)...else */
+            /* if callback for listxattr */
+            /* we need to invoke fuse_filter_xattr() twice. Once
+             * while counting size and then while filling buffer
+             */
+            len = dict_keys_join(NULL, 0, dict, fuse_filter_xattr);
+            if (len < 0)
+                goto out;
+
+            value = alloca(len + 1);
+            if (!value)
+                goto out;
+
+            len_next = dict_keys_join(value, len, dict, fuse_filter_xattr);
+            if (len_next != len)
+                gf_log(THIS->name, GF_LOG_ERROR, "sizes not equal %d != %d",
+                       len, len_next);
+
+            send_fuse_xattr(this, finh, value, len, state->size);
+        } /* if(state->name)...else */
+    } else {
+        /* facilitate retry from VFS */
+        if ((state->fd == NULL) && (op_errno == ENOENT)) {
+            op_errno = ESTALE;
+        }
+
+        /* if failure - no need to check if listxattr or getxattr */
+        if (op_errno != ENODATA && op_errno != ENOATTR) {
+            if (op_errno == ENOTSUP) {
+                GF_LOG_OCCASIONALLY(gf_fuse_xattr_enotsup_log, "glusterfs-fuse",
+                                    GF_LOG_ERROR,
+                                    "extended attribute not "
+                                    "supported by the backend "
+                                    "storage");
+            } else {
+                gf_log("glusterfs-fuse", GF_LOG_WARNING,
+                       "%" PRIu64 ": %s(%s) %s => -1 (%s)", frame->root->unique,
+                       gf_fop_list[frame->root->op], state->name,
+                       state->loc.path, strerror(op_errno));
+            }
+        } else {
+            gf_log("glusterfs-fuse", GF_LOG_DEBUG,
+                   "%" PRIu64 ": %s(%s) %s => -1 (%s)", frame->root->unique,
+                   gf_fop_list[frame->root->op], state->name, state->loc.path,
+                   strerror(op_errno));
+        } /* if(op_errno!= ENODATA)...else */
 
-                send_fuse_err (this, finh, op_errno);
-        } /* if(op_ret>=0)...else */
+        send_fuse_err(this, finh, op_errno);
+    } /* if(op_ret>=0)...else */
 
 out:
-        free_fuse_state (state);
-        STACK_DESTROY (frame->root);
+    free_fuse_state(state);
+    STACK_DESTROY(frame->root);
 
-        return 0;
+    return 0;
 }
 
-
 void
-fuse_getxattr_resume (fuse_state_t *state)
-{
-        char *value  = NULL;
-
-        if (!state->loc.inode) {
-                gf_log ("glusterfs-fuse", GF_LOG_WARNING,
-                        "%"PRIu64": GETXATTR %s/%"PRIu64" (%s) "
-                        "resolution failed",
-                        state->finh->unique,
-                        uuid_utoa (state->resolve.gfid),
-                        state->finh->nodeid, state->name);
-
-                send_fuse_err (state->this, state->finh,
-                               state->resolve.op_errno);
-                free_fuse_state (state);
-                return;
-        }
+fuse_getxattr_resume(fuse_state_t *state)
+{
+    char *value = NULL;
+
+    if (!state->loc.inode) {
+        gf_log("glusterfs-fuse", GF_LOG_WARNING,
+               "%" PRIu64 ": GETXATTR %s/%" PRIu64
+               " (%s) "
+               "resolution failed",
+               state->finh->unique, uuid_utoa(state->resolve.gfid),
+               state->finh->nodeid, state->name);
+
+        /* facilitate retry from VFS */
+        if (state->resolve.op_errno == ENOENT)
+            state->resolve.op_errno = ESTALE;
+
+        send_fuse_err(state->this, state->finh, state->resolve.op_errno);
+        free_fuse_state(state);
+        return;
+    }
 
 #ifdef GF_TEST_FFOP
-        state->fd = fd_lookup (state->loc.inode, state->finh->pid);
+    state->fd = fd_lookup(state->loc.inode, state->finh->pid);
 #endif /* GF_TEST_FFOP */
 
-        if (state->name &&
-            (strcmp (state->name, VIRTUAL_GFID_XATTR_KEY) == 0)) {
-                /* send glusterfs gfid in binary form */
-
-                value = GF_CALLOC (16 + 1, sizeof(char),
-                                   gf_common_mt_char);
-                if (!value) {
-                        send_fuse_err (state->this, state->finh, ENOMEM);
-                        goto internal_out;
-                }
-                memcpy (value, state->loc.inode->gfid, 16);
+    if (state->name && (strcmp(state->name, VIRTUAL_GFID_XATTR_KEY) == 0)) {
+        /* send glusterfs gfid in binary form */
 
-                send_fuse_xattr (THIS, state->finh, value, 16, state->size);
-                GF_FREE (value);
-        internal_out:
-                free_fuse_state (state);
-                return;
+        value = GF_MALLOC(16 + 1, gf_common_mt_char);
+        if (!value) {
+            send_fuse_err(state->this, state->finh, ENOMEM);
+            goto internal_out;
         }
+        memcpy(value, state->loc.inode->gfid, 16);
+        value[16] = '\0';
 
-        if (state->name &&
-            (strcmp (state->name, VIRTUAL_GFID_XATTR_KEY_STR) == 0)) {
-                /* transform binary gfid to canonical form */
+        send_fuse_xattr(THIS, state->finh, value, 16, state->size);
+        GF_FREE(value);
+    internal_out:
+        free_fuse_state(state);
+        return;
+    }
 
-                value = GF_CALLOC (UUID_CANONICAL_FORM_LEN + 1, sizeof(char),
-                                   gf_common_mt_char);
-                if (!value) {
-                        send_fuse_err (state->this, state->finh, ENOMEM);
-                        goto internal_out1;
-                }
-                uuid_utoa_r (state->loc.inode->gfid, value);
-
-                send_fuse_xattr (THIS, state->finh, value,
-                                 UUID_CANONICAL_FORM_LEN, state->size);
-                GF_FREE (value);
-        internal_out1:
-                free_fuse_state (state);
-                return;
+    if (state->name && (strcmp(state->name, VIRTUAL_GFID_XATTR_KEY_STR) == 0)) {
+        /* transform binary gfid to canonical form */
+
+        value = GF_CALLOC(UUID_CANONICAL_FORM_LEN + 1, sizeof(char),
+                          gf_common_mt_char);
+        if (!value) {
+            send_fuse_err(state->this, state->finh, ENOMEM);
+            goto internal_out1;
         }
+        uuid_utoa_r(state->loc.inode->gfid, value);
 
+        send_fuse_xattr(THIS, state->finh, value, UUID_CANONICAL_FORM_LEN,
+                        state->size);
+        GF_FREE(value);
+    internal_out1:
+        free_fuse_state(state);
+        return;
+    }
 
-        if (state->fd) {
-                gf_log ("glusterfs-fuse", GF_LOG_TRACE,
-                        "%"PRIu64": GETXATTR %p/%"PRIu64" (%s)", state->finh->unique,
-                        state->fd, state->finh->nodeid, state->name);
+    if (state->fd) {
+        gf_log("glusterfs-fuse", GF_LOG_TRACE,
+               "%" PRIu64 ": GETXATTR %p/%" PRIu64 " (%s)", state->finh->unique,
+               state->fd, state->finh->nodeid, state->name);
 
-                FUSE_FOP (state, fuse_xattr_cbk, GF_FOP_FGETXATTR,
-                          fgetxattr, state->fd, state->name, state->xdata);
-        } else {
-                gf_log ("glusterfs-fuse", GF_LOG_TRACE,
-                        "%"PRIu64": GETXATTR %s/%"PRIu64" (%s)", state->finh->unique,
-                        state->loc.path, state->finh->nodeid, state->name);
+        FUSE_FOP(state, fuse_xattr_cbk, GF_FOP_FGETXATTR, fgetxattr, state->fd,
+                 state->name, state->xdata);
+    } else {
+        gf_log("glusterfs-fuse", GF_LOG_TRACE,
+               "%" PRIu64 ": GETXATTR %s/%" PRIu64 " (%s)", state->finh->unique,
+               state->loc.path, state->finh->nodeid, state->name);
 
-                FUSE_FOP (state, fuse_xattr_cbk, GF_FOP_GETXATTR,
-                          getxattr, &state->loc, state->name, state->xdata);
-        }
+        FUSE_FOP(state, fuse_xattr_cbk, GF_FOP_GETXATTR, getxattr, &state->loc,
+                 state->name, state->xdata);
+    }
 }
 
-
 static void
-fuse_getxattr (xlator_t *this, fuse_in_header_t *finh, void *msg)
+fuse_getxattr(xlator_t *this, fuse_in_header_t *finh, void *msg,
+              struct iobuf *iobuf)
 {
-        struct fuse_getxattr_in *fgxi     = msg;
-        char                    *name     = (char *)(fgxi + 1);
-        fuse_state_t            *state    = NULL;
-        struct fuse_private     *priv     = NULL;
-        int                      rv       = 0;
-        int                     op_errno  = EINVAL;
-        char                    *newkey   = NULL;
+    struct fuse_getxattr_in *fgxi = msg;
+    char *name = (char *)(fgxi + 1);
+    fuse_state_t *state = NULL;
+    struct fuse_private *priv = NULL;
+    int rv = 0;
+    int op_errno = EINVAL;
+    char *newkey = NULL;
+    int ret = 0;
 
-        priv = this->private;
-        GET_STATE (this, finh, state);
+    priv = this->private;
+    GET_STATE(this, finh, state);
 
 #ifdef GF_DARWIN_HOST_OS
-        if (fgxi->position) {
-                /* position can be used only for
-                 * resource fork queries which we
-                 * don't support anyway... so handling
-                 * it separately is just sort of a
-                 * matter of aesthetics, not strictly
-                 * necessary.
-                 */
+    if (fgxi->position) {
+        /* position can be used only for
+         * resource fork queries which we
+         * don't support anyway... so handling
+         * it separately is just sort of a
+         * matter of aesthetics, not strictly
+         * necessary.
+         */
 
-                gf_log ("glusterfs-fuse", GF_LOG_WARNING,
-                        "%"PRIu64": GETXATTR %s/%"PRIu64" (%s):"
-                        "refusing positioned getxattr",
-                        finh->unique, state->loc.path, finh->nodeid, name);
-                op_errno = EINVAL;
-                goto err;
-        }
+        gf_log("glusterfs-fuse", GF_LOG_WARNING,
+               "%" PRIu64 ": GETXATTR %s/%" PRIu64
+               " (%s):"
+               "refusing positioned getxattr",
+               finh->unique, state->loc.path, finh->nodeid, name);
+        op_errno = EINVAL;
+        goto err;
+    }
 #endif
 
-        if (!priv->acl) {
-                if ((strcmp (name, POSIX_ACL_ACCESS_XATTR) == 0) ||
-                    (strcmp (name, POSIX_ACL_DEFAULT_XATTR) == 0)) {
-                        op_errno = ENOTSUP;
-                        goto err;
-                }
+    if (!priv->acl) {
+        if ((strcmp(name, POSIX_ACL_ACCESS_XATTR) == 0) ||
+            (strcmp(name, POSIX_ACL_DEFAULT_XATTR) == 0)) {
+            op_errno = ENOTSUP;
+            goto err;
         }
+    }
 
-        if (!priv->selinux) {
-                if (strncmp (name, "security.", 9) == 0) {
-                        op_errno = ENODATA;
-                        goto err;
-                }
-        }
+    ret = fuse_check_selinux_cap_xattr(priv, name);
+    if (ret) {
+        op_errno = ENODATA;
+        goto err;
+    }
 
-        fuse_resolve_inode_init (state, &state->resolve, finh->nodeid);
+    fuse_resolve_inode_init(state, &state->resolve, finh->nodeid);
 
-        rv = fuse_flip_xattr_ns (priv, name, &newkey);
-        if (rv) {
-                op_errno = ENOMEM;
-                goto err;
-        }
+    rv = fuse_flip_xattr_ns(priv, name, &newkey);
+    if (rv) {
+        op_errno = ENOMEM;
+        goto err;
+    }
 
-        state->size = fgxi->size;
-        state->name = newkey;
+    state->size = fgxi->size;
+    state->name = newkey;
 
-        fuse_resolve_and_resume (state, fuse_getxattr_resume);
+    fuse_resolve_and_resume(state, fuse_getxattr_resume);
 
-        return;
- err:
-        send_fuse_err (this, finh, op_errno);
-        free_fuse_state (state);
-        return;
+    return;
+err:
+    send_fuse_err(this, finh, op_errno);
+    free_fuse_state(state);
+    return;
 }
 
-
 void
-fuse_listxattr_resume (fuse_state_t *state)
-{
-        if (!state->loc.inode) {
-                gf_log ("glusterfs-fuse", GF_LOG_WARNING,
-                        "%"PRIu64": LISTXATTR %s/%"PRIu64
-                        "resolution failed", state->finh->unique,
-                        uuid_utoa (state->resolve.gfid), state->finh->nodeid);
-
-                send_fuse_err (state->this, state->finh,
-                               state->resolve.op_errno);
-                free_fuse_state (state);
-                return;
-        }
+fuse_listxattr_resume(fuse_state_t *state)
+{
+    if (!state->loc.inode) {
+        gf_log("glusterfs-fuse", GF_LOG_WARNING,
+               "%" PRIu64 ": LISTXATTR %s/%" PRIu64 "resolution failed",
+               state->finh->unique, uuid_utoa(state->resolve.gfid),
+               state->finh->nodeid);
+
+        /* facilitate retry from VFS */
+        if (state->resolve.op_errno == ENOENT)
+            state->resolve.op_errno = ESTALE;
+
+        send_fuse_err(state->this, state->finh, state->resolve.op_errno);
+        free_fuse_state(state);
+        return;
+    }
 
 #ifdef GF_TEST_FFOP
-        state->fd = fd_lookup (state->loc.inode, state->finh->pid);
+    state->fd = fd_lookup(state->loc.inode, state->finh->pid);
 #endif /* GF_TEST_FFOP */
 
-        if (state->fd) {
-                gf_log ("glusterfs-fuse", GF_LOG_TRACE,
-                        "%"PRIu64": LISTXATTR %p/%"PRIu64, state->finh->unique,
-                        state->fd, state->finh->nodeid);
+    if (state->fd) {
+        gf_log("glusterfs-fuse", GF_LOG_TRACE,
+               "%" PRIu64 ": LISTXATTR %p/%" PRIu64, state->finh->unique,
+               state->fd, state->finh->nodeid);
 
-                FUSE_FOP (state, fuse_xattr_cbk, GF_FOP_FGETXATTR,
-                          fgetxattr, state->fd, NULL, state->xdata);
-        } else {
-                gf_log ("glusterfs-fuse", GF_LOG_TRACE,
-                        "%"PRIu64": LISTXATTR %s/%"PRIu64, state->finh->unique,
-                        state->loc.path, state->finh->nodeid);
+        FUSE_FOP(state, fuse_xattr_cbk, GF_FOP_FGETXATTR, fgetxattr, state->fd,
+                 NULL, state->xdata);
+    } else {
+        gf_log("glusterfs-fuse", GF_LOG_TRACE,
+               "%" PRIu64 ": LISTXATTR %s/%" PRIu64, state->finh->unique,
+               state->loc.path, state->finh->nodeid);
 
-                FUSE_FOP (state, fuse_xattr_cbk, GF_FOP_GETXATTR,
-                          getxattr, &state->loc, NULL, state->xdata);
-        }
+        FUSE_FOP(state, fuse_xattr_cbk, GF_FOP_GETXATTR, getxattr, &state->loc,
+                 NULL, state->xdata);
+    }
 }
 
-
 static void
-fuse_listxattr (xlator_t *this, fuse_in_header_t *finh, void *msg)
+fuse_listxattr(xlator_t *this, fuse_in_header_t *finh, void *msg,
+               struct iobuf *iobuf)
 {
-        struct fuse_getxattr_in *fgxi = msg;
-        fuse_state_t *state = NULL;
+    struct fuse_getxattr_in *fgxi = msg;
+    fuse_state_t *state = NULL;
 
-        GET_STATE (this, finh, state);
+    GET_STATE(this, finh, state);
 
-        fuse_resolve_inode_init (state, &state->resolve, finh->nodeid);
+    fuse_resolve_inode_init(state, &state->resolve, finh->nodeid);
 
-        state->size = fgxi->size;
+    state->size = fgxi->size;
 
-        fuse_resolve_and_resume (state, fuse_listxattr_resume);
+    fuse_resolve_and_resume(state, fuse_listxattr_resume);
 
-        return;
+    return;
 }
 
-
 void
-fuse_removexattr_resume (fuse_state_t *state)
-{
-         if (!state->loc.inode) {
-                gf_log ("glusterfs-fuse", GF_LOG_DEBUG,
-                        "%"PRIu64": REMOVEXATTR %s/%"PRIu64" (%s) "
-                        "resolution failed",
-                        state->finh->unique, uuid_utoa (state->resolve.gfid),
-                        state->finh->nodeid, state->name);
-
-                send_fuse_err (state->this, state->finh,
-                               state->resolve.op_errno);
-                free_fuse_state (state);
-                return;
-        }
+fuse_removexattr_resume(fuse_state_t *state)
+{
+    if (!state->loc.inode) {
+        gf_log("glusterfs-fuse", GF_LOG_DEBUG,
+               "%" PRIu64 ": REMOVEXATTR %s/%" PRIu64
+               " (%s) "
+               "resolution failed",
+               state->finh->unique, uuid_utoa(state->resolve.gfid),
+               state->finh->nodeid, state->name);
+
+        /* facilitate retry from VFS */
+        if (state->resolve.op_errno == ENOENT)
+            state->resolve.op_errno = ESTALE;
+
+        send_fuse_err(state->this, state->finh, state->resolve.op_errno);
+        free_fuse_state(state);
+        return;
+    }
 
 #ifdef GF_TEST_FFOP
-        state->fd = fd_lookup (state->loc.inode, state->finh->pid);
+    state->fd = fd_lookup(state->loc.inode, state->finh->pid);
 #endif /* GF_TEST_FFOP */
 
-        if (state->fd) {
-                gf_log ("glusterfs-fuse", GF_LOG_TRACE,
-                        "%"PRIu64": REMOVEXATTR %p/%"PRIu64" (%s)", state->finh->unique,
-                        state->fd, state->finh->nodeid, state->name);
+    if (state->fd) {
+        gf_log("glusterfs-fuse", GF_LOG_TRACE,
+               "%" PRIu64 ": REMOVEXATTR %p/%" PRIu64 " (%s)",
+               state->finh->unique, state->fd, state->finh->nodeid,
+               state->name);
 
-                FUSE_FOP (state, fuse_err_cbk, GF_FOP_FREMOVEXATTR,
-                          fremovexattr, state->fd, state->name, state->xdata);
-        } else {
-                gf_log ("glusterfs-fuse", GF_LOG_TRACE,
-                        "%"PRIu64": REMOVEXATTR %s/%"PRIu64" (%s)", state->finh->unique,
-                        state->loc.path, state->finh->nodeid, state->name);
+        FUSE_FOP(state, fuse_removexattr_cbk, GF_FOP_FREMOVEXATTR, fremovexattr,
+                 state->fd, state->name, state->xdata);
+    } else {
+        gf_log("glusterfs-fuse", GF_LOG_TRACE,
+               "%" PRIu64 ": REMOVEXATTR %s/%" PRIu64 " (%s)",
+               state->finh->unique, state->loc.path, state->finh->nodeid,
+               state->name);
 
-                FUSE_FOP (state, fuse_err_cbk, GF_FOP_REMOVEXATTR,
-                          removexattr, &state->loc, state->name, state->xdata);
-        }
+        FUSE_FOP(state, fuse_removexattr_cbk, GF_FOP_REMOVEXATTR, removexattr,
+                 &state->loc, state->name, state->xdata);
+    }
 }
 
-
 static void
-fuse_removexattr (xlator_t *this, fuse_in_header_t *finh, void *msg)
+fuse_removexattr(xlator_t *this, fuse_in_header_t *finh, void *msg,
+                 struct iobuf *iobuf)
 {
-        char *name = msg;
+    char *name = msg;
 
-        fuse_state_t *state = NULL;
-        fuse_private_t *priv = NULL;
-        int32_t       ret = -1;
-        char *newkey = NULL;
+    fuse_state_t *state = NULL;
+    fuse_private_t *priv = NULL;
+    int32_t ret = -1;
+    char *newkey = NULL;
 
-        if (!strcmp (GFID_XATTR_KEY, name) || !strcmp (GF_XATTR_VOL_ID_KEY, name)) {
-                send_fuse_err (this, finh, EPERM);
-                GF_FREE (finh);
-                return;
-        }
+    if (!strcmp(GFID_XATTR_KEY, name) || !strcmp(GF_XATTR_VOL_ID_KEY, name)) {
+        send_fuse_err(this, finh, EPERM);
+        GF_FREE(finh);
+        return;
+    }
 
-        priv = this->private;
+    priv = this->private;
 
-        GET_STATE (this, finh, state);
+    GET_STATE(this, finh, state);
 
-        fuse_resolve_inode_init (state, &state->resolve, finh->nodeid);
+    fuse_resolve_inode_init(state, &state->resolve, finh->nodeid);
 
-        ret = fuse_flip_xattr_ns (priv, name, &newkey);
-        if (ret) {
-                send_fuse_err (this, finh, ENOMEM);
-                free_fuse_state (state);
-                return;
-        }
+    ret = fuse_flip_xattr_ns(priv, name, &newkey);
+    if (ret) {
+        send_fuse_err(this, finh, ENOMEM);
+        free_fuse_state(state);
+        return;
+    }
 
-        state->name = newkey;
+    state->name = newkey;
 
-        fuse_resolve_and_resume (state, fuse_removexattr_resume);
-        return;
+    fuse_resolve_and_resume(state, fuse_removexattr_resume);
+    return;
 }
 
-
 static int gf_fuse_lk_enosys_log;
 
 static int
-fuse_getlk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                int32_t op_ret, int32_t op_errno, struct gf_flock *lock,
-                dict_t *xdata)
+fuse_getlk_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+               int32_t op_ret, int32_t op_errno, struct gf_flock *lock,
+               dict_t *xdata)
 {
-        fuse_state_t *state = NULL;
-
-        state = frame->root->state;
-        struct fuse_lk_out flo = {{0, }, };
-
-        fuse_log_eh_fop(this, state, frame, op_ret, op_errno);
-
-        if (op_ret == 0) {
-                gf_log ("glusterfs-fuse", GF_LOG_TRACE,
-                        "%"PRIu64": ERR => 0", frame->root->unique);
-                flo.lk.type = lock->l_type;
-                flo.lk.pid  = lock->l_pid;
-                if (lock->l_type == F_UNLCK)
-                        flo.lk.start = flo.lk.end = 0;
-                else {
-                        flo.lk.start = lock->l_start;
-                        flo.lk.end = lock->l_len ?
-                                     (lock->l_start + lock->l_len - 1) :
-                                     OFFSET_MAX;
-                }
-                send_fuse_obj (this, state->finh, &flo);
+    fuse_state_t *state = NULL;
+
+    state = frame->root->state;
+    struct fuse_lk_out flo = {
+        {
+            0,
+        },
+    };
+
+    fuse_log_eh_fop(this, state, frame, op_ret, op_errno);
+
+    if (op_ret == 0) {
+        gf_log("glusterfs-fuse", GF_LOG_TRACE, "%" PRIu64 ": ERR => 0",
+               frame->root->unique);
+        flo.lk.type = lock->l_type;
+        flo.lk.pid = lock->l_pid;
+        if (lock->l_type == F_UNLCK)
+            flo.lk.start = flo.lk.end = 0;
+        else {
+            flo.lk.start = lock->l_start;
+            flo.lk.end = lock->l_len ? (lock->l_start + lock->l_len - 1)
+                                     : OFFSET_MAX;
+        }
+        send_fuse_obj(this, state->finh, &flo);
+    } else {
+        if (op_errno == ENOSYS) {
+            gf_fuse_lk_enosys_log++;
+            if (!(gf_fuse_lk_enosys_log % GF_UNIVERSAL_ANSWER)) {
+                gf_log("glusterfs-fuse", GF_LOG_ERROR,
+                       "GETLK not supported. loading "
+                       "'features/posix-locks' on server side "
+                       "will add GETLK support.");
+            }
         } else {
-                if (op_errno == ENOSYS) {
-                        gf_fuse_lk_enosys_log++;
-                        if (!(gf_fuse_lk_enosys_log % GF_UNIVERSAL_ANSWER)) {
-                                gf_log ("glusterfs-fuse", GF_LOG_ERROR,
-                                        "GETLK not supported. loading "
-                                        "'features/posix-locks' on server side "
-                                        "will add GETLK support.");
-                        }
-                } else {
-                        gf_log ("glusterfs-fuse", GF_LOG_WARNING,
-                                "%"PRIu64": ERR => -1 (%s)",
-                                frame->root->unique, strerror (op_errno));
-                }
-                send_fuse_err (this, state->finh, op_errno);
+            gf_log("glusterfs-fuse", GF_LOG_WARNING,
+                   "%" PRIu64 ": ERR => -1 (%s)", frame->root->unique,
+                   strerror(op_errno));
         }
+        send_fuse_err(this, state->finh, op_errno);
+    }
 
-        free_fuse_state (state);
-        STACK_DESTROY (frame->root);
+    free_fuse_state(state);
+    STACK_DESTROY(frame->root);
 
-        return 0;
+    return 0;
 }
 
-
 void
-fuse_getlk_resume (fuse_state_t *state)
+fuse_getlk_resume(fuse_state_t *state)
 {
-        gf_log ("glusterfs-fuse", GF_LOG_TRACE,
-                "%"PRIu64": GETLK %p", state->finh->unique, state->fd);
+    gf_log("glusterfs-fuse", GF_LOG_TRACE, "%" PRIu64 ": GETLK %p",
+           state->finh->unique, state->fd);
 
-        FUSE_FOP (state, fuse_getlk_cbk, GF_FOP_LK,
-                  lk, state->fd, F_GETLK, &state->lk_lock, state->xdata);
+    FUSE_FOP(state, fuse_getlk_cbk, GF_FOP_LK, lk, state->fd, F_GETLK,
+             &state->lk_lock, state->xdata);
 }
 
-
 static void
-fuse_getlk (xlator_t *this, fuse_in_header_t *finh, void *msg)
+fuse_getlk(xlator_t *this, fuse_in_header_t *finh, void *msg,
+           struct iobuf *iobuf)
 {
-        struct fuse_lk_in *fli = msg;
+    struct fuse_lk_in *fli = msg;
 
-        fuse_state_t *state = NULL;
-        fd_t         *fd = NULL;
+    fuse_state_t *state = NULL;
+    fd_t *fd = NULL;
 
-        fd = FH_TO_FD (fli->fh);
-        GET_STATE (this, finh, state);
-        state->fd = fd;
+    fd = FH_TO_FD(fli->fh);
+    GET_STATE(this, finh, state);
+    state->fd = fd;
 
-        fuse_resolve_fd_init (state, &state->resolve, fd);
+    fuse_resolve_fd_init(state, &state->resolve, fd);
 
-        convert_fuse_file_lock (&fli->lk, &state->lk_lock,
-                                fli->owner);
+    convert_fuse_file_lock(&fli->lk, &state->lk_lock, fli->owner);
 
-        state->lk_owner = fli->owner;
+    state->lk_owner = fli->owner;
 
-        fuse_resolve_and_resume (state, fuse_getlk_resume);
+    fuse_resolve_and_resume(state, fuse_getlk_resume);
 
-        return;
+    return;
 }
 
-
 static int
-fuse_setlk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                int32_t op_ret, int32_t op_errno, struct gf_flock *lock,
-                dict_t *xdata)
-{
-        uint32_t      op    = 0;
-        fuse_state_t *state = NULL;
+fuse_setlk_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+               int32_t op_ret, int32_t op_errno, struct gf_flock *lock,
+               dict_t *xdata)
+{
+    uint32_t op = 0;
+    fuse_state_t *state = NULL;
+    int ret = 0;
+
+    ret = fuse_interrupt_finish_fop(frame, this, _gf_true, (void **)&state);
+    GF_FREE(state->name);
+    dict_unref(state->xdata);
+    GF_FREE(state);
+    if (ret) {
+        return 0;
+    }
+
+    state = frame->root->state;
+    op = state->finh->opcode;
+
+    fuse_log_eh_fop(this, state, frame, op_ret, op_errno);
+
+    if (op_ret == 0) {
+        gf_log("glusterfs-fuse", GF_LOG_TRACE, "%" PRIu64 ": ERR => 0",
+               frame->root->unique);
+        fd_lk_insert_and_merge(state->fd,
+                               (op == FUSE_SETLK) ? F_SETLK : F_SETLKW,
+                               &state->lk_lock);
+
+        send_fuse_err(this, state->finh, 0);
+    } else {
+        if (op_errno == ENOSYS) {
+            gf_fuse_lk_enosys_log++;
+            if (!(gf_fuse_lk_enosys_log % GF_UNIVERSAL_ANSWER)) {
+                gf_log("glusterfs-fuse", GF_LOG_ERROR,
+                       "SETLK not supported. loading "
+                       "'features/posix-locks' on server side "
+                       "will add SETLK support.");
+            }
+        } else if (op_errno == EAGAIN) {
+            gf_log("glusterfs-fuse", GF_LOG_DEBUG,
+                   "Returning EAGAIN Flock: "
+                   "start=%llu, len=%llu, pid=%llu, lk-owner=%s",
+                   (unsigned long long)state->lk_lock.l_start,
+                   (unsigned long long)state->lk_lock.l_len,
+                   (unsigned long long)state->lk_lock.l_pid,
+                   lkowner_utoa(&frame->root->lk_owner));
+        } else {
+            gf_log("glusterfs-fuse", GF_LOG_WARNING,
+                   "%" PRIu64 ": ERR => -1 (%s)", frame->root->unique,
+                   strerror(op_errno));
+        }
 
-        state = frame->root->state;
-        op    = state->finh->opcode;
+        send_fuse_err(this, state->finh, op_errno);
+    }
 
-        fuse_log_eh_fop(this, state, frame, op_ret, op_errno);
+    free_fuse_state(state);
+    STACK_DESTROY(frame->root);
 
-        if (op_ret == 0) {
-                gf_log ("glusterfs-fuse", GF_LOG_TRACE,
-                        "%"PRIu64": ERR => 0", frame->root->unique);
-                fd_lk_insert_and_merge (state->fd,
-                                        (op == FUSE_SETLK) ? F_SETLK : F_SETLKW,
-                                        &state->lk_lock);
+    return 0;
+}
 
-                send_fuse_err (this, state->finh, 0);
-        } else {
-                if (op_errno == ENOSYS) {
-                        gf_fuse_lk_enosys_log++;
-                        if (!(gf_fuse_lk_enosys_log % GF_UNIVERSAL_ANSWER)) {
-                                gf_log ("glusterfs-fuse", GF_LOG_ERROR,
-                                        "SETLK not supported. loading "
-                                        "'features/posix-locks' on server side "
-                                        "will add SETLK support.");
-                        }
-                } else if (op_errno == EAGAIN) {
-                        gf_log ("glusterfs-fuse", GF_LOG_DEBUG,
-                                "Returning EAGAIN Flock: "
-                                "start=%llu, len=%llu, pid=%llu, lk-owner=%s",
-                                (unsigned long long) state->lk_lock.l_start,
-                                (unsigned long long) state->lk_lock.l_len,
-                                (unsigned long long) state->lk_lock.l_pid,
-                                lkowner_utoa (&frame->root->lk_owner));
-                } else  {
-                        gf_log ("glusterfs-fuse", GF_LOG_WARNING,
-                                "%"PRIu64": ERR => -1 (%s)",
-                                frame->root->unique, strerror (op_errno));
-                }
+static int
+fuse_setlk_interrupt_handler_cbk(call_frame_t *frame, void *cookie,
+                                 xlator_t *this, int32_t op_ret,
+                                 int32_t op_errno, dict_t *dict, dict_t *xdata)
+{
+    fuse_interrupt_state_t intstat = INTERRUPT_NONE;
+    fuse_interrupt_record_t *fir = cookie;
 
-                send_fuse_err (this, state->finh, op_errno);
-        }
+    intstat = op_ret >= 0 ? INTERRUPT_HANDLED : INTERRUPT_SQUELCHED;
 
-        free_fuse_state (state);
-        STACK_DESTROY (frame->root);
+    fuse_interrupt_finish_interrupt(this, fir, intstat, _gf_true, NULL);
 
-        return 0;
+    STACK_DESTROY(frame->root);
+
+    return 0;
 }
 
+static void
+fuse_setlk_interrupt_handler(xlator_t *this, fuse_interrupt_record_t *fir)
+{
+    fuse_state_t *state = NULL;
+    call_frame_t *frame = NULL;
+    char *xattr_name = NULL;
+    int ret = 0;
+
+    gf_log("glusterfs-fuse", GF_LOG_DEBUG,
+           "SETLK%s unique %" PRIu64 ": interrupt handler triggered",
+           fir->fuse_in_header.opcode == FUSE_SETLK ? "" : "W",
+           fir->fuse_in_header.unique);
+
+    state = fir->data;
+
+    ret = gf_asprintf(
+        &xattr_name, GF_XATTR_CLRLK_CMD ".tposix.kblocked.%hd,%jd-%jd",
+        state->lk_lock.l_whence, state->lk_lock.l_start, state->lk_lock.l_len);
+    if (ret == -1) {
+        xattr_name = NULL;
+        goto err;
+    }
+
+    frame = get_call_frame_for_req(state);
+    if (!frame) {
+        goto err;
+    }
+    frame->root->state = state;
+    frame->root->op = GF_FOP_GETXATTR;
+    frame->op = GF_FOP_GETXATTR;
+    state->name = xattr_name;
+
+    STACK_WIND_COOKIE(frame, fuse_setlk_interrupt_handler_cbk, fir,
+                      state->active_subvol,
+                      state->active_subvol->fops->fgetxattr, state->fd,
+                      xattr_name, state->xdata);
+
+    return;
+
+err:
+    GF_FREE(xattr_name);
+    fuse_interrupt_finish_interrupt(this, fir, INTERRUPT_SQUELCHED, _gf_false,
+                                    (void **)&state);
+    if (state) {
+        dict_unref(state->xdata);
+        GF_FREE(state);
+    }
+}
 
 void
-fuse_setlk_resume (fuse_state_t *state)
+fuse_setlk_resume(fuse_state_t *state)
 {
-        gf_log ("glusterfs-fuse", GF_LOG_TRACE,
-                "%"PRIu64": SETLK%s %p", state->finh->unique,
-                state->finh->opcode == FUSE_SETLK ? "" : "W", state->fd);
+    fuse_interrupt_record_t *fir = NULL;
+    fuse_state_t *state_clone = NULL;
 
-        FUSE_FOP (state, fuse_setlk_cbk, GF_FOP_LK, lk, state->fd,
-                  state->finh->opcode == FUSE_SETLK ? F_SETLK : F_SETLKW,
-                  &state->lk_lock, state->xdata);
-}
+    fir = fuse_interrupt_record_new(state->finh, fuse_setlk_interrupt_handler);
+    state_clone = gf_memdup(state, sizeof(*state));
+    if (state_clone) {
+        state_clone->xdata = dict_new();
+    }
+
+    if (!fir || !state_clone || !state_clone->xdata) {
+        if (fir) {
+            GF_FREE(fir);
+        }
+        if (state_clone) {
+            GF_FREE(state_clone);
+        }
+        send_fuse_err(state->this, state->finh, ENOMEM);
 
+        gf_log("glusterfs-fuse", GF_LOG_ERROR,
+               "SETLK%s unique %" PRIu64
+               ":"
+               " interrupt record allocation failed",
+               state->finh->opcode == FUSE_SETLK ? "" : "W",
+               state->finh->unique);
+        free_fuse_state(state);
+
+        return;
+    }
+    state_clone->name = NULL;
+    fir->data = state_clone;
+    fuse_interrupt_record_insert(state->this, fir);
+
+    gf_log("glusterfs-fuse", GF_LOG_TRACE, "%" PRIu64 ": SETLK%s %p",
+           state->finh->unique, state->finh->opcode == FUSE_SETLK ? "" : "W",
+           state->fd);
+
+    FUSE_FOP(state, fuse_setlk_cbk, GF_FOP_LK, lk, state->fd,
+             state->finh->opcode == FUSE_SETLK ? F_SETLK : F_SETLKW,
+             &state->lk_lock, state->xdata);
+}
 
 static void
-fuse_setlk (xlator_t *this, fuse_in_header_t *finh, void *msg)
+fuse_setlk(xlator_t *this, fuse_in_header_t *finh, void *msg,
+           struct iobuf *iobuf)
 {
-        struct fuse_lk_in *fli = msg;
+    struct fuse_lk_in *fli = msg;
 
-        fuse_state_t *state = NULL;
-        fd_t         *fd = NULL;
+    fuse_state_t *state = NULL;
+    fd_t *fd = NULL;
 
-        fd = FH_TO_FD (fli->fh);
-        GET_STATE (this, finh, state);
-        state->finh = finh;
-        state->fd = fd;
+    fd = FH_TO_FD(fli->fh);
+    GET_STATE(this, finh, state);
+    state->finh = finh;
+    state->fd = fd;
 
-        fuse_resolve_fd_init (state, &state->resolve, fd);
+    fuse_resolve_fd_init(state, &state->resolve, fd);
 
-        convert_fuse_file_lock (&fli->lk, &state->lk_lock,
-                                fli->owner);
+    convert_fuse_file_lock(&fli->lk, &state->lk_lock, fli->owner);
 
-        state->lk_owner = fli->owner;
+    state->lk_owner = fli->owner;
 
-        fuse_resolve_and_resume (state, fuse_setlk_resume);
+    fuse_resolve_and_resume(state, fuse_setlk_resume);
 
-        return;
+    return;
 }
 
-#if FUSE_KERNEL_MINOR_VERSION >= 11
+#if FUSE_KERNEL_MINOR_VERSION >= 11 && defined(HAVE_FUSE_NOTIFICATIONS)
 static void *
-notify_kernel_loop (void *data)
-{
-        xlator_t               *this = NULL;
-        fuse_private_t         *priv = NULL;
-        struct fuse_out_header *fouh = NULL;
-        int                     rv   = 0;
-
-        char inval_buf[INVAL_BUF_SIZE] = {0,};
+notify_kernel_loop(void *data)
+{
+    uint32_t len = 0;
+    ssize_t rv = 0;
+    xlator_t *this = NULL;
+    fuse_private_t *priv = NULL;
+    fuse_invalidate_node_t *node = NULL;
+    fuse_invalidate_node_t *tmp = NULL;
+    struct fuse_out_header *pfoh = NULL;
+    struct iovec iov_out = {
+        0,
+    };
+
+    this = data;
+    priv = this->private;
+
+    for (;;) {
+        pthread_mutex_lock(&priv->invalidate_mutex);
+        {
+            while (list_empty(&priv->invalidate_list))
+                pthread_cond_wait(&priv->invalidate_cond,
+                                  &priv->invalidate_mutex);
 
-        this = data;
-        priv = this->private;
+            node = list_entry(priv->invalidate_list.next,
+                              fuse_invalidate_node_t, next);
 
-        for (;;) {
-                rv = read (priv->revchan_in, inval_buf, sizeof (*fouh));
-                if (rv != sizeof (*fouh))
-                        break;
-                fouh = (struct fuse_out_header *)inval_buf;
-                rv = read (priv->revchan_in, inval_buf + sizeof (*fouh),
-                           fouh->len - sizeof (*fouh));
-                if (rv != fouh->len - sizeof (*fouh))
-                        break;
-                rv = write (priv->fd, inval_buf, fouh->len);
-                if (rv != fouh->len && !(rv == -1 && errno == ENOENT))
-                        break;
+            list_del_init(&node->next);
+            priv->invalidate_count--;
         }
+        pthread_mutex_unlock(&priv->invalidate_mutex);
 
-        close (priv->revchan_in);
-        close (priv->revchan_out);
-
-        gf_log ("glusterfs-fuse", GF_LOG_INFO,
-                "kernel notifier loop terminated");
-
-        return NULL;
-}
-#endif
+        pfoh = (struct fuse_out_header *)node->inval_buf;
+        memcpy(&len, &pfoh->len, sizeof(len));
+        /*
+         * a simple
+         *         len = pfoh->len;
+         * works on x86, but takes a multiple insn cycle hit
+         * when pfoh->len is not correctly aligned, possibly
+         * even stalling the insn pipeline.
+         * Other architectures will not be so forgiving. If
+         * we're lucky the memcpy will be inlined by the
+         * compiler, and might be as fast or faster without
+         * the risk of stalling the insn pipeline.
+         */
 
-static void
-fuse_init (xlator_t *this, fuse_in_header_t *finh, void *msg)
-{
-        struct fuse_init_in  *fini      = msg;
-        struct fuse_init_out  fino      = {0,};
-        fuse_private_t       *priv      = NULL;
-        int                   ret       = 0;
-#if FUSE_KERNEL_MINOR_VERSION >= 9
-        int                   pfd[2]    = {0,};
-        pthread_t             messenger;
-#endif
+        iov_out.iov_base = node->inval_buf;
+        iov_out.iov_len = len;
+        rv = sys_writev(priv->fd, &iov_out, 1);
+        check_and_dump_fuse_W(priv, &iov_out, 1, rv, node->errnomask);
 
-        priv = this->private;
+        GF_FREE(node);
 
-        if (priv->init_recvd) {
-                gf_log ("glusterfs-fuse", GF_LOG_ERROR,
-                        "got INIT after first message");
+        if (rv == -1 && errno == EBADF)
+            break;
 
-                close (priv->fd);
-                goto out;
+        if (rv != len && !(rv == -1 && errno == ENOENT)) {
+            gf_log("glusterfs-fuse", GF_LOG_INFO, "len: %u, rv: %zd, errno: %d",
+                   len, rv, errno);
         }
+    }
 
-        priv->init_recvd = 1;
-
-        if (fini->major != FUSE_KERNEL_VERSION) {
-                gf_log ("glusterfs-fuse", GF_LOG_ERROR,
-                        "unsupported FUSE protocol version %d.%d",
-                        fini->major, fini->minor);
+    gf_log("glusterfs-fuse", GF_LOG_ERROR, "kernel notifier loop terminated");
 
-                close (priv->fd);
-                goto out;
+    pthread_mutex_lock(&priv->invalidate_mutex);
+    {
+        priv->reverse_fuse_thread_started = _gf_false;
+        list_for_each_entry_safe(node, tmp, &priv->invalidate_list, next)
+        {
+            list_del_init(&node->next);
+            GF_FREE(node);
         }
-        priv->proto_minor = fini->minor;
+        priv->invalidate_count = 0;
+    }
+    pthread_mutex_unlock(&priv->invalidate_mutex);
 
-        fino.major = FUSE_KERNEL_VERSION;
-        fino.minor = FUSE_KERNEL_MINOR_VERSION;
-        fino.max_readahead = 1 << 17;
-        fino.max_write = 1 << 17;
-        fino.flags = FUSE_ASYNC_READ | FUSE_POSIX_LOCKS;
-#if FUSE_KERNEL_MINOR_VERSION >= 17
-	if (fini->minor >= 17)
-		fino.flags |= FUSE_FLOCK_LOCKS;
-#endif
-#if FUSE_KERNEL_MINOR_VERSION >= 12
-        if (fini->minor >= 12) {
-            /* let fuse leave the umask processing to us, so that it does not
-             * break extended POSIX ACL defaults on server */
-            fino.flags |= FUSE_DONT_MASK;
-        }
+    return NULL;
+}
 #endif
-#if FUSE_KERNEL_MINOR_VERSION >= 9
-        if (fini->minor >= 6 /* fuse_init_in has flags */ &&
-            fini->flags & FUSE_BIG_WRITES) {
-                /* no need for direct I/O mode by default if big writes are supported */
-                if (priv->direct_io_mode == 2)
-                        priv->direct_io_mode = 0;
-                fino.flags |= FUSE_BIG_WRITES;
-        }
 
-        /* Used for 'reverse invalidation of inode' */
-        if (fini->minor >= 12) {
-                if (pipe(pfd) == -1) {
-                        gf_log ("glusterfs-fuse", GF_LOG_ERROR,
-                                "cannot create pipe pair (%s)",
-                                strerror(errno));
+static void *
+timed_response_loop(void *data)
+{
+    ssize_t rv = 0;
+    size_t len = 0;
+    xlator_t *this = NULL;
+    fuse_private_t *priv = NULL;
+    fuse_timed_message_t *dmsg = NULL;
+    fuse_timed_message_t *tmp = NULL;
+    struct timespec now = {
+        0,
+    };
+    struct timespec delta = {
+        0,
+    };
+    struct iovec iovs[2] = {
+        {
+            0,
+        },
+    };
 
-                        close (priv->fd);
-                        goto out;
-                }
-                priv->revchan_in  = pfd[0];
-                priv->revchan_out = pfd[1];
-                ret = gf_thread_create (&messenger, NULL, notify_kernel_loop,
-					this);
-                if (ret != 0) {
-                        gf_log ("glusterfs-fuse", GF_LOG_ERROR,
-                                "failed to start messenger daemon (%s)",
-                                strerror(errno));
-
-                        close (priv->fd);
-                        goto out;
-                }
-                priv->reverse_fuse_thread_started = _gf_true;
-        } else {
-                /*
-                 * FUSE minor < 12 does not implement invalidate notifications.
-                 * This mechanism is required for fopen-keep-cache to operate
-                 * correctly. Disable and warn the user.
-                 */
-                if (priv->fopen_keep_cache) {
-                        gf_log("glusterfs-fuse", GF_LOG_WARNING, "FUSE version "
-                                "%d.%d does not support inval notifications. "
-                                "fopen-keep-cache disabled.", fini->major,
-                                fini->minor);
-                        priv->fopen_keep_cache = 0;
+    this = data;
+    priv = this->private;
+
+    for (;;) {
+        pthread_mutex_lock(&priv->timed_mutex);
+        {
+            while (list_empty(&priv->timed_list)) {
+                pthread_cond_wait(&priv->timed_cond, &priv->timed_mutex);
+            }
+
+            dmsg = list_entry(priv->timed_list.next, fuse_timed_message_t,
+                              next);
+            list_for_each_entry(tmp, &priv->timed_list, next)
+            {
+                if (timespec_cmp(&tmp->scheduled_ts, &dmsg->scheduled_ts) < 0) {
+                    dmsg = tmp;
                 }
-        }
+            }
 
-        if (fini->minor >= 13) {
-                fino.max_background = priv->background_qlen;
-                fino.congestion_threshold = priv->congestion_threshold;
+            list_del_init(&dmsg->next);
         }
-        if (fini->minor < 9)
-                *priv->msg0_len_p = sizeof(*finh) + FUSE_COMPAT_WRITE_IN_SIZE;
+        pthread_mutex_unlock(&priv->timed_mutex);
 
-        if (priv->use_readdirp) {
-                if (fini->flags & FUSE_DO_READDIRPLUS)
-                        fino.flags |= FUSE_DO_READDIRPLUS;
+        timespec_now(&now);
+        if (timespec_cmp(&now, &dmsg->scheduled_ts) < 0) {
+            timespec_sub(&now, &dmsg->scheduled_ts, &delta);
+            nanosleep(&delta, NULL);
         }
-#endif
-	if (priv->fopen_keep_cache == 2) {
-		/* If user did not explicitly set --fopen-keep-cache[=off],
-		   then check if kernel support FUSE_AUTO_INVAL_DATA and ...
-		*/
-#if FUSE_KERNEL_MINOR_VERSION >= 20
-		if (fini->flags & FUSE_AUTO_INVAL_DATA) {
-			/* ... enable fopen_keep_cache mode if supported.
-			*/
-			gf_log ("glusterfs-fuse", GF_LOG_DEBUG, "Detected "
-				"support for FUSE_AUTO_INVAL_DATA. Enabling "
-				"fopen_keep_cache automatically.");
-			fino.flags |= FUSE_AUTO_INVAL_DATA;
-			priv->fopen_keep_cache = 1;
-		} else
-#endif
-		{
-
-			gf_log ("glusterfs-fuse", GF_LOG_DEBUG, "No support "
-				"for FUSE_AUTO_INVAL_DATA. Disabling "
-				"fopen_keep_cache.");
-			/* ... else disable. */
-			priv->fopen_keep_cache = 0;
-		}
-	} else if (priv->fopen_keep_cache == 1) {
-		/* If user explicitly set --fopen-keep-cache[=on],
-		   then enable FUSE_AUTO_INVAL_DATA if possible.
-		*/
-#if FUSE_KERNEL_MINOR_VERSION >= 20
-		if (fini->flags & FUSE_AUTO_INVAL_DATA) {
-			gf_log ("glusterfs-fuse", GF_LOG_DEBUG, "fopen_keep_cache "
-				"is explicitly set. Enabling FUSE_AUTO_INVAL_DATA");
-			fino.flags |= FUSE_AUTO_INVAL_DATA;
-		} else
-#endif
-		{
-			gf_log ("glusterfs-fuse", GF_LOG_WARNING, "fopen_keep_cache "
-				"is explicitly set. Support for "
-				"FUSE_AUTO_INVAL_DATA is missing");
-		}
-	}
 
-#if FUSE_KERNEL_MINOR_VERSION >= 22
-	if (fini->flags & FUSE_ASYNC_DIO)
-		fino.flags |= FUSE_ASYNC_DIO;
-#endif
-        ret = send_fuse_obj (this, finh, &fino);
-        if (ret == 0)
-                gf_log ("glusterfs-fuse", GF_LOG_INFO,
-                        "FUSE inited with protocol versions:"
-                        " glusterfs %d.%d kernel %d.%d",
-                        FUSE_KERNEL_VERSION, FUSE_KERNEL_MINOR_VERSION,
-                        fini->major, fini->minor);
-        else {
-                gf_log ("glusterfs-fuse", GF_LOG_ERROR,
-                        "FUSE init failed (%s)", strerror (ret));
+        gf_log("glusterfs-fuse", GF_LOG_TRACE,
+               "sending timed "
+               "message of unique %" PRIu64,
+               dmsg->fuse_out_header.unique);
 
-                close (priv->fd);
+        len = dmsg->fuse_out_header.len;
+        iovs[0] = (struct iovec){&dmsg->fuse_out_header,
+                                 sizeof(struct fuse_out_header)};
+        iovs[1] = (struct iovec){dmsg->fuse_message_body,
+                                 len - sizeof(struct fuse_out_header)};
+        rv = sys_writev(priv->fd, iovs, 2);
+        check_and_dump_fuse_W(priv, iovs, 2, rv, dmsg->errnomask);
+
+        fuse_timed_message_free(dmsg);
+
+        if (rv == -1 && errno == EBADF) {
+            break;
         }
 
- out:
-        GF_FREE (finh);
-}
+        if (rv != len && !(rv == -1 && errno == ENOENT)) {
+            gf_log("glusterfs-fuse", GF_LOG_INFO,
+                   "len: %zu, rv: %zd, errno: %d", len, rv, errno);
+        }
+    }
 
+    gf_log("glusterfs-fuse", GF_LOG_ERROR, "timed response loop terminated");
 
-static void
-fuse_enosys (xlator_t *this, fuse_in_header_t *finh, void *msg)
-{
-        send_fuse_err (this, finh, ENOSYS);
+    pthread_mutex_lock(&priv->timed_mutex);
+    {
+        priv->timed_response_fuse_thread_started = _gf_false;
+        list_for_each_entry_safe(dmsg, tmp, &priv->timed_list, next)
+        {
+            list_del_init(&dmsg->next);
+            fuse_timed_message_free(dmsg);
+        }
+    }
+    pthread_mutex_unlock(&priv->timed_mutex);
 
-        GF_FREE (finh);
+    return NULL;
 }
 
-
 static void
-fuse_destroy (xlator_t *this, fuse_in_header_t *finh, void *msg)
-{
-        send_fuse_err (this, finh, 0);
+fuse_init(xlator_t *this, fuse_in_header_t *finh, void *msg,
+          struct iobuf *iobuf)
+{
+    struct fuse_init_in *fini = msg;
+    struct fuse_init_out fino = {
+        0,
+    };
+    fuse_private_t *priv = NULL;
+    size_t size = 0;
+    int ret = 0;
+#if FUSE_KERNEL_MINOR_VERSION >= 9
+    pthread_t messenger;
+#endif
+    pthread_t delayer;
 
-        GF_FREE (finh);
-}
+    priv = this->private;
 
+    if (priv->init_recvd) {
+        gf_log("glusterfs-fuse", GF_LOG_ERROR, "got INIT after first message");
 
+        sys_close(priv->fd);
+        goto out;
+    }
 
-struct fuse_first_lookup {
-        pthread_mutex_t  mutex;
-        pthread_cond_t   cond;
-        char             fin;
-};
+    priv->init_recvd = 1;
 
-int
-fuse_first_lookup_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                       int32_t op_ret, int32_t op_errno,
-                       inode_t *inode, struct iatt *buf, dict_t *xattr,
-                       struct iatt *postparent)
-{
-        struct fuse_first_lookup *stub = NULL;
+    if (fini->major != FUSE_KERNEL_VERSION) {
+        gf_log("glusterfs-fuse", GF_LOG_ERROR,
+               "unsupported FUSE protocol version %d.%d", fini->major,
+               fini->minor);
 
-        stub = frame->local;
+        sys_close(priv->fd);
+        goto out;
+    }
+    priv->proto_minor = fini->minor;
 
-        if (op_ret == 0) {
-                gf_log (this->name, GF_LOG_TRACE,
-                        "first lookup on root succeeded.");
-        } else {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "first lookup on root failed.");
-        }
+    fino.major = FUSE_KERNEL_VERSION;
+    fino.minor = FUSE_KERNEL_MINOR_VERSION;
+    fino.max_readahead = 1 << 17;
+    fino.max_write = 1 << 17;
+    fino.flags = FUSE_ASYNC_READ | FUSE_POSIX_LOCKS;
+#if FUSE_KERNEL_MINOR_VERSION >= 17
+    if (fini->minor >= 17)
+        fino.flags |= FUSE_FLOCK_LOCKS;
+#endif
+#if FUSE_KERNEL_MINOR_VERSION >= 12
+    if (fini->minor >= 12) {
+        /* let fuse leave the umask processing to us, so that it does not
+         * break extended POSIX ACL defaults on server */
+        fino.flags |= FUSE_DONT_MASK;
+    }
+#endif
+#if FUSE_KERNEL_MINOR_VERSION >= 9
+    if (fini->minor >= 6 /* fuse_init_in has flags */ &&
+        fini->flags & FUSE_BIG_WRITES) {
+        /* no need for direct I/O mode by default if big writes are supported */
+        if (priv->direct_io_mode == 2)
+            priv->direct_io_mode = 0;
+        fino.flags |= FUSE_BIG_WRITES;
+    }
+
+    /* Start the thread processing timed responses */
+    ret = gf_thread_create(&delayer, NULL, timed_response_loop, this,
+                           "fusedlyd");
+    if (ret != 0) {
+        gf_log("glusterfs-fuse", GF_LOG_ERROR,
+               "failed to start timed response thread (%s)", strerror(errno));
+
+        sys_close(priv->fd);
+        goto out;
+    }
+    priv->timed_response_fuse_thread_started = _gf_true;
+
+    /* Used for 'reverse invalidation of inode' */
+#ifdef HAVE_FUSE_NOTIFICATIONS
+    if (fini->minor >= 12) {
+        ret = gf_thread_create(&messenger, NULL, notify_kernel_loop, this,
+                               "fusenoti");
+        if (ret != 0) {
+            gf_log("glusterfs-fuse", GF_LOG_ERROR,
+                   "failed to start messenger daemon (%s)", strerror(errno));
 
-        pthread_mutex_lock (&stub->mutex);
-        {
-                stub->fin = 1;
-                pthread_cond_broadcast (&stub->cond);
+            sys_close(priv->fd);
+            goto out;
         }
-        pthread_mutex_unlock (&stub->mutex);
-
-        return 0;
-}
+        priv->reverse_fuse_thread_started = _gf_true;
+    } else
+#endif
+    {
+        /*
+         * FUSE minor < 12 does not implement invalidate notifications.
+         * This mechanism is required for fopen-keep-cache to operate
+         * correctly. Disable and warn the user.
+         */
+        if (priv->fopen_keep_cache) {
+            gf_log("glusterfs-fuse", GF_LOG_WARNING,
+                   "FUSE version "
+                   "%d.%d does not support inval notifications. "
+                   "fopen-keep-cache disabled.",
+                   fini->major, fini->minor);
+            priv->fopen_keep_cache = 0;
+        }
+    }
+
+    if (fini->minor >= 13) {
+        fino.max_background = priv->background_qlen;
+        fino.congestion_threshold = priv->congestion_threshold;
+    }
+    if (fini->minor < 9)
+        *priv->msg0_len_p = sizeof(*finh) + FUSE_COMPAT_WRITE_IN_SIZE;
+
+    if (priv->use_readdirp) {
+        if (fini->flags & FUSE_DO_READDIRPLUS)
+            fino.flags |= FUSE_DO_READDIRPLUS;
+    }
+#endif
+    if (priv->fopen_keep_cache == 2) {
+        /* If user did not explicitly set --fopen-keep-cache[=off],
+           then check if kernel support FUSE_AUTO_INVAL_DATA and ...
+        */
 
+        priv->fopen_keep_cache = 1;
 
-int
-fuse_first_lookup (xlator_t *this)
-{
-        fuse_private_t            *priv = NULL;
-        loc_t                      loc = {0, };
-        call_frame_t              *frame = NULL;
-        xlator_t                  *xl = NULL;
-        dict_t                    *dict = NULL;
-        struct fuse_first_lookup   stub;
-        uuid_t                     gfid;
-        int                        ret;
-
-        priv = this->private;
-
-        loc.path = "/";
-        loc.name = "";
-        loc.inode = fuse_ino_to_inode (1, this);
-        gf_uuid_copy (loc.gfid, loc.inode->gfid);
-        loc.parent = NULL;
-
-        dict = dict_new ();
-        frame = create_frame (this, this->ctx->pool);
-        if (!frame) {
-                gf_log ("fuse", GF_LOG_ERROR, "failed to create frame");
-                return -1;
+#if FUSE_KERNEL_MINOR_VERSION >= 20
+        if (fini->flags & FUSE_AUTO_INVAL_DATA) {
+            /* ... enable fopen_keep_cache mode if supported.
+             */
+            gf_log("glusterfs-fuse", GF_LOG_DEBUG,
+                   "Detected "
+                   "support for FUSE_AUTO_INVAL_DATA. Enabling "
+                   "fopen_keep_cache automatically.");
+
+            if (priv->fuse_auto_inval)
+                fino.flags |= FUSE_AUTO_INVAL_DATA;
+        } else
+#endif
+        {
+            if (priv->fuse_auto_inval) {
+                gf_log("glusterfs-fuse", GF_LOG_DEBUG,
+                       "No support for FUSE_AUTO_INVAL_DATA. Disabling "
+                       "fopen_keep_cache.");
+                /* ... else disable. */
+                priv->fopen_keep_cache = 0;
+            }
+        }
+    } else if (priv->fopen_keep_cache == 1) {
+        /* If user explicitly set --fopen-keep-cache[=on],
+           then enable FUSE_AUTO_INVAL_DATA if possible.
+        */
+#if FUSE_KERNEL_MINOR_VERSION >= 20
+        if (priv->fuse_auto_inval && (fini->flags & FUSE_AUTO_INVAL_DATA)) {
+            gf_log("glusterfs-fuse", GF_LOG_DEBUG,
+                   "fopen_keep_cache "
+                   "is explicitly set. Enabling FUSE_AUTO_INVAL_DATA");
+            fino.flags |= FUSE_AUTO_INVAL_DATA;
+        } else
+#endif
+        {
+            gf_log("glusterfs-fuse", GF_LOG_WARNING,
+                   "fopen_keep_cache "
+                   "is explicitly set. Support for "
+                   "FUSE_AUTO_INVAL_DATA is missing");
         }
+    }
 
-        frame->root->type = GF_OP_TYPE_FOP;
-
-        xl = priv->active_subvol;
-
-        pthread_mutex_init (&stub.mutex, NULL);
-        pthread_cond_init (&stub.cond, NULL);
-        stub.fin = 0;
-
-        frame->local = &stub;
-
-        memset (gfid, 0, 16);
-        gfid[15] = 1;
-        ret = dict_set_static_bin (dict, "gfid-req", gfid, 16);
-        if (ret)
-                gf_log (xl->name, GF_LOG_ERROR, "failed to set 'gfid-req'");
-
-        STACK_WIND (frame, fuse_first_lookup_cbk, xl, xl->fops->lookup,
-                    &loc, dict);
-        dict_unref (dict);
+#if FUSE_KERNEL_MINOR_VERSION >= 22
+    if (fini->flags & FUSE_ASYNC_DIO)
+        fino.flags |= FUSE_ASYNC_DIO;
+#endif
 
-        pthread_mutex_lock (&stub.mutex);
-        {
-                while (!stub.fin) {
-                        pthread_cond_wait (&stub.cond, &stub.mutex);
-                }
-        }
-        pthread_mutex_unlock (&stub.mutex);
+    size = sizeof(fino);
+#if FUSE_KERNEL_MINOR_VERSION >= 23
+    /* FUSE 7.23 and newer added attributes to the fuse_init_out struct */
+    if (fini->minor < 23) {
+        /* reduce the size, chop off unused attributes from &fino */
+        size = FUSE_COMPAT_22_INIT_OUT_SIZE;
+    }
+
+    /* Writeback cache support */
+    if (fini->minor >= 23) {
+        if (priv->kernel_writeback_cache)
+            fino.flags |= FUSE_WRITEBACK_CACHE;
+        fino.time_gran = priv->attr_times_granularity;
+    }
+#endif
 
-        pthread_mutex_destroy (&stub.mutex);
-        pthread_cond_destroy (&stub.cond);
+    ret = send_fuse_data(this, finh, &fino, size);
+    if (ret == 0)
+        gf_log("glusterfs-fuse", GF_LOG_INFO,
+               "FUSE inited with protocol versions:"
+               " glusterfs %d.%d kernel %d.%d",
+               FUSE_KERNEL_VERSION, FUSE_KERNEL_MINOR_VERSION, fini->major,
+               fini->minor);
+    else {
+        gf_log("glusterfs-fuse", GF_LOG_ERROR, "FUSE init failed (%s)",
+               strerror(ret));
 
-        frame->local = NULL;
-        STACK_DESTROY (frame->root);
+        sys_close(priv->fd);
+    }
 
-        return 0;
+out:
+    GF_FREE(finh);
 }
 
-
-int
-fuse_nameless_lookup (xlator_t *xl, uuid_t gfid, loc_t *loc)
+static void
+fuse_enosys(xlator_t *this, fuse_in_header_t *finh, void *msg,
+            struct iobuf *iobuf)
 {
-        int          ret          = -1;
-        dict_t      *xattr_req    = NULL;
-        struct iatt  iatt         = {0, };
-        inode_t     *linked_inode = NULL;
-
-        if ((loc == NULL) || (xl == NULL)) {
-                ret = -EINVAL;
-                goto out;
-        }
-
-        if (loc->inode == NULL) {
-                loc->inode = inode_new (xl->itable);
-                if (loc->inode == NULL) {
-                        ret = -ENOMEM;
-                        goto out;
-                }
-        }
+    send_fuse_err(this, finh, ENOSYS);
 
-        gf_uuid_copy (loc->gfid, gfid);
+    GF_FREE(finh);
+}
 
-        xattr_req = dict_new ();
-        if (xattr_req == NULL) {
-                ret = -ENOMEM;
-                goto out;
-        }
+static void
+fuse_destroy(xlator_t *this, fuse_in_header_t *finh, void *msg,
+             struct iobuf *iobuf)
+{
+    send_fuse_err(this, finh, 0);
 
-        ret = syncop_lookup (xl, loc, &iatt, NULL, xattr_req, NULL);
-        if (ret < 0)
-                goto out;
+    GF_FREE(finh);
+}
 
-        linked_inode = inode_link (loc->inode, NULL, NULL, &iatt);
-        inode_unref (loc->inode);
-        loc->inode = linked_inode;
+int
+fuse_first_lookup(xlator_t *this)
+{
+    fuse_private_t *priv = NULL;
+    loc_t loc = {
+        0,
+    };
+    xlator_t *xl = NULL;
+    dict_t *dict = NULL;
+    static uuid_t gfid = {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1};
+    int ret = -1;
+    struct iatt iatt = {
+        0,
+    };
+
+    priv = this->private;
+
+    loc.path = "/";
+    loc.name = "";
+    loc.inode = fuse_ino_to_inode(1, this);
+    gf_uuid_copy(loc.gfid, loc.inode->gfid);
+    loc.parent = NULL;
+
+    dict = dict_new();
+
+    xl = priv->active_subvol;
+
+    ret = dict_set_gfuuid(dict, "gfid-req", gfid, true);
+    if (ret) {
+        gf_log(xl->name, GF_LOG_ERROR, "failed to set 'gfid-req'");
+        goto out;
+    }
+
+    ret = syncop_lookup(xl, &loc, &iatt, NULL, dict, NULL);
+    DECODE_SYNCOP_ERR(ret);
+    if (ret < 0) {
+        gf_log(this->name, GF_LOG_ERROR, "first lookup on root failed (%s)",
+               strerror(errno));
+        /* NOTE: Treat it as an error case. */
+        /* goto out; */ /* commented for preventing coverity warning */
+    }
+    /* Remove comment of above goto statement if you are adding any
+       more code here, before 'out' label */
 
-        ret = 0;
 out:
-        if (xattr_req != NULL) {
-                dict_unref (xattr_req);
-        }
+    dict_unref(dict);
+    inode_unref(loc.inode);
 
-        return ret;
+    return ret;
 }
 
-
 int
-fuse_migrate_fd_open (xlator_t *this, fd_t *basefd, fd_t *oldfd,
-                      xlator_t *old_subvol, xlator_t *new_subvol)
-{
-        loc_t          loc       = {0, };
-        fd_t          *newfd     = NULL, *old_activefd = NULL;
-        fuse_fd_ctx_t *basefd_ctx = NULL;
-        fuse_fd_ctx_t *newfd_ctx = NULL;
-        int            ret       = 0, flags = 0;
-
-        ret = inode_path (basefd->inode, NULL, (char **)&loc.path);
-        if (ret < 0) {
-                gf_log ("glusterfs-fuse", GF_LOG_WARNING,
-                        "cannot construct path of gfid (%s) failed"
-                        "(old-subvolume:%s-%d new-subvolume:%s-%d)",
-                        uuid_utoa (basefd->inode->gfid),
-                        old_subvol->name, old_subvol->graph->id,
-                        new_subvol->name, new_subvol->graph->id);
-                goto out;
+fuse_nameless_lookup(xlator_t *this, xlator_t *xl, uuid_t gfid, loc_t *loc)
+{
+    int ret = -1;
+    dict_t *xattr_req = NULL;
+    struct iatt iatt = {
+        0,
+    };
+    inode_t *linked_inode = NULL;
+    uint64_t ctx_value = LOOKUP_NOT_NEEDED;
+
+    if ((loc == NULL) || (xl == NULL)) {
+        ret = -EINVAL;
+        goto out;
+    }
+
+    if (loc->inode == NULL) {
+        loc->inode = inode_new(xl->itable);
+        if (loc->inode == NULL) {
+            ret = -ENOMEM;
+            goto out;
         }
+    }
 
-        gf_uuid_copy (loc.gfid, basefd->inode->gfid);
-
-        loc.inode = inode_find (new_subvol->itable, basefd->inode->gfid);
-
-        if (loc.inode == NULL) {
-                ret = fuse_nameless_lookup (new_subvol, basefd->inode->gfid,
-                                            &loc);
-                if (ret < 0) {
-                        gf_log ("glusterfs-fuse", GF_LOG_WARNING,
-                                "name-less lookup of gfid (%s) failed (%s)"
-                                "(old-subvolume:%s-%d new-subvolume:%s-%d)",
-                                uuid_utoa (basefd->inode->gfid),
-                                strerror (-ret),
-                                old_subvol->name, old_subvol->graph->id,
-                                new_subvol->name, new_subvol->graph->id);
-                        ret = -1;
-                        goto out;
-                }
+    gf_uuid_copy(loc->gfid, gfid);
 
-        }
+    xattr_req = dict_new();
+    if (xattr_req == NULL) {
+        ret = -ENOMEM;
+        goto out;
+    }
 
-        basefd_ctx = fuse_fd_ctx_get (this, basefd);
-        GF_VALIDATE_OR_GOTO ("glusterfs-fuse", basefd_ctx, out);
-
-        newfd = fd_create (loc.inode, basefd->pid);
-        if (newfd == NULL) {
-                gf_log ("glusterfs-fuse", GF_LOG_WARNING,
-                        "cannot create new fd, hence not migrating basefd "
-                        "(ptr:%p inode-gfid:%s) "
-                        "(old-subvolume:%s-%d new-subvolume:%s-%d)", basefd,
-                        uuid_utoa (loc.inode->gfid),
-                        old_subvol->name, old_subvol->graph->id,
-                        new_subvol->name, new_subvol->graph->id);
-                ret = -1;
-                goto out;
-        }
+    ret = syncop_lookup(xl, loc, &iatt, NULL, xattr_req, NULL);
+    if (ret < 0)
+        goto out;
 
-        newfd->flags = basefd->flags;
-	if (newfd->lk_ctx)
-		fd_lk_ctx_unref (newfd->lk_ctx);
+    linked_inode = inode_link(loc->inode, NULL, NULL, &iatt);
+    if (linked_inode == loc->inode)
+        inode_ctx_set(linked_inode, this, &ctx_value);
 
-        newfd->lk_ctx = fd_lk_ctx_ref (oldfd->lk_ctx);
+    inode_unref(loc->inode);
+    loc->inode = linked_inode;
 
-        newfd_ctx = fuse_fd_ctx_check_n_create (this, newfd);
-        GF_VALIDATE_OR_GOTO ("glusterfs-fuse", newfd_ctx, out);
+    ret = 0;
+out:
+    if (xattr_req != NULL) {
+        dict_unref(xattr_req);
+    }
 
-        if (IA_ISDIR (basefd->inode->ia_type)) {
-                ret = syncop_opendir (new_subvol, &loc, newfd, NULL, NULL);
-        } else {
-                flags = basefd->flags & ~(O_CREAT | O_EXCL | O_TRUNC);
-                ret = syncop_open (new_subvol, &loc, flags, newfd, NULL, NULL);
-        }
+    return ret;
+}
 
+int
+fuse_migrate_fd_open(xlator_t *this, fd_t *basefd, fd_t *oldfd,
+                     xlator_t *old_subvol, xlator_t *new_subvol)
+{
+    loc_t loc = {
+        0,
+    };
+    fd_t *newfd = NULL, *old_activefd = NULL;
+    fuse_fd_ctx_t *basefd_ctx = NULL;
+    fuse_fd_ctx_t *newfd_ctx = NULL;
+    int ret = 0, flags = 0;
+
+    ret = inode_path(basefd->inode, NULL, (char **)&loc.path);
+    if (ret < 0) {
+        gf_log("glusterfs-fuse", GF_LOG_WARNING,
+               "cannot construct path of gfid (%s) failed"
+               "(old-subvolume:%s-%d new-subvolume:%s-%d)",
+               uuid_utoa(basefd->inode->gfid), old_subvol->name,
+               old_subvol->graph->id, new_subvol->name, new_subvol->graph->id);
+        goto out;
+    }
+
+    gf_uuid_copy(loc.gfid, basefd->inode->gfid);
+
+    loc.inode = inode_find(new_subvol->itable, basefd->inode->gfid);
+
+    if (loc.inode == NULL) {
+        ret = fuse_nameless_lookup(this, new_subvol, basefd->inode->gfid, &loc);
         if (ret < 0) {
-                gf_log ("glusterfs-fuse", GF_LOG_WARNING,
-                        "open on basefd (ptr:%p inode-gfid:%s) failed (%s)"
-                        "(old-subvolume:%s-%d new-subvolume:%s-%d)", basefd,
-                        uuid_utoa (basefd->inode->gfid), strerror (-ret),
-                        old_subvol->name, old_subvol->graph->id,
-                        new_subvol->name, new_subvol->graph->id);
-                ret = -1;
-                goto out;
-        }
-
-        fd_bind (newfd);
-
-        LOCK (&basefd->lock);
-        {
-                if (basefd_ctx->activefd != NULL) {
-                        old_activefd = basefd_ctx->activefd;
-                }
-
-                basefd_ctx->activefd = newfd;
-        }
-        UNLOCK (&basefd->lock);
-
-        if (old_activefd != NULL) {
-                fd_unref (old_activefd);
-        }
-
-        gf_log ("glusterfs-fuse", GF_LOG_INFO,
-                "migrated basefd (%p) to newfd (%p) (inode-gfid:%s)"
-                "(old-subvolume:%s-%d new-subvolume:%s-%d)", basefd, newfd,
-                uuid_utoa (basefd->inode->gfid),
-		old_subvol->name, old_subvol->graph->id,
-                new_subvol->name, new_subvol->graph->id);
-
-        ret = 0;
+            gf_log("glusterfs-fuse", GF_LOG_WARNING,
+                   "name-less lookup of gfid (%s) failed (%s)"
+                   "(old-subvolume:%s-%d new-subvolume:%s-%d)",
+                   uuid_utoa(basefd->inode->gfid), strerror(-ret),
+                   old_subvol->name, old_subvol->graph->id, new_subvol->name,
+                   new_subvol->graph->id);
+            ret = -1;
+            goto out;
+        }
+    }
+
+    basefd_ctx = fuse_fd_ctx_get(this, basefd);
+    GF_VALIDATE_OR_GOTO("glusterfs-fuse", basefd_ctx, out);
+
+    newfd = fd_create(loc.inode, basefd->pid);
+    if (newfd == NULL) {
+        gf_log("glusterfs-fuse", GF_LOG_WARNING,
+               "cannot create new fd, hence not migrating basefd "
+               "(ptr:%p inode-gfid:%s) "
+               "(old-subvolume:%s-%d new-subvolume:%s-%d)",
+               basefd, uuid_utoa(loc.inode->gfid), old_subvol->name,
+               old_subvol->graph->id, new_subvol->name, new_subvol->graph->id);
+        ret = -1;
+        goto out;
+    }
+
+    newfd->flags = basefd->flags;
+    if (newfd->lk_ctx)
+        fd_lk_ctx_unref(newfd->lk_ctx);
+
+    newfd->lk_ctx = fd_lk_ctx_ref(oldfd->lk_ctx);
+
+    newfd_ctx = fuse_fd_ctx_check_n_create(this, newfd);
+    GF_VALIDATE_OR_GOTO("glusterfs-fuse", newfd_ctx, out);
+
+    if (IA_ISDIR(basefd->inode->ia_type)) {
+        ret = syncop_opendir(new_subvol, &loc, newfd, NULL, NULL);
+    } else {
+        flags = basefd->flags & ~(O_CREAT | O_EXCL | O_TRUNC);
+        ret = syncop_open(new_subvol, &loc, flags, newfd, NULL, NULL);
+    }
+
+    if (ret < 0) {
+        gf_log("glusterfs-fuse", GF_LOG_WARNING,
+               "open on basefd (ptr:%p inode-gfid:%s) failed (%s)"
+               "(old-subvolume:%s-%d new-subvolume:%s-%d)",
+               basefd, uuid_utoa(basefd->inode->gfid), strerror(-ret),
+               old_subvol->name, old_subvol->graph->id, new_subvol->name,
+               new_subvol->graph->id);
+        ret = -1;
+        goto out;
+    }
+
+    fd_bind(newfd);
+
+    LOCK(&basefd->lock);
+    {
+        if (basefd_ctx->activefd != NULL) {
+            old_activefd = basefd_ctx->activefd;
+        }
+
+        basefd_ctx->activefd = newfd;
+    }
+    UNLOCK(&basefd->lock);
+
+    if (old_activefd != NULL) {
+        fd_unref(old_activefd);
+    }
+
+    gf_log("glusterfs-fuse", GF_LOG_INFO,
+           "migrated basefd (%p) to newfd (%p) (inode-gfid:%s)"
+           "(old-subvolume:%s-%d new-subvolume:%s-%d)",
+           basefd, newfd, uuid_utoa(basefd->inode->gfid), old_subvol->name,
+           old_subvol->graph->id, new_subvol->name, new_subvol->graph->id);
+
+    ret = 0;
 
 out:
-        loc_wipe (&loc);
+    loc_wipe(&loc);
 
-        return ret;
+    return ret;
 }
 
 int
-fuse_migrate_locks (xlator_t *this, fd_t *basefd, fd_t *oldfd,
-                    xlator_t *old_subvol, xlator_t *new_subvol)
+fuse_migrate_locks(xlator_t *this, fd_t *basefd, fd_t *oldfd,
+                   xlator_t *old_subvol, xlator_t *new_subvol)
 {
-        int	 ret      = -1;
-        dict_t	*lockinfo = NULL;
-        void    *ptr      = NULL;
-	fd_t    *newfd    = NULL;
-        fuse_fd_ctx_t *basefd_ctx         = NULL;
-
-
-	if (!oldfd->lk_ctx || fd_lk_ctx_empty (oldfd->lk_ctx))
-		return 0;
+    int ret = -1;
+    dict_t *lockinfo = NULL;
+    void *ptr = NULL;
+    fd_t *newfd = NULL;
+    fuse_fd_ctx_t *basefd_ctx = NULL;
 
-        basefd_ctx = fuse_fd_ctx_get (this, basefd);
-        GF_VALIDATE_OR_GOTO ("glusterfs-fuse", basefd_ctx, out);
-
-        LOCK (&basefd->lock);
-        {
-                newfd = fd_ref (basefd_ctx->activefd);
-        }
-        UNLOCK (&basefd->lock);
-
-        ret = syncop_fgetxattr (old_subvol, oldfd, &lockinfo,
-                                GF_XATTR_LOCKINFO_KEY, NULL, NULL);
-        if (ret < 0) {
-                gf_log (this->name, GF_LOG_WARNING,
-			"getting lockinfo failed while migrating locks"
-			"(oldfd:%p newfd:%p inode-gfid:%s)"
-			"(old-subvol:%s-%d new-subvol:%s-%d)",
-			oldfd, newfd, uuid_utoa (newfd->inode->gfid),
-			old_subvol->name, old_subvol->graph->id,
-			new_subvol->name, new_subvol->graph->id);
-                ret = -1;
-                goto out;
-        }
-
-        ret = dict_get_ptr (lockinfo, GF_XATTR_LOCKINFO_KEY, &ptr);
-        if (ptr == NULL) {
-                ret = 0;
-                gf_log (this->name, GF_LOG_INFO,
-                        "No lockinfo present on any of the bricks "
-			"(oldfd: %p newfd:%p inode-gfid:%s) "
-                        "(old-subvol:%s-%d new-subvol:%s-%d)",
-			oldfd, newfd, uuid_utoa (newfd->inode->gfid),
-			old_subvol->name, old_subvol->graph->id,
-			new_subvol->name, new_subvol->graph->id);
-
-                goto out;
-        }
+    if (!oldfd->lk_ctx || fd_lk_ctx_empty(oldfd->lk_ctx))
+        return 0;
 
-        ret = syncop_fsetxattr (new_subvol, newfd, lockinfo, 0, NULL, NULL);
-        if (ret < 0) {
-                gf_log (this->name, GF_LOG_WARNING,
-			"migrating locks failed (oldfd:%p newfd:%p "
-			"inode-gfid:%s) (old-subvol:%s-%d new-subvol:%s-%d)",
-			oldfd, newfd, uuid_utoa (newfd->inode->gfid),
-			old_subvol->name, old_subvol->graph->id,
-			new_subvol->name, new_subvol->graph->id);
-                ret = -1;
-                goto out;
-        }
+    basefd_ctx = fuse_fd_ctx_get(this, basefd);
+    GF_VALIDATE_OR_GOTO("glusterfs-fuse", basefd_ctx, out);
+
+    LOCK(&basefd->lock);
+    {
+        newfd = fd_ref(basefd_ctx->activefd);
+    }
+    UNLOCK(&basefd->lock);
+
+    ret = syncop_fgetxattr(old_subvol, oldfd, &lockinfo, GF_XATTR_LOCKINFO_KEY,
+                           NULL, NULL);
+    if (ret < 0) {
+        gf_log(this->name, GF_LOG_WARNING,
+               "getting lockinfo failed while migrating locks"
+               "(oldfd:%p newfd:%p inode-gfid:%s)"
+               "(old-subvol:%s-%d new-subvol:%s-%d)",
+               oldfd, newfd, uuid_utoa(newfd->inode->gfid), old_subvol->name,
+               old_subvol->graph->id, new_subvol->name, new_subvol->graph->id);
+        ret = -1;
+        goto out;
+    }
+
+    ret = dict_get_ptr(lockinfo, GF_XATTR_LOCKINFO_KEY, &ptr);
+    if (ptr == NULL) {
+        ret = 0;
+        gf_log(this->name, GF_LOG_INFO,
+               "No lockinfo present on any of the bricks "
+               "(oldfd: %p newfd:%p inode-gfid:%s) "
+               "(old-subvol:%s-%d new-subvol:%s-%d)",
+               oldfd, newfd, uuid_utoa(newfd->inode->gfid), old_subvol->name,
+               old_subvol->graph->id, new_subvol->name, new_subvol->graph->id);
+
+        goto out;
+    }
+
+    ret = syncop_fsetxattr(new_subvol, newfd, lockinfo, 0, NULL, NULL);
+    if (ret < 0) {
+        gf_log(this->name, GF_LOG_WARNING,
+               "migrating locks failed (oldfd:%p newfd:%p "
+               "inode-gfid:%s) (old-subvol:%s-%d new-subvol:%s-%d)",
+               oldfd, newfd, uuid_utoa(newfd->inode->gfid), old_subvol->name,
+               old_subvol->graph->id, new_subvol->name, new_subvol->graph->id);
+        ret = -1;
+        goto out;
+    }
 
 out:
-	if (newfd)
-		fd_unref (newfd);
+    if (newfd)
+        fd_unref(newfd);
 
-        if (lockinfo != NULL) {
-                dict_unref (lockinfo);
-        }
+    if (lockinfo != NULL) {
+        dict_unref(lockinfo);
+    }
 
-        return ret;
+    return ret;
 }
 
-
 int
-fuse_migrate_fd (xlator_t *this, fd_t *basefd, xlator_t *old_subvol,
-                 xlator_t *new_subvol)
-{
-        int            ret                = -1;
-        char           create_in_progress = 0;
-        fuse_fd_ctx_t *basefd_ctx         = NULL;
-        fd_t          *oldfd              = NULL;
-
-        basefd_ctx = fuse_fd_ctx_get (this, basefd);
-        GF_VALIDATE_OR_GOTO ("glusterfs-fuse", basefd_ctx, out);
-
-        LOCK (&basefd->lock);
-        {
-                oldfd = basefd_ctx->activefd ? basefd_ctx->activefd
-                        : basefd;
-                fd_ref (oldfd);
+fuse_migrate_fd(xlator_t *this, fd_t *basefd, xlator_t *old_subvol,
+                xlator_t *new_subvol)
+{
+    int ret = -1;
+    char create_in_progress = 0;
+    fuse_fd_ctx_t *basefd_ctx = NULL;
+    fd_t *oldfd = NULL;
+    dict_t *xdata = NULL;
+
+    basefd_ctx = fuse_fd_ctx_get(this, basefd);
+    GF_VALIDATE_OR_GOTO("glusterfs-fuse", basefd_ctx, out);
+
+    LOCK(&basefd->lock);
+    {
+        oldfd = basefd_ctx->activefd ? basefd_ctx->activefd : basefd;
+        fd_ref(oldfd);
+    }
+    UNLOCK(&basefd->lock);
+
+    LOCK(&oldfd->inode->lock);
+    {
+        if (gf_uuid_is_null(oldfd->inode->gfid)) {
+            create_in_progress = 1;
+        } else {
+            create_in_progress = 0;
         }
-        UNLOCK (&basefd->lock);
+    }
+    UNLOCK(&oldfd->inode->lock);
 
-        LOCK (&oldfd->inode->lock);
-        {
-                if (gf_uuid_is_null (oldfd->inode->gfid)) {
-                        create_in_progress = 1;
-                } else {
-                        create_in_progress = 0;
-                }
-        }
-        UNLOCK (&oldfd->inode->lock);
-
-        if (create_in_progress) {
-                gf_log ("glusterfs-fuse", GF_LOG_INFO,
-                        "create call on fd (%p) is in progress "
-                        "(basefd-ptr:%p basefd-inode.gfid:%s), "
-                        "hence deferring migration till application does an "
-                        "fd based operation on this fd"
-                        "(old-subvolume:%s-%d, new-subvolume:%s-%d)",
-                        oldfd, basefd, uuid_utoa (basefd->inode->gfid),
-                        old_subvol->name, old_subvol->graph->id,
-                        new_subvol->name, new_subvol->graph->id);
-
-                ret = 0;
-                goto out;
-        }
+    if (create_in_progress) {
+        gf_log("glusterfs-fuse", GF_LOG_INFO,
+               "create call on fd (%p) is in progress "
+               "(basefd-ptr:%p basefd-inode.gfid:%s), "
+               "hence deferring migration till application does an "
+               "fd based operation on this fd"
+               "(old-subvolume:%s-%d, new-subvolume:%s-%d)",
+               oldfd, basefd, uuid_utoa(basefd->inode->gfid), old_subvol->name,
+               old_subvol->graph->id, new_subvol->name, new_subvol->graph->id);
 
-        if (oldfd->inode->table->xl == old_subvol) {
-                ret = syncop_fsync (old_subvol, oldfd, 0, NULL, NULL);
-                if (ret < 0) {
-                        gf_log ("glusterfs-fuse", GF_LOG_WARNING,
-                                "syncop_fsync failed (%s) on fd (%p)"
-                                "(basefd:%p basefd-inode.gfid:%s) "
-                                "(old-subvolume:%s-%d new-subvolume:%s-%d)",
-                                strerror (-ret), oldfd, basefd,
-                                uuid_utoa (basefd->inode->gfid),
-                                old_subvol->name, old_subvol->graph->id,
-                                new_subvol->name, new_subvol->graph->id);
-                        ret = -1;
-                }
+        ret = 0;
+        goto out;
+    }
+
+    if (oldfd->inode->table->xl == old_subvol) {
+        if (IA_ISDIR(oldfd->inode->ia_type)) {
+            ret = syncop_fsyncdir(old_subvol, oldfd, 0, NULL, NULL);
         } else {
-                gf_log ("glusterfs-fuse", GF_LOG_WARNING,
-                        "basefd (ptr:%p inode-gfid:%s) was not "
-                        "migrated during previous graph switch"
-                        "(old-subvolume:%s-%d new-subvolume: %s-%d)", basefd,
-                        basefd->inode->gfid,
-                        old_subvol->name, old_subvol->graph->id,
-                        new_subvol->name, new_subvol->graph->id);
-        }
+            xdata = dict_new();
+            if (!xdata || dict_set_int8(xdata, "last-fsync", 1)) {
+                gf_log("glusterfs-fuse", GF_LOG_WARNING,
+                       "last-fsync set failed (%s) on fd (%p)"
+                       "(basefd:%p basefd-inode.gfid:%s) "
+                       "(old-subvolume:%s-%d new-subvolume:%s-%d)",
+                       strerror(ENOMEM), oldfd, basefd,
+                       uuid_utoa(basefd->inode->gfid), old_subvol->name,
+                       old_subvol->graph->id, new_subvol->name,
+                       new_subvol->graph->id);
+            }
 
-        ret = fuse_migrate_fd_open (this, basefd, oldfd, old_subvol,
-                                    new_subvol);
-        if (ret < 0) {
-                gf_log (this->name, GF_LOG_WARNING, "open corresponding to "
-                        "basefd (ptr:%p inode-gfid:%s) in new graph failed "
-                        "(old-subvolume:%s-%d new-subvolume:%s-%d)", basefd,
-                        uuid_utoa (basefd->inode->gfid), old_subvol->name,
-                        old_subvol->graph->id, new_subvol->name,
-                        new_subvol->graph->id);
-                goto out;
+            ret = syncop_fsync(old_subvol, oldfd, 0, NULL, NULL, xdata, NULL);
         }
 
-        ret = fuse_migrate_locks (this, basefd, oldfd, old_subvol,
-                                  new_subvol);
         if (ret < 0) {
-                gf_log (this->name, GF_LOG_WARNING,
-                        "migrating locks from old-subvolume (%s-%d) to "
-                        "new-subvolume (%s-%d) failed (inode-gfid:%s oldfd:%p "
-                        "basefd:%p)", old_subvol->name, old_subvol->graph->id,
-                        new_subvol->name, new_subvol->graph->id,
-                        uuid_utoa (basefd->inode->gfid), oldfd, basefd);
-
-        }
+            gf_log("glusterfs-fuse", GF_LOG_WARNING,
+                   "syncop_fsync(dir) failed (%s) on fd (%p)"
+                   "(basefd:%p basefd-inode.gfid:%s) "
+                   "(old-subvolume:%s-%d new-subvolume:%s-%d)",
+                   strerror(-ret), oldfd, basefd,
+                   uuid_utoa(basefd->inode->gfid), old_subvol->name,
+                   old_subvol->graph->id, new_subvol->name,
+                   new_subvol->graph->id);
+        }
+    } else {
+        gf_log("glusterfs-fuse", GF_LOG_WARNING,
+               "basefd (ptr:%p inode-gfid:%s) was not "
+               "migrated during previous graph switch"
+               "(old-subvolume:%s-%d new-subvolume: %s-%d)",
+               basefd, basefd->inode->gfid, old_subvol->name,
+               old_subvol->graph->id, new_subvol->name, new_subvol->graph->id);
+    }
+
+    ret = fuse_migrate_fd_open(this, basefd, oldfd, old_subvol, new_subvol);
+    if (ret < 0) {
+        gf_log(this->name, GF_LOG_WARNING,
+               "open corresponding to "
+               "basefd (ptr:%p inode-gfid:%s) in new graph failed "
+               "(old-subvolume:%s-%d new-subvolume:%s-%d)",
+               basefd, uuid_utoa(basefd->inode->gfid), old_subvol->name,
+               old_subvol->graph->id, new_subvol->name, new_subvol->graph->id);
+        goto out;
+    }
+
+    ret = fuse_migrate_locks(this, basefd, oldfd, old_subvol, new_subvol);
+    if (ret < 0) {
+        gf_log(this->name, GF_LOG_WARNING,
+               "migrating locks from old-subvolume (%s-%d) to "
+               "new-subvolume (%s-%d) failed (inode-gfid:%s oldfd:%p "
+               "basefd:%p)",
+               old_subvol->name, old_subvol->graph->id, new_subvol->name,
+               new_subvol->graph->id, uuid_utoa(basefd->inode->gfid), oldfd,
+               basefd);
+    }
 out:
-        if (ret < 0) {
-                gf_log (this->name, GF_LOG_WARNING, "migration of basefd "
-                        "(ptr:%p inode-gfid:%s) failed"
-                        "(old-subvolume:%s-%d new-subvolume:%s-%d)", basefd,
-                        oldfd ? uuid_utoa (oldfd->inode->gfid) : NULL,
-                        old_subvol->name, old_subvol->graph->id,
-                        new_subvol->name, new_subvol->graph->id);
-        }
+    if (ret < 0) {
+        gf_log(this->name, GF_LOG_WARNING,
+               "migration of basefd "
+               "(ptr:%p inode-gfid:%s) failed"
+               "(old-subvolume:%s-%d new-subvolume:%s-%d)",
+               basefd, oldfd ? uuid_utoa(oldfd->inode->gfid) : NULL,
+               old_subvol->name, old_subvol->graph->id, new_subvol->name,
+               new_subvol->graph->id);
+    }
 
-        fd_unref (oldfd);
+    fd_unref(oldfd);
 
-        return ret;
-}
+    if (xdata)
+        dict_unref(xdata);
 
+    return ret;
+}
 
 int
-fuse_handle_opened_fds (xlator_t *this, xlator_t *old_subvol,
-                        xlator_t *new_subvol)
-{
-        fuse_private_t *priv      = NULL;
-        fdentry_t      *fdentries = NULL;
-        uint32_t        count     = 0;
-        fdtable_t      *fdtable   = NULL;
-        int             i         = 0;
-        fd_t           *fd        = NULL;
-        int32_t         ret       = 0;
-        fuse_fd_ctx_t  *fdctx     = NULL;
-
-        priv = this->private;
-
-        fdtable = priv->fdtable;
-
-        fdentries = gf_fd_fdtable_copy_all_fds (fdtable, &count);
-        if (fdentries != NULL) {
-                for (i = 0; i < count; i++) {
-                        fd = fdentries[i].fd;
-                        if (fd == NULL)
-                                continue;
-
-                        ret = fuse_migrate_fd (this, fd, old_subvol,
-                                               new_subvol);
-
-                        fdctx = fuse_fd_ctx_get (this, fd);
-                        if (fdctx) {
-                                LOCK (&fd->lock);
-                                {
-                                        if (ret < 0) {
-                                                fdctx->migration_failed = 1;
-                                        } else {
-                                                fdctx->migration_failed = 0;
-                                        }
-                                }
-                                UNLOCK (&fd->lock);
-                        }
-                }
+fuse_handle_opened_fds(xlator_t *this, xlator_t *old_subvol,
+                       xlator_t *new_subvol)
+{
+    fuse_private_t *priv = NULL;
+    fdentry_t *fdentries = NULL;
+    uint32_t count = 0;
+    fdtable_t *fdtable = NULL;
+    int i = 0;
+    fd_t *fd = NULL;
+    int32_t ret = 0;
+    fuse_fd_ctx_t *fdctx = NULL;
+
+    priv = this->private;
+
+    fdtable = priv->fdtable;
+
+    fdentries = gf_fd_fdtable_copy_all_fds(fdtable, &count);
+    if (fdentries != NULL) {
+        for (i = 0; i < count; i++) {
+            fd = fdentries[i].fd;
+            if (fd == NULL)
+                continue;
 
-                for (i = 0; i < count ; i++) {
-                        fd = fdentries[i].fd;
-                        if (fd)
-                                fd_unref (fd);
+            ret = fuse_migrate_fd(this, fd, old_subvol, new_subvol);
+
+            fdctx = fuse_fd_ctx_get(this, fd);
+            if (fdctx) {
+                LOCK(&fd->lock);
+                {
+                    if (ret < 0) {
+                        fdctx->migration_failed = 1;
+                    } else {
+                        fdctx->migration_failed = 0;
+                    }
                 }
+                UNLOCK(&fd->lock);
+            }
+        }
 
-                GF_FREE (fdentries);
+        for (i = 0; i < count; i++) {
+            fd = fdentries[i].fd;
+            if (fd)
+                fd_unref(fd);
         }
 
-        return 0;
-}
+        GF_FREE(fdentries);
+    }
 
+    return 0;
+}
 
 static int
-fuse_handle_blocked_locks (xlator_t *this, xlator_t *old_subvol,
-                           xlator_t *new_subvol)
+fuse_handle_blocked_locks(xlator_t *this, xlator_t *old_subvol,
+                          xlator_t *new_subvol)
 {
-        return 0;
+    return 0;
 }
 
-
 static int
-fuse_graph_switch_task (void *data)
+fuse_graph_switch_task(void *data)
 {
-        fuse_graph_switch_args_t *args = NULL;
+    fuse_graph_switch_args_t *args = NULL;
 
-        args = data;
-        if (args == NULL) {
-                goto out;
-        }
+    args = data;
+    if (args == NULL) {
+        goto out;
+    }
 
-        /* don't change the order of handling open fds and blocked locks, since
-         * the act of opening files also reacquires granted locks in new graph.
-         */
-        fuse_handle_opened_fds (args->this, args->old_subvol, args->new_subvol);
+    /* don't change the order of handling open fds and blocked locks, since
+     * the act of opening files also reacquires granted locks in new graph.
+     */
+    fuse_handle_opened_fds(args->this, args->old_subvol, args->new_subvol);
 
-        fuse_handle_blocked_locks (args->this, args->old_subvol,
-                                   args->new_subvol);
+    fuse_handle_blocked_locks(args->this, args->old_subvol, args->new_subvol);
 
 out:
-        return 0;
+    return 0;
 }
 
-
 fuse_graph_switch_args_t *
-fuse_graph_switch_args_alloc (void)
+fuse_graph_switch_args_alloc(void)
 {
-        fuse_graph_switch_args_t *args = NULL;
+    fuse_graph_switch_args_t *args = NULL;
 
-        args = GF_CALLOC (1, sizeof (*args), gf_fuse_mt_graph_switch_args_t);
-        if (args == NULL) {
-                goto out;
-        }
+    args = GF_CALLOC(1, sizeof(*args), gf_fuse_mt_graph_switch_args_t);
+    if (args == NULL) {
+        goto out;
+    }
 
 out:
-        return args;
+    return args;
 }
 
-
 void
-fuse_graph_switch_args_destroy (fuse_graph_switch_args_t *args)
+fuse_graph_switch_args_destroy(fuse_graph_switch_args_t *args)
 {
-        if (args == NULL) {
-                goto out;
-        }
+    if (args == NULL) {
+        goto out;
+    }
 
-        GF_FREE (args);
+    GF_FREE(args);
 out:
-        return;
+    return;
 }
 
-
 int
-fuse_handle_graph_switch (xlator_t *this, xlator_t *old_subvol,
-                          xlator_t *new_subvol)
-{
-        call_frame_t             *frame = NULL;
-        int32_t                   ret   = -1;
-        fuse_graph_switch_args_t *args  = NULL;
-
-        frame = create_frame (this, this->ctx->pool);
-        if (frame == NULL) {
-                goto out;
-        }
-
-        args = fuse_graph_switch_args_alloc ();
-        if (args == NULL) {
-                goto out;
-        }
-
-        args->this = this;
-        args->old_subvol = old_subvol;
-        args->new_subvol = new_subvol;
-
-        ret = synctask_new (this->ctx->env, fuse_graph_switch_task, NULL, frame,
-                            args);
-        if (ret == -1) {
-                gf_log (this->name, GF_LOG_WARNING, "starting sync-task to "
-                        "handle graph switch failed");
-                goto out;
-        }
-
-        ret = 0;
+fuse_handle_graph_switch(xlator_t *this, xlator_t *old_subvol,
+                         xlator_t *new_subvol)
+{
+    call_frame_t *frame = NULL;
+    int32_t ret = -1;
+    fuse_graph_switch_args_t *args = NULL;
+
+    frame = create_frame(this, this->ctx->pool);
+    if (frame == NULL) {
+        goto out;
+    }
+
+    args = fuse_graph_switch_args_alloc();
+    if (args == NULL) {
+        goto out;
+    }
+
+    args->this = this;
+    args->old_subvol = old_subvol;
+    args->new_subvol = new_subvol;
+
+    ret = synctask_new(this->ctx->env, fuse_graph_switch_task, NULL, frame,
+                       args);
+    if (ret == -1) {
+        gf_log(this->name, GF_LOG_WARNING,
+               "starting sync-task to "
+               "handle graph switch failed");
+        goto out;
+    }
+
+    ret = 0;
 out:
-        if (args != NULL) {
-                fuse_graph_switch_args_destroy (args);
-        }
+    if (args != NULL) {
+        fuse_graph_switch_args_destroy(args);
+    }
 
-        if (frame != NULL) {
-                STACK_DESTROY (frame->root);
-        }
+    if (frame != NULL) {
+        STACK_DESTROY(frame->root);
+    }
 
-        return ret;
+    return ret;
 }
 
-
 int
-fuse_graph_sync (xlator_t *this)
-{
-        fuse_private_t *priv                = NULL;
-        int             need_first_lookup   = 0;
-        int             ret                 = 0;
-        xlator_t       *old_subvol          = NULL, *new_subvol = NULL;
-        uint64_t        winds_on_old_subvol = 0;
+fuse_graph_sync(xlator_t *this)
+{
+    fuse_private_t *priv = NULL;
+    int need_first_lookup = 0;
+    int ret = 0;
+    int new_graph_id = 0;
+    xlator_t *old_subvol = NULL, *new_subvol = NULL;
+    uint64_t winds_on_old_subvol = 0;
+
+    priv = this->private;
+
+    pthread_mutex_lock(&priv->sync_mutex);
+    {
+        if (!priv->next_graph)
+            goto unlock;
+
+        old_subvol = priv->active_subvol;
+        new_subvol = priv->active_subvol = priv->next_graph->top;
+        new_graph_id = priv->next_graph->id;
+        priv->next_graph = NULL;
+        need_first_lookup = 1;
+        if (old_subvol) {
+            priv->handle_graph_switch = _gf_true;
+        }
+
+        while (!priv->event_recvd) {
+            ret = pthread_cond_wait(&priv->sync_cond, &priv->sync_mutex);
+            if (ret != 0) {
+                gf_log(this->name, GF_LOG_DEBUG,
+                       "timedwait returned non zero value "
+                       "ret: %d errno: %d",
+                       ret, errno);
+                break;
+            }
+        }
+    }
+unlock:
+    pthread_mutex_unlock(&priv->sync_mutex);
 
-        priv = this->private;
+    if (need_first_lookup) {
+        gf_log("fuse", GF_LOG_INFO, "switched to graph %d", new_graph_id);
+        fuse_first_lookup(this);
+    }
 
-        pthread_mutex_lock (&priv->sync_mutex);
+    if ((old_subvol != NULL) && (new_subvol != NULL)) {
+        fuse_handle_graph_switch(this, old_subvol, new_subvol);
+
+        pthread_mutex_lock(&priv->sync_mutex);
         {
-                if (!priv->next_graph)
-                        goto unlock;
-
-                old_subvol = priv->active_subvol;
-                new_subvol = priv->active_subvol = priv->next_graph->top;
-                priv->next_graph = NULL;
-                need_first_lookup = 1;
-
-                while (!priv->event_recvd) {
-                        ret = pthread_cond_wait (&priv->sync_cond,
-                                                 &priv->sync_mutex);
-                        if (ret != 0) {
-                                  gf_log (this->name, GF_LOG_DEBUG,
-                                          "timedwait returned non zero value "
-                                          "ret: %d errno: %d", ret, errno);
-                                  break;
-                        }
-                }
+            old_subvol->switched = 1;
+            winds_on_old_subvol = old_subvol->winds;
+            priv->handle_graph_switch = _gf_false;
+            pthread_cond_broadcast(&priv->migrate_cond);
         }
-unlock:
-        pthread_mutex_unlock (&priv->sync_mutex);
+        pthread_mutex_unlock(&priv->sync_mutex);
 
-        if (need_first_lookup) {
-                fuse_first_lookup (this);
+        if (winds_on_old_subvol == 0) {
+            xlator_notify(old_subvol, GF_EVENT_PARENT_DOWN, old_subvol, NULL);
         }
+    }
 
-        if ((old_subvol != NULL) && (new_subvol != NULL)) {
-                fuse_handle_graph_switch (this, old_subvol, new_subvol);
+    return 0;
+}
 
-                pthread_mutex_lock (&priv->sync_mutex);
-                {
-                        old_subvol->switched = 1;
-                        winds_on_old_subvol = old_subvol->winds;
-                }
-                pthread_mutex_unlock (&priv->sync_mutex);
+int
+fuse_get_mount_status(xlator_t *this)
+{
+    int kid_status = -1;
+    fuse_private_t *priv = this->private;
 
-                if (winds_on_old_subvol == 0) {
-                        xlator_notify (old_subvol, GF_EVENT_PARENT_DOWN,
-                                       old_subvol, NULL);
-                }
-        }
+    if (sys_read(priv->status_pipe[0], &kid_status, sizeof(kid_status)) < 0) {
+        gf_log(this->name, GF_LOG_ERROR, "could not get mount status");
+        kid_status = -1;
+    }
+    gf_log(this->name, GF_LOG_DEBUG, "mount status is %d", kid_status);
 
-        return 0;
+    sys_close(priv->status_pipe[0]);
+    sys_close(priv->status_pipe[1]);
+    return kid_status;
 }
 
-int
-fuse_get_mount_status (xlator_t *this)
+static void
+fuse_dispatch(xlator_t *xl, gf_async_t *async)
 {
-        int             kid_status = -1;
-        fuse_private_t *priv = this->private;
+    fuse_async_t *fasync;
+    fuse_private_t *priv;
+    fuse_in_header_t *finh;
+    struct iobuf *iobuf;
 
-        if (read(priv->status_pipe[0],&kid_status, sizeof(kid_status)) < 0) {
-                gf_log (this->name, GF_LOG_ERROR, "could not get mount status");
-                kid_status = -1;
-        }
-        gf_log (this->name, GF_LOG_DEBUG, "mount status is %d", kid_status);
+    priv = xl->private;
+    fasync = caa_container_of(async, fuse_async_t, async);
+    finh = fasync->finh;
+    iobuf = fasync->iobuf;
 
-        close(priv->status_pipe[0]);
-        close(priv->status_pipe[1]);
-        return kid_status;
+    priv->fuse_ops[finh->opcode](xl, finh, fasync->msg, iobuf);
+
+    iobuf_unref(iobuf);
 }
 
+/* We need 512 extra buffer size for BATCH_FORGET fop. By tests, it is
+ * found to be reduces 'REALLOC()' in the loop */
+#define FUSE_EXTRA_ALLOC 512
+
 static void *
-fuse_thread_proc (void *data)
-{
-        char                     *mount_point = NULL;
-        xlator_t                 *this = NULL;
-        fuse_private_t           *priv = NULL;
-        ssize_t                   res = 0;
-        struct iobuf             *iobuf = NULL;
-        fuse_in_header_t         *finh = NULL;
-        struct iovec              iov_in[2];
-        void                     *msg = NULL;
-        const size_t              msg0_size = sizeof (*finh) + 128;
-        fuse_handler_t          **fuse_ops = NULL;
-        struct pollfd             pfd[2] = {{0,}};
-        gf_boolean_t              mount_finished = _gf_false;
-
-        this = data;
-        priv = this->private;
-        fuse_ops = priv->fuse_ops;
+fuse_thread_proc(void *data)
+{
+    char *mount_point = NULL;
+    xlator_t *this = NULL;
+    fuse_private_t *priv = NULL;
+    ssize_t res = 0;
+    struct iobuf *iobuf = NULL;
+    fuse_in_header_t *finh = NULL;
+    struct iovec iov_in[2] = {
+        {
+            0,
+        },
+    };
 
-        THIS = this;
+    void *msg = NULL;
+    size_t msg0_size = sizeof(*finh) + sizeof(struct fuse_write_in);
+    fuse_async_t *fasync;
+    struct pollfd pfd[2] = {{
+        0,
+    }};
+    uint32_t psize;
 
-        iov_in[0].iov_len = sizeof (*finh) + sizeof (struct fuse_write_in);
-        iov_in[1].iov_len = ((struct iobuf_pool *)this->ctx->iobuf_pool)
-                              ->default_page_size;
-        priv->msg0_len_p = &iov_in[0].iov_len;
-
-        for (;;) {
-                /* THIS has to be reset here */
-                THIS = this;
-
-                if (!mount_finished) {
-                        memset(pfd,0,sizeof(pfd));
-                        pfd[0].fd = priv->status_pipe[0];
-                        pfd[0].events = POLLIN | POLLHUP | POLLERR;
-                        pfd[1].fd = priv->fd;
-                        pfd[1].events = POLLIN | POLLHUP | POLLERR;
-                        if (poll(pfd,2,-1) < 0) {
-                                gf_log (this->name, GF_LOG_ERROR,
-                                        "poll error %s", strerror(errno));
-                                break;
-                        }
-                        if (pfd[0].revents & POLLIN) {
-                                if (fuse_get_mount_status(this) != 0) {
-                                        break;
-                                }
-                                mount_finished = _gf_true;
-                        }
-                        else if (pfd[0].revents) {
-                                gf_log (this->name, GF_LOG_ERROR,
-                                        "mount pipe closed without status");
-                                break;
-                        }
-                        if (!pfd[1].revents) {
-                                continue;
-                        }
-                }
+    this = data;
+    priv = this->private;
 
-                /*
-                 * We don't want to block on readv while we're still waiting
-                 * for mount status.  That means we only want to get here if
-                 * mount_status is true (meaning that our wait completed
-                 * already) or if we already called poll(2) on priv->fd to
-                 * make sure it's ready.
-                 */
+    THIS = this;
 
-                if (priv->init_recvd)
-                        fuse_graph_sync (this);
-
-                /* TODO: This place should always get maximum supported buffer
-                   size from 'fuse', which is as of today 128KB. If we bring in
-                   support for higher block sizes support, then we should be
-                   changing this one too */
-                iobuf = iobuf_get (this->ctx->iobuf_pool);
-
-                /* Add extra 128 byte to the first iov so that it can
-                 * accommodate "ordinary" non-write requests. It's not
-                 * guaranteed to be big enough, as SETXATTR and namespace
-                 * operations with very long names may grow behind it,
-                 * but it's good enough in most cases (and we can handle
-                 * rest via realloc).
-                 */
-                iov_in[0].iov_base = GF_CALLOC (1, msg0_size,
-                                                gf_fuse_mt_iov_base);
-
-                if (!iobuf || !iov_in[0].iov_base) {
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "Out of memory");
-                        if (iobuf)
-                                iobuf_unref (iobuf);
-                        GF_FREE (iov_in[0].iov_base);
-                        sleep (10);
-                        continue;
-                }
+    psize = ((struct iobuf_pool *)this->ctx->iobuf_pool)->default_page_size;
+    priv->msg0_len_p = &msg0_size;
 
-                iov_in[1].iov_base = iobuf->ptr;
-
-                res = readv (priv->fd, iov_in, 2);
-
-                if (res == -1) {
-                        if (errno == ENODEV || errno == EBADF) {
-                                gf_log ("glusterfs-fuse", GF_LOG_DEBUG,
-                                        "terminating upon getting %s when "
-                                        "reading /dev/fuse",
-                                        errno == ENODEV ? "ENODEV" : "EBADF");
-                                fuse_log_eh (this, "glusterfs-fuse: terminating"
-                                             " upon getting %s when "
-                                             "reading /dev/fuse",
-                                             errno == ENODEV ? "ENODEV":
-                                             "EBADF");
-                                break;
-                        }
-                        if (errno != EINTR) {
-                                gf_log ("glusterfs-fuse", GF_LOG_WARNING,
-                                        "read from /dev/fuse returned -1 (%s)",
-                                        strerror (errno));
-                                fuse_log_eh (this, "glusterfs-fuse: read from "
-                                             "/dev/fuse returned -1 (%s)",
-                                             strerror (errno));
-                        }
+    for (;;) {
+        /* THIS has to be reset here */
+        THIS = this;
 
-                        goto cont_err;
+        pthread_mutex_lock(&priv->sync_mutex);
+        {
+            if (!priv->mount_finished) {
+                memset(pfd, 0, sizeof(pfd));
+                pfd[0].fd = priv->status_pipe[0];
+                pfd[0].events = POLLIN | POLLHUP | POLLERR;
+                pfd[1].fd = priv->fd;
+                pfd[1].events = POLLIN | POLLHUP | POLLERR;
+                if (poll(pfd, 2, -1) < 0) {
+                    gf_log(this->name, GF_LOG_ERROR, "poll error %s",
+                           strerror(errno));
+                    pthread_mutex_unlock(&priv->sync_mutex);
+                    break;
                 }
-                if (res < sizeof (finh)) {
-                        gf_log ("glusterfs-fuse", GF_LOG_WARNING,
-                                "short read on /dev/fuse");
-                        fuse_log_eh (this, "glusterfs-fuse: short read on "
-                                     "/dev/fuse");
+                if (pfd[0].revents & POLLIN) {
+                    if (fuse_get_mount_status(this) != 0) {
+                        pthread_mutex_unlock(&priv->sync_mutex);
                         break;
+                    }
+                    priv->mount_finished = _gf_true;
+                } else if (pfd[0].revents) {
+                    gf_log(this->name, GF_LOG_ERROR,
+                           "mount pipe closed without status");
+                    pthread_mutex_unlock(&priv->sync_mutex);
+                    break;
                 }
+                if (!pfd[1].revents) {
+                    pthread_mutex_unlock(&priv->sync_mutex);
+                    continue;
+                }
+            }
+        }
+        pthread_mutex_unlock(&priv->sync_mutex);
 
-                finh = (fuse_in_header_t *)iov_in[0].iov_base;
+        /*
+         * We don't want to block on readv while we're still waiting
+         * for mount status.  That means we only want to get here if
+         * mount_status is true (meaning that our wait completed
+         * already) or if we already called poll(2) on priv->fd to
+         * make sure it's ready.
+         */
 
-                if (res != finh->len
-#ifdef GF_DARWIN_HOST_OS
-                    /* work around fuse4bsd/MacFUSE msg size miscalculation bug,
-                     * that is, payload size is not taken into account for
-                     * buffered writes
+        if (priv->init_recvd)
+            fuse_graph_sync(this);
+
+        /* TODO: This place should always get maximum supported buffer
+           size from 'fuse', which is as of today 128KB. If we bring in
+           support for higher block sizes support, then we should be
+           changing this one too */
+        iobuf = iobuf_get(this->ctx->iobuf_pool);
+
+        /* Add extra 512 byte to the first iov so that it can
+         * accommodate "ordinary" non-write requests. It's not
+         * guaranteed to be big enough, as SETXATTR and namespace
+         * operations with very long names may grow behind it,
+         * but it's good enough in most cases (and we can handle
+         * rest via realloc). */
+        iov_in[0].iov_base = GF_MALLOC(
+            sizeof(fuse_async_t) + msg0_size + FUSE_EXTRA_ALLOC,
+            gf_fuse_mt_iov_base);
+
+        if (!iobuf || !iov_in[0].iov_base) {
+            gf_log(this->name, GF_LOG_ERROR, "Out of memory");
+            if (iobuf)
+                iobuf_unref(iobuf);
+            GF_FREE(iov_in[0].iov_base);
+            sleep(10);
+            continue;
+        }
+
+        iov_in[1].iov_base = iobuf->ptr;
+
+        iov_in[0].iov_len = msg0_size;
+        iov_in[1].iov_len = psize;
+
+        res = sys_readv(priv->fd, iov_in, 2);
+
+        if (res == -1) {
+            if (errno == ENODEV || errno == EBADF) {
+                gf_log("glusterfs-fuse", GF_LOG_DEBUG,
+                       "terminating upon getting %s when "
+                       "reading /dev/fuse",
+                       errno == ENODEV ? "ENODEV" : "EBADF");
+                fuse_log_eh(this,
+                            "glusterfs-fuse: terminating"
+                            " upon getting %s when "
+                            "reading /dev/fuse",
+                            errno == ENODEV ? "ENODEV" : "EBADF");
+                break;
+            }
+            if (errno != EINTR) {
+                gf_log("glusterfs-fuse", GF_LOG_WARNING,
+                       "read from /dev/fuse returned -1 (%s)", strerror(errno));
+                fuse_log_eh(this,
+                            "glusterfs-fuse: read from "
+                            "/dev/fuse returned -1 (%s)",
+                            strerror(errno));
+                if (errno == EPERM) {
+                    /*
+                     * sleep a while to avoid busy looping
+                     * on EPERM condition
                      */
-                    && !(finh->opcode == FUSE_WRITE &&
-                         finh->len == sizeof(*finh) + sizeof(struct fuse_write_in) &&
-                         res == finh->len + ((struct fuse_write_in *)(finh + 1))->size)
-#endif
-                   ) {
-                        gf_log ("glusterfs-fuse", GF_LOG_WARNING,
-                                "inconsistent read on /dev/fuse");
-                        fuse_log_eh (this, "glusterfs-fuse: inconsistent read "
-                                     "on /dev/fuse");
-                        break;
+                    nanosleep(
+                        &(struct timespec){0,
+                                           priv->fuse_dev_eperm_ratelimit_ns},
+                        NULL);
                 }
+            }
 
-                priv->iobuf = iobuf;
-
-                if (finh->opcode == FUSE_WRITE)
-                        msg = iov_in[1].iov_base;
-                else {
-                        if (res > msg0_size) {
-                                void *b = GF_REALLOC (iov_in[0].iov_base, res);
-                                if (b) {
-                                        iov_in[0].iov_base = b;
-                                        finh = (fuse_in_header_t *)
-                                                 iov_in[0].iov_base;
-                                }
-                                else {
-                                        gf_log ("glusterfs-fuse", GF_LOG_ERROR,
-                                                "Out of memory");
-                                        send_fuse_err (this, finh, ENOMEM);
-
-                                        goto cont_err;
-                                }
-                        }
+            goto cont_err;
+        }
+        if (res < sizeof(*finh)) {
+            gf_log("glusterfs-fuse", GF_LOG_WARNING, "short read on /dev/fuse");
+            fuse_log_eh(this,
+                        "glusterfs-fuse: short read on "
+                        "/dev/fuse");
+            break;
+        }
+
+        finh = (fuse_in_header_t *)iov_in[0].iov_base;
 
-                        if (res > iov_in[0].iov_len)
-                                memcpy (iov_in[0].iov_base + iov_in[0].iov_len,
-                                        iov_in[1].iov_base,
-                                        res - iov_in[0].iov_len);
+        if (res != finh->len
+#ifdef GF_DARWIN_HOST_OS
+            /* work around fuse4bsd/MacFUSE msg size miscalculation bug,
+             * that is, payload size is not taken into account for
+             * buffered writes
+             */
+            && !(finh->opcode == FUSE_WRITE &&
+                 finh->len == sizeof(*finh) + sizeof(struct fuse_write_in) &&
+                 res == finh->len + ((struct fuse_write_in *)(finh + 1))->size)
+#endif
+        ) {
+            gf_log("glusterfs-fuse", GF_LOG_WARNING,
+                   "inconsistent read on /dev/fuse");
+            fuse_log_eh(this,
+                        "glusterfs-fuse: inconsistent read "
+                        "on /dev/fuse");
+            break;
+        }
+
+        /*
+         * This can be moved around a bit, but it's important to do it
+         * *after* the readv.  Otherwise, a graph switch could occur
+         * while we're in readv and we'll process the next request on
+         * the old graph before we come to the part of the loop above
+         * readv and check again.  That would be wrong.
+         */
+        if (priv->init_recvd)
+            fuse_graph_sync(this);
 
-                        msg = finh + 1;
+        if (finh->opcode == FUSE_WRITE)
+            msg = iov_in[1].iov_base;
+        else {
+            if (res > msg0_size + FUSE_EXTRA_ALLOC) {
+                void *b = GF_REALLOC(iov_in[0].iov_base,
+                                     sizeof(fuse_async_t) + res);
+                if (b) {
+                    iov_in[0].iov_base = b;
+                    finh = (fuse_in_header_t *)iov_in[0].iov_base;
+                } else {
+                    gf_log("glusterfs-fuse", GF_LOG_ERROR, "Out of memory");
+                    send_fuse_err(this, finh, ENOMEM);
+
+                    goto cont_err;
                 }
-                if (priv->uid_map_root &&
-                    finh->uid == priv->uid_map_root)
-                        finh->uid = 0;
+            }
 
-                if (finh->opcode >= FUSE_OP_HIGH)
-                        /* turn down MacFUSE specific messages */
-                        fuse_enosys (this, finh, msg);
-                else
-                        fuse_ops[finh->opcode] (this, finh, msg);
+            if (res > iov_in[0].iov_len) {
+                memcpy(iov_in[0].iov_base + iov_in[0].iov_len,
+                       iov_in[1].iov_base, res - iov_in[0].iov_len);
+                iov_in[0].iov_len = res;
+            }
 
-                iobuf_unref (iobuf);
-                continue;
+            msg = finh + 1;
+        }
+        if (priv->uid_map_root && finh->uid == priv->uid_map_root)
+            finh->uid = 0;
 
- cont_err:
-                iobuf_unref (iobuf);
-                GF_FREE (iov_in[0].iov_base);
+        if (finh->opcode >= FUSE_OP_HIGH) {
+            /* turn down MacFUSE specific messages */
+            fuse_enosys(this, finh, msg, NULL);
+            iobuf_unref(iobuf);
+        } else {
+            fasync = iov_in[0].iov_base + iov_in[0].iov_len;
+            fasync->finh = finh;
+            fasync->msg = msg;
+            fasync->iobuf = iobuf;
+            gf_async(&fasync->async, this, fuse_dispatch);
         }
 
-        /*
-         * We could be in all sorts of states with respect to iobuf and iov_in
-         * by the time we get here, and it's just not worth untangling them if
-         * we're about to kill ourselves anyway.
-         */
+        continue;
 
-        if (dict_get (this->options, ZR_MOUNTPOINT_OPT))
-                mount_point = data_to_str (dict_get (this->options,
-                                                     ZR_MOUNTPOINT_OPT));
-        if (mount_point) {
-                gf_log (this->name, GF_LOG_INFO,
-                        "unmounting %s", mount_point);
-        }
+    cont_err:
+        iobuf_unref(iobuf);
+        GF_FREE(iov_in[0].iov_base);
+        iov_in[0].iov_base = NULL;
+    }
 
-        /* Kill the whole process, not just this thread. */
-        kill (getpid(), SIGTERM);
-        return NULL;
-}
+    if (iov_in[0].iov_base)
+        GF_FREE(iov_in[0].iov_base);
+
+    /*
+     * We could be in all sorts of states with respect to iobuf and iov_in
+     * by the time we get here, and it's just not worth untangling them if
+     * we're about to kill ourselves anyway.
+     */
 
+    if (dict_get(this->options, ZR_MOUNTPOINT_OPT))
+        mount_point = data_to_str(dict_get(this->options, ZR_MOUNTPOINT_OPT));
+    if (mount_point) {
+        gf_log(this->name, GF_LOG_INFO, "initiating unmount of %s",
+               mount_point);
+    }
+
+    /* Kill the whole process, not just this thread. */
+    kill(getpid(), SIGTERM);
+    return NULL;
+}
 
 int32_t
-fuse_itable_dump (xlator_t  *this)
+fuse_itable_dump(xlator_t *this)
 {
-        if (!this)
-                 return -1;
+    fuse_private_t *priv = NULL;
 
-        gf_proc_dump_add_section("xlator.mount.fuse.itable");
-        inode_table_dump(this->itable, "xlator.mount.fuse.itable");
+    if (!this)
+        return -1;
 
-        return 0;
+    priv = this->private;
+
+    if (priv && priv->active_subvol) {
+        gf_proc_dump_add_section("xlator.mount.fuse.itable");
+        inode_table_dump(priv->active_subvol->itable,
+                         "xlator.mount.fuse.itable");
+    }
+    return 0;
 }
 
 int32_t
-fuse_priv_dump (xlator_t  *this)
-{
-        fuse_private_t  *private = NULL;
-
-        if (!this)
-                return -1;
-
-        private = this->private;
-
-        if (!private)
-                return -1;
-
-        gf_proc_dump_add_section("xlator.mount.fuse.priv");
-
-        gf_proc_dump_write("fd", "%d", private->fd);
-        gf_proc_dump_write("proto_minor", "%u",
-                            private->proto_minor);
-        gf_proc_dump_write("volfile", "%s",
-                            private->volfile?private->volfile:"None");
-        gf_proc_dump_write("volfile_size", "%d",
-                            private->volfile_size);
-        gf_proc_dump_write("mount_point", "%s",
-                            private->mount_point);
-        gf_proc_dump_write("iobuf", "%u",
-                            private->iobuf);
-        gf_proc_dump_write("fuse_thread_started", "%d",
-                            (int)private->fuse_thread_started);
-        gf_proc_dump_write("direct_io_mode", "%d",
-                            private->direct_io_mode);
-        gf_proc_dump_write("entry_timeout", "%lf",
-                            private->entry_timeout);
-        gf_proc_dump_write("attribute_timeout", "%lf",
-                            private->attribute_timeout);
-        gf_proc_dump_write("init_recvd", "%d",
-                            (int)private->init_recvd);
-        gf_proc_dump_write("strict_volfile_check", "%d",
-                            (int)private->strict_volfile_check);
-        gf_proc_dump_write("reverse_thread_started", "%d",
-                           (int)private->reverse_fuse_thread_started);
-        gf_proc_dump_write("use_readdirp", "%d", private->use_readdirp);
+fuse_priv_dump(xlator_t *this)
+{
+    fuse_private_t *private = NULL;
 
-        return 0;
+    if (!this)
+        return -1;
+
+   private
+    = this->private;
+
+    if (!private)
+        return -1;
+
+    gf_proc_dump_add_section("xlator.mount.fuse.priv");
+
+    gf_proc_dump_write("fd", "%d", private->fd);
+    gf_proc_dump_write("proto_minor", "%u", private->proto_minor);
+    gf_proc_dump_write("volfile", "%s",
+                       private->volfile ? private->volfile : "None");
+    gf_proc_dump_write("volfile_size", "%" GF_PRI_SIZET, private->volfile_size);
+    gf_proc_dump_write("mount_point", "%s", private->mount_point);
+    gf_proc_dump_write("fuse_thread_started", "%d",
+                       (int)private->fuse_thread_started);
+    gf_proc_dump_write("direct_io_mode", "%d", private->direct_io_mode);
+    gf_proc_dump_write("entry_timeout", "%lf", private->entry_timeout);
+    gf_proc_dump_write("attribute_timeout", "%lf", private->attribute_timeout);
+    gf_proc_dump_write("init_recvd", "%d", (int)private->init_recvd);
+    gf_proc_dump_write("strict_volfile_check", "%d",
+                       (int)private->strict_volfile_check);
+    gf_proc_dump_write("timed_response_thread_started", "%d",
+                       (int)private->timed_response_fuse_thread_started);
+    gf_proc_dump_write("reverse_thread_started", "%d",
+                       (int)private->reverse_fuse_thread_started);
+    gf_proc_dump_write("invalidate_limit", "%u", private->invalidate_limit);
+    gf_proc_dump_write("invalidate_queue_length", "%" PRIu64,
+                       private->invalidate_count);
+    gf_proc_dump_write("use_readdirp", "%d", private->use_readdirp);
+
+    return 0;
 }
 
 int
-fuse_history_dump (xlator_t *this)
+fuse_history_dump(xlator_t *this)
 {
-        int      ret    = -1;
-        char key_prefix[GF_DUMP_MAX_BUF_LEN] = {0,};
+    int ret = -1;
+    char key_prefix[GF_DUMP_MAX_BUF_LEN] = {
+        0,
+    };
+    fuse_private_t *priv = this->private;
 
-        GF_VALIDATE_OR_GOTO ("fuse", this, out);
-        GF_VALIDATE_OR_GOTO (this->name, this->history, out);
+    GF_VALIDATE_OR_GOTO("fuse", this, out);
 
-        gf_proc_dump_build_key (key_prefix, "xlator.mount.fuse",
-                                "history");
-        gf_proc_dump_add_section (key_prefix);
-        eh_dump (this->history, NULL, dump_history_fuse);
+    if (!priv->event_history)
+        goto out;
 
-        ret = 0;
+    GF_VALIDATE_OR_GOTO(this->name, this->history, out);
+
+    gf_proc_dump_build_key(key_prefix, "xlator.mount.fuse", "history");
+    gf_proc_dump_add_section("%s", key_prefix);
+    eh_dump(this->history, NULL, dump_history_fuse);
+
+    ret = 0;
 out:
-        return ret;
+    return ret;
 }
 
 int
-dump_history_fuse (circular_buffer_t *cb, void *data)
+dump_history_fuse(circular_buffer_t *cb, void *data)
 {
-        char       timestr[256] = {0,};
+    char timestr[GF_TIMESTR_SIZE] = {
+        0,
+    };
 
-        gf_time_fmt (timestr, sizeof timestr, cb->tv.tv_sec, gf_timefmt_F_HMS);
+    gf_time_fmt_tv(timestr, sizeof timestr, &cb->tv, gf_timefmt_F_HMS);
 
-        snprintf (timestr + strlen (timestr), 256 - strlen (timestr),
-                  ".%"GF_PRI_SUSECONDS, cb->tv.tv_usec);
-        gf_proc_dump_write ("TIME", "%s", timestr);
+    gf_proc_dump_write("TIME", "%s", timestr);
 
-        gf_proc_dump_write ("message", "%s\n", cb->data);
+    gf_proc_dump_write("message", "%s\n", (char *)cb->data);
 
-        return 0;
+    return 0;
 }
 
 int
-fuse_graph_setup (xlator_t *this, glusterfs_graph_t *graph)
+fuse_graph_setup(xlator_t *this, glusterfs_graph_t *graph)
 {
-        inode_table_t     *itable     = NULL;
-        int                ret        = 0, winds = 0;
-        fuse_private_t    *priv       = NULL;
-        glusterfs_graph_t *prev_graph = NULL;
-
-        priv = this->private;
-
-        pthread_mutex_lock (&priv->sync_mutex);
-        {
-                /* handle the case of more than one CHILD_UP on same graph */
-                if ((priv->active_subvol == graph->top) || graph->used) {
-                        goto unlock;
-                }
+    inode_table_t *itable = NULL;
+    int ret = 0, winds = 0;
+    fuse_private_t *priv = NULL;
+    glusterfs_graph_t *prev_graph = NULL;
 
-                itable = inode_table_new (0, graph->top);
-                if (!itable) {
-                        ret = -1;
-                        goto unlock;
-                }
+    priv = this->private;
 
-                ((xlator_t *)graph->top)->itable = itable;
+    pthread_mutex_lock(&priv->sync_mutex);
+    {
+        /* 1. handle the case of more than one CHILD_UP on same graph.
+         * 2. make sure graph is newer than current active_subvol.
+         */
+        if ((priv->active_subvol == graph->top) || graph->used ||
+            ((priv->active_subvol) &&
+             (priv->active_subvol->graph->id > graph->id))) {
+            goto unlock;
+        }
 
-                prev_graph = priv->next_graph;
+#if FUSE_KERNEL_MINOR_VERSION >= 11
+        itable = inode_table_with_invalidator(priv->lru_limit, graph->top,
+                                              fuse_inode_invalidate_fn, this);
+#else
+        itable = inode_table_new(0, graph->top);
+#endif
+        if (!itable) {
+            ret = -1;
+            goto unlock;
+        }
 
-                if ((prev_graph != NULL) && (prev_graph->id > graph->id)) {
-                        /* there was a race and an old graph was initialised
-                         * before new one.
-                         */
-                        prev_graph = graph;
-                } else {
-                        priv->next_graph = graph;
-                        priv->event_recvd = 0;
-                }
+        ((xlator_t *)graph->top)->itable = itable;
 
-                if (prev_graph != NULL)
-                        winds = ((xlator_t *)prev_graph->top)->winds;
+        prev_graph = priv->next_graph;
 
-		/* set post initializing next_graph i to preserve
-                 * critical section update and bails on error */
-                graph->used = 1;
+        if ((prev_graph != NULL) && (prev_graph->id > graph->id)) {
+            /* there was a race and an old graph was initialised
+             * before new one.
+             */
+            prev_graph = graph;
+        } else {
+            priv->next_graph = graph;
+            priv->event_recvd = 0;
         }
-        pthread_mutex_unlock (&priv->sync_mutex);
 
-        if ((prev_graph != NULL) && (winds == 0)) {
-                xlator_notify (prev_graph->top, GF_EVENT_PARENT_DOWN,
-                               prev_graph->top, NULL);
-        }
+        if (prev_graph != NULL)
+            winds = ((xlator_t *)prev_graph->top)->winds;
 
-        gf_log ("fuse", GF_LOG_INFO, "switched to graph %d",
-                ((graph) ? graph->id : 0));
+        /* set post initializing next_graph i to preserve
+         * critical section update and bails on error */
+        graph->used = 1;
+    }
+    pthread_mutex_unlock(&priv->sync_mutex);
 
-        return ret;
+    if ((prev_graph != NULL) && (winds == 0)) {
+        xlator_notify(prev_graph->top, GF_EVENT_PARENT_DOWN, prev_graph->top,
+                      NULL);
+    }
+
+    return ret;
 unlock:
-        pthread_mutex_unlock (&priv->sync_mutex);
+    pthread_mutex_unlock(&priv->sync_mutex);
 
-        return ret;
+    return ret;
 }
 
-
 int
-notify (xlator_t *this, int32_t event, void *data, ...)
+notify(xlator_t *this, int32_t event, void *data, ...)
 {
-        int32_t             ret     = 0;
-        fuse_private_t     *private = NULL;
-        gf_boolean_t        start_thread = _gf_false;
-        glusterfs_graph_t  *graph = NULL;
+    int i = 0;
+    int32_t ret = 0;
+    fuse_private_t *private = NULL;
+    gf_boolean_t start_thread = _gf_false;
+    glusterfs_graph_t *graph = NULL;
+    struct pollfd pfd = {0};
 
-        private = this->private;
+   private
+    = this->private;
 
-        graph = data;
+    graph = data;
 
-        gf_log ("fuse", GF_LOG_DEBUG, "got event %d on graph %d",
-                event, ((graph) ? graph->id : 0));
+    gf_log("fuse", GF_LOG_DEBUG, "got event %d on graph %d", event,
+           ((graph) ? graph->id : 0));
 
-        switch (event)
-        {
+    switch (event) {
         case GF_EVENT_GRAPH_NEW:
-                break;
+            break;
 
         case GF_EVENT_CHILD_UP:
         case GF_EVENT_CHILD_DOWN:
-        case GF_EVENT_CHILD_CONNECTING:
-        {
-                if (graph) {
-                        ret = fuse_graph_setup (this, graph);
-                        if (ret)
-                                gf_log (this->name, GF_LOG_WARNING,
-                                        "failed to setup the graph");
+        case GF_EVENT_CHILD_CONNECTING: {
+            if (graph) {
+                ret = fuse_graph_setup(this, graph);
+                if (ret)
+                    gf_log(this->name, GF_LOG_WARNING,
+                           "failed to setup the graph");
+            }
+
+            if ((event == GF_EVENT_CHILD_UP) ||
+                (event == GF_EVENT_CHILD_DOWN)) {
+                pthread_mutex_lock(&private->sync_mutex);
+                {
+                   private
+                    ->event_recvd = 1;
+                    pthread_cond_broadcast(&private->sync_cond);
                 }
-
-                if ((event == GF_EVENT_CHILD_UP)
-                    || (event == GF_EVENT_CHILD_DOWN)) {
-                        pthread_mutex_lock (&private->sync_mutex);
-                        {
-                                private->event_recvd = 1;
-                                pthread_cond_broadcast (&private->sync_cond);
-                        }
-                        pthread_mutex_unlock (&private->sync_mutex);
+                pthread_mutex_unlock(&private->sync_mutex);
+            }
+
+            pthread_mutex_lock(&private->sync_mutex);
+            {
+                if (!private->fuse_thread_started) {
+                   private
+                    ->fuse_thread_started = 1;
+                    start_thread = _gf_true;
                 }
-
-                pthread_mutex_lock (&private->sync_mutex);
-                {
-                        if (!private->fuse_thread_started) {
-                                private->fuse_thread_started = 1;
-                                start_thread = _gf_true;
-                        }
+            }
+            pthread_mutex_unlock(&private->sync_mutex);
+
+            if (start_thread) {
+               private
+                ->fuse_thread = GF_CALLOC(private->reader_thread_count,
+                                          sizeof(pthread_t),
+                                          gf_fuse_mt_pthread_t);
+                for (i = 0; i < private->reader_thread_count; i++) {
+                    ret = gf_thread_create(&private->fuse_thread[i], NULL,
+                                           fuse_thread_proc, this, "fuseproc");
+                    if (ret != 0) {
+                        gf_log(this->name, GF_LOG_DEBUG,
+                               "pthread_create() failed (%s)", strerror(errno));
+                        break;
+                    }
                 }
-                pthread_mutex_unlock (&private->sync_mutex);
-
-                if (start_thread) {
-                        ret = gf_thread_create (&private->fuse_thread, NULL,
-						fuse_thread_proc, this);
-                        if (ret != 0) {
-                                gf_log (this->name, GF_LOG_DEBUG,
-                                        "pthread_create() failed (%s)",
-                                        strerror (errno));
-                                break;
+            }
+
+            break;
+        }
+
+        case GF_EVENT_AUTH_FAILED: {
+            /* Authentication failure is an error and glusterfs should stop */
+            gf_log(this->name, GF_LOG_ERROR,
+                   "Server authenication failed. Shutting down.");
+            pthread_mutex_lock(&private->sync_mutex);
+            {
+                /*Wait for mount to finish*/
+                if (!private->mount_finished) {
+                    pfd.fd = private->status_pipe[0];
+                    pfd.events = POLLIN | POLLHUP | POLLERR;
+                    if (poll(&pfd, 1, -1) < 0) {
+                        gf_log(this->name, GF_LOG_ERROR, "poll error %s",
+                               strerror(errno));
+                        goto auth_fail_unlock;
+                    }
+                    if (pfd.revents & POLLIN) {
+                        if (fuse_get_mount_status(this) != 0) {
+                            goto auth_fail_unlock;
                         }
+                       private
+                        ->mount_finished = _gf_true;
+                    } else if (pfd.revents) {
+                        gf_log(this->name, GF_LOG_ERROR,
+                               "mount pipe closed without status");
+                        goto auth_fail_unlock;
+                    }
                 }
-
-                break;
-        }
-
-        case GF_EVENT_AUTH_FAILED:
-        {
-                /* Authentication failure is an error and glusterfs should stop */
-               gf_log (this->name, GF_LOG_ERROR, "Server authenication failed. Shutting down.");
-               fini (this);
-               break;
+            }
+        auth_fail_unlock:
+            pthread_mutex_unlock(&private->sync_mutex);
+            fini(this);
+            break;
         }
 
         default:
-                break;
-        }
+            break;
+    }
 
-        return ret;
+    return ret;
 }
 
 int32_t
-mem_acct_init (xlator_t *this)
+mem_acct_init(xlator_t *this)
 {
-        int     ret = -1;
+    int ret = -1;
 
-        if (!this)
-                return ret;
+    if (!this)
+        return ret;
 
-        ret = xlator_mem_acct_init (this, gf_fuse_mt_end + 1);
-
-        if (ret != 0) {
-                gf_log (this->name, GF_LOG_ERROR, "Memory accounting init"
-                                "failed");
-                return ret;
-        }
+    ret = xlator_mem_acct_init(this, gf_fuse_mt_end + 1);
 
+    if (ret != 0) {
+        gf_log(this->name, GF_LOG_ERROR,
+               "Memory accounting init"
+               "failed");
         return ret;
-}
+    }
 
+    return ret;
+}
 
 static fuse_handler_t *fuse_std_ops[FUSE_OP_HIGH] = {
-        [FUSE_LOOKUP]      = fuse_lookup,
-        [FUSE_FORGET]      = fuse_forget,
-        [FUSE_GETATTR]     = fuse_getattr,
-        [FUSE_SETATTR]     = fuse_setattr,
-        [FUSE_READLINK]    = fuse_readlink,
-        [FUSE_SYMLINK]     = fuse_symlink,
-        [FUSE_MKNOD]       = fuse_mknod,
-        [FUSE_MKDIR]       = fuse_mkdir,
-        [FUSE_UNLINK]      = fuse_unlink,
-        [FUSE_RMDIR]       = fuse_rmdir,
-        [FUSE_RENAME]      = fuse_rename,
-        [FUSE_LINK]        = fuse_link,
-        [FUSE_OPEN]        = fuse_open,
-        [FUSE_READ]        = fuse_readv,
-        [FUSE_WRITE]       = fuse_write,
-        [FUSE_STATFS]      = fuse_statfs,
-        [FUSE_RELEASE]     = fuse_release,
-        [FUSE_FSYNC]       = fuse_fsync,
-        [FUSE_SETXATTR]    = fuse_setxattr,
-        [FUSE_GETXATTR]    = fuse_getxattr,
-        [FUSE_LISTXATTR]   = fuse_listxattr,
-        [FUSE_REMOVEXATTR] = fuse_removexattr,
-        [FUSE_FLUSH]       = fuse_flush,
-        [FUSE_INIT]        = fuse_init,
-        [FUSE_OPENDIR]     = fuse_opendir,
-        [FUSE_READDIR]     = fuse_readdir,
-        [FUSE_RELEASEDIR]  = fuse_releasedir,
-        [FUSE_FSYNCDIR]    = fuse_fsyncdir,
-        [FUSE_GETLK]       = fuse_getlk,
-        [FUSE_SETLK]       = fuse_setlk,
-        [FUSE_SETLKW]      = fuse_setlk,
-        [FUSE_ACCESS]      = fuse_access,
-        [FUSE_CREATE]      = fuse_create,
-     /* [FUSE_INTERRUPT] */
-     /* [FUSE_BMAP] */
-        [FUSE_DESTROY]     = fuse_destroy,
-     /* [FUSE_IOCTL] */
-     /* [FUSE_POLL] */
-     /* [FUSE_NOTIFY_REPLY] */
+    [FUSE_LOOKUP] = fuse_lookup,
+    [FUSE_FORGET] = fuse_forget,
+    [FUSE_GETATTR] = fuse_getattr,
+    [FUSE_SETATTR] = fuse_setattr,
+    [FUSE_READLINK] = fuse_readlink,
+    [FUSE_SYMLINK] = fuse_symlink,
+    [FUSE_MKNOD] = fuse_mknod,
+    [FUSE_MKDIR] = fuse_mkdir,
+    [FUSE_UNLINK] = fuse_unlink,
+    [FUSE_RMDIR] = fuse_rmdir,
+    [FUSE_RENAME] = fuse_rename,
+    [FUSE_LINK] = fuse_link,
+    [FUSE_OPEN] = fuse_open,
+    [FUSE_READ] = fuse_readv,
+    [FUSE_WRITE] = fuse_write,
+    [FUSE_STATFS] = fuse_statfs,
+    [FUSE_RELEASE] = fuse_release,
+    [FUSE_FSYNC] = fuse_fsync,
+    [FUSE_SETXATTR] = fuse_setxattr,
+    [FUSE_GETXATTR] = fuse_getxattr,
+    [FUSE_LISTXATTR] = fuse_listxattr,
+    [FUSE_REMOVEXATTR] = fuse_removexattr,
+    [FUSE_FLUSH] = fuse_flush,
+    [FUSE_INIT] = fuse_init,
+    [FUSE_OPENDIR] = fuse_opendir,
+    [FUSE_READDIR] = fuse_readdir,
+    [FUSE_RELEASEDIR] = fuse_releasedir,
+    [FUSE_FSYNCDIR] = fuse_fsyncdir,
+    [FUSE_GETLK] = fuse_getlk,
+    [FUSE_SETLK] = fuse_setlk,
+    [FUSE_SETLKW] = fuse_setlk,
+    [FUSE_ACCESS] = fuse_access,
+    [FUSE_CREATE] = fuse_create,
+    [FUSE_INTERRUPT] = fuse_interrupt,
+    /* [FUSE_BMAP] */
+    [FUSE_DESTROY] = fuse_destroy,
+/* [FUSE_IOCTL] */
+/* [FUSE_POLL] */
+/* [FUSE_NOTIFY_REPLY] */
 
 #if FUSE_KERNEL_MINOR_VERSION >= 16
-	[FUSE_BATCH_FORGET]= fuse_batch_forget,
+    [FUSE_BATCH_FORGET] = fuse_batch_forget,
 #endif
 
 #if FUSE_KERNEL_MINOR_VERSION >= 19
 #ifdef FALLOC_FL_KEEP_SIZE
-	[FUSE_FALLOCATE]   = fuse_fallocate,
+    [FUSE_FALLOCATE] = fuse_fallocate,
 #endif /* FALLOC_FL_KEEP_SIZE */
 #endif
 
 #if FUSE_KERNEL_MINOR_VERSION >= 21
-	[FUSE_READDIRPLUS] = fuse_readdirp,
+    [FUSE_READDIRPLUS] = fuse_readdirp,
 #endif
-};
 
+#if FUSE_KERNEL_MINOR_VERSION >= 24 && HAVE_SEEK_HOLE
+    [FUSE_LSEEK] = fuse_lseek,
+#endif
 
-static fuse_handler_t *fuse_dump_ops[FUSE_OP_HIGH];
+#if FUSE_KERNEL_MINOR_VERSION >= 28
+    [FUSE_COPY_FILE_RANGE] = fuse_copy_file_range,
+#endif
+};
 
+static fuse_handler_t *fuse_dump_ops[FUSE_OP_HIGH];
 
 static void
-fuse_dumper (xlator_t *this, fuse_in_header_t *finh, void *msg)
+fuse_dumper(xlator_t *this, fuse_in_header_t *finh, void *msg,
+            struct iobuf *iobuf)
 {
-        fuse_private_t *priv = NULL;
-        struct iovec diov[3];
-        char r = 'R';
-        int ret = 0;
-
-        priv = this->private;
-
-        diov[0].iov_base = &r;
-        diov[0].iov_len  = 1;
-        diov[1].iov_base = finh;
-        diov[1].iov_len  = sizeof (*finh);
-        diov[2].iov_base = msg;
-        diov[2].iov_len  = finh->len - sizeof (*finh);
-
-        pthread_mutex_lock (&priv->fuse_dump_mutex);
-        ret = writev (priv->fuse_dump_fd, diov, 3);
-        pthread_mutex_unlock (&priv->fuse_dump_mutex);
-        if (ret == -1)
-                gf_log ("glusterfs-fuse", GF_LOG_ERROR,
-                        "failed to dump fuse message (R): %s",
-                        strerror (errno));
-
-        priv->fuse_ops0[finh->opcode] (this, finh, msg);
+    fuse_private_t *priv = NULL;
+    struct iovec diov[6] = {
+        {
+            0,
+        },
+    };
+    char r = 'R';
+    uint32_t fusedump_item_count = 3;
+    struct fusedump_timespec fts = {
+        0,
+    };
+    struct fusedump_signature fsig = {
+        0,
+    };
+
+    int ret = 0;
+
+    priv = this->private;
+
+    fusedump_setup_meta(diov, &r, &fusedump_item_count, &fts, &fsig);
+    diov[4] = (struct iovec){finh, sizeof(*finh)};
+    if (finh->opcode == FUSE_WRITE) {
+        /* WRITE has special data alignment, see comment in
+           fuse_write(). */
+        diov[4].iov_len += sizeof(struct fuse_write_in);
+    }
+    diov[5] = (struct iovec){msg, finh->len - diov[4].iov_len};
+
+    pthread_mutex_lock(&priv->fuse_dump_mutex);
+    ret = sys_writev(priv->fuse_dump_fd, diov, sizeof(diov) / sizeof(diov[0]));
+    pthread_mutex_unlock(&priv->fuse_dump_mutex);
+    if (ret == -1)
+        gf_log("glusterfs-fuse", GF_LOG_ERROR,
+               "failed to dump fuse message (R): %s", strerror(errno));
+
+    priv->fuse_ops0[finh->opcode](this, finh, msg, NULL);
 }
 
-
 int
-init (xlator_t *this_xl)
-{
-        int                ret = 0;
-        dict_t            *options = NULL;
-        char              *value_string = NULL;
-        cmd_args_t        *cmd_args = NULL;
-        char              *fsname = NULL;
-        fuse_private_t    *priv = NULL;
-        struct stat        stbuf = {0,};
-        int                i = 0;
-        int                xl_name_allocated = 0;
-        int                fsname_allocated = 0;
-        glusterfs_ctx_t   *ctx = NULL;
-        gf_boolean_t       sync_to_mount = _gf_false;
-        gf_boolean_t       fopen_keep_cache = _gf_false;
-        unsigned long      mntflags = 0;
-        char              *mnt_args = NULL;
-        eh_t              *event = NULL;
-
-        if (this_xl == NULL)
-                return -1;
-
-        if (this_xl->options == NULL)
-                return -1;
-
-        ctx = this_xl->ctx;
-        if (!ctx)
-                return -1;
-
-        options = this_xl->options;
-
-        if (this_xl->name == NULL) {
-                this_xl->name = gf_strdup ("fuse");
-                if (!this_xl->name) {
-                        gf_log ("glusterfs-fuse", GF_LOG_ERROR,
-                                "Out of memory");
-
-                        goto cleanup_exit;
-                }
-                xl_name_allocated = 1;
-        }
-
-        priv = GF_CALLOC (1, sizeof (*priv), gf_fuse_mt_fuse_private_t);
-        if (!priv) {
-                gf_log ("glusterfs-fuse", GF_LOG_ERROR,
-                        "Out of memory");
-
-                goto cleanup_exit;
-        }
-        this_xl->private = (void *) priv;
-        priv->mount_point = NULL;
-        priv->fd = -1;
-        priv->revchan_in = -1;
-        priv->revchan_out = -1;
-
-        /* get options from option dictionary */
-        ret = dict_get_str (options, ZR_MOUNTPOINT_OPT, &value_string);
-        if (ret == -1 || value_string == NULL) {
-                gf_log ("fuse", GF_LOG_ERROR,
-                        "Mandatory option 'mountpoint' is not specified.");
-                goto cleanup_exit;
-        }
+init(xlator_t *this_xl)
+{
+    int ret = 0;
+    dict_t *options = NULL;
+    char *value_string = NULL;
+    cmd_args_t *cmd_args = NULL;
+    char *fsname = NULL;
+    fuse_private_t *priv = NULL;
+    struct stat stbuf = {
+        0,
+    };
+    int i = 0;
+    int xl_name_allocated = 0;
+    int fsname_allocated = 0;
+    glusterfs_ctx_t *ctx = NULL;
+    gf_boolean_t sync_to_mount = _gf_false;
+    gf_boolean_t fopen_keep_cache = _gf_false;
+    char *mnt_args = NULL;
+    eh_t *event = NULL;
+
+    if (this_xl == NULL)
+        return -1;
 
-        if (stat (value_string, &stbuf) != 0) {
-                if (errno == ENOENT) {
-                        gf_log (this_xl->name, GF_LOG_ERROR,
-                                "%s %s does not exist",
-                                ZR_MOUNTPOINT_OPT, value_string);
-                } else if (errno == ENOTCONN) {
-                        gf_log (this_xl->name, GF_LOG_ERROR,
-                                "Mountpoint %s seems to have a stale "
-                                "mount, run 'umount %s' and try again.",
-                                value_string, value_string);
-                } else {
-                        gf_log (this_xl->name, GF_LOG_DEBUG,
-                                "%s %s : stat returned %s",
-                                ZR_MOUNTPOINT_OPT,
-                                value_string, strerror (errno));
-                }
-                goto cleanup_exit;
-        }
+    if (this_xl->options == NULL)
+        return -1;
 
-        if (S_ISDIR (stbuf.st_mode) == 0) {
-                gf_log (this_xl->name, GF_LOG_ERROR,
-                        "%s %s is not a directory",
-                        ZR_MOUNTPOINT_OPT, value_string);
-                goto cleanup_exit;
-        }
-        priv->mount_point = gf_strdup (value_string);
-        if (!priv->mount_point) {
-                gf_log ("glusterfs-fuse", GF_LOG_ERROR,
-                        "Out of memory");
+    ctx = this_xl->ctx;
+    if (!ctx)
+        return -1;
 
-                goto cleanup_exit;
+    options = this_xl->options;
+
+    if (this_xl->name == NULL) {
+        this_xl->name = gf_strdup("fuse");
+        if (!this_xl->name) {
+            gf_log("glusterfs-fuse", GF_LOG_ERROR, "Out of memory");
+
+            goto cleanup_exit;
+        }
+        xl_name_allocated = 1;
+    }
+
+    priv = GF_CALLOC(1, sizeof(*priv), gf_fuse_mt_fuse_private_t);
+    if (!priv) {
+        gf_log("glusterfs-fuse", GF_LOG_ERROR, "Out of memory");
+
+        goto cleanup_exit;
+    }
+    this_xl->private = (void *)priv;
+    priv->mount_point = NULL;
+    priv->fd = -1;
+
+    INIT_LIST_HEAD(&priv->invalidate_list);
+    pthread_cond_init(&priv->invalidate_cond, NULL);
+    pthread_mutex_init(&priv->invalidate_mutex, NULL);
+
+    INIT_LIST_HEAD(&priv->timed_list);
+    pthread_cond_init(&priv->timed_cond, NULL);
+    pthread_mutex_init(&priv->timed_mutex, NULL);
+
+    INIT_LIST_HEAD(&priv->interrupt_list);
+    pthread_mutex_init(&priv->interrupt_mutex, NULL);
+
+    pthread_mutex_init(&priv->fusedev_errno_cnt_mutex, NULL);
+
+    /* get options from option dictionary */
+    ret = dict_get_str(options, ZR_MOUNTPOINT_OPT, &value_string);
+    if (ret == -1 || value_string == NULL) {
+        gf_log("fuse", GF_LOG_ERROR,
+               "Mandatory option 'mountpoint' is not specified.");
+        goto cleanup_exit;
+    }
+
+    if (sys_stat(value_string, &stbuf) != 0) {
+        if (errno == ENOENT) {
+            gf_log(this_xl->name, GF_LOG_ERROR, "%s %s does not exist",
+                   ZR_MOUNTPOINT_OPT, value_string);
+        } else if (errno == ENOTCONN) {
+            gf_log(this_xl->name, GF_LOG_ERROR,
+                   "Mountpoint %s seems to have a stale "
+                   "mount, run 'umount %s' and try again.",
+                   value_string, value_string);
+        } else {
+            gf_log(this_xl->name, GF_LOG_DEBUG, "%s %s : stat returned %s",
+                   ZR_MOUNTPOINT_OPT, value_string, strerror(errno));
         }
+        goto cleanup_exit;
+    }
 
-        GF_OPTION_INIT (ZR_ATTR_TIMEOUT_OPT, priv->attribute_timeout, double,
-                        cleanup_exit);
-
-        GF_OPTION_INIT (ZR_ENTRY_TIMEOUT_OPT, priv->entry_timeout, double,
-                        cleanup_exit);
-
-        GF_OPTION_INIT (ZR_NEGATIVE_TIMEOUT_OPT, priv->negative_timeout, double,
-                        cleanup_exit);
-
-        GF_OPTION_INIT ("client-pid", priv->client_pid, int32, cleanup_exit);
-        /* have to check & register the presence of client-pid manually */
-        priv->client_pid_set = !!dict_get (this_xl->options, "client-pid");
-
-        GF_OPTION_INIT ("uid-map-root", priv->uid_map_root, uint32,
-                        cleanup_exit);
-
-        priv->direct_io_mode = 2;
-        ret = dict_get_str (options, ZR_DIRECT_IO_OPT, &value_string);
-        if (ret == 0) {
-                ret = gf_string2boolean (value_string, &priv->direct_io_mode);
-                GF_ASSERT (ret == 0);
-        }
+    if (S_ISDIR(stbuf.st_mode) == 0) {
+        gf_log(this_xl->name, GF_LOG_ERROR, "%s %s is not a directory",
+               ZR_MOUNTPOINT_OPT, value_string);
+        goto cleanup_exit;
+    }
+    priv->mount_point = gf_strdup(value_string);
+    if (!priv->mount_point) {
+        gf_log("glusterfs-fuse", GF_LOG_ERROR, "Out of memory");
 
-        GF_OPTION_INIT (ZR_STRICT_VOLFILE_CHECK, priv->strict_volfile_check,
-                        bool, cleanup_exit);
+        goto cleanup_exit;
+    }
 
-        GF_OPTION_INIT ("acl", priv->acl, bool, cleanup_exit);
+    GF_OPTION_INIT(ZR_ATTR_TIMEOUT_OPT, priv->attribute_timeout, double,
+                   cleanup_exit);
 
-        if (priv->uid_map_root)
-                priv->acl = 1;
+    GF_OPTION_INIT("reader-thread-count", priv->reader_thread_count, uint32,
+                   cleanup_exit);
 
-        GF_OPTION_INIT ("selinux", priv->selinux, bool, cleanup_exit);
+    GF_OPTION_INIT("auto-invalidation", priv->fuse_auto_inval, bool,
+                   cleanup_exit);
+    GF_OPTION_INIT(ZR_ENTRY_TIMEOUT_OPT, priv->entry_timeout, double,
+                   cleanup_exit);
 
-        GF_OPTION_INIT ("read-only", priv->read_only, bool, cleanup_exit);
+    GF_OPTION_INIT(ZR_NEGATIVE_TIMEOUT_OPT, priv->negative_timeout, double,
+                   cleanup_exit);
 
-        GF_OPTION_INIT ("enable-ino32", priv->enable_ino32, bool, cleanup_exit);
+    GF_OPTION_INIT("client-pid", priv->client_pid, int32, cleanup_exit);
+    /* have to check & register the presence of client-pid manually */
+    priv->client_pid_set = !!dict_get(this_xl->options, "client-pid");
 
-        GF_OPTION_INIT ("use-readdirp", priv->use_readdirp, bool, cleanup_exit);
+    GF_OPTION_INIT("uid-map-root", priv->uid_map_root, uint32, cleanup_exit);
 
-        priv->fuse_dump_fd = -1;
-        ret = dict_get_str (options, "dump-fuse", &value_string);
+    priv->direct_io_mode = 2;
+    ret = dict_get_str(options, ZR_DIRECT_IO_OPT, &value_string);
+    if (ret == 0) {
+        gf_boolean_t direct_io_mode_bool;
+        ret = gf_string2boolean(value_string, &direct_io_mode_bool);
         if (ret == 0) {
-                ret = unlink (value_string);
-                if (ret != -1 || errno == ENOENT)
-                        ret = open (value_string, O_RDWR|O_CREAT|O_EXCL,
-                                    S_IRUSR|S_IWUSR);
-                if (ret == -1) {
-                        gf_log ("glusterfs-fuse", GF_LOG_ERROR,
-                                "cannot open fuse dump file %s",
-                                 value_string);
-
-                        goto cleanup_exit;
-                }
-                priv->fuse_dump_fd = ret;
+            priv->direct_io_mode = direct_io_mode_bool ? 1 : 0;
         }
+    }
 
-        sync_to_mount = _gf_false;
-        ret = dict_get_str (options, "sync-to-mount", &value_string);
-        if (ret == 0) {
-                ret = gf_string2boolean (value_string,
-                                         &sync_to_mount);
-                GF_ASSERT (ret == 0);
-        }
+    GF_OPTION_INIT(ZR_STRICT_VOLFILE_CHECK, priv->strict_volfile_check, bool,
+                   cleanup_exit);
 
-	priv->fopen_keep_cache = 2;
-	if (dict_get (options, "fopen-keep-cache")) {
-		GF_OPTION_INIT("fopen-keep-cache", fopen_keep_cache, bool,
-			       cleanup_exit);
-		priv->fopen_keep_cache = fopen_keep_cache;
-	}
+    GF_OPTION_INIT("acl", priv->acl, bool, cleanup_exit);
 
-        GF_OPTION_INIT("gid-timeout", priv->gid_cache_timeout, int32,
-                cleanup_exit);
+    if (priv->uid_map_root)
+        priv->acl = 1;
 
-        GF_OPTION_INIT ("fuse-mountopts", priv->fuse_mountopts, str, cleanup_exit);
+    GF_OPTION_INIT("selinux", priv->selinux, bool, cleanup_exit);
 
-        if (gid_cache_init(&priv->gid_cache, priv->gid_cache_timeout) < 0) {
-                gf_log("glusterfs-fuse", GF_LOG_ERROR, "Failed to initialize "
-                        "group cache.");
-                goto cleanup_exit;
-        }
+    GF_OPTION_INIT("capability", priv->capability, bool, cleanup_exit);
 
-        /* default values seemed to work fine during testing */
-        GF_OPTION_INIT ("background-qlen", priv->background_qlen, int32,
-                        cleanup_exit);
-        GF_OPTION_INIT ("congestion-threshold", priv->congestion_threshold,
-                        int32, cleanup_exit);
+    GF_OPTION_INIT("read-only", priv->read_only, bool, cleanup_exit);
 
-        GF_OPTION_INIT("no-root-squash", priv->no_root_squash, bool,
-                       cleanup_exit);
-        /* change the client_pid to no-root-squash pid only if the
-           client is none of defrag process, hadoop access and gsyncd process.
-        */
-        if (!priv->client_pid_set) {
-                if (priv->no_root_squash == _gf_true) {
-                        priv->client_pid_set = _gf_true;
-                        priv->client_pid = GF_CLIENT_PID_NO_ROOT_SQUASH;
-                }
-        }
+    GF_OPTION_INIT("enable-ino32", priv->enable_ino32, bool, cleanup_exit);
 
-        /* user has set only background-qlen, not congestion-threshold,
-           use the fuse kernel driver formula to set congestion. ie, 75% */
-        if (dict_get (this_xl->options, "background-qlen") &&
-            !dict_get (this_xl->options, "congestion-threshold")) {
-                priv->congestion_threshold = (priv->background_qlen * 3) / 4;
-                gf_log (this_xl->name, GF_LOG_INFO,
-                        "setting congestion control as 75%% of "
-                        "background-queue length (ie, (.75 * %d) = %d",
-                        priv->background_qlen, priv->congestion_threshold);
-        }
+    GF_OPTION_INIT("use-readdirp", priv->use_readdirp, bool, cleanup_exit);
 
-        /* congestion should not be higher than background queue length */
-        if (priv->congestion_threshold > priv->background_qlen) {
-                gf_log (this_xl->name, GF_LOG_INFO,
-                        "setting congestion control same as "
-                        "background-queue length (%d)",
-                        priv->background_qlen);
-                priv->congestion_threshold = priv->background_qlen;
-        }
+    priv->fuse_dump_fd = -1;
+    ret = dict_get_str(options, "dump-fuse", &value_string);
+    if (ret == 0) {
+        ret = sys_unlink(value_string);
+        if (ret == -1 && errno != ENOENT) {
+            gf_log("glusterfs-fuse", GF_LOG_ERROR,
+                   "failed to remove old fuse dump file %s: %s", value_string,
+                   strerror(errno));
 
-        cmd_args = &this_xl->ctx->cmd_args;
-        fsname = cmd_args->volfile;
-        if (!fsname && cmd_args->volfile_server) {
-                if (cmd_args->volfile_id) {
-                        fsname = GF_MALLOC (
-                                   strlen (cmd_args->volfile_server) + 1 +
-                                   strlen (cmd_args->volfile_id) + 1,
-                                   gf_fuse_mt_fuse_private_t);
-                        if (!fsname) {
-                                gf_log ("glusterfs-fuse", GF_LOG_ERROR,
-                                        "Out of memory");
-                                goto cleanup_exit;
-                        }
-                        fsname_allocated = 1;
-                        strcpy (fsname, cmd_args->volfile_server);
-                        strcat (fsname, ":");
-                        strcat (fsname, cmd_args->volfile_id);
-                } else
-                        fsname = cmd_args->volfile_server;
+            goto cleanup_exit;
         }
-        if (!fsname)
-                fsname = "glusterfs";
+        ret = open(value_string, O_RDWR | O_CREAT | O_EXCL, S_IRUSR | S_IWUSR);
+        if (ret == -1) {
+            gf_log("glusterfs-fuse", GF_LOG_ERROR,
+                   "failed to open fuse dump file %s: %s", value_string,
+                   strerror(errno));
 
-        priv->fdtable = gf_fd_fdtable_alloc ();
-        if (priv->fdtable == NULL) {
-                gf_log ("glusterfs-fuse", GF_LOG_ERROR, "Out of memory");
-                goto cleanup_exit;
+            goto cleanup_exit;
         }
+        priv->fuse_dump_fd = ret;
+    }
 
-        if (priv->read_only)
-                mntflags |= MS_RDONLY;
-        gf_asprintf (&mnt_args, "%s%s%sallow_other,max_read=131072",
-                     priv->acl ? "" : "default_permissions,",
-                     priv->fuse_mountopts ? priv->fuse_mountopts : "",
-                     priv->fuse_mountopts ? "," : "");
-        if (!mnt_args)
-                goto cleanup_exit;
+    sync_to_mount = _gf_false;
+    ret = dict_get_str(options, "sync-to-mount", &value_string);
+    if (ret == 0) {
+        ret = gf_string2boolean(value_string, &sync_to_mount);
+        GF_ASSERT(ret == 0);
+    }
 
-        if (pipe(priv->status_pipe) < 0) {
-                gf_log (this_xl->name, GF_LOG_ERROR,
-                        "could not create pipe to separate mount process");
+    priv->fopen_keep_cache = 2;
+    if (dict_get(options, "fopen-keep-cache")) {
+        GF_OPTION_INIT("fopen-keep-cache", fopen_keep_cache, bool,
+                       cleanup_exit);
+        priv->fopen_keep_cache = fopen_keep_cache;
+    }
+
+    GF_OPTION_INIT("gid-timeout", priv->gid_cache_timeout, int32, cleanup_exit);
+
+    GF_OPTION_INIT("fuse-mountopts", priv->fuse_mountopts, str, cleanup_exit);
+
+    if (gid_cache_init(&priv->gid_cache, priv->gid_cache_timeout) < 0) {
+        gf_log("glusterfs-fuse", GF_LOG_ERROR,
+               "Failed to initialize "
+               "group cache.");
+        goto cleanup_exit;
+    }
+
+    GF_OPTION_INIT("resolve-gids", priv->resolve_gids, bool, cleanup_exit);
+
+    /* default values seemed to work fine during testing */
+    GF_OPTION_INIT("background-qlen", priv->background_qlen, int32,
+                   cleanup_exit);
+    GF_OPTION_INIT("congestion-threshold", priv->congestion_threshold, int32,
+                   cleanup_exit);
+
+    GF_OPTION_INIT("no-root-squash", priv->no_root_squash, bool, cleanup_exit);
+    /* change the client_pid to no-root-squash pid only if the
+       client is neither defrag process or gsyncd process.
+    */
+    if (!priv->client_pid_set) {
+        if (priv->no_root_squash == _gf_true) {
+            priv->client_pid_set = _gf_true;
+            priv->client_pid = GF_CLIENT_PID_NO_ROOT_SQUASH;
+        }
+    }
+
+    GF_OPTION_INIT("lru-limit", priv->lru_limit, uint32, cleanup_exit);
+
+    GF_OPTION_INIT("invalidate-limit", priv->invalidate_limit, uint32,
+                   cleanup_exit);
+
+    GF_OPTION_INIT("event-history", priv->event_history, bool, cleanup_exit);
+
+    GF_OPTION_INIT("thin-client", priv->thin_client, bool, cleanup_exit);
+
+    /* Writeback cache support */
+    GF_OPTION_INIT("kernel-writeback-cache", priv->kernel_writeback_cache, bool,
+                   cleanup_exit);
+    GF_OPTION_INIT("attr-times-granularity", priv->attr_times_granularity,
+                   int32, cleanup_exit);
+
+    GF_OPTION_INIT("flush-handle-interrupt", priv->flush_handle_interrupt, bool,
+                   cleanup_exit);
+
+    GF_OPTION_INIT("fuse-dev-eperm-ratelimit-ns",
+                   priv->fuse_dev_eperm_ratelimit_ns, uint32, cleanup_exit);
+
+    /* user has set only background-qlen, not congestion-threshold,
+       use the fuse kernel driver formula to set congestion. ie, 75% */
+    if (dict_get(this_xl->options, "background-qlen") &&
+        !dict_get(this_xl->options, "congestion-threshold")) {
+        priv->congestion_threshold = (priv->background_qlen * 3) / 4;
+        gf_log(this_xl->name, GF_LOG_INFO,
+               "setting congestion control as 75%% of "
+               "background-queue length (ie, (.75 * %d) = %d",
+               priv->background_qlen, priv->congestion_threshold);
+    }
+
+    /* congestion should not be higher than background queue length */
+    if (priv->congestion_threshold > priv->background_qlen) {
+        gf_log(this_xl->name, GF_LOG_INFO,
+               "setting congestion control same as "
+               "background-queue length (%d)",
+               priv->background_qlen);
+        priv->congestion_threshold = priv->background_qlen;
+    }
+
+    cmd_args = &this_xl->ctx->cmd_args;
+    fsname = cmd_args->volfile;
+    if (!fsname && cmd_args->volfile_server) {
+        if (cmd_args->volfile_id) {
+            int dir_len = 0;
+            if (cmd_args->subdir_mount)
+                dir_len = strlen(cmd_args->subdir_mount) + 1;
+            fsname = GF_MALLOC(strlen(cmd_args->volfile_server) + 1 +
+                                   strlen(cmd_args->volfile_id) + 1 + dir_len,
+                               gf_fuse_mt_fuse_private_t);
+            if (!fsname) {
+                gf_log("glusterfs-fuse", GF_LOG_ERROR, "Out of memory");
                 goto cleanup_exit;
+            }
+            fsname_allocated = 1;
+            strcpy(fsname, cmd_args->volfile_server);
+            strcat(fsname, ":");
+            strcat(fsname, cmd_args->volfile_id);
+            if (dir_len)
+                strcat(fsname, cmd_args->subdir_mount);
+        } else
+            fsname = cmd_args->volfile_server;
+    }
+    if (!fsname)
+        fsname = "glusterfs";
+
+    priv->fdtable = gf_fd_fdtable_alloc();
+    if (priv->fdtable == NULL) {
+        gf_log("glusterfs-fuse", GF_LOG_ERROR, "Out of memory");
+        goto cleanup_exit;
+    }
+
+    gf_asprintf(&mnt_args, "%s%s%s%sallow_other,max_read=131072",
+                priv->acl ? "" : "default_permissions,",
+                priv->read_only ? "ro," : "",
+                priv->fuse_mountopts ? priv->fuse_mountopts : "",
+                priv->fuse_mountopts ? "," : "");
+    if (!mnt_args)
+        goto cleanup_exit;
+
+    {
+        char *mnt_tok = NULL;
+        token_iter_t tit = {
+            0,
+        };
+        gf_boolean_t iter_end = _gf_false;
+
+        for (mnt_tok = token_iter_init(mnt_args, ',', &tit);;) {
+            iter_end = next_token(&mnt_tok, &tit);
+
+            if (strcmp(mnt_tok, "auto_unmount") == 0) {
+                priv->auto_unmount = _gf_true;
+                drop_token(mnt_tok, &tit);
+            }
+
+            if (iter_end)
+                break;
         }
+    }
+
+    if (pipe(priv->status_pipe) < 0) {
+        gf_log(this_xl->name, GF_LOG_ERROR,
+               "could not create pipe to separate mount process");
+        goto cleanup_exit;
+    }
+
+    priv->fd = gf_fuse_mount(priv->mount_point, fsname, mnt_args,
+                             sync_to_mount ? &ctx->mnt_pid : NULL,
+                             priv->status_pipe[1]);
+    if (priv->fd == -1)
+        goto cleanup_exit;
+    if (priv->auto_unmount) {
+        ret = gf_fuse_unmount_daemon(priv->mount_point, priv->fd);
+        if (ret == -1)
+            goto cleanup_exit;
+    }
 
-        priv->fd = gf_fuse_mount (priv->mount_point, fsname, mntflags, mnt_args,
-                                  sync_to_mount ? &ctx->mnt_pid : NULL,
-                                  priv->status_pipe[1]);
-        if (priv->fd == -1)
-                goto cleanup_exit;
-
-        event = eh_new (FUSE_EVENT_HISTORY_SIZE, _gf_false, NULL);
+    if (priv->event_history) {
+        event = eh_new(FUSE_EVENT_HISTORY_SIZE, _gf_false, NULL);
         if (!event) {
-                gf_log (this_xl->name, GF_LOG_ERROR,
-                        "could not create a new event history");
-                goto cleanup_exit;
+            gf_log(this_xl->name, GF_LOG_ERROR,
+                   "could not create a new event history");
+            goto cleanup_exit;
         }
 
         this_xl->history = event;
-
-        pthread_mutex_init (&priv->fuse_dump_mutex, NULL);
-        pthread_cond_init (&priv->sync_cond, NULL);
-        pthread_mutex_init (&priv->sync_mutex, NULL);
-        priv->event_recvd = 0;
-
-        for (i = 0; i < FUSE_OP_HIGH; i++) {
-                if (!fuse_std_ops[i])
-                        fuse_std_ops[i] = fuse_enosys;
-                if (!fuse_dump_ops[i])
-                        fuse_dump_ops[i] = fuse_dumper;
-        }
-        priv->fuse_ops = fuse_std_ops;
-        if (priv->fuse_dump_fd != -1) {
-                priv->fuse_ops0 = priv->fuse_ops;
-                priv->fuse_ops  = fuse_dump_ops;
-        }
-
-        if (fsname_allocated)
-                GF_FREE (fsname);
-        GF_FREE (mnt_args);
-        return 0;
+    }
+
+    pthread_mutex_init(&priv->fuse_dump_mutex, NULL);
+    pthread_cond_init(&priv->sync_cond, NULL);
+    pthread_cond_init(&priv->migrate_cond, NULL);
+    pthread_mutex_init(&priv->sync_mutex, NULL);
+    priv->event_recvd = 0;
+
+    for (i = 0; i < FUSE_OP_HIGH; i++) {
+        if (!fuse_std_ops[i])
+            fuse_std_ops[i] = fuse_enosys;
+        if (!fuse_dump_ops[i])
+            fuse_dump_ops[i] = fuse_dumper;
+    }
+    priv->fuse_ops = fuse_std_ops;
+    if (priv->fuse_dump_fd != -1) {
+        priv->fuse_ops0 = priv->fuse_ops;
+        priv->fuse_ops = fuse_dump_ops;
+    }
+
+    if (fsname_allocated)
+        GF_FREE(fsname);
+    GF_FREE(mnt_args);
+    return 0;
 
 cleanup_exit:
-        if (xl_name_allocated)
-                GF_FREE (this_xl->name);
-        if (fsname_allocated)
-                GF_FREE (fsname);
-        if (priv) {
-                GF_FREE (priv->mount_point);
-                if (priv->fd != -1)
-                        close (priv->fd);
-                if (priv->fuse_dump_fd != -1)
-                        close (priv->fuse_dump_fd);
-                GF_FREE (priv);
-        }
-        GF_FREE (mnt_args);
-        return -1;
+    if (xl_name_allocated)
+        GF_FREE(this_xl->name);
+    if (fsname_allocated)
+        GF_FREE(fsname);
+    if (priv) {
+        GF_FREE(priv->mount_point);
+        if (priv->fd != -1)
+            sys_close(priv->fd);
+        if (priv->fuse_dump_fd != -1)
+            sys_close(priv->fuse_dump_fd);
+        GF_FREE(priv);
+    }
+    GF_FREE(mnt_args);
+    return -1;
 }
 
-
 void
-fini (xlator_t *this_xl)
+fini(xlator_t *this_xl)
 {
-        fuse_private_t *priv = NULL;
-        char *mount_point = NULL;
+    fuse_private_t *priv = NULL;
+    char *mount_point = NULL;
 
-        if (this_xl == NULL)
-                return;
+    if (this_xl == NULL)
+        return;
 
-        if ((priv = this_xl->private) == NULL)
-                return;
+    if ((priv = this_xl->private) == NULL)
+        return;
 
-        pthread_mutex_lock (&priv->sync_mutex);
-        {
-                if (!(priv->fini_invoked)) {
-                        priv->fini_invoked = _gf_true;
-                } else {
-                        pthread_mutex_unlock (&priv->sync_mutex);
-                        return;
-                }
+    pthread_mutex_lock(&priv->sync_mutex);
+    {
+        if (!(priv->fini_invoked)) {
+            priv->fini_invoked = _gf_true;
+        } else {
+            pthread_mutex_unlock(&priv->sync_mutex);
+            return;
         }
-        pthread_mutex_unlock (&priv->sync_mutex);
-
-        if (dict_get (this_xl->options, ZR_MOUNTPOINT_OPT))
-                mount_point = data_to_str (dict_get (this_xl->options,
-                                                     ZR_MOUNTPOINT_OPT));
-        if (mount_point != NULL) {
-                gf_log (this_xl->name, GF_LOG_INFO,
-                        "Unmounting '%s'.", mount_point);
-
-                gf_fuse_unmount (mount_point, priv->fd);
-                close (priv->fuse_dump_fd);
-                dict_del (this_xl->options, ZR_MOUNTPOINT_OPT);
+    }
+    pthread_mutex_unlock(&priv->sync_mutex);
+
+    if (dict_get(this_xl->options, ZR_MOUNTPOINT_OPT))
+        mount_point = data_to_str(
+            dict_get(this_xl->options, ZR_MOUNTPOINT_OPT));
+    if (mount_point != NULL) {
+        if (!priv->auto_unmount) {
+            gf_log(this_xl->name, GF_LOG_INFO, "Unmounting '%s'.", mount_point);
+            gf_fuse_unmount(mount_point, priv->fd);
         }
-        /* Process should terminate once fuse xlator is finished.
-         * Required for AUTH_FAILED event.
-         */
-        kill (getpid (), SIGTERM);
+
+        gf_log(this_xl->name, GF_LOG_INFO, "Closing fuse connection to '%s'.",
+               mount_point);
+
+        sys_close(priv->fuse_dump_fd);
+        dict_del(this_xl->options, ZR_MOUNTPOINT_OPT);
+    }
+    /* Process should terminate once fuse xlator is finished.
+     * Required for AUTH_FAILED event.
+     */
+    kill(getpid(), SIGTERM);
 }
 
 struct xlator_fops fops;
 
-struct xlator_cbks cbks = {
-        .invalidate = fuse_invalidate,
-        .forget     = fuse_forget_cbk,
-        .release    = fuse_internal_release
-};
-
+struct xlator_cbks cbks = {.invalidate = fuse_invalidate,
+                           .forget = fuse_forget_cbk,
+                           .release = fuse_internal_release};
 
 struct xlator_dumpops dumpops = {
-        .priv  = fuse_priv_dump,
-        .inode = fuse_itable_dump,
-        .history = fuse_history_dump,
+    .priv = fuse_priv_dump,
+    .inode = fuse_itable_dump,
+    .history = fuse_history_dump,
 };
 
 struct volume_options options[] = {
-        { .key  = {"direct-io-mode"},
-          .type = GF_OPTION_TYPE_BOOL
-        },
-        { .key  = {ZR_MOUNTPOINT_OPT, "mount-point"},
-          .type = GF_OPTION_TYPE_PATH
-        },
-        { .key  = {ZR_DUMP_FUSE, "fuse-dumpfile"},
-          .type = GF_OPTION_TYPE_PATH
-        },
-        { .key  = {ZR_ATTR_TIMEOUT_OPT},
-          .type = GF_OPTION_TYPE_DOUBLE,
-          .default_value = "1.0"
-        },
-        { .key  = {ZR_ENTRY_TIMEOUT_OPT},
-          .type = GF_OPTION_TYPE_DOUBLE,
-          .default_value = "1.0"
-        },
-        { .key  = {ZR_NEGATIVE_TIMEOUT_OPT},
-          .type = GF_OPTION_TYPE_DOUBLE,
-          .default_value = "0.0"
-        },
-        { .key  = {ZR_STRICT_VOLFILE_CHECK},
-          .type = GF_OPTION_TYPE_BOOL,
-          .default_value = "false"
-        },
-        { .key  = {"client-pid"},
-          .type = GF_OPTION_TYPE_INT
-        },
-        { .key  = {"uid-map-root"},
-          .type = GF_OPTION_TYPE_INT
-        },
-        { .key  = {"sync-to-mount"},
-          .type = GF_OPTION_TYPE_BOOL
-        },
-        { .key = {"read-only"},
-          .type = GF_OPTION_TYPE_BOOL
-        },
-        { .key = {"fopen-keep-cache"},
-          .type = GF_OPTION_TYPE_BOOL,
-          .default_value = "false"
-        },
-        { .key = {"gid-timeout"},
-          .type = GF_OPTION_TYPE_INT,
-          .default_value = "2"
-        },
-        { .key = {"acl"},
-          .type = GF_OPTION_TYPE_BOOL,
-          .default_value = "false"
-        },
-        { .key = {"selinux"},
-          .type = GF_OPTION_TYPE_BOOL,
-          .default_value = "false"
-        },
-        { .key = {"enable-ino32"},
-          .type = GF_OPTION_TYPE_BOOL,
-          .default_value = "false"
-        },
-        { .key  = {"background-qlen"},
-          .type = GF_OPTION_TYPE_INT,
-          .default_value = "64",
-          .min = 16,
-          .max = (64 * GF_UNIT_KB),
-        },
-        { .key  = {"congestion-threshold"},
-          .type = GF_OPTION_TYPE_INT,
-          .default_value = "48",
-          .min = 12,
-          .max = (64 * GF_UNIT_KB),
-        },
-        { .key = {"fuse-mountopts"},
-          .type = GF_OPTION_TYPE_STR
-        },
-        { .key = {"use-readdirp"},
-          .type = GF_OPTION_TYPE_BOOL,
-          .default_value = "yes"
-        },
-        { .key = {"no-root-squash"},
-          .type = GF_OPTION_TYPE_BOOL,
-          .default_value = "false",
-          .description = "This is the mount option for disabling the "
-          "root squash for the client irrespective of whether the root-squash "
-          "option for the volume is set or not. But this option is honoured "
-          "only for the trusted clients. For non trusted clients this value "
-          "does not have any affect and the volume option for root-squash is "
-          "honoured.",
-        },
-        { .key = {NULL} },
+    {.key = {"direct-io-mode"}, .type = GF_OPTION_TYPE_BOOL},
+    {.key = {ZR_MOUNTPOINT_OPT, "mount-point"}, .type = GF_OPTION_TYPE_PATH},
+    {.key = {ZR_DUMP_FUSE, "fuse-dumpfile"}, .type = GF_OPTION_TYPE_PATH},
+    {.key = {ZR_ATTR_TIMEOUT_OPT},
+     .type = GF_OPTION_TYPE_DOUBLE,
+     .default_value = "1.0"},
+    {.key = {ZR_ENTRY_TIMEOUT_OPT},
+     .type = GF_OPTION_TYPE_DOUBLE,
+     .default_value = "1.0"},
+    {.key = {ZR_NEGATIVE_TIMEOUT_OPT},
+     .type = GF_OPTION_TYPE_DOUBLE,
+     .default_value = "0.0"},
+    {.key = {ZR_STRICT_VOLFILE_CHECK},
+     .type = GF_OPTION_TYPE_BOOL,
+     .default_value = "false"},
+    {.key = {"client-pid"}, .type = GF_OPTION_TYPE_INT},
+    {.key = {"uid-map-root"}, .type = GF_OPTION_TYPE_INT},
+    {.key = {"sync-to-mount"}, .type = GF_OPTION_TYPE_BOOL},
+    {.key = {"read-only"}, .type = GF_OPTION_TYPE_BOOL},
+    {.key = {"fopen-keep-cache"},
+     .type = GF_OPTION_TYPE_BOOL,
+     .default_value = "false"},
+    {.key = {"gid-timeout"},
+     .type = GF_OPTION_TYPE_INT,
+     .default_value = "300"},
+    {.key = {"resolve-gids"},
+     .type = GF_OPTION_TYPE_BOOL,
+     .default_value = "false"},
+    {.key = {"acl"}, .type = GF_OPTION_TYPE_BOOL, .default_value = "false"},
+    {.key = {"selinux"}, .type = GF_OPTION_TYPE_BOOL, .default_value = "false"},
+    {.key = {"enable-ino32"},
+     .type = GF_OPTION_TYPE_BOOL,
+     .default_value = "false"},
+    {
+        .key = {"background-qlen"},
+        .type = GF_OPTION_TYPE_INT,
+        .default_value = "64",
+        .min = 16,
+        .max = (64 * GF_UNIT_KB),
+    },
+    {
+        .key = {"congestion-threshold"},
+        .type = GF_OPTION_TYPE_INT,
+        .default_value = "48",
+        .min = 12,
+        .max = (64 * GF_UNIT_KB),
+    },
+    {.key = {"fuse-mountopts"}, .type = GF_OPTION_TYPE_STR},
+    {.key = {"use-readdirp"},
+     .type = GF_OPTION_TYPE_BOOL,
+     .default_value = "yes"},
+    {
+        .key = {"no-root-squash"},
+        .type = GF_OPTION_TYPE_BOOL,
+        .default_value = "false",
+        .description =
+            "This is the mount option for disabling the "
+            "root squash for the client irrespective of whether the "
+            "root-squash "
+            "option for the volume is set or not. But this option is honoured "
+            "only for the trusted clients. For non trusted clients this value "
+            "does not have any affect and the volume option for root-squash is "
+            "honoured.",
+    },
+    {.key = {"capability"},
+     .type = GF_OPTION_TYPE_BOOL,
+     .default_value = "false"},
+    {
+        .key = {"event-history"},
+        .type = GF_OPTION_TYPE_BOOL,
+        .default_value = "false",
+        .description = "This option can be used to enable or disable fuse "
+                       "event history.",
+    },
+    {
+        .key = {"thin-client"},
+        .type = GF_OPTION_TYPE_BOOL,
+        .default_value = "false",
+        .description = "Enables thin mount and connects via gfproxyd daemon.",
+    },
+    {
+        .key = {"reader-thread-count"},
+        .type = GF_OPTION_TYPE_INT,
+        .default_value = "1",
+        .min = 1,
+        .max = 64,
+        .description = "Sets fuse reader thread count.",
+    },
+    {
+        .key = {"kernel-writeback-cache"},
+        .type = GF_OPTION_TYPE_BOOL,
+        .default_value = "false",
+        .description = "Enables fuse in-kernel writeback cache.",
+    },
+    {
+        .key = {"attr-times-granularity"},
+        .type = GF_OPTION_TYPE_INT,
+        .default_value = "0",
+        .min = 0,
+        .max = 1000000000,
+        .description = "Supported granularity of file attribute times.",
+    },
+    {
+        .key = {"flush-handle-interrupt"},
+        .type = GF_OPTION_TYPE_BOOL,
+        .default_value = "false",
+        .description =
+            "Handle iterrupts in FLUSH handler (for testing purposes).",
+    },
+    {
+        .key = {"lru-limit"},
+        .type = GF_OPTION_TYPE_INT,
+        .default_value = "65536",
+        .min = 0,
+        .description = "makes glusterfs invalidate kernel inodes after "
+                       "reaching this limit (0 means 'unlimited')",
+    },
+    {
+        .key = {"invalidate-limit"},
+        .type = GF_OPTION_TYPE_INT,
+        .default_value = "0",
+        .min = 0,
+        .description = "suspend invalidations as of 'lru-limit' if the number "
+                       "of outstanding invalidations reaches this limit "
+                       "(0 means 'unlimited')",
+    },
+    {
+        .key = {"auto-invalidation"},
+        .type = GF_OPTION_TYPE_BOOL,
+        .default_value = "true",
+        .description = "controls whether fuse-kernel can auto-invalidate "
+                       "attribute, dentry and page-cache. Disable this only "
+                       "if same files/directories are not accessed across "
+                       "two different mounts concurrently",
+    },
+    {
+        .key = {"fuse-dev-eperm-ratelimit-ns"},
+        .type = GF_OPTION_TYPE_INT,
+        .default_value = "10000000", /* 0.01 sec */
+        .min = 0,
+        .max = 1000000000,
+        .description = "Rate limit reading from fuse device upon EPERM "
+                       "failure.",
+    },
+    {.key = {NULL}},
+};
+
+xlator_api_t xlator_api = {
+    .init = init,
+    .fini = fini,
+    .notify = notify,
+    .dumpops = &dumpops,
+    .mem_acct_init = mem_acct_init,
+    .op_version = {1}, /* Present from the initial version */
+    .fops = &fops,
+    .cbks = &cbks,
+    .options = options,
+    .identifier = "fuse",
+    .category = GF_MAINTAINED,
 };
diff --git a/xlators/mount/fuse/src/fuse-bridge.h b/xlators/mount/fuse/src/fuse-bridge.h
index 1a0d74cd4cd..4cb94c23cad 100644
--- a/xlators/mount/fuse/src/fuse-bridge.h
+++ b/xlators/mount/fuse/src/fuse-bridge.h
@@ -19,17 +19,12 @@
 #include <sys/time.h>
 #include <fnmatch.h>
 
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif /* _CONFIG_H */
-
-#include "glusterfs.h"
-#include "logging.h"
-#include "xlator.h"
-#include "defaults.h"
-#include "common-utils.h"
-#include "statedump.h"
+#include <glusterfs/glusterfs.h>
+#include <glusterfs/logging.h>
+#include <glusterfs/xlator.h>
+#include <glusterfs/defaults.h>
+#include <glusterfs/common-utils.h>
+#include <glusterfs/statedump.h>
 
 #ifdef GF_DARWIN_HOST_OS
 #include "fuse_kernel_macfuse.h"
@@ -40,374 +35,510 @@
 #include "fuse-mount.h"
 #include "fuse-mem-types.h"
 
-#include "list.h"
-#include "dict.h"
-#include "syncop.h"
-#include "gidcache.h"
+#include <glusterfs/list.h>
+#include <glusterfs/dict.h>
+#include <glusterfs/syncop.h>
+#include <glusterfs/gidcache.h>
 
 #if defined(GF_LINUX_HOST_OS) || defined(__FreeBSD__) || defined(__NetBSD__)
-#define FUSE_OP_HIGH (FUSE_READDIRPLUS + 1)
+
+/*
+ * TODO:
+ * So, with the addition of copy_file_range support, it might
+ * require a bump up of fuse kernel minor version (like it was
+ * done when support for lseek fop was added. But, as of now,
+ * the copy_file_range support has just landed in upstream
+ * kernel fuse module. So, until, there is a release of that
+ * fuse as part of a kernel, the FUSE_KERNEL_MINOR_VERSION
+ * from fuse_kernel.h in the contrib might not be changed.
+ * If so, then the highest op available should be based on
+ * the current minor version (which is 24). So, selectively
+ * determine. When, the minor version is changed to 28 in
+ * fuse_kernel.h from contrib (because in upstream linux
+ * kernel source tree, the kernel minor version which
+ * contains support for copy_file_range is 28), then remove
+ * the reference to FUSE_LSEEK below and just determine
+ * FUSE_OP_HIGH based on copy_file_range.
+ */
+#if FUSE_KERNEL_MINOR_VERSION >= 28
+#define FUSE_OP_HIGH (FUSE_COPY_FILE_RANGE + 1)
+#else
+#define FUSE_OP_HIGH (FUSE_LSEEK + 1)
+#endif
+
 #endif
 #ifdef GF_DARWIN_HOST_OS
 #define FUSE_OP_HIGH (FUSE_DESTROY + 1)
 #endif
-#define GLUSTERFS_XATTR_LEN_MAX  65536
+#define GLUSTERFS_XATTR_LEN_MAX 65536
 
 #define MAX_FUSE_PROC_DELAY 1
 
 typedef struct fuse_in_header fuse_in_header_t;
-typedef void (fuse_handler_t) (xlator_t *this, fuse_in_header_t *finh,
-                               void *msg);
+typedef void(fuse_handler_t)(xlator_t *this, fuse_in_header_t *finh, void *msg,
+                             struct iobuf *iobuf);
+
+enum fusedev_errno {
+    FUSEDEV_ENOENT,
+    FUSEDEV_ENOTDIR,
+    FUSEDEV_ENODEV,
+    FUSEDEV_EPERM,
+    FUSEDEV_ENOMEM,
+    FUSEDEV_ENOTCONN,
+    FUSEDEV_ECONNREFUSED,
+    FUSEDEV_EOVERFLOW,
+    FUSEDEV_EBUSY,
+    FUSEDEV_ENOTEMPTY,
+    FUSEDEV_EMAXPLUS
+};
 
 struct fuse_private {
-        int                  fd;
-        uint32_t             proto_minor;
-        char                *volfile;
-        size_t               volfile_size;
-        char                *mount_point;
-        struct iobuf        *iobuf;
-
-        pthread_t            fuse_thread;
-        char                 fuse_thread_started;
-
-        uint32_t             direct_io_mode;
-        size_t              *msg0_len_p;
-
-        double               entry_timeout;
-        double               negative_timeout;
-        double               attribute_timeout;
-
-        pthread_cond_t       sync_cond;
-        pthread_mutex_t      sync_mutex;
-        char                 event_recvd;
-
-        char                 init_recvd;
-
-        gf_boolean_t         strict_volfile_check;
-
-        fuse_handler_t     **fuse_ops;
-        fuse_handler_t     **fuse_ops0;
-        pthread_mutex_t      fuse_dump_mutex;
-        int                  fuse_dump_fd;
-
-        glusterfs_graph_t   *next_graph;
-        xlator_t            *active_subvol;
-
-        pid_t                client_pid;
-        gf_boolean_t         client_pid_set;
-        unsigned             uid_map_root;
-        gf_boolean_t         acl;
-        gf_boolean_t         selinux;
-        gf_boolean_t         read_only;
-	int32_t	             fopen_keep_cache;
-	int32_t		     gid_cache_timeout;
-        gf_boolean_t         enable_ino32;
-        /* This is the mount option for disabling the root-squash for the
-           mount irrespective of whether the root-squash option for the
-           volume is set or not. But this option is honoured only for
-           thr trusted clients. For non trusted clients this value does
-           not have any affect and the volume option for root-squash is
-           honoured.
-        */
-        gf_boolean_t        no_root_squash;
-        fdtable_t           *fdtable;
-	gid_cache_t	     gid_cache;
-        char                *fuse_mountopts;
-
-        /* For fuse-reverse-validation */
-        int                  revchan_in;
-        int                  revchan_out;
-        gf_boolean_t         reverse_fuse_thread_started;
-
-        /* For communicating with separate mount thread. */
-        int                  status_pipe[2];
-
-        /* for fuse queue length and congestion threshold */
-        int background_qlen;
-        int congestion_threshold;
-
-        /* for using fuse-kernel readdirp*/
-        gf_boolean_t use_readdirp;
-
-        /* fini started, helps prevent multiple epoll worker threads
-         * firing up the fini routine */
-        gf_boolean_t fini_invoked;
+    int fd;
+    uint32_t proto_minor;
+    char *volfile;
+    size_t volfile_size;
+    char *mount_point;
+    struct iobuf *iobuf;
+
+    pthread_t *fuse_thread;
+    uint32_t reader_thread_count;
+    char fuse_thread_started;
+
+    uint32_t direct_io_mode;
+    size_t *msg0_len_p;
+
+    double entry_timeout;
+    double negative_timeout;
+    double attribute_timeout;
+
+    pthread_cond_t sync_cond;
+    pthread_mutex_t sync_mutex;
+    char event_recvd;
+
+    char init_recvd;
+
+    gf_boolean_t strict_volfile_check;
+
+    fuse_handler_t **fuse_ops;
+    fuse_handler_t **fuse_ops0;
+    pthread_mutex_t fuse_dump_mutex;
+    int fuse_dump_fd;
+
+    glusterfs_graph_t *next_graph;
+    xlator_t *active_subvol;
+
+    pid_t client_pid;
+    gf_boolean_t client_pid_set;
+    unsigned uid_map_root;
+    gf_boolean_t acl;
+    gf_boolean_t selinux;
+    gf_boolean_t read_only;
+    int32_t fopen_keep_cache;
+    int32_t gid_cache_timeout;
+    gf_boolean_t enable_ino32;
+    /* This is the mount option for disabling the root-squash for the
+       mount irrespective of whether the root-squash option for the
+       volume is set or not. But this option is honoured only for
+       thr trusted clients. For non trusted clients this value does
+       not have any affect and the volume option for root-squash is
+       honoured.
+    */
+    gf_boolean_t no_root_squash;
+    fdtable_t *fdtable;
+    gid_cache_t gid_cache;
+    char *fuse_mountopts;
+
+    /* For fuse-reverse-validation */
+    struct list_head invalidate_list;
+    pthread_cond_t invalidate_cond;
+    pthread_mutex_t invalidate_mutex;
+    gf_boolean_t reverse_fuse_thread_started;
+    uint64_t invalidate_count;
+    /* For communicating with separate mount thread. */
+    int status_pipe[2];
+
+    /* for fuse queue length and congestion threshold */
+    int background_qlen;
+    int congestion_threshold;
+
+    /* for using fuse-kernel readdirp*/
+    gf_boolean_t use_readdirp;
+
+    /* fini started, helps prevent multiple epoll worker threads
+     * firing up the fini routine */
+    gf_boolean_t fini_invoked;
+
+    /* resolve gid with getgrouplist() instead of /proc/%d/status */
+    gf_boolean_t resolve_gids;
+
+    /* Enable or disable capability support */
+    gf_boolean_t capability;
+
+    /* Enable or disable event history */
+    gf_boolean_t event_history;
+
+    /* whether to run the unmount daemon */
+    gf_boolean_t auto_unmount;
+
+    /* Load the thin volfile, and connect to gfproxyd*/
+    gf_boolean_t thin_client;
+    gf_boolean_t mount_finished;
+    gf_boolean_t handle_graph_switch;
+    pthread_cond_t migrate_cond;
+
+    /* Writeback cache support */
+    gf_boolean_t kernel_writeback_cache;
+    int attr_times_granularity;
+
+    /* Delayed fuse response */
+    struct list_head timed_list;
+    pthread_cond_t timed_cond;
+    pthread_mutex_t timed_mutex;
+    gf_boolean_t timed_response_fuse_thread_started;
+
+    /* Interrupt subscription */
+    struct list_head interrupt_list;
+    pthread_mutex_t interrupt_mutex;
+
+    gf_boolean_t flush_handle_interrupt;
+    gf_boolean_t fuse_auto_inval;
+
+    /* LRU Limit, if not set, default is 64k for now */
+    uint32_t lru_limit;
+    uint32_t invalidate_limit;
+    uint32_t fuse_dev_eperm_ratelimit_ns;
+
+    /* counters for fusdev errnos */
+    uint8_t fusedev_errno_cnt[FUSEDEV_EMAXPLUS];
+    pthread_mutex_t fusedev_errno_cnt_mutex;
 };
 typedef struct fuse_private fuse_private_t;
 
+typedef uint64_t errnomask_t[2];
+#define MASK_ERRNO(mask, n) ((mask)[(n) >> 6] |= ((uint64_t)1 << ((n)&63)))
+#define GET_ERRNO_MASK(mask, n) ((mask)[(n) >> 6] & ((uint64_t)1 << ((n)&63)))
+#define ERRNOMASK_MAX (64 * (sizeof(errnomask_t) / sizeof(uint64_t)))
+
+#define INVAL_BUF_SIZE                                                         \
+    (sizeof(struct fuse_out_header) +                                          \
+     max(sizeof(struct fuse_notify_inval_inode_out),                           \
+         sizeof(struct fuse_notify_inval_entry_out) + NAME_MAX + 1))
+
+struct fuse_invalidate_node {
+    errnomask_t errnomask;
+    struct list_head next;
+    char inval_buf[INVAL_BUF_SIZE];
+};
+typedef struct fuse_invalidate_node fuse_invalidate_node_t;
+
+struct fuse_timed_message {
+    struct fuse_out_header fuse_out_header;
+    void *fuse_message_body;
+    struct timespec scheduled_ts;
+    errnomask_t errnomask;
+    struct list_head next;
+};
+typedef struct fuse_timed_message fuse_timed_message_t;
+
+enum fuse_interrupt_state {
+    INTERRUPT_NONE,
+    INTERRUPT_SQUELCHED,
+    INTERRUPT_HANDLED,
+    INTERRUPT_WAITING_HANDLER,
+};
+typedef enum fuse_interrupt_state fuse_interrupt_state_t;
+struct fuse_interrupt_record;
+typedef struct fuse_interrupt_record fuse_interrupt_record_t;
+typedef void (*fuse_interrupt_handler_t)(xlator_t *this,
+                                         fuse_interrupt_record_t *);
+struct fuse_interrupt_record {
+    fuse_in_header_t fuse_in_header;
+    void *data;
+    gf_boolean_t hit;
+    fuse_interrupt_state_t interrupt_state;
+    fuse_interrupt_handler_t interrupt_handler;
+    pthread_cond_t handler_cond;
+    pthread_mutex_t handler_mutex;
+    struct list_head next;
+};
+
 struct fuse_graph_switch_args {
-        xlator_t        *this;
-        xlator_t        *old_subvol;
-        xlator_t        *new_subvol;
+    xlator_t *this;
+    xlator_t *old_subvol;
+    xlator_t *new_subvol;
 };
 typedef struct fuse_graph_switch_args fuse_graph_switch_args_t;
 
-#define INVAL_BUF_SIZE (sizeof (struct fuse_out_header) +               \
-                        max (sizeof (struct fuse_notify_inval_inode_out), \
-                             sizeof (struct fuse_notify_inval_entry_out) + \
-                             NAME_MAX + 1))
-
 #define FUSE_EVENT_HISTORY_SIZE 1024
 
 #define _FH_TO_FD(fh) ((fd_t *)(uintptr_t)(fh))
 
-#define FH_TO_FD(fh) ((_FH_TO_FD (fh))?(fd_ref (_FH_TO_FD (fh))):((fd_t *) 0))
-
-#define FUSE_FOP(state, ret, op_num, fop, args ...)                     \
-        do {                                                            \
-                xlator_t     *xl     = NULL;                            \
-                call_frame_t *frame  = NULL;                            \
-                                                                        \
-                xl = state->active_subvol;                              \
-                if (!xl) {                                              \
-                        gf_log_callingfn (state->this->name, GF_LOG_ERROR, \
-                                          "No active subvolume");       \
-                        send_fuse_err (state->this, state->finh, ENOENT); \
-                        free_fuse_state (state);                        \
-                        break;                                          \
-                }                                                       \
-                                                                        \
-                frame = get_call_frame_for_req (state);                 \
-                if (!frame) {                                           \
-                         /* This is not completely clean, as some       \
-                          * earlier allocations might remain unfreed    \
-                          * if we return at this point, but still       \
-                          * better than trying to go on with a NULL     \
-                          * frame ...                                   \
-                          */                                            \
-                        send_fuse_err (state->this, state->finh, ENOMEM); \
-                        free_fuse_state (state);                        \
-                        /* ideally, need to 'return', but let the */    \
-                        /* calling function take care of it */          \
-                        break;                                          \
-                }                                                       \
-                                                                        \
-                frame->root->state = state;                             \
-                frame->root->op    = op_num;                            \
-                frame->op          = op_num;                            \
-                                                                        \
-                if (state->this->history)                               \
-                        gf_log_eh ("%"PRIu64", %s, path: (%s), gfid: "  \
-                                   "(%s)", frame->root->unique,         \
-                                   gf_fop_list[frame->root->op],        \
-                                   state->loc.path,                     \
-                                   (state->fd == NULL)?                 \
-                                   uuid_utoa (state->loc.gfid):         \
-                                   uuid_utoa (state->fd->inode->gfid)); \
-                STACK_WIND (frame, ret, xl, xl->fops->fop, args);       \
-        } while (0)
-
-#define GF_SELECT_LOG_LEVEL(_errno)                     \
+#define FH_TO_FD(fh) ((_FH_TO_FD(fh)) ? (fd_ref(_FH_TO_FD(fh))) : ((fd_t *)0))
+
+/* Use the same logic as the Linux NFS-client */
+#define GF_FUSE_SQUASH_INO(ino) (((uint32_t)ino) ^ (ino >> 32))
+
+#define FUSE_FOP(state, ret, op_num, fop, args...)                             \
+    do {                                                                       \
+        xlator_t *xl = NULL;                                                   \
+        call_frame_t *frame = NULL;                                            \
+                                                                               \
+        xl = state->active_subvol;                                             \
+        if (!xl) {                                                             \
+            gf_log_callingfn(state->this->name, GF_LOG_ERROR,                  \
+                             "No active subvolume");                           \
+            send_fuse_err(state->this, state->finh, ENOENT);                   \
+            free_fuse_state(state);                                            \
+            break;                                                             \
+        }                                                                      \
+                                                                               \
+        frame = get_call_frame_for_req(state);                                 \
+        if (!frame) {                                                          \
+            /* This is not completely clean, as some                           \
+             * earlier allocations might remain unfreed                        \
+             * if we return at this point, but still                           \
+             * better than trying to go on with a NULL                         \
+             * frame ...                                                       \
+             */                                                                \
+            send_fuse_err(state->this, state->finh, ENOMEM);                   \
+            free_fuse_state(state);                                            \
+            /* ideally, need to 'return', but let the */                       \
+            /* calling function take care of it */                             \
+            break;                                                             \
+        }                                                                      \
+                                                                               \
+        frame->root->state = state;                                            \
+        frame->root->op = op_num;                                              \
+        frame->op = op_num;                                                    \
+                                                                               \
+        if (state->this->history)                                              \
+            gf_log_eh("%" PRIu64                                               \
+                      ", %s, path: (%s), gfid: "                               \
+                      "(%s)",                                                  \
+                      frame->root->unique, gf_fop_list[frame->root->op],       \
+                      state->loc.path,                                         \
+                      (state->fd == NULL)                                      \
+                          ? uuid_utoa(state->loc.gfid)                         \
+                          : uuid_utoa(state->fd->inode->gfid));                \
+        STACK_WIND(frame, ret, xl, xl->fops->fop, args);                       \
+    } while (0)
+
+#define GF_SELECT_LOG_LEVEL(_errno)                                            \
         (((_errno == ENOENT) || (_errno == ESTALE))?    \
          GF_LOG_DEBUG)
 
-#define GET_STATE(this, finh, state)                                       \
-        do {                                                               \
-                state = get_fuse_state (this, finh);                       \
-                if (!state) {                                              \
-                        gf_log ("glusterfs-fuse",                          \
-                                GF_LOG_ERROR,                              \
-                                "FUSE message unique %"PRIu64" opcode %d:" \
-                                " state allocation failed",                \
-                                finh->unique, finh->opcode);               \
-                                                                           \
-                        send_fuse_err (this, finh, ENOMEM);                \
-                        GF_FREE (finh);                                    \
-                                                                           \
-                        return;                                            \
-                }                                                          \
-        } while (0)
-
-#define FUSE_ENTRY_CREATE(this, priv, finh, state, fci, op)             \
-                do {                                                    \
-                        if (priv->proto_minor >= 12)                    \
-                                state->mode &= ~fci->umask;             \
-                        if (priv->proto_minor >= 12 && priv->acl) {     \
-                                state->xdata = dict_new ();             \
-                                if (!state->xdata) {                    \
-                                        gf_log ("glusterfs-fuse",       \
-                                                GF_LOG_WARNING,         \
-                                                "%s failed to allocate " \
-                                                "a param dictionary", op); \
-                                        send_fuse_err (this, finh, ENOMEM); \
-                                        free_fuse_state (state);        \
-                                        return;                         \
-                                }                                       \
-                                state->umask = fci->umask;              \
-                                                                        \
-/* TODO: remove this after 3.4.0 release. keeping it for the            \
-   sake of backward compatibility with old (3.3.[01])                   \
-   releases till then. */                                               \
-                                ret = dict_set_int16 (state->xdata, "umask", \
-                                                      fci->umask);      \
-                                if (ret < 0) {                          \
-                                        gf_log ("glusterfs-fuse",       \
-                                                GF_LOG_WARNING,         \
-                                                "%s Failed adding umask"\
-                                                " to request", op);     \
-                                        dict_destroy (state->xdata);    \
-                                        send_fuse_err (this, finh, ENOMEM); \
-                                        free_fuse_state (state);        \
-                                        return;                         \
-                                }                                       \
-                                ret = dict_set_int16 (state->xdata, "mode", \
-                                                      fci->mode);       \
-                                if (ret < 0) {                          \
-                                        gf_log ("glusterfs-fuse",       \
-                                                GF_LOG_WARNING,         \
-                                                "%s Failed adding mode " \
-                                                "to request", op);         \
-                                        dict_destroy (state->xdata);    \
-                                        send_fuse_err (this, finh, ENOMEM); \
-                                        free_fuse_state (state);        \
-                                        return;                         \
-                                }                                       \
-                        }                                               \
-                } while (0)
-
-#define fuse_log_eh_fop(this, state, frame, op_ret, op_errno)               \
-        do {                                                            \
-                if (this->history) {                                    \
-                        if (state->fd)                                  \
-                                gf_log_eh ("op_ret: %d, op_errno: %d, " \
-                                           "%"PRIu64", %s () => %p, gfid: %s", \
-                                           op_ret, op_errno,            \
-                                           frame->root->unique,         \
-                                           gf_fop_list[frame->root->op], \
-                                           state->fd,                   \
-                                           uuid_utoa (state->fd->inode->gfid)); \
-                        else                                            \
-                                gf_log_eh ("op_ret: %d, op_errno: %d, " \
-                                           "%"PRIu64", %s () => %s, gfid: %s", \
-                                           op_ret, op_errno,            \
-                                           frame->root->unique,         \
-                                           gf_fop_list[frame->root->op], \
-                                           state->loc.path,             \
-                                           uuid_utoa (state->loc.gfid)); \
-                }                                                       \
-        } while(0)
-
-#define fuse_log_eh(this, args...)              \
-        do {                                    \
-                if (this->history)              \
-                        gf_log_eh(args);        \
-        } while (0)
+#define GET_STATE(this, finh, state)                                           \
+    do {                                                                       \
+        state = get_fuse_state(this, finh);                                    \
+        if (!state) {                                                          \
+            gf_log("glusterfs-fuse", GF_LOG_ERROR,                             \
+                   "FUSE message unique %" PRIu64                              \
+                   " opcode %d:"                                               \
+                   " state allocation failed",                                 \
+                   finh->unique, finh->opcode);                                \
+                                                                               \
+            send_fuse_err(this, finh, ENOMEM);                                 \
+            GF_FREE(finh);                                                     \
+                                                                               \
+            return;                                                            \
+        }                                                                      \
+    } while (0)
+
+#define FUSE_ENTRY_CREATE(this, priv, finh, state, fci, op)                    \
+    do {                                                                       \
+        if (priv->proto_minor >= 12)                                           \
+            state->mode &= ~fci->umask;                                        \
+        if (priv->proto_minor >= 12 && priv->acl) {                            \
+            state->xdata = dict_new();                                         \
+            if (!state->xdata) {                                               \
+                gf_log("glusterfs-fuse", GF_LOG_WARNING,                       \
+                       "%s failed to allocate "                                \
+                       "a param dictionary",                                   \
+                       op);                                                    \
+                send_fuse_err(this, finh, ENOMEM);                             \
+                free_fuse_state(state);                                        \
+                return;                                                        \
+            }                                                                  \
+            state->umask = fci->umask;                                         \
+        }                                                                      \
+    } while (0)
+
+#define fuse_log_eh_fop(this, state, frame, op_ret, op_errno)                  \
+    do {                                                                       \
+        fuse_private_t *priv = this->private;                                  \
+        if (this->history && priv->event_history) {                            \
+            if (state->fd)                                                     \
+                gf_log_eh(                                                     \
+                    "op_ret: %d, op_errno: %d, "                               \
+                    "%" PRIu64 ", %s () => %p, gfid: %s",                      \
+                    op_ret, op_errno, frame->root->unique,                     \
+                    gf_fop_list[frame->root->op], state->fd,                   \
+                    uuid_utoa(state->fd->inode->gfid));                        \
+            else                                                               \
+                gf_log_eh(                                                     \
+                    "op_ret: %d, op_errno: %d, "                               \
+                    "%" PRIu64 ", %s () => %s, gfid: %s",                      \
+                    op_ret, op_errno, frame->root->unique,                     \
+                    gf_fop_list[frame->root->op], state->loc.path,             \
+                    uuid_utoa(state->loc.gfid));                               \
+        }                                                                      \
+    } while (0)
+
+#define fuse_log_eh(this, args...)                                             \
+    do {                                                                       \
+        fuse_private_t *priv = this->private;                                  \
+        if (this->history && priv->event_history)                              \
+            gf_log_eh(args);                                                   \
+    } while (0)
 
 static inline xlator_t *
-fuse_active_subvol (xlator_t *fuse)
+fuse_active_subvol(xlator_t *fuse)
 {
-        fuse_private_t *priv = NULL;
+    fuse_private_t *priv = NULL;
 
-        priv = fuse->private;
+    priv = fuse->private;
 
-        return priv->active_subvol;
+    return priv->active_subvol;
 }
 
-
 typedef enum {
-        RESOLVE_MUST = 1,
-        RESOLVE_NOT,
-        RESOLVE_MAY,
-        RESOLVE_DONTCARE,
-        RESOLVE_EXACT
+    RESOLVE_MUST = 1,
+    RESOLVE_NOT,
+    RESOLVE_MAY,
+    RESOLVE_DONTCARE,
+    RESOLVE_EXACT
 } fuse_resolve_type_t;
 
-
 typedef struct {
-        fuse_resolve_type_t    type;
-        fd_t                  *fd;
-        char                  *path;
-        char                  *bname;
-        u_char                 gfid[16];
-	inode_t               *hint;
-        u_char                 pargfid[16];
-	inode_t               *parhint;
-        int                    op_ret;
-        int                    op_errno;
-        loc_t                  resolve_loc;
+    fuse_resolve_type_t type;
+    fd_t *fd;
+    char *path;
+    char *bname;
+    u_char gfid[16];
+    inode_t *hint;
+    u_char pargfid[16];
+    inode_t *parhint;
+    int op_ret;
+    int op_errno;
+    loc_t resolve_loc;
 } fuse_resolve_t;
 
-
 typedef struct {
-        void             *pool;
-        xlator_t         *this;
-	xlator_t         *active_subvol;
-        inode_table_t    *itable;
-        loc_t             loc;
-        loc_t             loc2;
-        fuse_in_header_t *finh;
-        int32_t           flags;
-        off_t             off;
-        size_t            size;
-        unsigned long     nlookup;
-        fd_t             *fd;
-        dict_t           *xattr;
-        dict_t           *xdata;
-        char             *name;
-        char              is_revalidate;
-        gf_boolean_t      truncate_needed;
-        gf_lock_t         lock;
-        uint64_t          lk_owner;
-
-        /* used within resolve_and_resume */
-        /* */
-        fuse_resolve_t resolve;
-        fuse_resolve_t resolve2;
-
-        loc_t        *loc_now;
-        fuse_resolve_t *resolve_now;
-
-        void *resume_fn;
-
-        int            valid;
-        int            mask;
-        dev_t          rdev;
-        mode_t         mode;
-        mode_t         umask;
-        struct iatt    attr;
-        struct gf_flock   lk_lock;
-        struct iovec   vector;
-
-        uuid_t         gfid;
-        uint32_t       io_flags;
-        int32_t        fd_no;
+    void *pool;
+    xlator_t *this;
+    xlator_t *active_subvol;
+    inode_table_t *itable;
+    loc_t loc;
+    loc_t loc2;
+    fuse_in_header_t *finh;
+    int32_t flags;
+
+    off_t off;
+    /*
+     * The man page of copy_file_range tells that the offset
+     * arguments are of type loff_t *. Here in fuse state, the values of
+     * those offsets are saved instead of pointers as the kernel sends
+     * the values of the offsets from those pointers instead of pointers.
+     * But the type loff_t is linux specific and is actually a typedef of
+     * off64_t. Hence using off64_t
+     */
+    off64_t off_in;  /* for copy_file_range source fd */
+    off64_t off_out; /* for copy_file_range destination fd */
+    size_t size;
+    unsigned long nlookup;
+    fd_t *fd;
+    fd_t *fd_dst; /* for copy_file_range destination */
+    dict_t *xattr;
+    dict_t *xdata;
+    char *name;
+    char is_revalidate;
+    gf_boolean_t truncate_needed;
+    gf_lock_t lock;
+    uint64_t lk_owner;
+
+    /* used within resolve_and_resume */
+    /* */
+    fuse_resolve_t resolve;
+    fuse_resolve_t resolve2;
+
+    loc_t *loc_now;
+    fuse_resolve_t *resolve_now;
+
+    void *resume_fn;
+
+    int valid;
+    int mask;
+    dev_t rdev;
+    mode_t mode;
+    mode_t umask;
+    struct iatt attr;
+    struct gf_flock lk_lock;
+    struct iovec vector;
+
+    uuid_t gfid;
+    uint32_t io_flags;
+    int32_t fd_no;
+
+    gf_seek_what_t whence;
+    struct iobuf *iobuf;
 } fuse_state_t;
 
 typedef struct {
-        uint32_t  open_flags;
-        char      migration_failed;
-        fd_t     *activefd;
+    uint32_t open_flags;
+    char migration_failed;
+    fd_t *activefd;
 } fuse_fd_ctx_t;
 
-typedef void (*fuse_resume_fn_t) (fuse_state_t *state);
+typedef void (*fuse_resume_fn_t)(fuse_state_t *state);
 
 GF_MUST_CHECK int32_t
-fuse_loc_fill (loc_t *loc, fuse_state_t *state, ino_t ino,
-               ino_t par, const char *name);
-call_frame_t *get_call_frame_for_req (fuse_state_t *state);
-fuse_state_t *get_fuse_state (xlator_t *this, fuse_in_header_t *finh);
-void free_fuse_state (fuse_state_t *state);
-void gf_fuse_stat2attr (struct iatt *st, struct fuse_attr *fa,
-                        gf_boolean_t enable_ino32);
-void gf_fuse_fill_dirent (gf_dirent_t *entry, struct fuse_dirent *fde,
-                          gf_boolean_t enable_ino32);
-uint64_t inode_to_fuse_nodeid (inode_t *inode);
-xlator_t *fuse_active_subvol (xlator_t *fuse);
-inode_t *fuse_ino_to_inode (uint64_t ino, xlator_t *fuse);
-int send_fuse_err (xlator_t *this, fuse_in_header_t *finh, int error);
-int fuse_gfid_set (fuse_state_t *state);
-int fuse_flip_xattr_ns (struct fuse_private *priv, char *okey, char **nkey);
-fuse_fd_ctx_t * __fuse_fd_ctx_check_n_create (xlator_t *this, fd_t *fd);
-fuse_fd_ctx_t * fuse_fd_ctx_check_n_create (xlator_t *this, fd_t *fd);
-
-int fuse_resolve_and_resume (fuse_state_t *state, fuse_resume_fn_t fn);
-int fuse_resolve_inode_init (fuse_state_t *state, fuse_resolve_t *resolve,
-			     ino_t ino);
-int fuse_resolve_entry_init (fuse_state_t *state, fuse_resolve_t *resolve,
-			     ino_t par, char *name);
-int fuse_resolve_fd_init (fuse_state_t *state, fuse_resolve_t *resolve,
-			  fd_t *fd);
-int fuse_ignore_xattr_set (fuse_private_t *priv, char *key);
-void fuse_fop_resume (fuse_state_t *state);
-int dump_history_fuse (circular_buffer_t *cb, void *data);
+fuse_loc_fill(loc_t *loc, fuse_state_t *state, ino_t ino, ino_t par,
+              const char *name);
+call_frame_t *
+get_call_frame_for_req(fuse_state_t *state);
+fuse_state_t *
+get_fuse_state(xlator_t *this, fuse_in_header_t *finh);
+void
+free_fuse_state(fuse_state_t *state);
+void
+gf_fuse_stat2attr(struct iatt *st, struct fuse_attr *fa,
+                  gf_boolean_t enable_ino32);
+void
+gf_fuse_fill_dirent(gf_dirent_t *entry, struct fuse_dirent *fde,
+                    gf_boolean_t enable_ino32);
+uint64_t
+inode_to_fuse_nodeid(inode_t *inode);
+xlator_t *
+fuse_active_subvol(xlator_t *fuse);
+inode_t *
+fuse_ino_to_inode(uint64_t ino, xlator_t *fuse);
+int
+send_fuse_err(xlator_t *this, fuse_in_header_t *finh, int error);
+int
+fuse_gfid_set(fuse_state_t *state);
+int
+fuse_flip_xattr_ns(struct fuse_private *priv, char *okey, char **nkey);
+fuse_fd_ctx_t *
+__fuse_fd_ctx_check_n_create(xlator_t *this, fd_t *fd);
+fuse_fd_ctx_t *
+fuse_fd_ctx_check_n_create(xlator_t *this, fd_t *fd);
+
+int
+fuse_resolve_and_resume(fuse_state_t *state, fuse_resume_fn_t fn);
+int
+fuse_resolve_inode_init(fuse_state_t *state, fuse_resolve_t *resolve,
+                        ino_t ino);
+int
+fuse_resolve_entry_init(fuse_state_t *state, fuse_resolve_t *resolve, ino_t par,
+                        char *name);
+int
+fuse_resolve_fd_init(fuse_state_t *state, fuse_resolve_t *resolve, fd_t *fd);
+int
+fuse_ignore_xattr_set(fuse_private_t *priv, char *key);
+void
+fuse_fop_resume(fuse_state_t *state);
+int
+dump_history_fuse(circular_buffer_t *cb, void *data);
+int
+fuse_check_selinux_cap_xattr(fuse_private_t *priv, char *name);
 #endif /* _GF_FUSE_BRIDGE_H_ */
diff --git a/xlators/mount/fuse/src/fuse-helpers.c b/xlators/mount/fuse/src/fuse-helpers.c
index a8d387fdd4b..a2b0ad11fe4 100644
--- a/xlators/mount/fuse/src/fuse-helpers.c
+++ b/xlators/mount/fuse/src/fuse-helpers.c
@@ -16,236 +16,295 @@
 #elif defined(__FreeBSD__)
 #include <sys/types.h>
 #include <libutil.h>
-#include <sys/user.h>
 #elif defined(CTL_KERN)
 #include <sys/sysctl.h>
 #endif
+#include <pwd.h>
 
 #include "fuse-bridge.h"
 
 static void
-fuse_resolve_wipe (fuse_resolve_t *resolve)
+fuse_resolve_wipe(fuse_resolve_t *resolve)
 {
-        GF_FREE ((void *)resolve->path);
+    GF_FREE((void *)resolve->path);
 
-        GF_FREE ((void *)resolve->bname);
+    GF_FREE((void *)resolve->bname);
 
-        if (resolve->fd)
-                fd_unref (resolve->fd);
+    if (resolve->fd)
+        fd_unref(resolve->fd);
 
-        loc_wipe (&resolve->resolve_loc);
+    loc_wipe(&resolve->resolve_loc);
 
-	if (resolve->hint) {
-		inode_unref (resolve->hint);
-		resolve->hint = 0;
-	}
+    if (resolve->hint) {
+        inode_unref(resolve->hint);
+        resolve->hint = 0;
+    }
 
-	if (resolve->parhint) {
-		inode_unref (resolve->parhint);
-		resolve->parhint = 0;
-	}
+    if (resolve->parhint) {
+        inode_unref(resolve->parhint);
+        resolve->parhint = 0;
+    }
 }
 
-
 void
-free_fuse_state (fuse_state_t *state)
+free_fuse_state(fuse_state_t *state)
 {
-        xlator_t       *this     = NULL;
-        fuse_private_t *priv     = NULL;
-        uint64_t        winds    = 0;
-        char            switched = 0;
+    xlator_t *this = NULL;
+    fuse_private_t *priv = NULL;
+    uint64_t winds = 0;
+    char switched = 0;
+
+    this = state->this;
+
+    priv = this->private;
+
+    loc_wipe(&state->loc);
+
+    loc_wipe(&state->loc2);
+
+    if (state->xdata) {
+        dict_unref(state->xdata);
+        state->xdata = (void *)0xaaaaeeee;
+    }
+    if (state->xattr)
+        dict_unref(state->xattr);
+
+    if (state->name) {
+        GF_FREE(state->name);
+        state->name = NULL;
+    }
+    if (state->fd) {
+        fd_unref(state->fd);
+        state->fd = (void *)0xfdfdfdfd;
+    }
+    if (state->finh) {
+        GF_FREE(state->finh);
+        state->finh = NULL;
+    }
+
+    fuse_resolve_wipe(&state->resolve);
+    fuse_resolve_wipe(&state->resolve2);
+
+    pthread_mutex_lock(&priv->sync_mutex);
+    {
+        winds = --state->active_subvol->winds;
+        switched = state->active_subvol->switched;
+    }
+    pthread_mutex_unlock(&priv->sync_mutex);
+
+    if ((winds == 0) && (switched)) {
+        xlator_notify(state->active_subvol, GF_EVENT_PARENT_DOWN,
+                      state->active_subvol, NULL);
+    }
 
-        this = state->this;
+#ifdef DEBUG
+    memset(state, 0x90, sizeof(*state));
+#endif
+    GF_FREE(state);
+    state = NULL;
+}
 
-        priv = this->private;
+fuse_state_t *
+get_fuse_state(xlator_t *this, fuse_in_header_t *finh)
+{
+    fuse_state_t *state = NULL;
+    xlator_t *active_subvol = NULL;
+    fuse_private_t *priv = NULL;
 
-        loc_wipe (&state->loc);
+    state = (void *)GF_CALLOC(1, sizeof(*state), gf_fuse_mt_fuse_state_t);
+    if (!state)
+        return NULL;
 
-        loc_wipe (&state->loc2);
+    state->this = THIS;
+    priv = this->private;
 
-        if (state->xdata) {
-                dict_unref (state->xdata);
-                state->xdata = (void *)0xaaaaeeee;
-        }
-        if (state->xattr)
-                dict_unref (state->xattr);
+    pthread_mutex_lock(&priv->sync_mutex);
+    {
+        while (priv->handle_graph_switch)
+            pthread_cond_wait(&priv->migrate_cond, &priv->sync_mutex);
+        active_subvol = fuse_active_subvol(state->this);
+        active_subvol->winds++;
+    }
+    pthread_mutex_unlock(&priv->sync_mutex);
 
-        if (state->name) {
-                GF_FREE (state->name);
-                state->name = NULL;
-        }
-        if (state->fd) {
-                fd_unref (state->fd);
-                state->fd = (void *)0xfdfdfdfd;
-        }
-        if (state->finh) {
-                GF_FREE (state->finh);
-                state->finh = NULL;
-        }
+    state->active_subvol = active_subvol;
+    state->itable = active_subvol->itable;
 
-        fuse_resolve_wipe (&state->resolve);
-        fuse_resolve_wipe (&state->resolve2);
+    state->pool = this->ctx->pool;
+    state->finh = finh;
+    state->this = this;
 
-        pthread_mutex_lock (&priv->sync_mutex);
-        {
-                winds = --state->active_subvol->winds;
-                switched = state->active_subvol->switched;
-        }
-        pthread_mutex_unlock (&priv->sync_mutex);
+    LOCK_INIT(&state->lock);
 
-        if ((winds == 0) && (switched)) {
-                xlator_notify (state->active_subvol, GF_EVENT_PARENT_DOWN,
-                               state->active_subvol, NULL);
-        }
-
-#ifdef DEBUG
-        memset (state, 0x90, sizeof (*state));
-#endif
-        GF_FREE (state);
-        state = NULL;
+    return state;
 }
 
-
-fuse_state_t *
-get_fuse_state (xlator_t *this, fuse_in_header_t *finh)
+void
+frame_fill_groups(call_frame_t *frame)
 {
-        fuse_state_t   *state         = NULL;
-	xlator_t       *active_subvol = NULL;
-        fuse_private_t *priv          = NULL;
-
-        state = (void *)GF_CALLOC (1, sizeof (*state),
-                                   gf_fuse_mt_fuse_state_t);
-        if (!state)
-                return NULL;
-
-	state->this = THIS;
-        priv = this->private;
-
-        pthread_mutex_lock (&priv->sync_mutex);
-        {
-                active_subvol = fuse_active_subvol (state->this);
-                active_subvol->winds++;
+#if defined(GF_LINUX_HOST_OS)
+    xlator_t *this = frame->this;
+    fuse_private_t *priv = this->private;
+    char filename[32];
+    char line[4096];
+    char *ptr = NULL;
+    long int id = 0;
+    char *saveptr = NULL;
+    char *endptr = NULL;
+    int ret = 0;
+    int ngroups = 0;
+    gid_t *mygroups = NULL;
+
+    if (priv->resolve_gids) {
+        struct passwd pwent;
+        char mystrs[1024];
+        struct passwd *result;
+
+        if (getpwuid_r(frame->root->uid, &pwent, mystrs, sizeof(mystrs),
+                       &result) != 0) {
+            gf_log(this->name, GF_LOG_ERROR,
+                   "getpwuid_r(%u) "
+                   "failed",
+                   frame->root->uid);
+            return;
+        }
+        if (result == 0) {
+            gf_log(this->name, GF_LOG_ERROR,
+                   "getpwuid_r(%u): "
+                   "no matching record",
+                   frame->root->uid);
+            return;
         }
-        pthread_mutex_unlock (&priv->sync_mutex);
-
-	state->active_subvol = active_subvol;
-	state->itable = active_subvol->itable;
-
-        state->pool = this->ctx->pool;
-        state->finh = finh;
-        state->this = this;
 
-        LOCK_INIT (&state->lock);
+        ngroups = gf_getgrouplist(result->pw_name, frame->root->gid, &mygroups);
+        if (ngroups == -1) {
+            gf_log(this->name, GF_LOG_ERROR,
+                   "could not map %s to "
+                   "group list (ngroups %d, max %d)",
+                   result->pw_name, ngroups, GF_MAX_AUX_GROUPS);
+            return;
+        }
 
-        return state;
-}
+        call_stack_set_groups(frame->root, ngroups, &mygroups);
+    } else {
+        FILE *fp = NULL;
 
+        ret = snprintf(filename, sizeof filename, "/proc/%d/status",
+                       frame->root->pid);
+        if (ret >= sizeof filename) {
+            gf_log(this->name, GF_LOG_ERROR, "procfs path exceeds buffer size");
+            goto out;
+        }
 
-#define FUSE_MAX_AUX_GROUPS 32 /* We can get only up to 32 aux groups from /proc */
-void
-frame_fill_groups (call_frame_t *frame)
-{
-#if defined(GF_LINUX_HOST_OS)
-        char         filename[32];
-        char         line[4096];
-        char        *ptr = NULL;
-        FILE        *fp = NULL;
-        int          idx = 0;
-        long int     id = 0;
-        char        *saveptr = NULL;
-        char        *endptr = NULL;
-        int          ret = 0;
-
-        ret = snprintf (filename, sizeof filename, "/proc/%d/status",
-                        frame->root->pid);
-        if (ret >= sizeof filename)
-                goto out;
+        fp = fopen(filename, "r");
+        if (!fp) {
+            gf_log(this->name, GF_LOG_ERROR, "failed to open %s: %s", filename,
+                   strerror(errno));
+            goto out;
+        }
 
-        fp = fopen (filename, "r");
-        if (!fp)
-                goto out;
+        for (;;) {
+            gf_boolean_t found_groups = _gf_false;
+            int idx = 0;
 
-        if (call_stack_alloc_groups (frame->root, FUSE_MAX_AUX_GROUPS) != 0)
+            if (call_stack_alloc_groups(frame->root, ngroups) != 0) {
+                gf_log(this->name, GF_LOG_ERROR,
+                       "failed to allocate gid buffer");
                 goto out;
+            }
 
-        while ((ptr = fgets (line, sizeof line, fp))) {
-                if (strncmp (ptr, "Groups:", 7) != 0)
-                        continue;
-
-                ptr = line + 8;
-
-                for (ptr = strtok_r (ptr, " \t\r\n", &saveptr);
-                     ptr;
-                     ptr = strtok_r (NULL, " \t\r\n", &saveptr)) {
-                        errno = 0;
-                        id = strtol (ptr, &endptr, 0);
-                        if (errno == ERANGE)
-                                break;
-                        if (!endptr || *endptr)
-                                break;
-                        frame->root->groups[idx++] = id;
-                        if (idx == FUSE_MAX_AUX_GROUPS)
-                                break;
+            while ((ptr = fgets(line, sizeof line, fp))) {
+                if (strncmp(ptr, "Groups:", 7) == 0) {
+                    found_groups = _gf_true;
+                    break;
                 }
-
+            }
+            if (!found_groups) {
+                gf_log(this->name, GF_LOG_ERROR, "cannot find gid list in %s",
+                       filename);
+                break;
+            }
+            ptr = line + 8;
+
+            for (ptr = strtok_r(ptr, " \t\r\n", &saveptr); ptr;
+                 ptr = strtok_r(NULL, " \t\r\n", &saveptr)) {
+                errno = 0;
+                id = strtol(ptr, &endptr, 0);
+                if (errno == ERANGE || !endptr || *endptr) {
+                    gf_log(this->name, GF_LOG_ERROR, "failed to parse %s",
+                           filename);
+                    break;
+                }
+                if (idx < call_stack_groups_capacity(frame->root))
+                    frame->root->groups[idx] = id;
+                idx++;
+                if (idx == GF_MAX_AUX_GROUPS)
+                    break;
+            }
+            if (idx > call_stack_groups_capacity(frame->root)) {
+                ngroups = idx;
+                rewind(fp);
+            } else {
                 frame->root->ngrps = idx;
                 break;
+            }
         }
-out:
+    out:
         if (fp)
-                fclose (fp);
+            fclose(fp);
+    }
 #elif defined(GF_SOLARIS_HOST_OS)
-        char         filename[32];
-        char         scratch[128];
-        prcred_t    *prcred = (prcred_t *) scratch;
-        FILE        *fp = NULL;
-        int          ret = 0;
-        int          ngrps;
-
-        ret = snprintf (filename, sizeof filename,
-                        "/proc/%d/cred", frame->root->pid);
-
-        if (ret < sizeof filename) {
-                fp = fopen (filename, "r");
-                if (fp != NULL) {
-                        if (fgets (scratch, sizeof scratch, fp) != NULL) {
-                                ngrps = MIN(prcred->pr_ngroups,
-                                            FUSE_MAX_AUX_GROUPS);
-                                if (call_stack_alloc_groups (frame->root,
-                                                             ngrps) != 0)
-                                        return;
-                        }
-                        fclose (fp);
-                 }
-         }
+    char filename[32];
+    char scratch[128];
+    prcred_t *prcred = (prcred_t *)scratch;
+    FILE *fp = NULL;
+    int ret = 0;
+    int ngrps;
+
+    ret = snprintf(filename, sizeof filename, "/proc/%d/cred",
+                   frame->root->pid);
+
+    if (ret < sizeof filename) {
+        fp = fopen(filename, "r");
+        if (fp != NULL) {
+            if (fgets(scratch, sizeof scratch, fp) != NULL) {
+                ngrps = MIN(prcred->pr_ngroups, GF_MAX_AUX_GROUPS);
+                if (call_stack_alloc_groups(frame->root, ngrps) != 0) {
+                    fclose(fp);
+                    return;
+                }
+            }
+            fclose(fp);
+        }
+    }
 #elif defined(CTL_KERN) /* DARWIN and *BSD */
-        /*
-           N.B. CTL_KERN is an enum on Linux. (Meaning, if it's not
-           obvious, that it's not subject to preprocessor directives
-           like '#if defined'.)
-           Unlike Linux, on Mac OS and the BSDs it is a #define. We
-           could test to see that KERN_PROC is defined, but, barring any
-           evidence to the contrary, I think that's overkill.
-           We might also test that GF_DARWIN_HOST_OS is defined, why
-           limit this to just Mac OS. It's equally valid for the BSDs
-           and we do have people building on NetBSD and FreeBSD.
-        */
-        int name[] = { CTL_KERN, KERN_PROC, KERN_PROC_PID, frame->root->pid };
-        size_t namelen = sizeof name / sizeof name[0];
-        struct kinfo_proc kp;
-        size_t kplen = sizeof(kp);
-        int i, ngroups;
-
-        if (sysctl(name, namelen, &kp, &kplen, NULL, 0) != 0)
-                return;
-        ngroups = MIN(kp.kp_eproc.e_ucred.cr_ngroups, NGROUPS_MAX);
-        if (call_stack_alloc_groups (frame->root, ngroups) != 0)
-                return;
-        for (i = 0; i < ngroups; i++)
-                frame->root->groups[i] = kp.kp_eproc.e_ucred.cr_groups[i];
-        frame->root->ngrps = ngroups;
+    /*
+       N.B. CTL_KERN is an enum on Linux. (Meaning, if it's not
+       obvious, that it's not subject to preprocessor directives
+       like '#if defined'.)
+       Unlike Linux, on Mac OS and the BSDs it is a #define. We
+       could test to see that KERN_PROC is defined, but, barring any
+       evidence to the contrary, I think that's overkill.
+       We might also test that GF_DARWIN_HOST_OS is defined, why
+       limit this to just Mac OS. It's equally valid for the BSDs
+       and we do have people building on NetBSD and FreeBSD.
+    */
+    int name[] = {CTL_KERN, KERN_PROC, KERN_PROC_PID, frame->root->pid};
+    size_t namelen = sizeof name / sizeof name[0];
+    struct kinfo_proc kp;
+    size_t kplen = sizeof(kp);
+    int i, ngroups;
+
+    if (sysctl(name, namelen, &kp, &kplen, NULL, 0) != 0)
+        return;
+    ngroups = MIN(kp.kp_eproc.e_ucred.cr_ngroups, NGROUPS_MAX);
+    if (call_stack_alloc_groups(frame->root, ngroups) != 0)
+        return;
+    for (i = 0; i < ngroups; i++)
+        frame->root->groups[i] = kp.kp_eproc.e_ucred.cr_groups[i];
+    frame->root->ngrps = ngroups;
 #else
-        frame->root->ngrps = 0;
+    frame->root->ngrps = 0;
 #endif /* GF_LINUX_HOST_OS */
 }
 
@@ -253,363 +312,377 @@ out:
  * Get the groups for the PID associated with this frame. If enabled,
  * use the gid cache to reduce group list collection.
  */
-static void get_groups(fuse_private_t *priv, call_frame_t *frame)
+static void
+get_groups(fuse_private_t *priv, call_frame_t *frame)
 {
-	int i;
-	const gid_list_t *gl;
-	gid_list_t agl;
-
-	if (!priv || !priv->gid_cache_timeout) {
-		frame_fill_groups(frame);
-		return;
-	}
-
-        if (-1 == priv->gid_cache_timeout) {
-                frame->root->ngrps = 0;
-                return;
-        }
+    int i;
+    const gid_list_t *gl;
+    gid_list_t agl;
+
+    if (!priv || !priv->gid_cache_timeout) {
+        frame_fill_groups(frame);
+        return;
+    }
 
-	gl = gid_cache_lookup(&priv->gid_cache, frame->root->pid,
-			      frame->root->uid, frame->root->gid);
-	if (gl) {
-		if (call_stack_alloc_groups (frame->root, gl->gl_count) != 0)
-			return;
-		frame->root->ngrps = gl->gl_count;
-		for (i = 0; i < gl->gl_count; i++)
-			frame->root->groups[i] = gl->gl_list[i];
-		gid_cache_release(&priv->gid_cache, gl);
-		return;
-	}
-
-	frame_fill_groups (frame);
-
-	agl.gl_id = frame->root->pid;
-	agl.gl_uid = frame->root->uid;
-	agl.gl_gid = frame->root->gid;
-	agl.gl_count = frame->root->ngrps;
-	agl.gl_list = GF_CALLOC(frame->root->ngrps, sizeof(gid_t),
-			gf_fuse_mt_gids_t);
-	if (!agl.gl_list)
-		return;
-
-	for (i = 0; i < frame->root->ngrps; i++)
-		agl.gl_list[i] = frame->root->groups[i];
-
-	if (gid_cache_add(&priv->gid_cache, &agl) != 1)
-		GF_FREE(agl.gl_list);
+    if (-1 == priv->gid_cache_timeout) {
+        frame->root->ngrps = 0;
+        return;
+    }
+
+    gl = gid_cache_lookup(&priv->gid_cache, frame->root->pid, frame->root->uid,
+                          frame->root->gid);
+    if (gl) {
+        if (call_stack_alloc_groups(frame->root, gl->gl_count) != 0) {
+            gid_cache_release(&priv->gid_cache, gl);
+            return;
+        }
+        frame->root->ngrps = gl->gl_count;
+        for (i = 0; i < gl->gl_count; i++)
+            frame->root->groups[i] = gl->gl_list[i];
+        gid_cache_release(&priv->gid_cache, gl);
+        return;
+    }
+
+    frame_fill_groups(frame);
+
+    agl.gl_id = frame->root->pid;
+    agl.gl_uid = frame->root->uid;
+    agl.gl_gid = frame->root->gid;
+    agl.gl_count = frame->root->ngrps;
+    agl.gl_list = GF_CALLOC(frame->root->ngrps, sizeof(gid_t),
+                            gf_fuse_mt_gids_t);
+    if (!agl.gl_list)
+        return;
+
+    for (i = 0; i < frame->root->ngrps; i++)
+        agl.gl_list[i] = frame->root->groups[i];
+
+    if (gid_cache_add(&priv->gid_cache, &agl) != 1)
+        GF_FREE(agl.gl_list);
 }
 
 call_frame_t *
-get_call_frame_for_req (fuse_state_t *state)
+get_call_frame_for_req(fuse_state_t *state)
 {
-        call_pool_t           *pool = NULL;
-        fuse_in_header_t      *finh = NULL;
-        call_frame_t          *frame = NULL;
-        xlator_t              *this = NULL;
-        fuse_private_t        *priv = NULL;
-
-        pool = state->pool;
-        finh = state->finh;
-        this = state->this;
-        priv = this->private;
-
-        frame = create_frame (this, pool);
-        if (!frame)
-                return NULL;
-
-        if (finh) {
-                frame->root->uid      = finh->uid;
-                frame->root->gid      = finh->gid;
-                frame->root->pid      = finh->pid;
-                frame->root->unique   = finh->unique;
-                set_lk_owner_from_uint64 (&frame->root->lk_owner,
-                                          state->lk_owner);
-        }
+    call_pool_t *pool = NULL;
+    fuse_in_header_t *finh = NULL;
+    call_frame_t *frame = NULL;
+    xlator_t *this = NULL;
+    fuse_private_t *priv = NULL;
 
-        get_groups(priv, frame);
+    pool = state->pool;
+    finh = state->finh;
+    this = state->this;
+    priv = this->private;
 
-        if (priv && priv->client_pid_set)
-                frame->root->pid = priv->client_pid;
+    frame = create_frame(this, pool);
+    if (!frame)
+        return NULL;
 
-        frame->root->type = GF_OP_TYPE_FOP;
+    if (finh) {
+        frame->root->uid = finh->uid;
+        frame->root->gid = finh->gid;
+        frame->root->pid = finh->pid;
+        set_lk_owner_from_uint64(&frame->root->lk_owner, state->lk_owner);
+    }
 
-        return frame;
-}
+    get_groups(priv, frame);
+
+    if (priv && priv->client_pid_set)
+        frame->root->pid = priv->client_pid;
 
+    frame->root->type = GF_OP_TYPE_FOP;
+
+    return frame;
+}
 
 inode_t *
-fuse_ino_to_inode (uint64_t ino, xlator_t *fuse)
+fuse_ino_to_inode(uint64_t ino, xlator_t *fuse)
 {
-        inode_t  *inode = NULL;
-        xlator_t *active_subvol = NULL;
-
-        if (ino == 1) {
-                active_subvol = fuse_active_subvol (fuse);
-                if (active_subvol)
-                        inode = active_subvol->itable->root;
-        } else {
-                inode = (inode_t *) (unsigned long) ino;
-                inode_ref (inode);
-        }
-
-        return inode;
+    inode_t *inode = NULL;
+    xlator_t *active_subvol = NULL;
+
+    if (ino == 1) {
+        active_subvol = fuse_active_subvol(fuse);
+        if (active_subvol)
+            inode = active_subvol->itable->root;
+    } else {
+        inode = (inode_t *)(unsigned long)ino;
+        inode_ref(inode);
+    }
+
+    return inode;
 }
 
 uint64_t
-inode_to_fuse_nodeid (inode_t *inode)
+inode_to_fuse_nodeid(inode_t *inode)
 {
-        if (!inode)
-		return 0;
-	if (__is_root_gfid (inode->gfid))
-                return 1;
+    if (!inode)
+        return 0;
+    if (__is_root_gfid(inode->gfid))
+        return 1;
 
-        return (unsigned long) inode;
+    return (unsigned long)inode;
 }
 
-
 GF_MUST_CHECK int32_t
-fuse_loc_fill (loc_t *loc, fuse_state_t *state, ino_t ino,
-               ino_t par, const char *name)
+fuse_loc_fill(loc_t *loc, fuse_state_t *state, ino_t ino, ino_t par,
+              const char *name)
 {
-        inode_t  *inode = NULL;
-        inode_t  *parent = NULL;
-        int32_t   ret = -1;
-        char     *path = NULL;
-        uuid_t    null_gfid = {0,};
-
-        /* resistance against multiple invocation of loc_fill not to get
-           reference leaks via inode_search() */
-
-        if (name) {
-                parent = loc->parent;
-                if (!parent) {
-                        parent = fuse_ino_to_inode (par, state->this);
-                        loc->parent = parent;
-                        if (parent)
-                                gf_uuid_copy (loc->pargfid, parent->gfid);
-                }
-
-                inode = loc->inode;
-                if (!inode && parent) {
-                        inode = inode_grep (parent->table, parent, name);
-                        loc->inode = inode;
-                }
-
-                ret = inode_path (parent, name, &path);
-                if (ret <= 0) {
-                        gf_log ("glusterfs-fuse", GF_LOG_DEBUG,
-                                "inode_path failed for %s/%s",
-                                (parent)?uuid_utoa (parent->gfid):"0", name);
-                        goto fail;
-                }
-                loc->path = path;
-        } else {
-                inode = loc->inode;
-                if (!inode) {
-                        inode = fuse_ino_to_inode (ino, state->this);
-                        loc->inode = inode;
-                        if (inode)
-                                gf_uuid_copy (loc->gfid, inode->gfid);
-                }
-
-                parent = loc->parent;
-                if (!parent) {
-                        parent = inode_parent (inode, null_gfid, NULL);
-                        loc->parent = parent;
-                        if (parent)
-                                gf_uuid_copy (loc->pargfid, parent->gfid);
+    inode_t *inode = NULL;
+    inode_t *parent = NULL;
+    int32_t ret = -1;
+    char *path = NULL;
+    uuid_t null_gfid = {
+        0,
+    };
+
+    /* resistance against multiple invocation of loc_fill not to get
+       reference leaks via inode_search() */
+
+    if (name) {
+        parent = loc->parent;
+        if (!parent) {
+            parent = fuse_ino_to_inode(par, state->this);
+            loc->parent = parent;
+            if (parent)
+                gf_uuid_copy(loc->pargfid, parent->gfid);
+        }
 
-                }
+        inode = loc->inode;
+        if (!inode && parent) {
+            inode = inode_grep(parent->table, parent, name);
+            loc->inode = inode;
+        }
 
-                ret = inode_path (inode, NULL, &path);
-                if (ret <= 0) {
-                        gf_log ("glusterfs-fuse", GF_LOG_DEBUG,
-                                "inode_path failed for %s",
-                                (inode) ? uuid_utoa (inode->gfid) : "0");
-                        goto fail;
-                }
-                loc->path = path;
+        ret = inode_path(parent, name, &path);
+        if (ret <= 0) {
+            gf_log("glusterfs-fuse", GF_LOG_DEBUG,
+                   "inode_path failed for %s/%s",
+                   (parent) ? uuid_utoa(parent->gfid) : "0", name);
+            goto fail;
+        }
+        loc->path = path;
+    } else {
+        inode = loc->inode;
+        if (!inode) {
+            inode = fuse_ino_to_inode(ino, state->this);
+            loc->inode = inode;
+            if (inode)
+                gf_uuid_copy(loc->gfid, inode->gfid);
         }
 
-        if (loc->path) {
-                loc->name = strrchr (loc->path, '/');
-                if (loc->name)
-                        loc->name++;
-                else
-                        loc->name = "";
+        parent = loc->parent;
+        if (!parent) {
+            parent = inode_parent(inode, null_gfid, NULL);
+            loc->parent = parent;
+            if (parent)
+                gf_uuid_copy(loc->pargfid, parent->gfid);
         }
 
-        if ((ino != 1) && (parent == NULL)) {
-                gf_log ("fuse-bridge", GF_LOG_DEBUG,
-                        "failed to search parent for %"PRId64"/%s (%"PRId64")",
-                        (ino_t)par, name, (ino_t)ino);
-                ret = -1;
-                goto fail;
+        ret = inode_path(inode, NULL, &path);
+        if (ret <= 0) {
+            gf_log("glusterfs-fuse", GF_LOG_DEBUG, "inode_path failed for %s",
+                   (inode) ? uuid_utoa(inode->gfid) : "0");
+            goto fail;
         }
-        ret = 0;
+        loc->path = path;
+    }
+
+    if (loc->path) {
+        loc->name = strrchr(loc->path, '/');
+        if (loc->name)
+            loc->name++;
+        else
+            loc->name = "";
+    }
+
+    if ((ino != 1) && (parent == NULL)) {
+        gf_log("fuse-bridge", GF_LOG_DEBUG,
+               "failed to search parent for %" PRId64 "/%s (%" PRId64 ")",
+               (ino_t)par, name, (ino_t)ino);
+        ret = -1;
+        goto fail;
+    }
+    ret = 0;
 fail:
-        /* this should not happen as inode_path returns -1 when buf is NULL
-           for sure */
-        if (path && !loc->path)
-                GF_FREE (path);
-        return ret;
+    /* this should not happen as inode_path returns -1 when buf is NULL
+       for sure */
+    if (path && !loc->path)
+        GF_FREE(path);
+    return ret;
 }
 
-/* Use the same logic as the Linux NFS-client */
-#define GF_FUSE_SQUASH_INO(ino) ((uint32_t) ino) ^ (ino >> 32)
-
 /* courtesy of folly */
 void
-gf_fuse_stat2attr (struct iatt *st, struct fuse_attr *fa, gf_boolean_t enable_ino32)
+gf_fuse_stat2attr(struct iatt *st, struct fuse_attr *fa,
+                  gf_boolean_t enable_ino32)
 {
-        if (enable_ino32)
-                fa->ino = GF_FUSE_SQUASH_INO(st->ia_ino);
-        else
-                fa->ino = st->ia_ino;
-
-        fa->size        = st->ia_size;
-        fa->blocks      = st->ia_blocks;
-        fa->atime       = st->ia_atime;
-        fa->mtime       = st->ia_mtime;
-        fa->ctime       = st->ia_ctime;
-        fa->atimensec   = st->ia_atime_nsec;
-        fa->mtimensec   = st->ia_mtime_nsec;
-        fa->ctimensec   = st->ia_ctime_nsec;
-        fa->mode        = st_mode_from_ia (st->ia_prot, st->ia_type);
-        fa->nlink       = st->ia_nlink;
-        fa->uid         = st->ia_uid;
-        fa->gid         = st->ia_gid;
-        fa->rdev        = makedev (ia_major (st->ia_rdev),
-                                   ia_minor (st->ia_rdev));
+    if (enable_ino32)
+        fa->ino = GF_FUSE_SQUASH_INO(st->ia_ino);
+    else
+        fa->ino = st->ia_ino;
+
+    fa->size = st->ia_size;
+    fa->blocks = st->ia_blocks;
+    fa->atime = st->ia_atime;
+    fa->mtime = st->ia_mtime;
+    fa->ctime = st->ia_ctime;
+    fa->atimensec = st->ia_atime_nsec;
+    fa->mtimensec = st->ia_mtime_nsec;
+    fa->ctimensec = st->ia_ctime_nsec;
+    fa->mode = st_mode_from_ia(st->ia_prot, st->ia_type);
+    fa->nlink = st->ia_nlink;
+    fa->uid = st->ia_uid;
+    fa->gid = st->ia_gid;
+    fa->rdev = makedev(ia_major(st->ia_rdev), ia_minor(st->ia_rdev));
 #if FUSE_KERNEL_MINOR_VERSION >= 9
-        fa->blksize     = st->ia_blksize;
+    fa->blksize = st->ia_blksize;
 #endif
 #ifdef GF_DARWIN_HOST_OS
-        fa->crtime      = (uint64_t)-1;
-        fa->crtimensec  = (uint32_t)-1;
-        fa->flags       = 0;
+    fa->crtime = (uint64_t)-1;
+    fa->crtimensec = (uint32_t)-1;
+    fa->flags = 0;
 #endif
 }
 
 void
-gf_fuse_fill_dirent (gf_dirent_t *entry, struct fuse_dirent *fde, gf_boolean_t enable_ino32)
+gf_fuse_fill_dirent(gf_dirent_t *entry, struct fuse_dirent *fde,
+                    gf_boolean_t enable_ino32)
 {
-        if (enable_ino32)
-                fde->ino = GF_FUSE_SQUASH_INO(entry->d_ino);
-        else
-                fde->ino = entry->d_ino;
-
-        fde->off         = entry->d_off;
-        fde->type        = entry->d_type;
-        fde->namelen     = strlen (entry->d_name);
-        strncpy (fde->name, entry->d_name, fde->namelen);
+    if (enable_ino32)
+        fde->ino = GF_FUSE_SQUASH_INO(entry->d_ino);
+    else
+        fde->ino = entry->d_ino;
+
+    fde->off = entry->d_off;
+    fde->type = entry->d_type;
+    fde->namelen = strlen(entry->d_name);
+    (void)memcpy(fde->name, entry->d_name, fde->namelen);
 }
 
 static int
-fuse_do_flip_xattr_ns (char *okey, const char *nns, char **nkey)
+fuse_do_flip_xattr_ns(char *okey, const char *nns, char **nkey)
 {
-        int   ret = 0;
-        char *key = NULL;
+    int ret = 0;
+    char *key = NULL;
 
-        okey = strchr (okey, '.');
-        GF_ASSERT (okey);
+    okey = strchr(okey, '.');
+    GF_ASSERT(okey);
 
-        key = GF_CALLOC (1, strlen (nns) + strlen(okey) + 1,
-                         gf_common_mt_char);
-        if (!key) {
-                ret = -1;
-                goto out;
-        }
+    int key_len = strlen(nns) + strlen(okey);
+    key = GF_MALLOC(key_len + 1, gf_common_mt_char);
+    if (!key) {
+        ret = -1;
+        goto out;
+    }
 
-        strcpy (key, nns);
-        strcat (key, okey);
+    strcpy(key, nns);
+    strcat(key, okey);
 
-        *nkey = key;
+    *nkey = key;
 
- out:
-        return ret;
+out:
+    return ret;
 }
 
 static int
-fuse_xattr_alloc_default (char *okey, char **nkey)
+fuse_xattr_alloc_default(char *okey, char **nkey)
 {
-        int ret = 0;
+    int ret = 0;
 
-        *nkey = gf_strdup (okey);
-        if (!*nkey)
-                ret = -1;
-        return ret;
+    *nkey = gf_strdup(okey);
+    if (!*nkey)
+        ret = -1;
+    return ret;
 }
 
-#define PRIV_XA_NS   "trusted"
+#define PRIV_XA_NS "trusted"
 #define UNPRIV_XA_NS "system"
 
 int
-fuse_flip_xattr_ns (fuse_private_t *priv, char *okey, char **nkey)
+fuse_flip_xattr_ns(fuse_private_t *priv, char *okey, char **nkey)
 {
-        int             ret       = 0;
-        gf_boolean_t    need_flip = _gf_false;
-
-        switch (priv->client_pid) {
-        case GF_CLIENT_PID_GSYNCD:
-                /* valid xattr(s): *xtime, volume-mark* */
-                gf_log("glusterfs-fuse", GF_LOG_DEBUG, "PID: %d, checking xattr(s): "
-                       "volume-mark*, *xtime", priv->client_pid);
-                if ( (strcmp (okey, UNPRIV_XA_NS".glusterfs.volume-mark") == 0)
-                     || (fnmatch (UNPRIV_XA_NS".glusterfs.volume-mark.*", okey, FNM_PERIOD) == 0)
-                     || (fnmatch (UNPRIV_XA_NS".glusterfs.*.xtime", okey, FNM_PERIOD) == 0) )
-                        need_flip = _gf_true;
-                break;
+    int ret = 0;
+    gf_boolean_t need_flip = _gf_false;
+
+    if (GF_CLIENT_PID_GSYNCD == priv->client_pid) {
+        /* valid xattr(s): *xtime, volume-mark* */
+        gf_log("glusterfs-fuse", GF_LOG_DEBUG,
+               "PID: %d, checking xattr(s): "
+               "volume-mark*, *xtime",
+               priv->client_pid);
+        if ((strcmp(okey, UNPRIV_XA_NS ".glusterfs.volume-mark") == 0) ||
+            (fnmatch(UNPRIV_XA_NS ".glusterfs.volume-mark.*", okey,
+                     FNM_PERIOD) == 0) ||
+            (fnmatch(UNPRIV_XA_NS ".glusterfs.*.xtime", okey, FNM_PERIOD) == 0))
+            need_flip = _gf_true;
+    }
+
+    if (need_flip) {
+        gf_log("glusterfs-fuse", GF_LOG_DEBUG,
+               "flipping %s to " PRIV_XA_NS " equivalent", okey);
+        ret = fuse_do_flip_xattr_ns(okey, PRIV_XA_NS, nkey);
+    } else {
+        /* if we cannot match, continue with what we got */
+        ret = fuse_xattr_alloc_default(okey, nkey);
+    }
+
+    return ret;
+}
 
-        case GF_CLIENT_PID_HADOOP:
-                /* valid xattr(s): pathinfo */
-                gf_log("glusterfs-fuse", GF_LOG_DEBUG, "PID: %d, checking xattr(s): "
-                       "pathinfo", priv->client_pid);
-                if (strcmp (okey, UNPRIV_XA_NS".glusterfs.pathinfo") == 0)
-                        need_flip = _gf_true;
-                break;
-        }
+int
+fuse_ignore_xattr_set(fuse_private_t *priv, char *key)
+{
+    int ret = 0;
 
-        if (need_flip) {
-                gf_log ("glusterfs-fuse", GF_LOG_DEBUG, "flipping %s to "PRIV_XA_NS" equivalent",
-                        okey);
-                ret = fuse_do_flip_xattr_ns (okey, PRIV_XA_NS, nkey);
-        } else {
-                /* if we cannot match, continue with what we got */
-                ret = fuse_xattr_alloc_default (okey, nkey);
-        }
+    /* don't mess with user namespace */
+    if (fnmatch("user.*", key, FNM_PERIOD) == 0)
+        goto out;
 
-        return ret;
+    if (priv->client_pid != GF_CLIENT_PID_GSYNCD)
+        goto out;
+
+    /* trusted NS check */
+    if (!((fnmatch("*.glusterfs.*.xtime", key, FNM_PERIOD) == 0) ||
+          (fnmatch("*.glusterfs.volume-mark", key, FNM_PERIOD) == 0) ||
+          (fnmatch("*.glusterfs.volume-mark.*", key, FNM_PERIOD) == 0) ||
+          (fnmatch("system.posix_acl_access", key, FNM_PERIOD) == 0) ||
+          (fnmatch("glusterfs.gfid.newfile", key, FNM_PERIOD) == 0) ||
+          (fnmatch("*.glusterfs.shard.block-size", key, FNM_PERIOD) == 0) ||
+          (fnmatch("*.glusterfs.shard.file-size", key, FNM_PERIOD) == 0)))
+        ret = -1;
+
+out:
+    gf_log("glusterfs-fuse", GF_LOG_DEBUG,
+           "%s setxattr: key [%s], "
+           " client pid [%d]",
+           (ret ? "disallowing" : "allowing"), key, priv->client_pid);
+
+    return ret;
 }
 
 int
-fuse_ignore_xattr_set (fuse_private_t *priv, char *key)
+fuse_check_selinux_cap_xattr(fuse_private_t *priv, char *name)
 {
-        int ret = 0;
+    int ret = -1;
 
-        /* don't mess with user namespace */
-        if (fnmatch ("user.*", key, FNM_PERIOD) == 0)
-                goto out;
+    if (strcmp(name, "security.selinux") &&
+        strcmp(name, "security.capability")) {
+        /* if xattr name is not of interest, no validations needed */
+        ret = 0;
+        goto out;
+    }
 
-        if (priv->client_pid != GF_CLIENT_PID_GSYNCD)
-                goto out;
+    if ((strcmp(name, "security.selinux") == 0) && (priv->selinux)) {
+        ret = 0;
+    }
 
-        /* trusted NS check */
-        if (!((fnmatch ("*.glusterfs.*.xtime", key, FNM_PERIOD) == 0)
-              || (fnmatch ("*.glusterfs.volume-mark",
-                           key, FNM_PERIOD) == 0)
-              || (fnmatch ("*.glusterfs.volume-mark.*",
-                           key, FNM_PERIOD) == 0)
-              || (fnmatch ("system.posix_acl_access",
-                           key, FNM_PERIOD) == 0)
-              || (fnmatch ("glusterfs.gfid.newfile",
-                           key, FNM_PERIOD) == 0)))
-                ret = -1;
-
- out:
-        gf_log ("glusterfs-fuse", GF_LOG_DEBUG, "%s setxattr: key [%s], "
-                " client pid [%d]", (ret ? "disallowing" : "allowing"), key,
-                priv->client_pid);
-
-        return ret;
+    if ((strcmp(name, "security.capability") == 0) &&
+        ((priv->capability) || (priv->selinux))) {
+        ret = 0;
+    }
+
+out:
+    return ret;
 }
diff --git a/xlators/mount/fuse/src/fuse-mem-types.h b/xlators/mount/fuse/src/fuse-mem-types.h
index 28b4dfbdd04..4fd8e58c523 100644
--- a/xlators/mount/fuse/src/fuse-mem-types.h
+++ b/xlators/mount/fuse/src/fuse-mem-types.h
@@ -11,18 +11,21 @@
 #ifndef __FUSE_MEM_TYPES_H__
 #define __FUSE_MEM_TYPES_H__
 
-#include "mem-types.h"
+#include <glusterfs/mem-types.h>
 
 enum gf_fuse_mem_types_ {
-        gf_fuse_mt_iovec  = gf_common_mt_end + 1,
-        gf_fuse_mt_fuse_private_t,
-        gf_fuse_mt_char,
-        gf_fuse_mt_iov_base,
-        gf_fuse_mt_fuse_state_t,
-        gf_fuse_mt_fd_ctx_t,
-        gf_fuse_mt_graph_switch_args_t,
-	gf_fuse_mt_gids_t,
-        gf_fuse_mt_end
+    gf_fuse_mt_iovec = gf_common_mt_end + 1,
+    gf_fuse_mt_fuse_private_t,
+    gf_fuse_mt_char,
+    gf_fuse_mt_iov_base,
+    gf_fuse_mt_fuse_state_t,
+    gf_fuse_mt_fd_ctx_t,
+    gf_fuse_mt_graph_switch_args_t,
+    gf_fuse_mt_gids_t,
+    gf_fuse_mt_invalidate_node_t,
+    gf_fuse_mt_pthread_t,
+    gf_fuse_mt_timed_message_t,
+    gf_fuse_mt_interrupt_record_t,
+    gf_fuse_mt_end
 };
 #endif
-
diff --git a/xlators/mount/fuse/src/fuse-resolve.c b/xlators/mount/fuse/src/fuse-resolve.c
index 2ddb31cd076..6206fd47325 100644
--- a/xlators/mount/fuse/src/fuse-resolve.c
+++ b/xlators/mount/fuse/src/fuse-resolve.c
@@ -7,206 +7,208 @@
    later), or the GNU General Public License, version 2 (GPLv2), in all
    cases as published by the Free Software Foundation.
 */
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
-
 #include "fuse-bridge.h"
 
 static int
-fuse_resolve_all (fuse_state_t *state);
+fuse_resolve_all(fuse_state_t *state);
 
-int fuse_resolve_continue (fuse_state_t *state);
-int fuse_resolve_entry_simple (fuse_state_t *state);
-int fuse_resolve_inode_simple (fuse_state_t *state);
-int fuse_migrate_fd (xlator_t *this, fd_t *fd, xlator_t *old_subvol,
-                     xlator_t *new_subvol);
+int
+fuse_resolve_continue(fuse_state_t *state);
+int
+fuse_resolve_entry_simple(fuse_state_t *state);
+int
+fuse_resolve_inode_simple(fuse_state_t *state);
+int
+fuse_migrate_fd(xlator_t *this, fd_t *fd, xlator_t *old_subvol,
+                xlator_t *new_subvol);
 
 fuse_fd_ctx_t *
-fuse_fd_ctx_get (xlator_t *this, fd_t *fd);
-
-gf_boolean_t fuse_inode_needs_lookup (inode_t *inode, xlator_t *this);
+fuse_fd_ctx_get(xlator_t *this, fd_t *fd);
 
 static int
-fuse_resolve_loc_touchup (fuse_state_t *state)
+fuse_resolve_loc_touchup(fuse_state_t *state)
 {
-        fuse_resolve_t *resolve = NULL;
-        loc_t          *loc     = NULL;
+    fuse_resolve_t *resolve = NULL;
+    loc_t *loc = NULL;
 
-        resolve = state->resolve_now;
-        loc     = state->loc_now;
+    resolve = state->resolve_now;
+    loc = state->loc_now;
 
-        loc_touchup (loc, resolve->bname);
-        return 0;
+    loc_touchup(loc, resolve->bname);
+    return 0;
 }
 
-
 int
-fuse_resolve_entry_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-			int op_ret, int op_errno, inode_t *inode,
-			struct iatt *buf, dict_t *xattr,
-			struct iatt *postparent)
+fuse_resolve_entry_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                       int op_ret, int op_errno, inode_t *inode,
+                       struct iatt *buf, dict_t *xattr, struct iatt *postparent)
 {
-        fuse_state_t   *state      = NULL;
-        fuse_resolve_t *resolve    = NULL;
-        inode_t        *link_inode = NULL;
-        loc_t          *resolve_loc   = NULL;
-
-        state = frame->root->state;
-        resolve = state->resolve_now;
-        resolve_loc = &resolve->resolve_loc;
-
-        STACK_DESTROY (frame->root);
-
-        if (op_ret == -1) {
-                gf_log (this->name, (op_errno == ENOENT)
-                        ? GF_LOG_DEBUG : GF_LOG_WARNING,
-                        "%s/%s: failed to resolve (%s)",
-                        uuid_utoa (resolve_loc->pargfid), resolve_loc->name,
-                        strerror (op_errno));
-                resolve->op_ret = -1;
-                resolve->op_errno = op_errno;
-                goto out;
-        }
-
-        link_inode = inode_link (inode, resolve_loc->parent,
-                                 resolve_loc->name, buf);
-
-	state->loc_now->inode = link_inode;
+    fuse_state_t *state = NULL;
+    fuse_resolve_t *resolve = NULL;
+    inode_t *link_inode = NULL;
+    loc_t *resolve_loc = NULL;
+    uint64_t ctx_value = LOOKUP_NOT_NEEDED;
+
+    state = frame->root->state;
+    resolve = state->resolve_now;
+    resolve_loc = &resolve->resolve_loc;
+
+    STACK_DESTROY(frame->root);
+
+    if (op_ret == -1) {
+        gf_log(this->name, (op_errno == ENOENT) ? GF_LOG_DEBUG : GF_LOG_WARNING,
+               "%s/%s: failed to resolve (%s)", uuid_utoa(resolve_loc->pargfid),
+               resolve_loc->name, strerror(op_errno));
+        resolve->op_ret = -1;
+        resolve->op_errno = op_errno;
+        goto out;
+    }
+
+    link_inode = inode_link(inode, resolve_loc->parent, resolve_loc->name, buf);
+    if (link_inode == inode)
+        inode_ctx_set(link_inode, this, &ctx_value);
+    state->loc_now->inode = link_inode;
 
 out:
-        loc_wipe (resolve_loc);
+    loc_wipe(resolve_loc);
 
-        fuse_resolve_continue (state);
-        return 0;
+    fuse_resolve_continue(state);
+    return 0;
 }
 
-
 int
-fuse_resolve_entry (fuse_state_t *state)
+fuse_resolve_entry(fuse_state_t *state)
 {
-	fuse_resolve_t   *resolve = NULL;
-	loc_t            *resolve_loc = NULL;
+    fuse_resolve_t *resolve = NULL;
+    loc_t *resolve_loc = NULL;
 
-	resolve = state->resolve_now;
-	resolve_loc = &resolve->resolve_loc;
+    resolve = state->resolve_now;
+    resolve_loc = &resolve->resolve_loc;
 
-	resolve_loc->parent = inode_ref (state->loc_now->parent);
-	gf_uuid_copy (resolve_loc->pargfid, state->loc_now->pargfid);
-        resolve_loc->name = resolve->bname;
-        resolve_loc->inode = inode_new (state->itable);
+    resolve_loc->parent = inode_ref(state->loc_now->parent);
+    gf_uuid_copy(resolve_loc->pargfid, state->loc_now->pargfid);
+    resolve_loc->name = resolve->bname;
 
-        inode_path (resolve_loc->parent, resolve_loc->name,
-                    (char **) &resolve_loc->path);
+    resolve_loc->inode = inode_grep(state->itable, resolve->parhint,
+                                    resolve->bname);
+    if (!resolve_loc->inode) {
+        resolve_loc->inode = inode_new(state->itable);
+    }
+    inode_path(resolve_loc->parent, resolve_loc->name,
+               (char **)&resolve_loc->path);
 
-        FUSE_FOP (state, fuse_resolve_entry_cbk, GF_FOP_LOOKUP,
-                  lookup, resolve_loc, NULL);
+    FUSE_FOP(state, fuse_resolve_entry_cbk, GF_FOP_LOOKUP, lookup, resolve_loc,
+             NULL);
 
-	return 0;
+    return 0;
 }
 
-
 int
-fuse_resolve_gfid_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                       int op_ret, int op_errno, inode_t *inode,
-                       struct iatt *buf, dict_t *xattr, struct iatt *postparent)
+fuse_resolve_gfid_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                      int op_ret, int op_errno, inode_t *inode,
+                      struct iatt *buf, dict_t *xattr, struct iatt *postparent)
 {
-        fuse_state_t   *state      = NULL;
-        fuse_resolve_t *resolve    = NULL;
-        inode_t        *link_inode = NULL;
-        loc_t          *loc_now   = NULL;
-
-        state = frame->root->state;
-        resolve = state->resolve_now;
-	loc_now = state->loc_now;
-
-        STACK_DESTROY (frame->root);
-
-        if (op_ret == -1) {
-                gf_log (this->name, (op_errno == ENOENT)
-                        ? GF_LOG_DEBUG : GF_LOG_WARNING,
-                        "%s: failed to resolve (%s)",
-                        uuid_utoa (resolve->resolve_loc.gfid),
-			strerror (op_errno));
-                loc_wipe (&resolve->resolve_loc);
-
-                /* resolve->op_ret can have 3 values: 0, -1, -2.
-                 * 0 : resolution was successful.
-                 * -1: parent inode could not be resolved.
-                 * -2: entry (inode corresponding to path) could not be resolved
-                 */
-
-                if (gf_uuid_is_null (resolve->gfid)) {
-                        resolve->op_ret = -1;
-                } else {
-                        resolve->op_ret = -2;
-                }
-
-                resolve->op_errno = op_errno;
-                goto out;
+    fuse_state_t *state = NULL;
+    fuse_resolve_t *resolve = NULL;
+    inode_t *link_inode = NULL;
+    loc_t *loc_now = NULL;
+    inode_t *tmp_inode = NULL;
+    uint64_t ctx_value = LOOKUP_NOT_NEEDED;
+
+    state = frame->root->state;
+    resolve = state->resolve_now;
+    loc_now = state->loc_now;
+
+    STACK_DESTROY(frame->root);
+
+    if (op_ret == -1) {
+        gf_log(this->name, (op_errno == ENOENT) ? GF_LOG_DEBUG : GF_LOG_WARNING,
+               "%s: failed to resolve (%s)",
+               uuid_utoa(resolve->resolve_loc.gfid), strerror(op_errno));
+        loc_wipe(&resolve->resolve_loc);
+
+        /* resolve->op_ret can have 3 values: 0, -1, -2.
+         * 0 : resolution was successful.
+         * -1: parent inode could not be resolved.
+         * -2: entry (inode corresponding to path) could not be resolved
+         */
+
+        if (gf_uuid_is_null(resolve->gfid)) {
+            resolve->op_ret = -1;
+        } else {
+            resolve->op_ret = -2;
         }
 
-        link_inode = inode_link (inode, NULL, NULL, buf);
+        resolve->op_errno = op_errno;
+        goto out;
+    }
 
-        loc_wipe (&resolve->resolve_loc);
+    link_inode = inode_link(inode, NULL, NULL, buf);
+    if (link_inode == inode)
+        inode_ctx_set(link_inode, this, &ctx_value);
 
-        if (!link_inode)
-                goto out;
+    loc_wipe(&resolve->resolve_loc);
 
-	if (!gf_uuid_is_null (resolve->gfid)) {
-		loc_now->inode = link_inode;
-		goto out;
-	}
+    if (!link_inode)
+        goto out;
 
-	loc_now->parent = link_inode;
-        gf_uuid_copy (loc_now->pargfid, link_inode->gfid);
+    if (!gf_uuid_is_null(resolve->gfid)) {
+        loc_now->inode = link_inode;
+        goto out;
+    }
 
-	fuse_resolve_entry (state);
+    loc_now->parent = link_inode;
+    gf_uuid_copy(loc_now->pargfid, link_inode->gfid);
 
-        return 0;
+    tmp_inode = inode_grep(state->itable, link_inode, resolve->bname);
+    if (tmp_inode && (!inode_needs_lookup(tmp_inode, THIS))) {
+        loc_now->inode = tmp_inode;
+        goto out;
+    }
+
+    inode_unref(tmp_inode);
+    fuse_resolve_entry(state);
+
+    return 0;
 out:
-        fuse_resolve_continue (state);
-        return 0;
+    fuse_resolve_continue(state);
+    return 0;
 }
 
-
 int
-fuse_resolve_gfid (fuse_state_t *state)
+fuse_resolve_gfid(fuse_state_t *state)
 {
-        fuse_resolve_t *resolve  = NULL;
-        loc_t          *resolve_loc = NULL;
-        int             ret      = 0;
-
-        resolve = state->resolve_now;
-        resolve_loc = &resolve->resolve_loc;
-
-        if (!gf_uuid_is_null (resolve->pargfid)) {
-                gf_uuid_copy (resolve_loc->gfid, resolve->pargfid);
-        } else if (!gf_uuid_is_null (resolve->gfid)) {
-                gf_uuid_copy (resolve_loc->gfid, resolve->gfid);
-        }
-
-	/* inode may already exist in case we are looking up an inode which was
-	   linked through readdirplus */
-	resolve_loc->inode = inode_find (state->itable, resolve_loc->gfid);
-	if (!resolve_loc->inode)
-		resolve_loc->inode = inode_new (state->itable);
-	ret = loc_path (resolve_loc, NULL);
-
-        if (ret <= 0) {
-                gf_log (THIS->name, GF_LOG_WARNING,
-                        "failed to get the path for inode %s",
-                        uuid_utoa (resolve->gfid));
-        }
-
-        FUSE_FOP (state, fuse_resolve_gfid_cbk, GF_FOP_LOOKUP,
-                  lookup, resolve_loc, NULL);
-
-        return 0;
+    fuse_resolve_t *resolve = NULL;
+    loc_t *resolve_loc = NULL;
+    int ret = 0;
+
+    resolve = state->resolve_now;
+    resolve_loc = &resolve->resolve_loc;
+
+    if (!gf_uuid_is_null(resolve->pargfid)) {
+        gf_uuid_copy(resolve_loc->gfid, resolve->pargfid);
+    } else if (!gf_uuid_is_null(resolve->gfid)) {
+        gf_uuid_copy(resolve_loc->gfid, resolve->gfid);
+    }
+
+    /* inode may already exist in case we are looking up an inode which was
+       linked through readdirplus */
+    resolve_loc->inode = inode_find(state->itable, resolve_loc->gfid);
+    if (!resolve_loc->inode)
+        resolve_loc->inode = inode_new(state->itable);
+    ret = loc_path(resolve_loc, NULL);
+
+    if (ret <= 0) {
+        gf_log(THIS->name, GF_LOG_WARNING,
+               "failed to get the path for inode %s", uuid_utoa(resolve->gfid));
+    }
+
+    FUSE_FOP(state, fuse_resolve_gfid_cbk, GF_FOP_LOOKUP, lookup, resolve_loc,
+             NULL);
+
+    return 0;
 }
 
-
 /*
  * Return value:
  * 0 - resolved parent and entry (as necessary)
@@ -215,434 +217,417 @@ fuse_resolve_gfid (fuse_state_t *state)
  */
 
 int
-fuse_resolve_parent_simple (fuse_state_t *state)
+fuse_resolve_parent_simple(fuse_state_t *state)
 {
-        fuse_resolve_t *resolve   = NULL;
-	loc_t          *loc       = NULL;
-        inode_t        *parent    = NULL;
-        inode_t        *inode     = NULL;
-
-        resolve = state->resolve_now;
-	loc = state->loc_now;
-
-	loc->name = resolve->bname;
-
-	parent = resolve->parhint;
-	if (parent->table == state->itable) {
-		if (fuse_inode_needs_lookup (parent, THIS))
-			return 1;
-
-		/* no graph switches since */
-		loc->parent = inode_ref (parent);
-		gf_uuid_copy (loc->pargfid, parent->gfid);
-		loc->inode = inode_grep (state->itable, parent, loc->name);
-
-                /* nodeid for root is 1 and we blindly take the latest graph's
-                 * table->root as the parhint and because of this there is
-                 * ambiguity whether the entry should have existed or not, and
-                 * we took the conservative approach of assuming entry should
-                 * have been there even though it need not have (bug #804592).
-                 */
-                if ((loc->inode == NULL)
-                    && __is_root_gfid (parent->gfid)) {
-                        /* non decisive result - entry missing */
-                        return -1;
-                }
-
-		/* decisive result - resolution success */
-		return 0;
-	}
-
-        parent = inode_find (state->itable, resolve->pargfid);
-	if (!parent) {
-		/* non decisive result - parent missing */
-		return 1;
-	}
-	if (fuse_inode_needs_lookup (parent, THIS)) {
-		inode_unref (parent);
-		return 1;
-	}
-
-	loc->parent = parent;
-        gf_uuid_copy (loc->pargfid, resolve->pargfid);
-
-	inode = inode_grep (state->itable, parent, loc->name);
-	if (inode) {
-		loc->inode = inode;
-		/* decisive result - resolution success */
-		return 0;
-	}
-
-	/* non decisive result - entry missing */
-        return -1;
-}
-
-
-int
-fuse_resolve_parent (fuse_state_t *state)
-{
-        int    ret = 0;
-
-        ret = fuse_resolve_parent_simple (state);
-        if (ret > 0) {
-                fuse_resolve_gfid (state);
-                return 0;
+    fuse_resolve_t *resolve = NULL;
+    loc_t *loc = NULL;
+    inode_t *parent = NULL;
+    inode_t *inode = NULL;
+    xlator_t *this = NULL;
+
+    resolve = state->resolve_now;
+    loc = state->loc_now;
+    this = state->this;
+
+    loc->name = resolve->bname;
+
+    parent = resolve->parhint;
+    if (parent->table == state->itable) {
+        if (inode_needs_lookup(parent, THIS))
+            return 1;
+
+        /* no graph switches since */
+        loc->parent = inode_ref(parent);
+        gf_uuid_copy(loc->pargfid, parent->gfid);
+        loc->inode = inode_grep(state->itable, parent, loc->name);
+
+        /* nodeid for root is 1 and we blindly take the latest graph's
+         * table->root as the parhint and because of this there is
+         * ambiguity whether the entry should have existed or not, and
+         * we took the conservative approach of assuming entry should
+         * have been there even though it need not have (bug #804592).
+         */
+
+        if (loc->inode && inode_needs_lookup(loc->inode, THIS)) {
+            inode_unref(loc->inode);
+            loc->inode = NULL;
+            return -1;
         }
 
-	if (ret < 0) {
-		fuse_resolve_entry (state);
-		return 0;
-	}
-
-        fuse_resolve_continue (state);
+        if ((loc->inode == NULL) && __is_root_gfid(parent->gfid)) {
+            /* non decisive result - entry missing */
+            return -1;
+        }
 
+        /* decisive result - resolution success */
         return 0;
-}
+    }
 
+    parent = inode_find(state->itable, resolve->pargfid);
+    if (!parent) {
+        /* non decisive result - parent missing */
+        return 1;
+    }
+    if (inode_needs_lookup(parent, THIS)) {
+        inode_unref(parent);
+        return 1;
+    }
 
-int
-fuse_resolve_inode_simple (fuse_state_t *state)
-{
-        fuse_resolve_t *resolve   = NULL;
-	loc_t          *loc = NULL;
-        inode_t        *inode     = NULL;
-
-        resolve = state->resolve_now;
-	loc = state->loc_now;
-
-	inode = resolve->hint;
-	if (inode->table == state->itable)
-		inode_ref (inode);
-	else
-		inode = inode_find (state->itable, resolve->gfid);
-
-        if (inode) {
-		if (!fuse_inode_needs_lookup (inode, THIS))
-			goto found;
-		/* inode was linked through readdirplus */
-		inode_unref (inode);
-	}
+    loc->parent = parent;
+    gf_uuid_copy(loc->pargfid, resolve->pargfid);
 
-        return 1;
-found:
-	loc->inode = inode;
-	return 0;
-}
+    inode = inode_grep(state->itable, parent, loc->name);
+    if (inode && !inode_needs_lookup(inode, this)) {
+        loc->inode = inode;
+        /* decisive result - resolution success */
+        return 0;
+    }
 
+    /* non decisive result - entry missing */
+    return -1;
+}
 
 int
-fuse_resolve_inode (fuse_state_t *state)
+fuse_resolve_parent(fuse_state_t *state)
 {
-        int                 ret = 0;
+    int ret = 0;
 
-        ret = fuse_resolve_inode_simple (state);
+    ret = fuse_resolve_parent_simple(state);
+    if (ret > 0) {
+        fuse_resolve_gfid(state);
+        return 0;
+    }
 
-        if (ret > 0) {
-                fuse_resolve_gfid (state);
-                return 0;
-        }
+    if (ret < 0) {
+        fuse_resolve_entry(state);
+        return 0;
+    }
 
-        fuse_resolve_continue (state);
+    fuse_resolve_continue(state);
 
-        return 0;
+    return 0;
 }
 
-
 int
-fuse_migrate_fd_task (void *data)
+fuse_resolve_inode_simple(fuse_state_t *state)
 {
-        int            ret        = -1;
-        fuse_state_t  *state      = NULL;
-        fd_t          *basefd     = NULL, *oldfd = NULL;
-        fuse_fd_ctx_t *basefd_ctx = NULL;
-        xlator_t      *old_subvol = NULL;
-
-        state = data;
-        if (state == NULL) {
-                goto out;
-        }
+    fuse_resolve_t *resolve = NULL;
+    loc_t *loc = NULL;
+    inode_t *inode = NULL;
+
+    resolve = state->resolve_now;
+    loc = state->loc_now;
+
+    inode = resolve->hint;
+    if (inode->table == state->itable)
+        inode_ref(inode);
+    else
+        inode = inode_find(state->itable, resolve->gfid);
+
+    if (inode) {
+        if (!inode_needs_lookup(inode, THIS))
+            goto found;
+        /* inode was linked through readdirplus */
+        inode_unref(inode);
+    }
+
+    return 1;
+found:
+    loc->inode = inode;
+    return 0;
+}
 
-        basefd = state->fd;
+int
+fuse_resolve_inode(fuse_state_t *state)
+{
+    int ret = 0;
 
-        basefd_ctx = fuse_fd_ctx_get (state->this, basefd);
-        if (!basefd_ctx)
-                goto out;
+    ret = fuse_resolve_inode_simple(state);
 
-        LOCK (&basefd->lock);
-        {
-                oldfd = basefd_ctx->activefd ? basefd_ctx->activefd : basefd;
-                fd_ref (oldfd);
-        }
-        UNLOCK (&basefd->lock);
+    if (ret > 0) {
+        fuse_resolve_gfid(state);
+        return 0;
+    }
 
-        old_subvol = oldfd->inode->table->xl;
+    fuse_resolve_continue(state);
 
-        ret = fuse_migrate_fd (state->this, basefd, old_subvol,
-                               state->active_subvol);
+    return 0;
+}
 
-        LOCK (&basefd->lock);
-        {
-                if (ret < 0) {
-                        basefd_ctx->migration_failed = 1;
-                } else {
-                        basefd_ctx->migration_failed = 0;
-                }
+int
+fuse_migrate_fd_task(void *data)
+{
+    int ret = -1;
+    fuse_state_t *state = NULL;
+    fd_t *basefd = NULL, *oldfd = NULL;
+    fuse_fd_ctx_t *basefd_ctx = NULL;
+    xlator_t *old_subvol = NULL;
+
+    state = data;
+    if (state == NULL) {
+        goto out;
+    }
+
+    basefd = state->fd;
+
+    basefd_ctx = fuse_fd_ctx_get(state->this, basefd);
+    if (!basefd_ctx)
+        goto out;
+
+    LOCK(&basefd->lock);
+    {
+        oldfd = basefd_ctx->activefd ? basefd_ctx->activefd : basefd;
+        fd_ref(oldfd);
+    }
+    UNLOCK(&basefd->lock);
+
+    old_subvol = oldfd->inode->table->xl;
+
+    ret = fuse_migrate_fd(state->this, basefd, old_subvol,
+                          state->active_subvol);
+
+    LOCK(&basefd->lock);
+    {
+        if (ret < 0) {
+            basefd_ctx->migration_failed = 1;
+        } else {
+            basefd_ctx->migration_failed = 0;
         }
-        UNLOCK (&basefd->lock);
+    }
+    UNLOCK(&basefd->lock);
 
-        ret = 0;
+    ret = 0;
 
 out:
-        if (oldfd)
-                fd_unref (oldfd);
+    if (oldfd)
+        fd_unref(oldfd);
 
-        return ret;
+    return ret;
 }
 
-
-static inline int
-fuse_migrate_fd_error (xlator_t *this, fd_t *fd)
+static int
+fuse_migrate_fd_error(xlator_t *this, fd_t *fd)
 {
-        fuse_fd_ctx_t *fdctx = NULL;
-        char           error = 0;
-
-        fdctx = fuse_fd_ctx_get (this, fd);
-        if (fdctx != NULL) {
-                if (fdctx->migration_failed) {
-                        error = 1;
-                }
+    fuse_fd_ctx_t *fdctx = NULL;
+    char error = 0;
+
+    fdctx = fuse_fd_ctx_get(this, fd);
+    if (fdctx != NULL) {
+        if (fdctx->migration_failed) {
+            error = 1;
         }
+    }
 
-        return error;
+    return error;
 }
 
-#define FUSE_FD_GET_ACTIVE_FD(activefd, basefd)                 \
-        do {                                                    \
-                LOCK (&basefd->lock);                           \
-                {                                               \
-                        activefd = basefd_ctx->activefd ?       \
-                                basefd_ctx->activefd : basefd;  \
-                        if (activefd != basefd) {               \
-                                fd_ref (activefd);              \
-                        }                                       \
-                }                                               \
-                UNLOCK (&basefd->lock);                         \
-                                                                \
-                if (activefd == basefd) {                       \
-                        fd_ref (activefd);                      \
-                }                                               \
-        } while (0);
-
+#define FUSE_FD_GET_ACTIVE_FD(activefd, basefd)                                \
+    do {                                                                       \
+        LOCK(&basefd->lock);                                                   \
+        {                                                                      \
+            activefd = basefd_ctx->activefd ? basefd_ctx->activefd : basefd;   \
+            if (activefd != basefd) {                                          \
+                fd_ref(activefd);                                              \
+            }                                                                  \
+        }                                                                      \
+        UNLOCK(&basefd->lock);                                                 \
+                                                                               \
+        if (activefd == basefd) {                                              \
+            fd_ref(activefd);                                                  \
+        }                                                                      \
+    } while (0);
 
 static int
-fuse_resolve_fd (fuse_state_t *state)
+fuse_resolve_fd(fuse_state_t *state)
 {
-        fuse_resolve_t *resolve            = NULL;
-	fd_t           *basefd             = NULL, *activefd = NULL;
-	xlator_t       *active_subvol      = NULL, *this = NULL;
-        int             ret                = 0;
-        char            fd_migration_error = 0;
-        fuse_fd_ctx_t  *basefd_ctx         = NULL;
-
-        resolve = state->resolve_now;
-
-        this = state->this;
-
-        basefd = resolve->fd;
-        basefd_ctx = fuse_fd_ctx_get (this, basefd);
-        if (basefd_ctx == NULL) {
-                gf_log (state->this->name, GF_LOG_WARNING,
-                        "fdctx is NULL for basefd (ptr:%p inode-gfid:%s), "
-                        "resolver erroring out with errno EINVAL",
-                        basefd, uuid_utoa (basefd->inode->gfid));
-                resolve->op_ret = -1;
-                resolve->op_errno = EINVAL;
-                goto resolve_continue;
-        }
-
-        FUSE_FD_GET_ACTIVE_FD (activefd, basefd);
-
+    fuse_resolve_t *resolve = NULL;
+    fd_t *basefd = NULL, *activefd = NULL;
+    xlator_t *active_subvol = NULL, *this = NULL;
+    int ret = 0;
+    char fd_migration_error = 0;
+    fuse_fd_ctx_t *basefd_ctx = NULL;
+
+    resolve = state->resolve_now;
+
+    this = state->this;
+
+    basefd = resolve->fd;
+    basefd_ctx = fuse_fd_ctx_get(this, basefd);
+    if (basefd_ctx == NULL) {
+        gf_log(state->this->name, GF_LOG_WARNING,
+               "fdctx is NULL for basefd (ptr:%p inode-gfid:%s), "
+               "resolver erroring out with errno EINVAL",
+               basefd, uuid_utoa(basefd->inode->gfid));
+        resolve->op_ret = -1;
+        resolve->op_errno = EINVAL;
+        goto resolve_continue;
+    }
+
+    FUSE_FD_GET_ACTIVE_FD(activefd, basefd);
+
+    active_subvol = activefd->inode->table->xl;
+
+    fd_migration_error = fuse_migrate_fd_error(state->this, basefd);
+    if (fd_migration_error) {
+        resolve->op_ret = -1;
+        resolve->op_errno = EBADF;
+    } else if (state->active_subvol != active_subvol) {
+        ret = synctask_new(state->this->ctx->env, fuse_migrate_fd_task, NULL,
+                           NULL, state);
+
+        fd_migration_error = fuse_migrate_fd_error(state->this, basefd);
+        fd_unref(activefd);
+
+        FUSE_FD_GET_ACTIVE_FD(activefd, basefd);
         active_subvol = activefd->inode->table->xl;
 
-        fd_migration_error = fuse_migrate_fd_error (state->this, basefd);
-        if (fd_migration_error) {
-                resolve->op_ret = -1;
-                resolve->op_errno = EBADF;
-        } else if (state->active_subvol != active_subvol) {
-                ret = synctask_new (state->this->ctx->env, fuse_migrate_fd_task,
-                                    NULL, NULL, state);
-
-                fd_migration_error = fuse_migrate_fd_error (state->this,
-                                                            basefd);
-                fd_unref (activefd);
-
-                FUSE_FD_GET_ACTIVE_FD (activefd, basefd);
-                active_subvol = activefd->inode->table->xl;
-
-                if ((ret == -1) || fd_migration_error
-                    || (state->active_subvol != active_subvol)) {
-                        if (ret == -1) {
-                                gf_log (state->this->name, GF_LOG_WARNING,
-                                        "starting sync-task to migrate "
-                                        "basefd (ptr:%p inode-gfid:%s) failed "
-                                        "(old-subvolume:%s-%d "
-                                        "new-subvolume:%s-%d)",
-                                        basefd,
-                                        uuid_utoa (basefd->inode->gfid),
-                                        active_subvol->name,
-                                        active_subvol->graph->id,
-                                        state->active_subvol->name,
-                                        state->active_subvol->graph->id);
-                        } else {
-                                gf_log (state->this->name, GF_LOG_WARNING,
-                                        "fd migration of basefd "
-                                        "(ptr:%p inode-gfid:%s) failed "
-                                        "(old-subvolume:%s-%d "
-                                        "new-subvolume:%s-%d)",
-                                        basefd,
-                                        uuid_utoa (basefd->inode->gfid),
-                                        active_subvol->name,
-                                        active_subvol->graph->id,
-                                        state->active_subvol->name,
-                                        state->active_subvol->graph->id);
-                        }
-
-                        resolve->op_ret = -1;
-                        resolve->op_errno = EBADF;
-                } else {
-                        gf_log (state->this->name, GF_LOG_DEBUG,
-                                "basefd (ptr:%p inode-gfid:%s) migrated "
-                                "successfully in resolver "
-                                "(old-subvolume:%s-%d new-subvolume:%s-%d)",
-                                basefd, uuid_utoa (basefd->inode->gfid),
-                                active_subvol->name, active_subvol->graph->id,
-                                state->active_subvol->name,
-                                state->active_subvol->graph->id);
-                }
+        if ((ret == -1) || fd_migration_error ||
+            (state->active_subvol != active_subvol)) {
+            if (ret == -1) {
+                gf_log(state->this->name, GF_LOG_WARNING,
+                       "starting sync-task to migrate "
+                       "basefd (ptr:%p inode-gfid:%s) failed "
+                       "(old-subvolume:%s-%d "
+                       "new-subvolume:%s-%d)",
+                       basefd, uuid_utoa(basefd->inode->gfid),
+                       active_subvol->name, active_subvol->graph->id,
+                       state->active_subvol->name,
+                       state->active_subvol->graph->id);
+            } else {
+                gf_log(state->this->name, GF_LOG_WARNING,
+                       "fd migration of basefd "
+                       "(ptr:%p inode-gfid:%s) failed "
+                       "(old-subvolume:%s-%d "
+                       "new-subvolume:%s-%d)",
+                       basefd, uuid_utoa(basefd->inode->gfid),
+                       active_subvol->name, active_subvol->graph->id,
+                       state->active_subvol->name,
+                       state->active_subvol->graph->id);
+            }
+
+            resolve->op_ret = -1;
+            resolve->op_errno = EBADF;
+        } else {
+            gf_log(state->this->name, GF_LOG_DEBUG,
+                   "basefd (ptr:%p inode-gfid:%s) migrated "
+                   "successfully in resolver "
+                   "(old-subvolume:%s-%d new-subvolume:%s-%d)",
+                   basefd, uuid_utoa(basefd->inode->gfid), active_subvol->name,
+                   active_subvol->graph->id, state->active_subvol->name,
+                   state->active_subvol->graph->id);
         }
+    }
 
-        if ((resolve->op_ret == -1) && (resolve->op_errno == EBADF)) {
-                gf_log ("fuse-resolve", GF_LOG_WARNING,
-                        "migration of basefd (ptr:%p inode-gfid:%s) "
-                        "did not complete, failing fop with EBADF "
-                        "(old-subvolume:%s-%d new-subvolume:%s-%d)", basefd,
-                        uuid_utoa (basefd->inode->gfid),
-                        active_subvol->name, active_subvol->graph->id,
-                        state->active_subvol->name,
-                        state->active_subvol->graph->id);
-        }
+    if ((resolve->op_ret == -1) && (resolve->op_errno == EBADF)) {
+        gf_log("fuse-resolve", GF_LOG_WARNING,
+               "migration of basefd (ptr:%p inode-gfid:%s) "
+               "did not complete, failing fop with EBADF "
+               "(old-subvolume:%s-%d new-subvolume:%s-%d)",
+               basefd, uuid_utoa(basefd->inode->gfid), active_subvol->name,
+               active_subvol->graph->id, state->active_subvol->name,
+               state->active_subvol->graph->id);
+    }
 
-        if (activefd != basefd) {
-                state->fd = fd_ref (activefd);
-                fd_unref (basefd);
-        }
+    if (activefd != basefd) {
+        state->fd = fd_ref(activefd);
+        fd_unref(basefd);
+    }
 
-	/* state->active_subvol = active_subvol; */
+    /* state->active_subvol = active_subvol; */
 
 resolve_continue:
-        if (activefd != NULL) {
-                fd_unref (activefd);
-        }
+    if (activefd != NULL) {
+        fd_unref(activefd);
+    }
 
-        fuse_resolve_continue (state);
+    fuse_resolve_continue(state);
 
-        return 0;
+    return 0;
 }
 
-
 int
-fuse_gfid_set (fuse_state_t *state)
+fuse_gfid_set(fuse_state_t *state)
 {
-        int   ret = 0;
+    int ret = 0;
 
-        if (gf_uuid_is_null (state->gfid))
-                goto out;
+    if (gf_uuid_is_null(state->gfid))
+        goto out;
 
-        if (!state->xdata)
-                state->xdata = dict_new ();
+    if (!state->xdata)
+        state->xdata = dict_new();
 
-        if (!state->xdata) {
-                ret = -1;
-                goto out;
-        }
+    if (!state->xdata) {
+        ret = -1;
+        goto out;
+    }
 
-        ret = dict_set_static_bin (state->xdata, "gfid-req",
-                                   state->gfid, sizeof (state->gfid));
+    ret = dict_set_gfuuid(state->xdata, "gfid-req", state->gfid, true);
 out:
-        return ret;
+    return ret;
 }
 
-
 int
-fuse_resolve_entry_init (fuse_state_t *state, fuse_resolve_t *resolve,
-			 ino_t par, char *name)
+fuse_resolve_entry_init(fuse_state_t *state, fuse_resolve_t *resolve, ino_t par,
+                        char *name)
 {
-	inode_t       *parent = NULL;
+    inode_t *parent = NULL;
 
-	parent = fuse_ino_to_inode (par, state->this);
-	gf_uuid_copy (resolve->pargfid, parent->gfid);
-	resolve->parhint = parent;
-	resolve->bname = gf_strdup (name);
+    parent = fuse_ino_to_inode(par, state->this);
+    gf_uuid_copy(resolve->pargfid, parent->gfid);
+    resolve->parhint = parent;
+    resolve->bname = gf_strdup(name);
 
-	return 0;
+    return 0;
 }
 
-
 int
-fuse_resolve_inode_init (fuse_state_t *state, fuse_resolve_t *resolve,
-			 ino_t ino)
+fuse_resolve_inode_init(fuse_state_t *state, fuse_resolve_t *resolve, ino_t ino)
 {
-	inode_t       *inode = NULL;
+    inode_t *inode = NULL;
 
-	inode = fuse_ino_to_inode (ino, state->this);
-	gf_uuid_copy (resolve->gfid, inode->gfid);
-	resolve->hint = inode;
+    inode = fuse_ino_to_inode(ino, state->this);
+    gf_uuid_copy(resolve->gfid, inode->gfid);
+    resolve->hint = inode;
 
-	return 0;
+    return 0;
 }
 
-
 int
-fuse_resolve_fd_init (fuse_state_t *state, fuse_resolve_t *resolve,
-		      fd_t *fd)
+fuse_resolve_fd_init(fuse_state_t *state, fuse_resolve_t *resolve, fd_t *fd)
 {
-	resolve->fd = fd_ref (fd);
+    resolve->fd = fd_ref(fd);
 
-	return 0;
+    return 0;
 }
 
-
 static int
-fuse_resolve (fuse_state_t *state)
+fuse_resolve(fuse_state_t *state)
 {
-        fuse_resolve_t   *resolve = NULL;
-
-        resolve = state->resolve_now;
+    fuse_resolve_t *resolve = NULL;
 
-        if (resolve->fd) {
+    resolve = state->resolve_now;
 
-                fuse_resolve_fd (state);
+    if (resolve->fd) {
+        fuse_resolve_fd(state);
 
-        } else if (!gf_uuid_is_null (resolve->pargfid)) {
+    } else if (!gf_uuid_is_null(resolve->pargfid)) {
+        fuse_resolve_parent(state);
 
-                fuse_resolve_parent (state);
+    } else if (!gf_uuid_is_null(resolve->gfid)) {
+        fuse_resolve_inode(state);
 
-        } else if (!gf_uuid_is_null (resolve->gfid)) {
+    } else {
+        fuse_resolve_all(state);
+    }
 
-                fuse_resolve_inode (state);
-
-        } else {
-                fuse_resolve_all (state);
-        }
-
-        return 0;
+    return 0;
 }
 
 static int
-fuse_resolve_done (fuse_state_t *state)
+fuse_resolve_done(fuse_state_t *state)
 {
-        fuse_fop_resume (state);
-        return 0;
+    fuse_fop_resume(state);
+    return 0;
 }
 
 /*
@@ -650,53 +635,49 @@ fuse_resolve_done (fuse_state_t *state)
  * state->resolve_now is used to decide which location/fd is to be resolved now
  */
 static int
-fuse_resolve_all (fuse_state_t *state)
+fuse_resolve_all(fuse_state_t *state)
 {
-        if (state->resolve_now == NULL) {
-
-                state->resolve_now = &state->resolve;
-                state->loc_now     = &state->loc;
-
-                fuse_resolve (state);
-
-        } else if (state->resolve_now == &state->resolve) {
+    if (state->resolve_now == NULL) {
+        state->resolve_now = &state->resolve;
+        state->loc_now = &state->loc;
 
-                state->resolve_now = &state->resolve2;
-                state->loc_now     = &state->loc2;
+        fuse_resolve(state);
 
-                fuse_resolve (state);
+    } else if (state->resolve_now == &state->resolve) {
+        state->resolve_now = &state->resolve2;
+        state->loc_now = &state->loc2;
 
-        } else if (state->resolve_now == &state->resolve2) {
+        fuse_resolve(state);
 
-                fuse_resolve_done (state);
+    } else if (state->resolve_now == &state->resolve2) {
+        fuse_resolve_done(state);
 
-        } else {
-                gf_log ("fuse-resolve", GF_LOG_ERROR,
-                        "Invalid pointer for state->resolve_now");
-        }
+    } else {
+        gf_log("fuse-resolve", GF_LOG_ERROR,
+               "Invalid pointer for state->resolve_now");
+    }
 
-        return 0;
+    return 0;
 }
 
-
 int
-fuse_resolve_continue (fuse_state_t *state)
+fuse_resolve_continue(fuse_state_t *state)
 {
-        fuse_resolve_loc_touchup (state);
+    fuse_resolve_loc_touchup(state);
 
-        fuse_resolve_all (state);
+    fuse_resolve_all(state);
 
-        return 0;
+    return 0;
 }
 
 int
-fuse_resolve_and_resume (fuse_state_t *state, fuse_resume_fn_t fn)
+fuse_resolve_and_resume(fuse_state_t *state, fuse_resume_fn_t fn)
 {
-        fuse_gfid_set (state);
+    fuse_gfid_set(state);
 
-        state->resume_fn = fn;
+    state->resume_fn = fn;
 
-        fuse_resolve_all (state);
+    fuse_resolve_all(state);
 
-        return 0;
+    return 0;
 }
diff --git a/xlators/mount/fuse/utils/mount.glusterfs.in b/xlators/mount/fuse/utils/mount.glusterfs.in
index 82660328363..ac4d94cb743 100755
--- a/xlators/mount/fuse/utils/mount.glusterfs.in
+++ b/xlators/mount/fuse/utils/mount.glusterfs.in
@@ -34,7 +34,7 @@ _init ()
 
     # check whether getfattr exists
     export PATH
-    getfattr=$(which getfattr 2>/dev/null);
+    getfattr=$(command -v getfattr 2>/dev/null)
     if [ $? -ne 0 ]; then
         warn "WARNING: getfattr not found, certain checks will be skipped.."
     fi
@@ -161,10 +161,6 @@ start_glusterfs ()
        cmd_line=$(echo "$cmd_line --volfile-max-fetch-attempts=$volfile_max_fetch_attempts")
     fi
 
-    if [ -n "$fopen_keep_cache" ]; then
-        cmd_line=$(echo "$cmd_line --fopen-keep-cache");
-    fi
-
     if [ -n "$volfile_check" ]; then
         cmd_line=$(echo "$cmd_line --volfile-check");
     fi
@@ -177,11 +173,43 @@ start_glusterfs ()
         cmd_line=$(echo "$cmd_line --aux-gfid-mount");
     fi
 
+    if [ -n "$resolve_gids" ]; then
+        cmd_line=$(echo "$cmd_line --resolve-gids");
+    fi
+
     if [ -n "$no_root_squash" ]; then
         cmd_line=$(echo "$cmd_line --no-root-squash");
     fi
 
-#options with values start here
+    if [ -n "$thin_client" ]; then
+        cmd_line=$(echo "$cmd_line --thin-client");
+    fi
+
+    if [ -n "$global_threading" ]; then
+        cmd_line=$(echo "$cmd_line --global-threading");
+    fi
+
+#options with optional values start here
+    if [ -n "$fopen_keep_cache" ]; then
+        cmd_line=$(echo "$cmd_line --fopen-keep-cache=$fopen_keep_cache");
+    fi
+
+#options with mandatory values start here
+    if [ -n "$halo_max_latency" ]; then
+      cmd_line=$(echo "$cmd_line --xlator-option \
+ *replicate*.halo-max-latency=$halo_max_latency");
+    fi
+
+    if [ -n "$halo_max_replicas" ]; then
+      cmd_line=$(echo "$cmd_line --xlator-option \
+ *replicate*.halo-max-replicas=$halo_max_replicas");
+    fi
+
+    if [ -n "$halo_min_replicas" ]; then
+      cmd_line=$(echo "$cmd_line --xlator-option \
+ *replicate*.halo-min-replicas=$halo_min_replicas");
+    fi
+
     if [ -n "$log_level" ]; then
         cmd_line=$(echo "$cmd_line --log-level=$log_level");
     fi
@@ -198,6 +226,18 @@ start_glusterfs ()
         cmd_line=$(echo "$cmd_line --use-readdirp=$use_readdirp");
     fi
 
+    if [ -n "$event_history" ]; then
+        cmd_line=$(echo "$cmd_line --event-history=$event_history");
+    fi
+
+    if [ -n "$reader_thread_count" ]; then
+        cmd_line=$(echo "$cmd_line --reader-thread-count=$reader_thread_count");
+    fi
+
+    if [ -n "$fuse_auto_invalidation" ]; then
+        cmd_line=$(echo "$cmd_line --auto-invalidation=$fuse_auto_invalidation");
+    fi
+
     if [ -n "$volume_name" ]; then
         cmd_line=$(echo "$cmd_line --volume-name=$volume_name");
     fi
@@ -218,6 +258,14 @@ start_glusterfs ()
         cmd_line=$(echo "$cmd_line --gid-timeout=$gid_timeout");
     fi
 
+    if [ -n "$lru_limit" ]; then
+        cmd_line=$(echo "$cmd_line --lru-limit=$lru_limit");
+    fi
+
+    if [ -n "$invalidate_limit" ]; then
+        cmd_line=$(echo "$cmd_line --invalidate-limit=$invalidate_limit");
+    fi
+
     if [ -n "$bg_qlen" ]; then
         cmd_line=$(echo "$cmd_line --background-qlen=$bg_qlen");
     fi
@@ -226,6 +274,10 @@ start_glusterfs ()
         cmd_line=$(echo "$cmd_line --congestion-threshold=$cong_threshold");
     fi
 
+    if [ -n "$oom_score_adj" ]; then
+        cmd_line=$(echo "$cmd_line --oom-score-adj=$oom_score_adj");
+    fi
+
     if [ -n "$fuse_mountopts" ]; then
         cmd_line=$(echo "$cmd_line --fuse-mountopts=$fuse_mountopts");
     fi
@@ -234,6 +286,28 @@ start_glusterfs ()
         cmd_line=$(echo "$cmd_line --xlator-option=$xlator_option");
     fi
 
+    if [ -n "$kernel_writeback_cache" ]; then
+        cmd_line=$(echo "$cmd_line --kernel-writeback-cache=$kernel_writeback_cache");
+    fi
+
+    if [ -n "$attr_times_granularity" ]; then
+        cmd_line=$(echo "$cmd_line --attr-times-granularity=$attr_times_granularity");
+    fi
+
+    if [ -n "$dump_fuse" ]; then
+        cmd_line=$(echo "$cmd_line --dump-fuse=$dump_fuse");
+    fi
+
+    if [ -n "$fuse_flush_handle_interrupt" ]; then
+        cmd_line=$(echo "$cmd_line --fuse-flush-handle-interrupt=$fuse_flush_handle_interrupt");
+    fi
+
+    if [ -n "$process_name" ]; then
+        cmd_line=$(echo "$cmd_line --process-name fuse.$process_name");
+    else
+        cmd_line=$(echo "$cmd_line --process-name fuse");
+    fi
+
     # if trasnport type is specified, we have to append it to
     # volume name, so that it fetches the right client vol file
 
@@ -289,10 +363,21 @@ start_glusterfs ()
         cmd_line=$(echo "$cmd_line --fuse-mountopts=$fuse_mountopts");
     fi
 
+    if [ -n "$subdir_mount" ]; then
+        cmd_line=$(echo "$cmd_line --subdir-mount=/$subdir_mount");
+    fi
+
+    if [ -n "$fuse_dev_eperm_ratelimit_ns" ]; then
+        cmd_line=$(echo "$cmd_line --fuse-dev-eperm-ratelimit-ns=$fuse_dev_eperm_ratelimit_ns");
+    fi
+
     cmd_line=$(echo "$cmd_line $mount_point");
     $cmd_line;
     if [ $? -ne 0 ]; then
-        warn "Mount failed. Please check the log file for more details."
+        # If this is true, then glusterfs process returned error without
+        # getting daemonized. We have made sure the logs are posted to
+        # 'stderr', so no need to point them to logfile.
+        warn "Mounting glusterfs on $mount_point failed."
         exit 1;
     fi
 
@@ -300,7 +385,9 @@ start_glusterfs ()
     inode=$( ${getinode} $mount_point 2>/dev/null);
     # this is required if the stat returns error
     if [ $? -ne 0 ]; then
-        warn "Mount failed. Please check the log file for more details."
+        # At this time, glusterfs got daemonized, and then later exited.
+        # These failures are only logged in log file.
+        warn "Mount failed. Check the log file ${log_file} for more details."
         umount $mount_point > /dev/null 2>&1;
         exit 1;
     fi
@@ -309,9 +396,9 @@ start_glusterfs ()
 print_usage ()
 {
 cat << EOF
-Usage: $0 <volumeserver>:<volumeid/volumeport> -o<options> <mountpoint>
+Usage: $0 <server>:<volume/subdir> <mountpoint> -o<options>
 Options:
-man 8 $0
+man 8 $(basename $0)
 To display the version number of the mount helper: $0 -V
 EOF
 }
@@ -407,6 +494,9 @@ with_options()
         "volume-id")
             volume_id=$value
             ;;
+        "subdir-mount")
+            subdir_mount=$value
+            ;;
         "volfile-check")
             volfile_check=$value
             ;;
@@ -425,6 +515,12 @@ with_options()
         "gid-timeout")
             gid_timeout=$value
             ;;
+        "lru-limit")
+            lru_limit=$value
+            ;;
+        "invalidate-limit")
+            invalidate_limit=$value
+            ;;
         "background-qlen")
             bg_qlen=$value
             ;;
@@ -440,6 +536,9 @@ with_options()
         "congestion-threshold")
             cong_threshold=$value
             ;;
+        "oom-score-adj")
+            oom_score_adj=$value
+            ;;
         "xlator-option")
             xlator_option=$value
             ;;
@@ -449,6 +548,15 @@ with_options()
         "use-readdirp")
             use_readdirp=$value
             ;;
+        "event-history")
+            event_history=$value
+            ;;
+        "reader-thread-count")
+            reader_thread_count=$value
+            ;;
+        "auto-invalidation")
+            fuse_auto_invalidation=$value
+	    ;;
         "no-root-squash")
             if [ $value = "yes" ] ||
                 [ $value = "on" ] ||
@@ -463,6 +571,45 @@ with_options()
                 [ $value = "false" ] ; then
                 no_root_squash=1;
             fi ;;
+        "kernel-writeback-cache")
+            kernel_writeback_cache=$value
+            ;;
+        "attr-times-granularity")
+            attr_times_granularity=$value
+            ;;
+        "dump-fuse")
+            dump_fuse=$value
+            ;;
+        "fuse-flush-handle-interrupt")
+            fuse_flush_handle_interrupt=$value
+            ;;
+        "fuse-dev-eperm-ratelimit-ns")
+            fuse_dev_eperm_ratelimit_ns=$value
+            ;;
+        "context"|"fscontext"|"defcontext"|"rootcontext")
+            # standard SElinux mount options to pass to the kernel
+            [ -z "$fuse_mountopts" ] || fuse_mountopts="$fuse_mountopts,"
+            fuse_mountopts="${fuse_mountopts}$key=\"$value\""
+            ;;
+        "halo-max-latency")
+            halo_max_latency=$value
+            ;;
+        "halo-max-replicas")
+            halo_max_replicas=$value
+            ;;
+        "halo-min-replicas")
+          halo_min_replicas=$value
+          ;;
+        "process-name")
+            process_name=$value
+            ;;
+        # Values that are optional
+        "fopen-keep-cache")
+            fopen_keep_cache="=$value"
+            ;;
+        x-*)
+            # comments or userspace application-specific options, drop them
+            ;;
         *)
             warn "Invalid option: $key"
             exit 1
@@ -487,9 +634,6 @@ without_options()
         "worm")
             worm=1
             ;;
-        "fopen-keep-cache")
-            fopen_keep_cache=1
-            ;;
         "enable-ino32")
             enable_ino32=1
             ;;
@@ -501,14 +645,46 @@ without_options()
                 aux_gfid_mount=1
             fi
             ;;
+        "thin-client")
+            thin_client=1
+            ;;
+        "resolve-gids")
+            resolve_gids=1
+            ;;
          # "mount -t glusterfs" sends this, but it's useless.
         "rw")
             ;;
+        "global-threading")
+            global_threading=1
+            ;;
+         # TODO: not sure how to handle this yet
+        "async"|"sync"|"dirsync"|\
+        "mand"|"nomand"|\
+        "silent"|"loud"|\
+        "iversion"|"noiversion"|\
+        "nofail")
+            warn "mount option '${option}' is not handled (yet?)"
+            ;;
+         # standard mount options to pass to the kernel
+        "atime"|"noatime"|"diratime"|"nodiratime"|\
+        "relatime"|"norelatime"|\
+        "strictatime"|"nostrictatime"|"lazyatime"|"nolazyatime"|\
+        "dev"|"nodev"|"exec"|"noexec"|"suid"|"nosuid"|"auto_unmount")
+            [ -z "$fuse_mountopts" ] || fuse_mountopts="$fuse_mountopts,"
+            fuse_mountopts="${fuse_mountopts}${option}"
+            ;;
          # these ones are interpreted during system initialization
-        "noauto")
+        "auto"|"noauto")
             ;;
         "_netdev")
             ;;
+        # Values that are optional
+        "fopen-keep-cache")
+            fopen_keep_cache="true"
+            ;;
+        x-*)
+            # comments or userspace application-specific options, drop them
+            ;;
         *)
             warn "Invalid option $option";
             exit 1
@@ -547,21 +723,23 @@ update_updatedb()
 
 main ()
 {
-    if [ "x${uname_s}" = "xLinux" ] ; then
+    if [ "x${uname_s}" = "xLinux" -a $# -ge 2 ] ; then
         volfile_loc=$1
         mount_point=$2
 
         ## `mount` specifies options as a last argument
         shift 2;
     fi
-    while getopts "Vo:hn" opt; do
+    while getopts "Vo:hns" opt; do
         case "${opt}" in
             o)
                 parse_options ${OPTARG};
 		shift 2;
                 ;;
             n)
-		shift 1;
+                ;;
+            s)
+                # accept+ignore sloppy mount, passed by autofs
                 ;;
             V)
                 ${cmd_line} -V;
@@ -584,10 +762,23 @@ main ()
     fi
 
     [ -r "$volfile_loc" ] || {
-        server_ip=$(echo "$volfile_loc" | sed -n 's/\([a-zA-Z0-9:.\-]*\):.*/\1/p');
+        # '%' included to support ipv6 link local addresses
+        server_ip=$(echo "$volfile_loc" | sed -n 's/\([a-zA-Z0-9:%,.\-]*\):.*/\1/p');
         volume_str=$(echo "$volfile_loc" | sed -n 's/.*:\([^ ]*\).*/\1/p');
         [ -n "$volume_str" ] && {
-            volume_id="$volume_str";
+            volume_id=$volume_str
+            volume_str_temp=$volume_str
+            first_char=$(echo "$volume_str" | cut -c 1)
+            [ ${first_char} = '/' ] && {
+                volume_str_temp=$(echo "$volume_str" | cut -c 2-)
+            }
+            volume_id_temp=$(echo "$volume_str_temp" | cut -f1 -d '/');
+            [ $(echo $volume_str_temp | grep -c "/") -eq 1 ] &&
+                    [ "$volume_id_temp" != "snaps" ]  && {
+                volume_id=$volume_id_temp;
+                [ ${first_char} = '/' ] && volume_id=/$volume_id;
+                subdir_mount=$(echo "$volume_str_temp" | cut -f2- -d '/');
+            }
         }
         volfile_loc="";
         [ -z "$volume_id" -o -z "$server_ip" ] && {
@@ -625,7 +816,7 @@ EOF
     }
 
     # Simple check to avoid multiple identical mounts
-    if grep -q "[[:space:]+]${mount_point}[[:space:]+]fuse" $mounttab; then
+    if grep -q "[[:space:]+]${mount_point}[[:space:]+]fuse.glusterfs" $mounttab; then
         warn "$0: according to mtab, GlusterFS is already mounted on" \
              "$mount_point"
         exit 32;
diff --git a/xlators/mount/fuse/utils/mount_glusterfs.in b/xlators/mount/fuse/utils/mount_glusterfs.in
index 50a301c26a8..3a5feb606d7 100755
--- a/xlators/mount/fuse/utils/mount_glusterfs.in
+++ b/xlators/mount/fuse/utils/mount_glusterfs.in
@@ -173,6 +173,10 @@ start_glusterfs ()
         cmd_line=$(echo "$cmd_line --no-root-squash");
     fi
 
+    if [ -n "$capability" ]; then
+         cmd_line=$(echo "$cmd_line --capability");
+    fi
+
 #options with values start here
     if [ -n "$log_level" ]; then
         cmd_line=$(echo "$cmd_line --log-level=$log_level");
@@ -230,9 +234,11 @@ start_glusterfs ()
         cmd_line=$(echo "$cmd_line --xlator-option=$xlator_option");
     fi
 
-    # for rdma volume, we have to fetch volfile with '.rdma' added
-    # to volume name, so that it fetches the right client vol file
-    volume_id_rdma="";
+    if [ -n "$process_name" ]; then
+        cmd_line=$(echo "$cmd_line --process-name fuse.$process_name");
+    else
+        cmd_line=$(echo "$cmd_line --process-name fuse");
+    fi
 
     if [ -z "$volfile_loc" ]; then
         if  [ -n "$server_ip" ]; then
@@ -272,15 +278,9 @@ start_glusterfs ()
 
             if [ -n "$transport" ]; then
                 cmd_line=$(echo "$cmd_line --volfile-server-transport=$transport");
-                if [ "$transport" = "rdma" ]; then
-                    volume_id_rdma=".rdma";
-                fi
             fi
 
             if [ -n "$volume_id" ]; then
-                if [ -n "$volume_id_rdma" ]; then
-                    volume_id="$volume_id$volume_id_rdma";
-                fi
                 cmd_line=$(echo "$cmd_line --volfile-id=$volume_id");
             fi
         fi
@@ -394,6 +394,9 @@ with_options()
                 [ $value = "false" ] ; then
                 no_root_squash=1;
             fi ;;
+        "process-name")
+            process_name=$value
+            ;;
         *)
             warn "Invalid option: $key"
             exit 1
@@ -440,6 +443,9 @@ without_options()
             ;;
         "_netdev")
             ;;
+        "capability")
+            capability=1
+            ;;
         *)
             warn "Invalid option $option";
             exit 1
@@ -463,6 +469,7 @@ parse_options()
 
 main ()
 {
+#if !defined(__FreeBSD__)
     ## `mount` on OSX specifies options as first argument
     echo $1|grep -q -- "-o"
     if [ $? -eq 0 ];  then
@@ -472,7 +479,7 @@ main ()
         volfile_loc=$1
         mount_point=$2
     fi
-
+#endif /* __FreeBSD__ */
     while getopts "Vo:h" opt; do
         case "${opt}" in
             o)
@@ -493,8 +500,15 @@ main ()
         esac
     done
 
+#ifdef __FreeBSD__
+    shift $((OPTIND - 1))
+    volfile_loc="$1"
+    mount_point="$2"
+#endif /* __FreeBSD__ */
+
     [ -r "$volfile_loc" ] || {
-        server_ip=$(echo "$volfile_loc" | sed -n 's/\([a-zA-Z0-9:.\-]*\):.*/\1/p');
+        # '%' included to support ipv6 link local addresses
+        server_ip=$(echo "$volfile_loc" | sed -n 's/\([a-zA-Z0-9:%.\-]*\):.*/\1/p');
         volume_str=$(echo "$volfile_loc" | sed -n 's/.*:\([^ ]*\).*/\1/p');
         [ -n "$volume_str" ] && {
             volume_id="$volume_str";
diff --git a/xlators/nfs/server/src/Makefile.am b/xlators/nfs/server/src/Makefile.am
index f91bbcf622a..01071a79a21 100644
--- a/xlators/nfs/server/src/Makefile.am
+++ b/xlators/nfs/server/src/Makefile.am
@@ -1,13 +1,22 @@
+if WITH_SERVER
 xlator_LTLIBRARIES = server.la
+endif
+
 xlatordir = $(libdir)/glusterfs/$(PACKAGE_VERSION)/xlator/nfs
 nfsrpclibdir = $(top_srcdir)/rpc/rpc-lib/src
-server_la_LDFLAGS = -module -avoid-version
+server_la_LDFLAGS = -module \
+        -export-symbols $(top_srcdir)/xlators/nfs/server/src/nfsserver.sym \
+        $(GF_XLATOR_LDFLAGS)
+
 server_la_SOURCES = nfs.c nfs-common.c nfs-fops.c nfs-inodes.c \
 	nfs-generics.c mount3.c nfs3-fh.c nfs3.c nfs3-helpers.c nlm4.c \
 	nlmcbk_svc.c mount3udp_svc.c acl3.c netgroups.c exports.c \
 	mount3-auth.c auth-cache.c
+
 server_la_LIBADD = $(top_builddir)/libglusterfs/src/libglusterfs.la \
-                   $(top_builddir)/api/src/libgfapi.la
+	$(top_builddir)/api/src/libgfapi.la \
+	$(top_builddir)/rpc/rpc-lib/src/libgfrpc.la \
+	$(top_builddir)/rpc/xdr/src/libgfxdr.la
 
 noinst_HEADERS = nfs.h nfs-common.h nfs-fops.h nfs-inodes.h nfs-generics.h \
 	mount3.h nfs3-fh.h nfs3.h nfs3-helpers.h nfs-mem-types.h nlm4.h \
@@ -15,13 +24,15 @@ noinst_HEADERS = nfs.h nfs-common.h nfs-fops.h nfs-inodes.h nfs-generics.h \
 
 AM_CPPFLAGS = $(GF_CPPFLAGS) \
 	-DLIBDIR=\"$(libdir)/glusterfs/$(PACKAGE_VERSION)/auth\" \
-	-I$(top_srcdir)/libglusterfs/src \
-	-I$(top_srcdir)/api/src \
+	-I$(top_srcdir)/libglusterfs/src -I$(top_srcdir)/api/src \
+	-I$(top_srcdir)/rpc/xdr/src/ -I$(top_builddir)/rpc/xdr/src/ \
 	-I$(nfsrpclibdir) -I$(CONTRIBDIR)/rbtree \
-	-I$(top_srcdir)/rpc/xdr/src/ -DDATADIR=\"$(localstatedir)\"
+	-DDATADIR=\"$(localstatedir)\"
 
 AM_CFLAGS = -Wall $(GF_CFLAGS)
 
 AM_LDFLAGS = -L$(xlatordir)
 
 CLEANFILES =
+
+EXTRA_DIST = nfsserver.sym
diff --git a/xlators/nfs/server/src/acl3.c b/xlators/nfs/server/src/acl3.c
index 230fb229579..7e3bbf16086 100644
--- a/xlators/nfs/server/src/acl3.c
+++ b/xlators/nfs/server/src/acl3.c
@@ -8,241 +8,247 @@
  * cases as published by the Free Software Foundation.
  */
 
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
-
-#include "defaults.h"
+#include <glusterfs/defaults.h>
 #include "rpcsvc.h"
-#include "dict.h"
-#include "xlator.h"
+#include <glusterfs/dict.h>
+#include <glusterfs/xlator.h>
 #include "nfs.h"
-#include "mem-pool.h"
-#include "logging.h"
+#include <glusterfs/mem-pool.h>
+#include <glusterfs/logging.h>
 #include "nfs-fops.h"
-#include "inode.h"
 #include "nfs3.h"
 #include "nfs-mem-types.h"
 #include "nfs3-helpers.h"
 #include "nfs3-fh.h"
 #include "nfs-generics.h"
 #include "acl3.h"
-#include "byte-order.h"
-#include "compat-errno.h"
+#include <glusterfs/byte-order.h>
+#include <glusterfs/compat-errno.h>
+#include "nfs-messages.h"
 
 static int
-acl3_nfs_acl_to_xattr (aclentry *ace, void *xattrbuf,
-                       int aclcount, int defacl);
+acl3_nfs_acl_to_xattr(aclentry *ace, void *xattrbuf, int aclcount, int defacl);
 
 static int
-acl3_nfs_acl_from_xattr (aclentry *ace, void *xattrbuf,
-                         int bufsize, int defacl);
+acl3_nfs_acl_from_xattr(aclentry *ace, void *xattrbuf, int bufsize, int defacl);
 
-typedef ssize_t (*acl3_serializer) (struct iovec outmsg, void *args);
+typedef ssize_t (*acl3_serializer)(struct iovec outmsg, void *args);
 
-extern void nfs3_call_state_wipe (nfs3_call_state_t *cs);
+extern void
+nfs3_call_state_wipe(nfs3_call_state_t *cs);
 
 extern nfs3_call_state_t *
-nfs3_call_state_init (struct nfs3_state *s, rpcsvc_request_t *req, xlator_t *v);
+nfs3_call_state_init(struct nfs3_state *s, rpcsvc_request_t *req, xlator_t *v);
 
 extern int
-nfs3_fh_validate (struct nfs3_fh *fh);
+nfs3_fh_validate(struct nfs3_fh *fh);
 
 extern void
-nfs3_stat_to_fattr3 (struct iatt *buf, fattr3 *fa);
-
-#define acl3_validate_nfs3_state(request, state, status, label, retval) \
-        do      {                                                       \
-                state = rpcsvc_request_program_private (request);       \
-                if (!state) {                                           \
-                        gf_log (GF_ACL, GF_LOG_ERROR, "NFSv3 state "    \
-                                "missing from RPC request");            \
-                        rpcsvc_request_seterr (req, SYSTEM_ERR);        \
-                        status = NFS3ERR_SERVERFAULT;                   \
-                        goto label;                                     \
-                }                                                       \
-        } while (0);                                                    \
-
-#define acl3_validate_gluster_fh(handle, status, errlabel)              \
-        do {                                                            \
-                if (!nfs3_fh_validate (handle)) {                       \
-                        gf_log (GF_ACL, GF_LOG_ERROR, "Bad Handle");    \
-                        status = NFS3ERR_BADHANDLE;                     \
-                        goto errlabel;                                  \
-                }                                                       \
-        } while (0)                                                     \
-
+nfs3_stat_to_fattr3(struct iatt *buf, fattr3 *fa);
+
+#define acl3_validate_nfs3_state(request, state, status, label, retval)        \
+    do {                                                                       \
+        state = rpcsvc_request_program_private(request);                       \
+        if (!state) {                                                          \
+            gf_msg(GF_ACL, GF_LOG_ERROR, errno, NFS_MSG_STATE_MISSING,         \
+                   "NFSv3 state "                                              \
+                   "missing from RPC request");                                \
+            rpcsvc_request_seterr(req, SYSTEM_ERR);                            \
+            status = NFS3ERR_SERVERFAULT;                                      \
+            goto label;                                                        \
+        }                                                                      \
+    } while (0);
+
+#define acl3_validate_gluster_fh(handle, status, errlabel)                     \
+    do {                                                                       \
+        if (!nfs3_fh_validate(handle)) {                                       \
+            gf_msg(GF_ACL, GF_LOG_ERROR, EINVAL, NFS_MSG_BAD_HANDLE,           \
+                   "Bad Handle");                                              \
+            status = NFS3ERR_BADHANDLE;                                        \
+            goto errlabel;                                                     \
+        }                                                                      \
+    } while (0)
 
 extern xlator_t *
-nfs3_fh_to_xlator (struct nfs3_state *nfs3, struct nfs3_fh *fh);
-
-#define acl3_map_fh_to_volume(nfs3state, handle, req, volume, status, label) \
-        do {                                                            \
-                char exportid[256], gfid[256];                          \
-                rpc_transport_t *trans = NULL;                          \
-                volume = nfs3_fh_to_xlator ((nfs3state), handle);      \
-                if (!volume) {                                          \
-                        gf_uuid_unparse (handle->exportid, exportid);       \
-                        gf_uuid_unparse (handle->gfid, gfid);               \
-                        trans = rpcsvc_request_transport (req);         \
-                        gf_log (GF_ACL, GF_LOG_ERROR, "Failed to map "  \
-                                "FH to vol: client=%s, exportid=%s, gfid=%s",\
-                                trans->peerinfo.identifier, exportid,   \
-                                gfid);                                  \
-                        gf_log (GF_ACL, GF_LOG_ERROR,                   \
-                                "Stale nfs client %s must be trying to "\
-                                "connect to a deleted volume, please "  \
-                                "unmount it.", trans->peerinfo.identifier);\
-                        status = NFS3ERR_STALE;                         \
-                        goto label;                                     \
-                } else {                                                \
-                        gf_log (GF_ACL, GF_LOG_TRACE, "FH to Volume: %s"\
-                                ,volume->name);                         \
-                        rpcsvc_request_set_private (req, volume);       \
-                }                                                       \
-        } while (0);                                                    \
-
-#define acl3_volume_started_check(nfs3state, vlm, rtval, erlbl)         \
-        do {                                                            \
-              if ((!nfs_subvolume_started (nfs_state (nfs3state->nfsx), vlm))){\
-                      gf_log (GF_ACL, GF_LOG_ERROR, "Volume is disabled: %s",\
-                              vlm->name);                               \
-                      rtval = RPCSVC_ACTOR_IGNORE;                      \
-                      goto erlbl;                                       \
-              }                                                         \
-        } while (0)                                                     \
-
-#define acl3_check_fh_resolve_status(cst, nfstat, erlabl)               \
-        do {                                                            \
-                xlator_t *xlatorp = NULL;                               \
-                char buf[256], gfid[256];                               \
-                rpc_transport_t *trans = NULL;                          \
-                if ((cst)->resolve_ret < 0) {                           \
-                        trans = rpcsvc_request_transport (cst->req);    \
-                        xlatorp = nfs3_fh_to_xlator (cst->nfs3state,    \
-                                                     &cst->resolvefh);  \
-                        gf_uuid_unparse (cst->resolvefh.gfid, gfid);       \
-                        snprintf (buf, sizeof (buf), "(%s) %s : %s",             \
-                                  trans->peerinfo.identifier,           \
-                                  xlatorp ? xlatorp->name : "ERR",      \
-                                  gfid);                                \
-                        gf_log (GF_ACL, GF_LOG_ERROR, "Unable to resolve FH"\
-                                ": %s", buf);                           \
-                        nfstat = nfs3_errno_to_nfsstat3 (cst->resolve_errno);\
-                        goto erlabl;                                    \
-                }                                                       \
-        } while (0)                                                     \
-
-#define acl3_handle_call_state_init(nfs3state, calls, rq, v, opstat, errlabel)\
-        do {                                                            \
-                calls = nfs3_call_state_init ((nfs3state), (rq), v); \
-                if (!calls) {                                           \
-                        gf_log (GF_ACL, GF_LOG_ERROR, "Failed to "      \
-                                "init call state");                     \
-                        opstat = NFS3ERR_SERVERFAULT;                   \
-                        rpcsvc_request_seterr (req, SYSTEM_ERR);        \
-                        goto errlabel;                                  \
-                }                                                       \
-        } while (0)                                                     \
-
+nfs3_fh_to_xlator(struct nfs3_state *nfs3, struct nfs3_fh *fh);
+
+#define acl3_map_fh_to_volume(nfs3state, handle, req, volume, status, label)   \
+    do {                                                                       \
+        char exportid[256], gfid[256];                                         \
+        rpc_transport_t *trans = NULL;                                         \
+        volume = nfs3_fh_to_xlator((nfs3state), handle);                       \
+        if (!volume) {                                                         \
+            gf_uuid_unparse(handle->exportid, exportid);                       \
+            gf_uuid_unparse(handle->gfid, gfid);                               \
+            trans = rpcsvc_request_transport(req);                             \
+            gf_msg(GF_ACL, GF_LOG_ERROR, 0, NFS_MSG_FH_TO_VOL_FAIL,            \
+                   "Failed to map "                                            \
+                   "FH to vol: client=%s, exportid=%s, gfid=%s",               \
+                   trans->peerinfo.identifier, exportid, gfid);                \
+            gf_msg(GF_ACL, GF_LOG_ERROR, ESTALE, NFS_MSG_VOLUME_ERROR,         \
+                   "Stale nfs client %s must be trying to "                    \
+                   "connect to a deleted volume, please "                      \
+                   "unmount it.",                                              \
+                   trans->peerinfo.identifier);                                \
+            status = NFS3ERR_STALE;                                            \
+            goto label;                                                        \
+        } else {                                                               \
+            gf_msg_trace(GF_ACL, 0, "FH to Volume: %s", volume->name);         \
+            rpcsvc_request_set_private(req, volume);                           \
+        }                                                                      \
+    } while (0);
+
+#define acl3_volume_started_check(nfs3state, vlm, rtval, erlbl)                \
+    do {                                                                       \
+        if ((!nfs_subvolume_started(nfs_state(nfs3state->nfsx), vlm))) {       \
+            gf_msg(GF_ACL, GF_LOG_ERROR, 0, NFS_MSG_VOL_DISABLE,               \
+                   "Volume is disabled: %s", vlm->name);                       \
+            rtval = RPCSVC_ACTOR_IGNORE;                                       \
+            goto erlbl;                                                        \
+        }                                                                      \
+    } while (0)
+
+#define acl3_check_fh_resolve_status(cst, nfstat, erlabl)                      \
+    do {                                                                       \
+        xlator_t *xlatorp = NULL;                                              \
+        char buf[256], gfid[GF_UUID_BUF_SIZE];                                 \
+        rpc_transport_t *trans = NULL;                                         \
+        if ((cst)->resolve_ret < 0) {                                          \
+            trans = rpcsvc_request_transport(cst->req);                        \
+            xlatorp = nfs3_fh_to_xlator(cst->nfs3state, &cst->resolvefh);      \
+            gf_uuid_unparse(cst->resolvefh.gfid, gfid);                        \
+            snprintf(buf, sizeof(buf), "(%s) %s : %s",                         \
+                     trans->peerinfo.identifier,                               \
+                     xlatorp ? xlatorp->name : "ERR", gfid);                   \
+            gf_msg(GF_ACL, GF_LOG_ERROR, cst->resolve_errno,                   \
+                   NFS_MSG_RESOLVE_FH_FAIL,                                    \
+                   "Unable to resolve "                                        \
+                   "FH: %s",                                                   \
+                   buf);                                                       \
+            nfstat = nfs3_errno_to_nfsstat3(cst->resolve_errno);               \
+            goto erlabl;                                                       \
+        }                                                                      \
+    } while (0)
+
+#define acl3_handle_call_state_init(nfs3state, calls, rq, v, opstat, errlabel) \
+    do {                                                                       \
+        calls = nfs3_call_state_init((nfs3state), (rq), v);                    \
+        if (!calls) {                                                          \
+            gf_msg(GF_ACL, GF_LOG_ERROR, 0, NFS_MSG_INIT_CALL_STAT_FAIL,       \
+                   "Failed to "                                                \
+                   "init call state");                                         \
+            opstat = NFS3ERR_SERVERFAULT;                                      \
+            rpcsvc_request_seterr(req, SYSTEM_ERR);                            \
+            goto errlabel;                                                     \
+        }                                                                      \
+    } while (0)
 
 int
-acl3svc_submit_reply (rpcsvc_request_t *req, void *arg, acl3_serializer sfunc)
+acl3svc_submit_reply(rpcsvc_request_t *req, void *arg, acl3_serializer sfunc)
 {
-        struct iovec            outmsg  = {0, };
-        struct iobuf            *iob    = NULL;
-        struct nfs3_state       *nfs3   = NULL;
-        int                     ret     = -1;
-        ssize_t                 msglen  = 0;
-        struct iobref           *iobref = NULL;
-
-        if (!req)
-                return -1;
-
-        nfs3 = (struct nfs3_state *)rpcsvc_request_program_private (req);
-        if (!nfs3) {
-                gf_log (GF_ACL, GF_LOG_ERROR, "mount state not found");
-                goto ret;
-        }
-
-        /* First, get the io buffer into which the reply in arg will
-         * be serialized.
-         */
-        iob = iobuf_get (nfs3->iobpool);
-        if (!iob) {
-                gf_log (GF_ACL, GF_LOG_ERROR, "Failed to get iobuf");
-                goto ret;
-        }
-
-        iobuf_to_iovec (iob, &outmsg);
-        /* Use the given serializer to translate the give C structure in arg
-         * to XDR format which will be written into the buffer in outmsg.
-         */
-        msglen = sfunc (outmsg, arg);
-        if (msglen < 0) {
-                gf_log (GF_ACL, GF_LOG_ERROR, "Failed to encode message");
-                goto ret;
-        }
-        outmsg.iov_len = msglen;
-
-        iobref = iobref_new ();
-        if (iobref == NULL) {
-                gf_log (GF_ACL, GF_LOG_ERROR, "Failed to get iobref");
-                goto ret;
-        }
-
-        ret = iobref_add (iobref, iob);
-        if (ret) {
-                gf_log (GF_ACL, GF_LOG_ERROR, "Failed to add iob to iobref");
-                goto ret;
-        }
-
-        /* Then, submit the message for transmission. */
-        ret = rpcsvc_submit_message (req, &outmsg, 1, NULL, 0, iobref);
-        if (ret == -1) {
-                gf_log (GF_ACL, GF_LOG_ERROR, "Reply submission failed");
-                goto ret;
-        }
-
-        ret = 0;
+    struct iovec outmsg = {
+        0,
+    };
+    struct iobuf *iob = NULL;
+    struct nfs3_state *nfs3 = NULL;
+    int ret = -1;
+    ssize_t msglen = 0;
+    struct iobref *iobref = NULL;
+
+    if (!req)
+        return -1;
+
+    nfs3 = (struct nfs3_state *)rpcsvc_request_program_private(req);
+    if (!nfs3) {
+        gf_msg(GF_ACL, GF_LOG_ERROR, EINVAL, NFS_MSG_MNT_STATE_NOT_FOUND,
+               "mount state not found");
+        goto ret;
+    }
+
+    /* First, get the io buffer into which the reply in arg will
+     * be serialized.
+     */
+    iob = iobuf_get(nfs3->iobpool);
+    if (!iob) {
+        gf_msg(GF_ACL, GF_LOG_ERROR, ENOMEM, NFS_MSG_NO_MEMORY,
+               "Failed to get iobuf");
+        goto ret;
+    }
+
+    iobuf_to_iovec(iob, &outmsg);
+    /* Use the given serializer to translate the give C structure in arg
+     * to XDR format which will be written into the buffer in outmsg.
+     */
+    msglen = sfunc(outmsg, arg);
+    if (msglen < 0) {
+        gf_msg(GF_ACL, GF_LOG_ERROR, errno, NFS_MSG_ENCODE_MSG_FAIL,
+               "Failed to encode message");
+        goto ret;
+    }
+    outmsg.iov_len = msglen;
+
+    iobref = iobref_new();
+    if (iobref == NULL) {
+        gf_msg(GF_ACL, GF_LOG_ERROR, ENOMEM, NFS_MSG_NO_MEMORY,
+               "Failed to get iobref");
+        goto ret;
+    }
+
+    ret = iobref_add(iobref, iob);
+    if (ret) {
+        gf_msg(GF_ACL, GF_LOG_ERROR, ENOMEM, NFS_MSG_NO_MEMORY,
+               "Failed to add iob to iobref");
+        goto ret;
+    }
+
+    /* Then, submit the message for transmission. */
+    ret = rpcsvc_submit_message(req, &outmsg, 1, NULL, 0, iobref);
+    if (ret == -1) {
+        gf_msg(GF_ACL, GF_LOG_ERROR, errno, NFS_MSG_REP_SUBMIT_FAIL,
+               "Reply submission failed");
+        goto ret;
+    }
+
+    ret = 0;
 ret:
-        if (iob)
-                iobuf_unref (iob);
-        if (iobref)
-                iobref_unref (iobref);
+    if (iob)
+        iobuf_unref(iob);
+    if (iobref)
+        iobref_unref(iobref);
 
-        return ret;
+    return ret;
 }
 
-
 int
-acl3svc_null (rpcsvc_request_t *req)
+acl3svc_null(rpcsvc_request_t *req)
 {
-        struct iovec    dummyvec = {0, };
+    struct iovec dummyvec = {
+        0,
+    };
 
-        if (!req) {
-                gf_log (GF_ACL, GF_LOG_ERROR, "Got NULL request!");
-                return 0;
-        }
-        rpcsvc_submit_generic (req, &dummyvec, 1,  NULL, 0, NULL);
+    if (!req) {
+        gf_msg(GF_ACL, GF_LOG_ERROR, EINVAL, NFS_MSG_INVALID_ENTRY,
+               "Got NULL request!");
         return 0;
+    }
+    rpcsvc_submit_generic(req, &dummyvec, 1, NULL, 0, NULL);
+    return 0;
 }
 
 int
-acl3_getacl_reply (rpcsvc_request_t *req, getaclreply *reply)
+acl3_getacl_reply(rpcsvc_request_t *req, getaclreply *reply)
 {
-        acl3svc_submit_reply (req, (void *)reply,
-                              (acl3_serializer)xdr_serialize_getaclreply);
-        return 0;
+    acl3svc_submit_reply(req, (void *)reply,
+                         (acl3_serializer)xdr_serialize_getaclreply);
+    return 0;
 }
 
 int
-acl3_setacl_reply (rpcsvc_request_t *req, setaclreply *reply)
+acl3_setacl_reply(rpcsvc_request_t *req, setaclreply *reply)
 {
-        acl3svc_submit_reply (req, (void *)reply,
-                              (acl3_serializer)xdr_serialize_setaclreply);
-        return 0;
+    acl3svc_submit_reply(req, (void *)reply,
+                         (acl3_serializer)xdr_serialize_setaclreply);
+    return 0;
 }
 
 /* acl3_getacl_cbk: fetch and decode the ACL in the POSIX_ACL_ACCESS_XATTR
@@ -250,62 +256,59 @@ acl3_setacl_reply (rpcsvc_request_t *req, setaclreply *reply)
  * The POSIX_ACL_ACCESS_XATTR can be set on files and directories.
  */
 int
-acl3_getacl_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                 int32_t op_ret, int32_t op_errno, dict_t *dict,
-                 dict_t *xdata)
+acl3_getacl_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                int32_t op_ret, int32_t op_errno, dict_t *dict, dict_t *xdata)
 {
-        nfsstat3                 stat = NFS3ERR_SERVERFAULT;
-        nfs3_call_state_t        *cs = NULL;
-        data_t                   *data = NULL;
-        getaclreply              *getaclreply = NULL;
-        int                      aclcount = 0;
-        int                      defacl = 1; /* DEFAULT ACL */
-
-        if (!frame->local) {
-                gf_log (GF_ACL, GF_LOG_ERROR, "Invalid argument,"
-                       " frame->local NULL");
-                return -EINVAL;
-        }
-        cs = frame->local;
-        getaclreply = &cs->args.getaclreply;
-        if ((op_ret < 0) && (op_errno != ENODATA && op_errno != ENOATTR)) {
-                stat = nfs3_cbk_errno_status (op_ret, op_errno);
-                goto err;
-        } else if (!dict) {
-                /* no ACL has been set */
-                stat = NFS3_OK;
-                goto err;
-        }
-
-        getaclreply->aclentry.aclentry_val = cs->aclentry;
-
-        /* getfacl: NFS USER ACL */
-        data = dict_get (dict, POSIX_ACL_ACCESS_XATTR);
-        if (data && data->data) {
-                aclcount = acl3_nfs_acl_from_xattr (cs->aclentry,
-                                                    data->data,
-                                                    data->len,
-                                                    !defacl);
-                if (aclcount < 0) {
-                        gf_log (GF_ACL, GF_LOG_ERROR,
-                                "Failed to get USER ACL");
-                        stat = nfs3_errno_to_nfsstat3 (-aclcount);
-                        goto err;
-                }
-                getaclreply->aclcount = aclcount;
-                getaclreply->aclentry.aclentry_len = aclcount;
-        }
-
-        acl3_getacl_reply (cs->req, getaclreply);
-        nfs3_call_state_wipe (cs);
-        return 0;
+    nfsstat3 stat = NFS3ERR_SERVERFAULT;
+    nfs3_call_state_t *cs = NULL;
+    data_t *data = NULL;
+    getaclreply *getaclreply = NULL;
+    int aclcount = 0;
+    int defacl = 1; /* DEFAULT ACL */
+
+    if (!frame->local) {
+        gf_msg(GF_ACL, GF_LOG_ERROR, EINVAL, NFS_MSG_INVALID_ENTRY,
+               "Invalid argument, frame->local NULL");
+        return -EINVAL;
+    }
+    cs = frame->local;
+    getaclreply = &cs->args.getaclreply;
+    if ((op_ret < 0) && (op_errno != ENODATA && op_errno != ENOATTR)) {
+        stat = nfs3_cbk_errno_status(op_ret, op_errno);
+        goto err;
+    } else if (!dict) {
+        /* no ACL has been set */
+        stat = NFS3_OK;
+        goto err;
+    }
+
+    getaclreply->aclentry.aclentry_val = cs->aclentry;
+
+    /* getfacl: NFS USER ACL */
+    data = dict_get(dict, POSIX_ACL_ACCESS_XATTR);
+    if (data && data->data) {
+        aclcount = acl3_nfs_acl_from_xattr(cs->aclentry, data->data, data->len,
+                                           !defacl);
+        if (aclcount < 0) {
+            gf_msg(GF_ACL, GF_LOG_ERROR, aclcount, NFS_MSG_GET_USER_ACL_FAIL,
+                   "Failed to get USER ACL");
+            stat = nfs3_errno_to_nfsstat3(-aclcount);
+            goto err;
+        }
+        getaclreply->aclcount = aclcount;
+        getaclreply->aclentry.aclentry_len = aclcount;
+    }
+
+    acl3_getacl_reply(cs->req, getaclreply);
+    nfs3_call_state_wipe(cs);
+    return 0;
 
 err:
-        if (getaclreply)
-                getaclreply->status = stat;
-        acl3_getacl_reply (cs->req, getaclreply);
-        nfs3_call_state_wipe (cs);
-        return 0;
+    if (getaclreply)
+        getaclreply->status = stat;
+    acl3_getacl_reply(cs->req, getaclreply);
+    nfs3_call_state_wipe(cs);
+    return 0;
 }
 
 /* acl3_default_getacl_cbk: fetch and decode the ACL set in the
@@ -317,604 +320,614 @@ err:
  * ACL that can be set in POSIX_ACL_DEFAULT_XATTR.
  */
 int
-acl3_default_getacl_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                         int32_t op_ret, int32_t op_errno, dict_t *dict,
-                         dict_t *xdata)
+acl3_default_getacl_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                        int32_t op_ret, int32_t op_errno, dict_t *dict,
+                        dict_t *xdata)
 {
-        nfsstat3                 stat = NFS3ERR_SERVERFAULT;
-        nfs3_call_state_t        *cs = NULL;
-        data_t                   *data = NULL;
-        getaclreply              *getaclreply = NULL;
-        int                      aclcount = 0;
-        int                      defacl = 1; /* DEFAULT ACL */
-        nfs_user_t               nfu = {0, };
-        int                      ret = -1;
-
-        if (!frame->local) {
-                gf_log (GF_ACL, GF_LOG_ERROR, "Invalid argument,"
-                       " frame->local NULL");
-                return -EINVAL;
-        }
-        cs = frame->local;
-        getaclreply = &cs->args.getaclreply;
-        if ((op_ret < 0) && (op_errno != ENODATA && op_errno != ENOATTR)) {
-                stat = nfs3_cbk_errno_status (op_ret, op_errno);
-                goto err;
-        } else if (!dict) {
-                /* no ACL has been set */
-                stat = NFS3_OK;
-                goto err;
-        }
-
-        getaclreply->daclentry.daclentry_val = cs->daclentry;
-
-        /* getfacl: NFS DEFAULT ACL */
-        data = dict_get (dict, POSIX_ACL_DEFAULT_XATTR);
-        if (data && data->data) {
-                aclcount = acl3_nfs_acl_from_xattr (cs->daclentry,
-                                                    data->data,
-                                                    data->len,
-                                                    defacl);
-                if (aclcount < 0) {
-                        gf_log (GF_ACL, GF_LOG_ERROR,
-                                "Failed to get DEFAULT ACL");
-                        stat = nfs3_errno_to_nfsstat3 (-aclcount);
-                        goto err;
-                }
-
-                getaclreply->daclcount = aclcount;
-                getaclreply->daclentry.daclentry_len = aclcount;
-        }
-
-        getaclreply->attr_follows = TRUE;
-        nfs_request_user_init (&nfu, cs->req);
-        ret = nfs_getxattr (cs->nfsx, cs->vol, &nfu, &cs->resolvedloc,
-                            POSIX_ACL_ACCESS_XATTR, NULL, acl3_getacl_cbk, cs);
-        if (ret < 0) {
-                stat = nfs3_errno_to_nfsstat3 (-ret);
-                goto err;
-        }
-
-        return 0;
+    nfsstat3 stat = NFS3ERR_SERVERFAULT;
+    nfs3_call_state_t *cs = NULL;
+    data_t *data = NULL;
+    getaclreply *getaclreply = NULL;
+    int aclcount = 0;
+    int defacl = 1; /* DEFAULT ACL */
+    nfs_user_t nfu = {
+        0,
+    };
+    int ret = -1;
+
+    if (!frame->local) {
+        gf_msg(GF_ACL, GF_LOG_ERROR, EINVAL, NFS_MSG_INVALID_ENTRY,
+               "Invalid argument, frame->local NULL");
+        return -EINVAL;
+    }
+    cs = frame->local;
+    getaclreply = &cs->args.getaclreply;
+    if ((op_ret < 0) && (op_errno != ENODATA && op_errno != ENOATTR)) {
+        stat = nfs3_cbk_errno_status(op_ret, op_errno);
+        goto err;
+    } else if (!dict) {
+        /* no ACL has been set */
+        stat = NFS3_OK;
+        goto err;
+    }
+
+    getaclreply->daclentry.daclentry_val = cs->daclentry;
+
+    /* getfacl: NFS DEFAULT ACL */
+    data = dict_get(dict, POSIX_ACL_DEFAULT_XATTR);
+    if (data && data->data) {
+        aclcount = acl3_nfs_acl_from_xattr(cs->daclentry, data->data, data->len,
+                                           defacl);
+        if (aclcount < 0) {
+            gf_msg(GF_ACL, GF_LOG_ERROR, aclcount, NFS_MSG_GET_DEF_ACL_FAIL,
+                   "Failed to get DEFAULT ACL");
+            stat = nfs3_errno_to_nfsstat3(-aclcount);
+            goto err;
+        }
+
+        getaclreply->daclcount = aclcount;
+        getaclreply->daclentry.daclentry_len = aclcount;
+    }
+
+    getaclreply->attr_follows = TRUE;
+    nfs_request_user_init(&nfu, cs->req);
+    ret = nfs_getxattr(cs->nfsx, cs->vol, &nfu, &cs->resolvedloc,
+                       POSIX_ACL_ACCESS_XATTR, NULL, acl3_getacl_cbk, cs);
+    if (ret < 0) {
+        stat = nfs3_errno_to_nfsstat3(-ret);
+        goto err;
+    }
+
+    return 0;
 
 err:
-        if (getaclreply)
-                getaclreply->status = stat;
-        acl3_getacl_reply (cs->req, getaclreply);
-        nfs3_call_state_wipe (cs);
-        return 0;
+    if (getaclreply)
+        getaclreply->status = stat;
+    acl3_getacl_reply(cs->req, getaclreply);
+    nfs3_call_state_wipe(cs);
+    return 0;
 }
 
-
 int
-acl3_stat_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                  int32_t op_ret, int32_t op_errno, struct iatt *buf,
-                  dict_t *xdata)
+acl3_stat_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int32_t op_ret,
+              int32_t op_errno, struct iatt *buf, dict_t *xdata)
 {
-        nfsstat3                        stat = NFS3ERR_SERVERFAULT;
-        nfs3_call_state_t              *cs = NULL;
-        getaclreply                     *getaclreply = NULL;
-        int                             ret = -1;
-        nfs_user_t                      nfu = {0, };
-        uint64_t                        deviceid = 0;
-
-        if (!frame->local) {
-                gf_log (GF_ACL, GF_LOG_ERROR, "Invalid argument,"
-                       " frame->local NULL");
-                return EINVAL;
-        }
-
-        cs = frame->local;
-        getaclreply = &cs->args.getaclreply;
-
-        if (op_ret == -1) {
-                stat = nfs3_cbk_errno_status (op_ret, op_errno);
-                goto err;
-        }
-
-        /* Fill the attrs before xattrs */
-        getaclreply->attr_follows = TRUE;
-        deviceid = nfs3_request_xlator_deviceid (cs->req);
-        nfs3_map_deviceid_to_statdev (buf, deviceid);
-        nfs3_stat_to_fattr3 (buf, &(getaclreply->attr));
-
-        nfs_request_user_init (&nfu, cs->req);
-        if (buf->ia_type == IA_IFDIR) {
-                ret = nfs_getxattr (cs->nfsx, cs->vol, &nfu, &cs->resolvedloc,
-                                    POSIX_ACL_DEFAULT_XATTR, NULL,
-                                    acl3_default_getacl_cbk, cs);
-        } else {
-                ret = nfs_getxattr (cs->nfsx, cs->vol, &nfu, &cs->resolvedloc,
-                                    POSIX_ACL_ACCESS_XATTR, NULL,
-                                    acl3_getacl_cbk, cs);
-        }
-
-        if (ret < 0) {
-                stat = nfs3_errno_to_nfsstat3 (-ret);
-                goto err;
-        }
-
-        return 0;
+    nfsstat3 stat = NFS3ERR_SERVERFAULT;
+    nfs3_call_state_t *cs = NULL;
+    getaclreply *getaclreply = NULL;
+    int ret = -1;
+    nfs_user_t nfu = {
+        0,
+    };
+    uint64_t deviceid = 0;
+
+    if (!frame->local) {
+        gf_msg(GF_ACL, GF_LOG_ERROR, EINVAL, NFS_MSG_INVALID_ENTRY,
+               "Invalid argument, frame->local NULL");
+        return EINVAL;
+    }
+
+    cs = frame->local;
+    getaclreply = &cs->args.getaclreply;
+
+    if (op_ret == -1) {
+        stat = nfs3_cbk_errno_status(op_ret, op_errno);
+        goto err;
+    }
+
+    /* Fill the attrs before xattrs */
+    getaclreply->attr_follows = TRUE;
+    deviceid = nfs3_request_xlator_deviceid(cs->req);
+    nfs3_map_deviceid_to_statdev(buf, deviceid);
+    nfs3_stat_to_fattr3(buf, &(getaclreply->attr));
+
+    nfs_request_user_init(&nfu, cs->req);
+    if (buf->ia_type == IA_IFDIR) {
+        ret = nfs_getxattr(cs->nfsx, cs->vol, &nfu, &cs->resolvedloc,
+                           POSIX_ACL_DEFAULT_XATTR, NULL,
+                           acl3_default_getacl_cbk, cs);
+    } else {
+        ret = nfs_getxattr(cs->nfsx, cs->vol, &nfu, &cs->resolvedloc,
+                           POSIX_ACL_ACCESS_XATTR, NULL, acl3_getacl_cbk, cs);
+    }
+
+    if (ret < 0) {
+        stat = nfs3_errno_to_nfsstat3(-ret);
+        goto err;
+    }
+
+    return 0;
 err:
-        getaclreply->status = stat;
-        acl3_getacl_reply (cs->req, getaclreply);
-        nfs3_call_state_wipe (cs);
-        return 0;
+    getaclreply->status = stat;
+    acl3_getacl_reply(cs->req, getaclreply);
+    nfs3_call_state_wipe(cs);
+    return 0;
 }
 
-
 int
-acl3_getacl_resume (void *carg)
+acl3_getacl_resume(void *carg)
 {
-        int                             ret = -1;
-        nfs3_call_state_t               *cs = NULL;
-        nfsstat3                        stat = NFS3ERR_SERVERFAULT;
-        nfs_user_t                      nfu = {0, };
-
-        if (!carg)
-                return ret;
+    int ret = -1;
+    nfs3_call_state_t *cs = NULL;
+    nfsstat3 stat = NFS3ERR_SERVERFAULT;
+    nfs_user_t nfu = {
+        0,
+    };
+
+    if (!carg)
+        return ret;
 
-        cs = (nfs3_call_state_t *)carg;
-        acl3_check_fh_resolve_status (cs, stat, acl3err);
-        nfs_request_user_init (&nfu, cs->req);
+    cs = (nfs3_call_state_t *)carg;
+    acl3_check_fh_resolve_status(cs, stat, acl3err);
+    nfs_request_user_init(&nfu, cs->req);
 
-        ret = nfs_stat (cs->nfsx, cs->vol, &nfu, &cs->resolvedloc,
-                        acl3_stat_cbk, cs);
-        stat = -ret;
+    ret = nfs_stat(cs->nfsx, cs->vol, &nfu, &cs->resolvedloc, acl3_stat_cbk,
+                   cs);
+    stat = -ret;
 acl3err:
-        if (ret < 0) {
-                gf_log (GF_ACL, GF_LOG_ERROR, "unable to open_and_resume");
-                cs->args.getaclreply.status = nfs3_errno_to_nfsstat3 (stat);
-                acl3_getacl_reply (cs->req, &cs->args.getaclreply);
-                nfs3_call_state_wipe (cs);
-        }
-
-        return ret;
+    if (ret < 0) {
+        gf_msg(GF_ACL, GF_LOG_ERROR, stat, NFS_MSG_OPEN_FAIL,
+               "unable to open_and_resume");
+        cs->args.getaclreply.status = nfs3_errno_to_nfsstat3(stat);
+        acl3_getacl_reply(cs->req, &cs->args.getaclreply);
+        nfs3_call_state_wipe(cs);
+    }
+
+    return ret;
 }
 
-
 int
-acl3svc_getacl (rpcsvc_request_t *req)
+acl3svc_getacl(rpcsvc_request_t *req)
 {
-        xlator_t                        *vol = NULL;
-        struct nfs_state               *nfs = NULL;
-        nfs3_state_t                   *nfs3 = NULL;
-        nfs3_call_state_t               *cs = NULL;
-        int                             ret = RPCSVC_ACTOR_ERROR;
-        nfsstat3                        stat = NFS3ERR_SERVERFAULT;
-        struct nfs3_fh                  fh, *fhp = NULL;
-        getaclargs                      getaclargs;
-        getaclreply                     getaclreply;
-
-        if (!req)
-                return ret;
-
-        acl3_validate_nfs3_state (req, nfs3, stat, rpcerr, ret);
-        nfs = nfs_state (nfs3->nfsx);
-        memset (&getaclargs, 0, sizeof (getaclargs));
-        memset (&getaclreply, 0, sizeof (getaclreply));
-        getaclargs.fh.n_bytes = (char *)&fh;
-        if (xdr_to_getaclargs(req->msg[0], &getaclargs) <= 0) {
-                gf_log (GF_ACL, GF_LOG_ERROR, "Error decoding args");
-                rpcsvc_request_seterr (req, GARBAGE_ARGS);
-                goto rpcerr;
-        }
-
-        /* Validate ACL mask */
-        if (getaclargs.mask & ~(NFS_ACL|NFS_ACLCNT|NFS_DFACL|NFS_DFACLCNT)) {
-                stat = NFS3ERR_INVAL;
-                goto acl3err;
-        }
-
-        fhp = &fh;
-        acl3_validate_gluster_fh (&fh, stat, acl3err);
-        acl3_map_fh_to_volume (nfs->nfs3state, fhp, req, vol, stat, acl3err);
-        acl3_volume_started_check (nfs3, vol, ret, rpcerr);
-        acl3_handle_call_state_init (nfs->nfs3state, cs, req,
-                                     vol, stat, acl3err);
-
-        cs->vol = vol;
-        cs->args.getaclreply.mask = getaclargs.mask;
+    xlator_t *vol = NULL;
+    struct nfs_state *nfs = NULL;
+    nfs3_state_t *nfs3 = NULL;
+    nfs3_call_state_t *cs = NULL;
+    int ret = RPCSVC_ACTOR_ERROR;
+    nfsstat3 stat = NFS3ERR_SERVERFAULT;
+    struct nfs3_fh fh, *fhp = NULL;
+    getaclargs getaclargs;
+    getaclreply getaclreply;
+
+    if (!req)
+        return ret;
 
-        ret = nfs3_fh_resolve_and_resume (cs, fhp, NULL, acl3_getacl_resume);
-        stat = nfs3_errno_to_nfsstat3 (-ret);
+    acl3_validate_nfs3_state(req, nfs3, stat, rpcerr, ret);
+    nfs = nfs_state(nfs3->nfsx);
+    memset(&getaclargs, 0, sizeof(getaclargs));
+    memset(&getaclreply, 0, sizeof(getaclreply));
+    getaclargs.fh.n_bytes = (char *)&fh;
+    if (xdr_to_getaclargs(req->msg[0], &getaclargs) <= 0) {
+        gf_msg(GF_ACL, GF_LOG_ERROR, errno, NFS_MSG_ARGS_DECODE_ERROR,
+               "Error decoding args");
+        rpcsvc_request_seterr(req, GARBAGE_ARGS);
+        goto rpcerr;
+    }
+
+    /* Validate ACL mask */
+    if (getaclargs.mask & ~(NFS_ACL | NFS_ACLCNT | NFS_DFACL | NFS_DFACLCNT)) {
+        stat = NFS3ERR_INVAL;
+        goto acl3err;
+    }
+
+    fhp = &fh;
+    acl3_validate_gluster_fh(&fh, stat, acl3err);
+    acl3_map_fh_to_volume(nfs->nfs3state, fhp, req, vol, stat, acl3err);
+    acl3_volume_started_check(nfs3, vol, ret, rpcerr);
+    acl3_handle_call_state_init(nfs->nfs3state, cs, req, vol, stat, acl3err);
+
+    cs->vol = vol;
+    cs->args.getaclreply.mask = getaclargs.mask;
+
+    ret = nfs3_fh_resolve_and_resume(cs, fhp, NULL, acl3_getacl_resume);
+    stat = nfs3_errno_to_nfsstat3(-ret);
 
 acl3err:
-        if (ret < 0) {
-                gf_log (GF_ACL, GF_LOG_ERROR, "unable to resolve and resume");
-                getaclreply.status = stat;
-                acl3_getacl_reply (req, &getaclreply);
-                nfs3_call_state_wipe (cs);
-                return 0;
-        }
+    if (ret < 0) {
+        gf_msg(GF_ACL, GF_LOG_ERROR, -ret, NFS_MSG_RESOLVE_ERROR,
+               "unable to resolve and resume");
+        getaclreply.status = stat;
+        acl3_getacl_reply(req, &getaclreply);
+        nfs3_call_state_wipe(cs);
+        return 0;
+    }
 
 rpcerr:
-        return ret;
+    return ret;
 }
 
 int
-acl3_setacl_cbk (call_frame_t *frame, void *cookie,
-                 xlator_t *this, int32_t op_ret, int32_t op_errno,
-                 dict_t *xdata)
+acl3_setacl_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
-        nfs3_call_state_t               *cs = NULL;
-        cs = frame->local;
-        if (op_ret < 0) {
-                nfsstat3 status = nfs3_cbk_errno_status (op_ret, op_errno);
-                cs->args.setaclreply.status = status;
-        }
+    nfs3_call_state_t *cs = NULL;
+    cs = frame->local;
+    if (op_ret < 0) {
+        nfsstat3 status = nfs3_cbk_errno_status(op_ret, op_errno);
+        cs->args.setaclreply.status = status;
+    }
 
-        acl3_setacl_reply (cs->req, &cs->args.setaclreply);
+    acl3_setacl_reply(cs->req, &cs->args.setaclreply);
 
-        nfs3_call_state_wipe (cs);
+    nfs3_call_state_wipe(cs);
 
-        return 0;
+    return 0;
 }
 
 int
-acl3_setacl_resume (void *carg)
+acl3_setacl_resume(void *carg)
 {
-        int                             ret = -1;
-        nfs3_call_state_t               *cs = NULL;
-        nfsstat3                        stat = NFS3ERR_SERVERFAULT;
-        nfs_user_t                      nfu = {0, };
-        dict_t                          *xattr = NULL;
-
-        if (!carg)
-                return ret;
-        cs = (nfs3_call_state_t *)carg;
-        acl3_check_fh_resolve_status (cs, stat, acl3err);
-        nfs_request_user_init (&nfu, cs->req);
-        xattr = dict_new();
-        if (cs->aclcount)
-        ret = dict_set_static_bin (xattr, POSIX_ACL_ACCESS_XATTR,
-                                   cs->aclxattr,
-                                   posix_acl_xattr_size (cs->aclcount));
-        if (cs->daclcount)
-        ret = dict_set_static_bin (xattr, POSIX_ACL_DEFAULT_XATTR,
-                                   cs->daclxattr,
-                                   posix_acl_xattr_size (cs->daclcount));
-
-        ret = nfs_setxattr (cs->nfsx, cs->vol, &nfu, &cs->resolvedloc, xattr,
-                            0, NULL, acl3_setacl_cbk, cs);
-        dict_unref (xattr);
+    int ret = -1;
+    nfs3_call_state_t *cs = NULL;
+    nfsstat3 stat = NFS3ERR_SERVERFAULT;
+    nfs_user_t nfu = {
+        0,
+    };
+    dict_t *xattr = NULL;
+
+    if (!carg)
+        return ret;
+    cs = (nfs3_call_state_t *)carg;
+    acl3_check_fh_resolve_status(cs, stat, acl3err);
+    nfs_request_user_init(&nfu, cs->req);
+    xattr = dict_new();
+    if (xattr == NULL) {
+        gf_msg(GF_NLM, GF_LOG_ERROR, ENOMEM, NFS_MSG_GFID_DICT_CREATE_FAIL,
+               "dict allocation failed");
+        goto acl3err;
+    }
+
+    if (cs->aclcount)
+        ret = dict_set_static_bin(xattr, POSIX_ACL_ACCESS_XATTR, cs->aclxattr,
+                                  posix_acl_xattr_size(cs->aclcount));
+    if (cs->daclcount)
+        ret = dict_set_static_bin(xattr, POSIX_ACL_DEFAULT_XATTR, cs->daclxattr,
+                                  posix_acl_xattr_size(cs->daclcount));
+
+    ret = nfs_setxattr(cs->nfsx, cs->vol, &nfu, &cs->resolvedloc, xattr, 0,
+                       NULL, acl3_setacl_cbk, cs);
+    dict_unref(xattr);
 
 acl3err:
-        if (ret < 0) {
-                stat = -ret;
-                gf_log (GF_ACL, GF_LOG_ERROR, "unable to open_and_resume");
-                cs->args.setaclreply.status = nfs3_errno_to_nfsstat3 (stat);
-                acl3_setacl_reply (cs->req, &cs->args.setaclreply);
-                nfs3_call_state_wipe (cs);
-        }
-
-        return ret;
+    if (ret < 0) {
+        stat = -ret;
+        gf_msg(GF_ACL, GF_LOG_ERROR, stat, NFS_MSG_OPEN_FAIL,
+               "unable to open_and_resume");
+        cs->args.setaclreply.status = nfs3_errno_to_nfsstat3(stat);
+        acl3_setacl_reply(cs->req, &cs->args.setaclreply);
+        nfs3_call_state_wipe(cs);
+    }
+
+    return ret;
 }
 
-
 int
-acl3svc_setacl (rpcsvc_request_t *req)
+acl3svc_setacl(rpcsvc_request_t *req)
 {
-        xlator_t                       *vol = NULL;
-        struct nfs_state               *nfs = NULL;
-        nfs3_state_t                   *nfs3 = NULL;
-        nfs3_call_state_t              *cs = NULL;
-        int                            ret = RPCSVC_ACTOR_ERROR;
-        nfsstat3                       stat = NFS3ERR_SERVERFAULT;
-        struct nfs3_fh                 fh;
-        struct nfs3_fh                 *fhp = NULL;
-        setaclargs                     setaclargs;
-        setaclreply                    setaclreply;
-        aclentry                       *daclentry = NULL;
-        aclentry                       *aclentry = NULL;
-        int                            aclerrno = 0;
-        int                            defacl = 1;
-
-        if (!req)
-                return ret;
-        aclentry =  GF_CALLOC (NFS_ACL_MAX_ENTRIES, sizeof(*aclentry),
-                               gf_nfs_mt_arr);
-        if (!aclentry) {
-                goto rpcerr;
-        }
-        daclentry = GF_CALLOC (NFS_ACL_MAX_ENTRIES, sizeof(*daclentry),
-                               gf_nfs_mt_arr);
-        if (!daclentry) {
-                goto rpcerr;
-        }
-
-        acl3_validate_nfs3_state (req, nfs3, stat, rpcerr, ret);
-        nfs = nfs_state (nfs3->nfsx);
-        memset (&setaclargs, 0, sizeof (setaclargs));
-        memset (&setaclreply, 0, sizeof (setaclreply));
-        memset (&fh, 0, sizeof (fh));
-        setaclargs.fh.n_bytes = (char *)&fh;
-        setaclargs.aclentry.aclentry_val = aclentry;
-        setaclargs.daclentry.daclentry_val = daclentry;
-        if (xdr_to_setaclargs(req->msg[0], &setaclargs) <= 0) {
-                gf_log (GF_ACL, GF_LOG_ERROR, "Error decoding args");
-                rpcsvc_request_seterr (req, GARBAGE_ARGS);
-                goto rpcerr;
-        }
-
-        /* Validate ACL mask */
-        if (setaclargs.mask & ~(NFS_ACL|NFS_ACLCNT|NFS_DFACL|NFS_DFACLCNT)) {
-                stat = NFS3ERR_INVAL;
-                goto acl3err;
-        }
-
-        fhp = &fh;
-        acl3_validate_gluster_fh (fhp, stat, acl3err);
-        acl3_map_fh_to_volume (nfs->nfs3state, fhp, req, vol, stat, acl3err);
-        acl3_volume_started_check (nfs3, vol, ret, rpcerr);
-        acl3_handle_call_state_init (nfs->nfs3state, cs, req,
-                                     vol, stat, acl3err);
-
-        cs->vol = vol;
-        cs->aclcount = setaclargs.aclcount;
-        cs->daclcount = setaclargs.daclcount;
-
-        /* setfacl: NFS USER ACL */
-        aclerrno = acl3_nfs_acl_to_xattr (aclentry,
-                                          cs->aclxattr,
-                                          cs->aclcount,
-                                          !defacl);
-        if (aclerrno < 0) {
-                gf_log (GF_ACL, GF_LOG_ERROR, "Failed to set USER ACL");
-                stat = nfs3_errno_to_nfsstat3 (-aclerrno);
-                goto acl3err;
-        }
-
-        /* setfacl: NFS DEFAULT ACL */
-        aclerrno = acl3_nfs_acl_to_xattr (daclentry,
-                                          cs->daclxattr,
-                                          cs->daclcount,
-                                          defacl);
-        if (aclerrno < 0) {
-                gf_log (GF_ACL, GF_LOG_ERROR, "Failed to set DEFAULT ACL");
-                stat = nfs3_errno_to_nfsstat3 (-aclerrno);
-                goto acl3err;
-        }
-
-        ret = nfs3_fh_resolve_and_resume (cs, fhp, NULL, acl3_setacl_resume);
-        stat = nfs3_errno_to_nfsstat3 (-ret);
+    xlator_t *vol = NULL;
+    struct nfs_state *nfs = NULL;
+    nfs3_state_t *nfs3 = NULL;
+    nfs3_call_state_t *cs = NULL;
+    int ret = RPCSVC_ACTOR_ERROR;
+    nfsstat3 stat = NFS3ERR_SERVERFAULT;
+    struct nfs3_fh fh;
+    struct nfs3_fh *fhp = NULL;
+    setaclargs setaclargs;
+    setaclreply setaclreply;
+    aclentry *daclentry = NULL;
+    aclentry *aclentry = NULL;
+    int aclerrno = 0;
+    int defacl = 1;
+
+    if (!req)
+        return ret;
+    aclentry = GF_CALLOC(NFS_ACL_MAX_ENTRIES, sizeof(*aclentry), gf_nfs_mt_arr);
+    if (!aclentry) {
+        goto rpcerr;
+    }
+    daclentry = GF_CALLOC(NFS_ACL_MAX_ENTRIES, sizeof(*daclentry),
+                          gf_nfs_mt_arr);
+    if (!daclentry) {
+        goto rpcerr;
+    }
+
+    acl3_validate_nfs3_state(req, nfs3, stat, rpcerr, ret);
+    nfs = nfs_state(nfs3->nfsx);
+    memset(&setaclargs, 0, sizeof(setaclargs));
+    memset(&setaclreply, 0, sizeof(setaclreply));
+    memset(&fh, 0, sizeof(fh));
+    setaclargs.fh.n_bytes = (char *)&fh;
+    setaclargs.aclentry.aclentry_val = aclentry;
+    setaclargs.daclentry.daclentry_val = daclentry;
+    if (xdr_to_setaclargs(req->msg[0], &setaclargs) <= 0) {
+        gf_msg(GF_ACL, GF_LOG_ERROR, errno, NFS_MSG_ARGS_DECODE_ERROR,
+               "Error decoding args");
+        rpcsvc_request_seterr(req, GARBAGE_ARGS);
+        goto rpcerr;
+    }
+
+    /* Validate ACL mask */
+    if (setaclargs.mask & ~(NFS_ACL | NFS_ACLCNT | NFS_DFACL | NFS_DFACLCNT)) {
+        stat = NFS3ERR_INVAL;
+        goto acl3err;
+    }
+
+    fhp = &fh;
+    acl3_validate_gluster_fh(fhp, stat, acl3err);
+    acl3_map_fh_to_volume(nfs->nfs3state, fhp, req, vol, stat, acl3err);
+    acl3_volume_started_check(nfs3, vol, ret, rpcerr);
+    acl3_handle_call_state_init(nfs->nfs3state, cs, req, vol, stat, acl3err);
+
+    cs->vol = vol;
+    cs->aclcount = setaclargs.aclcount;
+    cs->daclcount = setaclargs.daclcount;
+
+    /* setfacl: NFS USER ACL */
+    aclerrno = acl3_nfs_acl_to_xattr(aclentry, cs->aclxattr, cs->aclcount,
+                                     !defacl);
+    if (aclerrno < 0) {
+        gf_msg(GF_ACL, GF_LOG_ERROR, -aclerrno, NFS_MSG_SET_USER_ACL_FAIL,
+               "Failed to set USER ACL");
+        stat = nfs3_errno_to_nfsstat3(-aclerrno);
+        goto acl3err;
+    }
+
+    /* setfacl: NFS DEFAULT ACL */
+    aclerrno = acl3_nfs_acl_to_xattr(daclentry, cs->daclxattr, cs->daclcount,
+                                     defacl);
+    if (aclerrno < 0) {
+        gf_msg(GF_ACL, GF_LOG_ERROR, -aclerrno, NFS_MSG_SET_DEF_ACL_FAIL,
+               "Failed to set DEFAULT ACL");
+        stat = nfs3_errno_to_nfsstat3(-aclerrno);
+        goto acl3err;
+    }
+
+    ret = nfs3_fh_resolve_and_resume(cs, fhp, NULL, acl3_setacl_resume);
+    stat = nfs3_errno_to_nfsstat3(-ret);
 
 acl3err:
-        if (ret < 0) {
-                gf_log (GF_ACL, GF_LOG_ERROR, "unable to resolve and resume");
-                setaclreply.status = stat;
-                acl3_setacl_reply (req, &setaclreply);
-                nfs3_call_state_wipe (cs);
-                GF_FREE(aclentry);
-                GF_FREE(daclentry);
-                return 0;
-        }
+    if (ret < 0) {
+        gf_msg(GF_ACL, GF_LOG_ERROR, -ret, NFS_MSG_RESOLVE_ERROR,
+               "unable to resolve and resume");
+        setaclreply.status = stat;
+        acl3_setacl_reply(req, &setaclreply);
+        nfs3_call_state_wipe(cs);
+        GF_FREE(aclentry);
+        GF_FREE(daclentry);
+        return 0;
+    }
 
 rpcerr:
-        if (ret < 0)
-                nfs3_call_state_wipe (cs);
-        if (aclentry)
-                GF_FREE (aclentry);
-        if (daclentry)
-                GF_FREE (daclentry);
-        return ret;
+    if (ret < 0)
+        nfs3_call_state_wipe(cs);
+    if (aclentry)
+        GF_FREE(aclentry);
+    if (daclentry)
+        GF_FREE(daclentry);
+    return ret;
 }
 
-
-
-rpcsvc_actor_t  acl3svc_actors[ACL3_PROC_COUNT] = {
-        {"NULL",       ACL3_NULL,      acl3svc_null,   NULL,   0, DRC_NA},
-        {"GETACL",     ACL3_GETACL,    acl3svc_getacl, NULL,   0, DRC_NA},
-        {"SETACL",     ACL3_SETACL,    acl3svc_setacl, NULL,   0, DRC_NA},
+static rpcsvc_actor_t acl3svc_actors[ACL3_PROC_COUNT] = {
+    {"NULL", acl3svc_null, NULL, ACL3_NULL, DRC_NA, 0},
+    {"GETACL", acl3svc_getacl, NULL, ACL3_GETACL, DRC_NA, 0},
+    {"SETACL", acl3svc_setacl, NULL, ACL3_SETACL, DRC_NA, 0},
 };
 
-rpcsvc_program_t        acl3prog = {
-        .progname       = "ACL3",
-        .prognum        = ACL_PROGRAM,
-        .progver        = ACLV3_VERSION,
-        .progport       = GF_NFS3_PORT,
-        .actors         = acl3svc_actors,
-        .numactors      = ACL3_PROC_COUNT,
-        .min_auth       = AUTH_NULL,
+static rpcsvc_program_t acl3prog = {
+    .progname = "ACL3",
+    .prognum = ACL_PROGRAM,
+    .progver = ACLV3_VERSION,
+    .progport = GF_NFS3_PORT,
+    .actors = acl3svc_actors,
+    .numactors = ACL3_PROC_COUNT,
+    .min_auth = AUTH_NULL,
 };
 
 rpcsvc_program_t *
 acl3svc_init(xlator_t *nfsx)
 {
-        struct nfs3_state *ns = NULL;
-        struct nfs_state *nfs = NULL;
-        dict_t *options = NULL;
-        int ret = -1;
-        char *portstr = NULL;
-        static gf_boolean_t acl3_inited = _gf_false;
-
-        /* Already inited */
-        if (acl3_inited)
-                return &acl3prog;
-
-        nfs = (struct nfs_state*)nfsx->private;
-
-        ns = nfs->nfs3state;
-        if (!ns) {
-                gf_log (GF_ACL, GF_LOG_ERROR, "ACL3 init failed");
-                goto err;
-        }
-        acl3prog.private = ns;
-
-        options = dict_new ();
-
-        ret = gf_asprintf (&portstr, "%d", GF_ACL3_PORT);
-        if (ret == -1)
-                goto err;
+    struct nfs3_state *ns = NULL;
+    struct nfs_state *nfs = NULL;
+    dict_t *options = NULL;
+    int ret = -1;
+    static gf_boolean_t acl3_inited = _gf_false;
+
+    /* Already inited */
+    if (acl3_inited)
+        return &acl3prog;
 
-        ret = dict_set_dynstr (options, "transport.socket.listen-port",
-                               portstr);
-        if (ret == -1)
-                goto err;
-        ret = dict_set_str (options, "transport-type", "socket");
+    nfs = (struct nfs_state *)nfsx->private;
+
+    ns = nfs->nfs3state;
+    if (!ns) {
+        gf_msg(GF_ACL, GF_LOG_ERROR, EINVAL, NFS_MSG_ACL_INIT_FAIL,
+               "ACL3 init failed");
+        goto err;
+    }
+    acl3prog.private = ns;
+
+    options = dict_new();
+    if (options == NULL) {
+        gf_msg(GF_ACL, GF_LOG_ERROR, ENOMEM, NFS_MSG_GFID_DICT_CREATE_FAIL,
+               "dict allocation failed");
+        goto err;
+    }
+
+    ret = dict_set_str(options, "transport.socket.listen-port", GF_ACL3_PORT);
+    if (ret == -1)
+        goto err;
+    ret = dict_set_str(options, "transport-type", "socket");
+    if (ret == -1) {
+        gf_msg(GF_ACL, GF_LOG_ERROR, errno, NFS_MSG_DICT_SET_FAILED,
+               "dict_set_str error");
+        goto err;
+    }
+
+    if (nfs->allow_insecure) {
+        ret = dict_set_str(options, "rpc-auth-allow-insecure", "on");
         if (ret == -1) {
-                gf_log (GF_ACL, GF_LOG_ERROR, "dict_set_str error");
-                goto err;
-        }
-
-        if (nfs->allow_insecure) {
-                ret = dict_set_str (options, "rpc-auth-allow-insecure", "on");
-                if (ret == -1) {
-                        gf_log (GF_ACL, GF_LOG_ERROR, "dict_set_str error");
-                        goto err;
-                }
-                ret = dict_set_str (options, "rpc-auth.ports.insecure", "on");
-                if (ret == -1) {
-                        gf_log (GF_ACL, GF_LOG_ERROR, "dict_set_str error");
-                        goto err;
-                }
+            gf_msg(GF_ACL, GF_LOG_ERROR, errno, NFS_MSG_DICT_SET_FAILED,
+                   "dict_set_str error");
+            goto err;
         }
-
-        ret = dict_set_str (options, "transport.address-family", "inet");
+        ret = dict_set_str(options, "rpc-auth.ports.insecure", "on");
         if (ret == -1) {
-                gf_log (GF_ACL, GF_LOG_ERROR, "dict_set_str error");
-                goto err;
-        }
-
-        ret = rpcsvc_create_listeners (nfs->rpcsvc, options, "ACL");
-        if (ret == -1) {
-                gf_log (GF_ACL, GF_LOG_ERROR, "Unable to create listeners");
-                dict_unref (options);
-                goto err;
-        }
-
-        acl3_inited = _gf_true;
-        return &acl3prog;
+            gf_msg(GF_ACL, GF_LOG_ERROR, errno, NFS_MSG_DICT_SET_FAILED,
+                   "dict_set_str error");
+            goto err;
+        }
+    }
+
+    ret = dict_set_str(options, "transport.address-family", "inet");
+    if (ret == -1) {
+        gf_msg(GF_ACL, GF_LOG_ERROR, errno, NFS_MSG_DICT_SET_FAILED,
+               "dict_set_str error");
+        goto err;
+    }
+
+    ret = rpcsvc_create_listeners(nfs->rpcsvc, options, "ACL");
+    if (ret == -1) {
+        gf_msg(GF_ACL, GF_LOG_ERROR, errno, NFS_MSG_LISTENERS_CREATE_FAIL,
+               "Unable to create listeners");
+        goto err;
+    }
+
+    if (options)
+        dict_unref(options);
+
+    acl3_inited = _gf_true;
+    return &acl3prog;
 err:
-        return NULL;
+    if (options)
+        dict_unref(options);
+    return NULL;
 }
 
 static int
-acl3_nfs_acl_to_xattr (aclentry *ace,      /* ACL entries to be read */
-                       void     *xattrbuf, /* XATTR buf to be populated */
-                       int      aclcount,  /* No of ACLs to be read */
-                       int      defacl)    /* 1 if DEFAULT ACL */
+acl3_nfs_acl_to_xattr(aclentry *ace,  /* ACL entries to be read */
+                      void *xattrbuf, /* XATTR buf to be populated */
+                      int aclcount,   /* No of ACLs to be read */
+                      int defacl)     /* 1 if DEFAULT ACL */
 {
-        int                       idx      = 0;
-        posix_acl_xattr_header    *xheader = NULL;
-        posix_acl_xattr_entry     *xentry  = NULL;
+    int idx = 0;
+    posix_acl_xattr_header *xheader = NULL;
+    posix_acl_xattr_entry *xentry = NULL;
 
-        if ((!ace) || (!xattrbuf))
-                return (-EINVAL);
+    if ((!ace) || (!xattrbuf))
+        return (-EINVAL);
 
-        /* ACL count is ZERO, nothing to do */
-        if (!aclcount)
-                return (0);
+    /* ACL count is ZERO, nothing to do */
+    if (!aclcount)
+        return (0);
 
-        if ((aclcount < 0) || (aclcount > NFS_ACL_MAX_ENTRIES))
+    if ((aclcount < 0) || (aclcount > NFS_ACL_MAX_ENTRIES))
+        return (-EINVAL);
+
+    xheader = (posix_acl_xattr_header *)(xattrbuf);
+    xentry = (posix_acl_xattr_entry *)(xheader + 1);
+
+    /*
+     * For "default ACL", NFSv3 handles the 'type' differently
+     * i.e. by logical OR'ing 'type' with NFS_ACL_DEFAULT.
+     * Which the backend File system does not understand and
+     * that needs to be masked OFF.
+     */
+    xheader->version = POSIX_ACL_XATTR_VERSION;
+
+    for (idx = 0; idx < aclcount; idx++) {
+        xentry->tag = ace->type;
+        if (defacl)
+            xentry->tag &= ~NFS_ACL_DEFAULT;
+        xentry->perm = ace->perm;
+
+        switch (xentry->tag) {
+            case POSIX_ACL_USER:
+            case POSIX_ACL_GROUP:
+                if (xentry->perm & ~S_IRWXO)
+                    return (-EINVAL);
+                xentry->id = ace->uid;
+                break;
+            case POSIX_ACL_USER_OBJ:
+            case POSIX_ACL_GROUP_OBJ:
+            case POSIX_ACL_OTHER:
+                if (xentry->perm & ~S_IRWXO)
+                    return (-EINVAL);
+                xentry->id = POSIX_ACL_UNDEFINED_ID;
+                break;
+            case POSIX_ACL_MASK:
+                /* Solaris sometimes sets additional bits in
+                 * the mask.
+                 */
+                xentry->perm &= S_IRWXO;
+                xentry->id = POSIX_ACL_UNDEFINED_ID;
+                break;
+            default:
                 return (-EINVAL);
-
-        xheader = (posix_acl_xattr_header *) (xattrbuf);
-        xentry  = (posix_acl_xattr_entry  *) (xheader + 1);
-
-        /*
-         * For "default ACL", NFSv3 handles the 'type' differently
-         * i.e. by logical OR'ing 'type' with NFS_ACL_DEFAULT.
-         * Which the backend File system does not understand and
-         * that needs to be masked OFF.
-         */
-        xheader->version = POSIX_ACL_XATTR_VERSION;
-
-        for (idx = 0; idx < aclcount; idx++) {
-                xentry->tag  = ace->type;
-                if (defacl)
-                        xentry->tag &= ~NFS_ACL_DEFAULT;
-                xentry->perm = ace->perm;
-
-                switch (xentry->tag) {
-                case POSIX_ACL_USER:
-                case POSIX_ACL_GROUP:
-                        if (xentry->perm & ~S_IRWXO)
-                                return (-EINVAL);
-                        xentry->id = ace->uid;
-                        break;
-                case POSIX_ACL_USER_OBJ:
-                case POSIX_ACL_GROUP_OBJ:
-                case POSIX_ACL_OTHER:
-                        if (xentry->perm & ~S_IRWXO)
-                                return (-EINVAL);
-                        xentry->id = POSIX_ACL_UNDEFINED_ID;
-                        break;
-                case POSIX_ACL_MASK:
-                        /* Solaris sometimes sets additional bits in
-                         * the mask.
-                         */
-                        xentry->perm &= S_IRWXO;
-                        xentry->id = POSIX_ACL_UNDEFINED_ID;
-                        break;
-                default:
-                        return (-EINVAL);
-                }
-
-                xentry++;
-                ace++;
         }
 
-        /* SUCCESS */
-        return (0);
+        xentry++;
+        ace++;
+    }
+
+    /* SUCCESS */
+    return (0);
 }
 
 static int
-acl3_nfs_acl_from_xattr (aclentry  *ace,      /* ACL entries to be filled */
-                         void      *xattrbuf, /* XATTR buf to be read */
-                         int       bufsize,   /* Size of XATTR buffer */
-                         int       defacl)    /*  1 if DEFAULT ACL */
+acl3_nfs_acl_from_xattr(aclentry *ace,  /* ACL entries to be filled */
+                        void *xattrbuf, /* XATTR buf to be read */
+                        int bufsize,    /* Size of XATTR buffer */
+                        int defacl)     /*  1 if DEFAULT ACL */
 {
-        int                       idx       = 0;
-        ssize_t                   aclcount  = 0;
-        posix_acl_xattr_header    *xheader  = NULL;
-        posix_acl_xattr_entry     *xentry   = NULL;
-
-        if ((!xattrbuf) || (!ace))
+    int idx = 0;
+    ssize_t aclcount = 0;
+    posix_acl_xattr_header *xheader = NULL;
+    posix_acl_xattr_entry *xentry = NULL;
+
+    if ((!xattrbuf) || (!ace))
+        return (-EINVAL);
+
+    aclcount = posix_acl_xattr_count(bufsize);
+    if ((aclcount < 0) || (aclcount > NFS_ACL_MAX_ENTRIES))
+        return (-EINVAL);
+
+    xheader = (posix_acl_xattr_header *)(xattrbuf);
+    xentry = (posix_acl_xattr_entry *)(xheader + 1);
+
+    /* Check for supported POSIX ACL xattr version */
+    if (xheader->version != POSIX_ACL_XATTR_VERSION)
+        return (-ENOSYS);
+
+    for (idx = 0; idx < (int)aclcount; idx++) {
+        ace->type = xentry->tag;
+        if (defacl) {
+            /*
+             * SET the NFS_ACL_DEFAULT flag for default
+             * ACL which was masked OFF during setfacl().
+             */
+            ace->type |= NFS_ACL_DEFAULT;
+        }
+        ace->perm = (xentry->perm & S_IRWXO);
+
+        switch (xentry->tag) {
+            case POSIX_ACL_USER:
+            case POSIX_ACL_GROUP:
+                ace->uid = xentry->id;
+                break;
+            case POSIX_ACL_USER_OBJ:
+            case POSIX_ACL_GROUP_OBJ:
+            case POSIX_ACL_MASK:
+            case POSIX_ACL_OTHER:
+                ace->uid = POSIX_ACL_UNDEFINED_ID;
+                break;
+            default:
                 return (-EINVAL);
-
-        aclcount = posix_acl_xattr_count (bufsize);
-        if ((aclcount < 0) || (aclcount > NFS_ACL_MAX_ENTRIES))
-                return (-EINVAL);
-
-        xheader = (posix_acl_xattr_header *) (xattrbuf);
-        xentry  = (posix_acl_xattr_entry  *) (xheader + 1);
-
-        /* Check for supported POSIX ACL xattr version */
-        if (xheader->version != POSIX_ACL_XATTR_VERSION)
-                return (-ENOSYS);
-
-        for (idx = 0; idx < (int)aclcount; idx++) {
-                ace->type = xentry->tag;
-                if (defacl) {
-                        /*
-                         * SET the NFS_ACL_DEFAULT flag for default
-                         * ACL which was masked OFF during setfacl().
-                         */
-                        ace->type |= NFS_ACL_DEFAULT;
-                }
-                ace->perm = (xentry->perm & S_IRWXO);
-
-                switch (xentry->tag) {
-                case POSIX_ACL_USER:
-                case POSIX_ACL_GROUP:
-                        ace->uid = xentry->id;
-                        break;
-                case POSIX_ACL_USER_OBJ:
-                case POSIX_ACL_GROUP_OBJ:
-                case POSIX_ACL_MASK:
-                case POSIX_ACL_OTHER:
-                        ace->uid = POSIX_ACL_UNDEFINED_ID;
-                        break;
-                default:
-                        return (-EINVAL);
-                }
-
-
-                xentry++;
-                ace++;
         }
 
-        /* SUCCESS: ACL count */
-        return aclcount;
+        xentry++;
+        ace++;
+    }
+
+    /* SUCCESS: ACL count */
+    return aclcount;
 }
diff --git a/xlators/nfs/server/src/acl3.h b/xlators/nfs/server/src/acl3.h
index 3ccc587bd59..762fbb04a0f 100644
--- a/xlators/nfs/server/src/acl3.h
+++ b/xlators/nfs/server/src/acl3.h
@@ -11,30 +11,30 @@
 #ifndef _ACL3_H
 #define _ACL3_H
 
-#include "glusterfs-acl.h"
+#include <glusterfs/glusterfs-acl.h>
 
 #define ACL3_NULL 0
 #define ACL3_GETACL 1
 #define ACL3_SETACL 2
 #define ACL3_PROC_COUNT 3
 
-#define GF_ACL3_PORT            38469
-#define GF_ACL                  GF_NFS"-ACL"
+#define GF_ACL3_PORT "38469"
+#define GF_ACL GF_NFS "-ACL"
 
 /* Flags for the getacl/setacl mode */
-#define NFS_ACL                 0x0001
-#define NFS_ACLCNT              0x0002
-#define NFS_DFACL               0x0004
-#define NFS_DFACLCNT            0x0008
+#define NFS_ACL 0x0001
+#define NFS_ACLCNT 0x0002
+#define NFS_DFACL 0x0004
+#define NFS_DFACLCNT 0x0008
 
 /*
  * NFSv3, identifies the default ACL by NFS_ACL_DEFAULT. Gluster
  * NFS needs to mask it OFF before sending it up to POSIX layer
  * or File system layer.
  */
-#define NFS_ACL_DEFAULT             0x1000
+#define NFS_ACL_DEFAULT 0x1000
 
-#define NFS_ACL_MAX_ENTRIES         1024
+#define NFS_ACL_MAX_ENTRIES 1024
 
 rpcsvc_program_t *
 acl3svc_init(xlator_t *nfsx);
diff --git a/xlators/nfs/server/src/auth-cache.c b/xlators/nfs/server/src/auth-cache.c
index 01ebaed04d2..ffbf5b6cad6 100644
--- a/xlators/nfs/server/src/auth-cache.c
+++ b/xlators/nfs/server/src/auth-cache.c
@@ -11,40 +11,51 @@
    cases as published by the Free Software Foundation.
 */
 
+#include <glusterfs/refcount.h>
 #include "auth-cache.h"
 #include "nfs3.h"
 #include "exports.h"
+#include "nfs-messages.h"
 
 enum auth_cache_lookup_results {
-        ENTRY_FOUND     =  0,
-        ENTRY_NOT_FOUND = -1,
-        ENTRY_EXPIRED   = -2,
+    ENTRY_FOUND = 0,
+    ENTRY_NOT_FOUND = -1,
+    ENTRY_EXPIRED = -2,
 };
 
 struct auth_cache_entry {
-        time_t timestamp;
-        struct export_item *item;
+    GF_REF_DECL;  /* refcounting */
+    data_t *data; /* data_unref() on refcount == 0 */
+
+    time_t timestamp;
+    struct export_item *item;
 };
 
-/* Given a filehandle and an ip, creates a colon delimited hashkey that is
- * allocated on the stack.
+/* Given a filehandle and an ip, creates a colon delimited hashkey.
  */
-static inline void
-make_hashkey(char *hashkey, struct nfs3_fh *fh, const char *host)
+static char *
+make_hashkey(struct nfs3_fh *fh, const char *host)
 {
-        char    exportid[256]    = {0, };
-        char    gfid[256]        = {0, };
-        char    mountid[256]     = {0, };
-        size_t  nbytes           = 0;
-
-        gf_uuid_unparse (fh->exportid, exportid);
-        gf_uuid_unparse (fh->gfid, gfid);
-        gf_uuid_unparse (fh->mountid, mountid);
-        nbytes = strlen (exportid) + strlen (gfid) + strlen (host)
-                 + strlen (mountid) + 5;
-        hashkey = alloca (nbytes);
-        snprintf (hashkey, nbytes, "%s:%s:%s:%s", exportid, gfid,
-                  mountid, host);
+    char *hashkey = NULL;
+    char exportid[256] = {
+        0,
+    };
+    char mountid[256] = {
+        0,
+    };
+    size_t nbytes = 0;
+
+    gf_uuid_unparse(fh->exportid, exportid);
+    gf_uuid_unparse(fh->mountid, mountid);
+
+    nbytes = strlen(exportid) + strlen(host) + strlen(mountid) + 3;
+    hashkey = GF_MALLOC(nbytes, gf_common_mt_char);
+    if (!hashkey)
+        return NULL;
+
+    snprintf(hashkey, nbytes, "%s:%s:%s", exportid, mountid, host);
+
+    return hashkey;
 }
 
 /**
@@ -55,16 +66,47 @@ make_hashkey(char *hashkey, struct nfs3_fh *fh, const char *host)
  * @return : allocated auth cache struct, NULL if allocation failed.
  */
 struct auth_cache *
-auth_cache_init (time_t ttl_sec)
+auth_cache_init(time_t ttl_sec)
+{
+    struct auth_cache *cache = GF_CALLOC(1, sizeof(*cache),
+                                         gf_nfs_mt_auth_cache);
+
+    GF_VALIDATE_OR_GOTO("auth-cache", cache, out);
+
+    cache->cache_dict = dict_new();
+    if (!cache->cache_dict) {
+        GF_FREE(cache);
+        cache = NULL;
+        goto out;
+    }
+
+    LOCK_INIT(&cache->lock);
+    cache->ttl_sec = ttl_sec;
+out:
+    return cache;
+}
+
+/* auth_cache_entry_free -- called by refcounting subsystem on refcount == 0
+ *
+ * @to_free: auth_cache_entry that has refcount == 0 and needs to get free'd
+ */
+void
+auth_cache_entry_free(void *to_free)
 {
-        struct auth_cache *cache = GF_CALLOC (1, sizeof (*cache),
-                                              gf_nfs_mt_auth_cache);
+    struct auth_cache_entry *entry = to_free;
+    data_t *entry_data = NULL;
+
+    GF_VALIDATE_OR_GOTO(GF_NFS, entry, out);
+    GF_VALIDATE_OR_GOTO(GF_NFS, entry->data, out);
 
-        GF_VALIDATE_OR_GOTO ("auth-cache", cache, out);
+    entry_data = entry->data;
+    /* set data_t->data to NULL, otherwise data_unref() tries to free it */
+    entry_data->data = NULL;
+    data_unref(entry_data);
 
-        cache->ttl_sec = ttl_sec;
+    GF_FREE(entry);
 out:
-        return cache;
+    return;
 }
 
 /**
@@ -73,16 +115,143 @@ out:
  * @return: Pointer to an allocated auth cache entry, NULL if allocation
  *          failed.
  */
-struct auth_cache_entry *
-auth_cache_entry_init ()
+static struct auth_cache_entry *
+auth_cache_entry_init()
 {
-        struct auth_cache_entry *entry = NULL;
+    struct auth_cache_entry *entry = NULL;
 
-        entry = GF_CALLOC (1, sizeof (*entry), gf_nfs_mt_auth_cache_entry);
-        if (!entry)
-                gf_log (GF_NFS, GF_LOG_WARNING, "failed to allocate entry");
+    entry = GF_CALLOC(1, sizeof(*entry), gf_nfs_mt_auth_cache_entry);
+    if (!entry)
+        gf_msg(GF_NFS, GF_LOG_WARNING, ENOMEM, NFS_MSG_NO_MEMORY,
+               "failed to allocate entry");
+    else
+        GF_REF_INIT(entry, auth_cache_entry_free);
 
-        return entry;
+    return entry;
+}
+
+/**
+ * auth_cache_add -- Add an auth_cache_entry to the cache->dict
+ *
+ * @return: 0 on success, non-zero otherwise.
+ */
+static int
+auth_cache_add(struct auth_cache *cache, char *hashkey,
+               struct auth_cache_entry *entry)
+{
+    int ret = -1;
+    data_t *entry_data = NULL;
+    int hashkey_len;
+    GF_VALIDATE_OR_GOTO(GF_NFS, cache, out);
+    GF_VALIDATE_OR_GOTO(GF_NFS, cache->cache_dict, out);
+
+    /* FIXME: entry is passed as parameter, this can never fail? */
+    entry = GF_REF_GET(entry);
+    if (!entry) {
+        /* entry does not have any references */
+        ret = -1;
+        goto out;
+    }
+
+    entry_data = bin_to_data(entry, sizeof(*entry));
+    if (!entry_data) {
+        ret = -1;
+        GF_REF_PUT(entry);
+        goto out;
+    }
+
+    /* we'll take an extra ref on the data_t, it gets unref'd when the
+     * auth_cache_entry is released */
+    entry->data = data_ref(entry_data);
+
+    hashkey_len = strlen(hashkey);
+    LOCK(&cache->lock);
+    {
+        ret = dict_setn(cache->cache_dict, hashkey, hashkey_len, entry_data);
+    }
+    UNLOCK(&cache->lock);
+
+    if (ret) {
+        /* adding to dict failed */
+        GF_REF_PUT(entry);
+    }
+out:
+    return ret;
+}
+
+/**
+ * _auth_cache_expired -- Check if the auth_cache_entry has expired
+ *
+ * The auth_cache->lock should have been taken when this function is called.
+ *
+ * @return: true when the auth_cache_entry is expired, false otherwise.
+ */
+static int
+_auth_cache_expired(struct auth_cache *cache, struct auth_cache_entry *entry)
+{
+    return ((gf_time() - entry->timestamp) > cache->ttl_sec);
+}
+
+/**
+ * auth_cache_get -- Get the @hashkey entry from the cache->cache_dict
+ *
+ * @cache: The auth_cache that should contain the @entry.
+ * @haskkey: The key associated with the auth_cache_entry.
+ * @entry: The found auth_cache_entry, unmodified if not found/expired.
+ *
+ * The using the cache->dict requires locking, this function takes care of
+ * that. When the entry is found, but has expired, it will be removed from the
+ * cache_dict.
+ *
+ * @return: 0 when found, ENTRY_NOT_FOUND or ENTRY_EXPIRED otherwise.
+ */
+static enum auth_cache_lookup_results
+auth_cache_get(struct auth_cache *cache, char *hashkey,
+               struct auth_cache_entry **entry)
+{
+    enum auth_cache_lookup_results ret = ENTRY_NOT_FOUND;
+    data_t *entry_data = NULL;
+    struct auth_cache_entry *lookup_res = NULL;
+
+    GF_VALIDATE_OR_GOTO(GF_NFS, cache, out);
+    GF_VALIDATE_OR_GOTO(GF_NFS, cache->cache_dict, out);
+    GF_VALIDATE_OR_GOTO(GF_NFS, hashkey, out);
+
+    LOCK(&cache->lock);
+    {
+        entry_data = dict_get(cache->cache_dict, hashkey);
+        if (!entry_data)
+            goto unlock;
+
+        /* FIXME: this is dangerous use of entry_data */
+        lookup_res = GF_REF_GET((struct auth_cache_entry *)entry_data->data);
+        if (lookup_res == NULL) {
+            /* entry has been free'd */
+            ret = ENTRY_EXPIRED;
+            goto unlock;
+        }
+
+        if (_auth_cache_expired(cache, lookup_res)) {
+            ret = ENTRY_EXPIRED;
+            GF_REF_PUT(lookup_res->item);
+            lookup_res->item = NULL;
+
+            /* free entry and remove from the cache */
+            GF_FREE(lookup_res);
+            entry_data->data = NULL;
+            dict_del(cache->cache_dict, hashkey);
+
+            goto unlock;
+        }
+
+        *entry = lookup_res;
+        ret = ENTRY_FOUND;
+    }
+unlock:
+    UNLOCK(&cache->lock);
+
+out:
+    return ret;
 }
 
 /**
@@ -102,75 +271,98 @@ auth_cache_entry_init ()
  *          0 if found
  */
 enum auth_cache_lookup_results
-auth_cache_lookup (struct auth_cache *cache, struct nfs3_fh *fh,
-                   const char *host_addr, time_t *timestamp,
-                   gf_boolean_t *can_write)
+auth_cache_lookup(struct auth_cache *cache, struct nfs3_fh *fh,
+                  const char *host_addr, time_t *timestamp,
+                  gf_boolean_t *can_write)
 {
-        char                    *hashkey    = NULL;
-        data_t                  *entry_data = NULL;
-        struct auth_cache_entry *lookup_res = NULL;
-        int                      ret        = ENTRY_NOT_FOUND;
-
-        GF_VALIDATE_OR_GOTO (GF_NFS, cache, out);
-        GF_VALIDATE_OR_GOTO (GF_NFS, cache->cache_dict, out);
-        GF_VALIDATE_OR_GOTO (GF_NFS, fh, out);
-        GF_VALIDATE_OR_GOTO (GF_NFS, host_addr, out);
-        GF_VALIDATE_OR_GOTO (GF_NFS, timestamp, out);
-        GF_VALIDATE_OR_GOTO (GF_NFS, can_write, out);
-
-        make_hashkey (hashkey, fh, host_addr);
-
-        entry_data = dict_get (cache->cache_dict, hashkey);
-        if (!entry_data) {
-                gf_log (GF_NFS, GF_LOG_DEBUG, "could not find entry for %s",
-                        host_addr);
-                goto out;
-        }
+    char *hashkey = NULL;
+    struct auth_cache_entry *lookup_res = NULL;
+    enum auth_cache_lookup_results ret = ENTRY_NOT_FOUND;
+
+    GF_VALIDATE_OR_GOTO(GF_NFS, cache, out);
+    GF_VALIDATE_OR_GOTO(GF_NFS, fh, out);
+    GF_VALIDATE_OR_GOTO(GF_NFS, host_addr, out);
+    GF_VALIDATE_OR_GOTO(GF_NFS, timestamp, out);
+    GF_VALIDATE_OR_GOTO(GF_NFS, can_write, out);
+
+    hashkey = make_hashkey(fh, host_addr);
+    if (!hashkey) {
+        ret = -ENOMEM;
+        goto out;
+    }
+
+    ret = auth_cache_get(cache, hashkey, &lookup_res);
+    switch (ret) {
+        case ENTRY_FOUND:
+            *timestamp = lookup_res->timestamp;
+            *can_write = lookup_res->item->opts->rw;
+            GF_REF_PUT(lookup_res);
+            break;
+
+        case ENTRY_NOT_FOUND:
+            gf_msg_debug(GF_NFS, 0, "could not find entry for %s", host_addr);
+            break;
+
+        case ENTRY_EXPIRED:
+            gf_msg_debug(GF_NFS, 0, "entry for host %s has expired", host_addr);
+            break;
+    }
 
-        lookup_res = (struct auth_cache_entry *)(entry_data->data);
+out:
+    GF_FREE(hashkey);
 
-        if ((time (NULL) - lookup_res->timestamp) > cache->ttl_sec) {
-                gf_log (GF_NFS, GF_LOG_DEBUG, "entry for host %s has expired",
-                        host_addr);
-                GF_FREE (lookup_res);
-                entry_data->data = NULL;
-                /* Remove from the cache */
-                dict_del (cache->cache_dict, hashkey);
+    return ret;
+}
 
-                ret = ENTRY_EXPIRED;
-                goto out;
-        }
+/* auth_cache_entry_purge -- free up the auth_cache_entry
+ *
+ * This gets called through dict_foreach() by auth_cache_purge(). Each
+ * auth_cache_entry has a refcount which needs to be decremented. Once the
+ * auth_cache_entry reaches refcount == 0, auth_cache_entry_free() will call
+ * data_unref() to free the associated data_t.
+ *
+ * @d: dict that gets purged by auth_cache_purge()
+ * @k: hashkey of the current entry
+ * @v: data_t of the current entry
+ */
+int
+auth_cache_entry_purge(dict_t *d, char *k, data_t *v, void *_unused)
+{
+    struct auth_cache_entry *entry = (struct auth_cache_entry *)v->data;
 
-        *timestamp = lookup_res->timestamp;
-        *can_write = lookup_res->item->opts->rw;
+    if (entry)
+        GF_REF_PUT(entry);
 
-        ret = ENTRY_FOUND;
-out:
-        return ret;
+    return 0;
 }
 
 /**
- * auth_cache_purge -- Purge the dict in the cache and set
- *                     the dict pointer to NULL. It will be allocated
- *                     on the first insert into the dict.
+ * auth_cache_purge -- Purge the dict in the cache and create a new empty one.
  *
  * @cache: Cache to purge
  *
  */
 void
-auth_cache_purge (struct auth_cache *cache)
+auth_cache_purge(struct auth_cache *cache)
 {
-        dict_t *new_cache_dict = NULL;
-        dict_t *old_cache_dict = cache->cache_dict;
-
-        if (!cache || !cache->cache_dict)
-                goto out;
-
-        (void)__sync_lock_test_and_set (&cache->cache_dict, new_cache_dict);
-
-        dict_unref (old_cache_dict);
+    dict_t *new_cache_dict = dict_new();
+    dict_t *old_cache_dict = NULL;
+
+    if (!cache || !new_cache_dict)
+        goto out;
+
+    LOCK(&cache->lock);
+    {
+        old_cache_dict = cache->cache_dict;
+        cache->cache_dict = new_cache_dict;
+    }
+    UNLOCK(&cache->lock);
+
+    /* walk all entries and refcount-- with GF_REF_PUT() */
+    dict_foreach(old_cache_dict, auth_cache_entry_purge, NULL);
+    dict_unref(old_cache_dict);
 out:
-        return;
+    return;
 }
 
 /**
@@ -185,25 +377,24 @@ out:
  *
  */
 gf_boolean_t
-is_nfs_fh_cached (struct auth_cache *cache, struct nfs3_fh *fh,
-                  const char *host_addr)
+is_nfs_fh_cached(struct auth_cache *cache, struct nfs3_fh *fh,
+                 const char *host_addr)
 {
-        int          ret       = 0;
-        time_t       timestamp = 0;
-        gf_boolean_t cached    = _gf_false;
-        gf_boolean_t can_write = _gf_false;
+    int ret = 0;
+    time_t timestamp = 0;
+    gf_boolean_t cached = _gf_false;
+    gf_boolean_t can_write = _gf_false;
 
-        if (!fh)
-                goto out;
+    if (!fh)
+        goto out;
 
-        ret = auth_cache_lookup (cache, fh, host_addr, &timestamp, &can_write);
-        cached = (ret == ENTRY_FOUND);
+    ret = auth_cache_lookup(cache, fh, host_addr, &timestamp, &can_write);
+    cached = (ret == ENTRY_FOUND);
 
 out:
-        return cached;
+    return cached;
 }
 
-
 /**
  * is_nfs_fh_cached_and_writeable -- Checks if an NFS FH is cached for the given
  *                                   host and writable
@@ -216,22 +407,22 @@ out:
  *
  */
 gf_boolean_t
-is_nfs_fh_cached_and_writeable (struct auth_cache *cache, struct nfs3_fh *fh,
-                                const char *host_addr)
+is_nfs_fh_cached_and_writeable(struct auth_cache *cache, struct nfs3_fh *fh,
+                               const char *host_addr)
 {
-        int          ret       = 0;
-        time_t       timestamp = 0;
-        gf_boolean_t cached    = _gf_false;
-        gf_boolean_t writable  = _gf_false;
+    int ret = 0;
+    time_t timestamp = 0;
+    gf_boolean_t cached = _gf_false;
+    gf_boolean_t writable = _gf_false;
 
-        if (!fh)
-                goto out;
+    if (!fh)
+        goto out;
 
-        ret = auth_cache_lookup (cache, fh, host_addr, &timestamp, &writable);
-        cached = ((ret == ENTRY_FOUND) && writable);
+    ret = auth_cache_lookup(cache, fh, host_addr, &timestamp, &writable);
+    cached = ((ret == ENTRY_FOUND) && writable);
 
 out:
-        return cached;
+    return cached;
 }
 
 /**
@@ -248,64 +439,58 @@ out:
  *
  */
 int
-cache_nfs_fh (struct auth_cache *cache, struct nfs3_fh *fh,
-              const char *host_addr, struct export_item *export_item)
+cache_nfs_fh(struct auth_cache *cache, struct nfs3_fh *fh,
+             const char *host_addr, struct export_item *export_item)
 {
-        int                      ret        = -EINVAL;
-        char                    *hashkey    = NULL;
-        data_t                  *entry_data = NULL;
-        time_t                   timestamp  = 0;
-        gf_boolean_t             can_write  = _gf_false;
-        struct auth_cache_entry *entry      = NULL;
-
-        GF_VALIDATE_OR_GOTO (GF_NFS, host_addr, out);
-        GF_VALIDATE_OR_GOTO (GF_NFS, cache, out);
-        GF_VALIDATE_OR_GOTO (GF_NFS, fh, out);
-
-        /* If a dict has not been allocated already, allocate it. */
-        if (!cache->cache_dict) {
-                cache->cache_dict = dict_new ();
-                if (!cache->cache_dict) {
-                        ret = -ENOMEM;
-                        goto out;
-                }
-        }
-
-        /* If we could already find it in the cache, just return */
-        ret = auth_cache_lookup (cache, fh, host_addr, &timestamp, &can_write);
-        if (ret == 0) {
-                gf_log (GF_NFS, GF_LOG_TRACE, "found cached auth/fh for host "
-                        "%s", host_addr);
-                goto out;
-        }
+    int ret = -EINVAL;
+    char *hashkey = NULL;
+    time_t timestamp = 0;
+    gf_boolean_t can_write = _gf_false;
+    struct auth_cache_entry *entry = NULL;
+
+    GF_VALIDATE_OR_GOTO(GF_NFS, host_addr, out);
+    GF_VALIDATE_OR_GOTO(GF_NFS, cache, out);
+    GF_VALIDATE_OR_GOTO(GF_NFS, fh, out);
+
+    /* If we could already find it in the cache, just return */
+    ret = auth_cache_lookup(cache, fh, host_addr, &timestamp, &can_write);
+    if (ret == 0) {
+        gf_msg_trace(GF_NFS, 0,
+                     "found cached auth/fh for host "
+                     "%s",
+                     host_addr);
+        goto out;
+    }
+
+    hashkey = make_hashkey(fh, host_addr);
+    if (!hashkey) {
+        ret = -ENOMEM;
+        goto out;
+    }
+
+    entry = auth_cache_entry_init();
+    if (!entry) {
+        ret = -ENOMEM;
+        goto out;
+    }
+
+    entry->timestamp = gf_time();
+    /* Update entry->item if it is pointing to a different export_item */
+    if (entry->item && entry->item != export_item) {
+        GF_REF_PUT(entry->item);
+    }
+    entry->item = GF_REF_GET(export_item);
+
+    ret = auth_cache_add(cache, hashkey, entry);
+    GF_REF_PUT(entry);
+    if (ret)
+        goto out;
+
+    gf_msg_trace(GF_NFS, 0, "Caching file-handle (%s)", host_addr);
+    ret = 0;
 
-        make_hashkey (hashkey, fh, host_addr);
-
-        entry = auth_cache_entry_init ();
-        if (!entry) {
-                ret = -ENOMEM;
-                goto out;
-        }
-
-        entry->timestamp = time (NULL);
-        entry->item = export_item;
-
-        /* The cache entry will simply be the time that the entry
-         * was cached.
-         */
-        entry_data = bin_to_data (entry, sizeof (*entry));
-        if (!entry_data) {
-                GF_FREE (entry);
-                goto out;
-        }
-
-        ret = dict_set (cache->cache_dict, hashkey, entry_data);
-        if (ret == -1) {
-                GF_FREE (entry);
-                goto out;
-        }
-        gf_log (GF_NFS, GF_LOG_TRACE, "Caching file-handle (%s)", host_addr);
-        ret = 0;
 out:
-        return ret;
+    GF_FREE(hashkey);
+
+    return ret;
 }
diff --git a/xlators/nfs/server/src/auth-cache.h b/xlators/nfs/server/src/auth-cache.h
index 5f2f03c1cb8..4c9d09207f8 100644
--- a/xlators/nfs/server/src/auth-cache.h
+++ b/xlators/nfs/server/src/auth-cache.h
@@ -16,38 +16,37 @@
 #define _AUTH_CACHE_H_
 
 #include "nfs-mem-types.h"
-#include "mount3.h"
 #include "exports.h"
-#include "dict.h"
+#include <glusterfs/dict.h>
 #include "nfs3.h"
 
 struct auth_cache {
-        dict_t *cache_dict;      /* Dict holding fh -> authcache_entry */
-        time_t ttl_sec;          /* TTL of the auth cache in seconds */
+    gf_lock_t lock;     /* locking for the dict (and entries) */
+    dict_t *cache_dict; /* Dict holding fh -> authcache_entry */
+    time_t ttl_sec;     /* TTL of the auth cache in seconds */
 };
 
-
 /* Initializes the cache */
 struct auth_cache *
-auth_cache_init (time_t ttl_sec);
+auth_cache_init(time_t ttl_sec);
 
 /* Inserts FH into cache */
 int
-cache_nfs_fh (struct auth_cache *cache, struct nfs3_fh *fh,
-              const char *host_addr, struct export_item *export_item);
+cache_nfs_fh(struct auth_cache *cache, struct nfs3_fh *fh,
+             const char *host_addr, struct export_item *export_item);
 
 /* Checks if the filehandle cached & writable */
 gf_boolean_t
-is_nfs_fh_cached_and_writeable (struct auth_cache *cache, struct nfs3_fh *fh,
-                                const char *host_addr);
+is_nfs_fh_cached_and_writeable(struct auth_cache *cache, struct nfs3_fh *fh,
+                               const char *host_addr);
 
 /* Checks if the filehandle is cached */
 gf_boolean_t
-is_nfs_fh_cached (struct auth_cache *cache, struct nfs3_fh *fh,
-                  const char *host_addr);
+is_nfs_fh_cached(struct auth_cache *cache, struct nfs3_fh *fh,
+                 const char *host_addr);
 
 /* Purge the cache */
 void
-auth_cache_purge (struct auth_cache *cache);
+auth_cache_purge(struct auth_cache *cache);
 
 #endif /* _AUTH_CACHE_H_ */
diff --git a/xlators/nfs/server/src/exports.c b/xlators/nfs/server/src/exports.c
index 30810361785..d7e39934851 100644
--- a/xlators/nfs/server/src/exports.c
+++ b/xlators/nfs/server/src/exports.c
@@ -13,13 +13,18 @@
 */
 
 #include "exports.h"
-#include "hashfn.h"
-#include "parse-utils.h"
+#include <glusterfs/hashfn.h>
+#include <glusterfs/parse-utils.h>
+#include "nfs-messages.h"
 
-static void _exp_dict_destroy (dict_t *ng_dict);
-static void _export_options_print (const struct export_options *opts);
-static void _export_options_deinit (struct export_options *opts);
-static void _export_dir_deinit (struct export_dir *dir);
+static void
+_exp_dict_destroy(dict_t *ng_dict);
+static void
+_export_options_print(const struct export_options *opts);
+static void
+_export_options_deinit(struct export_options *opts);
+static void
+_export_dir_deinit(struct export_dir *dir);
 
 static struct parser *netgroup_parser;
 static struct parser *hostname_parser;
@@ -32,36 +37,36 @@ static struct parser *options_parser;
  *          failure: -1
  */
 static int
-_exp_init_parsers ()
+_exp_init_parsers()
 {
-        int ret = -1;
+    int ret = -1;
 
-        netgroup_parser = parser_init (NETGROUP_REGEX_PATTERN);
-        if (!netgroup_parser)
-                goto out;
+    netgroup_parser = parser_init(NETGROUP_REGEX_PATTERN);
+    if (!netgroup_parser)
+        goto out;
 
-        hostname_parser = parser_init (HOSTNAME_REGEX_PATTERN);
-        if (!hostname_parser)
-                goto out;
+    hostname_parser = parser_init(HOSTNAME_REGEX_PATTERN);
+    if (!hostname_parser)
+        goto out;
 
-        options_parser = parser_init (OPTIONS_REGEX_PATTERN);
-        if (!options_parser)
-                goto out;
+    options_parser = parser_init(OPTIONS_REGEX_PATTERN);
+    if (!options_parser)
+        goto out;
 
-        ret = 0;
+    ret = 0;
 out:
-        return ret;
+    return ret;
 }
 
 /**
  * _exp_deinit_parsers -- Free parsers used in this file
  */
 static void
-_exp_deinit_parsers ()
+_exp_deinit_parsers()
 {
-        parser_deinit (netgroup_parser);
-        parser_deinit (hostname_parser);
-        parser_deinit (options_parser);
+    parser_deinit(netgroup_parser);
+    parser_deinit(hostname_parser);
+    parser_deinit(options_parser);
 }
 
 /**
@@ -73,34 +78,35 @@ _exp_deinit_parsers ()
  * Not for external use.
  */
 struct exports_file *
-_exports_file_init ()
+_exports_file_init()
 {
-        struct exports_file *file = NULL;
+    struct exports_file *file = NULL;
 
-        file = GF_CALLOC (1, sizeof (*file), gf_common_mt_nfs_exports);
-        if (!file) {
-                gf_log (GF_EXP, GF_LOG_CRITICAL,
-                        "Failed to allocate file struct!");
-                goto out;
-        }
+    file = GF_CALLOC(1, sizeof(*file), gf_common_mt_nfs_exports);
+    if (!file) {
+        gf_msg(GF_EXP, GF_LOG_CRITICAL, ENOMEM, NFS_MSG_NO_MEMORY,
+               "Failed to allocate file struct!");
+        goto out;
+    }
 
-        file->exports_dict = dict_new ();
-        file->exports_map = dict_new ();
-        if (!file->exports_dict || !file->exports_map) {
-                gf_log (GF_EXP, GF_LOG_CRITICAL, "Failed to allocate dict!");
-                goto free_and_out;
-        }
+    file->exports_dict = dict_new();
+    file->exports_map = dict_new();
+    if (!file->exports_dict || !file->exports_map) {
+        gf_msg(GF_EXP, GF_LOG_CRITICAL, ENOMEM, NFS_MSG_NO_MEMORY,
+               "Failed to allocate dict!");
+        goto free_and_out;
+    }
 
-        goto out;
+    goto out;
 
 free_and_out:
-        if (file->exports_dict)
-                dict_unref (file->exports_dict);
+    if (file->exports_dict)
+        dict_unref(file->exports_dict);
 
-        GF_FREE (file);
-        file = NULL;
+    GF_FREE(file);
+    file = NULL;
 out:
-        return file;
+    return file;
 }
 
 /**
@@ -114,24 +120,24 @@ out:
  * Not for external use.
  */
 static int
-_exp_file_dict_destroy (dict_t *dict, char *key, data_t *val, void *tmp)
+_exp_file_dict_destroy(dict_t *dict, char *key, data_t *val, void *tmp)
 {
-        struct export_dir *dir = NULL;
+    struct export_dir *dir = NULL;
 
-        GF_VALIDATE_OR_GOTO (GF_EXP, dict, out);
+    GF_VALIDATE_OR_GOTO(GF_EXP, dict, out);
 
-        if (val) {
-                dir = (struct export_dir *)val->data;
+    if (val) {
+        dir = (struct export_dir *)val->data;
 
-                if (dir) {
-                        _export_dir_deinit (dir);
-                        val->data = NULL;
-                }
-                dict_del (dict, key);
+        if (dir) {
+            _export_dir_deinit(dir);
+            val->data = NULL;
         }
+        dict_del(dict, key);
+    }
 
 out:
-        return 0;
+    return 0;
 }
 
 /**
@@ -142,27 +148,25 @@ out:
  * Externally usable.
  */
 void
-exp_file_deinit (struct exports_file *expfile)
+exp_file_deinit(struct exports_file *expfile)
 {
-        if (!expfile)
-                goto out;
+    if (!expfile)
+        goto out;
 
-        if (expfile->exports_dict) {
-                dict_foreach (expfile->exports_dict, _exp_file_dict_destroy,
-                              NULL);
-                dict_unref (expfile->exports_dict);
-        }
+    if (expfile->exports_dict) {
+        dict_foreach(expfile->exports_dict, _exp_file_dict_destroy, NULL);
+        dict_unref(expfile->exports_dict);
+    }
 
-        if (expfile->exports_map) {
-                dict_foreach (expfile->exports_map, _exp_file_dict_destroy,
-                              NULL);
-                dict_unref (expfile->exports_map);
-        }
+    if (expfile->exports_map) {
+        dict_foreach(expfile->exports_map, _exp_file_dict_destroy, NULL);
+        dict_unref(expfile->exports_map);
+    }
 
-        GF_FREE (expfile->filename);
-        GF_FREE (expfile);
+    GF_FREE(expfile->filename);
+    GF_FREE(expfile);
 out:
-        return;
+    return;
 }
 
 /**
@@ -174,16 +178,16 @@ out:
  * Not for external use.
  */
 static struct export_dir *
-_export_dir_init ()
+_export_dir_init()
 {
-        struct export_dir *expdir  = GF_CALLOC (1, sizeof (*expdir),
-                                                gf_common_mt_nfs_exports);
+    struct export_dir *expdir = GF_CALLOC(1, sizeof(*expdir),
+                                          gf_common_mt_nfs_exports);
 
-        if (!expdir)
-                gf_log (GF_EXP, GF_LOG_CRITICAL,
-                        "Failed to allocate export dir structure!");
+    if (!expdir)
+        gf_msg(GF_EXP, GF_LOG_CRITICAL, ENOMEM, NFS_MSG_NO_MEMORY,
+               "Failed to allocate export dir structure!");
 
-        return expdir;
+    return expdir;
 }
 
 /**
@@ -194,19 +198,20 @@ _export_dir_init ()
  * Not for external use.
  */
 static void
-_export_dir_deinit (struct export_dir *dir)
+_export_dir_deinit(struct export_dir *dir)
 {
-        GF_VALIDATE_OR_GOTO (GF_EXP, dir, out);
-        GF_FREE (dir->dir_name);
-        _exp_dict_destroy (dir->netgroups);
-        _exp_dict_destroy (dir->hosts);
-        GF_FREE (dir);
+    GF_VALIDATE_OR_GOTO(GF_EXP, dir, out);
+    GF_FREE(dir->dir_name);
+    _exp_dict_destroy(dir->netgroups);
+    dict_unref(dir->netgroups);
+    _exp_dict_destroy(dir->hosts);
+    dict_unref(dir->hosts);
+    GF_FREE(dir);
 
 out:
-        return;
+    return;
 }
 
-
 /**
  * _export_item_print -- Print the elements in the export item.
  *
@@ -215,52 +220,55 @@ out:
  * Not for external use.
  */
 static void
-_export_item_print (const struct export_item *item)
+_export_item_print(const struct export_item *item)
 {
-        GF_VALIDATE_OR_GOTO (GF_EXP, item, out);
-        printf ("%s", item->name);
-        _export_options_print (item->opts);
+    GF_VALIDATE_OR_GOTO(GF_EXP, item, out);
+    printf("%s", item->name);
+    _export_options_print(item->opts);
 out:
-        return;
+    return;
 }
 
 /**
- * _export_item_init -- Initialize an export item structure
+ * _export_item_deinit -- Free memory used by an export item
  *
- * @return  : success: Pointer to an allocated exports item struct
- *            failure: NULL
+ * @expdir : Pointer to the export item to free
  *
  * Not for external use.
  */
-static struct export_item *
-_export_item_init ()
+static void
+_export_item_deinit(struct export_item *item)
 {
-        struct export_item *item = GF_CALLOC (1, sizeof (*item),
-                                              gf_common_mt_nfs_exports);
-
-        if (!item)
-                gf_log (GF_EXP, GF_LOG_CRITICAL,
-                        "Failed to allocate export item!");
+    if (!item)
+        return;
 
-        return item;
+    _export_options_deinit(item->opts);
+    GF_FREE(item->name);
+    GF_FREE(item);
 }
 
 /**
- * _export_item_deinit -- Free memory used by an export item
+ * _export_item_init -- Initialize an export item structure
  *
- * @expdir : Pointer to the export item to free
+ * @return  : success: Pointer to an allocated exports item struct
+ *            failure: NULL
  *
  * Not for external use.
  */
-static void
-_export_item_deinit (struct export_item *item)
+static struct export_item *
+_export_item_init()
 {
-        if (!item)
-                return;
+    struct export_item *item = GF_CALLOC(1, sizeof(*item),
+                                         gf_common_mt_nfs_exports);
 
-        _export_options_deinit (item->opts);
-        GF_FREE (item->name);
-        GF_FREE (item);
+    if (item) {
+        GF_REF_INIT(item, _export_item_deinit);
+    } else {
+        gf_msg(GF_EXP, GF_LOG_CRITICAL, ENOMEM, NFS_MSG_NO_MEMORY,
+               "Failed to allocate export item!");
+    }
+
+    return item;
 }
 
 /**
@@ -272,16 +280,16 @@ _export_item_deinit (struct export_item *item)
  * Not for external use.
  */
 static struct export_options *
-_export_options_init ()
+_export_options_init()
 {
-        struct export_options *opts = GF_CALLOC (1, sizeof (*opts),
-                                                 gf_common_mt_nfs_exports);
+    struct export_options *opts = GF_CALLOC(1, sizeof(*opts),
+                                            gf_common_mt_nfs_exports);
 
-        if (!opts)
-                gf_log (GF_EXP, GF_LOG_CRITICAL,
-                        "Failed to allocate options structure!");
+    if (!opts)
+        gf_msg(GF_EXP, GF_LOG_CRITICAL, ENOMEM, NFS_MSG_NO_MEMORY,
+               "Failed to allocate options structure!");
 
-        return opts;
+    return opts;
 }
 
 /**
@@ -292,14 +300,14 @@ _export_options_init ()
  * Not for external use.
  */
 static void
-_export_options_deinit (struct export_options *opts)
+_export_options_deinit(struct export_options *opts)
 {
-        if (!opts)
-                return;
+    if (!opts)
+        return;
 
-        GF_FREE (opts->anon_uid);
-        GF_FREE (opts->sec_type);
-        GF_FREE (opts);
+    GF_FREE(opts->anon_uid);
+    GF_FREE(opts->sec_type);
+    GF_FREE(opts);
 }
 
 /**
@@ -310,31 +318,31 @@ _export_options_deinit (struct export_options *opts)
  * Not for external use.
  */
 static void
-_export_options_print (const struct export_options *opts)
+_export_options_print(const struct export_options *opts)
 {
-        GF_VALIDATE_OR_GOTO (GF_EXP, opts, out);
+    GF_VALIDATE_OR_GOTO(GF_EXP, opts, out);
 
-        printf ("(");
-        if (opts->rw)
-                printf ("rw,");
-        else
-                printf ("ro,");
+    printf("(");
+    if (opts->rw)
+        printf("rw,");
+    else
+        printf("ro,");
 
-        if (opts->nosuid)
-                printf ("nosuid,");
+    if (opts->nosuid)
+        printf("nosuid,");
 
-        if (opts->root)
-                printf ("root,");
+    if (opts->root)
+        printf("root,");
 
-        if (opts->anon_uid)
-                printf ("anonuid=%s,", opts->anon_uid);
+    if (opts->anon_uid)
+        printf("anonuid=%s,", opts->anon_uid);
 
-        if (opts->sec_type)
-                printf ("sec=%s,", opts->sec_type);
+    if (opts->sec_type)
+        printf("sec=%s,", opts->sec_type);
 
-        printf (") ");
+    printf(") ");
 out:
-        return;
+    return;
 }
 
 /**
@@ -350,14 +358,14 @@ out:
  * Not for external use.
  */
 static int
-__exp_dict_free_walk (dict_t *dict, char *key, data_t *val, void *tmp)
+__exp_dict_free_walk(dict_t *dict, char *key, data_t *val, void *tmp)
 {
-        if (val) {
-                _export_item_deinit ((struct export_item *)val->data);
-                val->data = NULL;
-                dict_del (dict, key);
-        }
-        return 0;
+    if (val) {
+        GF_REF_PUT((struct export_item *)val->data);
+        val->data = NULL;
+        dict_del(dict, key);
+    }
+    return 0;
 }
 
 /**
@@ -369,14 +377,14 @@ __exp_dict_free_walk (dict_t *dict, char *key, data_t *val, void *tmp)
  * Not for external use.
  */
 static void
-_exp_dict_destroy (dict_t *ng_dict)
+_exp_dict_destroy(dict_t *ng_dict)
 {
-        if (!ng_dict)
-                goto out;
+    if (!ng_dict)
+        goto out;
 
-        dict_foreach (ng_dict, __exp_dict_free_walk, NULL);
+    dict_foreach(ng_dict, __exp_dict_free_walk, NULL);
 out:
-        return;
+    return;
 }
 
 /**
@@ -390,20 +398,22 @@ out:
  *           failure: NULL
  */
 struct export_dir *
-exp_file_dir_from_uuid (const struct exports_file *file,
-                        const uuid_t export_uuid)
+exp_file_dir_from_uuid(const struct exports_file *file,
+                       const uuid_t export_uuid)
 {
-        char               export_uuid_str[512] = {0, };
-        data_t            *dirdata              = NULL;
-        struct export_dir *dir                  = NULL;
+    char export_uuid_str[512] = {
+        0,
+    };
+    data_t *dirdata = NULL;
+    struct export_dir *dir = NULL;
 
-        gf_uuid_unparse (export_uuid, export_uuid_str);
+    gf_uuid_unparse(export_uuid, export_uuid_str);
 
-        dirdata = dict_get (file->exports_map, export_uuid_str);
-        if (dirdata)
-                dir = (struct export_dir *)dirdata->data;
+    dirdata = dict_get(file->exports_map, export_uuid_str);
+    if (dirdata)
+        dir = (struct export_dir *)dirdata->data;
 
-        return dir;
+    return dir;
 }
 
 /**
@@ -420,32 +430,36 @@ exp_file_dir_from_uuid (const struct exports_file *file,
  * Not for external use.
  */
 static void
-_exp_file_insert (struct exports_file *file, struct export_dir *dir)
+_exp_file_insert(struct exports_file *file, struct export_dir *dir)
 {
-        data_t   *dirdata              = NULL;
-        uint32_t  hashedval            = 0;
-        uuid_t    export_uuid          = {0, };
-        char      export_uuid_str[512] = {0, };
-        char     *dirdup               = NULL;
-
-        GF_VALIDATE_OR_GOTO (GF_EXP, file, out);
-        GF_VALIDATE_OR_GOTO (GF_EXP, dir, out);
-
-        dirdata = bin_to_data (dir, sizeof (*dir));
-        dict_set (file->exports_dict, dir->dir_name, dirdata);
-
-        dirdup = strdupa (dir->dir_name);
-        while (strlen (dirdup) > 0 && dirdup[0] == '/')
-                dirdup++;
-
-        hashedval = SuperFastHash (dirdup, strlen (dirdup));
-        memset (export_uuid, 0, sizeof (export_uuid));
-        memcpy (export_uuid, &hashedval, sizeof (hashedval));
-        gf_uuid_unparse (export_uuid, export_uuid_str);
-
-        dict_set (file->exports_map, export_uuid_str, dirdata);
+    data_t *dirdata = NULL;
+    uint32_t hashedval = 0;
+    uuid_t export_uuid = {
+        0,
+    };
+    char export_uuid_str[512] = {
+        0,
+    };
+    char *dirdup = NULL;
+
+    GF_VALIDATE_OR_GOTO(GF_EXP, file, out);
+    GF_VALIDATE_OR_GOTO(GF_EXP, dir, out);
+
+    dirdata = bin_to_data(dir, sizeof(*dir));
+    dict_set(file->exports_dict, dir->dir_name, dirdata);
+
+    dirdup = strdupa(dir->dir_name);
+    while (strlen(dirdup) > 0 && dirdup[0] == '/')
+        dirdup++;
+
+    hashedval = SuperFastHash(dirdup, strlen(dirdup));
+    memset(export_uuid, 0, sizeof(export_uuid));
+    memcpy(export_uuid, &hashedval, sizeof(hashedval));
+    gf_uuid_unparse(export_uuid, export_uuid_str);
+
+    dict_set(file->exports_map, export_uuid_str, dirdata);
 out:
-        return;
+    return;
 }
 
 /**
@@ -453,7 +467,7 @@ out:
  *
  * @dict : the dict to walk
  * @key  : the key in the dict we are currently on
- * @val  : the value in the dict assocated with the key
+ * @val  : the value in the dict associated with the key
  * @tmp  : Additional parameter data (not used)
  *
  * Passed as a function pointer to dict_foreach ().
@@ -461,12 +475,12 @@ out:
  * Not for external use.
  */
 static int
-__exp_item_print_walk (dict_t *dict, char *key, data_t *val, void *tmp)
+__exp_item_print_walk(dict_t *dict, char *key, data_t *val, void *tmp)
 {
-        if (val)
-                _export_item_print ((struct export_item *)val->data);
+    if (val)
+        _export_item_print((struct export_item *)val->data);
 
-        return 0;
+    return 0;
 }
 
 /**
@@ -474,7 +488,7 @@ __exp_item_print_walk (dict_t *dict, char *key, data_t *val, void *tmp)
  *
  * @dict : the dict to walk
  * @key  : the key in the dict we are currently on
- * @val  : the value in the dict assocated with the key
+ * @val  : the value in the dict associated with the key
  * @tmp  : Additional parameter data (not used)
  *
  * Passed as a function pointer to dict_foreach ().
@@ -482,23 +496,22 @@ __exp_item_print_walk (dict_t *dict, char *key, data_t *val, void *tmp)
  * Not for external use.
  */
 static int
-__exp_file_print_walk (dict_t *dict, char *key, data_t *val, void *tmp)
+__exp_file_print_walk(dict_t *dict, char *key, data_t *val, void *tmp)
 {
-        if (val) {
-                struct export_dir *dir = (struct export_dir *)val->data;
+    if (val) {
+        struct export_dir *dir = (struct export_dir *)val->data;
 
-                printf ("%s ", key);
+        printf("%s ", key);
 
-                if (dir->netgroups)
-                        dict_foreach (dir->netgroups, __exp_item_print_walk,
-                                      NULL);
+        if (dir->netgroups)
+            dict_foreach(dir->netgroups, __exp_item_print_walk, NULL);
 
-                if (dir->hosts)
-                        dict_foreach (dir->hosts, __exp_item_print_walk, NULL);
+        if (dir->hosts)
+            dict_foreach(dir->hosts, __exp_item_print_walk, NULL);
 
-                printf ("\n");
-        }
-        return 0;
+        printf("\n");
+    }
+    return 0;
 }
 
 /**
@@ -509,29 +522,29 @@ __exp_file_print_walk (dict_t *dict, char *key, data_t *val, void *tmp)
  * Not for external use.
  */
 void
-exp_file_print (const struct exports_file *file)
+exp_file_print(const struct exports_file *file)
 {
-        GF_VALIDATE_OR_GOTO (GF_EXP, file, out);
-        dict_foreach (file->exports_dict, __exp_file_print_walk, NULL);
+    GF_VALIDATE_OR_GOTO(GF_EXP, file, out);
+    dict_foreach(file->exports_dict, __exp_file_print_walk, NULL);
 out:
-        return;
+    return;
 }
 
-#define __exp_line_get_opt_val(val, equals, ret, errlabel)      \
-        do {                                                    \
-                (val) = (equals) + 1;                           \
-                if (!(*(val))) {                                \
-                        (ret) = 1;                              \
-                        goto errlabel;                          \
-                }                                               \
-        } while (0)                                             \
+#define __exp_line_get_opt_val(val, equals, ret, errlabel)                     \
+    do {                                                                       \
+        (val) = (equals) + 1;                                                  \
+        if (!(*(val))) {                                                       \
+            (ret) = 1;                                                         \
+            goto errlabel;                                                     \
+        }                                                                      \
+    } while (0)
 
 enum gf_exp_parse_status {
-        GF_EXP_PARSE_SUCCESS                 = 0,
-        GF_EXP_PARSE_ITEM_NOT_FOUND          = 1,
-        GF_EXP_PARSE_ITEM_FAILURE            = 2,
-        GF_EXP_PARSE_ITEM_NOT_IN_MOUNT_STATE = 3,
-        GF_EXP_PARSE_LINE_IGNORING           = 4,
+    GF_EXP_PARSE_SUCCESS = 0,
+    GF_EXP_PARSE_ITEM_NOT_FOUND = 1,
+    GF_EXP_PARSE_ITEM_FAILURE = 2,
+    GF_EXP_PARSE_ITEM_NOT_IN_MOUNT_STATE = 3,
+    GF_EXP_PARSE_LINE_IGNORING = 4,
 };
 
 /**
@@ -554,48 +567,48 @@ enum gf_exp_parse_status {
  * Not for external use.
  */
 static int
-__exp_line_opt_key_value_parse (char *option, struct export_options *opts)
+__exp_line_opt_key_value_parse(char *option, struct export_options *opts)
 {
-        char              *equals   = NULL;
-        char              *right    = NULL;
-        char              *strmatch = option;
-        int                ret      = -EINVAL;
+    char *equals = NULL;
+    char *right = NULL;
+    char *strmatch = option;
+    int ret = -EINVAL;
 
-        GF_VALIDATE_OR_GOTO (GF_EXP, option, out);
-        GF_VALIDATE_OR_GOTO (GF_EXP, opts, out);
+    GF_VALIDATE_OR_GOTO(GF_EXP, option, out);
+    GF_VALIDATE_OR_GOTO(GF_EXP, opts, out);
 
-        equals = strchr (option, '=');
-        if (!equals) {
-                ret = GF_EXP_PARSE_ITEM_FAILURE;
-                goto out;
-        }
-
-        *equals = 0;
-        /* Now that an '=' has been found the left side is the option and
-         * the right side is the value. We simply have to compare those and
-         * extract it.
-         */
-        if (strcmp (strmatch, "anonuid") == 0) {
-                *equals = '=';
-                /* Get the value for this option */
-                __exp_line_get_opt_val (right, equals, ret, out);
-                opts->anon_uid = gf_strdup (right);
-                GF_CHECK_ALLOC (opts->anon_uid, ret, out);
-        } else if (strcmp (strmatch, "sec") == 0) {
-                *equals = '=';
-                /* Get the value for this option */
-                __exp_line_get_opt_val (right, equals, ret, out);
-                opts->sec_type = gf_strdup (right);
-                GF_CHECK_ALLOC (opts->sec_type, ret, out);
-        } else {
-                *equals = '=';
-                ret = GF_EXP_PARSE_ITEM_FAILURE;
-                goto out;
-        }
+    equals = strchr(option, '=');
+    if (!equals) {
+        ret = GF_EXP_PARSE_ITEM_FAILURE;
+        goto out;
+    }
+
+    *equals = 0;
+    /* Now that an '=' has been found the left side is the option and
+     * the right side is the value. We simply have to compare those and
+     * extract it.
+     */
+    if (strcmp(strmatch, "anonuid") == 0) {
+        *equals = '=';
+        /* Get the value for this option */
+        __exp_line_get_opt_val(right, equals, ret, out);
+        opts->anon_uid = gf_strdup(right);
+        GF_CHECK_ALLOC(opts->anon_uid, ret, out);
+    } else if (strcmp(strmatch, "sec") == 0) {
+        *equals = '=';
+        /* Get the value for this option */
+        __exp_line_get_opt_val(right, equals, ret, out);
+        opts->sec_type = gf_strdup(right);
+        GF_CHECK_ALLOC(opts->sec_type, ret, out);
+    } else {
+        *equals = '=';
+        ret = GF_EXP_PARSE_ITEM_FAILURE;
+        goto out;
+    }
 
-        ret = GF_EXP_PARSE_SUCCESS;
+    ret = GF_EXP_PARSE_SUCCESS;
 out:
-        return ret;
+    return ret;
 }
 
 /**
@@ -614,83 +627,89 @@ out:
  * Not for external use.
  */
 static int
-__exp_line_opt_parse (const char *opt_str, struct export_options **exp_opts)
+__exp_line_opt_parse(const char *opt_str, struct export_options **exp_opts)
 {
-        struct export_options  *opts     = NULL;
-        char                   *strmatch = NULL;
-        int                    ret       = -EINVAL;
-        char                   *equals   = NULL;
+    struct export_options *opts = NULL;
+    char *strmatch = NULL;
+    int ret = -EINVAL;
+    char *equals = NULL;
 
-        ret = parser_set_string (options_parser, opt_str);
-        if (ret < 0)
-                goto out;
+    ret = parser_set_string(options_parser, opt_str);
+    if (ret < 0)
+        goto out;
 
-        while ((strmatch = parser_get_next_match (options_parser))) {
-                if (!opts) {
-                        /* If the options have not been allocated,
-                         * allocate it.
-                         */
-                        opts = _export_options_init ();
-                        if (!opts) {
-                                ret = -ENOMEM;
-                                parser_unset_string (options_parser);
-                                goto out;
-                        }
-                }
-
-                /* First,  check for all the boolean options Second, check for
-                 * an '=', and check the available options there. The string
-                 * parsing here gets slightly messy, but the concept itself
-                 * is pretty simple.
-                 */
-                equals = strchr (strmatch, '=');
-                if (strcmp (strmatch, "root") == 0)
-                        opts->root = _gf_true;
-                else if (strcmp (strmatch, "ro") == 0)
-                        opts->rw = _gf_false;
-                else if (strcmp (strmatch, "rw") == 0)
-                        opts->rw = _gf_true;
-                else if (strcmp (strmatch, "nosuid") == 0)
-                        opts->nosuid = _gf_true;
-                else if (equals) {
-                        ret = __exp_line_opt_key_value_parse (strmatch, opts);
-                        if (ret < 0) {
-                                /* This means invalid key value options were
-                                 * specified, or memory allocation failed.
-                                 * The ret value gets bubbled up to the caller.
-                                 */
-                                GF_FREE (strmatch);
-                                parser_unset_string (options_parser);
-                                _export_options_deinit (opts);
-                                goto out;
-                        }
-                } else
-                        gf_log (GF_EXP, GF_LOG_WARNING,
-                                "Could not find any valid options for "
-                                "string: %s", strmatch);
-                GF_FREE (strmatch);
+    while ((strmatch = parser_get_next_match(options_parser))) {
+        if (!opts) {
+            /* If the options have not been allocated,
+             * allocate it.
+             */
+            opts = _export_options_init();
+            if (!opts) {
+                ret = -ENOMEM;
+                parser_unset_string(options_parser);
+                GF_FREE(strmatch);
+                goto out;
+            }
         }
 
-        if (!opts) {
-                /* If opts is not allocated
-                 * that means no matches were found
-                 * which is a parse error. Not marking
-                 * it as "not found" because it is a parse
-                 * error to not have options.
+        /* First,  check for all the boolean options Second, check for
+         * an '=', and check the available options there. The string
+         * parsing here gets slightly messy, but the concept itself
+         * is pretty simple.
+         */
+        equals = strchr(strmatch, '=');
+        if (strcmp(strmatch, "root") == 0)
+            opts->root = _gf_true;
+        else if (strcmp(strmatch, "ro") == 0)
+            opts->rw = _gf_false;
+        else if (strcmp(strmatch, "rw") == 0)
+            opts->rw = _gf_true;
+        else if (strcmp(strmatch, "nosuid") == 0)
+            opts->nosuid = _gf_true;
+        else if (equals) {
+            ret = __exp_line_opt_key_value_parse(strmatch, opts);
+            if (ret < 0) {
+                /* This means invalid key value options were
+                 * specified, or memory allocation failed.
+                 * The ret value gets bubbled up to the caller.
                  */
-                ret = GF_EXP_PARSE_ITEM_FAILURE;
-                parser_unset_string (options_parser);
+                GF_FREE(strmatch);
+                parser_unset_string(options_parser);
+                _export_options_deinit(opts);
                 goto out;
+            }
+        } else {
+            /* Cannot change to gf_msg.
+             * gf_msg not giving output to STDOUT
+             * Bug id : BZ1215017
+             */
+            gf_log(GF_EXP, GF_LOG_WARNING,
+                   "Could not find any valid options for "
+                   "string: %s",
+                   strmatch);
         }
+        GF_FREE(strmatch);
+    }
+
+    if (!opts) {
+        /* If opts is not allocated
+         * that means no matches were found
+         * which is a parse error. Not marking
+         * it as "not found" because it is a parse
+         * error to not have options.
+         */
+        ret = GF_EXP_PARSE_ITEM_FAILURE;
+        parser_unset_string(options_parser);
+        goto out;
+    }
 
-        *exp_opts = opts;
-        parser_unset_string (options_parser);
-        ret = GF_EXP_PARSE_SUCCESS;
+    *exp_opts = opts;
+    parser_unset_string(options_parser);
+    ret = GF_EXP_PARSE_SUCCESS;
 out:
-        return ret;
+    return ret;
 }
 
-
 /**
  * __exp_line_ng_host_str_parse -- Parse the netgroup or host string
  *
@@ -707,74 +726,76 @@ out:
  * Not for external use.
  */
 static int
-__exp_line_ng_host_str_parse (char *str, struct export_item **exp_item)
+__exp_line_ng_host_str_parse(char *str, struct export_item **exp_item)
 {
-        struct export_item     *item      = NULL;
-        int                     ret       = -EINVAL;
-        char                   *parens    = NULL;
-        char                   *optstr    = NULL;
-        struct export_options  *exp_opts  = NULL;
-        char                   *item_name = NULL;
-
-        GF_VALIDATE_OR_GOTO (GF_EXP, str, out);
-        GF_VALIDATE_OR_GOTO (GF_EXP, exp_item, out);
-
-        /* A netgroup/host string looks like this:
-         * @test(sec=sys,rw,anonuid=0) or host(sec=sys,rw,anonuid=0)
-         * We want to extract the name, 'test' or 'host'
-         * Again, we could setup a regex and use it here,
-         * but its simpler to find the '(' and copy until
-         * there.
-         */
-        parens = strchr (str, '(');
-        if (!parens) {
-                /* Parse error if there are no parens. */
-                ret = GF_EXP_PARSE_ITEM_FAILURE;
-                goto out;
-        }
+    struct export_item *item = NULL;
+    int ret = -EINVAL;
+    char *parens = NULL;
+    char *optstr = NULL;
+    struct export_options *exp_opts = NULL;
+    char *item_name = NULL;
+
+    GF_VALIDATE_OR_GOTO(GF_EXP, str, out);
+    GF_VALIDATE_OR_GOTO(GF_EXP, exp_item, out);
+
+    /* A netgroup/host string looks like this:
+     * @test(sec=sys,rw,anonuid=0) or host(sec=sys,rw,anonuid=0)
+     * We want to extract the name, 'test' or 'host'
+     * Again, we could setup a regex and use it here,
+     * but its simpler to find the '(' and copy until
+     * there.
+     */
+    parens = strchr(str, '(');
+    if (!parens) {
+        /* Parse error if there are no parens. */
+        ret = GF_EXP_PARSE_ITEM_FAILURE;
+        goto out;
+    }
 
-        *parens = '\0'; /* Temporarily terminate it so we can do a copy */
+    *parens = '\0'; /* Temporarily terminate it so we can do a copy */
 
-        if (strlen (str) > FQDN_MAX_LEN) {
-                ret = GF_EXP_PARSE_ITEM_FAILURE;
-                goto out;
-        }
+    if (strlen(str) > FQDN_MAX_LEN) {
+        ret = GF_EXP_PARSE_ITEM_FAILURE;
+        goto out;
+    }
 
-        /* Strip leading whitespaces */
-        while (*str == ' ' || *str == '\t')
-                str++;
+    /* Strip leading whitespaces */
+    while (*str == ' ' || *str == '\t')
+        str++;
 
-        item_name = gf_strdup (str);
-        GF_CHECK_ALLOC (item_name, ret, out);
+    item_name = gf_strdup(str);
+    GF_CHECK_ALLOC(item_name, ret, out);
 
-        /* Initialize an export item for this */
-        item = _export_item_init ();
-        GF_CHECK_ALLOC (item, ret, free_and_out);
-        item->name = item_name;
+    gf_msg_trace(GF_EXP, 0, "found hostname/netgroup: %s", item_name);
 
-        *parens = '('; /* Restore the string */
+    /* Initialize an export item for this */
+    item = _export_item_init();
+    GF_CHECK_ALLOC(item, ret, free_and_out);
+    item->name = item_name;
 
-        /* Options start at the parantheses */
-        optstr = parens;
+    *parens = '('; /* Restore the string */
 
-        ret = __exp_line_opt_parse (optstr, &exp_opts);
-        if (ret != 0) {
-                /* Bubble up the error to the caller */
-                _export_item_deinit (item);
-                goto out;
-        }
+    /* Options start at the parentheses */
+    optstr = parens;
 
-        item->opts = exp_opts;
+    ret = __exp_line_opt_parse(optstr, &exp_opts);
+    if (ret != 0) {
+        /* Bubble up the error to the caller */
+        GF_REF_PUT(item);
+        goto out;
+    }
 
-        *exp_item = item;
+    item->opts = exp_opts;
 
-        ret = GF_EXP_PARSE_SUCCESS;
-        goto out;
+    *exp_item = item;
+
+    ret = GF_EXP_PARSE_SUCCESS;
+    goto out;
 
 free_and_out:
-        GF_FREE (item_name);
+    GF_FREE(item_name);
 out:
-        return ret;
+    return ret;
 }
 
 /**
@@ -805,71 +826,75 @@ out:
  * Not for external use.
  */
 static int
-__exp_line_ng_parse (const char *line, dict_t **ng_dict)
+__exp_line_ng_parse(const char *line, dict_t **ng_dict)
 {
-        dict_t                  *netgroups = NULL;
-        char                    *strmatch  = NULL;
-        int                      ret       = -EINVAL;
-        struct export_item      *exp_ng    = NULL;
-        data_t                  *ngdata    = NULL;
-
-        GF_VALIDATE_OR_GOTO (GF_EXP, line, out);
-        GF_VALIDATE_OR_GOTO (GF_EXP, ng_dict, out);
+    dict_t *netgroups = NULL;
+    char *strmatch = NULL;
+    int ret = -EINVAL;
+    struct export_item *exp_ng = NULL;
+    data_t *ngdata = NULL;
+
+    GF_VALIDATE_OR_GOTO(GF_EXP, line, out);
+    GF_VALIDATE_OR_GOTO(GF_EXP, ng_dict, out);
+
+    *ng_dict = NULL; /* Will be set if parse is successful */
+
+    /* Initialize a parser with the line to parse
+     * and the regex used to parse it.
+     */
+    ret = parser_set_string(netgroup_parser, line);
+    if (ret < 0) {
+        goto out;
+    }
 
-        *ng_dict = NULL; /* Will be set if parse is successful */
+    gf_msg_trace(GF_EXP, 0, "parsing line: %s", line);
 
-        /* Initialize a parser with the line to parse
-         * and the regex used to parse it.
-         */
-        ret = parser_set_string (netgroup_parser, line);
-        if (ret < 0) {
-                goto out;
+    while ((strmatch = parser_get_next_match(netgroup_parser))) {
+        if (!netgroups) {
+            /* Allocate a new dict to store the netgroups. */
+            netgroups = dict_new();
+            if (!netgroups) {
+                ret = -ENOMEM;
+                goto free_and_out;
+            }
         }
 
-        while ((strmatch = parser_get_next_match (netgroup_parser))) {
-                if (!netgroups) {
-                        /* Allocate a new dict to store the netgroups. */
-                        netgroups = dict_new ();
-                        if (!netgroups) {
-                                ret = -ENOMEM;
-                                goto free_and_out;
-                        }
-                }
-
-                ret = __exp_line_ng_host_str_parse (strmatch, &exp_ng);
-
-                if (ret != 0) {
-                        /* Parsing or other critical errors.
-                         * caller will handle return value.
-                         */
-                        _exp_dict_destroy (netgroups);
-                        goto free_and_out;
-                }
-
-                ngdata = bin_to_data (exp_ng, sizeof (*exp_ng));
-                dict_set (netgroups, exp_ng->name, ngdata);
-
-                /* Free this matched string and continue parsing. */
-                GF_FREE (strmatch);
-        }
+        gf_msg_trace(GF_EXP, 0, "parsing netgroup: %s", strmatch);
 
-        /* If the netgroups dict was not allocated, then we know that
-         * no matches were found.
-         */
-        if (!netgroups) {
-                ret = GF_EXP_PARSE_ITEM_NOT_FOUND;
-                parser_unset_string (netgroup_parser);
-                goto out;
+        ret = __exp_line_ng_host_str_parse(strmatch, &exp_ng);
+
+        if (ret != 0) {
+            /* Parsing or other critical errors.
+             * caller will handle return value.
+             */
+            _exp_dict_destroy(netgroups);
+            goto free_and_out;
         }
 
-        ret = GF_EXP_PARSE_SUCCESS;
-        *ng_dict = netgroups;
+        ngdata = bin_to_data(exp_ng, sizeof(*exp_ng));
+        dict_set(netgroups, exp_ng->name, ngdata);
+
+        /* Free this matched string and continue parsing. */
+        GF_FREE(strmatch);
+    }
+
+    /* If the netgroups dict was not allocated, then we know that
+     * no matches were found.
+     */
+    if (!netgroups) {
+        ret = GF_EXP_PARSE_ITEM_NOT_FOUND;
+        parser_unset_string(netgroup_parser);
+        goto out;
+    }
+
+    ret = GF_EXP_PARSE_SUCCESS;
+    *ng_dict = netgroups;
 
 free_and_out:
-        parser_unset_string (netgroup_parser);
-        GF_FREE (strmatch);
+    parser_unset_string(netgroup_parser);
+    GF_FREE(strmatch);
 out:
-        return ret;
+    return ret;
 }
 
 /**
@@ -900,73 +925,75 @@ out:
  * Not for external use.
  */
 static int
-__exp_line_host_parse (const char *line, dict_t **host_dict)
+__exp_line_host_parse(const char *line, dict_t **host_dict)
 {
-        dict_t                  *hosts    = NULL;
-        char                    *strmatch = NULL;
-        int                     ret       = -EINVAL;
-        struct export_item      *exp_host = NULL;
-        data_t                  *hostdata = NULL;
-
-        GF_VALIDATE_OR_GOTO (GF_EXP, line, out);
-        GF_VALIDATE_OR_GOTO (GF_EXP, host_dict, out);
+    dict_t *hosts = NULL;
+    char *strmatch = NULL;
+    int ret = -EINVAL;
+    struct export_item *exp_host = NULL;
+    data_t *hostdata = NULL;
+
+    GF_VALIDATE_OR_GOTO(GF_EXP, line, out);
+    GF_VALIDATE_OR_GOTO(GF_EXP, host_dict, out);
+
+    *host_dict = NULL;
+
+    /* Initialize a parser with the line to parse and the regex used to
+     * parse it.
+     */
+    ret = parser_set_string(hostname_parser, line);
+    if (ret < 0) {
+        goto out;
+    }
 
-        *host_dict = NULL;
+    gf_msg_trace(GF_EXP, 0, "parsing line: %s", line);
 
-        /* Initialize a parser with the line to parse and the regex used to
-         * parse it.
-         */
-        ret = parser_set_string (hostname_parser, line);
-        if (ret < 0) {
-                goto out;
+    while ((strmatch = parser_get_next_match(hostname_parser))) {
+        if (!hosts) {
+            /* Allocate a new dictto store the netgroups. */
+            hosts = dict_new();
+            GF_CHECK_ALLOC(hosts, ret, free_and_out);
         }
 
-        while ((strmatch = parser_get_next_match (hostname_parser))) {
-                if (!hosts) {
-                        /* Allocate a new dictto store the netgroups. */
-                        hosts = dict_new ();
-                        GF_CHECK_ALLOC (hosts, ret, free_and_out);
-                }
-
-                ret = __exp_line_ng_host_str_parse (strmatch, &exp_host);
+        gf_msg_trace(GF_EXP, 0, "parsing hostname: %s", strmatch);
 
-                if (ret != 0) {
-                        /* Parsing or other critical error, free allocated
-                         * memory and exit. The caller will handle the errors.
-                         */
-                        _exp_dict_destroy (hosts);
-                        goto free_and_out;
-                }
+        ret = __exp_line_ng_host_str_parse(strmatch, &exp_host);
 
-                /* Insert export item structure into the hosts dict. */
-                hostdata = bin_to_data (exp_host, sizeof (*exp_host));
-                dict_set (hosts, exp_host->name, hostdata);
+        if (ret != 0) {
+            /* Parsing or other critical error, free allocated
+             * memory and exit. The caller will handle the errors.
+             */
+            _exp_dict_destroy(hosts);
+            goto free_and_out;
+        }
 
+        /* Insert export item structure into the hosts dict. */
+        hostdata = bin_to_data(exp_host, sizeof(*exp_host));
+        dict_set(hosts, exp_host->name, hostdata);
 
-                /* Free this matched string and continue parsing.*/
-                GF_FREE (strmatch);
-        }
+        /* Free this matched string and continue parsing.*/
+        GF_FREE(strmatch);
+    }
 
-        /* If the hosts dict was not allocated, then we know that
-         * no matches were found.
-         */
-        if (!exp_host) {
-                ret = GF_EXP_PARSE_ITEM_NOT_FOUND;
-                parser_unset_string (hostname_parser);
-                goto out;
-        }
+    /* If the hosts dict was not allocated, then we know that
+     * no matches were found.
+     */
+    if (!exp_host) {
+        ret = GF_EXP_PARSE_ITEM_NOT_FOUND;
+        parser_unset_string(hostname_parser);
+        goto out;
+    }
 
-        ret = GF_EXP_PARSE_SUCCESS;
-        *host_dict = hosts;
+    ret = GF_EXP_PARSE_SUCCESS;
+    *host_dict = hosts;
 
 free_and_out:
-        parser_unset_string (hostname_parser);
-        GF_FREE (strmatch);
+    parser_unset_string(hostname_parser);
+    GF_FREE(strmatch);
 out:
-        return ret;
+    return ret;
 }
 
-
 /**
  * __exp_line_dir_parse -- Extract directory name from a line in the exports
  *                         file.
@@ -977,7 +1004,7 @@ out:
  *            it will point to a valid memory region that is allocated by
  *            this function.
  * @check_ms: If this variable is set then we cross check the directory line
- *            with whats in gluster's vol files and reject them if they don't
+ *            with what's in gluster's vol files and reject them if they don't
  *            match.
  *
  * @return : success: GF_EXP_PARSE_SUCCESS
@@ -991,70 +1018,71 @@ out:
  * Not for external use.
  */
 static int
-__exp_line_dir_parse (const char *line, char **dirname, struct mount3_state *ms)
+__exp_line_dir_parse(const char *line, char **dirname, struct mount3_state *ms)
 {
-        char                    *dir        = NULL;
-        char                    *delim      = NULL;
-        int                     ret         = -EINVAL;
-        char                    *linedup    = NULL;
-        struct mnt3_export      *mnt3export = NULL;
-        size_t                  dirlen      = 0;
-
-        GF_VALIDATE_OR_GOTO (GF_EXP, line, out);
-        GF_VALIDATE_OR_GOTO (GF_EXP, dirname, out);
-
-        /* Duplicate the line because we don't
-         * want to modify the original string.
-         */
-        linedup = strdupa (line);
-
-        /* We use strtok_r () here to split the string by space/tab and get the
-         * the result. We only need the first result of the split.
-         * a simple task. It is worth noting that dirnames always have to be
-         * validated against gluster's vol files so if they don't
-         * match it will be rejected.
+    char *dir = NULL;
+    char *delim = NULL;
+    int ret = -EINVAL;
+    char *linedup = NULL;
+    struct mnt3_export *mnt3export = NULL;
+    size_t dirlen = 0;
+
+    GF_VALIDATE_OR_GOTO(GF_EXP, line, out);
+    GF_VALIDATE_OR_GOTO(GF_EXP, dirname, out);
+
+    /* Duplicate the line because we don't
+     * want to modify the original string.
+     */
+    linedup = strdupa(line);
+
+    /* We use strtok_r () here to split the string by space/tab and get the
+     * the result. We only need the first result of the split.
+     * a simple task. It is worth noting that dirnames always have to be
+     * validated against gluster's vol files so if they don't
+     * match it will be rejected.
+     */
+    dir = linedup;
+    delim = linedup + strcspn(linedup, " \t");
+    *delim = 0;
+
+    if (ms) {
+        /* Match the directory name with an existing
+         * export in the mount state.
          */
-        dir = linedup;
-        delim = linedup + strcspn (linedup, " \t");
-        *delim = 0;
-
-        if (ms) {
-                /* Match the directory name with an existing
-                 * export in the mount state.
-                 */
-                mnt3export = mnt3_mntpath_to_export (ms, dir, _gf_true);
-                if (!mnt3export) {
-                        gf_log (GF_EXP, GF_LOG_DEBUG, "%s not in mount state, "
-                                                      "ignoring!", dir);
-                        ret = GF_EXP_PARSE_ITEM_NOT_IN_MOUNT_STATE;
-                        goto out;
-                }
+        mnt3export = mnt3_mntpath_to_export(ms, dir, _gf_true);
+        if (!mnt3export) {
+            gf_msg_debug(GF_EXP, 0,
+                         "%s not in mount state, "
+                         "ignoring!",
+                         dir);
+            ret = GF_EXP_PARSE_ITEM_NOT_IN_MOUNT_STATE;
+            goto out;
         }
+    }
+
+    /* Directories can be 1024 bytes in length, check
+     * that the argument provided adheres to
+     * that restriction.
+     */
+    if (strlen(dir) > DIR_MAX_LEN) {
+        ret = -EINVAL;
+        goto out;
+    }
 
-        /* Directories can be 1024 bytes in length, check
-         * that the argument provided adheres to
-         * that restriction.
-         */
-        if (strlen (dir) > DIR_MAX_LEN) {
-                ret = -EINVAL;
-                goto out;
-        }
-
-        /* Copy the result of the split */
-        dir = gf_strdup (dir);
-        GF_CHECK_ALLOC (dir, ret, out);
-
-        /* Ensure that trailing slashes are stripped before storing the name */
-        dirlen = strlen (dir);
-        if (dirlen > 0 && dir[dirlen - 1] == '/')
-                dir[dirlen - 1] = '\0';
+    /* Copy the result of the split */
+    dir = gf_strdup(dir);
+    GF_CHECK_ALLOC(dir, ret, out);
 
+    /* Ensure that trailing slashes are stripped before storing the name */
+    dirlen = strlen(dir);
+    if (dirlen > 0 && dir[dirlen - 1] == '/')
+        dir[dirlen - 1] = '\0';
 
-        /* Set the argument to point to the allocated string */
-        *dirname = dir;
-        ret = GF_EXP_PARSE_SUCCESS;
+    /* Set the argument to point to the allocated string */
+    *dirname = dir;
+    ret = GF_EXP_PARSE_SUCCESS;
 out:
-        return ret;
+    return ret;
 }
 
 /**
@@ -1102,121 +1130,126 @@ out:
  * Not for external use.
  */
 static int
-_exp_line_parse (const char *line, struct export_dir **dir,
-                 gf_boolean_t parse_full, struct mount3_state *ms)
+_exp_line_parse(const char *line, struct export_dir **dir,
+                gf_boolean_t parse_full, struct mount3_state *ms)
 {
-        struct export_dir *expdir           = NULL;
-        char              *dirstr           = NULL;
-        dict_t            *netgroups        = NULL;
-        dict_t            *hosts            = NULL;
-        int                ret              = -EINVAL;
-        gf_boolean_t       netgroups_failed = _gf_false;
-
-        GF_VALIDATE_OR_GOTO (GF_EXP, line, out);
-        GF_VALIDATE_OR_GOTO (GF_EXP, dir, out);
-
-        if (*line == '#' || *line == ' ' || *line == '\t'
-            || *line == '\0' || *line == '\n') {
-                ret = GF_EXP_PARSE_LINE_IGNORING;
-                goto out;
-        }
-
-        expdir = _export_dir_init ();
-        if (!expdir) {
-                *dir = NULL;
-                ret = -ENOMEM;
-                goto out;
-        }
-
-        /* Get the directory string from the line */
-        ret = __exp_line_dir_parse (line, &dirstr, ms);
-        if (ret < 0) {
-                gf_log (GF_EXP, GF_LOG_ERROR, "Parsing directory failed: %s",
-                        strerror (-ret));
-                /* If parsing the directory failed,
-                 * we should simply fail because there's
-                 * nothing else we can extract from the string to make
-                 * the data valuable.
-                 */
-                goto free_and_out;
-        }
-
-        /* Set the dir str */
-        expdir->dir_name = dirstr;
-
-        /* Parse the netgroup part of the string */
-        ret = __exp_line_ng_parse (line, &netgroups);
-        if (ret < 0) {
-                gf_log (GF_EXP, GF_LOG_ERROR, "Critical error: %s",
-                        strerror (-ret));
-                /* Return values less than 0
-                 * indicate critical failures (null parameters,
-                 * failure to allocate memory, etc).
-                 */
-                goto free_and_out;
-        }
-        if (ret != 0) {
-                if (ret == GF_EXP_PARSE_ITEM_FAILURE)
-                        gf_log (GF_EXP, GF_LOG_WARNING,
-                                "Error parsing netgroups for: %s", line);
-                /* Even though parsing failed for the netgroups we should let
-                 * host parsing proceed.
-                 */
-                netgroups_failed = _gf_true;
-        }
-
-        /* Parse the host part of the string */
-        ret = __exp_line_host_parse (line, &hosts);
-        if (ret < 0) {
-                gf_log (GF_EXP, GF_LOG_ERROR, "Critical error: %s",
-                        strerror (-ret));
-                goto free_and_out;
-        }
-        if (ret != 0) {
-                if (ret == GF_EXP_PARSE_ITEM_FAILURE)
-                        gf_log (GF_EXP, GF_LOG_WARNING,
-                                "Error parsing hosts for: %s", line);
-                /* If netgroups parsing failed, AND
-                 * host parsing failed, then theres something really
-                 * wrong with this line, so we're just going to
-                 * log it and fail out.
-                 */
-                if (netgroups_failed)
-                        goto free_and_out;
-        }
+    struct export_dir *expdir = NULL;
+    char *dirstr = NULL;
+    dict_t *netgroups = NULL;
+    dict_t *hosts = NULL;
+    int ret = -EINVAL;
+    gf_boolean_t netgroups_failed = _gf_false;
+
+    GF_VALIDATE_OR_GOTO(GF_EXP, line, out);
+    GF_VALIDATE_OR_GOTO(GF_EXP, dir, out);
+
+    if (*line == '#' || *line == ' ' || *line == '\t' || *line == '\0' ||
+        *line == '\n') {
+        ret = GF_EXP_PARSE_LINE_IGNORING;
+        goto out;
+    }
 
-        expdir->hosts = hosts;
-        expdir->netgroups = netgroups;
-        *dir = expdir;
+    expdir = _export_dir_init();
+    if (!expdir) {
+        *dir = NULL;
+        ret = -ENOMEM;
         goto out;
+    }
+
+    /* Get the directory string from the line */
+    ret = __exp_line_dir_parse(line, &dirstr, ms);
+    if (ret < 0) {
+        gf_msg(GF_EXP, GF_LOG_ERROR, 0, NFS_MSG_PARSE_DIR_FAIL,
+               "Parsing directory failed: %s", strerror(-ret));
+        /* If parsing the directory failed,
+         * we should simply fail because there's
+         * nothing else we can extract from the string to make
+         * the data valuable.
+         */
+        goto free_and_out;
+    }
+
+    /* Set the dir str */
+    expdir->dir_name = dirstr;
+
+    /* Parse the netgroup part of the string */
+    ret = __exp_line_ng_parse(line, &netgroups);
+    if (ret < 0) {
+        gf_msg(GF_EXP, GF_LOG_ERROR, -ret, NFS_MSG_PARSE_FAIL,
+               "Critical error: %s", strerror(-ret));
+        /* Return values less than 0
+         * indicate critical failures (null parameters,
+         * failure to allocate memory, etc).
+         */
+        goto free_and_out;
+    }
+    if (ret != 0) {
+        if (ret == GF_EXP_PARSE_ITEM_FAILURE)
+            /* Cannot change to gf_msg.
+             * gf_msg not giving output to STDOUT
+             * Bug id : BZ1215017
+             */
+            gf_log(GF_EXP, GF_LOG_WARNING, "Error parsing netgroups for: %s",
+                   line);
+        /* Even though parsing failed for the netgroups we should let
+         * host parsing proceed.
+         */
+        netgroups_failed = _gf_true;
+    }
+
+    /* Parse the host part of the string */
+    ret = __exp_line_host_parse(line, &hosts);
+    if (ret < 0) {
+        gf_msg(GF_EXP, GF_LOG_ERROR, -ret, NFS_MSG_PARSE_FAIL,
+               "Critical error: %s", strerror(-ret));
+        goto free_and_out;
+    }
+    if (ret != 0) {
+        if (ret == GF_EXP_PARSE_ITEM_FAILURE)
+            gf_msg(GF_EXP, GF_LOG_WARNING, 0, NFS_MSG_PARSE_FAIL,
+                   "Error parsing hosts for: %s", line);
+        /* If netgroups parsing failed, AND
+         * host parsing failed, then there's something really
+         * wrong with this line, so we're just going to
+         * log it and fail out.
+         */
+        if (netgroups_failed)
+            goto free_and_out;
+    }
+
+    expdir->hosts = hosts;
+    expdir->netgroups = netgroups;
+    *dir = expdir;
+    goto out;
 
 free_and_out:
-       _export_dir_deinit (expdir);
+    _export_dir_deinit(expdir);
 out:
-        return ret;
+    return ret;
 }
 
 struct export_item *
-exp_dir_get_netgroup (const struct export_dir *expdir, const char *netgroup)
+exp_dir_get_netgroup(const struct export_dir *expdir, const char *netgroup)
 {
-        struct export_item   *lookup_res = NULL;
-        data_t               *dict_res   = NULL;
+    struct export_item *lookup_res = NULL;
+    data_t *dict_res = NULL;
 
-        GF_VALIDATE_OR_GOTO (GF_EXP, expdir, out);
-        GF_VALIDATE_OR_GOTO (GF_EXP, netgroup, out);
+    GF_VALIDATE_OR_GOTO(GF_EXP, expdir, out);
+    GF_VALIDATE_OR_GOTO(GF_EXP, netgroup, out);
 
-        if (!expdir->netgroups)
-                goto out;
+    if (!expdir->netgroups)
+        goto out;
 
-        dict_res = dict_get (expdir->netgroups, (char *)netgroup);
-        if (!dict_res) {
-                gf_log (GF_EXP, GF_LOG_DEBUG, "%s not found for %s",
-                                netgroup, expdir->dir_name);
-        }
+    dict_res = dict_get(expdir->netgroups, (char *)netgroup);
+    if (!dict_res) {
+        gf_msg_debug(GF_EXP, 0, "%s not found for %s", netgroup,
+                     expdir->dir_name);
+        goto out;
+    }
 
-        lookup_res = (struct export_item *)dict_res->data;
+    lookup_res = (struct export_item *)dict_res->data;
 out:
-        return lookup_res;
+    return lookup_res;
 }
 /**
  * exp_dir_get_host -- Given a host string and an exports directory structure,
@@ -1230,35 +1263,33 @@ out:
  *          failure: NULL
  */
 struct export_item *
-exp_dir_get_host (const struct export_dir *expdir, const char *host)
+exp_dir_get_host(const struct export_dir *expdir, const char *host)
 {
-        struct export_item   *lookup_res = NULL;
-        data_t               *dict_res   = NULL;
+    struct export_item *lookup_res = NULL;
+    data_t *dict_res = NULL;
 
-        GF_VALIDATE_OR_GOTO (GF_EXP, expdir, out);
-        GF_VALIDATE_OR_GOTO (GF_EXP, host, out);
+    GF_VALIDATE_OR_GOTO(GF_EXP, expdir, out);
+    GF_VALIDATE_OR_GOTO(GF_EXP, host, out);
 
-        if (!expdir->hosts)
-                goto out;
+    if (!expdir->hosts)
+        goto out;
+
+    dict_res = dict_get(expdir->hosts, (char *)host);
+    if (!dict_res) {
+        gf_msg_debug(GF_EXP, 0, "%s not found for %s", host, expdir->dir_name);
 
-        dict_res = dict_get (expdir->hosts, (char *)host);
+        /* Check if wildcards are allowed for the host */
+        dict_res = dict_get(expdir->hosts, "*");
         if (!dict_res) {
-                gf_log (GF_EXP, GF_LOG_DEBUG, "%s not found for %s",
-                        host, expdir->dir_name);
-
-                /* Check if wildcards are allowed for the host */
-                dict_res = dict_get (expdir->hosts, "*");
-                if (!dict_res) {
-                        goto out;
-                }
+            goto out;
         }
+    }
 
-        lookup_res = (struct export_item *)dict_res->data;
+    lookup_res = (struct export_item *)dict_res->data;
 out:
-        return lookup_res;
+    return lookup_res;
 }
 
-
 /**
  * exp_file_get_dir -- Return an export dir given a directory name
  *                     Does a lookup from the dict in the file structure.
@@ -1270,38 +1301,37 @@ out:
  *           failure: NULL
  */
 struct export_dir *
-exp_file_get_dir (const struct exports_file *file, const char *dir)
+exp_file_get_dir(const struct exports_file *file, const char *dir)
 {
-        struct export_dir  *lookup_res = NULL;
-        data_t             *dict_res   = NULL;
-        char               *dirdup     = NULL;
-        size_t             dirlen      = 0;
+    struct export_dir *lookup_res = NULL;
+    data_t *dict_res = NULL;
+    char *dirdup = NULL;
+    size_t dirlen = 0;
 
-        GF_VALIDATE_OR_GOTO (GF_EXP, file, out);
-        GF_VALIDATE_OR_GOTO (GF_EXP, dir, out);
+    GF_VALIDATE_OR_GOTO(GF_EXP, file, out);
+    GF_VALIDATE_OR_GOTO(GF_EXP, dir, out);
 
-        dirlen = strlen (dir);
-        if (dirlen <= 0)
-                goto out;
+    dirlen = strlen(dir);
+    if (dirlen <= 0)
+        goto out;
 
-        dirdup = (char *)dir; /* Point at the directory */
+    dirdup = (char *)dir; /* Point at the directory */
 
-        /* If directories don't contain a leading slash */
-        if (*dir != '/') {
-                dirdup = alloca (dirlen + 2); /* Leading slash & null byte */
-                snprintf (dirdup, dirlen + 2, "/%s", dir);
-        }
+    /* If directories don't contain a leading slash */
+    if (*dir != '/') {
+        dirdup = alloca(dirlen + 2); /* Leading slash & null byte */
+        snprintf(dirdup, dirlen + 2, "/%s", dir);
+    }
 
-        dict_res = dict_get (file->exports_dict, dirdup);
-        if (!dict_res) {
-                gf_log (GF_EXP, GF_LOG_DEBUG, "%s not found in %s", dirdup,
-                        file->filename);
-                goto out;
-        }
+    dict_res = dict_get(file->exports_dict, dirdup);
+    if (!dict_res) {
+        gf_msg_debug(GF_EXP, 0, "%s not found in %s", dirdup, file->filename);
+        goto out;
+    }
 
-        lookup_res = (struct export_dir *)dict_res->data;
+    lookup_res = (struct export_dir *)dict_res->data;
 out:
-        return lookup_res;
+    return lookup_res;
 }
 
 /**
@@ -1323,129 +1353,132 @@ out:
  * Externally usable.
  */
 int
-exp_file_parse (const char *filepath, struct exports_file **expfile,
-                struct mount3_state *ms)
+exp_file_parse(const char *filepath, struct exports_file **expfile,
+               struct mount3_state *ms)
 {
-        FILE                *fp             = NULL;
-        struct exports_file *file           = NULL;
-        size_t               len            = 0;
-        int                  ret            = -EINVAL;
-        unsigned long        line_number    = 0;
-        char                *line           = NULL;
-        struct export_dir   *expdir         = NULL;
-
-        /* Sets whether we we should parse the entire file or just that which
-         * is present in the mount state */
-        gf_boolean_t    parse_complete_file = _gf_false;
-
-        GF_VALIDATE_OR_GOTO (GF_EXP, expfile, parse_done);
-
-        if (!ms) {
-                /* If mount state is null that means that we
-                 * should go through and parse the whole file
-                 * since we don't have anything to compare against.
-                 */
-                parse_complete_file = _gf_true;
+    FILE *fp = NULL;
+    struct exports_file *file = NULL;
+    size_t len = 0;
+    int ret = -EINVAL;
+    unsigned long line_number = 0;
+    char *line = NULL;
+    struct export_dir *expdir = NULL;
+
+    /* Sets whether we we should parse the entire file or just that which
+     * is present in the mount state */
+    gf_boolean_t parse_complete_file = _gf_false;
+
+    GF_VALIDATE_OR_GOTO(GF_EXP, expfile, parse_done);
+
+    if (!ms) {
+        /* If mount state is null that means that we
+         * should go through and parse the whole file
+         * since we don't have anything to compare against.
+         */
+        parse_complete_file = _gf_true;
+    }
+
+    fp = fopen(filepath, "r");
+    if (!fp) {
+        ret = -errno;
+        goto parse_done;
+    }
+
+    ret = _exp_init_parsers();
+    if (ret < 0)
+        goto parse_done;
+
+    /* Process the file line by line, with each line being parsed into
+     * an struct export_dir struct. If 'parse_complete_file' is set to TRUE
+     * then
+     */
+    while (getline(&line, &len, fp) != -1) {
+        line_number++;      /* Keeping track of line number allows us to
+                             * to log which line numbers were wrong
+                             */
+        strtok(line, "\n"); /* removes the newline character from
+                             * the line
+                             */
+
+        /* Parse the line from the file into an struct export_dir
+         * structure. The process is as follows:
+         * Given a line like :
+         * "/vol @test(sec=sys,rw,anonuid=0) 10.35.11.31(sec=sys,rw)"
+         *
+         * This function will allocate an export dir and set its name
+         * to '/vol', using the function _exp_line_dir_parse ().
+         *
+         * Then it will extract the netgroups from the line, in this
+         * case it would be '@test(sec=sys,rw,anonuid=0)', and set the
+         * item structure's name to '@test'.
+         * It will also extract the options from that string and parse
+         * them into an struct export_options which will be pointed
+         * to by the item structure. This will be put into a dict
+         * which will be pointed to by the export directory structure.
+         *
+         * The same process happens above for the host string
+         * '10.35.11.32(sec=sys,rw)'
+         */
+        ret = _exp_line_parse(line, &expdir, parse_complete_file, ms);
+        if (ret == -ENOMEM) {
+            /* If we get memory allocation errors, we really should
+             * not continue parsing, so just free the allocated
+             * memory and exit.
+             */
+            goto free_and_done;
         }
 
-        fp = fopen (filepath, "r");
-        if (!fp) {
-                ret = -errno;
-                goto parse_done;
+        if (ret < 0) {
+            gf_msg(GF_EXP, GF_LOG_ERROR, -ret, NFS_MSG_PARSE_FAIL,
+                   "Failed to parse line #%lu", line_number);
+            continue; /* Skip entering this line and continue */
         }
 
-        ret = _exp_init_parsers ();
-        if (ret < 0)
-                goto parse_done;
+        if (ret == GF_EXP_PARSE_LINE_IGNORING) {
+            /* This just means the line was empty or started with a
+             * '#' or a ' ' and we are ignoring it.
+             */
+            gf_msg_debug(GF_EXP, 0,
+                         "Ignoring line #%lu because it started "
+                         "with a %c",
+                         line_number, *line);
+            continue;
+        }
 
-        /* Process the file line by line, with each line being parsed into
-         * an struct export_dir struct. If 'parse_complete_file' is set to TRUE
-         * then
-         */
-        while (getline (&line, &len, fp) != -1) {
-                line_number++;  /* Keeping track of line number allows us to
-                                 * to log which line numbers were wrong
-                                 */
-                strtok (line, "\n");    /* removes the newline character from
-                                         * the line
-                                         */
-
-                /* Parse the line from the file into an struct export_dir
-                 * structure. The process is as follows:
-                 * Given a line like :
-                 * "/vol @test(sec=sys,rw,anonuid=0) 10.35.11.31(sec=sys,rw)"
-                 *
-                 * This function will allocate an export dir and set its name
-                 * to '/vol', using the function _exp_line_dir_parse ().
-                 *
-                 * Then it will extract the netgroups from the line, in this
-                 * case it would be '@test(sec=sys,rw,anonuid=0)', and set the
-                 * item structure's name to '@test'.
-                 * It will also extract the options from that string and parse
-                 * them into an struct export_options which will be pointed
-                 * to by the item structure. This will be put into a dict
-                 * which will be pointed to by the export directory structure.
-                 *
-                 * The same process happens above for the host string
-                 * '10.35.11.32(sec=sys,rw)'
-                 */
-                ret = _exp_line_parse (line, &expdir, parse_complete_file, ms);
-                if (ret == -ENOMEM) {
-                        /* If we get memory allocation errors, we really should
-                         * not continue parsing, so just free the allocated
-                         * memory and exit.
-                         */
-                        goto free_and_done;
-                }
-
-                if (ret < 0) {
-                        gf_log (GF_EXP, GF_LOG_ERROR,
-                                "Failed to parse line #%lu", line_number);
-                        continue; /* Skip entering this line and continue */
-                }
-
-                if (ret == GF_EXP_PARSE_LINE_IGNORING) {
-                        /* This just means the line was empty or started with a
-                         * '#' or a ' ' and we are ignoring it.
-                         */
-                        gf_log (GF_EXP, GF_LOG_DEBUG,
-                                "Ignoring line #%lu because it started "
-                                "with a %c", line_number, *line);
-                        continue;
-                }
-
-                if (!file) {
-                        file = _exports_file_init ();
-                        GF_CHECK_ALLOC_AND_LOG (GF_EXP, file, ret,
-                                                "Allocation error while "
-                                                "allocating file struct",
-                                                parse_done);
-
-                        file->filename = gf_strdup (filepath);
-                        GF_CHECK_ALLOC_AND_LOG (GF_EXP, file, ret,
-                                                "Allocation error while "
-                                                "duping filepath",
-                                                free_and_done);
-                }
-
-                /* If the parsing is successful store the export directory
-                 * in the file structure.
-                 */
-                _exp_file_insert (file, expdir);
+        if (!file) {
+            file = _exports_file_init();
+            GF_CHECK_ALLOC_AND_LOG(GF_EXP, file, ret,
+                                   "Allocation error while "
+                                   "allocating file struct",
+                                   free_and_done);
+
+            file->filename = gf_strdup(filepath);
+            GF_CHECK_ALLOC_AND_LOG(GF_EXP, file, ret,
+                                   "Allocation error while "
+                                   "duping filepath",
+                                   free_and_done);
         }
 
-        /* line got allocated through getline(), don't use GF_FREE() for it */
-        free (line);
+        /* If the parsing is successful store the export directory
+         * in the file structure.
+         */
+        _exp_file_insert(file, expdir);
+    }
 
-        *expfile = file;
-        goto parse_done;
+    /* line got allocated through getline(), don't use GF_FREE() for it */
+    free(line);
+
+    *expfile = file;
+    goto parse_done;
 
 free_and_done:
-        exp_file_deinit (file);
+    if (file)
+        exp_file_deinit(file);
+    _export_dir_deinit(expdir);
 
 parse_done:
-        if (fp)
-                fclose (fp);
-        _exp_deinit_parsers ();
-        return ret;
+    if (fp)
+        fclose(fp);
+    _exp_deinit_parsers();
+    return ret;
 }
diff --git a/xlators/nfs/server/src/exports.h b/xlators/nfs/server/src/exports.h
index 51a3cd668a4..ad35ad8c3a0 100644
--- a/xlators/nfs/server/src/exports.h
+++ b/xlators/nfs/server/src/exports.h
@@ -16,22 +16,22 @@
 #define _EXPORTS_H_
 
 #include "nfs-mem-types.h"
-#include "dict.h"
+#include <glusterfs/dict.h>
 #include "nfs.h"
 
-#define GF_EXP GF_NFS"-exports"
+#define GF_EXP GF_NFS "-exports"
 
-#define NETGROUP_REGEX_PATTERN  "(@([a-zA-Z0-9\\(=, .])+)())"
-#define HOSTNAME_REGEX_PATTERN  "[[:space:]]([a-zA-Z0-9.\\(=,*/)]+)"
-#define OPTIONS_REGEX_PATTERN   "([a-zA-Z0-9=\\.]+)"
+#define NETGROUP_REGEX_PATTERN "(@([a-zA-Z0-9\\(=, .])+)())"
+#define HOSTNAME_REGEX_PATTERN "[[:space:]]([a-zA-Z0-9.\\(=,*/:)-]+)"
+#define OPTIONS_REGEX_PATTERN "([a-zA-Z0-9=\\.]+)"
 
-#define NETGROUP_MAX_LEN        128
-#define FQDN_MAX_LEN            256
+#define NETGROUP_MAX_LEN 128
+#define FQDN_MAX_LEN 256
 
-#define SEC_OPTION_MAX          10
-#define UID_MAX_LEN             6
+#define SEC_OPTION_MAX 10
+#define UID_MAX_LEN 6
 
-#define DIR_MAX_LEN             1024
+#define DIR_MAX_LEN 1024
 
 /* The following 2 definitions are in mount3.h
  * but we don't want to include it because mount3.h
@@ -41,52 +41,53 @@
 struct mount3_state;
 
 extern struct mnt3_export *
-mnt3_mntpath_to_export (struct mount3_state *ms, const char *dirpath,
-                        gf_boolean_t export_parsing_match);
+mnt3_mntpath_to_export(struct mount3_state *ms, const char *dirpath,
+                       gf_boolean_t export_parsing_match);
 
 struct export_options {
-        gf_boolean_t    rw;                /* Read-write option */
-        gf_boolean_t    nosuid;            /* nosuid option */
-        gf_boolean_t    root;              /* root option */
-        char            *anon_uid;         /* anonuid option */
-        char            *sec_type;         /* X, for sec=X */
+    gf_boolean_t rw;     /* Read-write option */
+    gf_boolean_t nosuid; /* nosuid option */
+    gf_boolean_t root;   /* root option */
+    char *anon_uid;      /* anonuid option */
+    char *sec_type;      /* X, for sec=X */
 };
 
 struct export_item {
-        char                  *name;  /* Name of the export item */
-        struct export_options *opts;  /* NFS Options */
+    char *name;                  /* Name of the export item */
+    struct export_options *opts; /* NFS Options */
+    GF_REF_DECL;
 };
 
 struct export_dir {
-        char    *dir_name;      /* Directory */
-        dict_t  *netgroups;     /* Dict of netgroups */
-        dict_t  *hosts;         /* Dict of hosts */
+    char *dir_name;    /* Directory */
+    dict_t *netgroups; /* Dict of netgroups */
+    dict_t *hosts;     /* Dict of hosts */
 };
 
 struct exports_file {
-        char    *filename;      /* Filename */
-        dict_t  *exports_dict;  /* Dict of export_dir_t */
-        dict_t  *exports_map;   /* Map of SuperFastHash(<export>) -> expdir */
+    char *filename;       /* Filename */
+    dict_t *exports_dict; /* Dict of export_dir_t */
+    dict_t *exports_map;  /* Map of SuperFastHash(<export>) -> expdir */
 };
 
 void
-exp_file_deinit (struct exports_file *expfile);
+exp_file_deinit(struct exports_file *expfile);
 
 int
-exp_file_parse (const char *filepath, struct exports_file **expfile,
-                struct mount3_state *ms);
+exp_file_parse(const char *filepath, struct exports_file **expfile,
+               struct mount3_state *ms);
 
 struct export_dir *
-exp_file_get_dir (const struct exports_file *file, const char *dir);
+exp_file_get_dir(const struct exports_file *file, const char *dir);
 
 struct export_item *
-exp_dir_get_host (const struct export_dir *expdir, const char *host);
+exp_dir_get_host(const struct export_dir *expdir, const char *host);
 
 struct export_item *
-exp_dir_get_netgroup (const struct export_dir *expdir, const char *netgroup);
+exp_dir_get_netgroup(const struct export_dir *expdir, const char *netgroup);
 
 struct export_dir *
-exp_file_dir_from_uuid (const struct exports_file *file,
-                        const uuid_t export_uuid);
+exp_file_dir_from_uuid(const struct exports_file *file,
+                       const uuid_t export_uuid);
 
-#endif  /* _EXPORTS_H_ */
+#endif /* _EXPORTS_H_ */
diff --git a/xlators/nfs/server/src/mount3-auth.c b/xlators/nfs/server/src/mount3-auth.c
index 25246f96ab1..10e57c84cdb 100644
--- a/xlators/nfs/server/src/mount3-auth.c
+++ b/xlators/nfs/server/src/mount3-auth.c
@@ -38,7 +38,8 @@
 #include "mount3-auth.h"
 #include "exports.h"
 #include "netgroups.h"
-#include "mem-pool.h"
+#include <glusterfs/mem-pool.h>
+#include "nfs-messages.h"
 
 /**
  * mnt3_auth_params_init -- Initialize the mount3 authorization parameters
@@ -53,19 +54,18 @@
  * For external use.
  */
 struct mnt3_auth_params *
-mnt3_auth_params_init (struct mount3_state *ms)
+mnt3_auth_params_init(struct mount3_state *ms)
 {
-        struct mnt3_auth_params *auth_params = NULL;
+    struct mnt3_auth_params *auth_params = NULL;
 
-        auth_params = GF_MALLOC (sizeof (*auth_params),
-                                 gf_nfs_mt_mnt3_auth_params);
-        GF_VALIDATE_OR_GOTO (GF_MNT_AUTH, auth_params, out);
+    auth_params = GF_MALLOC(sizeof(*auth_params), gf_nfs_mt_mnt3_auth_params);
+    GF_VALIDATE_OR_GOTO(GF_MNT_AUTH, auth_params, out);
 
-        auth_params->ngfile = NULL;
-        auth_params->expfile = NULL;
-        auth_params->ms = ms;
+    auth_params->ngfile = NULL;
+    auth_params->expfile = NULL;
+    auth_params->ms = ms;
 out:
-        return auth_params;
+    return auth_params;
 }
 
 /**
@@ -76,23 +76,23 @@ out:
  * For external use.
  */
 void
-mnt3_auth_params_deinit (struct mnt3_auth_params *auth_params)
+mnt3_auth_params_deinit(struct mnt3_auth_params *auth_params)
 {
-        if (!auth_params)
-                goto out;
-
-        /* Atomically set the auth params in the mount state to NULL
-         * so subsequent fops will be denied while the auth params
-         * are being cleaned up.
-         */
-        (void)__sync_lock_test_and_set (&auth_params->ms->auth_params, NULL);
-
-        ng_file_deinit (auth_params->ngfile);
-        exp_file_deinit (auth_params->expfile);
-        auth_params->ms = NULL;
-        GF_FREE (auth_params);
+    if (!auth_params)
+        goto out;
+
+    /* Atomically set the auth params in the mount state to NULL
+     * so subsequent fops will be denied while the auth params
+     * are being cleaned up.
+     */
+    (void)__sync_lock_test_and_set(&auth_params->ms->auth_params, NULL);
+
+    ng_file_deinit(auth_params->ngfile);
+    exp_file_deinit(auth_params->expfile);
+    auth_params->ms = NULL;
+    GF_FREE(auth_params);
 out:
-        return;
+    return;
 }
 
 /**
@@ -107,31 +107,33 @@ out:
  * For external use.
  */
 int
-mnt3_auth_set_exports_auth (struct mnt3_auth_params *auth_params,
-                            const char *filename)
+mnt3_auth_set_exports_auth(struct mnt3_auth_params *auth_params,
+                           const char *filename)
 {
-        struct exports_file *expfile = NULL;
-        struct exports_file *oldfile = NULL;
-        int                  ret     = -EINVAL;
-
-        /* Validate args */
-        GF_VALIDATE_OR_GOTO (GF_MNT_AUTH, auth_params, out);
-        GF_VALIDATE_OR_GOTO (GF_MNT_AUTH, filename, out);
-
-        /* Parse the exports file and set the auth parameter */
-        ret = exp_file_parse (filename, &expfile, auth_params->ms);
-        if (ret < 0) {
-                gf_log (GF_MNT_AUTH, GF_LOG_ERROR, "Failed to load & parse file"
-                        " %s, see logs for more information", filename);
-                goto out;
-        }
-
-        /* Atomically set the file pointer */
-        oldfile = __sync_lock_test_and_set (&auth_params->expfile, expfile);
-        exp_file_deinit (oldfile);
-        ret = 0;
+    struct exports_file *expfile = NULL;
+    struct exports_file *oldfile = NULL;
+    int ret = -EINVAL;
+
+    /* Validate args */
+    GF_VALIDATE_OR_GOTO(GF_MNT_AUTH, auth_params, out);
+    GF_VALIDATE_OR_GOTO(GF_MNT_AUTH, filename, out);
+
+    /* Parse the exports file and set the auth parameter */
+    ret = exp_file_parse(filename, &expfile, auth_params->ms);
+    if (ret < 0) {
+        gf_msg(GF_MNT_AUTH, GF_LOG_ERROR, 0, NFS_MSG_LOAD_PARSE_ERROR,
+               "Failed to load & parse file"
+               " %s, see logs for more information",
+               filename);
+        goto out;
+    }
+
+    /* Atomically set the file pointer */
+    oldfile = __sync_lock_test_and_set(&auth_params->expfile, expfile);
+    exp_file_deinit(oldfile);
+    ret = 0;
 out:
-        return ret;
+    return ret;
 }
 
 /**
@@ -146,31 +148,33 @@ out:
  * For external use.
  */
 int
-mnt3_auth_set_netgroups_auth (struct mnt3_auth_params *auth_params,
-                              const char *filename)
+mnt3_auth_set_netgroups_auth(struct mnt3_auth_params *auth_params,
+                             const char *filename)
 {
-        struct netgroups_file *ngfile  = NULL;
-        struct netgroups_file *oldfile = NULL;
-        int                    ret     = -EINVAL;
-
-        /* Validate args */
-        GF_VALIDATE_OR_GOTO (GF_MNT_AUTH, auth_params, out);
-        GF_VALIDATE_OR_GOTO (GF_MNT_AUTH, filename, out);
-
-        ngfile = ng_file_parse (filename);
-        if (!ngfile) {
-                gf_log (GF_MNT_AUTH, GF_LOG_ERROR, "Failed to load file"
-                        " %s, see logs for more information", filename);
-                ret = -1;
-                goto out;
-        }
-
-        /* Atomically set the file pointer */
-        oldfile = __sync_lock_test_and_set (&auth_params->ngfile, ngfile);
-        ng_file_deinit (oldfile);
-        ret = 0;
+    struct netgroups_file *ngfile = NULL;
+    struct netgroups_file *oldfile = NULL;
+    int ret = -EINVAL;
+
+    /* Validate args */
+    GF_VALIDATE_OR_GOTO(GF_MNT_AUTH, auth_params, out);
+    GF_VALIDATE_OR_GOTO(GF_MNT_AUTH, filename, out);
+
+    ngfile = ng_file_parse(filename);
+    if (!ngfile) {
+        gf_msg(GF_MNT_AUTH, GF_LOG_ERROR, 0, NFS_MSG_LOAD_PARSE_ERROR,
+               "Failed to load file %s, see logs for more "
+               "information",
+               filename);
+        ret = -1;
+        goto out;
+    }
+
+    /* Atomically set the file pointer */
+    oldfile = __sync_lock_test_and_set(&auth_params->ngfile, ngfile);
+    ng_file_deinit(oldfile);
+    ret = 0;
 out:
-        return ret;
+    return ret;
 }
 
 /* Struct used to pass parameters to
@@ -178,8 +182,8 @@ out:
  * checks if an IP matches a subnet
  */
 struct _mnt3_subnet_match_s {
-        char                  *ip;   /* IP address to match */
-        struct export_item   **host; /* Host structure to set */
+    char *ip;                  /* IP address to match */
+    struct export_item **host; /* Host structure to set */
 };
 
 /**
@@ -193,40 +197,40 @@ struct _mnt3_subnet_match_s {
  *
  */
 static int
-_mnt3_auth_subnet_match (dict_t *dict, char *key, data_t *val, void *tmp)
+_mnt3_auth_subnet_match(dict_t *dict, char *key, data_t *val, void *tmp)
 {
-        struct  _mnt3_subnet_match_s *match = NULL;
-
-        match = (struct _mnt3_subnet_match_s *)tmp;
+    struct _mnt3_subnet_match_s *match = NULL;
 
-        if (!match)
-                return 0;
+    match = (struct _mnt3_subnet_match_s *)tmp;
 
-        if (!match->host)
-                return 0;
-
-        if (!match->ip)
-                return 0;
+    if (!match)
+        return 0;
 
-        /* Already found the host */
-        if (*(match->host))
-                return 0;
+    if (!match->host)
+        return 0;
 
-        /* Don't process anything that's not in CIDR */
-        if (!strchr (key, '/'))
-                return 0;
+    if (!match->ip)
+        return 0;
 
-        /* Strip out leading whitespaces */
-        while (*key == ' ')
-                key++;
+    /* Already found the host */
+    if (*(match->host))
+        return 0;
 
-        /* If we found that the IP was in the network, set the host
-         * to point to the value in the dict.
-         */
-        if (gf_is_ip_in_net (key, match->ip)) {
-                *(match->host) = (struct export_item *)val->data;
-        }
+    /* Don't process anything that's not in CIDR */
+    if (!strchr(key, '/'))
         return 0;
+
+    /* Strip out leading whitespaces */
+    while (*key == ' ')
+        key++;
+
+    /* If we found that the IP was in the network, set the host
+     * to point to the value in the dict.
+     */
+    if (gf_is_ip_in_net(key, match->ip)) {
+        *(match->host) = (struct export_item *)val->data;
+    }
+    return 0;
 }
 
 /**
@@ -253,67 +257,69 @@ _mnt3_auth_subnet_match (dict_t *dict, char *key, data_t *val, void *tmp)
  * Not for external use.
  */
 static struct export_item *
-_mnt3_auth_check_host_in_export (const struct exports_file *file,
-                                 const char *dir, const char *host,
-                                 struct nfs3_fh *fh)
+_mnt3_auth_check_host_in_export(const struct exports_file *file,
+                                const char *dir, const char *host,
+                                struct nfs3_fh *fh)
 {
-        struct export_dir           *expdir = NULL;
-        struct export_item          *host_s = NULL;
-        struct _mnt3_subnet_match_s  snet_match_s = {0, };
-
-        /* Validate args */
-        GF_VALIDATE_OR_GOTO (GF_MNT_AUTH, file, out);
-        GF_VALIDATE_OR_GOTO (GF_MNT_AUTH, host, out);
-
-        /* If the filehandle is defined, use that to perform authentication.
-         * All file operations that need authentication must follow this
-         * code path.
-         */
-        if (fh) {
-                expdir = exp_file_dir_from_uuid (file, fh->mountid);
-                if (!expdir)
-                        goto out;
-        } else {
-                /* Get the exports directory from the exports file */
-                expdir = exp_file_get_dir (file, dir);
-                if (!expdir)
-                        goto out;
-        }
-
-        /* Extract the host from the export directory */
-        host_s = exp_dir_get_host (expdir, host);
-        if (!host_s)
-                goto subnet_match;
-        else
-                goto out;
-
-        /* If the host is not found, we need to walk through the hosts
-         * in the exports directory and see if any of the "hosts" are actually
-         * networks (e.g. 10.5.153.0/24). If they are we should match the
-         * incoming network.
-         */
+    struct export_dir *expdir = NULL;
+    struct export_item *host_s = NULL;
+    struct _mnt3_subnet_match_s snet_match_s = {
+        0,
+    };
+
+    /* Validate args */
+    GF_VALIDATE_OR_GOTO(GF_MNT_AUTH, file, out);
+    GF_VALIDATE_OR_GOTO(GF_MNT_AUTH, host, out);
+
+    /* If the filehandle is defined, use that to perform authentication.
+     * All file operations that need authentication must follow this
+     * code path.
+     */
+    if (fh) {
+        expdir = exp_file_dir_from_uuid(file, fh->mountid);
+        if (!expdir)
+            goto out;
+    } else {
+        /* Get the exports directory from the exports file */
+        expdir = exp_file_get_dir(file, dir);
+        if (!expdir)
+            goto out;
+    }
+
+    /* Extract the host from the export directory */
+    host_s = exp_dir_get_host(expdir, host);
+    if (!host_s)
+        goto subnet_match;
+    else
+        goto out;
+
+    /* If the host is not found, we need to walk through the hosts
+     * in the exports directory and see if any of the "hosts" are actually
+     * networks (e.g. 10.5.153.0/24). If they are we should match the
+     * incoming network.
+     */
 subnet_match:
-        if (!expdir->hosts)
-                goto out;
-        snet_match_s.ip = (char *)host;
-        snet_match_s.host = &host_s;
-        dict_foreach (expdir->hosts, _mnt3_auth_subnet_match, &snet_match_s);
+    if (!expdir->hosts)
+        goto out;
+    snet_match_s.ip = (char *)host;
+    snet_match_s.host = &host_s;
+    dict_foreach(expdir->hosts, _mnt3_auth_subnet_match, &snet_match_s);
 out:
-        return host_s;
+    return host_s;
 }
 
 /* This struct represents all the parameters necessary to search through a
  * netgroups file to find a host.
  */
 struct ng_auth_search {
-        const char                  *search_for; /* strings to search for */
-        gf_boolean_t                 found;      /* mark true once found */
-        const struct netgroups_file *file;       /* netgroups file to search */
-        const char                  *expdir;
-        struct export_item          *expitem;    /* pointer to the export */
-        const struct exports_file   *expfile;
-        gf_boolean_t                 _is_host_dict; /* searching a host dict? */
-        struct netgroup_entry       *found_entry;   /* the entry we found! */
+    const char *search_for;            /* strings to search for */
+    gf_boolean_t found;                /* mark true once found */
+    const struct netgroups_file *file; /* netgroups file to search */
+    const char *expdir;
+    struct export_item *expitem; /* pointer to the export */
+    const struct exports_file *expfile;
+    gf_boolean_t _is_host_dict;         /* searching a host dict? */
+    struct netgroup_entry *found_entry; /* the entry we found! */
 };
 
 /**
@@ -329,65 +335,65 @@ struct ng_auth_search {
  * Not for external use.
  */
 static int
-__netgroup_dict_search (dict_t *dict, char *key, data_t *val, void *data)
+__netgroup_dict_search(dict_t *dict, char *key, data_t *val, void *data)
 {
-        struct ng_auth_search *ngsa    = NULL;
-        struct netgroup_entry *ngentry = NULL;
-        data_t                *hdata   = NULL;
-
-        /* 'ngsa' is the search params */
-        ngsa    = (struct ng_auth_search *)data;
-        ngentry = (struct netgroup_entry *)val->data;
-
-        if (ngsa->_is_host_dict) {
-                /* If are on a host dict, we can simply hash the search key
-                 * against the host dict and see if we find anything.
-                 */
-                hdata = dict_get (dict, (char *)ngsa->search_for);
-                if (hdata) {
-                        /* If it was found, log the message, mark the search
-                         * params dict as found and return.
-                         */
-                        gf_log (GF_MNT_AUTH, GF_LOG_DEBUG,
-                                "key %s was hashed and found", key);
-                        ngsa->found = _gf_true;
-                        ngsa->found_entry = (struct netgroup_entry *)hdata->data;
-                        goto out;
-                }
-        }
+    struct ng_auth_search *ngsa = NULL;
+    struct netgroup_entry *ngentry = NULL;
+    data_t *hdata = NULL;
 
-        /* If the key is what we are searching for, mark the item as
-         * found and return.
-         */
-        if (strcmp (key, ngsa->search_for) == 0) {
-                ngsa->found = _gf_true;
-                ngsa->found_entry = ngentry;
-                goto out;
-        }
+    /* 'ngsa' is the search params */
+    ngsa = (struct ng_auth_search *)data;
+    ngentry = (struct netgroup_entry *)val->data;
 
-        /* If we have a netgroup hosts dict, then search the dict using this
-         * same function.
+    if (ngsa->_is_host_dict) {
+        /* If are on a host dict, we can simply hash the search key
+         * against the host dict and see if we find anything.
          */
-        if (ngentry->netgroup_hosts) {
-                ngsa->_is_host_dict = _gf_true;
-                dict_foreach (ngentry->netgroup_hosts, __netgroup_dict_search,
-                                                        ngsa);
-        }
-
-        /* If that search was successful, just return */
-        if (ngsa->found)
-                goto out;
-
-        /* If we have a netgroup dict, then search the dict using this same
-         * function.
-         */
-        if (ngentry->netgroup_ngs) {
-                ngsa->_is_host_dict = _gf_false;
-                dict_foreach (ngentry->netgroup_ngs, __netgroup_dict_search,
-                                                        ngsa);
+        hdata = dict_get(dict, (char *)ngsa->search_for);
+        if (hdata) {
+            /* If it was found, log the message, mark the search
+             * params dict as found and return.
+             */
+            gf_msg_debug(GF_MNT_AUTH, errno,
+                         "key %s was hashed "
+                         "and found",
+                         key);
+            ngsa->found = _gf_true;
+            ngsa->found_entry = (struct netgroup_entry *)hdata->data;
+            goto out;
         }
+    }
+
+    /* If the key is what we are searching for, mark the item as
+     * found and return.
+     */
+    if (strcmp(key, ngsa->search_for) == 0) {
+        ngsa->found = _gf_true;
+        ngsa->found_entry = ngentry;
+        goto out;
+    }
+
+    /* If we have a netgroup hosts dict, then search the dict using this
+     * same function.
+     */
+    if (ngentry->netgroup_hosts) {
+        ngsa->_is_host_dict = _gf_true;
+        dict_foreach(ngentry->netgroup_hosts, __netgroup_dict_search, ngsa);
+    }
+
+    /* If that search was successful, just return */
+    if (ngsa->found)
+        goto out;
+
+    /* If we have a netgroup dict, then search the dict using this same
+     * function.
+     */
+    if (ngentry->netgroup_ngs) {
+        ngsa->_is_host_dict = _gf_false;
+        dict_foreach(ngentry->netgroup_ngs, __netgroup_dict_search, ngsa);
+    }
 out:
-        return 0;
+    return 0;
 }
 
 /**
@@ -413,54 +419,51 @@ out:
  * Not for external use.
  */
 static int
-__export_dir_lookup_netgroup (dict_t *dict, char *key, data_t *val,
-                                void *data)
+__export_dir_lookup_netgroup(dict_t *dict, char *key, data_t *val, void *data)
 {
-        struct ng_auth_search *ngsa    = NULL; /* Search params */
-        struct netgroups_file *nfile   = NULL; /* Netgroups file to search */
-        struct netgroup_entry *ngentry = NULL; /* Entry in the netgroups file */
-        struct export_dir     *tmpdir  = NULL;
-
-        ngsa  = (struct ng_auth_search *)data;
-        nfile = (struct netgroups_file *)ngsa->file;
-
-        GF_ASSERT ((*key == '@'));
-
-        /* We use ++key here because keys start with '@' for ngs */
-        ngentry = ng_file_get_netgroup (nfile, (key + 1));
-        if (!ngentry) {
-                gf_log (GF_MNT_AUTH, GF_LOG_DEBUG, "%s not found in %s",
-                        key, nfile->filename);
-                goto out;
-        }
-
-        tmpdir = exp_file_get_dir (ngsa->expfile, ngsa->expdir);
-        if (!tmpdir)
-                goto out;
-
-        ngsa->expitem = exp_dir_get_netgroup (tmpdir, key);
-        if (!ngsa->expitem)
-                goto out;
-
-        /* Run through the host dict */
-        if (ngentry->netgroup_hosts) {
-                ngsa->_is_host_dict = _gf_true;
-                dict_foreach (ngentry->netgroup_hosts, __netgroup_dict_search,
-                              ngsa);
-        }
-
-        /* If the above search was successful, just return */
-        if (ngsa->found)
-                goto out;
-
-        /* Run through the netgroups dict */
-        if (ngentry->netgroup_ngs) {
-                ngsa->_is_host_dict = _gf_false;
-                dict_foreach (ngentry->netgroup_ngs, __netgroup_dict_search,
-                              ngsa);
-        }
+    struct ng_auth_search *ngsa = NULL;    /* Search params */
+    struct netgroups_file *nfile = NULL;   /* Netgroups file to search */
+    struct netgroup_entry *ngentry = NULL; /* Entry in the netgroups file */
+    struct export_dir *tmpdir = NULL;
+
+    ngsa = (struct ng_auth_search *)data;
+    nfile = (struct netgroups_file *)ngsa->file;
+
+    GF_ASSERT((*key == '@'));
+
+    /* We use ++key here because keys start with '@' for ngs */
+    ngentry = ng_file_get_netgroup(nfile, (key + 1));
+    if (!ngentry) {
+        gf_msg_debug(GF_MNT_AUTH, 0, "%s not found in %s", key,
+                     nfile->filename);
+        goto out;
+    }
+
+    tmpdir = exp_file_get_dir(ngsa->expfile, ngsa->expdir);
+    if (!tmpdir)
+        goto out;
+
+    ngsa->expitem = exp_dir_get_netgroup(tmpdir, key);
+    if (!ngsa->expitem)
+        goto out;
+
+    /* Run through the host dict */
+    if (ngentry->netgroup_hosts) {
+        ngsa->_is_host_dict = _gf_true;
+        dict_foreach(ngentry->netgroup_hosts, __netgroup_dict_search, ngsa);
+    }
+
+    /* If the above search was successful, just return */
+    if (ngsa->found)
+        goto out;
+
+    /* Run through the netgroups dict */
+    if (ngentry->netgroup_ngs) {
+        ngsa->_is_host_dict = _gf_false;
+        dict_foreach(ngentry->netgroup_ngs, __netgroup_dict_search, ngsa);
+    }
 out:
-        return 0;
+    return 0;
 }
 
 /**
@@ -473,25 +476,26 @@ out:
  * @nfile : The netgroups file to set
  *
  */
-void _mnt3_auth_setup_search_params (struct ng_auth_search *params,
-                                     const char *host, const char *dir,
-                                     const struct netgroups_file *nfile,
-                                     const struct exports_file *expfile)
+void
+_mnt3_auth_setup_search_params(struct ng_auth_search *params, const char *host,
+                               const char *dir,
+                               const struct netgroups_file *nfile,
+                               const struct exports_file *expfile)
 {
-        GF_VALIDATE_OR_GOTO (GF_MNT_AUTH, params, out);
-        GF_VALIDATE_OR_GOTO (GF_MNT_AUTH, host, out);
-        GF_VALIDATE_OR_GOTO (GF_MNT_AUTH, nfile, out);
-
-        params->search_for = host;
-        params->found = _gf_false;
-        params->file = nfile;
-        params->_is_host_dict = _gf_false;
-        params->found_entry = NULL;
-        params->expitem = NULL;
-        params->expfile = expfile;
-        params->expdir = dir;
+    GF_VALIDATE_OR_GOTO(GF_MNT_AUTH, params, out);
+    GF_VALIDATE_OR_GOTO(GF_MNT_AUTH, host, out);
+    GF_VALIDATE_OR_GOTO(GF_MNT_AUTH, nfile, out);
+
+    params->search_for = host;
+    params->found = _gf_false;
+    params->file = nfile;
+    params->_is_host_dict = _gf_false;
+    params->found_entry = NULL;
+    params->expitem = NULL;
+    params->expfile = expfile;
+    params->expdir = dir;
 out:
-        return;
+    return;
 }
 
 /**
@@ -518,43 +522,44 @@ out:
  * Not for external use.
  */
 static struct netgroup_entry *
-_mnt3_auth_check_host_in_netgroup (const struct mnt3_auth_params *auth_params,
-                                   struct nfs3_fh *fh, const char *host,
-                                   const char *dir, struct export_item **item)
+_mnt3_auth_check_host_in_netgroup(const struct mnt3_auth_params *auth_params,
+                                  struct nfs3_fh *fh, const char *host,
+                                  const char *dir, struct export_item **item)
 {
-        struct export_dir     *expdir      = NULL;
-        struct ng_auth_search  ngsa        = {0, };
-        struct netgroup_entry *found_entry = NULL;
-        struct exports_file   *efile       = auth_params->expfile;
-        struct netgroups_file *nfile       = auth_params->ngfile;
-
-        /* Validate args */
-        GF_VALIDATE_OR_GOTO (GF_MNT_AUTH, nfile, out);
-        GF_VALIDATE_OR_GOTO (GF_MNT_AUTH, efile, out);
-        GF_VALIDATE_OR_GOTO (GF_MNT_AUTH, host, out);
-        GF_VALIDATE_OR_GOTO (GF_MNT_AUTH, item, out);
-
-        if (fh) {
-                expdir = exp_file_dir_from_uuid (efile, fh->mountid);
-                if (!expdir)
-                        goto out;
-        } else {
-                /* Get the exports directory */
-                expdir = exp_file_get_dir (efile, dir);
-                if (!expdir)
-                        goto out;
-        }
-
-        /* Setup search struct */
-        _mnt3_auth_setup_search_params (&ngsa, host, expdir->dir_name, nfile,
-                                        efile);
-
-        /* Do the search */
-        dict_foreach (expdir->netgroups, __export_dir_lookup_netgroup, &ngsa);
-        found_entry = ngsa.found_entry;
-        *item = ngsa.expitem;
+    struct export_dir *expdir = NULL;
+    struct ng_auth_search ngsa = {
+        0,
+    };
+    struct netgroup_entry *found_entry = NULL;
+    struct exports_file *efile = auth_params->expfile;
+    struct netgroups_file *nfile = auth_params->ngfile;
+
+    /* Validate args */
+    GF_VALIDATE_OR_GOTO(GF_MNT_AUTH, nfile, out);
+    GF_VALIDATE_OR_GOTO(GF_MNT_AUTH, efile, out);
+    GF_VALIDATE_OR_GOTO(GF_MNT_AUTH, host, out);
+    GF_VALIDATE_OR_GOTO(GF_MNT_AUTH, item, out);
+
+    if (fh) {
+        expdir = exp_file_dir_from_uuid(efile, fh->mountid);
+        if (!expdir)
+            goto out;
+    } else {
+        /* Get the exports directory */
+        expdir = exp_file_get_dir(efile, dir);
+        if (!expdir)
+            goto out;
+    }
+
+    /* Setup search struct */
+    _mnt3_auth_setup_search_params(&ngsa, host, expdir->dir_name, nfile, efile);
+
+    /* Do the search */
+    dict_foreach(expdir->netgroups, __export_dir_lookup_netgroup, &ngsa);
+    found_entry = ngsa.found_entry;
+    *item = ngsa.expitem;
 out:
-        return found_entry;
+    return found_entry;
 }
 
 /**
@@ -567,22 +572,22 @@ out:
  *
  */
 int
-check_rw_access (struct export_item *item)
+check_rw_access(struct export_item *item)
 {
-        struct export_options *opts   = NULL;
-        int                    ret    = -EROFS;
+    struct export_options *opts = NULL;
+    int ret = -EROFS;
 
-        if (!item)
-                goto out;
+    if (!item)
+        goto out;
 
-        opts = item->opts;
-        if (!opts)
-                goto out;
+    opts = item->opts;
+    if (!opts)
+        goto out;
 
-        if (opts->rw)
-                ret = 0;
+    if (opts->rw)
+        ret = 0;
 out:
-        return ret;
+    return ret;
 }
 
 /**
@@ -606,36 +611,32 @@ out:
  *          -EROFS  for unauthorized write operations (rm, mkdir, write)  *
  */
 int
-mnt3_auth_host (const struct mnt3_auth_params *auth_params, const char *host,
-                struct nfs3_fh *fh, const char *dir, gf_boolean_t is_write_op,
-                struct export_item **save_item)
+mnt3_auth_host(const struct mnt3_auth_params *auth_params, const char *host,
+               struct nfs3_fh *fh, const char *dir, gf_boolean_t is_write_op,
+               struct export_item **save_item)
 {
-        int                  auth_status_code = -EACCES;
-        struct export_item  *item             = NULL;
-
-        GF_VALIDATE_OR_GOTO (GF_MNT_AUTH, auth_params, out);
-        GF_VALIDATE_OR_GOTO (GF_MNT_AUTH, host, out);
-
-        /* Find the host in the exports file */
-        item = _mnt3_auth_check_host_in_export (auth_params->expfile, dir,
-                                                    host, fh);
-        if (item) {
-                auth_status_code = (is_write_op) ?
-                                   check_rw_access (item) : 0;
-                goto out;
-        }
+    int auth_status_code = -EACCES;
+    struct export_item *item = NULL;
 
-        /* Find the host in the netgroups file for the exports directory */
-        if (_mnt3_auth_check_host_in_netgroup (auth_params, fh, host, dir,
-                                               &item)) {
-                auth_status_code = (is_write_op) ?
-                                   check_rw_access (item) : 0;
-                goto out;
-        }
+    GF_VALIDATE_OR_GOTO(GF_MNT_AUTH, auth_params, out);
+    GF_VALIDATE_OR_GOTO(GF_MNT_AUTH, host, out);
+
+    /* Find the host in the exports file */
+    item = _mnt3_auth_check_host_in_export(auth_params->expfile, dir, host, fh);
+    if (item) {
+        auth_status_code = (is_write_op) ? check_rw_access(item) : 0;
+        goto out;
+    }
+
+    /* Find the host in the netgroups file for the exports directory */
+    if (_mnt3_auth_check_host_in_netgroup(auth_params, fh, host, dir, &item)) {
+        auth_status_code = (is_write_op) ? check_rw_access(item) : 0;
+        goto out;
+    }
 
 out:
-        if (save_item)
-                *save_item = item;
+    if (save_item)
+        *save_item = item;
 
-        return auth_status_code;
+    return auth_status_code;
 }
diff --git a/xlators/nfs/server/src/mount3-auth.h b/xlators/nfs/server/src/mount3-auth.h
index b25d4724fac..e50af5847d3 100644
--- a/xlators/nfs/server/src/mount3-auth.h
+++ b/xlators/nfs/server/src/mount3-auth.h
@@ -21,39 +21,39 @@
 #include "mount3.h"
 #include "nfs.h"
 
-#define GF_MNT_AUTH GF_NFS"-mount3-auth"
+#define GF_MNT_AUTH GF_NFS "-mount3-auth"
 
 struct mnt3_auth_params {
-        struct netgroups_file *ngfile;  /* The netgroup file to auth against */
-        struct exports_file   *expfile; /* The export file to auth against */
-        struct mount3_state   *ms;      /* The mount state that owns this */
+    struct netgroups_file *ngfile; /* The netgroup file to auth against */
+    struct exports_file *expfile;  /* The export file to auth against */
+    struct mount3_state *ms;       /* The mount state that owns this */
 };
 
 /* Initialize auth params struct */
 struct mnt3_auth_params *
-mnt3_auth_params_init (struct mount3_state *ms);
+mnt3_auth_params_init(struct mount3_state *ms);
 
 /* Set the netgroups file to use in the auth */
 int
-mnt3_auth_set_netgroups_auth (struct mnt3_auth_params *aps,
-                              const char *filename);
+mnt3_auth_set_netgroups_auth(struct mnt3_auth_params *aps,
+                             const char *filename);
 
 /* Set the exports file to use in the auth */
 int
-mnt3_auth_set_exports_auth (struct mnt3_auth_params *aps, const char *filename);
+mnt3_auth_set_exports_auth(struct mnt3_auth_params *aps, const char *filename);
 
 /* Check if a host is authorized to perform a mount / nfs-fop */
 int
-mnt3_auth_host (const struct mnt3_auth_params *aps, const char *host,
-                struct nfs3_fh *fh, const char *dir, gf_boolean_t is_write_op,
-                struct export_item **save_item);
+mnt3_auth_host(const struct mnt3_auth_params *aps, const char *host,
+               struct nfs3_fh *fh, const char *dir, gf_boolean_t is_write_op,
+               struct export_item **save_item);
 
 /* Free resources used by the auth params struct */
 void
-mnt3_auth_params_deinit (struct mnt3_auth_params *aps);
+mnt3_auth_params_deinit(struct mnt3_auth_params *aps);
 
 int
-mnt3_auth_fop_options_verify (const struct mnt3_auth_params *auth_params,
-                              const char *host, const char *dir);
+mnt3_auth_fop_options_verify(const struct mnt3_auth_params *auth_params,
+                             const char *host, const char *dir);
 
 #endif /* _MOUNT3_AUTH */
diff --git a/xlators/nfs/server/src/mount3.c b/xlators/nfs/server/src/mount3.c
index 63bb44fdcde..a34d9104c17 100644
--- a/xlators/nfs/server/src/mount3.c
+++ b/xlators/nfs/server/src/mount3.c
@@ -8,145 +8,149 @@
   cases as published by the Free Software Foundation.
 */
 
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
-
 #include "rpcsvc.h"
-#include "dict.h"
-#include "xlator.h"
+#include <glusterfs/dict.h>
+#include <glusterfs/xlator.h>
 #include "mount3.h"
 #include "xdr-nfs3.h"
 #include "msg-nfs3.h"
-#include "iobuf.h"
+#include <glusterfs/iobuf.h>
 #include "nfs-common.h"
 #include "nfs3-fh.h"
 #include "nfs-fops.h"
 #include "nfs-inodes.h"
 #include "nfs-generics.h"
-#include "locking.h"
-#include "iatt.h"
+#include <glusterfs/locking.h>
+#include <glusterfs/iatt.h>
 #include "nfs-mem-types.h"
 #include "nfs.h"
-#include "common-utils.h"
-#include "store.h"
+#include <glusterfs/common-utils.h>
+#include <glusterfs/store.h>
 #include "glfs-internal.h"
 #include "glfs.h"
 #include "mount3-auth.h"
-#include "hashfn.h"
+#include <glusterfs/hashfn.h>
+#include "nfs-messages.h"
 
 #include <errno.h>
 #include <sys/socket.h>
 #include <sys/uio.h>
 
-
 /* This macro will assist in freeing up entire link list
  * of host_auth_spec structure.
  */
-#define FREE_HOSTSPEC(exp) do {                                 \
-                struct host_auth_spec *host= exp->hostspec;     \
-                while (NULL != host){                           \
-                        struct host_auth_spec* temp = host;     \
-                        host = host->next;                      \
-                        if (NULL != temp->host_addr) {          \
-                                GF_FREE (temp->host_addr);      \
-                        }                                       \
-                        GF_FREE (temp);                         \
-                }                                               \
-                exp->hostspec = NULL;                           \
-        } while (0)
+#define FREE_HOSTSPEC(exp)                                                     \
+    do {                                                                       \
+        struct host_auth_spec *host = exp->hostspec;                           \
+        while (NULL != host) {                                                 \
+            struct host_auth_spec *temp = host;                                \
+            host = host->next;                                                 \
+            if (NULL != temp->host_addr) {                                     \
+                GF_FREE(temp->host_addr);                                      \
+            }                                                                  \
+            GF_FREE(temp);                                                     \
+        }                                                                      \
+        exp->hostspec = NULL;                                                  \
+    } while (0)
 
 /* Paths for export and netgroup files */
-const char *exports_file_path   = GLUSTERD_DEFAULT_WORKDIR "/nfs/exports";
+const char *exports_file_path = GLUSTERD_DEFAULT_WORKDIR "/nfs/exports";
 const char *netgroups_file_path = GLUSTERD_DEFAULT_WORKDIR "/nfs/netgroups";
 
-typedef ssize_t (*mnt3_serializer) (struct iovec outmsg, void *args);
+typedef ssize_t (*mnt3_serializer)(struct iovec outmsg, void *args);
 
 extern void *
-mount3udp_thread (void *argv);
+mount3udp_thread(void *argv);
 
-static inline void
-mnt3_export_free (struct mnt3_export *exp)
+static void
+mnt3_export_free(struct mnt3_export *exp)
 {
-        if (!exp)
-                return;
+    if (!exp)
+        return;
 
-        if (exp->exptype == MNT3_EXPTYPE_DIR)
-                FREE_HOSTSPEC (exp);
-        GF_FREE (exp->expname);
-        GF_FREE (exp->fullpath);
-        GF_FREE (exp);
+    if (exp->exptype == MNT3_EXPTYPE_DIR)
+        FREE_HOSTSPEC(exp);
+    GF_FREE(exp->expname);
+    GF_FREE(exp->fullpath);
+    GF_FREE(exp);
 }
 
 /* Generic reply function for MOUNTv3 specific replies. */
 int
-mnt3svc_submit_reply (rpcsvc_request_t *req, void *arg, mnt3_serializer sfunc)
-{
-        struct iovec            outmsg  = {0, };
-        struct iobuf            *iob    = NULL;
-        struct mount3_state     *ms     = NULL;
-        int                     ret     = -1;
-        ssize_t                 msglen  = 0;
-        struct iobref           *iobref = NULL;
-
-        if (!req)
-                return -1;
-
-        ms = (struct mount3_state *)rpcsvc_request_program_private (req);
-        if (!ms) {
-                gf_log (GF_MNT, GF_LOG_ERROR, "mount state not found");
-                goto ret;
-        }
-
-        /* First, get the io buffer into which the reply in arg will
-         * be serialized.
-         */
-        /* TODO: use 'xdrproc_t' instead of 'sfunc' to get the xdr-size */
-        iob = iobuf_get (ms->iobpool);
-        if (!iob) {
-                gf_log (GF_MNT, GF_LOG_ERROR, "Failed to get iobuf");
-                goto ret;
-        }
-
-        iobuf_to_iovec (iob, &outmsg);
-        /* Use the given serializer to translate the give C structure in arg
-         * to XDR format which will be written into the buffer in outmsg.
-         */
-        msglen = sfunc (outmsg, arg);
-        if (msglen < 0) {
-                gf_log (GF_MNT, GF_LOG_ERROR, "Failed to encode message");
-                goto ret;
-        }
-        outmsg.iov_len = msglen;
-
-        iobref = iobref_new ();
-        if (iobref == NULL) {
-                gf_log (GF_MNT, GF_LOG_ERROR, "Failed to get iobref");
-                goto ret;
-        }
-
-        ret = iobref_add (iobref, iob);
-        if (ret) {
-                gf_log (GF_MNT, GF_LOG_ERROR, "Failed to add iob to iobref");
-                goto ret;
-        }
-
-        /* Then, submit the message for transmission. */
-        ret = rpcsvc_submit_message (req, &outmsg, 1, NULL, 0, iobref);
-        if (ret == -1) {
-                gf_log (GF_MNT, GF_LOG_ERROR, "Reply submission failed");
-                goto ret;
-        }
+mnt3svc_submit_reply(rpcsvc_request_t *req, void *arg, mnt3_serializer sfunc)
+{
+    struct iovec outmsg = {
+        0,
+    };
+    struct iobuf *iob = NULL;
+    struct mount3_state *ms = NULL;
+    int ret = -1;
+    ssize_t msglen = 0;
+    struct iobref *iobref = NULL;
+
+    if (!req)
+        return -1;
 
-        ret = 0;
+    ms = (struct mount3_state *)rpcsvc_request_program_private(req);
+    if (!ms) {
+        gf_msg(GF_MNT, GF_LOG_ERROR, EINVAL, NFS_MSG_MNT_STATE_NOT_FOUND,
+               "mount state not found");
+        goto ret;
+    }
+
+    /* First, get the io buffer into which the reply in arg will
+     * be serialized.
+     */
+    /* TODO: use 'xdrproc_t' instead of 'sfunc' to get the xdr-size */
+    iob = iobuf_get(ms->iobpool);
+    if (!iob) {
+        gf_msg(GF_MNT, GF_LOG_ERROR, ENOMEM, NFS_MSG_NO_MEMORY,
+               "Failed to get iobuf");
+        goto ret;
+    }
+
+    iobuf_to_iovec(iob, &outmsg);
+    /* Use the given serializer to translate the give C structure in arg
+     * to XDR format which will be written into the buffer in outmsg.
+     */
+    msglen = sfunc(outmsg, arg);
+    if (msglen < 0) {
+        gf_msg(GF_MNT, GF_LOG_ERROR, 0, NFS_MSG_ENCODE_MSG_FAIL,
+               "Failed to encode message");
+        goto ret;
+    }
+    outmsg.iov_len = msglen;
+
+    iobref = iobref_new();
+    if (iobref == NULL) {
+        gf_msg(GF_MNT, GF_LOG_ERROR, ENOMEM, NFS_MSG_NO_MEMORY,
+               "Failed to get iobref");
+        goto ret;
+    }
+
+    ret = iobref_add(iobref, iob);
+    if (ret) {
+        gf_msg(GF_MNT, GF_LOG_ERROR, ENOMEM, NFS_MSG_NO_MEMORY,
+               "Failed to add iob to iobref");
+        goto ret;
+    }
+
+    /* Then, submit the message for transmission. */
+    ret = rpcsvc_submit_message(req, &outmsg, 1, NULL, 0, iobref);
+    if (ret == -1) {
+        gf_msg(GF_MNT, GF_LOG_ERROR, errno, NFS_MSG_REP_SUBMIT_FAIL,
+               "Reply submission failed");
+        goto ret;
+    }
+
+    ret = 0;
 ret:
-        if (NULL != iob)
-                iobuf_unref (iob);
-        if (NULL != iobref)
-                iobref_unref (iobref);
+    if (NULL != iob)
+        iobuf_unref(iob);
+    if (NULL != iobref)
+        iobref_unref(iobref);
 
-        return ret;
+    return ret;
 }
 /**
  * __mountdict_insert -- Insert a mount entry into the mount state
@@ -157,40 +161,38 @@ ret:
  * Not for external use.
  */
 void
-__mountdict_insert (struct mount3_state *ms, struct mountentry *me)
+__mountdict_insert(struct mount3_state *ms, struct mountentry *me)
 {
-        char   *exname = NULL;
-        char   *fpath  = NULL;
-        data_t *medata = NULL;
-
-        GF_VALIDATE_OR_GOTO (GF_MNT, ms, out);
-        GF_VALIDATE_OR_GOTO (GF_MNT, me, out);
-
-        /* We don't want export names with leading slashes */
-        exname = me->exname;
-        while (exname[0] == '/')
-                exname++;
-
-        /* Get the fullpath for the export */
-        fpath = me->fullpath;
-        if (me->has_full_path) {
-                while (fpath[0] == '/')
-                        fpath++;
-
-                /* Export names can either be just volumes or paths inside that
-                 * volume. */
-                exname = fpath;
-        }
-
-        snprintf (me->hashkey, sizeof (me->hashkey), "%s:%s", exname,
-                  me->hostname);
-
-        medata = bin_to_data (me, sizeof (*me));
-        dict_set (ms->mountdict, me->hashkey, medata);
-        gf_log (GF_MNT, GF_LOG_TRACE, "Inserted into mountdict: %s",
-                me->hashkey);
+    char *exname = NULL;
+    char *fpath = NULL;
+    data_t *medata = NULL;
+
+    GF_VALIDATE_OR_GOTO(GF_MNT, ms, out);
+    GF_VALIDATE_OR_GOTO(GF_MNT, me, out);
+
+    /* We don't want export names with leading slashes */
+    exname = me->exname;
+    while (exname[0] == '/')
+        exname++;
+
+    /* Get the fullpath for the export */
+    fpath = me->fullpath;
+    if (me->has_full_path) {
+        while (fpath[0] == '/')
+            fpath++;
+
+        /* Export names can either be just volumes or paths inside that
+         * volume. */
+        exname = fpath;
+    }
+
+    snprintf(me->hashkey, sizeof(me->hashkey), "%s:%s", exname, me->hostname);
+
+    medata = bin_to_data(me, sizeof(*me));
+    dict_set(ms->mountdict, me->hashkey, medata);
+    gf_msg_trace(GF_MNT, 0, "Inserted into mountdict: %s", me->hashkey);
 out:
-        return;
+    return;
 }
 
 /**
@@ -201,92 +203,91 @@ out:
  *
  * Not for external use.
  */
-inline void
-__mountdict_remove (struct mount3_state *ms, struct mountentry *me)
+void
+__mountdict_remove(struct mount3_state *ms, struct mountentry *me)
 {
-        dict_del (ms->mountdict, me->hashkey);
+    dict_del(ms->mountdict, me->hashkey);
 }
 
 /* Generic error reply function, just pass the err status
  * and it will do the rest, including transmission.
  */
 int
-mnt3svc_mnt_error_reply (rpcsvc_request_t *req, int mntstat)
+mnt3svc_mnt_error_reply(rpcsvc_request_t *req, int mntstat)
 {
-        mountres3       res;
+    mountres3 res;
 
-        if (!req)
-                return -1;
+    if (!req)
+        return -1;
 
-        res.fhs_status = mntstat;
-        mnt3svc_submit_reply (req, (void *)&res,
-                              (mnt3_serializer)xdr_serialize_mountres3);
+    res.fhs_status = mntstat;
+    mnt3svc_submit_reply(req, (void *)&res,
+                         (mnt3_serializer)xdr_serialize_mountres3);
 
-        return 0;
+    return 0;
 }
 
-
 mountstat3
-mnt3svc_errno_to_mnterr (int32_t errnum)
-{
-        mountstat3      stat;
-
-        switch (errnum) {
-
-                case 0:
-                        stat = MNT3_OK;
-                        break;
-                case ENOENT:
-                        stat = MNT3ERR_NOENT;
-                        break;
-                case EPERM:
-                        stat = MNT3ERR_PERM;
-                        break;
-                case EIO:
-                        stat = MNT3ERR_IO;
-                        break;
-                case EACCES:
-                        stat = MNT3ERR_ACCES;
-                        break;
-                case ENOTDIR:
-                        stat = MNT3ERR_NOTDIR;
-                        break;
-                case EINVAL:
-                        stat = MNT3ERR_INVAL;
-                        break;
-                case ENOSYS:
-                        stat = MNT3ERR_NOTSUPP;
-                        break;
-                case ENOMEM:
-                        stat = MNT3ERR_SERVERFAULT;
-                        break;
-                default:
-                        stat = MNT3ERR_SERVERFAULT;
-                        break;
-        }
+mnt3svc_errno_to_mnterr(int32_t errnum)
+{
+    mountstat3 stat;
+
+    switch (errnum) {
+        case 0:
+            stat = MNT3_OK;
+            break;
+        case ENOENT:
+            stat = MNT3ERR_NOENT;
+            break;
+        case EPERM:
+            stat = MNT3ERR_PERM;
+            break;
+        case EIO:
+            stat = MNT3ERR_IO;
+            break;
+        case EACCES:
+            stat = MNT3ERR_ACCES;
+            break;
+        case ENOTDIR:
+            stat = MNT3ERR_NOTDIR;
+            break;
+        case EINVAL:
+            stat = MNT3ERR_INVAL;
+            break;
+        case ENOSYS:
+            stat = MNT3ERR_NOTSUPP;
+            break;
+        case ENOMEM:
+            stat = MNT3ERR_SERVERFAULT;
+            break;
+        default:
+            stat = MNT3ERR_SERVERFAULT;
+            break;
+    }
 
-        return stat;
+    return stat;
 }
 
-
 mountres3
-mnt3svc_set_mountres3 (mountstat3 stat, struct nfs3_fh *fh, int *authflavor,
-                       u_int aflen)
+mnt3svc_set_mountres3(mountstat3 stat, struct nfs3_fh *fh, int *authflavor,
+                      u_int aflen)
 {
-        mountres3       res = {0, };
-        uint32_t        fhlen = 0;
+    mountres3 res = {
+        0,
+    };
+    uint32_t fhlen = 0;
 
-        res.fhs_status = stat;
+    res.fhs_status = stat;
 
-        if (fh)
-                fhlen = nfs3_fh_compute_size ();
+    if (fh)
+        fhlen = nfs3_fh_compute_size();
 
-        res.mountres3_u.mountinfo.fhandle.fhandle3_len = fhlen;
-        res.mountres3_u.mountinfo.fhandle.fhandle3_val = (char *)fh;
-        res.mountres3_u.mountinfo.auth_flavors.auth_flavors_val = authflavor;
-        res.mountres3_u.mountinfo.auth_flavors.auth_flavors_len = aflen;
+    res.mountres3_u.mountinfo.fhandle.fhandle3_len = fhlen;
+    res.mountres3_u.mountinfo.fhandle.fhandle3_val = (char *)fh;
+    res.mountres3_u.mountinfo.auth_flavors.auth_flavors_val = authflavor;
+    res.mountres3_u.mountinfo.auth_flavors.auth_flavors_len = aflen;
 
-        return res;
+    return res;
 }
 
 /* Read the rmtab from the store_handle and append (or not) the entries to the
@@ -295,160 +296,181 @@ mnt3svc_set_mountres3 (mountstat3 stat, struct nfs3_fh *fh, int *authflavor,
  * Requires the store_handle to be locked.
  */
 static int
-__mount_read_rmtab (gf_store_handle_t *sh, struct list_head *mountlist,
-                    gf_boolean_t append)
+__mount_read_rmtab(gf_store_handle_t *sh, struct list_head *mountlist,
+                   gf_boolean_t append)
 {
-        int                     ret = 0;
-        unsigned int            idx = 0;
-        struct mountentry       *me = NULL, *tmp = NULL;
-                                /* me->hostname is a char[MNTPATHLEN] */
-        char                    key[MNTPATHLEN + 11];
-
-        GF_ASSERT (sh && mountlist);
-
-        if (!gf_store_locked_local (sh)) {
-                gf_log (GF_MNT, GF_LOG_WARNING, "Not reading unlocked %s",
-                        sh->path);
-                return -1;
-        }
+    int ret = 0;
+    unsigned int idx = 0;
+    struct mountentry *me = NULL, *tmp = NULL;
+    /* me->hostname is a char[MNTPATHLEN] */
+    char key[MNTPATHLEN + 11];
 
-        if (!append) {
-                list_for_each_entry_safe (me, tmp, mountlist, mlist) {
-                        list_del (&me->mlist);
-                        GF_FREE (me);
-                }
-                me = NULL;
-        }
+    GF_ASSERT(sh && mountlist);
 
-        for (;;) {
-                char *value = NULL;
-
-                if (me && append) {
-                        /* do not add duplicates */
-                        list_for_each_entry (tmp, mountlist, mlist) {
-                                if (!strcmp(tmp->hostname, me->hostname) &&
-                                    !strcmp(tmp->exname, me->exname)) {
-                                        GF_FREE (me);
-                                        goto dont_add;
-                                }
-                        }
-                        list_add_tail (&me->mlist, mountlist);
-                } else if (me) {
-                        list_add_tail (&me->mlist, mountlist);
-                }
+    if (!gf_store_locked_local(sh)) {
+        gf_msg(GF_MNT, GF_LOG_WARNING, 0, NFS_MSG_READ_LOCKED,
+               "Not reading unlocked %s", sh->path);
+        return -1;
+    }
 
-dont_add:
-                me = GF_CALLOC (1, sizeof (*me), gf_nfs_mt_mountentry);
-                if (!me) {
-                        gf_log (GF_MNT, GF_LOG_ERROR, "Out of memory");
-                        ret = -1;
-                        goto out;
+    if (!append) {
+        list_for_each_entry_safe(me, tmp, mountlist, mlist)
+        {
+            list_del(&me->mlist);
+            GF_FREE(me);
+        }
+        me = NULL;
+    }
+
+    for (;;) {
+        char *value = NULL;
+
+        if (me && append) {
+            /* do not add duplicates */
+            list_for_each_entry(tmp, mountlist, mlist)
+            {
+                if (!strcmp(tmp->hostname, me->hostname) &&
+                    !strcmp(tmp->exname, me->exname)) {
+                    GF_FREE(me);
+                    goto dont_add;
                 }
-
-                INIT_LIST_HEAD (&me->mlist);
-
-                snprintf (key, 9 + MNTPATHLEN, "hostname-%d", idx);
-                ret = gf_store_retrieve_value (sh, key, &value);
-                if (ret)
-                        break;
-                strncpy (me->hostname, value, MNTPATHLEN);
-                GF_FREE (value);
-
-                snprintf (key, 11 + MNTPATHLEN, "mountpoint-%d", idx);
-                ret = gf_store_retrieve_value (sh, key, &value);
-                if (ret)
-                        break;
-                strncpy (me->exname, value, MNTPATHLEN);
-                GF_FREE (value);
-
-                idx++;
-                gf_log (GF_MNT, GF_LOG_TRACE, "Read entries %s:%s", me->hostname, me->exname);
-        }
-        gf_log (GF_MNT, GF_LOG_DEBUG, "Read %d entries from '%s'", idx, sh->path);
-        GF_FREE (me);
+            }
+            list_add_tail(&me->mlist, mountlist);
+        } else if (me) {
+            list_add_tail(&me->mlist, mountlist);
+        }
+
+    dont_add:
+        me = GF_CALLOC(1, sizeof(*me), gf_nfs_mt_mountentry);
+        if (!me) {
+            gf_msg(GF_MNT, GF_LOG_ERROR, ENOMEM, NFS_MSG_NO_MEMORY,
+                   "Out of memory");
+            ret = -1;
+            goto out;
+        }
+
+        INIT_LIST_HEAD(&me->mlist);
+
+        snprintf(key, 9 + MNTPATHLEN, "hostname-%d", idx);
+        ret = gf_store_retrieve_value(sh, key, &value);
+        if (ret)
+            break;
+        snprintf(me->hostname, MNTPATHLEN, "%s", value);
+        GF_FREE(value);
+
+        snprintf(key, 11 + MNTPATHLEN, "mountpoint-%d", idx);
+        ret = gf_store_retrieve_value(sh, key, &value);
+        if (ret)
+            break;
+        snprintf(me->exname, MNTPATHLEN, "%s", value);
+        GF_FREE(value);
+
+        idx++;
+        gf_msg_trace(GF_MNT, 0, "Read entries %s:%s", me->hostname, me->exname);
+    }
+    gf_msg_debug(GF_MNT, 0, "Read %d entries from '%s'", idx, sh->path);
+    GF_FREE(me);
 out:
-        return ret;
+    return ret;
 }
 
-/* Overwrite the contents of the rwtab with te in-memory client list.
+/* Overwrite the contents of the rwtab with the in-memory client list.
  * Fail gracefully if the stora_handle is not locked.
  */
 static void
 __mount_rewrite_rmtab(struct mount3_state *ms, gf_store_handle_t *sh)
 {
-        struct mountentry       *me = NULL;
-        char                    key[16];
-        int                     fd, ret;
-        unsigned int            idx = 0;
-
-        if (!gf_store_locked_local (sh)) {
-                gf_log (GF_MNT, GF_LOG_WARNING, "Not modifying unlocked %s",
-                        sh->path);
-                return;
-        }
+    struct mountentry *me = NULL;
+    char key[16];
+    int fd, ret;
+    unsigned int idx = 0;
+
+    if (!gf_store_locked_local(sh)) {
+        gf_msg(GF_MNT, GF_LOG_WARNING, 0, NFS_MSG_MODIFY_LOCKED,
+               "Not modifying unlocked %s", sh->path);
+        return;
+    }
 
-        fd = gf_store_mkstemp (sh);
-        if (fd == -1) {
-                gf_log (GF_MNT, GF_LOG_ERROR, "Failed to open %s", sh->path);
-                return;
-        }
+    fd = gf_store_mkstemp(sh);
+    if (fd == -1) {
+        gf_msg(GF_MNT, GF_LOG_ERROR, EINVAL, NFS_MSG_INVALID_ENTRY,
+               "Failed to open %s", sh->path);
+        return;
+    }
 
-        list_for_each_entry (me, &ms->mountlist, mlist) {
-                snprintf (key, 16, "hostname-%d", idx);
-                ret = gf_store_save_value (fd, key, me->hostname);
-                if (ret)
-                        goto fail;
+    list_for_each_entry(me, &ms->mountlist, mlist)
+    {
+        snprintf(key, 16, "hostname-%d", idx);
+        ret = gf_store_save_value(fd, key, me->hostname);
+        if (ret)
+            goto fail;
 
-                snprintf (key, 16, "mountpoint-%d", idx);
-                ret = gf_store_save_value (fd, key, me->exname);
-                if (ret)
-                        goto fail;
+        snprintf(key, 16, "mountpoint-%d", idx);
+        ret = gf_store_save_value(fd, key, me->exname);
+        if (ret)
+            goto fail;
 
-                idx++;
-        }
+        idx++;
+    }
 
-        gf_log (GF_MNT, GF_LOG_DEBUG, "Updated rmtab with %d entries", idx);
+    gf_msg_debug(GF_MNT, 0, "Updated rmtab with %d entries", idx);
 
-        if (gf_store_rename_tmppath (sh))
-                gf_log (GF_MNT, GF_LOG_ERROR, "Failed to overwrite rwtab %s",
-                        sh->path);
+    if (gf_store_rename_tmppath(sh))
+        gf_msg(GF_MNT, GF_LOG_ERROR, errno, NFS_MSG_RWTAB_OVERWRITE_FAIL,
+               "Failed to overwrite rwtab %s", sh->path);
 
-        return;
+    return;
 
 fail:
-        gf_log (GF_MNT, GF_LOG_ERROR, "Failed to update %s", sh->path);
-        gf_store_unlink_tmppath (sh);
+    gf_msg(GF_MNT, GF_LOG_ERROR, errno, NFS_MSG_UPDATE_FAIL,
+           "Failed to update %s", sh->path);
+    gf_store_unlink_tmppath(sh);
+}
+
+static gf_boolean_t
+mount_open_rmtab(const char *rmtab, gf_store_handle_t **sh)
+{
+    int ret = -1;
+
+    /* updating the rmtab is disabled, use in-memory only */
+    if (!rmtab || rmtab[0] == '\0')
+        return _gf_false;
+
+    ret = gf_store_handle_new(rmtab, sh);
+    if (ret) {
+        gf_log(GF_MNT, GF_LOG_WARNING, "Failed to open '%s'", rmtab);
+        return _gf_false;
+    }
+
+    return _gf_true;
 }
 
 /* Read the rmtab into a clean ms->mountlist.
  */
 static void
-mount_read_rmtab (struct mount3_state *ms)
+mount_read_rmtab(struct mount3_state *ms)
 {
-        gf_store_handle_t       *sh = NULL;
-        struct nfs_state        *nfs = NULL;
-        int                     ret;
+    gf_store_handle_t *sh = NULL;
+    struct nfs_state *nfs = NULL;
+    gf_boolean_t read_rmtab = _gf_false;
 
-        nfs = (struct nfs_state *)ms->nfsx->private;
+    nfs = (struct nfs_state *)ms->nfsx->private;
 
-        ret = gf_store_handle_new (nfs->rmtab, &sh);
-        if (ret) {
-                gf_log (GF_MNT, GF_LOG_WARNING, "Failed to open '%s'",
-                        nfs->rmtab);
-                return;
-        }
+    read_rmtab = mount_open_rmtab(nfs->rmtab, &sh);
+    if (!read_rmtab)
+        return;
 
-        if (gf_store_lock (sh)) {
-                gf_log (GF_MNT, GF_LOG_WARNING, "Failed to lock '%s'",
-                        nfs->rmtab);
-                goto out;
-        }
+    if (gf_store_lock(sh)) {
+        gf_msg(GF_MNT, GF_LOG_WARNING, 0, NFS_MSG_LOCK_FAIL,
+               "Failed to lock '%s'", nfs->rmtab);
+        goto out;
+    }
 
-        __mount_read_rmtab (sh, &ms->mountlist, _gf_false);
-        gf_store_unlock (sh);
+    __mount_read_rmtab(sh, &ms->mountlist, _gf_false);
+    gf_store_unlock(sh);
 
 out:
-        gf_store_handle_destroy (sh);
+    gf_store_handle_destroy(sh);
 }
 
 /* Write the ms->mountlist to the rmtab.
@@ -456,6 +478,7 @@ out:
  * The rmtab could be empty, or it can exists and have been updated by a
  * different storage server without our knowing.
  *
+ * 0. if opening the nfs->rmtab fails, return gracefully
  * 1. takes the store_handle lock on the current rmtab
  *    - blocks if an other storage server rewrites the rmtab at the same time
  * 2. [if new_rmtab] takes the store_handle lock on the new rmtab
@@ -468,74 +491,75 @@ out:
  * 9. releases the store_handle lock of the old rmtab
  */
 void
-mount_rewrite_rmtab (struct mount3_state *ms, char *new_rmtab)
+mount_rewrite_rmtab(struct mount3_state *ms, char *new_rmtab)
 {
-        gf_store_handle_t       *sh = NULL, *nsh = NULL;
-        struct nfs_state        *nfs = NULL;
-        int                     ret;
-        char                    *rmtab = NULL;
+    gf_store_handle_t *sh = NULL, *nsh = NULL;
+    struct nfs_state *nfs = NULL;
+    int ret;
+    char *rmtab = NULL;
+    gf_boolean_t got_old_rmtab = _gf_false;
+
+    nfs = (struct nfs_state *)ms->nfsx->private;
 
-        nfs = (struct nfs_state *)ms->nfsx->private;
+    got_old_rmtab = mount_open_rmtab(nfs->rmtab, &sh);
+    if (!got_old_rmtab && !new_rmtab)
+        return;
+
+    if (got_old_rmtab && gf_store_lock(sh)) {
+        gf_msg(GF_MNT, GF_LOG_WARNING, 0, NFS_MSG_REWRITE_ERROR,
+               "Not rewriting '%s'", nfs->rmtab);
+        goto free_sh;
+    }
 
-        ret = gf_store_handle_new (nfs->rmtab, &sh);
+    if (new_rmtab) {
+        ret = gf_store_handle_new(new_rmtab, &nsh);
         if (ret) {
-                gf_log (GF_MNT, GF_LOG_WARNING, "Failed to open '%s'",
-                        nfs->rmtab);
-                return;
+            gf_msg(GF_MNT, GF_LOG_WARNING, 0, NFS_MSG_OPEN_FAIL,
+                   "Failed to open '%s'", new_rmtab);
+            goto unlock_sh;
         }
 
-        if (gf_store_lock (sh)) {
-                gf_log (GF_MNT, GF_LOG_WARNING, "Not rewriting '%s'",
-                        nfs->rmtab);
-                goto free_sh;
+        if (gf_store_lock(nsh)) {
+            gf_msg(GF_MNT, GF_LOG_WARNING, 0, NFS_MSG_REWRITE_ERROR,
+                   "Not rewriting '%s'", new_rmtab);
+            goto free_nsh;
         }
+    }
 
-        if (new_rmtab) {
-                ret = gf_store_handle_new (new_rmtab, &nsh);
-                if (ret) {
-                        gf_log (GF_MNT, GF_LOG_WARNING, "Failed to open '%s'",
-                                new_rmtab);
-                        goto unlock_sh;
-                }
-
-                if (gf_store_lock (nsh)) {
-                        gf_log (GF_MNT, GF_LOG_WARNING, "Not rewriting '%s'",
-                                new_rmtab);
-                        goto free_nsh;
-                }
-        }
+    /* always read the currently used rmtab */
+    if (got_old_rmtab)
+        __mount_read_rmtab(sh, &ms->mountlist, _gf_true);
 
-        /* always read the currently used rmtab */
-        __mount_read_rmtab (sh, &ms->mountlist, _gf_true);
-
-        if (new_rmtab) {
-                /* read the new rmtab and write changes to the new location */
-                __mount_read_rmtab (nsh, &ms->mountlist, _gf_true);
-                __mount_rewrite_rmtab (ms, nsh);
-
-                /* replace the nfs->rmtab reference to the new rmtab */
-                rmtab = gf_strdup(new_rmtab);
-                if (rmtab == NULL) {
-                        gf_log (GF_MNT, GF_LOG_ERROR, "Out of memory, keeping "
-                                "%s as rmtab", nfs->rmtab);
-                } else {
-                        GF_FREE (nfs->rmtab);
-                        nfs->rmtab = rmtab;
-                }
+    if (new_rmtab) {
+        /* read the new rmtab and write changes to the new location */
+        __mount_read_rmtab(nsh, &ms->mountlist, _gf_true);
+        __mount_rewrite_rmtab(ms, nsh);
 
-                gf_store_unlock (nsh);
+        /* replace the nfs->rmtab reference to the new rmtab */
+        rmtab = gf_strdup(new_rmtab);
+        if (rmtab == NULL) {
+            gf_msg(GF_MNT, GF_LOG_ERROR, errno, NFS_MSG_NO_MEMORY,
+                   "Out of memory, keeping %s as rmtab", nfs->rmtab);
         } else {
-                /* rewrite the current (unchanged location) rmtab */
-                __mount_rewrite_rmtab (ms, sh);
+            GF_FREE(nfs->rmtab);
+            nfs->rmtab = rmtab;
         }
 
+        gf_store_unlock(nsh);
+    } else {
+        /* rewrite the current (unchanged location) rmtab */
+        __mount_rewrite_rmtab(ms, sh);
+    }
+
 free_nsh:
-        if (new_rmtab)
-                gf_store_handle_destroy (nsh);
+    if (new_rmtab)
+        gf_store_handle_destroy(nsh);
 unlock_sh:
-        gf_store_unlock (sh);
+    if (got_old_rmtab)
+        gf_store_unlock(sh);
 free_sh:
-        gf_store_handle_destroy (sh);
+    if (got_old_rmtab)
+        gf_store_handle_destroy(sh);
 }
 
 /* Add a new NFS-client to the ms->mountlist and update the rmtab if we can.
@@ -550,367 +574,372 @@ free_sh:
  * common issue for all(?) NFS-servers.
  */
 int
-mnt3svc_update_mountlist (struct mount3_state *ms, rpcsvc_request_t *req,
-                          const char *expname, const char *fullpath)
+mnt3svc_update_mountlist(struct mount3_state *ms, rpcsvc_request_t *req,
+                         const char *expname, const char *fullpath)
 {
-        struct mountentry       *me = NULL;
-        struct mountentry       *cur = NULL;
-        int                     ret = -1;
-        char                    *colon = NULL;
-        struct nfs_state        *nfs = NULL;
-        gf_store_handle_t       *sh = NULL;
-
-        if ((!ms) || (!req) || (!expname))
-                return -1;
+    struct mountentry *me = NULL;
+    struct mountentry *cur = NULL;
+    int ret = -1;
+    char *colon = NULL;
+    struct nfs_state *nfs = NULL;
+    gf_store_handle_t *sh = NULL;
+    gf_boolean_t update_rmtab = _gf_false;
+
+    if ((!ms) || (!req) || (!expname))
+        return -1;
 
-        me = (struct mountentry *)GF_CALLOC (1, sizeof (*me),
-                                             gf_nfs_mt_mountentry);
-        if (!me)
-                return -1;
+    me = (struct mountentry *)GF_CALLOC(1, sizeof(*me), gf_nfs_mt_mountentry);
+    if (!me)
+        return -1;
 
-        nfs = (struct nfs_state *)ms->nfsx->private;
+    nfs = (struct nfs_state *)ms->nfsx->private;
 
-        ret = gf_store_handle_new (nfs->rmtab, &sh);
-        if (ret) {
-                gf_log (GF_MNT, GF_LOG_WARNING, "Failed to open '%s'",
-                        nfs->rmtab);
-                goto free_err;
-        }
+    update_rmtab = mount_open_rmtab(nfs->rmtab, &sh);
 
-        strncpy (me->exname, expname, MNTPATHLEN);
-        /* Sometimes we don't care about the full path
-         * so a NULL value for fullpath is valid.
-         */
-        if (fullpath) {
-                if (strlen (fullpath) < MNTPATHLEN) {
-                        strcpy (me->fullpath, fullpath);
-                        me->has_full_path = _gf_true;
-                }
+    snprintf(me->exname, MNTPATHLEN, "%s", expname);
+    /* Sometimes we don't care about the full path
+     * so a NULL value for fullpath is valid.
+     */
+    if (fullpath) {
+        if (strlen(fullpath) < MNTPATHLEN) {
+            strcpy(me->fullpath, fullpath);
+            me->has_full_path = _gf_true;
         }
+    }
 
+    INIT_LIST_HEAD(&me->mlist);
+    /* Must get the IP or hostname of the client so we
+     * can map it into the mount entry.
+     */
+    ret = rpcsvc_transport_peername(req->trans, me->hostname, MNTPATHLEN);
+    if (ret == -1)
+        goto free_err;
 
-        INIT_LIST_HEAD (&me->mlist);
-        /* Must get the IP or hostname of the client so we
-         * can map it into the mount entry.
-         */
-        ret = rpcsvc_transport_peername (req->trans, me->hostname, MNTPATHLEN);
-        if (ret == -1)
-                goto free_err2;
-
-        colon = strrchr (me->hostname, ':');
-        if (colon) {
-                *colon = '\0';
-        }
-        LOCK (&ms->mountlock);
+    colon = strrchr(me->hostname, ':');
+    if (colon) {
+        *colon = '\0';
+    }
+    LOCK(&ms->mountlock);
+    {
+        /* in case locking fails, we just don't write the rmtab */
+        if (update_rmtab && gf_store_lock(sh)) {
+            gf_msg(GF_MNT, GF_LOG_WARNING, 0, NFS_MSG_LOCK_FAIL,
+                   "Failed to lock '%s', changes will not be "
+                   "written",
+                   nfs->rmtab);
+        } else if (update_rmtab) {
+            __mount_read_rmtab(sh, &ms->mountlist, _gf_false);
+        }
+
+        /* do not add duplicates */
+        list_for_each_entry(cur, &ms->mountlist, mlist)
         {
-                /* in case locking fails, we just don't write the rmtab */
-                if (gf_store_lock (sh)) {
-                        gf_log (GF_MNT, GF_LOG_WARNING, "Failed to lock '%s'"
-                                ", changes will not be written", nfs->rmtab);
-                } else {
-                        __mount_read_rmtab (sh, &ms->mountlist, _gf_false);
-                }
-
-                /* do not add duplicates */
-                list_for_each_entry (cur, &ms->mountlist, mlist) {
-                        if (!strcmp(cur->hostname, me->hostname) &&
-                            !strcmp(cur->exname, me->exname)) {
-                                GF_FREE (me);
-                                goto dont_add;
-                        }
-                }
-                list_add_tail (&me->mlist, &ms->mountlist);
-                __mountdict_insert (ms, me);
-
-                /* only write the rmtab in case it was locked */
-                if (gf_store_locked_local (sh))
-                        __mount_rewrite_rmtab (ms, sh);
-        }
+            if (!strcmp(cur->hostname, me->hostname) &&
+                !strcmp(cur->exname, me->exname)) {
+                GF_FREE(me);
+                goto dont_add;
+            }
+        }
+        list_add_tail(&me->mlist, &ms->mountlist);
+        __mountdict_insert(ms, me);
+
+        /* only write the rmtab in case it was locked */
+        if (update_rmtab && gf_store_locked_local(sh))
+            __mount_rewrite_rmtab(ms, sh);
+    }
 dont_add:
-        if (gf_store_locked_local (sh))
-                gf_store_unlock (sh);
+    if (update_rmtab && gf_store_locked_local(sh))
+        gf_store_unlock(sh);
 
-        UNLOCK (&ms->mountlock);
-
-free_err2:
-        gf_store_handle_destroy (sh);
+    UNLOCK(&ms->mountlock);
 
 free_err:
-        if (ret == -1)
-                GF_FREE (me);
+    if (update_rmtab)
+        gf_store_handle_destroy(sh);
 
-        return ret;
-}
+    if (ret == -1)
+        GF_FREE(me);
 
+    return ret;
+}
 
 int
-__mnt3_get_volume_id (struct mount3_state *ms, xlator_t *mntxl,
-                      uuid_t volumeid)
+__mnt3_get_volume_id(struct mount3_state *ms, xlator_t *mntxl, uuid_t volumeid)
 {
-        int                     ret = -1;
-        struct mnt3_export      *exp = NULL;
+    int ret = -1;
+    struct mnt3_export *exp = NULL;
 
-        if ((!ms) || (!mntxl))
-                return ret;
+    if ((!ms) || (!mntxl))
+        return ret;
 
-        LOCK (&ms->mountlock);
-        list_for_each_entry (exp, &ms->exportlist, explist) {
-                if (exp->vol == mntxl) {
-                        gf_uuid_copy (volumeid, exp->volumeid);
-                        ret = 0;
-                        goto out;
-                }
+    LOCK(&ms->mountlock);
+    list_for_each_entry(exp, &ms->exportlist, explist)
+    {
+        if (exp->vol == mntxl) {
+            gf_uuid_copy(volumeid, exp->volumeid);
+            ret = 0;
+            goto out;
         }
+    }
 
 out:
-        UNLOCK (&ms->mountlock);
-        return ret;
+    UNLOCK(&ms->mountlock);
+    return ret;
 }
 
 int
-__mnt3_build_mountid_from_path (const char *path, uuid_t mountid)
+__mnt3_build_mountid_from_path(const char *path, uuid_t mountid)
 {
-        uint32_t hashed_path = 0;
-        int      ret = -1;
-        size_t length;
+    uint32_t hashed_path = 0;
+    int ret = -1;
 
-        length = sizeof(mountid);
-        while (strlen (path) > 0 && path[0] == '/')
-                path++;
+    if (!path)
+        goto out;
 
-        /* Clear the mountid */
-        memset (mountid, 0, length);
+    while (strlen(path) > 0 && path[0] == '/')
+        path++;
 
-        hashed_path = SuperFastHash (path,  strlen (path));
-        if (hashed_path == 1) {
-                gf_log (GF_MNT, GF_LOG_WARNING, "failed to hash path: %s",
-                        path);
-                goto out;
-        }
+    /* Clear the mountid */
+    gf_uuid_clear(mountid);
 
-        memcpy (mountid, &hashed_path, sizeof (hashed_path));
-        ret = 0;
+    hashed_path = SuperFastHash(path, strlen(path));
+    if (hashed_path == 1) {
+        gf_msg(GF_MNT, GF_LOG_WARNING, 0, NFS_MSG_HASH_PATH_FAIL,
+               "failed to hash path: %s", path);
+        goto out;
+    }
+
+    memcpy(mountid, &hashed_path, sizeof(hashed_path));
+    ret = 0;
 out:
-        return ret;
+    return ret;
 }
 
 int
-__mnt3_get_mount_id (xlator_t *mntxl, uuid_t mountid)
+__mnt3_get_mount_id(xlator_t *mntxl, uuid_t mountid)
 {
-        int ret = -1;
-        uint32_t hashed_path = 0;
-        size_t length;
-
-        length = sizeof(mountid);
+    int ret = -1;
+    uint32_t hashed_path = 0;
 
-        /* first clear the mountid */
-        memset (mountid, 0, length);
+    /* first clear the mountid */
+    gf_uuid_clear(mountid);
 
-        hashed_path = SuperFastHash (mntxl->name, strlen (mntxl->name));
-        if (hashed_path == 1) {
-                gf_log (GF_MNT, GF_LOG_WARNING, "failed to hash xlator name: %s",
-                        mntxl->name);
-                goto out;
-        }
+    hashed_path = SuperFastHash(mntxl->name, strlen(mntxl->name));
+    if (hashed_path == 1) {
+        gf_msg(GF_MNT, GF_LOG_WARNING, 0, NFS_MSG_HASH_XLATOR_FAIL,
+               "failed to hash xlator name: %s", mntxl->name);
+        goto out;
+    }
 
-        memcpy (mountid, &hashed_path, sizeof (hashed_path));
-        ret = 0;
+    memcpy(mountid, &hashed_path, sizeof(hashed_path));
+    ret = 0;
 out:
-        return ret;
+    return ret;
 }
 
-
 int32_t
-mnt3svc_lookup_mount_cbk (call_frame_t *frame, void  *cookie,
-                          xlator_t *this, int32_t op_ret, int32_t op_errno,
-                          inode_t *inode, struct iatt *buf, dict_t *xattr,
-                          struct iatt *postparent)
-{
-        mountres3               res = {0, };
-        rpcsvc_request_t        *req = NULL;
-        struct nfs3_fh          fh = {{0}, };
-        struct mount3_state     *ms = NULL;
-        mountstat3              status = 0;
-        int                     autharr[10];
-        int                     autharrlen = 0;
-        rpcsvc_t                *svc = NULL;
-        xlator_t                *mntxl = NULL;
-        uuid_t                  volumeid = {0, };
-        char                    *path = NULL;
-        uuid_t                  mountid = {1, };
-        char                    fhstr[1536];
-
-        req = (rpcsvc_request_t *)frame->local;
-
-        if (!req)
-                return -1;
-
-        mntxl = (xlator_t *)cookie;
-        ms = (struct mount3_state *)rpcsvc_request_program_private (req);
-        if (!ms) {
-                gf_log (GF_MNT, GF_LOG_ERROR, "mount state not found");
-                op_ret = -1;
-                op_errno = EINVAL;
-        }
-
-        if (op_ret == -1) {
-                gf_log (GF_NFS, GF_LOG_ERROR, "error=%s", strerror (op_errno));
-                status = mnt3svc_errno_to_mnterr (op_errno);
-        }
-        if (status != MNT3_OK)
-                goto xmit_res;
-
-        path = GF_CALLOC (PATH_MAX, sizeof (char), gf_nfs_mt_char);
-        if (!path) {
-                gf_log (GF_MNT, GF_LOG_ERROR, "Out of memory");
-                goto xmit_res;
-        }
-
-        snprintf (path, PATH_MAX, "/%s", mntxl->name);
-        mnt3svc_update_mountlist (ms, req, path, NULL);
-        GF_FREE (path);
-        if (gf_nfs_dvm_off (nfs_state (ms->nfsx))) {
-                fh = nfs3_fh_build_indexed_root_fh (ms->nfsx->children, mntxl);
-                goto xmit_res;
-        }
+mnt3svc_lookup_mount_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                         int32_t op_ret, int32_t op_errno, inode_t *inode,
+                         struct iatt *buf, dict_t *xattr,
+                         struct iatt *postparent)
+{
+    mountres3 res = {
+        0,
+    };
+    rpcsvc_request_t *req = NULL;
+    struct nfs3_fh fh = {
+        {0},
+    };
+    struct mount3_state *ms = NULL;
+    mountstat3 status = 0;
+    int autharr[10];
+    int autharrlen = 0;
+    rpcsvc_t *svc = NULL;
+    xlator_t *mntxl = NULL;
+    uuid_t volumeid = {
+        0,
+    };
+    char *path = NULL;
+    uuid_t mountid = {
+        1,
+    };
+    char fhstr[1536];
+    int alloclen = 0;
+
+    req = (rpcsvc_request_t *)frame->local;
+
+    if (!req)
+        return -1;
 
-        __mnt3_get_mount_id (mntxl, mountid);
-        __mnt3_get_volume_id (ms, mntxl, volumeid);
-        fh = nfs3_fh_build_uuid_root_fh (volumeid, mountid);
+    mntxl = (xlator_t *)cookie;
+    ms = (struct mount3_state *)rpcsvc_request_program_private(req);
+    if (!ms) {
+        gf_msg(GF_MNT, GF_LOG_ERROR, EINVAL, NFS_MSG_MNT_STATE_NOT_FOUND,
+               "mount state not found");
+        op_ret = -1;
+        op_errno = EINVAL;
+    }
+
+    if (op_ret == -1) {
+        gf_msg(GF_NFS, GF_LOG_ERROR, op_errno, NFS_MSG_LOOKUP_MNT_ERROR,
+               "error=%s", strerror(op_errno));
+        status = mnt3svc_errno_to_mnterr(op_errno);
+    }
+    if (status != MNT3_OK)
+        goto xmit_res;
+
+    alloclen = strlen(mntxl->name) + 2;
+    path = GF_MALLOC(alloclen, gf_nfs_mt_char);
+    if (!path) {
+        gf_msg(GF_MNT, GF_LOG_ERROR, ENOMEM, NFS_MSG_NO_MEMORY,
+               "Memory allocation failed.");
+        goto xmit_res;
+    }
+
+    snprintf(path, alloclen, "/%s", mntxl->name);
+    mnt3svc_update_mountlist(ms, req, path, NULL);
+    GF_FREE(path);
+    if (gf_nfs_dvm_off(nfs_state(ms->nfsx))) {
+        fh = nfs3_fh_build_indexed_root_fh(ms->nfsx->children, mntxl);
+        goto xmit_res;
+    }
+
+    __mnt3_get_mount_id(mntxl, mountid);
+    __mnt3_get_volume_id(ms, mntxl, volumeid);
+    fh = nfs3_fh_build_uuid_root_fh(volumeid, mountid);
 
 xmit_res:
-        nfs3_fh_to_str (&fh, fhstr, sizeof (fhstr));
-        gf_log (GF_MNT, GF_LOG_DEBUG, "MNT reply: fh %s, status: %d", fhstr,
-                status);
-        if (op_ret == 0) {
-                svc = rpcsvc_request_service (req);
-                autharrlen = rpcsvc_auth_array (svc, mntxl->name, autharr,
-                                                10);
-        }
-
-        res = mnt3svc_set_mountres3 (status, &fh, autharr, autharrlen);
-        mnt3svc_submit_reply (req, (void *)&res,
-                              (mnt3_serializer)xdr_serialize_mountres3);
-
-        return 0;
+    nfs3_fh_to_str(&fh, fhstr, sizeof(fhstr));
+    gf_msg_debug(GF_MNT, 0, "MNT reply: fh %s, status: %d", fhstr, status);
+    if (op_ret == 0) {
+        svc = rpcsvc_request_service(req);
+        autharrlen = rpcsvc_auth_array(svc, mntxl->name, autharr, 10);
+    }
+
+    res = mnt3svc_set_mountres3(status, &fh, autharr, autharrlen);
+    mnt3svc_submit_reply(req, (void *)&res,
+                         (mnt3_serializer)xdr_serialize_mountres3);
+
+    return 0;
 }
 
-
 int
-mnt3_match_dirpath_export (const char *expname, const char *dirpath,
-                           gf_boolean_t export_parsing_match)
+mnt3_match_dirpath_export(const char *expname, const char *dirpath,
+                          gf_boolean_t export_parsing_match)
 {
-        int     ret = 0;
-        size_t  dlen;
-        char    *fullpath = NULL;
-        char    *second_slash = NULL;
-        char    *dirdup = NULL;
+    int ret = 0;
+    size_t dlen;
+    char *fullpath = NULL;
+    char *second_slash = NULL;
+    char *dirdup = NULL;
 
-        if ((!expname) || (!dirpath))
-                return 0;
+    if ((!expname) || (!dirpath))
+        return 0;
 
-        dirdup = strdupa (dirpath);
+    dirdup = strdupa(dirpath);
 
-        /* Some clients send a dirpath for mount that includes the slash at the
-         * end. String compare for searching the export will fail because our
-         * exports list does not include that slash. Remove the slash to
-         * compare.
-         */
-        dlen = strlen (dirdup);
-        if (dlen && dirdup[dlen - 1] == '/')
-                dirdup[dlen - 1] = '\0';
+    /* Some clients send a dirpath for mount that includes the slash at the
+     * end. String compare for searching the export will fail because our
+     * exports list does not include that slash. Remove the slash to
+     * compare.
+     */
+    dlen = strlen(dirdup);
+    if (dlen && dirdup[dlen - 1] == '/')
+        dirdup[dlen - 1] = '\0';
 
-        /* Here we try to match fullpaths with export names */
-        fullpath = dirdup;
+    /* Here we try to match fullpaths with export names */
+    fullpath = dirdup;
 
-        if (export_parsing_match) {
-                if (dirdup[0] == '/')
-                        fullpath = dirdup + 1;
+    if (export_parsing_match) {
+        if (dirdup[0] == '/')
+            fullpath = dirdup + 1;
 
-                second_slash = strchr (fullpath, '/');
-                if (second_slash)
-                        *second_slash = '\0';
-        }
+        second_slash = strchr(fullpath, '/');
+        if (second_slash)
+            *second_slash = '\0';
+    }
 
-        /* The export name begins with a slash so move it forward by one
-         * to ignore the slash when we want to compare the fullpath and
-         * export.
-         */
-        if (fullpath[0] != '/')
-                expname++;
+    /* The export name begins with a slash so move it forward by one
+     * to ignore the slash when we want to compare the fullpath and
+     * export.
+     */
+    if (fullpath[0] != '/')
+        expname++;
 
-        if (strcmp (expname, fullpath) == 0)
-                ret = 1;
+    if (strcmp(expname, fullpath) == 0)
+        ret = 1;
 
-        return ret;
+    return ret;
 }
 
-
 int
-mnt3svc_mount_inode (rpcsvc_request_t *req, struct mount3_state *ms,
-                     xlator_t * xl, inode_t *exportinode)
+mnt3svc_mount_inode(rpcsvc_request_t *req, struct mount3_state *ms,
+                    xlator_t *xl, inode_t *exportinode)
 {
-        int             ret = -EFAULT;
-        nfs_user_t      nfu = {0, };
-        loc_t           exportloc = {0, };
-
-        if ((!req) || (!xl) || (!ms) || (!exportinode))
-                return ret;
-
-        ret = nfs_inode_loc_fill (exportinode, &exportloc, NFS_RESOLVE_EXIST);
-        if (ret < 0) {
-                gf_log (GF_MNT, GF_LOG_ERROR, "Loc fill failed for export inode"
-                        ": gfid %s, volume: %s",
-                        uuid_utoa (exportinode->gfid), xl->name);
-                goto err;
-        }
-
-        /* To service the mount request, all we need to do
-         * is to send a lookup fop that returns the stat
-         * for the root of the child volume. This is
-         * used to build the root fh sent to the client.
-         */
-        nfs_request_user_init (&nfu, req);
-        ret = nfs_lookup (ms->nfsx, xl, &nfu, &exportloc,
-                          mnt3svc_lookup_mount_cbk, (void *)req);
+    int ret = -EFAULT;
+    nfs_user_t nfu = {
+        0,
+    };
+    loc_t exportloc = {
+        0,
+    };
+
+    if ((!req) || (!xl) || (!ms) || (!exportinode))
+        return ret;
 
-        nfs_loc_wipe (&exportloc);
+    ret = nfs_inode_loc_fill(exportinode, &exportloc, NFS_RESOLVE_EXIST);
+    if (ret < 0) {
+        gf_msg(GF_MNT, GF_LOG_ERROR, ret, NFS_MSG_INODE_LOC_FILL_ERROR,
+               "Loc fill failed for export inode"
+               ": gfid %s, volume: %s",
+               uuid_utoa(exportinode->gfid), xl->name);
+        goto err;
+    }
+
+    /* To service the mount request, all we need to do
+     * is to send a lookup fop that returns the stat
+     * for the root of the child volume. This is
+     * used to build the root fh sent to the client.
+     */
+    nfs_request_user_init(&nfu, req);
+    ret = nfs_lookup(ms->nfsx, xl, &nfu, &exportloc, mnt3svc_lookup_mount_cbk,
+                     (void *)req);
+
+    nfs_loc_wipe(&exportloc);
 err:
-        return ret;
+    return ret;
 }
 
-
 /* For a volume mount request, we just have to create loc on the root inode,
  * and send a lookup. In the lookup callback the mount reply is send along with
  * the file handle.
  */
 int
-mnt3svc_volume_mount (rpcsvc_request_t *req, struct mount3_state *ms,
-                      struct mnt3_export *exp)
-{
-        inode_t         *exportinode = NULL;
-        int             ret = -EFAULT;
-        uuid_t          rootgfid = {0, };
-
-        if ((!req) || (!exp) || (!ms))
-                return ret;
-
-        rootgfid[15] = 1;
-        exportinode = inode_find (exp->vol->itable, rootgfid);
-        if (!exportinode) {
-                gf_log (GF_MNT, GF_LOG_ERROR, "Failed to get root inode");
-                ret = -ENOENT;
-                goto err;
-        }
+mnt3svc_volume_mount(rpcsvc_request_t *req, struct mount3_state *ms,
+                     struct mnt3_export *exp)
+{
+    inode_t *exportinode = NULL;
+    int ret = -EFAULT;
+    static uuid_t rootgfid = {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1};
 
-        ret = mnt3svc_mount_inode (req, ms, exp->vol, exportinode);
-        inode_unref (exportinode);
+    if ((!req) || (!exp) || (!ms))
+        return ret;
+
+    exportinode = inode_find(exp->vol->itable, rootgfid);
+    if (!exportinode) {
+        gf_msg(GF_MNT, GF_LOG_ERROR, ENOENT, NFS_MSG_GET_ROOT_INODE_FAIL,
+               "Failed to get root inode");
+        ret = -ENOENT;
+        goto err;
+    }
+
+    ret = mnt3svc_mount_inode(req, ms, exp->vol, exportinode);
+    inode_unref(exportinode);
 
 err:
-        return ret;
+    return ret;
 }
 
-
 /* The catch with directory exports is that the first component of the export
  * name will be the name of the volume.
  * Any lookup that needs to be performed to build the directory's file handle
@@ -918,250 +947,280 @@ err:
  * we need to strip out the volume name first.
  */
 char *
-__volume_subdir (char *dirpath, char **volname)
+mnt3_get_volume_subdir(char *dirpath, char **volname)
 {
-        char    *subdir = NULL;
-        int     volname_len = 0;
-
-        if (!dirpath)
-                return NULL;
-
-        if (dirpath[0] == '/')
-                dirpath++;
-
-        subdir = index (dirpath, (int)'/');
-        if (!subdir)
-                goto out;
+    /* subdir points to the first / after the volume name while dirpath
+     * points to the first char of the volume name.
+     */
+    char *subdir = NULL;
+    int volname_len = 0;
+    static char *root = "/";
+
+    /* all callers are expected to pass a valid *dirpath */
+    GF_ASSERT(dirpath);
+
+    if (dirpath[0] == '/')
+        dirpath++;
+
+    subdir = index(dirpath, (int)'/');
+    if (!subdir) {
+        subdir = root;
+        volname_len = strlen(dirpath);
+    } else {
+        volname_len = subdir - dirpath;
+    }
 
-        if (!volname)
-                goto out;
+    if (!volname)
+        goto out;
 
-        if (!*volname)
-                goto out;
+    if (!*volname)
+        goto out;
 
-        /* subdir points to the first / after the volume name while dirpath
-         * points to the first char of the volume name.
-         */
-        volname_len = subdir - dirpath;
-        strncpy (*volname, dirpath, volname_len);
-        *(*volname + volname_len) = '\0';
+    strncpy(*volname, dirpath, volname_len);
+    *(*volname + volname_len) = '\0';
 out:
-        return subdir;
+    return subdir;
 }
 
-
 void
-mnt3_resolve_state_wipe (mnt3_resolve_t *mres)
+mnt3_resolve_state_wipe(mnt3_resolve_t *mres)
 {
-        if (!mres)
-                return;
-
-        nfs_loc_wipe (&mres->resolveloc);
-        GF_FREE (mres);
+    if (!mres)
+        return;
 
+    nfs_loc_wipe(&mres->resolveloc);
+    GF_FREE(mres);
 }
 
-
 /* Sets up the component argument to contain the next component in the path and
  * sets up path as an absolute path starting from the next component.
  */
 static char *
-setup_next_component (char *path, size_t plen, char *component, size_t clen)
+setup_next_component(char *path, size_t plen, char *component, size_t clen)
 {
-        char    *comp = NULL;
-        char    *nextcomp = NULL;
+    char *comp = NULL;
+    char *nextcomp = NULL;
 
-        if ((!path) || (!component))
-                return NULL;
+    if ((!path) || (!component))
+        return NULL;
 
-        strncpy (component, path, clen);
-        comp = index (component, (int)'/');
-        if (!comp)
-                goto err;
+    strncpy(component, path, clen);
+    comp = index(component, (int)'/');
+    if (!comp)
+        goto err;
 
-        comp++;
-        nextcomp = index (comp, (int)'/');
-        if (nextcomp) {
-                strncpy (path, nextcomp, plen);
-                *nextcomp = '\0';
-        } else
-                path[0] = '\0';
+    comp++;
+    nextcomp = index(comp, (int)'/');
+    if (nextcomp) {
+        strncpy(path, nextcomp, plen);
+        *nextcomp = '\0';
+    } else
+        path[0] = '\0';
 
 err:
-        return comp;
+    return comp;
 }
 
 int32_t
-mnt3_resolve_subdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                         int32_t op_ret, int32_t op_errno, inode_t *inode,
-                         struct iatt *buf, dict_t *xattr,
-                         struct iatt *postparent);
-
-int
-mnt3_parse_dir_exports (rpcsvc_request_t *req, struct mount3_state *ms,
-                        char *subdir);
+mnt3_resolve_subdir_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                        int32_t op_ret, int32_t op_errno, inode_t *inode,
+                        struct iatt *buf, dict_t *xattr,
+                        struct iatt *postparent);
 
 int32_t
-mnt3_readlink_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                   int32_t op_ret, int32_t op_errno, const char *path,
-                   struct iatt *buf, dict_t *xdata);
+mnt3_readlink_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                  int32_t op_ret, int32_t op_errno, const char *path,
+                  struct iatt *buf, dict_t *xdata);
 
 /* There are multiple components in the directory export path and each one
  * needs to be looked up one after the other.
  */
 int
-__mnt3_resolve_export_subdir_comp (mnt3_resolve_t *mres)
-{
-        char            dupsubdir[MNTPATHLEN];
-        char            *nextcomp = NULL;
-        int             ret = -EFAULT;
-        nfs_user_t      nfu = {0, };
-        uuid_t          gfid = {0, };
-
-        if (!mres)
-                return ret;
-
-        nextcomp = setup_next_component (mres->remainingdir,
-                                         sizeof (mres->remainingdir),
-                                         dupsubdir, sizeof (dupsubdir));
-        if (!nextcomp)
-                goto err;
-
-        /* Wipe the contents of the previous component */
-        gf_uuid_copy (gfid, mres->resolveloc.inode->gfid);
-        nfs_loc_wipe (&mres->resolveloc);
-        ret = nfs_entry_loc_fill (mres->exp->vol->itable, gfid, nextcomp,
-                                  &mres->resolveloc, NFS_RESOLVE_CREATE);
-        if ((ret < 0) && (ret != -2)) {
-                gf_log (GF_MNT, GF_LOG_ERROR, "Failed to resolve and create "
-                        "inode: parent gfid %s, entry %s",
-                        uuid_utoa (gfid), nextcomp);
-                ret = -EFAULT;
-                goto err;
-        }
+__mnt3_resolve_export_subdir_comp(mnt3_resolve_t *mres)
+{
+    char dupsubdir[MNTPATHLEN];
+    char *nextcomp = NULL;
+    int ret = -EFAULT;
+    nfs_user_t nfu = {
+        0,
+    };
+    uuid_t gfid = {
+        0,
+    };
+
+    if (!mres)
+        return ret;
 
-        nfs_request_user_init (&nfu, mres->req);
-        if (IA_ISLNK (mres->resolveloc.inode->ia_type)) {
-                ret = nfs_readlink (mres->mstate->nfsx, mres->exp->vol, &nfu,
-                                    &mres->resolveloc, mnt3_readlink_cbk, mres);
-                gf_log (GF_MNT, GF_LOG_DEBUG, "Symlink found , need to resolve"
-                                              " into directory handle");
-                goto err;
-        }
-        ret = nfs_lookup (mres->mstate->nfsx, mres->exp->vol, &nfu,
-                          &mres->resolveloc, mnt3_resolve_subdir_cbk, mres);
+    nextcomp = setup_next_component(mres->remainingdir,
+                                    sizeof(mres->remainingdir), dupsubdir,
+                                    sizeof(dupsubdir));
+    if (!nextcomp)
+        goto err;
+
+    /* Wipe the contents of the previous component */
+    gf_uuid_copy(gfid, mres->resolveloc.inode->gfid);
+    nfs_loc_wipe(&mres->resolveloc);
+    ret = nfs_entry_loc_fill(mres->mstate->nfsx, mres->exp->vol->itable, gfid,
+                             nextcomp, &mres->resolveloc, NFS_RESOLVE_CREATE,
+                             NULL);
+    if ((ret < 0) && (ret != -2)) {
+        gf_msg(GF_MNT, GF_LOG_ERROR, EFAULT, NFS_MSG_RESOLVE_INODE_FAIL,
+               "Failed to resolve and "
+               "create inode: parent gfid %s, entry %s",
+               uuid_utoa(gfid), nextcomp);
+        ret = -EFAULT;
+        goto err;
+    }
+
+    nfs_request_user_init(&nfu, mres->req);
+    if (IA_ISLNK(mres->resolveloc.inode->ia_type)) {
+        ret = nfs_readlink(mres->mstate->nfsx, mres->exp->vol, &nfu,
+                           &mres->resolveloc, mnt3_readlink_cbk, mres);
+        gf_msg_debug(GF_MNT, 0,
+                     "Symlink found , need to resolve"
+                     " into directory handle");
+        goto err;
+    }
+    ret = nfs_lookup(mres->mstate->nfsx, mres->exp->vol, &nfu,
+                     &mres->resolveloc, mnt3_resolve_subdir_cbk, mres);
 
 err:
-        return ret;
+    return ret;
 }
 
+int
+__mnt3_resolve_subdir(mnt3_resolve_t *mres);
+
+/*
+ * Per the AFR2 comments, this function performs the "fresh" lookup
+ * by deleting the inode from cache and calling __mnt3_resolve_subdir
+ * again.
+ */
+int
+__mnt3_fresh_lookup(mnt3_resolve_t *mres)
+{
+    inode_unlink(mres->resolveloc.inode, mres->resolveloc.parent,
+                 mres->resolveloc.name);
+    strncpy(mres->remainingdir, mres->resolveloc.path,
+            strlen(mres->resolveloc.path));
+    nfs_loc_wipe(&mres->resolveloc);
+    return __mnt3_resolve_subdir(mres);
+}
 
 int32_t
-mnt3_resolve_subdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                         int32_t op_ret, int32_t op_errno, inode_t *inode,
-                         struct iatt *buf, dict_t *xattr,
-                         struct iatt *postparent)
+mnt3_resolve_subdir_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                        int32_t op_ret, int32_t op_errno, inode_t *inode,
+                        struct iatt *buf, dict_t *xattr,
+                        struct iatt *postparent)
 {
-        mnt3_resolve_t          *mres = NULL;
-        mountstat3              mntstat = MNT3ERR_SERVERFAULT;
-        struct nfs3_fh          fh = {{0}, };
-        int                     autharr[10];
-        int                     autharrlen = 0;
-        rpcsvc_t                *svc = NULL;
-        mountres3               res = {0, };
-        xlator_t                *mntxl = NULL;
-        char                    *path = NULL;
-        struct mount3_state     *ms = NULL;
-        int                     authcode = 0;
-        char                    *authorized_host = NULL;
-        char                    *authorized_path = NULL;
-
-        mres = frame->local;
-        ms = mres->mstate;
-        mntxl = (xlator_t *)cookie;
-        if (op_ret == -1) {
-                gf_log (GF_NFS, GF_LOG_ERROR, "path=%s (%s)",
-                        mres->resolveloc.path, strerror (op_errno));
-                mntstat = mnt3svc_errno_to_mnterr (op_errno);
-                goto err;
+    mnt3_resolve_t *mres = NULL;
+    mountstat3 mntstat = MNT3ERR_SERVERFAULT;
+    struct nfs3_fh fh = {
+        {0},
+    };
+    int autharr[10];
+    int autharrlen = 0;
+    rpcsvc_t *svc = NULL;
+    mountres3 res = {
+        0,
+    };
+    xlator_t *mntxl = NULL;
+    char *path = NULL;
+    struct mount3_state *ms = NULL;
+    int authcode = 0;
+    char *authorized_host = NULL;
+    char *authorized_path = NULL;
+    inode_t *linked_inode = NULL;
+
+    mres = frame->local;
+    ms = mres->mstate;
+    mntxl = (xlator_t *)cookie;
+    if (op_ret == -1 && op_errno == ESTALE) {
+        /* Nuke inode from cache and try the LOOKUP
+         * request again. */
+        return __mnt3_fresh_lookup(mres);
+    } else if (op_ret == -1) {
+        gf_msg(GF_NFS, GF_LOG_ERROR, op_errno, NFS_MSG_RESOLVE_SUBDIR_FAIL,
+               "path=%s (%s)", mres->resolveloc.path, strerror(op_errno));
+        mntstat = mnt3svc_errno_to_mnterr(op_errno);
+        goto err;
+    }
+
+    linked_inode = inode_link(mres->resolveloc.inode, mres->resolveloc.parent,
+                              mres->resolveloc.name, buf);
+
+    if (linked_inode)
+        nfs_fix_generation(this, linked_inode);
+
+    nfs3_fh_build_child_fh(&mres->parentfh, buf, &fh);
+    if (strlen(mres->remainingdir) <= 0) {
+        int alloclen;
+        op_ret = -1;
+        mntstat = MNT3_OK;
+
+        /* Construct the full path */
+        int resolveloc_path_len = strlen(mres->resolveloc.path);
+        alloclen = strlen(mres->exp->expname) + resolveloc_path_len + 1;
+        mres->exp->fullpath = GF_MALLOC(alloclen, gf_nfs_mt_char);
+        if (!mres->exp->fullpath) {
+            gf_msg(GF_MNT, GF_LOG_ERROR, ENOMEM, NFS_MSG_NO_MEMORY,
+                   "Memory allocation failed.");
+            goto err;
+        }
+        snprintf(mres->exp->fullpath, alloclen, "%s%s", mres->exp->expname,
+                 mres->resolveloc.path);
+
+        /* Check if this path is authorized to be mounted */
+        authcode = mnt3_authenticate_request(
+            ms, mres->req, NULL, NULL, mres->exp->fullpath, &authorized_path,
+            &authorized_host, FALSE);
+        if (authcode != 0) {
+            mntstat = MNT3ERR_ACCES;
+            gf_msg_debug(GF_MNT, 0, "Client mount not allowed");
+            op_ret = -1;
+            goto err;
         }
 
-        inode_link (mres->resolveloc.inode, mres->resolveloc.parent,
-                    mres->resolveloc.name, buf);
-
-        nfs3_fh_build_child_fh (&mres->parentfh, buf, &fh);
-        if (strlen (mres->remainingdir) <= 0) {
-                size_t alloclen;
-                op_ret = -1;
-                mntstat = MNT3_OK;
-
-                /* Construct the full path */
-                alloclen = strlen (mres->exp->expname) +
-                                   strlen (mres->resolveloc.path) + 1;
-                mres->exp->fullpath = GF_CALLOC (alloclen, sizeof (char),
-                                                gf_nfs_mt_char);
-                if (!mres->exp->fullpath) {
-                        gf_log (GF_MNT, GF_LOG_CRITICAL, "Allocation failed.");
-                        goto err;
-                }
-                snprintf (mres->exp->fullpath, alloclen, "%s%s",
-                                mres->exp->expname, mres->resolveloc.path);
-
-                /* Check if this path is authorized to be mounted */
-                authcode = mnt3_authenticate_request (ms, mres->req, NULL, NULL,
-                                                      mres->exp->fullpath,
-                                                      &authorized_path,
-                                                      &authorized_host,
-                                                      FALSE);
-                if (authcode != 0) {
-                        mntstat = MNT3ERR_ACCES;
-                        gf_log (GF_MNT, GF_LOG_DEBUG,
-                                "Client mount not allowed");
-                        op_ret = -1;
-                        goto err;
-                }
-
-                path = GF_CALLOC (PATH_MAX, sizeof (char), gf_nfs_mt_char);
-                if (!path) {
-                        gf_log (GF_MNT, GF_LOG_ERROR, "Memory allocation "
-                                "failed");
-                        goto err;
-                }
-                /* Build mountid from the authorized path and stick it in the
-                 * filehandle that will get passed back to the client
-                 */
-                __mnt3_build_mountid_from_path (authorized_path, fh.mountid);
-
-                snprintf (path, PATH_MAX, "/%s%s", mres->exp->vol->name,
-                         mres->resolveloc.path);
-
-                mnt3svc_update_mountlist (mres->mstate, mres->req,
-                                          path, mres->exp->fullpath);
-                GF_FREE (path);
-        } else {
-                mres->parentfh = fh;
-                op_ret = __mnt3_resolve_export_subdir_comp (mres);
-                if (op_ret < 0)
-                        mntstat = mnt3svc_errno_to_mnterr (-op_ret);
+        alloclen = strlen(mres->exp->vol->name) + resolveloc_path_len + 2;
+        path = GF_MALLOC(alloclen, gf_nfs_mt_char);
+        if (!path) {
+            gf_msg(GF_MNT, GF_LOG_ERROR, ENOMEM, NFS_MSG_NO_MEMORY,
+                   "Memory allocation failed");
+            goto err;
         }
+        /* Build mountid from the authorized path and stick it in the
+         * filehandle that will get passed back to the client
+         */
+        __mnt3_build_mountid_from_path(authorized_path, fh.mountid);
+
+        snprintf(path, alloclen, "/%s%s", mres->exp->vol->name,
+                 mres->resolveloc.path);
+
+        mnt3svc_update_mountlist(mres->mstate, mres->req, path,
+                                 mres->exp->fullpath);
+        GF_FREE(path);
+    } else {
+        mres->parentfh = fh;
+        op_ret = __mnt3_resolve_export_subdir_comp(mres);
+        if (op_ret < 0)
+            mntstat = mnt3svc_errno_to_mnterr(-op_ret);
+    }
 err:
-        if (op_ret == -1) {
-                gf_log (GF_MNT, GF_LOG_DEBUG, "Mount reply status: %d",
-                        mntstat);
-                svc = rpcsvc_request_service (mres->req);
-                autharrlen = rpcsvc_auth_array (svc, mntxl->name, autharr,
-                                                10);
-
-                res = mnt3svc_set_mountres3 (mntstat, &fh, autharr, autharrlen);
-                mnt3svc_submit_reply (mres->req, (void *)&res,
-                                      (mnt3_serializer)xdr_serialize_mountres3);
-                mnt3_resolve_state_wipe (mres);
-        }
+    if (op_ret == -1) {
+        gf_msg_debug(GF_MNT, 0, "Mount reply status: %d", mntstat);
+        svc = rpcsvc_request_service(mres->req);
+        autharrlen = rpcsvc_auth_array(svc, mntxl->name, autharr, 10);
 
-        GF_FREE (authorized_path);
-        GF_FREE (authorized_host);
+        res = mnt3svc_set_mountres3(mntstat, &fh, autharr, autharrlen);
+        mnt3svc_submit_reply(mres->req, (void *)&res,
+                             (mnt3_serializer)xdr_serialize_mountres3);
+        mnt3_resolve_state_wipe(mres);
+    }
 
-        return 0;
+    GF_FREE(authorized_path);
+    GF_FREE(authorized_host);
+
+    return 0;
 }
 
 /* This function resolves symbolic link into directory path from
@@ -1188,103 +1247,106 @@ err:
  *            similar to udp
  */
 int32_t
-mnt3_readlink_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                   int32_t op_ret, int32_t op_errno, const char *path,
-                   struct iatt *buf, dict_t *xdata)
-{
-        mnt3_resolve_t           *mres            = NULL;
-        int                      ret              = -EFAULT;
-        char                     *real_loc        = NULL;
-        size_t                   path_len         = 0;
-        size_t                   parent_path_len  = 0;
-        char                     *parent_path     = NULL;
-        char                     *absolute_path   = NULL;
-        char                     *relative_path   = NULL;
-        int                      mntstat          = 0;
-
-        GF_ASSERT (frame);
-
-        mres = frame->local;
-        if (!mres || !path || (path[0] == '/') || (op_ret < 0))
-                goto mnterr;
-
-        /* Finding current location of symlink */
-        parent_path_len = strlen (mres->resolveloc.path) - strlen (mres->resolveloc.name);
-        parent_path = gf_strndup (mres->resolveloc.path, parent_path_len);
-        if (!parent_path) {
-                ret = -ENOMEM;
-                goto mnterr;
-        }
-
-        relative_path = gf_strdup (path);
-        if (!relative_path) {
-                ret = -ENOMEM;
-                goto mnterr;
-        }
-        /* Resolving into absolute path */
-        ret = gf_build_absolute_path (parent_path, relative_path, &absolute_path);
-        if (ret < 0) {
-                gf_log (GF_MNT, GF_LOG_ERROR, "Cannot resolve symlink, path"
-                               "is out of boundary from current location %s"
-                               "and with relative path %s pointed by symlink",
-                                parent_path, relative_path);
-
-                goto mnterr;
-        }
-
-        /* Building the actual mount path to be mounted */
-        path_len = strlen (mres->exp->vol->name) +  strlen (absolute_path)
-                   + strlen (mres->remainingdir) + 1;
-        real_loc = GF_CALLOC (1, path_len, gf_nfs_mt_char);
-        if (!real_loc) {
-                ret = -ENOMEM;
-                goto mnterr;
-        }
-        sprintf (real_loc , "%s%s", mres->exp->vol->name, absolute_path);
-        gf_path_strip_trailing_slashes (real_loc);
-
-        /* There may entries after symlink in the mount path,
-         * we should include remaining entries too */
-        if (strlen (mres->remainingdir) > 0)
-                strcat (real_loc, mres->remainingdir);
-
-        gf_log (GF_MNT, GF_LOG_DEBUG, "Resolved path is : %s%s "
-                        "and actual mount path is %s",
-                        absolute_path, mres->remainingdir, real_loc);
-
-        /* After the resolving the symlink , parsing should be done
-         * for the populated mount path
-         */
-        ret = mnt3_parse_dir_exports (mres->req, mres->mstate, real_loc);
-
-        if (ret) {
-                gf_log (GF_MNT, GF_LOG_ERROR,
-                        "Resolved into an unknown path %s%s "
-                        "from the current location of symlink %s",
-                         absolute_path, mres->remainingdir, parent_path);
-        }
-
-        GF_FREE (real_loc);
-        GF_FREE (absolute_path);
-        GF_FREE (parent_path);
-        GF_FREE (relative_path);
-
-        return ret;
+mnt3_readlink_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                  int32_t op_ret, int32_t op_errno, const char *path,
+                  struct iatt *buf, dict_t *xdata)
+{
+    mnt3_resolve_t *mres = NULL;
+    int ret = -EFAULT;
+    char *real_loc = NULL;
+    size_t path_len = 0;
+    size_t parent_path_len = 0;
+    char *parent_path = NULL;
+    char *absolute_path = NULL;
+    char *relative_path = NULL;
+    int mntstat = 0;
+
+    GF_ASSERT(frame);
+
+    mres = frame->local;
+    if (!mres || !path || (path[0] == '/') || (op_ret < 0))
+        goto mnterr;
+
+    /* Finding current location of symlink */
+    parent_path_len = strlen(mres->resolveloc.path) -
+                      strlen(mres->resolveloc.name);
+    parent_path = gf_strndup(mres->resolveloc.path, parent_path_len);
+    if (!parent_path) {
+        ret = -ENOMEM;
+        goto mnterr;
+    }
+
+    relative_path = gf_strdup(path);
+    if (!relative_path) {
+        ret = -ENOMEM;
+        goto mnterr;
+    }
+    /* Resolving into absolute path */
+    ret = gf_build_absolute_path(parent_path, relative_path, &absolute_path);
+    if (ret < 0) {
+        gf_msg(GF_MNT, GF_LOG_ERROR, ret, NFS_MSG_RESOLVE_SYMLINK_ERROR,
+               "Cannot resolve symlink, path is out of boundary "
+               "from current location %s and with relative path "
+               "%s pointed by symlink",
+               parent_path, relative_path);
+
+        goto mnterr;
+    }
+
+    /* Building the actual mount path to be mounted */
+    path_len = strlen(mres->exp->vol->name) + strlen(absolute_path) +
+               strlen(mres->remainingdir) + 1;
+    real_loc = GF_MALLOC(path_len, gf_nfs_mt_char);
+    if (!real_loc) {
+        ret = -ENOMEM;
+        goto mnterr;
+    }
+    snprintf(real_loc, path_len, "%s%s", mres->exp->vol->name, absolute_path);
+    gf_path_strip_trailing_slashes(real_loc);
+
+    /* There may entries after symlink in the mount path,
+     * we should include remaining entries too */
+    if (strlen(mres->remainingdir) > 0)
+        strcat(real_loc, mres->remainingdir);
+
+    gf_msg_debug(GF_MNT, 0,
+                 "Resolved path is : %s%s "
+                 "and actual mount path is %s",
+                 absolute_path, mres->remainingdir, real_loc);
+
+    /* After the resolving the symlink , parsing should be done
+     * for the populated mount path
+     */
+    ret = mnt3_parse_dir_exports(mres->req, mres->mstate, real_loc, _gf_true);
+
+    if (ret) {
+        gf_msg(GF_MNT, GF_LOG_ERROR, 0, NFS_MSG_RESOLVE_ERROR,
+               "Resolved into an unknown path %s%s "
+               "from the current location of symlink %s",
+               absolute_path, mres->remainingdir, parent_path);
+    }
+
+    GF_FREE(real_loc);
+    GF_FREE(absolute_path);
+    GF_FREE(parent_path);
+    GF_FREE(relative_path);
+
+    return ret;
 
 mnterr:
-        if (mres) {
-                mntstat = mnt3svc_errno_to_mnterr (-ret);
-                mnt3svc_mnt_error_reply (mres->req, mntstat);
-        } else
-                gf_log (GF_MNT, GF_LOG_CRITICAL,
-                        "mres == NULL, this should *never* happen");
-        if (absolute_path)
-                GF_FREE (absolute_path);
-        if (parent_path)
-                GF_FREE (parent_path);
-        if (relative_path)
-                GF_FREE (relative_path);
-        return ret;
+    if (mres) {
+        mntstat = mnt3svc_errno_to_mnterr(-ret);
+        mnt3svc_mnt_error_reply(mres->req, mntstat);
+    } else
+        gf_msg(GF_MNT, GF_LOG_CRITICAL, EINVAL, NFS_MSG_INVALID_ENTRY,
+               "mres == NULL, this should *never* happen");
+    if (absolute_path)
+        GF_FREE(absolute_path);
+    if (parent_path)
+        GF_FREE(parent_path);
+    if (relative_path)
+        GF_FREE(relative_path);
+    return ret;
 }
 
 /* We will always have to perform a hard lookup on all the components of a
@@ -1299,66 +1361,69 @@ mnterr:
  * of the exported directory can be built.
  */
 int
-__mnt3_resolve_subdir (mnt3_resolve_t *mres)
-{
-        char            dupsubdir[MNTPATHLEN];
-        char            *firstcomp = NULL;
-        int             ret = -EFAULT;
-        nfs_user_t      nfu = {0, };
-        uuid_t          rootgfid = {0, };
-
-        if (!mres)
-                return ret;
-
-        firstcomp = setup_next_component (mres->remainingdir,
-                                          sizeof (mres->remainingdir),
-                                          dupsubdir, sizeof (dupsubdir));
-        if (!firstcomp)
-                goto err;
-
-        rootgfid[15] = 1;
-        ret = nfs_entry_loc_fill (mres->exp->vol->itable, rootgfid, firstcomp,
-                                  &mres->resolveloc, NFS_RESOLVE_CREATE);
-        if ((ret < 0) && (ret != -2)) {
-                gf_log (GF_MNT, GF_LOG_ERROR, "Failed to resolve and create "
-                        "inode for volume root: %s", mres->exp->vol->name);
-                ret = -EFAULT;
-                goto err;
-        }
+__mnt3_resolve_subdir(mnt3_resolve_t *mres)
+{
+    char dupsubdir[MNTPATHLEN];
+    char *firstcomp = NULL;
+    int ret = -EFAULT;
+    nfs_user_t nfu = {
+        0,
+    };
+    static uuid_t rootgfid = {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1};
+
+    if (!mres)
+        return ret;
 
-        nfs_request_user_init (&nfu, mres->req);
-        if (IA_ISLNK (mres->resolveloc.inode->ia_type)) {
-                ret = nfs_readlink (mres->mstate->nfsx, mres->exp->vol, &nfu,
-                                    &mres->resolveloc, mnt3_readlink_cbk, mres);
-                gf_log (GF_MNT, GF_LOG_DEBUG, "Symlink found , need to resolve "
-                                              "into directory handle");
-                goto err;
-        }
-        ret = nfs_lookup (mres->mstate->nfsx, mres->exp->vol, &nfu,
-                          &mres->resolveloc, mnt3_resolve_subdir_cbk, mres);
+    firstcomp = setup_next_component(mres->remainingdir,
+                                     sizeof(mres->remainingdir), dupsubdir,
+                                     sizeof(dupsubdir));
+    if (!firstcomp)
+        goto err;
+
+    ret = nfs_entry_loc_fill(mres->mstate->nfsx, mres->exp->vol->itable,
+                             rootgfid, firstcomp, &mres->resolveloc,
+                             NFS_RESOLVE_CREATE, NULL);
+    if ((ret < 0) && (ret != -2)) {
+        gf_msg(GF_MNT, GF_LOG_ERROR, EFAULT, NFS_MSG_RESOLVE_INODE_FAIL,
+               "Failed to resolve and "
+               "create inode for volume root: %s",
+               mres->exp->vol->name);
+        ret = -EFAULT;
+        goto err;
+    }
+
+    nfs_request_user_init(&nfu, mres->req);
+    if (IA_ISLNK(mres->resolveloc.inode->ia_type)) {
+        ret = nfs_readlink(mres->mstate->nfsx, mres->exp->vol, &nfu,
+                           &mres->resolveloc, mnt3_readlink_cbk, mres);
+        gf_msg_debug(GF_MNT, 0,
+                     "Symlink found , need to resolve "
+                     "into directory handle");
+        goto err;
+    }
+    ret = nfs_lookup(mres->mstate->nfsx, mres->exp->vol, &nfu,
+                     &mres->resolveloc, mnt3_resolve_subdir_cbk, mres);
 
 err:
-        return ret;
+    return ret;
 }
 
-
 static gf_boolean_t
-mnt3_match_subnet_v4 (struct addrinfo *ai, uint32_t saddr, uint32_t mask)
+mnt3_match_subnet_v4(struct addrinfo *ai, uint32_t saddr, uint32_t mask)
 {
-        for (; ai; ai = ai->ai_next) {
-                struct sockaddr_in *sin = (struct sockaddr_in *)ai->ai_addr;
+    for (; ai; ai = ai->ai_next) {
+        struct sockaddr_in *sin = (struct sockaddr_in *)ai->ai_addr;
 
-                if (sin->sin_family != AF_INET)
-                        continue;
+        if (sin->sin_family != AF_INET)
+            continue;
 
-                if (mask_match (saddr, sin->sin_addr.s_addr, mask))
-                        return _gf_true;
-        }
+        if (mask_match(saddr, sin->sin_addr.s_addr, mask))
+            return _gf_true;
+    }
 
-        return _gf_false;
+    return _gf_false;
 }
 
-
 /**
  * This function will verify if the client is allowed to mount
  * the directory or not. Client's IP address will be compared with
@@ -1372,409 +1437,448 @@ mnt3_match_subnet_v4 (struct addrinfo *ai, uint32_t saddr, uint32_t mask)
  * TODO: Support IPv6 subnetwork
  */
 int
-mnt3_verify_auth (struct sockaddr_in *client_addr, struct mnt3_export *export)
-{
-        int                     retvalue = -EACCES;
-        int                     ret = 0;
-        struct host_auth_spec   *host = NULL;
-        struct sockaddr_in      *allowed_addr = NULL;
-        struct addrinfo         *allowed_addrinfo = NULL;
-
-        struct addrinfo         hint = {
-                .ai_family      = AF_INET,
-                .ai_protocol    = (int)IPPROTO_TCP,
-                .ai_flags       = AI_CANONNAME,
-        };
+mnt3_verify_auth(struct sockaddr_in *client_addr, struct mnt3_export *export)
+{
+    int retvalue = -EACCES;
+    int ret = 0;
+    struct host_auth_spec *host = NULL;
+    struct sockaddr_in *allowed_addr = NULL;
+    struct addrinfo *allowed_addrinfo = NULL;
+
+    struct addrinfo hint = {
+        .ai_family = AF_INET,
+        .ai_protocol = (int)IPPROTO_TCP,
+        .ai_flags = AI_CANONNAME,
+    };
+
+    /* Sanity check */
+    if ((NULL == client_addr) || (NULL == export) ||
+        (NULL == export->hostspec)) {
+        gf_msg(GF_MNT, GF_LOG_ERROR, EINVAL, NFS_MSG_INVALID_ENTRY,
+               "Invalid argument");
+        return retvalue;
+    }
+
+    host = export->hostspec;
+
+    /*
+     * Currently IPv4 subnetwork is supported i.e. AF_INET.
+     * TODO: IPv6 subnetwork i.e. AF_INET6.
+     */
+    if (client_addr->sin_family != AF_INET) {
+        gf_msg(GF_MNT, GF_LOG_ERROR, EAFNOSUPPORT, NFS_MSG_UNSUPPORTED_VERSION,
+               "Only IPv4 is supported for subdir-auth");
+        return retvalue;
+    }
 
-        /* Sanity check */
-        if ((NULL == client_addr) ||
-            (NULL == export) ||
-            (NULL == export->hostspec)) {
-                gf_log (GF_MNT, GF_LOG_ERROR, "Invalid argument");
-                return retvalue;
+    /* Try to see if the client IP matches the allowed IP list.*/
+    while (NULL != host) {
+        GF_ASSERT(host->host_addr);
+
+        if (NULL != allowed_addrinfo) {
+            freeaddrinfo(allowed_addrinfo);
+            allowed_addrinfo = NULL;
         }
 
-        host = export->hostspec;
+        /* Get the addrinfo for the allowed host (host_addr). */
+        ret = getaddrinfo(host->host_addr, NULL, &hint, &allowed_addrinfo);
+        if (0 != ret) {
+            /*
+             * getaddrinfo() FAILED for the host IP addr. Continue
+             * to search other allowed hosts in the  hostspec list.
+             */
+            gf_msg_debug(GF_MNT, 0, "getaddrinfo: %s\n", gai_strerror(ret));
+            host = host->next;
+            continue;
+        }
 
-        /*
-         * Currently IPv4 subnetwork is supported i.e. AF_INET.
-         * TODO: IPv6 subnetwork i.e. AF_INET6.
-         */
-        if (client_addr->sin_family != AF_INET) {
-                gf_log (GF_MNT, GF_LOG_ERROR,
-                        "Only IPv4 is supported for subdir-auth");
-                return retvalue;
+        allowed_addr = (struct sockaddr_in *)(allowed_addrinfo->ai_addr);
+        if (NULL == allowed_addr) {
+            gf_msg(GF_MNT, GF_LOG_ERROR, EINVAL, NFS_MSG_INVALID_ENTRY,
+                   "Invalid structure");
+            break;
         }
 
-        /* Try to see if the client IP matches the allowed IP list.*/
-        while (NULL != host){
-                GF_ASSERT (host->host_addr);
+        /* Check if the network addr of both IPv4 socket match */
+        if (mnt3_match_subnet_v4(allowed_addrinfo, client_addr->sin_addr.s_addr,
+                                 host->netmask)) {
+            retvalue = 0;
+            break;
+        }
 
-                if (NULL != allowed_addrinfo) {
-                        freeaddrinfo (allowed_addrinfo);
-                        allowed_addrinfo = NULL;
-                }
+        /* No match yet, continue the search */
+        host = host->next;
+    }
 
-                /* Get the addrinfo for the allowed host (host_addr). */
-                ret = getaddrinfo (host->host_addr, NULL,
-                                   &hint, &allowed_addrinfo);
-                if (0 != ret){
-                        /*
-                         * getaddrinfo() FAILED for the host IP addr. Continue
-                         * to search other allowed hosts in the  hostspec list.
-                         */
-                        gf_log (GF_MNT, GF_LOG_DEBUG,
-                                "getaddrinfo: %s\n", gai_strerror (ret));
-                        host = host->next;
-                        continue;
-                }
+    /* FREE the dynamic memory allocated by getaddrinfo() */
+    if (NULL != allowed_addrinfo) {
+        freeaddrinfo(allowed_addrinfo);
+    }
 
-                allowed_addr = (struct sockaddr_in *)(allowed_addrinfo->ai_addr);
-                if (NULL == allowed_addr) {
-                        gf_log (GF_MNT, GF_LOG_ERROR, "Invalid structure");
-                        break;
-                }
+    return retvalue;
+}
 
-                /* Check if the network addr of both IPv4 socket match */
-                if (mnt3_match_subnet_v4 (allowed_addrinfo,
-                                          client_addr->sin_addr.s_addr,
-                                          host->netmask)) {
-                        retvalue = 0;
-                        break;
-                }
+int
+mnt3_resolve_subdir(rpcsvc_request_t *req, struct mount3_state *ms,
+                    struct mnt3_export *exp, char *subdir,
+                    gf_boolean_t send_reply)
+{
+    mnt3_resolve_t *mres = NULL;
+    int ret = -EFAULT;
+    struct nfs3_fh pfh = GF_NFS3FH_STATIC_INITIALIZER;
+    struct sockaddr_in *sin = NULL;
 
-                /* No match yet, continue the search */
-               host = host->next;
-        }
+    if ((!req) || (!ms) || (!exp) || (!subdir))
+        return ret;
 
-        /* FREE the dynamic memory allocated by getaddrinfo() */
-        if (NULL != allowed_addrinfo) {
-               freeaddrinfo (allowed_addrinfo);
+    sin = (struct sockaddr_in *)(&(req->trans->peerinfo.sockaddr));
+
+    /* Need to check AUTH */
+    if (NULL != exp->hostspec) {
+        ret = mnt3_verify_auth(sin, exp);
+        if (0 != ret) {
+            gf_msg(GF_MNT, GF_LOG_ERROR, EACCES, NFS_MSG_AUTH_VERIFY_FAILED,
+                   "AUTH verification failed");
+            return ret;
         }
+    }
 
-        return retvalue;
-}
+    /* no reply is needed (WebNFS permissions checking), just return */
+    if (!send_reply)
+        return 0; /* no error, mnt3_verify_auth() allowed it */
 
-int
-mnt3_resolve_subdir (rpcsvc_request_t *req, struct mount3_state *ms,
-                     struct mnt3_export *exp, char *subdir)
-{
-        mnt3_resolve_t        *mres = NULL;
-        int                   ret = -EFAULT;
-        struct nfs3_fh        pfh = GF_NFS3FH_STATIC_INITIALIZER;
-        struct sockaddr_in    *sin = NULL;
-
-        if ((!req) || (!ms) || (!exp) || (!subdir))
-                return ret;
-
-        sin = (struct sockaddr_in *)(&(req->trans->peerinfo.sockaddr));
-
-        /* Need to check AUTH */
-        if (NULL != exp->hostspec) {
-                ret = mnt3_verify_auth (sin, exp);
-                if (0 != ret) {
-                        gf_log (GF_MNT,GF_LOG_ERROR,
-                                        "AUTH verification failed");
-                        return ret;
-                }
-        }
+    mres = GF_CALLOC(1, sizeof(mnt3_resolve_t), gf_nfs_mt_mnt3_resolve);
+    if (!mres) {
+        gf_msg(GF_MNT, GF_LOG_ERROR, ENOMEM, NFS_MSG_NO_MEMORY,
+               "Memory allocation failed");
+        goto err;
+    }
 
-        mres = GF_CALLOC (1, sizeof (mnt3_resolve_t), gf_nfs_mt_mnt3_resolve);
-        if (!mres) {
-                gf_log (GF_MNT, GF_LOG_ERROR, "Memory allocation failed");
-                goto err;
-        }
+    mres->exp = exp;
+    mres->mstate = ms;
+    mres->req = req;
+
+    snprintf(mres->remainingdir, MNTPATHLEN, "%s", subdir);
+    gf_path_strip_trailing_slashes(mres->remainingdir);
 
-        mres->exp = exp;
-        mres->mstate = ms;
-        mres->req = req;
-        strncpy (mres->remainingdir, subdir, MNTPATHLEN);
-        if (gf_nfs_dvm_off (nfs_state (ms->nfsx)))
-                pfh = nfs3_fh_build_indexed_root_fh (
-                                            mres->mstate->nfsx->children,
+    if (gf_nfs_dvm_off(nfs_state(ms->nfsx)))
+        pfh = nfs3_fh_build_indexed_root_fh(mres->mstate->nfsx->children,
                                             mres->exp->vol);
-        else
-                pfh = nfs3_fh_build_uuid_root_fh (exp->volumeid, exp->mountid);
+    else
+        pfh = nfs3_fh_build_uuid_root_fh(exp->volumeid, exp->mountid);
 
-        mres->parentfh = pfh;
-        ret = __mnt3_resolve_subdir (mres);
-        if (ret < 0) {
-                gf_log (GF_MNT, GF_LOG_ERROR,
-                        "Failed to resolve export dir: %s", mres->exp->expname);
-                GF_FREE (mres);
-        }
+    mres->parentfh = pfh;
+    ret = __mnt3_resolve_subdir(mres);
+    if (ret < 0) {
+        gf_msg(GF_MNT, GF_LOG_ERROR, ret, NFS_MSG_RESOLVE_SUBDIR_FAIL,
+               "Failed to resolve export dir: %s", mres->exp->expname);
+        GF_FREE(mres);
+    }
 
 err:
-        return ret;
+    return ret;
 }
 
-
 int
-mnt3_resolve_export_subdir (rpcsvc_request_t *req, struct mount3_state *ms,
-                            struct mnt3_export *exp)
+mnt3_resolve_export_subdir(rpcsvc_request_t *req, struct mount3_state *ms,
+                           struct mnt3_export *exp)
 {
-        char            *volume_subdir = NULL;
-        int             ret = -EFAULT;
+    char *volume_subdir = NULL;
+    int ret = -EFAULT;
 
-        if ((!req) || (!ms) || (!exp))
-                return ret;
+    if ((!req) || (!ms) || (!exp))
+        return ret;
 
-        volume_subdir = __volume_subdir (exp->expname, NULL);
-        if (!volume_subdir)
-                goto err;
+    volume_subdir = mnt3_get_volume_subdir(exp->expname, NULL);
 
-        ret = mnt3_resolve_subdir (req, ms, exp, volume_subdir);
-        if (ret < 0) {
-                gf_log (GF_MNT, GF_LOG_ERROR,
-                        "Failed to resolve export dir: %s", exp->expname);
-                goto err;
-        }
+    ret = mnt3_resolve_subdir(req, ms, exp, volume_subdir, _gf_true);
+    if (ret < 0) {
+        gf_msg(GF_MNT, GF_LOG_ERROR, ret, NFS_MSG_RESOLVE_SUBDIR_FAIL,
+               "Failed to resolve export dir: %s", exp->expname);
+        goto err;
+    }
 
 err:
-        return ret;
+    return ret;
 }
 
-
 int
-mnt3svc_mount (rpcsvc_request_t *req, struct mount3_state *ms,
-               struct mnt3_export *exp)
+mnt3svc_mount(rpcsvc_request_t *req, struct mount3_state *ms,
+              struct mnt3_export *exp)
 {
-        int     ret = -EFAULT;
+    int ret = -EFAULT;
 
-        if ((!req) || (!ms) || (!exp))
-                return ret;
+    if ((!req) || (!ms) || (!exp))
+        return ret;
 
-        if (exp->exptype == MNT3_EXPTYPE_VOLUME)
-                ret = mnt3svc_volume_mount (req, ms, exp);
-        else if (exp->exptype == MNT3_EXPTYPE_DIR)
-                ret = mnt3_resolve_export_subdir (req, ms, exp);
+    if (exp->exptype == MNT3_EXPTYPE_VOLUME)
+        ret = mnt3svc_volume_mount(req, ms, exp);
+    else if (exp->exptype == MNT3_EXPTYPE_DIR)
+        ret = mnt3_resolve_export_subdir(req, ms, exp);
 
-        return ret;
+    return ret;
 }
 
-
 /* mnt3_mntpath_to_xlator sets this to 1 if the mount is for a full
-* volume or 2 for a subdir in the volume.
-*
-* The parameter 'export_parsing_match' indicates whether this function
-* is being called by an exports parser or whether it is being called
-* during mount. The behavior is different since we don't have to resolve
-* the path when doing the parse.
-*/
+ * volume or 2 for a subdir in the volume.
+ *
+ * The parameter 'export_parsing_match' indicates whether this function
+ * is being called by an exports parser or whether it is being called
+ * during mount. The behavior is different since we don't have to resolve
+ * the path when doing the parse.
+ */
 struct mnt3_export *
-mnt3_mntpath_to_export (struct mount3_state *ms, const char *dirpath,
-                        gf_boolean_t export_parsing_match)
+mnt3_mntpath_to_export(struct mount3_state *ms, const char *dirpath,
+                       gf_boolean_t export_parsing_match)
 {
-        struct mnt3_export      *exp = NULL;
-        struct mnt3_export      *found = NULL;
+    struct mnt3_export *exp = NULL;
+    struct mnt3_export *found = NULL;
 
-        if ((!ms) || (!dirpath))
-                return NULL;
-
-        LOCK (&ms->mountlock);
-        list_for_each_entry (exp, &ms->exportlist, explist) {
-
-                /* Search for the an exact match with the volume */
-                if (mnt3_match_dirpath_export (exp->expname, dirpath,
-                                               export_parsing_match)) {
-                        found = exp;
-                        gf_log (GF_MNT, GF_LOG_DEBUG, "Found export volume: "
-                                "%s", exp->vol->name);
-                        goto foundexp;
-                }
-        }
+    if ((!ms) || (!dirpath))
+        return NULL;
 
-        gf_log (GF_MNT, GF_LOG_DEBUG, "Export not found");
+    LOCK(&ms->mountlock);
+    list_for_each_entry(exp, &ms->exportlist, explist)
+    {
+        /* Search for the an exact match with the volume */
+        if (mnt3_match_dirpath_export(exp->expname, dirpath,
+                                      export_parsing_match)) {
+            found = exp;
+            gf_msg_debug(GF_MNT, 0,
+                         "Found export volume: "
+                         "%s",
+                         exp->vol->name);
+            goto foundexp;
+        }
+    }
+
+    gf_msg_debug(GF_MNT, 0, "Export not found");
 foundexp:
-        UNLOCK (&ms->mountlock);
-        return found;
+    UNLOCK(&ms->mountlock);
+    return found;
 }
 
-
 static int
-mnt3_check_client_net_check (rpcsvc_t *svc, char *expvol,
-                             char *ipaddr, uint16_t port)
+mnt3_check_client_net_check(rpcsvc_t *svc, char *expvol, char *ipaddr,
+                            uint16_t port)
 {
-        int ret = RPCSVC_AUTH_REJECT;
-
-        if ((!svc) || (!expvol) || (!ipaddr))
-                goto err;
-
-        ret = rpcsvc_auth_check (svc, expvol, ipaddr);
-        if (ret == RPCSVC_AUTH_REJECT) {
-                gf_log (GF_MNT, GF_LOG_INFO, "Peer %s  not allowed", ipaddr);
-                goto err;
-        }
-
-        ret = rpcsvc_transport_privport_check (svc, expvol, port);
-        if (ret == RPCSVC_AUTH_REJECT) {
-                gf_log (GF_MNT, GF_LOG_INFO, "Peer %s rejected. Unprivileged "
-                        "port %d not allowed", ipaddr, port);
-                goto err;
-        }
-
-        ret = RPCSVC_AUTH_ACCEPT;
+    int ret = RPCSVC_AUTH_REJECT;
+
+    if ((!svc) || (!expvol) || (!ipaddr))
+        goto err;
+
+    ret = rpcsvc_auth_check(svc, expvol, ipaddr);
+    if (ret == RPCSVC_AUTH_REJECT) {
+        gf_msg(GF_MNT, GF_LOG_INFO, 0, NFS_MSG_PEER_NOT_ALLOWED,
+               "Peer %s  not allowed", ipaddr);
+        goto err;
+    }
+
+    ret = rpcsvc_transport_privport_check(svc, expvol, port);
+    if (ret == RPCSVC_AUTH_REJECT) {
+        gf_msg(GF_MNT, GF_LOG_INFO, errno, NFS_MSG_PEER_NOT_ALLOWED,
+               "Peer %s rejected. Unprivileged "
+               "port %d not allowed",
+               ipaddr, port);
+        goto err;
+    }
+
+    ret = RPCSVC_AUTH_ACCEPT;
 err:
-        return ret;
+    return ret;
 }
 
-static int
-mnt3_check_client_net_tcp (rpcsvc_request_t *req, char *volname)
-{
-        rpcsvc_t                *svc = NULL;
-        rpc_transport_t         *trans = NULL;
-        union gf_sock_union     sock_union;
-        socklen_t               socksize = sizeof (struct sockaddr_in);
-        char                    peer[RPCSVC_PEER_STRLEN] = {0,};
-        char                    *ipaddr = NULL;
-        uint16_t                port = 0;
-        int                     ret = RPCSVC_AUTH_REJECT;
-
-        if ((!req) || (!volname))
-                goto err;
-
-        svc = rpcsvc_request_service (req);
-        trans = rpcsvc_request_transport (req);
-        if ((!svc) || (!trans))
-                goto err;
-
-        ret = rpcsvc_transport_peeraddr (trans, peer, RPCSVC_PEER_STRLEN,
-                                         &sock_union.storage, socksize);
-        if (ret != 0) {
-                gf_log (GF_MNT, GF_LOG_WARNING,
-                                "Failed to get peer addr: %s",
-                                gai_strerror (ret));
-                ret = RPCSVC_AUTH_REJECT;
-                goto err;
-        }
-
-        /* peer[] gets IP:PORT formar, slash the port out */
-        if (!get_host_name ((char *)peer, &ipaddr))
-                ipaddr = peer;
-
-        port = ntohs (sock_union.sin.sin_port);
-
-        ret = mnt3_check_client_net_check (svc, volname, ipaddr, port);
+int
+mnt3_check_client_net_tcp(rpcsvc_request_t *req, char *volname)
+{
+    rpcsvc_t *svc = NULL;
+    rpc_transport_t *trans = NULL;
+    union gf_sock_union sock_union;
+    socklen_t socksize = sizeof(struct sockaddr_in);
+    char peer[RPCSVC_PEER_STRLEN] = {
+        0,
+    };
+    char *ipaddr = NULL;
+    uint16_t port = 0;
+    int ret = RPCSVC_AUTH_REJECT;
+
+    if ((!req) || (!volname))
+        goto err;
+
+    svc = rpcsvc_request_service(req);
+    trans = rpcsvc_request_transport(req);
+    if ((!svc) || (!trans))
+        goto err;
+
+    ret = rpcsvc_transport_peeraddr(trans, peer, RPCSVC_PEER_STRLEN,
+                                    &sock_union.storage, socksize);
+    if (ret != 0) {
+        gf_msg(GF_MNT, GF_LOG_WARNING, ENOENT, NFS_MSG_GET_PEER_ADDR_FAIL,
+               "Failed to get peer "
+               "addr: %s",
+               gai_strerror(ret));
+        ret = RPCSVC_AUTH_REJECT;
+        goto err;
+    }
+
+    /* peer[] gets IP:PORT formar, slash the port out */
+    if (!get_host_name((char *)peer, &ipaddr))
+        ipaddr = peer;
+
+    port = ntohs(sock_union.sin.sin_port);
+
+    ret = mnt3_check_client_net_check(svc, volname, ipaddr, port);
 err:
-        return ret;
+    return ret;
 }
 
 static int
-mnt3_check_client_net_udp (struct svc_req *req, char *volname, xlator_t *nfsx)
+mnt3_check_client_net_udp(struct svc_req *req, char *volname, xlator_t *nfsx)
 {
-        rpcsvc_t                *svc = NULL;
-        struct sockaddr_in      *sin = NULL;
-        char                    ipaddr[INET_ADDRSTRLEN + 1] = {0, };
-        uint16_t                port = 0;
-        int                     ret = RPCSVC_AUTH_REJECT;
-        struct nfs_state        *nfs = NULL;
-
-        if ((!req) || (!volname) || (!nfsx))
-                goto err;
+    rpcsvc_t *svc = NULL;
+    struct sockaddr_in *sin = NULL;
+    char ipaddr[INET_ADDRSTRLEN + 1] = {
+        0,
+    };
+    uint16_t port = 0;
+    int ret = RPCSVC_AUTH_REJECT;
+    struct nfs_state *nfs = NULL;
+
+    if ((!req) || (!volname) || (!nfsx))
+        goto err;
+
+#if !defined(_TIRPC_SVC_H)
+    sin = svc_getcaller(req->rq_xprt);
+#else
+    sin = (struct sockaddr_in *)svc_getcaller(req->rq_xprt);
+    /* TIRPC's svc_getcaller() returns a pointer to a sockaddr_in6, even
+     * though it might actually be an IPv4 address. It ought return a
+     * struct sockaddr and make the caller upcast it to the proper
+     * address family. Sigh.
+     */
+#endif
+    if (!sin)
+        goto err;
+    /* And let's make sure that it's actually an IPv4 address. */
 
-        sin = svc_getcaller (req->rq_xprt);
-        if (!sin)
-                goto err;
+    GF_ASSERT(sin->sin_family == AF_INET);
 
-        (void) inet_ntop (AF_INET, &sin->sin_addr, ipaddr, INET_ADDRSTRLEN);
+    (void)inet_ntop(AF_INET, &sin->sin_addr, ipaddr, INET_ADDRSTRLEN);
 
-        port = ntohs (sin->sin_port);
+    port = ntohs(sin->sin_port);
 
-        nfs = (struct nfs_state *)nfsx->private;
-        if (nfs != NULL)
-                svc = nfs->rpcsvc;
+    nfs = (struct nfs_state *)nfsx->private;
+    if (nfs != NULL)
+        svc = nfs->rpcsvc;
 
-        ret = mnt3_check_client_net_check (svc, volname, ipaddr, port);
+    ret = mnt3_check_client_net_check(svc, volname, ipaddr, port);
 err:
-        return ret;
+    return ret;
 }
 
-
 int
-mnt3_parse_dir_exports (rpcsvc_request_t *req, struct mount3_state *ms,
-                        char *subdir)
-{
-        char                    volname[1024] = {0, };
-        struct mnt3_export      *exp = NULL;
-        char                    *volname_ptr = NULL;
-        int                     ret = -ENOENT;
-        struct nfs_state        *nfs = NULL;
-
-        if ((!ms) || (!subdir))
-                return -1;
-
-        volname_ptr = volname;
-        subdir = __volume_subdir (subdir, &volname_ptr);
-        if (!subdir)
-                goto err;
-
-        exp = mnt3_mntpath_to_export (ms, volname, _gf_false);
-        if (!exp)
-                goto err;
-
-        nfs = (struct nfs_state *)ms->nfsx->private;
-        if (!nfs)
-                goto err;
-
-        if (!nfs_subvolume_started (nfs, exp->vol)) {
-                gf_log (GF_MNT, GF_LOG_DEBUG,
-                        "Volume %s not started", exp->vol->name);
-                goto err;
-        }
-
-        ret = mnt3_check_client_net_tcp (req, exp->vol->name);
-        if (ret == RPCSVC_AUTH_REJECT) {
-                gf_log (GF_MNT, GF_LOG_DEBUG, "Client mount not allowed");
-                ret = -EACCES;
-                goto err;
-        }
+mnt3_parse_dir_exports(rpcsvc_request_t *req, struct mount3_state *ms,
+                       char *path, gf_boolean_t send_reply)
+{
+    char volname[1024] = {
+        0,
+    };
+    struct mnt3_export *exp = NULL;
+    char *volname_ptr = NULL;
+    char *subdir = NULL;
+    int ret = -ENOENT;
+    struct nfs_state *nfs = NULL;
+
+    if ((!ms) || (!path))
+        return -1;
 
-        ret = mnt3_resolve_subdir (req, ms, exp, subdir);
-        if (ret < 0) {
-                gf_log (GF_MNT, GF_LOG_ERROR,
-                        "Failed to resolve export dir: %s", subdir);
-                goto err;
-        }
+    volname_ptr = volname;
+    subdir = mnt3_get_volume_subdir(path, &volname_ptr);
+
+    /* first try to match the full export/subdir */
+    exp = mnt3_mntpath_to_export(ms, path, _gf_false);
+    if (!exp) {
+        gf_msg_trace(GF_MNT, 0,
+                     "Could not find exact matching export "
+                     "for path=%s",
+                     path);
+        /* if no exact match is found, look for a fallback */
+        exp = mnt3_mntpath_to_export(ms, volname, _gf_true);
+        if (!exp) {
+            gf_msg_trace(GF_MNT, 0,
+                         "Could not find export for "
+                         "volume %s",
+                         volname);
+            goto err;
+        }
+    }
+    gf_msg_trace(GF_MNT, 0,
+                 "volume %s and export %s will be used for "
+                 "path %s",
+                 exp->vol->name, exp->expname, path);
+
+    nfs = (struct nfs_state *)ms->nfsx->private;
+    if (!nfs)
+        goto err;
+
+    if (!nfs_subvolume_started(nfs, exp->vol)) {
+        gf_msg_debug(GF_MNT, 0, "Volume %s not started", exp->vol->name);
+        goto err;
+    }
+
+    ret = mnt3_check_client_net_tcp(req, exp->vol->name);
+    if (ret == RPCSVC_AUTH_REJECT) {
+        gf_msg_debug(GF_MNT, 0, "Client mount not allowed");
+        ret = -EACCES;
+        goto err;
+    }
+
+    ret = mnt3_resolve_subdir(req, ms, exp, subdir, send_reply);
+    if (ret < 0) {
+        gf_msg(GF_MNT, GF_LOG_ERROR, ret, NFS_MSG_RESOLVE_SUBDIR_FAIL,
+               "Failed to resolve export dir: %s", subdir);
+        goto err;
+    }
 
 err:
-        return ret;
+    return ret;
 }
 
-
 int
-mnt3_find_export (rpcsvc_request_t *req, char *path, struct mnt3_export **e)
+mnt3_find_export(rpcsvc_request_t *req, char *path, struct mnt3_export **e)
 {
-        int                     ret = -EFAULT;
-        struct mount3_state     *ms = NULL;
-        struct mnt3_export      *exp = NULL;
+    int ret = -EFAULT;
+    struct mount3_state *ms = NULL;
+    struct mnt3_export *exp = NULL;
 
-        if ((!req) || (!path) || (!e))
-                return -1;
-
-        ms = (struct mount3_state *) rpcsvc_request_program_private (req);
-        if (!ms) {
-                gf_log (GF_MNT, GF_LOG_ERROR, "Mount state not present");
-                rpcsvc_request_seterr (req, SYSTEM_ERR);
-                goto err;
-        }
+    if ((!req) || (!path) || (!e))
+        return -1;
 
-        gf_log (GF_MNT, GF_LOG_DEBUG, "dirpath: %s", path);
-        exp = mnt3_mntpath_to_export (ms, path, _gf_false);
-        if (exp) {
-                ret = 0;
-                *e = exp;
-                goto err;
-        }
+    ms = (struct mount3_state *)rpcsvc_request_program_private(req);
+    if (!ms) {
+        gf_msg(GF_MNT, GF_LOG_ERROR, EINVAL, NFS_MSG_MNT_STATE_NOT_FOUND,
+               "Mount state not present");
+        rpcsvc_request_seterr(req, SYSTEM_ERR);
+        goto err;
+    }
+
+    gf_msg_debug(GF_MNT, 0, "dirpath: %s", path);
+    exp = mnt3_mntpath_to_export(ms, path, _gf_false);
+    if (exp) {
+        ret = 0;
+        *e = exp;
+        goto err;
+    }
 
-        if (!gf_mnt3_export_dirs(ms)) {
-                ret = -1;
-                goto err;
-        }
+    if (!gf_mnt3_export_dirs(ms)) {
+        ret = -1;
+        goto err;
+    }
 
-        ret = mnt3_parse_dir_exports (req, ms, path);
+    ret = mnt3_parse_dir_exports(req, ms, path, _gf_true);
 
 err:
-        return ret;
+    return ret;
 }
 
 /**
@@ -1787,25 +1891,29 @@ err:
  *          failure: NULL
  */
 char *
-_mnt3_get_peer_addr (const rpcsvc_request_t *req)
+_mnt3_get_peer_addr(const rpcsvc_request_t *req)
 {
-        rpc_transport_t         *trans = NULL;
-        struct sockaddr_storage sastorage = {0, };
-        char                    peer[RPCSVC_PEER_STRLEN] = {0, };
-        char                    *peerdup = NULL;
-        int                     ret = 0;
-
-        GF_VALIDATE_OR_GOTO (GF_NFS, req, out);
-
-        trans = rpcsvc_request_transport (req);
-        ret = rpcsvc_transport_peeraddr (trans, peer, RPCSVC_PEER_STRLEN,
-                                         &sastorage, sizeof (sastorage));
-        if (ret != 0)
-                goto out;
-
-        peerdup = gf_strdup (peer);
+    rpc_transport_t *trans = NULL;
+    struct sockaddr_storage sastorage = {
+        0,
+    };
+    char peer[RPCSVC_PEER_STRLEN] = {
+        0,
+    };
+    char *peerdup = NULL;
+    int ret = 0;
+
+    GF_VALIDATE_OR_GOTO(GF_NFS, req, out);
+
+    trans = rpcsvc_request_transport(req);
+    ret = rpcsvc_transport_peeraddr(trans, peer, RPCSVC_PEER_STRLEN, &sastorage,
+                                    sizeof(sastorage));
+    if (ret != 0)
+        goto out;
+
+    peerdup = gf_strdup(peer);
 out:
-        return peerdup;
+    return peerdup;
 }
 
 /**
@@ -1820,25 +1928,27 @@ out:
  *
  */
 char *
-_mnt3_get_host_from_peer (const char *peer_addr)
+_mnt3_get_host_from_peer(const char *peer_addr)
 {
-        char   *part       = NULL;
-        size_t host_len    = 0;
-        char   *colon      = NULL;
-
-        colon = strchr (peer_addr, ':');
-        if (!colon) {
-                gf_log (GF_MNT, GF_LOG_ERROR, "Bad peer %s", peer_addr);
-                goto out;
-        }
-
-        host_len = colon - peer_addr;
-        if (host_len < RPCSVC_PEER_STRLEN)
-                part = gf_strndup (peer_addr, host_len);
-        else
-                gf_log (GF_MNT, GF_LOG_ERROR, "Peer too long %s", peer_addr);
+    char *part = NULL;
+    size_t host_len = 0;
+    char *colon = NULL;
+
+    colon = strrchr(peer_addr, ':');
+    if (!colon) {
+        gf_msg(GF_MNT, GF_LOG_ERROR, 0, NFS_MSG_BAD_PEER, "Bad peer %s",
+               peer_addr);
+        goto out;
+    }
+
+    host_len = colon - peer_addr;
+    if (host_len < RPCSVC_PEER_STRLEN)
+        part = gf_strndup(peer_addr, host_len);
+    else
+        gf_msg(GF_MNT, GF_LOG_ERROR, 0, NFS_MSG_PEER_TOO_LONG,
+               "Peer too long %s", peer_addr);
 out:
-        return part;
+    return part;
 }
 
 /**
@@ -1848,14 +1958,14 @@ out:
  * a write operation.
  *
  */
-inline int
-mnt3_check_cached_fh (struct mount3_state *ms, struct nfs3_fh *fh,
-                      const char *host_addr, gf_boolean_t is_write_op)
+int
+mnt3_check_cached_fh(struct mount3_state *ms, struct nfs3_fh *fh,
+                     const char *host_addr, gf_boolean_t is_write_op)
 {
-        if (!is_write_op)
-                return is_nfs_fh_cached (ms->authcache, fh, host_addr);
+    if (!is_write_op)
+        return is_nfs_fh_cached(ms->authcache, fh, host_addr);
 
-        return is_nfs_fh_cached_and_writeable (ms->authcache, fh, host_addr);
+    return is_nfs_fh_cached_and_writeable(ms->authcache, fh, host_addr);
 }
 
 /**
@@ -1877,94 +1987,103 @@ mnt3_check_cached_fh (struct mount3_state *ms, struct nfs3_fh *fh,
  *          -EROFS  for unauthorized write operations (rm, mkdir, write)
  */
 int
-_mnt3_authenticate_req (struct mount3_state *ms, rpcsvc_request_t *req,
-                        struct nfs3_fh *fh, const char *path,
-                        char **authorized_export, char **authorized_host,
-                        gf_boolean_t is_write_op)
-{
-        char                    *peer_addr       = NULL;
-        char                    *host_addr_ip    = NULL;
-        char                    *host_addr_fqdn  = NULL;
-        int                     auth_status_code = -EACCES;
-        char                    *pathdup         = NULL;
-        size_t                  dlen             = 0;
-        char                    *auth_host       = NULL;
-        gf_boolean_t            fh_cached        = _gf_false;
-        struct export_item      *expitem         = NULL;
-
-        GF_VALIDATE_OR_GOTO (GF_MNT, ms, out);
-        GF_VALIDATE_OR_GOTO (GF_MNT, req, out);
-
-        peer_addr    = _mnt3_get_peer_addr (req);
-        host_addr_ip = _mnt3_get_host_from_peer (peer_addr);
-
-        if (!host_addr_ip || !peer_addr)
-                goto free_and_out;
-
-        if (path) {
-                /* Need to strip out trailing '/' */
-                pathdup = strdupa (path);
-                dlen = strlen (pathdup);
-                if (dlen > 0 && pathdup[dlen-1] == '/')
-                        pathdup[dlen-1] = '\0';
-        }
-
-        /* Check if the filehandle is cached */
-        fh_cached = mnt3_check_cached_fh (ms, fh, host_addr_ip, is_write_op);
-        if (fh_cached) {
-                gf_log (GF_MNT, GF_LOG_TRACE, "Found cached FH for %s",
-                        host_addr_ip);
-                auth_status_code = 0;
-                goto free_and_out;
-        }
-
-        /* Check if the IP is authorized */
-        auth_status_code = mnt3_auth_host (ms->auth_params, host_addr_ip,
-                                           fh, pathdup, is_write_op, &expitem);
-        if (auth_status_code != 0) {
-                /* If not, check if the FQDN is authorized */
-                host_addr_fqdn = gf_rev_dns_lookup (host_addr_ip);
-                auth_status_code = mnt3_auth_host (ms->auth_params,
-                                                   host_addr_fqdn,
-                                                   fh, pathdup, is_write_op,
-                                                   &expitem);
-                if (auth_status_code == 0)
-                        auth_host = host_addr_fqdn;
-        } else
-                auth_host = host_addr_ip;
-
-        /* Skip the lines that set authorized export &
-         * host if they are null.
-         */
-        if (!authorized_export || !authorized_host) {
-                /* Cache the file handle if it was authorized */
-                if (fh && auth_status_code == 0)
-                        cache_nfs_fh (ms->authcache, fh, host_addr_ip, expitem);
-
-                goto free_and_out;
-        }
-
-        if (!fh && auth_status_code == 0) {
-                *authorized_export = gf_strdup (pathdup);
-                if (!*authorized_export)
-                        gf_log (GF_MNT, GF_LOG_CRITICAL,
-                                "Allocation error when copying "
-                                "authorized path");
-
-                *authorized_host = gf_strdup (auth_host);
-                if (!*authorized_host)
-                        gf_log (GF_MNT, GF_LOG_CRITICAL,
-                                "Allocation error when copying "
-                                "authorized host");
-        }
+_mnt3_authenticate_req(struct mount3_state *ms, rpcsvc_request_t *req,
+                       struct nfs3_fh *fh, const char *path,
+                       char **authorized_export, char **authorized_host,
+                       gf_boolean_t is_write_op)
+{
+    char *peer_addr = NULL;
+    char *host_addr_ip = NULL;
+    char *host_addr_fqdn = NULL;
+    int auth_status_code = -EACCES;
+    char *pathdup = NULL;
+    size_t dlen = 0;
+    char *auth_host = NULL;
+    gf_boolean_t fh_cached = _gf_false;
+    struct export_item *expitem = NULL;
+
+    GF_VALIDATE_OR_GOTO(GF_MNT, ms, out);
+    GF_VALIDATE_OR_GOTO(GF_MNT, req, out);
+
+    peer_addr = _mnt3_get_peer_addr(req);
+
+    if (!peer_addr)
+        goto free_and_out;
+
+    host_addr_ip = _mnt3_get_host_from_peer(peer_addr);
+
+    if (!host_addr_ip)
+        goto free_and_out;
+
+    if (path) {
+        /* Need to strip out trailing '/' */
+        pathdup = strdupa(path);
+        dlen = strlen(pathdup);
+        if (dlen > 0 && pathdup[dlen - 1] == '/')
+            pathdup[dlen - 1] = '\0';
+    }
+
+    /* Check if the filehandle is cached */
+    fh_cached = mnt3_check_cached_fh(ms, fh, host_addr_ip, is_write_op);
+    if (fh_cached) {
+        gf_msg_trace(GF_MNT, 0, "Found cached FH for %s", host_addr_ip);
+        auth_status_code = 0;
+        goto free_and_out;
+    }
+
+    /* Check if the IP is authorized */
+    auth_status_code = mnt3_auth_host(ms->auth_params, host_addr_ip, fh,
+                                      pathdup, is_write_op, &expitem);
+
+    gf_msg_debug(GF_MNT, 0, "access from IP %s is %s", host_addr_ip,
+                 auth_status_code ? "denied" : "allowed");
+
+    if (auth_status_code != 0) {
+        /* If not, check if the FQDN is authorized */
+        host_addr_fqdn = gf_rev_dns_lookup(host_addr_ip);
+        auth_status_code = mnt3_auth_host(ms->auth_params, host_addr_fqdn, fh,
+                                          pathdup, is_write_op, &expitem);
+
+        gf_msg_debug(GF_MNT, 0, "access from FQDN %s is %s", host_addr_fqdn,
+                     auth_status_code ? "denied" : "allowed");
+
+        if (auth_status_code == 0)
+            auth_host = host_addr_fqdn;
+    } else
+        auth_host = host_addr_ip;
+
+    /* Skip the lines that set authorized export &
+     * host if they are null.
+     */
+    if (!authorized_export || !authorized_host) {
+        /* Cache the file handle if it was authorized */
+        if (fh && auth_status_code == 0)
+            cache_nfs_fh(ms->authcache, fh, host_addr_ip, expitem);
+
+        goto free_and_out;
+    }
+
+    if (!fh && auth_status_code == 0) {
+        *authorized_export = gf_strdup(pathdup);
+        if (!*authorized_export)
+            gf_msg(GF_MNT, GF_LOG_CRITICAL, ENOMEM, NFS_MSG_NO_MEMORY,
+                   "Allocation error when copying "
+                   "authorized path");
+
+        *authorized_host = gf_strdup(auth_host);
+        if (!*authorized_host)
+            gf_msg(GF_MNT, GF_LOG_CRITICAL, ENOMEM, NFS_MSG_NO_MEMORY,
+                   "Allocation error when copying "
+                   "authorized host");
+    }
 
 free_and_out:
-        /* Free allocated strings after doing the auth */
-        GF_FREE (peer_addr);
-        GF_FREE (host_addr_fqdn);
-        GF_FREE (host_addr_ip);
+    /* Free allocated strings after doing the auth */
+    GF_FREE(peer_addr);
+    GF_FREE(host_addr_fqdn);
+    GF_FREE(host_addr_ip);
 out:
-        return auth_status_code;
+    return auth_status_code;
 }
 
 /**
@@ -1992,751 +2111,814 @@ out:
  *          -EROFS  for unauthorized write operations (rm, mkdir, write)
  */
 int
-mnt3_authenticate_request (struct mount3_state *ms, rpcsvc_request_t *req,
-                           struct nfs3_fh *fh, const char *volname,
-                           const char *path, char **authorized_path,
-                           char **authorized_host, gf_boolean_t is_write_op)
-{
-        int          auth_status_code       = -EACCES;
-        char         *parent_path = NULL;
-        const char   *parent_old  = NULL;
-
-        GF_VALIDATE_OR_GOTO (GF_MNT, ms, out);
-        GF_VALIDATE_OR_GOTO (GF_MNT, req, out);
-
-        /* If this option is not set, just allow it through */
-        if (!ms->nfs->exports_auth) {
-                /* This function is called in a variety of use-cases (mount
-                 * + each fop) so path/authorized_path are not always present.
-                 * For the cases which it _is_ present we need to populate the
-                 * authorized_path. */
-                if (path && authorized_path)
-                        *authorized_path = gf_strdup (path);
-
-                auth_status_code = 0;
-                goto out;
-        }
-
-        /* First check if the path is allowed */
-        auth_status_code = _mnt3_authenticate_req (ms, req, fh, path,
-                                                   authorized_path,
-                                                   authorized_host,
-                                                   is_write_op);
-
-        /* If the filehandle is set, just exit since we have to make only
-         * one call to the function above
-         */
-        if (fh)
-                goto out;
-
-        parent_old = path;
-        while (auth_status_code != 0) {
-                /* Get the path's parent */
-                parent_path = gf_resolve_path_parent (parent_old);
-                if (!parent_path) /* Nothing left in the path to resolve */
-                        goto out;
-
-                /* Authenticate it */
-                auth_status_code = _mnt3_authenticate_req (ms, req, fh,
-                                                           parent_path,
-                                                           authorized_path,
-                                                           authorized_host,
-                                                           is_write_op);
-
-                parent_old = strdupa (parent_path); /* Copy the parent onto the
-                                                     * stack.
-                                                     */
-
-                GF_FREE (parent_path); /* Free the allocated parent string */
-        }
+mnt3_authenticate_request(struct mount3_state *ms, rpcsvc_request_t *req,
+                          struct nfs3_fh *fh, const char *volname,
+                          const char *path, char **authorized_path,
+                          char **authorized_host, gf_boolean_t is_write_op)
+{
+    int auth_status_code = -EACCES;
+    char *parent_path = NULL;
+    const char *parent_old = NULL;
+
+    GF_VALIDATE_OR_GOTO(GF_MNT, ms, out);
+    GF_VALIDATE_OR_GOTO(GF_MNT, req, out);
+
+    /* If this option is not set, just allow it through */
+    if (!ms->nfs->exports_auth) {
+        /* This function is called in a variety of use-cases (mount
+         * + each fop) so path/authorized_path are not always present.
+         * For the cases which it _is_ present we need to populate the
+         * authorized_path. */
+        if (path && authorized_path)
+            *authorized_path = gf_strdup(path);
+
+        auth_status_code = 0;
+        goto out;
+    }
+
+    /* First check if the path is allowed */
+    auth_status_code = _mnt3_authenticate_req(
+        ms, req, fh, path, authorized_path, authorized_host, is_write_op);
+
+    /* If the filehandle is set, just exit since we have to make only
+     * one call to the function above
+     */
+    if (fh)
+        goto out;
+
+    parent_old = path;
+    while (auth_status_code != 0) {
+        /* Get the path's parent */
+        parent_path = gf_resolve_path_parent(parent_old);
+        if (!parent_path) /* Nothing left in the path to resolve */
+            goto out;
+
+        /* Authenticate it */
+        auth_status_code = _mnt3_authenticate_req(ms, req, fh, parent_path,
+                                                  authorized_path,
+                                                  authorized_host, is_write_op);
+
+        parent_old = strdupa(parent_path); /* Copy the parent onto the
+                                            * stack.
+                                            */
+
+        GF_FREE(parent_path); /* Free the allocated parent string */
+    }
 
 out:
-        return auth_status_code;
+    return auth_status_code;
 }
 
 int
-mnt3svc_mnt (rpcsvc_request_t *req)
-{
-        struct iovec            pvec = {0, };
-        char                    path[MNTPATHLEN];
-        int                     ret = -1;
-        struct mount3_state     *ms = NULL;
-        mountstat3              mntstat = MNT3ERR_SERVERFAULT;
-        struct mnt3_export      *exp = NULL;
-        struct nfs_state        *nfs = NULL;
-        int                     authcode = 0;
-
-        if (!req)
-                return -1;
-
-        pvec.iov_base = path;
-        pvec.iov_len = MNTPATHLEN;
-        ret = xdr_to_mountpath (pvec, req->msg[0]);
-        if (ret == -1) {
-                gf_log (GF_MNT, GF_LOG_ERROR, "Failed to decode args");
-                rpcsvc_request_seterr (req, GARBAGE_ARGS);
-                goto rpcerr;
-        }
-
-        ms = (struct mount3_state *)rpcsvc_request_program_private (req);
-        if (!ms) {
-                gf_log (GF_MNT, GF_LOG_ERROR, "Mount state not present");
-                rpcsvc_request_seterr (req, SYSTEM_ERR);
-                ret = -1;
-                goto rpcerr;
-        }
-
-        nfs = (struct nfs_state *)ms->nfsx->private;
-        gf_log (GF_MNT, GF_LOG_DEBUG, "dirpath: %s", path);
-        ret = mnt3_find_export (req, path, &exp);
-        if (ret < 0) {
-                mntstat = mnt3svc_errno_to_mnterr (-ret);
-                goto mnterr;
-        } else if (!exp) {
-                /*
-                 * SPECIAL CASE: exp is NULL if "path" is subdir in
-                 * call to mnt3_find_export().
-                 *
-                 * This is subdir mount, we are already DONE!
-                 * nfs_subvolume_started() and mnt3_check_client_net_tcp()
-                 * validation are done in mnt3_parse_dir_exports()
-                 * which is invoked through mnt3_find_export().
-                 *
-                 * TODO: All mount should happen thorugh mnt3svc_mount()
-                 *       It needs more clean up.
-                 */
-                return (0);
-        }
-
-        if (!nfs_subvolume_started (nfs, exp->vol)) {
-                gf_log (GF_MNT, GF_LOG_DEBUG, "Volume %s not started",
-                        exp->vol->name);
-                ret = -1;
-                mntstat = MNT3ERR_NOENT;
-                goto mnterr;
-        }
-
-        ret = mnt3_check_client_net_tcp (req, exp->vol->name);
-        if (ret == RPCSVC_AUTH_REJECT) {
-                mntstat = MNT3ERR_ACCES;
-                gf_log (GF_MNT, GF_LOG_DEBUG, "Client mount not allowed");
-                ret = -1;
-                goto mnterr;
-        }
+mnt3svc_mnt(rpcsvc_request_t *req)
+{
+    struct iovec pvec = {
+        0,
+    };
+    char path[MNTPATHLEN];
+    int ret = -1;
+    struct mount3_state *ms = NULL;
+    mountstat3 mntstat = MNT3ERR_SERVERFAULT;
+    struct mnt3_export *exp = NULL;
+    struct nfs_state *nfs = NULL;
+    int authcode = 0;
+
+    if (!req)
+        return -1;
 
-        /* The second authentication check is the exports/netgroups
-         * check.
+    pvec.iov_base = path;
+    pvec.iov_len = MNTPATHLEN;
+    ret = xdr_to_mountpath(pvec, req->msg[0]);
+    if (ret == -1) {
+        gf_msg(GF_MNT, GF_LOG_ERROR, 0, NFS_MSG_ARGS_DECODE_ERROR,
+               "Failed to decode args");
+        rpcsvc_request_seterr(req, GARBAGE_ARGS);
+        goto rpcerr;
+    }
+
+    ms = (struct mount3_state *)rpcsvc_request_program_private(req);
+    if (!ms) {
+        gf_msg(GF_MNT, GF_LOG_ERROR, EINVAL, NFS_MSG_MNT_STATE_NOT_FOUND,
+               "Mount state not present");
+        rpcsvc_request_seterr(req, SYSTEM_ERR);
+        ret = -1;
+        goto rpcerr;
+    }
+
+    nfs = (struct nfs_state *)ms->nfsx->private;
+    gf_msg_debug(GF_MNT, 0, "dirpath: %s", path);
+    ret = mnt3_find_export(req, path, &exp);
+    if (ret < 0) {
+        mntstat = mnt3svc_errno_to_mnterr(-ret);
+        goto mnterr;
+    } else if (!exp) {
+        /*
+         * SPECIAL CASE: exp is NULL if "path" is subdir in
+         * call to mnt3_find_export().
+         *
+         * This is subdir mount, we are already DONE!
+         * nfs_subvolume_started() and mnt3_check_client_net_tcp()
+         * validation are done in mnt3_parse_dir_exports()
+         * which is invoked through mnt3_find_export().
+         *
+         * TODO: All mount should happen thorugh mnt3svc_mount()
+         *       It needs more clean up.
          */
-        authcode = mnt3_authenticate_request (ms, req, NULL, NULL, path, NULL,
-                                              NULL, _gf_false);
-        if (authcode != 0) {
-                mntstat = MNT3ERR_ACCES;
-                gf_log (GF_MNT, GF_LOG_DEBUG, "Client mount not allowed");
-                ret = -1;
-                goto mnterr;
-        }
-
-        ret = mnt3svc_mount (req, ms, exp);
-
-        if (ret < 0)
-                mntstat = mnt3svc_errno_to_mnterr (-ret);
+        return (0);
+    }
+
+    if (!nfs_subvolume_started(nfs, exp->vol)) {
+        gf_msg_debug(GF_MNT, 0, "Volume %s not started", exp->vol->name);
+        ret = -1;
+        mntstat = MNT3ERR_NOENT;
+        goto mnterr;
+    }
+
+    ret = mnt3_check_client_net_tcp(req, exp->vol->name);
+    if (ret == RPCSVC_AUTH_REJECT) {
+        mntstat = MNT3ERR_ACCES;
+        gf_msg_debug(GF_MNT, 0, "Client mount not allowed");
+        ret = -1;
+        goto mnterr;
+    }
+
+    /* The second authentication check is the exports/netgroups
+     * check.
+     */
+    authcode = mnt3_authenticate_request(ms, req, NULL, NULL, path, NULL, NULL,
+                                         _gf_false);
+    if (authcode != 0) {
+        mntstat = MNT3ERR_ACCES;
+        gf_msg_debug(GF_MNT, 0, "Client mount not allowed");
+        ret = -1;
+        goto mnterr;
+    }
+
+    ret = mnt3svc_mount(req, ms, exp);
+
+    if (ret < 0)
+        mntstat = mnt3svc_errno_to_mnterr(-ret);
 mnterr:
-        if (ret < 0) {
-                mnt3svc_mnt_error_reply (req, mntstat);
-                ret = 0;
-        }
+    if (ret < 0) {
+        mnt3svc_mnt_error_reply(req, mntstat);
+        ret = 0;
+    }
 
 rpcerr:
-        return ret;
+    return ret;
 }
 
-
 int
-mnt3svc_null (rpcsvc_request_t *req)
+mnt3svc_null(rpcsvc_request_t *req)
 {
-        struct iovec    dummyvec = {0, };
+    struct iovec dummyvec = {
+        0,
+    };
 
-        if (!req) {
-                gf_log (GF_MNT, GF_LOG_ERROR, "Got NULL request!");
-                return 0;
-        }
-        rpcsvc_submit_generic (req, &dummyvec, 1,  NULL, 0, NULL);
+    if (!req) {
+        gf_msg(GF_MNT, GF_LOG_ERROR, EINVAL, NFS_MSG_INVALID_ENTRY,
+               "Got NULL request!");
         return 0;
+    }
+    rpcsvc_submit_generic(req, &dummyvec, 1, NULL, 0, NULL);
+    return 0;
 }
 
-
 mountlist
-__build_mountlist (struct mount3_state *ms, int *count)
-{
-        struct mountbody        *mlist = NULL;
-        struct mountbody        *prev = NULL;
-        struct mountbody        *first = NULL;
-        size_t                  namelen = 0;
-        int                     ret = -1;
-        struct mountentry       *me = NULL;
-
-        if ((!ms) || (!count))
-                return NULL;
-
-        /* read rmtab, other peers might have updated it */
-        mount_read_rmtab(ms);
-
-        *count = 0;
-        gf_log (GF_MNT, GF_LOG_DEBUG, "Building mount list:");
-        list_for_each_entry (me, &ms->mountlist, mlist) {
-                namelen = strlen (me->exname);
-                mlist = GF_CALLOC (1, sizeof (*mlist), gf_nfs_mt_mountbody);
-                if (!mlist) {
-                        gf_log (GF_MNT, GF_LOG_ERROR, "Memory allocation"
-                                " failed");
-                        goto free_list;
-                }
-                if (!first)
-                        first = mlist;
-
-                mlist->ml_directory = GF_CALLOC (namelen + 2, sizeof (char),
-                                                 gf_nfs_mt_char);
-                if (!mlist->ml_directory) {
-                        gf_log (GF_MNT, GF_LOG_ERROR, "Memory allocation"
-                                " failed");
-                        goto free_list;
-                }
+__build_mountlist(struct mount3_state *ms, int *count)
+{
+    struct mountbody *mlist = NULL;
+    struct mountbody *prev = NULL;
+    struct mountbody *first = NULL;
+    size_t namelen = 0;
+    int ret = -1;
+    struct mountentry *me = NULL;
+
+    if ((!ms) || (!count))
+        return NULL;
 
-                strcpy (mlist->ml_directory, me->exname);
+    /* read rmtab, other peers might have updated it */
+    mount_read_rmtab(ms);
 
-                namelen = strlen (me->hostname);
-                mlist->ml_hostname = GF_CALLOC (namelen + 2, sizeof (char),
-                                                gf_nfs_mt_char);
-                if (!mlist->ml_hostname) {
-                        gf_log (GF_MNT, GF_LOG_ERROR, "Memory allocation"
-                                " failed");
-                        goto free_list;
-                }
+    *count = 0;
+    gf_msg_debug(GF_MNT, 0, "Building mount list:");
+    list_for_each_entry(me, &ms->mountlist, mlist)
+    {
+        namelen = strlen(me->exname);
+        mlist = GF_CALLOC(1, sizeof(*mlist), gf_nfs_mt_mountbody);
+        if (!mlist) {
+            gf_msg(GF_MNT, GF_LOG_ERROR, ENOMEM, NFS_MSG_NO_MEMORY,
+                   "Memory allocation failed");
+            goto free_list;
+        }
+        if (!first)
+            first = mlist;
 
-                strcat (mlist->ml_hostname, me->hostname);
+        mlist->ml_directory = GF_MALLOC(namelen + 2, gf_nfs_mt_char);
+        if (!mlist->ml_directory) {
+            gf_msg(GF_MNT, GF_LOG_ERROR, ENOMEM, NFS_MSG_NO_MEMORY,
+                   "Memory allocation failed");
+            goto free_list;
+        }
 
-                gf_log (GF_MNT, GF_LOG_DEBUG, "mount entry: dir: %s, host: %s",
-                        mlist->ml_directory, mlist->ml_hostname);
-                if (prev) {
-                        prev->ml_next = mlist;
-                        prev = mlist;
-                } else
-                        prev = mlist;
+        strcpy(mlist->ml_directory, me->exname);
 
-                (*count)++;
+        namelen = strlen(me->hostname);
+        mlist->ml_hostname = GF_MALLOC(namelen + 2, gf_nfs_mt_char);
+        if (!mlist->ml_hostname) {
+            gf_msg(GF_MNT, GF_LOG_ERROR, ENOMEM, NFS_MSG_NO_MEMORY,
+                   "Memory allocation failed");
+            goto free_list;
         }
 
-        ret = 0;
+        strcpy(mlist->ml_hostname, me->hostname);
+
+        gf_msg_debug(GF_MNT, 0, "mount entry: dir: %s, host: %s",
+                     mlist->ml_directory, mlist->ml_hostname);
+        if (prev) {
+            prev->ml_next = mlist;
+            prev = mlist;
+        } else
+            prev = mlist;
+
+        (*count)++;
+    }
+
+    ret = 0;
 
 free_list:
-        if (ret == -1) {
-                xdr_free_mountlist (first);
-                first = NULL;
-        }
+    if (ret == -1) {
+        xdr_free_mountlist(first);
+        first = NULL;
+    }
 
-        return first;
+    return first;
 }
 
-
 mountlist
-mnt3svc_build_mountlist (struct mount3_state *ms, int *count)
+mnt3svc_build_mountlist(struct mount3_state *ms, int *count)
 {
-        struct mountbody        *first = NULL;
+    struct mountbody *first = NULL;
 
-        LOCK (&ms->mountlock);
-        {
-                first = __build_mountlist (ms, count);
-        }
-        UNLOCK (&ms->mountlock);
+    LOCK(&ms->mountlock);
+    {
+        first = __build_mountlist(ms, count);
+    }
+    UNLOCK(&ms->mountlock);
 
-        return first;
+    return first;
 }
 
-
 int
-mnt3svc_dump (rpcsvc_request_t *req)
+mnt3svc_dump(rpcsvc_request_t *req)
 {
-        int                     ret = -1;
-        struct mount3_state     *ms = NULL;
-        mountlist               mlist;
-        mountstat3              mstat = 0;
-        mnt3_serializer         sfunc = NULL;
-        void                    *arg = NULL;
-
-
-        if (!req)
-                return -1;
+    int ret = -1;
+    struct mount3_state *ms = NULL;
+    mountlist mlist;
+    mountstat3 mstat = 0;
+    mnt3_serializer sfunc = NULL;
+    void *arg = NULL;
+
+    if (!req)
+        return -1;
 
-        ms = (struct mount3_state *)rpcsvc_request_program_private (req);
-        if (!ms) {
-                rpcsvc_request_seterr (req, SYSTEM_ERR);
-                goto rpcerr;
-        }
+    ms = (struct mount3_state *)rpcsvc_request_program_private(req);
+    if (!ms) {
+        rpcsvc_request_seterr(req, SYSTEM_ERR);
+        goto rpcerr;
+    }
 
-        sfunc = (mnt3_serializer)xdr_serialize_mountlist;
-        mlist = mnt3svc_build_mountlist (ms, &ret);
-        arg = &mlist;
+    sfunc = (mnt3_serializer)xdr_serialize_mountlist;
+    mlist = mnt3svc_build_mountlist(ms, &ret);
+    arg = &mlist;
 
-        if (!mlist) {
-                if (ret != 0) {
-                        rpcsvc_request_seterr (req, SYSTEM_ERR);
-                        ret = -1;
-                        goto rpcerr;
-                } else {
-                        arg = &mstat;
-                        sfunc = (mnt3_serializer)xdr_serialize_mountstat3;
-                }
+    if (!mlist) {
+        if (ret != 0) {
+            rpcsvc_request_seterr(req, SYSTEM_ERR);
+            ret = -1;
+            goto rpcerr;
+        } else {
+            arg = &mstat;
+            sfunc = (mnt3_serializer)xdr_serialize_mountstat3;
         }
+    }
 
-        mnt3svc_submit_reply (req, arg, sfunc);
+    mnt3svc_submit_reply(req, arg, sfunc);
 
-        xdr_free_mountlist (mlist);
-        ret = 0;
+    xdr_free_mountlist(mlist);
+    ret = 0;
 
 rpcerr:
-        return ret;
+    return ret;
 }
 
-
 int
-mnt3svc_umount (struct mount3_state *ms, char *dirpath, char *hostname)
+mnt3svc_umount(struct mount3_state *ms, char *dirpath, char *hostname)
 {
-        struct mountentry       *me = NULL;
-        int                     ret = -1;
-        gf_store_handle_t       *sh = NULL;
-        struct nfs_state        *nfs = NULL;
+    struct mountentry *me = NULL;
+    int ret = -1;
+    gf_store_handle_t *sh = NULL;
+    struct nfs_state *nfs = NULL;
+    gf_boolean_t update_rmtab = _gf_false;
 
-        if ((!ms) || (!dirpath) || (!hostname))
-                return -1;
+    if ((!ms) || (!dirpath) || (!hostname))
+        return -1;
 
-        nfs = (struct nfs_state *)ms->nfsx->private;
+    nfs = (struct nfs_state *)ms->nfsx->private;
 
-        ret = gf_store_handle_new (nfs->rmtab, &sh);
-        if (ret) {
-                gf_log (GF_MNT, GF_LOG_WARNING, "Failed to open '%s'",
-                        nfs->rmtab);
-                return 0;
-        }
+    update_rmtab = mount_open_rmtab(nfs->rmtab, &sh);
+    if (update_rmtab) {
+        ret = gf_store_lock(sh);
+        if (ret)
+            goto out_free;
+    }
 
-        ret = gf_store_lock (sh);
-        if (ret) {
-                goto out_free;
+    LOCK(&ms->mountlock);
+    {
+        if (update_rmtab)
+            __mount_read_rmtab(sh, &ms->mountlist, _gf_false);
+
+        if (list_empty(&ms->mountlist)) {
+            ret = 0;
+            goto out_unlock;
         }
 
-        LOCK (&ms->mountlock);
+        ret = -1;
+        list_for_each_entry(me, &ms->mountlist, mlist)
         {
-                __mount_read_rmtab (sh, &ms->mountlist, _gf_false);
-                if (list_empty (&ms->mountlist)) {
-                        ret = 0;
-                        goto out_unlock;
-                }
+            if ((strcmp(me->exname, dirpath) == 0) &&
+                (strcmp(me->hostname, hostname) == 0)) {
+                ret = 0;
+                break;
+            }
+        }
 
-                ret = -1;
-                list_for_each_entry (me, &ms->mountlist, mlist) {
-                       if ((strcmp (me->exname, dirpath) == 0) &&
-                            (strcmp (me->hostname, hostname) == 0)) {
-                               ret = 0;
-                               break;
-                       }
-                }
+        /* Need this check here because at the end of the search me
+         * might still be pointing to the last entry, which may not be
+         * the one we're looking for.
+         */
+        if (ret == -1) { /* Not found in list. */
+            gf_msg_trace(GF_MNT, 0, "Export not found");
+            goto out_unlock;
+        }
 
-                /* Need this check here because at the end of the search me
-                 * might still be pointing to the last entry, which may not be
-                 * the one we're looking for.
-                 */
-                if (ret == -1)  {/* Not found in list. */
-                        gf_log (GF_MNT, GF_LOG_TRACE, "Export not found");
-                        goto out_unlock;
-                }
+        if (!me)
+            goto out_unlock;
 
-                if (!me)
-                        goto out_unlock;
+        gf_msg_debug(GF_MNT, 0, "Unmounting: dir %s, host: %s", me->exname,
+                     me->hostname);
 
-                gf_log (GF_MNT, GF_LOG_DEBUG, "Unmounting: dir %s, host: %s",
-                        me->exname, me->hostname);
+        list_del(&me->mlist);
+        GF_FREE(me);
 
-                list_del (&me->mlist);
-                GF_FREE (me);
-                __mount_rewrite_rmtab (ms, sh);
-        }
+        if (update_rmtab)
+            __mount_rewrite_rmtab(ms, sh);
+    }
 out_unlock:
-        UNLOCK (&ms->mountlock);
-        gf_store_unlock (sh);
-out_free:
-        gf_store_handle_destroy (sh);
-        return ret;
-}
+    UNLOCK(&ms->mountlock);
 
+    if (update_rmtab)
+        gf_store_unlock(sh);
 
-int
-mnt3svc_umnt (rpcsvc_request_t *req)
-{
-        char                    hostname[MNTPATHLEN];
-        char                    dirpath[MNTPATHLEN];
-        struct iovec            pvec = {0, };
-        int                     ret = -1;
-        struct mount3_state     *ms = NULL;
-        mountstat3              mstat = MNT3_OK;
-        char                    *colon = NULL;
-
-        if (!req)
-                return -1;
-
-        /* Remove the mount point from the exports list. */
-        pvec.iov_base = dirpath;
-        pvec.iov_len = MNTPATHLEN;
-        ret = xdr_to_mountpath (pvec, req->msg[0]);
-        if (ret == -1) {
-                gf_log (GF_MNT, GF_LOG_ERROR, "Failed decode args");
-                rpcsvc_request_seterr (req, GARBAGE_ARGS);
-                goto rpcerr;
-        }
+out_free:
+    if (update_rmtab)
+        gf_store_handle_destroy(sh);
 
-        ms = (struct mount3_state *)rpcsvc_request_program_private (req);
-        if (!ms) {
-                gf_log (GF_MNT, GF_LOG_ERROR, "Mount state not present");
-                rpcsvc_request_seterr (req, SYSTEM_ERR);
-                ret = -1;
-                goto rpcerr;
-        }
+    return ret;
+}
 
-        ret = rpcsvc_transport_peername (req->trans, hostname, MNTPATHLEN);
-        if (ret != 0) {
-                gf_log (GF_MNT, GF_LOG_ERROR, "Failed to get remote name: %s",
-                        gai_strerror (ret));
-                goto rpcerr;
-        }
+int
+mnt3svc_umnt(rpcsvc_request_t *req)
+{
+    char hostname[MNTPATHLEN];
+    char dirpath[MNTPATHLEN];
+    struct iovec pvec = {
+        0,
+    };
+    int ret = -1;
+    struct mount3_state *ms = NULL;
+    mountstat3 mstat = MNT3_OK;
+    char *colon = NULL;
+
+    if (!req)
+        return -1;
 
-        colon = strrchr (hostname, ':');
-        if (colon) {
-                *colon= '\0';
-        }
-        gf_log (GF_MNT, GF_LOG_DEBUG, "dirpath: %s, hostname: %s", dirpath,
-                hostname);
-        ret = mnt3svc_umount (ms, dirpath, hostname);
+    /* Remove the mount point from the exports list. */
+    pvec.iov_base = dirpath;
+    pvec.iov_len = MNTPATHLEN;
+    ret = xdr_to_mountpath(pvec, req->msg[0]);
+    if (ret == -1) {
+        gf_msg(GF_MNT, GF_LOG_ERROR, 0, NFS_MSG_ARGS_DECODE_ERROR,
+               "Failed decode args");
+        rpcsvc_request_seterr(req, GARBAGE_ARGS);
+        goto rpcerr;
+    }
+
+    ms = (struct mount3_state *)rpcsvc_request_program_private(req);
+    if (!ms) {
+        gf_msg(GF_MNT, GF_LOG_ERROR, EINVAL, NFS_MSG_MNT_STATE_NOT_FOUND,
+               "Mount state not present");
+        rpcsvc_request_seterr(req, SYSTEM_ERR);
+        ret = -1;
+        goto rpcerr;
+    }
+
+    ret = rpcsvc_transport_peername(req->trans, hostname, MNTPATHLEN);
+    if (ret != 0) {
+        gf_msg(GF_MNT, GF_LOG_ERROR, ENOENT, NFS_MSG_GET_REMOTE_NAME_FAIL,
+               "Failed to get remote name: %s", gai_strerror(ret));
+        goto rpcerr;
+    }
+
+    colon = strrchr(hostname, ':');
+    if (colon) {
+        *colon = '\0';
+    }
+    gf_path_strip_trailing_slashes(dirpath);
+    gf_msg_debug(GF_MNT, 0, "dirpath: %s, hostname: %s", dirpath, hostname);
+    ret = mnt3svc_umount(ms, dirpath, hostname);
 
-        if (ret == -1) {
-                ret = 0;
-                mstat = MNT3ERR_NOENT;
-        }
-        /* FIXME: also take care of the corner case where the
-         * client was resolvable at mount but not at the umount - vice-versa.
-         */
-        mnt3svc_submit_reply (req, &mstat,
-                              (mnt3_serializer)xdr_serialize_mountstat3);
+    if (ret == -1) {
+        ret = 0;
+        mstat = MNT3ERR_NOENT;
+    }
+    /* FIXME: also take care of the corner case where the
+     * client was resolvable at mount but not at the umount - vice-versa.
+     */
+    mnt3svc_submit_reply(req, &mstat,
+                         (mnt3_serializer)xdr_serialize_mountstat3);
 
 rpcerr:
-        return ret;
+    return ret;
 }
 
-
 int
-__mnt3svc_umountall (struct mount3_state *ms)
+__mnt3svc_umountall(struct mount3_state *ms)
 {
-        struct mountentry       *me = NULL;
-        struct mountentry       *tmp = NULL;
+    struct mountentry *me = NULL;
+    struct mountentry *tmp = NULL;
 
-        if (!ms)
-                return -1;
-
-        if (list_empty (&ms->mountlist))
-                return 0;
+    if (!ms)
+        return -1;
 
-        list_for_each_entry_safe (me, tmp, &ms->mountlist, mlist) {
-                list_del (&me->mlist);       /* Remove from the mount list */
-                __mountdict_remove (ms, me); /* Remove from the mount dict */
-                GF_FREE (me);
-        }
+    if (list_empty(&ms->mountlist))
+        return 0;
 
-        dict_unref (ms->mountdict);
+    list_for_each_entry_safe(me, tmp, &ms->mountlist, mlist)
+    {
+        list_del(&me->mlist);       /* Remove from the mount list */
+        __mountdict_remove(ms, me); /* Remove from the mount dict */
+        GF_FREE(me);
+    }
 
-        return 0;
+    return 0;
 }
 
-
 int
-mnt3svc_umountall (struct mount3_state *ms)
+mnt3svc_umountall(struct mount3_state *ms)
 {
-        int     ret = -1;
-        if (!ms)
-                return -1;
+    int ret = -1;
+    if (!ms)
+        return -1;
 
-        LOCK (&ms->mountlock);
-        {
-               ret = __mnt3svc_umountall (ms);
-        }
-        UNLOCK (&ms->mountlock);
+    LOCK(&ms->mountlock);
+    {
+        ret = __mnt3svc_umountall(ms);
+    }
+    UNLOCK(&ms->mountlock);
 
-        return ret;
+    return ret;
 }
 
-
 int
-mnt3svc_umntall (rpcsvc_request_t *req)
+mnt3svc_umntall(rpcsvc_request_t *req)
 {
-        int                     ret = RPCSVC_ACTOR_ERROR;
-        struct mount3_state     *ms = NULL;
-        mountstat3              mstat = MNT3_OK;
+    int ret = RPCSVC_ACTOR_ERROR;
+    struct mount3_state *ms = NULL;
+    mountstat3 mstat = MNT3_OK;
 
-        if (!req)
-                return ret;
+    if (!req)
+        return ret;
 
-        ms = (struct mount3_state *)rpcsvc_request_program_private (req);
-        if (!ms) {
-                gf_log (GF_MNT, GF_LOG_ERROR, "Mount state not present");
-                rpcsvc_request_seterr (req, SYSTEM_ERR);
-                goto rpcerr;
-        }
+    ms = (struct mount3_state *)rpcsvc_request_program_private(req);
+    if (!ms) {
+        gf_msg(GF_MNT, GF_LOG_ERROR, EINVAL, NFS_MSG_MNT_STATE_NOT_FOUND,
+               "Mount state not present");
+        rpcsvc_request_seterr(req, SYSTEM_ERR);
+        goto rpcerr;
+    }
 
-        mnt3svc_umountall (ms);
-        mnt3svc_submit_reply (req, &mstat,
-                              (mnt3_serializer)xdr_serialize_mountstat3);
+    mnt3svc_umountall(ms);
+    mnt3svc_submit_reply(req, &mstat,
+                         (mnt3_serializer)xdr_serialize_mountstat3);
 
-        ret = RPCSVC_ACTOR_SUCCESS;
+    ret = RPCSVC_ACTOR_SUCCESS;
 rpcerr:
-        return ret;
+    return ret;
 }
 
-
 exports
-mnt3_xlchildren_to_exports (rpcsvc_t *svc, struct mount3_state *ms)
-{
-        struct exportnode       *elist = NULL;
-        struct exportnode       *prev = NULL;
-        struct exportnode       *first = NULL;
-        size_t                  namelen = 0;
-        int                     ret = -1;
-        char                    *addrstr = NULL;
-        struct mnt3_export      *ent = NULL;
-        struct nfs_state        *nfs = NULL;
-
-        if ((!ms) || (!svc))
-                return NULL;
-
-        nfs = (struct nfs_state *)ms->nfsx->private;
-        if (!nfs)
-                return NULL;
-
-        LOCK (&ms->mountlock);
-        list_for_each_entry(ent, &ms->exportlist, explist) {
-
-                /* If volume is not started yet, do not list it for tools like
-                 * showmount.
-                 */
-                if (!nfs_subvolume_started (nfs, ent->vol))
-                        continue;
-
-                namelen = strlen (ent->expname) + 1;
-                elist = GF_CALLOC (1, sizeof (*elist), gf_nfs_mt_exportnode);
-                if (!elist) {
-                        gf_log (GF_MNT, GF_LOG_ERROR, "Memory allocation"
-                                " failed");
-                        goto free_list;
-                }
-                 if (!first)
-                         first = elist;
-                elist->ex_dir = GF_CALLOC (namelen + 2, sizeof (char),
-                                           gf_nfs_mt_char);
-                if (!elist->ex_dir) {
-                        gf_log (GF_MNT, GF_LOG_ERROR, "Memory allocation"
-                                " failed");
-                        goto free_list;
-                }
-                strcpy (elist->ex_dir, ent->expname);
-
-                addrstr = rpcsvc_volume_allowed (svc->options,
-                                                 ent->vol->name);
-                elist->ex_groups = GF_CALLOC (1, sizeof (struct groupnode),
-                                              gf_nfs_mt_groupnode);
-                if (!elist->ex_groups) {
-                        gf_log (GF_MNT, GF_LOG_ERROR, "Memory allocation"
-                                " failed");
-                        goto free_list;
+mnt3_xlchildren_to_exports(rpcsvc_t *svc, struct mount3_state *ms)
+{
+    struct exportnode *elist = NULL;
+    struct exportnode *prev = NULL;
+    struct exportnode *first = NULL;
+    size_t namelen = 0;
+    int ret = -1;
+    char *addrstr = NULL;
+    struct mnt3_export *ent = NULL;
+    struct nfs_state *nfs = NULL;
+
+    if ((!ms) || (!svc))
+        return NULL;
+
+    nfs = (struct nfs_state *)ms->nfsx->private;
+    if (!nfs)
+        return NULL;
+
+    LOCK(&ms->mountlock);
+    list_for_each_entry(ent, &ms->exportlist, explist)
+    {
+        /* If volume is not started yet, do not list it for tools like
+         * showmount.
+         */
+        if (!nfs_subvolume_started(nfs, ent->vol))
+            continue;
+
+        elist = GF_CALLOC(1, sizeof(*elist), gf_nfs_mt_exportnode);
+        if (!elist) {
+            gf_msg(GF_MNT, GF_LOG_ERROR, ENOMEM, NFS_MSG_NO_MEMORY,
+                   "Memory allocation failed");
+            goto free_list;
+        }
+        if (!first)
+            first = elist;
+        namelen = strlen(ent->expname);
+        elist->ex_dir = GF_MALLOC(namelen + 2, gf_nfs_mt_char);
+        if (!elist->ex_dir) {
+            gf_msg(GF_MNT, GF_LOG_ERROR, ENOMEM, NFS_MSG_NO_MEMORY,
+                   "Memory allocation failed");
+            if (first == elist)
+                first = NULL;
+            xdr_free_exports_list(elist);
+            elist = NULL;
+            goto free_list;
+        }
+        strcpy(elist->ex_dir, ent->expname);
+
+        addrstr = rpcsvc_volume_allowed(svc->options, ent->vol->name);
+        if (addrstr) {
+            /* create a groupnode per allowed client */
+            char *pos = NULL;
+            char *addr = NULL;
+            char *addrs = NULL;
+            struct groupnode *group = NULL;
+            struct groupnode *prev_group = NULL;
+
+            /* strtok_r() modifies the string, dup it */
+            addrs = gf_strdup(addrstr);
+            if (!addrs)
+                goto free_list;
+
+            while (1) {
+                /* only pass addrs on the 1st call */
+                addr = strtok_r(group ? NULL : addrs, ",", &pos);
+                if (addr == NULL)
+                    /* no mode clients */
+                    break;
+
+                group = GF_CALLOC(1, sizeof(struct groupnode),
+                                  gf_nfs_mt_groupnode);
+                if (!group) {
+                    gf_msg(GF_MNT, GF_LOG_ERROR, ENOMEM, NFS_MSG_NO_MEMORY,
+                           "Memory "
+                           "allocation failed");
+                    GF_FREE(addrs);
+                    goto free_list;
                 }
-                /*This check has to be done after checking
-                 * elist->ex_groups allocation check to avoid resource leak;
-                */
-                if (addrstr)
-                        addrstr = gf_strdup (addrstr);
-                else
-                        addrstr = gf_strdup ("No Access");
-
-                if (!addrstr) {
-                        goto free_list;
+
+                group->gr_name = gf_strdup(addr);
+                if (!group->gr_name) {
+                    gf_msg(GF_MNT, GF_LOG_ERROR, ENOMEM, NFS_MSG_NO_MEMORY,
+                           "Memory "
+                           "allocation failed");
+                    GF_FREE(group);
+                    GF_FREE(addrs);
+                    goto free_list;
                 }
-                elist->ex_groups->gr_name = addrstr;
-                if (prev) {
-                        prev->ex_next = elist;
-                        prev = elist;
-                } else
-                        prev = elist;
+
+                /* chain the groups together */
+                if (!elist->ex_groups)
+                    elist->ex_groups = group;
+                else if (prev_group && !prev_group->gr_next)
+                    prev_group->gr_next = group;
+                prev_group = group;
+            }
+
+            GF_FREE(addrs);
+        } else {
+            elist->ex_groups = GF_CALLOC(1, sizeof(struct groupnode),
+                                         gf_nfs_mt_groupnode);
+            if (!elist->ex_groups) {
+                gf_msg(GF_MNT, GF_LOG_ERROR, ENOMEM, NFS_MSG_NO_MEMORY,
+                       "Memory allocation "
+                       "failed");
+                goto free_list;
+            }
+
+            addrstr = gf_strdup("No Access");
+            if (!addrstr)
+                goto free_list;
+
+            elist->ex_groups->gr_name = addrstr;
         }
 
-        ret = 0;
+        if (prev) {
+            prev->ex_next = elist;
+            prev = elist;
+        } else
+            prev = elist;
+    }
+
+    ret = 0;
 
 free_list:
-        UNLOCK (&ms->mountlock);
-        if (ret == -1) {
-                xdr_free_exports_list (first);
-                first = NULL;
-        }
+    UNLOCK(&ms->mountlock);
+    if (ret == -1) {
+        xdr_free_exports_list(first);
+        first = NULL;
+    }
 
-        return first;
+    return first;
 }
 
-
 int
-mnt3svc_export (rpcsvc_request_t *req)
+mnt3svc_export(rpcsvc_request_t *req)
 {
-        struct mount3_state     *ms = NULL;
-        exports                 elist = NULL;
-        int                     ret = -1;
+    struct mount3_state *ms = NULL;
+    exports elist = NULL;
+    int ret = -1;
 
-        if (!req)
-                return -1;
-
-        ms = (struct mount3_state *)rpcsvc_request_program_private (req);
-        if (!ms) {
-                gf_log (GF_MNT, GF_LOG_ERROR, "mount state not found");
-                rpcsvc_request_seterr (req, SYSTEM_ERR);
-                goto err;
-        }
-
-        /* Using the children translator names, build the export list */
-        elist = mnt3_xlchildren_to_exports (rpcsvc_request_service (req),
-                                            ms);
-        /* Do not return error when exports list is empty. An exports list can
-         * be empty when no subvolumes have come up. No point returning error
-         * and confusing the user.
-        if (!elist) {
-                gf_log (GF_MNT, GF_LOG_ERROR, "Failed to build exports list");
-                nfs_rpcsvc_request_seterr (req, SYSTEM_ERR);
-                goto err;
-        }
-        */
-
-        /* Note how the serializer is passed to the generic reply function. */
-        mnt3svc_submit_reply (req, &elist,
-                              (mnt3_serializer)xdr_serialize_exports);
+    if (!req)
+        return -1;
 
-        xdr_free_exports_list (elist);
-        ret = 0;
+    ms = (struct mount3_state *)rpcsvc_request_program_private(req);
+    if (!ms) {
+        gf_msg(GF_MNT, GF_LOG_ERROR, EINVAL, NFS_MSG_MNT_STATE_NOT_FOUND,
+               "mount state not found");
+        rpcsvc_request_seterr(req, SYSTEM_ERR);
+        goto err;
+    }
+
+    /* Using the children translator names, build the export list */
+    elist = mnt3_xlchildren_to_exports(rpcsvc_request_service(req), ms);
+    /* Do not return error when exports list is empty. An exports list can
+     * be empty when no subvolumes have come up. No point returning error
+     * and confusing the user.
+    if (!elist) {
+            gf_log (GF_MNT, GF_LOG_ERROR, "Failed to build exports list");
+            nfs_rpcsvc_request_seterr (req, SYSTEM_ERR);
+            goto err;
+    }
+    */
+
+    /* Note how the serializer is passed to the generic reply function. */
+    mnt3svc_submit_reply(req, &elist, (mnt3_serializer)xdr_serialize_exports);
+
+    xdr_free_exports_list(elist);
+    ret = 0;
 err:
-        return ret;
+    return ret;
 }
 
-
 /*
  * __mnt3udp_get_mstate() Fetches mount3_state from xlator
  * Linkage: Static
  * Usage: Used only for UDP MOUNT codepath
  */
 static struct mount3_state *
-__mnt3udp_get_mstate (xlator_t *nfsx)
+__mnt3udp_get_mstate(xlator_t *nfsx)
 {
-        struct nfs_state        *nfs = NULL;
-        struct mount3_state     *ms = NULL;
+    struct nfs_state *nfs = NULL;
+    struct mount3_state *ms = NULL;
 
-        if (nfsx == NULL)
-                return NULL;
+    if (nfsx == NULL)
+        return NULL;
 
-        nfs = (struct nfs_state *)nfsx->private;
-        if (nfs == NULL)
-                return NULL;
+    nfs = (struct nfs_state *)nfsx->private;
+    if (nfs == NULL)
+        return NULL;
 
-        ms = (struct mount3_state *)nfs->mstate;
-        return ms;
+    ms = (struct mount3_state *)nfs->mstate;
+    return ms;
 }
 
 extern int
-glfs_resolve_at (struct glfs *, xlator_t *, inode_t *,
-                 const char *, loc_t *, struct iatt *, int, int);
+glfs_resolve_at(struct glfs *, xlator_t *, inode_t *, const char *, loc_t *,
+                struct iatt *, int, int);
 
 extern struct glfs *
-glfs_new_from_ctx (glusterfs_ctx_t *);
+glfs_new_from_ctx(glusterfs_ctx_t *);
 
 extern void
-glfs_free_from_ctx (struct glfs *);
+glfs_free_from_ctx(struct glfs *);
 
 static inode_t *
-__mnt3udp_get_export_subdir_inode (struct svc_req *req, char *subdir,
-                                   char *expname, /* OUT */
-                                   struct mnt3_export *exp)
-{
-        inode_t                 *inode = NULL;
-        loc_t                   loc = {0, };
-        struct iatt             buf = {0, };
-        int                     ret = -1;
-        glfs_t                  *fs = NULL;
-
-        if ((!req) || (!subdir) || (!expname) || (!exp))
-                return NULL;
-
-        /* AUTH check for subdir i.e. nfs.export-dir */
-        if (exp->hostspec) {
-                struct sockaddr_in *sin = svc_getcaller (req->rq_xprt);
-                ret = mnt3_verify_auth (sin, exp);
-                if (ret) {
-                        gf_log (GF_MNT,GF_LOG_ERROR,
-                                "AUTH(nfs.export-dir) verification failed");
-                        errno = EACCES;
-                        return NULL;
-                }
-        }
+__mnt3udp_get_export_subdir_inode(struct svc_req *req, char *subdir,
+                                  char *expname, /* OUT */
+                                  struct mnt3_export *exp)
+{
+    inode_t *inode = NULL;
+    loc_t loc = {
+        0,
+    };
+    struct iatt buf = {
+        0,
+    };
+    int ret = -1;
+    glfs_t *fs = NULL;
+
+    if ((!req) || (!subdir) || (!expname) || (!exp))
+        return NULL;
 
-        /*
-         * IMP: glfs_t fs object is not used by glfs_resolve_at (). The main
-         * purpose is to not change the ABI of glfs_resolve_at () and not to
-         * pass a NULL object.
-         *
-         * TODO: Instead of linking against libgfapi.so, just for one API
-         * i.e. glfs_resolve_at(), It would be cleaner if PATH name to
-         * inode resolution code can be moved to libglusterfs.so or so.
-         * refer bugzilla for more details :
-         * https://bugzilla.redhat.com/show_bug.cgi?id=1161573
+    /* AUTH check for subdir i.e. nfs.export-dir */
+    if (exp->hostspec) {
+        struct sockaddr_in *sin = NULL;
+
+#if !defined(_TIRPC_SVC_H)
+        sin = svc_getcaller(req->rq_xprt);
+#else
+        sin = (struct sockaddr_in *)svc_getcaller(req->rq_xprt);
+        /* TIRPC's svc_getcaller() returns a pointer to a
+         * sockaddr_in6, even though it might actually be an
+         * IPv4 address. It ought return a struct sockaddr and
+         * make the caller upcast it to the proper address family.
          */
-        fs = glfs_new_from_ctx (exp->vol->ctx);
-        if (!fs)
-                return NULL;
+#endif
+        /* And let's make sure that it's actually an IPv4 address. */
+        GF_ASSERT(sin->sin_family == AF_INET);
 
-        ret = glfs_resolve_at (fs, exp->vol, NULL, subdir,
-                               &loc, &buf, 1 /* Follow link */,
-                               0 /* Hard lookup */);
+        ret = mnt3_verify_auth(sin, exp);
+        if (ret) {
+            gf_msg(GF_MNT, GF_LOG_ERROR, EACCES, NFS_MSG_AUTH_VERIFY_FAILED,
+                   "AUTH(nfs.export-dir) verification failed");
+            errno = EACCES;
+            return NULL;
+        }
+    }
+
+    /*
+     * IMP: glfs_t fs object is not used by glfs_resolve_at (). The main
+     * purpose is to not change the ABI of glfs_resolve_at () and not to
+     * pass a NULL object.
+     *
+     * TODO: Instead of linking against libgfapi.so, just for one API
+     * i.e. glfs_resolve_at(), It would be cleaner if PATH name to
+     * inode resolution code can be moved to libglusterfs.so or so.
+     * refer bugzilla for more details :
+     * https://bugzilla.redhat.com/show_bug.cgi?id=1161573
+     */
+    fs = glfs_new_from_ctx(exp->vol->ctx);
+    if (!fs)
+        return NULL;
 
-        glfs_free_from_ctx (fs);
+    ret = glfs_resolve_at(fs, exp->vol, NULL, subdir, &loc, &buf,
+                          1 /* Follow link */, 0 /* Hard lookup */);
 
-        if (ret != 0) {
-                loc_wipe (&loc);
-                return NULL;
-        }
+    glfs_free_from_ctx(fs);
+
+    if (ret != 0) {
+        loc_wipe(&loc);
+        return NULL;
+    }
 
-        inode = inode_ref (loc.inode);
-        snprintf (expname, PATH_MAX, "/%s%s", exp->vol->name, loc.path);
+    inode = inode_ref(loc.inode);
+    snprintf(expname, PATH_MAX, "/%s%s", exp->vol->name, loc.path);
 
-        loc_wipe (&loc);
+    loc_wipe(&loc);
 
-        return inode;
+    return inode;
 }
 
 static inode_t *
-__mnt3udp_get_export_volume_inode (struct svc_req *req, char *volpath,
-                                   char *expname, /* OUT */
-                                   struct mnt3_export *exp)
+__mnt3udp_get_export_volume_inode(struct svc_req *req, char *volpath,
+                                  char *expname, /* OUT */
+                                  struct mnt3_export *exp)
 {
-        char                    *rpath = NULL;
-        inode_t                 *inode = NULL;
+    char *rpath = NULL;
+    inode_t *inode = NULL;
 
-        if ((!req) || (!volpath) || (!expname) || (!exp))
-                return NULL;
+    if ((!req) || (!volpath) || (!expname) || (!exp))
+        return NULL;
 
-        rpath = strchr (volpath, '/');
-        if (rpath == NULL)
-                rpath = "/";
+    rpath = strchr(volpath, '/');
+    if (rpath == NULL)
+        rpath = "/";
 
-        inode = inode_from_path (exp->vol->itable, rpath);
-        snprintf (expname, PATH_MAX, "/%s", exp->vol->name);
+    inode = inode_from_path(exp->vol->itable, rpath);
+    snprintf(expname, PATH_MAX, "/%s", exp->vol->name);
 
-        return inode;
+    return inode;
 }
 
 /*
@@ -2745,161 +2927,161 @@ __mnt3udp_get_export_volume_inode (struct svc_req *req, char *volpath,
  * when nfs.mount-udp is ENABLED (set to TRUE/ON).
  */
 struct nfs3_fh *
-nfs3_rootfh (struct svc_req *req, xlator_t *nfsx,
-             char *path, char *expname /* OUT */)
-{
-        struct nfs3_fh          *fh = NULL;
-        inode_t                 *inode = NULL;
-        struct mnt3_export      *exp = NULL;
-        struct mount3_state     *ms = NULL;
-        struct nfs_state        *nfs = NULL;
-        int                     mnt3type = MNT3_EXPTYPE_DIR;
-        int                     ret = RPCSVC_AUTH_REJECT;
-
-        if ((!req) || (!nfsx) || (!path) || (!expname)) {
-                errno = EFAULT;
-                return NULL;
-        }
-
-        /*
-         * 1. First check if the MOUNT is for whole volume.
-         *      i.e. __mnt3udp_get_export_volume_inode ()
-         * 2. If NOT, then TRY for SUBDIR MOUNT.
-         *      i.e. __mnt3udp_get_export_subdir_inode ()
-         * 3. If a subdir is exported using nfs.export-dir,
-         *      then the mount type would be MNT3_EXPTYPE_DIR,
-         *      so make sure to find the proper path to be
-         *      resolved using __volume_subdir()
-         * 3. Make sure subdir export is allowed.
-         */
-        ms = __mnt3udp_get_mstate(nfsx);
-        if (!ms) {
-                errno = EFAULT;
-                return NULL;
-        }
-
-        exp = mnt3_mntpath_to_export (ms, path , _gf_false);
-        if (exp != NULL)
-                mnt3type = exp->exptype;
+nfs3_rootfh(struct svc_req *req, xlator_t *nfsx, char *path,
+            char *expname /* OUT */)
+{
+    struct nfs3_fh *fh = NULL;
+    inode_t *inode = NULL;
+    struct mnt3_export *exp = NULL;
+    struct mount3_state *ms = NULL;
+    struct nfs_state *nfs = NULL;
+    int mnt3type = MNT3_EXPTYPE_DIR;
+    int ret = RPCSVC_AUTH_REJECT;
+
+    if ((!req) || (!nfsx) || (!path) || (!expname)) {
+        errno = EFAULT;
+        return NULL;
+    }
+
+    /*
+     * 1. First check if the MOUNT is for whole volume.
+     *      i.e. __mnt3udp_get_export_volume_inode ()
+     * 2. If NOT, then TRY for SUBDIR MOUNT.
+     *      i.e. __mnt3udp_get_export_subdir_inode ()
+     * 3. If a subdir is exported using nfs.export-dir,
+     *      then the mount type would be MNT3_EXPTYPE_DIR,
+     *      so make sure to find the proper path to be
+     *      resolved using mnt3_get_volume_subdir()
+     * 3. Make sure subdir export is allowed.
+     */
+    ms = __mnt3udp_get_mstate(nfsx);
+    if (!ms) {
+        errno = EFAULT;
+        return NULL;
+    }
 
-        if (mnt3type == MNT3_EXPTYPE_DIR) {
-                char    volname [MNTPATHLEN] = {0, };
-                char    *volptr = volname;
+    exp = mnt3_mntpath_to_export(ms, path, _gf_false);
+    if (exp != NULL)
+        mnt3type = exp->exptype;
 
-                /* Subdir export (nfs3.export-dirs) check */
-                if (!gf_mnt3_export_dirs(ms)) {
-                        errno = EACCES;
-                        return NULL;
-                }
+    if (mnt3type == MNT3_EXPTYPE_DIR) {
+        char volname[MNTPATHLEN] = {
+            0,
+        };
+        char *volptr = volname;
 
-                path = __volume_subdir (path, &volptr);
-                if (exp == NULL)
-                        exp = mnt3_mntpath_to_export (ms, volname , _gf_false);
+        /* Subdir export (nfs3.export-dirs) check */
+        if (!gf_mnt3_export_dirs(ms)) {
+            errno = EACCES;
+            return NULL;
         }
 
-        if (exp == NULL) {
-                errno = ENOENT;
-                return NULL;
-        }
+        path = mnt3_get_volume_subdir(path, &volptr);
+        if (exp == NULL)
+            exp = mnt3_mntpath_to_export(ms, volname, _gf_false);
+    }
 
-        nfs = (struct nfs_state *)nfsx->private;
-        if (!nfs_subvolume_started (nfs, exp->vol)) {
-                errno = ENOENT;
-                return NULL;
-        }
+    if (exp == NULL) {
+        errno = ENOENT;
+        return NULL;
+    }
 
-        /* AUTH check: respect nfs.rpc-auth-allow/reject */
-        ret = mnt3_check_client_net_udp (req, exp->vol->name, nfsx);
-        if (ret == RPCSVC_AUTH_REJECT) {
-                errno = EACCES;
-                return NULL;
-        }
+    nfs = (struct nfs_state *)nfsx->private;
+    if (!nfs_subvolume_started(nfs, exp->vol)) {
+        errno = ENOENT;
+        return NULL;
+    }
 
-        switch (mnt3type) {
+    /* AUTH check: respect nfs.rpc-auth-allow/reject */
+    ret = mnt3_check_client_net_udp(req, exp->vol->name, nfsx);
+    if (ret == RPCSVC_AUTH_REJECT) {
+        errno = EACCES;
+        return NULL;
+    }
 
+    switch (mnt3type) {
         case MNT3_EXPTYPE_VOLUME:
-                inode = __mnt3udp_get_export_volume_inode (req, path,
-                                                           expname, exp);
-                break;
+            inode = __mnt3udp_get_export_volume_inode(req, path, expname, exp);
+            break;
 
         case MNT3_EXPTYPE_DIR:
-                inode = __mnt3udp_get_export_subdir_inode (req, path,
-                                                           expname, exp);
-                break;
+            inode = __mnt3udp_get_export_subdir_inode(req, path, expname, exp);
+            break;
 
         default:
-                /* Never reachable */
-                gf_log (GF_MNT, GF_LOG_ERROR, "Unknown MOUNT3 type");
-                errno = EFAULT;
-                goto err;
-        }
-
-        if (inode == NULL) {
-                /* Don't over-write errno */
-                if (!errno)
-                        errno = ENOENT;
-                goto err;
-        }
-
-        /* Build the inode from FH */
-        fh = GF_CALLOC (1, sizeof(*fh), gf_nfs_mt_nfs3_fh);
-        if (fh == NULL) {
-                errno = ENOMEM;
-                goto err;
-        }
-
-        (void) nfs3_build_fh (inode, exp->volumeid, fh);
+            /* Never reachable */
+            gf_msg(GF_MNT, GF_LOG_ERROR, EFAULT, NFS_MSG_UNKNOWN_MNT_TYPE,
+                   "Unknown MOUNT3 type");
+            errno = EFAULT;
+            goto err;
+    }
+
+    if (inode == NULL) {
+        /* Don't over-write errno */
+        if (!errno)
+            errno = ENOENT;
+        goto err;
+    }
+
+    /* Build the inode from FH */
+    fh = GF_CALLOC(1, sizeof(*fh), gf_nfs_mt_nfs3_fh);
+    if (fh == NULL) {
+        errno = ENOMEM;
+        goto err;
+    }
+
+    (void)nfs3_build_fh(inode, exp->volumeid, fh);
 
 err:
-        if (inode)
-                inode_unref (inode);
+    if (inode)
+        inode_unref(inode);
 
-        return fh;
+    return fh;
 }
 
 int
-mount3udp_add_mountlist (xlator_t *nfsx, char *host, char *export)
+mount3udp_add_mountlist(xlator_t *nfsx, char *host, char *export)
 {
-        struct mountentry       *me = NULL;
-        struct mount3_state     *ms = NULL;
+    struct mountentry *me = NULL;
+    struct mount3_state *ms = NULL;
 
-        if ((!host) || (!export) || (!nfsx))
-                return -1;
+    if ((!host) || (!export) || (!nfsx))
+        return -1;
 
-        ms = __mnt3udp_get_mstate (nfsx);
-        if (!ms)
-                return -1;
+    ms = __mnt3udp_get_mstate(nfsx);
+    if (!ms)
+        return -1;
 
-        me = GF_CALLOC (1, sizeof (*me), gf_nfs_mt_mountentry);
-        if (!me)
-                return -1;
+    me = GF_CALLOC(1, sizeof(*me), gf_nfs_mt_mountentry);
+    if (!me)
+        return -1;
 
-        strncpy (me->exname, export, MNTPATHLEN);
-        strncpy (me->hostname, host, MNTPATHLEN);
-        INIT_LIST_HEAD (&me->mlist);
-        LOCK (&ms->mountlock);
-        {
-                list_add_tail (&me->mlist, &ms->mountlist);
-                mount_rewrite_rmtab(ms, NULL);
-        }
-        UNLOCK (&ms->mountlock);
-        return 0;
+    snprintf(me->exname, MNTPATHLEN, "%s", export);
+    snprintf(me->hostname, MNTPATHLEN, "%s", host);
+    INIT_LIST_HEAD(&me->mlist);
+    LOCK(&ms->mountlock);
+    {
+        list_add_tail(&me->mlist, &ms->mountlist);
+        mount_rewrite_rmtab(ms, NULL);
+    }
+    UNLOCK(&ms->mountlock);
+    return 0;
 }
 
 int
-mount3udp_delete_mountlist (xlator_t *nfsx, char *hostname, char *export)
+mount3udp_delete_mountlist(xlator_t *nfsx, char *hostname, char *export)
 {
-        struct mount3_state     *ms = NULL;
+    struct mount3_state *ms = NULL;
 
-        if ((!hostname) || (!export) || (!nfsx))
-                return -1;
+    if ((!hostname) || (!export) || (!nfsx))
+        return -1;
 
-        ms = __mnt3udp_get_mstate (nfsx);
-        if (!ms)
-                return -1;
+    ms = __mnt3udp_get_mstate(nfsx);
+    if (!ms)
+        return -1;
 
-        mnt3svc_umount (ms, export, hostname);
-        return 0;
+    mnt3svc_umount(ms, export, hostname);
+    return 0;
 }
 
 /**
@@ -2914,87 +3096,88 @@ mount3udp_delete_mountlist (xlator_t *nfsx, char *hostname, char *export)
  * NB: This does not support IPv6 currently.
  */
 int
-mnt3_export_fill_hostspec (struct host_auth_spec* hostspec, const char* hostip)
-{
-        char     *ipdupstr = NULL;
-        char     *savptr = NULL;
-        char     *endptr = NULL;
-        char     *ip = NULL;
-        char     *token = NULL;
-        int      ret = -1;
-        long     prefixlen = IPv4_ADDR_SIZE; /* default */
-        uint32_t shiftbits = 0;
-        size_t   length = 0;
-
-        /* Create copy of the string so that the source won't change
-         */
-        ipdupstr = gf_strdup (hostip);
-        if (NULL == ipdupstr) {
-                gf_log (GF_MNT, GF_LOG_ERROR, "Memory allocation failed");
-                goto err;
-        }
-
-        ip = strtok_r (ipdupstr, "/", &savptr);
-        /* Validate the Hostname or IPv4 address
-         * TODO: IPv6 support for subdir auth.
-         */
-        length = strlen (ip);
-        if ((!valid_ipv4_address (ip, (int)length, _gf_false)) &&
-            (!valid_host_name (ip, (int)length))) {
-                gf_log (GF_MNT, GF_LOG_ERROR,
-                        "Invalid hostname or IPv4 address: %s", ip);
-                goto err;
-        }
-
-        hostspec->host_addr = gf_strdup (ip);
-        if (NULL == hostspec->host_addr) {
-                gf_log (GF_MNT, GF_LOG_ERROR, "Memory allocation failed");
-                goto err;
-        }
-
-        /**
-         * User provided CIDR address (xx.xx.xx.xx/n format) is split
-         * into HOST (IP addr or hostname) and network prefix(n) from
-         * which netmask would be calculated. This CIDR address may
-         * denote a single, distinct interface address or the beginning
-         * address of an entire network.
-         *
-         * e.g. the IPv4 block 192.168.100.0/24 represents the 256
-         * IPv4 addresses from 192.168.100.0 to 192.168.100.255.
-         * Therefore to check if an IP matches 192.168.100.0/24
-         * we should mask the IP with FFFFFF00 and compare it with
-         * host address part of CIDR.
-         *
-         * Refer: mask_match() in common-utils.c.
-         */
-        token = strtok_r (NULL, "/", &savptr);
-        if (token != NULL) {
-              prefixlen = strtol (token, &endptr, 10);
-              if ((errno != 0) || (*endptr != '\0') ||
-                  (prefixlen < 0) || (prefixlen > IPv4_ADDR_SIZE)) {
-                      gf_log (THIS->name, GF_LOG_WARNING,
-                              "Invalid IPv4 subnetwork mask");
-                      goto err;
-              }
-        }
-
-        /*
-         * 1. Calculate the network mask address.
-         * 2. Convert it into Big-Endian format.
-         * 3. Store it in hostspec netmask.
-         */
-        shiftbits = IPv4_ADDR_SIZE - prefixlen;
-        hostspec->netmask = htonl ((uint32_t)~0 << shiftbits);
-
-        ret = 0; /* SUCCESS */
+mnt3_export_fill_hostspec(struct host_auth_spec *hostspec, const char *hostip)
+{
+    char *ipdupstr = NULL;
+    char *savptr = NULL;
+    char *endptr = NULL;
+    char *ip = NULL;
+    char *token = NULL;
+    int ret = -1;
+    long prefixlen = IPv4_ADDR_SIZE; /* default */
+    uint32_t shiftbits = 0;
+    size_t length = 0;
+
+    /* Create copy of the string so that the source won't change
+     */
+    ipdupstr = gf_strdup(hostip);
+    if (NULL == ipdupstr) {
+        gf_msg(GF_MNT, GF_LOG_ERROR, ENOMEM, NFS_MSG_NO_MEMORY,
+               "Memory allocation failed");
+        goto err;
+    }
+
+    ip = strtok_r(ipdupstr, "/", &savptr);
+    /* Validate the Hostname or IPv4 address
+     * TODO: IPv6 support for subdir auth.
+     */
+    length = strlen(ip);
+    if ((!valid_ipv4_address(ip, (int)length, _gf_false)) &&
+        (!valid_host_name(ip, (int)length))) {
+        gf_msg(GF_MNT, GF_LOG_ERROR, EINVAL, NFS_MSG_INVALID_ENTRY,
+               "Invalid hostname or IPv4 address: %s", ip);
+        goto err;
+    }
+
+    hostspec->host_addr = gf_strdup(ip);
+    if (NULL == hostspec->host_addr) {
+        gf_msg(GF_MNT, GF_LOG_ERROR, ENOMEM, NFS_MSG_NO_MEMORY,
+               "Memory allocation failed");
+        goto err;
+    }
+
+    /**
+     * User provided CIDR address (xx.xx.xx.xx/n format) is split
+     * into HOST (IP addr or hostname) and network prefix(n) from
+     * which netmask would be calculated. This CIDR address may
+     * denote a single, distinct interface address or the beginning
+     * address of an entire network.
+     *
+     * e.g. the IPv4 block 192.168.100.0/24 represents the 256
+     * IPv4 addresses from 192.168.100.0 to 192.168.100.255.
+     * Therefore to check if an IP matches 192.168.100.0/24
+     * we should mask the IP with FFFFFF00 and compare it with
+     * host address part of CIDR.
+     *
+     * Refer: mask_match() in common-utils.c.
+     */
+    token = strtok_r(NULL, "/", &savptr);
+    if (token != NULL) {
+        prefixlen = strtol(token, &endptr, 10);
+        if ((errno != 0) || (*endptr != '\0') || (prefixlen < 0) ||
+            (prefixlen > IPv4_ADDR_SIZE)) {
+            gf_msg(THIS->name, GF_LOG_WARNING, EINVAL, NFS_MSG_INVALID_ENTRY,
+                   "Invalid IPv4 subnetwork mask");
+            goto err;
+        }
+    }
+
+    /*
+     * 1. Calculate the network mask address.
+     * 2. Convert it into Big-Endian format.
+     * 3. Store it in hostspec netmask.
+     */
+    shiftbits = IPv4_ADDR_SIZE - prefixlen;
+    hostspec->netmask = htonl((uint32_t)~0 << shiftbits);
+
+    ret = 0; /* SUCCESS */
 err:
-        if (NULL != ipdupstr) {
-                GF_FREE (ipdupstr);
-        }
-        return ret;
+    if (NULL != ipdupstr) {
+        GF_FREE(ipdupstr);
+    }
+    return ret;
 }
 
-
 /**
  * This function will parse the AUTH parameter passed along with
  * "export-dir" option. If AUTH parameter is present then it will be
@@ -3008,77 +3191,81 @@ err:
  * @return This function will return 0 on success and -1 on failure.
  */
 int
-mnt3_export_parse_auth_param (struct mnt3_export* exp, char* exportpath)
+mnt3_export_parse_auth_param(struct mnt3_export *exp, char *exportpath)
 {
-        char *token = NULL;
-        char *savPtr = NULL;
-        char *hostip = NULL;
-        struct host_auth_spec *host = NULL;
-        int ret = 0;
-
-        /* Using exportpath directly in strtok_r because we want
-         * to strip off AUTH parameter from exportpath. */
-        token = strtok_r (exportpath, "(", &savPtr);
-
-        /* Get the next token, which will be the AUTH parameter. */
-        token = strtok_r (NULL, ")", &savPtr);
+    char *token = NULL;
+    char *savPtr = NULL;
+    char *hostip = NULL;
+    struct host_auth_spec *host = NULL;
+    int ret = 0;
+
+    if (exportpath == NULL) {
+        gf_msg(GF_MNT, GF_LOG_ERROR, EINVAL, NFS_MSG_PARSE_HOSTSPEC_FAIL,
+               "Export path is NULL");
+        return -1;
+    }
 
-        if (NULL == token) {
-                /* If AUTH is not present then we should return success. */
-                return 0;
-        }
+    /* Using exportpath directly in strtok_r because we want
+     * to strip off AUTH parameter from exportpath. */
+    token = strtok_r(exportpath, "(", &savPtr);
 
-        /* Free any previously allocated hostspec structure. */
-        if (NULL != exp->hostspec) {
-                GF_FREE (exp->hostspec);
-                exp->hostspec = NULL;
-        }
+    /* Get the next token, which will be the AUTH parameter. */
+    token = strtok_r(NULL, ")", &savPtr);
 
-        exp->hostspec = GF_CALLOC (1,
-                        sizeof (*(exp->hostspec)),
-                        gf_nfs_mt_auth_spec);
-        if (NULL == exp->hostspec){
-                gf_log (GF_MNT, GF_LOG_ERROR, "Memory allocation failed");
-                return -1;
-        }
+    if (NULL == token) {
+        /* If AUTH is not present then we should return success. */
+        return 0;
+    }
+
+    /* Free any previously allocated hostspec structure. */
+    if (NULL != exp->hostspec) {
+        GF_FREE(exp->hostspec);
+        exp->hostspec = NULL;
+    }
+
+    exp->hostspec = GF_CALLOC(1, sizeof(*(exp->hostspec)), gf_nfs_mt_auth_spec);
+    if (NULL == exp->hostspec) {
+        gf_msg(GF_MNT, GF_LOG_ERROR, ENOMEM, NFS_MSG_NO_MEMORY,
+               "Memory allocation failed");
+        return -1;
+    }
 
-        /* AUTH parameter can have multiple entries. For each entry
-         * a host_auth_spec structure is created. */
-        host = exp->hostspec;
+    /* AUTH parameter can have multiple entries. For each entry
+     * a host_auth_spec structure is created. */
+    host = exp->hostspec;
 
-        hostip = strtok_r (token, "|", &savPtr);
+    hostip = strtok_r(token, "|", &savPtr);
 
-        /* Parse all AUTH parameters separated by '|' */
-        while (NULL != hostip){
-                ret = mnt3_export_fill_hostspec (host, hostip);
-                if (0 != ret) {
-                        gf_log(GF_MNT, GF_LOG_WARNING,
-                                        "Failed to parse hostspec: %s", hostip);
-                        goto err;
-                }
+    /* Parse all AUTH parameters separated by '|' */
+    while (NULL != hostip) {
+        ret = mnt3_export_fill_hostspec(host, hostip);
+        if (0 != ret) {
+            gf_msg(GF_MNT, GF_LOG_WARNING, 0, NFS_MSG_PARSE_HOSTSPEC_FAIL,
+                   "Failed to parse hostspec: %s", hostip);
+            goto err;
+        }
 
-                hostip = strtok_r (NULL, "|", &savPtr);
-                if (NULL == hostip) {
-                        break;
-                }
+        hostip = strtok_r(NULL, "|", &savPtr);
+        if (NULL == hostip) {
+            break;
+        }
 
-                host->next = GF_CALLOC (1, sizeof (*(host)),
-                                gf_nfs_mt_auth_spec);
-                if (NULL == host->next){
-                        gf_log (GF_MNT,GF_LOG_ERROR,
-                                        "Memory allocation failed");
-                        goto err;
-                }
-                host = host->next;
+        host->next = GF_CALLOC(1, sizeof(*(host)), gf_nfs_mt_auth_spec);
+        if (NULL == host->next) {
+            gf_msg(GF_MNT, GF_LOG_ERROR, ENOMEM, NFS_MSG_NO_MEMORY,
+                   "Memory allocation failed");
+            goto err;
         }
+        host = host->next;
+    }
 
-        /* In case of success return from here */
-        return 0;
+    /* In case of success return from here */
+    return 0;
 err:
-        /* In case of failure free up hostspec structure.  */
-        FREE_HOSTSPEC (exp);
+    /* In case of failure free up hostspec structure.  */
+    FREE_HOSTSPEC(exp);
 
-        return -1;
+    return -1;
 }
 
 /**
@@ -3087,404 +3274,416 @@ err:
  * exportpath format: <abspath>[(hostdesc[|hostspec|...])]
  */
 struct mnt3_export *
-mnt3_init_export_ent (struct mount3_state *ms, xlator_t *xl, char *exportpath,
-                      uuid_t volumeid)
+mnt3_init_export_ent(struct mount3_state *ms, xlator_t *xl, char *exportpath,
+                     uuid_t volumeid)
 {
-        struct mnt3_export      *exp = NULL;
-        int                     alloclen = 0;
-        int                     ret = -1;
+    struct mnt3_export *exp = NULL;
+    int alloclen = 0;
+    int ret = -1;
 
-        if ((!ms) || (!xl))
-                return NULL;
-
-        exp = GF_CALLOC (1, sizeof (*exp), gf_nfs_mt_mnt3_export);
-        if (!exp) {
-                gf_log (GF_MNT, GF_LOG_ERROR, "Memory allocation failed");
-                return NULL;
-        }
-
-        if (NULL != exportpath) {
-                /* If exportpath is not NULL then we should check if AUTH
-                 * parameter is present or not. If AUTH parameter is present
-                 * then it will be stripped and stored in mnt3_export (exp)
-                 * structure.
-                 */
-                if (0 != mnt3_export_parse_auth_param (exp, exportpath)){
-                        gf_log (GF_MNT, GF_LOG_ERROR,
-                                                "Failed to parse auth param");
-                        goto err;
-                }
-        }
-
-
-        INIT_LIST_HEAD (&exp->explist);
-        if (exportpath)
-                alloclen = strlen (xl->name) + 2 + strlen (exportpath);
-        else
-                alloclen = strlen (xl->name) + 2;
+    if ((!ms) || (!xl))
+        return NULL;
 
-        exp->expname = GF_CALLOC (alloclen, sizeof (char), gf_nfs_mt_char);
-        if (!exp->expname) {
-                gf_log (GF_MNT, GF_LOG_ERROR, "Memory allocation failed");
-                goto err;
-        }
+    exp = GF_CALLOC(1, sizeof(*exp), gf_nfs_mt_mnt3_export);
+    if (!exp) {
+        gf_msg(GF_MNT, GF_LOG_ERROR, ENOMEM, NFS_MSG_NO_MEMORY,
+               "Memory allocation failed");
+        return NULL;
+    }
 
-        if (exportpath) {
-                gf_log (GF_MNT, GF_LOG_TRACE, "Initing dir export: %s:%s",
-                        xl->name, exportpath);
-                exp->exptype = MNT3_EXPTYPE_DIR;
-                ret = snprintf (exp->expname, alloclen, "/%s%s", xl->name,
-                                exportpath);
-        } else {
-                gf_log (GF_MNT, GF_LOG_TRACE, "Initing volume export: %s",
-                        xl->name);
-                exp->exptype = MNT3_EXPTYPE_VOLUME;
-                ret = snprintf (exp->expname, alloclen, "/%s", xl->name);
-        }
-        if (ret < 0) {
-                gf_log (xl->name, GF_LOG_ERROR,
-                        "Failed to set the export name");
-                goto err;
-        }
-        /* Just copy without discrimination, we'll determine whether to
-         * actually use it when a mount request comes in and a file handle
-         * needs to be built.
+    if (NULL != exportpath) {
+        /* If exportpath is not NULL then we should check if AUTH
+         * parameter is present or not. If AUTH parameter is present
+         * then it will be stripped and stored in mnt3_export (exp)
+         * structure.
          */
-        gf_uuid_copy (exp->volumeid, volumeid);
-        exp->vol = xl;
-
-        /* On success we should return from here*/
-        return exp;
+        if (0 != mnt3_export_parse_auth_param(exp, exportpath)) {
+            gf_msg(GF_MNT, GF_LOG_ERROR, 0, NFS_MSG_PARSE_AUTH_PARAM_FAIL,
+                   "Failed to parse auth param");
+            goto err;
+        }
+    }
+
+    INIT_LIST_HEAD(&exp->explist);
+    if (exportpath)
+        alloclen = strlen(xl->name) + 2 + strlen(exportpath);
+    else
+        alloclen = strlen(xl->name) + 2;
+
+    exp->expname = GF_MALLOC(alloclen, gf_nfs_mt_char);
+    if (!exp->expname) {
+        gf_msg(GF_MNT, GF_LOG_ERROR, ENOMEM, NFS_MSG_NO_MEMORY,
+               "Memory allocation failed");
+        goto err;
+    }
+
+    if (exportpath) {
+        gf_msg_trace(GF_MNT, 0, "Initing dir export: %s:%s", xl->name,
+                     exportpath);
+        exp->exptype = MNT3_EXPTYPE_DIR;
+        ret = snprintf(exp->expname, alloclen, "/%s%s", xl->name, exportpath);
+    } else {
+        gf_msg_trace(GF_MNT, 0, "Initing volume export: %s", xl->name);
+        exp->exptype = MNT3_EXPTYPE_VOLUME;
+        ret = snprintf(exp->expname, alloclen, "/%s", xl->name);
+    }
+    if (ret < 0) {
+        gf_msg(xl->name, GF_LOG_ERROR, ret, NFS_MSG_SET_EXP_FAIL,
+               "Failed to set the export name");
+        goto err;
+    }
+    /* Just copy without discrimination, we'll determine whether to
+     * actually use it when a mount request comes in and a file handle
+     * needs to be built.
+     */
+    gf_uuid_copy(exp->volumeid, volumeid);
+    exp->vol = xl;
+
+    /* On success we should return from here*/
+    return exp;
 err:
-        /* On failure free exp and it's members.*/
-        if (NULL != exp) {
-                mnt3_export_free (exp);
-                exp = NULL;
-        }
+    /* On failure free exp and it's members.*/
+    if (NULL != exp) {
+        mnt3_export_free(exp);
+        exp = NULL;
+    }
 
-        return exp;
+    return exp;
 }
 
-
 int
-__mnt3_init_volume_direxports (struct mount3_state *ms, xlator_t *xlator,
-                               char *optstr, uuid_t volumeid)
-{
-        struct mnt3_export      *newexp = NULL;
-        int                     ret = -1;
-        char                    *savptr = NULL;
-        char                    *dupopt = NULL;
-        char                    *token = NULL;
-
-        if ((!ms) || (!xlator) || (!optstr))
-                return -1;
-
-        dupopt = strdupa (optstr);
-
-        token = strtok_r (dupopt, ",", &savptr);
-        while (token) {
-                newexp = mnt3_init_export_ent (ms, xlator, token, volumeid);
-                if (!newexp) {
-                        gf_log (GF_MNT, GF_LOG_ERROR, "Failed to init dir "
-                                "export: %s", token);
-                        ret = -1;
-                        goto err;
-                }
+__mnt3_init_volume_direxports(struct mount3_state *ms, xlator_t *xlator,
+                              char *optstr, uuid_t volumeid)
+{
+    struct mnt3_export *newexp = NULL;
+    int ret = -1;
+    char *savptr = NULL;
+    char *dupopt = NULL;
+    char *token = NULL;
+
+    if ((!ms) || (!xlator) || (!optstr))
+        return -1;
 
-                list_add_tail (&newexp->explist, &ms->exportlist);
-                token = strtok_r (NULL, ",", &savptr);
+    dupopt = strdupa(optstr);
+
+    token = strtok_r(dupopt, ",", &savptr);
+    while (token) {
+        newexp = mnt3_init_export_ent(ms, xlator, token, volumeid);
+        if (!newexp) {
+            gf_msg(GF_MNT, GF_LOG_ERROR, 0, NFS_MSG_INIT_DIR_EXP_FAIL,
+                   "Failed to init dir "
+                   "export: %s",
+                   token);
+            ret = -1;
+            goto err;
         }
 
-        ret = 0;
+        list_add_tail(&newexp->explist, &ms->exportlist);
+        token = strtok_r(NULL, ",", &savptr);
+    }
+
+    ret = 0;
 err:
-        return ret;
+    return ret;
 }
 
-
 int
-__mnt3_init_volume (struct mount3_state *ms, dict_t *opts, xlator_t *xlator)
+__mnt3_init_volume(struct mount3_state *ms, dict_t *opts, xlator_t *xlator)
 {
-        struct mnt3_export      *newexp = NULL;
-        int                     ret = -1;
-        char                    searchstr[1024];
-        char                    *optstr  = NULL;
-        uuid_t                  volumeid = {0, };
+    struct mnt3_export *newexp = NULL;
+    int ret = -1;
+    char searchstr[1024];
+    char *optstr = NULL;
+    uuid_t volumeid = {
+        0,
+    };
+
+    if ((!ms) || (!xlator) || (!opts))
+        return -1;
 
-        if ((!ms) || (!xlator) || (!opts))
-                return -1;
+    gf_uuid_clear(volumeid);
+    if (gf_nfs_dvm_off(nfs_state(ms->nfsx)))
+        goto no_dvm;
 
-        gf_uuid_clear (volumeid);
-        if (gf_nfs_dvm_off (nfs_state (ms->nfsx)))
-                goto no_dvm;
+    ret = snprintf(searchstr, 1024, "nfs3.%s.volume-id", xlator->name);
+    if (ret < 0) {
+        gf_msg(GF_MNT, GF_LOG_ERROR, ret, NFS_MSG_SNPRINTF_FAIL,
+               "snprintf failed");
+        ret = -1;
+        goto err;
+    }
 
-        ret = snprintf (searchstr, 1024, "nfs3.%s.volume-id", xlator->name);
+    if (dict_get(opts, searchstr)) {
+        ret = dict_get_str(opts, searchstr, &optstr);
         if (ret < 0) {
-                gf_log (GF_MNT, GF_LOG_ERROR, "snprintf failed");
-                ret = -1;
-                goto err;
-        }
-
-        if (dict_get (opts, searchstr)) {
-                ret = dict_get_str (opts, searchstr, &optstr);
-                if (ret < 0) {
-                        gf_log (GF_MNT, GF_LOG_ERROR, "Failed to read option"
-                                ": %s", searchstr);
-                        ret = -1;
-                        goto err;
-                }
-        } else {
-                gf_log (GF_MNT, GF_LOG_ERROR, "DVM is on but volume-id not "
-                        "given for volume: %s", xlator->name);
-                ret = -1;
-                goto err;
-        }
-
-        if (optstr) {
-                ret = gf_uuid_parse (optstr, volumeid);
-                if (ret < 0) {
-                        gf_log (GF_MNT, GF_LOG_ERROR, "Failed to parse volume "
-                                "UUID");
-                        ret = -1;
-                        goto err;
-                }
+            gf_msg(GF_MNT, GF_LOG_ERROR, ret, NFS_MSG_DICT_GET_FAILED,
+                   "Failed to read "
+                   "option: %s",
+                   searchstr);
+            ret = -1;
+            goto err;
+        }
+    } else {
+        gf_msg(GF_MNT, GF_LOG_ERROR, 0, NFS_MSG_VOLID_MISSING,
+               "DVM is on but volume-id not "
+               "given for volume: %s",
+               xlator->name);
+        ret = -1;
+        goto err;
+    }
+
+    if (optstr) {
+        ret = gf_uuid_parse(optstr, volumeid);
+        if (ret < 0) {
+            gf_msg(GF_MNT, GF_LOG_ERROR, ret, NFS_MSG_PARSE_VOL_UUID_FAIL,
+                   "Failed to parse "
+                   "volume UUID");
+            ret = -1;
+            goto err;
         }
+    }
 
 no_dvm:
-        ret = snprintf (searchstr, 1024, "nfs3.%s.export-dir", xlator->name);
+    ret = snprintf(searchstr, 1024, "nfs3.%s.export-dir", xlator->name);
+    if (ret < 0) {
+        gf_msg(GF_MNT, GF_LOG_ERROR, ret, NFS_MSG_SNPRINTF_FAIL,
+               "snprintf failed");
+        ret = -1;
+        goto err;
+    }
+
+    if (dict_get(opts, searchstr)) {
+        ret = dict_get_str(opts, searchstr, &optstr);
         if (ret < 0) {
-                gf_log (GF_MNT, GF_LOG_ERROR, "snprintf failed");
-                ret = -1;
-                goto err;
+            gf_msg(GF_MNT, GF_LOG_ERROR, ret, NFS_MSG_DICT_GET_FAILED,
+                   "Failed to read "
+                   "option: %s",
+                   searchstr);
+            ret = -1;
+            goto err;
         }
 
-        if (dict_get (opts, searchstr)) {
-                ret = dict_get_str (opts, searchstr, &optstr);
-                if (ret < 0) {
-                        gf_log (GF_MNT, GF_LOG_ERROR, "Failed to read option: "
-                                "%s", searchstr);
-                        ret = -1;
-                        goto err;
-                }
-
-                ret = __mnt3_init_volume_direxports (ms, xlator, optstr,
-                                                     volumeid);
-                if (ret == -1) {
-                        gf_log (GF_MNT, GF_LOG_ERROR, "Dir export setup failed"
-                                " for volume: %s", xlator->name);
-                        goto err;
-                }
+        ret = __mnt3_init_volume_direxports(ms, xlator, optstr, volumeid);
+        if (ret == -1) {
+            gf_msg(GF_MNT, GF_LOG_ERROR, 0, NFS_MSG_DIR_EXP_SETUP_FAIL,
+                   "Dir export "
+                   "setup failed for volume: %s",
+                   xlator->name);
+            goto err;
         }
+    }
 
-        if (ms->export_volumes) {
-                newexp = mnt3_init_export_ent (ms, xlator, NULL, volumeid);
-                if (!newexp) {
-                        ret = -1;
-                        goto err;
-                }
-
-                list_add_tail (&newexp->explist, &ms->exportlist);
+    if (ms->export_volumes) {
+        newexp = mnt3_init_export_ent(ms, xlator, NULL, volumeid);
+        if (!newexp) {
+            ret = -1;
+            goto err;
         }
-        ret = 0;
 
+        list_add_tail(&newexp->explist, &ms->exportlist);
+    }
+    ret = 0;
 
 err:
-        return ret;
+    return ret;
 }
 
-
 int
-__mnt3_init_volume_export (struct mount3_state *ms, dict_t *opts)
+__mnt3_init_volume_export(struct mount3_state *ms, dict_t *opts)
 {
-        int                     ret = -1;
-        char                    *optstr  = NULL;
-        /* On by default. */
-        gf_boolean_t            boolt = _gf_true;
-
-        if ((!ms) || (!opts))
-                return -1;
-
-        if (!dict_get (opts, "nfs3.export-volumes")) {
-                ret = 0;
-                goto err;
-        }
+    int ret = -1;
+    char *optstr = NULL;
+    /* On by default. */
+    gf_boolean_t boolt = _gf_true;
 
-        ret = dict_get_str (opts, "nfs3.export-volumes", &optstr);
-        if (ret < 0) {
-                gf_log (GF_MNT, GF_LOG_ERROR, "Failed to read option: "
-                        "nfs3.export-volumes");
-                ret = -1;
-                goto err;
-        }
+    if ((!ms) || (!opts))
+        return -1;
 
-        ret = gf_string2boolean (optstr, &boolt);
-        if (ret < 0) {
-                gf_log (GF_MNT, GF_LOG_ERROR, "Failed to convert"
-                        " string to boolean");
-        }
+    if (!dict_get(opts, "nfs3.export-volumes")) {
+        ret = 0;
+        goto err;
+    }
+
+    ret = dict_get_str(opts, "nfs3.export-volumes", &optstr);
+    if (ret < 0) {
+        gf_msg(GF_MNT, GF_LOG_ERROR, ret, NFS_MSG_DICT_GET_FAILED,
+               "Failed to read option: nfs3.export-volumes");
+        ret = -1;
+        goto err;
+    }
+
+    ret = gf_string2boolean(optstr, &boolt);
+    if (ret < 0) {
+        gf_msg(GF_MNT, GF_LOG_ERROR, ret, NFS_MSG_STR2BOOL_FAIL,
+               "Failed to convert string to boolean");
+    }
 
 err:
-        if (boolt == _gf_false) {
-                gf_log (GF_MNT, GF_LOG_TRACE, "Volume exports disabled");
-                ms->export_volumes = 0;
-        } else {
-                gf_log (GF_MNT, GF_LOG_TRACE, "Volume exports enabled");
-                ms->export_volumes = 1;
-        }
-
-        return ret;
+    if (boolt == _gf_false) {
+        gf_msg_trace(GF_MNT, 0, "Volume exports disabled");
+        ms->export_volumes = 0;
+    } else {
+        gf_msg_trace(GF_MNT, 0, "Volume exports enabled");
+        ms->export_volumes = 1;
+    }
+
+    return ret;
 }
 
-
 int
-__mnt3_init_dir_export (struct mount3_state *ms, dict_t *opts)
+__mnt3_init_dir_export(struct mount3_state *ms, dict_t *opts)
 {
-        int                     ret = -1;
-        char                    *optstr  = NULL;
-        /* On by default. */
-        gf_boolean_t            boolt = _gf_true;
-
-        if ((!ms) || (!opts))
-                return -1;
+    int ret = -1;
+    char *optstr = NULL;
+    /* On by default. */
+    gf_boolean_t boolt = _gf_true;
 
-        if (!dict_get (opts, "nfs3.export-dirs")) {
-                ret = 0;
-                goto err;
-        }
-
-        ret = dict_get_str (opts, "nfs3.export-dirs", &optstr);
-        if (ret < 0) {
-                gf_log (GF_MNT, GF_LOG_ERROR, "Failed to read option: "
-                        "nfs3.export-dirs");
-                ret = -1;
-                goto err;
-        }
+    if ((!ms) || (!opts))
+        return -1;
 
-        ret = gf_string2boolean (optstr, &boolt);
-        if (ret < 0) {
-                gf_log (GF_MNT, GF_LOG_ERROR, "Failed to convert"
-                        " string to boolean");
-         }
+    if (!dict_get(opts, "nfs3.export-dirs")) {
+        ret = 0;
+        goto err;
+    }
+
+    ret = dict_get_str(opts, "nfs3.export-dirs", &optstr);
+    if (ret < 0) {
+        gf_msg(GF_MNT, GF_LOG_ERROR, ret, NFS_MSG_DICT_GET_FAILED,
+               "Failed to read option: nfs3.export-dirs");
+        ret = -1;
+        goto err;
+    }
+
+    ret = gf_string2boolean(optstr, &boolt);
+    if (ret < 0) {
+        gf_msg(GF_MNT, GF_LOG_ERROR, ret, NFS_MSG_STR2BOOL_FAIL,
+               "Failed to convert string to boolean");
+    }
 
 err:
-        if (boolt == _gf_false) {
-                gf_log (GF_MNT, GF_LOG_TRACE, "Dir exports disabled");
-                ms->export_dirs = 0;
-        } else {
-                gf_log (GF_MNT, GF_LOG_TRACE, "Dir exports enabled");
-                ms->export_dirs = 1;
-        }
-
-        return ret;
+    if (boolt == _gf_false) {
+        gf_msg_trace(GF_MNT, 0, "Dir exports disabled");
+        ms->export_dirs = 0;
+    } else {
+        gf_msg_trace(GF_MNT, 0, "Dir exports enabled");
+        ms->export_dirs = 1;
+    }
+
+    return ret;
 }
 
-
 int
-mnt3_init_options (struct mount3_state *ms, dict_t *options)
-{
-        xlator_list_t   *volentry = NULL;
-        int             ret = -1;
-
-        if ((!ms) || (!options))
-                return -1;
-
-        __mnt3_init_volume_export (ms, options);
-        __mnt3_init_dir_export (ms, options);
-        volentry = ms->nfsx->children;
-        while (volentry) {
-                gf_log (GF_MNT, GF_LOG_TRACE, "Initing options for: %s",
-                        volentry->xlator->name);
-                ret = __mnt3_init_volume (ms, options, volentry->xlator);
-                if (ret < 0) {
-                        gf_log (GF_MNT, GF_LOG_ERROR, "Volume init failed");
-                        goto err;
-                }
+mnt3_init_options(struct mount3_state *ms, dict_t *options)
+{
+    xlator_list_t *volentry = NULL;
+    int ret = -1;
 
-                volentry = volentry->next;
+    if ((!ms) || (!options))
+        return -1;
+
+    __mnt3_init_volume_export(ms, options);
+    __mnt3_init_dir_export(ms, options);
+    volentry = ms->nfsx->children;
+    while (volentry) {
+        gf_msg_trace(GF_MNT, 0, "Initing options for: %s",
+                     volentry->xlator->name);
+        ret = __mnt3_init_volume(ms, options, volentry->xlator);
+        if (ret < 0) {
+            gf_msg(GF_MNT, GF_LOG_ERROR, ret, NFS_MSG_VOL_INIT_FAIL,
+                   "Volume init failed");
+            goto err;
         }
 
+        volentry = volentry->next;
+    }
 
-        ret = 0;
+    ret = 0;
 err:
-        return ret;
+    return ret;
 }
 
 struct mount3_state *
-mnt3_init_state (xlator_t *nfsx)
+mnt3_init_state(xlator_t *nfsx)
 {
-        struct mount3_state     *ms = NULL;
-        int                     ret = -1;
-
-        if (!nfsx)
-                return NULL;
+    struct mount3_state *ms = NULL;
+    int ret = -1;
 
-        ms = GF_CALLOC (1, sizeof (*ms), gf_nfs_mt_mount3_state);
-        if (!ms) {
-                gf_log (GF_MNT, GF_LOG_ERROR, "Memory allocation failed");
-                return NULL;
-        }
+    if (!nfsx)
+        return NULL;
 
-        ms->iobpool = nfsx->ctx->iobuf_pool;
-        ms->nfsx = nfsx;
-        INIT_LIST_HEAD (&ms->exportlist);
-        ret = mnt3_init_options (ms, nfsx->options);
-        if (ret < 0) {
-                gf_log (GF_MNT, GF_LOG_ERROR, "Options init failed");
-                return NULL;
-        }
+    ms = GF_CALLOC(1, sizeof(*ms), gf_nfs_mt_mount3_state);
+    if (!ms) {
+        gf_msg(GF_MNT, GF_LOG_ERROR, ENOMEM, NFS_MSG_NO_MEMORY,
+               "Memory allocation failed");
+        return NULL;
+    }
+
+    ms->iobpool = nfsx->ctx->iobuf_pool;
+    ms->nfsx = nfsx;
+    INIT_LIST_HEAD(&ms->exportlist);
+    ret = mnt3_init_options(ms, nfsx->options);
+    if (ret < 0) {
+        gf_msg(GF_MNT, GF_LOG_ERROR, ret, NFS_MSG_OPT_INIT_FAIL,
+               "Options init failed");
+        return NULL;
+    }
 
-        INIT_LIST_HEAD (&ms->mountlist);
-        LOCK_INIT (&ms->mountlock);
+    INIT_LIST_HEAD(&ms->mountlist);
+    LOCK_INIT(&ms->mountlock);
 
-        return ms;
+    return ms;
 }
 
 int
-mount_init_state (xlator_t *nfsx)
+mount_init_state(xlator_t *nfsx)
 {
-        int              ret = -1;
-        struct nfs_state *nfs = NULL;
-
-        if (!nfsx)
-                goto out;
-
-        nfs = (struct nfs_state *)nfs_state (nfsx);
-        /*Maintaining global state for MOUNT1 and MOUNT3*/
-        nfs->mstate =  mnt3_init_state (nfsx);
-        if (!nfs->mstate) {
-                gf_log (GF_NFS, GF_LOG_ERROR, "Failed to allocate"
-                        "mount state");
-                goto out;
-        }
-        ret = 0;
+    int ret = -1;
+    struct nfs_state *nfs = NULL;
+
+    if (!nfsx)
+        goto out;
+
+    nfs = (struct nfs_state *)nfs_state(nfsx);
+    /*Maintaining global state for MOUNT1 and MOUNT3*/
+    nfs->mstate = mnt3_init_state(nfsx);
+    if (!nfs->mstate) {
+        gf_msg(GF_NFS, GF_LOG_ERROR, ENOMEM, NFS_MSG_NO_MEMORY,
+               "Failed to allocate mount state");
+        goto out;
+    }
+    ret = 0;
 out:
-        return ret;
+    return ret;
 }
 
-rpcsvc_actor_t  mnt3svc_actors[MOUNT3_PROC_COUNT] = {
-        {"NULL",    MOUNT3_NULL,    mnt3svc_null,    NULL, 0, DRC_NA},
-        {"MNT",     MOUNT3_MNT,     mnt3svc_mnt,     NULL, 0, DRC_NA},
-        {"DUMP",    MOUNT3_DUMP,    mnt3svc_dump,    NULL, 0, DRC_NA},
-        {"UMNT",    MOUNT3_UMNT,    mnt3svc_umnt,    NULL, 0, DRC_NA},
-        {"UMNTALL", MOUNT3_UMNTALL, mnt3svc_umntall, NULL, 0, DRC_NA},
-        {"EXPORT",  MOUNT3_EXPORT,  mnt3svc_export,  NULL, 0, DRC_NA}
-};
-
-
+static rpcsvc_actor_t mnt3svc_actors[MOUNT3_PROC_COUNT] = {
+    {
+        "NULL",
+        mnt3svc_null,
+        NULL,
+        MOUNT3_NULL,
+        DRC_NA,
+    },
+    {"MNT", mnt3svc_mnt, NULL, MOUNT3_MNT, DRC_NA, 0},
+    {"DUMP", mnt3svc_dump, NULL, MOUNT3_DUMP, DRC_NA, 0},
+    {"UMNT", mnt3svc_umnt, NULL, MOUNT3_UMNT, DRC_NA, 0},
+    {"UMNTALL", mnt3svc_umntall, NULL, MOUNT3_UMNTALL, DRC_NA, 0},
+    {"EXPORT", mnt3svc_export, NULL, MOUNT3_EXPORT, DRC_NA, 0}};
 
 /* Static init parts are assigned here, dynamic ones are done in
  * mnt3svc_init and mnt3_init_state.
  * Making MOUNT3 a synctask so that the blocking DNS calls during rpc auth
  * gets offloaded to syncenv, keeping the main/poll thread unblocked
  */
-rpcsvc_program_t        mnt3prog = {
-                        .progname       = "MOUNT3",
-                        .prognum        = MOUNT_PROGRAM,
-                        .progver        = MOUNT_V3,
-                        .progport       = GF_MOUNTV3_PORT,
-                        .actors         = mnt3svc_actors,
-                        .numactors      = MOUNT3_PROC_COUNT,
-                        .min_auth       = AUTH_NULL,
-                        .synctask       = _gf_true,
+static rpcsvc_program_t mnt3prog = {
+    .progname = "MOUNT3",
+    .prognum = MOUNT_PROGRAM,
+    .progver = MOUNT_V3,
+    .progport = GF_MOUNTV3_PORT,
+    .actors = mnt3svc_actors,
+    .numactors = MOUNT3_PROC_COUNT,
+    .min_auth = AUTH_NULL,
+    .synctask = _gf_true,
 };
 
 /**
@@ -3498,47 +3697,82 @@ rpcsvc_program_t        mnt3prog = {
  *
  */
 int
-__mnt3_mounted_exports_walk (dict_t *dict, char *key, data_t *val, void *tmp)
+__mnt3_mounted_exports_walk(dict_t *dict, char *key, data_t *val, void *tmp)
 {
-        char                     *path             = NULL;
-        char                     *host_addr_ip     = NULL;
-        char                     *keydup           = NULL;
-        char                     *colon            = NULL;
-        struct mnt3_auth_params  *auth_params      = NULL;
-        int                       auth_status_code = 0;
-
-        gf_log (GF_MNT, GF_LOG_TRACE, "Checking if key %s is authorized.", key);
-
-        auth_params = (struct mnt3_auth_params *)tmp;
-
-        /* Since we haven't obtained a lock around the mount dict
-         * here, we want to duplicate the key and then process it.
-         * Otherwise we would potentially have a race condition
-         * by modifying the key in the dict when other threads
-         * are accessing it.
-         */
-        keydup = strdupa (key);
-
-        colon = strchr (keydup, ':');
-        if (!colon)
-                return 0;
-
-        *colon = '\0';
-
-        path = alloca (strlen (keydup) + 2);
-        snprintf (path, strlen (keydup) + 2, "/%s", keydup);
-
-        /* Host is one character after ':' */
-        host_addr_ip = colon + 1;
-        auth_status_code = mnt3_auth_host (auth_params, host_addr_ip, NULL,
-                                           path, _gf_false, NULL);
-        if (auth_status_code != 0) {
-                gf_log (GF_MNT, GF_LOG_ERROR,
-                                "%s is no longer authorized for %s",
-                                host_addr_ip, path);
-                mnt3svc_umount (auth_params->ms, path, host_addr_ip);
-        }
+    char *path = NULL;
+    char *host_addr_ip = NULL;
+    char *host_addr_fqdn = NULL;
+    char *keydup = NULL;
+    char *colon = NULL;
+    struct mnt3_auth_params *auth_params = NULL;
+    int ret = 0;
+    int auth_status_code = 0;
+
+    gf_msg_trace(GF_MNT, 0, "Checking if key %s is authorized.", key);
+
+    auth_params = (struct mnt3_auth_params *)tmp;
+
+    /* Since we haven't obtained a lock around the mount dict
+     * here, we want to duplicate the key and then process it.
+     * Otherwise we would potentially have a race condition
+     * by modifying the key in the dict when other threads
+     * are accessing it.
+     */
+    keydup = strdupa(key);
+
+    colon = strchr(keydup, ':');
+    if (!colon)
         return 0;
+
+    *colon = '\0';
+
+    path = alloca(strlen(keydup) + 2);
+    snprintf(path, strlen(keydup) + 2, "/%s", keydup);
+
+    /* Host is one character after ':' */
+    host_addr_ip = colon + 1;
+
+    /* Check if the IP is authorized */
+    auth_status_code = mnt3_auth_host(auth_params, host_addr_ip, NULL, path,
+                                      FALSE, NULL);
+    if (auth_status_code == 0) {
+        goto out;
+    }
+
+    ret = gf_get_hostname_from_ip(host_addr_ip, &host_addr_fqdn);
+    if (ret != 0) {
+        gf_msg(GF_MNT, GF_LOG_DEBUG, 0, NFS_MSG_AUTH_ERROR,
+               "Authorization failed for IP [%s], but name "
+               "resolution also failed!",
+               host_addr_ip);
+        goto unmount;
+    }
+
+    /* If not, check if the FQDN is authorized */
+    gf_msg(GF_MNT, GF_LOG_DEBUG, 0, NFS_MSG_AUTH_ERROR,
+           "Authorization failed for IP [%s], attempting to"
+           " auth hostname [%s]...",
+           host_addr_ip, host_addr_fqdn);
+
+    auth_status_code = mnt3_auth_host(auth_params, host_addr_fqdn, NULL, path,
+                                      FALSE, NULL);
+    if (auth_status_code == 0) {
+        gf_msg(GF_MNT, GF_LOG_DEBUG, 0, NFS_MSG_AUTH_ERROR,
+               "Authorization succeeded for "
+               "Client [IP=%s, Hostname=%s].",
+               host_addr_ip, host_addr_fqdn);
+        goto out;
+    }
+
+unmount:
+    gf_msg(GF_MNT, GF_LOG_ERROR, 0, NFS_MSG_AUTH_ERROR,
+           "Client [IP=%s, Hostname=%s] not authorized for this mount. "
+           "Unmounting!",
+           host_addr_ip, host_addr_fqdn);
+    mnt3svc_umount(auth_params->ms, path, host_addr_ip);
+out:
+    GF_FREE(host_addr_fqdn);
+    return 0;
 }
 
 /**
@@ -3550,14 +3784,12 @@ __mnt3_mounted_exports_walk (dict_t *dict, char *key, data_t *val, void *tmp)
  *
  */
 void
-_mnt3_invalidate_old_mounts (struct mount3_state *ms)
+_mnt3_invalidate_old_mounts(struct mount3_state *ms)
 {
-        gf_log (GF_MNT, GF_LOG_DEBUG, "Invalidating old mounts ...");
-        dict_foreach (ms->mountdict, __mnt3_mounted_exports_walk,
-                      ms->auth_params);
+    gf_msg_debug(GF_MNT, 0, "Invalidating old mounts ...");
+    dict_foreach(ms->mountdict, __mnt3_mounted_exports_walk, ms->auth_params);
 }
 
-
 /**
  * _mnt3_has_file_changed -- Checks if a file has changed on disk
  *
@@ -3570,25 +3802,25 @@ _mnt3_invalidate_old_mounts (struct mount3_state *ms)
  * Uses get_file_mtime () in common-utils.c
  */
 gf_boolean_t
-_mnt3_has_file_changed (const char *path, time_t *oldmtime)
+_mnt3_has_file_changed(const char *path, time_t *oldmtime)
 {
-        gf_boolean_t    changed = _gf_false;
-        time_t          mtime   = {0};
-        int             ret     = 0;
+    gf_boolean_t changed = _gf_false;
+    time_t mtime = {0};
+    int ret = 0;
 
-        GF_VALIDATE_OR_GOTO (GF_MNT, path, out);
-        GF_VALIDATE_OR_GOTO (GF_MNT, oldmtime, out);
+    GF_VALIDATE_OR_GOTO(GF_MNT, path, out);
+    GF_VALIDATE_OR_GOTO(GF_MNT, oldmtime, out);
 
-        ret = get_file_mtime (path, &mtime);
-        if (ret < 0)
-                goto out;
+    ret = get_file_mtime(path, &mtime);
+    if (ret < 0)
+        goto out;
 
-        if (mtime != *oldmtime) {
-                changed = _gf_true;
-                *oldmtime = mtime;
-        }
+    if (mtime != *oldmtime) {
+        changed = _gf_true;
+        *oldmtime = mtime;
+    }
 out:
-        return changed;
+    return changed;
 }
 
 /**
@@ -3601,81 +3833,82 @@ out:
  * @return: Always returns NULL
  */
 void *
-_mnt3_auth_param_refresh_thread (void *argv)
-{
-        struct          mount3_state *mstate    = (struct mount3_state *)argv;
-        char            *exp_file_path          = NULL;
-        char            *ng_file_path           = NULL;
-        size_t          nbytes                  = 0;
-        time_t          exp_time                = 0;
-        time_t          ng_time                 = 0;
-        gf_boolean_t    any_file_changed        = _gf_false;
-        int             ret                     = 0;
-
-        nbytes = strlen (exports_file_path) + 1;
-        exp_file_path = alloca (nbytes);
-        snprintf (exp_file_path, nbytes, "%s", exports_file_path);
-
-        nbytes = strlen (netgroups_file_path) + 1;
-        ng_file_path = alloca (nbytes);
-        snprintf (ng_file_path, nbytes, "%s", netgroups_file_path);
-
-        /* Set the initial timestamps to avoid reloading right after
-         * mnt3svc_init () spawns this thread */
-        get_file_mtime (exp_file_path, &exp_time);
-        get_file_mtime (ng_file_path, &ng_time);
-
-        while (_gf_true) {
-                if (mstate->stop_refresh)
-                        break;
-                any_file_changed = _gf_false;
-
-                /* Sleep before checking the file again */
-                sleep (mstate->nfs->auth_refresh_time_secs);
-
-                if (_mnt3_has_file_changed (exp_file_path, &exp_time)) {
-                        gf_log (GF_MNT, GF_LOG_INFO, "File %s changed, "
-                                                     "updating exports,",
-                                exp_file_path);
-
-                        ret = mnt3_auth_set_exports_auth (mstate->auth_params,
-                                                          exp_file_path);
-                        if (ret)
-                                gf_log (GF_MNT, GF_LOG_ERROR,
-                                        "Failed to set export auth params.");
-                        else
-                                any_file_changed = _gf_true;
-                }
-
-                if (_mnt3_has_file_changed (ng_file_path, &ng_time)) {
-                        gf_log (GF_MNT, GF_LOG_INFO, "File %s changed,"
-                                                     "updating netgroups",
-                                ng_file_path);
-
-                        ret = mnt3_auth_set_netgroups_auth (mstate->auth_params,
-                                                            ng_file_path);
-                        if (ret)
-                                gf_log (GF_MNT, GF_LOG_ERROR,
-                                        "Failed to set netgroup auth params.");
-                        else
-                                any_file_changed = _gf_true;
-                }
-
-                /* If no files changed, go back to sleep */
-                if (!any_file_changed)
-                        continue;
-
-                gf_log (GF_MNT, GF_LOG_INFO, "Purging auth cache.");
-                auth_cache_purge (mstate->authcache);
-
-                /* Walk through mounts that are no longer authorized
-                 * and unmount them on the server side. This will
-                 * cause subsequent file ops to fail with access denied.
-                 */
-                _mnt3_invalidate_old_mounts (mstate);
-        }
+_mnt3_auth_param_refresh_thread(void *argv)
+{
+    struct mount3_state *mstate = (struct mount3_state *)argv;
+    char *exp_file_path = NULL;
+    char *ng_file_path = NULL;
+    size_t nbytes = 0;
+    time_t exp_time = 0;
+    time_t ng_time = 0;
+    gf_boolean_t any_file_changed = _gf_false;
+    int ret = 0;
+
+    nbytes = strlen(exports_file_path) + 1;
+    exp_file_path = alloca(nbytes);
+    snprintf(exp_file_path, nbytes, "%s", exports_file_path);
+
+    nbytes = strlen(netgroups_file_path) + 1;
+    ng_file_path = alloca(nbytes);
+    snprintf(ng_file_path, nbytes, "%s", netgroups_file_path);
+
+    /* Set the initial timestamps to avoid reloading right after
+     * mnt3svc_init () spawns this thread */
+    get_file_mtime(exp_file_path, &exp_time);
+    get_file_mtime(ng_file_path, &ng_time);
+
+    while (_gf_true) {
+        if (mstate->stop_refresh)
+            break;
+        any_file_changed = _gf_false;
+
+        /* Sleep before checking the file again */
+        sleep(mstate->nfs->auth_refresh_time_secs);
+
+        if (_mnt3_has_file_changed(exp_file_path, &exp_time)) {
+            gf_msg(GF_MNT, GF_LOG_INFO, 0, NFS_MSG_UPDATING_EXP,
+                   "File %s changed, updating exports,", exp_file_path);
+
+            ret = mnt3_auth_set_exports_auth(mstate->auth_params,
+                                             exp_file_path);
+            if (ret)
+                gf_msg(GF_MNT, GF_LOG_ERROR, 0, NFS_MSG_SET_EXP_AUTH_PARAM_FAIL,
+                       "Failed to set export auth params.");
+            else
+                any_file_changed = _gf_true;
+        }
+
+        if (_mnt3_has_file_changed(ng_file_path, &ng_time)) {
+            gf_msg(GF_MNT, GF_LOG_INFO, 0, NFS_MSG_UPDATING_NET_GRP,
+                   "File %s changed,"
+                   "updating netgroups",
+                   ng_file_path);
+
+            ret = mnt3_auth_set_netgroups_auth(mstate->auth_params,
+                                               ng_file_path);
+            if (ret)
+                gf_msg(GF_MNT, GF_LOG_ERROR, 0, NFS_MSG_SET_NET_GRP_FAIL,
+                       "Failed to set netgroup auth params.");
+            else
+                any_file_changed = _gf_true;
+        }
+
+        /* If no files changed, go back to sleep */
+        if (!any_file_changed)
+            continue;
+
+        gf_msg(GF_MNT, GF_LOG_INFO, 0, NFS_MSG_PURGING_AUTH_CACHE,
+               "Purging auth cache.");
+        auth_cache_purge(mstate->authcache);
+
+        /* Walk through mounts that are no longer authorized
+         * and unmount them on the server side. This will
+         * cause subsequent file ops to fail with access denied.
+         */
+        _mnt3_invalidate_old_mounts(mstate);
+    }
 
-        return NULL;
+    return NULL;
 }
 
 /**
@@ -3691,51 +3924,50 @@ _mnt3_auth_param_refresh_thread (void *argv)
  *                    bubbled up from the functions we call to set the params.
  */
 int
-_mnt3_init_auth_params (struct mount3_state *mstate)
+_mnt3_init_auth_params(struct mount3_state *mstate)
 {
-        int      ret            = -EINVAL;
-        char     *exp_file_path = NULL;
-        char     *ng_file_path  = NULL;
-        size_t   nbytes         = 0;
-
-        GF_VALIDATE_OR_GOTO (GF_MNT, mstate, out);
-
-        mstate->auth_params = mnt3_auth_params_init (mstate);
-        if (!mstate->auth_params) {
-                gf_log (GF_MNT, GF_LOG_ERROR,
-                        "Failed to init mount auth params.");
-                ret = -ENOMEM;
-                goto out;
-        }
-
-        nbytes = strlen (exports_file_path) + 1;
-        exp_file_path = alloca (nbytes);
-        snprintf (exp_file_path, nbytes, "%s", exports_file_path);
-
-        ret = mnt3_auth_set_exports_auth (mstate->auth_params, exp_file_path);
-        if (ret < 0) {
-                gf_log (GF_MNT, GF_LOG_ERROR,
-                        "Failed to set export auth params.");
-                goto out;
-        }
-
-        nbytes = strlen (netgroups_file_path) + 1;
-        ng_file_path = alloca (nbytes);
-        snprintf (ng_file_path, nbytes, "%s", netgroups_file_path);
-
-        ret = mnt3_auth_set_netgroups_auth (mstate->auth_params, ng_file_path);
-        if (ret < 0) {
-                gf_log (GF_MNT, GF_LOG_ERROR,
-                        "Failed to set netgroup auth params.");
-                goto out;
-        }
-
-        ret = 0;
+    int ret = -EINVAL;
+    char *exp_file_path = NULL;
+    char *ng_file_path = NULL;
+    size_t nbytes = 0;
+
+    GF_VALIDATE_OR_GOTO(GF_MNT, mstate, out);
+
+    mstate->auth_params = mnt3_auth_params_init(mstate);
+    if (!mstate->auth_params) {
+        gf_msg(GF_MNT, GF_LOG_ERROR, ENOMEM, NFS_MSG_NO_MEMORY,
+               "Failed to init mount auth params.");
+        ret = -ENOMEM;
+        goto out;
+    }
+
+    nbytes = strlen(exports_file_path) + 1;
+    exp_file_path = alloca(nbytes);
+    snprintf(exp_file_path, nbytes, "%s", exports_file_path);
+
+    ret = mnt3_auth_set_exports_auth(mstate->auth_params, exp_file_path);
+    if (ret < 0) {
+        gf_msg(GF_MNT, GF_LOG_ERROR, ret, NFS_MSG_SET_EXP_AUTH_PARAM_FAIL,
+               "Failed to set export auth params.");
+        goto out;
+    }
+
+    nbytes = strlen(netgroups_file_path) + 1;
+    ng_file_path = alloca(nbytes);
+    snprintf(ng_file_path, nbytes, "%s", netgroups_file_path);
+
+    ret = mnt3_auth_set_netgroups_auth(mstate->auth_params, ng_file_path);
+    if (ret < 0) {
+        gf_msg(GF_MNT, GF_LOG_ERROR, ret, NFS_MSG_SET_EXP_AUTH_PARAM_FAIL,
+               "Failed to set netgroup auth params.");
+        goto out;
+    }
+
+    ret = 0;
 out:
-        return ret;
+    return ret;
 }
 
-
 /**
  * mnt3svc_deinit -- Function called by the nfs translator to cleanup all state
  *
@@ -3744,247 +3976,292 @@ out:
  *         as well as the threads that have been started.
  */
 void
-mnt3svc_deinit (xlator_t *nfsx)
+mnt3svc_deinit(xlator_t *nfsx)
 {
-        struct mount3_state *mstate = NULL;
-        struct nfs_state    *nfs    = NULL;
-
-        if (!nfsx || !nfsx->private)
-                return;
-
-        nfs = (struct nfs_state *)nfsx->private;
-        mstate = (struct mount3_state *)nfs->mstate;
-
-        if (nfs->refresh_auth) {
-                /* Mark as true and wait for thread to exit */
-                mstate->stop_refresh = _gf_true;
-                pthread_join (mstate->auth_refresh_thread, NULL);
-        }
+    struct mount3_state *mstate = NULL;
+    struct nfs_state *nfs = NULL;
 
-        if (nfs->exports_auth)
-                mnt3_auth_params_deinit (mstate->auth_params);
+    if (!nfsx || !nfsx->private)
+        return;
 
-        /* Unmount everything and clear mountdict */
-        mnt3svc_umountall (mstate);
+    nfs = (struct nfs_state *)nfsx->private;
+    mstate = (struct mount3_state *)nfs->mstate;
+
+    if (nfs->refresh_auth) {
+        /* Mark as true and wait for thread to exit */
+        mstate->stop_refresh = _gf_true;
+        pthread_join(mstate->auth_refresh_thread, NULL);
+    }
+
+    if (nfs->exports_auth)
+        mnt3_auth_params_deinit(mstate->auth_params);
+
+    /* Unmount everything and clear mountdict */
+    LOCK(&mstate->mountlock);
+    {
+        __mnt3svc_umountall(mstate);
+        dict_unref(mstate->mountdict);
+    }
+    UNLOCK(&mstate->mountlock);
 }
 
 rpcsvc_program_t *
-mnt3svc_init (xlator_t *nfsx)
+mnt3svc_init(xlator_t *nfsx)
 {
-        struct mount3_state     *mstate = NULL;
-        struct nfs_state        *nfs = NULL;
-        dict_t                  *options = NULL;
-        char                    *portstr = NULL;
-        int                      ret = -1;
-        pthread_t                udp_thread;
-
-        if (!nfsx || !nfsx->private)
-                return NULL;
-
-        nfs = (struct nfs_state *)nfsx->private;
-
-        gf_log (GF_MNT, GF_LOG_DEBUG, "Initing Mount v3 state");
-        mstate = (struct mount3_state *)nfs->mstate;
-        if (!mstate) {
-                gf_log (GF_MNT, GF_LOG_ERROR, "Mount v3 state init failed");
-                goto err;
-        }
-
-        mstate->nfs = nfs;
-
-        mstate->mountdict = dict_new ();
-        if (!mstate->mountdict) {
-                gf_log (GF_MNT, GF_LOG_ERROR,
-                        "Failed to setup mount dict. Allocation error.");
-                goto err;
-        }
-
-        if (nfs->exports_auth) {
-                ret = _mnt3_init_auth_params (mstate);
-                if (ret < 0)
-                        goto err;
+    struct mount3_state *mstate = NULL;
+    struct nfs_state *nfs = NULL;
+    dict_t *options = NULL;
+    char *portstr = NULL;
+    int ret = -1;
+    pthread_t udp_thread;
+
+    if (!nfsx || !nfsx->private)
+        return NULL;
 
-                mstate->authcache = auth_cache_init (nfs->auth_cache_ttl_sec);
-                if (!mstate->authcache) {
-                        ret = -ENOMEM;
-                        goto err;
-                }
+    nfs = (struct nfs_state *)nfsx->private;
 
-                mstate->stop_refresh = _gf_false; /* Allow thread to run */
-                pthread_create (&mstate->auth_refresh_thread, NULL,
-                                _mnt3_auth_param_refresh_thread, mstate);
-        } else
-                gf_log (GF_MNT, GF_LOG_WARNING, "Exports auth has been "
-                                                "disabled!");
+    gf_msg_debug(GF_MNT, 0, "Initing Mount v3 state");
+    mstate = (struct mount3_state *)nfs->mstate;
+    if (!mstate) {
+        gf_msg(GF_MNT, GF_LOG_ERROR, 0, NFS_MSG_MNT_STATE_INIT_FAIL,
+               "Mount v3 state init failed");
+        goto err;
+    }
 
-        mnt3prog.private = mstate;
-        options = dict_new ();
+    mstate->nfs = nfs;
 
-        ret = gf_asprintf (&portstr, "%d", GF_MOUNTV3_PORT);
-        if (ret == -1)
-                goto err;
+    mstate->mountdict = dict_new();
+    if (!mstate->mountdict) {
+        gf_msg(GF_MNT, GF_LOG_ERROR, ENOMEM, NFS_MSG_NO_MEMORY,
+               "Failed to setup mount dict. Allocation error.");
+        goto err;
+    }
 
-        ret = dict_set_dynstr (options, "transport.socket.listen-port",
-                               portstr);
-        if (ret == -1)
-                goto err;
+    if (nfs->exports_auth) {
+        ret = _mnt3_init_auth_params(mstate);
+        if (ret < 0)
+            goto err;
 
-        ret = dict_set_str (options, "transport-type", "socket");
-        if (ret == -1) {
-                gf_log (GF_NFS, GF_LOG_ERROR, "dict_set_str error");
-                goto err;
+        mstate->authcache = auth_cache_init(nfs->auth_cache_ttl_sec);
+        if (!mstate->authcache) {
+            ret = -ENOMEM;
+            goto err;
         }
 
-        if (nfs->allow_insecure) {
-                ret = dict_set_str (options, "rpc-auth-allow-insecure", "on");
-                if (ret == -1) {
-                        gf_log (GF_NFS, GF_LOG_ERROR, "dict_set_str error");
-                        goto err;
-                }
-                ret = dict_set_str (options, "rpc-auth.ports.insecure", "on");
-                if (ret == -1) {
-                        gf_log (GF_NFS, GF_LOG_ERROR, "dict_set_str error");
-                        goto err;
-                }
+        mstate->stop_refresh = _gf_false; /* Allow thread to run */
+        ret = gf_thread_create(&mstate->auth_refresh_thread, NULL,
+                               _mnt3_auth_param_refresh_thread, mstate,
+                               "nfsauth");
+        if (ret) {
+            gf_msg_debug(GF_MNT, GF_LOG_DEBUG, "Thread creation failed");
+        }
+
+    } else
+        gf_msg(GF_MNT, GF_LOG_INFO, 0, NFS_MSG_EXP_AUTH_DISABLED,
+               "Exports auth has been disabled!");
+
+    mnt3prog.private = mstate;
+    options = dict_new();
+    if (options == NULL) {
+        gf_msg(GF_NFS, GF_LOG_ERROR, ENOMEM, NFS_MSG_GFID_DICT_CREATE_FAIL,
+               "dict allocation failed");
+        goto err;
+    }
+
+    ret = gf_asprintf(&portstr, "%d", GF_MOUNTV3_PORT);
+    if (ret == -1)
+        goto err;
+
+    ret = dict_set_dynstr(options, "transport.socket.listen-port", portstr);
+    if (ret == -1)
+        goto err;
+
+    ret = dict_set_str(options, "transport-type", "socket");
+    if (ret == -1) {
+        gf_msg(GF_NFS, GF_LOG_ERROR, errno, NFS_MSG_DICT_SET_FAILED,
+               "dict_set_str error");
+        goto err;
+    }
+
+    if (nfs->allow_insecure) {
+        ret = dict_set_str(options, "rpc-auth-allow-insecure", "on");
+        if (ret == -1) {
+            gf_msg(GF_NFS, GF_LOG_ERROR, errno, NFS_MSG_DICT_SET_FAILED,
+                   "dict_set_str error");
+            goto err;
         }
-
-        ret= rpcsvc_create_listeners (nfs->rpcsvc, options, nfsx->name);
+        ret = dict_set_str(options, "rpc-auth.ports.insecure", "on");
         if (ret == -1) {
-                gf_log (GF_NFS, GF_LOG_ERROR, "Unable to create listeners");
-                dict_unref (options);
-                goto err;
+            gf_msg(GF_NFS, GF_LOG_ERROR, errno, NFS_MSG_DICT_SET_FAILED,
+                   "dict_set_str error");
+            goto err;
+        }
+    }
+
+    ret = rpcsvc_create_listeners(nfs->rpcsvc, options, nfsx->name);
+    if (ret == -1) {
+        gf_msg(GF_NFS, GF_LOG_ERROR, errno, NFS_MSG_LISTENERS_CREATE_FAIL,
+               "Unable to create listeners");
+        goto err;
+    }
+
+    if (nfs->mount_udp) {
+        ret = gf_thread_create(&udp_thread, NULL, mount3udp_thread, nfsx,
+                               "nfsudp");
+        if (ret) {
+            gf_msg_debug(GF_MNT, GF_LOG_DEBUG, "Thread creation failed");
         }
+    }
+    if (options)
+        dict_unref(options);
 
-        if (nfs->mount_udp) {
-                pthread_create (&udp_thread, NULL, mount3udp_thread, nfsx);
-        }
-        return &mnt3prog;
+    return &mnt3prog;
 err:
-        return NULL;
+    if (options)
+        dict_unref(options);
+    return NULL;
 }
 
-
-rpcsvc_actor_t  mnt1svc_actors[MOUNT1_PROC_COUNT] = {
-        {"NULL",    MOUNT1_NULL,    mnt3svc_null,   NULL, 0, DRC_NA},
-        {"MNT",     MOUNT1_MNT,     NULL,           NULL, 0, DRC_NA },
-        {"DUMP",    MOUNT1_DUMP,    mnt3svc_dump,   NULL, 0, DRC_NA},
-        {"UMNT",    MOUNT1_UMNT,    mnt3svc_umnt,   NULL, 0, DRC_NA},
-        {"UMNTALL", MOUNT1_UMNTALL, NULL,           NULL, 0, DRC_NA},
-        {"EXPORT",  MOUNT1_EXPORT,  mnt3svc_export, NULL, 0, DRC_NA}
-};
-
-rpcsvc_program_t        mnt1prog = {
-                        .progname       = "MOUNT1",
-                        .prognum        = MOUNT_PROGRAM,
-                        .progver        = MOUNT_V1,
-                        .progport       = GF_MOUNTV1_PORT,
-                        .actors         = mnt1svc_actors,
-                        .numactors      = MOUNT1_PROC_COUNT,
-                        .min_auth       = AUTH_NULL,
-                        .synctask       = _gf_true,
+static rpcsvc_actor_t mnt1svc_actors[MOUNT1_PROC_COUNT] = {
+    {"NULL", mnt3svc_null, NULL, MOUNT1_NULL, DRC_NA, 0},
+    {"MNT", NULL, NULL, MOUNT1_MNT, DRC_NA, 0},
+    {"DUMP", mnt3svc_dump, NULL, MOUNT1_DUMP, DRC_NA, 0},
+    {"UMNT", mnt3svc_umnt, NULL, MOUNT1_UMNT, DRC_NA, 0},
+    {"UMNTALL", NULL, NULL, MOUNT1_UMNTALL, DRC_NA, 0},
+    {"EXPORT", mnt3svc_export, NULL, MOUNT1_EXPORT, DRC_NA, 0}};
+
+static rpcsvc_program_t mnt1prog = {
+    .progname = "MOUNT1",
+    .prognum = MOUNT_PROGRAM,
+    .progver = MOUNT_V1,
+    .progport = GF_MOUNTV1_PORT,
+    .actors = mnt1svc_actors,
+    .numactors = MOUNT1_PROC_COUNT,
+    .min_auth = AUTH_NULL,
+    .synctask = _gf_true,
 };
 
-
 rpcsvc_program_t *
-mnt1svc_init (xlator_t *nfsx)
+mnt1svc_init(xlator_t *nfsx)
 {
-        struct mount3_state     *mstate = NULL;
-        struct nfs_state        *nfs = NULL;
-        dict_t                  *options = NULL;
-        char                    *portstr = NULL;
-        int                      ret = -1;
-
-        if (!nfsx || !nfsx->private)
-                return NULL;
-
-        nfs = (struct nfs_state *)nfsx->private;
-
-        gf_log (GF_MNT, GF_LOG_DEBUG, "Initing Mount v1 state");
-        mstate = (struct mount3_state *)nfs->mstate;
-        if (!mstate) {
-                gf_log (GF_MNT, GF_LOG_ERROR, "Mount v3 state init failed");
-                goto err;
-        }
+    struct mount3_state *mstate = NULL;
+    struct nfs_state *nfs = NULL;
+    dict_t *options = NULL;
+    char *portstr = NULL;
+    int ret = -1;
 
-        mnt1prog.private = mstate;
-
-        options = dict_new ();
-
-        ret = gf_asprintf (&portstr, "%d", GF_MOUNTV1_PORT);
-        if (ret == -1)
-                goto err;
+    if (!nfsx || !nfsx->private)
+        return NULL;
 
-        ret = dict_set_dynstr (options, "transport.socket.listen-port", portstr);
-        if (ret == -1)
-                goto err;
-        ret = dict_set_str (options, "transport-type", "socket");
+    nfs = (struct nfs_state *)nfsx->private;
+
+    gf_msg_debug(GF_MNT, GF_LOG_DEBUG, "Initing Mount v1 state");
+    mstate = (struct mount3_state *)nfs->mstate;
+    if (!mstate) {
+        gf_msg(GF_MNT, GF_LOG_ERROR, EINVAL, NFS_MSG_MNT_STATE_INIT_FAIL,
+               "Mount v3 state init failed");
+        goto err;
+    }
+
+    mnt1prog.private = mstate;
+
+    options = dict_new();
+    if (options == NULL) {
+        gf_msg(GF_NFS, GF_LOG_ERROR, ENOMEM, NFS_MSG_GFID_DICT_CREATE_FAIL,
+               "dict allocation failed");
+        goto err;
+    }
+
+    ret = gf_asprintf(&portstr, "%d", GF_MOUNTV1_PORT);
+    if (ret == -1)
+        goto err;
+
+    ret = dict_set_dynstr(options, "transport.socket.listen-port", portstr);
+    if (ret == -1)
+        goto err;
+    ret = dict_set_str(options, "transport-type", "socket");
+    if (ret == -1) {
+        gf_msg(GF_NFS, GF_LOG_ERROR, errno, NFS_MSG_DICT_SET_FAILED,
+               "dict_set_str error");
+        goto err;
+    }
+
+    if (nfs->allow_insecure) {
+        ret = dict_set_str(options, "rpc-auth-allow-insecure", "on");
         if (ret == -1) {
-                gf_log (GF_NFS, GF_LOG_ERROR, "dict_set_str error");
-                goto err;
-        }
-
-        if (nfs->allow_insecure) {
-                ret = dict_set_str (options, "rpc-auth-allow-insecure", "on");
-                if (ret == -1) {
-                        gf_log (GF_NFS, GF_LOG_ERROR, "dict_set_str error");
-                        goto err;
-                }
-                ret = dict_set_str (options, "rpc-auth.ports.insecure", "on");
-                if (ret == -1) {
-                        gf_log (GF_NFS, GF_LOG_ERROR, "dict_set_str error");
-                        goto err;
-                }
+            gf_msg(GF_NFS, GF_LOG_ERROR, errno, NFS_MSG_DICT_SET_FAILED,
+                   "dict_set_str error");
+            goto err;
         }
-
-        ret = rpcsvc_create_listeners (nfs->rpcsvc, options, nfsx->name);
+        ret = dict_set_str(options, "rpc-auth.ports.insecure", "on");
         if (ret == -1) {
-                gf_log (GF_NFS, GF_LOG_ERROR, "Unable to create listeners");
-                dict_unref (options);
-                goto err;
-        }
+            gf_msg(GF_NFS, GF_LOG_ERROR, errno, NFS_MSG_DICT_SET_FAILED,
+                   "dict_set_str error");
+            goto err;
+        }
+    }
+
+#ifdef IPV6_DEFAULT
+    ret = dict_set_str(options, "transport.address-family", "inet6");
+    if (ret == -1) {
+        gf_log(GF_NFS, GF_LOG_ERROR,
+               "dict_set_str error when trying to enable ipv6");
+        goto err;
+    }
+#endif
+
+    ret = rpcsvc_create_listeners(nfs->rpcsvc, options, nfsx->name);
+    if (ret == -1) {
+        gf_msg(GF_NFS, GF_LOG_ERROR, errno, NFS_MSG_LISTENERS_CREATE_FAIL,
+               "Unable to create listeners");
+        dict_unref(options);
+        goto err;
+    }
 
-        return &mnt1prog;
+    return &mnt1prog;
 err:
-        return NULL;
+    return NULL;
 }
 
 int
-mount_reconfigure_state (xlator_t *nfsx, dict_t *options)
+mount_reconfigure_state(xlator_t *nfsx, dict_t *options)
 {
-        int                    ret      = -1;
-        struct nfs_state       *nfs     = NULL;
-        struct mount3_state    *ms      = NULL;
-        struct mnt3_export     *exp     = NULL;
-        struct mnt3_export     *texp  = NULL;
-
-        if ((!nfsx) || (!options))
-                return (-1);
-
-        nfs = (struct nfs_state *)nfs_state (nfsx);
-        if (!nfs)
-                return (-1);
-
-        ms = nfs->mstate;
-        if (!ms)
-                return (-1);
-
-        /*
-         * Free() up the old export list. mnt3_init_options() will
-         * rebuild the export list from scratch. Do it with locking
-         * to avoid unnecessary race conditions.
-         */
-        LOCK (&ms->mountlock);
-        list_for_each_entry_safe (exp, texp, &ms->exportlist, explist) {
-                list_del (&exp->explist);
-                mnt3_export_free (exp);
-        }
-        ret = mnt3_init_options (ms, options);
-        UNLOCK (&ms->mountlock);
-
-        if (ret < 0) {
-                gf_log (GF_MNT, GF_LOG_ERROR, "Options reconfigure failed");
-                return (-1);
-        }
-
-        return (0);
+    int ret = -1;
+    struct nfs_state *nfs = NULL;
+    struct mount3_state *ms = NULL;
+    struct mnt3_export *exp = NULL;
+    struct mnt3_export *texp = NULL;
+
+    if ((!nfsx) || (!options))
+        return (-1);
+
+    nfs = (struct nfs_state *)nfs_state(nfsx);
+    if (!nfs)
+        return (-1);
+
+    ms = nfs->mstate;
+    if (!ms)
+        return (-1);
+
+    /*
+     * Free() up the old export list. mnt3_init_options() will
+     * rebuild the export list from scratch. Do it with locking
+     * to avoid unnecessary race conditions.
+     */
+    LOCK(&ms->mountlock);
+    list_for_each_entry_safe(exp, texp, &ms->exportlist, explist)
+    {
+        list_del(&exp->explist);
+        mnt3_export_free(exp);
+    }
+    ret = mnt3_init_options(ms, options);
+    UNLOCK(&ms->mountlock);
+
+    if (ret < 0) {
+        gf_msg(GF_MNT, GF_LOG_ERROR, ret, NFS_MSG_RECONF_FAIL,
+               "Options reconfigure failed");
+        return (-1);
+    }
+
+    return (0);
 }
diff --git a/xlators/nfs/server/src/mount3.h b/xlators/nfs/server/src/mount3.h
index 026300e886f..b185df835ae 100644
--- a/xlators/nfs/server/src/mount3.h
+++ b/xlators/nfs/server/src/mount3.h
@@ -11,176 +11,178 @@
 #ifndef _MOUNT3_H_
 #define _MOUNT3_H_
 
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
-
 #include "rpcsvc.h"
-#include "dict.h"
-#include "xlator.h"
-#include "iobuf.h"
+#include <glusterfs/dict.h>
+#include <glusterfs/xlator.h>
+#include <glusterfs/iobuf.h>
 #include "nfs.h"
-#include "list.h"
+#include <glusterfs/list.h>
 #include "xdr-nfs3.h"
-#include "locking.h"
+#include <glusterfs/locking.h>
 #include "nfs3-fh.h"
-#include "compat-uuid.h"
+#include <glusterfs/compat-uuid.h>
 #include "exports.h"
 #include "mount3-auth.h"
 #include "auth-cache.h"
 
 /* Registered with portmap */
-#define GF_MOUNTV3_PORT         38465
-#define GF_MOUNTV3_IOB          (2 * GF_UNIT_KB)
-#define GF_MOUNTV3_IOBPOOL      (GF_MOUNTV3_IOB * 50)
+#define GF_MOUNTV3_PORT 38465
+#define GF_MOUNTV3_IOB (2 * GF_UNIT_KB)
+#define GF_MOUNTV3_IOBPOOL (GF_MOUNTV3_IOB * 50)
 
-#define GF_MOUNTV1_PORT         38466
-#define GF_MNT                  GF_NFS"-mount"
+#define GF_MOUNTV1_PORT 38466
+#define GF_MNT GF_NFS "-mount"
 
 extern rpcsvc_program_t *
-mnt3svc_init (xlator_t *nfsx);
+mnt3svc_init(xlator_t *nfsx);
 
 extern rpcsvc_program_t *
-mnt1svc_init (xlator_t *nfsx);
+mnt1svc_init(xlator_t *nfsx);
 
 extern void
-mnt3svc_deinit (xlator_t *nfsx);
+mnt3svc_deinit(xlator_t *nfsx);
 
 extern int
-mount_init_state (xlator_t *nfsx);
+mount_init_state(xlator_t *nfsx);
 
 extern int
-mount_reconfigure_state (xlator_t *nfsx, dict_t *options);
+mount_reconfigure_state(xlator_t *nfsx, dict_t *options);
 
 void
-mount_rewrite_rmtab (struct mount3_state *ms, char *new_rmtab);
+mount_rewrite_rmtab(struct mount3_state *ms, char *new_rmtab);
 
 struct mnt3_export *
-mnt3_mntpath_to_export (struct mount3_state *ms, const char *dirpath,
-                        gf_boolean_t export_parsing_match);
+mnt3_mntpath_to_export(struct mount3_state *ms, const char *dirpath,
+                       gf_boolean_t export_parsing_match);
 
 extern int
-mnt3svc_update_mountlist (struct mount3_state *ms, rpcsvc_request_t *req,
-                          const char *expname, const char *fullpath);
+mnt3svc_update_mountlist(struct mount3_state *ms, rpcsvc_request_t *req,
+                         const char *expname, const char *fullpath);
 
 int
-mnt3_authenticate_request (struct mount3_state *ms, rpcsvc_request_t *req,
-                           struct nfs3_fh *fh, const char *volname,
-                           const char *path, char **authorized_path,
-                           char **authorized_host, gf_boolean_t is_write_op);
+mnt3_authenticate_request(struct mount3_state *ms, rpcsvc_request_t *req,
+                          struct nfs3_fh *fh, const char *volname,
+                          const char *path, char **authorized_path,
+                          char **authorized_host, gf_boolean_t is_write_op);
 
 /* Data structure used to store the list of mounts points currently
  * in use by NFS clients.
  */
 struct mountentry {
-        /* Links to mount3_state->mountlist.  */
-        struct list_head        mlist;
+    /* Links to mount3_state->mountlist.  */
+    struct list_head mlist;
 
-        /* The export name */
-        char                    exname[MNTPATHLEN];
-        char                    hostname[MNTPATHLEN];
-        char                    fullpath[MNTPATHLEN];
+    /* The export name */
+    char exname[MNTPATHLEN];
+    char hostname[MNTPATHLEN];
+    char fullpath[MNTPATHLEN];
 
-        gf_boolean_t            has_full_path;
+    gf_boolean_t has_full_path;
 
-        /* Since this is stored in a dict, we want to be able
-         * to find easily get the key we used to store
-         * the struct in our dict
-         */
-        char                    hashkey[MNTPATHLEN*2+2];
+    /* Since this is stored in a dict, we want to be able
+     * to find easily get the key we used to store
+     * the struct in our dict
+     */
+    char hashkey[MNTPATHLEN * 2 + 2];
 };
 
-#define MNT3_EXPTYPE_VOLUME     1
-#define MNT3_EXPTYPE_DIR        2
+#define MNT3_EXPTYPE_VOLUME 1
+#define MNT3_EXPTYPE_DIR 2
 
 /* Structure to hold export-dir AUTH parameter */
 struct host_auth_spec {
-        char                    *host_addr;    /* Allowed IP or host name */
-        uint32_t                netmask;       /* Network mask (Big-Endian) */
-        struct host_auth_spec   *next;         /* Pointer to next AUTH struct */
+    char *host_addr;             /* Allowed IP or host name */
+    uint32_t netmask;            /* Network mask (Big-Endian) */
+    struct host_auth_spec *next; /* Pointer to next AUTH struct */
 };
 
 struct mnt3_export {
-        struct list_head        explist;
-
-        /* The string that may contain either the volume name if the full volume
-         * is exported or the subdirectory in the volume.
-         */
-        char                    *expname;
-        /*
-         * IP address, hostname or subnets who are allowed to connect to expname
-         * subvolume or subdirectory
-         */
-        struct host_auth_spec*  hostspec;
-        xlator_t                *vol;
-        int                     exptype;
-
-        /* This holds the full path that the client requested including
-         * the volume name AND the subdirectory in the volume.
-         */
-        char                    *fullpath;
-
-        /* Extracted from nfs volume options if nfs.dynamicvolumes is on.
-         */
-        uuid_t                  volumeid;
-        uuid_t                  mountid;
+    struct list_head explist;
+
+    /* The string that may contain either the volume name if the full volume
+     * is exported or the subdirectory in the volume.
+     */
+    char *expname;
+    /*
+     * IP address, hostname or subnets who are allowed to connect to expname
+     * subvolume or subdirectory
+     */
+    struct host_auth_spec *hostspec;
+    xlator_t *vol;
+    int exptype;
+
+    /* This holds the full path that the client requested including
+     * the volume name AND the subdirectory in the volume.
+     */
+    char *fullpath;
+
+    /* Extracted from nfs volume options if nfs.dynamicvolumes is on.
+     */
+    uuid_t volumeid;
+    uuid_t mountid;
 };
 
 struct mount3_state {
-        xlator_t                *nfsx;
+    xlator_t *nfsx;
 
-        /* The NFS state that this belongs to */
-        struct nfs_state        *nfs;
+    /* The NFS state that this belongs to */
+    struct nfs_state *nfs;
 
-        /* The buffers for all network IO are got from this pool. */
-        struct iobuf_pool       *iobpool;
+    /* The buffers for all network IO are got from this pool. */
+    struct iobuf_pool *iobpool;
 
-        /* List of exports, can be volumes or directories in those volumes. */
-        struct list_head        exportlist;
+    /* List of exports, can be volumes or directories in those volumes. */
+    struct list_head exportlist;
 
-        /* List of current mount points over all the exports from this
-         * server.
-         */
-        struct list_head        mountlist;
+    /* List of current mount points over all the exports from this
+     * server.
+     */
+    struct list_head mountlist;
 
-        /* Dict of current mount points over all the exports from this
-         * server. Mirrors the mountlist above, but can be used for
-         * faster lookup in the event that there are several mounts.
-         * Currently, each NFSOP is validated against this dict: each
-         * op is checked to see if the host that operates on the path
-         * does in fact have an entry in the mount dict.
-         */
-        dict_t                  *mountdict;
+    /* Dict of current mount points over all the exports from this
+     * server. Mirrors the mountlist above, but can be used for
+     * faster lookup in the event that there are several mounts.
+     * Currently, each NFSOP is validated against this dict: each
+     * op is checked to see if the host that operates on the path
+     * does in fact have an entry in the mount dict.
+     */
+    dict_t *mountdict;
 
-        /* Used to protect the mountlist & the mount dict */
-        pthread_spinlock_t       mountlock;
+    /* Used to protect the mountlist & the mount dict */
+    gf_lock_t mountlock;
 
-        /* Used to insert additional authentication parameters */
-        struct mnt3_auth_params      *auth_params;
+    /* Used to insert additional authentication parameters */
+    struct mnt3_auth_params *auth_params;
 
-        /* Set to 0 if exporting full volumes is disabled. On by default. */
-        gf_boolean_t            export_volumes;
-        gf_boolean_t            export_dirs;
+    /* Set to 0 if exporting full volumes is disabled. On by default. */
+    gf_boolean_t export_volumes;
+    gf_boolean_t export_dirs;
 
-        pthread_t               auth_refresh_thread;
-        gf_boolean_t            stop_refresh;
+    pthread_t auth_refresh_thread;
+    gf_boolean_t stop_refresh;
 
-        struct auth_cache       *authcache;
+    struct auth_cache *authcache;
 };
 
-#define gf_mnt3_export_dirs(mst)        ((mst)->export_dirs)
+#define gf_mnt3_export_dirs(mst) ((mst)->export_dirs)
 
 struct mount3_resolve_state {
-        struct mnt3_export      *exp;
-        struct mount3_state     *mstate;
-        rpcsvc_request_t        *req;
+    struct mnt3_export *exp;
+    struct mount3_state *mstate;
+    rpcsvc_request_t *req;
 
-        char                    remainingdir[MNTPATHLEN];
-        loc_t                   resolveloc;
-        struct nfs3_fh          parentfh;
+    char remainingdir[MNTPATHLEN];
+    loc_t resolveloc;
+    struct nfs3_fh parentfh;
 };
 
 typedef struct mount3_resolve_state mnt3_resolve_t;
 
+int
+mnt3_parse_dir_exports(rpcsvc_request_t *req, struct mount3_state *ms,
+                       char *subdir, gf_boolean_t send_reply);
+
+char *
+mnt3_get_volume_subdir(char *path, char **volname);
+
 #endif
diff --git a/xlators/nfs/server/src/mount3udp_svc.c b/xlators/nfs/server/src/mount3udp_svc.c
index 4ad8e11ad49..1a2b0f85453 100644
--- a/xlators/nfs/server/src/mount3udp_svc.c
+++ b/xlators/nfs/server/src/mount3udp_svc.c
@@ -8,11 +8,11 @@
   cases as published by the Free Software Foundation.
 */
 
-
 #include "xdr-nfs3.h"
-#include "logging.h"
-#include "mem-pool.h"
+#include <glusterfs/logging.h>
+#include <glusterfs/mem-pool.h>
 #include "nfs-mem-types.h"
+#include "nfs-messages.h"
 #include "mount3.h"
 #include <stdio.h>
 #include <stdlib.h>
@@ -22,203 +22,217 @@
 #include <sys/socket.h>
 #include <netinet/in.h>
 
-
-extern struct nfs3_fh*
-nfs3_rootfh (struct svc_req *req, xlator_t *nfsx, char *dp, char *expname);
+extern struct nfs3_fh *
+nfs3_rootfh(struct svc_req *req, xlator_t *nfsx, char *dp, char *expname);
 
 extern mountres3
-mnt3svc_set_mountres3 (mountstat3 stat, struct nfs3_fh *fh,
-                       int *authflavor, u_int aflen);
+mnt3svc_set_mountres3(mountstat3 stat, struct nfs3_fh *fh, int *authflavor,
+                      u_int aflen);
 extern int
-mount3udp_add_mountlist (xlator_t *nfsx, char *host, char *expname);
+mount3udp_add_mountlist(xlator_t *nfsx, char *host, char *expname);
 
 extern int
-mount3udp_delete_mountlist (xlator_t *nfsx, char *host, char *expname);
+mount3udp_delete_mountlist(xlator_t *nfsx, char *host, char *expname);
 
 extern mountstat3
-mnt3svc_errno_to_mnterr (int32_t errnum);
-
+mnt3svc_errno_to_mnterr(int32_t errnum);
 
 /* only this thread will use this, no locking needed */
-char mnthost[INET_ADDRSTRLEN+1];
+char mnthost[INET_ADDRSTRLEN + 1];
 
 #define MNT3UDP_AUTH_LEN 1 /* Only AUTH_UNIX for now */
 
 mountres3 *
 mountudpproc3_mnt_3_svc(dirpath **dpp, struct svc_req *req)
 {
-        struct mountres3        *res = NULL;
-        int                     *autharr = NULL;
-        struct nfs3_fh          *fh = NULL;
-        char                    *mpath = NULL;
-        xlator_t                *nfsx = THIS;
-        char                    expname[PATH_MAX] = {0, };
-        mountstat3              stat = MNT3ERR_SERVERFAULT;
-
-        errno = 0; /* RESET errno */
-
-        mpath = (char *)*dpp;
-        while (*mpath == '/')
-                mpath++;
-
-        res = GF_CALLOC (1, sizeof(*res), gf_nfs_mt_mountres3);
-        if (res == NULL) {
-                gf_log (GF_MNT, GF_LOG_ERROR, "Unable to allocate memory");
-                goto err;
-        }
-        autharr = GF_CALLOC (MNT3UDP_AUTH_LEN, sizeof(int), gf_nfs_mt_int);
-        if (autharr == NULL) {
-                gf_log (GF_MNT, GF_LOG_ERROR, "Unable to allocate memory");
-                goto err;
-        }
-
-        autharr[0] = AUTH_UNIX;
-
-        fh = nfs3_rootfh (req, nfsx, mpath, (char *)expname);
-
-        /* FAILURE: No FH */
-        if (fh == NULL) {
-                gf_log (GF_MNT, GF_LOG_ERROR, "Unable to get fh for %s", mpath);
-                if (errno)
-                        stat = mnt3svc_errno_to_mnterr (errno);
-                *res = mnt3svc_set_mountres3 (stat, NULL /* fh */,
-                                              autharr, MNT3UDP_AUTH_LEN);
-                return res;
-        }
-
-        /* SUCCESS */
-        stat = MNT3_OK;
-        *res = mnt3svc_set_mountres3 (stat, fh, autharr, MNT3UDP_AUTH_LEN);
-        (void) mount3udp_add_mountlist (nfsx, mnthost, (char *) expname);
+    struct mountres3 *res = NULL;
+    int *autharr = NULL;
+    struct nfs3_fh *fh = NULL;
+    char *mpath = NULL;
+    xlator_t *nfsx = THIS;
+    char expname[PATH_MAX] = {
+        0,
+    };
+    mountstat3 stat = MNT3ERR_SERVERFAULT;
+
+    errno = 0; /* RESET errno */
+
+    mpath = (char *)*dpp;
+    while (*mpath == '/')
+        mpath++;
+
+    res = GF_CALLOC(1, sizeof(*res), gf_nfs_mt_mountres3);
+    if (res == NULL) {
+        gf_msg(GF_MNT, GF_LOG_ERROR, ENOMEM, NFS_MSG_NO_MEMORY,
+               "Unable to allocate memory");
+        goto err;
+    }
+    autharr = GF_CALLOC(MNT3UDP_AUTH_LEN, sizeof(int), gf_nfs_mt_int);
+    if (autharr == NULL) {
+        gf_msg(GF_MNT, GF_LOG_ERROR, ENOMEM, NFS_MSG_NO_MEMORY,
+               "Unable to allocate memory");
+        goto err;
+    }
+
+    autharr[0] = AUTH_UNIX;
+
+    fh = nfs3_rootfh(req, nfsx, mpath, (char *)expname);
+
+    /* FAILURE: No FH */
+    if (fh == NULL) {
+        gf_msg(GF_MNT, GF_LOG_ERROR, errno, NFS_MSG_GET_FH_FAIL,
+               "Unable to get fh for %s", mpath);
+        if (errno)
+            stat = mnt3svc_errno_to_mnterr(errno);
+        *res = mnt3svc_set_mountres3(stat, NULL /* fh */, autharr,
+                                     MNT3UDP_AUTH_LEN);
         return res;
-
- err:
-        GF_FREE (fh);
-        GF_FREE (res);
-        GF_FREE (autharr);
-        return NULL;
+    }
+
+    /* SUCCESS */
+    stat = MNT3_OK;
+    *res = mnt3svc_set_mountres3(stat, fh, autharr, MNT3UDP_AUTH_LEN);
+    (void)mount3udp_add_mountlist(nfsx, mnthost, (char *)expname);
+    return res;
+
+err:
+    GF_FREE(fh);
+    GF_FREE(res);
+    GF_FREE(autharr);
+    return NULL;
 }
 
 mountstat3 *
 mountudpproc3_umnt_3_svc(dirpath **dp, struct svc_req *req)
 {
-        mountstat3 *stat = NULL;
-        char       *mpath = (char *) *dp;
-        xlator_t   *nfsx = THIS;
-
-        stat = GF_CALLOC (1, sizeof(mountstat3), gf_nfs_mt_mountstat3);
-        if (stat == NULL) {
-                gf_log (GF_MNT, GF_LOG_ERROR, "Unable to allocate memory");
-                return NULL;
-        }
-        *stat = MNT3_OK;
-        (void) mount3udp_delete_mountlist (nfsx, mnthost, mpath);
-        return stat;
+    mountstat3 *stat = NULL;
+    char *mpath = (char *)*dp;
+    xlator_t *nfsx = THIS;
+
+    stat = GF_MALLOC(sizeof(mountstat3), gf_nfs_mt_mountstat3);
+    if (stat == NULL) {
+        gf_msg(GF_MNT, GF_LOG_ERROR, ENOMEM, NFS_MSG_NO_MEMORY,
+               "Unable to allocate memory");
+        return NULL;
+    }
+    *stat = MNT3_OK;
+    (void)mount3udp_delete_mountlist(nfsx, mnthost, mpath);
+    return stat;
 }
 
 static void
 mountudp_program_3(struct svc_req *rqstp, register SVCXPRT *transp)
 {
-        union {
-                dirpath mountudpproc3_mnt_3_arg;
-        } argument;
-        char                    *result = NULL;
-        xdrproc_t               _xdr_argument = NULL, _xdr_result = NULL;
-        char *(*local)(char *, struct svc_req *) = NULL;
-        mountres3               *res = NULL;
-        struct sockaddr_in      *sin = NULL;
-
-        sin = svc_getcaller (transp);
-        inet_ntop (AF_INET, &sin->sin_addr, mnthost, INET_ADDRSTRLEN+1);
-
-        switch (rqstp->rq_proc) {
+    union {
+        dirpath mountudpproc3_mnt_3_arg;
+    } argument;
+    char *result = NULL;
+    xdrproc_t _xdr_argument = NULL, _xdr_result = NULL;
+    char *(*local)(char *, struct svc_req *) = NULL;
+    mountres3 *res = NULL;
+    struct sockaddr_in *sin = NULL;
+
+#if !defined(_TIRPC_SVC_H)
+    sin = svc_getcaller(transp);
+#else
+    sin = (struct sockaddr_in *)svc_getcaller(transp);
+    /* TIRPC's svc_getcaller() returns a pointer to a sockaddr_in6, even
+     * though it might actually be an IPv4 address. It ought return a
+     * struct sockaddr and make the caller upcast it to the proper
+     * address family. Sigh.
+     */
+#endif
+    /* And let's make sure that it's actually an IPv4 address. */
+    GF_ASSERT(sin->sin_family == AF_INET);
+
+    inet_ntop(AF_INET, &sin->sin_addr, mnthost, INET_ADDRSTRLEN + 1);
+
+    switch (rqstp->rq_proc) {
         case NULLPROC:
-                (void) svc_sendreply (transp, (xdrproc_t) xdr_void,
-                                      (char *)NULL);
-                return;
+            (void)svc_sendreply(transp, (xdrproc_t)xdr_void, (char *)NULL);
+            return;
 
         case MOUNT3_MNT:
-                _xdr_argument = (xdrproc_t) xdr_dirpath;
-                _xdr_result = (xdrproc_t) xdr_mountres3;
-                local = (char *(*)(char *,
-                                   struct svc_req *)) mountudpproc3_mnt_3_svc;
-                break;
+            _xdr_argument = (xdrproc_t)xdr_dirpath;
+            _xdr_result = (xdrproc_t)xdr_mountres3;
+            local = (char *(*)(char *,
+                               struct svc_req *))mountudpproc3_mnt_3_svc;
+            break;
 
         case MOUNT3_UMNT:
-                _xdr_argument = (xdrproc_t) xdr_dirpath;
-                _xdr_result = (xdrproc_t) xdr_mountstat3;
-                local = (char *(*)(char *,
-                                   struct svc_req *)) mountudpproc3_umnt_3_svc;
-                break;
+            _xdr_argument = (xdrproc_t)xdr_dirpath;
+            _xdr_result = (xdrproc_t)xdr_mountstat3;
+            local = (char *(*)(char *,
+                               struct svc_req *))mountudpproc3_umnt_3_svc;
+            break;
 
         default:
-                svcerr_noproc (transp);
-                return;
-        }
-        memset ((char *)&argument, 0, sizeof (argument));
-        if (!svc_getargs (transp, (xdrproc_t) _xdr_argument,
-                          (caddr_t) &argument)) {
-                svcerr_decode (transp);
-                return;
-        }
-        result = (*local)((char *)&argument, rqstp);
-        if (result == NULL) {
-                gf_log (GF_MNT, GF_LOG_DEBUG, "PROC returned error");
-                svcerr_systemerr (transp);
-        }
-        if (result != NULL && !svc_sendreply(transp, (xdrproc_t) _xdr_result,
-                                             result)) {
-                gf_log (GF_MNT, GF_LOG_ERROR, "svc_sendreply returned error");
-                svcerr_systemerr (transp);
-        }
-        if (!svc_freeargs (transp, (xdrproc_t) _xdr_argument,
-                           (caddr_t) &argument)) {
-                gf_log (GF_MNT, GF_LOG_ERROR, "Unable to free arguments");
-        }
-        if (result == NULL)
-                return;
-        /* free the result */
-        switch (rqstp->rq_proc) {
+            svcerr_noproc(transp);
+            return;
+    }
+    memset((char *)&argument, 0, sizeof(argument));
+    if (!svc_getargs(transp, (xdrproc_t)_xdr_argument, (caddr_t)&argument)) {
+        svcerr_decode(transp);
+        return;
+    }
+    result = (*local)((char *)&argument, rqstp);
+    if (result == NULL) {
+        gf_msg_debug(GF_MNT, 0, "PROC returned error");
+        svcerr_systemerr(transp);
+    }
+    if (result != NULL &&
+        !svc_sendreply(transp, (xdrproc_t)_xdr_result, result)) {
+        gf_msg(GF_MNT, GF_LOG_ERROR, 0, NFS_MSG_SVC_ERROR,
+               "svc_sendreply returned error");
+        svcerr_systemerr(transp);
+    }
+    if (!svc_freeargs(transp, (xdrproc_t)_xdr_argument, (caddr_t)&argument)) {
+        gf_msg(GF_MNT, GF_LOG_ERROR, 0, NFS_MSG_ARG_FREE_FAIL,
+               "Unable to free arguments");
+    }
+    if (result == NULL)
+        return;
+    /* free the result */
+    switch (rqstp->rq_proc) {
         case MOUNT3_MNT:
-                res = (mountres3 *) result;
-                GF_FREE (res->mountres3_u.mountinfo.fhandle.fhandle3_val);
-                GF_FREE (res->mountres3_u.mountinfo.auth_flavors.auth_flavors_val);
-                GF_FREE (res);
-                break;
+            res = (mountres3 *)result;
+            GF_FREE(res->mountres3_u.mountinfo.fhandle.fhandle3_val);
+            GF_FREE(res->mountres3_u.mountinfo.auth_flavors.auth_flavors_val);
+            GF_FREE(res);
+            break;
 
         case MOUNT3_UMNT:
-                GF_FREE (result);
-                break;
-        }
-        return;
+            GF_FREE(result);
+            break;
+    }
+    return;
 }
 
 void *
-mount3udp_thread (void *argv)
+mount3udp_thread(void *argv)
 {
-        xlator_t         *nfsx   = argv;
-        register SVCXPRT *transp = NULL;
-
-        GF_ASSERT (nfsx);
-
-        if (glusterfs_this_set(nfsx)) {
-                gf_log (GF_MNT, GF_LOG_ERROR,
-                        "Failed to set xlator, nfs.mount-udp will not work");
-                return NULL;
-        }
-
-        transp = svcudp_create(RPC_ANYSOCK);
-        if (transp == NULL) {
-                gf_log (GF_MNT, GF_LOG_ERROR, "svcudp_create error");
-                return NULL;
-        }
-        if (!svc_register(transp, MOUNT_PROGRAM, MOUNT_V3,
-                          mountudp_program_3, IPPROTO_UDP)) {
-                gf_log (GF_MNT, GF_LOG_ERROR, "svc_register error");
-                return NULL;
-        }
-
-        svc_run ();
-        gf_log (GF_MNT, GF_LOG_ERROR, "svc_run returned");
+    xlator_t *nfsx = argv;
+    register SVCXPRT *transp = NULL;
+
+    GF_ASSERT(nfsx);
+
+    THIS = nfsx;
+
+    transp = svcudp_create(RPC_ANYSOCK);
+    if (transp == NULL) {
+        gf_msg(GF_MNT, GF_LOG_ERROR, 0, NFS_MSG_SVC_ERROR,
+               "svcudp_create error");
+        return NULL;
+    }
+    if (!svc_register(transp, MOUNT_PROGRAM, MOUNT_V3, mountudp_program_3,
+                      IPPROTO_UDP)) {
+        gf_msg(GF_MNT, GF_LOG_ERROR, 0, NFS_MSG_SVC_ERROR,
+               "svc_register error");
         return NULL;
+    }
+
+    svc_run();
+    gf_msg(GF_MNT, GF_LOG_ERROR, 0, NFS_MSG_SVC_RUN_RETURNED,
+           "svc_run returned");
+    return NULL;
 }
diff --git a/xlators/nfs/server/src/netgroups.c b/xlators/nfs/server/src/netgroups.c
index a505586961c..f68a7f14a26 100644
--- a/xlators/nfs/server/src/netgroups.c
+++ b/xlators/nfs/server/src/netgroups.c
@@ -13,11 +13,15 @@
 */
 
 #include "netgroups.h"
-#include "parse-utils.h"
+#include <glusterfs/parse-utils.h>
+#include "nfs-messages.h"
 
-static void _nge_print (const struct netgroup_entry *nge);
-static void _netgroup_entry_deinit (struct netgroup_entry *ptr);
-static void _netgroup_host_deinit (struct netgroup_host *host);
+static void
+_nge_print(const struct netgroup_entry *nge);
+static void
+_netgroup_entry_deinit(struct netgroup_entry *ptr);
+static void
+_netgroup_host_deinit(struct netgroup_host *host);
 
 static dict_t *__deleted_entries;
 static struct parser *ng_file_parser;
@@ -30,36 +34,36 @@ static struct parser *ng_host_parser;
  *          failure: -1
  */
 static int
-_ng_init_parsers ()
+_ng_init_parsers()
 {
-        int ret = -1;
-
-        /* Initialize the parsers. The only reason this should
-         * ever fail is because of 1) memory allocation errors
-         * 2) the regex in netgroups.h has been changed and no
-         * longer compiles.
-         */
-        ng_file_parser = parser_init (NG_FILE_PARSE_REGEX);
-        if (!ng_file_parser)
-                goto out;
-
-        ng_host_parser = parser_init (NG_HOST_PARSE_REGEX);
-        if (!ng_host_parser)
-                goto out;
-
-        ret = 0;
+    int ret = -1;
+
+    /* Initialize the parsers. The only reason this should
+     * ever fail is because of 1) memory allocation errors
+     * 2) the regex in netgroups.h has been changed and no
+     * longer compiles.
+     */
+    ng_file_parser = parser_init(NG_FILE_PARSE_REGEX);
+    if (!ng_file_parser)
+        goto out;
+
+    ng_host_parser = parser_init(NG_HOST_PARSE_REGEX);
+    if (!ng_host_parser)
+        goto out;
+
+    ret = 0;
 out:
-        return ret;
+    return ret;
 }
 
 /**
  * _ng_deinit_parsers - Free the parsers used in this file
  */
 static void
-_ng_deinit_parsers ()
+_ng_deinit_parsers()
 {
-        parser_deinit (ng_file_parser);
-        parser_deinit (ng_host_parser);
+    parser_deinit(ng_file_parser);
+    parser_deinit(ng_host_parser);
 }
 
 /**
@@ -70,18 +74,18 @@ _ng_deinit_parsers ()
  * Not for external use.
  */
 static struct netgroups_file *
-_netgroups_file_init ()
+_netgroups_file_init()
 {
-        struct netgroups_file *file  = GF_MALLOC (sizeof (*file),
-                                                  gf_common_mt_nfs_netgroups);
+    struct netgroups_file *file = GF_MALLOC(sizeof(*file),
+                                            gf_common_mt_nfs_netgroups);
 
-        if (!file)
-                goto out;
+    if (!file)
+        goto out;
 
-        file->filename     = NULL;
-        file->ng_file_dict = NULL;
+    file->filename = NULL;
+    file->ng_file_dict = NULL;
 out:
-        return file;
+    return file;
 }
 
 /**
@@ -99,17 +103,17 @@ out:
  * Not for external use.
  */
 static int
-__ngf_free_walk (dict_t *dict, char *key, data_t *val, void *tmp)
+__ngf_free_walk(dict_t *dict, char *key, data_t *val, void *tmp)
 {
-        struct netgroup_entry        *nge = NULL;
-
-        if (val) {
-                nge = (struct netgroup_entry *)val->data;
-                _netgroup_entry_deinit (nge);
-                val->data = NULL;
-                dict_del (dict, key); /* Remove the key from this dict */
-        }
-        return 0;
+    struct netgroup_entry *nge = NULL;
+
+    if (val) {
+        nge = (struct netgroup_entry *)val->data;
+        _netgroup_entry_deinit(nge);
+        val->data = NULL;
+        dict_del(dict, key); /* Remove the key from this dict */
+    }
+    return 0;
 }
 
 /**
@@ -127,10 +131,10 @@ __ngf_free_walk (dict_t *dict, char *key, data_t *val, void *tmp)
  * Not for external use.
  */
 static int
-__deleted_entries_free_walk (dict_t *dict, char *key, data_t *val, void *tmp)
+__deleted_entries_free_walk(dict_t *dict, char *key, data_t *val, void *tmp)
 {
-        dict_del (dict, key);
-        return 0;
+    dict_del(dict, key);
+    return 0;
 }
 
 /**
@@ -146,24 +150,26 @@ __deleted_entries_free_walk (dict_t *dict, char *key, data_t *val, void *tmp)
  * the memory allocated when parsing the file.
  */
 void
-ng_file_deinit (struct netgroups_file *ngfile)
+ng_file_deinit(struct netgroups_file *ngfile)
 {
-        GF_VALIDATE_OR_GOTO (GF_NG, ngfile, out);
+    if (!ngfile) {
+        return;
+    }
 
-        __deleted_entries = dict_new ();
-        GF_VALIDATE_OR_GOTO (GF_NG, __deleted_entries, out);
+    __deleted_entries = dict_new();
+    GF_VALIDATE_OR_GOTO(GF_NG, __deleted_entries, out);
 
-        GF_FREE (ngfile->filename);
-        dict_foreach (ngfile->ng_file_dict, __ngf_free_walk, NULL);
-        dict_unref (ngfile->ng_file_dict);
-        GF_FREE (ngfile);
+    GF_FREE(ngfile->filename);
+    dict_foreach(ngfile->ng_file_dict, __ngf_free_walk, NULL);
+    dict_unref(ngfile->ng_file_dict);
+    GF_FREE(ngfile);
 
-        /* Clean up temporary dict we used to store "freed" names */
-        dict_foreach (__deleted_entries, __deleted_entries_free_walk, NULL);
-        dict_unref (__deleted_entries);
-        __deleted_entries = NULL;
+    /* Clean up temporary dict we used to store "freed" names */
+    dict_foreach(__deleted_entries, __deleted_entries_free_walk, NULL);
+    dict_unref(__deleted_entries);
+    __deleted_entries = NULL;
 out:
-        return;
+    return;
 }
 
 /**
@@ -176,11 +182,11 @@ out:
  * Not for external use.
  */
 static struct netgroup_entry *
-_netgroup_entry_init ()
+_netgroup_entry_init()
 {
-        struct netgroup_entry *entry = GF_CALLOC (1, sizeof (*entry),
-                                                  gf_common_mt_nfs_netgroups);
-        return entry;
+    struct netgroup_entry *entry = GF_CALLOC(1, sizeof(*entry),
+                                             gf_common_mt_nfs_netgroups);
+    return entry;
 }
 
 /**
@@ -199,17 +205,17 @@ _netgroup_entry_init ()
  * Not for external use.
  */
 static int
-__ngh_free_walk (dict_t *dict, char *key, data_t *val, void *tmp)
+__ngh_free_walk(dict_t *dict, char *key, data_t *val, void *tmp)
 {
-        struct netgroup_host *ngh = NULL;
-
-        if (val) {
-                ngh = (struct netgroup_host *)val->data;
-                _netgroup_host_deinit (ngh);
-                val->data = NULL;
-                dict_del (dict, key);
-        }
-        return 0;
+    struct netgroup_host *ngh = NULL;
+
+    if (val) {
+        ngh = (struct netgroup_host *)val->data;
+        _netgroup_host_deinit(ngh);
+        val->data = NULL;
+        dict_del(dict, key);
+    }
+    return 0;
 }
 
 /**
@@ -228,23 +234,23 @@ __ngh_free_walk (dict_t *dict, char *key, data_t *val, void *tmp)
  * Not for external use.
  */
 static int
-__nge_free_walk (dict_t *dict, char *key, data_t *val, void *tmp)
+__nge_free_walk(dict_t *dict, char *key, data_t *val, void *tmp)
 {
-        struct netgroup_entry *nge = NULL;
+    struct netgroup_entry *nge = NULL;
 
-        GF_VALIDATE_OR_GOTO (GF_NG, dict, out);
+    GF_VALIDATE_OR_GOTO(GF_NG, dict, out);
 
-        if (val) {
-                nge = (struct netgroup_entry *)val->data;
-                if (!dict_get (__deleted_entries, key)) {
-                        _netgroup_entry_deinit (nge);
-                        val->data = NULL;
-                }
-                dict_del (dict, key);
+    if (val) {
+        nge = (struct netgroup_entry *)val->data;
+        if (!dict_get(__deleted_entries, key)) {
+            _netgroup_entry_deinit(nge);
+            val->data = NULL;
         }
+        dict_del(dict, key);
+    }
 
 out:
-        return 0;
+    return 0;
 }
 
 /**
@@ -261,53 +267,53 @@ out:
  * Not for external use.
  */
 static void
-_netgroup_entry_deinit (struct netgroup_entry *ngentry)
+_netgroup_entry_deinit(struct netgroup_entry *ngentry)
 {
-        dict_t  *ng_dict   = NULL;
-        dict_t  *host_dict = NULL;
-        char    *name      = NULL;
-        data_t  *dint      = NULL;
-
-        if (!ngentry)
-                return;
-
-        ng_dict = ngentry->netgroup_ngs;
-        host_dict = ngentry->netgroup_hosts;
-
-        if (ng_dict) {
-                /* Free the dict of netgroup entries */
-                dict_foreach (ng_dict, __nge_free_walk, NULL);
-                dict_unref (ng_dict);
-                ngentry->netgroup_ngs = NULL;
-        }
+    dict_t *ng_dict = NULL;
+    dict_t *host_dict = NULL;
+    char *name = NULL;
+    data_t *dint = NULL;
 
-        if (host_dict) {
-                /* Free the dict of host entries */
-                dict_foreach (host_dict, __ngh_free_walk, NULL);
-                dict_unref (host_dict);
-                ngentry->netgroup_hosts = NULL;
-        }
+    if (!ngentry)
+        return;
 
-        if (ngentry->netgroup_name) {
-                /* Keep track of the netgroup names we've deallocated
-                 * We need to do this because of the nature of this data
-                 * structure. This data structure may hold multiple
-                 * pointers to an already freed object, but these are
-                 * uniquely identifiable by the name. We keep track
-                 * of these names so when we encounter a key who has
-                 * an association to an already freed object, we don't
-                 * free it twice.
-                 */
-                name = strdupa (ngentry->netgroup_name);
+    ng_dict = ngentry->netgroup_ngs;
+    host_dict = ngentry->netgroup_hosts;
+
+    if (ng_dict) {
+        /* Free the dict of netgroup entries */
+        dict_foreach(ng_dict, __nge_free_walk, NULL);
+        dict_unref(ng_dict);
+        ngentry->netgroup_ngs = NULL;
+    }
+
+    if (host_dict) {
+        /* Free the dict of host entries */
+        dict_foreach(host_dict, __ngh_free_walk, NULL);
+        dict_unref(host_dict);
+        ngentry->netgroup_hosts = NULL;
+    }
+
+    if (ngentry->netgroup_name) {
+        /* Keep track of the netgroup names we've deallocated
+         * We need to do this because of the nature of this data
+         * structure. This data structure may hold multiple
+         * pointers to an already freed object, but these are
+         * uniquely identifiable by the name. We keep track
+         * of these names so when we encounter a key who has
+         * an association to an already freed object, we don't
+         * free it twice.
+         */
+        name = strdupa(ngentry->netgroup_name);
 
-                dint = int_to_data (1);
-                dict_set (__deleted_entries, name, dint);
+        dint = int_to_data(1);
+        dict_set(__deleted_entries, name, dint);
 
-                GF_FREE (ngentry->netgroup_name);
-                ngentry->netgroup_name = NULL;
-        }
+        GF_FREE(ngentry->netgroup_name);
+        ngentry->netgroup_name = NULL;
+    }
 
-        GF_FREE (ngentry);
+    GF_FREE(ngentry);
 }
 
 /**
@@ -321,11 +327,11 @@ _netgroup_entry_deinit (struct netgroup_entry *ngentry)
  * Not for external use.
  */
 static struct netgroup_host *
-_netgroup_host_init ()
+_netgroup_host_init()
 {
-        struct netgroup_host *host = GF_CALLOC (1, sizeof (*host),
-                                                gf_common_mt_nfs_netgroups);
-        return host;
+    struct netgroup_host *host = GF_CALLOC(1, sizeof(*host),
+                                           gf_common_mt_nfs_netgroups);
+    return host;
 }
 
 /**
@@ -339,23 +345,23 @@ _netgroup_host_init ()
  * Not for external use.
  */
 static void
-_netgroup_host_deinit (struct netgroup_host *host)
+_netgroup_host_deinit(struct netgroup_host *host)
 {
-        /* Validate args */
-        GF_VALIDATE_OR_GOTO (GF_NG, host, err);
+    /* Validate args */
+    GF_VALIDATE_OR_GOTO(GF_NG, host, err);
 
-        GF_FREE (host->hostname);
-        host->hostname = NULL;
+    GF_FREE(host->hostname);
+    host->hostname = NULL;
 
-        GF_FREE (host->user);
-        host->user = NULL;
+    GF_FREE(host->user);
+    host->user = NULL;
 
-        GF_FREE (host->domain);
-        host->domain = NULL;
+    GF_FREE(host->domain);
+    host->domain = NULL;
 
-        GF_FREE (host);
+    GF_FREE(host);
 err:
-        return;
+    return;
 }
 
 /**
@@ -374,19 +380,19 @@ err:
  * Not for external use.
  */
 static struct netgroup_entry *
-_nge_dict_get (dict_t *dict, const char *ngname)
+_nge_dict_get(dict_t *dict, const char *ngname)
 {
-        data_t *ngdata = NULL;
+    data_t *ngdata = NULL;
 
-        /* Validate args */
-        GF_VALIDATE_OR_GOTO (GF_NG, dict, err);
-        GF_VALIDATE_OR_GOTO (GF_NG, ngname, err);
+    /* Validate args */
+    GF_VALIDATE_OR_GOTO(GF_NG, dict, err);
+    GF_VALIDATE_OR_GOTO(GF_NG, ngname, err);
 
-        ngdata = dict_get (dict, (char *)ngname);
-        if (ngdata)
-                return (struct netgroup_entry *)ngdata->data;
+    ngdata = dict_get(dict, (char *)ngname);
+    if (ngdata)
+        return (struct netgroup_entry *)ngdata->data;
 err:
-        return NULL;
+    return NULL;
 }
 
 /**
@@ -402,17 +408,17 @@ err:
  * Not for external use.
  */
 static void
-_nge_dict_insert (dict_t *dict, struct netgroup_entry *nge)
+_nge_dict_insert(dict_t *dict, struct netgroup_entry *nge)
 {
-        data_t *ngdata = NULL;
+    data_t *ngdata = NULL;
 
-        GF_VALIDATE_OR_GOTO (GF_NG, dict, err);
-        GF_VALIDATE_OR_GOTO (GF_NG, nge, err);
+    GF_VALIDATE_OR_GOTO(GF_NG, dict, err);
+    GF_VALIDATE_OR_GOTO(GF_NG, nge, err);
 
-        ngdata = bin_to_data (nge, sizeof (*nge));
-        dict_set (dict, nge->netgroup_name, ngdata);
+    ngdata = bin_to_data(nge, sizeof(*nge));
+    dict_set(dict, nge->netgroup_name, ngdata);
 err:
-        return;
+    return;
 }
 
 /**
@@ -431,21 +437,21 @@ err:
  * Externally usable.
  */
 struct netgroup_host *
-ngh_dict_get (dict_t *dict, const char *hostname)
+ngh_dict_get(dict_t *dict, const char *hostname)
 {
-        data_t *ngdata = NULL;
+    data_t *ngdata = NULL;
 
-        GF_VALIDATE_OR_GOTO (GF_NG, dict, err);
-        GF_VALIDATE_OR_GOTO (GF_NG, hostname, err);
+    GF_VALIDATE_OR_GOTO(GF_NG, dict, err);
+    GF_VALIDATE_OR_GOTO(GF_NG, hostname, err);
 
-        ngdata = dict_get (dict, (char *)hostname);
-        if (!ngdata)
-                goto err;
+    ngdata = dict_get(dict, (char *)hostname);
+    if (!ngdata)
+        goto err;
 
-        return (struct netgroup_host *)ngdata->data;
+    return (struct netgroup_host *)ngdata->data;
 
 err:
-        return NULL;
+    return NULL;
 }
 
 /**
@@ -461,18 +467,18 @@ err:
  * Not for external use.
  */
 static void
-_ngh_dict_insert (dict_t *dict, struct netgroup_host *ngh)
+_ngh_dict_insert(dict_t *dict, struct netgroup_host *ngh)
 {
-        data_t *ngdata = NULL;
+    data_t *ngdata = NULL;
 
-        /* Validate args */
-        GF_VALIDATE_OR_GOTO (GF_NG, dict, err);
-        GF_VALIDATE_OR_GOTO (GF_NG, ngh, err);
+    /* Validate args */
+    GF_VALIDATE_OR_GOTO(GF_NG, dict, err);
+    GF_VALIDATE_OR_GOTO(GF_NG, ngh, err);
 
-        ngdata = bin_to_data (ngh, sizeof (*ngh));
-        dict_set (dict, ngh->hostname, ngdata);
+    ngdata = bin_to_data(ngh, sizeof(*ngh));
+    dict_set(dict, ngh->hostname, ngdata);
 err:
-        return;
+    return;
 }
 
 /**
@@ -486,15 +492,15 @@ err:
  * Not for external use.
  */
 static void
-_ngh_print (const struct netgroup_host *ngh)
+_ngh_print(const struct netgroup_host *ngh)
 {
-        /* Validate args */
-        GF_VALIDATE_OR_GOTO (GF_NG, ngh, err);
+    /* Validate args */
+    GF_VALIDATE_OR_GOTO(GF_NG, ngh, err);
 
-        printf ("(%s,%s,%s)", ngh->hostname, ngh->user ? ngh->user : "",
-                ngh->domain ? ngh->domain : "");
+    printf("(%s,%s,%s)", ngh->hostname, ngh->user ? ngh->user : "",
+           ngh->domain ? ngh->domain : "");
 err:
-        return;
+    return;
 }
 
 /**
@@ -514,12 +520,12 @@ err:
  * Not for external use.
  */
 static int
-__nge_print_walk (dict_t *dict, char *key, data_t *val, void *tmp)
+__nge_print_walk(dict_t *dict, char *key, data_t *val, void *tmp)
 {
-        if (val)
-                _nge_print ((struct netgroup_entry *)val->data);
+    if (val)
+        _nge_print((struct netgroup_entry *)val->data);
 
-        return 0;
+    return 0;
 }
 
 /**
@@ -540,12 +546,12 @@ __nge_print_walk (dict_t *dict, char *key, data_t *val, void *tmp)
  * Not for external use.
  */
 static int
-__ngh_print_walk (dict_t *dict, char *key, data_t *val, void *tmp)
+__ngh_print_walk(dict_t *dict, char *key, data_t *val, void *tmp)
 {
-        if (val)
-                _ngh_print ((struct netgroup_host *)val->data);
+    if (val)
+        _ngh_print((struct netgroup_host *)val->data);
 
-        return 0;
+    return 0;
 }
 
 /**
@@ -559,20 +565,20 @@ __ngh_print_walk (dict_t *dict, char *key, data_t *val, void *tmp)
  * Not for external use.
  */
 static void
-_nge_print (const struct netgroup_entry *nge)
+_nge_print(const struct netgroup_entry *nge)
 {
-        /* Validate args */
-        GF_VALIDATE_OR_GOTO (GF_NG, nge, err);
+    /* Validate args */
+    GF_VALIDATE_OR_GOTO(GF_NG, nge, err);
 
-        printf ("%s ", nge->netgroup_name);
-        if (nge->netgroup_ngs)
-                dict_foreach (nge->netgroup_ngs, __nge_print_walk, NULL);
+    printf("%s ", nge->netgroup_name);
+    if (nge->netgroup_ngs)
+        dict_foreach(nge->netgroup_ngs, __nge_print_walk, NULL);
 
-        if (nge->netgroup_hosts)
-                dict_foreach (nge->netgroup_hosts, __ngh_print_walk, NULL);
+    if (nge->netgroup_hosts)
+        dict_foreach(nge->netgroup_hosts, __ngh_print_walk, NULL);
 
 err:
-        return;
+    return;
 }
 
 /**
@@ -593,16 +599,16 @@ err:
  * Not for external use.
  */
 static int
-__ngf_print_walk (dict_t *dict, char *key, data_t *val, void *tmp)
+__ngf_print_walk(dict_t *dict, char *key, data_t *val, void *tmp)
 {
-        struct netgroup_entry *snge = NULL;
-
-        if (val) {
-                snge = (struct netgroup_entry *)val->data;
-                _nge_print (snge);
-                printf ("\n");
-        }
-        return 0;
+    struct netgroup_entry *snge = NULL;
+
+    if (val) {
+        snge = (struct netgroup_entry *)val->data;
+        _nge_print(snge);
+        printf("\n");
+    }
+    return 0;
 }
 
 /**
@@ -622,9 +628,9 @@ __ngf_print_walk (dict_t *dict, char *key, data_t *val, void *tmp)
  * Can be called on any valid 'struct netgroups_file *' type.
  */
 void
-ng_file_print (const struct netgroups_file *ngfile)
+ng_file_print(const struct netgroups_file *ngfile)
 {
-     dict_foreach (ngfile->ng_file_dict, __ngf_print_walk, NULL);
+    dict_foreach(ngfile->ng_file_dict, __ngf_print_walk, NULL);
 }
 
 /**
@@ -643,22 +649,21 @@ ng_file_print (const struct netgroups_file *ngfile)
  * *' as the lookup key.
  */
 struct netgroup_entry *
-ng_file_get_netgroup (const struct netgroups_file *ngfile, const char *netgroup)
+ng_file_get_netgroup(const struct netgroups_file *ngfile, const char *netgroup)
 {
-        data_t *ndata = NULL;
+    data_t *ndata = NULL;
 
-        GF_VALIDATE_OR_GOTO (GF_NG, ngfile, err);
-        GF_VALIDATE_OR_GOTO (GF_NG, netgroup, err);
+    GF_VALIDATE_OR_GOTO(GF_NG, ngfile, err);
+    GF_VALIDATE_OR_GOTO(GF_NG, netgroup, err);
 
-        ndata = dict_get (ngfile->ng_file_dict,
-                          (char *)netgroup);
-        if (!ndata)
-                goto err;
+    ndata = dict_get(ngfile->ng_file_dict, (char *)netgroup);
+    if (!ndata)
+        goto err;
 
-        return (struct netgroup_entry *)ndata->data;
+    return (struct netgroup_entry *)ndata->data;
 
 err:
-        return NULL;
+    return NULL;
 }
 
 /**
@@ -676,30 +681,30 @@ err:
  * Not for external use.
  */
 static gf_boolean_t
-__check_host_entry_str (const char *host_str)
+__check_host_entry_str(const char *host_str)
 {
-        unsigned int comma_count = 0;
-        unsigned int i           = 0;
-        gf_boolean_t str_valid   = _gf_true;
+    unsigned int comma_count = 0;
+    unsigned int i = 0;
+    gf_boolean_t str_valid = _gf_true;
 
-        GF_VALIDATE_OR_GOTO (GF_NG, host_str, out);
+    GF_VALIDATE_OR_GOTO(GF_NG, host_str, out);
 
-        for (i = 0; i < strlen (host_str); i++) {
-                if (host_str[i] == ',')
-                        comma_count++;
+    for (i = 0; i < strlen(host_str); i++) {
+        if (host_str[i] == ',')
+            comma_count++;
 
-                /* Spaces are not allowed in this string. e.g, (a,b,c) is valid
-                 * but (a, b,c) is not.
-                 */
-                if (host_str[i] == ' ') {
-                        str_valid = _gf_false;
-                        goto out;
-                }
+        /* Spaces are not allowed in this string. e.g, (a,b,c) is valid
+         * but (a, b,c) is not.
+         */
+        if (host_str[i] == ' ') {
+            str_valid = _gf_false;
+            goto out;
         }
+    }
 
-        str_valid = (comma_count == 2);
+    str_valid = (comma_count == 2);
 out:
-        return str_valid;
+    return str_valid;
 }
 
 /**
@@ -715,61 +720,65 @@ out:
  * Not for external use.
  */
 static int
-_parse_ng_host (char *ng_str, struct netgroup_host **ngh)
+_parse_ng_host(char *ng_str, struct netgroup_host **ngh)
 {
-        struct netgroup_host *ng_host  = NULL;
-        unsigned int          parts    = 0;
-        char                 *match    = NULL;
-        int                   ret      = -EINVAL;
-
-        GF_VALIDATE_OR_GOTO (GF_NG, ng_str, out);
-        GF_VALIDATE_OR_GOTO (GF_NG, ngh, out);
-
-        if (!__check_host_entry_str (ng_str)) {
-                ret = 1; /* Parse failed */
-                goto out;
-        }
-
-        ret = parser_set_string (ng_host_parser, ng_str);
-        if (ret < 0)
-                goto out;
-
-        ng_host = _netgroup_host_init ();
-        GF_CHECK_ALLOC (ng_host, ret, free_and_out); /* Sets ret to -ENOMEM on
-                                                      * failure.
-                                                      */
-        while ((match = parser_get_next_match (ng_host_parser)) != NULL) {
-                switch (parts) {
-                case 0:
-                        ng_host->hostname = match;
-                        break;
-                case 1:
-                        ng_host->user = match;
-                        break;
-                case 2:
-                        ng_host->domain = match;
-                        break;
-                default:
-                        GF_FREE (match);
-                        break;
-                };
-
-                /* We only allow three parts in the host string;
-                 * The format for the string is (a,b,c)
-                 */
-                parts++;
-                if (parts > 2)
-                        break;
-        }
+    struct netgroup_host *ng_host = NULL;
+    unsigned int parts = 0;
+    char *match = NULL;
+    int ret = -EINVAL;
+
+    GF_VALIDATE_OR_GOTO(GF_NG, ng_str, out);
+    GF_VALIDATE_OR_GOTO(GF_NG, ngh, out);
+
+    if (!__check_host_entry_str(ng_str)) {
+        ret = 1; /* Parse failed */
+        goto out;
+    }
+
+    ret = parser_set_string(ng_host_parser, ng_str);
+    if (ret < 0)
+        goto out;
+
+    gf_msg_trace(GF_NG, 0, "parsing host string: %s", ng_str);
+
+    ng_host = _netgroup_host_init();
+    GF_CHECK_ALLOC(ng_host, ret, free_and_out); /* Sets ret to -ENOMEM on
+                                                 * failure.
+                                                 */
+    while ((match = parser_get_next_match(ng_host_parser)) != NULL) {
+        gf_msg_trace(GF_NG, 0, "found match: %s (parts=%d)", match, parts);
+
+        switch (parts) {
+            case 0:
+                ng_host->hostname = match;
+                break;
+            case 1:
+                ng_host->user = match;
+                break;
+            case 2:
+                ng_host->domain = match;
+                break;
+            default:
+                GF_FREE(match);
+                break;
+        };
+
+        /* We only allow three parts in the host string;
+         * The format for the string is (a,b,c)
+         */
+        parts++;
+        if (parts > 2)
+            break;
+    }
 
-        /* Set the parameter */
-        *ngh = ng_host;
-        ret = 0;
+    /* Set the parameter */
+    *ngh = ng_host;
+    ret = 0;
 
 free_and_out:
-        parser_unset_string (ng_host_parser);
+    parser_unset_string(ng_host_parser);
 out:
-        return ret;
+    return ret;
 }
 
 /**
@@ -788,46 +797,47 @@ out:
  * Not for external use.
  */
 static int
-_ng_handle_host_part (char *match, struct netgroup_entry *ngp)
+_ng_handle_host_part(char *match, struct netgroup_entry *ngp)
 {
-        struct netgroup_host *ngh = NULL;
-        int                   ret = -EINVAL;
-
-        GF_VALIDATE_OR_GOTO (GF_NG, match, out);
-        GF_VALIDATE_OR_GOTO (GF_NG, ngp, out);
-
-        if (!ngp->netgroup_name) {
-                gf_log (GF_NG, GF_LOG_WARNING,
-                                "Invalid: Line starts with hostname!");
-                goto out;
-        }
-
-        /* Parse the host string and get a struct for it */
-        ret = _parse_ng_host (match, &ngh);
-        if (ret < 0) {
-                gf_log (GF_NG, GF_LOG_CRITICAL,
-                        "Critical error : %s", strerror (-ret));
-                goto out;
-        }
-        if (ret != 0) {
-                gf_log (GF_NG, GF_LOG_WARNING,
-                        "Parse error for: %s", match);
-                goto out;
-        }
-
+    struct netgroup_host *ngh = NULL;
+    int ret = -EINVAL;
+
+    GF_VALIDATE_OR_GOTO(GF_NG, match, out);
+    GF_VALIDATE_OR_GOTO(GF_NG, ngp, out);
+
+    if (!ngp->netgroup_name) {
+        gf_msg(GF_NG, GF_LOG_WARNING, EINVAL, NFS_MSG_INVALID_ENTRY,
+               "Invalid: Line starts with hostname!");
+        goto out;
+    }
+
+    /* Parse the host string and get a struct for it */
+    ret = _parse_ng_host(match, &ngh);
+    if (ret < 0) {
+        gf_msg(GF_NG, GF_LOG_CRITICAL, -ret, NFS_MSG_PARSE_FAIL,
+               "Critical error : %s", strerror(-ret));
+        goto out;
+    }
+    if (ret != 0) {
+        /* Cannot change to gf_msg
+         * gf_msg not giving output to STDOUT
+         * Bug id : BZ1215017
+         */
+        gf_log(GF_NG, GF_LOG_WARNING, "Parse error for: %s", match);
+        goto out;
+    }
 
-        /* Make dict for the parent entry's netgroup hosts */
-        if (!ngp->netgroup_hosts) {
-                ngp->netgroup_hosts = dict_new ();
-                GF_CHECK_ALLOC (ngp->netgroup_hosts, ret,
-                                out);
-        }
+    /* Make dict for the parent entry's netgroup hosts */
+    if (!ngp->netgroup_hosts) {
+        ngp->netgroup_hosts = dict_new();
+        GF_CHECK_ALLOC(ngp->netgroup_hosts, ret, out);
+    }
 
-        /* Insert this entry into the parent netgroup dict */
-        _ngh_dict_insert (ngp->netgroup_hosts, ngh);
+    /* Insert this entry into the parent netgroup dict */
+    _ngh_dict_insert(ngp->netgroup_hosts, ngh);
 
 out:
-        return ret;
+    return ret;
 }
 
 /**
@@ -849,29 +859,29 @@ out:
  * Not for external use.
  */
 static int
-_ng_setup_netgroup_entry (char *match, struct netgroups_file *file,
-                          struct netgroup_entry **ng_entry)
+_ng_setup_netgroup_entry(char *match, struct netgroups_file *file,
+                         struct netgroup_entry **ng_entry)
 {
-        struct netgroup_entry *nge           = NULL;
-        int                    ret           = -EINVAL;
+    struct netgroup_entry *nge = NULL;
+    int ret = -EINVAL;
 
-        GF_VALIDATE_OR_GOTO (GF_NG, match, out);
-        GF_VALIDATE_OR_GOTO (GF_NG, file, out);
-        GF_VALIDATE_OR_GOTO (GF_NG, ng_entry, out);
+    GF_VALIDATE_OR_GOTO(GF_NG, match, out);
+    GF_VALIDATE_OR_GOTO(GF_NG, file, out);
+    GF_VALIDATE_OR_GOTO(GF_NG, ng_entry, out);
 
-        nge = _netgroup_entry_init ();
-        GF_CHECK_ALLOC (nge, ret, out);
+    nge = _netgroup_entry_init();
+    GF_CHECK_ALLOC(nge, ret, out);
 
-        nge->netgroup_name = match;
+    nge->netgroup_name = match;
 
-        /* Insert this new entry into the file dict */
-        _nge_dict_insert (file->ng_file_dict, nge);
+    /* Insert this new entry into the file dict */
+    _nge_dict_insert(file->ng_file_dict, nge);
 
-        *ng_entry = nge;
+    *ng_entry = nge;
 
-        ret = 0;
+    ret = 0;
 out:
-        return ret;
+    return ret;
 }
 
 /**
@@ -940,112 +950,118 @@ out:
  * Not for external use.
  */
 static int
-_parse_ng_line (char *ng_str, struct netgroups_file *file,
-                struct netgroup_entry **ng_entry)
+_parse_ng_line(char *ng_str, struct netgroups_file *file,
+               struct netgroup_entry **ng_entry)
 {
-        struct netgroup_entry  *ngp         = NULL; /* Parent netgroup entry */
-        struct netgroup_entry  *nge         = NULL; /* Generic netgroup entry */
-        char                   *match       = NULL;
-        int                     ret         = -EINVAL;
-        unsigned int            num_entries = 0;
-
-        /* Validate arguments */
-        GF_VALIDATE_OR_GOTO (GF_NG, ng_str, out);
-        GF_VALIDATE_OR_GOTO (GF_NG, file, out);
-
-        if (*ng_str == ' ' || *ng_str == '\0' || *ng_str == '\n') {
-                ret = 0;
-                goto out;
-        }
+    struct netgroup_entry *ngp = NULL; /* Parent netgroup entry */
+    struct netgroup_entry *nge = NULL; /* Generic netgroup entry */
+    char *match = NULL;
+    int ret = -EINVAL;
+    unsigned int num_entries = 0;
 
-        ret = parser_set_string (ng_file_parser, ng_str);
-        if (ret < 0)
-                goto out;
+    /* Validate arguments */
+    GF_VALIDATE_OR_GOTO(GF_NG, ng_str, out);
+    GF_VALIDATE_OR_GOTO(GF_NG, file, out);
 
-        /* This is the first name in the line, and should be the
-         * parent netgroup entry.
-         */
-        match = parser_get_next_match (ng_file_parser);
-        if (!match) {
-                ret = 1;
-                gf_log (GF_NG, GF_LOG_WARNING, "Unable to find first match.");
-                gf_log (GF_NG, GF_LOG_WARNING, "Error parsing str: %s", ng_str);
-                goto out;
+    if (*ng_str == ' ' || *ng_str == '\0' || *ng_str == '\n') {
+        ret = 0;
+        goto out;
+    }
+
+    ret = parser_set_string(ng_file_parser, ng_str);
+    if (ret < 0)
+        goto out;
+
+    /* This is the first name in the line, and should be the
+     * parent netgroup entry.
+     */
+    match = parser_get_next_match(ng_file_parser);
+    if (!match) {
+        ret = 1;
+        gf_msg(GF_NG, GF_LOG_WARNING, 0, NFS_MSG_FIND_FIRST_MATCH_FAIL,
+               "Unable to find "
+               "first match.");
+        gf_msg(GF_NG, GF_LOG_WARNING, 0, NFS_MSG_PARSE_FAIL,
+               "Error parsing str: %s", ng_str);
+        goto out;
+    }
+
+    /* Lookup to see if the match already exists,
+     * if not, set the parent.
+     */
+    ngp = _nge_dict_get(file->ng_file_dict, match);
+    if (!ngp) {
+        ret = _ng_setup_netgroup_entry(match, file, &ngp);
+        if (ret < 0) {
+            /* Bubble up error to caller. We don't need to free ngp
+             * here because this can only fail if allocating the
+             * struct fails.
+             */
+            goto out;
         }
+    } else
+        GF_FREE(match);
 
-        /* Lookup to see if the match already exists,
-         * if not, set the parent.
+    if (!ngp->netgroup_ngs) {
+        /* If a netgroup dict has not been allocated
+         * for this parent, allocate it.
+         */
+        ngp->netgroup_ngs = dict_new();
+        GF_CHECK_ALLOC(ngp->netgroup_ngs, ret, out);
+        /* No need to free anything here since ngp is already
+         * a part of the file. When the file gets
+         * deallocated, we will free ngp.
          */
-        ngp = _nge_dict_get (file->ng_file_dict, match);
-        if (!ngp) {
-                ret = _ng_setup_netgroup_entry (match, file, &ngp);
+    }
+
+    while ((match = parser_get_next_match(ng_file_parser)) != NULL) {
+        num_entries++;
+        /* This means that we hit a host entry in the line */
+        if (*match == '(') {
+            ret = _ng_handle_host_part(match, ngp);
+            GF_FREE(match);
+            if (ret != 0) {
+                /* If parsing the host fails, bubble the error
+                 * code up to the caller.
+                 */
+                goto out;
+            }
+        } else {
+            nge = _nge_dict_get(file->ng_file_dict, match);
+            if (!nge) {
+                ret = _ng_setup_netgroup_entry(match, file, &nge);
                 if (ret < 0) {
-                        /* Bubble up error to caller. We don't need to free ngp
-                         * here because this can only fail if allocating the
-                         * struct fails.
-                         */
-                        goto out;
+                    /* Bubble up error to caller. We don't
+                     * need to free nge here because this
+                     * can only fail if allocating the
+                     * struct fails.
+                     */
+                    goto out;
                 }
-        } else
-                GF_FREE (match);
+            } else
+                GF_FREE(match);
 
-        if (!ngp->netgroup_ngs) {
-                /* If a netgroup dict has not been allocated
-                 * for this parent, allocate it.
-                 */
-                ngp->netgroup_ngs = dict_new ();
-                GF_CHECK_ALLOC (ngp->netgroup_ngs, ret, out);
-                /* No need to free anything here since ngp is already
-                 * a part of the file. When the file gets
-                 * deallocated, we will free ngp.
-                 */
+            /* Insert the netgroup into the parent's dict */
+            _nge_dict_insert(ngp->netgroup_ngs, nge);
         }
+    }
 
-        while ((match = parser_get_next_match (ng_file_parser)) != NULL) {
-                num_entries++;
-                /* This means that we hit a host entry in the line */
-                if (*match == '(') {
-                        ret = _ng_handle_host_part (match, ngp);
-                        GF_FREE (match);
-                        if (ret != 0) {
-                                /* If parsing the host fails, bubble the error
-                                 * code up to the caller.
-                                 */
-                                goto out;
-                        }
-                } else {
-                        nge = _nge_dict_get (file->ng_file_dict, match);
-                        if (!nge) {
-                                ret = _ng_setup_netgroup_entry (match, file,
-                                                                &nge);
-                                if (ret < 0) {
-                                        /* Bubble up error to caller. We don't
-                                         * need to free nge here because this
-                                         * can only fail if allocating the
-                                         * struct fails.
-                                         */
-                                        goto out;
-                                }
-                        } else
-                                GF_FREE (match);
-
-                        /* Insert the netgroup into the parent's dict */
-                        _nge_dict_insert (ngp->netgroup_ngs, nge);
-                }
-        }
+    /* If there are no entries on the RHS, log an error, but continue */
+    if (!num_entries) {
+        /* Cannot change to gf_msg
+         * gf_msg not giving output to STDOUT
+         * Bug id : BZ1215017
+         */
+        gf_log(GF_NG, GF_LOG_WARNING,
+               "No netgroups were specified except for the parent.");
+    }
 
-        /* If there are no entries on the RHS, log an error, but continue */
-        if (!num_entries) {
-                gf_log (GF_NG, GF_LOG_WARNING, "No netgroups were specified "
-                                               "except for the parent.");
-        }
-
-        *ng_entry = ngp;
-        ret = 0;
+    *ng_entry = ngp;
+    ret = 0;
 
 out:
-        parser_unset_string (ng_file_parser);
-        return ret;
+    parser_unset_string(ng_file_parser);
+    return ret;
 }
 
 /**
@@ -1063,80 +1079,83 @@ out:
  * Externally facing function
  */
 struct netgroups_file *
-ng_file_parse (const char *filepath)
+ng_file_parse(const char *filepath)
 {
-        FILE                  *fp    = NULL;
-        size_t                 len   = 0;
-        size_t                 read  = 0;
-        char                  *line  = NULL;
-        struct netgroups_file *file  = NULL;
-        struct netgroup_entry *nge   = NULL;
-        int                    ret   = 0;
-
-        GF_VALIDATE_OR_GOTO (GF_NG, filepath, err);
-
-        fp = fopen (filepath, "r");
-        if (!fp)
-                goto err;
-
-        file = _netgroups_file_init ();
-        if (!file)
-                goto err;
-
-        file->ng_file_dict = dict_new ();
-        if (!file->ng_file_dict) {
-                gf_log (GF_NG, GF_LOG_CRITICAL,
-                        "Failed to allocate netgroup file dict");
-                goto err;
+    FILE *fp = NULL;
+    size_t len = 0;
+    size_t read = 0;
+    char *line = NULL;
+    struct netgroups_file *file = NULL;
+    struct netgroup_entry *nge = NULL;
+    int ret = 0;
+
+    GF_VALIDATE_OR_GOTO(GF_NG, filepath, err);
+
+    fp = fopen(filepath, "r");
+    if (!fp)
+        goto err;
+
+    file = _netgroups_file_init();
+    if (!file)
+        goto err;
+
+    file->ng_file_dict = dict_new();
+    if (!file->ng_file_dict) {
+        gf_msg(GF_NG, GF_LOG_CRITICAL, ENOMEM, NFS_MSG_NO_MEMORY,
+               "Failed to allocate netgroup file dict");
+        goto err;
+    }
+
+    file->filename = gf_strdup(filepath);
+    if (!file->filename) {
+        gf_msg(GF_NG, GF_LOG_CRITICAL, errno, NFS_MSG_FILE_OP_FAILED,
+               "Failed to duplicate filename");
+        goto err;
+    }
+
+    ret = _ng_init_parsers();
+    if (ret < 0)
+        goto err;
+
+    /* Read the file line-by-line and parse it */
+    while ((read = getline(&line, &len, fp)) != -1) {
+        if (*line == '#') /* Lines starting with # are comments */
+            continue;
+
+        /* Parse the line into a netgroup entry */
+        ret = _parse_ng_line(line, file, &nge);
+        if (ret == -ENOMEM) {
+            gf_msg(GF_NG, GF_LOG_CRITICAL, ENOMEM, NFS_MSG_NO_MEMORY,
+                   "Allocation error "
+                   "while parsing line!");
+            goto err;
         }
-
-        file->filename = gf_strdup (filepath);
-        if (!file->filename) {
-                gf_log (GF_NG, GF_LOG_CRITICAL,
-                        "Failed to duplicate filename");
-                goto err;
+        if (ret != 0) {
+            gf_msg_debug(GF_NG, 0, "Failed to parse line %s", line);
+            continue;
         }
+    }
 
-        ret = _ng_init_parsers ();
-        if (ret < 0)
-                goto err;
-
-        /* Read the file line-by-line and parse it */
-        while ((read = getline (&line, &len, fp)) != -1) {
-                if (*line == '#') /* Lines starting with # are comments */
-                        continue;
-
-                /* Parse the line into a netgroup entry */
-                ret = _parse_ng_line (line, file, &nge);
-                if (ret == -ENOMEM) {
-                        gf_log (GF_NG, GF_LOG_CRITICAL, "Allocation error "
-                                                        "while parsing line!");
-                        ng_file_deinit (file);
-                        GF_FREE (line);
-                        goto err;
-                }
-                if (ret != 0) {
-                        gf_log (GF_NG, GF_LOG_DEBUG,
-                                "Failed to parse line %s", line);
-                        continue;
-                }
-        }
+    /* line got allocated through getline(), don't use GF_FREE() for it */
+    free(line);
 
-        /* line got allocated through getline(), don't use GF_FREE() for it */
-        free (line);
+    if (fp)
+        fclose(fp);
 
-        if (fp)
-                fclose(fp);
+    _ng_deinit_parsers();
 
-        return file;
+    return file;
 
 err:
-        if (file)
-                ng_file_deinit (file);
+    if (line)
+        free(line);
+
+    if (file)
+        ng_file_deinit(file);
 
-        _ng_deinit_parsers ();
+    _ng_deinit_parsers();
 
-        if (fp)
-                fclose (fp);
-        return NULL;
+    if (fp)
+        fclose(fp);
+    return NULL;
 }
diff --git a/xlators/nfs/server/src/netgroups.h b/xlators/nfs/server/src/netgroups.h
index c77a35a41f3..9c715f75d3c 100644
--- a/xlators/nfs/server/src/netgroups.h
+++ b/xlators/nfs/server/src/netgroups.h
@@ -16,39 +16,38 @@
 #define _NETGROUPS_H
 
 #include "nfs-mem-types.h"
-#include "dict.h"
+#include <glusterfs/dict.h>
 #include "nfs.h"
 
-#define GF_NG GF_NFS"-netgroup"
+#define GF_NG GF_NFS "-netgroup"
 
-#define NG_FILE_PARSE_REGEX "([a-zA-Z0-9.(,)]+)"
-#define NG_HOST_PARSE_REGEX "([a-zA-Z0-9.]+)"
+#define NG_FILE_PARSE_REGEX "([a-zA-Z0-9.(,)-]+)"
+#define NG_HOST_PARSE_REGEX "([a-zA-Z0-9.-]+)"
 
 struct netgroup_host {
-        char *hostname;         /* Hostname of entry */
-        char *user;             /* User field in the entry */
-        char *domain;           /* Domain field in the entry */
+    char *hostname; /* Hostname of entry */
+    char *user;     /* User field in the entry */
+    char *domain;   /* Domain field in the entry */
 };
 
 struct netgroup_entry {
-        char    *netgroup_name;         /* Name of the netgroup */
-        dict_t  *netgroup_ngs;          /* Dict of netgroups in this netgroup */
-        dict_t  *netgroup_hosts;        /* Dict of hosts in this netgroup. */
+    char *netgroup_name;    /* Name of the netgroup */
+    dict_t *netgroup_ngs;   /* Dict of netgroups in this netgroup */
+    dict_t *netgroup_hosts; /* Dict of hosts in this netgroup. */
 };
 
 struct netgroups_file {
-        char    *filename;         /* Filename on disk */
-        dict_t  *ng_file_dict;   /* Dict of netgroup entries */
+    char *filename;       /* Filename on disk */
+    dict_t *ng_file_dict; /* Dict of netgroup entries */
 };
 
 struct netgroups_file *
-ng_file_parse (const char *filepath);
+ng_file_parse(const char *filepath);
 
 struct netgroup_entry *
-ng_file_get_netgroup (const struct netgroups_file *ngfile,
-                      const char *netgroup);
+ng_file_get_netgroup(const struct netgroups_file *ngfile, const char *netgroup);
 
 void
-ng_file_deinit (struct netgroups_file *ngfile);
+ng_file_deinit(struct netgroups_file *ngfile);
 
 #endif /* _NETGROUPS_H */
diff --git a/xlators/nfs/server/src/nfs-common.c b/xlators/nfs/server/src/nfs-common.c
index 547e5b3a833..b8f6b6f318e 100644
--- a/xlators/nfs/server/src/nfs-common.c
+++ b/xlators/nfs/server/src/nfs-common.c
@@ -8,312 +8,298 @@
   cases as published by the Free Software Foundation.
 */
 
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
-
 #include "rpcsvc.h"
-#include "dict.h"
-#include "xlator.h"
+#include <glusterfs/dict.h>
+#include <glusterfs/xlator.h>
 #include "xdr-nfs3.h"
 #include "msg-nfs3.h"
-#include "iobuf.h"
+#include <glusterfs/iobuf.h>
 #include "nfs-common.h"
 #include "nfs-fops.h"
 #include "nfs-mem-types.h"
 #include "rpcsvc.h"
-#include "iatt.h"
+#include <glusterfs/iatt.h>
+#include "nfs-messages.h"
 
 #include <libgen.h>
 
 xlator_t *
-nfs_xlid_to_xlator (xlator_list_t *cl, uint8_t xlid)
+nfs_xlid_to_xlator(xlator_list_t *cl, uint8_t xlid)
 {
-        xlator_t        *xl = NULL;
-        uint8_t         id = 0;
-
-        while (id <= xlid) {
-                if (!cl) {
-                        xl = NULL;
-                        break;
-                }
-
-                xl = cl->xlator;
-                cl = cl->next;
-                id++;
+    xlator_t *xl = NULL;
+    uint8_t id = 0;
+
+    while (id <= xlid) {
+        if (!cl) {
+            xl = NULL;
+            break;
         }
 
-        return xl;
-}
+        xl = cl->xlator;
+        cl = cl->next;
+        id++;
+    }
 
+    return xl;
+}
 
 xlator_t *
-nfs_path_to_xlator (xlator_list_t *cl, char *path)
+nfs_path_to_xlator(xlator_list_t *cl, char *path)
 {
-        return NULL;
+    return NULL;
 }
 
-
 uint16_t
-nfs_xlator_to_xlid (xlator_list_t *cl, xlator_t *xl)
+nfs_xlator_to_xlid(xlator_list_t *cl, xlator_t *xl)
 {
-        uint16_t        xlid = 0;
+    uint16_t xlid = 0;
 
-        if ((!cl) || (!xl))
-                return 0;
+    if ((!cl) || (!xl))
+        return 0;
 
-        while (cl) {
-                if (xl == cl->xlator)
-                        break;
-                cl = cl->next;
-                ++xlid;
-        }
+    while (cl) {
+        if (xl == cl->xlator)
+            break;
+        cl = cl->next;
+        ++xlid;
+    }
 
-        return xlid;
+    return xlid;
 }
 
-
 xlator_t *
-nfs_mntpath_to_xlator (xlator_list_t *cl, char *path)
+nfs_mntpath_to_xlator(xlator_list_t *cl, char *path)
 {
-        char            *volname = NULL;
-        char            *volptr = NULL;
-        size_t           pathlen;
-        xlator_t        *targetxl = NULL;
-
-        if ((!cl) || (!path))
-                return NULL;
-
-        volname = strdupa (path);
-        pathlen = strlen (volname);
-        gf_log (GF_NFS, GF_LOG_TRACE, "Subvolume search: %s", path);
-        if (volname[0] == '/')
-                volptr = &volname[1];
-        else
-                volptr = &volname[0];
-
-        if (pathlen && volname[pathlen - 1] == '/')
-                volname[pathlen - 1] = '\0';
-
-        while (cl) {
-                if (strcmp (volptr, cl->xlator->name) == 0) {
-                        targetxl = cl->xlator;
-                        break;
-                }
-
-                cl = cl->next;
-        }
+    char *volname = NULL; /* volume name only */
+    char *volptr = NULL;  /* ptr to original volname */
+    size_t pathlen = -1;
+    xlator_t *targetxl = NULL;
+    int i = 0;
 
-        return targetxl;
+    if ((!cl) || (!path))
+        return NULL;
 
-}
+    gf_msg_trace(GF_NFS, 0, "Subvolume search: %s", path);
 
+    volname = volptr = gf_strdup(path);
+    if (!volname)
+        return NULL;
 
-/* Returns 1 if the stat seems to be filled with zeroes. */
-int
-nfs_zero_filled_stat (struct iatt *buf)
-{
-        if (!buf)
-                return 1;
+    if (volname[0] == '/')
+        volname++;
 
-        /* Do not use st_dev because it is transformed to store the xlator id
-         * in place of the device number. Do not use st_ino because by this time
-         * we've already mapped the root ino to 1 so it is not guaranteed to be
-         * 0.
-         */
-        if ((buf->ia_nlink == 0) && (buf->ia_ctime == 0))
-                return 1;
+    pathlen = strlen(volname);
+    for (i = 0; i < pathlen; i++) {
+        if (volname[i] == '/') {
+            volname[i] = '\0';
+            break;
+        }
+    }
 
-        return 0;
-}
+    while (cl) {
+        gf_msg_trace(GF_NFS, 0, "Volname: %s and cl->xlator->name: %s", volname,
+                     cl->xlator->name);
 
+        if (strcmp(volname, cl->xlator->name) == 0) {
+            targetxl = cl->xlator;
+            break;
+        }
+
+        cl = cl->next;
+    }
+
+    GF_FREE(volptr);
+
+    return targetxl;
+}
 
 void
-nfs_loc_wipe (loc_t *loc)
+nfs_loc_wipe(loc_t *loc)
 {
-        loc_wipe (loc);
+    loc_wipe(loc);
 }
 
-
 int
-nfs_loc_copy (loc_t *dst, loc_t *src)
+nfs_loc_copy(loc_t *dst, loc_t *src)
 {
-        int  ret = -1;
+    int ret = -1;
 
-        ret = loc_copy (dst, src);
+    ret = loc_copy(dst, src);
 
-        return ret;
+    return ret;
 }
 
-
 int
-nfs_loc_fill (loc_t *loc, inode_t *inode, inode_t *parent, char *path)
+nfs_loc_fill(loc_t *loc, inode_t *inode, inode_t *parent, char *path)
 {
-        int     ret = -EFAULT;
+    int ret = -EFAULT;
 
-        if (!loc)
-                return ret;
-
-        if (inode) {
-                loc->inode = inode_ref (inode);
-                if (!gf_uuid_is_null (inode->gfid))
-                        gf_uuid_copy (loc->gfid, inode->gfid);
-        }
+    if (!loc)
+        return ret;
 
-        if (parent)
-                loc->parent = inode_ref (parent);
-
-        if (path) {
-                loc->path = gf_strdup (path);
-                if (!loc->path) {
-                        gf_log (GF_NFS, GF_LOG_ERROR, "strdup failed");
-                        goto loc_wipe;
-                }
-                loc->name = strrchr (loc->path, '/');
-                if (loc->name)
-                        loc->name++;
+    if (inode) {
+        loc->inode = inode_ref(inode);
+        if (!gf_uuid_is_null(inode->gfid))
+            gf_uuid_copy(loc->gfid, inode->gfid);
+    }
+
+    if (parent)
+        loc->parent = inode_ref(parent);
+
+    if (path) {
+        loc->path = gf_strdup(path);
+        if (!loc->path) {
+            gf_msg(GF_NFS, GF_LOG_ERROR, ENOMEM, NFS_MSG_NO_MEMORY,
+                   "strdup failed");
+            goto loc_wipe;
         }
+        loc->name = strrchr(loc->path, '/');
+        if (loc->name)
+            loc->name++;
+    }
 
-        ret = 0;
+    ret = 0;
 loc_wipe:
-        if (ret < 0)
-                nfs_loc_wipe (loc);
+    if (ret < 0)
+        nfs_loc_wipe(loc);
 
-        return ret;
+    return ret;
 }
 
-
 int
-nfs_inode_loc_fill (inode_t *inode, loc_t *loc, int how)
+nfs_inode_loc_fill(inode_t *inode, loc_t *loc, int how)
 {
-        char            *resolvedpath = NULL;
-        inode_t         *parent = NULL;
-        int             ret = -EFAULT;
-
-        if ((!inode) || (!loc))
-                return ret;
-
-        /* If gfid is not null, then the inode is already linked to
-         * the inode table, and not a newly created one. For newly
-         * created inode, inode_path returns null gfid as the path.
-         */
-        if (!gf_uuid_is_null (inode->gfid)) {
-                ret = inode_path (inode, NULL, &resolvedpath);
-                if (ret < 0) {
-                        gf_log (GF_NFS, GF_LOG_ERROR, "path resolution failed "
-                                "%s", resolvedpath);
-                        goto err;
-                }
-        }
+    char *resolvedpath = NULL;
+    inode_t *parent = NULL;
+    int ret = -EFAULT;
 
-        if (resolvedpath == NULL) {
-                char tmp_path[GFID_STR_PFX_LEN + 1] = {0,};
-                snprintf (tmp_path, sizeof (tmp_path), "<gfid:%s>",
-                          uuid_utoa (loc->gfid));
-                resolvedpath = gf_strdup (tmp_path);
-        } else {
-                parent = inode_parent (inode, loc->pargfid, NULL);
-        }
+    if ((!inode) || (!loc))
+        return ret;
 
-        ret = nfs_loc_fill (loc, inode, parent, resolvedpath);
+    /* If gfid is not null, then the inode is already linked to
+     * the inode table, and not a newly created one. For newly
+     * created inode, inode_path returns null gfid as the path.
+     */
+    if (!gf_uuid_is_null(inode->gfid)) {
+        ret = inode_path(inode, NULL, &resolvedpath);
         if (ret < 0) {
-                gf_log (GF_NFS, GF_LOG_ERROR, "loc fill resolution failed %s",
-                                resolvedpath);
-                goto err;
+            gf_msg(GF_NFS, GF_LOG_ERROR, 0, NFS_MSG_PATH_RESOLVE_FAIL,
+                   "path resolution "
+                   "failed %s",
+                   resolvedpath);
+            goto err;
         }
-
-        ret = 0;
+    }
+
+    if (resolvedpath == NULL) {
+        char tmp_path[GFID_STR_PFX_LEN + 1] = {
+            0,
+        };
+        snprintf(tmp_path, sizeof(tmp_path), "<gfid:%s>", uuid_utoa(loc->gfid));
+        resolvedpath = gf_strdup(tmp_path);
+    } else {
+        parent = inode_parent(inode, loc->pargfid, NULL);
+    }
+
+    ret = nfs_loc_fill(loc, inode, parent, resolvedpath);
+    if (ret < 0) {
+        gf_msg(GF_NFS, GF_LOG_ERROR, -ret, NFS_MSG_LOC_FILL_RESOLVE_FAIL,
+               "loc fill resolution failed %s", resolvedpath);
+        goto err;
+    }
+
+    ret = 0;
 err:
-        if (parent)
-                inode_unref (parent);
+    if (parent)
+        inode_unref(parent);
 
-        GF_FREE (resolvedpath);
+    GF_FREE(resolvedpath);
 
-        return ret;
+    return ret;
 }
 
 int
-nfs_gfid_loc_fill (inode_table_t *itable, uuid_t gfid, loc_t *loc, int how)
+nfs_gfid_loc_fill(inode_table_t *itable, uuid_t gfid, loc_t *loc, int how)
 {
-        int             ret = -EFAULT;
-        inode_t         *inode = NULL;
-
-        if (!loc)
-                return ret;
-
-        inode = inode_find (itable, gfid);
-        if (!inode) {
-		gf_log (GF_NFS, GF_LOG_TRACE, "Inode not found in itable, will try to create one.");
-                if (how == NFS_RESOLVE_CREATE) {
-			gf_log (GF_NFS, GF_LOG_TRACE, "Inode needs to be created.");
-                        inode = inode_new (itable);
-                        if (!inode) {
-                                gf_log (GF_NFS, GF_LOG_ERROR, "Failed to "
-                                        "allocate memory");
-                                ret = -ENOMEM;
-                                goto err;
-                        }
-
-                } else {
-			gf_log (GF_NFS, GF_LOG_ERROR, "Inode not found in itable and no creation was requested.");
-                        ret = -ENOENT;
-                        goto err;
-                }
+    int ret = -EFAULT;
+    inode_t *inode = NULL;
+
+    if (!loc)
+        return ret;
+
+    inode = inode_find(itable, gfid);
+    if (!inode) {
+        gf_msg_trace(GF_NFS, 0,
+                     "Inode not found in itable, will "
+                     "try to create one.");
+        if (how == NFS_RESOLVE_CREATE) {
+            gf_msg_trace(GF_NFS, 0, "Inode needs to be created.");
+            inode = inode_new(itable);
+            if (!inode) {
+                gf_msg(GF_NFS, GF_LOG_ERROR, ENOMEM, NFS_MSG_NO_MEMORY,
+                       "Failed to "
+                       "allocate memory");
+                ret = -ENOMEM;
+                goto err;
+            }
+
         } else {
-		gf_log (GF_NFS, GF_LOG_TRACE, "Inode was found in the itable.");
-	}
+            gf_msg(GF_NFS, GF_LOG_ERROR, ENOENT, NFS_MSG_INODE_NOT_FOUND,
+                   "Inode not found in "
+                   "itable and no creation was requested.");
+            ret = -ENOENT;
+            goto err;
+        }
+    } else {
+        gf_msg_trace(GF_NFS, 0, "Inode was found in the itable.");
+    }
 
-        gf_uuid_copy (loc->gfid, gfid);
+    gf_uuid_copy(loc->gfid, gfid);
 
-        ret = nfs_inode_loc_fill (inode, loc, how);
-	if (ret < 0) {
-		gf_log (GF_NFS, GF_LOG_ERROR, "Inode loc filling failed.: %s", strerror (-ret));
-		goto err;
-	}
+    ret = nfs_inode_loc_fill(inode, loc, how);
+    if (ret < 0) {
+        gf_msg(GF_NFS, GF_LOG_ERROR, -ret, NFS_MSG_INODE_LOC_FILL_ERROR,
+               "Inode loc filling failed.: %s", strerror(-ret));
+        goto err;
+    }
 
 err:
-        if (inode)
-                inode_unref (inode);
-        return ret;
+    if (inode)
+        inode_unref(inode);
+    return ret;
 }
 
-
 int
-nfs_root_loc_fill (inode_table_t *itable, loc_t *loc)
+nfs_root_loc_fill(inode_table_t *itable, loc_t *loc)
 {
-        uuid_t  rootgfid = {0, };
+    static uuid_t rootgfid = {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1};
 
-        rootgfid[15] = 1;
-        return nfs_gfid_loc_fill (itable, rootgfid, loc, NFS_RESOLVE_EXIST);
+    return nfs_gfid_loc_fill(itable, rootgfid, loc, NFS_RESOLVE_EXIST);
 }
 
-
-
 int
-nfs_parent_inode_loc_fill (inode_t *parent, inode_t *entryinode, char *entry,
-                           loc_t *loc)
+nfs_parent_inode_loc_fill(inode_t *parent, inode_t *entryinode, char *entry,
+                          loc_t *loc)
 {
-        int             ret = -EFAULT;
-        char            *path = NULL;
+    int ret = -EFAULT;
+    char *path = NULL;
 
-        if ((!parent) || (!entry) || (!loc) || (!entryinode))
-                return ret;
+    if ((!parent) || (!entry) || (!loc) || (!entryinode))
+        return ret;
 
-        ret = inode_path (parent, entry, &path);
-        if (ret < 0) {
-                gf_log (GF_NFS, GF_LOG_ERROR, "path resolution failed %s",
-                                path);
-                goto err;
-        }
+    ret = inode_path(parent, entry, &path);
+    if (ret < 0) {
+        gf_msg(GF_NFS, GF_LOG_ERROR, -ret, NFS_MSG_PATH_RESOLVE_FAIL,
+               "path resolution failed %s", path);
+        goto err;
+    }
 
-        ret = nfs_loc_fill (loc, entryinode, parent, path);
-        GF_FREE (path);
+    ret = nfs_loc_fill(loc, entryinode, parent, path);
+    GF_FREE(path);
 err:
-        return ret;
+    return ret;
 }
 
-
 /* Returns -1 if parent is not available, return -2 if the entry is not
  * available. In case the return is going to be -2, and how = NFS_RESOLVE_CREATE
  * it does however fill in the loc so that it can be used to perform a lookup
@@ -321,145 +307,144 @@ err:
  * On other errors, return -3. 0 on success.
  */
 int
-nfs_entry_loc_fill (inode_table_t *itable, uuid_t pargfid, char *entry,
-                    loc_t *loc, int how)
+nfs_entry_loc_fill(xlator_t *this, inode_table_t *itable, uuid_t pargfid,
+                   char *entry, loc_t *loc, int how, gf_boolean_t *freshlookup)
 {
-        inode_t         *parent = NULL;
-        inode_t         *entryinode = NULL;
-        int             ret = -3;
-        char            *resolvedpath = NULL;
-        int             pret = -3;
-
-        if ((!itable) || (!entry) || (!loc))
-                return ret;
+    inode_t *parent = NULL;
+    inode_t *entryinode = NULL;
+    int ret = -3;
+    char *resolvedpath = NULL;
+    int pret = -3;
 
-        parent = inode_find (itable, pargfid);
-
-        ret = -1;
-        /* Will need hard resolution now */
-        if (!parent)
-                goto err;
-
-        gf_uuid_copy (loc->pargfid, pargfid);
-
-        ret = -2;
-        entryinode = inode_grep (itable, parent, entry);
-        if (!entryinode) {
-                if (how == NFS_RESOLVE_CREATE) {
-                        /* Even though we'll create the inode and the loc for
-                         * a missing inode, we still need to return -2 so
-                         * that the caller can use the filled loc to call
-                         * lookup.
-                         */
-                        entryinode = inode_new (itable);
-                        /* Cannot change ret because that must
-                         * continue to have -2.
-                         */
-                        pret = nfs_parent_inode_loc_fill (parent, entryinode,
-                                                          entry, loc);
-                        /* Only if parent loc fill fails, should we notify error
-                         * through ret, otherwise, we still need to force a
-                         * lookup by returning -2.
-                         */
-                        if (pret < 0)
-			         ret = -3;
-                }
-                goto err;
-        }
-
-        ret = inode_path (parent, entry, &resolvedpath);
-        if (ret < 0) {
-                gf_log (GF_NFS, GF_LOG_ERROR, "path resolution failed %s",
-                                resolvedpath);
-                ret = -3;
-                goto err;
-        }
+    if ((!itable) || (!entry) || (!loc))
+        return ret;
 
-        ret = nfs_loc_fill (loc, entryinode, parent, resolvedpath);
-        if (ret < 0) {
-                gf_log (GF_NFS, GF_LOG_ERROR, "loc_fill failed %s",
-                                resolvedpath);
+    parent = inode_find(itable, pargfid);
+
+    ret = -1;
+    /* Will need hard resolution now */
+    if (!parent || inode_ctx_get(parent, this, NULL))
+        goto err;
+
+    gf_uuid_copy(loc->pargfid, pargfid);
+
+    ret = -2;
+    entryinode = inode_grep(itable, parent, entry);
+    if (!entryinode || inode_ctx_get(entryinode, this, NULL)) {
+        if (how == NFS_RESOLVE_CREATE) {
+            /* Even though we'll create the inode and the loc for
+             * a missing inode, we still need to return -2 so
+             * that the caller can use the filled loc to call
+             * lookup.
+             */
+            if (!entryinode) {
+                entryinode = inode_new(itable);
+                if (freshlookup)
+                    *freshlookup = _gf_true;
+            }
+            /* Cannot change ret because that must
+             * continue to have -2.
+             */
+            pret = nfs_parent_inode_loc_fill(parent, entryinode, entry, loc);
+            /* Only if parent loc fill fails, should we notify error
+             * through ret, otherwise, we still need to force a
+             * lookup by returning -2.
+             */
+            if (pret < 0)
                 ret = -3;
         }
+        goto err;
+    }
+
+    ret = inode_path(parent, entry, &resolvedpath);
+    if (ret < 0) {
+        gf_msg(GF_NFS, GF_LOG_ERROR, -ret, NFS_MSG_PATH_RESOLVE_FAIL,
+               "path resolution failed %s", resolvedpath);
+        ret = -3;
+        goto err;
+    }
+
+    ret = nfs_loc_fill(loc, entryinode, parent, resolvedpath);
+    if (ret < 0) {
+        gf_msg(GF_NFS, GF_LOG_ERROR, 0, NFS_MSG_INODE_LOC_FILL_ERROR,
+               "loc_fill failed %s", resolvedpath);
+        ret = -3;
+    }
 
 err:
-        if (parent)
-                inode_unref (parent);
+    if (parent)
+        inode_unref(parent);
 
-        if (entryinode)
-                inode_unref (entryinode);
+    if (entryinode)
+        inode_unref(entryinode);
 
-        GF_FREE (resolvedpath);
+    GF_FREE(resolvedpath);
 
-        return ret;
+    return ret;
 }
 
-
 uint32_t
-nfs_hash_gfid (uuid_t gfid)
+nfs_hash_gfid(uuid_t gfid)
 {
-        uint32_t                hash = 0;
-        uint64_t                msb64 = 0;
-        uint64_t                lsb64 = 0;
-        uint32_t                a1 = 0;
-        uint32_t                a2 = 0;
-        uint32_t                a3 = 0;
-        uint32_t                a4 = 0;
-        uint32_t                b1 = 0;
-        uint32_t                b2 = 0;
+    uint32_t hash = 0;
+    uint64_t msb64 = 0;
+    uint64_t lsb64 = 0;
+    uint32_t a1 = 0;
+    uint32_t a2 = 0;
+    uint32_t a3 = 0;
+    uint32_t a4 = 0;
+    uint32_t b1 = 0;
+    uint32_t b2 = 0;
 
-        if (__is_root_gfid (gfid))
-                return 0x1;
+    if (__is_root_gfid(gfid))
+        return 0x1;
 
-        memcpy (&msb64, &gfid[8], 8);
-        memcpy (&lsb64, &gfid[0], 8);
+    memcpy(&msb64, &gfid[8], 8);
+    memcpy(&lsb64, &gfid[0], 8);
 
-        a1 = (msb64 << 32);
-        a2 = (msb64 >> 32);
-        a3 = (lsb64 << 32);
-        a4 = (lsb64 >> 32);
+    a1 = (msb64 << 32);
+    a2 = (msb64 >> 32);
+    a3 = (lsb64 << 32);
+    a4 = (lsb64 >> 32);
 
-        b1 = a1 ^ a4;
-        b2 = a2 ^ a3;
+    b1 = a1 ^ a4;
+    b2 = a2 ^ a3;
 
-        hash = b1 ^ b2;
+    hash = b1 ^ b2;
 
-        return hash;
+    return hash;
 }
 
-
 void
-nfs_fix_generation (xlator_t *this, inode_t *inode)
+nfs_fix_generation(xlator_t *this, inode_t *inode)
 {
-        uint64_t                 raw_ctx        = 0;
-        struct nfs_inode_ctx    *ictx           = NULL;
-        struct nfs_state        *priv           = NULL;
-        int                      ret            = -1;
-
-        if (!inode) {
-                return;
-        }
-        priv = this->private;
-
-        if (inode_ctx_get(inode,this,&raw_ctx) == 0) {
-                ictx = (struct nfs_inode_ctx *)raw_ctx;
-                ictx->generation = priv->generation;
+    uint64_t raw_ctx = 0;
+    struct nfs_inode_ctx *ictx = NULL;
+    struct nfs_state *priv = NULL;
+    int ret = -1;
+
+    if (!inode) {
+        return;
+    }
+    priv = this->private;
+
+    if (inode_ctx_get(inode, this, &raw_ctx) == 0) {
+        ictx = (struct nfs_inode_ctx *)(uintptr_t)raw_ctx;
+        ictx->generation = priv->generation;
+    } else {
+        ictx = GF_CALLOC(1, sizeof(struct nfs_inode_ctx), gf_nfs_mt_inode_ctx);
+        if (!ictx) {
+            gf_msg(this->name, GF_LOG_ERROR, ENOMEM, NFS_MSG_NO_MEMORY,
+                   "could not allocate nfs inode ctx");
+            return;
         }
-        else {
-                ictx = GF_CALLOC (1, sizeof (struct nfs_inode_ctx),
-                                  gf_nfs_mt_inode_ctx);
-                if (!ictx) {
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "could not allocate nfs inode ctx");
-                        return;
-                }
-                INIT_LIST_HEAD(&ictx->shares);
-                ictx->generation = priv->generation;
-                ret = inode_ctx_put (inode, this, (uint64_t)ictx);
-                if (ret) {
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "could not store nfs inode ctx");
-                        return;
-                }
+        INIT_LIST_HEAD(&ictx->shares);
+        ictx->generation = priv->generation;
+        ret = inode_ctx_put(inode, this, (uint64_t)(uintptr_t)ictx);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, NFS_MSG_INODE_CTX_STORE_FAIL,
+                   "could not store nfs inode ctx");
+            return;
         }
+    }
 }
diff --git a/xlators/nfs/server/src/nfs-common.h b/xlators/nfs/server/src/nfs-common.h
index 401484c0f3c..bd80d8b3be5 100644
--- a/xlators/nfs/server/src/nfs-common.h
+++ b/xlators/nfs/server/src/nfs-common.h
@@ -11,71 +11,63 @@
 #ifndef _NFS_COMMON_H_
 #define _NFS_COMMON_H_
 
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
-
 #include <unistd.h>
 
-#include "xlator.h"
+#include <glusterfs/xlator.h>
 #include "rpcsvc.h"
-#include "iatt.h"
-#include "compat-uuid.h"
+#include <glusterfs/iatt.h>
+#include <glusterfs/compat-uuid.h>
 
-//NFS_PATH_MAX hard-coded to 4096 as a work around for bug 2476.
-//nfs server crashes when path received is longer than PATH_MAX
-#define NFS_PATH_MAX    4096
-#define NFS_NAME_MAX    NAME_MAX
+// NFS_PATH_MAX hard-coded to 4096 as a work around for bug 2476.
+// nfs server crashes when path received is longer than PATH_MAX
+#define NFS_PATH_MAX 4096
+#define NFS_NAME_MAX NAME_MAX
 
 #define NFS_DEFAULT_CREATE_MODE 0600
 
 extern xlator_t *
-nfs_xlid_to_xlator (xlator_list_t *cl, uint8_t xlid);
+nfs_xlid_to_xlator(xlator_list_t *cl, uint8_t xlid);
 
 extern uint16_t
-nfs_xlator_to_xlid (xlator_list_t *cl, xlator_t *xl);
+nfs_xlator_to_xlid(xlator_list_t *cl, xlator_t *xl);
 
 extern xlator_t *
-nfs_path_to_xlator (xlator_list_t *cl, char *path);
+nfs_path_to_xlator(xlator_list_t *cl, char *path);
 
 extern xlator_t *
-nfs_mntpath_to_xlator (xlator_list_t *cl, char *path);
-
-extern int
-nfs_zero_filled_stat (struct iatt *buf);
+nfs_mntpath_to_xlator(xlator_list_t *cl, char *path);
 
 extern void
-nfs_loc_wipe (loc_t *loc);
+nfs_loc_wipe(loc_t *loc);
 
 extern int
-nfs_loc_copy (loc_t *dst, loc_t *src);
+nfs_loc_copy(loc_t *dst, loc_t *src);
 
 extern int
-nfs_loc_fill (loc_t *loc, inode_t *inode, inode_t *parent, char *path);
+nfs_loc_fill(loc_t *loc, inode_t *inode, inode_t *parent, char *path);
 
-#define NFS_RESOLVE_EXIST       1
-#define NFS_RESOLVE_CREATE      2
+#define NFS_RESOLVE_EXIST 1
+#define NFS_RESOLVE_CREATE 2
 
 extern int
-nfs_inode_loc_fill (inode_t *inode, loc_t *loc, int how);
+nfs_inode_loc_fill(inode_t *inode, loc_t *loc, int how);
 
 extern int
-nfs_ino_loc_fill (inode_table_t *itable, uuid_t gfid, loc_t *l);
+nfs_ino_loc_fill(inode_table_t *itable, uuid_t gfid, loc_t *l);
 
 extern int
-nfs_entry_loc_fill (inode_table_t *itable, uuid_t pargfid, char *entry,
-                    loc_t *loc, int how);
+nfs_entry_loc_fill(xlator_t *this, inode_table_t *itable, uuid_t pargfid,
+                   char *entry, loc_t *loc, int how, gf_boolean_t *freshlookup);
 
 extern int
-nfs_root_loc_fill (inode_table_t *itable, loc_t *loc);
+nfs_root_loc_fill(inode_table_t *itable, loc_t *loc);
 
 extern uint32_t
-nfs_hash_gfid (uuid_t gfid);
+nfs_hash_gfid(uuid_t gfid);
 
 extern int
-nfs_gfid_loc_fill (inode_table_t *itable, uuid_t gfid, loc_t *loc, int how);
+nfs_gfid_loc_fill(inode_table_t *itable, uuid_t gfid, loc_t *loc, int how);
 
 void
-nfs_fix_generation (xlator_t *this, inode_t *inode);
+nfs_fix_generation(xlator_t *this, inode_t *inode);
 #endif
diff --git a/xlators/nfs/server/src/nfs-fops.c b/xlators/nfs/server/src/nfs-fops.c
index c3f1b5e2fa6..4d8540c2c3e 100644
--- a/xlators/nfs/server/src/nfs-fops.c
+++ b/xlators/nfs/server/src/nfs-fops.c
@@ -8,383 +8,367 @@
   cases as published by the Free Software Foundation.
 */
 
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
-
-#include <grp.h>
 #include <pwd.h>
 
-#include "dict.h"
-#include "xlator.h"
-#include "iobuf.h"
-#include "call-stub.h"
-#include "stack.h"
+#include <glusterfs/dict.h>
+#include <glusterfs/xlator.h>
+#include <glusterfs/iobuf.h>
+#include <glusterfs/call-stub.h>
 #include "nfs.h"
 #include "nfs-fops.h"
-#include "inode.h"
 #include "nfs-common.h"
 #include "nfs3-helpers.h"
 #include "nfs-mem-types.h"
+#include "nfs-messages.h"
 #include <libgen.h>
 #include <semaphore.h>
 
 static int gf_auth_max_groups_nfs_log = 0;
 
 void
-nfs_fix_groups (xlator_t *this, call_stack_t *root)
+nfs_fix_groups(xlator_t *this, call_stack_t *root)
 {
-        struct passwd    mypw;
-        char             mystrs[1024];
-        struct passwd    *result;
-#ifdef GF_DARWIN_HOST_OS
-        /* BSD/DARWIN does not correctly uses gid_t in getgrouplist */
-        int              mygroups[GF_MAX_AUX_GROUPS];
-#else
-        gid_t            mygroups[GF_MAX_AUX_GROUPS];
-#endif
-        int              ngroups;
-        int              i;
-        int              max_groups;
-        struct nfs_state *priv = this->private;
-        const gid_list_t *agl;
-        gid_list_t gl;
-
-        if (!priv->server_aux_gids) {
-                return;
+    struct passwd mypw;
+    char mystrs[1024];
+    struct passwd *result;
+    gid_t *mygroups;
+    int ngroups;
+    int i;
+    int max_groups;
+    struct nfs_state *priv = this->private;
+    const gid_list_t *agl;
+    gid_list_t gl;
+
+    if (!priv->server_aux_gids) {
+        return;
+    }
+
+    /* RPC enforces the GF_AUTH_GLUSTERFS_MAX_GROUPS limit */
+    max_groups = GF_AUTH_GLUSTERFS_MAX_GROUPS(root->lk_owner.len,
+                                              AUTH_GLUSTERFS_v2);
+
+    agl = gid_cache_lookup(&priv->gid_cache, root->uid, 0, 0);
+    if (agl) {
+        if (agl->gl_count > max_groups) {
+            GF_LOG_OCCASIONALLY(gf_auth_max_groups_nfs_log, this->name,
+                                GF_LOG_WARNING,
+                                "too many groups, reducing %d -> %d",
+                                agl->gl_count, max_groups);
         }
 
-        /* RPC enforces the GF_AUTH_GLUSTERFS_MAX_GROUPS limit */
-        max_groups = GF_AUTH_GLUSTERFS_MAX_GROUPS(root->lk_owner.len);
-
-	agl = gid_cache_lookup(&priv->gid_cache, root->uid, 0, 0);
-	if (agl) {
-		if (agl->gl_count > max_groups) {
-			GF_LOG_OCCASIONALLY (gf_auth_max_groups_nfs_log,
-					this->name, GF_LOG_WARNING,
-					"too many groups, reducing %d -> %d",
-					agl->gl_count, max_groups);
-		}
-
-		for (ngroups = 0; ngroups < agl->gl_count
-				&& ngroups <= max_groups; ngroups++) {
-			root->groups[ngroups] = agl->gl_list[ngroups];
-		}
-		root->ngrps = ngroups;
-		gid_cache_release(&priv->gid_cache, agl);
-		return;
-	}
-
-	/* No cached list found. */
-        if (getpwuid_r(root->uid,&mypw,mystrs,sizeof(mystrs),&result) != 0) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "getpwuid_r(%u) failed", root->uid);
-                return;
+        for (ngroups = 0; ngroups < agl->gl_count && ngroups <= max_groups;
+             ngroups++) {
+            root->groups[ngroups] = agl->gl_list[ngroups];
         }
+        root->ngrps = ngroups;
+        gid_cache_release(&priv->gid_cache, agl);
+        return;
+    }
 
-        if (!result) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "getpwuid_r(%u) found nothing", root->uid);
-                return;
-        }
+    /* No cached list found. */
+    if (getpwuid_r(root->uid, &mypw, mystrs, sizeof(mystrs), &result) != 0) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, NFS_MSG_GETPWUID_FAIL,
+               "getpwuid_r(%u) failed", root->uid);
+        return;
+    }
 
-        gf_log (this->name, GF_LOG_TRACE, "mapped %u => %s",
-                root->uid, result->pw_name);
+    if (!result) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, NFS_MSG_GETPWUID_FAIL,
+               "getpwuid_r(%u) found nothing", root->uid);
+        return;
+    }
 
-        ngroups = GF_MAX_AUX_GROUPS;
-        if (getgrouplist(result->pw_name,root->gid,mygroups,&ngroups) == -1) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "could not map %s to group list", result->pw_name);
-                return;
-        }
+    gf_msg_trace(this->name, 0, "mapped %u => %s", root->uid, result->pw_name);
 
-	/* Add the group data to the cache. */
-	gl.gl_list = GF_CALLOC(ngroups, sizeof(gid_t), gf_nfs_mt_aux_gids);
-	if (gl.gl_list) {
-		/* It's not fatal if the alloc failed. */
-		gl.gl_id = root->uid;
-		gl.gl_uid = 0;
-		gl.gl_gid = 0;
-		gl.gl_count = ngroups;
-		memcpy(gl.gl_list, mygroups, sizeof(gid_t) * ngroups);
-		if (gid_cache_add(&priv->gid_cache, &gl) != 1)
-			GF_FREE(gl.gl_list);
-	}
-
-	/* RPC enforces the GF_AUTH_GLUSTERFS_MAX_GROUPS limit */
-	if (ngroups > max_groups) {
-		GF_LOG_OCCASIONALLY (gf_auth_max_groups_nfs_log,
-				     this->name, GF_LOG_WARNING,
-				     "too many groups, reducing %d -> %d",
-				     ngroups, max_groups);
-
-		ngroups = max_groups;
-	}
-
-	/* Copy data to the frame. */
-        for (i = 0; i < ngroups; ++i) {
-                gf_log (this->name, GF_LOG_TRACE,
-                        "%s is in group %u", result->pw_name, mygroups[i]);
-                root->groups[i] = mygroups[i];
-        }
-        root->ngrps = ngroups;
+    ngroups = gf_getgrouplist(result->pw_name, root->gid, &mygroups);
+    if (ngroups == -1) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, NFS_MSG_MAP_GRP_LIST_FAIL,
+               "could not map %s to group list", result->pw_name);
+        return;
+    }
+
+    /* RPC enforces the GF_AUTH_GLUSTERFS_MAX_GROUPS limit */
+    if (ngroups > max_groups) {
+        GF_LOG_OCCASIONALLY(
+            gf_auth_max_groups_nfs_log, this->name, GF_LOG_WARNING,
+            "too many groups, reducing %d -> %d", ngroups, max_groups);
+    }
+
+    /* Copy data to the frame. */
+    for (i = 0; i < ngroups && i < max_groups; ++i) {
+        gf_msg_trace(this->name, 0, "%s is in group %u", result->pw_name,
+                     mygroups[i]);
+        root->groups[i] = mygroups[i];
+    }
+    root->ngrps = ngroups;
+
+    /* Add the group data to the cache. */
+    gl.gl_list = mygroups;
+    gl.gl_id = root->uid;
+    gl.gl_uid = 0;
+    gl.gl_gid = 0;
+    gl.gl_count = ngroups;
+    if (gid_cache_add(&priv->gid_cache, &gl) != 1)
+        GF_FREE(mygroups);
 }
 
 struct nfs_fop_local *
-nfs_fop_local_init (xlator_t *nfsx)
+nfs_fop_local_init(xlator_t *nfsx)
 {
-        struct nfs_fop_local    *l = NULL;
+    struct nfs_fop_local *l = NULL;
 
-        if (!nfsx)
-                return NULL;
+    if (!nfsx)
+        return NULL;
 
-        l = mem_get (nfs_fop_mempool (nfsx));
+    l = mem_get(nfs_fop_mempool(nfsx));
+    if (!l) {
+        gf_msg_nomem(GF_NFS, GF_LOG_ERROR, 4096);
+        return NULL;
+    }
 
-        memset (l, 0, sizeof (*l));
-        return l;
+    memset(l, 0, sizeof(*l));
+    return l;
 }
 
 void
-nfs_fop_local_wipe (xlator_t *nfsx, struct nfs_fop_local *l)
+nfs_fop_local_wipe(xlator_t *nfsx, struct nfs_fop_local *l)
 {
-        if ((!nfsx) || (!l))
-                return;
+    if ((!nfsx) || (!l))
+        return;
 
-        if (l->iobref)
-                iobref_unref (l->iobref);
+    if (l->iobref)
+        iobref_unref(l->iobref);
 
-        if (l->parent)
-                inode_unref (l->parent);
+    if (l->parent)
+        inode_unref(l->parent);
 
-        if (l->inode)
-                inode_unref (l->inode);
+    if (l->inode)
+        inode_unref(l->inode);
 
-        if (l->newparent)
-                inode_unref (l->newparent);
+    if (l->newparent)
+        inode_unref(l->newparent);
 
-        if (l->dictgfid)
-                dict_unref (l->dictgfid);
+    if (l->dictgfid)
+        dict_unref(l->dictgfid);
 
-        mem_put (l);
+    mem_put(l);
 
-        return;
+    return;
 }
 
-#define nfs_stack_destroy(nfl, fram)                                    \
-        do {                                                            \
-                nfs_fop_local_wipe ((nfl)->nfsx, nfl);                  \
-                (fram)->local = NULL;                                   \
-                STACK_DESTROY ((fram)->root);                           \
-        } while (0)                                                     \
-
-
-pthread_mutex_t         ctr = PTHREAD_MUTEX_INITIALIZER;
-unsigned int            cval = 1;
+#define nfs_stack_destroy(nfl, fram)                                           \
+    do {                                                                       \
+        nfs_fop_local_wipe((nfl)->nfsx, nfl);                                  \
+        (fram)->local = NULL;                                                  \
+        STACK_DESTROY((fram)->root);                                           \
+    } while (0)
 
+pthread_mutex_t ctr = PTHREAD_MUTEX_INITIALIZER;
+unsigned int cval = 1;
 
 int
-nfs_frame_getctr ()
+nfs_frame_getctr()
 {
-        uint64_t val = 0;
-
-        pthread_mutex_lock (&ctr);
-        {
-                if (cval == 0)
-                        cval = 1;
-                val = cval;
-                cval++;
-        }
-        pthread_mutex_unlock (&ctr);
-
-        return val;
+    uint64_t val = 0;
+
+    pthread_mutex_lock(&ctr);
+    {
+        if (cval == 0)
+            cval = 1;
+        val = cval;
+        cval++;
+    }
+    pthread_mutex_unlock(&ctr);
+
+    return val;
 }
 
-
 call_frame_t *
-nfs_create_frame (xlator_t *xl, nfs_user_t *nfu)
+nfs_create_frame(xlator_t *xl, nfs_user_t *nfu)
 {
-        call_frame_t    *frame = NULL;
-        int             x = 0;
-        int             y = 0;
-
-        if ((!xl) || (!nfu) || (nfu->ngrps > NFS_NGROUPS))
-                return NULL;
-
-        frame = create_frame (xl, (call_pool_t *)xl->ctx->pool);
-        if (!frame)
-                goto err;
-	if (call_stack_alloc_groups (frame->root, nfu->ngrps) != 0) {
-		STACK_DESTROY (frame->root);
-		frame = NULL;
-		goto err;
-	}
-
-        frame->root->pid = NFS_PID;
-        frame->root->uid = nfu->uid;
-        frame->root->gid = nfu->gids[NFS_PRIMGID_IDX];
-        frame->root->lk_owner = nfu->lk_owner;
-
-        if (nfu->ngrps != 1) {
-                frame->root->ngrps = nfu->ngrps - 1;
-
-                gf_log (GF_NFS, GF_LOG_TRACE,"uid: %d, gid %d, gids: %d",
-                        frame->root->uid, frame->root->gid, frame->root->ngrps);
-                for(y = 0, x = 1;  y < frame->root->ngrps; x++,y++) {
-                        gf_log (GF_NFS, GF_LOG_TRACE, "gid: %d", nfu->gids[x]);
-                        frame->root->groups[y] = nfu->gids[x];
-                }
+    call_frame_t *frame = NULL;
+    int x = 0;
+    int y = 0;
+
+    if ((!xl) || (!nfu) || (nfu->ngrps > NFS_NGROUPS))
+        return NULL;
+
+    frame = create_frame(xl, (call_pool_t *)xl->ctx->pool);
+    if (!frame)
+        goto err;
+    if (call_stack_alloc_groups(frame->root, nfu->ngrps) != 0) {
+        STACK_DESTROY(frame->root);
+        frame = NULL;
+        goto err;
+    }
+
+    frame->root->pid = NFS_PID;
+    frame->root->uid = nfu->uid;
+    frame->root->gid = nfu->gids[NFS_PRIMGID_IDX];
+    memcpy(&frame->root->identifier, &nfu->identifier, UNIX_PATH_MAX);
+    frame->root->lk_owner = nfu->lk_owner;
+
+    if (nfu->ngrps != 1) {
+        frame->root->ngrps = nfu->ngrps - 1;
+
+        gf_msg_trace(GF_NFS, 0, "uid: %d, gid %d, gids: %d", frame->root->uid,
+                     frame->root->gid, frame->root->ngrps);
+        for (y = 0, x = 1; y < frame->root->ngrps; x++, y++) {
+            gf_msg_trace(GF_NFS, 0, "gid: %d", nfu->gids[x]);
+            frame->root->groups[y] = nfu->gids[x];
         }
+    }
 
-        /*
-         * It's tempting to do this *instead* of using nfu above, but we need
-         * to have those values in case nfs_fix_groups doesn't do anything.
-         */
-        nfs_fix_groups(xl,frame->root);
+    /*
+     * It's tempting to do this *instead* of using nfu above, but we need
+     * to have those values in case nfs_fix_groups doesn't do anything.
+     */
+    nfs_fix_groups(xl, frame->root);
 
 err:
-        return frame;
+    return frame;
 }
 
-#define nfs_fop_handle_frame_create(fram, xla, nfuser, retval, errlabel)      \
-        do {                                                                  \
-                fram = nfs_create_frame (xla, (nfuser));                \
-                if (!fram) {                                                  \
-                        retval = (-ENOMEM);                                   \
-                        gf_log (GF_NFS, GF_LOG_ERROR,"Frame creation failed");\
-                        goto errlabel;                                        \
-                }                                                             \
-        } while (0)                                                           \
+#define nfs_fop_handle_frame_create(fram, xla, nfuser, retval, errlabel)       \
+    do {                                                                       \
+        fram = nfs_create_frame(xla, (nfuser));                                \
+        if (!fram) {                                                           \
+            retval = (-ENOMEM);                                                \
+            gf_msg(GF_NFS, GF_LOG_ERROR, ENOMEM, NFS_MSG_NO_MEMORY,            \
+                   "Frame creation failed");                                   \
+            goto errlabel;                                                     \
+        }                                                                      \
+    } while (0)
 
 /* Look into the  inode and parent inode of a loc and save enough state
  * for us to determine in the callback whether to funge the ino in the stat buf
  * with 1 for the parent.
  */
-#define nfs_fop_save_root_ino(locl, loc)                                \
-        do {                                                            \
-                if (((loc)->inode) &&                                   \
-                    __is_root_gfid ((loc)->inode->gfid))                \
-                        (locl)->rootinode = 1;                          \
-                else if (((loc)->parent) &&                             \
-                         __is_root_gfid ((loc)->parent->gfid))          \
-                        (locl)->rootparentinode = 1;                    \
-        } while (0)
+#define nfs_fop_save_root_ino(locl, loc)                                       \
+    do {                                                                       \
+        if (((loc)->inode) && __is_root_gfid((loc)->inode->gfid))              \
+            (locl)->rootinode = 1;                                             \
+        else if (((loc)->parent) && __is_root_gfid((loc)->parent->gfid))       \
+            (locl)->rootparentinode = 1;                                       \
+    } while (0)
 
 /* Do the same for an fd */
-#define nfs_fop_save_root_fd_ino(locl, fdesc)                           \
-        do {                                                            \
-                if (__is_root_gfid ((fdesc)->inode->gfid))              \
-                        (locl)->rootinode = 1;                          \
-        } while (0)
-
+#define nfs_fop_save_root_fd_ino(locl, fdesc)                                  \
+    do {                                                                       \
+        if (__is_root_gfid((fdesc)->inode->gfid))                              \
+            (locl)->rootinode = 1;                                             \
+    } while (0)
 
 /* Use the state saved by the previous macro to funge the ino in the appropriate
  * structure.
  */
-#define nfs_fop_restore_root_ino(locl, fopret, preattr, postattr, prepar, postpar)  \
-        do {                                                                \
-                if (fopret == -1)                                           \
-                        break;                                              \
-                if ((locl)->rootinode) {                                    \
-                        if ((preattr)) {                                    \
-                                ((struct iatt *)(preattr))->ia_ino = 1;     \
-                                ((struct iatt *)(preattr))->ia_dev = 0;     \
-                        }                                                   \
-                        if ((postattr)) {                                   \
-                                ((struct iatt *)(postattr))->ia_ino = 1;    \
-                                ((struct iatt *)(postattr))->ia_dev = 0;    \
-                        }                                                   \
-                } else if ((locl)->rootparentinode) {                       \
-                        if ((prepar)) {                                     \
-                                ((struct iatt *)(prepar))->ia_ino = 1;      \
-                                ((struct iatt *)(prepar))->ia_dev = 0;      \
-                        }                                                   \
-                        if ((postpar)) {                                    \
-                                ((struct iatt *)(postpar))->ia_ino = 1;     \
-                                ((struct iatt *)(postpar))->ia_dev = 0;     \
-                        }                                                   \
-                }                                                           \
-        } while (0)                                                         \
+#define nfs_fop_restore_root_ino(locl, fopret, preattr, postattr, prepar,      \
+                                 postpar)                                      \
+    do {                                                                       \
+        if (fopret == -1)                                                      \
+            break;                                                             \
+        if ((locl)->rootinode) {                                               \
+            if ((preattr)) {                                                   \
+                ((struct iatt *)(preattr))->ia_ino = 1;                        \
+                ((struct iatt *)(preattr))->ia_dev = 0;                        \
+            }                                                                  \
+            if ((postattr)) {                                                  \
+                ((struct iatt *)(postattr))->ia_ino = 1;                       \
+                ((struct iatt *)(postattr))->ia_dev = 0;                       \
+            }                                                                  \
+        } else if ((locl)->rootparentinode) {                                  \
+            if ((prepar)) {                                                    \
+                ((struct iatt *)(prepar))->ia_ino = 1;                         \
+                ((struct iatt *)(prepar))->ia_dev = 0;                         \
+            }                                                                  \
+            if ((postpar)) {                                                   \
+                ((struct iatt *)(postpar))->ia_ino = 1;                        \
+                ((struct iatt *)(postpar))->ia_dev = 0;                        \
+            }                                                                  \
+        }                                                                      \
+    } while (0)
 
 /* If the newly created, inode's parent is root, we'll need to funge the ino
  * in the parent attr when we receive them in the callback.
  */
-#define nfs_fop_newloc_save_root_ino(locl, newloc)                      \
-        do {                                                            \
-                if (((newloc)->inode) &&                                \
-                    __is_root_gfid ((newloc)->inode->gfid))             \
-                        (locl)->newrootinode = 1;                       \
-                else if (((newloc)->parent) &&                          \
-                         __is_root_gfid ((newloc)->parent->gfid))       \
-                        (locl)->newrootparentinode = 1;                 \
-        } while (0)
-
-#define nfs_fop_newloc_restore_root_ino(locl, fopret, preattr, postattr, prepar, postpar)  \
-        do {                                                                   \
-                if (fopret == -1)                                              \
-                        break;                                                 \
+#define nfs_fop_newloc_save_root_ino(locl, newloc)                             \
+    do {                                                                       \
+        if (((newloc)->inode) && __is_root_gfid((newloc)->inode->gfid))        \
+            (locl)->newrootinode = 1;                                          \
+        else if (((newloc)->parent) && __is_root_gfid((newloc)->parent->gfid)) \
+            (locl)->newrootparentinode = 1;                                    \
+    } while (0)
+
+#define nfs_fop_newloc_restore_root_ino(locl, fopret, preattr, postattr,       \
+                                        prepar, postpar)                       \
+    do {                                                                       \
+        if (fopret == -1)                                                      \
+            break;                                                             \
                                                                                \
-                if ((locl)->newrootinode) {                                    \
-                        if ((preattr))                                         \
-                                ((struct iatt *)(preattr))->ia_ino = 1;        \
-                        if ((postattr))                                        \
-                                ((struct iatt *)(postattr))->ia_ino = 1;       \
-                } else if ((locl)->newrootparentinode) {                       \
-                        if ((prepar))                                          \
-                                ((struct iatt *)(prepar))->ia_ino = 1;         \
-                        if ((postpar))                                         \
-                                ((struct iatt *)(postpar))->ia_ino = 1;        \
-                }                                                              \
-        } while (0)                                                            \
+        if ((locl)->newrootinode) {                                            \
+            if ((preattr))                                                     \
+                ((struct iatt *)(preattr))->ia_ino = 1;                        \
+            if ((postattr))                                                    \
+                ((struct iatt *)(postattr))->ia_ino = 1;                       \
+        } else if ((locl)->newrootparentinode) {                               \
+            if ((prepar))                                                      \
+                ((struct iatt *)(prepar))->ia_ino = 1;                         \
+            if ((postpar))                                                     \
+                ((struct iatt *)(postpar))->ia_ino = 1;                        \
+        }                                                                      \
+    } while (0)
 
 dict_t *
-nfs_gfid_dict (inode_t *inode)
+nfs_gfid_dict(inode_t *inode)
 {
-        uuid_t  newgfid = {0, };
-        char    *dyngfid = NULL;
-        dict_t  *dictgfid = NULL;
-        int     ret = -1;
-        uuid_t  rootgfid = {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1};
-
-        dyngfid = GF_CALLOC (1, sizeof (uuid_t), gf_common_mt_char);
-        if (dyngfid == NULL)
-                return (NULL);
-
-        gf_uuid_generate (newgfid);
-
-        if (gf_uuid_compare (inode->gfid, rootgfid) == 0)
-                memcpy (dyngfid, rootgfid, sizeof (uuid_t));
-        else
-                memcpy (dyngfid, newgfid, sizeof (uuid_t));
-
-        dictgfid = dict_new ();
-        if (!dictgfid) {
-                gf_log (GF_NFS, GF_LOG_ERROR, "Failed to create gfid dict");
-                GF_FREE (dyngfid);
-                return (NULL);
-        }
-
-        ret = dict_set_bin (dictgfid, "gfid-req", dyngfid, sizeof (uuid_t));
-        if (ret < 0) {
-                GF_FREE (dyngfid);
-                dict_unref (dictgfid);
-                return (NULL);
-        }
-
-        return dictgfid;
+    uuid_t newgfid = {
+        0,
+    };
+    unsigned char *dyngfid = NULL;
+    dict_t *dictgfid = NULL;
+    int ret = -1;
+    static uuid_t rootgfid = {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1};
+
+    dyngfid = GF_MALLOC(sizeof(uuid_t), gf_common_mt_char);
+    if (dyngfid == NULL)
+        return (NULL);
+
+    gf_uuid_generate(newgfid);
+
+    if (gf_uuid_compare(inode->gfid, rootgfid) == 0)
+        memcpy(dyngfid, rootgfid, sizeof(uuid_t));
+    else
+        memcpy(dyngfid, newgfid, sizeof(uuid_t));
+
+    dictgfid = dict_new();
+    if (!dictgfid) {
+        gf_msg(GF_NFS, GF_LOG_ERROR, errno, NFS_MSG_GFID_DICT_CREATE_FAIL,
+               "Failed to create gfid dict");
+        GF_FREE(dyngfid);
+        return (NULL);
+    }
+
+    ret = dict_set_gfuuid(dictgfid, "gfid-req", dyngfid, false);
+    if (ret < 0) {
+        GF_FREE(dyngfid);
+        dict_unref(dictgfid);
+        return (NULL);
+    }
+
+    return dictgfid;
 }
 
-#define nfs_fop_gfid_setup(nflcl, inode, retval, erlbl)                 \
-        do {                                                            \
-                if (nflcl) {                                            \
-                        (nflcl)->dictgfid = nfs_gfid_dict (inode);      \
-                                                                        \
-                        if (!((nflcl)->dictgfid)) {                     \
-                                retval = -EFAULT;                       \
-                                goto erlbl;                             \
-                        }                                               \
-                }                                                       \
-        } while (0)                                                     \
+#define nfs_fop_gfid_setup(nflcl, inode, retval, erlbl)                        \
+    do {                                                                       \
+        if (nflcl) {                                                           \
+            (nflcl)->dictgfid = nfs_gfid_dict(inode);                          \
+                                                                               \
+            if (!((nflcl)->dictgfid)) {                                        \
+                retval = -EFAULT;                                              \
+                goto erlbl;                                                    \
+            }                                                                  \
+        }                                                                      \
+    } while (0)
 
 /* Fops Layer Explained
  * The fops layer has three types of functions. They can all be identified by
@@ -412,1286 +396,1237 @@ nfs_gfid_dict (inode_t *inode)
  */
 
 int32_t
-nfs_fop_lookup_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                    int32_t op_ret, int32_t op_errno, inode_t *inode,
-                    struct iatt *buf, dict_t *xattr, struct iatt *postparent)
+nfs_fop_lookup_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                   int32_t op_ret, int32_t op_errno, inode_t *inode,
+                   struct iatt *buf, dict_t *xattr, struct iatt *postparent)
 {
-        struct nfs_fop_local    *local = NULL;
-        fop_lookup_cbk_t        progcbk;
+    struct nfs_fop_local *local = NULL;
+    fop_lookup_cbk_t progcbk;
 
-        if (op_ret == 0) {
-                nfs_fix_generation(this,inode);
-        }
+    if (op_ret == 0) {
+        nfs_fix_generation(this, inode);
+    }
 
-        nfl_to_prog_data (local, progcbk, frame);
-        nfs_fop_restore_root_ino (local, op_ret, buf, NULL, NULL, postparent);
-        if (progcbk)
-                progcbk (frame, cookie, this, op_ret, op_errno, inode, buf,
-                         xattr, postparent);
+    nfl_to_prog_data(local, progcbk, frame);
+    nfs_fop_restore_root_ino(local, op_ret, buf, NULL, NULL, postparent);
+    if (progcbk)
+        progcbk(frame, cookie, this, op_ret, op_errno, inode, buf, xattr,
+                postparent);
 
-        nfs_stack_destroy (local, frame);
-        return 0;
+    nfs_stack_destroy(local, frame);
+    return 0;
 }
 
-
 int
-nfs_fop_lookup (xlator_t *nfsx, xlator_t *xl, nfs_user_t *nfu, loc_t *loc,
-                fop_lookup_cbk_t cbk, void *local)
+nfs_fop_lookup(xlator_t *nfsx, xlator_t *xl, nfs_user_t *nfu, loc_t *loc,
+               fop_lookup_cbk_t cbk, void *local)
 {
-        call_frame_t            *frame = NULL;
-        int                     ret = -EFAULT;
-        struct nfs_fop_local    *nfl = NULL;
+    call_frame_t *frame = NULL;
+    int ret = -EFAULT;
+    struct nfs_fop_local *nfl = NULL;
 
-        if ((!xl) || (!loc) || (!nfu))
-                return ret;
+    if ((!xl) || (!loc) || (!nfu))
+        return ret;
 
-        gf_log (GF_NFS, GF_LOG_TRACE, "Lookup: %s", loc->path);
-        nfs_fop_handle_frame_create (frame, nfsx, nfu, ret, err);
-        nfs_fop_handle_local_init (frame, nfsx, nfl, cbk, local, ret, err);
-        nfs_fop_save_root_ino (nfl, loc);
-        nfs_fop_gfid_setup (nfl, loc->inode, ret, err);
+    gf_msg_trace(GF_NFS, 0, "Lookup: %s", loc->path);
+    nfs_fop_handle_frame_create(frame, nfsx, nfu, ret, err);
+    nfs_fop_handle_local_init(frame, nfsx, nfl, cbk, local, ret, err);
+    nfs_fop_save_root_ino(nfl, loc);
+    nfs_fop_gfid_setup(nfl, loc->inode, ret, err);
 
-        STACK_WIND_COOKIE (frame, nfs_fop_lookup_cbk, xl, xl,
-                           xl->fops->lookup, loc, nfl->dictgfid);
+    STACK_WIND_COOKIE(frame, nfs_fop_lookup_cbk, xl, xl, xl->fops->lookup, loc,
+                      nfl->dictgfid);
 
-        ret = 0;
+    ret = 0;
 err:
-        if (ret < 0) {
-                if (frame)
-                        nfs_stack_destroy (nfl, frame);
-        }
+    if (ret < 0) {
+        if (frame)
+            nfs_stack_destroy(nfl, frame);
+    }
 
-        return ret;
+    return ret;
 }
 
 int32_t
-nfs_fop_access_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                    int32_t op_ret, int32_t op_errno, dict_t *xdata)
+nfs_fop_access_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                   int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
-        struct nfs_fop_local    *nfl = NULL;
-        fop_access_cbk_t        progcbk = NULL;
+    struct nfs_fop_local *nfl = NULL;
+    fop_access_cbk_t progcbk = NULL;
 
-        nfl_to_prog_data (nfl, progcbk, frame);
-        if (progcbk)
-                progcbk (frame, cookie, this, op_ret, op_errno, xdata);
+    nfl_to_prog_data(nfl, progcbk, frame);
+    if (progcbk)
+        progcbk(frame, cookie, this, op_ret, op_errno, xdata);
 
-        nfs_stack_destroy (nfl, frame);
-        return 0;
+    nfs_stack_destroy(nfl, frame);
+    return 0;
 }
 
 int
-nfs_fop_access (xlator_t *nfsx, xlator_t *xl, nfs_user_t *nfu, loc_t *loc,
-                int32_t accesstest, fop_access_cbk_t cbk, void *local)
+nfs_fop_access(xlator_t *nfsx, xlator_t *xl, nfs_user_t *nfu, loc_t *loc,
+               int32_t accesstest, fop_access_cbk_t cbk, void *local)
 {
-        call_frame_t            *frame = NULL;
-        int                     ret = -EFAULT;
-        struct nfs_fop_local    *nfl = NULL;
-        uint32_t                accessbits = 0;
-
-        if ((!xl) || (!loc) || (!nfu))
-                return ret;
-
-        gf_log (GF_NFS, GF_LOG_TRACE, "Access: %s", loc->path);
-        nfs_fop_handle_frame_create (frame, nfsx, nfu, ret, err);
-        nfs_fop_handle_local_init (frame, nfsx, nfl, cbk, local, ret, err);
-        nfs_fop_save_root_ino (nfl, loc);
-
-        accessbits = nfs3_request_to_accessbits (accesstest);
-        STACK_WIND_COOKIE (frame, nfs_fop_access_cbk, xl, xl, xl->fops->access,
-                           loc, accessbits, NULL);
-        ret = 0;
-err:
-        if (ret < 0) {
-                if (frame)
-                        nfs_stack_destroy (nfl, frame);
-        }
+    call_frame_t *frame = NULL;
+    int ret = -EFAULT;
+    struct nfs_fop_local *nfl = NULL;
+    uint32_t accessbits = 0;
 
+    if ((!xl) || (!loc) || (!nfu))
         return ret;
+
+    gf_msg_trace(GF_NFS, 0, "Access: %s", loc->path);
+    nfs_fop_handle_frame_create(frame, nfsx, nfu, ret, err);
+    nfs_fop_handle_local_init(frame, nfsx, nfl, cbk, local, ret, err);
+    nfs_fop_save_root_ino(nfl, loc);
+
+    accessbits = nfs3_request_to_accessbits(accesstest);
+    STACK_WIND_COOKIE(frame, nfs_fop_access_cbk, xl, xl, xl->fops->access, loc,
+                      accessbits, NULL);
+    ret = 0;
+err:
+    if (ret < 0) {
+        if (frame)
+            nfs_stack_destroy(nfl, frame);
+    }
+
+    return ret;
 }
 
 int32_t
-nfs_fop_stat_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                  int32_t op_ret, int32_t op_errno, struct iatt *buf,
-                  dict_t *xdata)
+nfs_fop_stat_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                 int32_t op_ret, int32_t op_errno, struct iatt *buf,
+                 dict_t *xdata)
 {
-        struct nfs_fop_local    *nfl = NULL;
-        fop_stat_cbk_t          progcbk = NULL;
+    struct nfs_fop_local *nfl = NULL;
+    fop_stat_cbk_t progcbk = NULL;
 
-        nfl_to_prog_data (nfl, progcbk, frame);
-        nfs_fop_restore_root_ino (nfl, op_ret, buf, NULL, NULL, NULL);
-        if (progcbk)
-                progcbk (frame, cookie, this, op_ret, op_errno, buf, xdata);
+    nfl_to_prog_data(nfl, progcbk, frame);
+    nfs_fop_restore_root_ino(nfl, op_ret, buf, NULL, NULL, NULL);
+    if (progcbk)
+        progcbk(frame, cookie, this, op_ret, op_errno, buf, xdata);
 
-        nfs_stack_destroy (nfl, frame);
-        return 0;
+    nfs_stack_destroy(nfl, frame);
+    return 0;
 }
 
-
 int
-nfs_fop_stat (xlator_t *nfsx, xlator_t *xl, nfs_user_t *nfu, loc_t *loc,
-              fop_stat_cbk_t cbk, void *local)
+nfs_fop_stat(xlator_t *nfsx, xlator_t *xl, nfs_user_t *nfu, loc_t *loc,
+             fop_stat_cbk_t cbk, void *local)
 {
-        call_frame_t            *frame = NULL;
-        int                     ret = -EFAULT;
-        struct nfs_fop_local    *nfl = NULL;
+    call_frame_t *frame = NULL;
+    int ret = -EFAULT;
+    struct nfs_fop_local *nfl = NULL;
 
-        if ((!xl) || (!loc) || (!nfu))
-                return ret;
+    if ((!xl) || (!loc) || (!nfu))
+        return ret;
 
-        gf_log (GF_NFS, GF_LOG_TRACE, "Stat: %s", loc->path);
-        nfs_fop_handle_frame_create (frame, nfsx, nfu, ret, err);
-        nfs_fop_handle_local_init (frame, nfsx, nfl, cbk, local, ret, err);
-        nfs_fop_save_root_ino (nfl, loc);
+    gf_msg_trace(GF_NFS, 0, "Stat: %s", loc->path);
+    nfs_fop_handle_frame_create(frame, nfsx, nfu, ret, err);
+    nfs_fop_handle_local_init(frame, nfsx, nfl, cbk, local, ret, err);
+    nfs_fop_save_root_ino(nfl, loc);
 
-        STACK_WIND_COOKIE (frame, nfs_fop_stat_cbk, xl, xl, xl->fops->stat,
-                           loc, NULL);
-        ret = 0;
+    STACK_WIND_COOKIE(frame, nfs_fop_stat_cbk, xl, xl, xl->fops->stat, loc,
+                      NULL);
+    ret = 0;
 err:
-        if (ret < 0) {
-                if (frame)
-                        nfs_stack_destroy (nfl, frame);
-        }
+    if (ret < 0) {
+        if (frame)
+            nfs_stack_destroy(nfl, frame);
+    }
 
-        return ret;
+    return ret;
 }
 
-
 int32_t
-nfs_fop_fstat_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                   int32_t op_ret, int32_t op_errno, struct iatt *buf,
-                   dict_t *xdata)
+nfs_fop_fstat_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                  int32_t op_ret, int32_t op_errno, struct iatt *buf,
+                  dict_t *xdata)
 {
-        struct nfs_fop_local    *nfl = NULL;
-        fop_fstat_cbk_t         progcbk = NULL;
+    struct nfs_fop_local *nfl = NULL;
+    fop_fstat_cbk_t progcbk = NULL;
 
-        nfl_to_prog_data (nfl, progcbk, frame);
-        nfs_fop_restore_root_ino (nfl, op_ret, buf, NULL, NULL, NULL);
-        if (progcbk)
-                progcbk (frame, cookie, this, op_ret, op_errno, buf, xdata);
+    nfl_to_prog_data(nfl, progcbk, frame);
+    nfs_fop_restore_root_ino(nfl, op_ret, buf, NULL, NULL, NULL);
+    if (progcbk)
+        progcbk(frame, cookie, this, op_ret, op_errno, buf, xdata);
 
-        nfs_stack_destroy (nfl, frame);
-        return 0;
+    nfs_stack_destroy(nfl, frame);
+    return 0;
 }
 
-
 int
-nfs_fop_fstat (xlator_t *nfsx, xlator_t *xl, nfs_user_t *nfu, fd_t *fd,
-               fop_fstat_cbk_t cbk, void *local)
+nfs_fop_fstat(xlator_t *nfsx, xlator_t *xl, nfs_user_t *nfu, fd_t *fd,
+              fop_fstat_cbk_t cbk, void *local)
 {
-        call_frame_t            *frame = NULL;
-        int                     ret = -EFAULT;
-        struct nfs_fop_local    *nfl = NULL;
+    call_frame_t *frame = NULL;
+    int ret = -EFAULT;
+    struct nfs_fop_local *nfl = NULL;
 
-        if ((!nfsx) || (!xl) || (!fd) || (!nfu))
-                return ret;
+    if ((!nfsx) || (!xl) || (!fd) || (!nfu))
+        return ret;
 
-        gf_log (GF_NFS, GF_LOG_TRACE, "FStat");
-        nfs_fop_handle_frame_create (frame, nfsx, nfu, ret, err);
-        nfs_fop_handle_local_init (frame, nfsx, nfl, cbk, local, ret, err);
-        nfs_fop_save_root_fd_ino (nfl, fd);
+    gf_msg_trace(GF_NFS, 0, "FStat");
+    nfs_fop_handle_frame_create(frame, nfsx, nfu, ret, err);
+    nfs_fop_handle_local_init(frame, nfsx, nfl, cbk, local, ret, err);
+    nfs_fop_save_root_fd_ino(nfl, fd);
 
-        STACK_WIND_COOKIE (frame, nfs_fop_fstat_cbk, xl, xl, xl->fops->fstat,
-                           fd, NULL);
+    STACK_WIND_COOKIE(frame, nfs_fop_fstat_cbk, xl, xl, xl->fops->fstat, fd,
+                      NULL);
 
-        ret = 0;
+    ret = 0;
 err:
-        if (ret < 0) {
-                if (frame)
-                        nfs_stack_destroy (nfl, frame);
-        }
+    if (ret < 0) {
+        if (frame)
+            nfs_stack_destroy(nfl, frame);
+    }
 
-        return ret;
+    return ret;
 }
 
-
 int32_t
-nfs_fop_opendir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                     int32_t op_ret, int32_t op_errno, fd_t *fd, dict_t *xdata)
+nfs_fop_opendir_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                    int32_t op_ret, int32_t op_errno, fd_t *fd, dict_t *xdata)
 {
-        struct nfs_fop_local    *nfl = NULL;
-        fop_opendir_cbk_t       progcbk = NULL;
-
-        nfl_to_prog_data (nfl, progcbk, frame);
-        if (progcbk)
-                progcbk (frame, cookie, this, op_ret, op_errno, fd, xdata);
-        nfs_stack_destroy (nfl, frame);
-        return 0;
+    struct nfs_fop_local *nfl = NULL;
+    fop_opendir_cbk_t progcbk = NULL;
+
+    nfl_to_prog_data(nfl, progcbk, frame);
+    if (progcbk)
+        progcbk(frame, cookie, this, op_ret, op_errno, fd, xdata);
+    nfs_stack_destroy(nfl, frame);
+    return 0;
 }
 
-
 int
-nfs_fop_opendir (xlator_t *nfsx, xlator_t *xl, nfs_user_t *nfu, loc_t *pathloc,
-                 fd_t *dirfd, fop_opendir_cbk_t cbk, void *local)
+nfs_fop_opendir(xlator_t *nfsx, xlator_t *xl, nfs_user_t *nfu, loc_t *pathloc,
+                fd_t *dirfd, fop_opendir_cbk_t cbk, void *local)
 {
-        call_frame_t            *frame = NULL;
-        int                     ret = -EFAULT;
-        struct nfs_fop_local    *nfl = NULL;
+    call_frame_t *frame = NULL;
+    int ret = -EFAULT;
+    struct nfs_fop_local *nfl = NULL;
 
-        if ((!nfsx) || (!xl) || (!pathloc) || (!dirfd) || (!nfu))
-                return ret;
+    if ((!nfsx) || (!xl) || (!pathloc) || (!dirfd) || (!nfu))
+        return ret;
 
-        gf_log (GF_NFS, GF_LOG_TRACE, "Opendir: %s", pathloc->path);
-        nfs_fop_handle_frame_create (frame, nfsx, nfu, ret, err);
-        nfs_fop_handle_local_init (frame, nfsx, nfl, cbk, local, ret, err);
+    gf_msg_trace(GF_NFS, 0, "Opendir: %s", pathloc->path);
+    nfs_fop_handle_frame_create(frame, nfsx, nfu, ret, err);
+    nfs_fop_handle_local_init(frame, nfsx, nfl, cbk, local, ret, err);
 
-        STACK_WIND_COOKIE (frame, nfs_fop_opendir_cbk, xl, xl,
-                           xl->fops->opendir, pathloc, dirfd, NULL);
-        ret = 0;
+    STACK_WIND_COOKIE(frame, nfs_fop_opendir_cbk, xl, xl, xl->fops->opendir,
+                      pathloc, dirfd, NULL);
+    ret = 0;
 
 err:
-        if (ret < 0) {
-                if (frame)
-                        nfs_stack_destroy (nfl, frame);
-        }
+    if (ret < 0) {
+        if (frame)
+            nfs_stack_destroy(nfl, frame);
+    }
 
-        return ret;
+    return ret;
 }
 
 int
-nfs_fop_flush_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                   int32_t op_ret, int32_t op_errno, dict_t *xdata)
+nfs_fop_flush_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                  int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
-        struct nfs_fop_local    *nfl = NULL;
-        fop_flush_cbk_t         progcbk = NULL;
+    struct nfs_fop_local *nfl = NULL;
+    fop_flush_cbk_t progcbk = NULL;
 
-        nfl_to_prog_data (nfl, progcbk, frame);
-        if (progcbk)
-                progcbk (frame, cookie, this, op_ret, op_errno, xdata);
+    nfl_to_prog_data(nfl, progcbk, frame);
+    if (progcbk)
+        progcbk(frame, cookie, this, op_ret, op_errno, xdata);
 
-        nfs_stack_destroy (nfl, frame);
-        return 0;
+    nfs_stack_destroy(nfl, frame);
+    return 0;
 }
 
-
 int
-nfs_fop_flush (xlator_t *nfsx, xlator_t *xl, nfs_user_t *nfu, fd_t *fd,
-               fop_flush_cbk_t cbk, void *local)
+nfs_fop_flush(xlator_t *nfsx, xlator_t *xl, nfs_user_t *nfu, fd_t *fd,
+              fop_flush_cbk_t cbk, void *local)
 {
-        call_frame_t            *frame = NULL;
-        int                     ret = -EFAULT;
-        struct nfs_fop_local    *nfl = NULL;
+    call_frame_t *frame = NULL;
+    int ret = -EFAULT;
+    struct nfs_fop_local *nfl = NULL;
 
-        if ((!nfsx) || (!xl) || (!fd) || (!nfu))
-                return ret;
+    if ((!nfsx) || (!xl) || (!fd) || (!nfu))
+        return ret;
 
-        nfs_fop_handle_frame_create (frame, nfsx, nfu, ret, err);
-        nfs_fop_handle_local_init (frame, nfsx, nfl, cbk, local, ret, err);
+    nfs_fop_handle_frame_create(frame, nfsx, nfu, ret, err);
+    nfs_fop_handle_local_init(frame, nfsx, nfl, cbk, local, ret, err);
 
-        STACK_WIND_COOKIE (frame, nfs_fop_flush_cbk, xl, xl, xl->fops->flush,
-                           fd, NULL);
-        ret = 0;
+    STACK_WIND_COOKIE(frame, nfs_fop_flush_cbk, xl, xl, xl->fops->flush, fd,
+                      NULL);
+    ret = 0;
 err:
-        if (ret < 0) {
-                if (frame)
-                        nfs_stack_destroy (nfl, frame);
-        }
+    if (ret < 0) {
+        if (frame)
+            nfs_stack_destroy(nfl, frame);
+    }
 
-        return ret;
+    return ret;
 }
 
-
 int32_t
-nfs_fop_readdirp_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                      int32_t op_ret, int32_t op_errno, gf_dirent_t *entries,
-                      dict_t *xdata)
+nfs_fop_readdirp_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                     int32_t op_ret, int32_t op_errno, gf_dirent_t *entries,
+                     dict_t *xdata)
 {
-        struct nfs_fop_local    *nfl = NULL;
-        fop_readdirp_cbk_t      progcbk = NULL;
+    struct nfs_fop_local *nfl = NULL;
+    fop_readdirp_cbk_t progcbk = NULL;
 
-        nfl_to_prog_data (nfl, progcbk, frame);
-        if (progcbk)
-                progcbk (frame, cookie, this, op_ret, op_errno, entries, xdata);
+    nfl_to_prog_data(nfl, progcbk, frame);
+    if (progcbk)
+        progcbk(frame, cookie, this, op_ret, op_errno, entries, xdata);
 
-        nfs_stack_destroy (nfl, frame);
+    nfs_stack_destroy(nfl, frame);
 
-        return 0;
+    return 0;
 }
 
-
 int
-nfs_fop_readdirp (xlator_t *nfsx, xlator_t *xl, nfs_user_t *nfu, fd_t *dirfd,
-                  size_t bufsize, off_t offset, fop_readdirp_cbk_t cbk,
-                  void *local)
+nfs_fop_readdirp(xlator_t *nfsx, xlator_t *xl, nfs_user_t *nfu, fd_t *dirfd,
+                 size_t bufsize, off_t offset, fop_readdirp_cbk_t cbk,
+                 void *local)
 {
-        call_frame_t            *frame = NULL;
-        int                     ret = -EFAULT;
-        struct nfs_fop_local    *nfl = NULL;
+    call_frame_t *frame = NULL;
+    int ret = -EFAULT;
+    struct nfs_fop_local *nfl = NULL;
 
-        if ((!nfsx) || (!xl) || (!dirfd) || (!nfu))
-                return ret;
+    if ((!nfsx) || (!xl) || (!dirfd) || (!nfu))
+        return ret;
 
-        gf_log (GF_NFS, GF_LOG_TRACE, "readdir");
-        nfs_fop_handle_frame_create (frame, nfsx, nfu, ret, err);
-        nfs_fop_handle_local_init (frame, nfsx, nfl, cbk, local, ret, err);
+    gf_msg_trace(GF_NFS, 0, "readdir");
+    nfs_fop_handle_frame_create(frame, nfsx, nfu, ret, err);
+    nfs_fop_handle_local_init(frame, nfsx, nfl, cbk, local, ret, err);
 
-        STACK_WIND_COOKIE (frame, nfs_fop_readdirp_cbk, xl, xl,
-                           xl->fops->readdirp, dirfd, bufsize, offset, 0);
+    STACK_WIND_COOKIE(frame, nfs_fop_readdirp_cbk, xl, xl, xl->fops->readdirp,
+                      dirfd, bufsize, offset, 0);
 
-        ret = 0;
+    ret = 0;
 err:
-        if (ret < 0) {
-                if (frame)
-                        nfs_stack_destroy (nfl, frame);
-        }
+    if (ret < 0) {
+        if (frame)
+            nfs_stack_destroy(nfl, frame);
+    }
 
-        return ret;
+    return ret;
 }
 
-
 int32_t
-nfs_fop_statfs_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                    int32_t op_ret, int32_t op_errno, struct statvfs *buf,
-                    dict_t *xdata)
+nfs_fop_statfs_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                   int32_t op_ret, int32_t op_errno, struct statvfs *buf,
+                   dict_t *xdata)
 {
+    struct nfs_fop_local *nfl = NULL;
+    fop_statfs_cbk_t progcbk = NULL;
 
-        struct nfs_fop_local    *nfl = NULL;
-        fop_statfs_cbk_t        progcbk = NULL;
+    nfl_to_prog_data(nfl, progcbk, frame);
+    if (progcbk)
+        progcbk(frame, cookie, this, op_ret, op_errno, buf, xdata);
 
-        nfl_to_prog_data (nfl, progcbk, frame);
-        if (progcbk)
-                progcbk (frame, cookie, this, op_ret, op_errno, buf, xdata);
-
-        nfs_stack_destroy (nfl, frame);
-        return 0;
+    nfs_stack_destroy(nfl, frame);
+    return 0;
 }
 
-
 int
-nfs_fop_statfs (xlator_t *nfsx, xlator_t *xl, nfs_user_t *nfu, loc_t *pathloc,
-                fop_statfs_cbk_t cbk, void *local)
+nfs_fop_statfs(xlator_t *nfsx, xlator_t *xl, nfs_user_t *nfu, loc_t *pathloc,
+               fop_statfs_cbk_t cbk, void *local)
 {
-        call_frame_t            *frame = NULL;
-        int                     ret = -EFAULT;
-        struct nfs_fop_local    *nfl = NULL;
+    call_frame_t *frame = NULL;
+    int ret = -EFAULT;
+    struct nfs_fop_local *nfl = NULL;
 
-        if ((!nfsx) || (!xl) || (!pathloc) || (!nfu))
-                return ret;
+    if ((!nfsx) || (!xl) || (!pathloc) || (!nfu))
+        return ret;
 
-        gf_log (GF_NFS, GF_LOG_TRACE, "Statfs: %s", pathloc->path);
-        nfs_fop_handle_frame_create (frame, nfsx, nfu, ret, err);
-        nfs_fop_handle_local_init (frame, nfsx, nfl, cbk, local, ret, err);
+    gf_msg_trace(GF_NFS, 0, "Statfs: %s", pathloc->path);
+    nfs_fop_handle_frame_create(frame, nfsx, nfu, ret, err);
+    nfs_fop_handle_local_init(frame, nfsx, nfl, cbk, local, ret, err);
 
-        STACK_WIND_COOKIE (frame, nfs_fop_statfs_cbk, xl, xl,
-                           xl->fops->statfs, pathloc, NULL);
-        ret = 0;
+    STACK_WIND_COOKIE(frame, nfs_fop_statfs_cbk, xl, xl, xl->fops->statfs,
+                      pathloc, NULL);
+    ret = 0;
 err:
-        if (ret < 0) {
-                if (frame)
-                        nfs_stack_destroy (nfl, frame);
-        }
+    if (ret < 0) {
+        if (frame)
+            nfs_stack_destroy(nfl, frame);
+    }
 
-        return ret;
+    return ret;
 }
 
-
 int32_t
-nfs_fop_create_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                    int32_t op_ret, int32_t op_errno, fd_t *fd, inode_t *inode,
-                    struct iatt *buf, struct iatt *preparent,
-                    struct iatt *postparent, dict_t *xdata)
+nfs_fop_create_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                   int32_t op_ret, int32_t op_errno, fd_t *fd, inode_t *inode,
+                   struct iatt *buf, struct iatt *preparent,
+                   struct iatt *postparent, dict_t *xdata)
 {
-        struct nfs_fop_local    *nfl = NULL;
-        fop_create_cbk_t        progcbk = NULL;
+    struct nfs_fop_local *nfl = NULL;
+    fop_create_cbk_t progcbk = NULL;
 
-        if (op_ret == 0) {
-                nfs_fix_generation(this,inode);
-        }
+    if (op_ret == 0) {
+        nfs_fix_generation(this, inode);
+    }
 
-        nfl_to_prog_data (nfl, progcbk, frame);
-        nfs_fop_restore_root_ino (nfl, op_ret, buf, NULL, preparent,
-                                  postparent);
-        if (progcbk)
-                progcbk (frame, cookie, this, op_ret, op_errno, fd, inode, buf,
-                         preparent, postparent, NULL);
+    nfl_to_prog_data(nfl, progcbk, frame);
+    nfs_fop_restore_root_ino(nfl, op_ret, buf, NULL, preparent, postparent);
+    if (progcbk)
+        progcbk(frame, cookie, this, op_ret, op_errno, fd, inode, buf,
+                preparent, postparent, NULL);
 
-        nfs_stack_destroy (nfl, frame);
-        return 0;
+    nfs_stack_destroy(nfl, frame);
+    return 0;
 }
 
-
 int
-nfs_fop_create (xlator_t *nfsx, xlator_t *xl, nfs_user_t *nfu, loc_t *pathloc,
-                int flags, mode_t mode, fd_t *fd, fop_create_cbk_t cbk,
-                void *local)
+nfs_fop_create(xlator_t *nfsx, xlator_t *xl, nfs_user_t *nfu, loc_t *pathloc,
+               int flags, mode_t mode, fd_t *fd, fop_create_cbk_t cbk,
+               void *local)
 {
-        call_frame_t            *frame = NULL;
-        int                     ret = -EFAULT;
-        struct nfs_fop_local    *nfl = NULL;
+    call_frame_t *frame = NULL;
+    int ret = -EFAULT;
+    struct nfs_fop_local *nfl = NULL;
 
-        if ((!nfsx) || (!xl) || (!pathloc) || (!nfu))
-                return ret;
+    if ((!nfsx) || (!xl) || (!pathloc) || (!nfu))
+        return ret;
 
-        gf_log (GF_NFS, GF_LOG_TRACE, "Create: %s", pathloc->path);
-        nfs_fop_handle_frame_create (frame, nfsx, nfu, ret, err);
-        nfs_fop_handle_local_init (frame, nfsx, nfl, cbk, local, ret, err);
-        nfs_fop_save_root_ino (nfl, pathloc);
-        nfs_fop_gfid_setup (nfl, pathloc->inode, ret, err);
+    gf_msg_trace(GF_NFS, 0, "Create: %s", pathloc->path);
+    nfs_fop_handle_frame_create(frame, nfsx, nfu, ret, err);
+    nfs_fop_handle_local_init(frame, nfsx, nfl, cbk, local, ret, err);
+    nfs_fop_save_root_ino(nfl, pathloc);
+    nfs_fop_gfid_setup(nfl, pathloc->inode, ret, err);
 
-        STACK_WIND_COOKIE (frame, nfs_fop_create_cbk, xl, xl, xl->fops->create,
-                           pathloc, flags, mode, 0, fd, nfl->dictgfid);
+    STACK_WIND_COOKIE(frame, nfs_fop_create_cbk, xl, xl, xl->fops->create,
+                      pathloc, flags, mode, 0, fd, nfl->dictgfid);
 
-        ret = 0;
+    ret = 0;
 err:
-        if (ret < 0) {
-                if (frame)
-                        nfs_stack_destroy (nfl, frame);
-        }
+    if (ret < 0) {
+        if (frame)
+            nfs_stack_destroy(nfl, frame);
+    }
 
-        return ret;
+    return ret;
 }
 
-
 int32_t
-nfs_fop_setattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                     int32_t op_ret, int32_t op_errno, struct iatt *pre,
-                     struct iatt *post, dict_t *xdata)
+nfs_fop_setattr_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                    int32_t op_ret, int32_t op_errno, struct iatt *pre,
+                    struct iatt *post, dict_t *xdata)
 {
-        struct nfs_fop_local    *nfl = NULL;
-        fop_setattr_cbk_t       progcbk = NULL;
-
-        nfl_to_prog_data (nfl, progcbk, frame);
-        nfs_fop_restore_root_ino (nfl, op_ret, pre, post, NULL, NULL);
-        if (progcbk)
-                progcbk (frame, cookie, this, op_ret, op_errno, pre, post,
-                         xdata);
-        nfs_stack_destroy (nfl, frame);
-        return 0;
+    struct nfs_fop_local *nfl = NULL;
+    fop_setattr_cbk_t progcbk = NULL;
+
+    nfl_to_prog_data(nfl, progcbk, frame);
+    nfs_fop_restore_root_ino(nfl, op_ret, pre, post, NULL, NULL);
+    if (progcbk)
+        progcbk(frame, cookie, this, op_ret, op_errno, pre, post, xdata);
+    nfs_stack_destroy(nfl, frame);
+    return 0;
 }
 
-
-
 int
-nfs_fop_setattr (xlator_t *nfsx, xlator_t *xl, nfs_user_t *nfu, loc_t *pathloc,
-                 struct iatt *buf, int32_t valid, fop_setattr_cbk_t cbk,
-                 void *local)
+nfs_fop_setattr(xlator_t *nfsx, xlator_t *xl, nfs_user_t *nfu, loc_t *pathloc,
+                struct iatt *buf, int32_t valid, fop_setattr_cbk_t cbk,
+                void *local)
 {
-        call_frame_t            *frame = NULL;
-        int                     ret = -EFAULT;
-        struct nfs_fop_local    *nfl = NULL;
+    call_frame_t *frame = NULL;
+    int ret = -EFAULT;
+    struct nfs_fop_local *nfl = NULL;
 
-        if ((!nfsx) || (!xl) || (!pathloc) || (!nfu))
-                return ret;
+    if ((!nfsx) || (!xl) || (!pathloc) || (!nfu))
+        return ret;
 
-        gf_log (GF_NFS, GF_LOG_TRACE, "Setattr: %s", pathloc->path);
-        nfs_fop_handle_frame_create (frame, nfsx, nfu, ret, err);
-        nfs_fop_handle_local_init (frame, nfsx, nfl, cbk, local, ret, err);
-        nfs_fop_save_root_ino (nfl, pathloc);
+    gf_msg_trace(GF_NFS, 0, "Setattr: %s", pathloc->path);
+    nfs_fop_handle_frame_create(frame, nfsx, nfu, ret, err);
+    nfs_fop_handle_local_init(frame, nfsx, nfl, cbk, local, ret, err);
+    nfs_fop_save_root_ino(nfl, pathloc);
 
-        STACK_WIND_COOKIE (frame, nfs_fop_setattr_cbk, xl, xl,
-                           xl->fops->setattr, pathloc, buf, valid, NULL);
-        ret = 0;
+    STACK_WIND_COOKIE(frame, nfs_fop_setattr_cbk, xl, xl, xl->fops->setattr,
+                      pathloc, buf, valid, NULL);
+    ret = 0;
 err:
-        if (ret < 0) {
-                if (frame)
-                        nfs_stack_destroy (nfl, frame);
-        }
+    if (ret < 0) {
+        if (frame)
+            nfs_stack_destroy(nfl, frame);
+    }
 
-        return ret;
+    return ret;
 }
 
-
 int32_t
-nfs_fop_mkdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                   int32_t op_ret, int32_t op_errno, inode_t *inode,
-                   struct iatt *buf, struct iatt *preparent,
-                   struct iatt *postparent, dict_t *xdata)
+nfs_fop_mkdir_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                  int32_t op_ret, int32_t op_errno, inode_t *inode,
+                  struct iatt *buf, struct iatt *preparent,
+                  struct iatt *postparent, dict_t *xdata)
 {
-        struct nfs_fop_local    *nfl = NULL;
-        fop_mkdir_cbk_t         progcbk = NULL;
-
-        if (op_ret == 0) {
-                nfs_fix_generation(this,inode);
-        }
-
-        nfl_to_prog_data (nfl, progcbk, frame);
-        nfs_fop_restore_root_ino (nfl, op_ret, buf, NULL,preparent, postparent);
-        if (progcbk)
-                progcbk (frame, cookie, this, op_ret, op_errno, inode, buf,
-                         preparent, postparent, xdata);
-        nfs_stack_destroy (nfl, frame);
-        return 0;
+    struct nfs_fop_local *nfl = NULL;
+    fop_mkdir_cbk_t progcbk = NULL;
+
+    if (op_ret == 0) {
+        nfs_fix_generation(this, inode);
+    }
+
+    nfl_to_prog_data(nfl, progcbk, frame);
+    nfs_fop_restore_root_ino(nfl, op_ret, buf, NULL, preparent, postparent);
+    if (progcbk)
+        progcbk(frame, cookie, this, op_ret, op_errno, inode, buf, preparent,
+                postparent, xdata);
+    nfs_stack_destroy(nfl, frame);
+    return 0;
 }
 
-
 int
-nfs_fop_mkdir (xlator_t *nfsx, xlator_t *xl, nfs_user_t *nfu, loc_t *pathloc,
-               mode_t mode, fop_mkdir_cbk_t cbk, void *local)
+nfs_fop_mkdir(xlator_t *nfsx, xlator_t *xl, nfs_user_t *nfu, loc_t *pathloc,
+              mode_t mode, fop_mkdir_cbk_t cbk, void *local)
 {
-        call_frame_t            *frame = NULL;
-        int                     ret = -EFAULT;
-        struct nfs_fop_local    *nfl = NULL;
-
-        if ((!nfsx) || (!xl) || (!pathloc) || (!nfu))
-                return ret;
-
-        gf_log (GF_NFS, GF_LOG_TRACE, "Mkdir: %s", pathloc->path);
-        nfs_fop_handle_frame_create (frame, nfsx, nfu, ret, err);
-        nfs_fop_handle_local_init (frame, nfsx, nfl, cbk, local, ret, err);
-        nfs_fop_save_root_ino (nfl, pathloc);
-        nfs_fop_gfid_setup (nfl, pathloc->inode, ret, err);
-
-        STACK_WIND_COOKIE  (frame, nfs_fop_mkdir_cbk, xl, xl, xl->fops->mkdir,
-                            pathloc, mode, 0, nfl->dictgfid);
-        ret = 0;
-err:
-        if (ret < 0) {
-                if (frame)
-                        nfs_stack_destroy (nfl, frame);
-        }
+    call_frame_t *frame = NULL;
+    int ret = -EFAULT;
+    struct nfs_fop_local *nfl = NULL;
 
+    if ((!nfsx) || (!xl) || (!pathloc) || (!nfu))
         return ret;
-}
 
+    gf_msg_trace(GF_NFS, 0, "Mkdir: %s", pathloc->path);
+    nfs_fop_handle_frame_create(frame, nfsx, nfu, ret, err);
+    nfs_fop_handle_local_init(frame, nfsx, nfl, cbk, local, ret, err);
+    nfs_fop_save_root_ino(nfl, pathloc);
+    nfs_fop_gfid_setup(nfl, pathloc->inode, ret, err);
 
-int32_t
-nfs_fop_symlink_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                     int32_t op_ret, int32_t op_errno, inode_t *inode,
-                     struct iatt *buf, struct iatt *preparent,
-                     struct iatt *postparent, dict_t *xdata)
-{
-        struct nfs_fop_local    *nfl = NULL;
-        fop_symlink_cbk_t       progcbk = NULL;
+    STACK_WIND_COOKIE(frame, nfs_fop_mkdir_cbk, xl, xl, xl->fops->mkdir,
+                      pathloc, mode, 0, nfl->dictgfid);
+    ret = 0;
+err:
+    if (ret < 0) {
+        if (frame)
+            nfs_stack_destroy(nfl, frame);
+    }
 
-        if (op_ret == 0) {
-                nfs_fix_generation(this,inode);
-        }
+    return ret;
+}
 
-        nfl_to_prog_data (nfl, progcbk, frame);
-        nfs_fop_restore_root_ino (nfl, op_ret,buf, NULL, preparent, postparent);
-        if (progcbk)
-                progcbk (frame, cookie, this, op_ret, op_errno, inode, buf,
-                         preparent, postparent, xdata);
-        nfs_stack_destroy (nfl, frame);
-        return 0;
+int32_t
+nfs_fop_symlink_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                    int32_t op_ret, int32_t op_errno, inode_t *inode,
+                    struct iatt *buf, struct iatt *preparent,
+                    struct iatt *postparent, dict_t *xdata)
+{
+    struct nfs_fop_local *nfl = NULL;
+    fop_symlink_cbk_t progcbk = NULL;
+
+    if (op_ret == 0) {
+        nfs_fix_generation(this, inode);
+    }
+
+    nfl_to_prog_data(nfl, progcbk, frame);
+    nfs_fop_restore_root_ino(nfl, op_ret, buf, NULL, preparent, postparent);
+    if (progcbk)
+        progcbk(frame, cookie, this, op_ret, op_errno, inode, buf, preparent,
+                postparent, xdata);
+    nfs_stack_destroy(nfl, frame);
+    return 0;
 }
 
 int
-nfs_fop_symlink (xlator_t *nfsx, xlator_t *xl, nfs_user_t *nfu, char *target,
-                 loc_t *pathloc, fop_symlink_cbk_t cbk, void *local)
+nfs_fop_symlink(xlator_t *nfsx, xlator_t *xl, nfs_user_t *nfu, char *target,
+                loc_t *pathloc, fop_symlink_cbk_t cbk, void *local)
 {
-        call_frame_t            *frame = NULL;
-        int                     ret = -EFAULT;
-        struct nfs_fop_local    *nfl = NULL;
-
-        if ((!nfsx) || (!xl) || (!pathloc) || (!target) || (!nfu))
-                return ret;
-
-        gf_log (GF_NFS, GF_LOG_TRACE, "Symlink: %s", pathloc->path);
-        nfs_fop_handle_frame_create (frame, nfsx, nfu, ret, err);
-        nfs_fop_handle_local_init (frame, nfsx, nfl, cbk, local, ret, err);
-        nfs_fop_save_root_ino (nfl, pathloc);
-        nfs_fop_gfid_setup (nfl, pathloc->inode, ret, err);
-
-        STACK_WIND_COOKIE  (frame, nfs_fop_symlink_cbk, xl, xl,
-                            xl->fops->symlink, target, pathloc,
-                            0, nfl->dictgfid);
-        ret = 0;
-err:
-        if (ret < 0) {
-                if (frame)
-                        nfs_stack_destroy (nfl, frame);
-        }
+    call_frame_t *frame = NULL;
+    int ret = -EFAULT;
+    struct nfs_fop_local *nfl = NULL;
 
+    if ((!nfsx) || (!xl) || (!pathloc) || (!target) || (!nfu))
         return ret;
-}
 
+    gf_msg_trace(GF_NFS, 0, "Symlink: %s", pathloc->path);
+    nfs_fop_handle_frame_create(frame, nfsx, nfu, ret, err);
+    nfs_fop_handle_local_init(frame, nfsx, nfl, cbk, local, ret, err);
+    nfs_fop_save_root_ino(nfl, pathloc);
+    nfs_fop_gfid_setup(nfl, pathloc->inode, ret, err);
+
+    STACK_WIND_COOKIE(frame, nfs_fop_symlink_cbk, xl, xl, xl->fops->symlink,
+                      target, pathloc, 0, nfl->dictgfid);
+    ret = 0;
+err:
+    if (ret < 0) {
+        if (frame)
+            nfs_stack_destroy(nfl, frame);
+    }
+
+    return ret;
+}
 
 int32_t
-nfs_fop_readlink_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                      int32_t op_ret, int32_t op_errno, const char *path,
-                      struct iatt *buf, dict_t *xdata)
+nfs_fop_readlink_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                     int32_t op_ret, int32_t op_errno, const char *path,
+                     struct iatt *buf, dict_t *xdata)
 {
-        struct nfs_fop_local    *nfl = NULL;
-        fop_readlink_cbk_t      progcbk = NULL;
-
-        nfl_to_prog_data (nfl, progcbk, frame);
-        nfs_fop_restore_root_ino (nfl, op_ret, buf, NULL, NULL, NULL);
-        if (progcbk)
-                progcbk (frame, cookie, this, op_ret, op_errno, path, buf,
-                         xdata);
-        nfs_stack_destroy (nfl, frame);
-        return 0;
+    struct nfs_fop_local *nfl = NULL;
+    fop_readlink_cbk_t progcbk = NULL;
+
+    nfl_to_prog_data(nfl, progcbk, frame);
+    nfs_fop_restore_root_ino(nfl, op_ret, buf, NULL, NULL, NULL);
+    if (progcbk)
+        progcbk(frame, cookie, this, op_ret, op_errno, path, buf, xdata);
+    nfs_stack_destroy(nfl, frame);
+    return 0;
 }
 
-
 int
-nfs_fop_readlink (xlator_t *nfsx, xlator_t *xl, nfs_user_t *nfu, loc_t *pathloc,
-                  size_t size, fop_readlink_cbk_t cbk, void *local)
+nfs_fop_readlink(xlator_t *nfsx, xlator_t *xl, nfs_user_t *nfu, loc_t *pathloc,
+                 size_t size, fop_readlink_cbk_t cbk, void *local)
 {
-        call_frame_t            *frame = NULL;
-        int                     ret = -EFAULT;
-        struct nfs_fop_local    *nfl = NULL;
+    call_frame_t *frame = NULL;
+    int ret = -EFAULT;
+    struct nfs_fop_local *nfl = NULL;
 
-        if ((!nfsx) || (!xl) || (!pathloc) || (!nfu))
-                return ret;
+    if ((!nfsx) || (!xl) || (!pathloc) || (!nfu))
+        return ret;
 
-        gf_log (GF_NFS, GF_LOG_TRACE, "Readlink: %s", pathloc->path);
-        nfs_fop_handle_frame_create (frame, nfsx, nfu, ret, err);
-        nfs_fop_handle_local_init (frame, nfsx, nfl, cbk, local, ret, err);
-        nfs_fop_save_root_ino (nfl, pathloc);
+    gf_msg_trace(GF_NFS, 0, "Readlink: %s", pathloc->path);
+    nfs_fop_handle_frame_create(frame, nfsx, nfu, ret, err);
+    nfs_fop_handle_local_init(frame, nfsx, nfl, cbk, local, ret, err);
+    nfs_fop_save_root_ino(nfl, pathloc);
 
-        STACK_WIND_COOKIE (frame, nfs_fop_readlink_cbk, xl, xl,
-                           xl->fops->readlink, pathloc, size, NULL);
-        ret = 0;
+    STACK_WIND_COOKIE(frame, nfs_fop_readlink_cbk, xl, xl, xl->fops->readlink,
+                      pathloc, size, NULL);
+    ret = 0;
 err:
-        if (ret < 0) {
-                if (frame)
-                        nfs_stack_destroy (nfl, frame);
-        }
+    if (ret < 0) {
+        if (frame)
+            nfs_stack_destroy(nfl, frame);
+    }
 
-        return ret;
+    return ret;
 }
 
-
 int32_t
-nfs_fop_mknod_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                   int32_t op_ret, int32_t op_errno, inode_t *inode,
-                   struct iatt *buf, struct iatt *preparent,
-                   struct iatt *postparent, dict_t *xdata)
+nfs_fop_mknod_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                  int32_t op_ret, int32_t op_errno, inode_t *inode,
+                  struct iatt *buf, struct iatt *preparent,
+                  struct iatt *postparent, dict_t *xdata)
 {
-        struct nfs_fop_local    *nfl = NULL;
-        fop_mknod_cbk_t         progcbk = NULL;
-
-        if (op_ret == 0) {
-                nfs_fix_generation(this,inode);
-        }
-
-        nfl_to_prog_data (nfl, progcbk, frame);
-        nfs_fop_restore_root_ino (nfl, op_ret,buf, NULL, preparent, postparent);
-        if (progcbk)
-                progcbk (frame, cookie, this, op_ret, op_errno, inode, buf,
-                         preparent, postparent, xdata);
-        nfs_stack_destroy (nfl, frame);
-        return 0;
+    struct nfs_fop_local *nfl = NULL;
+    fop_mknod_cbk_t progcbk = NULL;
+
+    if (op_ret == 0) {
+        nfs_fix_generation(this, inode);
+    }
+
+    nfl_to_prog_data(nfl, progcbk, frame);
+    nfs_fop_restore_root_ino(nfl, op_ret, buf, NULL, preparent, postparent);
+    if (progcbk)
+        progcbk(frame, cookie, this, op_ret, op_errno, inode, buf, preparent,
+                postparent, xdata);
+    nfs_stack_destroy(nfl, frame);
+    return 0;
 }
 
-
 int
-nfs_fop_mknod (xlator_t *nfsx, xlator_t *xl, nfs_user_t *nfu, loc_t *pathloc,
-               mode_t mode, dev_t dev, fop_mknod_cbk_t cbk, void *local)
+nfs_fop_mknod(xlator_t *nfsx, xlator_t *xl, nfs_user_t *nfu, loc_t *pathloc,
+              mode_t mode, dev_t dev, fop_mknod_cbk_t cbk, void *local)
 {
-        call_frame_t            *frame = NULL;
-        int                     ret = -EFAULT;
-        struct nfs_fop_local    *nfl = NULL;
-
-        if ((!nfsx) || (!xl) || (!pathloc) || (!nfu))
-                return ret;
-
-        gf_log (GF_NFS, GF_LOG_TRACE, "Mknod: %s", pathloc->path);
-        nfs_fop_handle_frame_create (frame, nfsx, nfu, ret, err);
-        nfs_fop_handle_local_init (frame, nfsx, nfl, cbk, local, ret, err);
-        nfs_fop_save_root_ino (nfl, pathloc);
-        nfs_fop_gfid_setup (nfl, pathloc->inode, ret, err);
-
-        STACK_WIND_COOKIE  (frame, nfs_fop_mknod_cbk, xl, xl, xl->fops->mknod,
-                            pathloc, mode, dev, 0, nfl->dictgfid);
-        ret = 0;
-err:
-        if (ret < 0) {
-                if (frame)
-                        nfs_stack_destroy (nfl, frame);
-        }
+    call_frame_t *frame = NULL;
+    int ret = -EFAULT;
+    struct nfs_fop_local *nfl = NULL;
 
+    if ((!nfsx) || (!xl) || (!pathloc) || (!nfu))
         return ret;
+
+    gf_msg_trace(GF_NFS, 0, "Mknod: %s", pathloc->path);
+    nfs_fop_handle_frame_create(frame, nfsx, nfu, ret, err);
+    nfs_fop_handle_local_init(frame, nfsx, nfl, cbk, local, ret, err);
+    nfs_fop_save_root_ino(nfl, pathloc);
+    nfs_fop_gfid_setup(nfl, pathloc->inode, ret, err);
+
+    STACK_WIND_COOKIE(frame, nfs_fop_mknod_cbk, xl, xl, xl->fops->mknod,
+                      pathloc, mode, dev, 0, nfl->dictgfid);
+    ret = 0;
+err:
+    if (ret < 0) {
+        if (frame)
+            nfs_stack_destroy(nfl, frame);
+    }
+
+    return ret;
 }
 
 int32_t
-nfs_fop_rmdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                   int32_t op_ret, int32_t op_errno, struct iatt *preparent,
-                   struct iatt *postparent, dict_t *xdata)
+nfs_fop_rmdir_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                  int32_t op_ret, int32_t op_errno, struct iatt *preparent,
+                  struct iatt *postparent, dict_t *xdata)
 {
-        struct nfs_fop_local    *nfl = frame->local;
-        fop_rmdir_cbk_t         progcbk = NULL;
-
-        nfl_to_prog_data (nfl, progcbk, frame);
-        nfs_fop_restore_root_ino (nfl, op_ret, NULL, NULL, preparent,
-                                  postparent);
-        if (progcbk)
-                progcbk (frame, cookie, this, op_ret, op_errno, preparent,
-                         postparent, NULL);
-        nfs_stack_destroy (nfl, frame);
-        return 0;
+    struct nfs_fop_local *nfl = frame->local;
+    fop_rmdir_cbk_t progcbk = NULL;
+
+    nfl_to_prog_data(nfl, progcbk, frame);
+    nfs_fop_restore_root_ino(nfl, op_ret, NULL, NULL, preparent, postparent);
+    if (progcbk)
+        progcbk(frame, cookie, this, op_ret, op_errno, preparent, postparent,
+                NULL);
+    nfs_stack_destroy(nfl, frame);
+    return 0;
 }
 
-
-
 int
-nfs_fop_rmdir (xlator_t *nfsx, xlator_t *xl, nfs_user_t *nfu, loc_t *pathloc,
-               fop_rmdir_cbk_t cbk, void *local)
+nfs_fop_rmdir(xlator_t *nfsx, xlator_t *xl, nfs_user_t *nfu, loc_t *pathloc,
+              fop_rmdir_cbk_t cbk, void *local)
 {
-        call_frame_t            *frame = NULL;
-        int                     ret = -EFAULT;
-        struct nfs_fop_local    *nfl = NULL;
+    call_frame_t *frame = NULL;
+    int ret = -EFAULT;
+    struct nfs_fop_local *nfl = NULL;
 
-        if ((!nfsx) || (!xl) || (!pathloc) || (!nfu))
-                return ret;
+    if ((!nfsx) || (!xl) || (!pathloc) || (!nfu))
+        return ret;
 
-        gf_log (GF_NFS, GF_LOG_TRACE, "Rmdir: %s", pathloc->path);
-        nfs_fop_handle_frame_create (frame, nfsx, nfu, ret, err);
-        nfs_fop_handle_local_init (frame, nfsx, nfl, cbk, local, ret, err);
-        nfs_fop_save_root_ino (nfl, pathloc);
+    gf_msg_trace(GF_NFS, 0, "Rmdir: %s", pathloc->path);
+    nfs_fop_handle_frame_create(frame, nfsx, nfu, ret, err);
+    nfs_fop_handle_local_init(frame, nfsx, nfl, cbk, local, ret, err);
+    nfs_fop_save_root_ino(nfl, pathloc);
 
-        STACK_WIND_COOKIE (frame, nfs_fop_rmdir_cbk, xl, xl, xl->fops->rmdir,
-                           pathloc, 0, NULL);
-        ret = 0;
+    STACK_WIND_COOKIE(frame, nfs_fop_rmdir_cbk, xl, xl, xl->fops->rmdir,
+                      pathloc, 0, NULL);
+    ret = 0;
 err:
-        if (ret < 0) {
-                if (frame)
-                        nfs_stack_destroy (nfl, frame);
-        }
-
-        return ret;
+    if (ret < 0) {
+        if (frame)
+            nfs_stack_destroy(nfl, frame);
+    }
 
+    return ret;
 }
 
-
 int32_t
-nfs_fop_unlink_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                    int32_t op_ret, int32_t op_errno, struct iatt *preparent,
-                    struct iatt *postparent, dict_t *xdata)
+nfs_fop_unlink_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                   int32_t op_ret, int32_t op_errno, struct iatt *preparent,
+                   struct iatt *postparent, dict_t *xdata)
 {
-        struct nfs_fop_local    *nfl = frame->local;
-        fop_unlink_cbk_t         progcbk = NULL;
-
-        nfl_to_prog_data (nfl, progcbk, frame);
-        nfs_fop_restore_root_ino (nfl, op_ret, NULL, NULL, preparent,
-                                  postparent);
-        if (progcbk)
-                progcbk (frame, cookie, this, op_ret, op_errno, preparent,
-                         postparent, xdata);
-        nfs_stack_destroy (nfl, frame);
-        return 0;
+    struct nfs_fop_local *nfl = frame->local;
+    fop_unlink_cbk_t progcbk = NULL;
+
+    nfl_to_prog_data(nfl, progcbk, frame);
+    nfs_fop_restore_root_ino(nfl, op_ret, NULL, NULL, preparent, postparent);
+    if (progcbk)
+        progcbk(frame, cookie, this, op_ret, op_errno, preparent, postparent,
+                xdata);
+    nfs_stack_destroy(nfl, frame);
+    return 0;
 }
 
-
 int
-nfs_fop_unlink (xlator_t *nfsx, xlator_t *xl, nfs_user_t *nfu, loc_t *pathloc,
-                fop_unlink_cbk_t cbk, void *local)
+nfs_fop_unlink(xlator_t *nfsx, xlator_t *xl, nfs_user_t *nfu, loc_t *pathloc,
+               fop_unlink_cbk_t cbk, void *local)
 {
-        call_frame_t            *frame = NULL;
-        int                     ret = -EFAULT;
-        struct nfs_fop_local    *nfl = NULL;
+    call_frame_t *frame = NULL;
+    int ret = -EFAULT;
+    struct nfs_fop_local *nfl = NULL;
 
-        if ((!nfsx) || (!xl) || (!pathloc) || (!nfu))
-                return ret;
+    if ((!nfsx) || (!xl) || (!pathloc) || (!nfu))
+        return ret;
 
-        gf_log (GF_NFS, GF_LOG_TRACE, "Unlink: %s", pathloc->path);
-        nfs_fop_handle_frame_create (frame, nfsx, nfu, ret, err);
-        nfs_fop_handle_local_init (frame, nfsx, nfl, cbk, local, ret, err);
-        nfs_fop_save_root_ino (nfl, pathloc);
+    gf_msg_trace(GF_NFS, 0, "Unlink: %s", pathloc->path);
+    nfs_fop_handle_frame_create(frame, nfsx, nfu, ret, err);
+    nfs_fop_handle_local_init(frame, nfsx, nfl, cbk, local, ret, err);
+    nfs_fop_save_root_ino(nfl, pathloc);
 
-        STACK_WIND_COOKIE (frame, nfs_fop_unlink_cbk, xl, xl,
-                           xl->fops->unlink, pathloc, 0, NULL);
-        ret = 0;
+    STACK_WIND_COOKIE(frame, nfs_fop_unlink_cbk, xl, xl, xl->fops->unlink,
+                      pathloc, 0, NULL);
+    ret = 0;
 err:
-        if (ret < 0) {
-                if (frame)
-                        nfs_stack_destroy (nfl, frame);
-        }
-
-        return ret;
+    if (ret < 0) {
+        if (frame)
+            nfs_stack_destroy(nfl, frame);
+    }
 
+    return ret;
 }
 
-
 int32_t
-nfs_fop_link_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                  int32_t op_ret, int32_t op_errno, inode_t *inode,
-                  struct iatt *buf, struct iatt *preparent,
-                  struct iatt *postparent, dict_t *xdata)
+nfs_fop_link_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                 int32_t op_ret, int32_t op_errno, inode_t *inode,
+                 struct iatt *buf, struct iatt *preparent,
+                 struct iatt *postparent, dict_t *xdata)
 {
-        struct nfs_fop_local    *nfl = NULL;
-        fop_link_cbk_t          progcbk = NULL;
+    struct nfs_fop_local *nfl = NULL;
+    fop_link_cbk_t progcbk = NULL;
 
-        if (op_ret == 0) {
-                nfs_fix_generation(this,inode);
-        }
+    if (op_ret == 0) {
+        nfs_fix_generation(this, inode);
+    }
 
-        nfl_to_prog_data (nfl, progcbk, frame);
-        nfs_fop_restore_root_ino (nfl, op_ret, buf, NULL, preparent,
-                                  postparent);
-        if (progcbk)
-                progcbk (frame, cookie, this, op_ret, op_errno, inode, buf,
-                         preparent, postparent, xdata);
+    nfl_to_prog_data(nfl, progcbk, frame);
+    nfs_fop_restore_root_ino(nfl, op_ret, buf, NULL, preparent, postparent);
+    if (progcbk)
+        progcbk(frame, cookie, this, op_ret, op_errno, inode, buf, preparent,
+                postparent, xdata);
 
-        nfs_stack_destroy (nfl, frame);
-        return 0;
+    nfs_stack_destroy(nfl, frame);
+    return 0;
 }
 
-
 int
-nfs_fop_link (xlator_t *nfsx, xlator_t *xl, nfs_user_t *nfu, loc_t *oldloc,
-              loc_t *newloc, fop_link_cbk_t cbk, void *local)
+nfs_fop_link(xlator_t *nfsx, xlator_t *xl, nfs_user_t *nfu, loc_t *oldloc,
+             loc_t *newloc, fop_link_cbk_t cbk, void *local)
 {
-        call_frame_t            *frame = NULL;
-        int                     ret = -EFAULT;
-        struct nfs_fop_local    *nfl = NULL;
-
-        if ((!nfsx) || (!xl) || (!oldloc) || (!newloc) || (!nfu))
-                return ret;
-
-        gf_log (GF_NFS, GF_LOG_TRACE, "Link: %s -> %s", newloc->path,
-                oldloc->path);
-        nfs_fop_handle_frame_create (frame, nfsx, nfu, ret, err);
-        nfs_fop_handle_local_init (frame, nfsx, nfl, cbk, local, ret, err);
-        nfs_fop_save_root_ino (nfl, newloc);
-
-        STACK_WIND_COOKIE (frame, nfs_fop_link_cbk, xl, xl, xl->fops->link,
-                           oldloc, newloc, NULL);
-        ret = 0;
-err:
-        if (ret < 0) {
-                if (frame)
-                        nfs_stack_destroy (nfl, frame);
-        }
+    call_frame_t *frame = NULL;
+    int ret = -EFAULT;
+    struct nfs_fop_local *nfl = NULL;
 
+    if ((!nfsx) || (!xl) || (!oldloc) || (!newloc) || (!nfu))
         return ret;
-}
 
+    gf_msg_trace(GF_NFS, 0, "Link: %s -> %s", newloc->path, oldloc->path);
+    nfs_fop_handle_frame_create(frame, nfsx, nfu, ret, err);
+    nfs_fop_handle_local_init(frame, nfsx, nfl, cbk, local, ret, err);
+    nfs_fop_save_root_ino(nfl, newloc);
 
-int32_t
-nfs_fop_rename_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                    int32_t op_ret, int32_t op_errno, struct iatt *buf,
-                    struct iatt *preoldparent, struct iatt *postoldparent,
-                    struct iatt *prenewparent, struct iatt *postnewparent,
-                    dict_t *xdata)
-{
+    STACK_WIND_COOKIE(frame, nfs_fop_link_cbk, xl, xl, xl->fops->link, oldloc,
+                      newloc, NULL);
+    ret = 0;
+err:
+    if (ret < 0) {
+        if (frame)
+            nfs_stack_destroy(nfl, frame);
+    }
 
-        struct nfs_fop_local    *nfl = NULL;
-        fop_rename_cbk_t        progcbk = NULL;
-
-        nfl_to_prog_data (nfl, progcbk, frame);
-        /* The preattr arg needs to be NULL instead of @buf because it is
-         * possible that the new parent is not root whereas the source dir
-         * could have been. That is handled in the next macro.
-         */
-        nfs_fop_restore_root_ino (nfl, op_ret, NULL, NULL, preoldparent,
-                                  postoldparent);
-        nfs_fop_newloc_restore_root_ino (nfl, op_ret, buf, NULL, prenewparent,
-                                         postnewparent);
-        if (progcbk)
-                progcbk (frame, cookie, this, op_ret, op_errno, buf,
-                         preoldparent, postoldparent, prenewparent,
-                         postnewparent, xdata);
-        nfs_stack_destroy (nfl, frame);
-        return 0;
+    return ret;
 }
 
+int32_t
+nfs_fop_rename_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                   int32_t op_ret, int32_t op_errno, struct iatt *buf,
+                   struct iatt *preoldparent, struct iatt *postoldparent,
+                   struct iatt *prenewparent, struct iatt *postnewparent,
+                   dict_t *xdata)
+{
+    struct nfs_fop_local *nfl = NULL;
+    fop_rename_cbk_t progcbk = NULL;
+
+    nfl_to_prog_data(nfl, progcbk, frame);
+    /* The preattr arg needs to be NULL instead of @buf because it is
+     * possible that the new parent is not root whereas the source dir
+     * could have been. That is handled in the next macro.
+     */
+    nfs_fop_restore_root_ino(nfl, op_ret, NULL, NULL, preoldparent,
+                             postoldparent);
+    nfs_fop_newloc_restore_root_ino(nfl, op_ret, buf, NULL, prenewparent,
+                                    postnewparent);
+    if (progcbk)
+        progcbk(frame, cookie, this, op_ret, op_errno, buf, preoldparent,
+                postoldparent, prenewparent, postnewparent, xdata);
+    nfs_stack_destroy(nfl, frame);
+    return 0;
+}
 
 int
-nfs_fop_rename (xlator_t *nfsx, xlator_t *xl, nfs_user_t *nfu, loc_t *oldloc,
-                loc_t *newloc, fop_rename_cbk_t cbk, void *local)
+nfs_fop_rename(xlator_t *nfsx, xlator_t *xl, nfs_user_t *nfu, loc_t *oldloc,
+               loc_t *newloc, fop_rename_cbk_t cbk, void *local)
 {
-        call_frame_t            *frame = NULL;
-        int                     ret = -EFAULT;
-        struct nfs_fop_local    *nfl = NULL;
-
-        if ((!nfsx) || (!xl) || (!oldloc) || (!newloc) || (!nfu))
-                return ret;
-
-        gf_log (GF_NFS, GF_LOG_TRACE, "Rename: %s -> %s", oldloc->path,
-                newloc->path);
-        nfs_fop_handle_frame_create (frame, nfsx, nfu, ret, err);
-        nfs_fop_handle_local_init (frame, nfsx, nfl, cbk, local, ret, err);
-        nfs_fop_save_root_ino (nfl, oldloc);
-        nfs_fop_newloc_save_root_ino (nfl, newloc);
-
-        STACK_WIND_COOKIE (frame, nfs_fop_rename_cbk, xl, xl,
-                           xl->fops->rename, oldloc, newloc, NULL);
-        ret = 0;
-err:
-        if (ret < 0) {
-                if (frame)
-                        nfs_stack_destroy (nfl, frame);
-        }
+    call_frame_t *frame = NULL;
+    int ret = -EFAULT;
+    struct nfs_fop_local *nfl = NULL;
 
+    if ((!nfsx) || (!xl) || (!oldloc) || (!newloc) || (!nfu))
         return ret;
-}
 
+    gf_msg_trace(GF_NFS, 0, "Rename: %s -> %s", oldloc->path, newloc->path);
+    nfs_fop_handle_frame_create(frame, nfsx, nfu, ret, err);
+    nfs_fop_handle_local_init(frame, nfsx, nfl, cbk, local, ret, err);
+    nfs_fop_save_root_ino(nfl, oldloc);
+    nfs_fop_newloc_save_root_ino(nfl, newloc);
+
+    STACK_WIND_COOKIE(frame, nfs_fop_rename_cbk, xl, xl, xl->fops->rename,
+                      oldloc, newloc, NULL);
+    ret = 0;
+err:
+    if (ret < 0) {
+        if (frame)
+            nfs_stack_destroy(nfl, frame);
+    }
+
+    return ret;
+}
 
 int32_t
-nfs_fop_open_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                  int32_t op_ret, int32_t op_errno, fd_t *fd, dict_t *xdata)
+nfs_fop_open_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                 int32_t op_ret, int32_t op_errno, fd_t *fd, dict_t *xdata)
 {
-        struct nfs_fop_local    *nfl = NULL;
-        fop_open_cbk_t          progcbk = NULL;
+    struct nfs_fop_local *nfl = NULL;
+    fop_open_cbk_t progcbk = NULL;
 
-        nfl_to_prog_data (nfl, progcbk, frame);
-        if (progcbk)
-                progcbk (frame, cookie, this, op_ret, op_errno, fd, xdata);
-        nfs_stack_destroy (nfl, frame);
+    nfl_to_prog_data(nfl, progcbk, frame);
+    if (progcbk)
+        progcbk(frame, cookie, this, op_ret, op_errno, fd, xdata);
+    nfs_stack_destroy(nfl, frame);
 
-        return 0;
+    return 0;
 }
 
 int
-nfs_fop_open (xlator_t *nfsx, xlator_t *xl, nfs_user_t *nfu, loc_t *loc,
-              int32_t flags, fd_t *fd, fop_open_cbk_t cbk,
-              void *local)
+nfs_fop_open(xlator_t *nfsx, xlator_t *xl, nfs_user_t *nfu, loc_t *loc,
+             int32_t flags, fd_t *fd, fop_open_cbk_t cbk, void *local)
 {
-        call_frame_t            *frame = NULL;
-        int                     ret = -EFAULT;
-        struct nfs_fop_local    *nfl = NULL;
+    call_frame_t *frame = NULL;
+    int ret = -EFAULT;
+    struct nfs_fop_local *nfl = NULL;
 
-        if ((!nfsx) || (!xl) || (!loc) || (!fd) || (!nfu))
-                return ret;
+    if ((!nfsx) || (!xl) || (!loc) || (!fd) || (!nfu))
+        return ret;
 
-        gf_log (GF_NFS, GF_LOG_TRACE, "Open: %s", loc->path);
-        nfs_fop_handle_frame_create (frame, nfsx, nfu, ret, err);
-        nfs_fop_handle_local_init (frame, nfsx, nfl, cbk, local, ret, err);
+    gf_msg_trace(GF_NFS, 0, "Open: %s", loc->path);
+    nfs_fop_handle_frame_create(frame, nfsx, nfu, ret, err);
+    nfs_fop_handle_local_init(frame, nfsx, nfl, cbk, local, ret, err);
 
-        STACK_WIND_COOKIE (frame, nfs_fop_open_cbk, xl, xl, xl->fops->open,
-                           loc, flags, fd, NULL);
-        ret = 0;
+    STACK_WIND_COOKIE(frame, nfs_fop_open_cbk, xl, xl, xl->fops->open, loc,
+                      flags, fd, NULL);
+    ret = 0;
 err:
-        if (ret < 0) {
-                if (frame)
-                        nfs_stack_destroy (nfl, frame);
-        }
+    if (ret < 0) {
+        if (frame)
+            nfs_stack_destroy(nfl, frame);
+    }
 
-        return ret;
+    return ret;
 }
 
-
 int32_t
-nfs_fop_writev_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                    int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
-                    struct iatt *postbuf, dict_t *xdata)
+nfs_fop_writev_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                   int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
+                   struct iatt *postbuf, dict_t *xdata)
 {
-        struct nfs_fop_local    *nfl = NULL;
-        fop_writev_cbk_t        progcbk = NULL;
+    struct nfs_fop_local *nfl = NULL;
+    fop_writev_cbk_t progcbk = NULL;
 
-        nfl_to_prog_data (nfl, progcbk, frame);
-        nfs_fop_restore_root_ino (nfl, op_ret, prebuf, postbuf, NULL, NULL);
-        if (progcbk)
-                progcbk (frame, cookie, this, op_ret, op_errno, prebuf,
-                         postbuf, xdata);
+    nfl_to_prog_data(nfl, progcbk, frame);
+    nfs_fop_restore_root_ino(nfl, op_ret, prebuf, postbuf, NULL, NULL);
+    if (progcbk)
+        progcbk(frame, cookie, this, op_ret, op_errno, prebuf, postbuf, xdata);
 
-        nfs_stack_destroy (nfl, frame);
+    nfs_stack_destroy(nfl, frame);
 
-        return 0;
+    return 0;
 }
 
-
 int
-nfs_fop_write (xlator_t *nfsx, xlator_t *xl, nfs_user_t *nfu, fd_t *fd,
-               struct iobref *srciobref, struct iovec *vector, int32_t count,
-               off_t offset, fop_writev_cbk_t cbk, void *local)
+nfs_fop_write(xlator_t *nfsx, xlator_t *xl, nfs_user_t *nfu, fd_t *fd,
+              struct iobref *srciobref, struct iovec *vector, int32_t count,
+              off_t offset, fop_writev_cbk_t cbk, void *local)
 {
-        call_frame_t            *frame = NULL;
-        int                     ret = -EFAULT;
-        struct nfs_fop_local    *nfl = NULL;
+    call_frame_t *frame = NULL;
+    int ret = -EFAULT;
+    struct nfs_fop_local *nfl = NULL;
+    int flags = 0;
+    nfs3_call_state_t *cs = local;
 
-        if ((!nfsx) || (!xl) || (!fd) || (!vector) || (!nfu) || (!srciobref))
-                return ret;
-
-        nfs_fop_handle_frame_create (frame, nfsx, nfu, ret, err);
-        nfs_fop_handle_local_init (frame, nfsx, nfl, cbk, local, ret, err);
-        nfs_fop_save_root_fd_ino (nfl, fd);
-/*
-        nfl->iobref = iobref_new ();
-        if (!nfl->iobref) {
-                gf_log (GF_NFS, GF_LOG_ERROR, "iobref creation failed");
-                ret = -ENOMEM;
-                goto err;
-        }
+    if ((!nfsx) || (!xl) || (!fd) || (!vector) || (!nfu) || (!srciobref))
+        return ret;
 
-        iobref_add (nfl->iobref, srciob);
-*/
-        STACK_WIND_COOKIE (frame, nfs_fop_writev_cbk, xl, xl,xl->fops->writev,
-                           fd, vector, count, offset, fd->flags, srciobref, NULL);
-        ret = 0;
+    nfs_fop_handle_frame_create(frame, nfsx, nfu, ret, err);
+    nfs_fop_handle_local_init(frame, nfsx, nfl, cbk, local, ret, err);
+    nfs_fop_save_root_fd_ino(nfl, fd);
+    /*
+            nfl->iobref = iobref_new ();
+            if (!nfl->iobref) {
+                    gf_log (GF_NFS, GF_LOG_ERROR, "iobref creation failed");
+                    ret = -ENOMEM;
+                    goto err;
+            }
+
+            iobref_add (nfl->iobref, srciob);
+    */
+
+    switch (cs->writetype) {
+        case UNSTABLE:
+            break;
+        case DATA_SYNC:
+            flags |= O_DSYNC;
+            break;
+        case FILE_SYNC:
+            flags |= O_SYNC;
+            break;
+    }
+
+    STACK_WIND_COOKIE(frame, nfs_fop_writev_cbk, xl, xl, xl->fops->writev, fd,
+                      vector, count, offset, flags, srciobref, NULL);
+    ret = 0;
 err:
-        if (ret < 0) {
-                if (frame)
-                        nfs_stack_destroy (nfl, frame);
-        }
+    if (ret < 0) {
+        if (frame)
+            nfs_stack_destroy(nfl, frame);
+    }
 
-        return ret;
+    return ret;
 }
 
-
 int32_t
-nfs_fop_fsync_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                   int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
-                   struct iatt *postbuf, dict_t *xdata)
+nfs_fop_fsync_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                  int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
+                  struct iatt *postbuf, dict_t *xdata)
 {
-        struct nfs_fop_local    *nfl = NULL;
-        fop_fsync_cbk_t         progcbk = NULL;
+    struct nfs_fop_local *nfl = NULL;
+    fop_fsync_cbk_t progcbk = NULL;
 
-        nfl_to_prog_data (nfl, progcbk, frame);
-        nfs_fop_restore_root_ino (nfl, op_ret, prebuf, postbuf, NULL, NULL);
-        if (progcbk)
-                progcbk (frame, cookie, this, op_ret, op_errno, prebuf,
-                         postbuf, xdata);
+    nfl_to_prog_data(nfl, progcbk, frame);
+    nfs_fop_restore_root_ino(nfl, op_ret, prebuf, postbuf, NULL, NULL);
+    if (progcbk)
+        progcbk(frame, cookie, this, op_ret, op_errno, prebuf, postbuf, xdata);
 
-        nfs_stack_destroy (nfl, frame);
-        return 0;
+    nfs_stack_destroy(nfl, frame);
+    return 0;
 }
 
-
-
 int
-nfs_fop_fsync (xlator_t *nfsx, xlator_t *xl, nfs_user_t *nfu, fd_t *fd,
-               int32_t datasync, fop_fsync_cbk_t cbk, void *local)
+nfs_fop_fsync(xlator_t *nfsx, xlator_t *xl, nfs_user_t *nfu, fd_t *fd,
+              int32_t datasync, fop_fsync_cbk_t cbk, void *local)
 {
-        call_frame_t            *frame = NULL;
-        int                     ret = -EFAULT;
-        struct nfs_fop_local    *nfl = NULL;
+    call_frame_t *frame = NULL;
+    int ret = -EFAULT;
+    struct nfs_fop_local *nfl = NULL;
 
-        if ((!nfsx) || (!xl) || (!fd))
-                return ret;
+    if ((!nfsx) || (!xl) || (!fd))
+        return ret;
 
-        nfs_fop_handle_frame_create (frame, nfsx, nfu, ret, err);
-        nfs_fop_handle_local_init (frame, nfsx, nfl, cbk, local, ret, err);
-        nfs_fop_save_root_fd_ino (nfl, fd);
+    nfs_fop_handle_frame_create(frame, nfsx, nfu, ret, err);
+    nfs_fop_handle_local_init(frame, nfsx, nfl, cbk, local, ret, err);
+    nfs_fop_save_root_fd_ino(nfl, fd);
 
-        STACK_WIND_COOKIE (frame, nfs_fop_fsync_cbk, xl, xl,
-                           xl->fops->fsync, fd, datasync, NULL);
-        ret = 0;
+    STACK_WIND_COOKIE(frame, nfs_fop_fsync_cbk, xl, xl, xl->fops->fsync, fd,
+                      datasync, NULL);
+    ret = 0;
 err:
-        if (ret < 0) {
-                if (frame)
-                        nfs_stack_destroy (nfl, frame);
-        }
+    if (ret < 0) {
+        if (frame)
+            nfs_stack_destroy(nfl, frame);
+    }
 
-        return ret;
+    return ret;
 }
 
-
 int32_t
-nfs_fop_readv_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                   int32_t op_ret, int32_t op_errno, struct iovec *vector,
-                   int32_t count, struct iatt *stbuf, struct iobref *iobref,
-                   dict_t *xdata)
+nfs_fop_readv_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                  int32_t op_ret, int32_t op_errno, struct iovec *vector,
+                  int32_t count, struct iatt *stbuf, struct iobref *iobref,
+                  dict_t *xdata)
 {
-        struct nfs_fop_local    *nfl = NULL;
-        fop_readv_cbk_t         progcbk = NULL;
+    struct nfs_fop_local *nfl = NULL;
+    fop_readv_cbk_t progcbk = NULL;
 
-        nfl_to_prog_data (nfl, progcbk, frame);
-        nfs_fop_restore_root_ino (nfl, op_ret, stbuf, NULL, NULL, NULL);
-        if (progcbk)
-                progcbk (frame, cookie, this, op_ret, op_errno, vector, count,
-                         stbuf, iobref, xdata);
+    nfl_to_prog_data(nfl, progcbk, frame);
+    nfs_fop_restore_root_ino(nfl, op_ret, stbuf, NULL, NULL, NULL);
+    if (progcbk)
+        progcbk(frame, cookie, this, op_ret, op_errno, vector, count, stbuf,
+                iobref, xdata);
 
-        nfs_stack_destroy (nfl, frame);
-        return 0;
+    nfs_stack_destroy(nfl, frame);
+    return 0;
 }
 
-
 int
-nfs_fop_read (xlator_t *nfsx, xlator_t *xl, nfs_user_t *nfu, fd_t *fd,
-              size_t size, off_t offset, fop_readv_cbk_t cbk, void *local)
+nfs_fop_read(xlator_t *nfsx, xlator_t *xl, nfs_user_t *nfu, fd_t *fd,
+             size_t size, off_t offset, fop_readv_cbk_t cbk, void *local)
 {
-        call_frame_t            *frame = NULL;
-        int                     ret = -EFAULT;
-        struct nfs_fop_local    *nfl = NULL;
+    call_frame_t *frame = NULL;
+    int ret = -EFAULT;
+    struct nfs_fop_local *nfl = NULL;
 
-        if ((!xl) || (!fd) || (!nfu))
-                return ret;
+    if ((!xl) || (!fd) || (!nfu))
+        return ret;
 
-        nfs_fop_handle_frame_create (frame, nfsx, nfu, ret, err);
-        nfs_fop_handle_local_init (frame, nfsx, nfl, cbk, local, ret, err);
-        nfs_fop_save_root_fd_ino (nfl, fd);
+    nfs_fop_handle_frame_create(frame, nfsx, nfu, ret, err);
+    nfs_fop_handle_local_init(frame, nfsx, nfl, cbk, local, ret, err);
+    nfs_fop_save_root_fd_ino(nfl, fd);
 
-        STACK_WIND_COOKIE (frame, nfs_fop_readv_cbk, xl, xl, xl->fops->readv,
-                           fd, size, offset, 0, NULL);
-        ret = 0;
+    STACK_WIND_COOKIE(frame, nfs_fop_readv_cbk, xl, xl, xl->fops->readv, fd,
+                      size, offset, 0, NULL);
+    ret = 0;
 err:
-        if (ret < 0) {
-                if (frame)
-                        nfs_stack_destroy (nfl, frame);
-        }
+    if (ret < 0) {
+        if (frame)
+            nfs_stack_destroy(nfl, frame);
+    }
 
-        return ret;
+    return ret;
 }
 
 int32_t
-nfs_fop_lk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                int32_t op_ret, int32_t op_errno, struct gf_flock *flock,
-                dict_t *xdata)
+nfs_fop_lk_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+               int32_t op_ret, int32_t op_errno, struct gf_flock *flock,
+               dict_t *xdata)
 {
-        struct nfs_fop_local    *nfl = NULL;
-        fop_lk_cbk_t         progcbk = NULL;
+    struct nfs_fop_local *nfl = NULL;
+    fop_lk_cbk_t progcbk = NULL;
 
-        nfl_to_prog_data (nfl, progcbk, frame);
+    nfl_to_prog_data(nfl, progcbk, frame);
 
-        if (!op_ret)
-                fd_lk_insert_and_merge (nfl->fd, nfl->cmd, &nfl->flock);
+    if (!op_ret)
+        fd_lk_insert_and_merge(nfl->fd, nfl->cmd, &nfl->flock);
 
-        fd_unref (nfl->fd);
+    fd_unref(nfl->fd);
 
-        if (progcbk)
-                progcbk (frame, cookie, this, op_ret, op_errno, flock, xdata);
+    if (progcbk)
+        progcbk(frame, cookie, this, op_ret, op_errno, flock, xdata);
 
-        nfs_stack_destroy (nfl, frame);
-        return 0;
+    nfs_stack_destroy(nfl, frame);
+    return 0;
 }
 
-
 int
-nfs_fop_lk (xlator_t *nfsx, xlator_t *xl, nfs_user_t *nfu, fd_t *fd,
-            int cmd, struct gf_flock *flock, fop_lk_cbk_t cbk, void *local)
+nfs_fop_lk(xlator_t *nfsx, xlator_t *xl, nfs_user_t *nfu, fd_t *fd, int cmd,
+           struct gf_flock *flock, fop_lk_cbk_t cbk, void *local)
 {
-        call_frame_t            *frame = NULL;
-        int                     ret = -EFAULT;
-        struct nfs_fop_local    *nfl = NULL;
+    call_frame_t *frame = NULL;
+    int ret = -EFAULT;
+    struct nfs_fop_local *nfl = NULL;
 
-        if ((!xl) || (!fd) || (!nfu))
-                return ret;
+    if ((!xl) || (!fd) || (!nfu))
+        return ret;
 
-        nfs_fop_handle_frame_create (frame, nfsx, nfu, ret, err);
-        nfs_fop_handle_local_init (frame, nfsx, nfl, cbk, local, ret, err);
+    nfs_fop_handle_frame_create(frame, nfsx, nfu, ret, err);
+    nfs_fop_handle_local_init(frame, nfsx, nfl, cbk, local, ret, err);
 
-        nfl->cmd   = cmd;
-        nfl->fd    = fd_ref (fd);
-        nfl->flock = *flock;
+    nfl->cmd = cmd;
+    nfl->fd = fd_ref(fd);
+    nfl->flock = *flock;
 
-        STACK_WIND_COOKIE (frame, nfs_fop_lk_cbk, xl, xl, xl->fops->lk,
-                           fd, cmd, flock, NULL);
-        ret = 0;
+    STACK_WIND_COOKIE(frame, nfs_fop_lk_cbk, xl, xl, xl->fops->lk, fd, cmd,
+                      flock, NULL);
+    ret = 0;
 err:
-        if (ret < 0) {
-                if (frame)
-                        nfs_stack_destroy (nfl, frame);
-        }
+    if (ret < 0) {
+        if (frame)
+            nfs_stack_destroy(nfl, frame);
+    }
 
-        return ret;
+    return ret;
 }
 
 int32_t
-nfs_fop_getxattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                int32_t op_ret, int32_t op_errno, dict_t *dict,
-                dict_t *xdata)
+nfs_fop_getxattr_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                     int32_t op_ret, int32_t op_errno, dict_t *dict,
+                     dict_t *xdata)
 {
-        struct nfs_fop_local    *nfl = NULL;
-        fop_getxattr_cbk_t         progcbk = NULL;
+    struct nfs_fop_local *nfl = NULL;
+    fop_getxattr_cbk_t progcbk = NULL;
 
-        nfl_to_prog_data (nfl, progcbk, frame);
+    nfl_to_prog_data(nfl, progcbk, frame);
 
-        if (progcbk)
-                progcbk (frame, cookie, this, op_ret, op_errno, dict, xdata);
+    if (progcbk)
+        progcbk(frame, cookie, this, op_ret, op_errno, dict, xdata);
 
-        nfs_stack_destroy (nfl, frame);
-        return 0;
+    nfs_stack_destroy(nfl, frame);
+    return 0;
 }
 
-
 int
-nfs_fop_getxattr (xlator_t *nfsx, xlator_t *xl, nfs_user_t *nfu, loc_t *loc,
-                  char *name, dict_t *xdata, fop_getxattr_cbk_t cbk, void *local)
+nfs_fop_getxattr(xlator_t *nfsx, xlator_t *xl, nfs_user_t *nfu, loc_t *loc,
+                 char *name, dict_t *xdata, fop_getxattr_cbk_t cbk, void *local)
 {
-        call_frame_t            *frame = NULL;
-        int                     ret = -EFAULT;
-        struct nfs_fop_local    *nfl = NULL;
+    call_frame_t *frame = NULL;
+    int ret = -EFAULT;
+    struct nfs_fop_local *nfl = NULL;
 
-        if ((!xl) || (!loc) || (!nfu))
-                return ret;
+    if ((!xl) || (!loc) || (!nfu))
+        return ret;
 
-        nfs_fop_handle_frame_create (frame, nfsx, nfu, ret, err);
-        nfs_fop_handle_local_init (frame, nfsx, nfl, cbk, local, ret, err);
+    nfs_fop_handle_frame_create(frame, nfsx, nfu, ret, err);
+    nfs_fop_handle_local_init(frame, nfsx, nfl, cbk, local, ret, err);
 
-        STACK_WIND_COOKIE (frame, nfs_fop_getxattr_cbk, xl, xl, xl->fops->getxattr,
-                           loc, name, NULL);
-        ret = 0;
+    STACK_WIND_COOKIE(frame, nfs_fop_getxattr_cbk, xl, xl, xl->fops->getxattr,
+                      loc, name, NULL);
+    ret = 0;
 err:
-        if (ret < 0) {
-                if (frame)
-                        nfs_stack_destroy (nfl, frame);
-        }
+    if (ret < 0) {
+        if (frame)
+            nfs_stack_destroy(nfl, frame);
+    }
 
-        return ret;
+    return ret;
 }
 
-
 int32_t
-nfs_fop_setxattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                int32_t op_ret, int32_t op_errno, dict_t *xdata)
+nfs_fop_setxattr_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                     int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
-        struct nfs_fop_local    *nfl = NULL;
-        fop_setxattr_cbk_t         progcbk = NULL;
+    struct nfs_fop_local *nfl = NULL;
+    fop_setxattr_cbk_t progcbk = NULL;
 
-        nfl_to_prog_data (nfl, progcbk, frame);
+    nfl_to_prog_data(nfl, progcbk, frame);
 
-        if (progcbk)
-                progcbk (frame, cookie, this, op_ret, op_errno, xdata);
+    if (progcbk)
+        progcbk(frame, cookie, this, op_ret, op_errno, xdata);
 
-        nfs_stack_destroy (nfl, frame);
-        return 0;
+    nfs_stack_destroy(nfl, frame);
+    return 0;
 }
 
-
 int
-nfs_fop_setxattr (xlator_t *nfsx, xlator_t *xl, nfs_user_t *nfu,
-                  loc_t *loc, dict_t *dict, int32_t flags, dict_t *xdata,
-                  fop_setxattr_cbk_t cbk, void *local)
+nfs_fop_setxattr(xlator_t *nfsx, xlator_t *xl, nfs_user_t *nfu, loc_t *loc,
+                 dict_t *dict, int32_t flags, dict_t *xdata,
+                 fop_setxattr_cbk_t cbk, void *local)
 {
-        call_frame_t            *frame = NULL;
-        int                     ret = -EFAULT;
-        struct nfs_fop_local    *nfl = NULL;
+    call_frame_t *frame = NULL;
+    int ret = -EFAULT;
+    struct nfs_fop_local *nfl = NULL;
 
-        if ((!xl) || (!loc) || (!nfu))
-                return ret;
+    if ((!xl) || (!loc) || (!nfu))
+        return ret;
 
-        nfs_fop_handle_frame_create (frame, nfsx, nfu, ret, err);
-        nfs_fop_handle_local_init (frame, nfsx, nfl, cbk, local, ret, err);
+    nfs_fop_handle_frame_create(frame, nfsx, nfu, ret, err);
+    nfs_fop_handle_local_init(frame, nfsx, nfl, cbk, local, ret, err);
 
-        STACK_WIND_COOKIE (frame, nfs_fop_setxattr_cbk, xl, xl, xl->fops->setxattr,
-                           loc, dict, flags, xdata);
-        ret = 0;
+    STACK_WIND_COOKIE(frame, nfs_fop_setxattr_cbk, xl, xl, xl->fops->setxattr,
+                      loc, dict, flags, xdata);
+    ret = 0;
 err:
-        if (ret < 0) {
-                if (frame)
-                        nfs_stack_destroy (nfl, frame);
-        }
+    if (ret < 0) {
+        if (frame)
+            nfs_stack_destroy(nfl, frame);
+    }
 
-        return ret;
+    return ret;
 }
 
-
 int32_t
-nfs_fop_truncate_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                      int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
-                      struct iatt *postbuf, dict_t *xdata)
+nfs_fop_truncate_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                     int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
+                     struct iatt *postbuf, dict_t *xdata)
 {
-        struct nfs_fop_local    *nfl = NULL;
-        fop_truncate_cbk_t      progcbk = NULL;
+    struct nfs_fop_local *nfl = NULL;
+    fop_truncate_cbk_t progcbk = NULL;
 
-        nfl_to_prog_data (nfl, progcbk, frame);
-        nfs_fop_restore_root_ino (nfl, op_ret, prebuf, postbuf, NULL, NULL);
-        if (progcbk)
-                progcbk (frame, cookie, this, op_ret, op_errno, prebuf,
-                         postbuf, xdata);
+    nfl_to_prog_data(nfl, progcbk, frame);
+    nfs_fop_restore_root_ino(nfl, op_ret, prebuf, postbuf, NULL, NULL);
+    if (progcbk)
+        progcbk(frame, cookie, this, op_ret, op_errno, prebuf, postbuf, xdata);
 
-        nfs_stack_destroy (nfl, frame);
-        return 0;
+    nfs_stack_destroy(nfl, frame);
+    return 0;
 }
 
-
 int
-nfs_fop_truncate (xlator_t *nfsx, xlator_t *xl, nfs_user_t *nfu, loc_t *loc,
-                  off_t offset, fop_truncate_cbk_t cbk, void *local)
+nfs_fop_truncate(xlator_t *nfsx, xlator_t *xl, nfs_user_t *nfu, loc_t *loc,
+                 off_t offset, fop_truncate_cbk_t cbk, void *local)
 {
-        call_frame_t            *frame = NULL;
-        int                     ret = -EFAULT;
-        struct nfs_fop_local    *nfl = NULL;
+    call_frame_t *frame = NULL;
+    int ret = -EFAULT;
+    struct nfs_fop_local *nfl = NULL;
 
-        if ((!nfsx) || (!xl) || (!loc) || (!nfu))
-                return ret;
+    if ((!nfsx) || (!xl) || (!loc) || (!nfu))
+        return ret;
 
-        nfs_fop_handle_frame_create (frame, nfsx, nfu, ret, err);
-        nfs_fop_handle_local_init (frame, nfsx, nfl, cbk, local, ret, err);
-        nfs_fop_save_root_ino (nfl, loc);
+    nfs_fop_handle_frame_create(frame, nfsx, nfu, ret, err);
+    nfs_fop_handle_local_init(frame, nfsx, nfl, cbk, local, ret, err);
+    nfs_fop_save_root_ino(nfl, loc);
 
-        STACK_WIND_COOKIE  (frame, nfs_fop_truncate_cbk, xl, xl,
-                            xl->fops->truncate, loc, offset, NULL);
+    STACK_WIND_COOKIE(frame, nfs_fop_truncate_cbk, xl, xl, xl->fops->truncate,
+                      loc, offset, NULL);
 
-        ret = 0;
+    ret = 0;
 err:
-        if (ret < 0) {
-                if (frame)
-                        nfs_stack_destroy (nfl, frame);
-        }
+    if (ret < 0) {
+        if (frame)
+            nfs_stack_destroy(nfl, frame);
+    }
 
-        return ret;
+    return ret;
 }
diff --git a/xlators/nfs/server/src/nfs-fops.h b/xlators/nfs/server/src/nfs-fops.h
index 44e99c66b06..005cb788a45 100644
--- a/xlators/nfs/server/src/nfs-fops.h
+++ b/xlators/nfs/server/src/nfs-fops.h
@@ -11,18 +11,13 @@
 #ifndef _NFS_FOPS_H_
 #define _NFS_FOPS_H_
 
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
-
-#include "dict.h"
-#include "xlator.h"
-#include "iobuf.h"
-#include "call-stub.h"
-#include "stack.h"
+#include <glusterfs/dict.h>
+#include <glusterfs/xlator.h>
+#include <glusterfs/iobuf.h>
+#include <glusterfs/call-stub.h>
 #include "nfs.h"
 #include "nfs-common.h"
+#include "nfs-messages.h"
 #include <semaphore.h>
 
 /* This structure used to communicate state between a fop and its callback.
@@ -43,206 +38,205 @@
  * That is initiated in nfs_init_subvolumes in nfs.c.
  */
 struct nfs_fop_local {
-        /* The local sent along by the user of the fop. */
-        void            *proglocal;
-
-        /* The address of the callback supplied by the user. After our
-         * callback is executed this one is called.
-         * The exact cast destination of this pointer will depend on the
-         * fop that is being called.
-         */
-        void            *progcbk;
-
-        /* Used only for write requests. */
-        struct iobref   *iobref;
-
-        inode_t         *parent;
-        inode_t         *newparent;
-        inode_t         *inode;
-
-        /* Set to 1 by nfs-inodes layer, which uses this to decide whether to
-         * link the newly allocated inode into the itable, in case the fop was
-         * successful.
-         */
-        int             newinode;
-
-        /* Used by nfs-fops layer in order to determine whether to funge the
-         * ino in a dir's stbuf. This funging of root ino is needed to ensure
-         * that the root ino remains 1 even when the NFS server has been
-         * restarted. Note that in distribute, a fresh lookup and a revalidate
-         * on the root inode returns two different inode numbers and this we
-         * need to handle by ourself.
-         */
-        int             rootinode;
-
-        /* This member is used to determine whether the new parent of a file
-         * being renamed is the root directory. If yes, the ino is funged.
-         */
-        int             newrootinode;
-        int             newrootparentinode;
-
-        /* Determines whether to funge the ino in the post and pre parent
-         * stbufs for a file/dir where the parent directory could be the root
-         * dir. Needed here because of the same reason as above.
-         */
-        int             rootparentinode;
-
-        char            path[NFS_NAME_MAX + 1];
-        char            newpath[NFS_NAME_MAX + 1];
-        xlator_t        *nfsx;
-        dict_t          *dictgfid;
-
-        fd_t            *fd;
-        int             cmd;
-        struct gf_flock flock;
+    /* The local sent along by the user of the fop. */
+    void *proglocal;
+
+    /* The address of the callback supplied by the user. After our
+     * callback is executed this one is called.
+     * The exact cast destination of this pointer will depend on the
+     * fop that is being called.
+     */
+    void *progcbk;
+
+    /* Used only for write requests. */
+    struct iobref *iobref;
+
+    inode_t *parent;
+    inode_t *newparent;
+    inode_t *inode;
+
+    /* Set to 1 by nfs-inodes layer, which uses this to decide whether to
+     * link the newly allocated inode into the itable, in case the fop was
+     * successful.
+     */
+    int newinode;
+
+    /* Used by nfs-fops layer in order to determine whether to funge the
+     * ino in a dir's stbuf. This funging of root ino is needed to ensure
+     * that the root ino remains 1 even when the NFS server has been
+     * restarted. Note that in distribute, a fresh lookup and a revalidate
+     * on the root inode returns two different inode numbers and this we
+     * need to handle by ourself.
+     */
+    int rootinode;
+
+    /* This member is used to determine whether the new parent of a file
+     * being renamed is the root directory. If yes, the ino is funged.
+     */
+    int newrootinode;
+    int newrootparentinode;
+
+    /* Determines whether to funge the ino in the post and pre parent
+     * stbufs for a file/dir where the parent directory could be the root
+     * dir. Needed here because of the same reason as above.
+     */
+    int rootparentinode;
+
+    char path[NFS_NAME_MAX + 1];
+    char newpath[NFS_NAME_MAX + 1];
+    xlator_t *nfsx;
+    dict_t *dictgfid;
+
+    fd_t *fd;
+    int cmd;
+    struct gf_flock flock;
 };
 
 extern struct nfs_fop_local *
-nfs_fop_local_init (xlator_t *xl);
+nfs_fop_local_init(xlator_t *xl);
 
 extern void
-nfs_fop_local_wipe (xlator_t *xl, struct nfs_fop_local *l);
-
-#define nfs_state(nfsxl)        (nfsxl)->private
-#define nfs_fop_mempool(nfxl)   (((struct nfs_state *)nfs_state(nfxl))->foppool)
-
-#define prog_data_to_nfl(nf,nflocal, fram, pcbk, plocal)                \
-        do {                                                            \
-                nflocal = nfs_fop_local_init (nf);                      \
-                if (nflocal) {                                          \
-                        nflocal->proglocal = plocal;                    \
-                        nflocal->progcbk = *VOID(&pcbk);                        \
-                        nflocal->nfsx = nf;                             \
-                        if (fram)                                       \
-                                ((call_frame_t *)fram)->local = nflocal;\
-                }                                                       \
-        } while (0)                                                     \
-
-
-
-#define nfl_to_prog_data(nflocal, pcbk, fram)                           \
-        do {                                                            \
-                nflocal = fram->local;                                  \
-                fram->local = nflocal->proglocal;                       \
-                pcbk = nflocal->progcbk;                                \
-        } while (0)                                                     \
-
-#define nfs_fop_handle_local_init(fram,nfx, nfloc, cbck,prgloc,retval,lab)  \
-        do {                                                                \
-                prog_data_to_nfl (nfx, nfloc, fram, cbck, prgloc);      \
-                if (!nfloc) {                                               \
-                        gf_log (GF_NFS,GF_LOG_ERROR,"Failed to init local");\
-                        retval = -ENOMEM;                                   \
-                        goto lab;                                           \
-                }                                                           \
-        } while (0)                                                         \
-
-extern int
-nfs_fop_fstat (xlator_t *nfsx, xlator_t *xl, nfs_user_t *nfu, fd_t *fd,
-               fop_stat_cbk_t cbk, void *local);
+nfs_fop_local_wipe(xlator_t *xl, struct nfs_fop_local *l);
+
+#define nfs_state(nfsxl) (nfsxl)->private
+#define nfs_fop_mempool(nfxl) (((struct nfs_state *)nfs_state(nfxl))->foppool)
+
+#define prog_data_to_nfl(nf, nflocal, fram, pcbk, plocal)                      \
+    do {                                                                       \
+        nflocal = nfs_fop_local_init(nf);                                      \
+        if (nflocal) {                                                         \
+            nflocal->proglocal = plocal;                                       \
+            nflocal->progcbk = *VOID(&pcbk);                                   \
+            nflocal->nfsx = nf;                                                \
+            if (fram)                                                          \
+                ((call_frame_t *)fram)->local = nflocal;                       \
+        }                                                                      \
+    } while (0)
+
+#define nfl_to_prog_data(nflocal, pcbk, fram)                                  \
+    do {                                                                       \
+        nflocal = fram->local;                                                 \
+        fram->local = nflocal->proglocal;                                      \
+        pcbk = nflocal->progcbk;                                               \
+    } while (0)
+
+#define nfs_fop_handle_local_init(fram, nfx, nfloc, cbck, prgloc, retval, lab) \
+    do {                                                                       \
+        prog_data_to_nfl(nfx, nfloc, fram, cbck, prgloc);                      \
+        if (!nfloc) {                                                          \
+            gf_msg(GF_NFS, GF_LOG_ERROR, ENOMEM, NFS_MSG_NO_MEMORY,            \
+                   "Failed to init local");                                    \
+            retval = -ENOMEM;                                                  \
+            goto lab;                                                          \
+        }                                                                      \
+    } while (0)
+
+extern int
+nfs_fop_fstat(xlator_t *nfsx, xlator_t *xl, nfs_user_t *nfu, fd_t *fd,
+              fop_stat_cbk_t cbk, void *local);
 
 extern int
-nfs_fop_readdirp (xlator_t *nfsx, xlator_t *xl, nfs_user_t *nfu, fd_t *dirfd,
-                  size_t bufsize, off_t offset, fop_readdir_cbk_t cbk,
-                  void *local);
+nfs_fop_readdirp(xlator_t *nfsx, xlator_t *xl, nfs_user_t *nfu, fd_t *dirfd,
+                 size_t bufsize, off_t offset, fop_readdir_cbk_t cbk,
+                 void *local);
 extern int
-nfs_fop_lookup (xlator_t *nfsx, xlator_t *xl, nfs_user_t *nfu, loc_t *loc,
-                fop_lookup_cbk_t cbk, void *local);
+nfs_fop_lookup(xlator_t *nfsx, xlator_t *xl, nfs_user_t *nfu, loc_t *loc,
+               fop_lookup_cbk_t cbk, void *local);
 
 extern int
-nfs_fop_create (xlator_t *nfsx, xlator_t *xl, nfs_user_t *nfu, loc_t *pathloc,
-                int flags, mode_t mode, fd_t *fd, fop_create_cbk_t cbk,
-                void *local);
+nfs_fop_create(xlator_t *nfsx, xlator_t *xl, nfs_user_t *nfu, loc_t *pathloc,
+               int flags, mode_t mode, fd_t *fd, fop_create_cbk_t cbk,
+               void *local);
 extern int
-nfs_fop_flush (xlator_t *nfsx, xlator_t *xl, nfs_user_t *nfu, fd_t *fd,
-               fop_flush_cbk_t cbk, void *local);
+nfs_fop_flush(xlator_t *nfsx, xlator_t *xl, nfs_user_t *nfu, fd_t *fd,
+              fop_flush_cbk_t cbk, void *local);
 
 extern int
-nfs_fop_mkdir (xlator_t *nfsx, xlator_t *xl, nfs_user_t *nfu, loc_t *pathloc,
-               mode_t mode, fop_mkdir_cbk_t cbk, void *local);
+nfs_fop_mkdir(xlator_t *nfsx, xlator_t *xl, nfs_user_t *nfu, loc_t *pathloc,
+              mode_t mode, fop_mkdir_cbk_t cbk, void *local);
 
 extern int
-nfs_fop_truncate (xlator_t *nfsx, xlator_t *xl, nfs_user_t *nfu, loc_t *loc,
-                  off_t offset, fop_truncate_cbk_t cbk, void *local);
+nfs_fop_truncate(xlator_t *nfsx, xlator_t *xl, nfs_user_t *nfu, loc_t *loc,
+                 off_t offset, fop_truncate_cbk_t cbk, void *local);
 
 extern int
-nfs_fop_read (xlator_t *nfsx, xlator_t *xl, nfs_user_t *nfu, fd_t *fd,
-              size_t size, off_t offset, fop_readv_cbk_t cbk, void *local);
+nfs_fop_read(xlator_t *nfsx, xlator_t *xl, nfs_user_t *nfu, fd_t *fd,
+             size_t size, off_t offset, fop_readv_cbk_t cbk, void *local);
 
 extern int
-nfs_fop_fsync (xlator_t *nfsx, xlator_t *xl, nfs_user_t *nfu, fd_t *fd,
-               int32_t datasync, fop_fsync_cbk_t cbk, void *local);
+nfs_fop_fsync(xlator_t *nfsx, xlator_t *xl, nfs_user_t *nfu, fd_t *fd,
+              int32_t datasync, fop_fsync_cbk_t cbk, void *local);
 
 extern int
-nfs_fop_write (xlator_t *nfsx, xlator_t *xl, nfs_user_t *nfu, fd_t *fd,
-               struct iobref *srciobref, struct iovec *vector, int32_t count,
-               off_t offset, fop_writev_cbk_t cbk, void *local);
+nfs_fop_write(xlator_t *nfsx, xlator_t *xl, nfs_user_t *nfu, fd_t *fd,
+              struct iobref *srciobref, struct iovec *vector, int32_t count,
+              off_t offset, fop_writev_cbk_t cbk, void *local);
 
 extern int
-nfs_fop_open (xlator_t *nfsx, xlator_t *xl, nfs_user_t *nfu, loc_t *loc,
-              int32_t flags, fd_t *fd, fop_open_cbk_t cbk,
-              void *local);
+nfs_fop_open(xlator_t *nfsx, xlator_t *xl, nfs_user_t *nfu, loc_t *loc,
+             int32_t flags, fd_t *fd, fop_open_cbk_t cbk, void *local);
 
 extern int
-nfs_fop_rename (xlator_t *nfsx, xlator_t *xl, nfs_user_t *nfu, loc_t *oldloc,
-                loc_t *newloc, fop_rename_cbk_t cbk, void *local);
+nfs_fop_rename(xlator_t *nfsx, xlator_t *xl, nfs_user_t *nfu, loc_t *oldloc,
+               loc_t *newloc, fop_rename_cbk_t cbk, void *local);
 
 extern int
-nfs_fop_link (xlator_t *nfsx, xlator_t *xl, nfs_user_t *nfu, loc_t *oldloc,
-              loc_t *newloc, fop_link_cbk_t cbk, void *local);
+nfs_fop_link(xlator_t *nfsx, xlator_t *xl, nfs_user_t *nfu, loc_t *oldloc,
+             loc_t *newloc, fop_link_cbk_t cbk, void *local);
 
 extern int
-nfs_fop_unlink (xlator_t *nfsx, xlator_t *xl, nfs_user_t *nfu, loc_t *pathloc,
-                fop_unlink_cbk_t cbk, void *local);
+nfs_fop_unlink(xlator_t *nfsx, xlator_t *xl, nfs_user_t *nfu, loc_t *pathloc,
+               fop_unlink_cbk_t cbk, void *local);
 
 extern int
-nfs_fop_rmdir (xlator_t *nfsx, xlator_t *xl, nfs_user_t *nfu, loc_t *pathloc,
-               fop_rmdir_cbk_t cbk, void *local);
+nfs_fop_rmdir(xlator_t *nfsx, xlator_t *xl, nfs_user_t *nfu, loc_t *pathloc,
+              fop_rmdir_cbk_t cbk, void *local);
 
 extern int
-nfs_fop_mknod (xlator_t *nfsx, xlator_t *xl, nfs_user_t *nfu, loc_t *pathloc,
-               mode_t mode, dev_t dev, fop_mknod_cbk_t cbk, void *local);
+nfs_fop_mknod(xlator_t *nfsx, xlator_t *xl, nfs_user_t *nfu, loc_t *pathloc,
+              mode_t mode, dev_t dev, fop_mknod_cbk_t cbk, void *local);
 
 extern int
-nfs_fop_readlink (xlator_t *nfsx, xlator_t *xl, nfs_user_t *nfu, loc_t *pathloc,
-                  size_t size, fop_readlink_cbk_t cbk, void *local);
+nfs_fop_readlink(xlator_t *nfsx, xlator_t *xl, nfs_user_t *nfu, loc_t *pathloc,
+                 size_t size, fop_readlink_cbk_t cbk, void *local);
 
 extern int
-nfs_fop_symlink (xlator_t *nfsx, xlator_t *xl, nfs_user_t *nfu, char *target,
-                 loc_t *pathloc, fop_symlink_cbk_t cbk, void *local);
+nfs_fop_symlink(xlator_t *nfsx, xlator_t *xl, nfs_user_t *nfu, char *target,
+                loc_t *pathloc, fop_symlink_cbk_t cbk, void *local);
 
 extern int
-nfs_fop_setattr (xlator_t *nfsx, xlator_t *xl, nfs_user_t *nfu, loc_t *pathloc,
-                 struct iatt *buf, int32_t valid, fop_setattr_cbk_t cbk,
-                 void *local);
+nfs_fop_setattr(xlator_t *nfsx, xlator_t *xl, nfs_user_t *nfu, loc_t *pathloc,
+                struct iatt *buf, int32_t valid, fop_setattr_cbk_t cbk,
+                void *local);
 
 extern int
-nfs_fop_statfs (xlator_t *nfsx, xlator_t *xl, nfs_user_t *nfu, loc_t *pathloc,
-                fop_statfs_cbk_t cbk, void *local);
+nfs_fop_statfs(xlator_t *nfsx, xlator_t *xl, nfs_user_t *nfu, loc_t *pathloc,
+               fop_statfs_cbk_t cbk, void *local);
 
 extern int
-nfs_fop_opendir (xlator_t *nfsx, xlator_t *xl, nfs_user_t *nfu, loc_t *pathloc,
-                 fd_t *dirfd, fop_opendir_cbk_t cbk, void *local);
+nfs_fop_opendir(xlator_t *nfsx, xlator_t *xl, nfs_user_t *nfu, loc_t *pathloc,
+                fd_t *dirfd, fop_opendir_cbk_t cbk, void *local);
 
 extern int
-nfs_fop_stat (xlator_t *nfsx, xlator_t *xl, nfs_user_t *nfu, loc_t *loc,
-              fop_stat_cbk_t cbk, void *local);
+nfs_fop_stat(xlator_t *nfsx, xlator_t *xl, nfs_user_t *nfu, loc_t *loc,
+             fop_stat_cbk_t cbk, void *local);
 
 extern int
-nfs_fop_access (xlator_t *nfsx, xlator_t *xl, nfs_user_t *nfu, loc_t *loc,
-                int32_t accesstest, fop_access_cbk_t cbk, void *local);
+nfs_fop_access(xlator_t *nfsx, xlator_t *xl, nfs_user_t *nfu, loc_t *loc,
+               int32_t accesstest, fop_access_cbk_t cbk, void *local);
 
 extern int
-nfs_fop_lk (xlator_t *nfsx, xlator_t *xl, nfs_user_t *nfu, fd_t *fd,
-            int cmd, struct gf_flock *flock, fop_lk_cbk_t cbk, void *local);
+nfs_fop_lk(xlator_t *nfsx, xlator_t *xl, nfs_user_t *nfu, fd_t *fd, int cmd,
+           struct gf_flock *flock, fop_lk_cbk_t cbk, void *local);
 
 extern int
-nfs_fop_getxattr (xlator_t *nfsx, xlator_t *xl, nfs_user_t *nfu, loc_t *loc,
-                  char *name, dict_t *xdata, fop_getxattr_cbk_t cbk, void *local);
+nfs_fop_getxattr(xlator_t *nfsx, xlator_t *xl, nfs_user_t *nfu, loc_t *loc,
+                 char *name, dict_t *xdata, fop_getxattr_cbk_t cbk,
+                 void *local);
 
 extern int
-nfs_fop_setxattr (xlator_t *nfsx, xlator_t *xl, nfs_user_t *nfu,
-                  loc_t *loc, dict_t *dict, int32_t flags, dict_t *xdata,
-                  fop_setxattr_cbk_t cbk, void *local);
+nfs_fop_setxattr(xlator_t *nfsx, xlator_t *xl, nfs_user_t *nfu, loc_t *loc,
+                 dict_t *dict, int32_t flags, dict_t *xdata,
+                 fop_setxattr_cbk_t cbk, void *local);
 
 #endif
diff --git a/xlators/nfs/server/src/nfs-generics.c b/xlators/nfs/server/src/nfs-generics.c
index cb32b7f1bab..009991877cb 100644
--- a/xlators/nfs/server/src/nfs-generics.c
+++ b/xlators/nfs/server/src/nfs-generics.c
@@ -8,338 +8,305 @@
   cases as published by the Free Software Foundation.
 */
 
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
-
-
 #include "string.h"
 
-#include "inode.h"
 #include "nfs.h"
 #include "nfs-fops.h"
 #include "nfs-inodes.h"
 #include "nfs-generics.h"
-#include "xlator.h"
-
+#include <glusterfs/xlator.h>
 
 int
-nfs_fstat (xlator_t *nfsx, xlator_t *xl, nfs_user_t *nfu, fd_t *fd,
-           fop_stat_cbk_t cbk, void *local)
+nfs_fstat(xlator_t *nfsx, xlator_t *xl, nfs_user_t *nfu, fd_t *fd,
+          fop_stat_cbk_t cbk, void *local)
 {
-        int             ret = -EFAULT;
+    int ret = -EFAULT;
 
-        if ((!nfsx) || (!xl) || (!fd) || (!nfu))
-                return ret;
-
-        ret = nfs_fop_fstat (nfsx, xl, nfu, fd, cbk, local);
+    if ((!nfsx) || (!xl) || (!fd) || (!nfu))
         return ret;
+
+    ret = nfs_fop_fstat(nfsx, xl, nfu, fd, cbk, local);
+    return ret;
 }
 
 int
-nfs_access (xlator_t *nfsx, xlator_t *xl, nfs_user_t *nfu, loc_t *pathloc,
-            int32_t accesstest, fop_access_cbk_t cbk, void *local)
+nfs_access(xlator_t *nfsx, xlator_t *xl, nfs_user_t *nfu, loc_t *pathloc,
+           int32_t accesstest, fop_access_cbk_t cbk, void *local)
 {
-        int             ret = -EFAULT;
+    int ret = -EFAULT;
 
-        if ((!nfsx) || (!xl) || (!pathloc) || (!nfu))
-                return ret;
+    if ((!nfsx) || (!xl) || (!pathloc) || (!nfu))
+        return ret;
 
-        ret = nfs_fop_access (nfsx, xl, nfu, pathloc, accesstest, cbk, local);
+    ret = nfs_fop_access(nfsx, xl, nfu, pathloc, accesstest, cbk, local);
 
-        return ret;
+    return ret;
 }
 
 int
-nfs_stat (xlator_t *nfsx, xlator_t *xl, nfs_user_t *nfu, loc_t *pathloc,
-          fop_stat_cbk_t cbk, void *local)
+nfs_stat(xlator_t *nfsx, xlator_t *xl, nfs_user_t *nfu, loc_t *pathloc,
+         fop_stat_cbk_t cbk, void *local)
 {
-        int             ret = -EFAULT;
+    int ret = -EFAULT;
 
-        if ((!nfsx) || (!xl) || (!pathloc) || (!nfu))
-                return ret;
+    if ((!nfsx) || (!xl) || (!pathloc) || (!nfu))
+        return ret;
 
-        ret = nfs_fop_stat (nfsx, xl, nfu, pathloc, cbk, local);
+    ret = nfs_fop_stat(nfsx, xl, nfu, pathloc, cbk, local);
 
-        return ret;
+    return ret;
 }
 
-
 int
-nfs_readdirp (xlator_t *nfsx, xlator_t *xl, nfs_user_t *nfu, fd_t *dirfd,
-              size_t bufsize, off_t offset, fop_readdir_cbk_t cbk, void *local)
+nfs_readdirp(xlator_t *nfsx, xlator_t *xl, nfs_user_t *nfu, fd_t *dirfd,
+             size_t bufsize, off_t offset, fop_readdir_cbk_t cbk, void *local)
 {
-        if ((!nfsx) || (!xl) || (!dirfd) || (!nfu))
-                return -EFAULT;
+    if ((!nfsx) || (!xl) || (!dirfd) || (!nfu))
+        return -EFAULT;
 
-        return nfs_fop_readdirp (nfsx, xl, nfu, dirfd, bufsize, offset, cbk,
-                                 local);
+    return nfs_fop_readdirp(nfsx, xl, nfu, dirfd, bufsize, offset, cbk, local);
 }
 
-
 int
-nfs_lookup (xlator_t *nfsx, xlator_t *xl, nfs_user_t *nfu, loc_t *pathloc,
-            fop_lookup_cbk_t cbk, void *local)
+nfs_lookup(xlator_t *nfsx, xlator_t *xl, nfs_user_t *nfu, loc_t *pathloc,
+           fop_lookup_cbk_t cbk, void *local)
 {
-        int             ret = -EFAULT;
-
-        if ((!nfsx) || (!xl) || (!pathloc) || (!nfu))
-                return ret;
+    int ret = -EFAULT;
 
-        ret = nfs_fop_lookup (nfsx, xl, nfu, pathloc, cbk, local);
+    if ((!nfsx) || (!xl) || (!pathloc) || (!nfu))
         return ret;
+
+    ret = nfs_fop_lookup(nfsx, xl, nfu, pathloc, cbk, local);
+    return ret;
 }
 
 int
-nfs_create (xlator_t *nfsx, xlator_t *xl, nfs_user_t *nfu, loc_t *pathloc,
-            int flags, mode_t mode, fop_create_cbk_t cbk, void *local)
+nfs_create(xlator_t *nfsx, xlator_t *xl, nfs_user_t *nfu, loc_t *pathloc,
+           int flags, mode_t mode, fop_create_cbk_t cbk, void *local)
 {
-        int                     ret = -EFAULT;
-
-        if ((!nfsx) || (!xl) || (!pathloc) || (!nfu))
-                return ret;
+    int ret = -EFAULT;
 
-        ret = nfs_inode_create (nfsx, xl, nfu, pathloc, flags, mode, cbk,local);
+    if ((!nfsx) || (!xl) || (!pathloc) || (!nfu))
         return ret;
-}
 
+    ret = nfs_inode_create(nfsx, xl, nfu, pathloc, flags, mode, cbk, local);
+    return ret;
+}
 
 int
-nfs_flush (xlator_t *nfsx, xlator_t *xl, nfs_user_t *nfu, fd_t *fd,
-           fop_flush_cbk_t cbk, void *local)
+nfs_flush(xlator_t *nfsx, xlator_t *xl, nfs_user_t *nfu, fd_t *fd,
+          fop_flush_cbk_t cbk, void *local)
 {
-        return nfs_fop_flush (nfsx, xl, nfu, fd, cbk, local);
+    return nfs_fop_flush(nfsx, xl, nfu, fd, cbk, local);
 }
 
-
-
 int
-nfs_mkdir (xlator_t *nfsx, xlator_t *xl, nfs_user_t *nfu, loc_t *pathloc,
-           mode_t mode, fop_mkdir_cbk_t cbk, void *local)
+nfs_mkdir(xlator_t *nfsx, xlator_t *xl, nfs_user_t *nfu, loc_t *pathloc,
+          mode_t mode, fop_mkdir_cbk_t cbk, void *local)
 {
-        int                     ret = -EFAULT;
+    int ret = -EFAULT;
 
-        if ((!nfsx) || (!xl) || (!pathloc) || (!nfu))
-                return ret;
-
-        ret = nfs_inode_mkdir (nfsx, xl, nfu, pathloc, mode, cbk, local);
+    if ((!nfsx) || (!xl) || (!pathloc) || (!nfu))
         return ret;
-}
-
 
+    ret = nfs_inode_mkdir(nfsx, xl, nfu, pathloc, mode, cbk, local);
+    return ret;
+}
 
 int
-nfs_truncate (xlator_t *nfsx, xlator_t *xl, nfs_user_t *nfu, loc_t *pathloc,
-              off_t offset, fop_truncate_cbk_t cbk, void *local)
+nfs_truncate(xlator_t *nfsx, xlator_t *xl, nfs_user_t *nfu, loc_t *pathloc,
+             off_t offset, fop_truncate_cbk_t cbk, void *local)
 {
-        int     ret = -EFAULT;
+    int ret = -EFAULT;
 
-        if ((!xl) || (!pathloc) || (!nfu))
-                return ret;
-
-        ret = nfs_fop_truncate (nfsx, xl, nfu, pathloc, offset, cbk, local);
+    if ((!xl) || (!pathloc) || (!nfu))
         return ret;
+
+    ret = nfs_fop_truncate(nfsx, xl, nfu, pathloc, offset, cbk, local);
+    return ret;
 }
 
 int
-nfs_read (xlator_t *nfsx, xlator_t *xl, nfs_user_t *nfu, fd_t *fd, size_t size,
-          off_t offset, fop_readv_cbk_t cbk, void *local)
+nfs_read(xlator_t *nfsx, xlator_t *xl, nfs_user_t *nfu, fd_t *fd, size_t size,
+         off_t offset, fop_readv_cbk_t cbk, void *local)
 {
-        return nfs_fop_read (nfsx, xl, nfu, fd, size, offset, cbk, local);
+    return nfs_fop_read(nfsx, xl, nfu, fd, size, offset, cbk, local);
 }
 
 int
-nfs_lk (xlator_t *nfsx, xlator_t *xl, nfs_user_t *nfu, fd_t *fd,
-        int cmd, struct gf_flock *flock, fop_lk_cbk_t cbk, void *local)
+nfs_lk(xlator_t *nfsx, xlator_t *xl, nfs_user_t *nfu, fd_t *fd, int cmd,
+       struct gf_flock *flock, fop_lk_cbk_t cbk, void *local)
 {
-        return nfs_fop_lk ( nfsx, xl, nfu, fd, cmd, flock, cbk, local);
+    return nfs_fop_lk(nfsx, xl, nfu, fd, cmd, flock, cbk, local);
 }
 
 int
-nfs_getxattr (xlator_t *nfsx, xlator_t *xl, nfs_user_t *nfu, loc_t *loc,
-              char *name, dict_t *xdata, fop_getxattr_cbk_t cbk, void *local)
+nfs_getxattr(xlator_t *nfsx, xlator_t *xl, nfs_user_t *nfu, loc_t *loc,
+             char *name, dict_t *xdata, fop_getxattr_cbk_t cbk, void *local)
 {
-        return nfs_fop_getxattr (nfsx, xl, nfu, loc, name, xdata, cbk, local);
+    return nfs_fop_getxattr(nfsx, xl, nfu, loc, name, xdata, cbk, local);
 }
 
 int
-nfs_setxattr (xlator_t *nfsx, xlator_t *xl, nfs_user_t *nfu,
-              loc_t *loc, dict_t *dict, int32_t flags, dict_t *xdata,
-              fop_setxattr_cbk_t cbk, void *local)
+nfs_setxattr(xlator_t *nfsx, xlator_t *xl, nfs_user_t *nfu, loc_t *loc,
+             dict_t *dict, int32_t flags, dict_t *xdata, fop_setxattr_cbk_t cbk,
+             void *local)
 {
-        return nfs_fop_setxattr (nfsx, xl, nfu, loc, dict, flags, xdata, cbk,
-                                local);
+    return nfs_fop_setxattr(nfsx, xl, nfu, loc, dict, flags, xdata, cbk, local);
 }
 
 int
-nfs_fsync (xlator_t *nfsx, xlator_t *xl, nfs_user_t *nfu, fd_t *fd,
-           int32_t datasync, fop_fsync_cbk_t cbk, void *local)
+nfs_fsync(xlator_t *nfsx, xlator_t *xl, nfs_user_t *nfu, fd_t *fd,
+          int32_t datasync, fop_fsync_cbk_t cbk, void *local)
 {
-        return nfs_fop_fsync (nfsx, xl, nfu, fd, datasync, cbk, local);
+    return nfs_fop_fsync(nfsx, xl, nfu, fd, datasync, cbk, local);
 }
 
-
 int
-nfs_write (xlator_t *nfsx, xlator_t *xl, nfs_user_t *nfu, fd_t *fd,
-           struct iobref *srciobref, struct iovec *vector, int32_t count,
-           off_t offset, fop_writev_cbk_t cbk, void *local)
+nfs_write(xlator_t *nfsx, xlator_t *xl, nfs_user_t *nfu, fd_t *fd,
+          struct iobref *srciobref, struct iovec *vector, int32_t count,
+          off_t offset, fop_writev_cbk_t cbk, void *local)
 {
-        return nfs_fop_write (nfsx, xl, nfu, fd, srciobref, vector, count,
-                              offset, cbk, local);
+    return nfs_fop_write(nfsx, xl, nfu, fd, srciobref, vector, count, offset,
+                         cbk, local);
 }
 
-
 int
-nfs_open (xlator_t *nfsx, xlator_t *xl, nfs_user_t *nfu, loc_t *pathloc,
-          int32_t flags, fop_open_cbk_t cbk, void *local)
+nfs_open(xlator_t *nfsx, xlator_t *xl, nfs_user_t *nfu, loc_t *pathloc,
+         int32_t flags, fop_open_cbk_t cbk, void *local)
 {
-        int             ret = -EFAULT;
-
-        if ((!nfsx) || (!xl) || (!pathloc) || (!nfu))
-                return ret;
+    int ret = -EFAULT;
 
-        ret = nfs_inode_open (nfsx, xl, nfu, pathloc, flags, cbk,
-                              local);
+    if ((!nfsx) || (!xl) || (!pathloc) || (!nfu))
         return ret;
-}
 
+    ret = nfs_inode_open(nfsx, xl, nfu, pathloc, flags, cbk, local);
+    return ret;
+}
 
 int
-nfs_rename (xlator_t *nfsx, xlator_t *xl, nfs_user_t *nfu, loc_t *oldloc,
-            loc_t *newloc, fop_rename_cbk_t cbk, void *local)
+nfs_rename(xlator_t *nfsx, xlator_t *xl, nfs_user_t *nfu, loc_t *oldloc,
+           loc_t *newloc, fop_rename_cbk_t cbk, void *local)
 {
-        int                     ret = -EFAULT;
-
-        if ((!nfsx) || (!xl) || (!oldloc) || (!newloc) || (!nfu))
-                return ret;
+    int ret = -EFAULT;
 
-        ret = nfs_inode_rename (nfsx, xl, nfu, oldloc, newloc, cbk, local);
+    if ((!nfsx) || (!xl) || (!oldloc) || (!newloc) || (!nfu))
         return ret;
-}
 
+    ret = nfs_inode_rename(nfsx, xl, nfu, oldloc, newloc, cbk, local);
+    return ret;
+}
 
 int
-nfs_link (xlator_t *nfsx, xlator_t *xl, nfs_user_t *nfu, loc_t *oldloc,
-          loc_t *newloc, fop_link_cbk_t cbk, void *local)
+nfs_link(xlator_t *nfsx, xlator_t *xl, nfs_user_t *nfu, loc_t *oldloc,
+         loc_t *newloc, fop_link_cbk_t cbk, void *local)
 {
-        int                     ret = -EFAULT;
+    int ret = -EFAULT;
 
-        if ((!nfsx) || (!xl) || (!oldloc) || (!newloc) || (!nfu))
-                return ret;
-
-        ret = nfs_inode_link (nfsx, xl, nfu, oldloc, newloc, cbk, local);
+    if ((!nfsx) || (!xl) || (!oldloc) || (!newloc) || (!nfu))
         return ret;
-}
 
+    ret = nfs_inode_link(nfsx, xl, nfu, oldloc, newloc, cbk, local);
+    return ret;
+}
 
 int
-nfs_unlink (xlator_t *nfsx, xlator_t *xl, nfs_user_t *nfu, loc_t *pathloc,
-            fop_unlink_cbk_t cbk, void *local)
+nfs_unlink(xlator_t *nfsx, xlator_t *xl, nfs_user_t *nfu, loc_t *pathloc,
+           fop_unlink_cbk_t cbk, void *local)
 {
-        int                     ret = -EFAULT;
-
-        if ((!xl) || (!pathloc) || (!nfu))
-                return ret;
+    int ret = -EFAULT;
 
-        ret = nfs_inode_unlink (nfsx, xl, nfu, pathloc, cbk, local);
+    if ((!xl) || (!pathloc) || (!nfu))
         return ret;
-}
 
+    ret = nfs_inode_unlink(nfsx, xl, nfu, pathloc, cbk, local);
+    return ret;
+}
 
 int
-nfs_rmdir (xlator_t *nfsx, xlator_t *xl, nfs_user_t *nfu, loc_t *path,
-           fop_rmdir_cbk_t cbk, void *local)
+nfs_rmdir(xlator_t *nfsx, xlator_t *xl, nfs_user_t *nfu, loc_t *path,
+          fop_rmdir_cbk_t cbk, void *local)
 {
-        int                     ret = -EFAULT;
+    int ret = -EFAULT;
 
-        if ((!nfsx) || (!xl) || (!path) || (!nfu))
-                return ret;
-
-        ret = nfs_inode_rmdir (nfsx, xl, nfu, path, cbk, local);
+    if ((!nfsx) || (!xl) || (!path) || (!nfu))
         return ret;
-}
 
+    ret = nfs_inode_rmdir(nfsx, xl, nfu, path, cbk, local);
+    return ret;
+}
 
 int
-nfs_mknod (xlator_t *nfsx, xlator_t *xl, nfs_user_t *nfu, loc_t *pathloc,
-           mode_t mode, dev_t dev, fop_mknod_cbk_t cbk, void *local)
+nfs_mknod(xlator_t *nfsx, xlator_t *xl, nfs_user_t *nfu, loc_t *pathloc,
+          mode_t mode, dev_t dev, fop_mknod_cbk_t cbk, void *local)
 {
-        int                     ret = -EFAULT;
-
-        if ((!nfsx) || (!xl) || (!pathloc) || (!nfu))
-                return ret;
+    int ret = -EFAULT;
 
-        ret = nfs_inode_mknod (nfsx, xl, nfu, pathloc, mode, dev, cbk, local);
+    if ((!nfsx) || (!xl) || (!pathloc) || (!nfu))
         return ret;
-}
 
+    ret = nfs_inode_mknod(nfsx, xl, nfu, pathloc, mode, dev, cbk, local);
+    return ret;
+}
 
 int
-nfs_readlink (xlator_t *nfsx, xlator_t *xl, nfs_user_t *nfu, loc_t *linkloc,
-              fop_readlink_cbk_t cbk, void *local)
+nfs_readlink(xlator_t *nfsx, xlator_t *xl, nfs_user_t *nfu, loc_t *linkloc,
+             fop_readlink_cbk_t cbk, void *local)
 {
-        int                     ret = -EFAULT;
-
-        if ((!nfsx) || (!xl) || (!linkloc) || (!nfu))
-                return ret;
+    int ret = -EFAULT;
 
-        ret = nfs_fop_readlink (nfsx, xl, nfu, linkloc, NFS_PATH_MAX, cbk,
-                                local);
+    if ((!nfsx) || (!xl) || (!linkloc) || (!nfu))
         return ret;
-}
 
+    ret = nfs_fop_readlink(nfsx, xl, nfu, linkloc, NFS_PATH_MAX, cbk, local);
+    return ret;
+}
 
 int
-nfs_symlink (xlator_t *nfsx, xlator_t *xl, nfs_user_t *nfu, char *target,
-             loc_t *linkloc, fop_symlink_cbk_t cbk, void *local)
+nfs_symlink(xlator_t *nfsx, xlator_t *xl, nfs_user_t *nfu, char *target,
+            loc_t *linkloc, fop_symlink_cbk_t cbk, void *local)
 {
-        int                     ret = -EFAULT;
+    int ret = -EFAULT;
 
-        if ((!nfsx) || (!xl) || (!linkloc) || (!target) || (!nfu))
-                return ret;
-
-        ret = nfs_inode_symlink (nfsx, xl, nfu, target, linkloc, cbk, local);
+    if ((!nfsx) || (!xl) || (!linkloc) || (!target) || (!nfu))
         return ret;
-}
-
 
+    ret = nfs_inode_symlink(nfsx, xl, nfu, target, linkloc, cbk, local);
+    return ret;
+}
 
 int
-nfs_setattr (xlator_t *nfsx, xlator_t *xl, nfs_user_t *nfu, loc_t *pathloc,
-             struct iatt *buf, int32_t valid, fop_setattr_cbk_t cbk,
-             void *local)
+nfs_setattr(xlator_t *nfsx, xlator_t *xl, nfs_user_t *nfu, loc_t *pathloc,
+            struct iatt *buf, int32_t valid, fop_setattr_cbk_t cbk, void *local)
 {
-        int     ret = -EFAULT;
-
-        if ((!nfsx) || (!xl) || (!pathloc) || (!nfu))
-                return ret;
+    int ret = -EFAULT;
 
-        ret = nfs_fop_setattr (nfsx, xl, nfu, pathloc, buf, valid, cbk, local);
+    if ((!nfsx) || (!xl) || (!pathloc) || (!nfu))
         return ret;
-}
 
+    ret = nfs_fop_setattr(nfsx, xl, nfu, pathloc, buf, valid, cbk, local);
+    return ret;
+}
 
 int
-nfs_statfs (xlator_t *nfsx, xlator_t *xl, nfs_user_t *nfu, loc_t *pathloc,
-            fop_statfs_cbk_t cbk, void *local)
+nfs_statfs(xlator_t *nfsx, xlator_t *xl, nfs_user_t *nfu, loc_t *pathloc,
+           fop_statfs_cbk_t cbk, void *local)
 {
-        int     ret = -EFAULT;
+    int ret = -EFAULT;
 
-        if ((!nfsx) || (!xl) || (!pathloc) || (!nfu))
-                return ret;
-
-        ret = nfs_fop_statfs (nfsx, xl, nfu, pathloc, cbk, local);
+    if ((!nfsx) || (!xl) || (!pathloc) || (!nfu))
         return ret;
+
+    ret = nfs_fop_statfs(nfsx, xl, nfu, pathloc, cbk, local);
+    return ret;
 }
 
 int
-nfs_opendir (xlator_t *nfsx, xlator_t *fopxl, nfs_user_t *nfu, loc_t *pathloc,
-             fop_opendir_cbk_t cbk, void *local)
+nfs_opendir(xlator_t *nfsx, xlator_t *fopxl, nfs_user_t *nfu, loc_t *pathloc,
+            fop_opendir_cbk_t cbk, void *local)
 {
-        if ((!nfsx) || (!fopxl) || (!pathloc) || (!nfu))
-                return -EFAULT;
+    if ((!nfsx) || (!fopxl) || (!pathloc) || (!nfu))
+        return -EFAULT;
 
-        return nfs_inode_opendir (nfsx, fopxl, nfu, pathloc, cbk, local);
+    return nfs_inode_opendir(nfsx, fopxl, nfu, pathloc, cbk, local);
 }
-
diff --git a/xlators/nfs/server/src/nfs-generics.h b/xlators/nfs/server/src/nfs-generics.h
index 01876d68e2b..07a79994a4b 100644
--- a/xlators/nfs/server/src/nfs-generics.h
+++ b/xlators/nfs/server/src/nfs-generics.h
@@ -11,23 +11,18 @@
 #ifndef _NFS_GENERICS_H_
 #define _NFS_GENERICS_H_
 
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
-
 #include "nfs.h"
-#include "xlator.h"
+#include <glusterfs/xlator.h>
 #include "nfs-fops.h"
 #include "nfs-inodes.h"
 
 struct nfs_direntcache {
-        gf_dirent_t entries;            /* Head of list of cached dirents. */
-        gf_dirent_t *next;              /* Pointer to the next entry that
-                                         * should be sent by readdir */
-        uint64_t prev_off;              /* Offset where the next read will
-                                         * happen.
-                                         */
+    gf_dirent_t entries; /* Head of list of cached dirents. */
+    gf_dirent_t *next;   /* Pointer to the next entry that
+                          * should be sent by readdir */
+    uint64_t prev_off;   /* Offset where the next read will
+                          * happen.
+                          */
 };
 
 /* WE're trying to abstract the fops interface from the NFS xlator so that
@@ -37,132 +32,130 @@ struct nfs_direntcache {
  * state is requires in the fd. E.g. the dirent cache for a directory fd_t.
  */
 typedef struct nfs_fop_fdcontext {
-        pthread_mutex_t         lock;
-        size_t                  dirent_bufsize;
-        off_t                   offset;
-        struct nfs_direntcache  *dcache;
-        xlator_t                *dirvol;
+    pthread_mutex_t lock;
+    size_t dirent_bufsize;
+    off_t offset;
+    struct nfs_direntcache *dcache;
+    xlator_t *dirvol;
 } nfs_fdctx_t;
 
 extern int
-nfs_fstat (xlator_t *nfsx, xlator_t *xl, nfs_user_t *nfu, fd_t *fd,
-           fop_stat_cbk_t cbk, void *local);
+nfs_fstat(xlator_t *nfsx, xlator_t *xl, nfs_user_t *nfu, fd_t *fd,
+          fop_stat_cbk_t cbk, void *local);
 
 extern int
-nfs_readdirp (xlator_t *nfsx, xlator_t *xl, nfs_user_t *nfu, fd_t *dirfd,
-              size_t bufsize, off_t offset, fop_readdir_cbk_t cbk, void *local);
-
+nfs_readdirp(xlator_t *nfsx, xlator_t *xl, nfs_user_t *nfu, fd_t *dirfd,
+             size_t bufsize, off_t offset, fop_readdir_cbk_t cbk, void *local);
 
 extern int
-nfs_lookup (xlator_t *nfsx, xlator_t *xl, nfs_user_t *nfu, loc_t *pathloc,
-            fop_lookup_cbk_t cbk, void *local);
+nfs_lookup(xlator_t *nfsx, xlator_t *xl, nfs_user_t *nfu, loc_t *pathloc,
+           fop_lookup_cbk_t cbk, void *local);
 
 extern int
-nfs_create (xlator_t *nfsx, xlator_t *xl, nfs_user_t *nfu, loc_t *pathloc,
-            int flags, mode_t mode, fop_create_cbk_t cbk, void *local);
+nfs_create(xlator_t *nfsx, xlator_t *xl, nfs_user_t *nfu, loc_t *pathloc,
+           int flags, mode_t mode, fop_create_cbk_t cbk, void *local);
 
 extern int
-nfs_flush (xlator_t *nfsx, xlator_t *xl, nfs_user_t *nfu, fd_t *fd,
-           fop_flush_cbk_t cbk, void *local);
+nfs_flush(xlator_t *nfsx, xlator_t *xl, nfs_user_t *nfu, fd_t *fd,
+          fop_flush_cbk_t cbk, void *local);
 
 extern int
-nfs_mkdir (xlator_t *nfsx, xlator_t *xl, nfs_user_t *nfu, loc_t *pathloc,
-           mode_t mode, fop_mkdir_cbk_t cbk, void *local);
+nfs_mkdir(xlator_t *nfsx, xlator_t *xl, nfs_user_t *nfu, loc_t *pathloc,
+          mode_t mode, fop_mkdir_cbk_t cbk, void *local);
 
 extern int
-nfs_truncate (xlator_t *nfsx, xlator_t *xl, nfs_user_t *nfu, loc_t *pathloc,
-              off_t offset, fop_truncate_cbk_t cbk, void *local);
+nfs_truncate(xlator_t *nfsx, xlator_t *xl, nfs_user_t *nfu, loc_t *pathloc,
+             off_t offset, fop_truncate_cbk_t cbk, void *local);
 
 extern int
-nfs_read (xlator_t *nfsx, xlator_t *xl, nfs_user_t *nfu, fd_t *fd, size_t size,
-          off_t offset, fop_readv_cbk_t cbk, void *local);
+nfs_read(xlator_t *nfsx, xlator_t *xl, nfs_user_t *nfu, fd_t *fd, size_t size,
+         off_t offset, fop_readv_cbk_t cbk, void *local);
 
 extern int
-nfs_fsync (xlator_t *nfsx, xlator_t *xl, nfs_user_t *nfu, fd_t *fd,
-           int32_t datasync, fop_fsync_cbk_t cbk, void *local);
+nfs_fsync(xlator_t *nfsx, xlator_t *xl, nfs_user_t *nfu, fd_t *fd,
+          int32_t datasync, fop_fsync_cbk_t cbk, void *local);
 
 extern int
-nfs_write (xlator_t *nfsx, xlator_t *xl, nfs_user_t *nfu, fd_t *fd,
-           struct iobref *srciobref, struct iovec *vector, int32_t count,
-           off_t offset, fop_writev_cbk_t cbk, void *local);
+nfs_write(xlator_t *nfsx, xlator_t *xl, nfs_user_t *nfu, fd_t *fd,
+          struct iobref *srciobref, struct iovec *vector, int32_t count,
+          off_t offset, fop_writev_cbk_t cbk, void *local);
 
 extern int
-nfs_open (xlator_t *nfsx, xlator_t *xl, nfs_user_t *nfu, loc_t *pathloc,
-          int32_t flags, fop_open_cbk_t cbk, void *local);
+nfs_open(xlator_t *nfsx, xlator_t *xl, nfs_user_t *nfu, loc_t *pathloc,
+         int32_t flags, fop_open_cbk_t cbk, void *local);
 
 extern int
-nfs_rename (xlator_t *nfsx, xlator_t *xl, nfs_user_t *nfu, loc_t *oldloc,
-            loc_t *newloc, fop_rename_cbk_t cbk, void *local);
+nfs_rename(xlator_t *nfsx, xlator_t *xl, nfs_user_t *nfu, loc_t *oldloc,
+           loc_t *newloc, fop_rename_cbk_t cbk, void *local);
 
 extern int
-nfs_link (xlator_t *nfsx, xlator_t *xl, nfs_user_t *nfu, loc_t *oldloc,
-          loc_t *newloc, fop_link_cbk_t cbk, void *local);
+nfs_link(xlator_t *nfsx, xlator_t *xl, nfs_user_t *nfu, loc_t *oldloc,
+         loc_t *newloc, fop_link_cbk_t cbk, void *local);
 
 extern int
-nfs_unlink (xlator_t *nfsx, xlator_t *xl, nfs_user_t *nfu, loc_t *pathloc,
-            fop_unlink_cbk_t cbk, void *local);
+nfs_unlink(xlator_t *nfsx, xlator_t *xl, nfs_user_t *nfu, loc_t *pathloc,
+           fop_unlink_cbk_t cbk, void *local);
 
 extern int
-nfs_rmdir (xlator_t *nfsx, xlator_t *xl, nfs_user_t *nfu, loc_t *pathloc,
-           fop_rmdir_cbk_t cbk, void *local);
+nfs_rmdir(xlator_t *nfsx, xlator_t *xl, nfs_user_t *nfu, loc_t *pathloc,
+          fop_rmdir_cbk_t cbk, void *local);
 
 extern int
-nfs_mknod (xlator_t *nfsx, xlator_t *xl, nfs_user_t *nfu, loc_t *pathloc,
-           mode_t mode, dev_t dev, fop_mknod_cbk_t cbk, void *local);
+nfs_mknod(xlator_t *nfsx, xlator_t *xl, nfs_user_t *nfu, loc_t *pathloc,
+          mode_t mode, dev_t dev, fop_mknod_cbk_t cbk, void *local);
 
 extern int
-nfs_readlink (xlator_t *nfsx, xlator_t *xl, nfs_user_t *nfu, loc_t *linkloc,
-              fop_readlink_cbk_t cbk, void *local);
+nfs_readlink(xlator_t *nfsx, xlator_t *xl, nfs_user_t *nfu, loc_t *linkloc,
+             fop_readlink_cbk_t cbk, void *local);
 
 extern int
-nfs_setattr (xlator_t *nfsx, xlator_t *xl, nfs_user_t *nfu, loc_t *pathloc,
-             struct iatt *buf, int32_t valid, fop_setattr_cbk_t cbk,
-             void *local);
+nfs_setattr(xlator_t *nfsx, xlator_t *xl, nfs_user_t *nfu, loc_t *pathloc,
+            struct iatt *buf, int32_t valid, fop_setattr_cbk_t cbk,
+            void *local);
 
 extern int
-nfs_statfs (xlator_t *nfsx, xlator_t *xl, nfs_user_t *nfu, loc_t *pathloc,
-            fop_statfs_cbk_t cbk, void *local);
+nfs_statfs(xlator_t *nfsx, xlator_t *xl, nfs_user_t *nfu, loc_t *pathloc,
+           fop_statfs_cbk_t cbk, void *local);
 
 extern int
-nfs_stat (xlator_t *nfsx, xlator_t *xl, nfs_user_t *nfu, loc_t *pathloc,
-          fop_stat_cbk_t cbk, void *local);
+nfs_stat(xlator_t *nfsx, xlator_t *xl, nfs_user_t *nfu, loc_t *pathloc,
+         fop_stat_cbk_t cbk, void *local);
 
 extern int
-nfs_symlink (xlator_t *nfsx, xlator_t *xl, nfs_user_t *nfu, char *target,
-             loc_t *linkloc, fop_symlink_cbk_t cbk, void *local);
+nfs_symlink(xlator_t *nfsx, xlator_t *xl, nfs_user_t *nfu, char *target,
+            loc_t *linkloc, fop_symlink_cbk_t cbk, void *local);
 
 /* Synchronous equivalents */
 
 extern call_stub_t *
-nfs_open_sync (xlator_t *xl, nfs_user_t *nfu, loc_t *pathloc,
-               int32_t flags);
+nfs_open_sync(xlator_t *xl, nfs_user_t *nfu, loc_t *pathloc, int32_t flags);
 
 extern call_stub_t *
-nfs_write_sync (xlator_t *xl, nfs_user_t *nfu, fd_t *fd, struct iobuf *srciob,
-                struct iovec *vec, int count, off_t offset);
+nfs_write_sync(xlator_t *xl, nfs_user_t *nfu, fd_t *fd, struct iobuf *srciob,
+               struct iovec *vec, int count, off_t offset);
 
 extern call_stub_t *
-nfs_read_sync (xlator_t *xl, nfs_user_t *nfu, fd_t *fd, size_t size,
-               off_t offset);
+nfs_read_sync(xlator_t *xl, nfs_user_t *nfu, fd_t *fd, size_t size,
+              off_t offset);
 
 extern int
-nfs_opendir (xlator_t *nfsx, xlator_t *fopxl, nfs_user_t *nfu, loc_t *pathloc,
-             fop_opendir_cbk_t cbk, void *local);
+nfs_opendir(xlator_t *nfsx, xlator_t *fopxl, nfs_user_t *nfu, loc_t *pathloc,
+            fop_opendir_cbk_t cbk, void *local);
 
 extern int
-nfs_access (xlator_t *nfsx, xlator_t *xl, nfs_user_t *nfu, loc_t *pathloc,
-            int32_t accesstest, fop_access_cbk_t cbk, void *local);
+nfs_access(xlator_t *nfsx, xlator_t *xl, nfs_user_t *nfu, loc_t *pathloc,
+           int32_t accesstest, fop_access_cbk_t cbk, void *local);
 extern int
-nfs_lk (xlator_t *nfsx, xlator_t *xl, nfs_user_t *nfu, fd_t *fd,
-        int cmd, struct gf_flock *flock, fop_lk_cbk_t cbk, void *local);
+nfs_lk(xlator_t *nfsx, xlator_t *xl, nfs_user_t *nfu, fd_t *fd, int cmd,
+       struct gf_flock *flock, fop_lk_cbk_t cbk, void *local);
 
 extern int
-nfs_getxattr (xlator_t *nfsx, xlator_t *xl, nfs_user_t *nfu, loc_t *loc,
-              char *name, dict_t *xdata, fop_getxattr_cbk_t cbk, void *local);
+nfs_getxattr(xlator_t *nfsx, xlator_t *xl, nfs_user_t *nfu, loc_t *loc,
+             char *name, dict_t *xdata, fop_getxattr_cbk_t cbk, void *local);
 
 extern int
-nfs_setxattr (xlator_t *nfsx, xlator_t *xl, nfs_user_t *nfu,
-              loc_t *loc, dict_t *dict, int32_t flags, dict_t *xdata,
-              fop_setxattr_cbk_t cbk, void *local);
+nfs_setxattr(xlator_t *nfsx, xlator_t *xl, nfs_user_t *nfu, loc_t *loc,
+             dict_t *dict, int32_t flags, dict_t *xdata, fop_setxattr_cbk_t cbk,
+             void *local);
 
 #endif
diff --git a/xlators/nfs/server/src/nfs-inodes.c b/xlators/nfs/server/src/nfs-inodes.c
index 63d5e8a198b..6f34ca705ba 100644
--- a/xlators/nfs/server/src/nfs-inodes.c
+++ b/xlators/nfs/server/src/nfs-inodes.c
@@ -8,601 +8,572 @@
   cases as published by the Free Software Foundation.
 */
 
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
-
 #include "string.h"
 
-#include "inode.h"
 #include "nfs.h"
 #include "nfs-inodes.h"
 #include "nfs-fops.h"
-#include "xlator.h"
+#include <glusterfs/xlator.h>
+#include "nfs-messages.h"
 
 #include <libgen.h>
 
-#define inodes_nfl_to_prog_data(nflocal, pcbk, fram)                    \
-        do {                                                            \
-                nflocal = fram->local;                                  \
-                fram->local = nflocal->proglocal;                       \
-                *VOID(&pcbk) = nflocal->progcbk;                                \
-                nfs_fop_local_wipe (nflocal->nfsx, nflocal);            \
-        } while (0)                                                     \
+#define inodes_nfl_to_prog_data(nflocal, pcbk, fram)                           \
+    do {                                                                       \
+        nflocal = fram->local;                                                 \
+        fram->local = nflocal->proglocal;                                      \
+        *VOID(&pcbk) = nflocal->progcbk;                                       \
+        nfs_fop_local_wipe(nflocal->nfsx, nflocal);                            \
+    } while (0)
 
 void
-nfl_inodes_init (struct nfs_fop_local *nfl, inode_t *inode, inode_t *parent,
-                 inode_t *newparent, const char *name, const char *newname)
+nfl_inodes_init(struct nfs_fop_local *nfl, inode_t *inode, inode_t *parent,
+                inode_t *newparent, const char *name, const char *newname)
 {
-        if (!nfl)
-                return;
-
-        if (inode)
-                nfl->inode = inode_ref (inode);
+    if (!nfl)
+        return;
 
-        if (parent)
-                nfl->parent = inode_ref (parent);
+    if (inode)
+        nfl->inode = inode_ref(inode);
 
-        if (newparent)
-                nfl->newparent = inode_ref (newparent);
+    if (parent)
+        nfl->parent = inode_ref(parent);
 
-        if (name)
-                strncpy (nfl->path, name, NFS_NAME_MAX);
+    if (newparent)
+        nfl->newparent = inode_ref(newparent);
 
-        if (newname)
-                strncpy (nfl->newpath, newname, NFS_NAME_MAX);
+    if (name)
+        snprintf(nfl->path, NFS_NAME_MAX, "%s", name);
 
-        return;
+    if (newname)
+        snprintf(nfl->newpath, NFS_NAME_MAX, "%s", newname);
+    return;
 }
 
-
 int32_t
-nfs_inode_create_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                      int32_t op_ret, int32_t op_errno, fd_t *fd, inode_t *inode
-                      , struct iatt *buf, struct iatt *preparent,
-                      struct iatt *postparent, dict_t *xdata)
+nfs_inode_create_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                     int32_t op_ret, int32_t op_errno, fd_t *fd, inode_t *inode,
+                     struct iatt *buf, struct iatt *preparent,
+                     struct iatt *postparent, dict_t *xdata)
 {
-        struct nfs_fop_local    *nfl = frame->local;
-        fop_create_cbk_t        progcbk = NULL;
-        inode_t                 *linked_inode = NULL;
+    struct nfs_fop_local *nfl = frame->local;
+    fop_create_cbk_t progcbk = NULL;
+    inode_t *linked_inode = NULL;
 
-        if (op_ret == -1)
-                goto do_not_link;
+    if (op_ret == -1)
+        goto do_not_link;
 
-        linked_inode = inode_link (inode, nfl->parent, nfl->path, buf);
+    linked_inode = inode_link(inode, nfl->parent, nfl->path, buf);
 
 do_not_link:
-        /* NFS does not need it, upper layers should not expect the pointer to
-         * be a valid fd.
-         */
-        fd_unref (fd);
-
-        inodes_nfl_to_prog_data (nfl, progcbk, frame);
-        if (progcbk)
-                progcbk (frame, cookie, this, op_ret, op_errno, fd, inode, buf,
-                         preparent, postparent, xdata);
-
-        if (linked_inode) {
-                inode_lookup (linked_inode);
-                inode_unref (linked_inode);
-        }
-
-        return 0;
+    /* NFS does not need it, upper layers should not expect the pointer to
+     * be a valid fd.
+     */
+    fd_unref(fd);
+
+    inodes_nfl_to_prog_data(nfl, progcbk, frame);
+    if (progcbk)
+        progcbk(frame, cookie, this, op_ret, op_errno, fd, inode, buf,
+                preparent, postparent, xdata);
+
+    if (linked_inode) {
+        inode_lookup(linked_inode);
+        inode_unref(linked_inode);
+    }
+
+    return 0;
 }
 
-
 int
-nfs_inode_create (xlator_t *nfsx, xlator_t *xl, nfs_user_t *nfu,
-                  loc_t *pathloc, int flags, int mode, fop_create_cbk_t cbk,
-                  void *local)
+nfs_inode_create(xlator_t *nfsx, xlator_t *xl, nfs_user_t *nfu, loc_t *pathloc,
+                 int flags, int mode, fop_create_cbk_t cbk, void *local)
 {
-        struct nfs_fop_local    *nfl = NULL;
-        int                     ret = -EFAULT;
-        fd_t                    *newfd = NULL;
-
-        if ((!nfsx) || (!xl) || (!pathloc) || (!nfu))
-                return ret;
-
-        nfs_fop_handle_local_init (NULL, nfsx, nfl, cbk, local, ret, err);
-
-        newfd = fd_create (pathloc->inode, 0);
-        if (!newfd) {
-                gf_log (GF_NFS, GF_LOG_ERROR, "Failed to create new fd");
-                ret = -ENOMEM;
-                goto wipe_nfl;
-        }
-
-        /* The parent and base name will be needed to link the new inode
-         * into the inode table.
-         */
-        nfl_inodes_init (nfl, pathloc->inode, pathloc->parent, NULL,
-                         pathloc->name, NULL);
-        ret = nfs_fop_create (nfsx, xl, nfu, pathloc, flags, mode, newfd,
-                              nfs_inode_create_cbk, nfl);
+    struct nfs_fop_local *nfl = NULL;
+    int ret = -EFAULT;
+    fd_t *newfd = NULL;
+
+    if ((!nfsx) || (!xl) || (!pathloc) || (!nfu))
+        return ret;
+
+    nfs_fop_handle_local_init(NULL, nfsx, nfl, cbk, local, ret, err);
+
+    newfd = fd_create(pathloc->inode, 0);
+    if (!newfd) {
+        gf_msg(GF_NFS, GF_LOG_ERROR, ENOMEM, NFS_MSG_NO_MEMORY,
+               "Failed to create new fd");
+        ret = -ENOMEM;
+        goto wipe_nfl;
+    }
+
+    /* The parent and base name will be needed to link the new inode
+     * into the inode table.
+     */
+    nfl_inodes_init(nfl, pathloc->inode, pathloc->parent, NULL, pathloc->name,
+                    NULL);
+    ret = nfs_fop_create(nfsx, xl, nfu, pathloc, flags, mode, newfd,
+                         nfs_inode_create_cbk, nfl);
 wipe_nfl:
-        if (ret < 0)
-                nfs_fop_local_wipe (xl, nfl);
+    if (ret < 0)
+        nfs_fop_local_wipe(xl, nfl);
 
 err:
-        return ret;
+    return ret;
 }
 
-
 int32_t
-nfs_inode_mkdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                     int32_t op_ret, int32_t op_errno, inode_t *inode,
-                     struct iatt *buf, struct iatt *preparent,
-                     struct iatt *postparent, dict_t *xdata)
+nfs_inode_mkdir_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                    int32_t op_ret, int32_t op_errno, inode_t *inode,
+                    struct iatt *buf, struct iatt *preparent,
+                    struct iatt *postparent, dict_t *xdata)
 {
-        struct nfs_fop_local    *nfl = frame->local;
-        fop_mkdir_cbk_t         progcbk = NULL;
-        inode_t                 *linked_inode = NULL;
+    struct nfs_fop_local *nfl = frame->local;
+    fop_mkdir_cbk_t progcbk = NULL;
+    inode_t *linked_inode = NULL;
 
-        if (op_ret == -1)
-                goto do_not_link;
+    if (op_ret == -1)
+        goto do_not_link;
 
-        linked_inode = inode_link (inode, nfl->parent, nfl->path, buf);
+    linked_inode = inode_link(inode, nfl->parent, nfl->path, buf);
 
 do_not_link:
-        inodes_nfl_to_prog_data (nfl, progcbk, frame);
-        if (progcbk)
-                progcbk (frame, cookie, this, op_ret, op_errno, inode, buf,
-                         preparent, postparent, xdata);
+    inodes_nfl_to_prog_data(nfl, progcbk, frame);
+    if (progcbk)
+        progcbk(frame, cookie, this, op_ret, op_errno, inode, buf, preparent,
+                postparent, xdata);
 
-        if (linked_inode) {
-                inode_lookup (linked_inode);
-                inode_unref (linked_inode);
-        }
+    if (linked_inode) {
+        inode_lookup(linked_inode);
+        inode_unref(linked_inode);
+    }
 
-        return 0;
+    return 0;
 }
 
-
 int
-nfs_inode_mkdir (xlator_t *nfsx, xlator_t *xl, nfs_user_t *nfu, loc_t *pathloc,
-                 int mode, fop_mkdir_cbk_t cbk, void *local)
+nfs_inode_mkdir(xlator_t *nfsx, xlator_t *xl, nfs_user_t *nfu, loc_t *pathloc,
+                int mode, fop_mkdir_cbk_t cbk, void *local)
 {
-        struct nfs_fop_local    *nfl = NULL;
-        int                     ret = -EFAULT;
+    struct nfs_fop_local *nfl = NULL;
+    int ret = -EFAULT;
 
-        if ((!nfsx) || (!xl) || (!pathloc) || (!nfu))
-                return ret;
+    if ((!nfsx) || (!xl) || (!pathloc) || (!nfu))
+        return ret;
 
-        nfs_fop_handle_local_init (NULL, nfsx, nfl, cbk, local, ret, err);
-        nfl_inodes_init (nfl, pathloc->inode, pathloc->parent, NULL,
-                         pathloc->name, NULL);
-        ret = nfs_fop_mkdir (nfsx, xl, nfu, pathloc, mode, nfs_inode_mkdir_cbk,
-                             nfl);
-        if (ret < 0)
-                nfs_fop_local_wipe (nfsx, nfl);
+    nfs_fop_handle_local_init(NULL, nfsx, nfl, cbk, local, ret, err);
+    nfl_inodes_init(nfl, pathloc->inode, pathloc->parent, NULL, pathloc->name,
+                    NULL);
+    ret = nfs_fop_mkdir(nfsx, xl, nfu, pathloc, mode, nfs_inode_mkdir_cbk, nfl);
+    if (ret < 0)
+        nfs_fop_local_wipe(nfsx, nfl);
 
 err:
-        return ret;
+    return ret;
 }
 
-
 int32_t
-nfs_inode_open_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                    int32_t op_ret, int32_t op_errno, fd_t *fd, dict_t *xdata)
+nfs_inode_open_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                   int32_t op_ret, int32_t op_errno, fd_t *fd, dict_t *xdata)
 {
-
-        struct nfs_fop_local    *nfl = NULL;
-        fop_open_cbk_t          progcbk = NULL;
-
-        if ((op_ret == -1) && (fd))
-                fd_unref (fd);
-        /* Not needed here since the fd is cached in higher layers and the bind
-         * must happen atomically when the fd gets added to the fd LRU.
-         */
-/*        else
-                fd_bind (fd);
-*/
-        inodes_nfl_to_prog_data (nfl, progcbk, frame);
-        if (progcbk)
-                progcbk (frame, cookie, this, op_ret, op_errno, fd, xdata);
-        return 0;
+    struct nfs_fop_local *nfl = NULL;
+    fop_open_cbk_t progcbk = NULL;
+
+    if ((op_ret == -1) && (fd))
+        fd_unref(fd);
+    /* Not needed here since the fd is cached in higher layers and the bind
+     * must happen atomically when the fd gets added to the fd LRU.
+     */
+    /*        else
+                    fd_bind (fd);
+    */
+    inodes_nfl_to_prog_data(nfl, progcbk, frame);
+    if (progcbk)
+        progcbk(frame, cookie, this, op_ret, op_errno, fd, xdata);
+    return 0;
 }
 
-
 int
-nfs_inode_open (xlator_t *nfsx, xlator_t *xl, nfs_user_t *nfu, loc_t *loc,
-                int32_t flags, fop_open_cbk_t cbk, void *local)
+nfs_inode_open(xlator_t *nfsx, xlator_t *xl, nfs_user_t *nfu, loc_t *loc,
+               int32_t flags, fop_open_cbk_t cbk, void *local)
 {
-        struct nfs_fop_local    *nfl = NULL;
-        fd_t                    *newfd = NULL;
-        int                     ret = -EFAULT;
+    struct nfs_fop_local *nfl = NULL;
+    fd_t *newfd = NULL;
+    int ret = -EFAULT;
 
-        if ((!nfsx) || (!xl) || (!loc) || (!nfu))
-                return ret;
+    if ((!nfsx) || (!xl) || (!loc) || (!nfu))
+        return ret;
 
-        newfd = fd_create (loc->inode, 0);
-        if (!newfd) {
-                gf_log (GF_NFS, GF_LOG_ERROR, "Failed to create fd");
-                ret = -ENOMEM;
-                goto err;
-        }
+    newfd = fd_create(loc->inode, 0);
+    if (!newfd) {
+        gf_msg(GF_NFS, GF_LOG_ERROR, ENOMEM, NFS_MSG_NO_MEMORY,
+               "Failed to create fd");
+        ret = -ENOMEM;
+        goto err;
+    }
 
-        nfs_fop_handle_local_init (NULL, nfsx, nfl, cbk, local, ret, fd_err);
-        ret = nfs_fop_open (nfsx, xl, nfu, loc, flags, newfd,
-                            nfs_inode_open_cbk, nfl);
+    nfs_fop_handle_local_init(NULL, nfsx, nfl, cbk, local, ret, fd_err);
+    ret = nfs_fop_open(nfsx, xl, nfu, loc, flags, newfd, nfs_inode_open_cbk,
+                       nfl);
 
-        if (ret < 0)
-                nfs_fop_local_wipe (xl, nfl);
+    if (ret < 0)
+        nfs_fop_local_wipe(xl, nfl);
 
 fd_err:
-        if (ret < 0)
-                if (newfd)
-                        fd_unref (newfd);
+    if (ret < 0)
+        if (newfd)
+            fd_unref(newfd);
 
 err:
 
-        return ret;
+    return ret;
 }
 
-
-
 int32_t
-nfs_inode_rename_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                      int32_t op_ret, int32_t op_errno, struct iatt *buf,
-                      struct iatt *preoldparent, struct iatt *postoldparent,
-                      struct iatt *prenewparent, struct iatt *postnewparent,
-                      dict_t *xdata)
+nfs_inode_rename_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                     int32_t op_ret, int32_t op_errno, struct iatt *buf,
+                     struct iatt *preoldparent, struct iatt *postoldparent,
+                     struct iatt *prenewparent, struct iatt *postnewparent,
+                     dict_t *xdata)
 {
-        struct nfs_fop_local    *nfl = NULL;
-        fop_rename_cbk_t        progcbk = NULL;
+    struct nfs_fop_local *nfl = NULL;
+    fop_rename_cbk_t progcbk = NULL;
 
-        nfl = frame->local;
-        if (op_ret == -1)
-                goto do_not_link;
+    nfl = frame->local;
+    if (op_ret == -1)
+        goto do_not_link;
 
-        inode_rename (this->itable, nfl->parent, nfl->path, nfl->newparent,
-                      nfl->newpath, nfl->inode, buf);
+    inode_rename(this->itable, nfl->parent, nfl->path, nfl->newparent,
+                 nfl->newpath, nfl->inode, buf);
 
 do_not_link:
-        inodes_nfl_to_prog_data (nfl, progcbk, frame);
-        if (progcbk)
-                progcbk (frame, cookie, this, op_ret, op_errno, buf,
-                         preoldparent, postoldparent, prenewparent,
-                         postnewparent, xdata);
-        return 0;
+    inodes_nfl_to_prog_data(nfl, progcbk, frame);
+    if (progcbk)
+        progcbk(frame, cookie, this, op_ret, op_errno, buf, preoldparent,
+                postoldparent, prenewparent, postnewparent, xdata);
+    return 0;
 }
 
-
 int
-nfs_inode_rename (xlator_t *nfsx, xlator_t *xl, nfs_user_t *nfu, loc_t *oldloc,
-                  loc_t *newloc, fop_rename_cbk_t cbk, void *local)
+nfs_inode_rename(xlator_t *nfsx, xlator_t *xl, nfs_user_t *nfu, loc_t *oldloc,
+                 loc_t *newloc, fop_rename_cbk_t cbk, void *local)
 {
-        struct nfs_fop_local    *nfl = NULL;
-        int                     ret = -EFAULT;
+    struct nfs_fop_local *nfl = NULL;
+    int ret = -EFAULT;
 
-        if ((!nfsx) || (!xl) || (!oldloc) || (!newloc))
-                return ret;
+    if ((!nfsx) || (!xl) || (!oldloc) || (!newloc))
+        return ret;
 
-        nfs_fop_handle_local_init (NULL, nfsx, nfl, cbk, local, ret, err);
-        nfl_inodes_init (nfl, oldloc->inode, oldloc->parent, newloc->parent,
-                         oldloc->name, newloc->name);
-        ret = nfs_fop_rename (nfsx,xl, nfu, oldloc, newloc, nfs_inode_rename_cbk
-                              , nfl);
+    nfs_fop_handle_local_init(NULL, nfsx, nfl, cbk, local, ret, err);
+    nfl_inodes_init(nfl, oldloc->inode, oldloc->parent, newloc->parent,
+                    oldloc->name, newloc->name);
+    ret = nfs_fop_rename(nfsx, xl, nfu, oldloc, newloc, nfs_inode_rename_cbk,
+                         nfl);
 
 err:
-        if (ret < 0)
-                nfs_fop_local_wipe (xl, nfl);
+    if (ret < 0)
+        nfs_fop_local_wipe(xl, nfl);
 
-        return ret;
+    return ret;
 }
 
-
 int32_t
-nfs_inode_link_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                    int32_t op_ret, int32_t op_errno, inode_t *inode,
-                    struct iatt *buf, struct iatt *preparent,
-                    struct iatt *postparent, dict_t *xdata)
+nfs_inode_link_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                   int32_t op_ret, int32_t op_errno, inode_t *inode,
+                   struct iatt *buf, struct iatt *preparent,
+                   struct iatt *postparent, dict_t *xdata)
 {
-        struct nfs_fop_local    *nfl = NULL;
-        fop_link_cbk_t          progcbk = NULL;
-        inode_t                 *linked_inode = NULL;
+    struct nfs_fop_local *nfl = NULL;
+    fop_link_cbk_t progcbk = NULL;
+    inode_t *linked_inode = NULL;
 
-        if (op_ret == -1)
-                goto do_not_link;
+    if (op_ret == -1)
+        goto do_not_link;
 
-        nfl = frame->local;
-        linked_inode = inode_link (inode, nfl->newparent, nfl->path, buf);
+    nfl = frame->local;
+    linked_inode = inode_link(inode, nfl->newparent, nfl->path, buf);
 
 do_not_link:
-        inodes_nfl_to_prog_data (nfl, progcbk, frame);
-        if (progcbk)
-                progcbk (frame, cookie, this, op_ret, op_errno, inode, buf,
-                         preparent, postparent, xdata);
+    inodes_nfl_to_prog_data(nfl, progcbk, frame);
+    if (progcbk)
+        progcbk(frame, cookie, this, op_ret, op_errno, inode, buf, preparent,
+                postparent, xdata);
 
-        if (linked_inode) {
-                inode_lookup (linked_inode);
-                inode_unref (linked_inode);
-        }
+    if (linked_inode) {
+        inode_lookup(linked_inode);
+        inode_unref(linked_inode);
+    }
 
-        return 0;
+    return 0;
 }
 
-
 int
-nfs_inode_link (xlator_t *nfsx, xlator_t *xl, nfs_user_t *nfu, loc_t *oldloc,
-                loc_t *newloc, fop_link_cbk_t cbk, void *local)
+nfs_inode_link(xlator_t *nfsx, xlator_t *xl, nfs_user_t *nfu, loc_t *oldloc,
+               loc_t *newloc, fop_link_cbk_t cbk, void *local)
 {
-        struct nfs_fop_local            *nfl = NULL;
-        int                             ret = -EFAULT;
+    struct nfs_fop_local *nfl = NULL;
+    int ret = -EFAULT;
 
-        if ((!nfsx) || (!xl) || (!oldloc) || (!newloc) || (!nfu))
-                return -EFAULT;
+    if ((!nfsx) || (!xl) || (!oldloc) || (!newloc) || (!nfu))
+        return -EFAULT;
 
-        nfs_fop_handle_local_init (NULL, nfsx, nfl, cbk, local, ret, err);
-        nfl_inodes_init (nfl, NULL, NULL, newloc->parent, newloc->name, NULL);
-        ret = nfs_fop_link (nfsx, xl, nfu, oldloc, newloc, nfs_inode_link_cbk,
-                            nfl);
+    nfs_fop_handle_local_init(NULL, nfsx, nfl, cbk, local, ret, err);
+    nfl_inodes_init(nfl, NULL, NULL, newloc->parent, newloc->name, NULL);
+    ret = nfs_fop_link(nfsx, xl, nfu, oldloc, newloc, nfs_inode_link_cbk, nfl);
 
 err:
-        if (ret < 0)
-                nfs_fop_local_wipe (xl, nfl);
+    if (ret < 0)
+        nfs_fop_local_wipe(xl, nfl);
 
-        return ret;
+    return ret;
 }
 
-
 int32_t
-nfs_inode_unlink_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                      int32_t op_ret, int32_t op_errno, struct iatt *preparent,
-                      struct iatt *postparent, dict_t *xdata)
+nfs_inode_unlink_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                     int32_t op_ret, int32_t op_errno, struct iatt *preparent,
+                     struct iatt *postparent, dict_t *xdata)
 {
-        struct nfs_fop_local    *nfl = NULL;
-        fop_unlink_cbk_t        progcbk = NULL;
+    struct nfs_fop_local *nfl = NULL;
+    fop_unlink_cbk_t progcbk = NULL;
 
-        nfl = frame->local;
+    nfl = frame->local;
 
-        if (op_ret == -1)
-                goto do_not_unlink;
+    if (op_ret == -1)
+        goto do_not_unlink;
 
-        inode_unlink (nfl->inode, nfl->parent, nfl->path);
-        inode_forget (nfl->inode, 0);
+    inode_unlink(nfl->inode, nfl->parent, nfl->path);
+    inode_forget(nfl->inode, 0);
 
 do_not_unlink:
-        inodes_nfl_to_prog_data (nfl, progcbk, frame);
-        if (progcbk)
-                progcbk (frame, cookie, this, op_ret, op_errno, preparent,
-                         postparent, xdata);
-        return 0;
+    inodes_nfl_to_prog_data(nfl, progcbk, frame);
+    if (progcbk)
+        progcbk(frame, cookie, this, op_ret, op_errno, preparent, postparent,
+                xdata);
+    return 0;
 }
 
-
 int
-nfs_inode_unlink (xlator_t *nfsx, xlator_t *xl, nfs_user_t *nfu, loc_t *pathloc,
-                  fop_unlink_cbk_t cbk, void *local)
+nfs_inode_unlink(xlator_t *nfsx, xlator_t *xl, nfs_user_t *nfu, loc_t *pathloc,
+                 fop_unlink_cbk_t cbk, void *local)
 {
-        struct nfs_fop_local            *nfl = NULL;
-        int                             ret = -EFAULT;
+    struct nfs_fop_local *nfl = NULL;
+    int ret = -EFAULT;
 
-        if ((!nfsx) || (!xl) || (!pathloc) || (!nfu))
-                return -EFAULT;
+    if ((!nfsx) || (!xl) || (!pathloc) || (!nfu))
+        return -EFAULT;
 
-        nfs_fop_handle_local_init (NULL, nfsx, nfl, cbk, local, ret, err);
-        nfl_inodes_init (nfl, pathloc->inode, pathloc->parent, NULL,
-                         pathloc->name, NULL);
-        ret = nfs_fop_unlink (nfsx, xl, nfu, pathloc, nfs_inode_unlink_cbk,nfl);
+    nfs_fop_handle_local_init(NULL, nfsx, nfl, cbk, local, ret, err);
+    nfl_inodes_init(nfl, pathloc->inode, pathloc->parent, NULL, pathloc->name,
+                    NULL);
+    ret = nfs_fop_unlink(nfsx, xl, nfu, pathloc, nfs_inode_unlink_cbk, nfl);
 
 err:
-        if (ret < 0)
-                nfs_fop_local_wipe (xl, nfl);
+    if (ret < 0)
+        nfs_fop_local_wipe(xl, nfl);
 
-        return ret;
+    return ret;
 }
 
-
 int32_t
-nfs_inode_rmdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                     int32_t op_ret, int32_t op_errno, struct iatt *preparent,
-                     struct iatt *postparent, dict_t *xdata)
+nfs_inode_rmdir_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                    int32_t op_ret, int32_t op_errno, struct iatt *preparent,
+                    struct iatt *postparent, dict_t *xdata)
 {
-        struct nfs_fop_local    *nfl = NULL;
-        fop_rmdir_cbk_t         progcbk = NULL;
+    struct nfs_fop_local *nfl = NULL;
+    fop_rmdir_cbk_t progcbk = NULL;
 
-        nfl = frame->local;
+    nfl = frame->local;
 
-        if (op_ret == -1)
-                goto do_not_unlink;
+    if (op_ret == -1)
+        goto do_not_unlink;
 
-        inode_unlink (nfl->inode, nfl->parent, nfl->path);
-        inode_forget (nfl->inode, 0);
+    inode_unlink(nfl->inode, nfl->parent, nfl->path);
+    inode_forget(nfl->inode, 0);
 
 do_not_unlink:
-        inodes_nfl_to_prog_data (nfl, progcbk, frame);
-        if (progcbk)
-                progcbk (frame, cookie, this, op_ret, op_errno, preparent,
-                         postparent, xdata);
+    inodes_nfl_to_prog_data(nfl, progcbk, frame);
+    if (progcbk)
+        progcbk(frame, cookie, this, op_ret, op_errno, preparent, postparent,
+                xdata);
 
-        return 0;
+    return 0;
 }
 
-
 int
-nfs_inode_rmdir (xlator_t *nfsx, xlator_t *xl, nfs_user_t *nfu, loc_t *pathloc,
-                 fop_rmdir_cbk_t cbk, void *local)
+nfs_inode_rmdir(xlator_t *nfsx, xlator_t *xl, nfs_user_t *nfu, loc_t *pathloc,
+                fop_rmdir_cbk_t cbk, void *local)
 {
-        struct nfs_fop_local    *nfl = NULL;
-        int                     ret = -EFAULT;
+    struct nfs_fop_local *nfl = NULL;
+    int ret = -EFAULT;
 
-        if ((!nfsx) || (!xl) || (!pathloc) || (!nfu))
-                return ret;
+    if ((!nfsx) || (!xl) || (!pathloc) || (!nfu))
+        return ret;
 
-        nfs_fop_handle_local_init (NULL, nfsx, nfl, cbk, local, ret, err);
-        nfl_inodes_init (nfl, pathloc->inode, pathloc->parent, NULL,
-                         pathloc->name, NULL);
+    nfs_fop_handle_local_init(NULL, nfsx, nfl, cbk, local, ret, err);
+    nfl_inodes_init(nfl, pathloc->inode, pathloc->parent, NULL, pathloc->name,
+                    NULL);
 
-        ret = nfs_fop_rmdir (nfsx, xl, nfu, pathloc, nfs_inode_rmdir_cbk, nfl);
+    ret = nfs_fop_rmdir(nfsx, xl, nfu, pathloc, nfs_inode_rmdir_cbk, nfl);
 
 err:
-        if (ret < 0)
-                nfs_fop_local_wipe (xl, nfl);
-        return ret;
+    if (ret < 0)
+        nfs_fop_local_wipe(xl, nfl);
+    return ret;
 }
 
-
 int32_t
-nfs_inode_mknod_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                     int32_t op_ret, int32_t op_errno, inode_t *inode,
-                     struct iatt *buf, struct iatt *preparent,
-                     struct iatt *postparent, dict_t *xdata)
+nfs_inode_mknod_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                    int32_t op_ret, int32_t op_errno, inode_t *inode,
+                    struct iatt *buf, struct iatt *preparent,
+                    struct iatt *postparent, dict_t *xdata)
 {
-        struct nfs_fop_local    *nfl = NULL;
-        fop_mknod_cbk_t         progcbk = NULL;
-        inode_t                 *linked_inode = NULL;
+    struct nfs_fop_local *nfl = NULL;
+    fop_mknod_cbk_t progcbk = NULL;
+    inode_t *linked_inode = NULL;
 
-        nfl = frame->local;
+    nfl = frame->local;
 
-        if (op_ret == -1)
-                goto do_not_link;
+    if (op_ret == -1)
+        goto do_not_link;
 
-        linked_inode = inode_link (inode, nfl->parent, nfl->path, buf);
+    linked_inode = inode_link(inode, nfl->parent, nfl->path, buf);
 
 do_not_link:
-        inodes_nfl_to_prog_data (nfl, progcbk, frame);
-        if (progcbk)
-                progcbk (frame, cookie, this, op_ret, op_errno, inode, buf,
-                         preparent, postparent, xdata);
+    inodes_nfl_to_prog_data(nfl, progcbk, frame);
+    if (progcbk)
+        progcbk(frame, cookie, this, op_ret, op_errno, inode, buf, preparent,
+                postparent, xdata);
 
-        if (linked_inode) {
-                inode_lookup (linked_inode);
-                inode_unref (linked_inode);
-        }
+    if (linked_inode) {
+        inode_lookup(linked_inode);
+        inode_unref(linked_inode);
+    }
 
-        return 0;
+    return 0;
 }
 
-
 int
-nfs_inode_mknod (xlator_t *nfsx, xlator_t *xl, nfs_user_t *nfu, loc_t *pathloc,
-                 mode_t mode, dev_t dev, fop_mknod_cbk_t cbk, void *local)
+nfs_inode_mknod(xlator_t *nfsx, xlator_t *xl, nfs_user_t *nfu, loc_t *pathloc,
+                mode_t mode, dev_t dev, fop_mknod_cbk_t cbk, void *local)
 {
-        struct nfs_fop_local    *nfl = NULL;
-        int                     ret = -EFAULT;
+    struct nfs_fop_local *nfl = NULL;
+    int ret = -EFAULT;
 
-        if ((!nfsx) || (!xl) || (!pathloc) || (!nfu))
-                return ret;
+    if ((!nfsx) || (!xl) || (!pathloc) || (!nfu))
+        return ret;
 
-        nfs_fop_handle_local_init (NULL, nfsx, nfl, cbk, local, ret, err);
-        nfl_inodes_init (nfl, pathloc->inode, pathloc->parent, NULL,
-                         pathloc->name, NULL);
+    nfs_fop_handle_local_init(NULL, nfsx, nfl, cbk, local, ret, err);
+    nfl_inodes_init(nfl, pathloc->inode, pathloc->parent, NULL, pathloc->name,
+                    NULL);
 
-        ret = nfs_fop_mknod (nfsx, xl, nfu, pathloc, mode, dev,
-                             nfs_inode_mknod_cbk, nfl);
+    ret = nfs_fop_mknod(nfsx, xl, nfu, pathloc, mode, dev, nfs_inode_mknod_cbk,
+                        nfl);
 
 err:
-        if (ret < 0)
-                nfs_fop_local_wipe (xl, nfl);
+    if (ret < 0)
+        nfs_fop_local_wipe(xl, nfl);
 
-        return ret;
+    return ret;
 }
 
-
 int32_t
-nfs_inode_symlink_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                       int32_t op_ret, int32_t op_errno, inode_t *inode,
-                       struct iatt *buf, struct iatt *preparent,
-                       struct iatt *postparent, dict_t *xdata)
+nfs_inode_symlink_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                      int32_t op_ret, int32_t op_errno, inode_t *inode,
+                      struct iatt *buf, struct iatt *preparent,
+                      struct iatt *postparent, dict_t *xdata)
 {
-        struct nfs_fop_local    *nfl = NULL;
-        fop_symlink_cbk_t       progcbk = NULL;
-        inode_t                 *linked_inode = NULL;
+    struct nfs_fop_local *nfl = NULL;
+    fop_symlink_cbk_t progcbk = NULL;
+    inode_t *linked_inode = NULL;
 
-        nfl = frame->local;
-        if (op_ret == -1)
-                goto do_not_link;
+    nfl = frame->local;
+    if (op_ret == -1)
+        goto do_not_link;
 
-        linked_inode = inode_link (inode, nfl->parent, nfl->path, buf);
+    linked_inode = inode_link(inode, nfl->parent, nfl->path, buf);
 
 do_not_link:
-        inodes_nfl_to_prog_data (nfl, progcbk, frame);
-        if (progcbk)
-                progcbk (frame, cookie, this, op_ret, op_errno, inode, buf,
-                         preparent, postparent, xdata);
+    inodes_nfl_to_prog_data(nfl, progcbk, frame);
+    if (progcbk)
+        progcbk(frame, cookie, this, op_ret, op_errno, inode, buf, preparent,
+                postparent, xdata);
 
-        if (linked_inode) {
-                inode_lookup (linked_inode);
-                inode_unref (linked_inode);
-        }
+    if (linked_inode) {
+        inode_lookup(linked_inode);
+        inode_unref(linked_inode);
+    }
 
-        return 0;
+    return 0;
 }
 
-
 int
-nfs_inode_symlink (xlator_t *nfsx, xlator_t *xl, nfs_user_t *nfu, char *target,
-                   loc_t *pathloc, fop_symlink_cbk_t cbk, void *local)
+nfs_inode_symlink(xlator_t *nfsx, xlator_t *xl, nfs_user_t *nfu, char *target,
+                  loc_t *pathloc, fop_symlink_cbk_t cbk, void *local)
 {
-        struct nfs_fop_local    *nfl = NULL;
-        int                     ret = -EFAULT;
+    struct nfs_fop_local *nfl = NULL;
+    int ret = -EFAULT;
 
-        if ((!nfsx) || (!xl) || (!target) || (!pathloc) || (!nfu))
-                return ret;
+    if ((!nfsx) || (!xl) || (!target) || (!pathloc) || (!nfu))
+        return ret;
 
-        nfs_fop_handle_local_init (NULL, nfsx, nfl, cbk, local, ret, err);
-        nfl_inodes_init (nfl, pathloc->inode, pathloc->parent, NULL,
-                         pathloc->name, NULL);
-        ret = nfs_fop_symlink (nfsx, xl, nfu, target, pathloc,
-                               nfs_inode_symlink_cbk, nfl);
+    nfs_fop_handle_local_init(NULL, nfsx, nfl, cbk, local, ret, err);
+    nfl_inodes_init(nfl, pathloc->inode, pathloc->parent, NULL, pathloc->name,
+                    NULL);
+    ret = nfs_fop_symlink(nfsx, xl, nfu, target, pathloc, nfs_inode_symlink_cbk,
+                          nfl);
 
 err:
-        if (ret < 0)
-                nfs_fop_local_wipe (xl, nfl);
+    if (ret < 0)
+        nfs_fop_local_wipe(xl, nfl);
 
-        return ret;
+    return ret;
 }
 
 int32_t
-nfs_inode_opendir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                       int32_t op_ret, int32_t op_errno, fd_t *fd, dict_t *xdata)
+nfs_inode_opendir_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                      int32_t op_ret, int32_t op_errno, fd_t *fd, dict_t *xdata)
 {
+    struct nfs_fop_local *nfl = NULL;
+    fop_open_cbk_t progcbk = NULL;
 
-        struct nfs_fop_local    *nfl = NULL;
-        fop_open_cbk_t          progcbk = NULL;
+    if (op_ret != -1)
+        fd_bind(fd);
 
-        if (op_ret != -1)
-                fd_bind (fd);
+    inodes_nfl_to_prog_data(nfl, progcbk, frame);
 
-        inodes_nfl_to_prog_data (nfl, progcbk, frame);
+    if (progcbk)
+        progcbk(frame, cookie, this, op_ret, op_errno, fd, xdata);
 
-        if (progcbk)
-                progcbk (frame, cookie, this, op_ret, op_errno, fd, xdata);
-
-        return 0;
+    return 0;
 }
 
-
 int
-nfs_inode_opendir (xlator_t *nfsx, xlator_t *xl, nfs_user_t *nfu, loc_t *loc,
-                   fop_opendir_cbk_t cbk, void *local)
+nfs_inode_opendir(xlator_t *nfsx, xlator_t *xl, nfs_user_t *nfu, loc_t *loc,
+                  fop_opendir_cbk_t cbk, void *local)
 {
-        struct nfs_fop_local    *nfl = NULL;
-        fd_t                    *newfd = NULL;
-        int                     ret = -EFAULT;
+    struct nfs_fop_local *nfl = NULL;
+    fd_t *newfd = NULL;
+    int ret = -EFAULT;
 
-        if ((!nfsx) || (!xl) || (!loc) || (!nfu))
-                return ret;
+    if ((!nfsx) || (!xl) || (!loc) || (!nfu))
+        return ret;
 
-        newfd = fd_create (loc->inode, 0);
-        if (!newfd) {
-                gf_log (GF_NFS, GF_LOG_ERROR, "Failed to create fd");
-                ret = -ENOMEM;
-                goto err;
-        }
+    newfd = fd_create(loc->inode, 0);
+    if (!newfd) {
+        gf_msg(GF_NFS, GF_LOG_ERROR, ENOMEM, NFS_MSG_NO_MEMORY,
+               "Failed to create fd");
+        ret = -ENOMEM;
+        goto err;
+    }
 
-        nfs_fop_handle_local_init (NULL, nfsx, nfl, cbk, local, ret, err);
-        ret = nfs_fop_opendir (nfsx, xl, nfu, loc, newfd,
-                               nfs_inode_opendir_cbk, nfl);
+    nfs_fop_handle_local_init(NULL, nfsx, nfl, cbk, local, ret, err);
+    ret = nfs_fop_opendir(nfsx, xl, nfu, loc, newfd, nfs_inode_opendir_cbk,
+                          nfl);
 
 err:
-        if (ret < 0) {
-                if (newfd)
-                        fd_unref (newfd);
-                nfs_fop_local_wipe (xl, nfl);
-        }
+    if (ret < 0) {
+        if (newfd)
+            fd_unref(newfd);
+        nfs_fop_local_wipe(xl, nfl);
+    }
 
-        return ret;
+    return ret;
 }
diff --git a/xlators/nfs/server/src/nfs-inodes.h b/xlators/nfs/server/src/nfs-inodes.h
index ba7a5712418..e8efd1e127c 100644
--- a/xlators/nfs/server/src/nfs-inodes.h
+++ b/xlators/nfs/server/src/nfs-inodes.h
@@ -11,66 +11,57 @@
 #ifndef _NFS_INODES_H_
 #define _NFS_INODES_H_
 
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
-
-#include "dict.h"
-#include "xlator.h"
-#include "iobuf.h"
-#include "call-stub.h"
-#include "stack.h"
+#include <glusterfs/dict.h>
+#include <glusterfs/xlator.h>
+#include <glusterfs/iobuf.h>
+#include <glusterfs/call-stub.h>
 #include "nfs-fops.h"
 
-
 extern int
-nfs_link_inode (inode_t *newi, inode_t *parent, char *name,
-                struct iatt *newstat);
+nfs_link_inode(inode_t *newi, inode_t *parent, char *name,
+               struct iatt *newstat);
 
 extern int
-nfs_inode_create (xlator_t *nfsx, xlator_t *xl, nfs_user_t *nfu,
-                  loc_t *pathloc, int flags, int mode, fop_create_cbk_t cbk,
-                  void *local);
+nfs_inode_create(xlator_t *nfsx, xlator_t *xl, nfs_user_t *nfu, loc_t *pathloc,
+                 int flags, int mode, fop_create_cbk_t cbk, void *local);
 
 extern int
-nfs_inode_mkdir (xlator_t *nfsx, xlator_t *xl, nfs_user_t *nfu, loc_t *pathloc,
-                 int mode, fop_mkdir_cbk_t cbk, void *local);
+nfs_inode_mkdir(xlator_t *nfsx, xlator_t *xl, nfs_user_t *nfu, loc_t *pathloc,
+                int mode, fop_mkdir_cbk_t cbk, void *local);
 
 extern int
-nfs_inode_open (xlator_t *nfsx, xlator_t *xl, nfs_user_t *nfu, loc_t *loc,
-                int32_t flags, fop_open_cbk_t cbk,
-                void *local);
+nfs_inode_open(xlator_t *nfsx, xlator_t *xl, nfs_user_t *nfu, loc_t *loc,
+               int32_t flags, fop_open_cbk_t cbk, void *local);
 
 extern int
-nfs_inode_rename (xlator_t *nfsx, xlator_t *xl, nfs_user_t *nfu, loc_t *oldloc,
-                  loc_t *newloc, fop_rename_cbk_t cbk, void *local);
+nfs_inode_rename(xlator_t *nfsx, xlator_t *xl, nfs_user_t *nfu, loc_t *oldloc,
+                 loc_t *newloc, fop_rename_cbk_t cbk, void *local);
 
 extern int
-nfs_inode_link (xlator_t *nfsx, xlator_t *xl, nfs_user_t *nfu, loc_t *oldloc,
-                loc_t *newloc, fop_link_cbk_t cbk, void *local);
+nfs_inode_link(xlator_t *nfsx, xlator_t *xl, nfs_user_t *nfu, loc_t *oldloc,
+               loc_t *newloc, fop_link_cbk_t cbk, void *local);
 
 extern int
-nfs_inode_unlink (xlator_t *nfsx, xlator_t *xl, nfs_user_t *nfu, loc_t *pathloc,
-                  fop_unlink_cbk_t cbk, void *local);
+nfs_inode_unlink(xlator_t *nfsx, xlator_t *xl, nfs_user_t *nfu, loc_t *pathloc,
+                 fop_unlink_cbk_t cbk, void *local);
 
 extern int
-nfs_inode_rmdir (xlator_t *nfsx, xlator_t *xl, nfs_user_t *nfu, loc_t *pathloc,
-                 fop_rmdir_cbk_t cbk, void *local);
+nfs_inode_rmdir(xlator_t *nfsx, xlator_t *xl, nfs_user_t *nfu, loc_t *pathloc,
+                fop_rmdir_cbk_t cbk, void *local);
 
 extern int
-nfs_inode_symlink (xlator_t *nfsx, xlator_t *xl, nfs_user_t *nfu, char *target,
-                   loc_t *pathloc, fop_symlink_cbk_t cbk, void *local);
+nfs_inode_symlink(xlator_t *nfsx, xlator_t *xl, nfs_user_t *nfu, char *target,
+                  loc_t *pathloc, fop_symlink_cbk_t cbk, void *local);
 
 extern int
-nfs_inode_opendir (xlator_t *nfsx, xlator_t *xl, nfs_user_t *nfu, loc_t *loc,
-                   fop_opendir_cbk_t cbk, void *local);
+nfs_inode_opendir(xlator_t *nfsx, xlator_t *xl, nfs_user_t *nfu, loc_t *loc,
+                  fop_opendir_cbk_t cbk, void *local);
 
 extern int
-nfs_inode_mknod (xlator_t *nfsx, xlator_t *xl, nfs_user_t *nfu, loc_t *pathloc,
-                 mode_t mode, dev_t dev, fop_mknod_cbk_t cbk, void *local);
+nfs_inode_mknod(xlator_t *nfsx, xlator_t *xl, nfs_user_t *nfu, loc_t *pathloc,
+                mode_t mode, dev_t dev, fop_mknod_cbk_t cbk, void *local);
 
 extern int
-nfs_inode_lookup (xlator_t *nfsx, xlator_t *xl, nfs_user_t *nfu, loc_t *pathloc,
-                  fop_lookup_cbk_t cbk, void *local);
+nfs_inode_lookup(xlator_t *nfsx, xlator_t *xl, nfs_user_t *nfu, loc_t *pathloc,
+                 fop_lookup_cbk_t cbk, void *local);
 #endif
diff --git a/xlators/nfs/server/src/nfs-mem-types.h b/xlators/nfs/server/src/nfs-mem-types.h
index 88c688f74f3..0a3c887eff1 100644
--- a/xlators/nfs/server/src/nfs-mem-types.h
+++ b/xlators/nfs/server/src/nfs-mem-types.h
@@ -8,49 +8,41 @@
   cases as published by the Free Software Foundation.
 */
 
-
 #ifndef __NFS_MEM_TYPES_H__
 #define __NFS_MEM_TYPES_H__
 
-#include "mem-types.h"
+#include <glusterfs/mem-types.h>
 
 enum gf_nfs_mem_types_ {
-        gf_nfs_mt_mountentry  = gf_common_mt_end + 1,
-        gf_nfs_mt_mountbody,
-        gf_nfs_mt_nfs_state,
-        gf_nfs_mt_char,
-        gf_nfs_mt_exportnode,
-        gf_nfs_mt_groupnode,
-        gf_nfs_mt_mount3_state,
-        gf_nfs_mt_write3args,
-        gf_nfs_mt_nfs3_export,
-        gf_nfs_mt_nfs3_state,
-        gf_nfs_mt_entry3,
-        gf_nfs_mt_entryp3,
-        gf_nfs_mt_nfs3_fd_entry,
-        gf_nfs_mt_nfs3_fh,
-        gf_nfs_mt_nfs_initer_list,
-        gf_nfs_mt_xlator_t,
-        gf_nfs_mt_list_head,
-        gf_nfs_mt_mnt3_resolve,
-        gf_nfs_mt_mnt3_export,
-        gf_nfs_mt_mnt3_auth_params,
-        gf_nfs_mt_int,
-        gf_nfs_mt_mountres3,
-        gf_nfs_mt_mountstat3,
-        gf_nfs_mt_inode_q,
-        gf_nfs_mt_nlm4_state,
-        gf_nfs_mt_nlm4_cm,
-        gf_nfs_mt_nlm4_fde,
-        gf_nfs_mt_nlm4_nlmclnt,
-        gf_nfs_mt_nlm4_share,
-        gf_nfs_mt_aux_gids,
-        gf_nfs_mt_inode_ctx,
-        gf_nfs_mt_auth_spec,
-        gf_nfs_mt_arr,
-        gf_nfs_mt_auth_cache,
-        gf_nfs_mt_auth_cache_entry,
-        gf_nfs_mt_end
+    gf_nfs_mt_mountentry = gf_common_mt_end + 1,
+    gf_nfs_mt_mountbody,
+    gf_nfs_mt_nfs_state,
+    gf_nfs_mt_char,
+    gf_nfs_mt_exportnode,
+    gf_nfs_mt_groupnode,
+    gf_nfs_mt_mount3_state,
+    gf_nfs_mt_nfs3_export,
+    gf_nfs_mt_nfs3_state,
+    gf_nfs_mt_entry3,
+    gf_nfs_mt_entryp3,
+    gf_nfs_mt_nfs3_fh,
+    gf_nfs_mt_nfs_initer_list,
+    gf_nfs_mt_xlator_t,
+    gf_nfs_mt_mnt3_resolve,
+    gf_nfs_mt_mnt3_export,
+    gf_nfs_mt_mnt3_auth_params,
+    gf_nfs_mt_int,
+    gf_nfs_mt_mountres3,
+    gf_nfs_mt_mountstat3,
+    gf_nfs_mt_nlm4_fde,
+    gf_nfs_mt_nlm4_nlmclnt,
+    gf_nfs_mt_nlm4_share,
+    gf_nfs_mt_inode_ctx,
+    gf_nfs_mt_auth_spec,
+    gf_nfs_mt_arr,
+    gf_nfs_mt_auth_cache,
+    gf_nfs_mt_auth_cache_entry,
+    gf_nfs_mt_nlm4_notify,
+    gf_nfs_mt_end
 };
 #endif
-
diff --git a/xlators/nfs/server/src/nfs-messages.h b/xlators/nfs/server/src/nfs-messages.h
index 9c634b22f55..04e15cbe78c 100644
--- a/xlators/nfs/server/src/nfs-messages.h
+++ b/xlators/nfs/server/src/nfs-messages.h
@@ -10,1656 +10,93 @@
 #ifndef _NFS_MESSAGES_H_
 #define _NFS_MESSAGES_H_
 
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
-
-#include "glfs-message-id.h"
-
-/*! \file nfs-messages.h
- *  \brief NFS log-message IDs and their descriptions
- *
- */
-
-/* NOTE: Rules for message additions
- * 1) Each instance of a message is _better_ left with a unique message ID, even
- *    if the message format is the same. Reasoning is that, if the message
- *    format needs to change in one instance, the other instances are not
- *    impacted or the new change does not change the ID of the instance being
- *    modified.
- * 2) Addition of a message,
- *       - Should increment the GLFS_NUM_MESSAGES
- *       - Append to the list of messages defined, towards the end
- *       - Retain macro naming as glfs_msg_X (for redability across developers)
- * NOTE: Rules for message format modifications
- * 3) Check acorss the code if the message ID macro in question is reused
- *    anywhere. If reused then then the modifications should ensure correctness
- *    everywhere, or needs a new message ID as (1) above was not adhered to. If
- *    not used anywhere, proceed with the required modification.
- * NOTE: Rules for message deletion
- * 4) Check (3) and if used anywhere else, then cannot be deleted. If not used
- *    anywhere, then can be deleted, but will leave a hole by design, as
- *    addition rules specify modification to the end of the list and not filling
- *    holes.
- */
-
-#define GLFS_NFS_BASE                   GLFS_MSGID_COMP_NFS
-#define GLFS_NFS_NUM_MESSAGES           201
-#define GLFS_MSGID_END                  (GLFS_NFS_BASE + GLFS_NFS_NUM_MESSAGES + 1)
-
-/* Messages with message IDs */
-#define glfs_msg_start_x GLFS_NFS_BASE, "Invalid: Start of messages"
-
-/*------------*/
-
-#define NFS_MSG_SVC_LOOKUP               (GLFS_NFS_BASE + 1)
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-
-#define NFS_MSG_SVC_PAR_LOOKUP          (GLFS_NFS_BASE + 2)
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-
-#define NFS_MSG_INVALID_ENTRY           (GLFS_NFS_BASE + 3)
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-
-#define NFS_MSG_INODE_LOC_FILL_ERROR    (GLFS_NFS_BASE + 4)
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-
-#define NFS_MSG_HARD_RESOLVE_FAIL       (GLFS_NFS_BASE + 5)
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-
-#define NFS_MSG_ARGS_DECODE_ERROR       (GLFS_NFS_BASE + 6)
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-
-#define NFS_MSG_LOOKUP_PROC_FAIL        (GLFS_NFS_BASE + 7)
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-
-#define NFS_MSG_ACCESS_CBK              (GLFS_NFS_BASE + 8)
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-
-#define NFS_MSG_READLINK_CBK            (GLFS_NFS_BASE + 9)
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-
-#define NFS_MSG_READLINK_PROC_FAIL      (GLFS_NFS_BASE + 10)
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-
-#define NFS_MSG_READ_CBK                (GLFS_NFS_BASE + 11)
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-
-#define NFS_MSG_ANONYMOUS_FD_FAIL       (GLFS_NFS_BASE + 12)
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define NFS_MSG_READ_FAIL               (GLFS_NFS_BASE + 13)
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-
-#define NFS_MSG_FSYNC_CBK               (GLFS_NFS_BASE + 14)
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-
-#define NFS_MSG_WRITE_CBK               (GLFS_NFS_BASE + 15)
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-
-#define NFS_MSG_STATE_WRONG             (GLFS_NFS_BASE + 16)
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-
-#define NFS_MSG_WRITE_FAIL              (GLFS_NFS_BASE + 17)
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-
-#define NFS_MSG_CREATE_SETATTR_CBK      (GLFS_NFS_BASE + 18)
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-
-#define NFS_MSG_CREATE_CBK              (GLFS_NFS_BASE + 19)
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-
-#define NFS_MSG_CREATE_STAT_CBK         (GLFS_NFS_BASE + 20)
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-
-
-#define NFS_MSG_CREATE_FAIL             (GLFS_NFS_BASE + 21)
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define NFS_MSG_MKDIR_SETATTR_CBK       (GLFS_NFS_BASE + 22)
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-
-
-#define NFS_MSG_MKDIR_CBK               (GLFS_NFS_BASE + 23)
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-
-#define NFS_MSG_DIR_OP_FAIL             (GLFS_NFS_BASE + 24)
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-
-#define NFS_MSG_SYMLINK_CBK             (GLFS_NFS_BASE + 25)
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-
-#define NFS_MSG_SYMLINK_FAIL           (GLFS_NFS_BASE + 26)
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-
-#define NFS_MSG_MKNOD_CBK               (GLFS_NFS_BASE + 27)
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-
-#define NFS_MSG_MKNOD_FAIL              (GLFS_NFS_BASE + 28)
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-
-#define NFS_MSG_OPT_INIT_FAIL           (GLFS_NFS_BASE + 29)
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-
-
-#define NFS_MSG_REMOVE_CBK              (GLFS_NFS_BASE + 30)
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-
-#define NFS_MSG_REMOVE_FAIL             (GLFS_NFS_BASE + 31)
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-
-#define NFS_MSG_RMDIR_CBK               (GLFS_NFS_BASE + 32)
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-
-#define NFS_MSG_RENAME_CBK              (GLFS_NFS_BASE + 33)
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-
-#define NFS_MSG_RENAME_FAIL             (GLFS_NFS_BASE + 34)
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-
-#define NFS_MSG_LINK_CBK                (GLFS_NFS_BASE + 35)
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-
-#define NFS_MSG_LINK_FAIL               (GLFS_NFS_BASE + 36)
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-
-#define NFS_MSG_READDIR_FSTAT_CBK       (GLFS_NFS_BASE + 37)
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-
-#define NFS_MSG_READDIR_CBK             (GLFS_NFS_BASE + 38)
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-
-#define NFS_MSG_READDIR_FAIL            (GLFS_NFS_BASE + 39)
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-
-#define NFS_MSG_READDIRP_FAIL           (GLFS_NFS_BASE + 40)
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-
-#define NFS_MSG_FSSTAT_STAT_CBK         (GLFS_NFS_BASE + 41)
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-
-#define NFS_MSG_FSSTAT_STATFS_CBK       (GLFS_NFS_BASE + 42)
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define NFS_MSG_FSTAT_FAIL              (GLFS_NFS_BASE + 43)
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-
-#define NFS_MSG_FSINFO_CBK              (GLFS_NFS_BASE + 44)
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-
-#define NFS_MSG_FSINFO_FAIL             (GLFS_NFS_BASE + 45)
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-
-#define NFS_MSG_PATHCONF_CBK            (GLFS_NFS_BASE + 46)
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-
-#define NFS_MSG_PATHCONF_FAIL           (GLFS_NFS_BASE + 47)
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-
-#define NFS_MSG_COMMIT_CBK              (GLFS_NFS_BASE + 48)
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-
-#define NFS_MSG_COMMIT_FAIL              (GLFS_NFS_BASE + 49)
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-
-#define NFS_MSG_PROT_INIT_ADD_FAIL      (GLFS_NFS_BASE + 50)
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define NFS_MSG_FORMAT_FAIL             (GLFS_NFS_BASE + 51)
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-
-#define NFS_MSG_SNPRINTF_FAIL           (GLFS_NFS_BASE + 52)
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-
-#define NFS_MSG_VOLID_MISSING           (GLFS_NFS_BASE + 53)
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-
-#define NFS_MSG_PARSE_VOL_UUID_FAIL     (GLFS_NFS_BASE + 54)
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define NFS_MSG_STR2BOOL_FAIL            (GLFS_NFS_BASE + 55)
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-
-#define NFS_MSG_SUBVOL_INIT_FAIL        (GLFS_NFS_BASE + 56)
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-
-#define NFS_MSG_NO_MEMORY               (GLFS_NFS_BASE + 57)
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define NFS_MSG_LISTENERS_CREATE_FAIL   (GLFS_NFS_BASE + 58)
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-
-#define NFS_MSG_STATE_INIT_FAIL         (GLFS_NFS_BASE + 59)
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-
-#define NFS_MSG_RECONF_FAIL             (GLFS_NFS_BASE + 60)
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define NFS_MSG_RECONF_SUBVOL_FAIL      (GLFS_NFS_BASE + 61)
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-
-#define NFS_MSG_STR_TOO_LONG            (GLFS_NFS_BASE + 62)
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-
-#define NFS_MSG_STATE_MISSING           (GLFS_NFS_BASE + 63)
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-
-#define NFS_MSG_INDEX_NOT_FOUND         (GLFS_NFS_BASE + 64)
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-
-#define NFS_MSG_EXPORT_ID_FAIL          (GLFS_NFS_BASE + 65)
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-
-#define NFS_MSG_NO_RW_ACCESS            (GLFS_NFS_BASE + 66)
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define NFS_MSG_BAD_HANDLE              (GLFS_NFS_BASE + 67)
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define NFS_MSG_RESOLVE_FH_FAIL         (GLFS_NFS_BASE + 68)
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define NFS_MSG_RESOLVE_STAT            (GLFS_NFS_BASE + 69)
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-#define NFS_MSG_VOL_DISABLE             (GLFS_NFS_BASE + 70)
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-
-#define NFS_MSG_INIT_CALL_STAT_FAIL     (GLFS_NFS_BASE + 71)
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-
-#define NFS_MSG_ENCODE_FAIL             (GLFS_NFS_BASE + 72)
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-
-#define NFS_MSG_SERIALIZE_REPLY_FAIL    (GLFS_NFS_BASE + 73)
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-
-#define NFS_MSG_SUBMIT_REPLY_FAIL       (GLFS_NFS_BASE + 74)
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-
-#define NFS_MSG_GETATTR_LOOKUP_CBK      (GLFS_NFS_BASE + 75)
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-
-#define NFS_MSG_GETATTR_STAT_CBK        (GLFS_NFS_BASE + 76)
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-
-#define NFS_MSG_STAT_FOP_FAIL           (GLFS_NFS_BASE + 77)
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-
-#define NFS_MSG_GETATTR_FAIL            (GLFS_NFS_BASE + 78)
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-
-#define NFS_MSG_TRUNCATE_CBK            (GLFS_NFS_BASE + 79)
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-
-#define NFS_MSG_SETATTR_STAT_CBK        (GLFS_NFS_BASE + 80)
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-
-#define NFS_MSG_TIMESTAMP_NO_SYNC       (GLFS_NFS_BASE + 81)
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-
-#define NFS_MSG_SETATTR_INVALID         (GLFS_NFS_BASE + 82)
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-
-#define NFS_MSG_SETATTR_FAIL            (GLFS_NFS_BASE + 83)
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-
-#define NFS_MSG_SETATTR_CBK             (GLFS_NFS_BASE + 84)
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-
-#define NFS_MSG_ACCESS_PROC_FAIL        (GLFS_NFS_BASE + 85)
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-
-#define NFS_MSG_PGM_NOT_FOUND           (GLFS_NFS_BASE + 86)
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-
-#define NFS_MSG_PGM_INIT_FAIL           (GLFS_NFS_BASE + 87)
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-
-#define NFS_MSG_PGM_REG_FAIL            (GLFS_NFS_BASE + 88)
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-
-#define NFS_MSG_LOOKUP_ROOT_FAIL        (GLFS_NFS_BASE + 89)
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-
-#define NFS_MSG_ROOT_LOC_INIT_FAIL      (GLFS_NFS_BASE + 90)
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-
-#define NFS_MSG_STARTUP_FAIL            (GLFS_NFS_BASE + 91)
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-
-#define NFS_MSG_XLATOR_INIT_FAIL        (GLFS_NFS_BASE + 92)
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-
-#define NFS_MSG_NFS_MAN_DISABLE         (GLFS_NFS_BASE + 93)
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-
-#define NFS_MSG_DICT_GET_FAILED         (GLFS_NFS_BASE + 94)
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-
-#define NFS_MSG_PARSE_FAIL              (GLFS_NFS_BASE + 95)
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-
-#define NFS_MSG_NLM_MAN_DISABLE         (GLFS_NFS_BASE + 96)
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-
-#define NFS_MSG_ACL_MAN_DISABLE         (GLFS_NFS_BASE + 97)
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-
-#define NFS_MSG_DICT_SET_FAILED         (GLFS_NFS_BASE + 98)
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-
-#define NFS_MSG_INIT_GRP_CACHE_FAIL     (GLFS_NFS_BASE + 99)
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-
-#define NFS_MSG_NO_PERM                 (GLFS_NFS_BASE + 100)
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-
-#define NFS_MSG_REG_FILE_ERROR          (GLFS_NFS_BASE + 101)
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-
-#define NFS_MSG_RPC_INIT_FAIL           (GLFS_NFS_BASE + 102)
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-
-#define NFS_MSG_RPC_CONFIG_FAIL         (GLFS_NFS_BASE + 103)
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-
-#define NFS_MSG_RECONFIG_PATH           (GLFS_NFS_BASE + 104)
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-
-#define NFS_MSG_RECONFIG_VALUE          (GLFS_NFS_BASE + 105)
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-
-#define NFS_MSG_RECONFIG_VOL            (GLFS_NFS_BASE + 106)
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-
-#define NFS_MSG_NLM_INFO                (GLFS_NFS_BASE + 107)
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-
-#define NFS_MSG_ACL_INFO                (GLFS_NFS_BASE + 108)
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-
-#define NFS_MSG_INIT_FAIL               (GLFS_NFS_BASE + 109)
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-
-#define NFS_MSG_STARTED                 (GLFS_NFS_BASE + 110)
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-
-#define NFS_MSG_VOL_NOT_FOUND           (GLFS_NFS_BASE + 111)
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-
-#define NFS_MSG_RECONFIG_ENABLE         (GLFS_NFS_BASE + 112)
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-
-#define NFS_MSG_RECONFIG_FAIL           (GLFS_NFS_BASE + 113)
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-
-#define NFS_MSG_MNT_STATE_NOT_FOUND     (GLFS_NFS_BASE + 114)
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-
-#define NFS_MSG_ENCODE_MSG_FAIL         (GLFS_NFS_BASE + 115)
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-
-#define NFS_MSG_REP_SUBMIT_FAIL         (GLFS_NFS_BASE + 116)
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-
-#define NFS_MSG_READ_LOCKED             (GLFS_NFS_BASE + 117)
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-
-#define NFS_MSG_MODIFY_LOCKED           (GLFS_NFS_BASE + 118)
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-
-#define NFS_MSG_RWTAB_OVERWRITE_FAIL    (GLFS_NFS_BASE + 119)
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-
-#define NFS_MSG_UPDATE_FAIL             (GLFS_NFS_BASE + 120)
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-
-#define NFS_MSG_OPEN_FAIL               (GLFS_NFS_BASE + 121)
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-
-#define NFS_MSG_LOCK_FAIL               (GLFS_NFS_BASE + 122)
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-
-#define NFS_MSG_REWRITE_ERROR           (GLFS_NFS_BASE + 123)
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-
-#define NFS_MSG_HASH_PATH_FAIL          (GLFS_NFS_BASE + 124)
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-
-#define NFS_MSG_LOOKUP_MNT_ERROR        (GLFS_NFS_BASE + 125)
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-
-#define NFS_MSG_GET_ROOT_INODE_FAIL     (GLFS_NFS_BASE + 126)
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-
-#define NFS_MSG_RESOLVE_INODE_FAIL      (GLFS_NFS_BASE + 127)
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-
-#define NFS_MSG_RESOLVE_SUBDIR_FAIL     (GLFS_NFS_BASE + 128)
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-
-#define NFS_MSG_RESOLVE_SYMLINK_ERROR   (GLFS_NFS_BASE + 129)
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-
-#define NFS_MSG_RESOLVE_ERROR           (GLFS_NFS_BASE + 130)
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-
-#define NFS_MSG_UNSUPPORTED_VERSION     (GLFS_NFS_BASE + 131)
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-
-#define NFS_MSG_AUTH_VERIFY_FAILED      (GLFS_NFS_BASE + 132)
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-
-#define NFS_MSG_PEER_NOT_ALLOWED        (GLFS_NFS_BASE + 133)
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-
-
-#define NFS_MSG_GET_PEER_ADDR_FAIL      (GLFS_NFS_BASE + 134)
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-
-#define NFS_MSG_BAD_PEER                (GLFS_NFS_BASE + 135)
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-
-#define NFS_MSG_PEER_TOO_LONG            (GLFS_NFS_BASE + 136)
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-
-#define NFS_MSG_CALLER_NOT_FOUND        (GLFS_NFS_BASE + 137)
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-
-#define NFS_MSG_GET_REMOTE_NAME_FAIL    (GLFS_NFS_BASE + 138)
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-
-#define NFS_MSG_UNKNOWN_MNT_TYPE        (GLFS_NFS_BASE + 139)
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-
-#define NFS_MSG_PARSE_HOSTSPEC_FAIL     (GLFS_NFS_BASE + 140)
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-
-#define NFS_MSG_PARSE_AUTH_PARAM_FAIL   (GLFS_NFS_BASE + 141)
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-
-#define NFS_MSG_SET_EXP_FAIL            (GLFS_NFS_BASE + 142)
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-
-#define NFS_MSG_INIT_DIR_EXP_FAIL       (GLFS_NFS_BASE + 143)
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-
-#define NFS_MSG_DIR_EXP_SETUP_FAIL      (GLFS_NFS_BASE + 144)
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-
-#define NFS_MSG_VOL_INIT_FAIL           (GLFS_NFS_BASE + 145)
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-
-#define NFS_MSG_AUTH_ERROR              (GLFS_NFS_BASE + 146)
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-
-#define NFS_MSG_UPDATING_EXP            (GLFS_NFS_BASE + 147)
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-
-#define NFS_MSG_SET_EXP_AUTH_PARAM_FAIL (GLFS_NFS_BASE + 148)
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-
-#define NFS_MSG_UPDATING_NET_GRP        (GLFS_NFS_BASE + 149)
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-
-#define NFS_MSG_SET_NET_GRP_FAIL        (GLFS_NFS_BASE + 150)
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-
-#define NFS_MSG_PURGING_AUTH_CACHE      (GLFS_NFS_BASE + 151)
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-
-#define NFS_MSG_MNT_STATE_INIT_FAIL     (GLFS_NFS_BASE + 152)
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-
-#define NFS_MSG_EXP_AUTH_DISABLED       (GLFS_NFS_BASE + 153)
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-
-#define NFS_MSG_FH_TO_VOL_FAIL          (GLFS_NFS_BASE + 154)
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-
-#define NFS_MSG_INODE_SHARES_NOT_FOUND  (GLFS_NFS_BASE + 155)
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-
-#define NFS_MSG_VOLUME_ERROR            (GLFS_NFS_BASE + 156)
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-
-#define NFS_MSG_GET_USER_ACL_FAIL       (GLFS_NFS_BASE + 157)
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-
-#define NFS_MSG_GET_DEF_ACL_FAIL        (GLFS_NFS_BASE + 158)
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-
-#define NFS_MSG_SET_USER_ACL_FAIL       (GLFS_NFS_BASE + 159)
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-
-#define NFS_MSG_SET_DEF_ACL_FAIL        (GLFS_NFS_BASE + 160)
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-
-#define NFS_MSG_ACL_INIT_FAIL           (GLFS_NFS_BASE + 161)
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-
-#define NFS_MSG_LOAD_PARSE_ERROR        (GLFS_NFS_BASE + 162)
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-
-#define NFS_MSG_CLNT_CALL_ERROR         (GLFS_NFS_BASE + 163)
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-
-#define NFS_MSG_CLNT_CREATE_ERROR       (GLFS_NFS_BASE + 164)
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-
-#define NFS_MSG_NLM_GRACE_PERIOD        (GLFS_NFS_BASE + 165)
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-
-#define NFS_MSG_RPC_CLNT_ERROR          (GLFS_NFS_BASE + 166)
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-
-#define NFS_MSG_GET_PORT_ERROR          (GLFS_NFS_BASE + 167)
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-
-#define NFS_MSG_NLMCLNT_NOT_FOUND       (GLFS_NFS_BASE + 168)
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-
-#define NFS_MSG_FD_LOOKUP_NULL          (GLFS_NFS_BASE + 169)
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-
-#define NFS_MSG_SM_NOTIFY               (GLFS_NFS_BASE + 170)
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-
-#define NFS_MSG_NLM_INIT_FAIL           (GLFS_NFS_BASE + 171)
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-
-#define NFS_MSG_START_ERROR             (GLFS_NFS_BASE + 172)
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-
-#define NFS_MSG_UNLINK_ERROR            (GLFS_NFS_BASE + 173)
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-
-#define NFS_MSG_SHARE_LIST_STORE_FAIL   (GLFS_NFS_BASE + 174)
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-
-#define NFS_MSG_CLIENT_NOT_FOUND        (GLFS_NFS_BASE + 175)
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-
-#define NFS_MSG_SHARE_CALL_FAIL         (GLFS_NFS_BASE + 176)
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-
-#define NFS_MSG_UNSHARE_CALL_FAIL       (GLFS_NFS_BASE + 177)
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-
-#define NFS_MSG_GET_PID_FAIL            (GLFS_NFS_BASE + 178)
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-
-#define NFS_MSG_ARG_FREE_FAIL           (GLFS_NFS_BASE + 179)
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-
-#define NFS_MSG_PMAP_UNSET_FAIL         (GLFS_NFS_BASE + 180)
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-
-#define NFS_MSG_UDP_SERV_FAIL           (GLFS_NFS_BASE + 181)
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-
-#define NFS_MSG_REG_NLMCBK_FAIL         (GLFS_NFS_BASE + 182)
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-
-#define NFS_MSG_TCP_SERV_FAIL           (GLFS_NFS_BASE + 183)
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-
-#define NFS_MSG_SVC_RUN_RETURNED        (GLFS_NFS_BASE + 184)
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-
-#define NFS_MSG_XLATOR_SET_FAIL         (GLFS_NFS_BASE + 185)
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-
-#define NFS_MSG_SVC_ERROR               (GLFS_NFS_BASE + 186)
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-
-#define NFS_MSG_GET_FH_FAIL             (GLFS_NFS_BASE + 187)
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-
-#define NFS_MSG_FIND_FIRST_MATCH_FAIL   (GLFS_NFS_BASE + 188)
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-
-#define NFS_MSG_NETGRP_NOT_FOUND        (GLFS_NFS_BASE + 189)
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-
-#define NFS_MSG_FILE_OP_FAILED          (GLFS_NFS_BASE + 190)
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-
-#define NFS_MSG_PATH_RESOLVE_FAIL       (GLFS_NFS_BASE + 191)
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-
-#define NFS_MSG_LOC_FILL_RESOLVE_FAIL   (GLFS_NFS_BASE + 192)
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-
-#define NFS_MSG_INODE_NOT_FOUND         (GLFS_NFS_BASE + 193)
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-
-#define NFS_MSG_INODE_CTX_STORE_FAIL    (GLFS_NFS_BASE + 194)
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-
-#define NFS_MSG_GETPWUID_FAIL           (GLFS_NFS_BASE + 195)
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-
-#define NFS_MSG_MAP_GRP_LIST_FAIL       (GLFS_NFS_BASE + 196)
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-
-#define NFS_MSG_PARSE_DIR_FAIL          (GLFS_NFS_BASE + 197)
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-
-#define NFS_MSG_LOOKUP_FAIL             (GLFS_NFS_BASE + 198)
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-
-#define NFS_MSG_STAT_ERROR              (GLFS_NFS_BASE + 199)
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-
-#define NFS_MSG_GFID_DICT_CREATE_FAIL   (GLFS_NFS_BASE + 200)
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-
-#define NFS_MSG_HASH_XLATOR_FAIL        (GLFS_NFS_BASE + 201)
-/*!
- * @messageid
- * @diagnosis
- * @recommendedaction
- *
- */
-
-/*------------*/
-#define glfs_msg_end_x GLFS_MSGID_END, "Invalid: End of messages"
-
+#include <glusterfs/glfs-message-id.h>
+
+/* To add new message IDs, append new identifiers at the end of the list.
+ *
+ * Never remove a message ID. If it's not used anymore, you can rename it or
+ * leave it as it is, but not delete it. This is to prevent reutilization of
+ * IDs by other messages.
+ *
+ * The component name must match one of the entries defined in
+ * glfs-message-id.h.
+ */
+
+GLFS_MSGID(
+    NFS, NFS_MSG_UNUSED_1, NFS_MSG_UNUSED_2, NFS_MSG_INVALID_ENTRY,
+    NFS_MSG_INODE_LOC_FILL_ERROR, NFS_MSG_HARD_RESOLVE_FAIL,
+    NFS_MSG_ARGS_DECODE_ERROR, NFS_MSG_LOOKUP_PROC_FAIL, NFS_MSG_UNUSED_8,
+    NFS_MSG_UNUSED_9, NFS_MSG_READLINK_PROC_FAIL, NFS_MSG_UNUSED_11,
+    NFS_MSG_ANONYMOUS_FD_FAIL, NFS_MSG_READ_FAIL, NFS_MSG_UNUSED_14,
+    NFS_MSG_UNUSED_15, NFS_MSG_STATE_WRONG, NFS_MSG_WRITE_FAIL,
+    NFS_MSG_UNUSED_18, NFS_MSG_UNUSED_19, NFS_MSG_UNUSED_20,
+    NFS_MSG_CREATE_FAIL, NFS_MSG_UNUSED_22, NFS_MSG_UNUSED_23,
+    NFS_MSG_DIR_OP_FAIL, NFS_MSG_UNUSED_25, NFS_MSG_SYMLINK_FAIL,
+    NFS_MSG_UNUSED_27, NFS_MSG_MKNOD_FAIL, NFS_MSG_OPT_INIT_FAIL,
+    NFS_MSG_UNUSED_30, NFS_MSG_REMOVE_FAIL, NFS_MSG_RMDIR_CBK,
+    NFS_MSG_UNUSED_33, NFS_MSG_RENAME_FAIL, NFS_MSG_UNUSED_35,
+    NFS_MSG_LINK_FAIL, NFS_MSG_UNUSED_37, NFS_MSG_UNUSED_38,
+    NFS_MSG_READDIR_FAIL, NFS_MSG_READDIRP_FAIL, NFS_MSG_UNUSED_41,
+    NFS_MSG_UNUSED_42, NFS_MSG_FSTAT_FAIL, NFS_MSG_UNUSED_44,
+    NFS_MSG_FSINFO_FAIL, NFS_MSG_UNUSED_46, NFS_MSG_PATHCONF_FAIL,
+    NFS_MSG_UNUSED_48, NFS_MSG_COMMIT_FAIL, NFS_MSG_PROT_INIT_ADD_FAIL,
+    NFS_MSG_FORMAT_FAIL, NFS_MSG_SNPRINTF_FAIL, NFS_MSG_VOLID_MISSING,
+    NFS_MSG_PARSE_VOL_UUID_FAIL, NFS_MSG_STR2BOOL_FAIL,
+    NFS_MSG_SUBVOL_INIT_FAIL, NFS_MSG_NO_MEMORY, NFS_MSG_LISTENERS_CREATE_FAIL,
+    NFS_MSG_STATE_INIT_FAIL, NFS_MSG_RECONF_FAIL, NFS_MSG_RECONF_SUBVOL_FAIL,
+    NFS_MSG_STR_TOO_LONG, NFS_MSG_STATE_MISSING, NFS_MSG_INDEX_NOT_FOUND,
+    NFS_MSG_EXPORT_ID_FAIL, NFS_MSG_NO_RW_ACCESS, NFS_MSG_BAD_HANDLE,
+    NFS_MSG_RESOLVE_FH_FAIL, NFS_MSG_RESOLVE_STAT, NFS_MSG_VOL_DISABLE,
+    NFS_MSG_INIT_CALL_STAT_FAIL, NFS_MSG_ENCODE_FAIL,
+    NFS_MSG_SERIALIZE_REPLY_FAIL, NFS_MSG_SUBMIT_REPLY_FAIL, NFS_MSG_UNUSED_75,
+    NFS_MSG_UNUSED_76, NFS_MSG_STAT_FOP_FAIL, NFS_MSG_GETATTR_FAIL,
+    NFS_MSG_UNUSED_79, NFS_MSG_UNUSED_80, NFS_MSG_TIMESTAMP_NO_SYNC,
+    NFS_MSG_SETATTR_INVALID, NFS_MSG_SETATTR_FAIL, NFS_MSG_UNUSED_84,
+    NFS_MSG_ACCESS_PROC_FAIL, NFS_MSG_PGM_NOT_FOUND, NFS_MSG_PGM_INIT_FAIL,
+    NFS_MSG_PGM_REG_FAIL, NFS_MSG_LOOKUP_ROOT_FAIL, NFS_MSG_ROOT_LOC_INIT_FAIL,
+    NFS_MSG_STARTUP_FAIL, NFS_MSG_XLATOR_INIT_FAIL, NFS_MSG_NFS_MAN_DISABLE,
+    NFS_MSG_DICT_GET_FAILED, NFS_MSG_PARSE_FAIL, NFS_MSG_NLM_MAN_DISABLE,
+    NFS_MSG_ACL_MAN_DISABLE, NFS_MSG_DICT_SET_FAILED,
+    NFS_MSG_INIT_GRP_CACHE_FAIL, NFS_MSG_NO_PERM, NFS_MSG_REG_FILE_ERROR,
+    NFS_MSG_RPC_INIT_FAIL, NFS_MSG_RPC_CONFIG_FAIL, NFS_MSG_RECONFIG_PATH,
+    NFS_MSG_RECONFIG_VALUE, NFS_MSG_RECONFIG_VOL, NFS_MSG_NLM_INFO,
+    NFS_MSG_ACL_INFO, NFS_MSG_INIT_FAIL, NFS_MSG_STARTED, NFS_MSG_VOL_NOT_FOUND,
+    NFS_MSG_RECONFIG_ENABLE, NFS_MSG_RECONFIG_FAIL, NFS_MSG_MNT_STATE_NOT_FOUND,
+    NFS_MSG_ENCODE_MSG_FAIL, NFS_MSG_REP_SUBMIT_FAIL, NFS_MSG_READ_LOCKED,
+    NFS_MSG_MODIFY_LOCKED, NFS_MSG_RWTAB_OVERWRITE_FAIL, NFS_MSG_UPDATE_FAIL,
+    NFS_MSG_OPEN_FAIL, NFS_MSG_LOCK_FAIL, NFS_MSG_REWRITE_ERROR,
+    NFS_MSG_HASH_PATH_FAIL, NFS_MSG_LOOKUP_MNT_ERROR,
+    NFS_MSG_GET_ROOT_INODE_FAIL, NFS_MSG_RESOLVE_INODE_FAIL,
+    NFS_MSG_RESOLVE_SUBDIR_FAIL, NFS_MSG_RESOLVE_SYMLINK_ERROR,
+    NFS_MSG_RESOLVE_ERROR, NFS_MSG_UNSUPPORTED_VERSION,
+    NFS_MSG_AUTH_VERIFY_FAILED, NFS_MSG_PEER_NOT_ALLOWED,
+    NFS_MSG_GET_PEER_ADDR_FAIL, NFS_MSG_BAD_PEER, NFS_MSG_PEER_TOO_LONG,
+    NFS_MSG_CALLER_NOT_FOUND, NFS_MSG_GET_REMOTE_NAME_FAIL,
+    NFS_MSG_UNKNOWN_MNT_TYPE, NFS_MSG_PARSE_HOSTSPEC_FAIL,
+    NFS_MSG_PARSE_AUTH_PARAM_FAIL, NFS_MSG_SET_EXP_FAIL,
+    NFS_MSG_INIT_DIR_EXP_FAIL, NFS_MSG_DIR_EXP_SETUP_FAIL,
+    NFS_MSG_VOL_INIT_FAIL, NFS_MSG_AUTH_ERROR, NFS_MSG_UPDATING_EXP,
+    NFS_MSG_SET_EXP_AUTH_PARAM_FAIL, NFS_MSG_UPDATING_NET_GRP,
+    NFS_MSG_SET_NET_GRP_FAIL, NFS_MSG_PURGING_AUTH_CACHE,
+    NFS_MSG_MNT_STATE_INIT_FAIL, NFS_MSG_EXP_AUTH_DISABLED,
+    NFS_MSG_FH_TO_VOL_FAIL, NFS_MSG_INODE_SHARES_NOT_FOUND,
+    NFS_MSG_VOLUME_ERROR, NFS_MSG_GET_USER_ACL_FAIL, NFS_MSG_GET_DEF_ACL_FAIL,
+    NFS_MSG_SET_USER_ACL_FAIL, NFS_MSG_SET_DEF_ACL_FAIL, NFS_MSG_ACL_INIT_FAIL,
+    NFS_MSG_LOAD_PARSE_ERROR, NFS_MSG_CLNT_CALL_ERROR,
+    NFS_MSG_CLNT_CREATE_ERROR, NFS_MSG_NLM_GRACE_PERIOD, NFS_MSG_RPC_CLNT_ERROR,
+    NFS_MSG_GET_PORT_ERROR, NFS_MSG_NLMCLNT_NOT_FOUND, NFS_MSG_FD_LOOKUP_NULL,
+    NFS_MSG_SM_NOTIFY, NFS_MSG_NLM_INIT_FAIL, NFS_MSG_START_ERROR,
+    NFS_MSG_UNLINK_ERROR, NFS_MSG_SHARE_LIST_STORE_FAIL,
+    NFS_MSG_CLIENT_NOT_FOUND, NFS_MSG_SHARE_CALL_FAIL,
+    NFS_MSG_UNSHARE_CALL_FAIL, NFS_MSG_GET_PID_FAIL, NFS_MSG_ARG_FREE_FAIL,
+    NFS_MSG_PMAP_UNSET_FAIL, NFS_MSG_UDP_SERV_FAIL, NFS_MSG_REG_NLMCBK_FAIL,
+    NFS_MSG_TCP_SERV_FAIL, NFS_MSG_SVC_RUN_RETURNED, NFS_MSG_XLATOR_SET_FAIL,
+    NFS_MSG_SVC_ERROR, NFS_MSG_GET_FH_FAIL, NFS_MSG_FIND_FIRST_MATCH_FAIL,
+    NFS_MSG_NETGRP_NOT_FOUND, NFS_MSG_FILE_OP_FAILED, NFS_MSG_PATH_RESOLVE_FAIL,
+    NFS_MSG_LOC_FILL_RESOLVE_FAIL, NFS_MSG_INODE_NOT_FOUND,
+    NFS_MSG_INODE_CTX_STORE_FAIL, NFS_MSG_GETPWUID_FAIL,
+    NFS_MSG_MAP_GRP_LIST_FAIL, NFS_MSG_PARSE_DIR_FAIL, NFS_MSG_LOOKUP_FAIL,
+    NFS_MSG_STAT_ERROR, NFS_MSG_GFID_DICT_CREATE_FAIL, NFS_MSG_HASH_XLATOR_FAIL,
+    NFS_MSG_ENABLE_THROTTLE_FAIL);
 
 #endif /* _NFS_MESSAGES_H_ */
-
-
-
-
diff --git a/xlators/nfs/server/src/nfs.c b/xlators/nfs/server/src/nfs.c
index 965aec7daa9..39b73f88ac3 100644
--- a/xlators/nfs/server/src/nfs.c
+++ b/xlators/nfs/server/src/nfs.c
@@ -12,1340 +12,1414 @@
  * Every other protocol version gets initialized from here.
  */
 
-
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
-
-#include "defaults.h"
+#include <glusterfs/defaults.h>
 #include "rpcsvc.h"
-#include "dict.h"
-#include "xlator.h"
+#include <glusterfs/dict.h>
+#include <glusterfs/xlator.h>
 #include "nfs.h"
-#include "mem-pool.h"
-#include "logging.h"
+#include <glusterfs/mem-pool.h>
+#include <glusterfs/logging.h>
 #include "nfs-fops.h"
-#include "inode.h"
 #include "mount3.h"
 #include "nfs3.h"
 #include "nfs-mem-types.h"
 #include "nfs3-helpers.h"
 #include "nlm4.h"
-#include "options.h"
+#include <glusterfs/options.h>
 #include "acl3.h"
 #include "rpc-drc.h"
-#include "syscall.h"
+#include <glusterfs/syscall.h>
+#include "rpcsvc.h"
+#include "nfs-messages.h"
+#include "glusterfs/statedump.h"
 
-#define OPT_SERVER_AUX_GIDS             "nfs.server-aux-gids"
-#define OPT_SERVER_GID_CACHE_TIMEOUT    "nfs.server.aux-gid-timeout"
-#define OPT_SERVER_RPC_STATD             "nfs.rpc-statd"
-#define OPT_SERVER_RPC_STATD_PIDFILE     "nfs.rpc-statd-pidfile"
+#define OPT_SERVER_AUX_GIDS "nfs.server-aux-gids"
+#define OPT_SERVER_GID_CACHE_TIMEOUT "nfs.server.aux-gid-timeout"
+#define OPT_SERVER_RPC_STATD "nfs.rpc-statd"
+#define OPT_SERVER_RPC_STATD_PIDFILE "nfs.rpc-statd-pidfile"
 #define OPT_SERVER_RPC_STATD_NOTIFY_PIDFILE "nfs.rpc-statd-notify-pidfile"
 
-#define NFS_DATADIR                     GLUSTERD_DEFAULT_WORKDIR "/nfs"
+#define NFS_DATADIR GLUSTERD_DEFAULT_WORKDIR "/nfs"
 
 /* Forward declaration */
-static int nfs_add_initer (struct list_head *list, nfs_version_initer_t init,
-                           gf_boolean_t required);
+static int
+nfs_add_initer(struct list_head *list, nfs_version_initer_t init,
+               gf_boolean_t required);
 
 static int
-nfs_init_version (xlator_t *this, nfs_version_initer_t init,
-                  gf_boolean_t required)
+nfs_init_version(xlator_t *this, nfs_version_initer_t init,
+                 gf_boolean_t required)
 {
-        int                       ret       = -1;
-        struct nfs_initer_list    *version  = NULL;
-        struct nfs_initer_list    *tmp      = NULL;
-        rpcsvc_program_t          *prog     = NULL;
-        struct list_head          *versions = NULL;
-        struct nfs_state          *nfs      = NULL;
-        gf_boolean_t              found     = _gf_false;
-
-        if ((!this) || (!this->private) || (!init))
-                return (-1);
-
-        nfs = (struct nfs_state *)this->private;
-
-        ret = nfs_add_initer (&nfs->versions, init, required);
-        if (ret == -1) {
-                gf_log (GF_NFS, GF_LOG_ERROR,
-                                "Failed to add protocol initializer");
-                goto err;
-        }
-
-        versions = &nfs->versions;
-        list_for_each_entry_safe (version, tmp, versions, list) {
-                prog = version->program;
-                if (version->init == init) {
-                        prog = init(this);
-                        if (!prog) {
-                                ret = -1;
-                                goto err;
-                        }
-                        version->program = prog;
-                        found = _gf_true;
-                        break;
-                }
-        }
-
-        /* program not added */
-        if (!found) {
-                gf_log (GF_NFS, GF_LOG_ERROR,
-                                "Program: %s NOT found", prog->progname);
-                goto err;
-        }
-
-        /* Check if nfs.port is configured */
-        if (nfs->override_portnum)
-                prog->progport = nfs->override_portnum;
-
-        gf_log (GF_NFS, GF_LOG_DEBUG, "Starting program: %s", prog->progname);
-
-        ret = rpcsvc_program_register (nfs->rpcsvc, prog);
-        if (ret == -1) {
-                gf_log (GF_NFS, GF_LOG_ERROR, "Program: %s init failed",
-                                                      prog->progname);
-                goto err;
-        }
-
-        /* Registration with portmapper is disabled, Nothing to do */
-        if (!nfs->register_portmap)
-                goto err;
+    int ret = -1;
+    struct nfs_initer_list *version = NULL;
+    struct nfs_initer_list *tmp = NULL;
+    rpcsvc_program_t *prog = NULL;
+    struct list_head *versions = NULL;
+    struct nfs_state *nfs = NULL;
+    gf_boolean_t found = _gf_false;
+
+    if ((!this) || (!this->private) || (!init))
+        return (-1);
 
-        ret = rpcsvc_program_register_portmap (prog, prog->progport);
-        if (ret == -1) {
-                gf_log (GF_NFS, GF_LOG_ERROR,
-                                "Program  %s registration failed",
-                                prog->progname);
+    nfs = (struct nfs_state *)this->private;
+
+    ret = nfs_add_initer(&nfs->versions, init, required);
+    if (ret == -1) {
+        gf_msg(GF_NFS, GF_LOG_ERROR, 0, NFS_MSG_PROT_INIT_ADD_FAIL,
+               "Failed to add protocol initializer");
+        goto err;
+    }
+
+    versions = &nfs->versions;
+    list_for_each_entry_safe(version, tmp, versions, list)
+    {
+        prog = version->program;
+        if (version->init == init) {
+            prog = init(this);
+            if (!prog) {
+                ret = -1;
                 goto err;
-        }
-        ret = 0; /* All well */
+            }
+            version->program = prog;
+            found = _gf_true;
+            break;
+        }
+    }
+
+    /* program not added */
+    if (!found) {
+        gf_msg(GF_NFS, GF_LOG_ERROR, 0, NFS_MSG_PGM_NOT_FOUND,
+               "Program: %s NOT found", prog->progname);
+        goto err;
+    }
+
+    /* Check if nfs.port is configured */
+    if (nfs->override_portnum)
+        prog->progport = nfs->override_portnum;
+
+    gf_msg_debug(GF_NFS, 0, "Starting program: %s", prog->progname);
+
+    ret = rpcsvc_program_register(nfs->rpcsvc, prog, _gf_false);
+    if (ret == -1) {
+        gf_msg(GF_NFS, GF_LOG_ERROR, 0, NFS_MSG_PGM_INIT_FAIL,
+               "Program: %s init failed", prog->progname);
+        goto err;
+    }
+
+    /* Registration with portmapper is disabled, Nothing to do */
+    if (!nfs->register_portmap)
+        goto err;
+
+    ret = rpcsvc_program_register_portmap(prog, prog->progport);
+    if (ret == -1) {
+        gf_msg(GF_NFS, GF_LOG_ERROR, 0, NFS_MSG_PGM_REG_FAIL,
+               "Program  %s registration failed", prog->progname);
+        goto err;
+    }
+    ret = 0; /* All well */
 err:
-        return ret;
+    return ret;
 }
 
 static int
-nfs_deinit_version (struct nfs_state *nfs, nfs_version_initer_t init)
+nfs_deinit_version(struct nfs_state *nfs, nfs_version_initer_t init)
 {
-        int                       ret       = -1;
-        struct nfs_initer_list    *version  = NULL;
-        struct nfs_initer_list    *tmp      = NULL;
-        rpcsvc_program_t          *prog     = NULL;
-        struct list_head          *versions = NULL;
+    int ret = -1;
+    struct nfs_initer_list *version = NULL;
+    struct nfs_initer_list *tmp = NULL;
+    rpcsvc_program_t *prog = NULL;
+    struct list_head *versions = NULL;
 
-        if ((!nfs) || (!init))
-                return (-1);
+    if ((!nfs) || (!init))
+        return (-1);
 
-        versions = &nfs->versions;
-        list_for_each_entry_safe (version, tmp, versions, list) {
-                prog = version->program;
-                if (version->init == init) {
-                        prog = version->program;
-                        ret = rpcsvc_program_unregister (nfs->rpcsvc, prog);
-                        if (ret != 0)
-                                return (-1);
-                        list_del (&version->list);
-                        GF_FREE (version);
-                        return (0);
-                }
+    versions = &nfs->versions;
+    list_for_each_entry_safe(version, tmp, versions, list)
+    {
+        prog = version->program;
+        if (version->init == init) {
+            prog = version->program;
+            ret = rpcsvc_program_unregister(nfs->rpcsvc, prog);
+            if (ret != 0)
+                return (-1);
+            list_del(&version->list);
+            GF_FREE(version);
+            return (0);
         }
+    }
 
-        return (-1);
+    return (-1);
 }
 
 static int
-nfs_reconfigure_acl3 (xlator_t *this)
+nfs_reconfigure_acl3(xlator_t *this)
 {
-        struct nfs_state          *nfs     = NULL;
+    struct nfs_state *nfs = NULL;
 
-        if ((!this) || (!this->private))
-                return (-1);
+    if ((!this) || (!this->private))
+        return (-1);
 
-        nfs = (struct nfs_state *)this->private;
+    nfs = (struct nfs_state *)this->private;
 
-        /* ACL is enabled */
-        if (nfs->enable_acl)
-                return nfs_init_version (this, acl3svc_init, _gf_false);
+    /* ACL is enabled */
+    if (nfs->enable_acl)
+        return nfs_init_version(this, acl3svc_init, _gf_false);
 
-        /* ACL is disabled */
-        return nfs_deinit_version (nfs, acl3svc_init);
+    /* ACL is disabled */
+    return nfs_deinit_version(nfs, acl3svc_init);
 }
 
 static int
-nfs_reconfigure_nlm4 (xlator_t *this)
+nfs_reconfigure_nlm4(xlator_t *this)
 {
-        struct nfs_state          *nfs     = NULL;
+    struct nfs_state *nfs = NULL;
 
-        if ((!this) || (!this->private))
-                return (-1);
+    if ((!this) || (!this->private))
+        return (-1);
 
-        nfs = (struct nfs_state *)this->private;
+    nfs = (struct nfs_state *)this->private;
 
-        /* NLM is enabled */
-        if (nfs->enable_nlm)
-                return nfs_init_version (this, nlm4svc_init, _gf_false);
+    /* NLM is enabled */
+    if (nfs->enable_nlm)
+        return nfs_init_version(this, nlm4svc_init, _gf_false);
 
-        /* NLM is disabled */
-        return nfs_deinit_version (nfs, nlm4svc_init);
+    /* NLM is disabled */
+    return nfs_deinit_version(nfs, nlm4svc_init);
 }
 
 static int
-nfs_program_register_portmap_all (struct nfs_state *nfs)
+nfs_program_register_portmap_all(struct nfs_state *nfs)
 {
-        struct list_head                *versions = NULL;
-        struct nfs_initer_list          *version = NULL;
-        struct nfs_initer_list          *tmp = NULL;
-        rpcsvc_program_t                *prog = NULL;
+    struct list_head *versions = NULL;
+    struct nfs_initer_list *version = NULL;
+    struct nfs_initer_list *tmp = NULL;
+    rpcsvc_program_t *prog = NULL;
 
-        if (nfs == NULL)
-                return (-1);
+    if (nfs == NULL)
+        return (-1);
 
-        versions = &nfs->versions;
-        list_for_each_entry_safe (version, tmp, versions, list) {
-                prog = version->program;
-                if (prog == NULL)
-                        continue;
-                if (nfs->override_portnum)
-                        prog->progport = nfs->override_portnum;
-                (void) rpcsvc_program_register_portmap (prog, prog->progport);
-        }
+    versions = &nfs->versions;
+    list_for_each_entry_safe(version, tmp, versions, list)
+    {
+        prog = version->program;
+        if (prog == NULL)
+            continue;
+        if (nfs->override_portnum)
+            prog->progport = nfs->override_portnum;
+        (void)rpcsvc_program_register_portmap(prog, prog->progport);
+#ifdef IPV6_DEFAULT
+        (void)rpcsvc_program_register_rpcbind6(prog, prog->progport);
+#endif
+    }
 
-        return (0);
+    return (0);
 }
 
 static int
-nfs_program_unregister_portmap_all (struct nfs_state *nfs)
+nfs_program_unregister_portmap_all(struct nfs_state *nfs)
 {
-        struct list_head                *versions = NULL;
-        struct nfs_initer_list          *version = NULL;
-        struct nfs_initer_list          *tmp = NULL;
-        rpcsvc_program_t                *prog = NULL;
+    struct list_head *versions = NULL;
+    struct nfs_initer_list *version = NULL;
+    struct nfs_initer_list *tmp = NULL;
+    rpcsvc_program_t *prog = NULL;
 
-        if (nfs == NULL)
-                return (-1);
+    if (nfs == NULL)
+        return (-1);
 
-        versions = &nfs->versions;
-        list_for_each_entry_safe (version, tmp, versions, list) {
-                prog = version->program;
-                if (prog == NULL)
-                        continue;
-                (void) rpcsvc_program_unregister_portmap (prog);
-        }
+    versions = &nfs->versions;
+    list_for_each_entry_safe(version, tmp, versions, list)
+    {
+        prog = version->program;
+        if (prog == NULL)
+            continue;
+        (void)rpcsvc_program_unregister_portmap(prog);
+#ifdef IPV6_DEFAULT
+        (void)rpcsvc_program_unregister_rpcbind6(prog);
+#endif
+    }
 
-        return (0);
+    return (0);
 }
 
 /* Every NFS version must call this function with the init function
  * for its particular version.
  */
 static int
-nfs_add_initer (struct list_head *list, nfs_version_initer_t init,
-                gf_boolean_t required)
+nfs_add_initer(struct list_head *list, nfs_version_initer_t init,
+               gf_boolean_t required)
 {
-        struct nfs_initer_list  *new = NULL;
-        if ((!list) || (!init))
-                return -1;
-
-        new = GF_CALLOC (1, sizeof (*new), gf_nfs_mt_nfs_initer_list);
-        if (!new) {
-                gf_log (GF_NFS, GF_LOG_ERROR, "Memory allocation failed");
-                return -1;
-        }
-
-        new->init = init;
-        new->required = required;
-        list_add_tail (&new->list, list);
-        return 0;
+    struct nfs_initer_list *new = NULL;
+    if ((!list) || (!init))
+        return -1;
+
+    new = GF_CALLOC(1, sizeof(*new), gf_nfs_mt_nfs_initer_list);
+    if (!new) {
+        gf_msg(GF_NFS, GF_LOG_ERROR, ENOMEM, NFS_MSG_NO_MEMORY,
+               "Memory allocation failed");
+        return -1;
+    }
+
+    new->init = init;
+    new->required = required;
+    list_add_tail(&new->list, list);
+    return 0;
 }
 
-
 int
-nfs_deinit_versions (struct list_head *versions, xlator_t *this)
+nfs_deinit_versions(struct list_head *versions, xlator_t *this)
 {
-        struct nfs_initer_list          *version = NULL;
-        struct nfs_initer_list          *tmp = NULL;
-        struct nfs_state                *nfs = NULL;
-
-        if ((!versions) || (!this))
-                return -1;
-
-        nfs = (struct nfs_state *)this->private;
-        list_for_each_entry_safe (version, tmp, versions, list) {
-                /* TODO: Add version specific destructor.
-                 * if (!version->deinit)
-                        goto err;
-
-                   version->deinit (this);
-                */
-                if (version->program)
-                        rpcsvc_program_unregister (nfs->rpcsvc,
-                                                   (version->program));
-
-                list_del (&version->list);
-                GF_FREE (version);
-        }
-
-        return 0;
-}
+    struct nfs_initer_list *version = NULL;
+    struct nfs_initer_list *tmp = NULL;
+    struct nfs_state *nfs = NULL;
+
+    if ((!versions) || (!this))
+        return -1;
+
+    nfs = (struct nfs_state *)this->private;
+    list_for_each_entry_safe(version, tmp, versions, list)
+    {
+        /* TODO: Add version specific destructor.
+         * if (!version->deinit)
+                goto err;
 
-int
-nfs_init_versions (struct nfs_state *nfs, xlator_t *this)
-{
-        struct nfs_initer_list          *version = NULL;
-        struct nfs_initer_list          *tmp = NULL;
-        rpcsvc_program_t                *prog = NULL;
-        int                             ret = -1;
-        struct list_head                *versions = NULL;
-
-        if ((!nfs) || (!this))
-                return -1;
-
-        gf_log (GF_NFS, GF_LOG_DEBUG, "Initing protocol versions");
-        versions = &nfs->versions;
-        list_for_each_entry_safe (version, tmp, versions, list) {
-                if (!version->init) {
-                        ret = -1;
-                        goto err;
-                }
-
-                prog = version->init (this);
-                if (!prog) {
-                        ret = -1;
-                        goto err;
-                }
-
-                version->program = prog;
-                if (nfs->override_portnum)
-                        prog->progport = nfs->override_portnum;
-                gf_log (GF_NFS, GF_LOG_DEBUG, "Starting program: %s",
-                        prog->progname);
-
-                ret = rpcsvc_program_register (nfs->rpcsvc, prog);
-                if (ret == -1) {
-                        gf_log (GF_NFS, GF_LOG_ERROR, "Program: %s init failed",
-                                                      prog->progname);
-                        goto err;
-                }
-                if (nfs->register_portmap) {
-                        ret = rpcsvc_program_register_portmap (prog,
-                                                               prog->progport);
-                        if (ret == -1) {
-                                gf_log (GF_NFS, GF_LOG_ERROR,
-                                        "%s program %s registration failed",
-                                        version->required ?
-                                                "Required" : "Optional",
-                                        prog->progname);
-
-                                /* fatal error if the program is required */
-                                if (version->required)
-                                        goto err;
-                        }
-                }
+           version->deinit (this);
+        */
+        if (version->program)
+            rpcsvc_program_unregister(nfs->rpcsvc, (version->program));
 
-        }
+        list_del(&version->list);
+        GF_FREE(version);
+    }
 
-        ret = 0;
-err:
-        return ret;
+    return 0;
 }
 
-
 int
-nfs_add_all_initiators (struct nfs_state *nfs)
+nfs_init_versions(struct nfs_state *nfs, xlator_t *this)
 {
-        int     ret = 0;
+    struct nfs_initer_list *version = NULL;
+    struct nfs_initer_list *tmp = NULL;
+    rpcsvc_program_t *prog = NULL;
+    int ret = -1;
+    struct list_head *versions = NULL;
 
-        /* Add the initializers for all versions. */
-        ret = nfs_add_initer (&nfs->versions, mnt3svc_init, _gf_true);
-        if (ret == -1) {
-                gf_log (GF_NFS, GF_LOG_ERROR, "Failed to add "
-                                "MOUNT3 protocol initializer");
-                goto ret;
+    if ((!nfs) || (!this))
+        return -1;
+
+    gf_msg_debug(GF_NFS, 0, "Initing protocol versions");
+    versions = &nfs->versions;
+    list_for_each_entry_safe(version, tmp, versions, list)
+    {
+        if (!version->init) {
+            ret = -1;
+            goto err;
         }
 
-        ret = nfs_add_initer (&nfs->versions, mnt1svc_init, _gf_true);
-        if (ret == -1) {
-                gf_log (GF_NFS, GF_LOG_ERROR, "Failed to add "
-                                "MOUNT1 protocol initializer");
-                goto ret;
+        prog = version->init(this);
+        if (!prog) {
+            ret = -1;
+            goto err;
         }
 
-        ret = nfs_add_initer (&nfs->versions, nfs3svc_init, _gf_true);
+        version->program = prog;
+        if (nfs->override_portnum)
+            prog->progport = nfs->override_portnum;
+        gf_msg_debug(GF_NFS, 0, "Starting program: %s", prog->progname);
+
+        ret = rpcsvc_program_register(nfs->rpcsvc, prog, _gf_false);
         if (ret == -1) {
-                gf_log (GF_NFS, GF_LOG_ERROR, "Failed to add "
-                                "NFS3 protocol initializer");
-                goto ret;
+            gf_msg(GF_NFS, GF_LOG_ERROR, 0, NFS_MSG_PGM_INIT_FAIL,
+                   "Program: %s init failed", prog->progname);
+            goto err;
+        }
+        if (nfs->register_portmap) {
+            ret = rpcsvc_program_register_portmap(prog, prog->progport);
+            if (ret == -1) {
+                gf_msg(GF_NFS, GF_LOG_ERROR, 0, NFS_MSG_PGM_REG_FAIL,
+                       "%s program  %s registration failed",
+                       version->required ? "Required" : "Optional",
+                       prog->progname);
+
+                /* fatal error if the program is required */
+                if (version->required)
+                    goto err;
+            }
+#ifdef IPV6_DEFAULT
+            ret = rpcsvc_program_register_rpcbind6(prog, prog->progport);
+            if (ret == -1) {
+                gf_msg(GF_NFS, GF_LOG_ERROR, 0, NFS_MSG_PGM_REG_FAIL,
+                       "Program (ipv6) %s registration failed", prog->progname);
+                goto err;
+            }
+#endif
         }
+    }
+
+    ret = 0;
+err:
+    return ret;
+}
 
-        if (nfs->enable_nlm == _gf_true) {
-                ret = nfs_add_initer (&nfs->versions, nlm4svc_init, _gf_false);
-                if (ret == -1) {
-                        gf_log (GF_NFS, GF_LOG_ERROR, "Failed to add protocol"
-                                " initializer");
-                        goto ret;
-                }
+int
+nfs_add_all_initiators(struct nfs_state *nfs)
+{
+    int ret = 0;
+
+    /* Add the initializers for all versions. */
+    ret = nfs_add_initer(&nfs->versions, mnt3svc_init, _gf_true);
+    if (ret == -1) {
+        gf_msg(GF_NFS, GF_LOG_ERROR, 0, NFS_MSG_PROT_INIT_ADD_FAIL,
+               "Failed to add MOUNT3 protocol initializer");
+        goto ret;
+    }
+
+    ret = nfs_add_initer(&nfs->versions, mnt1svc_init, _gf_true);
+    if (ret == -1) {
+        gf_msg(GF_NFS, GF_LOG_ERROR, 0, NFS_MSG_PROT_INIT_ADD_FAIL,
+               "Failed to add MOUNT1 protocol initializer");
+        goto ret;
+    }
+
+    ret = nfs_add_initer(&nfs->versions, nfs3svc_init, _gf_true);
+    if (ret == -1) {
+        gf_msg(GF_NFS, GF_LOG_ERROR, 0, NFS_MSG_PROT_INIT_ADD_FAIL,
+               "Failed to add NFS3 protocol initializer");
+        goto ret;
+    }
+
+    if (nfs->enable_nlm == _gf_true) {
+        ret = nfs_add_initer(&nfs->versions, nlm4svc_init, _gf_false);
+        if (ret == -1) {
+            gf_msg(GF_NFS, GF_LOG_ERROR, 0, NFS_MSG_PROT_INIT_ADD_FAIL,
+                   "Failed to add protocol initializer");
+            goto ret;
         }
+    }
 
-        if (nfs->enable_acl == _gf_true) {
-                ret = nfs_add_initer (&nfs->versions, acl3svc_init, _gf_false);
-                if (ret == -1) {
-                        gf_log (GF_NFS, GF_LOG_ERROR, "Failed to add "
-                                "ACL protocol initializer");
-                        goto ret;
-                }
+    if (nfs->enable_acl == _gf_true) {
+        ret = nfs_add_initer(&nfs->versions, acl3svc_init, _gf_false);
+        if (ret == -1) {
+            gf_msg(GF_NFS, GF_LOG_ERROR, 0, NFS_MSG_PROT_INIT_ADD_FAIL,
+                   "Failed to add ACL protocol initializer");
+            goto ret;
         }
+    }
 
-        ret = 0;
+    ret = 0;
 ret:
-        return ret;
+    return ret;
 }
 
-
 int
-nfs_subvolume_started (struct nfs_state *nfs, xlator_t *xl)
+nfs_subvolume_started(struct nfs_state *nfs, xlator_t *xl)
 {
-        int     x = 0;
-        int     started = 0;
-
-        if ((!nfs) || (!xl))
-                return 1;
-
-        LOCK (&nfs->svinitlock);
-        {
-                for (;x < nfs->allsubvols; ++x) {
-                        if (nfs->initedxl[x] == xl) {
-                                started = 1;
-                                goto unlock;
-                       }
-               }
+    int x = 0;
+    int started = 0;
+
+    if ((!nfs) || (!xl))
+        return 1;
+
+    LOCK(&nfs->svinitlock);
+    {
+        for (; x < nfs->allsubvols; ++x) {
+            if (nfs->initedxl[x] == xl) {
+                started = 1;
+                goto unlock;
+            }
         }
+    }
 unlock:
-        UNLOCK (&nfs->svinitlock);
+    UNLOCK(&nfs->svinitlock);
 
-        return started;
+    return started;
 }
 
-
 int
-nfs_subvolume_set_started (struct nfs_state *nfs, xlator_t *xl)
+nfs_subvolume_set_started(struct nfs_state *nfs, xlator_t *xl)
 {
-        int     x = 0;
-
-        if ((!nfs) || (!xl))
-                return 1;
-
-        LOCK (&nfs->svinitlock);
-        {
-                for (;x < nfs->allsubvols; ++x) {
-                        if (nfs->initedxl[x] == xl) {
-                                gf_log (GF_NFS, GF_LOG_DEBUG,
-                                        "Volume already started %s",
-                                        xl->name);
-                                break;
-                        }
-
-                        if (nfs->initedxl[x] == NULL) {
-                                nfs->initedxl[x] = xl;
-                                ++nfs->upsubvols;
-                                gf_log (GF_NFS, GF_LOG_DEBUG, "Starting up: %s "
-                                        ", vols started till now: %d", xl->name,
-                                        nfs->upsubvols);
-                                goto unlock;
-                        }
-               }
-        }
+    int x = 0;
+
+    if ((!nfs) || (!xl))
+        return 1;
+
+    LOCK(&nfs->svinitlock);
+    {
+        for (; x < nfs->allsubvols; ++x) {
+            if (nfs->initedxl[x] == xl) {
+                gf_msg_debug(GF_NFS, 0, "Volume already started %s", xl->name);
+                break;
+            }
+
+            if (nfs->initedxl[x] == NULL) {
+                nfs->initedxl[x] = xl;
+                ++nfs->upsubvols;
+                gf_msg_debug(GF_NFS, 0,
+                             "Starting up: %s "
+                             ", vols started till now: %d",
+                             xl->name, nfs->upsubvols);
+                goto unlock;
+            }
+        }
+    }
 unlock:
-        UNLOCK (&nfs->svinitlock);
+    UNLOCK(&nfs->svinitlock);
 
-        return 0;
+    return 0;
 }
 
-
 int32_t
-nfs_start_subvol_lookup_cbk (call_frame_t *frame, void *cookie,
-                             xlator_t *this, int32_t op_ret, int32_t op_errno,
-                             inode_t *inode, struct iatt *buf, dict_t *xattr,
-                             struct iatt *postparent)
+nfs_start_subvol_lookup_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                            int32_t op_ret, int32_t op_errno, inode_t *inode,
+                            struct iatt *buf, dict_t *xattr,
+                            struct iatt *postparent)
 {
-        if (op_ret == -1) {
-                gf_log (GF_NFS, GF_LOG_CRITICAL, "Failed to lookup root: %s",
-                        strerror (op_errno));
-                goto err;
-        }
-
-        nfs_subvolume_set_started (this->private, ((xlator_t *)cookie));
-        gf_log (GF_NFS, GF_LOG_TRACE, "Started %s", ((xlator_t *)cookie)->name);
+    if (op_ret == -1) {
+        gf_msg(GF_NFS, GF_LOG_CRITICAL, op_errno, NFS_MSG_LOOKUP_ROOT_FAIL,
+               "Failed to lookup root: %s", strerror(op_errno));
+        goto err;
+    }
+
+    nfs_subvolume_set_started(this->private, ((xlator_t *)cookie));
+    gf_msg_trace(GF_NFS, 0, "Started %s", ((xlator_t *)cookie)->name);
 err:
-        return 0;
+    return 0;
 }
 
-
 int
-nfs_startup_subvolume (xlator_t *nfsx, xlator_t *xl)
+nfs_startup_subvolume(xlator_t *nfsx, xlator_t *xl)
 {
-        int             ret = -1;
-        loc_t           rootloc = {0, };
-        nfs_user_t      nfu = {0, };
-
-        if ((!nfsx) || (!xl))
-                return -1;
-
-        if (nfs_subvolume_started (nfsx->private, xl)) {
-                gf_log (GF_NFS,GF_LOG_TRACE, "Subvolume already started: %s",
-                        xl->name);
-                ret = 0;
-                goto err;
-        }
-
-        ret = nfs_root_loc_fill (xl->itable, &rootloc);
-        if (ret == -1) {
-                gf_log (GF_NFS, GF_LOG_CRITICAL, "Failed to init root loc");
-                goto err;
-        }
-
-        nfs_user_root_create (&nfu);
-        ret = nfs_fop_lookup (nfsx, xl, &nfu, &rootloc,
-                              nfs_start_subvol_lookup_cbk,
-                              (void *)nfsx->private);
-        if (ret < 0) {
-                gf_log (GF_NFS, GF_LOG_CRITICAL, "Failed to lookup root: %s",
-                        strerror (-ret));
-                goto err;
-        }
-
-        nfs_loc_wipe (&rootloc);
+    int ret = -1;
+    loc_t rootloc = {
+        0,
+    };
+    nfs_user_t nfu = {
+        0,
+    };
+
+    if ((!nfsx) || (!xl))
+        return -1;
+
+    if (nfs_subvolume_started(nfsx->private, xl)) {
+        gf_msg_trace(GF_NFS, 0, "Subvolume already started: %s", xl->name);
+        ret = 0;
+        goto err;
+    }
+
+    ret = nfs_root_loc_fill(xl->itable, &rootloc);
+    if (ret == -1) {
+        gf_msg(GF_NFS, GF_LOG_CRITICAL, 0, NFS_MSG_ROOT_LOC_INIT_FAIL,
+               "Failed to init root loc");
+        goto err;
+    }
+
+    nfs_user_root_create(&nfu);
+    ret = nfs_fop_lookup(nfsx, xl, &nfu, &rootloc, nfs_start_subvol_lookup_cbk,
+                         (void *)nfsx->private);
+    if (ret < 0) {
+        gf_msg(GF_NFS, GF_LOG_CRITICAL, -ret, NFS_MSG_LOOKUP_ROOT_FAIL,
+               "Failed to lookup root: %s", strerror(-ret));
+        goto err;
+    }
+
+    nfs_loc_wipe(&rootloc);
 
 err:
-        return ret;
+    return ret;
 }
 
 int
-nfs_startup_subvolumes (xlator_t *nfsx)
+nfs_startup_subvolumes(xlator_t *nfsx)
 {
-        int                     ret = -1;
-        xlator_list_t           *cl = NULL;
-        struct nfs_state        *nfs = NULL;
-
-        if (!nfsx)
-                return -1;
-
-        nfs = nfsx->private;
-        cl = nfs->subvols;
-        while (cl) {
-                gf_log (GF_NFS, GF_LOG_DEBUG, "Starting subvolume: %s",
-                        cl->xlator->name);
-                ret = nfs_startup_subvolume (nfsx, cl->xlator);
-                if (ret == -1) {
-                        gf_log (GF_NFS, GF_LOG_CRITICAL, "Failed to start-up "
-                                "xlator: %s", cl->xlator->name);
-                        goto err;
-                }
-                cl = cl->next;
+    int ret = -1;
+    xlator_list_t *cl = NULL;
+    struct nfs_state *nfs = NULL;
+
+    if (!nfsx)
+        return -1;
+
+    nfs = nfsx->private;
+    cl = nfs->subvols;
+    while (cl) {
+        gf_msg_debug(GF_NFS, 0, "Starting subvolume: %s", cl->xlator->name);
+        ret = nfs_startup_subvolume(nfsx, cl->xlator);
+        if (ret == -1) {
+            gf_msg(GF_NFS, GF_LOG_CRITICAL, 0, NFS_MSG_STARTUP_FAIL,
+                   "Failed to start-up xlator: %s", cl->xlator->name);
+            goto err;
         }
+        cl = cl->next;
+    }
 
-        ret = 0;
+    ret = 0;
 err:
-        return ret;
+    return ret;
 }
 
-
 int
-nfs_init_subvolume (struct nfs_state *nfs, xlator_t *xl)
+nfs_init_subvolume(struct nfs_state *nfs, xlator_t *xl)
 {
-        unsigned int    lrusize = 0;
-        int             ret = -1;
-
-        if ((!nfs) || (!xl))
-                return -1;
-
-        lrusize = nfs->memfactor * GF_NFS_INODE_LRU_MULT;
-        xl->itable = inode_table_new (lrusize, xl);
-        if (!xl->itable) {
-                gf_log (GF_NFS, GF_LOG_CRITICAL, "Failed to allocate "
-                        "inode table");
-                goto err;
-        }
-        ret = 0;
+    unsigned int lrusize = 0;
+    int ret = -1;
+
+    if ((!nfs) || (!xl))
+        return -1;
+
+    lrusize = nfs->memfactor * GF_NFS_INODE_LRU_MULT;
+    xl->itable = inode_table_new(lrusize, xl);
+    if (!xl->itable) {
+        gf_msg(GF_NFS, GF_LOG_CRITICAL, ENOMEM, NFS_MSG_NO_MEMORY,
+               "Failed to allocate inode table");
+        goto err;
+    }
+    ret = 0;
 err:
-        return ret;
+    return ret;
 }
 
 int
-nfs_init_subvolumes (struct nfs_state *nfs, xlator_list_t *cl)
+nfs_init_subvolumes(struct nfs_state *nfs, xlator_list_t *cl)
 {
-        int             ret = -1;
-        unsigned int    lrusize = 0;
-        int             svcount = 0;
-
-        if ((!nfs) || (!cl))
-                return -1;
-
-        lrusize = nfs->memfactor * GF_NFS_INODE_LRU_MULT;
-        nfs->subvols = cl;
-        gf_log (GF_NFS, GF_LOG_TRACE, "inode table lru: %d", lrusize);
-
-        while (cl) {
-                gf_log (GF_NFS, GF_LOG_DEBUG, "Initing subvolume: %s",
-                        cl->xlator->name);
-                ret = nfs_init_subvolume (nfs, cl->xlator);
-                if (ret == -1) {
-                        gf_log (GF_NFS, GF_LOG_CRITICAL, "Failed to init "
-                                "xlator: %s", cl->xlator->name);
-                        goto err;
-                }
-                ++svcount;
-                cl = cl->next;
-        }
+    int ret = -1;
+    unsigned int lrusize = 0;
+    int svcount = 0;
 
-        LOCK_INIT (&nfs->svinitlock);
-        nfs->initedxl = GF_CALLOC (svcount, sizeof (xlator_t *),
-                                   gf_nfs_mt_xlator_t );
-        if (!nfs->initedxl) {
-                gf_log (GF_NFS, GF_LOG_ERROR, "Failed to allocated inited xls");
-                ret = -1;
-                goto err;
-        }
+    if ((!nfs) || (!cl))
+        return -1;
 
-        gf_log (GF_NFS, GF_LOG_TRACE, "Inited volumes: %d", svcount);
-        nfs->allsubvols = svcount;
-        ret = 0;
+    lrusize = nfs->memfactor * GF_NFS_INODE_LRU_MULT;
+    nfs->subvols = cl;
+    gf_msg_trace(GF_NFS, 0, "inode table lru: %d", lrusize);
+
+    while (cl) {
+        gf_msg_debug(GF_NFS, 0, "Initing subvolume: %s", cl->xlator->name);
+        ret = nfs_init_subvolume(nfs, cl->xlator);
+        if (ret == -1) {
+            gf_msg(GF_NFS, GF_LOG_CRITICAL, 0, NFS_MSG_XLATOR_INIT_FAIL,
+                   "Failed to init "
+                   "xlator: %s",
+                   cl->xlator->name);
+            goto err;
+        }
+        ++svcount;
+        cl = cl->next;
+    }
+
+    LOCK_INIT(&nfs->svinitlock);
+    nfs->initedxl = GF_CALLOC(svcount, sizeof(xlator_t *), gf_nfs_mt_xlator_t);
+    if (!nfs->initedxl) {
+        gf_msg(GF_NFS, GF_LOG_ERROR, ENOMEM, NFS_MSG_NO_MEMORY,
+               "Failed to allocated inited xls");
+        ret = -1;
+        goto err;
+    }
+
+    gf_msg_trace(GF_NFS, 0, "Inited volumes: %d", svcount);
+    nfs->allsubvols = svcount;
+    ret = 0;
 err:
-        return ret;
+    return ret;
 }
 
-
 int
-nfs_user_root_create (nfs_user_t *newnfu)
+nfs_user_root_create(nfs_user_t *newnfu)
 {
-        if (!newnfu)
-                return -1;
+    if (!newnfu)
+        return -1;
 
-        newnfu->uid = 0;
-        newnfu->gids[0] = 0;
-        newnfu->ngrps = 1;
+    newnfu->uid = 0;
+    newnfu->gids[0] = 0;
+    newnfu->ngrps = 1;
 
-        return 0;
+    return 0;
 }
 
-
 int
-nfs_user_create (nfs_user_t *newnfu, uid_t uid, gid_t gid, gid_t *auxgids,
-                 int auxcount)
+nfs_user_create(nfs_user_t *newnfu, uid_t uid, gid_t gid,
+                rpc_transport_t *trans, gid_t *auxgids, int auxcount)
 {
-        int     x = 1;
-        int     y = 0;
-
-        /* We test for GF_REQUEST_MAXGROUPS instead of  NFS_FOP_NGROUPS because
-         * the latter accounts for the @gid being in @auxgids, which is not the
-         * case here.
-         */
-        if ((!newnfu) || (auxcount > GF_REQUEST_MAXGROUPS))
-                return -1;
-
-        newnfu->uid = uid;
-        newnfu->gids[0] = gid;
-        newnfu->ngrps = 1;
-
-        gf_log (GF_NFS, GF_LOG_TRACE, "uid: %d, gid %d, gids: %d", uid, gid,
-                auxcount);
+    int x = 1;
+    int y = 0;
+
+    /* We test for GF_REQUEST_MAXGROUPS instead of  NFS_FOP_NGROUPS because
+     * the latter accounts for the @gid being in @auxgids, which is not the
+     * case here.
+     */
+    if ((!newnfu) || (auxcount > GF_REQUEST_MAXGROUPS))
+        return -1;
+
+    newnfu->uid = uid;
+    newnfu->gids[0] = gid;
+    newnfu->ngrps = 1;
+    if (trans) {
+        memcpy(&newnfu->identifier, trans->peerinfo.identifier, UNIX_PATH_MAX);
+    }
+
+    gf_msg_trace(GF_NFS, 0, "uid: %d, gid %d, gids: %d", uid, gid, auxcount);
+
+    if (!auxgids)
+        return 0;
 
-        if (!auxgids)
-                return 0;
+    for (; y < auxcount; ++x, ++y) {
+        newnfu->gids[x] = auxgids[y];
+        ++newnfu->ngrps;
+        gf_msg_trace(GF_NFS, 0, "gid: %d", auxgids[y]);
+    }
 
-        for (; y < auxcount; ++x,++y) {
-                newnfu->gids[x] = auxgids[y];
-                ++newnfu->ngrps;
-                gf_log (GF_NFS, GF_LOG_TRACE, "gid: %d", auxgids[y]);
-        }
-
-        return 0;
+    return 0;
 }
 
-
 void
-nfs_request_user_init (nfs_user_t *nfu, rpcsvc_request_t *req)
+nfs_request_user_init(nfs_user_t *nfu, rpcsvc_request_t *req)
 {
-        gid_t           *gidarr = NULL;
-        int             gids = 0;
+    gid_t *gidarr = NULL;
+    int gids = 0;
 
-        if ((!req) || (!nfu))
-                return;
+    if ((!req) || (!nfu))
+        return;
 
-        gidarr = rpcsvc_auth_unix_auxgids (req, &gids);
-        nfs_user_create (nfu, rpcsvc_request_uid (req),
-                         rpcsvc_request_gid (req), gidarr, gids);
+    gidarr = rpcsvc_auth_unix_auxgids(req, &gids);
+    nfs_user_create(nfu, rpcsvc_request_uid(req), rpcsvc_request_gid(req),
+                    rpcsvc_request_transport(req), gidarr, gids);
 
-        return;
+    return;
 }
 
 void
-nfs_request_primary_user_init (nfs_user_t *nfu, rpcsvc_request_t *req,
-                               uid_t uid, gid_t gid)
+nfs_request_primary_user_init(nfs_user_t *nfu, rpcsvc_request_t *req, uid_t uid,
+                              gid_t gid)
 {
-        gid_t           *gidarr = NULL;
-        int             gids = 0;
+    gid_t *gidarr = NULL;
+    int gids = 0;
 
-        if ((!req) || (!nfu))
-                return;
+    if ((!req) || (!nfu))
+        return;
 
-        gidarr = rpcsvc_auth_unix_auxgids (req, &gids);
-        nfs_user_create (nfu, uid, gid, gidarr, gids);
+    gidarr = rpcsvc_auth_unix_auxgids(req, &gids);
+    nfs_user_create(nfu, uid, gid, rpcsvc_request_transport(req), gidarr, gids);
 
-        return;
+    return;
 }
 
-
 int32_t
-mem_acct_init (xlator_t *this)
+mem_acct_init(xlator_t *this)
 {
-        int     ret = -1;
+    int ret = -1;
 
-        if (!this)
-                return ret;
+    if (!this)
+        return ret;
 
-        ret = xlator_mem_acct_init (this, gf_nfs_mt_end + 1);
-
-        if (ret != 0) {
-                gf_log(this->name, GF_LOG_ERROR, "Memory accounting init"
-                                "failed");
-                return ret;
-        }
+    ret = xlator_mem_acct_init(this, gf_nfs_mt_end + 1);
 
+    if (ret != 0) {
+        gf_msg(this->name, GF_LOG_ERROR, ENOMEM, NFS_MSG_NO_MEMORY,
+               "Memory accounting init failed");
         return ret;
-}
+    }
 
+    return ret;
+}
 
 struct nfs_state *
-nfs_init_state (xlator_t *this)
+nfs_init_state(xlator_t *this)
 {
-        struct nfs_state        *nfs = NULL;
-        int                     ret = -1;
-        unsigned int            fopspoolsize = 0;
-        char                    *optstr = NULL;
-        gf_boolean_t            boolt = _gf_false;
-        struct stat             stbuf = {0,};
-
-        if (!this)
-                return NULL;
-
-        if (!this->children) {
-                gf_log (GF_NFS, GF_LOG_INFO,
-                        "NFS is manually disabled: Exiting");
-                /* Nothing for nfs process to do, exit cleanly */
-                kill (getpid (), SIGTERM);
-        }
-
-        nfs = GF_CALLOC (1, sizeof (*nfs), gf_nfs_mt_nfs_state);
-        if (!nfs) {
-                gf_log (GF_NFS, GF_LOG_ERROR, "memory allocation failed");
-                return NULL;
+    struct nfs_state *nfs = NULL;
+    int i = 0, ret = -1;
+    unsigned int fopspoolsize = 0;
+    char *optstr = NULL;
+    gf_boolean_t boolt = _gf_false;
+    struct stat stbuf = {
+        0,
+    };
+
+    if (!this)
+        return NULL;
+
+    if (!this->children) {
+        gf_msg(GF_NFS, GF_LOG_INFO, 0, NFS_MSG_NFS_MAN_DISABLE,
+               "NFS is manually disabled: Exiting");
+        /* Nothing for nfs process to do, exit cleanly */
+        kill(getpid(), SIGTERM);
+    }
+
+    nfs = GF_CALLOC(1, sizeof(*nfs), gf_nfs_mt_nfs_state);
+    if (!nfs) {
+        gf_msg(GF_NFS, GF_LOG_ERROR, ENOMEM, NFS_MSG_NO_MEMORY,
+               "memory allocation failed");
+        return NULL;
+    }
+
+    nfs->memfactor = GF_NFS_DEFAULT_MEMFACTOR;
+    if (dict_get(this->options, "nfs.mem-factor")) {
+        ret = dict_get_str(this->options, "nfs.mem-factor", &optstr);
+        if (ret < 0) {
+            gf_msg(GF_NFS, GF_LOG_ERROR, -ret, NFS_MSG_DICT_GET_FAILED,
+                   "Failed to parse dict");
+            goto free_rpcsvc;
         }
 
-        nfs->memfactor = GF_NFS_DEFAULT_MEMFACTOR;
-        if (dict_get (this->options, "nfs.mem-factor")) {
-                ret = dict_get_str (this->options, "nfs.mem-factor",
-                                    &optstr);
-                if (ret < 0) {
-                        gf_log (GF_NFS, GF_LOG_ERROR, "Failed to parse dict");
-                        goto free_rpcsvc;
-                }
-
-                ret = gf_string2uint (optstr, &nfs->memfactor);
-                if (ret < 0) {
-                        gf_log (GF_NFS, GF_LOG_ERROR, "Failed to parse uint "
-                                "string");
-                        goto free_rpcsvc;
-                }
+        ret = gf_string2uint(optstr, &nfs->memfactor);
+        if (ret < 0) {
+            gf_msg(GF_NFS, GF_LOG_ERROR, errno, NFS_MSG_PARSE_FAIL,
+                   "Failed to parse uint string");
+            goto free_rpcsvc;
+        }
+    }
+
+    fopspoolsize = nfs->memfactor * GF_NFS_CONCURRENT_OPS_MULT;
+    /* FIXME: Really saddens me to see this as xlator wide. */
+    nfs->foppool = mem_pool_new(struct nfs_fop_local, fopspoolsize);
+    if (!nfs->foppool) {
+        gf_msg(GF_NFS, GF_LOG_CRITICAL, ENOMEM, NFS_MSG_NO_MEMORY,
+               "Failed to allocate fops local pool");
+        goto free_rpcsvc;
+    }
+
+    nfs->dynamicvolumes = GF_NFS_DVM_OFF;
+    if (dict_get(this->options, "nfs.dynamic-volumes")) {
+        ret = dict_get_str(this->options, "nfs.dynamic-volumes", &optstr);
+        if (ret < 0) {
+            gf_msg(GF_NFS, GF_LOG_ERROR, -ret, NFS_MSG_DICT_GET_FAILED,
+                   "Failed to parse dict");
+            goto free_foppool;
         }
 
-        fopspoolsize = nfs->memfactor * GF_NFS_CONCURRENT_OPS_MULT;
-        /* FIXME: Really saddens me to see this as xlator wide. */
-        nfs->foppool = mem_pool_new (struct nfs_fop_local, fopspoolsize);
-        if (!nfs->foppool) {
-                gf_log (GF_NFS, GF_LOG_CRITICAL, "Failed to allocate fops "
-                        "local pool");
-                goto free_rpcsvc;
+        ret = gf_string2boolean(optstr, &boolt);
+        if (ret < 0) {
+            gf_msg(GF_NFS, GF_LOG_ERROR, errno, NFS_MSG_PARSE_FAIL,
+                   "Failed to parse bool string");
+            goto free_foppool;
+        }
+
+        if (boolt == _gf_true)
+            nfs->dynamicvolumes = GF_NFS_DVM_ON;
+    }
+
+    nfs->enable_nlm = _gf_true;
+    ret = dict_get_str_boolean(this->options, "nfs.nlm", _gf_true);
+    if (ret == _gf_false) {
+        gf_msg(GF_NFS, GF_LOG_INFO, 0, NFS_MSG_NLM_MAN_DISABLE,
+               "NLM is manually disabled");
+        nfs->enable_nlm = _gf_false;
+    }
+
+    nfs->enable_acl = _gf_true;
+    ret = dict_get_str_boolean(this->options, "nfs.acl", _gf_true);
+    if (ret == _gf_false) {
+        gf_msg(GF_NFS, GF_LOG_INFO, 0, NFS_MSG_ACL_MAN_DISABLE,
+               "ACL is manually disabled");
+        nfs->enable_acl = _gf_false;
+    }
+
+    nfs->enable_ino32 = 0;
+    if (dict_get(this->options, "nfs.enable-ino32")) {
+        ret = dict_get_str(this->options, "nfs.enable-ino32", &optstr);
+        if (ret < 0) {
+            gf_msg(GF_NFS, GF_LOG_ERROR, -ret, NFS_MSG_PARSE_FAIL,
+                   "Failed to parse dict");
+            goto free_foppool;
         }
 
-        nfs->dynamicvolumes = GF_NFS_DVM_OFF;
-        if (dict_get (this->options, "nfs.dynamic-volumes")) {
-                ret = dict_get_str (this->options, "nfs.dynamic-volumes",
-                                    &optstr);
-                if (ret < 0) {
-                        gf_log (GF_NFS, GF_LOG_ERROR, "Failed to parse dict");
-                        goto free_foppool;
-                }
-
-                ret = gf_string2boolean (optstr, &boolt);
-                if (ret < 0) {
-                        gf_log (GF_NFS, GF_LOG_ERROR, "Failed to parse bool "
-                                "string");
-                        goto free_foppool;
-                }
-
-                if (boolt == _gf_true)
-                        nfs->dynamicvolumes = GF_NFS_DVM_ON;
+        ret = gf_string2boolean(optstr, &boolt);
+        if (ret < 0) {
+            gf_msg(GF_NFS, GF_LOG_ERROR, errno, NFS_MSG_PARSE_FAIL,
+                   "Failed to parse bool string");
+            goto free_foppool;
         }
 
-        nfs->enable_nlm = _gf_true;
-        ret = dict_get_str_boolean (this->options, "nfs.nlm", _gf_true);
-        if (ret == _gf_false) {
-                gf_log (GF_NFS, GF_LOG_INFO, "NLM is manually disabled");
-                nfs->enable_nlm = _gf_false;
-        }
+        if (boolt == _gf_true)
+            nfs->enable_ino32 = 1;
+    }
 
-        nfs->enable_acl = _gf_true;
-        ret = dict_get_str_boolean (this->options, "nfs.acl", _gf_true);
-        if (ret == _gf_false) {
-                gf_log (GF_NFS, GF_LOG_INFO, "ACL is manually disabled");
-                nfs->enable_acl = _gf_false;
+    if (dict_get(this->options, "nfs.port")) {
+        ret = dict_get_str(this->options, "nfs.port", &optstr);
+        if (ret < 0) {
+            gf_msg(GF_NFS, GF_LOG_ERROR, -ret, NFS_MSG_PARSE_FAIL,
+                   "Failed to parse dict");
+            goto free_foppool;
         }
 
-        nfs->enable_ino32 = 0;
-        if (dict_get (this->options, "nfs.enable-ino32")) {
-                ret = dict_get_str (this->options, "nfs.enable-ino32",
-                                    &optstr);
-                if (ret < 0) {
-                        gf_log (GF_NFS, GF_LOG_ERROR, "Failed to parse dict");
-                        goto free_foppool;
-                }
-
-                ret = gf_string2boolean (optstr, &boolt);
-                if (ret < 0) {
-                        gf_log (GF_NFS, GF_LOG_ERROR, "Failed to parse bool "
-                                "string");
-                        goto free_foppool;
-                }
-
-                if (boolt == _gf_true)
-                        nfs->enable_ino32 = 1;
+        ret = gf_string2uint(optstr, &nfs->override_portnum);
+        if (ret < 0) {
+            gf_msg(GF_NFS, GF_LOG_ERROR, errno, NFS_MSG_PARSE_FAIL,
+                   "Failed to parse uint string");
+            goto free_foppool;
         }
+    }
 
-        if (dict_get (this->options, "nfs.port")) {
-                ret = dict_get_str (this->options, "nfs.port",
-                                    &optstr);
-                if (ret < 0) {
-                        gf_log (GF_NFS, GF_LOG_ERROR, "Failed to parse dict");
-                        goto free_foppool;
-                }
-
-                ret = gf_string2uint (optstr, &nfs->override_portnum);
-                if (ret < 0) {
-                        gf_log (GF_NFS, GF_LOG_ERROR, "Failed to parse uint "
-                                "string");
-                        goto free_foppool;
-                }
+    if (dict_get(this->options, "transport.socket.bind-address")) {
+        ret = dict_get_str(this->options, "transport.socket.bind-address",
+                           &optstr);
+        if (ret < 0) {
+            gf_log(GF_NFS, GF_LOG_ERROR,
+                   "Failed to parse "
+                   "transport.socket.bind-address string");
+        } else {
+            this->instance_name = gf_strdup(optstr);
+            for (i = 0; i < strlen(this->instance_name); i++) {
+                if (this->instance_name[i] == '.' ||
+                    this->instance_name[i] == ':')
+                    this->instance_name[i] = '_';
+            }
         }
+    }
 
-        if (dict_get(this->options, "transport.socket.listen-port") == NULL) {
-                if (nfs->override_portnum)
-                        ret = gf_asprintf (&optstr, "%d",
-                                           nfs->override_portnum);
-                else
-                        ret = gf_asprintf (&optstr, "%d", GF_NFS3_PORT);
-                if (ret == -1) {
-                        gf_log (GF_NFS, GF_LOG_ERROR, "failed mem-allocation");
-                        goto free_foppool;
-                }
-                ret = dict_set_dynstr (this->options,
-                                       "transport.socket.listen-port", optstr);
-                if (ret == -1) {
-                        gf_log (GF_NFS, GF_LOG_ERROR, "dict_set_dynstr error");
-                        goto free_foppool;
-                }
+    if (dict_get(this->options, "transport.socket.listen-port") == NULL) {
+        if (nfs->override_portnum)
+            ret = gf_asprintf(&optstr, "%d", nfs->override_portnum);
+        else
+            ret = gf_asprintf(&optstr, "%d", GF_NFS3_PORT);
+        if (ret == -1) {
+            gf_msg(GF_NFS, GF_LOG_ERROR, ENOMEM, NFS_MSG_NO_MEMORY,
+                   "failed mem-allocation");
+            goto free_foppool;
         }
+        ret = dict_set_dynstr(this->options, "transport.socket.listen-port",
+                              optstr);
+        if (ret == -1) {
+            gf_msg(GF_NFS, GF_LOG_ERROR, 0, NFS_MSG_DICT_SET_FAILED,
+                   "dict_set_dynstr error");
+            goto free_foppool;
+        }
+    }
+
+#ifdef IPV6_DEFAULT
+    ret = dict_set_str(this->options, "transport.address-family", "inet6");
+    if (ret == -1) {
+        gf_log(GF_NFS, GF_LOG_ERROR, "dict_set_str error");
+        goto free_foppool;
+    }
+#endif
 
-        /* Right only socket support exists between nfs client and
-         * gluster nfs, so we can set default value as socket
-         */
-        ret = dict_set_str (this->options, "transport-type", "socket");
+    /* Right only socket support exists between nfs client and
+     * gluster nfs, so we can set default value as socket
+     */
+    ret = dict_set_str(this->options, "transport-type", "socket");
+    if (ret == -1) {
+        gf_msg(GF_NFS, GF_LOG_ERROR, 0, NFS_MSG_DICT_SET_FAILED,
+               "dict_set_str error");
+        goto free_foppool;
+    }
+
+    nfs->mount_udp = 0;
+    if (dict_get(this->options, "nfs.mount-udp")) {
+        ret = dict_get_str(this->options, "nfs.mount-udp", &optstr);
         if (ret == -1) {
-                gf_log (GF_NFS, GF_LOG_ERROR, "dict_set_str error");
-                goto free_foppool;
+            gf_msg(GF_NFS, GF_LOG_ERROR, -ret, NFS_MSG_PARSE_FAIL,
+                   "Failed to parse dict");
+            goto free_foppool;
         }
 
-        nfs->mount_udp = 0;
-        if (dict_get(this->options, "nfs.mount-udp")) {
-                ret = dict_get_str (this->options, "nfs.mount-udp", &optstr);
-                if (ret == -1) {
-                        gf_log (GF_NFS, GF_LOG_ERROR, "Failed to parse dict");
-                        goto free_foppool;
-                }
-
-                ret = gf_string2boolean (optstr, &boolt);
-                if (ret < 0) {
-                        gf_log (GF_NFS, GF_LOG_ERROR, "Failed to parse bool "
-                                "string");
-                        goto free_foppool;
-                }
-
-                if (boolt == _gf_true)
-                        nfs->mount_udp = 1;
+        ret = gf_string2boolean(optstr, &boolt);
+        if (ret < 0) {
+            gf_msg(GF_NFS, GF_LOG_ERROR, errno, NFS_MSG_PARSE_FAIL,
+                   "Failed to parse bool "
+                   "string");
+            goto free_foppool;
         }
 
-        nfs->exports_auth = GF_NFS_DEFAULT_EXPORT_AUTH;
-        if (dict_get(this->options, "nfs.exports-auth-enable")) {
-                ret = dict_get_str (this->options, "nfs.exports-auth-enable",
-                                    &optstr);
-                if (ret == -1) {
-                        gf_log (GF_NFS, GF_LOG_ERROR, "Failed to parse dict");
-                        goto free_foppool;
-                }
-
-                ret = gf_string2boolean (optstr, &boolt);
-                if (ret < 0) {
-                        gf_log (GF_NFS, GF_LOG_ERROR, "Failed to parse bool "
-                                "string");
-                        goto free_foppool;
-                }
-
-                if (boolt == _gf_true)
-                        nfs->exports_auth = 1;
-        }
+        if (boolt == _gf_true)
+            nfs->mount_udp = 1;
+    }
 
-        nfs->auth_refresh_time_secs = GF_NFS_DEFAULT_AUTH_REFRESH_INTERVAL_SEC;
-        if (dict_get (this->options, "nfs.auth-refresh-interval-sec")) {
-                ret = dict_get_str (this->options,
-                                    "nfs.auth-refresh-interval-sec", &optstr);
-                if (ret < 0) {
-                        gf_log (GF_NFS, GF_LOG_ERROR, "Failed to parse dict");
-                        goto free_foppool;
-                }
-
-                ret = gf_string2uint (optstr, &nfs->auth_refresh_time_secs);
-                if (ret < 0) {
-                        gf_log (GF_NFS, GF_LOG_ERROR, "Failed to parse uint "
-                                "string");
-                        goto free_foppool;
-                }
+    nfs->exports_auth = GF_NFS_DEFAULT_EXPORT_AUTH;
+    if (dict_get(this->options, "nfs.exports-auth-enable")) {
+        ret = dict_get_str(this->options, "nfs.exports-auth-enable", &optstr);
+        if (ret == -1) {
+            gf_msg(GF_NFS, GF_LOG_ERROR, -ret, NFS_MSG_PARSE_FAIL,
+                   "Failed to parse dict");
+            goto free_foppool;
         }
 
-        nfs->auth_cache_ttl_sec = GF_NFS_DEFAULT_AUTH_CACHE_TTL_SEC;
-        if (dict_get (this->options, "nfs.auth-cache-ttl-sec")) {
-                ret = dict_get_str (this->options,
-                                    "nfs.auth-cache-ttl-sec", &optstr);
-                if (ret < 0) {
-                        gf_log (GF_NFS, GF_LOG_ERROR, "Failed to parse dict");
-                        goto free_foppool;
-                }
-
-                ret = gf_string2uint (optstr, &nfs->auth_cache_ttl_sec);
-                if (ret < 0) {
-                        gf_log (GF_NFS, GF_LOG_ERROR, "Failed to parse uint "
-                                "string");
-                        goto free_foppool;
-                }
+        ret = gf_string2boolean(optstr, &boolt);
+        if (ret < 0) {
+            gf_msg(GF_NFS, GF_LOG_ERROR, errno, NFS_MSG_PARSE_FAIL,
+                   "Failed to parse bool string");
+            goto free_foppool;
         }
 
-        /* TODO: Make this a configurable option in case we don't want to read
-         * exports/netgroup files off disk when they change. */
-        nfs->refresh_auth = 1;
-
-        nfs->rmtab = gf_strdup (NFS_DATADIR "/rmtab");
-        if (dict_get(this->options, "nfs.mount-rmtab")) {
-                ret = dict_get_str (this->options, "nfs.mount-rmtab", &nfs->rmtab);
-                if (ret == -1) {
-                        gf_log (GF_NFS, GF_LOG_ERROR, "Failed to parse dict");
-                        goto free_foppool;
-                }
-        }
+        if (boolt == _gf_true)
+            nfs->exports_auth = 1;
+    }
 
-        /* support both options rpc-auth.ports.insecure and
-         * rpc-auth-allow-insecure for backward compatibility
-         */
-        nfs->allow_insecure = 1;
-        if (dict_get(this->options, "rpc-auth.ports.insecure")) {
-                ret = dict_get_str (this->options, "rpc-auth.ports.insecure",
-                                    &optstr);
-                if (ret < 0) {
-                        gf_log (GF_NFS, GF_LOG_ERROR, "Failed to parse dict");
-                        goto free_foppool;
-                }
-
-                ret = gf_string2boolean (optstr, &boolt);
-                if (ret < 0) {
-                        gf_log (GF_NFS, GF_LOG_ERROR, "Failed to parse bool "
-                                "string");
-                        goto free_foppool;
-                }
-
-                if (boolt == _gf_false)
-                        nfs->allow_insecure = 0;
+    nfs->auth_refresh_time_secs = GF_NFS_DEFAULT_AUTH_REFRESH_INTERVAL_SEC;
+    if (dict_get(this->options, "nfs.auth-refresh-interval-sec")) {
+        ret = dict_get_str(this->options, "nfs.auth-refresh-interval-sec",
+                           &optstr);
+        if (ret < 0) {
+            gf_msg(GF_NFS, GF_LOG_ERROR, -ret, NFS_MSG_PARSE_FAIL,
+                   "Failed to parse dict");
+            goto free_foppool;
         }
 
-        if (dict_get(this->options, "rpc-auth-allow-insecure")) {
-                ret = dict_get_str (this->options, "rpc-auth-allow-insecure",
-                                    &optstr);
-                if (ret < 0) {
-                        gf_log (GF_NFS, GF_LOG_ERROR, "Failed to parse dict");
-                        goto free_foppool;
-                }
-
-                ret = gf_string2boolean (optstr, &boolt);
-                if (ret < 0) {
-                        gf_log (GF_NFS, GF_LOG_ERROR, "Failed to parse bool "
-                                "string");
-                        goto free_foppool;
-                }
-
-                if (boolt == _gf_false)
-                        nfs->allow_insecure = 0;
+        ret = gf_string2uint(optstr, &nfs->auth_refresh_time_secs);
+        if (ret < 0) {
+            gf_msg(GF_NFS, GF_LOG_ERROR, errno, NFS_MSG_PARSE_FAIL,
+                   "Failed to parse uint string");
+            goto free_foppool;
         }
+    }
 
-        if (nfs->allow_insecure) {
-                /* blindly set both the options */
-                dict_del (this->options, "rpc-auth-allow-insecure");
-                ret = dict_set_str (this->options,
-                                    "rpc-auth-allow-insecure", "on");
-                if (ret == -1) {
-                        gf_log (GF_NFS, GF_LOG_ERROR, "dict_set_str error");
-                        goto free_foppool;
-                }
-                dict_del (this->options, "rpc-auth.ports.insecure");
-                ret = dict_set_str (this->options,
-                                    "rpc-auth.ports.insecure", "on");
-                if (ret == -1) {
-                        gf_log (GF_NFS, GF_LOG_ERROR, "dict_set_str error");
-                        goto free_foppool;
-                }
+    nfs->auth_cache_ttl_sec = GF_NFS_DEFAULT_AUTH_CACHE_TTL_SEC;
+    if (dict_get(this->options, "nfs.auth-cache-ttl-sec")) {
+        ret = dict_get_str(this->options, "nfs.auth-cache-ttl-sec", &optstr);
+        if (ret < 0) {
+            gf_msg(GF_NFS, GF_LOG_ERROR, -ret, NFS_MSG_PARSE_FAIL,
+                   "Failed to parse dict");
+            goto free_foppool;
         }
-	GF_OPTION_INIT (OPT_SERVER_RPC_STATD, nfs->rpc_statd, path, free_foppool);
 
-	GF_OPTION_INIT (OPT_SERVER_RPC_STATD_PIDFILE, nfs->rpc_statd_pid_file, path, free_foppool);
+        ret = gf_string2uint(optstr, &nfs->auth_cache_ttl_sec);
+        if (ret < 0) {
+            gf_msg(GF_NFS, GF_LOG_ERROR, errno, NFS_MSG_PARSE_FAIL,
+                   "Failed to parse uint string");
+            goto free_foppool;
+        }
+    }
 
-        GF_OPTION_INIT (OPT_SERVER_AUX_GIDS, nfs->server_aux_gids,
-                        bool, free_foppool);
-        GF_OPTION_INIT (OPT_SERVER_GID_CACHE_TIMEOUT,
-                        nfs->server_aux_gids_max_age,
-                        uint32, free_foppool);
+    /* TODO: Make this a configurable option in case we don't want to read
+     * exports/netgroup files off disk when they change. */
+    nfs->refresh_auth = 1;
 
-        if (gid_cache_init(&nfs->gid_cache, nfs->server_aux_gids_max_age) < 0) {
-                gf_log(GF_NFS, GF_LOG_ERROR, "Failed to initialize group cache.");
-                goto free_foppool;
+    nfs->rmtab = gf_strdup(NFS_DATADIR "/rmtab");
+    if (dict_get(this->options, "nfs.mount-rmtab")) {
+        ret = dict_get_str(this->options, "nfs.mount-rmtab", &nfs->rmtab);
+        if (ret == -1) {
+            gf_msg(GF_NFS, GF_LOG_ERROR, -ret, NFS_MSG_PARSE_FAIL,
+                   "Failed to parse dict");
+            goto free_foppool;
         }
 
-        ret = sys_access (nfs->rpc_statd, X_OK);
-        if (ret) {
-                gf_log (GF_NFS, GF_LOG_WARNING, "%s not enough permissions to"
-                        " access. Disabling NLM", nfs->rpc_statd);
-                nfs->enable_nlm = _gf_false;
+        /* check if writing the rmtab is disabled*/
+        if (nfs->rmtab && strcmp("/-", nfs->rmtab) == 0) {
+            GF_FREE(nfs->rmtab);
+            nfs->rmtab = NULL;
         }
+    }
 
-        ret = sys_stat (nfs->rpc_statd, &stbuf);
-        if (ret || !S_ISREG (stbuf.st_mode)) {
-                gf_log (GF_NFS, GF_LOG_WARNING, "%s not a regular file."
-                        " Disabling NLM", nfs->rpc_statd);
-                nfs->enable_nlm = _gf_false;
+    /* support both options rpc-auth.ports.insecure and
+     * rpc-auth-allow-insecure for backward compatibility
+     */
+    nfs->allow_insecure = 1;
+    if (dict_get(this->options, "rpc-auth.ports.insecure")) {
+        ret = dict_get_str(this->options, "rpc-auth.ports.insecure", &optstr);
+        if (ret < 0) {
+            gf_msg(GF_NFS, GF_LOG_ERROR, -ret, NFS_MSG_PARSE_FAIL,
+                   "Failed to parse dict");
+            goto free_foppool;
         }
 
-        nfs->rpcsvc =  rpcsvc_init (this, this->ctx,
-                                    this->options, fopspoolsize);
-        if (!nfs->rpcsvc) {
-                ret = -1;
-                gf_log (GF_NFS, GF_LOG_ERROR, "RPC service init failed");
-                goto free_foppool;
+        ret = gf_string2boolean(optstr, &boolt);
+        if (ret < 0) {
+            gf_msg(GF_NFS, GF_LOG_ERROR, errno, NFS_MSG_PARSE_FAIL,
+                   "Failed to parse bool "
+                   "string");
+            goto free_foppool;
         }
 
-        ret = rpcsvc_set_outstanding_rpc_limit (nfs->rpcsvc,
-                                                this->options,
-                                                RPCSVC_DEF_NFS_OUTSTANDING_RPC_LIMIT);
+        if (boolt == _gf_false)
+            nfs->allow_insecure = 0;
+    }
+
+    if (dict_get(this->options, "rpc-auth-allow-insecure")) {
+        ret = dict_get_str(this->options, "rpc-auth-allow-insecure", &optstr);
         if (ret < 0) {
-                gf_log (GF_NFS, GF_LOG_ERROR,
-                        "Failed to configure outstanding-rpc-limit");
-                goto free_foppool;
+            gf_msg(GF_NFS, GF_LOG_ERROR, -ret, NFS_MSG_PARSE_FAIL,
+                   "Failed to parse dict");
+            goto free_foppool;
         }
 
-        nfs->register_portmap = rpcsvc_register_portmap_enabled (nfs->rpcsvc);
+        ret = gf_string2boolean(optstr, &boolt);
+        if (ret < 0) {
+            gf_msg(GF_NFS, GF_LOG_ERROR, errno, NFS_MSG_PARSE_FAIL,
+                   "Failed to parse bool string");
+            goto free_foppool;
+        }
 
-        this->private = (void *)nfs;
-        INIT_LIST_HEAD (&nfs->versions);
-        nfs->generation = 1965;
+        if (boolt == _gf_false)
+            nfs->allow_insecure = 0;
+    }
 
-        ret = 0;
+    if (nfs->allow_insecure) {
+        /* blindly set both the options */
+        ret = dict_set_str(this->options, "rpc-auth-allow-insecure", "on");
+        if (ret == -1) {
+            gf_msg(GF_NFS, GF_LOG_ERROR, 0, NFS_MSG_DICT_SET_FAILED,
+                   "dict_set_str error");
+            goto free_foppool;
+        }
+        ret = dict_set_str(this->options, "rpc-auth.ports.insecure", "on");
+        if (ret == -1) {
+            gf_msg(GF_NFS, GF_LOG_ERROR, 0, NFS_MSG_DICT_SET_FAILED,
+                   "dict_set_str error");
+            goto free_foppool;
+        }
+    }
+
+    GF_OPTION_INIT("nfs.rdirplus", nfs->rdirplus, bool, free_foppool);
+
+    GF_OPTION_INIT(OPT_SERVER_RPC_STATD, nfs->rpc_statd, path, free_foppool);
+
+    GF_OPTION_INIT(OPT_SERVER_RPC_STATD_PIDFILE, nfs->rpc_statd_pid_file, path,
+                   free_foppool);
+
+    GF_OPTION_INIT(OPT_SERVER_AUX_GIDS, nfs->server_aux_gids, bool,
+                   free_foppool);
+    GF_OPTION_INIT(OPT_SERVER_GID_CACHE_TIMEOUT, nfs->server_aux_gids_max_age,
+                   uint32, free_foppool);
+
+    if (gid_cache_init(&nfs->gid_cache, nfs->server_aux_gids_max_age) < 0) {
+        gf_msg(GF_NFS, GF_LOG_ERROR, 0, NFS_MSG_INIT_GRP_CACHE_FAIL,
+               "Failed to initialize group cache.");
+        goto free_foppool;
+    }
+
+    ret = sys_access(nfs->rpc_statd, X_OK);
+    if (ret) {
+        gf_msg(GF_NFS, GF_LOG_WARNING, EPERM, NFS_MSG_NO_PERM,
+               "%s not enough permissions to access. Disabling NLM",
+               nfs->rpc_statd);
+        nfs->enable_nlm = _gf_false;
+    }
+
+    ret = sys_stat(nfs->rpc_statd, &stbuf);
+    if (ret || !S_ISREG(stbuf.st_mode)) {
+        gf_msg(GF_NFS, GF_LOG_WARNING, 0, NFS_MSG_REG_FILE_ERROR,
+               "%s not a regular file. Disabling NLM", nfs->rpc_statd);
+        nfs->enable_nlm = _gf_false;
+    }
+
+    nfs->rpcsvc = rpcsvc_init(this, this->ctx, this->options, fopspoolsize);
+    if (!nfs->rpcsvc) {
+        ret = -1;
+        gf_msg(GF_NFS, GF_LOG_ERROR, 0, NFS_MSG_RPC_INIT_FAIL,
+               "RPC service init failed");
+        goto free_foppool;
+    }
+
+    ret = rpcsvc_set_throttle_on(nfs->rpcsvc);
+    if (ret) {
+        gf_msg(GF_NFS, GF_LOG_ERROR, 0, NFS_MSG_ENABLE_THROTTLE_FAIL,
+               "Enabling throttle failed");
+        goto free_foppool;
+    }
+
+    ret = rpcsvc_set_outstanding_rpc_limit(
+        nfs->rpcsvc, this->options, RPCSVC_DEF_NFS_OUTSTANDING_RPC_LIMIT);
+    if (ret < 0) {
+        gf_msg(GF_NFS, GF_LOG_ERROR, 0, NFS_MSG_RPC_CONFIG_FAIL,
+               "Failed to configure outstanding-rpc-limit");
+        goto free_foppool;
+    }
+
+    nfs->register_portmap = rpcsvc_register_portmap_enabled(nfs->rpcsvc);
+
+    GF_OPTION_INIT("nfs.event-threads", nfs->event_threads, uint32,
+                   free_foppool);
+    gf_event_reconfigure_threads(this->ctx->event_pool, nfs->event_threads);
+
+    this->private = (void *)nfs;
+    INIT_LIST_HEAD(&nfs->versions);
+    nfs->generation = 1965;
+
+    ret = 0;
 
 free_foppool:
-        if (ret < 0)
-                mem_pool_destroy (nfs->foppool);
+    if (ret < 0)
+        mem_pool_destroy(nfs->foppool);
 
 free_rpcsvc:
-        /*
-         * rpcsvc_deinit */
-        if (ret < 0) {
-                GF_FREE (nfs);
-                nfs = NULL;
-        }
-
-        return nfs;
+    /*
+     * rpcsvc_deinit */
+    if (ret < 0) {
+        GF_FREE(nfs);
+        nfs = NULL;
+    }
+
+    return nfs;
 }
 
 int
-nfs_drc_init (xlator_t *this)
+nfs_drc_init(xlator_t *this)
 {
-        int       ret     = -1;
-        rpcsvc_t *svc     = NULL;
+    int ret = -1;
+    rpcsvc_t *svc = NULL;
 
-        GF_VALIDATE_OR_GOTO (GF_NFS, this, out);
-        GF_VALIDATE_OR_GOTO (GF_NFS, this->private, out);
+    GF_VALIDATE_OR_GOTO(GF_NFS, this, out);
+    GF_VALIDATE_OR_GOTO(GF_NFS, this->private, out);
 
-        svc = ((struct nfs_state *)(this->private))->rpcsvc;
-        if (!svc)
-                goto out;
+    svc = ((struct nfs_state *)(this->private))->rpcsvc;
+    if (!svc)
+        goto out;
 
-        ret = rpcsvc_drc_init (svc, this->options);
+    ret = rpcsvc_drc_init(svc, this->options);
 
- out:
-        return ret;
+out:
+    return ret;
 }
 
-int
-nfs_reconfigure_state (xlator_t *this, dict_t *options)
+static int
+nfs_reconfigure_state(xlator_t *this, dict_t *options)
 {
-        int                 ret = 0;
-        int                 keyindx = 0;
-        char                *rmtab = NULL;
-        char                *rpc_statd = NULL;
-        gf_boolean_t        optbool;
-        uint32_t            optuint32;
-        struct nfs_state    *nfs = NULL;
-        char                *blacklist_keys[] = {
-                                        "nfs.port",
-                                        "nfs.transport-type",
-                                        "nfs.mem-factor",
-                                        NULL};
-
-        GF_VALIDATE_OR_GOTO (GF_NFS, this, out);
-        GF_VALIDATE_OR_GOTO (GF_NFS, this->private, out);
-        GF_VALIDATE_OR_GOTO (GF_NFS, options, out);
-
-        nfs = (struct nfs_state *)this->private;
-
-        /* Black listed options can't be reconfigured, they need
-         * NFS to be restarted. There are two cases 1. SET 2. UNSET.
-         * 1. SET */
-        while (blacklist_keys[keyindx]) {
-                if (dict_get (options, blacklist_keys[keyindx])) {
-                        gf_log (GF_NFS, GF_LOG_ERROR,
-                                "Reconfiguring %s needs NFS restart",
-                                blacklist_keys[keyindx]);
-                        goto out;
-                }
-                keyindx ++;
-        }
-
-        /* UNSET for nfs.mem-factor */
-        if ((!dict_get (options, "nfs.mem-factor")) &&
-            (nfs->memfactor != GF_NFS_DEFAULT_MEMFACTOR)) {
-                gf_log (GF_NFS, GF_LOG_INFO,
-                        "Reconfiguring nfs.mem-factor needs NFS restart");
-                goto out;
-        }
-
-        /* UNSET for nfs.port */
-        if ((!dict_get (options, "nfs.port")) &&
-            (nfs->override_portnum)) {
-                gf_log (GF_NFS, GF_LOG_ERROR,
-                        "Reconfiguring nfs.port needs NFS restart");
-                goto out;
-        }
-
-        /* reconfig nfs.rpc-statd...  */
-        rpc_statd = GF_RPC_STATD_PROG;
-        if (dict_get (options, OPT_SERVER_RPC_STATD_PIDFILE)) {
-                ret = dict_get_str (options, "nfs.rpc-statd", &rpc_statd);
-                if (ret < 0) {
-                        gf_log (GF_NFS, GF_LOG_ERROR, "Failed to read "
-                                "reconfigured option: nfs.rpc-statd");
-                        goto out;
-                }
-        }
-
-        if (strcmp(nfs->rpc_statd, rpc_statd) != 0) {
-                gf_log (GF_NFS, GF_LOG_INFO,
-                        "Reconfiguring nfs.rpc-statd needs NFS restart");
-                goto out;
-        }
-
-        /* reconfig nfs.mount-rmtab */
-        rmtab = NFS_DATADIR "/rmtab";
-        if (dict_get (options, "nfs.mount-rmtab")) {
-                ret = dict_get_str (options, "nfs.mount-rmtab", &rmtab);
-                if (ret < 0) {
-                        gf_log (GF_NFS, GF_LOG_ERROR, "Failed to read "
-                                "reconfigured option: nfs.mount-rmtab");
-                        goto out;
-                }
-                gf_path_strip_trailing_slashes (rmtab);
-        }
-        if (strcmp (nfs->rmtab, rmtab) != 0) {
-                mount_rewrite_rmtab (nfs->mstate, rmtab);
-                gf_log (GF_NFS, GF_LOG_INFO,
-                                "Reconfigured nfs.mount-rmtab path: %s",
-                                nfs->rmtab);
-        }
-
-        GF_OPTION_RECONF (OPT_SERVER_AUX_GIDS, optbool,
-                                               options, bool, out);
-        if (nfs->server_aux_gids != optbool) {
-                nfs->server_aux_gids = optbool;
-                gf_log(GF_NFS, GF_LOG_INFO, "Reconfigured %s with value %d",
-                               OPT_SERVER_AUX_GIDS, optbool);
-        }
-
-        GF_OPTION_RECONF (OPT_SERVER_GID_CACHE_TIMEOUT, optuint32,
-                                               options, uint32, out);
-        if (nfs->server_aux_gids_max_age != optuint32) {
-                nfs->server_aux_gids_max_age = optuint32;
-                gid_cache_reconf (&nfs->gid_cache, optuint32);
-                gf_log(GF_NFS, GF_LOG_INFO, "Reconfigured %s with value %d",
-                               OPT_SERVER_GID_CACHE_TIMEOUT, optuint32);
-        }
-
-        /* reconfig nfs.dynamic-volumes */
-        ret = dict_get_str_boolean (options, "nfs.dynamic-volumes",
-                                             GF_NFS_DVM_OFF);
-        switch (ret) {
+    int ret = 0;
+    int keyindx = 0;
+    char *rmtab = NULL;
+    char *rpc_statd = NULL;
+    gf_boolean_t optbool;
+    uint32_t optuint32;
+    struct nfs_state *nfs = NULL;
+    static char *options_require_restart[] = {"nfs.port", "nfs.transport-type",
+                                              "nfs.mem-factor", NULL};
+
+    GF_VALIDATE_OR_GOTO(GF_NFS, this, out);
+    GF_VALIDATE_OR_GOTO(GF_NFS, this->private, out);
+    GF_VALIDATE_OR_GOTO(GF_NFS, options, out);
+
+    nfs = (struct nfs_state *)this->private;
+
+    /* Some listed options can't be reconfigured, they need
+     * NFS to be restarted. There are two cases 1. SET 2. UNSET.
+     * 1. SET */
+    while (options_require_restart[keyindx]) {
+        if (dict_get(options, options_require_restart[keyindx])) {
+            gf_msg(GF_NFS, GF_LOG_ERROR, 0, NFS_MSG_RECONFIG_FAIL,
+                   "Reconfiguring %s needs NFS restart",
+                   options_require_restart[keyindx]);
+            goto out;
+        }
+        keyindx++;
+    }
+
+    /* UNSET for nfs.mem-factor */
+    if ((!dict_get(options, "nfs.mem-factor")) &&
+        (nfs->memfactor != GF_NFS_DEFAULT_MEMFACTOR)) {
+        gf_msg(GF_NFS, GF_LOG_INFO, 0, NFS_MSG_RECONFIG_FAIL,
+               "Reconfiguring nfs.mem-factor needs NFS restart");
+        goto out;
+    }
+
+    /* UNSET for nfs.port */
+    if ((!dict_get(options, "nfs.port")) && (nfs->override_portnum)) {
+        gf_msg(GF_NFS, GF_LOG_ERROR, 0, NFS_MSG_RECONFIG_FAIL,
+               "Reconfiguring nfs.port needs NFS restart");
+        goto out;
+    }
+
+    /* reconfig nfs.rpc-statd...  */
+    rpc_statd = GF_RPC_STATD_PROG;
+    if (dict_get(options, OPT_SERVER_RPC_STATD_PIDFILE)) {
+        ret = dict_get_str(options, "nfs.rpc-statd", &rpc_statd);
+        if (ret < 0) {
+            gf_msg(GF_NFS, GF_LOG_ERROR, 0, NFS_MSG_READ_FAIL,
+                   "Failed to read reconfigured option: "
+                   "nfs.rpc-statd");
+            goto out;
+        }
+    }
+
+    if (strcmp(nfs->rpc_statd, rpc_statd) != 0) {
+        gf_msg(GF_NFS, GF_LOG_INFO, 0, NFS_MSG_RECONFIG_FAIL,
+               "Reconfiguring nfs.rpc-statd needs NFS restart");
+        goto out;
+    }
+
+    /* reconfig nfs.mount-rmtab */
+    rmtab = NFS_DATADIR "/rmtab";
+    if (dict_get(options, "nfs.mount-rmtab")) {
+        ret = dict_get_str(options, "nfs.mount-rmtab", &rmtab);
+        if (ret < 0) {
+            gf_msg(GF_NFS, GF_LOG_ERROR, 0, NFS_MSG_READ_FAIL,
+                   "Failed to read reconfigured option:"
+                   " nfs.mount-rmtab");
+            goto out;
+        }
+        gf_path_strip_trailing_slashes(rmtab);
+    }
+    /* check if writing the rmtab is disabled*/
+    if (strcmp("/-", rmtab) == 0) {
+        GF_FREE(nfs->rmtab);
+        nfs->rmtab = NULL;
+        gf_msg(GF_NFS, GF_LOG_INFO, 0, NFS_MSG_WRITE_FAIL,
+               "Disabled writing of nfs.mount-rmtab");
+    } else if (!nfs->rmtab || strcmp(nfs->rmtab, rmtab) != 0) {
+        mount_rewrite_rmtab(nfs->mstate, rmtab);
+        gf_msg(GF_NFS, GF_LOG_INFO, 0, NFS_MSG_RECONFIG_PATH,
+               "Reconfigured nfs.mount-rmtab path: %s", nfs->rmtab);
+    }
+
+    GF_OPTION_RECONF(OPT_SERVER_AUX_GIDS, optbool, options, bool, out);
+    if (nfs->server_aux_gids != optbool) {
+        nfs->server_aux_gids = optbool;
+        gf_msg(GF_NFS, GF_LOG_INFO, 0, NFS_MSG_RECONFIG_VALUE,
+               "Reconfigured %s with value %d", OPT_SERVER_AUX_GIDS, optbool);
+    }
+
+    GF_OPTION_RECONF(OPT_SERVER_GID_CACHE_TIMEOUT, optuint32, options, uint32,
+                     out);
+    if (nfs->server_aux_gids_max_age != optuint32) {
+        nfs->server_aux_gids_max_age = optuint32;
+        gid_cache_reconf(&nfs->gid_cache, optuint32);
+        gf_msg(GF_NFS, GF_LOG_INFO, 0, NFS_MSG_RECONFIG_VALUE,
+               "Reconfigured %s with value %d", OPT_SERVER_GID_CACHE_TIMEOUT,
+               optuint32);
+    }
+
+    GF_OPTION_RECONF("nfs.rdirplus", optbool, options, bool, out);
+    if (nfs->rdirplus != optbool) {
+        nfs->rdirplus = optbool;
+        gf_msg(GF_NFS, GF_LOG_INFO, 0, NFS_MSG_RECONFIG_VALUE,
+               "Reconfigured nfs.rdirplus with value %d", optbool);
+    }
+
+    /* reconfig nfs.dynamic-volumes */
+    ret = dict_get_str_boolean(options, "nfs.dynamic-volumes", GF_NFS_DVM_OFF);
+    switch (ret) {
         case GF_NFS_DVM_ON:
         case GF_NFS_DVM_OFF:
-                optbool = ret;
-                break;
+            optbool = ret;
+            break;
         default:
-                optbool = GF_NFS_DVM_OFF;
-                break;
-        }
-        if (nfs->dynamicvolumes != optbool) {
-                nfs->dynamicvolumes = optbool;
-                gf_log(GF_NFS, GF_LOG_INFO, "Reconfigured nfs.dynamic-volumes"
-                                            " with value %d", optbool);
-        }
-
-        optbool = _gf_false;
-        if (dict_get (options, "nfs.enable-ino32")) {
-                ret = dict_get_str_boolean (options, "nfs.enable-ino32",
-                                                      _gf_false);
-                if (ret < 0) {
-                        gf_log (GF_NFS, GF_LOG_ERROR,
-                                "Failed to read reconfigured option: "
-                                "nfs.enable-ino32");
-                        goto out;
-                }
-                optbool = ret;
-        }
-        if (nfs->enable_ino32 != optbool) {
-                nfs->enable_ino32 = optbool;
-                gf_log(GF_NFS, GF_LOG_INFO, "Reconfigured nfs.enable-ino32"
-                                            " with value %d", optbool);
-        }
-
-        /* nfs.nlm is enabled by default */
-        ret = dict_get_str_boolean (options, "nfs.nlm", _gf_true);
-        if (ret < 0) {
-                optbool = _gf_true;
-        } else {
-                optbool = ret;
-        }
-        if (nfs->enable_nlm != optbool) {
-                gf_log (GF_NFS, GF_LOG_INFO, "NLM is manually %s",
-                                (optbool ? "enabled":"disabled"));
-                nfs->enable_nlm = optbool;
-                nfs_reconfigure_nlm4 (this);
-        }
-
-        /* nfs.acl is enabled by default */
-        ret = dict_get_str_boolean (options, "nfs.acl", _gf_true);
+            optbool = GF_NFS_DVM_OFF;
+            break;
+    }
+    if (nfs->dynamicvolumes != optbool) {
+        nfs->dynamicvolumes = optbool;
+        gf_msg(GF_NFS, GF_LOG_INFO, 0, NFS_MSG_RECONFIG_VOL,
+               "Reconfigured nfs.dynamic-volumes with value %d", optbool);
+    }
+
+    optbool = _gf_false;
+    if (dict_get(options, "nfs.enable-ino32")) {
+        ret = dict_get_str_boolean(options, "nfs.enable-ino32", _gf_false);
         if (ret < 0) {
-                optbool = _gf_true;
-        } else {
-                optbool = ret;
-        }
-        if (nfs->enable_acl != optbool) {
-                gf_log (GF_NFS, GF_LOG_INFO, "ACL is manually %s",
-                                (optbool ? "enabled":"disabled"));
-                nfs->enable_acl = optbool;
-                nfs_reconfigure_acl3 (this);
-        }
-
-        ret = 0;
+            gf_msg(GF_NFS, GF_LOG_ERROR, 0, NFS_MSG_READ_FAIL,
+                   "Failed to read reconfigured option: "
+                   "nfs.enable-ino32");
+            goto out;
+        }
+        optbool = ret;
+    }
+    if (nfs->enable_ino32 != optbool) {
+        nfs->enable_ino32 = optbool;
+        gf_msg(GF_NFS, GF_LOG_INFO, 0, NFS_MSG_RECONFIG_ENABLE,
+               "Reconfigured nfs.enable-ino32 with value %d", optbool);
+    }
+
+    /* nfs.nlm is enabled by default */
+    ret = dict_get_str_boolean(options, "nfs.nlm", _gf_true);
+    if (ret < 0) {
+        optbool = _gf_true;
+    } else {
+        optbool = ret;
+    }
+    if (nfs->enable_nlm != optbool) {
+        gf_msg(GF_NFS, GF_LOG_INFO, 0, NFS_MSG_NLM_INFO,
+               "NLM is"
+               " manually %s",
+               (optbool ? "enabled" : "disabled"));
+        nfs->enable_nlm = optbool;
+        nfs_reconfigure_nlm4(this);
+    }
+
+    /* nfs.acl is enabled by default */
+    ret = dict_get_str_boolean(options, "nfs.acl", _gf_true);
+    if (ret < 0) {
+        optbool = _gf_true;
+    } else {
+        optbool = ret;
+    }
+    if (nfs->enable_acl != optbool) {
+        gf_msg(GF_NFS, GF_LOG_INFO, 0, NFS_MSG_ACL_INFO,
+               "ACL is "
+               "manually %s",
+               (optbool ? "enabled" : "disabled"));
+        nfs->enable_acl = optbool;
+        nfs_reconfigure_acl3(this);
+    }
+
+    GF_OPTION_RECONF("nfs.event-threads", nfs->event_threads, options, uint32,
+                     out);
+    gf_event_reconfigure_threads(this->ctx->event_pool, nfs->event_threads);
+
+    ret = 0;
 out:
-        return ret;
+    return ret;
 }
 
-
 /*
  * reconfigure() for NFS server xlator.
  */
 int
-reconfigure (xlator_t *this, dict_t *options)
+reconfigure(xlator_t *this, dict_t *options)
 {
-        int                      ret = 0;
-        struct nfs_state         *nfs = NULL;
-        gf_boolean_t             regpmap = _gf_true;
+    int ret = 0;
+    struct nfs_state *nfs = NULL;
+    gf_boolean_t regpmap = _gf_true;
 
-        if ((!this) || (!this->private) || (!options))
-                return (-1);
-
-        nfs = (struct nfs_state *)this->private;
+    if ((!this) || (!this->private) || (!options))
+        return (-1);
 
-        /* Reconfigure nfs options */
-        ret = nfs_reconfigure_state(this, options);
-        if (ret) {
-                gf_log (GF_NFS, GF_LOG_ERROR,
-                        "nfs reconfigure state failed");
-                return (-1);
-        }
+    nfs = (struct nfs_state *)this->private;
 
-        /* Reconfigure nfs3 options */
-        ret = nfs3_reconfigure_state(this, options);
-        if (ret) {
-                gf_log (GF_NFS, GF_LOG_ERROR,
-                        "nfs3 reconfigure state failed");
-                return (-1);
-        }
+    /* Reconfigure nfs options */
+    ret = nfs_reconfigure_state(this, options);
+    if (ret) {
+        gf_msg(GF_NFS, GF_LOG_ERROR, 0, NFS_MSG_RECONFIG_FAIL,
+               "nfs reconfigure state failed");
+        return (-1);
+    }
 
-        /* Reconfigure mount options */
-        ret = mount_reconfigure_state(this, options);
-        if (ret) {
-                gf_log (GF_NFS, GF_LOG_ERROR,
-                        "mount reconfigure state failed");
-                return (-1);
-        }
+    /* Reconfigure nfs3 options */
+    ret = nfs3_reconfigure_state(this, options);
+    if (ret) {
+        gf_msg(GF_NFS, GF_LOG_ERROR, 0, NFS_MSG_RECONFIG_FAIL,
+               "nfs3 reconfigure state failed");
+        return (-1);
+    }
 
-        /* Reconfigure rpc layer */
-        ret = rpcsvc_reconfigure_options (nfs->rpcsvc, options);
-        if (ret) {
-                gf_log (GF_NFS, GF_LOG_ERROR,
-                        "rpcsvc reconfigure options failed");
-                return (-1);
-        }
+    /* Reconfigure mount options */
+    ret = mount_reconfigure_state(this, options);
+    if (ret) {
+        gf_msg(GF_NFS, GF_LOG_ERROR, 0, NFS_MSG_RECONFIG_FAIL,
+               "mount reconfigure state failed");
+        return (-1);
+    }
 
-        /* Reconfigure rpc.outstanding-rpc-limit */
-        ret = rpcsvc_set_outstanding_rpc_limit (nfs->rpcsvc,
-                                       options,
-                                       RPCSVC_DEF_NFS_OUTSTANDING_RPC_LIMIT);
-        if (ret < 0) {
-                gf_log (GF_NFS, GF_LOG_ERROR,
-                        "Failed to reconfigure outstanding-rpc-limit");
-                return (-1);
-        }
+    /* Reconfigure rpc layer */
+    ret = rpcsvc_reconfigure_options(nfs->rpcsvc, options);
+    if (ret) {
+        gf_msg(GF_NFS, GF_LOG_ERROR, 0, NFS_MSG_RECONFIG_FAIL,
+               "rpcsvc reconfigure options failed");
+        return (-1);
+    }
+
+    /* Reconfigure rpc.outstanding-rpc-limit */
+    ret = rpcsvc_set_outstanding_rpc_limit(
+        nfs->rpcsvc, options, RPCSVC_DEF_NFS_OUTSTANDING_RPC_LIMIT);
+    if (ret < 0) {
+        gf_msg(GF_NFS, GF_LOG_ERROR, 0, NFS_MSG_RECONFIG_FAIL,
+               "Failed to reconfigure outstanding-rpc-limit");
+        return (-1);
+    }
 
-        regpmap = rpcsvc_register_portmap_enabled(nfs->rpcsvc);
-        if (nfs->register_portmap != regpmap) {
-                nfs->register_portmap = regpmap;
-                if (regpmap) {
-                        (void) nfs_program_register_portmap_all (nfs);
-                } else {
-                        (void) nfs_program_unregister_portmap_all (nfs);
-                }
+    regpmap = rpcsvc_register_portmap_enabled(nfs->rpcsvc);
+    if (nfs->register_portmap != regpmap) {
+        nfs->register_portmap = regpmap;
+        if (regpmap) {
+            (void)nfs_program_register_portmap_all(nfs);
+        } else {
+            (void)nfs_program_unregister_portmap_all(nfs);
         }
+    }
 
-        /* Reconfigure drc */
-        ret = rpcsvc_drc_reconfigure (nfs->rpcsvc, options);
-        if (ret) {
-                gf_log (GF_NFS, GF_LOG_ERROR,
-                        "rpcsvc DRC reconfigure failed");
-                return (-1);
-        }
+    /* Reconfigure drc */
+    ret = rpcsvc_drc_reconfigure(nfs->rpcsvc, options);
+    if (ret) {
+        gf_msg(GF_NFS, GF_LOG_ERROR, 0, NFS_MSG_RECONFIG_FAIL,
+               "rpcsvc DRC reconfigure failed");
+        return (-1);
+    }
 
-        return (0);
+    return (0);
 }
 
 /* Main init() routine for NFS server xlator. It inits NFS v3 protocol
@@ -1366,621 +1440,634 @@ reconfigure (xlator_t *this, dict_t *options)
  *      -1 (FAILURE) if any of them FAILS.
  */
 int
-init (xlator_t *this) {
-
-        struct nfs_state        *nfs = NULL;
-        int                     ret = -1;
+init(xlator_t *this)
+{
+    struct nfs_state *nfs = NULL;
+    int ret = -1;
 
-        if (!this)
-                return (-1);
+    if (!this)
+        return (-1);
 
-        nfs = nfs_init_state (this);
-        if (!nfs) {
-                gf_log (GF_NFS, GF_LOG_ERROR, "Failed to init nfs option");
-                return (-1);
-        }
+    nfs = nfs_init_state(this);
+    if (!nfs) {
+        gf_msg(GF_NFS, GF_LOG_ERROR, 0, NFS_MSG_OPT_INIT_FAIL,
+               "Failed to init nfs option");
+        return (-1);
+    }
 
-        ret = nfs_add_all_initiators (nfs);
-        if (ret) {
-                gf_log (GF_NFS, GF_LOG_ERROR, "Failed to add initiators");
-                return (-1);
-        }
+    ret = nfs_add_all_initiators(nfs);
+    if (ret) {
+        gf_msg(GF_NFS, GF_LOG_ERROR, 0, NFS_MSG_INIT_FAIL,
+               "Failed to add initiators");
+        return (-1);
+    }
 
-        ret = nfs_init_subvolumes (nfs, this->children);
-        if (ret) {
-                gf_log (GF_NFS, GF_LOG_CRITICAL, "Failed to init NFS exports");
-                return (-1);
-        }
+    ret = nfs_init_subvolumes(nfs, this->children);
+    if (ret) {
+        gf_msg(GF_NFS, GF_LOG_CRITICAL, 0, NFS_MSG_INIT_FAIL,
+               "Failed to init NFS exports");
+        return (-1);
+    }
 
-        ret = mount_init_state (this);
-        if (ret) {
-                gf_log (GF_NFS, GF_LOG_CRITICAL, "Failed to init Mount state");
-                return (-1);
-        }
+    ret = mount_init_state(this);
+    if (ret) {
+        gf_msg(GF_NFS, GF_LOG_CRITICAL, 0, NFS_MSG_INIT_FAIL,
+               "Failed to init Mount state");
+        return (-1);
+    }
 
-        ret = nlm4_init_state (this);
-        if (ret) {
-                gf_log (GF_NFS, GF_LOG_CRITICAL, "Failed to init NLM state");
-                return (-1);
-        }
+    ret = nlm4_init_state(this);
+    if (ret) {
+        gf_msg(GF_NFS, GF_LOG_CRITICAL, 0, NFS_MSG_INIT_FAIL,
+               "Failed to init NLM state");
+        return (-1);
+    }
 
-        ret = nfs_init_versions (nfs, this);
-        if (ret) {
-                gf_log (GF_NFS, GF_LOG_ERROR, "Failed to initialize protocols");
-                return (-1);
-        }
+    ret = nfs_init_versions(nfs, this);
+    if (ret) {
+        gf_msg(GF_NFS, GF_LOG_ERROR, 0, NFS_MSG_INIT_FAIL,
+               "Failed to initialize protocols");
+        return (-1);
+    }
 
-        ret = nfs_drc_init (this);
-        if (ret) {
-                gf_log (GF_NFS, GF_LOG_ERROR, "Failed to initialize DRC");
-                return (-1);
-        }
+    ret = nfs_drc_init(this);
+    if (ret) {
+        gf_msg(GF_NFS, GF_LOG_ERROR, 0, NFS_MSG_INIT_FAIL,
+               "Failed to initialize DRC");
+        return (-1);
+    }
 
-        gf_log (GF_NFS, GF_LOG_INFO, "NFS service started");
-        return (0); /* SUCCESS */
+    gf_msg(GF_NFS, GF_LOG_INFO, 0, NFS_MSG_STARTED, "NFS service started");
+    return (0); /* SUCCESS */
 }
 
-
 int
-notify (xlator_t *this, int32_t event, void *data, ...)
+notify(xlator_t *this, int32_t event, void *data, ...)
 {
-        xlator_t                *subvol = NULL;
-        struct nfs_state        *priv   = NULL;
+    xlator_t *subvol = NULL;
+    struct nfs_state *priv = NULL;
 
-        subvol = (xlator_t *)data;
+    subvol = (xlator_t *)data;
 
-        gf_log (GF_NFS, GF_LOG_TRACE, "Notification received: %d",
-                event);
+    gf_msg_trace(GF_NFS, 0, "Notification received: %d", event);
 
-        switch (event) {
+    switch (event) {
         case GF_EVENT_CHILD_UP:
-                nfs_startup_subvolume (this, subvol);
-                break;
+            nfs_startup_subvolume(this, subvol);
+            break;
 
-        case GF_EVENT_CHILD_MODIFIED:
-                priv = this->private;
-                ++(priv->generation);
-                break;
+        case GF_EVENT_SOME_DESCENDENT_DOWN:
+        case GF_EVENT_SOME_DESCENDENT_UP:
+            priv = this->private;
+            ++(priv->generation);
+            break;
 
         case GF_EVENT_PARENT_UP:
-                default_notify (this, GF_EVENT_PARENT_UP, data);
-                break;
-        }
+            default_notify(this, GF_EVENT_PARENT_UP, data);
+            break;
+    }
 
-        return 0;
+    return 0;
 }
 
-
-int
-fini (xlator_t *this)
+void
+fini(xlator_t *this)
 {
-
-        struct nfs_state        *nfs = NULL;
-
-        mnt3svc_deinit (this);
-        nfs = (struct nfs_state *)this->private;
-        gf_log (GF_NFS, GF_LOG_DEBUG, "NFS service going down");
-        nfs_deinit_versions (&nfs->versions, this);
-        return 0;
+    struct nfs_state *nfs = NULL;
+
+    mnt3svc_deinit(this);
+    nfs = (struct nfs_state *)this->private;
+    gf_msg_debug(GF_NFS, 0, "NFS service going down");
+    nfs_deinit_versions(&nfs->versions, this);
+    GF_FREE(this->instance_name);
+    return;
 }
 
 int32_t
-nfs_forget (xlator_t *this, inode_t *inode)
+nfs_forget(xlator_t *this, inode_t *inode)
 {
-        uint64_t                 ctx    = 0;
-        struct nfs_inode_ctx    *ictx   = NULL;
+    uint64_t ctx = 0;
+    struct nfs_inode_ctx *ictx = NULL;
 
-        if (inode_ctx_del (inode, this, &ctx))
-                return -1;
+    if (inode_ctx_del(inode, this, &ctx))
+        return -1;
 
-        ictx = (struct nfs_inode_ctx *)ctx;
-        GF_FREE (ictx);
+    ictx = (struct nfs_inode_ctx *)(uintptr_t)ctx;
+    GF_FREE(ictx);
 
-        return 0;
+    return 0;
 }
 
 gf_boolean_t
-_nfs_export_is_for_vol (char *exname, char *volname)
+_nfs_export_is_for_vol(char *exname, char *volname)
 {
-        gf_boolean_t    ret = _gf_false;
-        char            *tmp = NULL;
+    gf_boolean_t ret = _gf_false;
+    char *tmp = NULL;
 
-        tmp = exname;
-        if (tmp[0] == '/')
-                tmp++;
+    tmp = exname;
+    if (tmp[0] == '/')
+        tmp++;
 
-        if (!strcmp (tmp, volname))
-                ret = _gf_true;
+    if (!strcmp(tmp, volname))
+        ret = _gf_true;
 
-        return ret;
+    return ret;
 }
 
 int
-nfs_priv_to_dict (xlator_t *this, dict_t *dict)
+nfs_priv_to_dict(xlator_t *this, dict_t *dict, char *brickname)
 {
-        int                     ret = -1;
-        struct nfs_state        *priv = NULL;
-        struct mountentry       *mentry = NULL;
-        char                    *volname = NULL;
-        char                    key[1024] = {0,};
-        int                     count = 0;
-
-        GF_VALIDATE_OR_GOTO (THIS->name, this, out);
-        GF_VALIDATE_OR_GOTO (THIS->name, dict, out);
-
-        priv = this->private;
-        GF_ASSERT (priv);
+    int ret = -1;
+    struct nfs_state *priv = NULL;
+    struct mountentry *mentry = NULL;
+    char *volname = NULL;
+    char key[1024] = {
+        0,
+    };
+    int count = 0;
+
+    GF_VALIDATE_OR_GOTO(THIS->name, this, out);
+    GF_VALIDATE_OR_GOTO(THIS->name, dict, out);
+
+    priv = this->private;
+    GF_ASSERT(priv);
+
+    ret = dict_get_str(dict, "volname", &volname);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_ERROR, -ret, NFS_MSG_VOL_NOT_FOUND,
+               "Could not get volname");
+        goto out;
+    }
+
+    list_for_each_entry(mentry, &priv->mstate->mountlist, mlist)
+    {
+        if (!_nfs_export_is_for_vol(mentry->exname, volname))
+            continue;
+
+        snprintf(key, sizeof(key), "client%d.hostname", count);
+        ret = dict_set_str(dict, key, mentry->hostname);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, NFS_MSG_WRITE_FAIL,
+                   "Error writing hostname to dict");
+            goto out;
+        }
 
-        ret = dict_get_str (dict, "volname", &volname);
+        /* No connection data available yet in nfs server.
+         * Hence, setting to 0 to prevent cli failing
+         */
+        snprintf(key, sizeof(key), "client%d.bytesread", count);
+        ret = dict_set_uint64(dict, key, 0);
         if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "Could not get volname");
-                goto out;
+            gf_msg(this->name, GF_LOG_ERROR, 0, NFS_MSG_WRITE_FAIL,
+                   "Error writing bytes read to dict");
+            goto out;
         }
 
-        list_for_each_entry (mentry, &priv->mstate->mountlist, mlist) {
-                if (!_nfs_export_is_for_vol (mentry->exname, volname))
-                        continue;
-
-                memset (key, 0, sizeof (key));
-                snprintf (key, sizeof (key), "client%d.hostname", count);
-                ret = dict_set_str (dict, key, mentry->hostname);
-                if (ret) {
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "Error writing hostname to dict");
-                        goto out;
-                }
-
-                /* No connection data available yet in nfs server.
-                 * Hence, setting to 0 to prevent cli failing
-                 */
-                memset (key, 0, sizeof (key));
-                snprintf (key, sizeof (key), "client%d.bytesread", count);
-                ret = dict_set_uint64 (dict, key, 0);
-                if (ret) {
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "Error writing bytes read to dict");
-                        goto out;
-                }
-
-                memset (key, 0, sizeof (key));
-                snprintf (key, sizeof (key), "client%d.byteswrite", count);
-                ret = dict_set_uint64 (dict, key, 0);
-                if (ret) {
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "Error writing bytes write to dict");
-                        goto out;
-                }
-
-                count++;
+        snprintf(key, sizeof(key), "client%d.byteswrite", count);
+        ret = dict_set_uint64(dict, key, 0);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, NFS_MSG_WRITE_FAIL,
+                   "Error writing bytes write to dict");
+            goto out;
         }
 
-        ret = dict_set_int32 (dict, "clientcount", count);
-        if (ret)
-                gf_log (this->name, GF_LOG_ERROR,
-                        "Error writing client count to dict");
+        count++;
+    }
+
+    ret = dict_set_int32(dict, "clientcount", count);
+    if (ret)
+        gf_msg(this->name, GF_LOG_ERROR, 0, NFS_MSG_WRITE_FAIL,
+               "Error writing client count to dict");
 
 out:
-        gf_log (THIS->name, GF_LOG_DEBUG, "Returning %d", ret);
-        return ret;
+    gf_msg_debug(THIS->name, 0, "Returning %d", ret);
+    return ret;
 }
 
 extern int32_t
-nlm_priv (xlator_t *this);
+nlm_priv(xlator_t *this);
 
 int32_t
-nfs_priv (xlator_t *this)
+nfs_priv(xlator_t *this)
 {
-        int32_t ret = -1;
+    int32_t ret = -1;
+
+    /* DRC needs the global drc structure, xl is of no use to it. */
+    ret = rpcsvc_drc_priv(((struct nfs_state *)(this->private))->rpcsvc->drc);
+    if (ret) {
+        gf_msg_debug(this->name, 0, "Statedump of DRC failed");
+        goto out;
+    }
+
+    ret = nlm_priv(this);
+    if (ret) {
+        gf_msg_debug(this->name, 0, "Statedump of NLM failed");
+        goto out;
+    }
+out:
+    return ret;
+}
 
-        /* DRC needs the global drc structure, xl is of no use to it. */
-        ret = rpcsvc_drc_priv (((struct nfs_state *)(this->private))->rpcsvc->drc);
-        if (ret) {
-                gf_log (this->name, GF_LOG_DEBUG, "Statedump of DRC failed");
-                goto out;
-        }
+int32_t
+nfs_itable_dump(xlator_t *this)
+{
+    if (!this)
+        return -1;
 
-        ret = nlm_priv (this);
-        if (ret) {
-                gf_log (this->name, GF_LOG_DEBUG, "Statedump of NLM failed");
-                goto out;
-        }
- out:
-        return ret;
-}
+    if (this->next && this->next->itable) {
+        gf_proc_dump_add_section("xlator.nfs.itable");
+        inode_table_dump(this->next->itable, "xlator.nfs.itable");
+    }
 
+    return 0;
+}
 
 struct xlator_cbks cbks = {
-        .forget      = nfs_forget,
+    .forget = nfs_forget,
 };
 
 struct xlator_fops fops;
 
 struct xlator_dumpops dumpops = {
-        .priv           = nfs_priv,
-        .priv_to_dict   = nfs_priv_to_dict,
+    .priv = nfs_priv,
+    .priv_to_dict = nfs_priv_to_dict,
+    .inode = nfs_itable_dump,
 };
 
 /* TODO: If needed, per-volume options below can be extended to be export
  * specific also because after export-dir is introduced, a volume is not
  * necessarily an export whereas different subdirectories within that volume
  * can be and may need these options to be specified separately.
-*/
+ */
 struct volume_options options[] = {
-        { .key  = {"nfs3.read-size"},
-          .type = GF_OPTION_TYPE_SIZET,
-          .min  = GF_NFS3_RTMIN,
-          .max  = GF_NFS3_RTMAX,
-          .default_value = TOSTRING(GF_NFS3_RTPREF),
-          .description = "Size in which the client should issue read requests "
-                         "to the Gluster NFSv3 server. Must be a multiple of "
-                         "4KB (4096). Min and Max supported values are 4KB "
-                         "(4096) and 1MB (1048576) respectively. If the "
-                         "specified value is within the supported range but "
-                         "not a multiple of 4096, it is rounded up to the "
-                         "nearest multiple of 4096."
-        },
-        { .key  = {"nfs3.write-size"},
-          .type = GF_OPTION_TYPE_SIZET,
-          .min  = GF_NFS3_WTMIN,
-          .max  = GF_NFS3_WTMAX,
-          .default_value = TOSTRING(GF_NFS3_WTPREF),
-          .description = "Size in which the client should issue write requests "
-                         "to the Gluster NFSv3 server. Must be a multiple of "
-                         "1KB (1024). Min and Max supported values are "
-                         "4KB (4096) and 1MB(1048576) respectively. If the "
-                         "specified value is within the supported range but "
-                         "not a multiple of 4096, it is rounded up to the "
-                         "nearest multiple of 4096."
-        },
-        { .key  = {"nfs3.readdir-size"},
-          .type = GF_OPTION_TYPE_SIZET,
-          .min  = GF_NFS3_DTMIN,
-          .max  = GF_NFS3_DTMAX,
-          .default_value = TOSTRING(GF_NFS3_DTPREF),
-          .description = "Size in which the client should issue directory "
-                         "reading requests to the Gluster NFSv3 server. Must "
-                         "be a multiple of 1KB (1024). Min and Max supported "
-                         "values are 4KB (4096) and 1MB (1048576) respectively."
-                         "If the specified value is within the supported range "
-                         "but not a multiple of 4096, it is rounded up to the "
-                         "nearest multiple of 4096."
-        },
-        { .key  = {"nfs3.*.volume-access"},
-          .type = GF_OPTION_TYPE_STR,
-          .value = {"read-only", "read-write"},
-          .default_value = "read-write",
-          .description = "Type of access desired for this subvolume: "
-                         " read-only, read-write(default)"
-        },
-        { .key  = {"nfs3.*.trusted-write"},
-          .type = GF_OPTION_TYPE_BOOL,
-          .default_value = "off",
-          .description = "On an UNSTABLE write from client, return STABLE flag"
-                         " to force client to not send a COMMIT request. In "
-                         "some environments, combined with a replicated "
-                         "GlusterFS setup, this option can improve write "
-                         "performance. This flag allows user to trust Gluster"
-                         " replication logic to sync data to the disks and "
-                         "recover when required. COMMIT requests if received "
-                         "will be handled in a default manner by fsyncing."
-                         " STABLE writes are still handled in a sync manner. "
-                         "Off by default."
-
-        },
-        { .key  = {"nfs3.*.trusted-sync"},
-          .type = GF_OPTION_TYPE_BOOL,
-          .default_value = "off",
-          .description = "All writes and COMMIT requests are treated as async."
-                         " This implies that no write requests are guaranteed"
-                         " to be on server disks when the write reply is "
-                         "received at the NFS client. Trusted sync includes "
-                         " trusted-write behaviour. Off by default."
-
-        },
-        { .key  = {"nfs3.*.export-dir"},
-          .type = GF_OPTION_TYPE_PATH,
-          .default_value = "",
-          .description = "By default, all subvolumes of nfs are exported as "
-                         "individual exports. There are cases where a "
-                         "subdirectory or subdirectories in the volume need to "
-                         "be exported separately. This option can also be used "
-                         "in conjunction with nfs3.export-volumes option to "
-                         "restrict exports only to the subdirectories specified"
-                         " through this option. Must be an absolute path. Along"
-                         " with path allowed list of IPs/hostname can be "
-                         "associated with each subdirectory. If provided "
-                         "connection will allowed only from these IPs. By "
-                         "default connections from all IPs are allowed. "
-                         "Format: <dir>[(hostspec[|hostspec|...])][,...]. Where"
-                         " hostspec can be an IP address, hostname or an IP "
-                         "range in CIDR notation. "
-                         "e.g. /foo(192.168.1.0/24|host1|10.1.1.8),/host2."
-                         " NOTE: Care must be taken while configuring this "
-                         "option as invalid entries and/or unreachable DNS "
-                         "servers can introduce unwanted delay in all the mount"
-                         " calls."
-        },
-        { .key  = {"nfs3.export-dirs"},
-          .type = GF_OPTION_TYPE_BOOL,
-          .default_value = "on",
-          .description = "By default, all subvolumes of nfs are exported as "
-                         "individual exports. There are cases where a "
-                         "subdirectory or subdirectories in the volume need to "
-                         "be exported separately. Enabling this option allows "
-                         "any directory on a volumes to be exported separately."
-                         "Directory exports are enabled by default."
-        },
-        { .key  = {"nfs3.export-volumes"},
-          .type = GF_OPTION_TYPE_BOOL,
-          .default_value = "on",
-          .description = "Enable or disable exporting whole volumes, instead "
-                         "if used in conjunction with nfs3.export-dir, can "
-                         "allow setting up only subdirectories as exports. On "
-                         "by default."
-        },
-        { .key  = {"rpc-auth.auth-unix"},
-          .type = GF_OPTION_TYPE_BOOL,
-          .default_value = "on",
-          .description = "Disable or enable the AUTH_UNIX authentication type."
-                         "Must always be enabled for better interoperability. "
-                         "However, can be disabled if needed. Enabled by "
-                         "default"
-        },
-        { .key  = {"rpc-auth.auth-null"},
-          .type = GF_OPTION_TYPE_BOOL,
-          .default_value = "on",
-          .description = "Disable or enable the AUTH_NULL authentication type."
-                         "Must always be enabled. This option is here only to"
-                         " avoid unrecognized option warnings"
-        },
-        { .key  = {"rpc-auth.auth-unix.*"},
-          .type = GF_OPTION_TYPE_BOOL,
-          .default_value = "on",
-          .description = "Disable or enable the AUTH_UNIX authentication type "
-                         "for a particular exported volume overriding defaults"
-                         " and general setting for AUTH_UNIX scheme. Must "
-                         "always be enabled for better interoperability. "
-                         "However, can be disabled if needed. Enabled by "
-                         "default."
-        },
-        { .key  = {"rpc-auth.auth-unix.*.allow"},
-          .type = GF_OPTION_TYPE_STR,
-          .default_value = "on",
-          .description = "Disable or enable the AUTH_UNIX authentication type "
-                         "for a particular exported volume overriding defaults"
-                         " and general setting for AUTH_UNIX scheme. Must "
-                         "always be enabled for better interoperability. "
-                         "However, can be disabled if needed. Enabled by "
-                         "default."
-        },
-        { .key  = {"rpc-auth.auth-null.*"},
-          .type = GF_OPTION_TYPE_BOOL,
-          .default_value = "on",
-          .description = "Disable or enable the AUTH_NULL authentication type "
-                         "for a particular exported volume overriding defaults"
-                         " and general setting for AUTH_NULL. Must always be "
-                         "enabled. This option is here only to avoid "
-                         "unrecognized option warnings."
-        },
-        { .key  = {"rpc-auth.addr.allow"},
-          .type = GF_OPTION_TYPE_CLIENT_AUTH_ADDR,
-          .default_value = "all",
-          .description = "Allow a comma separated list of addresses and/or"
-                         " hostnames to connect to the server. By default, all"
-                         " connections are allowed. This allows users to "
-                         "define a general rule for all exported volumes."
-        },
-        { .key  = {"rpc-auth.addr.reject"},
-          .type = GF_OPTION_TYPE_CLIENT_AUTH_ADDR,
-          .default_value = "none",
-          .description = "Reject a comma separated list of addresses and/or"
-                         " hostnames from connecting to the server. By default,"
-                         " all connections are allowed. This allows users to"
-                         "define a general rule for all exported volumes."
-        },
-        { .key  = {"rpc-auth.addr.*.allow"},
-          .type = GF_OPTION_TYPE_CLIENT_AUTH_ADDR,
-          .default_value = "all",
-          .description = "Allow a comma separated list of addresses and/or"
-                         " hostnames to connect to the server. By default, all"
-                         " connections are allowed. This allows users to "
-                         "define a rule for a specific exported volume."
-        },
-        { .key  = {"rpc-auth.addr.*.reject"},
-          .type = GF_OPTION_TYPE_CLIENT_AUTH_ADDR,
-          .default_value = "none",
-          .description = "Reject a comma separated list of addresses and/or"
-                         " hostnames from connecting to the server. By default,"
-                         " all connections are allowed. This allows users to "
-                         "define a rule for a specific exported volume."
-        },
-        { .key  = {"rpc-auth.ports.insecure"},
-          .type = GF_OPTION_TYPE_BOOL,
-          .default_value = "off",
-          .description = "Allow client connections from unprivileged ports. By "
-                         "default only privileged ports are allowed. This is a"
-                         "global setting in case insecure ports are to be "
-                         "enabled for all exports using a single option."
-        },
-        { .key  = {"rpc-auth.ports.*.insecure"},
-          .type = GF_OPTION_TYPE_BOOL,
-          .default_value = "off",
-          .description = "Allow client connections from unprivileged ports. By "
-                         "default only privileged ports are allowed. Use this"
-                         " option to enable or disable insecure ports for "
-                         "a specific subvolume and to override the global "
-                         "setting set by the previous option."
-        },
-        { .key  = {"rpc-auth.addr.namelookup"},
-          .type = GF_OPTION_TYPE_BOOL,
-          .default_value = "off",
-          .description = "Users have the option of turning on name lookup for"
-                  " incoming client connections using this option. Use this "
-                  "option to turn on name lookups during address-based "
-                  "authentication. Turning this on will enable you to"
-                  " use hostnames in nfs.rpc-auth-* filters. In some "
-                  "setups, the name server can take too long to reply to DNS "
-                  "queries resulting in timeouts of mount requests. By "
-                  "default, name lookup is off"
-        },
-        { .key  = {"nfs.dynamic-volumes"},
-          .type = GF_OPTION_TYPE_BOOL,
-          .default_value = "off",
-          .description = "Internal option set to tell gnfs to use a different"
-                         " scheme for encoding file handles when DVM is being"
-                         " used."
-        },
-        { .key  = {"nfs3.*.volume-id"},
-          .type = GF_OPTION_TYPE_STR,
-          .default_value = "",
-          .description = "When nfs.dynamic-volumes is set, gnfs expects every "
-                         "subvolume to have this option set for it, so that "
-                         "gnfs can use this option to identify the volume. "
-                         "If all subvolumes do not have this option set, an "
-                         "error is reported."
-        },
-        { .key  = {"nfs.enable-ino32"},
-          .type = GF_OPTION_TYPE_BOOL,
-          .default_value = "no",
-          .description = "For nfs clients or apps that do not support 64-bit "
-                         "inode numbers, use this option to make NFS return "
-                         "32-bit inode numbers instead. Disabled by default, so"
-                         " NFS returns 64-bit inode numbers."
-        },
-        { .key  = {"rpc.register-with-portmap"},
-          .type = GF_OPTION_TYPE_BOOL,
-          .default_value = "on",
-          .description = "For systems that need to run multiple nfs servers, "
-                         "only one registration is possible with "
-                         "portmap service. Use this option to turn off portmap "
-                         "registration for Gluster NFS. On by default"
-        },
-        { .key  = {"rpc.outstanding-rpc-limit"},
-          .type = GF_OPTION_TYPE_INT,
-          .min  = RPCSVC_MIN_OUTSTANDING_RPC_LIMIT,
-          .max  = RPCSVC_MAX_OUTSTANDING_RPC_LIMIT,
-          .default_value = TOSTRING(RPCSVC_DEF_NFS_OUTSTANDING_RPC_LIMIT),
-          .description = "Parameter to throttle the number of incoming RPC "
-                         "requests from a client. 0 means no limit (can "
-                         "potentially run out of memory)"
-        },
-        { .key  = {"nfs.port"},
-          .type = GF_OPTION_TYPE_INT,
-          .min  = 1,
-          .max  = 0xffff,
-          .default_value = TOSTRING(GF_NFS3_PORT),
-          .description = "Use this option on systems that need Gluster NFS to "
-                         "be associated with a non-default port number."
-        },
-        { .key  = {"nfs.mem-factor"},
-          .type = GF_OPTION_TYPE_INT,
-          .min  = 1,
-          .max  = 1024,
-          .default_value = TOSTRING(GF_NFS_DEFAULT_MEMFACTOR),
-          .description = "Use this option to make NFS be faster on systems by "
-                         "using more memory. This option specifies a multiple "
-                         "that determines the total amount of memory used. "
-                         "Default value is 15. Increase to use more memory in "
-                         "order to improve performance for certain use cases."
-                         "Please consult gluster-users list before using this "
-                         "option."
-        },
-        { .key  = {"nfs.*.disable"},
-          .type = GF_OPTION_TYPE_BOOL,
-          .default_value = "false",
-          .description = "This option is used to start or stop the NFS server "
-                         "for individual volumes."
-        },
-        { .key  = {"nfs.nlm"},
-          .type = GF_OPTION_TYPE_BOOL,
-          .default_value = "on",
-          .description = "This option, if set to 'off', disables NLM server "
-                         "by not registering the service with the portmapper."
-                         " Set it to 'on' to re-enable it. Default value: 'on'"
-        },
-
-        { .key = {"nfs.mount-udp"},
-          .type = GF_OPTION_TYPE_BOOL,
-          .default_value = "off",
-          .description = "set the option to 'on' to enable mountd on UDP. "
-                         "Required for some Solaris and AIX NFS clients. "
-                         "The need for enabling this option often depends "
-                         "on the usage of NLM."
-        },
-        { .key = {"nfs.mount-rmtab"},
-          .type = GF_OPTION_TYPE_PATH,
-          .default_value = NFS_DATADIR "/rmtab",
-          .description = "Set the location of the cache file that is used to "
-                         "list all the NFS-clients that have connected "
-                         "through the MOUNT protocol. If this is on shared "
-                         "storage, all GlusterFS servers will update and "
-                         "output (with 'showmount') the same list."
-        },
-        { .key = {OPT_SERVER_RPC_STATD},
-          .type = GF_OPTION_TYPE_PATH,
-          .default_value = GF_RPC_STATD_PROG,
-          .description = "The executable of RPC statd utility. "
-                         "Defaults to " GF_RPC_STATD_PROG
-        },
-        { .key = {OPT_SERVER_RPC_STATD_PIDFILE},
-          .type = GF_OPTION_TYPE_PATH,
-          .default_value = GF_RPC_STATD_PIDFILE,
-          .description = "The pid file of RPC statd utility. "
-                         "Defaults to " GF_RPC_STATD_PIDFILE
-        },
-        { .key = {OPT_SERVER_AUX_GIDS},
-          .type = GF_OPTION_TYPE_BOOL,
-          .default_value = "off",
-          .description = "Let the server look up which groups a user belongs "
-                         "to, overwriting the list passed from the client. "
-                         "This enables support for group lists longer than "
-                         "can be passed through the NFS protocol, but is not "
-                         "secure unless users and groups are well synchronized "
-                         "between clients and servers."
-        },
-        { .key = {OPT_SERVER_GID_CACHE_TIMEOUT},
-          .type = GF_OPTION_TYPE_INT,
-          .min = 0,
-          .max = 3600,
-          .default_value = "5",
-          .description = "Number of seconds to cache auxiliary-GID data, when "
-                         OPT_SERVER_AUX_GIDS " is set."
-        },
-        { .key = {"nfs.acl"},
-          .type = GF_OPTION_TYPE_BOOL,
-          .default_value = "on",
-          .description = "This option is used to control ACL support for NFS."
-        },
-        { .key  = {"nfs.drc"},
-          .type = GF_OPTION_TYPE_STR,
-          .default_value = "off",
-          .description = "Enable Duplicate Request Cache in gNFS server to "
-                         "improve correctness of non-idempotent operations like "
-                         "write, delete, link, et al"
-        },
-        { .key  = {"nfs.drc-size"},
-          .type = GF_OPTION_TYPE_INT,
-          .default_value = "0x20000",
-          .description = "Sets the number of non-idempotent "
-                         "requests to cache in drc"
-        },
-        { .key = {"nfs.exports-auth-enable"},
-          .type = GF_OPTION_TYPE_BOOL,
-          .description = "Set the option to 'on' to enable exports/netgroup "
-                         "authentication in the NFS server and mount daemon."
-        },
-
-        { .key = {"nfs.auth-refresh-interval-sec"},
-          .type = GF_OPTION_TYPE_INT,
-          .description = "Frequency in seconds that the daemon should check for"
-                         " changes in the exports/netgroups file."
-        },
-
-       { .key = {"nfs.auth-cache-ttl-sec"},
-          .type = GF_OPTION_TYPE_INT,
-          .description = "Sets the TTL of an entry in the auth cache. Value is "
-                         "in seconds."
-        },
-
-        { .key  = {NULL} },
+    {.key = {"nfs3.read-size"},
+     .type = GF_OPTION_TYPE_SIZET,
+     .min = GF_NFS3_RTMIN,
+     .max = GF_NFS3_RTMAX,
+     .default_value = TOSTRING(GF_NFS3_RTPREF),
+     .description = "Size in which the client should issue read requests "
+                    "to the Gluster NFSv3 server. Must be a multiple of "
+                    "4KB (4096). Min and Max supported values are 4KB "
+                    "(4096) and 1MB (1048576) respectively. If the "
+                    "specified value is within the supported range but "
+                    "not a multiple of 4096, it is rounded up to the "
+                    "nearest multiple of 4096."},
+    {.key = {"nfs3.write-size"},
+     .type = GF_OPTION_TYPE_SIZET,
+     .min = GF_NFS3_WTMIN,
+     .max = GF_NFS3_WTMAX,
+     .default_value = TOSTRING(GF_NFS3_WTPREF),
+     .description = "Size in which the client should issue write requests "
+                    "to the Gluster NFSv3 server. Must be a multiple of "
+                    "1KB (1024). Min and Max supported values are "
+                    "4KB (4096) and 1MB(1048576) respectively. If the "
+                    "specified value is within the supported range but "
+                    "not a multiple of 4096, it is rounded up to the "
+                    "nearest multiple of 4096."},
+    {.key = {"nfs3.readdir-size"},
+     .type = GF_OPTION_TYPE_SIZET,
+     .min = GF_NFS3_DTMIN,
+     .max = GF_NFS3_DTMAX,
+     .default_value = TOSTRING(GF_NFS3_DTPREF),
+     .description = "Size in which the client should issue directory "
+                    "reading requests to the Gluster NFSv3 server. Must "
+                    "be a multiple of 1KB (1024). Min and Max supported "
+                    "values are 4KB (4096) and 1MB (1048576) respectively."
+                    "If the specified value is within the supported range "
+                    "but not a multiple of 4096, it is rounded up to the "
+                    "nearest multiple of 4096."},
+    {.key = {"nfs3.*.volume-access"},
+     .type = GF_OPTION_TYPE_STR,
+     .value = {"read-only", "read-write"},
+     .default_value = "read-write",
+     .description = "Type of access desired for this subvolume: "
+                    " read-only, read-write(default)"},
+    {.key = {"nfs3.*.trusted-write"},
+     .type = GF_OPTION_TYPE_BOOL,
+     .default_value = "off",
+     .description = "On an UNSTABLE write from client, return STABLE flag"
+                    " to force client to not send a COMMIT request. In "
+                    "some environments, combined with a replicated "
+                    "GlusterFS setup, this option can improve write "
+                    "performance. This flag allows user to trust Gluster"
+                    " replication logic to sync data to the disks and "
+                    "recover when required. COMMIT requests if received "
+                    "will be handled in a default manner by fsyncing."
+                    " STABLE writes are still handled in a sync manner. "
+                    "Off by default."
+
+    },
+    {.key = {"nfs3.*.trusted-sync"},
+     .type = GF_OPTION_TYPE_BOOL,
+     .default_value = "off",
+     .description = "All writes and COMMIT requests are treated as async."
+                    " This implies that no write requests are guaranteed"
+                    " to be on server disks when the write reply is "
+                    "received at the NFS client. Trusted sync includes "
+                    " trusted-write behaviour. Off by default."
+
+    },
+    {.key = {"nfs3.*.export-dir"},
+     .type = GF_OPTION_TYPE_PATH,
+     .default_value = "",
+     .description = "By default, all subvolumes of nfs are exported as "
+                    "individual exports. There are cases where a "
+                    "subdirectory or subdirectories in the volume need to "
+                    "be exported separately. This option can also be used "
+                    "in conjunction with nfs3.export-volumes option to "
+                    "restrict exports only to the subdirectories specified"
+                    " through this option. Must be an absolute path. Along"
+                    " with path allowed list of IPs/hostname can be "
+                    "associated with each subdirectory. If provided "
+                    "connection will allowed only from these IPs. By "
+                    "default connections from all IPs are allowed. "
+                    "Format: <dir>[(hostspec[|hostspec|...])][,...]. Where"
+                    " hostspec can be an IP address, hostname or an IP "
+                    "range in CIDR notation. "
+                    "e.g. /foo(192.168.1.0/24|host1|10.1.1.8),/host2."
+                    " NOTE: Care must be taken while configuring this "
+                    "option as invalid entries and/or unreachable DNS "
+                    "servers can introduce unwanted delay in all the mount"
+                    " calls."},
+    {.key = {"nfs3.export-dirs"},
+     .type = GF_OPTION_TYPE_BOOL,
+     .default_value = "on",
+     .description = "By default, all subvolumes of nfs are exported as "
+                    "individual exports. There are cases where a "
+                    "subdirectory or subdirectories in the volume need to "
+                    "be exported separately. Enabling this option allows "
+                    "any directory on a volumes to be exported separately."
+                    "Directory exports are enabled by default."},
+    {.key = {"nfs3.export-volumes"},
+     .type = GF_OPTION_TYPE_BOOL,
+     .default_value = "on",
+     .description = "Enable or disable exporting whole volumes, instead "
+                    "if used in conjunction with nfs3.export-dir, can "
+                    "allow setting up only subdirectories as exports. On "
+                    "by default."},
+    {.key = {"rpc-auth.auth-unix"},
+     .type = GF_OPTION_TYPE_BOOL,
+     .default_value = "on",
+     .description = "Disable or enable the AUTH_UNIX authentication type."
+                    "Must always be enabled for better interoperability. "
+                    "However, can be disabled if needed. Enabled by "
+                    "default"},
+    {.key = {"rpc-auth.auth-null"},
+     .type = GF_OPTION_TYPE_BOOL,
+     .default_value = "on",
+     .description = "Disable or enable the AUTH_NULL authentication type."
+                    "Must always be enabled. This option is here only to"
+                    " avoid unrecognized option warnings"},
+    {.key = {"rpc-auth.auth-unix.*"},
+     .type = GF_OPTION_TYPE_BOOL,
+     .default_value = "on",
+     .description = "Disable or enable the AUTH_UNIX authentication type "
+                    "for a particular exported volume overriding defaults"
+                    " and general setting for AUTH_UNIX scheme. Must "
+                    "always be enabled for better interoperability. "
+                    "However, can be disabled if needed. Enabled by "
+                    "default."},
+    {.key = {"rpc-auth.auth-unix.*.allow"},
+     .type = GF_OPTION_TYPE_STR,
+     .default_value = "on",
+     .description = "Disable or enable the AUTH_UNIX authentication type "
+                    "for a particular exported volume overriding defaults"
+                    " and general setting for AUTH_UNIX scheme. Must "
+                    "always be enabled for better interoperability. "
+                    "However, can be disabled if needed. Enabled by "
+                    "default."},
+    {.key = {"rpc-auth.auth-null.*"},
+     .type = GF_OPTION_TYPE_BOOL,
+     .default_value = "on",
+     .description = "Disable or enable the AUTH_NULL authentication type "
+                    "for a particular exported volume overriding defaults"
+                    " and general setting for AUTH_NULL. Must always be "
+                    "enabled. This option is here only to avoid "
+                    "unrecognized option warnings."},
+    {.key = {"rpc-auth.addr.allow"},
+     .type = GF_OPTION_TYPE_CLIENT_AUTH_ADDR,
+     .default_value = "all",
+     .description = "Allow a comma separated list of addresses and/or"
+                    " hostnames to connect to the server. By default, all"
+                    " connections are allowed. This allows users to "
+                    "define a general rule for all exported volumes."},
+    {.key = {"rpc-auth.addr.reject"},
+     .type = GF_OPTION_TYPE_CLIENT_AUTH_ADDR,
+     .default_value = "none",
+     .description = "Reject a comma separated list of addresses and/or"
+                    " hostnames from connecting to the server. By default,"
+                    " all connections are allowed. This allows users to "
+                    "define a general rule for all exported volumes."},
+    {.key = {"rpc-auth.addr.*.allow"},
+     .type = GF_OPTION_TYPE_CLIENT_AUTH_ADDR,
+     .default_value = "all",
+     .description = "Allow a comma separated list of addresses and/or"
+                    " hostnames to connect to the server. By default, all"
+                    " connections are allowed. This allows users to "
+                    "define a rule for a specific exported volume."},
+    {.key = {"rpc-auth.addr.*.reject"},
+     .type = GF_OPTION_TYPE_CLIENT_AUTH_ADDR,
+     .default_value = "none",
+     .description = "Reject a comma separated list of addresses and/or"
+                    " hostnames from connecting to the server. By default,"
+                    " all connections are allowed. This allows users to "
+                    "define a rule for a specific exported volume."},
+    {.key = {"rpc-auth.ports.insecure"},
+     .type = GF_OPTION_TYPE_BOOL,
+     .default_value = "off",
+     .description = "Allow client connections from unprivileged ports. By "
+                    "default only privileged ports are allowed. This is a"
+                    "global setting in case insecure ports are to be "
+                    "enabled for all exports using a single option."},
+    {.key = {"rpc-auth.ports.*.insecure"},
+     .type = GF_OPTION_TYPE_BOOL,
+     .default_value = "off",
+     .description = "Allow client connections from unprivileged ports. By "
+                    "default only privileged ports are allowed. Use this"
+                    " option to enable or disable insecure ports for "
+                    "a specific subvolume and to override the global "
+                    "setting set by the previous option."},
+    {.key = {"rpc-auth.addr.namelookup"},
+     .type = GF_OPTION_TYPE_BOOL,
+     .default_value = "off",
+     .description = "Users have the option of turning on name lookup for"
+                    " incoming client connections using this option. Use this "
+                    "option to turn on name lookups during address-based "
+                    "authentication. Turning this on will enable you to"
+                    " use hostnames in nfs.rpc-auth-* filters. In some "
+                    "setups, the name server can take too long to reply to DNS "
+                    "queries resulting in timeouts of mount requests. By "
+                    "default, name lookup is off"},
+    {.key = {"nfs.dynamic-volumes"},
+     .type = GF_OPTION_TYPE_BOOL,
+     .default_value = "off",
+     .description = "Internal option set to tell gnfs to use a different"
+                    " scheme for encoding file handles when DVM is being"
+                    " used."},
+    {.key = {"nfs3.*.volume-id"},
+     .type = GF_OPTION_TYPE_STR,
+     .default_value = "",
+     .description = "When nfs.dynamic-volumes is set, gnfs expects every "
+                    "subvolume to have this option set for it, so that "
+                    "gnfs can use this option to identify the volume. "
+                    "If all subvolumes do not have this option set, an "
+                    "error is reported."},
+    {.key = {"nfs.enable-ino32"},
+     .type = GF_OPTION_TYPE_BOOL,
+     .default_value = "no",
+     .description = "For nfs clients or apps that do not support 64-bit "
+                    "inode numbers, use this option to make NFS return "
+                    "32-bit inode numbers instead. Disabled by default, so"
+                    " NFS returns 64-bit inode numbers."},
+    {.key = {"rpc.register-with-portmap"},
+     .type = GF_OPTION_TYPE_BOOL,
+     .default_value = "on",
+     .description = "For systems that need to run multiple nfs servers, "
+                    "only one registration is possible with "
+                    "portmap service. Use this option to turn off portmap "
+                    "registration for Gluster NFS. On by default"},
+    {.key = {"rpc.outstanding-rpc-limit"},
+     .type = GF_OPTION_TYPE_INT,
+     .min = RPCSVC_MIN_OUTSTANDING_RPC_LIMIT,
+     .max = RPCSVC_MAX_OUTSTANDING_RPC_LIMIT,
+     .default_value = TOSTRING(RPCSVC_DEF_NFS_OUTSTANDING_RPC_LIMIT),
+     .description = "Parameter to throttle the number of incoming RPC "
+                    "requests from a client. 0 means no limit (can "
+                    "potentially run out of memory)"},
+    {.key = {"nfs.port"},
+     .type = GF_OPTION_TYPE_INT,
+     .min = 1,
+     .max = 0xffff,
+     .default_value = TOSTRING(GF_NFS3_PORT),
+     .description = "Use this option on systems that need Gluster NFS to "
+                    "be associated with a non-default port number."},
+    {.key = {"nfs.mem-factor"},
+     .type = GF_OPTION_TYPE_INT,
+     .min = 1,
+     .max = 1024,
+     .default_value = TOSTRING(GF_NFS_DEFAULT_MEMFACTOR),
+     .description = "Use this option to make NFS be faster on systems by "
+                    "using more memory. This option specifies a multiple "
+                    "that determines the total amount of memory used. "
+                    "Default value is 15. Increase to use more memory in "
+                    "order to improve performance for certain use cases."
+                    "Please consult gluster-users list before using this "
+                    "option."},
+    {.key = {"nfs.*.disable"},
+     .type = GF_OPTION_TYPE_BOOL,
+     .default_value = "false",
+     .description = "This option is used to start or stop the NFS server "
+                    "for individual volumes."},
+    {.key = {"nfs.nlm"},
+     .type = GF_OPTION_TYPE_BOOL,
+     .default_value = "on",
+     .description = "This option, if set to 'off', disables NLM server "
+                    "by not registering the service with the portmapper."
+                    " Set it to 'on' to re-enable it. Default value: 'on'"},
+
+    {.key = {"nfs.mount-udp"},
+     .type = GF_OPTION_TYPE_BOOL,
+     .default_value = "off",
+     .description = "set the option to 'on' to enable mountd on UDP. "
+                    "Required for some Solaris and AIX NFS clients. "
+                    "The need for enabling this option often depends "
+                    "on the usage of NLM."},
+    {.key = {"nfs.mount-rmtab"},
+     .type = GF_OPTION_TYPE_PATH,
+     .default_value = NFS_DATADIR "/rmtab",
+     .description = "Set the location of the cache file that is used to "
+                    "list all the NFS-clients that have connected "
+                    "through the MOUNT protocol. If this is on shared "
+                    "storage, all GlusterFS servers will update and "
+                    "output (with 'showmount') the same list. Set to "
+                    "\"/-\" to disable."},
+    {.key = {OPT_SERVER_RPC_STATD},
+     .type = GF_OPTION_TYPE_PATH,
+     .default_value = GF_RPC_STATD_PROG,
+     .description = "The executable of RPC statd utility. "
+                    "Defaults to " GF_RPC_STATD_PROG},
+    {.key = {OPT_SERVER_RPC_STATD_PIDFILE},
+     .type = GF_OPTION_TYPE_PATH,
+     .default_value = GF_RPC_STATD_PIDFILE,
+     .description = "The pid file of RPC statd utility. "
+                    "Defaults to " GF_RPC_STATD_PIDFILE},
+    {.key = {OPT_SERVER_AUX_GIDS},
+     .type = GF_OPTION_TYPE_BOOL,
+     .default_value = "off",
+     .description = "Let the server look up which groups a user belongs "
+                    "to, overwriting the list passed from the client. "
+                    "This enables support for group lists longer than "
+                    "can be passed through the NFS protocol, but is not "
+                    "secure unless users and groups are well synchronized "
+                    "between clients and servers."},
+    {.key = {OPT_SERVER_GID_CACHE_TIMEOUT},
+     .type = GF_OPTION_TYPE_INT,
+     .min = 0,
+     .max = 3600,
+     .default_value = "300",
+     .description = "Number of seconds to cache auxiliary-GID data, "
+                    "when " OPT_SERVER_AUX_GIDS " is set."},
+    {.key = {"nfs.acl"},
+     .type = GF_OPTION_TYPE_BOOL,
+     .default_value = "on",
+     .description = "This option is used to control ACL support for NFS."},
+    {.key = {"nfs.drc"},
+     .type = GF_OPTION_TYPE_STR,
+     .default_value = "off",
+     .description = "Enable Duplicate Request Cache in gNFS server to "
+                    "improve correctness of non-idempotent operations like "
+                    "write, delete, link, et al"},
+    {.key = {"nfs.drc-size"},
+     .type = GF_OPTION_TYPE_INT,
+     .default_value = "0x20000",
+     .description = "Sets the number of non-idempotent "
+                    "requests to cache in drc"},
+    {.key = {"nfs.exports-auth-enable"},
+     .type = GF_OPTION_TYPE_BOOL,
+     .description = "Set the option to 'on' to enable exports/netgroup "
+                    "authentication in the NFS server and mount daemon."},
+
+    {.key = {"nfs.auth-refresh-interval-sec"},
+     .type = GF_OPTION_TYPE_INT,
+     .description = "Frequency in seconds that the daemon should check for"
+                    " changes in the exports/netgroups file."},
+
+    {.key = {"nfs.auth-cache-ttl-sec"},
+     .type = GF_OPTION_TYPE_INT,
+     .description = "Sets the TTL of an entry in the auth cache. Value is "
+                    "in seconds."},
+    {.key = {"nfs.rdirplus"},
+     .type = GF_OPTION_TYPE_BOOL,
+     .default_value = "on",
+     .description = "When this option is set to off NFS falls back to "
+                    "standard readdir instead of readdirp"},
+    {
+        .key = {"nfs.event-threads"},
+        .type = GF_OPTION_TYPE_SIZET,
+        .min = 1,
+        .max = 32,
+        .default_value = "2",
+        .description = "Specifies the number of event threads to execute in"
+                       "in parallel. Larger values would help process"
+                       " responses faster, depending on available processing"
+                       " power. Range 1-32 threads.",
+        .op_version = {GD_OP_VERSION_4_0_0},
+        .flags = OPT_FLAG_SETTABLE,
+    },
+    {.key = {NULL}},
+};
+
+xlator_api_t xlator_api = {
+    .init = init,
+    .fini = fini,
+    .notify = notify,
+    .reconfigure = reconfigure,
+    .mem_acct_init = mem_acct_init,
+    .op_version = {1},
+    .dumpops = &dumpops,
+    .fops = &fops,
+    .cbks = &cbks,
+    .options = options,
+    .identifier = "gnfs",
+    .category = GF_MAINTAINED,
 };
diff --git a/xlators/nfs/server/src/nfs.h b/xlators/nfs/server/src/nfs.h
index 15aec5d7da9..e3daaed17a6 100644
--- a/xlators/nfs/server/src/nfs.h
+++ b/xlators/nfs/server/src/nfs.h
@@ -11,143 +11,144 @@
 #ifndef __NFS_H__
 #define __NFS_H__
 
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
-
 #include "rpcsvc.h"
-#include "dict.h"
-#include "xlator.h"
-#include "lkowner.h"
-#include "gidcache.h"
+#include <glusterfs/dict.h>
+#include <glusterfs/xlator.h>
+#include <glusterfs/lkowner.h>
+#include <glusterfs/gidcache.h>
 
-#define GF_NFS                  "nfs"
+#define GF_NFS "nfs"
 
-#define GF_NFS_CONCURRENT_OPS_MULT     15
+#define GF_NFS_CONCURRENT_OPS_MULT 15
 
-#define GF_NFS_INODE_LRU_MULT           6000
+#define GF_NFS_INODE_LRU_MULT 6000
 
-#define GF_RPC_MIN_THREADS      1
-#define GF_RPC_MAX_THREADS      16
+#define GF_RPC_MIN_THREADS 1
+#define GF_RPC_MAX_THREADS 16
 
-#define GF_NFS_DEFAULT_MEMFACTOR        15
-#define GF_NFS_MIN_MEMFACTOR            1
-#define GF_NFS_MAX_MEMFACTOR            30
+#define GF_NFS_DEFAULT_MEMFACTOR 15
+#define GF_NFS_MIN_MEMFACTOR 1
+#define GF_NFS_MAX_MEMFACTOR 30
 
-#define GF_NFS_DVM_ON                   1
-#define GF_NFS_DVM_OFF                  0
+#define GF_NFS_DVM_ON 1
+#define GF_NFS_DVM_OFF 0
 
 /* Disable using the exports file by default */
-#define GF_NFS_DEFAULT_EXPORT_AUTH      0
+#define GF_NFS_DEFAULT_EXPORT_AUTH 0
 
-#define GF_NFS_DEFAULT_AUTH_REFRESH_INTERVAL_SEC       2
-#define GF_NFS_DEFAULT_AUTH_CACHE_TTL_SEC              300 /* 5 min */
+#define GF_NFS_DEFAULT_AUTH_REFRESH_INTERVAL_SEC 2
+#define GF_NFS_DEFAULT_AUTH_CACHE_TTL_SEC 300 /* 5 min */
 
 /* This corresponds to the max 16 number of group IDs that are sent through an
  * RPC request. Since NFS is the only one going to set this, we can be safe
  * in keeping this size hardcoded.
  */
-#define GF_REQUEST_MAXGROUPS    16
+#define GF_REQUEST_MAXGROUPS 16
 
 /* Callback into a version-specific NFS protocol.
  * The return type is used by the nfs.c code to register the protocol.
  * with the RPC service.
  */
-typedef rpcsvc_program_t *(*nfs_version_initer_t) (xlator_t *nfsx);
+typedef rpcsvc_program_t *(*nfs_version_initer_t)(xlator_t *nfsx);
 
 /* List of version-specific protocol initiators */
 struct nfs_initer_list {
-        struct list_head list;
-        nfs_version_initer_t    init;
-        rpcsvc_program_t        *program;
-        gf_boolean_t            required;
+    struct list_head list;
+    nfs_version_initer_t init;
+    rpcsvc_program_t *program;
+    gf_boolean_t required;
 };
 
 struct nfs_state {
-        rpcsvc_t                *rpcsvc;
-        struct list_head        versions;
-        struct mount3_state     *mstate;
-        struct nfs3_state       *nfs3state;
-        struct nlm4_state       *nlm4state;
-        struct mem_pool         *foppool;
-        unsigned int            memfactor;
-        xlator_list_t           *subvols;
-
-        gf_lock_t               svinitlock;
-        int                     allsubvols;
-        int                     upsubvols;
-        xlator_t                **initedxl;
-        int                     subvols_started;
-        int                     dynamicvolumes;
-        int                     enable_ino32;
-        unsigned int            override_portnum;
-        int                     allow_insecure;
-        int                     enable_nlm;
-        int                     enable_acl;
-        int                     mount_udp;
-
-        /* Enable exports auth model */
-        int                     exports_auth;
-        /* Refresh auth params from disk periodically */
-        int                     refresh_auth;
-
-        unsigned int            auth_refresh_time_secs;
-        unsigned int            auth_cache_ttl_sec;
-
-        char                    *rmtab;
-        struct rpc_clnt         *rpc_clnt;
-        gf_boolean_t            server_aux_gids;
-	uint32_t		server_aux_gids_max_age;
-	gid_cache_t		gid_cache;
-        uint32_t                generation;
-        gf_boolean_t            register_portmap;
-        char                    *rpc_statd;
-        char                    *rpc_statd_pid_file;
+    rpcsvc_t *rpcsvc;
+    struct list_head versions;
+    struct mount3_state *mstate;
+    struct nfs3_state *nfs3state;
+    struct nlm4_state *nlm4state;
+    struct mem_pool *foppool;
+    unsigned int memfactor;
+    xlator_list_t *subvols;
+
+    gf_lock_t svinitlock;
+    int allsubvols;
+    int upsubvols;
+    xlator_t **initedxl;
+    int subvols_started;
+    int dynamicvolumes;
+    int enable_ino32;
+    unsigned int override_portnum;
+    int allow_insecure;
+    int enable_nlm;
+    int enable_acl;
+    int mount_udp;
+
+    /* Enable exports auth model */
+    int exports_auth;
+    /* Refresh auth params from disk periodically */
+    int refresh_auth;
+
+    unsigned int auth_refresh_time_secs;
+    unsigned int auth_cache_ttl_sec;
+
+    char *rmtab;
+    struct rpc_clnt *rpc_clnt;
+    gf_boolean_t server_aux_gids;
+    uint32_t server_aux_gids_max_age;
+    gid_cache_t gid_cache;
+    uint32_t generation;
+    gf_boolean_t register_portmap;
+    char *rpc_statd;
+    char *rpc_statd_pid_file;
+    gf_boolean_t rdirplus;
+    uint32_t event_threads;
 };
 
 struct nfs_inode_ctx {
-        struct list_head        shares;
-        uint32_t                generation;
+    struct list_head shares;
+    uint32_t generation;
 };
 
-#define gf_nfs_dvm_on(nfsstt)   (((struct nfs_state *)nfsstt)->dynamicvolumes == GF_NFS_DVM_ON)
-#define gf_nfs_dvm_off(nfsstt)  (((struct nfs_state *)nfsstt)->dynamicvolumes == GF_NFS_DVM_OFF)
-#define __gf_nfs_enable_ino32(nfsstt)     (((struct nfs_state *)nfsstt)->enable_ino32)
-#define gf_nfs_this_private     ((struct nfs_state *)((xlator_t *)THIS)->private)
-#define gf_nfs_enable_ino32()     (__gf_nfs_enable_ino32(gf_nfs_this_private))
+#define gf_nfs_dvm_on(nfsstt)                                                  \
+    (((struct nfs_state *)nfsstt)->dynamicvolumes == GF_NFS_DVM_ON)
+#define gf_nfs_dvm_off(nfsstt)                                                 \
+    (((struct nfs_state *)nfsstt)->dynamicvolumes == GF_NFS_DVM_OFF)
+#define __gf_nfs_enable_ino32(nfsstt)                                          \
+    (((struct nfs_state *)nfsstt)->enable_ino32)
+#define gf_nfs_this_private ((struct nfs_state *)((xlator_t *)THIS)->private)
+#define gf_nfs_enable_ino32() (__gf_nfs_enable_ino32(gf_nfs_this_private))
 
 /* We have one gid more than the glusterfs maximum since we pass the primary
  * gid as the first element of the array.
  */
-#define NFS_NGROUPS         (GF_REQUEST_MAXGROUPS + 1)
+#define NFS_NGROUPS (GF_REQUEST_MAXGROUPS + 1)
 
 /* Index of the primary gid */
-#define NFS_PRIMGID_IDX     0
+#define NFS_PRIMGID_IDX 0
 
 typedef struct nfs_user_info {
-        uid_t   uid;
-        gid_t   gids[NFS_NGROUPS];
-        int     ngrps;
-        gf_lkowner_t lk_owner;
+    uid_t uid;
+    gid_t gids[NFS_NGROUPS];
+    int ngrps;
+    gf_lkowner_t lk_owner;
+    char identifier[UNIX_PATH_MAX]; /* ip of user */
 } nfs_user_t;
 
 extern int
-nfs_user_root_create (nfs_user_t *newnfu);
+nfs_user_root_create(nfs_user_t *newnfu);
 
 extern int
-nfs_user_create (nfs_user_t *newnfu, uid_t uid, gid_t gid, gid_t *auxgids,
-                 int auxcount);
+nfs_user_create(nfs_user_t *newnfu, uid_t uid, gid_t gid,
+                rpc_transport_t *trans, gid_t *auxgids, int auxcount);
 
 extern void
-nfs_request_user_init (nfs_user_t *nfu, rpcsvc_request_t *req);
+nfs_request_user_init(nfs_user_t *nfu, rpcsvc_request_t *req);
 
 extern void
-nfs_request_primary_user_init (nfs_user_t *nfu, rpcsvc_request_t *req,
-                               uid_t uid, gid_t gid);
+nfs_request_primary_user_init(nfs_user_t *nfu, rpcsvc_request_t *req, uid_t uid,
+                              gid_t gid);
 extern int
-nfs_subvolume_started (struct nfs_state *nfs, xlator_t *xl);
+nfs_subvolume_started(struct nfs_state *nfs, xlator_t *xl);
 
 extern void
-nfs_fix_groups (xlator_t *this, call_stack_t *root);
+nfs_fix_groups(xlator_t *this, call_stack_t *root);
 #endif
diff --git a/xlators/nfs/server/src/nfs3-fh.c b/xlators/nfs/server/src/nfs3-fh.c
index 8d492c8c72d..caa3cfa6995 100644
--- a/xlators/nfs/server/src/nfs3-fh.c
+++ b/xlators/nfs/server/src/nfs3-fh.c
@@ -8,184 +8,179 @@
   cases as published by the Free Software Foundation.
 */
 
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
-
 #include "rpcsvc.h"
-#include "dict.h"
-#include "xlator.h"
+#include <glusterfs/dict.h>
+#include <glusterfs/xlator.h>
 #include "xdr-nfs3.h"
 #include "msg-nfs3.h"
-#include "iobuf.h"
+#include <glusterfs/iobuf.h>
 #include "nfs3-fh.h"
 #include "nfs-common.h"
-#include "iatt.h"
-#include "common-utils.h"
-
+#include <glusterfs/iatt.h>
+#include <glusterfs/common-utils.h>
+#include "nfs-messages.h"
 
 int
-nfs3_fh_validate (struct nfs3_fh *fh)
+nfs3_fh_validate(struct nfs3_fh *fh)
 {
-	if (!fh)
-		return 0;
+    if (!fh)
+        return 0;
 
-	if (fh->ident[0] != GF_NFSFH_IDENT0)
-		return 0;
+    if (fh->ident[0] != GF_NFSFH_IDENT0)
+        return 0;
 
-	if (fh->ident[1] != GF_NFSFH_IDENT1)
-		return 0;
+    if (fh->ident[1] != GF_NFSFH_IDENT1)
+        return 0;
 
-       if (fh->ident[2] != GF_NFSFH_IDENT2)
-               return 0;
+    if (fh->ident[2] != GF_NFSFH_IDENT2)
+        return 0;
 
-       if (fh->ident[3] != GF_NFSFH_IDENT3)
-               return 0;
+    if (fh->ident[3] != GF_NFSFH_IDENT3)
+        return 0;
 
-        return 1;
+    return 1;
 }
 
-
 void
-nfs3_fh_init (struct nfs3_fh *fh, struct iatt *buf)
+nfs3_fh_init(struct nfs3_fh *fh, struct iatt *buf)
 {
-        if ((!fh) || (!buf))
-                return;
+    if ((!fh) || (!buf))
+        return;
 
-        fh->ident[0] = GF_NFSFH_IDENT0;
-        fh->ident[1] = GF_NFSFH_IDENT1;
-        fh->ident[2] = GF_NFSFH_IDENT2;
-        fh->ident[3] = GF_NFSFH_IDENT3;
+    fh->ident[0] = GF_NFSFH_IDENT0;
+    fh->ident[1] = GF_NFSFH_IDENT1;
+    fh->ident[2] = GF_NFSFH_IDENT2;
+    fh->ident[3] = GF_NFSFH_IDENT3;
 
-        gf_uuid_copy (fh->gfid, buf->ia_gfid);
+    gf_uuid_copy(fh->gfid, buf->ia_gfid);
 }
 
-
 struct nfs3_fh
-nfs3_fh_build_indexed_root_fh (xlator_list_t *cl, xlator_t *xl)
+nfs3_fh_build_indexed_root_fh(xlator_list_t *cl, xlator_t *xl)
 {
-        struct nfs3_fh  fh = {{0}, };
-        struct iatt     buf = {0, };
-        uuid_t          root = {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1};
-
-        if ((!cl) || (!xl))
-                return fh;
+    struct nfs3_fh fh = {
+        {0},
+    };
+    struct iatt buf = {
+        0,
+    };
+    static uuid_t root = {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1};
+
+    if ((!cl) || (!xl))
+        return fh;
 
-        gf_uuid_copy (buf.ia_gfid, root);
-        nfs3_fh_init (&fh, &buf);
-        fh.exportid [15] = nfs_xlator_to_xlid (cl, xl);
+    gf_uuid_copy(buf.ia_gfid, root);
+    nfs3_fh_init(&fh, &buf);
+    fh.exportid[15] = nfs_xlator_to_xlid(cl, xl);
 
-        return fh;
+    return fh;
 }
 
-
 struct nfs3_fh
-nfs3_fh_build_uuid_root_fh (uuid_t volumeid, uuid_t mountid)
+nfs3_fh_build_uuid_root_fh(uuid_t volumeid, uuid_t mountid)
 {
-        struct nfs3_fh  fh = {{0}, };
-        struct iatt     buf = {0, };
-        uuid_t          root = {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1};
-
-        gf_uuid_copy (buf.ia_gfid, root);
-        nfs3_fh_init (&fh, &buf);
-        gf_uuid_copy (fh.exportid, volumeid);
-        gf_uuid_copy (fh.mountid, mountid);
-
-        return fh;
+    struct nfs3_fh fh = {
+        {0},
+    };
+    struct iatt buf = {
+        0,
+    };
+    static uuid_t root = {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1};
+
+    gf_uuid_copy(buf.ia_gfid, root);
+    nfs3_fh_init(&fh, &buf);
+    gf_uuid_copy(fh.exportid, volumeid);
+    gf_uuid_copy(fh.mountid, mountid);
+
+    return fh;
 }
 
-
 int
-nfs3_fh_is_root_fh (struct nfs3_fh *fh)
+nfs3_fh_is_root_fh(struct nfs3_fh *fh)
 {
-        uuid_t  rootgfid = {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1};
+    static uuid_t rootgfid = {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1};
 
-        if (!fh)
-                return 0;
+    if (!fh)
+        return 0;
 
-        if (gf_uuid_compare (fh->gfid, rootgfid) == 0)
-                return 1;
+    if (gf_uuid_compare(fh->gfid, rootgfid) == 0)
+        return 1;
 
-        return 0;
+    return 0;
 }
 
-
 void
-nfs3_fh_to_str (struct nfs3_fh *fh, char *str, size_t len)
+nfs3_fh_to_str(struct nfs3_fh *fh, char *str, size_t len)
 {
-        char            gfid[GF_UUID_BUF_SIZE];
-        char            exportid[GF_UUID_BUF_SIZE];
-        char            mountid[GF_UUID_BUF_SIZE];
+    char gfid[GF_UUID_BUF_SIZE];
+    char exportid[GF_UUID_BUF_SIZE];
+    char mountid[GF_UUID_BUF_SIZE];
 
-        if ((!fh) || (!str))
-                return;
+    if ((!fh) || (!str))
+        return;
 
-        snprintf (str, len, "FH: exportid %s, gfid %s, mountid %s",
-                  uuid_utoa_r (fh->exportid, exportid),
-                  uuid_utoa_r (fh->gfid, gfid),
-                  uuid_utoa_r (fh->mountid, mountid));
+    snprintf(str, len, "FH: exportid %s, gfid %s, mountid %s",
+             uuid_utoa_r(fh->exportid, exportid), uuid_utoa_r(fh->gfid, gfid),
+             uuid_utoa_r(fh->mountid, mountid));
 }
 
 void
-nfs3_log_fh (struct nfs3_fh *fh)
+nfs3_log_fh(struct nfs3_fh *fh)
 {
-        char    gfidstr[512];
-        char    exportidstr[512];
+    char gfidstr[512];
+    char exportidstr[512];
 
-        if (!fh)
-                return;
+    if (!fh)
+        return;
 
-        gf_log ("nfs3-fh", GF_LOG_TRACE, "filehandle: exportid "
-                "0x%s, gfid 0x%s",
-                 uuid_utoa_r (fh->exportid, exportidstr),
-                 uuid_utoa_r (fh->gfid, gfidstr));
+    gf_msg_trace("nfs3-fh", 0, "filehandle: exportid 0x%s, gfid 0x%s",
+                 uuid_utoa_r(fh->exportid, exportidstr),
+                 uuid_utoa_r(fh->gfid, gfidstr));
 }
 
 int
-nfs3_fh_build_parent_fh (struct nfs3_fh *child, struct iatt *newstat,
-                         struct nfs3_fh *newfh)
+nfs3_fh_build_parent_fh(struct nfs3_fh *child, struct iatt *newstat,
+                        struct nfs3_fh *newfh)
 {
-        if ((!child) || (!newstat) || (!newfh))
-                return -1;
+    if ((!child) || (!newstat) || (!newfh))
+        return -1;
 
-        nfs3_fh_init (newfh, newstat);
-        gf_uuid_copy (newfh->exportid, child->exportid);
-        return 0;
+    nfs3_fh_init(newfh, newstat);
+    gf_uuid_copy(newfh->exportid, child->exportid);
+    return 0;
 }
 
 int
-nfs3_build_fh (inode_t *inode, uuid_t exportid, struct nfs3_fh *newfh)
+nfs3_build_fh(inode_t *inode, uuid_t exportid, struct nfs3_fh *newfh)
 {
-        if (!newfh || !inode)
-                return -1;
-
-        newfh->ident[0] = GF_NFSFH_IDENT0;
-        newfh->ident[1] = GF_NFSFH_IDENT1;
-        newfh->ident[2] = GF_NFSFH_IDENT2;
-        newfh->ident[3] = GF_NFSFH_IDENT3;
-        gf_uuid_copy (newfh->gfid, inode->gfid);
-        gf_uuid_copy (newfh->exportid, exportid);
-        /*gf_uuid_copy (newfh->mountid, mountid);*/
-        return 0;
+    if (!newfh || !inode)
+        return -1;
+
+    newfh->ident[0] = GF_NFSFH_IDENT0;
+    newfh->ident[1] = GF_NFSFH_IDENT1;
+    newfh->ident[2] = GF_NFSFH_IDENT2;
+    newfh->ident[3] = GF_NFSFH_IDENT3;
+    gf_uuid_copy(newfh->gfid, inode->gfid);
+    gf_uuid_copy(newfh->exportid, exportid);
+    /*gf_uuid_copy (newfh->mountid, mountid);*/
+    return 0;
 }
 
 int
-nfs3_fh_build_child_fh (struct nfs3_fh *parent, struct iatt *newstat,
-                        struct nfs3_fh *newfh)
+nfs3_fh_build_child_fh(struct nfs3_fh *parent, struct iatt *newstat,
+                       struct nfs3_fh *newfh)
 {
-        if ((!parent) || (!newstat) || (!newfh))
-                return -1;
+    if ((!parent) || (!newstat) || (!newfh))
+        return -1;
 
-        nfs3_fh_init (newfh, newstat);
-        gf_uuid_copy (newfh->exportid, parent->exportid);
-        gf_uuid_copy (newfh->mountid, parent->mountid);
-        return 0;
+    nfs3_fh_init(newfh, newstat);
+    gf_uuid_copy(newfh->exportid, parent->exportid);
+    gf_uuid_copy(newfh->mountid, parent->mountid);
+    return 0;
 }
 
-
 uint32_t
-nfs3_fh_compute_size ()
+nfs3_fh_compute_size()
 {
-        return GF_NFSFH_STATIC_SIZE;
+    return GF_NFSFH_STATIC_SIZE;
 }
diff --git a/xlators/nfs/server/src/nfs3-fh.h b/xlators/nfs/server/src/nfs3-fh.h
index eb969e44e22..cd7916b892d 100644
--- a/xlators/nfs/server/src/nfs3-fh.h
+++ b/xlators/nfs/server/src/nfs3-fh.h
@@ -11,95 +11,91 @@
 #ifndef _NFS_FH_H_
 #define _NFS_FH_H_
 
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
-
-#include "xlator.h"
+#include <glusterfs/xlator.h>
 #include "xdr-nfs3.h"
-#include "iatt.h"
+#include <glusterfs/iatt.h>
 #include <sys/types.h>
-#include "compat-uuid.h"
+#include <glusterfs/compat-uuid.h>
 
 /* BIG FAT WARNING: The file handle code is tightly coupled to NFSv3 file
  * handles for now. This will change if and when we need v4. */
-#define GF_NFSFH_IDENT0         ':'
-#define GF_NFSFH_IDENT1         'O'
-#define GF_NFSFH_IDENT2         'G'
-#define GF_NFSFH_IDENT3         'L'
-#define GF_NFSFH_IDENT_SIZE     (sizeof(char) * 4)
-#define GF_NFSFH_STATIC_SIZE    (GF_NFSFH_IDENT_SIZE + (3*sizeof (uuid_t)))
-
-#define nfs3_fh_exportid_to_index(exprtid)      ((uint16_t)exprtid[15])
+#define GF_NFSFH_IDENT0 ':'
+#define GF_NFSFH_IDENT1 'O'
+#define GF_NFSFH_IDENT2 'G'
+#define GF_NFSFH_IDENT3 'L'
+#define GF_NFSFH_IDENT_SIZE (sizeof(char) * 4)
+#define GF_NFSFH_STATIC_SIZE (GF_NFSFH_IDENT_SIZE + (3 * sizeof(uuid_t)))
+
+#define nfs3_fh_exportid_to_index(exprtid) ((uint16_t)exprtid[15])
 /* ATTENTION: Change in size of the structure below should be reflected in the
  * GF_NFSFH_STATIC_SIZE.
  */
 struct nfs3_fh {
-
-        /* Used to ensure that a bunch of bytes are actually a GlusterFS NFS
-         * file handle. Should contain ":OGL"
-         */
-        char                    ident[4];
-
-        /* UUID that identifies an export. The value stored in exportid
-         * depends on the usage of gluster nfs. If the DVM is enabled using
-         * the nfs.dynamic-volumes option then exportid will contain the UUID
-         * of the volume so that gnfs is able to identify volumes uniquely
-         * through volume additions,deletions,migrations, etc.
-         *
-         * When not using dvm, exportid contains the index of the volume
-         * based on the position of the volume in the list of subvolumes
-         * for gnfs.
-         */
-        uuid_t              exportid;
-
-        /* File/dir gfid. */
-        uuid_t                  gfid;
-        uuid_t                  mountid;
-        /* This structure must be exactly NFS3_FHSIZE (64) bytes long.
-           Having the structure shorter results in buffer overflows
-           during XDR decoding.
-        */
-        unsigned char padding[NFS3_FHSIZE - GF_NFSFH_STATIC_SIZE];
+    /* Used to ensure that a bunch of bytes are actually a GlusterFS NFS
+     * file handle. Should contain ":OGL"
+     */
+    char ident[4];
+
+    /* UUID that identifies an export. The value stored in exportid
+     * depends on the usage of gluster nfs. If the DVM is enabled using
+     * the nfs.dynamic-volumes option then exportid will contain the UUID
+     * of the volume so that gnfs is able to identify volumes uniquely
+     * through volume additions,deletions,migrations, etc.
+     *
+     * When not using dvm, exportid contains the index of the volume
+     * based on the position of the volume in the list of subvolumes
+     * for gnfs.
+     */
+    uuid_t exportid;
+
+    /* File/dir gfid. */
+    uuid_t gfid;
+    uuid_t mountid;
+    /* This structure must be exactly NFS3_FHSIZE (64) bytes long.
+       Having the structure shorter results in buffer overflows
+       during XDR decoding.
+    */
+    unsigned char padding[NFS3_FHSIZE - GF_NFSFH_STATIC_SIZE];
 } __attribute__((__packed__));
 
-#define GF_NFS3FH_STATIC_INITIALIZER    {{0},}
+#define GF_NFS3FH_STATIC_INITIALIZER                                           \
+    {                                                                          \
+        {0},                                                                   \
+    }
 
 extern uint32_t
-nfs3_fh_compute_size ();
+nfs3_fh_compute_size();
 
 extern uint16_t
-nfs3_fh_hash_entry (uuid_t gfid);
+nfs3_fh_hash_entry(uuid_t gfid);
 
 extern int
-nfs3_fh_validate (struct nfs3_fh *fh);
+nfs3_fh_validate(struct nfs3_fh *fh);
 
 extern struct nfs3_fh
-nfs3_fh_build_indexed_root_fh (xlator_list_t *cl, xlator_t *xl);
+nfs3_fh_build_indexed_root_fh(xlator_list_t *cl, xlator_t *xl);
 
 extern int
-nfs3_fh_is_root_fh (struct nfs3_fh *fh);
+nfs3_fh_is_root_fh(struct nfs3_fh *fh);
 
 extern int
-nfs3_fh_build_child_fh (struct nfs3_fh *parent, struct iatt *newstat,
-                        struct nfs3_fh *newfh);
+nfs3_fh_build_child_fh(struct nfs3_fh *parent, struct iatt *newstat,
+                       struct nfs3_fh *newfh);
 
 extern void
-nfs3_log_fh (struct nfs3_fh *fh);
+nfs3_log_fh(struct nfs3_fh *fh);
 
 extern void
-nfs3_fh_to_str (struct nfs3_fh *fh, char *str, size_t len);
+nfs3_fh_to_str(struct nfs3_fh *fh, char *str, size_t len);
 
 extern int
-nfs3_fh_build_parent_fh (struct nfs3_fh *child, struct iatt *newstat,
-                         struct nfs3_fh *newfh);
+nfs3_fh_build_parent_fh(struct nfs3_fh *child, struct iatt *newstat,
+                        struct nfs3_fh *newfh);
 
 extern struct nfs3_fh
-nfs3_fh_build_uuid_root_fh (uuid_t volumeid, uuid_t mountid);
+nfs3_fh_build_uuid_root_fh(uuid_t volumeid, uuid_t mountid);
 
 extern int
-nfs3_build_fh (inode_t *inode, uuid_t exportid,
-               struct nfs3_fh *newfh);
+nfs3_build_fh(inode_t *inode, uuid_t exportid, struct nfs3_fh *newfh);
 
 #endif
diff --git a/xlators/nfs/server/src/nfs3-helpers.c b/xlators/nfs/server/src/nfs3-helpers.c
index fb630a643cf..897fb42b071 100644
--- a/xlators/nfs/server/src/nfs3-helpers.c
+++ b/xlators/nfs/server/src/nfs3-helpers.c
@@ -8,253 +8,241 @@
   cases as published by the Free Software Foundation.
 */
 
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
-
 #include <inttypes.h>
 
-#include "xlator.h"
+#include <glusterfs/xlator.h>
 #include "nfs3.h"
 #include "nfs3-fh.h"
 #include "msg-nfs3.h"
-#include "rbthash.h"
+#include <glusterfs/rbthash.h>
 #include "nfs-fops.h"
 #include "nfs-inodes.h"
 #include "nfs-generics.h"
 #include "nfs3-helpers.h"
 #include "nfs-mem-types.h"
-#include "iatt.h"
-#include "common-utils.h"
+#include <glusterfs/iatt.h>
+#include <glusterfs/common-utils.h>
+#include "nfs-messages.h"
 #include "mount3.h"
 #include <string.h>
 
 extern int
-nfs3_set_root_looked_up (struct nfs3_state *nfs3, struct nfs3_fh *rootfh);
+nfs3_set_root_looked_up(struct nfs3_state *nfs3, struct nfs3_fh *rootfh);
 
 extern int
-nfs3_is_root_looked_up (struct nfs3_state *nfs3, struct nfs3_fh *rootfh);
-
+nfs3_is_root_looked_up(struct nfs3_state *nfs3, struct nfs3_fh *rootfh);
 
-#define nfs3_call_resume(cst)                                   \
-        do {                                                    \
-                if (((cst)) && (cst)->resume_fn)                \
-                        (cst)->resume_fn (cst);                 \
-        } while (0)                                             \
+#define nfs3_call_resume(cst)                                                  \
+    do {                                                                       \
+        if (((cst)) && (cst)->resume_fn)                                       \
+            (cst)->resume_fn(cst);                                             \
+    } while (0)
 
-#define nfs3_call_resume_estale(csta)                           \
-        do {                                                    \
-                (csta)->resolve_ret = -1;                       \
-                (csta)->resolve_errno = ESTALE;                 \
-                nfs3_call_resume (csta);                        \
-        } while (0)                                             \
+#define nfs3_call_resume_estale(csta)                                          \
+    do {                                                                       \
+        (csta)->resolve_ret = -1;                                              \
+        (csta)->resolve_errno = ESTALE;                                        \
+        nfs3_call_resume(csta);                                                \
+    } while (0)
 
 struct nfs3stat_strerror {
-        nfsstat3        stat;
-        char            strerror[100];
+    nfsstat3 stat;
+    char strerror[100];
 };
 
-
 struct nfs3stat_strerror nfs3stat_strerror_table[] = {
-        { NFS3_OK,              "Call completed successfully."          },
-        { NFS3ERR_PERM,         "Not owner"                             },
-        { NFS3ERR_NOENT,        "No such file or directory"             },
-        { NFS3ERR_IO,           "I/O error"                             },
-        { NFS3ERR_NXIO,         "I/O error"                             },
-        { NFS3ERR_ACCES,        "Permission denied"                     },
-        { NFS3ERR_EXIST,        "File exists"                           },
-        { NFS3ERR_XDEV,         "Attempt to do a cross-device hard link"},
-        { NFS3ERR_NODEV,        "No such device"                        },
-        { NFS3ERR_NOTDIR,       "Not a directory"                       },
-        { NFS3ERR_ISDIR,        "Is a directory"                        },
-        { NFS3ERR_INVAL,        "Invalid argument for operation"        },
-        { NFS3ERR_FBIG,         "File too large"                        },
-        { NFS3ERR_NOSPC,        "No space left on device"               },
-        { NFS3ERR_ROFS,         "Read-only file system"                 },
-        { NFS3ERR_MLINK,        "Too many hard links"                   },
-        { NFS3ERR_NAMETOOLONG,  "Filename in operation was too long"    },
-        { NFS3ERR_NOTEMPTY,     "Directory not empty"                   },
-        { NFS3ERR_DQUOT,        "Resource (quota) hard limit exceeded"  },
-        { NFS3ERR_STALE,        "Invalid file handle"                   },
-        { NFS3ERR_REMOTE,       "Too many levels of remote in path"     },
-        { NFS3ERR_BADHANDLE,    "Illegal NFS file handle"               },
-        { NFS3ERR_NOT_SYNC,     "Update synchronization mismatch detected" },
-        { NFS3ERR_BAD_COOKIE,   "READDIR or READDIRPLUS cookie is stale"},
-        { NFS3ERR_NOTSUPP,      "Operation is not supported"            },
-        { NFS3ERR_TOOSMALL,     "Buffer or request is too small"        },
-        { NFS3ERR_SERVERFAULT,  "Error occurred on the server or IO Error" },
-        { NFS3ERR_BADTYPE,      "Type not supported by the server"      },
-        { NFS3ERR_JUKEBOX,      "Cannot complete server initiated request" },
-        { NFS3ERR_END_OF_LIST,  "IO Error"                              },
+    {NFS3_OK, "Call completed successfully."},
+    {NFS3ERR_PERM, "Not owner"},
+    {NFS3ERR_NOENT, "No such file or directory"},
+    {NFS3ERR_IO, "I/O error"},
+    {NFS3ERR_NXIO, "I/O error"},
+    {NFS3ERR_ACCES, "Permission denied"},
+    {NFS3ERR_EXIST, "File exists"},
+    {NFS3ERR_XDEV, "Attempt to do a cross-device hard link"},
+    {NFS3ERR_NODEV, "No such device"},
+    {NFS3ERR_NOTDIR, "Not a directory"},
+    {NFS3ERR_ISDIR, "Is a directory"},
+    {NFS3ERR_INVAL, "Invalid argument for operation"},
+    {NFS3ERR_FBIG, "File too large"},
+    {NFS3ERR_NOSPC, "No space left on device"},
+    {NFS3ERR_ROFS, "Read-only file system"},
+    {NFS3ERR_MLINK, "Too many hard links"},
+    {NFS3ERR_NAMETOOLONG, "Filename in operation was too long"},
+    {NFS3ERR_NOTEMPTY, "Directory not empty"},
+    {NFS3ERR_DQUOT, "Resource (quota) hard limit exceeded"},
+    {NFS3ERR_STALE, "Invalid file handle"},
+    {NFS3ERR_REMOTE, "Too many levels of remote in path"},
+    {NFS3ERR_BADHANDLE, "Illegal NFS file handle"},
+    {NFS3ERR_NOT_SYNC, "Update synchronization mismatch detected"},
+    {NFS3ERR_BAD_COOKIE, "READDIR or READDIRPLUS cookie is stale"},
+    {NFS3ERR_NOTSUPP, "Operation is not supported"},
+    {NFS3ERR_TOOSMALL, "Buffer or request is too small"},
+    {NFS3ERR_SERVERFAULT, "Error occurred on the server or IO Error"},
+    {NFS3ERR_BADTYPE, "Type not supported by the server"},
+    {NFS3ERR_JUKEBOX, "Cannot complete server initiated request"},
+    {NFS3ERR_END_OF_LIST, "IO Error"},
 
 };
 
-
 uint64_t
-nfs3_iatt_gfid_to_ino (struct iatt *buf)
+nfs3_iatt_gfid_to_ino(struct iatt *buf)
 {
-        uint64_t ino  = 0;
+    uint64_t ino = 0;
 
-        if (!buf)
-                return 0;
+    if (!buf)
+        return 0;
 
-        if (gf_nfs_enable_ino32()) {
-                ino = (uint32_t )nfs_hash_gfid (buf->ia_gfid);
-                goto hashout;
-        }
+    if (gf_nfs_enable_ino32()) {
+        ino = (uint32_t)nfs_hash_gfid(buf->ia_gfid);
+        goto hashout;
+    }
 
-        /* from posix its guaranteed to send unique ino */
-        ino = buf->ia_ino;
+    /* from posix its guaranteed to send unique ino */
+    ino = buf->ia_ino;
 
 hashout:
-        return ino;
+    return ino;
 }
 
-
 void
-nfs3_map_deviceid_to_statdev (struct iatt *ia, uint64_t deviceid)
+nfs3_map_deviceid_to_statdev(struct iatt *ia, uint64_t deviceid)
 {
-        if (!ia)
-                return;
+    if (!ia)
+        return;
 
-        ia->ia_dev = deviceid;
+    ia->ia_dev = deviceid;
 }
 
-
 struct nfs3_fh
-nfs3_extract_nfs3_fh (nfs_fh3 fh)
+nfs3_extract_nfs3_fh(nfs_fh3 fh)
 {
-        struct nfs3_fh          gfh;
+    struct nfs3_fh gfh;
 
-        memcpy (&gfh, fh.data.data_val, fh.data.data_len);
-        return gfh;
+    memcpy(&gfh, fh.data.data_val, fh.data.data_len);
+    return gfh;
 }
 
-
 struct nfs3_fh
-nfs3_extract_lookup_fh (lookup3args *args)
+nfs3_extract_lookup_fh(lookup3args *args)
 {
-        return nfs3_extract_nfs3_fh(args->what.dir);
+    return nfs3_extract_nfs3_fh(args->what.dir);
 }
 
-
 char *
-nfs3_extract_lookup_name (lookup3args *args)
+nfs3_extract_lookup_name(lookup3args *args)
 {
-        return args->what.name;
+    return args->what.name;
 }
 
-
 nfsstat3
-nfs3_errno_to_nfsstat3 (int errnum)
+nfs3_errno_to_nfsstat3(int errnum)
 {
-        nfsstat3        stat = NFS3_OK;
-
-        switch (errnum) {
+    nfsstat3 stat = NFS3_OK;
 
+    switch (errnum) {
         case 0:
-                stat = NFS3_OK;
-                break;
+            stat = NFS3_OK;
+            break;
 
         case EPERM:
-                stat = NFS3ERR_PERM;
-                break;
+            stat = NFS3ERR_PERM;
+            break;
 
         case ENOENT:
-                stat = NFS3ERR_NOENT;
-                break;
+            stat = NFS3ERR_NOENT;
+            break;
 
         case EACCES:
-                stat = NFS3ERR_ACCES;
-                break;
+            stat = NFS3ERR_ACCES;
+            break;
 
         case EEXIST:
-                stat = NFS3ERR_EXIST;
-                break;
+            stat = NFS3ERR_EXIST;
+            break;
 
         case EXDEV:
-                stat = NFS3ERR_XDEV;
-                break;
+            stat = NFS3ERR_XDEV;
+            break;
 
         case ENODEV:
-                stat = NFS3ERR_NODEV;
-                break;
+            stat = NFS3ERR_NODEV;
+            break;
 
         case EIO:
-                stat = NFS3ERR_IO;
-                break;
+            stat = NFS3ERR_IO;
+            break;
 
         case ENXIO:
-                stat = NFS3ERR_NXIO;
-                break;
+            stat = NFS3ERR_NXIO;
+            break;
 
         case ENOTDIR:
-                stat = NFS3ERR_NOTDIR;
-                break;
+            stat = NFS3ERR_NOTDIR;
+            break;
 
         case EISDIR:
-                stat = NFS3ERR_ISDIR;
-                break;
+            stat = NFS3ERR_ISDIR;
+            break;
 
         case EINVAL:
-                stat = NFS3ERR_INVAL;
-                break;
+            stat = NFS3ERR_INVAL;
+            break;
 
         case ENOSPC:
-                stat = NFS3ERR_NOSPC;
-                break;
+            stat = NFS3ERR_NOSPC;
+            break;
 
         case EROFS:
-                stat = NFS3ERR_ROFS;
-                break;
+            stat = NFS3ERR_ROFS;
+            break;
 
         case EFBIG:
-                stat = NFS3ERR_FBIG;
-                break;
+            stat = NFS3ERR_FBIG;
+            break;
 
         case EMLINK:
-                stat = NFS3ERR_MLINK;
-                break;
+            stat = NFS3ERR_MLINK;
+            break;
 
         case ENAMETOOLONG:
-                stat = NFS3ERR_NAMETOOLONG;
-                break;
+            stat = NFS3ERR_NAMETOOLONG;
+            break;
 
         case ENOTEMPTY:
-                stat = NFS3ERR_NOTEMPTY;
-                break;
+            stat = NFS3ERR_NOTEMPTY;
+            break;
 
         case EFAULT:
-                stat = NFS3ERR_SERVERFAULT;
-                break;
+            stat = NFS3ERR_SERVERFAULT;
+            break;
 
+        case ENOTSUP:
         case ENOSYS:
-                stat = NFS3ERR_NOTSUPP;
-                break;
+            stat = NFS3ERR_NOTSUPP;
+            break;
 
         case EBADF:
-                stat = NFS3ERR_BADTYPE;
-                break;
+            stat = NFS3ERR_BADTYPE;
+            break;
 
         case ESTALE:
-                stat = NFS3ERR_STALE;
-                break;
+            stat = NFS3ERR_STALE;
+            break;
 
         case ENOTCONN:
-                stat = NFS3ERR_IO;
-                break;
+            stat = NFS3ERR_IO;
+            break;
 
         case EDQUOT:
-                stat = NFS3ERR_DQUOT;
-                break;
+            stat = NFS3ERR_DQUOT;
+            break;
 
         default:
-                stat = NFS3ERR_SERVERFAULT;
-                break;
-        }
+            stat = NFS3ERR_SERVERFAULT;
+            break;
+    }
 
-        return stat;
+    return stat;
 }
 
 /*
@@ -262,3587 +250,3588 @@ nfs3_errno_to_nfsstat3 (int errnum)
  * 0 which means something came wrong from upper layer(s). If it
  * happens by any means, then set NFS3 status to NFS3ERR_SERVERFAULT.
  */
-inline nfsstat3
-nfs3_cbk_errno_status (int32_t op_ret, int32_t op_errno)
+nfsstat3
+nfs3_cbk_errno_status(int32_t op_ret, int32_t op_errno)
 {
-        if ((op_ret == -1) && (op_errno == 0)) {
-                return NFS3ERR_SERVERFAULT;
-        }
+    if ((op_ret == -1) && (op_errno == 0)) {
+        return NFS3ERR_SERVERFAULT;
+    }
 
-        return nfs3_errno_to_nfsstat3 (op_errno);
+    return nfs3_errno_to_nfsstat3(op_errno);
 }
 
 void
-nfs3_fill_lookup3res_error (lookup3res *res, nfsstat3 stat,
-                            struct iatt *dirstat)
+nfs3_fill_lookup3res_error(lookup3res *res, nfsstat3 stat, struct iatt *dirstat)
 {
-
-        memset (res, 0, sizeof (*res));
-        res->status = stat;
-        if (!dirstat) {
-                res->lookup3res_u.resfail.dir_attributes.attributes_follow = FALSE;
-        }
-
+    memset(res, 0, sizeof(*res));
+    res->status = stat;
+    if (!dirstat) {
+        res->lookup3res_u.resfail.dir_attributes.attributes_follow = FALSE;
+    }
 }
 
 void
-nfs3_stat_to_fattr3 (struct iatt *buf, fattr3 *fa)
+nfs3_stat_to_fattr3(struct iatt *buf, fattr3 *fa)
 {
-        if (buf == NULL || fa == NULL) {
-                errno = EINVAL;
-                return;
-        }
-
-        if (IA_ISDIR (buf->ia_type))
-                fa->type = NF3DIR;
-        else if (IA_ISREG (buf->ia_type))
-                fa->type = NF3REG;
-        else if (IA_ISCHR (buf->ia_type))
-                fa->type = NF3CHR;
-        else if (IA_ISBLK (buf->ia_type))
-                fa->type = NF3BLK;
-        else if (IA_ISFIFO (buf->ia_type))
-                fa->type = NF3FIFO;
-        else if (IA_ISLNK (buf->ia_type))
-                fa->type = NF3LNK;
-        else if (IA_ISSOCK (buf->ia_type))
-                fa->type = NF3SOCK;
-
-        if (IA_PROT_RUSR (buf->ia_prot))
-                fa->mode |= NFS3MODE_ROWNER;
-        if (IA_PROT_WUSR (buf->ia_prot))
-                fa->mode |= NFS3MODE_WOWNER;
-        if (IA_PROT_XUSR (buf->ia_prot))
-                fa->mode |= NFS3MODE_XOWNER;
-
-        if (IA_PROT_RGRP (buf->ia_prot))
-                fa->mode |= NFS3MODE_RGROUP;
-        if (IA_PROT_WGRP (buf->ia_prot))
-                fa->mode |= NFS3MODE_WGROUP;
-        if (IA_PROT_XGRP (buf->ia_prot))
-                fa->mode |= NFS3MODE_XGROUP;
-
-        if (IA_PROT_ROTH (buf->ia_prot))
-                fa->mode |= NFS3MODE_ROTHER;
-        if (IA_PROT_WOTH (buf->ia_prot))
-                fa->mode |= NFS3MODE_WOTHER;
-        if (IA_PROT_XOTH (buf->ia_prot))
-                fa->mode |= NFS3MODE_XOTHER;
-
-        if (IA_PROT_SUID (buf->ia_prot))
-                fa->mode |= NFS3MODE_SETXUID;
-        if (IA_PROT_SGID (buf->ia_prot))
-                fa->mode |= NFS3MODE_SETXGID;
-        if (IA_PROT_STCKY (buf->ia_prot))
-                fa->mode |= NFS3MODE_SAVESWAPTXT;
-
-        fa->nlink = buf->ia_nlink;
-        fa->uid = buf->ia_uid;
-        fa->gid = buf->ia_gid;
-        fa->size = buf->ia_size;
-        fa->used = (buf->ia_blocks * 512);
-
-        if ((IA_ISCHR (buf->ia_type) || IA_ISBLK (buf->ia_type))) {
-                fa->rdev.specdata1 = ia_major (buf->ia_rdev);
-                fa->rdev.specdata2 = ia_minor (buf->ia_rdev);
-        } else {
-                fa->rdev.specdata1 = 0;
-                fa->rdev.specdata2 = 0;
-        }
-
-        fa->fsid = buf->ia_dev;
-        fa->fileid = nfs3_iatt_gfid_to_ino (buf);
-
-        fa->atime.seconds = buf->ia_atime;
-        fa->atime.nseconds = buf->ia_atime_nsec;
-
-        fa->ctime.seconds = buf->ia_ctime;
-        fa->ctime.nseconds = buf->ia_ctime_nsec;
-
-        fa->mtime.seconds = buf->ia_mtime;
-        fa->mtime.nseconds = buf->ia_mtime_nsec;
+    if (buf == NULL || fa == NULL) {
+        errno = EINVAL;
+        return;
+    }
+
+    if (IA_ISDIR(buf->ia_type))
+        fa->type = NF3DIR;
+    else if (IA_ISREG(buf->ia_type))
+        fa->type = NF3REG;
+    else if (IA_ISCHR(buf->ia_type))
+        fa->type = NF3CHR;
+    else if (IA_ISBLK(buf->ia_type))
+        fa->type = NF3BLK;
+    else if (IA_ISFIFO(buf->ia_type))
+        fa->type = NF3FIFO;
+    else if (IA_ISLNK(buf->ia_type))
+        fa->type = NF3LNK;
+    else if (IA_ISSOCK(buf->ia_type))
+        fa->type = NF3SOCK;
+
+    if (IA_PROT_RUSR(buf->ia_prot))
+        fa->mode |= NFS3MODE_ROWNER;
+    if (IA_PROT_WUSR(buf->ia_prot))
+        fa->mode |= NFS3MODE_WOWNER;
+    if (IA_PROT_XUSR(buf->ia_prot))
+        fa->mode |= NFS3MODE_XOWNER;
+
+    if (IA_PROT_RGRP(buf->ia_prot))
+        fa->mode |= NFS3MODE_RGROUP;
+    if (IA_PROT_WGRP(buf->ia_prot))
+        fa->mode |= NFS3MODE_WGROUP;
+    if (IA_PROT_XGRP(buf->ia_prot))
+        fa->mode |= NFS3MODE_XGROUP;
+
+    if (IA_PROT_ROTH(buf->ia_prot))
+        fa->mode |= NFS3MODE_ROTHER;
+    if (IA_PROT_WOTH(buf->ia_prot))
+        fa->mode |= NFS3MODE_WOTHER;
+    if (IA_PROT_XOTH(buf->ia_prot))
+        fa->mode |= NFS3MODE_XOTHER;
+
+    if (IA_PROT_SUID(buf->ia_prot))
+        fa->mode |= NFS3MODE_SETXUID;
+    if (IA_PROT_SGID(buf->ia_prot))
+        fa->mode |= NFS3MODE_SETXGID;
+    if (IA_PROT_STCKY(buf->ia_prot))
+        fa->mode |= NFS3MODE_SAVESWAPTXT;
+
+    fa->nlink = buf->ia_nlink;
+    fa->uid = buf->ia_uid;
+    fa->gid = buf->ia_gid;
+    fa->size = buf->ia_size;
+    fa->used = (buf->ia_blocks * 512);
+
+    if ((IA_ISCHR(buf->ia_type) || IA_ISBLK(buf->ia_type))) {
+        fa->rdev.specdata1 = ia_major(buf->ia_rdev);
+        fa->rdev.specdata2 = ia_minor(buf->ia_rdev);
+    } else {
+        fa->rdev.specdata1 = 0;
+        fa->rdev.specdata2 = 0;
+    }
+
+    fa->fsid = buf->ia_dev;
+    fa->fileid = nfs3_iatt_gfid_to_ino(buf);
+
+    fa->atime.seconds = buf->ia_atime;
+    fa->atime.nseconds = buf->ia_atime_nsec;
+
+    fa->ctime.seconds = buf->ia_ctime;
+    fa->ctime.nseconds = buf->ia_ctime_nsec;
+
+    fa->mtime.seconds = buf->ia_mtime;
+    fa->mtime.nseconds = buf->ia_mtime_nsec;
 }
 
-
 post_op_attr
-nfs3_stat_to_post_op_attr (struct iatt *buf)
+nfs3_stat_to_post_op_attr(struct iatt *buf)
 {
-        post_op_attr    attr = {0, };
-        if (!buf)
-                return attr;
+    post_op_attr attr = {
+        0,
+    };
+    if (!buf)
+        return attr;
 
-        /* Some performance translators return zero-filled stats when they
-         * do not have up-to-date attributes. Need to handle this by not
-         * returning these zeroed out attrs.
-         */
-        attr.attributes_follow = FALSE;
-        if (nfs_zero_filled_stat (buf))
-                goto out;
+    /* Some performance translators return zero-filled stats when they
+     * do not have up-to-date attributes. Need to handle this by not
+     * returning these zeroed out attrs.
+     */
+    attr.attributes_follow = FALSE;
+    if (gf_is_zero_filled_stat(buf))
+        goto out;
 
-        nfs3_stat_to_fattr3 (buf, &(attr.post_op_attr_u.attributes));
-        attr.attributes_follow = TRUE;
+    nfs3_stat_to_fattr3(buf, &(attr.post_op_attr_u.attributes));
+    attr.attributes_follow = TRUE;
 
 out:
-        return attr;
+    return attr;
 }
 
-
 pre_op_attr
-nfs3_stat_to_pre_op_attr (struct iatt *pre)
-{
-        pre_op_attr     poa = {0, };
-
-        /* Some performance translators return zero-filled stats when they
-         * do not have up-to-date attributes. Need to handle this by not
-         * returning these zeroed out attrs.
-         */
-        poa.attributes_follow = FALSE;
-        if (nfs_zero_filled_stat (pre))
-                goto out;
-
-        poa.attributes_follow = TRUE;
-        poa.pre_op_attr_u.attributes.size = pre->ia_size;
-        poa.pre_op_attr_u.attributes.mtime.seconds = pre->ia_mtime;
-        poa.pre_op_attr_u.attributes.mtime.nseconds = pre->ia_mtime_nsec;
-        poa.pre_op_attr_u.attributes.ctime.seconds = pre->ia_ctime;
-        poa.pre_op_attr_u.attributes.ctime.nseconds = pre->ia_ctime_nsec;
+nfs3_stat_to_pre_op_attr(struct iatt *pre)
+{
+    pre_op_attr poa = {
+        0,
+    };
+
+    /* Some performance translators return zero-filled stats when they
+     * do not have up-to-date attributes. Need to handle this by not
+     * returning these zeroed out attrs.
+     */
+    poa.attributes_follow = FALSE;
+    if (gf_is_zero_filled_stat(pre))
+        goto out;
+
+    poa.attributes_follow = TRUE;
+    poa.pre_op_attr_u.attributes.size = pre->ia_size;
+    poa.pre_op_attr_u.attributes.mtime.seconds = pre->ia_mtime;
+    poa.pre_op_attr_u.attributes.mtime.nseconds = pre->ia_mtime_nsec;
+    poa.pre_op_attr_u.attributes.ctime.seconds = pre->ia_ctime;
+    poa.pre_op_attr_u.attributes.ctime.nseconds = pre->ia_ctime_nsec;
 
 out:
-        return poa;
+    return poa;
 }
 
 void
-nfs3_fill_lookup3res_success (lookup3res *res, nfsstat3 stat,
-                              struct nfs3_fh *fh, struct iatt *buf,
-                              struct iatt *postparent)
+nfs3_fill_lookup3res_success(lookup3res *res, nfsstat3 stat, struct nfs3_fh *fh,
+                             struct iatt *buf, struct iatt *postparent)
 {
-        post_op_attr    obj, dir;
-        uint32_t        fhlen = 0;
+    post_op_attr obj, dir;
+    uint32_t fhlen = 0;
 
-        res->status = stat;
-        if (fh) {
-                res->lookup3res_u.resok.object.data.data_val = (void *)fh;
-                fhlen = nfs3_fh_compute_size ();
-                res->lookup3res_u.resok.object.data.data_len = fhlen;
-        }
+    res->status = stat;
+    if (fh) {
+        res->lookup3res_u.resok.object.data.data_val = (void *)fh;
+        fhlen = nfs3_fh_compute_size();
+        res->lookup3res_u.resok.object.data.data_len = fhlen;
+    }
 
-        obj.attributes_follow = FALSE;
-        dir.attributes_follow = FALSE;
-        obj = nfs3_stat_to_post_op_attr (buf);
-        dir = nfs3_stat_to_post_op_attr (postparent);
+    obj.attributes_follow = FALSE;
+    dir.attributes_follow = FALSE;
+    obj = nfs3_stat_to_post_op_attr(buf);
+    dir = nfs3_stat_to_post_op_attr(postparent);
 
-        res->lookup3res_u.resok.obj_attributes = obj;
-        res->lookup3res_u.resok.dir_attributes = dir;
+    res->lookup3res_u.resok.obj_attributes = obj;
+    res->lookup3res_u.resok.dir_attributes = dir;
 }
 
-
 void
-nfs3_fill_lookup3res (lookup3res *res, nfsstat3 stat, struct nfs3_fh *newfh,
-                      struct iatt *buf, struct iatt *postparent,
-                      uint64_t deviceid)
+nfs3_fill_lookup3res(lookup3res *res, nfsstat3 stat, struct nfs3_fh *newfh,
+                     struct iatt *buf, struct iatt *postparent,
+                     uint64_t deviceid)
 {
-
-        memset (res, 0, sizeof (*res));
-        nfs3_map_deviceid_to_statdev (buf, deviceid);
-        nfs3_map_deviceid_to_statdev (postparent, deviceid);
-        if (stat != NFS3_OK)
-                nfs3_fill_lookup3res_error (res, stat, postparent);
-        else
-                nfs3_fill_lookup3res_success (res, stat, newfh, buf,
-                                              postparent);
+    memset(res, 0, sizeof(*res));
+    nfs3_map_deviceid_to_statdev(buf, deviceid);
+    nfs3_map_deviceid_to_statdev(postparent, deviceid);
+    if (stat != NFS3_OK)
+        nfs3_fill_lookup3res_error(res, stat, postparent);
+    else
+        nfs3_fill_lookup3res_success(res, stat, newfh, buf, postparent);
 }
 
 struct nfs3_fh
-nfs3_extract_getattr_fh (getattr3args *args)
+nfs3_extract_getattr_fh(getattr3args *args)
 {
-        return nfs3_extract_nfs3_fh(args->object);
+    return nfs3_extract_nfs3_fh(args->object);
 }
 
-
 void
-nfs3_fill_getattr3res (getattr3res *res, nfsstat3 stat, struct iatt *buf,
-                       uint64_t deviceid)
+nfs3_fill_getattr3res(getattr3res *res, nfsstat3 stat, struct iatt *buf,
+                      uint64_t deviceid)
 {
+    memset(res, 0, sizeof(*res));
+    res->status = stat;
+    if (stat != NFS3_OK)
+        return;
 
-        memset (res, 0, sizeof (*res));
-        res->status = stat;
-        if (stat != NFS3_OK)
-                return;
-
-        nfs3_map_deviceid_to_statdev (buf, deviceid);
-        nfs3_stat_to_fattr3 (buf, &(res->getattr3res_u.resok.obj_attributes));
+    nfs3_map_deviceid_to_statdev(buf, deviceid);
+    nfs3_stat_to_fattr3(buf, &(res->getattr3res_u.resok.obj_attributes));
 }
 
-
 struct nfs3_fh
-nfs3_extract_fsinfo_fh (fsinfo3args *args)
+nfs3_extract_fsinfo_fh(fsinfo3args *args)
 {
-        return nfs3_extract_nfs3_fh (args->fsroot);
+    return nfs3_extract_nfs3_fh(args->fsroot);
 }
 
-
 void
-nfs3_fill_fsinfo3res (struct nfs3_state *nfs3, fsinfo3res *res,
-                      nfsstat3 status, struct iatt *fsroot, uint64_t deviceid)
+nfs3_fill_fsinfo3res(struct nfs3_state *nfs3, fsinfo3res *res, nfsstat3 status,
+                     struct iatt *fsroot, uint64_t deviceid)
 {
-        fsinfo3resok    resok = {{0}, };
-        nfstime3        tdelta = GF_NFS3_TIMEDELTA_SECS;
-
-        memset (res, 0, sizeof (*res));
-        res->status = status;
-        if (status != NFS3_OK)
-                return;
+    fsinfo3resok resok = {
+        {0},
+    };
+    nfstime3 tdelta = GF_NFS3_TIMEDELTA_SECS;
 
-        nfs3_map_deviceid_to_statdev (fsroot, deviceid);
-        resok.obj_attributes = nfs3_stat_to_post_op_attr (fsroot);
-        resok.rtmax = nfs3->readsize;
-        resok.rtpref = nfs3->readsize;
-        resok.rtmult = GF_NFS3_RTMULT;
-        resok.wtmax = nfs3->writesize;
-        resok.wtpref = nfs3->writesize;
-        resok.wtmult = GF_NFS3_WTMULT;
-        resok.dtpref = nfs3->readdirsize;
-        resok.maxfilesize = GF_NFS3_MAXFILESIZE;
-        resok.time_delta = tdelta;
-        resok.properties = GF_NFS3_FS_PROP;
+    memset(res, 0, sizeof(*res));
+    res->status = status;
+    if (status != NFS3_OK)
+        return;
 
-        res->fsinfo3res_u.resok = resok;
+    nfs3_map_deviceid_to_statdev(fsroot, deviceid);
+    resok.obj_attributes = nfs3_stat_to_post_op_attr(fsroot);
+    resok.rtmax = nfs3->readsize;
+    resok.rtpref = nfs3->readsize;
+    resok.rtmult = GF_NFS3_RTMULT;
+    resok.wtmax = nfs3->writesize;
+    resok.wtpref = nfs3->writesize;
+    resok.wtmult = GF_NFS3_WTMULT;
+    resok.dtpref = nfs3->readdirsize;
+    resok.maxfilesize = GF_NFS3_MAXFILESIZE;
+    resok.time_delta = tdelta;
+    resok.properties = GF_NFS3_FS_PROP;
 
+    res->fsinfo3res_u.resok = resok;
 }
 
-
 void
-nfs3_prep_lookup3args (lookup3args *args, struct nfs3_fh *fh, char *name)
+nfs3_prep_lookup3args(lookup3args *args, struct nfs3_fh *fh, char *name)
 {
-        memset (args, 0, sizeof (*args));
-        args->what.dir.data.data_val = (void *)fh;
-        args->what.name = name;
+    memset(args, 0, sizeof(*args));
+    args->what.dir.data.data_val = (void *)fh;
+    args->what.name = name;
 }
 
-
 void
-nfs3_prep_getattr3args (getattr3args *args, struct nfs3_fh *fh)
+nfs3_prep_getattr3args(getattr3args *args, struct nfs3_fh *fh)
 {
-        memset (args, 0, sizeof (*args));
-        args->object.data.data_val = (void *)fh;
+    memset(args, 0, sizeof(*args));
+    args->object.data.data_val = (void *)fh;
 }
 
-
 void
-nfs3_prep_fsinfo3args (fsinfo3args *args, struct nfs3_fh *root)
+nfs3_prep_fsinfo3args(fsinfo3args *args, struct nfs3_fh *root)
 {
-        memset (args, 0, sizeof (*args));
-        args->fsroot.data.data_val = (void *)root;
+    memset(args, 0, sizeof(*args));
+    args->fsroot.data.data_val = (void *)root;
 }
 
-
 char *
 nfsstat3_strerror(int stat)
 {
-        int i;
-        for(i = 0; nfs3stat_strerror_table[i].stat != NFS3ERR_END_OF_LIST ; i++) {
-                if (nfs3stat_strerror_table[i].stat == stat)
-                        return nfs3stat_strerror_table[i].strerror;
-        }
+    int i;
+    for (i = 0; nfs3stat_strerror_table[i].stat != NFS3ERR_END_OF_LIST; i++) {
+        if (nfs3stat_strerror_table[i].stat == stat)
+            return nfs3stat_strerror_table[i].strerror;
+    }
 
-        return nfs3stat_strerror_table[i].strerror;
+    return nfs3stat_strerror_table[i].strerror;
 }
 
-
-
 void
-nfs3_prep_access3args (access3args *args, struct nfs3_fh *fh)
+nfs3_prep_access3args(access3args *args, struct nfs3_fh *fh)
 {
-        memset (args, 0, sizeof (*args));
-        args->object.data.data_val = (void *)fh;
+    memset(args, 0, sizeof(*args));
+    args->object.data.data_val = (void *)fh;
 }
 
-#define POSIX_READ      4
-#define POSIX_WRITE     2
-#define POSIX_EXEC      1
+#define POSIX_READ 4
+#define POSIX_WRITE 2
+#define POSIX_EXEC 1
 
 uint32_t
-nfs3_accessbits (int32_t accbits)
+nfs3_accessbits(int32_t accbits)
 {
-        uint32_t        accresult = 0;
+    uint32_t accresult = 0;
 
-        if (accbits & POSIX_READ)
-                accresult |= ACCESS3_READ;
+    if (accbits & POSIX_READ)
+        accresult |= ACCESS3_READ;
 
-        if (accbits & POSIX_WRITE)
-                accresult |= (ACCESS3_MODIFY | ACCESS3_EXTEND | ACCESS3_DELETE);
+    if (accbits & POSIX_WRITE)
+        accresult |= (ACCESS3_MODIFY | ACCESS3_EXTEND | ACCESS3_DELETE);
 
-        /* lookup on directory allowed only in case of execute permission */
-        if (accbits & POSIX_EXEC)
-                accresult |= (ACCESS3_EXECUTE | ACCESS3_LOOKUP);
+    /* lookup on directory allowed only in case of execute permission */
+    if (accbits & POSIX_EXEC)
+        accresult |= (ACCESS3_EXECUTE | ACCESS3_LOOKUP);
 
-        return accresult;
+    return accresult;
 }
 
 uint32_t
-nfs3_request_to_accessbits (int32_t accbits)
+nfs3_request_to_accessbits(int32_t accbits)
 {
-        uint32_t        acc_request = 0;
+    uint32_t acc_request = 0;
 
-        if (accbits & ACCESS3_READ)
-                acc_request |= POSIX_READ;
+    if (accbits & ACCESS3_READ)
+        acc_request |= POSIX_READ;
 
-        if (accbits & (ACCESS3_MODIFY | ACCESS3_EXTEND | ACCESS3_DELETE))
-                acc_request |= POSIX_WRITE;
+    if (accbits & (ACCESS3_MODIFY | ACCESS3_EXTEND | ACCESS3_DELETE))
+        acc_request |= POSIX_WRITE;
 
-        /* For lookup on directory check for execute permission */
-        if (accbits & (ACCESS3_EXECUTE | ACCESS3_LOOKUP))
-                acc_request |= POSIX_EXEC;
+    /* For lookup on directory check for execute permission */
+    if (accbits & (ACCESS3_EXECUTE | ACCESS3_LOOKUP))
+        acc_request |= POSIX_EXEC;
 
-        return acc_request;
+    return acc_request;
 }
 void
-nfs3_fill_access3res (access3res *res, nfsstat3 status, int32_t accbits,
-		      int32_t reqaccbits)
+nfs3_fill_access3res(access3res *res, nfsstat3 status, int32_t accbits,
+                     int32_t reqaccbits)
 {
-        uint32_t        accres = 0;
+    uint32_t accres = 0;
 
-        memset (res, 0, sizeof (*res));
-        res->status = status;
-        if (status != NFS3_OK)
-                return;
+    memset(res, 0, sizeof(*res));
+    res->status = status;
+    if (status != NFS3_OK)
+        return;
 
-        accres = nfs3_accessbits (accbits);
+    accres = nfs3_accessbits(accbits);
 
-	/* do not answer what was not asked */
-        res->access3res_u.resok.access = accres & reqaccbits;
+    /* do not answer what was not asked */
+    res->access3res_u.resok.access = accres & reqaccbits;
 }
 
 void
-nfs3_prep_readdir3args (readdir3args *ra, struct nfs3_fh *fh)
+nfs3_prep_readdir3args(readdir3args *ra, struct nfs3_fh *fh)
 {
-        memset (ra, 0, sizeof (*ra));
-        ra->dir.data.data_val = (void *)fh;
+    memset(ra, 0, sizeof(*ra));
+    ra->dir.data.data_val = (void *)fh;
 }
 
-
 int
-nfs3_is_dot_entry (char *entry)
+nfs3_is_dot_entry(char *entry)
 {
-        int     ret = 0;
+    int ret = 0;
 
-        if (!entry)
-                return 0;
+    if (!entry)
+        return 0;
 
-        if (strcmp (entry, ".") == 0)
-                ret = 1;
+    if (strcmp(entry, ".") == 0)
+        ret = 1;
 
-        return ret;
+    return ret;
 }
 
-
 int
-nfs3_is_parentdir_entry (char *entry)
+nfs3_is_parentdir_entry(char *entry)
 {
-        int     ret = 0;
+    int ret = 0;
 
-        if (!entry)
-                return 0;
+    if (!entry)
+        return 0;
 
-        if (strcmp (entry, "..") == 0)
-                ret = 1;
+    if (strcmp(entry, "..") == 0)
+        ret = 1;
 
-        return ret;
+    return ret;
 }
 
-
 void
-nfs3_funge_root_dotdot_dirent (gf_dirent_t *ent, struct nfs3_fh *dfh)
+nfs3_funge_root_dotdot_dirent(gf_dirent_t *ent, struct nfs3_fh *dfh)
 {
-        if ((!ent) || (!dfh))
-                return;
-
-        if (nfs3_fh_is_root_fh (dfh) &&
-            nfs3_is_parentdir_entry (ent->d_name)) {
-                ent->d_ino = 1;
-                ent->d_stat.ia_ino = 1;
-        }
+    if ((!ent) || (!dfh))
+        return;
 
-        if (nfs3_fh_is_root_fh (dfh) &&
-            nfs3_is_dot_entry (ent->d_name)) {
-                ent->d_ino = 1;
-                ent->d_stat.ia_ino = 1;
-        }
+    if (nfs3_fh_is_root_fh(dfh) && nfs3_is_parentdir_entry(ent->d_name)) {
+        ent->d_ino = 1;
+        ent->d_stat.ia_ino = 1;
+    }
 
+    if (nfs3_fh_is_root_fh(dfh) && nfs3_is_dot_entry(ent->d_name)) {
+        ent->d_ino = 1;
+        ent->d_stat.ia_ino = 1;
+    }
 }
 
-
 entry3 *
-nfs3_fill_entry3 (gf_dirent_t *entry, struct nfs3_fh *dfh)
-{
-        entry3          *ent = NULL;
-        if ((!entry) || (!dfh))
-                return NULL;
-
-        ent = GF_CALLOC (1, sizeof (*ent), gf_nfs_mt_entry3);
-        if (!ent)
-                return NULL;
-
-        gf_log (GF_NFS3, GF_LOG_TRACE, "Entry: %s", entry->d_name);
-
-        /* If the entry is . or .., we need to replace the physical ino and gen
-         * with 1 and 0 respectively if the directory is root. This funging is
-         * needed because there is no parent directory of the root. In that
-         * sense the behavior we provide is similar to the output of the
-         * command: "stat /.."
-         */
-        entry->d_ino = nfs3_iatt_gfid_to_ino (&entry->d_stat);
-        nfs3_funge_root_dotdot_dirent (entry, dfh);
-        ent->fileid = entry->d_ino;
-        ent->cookie = entry->d_off;
-        ent->name = GF_CALLOC ((strlen (entry->d_name) + 1), sizeof (char),
-                               gf_nfs_mt_char);
-        if (!ent->name) {
-                GF_FREE (ent);
-                ent = NULL;
-                goto err;
-        }
-        strcpy (ent->name, entry->d_name);
+nfs3_fill_entry3(gf_dirent_t *entry, struct nfs3_fh *dfh)
+{
+    entry3 *ent = NULL;
+    int name_len = 0;
+    if ((!entry) || (!dfh))
+        return NULL;
+
+    ent = GF_CALLOC(1, sizeof(*ent), gf_nfs_mt_entry3);
+    if (!ent)
+        return NULL;
+
+    gf_msg_trace(GF_NFS3, 0, "Entry: %s", entry->d_name);
+
+    /* If the entry is . or .., we need to replace the physical ino and gen
+     * with 1 and 0 respectively if the directory is root. This funging is
+     * needed because there is no parent directory of the root. In that
+     * sense the behavior we provide is similar to the output of the
+     * command: "stat /.."
+     */
+    entry->d_ino = nfs3_iatt_gfid_to_ino(&entry->d_stat);
+    nfs3_funge_root_dotdot_dirent(entry, dfh);
+    ent->fileid = entry->d_ino;
+    ent->cookie = entry->d_off;
+    name_len = strlen(entry->d_name);
+    ent->name = GF_MALLOC(name_len + 1, gf_nfs_mt_char);
+    if (!ent->name) {
+        GF_FREE(ent);
+        ent = NULL;
+        goto err;
+    }
+    strcpy(ent->name, entry->d_name);
+    ent->name[name_len] = '\0';
 
 err:
-        return ent;
+    return ent;
 }
 
-
 void
-nfs3_fill_post_op_fh3 (struct nfs3_fh *fh, post_op_fh3 *pfh)
+nfs3_fill_post_op_fh3(struct nfs3_fh *fh, post_op_fh3 *pfh)
 {
-        uint32_t        fhlen = 0;
+    uint32_t fhlen = 0;
 
-        if ((!fh) || (!pfh))
-                return;
+    if ((!fh) || (!pfh))
+        return;
 
-        pfh->handle_follows = 1;
-        fhlen = nfs3_fh_compute_size ();
-        pfh->post_op_fh3_u.handle.data.data_val = (void *)fh;
-        pfh->post_op_fh3_u.handle.data.data_len = fhlen;
+    pfh->handle_follows = 1;
+    fhlen = nfs3_fh_compute_size();
+    pfh->post_op_fh3_u.handle.data.data_val = (void *)fh;
+    pfh->post_op_fh3_u.handle.data.data_len = fhlen;
 }
 
-
 post_op_fh3
-nfs3_fh_to_post_op_fh3 (struct nfs3_fh *fh)
+nfs3_fh_to_post_op_fh3(struct nfs3_fh *fh)
 {
-        post_op_fh3     pfh = {0, };
-        char            *fhp = NULL;
+    post_op_fh3 pfh = {
+        0,
+    };
+    char *fhp = NULL;
 
-        if (!fh)
-                return pfh;
-
-        pfh.handle_follows = 1;
+    if (!fh)
+        return pfh;
 
-        fhp = GF_CALLOC (1, sizeof (*fh), gf_nfs_mt_char);
-        if (!fhp)
-                return pfh;
+    pfh.handle_follows = 1;
 
-        memcpy (fhp, fh, sizeof (*fh));
-        nfs3_fill_post_op_fh3 ((struct nfs3_fh *)fhp, &pfh);
+    fhp = GF_MALLOC(sizeof(*fh), gf_nfs_mt_char);
+    if (!fhp)
         return pfh;
-}
 
+    memcpy(fhp, fh, sizeof(*fh));
+    nfs3_fill_post_op_fh3((struct nfs3_fh *)fhp, &pfh);
+    return pfh;
+}
 
 entryp3 *
-nfs3_fill_entryp3 (gf_dirent_t *entry, struct nfs3_fh *dirfh, uint64_t devid)
+nfs3_fill_entryp3(gf_dirent_t *entry, struct nfs3_fh *dirfh, uint64_t devid)
+{
+    entryp3 *ent = NULL;
+    struct nfs3_fh newfh = {
+        {0},
+    };
+    int name_len = 0;
+
+    if ((!entry) || (!dirfh))
+        return NULL;
+
+    /* If the entry is . or .., we need to replace the physical ino and gen
+     * with 1 and 0 respectively if the directory is root. This funging is
+     * needed because there is no parent directory of the root. In that
+     * sense the behavior we provide is similar to the output of the
+     * command: "stat /.."
+     */
+    entry->d_ino = nfs3_iatt_gfid_to_ino(&entry->d_stat);
+    nfs3_funge_root_dotdot_dirent(entry, dirfh);
+    gf_msg_trace(GF_NFS3, 0, "Entry: %s, ino: %" PRIu64, entry->d_name,
+                 entry->d_ino);
+    ent = GF_CALLOC(1, sizeof(*ent), gf_nfs_mt_entryp3);
+    if (!ent)
+        return NULL;
+
+    ent->fileid = entry->d_ino;
+    ent->cookie = entry->d_off;
+    name_len = strlen(entry->d_name);
+    ent->name = GF_MALLOC(name_len + 1, gf_nfs_mt_char);
+    if (!ent->name) {
+        GF_FREE(ent);
+        ent = NULL;
+        goto err;
+    }
+    strcpy(ent->name, entry->d_name);
+    ent->name[name_len] = '\0';
+
+    nfs3_fh_build_child_fh(dirfh, &entry->d_stat, &newfh);
+    nfs3_map_deviceid_to_statdev(&entry->d_stat, devid);
+    /* *
+     * In tier volume, the readdirp send only to cold subvol
+     * which will populate in the 'T' file entries in the result.
+     * For such files an explicit stat call is required, by setting
+     * following argument client will perform the same.
+     *
+     * The inode value for 'T' files and directory is NULL, so just
+     * skip the check if it is directory.
+     */
+    if (!(IA_ISDIR(entry->d_stat.ia_type)) && (entry->inode == NULL))
+        ent->name_attributes.attributes_follow = FALSE;
+    else
+        ent->name_attributes = nfs3_stat_to_post_op_attr(&entry->d_stat);
+
+    ent->name_handle = nfs3_fh_to_post_op_fh3(&newfh);
+err:
+    return ent;
+}
+
+void
+nfs3_fill_readdir3res(readdir3res *res, nfsstat3 stat, struct nfs3_fh *dirfh,
+                      uint64_t cverf, struct iatt *dirstat,
+                      gf_dirent_t *entries, count3 count, int is_eof,
+                      uint64_t deviceid)
 {
-        entryp3         *ent = NULL;
-        struct nfs3_fh  newfh = {{0}, };
+    post_op_attr dirattr;
+    entry3 *ent = NULL;
+    entry3 *headentry = NULL;
+    entry3 *preventry = NULL;
+    count3 filled = 0;
+    gf_dirent_t *listhead = NULL;
 
-        if ((!entry) || (!dirfh))
-                return NULL;
+    memset(res, 0, sizeof(*res));
+    res->status = stat;
+    if (stat != NFS3_OK)
+        return;
 
-        /* If the entry is . or .., we need to replace the physical ino and gen
-         * with 1 and 0 respectively if the directory is root. This funging is
-         * needed because there is no parent directory of the root. In that
-         * sense the behavior we provide is similar to the output of the
-         * command: "stat /.."
-         */
-        entry->d_ino = nfs3_iatt_gfid_to_ino (&entry->d_stat);
-        nfs3_funge_root_dotdot_dirent (entry, dirfh);
-        gf_log (GF_NFS3, GF_LOG_TRACE, "Entry: %s, ino: %"PRIu64,
-                entry->d_name, entry->d_ino);
-        ent = GF_CALLOC (1, sizeof (*ent), gf_nfs_mt_entryp3);
+    nfs3_map_deviceid_to_statdev(dirstat, deviceid);
+    dirattr = nfs3_stat_to_post_op_attr(dirstat);
+    res->readdir3res_u.resok.dir_attributes = dirattr;
+    res->readdir3res_u.resok.reply.eof = (bool_t)is_eof;
+    memcpy(res->readdir3res_u.resok.cookieverf, &cverf, sizeof(cverf));
+
+    filled = NFS3_READDIR_RESOK_SIZE;
+    /* First entry is just the list head */
+    listhead = entries;
+    entries = entries->next;
+    while (((entries) && (entries != listhead)) && (filled < count)) {
+        /*
+        if ((strcmp (entries->d_name, ".") == 0) ||
+            (strcmp (entries->d_name, "..") == 0))
+                goto nextentry;
+                */
+        ent = nfs3_fill_entry3(entries, dirfh);
         if (!ent)
-                return NULL;
-
-        ent->fileid = entry->d_ino;
-        ent->cookie = entry->d_off;
-        ent->name = GF_CALLOC ((strlen (entry->d_name) + 1), sizeof (char),
-                               gf_nfs_mt_char);
-        if (!ent->name) {
-                GF_FREE (ent);
-                ent = NULL;
-                goto err;
-        }
-        strcpy (ent->name, entry->d_name);
+            break;
 
-        nfs3_fh_build_child_fh (dirfh, &entry->d_stat, &newfh);
-        nfs3_map_deviceid_to_statdev (&entry->d_stat, devid);
-        ent->name_attributes = nfs3_stat_to_post_op_attr (&entry->d_stat);
-        ent->name_handle = nfs3_fh_to_post_op_fh3 (&newfh);
-err:
-        return ent;
-}
+        if (!headentry)
+            headentry = ent;
 
+        if (preventry) {
+            preventry->nextentry = ent;
+            preventry = ent;
+        } else
+            preventry = ent;
 
-void
-nfs3_fill_readdir3res (readdir3res *res, nfsstat3 stat, struct nfs3_fh *dirfh,
-                       uint64_t cverf, struct iatt *dirstat,
-                       gf_dirent_t *entries, count3 count, int is_eof,
-                       uint64_t deviceid)
-{
-        post_op_attr    dirattr;
-        entry3          *ent = NULL;
-        entry3          *headentry = NULL;
-        entry3          *preventry = NULL;
-        count3          filled = 0;
-        gf_dirent_t     *listhead = NULL;
-
-        memset (res, 0, sizeof (*res));
-        res->status = stat;
-        if (stat != NFS3_OK)
-                return;
-
-        nfs3_map_deviceid_to_statdev (dirstat, deviceid);
-        dirattr = nfs3_stat_to_post_op_attr (dirstat);
-        res->readdir3res_u.resok.dir_attributes = dirattr;
-        res->readdir3res_u.resok.reply.eof = (bool_t)is_eof;
-        memcpy (res->readdir3res_u.resok.cookieverf, &cverf, sizeof (cverf));
-
-        filled = NFS3_READDIR_RESOK_SIZE;
-        /* First entry is just the list head */
-        listhead = entries;
+        filled += NFS3_ENTRY3_FIXED_SIZE + strlen(ent->name);
+        // nextentry:
         entries = entries->next;
-        while (((entries) && (entries != listhead)) && (filled < count)) {
-                /*
-                if ((strcmp (entries->d_name, ".") == 0) ||
-                    (strcmp (entries->d_name, "..") == 0))
-                        goto nextentry;
-                        */
-                ent = nfs3_fill_entry3 (entries, dirfh);
-                if (!ent)
-                        break;
-
-                if (!headentry)
-                        headentry = ent;
-
-                if (preventry) {
-                        preventry->nextentry = ent;
-                        preventry = ent;
-                } else
-                        preventry = ent;
-
-                filled += NFS3_ENTRY3_FIXED_SIZE + strlen (ent->name);
-//nextentry:
-                entries = entries->next;
-        }
+    }
 
-        res->readdir3res_u.resok.reply.entries = headentry;
+    res->readdir3res_u.resok.reply.entries = headentry;
 
-        return;
+    return;
 }
 
-
 void
-nfs3_fill_readdirp3res (readdirp3res *res, nfsstat3 stat,
-                        struct nfs3_fh *dirfh, uint64_t cverf,
-                        struct iatt *dirstat, gf_dirent_t *entries,
-                        count3 dircount, count3 maxcount, int is_eof,
-                        uint64_t deviceid)
-{
-        post_op_attr    dirattr;
-        entryp3         *ent = NULL;
-        entryp3         *headentry = NULL;
-        entryp3         *preventry = NULL;
-        count3          filled = 0;
-        gf_dirent_t     *listhead = NULL;
-        int             fhlen = 0;
+nfs3_fill_readdirp3res(readdirp3res *res, nfsstat3 stat, struct nfs3_fh *dirfh,
+                       uint64_t cverf, struct iatt *dirstat,
+                       gf_dirent_t *entries, count3 dircount, count3 maxcount,
+                       int is_eof, uint64_t deviceid)
+{
+    post_op_attr dirattr;
+    entryp3 *ent = NULL;
+    entryp3 *headentry = NULL;
+    entryp3 *preventry = NULL;
+    count3 filled = 0;
+    gf_dirent_t *listhead = NULL;
+    int fhlen = 0;
+
+    memset(res, 0, sizeof(*res));
+    res->status = stat;
+    if (stat != NFS3_OK)
+        return;
 
-        memset (res, 0, sizeof (*res));
-        res->status = stat;
-        if (stat != NFS3_OK)
-                return;
+    nfs3_map_deviceid_to_statdev(dirstat, deviceid);
+    dirattr = nfs3_stat_to_post_op_attr(dirstat);
+    res->readdirp3res_u.resok.dir_attributes = dirattr;
+    res->readdirp3res_u.resok.reply.eof = (bool_t)is_eof;
+    memcpy(res->readdirp3res_u.resok.cookieverf, &cverf, sizeof(cverf));
+
+    filled = NFS3_READDIR_RESOK_SIZE;
+    /* First entry is just the list head */
+    listhead = entries;
+    entries = entries->next;
+    while (((entries) && (entries != listhead)) && (filled < maxcount)) {
+        /* Linux does not display . and .. entries unless we provide
+         * these entries here.
+         */
+        /*                if ((strcmp (entries->d_name, ".") == 0) ||
+                            (strcmp (entries->d_name, "..") == 0))
+                                goto nextentry;
+                                */
+        ent = nfs3_fill_entryp3(entries, dirfh, deviceid);
+        if (!ent)
+            break;
 
-        nfs3_map_deviceid_to_statdev (dirstat, deviceid);
-        dirattr = nfs3_stat_to_post_op_attr (dirstat);
-        res->readdirp3res_u.resok.dir_attributes = dirattr;
-        res->readdirp3res_u.resok.reply.eof = (bool_t)is_eof;
-        memcpy (res->readdirp3res_u.resok.cookieverf, &cverf, sizeof (cverf));
+        if (!headentry)
+            headentry = ent;
 
-        filled = NFS3_READDIR_RESOK_SIZE;
-        /* First entry is just the list head */
-        listhead = entries;
+        if (preventry) {
+            preventry->nextentry = ent;
+            preventry = ent;
+        } else
+            preventry = ent;
+
+        fhlen = ent->name_handle.post_op_fh3_u.handle.data.data_len;
+        filled += NFS3_ENTRYP3_FIXED_SIZE + fhlen + strlen(ent->name);
+        // nextentry:
         entries = entries->next;
-        while (((entries) && (entries != listhead)) && (filled < maxcount)) {
-                /* Linux does not display . and .. entries unless we provide
-                 * these entries here.
-                 */
-/*                if ((strcmp (entries->d_name, ".") == 0) ||
-                    (strcmp (entries->d_name, "..") == 0))
-                        goto nextentry;
-                        */
-                ent = nfs3_fill_entryp3 (entries, dirfh, deviceid);
-                if (!ent)
-                        break;
-
-                if (!headentry)
-                        headentry = ent;
-
-                if (preventry) {
-                        preventry->nextentry = ent;
-                        preventry = ent;
-                } else
-                        preventry = ent;
-
-                fhlen = ent->name_handle.post_op_fh3_u.handle.data.data_len;
-                filled += NFS3_ENTRYP3_FIXED_SIZE + fhlen + strlen (ent->name);
-//nextentry:
-                entries = entries->next;
-        }
+    }
 
-        res->readdirp3res_u.resok.reply.entries = headentry;
+    res->readdirp3res_u.resok.reply.entries = headentry;
 
-        return;
+    return;
 }
 
-
 void
-nfs3_prep_readdirp3args (readdirp3args *ra, struct nfs3_fh *fh)
+nfs3_prep_readdirp3args(readdirp3args *ra, struct nfs3_fh *fh)
 {
-        memset (ra, 0, sizeof (*ra));
-        ra->dir.data.data_val = (void *)fh;
+    memset(ra, 0, sizeof(*ra));
+    ra->dir.data.data_val = (void *)fh;
 }
 
 void
-nfs3_free_readdirp3res (readdirp3res *res)
+nfs3_free_readdirp3res(readdirp3res *res)
 {
-        entryp3 *ent = NULL;
-        entryp3 *next = NULL;
-
-        if (!res)
-                return;
+    entryp3 *ent = NULL;
+    entryp3 *next = NULL;
 
-        ent = res->readdirp3res_u.resok.reply.entries;
-        while (ent) {
+    if (!res)
+        return;
 
-                next = ent->nextentry;
-                GF_FREE (ent->name);
-                GF_FREE (ent->name_handle.post_op_fh3_u.handle.data.data_val);
-                GF_FREE (ent);
-                ent = next;
-        }
+    ent = res->readdirp3res_u.resok.reply.entries;
+    while (ent) {
+        next = ent->nextentry;
+        GF_FREE(ent->name);
+        GF_FREE(ent->name_handle.post_op_fh3_u.handle.data.data_val);
+        GF_FREE(ent);
+        ent = next;
+    }
 
-        return;
+    return;
 }
 
-
 void
-nfs3_free_readdir3res (readdir3res *res)
+nfs3_free_readdir3res(readdir3res *res)
 {
-        entry3 *ent = NULL;
-        entry3 *next = NULL;
-
-        if (!res)
-                return;
+    entry3 *ent = NULL;
+    entry3 *next = NULL;
 
-        ent = res->readdir3res_u.resok.reply.entries;
-        while (ent) {
+    if (!res)
+        return;
 
-                next = ent->nextentry;
-                GF_FREE (ent->name);
-                GF_FREE (ent);
-                ent = next;
-        }
+    ent = res->readdir3res_u.resok.reply.entries;
+    while (ent) {
+        next = ent->nextentry;
+        GF_FREE(ent->name);
+        GF_FREE(ent);
+        ent = next;
+    }
 
-        return;
+    return;
 }
 
 void
-nfs3_prep_fsstat3args (fsstat3args *args, struct nfs3_fh *fh)
+nfs3_prep_fsstat3args(fsstat3args *args, struct nfs3_fh *fh)
 {
-        memset (args, 0, sizeof (*args));
-        args->fsroot.data.data_val = (char *)fh;
+    memset(args, 0, sizeof(*args));
+    args->fsroot.data.data_val = (char *)fh;
 }
 
-
 void
-nfs3_fill_fsstat3res (fsstat3res *res, nfsstat3 stat, struct statvfs *fsbuf,
-                      struct iatt *postbuf, uint64_t deviceid)
+nfs3_fill_fsstat3res(fsstat3res *res, nfsstat3 stat, struct statvfs *fsbuf,
+                     struct iatt *postbuf, uint64_t deviceid)
 {
-        post_op_attr    poa;
-        fsstat3resok    resok;
+    post_op_attr poa;
+    fsstat3resok resok;
 
-        memset (res, 0, sizeof (*res));
-        res->status = stat;
-        if (stat != NFS3_OK)
-                return;
+    memset(res, 0, sizeof(*res));
+    res->status = stat;
+    if (stat != NFS3_OK)
+        return;
 
-        nfs3_map_deviceid_to_statdev (postbuf, deviceid);
-        poa = nfs3_stat_to_post_op_attr (postbuf);
-        resok.tbytes = (size3)(fsbuf->f_frsize * fsbuf->f_blocks);
-        resok.fbytes = (size3)(fsbuf->f_frsize * fsbuf->f_bfree);
-        resok.abytes = (size3)(fsbuf->f_frsize * fsbuf->f_bavail);
-        resok.tfiles = (size3)(fsbuf->f_files);
-        resok.ffiles = (size3)(fsbuf->f_ffree);
-        resok.afiles = (size3)(fsbuf->f_favail);
-        resok.invarsec = 0;
+    nfs3_map_deviceid_to_statdev(postbuf, deviceid);
+    poa = nfs3_stat_to_post_op_attr(postbuf);
+    resok.tbytes = (size3)(fsbuf->f_frsize * fsbuf->f_blocks);
+    resok.fbytes = (size3)(fsbuf->f_frsize * fsbuf->f_bfree);
+    resok.abytes = (size3)(fsbuf->f_frsize * fsbuf->f_bavail);
+    resok.tfiles = (size3)(fsbuf->f_files);
+    resok.ffiles = (size3)(fsbuf->f_ffree);
+    resok.afiles = (size3)(fsbuf->f_favail);
+    resok.invarsec = 0;
 
-        resok.obj_attributes = poa;
-        res->fsstat3res_u.resok = resok;
+    resok.obj_attributes = poa;
+    res->fsstat3res_u.resok = resok;
 }
 
-
 int32_t
-nfs3_sattr3_to_setattr_valid (sattr3 *sattr, struct iatt *buf, mode_t *omode)
-{
-        int32_t         valid = 0;
-        ia_prot_t       prot = {0, };
-        mode_t          mode = 0;
-
-        if (!sattr)
-                return 0;
-
-        if (sattr->mode.set_it) {
-                valid |= GF_SET_ATTR_MODE;
-
-                if (sattr->mode.set_mode3_u.mode & NFS3MODE_ROWNER) {
-                        mode |= S_IRUSR;
-                        prot.owner.read = 1;
-                }
-                if (sattr->mode.set_mode3_u.mode & NFS3MODE_WOWNER) {
-                        mode |= S_IWUSR;
-                        prot.owner.write = 1;
-                }
-                if (sattr->mode.set_mode3_u.mode & NFS3MODE_XOWNER) {
-                        mode |= S_IXUSR;
-                        prot.owner.exec = 1;
-                }
-
-                if (sattr->mode.set_mode3_u.mode & NFS3MODE_RGROUP) {
-                        mode |= S_IRGRP;
-                        prot.group.read = 1;
-                }
-                if (sattr->mode.set_mode3_u.mode & NFS3MODE_WGROUP) {
-                        mode |= S_IWGRP;
-                        prot.group.write = 1;
-                }
-                if (sattr->mode.set_mode3_u.mode & NFS3MODE_XGROUP) {
-                        mode |= S_IXGRP;
-                        prot.group.exec = 1;
-                }
-
-                if (sattr->mode.set_mode3_u.mode & NFS3MODE_ROTHER) {
-                        mode |= S_IROTH;
-                        prot.other.read = 1;
-                }
-                if (sattr->mode.set_mode3_u.mode & NFS3MODE_WOTHER) {
-                        mode |= S_IWOTH;
-                        prot.other.write = 1;
-                }
-                if (sattr->mode.set_mode3_u.mode & NFS3MODE_XOTHER) {
-                        mode |= S_IXOTH;
-                        prot.other.exec = 1;
-                }
-
-                if (sattr->mode.set_mode3_u.mode & NFS3MODE_SETXUID) {
-                        mode |= S_ISUID;
-                        prot.suid = 1;
-                }
-                if (sattr->mode.set_mode3_u.mode & NFS3MODE_SETXGID) {
-                        mode |= S_ISGID;
-                        prot.sgid = 1;
-                }
-                if (sattr->mode.set_mode3_u.mode & NFS3MODE_SAVESWAPTXT) {
-                        mode |= S_ISVTX;
-                        prot.sticky = 1;
-                }
-
-                if (buf)
-                        buf->ia_prot = prot;
-                /* Create fop still requires the old mode_t style argument. */
-                if (omode)
-                        *omode = mode;
-        }
+nfs3_sattr3_to_setattr_valid(sattr3 *sattr, struct iatt *buf, mode_t *omode)
+{
+    int32_t valid = 0;
+    ia_prot_t prot = {
+        0,
+    };
+    mode_t mode = 0;
 
-        if (sattr->uid.set_it) {
-                valid |= GF_SET_ATTR_UID;
-                if (buf)
-                        buf->ia_uid = sattr->uid.set_uid3_u.uid;
-        }
+    if (!sattr)
+        return 0;
 
-        if (sattr->gid.set_it) {
-                valid |= GF_SET_ATTR_GID;
-                if (buf)
-                        buf->ia_gid = sattr->gid.set_gid3_u.gid;
-        }
+    if (sattr->mode.set_it) {
+        valid |= GF_SET_ATTR_MODE;
 
-        if (sattr->size.set_it) {
-                valid |= GF_SET_ATTR_SIZE;
-                if (buf)
-                        buf->ia_size = sattr->size.set_size3_u.size;
+        if (sattr->mode.set_mode3_u.mode & NFS3MODE_ROWNER) {
+            mode |= S_IRUSR;
+            prot.owner.read = 1;
         }
-
-        if (sattr->atime.set_it == SET_TO_CLIENT_TIME) {
-                valid |= GF_SET_ATTR_ATIME;
-                if (buf)
-                        buf->ia_atime = sattr->atime.set_atime_u.atime.seconds;
+        if (sattr->mode.set_mode3_u.mode & NFS3MODE_WOWNER) {
+            mode |= S_IWUSR;
+            prot.owner.write = 1;
+        }
+        if (sattr->mode.set_mode3_u.mode & NFS3MODE_XOWNER) {
+            mode |= S_IXUSR;
+            prot.owner.exec = 1;
         }
 
-        if (sattr->atime.set_it == SET_TO_SERVER_TIME) {
-                valid |= GF_SET_ATTR_ATIME;
-                if (buf)
-                        buf->ia_atime = time (NULL);
+        if (sattr->mode.set_mode3_u.mode & NFS3MODE_RGROUP) {
+            mode |= S_IRGRP;
+            prot.group.read = 1;
+        }
+        if (sattr->mode.set_mode3_u.mode & NFS3MODE_WGROUP) {
+            mode |= S_IWGRP;
+            prot.group.write = 1;
+        }
+        if (sattr->mode.set_mode3_u.mode & NFS3MODE_XGROUP) {
+            mode |= S_IXGRP;
+            prot.group.exec = 1;
         }
 
-        if (sattr->mtime.set_it == SET_TO_CLIENT_TIME) {
-                valid |= GF_SET_ATTR_MTIME;
-                if (buf)
-                        buf->ia_mtime = sattr->mtime.set_mtime_u.mtime.seconds;
+        if (sattr->mode.set_mode3_u.mode & NFS3MODE_ROTHER) {
+            mode |= S_IROTH;
+            prot.other.read = 1;
+        }
+        if (sattr->mode.set_mode3_u.mode & NFS3MODE_WOTHER) {
+            mode |= S_IWOTH;
+            prot.other.write = 1;
+        }
+        if (sattr->mode.set_mode3_u.mode & NFS3MODE_XOTHER) {
+            mode |= S_IXOTH;
+            prot.other.exec = 1;
         }
 
-        if (sattr->mtime.set_it == SET_TO_SERVER_TIME) {
-                valid |= GF_SET_ATTR_MTIME;
-                if (buf)
-                        buf->ia_mtime = time (NULL);
+        if (sattr->mode.set_mode3_u.mode & NFS3MODE_SETXUID) {
+            mode |= S_ISUID;
+            prot.suid = 1;
+        }
+        if (sattr->mode.set_mode3_u.mode & NFS3MODE_SETXGID) {
+            mode |= S_ISGID;
+            prot.sgid = 1;
+        }
+        if (sattr->mode.set_mode3_u.mode & NFS3MODE_SAVESWAPTXT) {
+            mode |= S_ISVTX;
+            prot.sticky = 1;
         }
 
-        return valid;
+        if (buf)
+            buf->ia_prot = prot;
+        /* Create fop still requires the old mode_t style argument. */
+        if (omode)
+            *omode = mode;
+    }
+
+    if (sattr->uid.set_it) {
+        valid |= GF_SET_ATTR_UID;
+        if (buf)
+            buf->ia_uid = sattr->uid.set_uid3_u.uid;
+    }
+
+    if (sattr->gid.set_it) {
+        valid |= GF_SET_ATTR_GID;
+        if (buf)
+            buf->ia_gid = sattr->gid.set_gid3_u.gid;
+    }
+
+    if (sattr->size.set_it) {
+        valid |= GF_SET_ATTR_SIZE;
+        if (buf)
+            buf->ia_size = sattr->size.set_size3_u.size;
+    }
+
+    if (sattr->atime.set_it == SET_TO_CLIENT_TIME) {
+        valid |= GF_SET_ATTR_ATIME;
+        if (buf)
+            buf->ia_atime = sattr->atime.set_atime_u.atime.seconds;
+    }
+
+    if (sattr->atime.set_it == SET_TO_SERVER_TIME) {
+        valid |= GF_SET_ATTR_ATIME;
+        if (buf)
+            buf->ia_atime = gf_time();
+    }
+
+    if (sattr->mtime.set_it == SET_TO_CLIENT_TIME) {
+        valid |= GF_SET_ATTR_MTIME;
+        if (buf)
+            buf->ia_mtime = sattr->mtime.set_mtime_u.mtime.seconds;
+    }
+
+    if (sattr->mtime.set_it == SET_TO_SERVER_TIME) {
+        valid |= GF_SET_ATTR_MTIME;
+        if (buf)
+            buf->ia_mtime = gf_time();
+    }
+
+    return valid;
 }
 
-
 wcc_data
-nfs3_stat_to_wcc_data (struct iatt *pre, struct iatt *post)
+nfs3_stat_to_wcc_data(struct iatt *pre, struct iatt *post)
 {
-        wcc_data        wd = {{0}, };
+    wcc_data wd = {
+        {0},
+    };
 
-        if (post)
-                wd.after = nfs3_stat_to_post_op_attr (post);
-        if (pre)
-                wd.before = nfs3_stat_to_pre_op_attr (pre);
+    if (post)
+        wd.after = nfs3_stat_to_post_op_attr(post);
+    if (pre)
+        wd.before = nfs3_stat_to_pre_op_attr(pre);
 
-        return wd;
+    return wd;
 }
 
 void
-nfs3_fill_create3res (create3res *res, nfsstat3 stat, struct nfs3_fh *newfh,
-                      struct iatt *newbuf, struct iatt *preparent,
-                      struct iatt *postparent, uint64_t deviceid)
+nfs3_fill_create3res(create3res *res, nfsstat3 stat, struct nfs3_fh *newfh,
+                     struct iatt *newbuf, struct iatt *preparent,
+                     struct iatt *postparent, uint64_t deviceid)
 {
-        post_op_attr    poa = {0, };
-        wcc_data        dirwcc = {{0}, };
+    post_op_attr poa = {
+        0,
+    };
+    wcc_data dirwcc = {
+        {0},
+    };
 
-        memset (res, 0, sizeof (*res));
-        res->status = stat;
-        if (stat != NFS3_OK)
-                return;
+    memset(res, 0, sizeof(*res));
+    res->status = stat;
+    if (stat != NFS3_OK)
+        return;
 
-        nfs3_fill_post_op_fh3 (newfh, &res->create3res_u.resok.obj);
-        nfs3_map_deviceid_to_statdev (newbuf, deviceid);
-        poa = nfs3_stat_to_post_op_attr (newbuf);
-        res->create3res_u.resok.obj_attributes = poa;
-        nfs3_map_deviceid_to_statdev (preparent, deviceid);
-        nfs3_map_deviceid_to_statdev (postparent, deviceid);
-        dirwcc = nfs3_stat_to_wcc_data (preparent, postparent);
+    nfs3_fill_post_op_fh3(newfh, &res->create3res_u.resok.obj);
+    nfs3_map_deviceid_to_statdev(newbuf, deviceid);
+    poa = nfs3_stat_to_post_op_attr(newbuf);
+    res->create3res_u.resok.obj_attributes = poa;
+    nfs3_map_deviceid_to_statdev(preparent, deviceid);
+    nfs3_map_deviceid_to_statdev(postparent, deviceid);
+    dirwcc = nfs3_stat_to_wcc_data(preparent, postparent);
 
-        res->create3res_u.resok.dir_wcc = dirwcc;
+    res->create3res_u.resok.dir_wcc = dirwcc;
 }
 
 void
-nfs3_prep_create3args (create3args *args, struct nfs3_fh *fh, char *name)
+nfs3_prep_create3args(create3args *args, struct nfs3_fh *fh, char *name)
 {
-
-        memset (args, 0, sizeof (*args));
-        args->where.dir.data.data_val = (void *)fh;
-        args->where.name = name;
+    memset(args, 0, sizeof(*args));
+    args->where.dir.data.data_val = (void *)fh;
+    args->where.name = name;
 }
 
 void
-nfs3_prep_setattr3args (setattr3args *args, struct nfs3_fh *fh)
+nfs3_prep_setattr3args(setattr3args *args, struct nfs3_fh *fh)
 {
-        memset (args, 0, sizeof (*args));
-        args->object.data.data_val = (void *)fh;
+    memset(args, 0, sizeof(*args));
+    args->object.data.data_val = (void *)fh;
 }
 
-
 void
-nfs3_fill_setattr3res (setattr3res *res, nfsstat3 stat, struct iatt *preop,
-                       struct iatt *postop, uint64_t deviceid)
+nfs3_fill_setattr3res(setattr3res *res, nfsstat3 stat, struct iatt *preop,
+                      struct iatt *postop, uint64_t deviceid)
 {
-        wcc_data        wcc;
-        memset (res, 0, sizeof (*res));
-        res->status = stat;
-        if (stat != NFS3_OK)
-                return;
+    wcc_data wcc;
+    memset(res, 0, sizeof(*res));
+    res->status = stat;
+    if (stat != NFS3_OK)
+        return;
 
-        nfs3_map_deviceid_to_statdev (preop, deviceid);
-        nfs3_map_deviceid_to_statdev (postop, deviceid);
-        wcc = nfs3_stat_to_wcc_data (preop, postop);
-        res->setattr3res_u.resok.obj_wcc = wcc;
+    nfs3_map_deviceid_to_statdev(preop, deviceid);
+    nfs3_map_deviceid_to_statdev(postop, deviceid);
+    wcc = nfs3_stat_to_wcc_data(preop, postop);
+    res->setattr3res_u.resok.obj_wcc = wcc;
 }
 
-
 void
-nfs3_prep_mkdir3args (mkdir3args *args, struct nfs3_fh *dirfh, char *name)
+nfs3_prep_mkdir3args(mkdir3args *args, struct nfs3_fh *dirfh, char *name)
 {
-
-        memset (args, 0, sizeof (*args));
-        args->where.dir.data.data_val = (void *)dirfh;
-        args->where.name = name;
+    memset(args, 0, sizeof(*args));
+    args->where.dir.data.data_val = (void *)dirfh;
+    args->where.name = name;
 }
 
-
 void
-nfs3_fill_mkdir3res (mkdir3res *res, nfsstat3 stat, struct nfs3_fh *fh,
-                     struct iatt *buf, struct iatt *preparent,
-                     struct iatt *postparent, uint64_t deviceid)
+nfs3_fill_mkdir3res(mkdir3res *res, nfsstat3 stat, struct nfs3_fh *fh,
+                    struct iatt *buf, struct iatt *preparent,
+                    struct iatt *postparent, uint64_t deviceid)
 {
-        wcc_data        dirwcc;
-        post_op_attr    poa;
+    wcc_data dirwcc;
+    post_op_attr poa;
 
-        memset (res, 0, sizeof (*res));
-        res->status = stat;
-        if (stat != NFS3_OK)
-                return;
-
-        nfs3_fill_post_op_fh3 (fh, &res->mkdir3res_u.resok.obj);
-        nfs3_map_deviceid_to_statdev (buf, deviceid);
-        poa = nfs3_stat_to_post_op_attr (buf);
-        nfs3_map_deviceid_to_statdev (preparent, deviceid);
-        nfs3_map_deviceid_to_statdev (postparent, deviceid);
-        dirwcc = nfs3_stat_to_wcc_data (preparent, postparent);
-        res->mkdir3res_u.resok.obj_attributes = poa;
-        res->mkdir3res_u.resok.dir_wcc = dirwcc;
+    memset(res, 0, sizeof(*res));
+    res->status = stat;
+    if (stat != NFS3_OK)
+        return;
 
+    nfs3_fill_post_op_fh3(fh, &res->mkdir3res_u.resok.obj);
+    nfs3_map_deviceid_to_statdev(buf, deviceid);
+    poa = nfs3_stat_to_post_op_attr(buf);
+    nfs3_map_deviceid_to_statdev(preparent, deviceid);
+    nfs3_map_deviceid_to_statdev(postparent, deviceid);
+    dirwcc = nfs3_stat_to_wcc_data(preparent, postparent);
+    res->mkdir3res_u.resok.obj_attributes = poa;
+    res->mkdir3res_u.resok.dir_wcc = dirwcc;
 }
 
-
 void
-nfs3_prep_symlink3args (symlink3args *args, struct nfs3_fh *dirfh, char *name,
-                        char *target)
+nfs3_prep_symlink3args(symlink3args *args, struct nfs3_fh *dirfh, char *name,
+                       char *target)
 {
-        memset (args, 0, sizeof (*args));
-        args->where.dir.data.data_val = (void *)dirfh;
-        args->where.name = name;
-        args->symlink.symlink_data = target;
+    memset(args, 0, sizeof(*args));
+    args->where.dir.data.data_val = (void *)dirfh;
+    args->where.name = name;
+    args->symlink.symlink_data = target;
 }
 
-
 void
-nfs3_fill_symlink3res (symlink3res *res, nfsstat3 stat, struct nfs3_fh *fh,
-                       struct iatt *buf, struct iatt *preparent,
-                       struct iatt *postparent, uint64_t deviceid)
+nfs3_fill_symlink3res(symlink3res *res, nfsstat3 stat, struct nfs3_fh *fh,
+                      struct iatt *buf, struct iatt *preparent,
+                      struct iatt *postparent, uint64_t deviceid)
 {
-        wcc_data        dirwcc;
-        post_op_attr    poa;
-
-        memset (res, 0, sizeof (*res));
-        res->status = stat;
-        if (stat != NFS3_OK)
-                return;
+    wcc_data dirwcc;
+    post_op_attr poa;
 
-        nfs3_fill_post_op_fh3 (fh, &res->symlink3res_u.resok.obj);
-        nfs3_map_deviceid_to_statdev (buf, deviceid);
-        poa = nfs3_stat_to_post_op_attr (buf);
-        nfs3_map_deviceid_to_statdev (postparent, deviceid);
-        nfs3_map_deviceid_to_statdev (preparent, deviceid);
-        dirwcc = nfs3_stat_to_wcc_data (preparent, postparent);
-        res->symlink3res_u.resok.obj_attributes = poa;
-        res->symlink3res_u.resok.dir_wcc = dirwcc;
+    memset(res, 0, sizeof(*res));
+    res->status = stat;
+    if (stat != NFS3_OK)
+        return;
 
+    nfs3_fill_post_op_fh3(fh, &res->symlink3res_u.resok.obj);
+    nfs3_map_deviceid_to_statdev(buf, deviceid);
+    poa = nfs3_stat_to_post_op_attr(buf);
+    nfs3_map_deviceid_to_statdev(postparent, deviceid);
+    nfs3_map_deviceid_to_statdev(preparent, deviceid);
+    dirwcc = nfs3_stat_to_wcc_data(preparent, postparent);
+    res->symlink3res_u.resok.obj_attributes = poa;
+    res->symlink3res_u.resok.dir_wcc = dirwcc;
 }
 
-
 void
-nfs3_prep_readlink3args (readlink3args *args, struct nfs3_fh *fh)
+nfs3_prep_readlink3args(readlink3args *args, struct nfs3_fh *fh)
 {
-
-        memset (args, 0, sizeof (*args));
-        args->symlink.data.data_val = (void *)fh;
+    memset(args, 0, sizeof(*args));
+    args->symlink.data.data_val = (void *)fh;
 }
 
-
 void
-nfs3_fill_readlink3res (readlink3res *res, nfsstat3 stat, char *path,
-                        struct iatt *buf, uint64_t deviceid)
+nfs3_fill_readlink3res(readlink3res *res, nfsstat3 stat, char *path,
+                       struct iatt *buf, uint64_t deviceid)
 {
-        post_op_attr    poa;
+    post_op_attr poa;
 
-        memset (res, 0, sizeof (*res));
-        res->status = stat;
+    memset(res, 0, sizeof(*res));
+    res->status = stat;
 
-        if (stat != NFS3_OK)
-                return;
+    if (stat != NFS3_OK)
+        return;
 
-        nfs3_map_deviceid_to_statdev (buf, deviceid);
-        poa = nfs3_stat_to_post_op_attr (buf);
-        res->readlink3res_u.resok.data = (void *)path;
-        res->readlink3res_u.resok.symlink_attributes = poa;
+    nfs3_map_deviceid_to_statdev(buf, deviceid);
+    poa = nfs3_stat_to_post_op_attr(buf);
+    res->readlink3res_u.resok.data = (void *)path;
+    res->readlink3res_u.resok.symlink_attributes = poa;
 }
 
-
 void
-nfs3_prep_mknod3args (mknod3args *args, struct nfs3_fh *fh, char *name)
+nfs3_prep_mknod3args(mknod3args *args, struct nfs3_fh *fh, char *name)
 {
-        memset (args, 0, sizeof (*args));
-        args->where.dir.data.data_val = (void *)fh;
-        args->where.name = name;
-
+    memset(args, 0, sizeof(*args));
+    args->where.dir.data.data_val = (void *)fh;
+    args->where.name = name;
 }
 
 void
-nfs3_fill_mknod3res (mknod3res *res, nfsstat3 stat, struct nfs3_fh *fh,
-                     struct iatt *buf, struct iatt *preparent,
-                     struct iatt *postparent, uint64_t deviceid)
+nfs3_fill_mknod3res(mknod3res *res, nfsstat3 stat, struct nfs3_fh *fh,
+                    struct iatt *buf, struct iatt *preparent,
+                    struct iatt *postparent, uint64_t deviceid)
 {
-        post_op_attr    poa;
-        wcc_data        wccdir;
-
-        memset (res, 0, sizeof (*res));
-        res->status = stat;
-        if (stat != NFS3_OK)
-                return;
+    post_op_attr poa;
+    wcc_data wccdir;
 
-        nfs3_fill_post_op_fh3 (fh, &res->mknod3res_u.resok.obj);
-        nfs3_map_deviceid_to_statdev (buf, deviceid);
-        poa = nfs3_stat_to_post_op_attr (buf);
-        nfs3_map_deviceid_to_statdev (preparent, deviceid);
-        nfs3_map_deviceid_to_statdev (postparent, deviceid);
-        wccdir = nfs3_stat_to_wcc_data (preparent, postparent);
-        res->mknod3res_u.resok.obj_attributes = poa;
-        res->mknod3res_u.resok.dir_wcc = wccdir;
+    memset(res, 0, sizeof(*res));
+    res->status = stat;
+    if (stat != NFS3_OK)
+        return;
 
+    nfs3_fill_post_op_fh3(fh, &res->mknod3res_u.resok.obj);
+    nfs3_map_deviceid_to_statdev(buf, deviceid);
+    poa = nfs3_stat_to_post_op_attr(buf);
+    nfs3_map_deviceid_to_statdev(preparent, deviceid);
+    nfs3_map_deviceid_to_statdev(postparent, deviceid);
+    wccdir = nfs3_stat_to_wcc_data(preparent, postparent);
+    res->mknod3res_u.resok.obj_attributes = poa;
+    res->mknod3res_u.resok.dir_wcc = wccdir;
 }
 
-
 void
-nfs3_fill_remove3res (remove3res *res, nfsstat3 stat, struct iatt *preparent,
-                      struct iatt *postparent, uint64_t deviceid)
+nfs3_fill_remove3res(remove3res *res, nfsstat3 stat, struct iatt *preparent,
+                     struct iatt *postparent, uint64_t deviceid)
 {
-        wcc_data        dirwcc;
+    wcc_data dirwcc;
 
-        memset (res, 0, sizeof (*res));
-        res->status = stat;
-        if (stat != NFS3_OK)
-                return;
+    memset(res, 0, sizeof(*res));
+    res->status = stat;
+    if (stat != NFS3_OK)
+        return;
 
-        nfs3_map_deviceid_to_statdev (preparent, deviceid);
-        nfs3_map_deviceid_to_statdev (postparent, deviceid);
-        dirwcc = nfs3_stat_to_wcc_data (preparent, postparent);
-        res->remove3res_u.resok.dir_wcc = dirwcc;
+    nfs3_map_deviceid_to_statdev(preparent, deviceid);
+    nfs3_map_deviceid_to_statdev(postparent, deviceid);
+    dirwcc = nfs3_stat_to_wcc_data(preparent, postparent);
+    res->remove3res_u.resok.dir_wcc = dirwcc;
 }
 
-
 void
-nfs3_prep_remove3args (remove3args *args, struct nfs3_fh *fh, char *name)
+nfs3_prep_remove3args(remove3args *args, struct nfs3_fh *fh, char *name)
 {
-        memset (args, 0, sizeof (*args));
-        args->object.dir.data.data_val = (void *)fh;
-        args->object.name = name;
+    memset(args, 0, sizeof(*args));
+    args->object.dir.data.data_val = (void *)fh;
+    args->object.name = name;
 }
 
-
 void
-nfs3_prep_rmdir3args (rmdir3args *args, struct nfs3_fh *fh, char *name)
+nfs3_prep_rmdir3args(rmdir3args *args, struct nfs3_fh *fh, char *name)
 {
-        memset (args, 0, sizeof (*args));
-        args->object.dir.data.data_val = (void *)fh;
-        args->object.name = name;
+    memset(args, 0, sizeof(*args));
+    args->object.dir.data.data_val = (void *)fh;
+    args->object.name = name;
 }
 
-
 void
-nfs3_fill_rmdir3res (rmdir3res *res, nfsstat3 stat, struct iatt *preparent,
-                     struct iatt *postparent, uint64_t deviceid)
+nfs3_fill_rmdir3res(rmdir3res *res, nfsstat3 stat, struct iatt *preparent,
+                    struct iatt *postparent, uint64_t deviceid)
 {
-        wcc_data        dirwcc;
-        memset (res, 0, sizeof (*res));
-        res->status = stat;
+    wcc_data dirwcc;
+    memset(res, 0, sizeof(*res));
+    res->status = stat;
 
-        if (stat != NFS3_OK)
-                return;
+    if (stat != NFS3_OK)
+        return;
 
-        nfs3_map_deviceid_to_statdev (postparent, deviceid);
-        nfs3_map_deviceid_to_statdev (preparent, deviceid);
-        dirwcc = nfs3_stat_to_wcc_data (preparent, postparent);
-        res->rmdir3res_u.resok.dir_wcc = dirwcc;
+    nfs3_map_deviceid_to_statdev(postparent, deviceid);
+    nfs3_map_deviceid_to_statdev(preparent, deviceid);
+    dirwcc = nfs3_stat_to_wcc_data(preparent, postparent);
+    res->rmdir3res_u.resok.dir_wcc = dirwcc;
 }
 
-
 void
-nfs3_prep_link3args (link3args *args, struct nfs3_fh *target,
-                     struct nfs3_fh * dirfh, char *name)
+nfs3_prep_link3args(link3args *args, struct nfs3_fh *target,
+                    struct nfs3_fh *dirfh, char *name)
 {
-        memset (args, 0, sizeof (*args));
-        args->file.data.data_val = (void *)target;
-        args->link.dir.data.data_val = (void *)dirfh;
-        args->link.name = name;
+    memset(args, 0, sizeof(*args));
+    args->file.data.data_val = (void *)target;
+    args->link.dir.data.data_val = (void *)dirfh;
+    args->link.name = name;
 }
 
-
 void
-nfs3_fill_link3res (link3res *res, nfsstat3 stat, struct iatt *buf,
-                    struct iatt *preparent, struct iatt *postparent,
-                    uint64_t deviceid)
+nfs3_fill_link3res(link3res *res, nfsstat3 stat, struct iatt *buf,
+                   struct iatt *preparent, struct iatt *postparent,
+                   uint64_t deviceid)
 {
-        post_op_attr    poa;
-        wcc_data        dirwcc;
+    post_op_attr poa;
+    wcc_data dirwcc;
 
-        memset (res, 0, sizeof (*res));
-        res->status = stat;
-        if (stat != NFS3_OK)
-                return;
+    memset(res, 0, sizeof(*res));
+    res->status = stat;
+    if (stat != NFS3_OK)
+        return;
 
-        nfs3_map_deviceid_to_statdev (preparent, deviceid);
-        nfs3_map_deviceid_to_statdev (postparent, deviceid);
-        nfs3_map_deviceid_to_statdev (buf,deviceid);
-        poa = nfs3_stat_to_post_op_attr (buf);
-        dirwcc = nfs3_stat_to_wcc_data (preparent, postparent);
-        res->link3res_u.resok.file_attributes = poa;
-        res->link3res_u.resok.linkdir_wcc = dirwcc;
+    nfs3_map_deviceid_to_statdev(preparent, deviceid);
+    nfs3_map_deviceid_to_statdev(postparent, deviceid);
+    nfs3_map_deviceid_to_statdev(buf, deviceid);
+    poa = nfs3_stat_to_post_op_attr(buf);
+    dirwcc = nfs3_stat_to_wcc_data(preparent, postparent);
+    res->link3res_u.resok.file_attributes = poa;
+    res->link3res_u.resok.linkdir_wcc = dirwcc;
 }
 
-
-
 void
-nfs3_prep_rename3args (rename3args *args, struct nfs3_fh *olddirfh,
-                       char *oldname, struct nfs3_fh *newdirfh, char *newname)
+nfs3_prep_rename3args(rename3args *args, struct nfs3_fh *olddirfh,
+                      char *oldname, struct nfs3_fh *newdirfh, char *newname)
 {
-        memset (args, 0, sizeof (*args));
-
-        args->from.name = oldname;
-        args->from.dir.data.data_val = (void *)olddirfh;
-        args->to.name = newname;
-        args->to.dir.data.data_val = (void *)newdirfh;
+    memset(args, 0, sizeof(*args));
 
+    args->from.name = oldname;
+    args->from.dir.data.data_val = (void *)olddirfh;
+    args->to.name = newname;
+    args->to.dir.data.data_val = (void *)newdirfh;
 }
 
-
 void
-nfs3_fill_rename3res (rename3res *res, nfsstat3 stat, struct iatt *buf,
-                      struct iatt *preoldparent, struct iatt *postoldparent,
-                      struct iatt *prenewparent, struct iatt *postnewparent,
-                      uint64_t deviceid)
+nfs3_fill_rename3res(rename3res *res, nfsstat3 stat, struct iatt *buf,
+                     struct iatt *preoldparent, struct iatt *postoldparent,
+                     struct iatt *prenewparent, struct iatt *postnewparent,
+                     uint64_t deviceid)
 
 {
-        wcc_data        dirwcc;
+    wcc_data dirwcc;
 
-        memset (res, 0, sizeof (*res));
-        res->status = stat;
-        if (stat != NFS3_OK)
-                return;
+    memset(res, 0, sizeof(*res));
+    res->status = stat;
+    if (stat != NFS3_OK)
+        return;
 
-        nfs3_map_deviceid_to_statdev (preoldparent, deviceid);
-        nfs3_map_deviceid_to_statdev (postoldparent, deviceid);
-        nfs3_map_deviceid_to_statdev (prenewparent, deviceid);
-        nfs3_map_deviceid_to_statdev (postnewparent, deviceid);
-        nfs3_map_deviceid_to_statdev (buf, deviceid);
-        dirwcc = nfs3_stat_to_wcc_data (preoldparent, postoldparent);
-        res->rename3res_u.resok.fromdir_wcc = dirwcc;
-        dirwcc = nfs3_stat_to_wcc_data (prenewparent, postnewparent);
-        res->rename3res_u.resok.todir_wcc = dirwcc;
+    nfs3_map_deviceid_to_statdev(preoldparent, deviceid);
+    nfs3_map_deviceid_to_statdev(postoldparent, deviceid);
+    nfs3_map_deviceid_to_statdev(prenewparent, deviceid);
+    nfs3_map_deviceid_to_statdev(postnewparent, deviceid);
+    nfs3_map_deviceid_to_statdev(buf, deviceid);
+    dirwcc = nfs3_stat_to_wcc_data(preoldparent, postoldparent);
+    res->rename3res_u.resok.fromdir_wcc = dirwcc;
+    dirwcc = nfs3_stat_to_wcc_data(prenewparent, postnewparent);
+    res->rename3res_u.resok.todir_wcc = dirwcc;
 }
 
-
 void
-nfs3_prep_write3args (write3args *args, struct nfs3_fh *fh)
+nfs3_prep_write3args(write3args *args, struct nfs3_fh *fh)
 {
-        memset (args, 0, sizeof (*args));
-        args->file.data.data_val = (void *)fh;
+    memset(args, 0, sizeof(*args));
+    args->file.data.data_val = (void *)fh;
 }
 
-
 void
-nfs3_fill_write3res (write3res *res, nfsstat3 stat, count3 count,
-                     stable_how stable, uint64_t wverf, struct iatt *prestat,
-                     struct iatt *poststat, uint64_t deviceid)
+nfs3_fill_write3res(write3res *res, nfsstat3 stat, count3 count,
+                    stable_how stable, uint64_t wverf, struct iatt *prestat,
+                    struct iatt *poststat, uint64_t deviceid)
 {
-        write3resok     resok;
-        memset (res, 0, sizeof (*res));
-        res->status = stat;
-        if (stat != NFS3_OK)
-                return;
+    write3resok resok;
+    memset(res, 0, sizeof(*res));
+    res->status = stat;
+    if (stat != NFS3_OK)
+        return;
 
-        nfs3_map_deviceid_to_statdev (prestat, deviceid);
-        nfs3_map_deviceid_to_statdev (poststat, deviceid);
-        resok.file_wcc = nfs3_stat_to_wcc_data (prestat, poststat);
-        resok.count = count;
-        resok.committed = stable;
-        memcpy (resok.verf, &wverf, sizeof (wverf));
+    nfs3_map_deviceid_to_statdev(prestat, deviceid);
+    nfs3_map_deviceid_to_statdev(poststat, deviceid);
+    resok.file_wcc = nfs3_stat_to_wcc_data(prestat, poststat);
+    resok.count = count;
+    resok.committed = stable;
+    memcpy(resok.verf, &wverf, sizeof(wverf));
 
-        res->write3res_u.resok = resok;
+    res->write3res_u.resok = resok;
 }
 
-
 void
-nfs3_prep_commit3args (commit3args *args, struct nfs3_fh *fh)
+nfs3_prep_commit3args(commit3args *args, struct nfs3_fh *fh)
 {
-        memset (args, 0, sizeof (*args));
-        args->file.data.data_val = (void *)fh;
+    memset(args, 0, sizeof(*args));
+    args->file.data.data_val = (void *)fh;
 }
 
-
 void
-nfs3_fill_commit3res (commit3res *res, nfsstat3 stat, uint64_t wverf,
-                      struct iatt *prestat, struct iatt *poststat,
-                      uint64_t deviceid)
+nfs3_fill_commit3res(commit3res *res, nfsstat3 stat, uint64_t wverf,
+                     struct iatt *prestat, struct iatt *poststat,
+                     uint64_t deviceid)
 {
-        memset (res, 0, sizeof (*res));
-        res->status = stat;
-        if (stat != NFS3_OK)
-                return;
+    memset(res, 0, sizeof(*res));
+    res->status = stat;
+    if (stat != NFS3_OK)
+        return;
 
-        nfs3_map_deviceid_to_statdev (poststat, deviceid);
-        nfs3_map_deviceid_to_statdev (prestat, deviceid);
-        res->commit3res_u.resok.file_wcc = nfs3_stat_to_wcc_data (prestat,
-                                                                  poststat);
-        memcpy (res->commit3res_u.resok.verf, &wverf, sizeof (wverf));
+    nfs3_map_deviceid_to_statdev(poststat, deviceid);
+    nfs3_map_deviceid_to_statdev(prestat, deviceid);
+    res->commit3res_u.resok.file_wcc = nfs3_stat_to_wcc_data(prestat, poststat);
+    memcpy(res->commit3res_u.resok.verf, &wverf, sizeof(wverf));
 }
 
 void
-nfs3_fill_read3res (read3res *res, nfsstat3 stat, count3 count,
-                    struct iatt *poststat, int is_eof, uint64_t deviceid)
+nfs3_fill_read3res(read3res *res, nfsstat3 stat, count3 count,
+                   struct iatt *poststat, int is_eof, uint64_t deviceid)
 {
-        post_op_attr    poa;
+    post_op_attr poa;
 
-        memset (res, 0, sizeof (*res));
-        res->status = stat;
-        if (stat != NFS3_OK)
-                return;
+    memset(res, 0, sizeof(*res));
+    res->status = stat;
+    if (stat != NFS3_OK)
+        return;
 
-        nfs3_map_deviceid_to_statdev (poststat, deviceid);
-        poa = nfs3_stat_to_post_op_attr (poststat);
-        res->read3res_u.resok.file_attributes = poa;
-        res->read3res_u.resok.count = count;
-        res->read3res_u.resok.eof = is_eof;
-        res->read3res_u.resok.data.data_len = count;
+    nfs3_map_deviceid_to_statdev(poststat, deviceid);
+    poa = nfs3_stat_to_post_op_attr(poststat);
+    res->read3res_u.resok.file_attributes = poa;
+    res->read3res_u.resok.count = count;
+    res->read3res_u.resok.eof = is_eof;
+    res->read3res_u.resok.data.data_len = count;
 }
 
-
 void
-nfs3_prep_read3args (read3args *args, struct nfs3_fh *fh)
+nfs3_prep_read3args(read3args *args, struct nfs3_fh *fh)
 {
-        memset (args, 0, sizeof (*args));
-        args->file.data.data_val = (void *)fh;
+    memset(args, 0, sizeof(*args));
+    args->file.data.data_val = (void *)fh;
 }
 
-
 void
-nfs3_fill_pathconf3res (pathconf3res *res, nfsstat3 stat, struct iatt *buf,
-                        uint64_t deviceid)
+nfs3_fill_pathconf3res(pathconf3res *res, nfsstat3 stat, struct iatt *buf,
+                       uint64_t deviceid)
 {
-        pathconf3resok  resok;
+    pathconf3resok resok;
 
-        memset (res, 0, sizeof (*res));
-        res->status = stat;
-        if (stat != NFS3_OK)
-                return;
+    memset(res, 0, sizeof(*res));
+    res->status = stat;
+    if (stat != NFS3_OK)
+        return;
 
-        nfs3_map_deviceid_to_statdev (buf, deviceid);
-        resok.obj_attributes = nfs3_stat_to_post_op_attr (buf);
-        resok.linkmax = 256;
-        resok.name_max = NFS_NAME_MAX;
-        resok.no_trunc = TRUE;
-        resok.chown_restricted = FALSE;
-        resok.case_insensitive = FALSE;
-        resok.case_preserving = TRUE;
+    nfs3_map_deviceid_to_statdev(buf, deviceid);
+    resok.obj_attributes = nfs3_stat_to_post_op_attr(buf);
+    resok.linkmax = 256;
+    resok.name_max = NFS_NAME_MAX;
+    resok.no_trunc = TRUE;
+    resok.chown_restricted = FALSE;
+    resok.case_insensitive = FALSE;
+    resok.case_preserving = TRUE;
 
-        res->pathconf3res_u.resok = resok;
+    res->pathconf3res_u.resok = resok;
 }
 
-
 void
-nfs3_prep_pathconf3args (pathconf3args *args, struct nfs3_fh *fh)
+nfs3_prep_pathconf3args(pathconf3args *args, struct nfs3_fh *fh)
 {
-        memset (args, 0, sizeof (*args));
-        args->object.data.data_val = (void *)fh;
+    memset(args, 0, sizeof(*args));
+    args->object.data.data_val = (void *)fh;
 }
 
-
 int
-nfs3_verify_dircookie (struct nfs3_state *nfs3, fd_t *dirfd, cookie3 cookie,
-                       uint64_t cverf, nfsstat3 *stat)
+nfs3_verify_dircookie(struct nfs3_state *nfs3, fd_t *dirfd, cookie3 cookie,
+                      uint64_t cverf, nfsstat3 *stat)
 {
-        int             ret = -1;
+    int ret = -1;
 
-        if ((!nfs3) || (!dirfd))
-                return -1;
+    if ((!nfs3) || (!dirfd))
+        return -1;
 
-        /* Can assume that this is first read on the dir, so cookie check
-         * is successful by default.
-         */
-        if (cookie == 0)
-                return 0;
-
-        gf_log (GF_NFS3, GF_LOG_TRACE, "Verifying cookie: cverf: %"PRIu64
-                ", cookie: %"PRIu64, cverf, cookie);
-        /* The cookie bad, no way cverf will be zero with a non-zero cookie. */
-        if ((cverf == 0) && (cookie != 0)) {
-                gf_log (GF_NFS3, GF_LOG_TRACE, "Bad cookie requested");
-                if (stat)
-                        *stat = NFS3ERR_BAD_COOKIE;
-                goto err;
-        }
+    /* Can assume that this is first read on the dir, so cookie check
+     * is successful by default.
+     */
+    if (cookie == 0)
+        return 0;
 
-        /* Yes, its true, our cookie is simply the fd_t address.
-         * NOTE: We used have the check for cookieverf but VMWare client sends
-         * a readdirp requests even after we've told it that EOF has been
-         * reached on the directory. This causes a problem because we close a
-         * dir fd_t after reaching EOF. The next readdirp sent by VMWare
-         * contains the address of the closed fd_t as cookieverf. Since we
-         * closed that fd_t, this readdirp results in a new opendir which will
-         * give an fd_t that will fail this check below.
-         */
-/*        if ((cverf != (uint64_t)dirfd)) {
-                gf_log (GF_NFS3, GF_LOG_TRACE, "Cookieverf does not match");
-                if (stat)
-                        *stat = NFS3ERR_BAD_COOKIE;
-                goto err;
-        }
-*/
-        gf_log (GF_NFS3, GF_LOG_TRACE, "Cookie verified");
+    gf_msg_trace(GF_NFS3, 0,
+                 "Verifying cookie: cverf: %" PRIu64 ", cookie: %" PRIu64,
+                 cverf, cookie);
+    /* The cookie bad, no way cverf will be zero with a non-zero cookie. */
+    if ((cverf == 0) && (cookie != 0)) {
+        gf_msg_trace(GF_NFS3, 0, "Bad cookie requested");
         if (stat)
-                *stat = NFS3_OK;
-        ret = 0;
+            *stat = NFS3ERR_BAD_COOKIE;
+        goto err;
+    }
+
+    /* Yes, its true, our cookie is simply the fd_t address.
+     * NOTE: We used have the check for cookieverf but VMWare client sends
+     * a readdirp requests even after we've told it that EOF has been
+     * reached on the directory. This causes a problem because we close a
+     * dir fd_t after reaching EOF. The next readdirp sent by VMWare
+     * contains the address of the closed fd_t as cookieverf. Since we
+     * closed that fd_t, this readdirp results in a new opendir which will
+     * give an fd_t that will fail this check below.
+     */
+    /*        if ((cverf != (uint64_t)dirfd)) {
+                    gf_log (GF_NFS3, GF_LOG_TRACE, "Cookieverf does not match");
+                    if (stat)
+                            *stat = NFS3ERR_BAD_COOKIE;
+                    goto err;
+            }
+    */
+    gf_msg_trace(GF_NFS3, 0, "Cookie verified");
+    if (stat)
+        *stat = NFS3_OK;
+    ret = 0;
 err:
-        return ret;
+    return ret;
 }
 
-
 void
-nfs3_stat_to_errstr (uint32_t xid, char *op, nfsstat3 stat, int pstat,
-                     char *errstr, size_t len)
+nfs3_stat_to_errstr(uint32_t xid, char *op, nfsstat3 stat, int pstat,
+                    char *errstr, size_t len)
 {
-        if ((!op) || (!errstr))
-                return;
+    if ((!op) || (!errstr))
+        return;
 
-        snprintf (errstr, len, "XID: %x, %s: NFS: %d(%s), POSIX: %d(%s)",
-                  xid, op,stat, nfsstat3_strerror (stat), pstat,
-                  strerror (pstat));
+    snprintf(errstr, len, "XID: %x, %s: NFS: %d(%s), POSIX: %d(%s)", xid, op,
+             stat, nfsstat3_strerror(stat), pstat, strerror(pstat));
 }
 
 void
-nfs3_log_common_call (uint32_t xid, char *op, struct nfs3_fh *fh)
+nfs3_log_common_call(uint32_t xid, char *op, struct nfs3_fh *fh)
 {
-        char    fhstr[1024];
+    char fhstr[1024];
 
-        if (THIS->ctx->log.loglevel < GF_LOG_DEBUG)
-                return;
+    if (THIS->ctx->log.loglevel < GF_LOG_DEBUG)
+        return;
 
-        nfs3_fh_to_str (fh, fhstr, sizeof (fhstr));
-        gf_log (GF_NFS3, GF_LOG_DEBUG, "XID: %x, %s: args: %s", xid, op,
-                fhstr);
+    nfs3_fh_to_str(fh, fhstr, sizeof(fhstr));
+    gf_msg_debug(GF_NFS3, 0, "XID: %x, %s: args: %s", xid, op, fhstr);
 }
 
-
 void
-nfs3_log_fh_entry_call (uint32_t xid, char *op, struct nfs3_fh *fh,
-                        char *name)
+nfs3_log_fh_entry_call(uint32_t xid, char *op, struct nfs3_fh *fh, char *name)
 {
-        char    fhstr[1024];
+    char fhstr[1024];
 
-        if (THIS->ctx->log.loglevel < GF_LOG_DEBUG)
-                return;
-        nfs3_fh_to_str (fh, fhstr, sizeof (fhstr));
-        gf_log (GF_NFS3, GF_LOG_DEBUG, "XID: %x, %s: args: %s, name: %s", xid,
-                op, fhstr, name);
+    if (THIS->ctx->log.loglevel < GF_LOG_DEBUG)
+        return;
+    nfs3_fh_to_str(fh, fhstr, sizeof(fhstr));
+    gf_msg_debug(GF_NFS3, 0, "XID: %x, %s: args: %s, name: %s", xid, op, fhstr,
+                 name);
 }
 
-
 void
-nfs3_log_rename_call (uint32_t xid, struct nfs3_fh *src, char *sname,
-                      struct nfs3_fh *dst, char *dname)
+nfs3_log_rename_call(uint32_t xid, struct nfs3_fh *src, char *sname,
+                     struct nfs3_fh *dst, char *dname)
 {
-        char    sfhstr[1024];
-        char    dfhstr[1024];
+    char sfhstr[1024];
+    char dfhstr[1024];
 
-        if (THIS->ctx->log.loglevel < GF_LOG_DEBUG)
-                return;
-        nfs3_fh_to_str (src, sfhstr, sizeof (sfhstr));
-        nfs3_fh_to_str (dst, dfhstr, sizeof (dfhstr));
-        gf_log (GF_NFS3, GF_LOG_DEBUG, "XID: %x, RENAME: args: Src: %s, "
-                "name: %s, Dst: %s, name: %s", xid, sfhstr, sname, dfhstr,
-                dname);
+    if (THIS->ctx->log.loglevel < GF_LOG_DEBUG)
+        return;
+    nfs3_fh_to_str(src, sfhstr, sizeof(sfhstr));
+    nfs3_fh_to_str(dst, dfhstr, sizeof(dfhstr));
+    gf_msg_debug(GF_NFS3, 0,
+                 "XID: %x, RENAME: args: Src: %s, "
+                 "name: %s, Dst: %s, name: %s",
+                 xid, sfhstr, sname, dfhstr, dname);
 }
 
-
-
 void
-nfs3_log_create_call (uint32_t xid, struct nfs3_fh *fh, char *name,
-                      createmode3 mode)
+nfs3_log_create_call(uint32_t xid, struct nfs3_fh *fh, char *name,
+                     createmode3 mode)
 {
-        char    fhstr[1024];
-        char    *modestr = NULL;
-        char    exclmode[] = "EXCLUSIVE";
-        char    unchkd[] = "UNCHECKED";
-        char    guarded[] = "GUARDED";
+    char fhstr[1024];
+    char *modestr = NULL;
+    char exclmode[] = "EXCLUSIVE";
+    char unchkd[] = "UNCHECKED";
+    char guarded[] = "GUARDED";
 
-        if (THIS->ctx->log.loglevel < GF_LOG_DEBUG)
-                return;
-        nfs3_fh_to_str (fh, fhstr, sizeof (fhstr));
-        if (mode == EXCLUSIVE)
-                modestr = exclmode;
-        else if (mode == GUARDED)
-                modestr = guarded;
-        else
-                modestr = unchkd;
+    if (THIS->ctx->log.loglevel < GF_LOG_DEBUG)
+        return;
+    nfs3_fh_to_str(fh, fhstr, sizeof(fhstr));
+    if (mode == EXCLUSIVE)
+        modestr = exclmode;
+    else if (mode == GUARDED)
+        modestr = guarded;
+    else
+        modestr = unchkd;
 
-        gf_log (GF_NFS3, GF_LOG_DEBUG, "XID: %x, CREATE: args: %s, name: %s,"
-                " mode: %s", xid, fhstr, name, modestr);
+    gf_msg_debug(GF_NFS3, 0,
+                 "XID: %x, CREATE: args: %s, name: %s,"
+                 " mode: %s",
+                 xid, fhstr, name, modestr);
 }
 
-
 void
-nfs3_log_mknod_call (uint32_t xid, struct nfs3_fh *fh, char *name, int type)
+nfs3_log_mknod_call(uint32_t xid, struct nfs3_fh *fh, char *name, int type)
 {
-        char    fhstr[1024];
-        char    *modestr = NULL;
-        char    chr[] = "CHAR";
-        char    blk[] = "BLK";
-        char    sock[] = "SOCK";
-        char    fifo[] = "FIFO";
+    char fhstr[1024];
+    char *modestr = NULL;
+    char chr[] = "CHAR";
+    char blk[] = "BLK";
+    char sock[] = "SOCK";
+    char fifo[] = "FIFO";
 
-        if (THIS->ctx->log.loglevel < GF_LOG_DEBUG)
-                return;
-        nfs3_fh_to_str (fh, fhstr, sizeof (fhstr));
-        if (type == NF3CHR)
-                modestr = chr;
-        else if (type == NF3BLK)
-                modestr = blk;
-        else if (type == NF3SOCK)
-                modestr = sock;
-        else
-                modestr = fifo;
+    if (THIS->ctx->log.loglevel < GF_LOG_DEBUG)
+        return;
+    nfs3_fh_to_str(fh, fhstr, sizeof(fhstr));
+    if (type == NF3CHR)
+        modestr = chr;
+    else if (type == NF3BLK)
+        modestr = blk;
+    else if (type == NF3SOCK)
+        modestr = sock;
+    else
+        modestr = fifo;
 
-        gf_log (GF_NFS3, GF_LOG_DEBUG, "XID: %x, MKNOD: args: %s, name: %s,"
-                " type: %s", xid, fhstr, name, modestr);
+    gf_msg_debug(GF_NFS3, 0,
+                 "XID: %x, MKNOD: args: %s, name: %s,"
+                 " type: %s",
+                 xid, fhstr, name, modestr);
 }
 
-
-
 void
-nfs3_log_symlink_call (uint32_t xid, struct nfs3_fh *fh, char *name, char *tgt)
+nfs3_log_symlink_call(uint32_t xid, struct nfs3_fh *fh, char *name, char *tgt)
 {
-        char    fhstr[1024];
+    char fhstr[1024];
 
-        if (THIS->ctx->log.loglevel < GF_LOG_DEBUG)
-                return;
-        nfs3_fh_to_str (fh, fhstr, sizeof (fhstr));
-        gf_log (GF_NFS3, GF_LOG_DEBUG, "XID: %x, SYMLINK: args: %s, name: %s,"
-                " target: %s", xid, fhstr, name, tgt);
+    if (THIS->ctx->log.loglevel < GF_LOG_DEBUG)
+        return;
+    nfs3_fh_to_str(fh, fhstr, sizeof(fhstr));
+    gf_msg_debug(GF_NFS3, 0,
+                 "XID: %x, SYMLINK: args: %s, name: %s,"
+                 " target: %s",
+                 xid, fhstr, name, tgt);
 }
 
-
 void
-nfs3_log_link_call (uint32_t xid, struct nfs3_fh *fh, char *name,
-                    struct nfs3_fh *tgt)
+nfs3_log_link_call(uint32_t xid, struct nfs3_fh *fh, char *name,
+                   struct nfs3_fh *tgt)
 {
-        char    dfhstr[1024];
-        char    tfhstr[1024];
+    char dfhstr[1024];
+    char tfhstr[1024];
 
-        if (THIS->ctx->log.loglevel < GF_LOG_DEBUG)
-                return;
-        nfs3_fh_to_str (fh, dfhstr, sizeof (dfhstr));
-        nfs3_fh_to_str (tgt, tfhstr, sizeof (tfhstr));
-        gf_log (GF_NFS3, GF_LOG_DEBUG, "XID: %x, LINK: args: %s, name: %s,"
-                " target: %s", xid, dfhstr, name, tfhstr);
+    if (THIS->ctx->log.loglevel < GF_LOG_DEBUG)
+        return;
+    nfs3_fh_to_str(fh, dfhstr, sizeof(dfhstr));
+    nfs3_fh_to_str(tgt, tfhstr, sizeof(tfhstr));
+    gf_msg_debug(GF_NFS3, 0,
+                 "XID: %x, LINK: args: %s, name: %s,"
+                 " target: %s",
+                 xid, dfhstr, name, tfhstr);
 }
 
-
 void
-nfs3_log_rw_call (uint32_t xid, char *op, struct nfs3_fh *fh, offset3 offt,
-                  count3 count, int stablewrite)
+nfs3_log_rw_call(uint32_t xid, char *op, struct nfs3_fh *fh, offset3 offt,
+                 count3 count, int stablewrite)
 {
-        char    fhstr[1024];
-
-        if (THIS->ctx->log.loglevel < GF_LOG_DEBUG)
-                return;
-        nfs3_fh_to_str (fh, fhstr, sizeof (fhstr));
-        if (stablewrite == -1)
-                gf_log (GF_NFS3, GF_LOG_DEBUG, "XID: %x, %s: args: %s, offset:"
-                        " %"PRIu64",  count: %"PRIu32, xid, op, fhstr, offt,
-                        count);
-        else
-                gf_log (GF_NFS3, GF_LOG_DEBUG, "XID: %x, %s: args: %s, offset:"
-                        " %"PRIu64",  count: %"PRIu32", %s", xid, op, fhstr,
-                        offt, count,
-                        (stablewrite == UNSTABLE)?"UNSTABLE":"STABLE");
+    char fhstr[1024];
 
+    if (THIS->ctx->log.loglevel < GF_LOG_DEBUG)
+        return;
+    nfs3_fh_to_str(fh, fhstr, sizeof(fhstr));
+    if (stablewrite == -1)
+        gf_msg_debug(GF_NFS3, 0,
+                     "XID: %x, %s: args: %s, offset:"
+                     " %" PRIu64 ",  count: %" PRIu32,
+                     xid, op, fhstr, offt, count);
+    else
+        gf_msg_debug(GF_NFS3, 0,
+                     "XID: %x, %s: args: %s, offset:"
+                     " %" PRIu64 ",  count: %" PRIu32 ", %s",
+                     xid, op, fhstr, offt, count,
+                     (stablewrite == UNSTABLE) ? "UNSTABLE" : "STABLE");
 }
 
-
 int
-nfs3_getattr_loglevel (nfsstat3 stat) {
-
-	int ll = GF_LOG_DEBUG;
-
-	switch (stat) {
+nfs3_getattr_loglevel(nfsstat3 stat)
+{
+    int ll = GF_LOG_DEBUG;
 
+    switch (stat) {
         case NFS3ERR_PERM:
-		ll = GF_LOG_WARNING;
-                break;
+            ll = GF_LOG_WARNING;
+            break;
 
         case NFS3ERR_NOENT:
-		ll = GF_LOG_WARNING;
-                break;
+            ll = GF_LOG_WARNING;
+            break;
 
         case NFS3ERR_ACCES:
-		ll = GF_LOG_WARNING;
-                break;
+            ll = GF_LOG_WARNING;
+            break;
 
         case NFS3ERR_EXIST:
-		ll = GF_LOG_WARNING;
-                break;
+            ll = GF_LOG_WARNING;
+            break;
 
         case NFS3ERR_XDEV:
-		ll = GF_LOG_WARNING;
-                break;
+            ll = GF_LOG_WARNING;
+            break;
 
         case NFS3ERR_NODEV:
-		ll = GF_LOG_WARNING;
-                break;
+            ll = GF_LOG_WARNING;
+            break;
 
         case NFS3ERR_IO:
-		ll = GF_LOG_WARNING;
-                break;
+            ll = GF_LOG_WARNING;
+            break;
 
         case NFS3ERR_NXIO:
-		ll = GF_LOG_WARNING;
-                break;
+            ll = GF_LOG_WARNING;
+            break;
 
         case NFS3ERR_NOTDIR:
-		ll = GF_LOG_WARNING;
-                break;
+            ll = GF_LOG_WARNING;
+            break;
 
         case NFS3ERR_ISDIR:
-		ll = GF_LOG_WARNING;
-                break;
+            ll = GF_LOG_WARNING;
+            break;
 
         case NFS3ERR_INVAL:
-		ll = GF_LOG_WARNING;
-                break;
+            ll = GF_LOG_WARNING;
+            break;
 
         case NFS3ERR_NOSPC:
-		ll = GF_LOG_WARNING;
-                break;
+            ll = GF_LOG_WARNING;
+            break;
 
         case NFS3ERR_ROFS:
-		ll = GF_LOG_WARNING;
-                break;
+            ll = GF_LOG_WARNING;
+            break;
 
         case NFS3ERR_FBIG:
-		ll = GF_LOG_WARNING;
-                break;
+            ll = GF_LOG_WARNING;
+            break;
 
         case NFS3ERR_MLINK:
-		ll = GF_LOG_WARNING;
-                break;
+            ll = GF_LOG_WARNING;
+            break;
 
         case NFS3ERR_NAMETOOLONG:
-		ll = GF_LOG_WARNING;
-                break;
+            ll = GF_LOG_WARNING;
+            break;
 
         case NFS3ERR_NOTEMPTY:
-		ll = GF_LOG_WARNING;
-                break;
+            ll = GF_LOG_WARNING;
+            break;
 
         case NFS3ERR_SERVERFAULT:
-		ll = GF_LOG_WARNING;
-                break;
+            ll = GF_LOG_WARNING;
+            break;
 
         case NFS3ERR_NOTSUPP:
-		ll = GF_LOG_WARNING;
-                break;
+            ll = GF_LOG_WARNING;
+            break;
 
         case NFS3ERR_BADHANDLE:
-		ll = GF_LOG_WARNING;
-                break;
+            ll = GF_LOG_WARNING;
+            break;
 
         case NFS3ERR_STALE:
-		ll = GF_LOG_WARNING;
-                break;
+            ll = GF_LOG_WARNING;
+            break;
 
         case NFS3ERR_DQUOT:
-		ll = GF_LOG_WARNING;
-                break;
+            ll = GF_LOG_WARNING;
+            break;
 
         default:
-		ll = GF_LOG_DEBUG;
-                break;
-        }
+            ll = GF_LOG_DEBUG;
+            break;
+    }
 
-        return ll;
+    return ll;
 }
 
-
 int
-nfs3_setattr_loglevel (nfsstat3 stat) {
-
-	int ll = GF_LOG_DEBUG;
-
-	switch (stat) {
+nfs3_setattr_loglevel(nfsstat3 stat)
+{
+    int ll = GF_LOG_DEBUG;
 
+    switch (stat) {
         case NFS3ERR_NOENT:
-		ll = GF_LOG_WARNING;
-                break;
+            ll = GF_LOG_WARNING;
+            break;
 
         case NFS3ERR_EXIST:
-		ll = GF_LOG_WARNING;
-                break;
+            ll = GF_LOG_WARNING;
+            break;
 
         case NFS3ERR_XDEV:
-		ll = GF_LOG_WARNING;
-                break;
+            ll = GF_LOG_WARNING;
+            break;
 
         case NFS3ERR_NODEV:
-		ll = GF_LOG_WARNING;
-                break;
+            ll = GF_LOG_WARNING;
+            break;
 
         case NFS3ERR_IO:
-		ll = GF_LOG_WARNING;
-                break;
+            ll = GF_LOG_WARNING;
+            break;
 
         case NFS3ERR_NXIO:
-		ll = GF_LOG_WARNING;
-                break;
+            ll = GF_LOG_WARNING;
+            break;
 
         case NFS3ERR_NOTDIR:
-		ll = GF_LOG_WARNING;
-                break;
+            ll = GF_LOG_WARNING;
+            break;
 
         case NFS3ERR_ISDIR:
-		ll = GF_LOG_WARNING;
-                break;
+            ll = GF_LOG_WARNING;
+            break;
 
         case NFS3ERR_INVAL:
-		ll = GF_LOG_WARNING;
-                break;
+            ll = GF_LOG_WARNING;
+            break;
 
         case NFS3ERR_NOSPC:
-		ll = GF_LOG_WARNING;
-                break;
+            ll = GF_LOG_WARNING;
+            break;
 
         case NFS3ERR_ROFS:
-		ll = GF_LOG_WARNING;
-                break;
+            ll = GF_LOG_WARNING;
+            break;
 
         case NFS3ERR_FBIG:
-		ll = GF_LOG_WARNING;
-                break;
+            ll = GF_LOG_WARNING;
+            break;
 
         case NFS3ERR_MLINK:
-		ll = GF_LOG_WARNING;
-                break;
+            ll = GF_LOG_WARNING;
+            break;
 
         case NFS3ERR_NAMETOOLONG:
-		ll = GF_LOG_WARNING;
-                break;
+            ll = GF_LOG_WARNING;
+            break;
 
         case NFS3ERR_NOTEMPTY:
-		ll = GF_LOG_WARNING;
-                break;
+            ll = GF_LOG_WARNING;
+            break;
 
         case NFS3ERR_SERVERFAULT:
-		ll = GF_LOG_WARNING;
-                break;
+            ll = GF_LOG_WARNING;
+            break;
 
         case NFS3ERR_NOTSUPP:
-		ll = GF_LOG_WARNING;
-                break;
+            ll = GF_LOG_WARNING;
+            break;
 
         case NFS3ERR_BADHANDLE:
-		ll = GF_LOG_WARNING;
-                break;
+            ll = GF_LOG_WARNING;
+            break;
 
         case NFS3ERR_STALE:
-		ll = GF_LOG_WARNING;
-                break;
+            ll = GF_LOG_WARNING;
+            break;
 
         case NFS3ERR_DQUOT:
-		ll = GF_LOG_WARNING;
-                break;
+            ll = GF_LOG_WARNING;
+            break;
 
         default:
-		ll = GF_LOG_DEBUG;
-                break;
-        }
+            ll = GF_LOG_DEBUG;
+            break;
+    }
 
-        return ll;
+    return ll;
 }
 
-
 int
-nfs3_lookup_loglevel (nfsstat3 stat) {
-
-	int ll = GF_LOG_DEBUG;
-
-	switch (stat) {
+nfs3_lookup_loglevel(nfsstat3 stat)
+{
+    int ll = GF_LOG_DEBUG;
 
+    switch (stat) {
         case NFS3ERR_PERM:
-		ll = GF_LOG_WARNING;
-                break;
+            ll = GF_LOG_WARNING;
+            break;
 
         case NFS3ERR_ACCES:
-		ll = GF_LOG_WARNING;
-                break;
+            ll = GF_LOG_WARNING;
+            break;
 
         case NFS3ERR_EXIST:
-		ll = GF_LOG_WARNING;
-                break;
+            ll = GF_LOG_WARNING;
+            break;
 
         case NFS3ERR_XDEV:
-		ll = GF_LOG_WARNING;
-                break;
+            ll = GF_LOG_WARNING;
+            break;
 
         case NFS3ERR_NODEV:
-		ll = GF_LOG_WARNING;
-                break;
+            ll = GF_LOG_WARNING;
+            break;
 
         case NFS3ERR_IO:
-		ll = GF_LOG_WARNING;
-                break;
+            ll = GF_LOG_WARNING;
+            break;
 
         case NFS3ERR_NXIO:
-		ll = GF_LOG_WARNING;
-                break;
+            ll = GF_LOG_WARNING;
+            break;
 
         case NFS3ERR_NOTDIR:
-		ll = GF_LOG_WARNING;
-                break;
+            ll = GF_LOG_WARNING;
+            break;
 
         case NFS3ERR_ISDIR:
-		ll = GF_LOG_WARNING;
-                break;
+            ll = GF_LOG_WARNING;
+            break;
 
         case NFS3ERR_INVAL:
-		ll = GF_LOG_WARNING;
-                break;
+            ll = GF_LOG_WARNING;
+            break;
 
         case NFS3ERR_NOSPC:
-		ll = GF_LOG_WARNING;
-                break;
+            ll = GF_LOG_WARNING;
+            break;
 
         case NFS3ERR_ROFS:
-		ll = GF_LOG_WARNING;
-                break;
+            ll = GF_LOG_WARNING;
+            break;
 
         case NFS3ERR_FBIG:
-		ll = GF_LOG_WARNING;
-                break;
+            ll = GF_LOG_WARNING;
+            break;
 
         case NFS3ERR_MLINK:
-		ll = GF_LOG_WARNING;
-                break;
+            ll = GF_LOG_WARNING;
+            break;
 
         case NFS3ERR_NAMETOOLONG:
-		ll = GF_LOG_WARNING;
-                break;
+            ll = GF_LOG_WARNING;
+            break;
 
         case NFS3ERR_NOTEMPTY:
-		ll = GF_LOG_WARNING;
-                break;
+            ll = GF_LOG_WARNING;
+            break;
 
         case NFS3ERR_SERVERFAULT:
-		ll = GF_LOG_WARNING;
-                break;
+            ll = GF_LOG_WARNING;
+            break;
 
         case NFS3ERR_NOTSUPP:
-		ll = GF_LOG_WARNING;
-                break;
+            ll = GF_LOG_WARNING;
+            break;
 
         case NFS3ERR_BADHANDLE:
-		ll = GF_LOG_WARNING;
-                break;
+            ll = GF_LOG_WARNING;
+            break;
 
         case NFS3ERR_STALE:
-		ll = GF_LOG_WARNING;
-                break;
+            ll = GF_LOG_WARNING;
+            break;
 
         case NFS3ERR_DQUOT:
-		ll = GF_LOG_WARNING;
-                break;
+            ll = GF_LOG_WARNING;
+            break;
 
         default:
-		ll = GF_LOG_DEBUG;
-                break;
-        }
+            ll = GF_LOG_DEBUG;
+            break;
+    }
 
-        return ll;
+    return ll;
 }
 
-
 int
-nfs3_access_loglevel (nfsstat3 stat) {
-
-	int ll = GF_LOG_DEBUG;
-
-	switch (stat) {
+nfs3_access_loglevel(nfsstat3 stat)
+{
+    int ll = GF_LOG_DEBUG;
 
+    switch (stat) {
         case NFS3ERR_NOENT:
-		ll = GF_LOG_WARNING;
-                break;
+            ll = GF_LOG_WARNING;
+            break;
 
         case NFS3ERR_EXIST:
-		ll = GF_LOG_WARNING;
-                break;
+            ll = GF_LOG_WARNING;
+            break;
 
         case NFS3ERR_XDEV:
-		ll = GF_LOG_WARNING;
-                break;
+            ll = GF_LOG_WARNING;
+            break;
 
         case NFS3ERR_NODEV:
-		ll = GF_LOG_WARNING;
-                break;
+            ll = GF_LOG_WARNING;
+            break;
 
         case NFS3ERR_IO:
-		ll = GF_LOG_WARNING;
-                break;
+            ll = GF_LOG_WARNING;
+            break;
 
         case NFS3ERR_NXIO:
-		ll = GF_LOG_WARNING;
-                break;
+            ll = GF_LOG_WARNING;
+            break;
 
         case NFS3ERR_NOTDIR:
-		ll = GF_LOG_WARNING;
-                break;
+            ll = GF_LOG_WARNING;
+            break;
 
         case NFS3ERR_ISDIR:
-		ll = GF_LOG_WARNING;
-                break;
+            ll = GF_LOG_WARNING;
+            break;
 
         case NFS3ERR_INVAL:
-		ll = GF_LOG_WARNING;
-                break;
+            ll = GF_LOG_WARNING;
+            break;
 
         case NFS3ERR_NOSPC:
-		ll = GF_LOG_WARNING;
-                break;
+            ll = GF_LOG_WARNING;
+            break;
 
         case NFS3ERR_ROFS:
-		ll = GF_LOG_WARNING;
-                break;
+            ll = GF_LOG_WARNING;
+            break;
 
         case NFS3ERR_FBIG:
-		ll = GF_LOG_WARNING;
-                break;
+            ll = GF_LOG_WARNING;
+            break;
 
         case NFS3ERR_MLINK:
-		ll = GF_LOG_WARNING;
-                break;
+            ll = GF_LOG_WARNING;
+            break;
 
         case NFS3ERR_NAMETOOLONG:
-		ll = GF_LOG_WARNING;
-                break;
+            ll = GF_LOG_WARNING;
+            break;
 
         case NFS3ERR_NOTEMPTY:
-		ll = GF_LOG_WARNING;
-                break;
+            ll = GF_LOG_WARNING;
+            break;
 
         case NFS3ERR_SERVERFAULT:
-		ll = GF_LOG_WARNING;
-                break;
+            ll = GF_LOG_WARNING;
+            break;
 
         case NFS3ERR_NOTSUPP:
-		ll = GF_LOG_WARNING;
-                break;
+            ll = GF_LOG_WARNING;
+            break;
 
         case NFS3ERR_BADHANDLE:
-		ll = GF_LOG_WARNING;
-                break;
+            ll = GF_LOG_WARNING;
+            break;
 
         case NFS3ERR_STALE:
-		ll = GF_LOG_WARNING;
-                break;
+            ll = GF_LOG_WARNING;
+            break;
 
         case NFS3ERR_DQUOT:
-		ll = GF_LOG_WARNING;
-                break;
+            ll = GF_LOG_WARNING;
+            break;
 
         default:
-		ll = GF_LOG_DEBUG;
-                break;
-        }
+            ll = GF_LOG_DEBUG;
+            break;
+    }
 
-        return ll;
+    return ll;
 }
 
-
 int
-nfs3_readlink_loglevel (nfsstat3 stat) {
-
-	int ll = GF_LOG_DEBUG;
-
-	switch (stat) {
+nfs3_readlink_loglevel(nfsstat3 stat)
+{
+    int ll = GF_LOG_DEBUG;
 
+    switch (stat) {
         case NFS3ERR_EXIST:
-		ll = GF_LOG_WARNING;
-                break;
+            ll = GF_LOG_WARNING;
+            break;
 
         case NFS3ERR_XDEV:
-		ll = GF_LOG_WARNING;
-                break;
+            ll = GF_LOG_WARNING;
+            break;
 
         case NFS3ERR_NODEV:
-		ll = GF_LOG_WARNING;
-                break;
+            ll = GF_LOG_WARNING;
+            break;
 
         case NFS3ERR_IO:
-		ll = GF_LOG_WARNING;
-                break;
+            ll = GF_LOG_WARNING;
+            break;
 
         case NFS3ERR_NXIO:
-		ll = GF_LOG_WARNING;
-                break;
+            ll = GF_LOG_WARNING;
+            break;
 
         case NFS3ERR_NOTDIR:
-		ll = GF_LOG_WARNING;
-                break;
+            ll = GF_LOG_WARNING;
+            break;
 
         case NFS3ERR_ISDIR:
-		ll = GF_LOG_WARNING;
-                break;
+            ll = GF_LOG_WARNING;
+            break;
 
         case NFS3ERR_INVAL:
-		ll = GF_LOG_WARNING;
-                break;
+            ll = GF_LOG_WARNING;
+            break;
 
         case NFS3ERR_NOSPC:
-		ll = GF_LOG_WARNING;
-                break;
+            ll = GF_LOG_WARNING;
+            break;
 
         case NFS3ERR_ROFS:
-		ll = GF_LOG_WARNING;
-                break;
+            ll = GF_LOG_WARNING;
+            break;
 
         case NFS3ERR_FBIG:
-		ll = GF_LOG_WARNING;
-                break;
+            ll = GF_LOG_WARNING;
+            break;
 
         case NFS3ERR_MLINK:
-		ll = GF_LOG_WARNING;
-                break;
+            ll = GF_LOG_WARNING;
+            break;
 
         case NFS3ERR_NOTEMPTY:
-		ll = GF_LOG_WARNING;
-                break;
+            ll = GF_LOG_WARNING;
+            break;
 
         case NFS3ERR_SERVERFAULT:
-		ll = GF_LOG_WARNING;
-                break;
+            ll = GF_LOG_WARNING;
+            break;
 
         case NFS3ERR_NOTSUPP:
-		ll = GF_LOG_WARNING;
-                break;
+            ll = GF_LOG_WARNING;
+            break;
 
         case NFS3ERR_BADHANDLE:
-		ll = GF_LOG_WARNING;
-                break;
+            ll = GF_LOG_WARNING;
+            break;
 
         case NFS3ERR_STALE:
-		ll = GF_LOG_WARNING;
-                break;
+            ll = GF_LOG_WARNING;
+            break;
 
         case NFS3ERR_DQUOT:
-		ll = GF_LOG_WARNING;
-                break;
+            ll = GF_LOG_WARNING;
+            break;
 
         default:
-		ll = GF_LOG_DEBUG;
-                break;
-        }
+            ll = GF_LOG_DEBUG;
+            break;
+    }
 
-        return ll;
+    return ll;
 }
 
 int
-nfs3_read_loglevel (nfsstat3 stat) {
-
-	int ll = GF_LOG_DEBUG;
-
-	switch (stat) {
+nfs3_read_loglevel(nfsstat3 stat)
+{
+    int ll = GF_LOG_DEBUG;
 
+    switch (stat) {
         case NFS3ERR_NOENT:
-		ll = GF_LOG_WARNING;
-                break;
+            ll = GF_LOG_WARNING;
+            break;
 
         case NFS3ERR_EXIST:
-		ll = GF_LOG_WARNING;
-                break;
+            ll = GF_LOG_WARNING;
+            break;
 
         case NFS3ERR_XDEV:
-		ll = GF_LOG_WARNING;
-                break;
+            ll = GF_LOG_WARNING;
+            break;
 
         case NFS3ERR_NODEV:
-		ll = GF_LOG_WARNING;
-                break;
+            ll = GF_LOG_WARNING;
+            break;
 
         case NFS3ERR_IO:
-		ll = GF_LOG_WARNING;
-                break;
+            ll = GF_LOG_WARNING;
+            break;
 
         case NFS3ERR_NXIO:
-		ll = GF_LOG_WARNING;
-                break;
+            ll = GF_LOG_WARNING;
+            break;
 
         case NFS3ERR_NOTDIR:
-		ll = GF_LOG_WARNING;
-                break;
+            ll = GF_LOG_WARNING;
+            break;
 
         case NFS3ERR_ISDIR:
-		ll = GF_LOG_WARNING;
-                break;
+            ll = GF_LOG_WARNING;
+            break;
 
         case NFS3ERR_INVAL:
-		ll = GF_LOG_WARNING;
-                break;
+            ll = GF_LOG_WARNING;
+            break;
 
         case NFS3ERR_NOSPC:
-		ll = GF_LOG_WARNING;
-                break;
+            ll = GF_LOG_WARNING;
+            break;
 
         case NFS3ERR_ROFS:
-		ll = GF_LOG_WARNING;
-                break;
+            ll = GF_LOG_WARNING;
+            break;
 
         case NFS3ERR_FBIG:
-		ll = GF_LOG_WARNING;
-                break;
+            ll = GF_LOG_WARNING;
+            break;
 
         case NFS3ERR_MLINK:
-		ll = GF_LOG_WARNING;
-                break;
+            ll = GF_LOG_WARNING;
+            break;
 
         case NFS3ERR_NAMETOOLONG:
-		ll = GF_LOG_WARNING;
-                break;
+            ll = GF_LOG_WARNING;
+            break;
 
         case NFS3ERR_NOTEMPTY:
-		ll = GF_LOG_WARNING;
-                break;
+            ll = GF_LOG_WARNING;
+            break;
 
         case NFS3ERR_SERVERFAULT:
-		ll = GF_LOG_WARNING;
-                break;
+            ll = GF_LOG_WARNING;
+            break;
 
         case NFS3ERR_NOTSUPP:
-		ll = GF_LOG_WARNING;
-                break;
+            ll = GF_LOG_WARNING;
+            break;
 
         case NFS3ERR_BADHANDLE:
-		ll = GF_LOG_WARNING;
-                break;
+            ll = GF_LOG_WARNING;
+            break;
 
         case NFS3ERR_STALE:
-		ll = GF_LOG_WARNING;
-                break;
+            ll = GF_LOG_WARNING;
+            break;
 
         case NFS3ERR_DQUOT:
-		ll = GF_LOG_WARNING;
-                break;
+            ll = GF_LOG_WARNING;
+            break;
 
         default:
-		ll = GF_LOG_DEBUG;
-                break;
-        }
+            ll = GF_LOG_DEBUG;
+            break;
+    }
 
-        return ll;
+    return ll;
 }
 
-
 int
-nfs3_write_loglevel (nfsstat3 stat) {
-
-	int ll = GF_LOG_DEBUG;
-
-	switch (stat) {
+nfs3_write_loglevel(nfsstat3 stat)
+{
+    int ll = GF_LOG_DEBUG;
 
+    switch (stat) {
         case NFS3ERR_NOENT:
-		ll = GF_LOG_WARNING;
-                break;
+            ll = GF_LOG_WARNING;
+            break;
 
         case NFS3ERR_EXIST:
-		ll = GF_LOG_WARNING;
-                break;
+            ll = GF_LOG_WARNING;
+            break;
 
         case NFS3ERR_XDEV:
-		ll = GF_LOG_WARNING;
-                break;
+            ll = GF_LOG_WARNING;
+            break;
 
         case NFS3ERR_NODEV:
-		ll = GF_LOG_WARNING;
-                break;
+            ll = GF_LOG_WARNING;
+            break;
 
         case NFS3ERR_IO:
-		ll = GF_LOG_WARNING;
-                break;
+            ll = GF_LOG_WARNING;
+            break;
 
         case NFS3ERR_NXIO:
-		ll = GF_LOG_WARNING;
-                break;
+            ll = GF_LOG_WARNING;
+            break;
 
         case NFS3ERR_NOTDIR:
-		ll = GF_LOG_WARNING;
-                break;
+            ll = GF_LOG_WARNING;
+            break;
 
         case NFS3ERR_ISDIR:
-		ll = GF_LOG_WARNING;
-                break;
+            ll = GF_LOG_WARNING;
+            break;
 
         case NFS3ERR_INVAL:
-		ll = GF_LOG_WARNING;
-                break;
+            ll = GF_LOG_WARNING;
+            break;
 
         case NFS3ERR_NOSPC:
-		ll = GF_LOG_WARNING;
-                break;
+            ll = GF_LOG_WARNING;
+            break;
 
         case NFS3ERR_ROFS:
-		ll = GF_LOG_WARNING;
-                break;
+            ll = GF_LOG_WARNING;
+            break;
 
         case NFS3ERR_FBIG:
-		ll = GF_LOG_WARNING;
-                break;
+            ll = GF_LOG_WARNING;
+            break;
 
         case NFS3ERR_MLINK:
-		ll = GF_LOG_WARNING;
-                break;
+            ll = GF_LOG_WARNING;
+            break;
 
         case NFS3ERR_NAMETOOLONG:
-		ll = GF_LOG_WARNING;
-                break;
+            ll = GF_LOG_WARNING;
+            break;
 
         case NFS3ERR_NOTEMPTY:
-		ll = GF_LOG_WARNING;
-                break;
+            ll = GF_LOG_WARNING;
+            break;
 
         case NFS3ERR_SERVERFAULT:
-		ll = GF_LOG_WARNING;
-                break;
+            ll = GF_LOG_WARNING;
+            break;
 
         case NFS3ERR_NOTSUPP:
-		ll = GF_LOG_WARNING;
-                break;
+            ll = GF_LOG_WARNING;
+            break;
 
         case NFS3ERR_BADHANDLE:
-		ll = GF_LOG_WARNING;
-                break;
+            ll = GF_LOG_WARNING;
+            break;
 
         case NFS3ERR_STALE:
-		ll = GF_LOG_WARNING;
-                break;
+            ll = GF_LOG_WARNING;
+            break;
 
         case NFS3ERR_DQUOT:
-		ll = GF_LOG_WARNING;
-                break;
+            ll = GF_LOG_WARNING;
+            break;
 
         default:
-		ll = GF_LOG_DEBUG;
-                break;
-        }
+            ll = GF_LOG_DEBUG;
+            break;
+    }
 
-        return ll;
+    return ll;
 }
 
-
 int
-nfs3_create_loglevel (nfsstat3 stat) {
-
-	int ll = GF_LOG_DEBUG;
-
-	switch (stat) {
+nfs3_create_loglevel(nfsstat3 stat)
+{
+    int ll = GF_LOG_DEBUG;
 
+    switch (stat) {
         case NFS3ERR_NOENT:
-		ll = GF_LOG_WARNING;
-                break;
+            ll = GF_LOG_WARNING;
+            break;
 
         case NFS3ERR_EXIST:
-		ll = GF_LOG_WARNING;
-                break;
+            ll = GF_LOG_WARNING;
+            break;
 
         case NFS3ERR_XDEV:
-		ll = GF_LOG_WARNING;
-                break;
+            ll = GF_LOG_WARNING;
+            break;
 
         case NFS3ERR_NODEV:
-		ll = GF_LOG_WARNING;
-                break;
+            ll = GF_LOG_WARNING;
+            break;
 
         case NFS3ERR_IO:
-		ll = GF_LOG_WARNING;
-                break;
+            ll = GF_LOG_WARNING;
+            break;
 
         case NFS3ERR_NXIO:
-		ll = GF_LOG_WARNING;
-                break;
+            ll = GF_LOG_WARNING;
+            break;
 
         case NFS3ERR_NOTDIR:
-		ll = GF_LOG_WARNING;
-                break;
+            ll = GF_LOG_WARNING;
+            break;
 
         case NFS3ERR_ISDIR:
-		ll = GF_LOG_WARNING;
-                break;
+            ll = GF_LOG_WARNING;
+            break;
 
         case NFS3ERR_INVAL:
-		ll = GF_LOG_WARNING;
-                break;
+            ll = GF_LOG_WARNING;
+            break;
 
         case NFS3ERR_FBIG:
-		ll = GF_LOG_WARNING;
-                break;
+            ll = GF_LOG_WARNING;
+            break;
 
         case NFS3ERR_MLINK:
-		ll = GF_LOG_WARNING;
-                break;
+            ll = GF_LOG_WARNING;
+            break;
 
         case NFS3ERR_NOTEMPTY:
-		ll = GF_LOG_WARNING;
-                break;
+            ll = GF_LOG_WARNING;
+            break;
 
         case NFS3ERR_SERVERFAULT:
-		ll = GF_LOG_WARNING;
-                break;
+            ll = GF_LOG_WARNING;
+            break;
 
         case NFS3ERR_NOTSUPP:
-		ll = GF_LOG_WARNING;
-                break;
+            ll = GF_LOG_WARNING;
+            break;
 
         case NFS3ERR_BADHANDLE:
-		ll = GF_LOG_WARNING;
-                break;
+            ll = GF_LOG_WARNING;
+            break;
 
         case NFS3ERR_STALE:
-		ll = GF_LOG_WARNING;
-                break;
+            ll = GF_LOG_WARNING;
+            break;
 
         default:
-		ll = GF_LOG_DEBUG;
-                break;
-        }
+            ll = GF_LOG_DEBUG;
+            break;
+    }
 
-        return ll;
+    return ll;
 }
 
-
 int
-nfs3_mkdir_loglevel (nfsstat3 stat) {
-
-	int ll = GF_LOG_DEBUG;
-
-	switch (stat) {
+nfs3_mkdir_loglevel(nfsstat3 stat)
+{
+    int ll = GF_LOG_DEBUG;
 
+    switch (stat) {
         case NFS3ERR_NOENT:
-		ll = GF_LOG_WARNING;
-                break;
+            ll = GF_LOG_WARNING;
+            break;
 
         case NFS3ERR_XDEV:
-		ll = GF_LOG_WARNING;
-                break;
+            ll = GF_LOG_WARNING;
+            break;
 
         case NFS3ERR_NODEV:
-		ll = GF_LOG_WARNING;
-                break;
+            ll = GF_LOG_WARNING;
+            break;
 
         case NFS3ERR_IO:
-		ll = GF_LOG_WARNING;
-                break;
+            ll = GF_LOG_WARNING;
+            break;
 
         case NFS3ERR_NXIO:
-		ll = GF_LOG_WARNING;
-                break;
+            ll = GF_LOG_WARNING;
+            break;
 
         case NFS3ERR_NOTDIR:
-		ll = GF_LOG_WARNING;
-                break;
+            ll = GF_LOG_WARNING;
+            break;
 
         case NFS3ERR_ISDIR:
-		ll = GF_LOG_WARNING;
-                break;
+            ll = GF_LOG_WARNING;
+            break;
 
         case NFS3ERR_INVAL:
-		ll = GF_LOG_WARNING;
-                break;
+            ll = GF_LOG_WARNING;
+            break;
 
         case NFS3ERR_FBIG:
-		ll = GF_LOG_WARNING;
-                break;
+            ll = GF_LOG_WARNING;
+            break;
 
         case NFS3ERR_MLINK:
-		ll = GF_LOG_WARNING;
-                break;
+            ll = GF_LOG_WARNING;
+            break;
 
         case NFS3ERR_NOTEMPTY:
-		ll = GF_LOG_WARNING;
-                break;
+            ll = GF_LOG_WARNING;
+            break;
 
         case NFS3ERR_SERVERFAULT:
-		ll = GF_LOG_WARNING;
-                break;
+            ll = GF_LOG_WARNING;
+            break;
 
         case NFS3ERR_NOTSUPP:
-		ll = GF_LOG_WARNING;
-                break;
+            ll = GF_LOG_WARNING;
+            break;
 
         case NFS3ERR_BADHANDLE:
-		ll = GF_LOG_WARNING;
-                break;
+            ll = GF_LOG_WARNING;
+            break;
 
         case NFS3ERR_STALE:
-		ll = GF_LOG_WARNING;
-                break;
+            ll = GF_LOG_WARNING;
+            break;
 
         case NFS3ERR_DQUOT:
-		ll = GF_LOG_WARNING;
-                break;
+            ll = GF_LOG_WARNING;
+            break;
 
         default:
-		ll = GF_LOG_DEBUG;
-                break;
-        }
+            ll = GF_LOG_DEBUG;
+            break;
+    }
 
-        return ll;
+    return ll;
 }
 
-
 int
-nfs3_symlink_loglevel (nfsstat3 stat) {
-
-	int ll = GF_LOG_DEBUG;
-
-	switch (stat) {
+nfs3_symlink_loglevel(nfsstat3 stat)
+{
+    int ll = GF_LOG_DEBUG;
 
+    switch (stat) {
         case NFS3ERR_XDEV:
-		ll = GF_LOG_WARNING;
-                break;
+            ll = GF_LOG_WARNING;
+            break;
 
         case NFS3ERR_NODEV:
-		ll = GF_LOG_WARNING;
-                break;
+            ll = GF_LOG_WARNING;
+            break;
 
         case NFS3ERR_IO:
-		ll = GF_LOG_WARNING;
-                break;
+            ll = GF_LOG_WARNING;
+            break;
 
         case NFS3ERR_NXIO:
-		ll = GF_LOG_WARNING;
-                break;
+            ll = GF_LOG_WARNING;
+            break;
 
         case NFS3ERR_NOTDIR:
-		ll = GF_LOG_WARNING;
-                break;
+            ll = GF_LOG_WARNING;
+            break;
 
         case NFS3ERR_ISDIR:
-		ll = GF_LOG_WARNING;
-                break;
+            ll = GF_LOG_WARNING;
+            break;
 
         case NFS3ERR_INVAL:
-		ll = GF_LOG_WARNING;
-                break;
+            ll = GF_LOG_WARNING;
+            break;
 
         case NFS3ERR_FBIG:
-		ll = GF_LOG_WARNING;
-                break;
+            ll = GF_LOG_WARNING;
+            break;
 
         case NFS3ERR_MLINK:
-		ll = GF_LOG_WARNING;
-                break;
+            ll = GF_LOG_WARNING;
+            break;
 
         case NFS3ERR_NOTEMPTY:
-		ll = GF_LOG_WARNING;
-                break;
+            ll = GF_LOG_WARNING;
+            break;
 
         case NFS3ERR_SERVERFAULT:
-		ll = GF_LOG_WARNING;
-                break;
+            ll = GF_LOG_WARNING;
+            break;
 
         case NFS3ERR_NOTSUPP:
-		ll = GF_LOG_WARNING;
-                break;
+            ll = GF_LOG_WARNING;
+            break;
 
         case NFS3ERR_BADHANDLE:
-		ll = GF_LOG_WARNING;
-                break;
+            ll = GF_LOG_WARNING;
+            break;
 
         case NFS3ERR_STALE:
-		ll = GF_LOG_WARNING;
-                break;
+            ll = GF_LOG_WARNING;
+            break;
 
         case NFS3ERR_DQUOT:
-		ll = GF_LOG_WARNING;
-                break;
+            ll = GF_LOG_WARNING;
+            break;
 
         default:
-		ll = GF_LOG_DEBUG;
-                break;
-        }
+            ll = GF_LOG_DEBUG;
+            break;
+    }
 
-        return ll;
+    return ll;
 }
 
-
 int
-nfs3_mknod_loglevel (nfsstat3 stat) {
-
-	int ll = GF_LOG_DEBUG;
-
-	switch (stat) {
+nfs3_mknod_loglevel(nfsstat3 stat)
+{
+    int ll = GF_LOG_DEBUG;
 
+    switch (stat) {
         case NFS3ERR_NOENT:
-		ll = GF_LOG_WARNING;
-                break;
+            ll = GF_LOG_WARNING;
+            break;
 
         case NFS3ERR_XDEV:
-		ll = GF_LOG_WARNING;
-                break;
+            ll = GF_LOG_WARNING;
+            break;
 
         case NFS3ERR_NODEV:
-		ll = GF_LOG_WARNING;
-                break;
+            ll = GF_LOG_WARNING;
+            break;
 
         case NFS3ERR_IO:
-		ll = GF_LOG_WARNING;
-                break;
+            ll = GF_LOG_WARNING;
+            break;
 
         case NFS3ERR_NXIO:
-		ll = GF_LOG_WARNING;
-                break;
+            ll = GF_LOG_WARNING;
+            break;
 
         case NFS3ERR_NOTDIR:
-		ll = GF_LOG_WARNING;
-                break;
+            ll = GF_LOG_WARNING;
+            break;
 
         case NFS3ERR_ISDIR:
-		ll = GF_LOG_WARNING;
-                break;
+            ll = GF_LOG_WARNING;
+            break;
 
         case NFS3ERR_INVAL:
-		ll = GF_LOG_WARNING;
-                break;
+            ll = GF_LOG_WARNING;
+            break;
 
         case NFS3ERR_FBIG:
-		ll = GF_LOG_WARNING;
-                break;
+            ll = GF_LOG_WARNING;
+            break;
 
         case NFS3ERR_MLINK:
-		ll = GF_LOG_WARNING;
-                break;
+            ll = GF_LOG_WARNING;
+            break;
 
         case NFS3ERR_NOTEMPTY:
-		ll = GF_LOG_WARNING;
-                break;
+            ll = GF_LOG_WARNING;
+            break;
 
         case NFS3ERR_SERVERFAULT:
-		ll = GF_LOG_WARNING;
-                break;
+            ll = GF_LOG_WARNING;
+            break;
 
         case NFS3ERR_NOTSUPP:
-		ll = GF_LOG_WARNING;
-                break;
+            ll = GF_LOG_WARNING;
+            break;
 
         case NFS3ERR_BADHANDLE:
-		ll = GF_LOG_WARNING;
-                break;
+            ll = GF_LOG_WARNING;
+            break;
 
         case NFS3ERR_STALE:
-		ll = GF_LOG_WARNING;
-                break;
+            ll = GF_LOG_WARNING;
+            break;
 
         default:
-		ll = GF_LOG_DEBUG;
-                break;
-        }
+            ll = GF_LOG_DEBUG;
+            break;
+    }
 
-        return ll;
+    return ll;
 }
 
 int
-nfs3_remove_loglevel (nfsstat3 stat) {
-
-	int ll = GF_LOG_DEBUG;
-
-	switch (stat) {
+nfs3_remove_loglevel(nfsstat3 stat)
+{
+    int ll = GF_LOG_DEBUG;
 
+    switch (stat) {
         case NFS3ERR_EXIST:
-		ll = GF_LOG_WARNING;
-                break;
+            ll = GF_LOG_WARNING;
+            break;
 
         case NFS3ERR_XDEV:
-		ll = GF_LOG_WARNING;
-                break;
+            ll = GF_LOG_WARNING;
+            break;
 
         case NFS3ERR_NODEV:
-		ll = GF_LOG_WARNING;
-                break;
+            ll = GF_LOG_WARNING;
+            break;
 
         case NFS3ERR_IO:
-		ll = GF_LOG_WARNING;
-                break;
+            ll = GF_LOG_WARNING;
+            break;
 
         case NFS3ERR_NXIO:
-		ll = GF_LOG_WARNING;
-                break;
+            ll = GF_LOG_WARNING;
+            break;
 
         case NFS3ERR_NOTDIR:
-		ll = GF_LOG_WARNING;
-                break;
+            ll = GF_LOG_WARNING;
+            break;
 
         case NFS3ERR_INVAL:
-		ll = GF_LOG_WARNING;
-                break;
+            ll = GF_LOG_WARNING;
+            break;
 
         case NFS3ERR_NOSPC:
-		ll = GF_LOG_WARNING;
-                break;
+            ll = GF_LOG_WARNING;
+            break;
 
         case NFS3ERR_FBIG:
-		ll = GF_LOG_WARNING;
-                break;
+            ll = GF_LOG_WARNING;
+            break;
 
         case NFS3ERR_MLINK:
-		ll = GF_LOG_WARNING;
-                break;
+            ll = GF_LOG_WARNING;
+            break;
 
         case NFS3ERR_SERVERFAULT:
-		ll = GF_LOG_WARNING;
-                break;
+            ll = GF_LOG_WARNING;
+            break;
 
         case NFS3ERR_NOTSUPP:
-		ll = GF_LOG_WARNING;
-                break;
+            ll = GF_LOG_WARNING;
+            break;
 
         case NFS3ERR_BADHANDLE:
-		ll = GF_LOG_WARNING;
-                break;
+            ll = GF_LOG_WARNING;
+            break;
 
         case NFS3ERR_STALE:
-		ll = GF_LOG_WARNING;
-                break;
+            ll = GF_LOG_WARNING;
+            break;
 
         case NFS3ERR_DQUOT:
-		ll = GF_LOG_WARNING;
-                break;
+            ll = GF_LOG_WARNING;
+            break;
 
         default:
-		ll = GF_LOG_DEBUG;
-                break;
-        }
+            ll = GF_LOG_DEBUG;
+            break;
+    }
 
-        return ll;
+    return ll;
 }
 
-
 int
-nfs3_rmdir_loglevel (nfsstat3 stat) {
-
-	int ll = GF_LOG_DEBUG;
-
-	switch (stat) {
+nfs3_rmdir_loglevel(nfsstat3 stat)
+{
+    int ll = GF_LOG_DEBUG;
 
+    switch (stat) {
         case NFS3ERR_EXIST:
-		ll = GF_LOG_WARNING;
-                break;
+            ll = GF_LOG_WARNING;
+            break;
 
         case NFS3ERR_XDEV:
-		ll = GF_LOG_WARNING;
-                break;
+            ll = GF_LOG_WARNING;
+            break;
 
         case NFS3ERR_NODEV:
-		ll = GF_LOG_WARNING;
-                break;
+            ll = GF_LOG_WARNING;
+            break;
 
         case NFS3ERR_IO:
-		ll = GF_LOG_WARNING;
-                break;
+            ll = GF_LOG_WARNING;
+            break;
 
         case NFS3ERR_NXIO:
-		ll = GF_LOG_WARNING;
-                break;
+            ll = GF_LOG_WARNING;
+            break;
 
         case NFS3ERR_NOTDIR:
-		ll = GF_LOG_WARNING;
-                break;
+            ll = GF_LOG_WARNING;
+            break;
 
         case NFS3ERR_INVAL:
-		ll = GF_LOG_WARNING;
-                break;
+            ll = GF_LOG_WARNING;
+            break;
 
         case NFS3ERR_NOSPC:
-		ll = GF_LOG_WARNING;
-                break;
+            ll = GF_LOG_WARNING;
+            break;
 
         case NFS3ERR_FBIG:
-		ll = GF_LOG_WARNING;
-                break;
+            ll = GF_LOG_WARNING;
+            break;
 
         case NFS3ERR_MLINK:
-		ll = GF_LOG_WARNING;
-                break;
+            ll = GF_LOG_WARNING;
+            break;
 
         case NFS3ERR_SERVERFAULT:
-		ll = GF_LOG_WARNING;
-                break;
+            ll = GF_LOG_WARNING;
+            break;
 
         case NFS3ERR_NOTSUPP:
-		ll = GF_LOG_WARNING;
-                break;
+            ll = GF_LOG_WARNING;
+            break;
 
         case NFS3ERR_BADHANDLE:
-		ll = GF_LOG_WARNING;
-                break;
+            ll = GF_LOG_WARNING;
+            break;
 
         case NFS3ERR_STALE:
-		ll = GF_LOG_WARNING;
-                break;
+            ll = GF_LOG_WARNING;
+            break;
 
         case NFS3ERR_DQUOT:
-		ll = GF_LOG_WARNING;
-                break;
+            ll = GF_LOG_WARNING;
+            break;
 
         default:
-		ll = GF_LOG_DEBUG;
-                break;
-        }
+            ll = GF_LOG_DEBUG;
+            break;
+    }
 
-        return ll;
+    return ll;
 }
 
-
 int
-nfs3_rename_loglevel (nfsstat3 stat) {
-
-	int ll = GF_LOG_DEBUG;
-
-	switch (stat) {
+nfs3_rename_loglevel(nfsstat3 stat)
+{
+    int ll = GF_LOG_DEBUG;
 
+    switch (stat) {
         case NFS3ERR_XDEV:
-		ll = GF_LOG_WARNING;
-                break;
+            ll = GF_LOG_WARNING;
+            break;
 
         case NFS3ERR_NODEV:
-		ll = GF_LOG_WARNING;
-                break;
+            ll = GF_LOG_WARNING;
+            break;
 
         case NFS3ERR_IO:
-		ll = GF_LOG_WARNING;
-                break;
+            ll = GF_LOG_WARNING;
+            break;
 
         case NFS3ERR_NXIO:
-		ll = GF_LOG_WARNING;
-                break;
+            ll = GF_LOG_WARNING;
+            break;
 
         case NFS3ERR_NOTDIR:
-		ll = GF_LOG_WARNING;
-                break;
+            ll = GF_LOG_WARNING;
+            break;
 
         case NFS3ERR_ISDIR:
-		ll = GF_LOG_WARNING;
-                break;
+            ll = GF_LOG_WARNING;
+            break;
 
         case NFS3ERR_INVAL:
-		ll = GF_LOG_WARNING;
-                break;
+            ll = GF_LOG_WARNING;
+            break;
 
         case NFS3ERR_NOSPC:
-		ll = GF_LOG_WARNING;
-                break;
+            ll = GF_LOG_WARNING;
+            break;
 
         case NFS3ERR_FBIG:
-		ll = GF_LOG_WARNING;
-                break;
+            ll = GF_LOG_WARNING;
+            break;
 
         case NFS3ERR_MLINK:
-		ll = GF_LOG_WARNING;
-                break;
+            ll = GF_LOG_WARNING;
+            break;
 
         case NFS3ERR_NOTEMPTY:
-		ll = GF_LOG_WARNING;
-                break;
+            ll = GF_LOG_WARNING;
+            break;
 
         case NFS3ERR_SERVERFAULT:
-		ll = GF_LOG_WARNING;
-                break;
+            ll = GF_LOG_WARNING;
+            break;
 
         case NFS3ERR_NOTSUPP:
-		ll = GF_LOG_WARNING;
-                break;
+            ll = GF_LOG_WARNING;
+            break;
 
         case NFS3ERR_BADHANDLE:
-		ll = GF_LOG_WARNING;
-                break;
+            ll = GF_LOG_WARNING;
+            break;
 
         case NFS3ERR_STALE:
-		ll = GF_LOG_WARNING;
-                break;
+            ll = GF_LOG_WARNING;
+            break;
 
         case NFS3ERR_DQUOT:
-		ll = GF_LOG_WARNING;
-                break;
+            ll = GF_LOG_WARNING;
+            break;
 
         default:
-		ll = GF_LOG_DEBUG;
-                break;
-        }
+            ll = GF_LOG_DEBUG;
+            break;
+    }
 
-        return ll;
+    return ll;
 }
 
-
 int
-nfs3_link_loglevel (nfsstat3 stat) {
-
-	int ll = GF_LOG_DEBUG;
-
-	switch (stat) {
+nfs3_link_loglevel(nfsstat3 stat)
+{
+    int ll = GF_LOG_DEBUG;
 
+    switch (stat) {
         case NFS3ERR_XDEV:
-		ll = GF_LOG_WARNING;
-                break;
+            ll = GF_LOG_WARNING;
+            break;
 
         case NFS3ERR_NODEV:
-		ll = GF_LOG_WARNING;
-                break;
+            ll = GF_LOG_WARNING;
+            break;
 
         case NFS3ERR_IO:
-		ll = GF_LOG_WARNING;
-                break;
+            ll = GF_LOG_WARNING;
+            break;
 
         case NFS3ERR_NXIO:
-		ll = GF_LOG_WARNING;
-                break;
+            ll = GF_LOG_WARNING;
+            break;
 
         case NFS3ERR_INVAL:
-		ll = GF_LOG_WARNING;
-                break;
+            ll = GF_LOG_WARNING;
+            break;
 
         case NFS3ERR_FBIG:
-		ll = GF_LOG_WARNING;
-                break;
+            ll = GF_LOG_WARNING;
+            break;
 
         case NFS3ERR_MLINK:
-		ll = GF_LOG_WARNING;
-                break;
+            ll = GF_LOG_WARNING;
+            break;
 
         case NFS3ERR_NOTEMPTY:
-		ll = GF_LOG_WARNING;
-                break;
+            ll = GF_LOG_WARNING;
+            break;
 
         case NFS3ERR_SERVERFAULT:
-		ll = GF_LOG_WARNING;
-                break;
+            ll = GF_LOG_WARNING;
+            break;
 
         case NFS3ERR_NOTSUPP:
-		ll = GF_LOG_WARNING;
-                break;
+            ll = GF_LOG_WARNING;
+            break;
 
         case NFS3ERR_BADHANDLE:
-		ll = GF_LOG_WARNING;
-                break;
+            ll = GF_LOG_WARNING;
+            break;
 
         case NFS3ERR_STALE:
-		ll = GF_LOG_WARNING;
-                break;
+            ll = GF_LOG_WARNING;
+            break;
 
         case NFS3ERR_DQUOT:
-		ll = GF_LOG_WARNING;
-                break;
+            ll = GF_LOG_WARNING;
+            break;
 
         default:
-		ll = GF_LOG_DEBUG;
-                break;
-        }
+            ll = GF_LOG_DEBUG;
+            break;
+    }
 
-        return ll;
+    return ll;
 }
 
-
 int
-nfs3_readdir_loglevel (nfsstat3 stat) {
-
-	int ll = GF_LOG_DEBUG;
-
-	switch (stat) {
+nfs3_readdir_loglevel(nfsstat3 stat)
+{
+    int ll = GF_LOG_DEBUG;
 
+    switch (stat) {
         case NFS3ERR_NOENT:
-		ll = GF_LOG_WARNING;
-                break;
+            ll = GF_LOG_WARNING;
+            break;
 
         case NFS3ERR_EXIST:
-		ll = GF_LOG_WARNING;
-                break;
+            ll = GF_LOG_WARNING;
+            break;
 
         case NFS3ERR_XDEV:
-		ll = GF_LOG_WARNING;
-                break;
+            ll = GF_LOG_WARNING;
+            break;
 
         case NFS3ERR_NODEV:
-		ll = GF_LOG_WARNING;
-                break;
+            ll = GF_LOG_WARNING;
+            break;
 
         case NFS3ERR_IO:
-		ll = GF_LOG_WARNING;
-                break;
+            ll = GF_LOG_WARNING;
+            break;
 
         case NFS3ERR_NXIO:
-		ll = GF_LOG_WARNING;
-                break;
+            ll = GF_LOG_WARNING;
+            break;
 
         case NFS3ERR_NOTDIR:
-		ll = GF_LOG_WARNING;
-                break;
+            ll = GF_LOG_WARNING;
+            break;
 
         case NFS3ERR_ISDIR:
-		ll = GF_LOG_WARNING;
-                break;
+            ll = GF_LOG_WARNING;
+            break;
 
         case NFS3ERR_INVAL:
-		ll = GF_LOG_WARNING;
-                break;
+            ll = GF_LOG_WARNING;
+            break;
 
         case NFS3ERR_NOSPC:
-		ll = GF_LOG_WARNING;
-                break;
+            ll = GF_LOG_WARNING;
+            break;
 
         case NFS3ERR_ROFS:
-		ll = GF_LOG_WARNING;
-                break;
+            ll = GF_LOG_WARNING;
+            break;
 
         case NFS3ERR_FBIG:
-		ll = GF_LOG_WARNING;
-                break;
+            ll = GF_LOG_WARNING;
+            break;
 
         case NFS3ERR_MLINK:
-		ll = GF_LOG_WARNING;
-                break;
+            ll = GF_LOG_WARNING;
+            break;
 
         case NFS3ERR_NAMETOOLONG:
-		ll = GF_LOG_WARNING;
-                break;
+            ll = GF_LOG_WARNING;
+            break;
 
         case NFS3ERR_NOTEMPTY:
-		ll = GF_LOG_WARNING;
-                break;
+            ll = GF_LOG_WARNING;
+            break;
 
         case NFS3ERR_SERVERFAULT:
-		ll = GF_LOG_WARNING;
-                break;
+            ll = GF_LOG_WARNING;
+            break;
 
         case NFS3ERR_NOTSUPP:
-		ll = GF_LOG_WARNING;
-                break;
+            ll = GF_LOG_WARNING;
+            break;
 
         case NFS3ERR_BADHANDLE:
-		ll = GF_LOG_WARNING;
-                break;
+            ll = GF_LOG_WARNING;
+            break;
 
         case NFS3ERR_STALE:
-		ll = GF_LOG_WARNING;
-                break;
+            ll = GF_LOG_WARNING;
+            break;
 
         case NFS3ERR_DQUOT:
-		ll = GF_LOG_WARNING;
-                break;
+            ll = GF_LOG_WARNING;
+            break;
 
         default:
-		ll = GF_LOG_DEBUG;
-                break;
-        }
+            ll = GF_LOG_DEBUG;
+            break;
+    }
 
-        return ll;
+    return ll;
 }
 
-
 int
-nfs3_fsstat_loglevel (nfsstat3 stat) {
-
-	int ll = GF_LOG_DEBUG;
-
-	switch (stat) {
+nfs3_fsstat_loglevel(nfsstat3 stat)
+{
+    int ll = GF_LOG_DEBUG;
 
+    switch (stat) {
         case NFS3ERR_PERM:
-		ll = GF_LOG_WARNING;
-                break;
+            ll = GF_LOG_WARNING;
+            break;
 
         case NFS3ERR_NOENT:
-		ll = GF_LOG_WARNING;
-                break;
+            ll = GF_LOG_WARNING;
+            break;
 
         case NFS3ERR_ACCES:
-		ll = GF_LOG_WARNING;
-                break;
+            ll = GF_LOG_WARNING;
+            break;
 
         case NFS3ERR_EXIST:
-		ll = GF_LOG_WARNING;
-                break;
+            ll = GF_LOG_WARNING;
+            break;
 
         case NFS3ERR_XDEV:
-		ll = GF_LOG_WARNING;
-                break;
+            ll = GF_LOG_WARNING;
+            break;
 
         case NFS3ERR_NODEV:
-		ll = GF_LOG_WARNING;
-                break;
+            ll = GF_LOG_WARNING;
+            break;
 
         case NFS3ERR_IO:
-		ll = GF_LOG_WARNING;
-                break;
+            ll = GF_LOG_WARNING;
+            break;
 
         case NFS3ERR_NXIO:
-		ll = GF_LOG_WARNING;
-                break;
+            ll = GF_LOG_WARNING;
+            break;
 
         case NFS3ERR_NOTDIR:
-		ll = GF_LOG_WARNING;
-                break;
+            ll = GF_LOG_WARNING;
+            break;
 
         case NFS3ERR_ISDIR:
-		ll = GF_LOG_WARNING;
-                break;
+            ll = GF_LOG_WARNING;
+            break;
 
         case NFS3ERR_INVAL:
-		ll = GF_LOG_WARNING;
-                break;
+            ll = GF_LOG_WARNING;
+            break;
 
         case NFS3ERR_NOSPC:
-		ll = GF_LOG_WARNING;
-                break;
+            ll = GF_LOG_WARNING;
+            break;
 
         case NFS3ERR_ROFS:
-		ll = GF_LOG_WARNING;
-                break;
+            ll = GF_LOG_WARNING;
+            break;
 
         case NFS3ERR_FBIG:
-		ll = GF_LOG_WARNING;
-                break;
+            ll = GF_LOG_WARNING;
+            break;
 
         case NFS3ERR_MLINK:
-		ll = GF_LOG_WARNING;
-                break;
+            ll = GF_LOG_WARNING;
+            break;
 
         case NFS3ERR_NAMETOOLONG:
-		ll = GF_LOG_WARNING;
-                break;
+            ll = GF_LOG_WARNING;
+            break;
 
         case NFS3ERR_NOTEMPTY:
-		ll = GF_LOG_WARNING;
-                break;
+            ll = GF_LOG_WARNING;
+            break;
 
         case NFS3ERR_SERVERFAULT:
-		ll = GF_LOG_WARNING;
-                break;
+            ll = GF_LOG_WARNING;
+            break;
 
         case NFS3ERR_NOTSUPP:
-		ll = GF_LOG_WARNING;
-                break;
+            ll = GF_LOG_WARNING;
+            break;
 
         case NFS3ERR_BADHANDLE:
-		ll = GF_LOG_WARNING;
-                break;
+            ll = GF_LOG_WARNING;
+            break;
 
         case NFS3ERR_STALE:
-		ll = GF_LOG_WARNING;
-                break;
+            ll = GF_LOG_WARNING;
+            break;
 
         case NFS3ERR_DQUOT:
-		ll = GF_LOG_WARNING;
-                break;
+            ll = GF_LOG_WARNING;
+            break;
 
         default:
-		ll = GF_LOG_DEBUG;
-                break;
-        }
+            ll = GF_LOG_DEBUG;
+            break;
+    }
 
-        return ll;
+    return ll;
 }
 
 struct nfs3op_str {
-	int	op;
-	char	str[100];
+    int op;
+    char str[100];
 };
 
 struct nfs3op_str nfs3op_strings[] = {
-	{ NFS3_NULL, "NULL"},
-	{ NFS3_GETATTR, "GETATTR"},
-	{ NFS3_SETATTR, "SETATTR"},
-	{ NFS3_LOOKUP, "LOOKUP"},
-	{ NFS3_ACCESS, "ACCESS"},
-	{ NFS3_READLINK, "READLINK"},
-	{ NFS3_READ, "READ"},
-	{ NFS3_WRITE, "WRITE"},
-	{ NFS3_CREATE, "CREATE"},
-	{ NFS3_MKDIR, "MKDIR"},
-	{ NFS3_SYMLINK, "SYMLINK"},
-	{ NFS3_MKNOD, "MKNOD"},
-	{ NFS3_REMOVE, "REMOVE"},
-	{ NFS3_RMDIR, "RMDIR"},
-	{ NFS3_RENAME, "RENAME"},
-	{ NFS3_LINK, "LINK"},
-	{ NFS3_READDIR, "READDIR"},
-	{ NFS3_READDIRP, "READDIRP"},
-	{ NFS3_FSSTAT, "FSSTAT"},
-	{ NFS3_FSINFO, "FSINFO"},
-	{ NFS3_PATHCONF, "PATHCONF"},
-	{ NFS3_COMMIT, "COMMIT"},
+    {NFS3_NULL, "NULL"},         {NFS3_GETATTR, "GETATTR"},
+    {NFS3_SETATTR, "SETATTR"},   {NFS3_LOOKUP, "LOOKUP"},
+    {NFS3_ACCESS, "ACCESS"},     {NFS3_READLINK, "READLINK"},
+    {NFS3_READ, "READ"},         {NFS3_WRITE, "WRITE"},
+    {NFS3_CREATE, "CREATE"},     {NFS3_MKDIR, "MKDIR"},
+    {NFS3_SYMLINK, "SYMLINK"},   {NFS3_MKNOD, "MKNOD"},
+    {NFS3_REMOVE, "REMOVE"},     {NFS3_RMDIR, "RMDIR"},
+    {NFS3_RENAME, "RENAME"},     {NFS3_LINK, "LINK"},
+    {NFS3_READDIR, "READDIR"},   {NFS3_READDIRP, "READDIRP"},
+    {NFS3_FSSTAT, "FSSTAT"},     {NFS3_FSINFO, "FSINFO"},
+    {NFS3_PATHCONF, "PATHCONF"}, {NFS3_COMMIT, "COMMIT"},
 };
 
 int
-nfs3_loglevel (int nfs_op, nfsstat3 stat) {
-
-	int	ll = GF_LOG_DEBUG;
+nfs3_loglevel(int nfs_op, nfsstat3 stat)
+{
+    int ll = GF_LOG_DEBUG;
 
-	switch (nfs_op) {
-	case NFS3_GETATTR:
-		ll = nfs3_getattr_loglevel (stat);
-		break;
+    switch (nfs_op) {
+        case NFS3_GETATTR:
+            ll = nfs3_getattr_loglevel(stat);
+            break;
 
-	case NFS3_SETATTR:
-		ll = nfs3_setattr_loglevel (stat);
-		break;
+        case NFS3_SETATTR:
+            ll = nfs3_setattr_loglevel(stat);
+            break;
 
-	case NFS3_LOOKUP:
-		ll = nfs3_lookup_loglevel (stat);
-		break;
+        case NFS3_LOOKUP:
+            ll = nfs3_lookup_loglevel(stat);
+            break;
 
-	case NFS3_ACCESS:
-		ll = nfs3_access_loglevel (stat);
-		break;
+        case NFS3_ACCESS:
+            ll = nfs3_access_loglevel(stat);
+            break;
 
-	case NFS3_READLINK:
-		ll = nfs3_readlink_loglevel (stat);
-		break;
+        case NFS3_READLINK:
+            ll = nfs3_readlink_loglevel(stat);
+            break;
 
-	case NFS3_READ:
-		ll = nfs3_read_loglevel (stat);
-		break;
+        case NFS3_READ:
+            ll = nfs3_read_loglevel(stat);
+            break;
 
-	case NFS3_WRITE:
-		ll = nfs3_write_loglevel (stat);
-		break;
+        case NFS3_WRITE:
+            ll = nfs3_write_loglevel(stat);
+            break;
 
-	case NFS3_CREATE:
-		ll = nfs3_create_loglevel (stat);
-		break;
+        case NFS3_CREATE:
+            ll = nfs3_create_loglevel(stat);
+            break;
 
-	case NFS3_MKDIR:
-		ll = nfs3_mkdir_loglevel (stat);
-		break;
+        case NFS3_MKDIR:
+            ll = nfs3_mkdir_loglevel(stat);
+            break;
 
-	case NFS3_SYMLINK:
-		ll = nfs3_symlink_loglevel (stat);
-		break;
+        case NFS3_SYMLINK:
+            ll = nfs3_symlink_loglevel(stat);
+            break;
 
-	case NFS3_MKNOD:
-		ll = nfs3_mknod_loglevel (stat);
-		break;
+        case NFS3_MKNOD:
+            ll = nfs3_mknod_loglevel(stat);
+            break;
 
-	case NFS3_REMOVE:
-		ll = nfs3_remove_loglevel (stat);
-		break;
+        case NFS3_REMOVE:
+            ll = nfs3_remove_loglevel(stat);
+            break;
 
-	case NFS3_RMDIR:
-		ll = nfs3_rmdir_loglevel (stat);
-		break;
+        case NFS3_RMDIR:
+            ll = nfs3_rmdir_loglevel(stat);
+            break;
 
-	case NFS3_RENAME:
-		ll = nfs3_rename_loglevel (stat);
-		break;
+        case NFS3_RENAME:
+            ll = nfs3_rename_loglevel(stat);
+            break;
 
-	case NFS3_LINK:
-		ll = nfs3_link_loglevel (stat);
-		break;
+        case NFS3_LINK:
+            ll = nfs3_link_loglevel(stat);
+            break;
 
-	case NFS3_READDIR:
-		ll = nfs3_readdir_loglevel (stat);
-		break;
+        case NFS3_READDIR:
+            ll = nfs3_readdir_loglevel(stat);
+            break;
 
-	case NFS3_READDIRP:
-		ll = nfs3_readdir_loglevel (stat);
-		break;
+        case NFS3_READDIRP:
+            ll = nfs3_readdir_loglevel(stat);
+            break;
 
-	case NFS3_FSSTAT:
-		ll = nfs3_fsstat_loglevel (stat);
-		break;
+        case NFS3_FSSTAT:
+            ll = nfs3_fsstat_loglevel(stat);
+            break;
 
-	case NFS3_FSINFO:
-		ll = nfs3_fsstat_loglevel (stat);
-		break;
+        case NFS3_FSINFO:
+            ll = nfs3_fsstat_loglevel(stat);
+            break;
 
-	case NFS3_PATHCONF:
-		ll = nfs3_fsstat_loglevel (stat);
-		break;
+        case NFS3_PATHCONF:
+            ll = nfs3_fsstat_loglevel(stat);
+            break;
 
-	case NFS3_COMMIT:
-		ll = nfs3_write_loglevel (stat);
-		break;
+        case NFS3_COMMIT:
+            ll = nfs3_write_loglevel(stat);
+            break;
 
-	default:
-		ll = GF_LOG_DEBUG;
-		break;
-	}
+        default:
+            ll = GF_LOG_DEBUG;
+            break;
+    }
 
-	return ll;
+    return ll;
 }
 
 void
-nfs3_log_common_res (uint32_t xid, int op, nfsstat3 stat, int pstat)
+nfs3_log_common_res(uint32_t xid, int op, nfsstat3 stat, int pstat,
+                    const char *path)
 {
-        char    errstr[1024];
-        int     ll = nfs3_loglevel (op, stat);
+    char errstr[1024];
+    int ll = nfs3_loglevel(op, stat);
 
-        if (THIS->ctx->log.loglevel < ll)
-                return;
-        nfs3_stat_to_errstr (xid, nfs3op_strings[op].str, stat, pstat, errstr, sizeof (errstr));
-        gf_log (GF_NFS3, ll, "%s", errstr);
+    if (THIS->ctx->log.loglevel < ll)
+        return;
+    nfs3_stat_to_errstr(xid, nfs3op_strings[op].str, stat, pstat, errstr,
+                        sizeof(errstr));
+    if (ll == GF_LOG_DEBUG)
+        gf_msg_debug(GF_NFS3, 0, "%s => (%s)", path, errstr);
+    else
+        gf_msg(GF_NFS3, ll, errno, NFS_MSG_STAT_ERROR, "%s => (%s)", path,
+               errstr);
 }
 
 void
-nfs3_log_readlink_res (uint32_t xid, nfsstat3 stat, int pstat, char *linkpath)
+nfs3_log_readlink_res(uint32_t xid, nfsstat3 stat, int pstat, char *linkpath,
+                      const char *path)
 {
-        char    errstr[1024];
-        int     ll = nfs3_loglevel (NFS3_READLINK, stat);
+    char errstr[1024];
+    int ll = nfs3_loglevel(NFS3_READLINK, stat);
 
-        if (THIS->ctx->log.loglevel < ll)
-                return;
-
-        nfs3_stat_to_errstr (xid, "READLINK", stat, pstat, errstr, sizeof (errstr));
-        gf_log (GF_NFS3, ll, "%s, target: %s",
-                errstr, linkpath);
+    if (THIS->ctx->log.loglevel < ll)
+        return;
 
+    nfs3_stat_to_errstr(xid, "READLINK", stat, pstat, errstr, sizeof(errstr));
+    if (ll == GF_LOG_DEBUG)
+        gf_msg_debug(GF_NFS3, 0, "%s => (%s), target: %s", path, errstr,
+                     linkpath);
+    else
+        gf_msg(GF_NFS3, ll, errno, NFS_MSG_STAT_ERROR, "%s => (%s) target: %s",
+               path, errstr, linkpath);
 }
 
 void
-nfs3_log_read_res (uint32_t xid, nfsstat3 stat, int pstat, count3 count,
-                   int is_eof, struct iovec *vec, int32_t veccount)
+nfs3_log_read_res(uint32_t xid, nfsstat3 stat, int pstat, count3 count,
+                  int is_eof, struct iovec *vec, int32_t veccount,
+                  const char *path)
 {
-        char    errstr[1024];
-        int     ll = GF_LOG_DEBUG;
+    char errstr[1024];
+    int ll = GF_LOG_DEBUG;
 
-	ll = nfs3_loglevel (NFS3_READ, stat);
-        if (THIS->ctx->log.loglevel < ll)
-                return;
-        nfs3_stat_to_errstr (xid, "READ", stat, pstat, errstr, sizeof (errstr));
-        if (vec)
-                gf_log (GF_NFS3, ll, "%s, count: %"PRIu32", is_eof:"
-                        " %d, vector: count: %d, len: %zd", errstr, count,
-                        is_eof, veccount, vec->iov_len);
+    ll = nfs3_loglevel(NFS3_READ, stat);
+    if (THIS->ctx->log.loglevel < ll)
+        return;
+    nfs3_stat_to_errstr(xid, "READ", stat, pstat, errstr, sizeof(errstr));
+    if (vec)
+        if (ll == GF_LOG_DEBUG)
+            gf_msg_debug(GF_NFS3, 0,
+                         "%s => (%s), count: %" PRIu32
+                         ", is_eof:"
+                         " %d, vector: count: %d, len: %zd",
+                         path, errstr, count, is_eof, veccount, vec->iov_len);
         else
-                gf_log (GF_NFS3, ll, "%s, count: %"PRIu32", is_eof:"
-                        " %d", errstr, count, is_eof);
-}
-
-
-void
-nfs3_log_write_res (uint32_t xid, nfsstat3 stat, int pstat, count3 count,
-                    int stable, uint64_t wverf)
-{
-        char    errstr[1024];
-        int     ll = nfs3_loglevel (NFS3_WRITE, stat);
-
-        if (THIS->ctx->log.loglevel < ll)
-                 return;
+            gf_msg(GF_NFS3, ll, errno, NFS_MSG_STAT_ERROR,
+                   "%s => (%s), count: %" PRIu32
+                   ", is_eof:"
+                   " %d, vector: count: %d, len: %zd",
+                   path, errstr, count, is_eof, veccount, vec->iov_len);
+    else if (ll == GF_LOG_DEBUG)
+        gf_msg_debug(GF_NFS3, 0,
+                     "%s => (%s), count: %" PRIu32
+                     ", is_eof:"
+                     " %d",
+                     path, errstr, count, is_eof);
+    else
+        gf_msg(GF_NFS3, ll, errno, NFS_MSG_STAT_ERROR,
+               "%s => (%s), count: %" PRIu32
+               ", is_eof:"
+               " %d",
+               path, errstr, count, is_eof);
+}
+
+void
+nfs3_log_write_res(uint32_t xid, nfsstat3 stat, int pstat, count3 count,
+                   int stable, uint64_t wverf, const char *path)
+{
+    char errstr[1024];
+    int ll = nfs3_loglevel(NFS3_WRITE, stat);
+
+    if (THIS->ctx->log.loglevel < ll)
+        return;
 
-        nfs3_stat_to_errstr (xid, "WRITE", stat, pstat, errstr, sizeof (errstr));
-        gf_log (GF_NFS3, ll, "%s, count: %"PRIu32", %s,wverf: %"PRIu64
-                , errstr, count, (stable == UNSTABLE)?"UNSTABLE":"STABLE",
-                wverf);
+    nfs3_stat_to_errstr(xid, "WRITE", stat, pstat, errstr, sizeof(errstr));
+    if (ll == GF_LOG_DEBUG)
+        gf_msg_debug(GF_NFS3, 0,
+                     "%s => (%s), count: %" PRIu32
+                     ", %s,wverf: "
+                     "%" PRIu64,
+                     path, errstr, count,
+                     (stable == UNSTABLE) ? "UNSTABLE" : "STABLE", wverf);
+    else
+        gf_msg(GF_NFS3, ll, errno, NFS_MSG_STAT_ERROR,
+               "%s => (%s), count: %" PRIu32 ", %s,wverf: %" PRIu64, path,
+               errstr, count, (stable == UNSTABLE) ? "UNSTABLE" : "STABLE",
+               wverf);
 }
 
-
 void
-nfs3_log_newfh_res (uint32_t xid, int op, nfsstat3 stat, int pstat,
-                    struct nfs3_fh *newfh)
+nfs3_log_newfh_res(uint32_t xid, int op, nfsstat3 stat, int pstat,
+                   struct nfs3_fh *newfh, const char *path)
 {
-        char    errstr[1024];
-        char    fhstr[1024];
-        int     ll = nfs3_loglevel (op, stat);
+    char errstr[1024];
+    char fhstr[1024];
+    int ll = nfs3_loglevel(op, stat);
 
-        if (THIS->ctx->log.loglevel < ll)
-                return;
-        nfs3_stat_to_errstr (xid, nfs3op_strings[op].str, stat, pstat, errstr, sizeof (errstr));
-        nfs3_fh_to_str (newfh, fhstr, sizeof (fhstr));
+    if (THIS->ctx->log.loglevel < ll)
+        return;
+    nfs3_stat_to_errstr(xid, nfs3op_strings[op].str, stat, pstat, errstr,
+                        sizeof(errstr));
+    nfs3_fh_to_str(newfh, fhstr, sizeof(fhstr));
 
-        gf_log (GF_NFS3, nfs3_loglevel (op, stat), "%s, %s", errstr, fhstr);
+    if (ll == GF_LOG_DEBUG)
+        gf_msg_debug(GF_NFS3, 0, "%s => (%s), %s", path, errstr, fhstr);
+    else
+        gf_msg(GF_NFS3, nfs3_loglevel(op, stat), errno, NFS_MSG_STAT_ERROR,
+               "%s => (%s), %s", path, errstr, fhstr);
 }
 
-
 void
-nfs3_log_readdir_res (uint32_t xid, nfsstat3 stat, int pstat, uint64_t cverf,
-                      count3 count, int is_eof)
+nfs3_log_readdir_res(uint32_t xid, nfsstat3 stat, int pstat, uint64_t cverf,
+                     count3 count, int is_eof, const char *path)
 {
-        char    errstr[1024];
-        int     ll = nfs3_loglevel (NFS3_READDIR, stat);
+    char errstr[1024];
+    int ll = nfs3_loglevel(NFS3_READDIR, stat);
 
-        if (THIS->ctx->log.loglevel < ll)
-                return;
-        nfs3_stat_to_errstr (xid, "READDIR", stat, pstat, errstr, sizeof (errstr));
-        gf_log (GF_NFS3, ll, "%s, count: %"PRIu32", cverf: %"PRIu64
-                ", is_eof: %d", errstr, count, cverf, is_eof);
+    if (THIS->ctx->log.loglevel < ll)
+        return;
+    nfs3_stat_to_errstr(xid, "READDIR", stat, pstat, errstr, sizeof(errstr));
+    if (ll == GF_LOG_DEBUG)
+        gf_msg_debug(GF_NFS3, 0,
+                     "%s => (%s), count: %" PRIu32 ", cverf: %" PRIu64
+                     ", is_eof: %d",
+                     path, errstr, count, cverf, is_eof);
+    else
+        gf_msg(GF_NFS3, ll, errno, NFS_MSG_STAT_ERROR,
+               "%s => (%s), count: %" PRIu32 ", cverf: %" PRIu64 ", is_eof: %d",
+               path, errstr, count, cverf, is_eof);
 }
 
-
 void
-nfs3_log_readdirp_res (uint32_t xid, nfsstat3 stat, int pstat, uint64_t cverf,
-                       count3 dircount, count3 maxcount, int is_eof)
+nfs3_log_readdirp_res(uint32_t xid, nfsstat3 stat, int pstat, uint64_t cverf,
+                      count3 dircount, count3 maxcount, int is_eof,
+                      const char *path)
 {
-        char    errstr[1024];
-	int	ll = nfs3_loglevel (NFS3_READDIRP, stat);
+    char errstr[1024];
+    int ll = nfs3_loglevel(NFS3_READDIRP, stat);
 
-        if (THIS->ctx->log.loglevel < ll)
-                return;
-        nfs3_stat_to_errstr (xid, "READDIRPLUS", stat, pstat, errstr, sizeof (errstr));
-        gf_log (GF_NFS3, ll, "%s, dircount: %"PRIu32", maxcount: %"
-                PRIu32", cverf: %"PRIu64", is_eof: %d", errstr, dircount,
-                maxcount, cverf, is_eof);
+    if (THIS->ctx->log.loglevel < ll)
+        return;
+    nfs3_stat_to_errstr(xid, "READDIRPLUS", stat, pstat, errstr,
+                        sizeof(errstr));
+    if (ll == GF_LOG_DEBUG)
+        gf_msg_debug(GF_NFS3, 0,
+                     "%s => (%s), dircount: %" PRIu32 ", maxcount: %" PRIu32
+                     ", cverf: %" PRIu64 ", is_eof: %d",
+                     path, errstr, dircount, maxcount, cverf, is_eof);
+    else
+        gf_msg(GF_NFS3, ll, errno, NFS_MSG_STAT_ERROR,
+               "%s => (%s), dircount: %" PRIu32 ", maxcount: %" PRIu32
+               ", cverf: %" PRIu64 ", is_eof: %d",
+               path, errstr, dircount, maxcount, cverf, is_eof);
 }
 
-
 void
-nfs3_log_commit_res (uint32_t xid, nfsstat3 stat, int pstat, uint64_t wverf)
+nfs3_log_commit_res(uint32_t xid, nfsstat3 stat, int pstat, uint64_t wverf,
+                    const char *path)
 {
-        char    errstr[1024];
-	int	ll = nfs3_loglevel (NFS3_COMMIT, stat);
+    char errstr[1024];
+    int ll = nfs3_loglevel(NFS3_COMMIT, stat);
 
-        if (THIS->ctx->log.loglevel < ll)
-                return;
-        nfs3_stat_to_errstr (xid, "COMMIT", stat, pstat, errstr, sizeof (errstr));
-        gf_log (GF_NFS3, ll, "%s, wverf: %"PRIu64, errstr, wverf);
+    if (THIS->ctx->log.loglevel < ll)
+        return;
+    nfs3_stat_to_errstr(xid, "COMMIT", stat, pstat, errstr, sizeof(errstr));
+    if (ll == GF_LOG_DEBUG)
+        gf_msg_debug(GF_NFS3, 0, "%s => (%s), wverf: %" PRIu64, path, errstr,
+                     wverf);
+    else
+        gf_msg(GF_NFS3, ll, errno, NFS_MSG_STAT_ERROR,
+               "%s => (%s), wverf: %" PRIu64, path, errstr, wverf);
 }
 
-
 void
-nfs3_log_readdir_call (uint32_t xid, struct nfs3_fh *fh, count3 dircount,
-                       count3 maxcount)
+nfs3_log_readdir_call(uint32_t xid, struct nfs3_fh *fh, count3 dircount,
+                      count3 maxcount)
 {
-        char    fhstr[1024];
+    char fhstr[1024];
 
-        if (THIS->ctx->log.loglevel < GF_LOG_DEBUG)
-                return;
+    if (THIS->ctx->log.loglevel < GF_LOG_DEBUG)
+        return;
 
-        nfs3_fh_to_str (fh, fhstr, sizeof (fhstr));
+    nfs3_fh_to_str(fh, fhstr, sizeof(fhstr));
 
-        if (maxcount == 0)
-                gf_log (GF_NFS3, GF_LOG_DEBUG, "XID: %x, READDIR: args: %s,"
-                        " count: %d", xid, fhstr, (uint32_t)dircount);
-        else
-                gf_log (GF_NFS3, GF_LOG_DEBUG, "XID: %x, READDIRPLUS: args: %s,"
-                        " dircount: %d, maxcount: %d", xid, fhstr,
-                        (uint32_t)dircount, (uint32_t)maxcount);
+    if (maxcount == 0)
+        gf_msg_debug(GF_NFS3, 0,
+                     "XID: %x, READDIR: args: %s,"
+                     " count: %d",
+                     xid, fhstr, (uint32_t)dircount);
+    else
+        gf_msg_debug(GF_NFS3, 0,
+                     "XID: %x, READDIRPLUS: args: %s,"
+                     " dircount: %d, maxcount: %d",
+                     xid, fhstr, (uint32_t)dircount, (uint32_t)maxcount);
 }
 
-
 int
-nfs3_fh_resolve_inode_done (nfs3_call_state_t *cs, inode_t *inode)
+nfs3_fh_resolve_inode_done(nfs3_call_state_t *cs, inode_t *inode)
 {
-        int             ret = -EFAULT;
+    int ret = -EFAULT;
 
-        if ((!cs) || (!inode))
-                return ret;
+    if ((!cs) || (!inode))
+        return ret;
 
-        gf_log (GF_NFS3, GF_LOG_TRACE, "FH inode resolved");
-        ret = nfs_inode_loc_fill (inode, &cs->resolvedloc, NFS_RESOLVE_EXIST);
-        if (ret < 0) {
-                gf_log (GF_NFS3, GF_LOG_ERROR, "inode loc fill failed");
-                goto err;
-        }
+    gf_msg_trace(GF_NFS3, 0, "FH inode resolved");
+    ret = nfs_inode_loc_fill(inode, &cs->resolvedloc, NFS_RESOLVE_EXIST);
+    if (ret < 0) {
+        gf_msg(GF_NFS3, GF_LOG_ERROR, -ret, NFS_MSG_INODE_LOC_FILL_ERROR,
+               "inode loc fill failed");
+        goto err;
+    }
 
-        nfs3_call_resume (cs);
+    nfs3_call_resume(cs);
 
 err:
-        return ret;
+    return ret;
 }
 
-
 int32_t
-nfs3_fh_resolve_entry_lookup_cbk (call_frame_t *frame, void *cookie,
-                                  xlator_t *this, int32_t op_ret,
-                                  int32_t op_errno, inode_t *inode,
-                                  struct iatt *buf, dict_t *xattr,
-                                  struct iatt *postparent)
-{
-        nfs3_call_state_t       *cs = NULL;
-        inode_t                 *linked_inode = NULL;
-
-        cs = frame->local;
-        cs->resolve_ret = op_ret;
-        cs->resolve_errno = op_errno;
-
-        if (op_ret == -1) {
-                gf_log (GF_NFS3, (op_errno == ENOENT ? GF_LOG_TRACE : GF_LOG_ERROR),
-                        "Lookup failed: %s: %s",
-                        cs->resolvedloc.path, strerror (op_errno));
-                goto err;
-        } else
-                gf_log (GF_NFS3, GF_LOG_TRACE, "Entry looked up: %s",
-                        cs->resolvedloc.path);
-
-	memcpy (&cs->stbuf, buf, sizeof (*buf));
-	memcpy (&cs->postparent, postparent, sizeof (*postparent));
-        linked_inode = inode_link (inode, cs->resolvedloc.parent,
-                                   cs->resolvedloc.name, buf);
-        if (linked_inode) {
-                inode_lookup (linked_inode);
-                inode_unref (cs->resolvedloc.inode);
-                cs->resolvedloc.inode = linked_inode;
+nfs3_fh_resolve_entry_lookup_cbk(call_frame_t *frame, void *cookie,
+                                 xlator_t *this, int32_t op_ret,
+                                 int32_t op_errno, inode_t *inode,
+                                 struct iatt *buf, dict_t *xattr,
+                                 struct iatt *postparent)
+{
+    nfs3_call_state_t *cs = NULL;
+    inode_t *linked_inode = NULL;
+
+    cs = frame->local;
+    cs->resolve_ret = op_ret;
+    cs->resolve_errno = op_errno;
+
+    if (op_ret == -1) {
+        if (op_errno == ENOENT) {
+            gf_msg_trace(GF_NFS3, 0, "Lookup failed: %s: %s",
+                         cs->resolvedloc.path, strerror(op_errno));
+        } else {
+            gf_msg(GF_NFS3, GF_LOG_ERROR, op_errno, NFS_MSG_LOOKUP_FAIL,
+                   "Lookup failed: %s: %s", cs->resolvedloc.path,
+                   strerror(op_errno));
         }
+        goto err;
+    } else
+        gf_msg_trace(GF_NFS3, 0, "Entry looked up: %s", cs->resolvedloc.path);
+
+    memcpy(&cs->stbuf, buf, sizeof(*buf));
+    memcpy(&cs->postparent, postparent, sizeof(*postparent));
+    linked_inode = inode_link(inode, cs->resolvedloc.parent,
+                              cs->resolvedloc.name, buf);
+    if (linked_inode) {
+        nfs_fix_generation(this, linked_inode);
+        inode_lookup(linked_inode);
+        inode_unref(cs->resolvedloc.inode);
+        cs->resolvedloc.inode = linked_inode;
+    } else {
+        /* nfs3_fh_resolve_entry_hard() use to resolve entire path if needed.
+         * So the ctx for inode obtained from here need to set properly,
+         * otherwise it may result in a crash.
+         */
+        nfs_fix_generation(this, inode);
+    }
 err:
-        nfs3_call_resume (cs);
-        return 0;
+    nfs3_call_resume(cs);
+    return 0;
 }
 
-
 int32_t
-nfs3_fh_resolve_inode_lookup_cbk (call_frame_t *frame, void *cookie,
-                                  xlator_t *this, int32_t op_ret,
-                                  int32_t op_errno, inode_t *inode,
-                                  struct iatt *buf, dict_t *xattr,
-                                  struct iatt *postparent)
-{
-        nfs3_call_state_t       *cs = NULL;
-        inode_t                 *linked_inode = NULL;
-
-        cs = frame->local;
-        cs->resolve_ret = op_ret;
-        cs->resolve_errno = op_errno;
-
-        if (op_ret == -1) {
-                gf_log (GF_NFS3, (op_errno == ENOENT ? GF_LOG_TRACE : GF_LOG_ERROR),
-                        "Lookup failed: %s: %s",
-                        cs->resolvedloc.path, strerror (op_errno));
-                nfs3_call_resume (cs);
-                goto err;
-        }
+nfs3_fh_resolve_inode_lookup_cbk(call_frame_t *frame, void *cookie,
+                                 xlator_t *this, int32_t op_ret,
+                                 int32_t op_errno, inode_t *inode,
+                                 struct iatt *buf, dict_t *xattr,
+                                 struct iatt *postparent)
+{
+    nfs3_call_state_t *cs = NULL;
+    inode_t *linked_inode = NULL;
 
-	memcpy (&cs->stbuf, buf, sizeof(*buf));
-	memcpy (&cs->postparent, buf, sizeof(*postparent));
-        linked_inode = inode_link (inode, cs->resolvedloc.parent,
-                                   cs->resolvedloc.name, buf);
-        if (linked_inode) {
-                inode_lookup (linked_inode);
-		inode_unref (cs->resolvedloc.inode);
-		cs->resolvedloc.inode = linked_inode;
-        }
+    cs = frame->local;
+    cs->resolve_ret = op_ret;
+    cs->resolve_errno = op_errno;
 
-        /* If it is an entry lookup and we landed in the callback for hard
-         * inode resolution, it means the parent inode was not available and
-         * had to be resolved first. Now that is done, lets head back into
-         * entry resolution.
-         */
-        if (cs->resolventry)
-                nfs3_fh_resolve_entry_hard (cs);
-	else
-		nfs3_call_resume (cs);
+    if (op_ret == -1) {
+        if (op_errno == ENOENT) {
+            gf_msg_trace(GF_NFS3, 0, "Lookup failed: %s: %s",
+                         cs->resolvedloc.path, strerror(op_errno));
+        } else {
+            gf_msg(GF_NFS3, GF_LOG_ERROR, op_errno, NFS_MSG_LOOKUP_FAIL,
+                   "Lookup failed: %s: %s", cs->resolvedloc.path,
+                   strerror(op_errno));
+        }
+        nfs3_call_resume(cs);
+        goto err;
+    }
+
+    memcpy(&cs->stbuf, buf, sizeof(*buf));
+    memcpy(&cs->postparent, buf, sizeof(*postparent));
+    linked_inode = inode_link(inode, cs->resolvedloc.parent,
+                              cs->resolvedloc.name, buf);
+    if (linked_inode) {
+        nfs_fix_generation(this, linked_inode);
+        inode_lookup(linked_inode);
+        inode_unref(cs->resolvedloc.inode);
+        cs->resolvedloc.inode = linked_inode;
+    }
+
+    /* If it is an entry lookup and we landed in the callback for hard
+     * inode resolution, it means the parent inode was not available and
+     * had to be resolved first. Now that is done, lets head back into
+     * entry resolution.
+     */
+    if (cs->resolventry)
+        nfs3_fh_resolve_entry_hard(cs);
+    else
+        nfs3_call_resume(cs);
 err:
-        return 0;
+    return 0;
 }
 
-
-
 /* Needs no extra argument since it knows that the fh to be resolved is in
  * resolvefh and that it needs to start looking from the root.
  */
 int
-nfs3_fh_resolve_inode_hard (nfs3_call_state_t *cs)
-{
-        int             ret = -EFAULT;
-        nfs_user_t      nfu = {0, };
-
-        if (!cs)
-                return ret;
-
-        gf_log (GF_NFS3, GF_LOG_TRACE, "FH hard resolution for: gfid 0x%s",
-                uuid_utoa (cs->resolvefh.gfid));
-	cs->hardresolved = 1;
-        nfs_loc_wipe (&cs->resolvedloc);
-        ret = nfs_gfid_loc_fill (cs->vol->itable, cs->resolvefh.gfid,
-                                 &cs->resolvedloc, NFS_RESOLVE_CREATE);
-        if (ret < 0) {
-                gf_log (GF_NFS3, GF_LOG_ERROR, "Failed to fill loc using gfid: "
-                        "%s", strerror (-ret));
-                goto out;
-        }
+nfs3_fh_resolve_inode_hard(nfs3_call_state_t *cs)
+{
+    int ret = -EFAULT;
+    nfs_user_t nfu = {
+        0,
+    };
+
+    if (!cs)
+        return ret;
 
-        nfs_user_root_create (&nfu);
-        ret = nfs_lookup (cs->nfsx, cs->vol, &nfu, &cs->resolvedloc,
-                          nfs3_fh_resolve_inode_lookup_cbk, cs);
+    gf_msg_trace(GF_NFS3, 0, "FH hard resolution for: gfid 0x%s",
+                 uuid_utoa(cs->resolvefh.gfid));
+    cs->hardresolved = 1;
+    nfs_loc_wipe(&cs->resolvedloc);
+    ret = nfs_gfid_loc_fill(cs->vol->itable, cs->resolvefh.gfid,
+                            &cs->resolvedloc, NFS_RESOLVE_CREATE);
+    if (ret < 0) {
+        gf_msg(GF_NFS3, GF_LOG_ERROR, -ret, NFS_MSG_INODE_LOC_FILL_ERROR,
+               "Failed to fill loc using gfid: "
+               "%s",
+               strerror(-ret));
+        goto out;
+    }
+
+    nfs_user_root_create(&nfu);
+    ret = nfs_lookup(cs->nfsx, cs->vol, &nfu, &cs->resolvedloc,
+                     nfs3_fh_resolve_inode_lookup_cbk, cs);
 
 out:
-        return ret;
+    return ret;
 }
 
-
 int
-nfs3_fh_resolve_entry_hard (nfs3_call_state_t *cs)
-{
-        int             ret = -EFAULT;
-        nfs_user_t      nfu = {0, };
-
-        if (!cs)
-                return ret;
-
-        nfs_loc_wipe (&cs->resolvedloc);
-        nfs_user_root_create (&nfu);
-        gf_log (GF_NFS3, GF_LOG_TRACE, "FH hard resolution: gfid: %s "
-                ", entry: %s", uuid_utoa (cs->resolvefh.gfid),
-                cs->resolventry);
-
-        ret = nfs_entry_loc_fill (cs->vol->itable, cs->resolvefh.gfid,
-                                  cs->resolventry, &cs->resolvedloc,
-                                  NFS_RESOLVE_CREATE);
-
-        if (ret == -2) {
-                gf_log (GF_NFS3, GF_LOG_TRACE, "Entry needs lookup: %s",
-                        cs->resolvedloc.path);
-		/* If the NFS op is lookup, let the resume callback
-		 * handle the sending of the lookup fop. Similarly,
-		 * if the NFS op is create, let the create call
-		 * go ahead in the resume callback so that an EEXIST gets
-		 * handled at posix without an extra fop at this point.
-		 */
-                if (nfs3_lookup_op (cs) ||
-		    (nfs3_create_op (cs) && !nfs3_create_exclusive_op (cs))) {
-                        cs->lookuptype = GF_NFS3_FRESH;
-                        cs->resolve_ret = 0;
-                        nfs3_call_resume (cs);
-                } else {
-			cs->hardresolved = 1;
-                        nfs_lookup (cs->nfsx, cs->vol, &nfu, &cs->resolvedloc,
-                                    nfs3_fh_resolve_entry_lookup_cbk, cs);
-		}
-                ret = 0;
-        } else if (ret == -1) {
-                gf_log (GF_NFS3, GF_LOG_TRACE, "Entry needs parent lookup: %s",
-                        cs->resolvedloc.path);
-                ret = nfs3_fh_resolve_inode_hard (cs);
-        } else if (ret == 0) {
-                cs->resolve_ret = 0;
-                nfs3_call_resume (cs);
-        }
+nfs3_fh_resolve_entry_hard(nfs3_call_state_t *cs)
+{
+    int ret = -EFAULT;
+    nfs_user_t nfu = {
+        0,
+    };
+    gf_boolean_t freshlookup = _gf_false;
 
+    if (!cs)
         return ret;
+
+    nfs_loc_wipe(&cs->resolvedloc);
+    nfs_user_root_create(&nfu);
+    gf_msg_trace(GF_NFS3, 0,
+                 "FH hard resolution: gfid: %s "
+                 ", entry: %s",
+                 uuid_utoa(cs->resolvefh.gfid), cs->resolventry);
+
+    ret = nfs_entry_loc_fill(cs->nfsx, cs->vol->itable, cs->resolvefh.gfid,
+                             cs->resolventry, &cs->resolvedloc,
+                             NFS_RESOLVE_CREATE, &freshlookup);
+
+    if (ret == -2) {
+        gf_msg_trace(GF_NFS3, 0, "Entry needs lookup: %s",
+                     cs->resolvedloc.path);
+        /* If the NFS op is lookup, let the resume callback
+         * handle the sending of the lookup fop. Similarly,
+         * if the NFS op is create, let the create call
+         * go ahead in the resume callback so that an EEXIST gets
+         * handled at posix without an extra fop at this point.
+         */
+        if (freshlookup &&
+            (nfs3_lookup_op(cs) ||
+             (nfs3_create_op(cs) && !nfs3_create_exclusive_op(cs)))) {
+            cs->lookuptype = GF_NFS3_FRESH;
+            cs->resolve_ret = 0;
+            cs->hardresolved = 0;
+            nfs3_call_resume(cs);
+        } else {
+            cs->hardresolved = 1;
+            nfs_lookup(cs->nfsx, cs->vol, &nfu, &cs->resolvedloc,
+                       nfs3_fh_resolve_entry_lookup_cbk, cs);
+        }
+        ret = 0;
+    } else if (ret == -1) {
+        gf_msg_trace(GF_NFS3, 0, "Entry needs parent lookup: %s",
+                     cs->resolvedloc.path);
+        ret = nfs3_fh_resolve_inode_hard(cs);
+    } else if (ret == 0) {
+        cs->resolve_ret = 0;
+        nfs3_call_resume(cs);
+    }
+
+    return ret;
 }
 
 int
-nfs3_fh_resolve_inode (nfs3_call_state_t *cs)
+nfs3_fh_resolve_inode(nfs3_call_state_t *cs)
 {
-        inode_t         *inode = NULL;
-        int             ret = -EFAULT;
+    inode_t *inode = NULL;
+    int ret = -EFAULT;
+    xlator_t *this = NULL;
 
-        if (!cs)
-                return ret;
+    if (!cs)
+        return ret;
 
-        gf_log (GF_NFS3, GF_LOG_TRACE, "FH needs inode resolution");
-        gf_uuid_copy (cs->resolvedloc.gfid, cs->resolvefh.gfid);
-        inode = inode_find (cs->vol->itable, cs->resolvefh.gfid);
-        if (!inode)
-                ret = nfs3_fh_resolve_inode_hard (cs);
-        else
-                ret = nfs3_fh_resolve_inode_done (cs, inode);
+    this = cs->nfsx;
+    gf_msg_trace(GF_NFS3, 0, "FH needs inode resolution");
+    gf_uuid_copy(cs->resolvedloc.gfid, cs->resolvefh.gfid);
 
-        if (inode)
-                inode_unref (inode);
+    inode = inode_find(cs->vol->itable, cs->resolvefh.gfid);
+    if (!inode || inode_ctx_get(inode, this, NULL))
+        ret = nfs3_fh_resolve_inode_hard(cs);
+    else
+        ret = nfs3_fh_resolve_inode_done(cs, inode);
 
-        return ret;
+    if (inode)
+        inode_unref(inode);
+
+    return ret;
 }
 
 int
-nfs3_fh_resolve_entry (nfs3_call_state_t *cs)
+nfs3_fh_resolve_entry(nfs3_call_state_t *cs)
 {
-        int     ret = -EFAULT;
+    int ret = -EFAULT;
 
-        if (!cs)
-                return ret;
+    if (!cs)
+        return ret;
 
-        return nfs3_fh_resolve_entry_hard (cs);
+    return nfs3_fh_resolve_entry_hard(cs);
 }
 
-
 int
-nfs3_fh_resolve_resume (nfs3_call_state_t *cs)
+nfs3_fh_resolve_resume(nfs3_call_state_t *cs)
 {
-        int     ret = -EFAULT;
+    int ret = -EFAULT;
 
-        if (!cs)
-                return ret;
+    if (!cs)
+        return ret;
 
-        if (cs->resolve_ret < 0)
-                goto err_resume_call;
+    if (cs->resolve_ret < 0)
+        goto err_resume_call;
 
-        if (!cs->resolventry)
-                ret = nfs3_fh_resolve_inode (cs);
-        else
-                ret = nfs3_fh_resolve_entry (cs);
+    if (!cs->resolventry)
+        ret = nfs3_fh_resolve_inode(cs);
+    else
+        ret = nfs3_fh_resolve_entry(cs);
 
 err_resume_call:
-        if (ret < 0) {
-                cs->resolve_ret = -1;
-                cs->resolve_errno = EFAULT;
-                nfs3_call_resume (cs);
-                ret = 0;
-        }
+    if (ret < 0) {
+        cs->resolve_ret = -1;
+        cs->resolve_errno = EFAULT;
+        nfs3_call_resume(cs);
+        ret = 0;
+    }
 
-        return ret;
+    return ret;
 }
 
-
 int32_t
-nfs3_fh_resolve_root_lookup_cbk (call_frame_t *frame, void *cookie,
-                                 xlator_t *this, int32_t op_ret,
-                                 int32_t op_errno, inode_t *inode,
-                                 struct iatt *buf, dict_t *xattr,
-                                 struct iatt *postparent)
-{
-        nfs3_call_state_t       *cs = NULL;
-
-        cs = frame->local;
-        cs->resolve_ret = op_ret;
-        cs->resolve_errno = op_errno;
-
-        if (op_ret == -1) {
-                gf_log (GF_NFS3, GF_LOG_ERROR, "Root lookup failed: %s",
-                        strerror (op_errno));
-                goto err;
-        } else
-                gf_log (GF_NFS3, GF_LOG_TRACE, "Root looked up: %s",
-                        cs->resolvedloc.path);
-
-        nfs3_set_root_looked_up (cs->nfs3state, &cs->resolvefh);
+nfs3_fh_resolve_root_lookup_cbk(call_frame_t *frame, void *cookie,
+                                xlator_t *this, int32_t op_ret,
+                                int32_t op_errno, inode_t *inode,
+                                struct iatt *buf, dict_t *xattr,
+                                struct iatt *postparent)
+{
+    nfs3_call_state_t *cs = NULL;
+
+    cs = frame->local;
+    cs->resolve_ret = op_ret;
+    cs->resolve_errno = op_errno;
+
+    if (op_ret == -1) {
+        gf_msg(GF_NFS3, GF_LOG_ERROR, op_errno, NFS_MSG_LOOKUP_ROOT_FAIL,
+               "Root lookup failed: %s", strerror(op_errno));
+        goto err;
+    } else
+        gf_msg_trace(GF_NFS3, 0, "Root looked up: %s", cs->resolvedloc.path);
+
+    nfs3_set_root_looked_up(cs->nfs3state, &cs->resolvefh);
 err:
-        nfs3_fh_resolve_resume (cs);
-        return 0;
+    nfs3_fh_resolve_resume(cs);
+    return 0;
 }
 
-
 int
-nfs3_fh_resolve_root (nfs3_call_state_t *cs)
+nfs3_fh_resolve_root(nfs3_call_state_t *cs)
 {
-        int             ret = -EFAULT;
-        nfs_user_t      nfu = {0, };
+    int ret = -EFAULT;
+    nfs_user_t nfu = {
+        0,
+    };
 
-        if (!cs)
-                return ret;
+    if (!cs)
+        return ret;
 
-        if (nfs3_is_root_looked_up (cs->nfs3state, &cs->resolvefh)) {
-                ret = nfs3_fh_resolve_resume (cs);
-                goto out;
-        }
+    if (nfs3_is_root_looked_up(cs->nfs3state, &cs->resolvefh)) {
+        ret = nfs3_fh_resolve_resume(cs);
+        goto out;
+    }
 
-        nfs_user_root_create (&nfu);
-        gf_log (GF_NFS3, GF_LOG_TRACE, "Root needs lookup");
-        ret = nfs_root_loc_fill (cs->vol->itable, &cs->resolvedloc);
-	if (ret < 0) {
-		gf_log (GF_NFS3, GF_LOG_ERROR, "Failed to lookup root from itable: %s",
-			strerror (-ret));
-		goto out;
-	}
+    nfs_user_root_create(&nfu);
+    gf_msg_trace(GF_NFS3, 0, "Root needs lookup");
+    ret = nfs_root_loc_fill(cs->vol->itable, &cs->resolvedloc);
+    if (ret < 0) {
+        gf_msg(GF_NFS3, GF_LOG_ERROR, -ret, NFS_MSG_LOOKUP_ROOT_FAIL,
+               "Failed to lookup root from itable: %s", strerror(-ret));
+        goto out;
+    }
 
-        ret = nfs_lookup (cs->nfsx, cs->vol, &nfu, &cs->resolvedloc,
-                          nfs3_fh_resolve_root_lookup_cbk, cs);
+    ret = nfs_lookup(cs->nfsx, cs->vol, &nfu, &cs->resolvedloc,
+                     nfs3_fh_resolve_root_lookup_cbk, cs);
 
 out:
-        return ret;
+    return ret;
 }
 
 /**
@@ -3852,25 +3841,27 @@ out:
  * @req : The request to get host/peer from
  */
 int
-__nfs3_fh_auth_get_peer (const rpcsvc_request_t *req, char *peer)
-{
-        struct sockaddr_storage sastorage       = {0, };
-        rpc_transport_t         *trans          = NULL;
-        int                     ret             = 0;
-
-        /* Why do we pass in the peer here and then
-         * store it rather than malloc() and return a char * ? We want to avoid
-         * heap allocations in the IO path as much as possible for speed
-         * so we try to keep all allocations on the stack.
-         */
-        trans = rpcsvc_request_transport (req);
-        ret = rpcsvc_transport_peeraddr (trans, peer, RPCSVC_PEER_STRLEN,
-                                         &sastorage, sizeof (sastorage));
-        if (ret != 0) {
-                gf_log (GF_NFS3, GF_LOG_WARNING, "Failed to get peer addr: %s",
-                        gai_strerror (ret));
-        }
-        return ret;
+__nfs3_fh_auth_get_peer(const rpcsvc_request_t *req, char *peer)
+{
+    struct sockaddr_storage sastorage = {
+        0,
+    };
+    rpc_transport_t *trans = NULL;
+    int ret = 0;
+
+    /* Why do we pass in the peer here and then
+     * store it rather than malloc() and return a char * ? We want to avoid
+     * heap allocations in the IO path as much as possible for speed
+     * so we try to keep all allocations on the stack.
+     */
+    trans = rpcsvc_request_transport(req);
+    ret = rpcsvc_transport_peeraddr(trans, peer, RPCSVC_PEER_STRLEN, &sastorage,
+                                    sizeof(sastorage));
+    if (ret != 0) {
+        gf_msg(GF_NFS3, GF_LOG_WARNING, 0, NFS_MSG_GET_PEER_ADDR_FAIL,
+               "Failed to get peer addr: %s", gai_strerror(ret));
+    }
+    return ret;
 }
 
 /*
@@ -3882,45 +3873,45 @@ __nfs3_fh_auth_get_peer (const rpcsvc_request_t *req, char *peer)
  *          -EACCES for completely unauthorized fop
  *          -EROFS  for unauthorized write operations (rm, mkdir, write)
  */
-inline int
-nfs3_fh_auth_nfsop (nfs3_call_state_t *cs, gf_boolean_t is_write_op)
+int
+nfs3_fh_auth_nfsop(nfs3_call_state_t *cs, gf_boolean_t is_write_op)
 {
-        struct nfs_state    *nfs = NULL;
-        struct mount3_state *ms  = NULL;
+    struct nfs_state *nfs = NULL;
+    struct mount3_state *ms = NULL;
 
-        nfs = (struct nfs_state *)cs->nfsx->private;
-        ms  = (struct mount3_state *)nfs->mstate;
-        return  mnt3_authenticate_request (ms, cs->req, &cs->resolvefh, NULL,
-                                           NULL, NULL, NULL, is_write_op);
+    nfs = (struct nfs_state *)cs->nfsx->private;
+    ms = (struct mount3_state *)nfs->mstate;
+    return mnt3_authenticate_request(ms, cs->req, &cs->resolvefh, NULL, NULL,
+                                     NULL, NULL, is_write_op);
 }
 
 int
-nfs3_fh_resolve_and_resume (nfs3_call_state_t *cs, struct nfs3_fh *fh,
-                            char *entry, nfs3_resume_fn_t resum_fn)
+nfs3_fh_resolve_and_resume(nfs3_call_state_t *cs, struct nfs3_fh *fh,
+                           char *entry, nfs3_resume_fn_t resum_fn)
 {
-        int     ret = -EFAULT;
+    int ret = -EFAULT;
 
-        if ((!cs) || (!fh))
-                return ret;
-
-        cs->resume_fn = resum_fn;
-        cs->resolvefh = *fh;
-        cs->hashidx = 0;
+    if ((!cs) || (!fh))
+        return ret;
 
-        /* Check if the resolution is:
-         * a. fh resolution
-         *
-         * or
-         *
-         * b. (fh, basename) resolution
-         */
-        if (entry) {    /* b */
-                cs->resolventry = gf_strdup (entry);
-                if (!cs->resolventry)
-                        goto err;
-        }
+    cs->resume_fn = resum_fn;
+    cs->resolvefh = *fh;
+    cs->hashidx = 0;
+
+    /* Check if the resolution is:
+     * a. fh resolution
+     *
+     * or
+     *
+     * b. (fh, basename) resolution
+     */
+    if (entry) { /* b */
+        cs->resolventry = gf_strdup(entry);
+        if (!cs->resolventry)
+            goto err;
+    }
 
-        ret = nfs3_fh_resolve_root (cs);
+    ret = nfs3_fh_resolve_root(cs);
 err:
-        return ret;
+    return ret;
 }
diff --git a/xlators/nfs/server/src/nfs3-helpers.h b/xlators/nfs/server/src/nfs3-helpers.h
index 626aa159d3a..aead636c831 100644
--- a/xlators/nfs/server/src/nfs3-helpers.h
+++ b/xlators/nfs/server/src/nfs3-helpers.h
@@ -10,13 +10,8 @@
 
 #ifndef _NFS3_HELPER_H_
 #define _NFS3_HELPER_H_
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
-
 
-#include "xlator.h"
+#include <glusterfs/xlator.h>
 #include "nfs3.h"
 #include "nfs3-fh.h"
 #include "msg-nfs3.h"
@@ -24,41 +19,40 @@
 
 #include <sys/statvfs.h>
 
-#define GF_NFS3_FD_CACHED       0xcaced
+#define GF_NFS3_FD_CACHED 0xcaced
 
 extern struct nfs3_fh
-nfs3_extract_lookup_fh (lookup3args *args);
+nfs3_extract_lookup_fh(lookup3args *args);
 
 extern char *
-nfs3_extract_lookup_name (lookup3args *args);
+nfs3_extract_lookup_name(lookup3args *args);
 
 extern nfsstat3
-nfs3_errno_to_nfsstat3 (int errnum);
+nfs3_errno_to_nfsstat3(int errnum);
 
-extern nfsstat3
-nfs3_cbk_errno_status (int32_t, int32_t);
+extern nfsstat3 nfs3_cbk_errno_status(int32_t, int32_t);
 
 extern void
-nfs3_fill_lookup3res (lookup3res *res, nfsstat3 stat, struct nfs3_fh *newfh,
-                      struct iatt *stbuf, struct iatt *postparent,
-                      uint64_t deviceid);
+nfs3_fill_lookup3res(lookup3res *res, nfsstat3 stat, struct nfs3_fh *newfh,
+                     struct iatt *stbuf, struct iatt *postparent,
+                     uint64_t deviceid);
 
 extern post_op_attr
-nfs3_stat_to_post_op_attr (struct iatt *buf);
+nfs3_stat_to_post_op_attr(struct iatt *buf);
 
 extern struct nfs3_fh
-nfs3_extract_getattr_fh (getattr3args *args);
+nfs3_extract_getattr_fh(getattr3args *args);
 
 extern void
-nfs3_fill_getattr3res (getattr3res *res, nfsstat3 stat, struct iatt *buf,
-                       uint64_t deviceid);
+nfs3_fill_getattr3res(getattr3res *res, nfsstat3 stat, struct iatt *buf,
+                      uint64_t deviceid);
 
 extern struct nfs3_fh
-nfs3_extract_fsinfo_fh (fsinfo3args *args);
+nfs3_extract_fsinfo_fh(fsinfo3args *args);
 
 extern void
-nfs3_fill_fsinfo3res (struct nfs3_state *nfs3, fsinfo3res *res,
-                      nfsstat3 status, struct iatt *fsroot,uint64_t deviceid);
+nfs3_fill_fsinfo3res(struct nfs3_state *nfs3, fsinfo3res *res, nfsstat3 status,
+                     struct iatt *fsroot, uint64_t deviceid);
 
 /* Functions containing _prep_ are used specifically to work around
  * the memory allocations that happen inside Sun RPC library.
@@ -78,266 +72,268 @@ nfs3_fill_fsinfo3res (struct nfs3_state *nfs3, fsinfo3res *res,
  * looking through the glibc/sunrpc sources.
  */
 extern void
-nfs3_prep_lookup3args (lookup3args *args, struct nfs3_fh *fh, char *name);
+nfs3_prep_lookup3args(lookup3args *args, struct nfs3_fh *fh, char *name);
 
 extern void
-nfs3_prep_getattr3args (getattr3args *args, struct nfs3_fh *fh);
+nfs3_prep_getattr3args(getattr3args *args, struct nfs3_fh *fh);
 
 extern void
-nfs3_prep_fsinfo3args (fsinfo3args *args, struct nfs3_fh *root);
+nfs3_prep_fsinfo3args(fsinfo3args *args, struct nfs3_fh *root);
 
 extern char *
 nfsstat3_strerror(int stat);
 
 extern void
-nfs3_prep_access3args (access3args *args, struct nfs3_fh *fh);
+nfs3_prep_access3args(access3args *args, struct nfs3_fh *fh);
 
 extern void
-nfs3_fill_access3res (access3res *res, nfsstat3 status, int32_t accbits,
-		      int32_t reqaccbits);
+nfs3_fill_access3res(access3res *res, nfsstat3 status, int32_t accbits,
+                     int32_t reqaccbits);
 
 extern char *
-nfs3_fhcache_getpath (struct nfs3_state *nfs3, struct nfs3_fh *fh);
+nfs3_fhcache_getpath(struct nfs3_state *nfs3, struct nfs3_fh *fh);
 
 extern int
-nfs3_fhcache_putpath (struct nfs3_state *nfs3, struct nfs3_fh *fh, char *path);
+nfs3_fhcache_putpath(struct nfs3_state *nfs3, struct nfs3_fh *fh, char *path);
 
 extern void
-nfs3_prep_readdir3args (readdir3args *ra, struct nfs3_fh *fh);
+nfs3_prep_readdir3args(readdir3args *ra, struct nfs3_fh *fh);
 
 extern void
-nfs3_fill_readdir3res (readdir3res *res, nfsstat3 stat, struct nfs3_fh *dfh,
-                       uint64_t cverf, struct iatt *dirstat,
-                       gf_dirent_t *entries, count3 count, int is_eof,
-                       uint64_t deviceid);
+nfs3_fill_readdir3res(readdir3res *res, nfsstat3 stat, struct nfs3_fh *dfh,
+                      uint64_t cverf, struct iatt *dirstat,
+                      gf_dirent_t *entries, count3 count, int is_eof,
+                      uint64_t deviceid);
 
 extern void
-nfs3_prep_readdirp3args (readdirp3args *ra, struct nfs3_fh *fh);
+nfs3_prep_readdirp3args(readdirp3args *ra, struct nfs3_fh *fh);
 
 extern void
-nfs3_fill_readdirp3res (readdirp3res *res, nfsstat3 stat,
-                        struct nfs3_fh *dirfh, uint64_t cverf,
-                        struct iatt *dirstat, gf_dirent_t *entries,
-                        count3 dircount, count3 maxcount, int is_eof,
-                        uint64_t deviceid);
+nfs3_fill_readdirp3res(readdirp3res *res, nfsstat3 stat, struct nfs3_fh *dirfh,
+                       uint64_t cverf, struct iatt *dirstat,
+                       gf_dirent_t *entries, count3 dircount, count3 maxcount,
+                       int is_eof, uint64_t deviceid);
 
 extern void
-nfs3_free_readdirp3res (readdirp3res *res);
+nfs3_free_readdirp3res(readdirp3res *res);
 
 extern void
-nfs3_free_readdir3res (readdir3res *res);
+nfs3_free_readdir3res(readdir3res *res);
 
 extern void
-nfs3_prep_fsstat3args (fsstat3args *args, struct nfs3_fh *fh);
+nfs3_prep_fsstat3args(fsstat3args *args, struct nfs3_fh *fh);
 
 extern void
-nfs3_fill_fsstat3res (fsstat3res *res, nfsstat3 stat, struct statvfs *fsbuf,
-                      struct iatt *postbuf, uint64_t deviceid);
+nfs3_fill_fsstat3res(fsstat3res *res, nfsstat3 stat, struct statvfs *fsbuf,
+                     struct iatt *postbuf, uint64_t deviceid);
 
 extern int32_t
-nfs3_sattr3_to_setattr_valid (sattr3 *sattr, struct iatt *buf, mode_t *omode);
+nfs3_sattr3_to_setattr_valid(sattr3 *sattr, struct iatt *buf, mode_t *omode);
 extern void
-nfs3_fill_create3res (create3res *res, nfsstat3 stat, struct nfs3_fh *newfh,
-                      struct iatt *newbuf, struct iatt *preparent,
-                      struct iatt *postparent, uint64_t deviceid);
+nfs3_fill_create3res(create3res *res, nfsstat3 stat, struct nfs3_fh *newfh,
+                     struct iatt *newbuf, struct iatt *preparent,
+                     struct iatt *postparent, uint64_t deviceid);
 
 extern void
-nfs3_prep_create3args (create3args *args, struct nfs3_fh *fh, char *name);
+nfs3_prep_create3args(create3args *args, struct nfs3_fh *fh, char *name);
 
 extern void
-nfs3_prep_setattr3args (setattr3args *args, struct nfs3_fh *fh);
+nfs3_prep_setattr3args(setattr3args *args, struct nfs3_fh *fh);
 
 extern void
-nfs3_fill_setattr3res (setattr3res *res, nfsstat3 stat, struct iatt *preop,
-                       struct iatt *postop, uint64_t deviceid);
+nfs3_fill_setattr3res(setattr3res *res, nfsstat3 stat, struct iatt *preop,
+                      struct iatt *postop, uint64_t deviceid);
 
 extern void
-nfs3_prep_mkdir3args (mkdir3args *args, struct nfs3_fh *dirfh, char *name);
+nfs3_prep_mkdir3args(mkdir3args *args, struct nfs3_fh *dirfh, char *name);
 
 extern void
-nfs3_fill_mkdir3res (mkdir3res *res, nfsstat3 stat, struct nfs3_fh *fh,
-                     struct iatt *buf, struct iatt *preparent,
-                     struct iatt *postparent, uint64_t deviceid);
+nfs3_fill_mkdir3res(mkdir3res *res, nfsstat3 stat, struct nfs3_fh *fh,
+                    struct iatt *buf, struct iatt *preparent,
+                    struct iatt *postparent, uint64_t deviceid);
 
 extern void
-nfs3_prep_symlink3args (symlink3args *args, struct nfs3_fh *dirfh, char *name,
-                        char *target);
+nfs3_prep_symlink3args(symlink3args *args, struct nfs3_fh *dirfh, char *name,
+                       char *target);
 
 extern void
-nfs3_fill_symlink3res (symlink3res *res, nfsstat3 stat, struct nfs3_fh *fh,
-                       struct iatt *buf, struct iatt *preparent,
-                       struct iatt *postparent, uint64_t deviceid);
+nfs3_fill_symlink3res(symlink3res *res, nfsstat3 stat, struct nfs3_fh *fh,
+                      struct iatt *buf, struct iatt *preparent,
+                      struct iatt *postparent, uint64_t deviceid);
 
 extern void
-nfs3_prep_readlink3args (readlink3args *args, struct nfs3_fh *fh);
+nfs3_prep_readlink3args(readlink3args *args, struct nfs3_fh *fh);
 
 extern void
-nfs3_fill_readlink3res (readlink3res *res, nfsstat3 stat, char *path,
-                        struct iatt *buf, uint64_t deviceid);
+nfs3_fill_readlink3res(readlink3res *res, nfsstat3 stat, char *path,
+                       struct iatt *buf, uint64_t deviceid);
 
 extern void
-nfs3_prep_mknod3args (mknod3args *args, struct nfs3_fh *fh, char *name);
+nfs3_prep_mknod3args(mknod3args *args, struct nfs3_fh *fh, char *name);
 
 extern void
-nfs3_fill_mknod3res (mknod3res *res, nfsstat3 stat, struct nfs3_fh *fh,
-                     struct iatt *buf, struct iatt *preparent,
-                     struct iatt *postparent, uint64_t deviceid);
+nfs3_fill_mknod3res(mknod3res *res, nfsstat3 stat, struct nfs3_fh *fh,
+                    struct iatt *buf, struct iatt *preparent,
+                    struct iatt *postparent, uint64_t deviceid);
 
 extern void
-nfs3_fill_remove3res (remove3res *res, nfsstat3 stat, struct iatt *preparent,
-                      struct iatt *postparent, uint64_t deviceid);
+nfs3_fill_remove3res(remove3res *res, nfsstat3 stat, struct iatt *preparent,
+                     struct iatt *postparent, uint64_t deviceid);
 extern void
-nfs3_prep_remove3args (remove3args *args, struct nfs3_fh *fh, char *name);
+nfs3_prep_remove3args(remove3args *args, struct nfs3_fh *fh, char *name);
 
 extern void
-nfs3_fill_rmdir3res (rmdir3res *res, nfsstat3 stat, struct iatt *preparent,
-                     struct iatt *postparent, uint64_t deviceid);
+nfs3_fill_rmdir3res(rmdir3res *res, nfsstat3 stat, struct iatt *preparent,
+                    struct iatt *postparent, uint64_t deviceid);
 
 extern void
-nfs3_prep_rmdir3args (rmdir3args *args, struct nfs3_fh *fh, char *name);
+nfs3_prep_rmdir3args(rmdir3args *args, struct nfs3_fh *fh, char *name);
 
 extern void
-nfs3_fill_link3res (link3res *res, nfsstat3 stat, struct iatt *buf,
-                    struct iatt *preparent, struct iatt *postparent,
-                    uint64_t deviceid);
+nfs3_fill_link3res(link3res *res, nfsstat3 stat, struct iatt *buf,
+                   struct iatt *preparent, struct iatt *postparent,
+                   uint64_t deviceid);
 
 extern void
-nfs3_prep_link3args (link3args *args, struct nfs3_fh *target,
-                     struct nfs3_fh * dirfh, char *name);
+nfs3_prep_link3args(link3args *args, struct nfs3_fh *target,
+                    struct nfs3_fh *dirfh, char *name);
 
 extern void
-nfs3_prep_rename3args (rename3args *args, struct nfs3_fh *olddirfh,
-                       char *oldname, struct nfs3_fh *newdirfh,
-                       char *newname);
+nfs3_prep_rename3args(rename3args *args, struct nfs3_fh *olddirfh,
+                      char *oldname, struct nfs3_fh *newdirfh, char *newname);
 
 extern void
-nfs3_fill_rename3res (rename3res *res, nfsstat3 stat, struct iatt *buf,
-                      struct iatt *preoldparent, struct iatt *postoldparent,
-                      struct iatt *prenewparent, struct iatt *postnewparent,
-                      uint64_t deviceid);
+nfs3_fill_rename3res(rename3res *res, nfsstat3 stat, struct iatt *buf,
+                     struct iatt *preoldparent, struct iatt *postoldparent,
+                     struct iatt *prenewparent, struct iatt *postnewparent,
+                     uint64_t deviceid);
 
 extern void
-nfs3_prep_write3args (write3args *args, struct nfs3_fh *fh);
+nfs3_prep_write3args(write3args *args, struct nfs3_fh *fh);
 
 extern void
-nfs3_fill_write3res (write3res *res, nfsstat3 stat, count3 count,
-                     stable_how stable, uint64_t wverf, struct iatt *prestat,
-                     struct iatt *poststat, uint64_t deviceid);
+nfs3_fill_write3res(write3res *res, nfsstat3 stat, count3 count,
+                    stable_how stable, uint64_t wverf, struct iatt *prestat,
+                    struct iatt *poststat, uint64_t deviceid);
 
 extern void
-nfs3_prep_commit3args (commit3args *args, struct nfs3_fh *fh);
+nfs3_prep_commit3args(commit3args *args, struct nfs3_fh *fh);
 
 extern void
-nfs3_fill_commit3res (commit3res *res, nfsstat3 stat, uint64_t wverf,
-                      struct iatt *prestat, struct iatt *poststat,
-                      uint64_t deviceid);
+nfs3_fill_commit3res(commit3res *res, nfsstat3 stat, uint64_t wverf,
+                     struct iatt *prestat, struct iatt *poststat,
+                     uint64_t deviceid);
 
 extern void
-nfs3_fill_read3res (read3res *res, nfsstat3 stat, count3 count,
-                    struct iatt *poststat, int is_eof, uint64_t deviceid);
+nfs3_fill_read3res(read3res *res, nfsstat3 stat, count3 count,
+                   struct iatt *poststat, int is_eof, uint64_t deviceid);
 
 extern void
-nfs3_prep_read3args (read3args *args, struct nfs3_fh *fh);
+nfs3_prep_read3args(read3args *args, struct nfs3_fh *fh);
 
 extern void
-nfs3_prep_pathconf3args (pathconf3args *args, struct nfs3_fh *fh);
+nfs3_prep_pathconf3args(pathconf3args *args, struct nfs3_fh *fh);
 
 extern void
-nfs3_fill_pathconf3res (pathconf3res *res, nfsstat3 stat, struct iatt *buf,
-                        uint64_t deviceid);
+nfs3_fill_pathconf3res(pathconf3res *res, nfsstat3 stat, struct iatt *buf,
+                       uint64_t deviceid);
 
 extern int
-nfs3_cached_inode_opened (xlator_t *nfsxl, inode_t *inode);
+nfs3_cached_inode_opened(xlator_t *nfsxl, inode_t *inode);
 
 extern void
-nfs3_log_common_res (uint32_t xid, int op, nfsstat3 stat, int pstat);
+nfs3_log_common_res(uint32_t xid, int op, nfsstat3 stat, int pstat,
+                    const char *path);
 
 extern void
-nfs3_log_readlink_res (uint32_t xid, nfsstat3 stat, int pstat, char *linkpath);
+nfs3_log_readlink_res(uint32_t xid, nfsstat3 stat, int pstat, char *linkpath,
+                      const char *path);
 
 extern void
-nfs3_log_read_res (uint32_t xid, nfsstat3 stat, int pstat, count3 count,
-                   int is_eof, struct iovec *vec, int32_t vcount);
+nfs3_log_read_res(uint32_t xid, nfsstat3 stat, int pstat, count3 count,
+                  int is_eof, struct iovec *vec, int32_t vcount,
+                  const char *path);
 
 extern void
-nfs3_log_write_res (uint32_t xid, nfsstat3 stat, int pstat, count3 count,
-                    int stable, uint64_t wverf);
+nfs3_log_write_res(uint32_t xid, nfsstat3 stat, int pstat, count3 count,
+                   int stable, uint64_t wverf, const char *path);
 
 extern void
-nfs3_log_newfh_res (uint32_t xid, int op, nfsstat3 stat, int pstat,
-                    struct nfs3_fh *newfh);
+nfs3_log_newfh_res(uint32_t xid, int op, nfsstat3 stat, int pstat,
+                   struct nfs3_fh *newfh, const char *path);
 
 extern void
-nfs3_log_readdir_res (uint32_t xid, nfsstat3 stat, int pstat, uint64_t cverf,
-                      count3 count, int is_eof);
+nfs3_log_readdir_res(uint32_t xid, nfsstat3 stat, int pstat, uint64_t cverf,
+                     count3 count, int is_eof, const char *path);
 
 extern void
-nfs3_log_readdirp_res (uint32_t xid, nfsstat3 stat, int pstat, uint64_t cverf,
-                       count3 dircount, count3 maxcount, int is_eof);
+nfs3_log_readdirp_res(uint32_t xid, nfsstat3 stat, int pstat, uint64_t cverf,
+                      count3 dircount, count3 maxcount, int is_eof,
+                      const char *path);
 
 extern void
-nfs3_log_commit_res (uint32_t xid, nfsstat3 stat, int pstat, uint64_t wverf);
+nfs3_log_commit_res(uint32_t xid, nfsstat3 stat, int pstat, uint64_t wverf,
+                    const char *path);
 
 extern void
-nfs3_log_common_call (uint32_t xid, char *op, struct nfs3_fh *fh);
+nfs3_log_common_call(uint32_t xid, char *op, struct nfs3_fh *fh);
 
 extern void
-nfs3_log_fh_entry_call (uint32_t xid, char *op, struct nfs3_fh *fh,
-                        char *name);
+nfs3_log_fh_entry_call(uint32_t xid, char *op, struct nfs3_fh *fh, char *name);
 
 extern void
-nfs3_log_rw_call (uint32_t xid, char *op, struct nfs3_fh *fh, offset3 offt,
-                  count3 count, int stablewrite);
+nfs3_log_rw_call(uint32_t xid, char *op, struct nfs3_fh *fh, offset3 offt,
+                 count3 count, int stablewrite);
 
 extern void
-nfs3_log_create_call (uint32_t xid, struct nfs3_fh *fh, char *name,
-                      createmode3 mode);
+nfs3_log_create_call(uint32_t xid, struct nfs3_fh *fh, char *name,
+                     createmode3 mode);
 
 extern void
-nfs3_log_symlink_call (uint32_t xid, struct nfs3_fh *fh, char *name, char *tgt);
+nfs3_log_symlink_call(uint32_t xid, struct nfs3_fh *fh, char *name, char *tgt);
 
 extern void
-nfs3_log_mknod_call (uint32_t xid, struct nfs3_fh *fh, char *name, int type);
+nfs3_log_mknod_call(uint32_t xid, struct nfs3_fh *fh, char *name, int type);
 
 extern void
-nfs3_log_rename_call (uint32_t xid, struct nfs3_fh *src, char *sname,
-                      struct nfs3_fh *dst, char *dname);
+nfs3_log_rename_call(uint32_t xid, struct nfs3_fh *src, char *sname,
+                     struct nfs3_fh *dst, char *dname);
 
 extern void
-nfs3_log_link_call (uint32_t xid, struct nfs3_fh *fh, char *name,
-                    struct nfs3_fh *tgt);
+nfs3_log_link_call(uint32_t xid, struct nfs3_fh *fh, char *name,
+                   struct nfs3_fh *tgt);
 
 extern void
-nfs3_log_readdir_call (uint32_t xid, struct nfs3_fh *fh, count3 dircount,
-                       count3 maxcount);
+nfs3_log_readdir_call(uint32_t xid, struct nfs3_fh *fh, count3 dircount,
+                      count3 maxcount);
 
 extern int
-nfs3_fh_resolve_entry_hard (nfs3_call_state_t *cs);
+nfs3_fh_resolve_entry_hard(nfs3_call_state_t *cs);
 
 extern int
-nfs3_fh_resolve_inode (nfs3_call_state_t *cs);
+nfs3_fh_resolve_inode(nfs3_call_state_t *cs);
 
 extern int
-nfs3_fh_resolve_entry (nfs3_call_state_t *cs);
+nfs3_fh_resolve_entry(nfs3_call_state_t *cs);
 
 extern int
-nfs3_fh_resolve_and_resume (nfs3_call_state_t *cs, struct nfs3_fh *fh,
-                            char *entry, nfs3_resume_fn_t resum_fn);
+nfs3_fh_resolve_and_resume(nfs3_call_state_t *cs, struct nfs3_fh *fh,
+                           char *entry, nfs3_resume_fn_t resum_fn);
 
 extern int
-nfs3_verify_dircookie (struct nfs3_state *nfs3, fd_t *dirfd, cookie3 cookie,
-                       uint64_t cverf, nfsstat3 *stat);
+nfs3_verify_dircookie(struct nfs3_state *nfs3, fd_t *dirfd, cookie3 cookie,
+                      uint64_t cverf, nfsstat3 *stat);
 
 extern int
-nfs3_is_parentdir_entry (char *entry);
+nfs3_is_parentdir_entry(char *entry);
 
 uint32_t
-nfs3_request_to_accessbits (int32_t accbits);
+nfs3_request_to_accessbits(int32_t accbits);
 
 extern int
-nfs3_fh_auth_nfsop (nfs3_call_state_t *cs, gf_boolean_t is_write_op);
+nfs3_fh_auth_nfsop(nfs3_call_state_t *cs, gf_boolean_t is_write_op);
 
 void
-nfs3_map_deviceid_to_statdev (struct iatt *ia, uint64_t deviceid);
+nfs3_map_deviceid_to_statdev(struct iatt *ia, uint64_t deviceid);
 
 #endif
diff --git a/xlators/nfs/server/src/nfs3.c b/xlators/nfs/server/src/nfs3.c
index 1a94c00ce8c..f9042bc3b3f 100644
--- a/xlators/nfs/server/src/nfs3.c
+++ b/xlators/nfs/server/src/nfs3.c
@@ -8,21 +8,16 @@
   cases as published by the Free Software Foundation.
 */
 
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
-
 #include "rpcsvc.h"
-#include "dict.h"
-#include "xlator.h"
+#include <glusterfs/dict.h>
+#include <glusterfs/xlator.h>
 #include "mount3.h"
 #include "xdr-nfs3.h"
 #include "msg-nfs3.h"
-#include "iobuf.h"
+#include <glusterfs/iobuf.h>
 #include "nfs3.h"
-#include "mem-pool.h"
-#include "logging.h"
+#include <glusterfs/mem-pool.h>
+#include <glusterfs/logging.h>
 #include "nfs-common.h"
 #include "nfs-fops.h"
 #include "nfs-inodes.h"
@@ -32,2052 +27,2151 @@
 #include "nfs.h"
 #include "xdr-rpc.h"
 #include "xdr-generic.h"
+#include "nfs-messages.h"
+#include "glfs-internal.h"
 
 #include <sys/socket.h>
 #include <sys/uio.h>
 #include <sys/statvfs.h>
 #include <time.h>
 
-#define nfs3_validate_strlen_or_goto(str, len, label, status, retval)   \
-        do      {                                                       \
-                if ((str)) {                                            \
-                        if (strlen ((str)) > (len)) {                   \
-                                gf_log (GF_NFS3, GF_LOG_ERROR, "strlen "\
-                                        "too long");                    \
-                                status = NFS3ERR_NAMETOOLONG;           \
-                                retval = -ENAMETOOLONG;                 \
-                                goto label;                             \
-                        }                                               \
-                }                                                       \
-        } while (0);                                                    \
-
-#define nfs3_validate_nfs3_state(request, state, status, label, retval) \
-        do      {                                                       \
-                state = rpcsvc_request_program_private (request);       \
-                if (!state) {                                            \
-                        gf_log (GF_NFS3, GF_LOG_ERROR, "NFSv3 state "   \
-                                "missing from RPC request");            \
-                        status = NFS3ERR_SERVERFAULT;                   \
-                        ret = -EFAULT;                                  \
-                        goto label;                                     \
-                }                                                       \
-        } while (0);                                                    \
-
+#define nfs3_validate_strlen_or_goto(str, len, label, status, retval)          \
+    do {                                                                       \
+        if ((str)) {                                                           \
+            if (strlen((str)) > (len)) {                                       \
+                gf_msg(GF_NFS3, GF_LOG_ERROR, ENAMETOOLONG,                    \
+                       NFS_MSG_STR_TOO_LONG, "strlen too long");               \
+                status = NFS3ERR_NAMETOOLONG;                                  \
+                retval = -ENAMETOOLONG;                                        \
+                goto label;                                                    \
+            }                                                                  \
+        }                                                                      \
+    } while (0);
+
+#define nfs3_validate_nfs3_state(request, state, status, label, retval)        \
+    do {                                                                       \
+        state = rpcsvc_request_program_private(request);                       \
+        if (!state) {                                                          \
+            gf_msg(GF_NFS3, GF_LOG_ERROR, EFAULT, NFS_MSG_STATE_MISSING,       \
+                   "NFSv3 state "                                              \
+                   "missing from RPC request");                                \
+            status = NFS3ERR_SERVERFAULT;                                      \
+            ret = -EFAULT;                                                     \
+            goto label;                                                        \
+        }                                                                      \
+    } while (0);
 
 struct nfs3_export *
-__nfs3_get_export_by_index (struct nfs3_state *nfs3, uuid_t exportid)
+__nfs3_get_export_by_index(struct nfs3_state *nfs3, uuid_t exportid)
 {
-        struct nfs3_export      *exp = NULL;
-        int                     index = 0;
-        int                     searchindex = 0;
+    struct nfs3_export *exp = NULL;
+    int index = 0;
+    int searchindex = 0;
 
-        searchindex = nfs3_fh_exportid_to_index (exportid);
-        list_for_each_entry (exp, &nfs3->exports, explist) {
-                if (searchindex == index)
-                        goto found;
+    searchindex = nfs3_fh_exportid_to_index(exportid);
+    list_for_each_entry(exp, &nfs3->exports, explist)
+    {
+        if (searchindex == index)
+            goto found;
 
-                ++index;
-        }
+        ++index;
+    }
 
-        exp = NULL;
-        gf_log (GF_NFS, GF_LOG_ERROR, "searchindex=%d not found", searchindex);
+    exp = NULL;
+    gf_msg(GF_NFS3, GF_LOG_ERROR, 0, NFS_MSG_INDEX_NOT_FOUND,
+           "searchindex=%d not found", searchindex);
 found:
-        return exp;
+    return exp;
 }
 
-
 struct nfs3_export *
-__nfs3_get_export_by_volumeid (struct nfs3_state *nfs3, uuid_t exportid)
+__nfs3_get_export_by_volumeid(struct nfs3_state *nfs3, uuid_t exportid)
 {
-        struct nfs3_export      *exp = NULL;
+    struct nfs3_export *exp = NULL;
 
-        list_for_each_entry (exp, &nfs3->exports, explist) {
-                if (!gf_uuid_compare (exportid, exp->volumeid))
-                        goto found;
-        }
+    list_for_each_entry(exp, &nfs3->exports, explist)
+    {
+        if (!gf_uuid_compare(exportid, exp->volumeid))
+            goto found;
+    }
 
-        exp = NULL;
+    exp = NULL;
 found:
-        return exp;
+    return exp;
 }
 
-
 struct nfs3_export *
-__nfs3_get_export_by_exportid (struct nfs3_state *nfs3, uuid_t exportid)
+__nfs3_get_export_by_exportid(struct nfs3_state *nfs3, uuid_t exportid)
 {
-        struct nfs3_export      *exp = NULL;
+    struct nfs3_export *exp = NULL;
 
-        if (!nfs3)
-                return exp;
+    if (!nfs3)
+        return exp;
 
-        if (gf_nfs_dvm_off (nfs_state(nfs3->nfsx)))
-                exp = __nfs3_get_export_by_index (nfs3, exportid);
-        else
-                exp = __nfs3_get_export_by_volumeid (nfs3, exportid);
+    if (gf_nfs_dvm_off(nfs_state(nfs3->nfsx)))
+        exp = __nfs3_get_export_by_index(nfs3, exportid);
+    else
+        exp = __nfs3_get_export_by_volumeid(nfs3, exportid);
 
-        return exp;
+    return exp;
 }
 
-
 int
-nfs3_export_access (struct nfs3_state *nfs3, uuid_t exportid)
+nfs3_export_access(struct nfs3_state *nfs3, uuid_t exportid)
 {
-        int                     ret = GF_NFS3_VOLACCESS_RO;
-        struct nfs3_export      *exp = NULL;
+    int ret = GF_NFS3_VOLACCESS_RO;
+    struct nfs3_export *exp = NULL;
 
-        GF_VALIDATE_OR_GOTO (GF_NFS3, nfs3, err);
+    GF_VALIDATE_OR_GOTO(GF_NFS3, nfs3, err);
 
-        exp = __nfs3_get_export_by_exportid (nfs3, exportid);
+    exp = __nfs3_get_export_by_exportid(nfs3, exportid);
 
-        if (!exp) {
-                gf_log (GF_NFS3, GF_LOG_ERROR, "Failed to get export by ID");
-                goto err;
-        }
+    if (!exp) {
+        gf_msg(GF_NFS3, GF_LOG_ERROR, 0, NFS_MSG_EXPORT_ID_FAIL,
+               "Failed to get export by ID");
+        goto err;
+    }
 
-        ret = exp->access;
+    ret = exp->access;
 
 err:
-        return ret;
+    return ret;
 }
 
-#define nfs3_check_rw_volaccess(nfs3state, exid, status, label)         \
-        do {                                                            \
-                if (nfs3_export_access (nfs3state,exid)!=GF_NFS3_VOLACCESS_RW){\
-                        gf_log (GF_NFS3, GF_LOG_ERROR, "No read-write access");\
-                        status = NFS3ERR_ROFS;                          \
-                        goto label;                                     \
-                }                                                       \
-        } while (0)                                                     \
-
-
+#define nfs3_check_rw_volaccess(nfs3state, exid, status, label)                \
+    do {                                                                       \
+        if (nfs3_export_access(nfs3state, exid) != GF_NFS3_VOLACCESS_RW) {     \
+            gf_msg(GF_NFS3, GF_LOG_ERROR, EACCES, NFS_MSG_NO_RW_ACCESS,        \
+                   "No read-write access");                                    \
+            status = NFS3ERR_ROFS;                                             \
+            goto label;                                                        \
+        }                                                                      \
+    } while (0)
 
 xlator_t *
-nfs3_fh_to_xlator (struct nfs3_state *nfs3, struct nfs3_fh *fh)
+nfs3_fh_to_xlator(struct nfs3_state *nfs3, struct nfs3_fh *fh)
 {
-        xlator_t                *vol = NULL;
-        struct nfs3_export      *exp = NULL;
+    xlator_t *vol = NULL;
+    struct nfs3_export *exp = NULL;
 
-        GF_VALIDATE_OR_GOTO (GF_NFS3, nfs3, out);
-        GF_VALIDATE_OR_GOTO (GF_NFS3, fh, out);
+    GF_VALIDATE_OR_GOTO(GF_NFS3, nfs3, out);
+    GF_VALIDATE_OR_GOTO(GF_NFS3, fh, out);
 
-        exp = __nfs3_get_export_by_exportid (nfs3, fh->exportid);
-        if (!exp)
-                goto out;
+    exp = __nfs3_get_export_by_exportid(nfs3, fh->exportid);
+    if (!exp)
+        goto out;
 
-        vol = exp->subvol;
+    vol = exp->subvol;
 out:
-        return vol;
+    return vol;
 }
 
-
 int
-nfs3_is_root_looked_up (struct nfs3_state *nfs3, struct nfs3_fh *rootfh)
+nfs3_is_root_looked_up(struct nfs3_state *nfs3, struct nfs3_fh *rootfh)
 {
-        struct nfs3_export      *exp = NULL;
-        int                     ret = 0;
+    struct nfs3_export *exp = NULL;
+    int ret = 0;
 
-        GF_VALIDATE_OR_GOTO (GF_NFS3, nfs3, out);
-        GF_VALIDATE_OR_GOTO (GF_NFS3, rootfh, out);
+    GF_VALIDATE_OR_GOTO(GF_NFS3, nfs3, out);
+    GF_VALIDATE_OR_GOTO(GF_NFS3, rootfh, out);
 
-        exp = __nfs3_get_export_by_exportid (nfs3, rootfh->exportid);
-        if (!exp)
-                goto out;
+    exp = __nfs3_get_export_by_exportid(nfs3, rootfh->exportid);
+    if (!exp)
+        goto out;
 
-        ret = exp->rootlookedup;
+    ret = exp->rootlookedup;
 out:
-        return ret;
+    return ret;
 }
 
-
 int
-nfs3_set_root_looked_up (struct nfs3_state *nfs3, struct nfs3_fh *rootfh)
+nfs3_set_root_looked_up(struct nfs3_state *nfs3, struct nfs3_fh *rootfh)
 {
-        struct nfs3_export      *exp = NULL;
-        int                     ret = 0;
+    struct nfs3_export *exp = NULL;
+    int ret = 0;
 
-        GF_VALIDATE_OR_GOTO (GF_NFS3, nfs3, out);
-        GF_VALIDATE_OR_GOTO (GF_NFS3, rootfh, out);
+    GF_VALIDATE_OR_GOTO(GF_NFS3, nfs3, out);
+    GF_VALIDATE_OR_GOTO(GF_NFS3, rootfh, out);
 
-        exp = __nfs3_get_export_by_exportid (nfs3, rootfh->exportid);
-        if (!exp)
-                goto out;
+    exp = __nfs3_get_export_by_exportid(nfs3, rootfh->exportid);
+    if (!exp)
+        goto out;
 
-        exp->rootlookedup = 1;
+    exp->rootlookedup = 1;
 out:
-        return ret;
-}
-
-
-#define nfs3_map_fh_to_volume(nfs3state, handle, req, volume, status, label) \
-        do {                                                            \
-                char exportid[256], gfid[256];                          \
-                rpc_transport_t *trans = NULL;                          \
-                volume = nfs3_fh_to_xlator ((nfs3state), handle);       \
-                if (!volume) {                                          \
-                        gf_uuid_unparse (handle->exportid, exportid);      \
-                        gf_uuid_unparse (handle->gfid, gfid);              \
-                        trans = rpcsvc_request_transport (req);         \
-                        GF_LOG_OCCASIONALLY (nfs3state->occ_logger,     \
-                                GF_NFS3, GF_LOG_ERROR, "Failed to map " \
-                                "FH to vol: client=%s, exportid=%s, "   \
-                                "gfid=%s", trans->peerinfo.identifier,  \
-                                exportid, gfid);                        \
-                        GF_LOG_OCCASIONALLY (nfs3state->occ_logger,     \
-                                GF_NFS3, GF_LOG_ERROR, "Stale nfs "     \
-                                "client %s must be trying to connect to"\
-                                " a deleted volume, please unmount it.",\
-                                trans->peerinfo.identifier);            \
-                        status = NFS3ERR_STALE;                         \
-                        goto label;                                     \
-                } else {                                                \
-                        gf_log (GF_NFS3, GF_LOG_TRACE, "FH to Volume:"  \
-                                "%s", volume->name);                    \
-                        rpcsvc_request_set_private (req, volume);       \
-                }                                                       \
-        } while (0);                                                    \
-
-
-#define nfs3_validate_gluster_fh(handle, status, errlabel)              \
-        do {                                                            \
-                if (!nfs3_fh_validate (handle)) {                       \
-                        gf_log (GF_NFS3, GF_LOG_ERROR, "Bad Handle");   \
-                        status = NFS3ERR_BADHANDLE;                     \
-                        goto errlabel;                                  \
-                }                                                       \
-        } while (0)                                                     \
-
-
-#define nfs3_check_fh_auth_status(cst, nfstat, is_write_op, erlabl)     \
-        do {                                                            \
-                xlator_t *xlatorp = NULL;                               \
-                char buf[256], gfid[256];                               \
-                rpc_transport_t *trans = NULL;                          \
-                cst->resolve_ret = cst->resolve_errno =                 \
-                        nfs3_fh_auth_nfsop (cst, is_write_op);          \
-                if ((cst)->resolve_ret < 0) {                           \
-                        trans = rpcsvc_request_transport (cst->req);    \
-                        xlatorp = nfs3_fh_to_xlator (cst->nfs3state,    \
-                                                     &cst->resolvefh);  \
-                        gf_uuid_unparse (cst->resolvefh.gfid, gfid);       \
-                        sprintf (buf, "(%s) %s : %s",                   \
-                                 trans->peerinfo.identifier,            \
-                        xlatorp ? xlatorp->name : "ERR", gfid);         \
-                        gf_log (GF_NFS3, GF_LOG_ERROR, "Unable to resolve FH"\
-                                ": %s", buf);                           \
-                        nfstat = nfs3_errno_to_nfsstat3 (-cst->resolve_errno);\
-                        goto erlabl;                                    \
-                }                                                       \
-        } while (0)                                                     \
-
-#define nfs3_check_fh_resolve_status(cst, nfstat, erlabl)               \
-        do {                                                            \
-                xlator_t *xlatorp = NULL;                               \
-                char buf[256], gfid[256];                               \
-                rpc_transport_t *trans = NULL;                          \
-                if ((cst)->resolve_ret < 0) {                           \
-                        trans = rpcsvc_request_transport (cst->req);    \
-                        xlatorp = nfs3_fh_to_xlator (cst->nfs3state,    \
-                                                     &cst->resolvefh);  \
-                        gf_uuid_unparse (cst->resolvefh.gfid, gfid);       \
-                        snprintf (buf, sizeof (buf), "(%s) %s : %s",             \
-                                  trans->peerinfo.identifier,           \
-                                  xlatorp ? xlatorp->name : "ERR",      \
-                                  gfid );                                \
-                        gf_log (GF_NFS3, GF_LOG_ERROR, "%s: %s",        \
-                                strerror(cst->resolve_errno), buf);     \
-                        nfstat = nfs3_errno_to_nfsstat3 (cst->resolve_errno);\
-                        goto erlabl;                                    \
-                }                                                       \
-        } while (0)                                                     \
-
-#define nfs3_check_new_fh_resolve_status(cst, nfstat, erlabl)           \
-        do {                                                            \
-                xlator_t *xlatorp = NULL;                               \
-                char buf[256], gfid[256];                               \
-                rpc_transport_t *trans = NULL;                          \
-                if (((cst)->resolve_ret < 0) &&                         \
-                    ((cst)->resolve_errno != ENOENT)) {                 \
-                        trans = rpcsvc_request_transport (cst->req);    \
-                        xlatorp = nfs3_fh_to_xlator (cst->nfs3state,    \
-                                                     &cst->resolvefh);  \
-                        gf_uuid_unparse (cst->resolvefh.gfid, gfid);       \
-                        snprintf (buf, sizeof (buf), "(%s) %s : %s",             \
-                                  trans->peerinfo.identifier,     \
-                                  xlatorp ? xlatorp->name : "ERR",      \
-                                  gfid);         \
-                        gf_log (GF_NFS3, GF_LOG_ERROR, "%s: %s",        \
-                                strerror(cst->resolve_errno), buf);     \
-                        nfstat = nfs3_errno_to_nfsstat3 (cs->resolve_errno);\
-                        goto erlabl;                                    \
-                }                                                       \
-        } while (0)                                                     \
-
+    return ret;
+}
+
+#define nfs3_map_fh_to_volume(nfs3state, handle, req, volume, status, label)   \
+    do {                                                                       \
+        char exportid[256], gfid[256];                                         \
+        rpc_transport_t *trans = NULL;                                         \
+        volume = nfs3_fh_to_xlator((nfs3state), handle);                       \
+        if (!volume) {                                                         \
+            gf_uuid_unparse(handle->exportid, exportid);                       \
+            gf_uuid_unparse(handle->gfid, gfid);                               \
+            trans = rpcsvc_request_transport(req);                             \
+            GF_LOG_OCCASIONALLY(nfs3state->occ_logger, GF_NFS3, GF_LOG_ERROR,  \
+                                "Failed to map "                               \
+                                "FH to vol: client=%s, exportid=%s, "          \
+                                "gfid=%s",                                     \
+                                trans->peerinfo.identifier, exportid, gfid);   \
+            GF_LOG_OCCASIONALLY(nfs3state->occ_logger, GF_NFS3, GF_LOG_ERROR,  \
+                                "Stale nfs "                                   \
+                                "client %s must be trying to connect to"       \
+                                " a deleted volume, please unmount it.",       \
+                                trans->peerinfo.identifier);                   \
+            status = NFS3ERR_STALE;                                            \
+            goto label;                                                        \
+        } else {                                                               \
+            gf_msg_trace(GF_NFS3, 0,                                           \
+                         "FH to Volume:"                                       \
+                         "%s",                                                 \
+                         volume->name);                                        \
+            rpcsvc_request_set_private(req, volume);                           \
+        }                                                                      \
+    } while (0);
+
+#define nfs3_validate_gluster_fh(handle, status, errlabel)                     \
+    do {                                                                       \
+        if (!nfs3_fh_validate(handle)) {                                       \
+            gf_msg(GF_NFS3, GF_LOG_ERROR, 0, NFS_MSG_BAD_HANDLE,               \
+                   "Bad Handle");                                              \
+            status = NFS3ERR_BADHANDLE;                                        \
+            goto errlabel;                                                     \
+        }                                                                      \
+    } while (0)
+
+#define nfs3_check_fh_auth_status(cst, nfstat, is_write_op, erlabl)            \
+    do {                                                                       \
+        int auth_ret = 0;                                                      \
+        int auth_errno = 0;                                                    \
+        xlator_t *xlatorp = NULL;                                              \
+        char buf[256], gfid[GF_UUID_BUF_SIZE];                                 \
+        rpc_transport_t *trans = NULL;                                         \
+                                                                               \
+        auth_ret = auth_errno = nfs3_fh_auth_nfsop(cst, is_write_op);          \
+        if (auth_ret < 0) {                                                    \
+            trans = rpcsvc_request_transport(cst->req);                        \
+            xlatorp = nfs3_fh_to_xlator(cst->nfs3state, &cst->resolvefh);      \
+            gf_uuid_unparse(cst->resolvefh.gfid, gfid);                        \
+            sprintf(buf, "(%s) %s : %s", trans->peerinfo.identifier,           \
+                    xlatorp ? xlatorp->name : "ERR", gfid);                    \
+            gf_msg(GF_NFS3, GF_LOG_ERROR, 0, NFS_MSG_RESOLVE_FH_FAIL,          \
+                   "Unable to "                                                \
+                   "resolve FH: %s",                                           \
+                   buf);                                                       \
+            nfstat = nfs3_errno_to_nfsstat3(-auth_errno);                      \
+            goto erlabl;                                                       \
+        }                                                                      \
+    } while (0)
+
+#define nfs3_check_fh_resolve_status(cst, nfstat, erlabl)                      \
+    do {                                                                       \
+        xlator_t *xlatorp = NULL;                                              \
+        char buf[256], gfid[GF_UUID_BUF_SIZE];                                 \
+        rpc_transport_t *trans = NULL;                                         \
+        if ((cst)->resolve_ret < 0) {                                          \
+            trans = rpcsvc_request_transport(cst->req);                        \
+            xlatorp = nfs3_fh_to_xlator(cst->nfs3state, &cst->resolvefh);      \
+            gf_uuid_unparse(cst->resolvefh.gfid, gfid);                        \
+            snprintf(buf, sizeof(buf), "(%s) %s : %s",                         \
+                     trans->peerinfo.identifier,                               \
+                     xlatorp ? xlatorp->name : "ERR", gfid);                   \
+            gf_msg(GF_NFS3, GF_LOG_ERROR, 0, NFS_MSG_RESOLVE_STAT, "%s: %s",   \
+                   strerror(cst->resolve_errno), buf);                         \
+            nfstat = nfs3_errno_to_nfsstat3(cst->resolve_errno);               \
+            goto erlabl;                                                       \
+        }                                                                      \
+    } while (0)
+
+#define nfs3_check_new_fh_resolve_status(cst, nfstat, erlabl)                  \
+    do {                                                                       \
+        xlator_t *xlatorp = NULL;                                              \
+        char buf[256], gfid[GF_UUID_BUF_SIZE];                                 \
+        rpc_transport_t *trans = NULL;                                         \
+        if (((cst)->resolve_ret < 0) && ((cst)->resolve_errno != ENOENT)) {    \
+            trans = rpcsvc_request_transport(cst->req);                        \
+            xlatorp = nfs3_fh_to_xlator(cst->nfs3state, &cst->resolvefh);      \
+            gf_uuid_unparse(cst->resolvefh.gfid, gfid);                        \
+            snprintf(buf, sizeof(buf), "(%s) %s : %s",                         \
+                     trans->peerinfo.identifier,                               \
+                     xlatorp ? xlatorp->name : "ERR", gfid);                   \
+            gf_msg(GF_NFS3, GF_LOG_ERROR, 0, NFS_MSG_RESOLVE_STAT, "%s: %s",   \
+                   strerror(cst->resolve_errno), buf);                         \
+            nfstat = nfs3_errno_to_nfsstat3(cs->resolve_errno);                \
+            goto erlabl;                                                       \
+        }                                                                      \
+    } while (0)
 
 int
-__nfs3_get_volume_id (struct nfs3_state *nfs3, xlator_t *xl,
-                      uuid_t volumeid)
+__nfs3_get_volume_id(struct nfs3_state *nfs3, xlator_t *xl, uuid_t volumeid)
 {
-        int                     ret = -1;
-        struct nfs3_export      *exp = NULL;
+    int ret = -1;
+    struct nfs3_export *exp = NULL;
 
-        GF_VALIDATE_OR_GOTO (GF_NFS3, nfs3, out);
-        GF_VALIDATE_OR_GOTO (GF_NFS3, xl, out);
+    GF_VALIDATE_OR_GOTO(GF_NFS3, nfs3, out);
+    GF_VALIDATE_OR_GOTO(GF_NFS3, xl, out);
 
-        list_for_each_entry (exp, &nfs3->exports, explist) {
-                if (exp->subvol == xl) {
-                        gf_uuid_copy (volumeid, exp->volumeid);
-                        ret = 0;
-                        goto out;
-                }
+    list_for_each_entry(exp, &nfs3->exports, explist)
+    {
+        if (exp->subvol == xl) {
+            gf_uuid_copy(volumeid, exp->volumeid);
+            ret = 0;
+            goto out;
         }
+    }
 
 out:
-        return ret;
+    return ret;
+}
+
+static int
+nfs3_funge_webnfs_zerolen_fh(rpcsvc_request_t *req, struct nfs3_state *nfs3st,
+                             struct nfs3_fh *fhd, char *name)
+{
+    xlator_t *fungexl = NULL;
+    struct nfs_state *nfs = NULL;
+    glfs_t *fs = NULL;
+    loc_t loc = {
+        0,
+    };
+    int ret = -1;
+    char *subdir = NULL;
+    char volname[NAME_MAX] = {
+        0,
+    };
+
+    fungexl = nfs_mntpath_to_xlator(nfs3st->exportslist, name);
+    if (!fungexl) {
+        gf_msg_trace(GF_NFS3, 0, "failed to find xlator for volume");
+        ret = -ENOENT;
+        goto out;
+    }
+    /* fungexl is valid, set for nfs3_request_xlator_deviceid() */
+    rpcsvc_request_set_private(req, fungexl);
+
+    /* Permission checks are done through mnt3_parse_dir_exports(). The
+     * "nfs.export-dir" option gets checked as well. */
+    nfs = nfs_state(nfs3st->nfsx);
+    ret = mnt3_parse_dir_exports(req, nfs->mstate, name, _gf_false);
+    if (ret) {
+        gf_msg_trace(GF_NFS3, -ret, "mounting not possible");
+        goto out;
+    }
+
+    /* glfs_resolve_at copied from UDP MNT support */
+    fs = glfs_new_from_ctx(fungexl->ctx);
+    if (!fs) {
+        gf_msg_trace(GF_NFS3, 0, "failed to create glfs instance");
+        ret = -ENOENT;
+        goto out;
+    }
+
+    /* split name "volname/sub/dir/s" into pieces */
+    subdir = mnt3_get_volume_subdir(name, (char **)&volname);
+
+    ret = glfs_resolve_at(fs, fungexl, NULL, subdir, &loc, NULL, 1, 0);
+    if (ret != 0) {
+        gf_msg_trace(GF_NFS3, 0, "failed to resolve %s", subdir);
+        ret = -ENOENT;
+        goto out;
+    }
+
+    /* resolved subdir, copy gfid for the fh */
+    gf_uuid_copy(fhd->gfid, loc.gfid);
+    loc_wipe(&loc);
+
+    if (gf_nfs_dvm_off(nfs_state(nfs3st->nfsx)))
+        fhd->exportid[15] = nfs_xlator_to_xlid(nfs3st->exportslist, fungexl);
+    else {
+        if (__nfs3_get_volume_id(nfs3st, fungexl, fhd->exportid) < 0) {
+            ret = -ESTALE;
+            goto out;
+        }
+    }
+
+    ret = 0;
+out:
+    if (fs)
+        glfs_free_from_ctx(fs);
+
+    return ret;
 }
 
+/*
+ * This macro checks if the volume is started or not.
+ * If it is not started, it closes the client connection & logs it.
+ *
+ * Why do we do this?
+ *
+ * There is a "race condition" where gNFSd may start listening for RPC requests
+ * prior to the volume being started. Presumably, that is why this macro exists
+ * in the first place. In the NFS kernel client (specifically Linux's NFS
+ * kernel client), they establish a TCP connection to our endpoint and
+ * (re-)send requests. If we ignore the request, and return nothing back,
+ * the NFS kernel client waits forever for our response. If for some reason,
+ * the TCP connection were to die, and re-establish, the requests are
+ * retransmitted and everything begins working as expected
+ *
+ * Now, this is clearly bad behavior on the client side,
+ * but in order to make every user's life easier,
+ * gNFSd should simply disconnect the TCP connection if it sees requests
+ * before it is ready to accept them.
+ *
+ */
 
-#define nfs3_funge_solaris_zerolen_fh(nfs3st, fhd, enam, nfsst, erl)    \
-        do {                                                            \
-                xlator_t        *fungexl = NULL;                        \
-                uuid_t          zero = {0, };                           \
-                fungexl =nfs_mntpath_to_xlator ((nfs3st)->exportslist,enam);\
-                if (!fungexl) {                                         \
-                        (nfsst) = NFS3ERR_NOENT;                        \
-                        goto erl;                                       \
-                }                                                       \
-                                                                        \
-                gf_uuid_copy ((fhd)->gfid, zero);                          \
-                (fhd)->gfid[15] = 1;                                    \
-                (enam) = NULL;                                          \
-                if ((gf_nfs_dvm_off (nfs_state (nfs3st->nfsx))))        \
-                        (fhd)->exportid[15] = nfs_xlator_to_xlid ((nfs3st)->exportslist, fungexl);                                                 \
-                else {                                                  \
-                        if(__nfs3_get_volume_id ((nfs3st), fungexl, (fhd)->exportid) < 0) { \
-                                (nfsst) = NFS3ERR_STALE;                \
-                                goto erl;                               \
-                        }                                               \
-                }                                                       \
-        } while (0)                                                     \
-
-
-#define nfs3_volume_started_check(nf3stt, vlm, rtval, erlbl)            \
-        do {                                                            \
-              if ((!nfs_subvolume_started (nfs_state (nf3stt->nfsx), vlm))){\
-                      gf_log (GF_NFS3, GF_LOG_ERROR, "Volume is disabled: %s",\
-                              vlm->name);                               \
-                      rtval = RPCSVC_ACTOR_IGNORE;                      \
-                      goto erlbl;                                       \
-              }                                                         \
-        } while (0)                                                     \
+#define nfs3_volume_started_check(nf3stt, vlm, rtval, erlbl)                   \
+    do {                                                                       \
+        if ((!nfs_subvolume_started(nfs_state(nf3stt->nfsx), vlm))) {          \
+            gf_msg(GF_NFS3, GF_LOG_ERROR, 0, NFS_MSG_VOL_DISABLE,              \
+                   "Volume is disabled: %s", vlm->name);                       \
+            nfs3_disconnect_transport(req->trans);                             \
+            rtval = RPCSVC_ACTOR_IGNORE;                                       \
+            goto erlbl;                                                        \
+        }                                                                      \
+    } while (0)
 
+void
+nfs3_disconnect_transport(rpc_transport_t *transport)
+{
+    int ret = 0;
+
+    GF_VALIDATE_OR_GOTO(GF_NFS3, transport, out);
+
+    ret = rpc_transport_disconnect(transport, _gf_false);
+    if (ret != 0) {
+        gf_log(GF_NFS3, GF_LOG_WARNING,
+               "Unable to close client connection to %s.",
+               transport->peerinfo.identifier);
+    } else {
+        gf_log(GF_NFS3, GF_LOG_WARNING, "Closed client connection to %s.",
+               transport->peerinfo.identifier);
+    }
+out:
+    return;
+}
 
 int
-nfs3_export_sync_trusted (struct nfs3_state *nfs3, uuid_t exportid)
+nfs3_export_sync_trusted(struct nfs3_state *nfs3, uuid_t exportid)
 {
-        struct nfs3_export      *exp = NULL;
-        int                     ret = 0;
+    struct nfs3_export *exp = NULL;
+    int ret = 0;
 
-        GF_VALIDATE_OR_GOTO (GF_NFS3, nfs3, err);
+    GF_VALIDATE_OR_GOTO(GF_NFS3, nfs3, err);
 
-        exp = __nfs3_get_export_by_exportid (nfs3, exportid);
-        if (!exp)
-                goto err;
+    exp = __nfs3_get_export_by_exportid(nfs3, exportid);
+    if (!exp)
+        goto err;
 
-        ret = exp->trusted_sync;
+    ret = exp->trusted_sync;
 err:
-        return ret;
+    return ret;
 }
 
-
 int
-nfs3_export_write_trusted (struct nfs3_state *nfs3, uuid_t exportid)
+nfs3_export_write_trusted(struct nfs3_state *nfs3, uuid_t exportid)
 {
-        struct nfs3_export      *exp = NULL;
-        int                     ret = 0;
+    struct nfs3_export *exp = NULL;
+    int ret = 0;
 
-        GF_VALIDATE_OR_GOTO (GF_NFS3, nfs3, err);
+    GF_VALIDATE_OR_GOTO(GF_NFS3, nfs3, err);
 
-        exp = __nfs3_get_export_by_exportid (nfs3, exportid);
-        if (!exp)
-                goto err;
+    exp = __nfs3_get_export_by_exportid(nfs3, exportid);
+    if (!exp)
+        goto err;
 
-        ret = exp->trusted_write;
+    ret = exp->trusted_write;
 err:
-        return ret;
+    return ret;
 }
 
 int
-nfs3_solaris_zerolen_fh (struct nfs3_fh *fh, int fhlen)
+nfs3_solaris_zerolen_fh(struct nfs3_fh *fh, int fhlen)
 {
-        if (!fh)
-                return 0;
+    if (!fh)
+        return 0;
 
-        if (nfs3_fh_validate (fh))
-                return 0;
+    if (nfs3_fh_validate(fh))
+        return 0;
 
-        if (fhlen == 0)
-                return 1;
+    if (fhlen == 0) {
+        gf_msg_trace(GF_NFS3, 0, "received WebNFS request");
+        return 1;
+    }
 
-        return 0;
+    return 0;
 }
 
-
 /* Function pointer that represents the generic prototypes of functions used
  * to serialize NFS3 message structures into the XDR format.
  * For usage, see the nfs3svc_XXX_cbk functions.
  */
-typedef ssize_t (*nfs3_serializer) (struct iovec outmsg, void *args);
+typedef ssize_t (*nfs3_serializer)(struct iovec outmsg, void *args);
 
-nfs3_call_state_t *
-nfs3_call_state_init (struct nfs3_state *s, rpcsvc_request_t *req, xlator_t *v)
+static void
+__nfs3_call_state_wipe(nfs3_call_state_t *cs)
 {
-        nfs3_call_state_t       *cs = NULL;
+    if (cs->fd) {
+        gf_msg_trace(GF_NFS3, 0, "fd 0x%lx ref: %" PRId64, (long)cs->fd,
+                     GF_ATOMIC_GET(cs->fd->refcount));
+        fd_unref(cs->fd);
+    }
 
-        GF_VALIDATE_OR_GOTO (GF_NFS3, s, err);
-        GF_VALIDATE_OR_GOTO (GF_NFS3, req, err);
-        GF_VALIDATE_OR_GOTO (GF_NFS3, v, err);
+    GF_FREE(cs->resolventry);
 
-        cs = (nfs3_call_state_t *) mem_get (s->localpool);
-        if (!cs) {
-                gf_log (GF_NFS3, GF_LOG_ERROR, "out of memory");
-                return NULL;
-        }
+    GF_FREE(cs->pathname);
 
-        memset (cs, 0, sizeof (*cs));
-        INIT_LIST_HEAD (&cs->entries.list);
-        INIT_LIST_HEAD (&cs->openwait_q);
-        cs->operrno = EINVAL;
-        cs->req = req;
-        cs->vol = v;
-        cs->nfsx = s->nfsx;
-        cs->nfs3state = s;
-err:
-        return cs;
-}
+    if (!list_empty(&cs->entries.list))
+        gf_dirent_free(&cs->entries);
 
-void
-nfs3_call_state_wipe (nfs3_call_state_t *cs)
-{
-        if (!cs)
-                return;
-
-        if (cs->fd) {
-                gf_log (GF_NFS3, GF_LOG_TRACE, "fd 0x%lx ref: %d",
-                        (long)cs->fd, cs->fd->refcount);
-                fd_unref (cs->fd);
-        }
-
-        GF_FREE (cs->resolventry);
-
-        GF_FREE (cs->pathname);
-
-        if (!list_empty (&cs->entries.list))
-                gf_dirent_free (&cs->entries);
-
-        nfs_loc_wipe (&cs->oploc);
-        nfs_loc_wipe (&cs->resolvedloc);
-        if (cs->iob)
-                iobuf_unref (cs->iob);
-        if (cs->iobref)
-                iobref_unref (cs->iobref);
-        if (cs->trans)
-                rpc_transport_unref (cs->trans);
-        memset (cs, 0, sizeof (*cs));
-        mem_put (cs);
-        /* Already refd by fd_lookup, so no need to ref again. */
+    nfs_loc_wipe(&cs->oploc);
+    nfs_loc_wipe(&cs->resolvedloc);
+    if (cs->iob)
+        iobuf_unref(cs->iob);
+    if (cs->iobref)
+        iobref_unref(cs->iobref);
+    if (cs->trans)
+        rpc_transport_unref(cs->trans);
+    memset(cs, 0, sizeof(*cs));
+    mem_put(cs);
+    /* Already refd by fd_lookup, so no need to ref again. */
 }
 
+nfs3_call_state_t *
+nfs3_call_state_init(struct nfs3_state *s, rpcsvc_request_t *req, xlator_t *v)
+{
+    nfs3_call_state_t *cs = NULL;
+
+    GF_VALIDATE_OR_GOTO(GF_NFS3, s, err);
+    GF_VALIDATE_OR_GOTO(GF_NFS3, req, err);
+    /* GF_VALIDATE_OR_GOTO (GF_NFS3, v, err); NLM sets this later */
+
+    cs = (nfs3_call_state_t *)mem_get(s->localpool);
+    if (!cs) {
+        gf_msg(GF_NFS3, GF_LOG_ERROR, ENOMEM, NFS_MSG_NO_MEMORY,
+               "out of memory");
+        return NULL;
+    }
+
+    memset(cs, 0, sizeof(*cs));
+    GF_REF_INIT(cs, __nfs3_call_state_wipe);
+    INIT_LIST_HEAD(&cs->entries.list);
+    INIT_LIST_HEAD(&cs->openwait_q);
+    cs->operrno = EINVAL;
+    cs->req = req;
+    cs->vol = v;
+    cs->nfsx = s->nfsx;
+    cs->nfs3state = s;
+err:
+    return cs;
+}
 
-#define nfs3_handle_call_state_init(nfs3state, calls, rq, vl ,opstat, errlabel)\
-        do {                                                            \
-                calls = nfs3_call_state_init ((nfs3state), (rq), (vl)); \
-                if (!calls) {                                           \
-                        gf_log (GF_NFS3, GF_LOG_ERROR, "Failed to "     \
-                                "init call state");                     \
-                        opstat = NFS3ERR_SERVERFAULT;                   \
-                        goto errlabel;                                  \
-                }                                                       \
-        } while (0)                                                     \
-
-
+void
+nfs3_call_state_wipe(nfs3_call_state_t *cs)
+{
+    if (!cs) {
+        gf_log_callingfn("nfs", GF_LOG_WARNING, "nfs calling state NULL");
+        return;
+    }
+    GF_REF_PUT(cs);
+}
+
+#define nfs3_handle_call_state_init(nfs3state, calls, rq, vl, opstat,          \
+                                    errlabel)                                  \
+    do {                                                                       \
+        calls = nfs3_call_state_init((nfs3state), (rq), (vl));                 \
+        if (!calls) {                                                          \
+            gf_msg(GF_NFS3, GF_LOG_ERROR, 0, NFS_MSG_INIT_CALL_STAT_FAIL,      \
+                   "Failed to"                                                 \
+                   " init call state");                                        \
+            opstat = NFS3ERR_SERVERFAULT;                                      \
+            goto errlabel;                                                     \
+        }                                                                      \
+    } while (0)
 
 struct iobuf *
-nfs3_serialize_reply (rpcsvc_request_t *req, void *arg, nfs3_serializer sfunc,
-                      struct iovec *outmsg)
-{
-        struct nfs3_state       *nfs3 = NULL;
-        struct iobuf            *iob = NULL;
-        ssize_t                 retlen = -1;
-
-        nfs3 = (struct nfs3_state *)rpcsvc_request_program_private (req);
-        if (!nfs3) {
-                gf_log (GF_NFS3, GF_LOG_ERROR, "NFSv3 state not found in RPC"
-                        " request");
-                goto ret;
-        }
-
-        /* First, get the io buffer into which the reply in arg will
-         * be serialized.
-         */
-        /* TODO: get rid of 'sfunc' and use 'xdrproc_t' so we
-           can have 'xdr_sizeof' */
-        iob = iobuf_get (nfs3->iobpool);
-        if (!iob) {
-                gf_log (GF_NFS3, GF_LOG_ERROR, "Failed to get iobuf");
-                goto ret;
-        }
-
-        iobuf_to_iovec (iob, outmsg);
-        /* Use the given serializer to translate the give C structure in arg
-         * to XDR format which will be written into the buffer in outmsg.
-         */
-        /* retlen is used to received the error since size_t is unsigned and we
-         * need -1 for error notification during encoding.
-         */
-        retlen = sfunc (*outmsg, arg);
-        if (retlen == -1) {
-                gf_log (GF_NFS3, GF_LOG_ERROR, "Failed to encode message");
-                goto ret;
-        }
-
-        outmsg->iov_len = retlen;
+nfs3_serialize_reply(rpcsvc_request_t *req, void *arg, nfs3_serializer sfunc,
+                     struct iovec *outmsg)
+{
+    struct nfs3_state *nfs3 = NULL;
+    struct iobuf *iob = NULL;
+    ssize_t retlen = -1;
+
+    nfs3 = (struct nfs3_state *)rpcsvc_request_program_private(req);
+    if (!nfs3) {
+        gf_msg(GF_NFS3, GF_LOG_ERROR, EINVAL, NFS_MSG_STATE_MISSING,
+               "NFSv3 state not found in RPC request");
+        goto ret;
+    }
+
+    /* First, get the io buffer into which the reply in arg will
+     * be serialized.
+     */
+    /* TODO: get rid of 'sfunc' and use 'xdrproc_t' so we
+       can have 'xdr_sizeof' */
+    iob = iobuf_get(nfs3->iobpool);
+    if (!iob) {
+        gf_msg(GF_NFS3, GF_LOG_ERROR, ENOMEM, NFS_MSG_NO_MEMORY,
+               "Failed to get iobuf");
+        goto ret;
+    }
+
+    iobuf_to_iovec(iob, outmsg);
+    /* Use the given serializer to translate the give C structure in arg
+     * to XDR format which will be written into the buffer in outmsg.
+     */
+    /* retlen is used to received the error since size_t is unsigned and we
+     * need -1 for error notification during encoding.
+     */
+    retlen = sfunc(*outmsg, arg);
+    if (retlen == -1) {
+        gf_msg(GF_NFS3, GF_LOG_ERROR, 0, NFS_MSG_ENCODE_FAIL,
+               "Failed to encode message");
+        goto ret;
+    }
+
+    outmsg->iov_len = retlen;
 ret:
-        if (retlen == -1) {
-                iobuf_unref (iob);
-                iob = NULL;
-        }
+    if (retlen == -1) {
+        iobuf_unref(iob);
+        iob = NULL;
+    }
 
-        return iob;
+    return iob;
 }
 
-
-
 /* Generic reply function for NFSv3 specific replies. */
 int
-nfs3svc_submit_reply (rpcsvc_request_t *req, void *arg, nfs3_serializer sfunc)
-{
-        struct iovec            outmsg = {0, };
-        struct iobuf            *iob = NULL;
-        int                     ret = -1;
-        struct iobref           *iobref = NULL;
-
-        if (!req)
-                return -1;
-
-        iob = nfs3_serialize_reply (req, arg, sfunc, &outmsg);
-        if (!iob) {
-                gf_log (GF_NFS3, GF_LOG_ERROR, "Failed to serialize reply");
-                goto ret;
-        }
-
-        iobref = iobref_new ();
-        if (!iobref) {
-                gf_log (GF_NFS3, GF_LOG_ERROR, "failed on iobref_new()");
-                goto ret;
-        }
-
-        ret = iobref_add (iobref, iob);
-        if (ret) {
-                gf_log (GF_NFS3, GF_LOG_ERROR, "Failed to add iob to iobref");
-                goto ret;
-        }
-
-        /* Then, submit the message for transmission. */
-        ret = rpcsvc_submit_message (req, &outmsg, 1, NULL, 0, iobref);
-        if (ret == -1) {
-                gf_log (GF_NFS3, GF_LOG_ERROR, "Reply submission failed");
-                goto ret;
-        }
-
-        ret = 0;
+nfs3svc_submit_reply(rpcsvc_request_t *req, void *arg, nfs3_serializer sfunc)
+{
+    struct iovec outmsg = {
+        0,
+    };
+    struct iobuf *iob = NULL;
+    int ret = -1;
+    struct iobref *iobref = NULL;
+
+    if (!req)
+        return -1;
+
+    iob = nfs3_serialize_reply(req, arg, sfunc, &outmsg);
+    if (!iob) {
+        gf_msg(GF_NFS3, GF_LOG_ERROR, 0, NFS_MSG_SERIALIZE_REPLY_FAIL,
+               "Failed to serialize reply");
+        goto ret;
+    }
+
+    iobref = iobref_new();
+    if (!iobref) {
+        gf_msg(GF_NFS3, GF_LOG_ERROR, ENOMEM, NFS_MSG_NO_MEMORY,
+               "failed on iobref_new()");
+        goto ret;
+    }
+
+    ret = iobref_add(iobref, iob);
+    if (ret) {
+        gf_msg(GF_NFS3, GF_LOG_ERROR, ENOMEM, NFS_MSG_NO_MEMORY,
+               "Failed to add iob to iobref");
+        goto ret;
+    }
+
+    /* Then, submit the message for transmission. */
+    ret = rpcsvc_submit_message(req, &outmsg, 1, NULL, 0, iobref);
+    if (ret == -1) {
+        gf_msg(GF_NFS3, GF_LOG_ERROR, 0, NFS_MSG_SUBMIT_REPLY_FAIL,
+               "Reply submission failed");
+        goto ret;
+    }
+
+    ret = 0;
 ret:
-        /* Now that we've done our job of handing the message to the RPC layer
-         * we can safely unref the iob in the hope that RPC layer must have
-         * ref'ed the iob on receiving into the txlist.
-         */
-        if (NULL != iob)
-                iobuf_unref (iob);
-        if (NULL != iobref)
-                iobref_unref (iobref);
-        return ret;
+    /* Now that we've done our job of handing the message to the RPC layer
+     * we can safely unref the iob in the hope that RPC layer must have
+     * ref'ed the iob on receiving into the txlist.
+     */
+    if (NULL != iob)
+        iobuf_unref(iob);
+    if (NULL != iobref)
+        iobref_unref(iobref);
+    return ret;
 }
 
-
 int
-nfs3svc_submit_vector_reply (rpcsvc_request_t *req, void *arg,
-                             nfs3_serializer sfunc, struct iovec *payload,
-                             int vcount, struct iobref *iobref)
-{
-        struct iovec            outmsg = {0, };
-        struct iobuf            *iob = NULL;
-        int                     ret = -1;
-        int                     new_iobref = 0;
-
-        if (!req)
-                return -1;
-
-        iob = nfs3_serialize_reply (req, arg, sfunc, &outmsg);
-        if (!iob) {
-                gf_log (GF_NFS3, GF_LOG_ERROR, "Failed to serialize reply");
-                goto ret;
-        }
-        if (iobref == NULL) {
-                iobref = iobref_new ();
-                if (!iobref) {
-                        gf_log (GF_NFS3, GF_LOG_ERROR, "failed on iobref_new");
-                        goto ret;
-                }
-                new_iobref = 1;
-        }
-
-        ret = iobref_add (iobref, iob);
-        if (ret) {
-                gf_log (GF_NFS3, GF_LOG_ERROR, "Failed to add iob to iobref");
-                goto ret;
-        }
-
-        /* Then, submit the message for transmission. */
-        ret = rpcsvc_submit_message (req, &outmsg, 1, payload, vcount, iobref);
-        if (ret == -1) {
-                gf_log (GF_NFS3, GF_LOG_ERROR, "Reply submission failed");
-                goto ret;
-        }
-
-        ret = 0;
+nfs3svc_submit_vector_reply(rpcsvc_request_t *req, void *arg,
+                            nfs3_serializer sfunc, struct iovec *payload,
+                            int vcount, struct iobref *iobref)
+{
+    struct iovec outmsg = {
+        0,
+    };
+    struct iobuf *iob = NULL;
+    int ret = -1;
+    int new_iobref = 0;
+
+    if (!req)
+        return -1;
+
+    iob = nfs3_serialize_reply(req, arg, sfunc, &outmsg);
+    if (!iob) {
+        gf_msg(GF_NFS3, GF_LOG_ERROR, 0, NFS_MSG_SERIALIZE_REPLY_FAIL,
+               "Failed to serialize reply");
+        goto ret;
+    }
+    if (iobref == NULL) {
+        iobref = iobref_new();
+        if (!iobref) {
+            gf_msg(GF_NFS3, GF_LOG_ERROR, ENOMEM, NFS_MSG_NO_MEMORY,
+                   "failed on iobref_new");
+            goto ret;
+        }
+        new_iobref = 1;
+    }
+
+    ret = iobref_add(iobref, iob);
+    if (ret) {
+        gf_msg(GF_NFS3, GF_LOG_ERROR, ENOMEM, NFS_MSG_NO_MEMORY,
+               "Failed to add iob to iobref");
+        goto ret;
+    }
+
+    /* Then, submit the message for transmission. */
+    ret = rpcsvc_submit_message(req, &outmsg, 1, payload, vcount, iobref);
+    if (ret == -1) {
+        gf_msg(GF_NFS3, GF_LOG_ERROR, 0, NFS_MSG_SUBMIT_REPLY_FAIL,
+               "Reply submission failed");
+        goto ret;
+    }
+
+    ret = 0;
 ret:
-        /* Now that we've done our job of handing the message to the RPC layer
-         * we can safely unref the iob in the hope that RPC layer must have
-         * ref'ed the iob on receiving into the txlist.
-         */
-        if (NULL != iob)
-                iobuf_unref (iob);
-        if (new_iobref)
-                iobref_unref (iobref);
-        return ret;
+    /* Now that we've done our job of handing the message to the RPC layer
+     * we can safely unref the iob in the hope that RPC layer must have
+     * ref'ed the iob on receiving into the txlist.
+     */
+    if (NULL != iob)
+        iobuf_unref(iob);
+    if (new_iobref)
+        iobref_unref(iobref);
+    return ret;
 }
 
 uint64_t
-nfs3_request_xlator_deviceid (rpcsvc_request_t *rq)
-{
-        struct nfs3_state       *nfs3 = NULL;
-        xlator_t                *xl = NULL;
-        uint64_t                devid = 0;
-        uuid_t                  volumeid = {0, };
-
-        if (!rq)
-                return 0;
-
-        xl = rpcsvc_request_private (rq);
-        nfs3 = rpcsvc_request_program_private (rq);
-        if (gf_nfs_dvm_off (nfs_state (nfs3->nfsx)))
-                devid = (uint64_t)nfs_xlator_to_xlid (nfs3->exportslist, xl);
-        else {
-                __nfs3_get_volume_id (nfs3, xl, volumeid);
-                memcpy (&devid, &volumeid[8], sizeof (devid));
-        }
+nfs3_request_xlator_deviceid(rpcsvc_request_t *rq)
+{
+    struct nfs3_state *nfs3 = NULL;
+    xlator_t *xl = NULL;
+    uint64_t devid = 0;
+    uuid_t volumeid = {
+        0,
+    };
 
-        return devid;
-}
+    if (!rq)
+        return 0;
+
+    xl = rpcsvc_request_private(rq);
+    nfs3 = rpcsvc_request_program_private(rq);
+    if (nfs3 && (gf_nfs_dvm_off(nfs_state(nfs3->nfsx))))
+        devid = (uint64_t)nfs_xlator_to_xlid(nfs3->exportslist, xl);
+    else {
+        __nfs3_get_volume_id(nfs3, xl, volumeid);
+        memcpy(&devid, &volumeid[8], sizeof(devid));
+    }
 
+    return devid;
+}
 
 int
-nfs3svc_null (rpcsvc_request_t *req)
+nfs3svc_null(rpcsvc_request_t *req)
 {
-        struct iovec    dummyvec = {0, };
-        if (!req)
-                return RPCSVC_ACTOR_ERROR;
-        rpcsvc_submit_generic (req, &dummyvec, 1,  NULL, 0, NULL);
-        return RPCSVC_ACTOR_SUCCESS;
+    struct iovec dummyvec = {
+        0,
+    };
+    if (!req)
+        return RPCSVC_ACTOR_ERROR;
+    rpcsvc_submit_generic(req, &dummyvec, 1, NULL, 0, NULL);
+    return RPCSVC_ACTOR_SUCCESS;
 }
 
-
 int
-nfs3_getattr_reply (rpcsvc_request_t *req, nfsstat3 status, struct iatt *buf)
+nfs3_getattr_reply(rpcsvc_request_t *req, nfsstat3 status, struct iatt *buf)
 {
-        getattr3res     res;
-        uint64_t        deviceid = 0;
+    getattr3res res;
+    uint64_t deviceid = 0;
 
-        deviceid = nfs3_request_xlator_deviceid (req);
-        nfs3_fill_getattr3res (&res, status, buf, deviceid);
-        nfs3svc_submit_reply (req, &res,
-                              (nfs3_serializer)xdr_serialize_getattr3res);
+    deviceid = nfs3_request_xlator_deviceid(req);
+    nfs3_fill_getattr3res(&res, status, buf, deviceid);
+    nfs3svc_submit_reply(req, &res, (nfs3_serializer)xdr_serialize_getattr3res);
 
-        return 0;
+    return 0;
 }
 
-
 int32_t
-nfs3svc_getattr_lookup_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                            int32_t op_ret, int32_t op_errno, inode_t *inode,
-                            struct iatt *buf, dict_t *xattr,
-                            struct iatt *postparent)
+nfs3svc_getattr_lookup_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                           int32_t op_ret, int32_t op_errno, inode_t *inode,
+                           struct iatt *buf, dict_t *xattr,
+                           struct iatt *postparent)
 {
-        nfsstat3                status = NFS3_OK;
-        nfs3_call_state_t       *cs = NULL;
+    nfsstat3 status = NFS3_OK;
+    nfs3_call_state_t *cs = NULL;
 
-        cs = frame->local;
+    cs = frame->local;
 
-        /*
-         * Somewhat counter-intuitively, we don't need to look for sh-failed
-         * here. Failing this getattr will generate a new lookup from the
-         * client, and nfs_fop_lookup_cbk will detect any self-heal failures.
-         */
+    /*
+     * Somewhat counter-intuitively, we don't need to look for sh-failed
+     * here. Failing this getattr will generate a new lookup from the
+     * client, and nfs_fop_lookup_cbk will detect any self-heal failures.
+     */
 
-        if (op_ret == -1) {
-                gf_log (GF_NFS, GF_LOG_WARNING,
-                        "%x: %s => -1 (%s)", rpcsvc_request_xid (cs->req),
-                        cs->resolvedloc.path, strerror (op_errno));
-                status = nfs3_cbk_errno_status (op_ret, op_errno);
-        }
-        else {
-                nfs_fix_generation(this,inode);
-        }
+    if (op_ret == -1) {
+        status = nfs3_cbk_errno_status(op_ret, op_errno);
+    } else {
+        nfs_fix_generation(this, inode);
+    }
 
-        nfs3_log_common_res (rpcsvc_request_xid (cs->req), NFS3_GETATTR,
-                        status, op_errno);
+    nfs3_log_common_res(rpcsvc_request_xid(cs->req), NFS3_GETATTR, status,
+                        op_errno, cs->resolvedloc.path);
 
-        nfs3_getattr_reply (cs->req, status, buf);
-        nfs3_call_state_wipe (cs);
+    nfs3_getattr_reply(cs->req, status, buf);
+    nfs3_call_state_wipe(cs);
 
-        return 0;
+    return 0;
 }
 
-
 int32_t
-nfs3svc_getattr_stat_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                          int32_t op_ret, int32_t op_errno, struct iatt *buf,
-                          dict_t *xdata)
+nfs3svc_getattr_stat_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                         int32_t op_ret, int32_t op_errno, struct iatt *buf,
+                         dict_t *xdata)
 {
-        nfsstat3                status = NFS3_OK;
-        nfs3_call_state_t       *cs = NULL;
+    nfsstat3 status = NFS3_OK;
+    nfs3_call_state_t *cs = NULL;
 
-        cs = frame->local;
+    cs = frame->local;
 
-        if (op_ret == -1) {
-                gf_log (GF_NFS, GF_LOG_WARNING,
-                        "%x: %s => -1 (%s)", rpcsvc_request_xid (cs->req),
-                        cs->resolvedloc.path, strerror (op_errno));
-                status = nfs3_cbk_errno_status (op_ret, op_errno);
+    if (op_ret == -1) {
+        /* Prevent crashes for the case where this call fails
+         * and buf is left in a NULL state, yet the op_errno == 0.
+         */
+        if (!buf && op_errno == 0) {
+            op_errno = EIO;
         }
+        status = nfs3_cbk_errno_status(op_ret, op_errno);
+    }
 
-        nfs3_log_common_res (rpcsvc_request_xid (cs->req), NFS3_GETATTR,
-                        status, op_errno);
+    nfs3_log_common_res(rpcsvc_request_xid(cs->req), NFS3_GETATTR, status,
+                        op_errno, cs->resolvedloc.path);
 
-        nfs3_getattr_reply (cs->req, status, buf);
-        nfs3_call_state_wipe (cs);
+    nfs3_getattr_reply(cs->req, status, buf);
+    nfs3_call_state_wipe(cs);
 
-        return 0;
+    return 0;
 }
 
-
 int
-nfs3_getattr_resume (void *carg)
-{
-        nfsstat3                        stat = NFS3ERR_SERVERFAULT;
-        int                             ret = -EFAULT;
-        nfs_user_t                      nfu = {0, };
-        nfs3_call_state_t               *cs = NULL;
-        uint64_t                         raw_ctx = 0;
-        struct nfs_inode_ctx            *ictx = NULL;
-        struct nfs_state                *priv = NULL;
-
-        if (!carg)
-                return ret;
-
-        cs = (nfs3_call_state_t *)carg;
-        nfs3_check_fh_auth_status (cs, stat, _gf_false, nfs3err);
-        nfs3_check_fh_resolve_status (cs, stat, nfs3err);
-        nfs_request_user_init (&nfu, cs->req);
-        /* If inode which is to be getattr'd is the root, we need to do a
-         * lookup instead because after a server reboot, it is not necessary
-         * for the root to have been looked up when the getattr on the root is
-         * sent. AND, this causes a problem for stat-prefetch in that it
-         * expects even the root inode to have been looked up.
-
-        if (__is_root_gfid (cs->resolvedloc.inode->gfid))
-                ret = nfs_lookup (cs->nfsx, cs->vol, &nfu, &cs->resolvedloc,
-                                  nfs3svc_getattr_lookup_cbk, cs);
-        else
-                ret = nfs_stat (cs->nfsx, cs->vol, &nfu, &cs->resolvedloc,
-        */
-
-        if (cs->hardresolved) {
-                ret = -EFAULT;
-                stat = NFS3_OK;
-                goto nfs3err;
-        }
-
-        /*
-         * If brick state changed, we need to force a proper lookup cycle (as
-         * would happen in native protocol) to do self-heal checks. We detect
-         * this by comparing the generation number for the last successful
-         * creation/lookup on the inode to the current number, so inodes that
-         * haven't been validated since the state change are affected.
-         */
-        if (inode_ctx_get(cs->resolvedloc.inode,cs->nfsx,&raw_ctx) == 0) {
-                ictx = (struct nfs_inode_ctx *)raw_ctx;
-                priv = cs->nfsx->private;
-                if (ictx->generation != priv->generation) {
-                        ret = nfs_lookup (cs->nfsx, cs->vol, &nfu,
-                                          &cs->resolvedloc,
-                                          nfs3svc_getattr_lookup_cbk, cs);
-                        goto check_err;
-                }
-        }
+nfs3_getattr_resume(void *carg)
+{
+    nfsstat3 stat = NFS3ERR_SERVERFAULT;
+    int ret = -EFAULT;
+    nfs_user_t nfu = {
+        0,
+    };
+    nfs3_call_state_t *cs = NULL;
+    uint64_t raw_ctx = 0;
+    struct nfs_inode_ctx *ictx = NULL;
+    struct nfs_state *priv = NULL;
+
+    if (!carg)
+        return ret;
 
-        ret = nfs_stat (cs->nfsx, cs->vol, &nfu, &cs->resolvedloc,
-                        nfs3svc_getattr_stat_cbk, cs);
+    cs = (nfs3_call_state_t *)carg;
+    nfs3_check_fh_auth_status(cs, stat, _gf_false, nfs3err);
+    nfs3_check_fh_resolve_status(cs, stat, nfs3err);
+    nfs_request_user_init(&nfu, cs->req);
+    /* If inode which is to be getattr'd is the root, we need to do a
+     * lookup instead because after a server reboot, it is not necessary
+     * for the root to have been looked up when the getattr on the root is
+     * sent. AND, this causes a problem for stat-prefetch in that it
+     * expects even the root inode to have been looked up.
+
+    if (__is_root_gfid (cs->resolvedloc.inode->gfid))
+            ret = nfs_lookup (cs->nfsx, cs->vol, &nfu, &cs->resolvedloc,
+                              nfs3svc_getattr_lookup_cbk, cs);
+    else
+            ret = nfs_stat (cs->nfsx, cs->vol, &nfu, &cs->resolvedloc,
+    */
+
+    if (cs->hardresolved) {
+        ret = -EFAULT;
+        stat = NFS3_OK;
+        goto nfs3err;
+    }
+
+    /*
+     * If brick state changed, we need to force a proper lookup cycle (as
+     * would happen in native protocol) to do self-heal checks. We detect
+     * this by comparing the generation number for the last successful
+     * creation/lookup on the inode to the current number, so inodes that
+     * haven't been validated since the state change are affected.
+     */
+    if (inode_ctx_get(cs->resolvedloc.inode, cs->nfsx, &raw_ctx) == 0) {
+        ictx = (struct nfs_inode_ctx *)(uintptr_t)raw_ctx;
+        priv = cs->nfsx->private;
+        if (ictx->generation != priv->generation) {
+            ret = nfs_lookup(cs->nfsx, cs->vol, &nfu, &cs->resolvedloc,
+                             nfs3svc_getattr_lookup_cbk, cs);
+            goto check_err;
+        }
+    }
+
+    ret = nfs_stat(cs->nfsx, cs->vol, &nfu, &cs->resolvedloc,
+                   nfs3svc_getattr_stat_cbk, cs);
 
 check_err:
-        if (ret < 0) {
-                gf_log (GF_NFS3, GF_LOG_ERROR, "Stat fop failed: %s: %s",
-                        cs->oploc.path, strerror (-ret));
-                stat = nfs3_errno_to_nfsstat3 (-ret);
-        }
+    if (ret < 0) {
+        gf_msg(GF_NFS3, GF_LOG_ERROR, -ret, NFS_MSG_STAT_FOP_FAIL,
+               "Stat fop failed: %s: %s", cs->oploc.path, strerror(-ret));
+        stat = nfs3_errno_to_nfsstat3(-ret);
+    }
 
 nfs3err:
-        if (ret < 0) {
-                nfs3_log_common_res (rpcsvc_request_xid (cs->req),
-                                     NFS3_GETATTR, stat, -ret);
-                nfs3_getattr_reply (cs->req, stat, &cs->stbuf);
-                nfs3_call_state_wipe (cs);
-                ret = 0;
-        }
+    if (ret < 0) {
+        nfs3_log_common_res(rpcsvc_request_xid(cs->req), NFS3_GETATTR, stat,
+                            -ret, cs->resolvedloc.path);
+        nfs3_getattr_reply(cs->req, stat, &cs->stbuf);
+        nfs3_call_state_wipe(cs);
+        ret = 0;
+    }
 
-        return ret;
+    return ret;
 }
 
-
 int
-nfs3_getattr (rpcsvc_request_t *req, struct nfs3_fh *fh)
+nfs3_getattr(rpcsvc_request_t *req, struct nfs3_fh *fh)
 {
-        xlator_t                        *vol = NULL;
-        nfsstat3                        stat = NFS3ERR_SERVERFAULT;
-        int                             ret = -EFAULT;
-        struct nfs3_state               *nfs3 = NULL;
-        nfs3_call_state_t               *cstate = NULL;
+    xlator_t *vol = NULL;
+    nfsstat3 stat = NFS3ERR_SERVERFAULT;
+    int ret = -EFAULT;
+    struct nfs3_state *nfs3 = NULL;
+    nfs3_call_state_t *cstate = NULL;
 
-        GF_VALIDATE_OR_GOTO (GF_NFS3, req, out);
-        GF_VALIDATE_OR_GOTO (GF_NFS3, fh, out);
+    GF_VALIDATE_OR_GOTO(GF_NFS3, req, out);
+    GF_VALIDATE_OR_GOTO(GF_NFS3, fh, out);
 
-        nfs3_log_common_call (rpcsvc_request_xid (req), "GETATTR", fh);
-        nfs3_validate_gluster_fh (fh, stat, nfs3err);
-        nfs3_validate_nfs3_state (req, nfs3, stat, nfs3err, ret);
-        nfs3_map_fh_to_volume (nfs3, fh, req, vol, stat, nfs3err);
-        nfs3_volume_started_check (nfs3, vol, ret, out);
-        nfs3_handle_call_state_init (nfs3, cstate, req, vol, stat, nfs3err);
+    nfs3_log_common_call(rpcsvc_request_xid(req), "GETATTR", fh);
+    nfs3_validate_gluster_fh(fh, stat, nfs3err);
+    nfs3_validate_nfs3_state(req, nfs3, stat, nfs3err, ret);
+    nfs3_map_fh_to_volume(nfs3, fh, req, vol, stat, nfs3err);
+    nfs3_volume_started_check(nfs3, vol, ret, out);
+    nfs3_handle_call_state_init(nfs3, cstate, req, vol, stat, nfs3err);
 
-        ret = nfs3_fh_resolve_and_resume (cstate, fh, NULL,nfs3_getattr_resume);
-        if (ret < 0)
-                stat = nfs3_errno_to_nfsstat3 (-ret);
+    ret = nfs3_fh_resolve_and_resume(cstate, fh, NULL, nfs3_getattr_resume);
+    if (ret < 0)
+        stat = nfs3_errno_to_nfsstat3(-ret);
 
 nfs3err:
-        if (ret < 0) {
-                nfs3_log_common_res (rpcsvc_request_xid (req), NFS3_GETATTR,
-                                     stat, -ret);
-                nfs3_getattr_reply (req, stat, NULL);
-                ret = 0;
-                nfs3_call_state_wipe (cstate);
-        }
+    if (ret < 0) {
+        nfs3_log_common_res(rpcsvc_request_xid(req), NFS3_GETATTR, stat, -ret,
+                            NULL);
+        nfs3_getattr_reply(req, stat, NULL);
+        ret = 0;
+        nfs3_call_state_wipe(cstate);
+    }
 out:
-        return ret;
+    return ret;
 }
 
-
 int
-nfs3svc_getattr (rpcsvc_request_t *req)
+nfs3svc_getattr(rpcsvc_request_t *req)
 {
-        struct nfs3_fh          fh = {{0}, };
-        getattr3args            args;
-        int                     ret = RPCSVC_ACTOR_ERROR;
+    struct nfs3_fh fh = {
+        {0},
+    };
+    getattr3args args;
+    int ret = RPCSVC_ACTOR_ERROR;
 
-        if (!req)
-                return ret;
-
-        nfs3_prep_getattr3args (&args, &fh);
-        if (xdr_to_getattr3args (req->msg[0], &args) <= 0) {
-                gf_log (GF_NFS3, GF_LOG_ERROR, "Error decoding args");
-                rpcsvc_request_seterr (req, GARBAGE_ARGS);
-                goto rpcerr;
-        }
+    if (!req)
+        return ret;
 
-        ret = nfs3_getattr (req, &fh);
-        if ((ret < 0) && (ret != RPCSVC_ACTOR_IGNORE)) {
-                gf_log (GF_NFS3, GF_LOG_ERROR, "GETATTR procedure failed");
-                rpcsvc_request_seterr (req, SYSTEM_ERR);
-                ret = RPCSVC_ACTOR_ERROR;
-        }
+    nfs3_prep_getattr3args(&args, &fh);
+    if (xdr_to_getattr3args(req->msg[0], &args) <= 0) {
+        gf_msg(GF_NFS3, GF_LOG_ERROR, 0, NFS_MSG_ARGS_DECODE_ERROR,
+               "Error decoding args");
+        rpcsvc_request_seterr(req, GARBAGE_ARGS);
+        goto rpcerr;
+    }
+
+    ret = nfs3_getattr(req, &fh);
+    if ((ret < 0) && (ret != RPCSVC_ACTOR_IGNORE)) {
+        gf_msg(GF_NFS3, GF_LOG_ERROR, -ret, NFS_MSG_GETATTR_FAIL,
+               "GETATTR procedure failed");
+        rpcsvc_request_seterr(req, SYSTEM_ERR);
+        ret = RPCSVC_ACTOR_ERROR;
+    }
 
 rpcerr:
-        return ret;
+    return ret;
 }
 
-
 int
-nfs3_setattr_reply (rpcsvc_request_t *req, nfsstat3 stat, struct iatt *preop,
-                    struct iatt *postop)
+nfs3_setattr_reply(rpcsvc_request_t *req, nfsstat3 stat, struct iatt *preop,
+                   struct iatt *postop)
 {
-        setattr3res     res = {0, };
-        uint64_t        deviceid = 0;
+    setattr3res res = {
+        0,
+    };
+    uint64_t deviceid = 0;
 
-        deviceid = nfs3_request_xlator_deviceid (req);
-        nfs3_fill_setattr3res (&res, stat, preop, postop, deviceid);
-        nfs3svc_submit_reply (req, (void *)&res,
-                              (nfs3_serializer) xdr_serialize_setattr3res);
-        return 0;
+    deviceid = nfs3_request_xlator_deviceid(req);
+    nfs3_fill_setattr3res(&res, stat, preop, postop, deviceid);
+    nfs3svc_submit_reply(req, (void *)&res,
+                         (nfs3_serializer)xdr_serialize_setattr3res);
+    return 0;
 }
 
-
 int32_t
-nfs3svc_truncate_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                      int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
-                      struct iatt *postbuf, dict_t *xdata)
-{
-        nfsstat3                stat = NFS3ERR_SERVERFAULT;
-        struct iatt             *prestat = NULL;
-        nfs3_call_state_t       *cs = NULL;
-
-        cs = frame->local;
-        if (op_ret == -1) {
-                gf_log (GF_NFS, GF_LOG_WARNING,
-                        "%x: %s => -1 (%s)", rpcsvc_request_xid (cs->req),
-                        cs->resolvedloc.path, strerror (op_errno));
-                stat = nfs3_cbk_errno_status (op_ret, op_errno);
-                goto nfs3err;
-        }
-
-        /* If the first stat was got from the guarded setattr callback, or
-         * from an earlier setattr call then we'll need to use that stat
-         * instead of the preop returned here.
-         */
-        if (cs->preparent.ia_ino != 0)
-                prestat = &cs->preparent;
-        else
-                prestat = prebuf;
-
-        stat = NFS3_OK;
+nfs3svc_truncate_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                     int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
+                     struct iatt *postbuf, dict_t *xdata)
+{
+    nfsstat3 stat = NFS3ERR_SERVERFAULT;
+    struct iatt *prestat = NULL;
+    nfs3_call_state_t *cs = NULL;
+
+    cs = frame->local;
+    if (op_ret == -1) {
+        stat = nfs3_cbk_errno_status(op_ret, op_errno);
+        goto nfs3err;
+    }
+
+    /* If the first stat was got from the guarded setattr callback, or
+     * from an earlier setattr call then we'll need to use that stat
+     * instead of the preop returned here.
+     */
+    if (cs->preparent.ia_ino != 0)
+        prestat = &cs->preparent;
+    else
+        prestat = prebuf;
+
+    stat = NFS3_OK;
 nfs3err:
-        nfs3_log_common_res (rpcsvc_request_xid (cs->req), NFS3_SETATTR, stat,
-                             op_errno);
-        nfs3_setattr_reply (cs->req, stat, prestat, postbuf);
-        nfs3_call_state_wipe (cs);
+    nfs3_log_common_res(rpcsvc_request_xid(cs->req), NFS3_SETATTR, stat,
+                        op_errno, cs->resolvedloc.path);
+    nfs3_setattr_reply(cs->req, stat, prestat, postbuf);
+    nfs3_call_state_wipe(cs);
 
-        return 0;
+    return 0;
 }
 
-
 int32_t
-nfs3svc_setattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                     int32_t op_ret, int32_t op_errno, struct iatt *preop,
-                     struct iatt *postop, dict_t *xdata)
-{
-        nfsstat3                stat = NFS3ERR_SERVERFAULT;
-        int                     ret = -1;
-        struct iatt             *prebuf = NULL;
-        nfs_user_t              nfu = {0, };
-        nfs3_call_state_t       *cs = NULL;
-
-        cs = frame->local;
-        if (op_ret == -1) {
-                gf_log (GF_NFS, GF_LOG_WARNING,
-                        "%x: %s => -1 (%s)", rpcsvc_request_xid (cs->req),
-                        cs->resolvedloc.path, strerror (op_errno));
-                stat = nfs3_cbk_errno_status (op_ret, op_errno);
-                goto nfs3err;
-        }
+nfs3svc_setattr_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                    int32_t op_ret, int32_t op_errno, struct iatt *preop,
+                    struct iatt *postop, dict_t *xdata)
+{
+    nfsstat3 stat = NFS3ERR_SERVERFAULT;
+    int ret = -1;
+    struct iatt *prebuf = NULL;
+    nfs_user_t nfu = {
+        0,
+    };
+    nfs3_call_state_t *cs = NULL;
+
+    cs = frame->local;
+    if (op_ret == -1) {
+        stat = nfs3_cbk_errno_status(op_ret, op_errno);
+        goto nfs3err;
+    }
+
+    prebuf = preop;
+    /* Store the current preop in case we need to send a truncate,
+     * in which case the preop to be returned will be this one.
+     */
+    cs->preparent = *preop;
+
+    /* Only truncate if the size is not already same as the requested
+     * truncation and also only if this is not a directory.
+     */
+    if ((gf_attr_size_set(cs->setattr_valid)) && (!IA_ISDIR(postop->ia_type)) &&
+        (preop->ia_size != cs->attr_in.ia_size)) {
+        nfs_request_user_init(&nfu, cs->req);
+        ret = nfs_truncate(cs->nfsx, cs->vol, &nfu, &cs->resolvedloc,
+                           cs->attr_in.ia_size, nfs3svc_truncate_cbk, cs);
 
-        prebuf = preop;
-        /* Store the current preop in case we need to send a truncate,
-         * in which case the preop to be returned will be this one.
-         */
-        cs->preparent = *preop;
-
-        /* Only truncate if the size is not already same as the requested
-         * truncation and also only if this is not a directory.
-         */
-        if ((gf_attr_size_set (cs->setattr_valid)) &&
-            (!IA_ISDIR (postop->ia_type)) &&
-            (preop->ia_size != cs->stbuf.ia_size)) {
-                nfs_request_user_init (&nfu, cs->req);
-                ret = nfs_truncate (cs->nfsx, cs->vol, &nfu, &cs->resolvedloc,
-                                    cs->stbuf.ia_size, nfs3svc_truncate_cbk,cs);
-
-                if (ret < 0)
-                        stat = nfs3_errno_to_nfsstat3 (-ret);
-        } else {
-                ret = -1;       /* Force a reply in the branch below. */
-                stat = NFS3_OK;
-        }
+        if (ret < 0)
+            stat = nfs3_errno_to_nfsstat3(-ret);
+    } else {
+        ret = -1; /* Force a reply in the branch below. */
+        stat = NFS3_OK;
+    }
 
 nfs3err:
-        if (ret < 0) {
-                nfs3_log_common_res (rpcsvc_request_xid (cs->req),
-                                     NFS3_SETATTR, stat, op_errno);
-                nfs3_setattr_reply (cs->req, stat, prebuf, postop);
-                nfs3_call_state_wipe (cs);
-        }
+    if (ret < 0) {
+        nfs3_log_common_res(rpcsvc_request_xid(cs->req), NFS3_SETATTR, stat,
+                            op_errno, cs->resolvedloc.path);
+        nfs3_setattr_reply(cs->req, stat, prebuf, postop);
+        nfs3_call_state_wipe(cs);
+    }
 
-        return 0;
+    return 0;
 }
 
-
-
 int32_t
-nfs3svc_setattr_stat_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                          int32_t op_ret, int32_t op_errno, struct iatt *buf,
-                          dict_t *xdata)
+nfs3svc_setattr_stat_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                         int32_t op_ret, int32_t op_errno, struct iatt *buf,
+                         dict_t *xdata)
 {
-
-        int                     ret = -EFAULT;
-        nfsstat3                stat = NFS3ERR_SERVERFAULT;
-        nfs_user_t              nfu = {0, };
-        nfs3_call_state_t       *cs = NULL;
-
-        cs = frame->local;
-        if (op_ret == -1) {
-                gf_log (GF_NFS, GF_LOG_WARNING,
-                        "%x: %s => -1 (%s)", rpcsvc_request_xid (cs->req),
-                        cs->resolvedloc.path, strerror (op_errno));
-                stat = nfs3_cbk_errno_status (op_ret, op_errno);
-                goto nfs3err;
-        }
-
-        if (buf->ia_ctime != cs->timestamp.seconds) {
-                gf_log (GF_NFS3, GF_LOG_ERROR, "Timestamps not in sync");
-                stat = NFS3ERR_NOT_SYNC;
-                goto nfs3err;
-        }
-
-        /* Not a clean way but no motivation to add a new member to local. */
-        cs->preparent = *buf;
-        nfs_request_user_init (&nfu, cs->req);
-        ret = nfs_setattr (cs->nfsx, cs->vol, &nfu, &cs->resolvedloc,&cs->stbuf,
-                           cs->setattr_valid, nfs3svc_setattr_cbk, cs);
-        if (ret < 0)
-                stat = nfs3_errno_to_nfsstat3 (-ret);
+    int ret = -EFAULT;
+    nfsstat3 stat = NFS3ERR_SERVERFAULT;
+    nfs_user_t nfu = {
+        0,
+    };
+    nfs3_call_state_t *cs = NULL;
+
+    cs = frame->local;
+    if (op_ret == -1) {
+        stat = nfs3_cbk_errno_status(op_ret, op_errno);
+        goto nfs3err;
+    }
+
+    if (buf->ia_ctime != cs->timestamp.seconds) {
+        gf_msg(GF_NFS3, GF_LOG_ERROR, 0, NFS_MSG_TIMESTAMP_NO_SYNC,
+               "Timestamps not in sync");
+        stat = NFS3ERR_NOT_SYNC;
+        goto nfs3err;
+    }
+
+    /* Not a clean way but no motivation to add a new member to local. */
+    cs->preparent = *buf;
+    nfs_request_user_init(&nfu, cs->req);
+    ret = nfs_setattr(cs->nfsx, cs->vol, &nfu, &cs->resolvedloc, &cs->stbuf,
+                      cs->setattr_valid, nfs3svc_setattr_cbk, cs);
+    if (ret < 0)
+        stat = nfs3_errno_to_nfsstat3(-ret);
 
 nfs3err:
-        if (ret < 0) {
-                nfs3_log_common_res (rpcsvc_request_xid (cs->req),
-                                     NFS3_SETATTR, stat, op_errno);
-                nfs3_setattr_reply (cs->req, stat, NULL, NULL);
-                nfs3_call_state_wipe (cs);
-        }
+    if (ret < 0) {
+        nfs3_log_common_res(rpcsvc_request_xid(cs->req), NFS3_SETATTR, stat,
+                            op_errno, cs->resolvedloc.path);
+        nfs3_setattr_reply(cs->req, stat, NULL, NULL);
+        nfs3_call_state_wipe(cs);
+    }
 
-        return 0;
+    return 0;
 }
 
-
 int
-nfs3_setattr_resume (void *carg)
+nfs3_setattr_resume(void *carg)
 {
-        nfsstat3                        stat = NFS3ERR_SERVERFAULT;
-        int                             ret = -EFAULT;
-        nfs_user_t                      nfu = {0, };
-        nfs3_call_state_t               *cs = NULL;
+    nfsstat3 stat = NFS3ERR_SERVERFAULT;
+    int ret = -EFAULT;
+    nfs_user_t nfu = {
+        0,
+    };
+    nfs3_call_state_t *cs = NULL;
 
-        if (!carg)
-                return ret;
+    if (!carg)
+        return ret;
 
-        cs = (nfs3_call_state_t *)carg;
-        nfs3_check_fh_resolve_status (cs, stat, nfs3err);
-        nfs_request_user_init (&nfu, cs->req);
-        ret = nfs_setattr (cs->nfsx, cs->vol, &nfu, &cs->resolvedloc,
-                           &cs->stbuf, cs->setattr_valid,
-                           nfs3svc_setattr_cbk, cs);
+    cs = (nfs3_call_state_t *)carg;
+    nfs3_check_fh_resolve_status(cs, stat, nfs3err);
+    nfs_request_user_init(&nfu, cs->req);
+    ret = nfs_setattr(cs->nfsx, cs->vol, &nfu, &cs->resolvedloc, &cs->attr_in,
+                      cs->setattr_valid, nfs3svc_setattr_cbk, cs);
 
-        if (ret < 0)
-                stat = nfs3_errno_to_nfsstat3 (-ret);
+    if (ret < 0)
+        stat = nfs3_errno_to_nfsstat3(-ret);
 
 nfs3err:
-        if (ret < 0) {
-                nfs3_log_common_res (rpcsvc_request_xid (cs->req),
-                                     NFS3_SETATTR, stat, -ret);
-                nfs3_setattr_reply (cs->req, stat, NULL, NULL);
-                nfs3_call_state_wipe (cs);
-        }
+    if (ret < 0) {
+        nfs3_log_common_res(rpcsvc_request_xid(cs->req), NFS3_SETATTR, stat,
+                            -ret, cs->resolvedloc.path);
+        nfs3_setattr_reply(cs->req, stat, NULL, NULL);
+        nfs3_call_state_wipe(cs);
+    }
 
-        return ret;
+    return ret;
 }
 
-
 int
-nfs3_setattr (rpcsvc_request_t *req, struct nfs3_fh *fh, sattr3 *sattr,
-              sattrguard3 *guard)
-{
-        xlator_t                        *vol = NULL;
-        nfsstat3                        stat = NFS3ERR_SERVERFAULT;
-        int                             ret = -EFAULT;
-        struct nfs3_state               *nfs3 = NULL;
-        nfs3_call_state_t               *cs = NULL;
-
-        GF_VALIDATE_OR_GOTO (GF_NFS3, req, out);
-        GF_VALIDATE_OR_GOTO (GF_NFS3, fh, out);
-        GF_VALIDATE_OR_GOTO (GF_NFS3, sattr, out);
-        GF_VALIDATE_OR_GOTO (GF_NFS3, guard, out);
-
-        nfs3_log_common_call (rpcsvc_request_xid (req), "SETATTR", fh);
-        nfs3_validate_gluster_fh (fh, stat, nfs3err);
-        nfs3_validate_nfs3_state (req, nfs3, stat, nfs3err, ret);
-        nfs3_map_fh_to_volume (nfs3, fh, req, vol, stat, nfs3err);
-        nfs3_volume_started_check (nfs3, vol, ret, out);
-        nfs3_check_rw_volaccess (nfs3, fh->exportid, stat, nfs3err);
-        nfs3_handle_call_state_init (nfs3, cs, req, vol, stat, nfs3err);
-
-        cs->setattr_valid = nfs3_sattr3_to_setattr_valid (sattr, &cs->stbuf,
-                                                          NULL);
-        if (guard->check) {
-                gf_log (GF_NFS3, GF_LOG_TRACE, "Guard check required");
-                cs->timestamp = guard->sattrguard3_u.obj_ctime;
-                cs->sattrguardcheck = 1;
-        } else {
-                gf_log (GF_NFS3, GF_LOG_TRACE, "Guard check not required");
-                cs->sattrguardcheck = 0;
-        }
-
-        if (!cs->setattr_valid) {
-                ret = -EINVAL;  /* Force a reply */
-                stat = NFS3_OK;
-                gf_log (GF_NFS3, GF_LOG_ERROR, "cs->setattr_valid is invalid");
-                goto nfs3err;
-        }
+nfs3_setattr(rpcsvc_request_t *req, struct nfs3_fh *fh, sattr3 *sattr,
+             sattrguard3 *guard)
+{
+    xlator_t *vol = NULL;
+    nfsstat3 stat = NFS3ERR_SERVERFAULT;
+    int ret = -EFAULT;
+    struct nfs3_state *nfs3 = NULL;
+    nfs3_call_state_t *cs = NULL;
+
+    GF_VALIDATE_OR_GOTO(GF_NFS3, req, out);
+    GF_VALIDATE_OR_GOTO(GF_NFS3, fh, out);
+    GF_VALIDATE_OR_GOTO(GF_NFS3, sattr, out);
+    GF_VALIDATE_OR_GOTO(GF_NFS3, guard, out);
+
+    nfs3_log_common_call(rpcsvc_request_xid(req), "SETATTR", fh);
+    nfs3_validate_gluster_fh(fh, stat, nfs3err);
+    nfs3_validate_nfs3_state(req, nfs3, stat, nfs3err, ret);
+    nfs3_map_fh_to_volume(nfs3, fh, req, vol, stat, nfs3err);
+    nfs3_volume_started_check(nfs3, vol, ret, out);
+    nfs3_check_rw_volaccess(nfs3, fh->exportid, stat, nfs3err);
+    nfs3_handle_call_state_init(nfs3, cs, req, vol, stat, nfs3err);
+
+    cs->setattr_valid = nfs3_sattr3_to_setattr_valid(sattr, &cs->attr_in, NULL);
+    if (guard->check) {
+        gf_msg_trace(GF_NFS3, 0, "Guard check required");
+        cs->timestamp = guard->sattrguard3_u.obj_ctime;
+        cs->sattrguardcheck = 1;
+    } else {
+        gf_msg_trace(GF_NFS3, 0, "Guard check not required");
+        cs->sattrguardcheck = 0;
+    }
+
+    if (!cs->setattr_valid) {
+        ret = -EINVAL; /* Force a reply */
+        stat = NFS3_OK;
+        gf_msg(GF_NFS3, GF_LOG_ERROR, EINVAL, NFS_MSG_SETATTR_INVALID,
+               "cs->setattr_valid is invalid");
+        goto nfs3err;
+    }
 
-        ret = nfs3_fh_resolve_and_resume (cs, fh, NULL, nfs3_setattr_resume);
-        if (ret < 0)
-                stat = nfs3_errno_to_nfsstat3 (-ret);
+    ret = nfs3_fh_resolve_and_resume(cs, fh, NULL, nfs3_setattr_resume);
+    if (ret < 0)
+        stat = nfs3_errno_to_nfsstat3(-ret);
 
 nfs3err:
-        if (ret < 0) {
-                nfs3_log_common_res (rpcsvc_request_xid (req), NFS3_SETATTR,
-                                     stat, -ret);
-                nfs3_setattr_reply (req, stat, NULL, NULL);
-                nfs3_call_state_wipe (cs);
-                /* Ret must be 0 after this so that the caller does not
-                 * also send an RPC reply.
-                 */
-                ret = 0;
-        }
+    if (ret < 0) {
+        nfs3_log_common_res(rpcsvc_request_xid(req), NFS3_SETATTR, stat, -ret,
+                            cs ? cs->resolvedloc.path : NULL);
+        nfs3_setattr_reply(req, stat, NULL, NULL);
+        nfs3_call_state_wipe(cs);
+        /* Ret must be 0 after this so that the caller does not
+         * also send an RPC reply.
+         */
+        ret = 0;
+    }
 out:
-        return ret;
+    return ret;
 }
 
-
-
 int
-nfs3svc_setattr (rpcsvc_request_t *req)
-{
-        struct nfs3_fh          fh = {{0}, };
-        setattr3args            args;
-        int                     ret = RPCSVC_ACTOR_ERROR;
-
-        GF_VALIDATE_OR_GOTO (GF_NFS3, req, rpcerr);
-
-        nfs3_prep_setattr3args (&args, &fh);
-        if (xdr_to_setattr3args (req->msg[0], &args) <= 0) {
-                gf_log (GF_NFS3, GF_LOG_ERROR, "Error decoding args");
-                rpcsvc_request_seterr (req, GARBAGE_ARGS);
-                goto rpcerr;
-        }
-
-        ret = nfs3_setattr (req, &fh, &args.new_attributes, &args.guard);
-        if ((ret < 0) && (ret != RPCSVC_ACTOR_IGNORE)) {
-                gf_log (GF_NFS3, GF_LOG_ERROR, "SETATTR procedure failed");
-                rpcsvc_request_seterr (req, SYSTEM_ERR);
-                ret = RPCSVC_ACTOR_ERROR;
-        }
+nfs3svc_setattr(rpcsvc_request_t *req)
+{
+    struct nfs3_fh fh = {
+        {0},
+    };
+    setattr3args args;
+    int ret = RPCSVC_ACTOR_ERROR;
+
+    GF_VALIDATE_OR_GOTO(GF_NFS3, req, rpcerr);
+
+    nfs3_prep_setattr3args(&args, &fh);
+    if (xdr_to_setattr3args(req->msg[0], &args) <= 0) {
+        gf_msg(GF_NFS3, GF_LOG_ERROR, 0, NFS_MSG_ARGS_DECODE_ERROR,
+               "Error decoding args");
+        rpcsvc_request_seterr(req, GARBAGE_ARGS);
+        goto rpcerr;
+    }
+
+    ret = nfs3_setattr(req, &fh, &args.new_attributes, &args.guard);
+    if ((ret < 0) && (ret != RPCSVC_ACTOR_IGNORE)) {
+        gf_msg(GF_NFS3, GF_LOG_ERROR, -ret, NFS_MSG_SETATTR_FAIL,
+               "SETATTR procedure failed");
+        rpcsvc_request_seterr(req, SYSTEM_ERR);
+        ret = RPCSVC_ACTOR_ERROR;
+    }
 
 rpcerr:
-        return ret;
-
+    return ret;
 }
 
-
 int
-nfs3_lookup_reply (rpcsvc_request_t *req, nfsstat3 stat, struct nfs3_fh *newfh,
-                   struct iatt *stbuf, struct iatt *postparent)
+nfs3_lookup_reply(rpcsvc_request_t *req, nfsstat3 stat, struct nfs3_fh *newfh,
+                  struct iatt *stbuf, struct iatt *postparent)
 {
-        lookup3res      res = {0, };
-        uint64_t        deviceid = 0;
+    lookup3res res = {
+        0,
+    };
+    uint64_t deviceid = 0;
 
-        deviceid = nfs3_request_xlator_deviceid (req);
-        nfs3_fill_lookup3res (&res, stat, newfh, stbuf, postparent, deviceid);
-        return nfs3svc_submit_reply (req, &res,
-                                     (nfs3_serializer)xdr_serialize_lookup3res);
+    deviceid = nfs3_request_xlator_deviceid(req);
+    nfs3_fill_lookup3res(&res, stat, newfh, stbuf, postparent, deviceid);
+    return nfs3svc_submit_reply(req, &res,
+                                (nfs3_serializer)xdr_serialize_lookup3res);
 }
 
 int
-nfs3_lookup_resume (void *carg);
-
+nfs3_lookup_resume(void *carg);
 
 int
-nfs3_fresh_lookup (nfs3_call_state_t *cs)
-{
-        int     ret = -EFAULT;
-        char    *oldresolventry = NULL;
-
-        GF_VALIDATE_OR_GOTO (GF_NFS3, cs, err);
-        gf_log (GF_NFS3, GF_LOG_DEBUG, "inode needs fresh lookup");
-        inode_unlink (cs->resolvedloc.inode, cs->resolvedloc.parent,
-                      cs->resolventry);
-        nfs_loc_wipe (&cs->resolvedloc);
-
-        /* Store pointer to currently allocated resolventry because it gets over
-         * written in fh_resolve_and_resume.
-         */
-        oldresolventry = cs->resolventry;
-        cs->lookuptype = GF_NFS3_FRESH;
-        ret = nfs3_fh_resolve_and_resume (cs, &cs->resolvefh, cs->resolventry,
-                                          nfs3_lookup_resume);
-        /* Allocated in the previous call to fh_resolve_and_resume using the
-         * same call_state.
-         */
-        GF_FREE (oldresolventry);
+nfs3_fresh_lookup(nfs3_call_state_t *cs)
+{
+    int ret = -EFAULT;
+    char *oldresolventry = NULL;
+
+    GF_VALIDATE_OR_GOTO(GF_NFS3, cs, err);
+    gf_msg_debug(GF_NFS3, 0, "inode needs fresh lookup");
+    inode_unlink(cs->resolvedloc.inode, cs->resolvedloc.parent,
+                 cs->resolventry);
+    nfs_loc_wipe(&cs->resolvedloc);
+
+    /* Store pointer to currently allocated resolventry because it gets over
+     * written in fh_resolve_and_resume.
+     */
+    oldresolventry = cs->resolventry;
+    cs->lookuptype = GF_NFS3_FRESH;
+    ret = nfs3_fh_resolve_and_resume(cs, &cs->resolvefh, cs->resolventry,
+                                     nfs3_lookup_resume);
+    /* Allocated in the previous call to fh_resolve_and_resume using the
+     * same call_state.
+     */
+    GF_FREE(oldresolventry);
 err:
-        return ret;
+    return ret;
 }
 
 int
-nfs3svc_lookup_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                    int32_t op_ret, int32_t op_errno, inode_t *inode,
-                    struct iatt *buf, dict_t *xattr, struct iatt *postparent)
-{
-        struct nfs3_fh                  newfh = {{0}, };
-        nfsstat3                        status = NFS3_OK;
-        nfs3_call_state_t               *cs = NULL;
-        inode_t                         *oldinode = NULL;
-
-        cs = frame->local;
-        if (op_ret == -1) {
-                gf_log (GF_NFS,
-                        (op_errno == ENOENT ? GF_LOG_TRACE : GF_LOG_WARNING),
-                        "%x: %s => -1 (%s)", rpcsvc_request_xid (cs->req),
-                        cs->resolvedloc.path, strerror (op_errno));
-                status = nfs3_cbk_errno_status (op_ret, op_errno);
-                goto xmit_res;
-        }
-
-        nfs3_fh_build_child_fh (&cs->parent, buf, &newfh);
-        oldinode = inode_link (inode, cs->resolvedloc.parent,
-                               cs->resolvedloc.name, buf);
+nfs3svc_lookup_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                   int32_t op_ret, int32_t op_errno, inode_t *inode,
+                   struct iatt *buf, dict_t *xattr, struct iatt *postparent)
+{
+    struct nfs3_fh newfh = {
+        {0},
+    };
+    nfsstat3 status = NFS3_OK;
+    nfs3_call_state_t *cs = NULL;
+    inode_t *oldinode = NULL;
+
+    cs = frame->local;
+    if (op_ret == -1) {
+        status = nfs3_cbk_errno_status(op_ret, op_errno);
+        goto xmit_res;
+    }
+
+    nfs3_fh_build_child_fh(&cs->parent, buf, &newfh);
+    oldinode = inode_link(inode, cs->resolvedloc.parent, cs->resolvedloc.name,
+                          buf);
 xmit_res:
-        /* Only send fresh lookup if it was a revalidate that failed. */
-        if ((op_ret ==  -1) && (nfs3_is_revalidate_lookup (cs))) {
-                op_ret = nfs3_fresh_lookup (cs);
-                goto out;
-        }
-
-        nfs3_log_newfh_res (rpcsvc_request_xid (cs->req), NFS3_LOOKUP,
-                        status, op_errno, &newfh);
-        nfs3_lookup_reply (cs->req, status, &newfh, buf, postparent);
-        nfs3_call_state_wipe (cs);
+    /* Only send fresh lookup if it was a revalidate that failed. */
+    if ((op_ret == -1) && (nfs3_is_revalidate_lookup(cs))) {
+        op_ret = nfs3_fresh_lookup(cs);
+        goto out;
+    }
+
+    nfs3_log_newfh_res(rpcsvc_request_xid(cs->req), NFS3_LOOKUP, status,
+                       op_errno, &newfh, cs->resolvedloc.path);
+    nfs3_lookup_reply(cs->req, status, &newfh, buf, postparent);
+    nfs3_call_state_wipe(cs);
 out:
-        if (oldinode) {
-                inode_lookup (oldinode);
-                inode_unref (oldinode);
-        }
-        return 0;
+    if (oldinode) {
+        inode_lookup(oldinode);
+        inode_unref(oldinode);
+    }
+    return 0;
 }
 
-
 int
-nfs3svc_lookup_parentdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                              int32_t op_ret, int32_t op_errno, inode_t *inode,
-                              struct iatt *buf, dict_t *xattr,
-                              struct iatt *postparent)
-{
-        struct nfs3_fh                  newfh = {{0}, };
-        nfsstat3                        status = NFS3_OK;
-        nfs3_call_state_t               *cs = NULL;
-        uuid_t                          volumeid = {0, };
-        uuid_t                          mountid = {1, };
-        struct nfs3_state               *nfs3 = NULL;
-
-        cs = frame->local;
-        if (op_ret == -1) {
-                gf_log (GF_NFS, GF_LOG_WARNING,
-                        "%x: %s => -1 (%s)", rpcsvc_request_xid (cs->req),
-                        cs->resolvedloc.path, strerror (op_errno));
-                status = nfs3_cbk_errno_status (op_ret, op_errno);
-                goto xmit_res;
-        }
-
-        nfs3 = cs->nfs3state;
-        /* If the buf inode shows that this is a root dir's buf, then the file
-         * handle needs to be specially crafted, in all other cases, we'll just
-         * create the handle normally using the buffer of the parent dir.
-         */
-        if (buf->ia_ino != 1) {
-                nfs3_fh_build_parent_fh (&cs->fh, buf, &newfh);
-                goto xmit_res;
-        }
-
-        if (gf_nfs_dvm_off (nfs_state (nfs3->nfsx)))
-                newfh = nfs3_fh_build_indexed_root_fh (nfs3->exportslist,
-                                                       cs->vol);
-        else {
-                __nfs3_get_volume_id (nfs3, cs->vol, volumeid);
-                newfh = nfs3_fh_build_uuid_root_fh (volumeid, mountid);
-        }
+nfs3svc_lookup_parentdir_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                             int32_t op_ret, int32_t op_errno, inode_t *inode,
+                             struct iatt *buf, dict_t *xattr,
+                             struct iatt *postparent)
+{
+    struct nfs3_fh newfh = {
+        {0},
+    };
+    nfsstat3 status = NFS3_OK;
+    nfs3_call_state_t *cs = NULL;
+    uuid_t volumeid = {
+        0,
+    };
+    uuid_t mountid = {
+        1,
+    };
+    struct nfs3_state *nfs3 = NULL;
+
+    cs = frame->local;
+    if (op_ret == -1) {
+        status = nfs3_cbk_errno_status(op_ret, op_errno);
+        goto xmit_res;
+    }
+
+    nfs3 = cs->nfs3state;
+    /* If the buf inode shows that this is a root dir's buf, then the file
+     * handle needs to be specially crafted, in all other cases, we'll just
+     * create the handle normally using the buffer of the parent dir.
+     */
+    if (buf->ia_ino != 1) {
+        nfs3_fh_build_parent_fh(&cs->fh, buf, &newfh);
+        goto xmit_res;
+    }
+
+    if (gf_nfs_dvm_off(nfs_state(nfs3->nfsx)))
+        newfh = nfs3_fh_build_indexed_root_fh(nfs3->exportslist, cs->vol);
+    else {
+        __nfs3_get_volume_id(nfs3, cs->vol, volumeid);
+        newfh = nfs3_fh_build_uuid_root_fh(volumeid, mountid);
+    }
 
 xmit_res:
-        nfs3_log_newfh_res (rpcsvc_request_xid (cs->req), NFS3_LOOKUP, status,
-                            op_errno, &newfh);
-        nfs3_lookup_reply (cs->req, status, &newfh, buf, postparent);
-        nfs3_call_state_wipe (cs);
+    nfs3_log_newfh_res(rpcsvc_request_xid(cs->req), NFS3_LOOKUP, status,
+                       op_errno, &newfh, cs->resolvedloc.path);
+    nfs3_lookup_reply(cs->req, status, &newfh, buf, postparent);
+    nfs3_call_state_wipe(cs);
 
-        return 0;
+    return 0;
 }
 
-
-
 int
-nfs3_lookup_parentdir_resume (void *carg)
-{
-        nfsstat3                        stat = NFS3ERR_SERVERFAULT;
-        int                             ret = -EFAULT;
-        nfs_user_t                      nfu = {0, };
-        nfs3_call_state_t               *cs = NULL;
-        inode_t                         *parent = NULL;
-
-        if (!carg) {
-                gf_log (GF_NFS3, GF_LOG_ERROR, "Invalid argument,"
-                        " carg value NULL");
-                return EINVAL;
-        }
-
-        cs = (nfs3_call_state_t *)carg;
-        nfs3_check_fh_auth_status (cs, stat, _gf_false, nfs3err);
-        nfs3_check_fh_resolve_status (cs, stat, nfs3err);
-
-        /* At this point now, the loc in cs is for the directory file handle
-         * sent by the client. This loc needs to be transformed into a loc that
-         * represents the parent dir of cs->resolvedloc.inode.
-         *
-         * EXCEPT in the case where the .. is a parent of the root directory.
-         * In this case we'll be returning the file handle and attributes of the
-         * root itself.
-         */
-        nfs_request_user_init (&nfu, cs->req);
+nfs3_lookup_parentdir_resume(void *carg)
+{
+    nfsstat3 stat = NFS3ERR_SERVERFAULT;
+    int ret = -EFAULT;
+    nfs_user_t nfu = {
+        0,
+    };
+    nfs3_call_state_t *cs = NULL;
+    inode_t *parent = NULL;
+
+    if (!carg) {
+        gf_msg(GF_NFS3, GF_LOG_ERROR, EINVAL, NFS_MSG_INVALID_ENTRY,
+               "Invalid argument, carg value NULL");
+        return EINVAL;
+    }
+
+    cs = (nfs3_call_state_t *)carg;
+    nfs3_check_fh_auth_status(cs, stat, _gf_false, nfs3err);
+    nfs3_check_fh_resolve_status(cs, stat, nfs3err);
+
+    /* At this point now, the loc in cs is for the directory file handle
+     * sent by the client. This loc needs to be transformed into a loc that
+     * represents the parent dir of cs->resolvedloc.inode.
+     *
+     * EXCEPT in the case where the .. is a parent of the root directory.
+     * In this case we'll be returning the file handle and attributes of the
+     * root itself.
+     */
+    nfs_request_user_init(&nfu, cs->req);
+
+    /* Save the file handle from the LOOKUP request. We'll use this to
+     * build the file handle of the parent directory in case the parent is
+     * not root dir.
+     */
+    cs->fh = cs->resolvefh;
+
+    /* If fh is that of the root, the resolvedloc will already contain
+     * the loc for root. After that, we'll send lookup for the root dir
+     * itself since we cannot send the lookup on the parent of root.
+     *
+     * For all other cases, we'll send the lookup on the parent of the
+     * given directory file handle.
+     */
+    if (!nfs3_fh_is_root_fh(&cs->fh)) {
+        parent = inode_ref(cs->resolvedloc.parent);
+        nfs_loc_wipe(&cs->resolvedloc);
+        ret = nfs_inode_loc_fill(parent, &cs->resolvedloc, NFS_RESOLVE_CREATE);
 
-        /* Save the file handle from the LOOKUP request. We'll use this to
-         * build the file handle of the parent directory in case the parent is
-         * not root dir.
-         */
-        cs->fh = cs->resolvefh;
-
-        /* If fh is that of the root, the resolvedloc will already contain
-         * the loc for root. After that, we'll send lookup for the root dir
-         * itself since we cannot send the lookup on the parent of root.
-         *
-         * For all other cases, we'll send the lookup on the parent of the
-         * given directory file handle.
-         */
-        if (!nfs3_fh_is_root_fh (&cs->fh)) {
-                parent = inode_ref (cs->resolvedloc.parent);
-                nfs_loc_wipe (&cs->resolvedloc);
-                ret = nfs_inode_loc_fill (parent, &cs->resolvedloc,
-                                          NFS_RESOLVE_CREATE);
-
-                if (ret < 0) {
-                        gf_log (GF_NFS3, GF_LOG_ERROR, "nfs_inode_loc_fill"
-                                " error");
-                        goto errtostat;
-                }
+        if (ret < 0) {
+            gf_msg(GF_NFS3, GF_LOG_ERROR, -ret, NFS_MSG_INODE_LOC_FILL_ERROR,
+                   "nfs_inode_loc_fill"
+                   " error");
+            goto errtostat;
         }
+    }
 
-        ret = nfs_lookup (cs->nfsx, cs->vol, &nfu, &cs->resolvedloc,
-                          nfs3svc_lookup_parentdir_cbk, cs);
+    ret = nfs_lookup(cs->nfsx, cs->vol, &nfu, &cs->resolvedloc,
+                     nfs3svc_lookup_parentdir_cbk, cs);
 errtostat:
-        if (ret < 0)
-                stat = nfs3_errno_to_nfsstat3 (-ret);
+    if (ret < 0)
+        stat = nfs3_errno_to_nfsstat3(-ret);
 
 nfs3err:
-        if (ret < 0) {
-                nfs3_log_common_res (rpcsvc_request_xid (cs->req), NFS3_LOOKUP,
-                                     stat, -ret);
-                nfs3_lookup_reply (cs->req, stat, NULL, NULL, NULL);
-                nfs3_call_state_wipe (cs);
-        }
+    if (ret < 0) {
+        nfs3_log_common_res(rpcsvc_request_xid(cs->req), NFS3_LOOKUP, stat,
+                            -ret, cs->resolvedloc.path);
+        nfs3_lookup_reply(cs->req, stat, NULL, NULL, NULL);
+        nfs3_call_state_wipe(cs);
+    }
 
-        if (parent)
-                inode_unref (parent);
+    if (parent)
+        inode_unref(parent);
 
-        return ret;
+    return ret;
 }
 
-
 int
-nfs3_lookup_resume (void *carg)
-{
-        nfsstat3                        stat = NFS3ERR_SERVERFAULT;
-        int                             ret = -EFAULT;
-        nfs_user_t                      nfu = {0, };
-        nfs3_call_state_t               *cs = NULL;
-        struct nfs3_fh                  newfh = {{0},};
-
-        if (!carg) {
-                gf_log (GF_NFS3, GF_LOG_ERROR, "Invalid argument,"
-                        " carg value NULL");
-                return EINVAL;
-        }
-
-        cs = (nfs3_call_state_t *)carg;
-        nfs3_check_fh_auth_status (cs, stat, _gf_false, nfs3err);
-        nfs3_check_fh_resolve_status (cs, stat, nfs3err);
-        cs->parent = cs->resolvefh;
-
-	if (cs->hardresolved) {
-		stat = NFS3_OK;
-		nfs3_fh_build_child_fh (&cs->parent, &cs->stbuf, &newfh);
-		goto nfs3err;
-	}
+nfs3_lookup_resume(void *carg)
+{
+    nfsstat3 stat = NFS3ERR_SERVERFAULT;
+    int ret = -EFAULT;
+    nfs_user_t nfu = {
+        0,
+    };
+    nfs3_call_state_t *cs = NULL;
+    struct nfs3_fh newfh = {
+        {0},
+    };
+
+    if (!carg) {
+        gf_msg(GF_NFS3, GF_LOG_ERROR, EINVAL, NFS_MSG_INVALID_ENTRY,
+               "Invalid argument, carg value NULL");
+        return EINVAL;
+    }
+
+    cs = (nfs3_call_state_t *)carg;
+    nfs3_check_fh_auth_status(cs, stat, _gf_false, nfs3err);
+    nfs3_check_fh_resolve_status(cs, stat, nfs3err);
+    cs->parent = cs->resolvefh;
+
+    if (cs->hardresolved) {
+        stat = NFS3_OK;
+        nfs3_fh_build_child_fh(&cs->parent, &cs->stbuf, &newfh);
+        goto nfs3err;
+    }
 
-        nfs_request_user_init (&nfu, cs->req);
-        ret = nfs_lookup (cs->nfsx, cs->vol, &nfu, &cs->resolvedloc,
-                          nfs3svc_lookup_cbk, cs);
-        if (ret < 0)
-                stat = nfs3_errno_to_nfsstat3 (-ret);
+    nfs_request_user_init(&nfu, cs->req);
+    ret = nfs_lookup(cs->nfsx, cs->vol, &nfu, &cs->resolvedloc,
+                     nfs3svc_lookup_cbk, cs);
+    if (ret < 0)
+        stat = nfs3_errno_to_nfsstat3(-ret);
 
 nfs3err:
-        if (ret < 0) {
-                nfs3_log_common_res (rpcsvc_request_xid (cs->req), NFS3_LOOKUP,
-                                     stat, -ret);
-                nfs3_lookup_reply (cs->req, stat, &newfh, &cs->stbuf,
-                                   &cs->postparent);
-                nfs3_call_state_wipe (cs);
-        }
+    if (ret < 0) {
+        nfs3_log_common_res(rpcsvc_request_xid(cs->req), NFS3_LOOKUP, stat,
+                            -ret, cs->resolvedloc.path);
+        nfs3_lookup_reply(cs->req, stat, &newfh, &cs->stbuf, &cs->postparent);
+        nfs3_call_state_wipe(cs);
+    }
 
-        return ret;
+    return ret;
 }
 
-
 int
-nfs3_lookup (rpcsvc_request_t *req, struct nfs3_fh *fh, int fhlen, char *name)
-{
-        xlator_t                        *vol = NULL;
-        nfsstat3                        stat = NFS3ERR_SERVERFAULT;
-        int                             ret = -EFAULT;
-        struct nfs3_state               *nfs3 = NULL;
-        nfs3_call_state_t               *cs = NULL;
-
-        GF_VALIDATE_OR_GOTO (GF_NFS3, req, out);
-        GF_VALIDATE_OR_GOTO (GF_NFS3, fh, out);
-        GF_VALIDATE_OR_GOTO (GF_NFS3, name, out);
-
-        nfs3_log_fh_entry_call (rpcsvc_request_xid (req), "LOOKUP", fh,
-                                name);
-        nfs3_validate_nfs3_state (req, nfs3, stat, nfs3err, ret);
-        if (nfs3_solaris_zerolen_fh (fh, fhlen))
-                nfs3_funge_solaris_zerolen_fh (nfs3, fh, name, stat, nfs3err);
-        else
-                nfs3_validate_gluster_fh (fh, stat, nfs3err);
-        nfs3_validate_strlen_or_goto (name, NFS_NAME_MAX, nfs3err, stat, ret);
-        nfs3_map_fh_to_volume (nfs3, fh, req, vol, stat, nfs3err);
-        nfs3_volume_started_check (nfs3, vol, ret, out);
-        nfs3_handle_call_state_init (nfs3, cs, req, vol, stat, nfs3err);
-
-        cs->lookuptype = GF_NFS3_REVALIDATE;
-        ret = nfs3_fh_resolve_and_resume (cs, fh, name,
-                                          nfs3_lookup_resume);
+nfs3_lookup(rpcsvc_request_t *req, struct nfs3_fh *fh, int fhlen, char *name)
+{
+    xlator_t *vol = NULL;
+    nfsstat3 stat = NFS3ERR_SERVERFAULT;
+    int ret = -EFAULT;
+    struct nfs3_state *nfs3 = NULL;
+    nfs3_call_state_t *cs = NULL;
+
+    GF_VALIDATE_OR_GOTO(GF_NFS3, req, out);
+    GF_VALIDATE_OR_GOTO(GF_NFS3, fh, out);
+    GF_VALIDATE_OR_GOTO(GF_NFS3, name, out);
+
+    nfs3_log_fh_entry_call(rpcsvc_request_xid(req), "LOOKUP", fh, name);
+    nfs3_validate_nfs3_state(req, nfs3, stat, nfs3err, ret);
+    if (nfs3_solaris_zerolen_fh(fh, fhlen)) {
+        ret = nfs3_funge_webnfs_zerolen_fh(req, nfs3, fh, name);
+        if (ret < 0)
+            goto nfs3err;
 
-        if (ret < 0) {
-                gf_log (GF_NFS, GF_LOG_ERROR, "failed to start hard reslove");
-                stat = nfs3_errno_to_nfsstat3 (-ret);
-        }
+        /* this fh means we're doing a mount, name is no more useful */
+        name = NULL;
+    } else
+        nfs3_validate_gluster_fh(fh, stat, nfs3err);
+    nfs3_validate_strlen_or_goto(name, NFS_NAME_MAX, nfs3err, stat, ret);
+    nfs3_map_fh_to_volume(nfs3, fh, req, vol, stat, nfs3err);
+    nfs3_volume_started_check(nfs3, vol, ret, out);
+    nfs3_handle_call_state_init(nfs3, cs, req, vol, stat, nfs3err);
+
+    cs->lookuptype = GF_NFS3_REVALIDATE;
+    ret = nfs3_fh_resolve_and_resume(cs, fh, name, nfs3_lookup_resume);
+
+    if (ret < 0) {
+        gf_msg(GF_NFS, GF_LOG_ERROR, -ret, NFS_MSG_HARD_RESOLVE_FAIL,
+               "failed to start hard resolve");
+    }
 
 nfs3err:
-        if (ret < 0) {
-                nfs3_log_common_res (rpcsvc_request_xid (req), NFS3_LOOKUP,
-                                     stat,
-                                     -ret);
-                nfs3_lookup_reply (req, stat, NULL, NULL, NULL);
-                nfs3_call_state_wipe (cs);
-                /* Ret must be 0 after this so that the caller does not
-                 * also send an RPC reply.
-                 */
-                ret = 0;
-        }
+    if (ret < 0) {
+        stat = nfs3_errno_to_nfsstat3(-ret);
+        nfs3_log_common_res(rpcsvc_request_xid(req), NFS3_LOOKUP, stat, -ret,
+                            cs ? cs->resolvedloc.path : NULL);
+        nfs3_lookup_reply(req, stat, NULL, NULL, NULL);
+        if (cs)
+            nfs3_call_state_wipe(cs);
+        /* Ret must be 0 after this so that the caller does not
+         * also send an RPC reply.
+         */
+        ret = 0;
+    }
 out:
-        return ret;
+    return ret;
 }
 
-
 int
-nfs3svc_lookup (rpcsvc_request_t *req)
-{
-        char                    name[NFS_PATH_MAX];
-        struct nfs3_fh          fh = {{0}, };
-        lookup3args             args;
-        int                     ret = RPCSVC_ACTOR_ERROR;
-
-        GF_VALIDATE_OR_GOTO (GF_NFS, req, rpcerr);
-
-        nfs3_prep_lookup3args (&args, &fh, name);
-        if (xdr_to_lookup3args (req->msg[0], &args) <= 0) {
-                gf_log (GF_NFS3, GF_LOG_ERROR, "Error decoding args");
-                rpcsvc_request_seterr (req, GARBAGE_ARGS);
-                goto rpcerr;
-        }
-
-        ret = nfs3_lookup (req, &fh, args.what.dir.data.data_len, name);
-        if ((ret < 0) && (ret != RPCSVC_ACTOR_IGNORE)) {
-                gf_log (GF_NFS3, GF_LOG_ERROR, "LOOKUP procedure failed");
-                rpcsvc_request_seterr (req, SYSTEM_ERR);
-                ret = RPCSVC_ACTOR_ERROR;
-        }
+nfs3svc_lookup(rpcsvc_request_t *req)
+{
+    char name[NFS_PATH_MAX];
+    struct nfs3_fh fh = {
+        {0},
+    };
+    lookup3args args;
+    int ret = RPCSVC_ACTOR_ERROR;
+
+    GF_VALIDATE_OR_GOTO(GF_NFS, req, rpcerr);
+
+    nfs3_prep_lookup3args(&args, &fh, name);
+    if (xdr_to_lookup3args(req->msg[0], &args) <= 0) {
+        gf_msg(GF_NFS3, GF_LOG_ERROR, 0, NFS_MSG_ARGS_DECODE_ERROR,
+               "Error decoding args");
+        rpcsvc_request_seterr(req, GARBAGE_ARGS);
+        goto rpcerr;
+    }
+
+    ret = nfs3_lookup(req, &fh, args.what.dir.data.data_len, name);
+    if ((ret < 0) && (ret != RPCSVC_ACTOR_IGNORE)) {
+        gf_msg(GF_NFS3, GF_LOG_ERROR, -ret, NFS_MSG_LOOKUP_PROC_FAIL,
+               "LOOKUP procedure failed");
+        rpcsvc_request_seterr(req, SYSTEM_ERR);
+        ret = RPCSVC_ACTOR_ERROR;
+    }
 
 rpcerr:
-        return ret;
+    return ret;
 }
 
-
 int
-nfs3_access_reply (rpcsvc_request_t *req, nfsstat3 status, int32_t accbits,
-		   int32_t reqaccbits)
+nfs3_access_reply(rpcsvc_request_t *req, nfsstat3 status, int32_t accbits,
+                  int32_t reqaccbits)
 {
-        access3res      res;
+    access3res res;
 
-        nfs3_fill_access3res (&res, status, accbits, reqaccbits);
-        nfs3svc_submit_reply (req, &res,
-                              (nfs3_serializer)xdr_serialize_access3res);
-        return 0;
+    nfs3_fill_access3res(&res, status, accbits, reqaccbits);
+    nfs3svc_submit_reply(req, &res, (nfs3_serializer)xdr_serialize_access3res);
+    return 0;
 }
 
-
 int32_t
-nfs3svc_access_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                    int32_t op_ret, int32_t op_errno, dict_t *xdata)
+nfs3svc_access_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                   int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
-        nfsstat3                status = NFS3_OK;
-        nfs3_call_state_t       *cs = NULL;
+    nfsstat3 status = NFS3_OK;
+    nfs3_call_state_t *cs = NULL;
 
-        cs = frame->local;
+    cs = frame->local;
 
-        if (op_ret == -1) {
-                gf_log (GF_NFS, GF_LOG_WARNING,
-                        "%x: %s => -1 (%s)", rpcsvc_request_xid (cs->req),
-                        cs->resolvedloc.path, strerror (op_errno));
-                status = nfs3_cbk_errno_status (op_ret, op_errno);
-        }
+    if (op_ret == -1) {
+        status = nfs3_cbk_errno_status(op_ret, op_errno);
+    }
 
-        nfs3_log_common_res (rpcsvc_request_xid (cs->req), NFS3_ACCESS,
-                        status, op_errno);
-        nfs3_access_reply (cs->req, status, op_errno, cs->accessbits);
-        nfs3_call_state_wipe (cs);
+    nfs3_log_common_res(rpcsvc_request_xid(cs->req), NFS3_ACCESS, status,
+                        op_errno, cs->resolvedloc.path);
+    nfs3_access_reply(cs->req, status, op_errno, cs->accessbits);
+    nfs3_call_state_wipe(cs);
 
-        return 0;
+    return 0;
 }
 
 int
-nfs3_access_resume (void *carg)
-{
-        nfsstat3                stat = NFS3ERR_SERVERFAULT;
-        int                     ret = -EFAULT;
-        nfs_user_t              nfu = {0, };
-        nfs3_call_state_t       *cs = NULL;
-
-        if (!carg) {
-                gf_log (GF_NFS3, GF_LOG_ERROR, "Invalid argument,"
-                        " carg value NULL");
-                return EINVAL;
-        }
-
-        cs = (nfs3_call_state_t *)carg;
-
-        /* Additional checks on the NFS file handle
-         * go here. The path for an NFS ACCESS call
-         * goes like this:
-         * nfs3_access -> nfs3_fh_resolve_and_resume -> nfs3_resolve_resume ->
-         * nfs3_access_resume -> <macro/function performs check on FH> ->
-         * <continue or return from function based on check.> ('goto nfs3err'
-         * terminates this function and writes the appropriate response to the
-         * client). It is important that you do NOT stick any sort of check
-         * on the file handle outside of the nfs3_##OP_resume functions.
-         */
-        nfs3_check_fh_auth_status (cs, stat, _gf_false, nfs3err);
-        nfs3_check_fh_resolve_status (cs, stat, nfs3err);
-        cs->fh = cs->resolvefh;
-        nfs_request_user_init (&nfu, cs->req);
-        ret = nfs_access (cs->nfsx, cs->vol, &nfu, &cs->resolvedloc,
-                          cs->accessbits, nfs3svc_access_cbk, cs);
-        if (ret < 0)
-                stat = nfs3_errno_to_nfsstat3 (-ret);
+nfs3_access_resume(void *carg)
+{
+    nfsstat3 stat = NFS3ERR_SERVERFAULT;
+    int ret = -EFAULT;
+    nfs_user_t nfu = {
+        0,
+    };
+    nfs3_call_state_t *cs = NULL;
+
+    if (!carg) {
+        gf_msg(GF_NFS3, GF_LOG_ERROR, EINVAL, NFS_MSG_INVALID_ENTRY,
+               "Invalid argument, carg value NULL");
+        return EINVAL;
+    }
+
+    cs = (nfs3_call_state_t *)carg;
+
+    /* Additional checks on the NFS file handle
+     * go here. The path for an NFS ACCESS call
+     * goes like this:
+     * nfs3_access -> nfs3_fh_resolve_and_resume -> nfs3_resolve_resume ->
+     * nfs3_access_resume -> <macro/function performs check on FH> ->
+     * <continue or return from function based on check.> ('goto nfs3err'
+     * terminates this function and writes the appropriate response to the
+     * client). It is important that you do NOT stick any sort of check
+     * on the file handle outside of the nfs3_##OP_resume functions.
+     */
+    nfs3_check_fh_auth_status(cs, stat, _gf_false, nfs3err);
+    nfs3_check_fh_resolve_status(cs, stat, nfs3err);
+    cs->fh = cs->resolvefh;
+    nfs_request_user_init(&nfu, cs->req);
+    ret = nfs_access(cs->nfsx, cs->vol, &nfu, &cs->resolvedloc, cs->accessbits,
+                     nfs3svc_access_cbk, cs);
+    if (ret < 0)
+        stat = nfs3_errno_to_nfsstat3(-ret);
 
 nfs3err:
-        if (ret < 0) {
-                nfs3_log_common_res (rpcsvc_request_xid (cs->req), NFS3_ACCESS,
-                                     stat, -ret);
-                nfs3_access_reply (cs->req, stat, 0, 0);
-                nfs3_call_state_wipe (cs);
-                ret = 0;
-        }
+    if (ret < 0) {
+        nfs3_log_common_res(rpcsvc_request_xid(cs->req), NFS3_ACCESS, stat,
+                            -ret, cs->resolvedloc.path);
+        nfs3_access_reply(cs->req, stat, 0, 0);
+        nfs3_call_state_wipe(cs);
+        ret = 0;
+    }
 
-        return ret;
+    return ret;
 }
 
-
 int
-nfs3_access (rpcsvc_request_t *req, struct nfs3_fh *fh, uint32_t accbits)
-{
-        xlator_t                *vol = NULL;
-        struct nfs3_state       *nfs3 = NULL;
-        nfsstat3                stat = NFS3ERR_SERVERFAULT;
-        int                     ret = -EFAULT;
-        nfs3_call_state_t       *cs = NULL;
-
-        GF_VALIDATE_OR_GOTO (GF_NFS, req, out);
-        GF_VALIDATE_OR_GOTO (GF_NFS, fh, out);
-        nfs3_log_common_call (rpcsvc_request_xid (req), "ACCESS", fh);
-        nfs3_validate_gluster_fh (fh, stat, nfs3err);
-        nfs3_validate_nfs3_state (req, nfs3, stat, nfs3err, ret);
-        nfs3_map_fh_to_volume (nfs3, fh, req, vol, stat, nfs3err);
-        nfs3_volume_started_check (nfs3, vol, ret, out);
-        nfs3_handle_call_state_init (nfs3, cs, req, vol, stat, nfs3err);
-        cs->accessbits = accbits;
-
-        ret = nfs3_fh_resolve_and_resume (cs, fh, NULL, nfs3_access_resume);
-        if (ret < 0)
-                stat = nfs3_errno_to_nfsstat3 (-ret);
+nfs3_access(rpcsvc_request_t *req, struct nfs3_fh *fh, uint32_t accbits)
+{
+    xlator_t *vol = NULL;
+    struct nfs3_state *nfs3 = NULL;
+    nfsstat3 stat = NFS3ERR_SERVERFAULT;
+    int ret = -EFAULT;
+    nfs3_call_state_t *cs = NULL;
+
+    GF_VALIDATE_OR_GOTO(GF_NFS, req, out);
+    GF_VALIDATE_OR_GOTO(GF_NFS, fh, out);
+    nfs3_log_common_call(rpcsvc_request_xid(req), "ACCESS", fh);
+    nfs3_validate_gluster_fh(fh, stat, nfs3err);
+    nfs3_validate_nfs3_state(req, nfs3, stat, nfs3err, ret);
+    nfs3_map_fh_to_volume(nfs3, fh, req, vol, stat, nfs3err);
+    nfs3_volume_started_check(nfs3, vol, ret, out);
+    nfs3_handle_call_state_init(nfs3, cs, req, vol, stat, nfs3err);
+    cs->accessbits = accbits;
+
+    ret = nfs3_fh_resolve_and_resume(cs, fh, NULL, nfs3_access_resume);
+    if (ret < 0)
+        stat = nfs3_errno_to_nfsstat3(-ret);
 
 nfs3err:
-        if (ret < 0) {
-                nfs3_log_common_res (rpcsvc_request_xid (req), NFS3_ACCESS,
-                                     stat, -ret);
-                nfs3_access_reply (req, stat, 0, 0);
-                nfs3_call_state_wipe (cs);
-                ret = 0;
-        }
+    if (ret < 0) {
+        nfs3_log_common_res(rpcsvc_request_xid(req), NFS3_ACCESS, stat, -ret,
+                            cs ? cs->resolvedloc.path : NULL);
+        nfs3_access_reply(req, stat, 0, 0);
+        nfs3_call_state_wipe(cs);
+        ret = 0;
+    }
 out:
-        return ret;
+    return ret;
 }
 
-
 int
-nfs3svc_access (rpcsvc_request_t *req)
+nfs3svc_access(rpcsvc_request_t *req)
 {
-        struct nfs3_fh  fh = {{0}, };
-        access3args     args;
-        int             ret = RPCSVC_ACTOR_ERROR;
+    struct nfs3_fh fh = {
+        {0},
+    };
+    access3args args;
+    int ret = RPCSVC_ACTOR_ERROR;
 
-        if (!req)
-                return ret;
-
-        nfs3_prep_access3args (&args, &fh);
-        if (xdr_to_access3args (req->msg[0], &args) <= 0) {
-                gf_log (GF_NFS3, GF_LOG_ERROR, "Error decoding args");
-                rpcsvc_request_seterr (req, GARBAGE_ARGS);
-                goto rpcerr;
-        }
+    if (!req)
+        return ret;
 
-        ret = nfs3_access (req, &fh, args.access);
-        if ((ret < 0) && (ret != RPCSVC_ACTOR_IGNORE)) {
-                gf_log (GF_NFS3, GF_LOG_ERROR, "ACCESS procedure failed");
-                rpcsvc_request_seterr (req, SYSTEM_ERR);
-                ret = RPCSVC_ACTOR_ERROR;
-        }
+    nfs3_prep_access3args(&args, &fh);
+    if (xdr_to_access3args(req->msg[0], &args) <= 0) {
+        gf_msg(GF_NFS3, GF_LOG_ERROR, 0, NFS_MSG_ARGS_DECODE_ERROR,
+               "Error decoding args");
+        rpcsvc_request_seterr(req, GARBAGE_ARGS);
+        goto rpcerr;
+    }
+
+    ret = nfs3_access(req, &fh, args.access);
+    if ((ret < 0) && (ret != RPCSVC_ACTOR_IGNORE)) {
+        gf_msg(GF_NFS3, GF_LOG_ERROR, -ret, NFS_MSG_ACCESS_PROC_FAIL,
+               "ACCESS procedure failed");
+        rpcsvc_request_seterr(req, SYSTEM_ERR);
+        ret = RPCSVC_ACTOR_ERROR;
+    }
 
 rpcerr:
-        return ret;
+    return ret;
 }
 
-
 int
-nfs3_readlink_reply (rpcsvc_request_t *req, nfsstat3 stat, char *path,
-                     struct iatt *buf)
+nfs3_readlink_reply(rpcsvc_request_t *req, nfsstat3 stat, char *path,
+                    struct iatt *buf)
 {
-        readlink3res    res = {0, };
-        uint64_t        deviceid = 0;
+    readlink3res res = {
+        0,
+    };
+    uint64_t deviceid = 0;
 
-        deviceid = nfs3_request_xlator_deviceid (req);
-        nfs3_fill_readlink3res (&res, stat, path, buf, deviceid);
-        nfs3svc_submit_reply (req, (void *)&res,
-                              (nfs3_serializer)xdr_serialize_readlink3res);
+    deviceid = nfs3_request_xlator_deviceid(req);
+    nfs3_fill_readlink3res(&res, stat, path, buf, deviceid);
+    nfs3svc_submit_reply(req, (void *)&res,
+                         (nfs3_serializer)xdr_serialize_readlink3res);
 
-        return 0;
+    return 0;
 }
 
-
 int32_t
-nfs3svc_readlink_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                      int32_t op_ret, int32_t op_errno, const char *path,
-                      struct iatt *buf, dict_t *xdata)
-{
-        nfsstat3                stat = NFS3ERR_SERVERFAULT;
-        nfs3_call_state_t       *cs = NULL;
-
-        cs = frame->local;
-        if (op_ret == -1) {
-                gf_log (GF_NFS, GF_LOG_WARNING,
-                        "%x: %s => -1 (%s)", rpcsvc_request_xid (cs->req),
-                        cs->resolvedloc.path, strerror (op_errno));
-                stat = nfs3_cbk_errno_status (op_ret, op_errno);
-                goto nfs3err;
-        }
+nfs3svc_readlink_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                     int32_t op_ret, int32_t op_errno, const char *path,
+                     struct iatt *buf, dict_t *xdata)
+{
+    nfsstat3 stat = NFS3ERR_SERVERFAULT;
+    nfs3_call_state_t *cs = NULL;
 
-        stat = NFS3_OK;
+    cs = frame->local;
+    if (op_ret == -1) {
+        stat = nfs3_cbk_errno_status(op_ret, op_errno);
+        goto nfs3err;
+    }
+
+    stat = NFS3_OK;
 
 nfs3err:
-        nfs3_log_readlink_res (rpcsvc_request_xid (cs->req), stat, op_errno,
-                               (char *)path);
-        nfs3_readlink_reply (cs->req, stat, (char *)path, buf);
-        nfs3_call_state_wipe (cs);
+    nfs3_log_readlink_res(rpcsvc_request_xid(cs->req), stat, op_errno,
+                          (char *)path, cs->resolvedloc.path);
+    nfs3_readlink_reply(cs->req, stat, (char *)path, buf);
+    nfs3_call_state_wipe(cs);
 
-        return 0;
+    return 0;
 }
 
-
 int
-nfs3_readlink_resume (void *carg)
-{
-        nfsstat3                        stat = NFS3ERR_SERVERFAULT;
-        int                             ret = -EFAULT;
-        nfs3_call_state_t               *cs = NULL;
-        nfs_user_t                      nfu = {0, };
-
-        if (!carg)
-                return ret;
-
-        cs = (nfs3_call_state_t *)carg;
-        nfs3_check_fh_auth_status (cs, stat, _gf_false, nfs3err);
-        nfs3_check_fh_resolve_status (cs, stat, nfs3err);
-        nfs_request_user_init (&nfu, cs->req);
-        ret = nfs_readlink (cs->nfsx, cs->vol, &nfu, &cs->resolvedloc,
-                            nfs3svc_readlink_cbk, cs);
-        if (ret < 0)
-                stat = nfs3_errno_to_nfsstat3 (-ret);
-
-nfs3err:
-        if (ret < 0) {
-                nfs3_log_common_res (rpcsvc_request_xid (cs->req),
-                                     NFS3_READLINK, stat, -ret);
-                nfs3_readlink_reply (cs->req, stat, NULL, NULL);
-                nfs3_call_state_wipe (cs);
-        }
+nfs3_readlink_resume(void *carg)
+{
+    nfsstat3 stat = NFS3ERR_SERVERFAULT;
+    int ret = -EFAULT;
+    nfs3_call_state_t *cs = NULL;
+    nfs_user_t nfu = {
+        0,
+    };
 
+    if (!carg)
         return ret;
-}
-
 
-int
-nfs3_readlink (rpcsvc_request_t *req, struct nfs3_fh *fh)
-{
-        xlator_t                        *vol = NULL;
-        nfsstat3                        stat = NFS3ERR_SERVERFAULT;
-        int                             ret = -EFAULT;
-        struct nfs3_state               *nfs3 = NULL;
-        nfs3_call_state_t               *cs = NULL;
+    cs = (nfs3_call_state_t *)carg;
+    nfs3_check_fh_auth_status(cs, stat, _gf_false, nfs3err);
+    nfs3_check_fh_resolve_status(cs, stat, nfs3err);
+    nfs_request_user_init(&nfu, cs->req);
+    ret = nfs_readlink(cs->nfsx, cs->vol, &nfu, &cs->resolvedloc,
+                       nfs3svc_readlink_cbk, cs);
+    if (ret < 0)
+        stat = nfs3_errno_to_nfsstat3(-ret);
 
-        if ((!req) || (!fh)) {
-                gf_log (GF_NFS3, GF_LOG_ERROR, "Bad arguments");
-                return -1;
-        }
+nfs3err:
+    if (ret < 0) {
+        nfs3_log_common_res(rpcsvc_request_xid(cs->req), NFS3_READLINK, stat,
+                            -ret, cs->resolvedloc.path);
+        nfs3_readlink_reply(cs->req, stat, NULL, NULL);
+        nfs3_call_state_wipe(cs);
+    }
 
-        nfs3_log_common_call (rpcsvc_request_xid (req), "READLINK", fh);
-        nfs3_validate_gluster_fh (fh, stat, nfs3err);
-        nfs3_validate_nfs3_state (req, nfs3, stat, nfs3err, ret);
-        nfs3_map_fh_to_volume (nfs3, fh, req, vol, stat, nfs3err);
-        nfs3_volume_started_check (nfs3, vol, ret, out);
-        nfs3_handle_call_state_init (nfs3, cs, req, vol, stat, nfs3err);
+    return ret;
+}
 
-        ret = nfs3_fh_resolve_and_resume (cs, fh, NULL, nfs3_readlink_resume);
-        if (ret < 0)
-                stat = nfs3_errno_to_nfsstat3 (-ret);
+int
+nfs3_readlink(rpcsvc_request_t *req, struct nfs3_fh *fh)
+{
+    xlator_t *vol = NULL;
+    nfsstat3 stat = NFS3ERR_SERVERFAULT;
+    int ret = -EFAULT;
+    struct nfs3_state *nfs3 = NULL;
+    nfs3_call_state_t *cs = NULL;
+
+    if ((!req) || (!fh)) {
+        gf_msg(GF_NFS3, GF_LOG_ERROR, EINVAL, NFS_MSG_INVALID_ENTRY,
+               "Bad arguments");
+        return -1;
+    }
+
+    nfs3_log_common_call(rpcsvc_request_xid(req), "READLINK", fh);
+    nfs3_validate_gluster_fh(fh, stat, nfs3err);
+    nfs3_validate_nfs3_state(req, nfs3, stat, nfs3err, ret);
+    nfs3_map_fh_to_volume(nfs3, fh, req, vol, stat, nfs3err);
+    nfs3_volume_started_check(nfs3, vol, ret, out);
+    nfs3_handle_call_state_init(nfs3, cs, req, vol, stat, nfs3err);
+
+    ret = nfs3_fh_resolve_and_resume(cs, fh, NULL, nfs3_readlink_resume);
+    if (ret < 0)
+        stat = nfs3_errno_to_nfsstat3(-ret);
 
 nfs3err:
-        if (ret < 0) {
-                nfs3_log_common_res (rpcsvc_request_xid (req), NFS3_READLINK,
-                                     stat, -ret);
-                nfs3_readlink_reply (req, stat, NULL, NULL);
-                nfs3_call_state_wipe (cs);
-                /* Ret must be 0 after this so that the caller does not
-                 * also send an RPC reply.
-                 */
-                ret = 0;
-        }
+    if (ret < 0) {
+        nfs3_log_common_res(rpcsvc_request_xid(req), NFS3_READLINK, stat, -ret,
+                            cs ? cs->resolvedloc.path : NULL);
+        nfs3_readlink_reply(req, stat, NULL, NULL);
+        nfs3_call_state_wipe(cs);
+        /* Ret must be 0 after this so that the caller does not
+         * also send an RPC reply.
+         */
+        ret = 0;
+    }
 out:
-        return ret;
+    return ret;
 }
 
-
 int
-nfs3svc_readlink (rpcsvc_request_t *req)
+nfs3svc_readlink(rpcsvc_request_t *req)
 {
-        struct nfs3_fh          fh = {{0}, };
-        readlink3args           args;
-        int                     ret = RPCSVC_ACTOR_ERROR;
+    struct nfs3_fh fh = {
+        {0},
+    };
+    readlink3args args;
+    int ret = RPCSVC_ACTOR_ERROR;
 
-        if (!req)
-                return ret;
+    if (!req)
+        return ret;
 
-        nfs3_prep_readlink3args (&args, &fh);
-        if (xdr_to_readlink3args (req->msg[0], &args) <= 0) {
-                gf_log (GF_NFS3, GF_LOG_ERROR, "Error decoding args");
-                rpcsvc_request_seterr (req, GARBAGE_ARGS);
-                goto rpcerr;
-        }
-
-        ret = nfs3_readlink (req, &fh);
-        if ((ret < 0) && (ret != RPCSVC_ACTOR_IGNORE)) {
-                gf_log (GF_NFS3, GF_LOG_ERROR, "READLINK procedure failed");
-                rpcsvc_request_seterr (req, SYSTEM_ERR);
-                ret = RPCSVC_ACTOR_ERROR;
-        }
+    nfs3_prep_readlink3args(&args, &fh);
+    if (xdr_to_readlink3args(req->msg[0], &args) <= 0) {
+        gf_msg(GF_NFS3, GF_LOG_ERROR, 0, NFS_MSG_ARGS_DECODE_ERROR,
+               "Error decoding args");
+        rpcsvc_request_seterr(req, GARBAGE_ARGS);
+        goto rpcerr;
+    }
+
+    ret = nfs3_readlink(req, &fh);
+    if ((ret < 0) && (ret != RPCSVC_ACTOR_IGNORE)) {
+        gf_msg(GF_NFS3, GF_LOG_ERROR, -ret, NFS_MSG_READLINK_PROC_FAIL,
+               "READLINK procedure failed");
+        rpcsvc_request_seterr(req, SYSTEM_ERR);
+        ret = RPCSVC_ACTOR_ERROR;
+    }
 
 rpcerr:
-        return ret;
+    return ret;
 }
 
-
 int
-nfs3_read_reply (rpcsvc_request_t *req, nfsstat3 stat, count3 count,
-                 struct iovec *vec, int vcount, struct iobref *iobref,
-                 struct iatt *poststat, int is_eof)
-{
-        read3res                res = {0, };
-        uint64_t                deviceid = 0;
-
-        deviceid = nfs3_request_xlator_deviceid (req);
-        nfs3_fill_read3res (&res, stat, count, poststat, is_eof, deviceid);
-        if (stat == NFS3_OK) {
-                xdr_vector_round_up (vec, vcount, count);
-                /* iob can be zero if the file size was zero. If so, op_ret
-                 * would be 0 and count = 0.
-                 */
-
-                if (count != 0) {
-                        nfs3svc_submit_vector_reply (req, (void *)&res,
-                                                     (nfs3_serializer)
-                                                  xdr_serialize_read3res_nocopy,
-                                                    vec, vcount, iobref);
-                } else
-
-                nfs3svc_submit_reply (req, (void *)&res,
-                                              (nfs3_serializer)
-                                              xdr_serialize_read3res_nocopy);
+nfs3_read_reply(rpcsvc_request_t *req, nfsstat3 stat, count3 count,
+                struct iovec *vec, int vcount, struct iobref *iobref,
+                struct iatt *poststat, int is_eof)
+{
+    read3res res = {
+        0,
+    };
+    uint64_t deviceid = 0;
+
+    deviceid = nfs3_request_xlator_deviceid(req);
+    nfs3_fill_read3res(&res, stat, count, poststat, is_eof, deviceid);
+    if (stat == NFS3_OK) {
+        xdr_vector_round_up(vec, vcount, count);
+        /* iob can be zero if the file size was zero. If so, op_ret
+         * would be 0 and count = 0.
+         */
+
+        if (count != 0) {
+            nfs3svc_submit_vector_reply(
+                req, (void *)&res,
+                (nfs3_serializer)xdr_serialize_read3res_nocopy, vec, vcount,
+                iobref);
         } else
-                nfs3svc_submit_reply (req, (void *)&res,
-                                      (nfs3_serializer)
-                                      xdr_serialize_read3res_nocopy);
 
-        return 0;
-}
+            nfs3svc_submit_reply(
+                req, (void *)&res,
+                (nfs3_serializer)xdr_serialize_read3res_nocopy);
+    } else
+        nfs3svc_submit_reply(req, (void *)&res,
+                             (nfs3_serializer)xdr_serialize_read3res_nocopy);
 
+    return 0;
+}
 
 int32_t
-nfs3svc_read_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                  int32_t op_ret, int32_t op_errno, struct iovec *vector,
-                  int32_t count, struct iatt *stbuf, struct iobref *iobref,
-                  dict_t *xdata)
-{
-        nfsstat3                stat = NFS3ERR_SERVERFAULT;
-        int                     is_eof = 0;
-        nfs3_call_state_t       *cs = NULL;
-
-        cs = frame->local;
-        if (op_ret == -1) {
-                gf_log (GF_NFS, GF_LOG_WARNING,
-                        "%x: %s => -1 (%s)", rpcsvc_request_xid (cs->req),
-                        cs->resolvedloc.path, strerror (op_errno));
-                stat = nfs3_cbk_errno_status (op_ret, op_errno);
-                goto err;
-        } else
-                stat = NFS3_OK;
+nfs3svc_read_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                 int32_t op_ret, int32_t op_errno, struct iovec *vector,
+                 int32_t count, struct iatt *stbuf, struct iobref *iobref,
+                 dict_t *xdata)
+{
+    nfsstat3 stat = NFS3ERR_SERVERFAULT;
+    int is_eof = 0;
+    nfs3_call_state_t *cs = NULL;
+
+    cs = frame->local;
+    if (op_ret == -1) {
+        stat = nfs3_cbk_errno_status(op_ret, op_errno);
+        goto err;
+    } else
+        stat = NFS3_OK;
 
-        if (op_errno == ENOENT)
-                is_eof = 1;
+    if (op_errno == ENOENT)
+        is_eof = 1;
 
 err:
-        nfs3_log_read_res (rpcsvc_request_xid (cs->req), stat, op_errno,
-                           op_ret, is_eof, vector, count);
-        nfs3_read_reply (cs->req, stat, op_ret, vector, count, iobref, stbuf,
-                         is_eof);
-        nfs3_call_state_wipe (cs);
+    nfs3_log_read_res(rpcsvc_request_xid(cs->req), stat, op_errno, op_ret,
+                      is_eof, vector, count, cs->resolvedloc.path);
+    nfs3_read_reply(cs->req, stat, op_ret, vector, count, iobref, stbuf,
+                    is_eof);
+    nfs3_call_state_wipe(cs);
 
-        return 0;
+    return 0;
 }
 
-
 int
-nfs3_read_fd_resume (void *carg)
+nfs3_read_fd_resume(void *carg)
 {
-        nfsstat3                        stat = NFS3ERR_SERVERFAULT;
-        int                             ret = -EFAULT;
-        nfs_user_t                      nfu = {0, };
-        nfs3_call_state_t               *cs = NULL;
+    nfsstat3 stat = NFS3ERR_SERVERFAULT;
+    int ret = -EFAULT;
+    nfs_user_t nfu = {
+        0,
+    };
+    nfs3_call_state_t *cs = NULL;
 
-        if (!carg)
-                return ret;
+    if (!carg)
+        return ret;
 
-        cs = (nfs3_call_state_t *)carg;
-        nfs3_check_fh_resolve_status (cs, stat, nfs3err);
-        nfs_request_user_init (&nfu, cs->req);
-        ret = nfs_read (cs->nfsx, cs->vol, &nfu, cs->fd, cs->datacount,
-                        cs->dataoffset, nfs3svc_read_cbk, cs);
-        if (ret < 0)
-                stat = nfs3_errno_to_nfsstat3 (-ret);
+    cs = (nfs3_call_state_t *)carg;
+    nfs3_check_fh_resolve_status(cs, stat, nfs3err);
+    nfs_request_user_init(&nfu, cs->req);
+    ret = nfs_read(cs->nfsx, cs->vol, &nfu, cs->fd, cs->datacount,
+                   cs->dataoffset, nfs3svc_read_cbk, cs);
+    if (ret < 0)
+        stat = nfs3_errno_to_nfsstat3(-ret);
 nfs3err:
-        if (ret < 0) {
-                nfs3_log_common_res (rpcsvc_request_xid (cs->req), NFS3_READ,
-                                     stat, -ret);
-                nfs3_read_reply (cs->req, stat, 0, NULL, 0, NULL, NULL, 0);
-                nfs3_call_state_wipe (cs);
-        }
+    if (ret < 0) {
+        nfs3_log_common_res(rpcsvc_request_xid(cs->req), NFS3_READ, stat, -ret,
+                            cs->resolvedloc.path);
+        nfs3_read_reply(cs->req, stat, 0, NULL, 0, NULL, NULL, 0);
+        nfs3_call_state_wipe(cs);
+    }
 
-        return ret;
+    return ret;
 }
 
-
 int
-nfs3_read_resume (void *carg)
-{
-        nfsstat3                        stat = NFS3ERR_SERVERFAULT;
-        int                             ret = -EFAULT;
-        nfs3_call_state_t               *cs = NULL;
-        fd_t                            *fd = NULL;
-
-        if (!carg)
-                return ret;
-
-        cs = (nfs3_call_state_t *)carg;
-        nfs3_check_fh_auth_status (cs, stat, _gf_false, nfs3err);
-        nfs3_check_fh_resolve_status (cs, stat, nfs3err);
-        fd = fd_anonymous (cs->resolvedloc.inode);
-        if (!fd) {
-                gf_log (GF_NFS3, GF_LOG_ERROR, "Failed to create anonymous fd");
-                goto nfs3err;
-        }
+nfs3_read_resume(void *carg)
+{
+    nfsstat3 stat = NFS3ERR_SERVERFAULT;
+    int ret = -EFAULT;
+    nfs3_call_state_t *cs = NULL;
+    fd_t *fd = NULL;
 
-        cs->fd = fd;
-        nfs3_read_fd_resume (cs);
-        ret = 0;
+    if (!carg)
+        return ret;
+
+    cs = (nfs3_call_state_t *)carg;
+    nfs3_check_fh_auth_status(cs, stat, _gf_false, nfs3err);
+    nfs3_check_fh_resolve_status(cs, stat, nfs3err);
+    fd = fd_anonymous(cs->resolvedloc.inode);
+    if (!fd) {
+        gf_msg(GF_NFS3, GF_LOG_ERROR, 0, NFS_MSG_ANONYMOUS_FD_FAIL,
+               "Failed to create anonymous fd");
+        goto nfs3err;
+    }
+
+    cs->fd = fd;
+    nfs3_read_fd_resume(cs);
+    ret = 0;
 nfs3err:
-        if (ret < 0) {
-                nfs3_log_common_res (rpcsvc_request_xid (cs->req), NFS3_READ,
-                                     stat, -ret);
-                nfs3_read_reply (cs->req, stat, 0, NULL,0, NULL, NULL, 0);
-                nfs3_call_state_wipe (cs);
-        }
+    if (ret < 0) {
+        nfs3_log_common_res(rpcsvc_request_xid(cs->req), NFS3_READ, stat, -ret,
+                            cs->resolvedloc.path);
+        nfs3_read_reply(cs->req, stat, 0, NULL, 0, NULL, NULL, 0);
+        nfs3_call_state_wipe(cs);
+    }
 
-        return ret;
+    return ret;
 }
 
 int
-nfs3_read (rpcsvc_request_t *req, struct nfs3_fh *fh, offset3 offset,
-           count3 count)
-{
-        xlator_t                        *vol = NULL;
-        nfsstat3                        stat = NFS3ERR_SERVERFAULT;
-        int                             ret = -EFAULT;
-        struct nfs3_state               *nfs3 = NULL;
-        nfs3_call_state_t               *cs = NULL;
-
-        if ((!req) || (!fh)) {
-                gf_log (GF_NFS3, GF_LOG_ERROR, "Bad arguments");
-                return -1;
-        }
-
-        nfs3_log_rw_call (rpcsvc_request_xid (req), "READ", fh, offset,
-                          count, -1);
-        nfs3_validate_gluster_fh (fh, stat, nfs3err);
-        nfs3_validate_nfs3_state (req, nfs3, stat, nfs3err, ret);
-        nfs3_map_fh_to_volume (nfs3, fh, req, vol, stat, nfs3err);
-        nfs3_volume_started_check (nfs3, vol, ret, out);
-        nfs3_handle_call_state_init (nfs3, cs, req, vol, stat, nfs3err);
-
-        cs->datacount = count;
-        cs->dataoffset = offset;
-        ret = nfs3_fh_resolve_and_resume (cs, fh, NULL, nfs3_read_resume);
-        if (ret < 0)
-                stat = nfs3_errno_to_nfsstat3 (-ret);
+nfs3_read(rpcsvc_request_t *req, struct nfs3_fh *fh, offset3 offset,
+          count3 count)
+{
+    xlator_t *vol = NULL;
+    nfsstat3 stat = NFS3ERR_SERVERFAULT;
+    int ret = -EFAULT;
+    struct nfs3_state *nfs3 = NULL;
+    nfs3_call_state_t *cs = NULL;
+
+    if ((!req) || (!fh)) {
+        gf_msg(GF_NFS3, GF_LOG_ERROR, EINVAL, NFS_MSG_INVALID_ENTRY,
+               "Bad arguments");
+        return -1;
+    }
+
+    nfs3_log_rw_call(rpcsvc_request_xid(req), "READ", fh, offset, count, -1);
+    nfs3_validate_gluster_fh(fh, stat, nfs3err);
+    nfs3_validate_nfs3_state(req, nfs3, stat, nfs3err, ret);
+    nfs3_map_fh_to_volume(nfs3, fh, req, vol, stat, nfs3err);
+    nfs3_volume_started_check(nfs3, vol, ret, out);
+    nfs3_handle_call_state_init(nfs3, cs, req, vol, stat, nfs3err);
+
+    cs->datacount = count;
+    cs->dataoffset = offset;
+    ret = nfs3_fh_resolve_and_resume(cs, fh, NULL, nfs3_read_resume);
+    if (ret < 0)
+        stat = nfs3_errno_to_nfsstat3(-ret);
 
 nfs3err:
-        if (ret < 0) {
-                nfs3_log_common_res (rpcsvc_request_xid (req), NFS3_READ, stat,
-                                     -ret);
-                nfs3_read_reply (req, stat, 0, NULL,0, NULL, NULL, 0);
-                nfs3_call_state_wipe (cs);
-                ret = 0;
-        }
+    if (ret < 0) {
+        nfs3_log_common_res(rpcsvc_request_xid(req), NFS3_READ, stat, -ret,
+                            cs ? cs->resolvedloc.path : NULL);
+        nfs3_read_reply(req, stat, 0, NULL, 0, NULL, NULL, 0);
+        nfs3_call_state_wipe(cs);
+        ret = 0;
+    }
 out:
-        return ret;
+    return ret;
 }
 
-
 int
-nfs3svc_read (rpcsvc_request_t *req)
+nfs3svc_read(rpcsvc_request_t *req)
 {
-        struct nfs3_fh          fh = {{0}, };
-        read3args               args;
-        int                     ret = RPCSVC_ACTOR_ERROR;
-
-        if (!req)
-                return ret;
+    struct nfs3_fh fh = {
+        {0},
+    };
+    read3args args;
+    int ret = RPCSVC_ACTOR_ERROR;
 
-        nfs3_prep_read3args (&args, &fh);
-        if (xdr_to_read3args (req->msg[0], &args) <= 0) {
-                gf_log (GF_NFS3, GF_LOG_ERROR, "Error decoding args");
-                rpcsvc_request_seterr (req, GARBAGE_ARGS);
-                goto rpcerr;
-        }
+    if (!req)
+        return ret;
 
-        ret = nfs3_read (req, &fh, args.offset, args.count);
-        if ((ret < 0) && (ret != RPCSVC_ACTOR_IGNORE)) {
-                gf_log (GF_NFS3, GF_LOG_ERROR, "READ procedure failed");
-                rpcsvc_request_seterr (req, SYSTEM_ERR);
-                ret = RPCSVC_ACTOR_ERROR;
-        }
+    nfs3_prep_read3args(&args, &fh);
+    if (xdr_to_read3args(req->msg[0], &args) <= 0) {
+        gf_msg(GF_NFS3, GF_LOG_ERROR, 0, NFS_MSG_ARGS_DECODE_ERROR,
+               "Error decoding args");
+        rpcsvc_request_seterr(req, GARBAGE_ARGS);
+        goto rpcerr;
+    }
+
+    ret = nfs3_read(req, &fh, args.offset, args.count);
+    if ((ret < 0) && (ret != RPCSVC_ACTOR_IGNORE)) {
+        gf_msg(GF_NFS3, GF_LOG_ERROR, -ret, NFS_MSG_READ_FAIL,
+               "READ procedure failed");
+        rpcsvc_request_seterr(req, SYSTEM_ERR);
+        ret = RPCSVC_ACTOR_ERROR;
+    }
 
 rpcerr:
-        return ret;
+    return ret;
 }
 
-
 int
-nfs3_write_reply (rpcsvc_request_t *req, nfsstat3 stat, count3 count,
-                  stable_how stable, uint64_t wverf, struct iatt *prestat,
-                  struct iatt *poststat)
+nfs3_write_reply(rpcsvc_request_t *req, nfsstat3 stat, count3 count,
+                 stable_how stable, uint64_t wverf, struct iatt *prestat,
+                 struct iatt *poststat)
 {
-        write3res       res = {0, };
-        uint64_t        deviceid = 0;
+    write3res res = {
+        0,
+    };
+    uint64_t deviceid = 0;
 
-        deviceid = nfs3_request_xlator_deviceid (req);
-        nfs3_fill_write3res (&res, stat, count, stable, wverf, prestat,
-                             poststat, deviceid);
-        nfs3svc_submit_reply (req, (void *)&res,
-                              (nfs3_serializer)xdr_serialize_write3res);
+    deviceid = nfs3_request_xlator_deviceid(req);
+    nfs3_fill_write3res(&res, stat, count, stable, wverf, prestat, poststat,
+                        deviceid);
+    nfs3svc_submit_reply(req, (void *)&res,
+                         (nfs3_serializer)xdr_serialize_write3res);
 
-        return 0;
+    return 0;
 }
 
 int32_t
-nfs3svc_write_fsync_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                         int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
-                         struct iatt *postbuf, dict_t *xdata)
-{
-        struct nfs3_state       *nfs3 = NULL;
-        nfsstat3                stat = NFS3ERR_SERVERFAULT;
-        nfs3_call_state_t       *cs = NULL;
-
-        cs = frame->local;
-        nfs3 = rpcsvc_request_program_private (cs->req);
-
-        if (op_ret == -1) {
-                gf_log (GF_NFS, GF_LOG_WARNING,
-                        "%x: %s => -1 (%s)", rpcsvc_request_xid (cs->req),
-                        cs->resolvedloc.path, strerror (op_errno));
-                stat = nfs3_cbk_errno_status (op_ret, op_errno);
-        } else
-                stat = NFS3_OK;
+nfs3svc_write_fsync_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                        int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
+                        struct iatt *postbuf, dict_t *xdata)
+{
+    struct nfs3_state *nfs3 = NULL;
+    nfsstat3 stat = NFS3ERR_SERVERFAULT;
+    nfs3_call_state_t *cs = NULL;
 
-        nfs3_log_write_res (rpcsvc_request_xid (cs->req), stat, op_errno,
-                            cs->maxcount, cs->writetype, nfs3->serverstart);
-        nfs3_write_reply (cs->req, stat, cs->maxcount, cs->writetype,
-                          nfs3->serverstart, &cs->stbuf, postbuf);
-        nfs3_call_state_wipe (cs);
-        return 0;
-}
+    cs = frame->local;
+    nfs3 = rpcsvc_request_program_private(cs->req);
 
+    if (op_ret == -1) {
+        stat = nfs3_cbk_errno_status(op_ret, op_errno);
+    } else
+        stat = NFS3_OK;
 
+    nfs3_log_write_res(rpcsvc_request_xid(cs->req), stat, op_errno,
+                       cs->maxcount, cs->writetype, nfs3->serverstart,
+                       cs->resolvedloc.path);
+    nfs3_write_reply(cs->req, stat, cs->maxcount, cs->writetype,
+                     nfs3->serverstart, &cs->stbuf, postbuf);
+    nfs3_call_state_wipe(cs);
+    return 0;
+}
 
 /*
  * Before going into the write reply logic, here is a matrix that shows the
@@ -2111,3077 +2205,3029 @@ nfs3svc_write_fsync_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
  *
  */
 int32_t
-nfs3svc_write_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                   int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
-                   struct iatt *postbuf, dict_t *xdata)
-{
-        nfsstat3                stat = NFS3ERR_SERVERFAULT;
-        nfs3_call_state_t       *cs = NULL;
-        struct nfs3_state       *nfs3 = NULL;
-
-        cs = frame->local;
-        nfs3 = rpcsvc_request_program_private (cs->req);
-        if (op_ret == -1) {
-                gf_log (GF_NFS, GF_LOG_WARNING,
-                        "%x: %s => -1 (%s)", rpcsvc_request_xid (cs->req),
-                        cs->resolvedloc.path, strerror (op_errno));
-                stat = nfs3_cbk_errno_status (op_ret, op_errno);
-                goto err;
-        }
+nfs3svc_write_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                  int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
+                  struct iatt *postbuf, dict_t *xdata)
+{
+    nfsstat3 stat = NFS3ERR_SERVERFAULT;
+    nfs3_call_state_t *cs = NULL;
+    struct nfs3_state *nfs3 = NULL;
 
-        stat = NFS3_OK;
-        cs->maxcount = op_ret;
+    cs = frame->local;
+    nfs3 = rpcsvc_request_program_private(cs->req);
+    if (op_ret == -1) {
+        stat = nfs3_cbk_errno_status(op_ret, op_errno);
+        goto err;
+    }
+
+    stat = NFS3_OK;
+    cs->maxcount = op_ret;
 
 err:
-	nfs3_log_write_res (rpcsvc_request_xid (cs->req), stat,
-			    op_errno, cs->maxcount, cs->writetype,
-			    nfs3->serverstart);
-	nfs3_write_reply (cs->req, stat, cs->maxcount,
-			  cs->writetype, nfs3->serverstart, prebuf,
-			  postbuf);
-	nfs3_call_state_wipe (cs);
+    nfs3_log_write_res(rpcsvc_request_xid(cs->req), stat, op_errno,
+                       cs->maxcount, cs->writetype, nfs3->serverstart,
+                       cs->resolvedloc.path);
+    nfs3_write_reply(cs->req, stat, cs->maxcount, cs->writetype,
+                     nfs3->serverstart, prebuf, postbuf);
+    nfs3_call_state_wipe(cs);
 
-        return 0;
+    return 0;
 }
 
-
 int
-__nfs3_write_resume (nfs3_call_state_t *cs)
-{
-        int                             ret = -EFAULT;
-        nfs_user_t                      nfu = {0, };
-
-        if (!cs)
-                return ret;
-
-        nfs_request_user_init (&nfu, cs->req);
-        /* It is possible that the RPC record contains more bytes than
-         * than the size of write requested in this request. This means,
-         * that in the RPC message buffer, there could be more bytes
-         * beyind the @count bytes. Since @payload is referring to the write
-         * data directly inside the RPC request buffer(..since we performed a
-         * no-copy deXDRing..), we might end up writing more data than
-         * requested, because till now payload.iov_len accounts for all the
-         * bytes not just the write request bytes. These extra bytes are present
-         * as a requirement of the XDR encoding to round up the all string and
-         * opaque data buffers to multiples of 4 bytes.
-         */
-        cs->datavec.iov_len = cs->datacount;
-        ret = nfs_write (cs->nfsx, cs->vol, &nfu, cs->fd, cs->iobref,
-                         &cs->datavec, 1, cs->dataoffset, nfs3svc_write_cbk,
-                         cs);
+__nfs3_write_resume(nfs3_call_state_t *cs)
+{
+    int ret = -EFAULT;
+    nfs_user_t nfu = {
+        0,
+    };
 
+    if (!cs)
         return ret;
-}
 
+    nfs_request_user_init(&nfu, cs->req);
+    /* It is possible that the RPC record contains more bytes than
+     * than the size of write requested in this request. This means,
+     * that in the RPC message buffer, there could be more bytes
+     * beyind the @count bytes. Since @payload is referring to the write
+     * data directly inside the RPC request buffer(..since we performed a
+     * no-copy deXDRing..), we might end up writing more data than
+     * requested, because till now payload.iov_len accounts for all the
+     * bytes not just the write request bytes. These extra bytes are present
+     * as a requirement of the XDR encoding to round up the all string and
+     * opaque data buffers to multiples of 4 bytes.
+     */
+    cs->datavec.iov_len = cs->datacount;
+    ret = nfs_write(cs->nfsx, cs->vol, &nfu, cs->fd, cs->iobref, &cs->datavec,
+                    1, cs->dataoffset, nfs3svc_write_cbk, cs);
+
+    return ret;
+}
 
 int
-nfs3_write_resume (void *carg)
-{
-        nfsstat3                stat = NFS3ERR_SERVERFAULT;
-        int                     ret = -EFAULT;
-        nfs3_call_state_t       *cs = NULL;
-        fd_t                    *fd = NULL;
-
-        if (!carg)
-                return ret;
-
-        cs = (nfs3_call_state_t *)carg;
-        nfs3_check_fh_auth_status (cs, stat, _gf_true, nfs3err);
-        nfs3_check_fh_resolve_status (cs, stat, nfs3err);
-        fd = fd_anonymous (cs->resolvedloc.inode);
-        if (!fd) {
-                gf_log (GF_NFS3, GF_LOG_ERROR, "Failed to create anonymous fd");
-                goto nfs3err;
-        }
+nfs3_write_resume(void *carg)
+{
+    nfsstat3 stat = NFS3ERR_SERVERFAULT;
+    int ret = -EFAULT;
+    nfs3_call_state_t *cs = NULL;
+    fd_t *fd = NULL;
 
-        cs->fd = fd;    /* Gets unrefd when the call state is wiped. */
+    if (!carg)
+        return ret;
 
-/*
-  enum stable_how {
-  UNSTABLE = 0,
-  DATA_SYNC = 1,
-  FILE_SYNC = 2,
-  };
-*/
-	switch (cs->writetype) {
-	case UNSTABLE:
-		break;
-	case DATA_SYNC:
-		fd->flags |= O_DSYNC;
-		break;
-	case FILE_SYNC:
-		fd->flags |= O_SYNC;
-		break;
-	}
-
-        ret = __nfs3_write_resume (cs);
-        if (ret < 0)
-                stat = nfs3_errno_to_nfsstat3 (-ret);
+    cs = (nfs3_call_state_t *)carg;
+    nfs3_check_fh_auth_status(cs, stat, _gf_true, nfs3err);
+    nfs3_check_fh_resolve_status(cs, stat, nfs3err);
+    fd = fd_anonymous(cs->resolvedloc.inode);
+    if (!fd) {
+        gf_msg(GF_NFS3, GF_LOG_ERROR, 0, NFS_MSG_ANONYMOUS_FD_FAIL,
+               "Failed to create anonymous fd");
+        goto nfs3err;
+    }
+
+    cs->fd = fd; /* Gets unrefd when the call state is wiped. */
+
+    ret = __nfs3_write_resume(cs);
+    if (ret < 0)
+        stat = nfs3_errno_to_nfsstat3(-ret);
 nfs3err:
-        if (ret < 0) {
-                nfs3_log_common_res (rpcsvc_request_xid (cs->req), NFS3_WRITE,
-                                     stat, -ret);
-                nfs3_write_reply (cs->req, stat, 0, cs->writetype, 0, NULL,
-                                  NULL);
-                nfs3_call_state_wipe (cs);
-        }
-        return ret;
+    if (ret < 0) {
+        nfs3_log_common_res(rpcsvc_request_xid(cs->req), NFS3_WRITE, stat, -ret,
+                            cs->resolvedloc.path);
+        nfs3_write_reply(cs->req, stat, 0, cs->writetype, 0, NULL, NULL);
+        nfs3_call_state_wipe(cs);
+    }
+    return ret;
 }
 
-
 int
-nfs3_write (rpcsvc_request_t *req, struct nfs3_fh *fh, offset3 offset,
-            count3 count, stable_how stable, struct iovec payload,
-            struct iobref *iobref)
-{
-        xlator_t                        *vol = NULL;
-        nfsstat3                        stat = NFS3ERR_SERVERFAULT;
-        int                             ret = -EFAULT;
-        struct nfs3_state               *nfs3 = NULL;
-        nfs3_call_state_t               *cs = NULL;
-
-        if ((!req) || (!fh) || (!payload.iov_base)) {
-                gf_log (GF_NFS3, GF_LOG_ERROR, "Bad arguments");
-                return -1;
-        }
-
-        nfs3_log_rw_call (rpcsvc_request_xid (req), "WRITE", fh, offset,
-                          count, stable);
-        nfs3_validate_gluster_fh (fh, stat, nfs3err);
-        nfs3_validate_nfs3_state (req, nfs3, stat, nfs3err, ret);
-        nfs3_map_fh_to_volume (nfs3, fh, req, vol, stat, nfs3err);
-        nfs3_volume_started_check (nfs3, vol, ret, out);
-        nfs3_check_rw_volaccess (nfs3, fh->exportid, stat, nfs3err);
-        nfs3_handle_call_state_init (nfs3, cs, req, vol, stat, nfs3err);
-        cs->datacount = count;
-        cs->dataoffset = offset;
-        cs->writetype = stable;
-        cs->iobref = iobref;
-        cs->datavec = payload;
-
-        ret = nfs3_fh_resolve_and_resume (cs, fh, NULL, nfs3_write_resume);
-        if (ret < 0)
-                stat = nfs3_errno_to_nfsstat3 (-ret);
+nfs3_write(rpcsvc_request_t *req, struct nfs3_fh *fh, offset3 offset,
+           count3 count, stable_how stable, struct iovec payload,
+           struct iobref *iobref)
+{
+    xlator_t *vol = NULL;
+    nfsstat3 stat = NFS3ERR_SERVERFAULT;
+    int ret = -EFAULT;
+    struct nfs3_state *nfs3 = NULL;
+    nfs3_call_state_t *cs = NULL;
+
+    if ((!req) || (!fh) || (!payload.iov_base)) {
+        gf_msg(GF_NFS3, GF_LOG_ERROR, EINVAL, NFS_MSG_INVALID_ENTRY,
+               "Bad arguments");
+        return -1;
+    }
+
+    nfs3_log_rw_call(rpcsvc_request_xid(req), "WRITE", fh, offset, count,
+                     stable);
+    nfs3_validate_gluster_fh(fh, stat, nfs3err);
+    nfs3_validate_nfs3_state(req, nfs3, stat, nfs3err, ret);
+    nfs3_map_fh_to_volume(nfs3, fh, req, vol, stat, nfs3err);
+    nfs3_volume_started_check(nfs3, vol, ret, out);
+    nfs3_check_rw_volaccess(nfs3, fh->exportid, stat, nfs3err);
+    nfs3_handle_call_state_init(nfs3, cs, req, vol, stat, nfs3err);
+    cs->datacount = count;
+    cs->dataoffset = offset;
+    cs->writetype = stable;
+    cs->iobref = iobref;
+    cs->datavec = payload;
+
+    ret = nfs3_fh_resolve_and_resume(cs, fh, NULL, nfs3_write_resume);
+    if (ret < 0)
+        stat = nfs3_errno_to_nfsstat3(-ret);
 
 nfs3err:
-        if (ret < 0) {
-                nfs3_log_common_res (rpcsvc_request_xid (req), NFS3_WRITE,
-                                     stat, -ret);
-                nfs3_write_reply (req, stat, 0, stable, 0, NULL, NULL);
-                nfs3_call_state_wipe (cs);
-                ret = 0;
-        }
+    if (ret < 0) {
+        nfs3_log_common_res(rpcsvc_request_xid(req), NFS3_WRITE, stat, -ret,
+                            cs ? cs->resolvedloc.path : NULL);
+        nfs3_write_reply(req, stat, 0, stable, 0, NULL, NULL);
+        nfs3_call_state_wipe(cs);
+        ret = 0;
+    }
 out:
-        return ret;
+    return ret;
 }
 
-#define NFS3_VECWRITE_READFHLEN         1
-#define NFS3_VECWRITE_READFH            2
-#define NFS3_VECWRITE_READREST          3
-
-#define NFS3_WRITE_POSTFH_SIZE          20
+#define NFS3_VECWRITE_READFHLEN 1
+#define NFS3_VECWRITE_READFH 2
+#define NFS3_VECWRITE_READREST 3
 
+#define NFS3_WRITE_POSTFH_SIZE 20
 
 int
-nfs3svc_write_vecsizer (int state, ssize_t *readsize, char *base_addr,
-                        char *curr_addr)
-{
-        int      ret     = 0;
-        uint32_t fhlen   = 0;
-        uint32_t fhlen_n = 0;
-
-        if (state == 0) {
-                ret = NFS3_VECWRITE_READFHLEN;
-                *readsize = 4;
-        } else if (state == NFS3_VECWRITE_READFHLEN) {
-                fhlen_n = *(uint32_t *)(curr_addr - 4);
-                fhlen = ntohl (fhlen_n);
-                *readsize = xdr_length_round_up (fhlen, NFS3_FHSIZE);
-                ret = NFS3_VECWRITE_READFH;
-        } else if (state == NFS3_VECWRITE_READFH) {
-                *readsize = NFS3_WRITE_POSTFH_SIZE;
-                ret = NFS3_VECWRITE_READREST;
-        } else if (state == NFS3_VECWRITE_READREST) {
-                ret = 0;
-                *readsize = 0;
-        } else
-                gf_log ("nfs", GF_LOG_ERROR, "state wrong");
+nfs3svc_write_vecsizer(int state, ssize_t *readsize, char *base_addr,
+                       char *curr_addr)
+{
+    int ret = 0;
+    uint32_t fhlen = 0;
+    uint32_t fhlen_n = 0;
+
+    if (state == 0) {
+        ret = NFS3_VECWRITE_READFHLEN;
+        *readsize = 4;
+    } else if (state == NFS3_VECWRITE_READFHLEN) {
+        fhlen_n = *(uint32_t *)(curr_addr - 4);
+        fhlen = ntohl(fhlen_n);
+        *readsize = xdr_length_round_up(fhlen, NFS3_FHSIZE);
+        ret = NFS3_VECWRITE_READFH;
+    } else if (state == NFS3_VECWRITE_READFH) {
+        *readsize = NFS3_WRITE_POSTFH_SIZE;
+        ret = NFS3_VECWRITE_READREST;
+    } else if (state == NFS3_VECWRITE_READREST) {
+        ret = 0;
+        *readsize = 0;
+    } else
+        gf_msg("nfs", GF_LOG_ERROR, 0, NFS_MSG_STATE_WRONG, "state wrong");
 
-        return ret;
+    return ret;
 }
 
-
 int
-nfs3svc_write (rpcsvc_request_t *req)
-{
-        struct nfs3_fh          fh = {{0}, };
-        write3args              args;
-        int                     ret = RPCSVC_ACTOR_ERROR;
-
-        if (!req)
-                return ret;
-        nfs3_prep_write3args (&args, &fh);
-        if (xdr_to_write3args (req->msg[0], &args) <= 0) {
-                gf_log (GF_NFS3, GF_LOG_ERROR, "Error decoding args");
-                rpcsvc_request_seterr (req, GARBAGE_ARGS);
-                goto rpcerr;
-        }
-
-        /* To ensure that the iobuf for the current record does not
-         * get returned to the iobpool, we need to keep a reference for
-         * ourselves because the RPC call handler who called us will unref its
-         * own ref of the record's iobuf when it is done handling the request.
-         */
+nfs3svc_write(rpcsvc_request_t *req)
+{
+    struct nfs3_fh fh = {
+        {0},
+    };
+    write3args args;
+    int ret = RPCSVC_ACTOR_ERROR;
 
-        ret = nfs3_write (req, &fh, args.offset, args.count, args.stable,
-                          req->msg[1], rpcsvc_request_iobref_ref (req));
-        if ((ret < 0) && (ret != RPCSVC_ACTOR_IGNORE)) {
-                gf_log (GF_NFS3, GF_LOG_ERROR, "WRITE procedure failed");
-                rpcsvc_request_seterr (req, SYSTEM_ERR);
-                ret = RPCSVC_ACTOR_ERROR;
-        }
+    if (!req)
+        return ret;
+    nfs3_prep_write3args(&args, &fh);
+    if (xdr_to_write3args(req->msg[0], &args) <= 0) {
+        gf_msg(GF_NFS3, GF_LOG_ERROR, 0, NFS_MSG_ARGS_DECODE_ERROR,
+               "Error decoding args");
+        rpcsvc_request_seterr(req, GARBAGE_ARGS);
+        goto rpcerr;
+    }
+
+    /* To ensure that the iobuf for the current record does not
+     * get returned to the iobpool, we need to keep a reference for
+     * ourselves because the RPC call handler who called us will unref its
+     * own ref of the record's iobuf when it is done handling the request.
+     */
+
+    ret = nfs3_write(req, &fh, args.offset, args.count, args.stable,
+                     req->msg[1], rpcsvc_request_iobref_ref(req));
+    if ((ret < 0) && (ret != RPCSVC_ACTOR_IGNORE)) {
+        gf_msg(GF_NFS3, GF_LOG_ERROR, -ret, NFS_MSG_WRITE_FAIL,
+               "WRITE procedure failed");
+        rpcsvc_request_seterr(req, SYSTEM_ERR);
+        ret = RPCSVC_ACTOR_ERROR;
+    }
 
 rpcerr:
-        return ret;
+    return ret;
 }
 
-
 int
-nfs3_create_reply (rpcsvc_request_t *req, nfsstat3 stat, struct nfs3_fh *newfh,
-                   struct iatt *newbuf, struct iatt *preparent,
-                   struct iatt *postparent)
+nfs3_create_reply(rpcsvc_request_t *req, nfsstat3 stat, struct nfs3_fh *newfh,
+                  struct iatt *newbuf, struct iatt *preparent,
+                  struct iatt *postparent)
 {
-        create3res      res = {0, };
-        uint64_t        deviceid = 0;
+    create3res res = {
+        0,
+    };
+    uint64_t deviceid = 0;
 
-        deviceid = nfs3_request_xlator_deviceid (req);
-        nfs3_fill_create3res (&res, stat, newfh, newbuf, preparent, postparent,
-                              deviceid);
-        nfs3svc_submit_reply (req, (void *)&res,
-                              (nfs3_serializer)xdr_serialize_create3res);
-        return 0;
+    deviceid = nfs3_request_xlator_deviceid(req);
+    nfs3_fill_create3res(&res, stat, newfh, newbuf, preparent, postparent,
+                         deviceid);
+    nfs3svc_submit_reply(req, (void *)&res,
+                         (nfs3_serializer)xdr_serialize_create3res);
+    return 0;
 }
 
-
 int32_t
-nfs3svc_create_setattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                            int32_t op_ret, int32_t op_errno,
-                            struct iatt *preop, struct iatt *postop, dict_t *xdata)
-{
-        nfsstat3                stat = NFS3ERR_SERVERFAULT;
-        nfs3_call_state_t       *cs = NULL;
-
-        cs = frame->local;
-        if (op_ret == -1) {
-                gf_log (GF_NFS, GF_LOG_WARNING,
-                        "%x: %s => -1 (%s)", rpcsvc_request_xid (cs->req),
-                        cs->resolvedloc.path, strerror (op_errno));
-                stat = nfs3_cbk_errno_status (op_ret, op_errno);
-                goto nfs3err;
-        }
+nfs3svc_create_setattr_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                           int32_t op_ret, int32_t op_errno, struct iatt *preop,
+                           struct iatt *postop, dict_t *xdata)
+{
+    nfsstat3 stat = NFS3ERR_SERVERFAULT;
+    nfs3_call_state_t *cs = NULL;
 
-        stat = NFS3_OK;
+    cs = frame->local;
+    if (op_ret == -1) {
+        stat = nfs3_cbk_errno_status(op_ret, op_errno);
+        goto nfs3err;
+    }
+
+    stat = NFS3_OK;
 nfs3err:
-        nfs3_log_newfh_res (rpcsvc_request_xid (cs->req), NFS3_CREATE, stat,
-                            op_errno, &cs->fh);
-        nfs3_create_reply (cs->req, stat, &cs->fh, postop, &cs->preparent,
-                           &cs->postparent);
-        nfs3_call_state_wipe (cs);
+    nfs3_log_newfh_res(rpcsvc_request_xid(cs->req), NFS3_CREATE, stat, op_errno,
+                       &cs->fh, cs->resolvedloc.path);
+    nfs3_create_reply(cs->req, stat, &cs->fh, postop, &cs->preparent,
+                      &cs->postparent);
+    nfs3_call_state_wipe(cs);
 
-        return 0;
+    return 0;
 }
 
-
 int32_t
-nfs3svc_create_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                    int32_t op_ret, int32_t op_errno, fd_t *fd, inode_t *inode,
-                    struct iatt *buf, struct iatt *preparent,
-                    struct iatt *postparent, dict_t *xdata)
+nfs3svc_create_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                   int32_t op_ret, int32_t op_errno, fd_t *fd, inode_t *inode,
+                   struct iatt *buf, struct iatt *preparent,
+                   struct iatt *postparent, dict_t *xdata)
 {
-        nfsstat3                stat = NFS3ERR_SERVERFAULT;
-        int                     ret = -EFAULT;
-        nfs_user_t              nfu = {0, };
-        nfs3_call_state_t       *cs = NULL;
-        inode_t                 *oldinode = NULL;
-
-        cs = frame->local;
-        if (op_ret == -1) {
-                gf_log (GF_NFS, GF_LOG_WARNING,
-                        "%x: %s => -1 (%s)", rpcsvc_request_xid (cs->req),
-                        cs->resolvedloc.path, strerror (op_errno));
-                stat = nfs3_cbk_errno_status (op_ret, op_errno);
-                goto nfs3err;
-        }
-
-        nfs3_fh_build_child_fh (&cs->parent, buf, &cs->fh);
-        oldinode = inode_link (inode, cs->resolvedloc.parent,
-                               cs->resolvedloc.name, buf);
-
-        /* Means no attributes were required to be set. */
-        if (!cs->setattr_valid) {
-                stat = NFS3_OK;
-                ret = -1;
-                goto nfs3err;
-        }
-
-        cs->preparent = *preparent;
-        cs->postparent = *postparent;
-        nfs_request_user_init (&nfu, cs->req);
-        gf_uuid_copy (cs->resolvedloc.gfid, inode->gfid);
-        ret = nfs_setattr (cs->nfsx, cs->vol, &nfu, &cs->resolvedloc,&cs->stbuf,
-                           cs->setattr_valid, nfs3svc_create_setattr_cbk, cs);
-        if (ret < 0)
-                stat = nfs3_errno_to_nfsstat3 (-ret);
+    nfsstat3 stat = NFS3ERR_SERVERFAULT;
+    int ret = -EFAULT;
+    nfs_user_t nfu = {
+        0,
+    };
+    nfs3_call_state_t *cs = NULL;
+    inode_t *oldinode = NULL;
+
+    cs = frame->local;
+    if (op_ret == -1) {
+        stat = nfs3_cbk_errno_status(op_ret, op_errno);
+        goto nfs3err;
+    }
+
+    nfs3_fh_build_child_fh(&cs->parent, buf, &cs->fh);
+    oldinode = inode_link(inode, cs->resolvedloc.parent, cs->resolvedloc.name,
+                          buf);
+
+    /* Means no attributes were required to be set. */
+    if (!cs->setattr_valid) {
+        stat = NFS3_OK;
+        ret = -1;
+        goto nfs3err;
+    }
+
+    cs->preparent = *preparent;
+    cs->postparent = *postparent;
+    nfs_request_user_init(&nfu, cs->req);
+    gf_uuid_copy(cs->resolvedloc.gfid, oldinode->gfid);
+    ret = nfs_setattr(cs->nfsx, cs->vol, &nfu, &cs->resolvedloc, &cs->stbuf,
+                      cs->setattr_valid, nfs3svc_create_setattr_cbk, cs);
+    if (ret < 0)
+        stat = nfs3_errno_to_nfsstat3(-ret);
 
 nfs3err:
-        if (oldinode) {
-                inode_lookup (oldinode);
-                inode_unref (oldinode);
-        }
+    if (oldinode) {
+        inode_lookup(oldinode);
+        inode_unref(oldinode);
+    }
 
-        if (ret < 0) {
-                nfs3_log_newfh_res (rpcsvc_request_xid (cs->req), NFS3_CREATE,
-                                    stat, op_errno, &cs->fh);
-                nfs3_create_reply (cs->req, stat, &cs->fh, buf, preparent,
-                                   postparent);
-                nfs3_call_state_wipe (cs);
-        }
+    if (ret < 0) {
+        nfs3_log_newfh_res(rpcsvc_request_xid(cs->req), NFS3_CREATE, stat,
+                           op_errno, &cs->fh, cs->resolvedloc.path);
+        nfs3_create_reply(cs->req, stat, &cs->fh, buf, preparent, postparent);
+        nfs3_call_state_wipe(cs);
+    }
 
-        return 0;
+    return 0;
 }
 
 int
-nfs3_create_common (nfs3_call_state_t *cs)
+nfs3_create_common(nfs3_call_state_t *cs)
 {
-        int                     ret = -EFAULT;
-        int                     flags = 0;
-        nfs_user_t              nfu = {0, };
-        uid_t                   uid = 0;
-        gid_t                   gid = 0;
-
-        if (!cs)
-                return ret;
-
-        if (cs->createmode == GUARDED)
-                flags = (O_RDWR | O_EXCL);
-        else
-                flags = O_RDWR;
-
-        if (gf_attr_uid_set (cs->setattr_valid)) {
-                uid = cs->stbuf.ia_uid;
-                cs->setattr_valid &= ~GF_SET_ATTR_UID;
-        } else
-                uid = rpcsvc_request_uid (cs->req);
-
-        if (gf_attr_gid_set (cs->setattr_valid)) {
-                gid = cs->stbuf.ia_gid;
-                cs->setattr_valid &= ~GF_SET_ATTR_GID;
-        } else
-                gid = rpcsvc_request_gid (cs->req);
-
-        nfs_request_primary_user_init (&nfu, cs->req, uid, gid);
-        /* We can avoid sending the setattr call later if only the mode is
-         * required to be set. This is possible because the create fop allows
-         * us to specify a mode arg.
-         */
-        if (cs->setattr_valid & GF_SET_ATTR_MODE) {
-                cs->setattr_valid &= ~GF_SET_ATTR_MODE;
-                ret = nfs_create (cs->nfsx, cs->vol, &nfu, &cs->resolvedloc,
-                                  flags, cs->mode, nfs3svc_create_cbk, cs);
-        } else
-                ret = nfs_create (cs->nfsx, cs->vol, &nfu, &cs->resolvedloc,
-                                  flags, NFS_DEFAULT_CREATE_MODE,
-                                  nfs3svc_create_cbk, cs);
+    int ret = -EFAULT;
+    int flags = 0;
+    nfs_user_t nfu = {
+        0,
+    };
+    uid_t uid = 0;
+    gid_t gid = 0;
 
+    if (!cs)
         return ret;
-}
 
+    if (cs->createmode == GUARDED)
+        flags = (O_RDWR | O_EXCL);
+    else
+        flags = O_RDWR;
+
+    if (gf_attr_uid_set(cs->setattr_valid)) {
+        uid = cs->stbuf.ia_uid;
+        cs->setattr_valid &= ~GF_SET_ATTR_UID;
+    } else
+        uid = rpcsvc_request_uid(cs->req);
+
+    if (gf_attr_gid_set(cs->setattr_valid)) {
+        gid = cs->stbuf.ia_gid;
+        cs->setattr_valid &= ~GF_SET_ATTR_GID;
+    } else
+        gid = rpcsvc_request_gid(cs->req);
+
+    nfs_request_primary_user_init(&nfu, cs->req, uid, gid);
+    /* We can avoid sending the setattr call later if only the mode is
+     * required to be set. This is possible because the create fop allows
+     * us to specify a mode arg.
+     */
+    if (cs->setattr_valid & GF_SET_ATTR_MODE) {
+        cs->setattr_valid &= ~GF_SET_ATTR_MODE;
+        ret = nfs_create(cs->nfsx, cs->vol, &nfu, &cs->resolvedloc, flags,
+                         cs->mode, nfs3svc_create_cbk, cs);
+    } else
+        ret = nfs_create(cs->nfsx, cs->vol, &nfu, &cs->resolvedloc, flags,
+                         NFS_DEFAULT_CREATE_MODE, nfs3svc_create_cbk, cs);
+
+    return ret;
+}
 
 int32_t
-nfs3svc_create_stat_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                         int32_t op_ret, int32_t op_errno, struct iatt *buf,
-                         dict_t *xdata)
+nfs3svc_create_stat_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                        int32_t op_ret, int32_t op_errno, struct iatt *buf,
+                        dict_t *xdata)
 {
-        int                     ret = -EFAULT;
-        nfsstat3                stat = NFS3ERR_SERVERFAULT;
-        nfs_user_t              nfu = {0, };
-        nfs3_call_state_t       *cs = NULL;
-
-        cs = frame->local;
-        nfs_request_user_init (&nfu, cs->req);
-        if (op_ret == -1) {
-                gf_log (GF_NFS, GF_LOG_WARNING,
-                        "%x: %s => -1 (%s)", rpcsvc_request_xid (cs->req),
-                        cs->resolvedloc.path, strerror (op_errno));
-                ret = -op_errno;
-                stat = nfs3_cbk_errno_status (op_ret, op_errno);
-                goto nfs3err;
-        }
-
-        if ((cs->stbuf.ia_mtime == buf->ia_mtime) &&
-            (cs->stbuf.ia_atime == buf->ia_atime)) {
-                gf_log (GF_NFS3, GF_LOG_DEBUG,
-                        "Create req retransmitted verf %x %x",
-                        cs->stbuf.ia_mtime, cs->stbuf.ia_atime);
-                stat = NFS3_OK;
-                nfs3_fh_build_child_fh (&cs->parent, buf, &cs->fh);
-        } else {
-                gf_log (GF_NFS3, GF_LOG_DEBUG,
-                        "File already exist new_verf %x %x"
-                        "old_verf %x %x", cs->stbuf.ia_mtime,
-                        cs->stbuf.ia_atime,
-                        buf->ia_mtime, buf->ia_atime);
-                stat = NFS3ERR_EXIST;
-        }
+    int ret = -EFAULT;
+    nfsstat3 stat = NFS3ERR_SERVERFAULT;
+    nfs_user_t nfu = {
+        0,
+    };
+    nfs3_call_state_t *cs = NULL;
+
+    cs = frame->local;
+    nfs_request_user_init(&nfu, cs->req);
+    if (op_ret == -1) {
+        ret = -op_errno;
+        stat = nfs3_cbk_errno_status(op_ret, op_errno);
+        goto nfs3err;
+    }
+
+    if ((cs->stbuf.ia_mtime == buf->ia_mtime) &&
+        (cs->stbuf.ia_atime == buf->ia_atime)) {
+        gf_msg_debug(GF_NFS3, 0,
+                     "Create req retransmitted verf %" PRId64 " %" PRId64,
+                     cs->stbuf.ia_mtime, cs->stbuf.ia_atime);
+        stat = NFS3_OK;
+        nfs3_fh_build_child_fh(&cs->parent, buf, &cs->fh);
+    } else {
+        gf_msg_debug(GF_NFS3, 0,
+                     "File already exist new_verf %" PRId64 " %" PRId64
+                     "old_verf %" PRId64 " %" PRId64,
+                     cs->stbuf.ia_mtime, cs->stbuf.ia_atime, buf->ia_mtime,
+                     buf->ia_atime);
+        stat = NFS3ERR_EXIST;
+    }
 
 nfs3err:
-        if (ret < 0) {
-                nfs3_log_common_res (rpcsvc_request_xid (cs->req), NFS3_CREATE,
-                                     stat, op_errno);
-                nfs3_create_reply (cs->req, stat, &cs->fh, buf, NULL, NULL);
-                nfs3_call_state_wipe (cs);
-        }
+    if (ret < 0) {
+        nfs3_log_common_res(rpcsvc_request_xid(cs->req), NFS3_CREATE, stat,
+                            op_errno, cs->resolvedloc.path);
+        nfs3_create_reply(cs->req, stat, &cs->fh, buf, NULL, NULL);
+        nfs3_call_state_wipe(cs);
+    }
 
-        return 0;
+    return 0;
 }
 
-
 int
-nfs3_create_exclusive (nfs3_call_state_t *cs)
-{
-        int                     ret = -EFAULT;
-        nfs_user_t              nfu = {0, };
-
-        if (!cs)
-                return ret;
-
-        /* Storing verifier as a mtime and atime attribute, to store it
-         * in stable storage */
-        memcpy (&cs->stbuf.ia_atime, &cs->cookieverf,
-                sizeof (cs->stbuf.ia_atime));
-        memcpy (&cs->stbuf.ia_mtime,
-                ((char *) &cs->cookieverf) + sizeof (cs->stbuf.ia_atime),
-                sizeof (cs->stbuf.ia_mtime));
-        cs->setattr_valid |= GF_SET_ATTR_ATIME;
-        cs->setattr_valid |= GF_SET_ATTR_MTIME;
-        nfs_request_user_init (&nfu, cs->req);
-
-        /* If the file already existed we need to get that attributes so we can
-         * compare and check whether a previous create operation was
-         * interrupted due to server failure or dropped packets.
-         */
-        if ((cs->resolve_ret == 0) ||
-            ((cs->resolve_ret == -1) && (cs->resolve_errno != ENOENT))) {
-                ret = nfs_stat (cs->nfsx, cs->vol, &nfu, &cs->resolvedloc,
-                                nfs3svc_create_stat_cbk, cs);
-                goto nfs3err;
-        }
+nfs3_create_exclusive(nfs3_call_state_t *cs)
+{
+    int ret = -EFAULT;
+    nfs_user_t nfu = {
+        0,
+    };
 
-        ret = nfs3_create_common (cs);
-nfs3err:
+    if (!cs)
         return ret;
-}
 
+    /* Storing verifier as a mtime and atime attribute, to store it
+     * in stable storage */
+    memcpy(&cs->stbuf.ia_atime, &cs->cookieverf, sizeof(cs->stbuf.ia_atime));
+    memcpy(&cs->stbuf.ia_mtime,
+           ((char *)&cs->cookieverf) + sizeof(cs->stbuf.ia_atime),
+           sizeof(cs->stbuf.ia_mtime));
+    cs->setattr_valid |= GF_SET_ATTR_ATIME;
+    cs->setattr_valid |= GF_SET_ATTR_MTIME;
+    nfs_request_user_init(&nfu, cs->req);
+
+    /* If the file already existed we need to get that attributes so we can
+     * compare and check whether a previous create operation was
+     * interrupted due to server failure or dropped packets.
+     */
+    if ((cs->resolve_ret == 0) ||
+        ((cs->resolve_ret == -1) && (cs->resolve_errno != ENOENT))) {
+        ret = nfs_stat(cs->nfsx, cs->vol, &nfu, &cs->resolvedloc,
+                       nfs3svc_create_stat_cbk, cs);
+        goto nfs3err;
+    }
+
+    ret = nfs3_create_common(cs);
+nfs3err:
+    return ret;
+}
 
 int
-nfs3_create_resume (void *carg)
+nfs3_create_resume(void *carg)
 {
-        nfsstat3                        stat = NFS3ERR_SERVERFAULT;
-        int                             ret = -EFAULT;
-        nfs3_call_state_t               *cs = NULL;
+    nfsstat3 stat = NFS3ERR_SERVERFAULT;
+    int ret = -EFAULT;
+    nfs3_call_state_t *cs = NULL;
 
-        if (!carg)
-                return ret;
+    if (!carg)
+        return ret;
 
-        cs = (nfs3_call_state_t *)carg;
-        nfs3_check_fh_auth_status (cs, stat, _gf_true, nfs3err);
-        nfs3_check_new_fh_resolve_status (cs, stat, nfs3err);
-        if (cs->createmode == EXCLUSIVE)
-                ret = nfs3_create_exclusive (cs);
-        else
-                ret = nfs3_create_common (cs);
+    cs = (nfs3_call_state_t *)carg;
+    nfs3_check_fh_auth_status(cs, stat, _gf_true, nfs3err);
+    nfs3_check_new_fh_resolve_status(cs, stat, nfs3err);
+    if (cs->createmode == EXCLUSIVE)
+        ret = nfs3_create_exclusive(cs);
+    else
+        ret = nfs3_create_common(cs);
 
-        /* Handle a failure return from either of the create functions above. */
-        if (ret < 0)
-                stat = nfs3_errno_to_nfsstat3 (-ret);
+    /* Handle a failure return from either of the create functions above. */
+    if (ret < 0)
+        stat = nfs3_errno_to_nfsstat3(-ret);
 
 nfs3err:
-        if (ret < 0) {
-                nfs3_log_common_res (rpcsvc_request_xid (cs->req), NFS3_CREATE,
-                                     stat, -ret);
-                nfs3_create_reply (cs->req, stat, NULL, NULL, NULL, NULL);
-                nfs3_call_state_wipe (cs);
-        }
+    if (ret < 0) {
+        nfs3_log_common_res(rpcsvc_request_xid(cs->req), NFS3_CREATE, stat,
+                            -ret, cs->resolvedloc.path);
+        nfs3_create_reply(cs->req, stat, NULL, NULL, NULL, NULL);
+        nfs3_call_state_wipe(cs);
+    }
 
-        return ret;
+    return ret;
 }
 
 int
-nfs3_create (rpcsvc_request_t *req, struct nfs3_fh *dirfh, char *name,
-             createmode3 mode, sattr3 *sattr, uint64_t cverf)
-{
-        xlator_t                        *vol = NULL;
-        nfsstat3                        stat = NFS3ERR_SERVERFAULT;
-        int                             ret = -EFAULT;
-        struct nfs3_state               *nfs3 = NULL;
-        nfs3_call_state_t               *cs = NULL;
-
-        if ((!req) || (!dirfh) || (!name) || (!sattr))
-                return -1;
-
-        nfs3_log_create_call (rpcsvc_request_xid (req), dirfh, name, mode);
-        nfs3_validate_gluster_fh (dirfh, stat, nfs3err);
-        nfs3_validate_nfs3_state (req, nfs3, stat, nfs3err, ret);
-        nfs3_validate_strlen_or_goto (name, NFS_NAME_MAX, nfs3err, stat, ret);
-        nfs3_map_fh_to_volume (nfs3, dirfh, req, vol, stat, nfs3err);
-        nfs3_volume_started_check (nfs3, vol, ret, out);
-        nfs3_check_rw_volaccess (nfs3, dirfh->exportid, stat, nfs3err);
-        nfs3_handle_call_state_init (nfs3, cs, req, vol, stat, nfs3err);
-
-        cs->cookieverf = cverf;
-        /*In Exclusive create client is supposed to send cverf instead of
-         * sattr*/
-        if (mode != EXCLUSIVE)
-                cs->setattr_valid = nfs3_sattr3_to_setattr_valid (sattr,
-                                                                  &cs->stbuf,
-                                                                  &cs->mode);
-        cs->createmode = mode;
-        cs->parent = *dirfh;
-
-        ret = nfs3_fh_resolve_and_resume (cs, dirfh, name, nfs3_create_resume);
-        if (ret < 0)
-                stat = nfs3_errno_to_nfsstat3 (-ret);
+nfs3_create(rpcsvc_request_t *req, struct nfs3_fh *dirfh, char *name,
+            createmode3 mode, sattr3 *sattr, uint64_t cverf)
+{
+    xlator_t *vol = NULL;
+    nfsstat3 stat = NFS3ERR_SERVERFAULT;
+    int ret = -EFAULT;
+    struct nfs3_state *nfs3 = NULL;
+    nfs3_call_state_t *cs = NULL;
+
+    if ((!req) || (!dirfh) || (!name) || (!sattr))
+        return -1;
+
+    nfs3_log_create_call(rpcsvc_request_xid(req), dirfh, name, mode);
+    nfs3_validate_gluster_fh(dirfh, stat, nfs3err);
+    nfs3_validate_nfs3_state(req, nfs3, stat, nfs3err, ret);
+    nfs3_validate_strlen_or_goto(name, NFS_NAME_MAX, nfs3err, stat, ret);
+    nfs3_map_fh_to_volume(nfs3, dirfh, req, vol, stat, nfs3err);
+    nfs3_volume_started_check(nfs3, vol, ret, out);
+    nfs3_check_rw_volaccess(nfs3, dirfh->exportid, stat, nfs3err);
+    nfs3_handle_call_state_init(nfs3, cs, req, vol, stat, nfs3err);
+
+    cs->cookieverf = cverf;
+    /*In Exclusive create client is supposed to send cverf instead of
+     * sattr*/
+    if (mode != EXCLUSIVE)
+        cs->setattr_valid = nfs3_sattr3_to_setattr_valid(sattr, &cs->stbuf,
+                                                         &cs->mode);
+    cs->createmode = mode;
+    cs->parent = *dirfh;
+
+    ret = nfs3_fh_resolve_and_resume(cs, dirfh, name, nfs3_create_resume);
+    if (ret < 0)
+        stat = nfs3_errno_to_nfsstat3(-ret);
 
 nfs3err:
-        if (ret < 0) {
-                nfs3_log_common_res (rpcsvc_request_xid (req), NFS3_CREATE,
-                                     stat, -ret);
-                nfs3_create_reply (req, stat, NULL, NULL, NULL, NULL);
-                nfs3_call_state_wipe (cs);
-                ret = 0;
-        }
+    if (ret < 0) {
+        nfs3_log_common_res(rpcsvc_request_xid(req), NFS3_CREATE, stat, -ret,
+                            cs ? cs->resolvedloc.path : NULL);
+        nfs3_create_reply(req, stat, NULL, NULL, NULL, NULL);
+        nfs3_call_state_wipe(cs);
+        ret = 0;
+    }
 out:
-        return ret;
+    return ret;
 }
 
-
 int
-nfs3svc_create (rpcsvc_request_t *req)
-{
-        char            name[NFS_PATH_MAX];
-        struct nfs3_fh  dirfh = {{0}, };
-        create3args     args;
-        int             ret   = RPCSVC_ACTOR_ERROR;
-        uint64_t        cverf = 0;
-        uint64_t       *cval;
-
-        if (!req)
-                return ret;
-
-        nfs3_prep_create3args (&args, &dirfh, name);
-        if (xdr_to_create3args (req->msg[0], &args) <= 0) {
-                gf_log (GF_NFS3, GF_LOG_ERROR, "Error decoding args");
-                rpcsvc_request_seterr (req, GARBAGE_ARGS);
-                goto rpcerr;
-        }
-
-        cval = (uint64_t *)args.how.createhow3_u.verf;
-        cverf = *cval;
+nfs3svc_create(rpcsvc_request_t *req)
+{
+    char name[NFS_PATH_MAX];
+    struct nfs3_fh dirfh = {
+        {0},
+    };
+    create3args args;
+    int ret = RPCSVC_ACTOR_ERROR;
+    uint64_t cverf = 0;
+    uint64_t *cval;
+
+    if (!req)
+        return ret;
 
-        ret = nfs3_create (req, &dirfh, name, args.how.mode,
-                           &args.how.createhow3_u.obj_attributes, cverf);
-        if ((ret < 0) && (ret != RPCSVC_ACTOR_IGNORE)) {
-                gf_log (GF_NFS3, GF_LOG_ERROR, "CREATE procedure failed");
-                rpcsvc_request_seterr (req, SYSTEM_ERR);
-                ret = RPCSVC_ACTOR_ERROR;
-        }
+    nfs3_prep_create3args(&args, &dirfh, name);
+    if (xdr_to_create3args(req->msg[0], &args) <= 0) {
+        gf_msg(GF_NFS3, GF_LOG_ERROR, 0, NFS_MSG_ARGS_DECODE_ERROR,
+               "Error decoding args");
+        rpcsvc_request_seterr(req, GARBAGE_ARGS);
+        goto rpcerr;
+    }
+
+    cval = (uint64_t *)args.how.createhow3_u.verf;
+    cverf = *cval;
+
+    ret = nfs3_create(req, &dirfh, name, args.how.mode,
+                      &args.how.createhow3_u.obj_attributes, cverf);
+    if ((ret < 0) && (ret != RPCSVC_ACTOR_IGNORE)) {
+        gf_msg(GF_NFS3, GF_LOG_ERROR, -ret, NFS_MSG_CREATE_FAIL,
+               "CREATE procedure failed");
+        rpcsvc_request_seterr(req, SYSTEM_ERR);
+        ret = RPCSVC_ACTOR_ERROR;
+    }
 
 rpcerr:
-        return ret;
+    return ret;
 }
 
-
 int
-nfs3_mkdir_reply (rpcsvc_request_t *req, nfsstat3 stat, struct nfs3_fh *fh,
-                  struct iatt *buf, struct iatt *preparent,
-                  struct iatt *postparent)
+nfs3_mkdir_reply(rpcsvc_request_t *req, nfsstat3 stat, struct nfs3_fh *fh,
+                 struct iatt *buf, struct iatt *preparent,
+                 struct iatt *postparent)
 {
-        mkdir3res       res = {0, };
-        uint64_t        deviceid = 0;
+    mkdir3res res = {
+        0,
+    };
+    uint64_t deviceid = 0;
 
-        deviceid = nfs3_request_xlator_deviceid (req);
-        nfs3_fill_mkdir3res (&res, stat, fh, buf, preparent, postparent,
-                             deviceid);
-        nfs3svc_submit_reply (req, &res,
-                              (nfs3_serializer)xdr_serialize_mkdir3res);
-        return 0;
+    deviceid = nfs3_request_xlator_deviceid(req);
+    nfs3_fill_mkdir3res(&res, stat, fh, buf, preparent, postparent, deviceid);
+    nfs3svc_submit_reply(req, &res, (nfs3_serializer)xdr_serialize_mkdir3res);
+    return 0;
 }
 
 int32_t
-nfs3svc_mkdir_setattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                           int32_t op_ret, int32_t op_errno,
-                           struct iatt *preop, struct iatt *postop, dict_t *xdata)
-{
-        nfsstat3                stat = NFS3ERR_SERVERFAULT;
-        nfs3_call_state_t       *cs = NULL;
-
-        cs = frame->local;
-        if (op_ret == -1) {
-                gf_log (GF_NFS, GF_LOG_WARNING,
-                        "%x: %s => -1 (%s)", rpcsvc_request_xid (cs->req),
-                        cs->resolvedloc.path, strerror (op_errno));
-                stat = nfs3_cbk_errno_status (op_ret, op_errno);
-                goto nfs3err;
-        }
+nfs3svc_mkdir_setattr_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                          int32_t op_ret, int32_t op_errno, struct iatt *preop,
+                          struct iatt *postop, dict_t *xdata)
+{
+    nfsstat3 stat = NFS3ERR_SERVERFAULT;
+    nfs3_call_state_t *cs = NULL;
 
-        stat = NFS3_OK;
+    cs = frame->local;
+    if (op_ret == -1) {
+        stat = nfs3_cbk_errno_status(op_ret, op_errno);
+        goto nfs3err;
+    }
+
+    stat = NFS3_OK;
 nfs3err:
-        nfs3_log_newfh_res (rpcsvc_request_xid (cs->req), NFS3_MKDIR, stat,
-                            op_errno, &cs->fh);
-        nfs3_mkdir_reply (cs->req, stat, &cs->fh, postop, &cs->preparent,
-                          &cs->postparent);
-        nfs3_call_state_wipe (cs);
+    nfs3_log_newfh_res(rpcsvc_request_xid(cs->req), NFS3_MKDIR, stat, op_errno,
+                       &cs->fh, cs->resolvedloc.path);
+    nfs3_mkdir_reply(cs->req, stat, &cs->fh, postop, &cs->preparent,
+                     &cs->postparent);
+    nfs3_call_state_wipe(cs);
 
-        return 0;
+    return 0;
 }
 
-
 int32_t
-nfs3svc_mkdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                   int32_t op_ret, int32_t op_errno, inode_t *inode,
-                   struct iatt *buf, struct iatt *preparent,
-                   struct iatt *postparent, dict_t *xdata)
+nfs3svc_mkdir_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                  int32_t op_ret, int32_t op_errno, inode_t *inode,
+                  struct iatt *buf, struct iatt *preparent,
+                  struct iatt *postparent, dict_t *xdata)
 {
-        nfsstat3                        stat = NFS3ERR_SERVERFAULT;
-        int                             ret = -EFAULT;
-        nfs_user_t                      nfu = {0, };
-        nfs3_call_state_t               *cs = NULL;
-
-        cs = frame->local;
-        if (op_ret == -1) {
-                gf_log (GF_NFS, GF_LOG_WARNING,
-                        "%x: %s => -1 (%s)", rpcsvc_request_xid (cs->req),
-                        cs->resolvedloc.path, strerror (op_errno));
-                stat = nfs3_cbk_errno_status (op_ret, op_errno);
-                goto nfs3err;
-        }
+    nfsstat3 stat = NFS3ERR_SERVERFAULT;
+    int ret = -EFAULT;
+    nfs_user_t nfu = {
+        0,
+    };
+    nfs3_call_state_t *cs = NULL;
 
-        nfs3_fh_build_child_fh (&cs->parent, buf, &cs->fh);
+    cs = frame->local;
+    if (op_ret == -1) {
+        stat = nfs3_cbk_errno_status(op_ret, op_errno);
+        goto nfs3err;
+    }
 
-        /* Means no attributes were required to be set. */
-        if (!cs->setattr_valid) {
-                stat = NFS3_OK;
-                goto nfs3err;
-        }
+    nfs3_fh_build_child_fh(&cs->parent, buf, &cs->fh);
 
-        cs->preparent = *preparent;
-        cs->postparent = *postparent;
-        nfs_request_user_init (&nfu, cs->req);
-        ret = nfs_setattr (cs->nfsx, cs->vol, &nfu, &cs->resolvedloc,&cs->stbuf,
-                           cs->setattr_valid, nfs3svc_mkdir_setattr_cbk, cs);
-        if (ret < 0)
-                stat = nfs3_errno_to_nfsstat3 (-ret);
+    /* Means no attributes were required to be set. */
+    if (!cs->setattr_valid) {
+        stat = NFS3_OK;
+        goto nfs3err;
+    }
+
+    cs->preparent = *preparent;
+    cs->postparent = *postparent;
+    nfs_request_user_init(&nfu, cs->req);
+    ret = nfs_setattr(cs->nfsx, cs->vol, &nfu, &cs->resolvedloc, &cs->stbuf,
+                      cs->setattr_valid, nfs3svc_mkdir_setattr_cbk, cs);
+    if (ret < 0)
+        stat = nfs3_errno_to_nfsstat3(-ret);
 
 nfs3err:
-        if (ret < 0) {
-                nfs3_log_newfh_res (rpcsvc_request_xid (cs->req), NFS3_MKDIR,
-                                    stat, op_errno, &cs->fh);
-                nfs3_mkdir_reply (cs->req, stat, &cs->fh, buf, preparent,
-                                  postparent);
-                nfs3_call_state_wipe (cs);
-        }
+    if (ret < 0) {
+        nfs3_log_newfh_res(rpcsvc_request_xid(cs->req), NFS3_MKDIR, stat,
+                           op_errno, &cs->fh, cs->resolvedloc.path);
+        nfs3_mkdir_reply(cs->req, stat, &cs->fh, buf, preparent, postparent);
+        nfs3_call_state_wipe(cs);
+    }
 
-        return 0;
+    return 0;
 }
 
-
 int
-nfs3_mkdir_resume (void *carg)
+nfs3_mkdir_resume(void *carg)
 {
-        nfsstat3                        stat = NFS3ERR_SERVERFAULT;
-        int                             ret = -EFAULT;
-        nfs_user_t                      nfu = {0, };
-        nfs3_call_state_t               *cs = NULL;
+    nfsstat3 stat = NFS3ERR_SERVERFAULT;
+    int ret = -EFAULT;
+    nfs_user_t nfu = {
+        0,
+    };
+    nfs3_call_state_t *cs = NULL;
 
-        if (!carg)
-                return ret;
+    if (!carg)
+        return ret;
 
-        cs = (nfs3_call_state_t *)carg;
-        nfs3_check_new_fh_resolve_status (cs, stat, nfs3err);
-        nfs_request_user_init (&nfu, cs->req);
+    cs = (nfs3_call_state_t *)carg;
+    nfs3_check_new_fh_resolve_status(cs, stat, nfs3err);
+    nfs_request_user_init(&nfu, cs->req);
 
-        if (gf_attr_mode_set (cs->setattr_valid)) {
-                cs->setattr_valid &= ~GF_SET_ATTR_MODE;
-                ret = nfs_mkdir (cs->nfsx, cs->vol, &nfu, &cs->resolvedloc,
-                                 cs->mode, nfs3svc_mkdir_cbk, cs);
-        } else
-                ret = nfs_mkdir (cs->nfsx, cs->vol, &nfu, &cs->resolvedloc,
-                                 cs->mode, nfs3svc_mkdir_cbk, cs);
+    if (gf_attr_mode_set(cs->setattr_valid)) {
+        cs->setattr_valid &= ~GF_SET_ATTR_MODE;
+        ret = nfs_mkdir(cs->nfsx, cs->vol, &nfu, &cs->resolvedloc, cs->mode,
+                        nfs3svc_mkdir_cbk, cs);
+    } else
+        ret = nfs_mkdir(cs->nfsx, cs->vol, &nfu, &cs->resolvedloc, cs->mode,
+                        nfs3svc_mkdir_cbk, cs);
 
-        if (ret < 0)
-                stat = nfs3_errno_to_nfsstat3 (-ret);
+    if (ret < 0)
+        stat = nfs3_errno_to_nfsstat3(-ret);
 
 nfs3err:
-        if (ret < 0) {
-                nfs3_log_common_res (rpcsvc_request_xid (cs->req), NFS3_MKDIR,
-                                     stat, -ret);
-                nfs3_mkdir_reply (cs->req, stat, NULL, NULL, NULL, NULL);
-                nfs3_call_state_wipe (cs);
-        }
+    if (ret < 0) {
+        nfs3_log_common_res(rpcsvc_request_xid(cs->req), NFS3_MKDIR, stat, -ret,
+                            cs->resolvedloc.path);
+        nfs3_mkdir_reply(cs->req, stat, NULL, NULL, NULL, NULL);
+        nfs3_call_state_wipe(cs);
+    }
 
-        return 0;
+    return 0;
 }
 
-
-
 int
-nfs3_mkdir (rpcsvc_request_t *req, struct nfs3_fh *dirfh, char *name,
-            sattr3 *sattr)
-{
-        xlator_t                        *vol = NULL;
-        nfsstat3                        stat = NFS3ERR_SERVERFAULT;
-        int                             ret = -EFAULT;
-        struct nfs3_state               *nfs3 = NULL;
-        nfs3_call_state_t               *cs = NULL;
-
-        if ((!req) || (!dirfh) || (!name) || (!sattr)) {
-                gf_log (GF_NFS3, GF_LOG_ERROR, "Bad arguments");
-                return -1;
-        }
-
-        nfs3_log_fh_entry_call (rpcsvc_request_xid (req), "MKDIR", dirfh,
-                                name);
-        nfs3_validate_gluster_fh (dirfh, stat, nfs3err);
-        nfs3_validate_nfs3_state (req, nfs3, stat, nfs3err, ret);
-        nfs3_validate_strlen_or_goto (name, NFS_NAME_MAX, nfs3err, stat, ret);
-        nfs3_map_fh_to_volume (nfs3, dirfh, req, vol, stat, nfs3err);
-        nfs3_volume_started_check (nfs3, vol, ret, out);
-        nfs3_check_rw_volaccess (nfs3, dirfh->exportid, stat, nfs3err);
-        nfs3_handle_call_state_init (nfs3, cs, req, vol, stat, nfs3err);
-
-        cs->parent = *dirfh;
-        cs->setattr_valid = nfs3_sattr3_to_setattr_valid (sattr, &cs->stbuf,
-                                                          &cs->mode);
-        ret = nfs3_fh_resolve_and_resume (cs, dirfh, name, nfs3_mkdir_resume);
-        if (ret < 0)
-                stat = nfs3_errno_to_nfsstat3 (-ret);
+nfs3_mkdir(rpcsvc_request_t *req, struct nfs3_fh *dirfh, char *name,
+           sattr3 *sattr)
+{
+    xlator_t *vol = NULL;
+    nfsstat3 stat = NFS3ERR_SERVERFAULT;
+    int ret = -EFAULT;
+    struct nfs3_state *nfs3 = NULL;
+    nfs3_call_state_t *cs = NULL;
+
+    if ((!req) || (!dirfh) || (!name) || (!sattr)) {
+        gf_msg(GF_NFS3, GF_LOG_ERROR, EINVAL, NFS_MSG_INVALID_ENTRY,
+               "Bad arguments");
+        return -1;
+    }
+
+    nfs3_log_fh_entry_call(rpcsvc_request_xid(req), "MKDIR", dirfh, name);
+    nfs3_validate_gluster_fh(dirfh, stat, nfs3err);
+    nfs3_validate_nfs3_state(req, nfs3, stat, nfs3err, ret);
+    nfs3_validate_strlen_or_goto(name, NFS_NAME_MAX, nfs3err, stat, ret);
+    nfs3_map_fh_to_volume(nfs3, dirfh, req, vol, stat, nfs3err);
+    nfs3_volume_started_check(nfs3, vol, ret, out);
+    nfs3_check_rw_volaccess(nfs3, dirfh->exportid, stat, nfs3err);
+    nfs3_handle_call_state_init(nfs3, cs, req, vol, stat, nfs3err);
+
+    cs->parent = *dirfh;
+    cs->setattr_valid = nfs3_sattr3_to_setattr_valid(sattr, &cs->stbuf,
+                                                     &cs->mode);
+    ret = nfs3_fh_resolve_and_resume(cs, dirfh, name, nfs3_mkdir_resume);
+    if (ret < 0)
+        stat = nfs3_errno_to_nfsstat3(-ret);
 
 nfs3err:
-        if (ret < 0) {
-                nfs3_log_common_res (rpcsvc_request_xid (req), NFS3_MKDIR,
-                                     stat, -ret);
-                nfs3_mkdir_reply (req, stat, NULL, NULL, NULL, NULL);
-                nfs3_call_state_wipe (cs);
-                ret = 0;
-        }
+    if (ret < 0) {
+        nfs3_log_common_res(rpcsvc_request_xid(req), NFS3_MKDIR, stat, -ret,
+                            cs ? cs->resolvedloc.path : NULL);
+        nfs3_mkdir_reply(req, stat, NULL, NULL, NULL, NULL);
+        nfs3_call_state_wipe(cs);
+        ret = 0;
+    }
 out:
-        return ret;
+    return ret;
 }
 
-
 int
-nfs3svc_mkdir (rpcsvc_request_t *req)
-{
-        char                    name[NFS_PATH_MAX];
-        struct nfs3_fh          dirfh = {{0}, };
-        mkdir3args              args;
-        int                     ret = RPCSVC_ACTOR_ERROR;
-
-        if (!req)
-                return ret;
-        nfs3_prep_mkdir3args (&args, &dirfh, name);
-        if (xdr_to_mkdir3args (req->msg[0], &args) <= 0) {
-                gf_log (GF_NFS3, GF_LOG_ERROR, "Error decoding args");
-                rpcsvc_request_seterr (req, GARBAGE_ARGS);
-                goto rpcerr;
-        }
+nfs3svc_mkdir(rpcsvc_request_t *req)
+{
+    char name[NFS_PATH_MAX];
+    struct nfs3_fh dirfh = {
+        {0},
+    };
+    mkdir3args args;
+    int ret = RPCSVC_ACTOR_ERROR;
 
-        ret = nfs3_mkdir (req, &dirfh, name, &args.attributes);
-        if ((ret < 0) && (ret != RPCSVC_ACTOR_IGNORE)) {
-                gf_log (GF_NFS3, GF_LOG_ERROR, "MKDIR procedure failed");
-                rpcsvc_request_seterr (req, SYSTEM_ERR);
-                ret = RPCSVC_ACTOR_ERROR;
-        }
+    if (!req)
+        return ret;
+    nfs3_prep_mkdir3args(&args, &dirfh, name);
+    if (xdr_to_mkdir3args(req->msg[0], &args) <= 0) {
+        gf_msg(GF_NFS3, GF_LOG_ERROR, 0, NFS_MSG_ARGS_DECODE_ERROR,
+               "Error decoding args");
+        rpcsvc_request_seterr(req, GARBAGE_ARGS);
+        goto rpcerr;
+    }
+
+    ret = nfs3_mkdir(req, &dirfh, name, &args.attributes);
+    if ((ret < 0) && (ret != RPCSVC_ACTOR_IGNORE)) {
+        gf_msg(GF_NFS3, GF_LOG_ERROR, -ret, NFS_MSG_DIR_OP_FAIL,
+               "MKDIR procedure failed");
+        rpcsvc_request_seterr(req, SYSTEM_ERR);
+        ret = RPCSVC_ACTOR_ERROR;
+    }
 
 rpcerr:
-        return ret;
+    return ret;
 }
 
-
 int
-nfs3_symlink_reply (rpcsvc_request_t *req, nfsstat3 stat, struct nfs3_fh *fh,
-                    struct iatt *buf, struct iatt *preparent,
-                    struct iatt *postparent)
+nfs3_symlink_reply(rpcsvc_request_t *req, nfsstat3 stat, struct nfs3_fh *fh,
+                   struct iatt *buf, struct iatt *preparent,
+                   struct iatt *postparent)
 {
-        symlink3res     res = {0, };
-        uint64_t        deviceid = 0;
+    symlink3res res = {
+        0,
+    };
+    uint64_t deviceid = 0;
 
-        deviceid = nfs3_request_xlator_deviceid (req);
-        nfs3_fill_symlink3res (&res, stat, fh, buf, preparent, postparent,
-                               deviceid);
-        nfs3svc_submit_reply (req, (void *)&res,
-                              (nfs3_serializer)xdr_serialize_symlink3res);
+    deviceid = nfs3_request_xlator_deviceid(req);
+    nfs3_fill_symlink3res(&res, stat, fh, buf, preparent, postparent, deviceid);
+    nfs3svc_submit_reply(req, (void *)&res,
+                         (nfs3_serializer)xdr_serialize_symlink3res);
 
-        return 0;
+    return 0;
 }
 
-
 int32_t
-nfs3svc_symlink_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                     int32_t op_ret, int32_t op_errno, inode_t *inode,
-                     struct iatt *buf, struct iatt *preparent,
-                     struct iatt *postparent, dict_t *xdata)
-{
-        nfsstat3                        stat = NFS3ERR_SERVERFAULT;
-        nfs3_call_state_t               *cs = NULL;
-
-        cs = frame->local;
-        if (op_ret == -1) {
-                gf_log (GF_NFS, GF_LOG_WARNING,
-                        "%x: %s => -1 (%s)", rpcsvc_request_xid (cs->req),
-                        cs->resolvedloc.path, strerror (op_errno));
-                stat = nfs3_cbk_errno_status (op_ret, op_errno);
-                goto nfs3err;
-        }
+nfs3svc_symlink_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                    int32_t op_ret, int32_t op_errno, inode_t *inode,
+                    struct iatt *buf, struct iatt *preparent,
+                    struct iatt *postparent, dict_t *xdata)
+{
+    nfsstat3 stat = NFS3ERR_SERVERFAULT;
+    nfs3_call_state_t *cs = NULL;
 
-        nfs3_fh_build_child_fh (&cs->parent, buf, &cs->fh);
-        stat = NFS3_OK;
+    cs = frame->local;
+    if (op_ret == -1) {
+        stat = nfs3_cbk_errno_status(op_ret, op_errno);
+        goto nfs3err;
+    }
+
+    nfs3_fh_build_child_fh(&cs->parent, buf, &cs->fh);
+    stat = NFS3_OK;
 
 nfs3err:
-        nfs3_log_newfh_res (rpcsvc_request_xid (cs->req), NFS3_SYMLINK, stat,
-                            op_errno, &cs->fh);
-        nfs3_symlink_reply (cs->req, stat, &cs->fh, buf, preparent,
-                            postparent);
-        nfs3_call_state_wipe (cs);
-        return 0;
+    nfs3_log_newfh_res(rpcsvc_request_xid(cs->req), NFS3_SYMLINK, stat,
+                       op_errno, &cs->fh, cs->resolvedloc.path);
+    nfs3_symlink_reply(cs->req, stat, &cs->fh, buf, preparent, postparent);
+    nfs3_call_state_wipe(cs);
+    return 0;
 }
 
-
 int
-nfs3_symlink_resume (void *carg)
+nfs3_symlink_resume(void *carg)
 {
-        nfsstat3                        stat = NFS3ERR_SERVERFAULT;
-        int                             ret = -EFAULT;
-        nfs3_call_state_t               *cs = NULL;
-        nfs_user_t                      nfu = {0, };
+    nfsstat3 stat = NFS3ERR_SERVERFAULT;
+    int ret = -EFAULT;
+    nfs3_call_state_t *cs = NULL;
+    nfs_user_t nfu = {
+        0,
+    };
 
-        if (!carg)
-                return ret;
+    if (!carg)
+        return ret;
 
-        cs = (nfs3_call_state_t *)carg;
-        nfs3_check_new_fh_resolve_status (cs, stat, nfs3err);
-        nfs_request_user_init (&nfu, cs->req);
-        ret = nfs_symlink (cs->nfsx, cs->vol, &nfu, cs->pathname,
-                           &cs->resolvedloc, nfs3svc_symlink_cbk, cs);
-        if (ret < 0)
-                stat = nfs3_errno_to_nfsstat3 (-ret);
+    cs = (nfs3_call_state_t *)carg;
+    nfs3_check_new_fh_resolve_status(cs, stat, nfs3err);
+    nfs_request_user_init(&nfu, cs->req);
+    ret = nfs_symlink(cs->nfsx, cs->vol, &nfu, cs->pathname, &cs->resolvedloc,
+                      nfs3svc_symlink_cbk, cs);
+    if (ret < 0)
+        stat = nfs3_errno_to_nfsstat3(-ret);
 
 nfs3err:
-        if (ret < 0) {
-                nfs3_log_common_res (rpcsvc_request_xid (cs->req),
-                                     NFS3_SYMLINK, stat, -ret);
-                nfs3_symlink_reply (cs->req, stat, NULL, NULL, NULL, NULL);
-                nfs3_call_state_wipe (cs);
-        }
+    if (ret < 0) {
+        nfs3_log_common_res(rpcsvc_request_xid(cs->req), NFS3_SYMLINK, stat,
+                            -ret, cs->resolvedloc.path);
+        nfs3_symlink_reply(cs->req, stat, NULL, NULL, NULL, NULL);
+        nfs3_call_state_wipe(cs);
+    }
 
-        return ret;
+    return ret;
 }
 
-
 int
-nfs3_symlink (rpcsvc_request_t *req, struct nfs3_fh *dirfh, char *name,
-              char *target, sattr3 *sattr)
-{
-        xlator_t                        *vol = NULL;
-        nfsstat3                        stat = NFS3ERR_SERVERFAULT;
-        int                             ret = -EFAULT;
-        struct nfs3_state               *nfs3 = NULL;
-        nfs3_call_state_t               *cs = NULL;
-
-        if ((!req) || (!dirfh) || (!name) || (!target) || (!sattr)) {
-                gf_log (GF_NFS3, GF_LOG_ERROR, "Bad arguments");
-                return -1;
-        }
-
-        nfs3_log_symlink_call (rpcsvc_request_xid (req), dirfh, name,
-                               target);
-        nfs3_validate_gluster_fh (dirfh, stat, nfs3err);
-        nfs3_validate_nfs3_state (req, nfs3, stat, nfs3err, ret);
-        nfs3_validate_strlen_or_goto (name, NFS_NAME_MAX, nfs3err, stat, ret);
-        nfs3_map_fh_to_volume (nfs3, dirfh, req, vol, stat, nfs3err);
-        nfs3_volume_started_check (nfs3, vol, ret, out);
-        nfs3_check_rw_volaccess (nfs3, dirfh->exportid, stat, nfs3err);
-        nfs3_handle_call_state_init (nfs3, cs, req, vol, stat, nfs3err);
-
-        cs->parent = *dirfh;
-        cs->pathname = gf_strdup (target);
-        if (!cs->pathname) {
-                ret = -1;
-                stat = NFS3ERR_SERVERFAULT;
-                goto nfs3err;
-        }
-
-        ret = nfs3_fh_resolve_and_resume (cs, dirfh, name, nfs3_symlink_resume);
-        if (ret < 0)
-                stat = nfs3_errno_to_nfsstat3 (-ret);
+nfs3_symlink(rpcsvc_request_t *req, struct nfs3_fh *dirfh, char *name,
+             char *target, sattr3 *sattr)
+{
+    xlator_t *vol = NULL;
+    nfsstat3 stat = NFS3ERR_SERVERFAULT;
+    int ret = -EFAULT;
+    struct nfs3_state *nfs3 = NULL;
+    nfs3_call_state_t *cs = NULL;
+
+    if ((!req) || (!dirfh) || (!name) || (!target) || (!sattr)) {
+        gf_msg(GF_NFS3, GF_LOG_ERROR, EINVAL, NFS_MSG_INVALID_ENTRY,
+               "Bad arguments");
+        return -1;
+    }
+
+    nfs3_log_symlink_call(rpcsvc_request_xid(req), dirfh, name, target);
+    nfs3_validate_gluster_fh(dirfh, stat, nfs3err);
+    nfs3_validate_nfs3_state(req, nfs3, stat, nfs3err, ret);
+    nfs3_validate_strlen_or_goto(name, NFS_NAME_MAX, nfs3err, stat, ret);
+    nfs3_map_fh_to_volume(nfs3, dirfh, req, vol, stat, nfs3err);
+    nfs3_volume_started_check(nfs3, vol, ret, out);
+    nfs3_check_rw_volaccess(nfs3, dirfh->exportid, stat, nfs3err);
+    nfs3_handle_call_state_init(nfs3, cs, req, vol, stat, nfs3err);
+
+    cs->parent = *dirfh;
+    cs->pathname = gf_strdup(target);
+    if (!cs->pathname) {
+        ret = -1;
+        stat = NFS3ERR_SERVERFAULT;
+        goto nfs3err;
+    }
+
+    ret = nfs3_fh_resolve_and_resume(cs, dirfh, name, nfs3_symlink_resume);
+    if (ret < 0)
+        stat = nfs3_errno_to_nfsstat3(-ret);
 
 nfs3err:
-        if (ret < 0) {
-                nfs3_log_common_res (rpcsvc_request_xid (req), NFS3_SYMLINK,
-                                     stat, -ret);
-                nfs3_symlink_reply (req, stat, NULL, NULL, NULL, NULL);
-                nfs3_call_state_wipe (cs);
-                /* Ret must be 0 after this so that the caller does not
-                 * also send an RPC reply.
-                 */
-                ret = 0;
-        }
+    if (ret < 0) {
+        nfs3_log_common_res(rpcsvc_request_xid(req), NFS3_SYMLINK, stat, -ret,
+                            cs ? cs->resolvedloc.path : NULL);
+        nfs3_symlink_reply(req, stat, NULL, NULL, NULL, NULL);
+        nfs3_call_state_wipe(cs);
+        /* Ret must be 0 after this so that the caller does not
+         * also send an RPC reply.
+         */
+        ret = 0;
+    }
 out:
-        return ret;
+    return ret;
 }
 
-
 int
-nfs3svc_symlink (rpcsvc_request_t *req)
-{
-        char                    name[NFS_PATH_MAX];
-        struct nfs3_fh          dirfh = {{0}, };
-        char                    target[NFS_PATH_MAX];
-        symlink3args            args;
-        int                     ret = RPCSVC_ACTOR_ERROR;
-
-        if (!req)
-                return ret;
-        nfs3_prep_symlink3args (&args, &dirfh, name, target);
-        if (xdr_to_symlink3args (req->msg[0], &args) <= 0) {
-                gf_log (GF_NFS3, GF_LOG_ERROR, "Error decoding args");
-                rpcsvc_request_seterr (req, GARBAGE_ARGS);
-                goto rpcerr;
-        }
+nfs3svc_symlink(rpcsvc_request_t *req)
+{
+    char name[NFS_PATH_MAX];
+    struct nfs3_fh dirfh = {
+        {0},
+    };
+    char target[NFS_PATH_MAX];
+    symlink3args args;
+    int ret = RPCSVC_ACTOR_ERROR;
 
-        ret = nfs3_symlink (req, &dirfh, name, target,
-                            &args.symlink.symlink_attributes);
-        if ((ret < 0) && (ret != RPCSVC_ACTOR_IGNORE)) {
-                gf_log (GF_NFS3, GF_LOG_ERROR, "SYMLINK procedure failed");
-                rpcsvc_request_seterr (req, SYSTEM_ERR);
-                ret = RPCSVC_ACTOR_ERROR;
-        }
+    if (!req)
+        return ret;
+    nfs3_prep_symlink3args(&args, &dirfh, name, target);
+    if (xdr_to_symlink3args(req->msg[0], &args) <= 0) {
+        gf_msg(GF_NFS3, GF_LOG_ERROR, 0, NFS_MSG_ARGS_DECODE_ERROR,
+               "Error decoding args");
+        rpcsvc_request_seterr(req, GARBAGE_ARGS);
+        goto rpcerr;
+    }
+
+    ret = nfs3_symlink(req, &dirfh, name, target,
+                       &args.symlink.symlink_attributes);
+    if ((ret < 0) && (ret != RPCSVC_ACTOR_IGNORE)) {
+        gf_msg(GF_NFS3, GF_LOG_ERROR, EXDEV, NFS_MSG_SYMLINK_FAIL,
+               "SYMLINK procedure failed");
+        rpcsvc_request_seterr(req, SYSTEM_ERR);
+        ret = RPCSVC_ACTOR_ERROR;
+    }
 
 rpcerr:
-        return ret;
+    return ret;
 }
 
-
-int
-nfs3_mknod_reply (rpcsvc_request_t *req, nfsstat3 stat, struct nfs3_fh *fh,
-                  struct iatt *buf, struct iatt *preparent,
-                  struct iatt *postparent)
+static int
+nfs3_mknod_reply(rpcsvc_request_t *req, nfsstat3 stat, struct nfs3_fh *fh,
+                 struct iatt *buf, struct iatt *preparent,
+                 struct iatt *postparent)
 {
-        mknod3res       res = {0, };
-        uint64_t        deviceid = 0;
+    mknod3res res = {
+        0,
+    };
+    uint64_t deviceid = 0;
 
-        deviceid = nfs3_request_xlator_deviceid (req);
-        nfs3_fill_mknod3res (&res, stat, fh, buf, preparent, postparent,
-                             deviceid);
-        nfs3svc_submit_reply (req, (void *)&res,
-                              (nfs3_serializer)xdr_serialize_mknod3res);
+    deviceid = nfs3_request_xlator_deviceid(req);
+    nfs3_fill_mknod3res(&res, stat, fh, buf, preparent, postparent, deviceid);
+    nfs3svc_submit_reply(req, (void *)&res,
+                         (nfs3_serializer)xdr_serialize_mknod3res);
 
-        return 0;
+    return 0;
 }
 
 int32_t
-nfs3svc_mknod_setattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                           int32_t op_ret, int32_t op_errno,
-                           struct iatt *preop, struct iatt *postop, dict_t *xdata)
-{
-        nfsstat3                stat = NFS3ERR_SERVERFAULT;
-        nfs3_call_state_t       *cs = NULL;
-
-        cs = frame->local;
-        if (op_ret == -1) {
-                gf_log (GF_NFS, GF_LOG_WARNING,
-                        "%x: %s => -1 (%s)", rpcsvc_request_xid (cs->req),
-                        cs->resolvedloc.path, strerror (op_errno));
-                stat = nfs3_cbk_errno_status (op_ret, op_errno);
-                goto nfs3err;
-        }
+nfs3svc_mknod_setattr_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                          int32_t op_ret, int32_t op_errno, struct iatt *preop,
+                          struct iatt *postop, dict_t *xdata)
+{
+    nfsstat3 stat = NFS3ERR_SERVERFAULT;
+    nfs3_call_state_t *cs = NULL;
 
-        stat = NFS3_OK;
+    cs = frame->local;
+    if (op_ret == -1) {
+        stat = nfs3_cbk_errno_status(op_ret, op_errno);
+        goto nfs3err;
+    }
+
+    stat = NFS3_OK;
 nfs3err:
-        nfs3_log_newfh_res (rpcsvc_request_xid (cs->req), NFS3_MKNOD, stat,
-                            op_errno, &cs->fh);
-        nfs3_mknod_reply (cs->req, stat, &cs->fh, postop, &cs->preparent,
-                          &cs->postparent);
-        nfs3_call_state_wipe (cs);
-        return 0;
+    nfs3_log_newfh_res(rpcsvc_request_xid(cs->req), NFS3_MKNOD, stat, op_errno,
+                       &cs->fh, cs->resolvedloc.path);
+    nfs3_mknod_reply(cs->req, stat, &cs->fh, postop, &cs->preparent,
+                     &cs->postparent);
+    nfs3_call_state_wipe(cs);
+    return 0;
 }
 
-
-
 int32_t
-nfs3svc_mknod_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                   int32_t op_ret, int32_t op_errno, inode_t *inode,
-                   struct iatt *buf, struct iatt *preparent,
-                   struct iatt *postparent, dict_t *xdata)
+nfs3svc_mknod_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                  int32_t op_ret, int32_t op_errno, inode_t *inode,
+                  struct iatt *buf, struct iatt *preparent,
+                  struct iatt *postparent, dict_t *xdata)
 {
-        nfsstat3                        stat = NFS3ERR_SERVERFAULT;
-        int                             ret = -1;
-        nfs_user_t                      nfu = {0, };
-        nfs3_call_state_t               *cs = NULL;
-
-        cs = frame->local;
-        if (op_ret == -1) {
-                gf_log (GF_NFS, GF_LOG_WARNING,
-                        "%x: %s => -1 (%s)", rpcsvc_request_xid (cs->req),
-                        cs->resolvedloc.path, strerror (op_errno));
-                stat = nfs3_cbk_errno_status (op_ret, op_errno);
-                goto nfs3err;
-        }
+    nfsstat3 stat = NFS3ERR_SERVERFAULT;
+    int ret = -1;
+    nfs_user_t nfu = {
+        0,
+    };
+    nfs3_call_state_t *cs = NULL;
 
-        nfs3_fh_build_child_fh (&cs->parent, buf, &cs->fh);
+    cs = frame->local;
+    if (op_ret == -1) {
+        stat = nfs3_cbk_errno_status(op_ret, op_errno);
+        goto nfs3err;
+    }
 
-        /* Means no attributes were required to be set. */
-        if (!cs->setattr_valid) {
-                stat = NFS3_OK;
-                ret = -1;
-                goto nfs3err;
-        }
+    nfs3_fh_build_child_fh(&cs->parent, buf, &cs->fh);
 
-        cs->preparent = *preparent;
-        cs->postparent = *postparent;
-        nfs_request_user_init (&nfu, cs->req);
-        ret = nfs_setattr (cs->nfsx, cs->vol, &nfu, &cs->resolvedloc,&cs->stbuf,
-                           cs->setattr_valid, nfs3svc_mknod_setattr_cbk, cs);
-        if (ret < 0)
-                stat = nfs3_errno_to_nfsstat3 (-ret);
+    /* Means no attributes were required to be set. */
+    if (!cs->setattr_valid) {
+        stat = NFS3_OK;
+        ret = -1;
+        goto nfs3err;
+    }
+
+    cs->preparent = *preparent;
+    cs->postparent = *postparent;
+    nfs_request_user_init(&nfu, cs->req);
+    ret = nfs_setattr(cs->nfsx, cs->vol, &nfu, &cs->resolvedloc, &cs->stbuf,
+                      cs->setattr_valid, nfs3svc_mknod_setattr_cbk, cs);
+    if (ret < 0)
+        stat = nfs3_errno_to_nfsstat3(-ret);
 nfs3err:
-        if (ret < 0) {
-                nfs3_log_newfh_res (rpcsvc_request_xid (cs->req), NFS3_MKNOD,
-                                    stat,
-                                    op_errno, &cs->fh);
-                nfs3_mknod_reply (cs->req, stat, &cs->fh, buf, preparent,
-                                  postparent);
-                nfs3_call_state_wipe (cs);
-        }
+    if (ret < 0) {
+        nfs3_log_newfh_res(rpcsvc_request_xid(cs->req), NFS3_MKNOD, stat,
+                           op_errno, &cs->fh, cs->resolvedloc.path);
+        nfs3_mknod_reply(cs->req, stat, &cs->fh, buf, preparent, postparent);
+        nfs3_call_state_wipe(cs);
+    }
 
-        return 0;
+    return 0;
 }
 
+static int
+nfs3_mknod_device(nfs3_call_state_t *cs)
+{
+    int ret = -EFAULT;
+    dev_t devnum = 0;
+    mode_t mode = 0;
+    nfs_user_t nfu = {
+        0,
+    };
 
-int
-nfs3_mknod_device (nfs3_call_state_t *cs)
-{
-        int                             ret = -EFAULT;
-        dev_t                           devnum = 0;
-        mode_t                          mode = 0;
-        nfs_user_t                      nfu = {0, };
-
-        if (!cs)
-                return ret;
-
-        devnum = makedev (cs->devnums.specdata1, cs->devnums.specdata2);
-        if (cs->mknodtype == NF3CHR)
-                mode = S_IFCHR;
-        else
-                mode = S_IFBLK;
-
-        nfs_request_user_init (&nfu, cs->req);
-        if (gf_attr_mode_set (cs->setattr_valid)) {
-                cs->setattr_valid &= ~GF_SET_ATTR_MODE;
-                mode |= cs->mode;
-                ret = nfs_mknod (cs->nfsx, cs->vol, &nfu, &cs->resolvedloc,
-                                 mode, devnum, nfs3svc_mknod_cbk, cs);
-        } else
-                ret = nfs_mknod (cs->nfsx, cs->vol, &nfu, &cs->resolvedloc,
-                                 mode, devnum, nfs3svc_mknod_cbk, cs);
-
+    if (!cs)
         return ret;
-}
 
+    devnum = makedev(cs->devnums.specdata1, cs->devnums.specdata2);
+    if (cs->mknodtype == NF3CHR)
+        mode = S_IFCHR;
+    else
+        mode = S_IFBLK;
 
-int
-nfs3_mknod_fifo (nfs3_call_state_t *cs)
-{
-        int                             ret = -EFAULT;
-        nfs_user_t                      nfu = {0, };
-        mode_t                          mode = S_IFIFO;
+    nfs_request_user_init(&nfu, cs->req);
+    if (gf_attr_mode_set(cs->setattr_valid)) {
+        cs->setattr_valid &= ~GF_SET_ATTR_MODE;
+        mode |= cs->mode;
+        ret = nfs_mknod(cs->nfsx, cs->vol, &nfu, &cs->resolvedloc, mode, devnum,
+                        nfs3svc_mknod_cbk, cs);
+    } else
+        ret = nfs_mknod(cs->nfsx, cs->vol, &nfu, &cs->resolvedloc, mode, devnum,
+                        nfs3svc_mknod_cbk, cs);
 
-        if (!cs)
-                return ret;
+    return ret;
+}
 
-        nfs_request_user_init (&nfu, cs->req);
-        if (gf_attr_mode_set (cs->setattr_valid)) {
-                cs->setattr_valid &= ~GF_SET_ATTR_MODE;
-                mode |= cs->mode;
-                ret = nfs_mknod (cs->nfsx, cs->vol, &nfu, &cs->resolvedloc,
-                                 mode, 0, nfs3svc_mknod_cbk, cs);
-        } else
-                ret = nfs_mknod (cs->nfsx, cs->vol, &nfu, &cs->resolvedloc,
-                                 mode, 0, nfs3svc_mknod_cbk, cs);
+static int
+nfs3_mknod_fifo(nfs3_call_state_t *cs, mode_t mode)
+{
+    int ret = -EFAULT;
+    nfs_user_t nfu = {
+        0,
+    };
 
+    if (!cs)
         return ret;
-}
 
+    nfs_request_user_init(&nfu, cs->req);
+    if (gf_attr_mode_set(cs->setattr_valid)) {
+        cs->setattr_valid &= ~GF_SET_ATTR_MODE;
+        mode |= cs->mode;
+        ret = nfs_mknod(cs->nfsx, cs->vol, &nfu, &cs->resolvedloc, mode, 0,
+                        nfs3svc_mknod_cbk, cs);
+    } else
+        ret = nfs_mknod(cs->nfsx, cs->vol, &nfu, &cs->resolvedloc, mode, 0,
+                        nfs3svc_mknod_cbk, cs);
 
-int
-nfs3_mknod_resume (void *carg)
-{
-        nfsstat3                        stat = NFS3ERR_SERVERFAULT;
-        int                             ret = -EFAULT;
-        nfs3_call_state_t               *cs = NULL;
+    return ret;
+}
 
-        if (!carg)
-                return ret;
+static int
+nfs3_mknod_resume(void *carg)
+{
+    nfsstat3 stat = NFS3ERR_SERVERFAULT;
+    int ret = -EFAULT;
+    nfs3_call_state_t *cs = NULL;
 
-        cs = (nfs3_call_state_t *)carg;
-        nfs3_check_new_fh_resolve_status (cs, stat, nfs3err);
-        switch (cs->mknodtype) {
+    if (!carg)
+        return ret;
 
+    cs = (nfs3_call_state_t *)carg;
+    nfs3_check_new_fh_resolve_status(cs, stat, nfs3err);
+    switch (cs->mknodtype) {
         case NF3CHR:
         case NF3BLK:
-                ret = nfs3_mknod_device (cs);
-                break;
+            ret = nfs3_mknod_device(cs);
+            break;
         case NF3SOCK:
+            ret = nfs3_mknod_fifo(cs, S_IFSOCK);
+            break;
         case NF3FIFO:
-                ret = nfs3_mknod_fifo (cs);
-                break;
+            ret = nfs3_mknod_fifo(cs, S_IFIFO);
+            break;
         default:
-                ret = -EBADF;
-                break;
-        }
+            ret = -EBADF;
+            break;
+    }
 
-        if (ret < 0)
-                stat = nfs3_errno_to_nfsstat3 (-ret);
+    if (ret < 0)
+        stat = nfs3_errno_to_nfsstat3(-ret);
 
 nfs3err:
-        if (ret < 0) {
-                nfs3_log_common_res (rpcsvc_request_xid (cs->req), NFS3_MKNOD,
-                                     stat, -ret);
-                nfs3_mknod_reply (cs->req, stat, NULL, NULL, NULL, NULL);
-                nfs3_call_state_wipe (cs);
-        }
+    if (ret < 0) {
+        nfs3_log_common_res(rpcsvc_request_xid(cs->req), NFS3_MKNOD, stat, -ret,
+                            cs->resolvedloc.path);
+        nfs3_mknod_reply(cs->req, stat, NULL, NULL, NULL, NULL);
+        nfs3_call_state_wipe(cs);
+    }
 
-        return ret;
+    return ret;
 }
 
-
-
 int
-nfs3_mknod (rpcsvc_request_t *req, struct nfs3_fh *fh, char *name,
-            mknoddata3 *nodedata)
-{
-        xlator_t                        *vol = NULL;
-        nfsstat3                        stat = NFS3ERR_SERVERFAULT;
-        int                             ret = -EFAULT;
-        struct nfs3_state               *nfs3 = NULL;
-        nfs3_call_state_t               *cs = NULL;
-        sattr3                          *sattr = NULL;
-
-        if ((!req) || (!fh) || (!name) || (!nodedata)) {
-                gf_log (GF_NFS3, GF_LOG_ERROR, "Bad arguments");
-                return -1;
-        }
-
-        nfs3_log_mknod_call (rpcsvc_request_xid (req), fh, name,
-                             nodedata->type);
-        nfs3_validate_gluster_fh (fh, stat, nfs3err);
-        nfs3_validate_nfs3_state (req, nfs3, stat, nfs3err, ret);
-        nfs3_validate_strlen_or_goto (name, NFS_NAME_MAX, nfs3err, stat, ret);
-        nfs3_map_fh_to_volume (nfs3, fh, req, vol, stat, nfs3err);
-        nfs3_volume_started_check (nfs3, vol, ret, out);
-        nfs3_check_rw_volaccess (nfs3, fh->exportid, stat, nfs3err);
-        nfs3_handle_call_state_init (nfs3, cs, req, vol, stat, nfs3err);
-
-        cs->mknodtype = nodedata->type;
-        switch (nodedata->type) {
+nfs3_mknod(rpcsvc_request_t *req, struct nfs3_fh *fh, char *name,
+           mknoddata3 *nodedata)
+{
+    xlator_t *vol = NULL;
+    nfsstat3 stat = NFS3ERR_SERVERFAULT;
+    int ret = -EFAULT;
+    struct nfs3_state *nfs3 = NULL;
+    nfs3_call_state_t *cs = NULL;
+    sattr3 *sattr = NULL;
+
+    if ((!req) || (!fh) || (!name) || (!nodedata)) {
+        gf_msg(GF_NFS3, GF_LOG_ERROR, EINVAL, NFS_MSG_INVALID_ENTRY,
+               "Bad arguments");
+        return -1;
+    }
+
+    nfs3_log_mknod_call(rpcsvc_request_xid(req), fh, name, nodedata->type);
+    nfs3_validate_gluster_fh(fh, stat, nfs3err);
+    nfs3_validate_nfs3_state(req, nfs3, stat, nfs3err, ret);
+    nfs3_validate_strlen_or_goto(name, NFS_NAME_MAX, nfs3err, stat, ret);
+    nfs3_map_fh_to_volume(nfs3, fh, req, vol, stat, nfs3err);
+    nfs3_volume_started_check(nfs3, vol, ret, out);
+    nfs3_check_rw_volaccess(nfs3, fh->exportid, stat, nfs3err);
+    nfs3_handle_call_state_init(nfs3, cs, req, vol, stat, nfs3err);
+
+    cs->mknodtype = nodedata->type;
+    switch (nodedata->type) {
         case NF3CHR:
         case NF3BLK:
-                cs->devnums = nodedata->mknoddata3_u.device.spec;
-                sattr = &nodedata->mknoddata3_u.device.dev_attributes;
-                cs->setattr_valid = nfs3_sattr3_to_setattr_valid (sattr,
-                                                                  &cs->stbuf,
-                                                                  &cs->mode);
-                break;
+            cs->devnums = nodedata->mknoddata3_u.device.spec;
+            sattr = &nodedata->mknoddata3_u.device.dev_attributes;
+            cs->setattr_valid = nfs3_sattr3_to_setattr_valid(sattr, &cs->stbuf,
+                                                             &cs->mode);
+            break;
         case NF3SOCK:
         case NF3FIFO:
-                sattr = &nodedata->mknoddata3_u.pipe_attributes;
-                cs->setattr_valid = nfs3_sattr3_to_setattr_valid (sattr,
-                                                                  &cs->stbuf,
-                                                                  &cs->mode);
-                break;
+            sattr = &nodedata->mknoddata3_u.pipe_attributes;
+            cs->setattr_valid = nfs3_sattr3_to_setattr_valid(sattr, &cs->stbuf,
+                                                             &cs->mode);
+            break;
         default:
-                ret = -EBADF;
-                break;
-        }
+            ret = -EBADF;
+            goto nfs3err;
+    }
 
-        cs->parent = *fh;
-        ret = nfs3_fh_resolve_and_resume (cs, fh, name, nfs3_mknod_resume);
-        if (ret < 0)
-                stat = nfs3_errno_to_nfsstat3 (-ret);
+    cs->parent = *fh;
+    ret = nfs3_fh_resolve_and_resume(cs, fh, name, nfs3_mknod_resume);
+    if (ret < 0)
+        stat = nfs3_errno_to_nfsstat3(-ret);
 
 nfs3err:
-        if (ret < 0) {
-                nfs3_log_common_res (rpcsvc_request_xid (req), NFS3_MKNOD,
-                                     stat, -ret);
-                nfs3_mknod_reply (req, stat, NULL, NULL, NULL, NULL);
-                /* Ret must be 0 after this so that the caller does not
-                 * also send an RPC reply.
-                 */
-                nfs3_call_state_wipe (cs);
-                ret = 0;
-        }
+    if (ret < 0) {
+        nfs3_log_common_res(rpcsvc_request_xid(req), NFS3_MKNOD, stat, -ret,
+                            cs ? cs->resolvedloc.path : NULL);
+        nfs3_mknod_reply(req, stat, NULL, NULL, NULL, NULL);
+        /* Ret must be 0 after this so that the caller does not
+         * also send an RPC reply.
+         */
+        nfs3_call_state_wipe(cs);
+        ret = 0;
+    }
 out:
-        return ret;
+    return ret;
 }
 
-
 int
-nfs3svc_mknod (rpcsvc_request_t *req)
-{
-        char                    name[NFS_PATH_MAX];
-        struct nfs3_fh          fh = {{0}, };
-        mknod3args              args;
-        int                     ret = RPCSVC_ACTOR_ERROR;
-
-        if (!req)
-                return ret;
-        nfs3_prep_mknod3args (&args, &fh, name);
-        if (xdr_to_mknod3args (req->msg[0], &args) <= 0) {
-                gf_log (GF_NFS3, GF_LOG_ERROR, "Error decoding args");
-                rpcsvc_request_seterr (req, GARBAGE_ARGS);
-                goto rpcerr;
-        }
+nfs3svc_mknod(rpcsvc_request_t *req)
+{
+    char name[NFS_PATH_MAX];
+    struct nfs3_fh fh = {
+        {0},
+    };
+    mknod3args args;
+    int ret = RPCSVC_ACTOR_ERROR;
 
-        ret = nfs3_mknod (req, &fh, name, &args.what);
-        if ((ret < 0) && (ret != RPCSVC_ACTOR_IGNORE)) {
-                gf_log (GF_NFS3, GF_LOG_ERROR, "MKNOD procedure failed");
-                rpcsvc_request_seterr (req, SYSTEM_ERR);
-                ret = RPCSVC_ACTOR_ERROR;
-        }
+    if (!req)
+        return ret;
+    nfs3_prep_mknod3args(&args, &fh, name);
+    if (xdr_to_mknod3args(req->msg[0], &args) <= 0) {
+        gf_msg(GF_NFS3, GF_LOG_ERROR, 0, NFS_MSG_ARGS_DECODE_ERROR,
+               "Error decoding args");
+        rpcsvc_request_seterr(req, GARBAGE_ARGS);
+        goto rpcerr;
+    }
+
+    ret = nfs3_mknod(req, &fh, name, &args.what);
+    if ((ret < 0) && (ret != RPCSVC_ACTOR_IGNORE)) {
+        gf_msg(GF_NFS3, GF_LOG_ERROR, -ret, NFS_MSG_MKNOD_FAIL,
+               "MKNOD procedure failed");
+        rpcsvc_request_seterr(req, SYSTEM_ERR);
+        ret = RPCSVC_ACTOR_ERROR;
+    }
 
 rpcerr:
-        return ret;
+    return ret;
 }
 
 int
-nfs3_remove_reply (rpcsvc_request_t *req, nfsstat3 stat, struct iatt *preparent
-                   , struct iatt *postparent)
+nfs3_remove_reply(rpcsvc_request_t *req, nfsstat3 stat, struct iatt *preparent,
+                  struct iatt *postparent)
 {
-        remove3res      res = {0, };
-        uint64_t        deviceid = 0;
+    remove3res res = {
+        0,
+    };
+    uint64_t deviceid = 0;
 
-        deviceid = nfs3_request_xlator_deviceid (req);
-        nfs3_fill_remove3res (&res, stat, preparent, postparent, deviceid);
-        nfs3svc_submit_reply (req, (void *)&res,
-                              (nfs3_serializer)xdr_serialize_remove3res);
-        return 0;
+    deviceid = nfs3_request_xlator_deviceid(req);
+    nfs3_fill_remove3res(&res, stat, preparent, postparent, deviceid);
+    nfs3svc_submit_reply(req, (void *)&res,
+                         (nfs3_serializer)xdr_serialize_remove3res);
+    return 0;
 }
 
-
-
 int32_t
-nfs3svc_remove_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                    int32_t op_ret, int32_t op_errno, struct iatt *preparent,
-                    struct iatt *postparent, dict_t *xdata)
+nfs3svc_remove_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                   int32_t op_ret, int32_t op_errno, struct iatt *preparent,
+                   struct iatt *postparent, dict_t *xdata)
 {
-        nfsstat3                stat = NFS3ERR_SERVERFAULT;
-        nfs3_call_state_t       *cs = NULL;
+    nfsstat3 stat = NFS3ERR_SERVERFAULT;
+    nfs3_call_state_t *cs = NULL;
 
-        cs = frame->local;
-        if (op_ret == -1) {
-                gf_log (GF_NFS, GF_LOG_WARNING,
-                        "%x: %s => -1 (%s)", rpcsvc_request_xid (cs->req),
-                        cs->resolvedloc.path, strerror (op_errno));
-                stat = nfs3_cbk_errno_status (op_ret, op_errno);
-        }
+    cs = frame->local;
+    if (op_ret == -1) {
+        stat = nfs3_cbk_errno_status(op_ret, op_errno);
+    }
 
-        if (op_ret == 0)
-                stat = NFS3_OK;
+    if (op_ret == 0)
+        stat = NFS3_OK;
 
-        nfs3_log_common_res (rpcsvc_request_xid (cs->req), NFS3_REMOVE, stat,
-                             op_errno);
-        nfs3_remove_reply (cs->req, stat, preparent, postparent);
-        nfs3_call_state_wipe (cs);
+    nfs3_log_common_res(rpcsvc_request_xid(cs->req), NFS3_REMOVE, stat,
+                        op_errno, cs->resolvedloc.path);
+    nfs3_remove_reply(cs->req, stat, preparent, postparent);
+    nfs3_call_state_wipe(cs);
 
-        return 0;
+    return 0;
 }
 
-
 int
-__nfs3_remove (nfs3_call_state_t *cs)
-{
-        int                             ret = -EFAULT;
-        nfs_user_t                      nfu = {0, };
-        ia_type_t                       type = 0;
-
-        if (!cs)
-                return ret;
-        type = cs->resolvedloc.inode->ia_type;
-        nfs_request_user_init (&nfu, cs->req);
-        if (IA_ISDIR (type))
-                ret = nfs_rmdir (cs->nfsx, cs->vol, &nfu, &cs->resolvedloc,
-                                 nfs3svc_remove_cbk, cs);
-        else
-                ret = nfs_unlink (cs->nfsx, cs->vol, &nfu, &cs->resolvedloc,
-                                  nfs3svc_remove_cbk, cs);
+__nfs3_remove(nfs3_call_state_t *cs)
+{
+    int ret = -EFAULT;
+    nfs_user_t nfu = {
+        0,
+    };
+    ia_type_t type = 0;
 
+    if (!cs)
         return ret;
-}
+    type = cs->resolvedloc.inode->ia_type;
+    nfs_request_user_init(&nfu, cs->req);
+    if (IA_ISDIR(type))
+        ret = nfs_rmdir(cs->nfsx, cs->vol, &nfu, &cs->resolvedloc,
+                        nfs3svc_remove_cbk, cs);
+    else
+        ret = nfs_unlink(cs->nfsx, cs->vol, &nfu, &cs->resolvedloc,
+                         nfs3svc_remove_cbk, cs);
 
+    return ret;
+}
 
 int
-nfs3_remove_resume (void *carg)
+nfs3_remove_resume(void *carg)
 {
-        nfsstat3                        stat = NFS3ERR_SERVERFAULT;
-        int                             ret = -EFAULT;
-        nfs3_call_state_t               *cs = NULL;
+    nfsstat3 stat = NFS3ERR_SERVERFAULT;
+    int ret = -EFAULT;
+    nfs3_call_state_t *cs = NULL;
 
-        if (!carg)
-                return ret;
+    if (!carg)
+        return ret;
 
-        cs = (nfs3_call_state_t *)carg;
-        nfs3_check_fh_resolve_status (cs, stat, nfs3err);
-        ret = __nfs3_remove (cs);
-        if (ret < 0)
-                stat = nfs3_errno_to_nfsstat3 (-ret);
+    cs = (nfs3_call_state_t *)carg;
+    nfs3_check_fh_resolve_status(cs, stat, nfs3err);
+    ret = __nfs3_remove(cs);
+    if (ret < 0)
+        stat = nfs3_errno_to_nfsstat3(-ret);
 
 nfs3err:
-        if (ret < 0) {
-                nfs3_log_common_res (rpcsvc_request_xid (cs->req), NFS3_REMOVE,
-                                     stat, -ret);
-                nfs3_remove_reply (cs->req, stat, NULL, NULL);
-                nfs3_call_state_wipe (cs);
-        }
+    if (ret < 0) {
+        nfs3_log_common_res(rpcsvc_request_xid(cs->req), NFS3_REMOVE, stat,
+                            -ret, cs->resolvedloc.path);
+        nfs3_remove_reply(cs->req, stat, NULL, NULL);
+        nfs3_call_state_wipe(cs);
+    }
 
-        return ret;
+    return ret;
 }
 
-
 int
-nfs3_remove (rpcsvc_request_t *req, struct nfs3_fh *fh, char *name)
-{
-        xlator_t                        *vol = NULL;
-        nfsstat3                        stat = NFS3ERR_SERVERFAULT;
-        int                             ret = -EFAULT;
-        struct nfs3_state               *nfs3 = NULL;
-        nfs3_call_state_t               *cs = NULL;
-
-        if ((!req) || (!fh) || (!name)) {
-                gf_log (GF_NFS3, GF_LOG_ERROR, "Bad arguments");
-                return -1;
-        }
-
-        nfs3_log_fh_entry_call (rpcsvc_request_xid (req), "REMOVE", fh,
-                                name);
-        nfs3_validate_gluster_fh (fh, stat, nfs3err);
-        nfs3_validate_nfs3_state (req, nfs3, stat, nfs3err, ret);
-        nfs3_validate_strlen_or_goto (name, NFS_NAME_MAX, nfs3err, stat, ret);
-        nfs3_map_fh_to_volume (nfs3, fh, req, vol, stat, nfs3err);
-        nfs3_volume_started_check (nfs3, vol, ret, out);
-        nfs3_check_rw_volaccess (nfs3, fh->exportid, stat, nfs3err);
-        nfs3_handle_call_state_init (nfs3, cs, req, vol, stat, nfs3err);
-
-        ret = nfs3_fh_resolve_and_resume (cs, fh, name, nfs3_remove_resume);
-        if (ret < 0)
-                stat = nfs3_errno_to_nfsstat3 (-ret);
+nfs3_remove(rpcsvc_request_t *req, struct nfs3_fh *fh, char *name)
+{
+    xlator_t *vol = NULL;
+    nfsstat3 stat = NFS3ERR_SERVERFAULT;
+    int ret = -EFAULT;
+    struct nfs3_state *nfs3 = NULL;
+    nfs3_call_state_t *cs = NULL;
+
+    if ((!req) || (!fh) || (!name)) {
+        gf_msg(GF_NFS3, GF_LOG_ERROR, EINVAL, NFS_MSG_INVALID_ENTRY,
+               "Bad arguments");
+        return -1;
+    }
+
+    nfs3_log_fh_entry_call(rpcsvc_request_xid(req), "REMOVE", fh, name);
+    nfs3_validate_gluster_fh(fh, stat, nfs3err);
+    nfs3_validate_nfs3_state(req, nfs3, stat, nfs3err, ret);
+    nfs3_validate_strlen_or_goto(name, NFS_NAME_MAX, nfs3err, stat, ret);
+    nfs3_map_fh_to_volume(nfs3, fh, req, vol, stat, nfs3err);
+    nfs3_volume_started_check(nfs3, vol, ret, out);
+    nfs3_check_rw_volaccess(nfs3, fh->exportid, stat, nfs3err);
+    nfs3_handle_call_state_init(nfs3, cs, req, vol, stat, nfs3err);
+
+    ret = nfs3_fh_resolve_and_resume(cs, fh, name, nfs3_remove_resume);
+    if (ret < 0)
+        stat = nfs3_errno_to_nfsstat3(-ret);
 
 nfs3err:
-        if (ret < 0) {
-                nfs3_log_common_res (rpcsvc_request_xid (req), NFS3_REMOVE,
-                                     stat, -ret);
-                nfs3_remove_reply (req, stat, NULL, NULL);
-                nfs3_call_state_wipe (cs);
-                /* Ret must be 0 after this so that the caller does not
-                 * also send an RPC reply.
-                 */
-                ret = 0;
-        }
+    if (ret < 0) {
+        nfs3_log_common_res(rpcsvc_request_xid(req), NFS3_REMOVE, stat, -ret,
+                            cs ? cs->resolvedloc.path : NULL);
+        nfs3_remove_reply(req, stat, NULL, NULL);
+        nfs3_call_state_wipe(cs);
+        /* Ret must be 0 after this so that the caller does not
+         * also send an RPC reply.
+         */
+        ret = 0;
+    }
 out:
-        return ret;
+    return ret;
 }
 
-
 int
-nfs3svc_remove (rpcsvc_request_t *req)
-{
-        char                    name[NFS_PATH_MAX];
-        struct nfs3_fh          fh = {{0}, };
-        remove3args             args;
-        int                     ret = RPCSVC_ACTOR_ERROR;
-
-        if (!req)
-                return ret;
-        nfs3_prep_remove3args (&args, &fh, name);
-        if (xdr_to_remove3args (req->msg[0], &args) <= 0) {
-                gf_log (GF_NFS3, GF_LOG_ERROR, "Error decoding args");
-                rpcsvc_request_seterr (req, GARBAGE_ARGS);
-                goto rpcerr;
-        }
+nfs3svc_remove(rpcsvc_request_t *req)
+{
+    char name[NFS_PATH_MAX];
+    struct nfs3_fh fh = {
+        {0},
+    };
+    remove3args args;
+    int ret = RPCSVC_ACTOR_ERROR;
 
-        ret = nfs3_remove (req, &fh, name);
-        if ((ret < 0) && (ret != RPCSVC_ACTOR_IGNORE)) {
-                gf_log (GF_NFS3, GF_LOG_ERROR, "REMOVE procedure failed");
-                rpcsvc_request_seterr (req, SYSTEM_ERR);
-                ret = RPCSVC_ACTOR_ERROR;
-        }
+    if (!req)
+        return ret;
+    nfs3_prep_remove3args(&args, &fh, name);
+    if (xdr_to_remove3args(req->msg[0], &args) <= 0) {
+        gf_msg(GF_NFS3, GF_LOG_ERROR, 0, NFS_MSG_ARGS_DECODE_ERROR,
+               "Error decoding args");
+        rpcsvc_request_seterr(req, GARBAGE_ARGS);
+        goto rpcerr;
+    }
+
+    ret = nfs3_remove(req, &fh, name);
+    if ((ret < 0) && (ret != RPCSVC_ACTOR_IGNORE)) {
+        gf_msg(GF_NFS3, GF_LOG_ERROR, 0, NFS_MSG_REMOVE_FAIL,
+               "REMOVE procedure failed");
+        rpcsvc_request_seterr(req, SYSTEM_ERR);
+        ret = RPCSVC_ACTOR_ERROR;
+    }
 
 rpcerr:
-        return ret;
+    return ret;
 }
 
-
 int
-nfs3_rmdir_reply (rpcsvc_request_t *req, nfsstat3 stat, struct iatt *preparent,
-                  struct iatt *postparent)
+nfs3_rmdir_reply(rpcsvc_request_t *req, nfsstat3 stat, struct iatt *preparent,
+                 struct iatt *postparent)
 {
-        rmdir3res       res = {0, };
-        uint64_t        deviceid = 0;
+    rmdir3res res = {
+        0,
+    };
+    uint64_t deviceid = 0;
 
-        deviceid = nfs3_request_xlator_deviceid (req);
-        nfs3_fill_rmdir3res (&res, stat, preparent, postparent, deviceid);
-        nfs3svc_submit_reply (req, (void *)&res,
-                              (nfs3_serializer)xdr_serialize_rmdir3res);
-        return 0;
+    deviceid = nfs3_request_xlator_deviceid(req);
+    nfs3_fill_rmdir3res(&res, stat, preparent, postparent, deviceid);
+    nfs3svc_submit_reply(req, (void *)&res,
+                         (nfs3_serializer)xdr_serialize_rmdir3res);
+    return 0;
 }
 
-
 int32_t
-nfs3svc_rmdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                   int32_t op_ret, int32_t op_errno, struct iatt *preparent,
-                   struct iatt *postparent, dict_t *xdata)
+nfs3svc_rmdir_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                  int32_t op_ret, int32_t op_errno, struct iatt *preparent,
+                  struct iatt *postparent, dict_t *xdata)
 {
-        nfsstat3                stat = NFS3ERR_SERVERFAULT;
-        nfs3_call_state_t       *cs = NULL;
+    nfsstat3 stat = NFS3ERR_SERVERFAULT;
+    nfs3_call_state_t *cs = NULL;
 
-        cs = frame->local;
-        if (op_ret == -1) {
-                gf_log (GF_NFS, GF_LOG_WARNING,
-                        "%x: %s => -1 (%s)", rpcsvc_request_xid (cs->req),
-                        cs->resolvedloc.path, strerror (op_errno));
-                stat = nfs3_cbk_errno_status (op_ret, op_errno);
-        } else {
-                stat = NFS3_OK;
-        }
+    cs = frame->local;
+    if (op_ret == -1) {
+        gf_msg(GF_NFS, GF_LOG_WARNING, op_errno, NFS_MSG_RMDIR_CBK,
+               "%x: %s => -1 (%s)", rpcsvc_request_xid(cs->req),
+               cs->resolvedloc.path, strerror(op_errno));
+        stat = nfs3_cbk_errno_status(op_ret, op_errno);
+    } else {
+        stat = NFS3_OK;
+    }
 
-        nfs3_log_common_res (rpcsvc_request_xid (cs->req), NFS3_RMDIR, stat,
-                             op_errno);
-        nfs3_rmdir_reply (cs->req, stat, preparent, postparent);
-        nfs3_call_state_wipe (cs);
+    nfs3_log_common_res(rpcsvc_request_xid(cs->req), NFS3_RMDIR, stat, op_errno,
+                        cs->resolvedloc.path);
+    nfs3_rmdir_reply(cs->req, stat, preparent, postparent);
+    nfs3_call_state_wipe(cs);
 
-        return 0;
+    return 0;
 }
 
 int
-nfs3_rmdir_resume (void *carg)
+nfs3_rmdir_resume(void *carg)
 {
-        nfsstat3                        stat = NFS3ERR_SERVERFAULT;
-        int                             ret = -EFAULT;
-        nfs3_call_state_t               *cs = NULL;
-        nfs_user_t                      nfu = {0, };
+    nfsstat3 stat = NFS3ERR_SERVERFAULT;
+    int ret = -EFAULT;
+    nfs3_call_state_t *cs = NULL;
+    nfs_user_t nfu = {
+        0,
+    };
 
-        if (!carg)
-                return ret;
+    if (!carg)
+        return ret;
 
-        cs = (nfs3_call_state_t *)carg;
-        nfs3_check_fh_resolve_status (cs, stat, nfs3err);
-        nfs_request_user_init (&nfu, cs->req);
-        ret = nfs_rmdir (cs->nfsx, cs->vol, &nfu, &cs->resolvedloc,
-                         nfs3svc_rmdir_cbk, cs);
-        if (ret < 0)
-                stat = nfs3_errno_to_nfsstat3 (-ret);
+    cs = (nfs3_call_state_t *)carg;
+    nfs3_check_fh_resolve_status(cs, stat, nfs3err);
+    nfs_request_user_init(&nfu, cs->req);
+    ret = nfs_rmdir(cs->nfsx, cs->vol, &nfu, &cs->resolvedloc,
+                    nfs3svc_rmdir_cbk, cs);
+    if (ret < 0)
+        stat = nfs3_errno_to_nfsstat3(-ret);
 
 nfs3err:
-        if (ret < 0) {
-                nfs3_log_common_res (rpcsvc_request_xid (cs->req), NFS3_RMDIR,
-                                     stat, -ret);
-                nfs3_rmdir_reply (cs->req, stat, NULL, NULL);
-                nfs3_call_state_wipe (cs);
-        }
+    if (ret < 0) {
+        nfs3_log_common_res(rpcsvc_request_xid(cs->req), NFS3_RMDIR, stat, -ret,
+                            cs->resolvedloc.path);
+        nfs3_rmdir_reply(cs->req, stat, NULL, NULL);
+        nfs3_call_state_wipe(cs);
+    }
 
-        return ret;
+    return ret;
 }
 
-
-
 int
-nfs3_rmdir (rpcsvc_request_t *req, struct nfs3_fh *fh, char *name)
-{
-        xlator_t                        *vol = NULL;
-        nfsstat3                        stat = NFS3ERR_SERVERFAULT;
-        int                             ret = -EFAULT;
-        struct nfs3_state               *nfs3 = NULL;
-        nfs3_call_state_t               *cs = NULL;
-
-        if ((!req) || (!fh) || (!name)) {
-                gf_log (GF_NFS3, GF_LOG_ERROR, "Bad arguments");
-                return -1;
-        }
-
-        nfs3_log_fh_entry_call (rpcsvc_request_xid (req), "RMDIR", fh,
-                                name);
-        nfs3_validate_gluster_fh (fh, stat, nfs3err);
-        nfs3_validate_nfs3_state (req, nfs3, stat, nfs3err, ret);
-        nfs3_validate_strlen_or_goto (name, NFS_NAME_MAX, nfs3err, stat, ret);
-        nfs3_map_fh_to_volume (nfs3, fh, req, vol, stat, nfs3err);
-        nfs3_volume_started_check (nfs3, vol, ret, out);
-        nfs3_check_rw_volaccess (nfs3, fh->exportid, stat, nfs3err);
-        nfs3_handle_call_state_init (nfs3, cs, req, vol, stat, nfs3err);
-
-        ret = nfs3_fh_resolve_and_resume (cs, fh, name, nfs3_rmdir_resume);
-        if (ret < 0)
-                stat = nfs3_errno_to_nfsstat3 (-ret);
+nfs3_rmdir(rpcsvc_request_t *req, struct nfs3_fh *fh, char *name)
+{
+    xlator_t *vol = NULL;
+    nfsstat3 stat = NFS3ERR_SERVERFAULT;
+    int ret = -EFAULT;
+    struct nfs3_state *nfs3 = NULL;
+    nfs3_call_state_t *cs = NULL;
+
+    if ((!req) || (!fh) || (!name)) {
+        gf_msg(GF_NFS3, GF_LOG_ERROR, EINVAL, NFS_MSG_INVALID_ENTRY,
+               "Bad arguments");
+        return -1;
+    }
+
+    nfs3_log_fh_entry_call(rpcsvc_request_xid(req), "RMDIR", fh, name);
+    nfs3_validate_gluster_fh(fh, stat, nfs3err);
+    nfs3_validate_nfs3_state(req, nfs3, stat, nfs3err, ret);
+    nfs3_validate_strlen_or_goto(name, NFS_NAME_MAX, nfs3err, stat, ret);
+    nfs3_map_fh_to_volume(nfs3, fh, req, vol, stat, nfs3err);
+    nfs3_volume_started_check(nfs3, vol, ret, out);
+    nfs3_check_rw_volaccess(nfs3, fh->exportid, stat, nfs3err);
+    nfs3_handle_call_state_init(nfs3, cs, req, vol, stat, nfs3err);
+
+    ret = nfs3_fh_resolve_and_resume(cs, fh, name, nfs3_rmdir_resume);
+    if (ret < 0)
+        stat = nfs3_errno_to_nfsstat3(-ret);
 
 nfs3err:
-        if (ret < 0) {
-                nfs3_log_common_res (rpcsvc_request_xid (req), NFS3_RMDIR,
-                                     stat, -ret);
-                nfs3_rmdir_reply (req, stat, NULL, NULL);
-                nfs3_call_state_wipe (cs);
-                /* Ret must be 0 after this so that the caller does not
-                 * also send an RPC reply.
-                 */
-                ret = 0;
-        }
+    if (ret < 0) {
+        nfs3_log_common_res(rpcsvc_request_xid(req), NFS3_RMDIR, stat, -ret,
+                            cs ? cs->resolvedloc.path : NULL);
+        nfs3_rmdir_reply(req, stat, NULL, NULL);
+        nfs3_call_state_wipe(cs);
+        /* Ret must be 0 after this so that the caller does not
+         * also send an RPC reply.
+         */
+        ret = 0;
+    }
 out:
-        return ret;
+    return ret;
 }
 
-
 int
-nfs3svc_rmdir (rpcsvc_request_t *req)
-{
-        char                    name[NFS_PATH_MAX];
-        struct nfs3_fh          fh = {{0}, };
-        rmdir3args              args;
-        int                     ret = RPCSVC_ACTOR_ERROR;
-
-        if (!req)
-                return ret;
-        nfs3_prep_rmdir3args (&args, &fh, name);
-        if (xdr_to_rmdir3args (req->msg[0], &args) <= 0) {
-                gf_log (GF_NFS3, GF_LOG_ERROR, "Error decoding args");
-                rpcsvc_request_seterr (req, GARBAGE_ARGS);
-                goto rpcerr;
-        }
+nfs3svc_rmdir(rpcsvc_request_t *req)
+{
+    char name[NFS_PATH_MAX];
+    struct nfs3_fh fh = {
+        {0},
+    };
+    rmdir3args args;
+    int ret = RPCSVC_ACTOR_ERROR;
 
-        ret = nfs3_rmdir (req, &fh, name);
-        if ((ret < 0) && (ret != RPCSVC_ACTOR_IGNORE)) {
-                gf_log (GF_NFS3, GF_LOG_ERROR, "RMDIR procedure failed");
-                rpcsvc_request_seterr (req, SYSTEM_ERR);
-                ret = RPCSVC_ACTOR_ERROR;
-        }
+    if (!req)
+        return ret;
+    nfs3_prep_rmdir3args(&args, &fh, name);
+    if (xdr_to_rmdir3args(req->msg[0], &args) <= 0) {
+        gf_msg(GF_NFS3, GF_LOG_ERROR, 0, NFS_MSG_ARGS_DECODE_ERROR,
+               "Error decoding args");
+        rpcsvc_request_seterr(req, GARBAGE_ARGS);
+        goto rpcerr;
+    }
+
+    ret = nfs3_rmdir(req, &fh, name);
+    if ((ret < 0) && (ret != RPCSVC_ACTOR_IGNORE)) {
+        gf_msg(GF_NFS3, GF_LOG_ERROR, -ret, NFS_MSG_DIR_OP_FAIL,
+               "RMDIR procedure failed");
+        rpcsvc_request_seterr(req, SYSTEM_ERR);
+        ret = RPCSVC_ACTOR_ERROR;
+    }
 
 rpcerr:
-        return ret;
+    return ret;
 }
 
-
 int
-nfs3_rename_reply (rpcsvc_request_t *req, nfsstat3 stat, struct iatt *buf,
-                   struct iatt *preoldparent, struct iatt *postoldparent,
-                   struct iatt *prenewparent, struct iatt *postnewparent)
+nfs3_rename_reply(rpcsvc_request_t *req, nfsstat3 stat, struct iatt *buf,
+                  struct iatt *preoldparent, struct iatt *postoldparent,
+                  struct iatt *prenewparent, struct iatt *postnewparent)
 {
-        rename3res      res = {0, };
-        uint64_t        deviceid = 0;
+    rename3res res = {
+        0,
+    };
+    uint64_t deviceid = 0;
 
-        deviceid = nfs3_request_xlator_deviceid (req);
-        nfs3_fill_rename3res (&res, stat, buf, preoldparent, postoldparent,
-                              prenewparent, postnewparent, deviceid);
+    deviceid = nfs3_request_xlator_deviceid(req);
+    nfs3_fill_rename3res(&res, stat, buf, preoldparent, postoldparent,
+                         prenewparent, postnewparent, deviceid);
 
-        nfs3svc_submit_reply (req, (void *)&res,
-                              (nfs3_serializer) xdr_serialize_rename3res);
+    nfs3svc_submit_reply(req, (void *)&res,
+                         (nfs3_serializer)xdr_serialize_rename3res);
 
-        return 0;
+    return 0;
 }
 
-
-
 int32_t
-nfs3svc_rename_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                    int32_t op_ret, int32_t op_errno, struct iatt *buf,
-                    struct iatt *preoldparent, struct iatt *postoldparent,
-                    struct iatt *prenewparent, struct iatt *postnewparent,
-                    dict_t *xdata)
+nfs3svc_rename_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                   int32_t op_ret, int32_t op_errno, struct iatt *buf,
+                   struct iatt *preoldparent, struct iatt *postoldparent,
+                   struct iatt *prenewparent, struct iatt *postnewparent,
+                   dict_t *xdata)
 {
-        int                     ret = -EFAULT;
-        nfsstat3                stat = NFS3ERR_SERVERFAULT;
-        nfs3_call_state_t       *cs = NULL;
-
-        cs = frame->local;
-        if (op_ret == -1) {
-                gf_log (GF_NFS, GF_LOG_WARNING,
-                        "%x: rename %s -> %s => -1 (%s)",
-                        rpcsvc_request_xid (cs->req), cs->oploc.path,
-                        cs->resolvedloc.path, strerror (op_errno));
-                stat = nfs3_cbk_errno_status (op_ret, op_errno);
-                goto nfs3err;
-        }
+    int ret = -EFAULT;
+    nfsstat3 stat = NFS3ERR_SERVERFAULT;
+    nfs3_call_state_t *cs = NULL;
 
-        stat = NFS3_OK;
+    cs = frame->local;
+    if (op_ret == -1) {
+        stat = nfs3_cbk_errno_status(op_ret, op_errno);
+        goto nfs3err;
+    }
+
+    stat = NFS3_OK;
 nfs3err:
-        nfs3_log_common_res (rpcsvc_request_xid (cs->req), NFS3_RENAME, stat,
-                             -ret);
-        nfs3_rename_reply (cs->req, stat, buf, preoldparent, postoldparent,
-                           prenewparent, postnewparent);
-        nfs3_call_state_wipe (cs);
-        return 0;
+    nfs3_log_common_res(rpcsvc_request_xid(cs->req), NFS3_RENAME, stat, -ret,
+                        cs->resolvedloc.path);
+    nfs3_rename_reply(cs->req, stat, buf, preoldparent, postoldparent,
+                      prenewparent, postnewparent);
+    nfs3_call_state_wipe(cs);
+    return 0;
 }
 
-
 int
-nfs3_rename_resume_dst (void *carg)
-{
-        nfsstat3                        stat = NFS3ERR_SERVERFAULT;
-        int                             ret = -EFAULT;
-        nfs3_call_state_t               *cs = NULL;
-        nfs_user_t                      nfu = {0, };
-
-        if (!carg)
-                return ret;
-
-        cs = (nfs3_call_state_t *)carg;
-        nfs3_check_new_fh_resolve_status (cs, stat, nfs3err);
-        cs->parent = cs->resolvefh;
-        nfs_request_user_init (&nfu, cs->req);
-        ret = nfs_rename (cs->nfsx, cs->vol, &nfu, &cs->oploc, &cs->resolvedloc,
-                          nfs3svc_rename_cbk, cs);
-        if (ret < 0)
-                stat = nfs3_errno_to_nfsstat3 (-ret);
-
-nfs3err:
-        if (ret < 0) {
-                nfs3_log_common_res (rpcsvc_request_xid (cs->req), NFS3_RENAME,
-                                     stat, -ret);
-                nfs3_rename_reply (cs->req, stat, NULL, NULL, NULL, NULL, NULL);
-                nfs3_call_state_wipe (cs);
-        }
+nfs3_rename_resume_dst(void *carg)
+{
+    nfsstat3 stat = NFS3ERR_SERVERFAULT;
+    int ret = -EFAULT;
+    nfs3_call_state_t *cs = NULL;
+    nfs_user_t nfu = {
+        0,
+    };
 
+    if (!carg)
         return ret;
-}
 
+    cs = (nfs3_call_state_t *)carg;
+    nfs3_check_new_fh_resolve_status(cs, stat, nfs3err);
+    cs->parent = cs->resolvefh;
+    nfs_request_user_init(&nfu, cs->req);
+    ret = nfs_rename(cs->nfsx, cs->vol, &nfu, &cs->oploc, &cs->resolvedloc,
+                     nfs3svc_rename_cbk, cs);
+    if (ret < 0)
+        stat = nfs3_errno_to_nfsstat3(-ret);
+
+nfs3err:
+    if (ret < 0) {
+        nfs3_log_common_res(rpcsvc_request_xid(cs->req), NFS3_RENAME, stat,
+                            -ret, cs->resolvedloc.path);
+        nfs3_rename_reply(cs->req, stat, NULL, NULL, NULL, NULL, NULL);
+        nfs3_call_state_wipe(cs);
+    }
 
+    return ret;
+}
 
 int
-nfs3_rename_resume_src (void *carg)
+nfs3_rename_resume_src(void *carg)
 {
-        nfsstat3                        stat = NFS3ERR_SERVERFAULT;
-        int                             ret = -EFAULT;
-        nfs3_call_state_t               *cs = NULL;
+    nfsstat3 stat = NFS3ERR_SERVERFAULT;
+    int ret = -EFAULT;
+    nfs3_call_state_t *cs = NULL;
 
-        if (!carg)
-                return ret;
+    if (!carg)
+        return ret;
 
-        cs = (nfs3_call_state_t *)carg;
-        nfs3_check_fh_resolve_status (cs, stat, nfs3err);
-        /* Copy the resolved loc for the source file into another loc
-         * for safekeeping till we resolve the dest loc.
-         */
-        nfs_loc_copy (&cs->oploc, &cs->resolvedloc);
-        nfs_loc_wipe (&cs->resolvedloc);
-        GF_FREE (cs->resolventry);
+    cs = (nfs3_call_state_t *)carg;
+    nfs3_check_fh_resolve_status(cs, stat, nfs3err);
+    /* Copy the resolved loc for the source file into another loc
+     * for safekeeping till we resolve the dest loc.
+     */
+    nfs_loc_copy(&cs->oploc, &cs->resolvedloc);
+    nfs_loc_wipe(&cs->resolvedloc);
+    GF_FREE(cs->resolventry);
 
-        ret = nfs3_fh_resolve_and_resume (cs, &cs->fh, cs->pathname,
-                                          nfs3_rename_resume_dst);
-        if (ret < 0)
-                stat = nfs3_errno_to_nfsstat3 (-ret);
+    ret = nfs3_fh_resolve_and_resume(cs, &cs->fh, cs->pathname,
+                                     nfs3_rename_resume_dst);
+    if (ret < 0)
+        stat = nfs3_errno_to_nfsstat3(-ret);
 
 nfs3err:
-        if (ret < 0) {
-                nfs3_log_common_res (rpcsvc_request_xid (cs->req), NFS3_RENAME,
-                                     stat, -ret);
-                nfs3_rename_reply (cs->req, stat, NULL, NULL, NULL, NULL, NULL);
-                nfs3_call_state_wipe (cs);
-        }
+    if (ret < 0) {
+        nfs3_log_common_res(rpcsvc_request_xid(cs->req), NFS3_RENAME, stat,
+                            -ret, cs->resolvedloc.path);
+        nfs3_rename_reply(cs->req, stat, NULL, NULL, NULL, NULL, NULL);
+        nfs3_call_state_wipe(cs);
+    }
 
-        return ret;
+    return ret;
 }
 
-
 int
-nfs3_rename (rpcsvc_request_t *req, struct nfs3_fh *olddirfh, char *oldname,
-             struct nfs3_fh *newdirfh, char *newname)
-{
-        xlator_t                        *vol = NULL;
-        nfsstat3                        stat = NFS3ERR_SERVERFAULT;
-        int                             ret = -EFAULT;
-        struct nfs3_state               *nfs3 = NULL;
-        nfs3_call_state_t               *cs = NULL;
-
-        if ((!req) || (!olddirfh) || (!oldname) || (!newdirfh) || (!newname)) {
-                gf_log (GF_NFS3, GF_LOG_ERROR, "Bad arguments");
-                return -1;
-        }
-
-        nfs3_log_rename_call (rpcsvc_request_xid (req), olddirfh, oldname,
-                              newdirfh, newname);
-        nfs3_validate_gluster_fh (olddirfh, stat, nfs3err);
-        nfs3_validate_gluster_fh (newdirfh, stat, nfs3err);
-        nfs3_validate_nfs3_state (req, nfs3, stat, nfs3err, ret);
-        nfs3_validate_strlen_or_goto(oldname, NFS_NAME_MAX, nfs3err, stat, ret);
-        nfs3_validate_strlen_or_goto(newname, NFS_NAME_MAX, nfs3err, stat, ret);
-        nfs3_map_fh_to_volume (nfs3, olddirfh, req, vol, stat, nfs3err);
-        nfs3_volume_started_check (nfs3, vol, ret, out);
-        nfs3_check_rw_volaccess (nfs3, olddirfh->exportid, stat, nfs3err);
-        nfs3_handle_call_state_init (nfs3, cs, req, vol, stat, nfs3err);
-
-        /* While we resolve the source (fh, name) pair, we need to keep a copy
-         * of the dest (fh,name) pair.
-         */
-        cs->fh = *newdirfh;
-        cs->pathname = gf_strdup (newname);
-        if (!cs->pathname) {
-                stat = NFS3ERR_SERVERFAULT;
-                ret = -1;
-                goto nfs3err;
-        }
-
-        ret = nfs3_fh_resolve_and_resume (cs, olddirfh, oldname,
-                                          nfs3_rename_resume_src);
-        if (ret < 0)
-                stat = nfs3_errno_to_nfsstat3 (-ret);
+nfs3_rename(rpcsvc_request_t *req, struct nfs3_fh *olddirfh, char *oldname,
+            struct nfs3_fh *newdirfh, char *newname)
+{
+    xlator_t *vol = NULL;
+    nfsstat3 stat = NFS3ERR_SERVERFAULT;
+    int ret = -EFAULT;
+    struct nfs3_state *nfs3 = NULL;
+    nfs3_call_state_t *cs = NULL;
+
+    if ((!req) || (!olddirfh) || (!oldname) || (!newdirfh) || (!newname)) {
+        gf_msg(GF_NFS3, GF_LOG_ERROR, EINVAL, NFS_MSG_INVALID_ENTRY,
+               "Bad arguments");
+        return -1;
+    }
+
+    nfs3_log_rename_call(rpcsvc_request_xid(req), olddirfh, oldname, newdirfh,
+                         newname);
+    nfs3_validate_gluster_fh(olddirfh, stat, nfs3err);
+    nfs3_validate_gluster_fh(newdirfh, stat, nfs3err);
+    nfs3_validate_nfs3_state(req, nfs3, stat, nfs3err, ret);
+    nfs3_validate_strlen_or_goto(oldname, NFS_NAME_MAX, nfs3err, stat, ret);
+    nfs3_validate_strlen_or_goto(newname, NFS_NAME_MAX, nfs3err, stat, ret);
+    nfs3_map_fh_to_volume(nfs3, olddirfh, req, vol, stat, nfs3err);
+    nfs3_volume_started_check(nfs3, vol, ret, out);
+    nfs3_check_rw_volaccess(nfs3, olddirfh->exportid, stat, nfs3err);
+    nfs3_handle_call_state_init(nfs3, cs, req, vol, stat, nfs3err);
+
+    /* While we resolve the source (fh, name) pair, we need to keep a copy
+     * of the dest (fh,name) pair.
+     */
+    cs->fh = *newdirfh;
+    cs->pathname = gf_strdup(newname);
+    if (!cs->pathname) {
+        stat = NFS3ERR_SERVERFAULT;
+        ret = -1;
+        goto nfs3err;
+    }
+
+    ret = nfs3_fh_resolve_and_resume(cs, olddirfh, oldname,
+                                     nfs3_rename_resume_src);
+    if (ret < 0)
+        stat = nfs3_errno_to_nfsstat3(-ret);
 
 nfs3err:
-        if (ret < 0) {
-                nfs3_log_common_res (rpcsvc_request_xid (req), NFS3_RENAME,
-                                     stat, -ret);
-                nfs3_rename_reply (req, stat, NULL, NULL, NULL, NULL, NULL);
-                nfs3_call_state_wipe (cs);
-                /* Ret must be 0 after this so that the caller does not
-                 * also send an RPC reply.
-                 */
-                ret = 0;
-        }
+    if (ret < 0) {
+        nfs3_log_common_res(rpcsvc_request_xid(req), NFS3_RENAME, stat, -ret,
+                            cs ? cs->resolvedloc.path : NULL);
+        nfs3_rename_reply(req, stat, NULL, NULL, NULL, NULL, NULL);
+        nfs3_call_state_wipe(cs);
+        /* Ret must be 0 after this so that the caller does not
+         * also send an RPC reply.
+         */
+        ret = 0;
+    }
 out:
-        return ret;
+    return ret;
 }
 
-
 int
-nfs3svc_rename (rpcsvc_request_t *req)
-{
-        char                    newname[NFS_PATH_MAX];
-        char                    oldname[NFS_PATH_MAX];
-        struct nfs3_fh          olddirfh = {{0}, };
-        struct nfs3_fh          newdirfh = {{0}, };
-        rename3args             args;
-        int                     ret = RPCSVC_ACTOR_ERROR;
-
-        if (!req)
-                return ret;
-        nfs3_prep_rename3args (&args, &olddirfh, oldname, &newdirfh, newname);
-        if (xdr_to_rename3args (req->msg[0], &args) <= 0) {
-                gf_log (GF_NFS3, GF_LOG_ERROR, "Error decoding args");
-                rpcsvc_request_seterr (req, GARBAGE_ARGS);
-                goto rpcerr;
-        }
-
-        ret = nfs3_rename (req, &olddirfh, oldname, &newdirfh, newname);
-        if ((ret < 0) && (ret != RPCSVC_ACTOR_IGNORE)) {
-                gf_log (GF_NFS3, GF_LOG_ERROR, "RENAME procedure failed");
-                rpcsvc_request_seterr (req, SYSTEM_ERR);
-                ret = RPCSVC_ACTOR_ERROR;
-        }
+nfs3svc_rename(rpcsvc_request_t *req)
+{
+    char newname[NFS_PATH_MAX];
+    char oldname[NFS_PATH_MAX];
+    struct nfs3_fh olddirfh = {
+        {0},
+    };
+    struct nfs3_fh newdirfh = {
+        {0},
+    };
+    rename3args args;
+    int ret = RPCSVC_ACTOR_ERROR;
+
+    if (!req)
+        return ret;
+    nfs3_prep_rename3args(&args, &olddirfh, oldname, &newdirfh, newname);
+    if (xdr_to_rename3args(req->msg[0], &args) <= 0) {
+        gf_msg(GF_NFS3, GF_LOG_ERROR, 0, NFS_MSG_ARGS_DECODE_ERROR,
+               "Error decoding args");
+        rpcsvc_request_seterr(req, GARBAGE_ARGS);
+        goto rpcerr;
+    }
+
+    ret = nfs3_rename(req, &olddirfh, oldname, &newdirfh, newname);
+    if ((ret < 0) && (ret != RPCSVC_ACTOR_IGNORE)) {
+        gf_msg(GF_NFS3, GF_LOG_ERROR, 0, NFS_MSG_RENAME_FAIL,
+               "RENAME procedure failed");
+        rpcsvc_request_seterr(req, SYSTEM_ERR);
+        ret = RPCSVC_ACTOR_ERROR;
+    }
 
 rpcerr:
-        return ret;
+    return ret;
 }
 
-
 int
-nfs3_link_reply (rpcsvc_request_t *req, nfsstat3 stat, struct iatt *buf,
-                 struct iatt *preparent, struct iatt *postparent)
+nfs3_link_reply(rpcsvc_request_t *req, nfsstat3 stat, struct iatt *buf,
+                struct iatt *preparent, struct iatt *postparent)
 {
-        link3res        res = {0, };
-        uint64_t        deviceid = 0;
+    link3res res = {
+        0,
+    };
+    uint64_t deviceid = 0;
 
-        deviceid = nfs3_request_xlator_deviceid (req);
-        nfs3_fill_link3res (&res, stat, buf, preparent, postparent, deviceid);
-        nfs3svc_submit_reply (req, (void *)&res,
-                              (nfs3_serializer)xdr_serialize_link3res);
+    deviceid = nfs3_request_xlator_deviceid(req);
+    nfs3_fill_link3res(&res, stat, buf, preparent, postparent, deviceid);
+    nfs3svc_submit_reply(req, (void *)&res,
+                         (nfs3_serializer)xdr_serialize_link3res);
 
-        return 0;
+    return 0;
 }
 
-
 int32_t
-nfs3svc_link_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                  int32_t op_ret, int32_t op_errno, inode_t *inode,
-                  struct iatt *buf, struct iatt *preparent,
-                  struct iatt *postparent, dict_t *xdata)
-{
-        nfsstat3                stat = NFS3ERR_SERVERFAULT;
-        nfs3_call_state_t       *cs = NULL;
-
-        cs = frame->local;
-        if (op_ret == -1) {
-                gf_log (GF_NFS, GF_LOG_WARNING,
-                        "%x: link %s <- %s => -1 (%s)",
-                        rpcsvc_request_xid (cs->req), cs->oploc.path,
-                        cs->resolvedloc.path, strerror (op_errno));
-                stat = nfs3_cbk_errno_status (op_ret, op_errno);
-        } else
-                stat = NFS3_OK;
+nfs3svc_link_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                 int32_t op_ret, int32_t op_errno, inode_t *inode,
+                 struct iatt *buf, struct iatt *preparent,
+                 struct iatt *postparent, dict_t *xdata)
+{
+    nfsstat3 stat = NFS3ERR_SERVERFAULT;
+    nfs3_call_state_t *cs = NULL;
+
+    cs = frame->local;
+    if (op_ret == -1) {
+        stat = nfs3_cbk_errno_status(op_ret, op_errno);
+    } else
+        stat = NFS3_OK;
 
-        nfs3_log_common_res (rpcsvc_request_xid (cs->req), NFS3_LINK, stat,
-                             op_errno);
-        nfs3_link_reply (cs->req, stat, buf, preparent, postparent);
-        nfs3_call_state_wipe (cs);
+    nfs3_log_common_res(rpcsvc_request_xid(cs->req), NFS3_LINK, stat, op_errno,
+                        cs->resolvedloc.path);
+    nfs3_link_reply(cs->req, stat, buf, preparent, postparent);
+    nfs3_call_state_wipe(cs);
 
-        return 0;
+    return 0;
 }
 
-
 int
-nfs3_link_resume_lnk (void *carg)
+nfs3_link_resume_lnk(void *carg)
 {
-        nfsstat3                stat = NFS3ERR_SERVERFAULT;
-        int                     ret = -EFAULT;
-        nfs3_call_state_t       *cs = NULL;
-        nfs_user_t              nfu = {0, };
+    nfsstat3 stat = NFS3ERR_SERVERFAULT;
+    int ret = -EFAULT;
+    nfs3_call_state_t *cs = NULL;
+    nfs_user_t nfu = {
+        0,
+    };
 
-        if (!carg)
-                return ret;
+    if (!carg)
+        return ret;
 
-        cs = (nfs3_call_state_t *)carg;
-        nfs3_check_new_fh_resolve_status (cs, stat, nfs3err);
+    cs = (nfs3_call_state_t *)carg;
+    nfs3_check_new_fh_resolve_status(cs, stat, nfs3err);
 
-        nfs_request_user_init (&nfu, cs->req);
-        ret = nfs_link (cs->nfsx, cs->vol, &nfu, &cs->oploc, &cs->resolvedloc,
-                        nfs3svc_link_cbk, cs);
-        if (ret < 0)
-                stat = nfs3_errno_to_nfsstat3 (-ret);
+    nfs_request_user_init(&nfu, cs->req);
+    ret = nfs_link(cs->nfsx, cs->vol, &nfu, &cs->oploc, &cs->resolvedloc,
+                   nfs3svc_link_cbk, cs);
+    if (ret < 0)
+        stat = nfs3_errno_to_nfsstat3(-ret);
 
 nfs3err:
-        if (ret < 0) {
-                nfs3_log_common_res (rpcsvc_request_xid (cs->req), NFS3_LINK,
-                                     stat, -ret);
-                nfs3_link_reply (cs->req, stat, NULL, NULL, NULL);
-                nfs3_call_state_wipe (cs);
-        }
-        return ret;
+    if (ret < 0) {
+        nfs3_log_common_res(rpcsvc_request_xid(cs->req), NFS3_LINK, stat, -ret,
+                            cs->resolvedloc.path);
+        nfs3_link_reply(cs->req, stat, NULL, NULL, NULL);
+        nfs3_call_state_wipe(cs);
+    }
+    return ret;
 }
 
-
 int
-nfs3_link_resume_tgt (void *carg)
+nfs3_link_resume_tgt(void *carg)
 {
-        nfsstat3                stat = NFS3ERR_SERVERFAULT;
-        int                     ret = -EFAULT;
-        nfs3_call_state_t       *cs = NULL;
+    nfsstat3 stat = NFS3ERR_SERVERFAULT;
+    int ret = -EFAULT;
+    nfs3_call_state_t *cs = NULL;
 
-        if (!carg)
-                return ret;
+    if (!carg)
+        return ret;
 
-        cs = (nfs3_call_state_t *)carg;
-        nfs3_check_fh_resolve_status (cs, stat, nfs3err);
-        nfs_loc_copy (&cs->oploc, &cs->resolvedloc);
-        nfs_loc_wipe (&cs->resolvedloc);
+    cs = (nfs3_call_state_t *)carg;
+    nfs3_check_fh_resolve_status(cs, stat, nfs3err);
+    nfs_loc_copy(&cs->oploc, &cs->resolvedloc);
+    nfs_loc_wipe(&cs->resolvedloc);
 
-        ret = nfs3_fh_resolve_and_resume (cs, &cs->fh, cs->pathname,
-                                          nfs3_link_resume_lnk);
-        if (ret < 0)
-                stat = nfs3_errno_to_nfsstat3 (-ret);
+    ret = nfs3_fh_resolve_and_resume(cs, &cs->fh, cs->pathname,
+                                     nfs3_link_resume_lnk);
+    if (ret < 0)
+        stat = nfs3_errno_to_nfsstat3(-ret);
 
 nfs3err:
-        if (ret < 0) {
-                nfs3_log_common_res (rpcsvc_request_xid (cs->req), NFS3_LINK,
-                                     stat, -ret);
-                nfs3_link_reply (cs->req, stat, NULL, NULL, NULL);
-                nfs3_call_state_wipe (cs);
-        }
+    if (ret < 0) {
+        nfs3_log_common_res(rpcsvc_request_xid(cs->req), NFS3_LINK, stat, -ret,
+                            cs->resolvedloc.path);
+        nfs3_link_reply(cs->req, stat, NULL, NULL, NULL);
+        nfs3_call_state_wipe(cs);
+    }
 
-        return ret;
+    return ret;
 }
 
-
 int
-nfs3_link (rpcsvc_request_t *req, struct nfs3_fh *targetfh,
-           struct nfs3_fh *dirfh, char *newname)
-{
-        xlator_t                *vol = NULL;
-        nfsstat3                stat = NFS3ERR_SERVERFAULT;
-        int                     ret = -EFAULT;
-        struct nfs3_state       *nfs3 = NULL;
-        nfs3_call_state_t       *cs = NULL;
-
-        if ((!req) || (!targetfh) || (!dirfh) || (!newname)) {
-                gf_log (GF_NFS3, GF_LOG_ERROR, "Bad arguments");
-                return -1;
-        }
-
-        nfs3_validate_gluster_fh (dirfh, stat, nfs3err);
-        nfs3_validate_gluster_fh (targetfh, stat, nfs3err);
-        nfs3_validate_nfs3_state (req, nfs3, stat, nfs3err, ret);
-        nfs3_validate_strlen_or_goto(newname, NFS_NAME_MAX, nfs3err, stat, ret);
-        nfs3_map_fh_to_volume (nfs3, dirfh, req, vol, stat, nfs3err);
-        nfs3_volume_started_check (nfs3, vol, ret, out);
-        nfs3_check_rw_volaccess (nfs3, dirfh->exportid, stat, nfs3err);
-        nfs3_handle_call_state_init (nfs3, cs, req, vol, stat, nfs3err);
-
-        cs->fh = *dirfh;
-        cs->pathname = gf_strdup (newname);
-        if (!cs->pathname) {
-                stat = NFS3ERR_SERVERFAULT;
-                ret = -1;
-                goto nfs3err;
-        }
-
-        ret = nfs3_fh_resolve_and_resume (cs, targetfh, NULL,
-                                          nfs3_link_resume_tgt);
-        if (ret < 0)
-                stat = nfs3_errno_to_nfsstat3 (-ret);
+nfs3_link(rpcsvc_request_t *req, struct nfs3_fh *targetfh,
+          struct nfs3_fh *dirfh, char *newname)
+{
+    xlator_t *vol = NULL;
+    nfsstat3 stat = NFS3ERR_SERVERFAULT;
+    int ret = -EFAULT;
+    struct nfs3_state *nfs3 = NULL;
+    nfs3_call_state_t *cs = NULL;
+
+    if ((!req) || (!targetfh) || (!dirfh) || (!newname)) {
+        gf_msg(GF_NFS3, GF_LOG_ERROR, EINVAL, NFS_MSG_INVALID_ENTRY,
+               "Bad arguments");
+        return -1;
+    }
+
+    nfs3_validate_gluster_fh(dirfh, stat, nfs3err);
+    nfs3_validate_gluster_fh(targetfh, stat, nfs3err);
+    nfs3_validate_nfs3_state(req, nfs3, stat, nfs3err, ret);
+    nfs3_validate_strlen_or_goto(newname, NFS_NAME_MAX, nfs3err, stat, ret);
+    nfs3_map_fh_to_volume(nfs3, dirfh, req, vol, stat, nfs3err);
+    nfs3_volume_started_check(nfs3, vol, ret, out);
+    nfs3_check_rw_volaccess(nfs3, dirfh->exportid, stat, nfs3err);
+    nfs3_handle_call_state_init(nfs3, cs, req, vol, stat, nfs3err);
+
+    cs->fh = *dirfh;
+    cs->pathname = gf_strdup(newname);
+    if (!cs->pathname) {
+        stat = NFS3ERR_SERVERFAULT;
+        ret = -1;
+        goto nfs3err;
+    }
+
+    ret = nfs3_fh_resolve_and_resume(cs, targetfh, NULL, nfs3_link_resume_tgt);
+    if (ret < 0)
+        stat = nfs3_errno_to_nfsstat3(-ret);
 
 nfs3err:
-        if (ret < 0) {
-                nfs3_log_common_res (rpcsvc_request_xid (req), NFS3_LINK, stat,
-                                     -ret);
-                nfs3_link_reply (req, stat, NULL, NULL, NULL);
-                nfs3_call_state_wipe (cs);
-                /* Ret must be 0 after this so that the caller does not
-                 * also send an RPC reply.
-                 */
-                ret = 0;
-        }
+    if (ret < 0) {
+        nfs3_log_common_res(rpcsvc_request_xid(req), NFS3_LINK, stat, -ret,
+                            cs ? cs->pathname : NULL);
+        nfs3_link_reply(req, stat, NULL, NULL, NULL);
+        nfs3_call_state_wipe(cs);
+        /* Ret must be 0 after this so that the caller does not
+         * also send an RPC reply.
+         */
+        ret = 0;
+    }
 out:
-        return ret;
+    return ret;
 }
 
 int
-nfs3svc_link (rpcsvc_request_t *req)
-{
-        char                    newpath[NFS_PATH_MAX];
-        struct nfs3_fh          dirfh = {{0}, };
-        struct nfs3_fh          targetfh = {{0}, };
-        link3args               args;
-        int                     ret = RPCSVC_ACTOR_ERROR;
-
-        if (!req)
-                return ret;
-        nfs3_prep_link3args (&args, &targetfh, &dirfh, newpath);
-        if (xdr_to_link3args (req->msg[0], &args) <= 0) {
-                gf_log (GF_NFS3, GF_LOG_ERROR, "Error decoding args");
-                rpcsvc_request_seterr (req, GARBAGE_ARGS);
-                goto rpcerr;
-        }
-
-        ret = nfs3_link (req, &targetfh, &dirfh, newpath);
-        if ((ret < 0) && (ret != RPCSVC_ACTOR_IGNORE)) {
-                gf_log (GF_NFS3, GF_LOG_ERROR, "LINK procedure failed");
-                rpcsvc_request_seterr (req, SYSTEM_ERR);
-                ret = RPCSVC_ACTOR_ERROR;
-        }
+nfs3svc_link(rpcsvc_request_t *req)
+{
+    char newpath[NFS_PATH_MAX];
+    struct nfs3_fh dirfh = {
+        {0},
+    };
+    struct nfs3_fh targetfh = {
+        {0},
+    };
+    link3args args;
+    int ret = RPCSVC_ACTOR_ERROR;
+
+    if (!req)
+        return ret;
+    nfs3_prep_link3args(&args, &targetfh, &dirfh, newpath);
+    if (xdr_to_link3args(req->msg[0], &args) <= 0) {
+        gf_msg(GF_NFS3, GF_LOG_ERROR, 0, NFS_MSG_ARGS_DECODE_ERROR,
+               "Error decoding args");
+        rpcsvc_request_seterr(req, GARBAGE_ARGS);
+        goto rpcerr;
+    }
+
+    ret = nfs3_link(req, &targetfh, &dirfh, newpath);
+    if ((ret < 0) && (ret != RPCSVC_ACTOR_IGNORE)) {
+        gf_msg(GF_NFS3, GF_LOG_ERROR, EXDEV, NFS_MSG_LINK_FAIL,
+               "LINK procedure failed");
+        rpcsvc_request_seterr(req, SYSTEM_ERR);
+        ret = RPCSVC_ACTOR_ERROR;
+    }
 
 rpcerr:
-        return ret;
+    return ret;
 }
 
-
 int
-nfs3_readdirp_reply (rpcsvc_request_t *req, nfsstat3 stat,struct nfs3_fh *dirfh,
-                     uint64_t cverf, struct iatt *dirstat, gf_dirent_t *entries,
-                     count3 dircount, count3 maxcount, int is_eof)
+nfs3_readdirp_reply(rpcsvc_request_t *req, nfsstat3 stat, struct nfs3_fh *dirfh,
+                    uint64_t cverf, struct iatt *dirstat, gf_dirent_t *entries,
+                    count3 dircount, count3 maxcount, int is_eof)
 {
-        readdirp3res    res = {0, };
-        uint64_t        deviceid = 0;
+    readdirp3res res = {
+        0,
+    };
+    uint64_t deviceid = 0;
 
-        deviceid = nfs3_request_xlator_deviceid (req);
-        nfs3_fill_readdirp3res (&res, stat, dirfh, cverf, dirstat, entries,
-                                dircount, maxcount, is_eof, deviceid);
-        nfs3svc_submit_reply (req, (void *)&res,
-                              (nfs3_serializer) xdr_serialize_readdirp3res);
-        nfs3_free_readdirp3res (&res);
+    deviceid = nfs3_request_xlator_deviceid(req);
+    nfs3_fill_readdirp3res(&res, stat, dirfh, cverf, dirstat, entries, dircount,
+                           maxcount, is_eof, deviceid);
+    nfs3svc_submit_reply(req, (void *)&res,
+                         (nfs3_serializer)xdr_serialize_readdirp3res);
+    nfs3_free_readdirp3res(&res);
 
-        return 0;
+    return 0;
 }
 
-
 int
-nfs3_readdir_reply (rpcsvc_request_t *req, nfsstat3 stat, struct nfs3_fh *dirfh,
-                    uint64_t cverf, struct iatt *dirstat, gf_dirent_t *entries,
-                    count3 count, int is_eof)
+nfs3_readdir_reply(rpcsvc_request_t *req, nfsstat3 stat, struct nfs3_fh *dirfh,
+                   uint64_t cverf, struct iatt *dirstat, gf_dirent_t *entries,
+                   count3 count, int is_eof)
 {
-        readdir3res     res = {0, };
-        uint64_t        deviceid = 0;
+    readdir3res res = {
+        0,
+    };
+    uint64_t deviceid = 0;
 
-        deviceid = nfs3_request_xlator_deviceid (req);
-        nfs3_fill_readdir3res (&res, stat, dirfh, cverf, dirstat, entries, count
-                               , is_eof, deviceid);
-        nfs3svc_submit_reply (req, (void *)&res,
-                              (nfs3_serializer) xdr_serialize_readdir3res);
-        nfs3_free_readdir3res (&res);
+    deviceid = nfs3_request_xlator_deviceid(req);
+    nfs3_fill_readdir3res(&res, stat, dirfh, cverf, dirstat, entries, count,
+                          is_eof, deviceid);
+    nfs3svc_submit_reply(req, (void *)&res,
+                         (nfs3_serializer)xdr_serialize_readdir3res);
+    nfs3_free_readdir3res(&res);
 
-        return 0;
+    return 0;
 }
 
-
 int32_t
-nfs3svc_readdir_fstat_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                           int32_t op_ret, int32_t op_errno, struct iatt *buf,
-                           dict_t *xdata)
-{
-        nfsstat3                stat = NFS3ERR_SERVERFAULT;
-        int                     is_eof = 0;
-        nfs3_call_state_t       *cs = NULL;
-
-        cs = frame->local;
-        if (op_ret == -1) {
-                gf_log (GF_NFS, GF_LOG_WARNING,
-                        "%x: %s => -1 (%s)", rpcsvc_request_xid (cs->req),
-                        cs->resolvedloc.path, strerror (op_errno));
-                stat = nfs3_cbk_errno_status (op_ret, op_errno);
-                goto nfs3err;
-        }
+nfs3svc_readdir_fstat_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                          int32_t op_ret, int32_t op_errno, struct iatt *buf,
+                          dict_t *xdata)
+{
+    nfsstat3 stat = NFS3ERR_SERVERFAULT;
+    int is_eof = 0;
+    nfs3_call_state_t *cs = NULL;
 
-        /* Check whether we encountered a end of directory stream while
-         * readdir'ing.
-         */
-        if (cs->operrno == ENOENT) {
-                gf_log (GF_NFS3, GF_LOG_TRACE, "Reached end-of-directory");
-                is_eof = 1;
-        }
+    cs = frame->local;
+    if (op_ret == -1) {
+        stat = nfs3_cbk_errno_status(op_ret, op_errno);
+        goto nfs3err;
+    }
 
-        stat = NFS3_OK;
+    /* Check whether we encountered a end of directory stream while
+     * readdir'ing.
+     */
+    if (cs->operrno == ENOENT) {
+        gf_msg_trace(GF_NFS3, 0, "Reached end-of-directory");
+        is_eof = 1;
+    }
 
-        /* do inode linking here */
-        gf_link_inodes_from_dirent (this, cs->fd->inode, &cs->entries);
+    stat = NFS3_OK;
 
-nfs3err:
-        if (cs->maxcount == 0) {
-                nfs3_log_readdir_res (rpcsvc_request_xid (cs->req), stat,
-                                      op_errno, (uintptr_t)cs->fd,
-                                      cs->dircount, is_eof);
-                nfs3_readdir_reply (cs->req, stat, &cs->parent,
-                                    (uintptr_t)cs->fd, buf, &cs->entries,
-                                    cs->dircount, is_eof);
-        } else {
-                nfs3_log_readdirp_res (rpcsvc_request_xid (cs->req), stat,
-                                       op_errno, (uintptr_t)cs->fd,
-                                       cs->dircount, cs->maxcount, is_eof);
-                nfs3_readdirp_reply (cs->req, stat, &cs->parent,
-                                     (uintptr_t)cs->fd, buf,
-                                     &cs->entries, cs->dircount,
-                                     cs->maxcount, is_eof);
-        }
-
-        if (is_eof) {
-                /* do nothing */
-        }
+    /* do inode linking here */
+    gf_link_inodes_from_dirent(this, cs->fd->inode, &cs->entries);
 
-        nfs3_call_state_wipe (cs);
-        return 0;
+nfs3err:
+    if (cs->maxcount == 0) {
+        nfs3_log_readdir_res(rpcsvc_request_xid(cs->req), stat, op_errno,
+                             (uintptr_t)cs->fd, cs->dircount, is_eof,
+                             cs->resolvedloc.path);
+        nfs3_readdir_reply(cs->req, stat, &cs->parent, (uintptr_t)cs->fd, buf,
+                           &cs->entries, cs->dircount, is_eof);
+    } else {
+        nfs3_log_readdirp_res(rpcsvc_request_xid(cs->req), stat, op_errno,
+                              (uintptr_t)cs->fd, cs->dircount, cs->maxcount,
+                              is_eof, cs->resolvedloc.path);
+        nfs3_readdirp_reply(cs->req, stat, &cs->parent, (uintptr_t)cs->fd, buf,
+                            &cs->entries, cs->dircount, cs->maxcount, is_eof);
+    }
+
+    if (is_eof) {
+        /* do nothing */
+    }
+
+    nfs3_call_state_wipe(cs);
+    return 0;
 }
 
-
 int32_t
-nfs3svc_readdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                     int32_t op_ret, int32_t op_errno, gf_dirent_t *entries,
-                     dict_t *xdata)
+nfs3svc_readdir_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                    int32_t op_ret, int32_t op_errno, gf_dirent_t *entries,
+                    dict_t *xdata)
 {
-        nfsstat3                stat = NFS3ERR_SERVERFAULT;
-        int                     ret = -EFAULT;
-        nfs_user_t              nfu = {0, };
-        nfs3_call_state_t       *cs = NULL;
-
-        cs = frame->local;
-        if (op_ret == -1) {
-                gf_log (GF_NFS, GF_LOG_WARNING,
-                        "%x: %s => -1 (%s)", rpcsvc_request_xid (cs->req),
-                        cs->resolvedloc.path, strerror (op_errno));
-                stat = nfs3_cbk_errno_status (op_ret, op_errno);
-                goto err;
-        }
-
-        cs->operrno = op_errno;
-        list_splice_init (&entries->list, &cs->entries.list);
-        nfs_request_user_init (&nfu, cs->req);
-        ret = nfs_fstat (cs->nfsx, cs->vol, &nfu, cs->fd,
-                         nfs3svc_readdir_fstat_cbk, cs);
-        if (ret < 0) {
-                op_ret = -1;
-                stat = nfs3_errno_to_nfsstat3 (-ret);
-                op_errno = -ret;
-        }
+    nfsstat3 stat = NFS3ERR_SERVERFAULT;
+    int ret = -EFAULT;
+    nfs_user_t nfu = {
+        0,
+    };
+    nfs3_call_state_t *cs = NULL;
+
+    cs = frame->local;
+    if (op_ret == -1) {
+        stat = nfs3_cbk_errno_status(op_ret, op_errno);
+        goto err;
+    }
+
+    cs->operrno = op_errno;
+    list_splice_init(&entries->list, &cs->entries.list);
+    nfs_request_user_init(&nfu, cs->req);
+    ret = nfs_fstat(cs->nfsx, cs->vol, &nfu, cs->fd, nfs3svc_readdir_fstat_cbk,
+                    cs);
+    if (ret < 0) {
+        op_ret = -1;
+        stat = nfs3_errno_to_nfsstat3(-ret);
+        op_errno = -ret;
+    }
 
 err:
-        if (op_ret >= 0)
-                goto ret;
-
-        if (cs->maxcount == 0) {
-                nfs3_log_common_res (rpcsvc_request_xid (cs->req),
-                                     NFS3_READDIR, stat, op_errno);
-                nfs3_readdir_reply (cs->req, stat, NULL, 0, NULL, NULL, 0, 0);
-        } else {
-                nfs3_log_common_res (rpcsvc_request_xid (cs->req),
-                                     NFS3_READDIRP, stat, op_errno);
-                nfs3_readdirp_reply (cs->req, stat, NULL, 0, NULL, NULL,
-                                     0, 0, 0);
-        }
-
-        /* For directories, we force a purge from the fd cache on close
-         * so that next time the dir is read, we'll get any changed directory
-         * entries.
-         */
-        nfs3_call_state_wipe (cs);
+    if (op_ret >= 0)
+        goto ret;
+
+    if (cs->maxcount == 0) {
+        nfs3_log_common_res(rpcsvc_request_xid(cs->req), NFS3_READDIR, stat,
+                            op_errno, cs->resolvedloc.path);
+        nfs3_readdir_reply(cs->req, stat, NULL, 0, NULL, NULL, 0, 0);
+    } else {
+        nfs3_log_common_res(rpcsvc_request_xid(cs->req), NFS3_READDIRP, stat,
+                            op_errno, cs->resolvedloc.path);
+        nfs3_readdirp_reply(cs->req, stat, NULL, 0, NULL, NULL, 0, 0, 0);
+    }
+
+    /* For directories, we force a purge from the fd cache on close
+     * so that next time the dir is read, we'll get any changed directory
+     * entries.
+     */
+    nfs3_call_state_wipe(cs);
 ret:
-        return 0;
+    return 0;
 }
 
 int
-nfs3_readdir_process (nfs3_call_state_t *cs)
+nfs3_readdir_process(nfs3_call_state_t *cs)
 {
-        int                     ret = -EFAULT;
-        nfs_user_t              nfu = {0, };
+    int ret = -EFAULT;
+    nfs_user_t nfu = {
+        0,
+    };
 
-        if (!cs)
-                return ret;
-
-        nfs_request_user_init (&nfu, cs->req);
-        ret = nfs_readdirp (cs->nfsx, cs->vol, &nfu, cs->fd, cs->dircount,
-                            cs->cookie, nfs3svc_readdir_cbk, cs);
+    if (!cs)
         return ret;
-}
 
+    nfs_request_user_init(&nfu, cs->req);
+    ret = nfs_readdirp(cs->nfsx, cs->vol, &nfu, cs->fd, cs->dircount,
+                       cs->cookie, nfs3svc_readdir_cbk, cs);
+    return ret;
+}
 
 int
-nfs3_readdir_read_resume (void *carg)
+nfs3_readdir_read_resume(void *carg)
 {
-        nfsstat3                stat = NFS3ERR_SERVERFAULT;
-        int                     ret = -EFAULT;
-        nfs3_call_state_t       *cs = NULL;
-        struct nfs3_state       *nfs3 = NULL;
-
-        if (!carg)
-                return ret;
+    nfsstat3 stat = NFS3ERR_SERVERFAULT;
+    int ret = -EFAULT;
+    nfs3_call_state_t *cs = NULL;
+    struct nfs3_state *nfs3 = NULL;
 
-        cs = (nfs3_call_state_t *)carg;
-        nfs3_check_fh_resolve_status (cs, stat, nfs3err);
-        nfs3 = rpcsvc_request_program_private (cs->req);
-        ret = nfs3_verify_dircookie (nfs3, cs->fd, cs->cookie, cs->cookieverf,
-                                     &stat);
-        if (ret < 0)    /* Stat already set by verifier function above. */
-                goto nfs3err;
+    if (!carg)
+        return ret;
 
-        ret = nfs3_readdir_process (cs);
-        if (ret < 0)
-                stat = nfs3_errno_to_nfsstat3 (-ret);
+    cs = (nfs3_call_state_t *)carg;
+    nfs3_check_fh_resolve_status(cs, stat, nfs3err);
+    nfs3 = rpcsvc_request_program_private(cs->req);
+    ret = nfs3_verify_dircookie(nfs3, cs->fd, cs->cookie, cs->cookieverf,
+                                &stat);
+    if (ret < 0) /* Stat already set by verifier function above. */
+        goto nfs3err;
+
+    ret = nfs3_readdir_process(cs);
+    if (ret < 0)
+        stat = nfs3_errno_to_nfsstat3(-ret);
 nfs3err:
-        if (ret < 0) {
-                if (cs->maxcount == 0) {
-                        nfs3_log_common_res (rpcsvc_request_xid (cs->req),
-                                             NFS3_READDIR, stat, -ret);
-                        nfs3_readdir_reply (cs->req, stat, NULL, 0, NULL, NULL,
-                                            0, 0);
-                } else {
-                        nfs3_log_common_res (rpcsvc_request_xid (cs->req),
-                                             NFS3_READDIRP, stat, -ret);
-                        nfs3_readdirp_reply (cs->req, stat, NULL, 0, NULL, NULL,
-                                             0, 0, 0);
-                }
-                nfs3_call_state_wipe (cs);
+    if (ret < 0) {
+        if (cs->maxcount == 0) {
+            nfs3_log_common_res(rpcsvc_request_xid(cs->req), NFS3_READDIR, stat,
+                                -ret, cs->resolvedloc.path);
+            nfs3_readdir_reply(cs->req, stat, NULL, 0, NULL, NULL, 0, 0);
+        } else {
+            nfs3_log_common_res(rpcsvc_request_xid(cs->req), NFS3_READDIRP,
+                                stat, -ret, cs->resolvedloc.path);
+            nfs3_readdirp_reply(cs->req, stat, NULL, 0, NULL, NULL, 0, 0, 0);
         }
+        nfs3_call_state_wipe(cs);
+    }
 
-        return 0;
+    return 0;
 }
 
-
 int32_t
-nfs3svc_readdir_opendir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                             int32_t op_ret, int32_t op_errno, fd_t *fd,
-                             dict_t *xdata)
+nfs3svc_readdir_opendir_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                            int32_t op_ret, int32_t op_errno, fd_t *fd,
+                            dict_t *xdata)
 {
-        /*
-         * We don't really need this, it's just an artifact of forcing the
-         * opendir to happen.
-         */
-        if (fd) {
-                fd_unref(fd);
-        }
+    /*
+     * We don't really need this, it's just an artifact of forcing the
+     * opendir to happen.
+     */
+    if (fd) {
+        fd_unref(fd);
+    }
 
-        return 0;
+    return 0;
 }
 
-
 int
-nfs3_readdir_open_resume (void *carg)
-{
-        nfsstat3                stat = NFS3ERR_SERVERFAULT;
-        int                     ret = -EFAULT;
-        nfs3_call_state_t       *cs = NULL;
-        nfs_user_t               nfu = {0, };
-
-        if (!carg)
-                return ret;
-
-        cs = (nfs3_call_state_t *)carg;
-        nfs3_check_fh_resolve_status (cs, stat, nfs3err);
-        cs->fd = fd_anonymous (cs->resolvedloc.inode);
-        if (!cs->fd) {
-                gf_log (GF_NFS3, GF_LOG_ERROR, "Faile to create anonymous fd");
-                goto nfs3err;
-        }
+nfs3_readdir_open_resume(void *carg)
+{
+    nfsstat3 stat = NFS3ERR_SERVERFAULT;
+    int ret = -EFAULT;
+    nfs3_call_state_t *cs = NULL;
+    nfs_user_t nfu = {
+        0,
+    };
 
-        /*
-         * NFS client will usually send us a readdirp without an opendir,
-         * which would cause us to skip our usual self-heal checks which occur
-         * in opendir for native protocol. To make sure those checks do happen,
-         * our most reliable option is to do our own opendir for any readdirp
-         * at the beginning of the directory.
-         */
-        if (cs->cookie == 0) {
-                nfs_request_user_init (&nfu, cs->req);
-                ret = nfs_opendir (cs->nfsx, cs->vol, &nfu, &cs->resolvedloc,
-                                   nfs3svc_readdir_opendir_cbk, cs);
-                if (ret < 0) {
-                        gf_log (GF_NFS3, GF_LOG_ERROR, "auto-opendir failed");
-                }
+    if (!carg)
+        return ret;
+
+    cs = (nfs3_call_state_t *)carg;
+    nfs3_check_fh_resolve_status(cs, stat, nfs3err);
+    cs->fd = fd_anonymous(cs->resolvedloc.inode);
+    if (!cs->fd) {
+        gf_msg(GF_NFS3, GF_LOG_ERROR, 0, NFS_MSG_ANONYMOUS_FD_FAIL,
+               "Fail to create anonymous fd");
+        goto nfs3err;
+    }
+
+    /*
+     * NFS client will usually send us a readdirp without an opendir,
+     * which would cause us to skip our usual self-heal checks which occur
+     * in opendir for native protocol. To make sure those checks do happen,
+     * our most reliable option is to do our own opendir for any readdirp
+     * at the beginning of the directory.
+     */
+    if (cs->cookie == 0) {
+        nfs_request_user_init(&nfu, cs->req);
+        ret = nfs_opendir(cs->nfsx, cs->vol, &nfu, &cs->resolvedloc,
+                          nfs3svc_readdir_opendir_cbk, cs);
+        if (ret < 0) {
+            gf_msg(GF_NFS3, GF_LOG_ERROR, -ret, NFS_MSG_DIR_OP_FAIL,
+                   "auto-opendir failed");
         }
+    }
 
-        ret = nfs3_readdir_read_resume (cs);
-        if (ret < 0)
-                stat = nfs3_errno_to_nfsstat3 (-ret);
+    ret = nfs3_readdir_read_resume(cs);
+    if (ret < 0)
+        stat = nfs3_errno_to_nfsstat3(-ret);
 
 nfs3err:
-        if (ret < 0) {
-                if (cs->maxcount == 0) {
-                        nfs3_log_common_res (rpcsvc_request_xid (cs->req),
-                                             NFS3_READDIR, stat, -ret);
-                        nfs3_readdir_reply (cs->req, stat, NULL, 0, NULL, NULL,
-                                            0, 0);
-                } else {
-                        nfs3_log_common_res (rpcsvc_request_xid (cs->req),
-                                             NFS3_READDIRP, stat, -ret);
-                        nfs3_readdirp_reply (cs->req, stat, NULL, 0, NULL, NULL,
-                                             0, 0, 0);
-                }
-                nfs3_call_state_wipe (cs);
+    if (ret < 0) {
+        if (cs->maxcount == 0) {
+            nfs3_log_common_res(rpcsvc_request_xid(cs->req), NFS3_READDIR, stat,
+                                -ret, cs->resolvedloc.path);
+            nfs3_readdir_reply(cs->req, stat, NULL, 0, NULL, NULL, 0, 0);
+        } else {
+            nfs3_log_common_res(rpcsvc_request_xid(cs->req), NFS3_READDIRP,
+                                stat, -ret, cs->resolvedloc.path);
+            nfs3_readdirp_reply(cs->req, stat, NULL, 0, NULL, NULL, 0, 0, 0);
         }
+        nfs3_call_state_wipe(cs);
+    }
 
-        return ret;
+    return ret;
 }
 
-
-
 int
-nfs3_readdir (rpcsvc_request_t *req, struct nfs3_fh *fh, cookie3 cookie,
-              uint64_t cverf, count3 dircount, count3 maxcount)
-{
-        xlator_t                *vol = NULL;
-        nfsstat3                stat = NFS3ERR_SERVERFAULT;
-        int                     ret = -EFAULT;
-        struct nfs3_state       *nfs3 = NULL;
-        nfs3_call_state_t       *cs = NULL;
-
-        if ((!req) || (!fh)) {
-                gf_log (GF_NFS3, GF_LOG_ERROR, "Bad arguments");
-                return -1;
-        }
-
-        nfs3_log_readdir_call (rpcsvc_request_xid (req), fh, dircount,
-                               maxcount);
-        nfs3_validate_gluster_fh (fh, stat, nfs3err);
-        nfs3_validate_nfs3_state (req, nfs3, stat, nfs3err, ret);
-        nfs3_map_fh_to_volume (nfs3, fh, req, vol, stat, nfs3err);
-        nfs3_volume_started_check (nfs3, vol, ret, out);
-        nfs3_handle_call_state_init (nfs3, cs, req, vol, stat, nfs3err);
-
-        cs->cookieverf = cverf;
-        cs->dircount = dircount;
-        cs->maxcount = maxcount;
-        cs->cookie = cookie;
-        cs->parent = *fh;
-        ret = nfs3_fh_resolve_and_resume (cs, fh, NULL,
-                                          nfs3_readdir_open_resume);
-        if (ret < 0)
-                stat = nfs3_errno_to_nfsstat3 (-ret);
+nfs3_readdir(rpcsvc_request_t *req, struct nfs3_fh *fh, cookie3 cookie,
+             uint64_t cverf, count3 dircount, count3 maxcount)
+{
+    xlator_t *vol = NULL;
+    nfsstat3 stat = NFS3ERR_SERVERFAULT;
+    int ret = -EFAULT;
+    struct nfs3_state *nfs3 = NULL;
+    nfs3_call_state_t *cs = NULL;
+    struct nfs_state *nfs = NULL;
+    gf_boolean_t is_readdirp = !!maxcount;
+
+    if ((!req) || (!fh)) {
+        gf_msg(GF_NFS3, GF_LOG_ERROR, EINVAL, NFS_MSG_INVALID_ENTRY,
+               "Bad arguments");
+        return -1;
+    }
+
+    nfs3_log_readdir_call(rpcsvc_request_xid(req), fh, dircount, maxcount);
+    nfs3_validate_gluster_fh(fh, stat, nfs3err);
+    nfs3_validate_nfs3_state(req, nfs3, stat, nfs3err, ret);
+    nfs3_map_fh_to_volume(nfs3, fh, req, vol, stat, nfs3err);
+    nfs3_volume_started_check(nfs3, vol, ret, out);
+    nfs3_handle_call_state_init(nfs3, cs, req, vol, stat, nfs3err);
+    nfs = nfs_state(nfs3->nfsx);
+
+    if (is_readdirp && !nfs->rdirplus) {
+        ret = -ENOTSUP;
+        stat = nfs3_errno_to_nfsstat3(-ret);
+        goto nfs3err;
+    }
+
+    cs->cookieverf = cverf;
+    cs->dircount = dircount;
+    cs->maxcount = maxcount;
+    cs->cookie = cookie;
+    cs->parent = *fh;
+    ret = nfs3_fh_resolve_and_resume(cs, fh, NULL, nfs3_readdir_open_resume);
+    if (ret < 0)
+        stat = nfs3_errno_to_nfsstat3(-ret);
 
 nfs3err:
-        if (ret < 0) {
-                if (maxcount == 0) {
-                        nfs3_log_common_res (rpcsvc_request_xid (req),
-                                             NFS3_READDIR, stat, -ret);
-                        nfs3_readdir_reply (req, stat, NULL, 0, NULL, NULL, 0,
-                                            0);
-                } else {
-                        nfs3_log_common_res (rpcsvc_request_xid (req),
-                                             NFS3_READDIRP, stat, -ret);
-                        nfs3_readdirp_reply (req, stat, NULL, 0, NULL, NULL, 0,
-                                             0, 0);
-                }
-                /* Ret must be NULL after this so that the caller does not
-                 * also send an RPC reply.
-                 */
-                ret = 0;
-                nfs3_call_state_wipe (cs);
+    if (ret < 0) {
+        if (!is_readdirp) {
+            nfs3_log_common_res(rpcsvc_request_xid(req), NFS3_READDIR, stat,
+                                -ret, cs ? cs->resolvedloc.path : NULL);
+            nfs3_readdir_reply(req, stat, NULL, 0, NULL, NULL, 0, 0);
+        } else {
+            nfs3_log_common_res(rpcsvc_request_xid(req), NFS3_READDIRP, stat,
+                                -ret, cs ? cs->resolvedloc.path : NULL);
+            nfs3_readdirp_reply(req, stat, NULL, 0, NULL, NULL, 0, 0, 0);
         }
+        /* Ret must be NULL after this so that the caller does not
+         * also send an RPC reply.
+         */
+        ret = 0;
+        nfs3_call_state_wipe(cs);
+    }
 out:
-        return ret;
+    return ret;
 }
 
-
 int
-nfs3svc_readdir (rpcsvc_request_t *req)
-{
-        readdir3args    ra;
-        struct nfs3_fh  fh = {{0},};
-        int             ret = RPCSVC_ACTOR_ERROR;
-        uint64_t        verf = 0;
-        uint64_t       *cval;
-
-        if (!req)
-                return ret;
-        nfs3_prep_readdir3args (&ra, &fh);
-        if (xdr_to_readdir3args (req->msg[0], &ra) <= 0) {
-                gf_log (GF_NFS3, GF_LOG_ERROR, "Error decoding args");
-                rpcsvc_request_seterr (req, GARBAGE_ARGS);
-                goto rpcerr;
-        }
-        cval = (uint64_t *) ra.cookieverf;
-        verf =  *cval;
-
-        ret = nfs3_readdir (req, &fh, ra.cookie, verf, ra.count, 0);
-        if ((ret < 0) && (ret != RPCSVC_ACTOR_IGNORE)) {
-                gf_log (GF_NFS3, GF_LOG_ERROR, "READDIR procedure failed");
-                rpcsvc_request_seterr (req, SYSTEM_ERR);
-                ret = RPCSVC_ACTOR_ERROR;
-        }
+nfs3svc_readdir(rpcsvc_request_t *req)
+{
+    readdir3args ra;
+    struct nfs3_fh fh = {
+        {0},
+    };
+    int ret = RPCSVC_ACTOR_ERROR;
+    uint64_t verf = 0;
+    uint64_t *cval;
 
-rpcerr:
+    if (!req)
         return ret;
-}
+    nfs3_prep_readdir3args(&ra, &fh);
+    if (xdr_to_readdir3args(req->msg[0], &ra) <= 0) {
+        gf_msg(GF_NFS3, GF_LOG_ERROR, 0, NFS_MSG_ARGS_DECODE_ERROR,
+               "Error decoding args");
+        rpcsvc_request_seterr(req, GARBAGE_ARGS);
+        goto rpcerr;
+    }
+    cval = (uint64_t *)ra.cookieverf;
+    verf = *cval;
+
+    ret = nfs3_readdir(req, &fh, ra.cookie, verf, ra.count, 0);
+    if ((ret < 0) && (ret != RPCSVC_ACTOR_IGNORE)) {
+        gf_msg(GF_NFS3, GF_LOG_ERROR, -ret, NFS_MSG_READDIR_FAIL,
+               "READDIR procedure failed");
+        rpcsvc_request_seterr(req, SYSTEM_ERR);
+        ret = RPCSVC_ACTOR_ERROR;
+    }
 
+rpcerr:
+    return ret;
+}
 
 int
-nfs3svc_readdirp (rpcsvc_request_t *req)
-{
-        readdirp3args   ra;
-        struct nfs3_fh  fh = {{0},};
-        int             ret = RPCSVC_ACTOR_ERROR;
-        uint64_t        cverf = 0;
-        uint64_t       *cval;
-
-        if (!req)
-                return ret;
-        nfs3_prep_readdirp3args (&ra, &fh);
-        if (xdr_to_readdirp3args (req->msg[0], &ra) <= 0) {
-                gf_log (GF_NFS3, GF_LOG_ERROR, "Error decoding args");
-                rpcsvc_request_seterr (req, GARBAGE_ARGS);
-                goto rpcerr;
-        }
-        cval = (uint64_t *) ra.cookieverf;
-        cverf = *cval;
-
-        ret = nfs3_readdir (req, &fh, ra.cookie, cverf, ra.dircount,
-                            ra.maxcount);
-        if ((ret < 0) && (ret != RPCSVC_ACTOR_IGNORE)) {
-                gf_log (GF_NFS3, GF_LOG_ERROR, "READDIRP procedure failed");
-                rpcsvc_request_seterr (req, SYSTEM_ERR);
-                ret = RPCSVC_ACTOR_ERROR;
-        }
+nfs3svc_readdirp(rpcsvc_request_t *req)
+{
+    readdirp3args ra;
+    struct nfs3_fh fh = {
+        {0},
+    };
+    int ret = RPCSVC_ACTOR_ERROR;
+    uint64_t cverf = 0;
+    uint64_t *cval;
 
-rpcerr:
+    if (!req)
         return ret;
-}
+    nfs3_prep_readdirp3args(&ra, &fh);
+    if (xdr_to_readdirp3args(req->msg[0], &ra) <= 0) {
+        gf_msg(GF_NFS3, GF_LOG_ERROR, 0, NFS_MSG_ARGS_DECODE_ERROR,
+               "Error decoding args");
+        rpcsvc_request_seterr(req, GARBAGE_ARGS);
+        goto rpcerr;
+    }
+    cval = (uint64_t *)ra.cookieverf;
+    cverf = *cval;
+
+    ret = nfs3_readdir(req, &fh, ra.cookie, cverf, ra.dircount, ra.maxcount);
+    if ((ret < 0) && (ret != RPCSVC_ACTOR_IGNORE)) {
+        gf_msg(GF_NFS3, GF_LOG_ERROR, -ret, NFS_MSG_READDIRP_FAIL,
+               "READDIRP procedure failed");
+        rpcsvc_request_seterr(req, SYSTEM_ERR);
+        ret = RPCSVC_ACTOR_ERROR;
+    }
 
+rpcerr:
+    return ret;
+}
 
 int
-nfs3_fsstat_reply (rpcsvc_request_t *req, nfsstat3 stat, struct statvfs *fsbuf,
-                   struct iatt *postbuf)
+nfs3_fsstat_reply(rpcsvc_request_t *req, nfsstat3 stat, struct statvfs *fsbuf,
+                  struct iatt *postbuf)
 {
-        fsstat3res      res = {0, };
-        uint64_t        deviceid = 0;
-
-        deviceid = nfs3_request_xlator_deviceid (req);
-        nfs3_fill_fsstat3res (&res, stat, fsbuf, postbuf, deviceid);
-        return nfs3svc_submit_reply (req, &res,
-                                     (nfs3_serializer)xdr_serialize_fsstat3res);
+    fsstat3res res = {
+        0,
+    };
+    uint64_t deviceid = 0;
 
+    deviceid = nfs3_request_xlator_deviceid(req);
+    nfs3_fill_fsstat3res(&res, stat, fsbuf, postbuf, deviceid);
+    return nfs3svc_submit_reply(req, &res,
+                                (nfs3_serializer)xdr_serialize_fsstat3res);
 }
 
-
 int32_t
-nfs3_fsstat_stat_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                      int32_t op_ret, int32_t op_errno, struct iatt *buf,
-                      dict_t *xdata)
-{
-        nfsstat3                stat = NFS3ERR_SERVERFAULT;
-        nfs3_call_state_t       *cs = NULL;
-
-        cs = frame->local;
-        if (op_ret == -1) {
-                gf_log (GF_NFS, GF_LOG_WARNING,
-                        "%x: %s => -1 (%s)", rpcsvc_request_xid (cs->req),
-                        cs->resolvedloc.path, strerror (op_errno));
-                stat = nfs3_cbk_errno_status (op_ret, op_errno);
-        } else
-                stat = NFS3_OK;
+nfs3_fsstat_stat_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                     int32_t op_ret, int32_t op_errno, struct iatt *buf,
+                     dict_t *xdata)
+{
+    nfsstat3 stat = NFS3ERR_SERVERFAULT;
+    nfs3_call_state_t *cs = NULL;
 
-        nfs3_log_common_res (rpcsvc_request_xid (cs->req), NFS3_FSSTAT, stat,
-                             op_errno);
-        nfs3_fsstat_reply (cs->req, stat, &cs->fsstat, buf);
-        nfs3_call_state_wipe (cs);
-        return 0;
-}
+    cs = frame->local;
+    if (op_ret == -1) {
+        stat = nfs3_cbk_errno_status(op_ret, op_errno);
+    } else
+        stat = NFS3_OK;
 
+    nfs3_log_common_res(rpcsvc_request_xid(cs->req), NFS3_FSSTAT, stat,
+                        op_errno, cs->resolvedloc.path);
+    nfs3_fsstat_reply(cs->req, stat, &cs->fsstat, buf);
+    nfs3_call_state_wipe(cs);
+    return 0;
+}
 
 int32_t
-nfs3_fsstat_statfs_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                        int32_t op_ret, int32_t op_errno, struct statvfs *buf,
-                        dict_t *xdata)
-{
-        nfs_user_t              nfu = {0, };
-        int                     ret = -EFAULT;
-        nfsstat3                stat = NFS3ERR_SERVERFAULT;
-        nfs3_call_state_t       *cs = NULL;
-
-        cs = frame->local;
-        if (op_ret == -1) {
-                gf_log (GF_NFS, GF_LOG_WARNING,
-                        "%x: %s => -1 (%s)", rpcsvc_request_xid (cs->req),
-                        cs->resolvedloc.path, strerror (op_errno));
-                ret = -op_errno;
-                stat = nfs3_cbk_errno_status (op_ret, op_errno);
-                goto err;
-        }
-
-        /* Then get the stat for the fs root in order to fill in the
-         * post_op_attr.
-         */
-        cs->fsstat = *buf;
-        nfs_request_user_init (&nfu, cs->req);
-        ret = nfs_stat (cs->nfsx, cs->vol, &nfu, &cs->resolvedloc,
-                        nfs3_fsstat_stat_cbk, cs);
-        if (ret < 0)
-                stat = nfs3_errno_to_nfsstat3 (-ret);
+nfs3_fsstat_statfs_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                       int32_t op_ret, int32_t op_errno, struct statvfs *buf,
+                       dict_t *xdata)
+{
+    nfs_user_t nfu = {
+        0,
+    };
+    int ret = -EFAULT;
+    nfsstat3 stat = NFS3ERR_SERVERFAULT;
+    nfs3_call_state_t *cs = NULL;
+
+    cs = frame->local;
+    if (op_ret == -1) {
+        ret = -op_errno;
+        stat = nfs3_cbk_errno_status(op_ret, op_errno);
+        goto err;
+    }
+
+    /* Then get the stat for the fs root in order to fill in the
+     * post_op_attr.
+     */
+    cs->fsstat = *buf;
+    nfs_request_user_init(&nfu, cs->req);
+    ret = nfs_stat(cs->nfsx, cs->vol, &nfu, &cs->resolvedloc,
+                   nfs3_fsstat_stat_cbk, cs);
+    if (ret < 0)
+        stat = nfs3_errno_to_nfsstat3(-ret);
 
 err:
-        if (ret < 0) {
-                nfs3_log_common_res (rpcsvc_request_xid (cs->req), NFS3_FSSTAT,
-                                     stat, -ret);
-                nfs3_fsstat_reply (cs->req, stat, NULL, NULL);
-                nfs3_call_state_wipe (cs);
-        }
+    if (ret < 0) {
+        nfs3_log_common_res(rpcsvc_request_xid(cs->req), NFS3_FSSTAT, stat,
+                            -ret, cs->resolvedloc.path);
+        nfs3_fsstat_reply(cs->req, stat, NULL, NULL);
+        nfs3_call_state_wipe(cs);
+    }
 
-        return 0;
+    return 0;
 }
 
-
 int
-nfs3_fsstat_resume (void *carg)
-{
-        nfsstat3                stat = NFS3ERR_SERVERFAULT;
-        int                     ret = -EFAULT;
-        nfs3_call_state_t       *cs = NULL;
-        nfs_user_t              nfu = {0, };
-
-        if (!carg)
-                return ret;
-
-        cs = (nfs3_call_state_t *)carg;
-        nfs3_check_fh_resolve_status (cs, stat, nfs3err);
-        nfs_request_user_init (&nfu, cs->req);
-        /* First, we need to get the statvfs for the subvol */
-        ret = nfs_statfs (cs->nfsx, cs->vol, &nfu, &cs->resolvedloc,
-                          nfs3_fsstat_statfs_cbk, cs);
-        if (ret < 0)
-                stat = nfs3_errno_to_nfsstat3 (-ret);
-
-nfs3err:
-        if (ret < 0) {
-                nfs3_log_common_res (rpcsvc_request_xid (cs->req), NFS3_FSSTAT,
-                                     stat, -ret);
-                nfs3_fsstat_reply (cs->req, stat, NULL, NULL);
-                nfs3_call_state_wipe (cs);
-        }
+nfs3_fsstat_resume(void *carg)
+{
+    nfsstat3 stat = NFS3ERR_SERVERFAULT;
+    int ret = -EFAULT;
+    nfs3_call_state_t *cs = NULL;
+    nfs_user_t nfu = {
+        0,
+    };
 
+    if (!carg)
         return ret;
-}
 
+    cs = (nfs3_call_state_t *)carg;
+    nfs3_check_fh_resolve_status(cs, stat, nfs3err);
+    nfs_request_user_init(&nfu, cs->req);
+    /* First, we need to get the statvfs for the subvol */
+    ret = nfs_statfs(cs->nfsx, cs->vol, &nfu, &cs->resolvedloc,
+                     nfs3_fsstat_statfs_cbk, cs);
+    if (ret < 0)
+        stat = nfs3_errno_to_nfsstat3(-ret);
 
+nfs3err:
+    if (ret < 0) {
+        nfs3_log_common_res(rpcsvc_request_xid(cs->req), NFS3_FSSTAT, stat,
+                            -ret, cs->resolvedloc.path);
+        nfs3_fsstat_reply(cs->req, stat, NULL, NULL);
+        nfs3_call_state_wipe(cs);
+    }
 
-int
-nfs3_fsstat (rpcsvc_request_t *req, struct nfs3_fh *fh)
-{
-        xlator_t                        *vol = NULL;
-        nfsstat3                        stat = NFS3ERR_SERVERFAULT;
-        int                             ret = -EFAULT;
-        struct nfs3_state               *nfs3 = NULL;
-        nfs3_call_state_t               *cs = NULL;
-
-        if ((!req) || (!fh)) {
-                gf_log (GF_NFS3, GF_LOG_ERROR, "Bad arguments");
-                return -1;
-        }
-
-        nfs3_log_common_call (rpcsvc_request_xid (req), "FSSTAT", fh);
-        nfs3_validate_gluster_fh (fh, stat, nfs3err);
-        nfs3_validate_nfs3_state (req, nfs3, stat, nfs3err, ret);
-        nfs3_map_fh_to_volume (nfs3, fh, req, vol, stat, nfs3err);
-        nfs3_volume_started_check (nfs3, vol, ret, out);
-        nfs3_handle_call_state_init (nfs3, cs, req, vol, stat, nfs3err);
+    return ret;
+}
 
-        ret = nfs3_fh_resolve_and_resume (cs, fh, NULL, nfs3_fsstat_resume);
-        if (ret < 0)
-                stat = nfs3_errno_to_nfsstat3 (-ret);
+int
+nfs3_fsstat(rpcsvc_request_t *req, struct nfs3_fh *fh)
+{
+    xlator_t *vol = NULL;
+    nfsstat3 stat = NFS3ERR_SERVERFAULT;
+    int ret = -EFAULT;
+    struct nfs3_state *nfs3 = NULL;
+    nfs3_call_state_t *cs = NULL;
+
+    if ((!req) || (!fh)) {
+        gf_msg(GF_NFS3, GF_LOG_ERROR, EINVAL, NFS_MSG_INVALID_ENTRY,
+               "Bad arguments");
+        return -1;
+    }
+
+    nfs3_log_common_call(rpcsvc_request_xid(req), "FSSTAT", fh);
+    nfs3_validate_gluster_fh(fh, stat, nfs3err);
+    nfs3_validate_nfs3_state(req, nfs3, stat, nfs3err, ret);
+    nfs3_map_fh_to_volume(nfs3, fh, req, vol, stat, nfs3err);
+    nfs3_volume_started_check(nfs3, vol, ret, out);
+    nfs3_handle_call_state_init(nfs3, cs, req, vol, stat, nfs3err);
+
+    ret = nfs3_fh_resolve_and_resume(cs, fh, NULL, nfs3_fsstat_resume);
+    if (ret < 0)
+        stat = nfs3_errno_to_nfsstat3(-ret);
 
 nfs3err:
-        if (ret < 0) {
-                nfs3_log_common_res (rpcsvc_request_xid (req), NFS3_FSSTAT,
-                                     stat, -ret);
-                nfs3_fsstat_reply (req, stat, NULL, NULL);
-                nfs3_call_state_wipe (cs);
-                /* Ret must be 0 after this so that the caller does not
-                 * also send an RPC reply.
-                 */
-                ret = 0;
-        }
+    if (ret < 0) {
+        nfs3_log_common_res(rpcsvc_request_xid(req), NFS3_FSSTAT, stat, -ret,
+                            cs ? cs->resolvedloc.path : NULL);
+        nfs3_fsstat_reply(req, stat, NULL, NULL);
+        nfs3_call_state_wipe(cs);
+        /* Ret must be 0 after this so that the caller does not
+         * also send an RPC reply.
+         */
+        ret = 0;
+    }
 out:
-        return ret;
+    return ret;
 }
 
-
 int
-nfs3svc_fsstat (rpcsvc_request_t *req)
-{
-        struct nfs3_fh          fh = {{0}, };
-        fsstat3args             args;
-        int                     ret = RPCSVC_ACTOR_ERROR;
-
-        if (!req)
-                return ret;
-        nfs3_prep_fsstat3args (&args, &fh);
-        if (xdr_to_fsstat3args (req->msg[0], &args) <= 0) {
-                gf_log (GF_NFS3, GF_LOG_ERROR, "Error decoding args");
-                rpcsvc_request_seterr (req, GARBAGE_ARGS);
-                goto rpcerr;
-        }
+nfs3svc_fsstat(rpcsvc_request_t *req)
+{
+    struct nfs3_fh fh = {
+        {0},
+    };
+    fsstat3args args;
+    int ret = RPCSVC_ACTOR_ERROR;
 
-        ret = nfs3_fsstat (req, &fh);
-        if ((ret < 0) && (ret != RPCSVC_ACTOR_IGNORE)) {
-                gf_log (GF_NFS3, GF_LOG_ERROR, "FSTAT procedure failed");
-                rpcsvc_request_seterr (req, SYSTEM_ERR);
-                ret = RPCSVC_ACTOR_ERROR;
-        }
+    if (!req)
+        return ret;
+    nfs3_prep_fsstat3args(&args, &fh);
+    if (xdr_to_fsstat3args(req->msg[0], &args) <= 0) {
+        gf_msg(GF_NFS3, GF_LOG_ERROR, 0, NFS_MSG_ARGS_DECODE_ERROR,
+               "Error decoding args");
+        rpcsvc_request_seterr(req, GARBAGE_ARGS);
+        goto rpcerr;
+    }
+
+    ret = nfs3_fsstat(req, &fh);
+    if ((ret < 0) && (ret != RPCSVC_ACTOR_IGNORE)) {
+        gf_msg(GF_NFS3, GF_LOG_ERROR, 0, NFS_MSG_FSTAT_FAIL,
+               "FSTAT procedure failed");
+        rpcsvc_request_seterr(req, SYSTEM_ERR);
+        ret = RPCSVC_ACTOR_ERROR;
+    }
 
 rpcerr:
-        return ret;
+    return ret;
 }
 
-
 int
-nfs3_fsinfo_reply (rpcsvc_request_t *req, nfsstat3 status, struct iatt *fsroot)
+nfs3_fsinfo_reply(rpcsvc_request_t *req, nfsstat3 status, struct iatt *fsroot)
 {
-        fsinfo3res              res;
-        struct nfs3_state       *nfs3 = NULL;
-        uint64_t                deviceid = 0;
+    fsinfo3res res;
+    struct nfs3_state *nfs3 = NULL;
+    uint64_t deviceid = 0;
 
-        deviceid = nfs3_request_xlator_deviceid (req);
-        nfs3 = rpcsvc_request_program_private (req);
-        nfs3_fill_fsinfo3res (nfs3, &res, status, fsroot, deviceid);
+    deviceid = nfs3_request_xlator_deviceid(req);
+    nfs3 = rpcsvc_request_program_private(req);
+    nfs3_fill_fsinfo3res(nfs3, &res, status, fsroot, deviceid);
 
-        nfs3svc_submit_reply (req, &res,
-                              (nfs3_serializer)xdr_serialize_fsinfo3res);
-        return 0;
+    nfs3svc_submit_reply(req, &res, (nfs3_serializer)xdr_serialize_fsinfo3res);
+    return 0;
 }
 
-
 int32_t
-nfs3svc_fsinfo_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                    int32_t op_ret, int32_t op_errno, struct iatt *buf,
-                    dict_t *xdata)
+nfs3svc_fsinfo_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                   int32_t op_ret, int32_t op_errno, struct iatt *buf,
+                   dict_t *xdata)
 {
-        nfsstat3                status = NFS3ERR_SERVERFAULT;
-        nfs3_call_state_t       *cs = NULL;
+    nfsstat3 status = NFS3ERR_SERVERFAULT;
+    nfs3_call_state_t *cs = NULL;
 
-        cs = frame->local;
+    cs = frame->local;
 
-        if (op_ret == -1) {
-                gf_log (GF_NFS, GF_LOG_WARNING,
-                        "%x: %s => -1 (%s)", rpcsvc_request_xid (cs->req),
-                        cs->resolvedloc.path, strerror (op_errno));
-                status = nfs3_cbk_errno_status (op_ret, op_errno);
-        }else
-                status = NFS3_OK;
+    if (op_ret == -1) {
+        status = nfs3_cbk_errno_status(op_ret, op_errno);
+    } else
+        status = NFS3_OK;
 
-        nfs3_log_common_res (rpcsvc_request_xid (cs->req), NFS3_FSINFO, status,
-                             op_errno);
+    nfs3_log_common_res(rpcsvc_request_xid(cs->req), NFS3_FSINFO, status,
+                        op_errno, cs->resolvedloc.path);
 
-        nfs3_fsinfo_reply (cs->req, status, buf);
-        nfs3_call_state_wipe (cs);
+    nfs3_fsinfo_reply(cs->req, status, buf);
+    nfs3_call_state_wipe(cs);
 
-        return 0;
+    return 0;
 }
 
-
 int
-nfs3_fsinfo_resume (void *carg)
+nfs3_fsinfo_resume(void *carg)
 {
-        nfsstat3                        stat = NFS3ERR_SERVERFAULT;
-        int                             ret = -EFAULT;
-        nfs_user_t                      nfu = {0, };
-        nfs3_call_state_t               *cs = NULL;
+    nfsstat3 stat = NFS3ERR_SERVERFAULT;
+    int ret = -EFAULT;
+    nfs_user_t nfu = {
+        0,
+    };
+    nfs3_call_state_t *cs = NULL;
 
+    if (!carg)
+        return ret;
 
-        if (!carg)
-                return ret;
-
-        cs = (nfs3_call_state_t *)carg;
-        nfs3_check_fh_resolve_status (cs, stat, nfs3err);
-        nfs_request_user_init (&nfu, cs->req);
+    cs = (nfs3_call_state_t *)carg;
+    nfs3_check_fh_resolve_status(cs, stat, nfs3err);
+    nfs_request_user_init(&nfu, cs->req);
 
-        ret = nfs_stat (cs->nfsx, cs->vol, &nfu, &cs->resolvedloc,
-                        nfs3svc_fsinfo_cbk, cs);
-        if (ret < 0)
-                stat = nfs3_errno_to_nfsstat3 (-ret);
+    ret = nfs_stat(cs->nfsx, cs->vol, &nfu, &cs->resolvedloc,
+                   nfs3svc_fsinfo_cbk, cs);
+    if (ret < 0)
+        stat = nfs3_errno_to_nfsstat3(-ret);
 
 nfs3err:
-        if (ret < 0) {
-                nfs3_log_common_res (rpcsvc_request_xid (cs->req), NFS3_FSINFO,
-                                     stat, -ret);
-                nfs3_fsinfo_reply (cs->req, stat, NULL);
-                nfs3_call_state_wipe (cs);
-        }
+    if (ret < 0) {
+        nfs3_log_common_res(rpcsvc_request_xid(cs->req), NFS3_FSINFO, stat,
+                            -ret, cs->resolvedloc.path);
+        nfs3_fsinfo_reply(cs->req, stat, NULL);
+        nfs3_call_state_wipe(cs);
+    }
 
-        return ret;
+    return ret;
 }
 
-
 int
-nfs3_fsinfo (rpcsvc_request_t *req, struct nfs3_fh *fh)
-{
-        xlator_t                        *vol = NULL;
-        nfsstat3                        stat = NFS3ERR_SERVERFAULT;
-        int                             ret = -EFAULT;
-        struct nfs3_state               *nfs3 = NULL;
-        nfs3_call_state_t               *cs = NULL;
-
-        if ((!req) || (!fh)) {
-                gf_log (GF_NFS3, GF_LOG_ERROR, "Bad arguments");
-                return -1;
-        }
-
-        nfs3_log_common_call (rpcsvc_request_xid (req), "FSINFO", fh);
-        nfs3_validate_gluster_fh (fh, stat, nfs3err);
-        nfs3_validate_nfs3_state (req, nfs3, stat, nfs3err, ret);
-        nfs3_map_fh_to_volume (nfs3, fh, req, vol, stat, nfs3err);
-        nfs3_volume_started_check (nfs3, vol, ret, out);
-        nfs3_handle_call_state_init (nfs3, cs, req, vol, stat, nfs3err);
-
-        ret = nfs3_fh_resolve_and_resume (cs, fh, NULL, nfs3_fsinfo_resume);
-        if (ret < 0)
-                stat = nfs3_errno_to_nfsstat3 (-ret);
+nfs3_fsinfo(rpcsvc_request_t *req, struct nfs3_fh *fh)
+{
+    xlator_t *vol = NULL;
+    nfsstat3 stat = NFS3ERR_SERVERFAULT;
+    int ret = -EFAULT;
+    struct nfs3_state *nfs3 = NULL;
+    nfs3_call_state_t *cs = NULL;
+
+    if ((!req) || (!fh)) {
+        gf_msg(GF_NFS3, GF_LOG_ERROR, EINVAL, NFS_MSG_INVALID_ENTRY,
+               "Bad arguments");
+        return -1;
+    }
+
+    nfs3_log_common_call(rpcsvc_request_xid(req), "FSINFO", fh);
+    nfs3_validate_gluster_fh(fh, stat, nfs3err);
+    nfs3_validate_nfs3_state(req, nfs3, stat, nfs3err, ret);
+    nfs3_map_fh_to_volume(nfs3, fh, req, vol, stat, nfs3err);
+    nfs3_volume_started_check(nfs3, vol, ret, out);
+    nfs3_handle_call_state_init(nfs3, cs, req, vol, stat, nfs3err);
+
+    ret = nfs3_fh_resolve_and_resume(cs, fh, NULL, nfs3_fsinfo_resume);
+    if (ret < 0)
+        stat = nfs3_errno_to_nfsstat3(-ret);
 
 nfs3err:
-        if (ret < 0) {
-                nfs3_log_common_res (rpcsvc_request_xid (req), NFS3_FSINFO,
-                                     stat, -ret);
-                nfs3_fsinfo_reply (req, stat, NULL);
-                nfs3_call_state_wipe (cs);
-                ret = 0;
-        }
+    if (ret < 0) {
+        nfs3_log_common_res(rpcsvc_request_xid(req), NFS3_FSINFO, stat, -ret,
+                            cs ? cs->resolvedloc.path : NULL);
+        nfs3_fsinfo_reply(req, stat, NULL);
+        nfs3_call_state_wipe(cs);
+        ret = 0;
+    }
 out:
-        return ret;
+    return ret;
 }
 
-
 int
-nfs3svc_fsinfo (rpcsvc_request_t *req)
+nfs3svc_fsinfo(rpcsvc_request_t *req)
 {
-        int             ret = RPCSVC_ACTOR_ERROR;
-        fsinfo3args     args;
-        struct nfs3_fh  root = {{0}, };
+    int ret = RPCSVC_ACTOR_ERROR;
+    fsinfo3args args;
+    struct nfs3_fh root = {
+        {0},
+    };
 
-        if (!req)
-                return ret;
+    if (!req)
+        return ret;
 
-        nfs3_prep_fsinfo3args (&args, &root);
-        if (xdr_to_fsinfo3args (req->msg[0], &args) <= 0) {
-                gf_log (GF_NFS3, GF_LOG_ERROR, "Error decoding arguments");
-                rpcsvc_request_seterr (req, GARBAGE_ARGS);
-                goto rpcerr;
-        }
-
-        ret = nfs3_fsinfo (req, &root);
-        if ((ret < 0) && (ret != RPCSVC_ACTOR_IGNORE)) {
-                gf_log (GF_NFS3, GF_LOG_ERROR, "FSINFO procedure failed");
-                rpcsvc_request_seterr (req, SYSTEM_ERR);
-                ret = RPCSVC_ACTOR_ERROR;
-        }
+    nfs3_prep_fsinfo3args(&args, &root);
+    if (xdr_to_fsinfo3args(req->msg[0], &args) <= 0) {
+        gf_msg(GF_NFS3, GF_LOG_ERROR, 0, NFS_MSG_ARGS_DECODE_ERROR,
+               "Error decoding arguments");
+        rpcsvc_request_seterr(req, GARBAGE_ARGS);
+        goto rpcerr;
+    }
+
+    ret = nfs3_fsinfo(req, &root);
+    if ((ret < 0) && (ret != RPCSVC_ACTOR_IGNORE)) {
+        gf_msg(GF_NFS3, GF_LOG_ERROR, 0, NFS_MSG_FSINFO_FAIL,
+               "FSINFO procedure failed");
+        rpcsvc_request_seterr(req, SYSTEM_ERR);
+        ret = RPCSVC_ACTOR_ERROR;
+    }
 
 rpcerr:
-        return ret;
+    return ret;
 }
 
-
 int
-nfs3_pathconf_reply (rpcsvc_request_t *req, nfsstat3 stat, struct iatt *buf)
+nfs3_pathconf_reply(rpcsvc_request_t *req, nfsstat3 stat, struct iatt *buf)
 {
-        pathconf3res    res = {0, };
-        uint64_t        deviceid = 0;
+    pathconf3res res = {
+        0,
+    };
+    uint64_t deviceid = 0;
 
-        deviceid = nfs3_request_xlator_deviceid (req);
-        nfs3_fill_pathconf3res (&res, stat, buf, deviceid);
-        nfs3svc_submit_reply (req, (void *)&res,
-                              (nfs3_serializer)xdr_serialize_pathconf3res);
-        return 0;
+    deviceid = nfs3_request_xlator_deviceid(req);
+    nfs3_fill_pathconf3res(&res, stat, buf, deviceid);
+    nfs3svc_submit_reply(req, (void *)&res,
+                         (nfs3_serializer)xdr_serialize_pathconf3res);
+    return 0;
 }
 
-
 int32_t
-nfs3svc_pathconf_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                      int32_t op_ret, int32_t op_errno, struct iatt *buf,
-                      dict_t *xdata)
-{
-        struct iatt             *sbuf = NULL;
-        nfs3_call_state_t       *cs = NULL;
-        nfsstat3                stat = NFS3ERR_SERVERFAULT;
-
-        cs = frame->local;
-        if (op_ret == -1) {
-                gf_log (GF_NFS, GF_LOG_WARNING,
-                        "%x: %s => -1 (%s)", rpcsvc_request_xid (cs->req),
-                        cs->resolvedloc.path, strerror (op_errno));
-                stat = nfs3_cbk_errno_status (op_ret, op_errno);
-        } else {
-                /* If stat fop failed, we can still send the other components
-                 * in a pathconf reply.
-                 */
-                sbuf = buf;
-                stat = NFS3_OK;
-        }
+nfs3svc_pathconf_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                     int32_t op_ret, int32_t op_errno, struct iatt *buf,
+                     dict_t *xdata)
+{
+    struct iatt *sbuf = NULL;
+    nfs3_call_state_t *cs = NULL;
+    nfsstat3 stat = NFS3ERR_SERVERFAULT;
 
-        nfs3_log_common_res (rpcsvc_request_xid (cs->req), NFS3_PATHCONF, stat,
-                             op_errno);
-        nfs3_pathconf_reply (cs->req, stat, sbuf);
-        nfs3_call_state_wipe (cs);
+    cs = frame->local;
+    if (op_ret == -1) {
+        stat = nfs3_cbk_errno_status(op_ret, op_errno);
+    } else {
+        /* If stat fop failed, we can still send the other components
+         * in a pathconf reply.
+         */
+        sbuf = buf;
+        stat = NFS3_OK;
+    }
 
-        return 0;
-}
+    nfs3_log_common_res(rpcsvc_request_xid(cs->req), NFS3_PATHCONF, stat,
+                        op_errno, cs->resolvedloc.path);
+    nfs3_pathconf_reply(cs->req, stat, sbuf);
+    nfs3_call_state_wipe(cs);
 
+    return 0;
+}
 
 int
-nfs3_pathconf_resume (void *carg)
+nfs3_pathconf_resume(void *carg)
 {
-        nfsstat3                        stat = NFS3ERR_SERVERFAULT;
-        int                             ret = -EFAULT;
-        nfs_user_t                      nfu = {0, };
-        nfs3_call_state_t               *cs = NULL;
+    nfsstat3 stat = NFS3ERR_SERVERFAULT;
+    int ret = -EFAULT;
+    nfs_user_t nfu = {
+        0,
+    };
+    nfs3_call_state_t *cs = NULL;
 
-        if (!carg)
-                return ret;
+    if (!carg)
+        return ret;
 
-        cs = (nfs3_call_state_t *)carg;
-        nfs3_check_fh_resolve_status (cs, stat, nfs3err);
-        nfs_request_user_init (&nfu, cs->req);
-        ret = nfs_stat (cs->nfsx, cs->vol, &nfu, &cs->resolvedloc,
-                        nfs3svc_pathconf_cbk, cs);
-        if (ret < 0)
-                stat = nfs3_errno_to_nfsstat3 (-ret);
+    cs = (nfs3_call_state_t *)carg;
+    nfs3_check_fh_resolve_status(cs, stat, nfs3err);
+    nfs_request_user_init(&nfu, cs->req);
+    ret = nfs_stat(cs->nfsx, cs->vol, &nfu, &cs->resolvedloc,
+                   nfs3svc_pathconf_cbk, cs);
+    if (ret < 0)
+        stat = nfs3_errno_to_nfsstat3(-ret);
 nfs3err:
-        if (ret < 0) {
-                nfs3_log_common_res (rpcsvc_request_xid (cs->req),
-                                     NFS3_PATHCONF, stat, -ret);
-                nfs3_pathconf_reply (cs->req, stat, NULL);
-                nfs3_call_state_wipe (cs);
-        }
+    if (ret < 0) {
+        nfs3_log_common_res(rpcsvc_request_xid(cs->req), NFS3_PATHCONF, stat,
+                            -ret, cs->resolvedloc.path);
+        nfs3_pathconf_reply(cs->req, stat, NULL);
+        nfs3_call_state_wipe(cs);
+    }
 
-        return ret;
+    return ret;
 }
 
 int
-nfs3_pathconf (rpcsvc_request_t *req, struct nfs3_fh *fh)
-{
-        xlator_t                        *vol = NULL;
-        nfsstat3                        stat = NFS3ERR_SERVERFAULT;
-        int                             ret = -EFAULT;
-        struct nfs3_state               *nfs3 = NULL;
-        nfs3_call_state_t               *cs = NULL;
-
-        if ((!req) || (!fh)) {
-                gf_log (GF_NFS3, GF_LOG_ERROR, "Bad arguments");
-                return -1;
-        }
-
-        nfs3_log_common_call (rpcsvc_request_xid (req), "PATHCONF", fh);
-        nfs3_validate_gluster_fh (fh, stat, nfs3err);
-        nfs3_validate_nfs3_state (req, nfs3, stat, nfs3err, ret);
-        nfs3_map_fh_to_volume (nfs3, fh, req, vol, stat, nfs3err);
-        nfs3_volume_started_check (nfs3, vol, ret, out);
-        nfs3_handle_call_state_init (nfs3, cs, req, vol, stat, nfs3err);
-
-        ret = nfs3_fh_resolve_and_resume (cs, fh, NULL, nfs3_pathconf_resume);
-        if (ret < 0)
-                stat = nfs3_errno_to_nfsstat3 (-ret);
+nfs3_pathconf(rpcsvc_request_t *req, struct nfs3_fh *fh)
+{
+    xlator_t *vol = NULL;
+    nfsstat3 stat = NFS3ERR_SERVERFAULT;
+    int ret = -EFAULT;
+    struct nfs3_state *nfs3 = NULL;
+    nfs3_call_state_t *cs = NULL;
+
+    if ((!req) || (!fh)) {
+        gf_msg(GF_NFS3, GF_LOG_ERROR, EINVAL, NFS_MSG_INVALID_ENTRY,
+               "Bad arguments");
+        return -1;
+    }
+
+    nfs3_log_common_call(rpcsvc_request_xid(req), "PATHCONF", fh);
+    nfs3_validate_gluster_fh(fh, stat, nfs3err);
+    nfs3_validate_nfs3_state(req, nfs3, stat, nfs3err, ret);
+    nfs3_map_fh_to_volume(nfs3, fh, req, vol, stat, nfs3err);
+    nfs3_volume_started_check(nfs3, vol, ret, out);
+    nfs3_handle_call_state_init(nfs3, cs, req, vol, stat, nfs3err);
+
+    ret = nfs3_fh_resolve_and_resume(cs, fh, NULL, nfs3_pathconf_resume);
+    if (ret < 0)
+        stat = nfs3_errno_to_nfsstat3(-ret);
 
 nfs3err:
-        if (ret < 0) {
-                nfs3_log_common_res (rpcsvc_request_xid (req), NFS3_PATHCONF,
-                                     stat, -ret);
-                nfs3_pathconf_reply (req, stat, NULL);
-                nfs3_call_state_wipe (cs);
-                /* Ret must be 0 after this so that the caller does not
-                 * also send an RPC reply.
-                 */
-                ret = 0;
-        }
+    if (ret < 0) {
+        nfs3_log_common_res(rpcsvc_request_xid(req), NFS3_PATHCONF, stat, -ret,
+                            cs ? cs->resolvedloc.path : NULL);
+        nfs3_pathconf_reply(req, stat, NULL);
+        nfs3_call_state_wipe(cs);
+        /* Ret must be 0 after this so that the caller does not
+         * also send an RPC reply.
+         */
+        ret = 0;
+    }
 out:
-        return ret;
+    return ret;
 }
 
-
 int
-nfs3svc_pathconf (rpcsvc_request_t *req)
-{
-        struct nfs3_fh          fh = {{0}, };
-        pathconf3args           args;
-        int                     ret = RPCSVC_ACTOR_ERROR;
-
-        if (!req)
-                return ret;
-        nfs3_prep_pathconf3args (&args, &fh);
-        if (xdr_to_pathconf3args (req->msg[0], &args) <= 0) {
-                gf_log (GF_NFS3, GF_LOG_ERROR, "Error decoding args");
-                rpcsvc_request_seterr (req, GARBAGE_ARGS);
-                goto rpcerr;
-        }
+nfs3svc_pathconf(rpcsvc_request_t *req)
+{
+    struct nfs3_fh fh = {
+        {0},
+    };
+    pathconf3args args;
+    int ret = RPCSVC_ACTOR_ERROR;
 
-        ret = nfs3_pathconf (req, &fh);
-        if ((ret < 0) && (ret != RPCSVC_ACTOR_IGNORE)) {
-                gf_log (GF_NFS3, GF_LOG_ERROR, "PATHCONF procedure failed");
-                rpcsvc_request_seterr (req, SYSTEM_ERR);
-                ret = RPCSVC_ACTOR_ERROR;
-        }
+    if (!req)
+        return ret;
+    nfs3_prep_pathconf3args(&args, &fh);
+    if (xdr_to_pathconf3args(req->msg[0], &args) <= 0) {
+        gf_msg(GF_NFS3, GF_LOG_ERROR, 0, NFS_MSG_ARGS_DECODE_ERROR,
+               "Error decoding args");
+        rpcsvc_request_seterr(req, GARBAGE_ARGS);
+        goto rpcerr;
+    }
+
+    ret = nfs3_pathconf(req, &fh);
+    if ((ret < 0) && (ret != RPCSVC_ACTOR_IGNORE)) {
+        gf_msg(GF_NFS3, GF_LOG_ERROR, -ret, NFS_MSG_PATHCONF_FAIL,
+               "PATHCONF procedure failed");
+        rpcsvc_request_seterr(req, SYSTEM_ERR);
+        ret = RPCSVC_ACTOR_ERROR;
+    }
 
 rpcerr:
-        return ret;
+    return ret;
 }
 
 int
-nfs3_commit_reply (rpcsvc_request_t *req, nfsstat3 stat, uint64_t wverf,
-                   struct iatt *prestat, struct iatt *poststat)
+nfs3_commit_reply(rpcsvc_request_t *req, nfsstat3 stat, uint64_t wverf,
+                  struct iatt *prestat, struct iatt *poststat)
 {
-        commit3res      res = {0, };
-        uint64_t        deviceid = 0;
+    commit3res res = {
+        0,
+    };
+    uint64_t deviceid = 0;
 
-        deviceid = nfs3_request_xlator_deviceid (req);
-        nfs3_fill_commit3res (&res, stat, wverf, prestat, poststat, deviceid);
-        nfs3svc_submit_reply (req, (void *)&res,
-                              (nfs3_serializer)xdr_serialize_commit3res);
+    deviceid = nfs3_request_xlator_deviceid(req);
+    nfs3_fill_commit3res(&res, stat, wverf, prestat, poststat, deviceid);
+    nfs3svc_submit_reply(req, (void *)&res,
+                         (nfs3_serializer)xdr_serialize_commit3res);
 
-        return 0;
+    return 0;
 }
 
-
 int32_t
-nfs3svc_commit_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                    int32_t op_ret, int32_t op_errno, dict_t *xdata)
-{
-        nfsstat3                stat = NFS3ERR_SERVERFAULT;
-        nfs3_call_state_t       *cs = NULL;
-        struct nfs3_state       *nfs3 = NULL;
-
-        cs = frame->local;
-        if (op_ret == -1) {
-                gf_log (GF_NFS, GF_LOG_WARNING,
-                        "%x: %s => -1 (%s)", rpcsvc_request_xid (cs->req),
-                        cs->resolvedloc.path, strerror (op_errno));
-                stat = nfs3_cbk_errno_status (op_ret, op_errno);
-        } else
-                stat = NFS3_OK;
+nfs3svc_commit_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                   int32_t op_ret, int32_t op_errno, dict_t *xdata)
+{
+    nfsstat3 stat = NFS3ERR_SERVERFAULT;
+    nfs3_call_state_t *cs = NULL;
+    struct nfs3_state *nfs3 = NULL;
+
+    cs = frame->local;
+    if (op_ret == -1) {
+        stat = nfs3_cbk_errno_status(op_ret, op_errno);
+    } else
+        stat = NFS3_OK;
 
-        nfs3 = rpcsvc_request_program_private (cs->req);
-        nfs3_log_commit_res (rpcsvc_request_xid (cs->req), stat, op_errno,
-                             nfs3->serverstart);
-        nfs3_commit_reply (cs->req, stat, nfs3->serverstart, NULL, NULL);
-        nfs3_call_state_wipe (cs);
+    nfs3 = rpcsvc_request_program_private(cs->req);
+    nfs3_log_commit_res(rpcsvc_request_xid(cs->req), stat, op_errno,
+                        nfs3->serverstart, cs->resolvedloc.path);
+    nfs3_commit_reply(cs->req, stat, nfs3->serverstart, NULL, NULL);
+    nfs3_call_state_wipe(cs);
 
-        return 0;
+    return 0;
 }
 
 int
-nfs3_commit_resume (void *carg)
+nfs3_commit_resume(void *carg)
 {
-        nfsstat3                stat = NFS3ERR_SERVERFAULT;
-        int                     ret = -EFAULT;
-        nfs_user_t              nfu = {0, };
-        nfs3_call_state_t       *cs = NULL;
+    nfsstat3 stat = NFS3ERR_SERVERFAULT;
+    int ret = -EFAULT;
+    nfs_user_t nfu = {
+        0,
+    };
+    nfs3_call_state_t *cs = NULL;
 
-        if (!carg)
-                return ret;
+    if (!carg)
+        return ret;
 
-        cs = (nfs3_call_state_t *)carg;
-        nfs3_check_fh_resolve_status (cs, stat, nfs3err);
+    cs = (nfs3_call_state_t *)carg;
+    nfs3_check_fh_resolve_status(cs, stat, nfs3err);
 
-        if (nfs3_export_sync_trusted (cs->nfs3state, cs->resolvefh.exportid)) {
-                ret = -1;
-                stat = NFS3_OK;
-                goto nfs3err;
-        }
+    if (nfs3_export_sync_trusted(cs->nfs3state, cs->resolvefh.exportid)) {
+        ret = -1;
+        stat = NFS3_OK;
+        goto nfs3err;
+    }
 
-        nfs_request_user_init (&nfu, cs->req);
-        ret = nfs_flush (cs->nfsx, cs->vol, &nfu, cs->fd,
-                         nfs3svc_commit_cbk, cs);
-        if (ret < 0)
-                stat = nfs3_errno_to_nfsstat3 (-ret);
+    nfs_request_user_init(&nfu, cs->req);
+    ret = nfs_flush(cs->nfsx, cs->vol, &nfu, cs->fd, nfs3svc_commit_cbk, cs);
+    if (ret < 0)
+        stat = nfs3_errno_to_nfsstat3(-ret);
 
 nfs3err:
-        if (ret < 0) {
-                nfs3_log_common_res (rpcsvc_request_xid (cs->req), NFS3_COMMIT,
-                                     stat, -ret);
-                nfs3_commit_reply (cs->req, stat, cs->nfs3state->serverstart,
-                                   NULL, NULL);
-                nfs3_call_state_wipe (cs);
-                ret = 0;
-        }
+    if (ret < 0) {
+        nfs3_log_common_res(rpcsvc_request_xid(cs->req), NFS3_COMMIT, stat,
+                            -ret, cs->resolvedloc.path);
+        nfs3_commit_reply(cs->req, stat, cs->nfs3state->serverstart, NULL,
+                          NULL);
+        nfs3_call_state_wipe(cs);
+        ret = 0;
+    }
 
-        return 0;
+    return 0;
 }
 
-
 int
-nfs3_commit_open_resume (void *carg)
+nfs3_commit_open_resume(void *carg)
 {
-        nfsstat3                stat = NFS3ERR_SERVERFAULT;
-        int                     ret = -EFAULT;
-        nfs3_call_state_t       *cs = NULL;
-
-        if (!carg)
-                return ret;
+    nfsstat3 stat = NFS3ERR_SERVERFAULT;
+    int ret = -EFAULT;
+    nfs3_call_state_t *cs = NULL;
 
-        cs = (nfs3_call_state_t *)carg;
-        nfs3_check_fh_resolve_status (cs, stat, nfs3err);
-        cs->fd = fd_anonymous (cs->resolvedloc.inode);
-        if (!cs->fd) {
-                gf_log (GF_NFS3, GF_LOG_ERROR, "Failed to create anonymous fd.");
-                goto nfs3err;
-        }
+    if (!carg)
+        return ret;
 
-        ret = nfs3_commit_resume (cs);
-        if (ret < 0)
-                stat = nfs3_errno_to_nfsstat3 (-ret);
+    cs = (nfs3_call_state_t *)carg;
+    nfs3_check_fh_resolve_status(cs, stat, nfs3err);
+    cs->fd = fd_anonymous(cs->resolvedloc.inode);
+    if (!cs->fd) {
+        gf_msg(GF_NFS3, GF_LOG_ERROR, 0, NFS_MSG_ANONYMOUS_FD_FAIL,
+               "Failed to create anonymous fd.");
+        goto nfs3err;
+    }
+
+    ret = nfs3_commit_resume(cs);
+    if (ret < 0)
+        stat = nfs3_errno_to_nfsstat3(-ret);
 nfs3err:
-        if (ret < 0) {
-                nfs3_log_common_res (rpcsvc_request_xid (cs->req), NFS3_COMMIT,
-                                     stat, -ret);
-                nfs3_commit_reply (cs->req, stat, 0, NULL, NULL);
-                nfs3_call_state_wipe (cs);
-        }
+    if (ret < 0) {
+        nfs3_log_common_res(rpcsvc_request_xid(cs->req), NFS3_COMMIT, stat,
+                            -ret, cs->resolvedloc.path);
+        nfs3_commit_reply(cs->req, stat, 0, NULL, NULL);
+        nfs3_call_state_wipe(cs);
+    }
 
-        return ret;
+    return ret;
 }
 
-
-
 int
-nfs3_commit (rpcsvc_request_t *req, struct nfs3_fh *fh, offset3 offset,
-             count3 count)
-{
-        xlator_t                        *vol = NULL;
-        nfsstat3                        stat = NFS3ERR_SERVERFAULT;
-        int                             ret = -EFAULT;
-        struct nfs3_state               *nfs3 = NULL;
-        nfs3_call_state_t               *cs = NULL;
-
-        if ((!req) || (!fh)) {
-                gf_log (GF_NFS3, GF_LOG_ERROR, "Bad arguments");
-                return -1;
-        }
-
-        nfs3_log_rw_call (rpcsvc_request_xid (req), "COMMIT", fh, offset,
-                          count, -1);
-        nfs3_validate_gluster_fh (fh, stat, nfs3err);
-        nfs3_validate_nfs3_state (req, nfs3, stat, nfs3err, ret);
-        nfs3_map_fh_to_volume (nfs3, fh, req, vol, stat, nfs3err);
-        nfs3_volume_started_check (nfs3, vol, ret, out);
-        nfs3_check_rw_volaccess (nfs3, fh->exportid, stat, nfs3err);
-        nfs3_handle_call_state_init (nfs3, cs, req, vol, stat, nfs3err);
-
-        cs->datacount = count;
-        cs->dataoffset = offset;
-        ret = nfs3_fh_resolve_and_resume (cs, fh, NULL,
-                                          nfs3_commit_open_resume);
-        if (ret < 0)
-                stat = nfs3_errno_to_nfsstat3 (-ret);
+nfs3_commit(rpcsvc_request_t *req, struct nfs3_fh *fh, offset3 offset,
+            count3 count)
+{
+    xlator_t *vol = NULL;
+    nfsstat3 stat = NFS3ERR_SERVERFAULT;
+    int ret = -EFAULT;
+    struct nfs3_state *nfs3 = NULL;
+    nfs3_call_state_t *cs = NULL;
+
+    if ((!req) || (!fh)) {
+        gf_msg(GF_NFS3, GF_LOG_ERROR, EINVAL, NFS_MSG_INVALID_ENTRY,
+               "Bad arguments");
+        return -1;
+    }
+
+    nfs3_log_rw_call(rpcsvc_request_xid(req), "COMMIT", fh, offset, count, -1);
+    nfs3_validate_gluster_fh(fh, stat, nfs3err);
+    nfs3_validate_nfs3_state(req, nfs3, stat, nfs3err, ret);
+    nfs3_map_fh_to_volume(nfs3, fh, req, vol, stat, nfs3err);
+    nfs3_volume_started_check(nfs3, vol, ret, out);
+    nfs3_check_rw_volaccess(nfs3, fh->exportid, stat, nfs3err);
+    nfs3_handle_call_state_init(nfs3, cs, req, vol, stat, nfs3err);
+
+    cs->datacount = count;
+    cs->dataoffset = offset;
+    ret = nfs3_fh_resolve_and_resume(cs, fh, NULL, nfs3_commit_open_resume);
+    if (ret < 0)
+        stat = nfs3_errno_to_nfsstat3(-ret);
 
 nfs3err:
-        if (ret < 0) {
-                nfs3_log_common_res (rpcsvc_request_xid (req), NFS3_COMMIT,
-                                     stat, -ret);
-                nfs3_commit_reply (req, stat, 0, NULL, NULL);
-                nfs3_call_state_wipe (cs);
-                ret = 0;
-        }
+    if (ret < 0) {
+        nfs3_log_common_res(rpcsvc_request_xid(req), NFS3_COMMIT, stat, -ret,
+                            cs ? cs->resolvedloc.path : NULL);
+        nfs3_commit_reply(req, stat, 0, NULL, NULL);
+        nfs3_call_state_wipe(cs);
+        ret = 0;
+    }
 out:
-        return ret;
+    return ret;
 }
 
-
-
 int
-nfs3svc_commit (rpcsvc_request_t *req)
-{
-        struct nfs3_fh          fh = {{0}, };
-        commit3args             args;
-        int                     ret = RPCSVC_ACTOR_ERROR;
-
-        if (!req)
-                return ret;
-        nfs3_prep_commit3args (&args, &fh);
-        if (xdr_to_commit3args (req->msg[0], &args) <= 0) {
-                gf_log (GF_NFS3, GF_LOG_ERROR, "Error decoding args");
-                rpcsvc_request_seterr (req, GARBAGE_ARGS);
-                goto rpcerr;
-        }
-
-        ret = nfs3_commit (req, &fh, args.offset, args.count);
-        if ((ret < 0) && (ret != RPCSVC_ACTOR_IGNORE)) {
-                gf_log (GF_NFS3, GF_LOG_ERROR, "COMMIT procedure failed");
-                rpcsvc_request_seterr (req, SYSTEM_ERR);
-                ret = RPCSVC_ACTOR_ERROR;
-        }
+nfs3svc_commit(rpcsvc_request_t *req)
+{
+    struct nfs3_fh fh = {
+        {0},
+    };
+    commit3args args;
+    int ret = RPCSVC_ACTOR_ERROR;
 
-rpcerr:
+    if (!req)
         return ret;
-}
-
+    nfs3_prep_commit3args(&args, &fh);
+    if (xdr_to_commit3args(req->msg[0], &args) <= 0) {
+        gf_msg(GF_NFS3, GF_LOG_ERROR, 0, NFS_MSG_ARGS_DECODE_ERROR,
+               "Error decoding args");
+        rpcsvc_request_seterr(req, GARBAGE_ARGS);
+        goto rpcerr;
+    }
+
+    ret = nfs3_commit(req, &fh, args.offset, args.count);
+    if ((ret < 0) && (ret != RPCSVC_ACTOR_IGNORE)) {
+        gf_msg(GF_NFS3, GF_LOG_ERROR, 0, NFS_MSG_COMMIT_FAIL,
+               "COMMIT procedure failed");
+        rpcsvc_request_seterr(req, SYSTEM_ERR);
+        ret = RPCSVC_ACTOR_ERROR;
+    }
 
-rpcsvc_actor_t          nfs3svc_actors[NFS3_PROC_COUNT] = {
-        {"NULL",        NFS3_NULL,      nfs3svc_null,      NULL,   0, DRC_IDEMPOTENT},
-        {"GETATTR",     NFS3_GETATTR,   nfs3svc_getattr,   NULL,   0, DRC_IDEMPOTENT},
-        {"SETATTR",     NFS3_SETATTR,   nfs3svc_setattr,   NULL,   0, DRC_NON_IDEMPOTENT},
-        {"LOOKUP",      NFS3_LOOKUP,    nfs3svc_lookup,    NULL,   0, DRC_IDEMPOTENT},
-        {"ACCESS",      NFS3_ACCESS,    nfs3svc_access,    NULL,   0, DRC_IDEMPOTENT},
-        {"READLINK",    NFS3_READLINK,  nfs3svc_readlink,  NULL,   0, DRC_IDEMPOTENT},
-        {"READ",        NFS3_READ,      nfs3svc_read,      NULL,   0, DRC_IDEMPOTENT},
-        {"WRITE",       NFS3_WRITE,     nfs3svc_write,     nfs3svc_write_vecsizer, 0, DRC_NON_IDEMPOTENT},
-        {"CREATE",      NFS3_CREATE,    nfs3svc_create,    NULL,   0, DRC_NON_IDEMPOTENT},
-        {"MKDIR",       NFS3_MKDIR,     nfs3svc_mkdir,     NULL,   0, DRC_NON_IDEMPOTENT},
-        {"SYMLINK",     NFS3_SYMLINK,   nfs3svc_symlink,   NULL,   0, DRC_NON_IDEMPOTENT},
-        {"MKNOD",       NFS3_MKNOD,     nfs3svc_mknod,     NULL,   0, DRC_NON_IDEMPOTENT},
-        {"REMOVE",      NFS3_REMOVE,    nfs3svc_remove,    NULL,   0, DRC_NON_IDEMPOTENT},
-        {"RMDIR",       NFS3_RMDIR,     nfs3svc_rmdir,     NULL,   0, DRC_NON_IDEMPOTENT},
-        {"RENAME",      NFS3_RENAME,    nfs3svc_rename,    NULL,   0, DRC_NON_IDEMPOTENT},
-        {"LINK",        NFS3_LINK,      nfs3svc_link,      NULL,   0, DRC_NON_IDEMPOTENT},
-        {"READDIR",     NFS3_READDIR,   nfs3svc_readdir,   NULL,   0, DRC_IDEMPOTENT},
-        {"READDIRPLUS", NFS3_READDIRP,  nfs3svc_readdirp,  NULL,   0, DRC_IDEMPOTENT},
-        {"FSSTAT",      NFS3_FSSTAT,    nfs3svc_fsstat,    NULL,   0, DRC_IDEMPOTENT},
-        {"FSINFO",      NFS3_FSINFO,    nfs3svc_fsinfo,    NULL,   0, DRC_IDEMPOTENT},
-        {"PATHCONF",    NFS3_PATHCONF,  nfs3svc_pathconf,  NULL,   0, DRC_IDEMPOTENT},
-        {"COMMIT",      NFS3_COMMIT,    nfs3svc_commit,    NULL,   0, DRC_IDEMPOTENT}
-};
-
-
-rpcsvc_program_t        nfs3prog = {
-                        .progname       = "NFS3",
-                        .prognum        = NFS_PROGRAM,
-                        .progver        = NFS_V3,
-                        .progport       = GF_NFS3_PORT,
-                        .actors         = nfs3svc_actors,
-                        .numactors      = NFS3_PROC_COUNT,
-
-                        /* Requests like FSINFO are sent before an auth scheme
-                         * is inited by client. See RFC 2623, Section 2.3.2. */
-                        .min_auth       = AUTH_NULL,
+rpcerr:
+    return ret;
+}
+
+static rpcsvc_actor_t nfs3svc_actors[NFS3_PROC_COUNT] = {
+    {"NULL", nfs3svc_null, NULL, NFS3_NULL, DRC_IDEMPOTENT, 0},
+    {"GETATTR", nfs3svc_getattr, NULL, NFS3_GETATTR, DRC_IDEMPOTENT, 0},
+    {"SETATTR", nfs3svc_setattr, NULL, NFS3_SETATTR, DRC_NON_IDEMPOTENT, 0},
+    {"LOOKUP", nfs3svc_lookup, NULL, NFS3_LOOKUP, DRC_IDEMPOTENT, 0},
+    {"ACCESS", nfs3svc_access, NULL, NFS3_ACCESS, DRC_IDEMPOTENT, 0},
+    {"READLINK", nfs3svc_readlink, NULL, NFS3_READLINK, DRC_IDEMPOTENT, 0},
+    {"READ", nfs3svc_read, NULL, NFS3_READ, DRC_IDEMPOTENT, 0},
+    {"WRITE", nfs3svc_write, nfs3svc_write_vecsizer, NFS3_WRITE, DRC_IDEMPOTENT,
+     0},
+    {"CREATE", nfs3svc_create, NULL, NFS3_CREATE, DRC_NON_IDEMPOTENT, 0},
+    {"MKDIR", nfs3svc_mkdir, NULL, NFS3_MKDIR, DRC_NON_IDEMPOTENT, 0},
+    {"SYMLINK", nfs3svc_symlink, NULL, NFS3_SYMLINK, DRC_NON_IDEMPOTENT, 0},
+    {"MKNOD", nfs3svc_mknod, NULL, NFS3_MKNOD, DRC_NON_IDEMPOTENT, 0},
+    {"REMOVE", nfs3svc_remove, NULL, NFS3_REMOVE, DRC_NON_IDEMPOTENT, 0},
+    {"RMDIR", nfs3svc_rmdir, NULL, NFS3_RMDIR, DRC_NON_IDEMPOTENT, 0},
+    {"RENAME", nfs3svc_rename, NULL, NFS3_RENAME, DRC_NON_IDEMPOTENT, 0},
+    {"LINK", nfs3svc_link, NULL, NFS3_LINK, DRC_NON_IDEMPOTENT, 0},
+    {"READDIR", nfs3svc_readdir, NULL, NFS3_READDIR, DRC_IDEMPOTENT, 0},
+    {"READDIRPLUS", nfs3svc_readdirp, NULL, NFS3_READDIRP, DRC_IDEMPOTENT, 0},
+    {"FSSTAT", nfs3svc_fsstat, NULL, NFS3_FSSTAT, DRC_IDEMPOTENT, 0},
+    {"FSINFO", nfs3svc_fsinfo, NULL, NFS3_FSINFO, DRC_IDEMPOTENT, 0},
+    {"PATHCONF", nfs3svc_pathconf, NULL, NFS3_PATHCONF, DRC_IDEMPOTENT, 0},
+    {"COMMIT", nfs3svc_commit, NULL, NFS3_COMMIT, DRC_IDEMPOTENT, 0}};
+
+static rpcsvc_program_t nfs3prog = {
+    .progname = "NFS3",
+    .prognum = NFS_PROGRAM,
+    .progver = NFS_V3,
+    .progport = GF_NFS3_PORT,
+    .actors = nfs3svc_actors,
+    .numactors = NFS3_PROC_COUNT,
+
+    /* Requests like FSINFO are sent before an auth scheme
+     * is inited by client. See RFC 2623, Section 2.3.2. */
+    .min_auth = AUTH_NULL,
 };
 
 /*
@@ -5190,486 +5236,503 @@ rpcsvc_program_t        nfs3prog = {
  * 1MB (GF_NFS3_FILE_IO_SIZE_MAX).
  */
 void
-nfs3_iosize_roundup_4KB (uint64_t *ioszptr)
+nfs3_iosize_roundup_4KB(uint64_t *ioszptr)
 {
-        uint64_t iosize;
-        uint64_t iopages;
+    uint64_t iosize;
+    uint64_t iopages;
 
-        if (!ioszptr)
-                return;
+    if (!ioszptr)
+        return;
 
-        iosize  = *ioszptr;
-        iopages = (iosize + GF_NFS3_IO_SIZE -1) >> GF_NFS3_IO_SHIFT;
-        iosize  = (iopages * GF_NFS3_IO_SIZE);
+    iosize = *ioszptr;
+    iopages = (iosize + GF_NFS3_IO_SIZE - 1) >> GF_NFS3_IO_SHIFT;
+    iosize = (iopages * GF_NFS3_IO_SIZE);
 
-        /* Double check - boundary conditions */
-        if (iosize < GF_NFS3_FILE_IO_SIZE_MIN) {
-                iosize = GF_NFS3_FILE_IO_SIZE_MIN;
-        } else if (iosize > GF_NFS3_FILE_IO_SIZE_MAX) {
-                iosize = GF_NFS3_FILE_IO_SIZE_MAX;
-        }
+    /* Double check - boundary conditions */
+    if (iosize < GF_NFS3_FILE_IO_SIZE_MIN) {
+        iosize = GF_NFS3_FILE_IO_SIZE_MIN;
+    } else if (iosize > GF_NFS3_FILE_IO_SIZE_MAX) {
+        iosize = GF_NFS3_FILE_IO_SIZE_MAX;
+    }
 
-        *ioszptr = iosize;
+    *ioszptr = iosize;
 }
 
 int
-nfs3_init_options (struct nfs3_state *nfs3, dict_t *options)
-{
-        int      ret = -1;
-        char     *optstr = NULL;
-        uint64_t size64 = 0;
-
-        if ((!nfs3) || (!options))
-                return -1;
-
-        /* nfs3.read-size */
-        nfs3->readsize = GF_NFS3_RTPREF;
-        if (dict_get (options, "nfs3.read-size")) {
-                ret = dict_get_str (options, "nfs3.read-size", &optstr);
-                if (ret < 0) {
-                        gf_log (GF_NFS3, GF_LOG_ERROR, "Failed to read "
-                                " option: nfs3.read-size");
-                        ret = -1;
-                        goto err;
-                }
-
-                ret = gf_string2uint64 (optstr, &size64);
-                if (ret == -1) {
-                        gf_log (GF_NFS3, GF_LOG_ERROR, "Failed to format"
-                                " option: nfs3.read-size");
-                        ret = -1;
-                        goto err;
-                }
-
-                nfs3_iosize_roundup_4KB (&size64);
-                nfs3->readsize = size64;
-        }
-
-        /* nfs3.write-size */
-        nfs3->writesize = GF_NFS3_WTPREF;
-        if (dict_get (options, "nfs3.write-size")) {
-                ret = dict_get_str (options, "nfs3.write-size", &optstr);
-                if (ret < 0) {
-                        gf_log (GF_NFS3, GF_LOG_ERROR, "Failed to read "
-                                " option: nfs3.write-size");
-                        ret = -1;
-                        goto err;
-                }
-
-                ret = gf_string2uint64 (optstr, &size64);
-                if (ret == -1) {
-                        gf_log (GF_NFS3, GF_LOG_ERROR, "Failed to format"
-                                " option: nfs3.write-size");
-                        ret = -1;
-                        goto err;
-                }
-
-                nfs3_iosize_roundup_4KB (&size64);
-                nfs3->writesize = size64;
-        }
-
-        /* nfs3.readdir.size */
-        nfs3->readdirsize = GF_NFS3_DTPREF;
-        if (dict_get (options, "nfs3.readdir-size")) {
-                ret = dict_get_str (options,"nfs3.readdir-size", &optstr);
-                if (ret < 0) {
-                        gf_log (GF_NFS3, GF_LOG_ERROR, "Failed to read"
-                                " option: nfs3.readdir-size");
-                        ret = -1;
-                        goto err;
-                }
-
-                ret = gf_string2uint64 (optstr, &size64);
-                if (ret == -1) {
-                        gf_log (GF_NFS3, GF_LOG_ERROR, "Failed to format"
-                                " option: nfs3.readdir-size");
-                        ret = -1;
-                        goto err;
-                }
-
-                nfs3_iosize_roundup_4KB (&size64);
-                nfs3->readdirsize = size64;
-        }
-
-        /* We want to use the size of the biggest param for the io buffer size.
-         */
-        nfs3->iobsize = nfs3->readsize;
-        if (nfs3->iobsize < nfs3->writesize)
-                nfs3->iobsize = nfs3->writesize;
-        if (nfs3->iobsize < nfs3->readdirsize)
-                nfs3->iobsize = nfs3->readdirsize;
-
-        /* But this is the true size of each iobuf. We need this size to
-         * accommodate the NFS headers also in the same buffer. */
-        nfs3->iobsize = nfs3->iobsize * 2;
-
-        ret = 0;
-err:
-        return ret;
-}
-
-int
-nfs3_init_subvolume_options (xlator_t *nfsx,
-                             struct nfs3_export *exp,
-                             dict_t *options)
+nfs3_init_options(struct nfs3_state *nfs3, dict_t *options)
 {
-        int             ret = -1;
-        char            *optstr = NULL;
-        char            searchkey[1024];
-        char            *name = NULL;
-        gf_boolean_t    boolt = _gf_false;
-        uuid_t          volumeid = {0, };
-
-        if ((!nfsx) || (!exp))
-                return -1;
+    int ret = -1;
+    char *optstr = NULL;
+    uint64_t size64 = 0;
 
-        /* For init, fetch options from xlator but for
-         * reconfigure, take the parameter */
-        if (!options)
-                options = nfsx->options;
+    if ((!nfs3) || (!options))
+        return -1;
 
-        if (!options)
-                return (-1);
-
-        gf_uuid_clear (volumeid);
-        if (gf_nfs_dvm_off (nfs_state (nfsx)))
-                goto no_dvm;
-
-        ret = snprintf (searchkey, 1024, "nfs3.%s.volume-id",exp->subvol->name);
+    /* nfs3.read-size */
+    nfs3->readsize = GF_NFS3_RTPREF;
+    if (dict_get(options, "nfs3.read-size")) {
+        ret = dict_get_str(options, "nfs3.read-size", &optstr);
         if (ret < 0) {
-                gf_log (GF_MNT, GF_LOG_ERROR, "snprintf failed");
-                ret = -1;
-                goto err;
+            gf_msg(GF_NFS3, GF_LOG_ERROR, 0, NFS_MSG_READ_FAIL,
+                   "Failed to read option: nfs3.read-size");
+            ret = -1;
+            goto err;
         }
 
-        if (dict_get (options, searchkey)) {
-                ret = dict_get_str (options, searchkey, &optstr);
-                if (ret < 0) {
-                        gf_log (GF_MNT, GF_LOG_ERROR, "Failed to read option"
-                                ": %s", searchkey);
-                        ret = -1;
-                        goto err;
-                }
-        } else {
-                gf_log (GF_MNT, GF_LOG_ERROR, "DVM is on but volume-id not "
-                        "given for volume: %s", exp->subvol->name);
-                ret = -1;
-                goto err;
+        ret = gf_string2uint64(optstr, &size64);
+        if (ret == -1) {
+            gf_msg(GF_NFS3, GF_LOG_ERROR, 0, NFS_MSG_FORMAT_FAIL,
+                   "Failed to format option: nfs3.read-size");
+            ret = -1;
+            goto err;
         }
 
-        if (optstr) {
-                ret = gf_uuid_parse (optstr, volumeid);
-                if (ret < 0) {
-                        gf_log (GF_MNT, GF_LOG_ERROR, "Failed to parse volume "
-                                "UUID");
-                        ret = -1;
-                        goto err;
-                }
-                gf_uuid_copy (exp->volumeid, volumeid);
-        }
+        nfs3_iosize_roundup_4KB(&size64);
+        nfs3->readsize = size64;
+    }
 
-no_dvm:
-        /* Volume Access */
-        name = exp->subvol->name;
-        ret = snprintf (searchkey, 1024, "nfs3.%s.volume-access", name);
+    /* nfs3.write-size */
+    nfs3->writesize = GF_NFS3_WTPREF;
+    if (dict_get(options, "nfs3.write-size")) {
+        ret = dict_get_str(options, "nfs3.write-size", &optstr);
         if (ret < 0) {
-                gf_log (GF_NFS3, GF_LOG_ERROR, "snprintf failed");
-                ret = -1;
-                goto err;
+            gf_msg(GF_NFS3, GF_LOG_ERROR, 0, NFS_MSG_READ_FAIL,
+                   "Failed to read  option: nfs3.write-size");
+            ret = -1;
+            goto err;
         }
 
-        exp->access = GF_NFS3_DEFAULT_VOLACCESS;
-        if (dict_get (options, searchkey)) {
-                ret = dict_get_str (options, searchkey, &optstr);
-                if (ret < 0) {
-                        gf_log (GF_NFS3, GF_LOG_ERROR, "Failed to read "
-                                " option: %s", searchkey);
-                        ret = -1;
-                        goto err;
-                }
-
-                if (strcmp (optstr, "read-only") == 0)
-                        exp->access = GF_NFS3_VOLACCESS_RO;
+        ret = gf_string2uint64(optstr, &size64);
+        if (ret == -1) {
+            gf_msg(GF_NFS3, GF_LOG_ERROR, 0, NFS_MSG_FORMAT_FAIL,
+                   "Failed to format option: nfs3.write-size");
+            ret = -1;
+            goto err;
         }
 
-        ret = snprintf (searchkey, 1024, "rpc-auth.%s.unix", name);
+        nfs3_iosize_roundup_4KB(&size64);
+        nfs3->writesize = size64;
+    }
+
+    /* nfs3.readdir.size */
+    nfs3->readdirsize = GF_NFS3_DTPREF;
+    if (dict_get(options, "nfs3.readdir-size")) {
+        ret = dict_get_str(options, "nfs3.readdir-size", &optstr);
         if (ret < 0) {
-                gf_log (GF_NFS3, GF_LOG_ERROR, "snprintf failed");
-                ret = -1;
-                goto err;
+            gf_msg(GF_NFS3, GF_LOG_ERROR, 0, NFS_MSG_READ_FAIL,
+                   "Failed to read option: nfs3.readdir-size");
+            ret = -1;
+            goto err;
         }
 
-        if (dict_get (options, searchkey)) {
-                ret = dict_get_str (options, searchkey, &optstr);
-                if (ret < 0) {
-                        gf_log (GF_NFS3, GF_LOG_ERROR, "Failed to read "
-                                " option: %s", searchkey);
-                        ret = -1;
-                        goto err;
-                }
+        ret = gf_string2uint64(optstr, &size64);
+        if (ret == -1) {
+            gf_msg(GF_NFS3, GF_LOG_ERROR, 0, NFS_MSG_FORMAT_FAIL,
+                   "Failed to format option: nfs3.readdir-size");
+            ret = -1;
+            goto err;
         }
 
-        exp->trusted_sync = 0;
-        ret = snprintf (searchkey, 1024, "nfs3.%s.trusted-sync", name);
-        if (ret < 0) {
-                gf_log (GF_NFS3, GF_LOG_ERROR, "snprintf failed");
-                ret = -1;
-                goto err;
-        }
+        nfs3_iosize_roundup_4KB(&size64);
+        nfs3->readdirsize = size64;
+    }
 
-        if (dict_get (options, searchkey)) {
-                ret = dict_get_str (options, searchkey, &optstr);
-                if (ret < 0) {
-                        gf_log (GF_NFS3, GF_LOG_ERROR, "Failed to read "
-                                " option: %s", searchkey);
-                        ret = -1;
-                        goto err;
-                }
-
-                ret = gf_string2boolean (optstr, &boolt);
-                if (ret < 0) {
-                        gf_log (GF_NFS3, GF_LOG_ERROR, "Failed to convert str "
-                                "to gf_boolean_t");
-                        ret = -1;
-                        goto err;
-                }
-
-                if (boolt == _gf_true)
-                        exp->trusted_sync = 1;
-        }
+    /* We want to use the size of the biggest param for the io buffer size.
+     */
+    nfs3->iobsize = nfs3->readsize;
+    if (nfs3->iobsize < nfs3->writesize)
+        nfs3->iobsize = nfs3->writesize;
+    if (nfs3->iobsize < nfs3->readdirsize)
+        nfs3->iobsize = nfs3->readdirsize;
+
+    /* But this is the true size of each iobuf. We need this size to
+     * accommodate the NFS headers also in the same buffer. */
+    nfs3->iobsize = nfs3->iobsize * 2;
 
-        exp->trusted_write = 0;
-        ret = snprintf (searchkey, 1024, "nfs3.%s.trusted-write", name);
+    ret = 0;
+err:
+    return ret;
+}
+
+int
+nfs3_init_subvolume_options(xlator_t *nfsx, struct nfs3_export *exp,
+                            dict_t *options)
+{
+    int ret = -1;
+    char *optstr = NULL;
+    char searchkey[1024];
+    char *name = NULL;
+    gf_boolean_t boolt = _gf_false;
+    uuid_t volumeid = {
+        0,
+    };
+
+    if ((!nfsx) || (!exp))
+        return -1;
+
+    /* For init, fetch options from xlator but for
+     * reconfigure, take the parameter */
+    if (!options)
+        options = nfsx->options;
+
+    if (!options)
+        return (-1);
+
+    gf_uuid_clear(volumeid);
+    if (gf_nfs_dvm_off(nfs_state(nfsx)))
+        goto no_dvm;
+
+    ret = snprintf(searchkey, sizeof(searchkey), "nfs3.%s.volume-id",
+                   exp->subvol->name);
+    if (ret < 0) {
+        gf_msg(GF_MNT, GF_LOG_ERROR, 0, NFS_MSG_SNPRINTF_FAIL,
+               "snprintf failed");
+        ret = -1;
+        goto err;
+    }
+
+    if (dict_get(options, searchkey)) {
+        ret = dict_get_str(options, searchkey, &optstr);
+        if (ret < 0) {
+            gf_msg(GF_MNT, GF_LOG_ERROR, 0, NFS_MSG_READ_FAIL,
+                   "Failed to read option: %s", searchkey);
+            ret = -1;
+            goto err;
+        }
+    } else {
+        gf_msg(GF_MNT, GF_LOG_ERROR, 0, NFS_MSG_VOLID_MISSING,
+               "DVM is"
+               " on but volume-id not given for volume: %s",
+               exp->subvol->name);
+        ret = -1;
+        goto err;
+    }
+
+    if (optstr) {
+        ret = gf_uuid_parse(optstr, volumeid);
         if (ret < 0) {
-                gf_log (GF_NFS3, GF_LOG_ERROR, "snprintf failed");
-                ret = -1;
-                goto err;
+            gf_msg(GF_MNT, GF_LOG_ERROR, 0, NFS_MSG_PARSE_VOL_UUID_FAIL,
+                   "Failed to parse volume UUID");
+            ret = -1;
+            goto err;
         }
+        gf_uuid_copy(exp->volumeid, volumeid);
+    }
 
-        if (dict_get (options, searchkey)) {
-                ret = dict_get_str (options, searchkey, &optstr);
-                if (ret < 0) {
-                        gf_log (GF_NFS3, GF_LOG_ERROR, "Failed to read "
-                                " option: %s", searchkey);
-                        ret = -1;
-                        goto err;
-                }
-
-                ret = gf_string2boolean (optstr, &boolt);
-                if (ret < 0) {
-                        gf_log (GF_NFS3, GF_LOG_ERROR, "Failed to convert str "
-                                "to gf_boolean_t");
-                        ret = -1;
-                        goto err;
-                }
-
-                if (boolt == _gf_true)
-                        exp->trusted_write = 1;
+no_dvm:
+    /* Volume Access */
+    name = exp->subvol->name;
+    ret = snprintf(searchkey, sizeof(searchkey), "nfs3.%s.volume-access", name);
+    if (ret < 0) {
+        gf_msg(GF_NFS3, GF_LOG_ERROR, 0, NFS_MSG_SNPRINTF_FAIL,
+               "snprintf failed");
+        ret = -1;
+        goto err;
+    }
+
+    exp->access = GF_NFS3_DEFAULT_VOLACCESS;
+    if (dict_get(options, searchkey)) {
+        ret = dict_get_str(options, searchkey, &optstr);
+        if (ret < 0) {
+            gf_msg(GF_NFS3, GF_LOG_ERROR, 0, NFS_MSG_READ_FAIL,
+                   "Failed to read option: %s", searchkey);
+            ret = -1;
+            goto err;
+        }
+
+        if (strcmp(optstr, "read-only") == 0)
+            exp->access = GF_NFS3_VOLACCESS_RO;
+    }
+
+    ret = snprintf(searchkey, sizeof(searchkey), "rpc-auth.%s.unix", name);
+    if (ret < 0) {
+        gf_msg(GF_NFS3, GF_LOG_ERROR, 0, NFS_MSG_SNPRINTF_FAIL,
+               "snprintf failed");
+        ret = -1;
+        goto err;
+    }
+
+    if (dict_get(options, searchkey)) {
+        ret = dict_get_str(options, searchkey, &optstr);
+        if (ret < 0) {
+            gf_msg(GF_NFS3, GF_LOG_ERROR, 0, NFS_MSG_READ_FAIL,
+                   "Failed to read option: %s", searchkey);
+            ret = -1;
+            goto err;
+        }
+    }
+
+    exp->trusted_sync = 0;
+    ret = snprintf(searchkey, sizeof(searchkey), "nfs3.%s.trusted-sync", name);
+    if (ret < 0) {
+        gf_msg(GF_NFS3, GF_LOG_ERROR, 0, NFS_MSG_SNPRINTF_FAIL,
+               "snprintf failed");
+        ret = -1;
+        goto err;
+    }
+
+    if (dict_get(options, searchkey)) {
+        ret = dict_get_str(options, searchkey, &optstr);
+        if (ret < 0) {
+            gf_msg(GF_NFS3, GF_LOG_ERROR, 0, NFS_MSG_READ_FAIL,
+                   "Failed to read option: %s", searchkey);
+            ret = -1;
+            goto err;
         }
 
-        /* If trusted-sync is on, then we also switch on trusted-write because
-         * tw is included in ts. In write logic, we're then only checking for
-         * tw.
-         */
-        if (exp->trusted_sync)
-                exp->trusted_write = 1;
+        ret = gf_string2boolean(optstr, &boolt);
+        if (ret < 0) {
+            gf_msg(GF_NFS3, GF_LOG_ERROR, 0, NFS_MSG_STR2BOOL_FAIL,
+                   "Failed to convert str "
+                   "to gf_boolean_t");
+            ret = -1;
+            goto err;
+        }
+
+        if (boolt == _gf_true)
+            exp->trusted_sync = 1;
+    }
+
+    exp->trusted_write = 0;
+    ret = snprintf(searchkey, sizeof(searchkey), "nfs3.%s.trusted-write", name);
+    if (ret < 0) {
+        gf_msg(GF_NFS3, GF_LOG_ERROR, 0, NFS_MSG_SNPRINTF_FAIL,
+               "snprintf failed");
+        ret = -1;
+        goto err;
+    }
+
+    if (dict_get(options, searchkey)) {
+        ret = dict_get_str(options, searchkey, &optstr);
+        if (ret < 0) {
+            gf_msg(GF_NFS3, GF_LOG_ERROR, 0, NFS_MSG_READ_FAIL,
+                   "Failed to read option: %s", searchkey);
+            ret = -1;
+            goto err;
+        }
 
-        gf_log (GF_NFS3, GF_LOG_TRACE, "%s: %s, %s, %s", exp->subvol->name,
-                (exp->access == GF_NFS3_VOLACCESS_RO)?"read-only":"read-write",
-                (exp->trusted_sync == 0)?"no trusted_sync":"trusted_sync",
-                (exp->trusted_write == 0)?"no trusted_write":"trusted_write");
-        ret = 0;
+        ret = gf_string2boolean(optstr, &boolt);
+        if (ret < 0) {
+            gf_msg(GF_NFS3, GF_LOG_ERROR, 0, NFS_MSG_STR2BOOL_FAIL,
+                   "Failed to convert str"
+                   " to gf_boolean_t");
+            ret = -1;
+            goto err;
+        }
+
+        if (boolt == _gf_true)
+            exp->trusted_write = 1;
+    }
+
+    /* If trusted-sync is on, then we also switch on trusted-write because
+     * tw is included in ts. In write logic, we're then only checking for
+     * tw.
+     */
+    if (exp->trusted_sync)
+        exp->trusted_write = 1;
+
+    gf_msg_trace(
+        GF_NFS3, 0, "%s: %s, %s, %s", exp->subvol->name,
+        (exp->access == GF_NFS3_VOLACCESS_RO) ? "read-only" : "read-write",
+        (exp->trusted_sync == 0) ? "no trusted_sync" : "trusted_sync",
+        (exp->trusted_write == 0) ? "no trusted_write" : "trusted_write");
+    ret = 0;
 err:
-        return ret;
+    return ret;
 }
 
-
 struct nfs3_export *
-nfs3_init_subvolume (struct nfs3_state *nfs3, xlator_t *subvol)
+nfs3_init_subvolume(struct nfs3_state *nfs3, xlator_t *subvol)
 {
-        int                     ret = -1;
-        struct nfs3_export      *exp = NULL;
+    int ret = -1;
+    struct nfs3_export *exp = NULL;
 
-        if ((!nfs3) || (!subvol))
-                return NULL;
+    if ((!nfs3) || (!subvol))
+        return NULL;
 
-        exp = GF_CALLOC (1, sizeof (*exp), gf_nfs_mt_nfs3_export);
-        exp->subvol = subvol;
-        INIT_LIST_HEAD (&exp->explist);
-        gf_log (GF_NFS3, GF_LOG_TRACE, "Initing state: %s", exp->subvol->name);
+    exp = GF_CALLOC(1, sizeof(*exp), gf_nfs_mt_nfs3_export);
+    exp->subvol = subvol;
+    INIT_LIST_HEAD(&exp->explist);
+    gf_msg_trace(GF_NFS3, 0, "Initing state: %s", exp->subvol->name);
 
-        ret = nfs3_init_subvolume_options (nfs3->nfsx, exp, NULL);
-        if (ret == -1) {
-                gf_log (GF_NFS3, GF_LOG_ERROR, "Failed to init subvol");
-                goto exp_free;
-        }
+    ret = nfs3_init_subvolume_options(nfs3->nfsx, exp, NULL);
+    if (ret == -1) {
+        gf_msg(GF_NFS3, GF_LOG_ERROR, 0, NFS_MSG_SUBVOL_INIT_FAIL,
+               "Failed to init subvol");
+        goto exp_free;
+    }
 
-        ret = 0;
+    ret = 0;
 exp_free:
-        if (ret < 0) {
-                GF_FREE (exp);
-                exp = NULL;
-        }
+    if (ret < 0) {
+        GF_FREE(exp);
+        exp = NULL;
+    }
 
-        return exp;
+    return exp;
 }
 
-
 int
-nfs3_init_subvolumes (struct nfs3_state *nfs3)
-{
-        int                     ret = -1;
-        struct xlator_list      *xl_list = NULL;
-        struct nfs3_export      *exp = NULL;
-
-        if (!nfs3)
-                return -1;
-
-        xl_list = nfs3->nfsx->children;
-
-        while (xl_list) {
-                exp = nfs3_init_subvolume (nfs3, xl_list->xlator);
-                if (!exp) {
-                        gf_log (GF_NFS3, GF_LOG_ERROR, "Failed to init subvol: "
-                                "%s", xl_list->xlator->name);
-                        goto err;
-                }
-                list_add_tail (&exp->explist, &nfs3->exports);
-                xl_list = xl_list->next;
-        }
-
-        ret = 0;
-err:
-        return ret;
-}
-
-
-struct nfs3_state *
-nfs3_init_state (xlator_t *nfsx)
+nfs3_init_subvolumes(struct nfs3_state *nfs3)
 {
-        struct nfs3_state       *nfs3 = NULL;
-        int                     ret = -1;
-        unsigned int            localpool = 0;
-        struct nfs_state        *nfs = NULL;
-
-        if ((!nfsx) || (!nfsx->private))
-                return NULL;
-
-        nfs3 = (struct nfs3_state *)GF_CALLOC (1, sizeof (*nfs3),
-                                    gf_nfs_mt_nfs3_state);
-        if (!nfs3) {
-                gf_log (GF_NFS3, GF_LOG_ERROR, "Memory allocation failed");
-                return NULL;
-        }
+    int ret = -1;
+    struct xlator_list *xl_list = NULL;
+    struct nfs3_export *exp = NULL;
 
-        nfs = nfsx->private;
-        ret = nfs3_init_options (nfs3, nfsx->options);
-        if (ret == -1) {
-                gf_log (GF_NFS3, GF_LOG_ERROR, "Failed to init options");
-                goto ret;
-        }
+    if (!nfs3)
+        return -1;
 
-        nfs3->iobpool = nfsx->ctx->iobuf_pool;
-
-        localpool = nfs->memfactor * GF_NFS_CONCURRENT_OPS_MULT;
-        gf_log (GF_NFS3, GF_LOG_TRACE, "local pool: %d", localpool);
-        nfs3->localpool = mem_pool_new (nfs3_call_state_t, localpool);
-        if (!nfs3->localpool) {
-                gf_log (GF_NFS3, GF_LOG_ERROR, "local mempool creation failed");
-                ret = -1;
-                goto ret;
-        }
+    xl_list = nfs3->nfsx->children;
 
-        nfs3->nfsx = nfsx;
-        nfs3->exportslist = nfsx->children;
-        INIT_LIST_HEAD (&nfs3->exports);
-        ret = nfs3_init_subvolumes (nfs3);
-        if (ret == -1) {
-                gf_log (GF_NFS3, GF_LOG_ERROR, "Failed to init per-subvolume "
-                        "state");
-                goto free_localpool;
+    while (xl_list) {
+        exp = nfs3_init_subvolume(nfs3, xl_list->xlator);
+        if (!exp) {
+            gf_msg(GF_NFS3, GF_LOG_ERROR, 0, NFS_MSG_SUBVOL_INIT_FAIL,
+                   "Failed to init "
+                   "subvol: %s",
+                   xl_list->xlator->name);
+            goto err;
         }
+        list_add_tail(&exp->explist, &nfs3->exports);
+        xl_list = xl_list->next;
+    }
 
-        nfs3->serverstart = (uint64_t)time (NULL);
-        INIT_LIST_HEAD (&nfs3->fdlru);
-        LOCK_INIT (&nfs3->fdlrulock);
-        nfs3->fdcount = 0;
-
-        ret = rpcsvc_create_listeners (nfs->rpcsvc, nfsx->options, nfsx->name);
-        if (ret == -1) {
-                gf_log (GF_NFS, GF_LOG_ERROR, "Unable to create listeners");
-                goto free_localpool;
-        }
+    ret = 0;
+err:
+    return ret;
+}
 
-        nfs->nfs3state = nfs3;
-        ret = 0;
+struct nfs3_state *
+nfs3_init_state(xlator_t *nfsx)
+{
+    struct nfs3_state *nfs3 = NULL;
+    int ret = -1;
+    unsigned int localpool = 0;
+    struct nfs_state *nfs = NULL;
+
+    if ((!nfsx) || (!nfsx->private))
+        return NULL;
+
+    nfs3 = (struct nfs3_state *)GF_CALLOC(1, sizeof(*nfs3),
+                                          gf_nfs_mt_nfs3_state);
+    if (!nfs3) {
+        gf_msg(GF_NFS3, GF_LOG_ERROR, ENOMEM, NFS_MSG_NO_MEMORY,
+               "Memory allocation failed");
+        return NULL;
+    }
+
+    nfs = nfsx->private;
+    ret = nfs3_init_options(nfs3, nfsx->options);
+    if (ret == -1) {
+        gf_msg(GF_NFS3, GF_LOG_ERROR, 0, NFS_MSG_OPT_INIT_FAIL,
+               "Failed to init options");
+        goto ret;
+    }
+
+    nfs3->iobpool = nfsx->ctx->iobuf_pool;
+
+    localpool = nfs->memfactor * GF_NFS_CONCURRENT_OPS_MULT;
+    gf_msg_trace(GF_NFS3, 0, "local pool: %d", localpool);
+    nfs3->localpool = mem_pool_new_ctx(nfsx->ctx, nfs3_call_state_t, localpool);
+    if (!nfs3->localpool) {
+        gf_msg(GF_NFS3, GF_LOG_ERROR, ENOMEM, NFS_MSG_NO_MEMORY,
+               "local mempool creation failed");
+        ret = -1;
+        goto ret;
+    }
+
+    nfs3->nfsx = nfsx;
+    nfs3->exportslist = nfsx->children;
+    INIT_LIST_HEAD(&nfs3->exports);
+    ret = nfs3_init_subvolumes(nfs3);
+    if (ret == -1) {
+        gf_msg(GF_NFS3, GF_LOG_ERROR, 0, NFS_MSG_SUBVOL_INIT_FAIL,
+               "Failed to init per-subvolume state");
+        goto free_localpool;
+    }
+
+    nfs3->serverstart = (uint64_t)gf_time();
+    INIT_LIST_HEAD(&nfs3->fdlru);
+    LOCK_INIT(&nfs3->fdlrulock);
+    nfs3->fdcount = 0;
+
+    ret = rpcsvc_create_listeners(nfs->rpcsvc, nfsx->options, nfsx->name);
+    if (ret == -1) {
+        gf_msg(GF_NFS, GF_LOG_ERROR, 0, NFS_MSG_LISTENERS_CREATE_FAIL,
+               "Unable to create listeners");
+        goto free_localpool;
+    }
+
+    nfs->nfs3state = nfs3;
+    ret = 0;
 
 free_localpool:
-        if (ret == -1)
-                mem_pool_destroy (nfs3->localpool);
+    if (ret == -1)
+        mem_pool_destroy(nfs3->localpool);
 
 ret:
-        if (ret == -1) {
-                GF_FREE (nfs3);
-                nfs3 = NULL;
-        }
+    if (ret == -1) {
+        GF_FREE(nfs3);
+        nfs3 = NULL;
+    }
 
-        return nfs3;
+    return nfs3;
 }
 
-
 rpcsvc_program_t *
-nfs3svc_init (xlator_t *nfsx)
+nfs3svc_init(xlator_t *nfsx)
 {
-        struct nfs3_state       *nfs3 = NULL;
+    struct nfs3_state *nfs3 = NULL;
 
-        if (!nfsx)
-                return NULL;
+    if (!nfsx)
+        return NULL;
 
-        nfs3 = nfs3_init_state (nfsx);
-        if (!nfs3) {
-                gf_log (GF_NFS3, GF_LOG_ERROR, "NFSv3 state init failed");
-                return NULL;
-        }
+    nfs3 = nfs3_init_state(nfsx);
+    if (!nfs3) {
+        gf_msg(GF_NFS3, GF_LOG_ERROR, 0, NFS_MSG_STATE_INIT_FAIL,
+               "NFSv3 state init failed");
+        return NULL;
+    }
 
-        nfs3prog.private = nfs3;
+    nfs3prog.private = nfs3;
 
-        return &nfs3prog;
+    return &nfs3prog;
 }
 
 int
-nfs3_reconfigure_state (xlator_t *nfsx, dict_t *options)
-{
-        int                   ret   = -1;
-        struct nfs3_export    *exp  = NULL;
-        struct nfs_state      *nfs  = NULL;
-        struct nfs3_state     *nfs3 = NULL;
-
-        if ((!nfsx) || (!nfsx->private) || (!options))
-                goto out;
-
-        nfs = (struct nfs_state *)nfsx->private;
-        nfs3 = nfs->nfs3state;
-        if (!nfs3)
-                goto out;
-
-        ret = nfs3_init_options (nfs3, options);
+nfs3_reconfigure_state(xlator_t *nfsx, dict_t *options)
+{
+    int ret = -1;
+    struct nfs3_export *exp = NULL;
+    struct nfs_state *nfs = NULL;
+    struct nfs3_state *nfs3 = NULL;
+
+    if ((!nfsx) || (!nfsx->private) || (!options))
+        goto out;
+
+    nfs = (struct nfs_state *)nfsx->private;
+    nfs3 = nfs->nfs3state;
+    if (!nfs3)
+        goto out;
+
+    ret = nfs3_init_options(nfs3, options);
+    if (ret) {
+        gf_msg(GF_NFS3, GF_LOG_ERROR, 0, NFS_MSG_RECONF_FAIL,
+               "Failed to reconfigure options");
+        goto out;
+    }
+
+    list_for_each_entry(exp, &nfs3->exports, explist)
+    {
+        ret = nfs3_init_subvolume_options(nfsx, exp, options);
         if (ret) {
-                gf_log (GF_NFS3, GF_LOG_ERROR,
-                        "Failed to reconfigure options");
-                goto out;
+            gf_msg(GF_NFS3, GF_LOG_ERROR, 0, NFS_MSG_RECONF_SUBVOL_FAIL,
+                   "Failed to reconfigure subvol options");
+            goto out;
         }
+    }
 
-        list_for_each_entry (exp, &nfs3->exports, explist) {
-                ret = nfs3_init_subvolume_options (nfsx, exp, options);
-                if (ret) {
-                        gf_log (GF_NFS3, GF_LOG_ERROR,
-                                "Failed to reconfigure subvol options");
-                        goto out;
-                }
-        }
-
-        ret = 0;
+    ret = 0;
 out:
-        return ret;
+    return ret;
 }
diff --git a/xlators/nfs/server/src/nfs3.h b/xlators/nfs/server/src/nfs3.h
index f1b89fe17a8..cdb7e03a619 100644
--- a/xlators/nfs/server/src/nfs3.h
+++ b/xlators/nfs/server/src/nfs3.h
@@ -11,32 +11,27 @@
 #ifndef _NFS3_H_
 #define _NFS3_H_
 
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
-
 #include "rpcsvc.h"
-#include "dict.h"
-#include "xlator.h"
-#include "iobuf.h"
+#include <glusterfs/dict.h>
+#include <glusterfs/xlator.h>
+#include <glusterfs/iobuf.h>
 #include "nfs.h"
 #include "nfs3-fh.h"
 #include "nfs-common.h"
 #include "xdr-nfs3.h"
-#include "mem-pool.h"
+#include <glusterfs/mem-pool.h>
 #include "nlm4.h"
 #include "acl3-xdr.h"
 #include "acl3.h"
+#include <glusterfs/refcount.h>
 #include <sys/statvfs.h>
 
-#define GF_NFS3                 GF_NFS"-nfsv3"
-
-#define GF_NFS3_DEFAULT_MEMFACTOR       15
-#define GF_NFS3_IOBPOOL_MULT            GF_NFS_CONCURRENT_OPS_MULT
-#define GF_NFS3_CLTABLE_BUCKETS_MULT    2
-#define GF_NFS3_FDTABLE_BUCKETS_MULT    2
+#define GF_NFS3 GF_NFS "-nfsv3"
 
+#define GF_NFS3_DEFAULT_MEMFACTOR 15
+#define GF_NFS3_IOBPOOL_MULT GF_NFS_CONCURRENT_OPS_MULT
+#define GF_NFS3_CLTABLE_BUCKETS_MULT 2
+#define GF_NFS3_FDTABLE_BUCKETS_MULT 2
 
 /* Static values used for FSINFO
  * To change the maximum rsize and wsize supported by the NFS client, adjust
@@ -47,136 +42,143 @@
  *
  * NB: For Kernel-NFS, NFS_MAX_FILE_IO_SIZE is 1048576U (1MB).
  */
-#define GF_NFS3_FILE_IO_SIZE_MAX     (1  * GF_UNIT_MB) /* 1048576 */
-#define GF_NFS3_FILE_IO_SIZE_MIN     (4  * GF_UNIT_KB) /* 4096 */
+#define GF_NFS3_FILE_IO_SIZE_MAX (1 * GF_UNIT_MB) /* 1048576 */
+#define GF_NFS3_FILE_IO_SIZE_MIN (4 * GF_UNIT_KB) /* 4096 */
 
-#define GF_NFS3_FILE_IO_SIZE_DEF     GF_NFS3_FILE_IO_SIZE_MAX
+#define GF_NFS3_FILE_IO_SIZE_DEF GF_NFS3_FILE_IO_SIZE_MAX
 
-#define GF_NFS3_RTMAX          GF_NFS3_FILE_IO_SIZE_MAX
-#define GF_NFS3_RTMIN          GF_NFS3_FILE_IO_SIZE_MIN
-#define GF_NFS3_RTPREF         GF_NFS3_FILE_IO_SIZE_DEF
-#define GF_NFS3_RTMULT         GF_NFS3_FILE_IO_SIZE_MIN
+#define GF_NFS3_RTMAX GF_NFS3_FILE_IO_SIZE_MAX
+#define GF_NFS3_RTMIN GF_NFS3_FILE_IO_SIZE_MIN
+#define GF_NFS3_RTPREF GF_NFS3_FILE_IO_SIZE_DEF
+#define GF_NFS3_RTMULT GF_NFS3_FILE_IO_SIZE_MIN
 
-#define GF_NFS3_WTMAX          GF_NFS3_FILE_IO_SIZE_MAX
-#define GF_NFS3_WTMIN          GF_NFS3_FILE_IO_SIZE_MIN
-#define GF_NFS3_WTPREF         GF_NFS3_FILE_IO_SIZE_DEF
-#define GF_NFS3_WTMULT         GF_NFS3_FILE_IO_SIZE_MIN
+#define GF_NFS3_WTMAX GF_NFS3_FILE_IO_SIZE_MAX
+#define GF_NFS3_WTMIN GF_NFS3_FILE_IO_SIZE_MIN
+#define GF_NFS3_WTPREF GF_NFS3_FILE_IO_SIZE_DEF
+#define GF_NFS3_WTMULT GF_NFS3_FILE_IO_SIZE_MIN
 
 /* This can be tuned through nfs.readdir-size */
-#define GF_NFS3_DTMAX          GF_NFS3_FILE_IO_SIZE_MAX
-#define GF_NFS3_DTMIN          GF_NFS3_FILE_IO_SIZE_MIN
-#define GF_NFS3_DTPREF         GF_NFS3_FILE_IO_SIZE_DEF
+#define GF_NFS3_DTMAX GF_NFS3_FILE_IO_SIZE_MAX
+#define GF_NFS3_DTMIN GF_NFS3_FILE_IO_SIZE_MIN
+#define GF_NFS3_DTPREF GF_NFS3_FILE_IO_SIZE_DEF
 
-#define GF_NFS3_MAXFILESIZE    (1 * GF_UNIT_PB)
+#define GF_NFS3_MAXFILESIZE (1 * GF_UNIT_PB)
 
-#define GF_NFS3_IO_SIZE        4096 /* 4-KB */
-#define GF_NFS3_IO_SHIFT       12   /* 2^12 = 4KB */
+#define GF_NFS3_IO_SIZE 4096 /* 4-KB */
+#define GF_NFS3_IO_SHIFT 12  /* 2^12 = 4KB */
 
 /* FIXME: Handle time resolutions */
-#define GF_NFS3_TIMEDELTA_SECS     {1,0}
-#define GF_NFS3_TIMEDELTA_NSECS    {0,1}
-#define GF_NFS3_TIMEDELTA_MSECS    {0,1000000}
-
-#define GF_NFS3_FS_PROP    (FSF3_LINK | FSF3_SYMLINK | FSF3_HOMOGENEOUS | FSF3_CANSETTIME)
-
-#define GF_NFS3_DIRFD_VALID        1
-#define GF_NFS3_DIRFD_INVALID      0
-
-#define GF_NFS3_VOLACCESS_RW    1
-#define GF_NFS3_VOLACCESS_RO    2
-
-
-#define GF_NFS3_FDCACHE_SIZE    512
+#define GF_NFS3_TIMEDELTA_SECS                                                 \
+    {                                                                          \
+        1, 0                                                                   \
+    }
+#define GF_NFS3_TIMEDELTA_NSECS                                                \
+    {                                                                          \
+        0, 1                                                                   \
+    }
+#define GF_NFS3_TIMEDELTA_MSECS                                                \
+    {                                                                          \
+        0, 1000000                                                             \
+    }
+
+#define GF_NFS3_FS_PROP                                                        \
+    (FSF3_LINK | FSF3_SYMLINK | FSF3_HOMOGENEOUS | FSF3_CANSETTIME)
+
+#define GF_NFS3_DIRFD_VALID 1
+#define GF_NFS3_DIRFD_INVALID 0
+
+#define GF_NFS3_VOLACCESS_RW 1
+#define GF_NFS3_VOLACCESS_RO 2
+
+#define GF_NFS3_FDCACHE_SIZE 512
 /* This should probably be moved to a more generic layer so that if needed
  * different versions of NFS protocol can use the same thing.
  */
 struct nfs3_fd_entry {
-        fd_t                    *cachedfd;
-        struct list_head        list;
+    fd_t *cachedfd;
+    struct list_head list;
 };
 
 /* Per subvolume nfs3 specific state */
 struct nfs3_export {
-        struct list_head        explist;
-        xlator_t                *subvol;
-        uuid_t                  volumeid;
-        int                     access;
-        int                     trusted_sync;
-        int                     trusted_write;
-        int                     rootlookedup;
+    struct list_head explist;
+    xlator_t *subvol;
+    uuid_t volumeid;
+    int access;
+    int trusted_sync;
+    int trusted_write;
+    int rootlookedup;
 };
 
-#define GF_NFS3_DEFAULT_VOLACCESS       (GF_NFS3_VOLACCESS_RW)
+#define GF_NFS3_DEFAULT_VOLACCESS (GF_NFS3_VOLACCESS_RW)
 
 /* The NFSv3 protocol state */
 typedef struct nfs3_state {
-
-        /* The NFS xlator pointer. The NFS xlator can be running
-         * multiple versions of the NFS protocol.
-         */
-        xlator_t                *nfsx;
-
-        /* The iob pool from which memory allocations are made for receiving
-         * and sending network messages.
-         */
-        struct iobuf_pool       *iobpool;
-
-        /* List of child subvolumes for the NFSv3 protocol.
-         * Right now, is simply referring to the list of children in nfsx above.
-         */
-        xlator_list_t           *exportslist;
-
-        struct list_head        exports;
-        /* Mempool for allocations of struct nfs3_local */
-        struct mem_pool         *localpool;
-
-        /* Server start-up timestamp, currently used for write verifier. */
-        uint64_t                serverstart;
-
-        /* NFSv3 Protocol configurables */
-        uint64_t                readsize;
-        uint64_t                writesize;
-        uint64_t                readdirsize;
-
-        /* Size of the iobufs used, depends on the sizes of the three params
-         * above.
-         */
-        uint64_t                iobsize;
-
-        struct list_head        fdlru;
-        gf_lock_t               fdlrulock;
-        int                     fdcount;
-        uint32_t                occ_logger;
+    /* The NFS xlator pointer. The NFS xlator can be running
+     * multiple versions of the NFS protocol.
+     */
+    xlator_t *nfsx;
+
+    /* The iob pool from which memory allocations are made for receiving
+     * and sending network messages.
+     */
+    struct iobuf_pool *iobpool;
+
+    /* List of child subvolumes for the NFSv3 protocol.
+     * Right now, is simply referring to the list of children in nfsx above.
+     */
+    xlator_list_t *exportslist;
+
+    struct list_head exports;
+    /* Mempool for allocations of struct nfs3_local */
+    struct mem_pool *localpool;
+
+    /* Server start-up timestamp, currently used for write verifier. */
+    uint64_t serverstart;
+
+    /* NFSv3 Protocol configurables */
+    uint64_t readsize;
+    uint64_t writesize;
+    uint64_t readdirsize;
+
+    /* Size of the iobufs used, depends on the sizes of the three params
+     * above.
+     */
+    uint64_t iobsize;
+
+    struct list_head fdlru;
+    gf_lock_t fdlrulock;
+    int fdcount;
+    uint32_t occ_logger;
 } nfs3_state_t;
 
 typedef enum nfs3_lookup_type {
-        GF_NFS3_REVALIDATE = 1,
-        GF_NFS3_FRESH,
+    GF_NFS3_REVALIDATE = 1,
+    GF_NFS3_FRESH,
 } nfs3_lookup_type_t;
 
 typedef union args_ {
-        nlm4_stat nlm4_stat;
-        nlm4_holder nlm4_holder;
-        nlm4_lock nlm4_lock;
-        nlm4_share nlm4_share;
-        nlm4_testrply nlm4_testrply;
-        nlm4_testres nlm4_testres;
-        nlm4_testargs nlm4_testargs;
-        nlm4_res nlm4_res;
-        nlm4_lockargs nlm4_lockargs;
-        nlm4_cancargs nlm4_cancargs;
-        nlm4_unlockargs nlm4_unlockargs;
-        nlm4_shareargs nlm4_shareargs;
-        nlm4_shareres nlm4_shareres;
-        nlm4_freeallargs nlm4_freeallargs;
-        getaclargs getaclargs;
-        setaclargs setaclargs;
-        getaclreply getaclreply;
-        setaclreply setaclreply;
+    nlm4_stat nlm4_stat;
+    nlm4_holder nlm4_holder;
+    nlm4_lock nlm4_lock;
+    nlm4_share nlm4_share;
+    nlm4_testrply nlm4_testrply;
+    nlm4_testres nlm4_testres;
+    nlm4_testargs nlm4_testargs;
+    nlm4_res nlm4_res;
+    nlm4_lockargs nlm4_lockargs;
+    nlm4_cancargs nlm4_cancargs;
+    nlm4_unlockargs nlm4_unlockargs;
+    nlm4_shareargs nlm4_shareargs;
+    nlm4_shareres nlm4_shareres;
+    nlm4_freeallargs nlm4_freeallargs;
+    getaclargs getaclargs;
+    setaclargs setaclargs;
+    getaclreply getaclreply;
+    setaclreply setaclreply;
 } args;
 
-
-typedef int (*nfs3_resume_fn_t) (void *cs);
+typedef int (*nfs3_resume_fn_t)(void *cs);
 /* Structure used to communicate state between a fop and its callback.
  * Not all members are used at all times. Usage is fop and NFS request
  * dependent.
@@ -189,77 +191,80 @@ typedef int (*nfs3_resume_fn_t) (void *cs);
  * Imagine the chaos if we need a mem-pool for each one of those sub-structures.
  */
 struct nfs3_local {
-        rpcsvc_request_t        *req;
-        xlator_t                *vol;
-        nfs3_resume_fn_t        resume_fn;
-        xlator_t                *nfsx;
-        struct nfs3_state       *nfs3state;
-
-        /* The list hook to attach this call state to the inode's queue till
-         * the opening of the fd on the inode completes.
-         */
-        struct list_head        openwait_q;
-
-        /* Per-NFSv3 Op state */
-        struct nfs3_fh          parent;
-        struct nfs3_fh          fh;
-        fd_t                    *fd;
-        uint32_t                accessbits;
-        int                     operrno;
-        count3                  dircount;
-        count3                  maxcount;
-        struct statvfs          fsstat;
-        gf_dirent_t             entries;
-        struct iatt             stbuf;
-        struct iatt             preparent;
-        struct iatt             postparent;
-        int32_t                 setattr_valid;
-        nfstime3                timestamp;
-        loc_t                   oploc;
-        int                     writetype;
-        count3                  datacount;
-        offset3                 dataoffset;
-        struct iobuf            *iob;
-        struct iobref           *iobref;
-        createmode3             createmode;
-        uint64_t                cookieverf;
-        int                     sattrguardcheck;
-        char                    *pathname;
-        ftype3                  mknodtype;
-        specdata3               devnums;
-        cookie3                 cookie;
-        struct iovec            datavec;
-        mode_t                  mode;
-
-        /* NFSv3 FH resolver state */
-        int                     hardresolved;
-        struct nfs3_fh          resolvefh;
-        loc_t                   resolvedloc;
-        int                     resolve_ret;
-        int                     resolve_errno;
-        int                     hashidx;
-        fd_t                    *resolve_dir_fd;
-        char                    *resolventry;
-        nfs3_lookup_type_t      lookuptype;
-        gf_dirent_t             *hashmatch;
-        gf_dirent_t             *entrymatch;
-        off_t                   lastentryoffset;
-        struct flock            flock;
-        args                    args;
-        nlm4_lkowner_t          lkowner;
-        char                    cookiebytes[1024];
-        struct nfs3_fh          lockfh;
-        int                     monitor;
-        rpc_transport_t         *trans;
-        call_frame_t            *frame;
-
-        /* ACL */
-        aclentry                aclentry[NFS_ACL_MAX_ENTRIES];
-        aclentry                daclentry[NFS_ACL_MAX_ENTRIES];
-        int                     aclcount;
-        char                    aclxattr[NFS_ACL_MAX_ENTRIES*8 + 4];
-        int                     daclcount;
-        char                    daclxattr[NFS_ACL_MAX_ENTRIES*8 + 4];
+    GF_REF_DECL;
+
+    rpcsvc_request_t *req;
+    xlator_t *vol;
+    nfs3_resume_fn_t resume_fn;
+    xlator_t *nfsx;
+    struct nfs3_state *nfs3state;
+
+    /* The list hook to attach this call state to the inode's queue till
+     * the opening of the fd on the inode completes.
+     */
+    struct list_head openwait_q;
+
+    /* Per-NFSv3 Op state */
+    struct nfs3_fh parent;
+    struct nfs3_fh fh;
+    fd_t *fd;
+    uint32_t accessbits;
+    int operrno;
+    count3 dircount;
+    count3 maxcount;
+    struct statvfs fsstat;
+    gf_dirent_t entries;
+    struct iatt stbuf;
+    struct iatt preparent;
+    struct iatt postparent;
+    int32_t setattr_valid;
+    nfstime3 timestamp;
+    loc_t oploc;
+    int writetype;
+    count3 datacount;
+    offset3 dataoffset;
+    struct iobuf *iob;
+    struct iobref *iobref;
+    createmode3 createmode;
+    uint64_t cookieverf;
+    int sattrguardcheck;
+    char *pathname;
+    ftype3 mknodtype;
+    specdata3 devnums;
+    cookie3 cookie;
+    struct iovec datavec;
+    mode_t mode;
+    struct iatt attr_in;
+
+    /* NFSv3 FH resolver state */
+    int hardresolved;
+    struct nfs3_fh resolvefh;
+    loc_t resolvedloc;
+    int resolve_ret;
+    int resolve_errno;
+    int hashidx;
+    fd_t *resolve_dir_fd;
+    char *resolventry;
+    nfs3_lookup_type_t lookuptype;
+    gf_dirent_t *hashmatch;
+    gf_dirent_t *entrymatch;
+    off_t lastentryoffset;
+    struct flock flock;
+    args args;
+    nlm4_lkowner_t lkowner;
+    char cookiebytes[1024];
+    struct nfs3_fh lockfh;
+    int monitor;
+    rpc_transport_t *trans;
+    call_frame_t *frame;
+
+    /* ACL */
+    aclentry aclentry[NFS_ACL_MAX_ENTRIES];
+    aclentry daclentry[NFS_ACL_MAX_ENTRIES];
+    int aclcount;
+    char aclxattr[NFS_ACL_MAX_ENTRIES * 8 + 4];
+    int daclcount;
+    char daclxattr[NFS_ACL_MAX_ENTRIES * 8 + 4];
 };
 
 #define nfs3_is_revalidate_lookup(cst) ((cst)->lookuptype == GF_NFS3_REVALIDATE)
@@ -271,17 +276,17 @@ typedef struct nfs3_local nfs3_call_state_t;
 
 /* Queue of ops waiting for open fop to return. */
 struct inode_op_queue {
-        struct list_head        opq;
-        pthread_mutex_t         qlock;
+    struct list_head opq;
+    pthread_mutex_t qlock;
 };
 
 extern rpcsvc_program_t *
-nfs3svc_init (xlator_t *nfsx);
+nfs3svc_init(xlator_t *nfsx);
 
 extern int
-nfs3_reconfigure_state (xlator_t *nfsx, dict_t *options);
+nfs3_reconfigure_state(xlator_t *nfsx, dict_t *options);
 
 extern uint64_t
-nfs3_request_xlator_deviceid (rpcsvc_request_t *req);
+nfs3_request_xlator_deviceid(rpcsvc_request_t *req);
 
 #endif
diff --git a/xlators/nfs/server/src/nfsserver.sym b/xlators/nfs/server/src/nfsserver.sym
new file mode 100644
index 00000000000..dce7d964e9e
--- /dev/null
+++ b/xlators/nfs/server/src/nfsserver.sym
@@ -0,0 +1,12 @@
+exp_file_parse
+exp_file_print
+exp_file_get_dir
+exp_dir_get_host
+exp_dir_get_netgroup
+exp_file_dir_from_uuid
+exp_file_deinit
+ng_file_parse
+ng_file_get_netgroup
+ng_file_print
+ng_file_deinit
+xlator_api
diff --git a/xlators/nfs/server/src/nlm4.c b/xlators/nfs/server/src/nlm4.c
index 37fc9c37699..577e8543966 100644
--- a/xlators/nfs/server/src/nlm4.c
+++ b/xlators/nfs/server/src/nlm4.c
@@ -8,20 +8,15 @@
   cases as published by the Free Software Foundation.
 */
 
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
-
-#include "defaults.h"
+#include <glusterfs/defaults.h>
 #include "rpcsvc.h"
-#include "dict.h"
-#include "xlator.h"
+#include <glusterfs/dict.h>
+#include <glusterfs/xlator.h>
 #include "nfs.h"
-#include "mem-pool.h"
-#include "logging.h"
+#include <glusterfs/mem-pool.h>
+#include <glusterfs/logging.h>
+#include <glusterfs/syscall.h>
 #include "nfs-fops.h"
-#include "inode.h"
 #include "mount3.h"
 #include "nfs3.h"
 #include "nfs-mem-types.h"
@@ -33,27 +28,28 @@
 #include "nfs-generics.h"
 #include "rpc-clnt.h"
 #include "nsm-xdr.h"
-#include "run.h"
+#include <glusterfs/run.h>
+#include "nfs-messages.h"
 #include <unistd.h>
 #include <rpc/pmap_clnt.h>
 #include <rpc/rpc.h>
 #include <rpc/xdr.h>
-#include <statedump.h>
+#include <glusterfs/statedump.h>
 
-#ifdef __NetBSD__
 #define KILLALL_CMD "pkill"
-#else
-#define KILLALL_CMD "killall"
-#endif
 
 /* TODO:
  * 1) 2 opens racing .. creating an fd leak.
- * 2) use mempool for nlmclnt - destroy if no fd exists, create during 1st call
+ * 2) use GF_REF_* for nlm_clnt_t
  */
 
-typedef ssize_t (*nlm4_serializer) (struct iovec outmsg, void *args);
+typedef ssize_t (*nlm4_serializer)(struct iovec outmsg, void *args);
+
+extern void
+nfs3_call_state_wipe(nfs3_call_state_t *cs);
 
-extern void nfs3_call_state_wipe (nfs3_call_state_t *cs);
+nfs3_call_state_t *
+nfs3_call_state_init(struct nfs3_state *s, rpcsvc_request_t *req, xlator_t *v);
 
 struct list_head nlm_client_list;
 gf_lock_t nlm_client_list_lk;
@@ -61,1909 +57,2047 @@ gf_lock_t nlm_client_list_lk;
 /* race on this is harmless */
 int nlm_grace_period = 50;
 
-#define nlm4_validate_nfs3_state(request, state, status, label, retval) \
-        do      {                                                       \
-                state = rpcsvc_request_program_private (request);       \
-                if (!state) {                                           \
-                        gf_log (GF_NLM, GF_LOG_ERROR, "NFSv3 state "    \
-                                "missing from RPC request");            \
-                        rpcsvc_request_seterr (req, SYSTEM_ERR);        \
-                        status = nlm4_failed;                           \
-                        goto label;                                     \
-                }                                                       \
-        } while (0);                                                    \
-
-nfs3_call_state_t *
-nfs3_call_state_init (struct nfs3_state *s, rpcsvc_request_t *req, xlator_t *v);
-
-#define nlm4_handle_call_state_init(nfs3state, calls, rq, opstat, errlabel)\
-        do {                                                            \
-                calls = nlm4_call_state_init ((nfs3state), (rq));       \
-                if (!calls) {                                           \
-                        gf_log (GF_NLM, GF_LOG_ERROR, "Failed to "      \
-                                "init call state");                     \
-                        opstat = nlm4_failed;                           \
-                        rpcsvc_request_seterr (req, SYSTEM_ERR);        \
-                        goto errlabel;                                  \
-                }                                                       \
-        } while (0)                                                     \
-
-#define nlm4_validate_gluster_fh(handle, status, errlabel)              \
-        do {                                                            \
-                if (!nfs3_fh_validate (handle)) {                       \
-                        status = nlm4_stale_fh;                         \
-                        goto errlabel;                                  \
-                }                                                       \
-        } while (0)                                                     \
+static gf_boolean_t nlm4_inited = _gf_false;
+
+#define nlm4_validate_nfs3_state(request, state, status, label, retval)        \
+    do {                                                                       \
+        state = rpcsvc_request_program_private(request);                       \
+        if (!state) {                                                          \
+            gf_msg(GF_NLM, GF_LOG_ERROR, errno, NFS_MSG_STATE_MISSING,         \
+                   "NFSv3 state "                                              \
+                   "missing from RPC request");                                \
+            rpcsvc_request_seterr(req, SYSTEM_ERR);                            \
+            status = nlm4_failed;                                              \
+            goto label;                                                        \
+        }                                                                      \
+    } while (0);
+
+#define nlm4_handle_call_state_init(nfs3state, calls, rq, opstat, errlabel)    \
+    do {                                                                       \
+        calls = nlm4_call_state_init((nfs3state), (rq));                       \
+        if (!calls) {                                                          \
+            gf_msg(GF_NLM, GF_LOG_ERROR, errno, NFS_MSG_INIT_CALL_STAT_FAIL,   \
+                   "Failed to "                                                \
+                   "init call state");                                         \
+            opstat = nlm4_failed;                                              \
+            rpcsvc_request_seterr(req, SYSTEM_ERR);                            \
+            goto errlabel;                                                     \
+        }                                                                      \
+    } while (0)
+
+#define nlm4_validate_gluster_fh(handle, status, errlabel)                     \
+    do {                                                                       \
+        if (!nfs3_fh_validate(handle)) {                                       \
+            status = nlm4_stale_fh;                                            \
+            goto errlabel;                                                     \
+        }                                                                      \
+    } while (0)
 
 xlator_t *
-nfs3_fh_to_xlator (struct nfs3_state *nfs3, struct nfs3_fh *fh);
-
-#define nlm4_map_fh_to_volume(nfs3state, handle, req, volume, status, label) \
-        do {                                                            \
-                char exportid[256], gfid[256];                          \
-                rpc_transport_t *trans = NULL;                          \
-                volume = nfs3_fh_to_xlator ((nfs3state), &handle);      \
-                if (!volume) {                                          \
-                        gf_uuid_unparse (handle.exportid, exportid);       \
-                        gf_uuid_unparse (handle.gfid, gfid);               \
-                        trans = rpcsvc_request_transport (req);         \
-                        gf_log (GF_NLM, GF_LOG_ERROR, "Failed to map "  \
-                                "FH to vol: client=%s, exportid=%s, gfid=%s",\
-                                trans->peerinfo.identifier, exportid,   \
-                                gfid);                                  \
-                        gf_log (GF_NLM, GF_LOG_ERROR,                   \
-                                "Stale nfs client %s must be trying to "\
-                                "connect to a deleted volume, please "  \
-                                "unmount it.", trans->peerinfo.identifier);\
-                        status = nlm4_stale_fh;                         \
-                        goto label;                                     \
-                } else {                                                \
-                        gf_log (GF_NLM, GF_LOG_TRACE, "FH to Volume: %s"\
-                                ,volume->name);                         \
-                        rpcsvc_request_set_private (req, volume);       \
-                }                                                       \
-        } while (0);                                                    \
-
-#define nlm4_volume_started_check(nfs3state, vlm, rtval, erlbl)         \
-        do {                                                            \
-              if ((!nfs_subvolume_started (nfs_state (nfs3state->nfsx), vlm))){\
-                      gf_log (GF_NLM, GF_LOG_ERROR, "Volume is disabled: %s",\
-                              vlm->name);                               \
-                      rtval = RPCSVC_ACTOR_IGNORE;                      \
-                      goto erlbl;                                       \
-              }                                                         \
-        } while (0)                                                     \
-
-#define nlm4_check_fh_resolve_status(cst, nfstat, erlabl)               \
-        do {                                                            \
-                xlator_t *xlatorp = NULL;                               \
-                char buf[256], gfid[256];                               \
-                rpc_transport_t *trans = NULL;                          \
-                if ((cst)->resolve_ret < 0) {                           \
-                        trans = rpcsvc_request_transport (cst->req);    \
-                        xlatorp = nfs3_fh_to_xlator (cst->nfs3state,    \
-                                                     &cst->resolvefh);  \
-                        gf_uuid_unparse (cst->resolvefh.gfid, gfid);       \
-                        snprintf (buf, sizeof (buf), "(%s) %s : %s",             \
-                                  trans->peerinfo.identifier,           \
-                                  xlatorp ? xlatorp->name : "ERR",      \
-                                  gfid);                                \
-                        gf_log (GF_NLM, GF_LOG_ERROR, "Unable to resolve FH"\
-                                ": %s", buf);                           \
-                        nfstat = nlm4_errno_to_nlm4stat (cst->resolve_errno);\
-                        goto erlabl;                                    \
-                }                                                       \
-        } while (0)                                                     \
-
+nfs3_fh_to_xlator(struct nfs3_state *nfs3, struct nfs3_fh *fh);
+
+#define nlm4_map_fh_to_volume(nfs3state, handle, req, volume, status, label)   \
+    do {                                                                       \
+        char exportid[256], gfid[256];                                         \
+        rpc_transport_t *trans = NULL;                                         \
+        volume = nfs3_fh_to_xlator((nfs3state), &handle);                      \
+        if (!volume) {                                                         \
+            gf_uuid_unparse(handle.exportid, exportid);                        \
+            gf_uuid_unparse(handle.gfid, gfid);                                \
+            trans = rpcsvc_request_transport(req);                             \
+            gf_msg(GF_NLM, GF_LOG_ERROR, errno, NFS_MSG_FH_TO_VOL_FAIL,        \
+                   "Failed to map "                                            \
+                   "FH to vol: client=%s, exportid=%s, gfid=%s",               \
+                   trans->peerinfo.identifier, exportid, gfid);                \
+            gf_msg(GF_NLM, GF_LOG_ERROR, errno, NFS_MSG_VOLUME_ERROR,          \
+                   "Stale nfs client %s must be trying to "                    \
+                   "connect to a deleted volume, please "                      \
+                   "unmount it.",                                              \
+                   trans->peerinfo.identifier);                                \
+            status = nlm4_stale_fh;                                            \
+            goto label;                                                        \
+        } else {                                                               \
+            gf_msg_trace(GF_NLM, 0, "FH to Volume: %s", volume->name);         \
+            rpcsvc_request_set_private(req, volume);                           \
+        }                                                                      \
+    } while (0);
+
+#define nlm4_volume_started_check(nfs3state, vlm, rtval, erlbl)                \
+    do {                                                                       \
+        if ((!nfs_subvolume_started(nfs_state(nfs3state->nfsx), vlm))) {       \
+            gf_msg(GF_NLM, GF_LOG_ERROR, 0, NFS_MSG_VOL_DISABLE,               \
+                   "Volume is disabled: %s", vlm->name);                       \
+            rtval = RPCSVC_ACTOR_IGNORE;                                       \
+            goto erlbl;                                                        \
+        }                                                                      \
+    } while (0)
+
+#define nlm4_check_fh_resolve_status(cst, nfstat, erlabl)                      \
+    do {                                                                       \
+        xlator_t *xlatorp = NULL;                                              \
+        char buf[256], gfid[GF_UUID_BUF_SIZE];                                 \
+        rpc_transport_t *trans = NULL;                                         \
+        if ((cst)->resolve_ret < 0) {                                          \
+            trans = rpcsvc_request_transport(cst->req);                        \
+            xlatorp = nfs3_fh_to_xlator(cst->nfs3state, &cst->resolvefh);      \
+            gf_uuid_unparse(cst->resolvefh.gfid, gfid);                        \
+            snprintf(buf, sizeof(buf), "(%s) %s : %s",                         \
+                     trans->peerinfo.identifier,                               \
+                     xlatorp ? xlatorp->name : "ERR", gfid);                   \
+            gf_msg(GF_NLM, GF_LOG_ERROR, 0, NFS_MSG_RESOLVE_FH_FAIL,           \
+                   "Unable to resolve FH"                                      \
+                   ": %s",                                                     \
+                   buf);                                                       \
+            nfstat = nlm4_errno_to_nlm4stat(cst->resolve_errno);               \
+            goto erlabl;                                                       \
+        }                                                                      \
+    } while (0)
 
 void
-nlm4_prep_nlm4_testargs (nlm4_testargs *args, struct nfs3_fh *fh,
-                         nlm4_lkowner_t *oh, char *cookiebytes)
+nlm4_prep_nlm4_testargs(nlm4_testargs *args, struct nfs3_fh *fh,
+                        nlm4_lkowner_t *oh, char *cookiebytes)
 {
-        memset (args, 0, sizeof (*args));
-        args->alock.fh.nlm4_netobj_val = (void *)fh;
-        args->alock.oh.nlm4_netobj_val = (void *)oh;
-        args->cookie.nlm4_netobj_val = (void *)cookiebytes;
+    memset(args, 0, sizeof(*args));
+    args->alock.fh.nlm4_netobj_val = (void *)fh;
+    args->alock.oh.nlm4_netobj_val = (void *)oh;
+    args->cookie.nlm4_netobj_val = (void *)cookiebytes;
 }
 
 void
-nlm4_prep_nlm4_lockargs (nlm4_lockargs *args, struct nfs3_fh *fh,
-                         nlm4_lkowner_t *oh, char *cookiebytes)
+nlm4_prep_nlm4_lockargs(nlm4_lockargs *args, struct nfs3_fh *fh,
+                        nlm4_lkowner_t *oh, char *cookiebytes)
 {
-        memset (args, 0, sizeof (*args));
-        args->alock.fh.nlm4_netobj_val = (void *)fh;
-        args->alock.oh.nlm4_netobj_val = (void *)oh;
-        args->cookie.nlm4_netobj_val = (void *)cookiebytes;
+    memset(args, 0, sizeof(*args));
+    args->alock.fh.nlm4_netobj_val = (void *)fh;
+    args->alock.oh.nlm4_netobj_val = (void *)oh;
+    args->cookie.nlm4_netobj_val = (void *)cookiebytes;
 }
 
 void
-nlm4_prep_nlm4_cancargs (nlm4_cancargs *args, struct nfs3_fh *fh,
-                           nlm4_lkowner_t *oh, char *cookiebytes)
+nlm4_prep_nlm4_cancargs(nlm4_cancargs *args, struct nfs3_fh *fh,
+                        nlm4_lkowner_t *oh, char *cookiebytes)
 {
-        memset (args, 0, sizeof (*args));
-        args->alock.fh.nlm4_netobj_val = (void *)fh;
-        args->alock.oh.nlm4_netobj_val = (void *)oh;
-        args->cookie.nlm4_netobj_val = (void *)cookiebytes;
+    memset(args, 0, sizeof(*args));
+    args->alock.fh.nlm4_netobj_val = (void *)fh;
+    args->alock.oh.nlm4_netobj_val = (void *)oh;
+    args->cookie.nlm4_netobj_val = (void *)cookiebytes;
 }
 
 void
-nlm4_prep_nlm4_unlockargs (nlm4_unlockargs *args, struct nfs3_fh *fh,
-                           nlm4_lkowner_t *oh, char *cookiebytes)
+nlm4_prep_nlm4_unlockargs(nlm4_unlockargs *args, struct nfs3_fh *fh,
+                          nlm4_lkowner_t *oh, char *cookiebytes)
 {
-        memset (args, 0, sizeof (*args));
-        args->alock.fh.nlm4_netobj_val = (void *)fh;
-        args->alock.oh.nlm4_netobj_val = (void *)oh;
-        args->cookie.nlm4_netobj_val = (void *)cookiebytes;
+    memset(args, 0, sizeof(*args));
+    args->alock.fh.nlm4_netobj_val = (void *)fh;
+    args->alock.oh.nlm4_netobj_val = (void *)oh;
+    args->cookie.nlm4_netobj_val = (void *)cookiebytes;
 }
 
 void
-nlm4_prep_shareargs (nlm4_shareargs *args, struct nfs3_fh *fh,
-                     nlm4_lkowner_t *oh, char *cookiebytes)
+nlm4_prep_shareargs(nlm4_shareargs *args, struct nfs3_fh *fh,
+                    nlm4_lkowner_t *oh, char *cookiebytes)
 {
-        memset (args, 0, sizeof (*args));
-        args->share.fh.nlm4_netobj_val = (void *)fh;
-        args->share.oh.nlm4_netobj_val = (void *)oh;
-        args->cookie.nlm4_netobj_val = (void *)cookiebytes;
+    memset(args, 0, sizeof(*args));
+    args->share.fh.nlm4_netobj_val = (void *)fh;
+    args->share.oh.nlm4_netobj_val = (void *)oh;
+    args->cookie.nlm4_netobj_val = (void *)cookiebytes;
 }
 
 void
-nlm4_prep_freeallargs (nlm4_freeallargs *args, nlm4_lkowner_t *oh)
+nlm4_prep_freeallargs(nlm4_freeallargs *args, nlm4_lkowner_t *oh)
 {
-        memset (args, 0, sizeof (*args));
-        args->name = (void *)oh;
+    memset(args, 0, sizeof(*args));
+    args->name = (void *)oh;
 }
 
 void
-nlm_copy_lkowner (gf_lkowner_t *dst, nlm4_netobj *src)
+nlm_copy_lkowner(gf_lkowner_t *dst, nlm4_netobj *src)
 {
-        dst->len = src->nlm4_netobj_len;
-        memcpy (dst->data, src->nlm4_netobj_val, dst->len);
+    dst->len = src->nlm4_netobj_len;
+    memcpy(dst->data, src->nlm4_netobj_val, dst->len);
 }
 
 int
-nlm_is_oh_same_lkowner (gf_lkowner_t *a, nlm4_netobj *b)
+nlm_is_oh_same_lkowner(gf_lkowner_t *a, nlm4_netobj *b)
 {
-        if (!a || !b) {
-                gf_log (GF_NLM, GF_LOG_ERROR, "invalid args");
-                return -1;
-        }
+    if (!a || !b) {
+        gf_msg(GF_NLM, GF_LOG_ERROR, EINVAL, NFS_MSG_INVALID_ENTRY,
+               "invalid args");
+        return -1;
+    }
 
-        return (a->len == b->nlm4_netobj_len &&
-                !memcmp (a->data, b->nlm4_netobj_val, a->len));
+    return (a->len == b->nlm4_netobj_len &&
+            !memcmp(a->data, b->nlm4_netobj_val, a->len));
 }
 
 nlm4_stats
-nlm4_errno_to_nlm4stat (int errnum)
+nlm4_errno_to_nlm4stat(int errnum)
 {
-        nlm4_stats        stat = nlm4_denied;
+    nlm4_stats stat = nlm4_denied;
 
-        switch (errnum) {
+    switch (errnum) {
         case 0:
-                stat = nlm4_granted;
-                break;
+            stat = nlm4_granted;
+            break;
         case EROFS:
-                stat = nlm4_rofs;
-                break;
+            stat = nlm4_rofs;
+            break;
         case ESTALE:
-                stat = nlm4_stale_fh;
-                break;
+            stat = nlm4_stale_fh;
+            break;
         case ENOLCK:
-                stat = nlm4_failed;
-                break;
+            stat = nlm4_failed;
+            break;
         default:
-                stat = nlm4_denied;
-                break;
-        }
+            stat = nlm4_denied;
+            break;
+    }
 
-        return stat;
+    return stat;
 }
 
 nfs3_call_state_t *
-nlm4_call_state_init (struct nfs3_state *s, rpcsvc_request_t *req)
+nlm4_call_state_init(struct nfs3_state *s, rpcsvc_request_t *req)
 {
-        nfs3_call_state_t       *cs = NULL;
+    nfs3_call_state_t *cs = NULL;
 
-        if ((!s) || (!req))
-                return NULL;
+    if ((!s) || (!req))
+        return NULL;
 
-        cs = (nfs3_call_state_t *) mem_get (s->localpool);
-        if (!cs)
-                return NULL;
+    cs = nfs3_call_state_init(s, req, NULL);
+    if (!cs)
+        return NULL;
 
-        memset (cs, 0, sizeof (*cs));
-        INIT_LIST_HEAD (&cs->entries.list);
-        INIT_LIST_HEAD (&cs->openwait_q);
-        cs->operrno = EINVAL;
-        cs->req = req;
-        cs->nfsx = s->nfsx;
-        cs->nfs3state = s;
-        cs->monitor = 1;
+    cs->monitor = 1;
 
-        return cs;
+    return cs;
 }
 
 int
-nlm_monitor (char *caller_name)
-{
-        nlm_client_t *nlmclnt = NULL;
-        int monitor = -1;
-
-        LOCK (&nlm_client_list_lk);
-        list_for_each_entry (nlmclnt, &nlm_client_list, nlm_clients) {
-                if (!strcmp(caller_name, nlmclnt->caller_name)) {
-                        monitor = nlmclnt->nsm_monitor;
-                        nlmclnt->nsm_monitor = 1;
-                        break;
-                }
+nlm_monitor(char *caller_name)
+{
+    nlm_client_t *nlmclnt = NULL;
+    int monitor = -1;
+
+    LOCK(&nlm_client_list_lk);
+    list_for_each_entry(nlmclnt, &nlm_client_list, nlm_clients)
+    {
+        if (!strcmp(caller_name, nlmclnt->caller_name)) {
+            monitor = nlmclnt->nsm_monitor;
+            nlmclnt->nsm_monitor = 1;
+            break;
         }
-        UNLOCK (&nlm_client_list_lk);
+    }
+    UNLOCK(&nlm_client_list_lk);
 
-        if (monitor == -1)
-                gf_log (GF_NLM, GF_LOG_ERROR, "%s was not found in "
-                        "the nlmclnt list", caller_name);
+    if (monitor == -1)
+        gf_msg(GF_NLM, GF_LOG_ERROR, 0, NFS_MSG_CALLER_NOT_FOUND,
+               "%s was not found in the nlmclnt list", caller_name);
 
-        return monitor;
+    return monitor;
 }
 
 rpc_clnt_t *
-nlm_get_rpc_clnt (char *caller_name)
-{
-        nlm_client_t *nlmclnt = NULL;
-        int nlmclnt_found = 0;
-        rpc_clnt_t *rpc_clnt = NULL;
-
-        LOCK (&nlm_client_list_lk);
-        list_for_each_entry (nlmclnt, &nlm_client_list, nlm_clients) {
-                if (!strcmp(caller_name, nlmclnt->caller_name)) {
-                        nlmclnt_found = 1;
-                        break;
-                }
-        }
-        if (!nlmclnt_found)
-                goto ret;
-        if (nlmclnt->rpc_clnt)
-                rpc_clnt = rpc_clnt_ref (nlmclnt->rpc_clnt);
+nlm_get_rpc_clnt(char *caller_name)
+{
+    nlm_client_t *nlmclnt = NULL;
+    int nlmclnt_found = 0;
+    rpc_clnt_t *rpc_clnt = NULL;
+
+    LOCK(&nlm_client_list_lk);
+    list_for_each_entry(nlmclnt, &nlm_client_list, nlm_clients)
+    {
+        if (!strcmp(caller_name, nlmclnt->caller_name)) {
+            nlmclnt_found = 1;
+            break;
+        }
+    }
+    if (!nlmclnt_found)
+        goto ret;
+    if (nlmclnt->rpc_clnt)
+        rpc_clnt = rpc_clnt_ref(nlmclnt->rpc_clnt);
 ret:
-        UNLOCK (&nlm_client_list_lk);
-        return rpc_clnt;
+    UNLOCK(&nlm_client_list_lk);
+    return rpc_clnt;
 }
 
-int
-nlm_set_rpc_clnt (rpc_clnt_t *rpc_clnt, char *caller_name)
-{
-        nlm_client_t *nlmclnt = NULL;
-        int nlmclnt_found = 0;
-        int ret = -1;
-
-        LOCK (&nlm_client_list_lk);
-        list_for_each_entry (nlmclnt, &nlm_client_list, nlm_clients) {
-                if (!strcmp(caller_name, nlmclnt->caller_name)) {
-                        nlmclnt_found = 1;
-                        break;
-                }
-        }
+static void
+nlm_client_free(nlm_client_t *nlmclnt)
+{
+    nlm_fde_t *fde = NULL, *tmp = NULL;
 
-        if (!nlmclnt_found) {
-                nlmclnt = GF_CALLOC (1, sizeof(*nlmclnt),
-                                     gf_nfs_mt_nlm4_nlmclnt);
-                if (nlmclnt == NULL)
-                        goto ret;
+    gf_msg_trace(GF_NLM, 0, "removing nlm-client %s from the list",
+                 nlmclnt->caller_name);
 
-                INIT_LIST_HEAD(&nlmclnt->fdes);
-                INIT_LIST_HEAD(&nlmclnt->nlm_clients);
-                INIT_LIST_HEAD(&nlmclnt->shares);
+    list_for_each_entry_safe(fde, tmp, &nlmclnt->fdes, fde_list)
+    {
+        fd_unref(fde->fd);
+        list_del(&fde->fde_list);
+        GF_FREE(fde);
+    }
 
-                list_add (&nlmclnt->nlm_clients, &nlm_client_list);
-                nlmclnt->caller_name = gf_strdup (caller_name);
-        }
+    list_del(&nlmclnt->fdes);
+    list_del(&nlmclnt->nlm_clients);
+    list_del(&nlmclnt->shares);
 
-        if (nlmclnt->rpc_clnt == NULL) {
-                nlmclnt->rpc_clnt = rpc_clnt_ref (rpc_clnt);
-        }
-        ret = 0;
-ret:
-        UNLOCK (&nlm_client_list_lk);
-        return ret;
-}
+    GF_FREE(nlmclnt->caller_name);
 
-int
-nlm_unset_rpc_clnt (rpc_clnt_t *rpc)
-{
-        nlm_client_t *nlmclnt = NULL;
-        rpc_clnt_t *rpc_clnt = NULL;
-
-        LOCK (&nlm_client_list_lk);
-        list_for_each_entry (nlmclnt, &nlm_client_list, nlm_clients) {
-                if (rpc == nlmclnt->rpc_clnt) {
-                        rpc_clnt = nlmclnt->rpc_clnt;
-                        nlmclnt->rpc_clnt = NULL;
-                        break;
-                }
-        }
-        UNLOCK (&nlm_client_list_lk);
-        if (rpc_clnt == NULL) {
-                return -1;
-        }
-        if (rpc_clnt) {
-                /* cleanup the saved-frames before last unref */
-                rpc_clnt_connection_cleanup (&rpc_clnt->conn);
+    if (nlmclnt->rpc_clnt) {
+        /* cleanup the saved-frames before last unref */
+        rpc_clnt_connection_cleanup(&nlmclnt->rpc_clnt->conn);
+        /* rpc_clnt_connection_cleanup() calls rpc_clnt_unref() */
+    }
 
-                rpc_clnt_unref (rpc_clnt);
-        }
-        return 0;
+    GF_FREE(nlmclnt);
 }
 
 int
-nlm_add_nlmclnt (char *caller_name)
-{
-        nlm_client_t *nlmclnt = NULL;
-        int nlmclnt_found = 0;
-        int ret = -1;
-
-        LOCK (&nlm_client_list_lk);
-        list_for_each_entry (nlmclnt, &nlm_client_list, nlm_clients) {
-                if (!strcmp(caller_name, nlmclnt->caller_name)) {
-                        nlmclnt_found = 1;
-                        break;
-                }
-        }
-        if (!nlmclnt_found) {
-                nlmclnt = GF_CALLOC (1, sizeof(*nlmclnt),
-                                     gf_nfs_mt_nlm4_nlmclnt);
-                if (nlmclnt == NULL) {
-                        gf_log (GF_NLM, GF_LOG_DEBUG, "malloc error");
-                        goto ret;
-                }
-
-                INIT_LIST_HEAD(&nlmclnt->fdes);
-                INIT_LIST_HEAD(&nlmclnt->nlm_clients);
-                INIT_LIST_HEAD(&nlmclnt->shares);
-
-                list_add (&nlmclnt->nlm_clients, &nlm_client_list);
-                nlmclnt->caller_name = gf_strdup (caller_name);
-        }
-        ret = 0;
+nlm_set_rpc_clnt(rpc_clnt_t *rpc_clnt, char *caller_name)
+{
+    nlm_client_t *nlmclnt = NULL;
+    int nlmclnt_found = 0;
+    int ret = -1;
+
+    LOCK(&nlm_client_list_lk);
+    list_for_each_entry(nlmclnt, &nlm_client_list, nlm_clients)
+    {
+        if (!strcmp(caller_name, nlmclnt->caller_name)) {
+            nlmclnt_found = 1;
+            break;
+        }
+    }
+
+    if (!nlmclnt_found) {
+        nlmclnt = GF_CALLOC(1, sizeof(*nlmclnt), gf_nfs_mt_nlm4_nlmclnt);
+        if (nlmclnt == NULL)
+            goto ret;
+
+        INIT_LIST_HEAD(&nlmclnt->fdes);
+        INIT_LIST_HEAD(&nlmclnt->nlm_clients);
+        INIT_LIST_HEAD(&nlmclnt->shares);
+
+        list_add(&nlmclnt->nlm_clients, &nlm_client_list);
+        nlmclnt->caller_name = gf_strdup(caller_name);
+    }
+
+    if (nlmclnt->rpc_clnt == NULL) {
+        nlmclnt->rpc_clnt = rpc_clnt_ref(rpc_clnt);
+    }
+    ret = 0;
 ret:
-        UNLOCK (&nlm_client_list_lk);
-        return ret;
+    UNLOCK(&nlm_client_list_lk);
+    return ret;
 }
 
 int
-nlm4svc_submit_reply (rpcsvc_request_t *req, void *arg, nlm4_serializer sfunc)
-{
-        struct iovec            outmsg  = {0, };
-        struct iobuf            *iob    = NULL;
-        struct nfs3_state       *nfs3   = NULL;
-        int                     ret     = -1;
-        ssize_t                 msglen  = 0;
-        struct iobref           *iobref = NULL;
-
-        if (!req)
-                return -1;
-
-        nfs3 = (struct nfs3_state *)rpcsvc_request_program_private (req);
-        if (!nfs3) {
-                gf_log (GF_NLM, GF_LOG_ERROR, "mount state not found");
-                goto ret;
-        }
+nlm_unset_rpc_clnt(rpc_clnt_t *rpc)
+{
+    nlm_client_t *nlmclnt = NULL;
 
-        /* First, get the io buffer into which the reply in arg will
-         * be serialized.
-         */
-        iob = iobuf_get (nfs3->iobpool);
-        if (!iob) {
-                gf_log (GF_NLM, GF_LOG_ERROR, "Failed to get iobuf");
-                goto ret;
+    LOCK(&nlm_client_list_lk);
+    list_for_each_entry(nlmclnt, &nlm_client_list, nlm_clients)
+    {
+        if (rpc == nlmclnt->rpc_clnt) {
+            nlm_client_free(nlmclnt);
+            break;
         }
+    }
+    UNLOCK(&nlm_client_list_lk);
 
-        iobuf_to_iovec (iob, &outmsg);
-        /* Use the given serializer to translate the give C structure in arg
-         * to XDR format which will be written into the buffer in outmsg.
-         */
-        msglen = sfunc (outmsg, arg);
-        if (msglen < 0) {
-                gf_log (GF_NLM, GF_LOG_ERROR, "Failed to encode message");
-                goto ret;
-        }
-        outmsg.iov_len = msglen;
+    return 0;
+}
 
-        iobref = iobref_new ();
-        if (iobref == NULL) {
-                gf_log (GF_NLM, GF_LOG_ERROR, "Failed to get iobref");
-                goto ret;
+int
+nlm_add_nlmclnt(char *caller_name)
+{
+    nlm_client_t *nlmclnt = NULL;
+    int nlmclnt_found = 0;
+    int ret = -1;
+
+    LOCK(&nlm_client_list_lk);
+    list_for_each_entry(nlmclnt, &nlm_client_list, nlm_clients)
+    {
+        if (!strcmp(caller_name, nlmclnt->caller_name)) {
+            nlmclnt_found = 1;
+            break;
+        }
+    }
+    if (!nlmclnt_found) {
+        nlmclnt = GF_CALLOC(1, sizeof(*nlmclnt), gf_nfs_mt_nlm4_nlmclnt);
+        if (nlmclnt == NULL) {
+            gf_msg(GF_NLM, GF_LOG_ERROR, ENOMEM, NFS_MSG_NO_MEMORY,
+                   "malloc error");
+            goto ret;
         }
 
-        ret = iobref_add (iobref, iob);
-        if (ret) {
-                gf_log (GF_NLM, GF_LOG_ERROR, "Failed to add iob to iobref");
-                goto ret;
-        }
+        INIT_LIST_HEAD(&nlmclnt->fdes);
+        INIT_LIST_HEAD(&nlmclnt->nlm_clients);
+        INIT_LIST_HEAD(&nlmclnt->shares);
 
-        /* Then, submit the message for transmission. */
-        ret = rpcsvc_submit_message (req, &outmsg, 1, NULL, 0, iobref);
-        if (ret == -1) {
-                gf_log (GF_NLM, GF_LOG_ERROR, "Reply submission failed");
-                goto ret;
-        }
+        list_add(&nlmclnt->nlm_clients, &nlm_client_list);
+        nlmclnt->caller_name = gf_strdup(caller_name);
+    }
+    ret = 0;
+ret:
+    UNLOCK(&nlm_client_list_lk);
+    return ret;
+}
 
-        ret = 0;
+int
+nlm4svc_submit_reply(rpcsvc_request_t *req, void *arg, nlm4_serializer sfunc)
+{
+    struct iovec outmsg = {
+        0,
+    };
+    struct iobuf *iob = NULL;
+    struct nfs3_state *nfs3 = NULL;
+    int ret = -1;
+    ssize_t msglen = 0;
+    struct iobref *iobref = NULL;
+
+    if (!req)
+        return -1;
+
+    nfs3 = (struct nfs3_state *)rpcsvc_request_program_private(req);
+    if (!nfs3) {
+        gf_msg(GF_NLM, GF_LOG_ERROR, EINVAL, NFS_MSG_MNT_STATE_NOT_FOUND,
+               "mount state not found");
+        goto ret;
+    }
+
+    /* First, get the io buffer into which the reply in arg will
+     * be serialized.
+     */
+    iob = iobuf_get(nfs3->iobpool);
+    if (!iob) {
+        gf_msg(GF_NLM, GF_LOG_ERROR, ENOMEM, NFS_MSG_NO_MEMORY,
+               "Failed to get iobuf");
+        goto ret;
+    }
+
+    iobuf_to_iovec(iob, &outmsg);
+    /* Use the given serializer to translate the give C structure in arg
+     * to XDR format which will be written into the buffer in outmsg.
+     */
+    msglen = sfunc(outmsg, arg);
+    if (msglen < 0) {
+        gf_msg(GF_NLM, GF_LOG_ERROR, errno, NFS_MSG_ENCODE_MSG_FAIL,
+               "Failed to encode message");
+        goto ret;
+    }
+    outmsg.iov_len = msglen;
+
+    iobref = iobref_new();
+    if (iobref == NULL) {
+        gf_msg(GF_NLM, GF_LOG_ERROR, ENOMEM, NFS_MSG_NO_MEMORY,
+               "Failed to get iobref");
+        goto ret;
+    }
+
+    ret = iobref_add(iobref, iob);
+    if (ret) {
+        gf_msg(GF_NLM, GF_LOG_ERROR, ENOMEM, NFS_MSG_NO_MEMORY,
+               "Failed to add iob to iobref");
+        goto ret;
+    }
+
+    /* Then, submit the message for transmission. */
+    ret = rpcsvc_submit_message(req, &outmsg, 1, NULL, 0, iobref);
+    if (ret == -1) {
+        gf_msg(GF_NLM, GF_LOG_ERROR, errno, NFS_MSG_REP_SUBMIT_FAIL,
+               "Reply submission failed");
+        goto ret;
+    }
+
+    ret = 0;
 ret:
-        if (iob)
-                iobuf_unref (iob);
-        if (iobref)
-                iobref_unref (iobref);
+    if (iob)
+        iobuf_unref(iob);
+    if (iobref)
+        iobref_unref(iobref);
 
-        return ret;
+    return ret;
 }
 
-typedef int (*nlm4_resume_fn_t) (void *cs);
+typedef int (*nlm4_resume_fn_t)(void *cs);
 
 int32_t
-nlm4_file_open_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                    int32_t op_ret, int32_t op_errno, fd_t *fd, dict_t *xdata)
+nlm4_file_open_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                   int32_t op_ret, int32_t op_errno, fd_t *fd, dict_t *xdata)
 {
-        nfs3_call_state_t *cs = frame->local;
+    nfs3_call_state_t *cs = frame->local;
 
-        if (op_ret == 0)
-                fd_bind (cs->fd);
-        cs->resolve_ret = op_ret;
-        cs->resume_fn (cs);
-        frame->local = NULL;
-        STACK_DESTROY (frame->root);
-        return 0;
+    if (op_ret == 0)
+        fd_bind(cs->fd);
+    cs->resolve_ret = op_ret;
+    cs->resume_fn(cs);
+
+    frame->local = NULL;
+    STACK_DESTROY(frame->root);
+    GF_REF_PUT(cs);
+
+    return 0;
 }
 
 void *
 nsm_monitor(void *arg)
 {
-        CLIENT *clnt = NULL;
-        enum clnt_stat ret;
-        struct mon nsm_mon;
-        struct sm_stat_res res;
-        struct timeval tout = { 5, 0 };
-        char *host = NULL;
-
-        host = arg;
-        nsm_mon.mon_id.mon_name = gf_strdup(host);
-        nsm_mon.mon_id.my_id.my_name = gf_strdup("localhost");
-        nsm_mon.mon_id.my_id.my_prog = NLMCBK_PROGRAM;
-        nsm_mon.mon_id.my_id.my_vers = NLMCBK_V1;
-        nsm_mon.mon_id.my_id.my_proc = NLMCBK_SM_NOTIFY;
-        /* nothing to put in the private data */
+    CLIENT *clnt = NULL;
+    enum clnt_stat ret;
+    struct mon nsm_mon;
+    struct sm_stat_res res;
+    struct timeval tout = {5, 0};
+    char *host = NULL;
+
+    host = arg;
+    nsm_mon.mon_id.mon_name = gf_strdup(host);
+    nsm_mon.mon_id.my_id.my_name = gf_strdup("localhost");
+    nsm_mon.mon_id.my_id.my_prog = NLMCBK_PROGRAM;
+    nsm_mon.mon_id.my_id.my_vers = NLMCBK_V1;
+    nsm_mon.mon_id.my_id.my_proc = NLMCBK_SM_NOTIFY;
+    /* nothing to put in the private data */
 #define SM_PROG 100024
 #define SM_VERS 1
 #define SM_MON 2
 
-        /* create a connection to nsm on the localhost */
-        clnt = clnt_create("localhost", SM_PROG, SM_VERS, "tcp");
-        if(!clnt)
-        {
-                gf_log (GF_NLM, GF_LOG_ERROR, "%s",
-                        clnt_spcreateerror ("Clnt_create()"));
-                goto out;
-        }
-
-        ret = clnt_call(clnt, SM_MON,
-                        (xdrproc_t) xdr_mon, (caddr_t) & nsm_mon,
-                        (xdrproc_t) xdr_sm_stat_res, (caddr_t) & res, tout);
-        if(ret != RPC_SUCCESS)
-        {
-                gf_log (GF_NLM, GF_LOG_ERROR, "clnt_call(): %s",
-                        clnt_sperrno(ret));
-                goto out;
-        }
-        if(res.res_stat != STAT_SUCC)
-        {
-                gf_log (GF_NLM, GF_LOG_ERROR, "clnt_call(): %s",
-                        clnt_sperrno(ret));
-                goto out;
-        }
+    /* create a connection to nsm on the localhost */
+    clnt = clnt_create("localhost", SM_PROG, SM_VERS, "tcp");
+    if (!clnt) {
+        gf_msg(GF_NLM, GF_LOG_ERROR, 0, NFS_MSG_CLNT_CREATE_ERROR, "%s",
+               clnt_spcreateerror("Clnt_create()"));
+        goto out;
+    }
+
+    ret = clnt_call(clnt, SM_MON, (xdrproc_t)xdr_mon, (caddr_t)&nsm_mon,
+                    (xdrproc_t)xdr_sm_stat_res, (caddr_t)&res, tout);
+    if (ret != RPC_SUCCESS) {
+        gf_msg(GF_NLM, GF_LOG_ERROR, 0, NFS_MSG_CLNT_CALL_ERROR,
+               "clnt_call(): %s", clnt_sperrno(ret));
+        goto out;
+    }
+    if (res.res_stat != STAT_SUCC) {
+        gf_msg(GF_NLM, GF_LOG_ERROR, 0, NFS_MSG_CLNT_CALL_ERROR,
+               "clnt_call(): %s", clnt_sperrno(ret));
+        goto out;
+    }
 
 out:
-        GF_FREE(nsm_mon.mon_id.mon_name);
-        GF_FREE(nsm_mon.mon_id.my_id.my_name);
-        if (clnt != NULL)
-                clnt_destroy(clnt);
-        return NULL;
+    GF_FREE(nsm_mon.mon_id.mon_name);
+    GF_FREE(nsm_mon.mon_id.my_id.my_name);
+    if (clnt != NULL)
+        clnt_destroy(clnt);
+    return NULL;
 }
 
 nlm_client_t *
-__nlm_get_uniq (char *caller_name)
+__nlm_get_uniq(char *caller_name)
 {
-        nlm_client_t *nlmclnt = NULL;
+    nlm_client_t *nlmclnt = NULL;
 
-        if (!caller_name)
-                return NULL;
+    if (!caller_name)
+        return NULL;
 
-        list_for_each_entry (nlmclnt, &nlm_client_list, nlm_clients) {
-                if (!strcmp(caller_name, nlmclnt->caller_name))
-                        return nlmclnt;
-        }
+    list_for_each_entry(nlmclnt, &nlm_client_list, nlm_clients)
+    {
+        if (!strcmp(caller_name, nlmclnt->caller_name))
+            return nlmclnt;
+    }
 
-        return NULL;
+    return NULL;
 }
 
 nlm_client_t *
-nlm_get_uniq (char *caller_name)
+nlm_get_uniq(char *caller_name)
 {
-        nlm_client_t *nlmclnt = NULL;
+    nlm_client_t *nlmclnt = NULL;
 
-        LOCK (&nlm_client_list_lk);
-        nlmclnt = __nlm_get_uniq (caller_name);
-        UNLOCK (&nlm_client_list_lk);
+    LOCK(&nlm_client_list_lk);
+    nlmclnt = __nlm_get_uniq(caller_name);
+    UNLOCK(&nlm_client_list_lk);
 
-        return nlmclnt;
+    return nlmclnt;
 }
 
-
 int
 nlm4_file_open_and_resume(nfs3_call_state_t *cs, nlm4_resume_fn_t resume)
 {
-        fd_t *fd = NULL;
-        int ret = -1;
-        int flags = 0;
-        nlm_client_t *nlmclnt = NULL;
-        call_frame_t *frame = NULL;
-
-        if (cs->args.nlm4_lockargs.exclusive == _gf_false)
-                flags = O_RDONLY;
-        else
-                flags = O_WRONLY;
-
-        nlmclnt = nlm_get_uniq (cs->args.nlm4_lockargs.alock.caller_name);
-        if (nlmclnt == NULL) {
-                gf_log (GF_NLM, GF_LOG_ERROR, "nlm_get_uniq() returned NULL");
-                ret = -ENOLCK;
-                goto err;
-        }
-        cs->resume_fn = resume;
-        fd = fd_lookup_uint64 (cs->resolvedloc.inode, (uint64_t)nlmclnt);
-        if (fd) {
-                cs->fd = fd;
-                cs->resolve_ret = 0;
-                cs->resume_fn(cs);
-                ret = 0;
-                goto err;
-        }
-
-        fd = fd_create_uint64 (cs->resolvedloc.inode, (uint64_t)nlmclnt);
-        if (fd == NULL) {
-                gf_log (GF_NLM, GF_LOG_ERROR, "fd_create_uint64() returned NULL");
-                ret = -ENOLCK;
-                goto err;
-        }
-
+    fd_t *fd = NULL;
+    int ret = -1;
+    int flags = 0;
+    nlm_client_t *nlmclnt = NULL;
+    call_frame_t *frame = NULL;
+
+    if (cs->args.nlm4_lockargs.exclusive == _gf_false)
+        flags = O_RDONLY;
+    else
+        flags = O_WRONLY;
+
+    nlmclnt = nlm_get_uniq(cs->args.nlm4_lockargs.alock.caller_name);
+    if (nlmclnt == NULL) {
+        gf_msg(GF_NLM, GF_LOG_ERROR, ENOLCK, NFS_MSG_NO_MEMORY,
+               "nlm_get_uniq() "
+               "returned NULL");
+        ret = -ENOLCK;
+        goto err;
+    }
+    cs->resume_fn = resume;
+    fd = fd_lookup_uint64(cs->resolvedloc.inode, (uint64_t)(uintptr_t)nlmclnt);
+    if (fd) {
         cs->fd = fd;
-
-        frame = create_frame (cs->nfsx, cs->nfsx->ctx->pool);
-        if (!frame) {
-                gf_log (GF_NLM, GF_LOG_ERROR, "unable to create frame");
-                ret = -ENOMEM;
-                goto err;
-        }
-
-        frame->root->pid = NFS_PID;
-        frame->root->uid = rpcsvc_request_uid (cs->req);
-        frame->root->gid = rpcsvc_request_gid (cs->req);
-        frame->local = cs;
-        nfs_fix_groups (cs->nfsx, frame->root);
-
-        STACK_WIND_COOKIE (frame, nlm4_file_open_cbk, cs->vol, cs->vol,
-                          cs->vol->fops->open, &cs->resolvedloc, flags,
-                          cs->fd, NULL);
+        cs->resolve_ret = 0;
+        cs->resume_fn(cs);
         ret = 0;
+        goto err;
+    }
+
+    fd = fd_create_uint64(cs->resolvedloc.inode, (uint64_t)(uintptr_t)nlmclnt);
+    if (fd == NULL) {
+        gf_msg(GF_NLM, GF_LOG_ERROR, ENOLCK, NFS_MSG_NO_MEMORY,
+               "fd_create_uint64() returned NULL");
+        ret = -ENOLCK;
+        goto err;
+    }
+
+    cs->fd = fd;
+
+    frame = create_frame(cs->nfsx, cs->nfsx->ctx->pool);
+    if (!frame) {
+        gf_msg(GF_NLM, GF_LOG_ERROR, ENOMEM, NFS_MSG_NO_MEMORY,
+               "unable to create frame");
+        ret = -ENOMEM;
+        goto err;
+    }
+
+    frame->root->pid = NFS_PID;
+    frame->root->uid = rpcsvc_request_uid(cs->req);
+    frame->root->gid = rpcsvc_request_gid(cs->req);
+    frame->local = GF_REF_GET(cs);
+    nfs_fix_groups(cs->nfsx, frame->root);
+
+    STACK_WIND_COOKIE(frame, nlm4_file_open_cbk, cs->vol, cs->vol,
+                      cs->vol->fops->open, &cs->resolvedloc, flags, cs->fd,
+                      NULL);
+    ret = 0;
 err:
-        return ret;
+    return ret;
 }
 
 int
-nlm4_generic_reply (rpcsvc_request_t *req, nlm4_netobj cookie, nlm4_stats stat)
+nlm4_generic_reply(rpcsvc_request_t *req, nlm4_netobj cookie, nlm4_stats stat)
 {
-        nlm4_res res;
+    nlm4_res res;
 
-        memset (&res, 0, sizeof (res));
-        res.cookie = cookie;
-        res.stat.stat = stat;
+    memset(&res, 0, sizeof(res));
+    res.cookie = cookie;
+    res.stat.stat = stat;
 
-        nlm4svc_submit_reply (req, (void *)&res,
-                              (nlm4_serializer)xdr_serialize_nlm4_res);
-        return 0;
+    nlm4svc_submit_reply(req, (void *)&res,
+                         (nlm4_serializer)xdr_serialize_nlm4_res);
+    return 0;
 }
 
 int
-nlm4svc_null (rpcsvc_request_t *req)
+nlm4svc_null(rpcsvc_request_t *req)
 {
-        struct iovec    dummyvec = {0, };
+    struct iovec dummyvec = {
+        0,
+    };
 
-        if (!req) {
-                gf_log (GF_NLM, GF_LOG_ERROR, "Got NULL request!");
-                return 0;
-        }
-        rpcsvc_submit_generic (req, &dummyvec, 1,  NULL, 0, NULL);
+    if (!req) {
+        gf_msg(GF_NLM, GF_LOG_ERROR, EINVAL, NFS_MSG_INVALID_ENTRY,
+               "Got NULL request!");
         return 0;
+    }
+    rpcsvc_submit_generic(req, &dummyvec, 1, NULL, 0, NULL);
+    return 0;
 }
 
 int
-nlm4_gf_flock_to_holder (nlm4_holder *holder, struct gf_flock *flock)
+nlm4_gf_flock_to_holder(nlm4_holder *holder, struct gf_flock *flock)
 {
-        switch (flock->l_type) {
+    switch (flock->l_type) {
         case GF_LK_F_WRLCK:
-                holder->exclusive = 1;
-                break;
-        }
+            holder->exclusive = 1;
+            break;
+    }
 
-        holder->svid = flock->l_pid;
-        holder->l_offset = flock->l_start;
-        holder->l_len = flock->l_len;
-        return 0;
+    holder->svid = flock->l_pid;
+    holder->l_offset = flock->l_start;
+    holder->l_len = flock->l_len;
+    return 0;
 }
 
 int
-nlm4_lock_to_gf_flock (struct gf_flock *flock, nlm4_lock *lock, int excl)
-{
-        flock->l_pid = lock->svid;
-        flock->l_start = lock->l_offset;
-        flock->l_len = lock->l_len;
-        if (excl)
-                flock->l_type = F_WRLCK;
-        else
-                flock->l_type = F_RDLCK;
-        flock->l_whence = SEEK_SET;
-        nlm_copy_lkowner (&flock->l_owner, &lock->oh);
-        return 0;
+nlm4_lock_to_gf_flock(struct gf_flock *flock, nlm4_lock *lock, int excl)
+{
+    flock->l_pid = lock->svid;
+    flock->l_start = lock->l_offset;
+    flock->l_len = lock->l_len;
+    if (excl)
+        flock->l_type = F_WRLCK;
+    else
+        flock->l_type = F_RDLCK;
+    flock->l_whence = SEEK_SET;
+    nlm_copy_lkowner(&flock->l_owner, &lock->oh);
+    return 0;
 }
 
 rpc_clnt_procedure_t nlm4_clnt_actors[NLM4_PROC_COUNT] = {
-        [NLM4_NULL] = {"NULL", NULL},
-        [NLM4_GRANTED] = {"GRANTED", NULL},
+    [NLM4_NULL] = {"NULL", NULL},
+    [NLM4_GRANTED] = {"GRANTED", NULL},
 };
 
 char *nlm4_clnt_names[NLM4_PROC_COUNT] = {
-        [NLM4_NULL] = "NULL",
-        [NLM4_GRANTED] = "GRANTED",
+    [NLM4_NULL] = "NULL",
+    [NLM4_GRANTED] = "GRANTED",
 };
 
 rpc_clnt_prog_t nlm4clntprog = {
-        .progname = "NLMv4",
-        .prognum = NLM_PROGRAM,
-        .progver = NLM_V4,
-        .numproc = NLM4_PROC_COUNT,
-        .proctable = nlm4_clnt_actors,
-        .procnames = nlm4_clnt_names,
+    .progname = "NLMv4",
+    .prognum = NLM_PROGRAM,
+    .progver = NLM_V4,
+    .numproc = NLM4_PROC_COUNT,
+    .proctable = nlm4_clnt_actors,
+    .procnames = nlm4_clnt_names,
 };
 
 int
-nlm4_test_reply (nfs3_call_state_t *cs, nlm4_stats stat, struct gf_flock *flock)
+nlm4_test_reply(nfs3_call_state_t *cs, nlm4_stats stat, struct gf_flock *flock)
 {
-        nlm4_testres res;
+    nlm4_testres res;
 
-        memset (&res, 0, sizeof (res));
-        res.cookie = cs->args.nlm4_testargs.cookie;
-        res.stat.stat = stat;
-        if (stat == nlm4_denied)
-                nlm4_gf_flock_to_holder (&res.stat.nlm4_testrply_u.holder,
-                                         flock);
+    memset(&res, 0, sizeof(res));
+    res.cookie = cs->args.nlm4_testargs.cookie;
+    res.stat.stat = stat;
+    if ((stat == nlm4_denied) && flock)
+        nlm4_gf_flock_to_holder(&res.stat.nlm4_testrply_u.holder, flock);
 
-        nlm4svc_submit_reply (cs->req, (void *)&res,
-                              (nlm4_serializer)xdr_serialize_nlm4_testres);
-        return 0;
+    nlm4svc_submit_reply(cs->req, (void *)&res,
+                         (nlm4_serializer)xdr_serialize_nlm4_testres);
+    return 0;
 }
 
 int
-nlm4svc_test_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                  int32_t op_ret, int32_t op_errno, struct gf_flock *flock,
-                  dict_t *xdata)
+nlm4svc_test_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                 int32_t op_ret, int32_t op_errno, struct gf_flock *flock,
+                 dict_t *xdata)
 {
-        nlm4_stats                      stat = nlm4_denied;
-        nfs3_call_state_t              *cs = NULL;
+    nlm4_stats stat = nlm4_denied;
+    nfs3_call_state_t *cs = NULL;
 
-        cs = frame->local;
-        if (op_ret == -1) {
-                stat = nlm4_errno_to_nlm4stat (op_errno);
-                goto err;
-        } else if (flock->l_type == F_UNLCK)
-                stat = nlm4_granted;
+    cs = frame->local;
+    if (op_ret == -1) {
+        stat = nlm4_errno_to_nlm4stat(op_errno);
+        goto err;
+    } else if (flock->l_type == F_UNLCK)
+        stat = nlm4_granted;
 
 err:
-        nlm4_test_reply (cs, stat, flock);
-        nfs3_call_state_wipe (cs);
-        return 0;
+    nlm4_test_reply(cs, stat, flock);
+    nfs3_call_state_wipe(cs);
+    return 0;
 }
 
 int
-nlm4_test_fd_resume (void *carg)
-{
-        int                             ret = -EFAULT;
-        nfs_user_t                      nfu = {0, };
-        nfs3_call_state_t               *cs = NULL;
-        struct gf_flock                 flock = {0, };
+nlm4_test_fd_resume(void *carg)
+{
+    int ret = -EFAULT;
+    nfs_user_t nfu = {
+        0,
+    };
+    nfs3_call_state_t *cs = NULL;
+    struct gf_flock flock = {
+        0,
+    };
+
+    if (!carg)
+        return ret;
 
-        if (!carg)
-                return ret;
+    cs = GF_REF_GET((nfs3_call_state_t *)carg);
+    nfs_request_user_init(&nfu, cs->req);
+    nlm4_lock_to_gf_flock(&flock, &cs->args.nlm4_testargs.alock,
+                          cs->args.nlm4_testargs.exclusive);
+    nlm_copy_lkowner(&nfu.lk_owner, &cs->args.nlm4_testargs.alock.oh);
+    ret = nfs_lk(cs->nfsx, cs->vol, &nfu, cs->fd, F_GETLK, &flock,
+                 nlm4svc_test_cbk, cs);
 
-        cs = (nfs3_call_state_t *)carg;
-        nfs_request_user_init (&nfu, cs->req);
-        nlm4_lock_to_gf_flock (&flock, &cs->args.nlm4_testargs.alock,
-                               cs->args.nlm4_testargs.exclusive);
-        nlm_copy_lkowner (&nfu.lk_owner, &cs->args.nlm4_testargs.alock.oh);
-        ret = nfs_lk (cs->nfsx, cs->vol, &nfu, cs->fd, F_GETLK, &flock,
-                      nlm4svc_test_cbk, cs);
+    GF_REF_PUT(cs);
 
-        return ret;
+    return ret;
 }
 
-
 int
-nlm4_test_resume (void *carg)
+nlm4_test_resume(void *carg)
 {
-        nlm4_stats                      stat = nlm4_failed;
-        int                             ret = -1;
-        nfs3_call_state_t               *cs = NULL;
-        fd_t                            *fd = NULL;
+    nlm4_stats stat = nlm4_failed;
+    int ret = -1;
+    nfs3_call_state_t *cs = NULL;
+    fd_t *fd = NULL;
 
-        if (!carg)
-                return ret;
+    if (!carg)
+        return ret;
 
-        cs = (nfs3_call_state_t *)carg;
-        nlm4_check_fh_resolve_status (cs, stat, nlm4err);
-        fd = fd_anonymous (cs->resolvedloc.inode);
-        if (!fd)
-                goto nlm4err;
-        cs->fd = fd;
-        ret = nlm4_test_fd_resume (cs);
+    cs = GF_REF_GET((nfs3_call_state_t *)carg);
+    nlm4_check_fh_resolve_status(cs, stat, nlm4err);
+    fd = fd_anonymous(cs->resolvedloc.inode);
+    if (!fd)
+        goto nlm4err;
+    cs->fd = fd;
+    ret = nlm4_test_fd_resume(cs);
 
 nlm4err:
-        if (ret < 0) {
-                gf_log (GF_NLM, GF_LOG_ERROR, "unable to open_and_resume");
-                stat = nlm4_errno_to_nlm4stat (-ret);
-                nlm4_test_reply (cs, stat, NULL);
-                nfs3_call_state_wipe (cs);
-        }
+    if (ret < 0) {
+        gf_msg(GF_NLM, GF_LOG_ERROR, -ret, NFS_MSG_OPEN_FAIL,
+               "unable to open_and_resume");
+        stat = nlm4_errno_to_nlm4stat(-ret);
+        nlm4_test_reply(cs, stat, NULL);
+        nfs3_call_state_wipe(cs);
+    }
 
-        return ret;
+    GF_REF_PUT(cs);
+
+    return ret;
 }
 
 int
-nlm4svc_test (rpcsvc_request_t *req)
-{
-        xlator_t                        *vol = NULL;
-        nlm4_stats                      stat = nlm4_failed;
-        struct nfs_state               *nfs = NULL;
-        nfs3_state_t                   *nfs3 = NULL;
-        nfs3_call_state_t               *cs = NULL;
-        int                             ret = RPCSVC_ACTOR_ERROR;
-        struct nfs3_fh                  fh = {{0}, };
-
-        if (!req)
-                return ret;
-
-        nlm4_validate_nfs3_state (req, nfs3, stat, rpcerr, ret);
-        nfs = nfs_state (nfs3->nfsx);
-        nlm4_handle_call_state_init (nfs->nfs3state, cs, req,
-                                     stat, rpcerr);
-
-        nlm4_prep_nlm4_testargs (&cs->args.nlm4_testargs, &fh, &cs->lkowner,
-                                 cs->cookiebytes);
-        if (xdr_to_nlm4_testargs(req->msg[0], &cs->args.nlm4_testargs) <= 0) {
-                gf_log (GF_NLM, GF_LOG_ERROR, "Error decoding args");
-                rpcsvc_request_seterr (req, GARBAGE_ARGS);
-                goto rpcerr;
-        }
-
-        nlm4_validate_gluster_fh (&fh, stat, nlm4err);
-        nlm4_map_fh_to_volume (cs->nfs3state, fh, req, vol, stat, nlm4err);
+nlm4svc_test(rpcsvc_request_t *req)
+{
+    xlator_t *vol = NULL;
+    nlm4_stats stat = nlm4_failed;
+    struct nfs_state *nfs = NULL;
+    nfs3_state_t *nfs3 = NULL;
+    nfs3_call_state_t *cs = NULL;
+    int ret = RPCSVC_ACTOR_ERROR;
+    struct nfs3_fh fh = {
+        {0},
+    };
+
+    if (!req)
+        return ret;
 
-        if (nlm_grace_period) {
-                gf_log (GF_NLM, GF_LOG_WARNING, "NLM in grace period");
-                stat = nlm4_denied_grace_period;
-                nlm4_test_reply (cs, stat, NULL);
-                nfs3_call_state_wipe (cs);
-                return 0;
-        }
+    nlm4_validate_nfs3_state(req, nfs3, stat, rpcerr, ret);
+    nfs = nfs_state(nfs3->nfsx);
+    nlm4_handle_call_state_init(nfs->nfs3state, cs, req, stat, rpcerr);
+
+    nlm4_prep_nlm4_testargs(&cs->args.nlm4_testargs, &fh, &cs->lkowner,
+                            cs->cookiebytes);
+    if (xdr_to_nlm4_testargs(req->msg[0], &cs->args.nlm4_testargs) <= 0) {
+        gf_msg(GF_NLM, GF_LOG_ERROR, errno, NFS_MSG_ARGS_DECODE_ERROR,
+               "Error decoding args");
+        rpcsvc_request_seterr(req, GARBAGE_ARGS);
+        goto rpcerr;
+    }
+
+    nlm4_validate_gluster_fh(&fh, stat, nlm4err);
+    nlm4_map_fh_to_volume(cs->nfs3state, fh, req, vol, stat, nlm4err);
+
+    if (nlm_grace_period) {
+        gf_msg(GF_NLM, GF_LOG_WARNING, 0, NFS_MSG_NLM_GRACE_PERIOD,
+               "NLM in grace period");
+        stat = nlm4_denied_grace_period;
+        nlm4_test_reply(cs, stat, NULL);
+        nfs3_call_state_wipe(cs);
+        return 0;
+    }
 
-        cs->vol = vol;
-        nlm4_volume_started_check (nfs3, vol, ret, rpcerr);
+    cs->vol = vol;
+    nlm4_volume_started_check(nfs3, vol, ret, rpcerr);
 
-        ret = nfs3_fh_resolve_and_resume (cs, &fh,
-                                          NULL, nlm4_test_resume);
+    ret = nfs3_fh_resolve_and_resume(cs, &fh, NULL, nlm4_test_resume);
 
 nlm4err:
-        if (ret < 0) {
-                gf_log (GF_NLM, GF_LOG_ERROR, "unable to resolve and resume");
-                nlm4_test_reply (cs, stat, NULL);
-                nfs3_call_state_wipe (cs);
-                return 0;
-        }
+    if (ret < 0) {
+        gf_msg(GF_NLM, GF_LOG_ERROR, -ret, NFS_MSG_RESOLVE_ERROR,
+               "unable to resolve and resume");
+        nlm4_test_reply(cs, stat, NULL);
+        nfs3_call_state_wipe(cs);
+        return 0;
+    }
 
 rpcerr:
-        if (ret < 0)
-                nfs3_call_state_wipe (cs);
+    if (ret < 0)
+        nfs3_call_state_wipe(cs);
 
-        return ret;
+    return ret;
 }
 
-int
-nlm4svc_send_granted_cbk (struct rpc_req *req, struct iovec *iov, int count,
-                          void *myframe)
+struct nlm4_notify_args {
+    GF_REF_DECL; /* refcounting */
+
+    nfs3_call_state_t *cs; /* call state, w/ lock request details */
+    call_frame_t *frame;   /* frame to us for the reply */
+};
+
+static int
+nlm4svc_send_granted_cbk(struct rpc_req *req, struct iovec *iov, int count,
+                         void *myframe)
 {
-        STACK_DESTROY (((call_frame_t*)myframe)->root);
-        return 0;
+    call_frame_t *frame = myframe;
+    struct nlm4_notify_args *args = frame->local;
+
+    GF_REF_PUT(args);
+    return 0;
 }
 
-void
-nlm4svc_send_granted (nfs3_call_state_t *cs);
+static void
+nlm4_notify_free(struct nlm4_notify_args *ncf)
+{
+    GF_REF_PUT(ncf->cs);
+    STACK_DESTROY(ncf->frame->root);
+    GF_FREE(ncf);
+}
+
+static struct nlm4_notify_args *
+nlm4_notify_init(nfs3_call_state_t *cs)
+{
+    struct nlm4_notify_args *ncf = NULL;
+
+    ncf = GF_CALLOC(1, sizeof(struct nlm4_notify_args), gf_nfs_mt_nlm4_notify);
+    if (!ncf)
+        /* GF_CALLOW will log the ENOMEM error */
+        goto out;
+
+    GF_REF_INIT(ncf, nlm4_notify_free);
+    ncf->cs = GF_REF_GET(cs);
+
+out:
+    return ncf;
+}
+
+static int
+nlm_handle_connect(struct rpc_clnt *rpc_clnt, struct nlm4_notify_args *ncf);
 
 int
-nlm_rpcclnt_notify (struct rpc_clnt *rpc_clnt, void *mydata,
-                    rpc_clnt_event_t fn, void *data)
+nlm_rpcclnt_notify(struct rpc_clnt *rpc_clnt, void *mydata, rpc_clnt_event_t fn,
+                   void *data)
 {
-        int                ret         = 0;
-        char              *caller_name = NULL;
-        nfs3_call_state_t *cs          = NULL;
+    struct nlm4_notify_args *ncf = mydata;
 
-        cs = mydata;
-        caller_name = cs->args.nlm4_lockargs.alock.caller_name;
+    GF_VALIDATE_OR_GOTO("NLM4-notify", ncf, out);
 
-        switch (fn) {
+    switch (fn) {
         case RPC_CLNT_CONNECT:
-                ret = nlm_set_rpc_clnt (rpc_clnt, caller_name);
-                if (ret == -1) {
-                        gf_log (GF_NLM, GF_LOG_ERROR, "Failed to set rpc clnt");
-                        goto err;
-                }
-                rpc_clnt_unref (rpc_clnt);
-                nlm4svc_send_granted (cs);
-
-                break;
+            nlm_handle_connect(rpc_clnt, ncf);
+            break;
 
         case RPC_CLNT_MSG:
-                break;
+            break;
 
         case RPC_CLNT_DISCONNECT:
-                nlm_unset_rpc_clnt (rpc_clnt);
-                break;
-        default:
-                break;
-        }
+            nlm_unset_rpc_clnt(rpc_clnt);
+            break;
 
- err:
-        return 0;
+        case RPC_CLNT_DESTROY:
+            GF_REF_PUT(ncf);
+            break;
+
+        default:
+            break;
+    }
+out:
+    return 0;
 }
 
 void *
-nlm4_establish_callback (void *csarg)
+nlm4_establish_callback(nfs3_call_state_t *cs, call_frame_t *cbk_frame)
 {
-        int                           ret                        = -1;
-        nfs3_call_state_t            *cs                         = NULL;
-        union gf_sock_union           sock_union;
-        dict_t                       *options                    = NULL;
-        char                          peerip[INET6_ADDRSTRLEN+1] = {0};
-        char                         *portstr                    = NULL;
-        char                          myip[INET6_ADDRSTRLEN+1]   = {0};
-        rpc_clnt_t                   *rpc_clnt                   = NULL;
-        int                           port                       = -1;
-
+    int ret = -1;
+    union gf_sock_union sock_union;
+    dict_t *options = NULL;
+    char peerip[INET6_ADDRSTRLEN + 1] = {0};
+    char *portstr = NULL;
+    char myip[INET6_ADDRSTRLEN + 1] = {0};
+    rpc_clnt_t *rpc_clnt = NULL;
+    int port = -1;
+    struct nlm4_notify_args *ncf = NULL;
 
-        cs = (nfs3_call_state_t *) csarg;
-        glusterfs_this_set (cs->nfsx);
+    GF_ASSERT(cs->nfsx);
+    THIS = cs->nfsx;
 
-        rpc_transport_get_peeraddr (cs->trans, NULL, 0, &sock_union.storage,
-                                    sizeof (sock_union.storage));
+    rpc_transport_get_peeraddr(cs->trans, NULL, 0, &sock_union.storage,
+                               sizeof(sock_union.storage));
 
-        switch (sock_union.sa.sa_family) {
+    switch (sock_union.sa.sa_family) {
         case AF_INET6:
-        /* can not come here as NLM listens on IPv4 */
-                gf_log (GF_NLM, GF_LOG_ERROR, "NLM is not supported on IPv6 in"
-                        " this release");
-                goto err;
-/*
-                inet_ntop (AF_INET6,
-                           &((struct sockaddr_in6 *)sockaddr)->sin6_addr,
-                           peerip, INET6_ADDRSTRLEN+1);
-                break;
-*/
+            /* can not come here as NLM listens on IPv4 */
+            gf_msg(GF_NLM, GF_LOG_ERROR, EAFNOSUPPORT,
+                   NFS_MSG_UNSUPPORTED_VERSION,
+                   "NLM is not supported on IPv6 in this release");
+            goto err;
+            /*
+                            inet_ntop (AF_INET6,
+                                       &((struct sockaddr_in6
+               *)sockaddr)->sin6_addr, peerip, INET6_ADDRSTRLEN+1); break;
+            */
         case AF_INET:
-                inet_ntop (AF_INET, &sock_union.sin.sin_addr, peerip,
-                           INET6_ADDRSTRLEN+1);
-                inet_ntop (AF_INET, &(((struct sockaddr_in *)&cs->trans->myinfo.sockaddr)->sin_addr),
-                           myip, INET6_ADDRSTRLEN + 1);
-
-                break;
+            inet_ntop(AF_INET, &sock_union.sin.sin_addr, peerip,
+                      INET6_ADDRSTRLEN + 1);
+            inet_ntop(AF_INET,
+                      &(((struct sockaddr_in *)&cs->trans->myinfo.sockaddr)
+                            ->sin_addr),
+                      myip, INET6_ADDRSTRLEN + 1);
+
+            break;
         default:
-                break;
-                /* FIXME: handle the error */
-        }
-
-        /* looks like libc rpc supports only ipv4 */
-        port = pmap_getport (&sock_union.sin, NLM_PROGRAM,
-                             NLM_V4, IPPROTO_TCP);
-
-        if (port == 0) {
-                gf_log (GF_NLM, GF_LOG_ERROR,
-                        "Unable to get NLM port of the client."
-                        " Is the firewall running on client?"
-                        " OR Are RPC services running (rpcinfo -p)?");
-                goto err;
-        }
-
-        options = dict_new();
-        ret = dict_set_str (options, "transport-type", "socket");
-        if (ret == -1) {
-                gf_log (GF_NLM, GF_LOG_ERROR, "dict_set_str error");
-                goto err;
-        }
-
-        ret = dict_set_dynstr (options, "remote-host", gf_strdup (peerip));
-        if (ret == -1) {
-                gf_log (GF_NLM, GF_LOG_ERROR, "dict_set_str error");
-                goto err;
-        }
-
-        ret = gf_asprintf (&portstr, "%d", port);
-        if (ret == -1)
-                goto err;
-
-        ret = dict_set_dynstr (options, "remote-port",
-                               portstr);
-        if (ret == -1) {
-                gf_log (GF_NLM, GF_LOG_ERROR, "dict_set_dynstr error");
-                goto err;
-        }
-
-        /* needed in case virtual IP is used */
-        ret = dict_set_dynstr (options, "transport.socket.source-addr",
-                               gf_strdup (myip));
-        if (ret == -1)
-                goto err;
-
-        ret = dict_set_str (options, "auth-null", "on");
-        if (ret == -1) {
-                gf_log (GF_NLM, GF_LOG_ERROR, "dict_set_dynstr error");
-                goto err;
-        }
-
-        /* TODO: is 32 frames in transit enough ? */
-        rpc_clnt = rpc_clnt_new (options, cs->nfsx->ctx, "NLM-client", 32);
-        if (rpc_clnt == NULL) {
-                gf_log (GF_NLM, GF_LOG_ERROR, "rpc_clnt NULL");
-                goto err;
-        }
-
-        ret = rpc_clnt_register_notify (rpc_clnt, nlm_rpcclnt_notify, cs);
-        if (ret == -1) {
-                gf_log (GF_NLM, GF_LOG_ERROR,"rpc_clnt_register_connect error");
-                goto err;
-        }
-
-        /* After this connect succeeds, granted msg is sent in notify */
-        ret = rpc_transport_connect (rpc_clnt->conn.trans, port);
-
-        if (ret == -1 && EINPROGRESS == errno)
-                ret = 0;
+            break;
+            /* FIXME: handle the error */
+    }
+
+    /* looks like libc rpc supports only ipv4 */
+    port = pmap_getport(&sock_union.sin, NLM_PROGRAM, NLM_V4, IPPROTO_TCP);
+
+    if (port == 0) {
+        gf_msg(GF_NLM, GF_LOG_ERROR, 0, NFS_MSG_GET_PORT_ERROR,
+               "Unable to get NLM port of the client."
+               " Is the firewall running on client?"
+               " OR Are RPC services running (rpcinfo -p)?");
+        goto err;
+    }
+
+    options = dict_new();
+    if (options == NULL) {
+        gf_msg(GF_NLM, GF_LOG_ERROR, ENOMEM, NFS_MSG_GFID_DICT_CREATE_FAIL,
+               "dict allocation failed");
+        goto err;
+    }
+
+    ret = dict_set_str(options, "transport-type", "socket");
+    if (ret == -1) {
+        gf_msg(GF_NLM, GF_LOG_ERROR, errno, NFS_MSG_DICT_SET_FAILED,
+               "dict_set_str error");
+        goto err;
+    }
+
+    ret = dict_set_dynstr(options, "remote-host", gf_strdup(peerip));
+    if (ret == -1) {
+        gf_msg(GF_NLM, GF_LOG_ERROR, errno, NFS_MSG_DICT_SET_FAILED,
+               "dict_set_str error");
+        goto err;
+    }
+
+    ret = gf_asprintf(&portstr, "%d", port);
+    if (ret == -1)
+        goto err;
+
+    ret = dict_set_dynstr(options, "remote-port", portstr);
+    if (ret == -1) {
+        gf_msg(GF_NLM, GF_LOG_ERROR, errno, NFS_MSG_DICT_SET_FAILED,
+               "dict_set_dynstr error");
+        goto err;
+    }
+
+    /* needed in case virtual IP is used */
+    ret = dict_set_dynstr(options, "transport.socket.source-addr",
+                          gf_strdup(myip));
+    if (ret == -1)
+        goto err;
+
+    ret = dict_set_str(options, "auth-null", "on");
+    if (ret == -1) {
+        gf_msg(GF_NLM, GF_LOG_ERROR, errno, NFS_MSG_DICT_SET_FAILED,
+               "dict_set_dynstr error");
+        goto err;
+    }
+
+    ncf = nlm4_notify_init(cs);
+    if (!ncf) {
+        ret = -1;
+        goto err;
+    }
+
+    ncf->frame = cbk_frame;
+    ncf->frame->local = ncf;
+
+    /* TODO: is 32 frames in transit enough ? */
+    rpc_clnt = rpc_clnt_new(options, cs->nfsx, "NLM-client", 32);
+    if (rpc_clnt == NULL) {
+        gf_msg(GF_NLM, GF_LOG_ERROR, EINVAL, NFS_MSG_INVALID_ENTRY,
+               "rpc_clnt NULL");
+        goto err;
+    }
+
+    ret = rpc_clnt_register_notify(rpc_clnt, nlm_rpcclnt_notify, ncf);
+    if (ret == -1) {
+        gf_msg(GF_NLM, GF_LOG_ERROR, errno, NFS_MSG_RPC_CLNT_ERROR,
+               "rpc_clnt_register_connect error");
+        goto err;
+    }
+
+    /* After this connect succeeds, granted msg is sent in notify */
+    ret = rpc_transport_connect(rpc_clnt->conn.trans, port);
+
+    if (ret == -1 && EINPROGRESS == errno)
+        ret = 0;
 
 err:
-        if (ret == -1 && rpc_clnt) {
-                rpc_clnt_unref (rpc_clnt);
-        }
-
-        return rpc_clnt;
-}
-
-void
-nlm4svc_send_granted (nfs3_call_state_t *cs)
-{
-        int ret = -1;
-        rpc_clnt_t *rpc_clnt = NULL;
-        struct iovec            outmsg = {0, };
-        nlm4_testargs testargs;
-        struct iobuf *iobuf = NULL;
-        struct iobref *iobref = NULL;
-        char peerip[INET6_ADDRSTRLEN+1];
-        union gf_sock_union sock_union;
-
-        rpc_clnt = nlm_get_rpc_clnt (cs->args.nlm4_lockargs.alock.caller_name);
-        if (rpc_clnt == NULL) {
-                nlm4_establish_callback ((void*)cs);
-                return;
-        }
+    if (options)
+        dict_unref(options);
+    if (ret == -1) {
+        if (rpc_clnt)
+            rpc_clnt_unref(rpc_clnt);
+        if (ncf)
+            GF_REF_PUT(ncf);
+    }
+
+    return rpc_clnt;
+}
+
+static void
+nlm4svc_send_granted(struct nlm4_notify_args *ncf)
+{
+    int ret = -1;
+    nfs3_call_state_t *cs = ncf->cs;
+    rpc_clnt_t *rpc_clnt = NULL;
+    struct iovec outmsg = {
+        0,
+    };
+    nlm4_testargs testargs;
+    struct iobuf *iobuf = NULL;
+    struct iobref *iobref = NULL;
+    char peerip[INET6_ADDRSTRLEN + 1];
+    union gf_sock_union sock_union;
+
+    rpc_clnt = nlm_get_rpc_clnt(cs->args.nlm4_lockargs.alock.caller_name);
+    if (rpc_clnt == NULL) {
+        nlm4_establish_callback(cs, ncf->frame);
+        return;
+    }
 
-        rpc_transport_get_peeraddr (cs->trans, NULL, 0, &sock_union.storage,
-                                    sizeof (sock_union.storage));
+    rpc_transport_get_peeraddr(cs->trans, NULL, 0, &sock_union.storage,
+                               sizeof(sock_union.storage));
 
-        switch (sock_union.sa.sa_family) {
+    switch (sock_union.sa.sa_family) {
         case AF_INET6:
-                inet_ntop (AF_INET6, &sock_union.sin6.sin6_addr, peerip,
-                           INET6_ADDRSTRLEN+1);
-                break;
+            inet_ntop(AF_INET6, &sock_union.sin6.sin6_addr, peerip,
+                      INET6_ADDRSTRLEN + 1);
+            break;
         case AF_INET:
-                inet_ntop (AF_INET, &sock_union.sin.sin_addr, peerip,
-                           INET6_ADDRSTRLEN+1);
-                break;
+            inet_ntop(AF_INET, &sock_union.sin.sin_addr, peerip,
+                      INET6_ADDRSTRLEN + 1);
+            break;
         default:
-                break;
-        }
-
-        testargs.cookie = cs->args.nlm4_lockargs.cookie;
-        testargs.exclusive = cs->args.nlm4_lockargs.exclusive;
-        testargs.alock = cs->args.nlm4_lockargs.alock;
-
-        iobuf = iobuf_get (cs->nfs3state->iobpool);
-        if (!iobuf) {
-                gf_log (GF_NLM, GF_LOG_ERROR, "Failed to get iobuf");
-                goto ret;
-        }
-
-        iobuf_to_iovec (iobuf, &outmsg);
-        /* Use the given serializer to translate the give C structure in arg
-         * to XDR format which will be written into the buffer in outmsg.
-         */
-        outmsg.iov_len = xdr_serialize_nlm4_testargs (outmsg, &testargs);
-
-        iobref = iobref_new ();
-        if (iobref == NULL) {
-                gf_log (GF_NLM, GF_LOG_ERROR, "Failed to get iobref");
-                goto ret;
-        }
-
-        ret = iobref_add (iobref, iobuf);
-        if (ret) {
-                gf_log (GF_NLM, GF_LOG_ERROR, "Failed to add iob to iobref");
-                goto ret;
-        }
-
-        ret = rpc_clnt_submit (rpc_clnt, &nlm4clntprog, NLM4_GRANTED,
-                               nlm4svc_send_granted_cbk, &outmsg, 1,
-                               NULL, 0, iobref, cs->frame, NULL, 0,
-                               NULL, 0, NULL);
-
-        if (ret < 0) {
-                gf_log (GF_NLM, GF_LOG_ERROR, "rpc_clnt_submit error");
-                goto ret;
-        }
+            break;
+    }
+
+    testargs.cookie = cs->args.nlm4_lockargs.cookie;
+    testargs.exclusive = cs->args.nlm4_lockargs.exclusive;
+    testargs.alock = cs->args.nlm4_lockargs.alock;
+
+    iobuf = iobuf_get(cs->nfs3state->iobpool);
+    if (!iobuf) {
+        gf_msg(GF_NLM, GF_LOG_ERROR, ENOMEM, NFS_MSG_NO_MEMORY,
+               "Failed to get iobuf");
+        goto ret;
+    }
+
+    iobuf_to_iovec(iobuf, &outmsg);
+    /* Use the given serializer to translate the give C structure in arg
+     * to XDR format which will be written into the buffer in outmsg.
+     */
+    outmsg.iov_len = xdr_serialize_nlm4_testargs(outmsg, &testargs);
+
+    iobref = iobref_new();
+    if (iobref == NULL) {
+        gf_msg(GF_NLM, GF_LOG_ERROR, ENOMEM, NFS_MSG_NO_MEMORY,
+               "Failed to get iobref");
+        goto ret;
+    }
+
+    ret = iobref_add(iobref, iobuf);
+    if (ret) {
+        gf_msg(GF_NLM, GF_LOG_ERROR, ENOMEM, NFS_MSG_NO_MEMORY,
+               "Failed to add iob to iobref");
+        goto ret;
+    }
+
+    GF_REF_GET(ncf);
+    ret = rpc_clnt_submit(rpc_clnt, &nlm4clntprog, NLM4_GRANTED,
+                          nlm4svc_send_granted_cbk, &outmsg, 1, NULL, 0, iobref,
+                          ncf->frame, NULL, 0, NULL, 0, NULL);
+
+    if (ret < 0) {
+        gf_msg(GF_NLM, GF_LOG_ERROR, -ret, NFS_MSG_RPC_CLNT_ERROR,
+               "rpc_clnt_submit error");
+        goto ret;
+    }
 ret:
-        if (iobref)
-                iobref_unref (iobref);
-        if (iobuf)
-                iobuf_unref (iobuf);
+    if (iobref)
+        iobref_unref(iobref);
+    if (iobuf)
+        iobuf_unref(iobuf);
 
-        rpc_clnt_unref (rpc_clnt);
-        nfs3_call_state_wipe (cs);
-        return;
+    rpc_clnt_unref(rpc_clnt);
+    return;
 }
 
 int
-nlm_cleanup_fds (char *caller_name)
-{
-        int nlmclnt_found = 0;
-        nlm_fde_t *fde = NULL, *tmp = NULL;
-        nlm_client_t *nlmclnt = NULL;
-
-        LOCK (&nlm_client_list_lk);
-        list_for_each_entry (nlmclnt,
-                             &nlm_client_list, nlm_clients) {
-                if (!strcmp(caller_name, nlmclnt->caller_name)) {
-                        nlmclnt_found = 1;
-                        break;
-                }
-        }
-
-        if (!nlmclnt_found)
-                goto ret;
-
-        if (list_empty (&nlmclnt->fdes))
-                goto ret;
+nlm_cleanup_fds(char *caller_name)
+{
+    int nlmclnt_found = 0;
+    nlm_client_t *nlmclnt = NULL;
 
-        list_for_each_entry_safe (fde, tmp, &nlmclnt->fdes, fde_list) {
-                fd_unref (fde->fd);
-                list_del (&fde->fde_list);
-                GF_FREE (fde);
+    LOCK(&nlm_client_list_lk);
+    list_for_each_entry(nlmclnt, &nlm_client_list, nlm_clients)
+    {
+        if (!strcmp(caller_name, nlmclnt->caller_name)) {
+            nlmclnt_found = 1;
+            break;
         }
+    }
+
+    if (!nlmclnt_found)
+        goto ret;
 
+    nlm_client_free(nlmclnt);
 ret:
-        UNLOCK (&nlm_client_list_lk);
-        return 0;
+    UNLOCK(&nlm_client_list_lk);
+    return 0;
 }
 
 void
-nlm_search_and_delete (fd_t *fd, char *caller_name)
-{
-        nlm_fde_t *fde = NULL;
-        nlm_client_t *nlmclnt = NULL;
-        int nlmclnt_found = 0;
-        int fde_found = 0;
-        int transit_cnt = 0;
-
-        LOCK (&nlm_client_list_lk);
-        list_for_each_entry (nlmclnt,
-                             &nlm_client_list, nlm_clients) {
-                if (!strcmp(caller_name, nlmclnt->caller_name)) {
-                        nlmclnt_found = 1;
-                        break;
-                }
+nlm_search_and_delete(fd_t *fd, nlm4_lock *lk)
+{
+    nlm_fde_t *fde = NULL;
+    nlm_client_t *nlmclnt = NULL;
+    int nlmclnt_found = 0;
+    int fde_found = 0;
+    int transit_cnt = 0;
+
+    LOCK(&nlm_client_list_lk);
+    list_for_each_entry(nlmclnt, &nlm_client_list, nlm_clients)
+    {
+        if (!strcmp(lk->caller_name, nlmclnt->caller_name)) {
+            nlmclnt_found = 1;
+            break;
         }
+    }
 
-        if (!nlmclnt_found)
-                goto ret;
+    if (!nlmclnt_found)
+        goto ret;
 
-        list_for_each_entry (fde, &nlmclnt->fdes, fde_list) {
-                if (fde->fd == fd) {
-                        fde_found = 1;
-                        break;
-                }
+    list_for_each_entry(fde, &nlmclnt->fdes, fde_list)
+    {
+        if (fde->fd == fd) {
+            fde_found = 1;
+            break;
         }
+    }
 
-        if (!fde_found)
-                goto ret;
-        transit_cnt = fde->transit_cnt;
-        if (transit_cnt)
-                goto ret;
-        list_del (&fde->fde_list);
+    if (!fde_found)
+        goto ret;
+    transit_cnt = fde->transit_cnt;
+    if (transit_cnt)
+        goto ret;
+    list_del(&fde->fde_list);
 
 ret:
-        UNLOCK (&nlm_client_list_lk);
+    UNLOCK(&nlm_client_list_lk);
 
-        if (fde_found && !transit_cnt) {
-                fd_unref (fde->fd);
-                GF_FREE (fde);
-        }
-        return;
+    if (fde_found && !transit_cnt) {
+        fd_unref(fde->fd);
+        GF_FREE(fde);
+    }
+    return;
 }
 
 int
-nlm_dec_transit_count (fd_t *fd, char *caller_name)
-{
-        nlm_fde_t *fde = NULL;
-        nlm_client_t *nlmclnt = NULL;
-        int nlmclnt_found = 0;
-        int fde_found = 0;
-        int transit_cnt = -1;
-
-        LOCK (&nlm_client_list_lk);
-        list_for_each_entry (nlmclnt,
-                             &nlm_client_list, nlm_clients) {
-                if (!strcmp(caller_name, nlmclnt->caller_name)) {
-                        nlmclnt_found = 1;
-                        break;
-                }
-        }
-
-        if (!nlmclnt_found) {
-                gf_log (GF_NLM, GF_LOG_ERROR, "nlmclnt not found");
-                nlmclnt = NULL;
-                goto ret;
-        }
-
-        list_for_each_entry (fde, &nlmclnt->fdes, fde_list) {
-                if (fde->fd == fd) {
-                        fde_found = 1;
-                        break;
-                }
-        }
-
-        if (fde_found) {
-                transit_cnt = --fde->transit_cnt;
-                goto ret;
-        }
+nlm_dec_transit_count(fd_t *fd, char *caller_name)
+{
+    nlm_fde_t *fde = NULL;
+    nlm_client_t *nlmclnt = NULL;
+    int nlmclnt_found = 0;
+    int fde_found = 0;
+    int transit_cnt = -1;
+
+    LOCK(&nlm_client_list_lk);
+    list_for_each_entry(nlmclnt, &nlm_client_list, nlm_clients)
+    {
+        if (!strcmp(caller_name, nlmclnt->caller_name)) {
+            nlmclnt_found = 1;
+            break;
+        }
+    }
+
+    if (!nlmclnt_found) {
+        gf_msg(GF_NLM, GF_LOG_ERROR, 0, NFS_MSG_NLMCLNT_NOT_FOUND,
+               "nlmclnt not found");
+        nlmclnt = NULL;
+        goto ret;
+    }
+
+    list_for_each_entry(fde, &nlmclnt->fdes, fde_list)
+    {
+        if (fde->fd == fd) {
+            fde_found = 1;
+            break;
+        }
+    }
+
+    if (fde_found) {
+        transit_cnt = --fde->transit_cnt;
+        goto ret;
+    }
 ret:
 
-        UNLOCK (&nlm_client_list_lk);
-        return transit_cnt;
+    UNLOCK(&nlm_client_list_lk);
+    return transit_cnt;
 }
 
-
 nlm_client_t *
-nlm_search_and_add (fd_t *fd, char *caller_name)
-{
-        nlm_fde_t *fde = NULL;
-        nlm_client_t *nlmclnt = NULL;
-        int nlmclnt_found = 0;
-        int fde_found = 0;
-
-        LOCK (&nlm_client_list_lk);
-        list_for_each_entry (nlmclnt,
-                             &nlm_client_list, nlm_clients) {
-                if (!strcmp(caller_name, nlmclnt->caller_name)) {
-                        nlmclnt_found = 1;
-                        break;
-                }
-        }
+nlm_search_and_add(fd_t *fd, char *caller_name)
+{
+    nlm_fde_t *fde = NULL;
+    nlm_client_t *nlmclnt = NULL;
+    int nlmclnt_found = 0;
+    int fde_found = 0;
 
-        if (!nlmclnt_found) {
-                gf_log (GF_NLM, GF_LOG_ERROR, "nlmclnt not found");
-                nlmclnt = NULL;
-                goto ret;
+    LOCK(&nlm_client_list_lk);
+    list_for_each_entry(nlmclnt, &nlm_client_list, nlm_clients)
+    {
+        if (!strcmp(caller_name, nlmclnt->caller_name)) {
+            nlmclnt_found = 1;
+            break;
         }
+    }
 
-        list_for_each_entry (fde, &nlmclnt->fdes, fde_list) {
-                if (fde->fd == fd) {
-                        fde_found = 1;
-                        break;
-                }
+    if (!nlmclnt_found) {
+        gf_msg(GF_NLM, GF_LOG_ERROR, 0, NFS_MSG_NLMCLNT_NOT_FOUND,
+               "nlmclnt not found");
+        nlmclnt = NULL;
+        goto ret;
+    }
+
+    list_for_each_entry(fde, &nlmclnt->fdes, fde_list)
+    {
+        if (fde->fd == fd) {
+            fde_found = 1;
+            break;
         }
+    }
 
-        if (fde_found)
-                goto ret;
+    if (fde_found)
+        goto ret;
 
-        fde = GF_CALLOC (1, sizeof (*fde), gf_nfs_mt_nlm4_fde);
+    fde = GF_CALLOC(1, sizeof(*fde), gf_nfs_mt_nlm4_fde);
 
-        fde->fd = fd_ref (fd);
-        list_add (&fde->fde_list, &nlmclnt->fdes);
+    fde->fd = fd_ref(fd);
+    list_add(&fde->fde_list, &nlmclnt->fdes);
 ret:
-        if (nlmclnt_found && fde)
-                fde->transit_cnt++;
-        UNLOCK (&nlm_client_list_lk);
-        return nlmclnt;
+    if (nlmclnt_found && fde)
+        fde->transit_cnt++;
+    UNLOCK(&nlm_client_list_lk);
+    return nlmclnt;
 }
 
 int
-nlm4svc_lock_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                  int32_t op_ret, int32_t op_errno, struct gf_flock *flock,
-                  dict_t *xdata)
-{
-        nlm4_stats                       stat        = nlm4_denied;
-        int                              transit_cnt = -1;
-        char                            *caller_name = NULL;
-        nfs3_call_state_t               *cs          = NULL;
-        pthread_t                        thr;
-
-        cs = frame->local;
-        caller_name = cs->args.nlm4_lockargs.alock.caller_name;
-        transit_cnt = nlm_dec_transit_count (cs->fd, caller_name);
-
-        if (op_ret == -1) {
-                if (transit_cnt == 0)
-                        nlm_search_and_delete (cs->fd, caller_name);
-                stat = nlm4_errno_to_nlm4stat (op_errno);
-                goto err;
-        } else {
-                stat = nlm4_granted;
-                if (cs->monitor && !nlm_monitor (caller_name)) {
-                        /* FIXME: handle nsm_monitor failure */
-                        pthread_create (&thr, NULL, nsm_monitor, (void*)caller_name);
-                }
-        }
+nlm4svc_lock_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                 int32_t op_ret, int32_t op_errno, struct gf_flock *flock,
+                 dict_t *xdata)
+{
+    nlm4_stats stat = nlm4_denied;
+    int transit_cnt = -1;
+    char *caller_name = NULL;
+    nfs3_call_state_t *cs = NULL;
+    pthread_t thr;
+    struct nlm4_notify_args *ncf = NULL;
+
+    cs = frame->local;
+    caller_name = cs->args.nlm4_lockargs.alock.caller_name;
+    transit_cnt = nlm_dec_transit_count(cs->fd, caller_name);
+
+    if (op_ret == -1) {
+        if (transit_cnt == 0)
+            nlm_search_and_delete(cs->fd, &cs->args.nlm4_lockargs.alock);
+        stat = nlm4_errno_to_nlm4stat(op_errno);
+        goto err;
+    } else {
+        stat = nlm4_granted;
+        if (cs->monitor && !nlm_monitor(caller_name)) {
+            /* FIXME: handle nsm_monitor failure */
+            (void)gf_thread_create(&thr, NULL, nsm_monitor, (void *)caller_name,
+                                   "nlmmon");
+        }
+    }
 
 err:
-        if (cs->args.nlm4_lockargs.block) {
-                cs->frame = copy_frame (frame);
-                frame->local = NULL;
-                nlm4svc_send_granted (cs);
-        } else {
-                nlm4_generic_reply (cs->req, cs->args.nlm4_lockargs.cookie,
-                                    stat);
-                nfs3_call_state_wipe (cs);
+    if (cs->args.nlm4_lockargs.block) {
+        ncf = nlm4_notify_init(cs);
+        if (ncf) {
+            ncf->frame = copy_frame(frame);
+            ncf->frame->local = ncf;
+            nlm4svc_send_granted(ncf);
         }
-        return 0;
+    } else {
+        nlm4_generic_reply(cs->req, cs->args.nlm4_lockargs.cookie, stat);
+        nfs3_call_state_wipe(cs);
+    }
+
+    return 0;
 }
 
 int
-nlm4_lock_fd_resume (void *carg)
-{
-        nlm4_stats                      stat = nlm4_denied;
-        int                             ret = -EFAULT;
-        nfs_user_t                      nfu = {0, };
-        nfs3_call_state_t               *cs = NULL;
-        struct gf_flock                 flock = {0, };
-
-        if (!carg)
-                return ret;
-
-        cs = (nfs3_call_state_t *)carg;
-        nlm4_check_fh_resolve_status (cs, stat, nlm4err);
-        (void) nlm_search_and_add (cs->fd,
-                                   cs->args.nlm4_lockargs.alock.caller_name);
-        nfs_request_user_init (&nfu, cs->req);
-        nlm4_lock_to_gf_flock (&flock, &cs->args.nlm4_lockargs.alock,
-                               cs->args.nlm4_lockargs.exclusive);
-        nlm_copy_lkowner (&nfu.lk_owner, &cs->args.nlm4_lockargs.alock.oh);
-        if (cs->args.nlm4_lockargs.block) {
-                nlm4_generic_reply (cs->req, cs->args.nlm4_lockargs.cookie,
-                                    nlm4_blocked);
-                ret = nfs_lk (cs->nfsx, cs->vol, &nfu, cs->fd, F_SETLKW,
-                              &flock, nlm4svc_lock_cbk, cs);
-                /* FIXME: handle error from nfs_lk() specially  by just
-                 * cleaning up cs and unblock the client lock request.
-                 */
-                ret = 0;
-        } else
-                ret = nfs_lk (cs->nfsx, cs->vol, &nfu, cs->fd, F_SETLK,
-                              &flock, nlm4svc_lock_cbk, cs);
+nlm4_lock_fd_resume(void *carg)
+{
+    nlm4_stats stat = nlm4_denied;
+    int ret = -EFAULT;
+    nfs_user_t nfu = {
+        0,
+    };
+    nfs3_call_state_t *cs = NULL;
+    struct gf_flock flock = {
+        0,
+    };
+
+    if (!carg)
+        return ret;
+
+    cs = GF_REF_GET((nfs3_call_state_t *)carg);
+    nlm4_check_fh_resolve_status(cs, stat, nlm4err);
+    (void)nlm_search_and_add(cs->fd, cs->args.nlm4_lockargs.alock.caller_name);
+    nfs_request_user_init(&nfu, cs->req);
+    nlm4_lock_to_gf_flock(&flock, &cs->args.nlm4_lockargs.alock,
+                          cs->args.nlm4_lockargs.exclusive);
+    nlm_copy_lkowner(&nfu.lk_owner, &cs->args.nlm4_lockargs.alock.oh);
+    if (cs->args.nlm4_lockargs.block) {
+        nlm4_generic_reply(cs->req, cs->args.nlm4_lockargs.cookie,
+                           nlm4_blocked);
+        ret = nfs_lk(cs->nfsx, cs->vol, &nfu, cs->fd, F_SETLKW, &flock,
+                     nlm4svc_lock_cbk, cs);
+        /* FIXME: handle error from nfs_lk() specially  by just
+         * cleaning up cs and unblock the client lock request.
+         */
+        ret = 0;
+    } else
+        ret = nfs_lk(cs->nfsx, cs->vol, &nfu, cs->fd, F_SETLK, &flock,
+                     nlm4svc_lock_cbk, cs);
 
 nlm4err:
-        if (ret < 0) {
-                stat = nlm4_errno_to_nlm4stat (-ret);
-                gf_log (GF_NLM, GF_LOG_ERROR, "unable to call lk()");
-                nlm4_generic_reply (cs->req, cs->args.nlm4_lockargs.cookie,
-                                    stat);
-                nfs3_call_state_wipe (cs);
-        }
+    if (ret < 0) {
+        stat = nlm4_errno_to_nlm4stat(-ret);
+        gf_msg(GF_NLM, GF_LOG_ERROR, stat, NFS_MSG_LOCK_FAIL,
+               "unable to call lk()");
+        nlm4_generic_reply(cs->req, cs->args.nlm4_lockargs.cookie, stat);
+        nfs3_call_state_wipe(cs);
+    }
 
-        return ret;
-}
+    GF_REF_PUT(cs);
 
+    return ret;
+}
 
 int
-nlm4_lock_resume (void *carg)
+nlm4_lock_resume(void *carg)
 {
-        nlm4_stats                      stat = nlm4_failed;
-        int                             ret  = -1;
-        nfs3_call_state_t              *cs   = NULL;
+    nlm4_stats stat = nlm4_failed;
+    int ret = -1;
+    nfs3_call_state_t *cs = NULL;
 
-        if (!carg)
-                return ret;
+    if (!carg)
+        return ret;
 
-        cs = (nfs3_call_state_t *)carg;
-        nlm4_check_fh_resolve_status (cs, stat, nlm4err);
-        ret = nlm4_file_open_and_resume (cs, nlm4_lock_fd_resume);
+    cs = GF_REF_GET((nfs3_call_state_t *)carg);
+    nlm4_check_fh_resolve_status(cs, stat, nlm4err);
+    ret = nlm4_file_open_and_resume(cs, nlm4_lock_fd_resume);
 
 nlm4err:
-        if (ret < 0) {
-                gf_log (GF_NLM, GF_LOG_ERROR, "unable to open and resume");
-                stat = nlm4_errno_to_nlm4stat (-ret);
-                nlm4_generic_reply (cs->req, cs->args.nlm4_lockargs.cookie,
-                                    stat);
-                nfs3_call_state_wipe (cs);
-        }
+    if (ret < 0) {
+        gf_msg(GF_NLM, GF_LOG_ERROR, -ret, NFS_MSG_OPEN_FAIL,
+               "unable to open and resume");
+        stat = nlm4_errno_to_nlm4stat(-ret);
+        nlm4_generic_reply(cs->req, cs->args.nlm4_lockargs.cookie, stat);
+        nfs3_call_state_wipe(cs);
+    }
 
-        return ret;
+    GF_REF_PUT(cs);
+
+    return ret;
 }
 
 int
-nlm4svc_lock_common (rpcsvc_request_t *req, int mon)
-{
-        int                              ret      = RPCSVC_ACTOR_ERROR;
-        nlm4_stats                       stat     = nlm4_failed;
-        struct nfs3_fh                   fh       = {{0}, };
-        xlator_t                        *vol      = NULL;
-        nfs3_state_t                    *nfs3     = NULL;
-        nfs3_call_state_t               *cs       = NULL;
-        struct nfs_state                *nfs      = NULL;
-
-        if (!req)
-                return ret;
-
-        nlm4_validate_nfs3_state (req, nfs3, stat, rpcerr, ret);
-        nfs = nfs_state (nfs3->nfsx);
-        nlm4_handle_call_state_init (nfs->nfs3state, cs, req,
-                                     stat, rpcerr);
-
-        nlm4_prep_nlm4_lockargs (&cs->args.nlm4_lockargs, &cs->lockfh,
-                                 &cs->lkowner, cs->cookiebytes);
-        if (xdr_to_nlm4_lockargs(req->msg[0], &cs->args.nlm4_lockargs) <= 0) {
-                gf_log (GF_NLM, GF_LOG_ERROR, "Error decoding args");
-                rpcsvc_request_seterr (req, GARBAGE_ARGS);
-                goto rpcerr;
-        }
+nlm4svc_lock_common(rpcsvc_request_t *req, int mon)
+{
+    int ret = RPCSVC_ACTOR_ERROR;
+    nlm4_stats stat = nlm4_failed;
+    struct nfs3_fh fh = {
+        {0},
+    };
+    xlator_t *vol = NULL;
+    nfs3_state_t *nfs3 = NULL;
+    nfs3_call_state_t *cs = NULL;
+    struct nfs_state *nfs = NULL;
+
+    if (!req)
+        return ret;
 
-        fh = cs->lockfh;
-        cs->monitor = mon;
-        nlm4_validate_gluster_fh (&fh, stat, nlm4err);
-        nlm4_map_fh_to_volume (cs->nfs3state, fh, req, vol, stat, nlm4err);
-
-        if (nlm_grace_period && !cs->args.nlm4_lockargs.reclaim) {
-                gf_log (GF_NLM, GF_LOG_WARNING, "NLM in grace period");
-                stat = nlm4_denied_grace_period;
-                nlm4_generic_reply (req, cs->args.nlm4_unlockargs.cookie, stat);
-                nfs3_call_state_wipe (cs);
-                return 0;
-        }
+    nlm4_validate_nfs3_state(req, nfs3, stat, rpcerr, ret);
+    nfs = nfs_state(nfs3->nfsx);
+    nlm4_handle_call_state_init(nfs->nfs3state, cs, req, stat, rpcerr);
+
+    nlm4_prep_nlm4_lockargs(&cs->args.nlm4_lockargs, &cs->lockfh, &cs->lkowner,
+                            cs->cookiebytes);
+    if (xdr_to_nlm4_lockargs(req->msg[0], &cs->args.nlm4_lockargs) <= 0) {
+        gf_msg(GF_NLM, GF_LOG_ERROR, errno, NFS_MSG_ARGS_DECODE_ERROR,
+               "Error decoding args");
+        rpcsvc_request_seterr(req, GARBAGE_ARGS);
+        goto rpcerr;
+    }
+
+    fh = cs->lockfh;
+    cs->monitor = mon;
+    nlm4_validate_gluster_fh(&fh, stat, nlm4err);
+    nlm4_map_fh_to_volume(cs->nfs3state, fh, req, vol, stat, nlm4err);
+
+    if (nlm_grace_period && !cs->args.nlm4_lockargs.reclaim) {
+        gf_msg(GF_NLM, GF_LOG_WARNING, 0, NFS_MSG_NLM_GRACE_PERIOD,
+               "NLM in grace period");
+        stat = nlm4_denied_grace_period;
+        nlm4_generic_reply(req, cs->args.nlm4_unlockargs.cookie, stat);
+        nfs3_call_state_wipe(cs);
+        return 0;
+    }
 
-        cs->vol = vol;
-        cs->trans = rpcsvc_request_transport_ref(req);
-        nlm4_volume_started_check (nfs3, vol, ret, rpcerr);
+    cs->vol = vol;
+    cs->trans = rpcsvc_request_transport_ref(req);
+    nlm4_volume_started_check(nfs3, vol, ret, rpcerr);
 
-        ret = nlm_add_nlmclnt (cs->args.nlm4_lockargs.alock.caller_name);
+    ret = nlm_add_nlmclnt(cs->args.nlm4_lockargs.alock.caller_name);
 
-        ret = nfs3_fh_resolve_and_resume (cs, &fh,
-                                          NULL, nlm4_lock_resume);
+    ret = nfs3_fh_resolve_and_resume(cs, &fh, NULL, nlm4_lock_resume);
 
 nlm4err:
-        if (ret < 0) {
-                gf_log (GF_NLM, GF_LOG_ERROR, "unable to resolve and resume");
-                nlm4_generic_reply (cs->req, cs->args.nlm4_lockargs.cookie,
-                                    stat);
-                nfs3_call_state_wipe (cs);
-                return 0;
-        }
+    if (ret < 0) {
+        gf_msg(GF_NLM, GF_LOG_ERROR, -ret, NFS_MSG_RESOLVE_ERROR,
+               "unable to resolve and resume");
+        nlm4_generic_reply(cs->req, cs->args.nlm4_lockargs.cookie, stat);
+        nfs3_call_state_wipe(cs);
+        return 0;
+    }
 
 rpcerr:
-        if (ret < 0) {
-                nfs3_call_state_wipe (cs);
-        }
+    if (ret < 0) {
+        nfs3_call_state_wipe(cs);
+    }
 
-        return ret;
+    return ret;
 }
 
 int
-nlm4svc_lock (rpcsvc_request_t *req)
+nlm4svc_lock(rpcsvc_request_t *req)
 {
-        return nlm4svc_lock_common (req, 1);
+    return nlm4svc_lock_common(req, 1);
 }
 
 int
-nlm4svc_nm_lock (rpcsvc_request_t *req)
+nlm4svc_nm_lock(rpcsvc_request_t *req)
 {
-        return nlm4svc_lock_common (req, 0);
+    return nlm4svc_lock_common(req, 0);
 }
 
 int
-nlm4svc_cancel_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                    int32_t op_ret, int32_t op_errno, struct gf_flock *flock,
-                    dict_t *xdata)
-{
-        nlm4_stats                      stat = nlm4_denied;
-        nfs3_call_state_t              *cs = NULL;
-
-        cs = frame->local;
-        if (op_ret == -1) {
-                stat = nlm4_errno_to_nlm4stat (op_errno);
-                goto err;
-        } else
-                stat = nlm4_granted;
+nlm4svc_cancel_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                   int32_t op_ret, int32_t op_errno, struct gf_flock *flock,
+                   dict_t *xdata)
+{
+    nlm4_stats stat = nlm4_denied;
+    nfs3_call_state_t *cs = NULL;
+
+    cs = frame->local;
+    if (op_ret == -1) {
+        stat = nlm4_errno_to_nlm4stat(op_errno);
+        goto err;
+    } else {
+        stat = nlm4_granted;
+        nlm_search_and_delete(cs->fd, &cs->args.nlm4_lockargs.alock);
+    }
 
 err:
-        nlm4_generic_reply (cs->req, cs->args.nlm4_cancargs.cookie,
-                            stat);
-        nfs3_call_state_wipe (cs);
-        return 0;
+    nlm4_generic_reply(cs->req, cs->args.nlm4_cancargs.cookie, stat);
+    nfs3_call_state_wipe(cs);
+    return 0;
 }
 
 int
-nlm4_cancel_fd_resume (void *carg)
-{
-        int                             ret = -EFAULT;
-        nfs_user_t                      nfu = {0, };
-        nfs3_call_state_t               *cs = NULL;
-        struct gf_flock                 flock = {0, };
+nlm4_cancel_fd_resume(void *carg)
+{
+    int ret = -EFAULT;
+    nfs_user_t nfu = {
+        0,
+    };
+    nfs3_call_state_t *cs = NULL;
+    struct gf_flock flock = {
+        0,
+    };
+
+    if (!carg)
+        return ret;
 
-        if (!carg)
-                return ret;
+    cs = GF_REF_GET((nfs3_call_state_t *)carg);
+    nfs_request_user_init(&nfu, cs->req);
+    nlm4_lock_to_gf_flock(&flock, &cs->args.nlm4_cancargs.alock,
+                          cs->args.nlm4_cancargs.exclusive);
+    nlm_copy_lkowner(&nfu.lk_owner, &cs->args.nlm4_cancargs.alock.oh);
+    flock.l_type = F_UNLCK;
+    ret = nfs_lk(cs->nfsx, cs->vol, &nfu, cs->fd, F_SETLK, &flock,
+                 nlm4svc_cancel_cbk, cs);
 
-        cs = (nfs3_call_state_t *)carg;
-        nfs_request_user_init (&nfu, cs->req);
-        nlm4_lock_to_gf_flock (&flock, &cs->args.nlm4_cancargs.alock,
-                                cs->args.nlm4_cancargs.exclusive);
-        nlm_copy_lkowner (&nfu.lk_owner, &cs->args.nlm4_cancargs.alock.oh);
-        flock.l_type = F_UNLCK;
-        ret = nfs_lk (cs->nfsx, cs->vol, &nfu, cs->fd, F_SETLK,
-                      &flock, nlm4svc_cancel_cbk, cs);
+    GF_REF_PUT(cs);
 
-        return ret;
+    return ret;
 }
 
 int
-nlm4_cancel_resume (void *carg)
+nlm4_cancel_resume(void *carg)
 {
-        nlm4_stats                      stat = nlm4_failed;
-        int                             ret = -EFAULT;
-        nfs3_call_state_t               *cs = NULL;
-        nlm_client_t                    *nlmclnt = NULL;
+    nlm4_stats stat = nlm4_failed;
+    int ret = -EFAULT;
+    nfs3_call_state_t *cs = NULL;
+    nlm_client_t *nlmclnt = NULL;
 
-        if (!carg)
-                return ret;
-
-        cs = (nfs3_call_state_t *)carg;
-        nlm4_check_fh_resolve_status (cs, stat, nlm4err);
+    if (!carg)
+        return ret;
 
-        nlmclnt = nlm_get_uniq (cs->args.nlm4_cancargs.alock.caller_name);
-        if (nlmclnt == NULL) {
-                gf_log (GF_NLM, GF_LOG_ERROR, "nlm_get_uniq() returned NULL");
-                goto nlm4err;
-        }
-        cs->fd = fd_lookup_uint64 (cs->resolvedloc.inode, (uint64_t)nlmclnt);
-        if (cs->fd == NULL) {
-                gf_log (GF_NLM, GF_LOG_ERROR, "fd_lookup_uint64 retrned NULL");
-                goto nlm4err;
-        }
-        ret = nlm4_cancel_fd_resume (cs);
+    cs = GF_REF_GET((nfs3_call_state_t *)carg);
+    nlm4_check_fh_resolve_status(cs, stat, nlm4err);
+
+    nlmclnt = nlm_get_uniq(cs->args.nlm4_cancargs.alock.caller_name);
+    if (nlmclnt == NULL) {
+        gf_msg(GF_NLM, GF_LOG_ERROR, ENOLCK, NFS_MSG_NO_MEMORY,
+               "nlm_get_uniq() returned NULL");
+        goto nlm4err;
+    }
+    cs->fd = fd_lookup_uint64(cs->resolvedloc.inode,
+                              (uint64_t)(uintptr_t)nlmclnt);
+    if (cs->fd == NULL) {
+        gf_msg(GF_NLM, GF_LOG_ERROR, errno, NFS_MSG_FD_LOOKUP_NULL,
+               "fd_lookup_uint64 retrned NULL");
+        goto nlm4err;
+    }
+    ret = nlm4_cancel_fd_resume(cs);
 
 nlm4err:
-        if (ret < 0) {
-                gf_log (GF_NLM, GF_LOG_WARNING, "unable to unlock_fd_resume()");
-                stat = nlm4_errno_to_nlm4stat (-ret);
-                nlm4_generic_reply (cs->req, cs->args.nlm4_cancargs.cookie,
-                                    stat);
+    if (ret < 0) {
+        gf_msg(GF_NLM, GF_LOG_WARNING, -ret, NFS_MSG_LOCK_FAIL,
+               "unable to unlock_fd_resume()");
+        stat = nlm4_errno_to_nlm4stat(-ret);
+        nlm4_generic_reply(cs->req, cs->args.nlm4_cancargs.cookie, stat);
 
-                nfs3_call_state_wipe (cs);
-        }
-        /* clean up is taken care of */
-        return 0;
+        nfs3_call_state_wipe(cs);
+    }
+
+    GF_REF_PUT(cs);
+
+    /* clean up is taken care of */
+    return 0;
 }
 
 int
-nlm4svc_cancel (rpcsvc_request_t *req)
-{
-        xlator_t                        *vol = NULL;
-        nlm4_stats                      stat = nlm4_failed;
-        struct nfs_state               *nfs = NULL;
-        nfs3_state_t                   *nfs3 = NULL;
-        nfs3_call_state_t               *cs = NULL;
-        int                             ret = RPCSVC_ACTOR_ERROR;
-        struct nfs3_fh                  fh = {{0}, };
-
-        if (!req)
-                return ret;
-
-        nlm4_validate_nfs3_state (req, nfs3, stat, rpcerr, ret);
-        nfs = nfs_state (nfs3->nfsx);
-        nlm4_handle_call_state_init (nfs->nfs3state, cs, req,
-                                     stat, rpcerr);
-
-        nlm4_prep_nlm4_cancargs (&cs->args.nlm4_cancargs, &fh, &cs->lkowner,
-                                 cs->cookiebytes);
-        if (xdr_to_nlm4_cancelargs(req->msg[0], &cs->args.nlm4_cancargs) <= 0) {
-                gf_log (GF_NLM, GF_LOG_ERROR, "Error decoding args");
-                rpcsvc_request_seterr (req, GARBAGE_ARGS);
-                goto rpcerr;
-        }
-
-        nlm4_validate_gluster_fh (&fh, stat, nlm4err);
-        nlm4_map_fh_to_volume (cs->nfs3state, fh, req, vol, stat, nlm4err);
+nlm4svc_cancel(rpcsvc_request_t *req)
+{
+    xlator_t *vol = NULL;
+    nlm4_stats stat = nlm4_failed;
+    struct nfs_state *nfs = NULL;
+    nfs3_state_t *nfs3 = NULL;
+    nfs3_call_state_t *cs = NULL;
+    int ret = RPCSVC_ACTOR_ERROR;
+    struct nfs3_fh fh = {
+        {0},
+    };
+
+    if (!req)
+        return ret;
 
-        if (nlm_grace_period) {
-                gf_log (GF_NLM, GF_LOG_WARNING, "NLM in grace period");
-                stat = nlm4_denied_grace_period;
-                nlm4_generic_reply (req, cs->args.nlm4_unlockargs.cookie, stat);
-                nfs3_call_state_wipe (cs);
-                return 0;
-        }
+    nlm4_validate_nfs3_state(req, nfs3, stat, rpcerr, ret);
+    nfs = nfs_state(nfs3->nfsx);
+    nlm4_handle_call_state_init(nfs->nfs3state, cs, req, stat, rpcerr);
+
+    nlm4_prep_nlm4_cancargs(&cs->args.nlm4_cancargs, &fh, &cs->lkowner,
+                            cs->cookiebytes);
+    if (xdr_to_nlm4_cancelargs(req->msg[0], &cs->args.nlm4_cancargs) <= 0) {
+        gf_msg(GF_NLM, GF_LOG_ERROR, errno, NFS_MSG_ARGS_DECODE_ERROR,
+               "Error decoding args");
+        rpcsvc_request_seterr(req, GARBAGE_ARGS);
+        goto rpcerr;
+    }
+
+    nlm4_validate_gluster_fh(&fh, stat, nlm4err);
+    nlm4_map_fh_to_volume(cs->nfs3state, fh, req, vol, stat, nlm4err);
+
+    if (nlm_grace_period) {
+        gf_msg(GF_NLM, GF_LOG_WARNING, 0, NFS_MSG_NLM_GRACE_PERIOD,
+               "NLM in grace period");
+        stat = nlm4_denied_grace_period;
+        nlm4_generic_reply(req, cs->args.nlm4_unlockargs.cookie, stat);
+        nfs3_call_state_wipe(cs);
+        return 0;
+    }
 
-        cs->vol = vol;
-        nlm4_volume_started_check (nfs3, vol, ret, rpcerr);
+    cs->vol = vol;
+    nlm4_volume_started_check(nfs3, vol, ret, rpcerr);
 
-        ret = nfs3_fh_resolve_and_resume (cs, &fh,
-                                          NULL, nlm4_cancel_resume);
+    ret = nfs3_fh_resolve_and_resume(cs, &fh, NULL, nlm4_cancel_resume);
 
 nlm4err:
-        if (ret < 0) {
-                gf_log (GF_NLM, GF_LOG_ERROR, "unable to resolve and resume");
-                nlm4_generic_reply (cs->req, cs->args.nlm4_cancargs.cookie,
-                                    stat);
-                nfs3_call_state_wipe (cs);
-                return 0;
-        }
+    if (ret < 0) {
+        gf_msg(GF_NLM, GF_LOG_ERROR, -ret, NFS_MSG_RESOLVE_ERROR,
+               "unable to resolve and resume");
+        nlm4_generic_reply(cs->req, cs->args.nlm4_cancargs.cookie, stat);
+        nfs3_call_state_wipe(cs);
+        return 0;
+    }
 
 rpcerr:
-        if (ret < 0) {
-                nfs3_call_state_wipe (cs);
-        }
-        return ret;
+    if (ret < 0) {
+        nfs3_call_state_wipe(cs);
+    }
+    return ret;
 }
 
 int
-nlm4svc_unlock_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                    int32_t op_ret, int32_t op_errno, struct gf_flock *flock,
-                    dict_t *xdata)
-{
-        nlm4_stats                      stat = nlm4_denied;
-        nfs3_call_state_t              *cs = NULL;
-
-        cs = frame->local;
-        if (op_ret == -1) {
-                stat = nlm4_errno_to_nlm4stat (op_errno);
-                goto err;
-        } else {
-                stat = nlm4_granted;
-                if (flock->l_type == F_UNLCK)
-                        nlm_search_and_delete (cs->fd,
-                                               cs->args.nlm4_unlockargs.alock.caller_name);
-        }
+nlm4svc_unlock_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                   int32_t op_ret, int32_t op_errno, struct gf_flock *flock,
+                   dict_t *xdata)
+{
+    nlm4_stats stat = nlm4_denied;
+    nfs3_call_state_t *cs = NULL;
+
+    cs = GF_REF_GET((nfs3_call_state_t *)frame->local);
+    if (op_ret == -1) {
+        stat = nlm4_errno_to_nlm4stat(op_errno);
+        goto err;
+    } else {
+        stat = nlm4_granted;
+        if (flock->l_type == F_UNLCK)
+            nlm_search_and_delete(cs->fd, &cs->args.nlm4_unlockargs.alock);
+    }
 
 err:
-        nlm4_generic_reply (cs->req, cs->args.nlm4_unlockargs.cookie, stat);
-        nfs3_call_state_wipe (cs);
-        return 0;
+    nlm4_generic_reply(cs->req, cs->args.nlm4_unlockargs.cookie, stat);
+    GF_REF_PUT(cs);
+    return 0;
 }
 
 int
-nlm4_unlock_fd_resume (void *carg)
-{
-        int                             ret = -EFAULT;
-        nfs_user_t                      nfu = {0, };
-        nfs3_call_state_t               *cs = NULL;
-        struct gf_flock                 flock = {0, };
-
-        if (!carg)
-                return ret;
-        cs = (nfs3_call_state_t *)carg;
-        nfs_request_user_init (&nfu, cs->req);
-        nlm4_lock_to_gf_flock (&flock, &cs->args.nlm4_unlockargs.alock, 0);
-        nlm_copy_lkowner (&nfu.lk_owner, &cs->args.nlm4_unlockargs.alock.oh);
-        flock.l_type = F_UNLCK;
-        ret = nfs_lk (cs->nfsx, cs->vol, &nfu, cs->fd, F_SETLK,
-                      &flock, nlm4svc_unlock_cbk, cs);
-
+nlm4_unlock_fd_resume(void *carg)
+{
+    int ret = -EFAULT;
+    nfs_user_t nfu = {
+        0,
+    };
+    nfs3_call_state_t *cs = NULL;
+    struct gf_flock flock = {
+        0,
+    };
+
+    if (!carg)
         return ret;
+    cs = GF_REF_GET((nfs3_call_state_t *)carg);
+    nfs_request_user_init(&nfu, cs->req);
+    nlm4_lock_to_gf_flock(&flock, &cs->args.nlm4_unlockargs.alock, 0);
+    nlm_copy_lkowner(&nfu.lk_owner, &cs->args.nlm4_unlockargs.alock.oh);
+    flock.l_type = F_UNLCK;
+    ret = nfs_lk(cs->nfsx, cs->vol, &nfu, cs->fd, F_SETLK, &flock,
+                 nlm4svc_unlock_cbk, cs);
+
+    GF_REF_PUT(cs);
+
+    return ret;
 }
 
 int
-nlm4_unlock_resume (void *carg)
+nlm4_unlock_resume(void *carg)
 {
-        nlm4_stats                      stat = nlm4_failed;
-        int                             ret = -1;
-        nfs3_call_state_t               *cs = NULL;
-        nlm_client_t                    *nlmclnt = NULL;
+    nlm4_stats stat = nlm4_failed;
+    int ret = -1;
+    nfs3_call_state_t *cs = NULL;
+    nlm_client_t *nlmclnt = NULL;
+    char *caller_name = NULL;
 
-        if (!carg)
-                return ret;
+    if (!carg)
+        return ret;
 
-        cs = (nfs3_call_state_t *)carg;
-        nlm4_check_fh_resolve_status (cs, stat, nlm4err);
-
-        nlmclnt = nlm_get_uniq (cs->args.nlm4_unlockargs.alock.caller_name);
-        if (nlmclnt == NULL) {
-                stat = nlm4_granted;
-                gf_log (GF_NLM, GF_LOG_WARNING, "nlm_get_uniq() returned NULL");
-                goto nlm4err;
-        }
-        cs->fd = fd_lookup_uint64 (cs->resolvedloc.inode, (uint64_t)nlmclnt);
-        if (cs->fd == NULL) {
-                stat = nlm4_granted;
-                gf_log (GF_NLM, GF_LOG_WARNING, "fd_lookup_uint64() returned "
-                        "NULL");
-                goto nlm4err;
-        }
-        ret = nlm4_unlock_fd_resume (cs);
+    cs = GF_REF_GET((nfs3_call_state_t *)carg);
+    nlm4_check_fh_resolve_status(cs, stat, nlm4err);
+    caller_name = cs->args.nlm4_unlockargs.alock.caller_name;
+
+    nlmclnt = nlm_get_uniq(caller_name);
+    if (nlmclnt == NULL) {
+        stat = nlm4_granted;
+        gf_msg(GF_NLM, GF_LOG_WARNING, ENOLCK, NFS_MSG_NO_MEMORY,
+               "nlm_get_uniq() returned NULL for %s", caller_name);
+        goto nlm4err;
+    }
+    cs->fd = fd_lookup_uint64(cs->resolvedloc.inode,
+                              (uint64_t)(uintptr_t)nlmclnt);
+    if (cs->fd == NULL) {
+        stat = nlm4_granted;
+        gf_msg(GF_NLM, GF_LOG_WARNING, 0, NFS_MSG_FD_LOOKUP_NULL,
+               "fd_lookup_uint64() returned NULL");
+        goto nlm4err;
+    }
+    ret = nlm4_unlock_fd_resume(cs);
 
 nlm4err:
-        if (ret < 0) {
-                gf_log (GF_NLM, GF_LOG_WARNING, "unable to unlock_fd_resume");
-                stat = nlm4_errno_to_nlm4stat (-ret);
-                nlm4_generic_reply (cs->req, cs->args.nlm4_unlockargs.cookie,
-                                    stat);
+    if (ret < 0) {
+        gf_msg(GF_NLM, GF_LOG_WARNING, -ret, NFS_MSG_LOCK_FAIL,
+               "unable to unlock_fd_resume");
+        stat = nlm4_errno_to_nlm4stat(-ret);
+        nlm4_generic_reply(cs->req, cs->args.nlm4_unlockargs.cookie, stat);
 
-                nfs3_call_state_wipe (cs);
-        }
-        /* we have already taken care of cleanup */
-        return 0;
+        nfs3_call_state_wipe(cs);
+    }
+
+    GF_REF_PUT(cs);
+
+    /* we have already taken care of cleanup */
+    return 0;
 }
 
 int
-nlm4svc_unlock (rpcsvc_request_t *req)
-{
-        xlator_t                        *vol = NULL;
-        nlm4_stats                      stat = nlm4_failed;
-        struct nfs_state               *nfs = NULL;
-        nfs3_state_t                   *nfs3 = NULL;
-        nfs3_call_state_t               *cs = NULL;
-        int                             ret = RPCSVC_ACTOR_ERROR;
-        struct nfs3_fh                  fh = {{0}, };
-
-        if (!req)
-                return ret;
-
-        nlm4_validate_nfs3_state (req, nfs3, stat, rpcerr, ret);
-        nfs = nfs_state (nfs3->nfsx);
-        nlm4_handle_call_state_init (nfs->nfs3state, cs, req,
-                                     stat, rpcerr);
-
-        nlm4_prep_nlm4_unlockargs (&cs->args.nlm4_unlockargs, &fh, &cs->lkowner,
-                                   cs->cookiebytes);
-        if (xdr_to_nlm4_unlockargs(req->msg[0], &cs->args.nlm4_unlockargs) <= 0)
-        {
-                gf_log (GF_NLM, GF_LOG_ERROR, "Error decoding args");
-                rpcsvc_request_seterr (req, GARBAGE_ARGS);
-                goto rpcerr;
-        }
-
-        nlm4_validate_gluster_fh (&fh, stat, nlm4err);
-        nlm4_map_fh_to_volume (cs->nfs3state, fh, req, vol, stat, nlm4err);
+nlm4svc_unlock(rpcsvc_request_t *req)
+{
+    xlator_t *vol = NULL;
+    nlm4_stats stat = nlm4_failed;
+    struct nfs_state *nfs = NULL;
+    nfs3_state_t *nfs3 = NULL;
+    nfs3_call_state_t *cs = NULL;
+    int ret = RPCSVC_ACTOR_ERROR;
+    struct nfs3_fh fh = {
+        {0},
+    };
+
+    if (!req)
+        return ret;
 
-        if (nlm_grace_period) {
-                gf_log (GF_NLM, GF_LOG_WARNING, "NLM in grace period");
-                stat = nlm4_denied_grace_period;
-                nlm4_generic_reply (req, cs->args.nlm4_unlockargs.cookie, stat);
-                nfs3_call_state_wipe (cs);
-                return 0;
-        }
+    nlm4_validate_nfs3_state(req, nfs3, stat, rpcerr, ret);
+    nfs = nfs_state(nfs3->nfsx);
+    nlm4_handle_call_state_init(nfs->nfs3state, cs, req, stat, rpcerr);
+
+    nlm4_prep_nlm4_unlockargs(&cs->args.nlm4_unlockargs, &fh, &cs->lkowner,
+                              cs->cookiebytes);
+    if (xdr_to_nlm4_unlockargs(req->msg[0], &cs->args.nlm4_unlockargs) <= 0) {
+        gf_msg(GF_NLM, GF_LOG_ERROR, errno, NFS_MSG_ARGS_DECODE_ERROR,
+               "Error decoding args");
+        rpcsvc_request_seterr(req, GARBAGE_ARGS);
+        goto rpcerr;
+    }
+
+    nlm4_validate_gluster_fh(&fh, stat, nlm4err);
+    nlm4_map_fh_to_volume(cs->nfs3state, fh, req, vol, stat, nlm4err);
+
+    if (nlm_grace_period) {
+        gf_msg(GF_NLM, GF_LOG_WARNING, 0, NFS_MSG_NLM_GRACE_PERIOD,
+               "NLM in grace period");
+        stat = nlm4_denied_grace_period;
+        nlm4_generic_reply(req, cs->args.nlm4_unlockargs.cookie, stat);
+        nfs3_call_state_wipe(cs);
+        return 0;
+    }
 
-        cs->vol = vol;
-        /* FIXME: check if trans is being used at all for unlock */
-        cs->trans = rpcsvc_request_transport_ref(req);
-        nlm4_volume_started_check (nfs3, vol, ret, rpcerr);
+    cs->vol = vol;
+    /* FIXME: check if trans is being used at all for unlock */
+    cs->trans = rpcsvc_request_transport_ref(req);
+    nlm4_volume_started_check(nfs3, vol, ret, rpcerr);
 
-        ret = nfs3_fh_resolve_and_resume (cs, &fh,
-                                          NULL, nlm4_unlock_resume);
+    ret = nfs3_fh_resolve_and_resume(cs, &fh, NULL, nlm4_unlock_resume);
 
 nlm4err:
-        if (ret < 0) {
-                gf_log (GF_NLM, GF_LOG_ERROR, "unable to resolve and resume");
-                nlm4_generic_reply (req, cs->args.nlm4_unlockargs.cookie, stat);
-                nfs3_call_state_wipe (cs);
-                return 0;
-        }
+    if (ret < 0) {
+        gf_msg(GF_NLM, GF_LOG_ERROR, -ret, NFS_MSG_RESOLVE_ERROR,
+               "unable to resolve and resume");
+        nlm4_generic_reply(req, cs->args.nlm4_unlockargs.cookie, stat);
+        nfs3_call_state_wipe(cs);
+        return 0;
+    }
 
 rpcerr:
-        if (ret < 0) {
-                nfs3_call_state_wipe (cs);
-        }
-        return ret;
+    if (ret < 0) {
+        nfs3_call_state_wipe(cs);
+    }
+    return ret;
 }
 
 int
-nlm4_share_reply (nfs3_call_state_t *cs, nlm4_stats stat)
+nlm4_share_reply(nfs3_call_state_t *cs, nlm4_stats stat)
 {
-        nlm4_shareres  res = {{0}, 0, 0};
+    nlm4_shareres res = {{0}, 0, 0};
 
-        if (!cs)
-                return -1;
+    if (!cs)
+        return -1;
 
-        res.cookie = cs->args.nlm4_shareargs.cookie;
-        res.stat = stat;
-        res.sequence = 0;
+    res.cookie = cs->args.nlm4_shareargs.cookie;
+    res.stat = stat;
+    res.sequence = 0;
 
-        nlm4svc_submit_reply (cs->req, (void *)&res,
-                              (nlm4_serializer)xdr_serialize_nlm4_shareres);
-        return 0;
+    nlm4svc_submit_reply(cs->req, (void *)&res,
+                         (nlm4_serializer)xdr_serialize_nlm4_shareres);
+    return 0;
 }
 
 nlm_share_t *
-nlm4_share_new ()
+nlm4_share_new()
 {
-        nlm_share_t      *share     = NULL;
+    nlm_share_t *share = NULL;
 
-        share = GF_CALLOC (1, sizeof (nlm_share_t),
-                           gf_nfs_mt_nlm4_share);
-        if (!share)
-                goto out;
+    share = GF_CALLOC(1, sizeof(nlm_share_t), gf_nfs_mt_nlm4_share);
+    if (!share)
+        goto out;
 
-        INIT_LIST_HEAD (&share->client_list);
-        INIT_LIST_HEAD (&share->inode_list);
- out:
-        return share;
+    INIT_LIST_HEAD(&share->client_list);
+    INIT_LIST_HEAD(&share->inode_list);
+out:
+    return share;
 }
 
 int
-nlm4_add_share_to_inode (nlm_share_t *share)
-{
-        int                    ret     = -1;
-        uint64_t               ctx     = 0;
-        struct list_head      *head    = NULL;
-        xlator_t              *this    = NULL;
-        inode_t               *inode   = NULL;
-        struct nfs_inode_ctx  *ictx    = NULL;
-        struct nfs_state      *priv    = NULL;
+nlm4_add_share_to_inode(nlm_share_t *share)
+{
+    int ret = -1;
+    uint64_t ctx = 0;
+    struct list_head *head = NULL;
+    xlator_t *this = NULL;
+    inode_t *inode = NULL;
+    struct nfs_inode_ctx *ictx = NULL;
+    struct nfs_state *priv = NULL;
+
+    this = THIS;
+    priv = this->private;
+    inode = share->inode;
+    ret = inode_ctx_get(inode, this, &ctx);
+
+    if (ret == -1) {
+        ictx = GF_CALLOC(1, sizeof(struct nfs_inode_ctx), gf_nfs_mt_inode_ctx);
+        if (!ictx) {
+            gf_msg(this->name, GF_LOG_ERROR, ENOMEM, NFS_MSG_NO_MEMORY,
+                   "could not allocate nfs inode ctx");
+            ret = -1;
+            goto out;
+        }
+        ictx->generation = priv->generation;
 
-        this = THIS;
-        priv = this->private;
-        inode = share->inode;
-        ret = inode_ctx_get (inode, this, &ctx);
-
-        if (ret == -1) {
-                ictx = GF_CALLOC (1, sizeof (struct nfs_inode_ctx),
-                                  gf_nfs_mt_inode_ctx);
-                if (!ictx ) {
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "could not allocate nfs inode ctx");
-                        ret = -1;
-                        goto out;
-                }
-                ictx->generation = priv->generation;
-
-                head = &ictx->shares;
-                INIT_LIST_HEAD (head);
-
-                ret = inode_ctx_put (inode, this, (uint64_t)ictx);
-                if (ret) {
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "could not store share list");
-                        goto out;
-                }
-        }
-        else {
-                ictx = (struct nfs_inode_ctx *)ctx;
-                head = &ictx->shares;
-        }
-
-        list_add (&share->inode_list, head);
-
- out:
-        if (ret && head)
-                GF_FREE (head);
-
-        return ret;
-}
-
-int
-nlm4_approve_share_reservation (nfs3_call_state_t *cs)
-{
-        int                       ret               = -1;
-        uint64_t                  ctx               = 0;
-        fsh_mode                  req_mode          = 0;
-        fsh_access                req_access        = 0;
-        inode_t                  *inode             = NULL;
-        nlm_share_t              *share             = NULL;
-        struct list_head         *head              = NULL;
-        struct nfs_inode_ctx     *ictx              = NULL;
-
-        if (!cs)
-                goto out;
-
-        inode = cs->resolvedloc.inode;
+        head = &ictx->shares;
+        INIT_LIST_HEAD(head);
 
-        ret = inode_ctx_get (inode, THIS, &ctx);
+        ret = inode_ctx_put(inode, this, (uint64_t)(uintptr_t)ictx);
         if (ret) {
-                ret = 0;
-                goto out;
+            gf_msg(this->name, GF_LOG_ERROR, 0, NFS_MSG_SHARE_LIST_STORE_FAIL,
+                   "could not store share list");
+            goto out;
         }
-        ictx = (struct nfs_inode_ctx *)ctx;
-
+    } else {
+        ictx = (struct nfs_inode_ctx *)(uintptr_t)ctx;
         head = &ictx->shares;
-        if (!head || list_empty (head))
-                goto out;
+    }
 
-        req_mode = cs->args.nlm4_shareargs.share.mode;
-        req_access = cs->args.nlm4_shareargs.share.access;
+    list_add(&share->inode_list, head);
 
-        list_for_each_entry (share, head, inode_list) {
-                ret = (((req_mode & share->access) == 0) &&
-                       ((req_access & share->mode) == 0));
-                if (!ret) {
-                        ret = -1;
-                        goto out;
-                }
-        }
-        ret = 0;
+out:
+    if (ret && head)
+        GF_FREE(head);
 
- out:
-         return ret;
+    return ret;
 }
 
 int
-nlm4_create_share_reservation (nfs3_call_state_t *cs)
+nlm4_approve_share_reservation(nfs3_call_state_t *cs)
 {
-        int                       ret        = -1;
-        nlm_share_t              *share      = NULL;
-        nlm_client_t             *client     = NULL;
-        inode_t                  *inode      = NULL;
+    int ret = -1;
+    uint64_t ctx = 0;
+    fsh_mode req_mode = 0;
+    fsh_access req_access = 0;
+    inode_t *inode = NULL;
+    nlm_share_t *share = NULL;
+    struct list_head *head = NULL;
+    struct nfs_inode_ctx *ictx = NULL;
 
-        LOCK (&nlm_client_list_lk);
+    if (!cs)
+        goto out;
 
-        inode = inode_ref (cs->resolvedloc.inode);
-        if (!inode) {
-                gf_log (GF_NLM, GF_LOG_ERROR, "inode not found");
-                goto out;
-        }
+    inode = cs->resolvedloc.inode;
 
-        client = __nlm_get_uniq (cs->args.nlm4_shareargs.share.caller_name);
-        if (!client) {
-                /* DO NOT add client. the client is supposed
-                   to be here, since nlm4svc_share adds it */
-                gf_log (GF_NLM, GF_LOG_ERROR, "client not found");
-                goto out;
-        }
+    ret = inode_ctx_get(inode, THIS, &ctx);
+    if (ret) {
+        ret = 0;
+        goto out;
+    }
+    ictx = (struct nfs_inode_ctx *)(uintptr_t)ctx;
 
-        ret = nlm4_approve_share_reservation (cs);
-        if (ret)
-                goto out;
+    head = &ictx->shares;
+    if (!head || list_empty(head))
+        goto out;
 
-        share = nlm4_share_new ();
-        if (!share) {
-                ret = -1;
-                goto out;
-        }
+    req_mode = cs->args.nlm4_shareargs.share.mode;
+    req_access = cs->args.nlm4_shareargs.share.access;
 
-        share->inode  = inode;
-        share->mode   = cs->args.nlm4_shareargs.share.mode;
-        share->access = cs->args.nlm4_shareargs.share.access;
-        nlm_copy_lkowner (&share->lkowner,
-                          &cs->args.nlm4_shareargs.share.oh);
+    list_for_each_entry(share, head, inode_list)
+    {
+        ret = (((req_mode & share->access) == 0) &&
+               ((req_access & share->mode) == 0));
+        if (!ret) {
+            ret = -1;
+            goto out;
+        }
+    }
+    ret = 0;
 
-        ret = nlm4_add_share_to_inode (share);
-        if (ret)
-                goto out;
+out:
+    return ret;
+}
 
-        list_add (&share->client_list, &client->shares);
+int
+nlm4_create_share_reservation(nfs3_call_state_t *cs)
+{
+    int ret = -1;
+    nlm_share_t *share = NULL;
+    nlm_client_t *client = NULL;
+    inode_t *inode = NULL;
+
+    LOCK(&nlm_client_list_lk);
+
+    inode = inode_ref(cs->resolvedloc.inode);
+    if (!inode) {
+        gf_msg(GF_NLM, GF_LOG_ERROR, 0, NFS_MSG_INODE_NOT_FOUND,
+               "inode not found");
+        goto out;
+    }
+
+    client = __nlm_get_uniq(cs->args.nlm4_shareargs.share.caller_name);
+    if (!client) {
+        /* DO NOT add client. the client is supposed
+           to be here, since nlm4svc_share adds it */
+        gf_msg(GF_NLM, GF_LOG_ERROR, 0, NFS_MSG_CLIENT_NOT_FOUND,
+               "client not found");
+        goto out;
+    }
+
+    ret = nlm4_approve_share_reservation(cs);
+    if (ret)
+        goto out;
+
+    share = nlm4_share_new();
+    if (!share) {
+        ret = -1;
+        goto out;
+    }
+
+    share->inode = inode;
+    share->mode = cs->args.nlm4_shareargs.share.mode;
+    share->access = cs->args.nlm4_shareargs.share.access;
+    nlm_copy_lkowner(&share->lkowner, &cs->args.nlm4_shareargs.share.oh);
+
+    ret = nlm4_add_share_to_inode(share);
+    if (ret)
+        goto out;
+
+    list_add(&share->client_list, &client->shares);
 
- out:
-        if (ret && inode) {
-                inode_unref (inode);
-                GF_FREE (share);
-        }
+out:
+    if (ret && inode) {
+        inode_unref(inode);
+        GF_FREE(share);
+    }
 
-        UNLOCK (&nlm_client_list_lk);
-        return ret;
+    UNLOCK(&nlm_client_list_lk);
+    return ret;
 }
 
 /*
@@ -1972,581 +2106,681 @@ nlm4_create_share_reservation (nfs3_call_state_t *cs)
   at *nfs xlator level only*, in memory
 */
 int
-nlm4_share_resume (void *call_state)
+nlm4_share_resume(void *call_state)
 {
-        int                             ret             = -1;
-        nlm4_stats                      stat            = nlm4_failed;
-        nfs3_call_state_t              *cs              = NULL;
+    int ret = -1;
+    nlm4_stats stat = nlm4_failed;
+    nfs3_call_state_t *cs = NULL;
 
-        if (!call_state)
-                return ret;
+    if (!call_state)
+        return ret;
 
-        cs = (nfs3_call_state_t *)call_state;
-        nlm4_check_fh_resolve_status (cs, stat, out);
+    cs = (nfs3_call_state_t *)call_state;
+    nlm4_check_fh_resolve_status(cs, stat, out);
 
-        ret = nlm4_create_share_reservation (cs);
-        if (!ret)
-                stat = nlm4_granted;
+    ret = nlm4_create_share_reservation(cs);
+    if (!ret)
+        stat = nlm4_granted;
 
- out:
-        nlm4_share_reply (cs, stat);
-        nfs3_call_state_wipe (cs);
-        return 0;
+out:
+    nlm4_share_reply(cs, stat);
+    nfs3_call_state_wipe(cs);
+    return 0;
 }
 
 int
-nlm4svc_share (rpcsvc_request_t *req)
-{
-        nlm4_stats                    stat      = nlm4_failed;
-        xlator_t                     *vol       = NULL;
-        nfs3_state_t                 *nfs3      = NULL;
-        nfs3_call_state_t            *cs        = NULL;
-        struct nfs_state             *nfs       = NULL;
-        struct nfs3_fh                fh        = {{0}, };
-        int                           ret       = RPCSVC_ACTOR_ERROR;
-
-        if (!req)
-                return ret;
-
-        nlm4_validate_nfs3_state (req, nfs3, stat, rpcerr, ret);
-        nfs = nfs_state (nfs3->nfsx);
-        nlm4_handle_call_state_init (nfs->nfs3state, cs, req,
-                                     stat, rpcerr);
-
-        nlm4_prep_shareargs (&cs->args.nlm4_shareargs, &cs->lockfh,
-                             &cs->lkowner, cs->cookiebytes);
-
-        if (xdr_to_nlm4_shareargs (req->msg[0],
-                                   &cs->args.nlm4_shareargs) <= 0) {
-                gf_log (GF_NLM, GF_LOG_ERROR, "Error decoding SHARE args");
-                rpcsvc_request_seterr (req, GARBAGE_ARGS);
-                goto rpcerr;
-        }
+nlm4svc_share(rpcsvc_request_t *req)
+{
+    nlm4_stats stat = nlm4_failed;
+    xlator_t *vol = NULL;
+    nfs3_state_t *nfs3 = NULL;
+    nfs3_call_state_t *cs = NULL;
+    struct nfs_state *nfs = NULL;
+    struct nfs3_fh fh = {
+        {0},
+    };
+    int ret = RPCSVC_ACTOR_ERROR;
+
+    if (!req)
+        return ret;
 
-        fh = cs->lockfh;
-        nlm4_validate_gluster_fh (&fh, stat, nlm4err);
-        nlm4_map_fh_to_volume (cs->nfs3state, fh, req,
-                               vol, stat, nlm4err);
-
-        if (nlm_grace_period && !cs->args.nlm4_shareargs.reclaim) {
-                gf_log (GF_NLM, GF_LOG_DEBUG, "NLM in grace period");
-                stat = nlm4_denied_grace_period;
-                nlm4_share_reply (cs, stat);
-                nfs3_call_state_wipe (cs);
-                return 0;
-        }
+    nlm4_validate_nfs3_state(req, nfs3, stat, rpcerr, ret);
+    nfs = nfs_state(nfs3->nfsx);
+    nlm4_handle_call_state_init(nfs->nfs3state, cs, req, stat, rpcerr);
+
+    nlm4_prep_shareargs(&cs->args.nlm4_shareargs, &cs->lockfh, &cs->lkowner,
+                        cs->cookiebytes);
+
+    if (xdr_to_nlm4_shareargs(req->msg[0], &cs->args.nlm4_shareargs) <= 0) {
+        gf_msg(GF_NLM, GF_LOG_ERROR, errno, NFS_MSG_ARGS_DECODE_ERROR,
+               "Error decoding SHARE args");
+        rpcsvc_request_seterr(req, GARBAGE_ARGS);
+        goto rpcerr;
+    }
+
+    fh = cs->lockfh;
+    nlm4_validate_gluster_fh(&fh, stat, nlm4err);
+    nlm4_map_fh_to_volume(cs->nfs3state, fh, req, vol, stat, nlm4err);
+
+    if (nlm_grace_period && !cs->args.nlm4_shareargs.reclaim) {
+        gf_msg_debug(GF_NLM, 0, "NLM in grace period");
+        stat = nlm4_denied_grace_period;
+        nlm4_share_reply(cs, stat);
+        nfs3_call_state_wipe(cs);
+        return 0;
+    }
 
-        cs->vol = vol;
-        cs->trans = rpcsvc_request_transport_ref(req);
-        nlm4_volume_started_check (nfs3, vol, ret, rpcerr);
+    cs->vol = vol;
+    cs->trans = rpcsvc_request_transport_ref(req);
+    nlm4_volume_started_check(nfs3, vol, ret, rpcerr);
 
-        ret = nlm_add_nlmclnt (cs->args.nlm4_shareargs.share.caller_name);
+    ret = nlm_add_nlmclnt(cs->args.nlm4_shareargs.share.caller_name);
 
-        ret = nfs3_fh_resolve_and_resume (cs, &fh, NULL, nlm4_share_resume);
+    ret = nfs3_fh_resolve_and_resume(cs, &fh, NULL, nlm4_share_resume);
 
- nlm4err:
-        if (ret < 0) {
-                gf_log (GF_NLM, GF_LOG_ERROR, "SHARE call failed");
-                nlm4_share_reply (cs, stat);
-                nfs3_call_state_wipe (cs);
-                return 0;
-        }
+nlm4err:
+    if (ret < 0) {
+        gf_msg(GF_NLM, GF_LOG_ERROR, -ret, NFS_MSG_SHARE_CALL_FAIL,
+               "SHARE call failed");
+        nlm4_share_reply(cs, stat);
+        nfs3_call_state_wipe(cs);
+        return 0;
+    }
 
- rpcerr:
-        if (ret < 0)
-                nfs3_call_state_wipe (cs);
+rpcerr:
+    if (ret < 0)
+        nfs3_call_state_wipe(cs);
 
-        return ret;
+    return ret;
 }
 
 int
-nlm4_remove_share_reservation (nfs3_call_state_t *cs)
-{
-        int                      ret        = -1;
-        uint64_t                 ctx        = 0;
-        fsh_mode                 req_mode   = 0;
-        fsh_access               req_access = 0;
-        nlm_share_t             *share      = NULL;
-        nlm_share_t             *tmp        = NULL;
-        nlm_client_t            *client     = NULL;
-        char                    *caller     = NULL;
-        inode_t                 *inode      = NULL;
-        xlator_t                *this       = NULL;
-        struct list_head        *head       = NULL;
-        nlm4_shareargs          *args       = NULL;
-        struct nfs_inode_ctx    *ictx       = NULL;
-
-        LOCK (&nlm_client_list_lk);
-
-        args = &cs->args.nlm4_shareargs;
-        caller = args->share.caller_name;
-
-        client = __nlm_get_uniq (caller);
-        if (!client) {
-                gf_log (GF_NLM, GF_LOG_ERROR,
-                        "client not found: %s", caller);
-                goto out;
-        }
-
-        inode = cs->resolvedloc.inode;
-        if (!inode) {
-                gf_log (GF_NLM, GF_LOG_ERROR,
-                        "inode not found: client: %s", caller);
-                goto out;
-        }
-
-        this = THIS;
-        ret = inode_ctx_get (inode, this, &ctx);
+nlm4_remove_share_reservation(nfs3_call_state_t *cs)
+{
+    int ret = -1;
+    uint64_t ctx = 0;
+    fsh_mode req_mode = 0;
+    fsh_access req_access = 0;
+    nlm_share_t *share = NULL;
+    nlm_share_t *tmp = NULL;
+    nlm_client_t *client = NULL;
+    char *caller = NULL;
+    inode_t *inode = NULL;
+    xlator_t *this = NULL;
+    struct list_head *head = NULL;
+    nlm4_shareargs *args = NULL;
+    struct nfs_inode_ctx *ictx = NULL;
+
+    LOCK(&nlm_client_list_lk);
+
+    args = &cs->args.nlm4_shareargs;
+    caller = args->share.caller_name;
+
+    client = __nlm_get_uniq(caller);
+    if (!client) {
+        gf_msg(GF_NLM, GF_LOG_ERROR, 0, NFS_MSG_CLIENT_NOT_FOUND,
+               "client not found: %s", caller);
+        goto out;
+    }
+
+    inode = cs->resolvedloc.inode;
+    if (!inode) {
+        gf_msg(GF_NLM, GF_LOG_ERROR, 0, NFS_MSG_INODE_NOT_FOUND,
+               "inode not found: client: %s", caller);
+        goto out;
+    }
+
+    this = THIS;
+    ret = inode_ctx_get(inode, this, &ctx);
+    if (ret) {
+        gf_msg(GF_NLM, GF_LOG_ERROR, 0, NFS_MSG_INODE_SHARES_NOT_FOUND,
+               "no shares found for inode:"
+               "gfid: %s; client: %s",
+               inode->gfid, caller);
+        goto out;
+    }
+    ictx = (struct nfs_inode_ctx *)(uintptr_t)ctx;
+
+    head = &ictx->shares;
+    if (list_empty(head)) {
+        ret = -1;
+        goto out;
+    }
+
+    ret = 0;
+    req_mode = args->share.mode;
+    req_access = args->share.access;
+
+    list_for_each_entry_safe(share, tmp, head, inode_list)
+    {
+        ret = ((req_mode == share->mode) && (req_access == share->access) &&
+               nlm_is_oh_same_lkowner(&share->lkowner, &args->share.oh));
         if (ret) {
-                gf_log (GF_NLM, GF_LOG_ERROR,
-                        "no shares found for inode:"
-                        "gfid: %s; client: %s",
-                        inode->gfid, caller);
-                goto out;
-        }
-        ictx = (struct nfs_inode_ctx *)ctx;
-
-        head = &ictx->shares;
-        if (list_empty (head)) {
-                ret = -1;
-                goto out;
+            list_del(&share->client_list);
+            list_del(&share->inode_list);
+            inode_unref(share->inode);
+            GF_FREE(share);
+            break;
         }
+    }
 
-        ret = 0;
-        req_mode = args->share.mode;
-        req_access = args->share.access;
-
-        list_for_each_entry_safe (share, tmp, head, inode_list) {
-                ret = ((req_mode == share->mode) &&
-                       (req_access == share->access) &&
-                       nlm_is_oh_same_lkowner (&share->lkowner, &args->share.oh));
-                if (ret) {
-                        list_del (&share->client_list);
-                        list_del (&share->inode_list);
-                        inode_unref (share->inode);
-                        GF_FREE (share);
-                        break;
-                }
-        }
-
-        ret = 0;
- out:
-        UNLOCK (&nlm_client_list_lk);
-        return ret;
-
+    ret = 0;
+out:
+    UNLOCK(&nlm_client_list_lk);
+    return ret;
 }
 
 int
-nlm4_unshare_resume (void *call_state)
+nlm4_unshare_resume(void *call_state)
 {
-        int                ret        = -1;
-        nlm4_stats         stat       = nlm4_failed;
-        nfs3_call_state_t *cs         = NULL;
+    int ret = -1;
+    nlm4_stats stat = nlm4_failed;
+    nfs3_call_state_t *cs = NULL;
 
-        if (!call_state)
-                return ret;
+    if (!call_state)
+        return ret;
 
-        cs = (nfs3_call_state_t *)call_state;
+    cs = (nfs3_call_state_t *)call_state;
 
-        nlm4_check_fh_resolve_status (cs, stat, out);
-        ret = nlm4_remove_share_reservation (cs);
-        if (!ret)
-                stat = nlm4_granted;
+    nlm4_check_fh_resolve_status(cs, stat, out);
+    ret = nlm4_remove_share_reservation(cs);
+    if (!ret)
+        stat = nlm4_granted;
 
- out:
-        nlm4_share_reply (cs, stat);
-        nfs3_call_state_wipe (cs);
-        return 0;
+out:
+    nlm4_share_reply(cs, stat);
+    nfs3_call_state_wipe(cs);
+    return 0;
 }
 
 int
-nlm4svc_unshare (rpcsvc_request_t *req)
-{
-        nlm4_stats                    stat      = nlm4_failed;
-        xlator_t                     *vol       = NULL;
-        nfs3_state_t                 *nfs3      = NULL;
-        nfs3_call_state_t            *cs        = NULL;
-        struct nfs_state             *nfs       = NULL;
-        struct nfs3_fh                fh        = {{0}, };
-        int                           ret       = RPCSVC_ACTOR_ERROR;
-
-        if (!req)
-                return ret;
-
-        nlm4_validate_nfs3_state (req, nfs3, stat, rpcerr, ret);
-        nfs = nfs_state (nfs3->nfsx);
-        nlm4_handle_call_state_init (nfs->nfs3state, cs, req,
-                                     stat, rpcerr);
-
-        nlm4_prep_shareargs (&cs->args.nlm4_shareargs, &cs->lockfh,
-                             &cs->lkowner, cs->cookiebytes);
-
-        if (xdr_to_nlm4_shareargs (req->msg[0],
-                                   &cs->args.nlm4_shareargs) <= 0) {
-                gf_log (GF_NLM, GF_LOG_ERROR, "Error decoding UNSHARE args");
-                rpcsvc_request_seterr (req, GARBAGE_ARGS);
-                goto rpcerr;
-        }
+nlm4svc_unshare(rpcsvc_request_t *req)
+{
+    nlm4_stats stat = nlm4_failed;
+    xlator_t *vol = NULL;
+    nfs3_state_t *nfs3 = NULL;
+    nfs3_call_state_t *cs = NULL;
+    struct nfs_state *nfs = NULL;
+    struct nfs3_fh fh = {
+        {0},
+    };
+    int ret = RPCSVC_ACTOR_ERROR;
+
+    if (!req)
+        return ret;
 
-        fh = cs->lockfh;
-        nlm4_validate_gluster_fh (&fh, stat, nlm4err);
-        nlm4_map_fh_to_volume (cs->nfs3state, fh, req,
-                               vol, stat, nlm4err);
-
-        if (nlm_grace_period && !cs->args.nlm4_shareargs.reclaim) {
-                gf_log (GF_NLM, GF_LOG_DEBUG, "NLM in grace period");
-                stat = nlm4_denied_grace_period;
-                nlm4_share_reply (cs, stat);
-                nfs3_call_state_wipe (cs);
-                return 0;
-        }
+    nlm4_validate_nfs3_state(req, nfs3, stat, rpcerr, ret);
+    nfs = nfs_state(nfs3->nfsx);
+    nlm4_handle_call_state_init(nfs->nfs3state, cs, req, stat, rpcerr);
+
+    nlm4_prep_shareargs(&cs->args.nlm4_shareargs, &cs->lockfh, &cs->lkowner,
+                        cs->cookiebytes);
+
+    if (xdr_to_nlm4_shareargs(req->msg[0], &cs->args.nlm4_shareargs) <= 0) {
+        gf_msg(GF_NLM, GF_LOG_ERROR, errno, NFS_MSG_ARGS_DECODE_ERROR,
+               "Error decoding UNSHARE args");
+        rpcsvc_request_seterr(req, GARBAGE_ARGS);
+        goto rpcerr;
+    }
+
+    fh = cs->lockfh;
+    nlm4_validate_gluster_fh(&fh, stat, nlm4err);
+    nlm4_map_fh_to_volume(cs->nfs3state, fh, req, vol, stat, nlm4err);
+
+    if (nlm_grace_period && !cs->args.nlm4_shareargs.reclaim) {
+        gf_msg_debug(GF_NLM, 0, "NLM in grace period");
+        stat = nlm4_denied_grace_period;
+        nlm4_share_reply(cs, stat);
+        nfs3_call_state_wipe(cs);
+        return 0;
+    }
 
-        cs->vol = vol;
-        cs->trans = rpcsvc_request_transport_ref(req);
-        nlm4_volume_started_check (nfs3, vol, ret, rpcerr);
+    cs->vol = vol;
+    cs->trans = rpcsvc_request_transport_ref(req);
+    nlm4_volume_started_check(nfs3, vol, ret, rpcerr);
 
-        ret = nfs3_fh_resolve_and_resume (cs, &fh, NULL,
-                                          nlm4_unshare_resume);
+    ret = nfs3_fh_resolve_and_resume(cs, &fh, NULL, nlm4_unshare_resume);
 
- nlm4err:
-        if (ret < 0) {
-                gf_log (GF_NLM, GF_LOG_ERROR, "UNSHARE call failed");
-                nlm4_share_reply (cs, stat);
-                ret = 0;
-                return 0;
-        }
+nlm4err:
+    if (ret < 0) {
+        gf_msg(GF_NLM, GF_LOG_ERROR, -ret, NFS_MSG_UNSHARE_CALL_FAIL,
+               "UNSHARE call failed");
+        nlm4_share_reply(cs, stat);
+        ret = 0;
+        return 0;
+    }
+
+rpcerr:
+    if (ret < 0)
+        nfs3_call_state_wipe(cs);
 
- rpcerr:
-        if (ret < 0)
-                nfs3_call_state_wipe (cs);
+    return ret;
+}
 
-        return ret;
+int
+nlm4_free_all_shares(char *caller_name)
+{
+    nlm_share_t *share = NULL;
+    nlm_share_t *tmp = NULL;
+    nlm_client_t *client = NULL;
+
+    LOCK(&nlm_client_list_lk);
+
+    client = __nlm_get_uniq(caller_name);
+    if (!client) {
+        gf_msg_debug(GF_NLM, 0, "client not found: %s", caller_name);
+        goto out;
+    }
+
+    list_for_each_entry_safe(share, tmp, &client->shares, client_list)
+    {
+        list_del(&share->inode_list);
+        list_del(&share->client_list);
+        inode_unref(share->inode);
+        GF_FREE(share);
+    }
+out:
+    UNLOCK(&nlm_client_list_lk);
+    return 0;
 }
 
 int
-nlm4_free_all_shares (char *caller_name)
+nlm4svc_free_all(rpcsvc_request_t *req)
 {
-        nlm_share_t             *share      = NULL;
-        nlm_share_t             *tmp        = NULL;
-        nlm_client_t            *client     = NULL;
+    int ret = RPCSVC_ACTOR_ERROR;
+    nlm4_stats stat = nlm4_failed;
+    nfs3_state_t *nfs3 = NULL;
+    nfs3_call_state_t *cs = NULL;
+    struct nfs_state *nfs = NULL;
 
-        LOCK (&nlm_client_list_lk);
+    nlm4_validate_nfs3_state(req, nfs3, stat, err, ret);
+    nfs = nfs_state(nfs3->nfsx);
+    nlm4_handle_call_state_init(nfs->nfs3state, cs, req, stat, err);
 
-        client = __nlm_get_uniq (caller_name);
-        if (!client) {
-                gf_log (GF_NLM, GF_LOG_DEBUG,
-                        "client not found: %s", caller_name);
-                goto out;
-        }
+    nlm4_prep_freeallargs(&cs->args.nlm4_freeallargs, &cs->lkowner);
 
-        list_for_each_entry_safe (share, tmp, &client->shares, client_list) {
-                list_del (&share->inode_list);
-                list_del (&share->client_list);
-                inode_unref (share->inode);
-                GF_FREE (share);
-        }
- out:
-        UNLOCK (&nlm_client_list_lk);
-        return 0;
+    if (xdr_to_nlm4_freeallargs(req->msg[0], &cs->args.nlm4_freeallargs) <= 0) {
+        gf_msg(GF_NLM, GF_LOG_ERROR, errno, NFS_MSG_ARGS_DECODE_ERROR,
+               "Error decoding FREE_ALL args");
+        rpcsvc_request_seterr(req, GARBAGE_ARGS);
+        goto err;
+    }
+
+    ret = nlm4_free_all_shares(cs->args.nlm4_freeallargs.name);
+    if (ret)
+        goto err;
+
+    ret = nlm_cleanup_fds(cs->args.nlm4_freeallargs.name);
+    if (ret)
+        goto err;
+
+err:
+    nfs3_call_state_wipe(cs);
+    if (ret)
+        gf_msg_debug(GF_NLM, 0, "error in free all; stat: %d", stat);
+    return ret;
 }
 
-int
-nlm4svc_free_all (rpcsvc_request_t *req)
-{
-        int                           ret       = RPCSVC_ACTOR_ERROR;
-        nlm4_stats                    stat      = nlm4_failed;
-        nfs3_state_t                 *nfs3      = NULL;
-        nfs3_call_state_t            *cs        = NULL;
-        struct nfs_state             *nfs       = NULL;
-
-        nlm4_validate_nfs3_state (req, nfs3, stat, err, ret);
-        nfs = nfs_state (nfs3->nfsx);
-        nlm4_handle_call_state_init (nfs->nfs3state, cs,
-                                     req, stat, err);
-
-        nlm4_prep_freeallargs (&cs->args.nlm4_freeallargs,
-                               &cs->lkowner);
-
-        if (xdr_to_nlm4_freeallargs (req->msg[0],
-                                     &cs->args.nlm4_freeallargs) <= 0) {
-                gf_log (GF_NLM, GF_LOG_ERROR, "Error decoding FREE_ALL args");
-                rpcsvc_request_seterr (req, GARBAGE_ARGS);
-                goto err;
-        }
+void
+nlm4svc_sm_notify(struct nlm_sm_status *status)
+{
+    gf_msg(GF_NLM, GF_LOG_INFO, 0, NFS_MSG_SM_NOTIFY,
+           "sm_notify: "
+           "%s, state: %d",
+           status->mon_name, status->state);
+    nlm_cleanup_fds(status->mon_name);
+}
+
+/* RPC_CLNT_CONNECT gets called on (re)connects and should be able to handle
+ * different NLM requests. */
+static int
+nlm_handle_connect(struct rpc_clnt *rpc_clnt, struct nlm4_notify_args *ncf)
+{
+    int ret = -1;
+    int nlm_proc = NLM4_NULL;
+    nfs3_call_state_t *cs = NULL;
+    struct nlm4_lock *alock = NULL;
+    char *caller_name = NULL;
+
+    cs = GF_REF_GET(ncf->cs);
+    if (!cs || !cs->req) {
+        gf_msg(GF_NLM, GF_LOG_ERROR, EINVAL, NFS_MSG_RPC_CLNT_ERROR,
+               "Spurious notify?!");
+        goto out;
+    }
+
+    /* NLM4_* actions from nlm4.h */
+    if (cs->req->prognum == NLM_PROGRAM) {
+        nlm_proc = cs->req->procnum;
+    } else {
+        /* hmm, cs->req has not been filled completely */
+        if (cs->resume_fn == nlm4_lock_fd_resume)
+            nlm_proc = NLM4_LOCK;
+        else if (cs->resume_fn == nlm4_cancel_fd_resume)
+            nlm_proc = NLM4_CANCEL;
+        else if (cs->resume_fn == nlm4_unlock_fd_resume)
+            nlm_proc = NLM4_UNLOCK;
+        else {
+            gf_msg(GF_NLM, GF_LOG_ERROR, 0, NFS_MSG_RPC_CLNT_ERROR,
+                   "(re)connect with an "
+                   "unexpected NLM4 procedure (%d)",
+                   nlm_proc);
+            goto out;
+        }
+    }
+
+    switch (nlm_proc) {
+        case NLM4_LOCK:
+            alock = &cs->args.nlm4_lockargs.alock;
+            caller_name = alock->caller_name;
+
+            ret = nlm_set_rpc_clnt(rpc_clnt, caller_name);
+            if (ret == -1) {
+                gf_msg(GF_NLM, GF_LOG_ERROR, 0, NFS_MSG_RPC_CLNT_ERROR,
+                       "Failed to set "
+                       "rpc clnt");
+                goto out;
+            }
 
-        ret = nlm4_free_all_shares (cs->args.nlm4_freeallargs.name);
-        if (ret)
-                goto err;
+            /* extra ref taken with nlm_set_rpc_clnt() */
+            rpc_clnt_unref(rpc_clnt);
 
-        ret = nlm_cleanup_fds (cs->args.nlm4_freeallargs.name);
-        if (ret)
-                goto err;
+            nlm4svc_send_granted(ncf);
+            break;
 
- err:
-        nfs3_call_state_wipe (cs);
-        if (ret)
-                gf_log (GF_NLM, GF_LOG_DEBUG,
-                        "error in free all; stat: %d", stat);
-        return ret;
+        case NLM4_CANCEL:
+            /* alock = &cs->args.nlm4_cancargs.alock; */
+            ret = nlm4svc_cancel(cs->req);
+            break;
 
-}
+        case NLM4_UNLOCK:
+            /* alock = &cs->args.nlm4_unlockargs.alock; */
+            ret = nlm4svc_unlock(cs->req);
+            break;
 
-void
-nlm4svc_sm_notify (struct nlm_sm_status *status)
-{
-        gf_log (GF_NLM, GF_LOG_INFO, "sm_notify: %s, state: %d",
-                status->mon_name,
-                status->state);
-        nlm_cleanup_fds (status->mon_name);
-}
-
-rpcsvc_actor_t  nlm4svc_actors[NLM4_PROC_COUNT] = {
-        /* 0 */
-        {"NULL",       NLM4_NULL,         nlm4svc_null,      NULL, 0, DRC_IDEMPOTENT},
-        {"TEST",       NLM4_TEST,         nlm4svc_test,      NULL, 0, DRC_IDEMPOTENT},
-        {"LOCK",       NLM4_LOCK,         nlm4svc_lock,      NULL, 0, DRC_NON_IDEMPOTENT},
-        {"CANCEL",     NLM4_CANCEL,       nlm4svc_cancel,    NULL, 0, DRC_NON_IDEMPOTENT},
-        {"UNLOCK",     NLM4_UNLOCK,       nlm4svc_unlock,    NULL, 0, DRC_NON_IDEMPOTENT},
-        /* 5 */
-        {"GRANTED",    NLM4_GRANTED,      NULL,              NULL, 0, DRC_NA},
-        {"TEST",       NLM4_TEST_MSG,     NULL,              NULL, 0, DRC_NA},
-        {"LOCK",       NLM4_LOCK_MSG,     NULL,              NULL, 0, DRC_NA},
-        {"CANCEL",     NLM4_CANCEL_MSG,   NULL,              NULL, 0, DRC_NA},
-        {"UNLOCK",     NLM4_UNLOCK_MSG,   NULL,              NULL, 0, DRC_NA},
-        /* 10 */
-        {"GRANTED",    NLM4_GRANTED_MSG,  NULL,              NULL, 0, DRC_NA},
-        {"TEST",       NLM4_TEST_RES,     NULL,              NULL, 0, DRC_NA},
-        {"LOCK",       NLM4_LOCK_RES,     NULL,              NULL, 0, DRC_NA},
-        {"CANCEL",     NLM4_CANCEL_RES,   NULL,              NULL, 0, DRC_NA},
-        {"UNLOCK",     NLM4_UNLOCK_RES,   NULL,              NULL, 0, DRC_NA},
-        /* 15 ; procedures 17,18,19 are not defined by nlm */
-        {"GRANTED",    NLM4_GRANTED_RES,  NULL,              NULL, 0, DRC_NA},
-        {"SM_NOTIFY",  NLM4_SM_NOTIFY,    NULL,              NULL, 0, DRC_NA},
-        {"SEVENTEEN",  NLM4_SEVENTEEN,    NULL,              NULL, 0, DRC_NA},
-        {"EIGHTEEN",   NLM4_EIGHTEEN,     NULL,              NULL, 0, DRC_NA},
-        {"NINETEEN",   NLM4_NINETEEN,     NULL,              NULL, 0, DRC_NA},
-        /* 20 */
-        {"SHARE",      NLM4_SHARE,        nlm4svc_share,     NULL, 0, DRC_NON_IDEMPOTENT},
-        {"UNSHARE",    NLM4_UNSHARE,      nlm4svc_unshare,   NULL, 0, DRC_NON_IDEMPOTENT},
-        {"NM_LOCK",    NLM4_NM_LOCK,      nlm4svc_nm_lock,   NULL, 0, DRC_NON_IDEMPOTENT},
-        {"FREE_ALL",   NLM4_FREE_ALL,     nlm4svc_free_all,  NULL, 0, DRC_IDEMPOTENT},
-};
+        default:
+            gf_msg(GF_NLM, GF_LOG_ERROR, 0, NFS_MSG_RPC_CLNT_ERROR,
+                   "(re)connect with an unexpected NLM4 procedure "
+                   "(%d)",
+                   nlm_proc);
+    }
 
-rpcsvc_program_t        nlm4prog = {
-        .progname       = "NLM4",
-        .prognum        = NLM_PROGRAM,
-        .progver        = NLM_V4,
-        .progport       = GF_NLM4_PORT,
-        .actors         = nlm4svc_actors,
-        .numactors      = NLM4_PROC_COUNT,
-        .min_auth       = AUTH_NULL,
+out:
+    if (cs)
+        GF_REF_PUT(cs);
+
+    return ret;
+}
+
+static rpcsvc_actor_t nlm4svc_actors[NLM4_PROC_COUNT] = {
+    /* 0 */
+    {"NULL", nlm4svc_null, NULL, NLM4_NULL, DRC_IDEMPOTENT, 0},
+    {"TEST", nlm4svc_test, NULL, NLM4_TEST, DRC_IDEMPOTENT, 0},
+    {"LOCK", nlm4svc_lock, NULL, NLM4_LOCK, DRC_IDEMPOTENT, 0},
+    {"CANCEL", nlm4svc_cancel, NULL, NLM4_CANCEL, DRC_NON_IDEMPOTENT, 0},
+    {"UNLOCK", nlm4svc_unlock, NULL, NLM4_UNLOCK, DRC_NON_IDEMPOTENT, 0},
+    /* 5 */
+    {"GRANTED", NULL, NULL, NLM4_GRANTED, DRC_NA, 0},
+    {"TEST", NULL, NULL, NLM4_TEST_MSG, DRC_NA, 0},
+    {"LOCK", NULL, NULL, NLM4_LOCK_MSG, DRC_NA, 0},
+    {"CANCEL", NULL, NULL, NLM4_CANCEL_MSG, DRC_NA, 0},
+    {"UNLOCK", NULL, NULL, NLM4_UNLOCK_MSG, DRC_NA, 0},
+    /* 10 */
+    {"GRANTED", NULL, NULL, NLM4_GRANTED_MSG, DRC_NA, 0},
+    {"TEST", NULL, NULL, NLM4_TEST_RES, DRC_NA, 0},
+    {"LOCK", NULL, NULL, NLM4_LOCK_RES, DRC_NA, 0},
+    {"CANCEL", NULL, NULL, NLM4_CANCEL_RES, DRC_NA, 0},
+    {"UNLOCK", NULL, NULL, NLM4_UNLOCK_RES, DRC_NA, 0},
+    /* 15 ; procedures 17,18,19 are not defined by nlm */
+    {"GRANTED", NULL, NULL, NLM4_GRANTED_RES, DRC_NA, 0},
+    {"SM_NOTIFY", NULL, NULL, NLM4_SM_NOTIFY, DRC_NA, 0},
+    {"SEVENTEEN", NULL, NULL, NLM4_SEVENTEEN, DRC_NA, 0},
+    {"EIGHTEEN", NULL, NULL, NLM4_EIGHTEEN, DRC_NA, 0},
+    {"NINETEEN", NULL, NULL, NLM4_NINETEEN, DRC_NA, 0},
+    /* 20 */
+    {"SHARE", nlm4svc_share, NULL, NLM4_SHARE, DRC_NON_IDEMPOTENT, 0},
+    {"UNSHARE", nlm4svc_unshare, NULL, NLM4_UNSHARE, DRC_NON_IDEMPOTENT, 0},
+    {"NM_LOCK", nlm4svc_nm_lock, NULL, NLM4_NM_LOCK, DRC_NON_IDEMPOTENT, 0},
+    {"FREE_ALL", nlm4svc_free_all, NULL, NLM4_FREE_ALL, DRC_IDEMPOTENT, 0},
 };
 
+static rpcsvc_program_t nlm4prog = {
+    .progname = "NLM4",
+    .prognum = NLM_PROGRAM,
+    .progver = NLM_V4,
+    .progport = GF_NLM4_PORT,
+    .actors = nlm4svc_actors,
+    .numactors = NLM4_PROC_COUNT,
+    .min_auth = AUTH_NULL,
+};
 
 int
-nlm4_init_state (xlator_t *nfsx)
+nlm4_init_state(xlator_t *nfsx)
 {
-        return 0;
+    return 0;
 }
 
-extern void *nsm_thread (void *argv);
+extern void *
+nsm_thread(void *argv);
 
-void nlm_grace_period_over(void *arg)
+void
+nlm_grace_period_over(void *arg)
 {
-        nlm_grace_period = 0;
+    nlm_grace_period = 0;
 }
 
 rpcsvc_program_t *
 nlm4svc_init(xlator_t *nfsx)
 {
-        struct nfs3_state *ns = NULL;
-        struct nfs_state *nfs = NULL;
-        dict_t *options = NULL;
-        int ret = -1;
-        char *portstr = NULL;
-        pthread_t thr;
-        struct timespec timeout = {0,};
-        FILE   *pidfile = NULL;
-        pid_t   pid     = -1;
-        static gf_boolean_t nlm4_inited = _gf_false;
-
-        /* Already inited */
-        if (nlm4_inited)
-                return &nlm4prog;
-
-        nfs = (struct nfs_state*)nfsx->private;
-
-        ns = nfs->nfs3state;
-        if (!ns) {
-                gf_log (GF_NLM, GF_LOG_ERROR, "NLM4 init failed");
-                goto err;
-        }
-        nlm4prog.private = ns;
-
-        options = dict_new ();
-
-        ret = gf_asprintf (&portstr, "%d", GF_NLM4_PORT);
-        if (ret == -1)
-                goto err;
-
-        ret = dict_set_dynstr (options, "transport.socket.listen-port",
-                               portstr);
-        if (ret == -1)
-                goto err;
-        ret = dict_set_str (options, "transport-type", "socket");
-        if (ret == -1) {
-                gf_log (GF_NLM, GF_LOG_ERROR, "dict_set_str error");
-                goto err;
-        }
-
-        if (nfs->allow_insecure) {
-                ret = dict_set_str (options, "rpc-auth-allow-insecure", "on");
-                if (ret == -1) {
-                        gf_log (GF_NLM, GF_LOG_ERROR, "dict_set_str error");
-                        goto err;
-                }
-                ret = dict_set_str (options, "rpc-auth.ports.insecure", "on");
-                if (ret == -1) {
-                        gf_log (GF_NLM, GF_LOG_ERROR, "dict_set_str error");
-                        goto err;
-                }
-        }
+    struct nfs3_state *ns = NULL;
+    struct nfs_state *nfs = NULL;
+    dict_t *options = NULL;
+    int ret = -1;
+    char *portstr = NULL;
+    pthread_t thr;
+    struct timespec timeout = {
+        0,
+    };
+    FILE *pidfile = NULL;
+    pid_t pid = -1;
+
+    /* Already inited */
+    if (nlm4_inited)
+        return &nlm4prog;
 
-        ret = dict_set_str (options, "transport.address-family", "inet");
+    nfs = (struct nfs_state *)nfsx->private;
+
+    ns = nfs->nfs3state;
+    if (!ns) {
+        gf_msg(GF_NLM, GF_LOG_ERROR, EINVAL, NFS_MSG_NLM_INIT_FAIL,
+               "NLM4 init failed");
+        goto err;
+    }
+    nlm4prog.private = ns;
+
+    options = dict_new();
+    if (options == NULL) {
+        gf_msg(GF_NLM, GF_LOG_ERROR, ENOMEM, NFS_MSG_GFID_DICT_CREATE_FAIL,
+               "dict allocation failed");
+        goto err;
+    }
+
+    ret = gf_asprintf(&portstr, "%d", GF_NLM4_PORT);
+    if (ret == -1)
+        goto err;
+
+    ret = dict_set_dynstr(options, "transport.socket.listen-port", portstr);
+    if (ret == -1)
+        goto err;
+    ret = dict_set_str(options, "transport-type", "socket");
+    if (ret == -1) {
+        gf_msg(GF_NLM, GF_LOG_ERROR, errno, NFS_MSG_DICT_SET_FAILED,
+               "dict_set_str error");
+        goto err;
+    }
+
+    if (nfs->allow_insecure) {
+        ret = dict_set_str(options, "rpc-auth-allow-insecure", "on");
         if (ret == -1) {
-                gf_log (GF_NLM, GF_LOG_ERROR, "dict_set_str error");
-                goto err;
+            gf_msg(GF_NLM, GF_LOG_ERROR, errno, NFS_MSG_DICT_SET_FAILED,
+                   "dict_set_str error");
+            goto err;
         }
-
-        ret = rpcsvc_create_listeners (nfs->rpcsvc, options, "NLM");
+        ret = dict_set_str(options, "rpc-auth.ports.insecure", "on");
         if (ret == -1) {
-                gf_log (GF_NLM, GF_LOG_ERROR, "Unable to create listeners");
-                dict_unref (options);
-                goto err;
-        }
-        INIT_LIST_HEAD(&nlm_client_list);
-        LOCK_INIT (&nlm_client_list_lk);
-
-        /* unlink sm-notify.pid so that when we restart rpc.statd/sm-notify
-         * it thinks that the machine has restarted and sends NOTIFY to clients.
-         */
-
-        /* TODO:
-           notify/rpc.statd is done differently on OSX
-
-           On OSX rpc.statd is controlled by rpc.lockd and are part for launchd
-           (unified service management framework)
-
-           A runcmd() should be invoking "launchctl start com.apple.lockd"
-           instead. This is still a theory but we need to thoroughly test it
-           out. Until then NLM support is non-existent on OSX.
-        */
-        ret = unlink (GF_SM_NOTIFY_PIDFILE);
-        if (ret == -1 && errno != ENOENT) {
-                gf_log (GF_NLM, GF_LOG_ERROR, "unable to unlink %s: %d",
-                        GF_SM_NOTIFY_PIDFILE, errno);
-                goto err;
-        }
-        /* temporary work around to restart statd, not distro/OS independent.
-         * Need to figure out a more graceful way
-         * killall will cause problems on solaris.
+            gf_msg(GF_NLM, GF_LOG_ERROR, errno, NFS_MSG_DICT_SET_FAILED,
+                   "dict_set_str error");
+            goto err;
+        }
+    }
+
+    ret = dict_set_str(options, "transport.address-family", "inet");
+    if (ret == -1) {
+        gf_msg(GF_NLM, GF_LOG_ERROR, errno, NFS_MSG_DICT_SET_FAILED,
+               "dict_set_str error");
+        goto err;
+    }
+
+    ret = rpcsvc_create_listeners(nfs->rpcsvc, options, "NLM");
+    if (ret == -1) {
+        gf_msg(GF_NLM, GF_LOG_ERROR, errno, NFS_MSG_LISTENERS_CREATE_FAIL,
+               "Unable to create listeners");
+        goto err;
+    }
+    INIT_LIST_HEAD(&nlm_client_list);
+    LOCK_INIT(&nlm_client_list_lk);
+
+    /* unlink sm-notify.pid so that when we restart rpc.statd/sm-notify
+     * it thinks that the machine has restarted and sends NOTIFY to clients.
+     */
+
+    /* TODO:
+       notify/rpc.statd is done differently on OSX
+
+       On OSX rpc.statd is controlled by rpc.lockd and are part for launchd
+       (unified service management framework)
+
+       A runcmd() should be invoking "launchctl start com.apple.lockd"
+       instead. This is still a theory but we need to thoroughly test it
+       out. Until then NLM support is non-existent on OSX.
+    */
+    ret = sys_unlink(GF_SM_NOTIFY_PIDFILE);
+    if (ret == -1 && errno != ENOENT) {
+        gf_msg(GF_NLM, GF_LOG_ERROR, errno, NFS_MSG_UNLINK_ERROR,
+               "unable to unlink %s: %d", GF_SM_NOTIFY_PIDFILE, errno);
+        goto err;
+    }
+    /* temporary work around to restart statd, not distro/OS independent.
+     * Need to figure out a more graceful way
+     * killall will cause problems on solaris.
+     */
+
+    char *pid_file = GF_RPC_STATD_PIDFILE;
+    if (nfs->rpc_statd_pid_file)
+        pid_file = nfs->rpc_statd_pid_file;
+    pidfile = fopen(pid_file, "r");
+    if (pidfile) {
+        ret = fscanf(pidfile, "%d", &pid);
+        if (ret <= 0) {
+            gf_msg(GF_NLM, GF_LOG_WARNING, errno, NFS_MSG_GET_PID_FAIL,
+                   "unable to get pid of "
+                   "rpc.statd from %s ",
+                   GF_RPC_STATD_PIDFILE);
+            ret = runcmd(KILLALL_CMD, "-9", "rpc.statd", NULL);
+        } else
+            kill(pid, SIGKILL);
+
+        fclose(pidfile);
+    } else {
+        gf_msg(GF_NLM, GF_LOG_WARNING, errno, NFS_MSG_OPEN_FAIL,
+               "opening %s of rpc.statd failed (%s)", pid_file,
+               strerror(errno));
+        /* if ret == -1, do nothing - case either statd was not
+         * running or was running in valgrind mode
          */
-
-        char *pid_file = GF_RPC_STATD_PIDFILE;
-        if (nfs->rpc_statd_pid_file)
-                pid_file = nfs->rpc_statd_pid_file;
-        pidfile = fopen (pid_file, "r");
-        if (pidfile) {
-                ret = fscanf (pidfile, "%d", &pid);
-                if (ret <= 0) {
-                        gf_log (GF_NLM, GF_LOG_WARNING, "unable to get pid of "
-                                "rpc.statd from %s ", GF_RPC_STATD_PIDFILE);
-                        ret = runcmd (KILLALL_CMD, "-9", "rpc.statd", NULL);
-                } else
-                        kill (pid, SIGKILL);
-
-                fclose (pidfile);
-        } else {
-                gf_log (GF_NLM, GF_LOG_WARNING, "opening %s of "
-                        "rpc.statd failed (%s)", pid_file, strerror (errno));
-                /* if ret == -1, do nothing - case either statd was not
-                 * running or was running in valgrind mode
-                 */
-                ret = runcmd (KILLALL_CMD, "-9", "rpc.statd", NULL);
-        }
-
-        ret = unlink (GF_RPC_STATD_PIDFILE);
-        if (ret == -1 && errno != ENOENT) {
-                gf_log (GF_NLM, GF_LOG_ERROR, "unable to unlink %s", pid_file);
-                goto err;
-        }
-
-        ret = runcmd (nfs->rpc_statd, NULL);
-        if (ret == -1) {
-                gf_log (GF_NLM, GF_LOG_ERROR, "unable to start %s",
-                        nfs->rpc_statd);
-                goto err;
-        }
-
-
-        pthread_create (&thr, NULL, nsm_thread, (void*)NULL);
-
-        timeout.tv_sec = nlm_grace_period;
-        timeout.tv_nsec = 0;
-
-        gf_timer_call_after (nfsx->ctx, timeout, nlm_grace_period_over, NULL);
-        nlm4_inited = _gf_true;
-        return &nlm4prog;
+        ret = runcmd(KILLALL_CMD, "-9", "rpc.statd", NULL);
+    }
+
+    ret = sys_unlink(GF_RPC_STATD_PIDFILE);
+    if (ret == -1 && errno != ENOENT) {
+        gf_msg(GF_NLM, GF_LOG_ERROR, errno, NFS_MSG_UNLINK_ERROR,
+               "unable to unlink %s", pid_file);
+        goto err;
+    }
+
+    ret = runcmd(nfs->rpc_statd, NULL);
+    if (ret == -1) {
+        gf_msg(GF_NLM, GF_LOG_ERROR, errno, NFS_MSG_START_ERROR,
+               "unable to start %s", nfs->rpc_statd);
+        goto err;
+    }
+
+    (void)gf_thread_create(&thr, NULL, nsm_thread, nfsx, "nfsnsm");
+
+    timeout.tv_sec = nlm_grace_period;
+    timeout.tv_nsec = 0;
+
+    gf_timer_call_after(nfsx->ctx, timeout, nlm_grace_period_over, NULL);
+    nlm4_inited = _gf_true;
+
+    if (options)
+        dict_unref(options);
+    return &nlm4prog;
 err:
-        return NULL;
+    if (options)
+        dict_unref(options);
+    return NULL;
 }
 
 int32_t
-nlm_priv (xlator_t *this)
+nlm_priv(xlator_t *this)
 {
-        int32_t                  ret                       = -1;
-        uint32_t                 client_count              = 0;
-        uint64_t                 file_count                = 0;
-        nlm_client_t            *client                    = NULL;
-        nlm_fde_t               *fde                       = NULL;
-        char                     key[GF_DUMP_MAX_BUF_LEN]  = {0};
-        char                     gfid_str[64]              = {0};
-
-        gf_proc_dump_add_section("nfs.nlm");
-
-        if (TRY_LOCK (&nlm_client_list_lk))
-                goto out;
+    int32_t ret = -1;
+    uint32_t client_count = 0;
+    uint64_t file_count = 0;
+    nlm_client_t *client = NULL;
+    nlm_fde_t *fde = NULL;
+    char key[GF_DUMP_MAX_BUF_LEN] = {0};
+    char gfid_str[64] = {0};
 
-        list_for_each_entry (client, &nlm_client_list, nlm_clients) {
+    gf_proc_dump_add_section("nfs.nlm");
 
-                gf_proc_dump_build_key (key, "client", "%d.hostname", client_count);
-                gf_proc_dump_write (key, "%s\n", client->caller_name);
+    if ((nlm4_inited == _gf_false) || TRY_LOCK(&nlm_client_list_lk))
+        goto out;
 
-                file_count = 0;
-                list_for_each_entry (fde, &client->fdes, fde_list) {
-                        gf_proc_dump_build_key (key, "file", "%ld.gfid", file_count);
-                        memset (gfid_str, 0, 64);
-                        uuid_utoa_r (fde->fd->inode->gfid, gfid_str);
-                        gf_proc_dump_write (key, "%s", gfid_str);
-                        file_count++;
-                }
+    list_for_each_entry(client, &nlm_client_list, nlm_clients)
+    {
+        gf_proc_dump_build_key(key, "client", "%d.hostname", client_count);
+        gf_proc_dump_write(key, "%s\n", client->caller_name);
 
-                gf_proc_dump_build_key (key, "client", "files-locked");
-                gf_proc_dump_write (key, "%ld\n", file_count);
-                client_count++;
+        file_count = 0;
+        list_for_each_entry(fde, &client->fdes, fde_list)
+        {
+            gf_proc_dump_build_key(key, "file", "%" PRIu64 ".gfid", file_count);
+            memset(gfid_str, 0, 64);
+            uuid_utoa_r(fde->fd->inode->gfid, gfid_str);
+            gf_proc_dump_write(key, "%s", gfid_str);
+            file_count++;
         }
 
-        gf_proc_dump_build_key (key, "nlm", "client-count");
-        gf_proc_dump_write (key, "%d", client_count);
-        ret = 0;
-        UNLOCK (&nlm_client_list_lk);
+        gf_proc_dump_build_key(key, "client", "files-locked");
+        gf_proc_dump_write(key, "%" PRIu64 "\n", file_count);
+        client_count++;
+    }
 
- out:
-        if (ret) {
-                gf_proc_dump_build_key (key, "nlm", "statedump_error");
-                gf_proc_dump_write (key, "Unable to dump nlm state because "
-                                    "nlm_client_list_lk lock couldn't be acquired");
-        }
+    gf_proc_dump_build_key(key, "nlm", "client-count");
+    gf_proc_dump_write(key, "%d", client_count);
+    ret = 0;
+    UNLOCK(&nlm_client_list_lk);
 
-        return ret;
+out:
+    if (ret) {
+        gf_proc_dump_build_key(key, "nlm", "statedump_error");
+        gf_proc_dump_write(key,
+                           "Unable to dump nlm state because "
+                           "nlm is not initialised or nlm_client_list_lk "
+                           "lock couldn't be acquired");
+    }
+
+    return ret;
 }
diff --git a/xlators/nfs/server/src/nlm4.h b/xlators/nfs/server/src/nlm4.h
index 0ab03d8c575..a22032cac69 100644
--- a/xlators/nfs/server/src/nlm4.h
+++ b/xlators/nfs/server/src/nlm4.h
@@ -11,106 +11,101 @@
 #ifndef _NLM4_H_
 #define _NLM4_H_
 
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
-
 #include <sys/types.h>
 #include <signal.h>
 #include "rpcsvc.h"
-#include "dict.h"
-#include "xlator.h"
-#include "iobuf.h"
+#include <glusterfs/dict.h>
+#include <glusterfs/xlator.h>
+#include <glusterfs/iobuf.h>
 #include "nfs.h"
-#include "list.h"
+#include <glusterfs/list.h>
 #include "xdr-nfs3.h"
-#include "locking.h"
+#include <glusterfs/locking.h>
 #include "nfs3-fh.h"
-#include "compat-uuid.h"
+#include <glusterfs/compat-uuid.h>
 #include "nlm4-xdr.h"
-#include "lkowner.h"
+#include <glusterfs/lkowner.h>
 
-#define NLM4_NULL          0
-#define NLM4_TEST          1
-#define NLM4_LOCK          2
-#define NLM4_CANCEL        3
-#define NLM4_UNLOCK        4
-#define NLM4_GRANTED       5
-#define NLM4_TEST_MSG      6
-#define NLM4_LOCK_MSG      7
-#define NLM4_CANCEL_MSG    8
-#define NLM4_UNLOCK_MSG    9
-#define NLM4_GRANTED_MSG   10
-#define NLM4_TEST_RES      11
-#define NLM4_LOCK_RES      12
-#define NLM4_CANCEL_RES    13
-#define NLM4_UNLOCK_RES    14
-#define NLM4_GRANTED_RES   15
-#define NLM4_SM_NOTIFY     16
-#define NLM4_SEVENTEEN     17
-#define NLM4_EIGHTEEN      18
-#define NLM4_NINETEEN      19
-#define NLM4_SHARE         20
-#define NLM4_UNSHARE       21
-#define NLM4_NM_LOCK       22
-#define NLM4_FREE_ALL      23
-#define NLM4_PROC_COUNT    24
+#define NLM4_NULL 0
+#define NLM4_TEST 1
+#define NLM4_LOCK 2
+#define NLM4_CANCEL 3
+#define NLM4_UNLOCK 4
+#define NLM4_GRANTED 5
+#define NLM4_TEST_MSG 6
+#define NLM4_LOCK_MSG 7
+#define NLM4_CANCEL_MSG 8
+#define NLM4_UNLOCK_MSG 9
+#define NLM4_GRANTED_MSG 10
+#define NLM4_TEST_RES 11
+#define NLM4_LOCK_RES 12
+#define NLM4_CANCEL_RES 13
+#define NLM4_UNLOCK_RES 14
+#define NLM4_GRANTED_RES 15
+#define NLM4_SM_NOTIFY 16
+#define NLM4_SEVENTEEN 17
+#define NLM4_EIGHTEEN 18
+#define NLM4_NINETEEN 19
+#define NLM4_SHARE 20
+#define NLM4_UNSHARE 21
+#define NLM4_NM_LOCK 22
+#define NLM4_FREE_ALL 23
+#define NLM4_PROC_COUNT 24
 
 /* Registered with portmap */
-#define GF_NLM4_PORT            38468
-#define GF_NLM                  GF_NFS"-NLM"
+#define GF_NLM4_PORT 38468
+#define GF_NLM GF_NFS "-NLM"
 #if defined(GF_DARWIN_HOST_OS)
-#define GF_RPC_STATD_PROG       "/usr/sbin/rpc.statd"
-#define GF_RPC_STATD_PIDFILE    "/var/run/statd.pid"
-#define GF_SM_NOTIFY_PIDFILE    "/var/run/statd.notify.pid"
+#define GF_RPC_STATD_PROG "/usr/sbin/rpc.statd"
+#define GF_RPC_STATD_PIDFILE "/var/run/statd.pid"
+#define GF_SM_NOTIFY_PIDFILE "/var/run/statd.notify.pid"
 #elif defined(__NetBSD__)
-#define GF_RPC_STATD_PROG       "/usr/sbin/rpc.statd"
-#define GF_RPC_STATD_PIDFILE    "/var/run/rpc.statd.pid"
-#define GF_SM_NOTIFY_PIDFILE    "/var/run/inexistant.pid"
+#define GF_RPC_STATD_PROG "/usr/sbin/rpc.statd"
+#define GF_RPC_STATD_PIDFILE "/var/run/rpc.statd.pid"
+#define GF_SM_NOTIFY_PIDFILE "/var/run/inexistent.pid"
 #else
-#define GF_RPC_STATD_PROG       "/sbin/rpc.statd"
-#define GF_RPC_STATD_PIDFILE    "/var/run/rpc.statd.pid"
-#define GF_SM_NOTIFY_PIDFILE    "/var/run/sm-notify.pid"
+#define GF_RPC_STATD_PROG "/sbin/rpc.statd"
+#define GF_RPC_STATD_PIDFILE "/var/run/rpc.statd.pid"
+#define GF_SM_NOTIFY_PIDFILE "/var/run/sm-notify.pid"
 #endif
 
 extern rpcsvc_program_t *
-nlm4svc_init (xlator_t *nfsx);
+nlm4svc_init(xlator_t *nfsx);
 
 extern int
-nlm4_init_state (xlator_t *nfsx);
+nlm4_init_state(xlator_t *nfsx);
 
 #define NLM_PROGRAM 100021
 #define NLM_V4 4
 
 typedef struct nlm4_lwowner {
-        char temp[1024];
+    char temp[1024];
 } nlm4_lkowner_t;
 
 typedef struct nlm_client {
-        struct sockaddr_storage sa;
-        pid_t uniq;
-        struct list_head nlm_clients;
-        struct list_head fdes;
-        struct list_head shares;
-        struct rpc_clnt *rpc_clnt;
-        char *caller_name;
-        int nsm_monitor;
+    struct sockaddr_storage sa;
+    pid_t uniq;
+    struct list_head nlm_clients;
+    struct list_head fdes;
+    struct list_head shares;
+    struct rpc_clnt *rpc_clnt;
+    char *caller_name;
+    int nsm_monitor;
 } nlm_client_t;
 
 typedef struct nlm_share {
-        struct list_head     client_list;
-        struct list_head     inode_list;
-        gf_lkowner_t         lkowner;
-        inode_t             *inode;
-        fsh_mode             mode;
-        fsh_access           access;
+    struct list_head client_list;
+    struct list_head inode_list;
+    gf_lkowner_t lkowner;
+    inode_t *inode;
+    fsh_mode mode;
+    fsh_access access;
 } nlm_share_t;
 
 typedef struct nlm_fde {
-        struct list_head fde_list;
-        fd_t *fd;
-        int transit_cnt;
+    struct list_head fde_list;
+    fd_t *fd;
+    int transit_cnt;
 } nlm_fde_t;
 
 #endif
diff --git a/xlators/nfs/server/src/nlmcbk_svc.c b/xlators/nfs/server/src/nlmcbk_svc.c
index 20d3728d031..eaa7b916190 100644
--- a/xlators/nfs/server/src/nlmcbk_svc.c
+++ b/xlators/nfs/server/src/nlmcbk_svc.c
@@ -14,7 +14,8 @@
  */
 
 #include "nlm4.h"
-#include "logging.h"
+#include <glusterfs/logging.h>
+#include "nfs-messages.h"
 #include <stdio.h>
 #include <stdlib.h>
 #include <rpc/pmap_clnt.h>
@@ -24,93 +25,110 @@
 #include <netinet/in.h>
 
 #ifndef SIG_PF
-#define SIG_PF void(*)(int)
+#define SIG_PF void (*)(int)
 #endif
 
 void
-nlm4svc_sm_notify (struct nlm_sm_status *status);
+nlm4svc_sm_notify(struct nlm_sm_status *status);
 
-void *nlmcbk_sm_notify_0_svc(struct nlm_sm_status *status, struct svc_req *req)
+void *
+nlmcbk_sm_notify_0_svc(struct nlm_sm_status *status, struct svc_req *req)
 {
-        nlm4svc_sm_notify (status);
-        return NULL;
+    nlm4svc_sm_notify(status);
+    return NULL;
 }
 
 static void
 nlmcbk_program_0(struct svc_req *rqstp, register SVCXPRT *transp)
 {
-	union {
-		struct nlm_sm_status nlmcbk_sm_notify_0_arg;
-	} argument;
-	char *result;
-	xdrproc_t _xdr_argument, _xdr_result;
-	char *(*local)(char *, struct svc_req *);
-
-	switch (rqstp->rq_proc) {
-	case NULLPROC:
-		(void) svc_sendreply (transp, (xdrproc_t) xdr_void, (char *)NULL);
-		return;
-
-	case NLMCBK_SM_NOTIFY:
-		_xdr_argument = (xdrproc_t) xdr_nlm_sm_status;
-		_xdr_result = (xdrproc_t) xdr_void;
-		local = (char *(*)(char *, struct svc_req *)) nlmcbk_sm_notify_0_svc;
-		break;
-
-	default:
-		svcerr_noproc (transp);
-		return;
-	}
-	memset ((char *)&argument, 0, sizeof (argument));
-	if (!svc_getargs (transp, (xdrproc_t) _xdr_argument, (caddr_t) &argument)) {
-		svcerr_decode (transp);
-		return;
-	}
-	result = (*local)((char *)&argument, rqstp);
-	if (!svc_sendreply(transp, (xdrproc_t) _xdr_result, result)) {
-		svcerr_systemerr (transp);
-	}
-
-	if (!svc_freeargs (transp, (xdrproc_t) _xdr_argument, (caddr_t) &argument)) {
-		gf_log (GF_NLM, GF_LOG_ERROR, "unable to free arguments");
-                return;
-	}
-	return;
+    union {
+        struct nlm_sm_status nlmcbk_sm_notify_0_arg;
+    } argument;
+    char *result;
+    xdrproc_t _xdr_argument, _xdr_result;
+    char *(*local)(char *, struct svc_req *);
+
+    switch (rqstp->rq_proc) {
+        case NULLPROC:
+            (void)svc_sendreply(transp, (xdrproc_t)xdr_void, (char *)NULL);
+            return;
+
+        case NLMCBK_SM_NOTIFY:
+            _xdr_argument = (xdrproc_t)xdr_nlm_sm_status;
+            _xdr_result = (xdrproc_t)xdr_void;
+            local = (char *(*)(char *, struct svc_req *))nlmcbk_sm_notify_0_svc;
+            break;
+
+        default:
+            svcerr_noproc(transp);
+            return;
+    }
+    memset((char *)&argument, 0, sizeof(argument));
+    if (!svc_getargs(transp, (xdrproc_t)_xdr_argument, (caddr_t)&argument)) {
+        svcerr_decode(transp);
+        return;
+    }
+    result = (*local)((char *)&argument, rqstp);
+    if (!svc_sendreply(transp, (xdrproc_t)_xdr_result, result)) {
+        svcerr_systemerr(transp);
+    }
+
+    if (!svc_freeargs(transp, (xdrproc_t)_xdr_argument, (caddr_t)&argument)) {
+        gf_msg(GF_NLM, GF_LOG_ERROR, 0, NFS_MSG_ARG_FREE_FAIL,
+               "unable to free arguments");
+        return;
+    }
+    return;
 }
 
 void *
-nsm_thread (void *argv)
+nsm_thread(void *argv)
 {
-	register SVCXPRT *transp;
-        int ret = 0;
-
-        ret = pmap_unset (NLMCBK_PROGRAM, NLMCBK_V1);
-        if (ret == 0) {
-                gf_log (GF_NLM, GF_LOG_ERROR, "pmap_unset failed");
-                return NULL;
-        }
-        transp = svcudp_create(RPC_ANYSOCK);
-	if (transp == NULL) {
-		gf_log (GF_NLM, GF_LOG_ERROR, "cannot create udp service.");
-                return NULL;
-	}
-	if (!svc_register(transp, NLMCBK_PROGRAM, NLMCBK_V1, nlmcbk_program_0, IPPROTO_UDP)) {
-		gf_log (GF_NLM, GF_LOG_ERROR, "unable to register (NLMCBK_PROGRAM, NLMCBK_V0, udp).");
-                return NULL;
-	}
-
-	transp = svctcp_create(RPC_ANYSOCK, 0, 0);
-	if (transp == NULL) {
-		gf_log (GF_NLM, GF_LOG_ERROR, "cannot create tcp service.");
-                return NULL;
-	}
-	if (!svc_register(transp, NLMCBK_PROGRAM, NLMCBK_V1, nlmcbk_program_0, IPPROTO_TCP)) {
-		gf_log (GF_NLM, GF_LOG_ERROR, "unable to register (NLMCBK_PROGRAM, NLMCBK_V0, tcp).");
-                return NULL;
-	}
-
-	svc_run ();
-	gf_log (GF_NLM, GF_LOG_ERROR, "svc_run returned");
+    xlator_t *nfsx = argv;
+    register SVCXPRT *transp;
+    int ret = 0;
+
+    GF_ASSERT(nfsx);
+
+    THIS = nfsx;
+
+    ret = pmap_unset(NLMCBK_PROGRAM, NLMCBK_V1);
+    if (ret == 0) {
+        gf_msg(GF_NLM, GF_LOG_ERROR, 0, NFS_MSG_PMAP_UNSET_FAIL,
+               "pmap_unset failed");
+        return NULL;
+    }
+    transp = svcudp_create(RPC_ANYSOCK);
+    if (transp == NULL) {
+        gf_msg(GF_NLM, GF_LOG_ERROR, errno, NFS_MSG_UDP_SERV_FAIL,
+               "cannot create udp service.");
         return NULL;
-	/* NOTREACHED */
+    }
+    if (!svc_register(transp, NLMCBK_PROGRAM, NLMCBK_V1, nlmcbk_program_0,
+                      IPPROTO_UDP)) {
+        gf_msg(GF_NLM, GF_LOG_ERROR, 0, NFS_MSG_REG_NLMCBK_FAIL,
+               "unable to register (NLMCBK_PROGRAM, "
+               "NLMCBK_V0, udp).");
+        return NULL;
+    }
+
+    transp = svctcp_create(RPC_ANYSOCK, 0, 0);
+    if (transp == NULL) {
+        gf_msg(GF_NLM, GF_LOG_ERROR, errno, NFS_MSG_TCP_SERV_FAIL,
+               "cannot create tcp service.");
+        return NULL;
+    }
+    if (!svc_register(transp, NLMCBK_PROGRAM, NLMCBK_V1, nlmcbk_program_0,
+                      IPPROTO_TCP)) {
+        gf_msg(GF_NLM, GF_LOG_ERROR, 0, NFS_MSG_REG_NLMCBK_FAIL,
+               "unable to register (NLMCBK_PROGRAM, "
+               "NLMCBK_V0, tcp).");
+        return NULL;
+    }
+
+    svc_run();
+    gf_msg(GF_NLM, GF_LOG_ERROR, 0, NFS_MSG_SVC_RUN_RETURNED,
+           "svc_run returned");
+    return NULL;
+    /* NOTREACHED */
 }
diff --git a/xlators/performance/Makefile.am b/xlators/performance/Makefile.am
index a494190baac..e95725acb8c 100644
--- a/xlators/performance/Makefile.am
+++ b/xlators/performance/Makefile.am
@@ -1,3 +1,4 @@
-SUBDIRS = write-behind read-ahead readdir-ahead io-threads io-cache symlink-cache quick-read md-cache open-behind
+SUBDIRS = write-behind read-ahead readdir-ahead io-threads io-cache \
+	quick-read md-cache open-behind nl-cache
 
 CLEANFILES = 
diff --git a/xlators/performance/io-cache/src/Makefile.am b/xlators/performance/io-cache/src/Makefile.am
index 155be9988c9..bfa34ce5502 100644
--- a/xlators/performance/io-cache/src/Makefile.am
+++ b/xlators/performance/io-cache/src/Makefile.am
@@ -1,14 +1,15 @@
 xlator_LTLIBRARIES = io-cache.la
 xlatordir = $(libdir)/glusterfs/$(PACKAGE_VERSION)/xlator/performance
 
-io_cache_la_LDFLAGS = -module -avoid-version 
+io_cache_la_LDFLAGS = -module $(GF_XLATOR_DEFAULT_LDFLAGS)
 
 io_cache_la_SOURCES = io-cache.c page.c ioc-inode.c
 io_cache_la_LIBADD = $(top_builddir)/libglusterfs/src/libglusterfs.la
 
-noinst_HEADERS = io-cache.h ioc-mem-types.h
+noinst_HEADERS = io-cache.h ioc-mem-types.h io-cache-messages.h
 
 AM_CPPFLAGS = $(GF_CPPFLAGS) -I$(top_srcdir)/libglusterfs/src \
+	-I$(top_srcdir)/rpc/xdr/src -I$(top_builddir)/rpc/xdr/src \
 	-I$(CONTRIBDIR)/rbtree
 
 AM_CFLAGS = -Wall $(GF_CFLAGS)
diff --git a/xlators/performance/io-cache/src/io-cache-messages.h b/xlators/performance/io-cache/src/io-cache-messages.h
new file mode 100644
index 00000000000..38ad0b14d0e
--- /dev/null
+++ b/xlators/performance/io-cache/src/io-cache-messages.h
@@ -0,0 +1,69 @@
+/*Copyright (c) 2015 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
+*/
+
+#ifndef _IO_CACHE_MESSAGES_H_
+#define _IO_CACHE_MESSAGES_H_
+
+#include <glusterfs/glfs-message-id.h>
+
+/* To add new message IDs, append new identifiers at the end of the list.
+ *
+ * Never remove a message ID. If it's not used anymore, you can rename it or
+ * leave it as it is, but not delete it. This is to prevent reutilization of
+ * IDs by other messages.
+ *
+ * The component name must match one of the entries defined in
+ * glfs-message-id.h.
+ */
+
+GLFS_MSGID(IO_CACHE, IO_CACHE_MSG_ENFORCEMENT_FAILED,
+           IO_CACHE_MSG_INVALID_ARGUMENT,
+           IO_CACHE_MSG_XLATOR_CHILD_MISCONFIGURED, IO_CACHE_MSG_NO_MEMORY,
+           IO_CACHE_MSG_VOL_MISCONFIGURED, IO_CACHE_MSG_INODE_NULL,
+           IO_CACHE_MSG_PAGE_WAIT_VALIDATE, IO_CACHE_MSG_STR_COVERSION_FAILED,
+           IO_CACHE_MSG_WASTED_COPY, IO_CACHE_MSG_SET_FD_FAILED,
+           IO_CACHE_MSG_TABLE_NULL, IO_CACHE_MSG_MEMORY_INIT_FAILED,
+           IO_CACHE_MSG_NO_CACHE_SIZE_OPT, IO_CACHE_MSG_NOT_RECONFIG_CACHE_SIZE,
+           IO_CACHE_MSG_CREATE_MEM_POOL_FAILED,
+           IO_CACHE_MSG_ALLOC_MEM_POOL_FAILED, IO_CACHE_MSG_NULL_PAGE_WAIT,
+           IO_CACHE_MSG_FRAME_NULL, IO_CACHE_MSG_PAGE_FAULT,
+           IO_CACHE_MSG_SERVE_READ_REQUEST, IO_CACHE_MSG_LOCAL_NULL,
+           IO_CACHE_MSG_DEFAULTING_TO_OLD);
+
+#define IO_CACHE_MSG_NO_MEMORY_STR "out of memory"
+#define IO_CACHE_MSG_ENFORCEMENT_FAILED_STR "inode context is NULL"
+#define IO_CACHE_MSG_SET_FD_FAILED_STR "failed to set fd ctx"
+#define IO_CACHE_MSG_TABLE_NULL_STR "table is NULL"
+#define IO_CACHE_MSG_MEMORY_INIT_FAILED_STR "Memory accounting init failed"
+#define IO_CACHE_MSG_NO_CACHE_SIZE_OPT_STR "could not get cache-size option"
+#define IO_CACHE_MSG_INVALID_ARGUMENT_STR                                      \
+    "file size is greater than the max size"
+#define IO_CACHE_MSG_NOT_RECONFIG_CACHE_SIZE_STR "Not reconfiguring cache-size"
+#define IO_CACHE_MSG_XLATOR_CHILD_MISCONFIGURED_STR                            \
+    "FATAL: io-cache not configured with exactly one child"
+#define IO_CACHE_MSG_VOL_MISCONFIGURED_STR "dangling volume. check volfile"
+#define IO_CACHE_MSG_CREATE_MEM_POOL_FAILED_STR                                \
+    "failed to create local_t's memory pool"
+#define IO_CACHE_MSG_ALLOC_MEM_POOL_FAILED_STR "Unable to allocate mem_pool"
+#define IO_CACHE_MSG_STR_COVERSION_FAILED_STR                                  \
+    "asprintf failed while converting prt to str"
+#define IO_CACHE_MSG_INODE_NULL_STR "ioc_inode is NULL"
+#define IO_CACHE_MSG_PAGE_WAIT_VALIDATE_STR                                    \
+    "cache validate called without any page waiting to be validated"
+#define IO_CACHE_MSG_NULL_PAGE_WAIT_STR "asked to wait on a NULL page"
+#define IO_CACHE_MSG_WASTED_COPY_STR "wasted copy"
+#define IO_CACHE_MSG_FRAME_NULL_STR "frame>root>rsp_refs is null"
+#define IO_CACHE_MSG_PAGE_FAULT_STR "page fault on a NULL frame"
+#define IO_CACHE_MSG_SERVE_READ_REQUEST_STR                                    \
+    "NULL page has been provided to serve read request"
+#define IO_CACHE_MSG_LOCAL_NULL_STR "local is NULL"
+#define IO_CACHE_MSG_DEFAULTING_TO_OLD_STR                                     \
+    "minimum size of file that can be cached is greater than maximum size. "   \
+    "Hence Defaulting to old value"
+#endif /* _IO_CACHE_MESSAGES_H_ */
diff --git a/xlators/performance/io-cache/src/io-cache.c b/xlators/performance/io-cache/src/io-cache.c
index bfda85d4926..9375d29c17f 100644
--- a/xlators/performance/io-cache/src/io-cache.c
+++ b/xlators/performance/io-cache/src/io-cache.c
@@ -8,43 +8,38 @@
   cases as published by the Free Software Foundation.
 */
 
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
-
-#include "glusterfs.h"
-#include "logging.h"
-#include "dict.h"
-#include "xlator.h"
+#include <math.h>
+#include <glusterfs/glusterfs.h>
+#include <glusterfs/logging.h>
+#include <glusterfs/dict.h>
+#include <glusterfs/xlator.h>
 #include "io-cache.h"
 #include "ioc-mem-types.h"
-#include "statedump.h"
+#include <glusterfs/statedump.h>
 #include <assert.h>
 #include <sys/time.h>
-
+#include "io-cache-messages.h"
 int ioc_log2_page_size;
 
 uint32_t
-ioc_get_priority (ioc_table_t *table, const char *path);
+ioc_get_priority(ioc_table_t *table, const char *path);
 
 struct volume_options options[];
 
-
-static inline uint32_t
-ioc_hashfn (void *data, int len)
+static uint32_t
+ioc_hashfn(void *data, int len)
 {
-        off_t offset;
+    off_t offset;
 
-        offset = *(off_t *) data;
+    offset = *(off_t *)data;
 
-        return (offset >> ioc_log2_page_size);
+    return (offset >> ioc_log2_page_size);
 }
 
 /* TODO: This function is not used, uncomment when we find a
          usage for this function.
 
-static inline ioc_inode_t *
+static ioc_inode_t *
 ioc_inode_reupdate (ioc_inode_t *ioc_inode)
 {
         ioc_table_t *table = NULL;
@@ -58,7 +53,7 @@ ioc_inode_reupdate (ioc_inode_t *ioc_inode)
 }
 
 
-static inline ioc_inode_t *
+static ioc_inode_t *
 ioc_get_inode (dict_t *dict, char *name)
 {
         ioc_inode_t *ioc_inode      = NULL;
@@ -83,21 +78,72 @@ ioc_get_inode (dict_t *dict, char *name)
 }
 */
 
-int32_t
-ioc_inode_need_revalidate (ioc_inode_t *ioc_inode)
+int
+ioc_update_pages(call_frame_t *frame, ioc_inode_t *ioc_inode,
+                 struct iovec *vector, int32_t count, int op_ret, off_t offset)
 {
-        int8_t          need_revalidate = 0;
-        struct timeval  tv              = {0,};
-        ioc_table_t    *table           = NULL;
+    size_t size = 0;
+    off_t rounded_offset = 0, rounded_end = 0, trav_offset = 0,
+          write_offset = 0;
+    off_t page_offset = 0, page_end = 0;
+    ioc_page_t *trav = NULL;
 
-        table = ioc_inode->table;
+    size = iov_length(vector, count);
+    size = min(size, op_ret);
 
-        gettimeofday (&tv, NULL);
+    rounded_offset = gf_floor(offset, ioc_inode->table->page_size);
+    rounded_end = gf_roof(offset + size, ioc_inode->table->page_size);
 
-        if (time_elapsed (&tv, &ioc_inode->cache.tv) >= table->cache_timeout)
-                need_revalidate = 1;
+    trav_offset = rounded_offset;
+    ioc_inode_lock(ioc_inode);
+    {
+        while (trav_offset < rounded_end) {
+            trav = __ioc_page_get(ioc_inode, trav_offset);
+            if (trav && trav->ready) {
+                if (trav_offset == rounded_offset)
+                    page_offset = offset - rounded_offset;
+                else
+                    page_offset = 0;
+
+                if ((trav_offset + ioc_inode->table->page_size) >=
+                    rounded_end) {
+                    page_end = trav->size - (rounded_end - (offset + size));
+                } else {
+                    page_end = trav->size;
+                }
 
-        return need_revalidate;
+                iov_range_copy(trav->vector, trav->count, page_offset, vector,
+                               count, write_offset, page_end - page_offset);
+            } else if (trav) {
+                if (!trav->waitq)
+                    ioc_inode->table->cache_used -= __ioc_page_destroy(trav);
+            }
+
+            if (trav_offset == rounded_offset)
+                write_offset += (ioc_inode->table->page_size -
+                                 (offset - rounded_offset));
+            else
+                write_offset += ioc_inode->table->page_size;
+
+            trav_offset += ioc_inode->table->page_size;
+        }
+    }
+    ioc_inode_unlock(ioc_inode);
+
+    return 0;
+}
+
+static gf_boolean_t
+ioc_inode_need_revalidate(ioc_inode_t *ioc_inode)
+{
+    ioc_table_t *table = NULL;
+
+    GF_ASSERT(ioc_inode);
+    table = ioc_inode->table;
+    GF_ASSERT(table);
+
+    return (gf_time() - ioc_inode->cache.last_revalidate >=
+            table->cache_timeout);
 }
 
 /*
@@ -108,193 +154,201 @@ ioc_inode_need_revalidate (ioc_inode_t *ioc_inode)
  * assumes lock is held
  */
 int64_t
-__ioc_inode_flush (ioc_inode_t *ioc_inode)
+__ioc_inode_flush(ioc_inode_t *ioc_inode)
 {
-        ioc_page_t *curr         = NULL, *next = NULL;
-        int64_t     destroy_size = 0;
-        int64_t     ret          = 0;
+    ioc_page_t *curr = NULL, *next = NULL;
+    int64_t destroy_size = 0;
+    int64_t ret = 0;
 
-        list_for_each_entry_safe (curr, next, &ioc_inode->cache.page_lru,
-                                  page_lru) {
-                ret = __ioc_page_destroy (curr);
+    list_for_each_entry_safe(curr, next, &ioc_inode->cache.page_lru, page_lru)
+    {
+        ret = __ioc_page_destroy(curr);
 
-                if (ret != -1)
-                        destroy_size += ret;
-        }
+        if (ret != -1)
+            destroy_size += ret;
+    }
 
-        return destroy_size;
+    return destroy_size;
 }
 
 void
-ioc_inode_flush (ioc_inode_t *ioc_inode)
+ioc_inode_flush(ioc_inode_t *ioc_inode)
 {
-        int64_t destroy_size = 0;
+    int64_t destroy_size = 0;
 
-        ioc_inode_lock (ioc_inode);
-        {
-                destroy_size = __ioc_inode_flush (ioc_inode);
-        }
-        ioc_inode_unlock (ioc_inode);
+    ioc_inode_lock(ioc_inode);
+    {
+        destroy_size = __ioc_inode_flush(ioc_inode);
+    }
+    ioc_inode_unlock(ioc_inode);
 
-        if (destroy_size) {
-                ioc_table_lock (ioc_inode->table);
-                {
-                        ioc_inode->table->cache_used -= destroy_size;
-                }
-                ioc_table_unlock (ioc_inode->table);
+    if (destroy_size) {
+        ioc_table_lock(ioc_inode->table);
+        {
+            ioc_inode->table->cache_used -= destroy_size;
         }
+        ioc_table_unlock(ioc_inode->table);
+    }
 
-        return;
+    return;
 }
 
 int32_t
-ioc_setattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                 int32_t op_ret, int32_t op_errno,
-                 struct iatt *preop, struct iatt *postop, dict_t *xdata)
+ioc_setattr_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                int32_t op_ret, int32_t op_errno, struct iatt *preop,
+                struct iatt *postop, dict_t *xdata)
 {
-        STACK_UNWIND_STRICT (setattr, frame, op_ret, op_errno, preop, postop,
-                             xdata);
-        return 0;
+    STACK_UNWIND_STRICT(setattr, frame, op_ret, op_errno, preop, postop, xdata);
+    return 0;
 }
 
 int32_t
-ioc_setattr (call_frame_t *frame, xlator_t *this, loc_t *loc,
-             struct iatt *stbuf, int32_t valid, dict_t *xdata)
+ioc_setattr(call_frame_t *frame, xlator_t *this, loc_t *loc, struct iatt *stbuf,
+            int32_t valid, dict_t *xdata)
 {
-        uint64_t ioc_inode = 0;
+    uint64_t ioc_inode = 0;
 
-        inode_ctx_get (loc->inode, this, &ioc_inode);
+    inode_ctx_get(loc->inode, this, &ioc_inode);
 
-        if (ioc_inode
-            && ((valid & GF_SET_ATTR_ATIME)
-                || (valid & GF_SET_ATTR_MTIME)))
-                ioc_inode_flush ((ioc_inode_t *)(long)ioc_inode);
+    if (ioc_inode &&
+        ((valid & GF_SET_ATTR_ATIME) || (valid & GF_SET_ATTR_MTIME)))
+        ioc_inode_flush((ioc_inode_t *)(long)ioc_inode);
 
-        STACK_WIND (frame, ioc_setattr_cbk, FIRST_CHILD (this),
-                    FIRST_CHILD (this)->fops->setattr, loc, stbuf, valid, xdata);
+    STACK_WIND(frame, ioc_setattr_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->setattr, loc, stbuf, valid, xdata);
 
-        return 0;
+    return 0;
 }
 
 int32_t
-ioc_lookup_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                int32_t op_ret,	int32_t op_errno, inode_t *inode,
-                struct iatt *stbuf, dict_t *xdata, struct iatt *postparent)
+ioc_inode_update(xlator_t *this, inode_t *inode, char *path, struct iatt *iabuf)
 {
-        ioc_inode_t *ioc_inode         = NULL;
-        ioc_table_t *table             = NULL;
-        uint8_t      cache_still_valid = 0;
-        uint64_t     tmp_ioc_inode     = 0;
-        uint32_t     weight            = 0xffffffff;
-        const char  *path              = NULL;
-        ioc_local_t *local             = NULL;
-
-        if (op_ret != 0)
-                goto out;
+    ioc_table_t *table = NULL;
+    uint64_t tmp_ioc_inode = 0;
+    ioc_inode_t *ioc_inode = NULL;
+    uint32_t weight = 0xffffffff;
+    gf_boolean_t cache_still_valid = _gf_false;
 
-        local = frame->local;
-        if (local == NULL) {
-                op_ret = -1;
-                op_errno = EINVAL;
-                goto out;
-        }
+    if (!this || !inode)
+        goto out;
 
-        if (!this || !this->private) {
-                op_ret = -1;
-                op_errno = EINVAL;
-                goto out;
+    table = this->private;
+
+    LOCK(&inode->lock);
+    {
+        (void)__inode_ctx_get(inode, this, &tmp_ioc_inode);
+        ioc_inode = (ioc_inode_t *)(long)tmp_ioc_inode;
+
+        if (!ioc_inode) {
+            weight = ioc_get_priority(table, path);
+
+            ioc_inode = ioc_inode_create(table, inode, weight);
+
+            (void)__inode_ctx_put(inode, this, (uint64_t)(long)ioc_inode);
         }
+    }
+    UNLOCK(&inode->lock);
 
-        table = this->private;
+    ioc_inode_lock(ioc_inode);
+    {
+        if (ioc_inode->cache.mtime == 0) {
+            ioc_inode->cache.mtime = iabuf->ia_mtime;
+            ioc_inode->cache.mtime_nsec = iabuf->ia_mtime_nsec;
+        }
 
-        path = local->file_loc.path;
+        ioc_inode->ia_size = iabuf->ia_size;
+    }
+    ioc_inode_unlock(ioc_inode);
 
-        LOCK (&inode->lock);
-        {
-                __inode_ctx_get (inode, this, &tmp_ioc_inode);
-                ioc_inode = (ioc_inode_t *)(long)tmp_ioc_inode;
+    cache_still_valid = ioc_cache_still_valid(ioc_inode, iabuf);
 
-                if (!ioc_inode) {
-                        weight = ioc_get_priority (table, path);
+    if (!cache_still_valid) {
+        ioc_inode_flush(ioc_inode);
+    }
 
-                        ioc_inode = ioc_inode_update (table, inode,
-                                                      weight);
+    ioc_table_lock(ioc_inode->table);
+    {
+        list_move_tail(&ioc_inode->inode_lru,
+                       &table->inode_lru[ioc_inode->weight]);
+    }
+    ioc_table_unlock(ioc_inode->table);
 
-                        __inode_ctx_put (inode, this,
-                                         (uint64_t)(long)ioc_inode);
-                }
-        }
-        UNLOCK (&inode->lock);
+out:
+    return 0;
+}
 
-        ioc_inode_lock (ioc_inode);
-        {
-                if (ioc_inode->cache.mtime == 0) {
-                        ioc_inode->cache.mtime = stbuf->ia_mtime;
-                        ioc_inode->cache.mtime_nsec = stbuf->ia_mtime_nsec;
-                }
+int32_t
+ioc_lookup_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+               int32_t op_ret, int32_t op_errno, inode_t *inode,
+               struct iatt *stbuf, dict_t *xdata, struct iatt *postparent)
+{
+    ioc_local_t *local = NULL;
 
-                ioc_inode->ia_size = stbuf->ia_size;
-        }
-        ioc_inode_unlock (ioc_inode);
+    if (op_ret != 0)
+        goto out;
 
-        cache_still_valid = ioc_cache_still_valid (ioc_inode,
-                                                   stbuf);
+    local = frame->local;
+    if (local == NULL) {
+        op_ret = -1;
+        op_errno = EINVAL;
+        goto out;
+    }
 
-        if (!cache_still_valid) {
-                ioc_inode_flush (ioc_inode);
-        }
+    if (!this || !this->private) {
+        op_ret = -1;
+        op_errno = EINVAL;
+        goto out;
+    }
 
-        ioc_table_lock (ioc_inode->table);
-        {
-                list_move_tail (&ioc_inode->inode_lru,
-                                &table->inode_lru[ioc_inode->weight]);
-        }
-        ioc_table_unlock (ioc_inode->table);
+    ioc_inode_update(this, inode, (char *)local->file_loc.path, stbuf);
 
 out:
-        if (frame->local != NULL) {
-                local = frame->local;
-                loc_wipe (&local->file_loc);
-        }
+    if (frame->local != NULL) {
+        local = frame->local;
+        loc_wipe(&local->file_loc);
+    }
 
-        STACK_UNWIND_STRICT (lookup, frame, op_ret, op_errno, inode, stbuf,
-                             xdata, postparent);
-        return 0;
+    STACK_UNWIND_STRICT(lookup, frame, op_ret, op_errno, inode, stbuf, xdata,
+                        postparent);
+    return 0;
 }
 
 int32_t
-ioc_lookup (call_frame_t *frame, xlator_t *this, loc_t *loc,
-            dict_t *xdata)
+ioc_lookup(call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xdata)
 {
-        ioc_local_t *local    = NULL;
-        int32_t      op_errno = -1, ret = -1;
+    ioc_local_t *local = NULL;
+    int32_t op_errno = -1, ret = -1;
 
-        local = mem_get0 (this->local_pool);
-        if (local == NULL) {
-                op_errno = ENOMEM;
-                gf_log (this->name, GF_LOG_ERROR, "out of memory");
-                goto unwind;
-        }
+    local = mem_get0(this->local_pool);
+    if (local == NULL) {
+        op_errno = ENOMEM;
+        gf_smsg(this->name, GF_LOG_ERROR, 0, IO_CACHE_MSG_NO_MEMORY, NULL);
+        goto unwind;
+    }
 
-        ret = loc_copy (&local->file_loc, loc);
-        if (ret != 0) {
-                op_errno = ENOMEM;
-                gf_log (this->name, GF_LOG_ERROR, "out of memory");
-                goto unwind;
-        }
+    ret = loc_copy(&local->file_loc, loc);
+    if (ret != 0) {
+        op_errno = ENOMEM;
+        gf_smsg(this->name, GF_LOG_ERROR, 0, IO_CACHE_MSG_NO_MEMORY, NULL);
+        goto unwind;
+    }
 
-        frame->local = local;
+    frame->local = local;
 
-        STACK_WIND (frame, ioc_lookup_cbk, FIRST_CHILD (this),
-                    FIRST_CHILD (this)->fops->lookup, loc, xdata);
+    STACK_WIND(frame, ioc_lookup_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->lookup, loc, xdata);
 
-        return 0;
+    return 0;
 
 unwind:
-        STACK_UNWIND_STRICT (lookup, frame, -1, op_errno, NULL, NULL,
-                             NULL, NULL);
+    if (local != NULL) {
+        loc_wipe(&local->file_loc);
+        mem_put(local);
+    }
 
-        return 0;
+    STACK_UNWIND_STRICT(lookup, frame, -1, op_errno, NULL, NULL, NULL, NULL);
+
+    return 0;
 }
 
 /*
@@ -306,31 +360,29 @@ unwind:
  *
  */
 int32_t
-ioc_forget (xlator_t *this, inode_t *inode)
+ioc_forget(xlator_t *this, inode_t *inode)
 {
-        uint64_t ioc_inode = 0;
+    uint64_t ioc_inode = 0;
 
-        inode_ctx_get (inode, this, &ioc_inode);
+    inode_ctx_get(inode, this, &ioc_inode);
 
-        if (ioc_inode)
-                ioc_inode_destroy ((ioc_inode_t *)(long)ioc_inode);
+    if (ioc_inode)
+        ioc_inode_destroy((ioc_inode_t *)(long)ioc_inode);
 
-        return 0;
+    return 0;
 }
 
 static int32_t
 ioc_invalidate(xlator_t *this, inode_t *inode)
 {
-	uint64_t     ioc_addr = 0;
-	ioc_inode_t *ioc_inode = NULL;
+    uint64_t ioc_inode = 0;
 
-	inode_ctx_get(inode, this, (uint64_t *) &ioc_addr);
-	ioc_inode = (void *) ioc_addr;
+    inode_ctx_get(inode, this, &ioc_inode);
 
-	if (ioc_inode)
-		ioc_inode_flush(ioc_inode);
+    if (ioc_inode)
+        ioc_inode_flush((ioc_inode_t *)(uintptr_t)ioc_inode);
 
-	return 0;
+    return 0;
 }
 
 /*
@@ -345,104 +397,103 @@ ioc_invalidate(xlator_t *this, inode_t *inode)
  *
  */
 int32_t
-ioc_cache_validate_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                        int32_t op_ret, int32_t op_errno, struct iatt *stbuf,
-                        dict_t *xdata)
+ioc_cache_validate_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                       int32_t op_ret, int32_t op_errno, struct iatt *stbuf,
+                       dict_t *xdata)
 {
-        ioc_local_t *local        = NULL;
-        ioc_inode_t *ioc_inode    = NULL;
-        size_t       destroy_size = 0;
-        struct iatt *local_stbuf  = NULL;
-
-        local = frame->local;
-        ioc_inode = local->inode;
-        local_stbuf = stbuf;
-
-        if ((op_ret == -1) ||
-            ((op_ret >= 0) && !ioc_cache_still_valid(ioc_inode, stbuf))) {
-                gf_log (ioc_inode->table->xl->name, GF_LOG_DEBUG,
-                        "cache for inode(%p) is invalid. flushing all pages",
-                        ioc_inode);
-                /* NOTE: only pages with no waiting frames are flushed by
-                 * ioc_inode_flush. page_fault will be generated for all
-                 * the pages which have waiting frames by ioc_inode_wakeup()
-                 */
-                ioc_inode_lock (ioc_inode);
-                {
-                        destroy_size = __ioc_inode_flush (ioc_inode);
-                        if (op_ret >= 0) {
-                                ioc_inode->cache.mtime = stbuf->ia_mtime;
-                                ioc_inode->cache.mtime_nsec
-                                        = stbuf->ia_mtime_nsec;
-                        }
-                }
-                ioc_inode_unlock (ioc_inode);
-                local_stbuf = NULL;
-        }
-
-        if (destroy_size) {
-                ioc_table_lock (ioc_inode->table);
-                {
-                        ioc_inode->table->cache_used -= destroy_size;
-                }
-                ioc_table_unlock (ioc_inode->table);
+    ioc_local_t *local = NULL;
+    ioc_inode_t *ioc_inode = NULL;
+    size_t destroy_size = 0;
+    struct iatt *local_stbuf = NULL;
+
+    local = frame->local;
+    ioc_inode = local->inode;
+    local_stbuf = stbuf;
+
+    if ((op_ret == -1) ||
+        ((op_ret >= 0) && !ioc_cache_still_valid(ioc_inode, stbuf))) {
+        gf_msg_debug(ioc_inode->table->xl->name, 0,
+                     "cache for inode(%p) is invalid. flushing all pages",
+                     ioc_inode);
+        /* NOTE: only pages with no waiting frames are flushed by
+         * ioc_inode_flush. page_fault will be generated for all
+         * the pages which have waiting frames by ioc_inode_wakeup()
+         */
+        ioc_inode_lock(ioc_inode);
+        {
+            destroy_size = __ioc_inode_flush(ioc_inode);
+            if (op_ret >= 0) {
+                ioc_inode->cache.mtime = stbuf->ia_mtime;
+                ioc_inode->cache.mtime_nsec = stbuf->ia_mtime_nsec;
+            }
+        }
+        ioc_inode_unlock(ioc_inode);
+        local_stbuf = NULL;
+    }
+
+    if (destroy_size) {
+        ioc_table_lock(ioc_inode->table);
+        {
+            ioc_inode->table->cache_used -= destroy_size;
         }
+        ioc_table_unlock(ioc_inode->table);
+    }
 
-        if (op_ret < 0)
-                local_stbuf = NULL;
+    if (op_ret < 0)
+        local_stbuf = NULL;
 
-        ioc_inode_lock (ioc_inode);
-        {
-                gettimeofday (&ioc_inode->cache.tv, NULL);
-        }
-        ioc_inode_unlock (ioc_inode);
+    ioc_inode_lock(ioc_inode);
+    {
+        ioc_inode->cache.last_revalidate = gf_time();
+    }
+    ioc_inode_unlock(ioc_inode);
 
-        ioc_inode_wakeup (frame, ioc_inode, local_stbuf);
+    ioc_inode_wakeup(frame, ioc_inode, local_stbuf);
 
-        /* any page-fault initiated by ioc_inode_wakeup() will have its own
-         * fd_ref on fd, safe to unref validate frame's private copy
-         */
-        fd_unref (local->fd);
+    /* any page-fault initiated by ioc_inode_wakeup() will have its own
+     * fd_ref on fd, safe to unref validate frame's private copy
+     */
+    fd_unref(local->fd);
+    dict_unref(local->xattr_req);
 
-        STACK_DESTROY (frame->root);
+    STACK_DESTROY(frame->root);
 
-        return 0;
+    return 0;
 }
 
 int32_t
-ioc_wait_on_inode (ioc_inode_t *ioc_inode, ioc_page_t *page)
+ioc_wait_on_inode(ioc_inode_t *ioc_inode, ioc_page_t *page)
 {
-        ioc_waitq_t *waiter     = NULL, *trav = NULL;
-        uint32_t     page_found = 0;
-        int32_t      ret        = 0;
+    ioc_waitq_t *waiter = NULL, *trav = NULL;
+    uint32_t page_found = 0;
+    int32_t ret = 0;
 
-        trav = ioc_inode->waitq;
+    trav = ioc_inode->waitq;
 
-        while (trav) {
-                if (trav->data == page) {
-                        page_found = 1;
-                        break;
-                }
-                trav = trav->next;
+    while (trav) {
+        if (trav->data == page) {
+            page_found = 1;
+            break;
         }
+        trav = trav->next;
+    }
 
-        if (!page_found) {
-                waiter = GF_CALLOC (1, sizeof (ioc_waitq_t),
-                                    gf_ioc_mt_ioc_waitq_t);
-                if (waiter == NULL) {
-                        gf_log (ioc_inode->table->xl->name, GF_LOG_ERROR,
-                                "out of memory");
-                        ret = -ENOMEM;
-                        goto out;
-                }
-
-                waiter->data = page;
-                waiter->next = ioc_inode->waitq;
-                ioc_inode->waitq = waiter;
+    if (!page_found) {
+        waiter = GF_CALLOC(1, sizeof(ioc_waitq_t), gf_ioc_mt_ioc_waitq_t);
+        if (waiter == NULL) {
+            gf_smsg(ioc_inode->table->xl->name, GF_LOG_ERROR, ENOMEM,
+                    IO_CACHE_MSG_NO_MEMORY, NULL);
+            ret = -ENOMEM;
+            goto out;
         }
 
+        waiter->data = page;
+        waiter->next = ioc_inode->waitq;
+        ioc_inode->waitq = waiter;
+    }
+
 out:
-        return ret;
+    return ret;
 }
 
 /*
@@ -454,74 +505,77 @@ out:
  *
  */
 int32_t
-ioc_cache_validate (call_frame_t *frame, ioc_inode_t *ioc_inode, fd_t *fd,
-                    ioc_page_t *page)
+ioc_cache_validate(call_frame_t *frame, ioc_inode_t *ioc_inode, fd_t *fd,
+                   ioc_page_t *page)
 {
-        call_frame_t *validate_frame = NULL;
-        ioc_local_t  *validate_local = NULL;
-        ioc_local_t  *local          = NULL;
-        int32_t       ret            = 0;
-
-        local = frame->local;
-        validate_local = mem_get0 (THIS->local_pool);
-        if (validate_local == NULL) {
-                ret = -1;
-                local->op_ret = -1;
-                local->op_errno = ENOMEM;
-                gf_log (ioc_inode->table->xl->name, GF_LOG_ERROR,
-                        "out of memory");
-                goto out;
-        }
-
-        validate_frame = copy_frame (frame);
-        if (validate_frame == NULL) {
-                ret = -1;
-                local->op_ret = -1;
-                local->op_errno = ENOMEM;
-                mem_put (validate_local);
-                gf_log (ioc_inode->table->xl->name, GF_LOG_ERROR,
-                        "out of memory");
-                goto out;
-        }
-
-        validate_local->fd = fd_ref (fd);
-        validate_local->inode = ioc_inode;
-        validate_frame->local = validate_local;
-
-        STACK_WIND (validate_frame, ioc_cache_validate_cbk,
-                    FIRST_CHILD (frame->this),
-                    FIRST_CHILD (frame->this)->fops->fstat, fd, NULL);
+    call_frame_t *validate_frame = NULL;
+    ioc_local_t *validate_local = NULL;
+    ioc_local_t *local = NULL;
+    int32_t ret = 0;
+
+    local = frame->local;
+    validate_local = mem_get0(THIS->local_pool);
+    if (validate_local == NULL) {
+        ret = -1;
+        local->op_ret = -1;
+        local->op_errno = ENOMEM;
+        gf_smsg(ioc_inode->table->xl->name, GF_LOG_ERROR, 0,
+                IO_CACHE_MSG_NO_MEMORY, NULL);
+        goto out;
+    }
+
+    validate_frame = copy_frame(frame);
+    if (validate_frame == NULL) {
+        ret = -1;
+        local->op_ret = -1;
+        local->op_errno = ENOMEM;
+        mem_put(validate_local);
+        gf_smsg(ioc_inode->table->xl->name, GF_LOG_ERROR, 0,
+                IO_CACHE_MSG_NO_MEMORY, NULL);
+        goto out;
+    }
+
+    validate_local->fd = fd_ref(fd);
+    validate_local->inode = ioc_inode;
+    if (local && local->xattr_req)
+        validate_local->xattr_req = dict_ref(local->xattr_req);
+    validate_frame->local = validate_local;
+
+    STACK_WIND(validate_frame, ioc_cache_validate_cbk, FIRST_CHILD(frame->this),
+               FIRST_CHILD(frame->this)->fops->fstat, fd,
+               validate_local->xattr_req);
 
 out:
-        return ret;
+    return ret;
 }
 
-static inline uint32_t
-is_match (const char *path, const char *pattern)
+static uint32_t
+is_match(const char *path, const char *pattern)
 {
-        int32_t ret = 0;
+    int32_t ret = 0;
 
-        ret = fnmatch (pattern, path, FNM_NOESCAPE);
+    ret = fnmatch(pattern, path, FNM_NOESCAPE);
 
-        return (ret == 0);
+    return (ret == 0);
 }
 
 uint32_t
-ioc_get_priority (ioc_table_t *table, const char *path)
+ioc_get_priority(ioc_table_t *table, const char *path)
 {
-        uint32_t             priority = 1;
-        struct ioc_priority *curr     = NULL;
+    uint32_t priority = 1;
+    struct ioc_priority *curr = NULL;
 
-        if (list_empty(&table->priority_list))
-                return priority;
+    if (list_empty(&table->priority_list) || !path)
+        return priority;
 
-        priority = 0;
-        list_for_each_entry (curr, &table->priority_list, list) {
-                if (is_match (path, curr->pattern))
-                        priority = curr->priority;
-        }
+    priority = 0;
+    list_for_each_entry(curr, &table->priority_list, list)
+    {
+        if (is_match(path, curr->pattern))
+            priority = curr->priority;
+    }
 
-        return priority;
+    return priority;
 }
 
 /*
@@ -536,75 +590,68 @@ ioc_get_priority (ioc_table_t *table, const char *path)
  *
  */
 int32_t
-ioc_open_cbk (call_frame_t *frame, void *cookie, xlator_t *this, int32_t op_ret,
-              int32_t op_errno, fd_t *fd, dict_t *xdata)
+ioc_open_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int32_t op_ret,
+             int32_t op_errno, fd_t *fd, dict_t *xdata)
 {
-        uint64_t     tmp_ioc_inode = 0;
-        ioc_local_t *local         = NULL;
-        ioc_table_t *table         = NULL;
-        ioc_inode_t *ioc_inode     = NULL;
-        uint32_t     weight        = 0xffffffff;
+    uint64_t tmp_ioc_inode = 0;
+    ioc_local_t *local = NULL;
+    ioc_table_t *table = NULL;
+    ioc_inode_t *ioc_inode = NULL;
+
+    local = frame->local;
+    if (!this || !this->private) {
+        op_ret = -1;
+        op_errno = EINVAL;
+        goto out;
+    }
+
+    table = this->private;
+
+    if (op_ret != -1) {
+        inode_ctx_get(fd->inode, this, &tmp_ioc_inode);
+        ioc_inode = (ioc_inode_t *)(long)tmp_ioc_inode;
 
-        local = frame->local;
-        if (!this || !this->private) {
-                op_ret = -1;
-                op_errno = EINVAL;
-                goto out;
+        // TODO: see why inode context is NULL and handle it.
+        if (!ioc_inode) {
+            gf_smsg(this->name, GF_LOG_ERROR, EINVAL,
+                    IO_CACHE_MSG_ENFORCEMENT_FAILED, "inode-gfid=%s",
+                    uuid_utoa(fd->inode->gfid), NULL);
+            goto out;
         }
 
-        table = this->private;
-
-        if (op_ret != -1) {
-                inode_ctx_get (fd->inode, this, &tmp_ioc_inode);
-                ioc_inode = (ioc_inode_t *)(long)tmp_ioc_inode;
-
-                //TODO: see why inode context is NULL and handle it.
-                if (!ioc_inode) {
-                        gf_log (this->name, GF_LOG_ERROR, "inode context is "
-                                "NULL (%s)", uuid_utoa (fd->inode->gfid));
-                        goto out;
-                }
-
-                ioc_table_lock (ioc_inode->table);
-                {
-                        list_move_tail (&ioc_inode->inode_lru,
-                                        &table->inode_lru[ioc_inode->weight]);
-                }
-                ioc_table_unlock (ioc_inode->table);
+        ioc_table_lock(ioc_inode->table);
+        {
+            list_move_tail(&ioc_inode->inode_lru,
+                           &table->inode_lru[ioc_inode->weight]);
+        }
+        ioc_table_unlock(ioc_inode->table);
 
-                ioc_inode_lock (ioc_inode);
-                {
-                        if ((table->min_file_size > ioc_inode->ia_size)
-                            || ((table->max_file_size > 0)
-                                && (table->max_file_size < ioc_inode->ia_size))) {
-                                fd_ctx_set (fd, this, 1);
-                        }
-                }
-                ioc_inode_unlock (ioc_inode);
-
-                /* If O_DIRECT open, we disable caching on it */
-                if ((local->flags & O_DIRECT)){
-                        /* O_DIRECT is only for one fd, not the inode
-                         * as a whole
-                         */
-                        fd_ctx_set (fd, this, 1);
-                }
+        ioc_inode_lock(ioc_inode);
+        {
+            if ((table->min_file_size > ioc_inode->ia_size) ||
+                ((table->max_file_size > 0) &&
+                 (table->max_file_size < ioc_inode->ia_size))) {
+                fd_ctx_set(fd, this, 1);
+            }
+        }
+        ioc_inode_unlock(ioc_inode);
 
-                /* weight = 0, we disable caching on it */
-                if (weight == 0) {
-                        /* we allow a pattern-matched cache disable this way
-                         */
-                        fd_ctx_set (fd, this, 1);
-                }
+        /* If O_DIRECT open, we disable caching on it */
+        if ((local->flags & O_DIRECT)) {
+            /* O_DIRECT is only for one fd, not the inode
+             * as a whole
+             */
+            fd_ctx_set(fd, this, 1);
         }
+    }
 
 out:
-        mem_put (local);
-        frame->local = NULL;
+    mem_put(local);
+    frame->local = NULL;
 
-        STACK_UNWIND_STRICT (open, frame, op_ret, op_errno, fd, xdata);
+    STACK_UNWIND_STRICT(open, frame, op_ret, op_errno, fd, xdata);
 
-        return 0;
+    return 0;
 }
 
 /*
@@ -621,184 +668,175 @@ out:
  *
  */
 int32_t
-ioc_create_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                int32_t op_ret,	int32_t op_errno, fd_t *fd,
-                inode_t *inode,	struct iatt *buf, struct iatt *preparent,
-                struct iatt *postparent, dict_t *xdata)
+ioc_create_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+               int32_t op_ret, int32_t op_errno, fd_t *fd, inode_t *inode,
+               struct iatt *buf, struct iatt *preparent,
+               struct iatt *postparent, dict_t *xdata)
 {
-        ioc_local_t *local     = NULL;
-        ioc_table_t *table     = NULL;
-        ioc_inode_t *ioc_inode = NULL;
-        uint32_t     weight    = 0xffffffff;
-        const char  *path      = NULL;
-        int          ret       = -1;
-
-        local = frame->local;
-        if (!this || !this->private) {
-                op_ret = -1;
-                op_errno = EINVAL;
-                goto out;
-        }
-
-        table = this->private;
-        path = local->file_loc.path;
-
-        if (op_ret != -1) {
-                /* assign weight */
-                weight = ioc_get_priority (table, path);
-
-                ioc_inode = ioc_inode_update (table, inode, weight);
-
-                ioc_inode_lock (ioc_inode);
-                {
-                        ioc_inode->cache.mtime = buf->ia_mtime;
-                        ioc_inode->cache.mtime_nsec = buf->ia_mtime_nsec;
-                        ioc_inode->ia_size = buf->ia_size;
-
-                        if ((table->min_file_size > ioc_inode->ia_size)
-                            || ((table->max_file_size > 0)
-                                && (table->max_file_size < ioc_inode->ia_size))) {
-                                ret = fd_ctx_set (fd, this, 1);
-                                if (ret)
-                                        gf_log (this->name, GF_LOG_WARNING,
-                                                "%s: failed to set fd ctx",
-                                                local->file_loc.path);
-                        }
-                }
-                ioc_inode_unlock (ioc_inode);
-
-                inode_ctx_put (fd->inode, this,
-                               (uint64_t)(long)ioc_inode);
-
-                /* If O_DIRECT open, we disable caching on it */
-                if (local->flags & O_DIRECT) {
-                        /*
-                         * O_DIRECT is only for one fd, not the inode
-                         * as a whole */
-                        ret = fd_ctx_set (fd, this, 1);
-                        if (ret)
-                                gf_log (this->name, GF_LOG_WARNING,
-                                        "%s: failed to set fd ctx",
-                                        local->file_loc.path);
-                }
-
-                /* if weight == 0, we disable caching on it */
-                if (!weight) {
-                        /* we allow a pattern-matched cache disable this way */
-                        ret = fd_ctx_set (fd, this, 1);
-                        if (ret)
-                                gf_log (this->name, GF_LOG_WARNING,
-                                        "%s: failed to set fd ctx",
-                                        local->file_loc.path);
-                }
-
-        }
+    ioc_local_t *local = NULL;
+    ioc_table_t *table = NULL;
+    ioc_inode_t *ioc_inode = NULL;
+    uint32_t weight = 0xffffffff;
+    const char *path = NULL;
+    int ret = -1;
+
+    local = frame->local;
+    if (!this || !this->private) {
+        op_ret = -1;
+        op_errno = EINVAL;
+        goto out;
+    }
+
+    table = this->private;
+    path = local->file_loc.path;
+
+    if (op_ret != -1) {
+        /* assign weight */
+        weight = ioc_get_priority(table, path);
+
+        ioc_inode = ioc_inode_create(table, inode, weight);
+
+        ioc_inode_lock(ioc_inode);
+        {
+            ioc_inode->cache.mtime = buf->ia_mtime;
+            ioc_inode->cache.mtime_nsec = buf->ia_mtime_nsec;
+            ioc_inode->ia_size = buf->ia_size;
+
+            if ((table->min_file_size > ioc_inode->ia_size) ||
+                ((table->max_file_size > 0) &&
+                 (table->max_file_size < ioc_inode->ia_size))) {
+                ret = fd_ctx_set(fd, this, 1);
+                if (ret)
+                    gf_smsg(this->name, GF_LOG_WARNING, ENOMEM,
+                            IO_CACHE_MSG_SET_FD_FAILED, "path=%s",
+                            local->file_loc.path, NULL);
+            }
+        }
+        ioc_inode_unlock(ioc_inode);
+
+        inode_ctx_put(fd->inode, this, (uint64_t)(long)ioc_inode);
+
+        /* If O_DIRECT open, we disable caching on it */
+        if (local->flags & O_DIRECT) {
+            /*
+             * O_DIRECT is only for one fd, not the inode
+             * as a whole */
+            ret = fd_ctx_set(fd, this, 1);
+            if (ret)
+                gf_smsg(this->name, GF_LOG_WARNING, ENOMEM,
+                        IO_CACHE_MSG_SET_FD_FAILED, "path=%s",
+                        local->file_loc.path, NULL);
+        }
+
+        /* if weight == 0, we disable caching on it */
+        if (!weight) {
+            /* we allow a pattern-matched cache disable this way */
+            ret = fd_ctx_set(fd, this, 1);
+            if (ret)
+                gf_smsg(this->name, GF_LOG_WARNING, ENOMEM,
+                        IO_CACHE_MSG_SET_FD_FAILED, "path=%s",
+                        local->file_loc.path, NULL);
+        }
+    }
 
 out:
-        frame->local = NULL;
-        mem_put (local);
+    frame->local = NULL;
+    mem_put(local);
 
-        STACK_UNWIND_STRICT (create, frame, op_ret, op_errno, fd, inode, buf,
-                             preparent, postparent, xdata);
+    STACK_UNWIND_STRICT(create, frame, op_ret, op_errno, fd, inode, buf,
+                        preparent, postparent, xdata);
 
-        return 0;
+    return 0;
 }
 
-
 int32_t
-ioc_mknod_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-               int32_t op_ret, int32_t op_errno, inode_t *inode,
-               struct iatt *buf, struct iatt *preparent,
-               struct iatt *postparent, dict_t *xdata)
+ioc_mknod_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int32_t op_ret,
+              int32_t op_errno, inode_t *inode, struct iatt *buf,
+              struct iatt *preparent, struct iatt *postparent, dict_t *xdata)
 {
-        ioc_local_t *local     = NULL;
-        ioc_table_t *table     = NULL;
-        ioc_inode_t *ioc_inode = NULL;
-        uint32_t     weight    = 0xffffffff;
-        const char  *path      = NULL;
+    ioc_local_t *local = NULL;
+    ioc_table_t *table = NULL;
+    ioc_inode_t *ioc_inode = NULL;
+    uint32_t weight = 0xffffffff;
+    const char *path = NULL;
 
-        local = frame->local;
-        if (!this || !this->private) {
-                op_ret = -1;
-                op_errno = EINVAL;
-                goto out;
-        }
-
-        table = this->private;
-        path = local->file_loc.path;
+    local = frame->local;
+    if (!this || !this->private) {
+        op_ret = -1;
+        op_errno = EINVAL;
+        goto out;
+    }
 
-        if (op_ret != -1) {
-                /* assign weight */
-                weight = ioc_get_priority (table, path);
+    table = this->private;
+    path = local->file_loc.path;
 
-                ioc_inode = ioc_inode_update (table, inode, weight);
+    if (op_ret != -1) {
+        /* assign weight */
+        weight = ioc_get_priority(table, path);
 
-                ioc_inode_lock (ioc_inode);
-                {
-                        ioc_inode->cache.mtime = buf->ia_mtime;
-                        ioc_inode->cache.mtime_nsec = buf->ia_mtime_nsec;
-                        ioc_inode->ia_size = buf->ia_size;
-                }
-                ioc_inode_unlock (ioc_inode);
+        ioc_inode = ioc_inode_create(table, inode, weight);
 
-                inode_ctx_put (inode, this,
-                               (uint64_t)(long)ioc_inode);
+        ioc_inode_lock(ioc_inode);
+        {
+            ioc_inode->cache.mtime = buf->ia_mtime;
+            ioc_inode->cache.mtime_nsec = buf->ia_mtime_nsec;
+            ioc_inode->ia_size = buf->ia_size;
         }
+        ioc_inode_unlock(ioc_inode);
+
+        inode_ctx_put(inode, this, (uint64_t)(long)ioc_inode);
+    }
 
 out:
-        frame->local = NULL;
+    frame->local = NULL;
 
-        loc_wipe (&local->file_loc);
-        mem_put (local);
+    loc_wipe(&local->file_loc);
+    mem_put(local);
 
-        STACK_UNWIND_STRICT (mknod, frame, op_ret, op_errno, inode, buf,
-                             preparent, postparent, xdata);
-        return 0;
+    STACK_UNWIND_STRICT(mknod, frame, op_ret, op_errno, inode, buf, preparent,
+                        postparent, xdata);
+    return 0;
 }
 
-
 int
-ioc_mknod (call_frame_t *frame, xlator_t *this, loc_t *loc, mode_t mode,
-           dev_t rdev, mode_t umask, dict_t *xdata)
+ioc_mknod(call_frame_t *frame, xlator_t *this, loc_t *loc, mode_t mode,
+          dev_t rdev, mode_t umask, dict_t *xdata)
 {
-        ioc_local_t *local    = NULL;
-        int32_t      op_errno = -1, ret = -1;
+    ioc_local_t *local = NULL;
+    int32_t op_errno = -1, ret = -1;
 
-        local = mem_get0 (this->local_pool);
-        if (local == NULL) {
-                op_errno = ENOMEM;
-                gf_log (this->name, GF_LOG_ERROR, "out of memory");
-                goto unwind;
-        }
+    local = mem_get0(this->local_pool);
+    if (local == NULL) {
+        op_errno = ENOMEM;
+        gf_smsg(this->name, GF_LOG_ERROR, 0, IO_CACHE_MSG_NO_MEMORY, NULL);
+        goto unwind;
+    }
 
-        ret = loc_copy (&local->file_loc, loc);
-        if (ret != 0) {
-                op_errno = ENOMEM;
-                gf_log (this->name, GF_LOG_ERROR, "out of memory");
-                goto unwind;
-        }
+    ret = loc_copy(&local->file_loc, loc);
+    if (ret != 0) {
+        op_errno = ENOMEM;
+        gf_smsg(this->name, GF_LOG_ERROR, 0, IO_CACHE_MSG_NO_MEMORY, NULL);
+        goto unwind;
+    }
 
-        frame->local = local;
+    frame->local = local;
 
-        STACK_WIND (frame, ioc_mknod_cbk,
-                    FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->mknod,
-                    loc, mode, rdev, umask, xdata);
-        return 0;
+    STACK_WIND(frame, ioc_mknod_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->mknod, loc, mode, rdev, umask, xdata);
+    return 0;
 
 unwind:
-        if (local != NULL) {
-                loc_wipe (&local->file_loc);
-                mem_put (local);
-        }
+    if (local != NULL) {
+        loc_wipe(&local->file_loc);
+        mem_put(local);
+    }
 
-        STACK_UNWIND_STRICT (mknod, frame, -1, op_errno, NULL, NULL,
-                             NULL, NULL, NULL);
+    STACK_UNWIND_STRICT(mknod, frame, -1, op_errno, NULL, NULL, NULL, NULL,
+                        NULL);
 
-        return 0;
+    return 0;
 }
 
-
 /*
  * ioc_open - open fop for io cache
  * @frame:
@@ -808,30 +846,28 @@ unwind:
  *
  */
 int32_t
-ioc_open (call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t flags,
-          fd_t *fd, dict_t *xdata)
+ioc_open(call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t flags,
+         fd_t *fd, dict_t *xdata)
 {
+    ioc_local_t *local = NULL;
 
-        ioc_local_t *local = NULL;
-
-        local = mem_get0 (this->local_pool);
-        if (local == NULL) {
-                gf_log (this->name, GF_LOG_ERROR, "out of memory");
-                STACK_UNWIND_STRICT (open, frame, -1, ENOMEM, NULL, NULL);
-                return 0;
-        }
+    local = mem_get0(this->local_pool);
+    if (local == NULL) {
+        gf_smsg(this->name, GF_LOG_ERROR, ENOMEM, IO_CACHE_MSG_NO_MEMORY, NULL);
+        STACK_UNWIND_STRICT(open, frame, -1, ENOMEM, NULL, NULL);
+        return 0;
+    }
 
-        local->flags = flags;
-        local->file_loc.path = loc->path;
-        local->file_loc.inode = loc->inode;
+    local->flags = flags;
+    local->file_loc.path = loc->path;
+    local->file_loc.inode = loc->inode;
 
-        frame->local = local;
+    frame->local = local;
 
-        STACK_WIND (frame, ioc_open_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->open, loc, flags, fd,
-                    xdata);
+    STACK_WIND(frame, ioc_open_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->open, loc, flags, fd, xdata);
 
-        return 0;
+    return 0;
 }
 
 /*
@@ -845,32 +881,29 @@ ioc_open (call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t flags,
  *
  */
 int32_t
-ioc_create (call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t flags,
-            mode_t mode, mode_t umask, fd_t *fd, dict_t *xdata)
+ioc_create(call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t flags,
+           mode_t mode, mode_t umask, fd_t *fd, dict_t *xdata)
 {
-        ioc_local_t *local = NULL;
-
-        local = mem_get0 (this->local_pool);
-        if (local == NULL) {
-                gf_log (this->name, GF_LOG_ERROR, "out of memory");
-                STACK_UNWIND_STRICT (create, frame, -1, ENOMEM, NULL, NULL,
-                                     NULL, NULL, NULL, NULL);
-                return 0;
-        }
-
-        local->flags = flags;
-        local->file_loc.path = loc->path;
-        frame->local = local;
-
-        STACK_WIND (frame, ioc_create_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->create, loc, flags, mode,
-                    umask, fd, xdata);
+    ioc_local_t *local = NULL;
 
+    local = mem_get0(this->local_pool);
+    if (local == NULL) {
+        gf_smsg(this->name, GF_LOG_ERROR, ENOMEM, IO_CACHE_MSG_NO_MEMORY, NULL);
+        STACK_UNWIND_STRICT(create, frame, -1, ENOMEM, NULL, NULL, NULL, NULL,
+                            NULL, NULL);
         return 0;
-}
+    }
 
+    local->flags = flags;
+    local->file_loc.path = loc->path;
+    frame->local = local;
 
+    STACK_WIND(frame, ioc_create_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->create, loc, flags, mode, umask, fd,
+               xdata);
 
+    return 0;
+}
 
 /*
  * ioc_release - release fop for io cache
@@ -881,49 +914,26 @@ ioc_create (call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t flags,
  *
  */
 int32_t
-ioc_release (xlator_t *this, fd_t *fd)
+ioc_release(xlator_t *this, fd_t *fd)
 {
-        return 0;
+    return 0;
 }
 
-/*
- * ioc_readv_disabled_cbk
- * @frame:
- * @cookie:
- * @this:
- * @op_ret:
- * @op_errno:
- * @vector:
- * @count:
- *
- */
 int32_t
-ioc_readv_disabled_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                        int32_t op_ret,	int32_t op_errno, struct iovec *vector,
-                        int32_t count, struct iatt *stbuf,
-                        struct iobref *iobref, dict_t *xdata)
+ioc_need_prune(ioc_table_t *table)
 {
-        STACK_UNWIND_STRICT (readv, frame, op_ret, op_errno, vector, count,
-                             stbuf, iobref, xdata);
-        return 0;
-}
+    int64_t cache_difference = 0;
 
+    ioc_table_lock(table);
+    {
+        cache_difference = table->cache_used - table->cache_size;
+    }
+    ioc_table_unlock(table);
 
-int32_t
-ioc_need_prune (ioc_table_t *table)
-{
-        int64_t cache_difference = 0;
-
-        ioc_table_lock (table);
-        {
-                cache_difference = table->cache_used - table->cache_size;
-        }
-        ioc_table_unlock (table);
-
-        if (cache_difference > 0)
-                return 1;
-        else
-                return 0;
+    if (cache_difference > 0)
+        return 1;
+    else
+        return 0;
 }
 
 /*
@@ -935,159 +945,151 @@ ioc_need_prune (ioc_table_t *table)
  *
  */
 void
-ioc_dispatch_requests (call_frame_t *frame, ioc_inode_t *ioc_inode, fd_t *fd,
-                       off_t offset, size_t size)
+ioc_dispatch_requests(call_frame_t *frame, ioc_inode_t *ioc_inode, fd_t *fd,
+                      off_t offset, size_t size)
 {
-        ioc_local_t *local               = NULL;
-        ioc_table_t *table               = NULL;
-        ioc_page_t  *trav                = NULL;
-        ioc_waitq_t *waitq               = NULL;
-        off_t        rounded_offset      = 0;
-        off_t        rounded_end         = 0;
-        off_t        trav_offset         = 0;
-        int32_t      fault               = 0;
-        size_t       trav_size           = 0;
-        off_t        local_offset        = 0;
-        int32_t      ret                 = -1;
-        int8_t       need_validate       = 0;
-        int8_t       might_need_validate = 0;  /*
-                                                * if a page exists, do we need
-                                                * to validate it?
-                                                */
-        local = frame->local;
-        table = ioc_inode->table;
-
-        rounded_offset = floor (offset, table->page_size);
-        rounded_end = roof (offset + size, table->page_size);
-        trav_offset = rounded_offset;
-
-        /* once a frame does read, it should be waiting on something */
-        local->wait_count++;
-
-        /* Requested region can fall in three different pages,
-         * 1. Ready - region is already in cache, we just have to serve it.
-         * 2. In-transit - page fault has been generated on this page, we need
-         *    to wait till the page is ready
-         * 3. Fault - page is not in cache, we have to generate a page fault
-         */
+    ioc_local_t *local = NULL;
+    ioc_table_t *table = NULL;
+    ioc_page_t *trav = NULL;
+    ioc_waitq_t *waitq = NULL;
+    off_t rounded_offset = 0;
+    off_t rounded_end = 0;
+    off_t trav_offset = 0;
+    int32_t fault = 0;
+    size_t trav_size = 0;
+    off_t local_offset = 0;
+    int32_t ret = -1;
+    int8_t need_validate = 0;
+    int8_t might_need_validate = 0; /*
+                                     * if a page exists, do we need
+                                     * to validate it?
+                                     */
+    local = frame->local;
+    table = ioc_inode->table;
+
+    rounded_offset = gf_floor(offset, table->page_size);
+    rounded_end = gf_roof(offset + size, table->page_size);
+    trav_offset = rounded_offset;
+
+    /* once a frame does read, it should be waiting on something */
+    local->wait_count++;
+
+    /* Requested region can fall in three different pages,
+     * 1. Ready - region is already in cache, we just have to serve it.
+     * 2. In-transit - page fault has been generated on this page, we need
+     *    to wait till the page is ready
+     * 3. Fault - page is not in cache, we have to generate a page fault
+     */
+
+    might_need_validate = ioc_inode_need_revalidate(ioc_inode);
+
+    while (trav_offset < rounded_end) {
+        ioc_inode_lock(ioc_inode);
+        {
+            /* look for requested region in the cache */
+            trav = __ioc_page_get(ioc_inode, trav_offset);
 
-        might_need_validate = ioc_inode_need_revalidate (ioc_inode);
+            local_offset = max(trav_offset, offset);
+            trav_size = min(((offset + size) - local_offset), table->page_size);
 
-        while (trav_offset < rounded_end) {
-                ioc_inode_lock (ioc_inode);
-                {
-                        /* look for requested region in the cache */
-                        trav = __ioc_page_get (ioc_inode, trav_offset);
-
-                        local_offset = max (trav_offset, offset);
-                        trav_size = min (((offset+size) - local_offset),
-                                         table->page_size);
-
-                        if (!trav) {
-                                /* page not in cache, we need to generate page
-                                 * fault
-                                 */
-                                trav = __ioc_page_create (ioc_inode,
-                                                          trav_offset);
-                                fault = 1;
-                                if (!trav) {
-                                        gf_log (frame->this->name,
-                                                GF_LOG_CRITICAL,
-                                                "out of memory");
-                                        local->op_ret = -1;
-                                        local->op_errno = ENOMEM;
-                                        ioc_inode_unlock (ioc_inode);
-                                        goto out;
-                                }
-                        }
+            if (!trav) {
+                /* page not in cache, we need to generate page
+                 * fault
+                 */
+                trav = __ioc_page_create(ioc_inode, trav_offset);
+                fault = 1;
+                if (!trav) {
+                    gf_smsg(frame->this->name, GF_LOG_CRITICAL, ENOMEM,
+                            IO_CACHE_MSG_NO_MEMORY, NULL);
+                    local->op_ret = -1;
+                    local->op_errno = ENOMEM;
+                    ioc_inode_unlock(ioc_inode);
+                    goto out;
+                }
+            }
+
+            __ioc_wait_on_page(trav, frame, local_offset, trav_size);
+
+            if (trav->ready) {
+                /* page found in cache */
+                if (!might_need_validate && !ioc_inode->waitq) {
+                    /* fresh enough */
+                    gf_msg_trace(frame->this->name, 0,
+                                 "cache hit for "
+                                 "trav_offset=%" PRId64
+                                 "/local_"
+                                 "offset=%" PRId64 "",
+                                 trav_offset, local_offset);
+                    waitq = __ioc_page_wakeup(trav, trav->op_errno);
+                } else {
+                    /* if waitq already exists, fstat
+                     * revalidate is
+                     * already on the way
+                     */
+                    if (!ioc_inode->waitq) {
+                        need_validate = 1;
+                    }
+
+                    ret = ioc_wait_on_inode(ioc_inode, trav);
+                    if (ret < 0) {
+                        local->op_ret = -1;
+                        local->op_errno = -ret;
+                        need_validate = 0;
 
-                        __ioc_wait_on_page (trav, frame, local_offset,
-                                            trav_size);
-
-                        if (trav->ready) {
-                                /* page found in cache */
-                                if (!might_need_validate && !ioc_inode->waitq) {
-                                        /* fresh enough */
-                                        gf_log (frame->this->name, GF_LOG_TRACE,
-                                                "cache hit for trav_offset=%"
-                                                PRId64"/local_offset=%"PRId64"",
-                                                trav_offset, local_offset);
-                                        waitq = __ioc_page_wakeup (trav,
-                                                                   trav->op_errno);
-                                } else {
-                                        /* if waitq already exists, fstat
-                                         * revalidate is
-                                         * already on the way
-                                         */
-                                        if (!ioc_inode->waitq) {
-                                                need_validate = 1;
-                                        }
-
-                                        ret = ioc_wait_on_inode (ioc_inode,
-                                                                 trav);
-                                        if (ret < 0) {
-                                                local->op_ret = -1;
-                                                local->op_errno = -ret;
-                                                need_validate = 0;
-
-                                                waitq = __ioc_page_wakeup (trav,
-                                                                           trav->op_errno);
-                                                ioc_inode_unlock (ioc_inode);
-
-                                                ioc_waitq_return (waitq);
-                                                waitq = NULL;
-                                                goto out;
-                                        }
-                                }
-                        }
+                        waitq = __ioc_page_wakeup(trav, trav->op_errno);
+                        ioc_inode_unlock(ioc_inode);
 
+                        ioc_waitq_return(waitq);
+                        waitq = NULL;
+                        goto out;
+                    }
                 }
-                ioc_inode_unlock (ioc_inode);
+            }
+        }
+        ioc_inode_unlock(ioc_inode);
 
-                ioc_waitq_return (waitq);
-                waitq = NULL;
+        ioc_waitq_return(waitq);
+        waitq = NULL;
 
-                if (fault) {
-                        fault = 0;
-                        /* new page created, increase the table->cache_used */
-                        ioc_page_fault (ioc_inode, frame, fd, trav_offset);
-                }
+        if (fault) {
+            fault = 0;
+            /* new page created, increase the table->cache_used */
+            ioc_page_fault(ioc_inode, frame, fd, trav_offset);
+        }
 
-                if (need_validate) {
-                        need_validate = 0;
-                        gf_log (frame->this->name, GF_LOG_TRACE,
-                                "sending validate request for "
-                                "inode(%s) at offset=%"PRId64"",
-                                uuid_utoa (fd->inode->gfid), trav_offset);
-                        ret = ioc_cache_validate (frame, ioc_inode, fd, trav);
-                        if (ret == -1) {
-                                ioc_inode_lock (ioc_inode);
-                                {
-                                        waitq = __ioc_page_wakeup (trav,
-                                                                   trav->op_errno);
-                                }
-                                ioc_inode_unlock (ioc_inode);
-
-                                ioc_waitq_return (waitq);
-                                waitq = NULL;
-                                goto out;
-                        }
+        if (need_validate) {
+            need_validate = 0;
+            gf_msg_trace(frame->this->name, 0,
+                         "sending validate request for "
+                         "inode(%s) at offset=%" PRId64 "",
+                         uuid_utoa(fd->inode->gfid), trav_offset);
+            ret = ioc_cache_validate(frame, ioc_inode, fd, trav);
+            if (ret == -1) {
+                ioc_inode_lock(ioc_inode);
+                {
+                    waitq = __ioc_page_wakeup(trav, trav->op_errno);
                 }
+                ioc_inode_unlock(ioc_inode);
 
-                trav_offset += table->page_size;
+                ioc_waitq_return(waitq);
+                waitq = NULL;
+                goto out;
+            }
         }
 
+        trav_offset += table->page_size;
+    }
+
 out:
-        ioc_frame_return (frame);
+    ioc_frame_return(frame);
 
-        if (ioc_need_prune (ioc_inode->table)) {
-                ioc_prune (ioc_inode->table);
-        }
+    if (ioc_need_prune(ioc_inode->table)) {
+        ioc_prune(ioc_inode->table);
+    }
 
-        return;
+    return;
 }
 
-
 /*
  * ioc_readv -
  *
@@ -1099,103 +1101,108 @@ out:
  *
  */
 int32_t
-ioc_readv (call_frame_t *frame, xlator_t *this, fd_t *fd,
-           size_t size, off_t offset, uint32_t flags, dict_t *xdata)
+ioc_readv(call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
+          off_t offset, uint32_t flags, dict_t *xdata)
 {
-        uint64_t     tmp_ioc_inode = 0;
-        ioc_inode_t *ioc_inode     = NULL;
-        ioc_local_t *local         = NULL;
-        uint32_t     weight        = 0;
-        ioc_table_t *table         = NULL;
-        int32_t      op_errno      = -1;
-
-        if (!this) {
-                goto out;
-        }
-
-        inode_ctx_get (fd->inode, this, &tmp_ioc_inode);
-        ioc_inode = (ioc_inode_t *)(long)tmp_ioc_inode;
-        if (!ioc_inode) {
-                /* caching disabled, go ahead with normal readv */
-                STACK_WIND (frame, ioc_readv_disabled_cbk,
-                            FIRST_CHILD (frame->this),
-                            FIRST_CHILD (frame->this)->fops->readv, fd, size,
-                            offset, flags, xdata);
-                return 0;
-        }
+    uint64_t tmp_ioc_inode = 0;
+    ioc_inode_t *ioc_inode = NULL;
+    ioc_local_t *local = NULL;
+    uint32_t weight = 0;
+    ioc_table_t *table = NULL;
+    int32_t op_errno = EINVAL;
+
+    if (!this) {
+        goto out;
+    }
+
+    inode_ctx_get(fd->inode, this, &tmp_ioc_inode);
+    ioc_inode = (ioc_inode_t *)(long)tmp_ioc_inode;
+    if (!ioc_inode) {
+        /* caching disabled, go ahead with normal readv */
+        STACK_WIND_TAIL(frame, FIRST_CHILD(this),
+                        FIRST_CHILD(this)->fops->readv, fd, size, offset, flags,
+                        xdata);
+        return 0;
+    }
 
+    if (flags & O_DIRECT) {
+        /* disable caching for this fd, if O_DIRECT is used */
+        STACK_WIND_TAIL(frame, FIRST_CHILD(this),
+                        FIRST_CHILD(this)->fops->readv, fd, size, offset, flags,
+                        xdata);
+        return 0;
+    }
 
-        table = this->private;
+    table = this->private;
 
-        if (!table) {
-                gf_log (this->name, GF_LOG_ERROR, "table is null");
-                op_errno = EINVAL;
-                goto out;
-        }
+    if (!table) {
+        gf_smsg(this->name, GF_LOG_ERROR, EINVAL, IO_CACHE_MSG_TABLE_NULL,
+                NULL);
+        op_errno = EINVAL;
+        goto out;
+    }
 
-        ioc_inode_lock (ioc_inode);
-        {
-                if (!ioc_inode->cache.page_table) {
-                        ioc_inode->cache.page_table
-                                = rbthash_table_init
-                                (IOC_PAGE_TABLE_BUCKET_COUNT,
-                                 ioc_hashfn, NULL, 0,
-                                 table->mem_pool);
-
-                        if (ioc_inode->cache.page_table == NULL) {
-                                op_errno = ENOMEM;
-                                ioc_inode_unlock (ioc_inode);
-                                goto out;
-                        }
-                }
-        }
-        ioc_inode_unlock (ioc_inode);
-
-        if (!fd_ctx_get (fd, this, NULL)) {
-                /* disable caching for this fd, go ahead with normal readv */
-                STACK_WIND (frame, ioc_readv_disabled_cbk,
-                            FIRST_CHILD (frame->this),
-                            FIRST_CHILD (frame->this)->fops->readv, fd, size,
-                            offset, flags, xdata);
-                return 0;
-        }
+    ioc_inode_lock(ioc_inode);
+    {
+        if (!ioc_inode->cache.page_table) {
+            ioc_inode->cache.page_table = rbthash_table_init(
+                this->ctx, IOC_PAGE_TABLE_BUCKET_COUNT, ioc_hashfn, NULL, 0,
+                table->mem_pool);
 
-        local = mem_get0 (this->local_pool);
-        if (local == NULL) {
-                gf_log (this->name, GF_LOG_ERROR, "out of memory");
+            if (ioc_inode->cache.page_table == NULL) {
                 op_errno = ENOMEM;
+                ioc_inode_unlock(ioc_inode);
                 goto out;
+            }
         }
+    }
+    ioc_inode_unlock(ioc_inode);
 
-        INIT_LIST_HEAD (&local->fill_list);
-
-        frame->local = local;
-        local->pending_offset = offset;
-        local->pending_size = size;
-        local->offset = offset;
-        local->size = size;
-        local->inode = ioc_inode;
-
-        gf_log (this->name, GF_LOG_TRACE,
-                "NEW REQ (%p) offset = %"PRId64" && size = %"GF_PRI_SIZET"",
-                frame, offset, size);
-
-        weight = ioc_inode->weight;
-
-        ioc_table_lock (ioc_inode->table);
-        {
-                list_move_tail (&ioc_inode->inode_lru,
-                                &ioc_inode->table->inode_lru[weight]);
-        }
-        ioc_table_unlock (ioc_inode->table);
-
-        ioc_dispatch_requests (frame, ioc_inode, fd, offset, size);
+    if (!fd_ctx_get(fd, this, NULL)) {
+        /* disable caching for this fd, go ahead with normal readv */
+        STACK_WIND_TAIL(frame, FIRST_CHILD(this),
+                        FIRST_CHILD(this)->fops->readv, fd, size, offset, flags,
+                        xdata);
         return 0;
+    }
+
+    local = mem_get0(this->local_pool);
+    if (local == NULL) {
+        gf_smsg(this->name, GF_LOG_ERROR, ENOMEM, IO_CACHE_MSG_NO_MEMORY, NULL);
+        op_errno = ENOMEM;
+        goto out;
+    }
+
+    INIT_LIST_HEAD(&local->fill_list);
+
+    frame->local = local;
+    local->pending_offset = offset;
+    local->pending_size = size;
+    local->offset = offset;
+    local->size = size;
+    local->inode = ioc_inode;
+    local->xattr_req = dict_ref(xdata);
+
+    gf_msg_trace(this->name, 0,
+                 "NEW REQ (%p) offset "
+                 "= %" PRId64 " && size = %" GF_PRI_SIZET "",
+                 frame, offset, size);
+
+    weight = ioc_inode->weight;
+
+    ioc_table_lock(ioc_inode->table);
+    {
+        list_move_tail(&ioc_inode->inode_lru,
+                       &ioc_inode->table->inode_lru[weight]);
+    }
+    ioc_table_unlock(ioc_inode->table);
+
+    ioc_dispatch_requests(frame, ioc_inode, fd, offset, size);
+    return 0;
 
 out:
-        STACK_UNWIND_STRICT (readv, frame, -1, op_errno, NULL, 0, NULL, NULL,
-                             NULL);
-        return 0;
+    STACK_UNWIND_STRICT(readv, frame, -1, op_errno, NULL, 0, NULL, NULL, NULL);
+    return 0;
 }
 
 /*
@@ -1209,22 +1216,31 @@ out:
  *
  */
 int32_t
-ioc_writev_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                int32_t op_ret,	int32_t op_errno, struct iatt *prebuf,
-                struct iatt *postbuf, dict_t *xdata)
+ioc_writev_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+               int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
+               struct iatt *postbuf, dict_t *xdata)
 {
-        ioc_local_t *local     = NULL;
-        uint64_t     ioc_inode = 0;
-
-        local = frame->local;
-        inode_ctx_get (local->fd->inode, this, &ioc_inode);
-
-        if (ioc_inode)
-                ioc_inode_flush ((ioc_inode_t *)(long)ioc_inode);
-
-        STACK_UNWIND_STRICT (writev, frame, op_ret, op_errno, prebuf, postbuf,
-                             xdata);
-        return 0;
+    ioc_local_t *local = NULL;
+    uint64_t ioc_inode = 0;
+
+    local = frame->local;
+    frame->local = NULL;
+    inode_ctx_get(local->fd->inode, this, &ioc_inode);
+
+    if (op_ret >= 0) {
+        ioc_update_pages(frame, (ioc_inode_t *)(long)ioc_inode, local->vector,
+                         local->op_ret, op_ret, local->offset);
+    }
+
+    STACK_UNWIND_STRICT(writev, frame, op_ret, op_errno, prebuf, postbuf,
+                        xdata);
+    if (local->iobref) {
+        iobref_unref(local->iobref);
+        GF_FREE(local->vector);
+    }
+
+    mem_put(local);
+    return 0;
 }
 
 /*
@@ -1239,34 +1255,38 @@ ioc_writev_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
  *
  */
 int32_t
-ioc_writev (call_frame_t *frame, xlator_t *this, fd_t *fd,
-            struct iovec *vector, int32_t count, off_t offset,
-            uint32_t flags, struct iobref *iobref, dict_t *xdata)
+ioc_writev(call_frame_t *frame, xlator_t *this, fd_t *fd, struct iovec *vector,
+           int32_t count, off_t offset, uint32_t flags, struct iobref *iobref,
+           dict_t *xdata)
 {
-        ioc_local_t *local     = NULL;
-        uint64_t     ioc_inode = 0;
+    ioc_local_t *local = NULL;
+    uint64_t ioc_inode = 0;
 
-        local = mem_get0 (this->local_pool);
-        if (local == NULL) {
-                gf_log (this->name, GF_LOG_ERROR, "out of memory");
+    local = mem_get0(this->local_pool);
+    if (local == NULL) {
+        gf_smsg(this->name, GF_LOG_ERROR, ENOMEM, IO_CACHE_MSG_NO_MEMORY, NULL);
 
-                STACK_UNWIND_STRICT (writev, frame, -1, ENOMEM, NULL, NULL, NULL);
-                return 0;
-        }
+        STACK_UNWIND_STRICT(writev, frame, -1, ENOMEM, NULL, NULL, NULL);
+        return 0;
+    }
 
-        /* TODO: why is it not fd_ref'ed */
-        local->fd = fd;
-        frame->local = local;
+    /* TODO: why is it not fd_ref'ed */
+    local->fd = fd;
+    frame->local = local;
 
-        inode_ctx_get (fd->inode, this, &ioc_inode);
-        if (ioc_inode)
-                ioc_inode_flush ((ioc_inode_t *)(long)ioc_inode);
+    inode_ctx_get(fd->inode, this, &ioc_inode);
+    if (ioc_inode) {
+        local->iobref = iobref_ref(iobref);
+        local->vector = iov_dup(vector, count);
+        local->op_ret = count;
+        local->offset = offset;
+    }
 
-        STACK_WIND (frame, ioc_writev_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->writev, fd, vector, count, offset,
-                    flags, iobref, xdata);
+    STACK_WIND(frame, ioc_writev_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->writev, fd, vector, count, offset,
+               flags, iobref, xdata);
 
-        return 0;
+    return 0;
 }
 
 /*
@@ -1281,17 +1301,15 @@ ioc_writev (call_frame_t *frame, xlator_t *this, fd_t *fd,
  *
  */
 int32_t
-ioc_truncate_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                  int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
-                  struct iatt *postbuf, dict_t *xdata)
+ioc_truncate_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                 int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
+                 struct iatt *postbuf, dict_t *xdata)
 {
-
-        STACK_UNWIND_STRICT (truncate, frame, op_ret, op_errno, prebuf,
-                             postbuf, xdata);
-        return 0;
+    STACK_UNWIND_STRICT(truncate, frame, op_ret, op_errno, prebuf, postbuf,
+                        xdata);
+    return 0;
 }
 
-
 /*
  * ioc_ftruncate_cbk -
  *
@@ -1304,17 +1322,15 @@ ioc_truncate_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
  *
  */
 int32_t
-ioc_ftruncate_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                   int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
-                   struct iatt *postbuf, dict_t *xdata)
+ioc_ftruncate_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                  int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
+                  struct iatt *postbuf, dict_t *xdata)
 {
-
-        STACK_UNWIND_STRICT (ftruncate, frame, op_ret, op_errno, prebuf,
-                             postbuf, xdata);
-        return 0;
+    STACK_UNWIND_STRICT(ftruncate, frame, op_ret, op_errno, prebuf, postbuf,
+                        xdata);
+    return 0;
 }
 
-
 /*
  * ioc_truncate -
  *
@@ -1325,19 +1341,19 @@ ioc_ftruncate_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
  *
  */
 int32_t
-ioc_truncate (call_frame_t *frame, xlator_t *this, loc_t *loc, off_t offset,
-              dict_t *xdata)
+ioc_truncate(call_frame_t *frame, xlator_t *this, loc_t *loc, off_t offset,
+             dict_t *xdata)
 {
-        uint64_t ioc_inode = 0;
+    uint64_t ioc_inode = 0;
 
-        inode_ctx_get (loc->inode, this, &ioc_inode);
+    inode_ctx_get(loc->inode, this, &ioc_inode);
 
-        if (ioc_inode)
-                ioc_inode_flush ((ioc_inode_t *)(long)ioc_inode);
+    if (ioc_inode)
+        ioc_inode_flush((ioc_inode_t *)(long)ioc_inode);
 
-        STACK_WIND (frame, ioc_truncate_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->truncate, loc, offset, xdata);
-        return 0;
+    STACK_WIND(frame, ioc_truncate_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->truncate, loc, offset, xdata);
+    return 0;
 }
 
 /*
@@ -1350,705 +1366,719 @@ ioc_truncate (call_frame_t *frame, xlator_t *this, loc_t *loc, off_t offset,
  *
  */
 int32_t
-ioc_ftruncate (call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
-               dict_t *xdata)
+ioc_ftruncate(call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
+              dict_t *xdata)
 {
-        uint64_t ioc_inode = 0;
+    uint64_t ioc_inode = 0;
 
-        inode_ctx_get (fd->inode, this, &ioc_inode);
+    inode_ctx_get(fd->inode, this, &ioc_inode);
 
-        if (ioc_inode)
-                ioc_inode_flush ((ioc_inode_t *)(long)ioc_inode);
+    if (ioc_inode)
+        ioc_inode_flush((ioc_inode_t *)(long)ioc_inode);
 
-        STACK_WIND (frame, ioc_ftruncate_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->ftruncate, fd, offset, xdata);
-        return 0;
+    STACK_WIND(frame, ioc_ftruncate_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->ftruncate, fd, offset, xdata);
+    return 0;
 }
 
 int32_t
-ioc_lk_cbk (call_frame_t *frame, void *cookie, xlator_t *this, int32_t op_ret,
-            int32_t op_errno, struct gf_flock *lock, dict_t *xdata)
+ioc_lk_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int32_t op_ret,
+           int32_t op_errno, struct gf_flock *lock, dict_t *xdata)
 {
-        STACK_UNWIND_STRICT (lk, frame, op_ret, op_errno, lock, xdata);
-        return 0;
+    STACK_UNWIND_STRICT(lk, frame, op_ret, op_errno, lock, xdata);
+    return 0;
 }
 
 int32_t
-ioc_lk (call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t cmd,
-        struct gf_flock *lock, dict_t *xdata)
+ioc_lk(call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t cmd,
+       struct gf_flock *lock, dict_t *xdata)
 {
-        ioc_inode_t *ioc_inode = NULL;
-        uint64_t     tmp_inode = 0;
-
-        inode_ctx_get (fd->inode, this, &tmp_inode);
-        ioc_inode = (ioc_inode_t *)(long)tmp_inode;
-        if (!ioc_inode) {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "inode context is NULL: returning EBADFD");
-                STACK_UNWIND_STRICT (lk, frame, -1, EBADFD, NULL, NULL);
-                return 0;
-        }
+    ioc_inode_t *ioc_inode = NULL;
+    uint64_t tmp_inode = 0;
+
+    inode_ctx_get(fd->inode, this, &tmp_inode);
+    ioc_inode = (ioc_inode_t *)(long)tmp_inode;
+    if (!ioc_inode) {
+        gf_msg_debug(this->name, EBADFD,
+                     "inode context is NULL: returning EBADFD");
+        STACK_UNWIND_STRICT(lk, frame, -1, EBADFD, NULL, NULL);
+        return 0;
+    }
 
-        ioc_inode_lock (ioc_inode);
-        {
-                gettimeofday (&ioc_inode->cache.tv, NULL);
-        }
-        ioc_inode_unlock (ioc_inode);
+    ioc_inode_lock(ioc_inode);
+    {
+        ioc_inode->cache.last_revalidate = gf_time();
+    }
+    ioc_inode_unlock(ioc_inode);
 
-        STACK_WIND (frame, ioc_lk_cbk, FIRST_CHILD (this),
-                    FIRST_CHILD (this)->fops->lk, fd, cmd, lock, xdata);
+    STACK_WIND(frame, ioc_lk_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->lk, fd, cmd, lock, xdata);
 
-        return 0;
+    return 0;
 }
 
 int
-ioc_readdirp_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                  int op_ret, int op_errno, gf_dirent_t *entries, dict_t *xdata)
+ioc_readdirp_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int op_ret,
+                 int op_errno, gf_dirent_t *entries, dict_t *xdata)
 {
-        gf_dirent_t *entry = NULL;
+    gf_dirent_t *entry = NULL;
+    char *path = NULL;
+    fd_t *fd = NULL;
 
-        if (op_ret <= 0)
-                goto unwind;
+    fd = frame->local;
+    frame->local = NULL;
 
-        list_for_each_entry (entry, &entries->list, list) {
-                /* TODO: fill things */
-        }
+    if (op_ret <= 0)
+        goto unwind;
+
+    list_for_each_entry(entry, &entries->list, list)
+    {
+        inode_path(fd->inode, entry->d_name, &path);
+        ioc_inode_update(this, entry->inode, path, &entry->d_stat);
+        GF_FREE(path);
+        path = NULL;
+    }
 
 unwind:
-        STACK_UNWIND_STRICT (readdirp, frame, op_ret, op_errno, entries, xdata);
+    STACK_UNWIND_STRICT(readdirp, frame, op_ret, op_errno, entries, xdata);
 
-        return 0;
+    return 0;
 }
+
 int
-ioc_readdirp (call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
-              off_t offset, dict_t *dict)
+ioc_readdirp(call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
+             off_t offset, dict_t *dict)
 {
-        STACK_WIND (frame, ioc_readdirp_cbk,
-                    FIRST_CHILD(this), FIRST_CHILD(this)->fops->readdirp,
-                    fd, size, offset, dict);
+    frame->local = fd;
 
-        return 0;
+    STACK_WIND(frame, ioc_readdirp_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->readdirp, fd, size, offset, dict);
+
+    return 0;
 }
 
 static int32_t
 ioc_discard_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
-		int32_t op_ret, int32_t op_errno, struct iatt *pre,
-		struct iatt *post, dict_t *xdata)
+                int32_t op_ret, int32_t op_errno, struct iatt *pre,
+                struct iatt *post, dict_t *xdata)
 {
-	STACK_UNWIND_STRICT(discard, frame, op_ret, op_errno, pre, post, xdata);
-	return 0;
+    STACK_UNWIND_STRICT(discard, frame, op_ret, op_errno, pre, post, xdata);
+    return 0;
 }
 
 static int32_t
 ioc_discard(call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
-	    size_t len, dict_t *xdata)
+            size_t len, dict_t *xdata)
 {
-	uint64_t ioc_inode = 0;
+    uint64_t ioc_inode = 0;
 
-	inode_ctx_get (fd->inode, this, &ioc_inode);
+    inode_ctx_get(fd->inode, this, &ioc_inode);
 
-	if (ioc_inode)
-		ioc_inode_flush ((ioc_inode_t *)(long)ioc_inode);
+    if (ioc_inode)
+        ioc_inode_flush((ioc_inode_t *)(long)ioc_inode);
 
-	STACK_WIND(frame, ioc_discard_cbk, FIRST_CHILD(this),
-		   FIRST_CHILD(this)->fops->discard, fd, offset, len, xdata);
-       return 0;
+    STACK_WIND(frame, ioc_discard_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->discard, fd, offset, len, xdata);
+    return 0;
 }
 
 static int32_t
 ioc_zerofill_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
-                int32_t op_ret, int32_t op_errno, struct iatt *pre,
-                struct iatt *post, dict_t *xdata)
+                 int32_t op_ret, int32_t op_errno, struct iatt *pre,
+                 struct iatt *post, dict_t *xdata)
 {
-        STACK_UNWIND_STRICT(zerofill, frame, op_ret,
-                            op_errno, pre, post, xdata);
-        return 0;
+    STACK_UNWIND_STRICT(zerofill, frame, op_ret, op_errno, pre, post, xdata);
+    return 0;
 }
 
 static int32_t
 ioc_zerofill(call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
-            off_t len, dict_t *xdata)
+             off_t len, dict_t *xdata)
 {
-        uint64_t ioc_inode = 0;
+    uint64_t ioc_inode = 0;
 
-        inode_ctx_get (fd->inode, this, &ioc_inode);
+    inode_ctx_get(fd->inode, this, &ioc_inode);
 
-        if (ioc_inode)
-                ioc_inode_flush ((ioc_inode_t *)(long)ioc_inode);
+    if (ioc_inode)
+        ioc_inode_flush((ioc_inode_t *)(long)ioc_inode);
 
-        STACK_WIND(frame, ioc_zerofill_cbk, FIRST_CHILD(this),
-                   FIRST_CHILD(this)->fops->zerofill, fd, offset, len, xdata);
-       return 0;
+    STACK_WIND(frame, ioc_zerofill_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->zerofill, fd, offset, len, xdata);
+    return 0;
 }
 
-
 int32_t
-ioc_get_priority_list (const char *opt_str, struct list_head *first)
+ioc_get_priority_list(const char *opt_str, struct list_head *first)
 {
-        int32_t              max_pri    = 1;
-        char                *tmp_str    = NULL;
-        char                *tmp_str1   = NULL;
-        char                *tmp_str2   = NULL;
-        char                *dup_str    = NULL;
-        char                *stripe_str = NULL;
-        char                *pattern    = NULL;
-        char                *priority   = NULL;
-        char                *string     = NULL;
-        struct ioc_priority *curr       = NULL, *tmp = NULL;
-
-        string = gf_strdup (opt_str);
-        if (string == NULL) {
-                max_pri = -1;
-                goto out;
-        }
-
-        /* Get the pattern for cache priority.
-         * "option priority *.jpg:1,abc*:2" etc
-         */
-        /* TODO: inode_lru in table is statically hard-coded to 5,
-         * should be changed to run-time configuration
-         */
-        stripe_str = strtok_r (string, ",", &tmp_str);
-        while (stripe_str) {
-                curr = GF_CALLOC (1, sizeof (struct ioc_priority),
-                                  gf_ioc_mt_ioc_priority);
-                if (curr == NULL) {
-                        max_pri = -1;
-                        goto out;
-                }
-
-                list_add_tail (&curr->list, first);
-
-                dup_str = gf_strdup (stripe_str);
-                if (dup_str == NULL) {
-                        max_pri = -1;
-                        goto out;
-                }
-
-                pattern = strtok_r (dup_str, ":", &tmp_str1);
-                if (!pattern) {
-                        max_pri = -1;
-                        goto out;
-                }
-
-                priority = strtok_r (NULL, ":", &tmp_str1);
-                if (!priority) {
-                        max_pri = -1;
-                        goto out;
-                }
-
-                gf_log ("io-cache", GF_LOG_TRACE,
-                        "ioc priority : pattern %s : priority %s",
-                        pattern,
-                        priority);
-
-                curr->pattern = gf_strdup (pattern);
-                if (curr->pattern == NULL) {
-                        max_pri = -1;
-                        goto out;
-                }
-
-                curr->priority = strtol (priority, &tmp_str2, 0);
-                if (tmp_str2 && (*tmp_str2)) {
-                        max_pri = -1;
-                        goto out;
-                } else {
-                        max_pri = max (max_pri, curr->priority);
-                }
-
-                GF_FREE (dup_str);
-                dup_str = NULL;
-
-                stripe_str = strtok_r (NULL, ",", &tmp_str);
-        }
+    int32_t max_pri = 1;
+    char *tmp_str = NULL;
+    char *tmp_str1 = NULL;
+    char *tmp_str2 = NULL;
+    char *dup_str = NULL;
+    char *stripe_str = NULL;
+    char *pattern = NULL;
+    char *priority = NULL;
+    char *string = NULL;
+    struct ioc_priority *curr = NULL, *tmp = NULL;
+
+    string = gf_strdup(opt_str);
+    if (string == NULL) {
+        max_pri = -1;
+        goto out;
+    }
+
+    /* Get the pattern for cache priority.
+     * "option priority *.jpg:1,abc*:2" etc
+     */
+    /* TODO: inode_lru in table is statically hard-coded to 5,
+     * should be changed to run-time configuration
+     */
+    stripe_str = strtok_r(string, ",", &tmp_str);
+    while (stripe_str) {
+        curr = GF_CALLOC(1, sizeof(struct ioc_priority),
+                         gf_ioc_mt_ioc_priority);
+        if (curr == NULL) {
+            max_pri = -1;
+            goto out;
+        }
+
+        list_add_tail(&curr->list, first);
+
+        dup_str = gf_strdup(stripe_str);
+        if (dup_str == NULL) {
+            max_pri = -1;
+            goto out;
+        }
+
+        pattern = strtok_r(dup_str, ":", &tmp_str1);
+        if (!pattern) {
+            max_pri = -1;
+            goto out;
+        }
+
+        priority = strtok_r(NULL, ":", &tmp_str1);
+        if (!priority) {
+            max_pri = -1;
+            goto out;
+        }
+
+        gf_msg_trace("io-cache", 0, "ioc priority : pattern %s : priority %s",
+                     pattern, priority);
+
+        curr->pattern = gf_strdup(pattern);
+        if (curr->pattern == NULL) {
+            max_pri = -1;
+            goto out;
+        }
+
+        curr->priority = strtol(priority, &tmp_str2, 0);
+        if (tmp_str2 && (*tmp_str2)) {
+            max_pri = -1;
+            goto out;
+        } else {
+            max_pri = max(max_pri, curr->priority);
+        }
+
+        GF_FREE(dup_str);
+        dup_str = NULL;
+
+        stripe_str = strtok_r(NULL, ",", &tmp_str);
+    }
 out:
-        GF_FREE (string);
+    GF_FREE(string);
 
-        GF_FREE (dup_str);
+    GF_FREE(dup_str);
 
-        if (max_pri == -1) {
-                list_for_each_entry_safe (curr, tmp, first, list) {
-                        list_del_init (&curr->list);
-                        GF_FREE (curr->pattern);
-                        GF_FREE (curr);
-                }
+    if (max_pri == -1) {
+        list_for_each_entry_safe(curr, tmp, first, list)
+        {
+            list_del_init(&curr->list);
+            GF_FREE(curr->pattern);
+            GF_FREE(curr);
         }
+    }
 
-        return max_pri;
+    return max_pri;
 }
 
 int32_t
-mem_acct_init (xlator_t *this)
+mem_acct_init(xlator_t *this)
 {
-        int     ret = -1;
-
-        if (!this)
-                return ret;
+    int ret = -1;
 
-        ret = xlator_mem_acct_init (this, gf_ioc_mt_end + 1);
+    if (!this)
+        return ret;
 
-        if (ret != 0) {
-                gf_log (this->name, GF_LOG_ERROR, "Memory accounting init"
-                        "failed");
-                return ret;
-        }
+    ret = xlator_mem_acct_init(this, gf_ioc_mt_end + 1);
 
+    if (ret != 0) {
+        gf_smsg(this->name, GF_LOG_ERROR, ENOMEM,
+                IO_CACHE_MSG_MEMORY_INIT_FAILED, NULL);
         return ret;
-}
+    }
 
+    return ret;
+}
 
 static gf_boolean_t
-check_cache_size_ok (xlator_t *this, uint64_t cache_size)
+check_cache_size_ok(xlator_t *this, uint64_t cache_size)
 {
-        gf_boolean_t            ret = _gf_true;
-        uint64_t                total_mem = 0;
-        uint64_t                max_cache_size = 0;
-        volume_option_t         *opt = NULL;
-
-        GF_ASSERT (this);
-        opt = xlator_volume_option_get (this, "cache-size");
-        if (!opt) {
-                ret = _gf_false;
-                gf_log (this->name, GF_LOG_ERROR,
-                        "could not get cache-size option");
-                goto out;
-        }
-
-        total_mem = get_mem_size ();
-        if (-1 == total_mem)
-                max_cache_size = opt->max;
-        else
-                max_cache_size = total_mem;
-
-        gf_log (this->name, GF_LOG_DEBUG, "Max cache size is %"PRIu64,
-                max_cache_size);
-
-        if (cache_size > max_cache_size) {
-                ret = _gf_false;
-                gf_log (this->name, GF_LOG_ERROR, "Cache size %"PRIu64
-                        " is greater than the max size of %"PRIu64,
-                        cache_size, max_cache_size);
-                goto out;
-        }
+    gf_boolean_t ret = _gf_true;
+    uint64_t total_mem = 0;
+    uint64_t max_cache_size = 0;
+    volume_option_t *opt = NULL;
+
+    GF_ASSERT(this);
+    opt = xlator_volume_option_get(this, "cache-size");
+    if (!opt) {
+        ret = _gf_false;
+        gf_smsg(this->name, GF_LOG_ERROR, EINVAL,
+                IO_CACHE_MSG_NO_CACHE_SIZE_OPT, NULL);
+        goto out;
+    }
+
+    total_mem = get_mem_size();
+    if (-1 == total_mem)
+        max_cache_size = opt->max;
+    else
+        max_cache_size = total_mem;
+
+    gf_msg_debug(this->name, 0, "Max cache size is %" PRIu64, max_cache_size);
+
+    if (cache_size > max_cache_size) {
+        ret = _gf_false;
+        gf_smsg(this->name, GF_LOG_ERROR, 0, IO_CACHE_MSG_INVALID_ARGUMENT,
+                "Cache-size=%" PRIu64, cache_size, "max-size=%" PRIu64,
+                max_cache_size, NULL);
+        goto out;
+    }
 out:
-        return ret;
+    return ret;
 }
 
 int
-reconfigure (xlator_t *this, dict_t *options)
+reconfigure(xlator_t *this, dict_t *options)
 {
-        data_t      *data              = NULL;
-        ioc_table_t *table             = NULL;
-        int          ret               = -1;
-        uint64_t      cache_size_new    = 0;
-        if (!this || !this->private)
-                goto out;
+    data_t *data = NULL;
+    ioc_table_t *table = NULL;
+    int ret = -1;
+    uint64_t cache_size_new = 0;
+    if (!this || !this->private)
+        goto out;
 
-        table = this->private;
+    table = this->private;
 
-        ioc_table_lock (table);
-        {
-                GF_OPTION_RECONF ("cache-timeout", table->cache_timeout,
-                                  options, int32, unlock);
+    ioc_table_lock(table);
+    {
+        GF_OPTION_RECONF("pass-through", this->pass_through, options, bool,
+                         unlock);
 
-                data = dict_get (options, "priority");
-                if (data) {
-                        char *option_list = data_to_str (data);
+        GF_OPTION_RECONF("cache-timeout", table->cache_timeout, options, int32,
+                         unlock);
 
-                        gf_log (this->name, GF_LOG_TRACE,
-                                "option path %s", option_list);
-                        /* parse the list of pattern:priority */
-                        table->max_pri = ioc_get_priority_list (option_list,
-                                                                &table->priority_list);
+        data = dict_get(options, "priority");
+        if (data) {
+            char *option_list = data_to_str(data);
 
-                        if (table->max_pri == -1) {
-                                goto unlock;
-                        }
-                        table->max_pri ++;
-                }
+            gf_msg_trace(this->name, 0, "option path %s", option_list);
+            /* parse the list of pattern:priority */
+            table->max_pri = ioc_get_priority_list(option_list,
+                                                   &table->priority_list);
 
-                GF_OPTION_RECONF ("max-file-size", table->max_file_size,
-                                  options, size_uint64, unlock);
+            if (table->max_pri == -1) {
+                goto unlock;
+            }
+            table->max_pri++;
+        }
 
-                GF_OPTION_RECONF ("min-file-size", table->min_file_size,
-                                  options, size_uint64, unlock);
+        GF_OPTION_RECONF("max-file-size", table->max_file_size, options,
+                         size_uint64, unlock);
 
-                if ((table->max_file_size <= UINT64_MAX) &&
-                    (table->min_file_size > table->max_file_size)) {
-                        gf_log (this->name, GF_LOG_ERROR, "minimum size (%"
-                                PRIu64") of a file that can be cached is "
-                                "greater than maximum size (%"PRIu64"). "
-                                "Hence Defaulting to old value",
-                                table->min_file_size, table->max_file_size);
-                        goto unlock;
-                }
+        GF_OPTION_RECONF("min-file-size", table->min_file_size, options,
+                         size_uint64, unlock);
 
-                GF_OPTION_RECONF ("cache-size", cache_size_new,
-                                  options, size_uint64, unlock);
-                if (!check_cache_size_ok (this, cache_size_new)) {
-                        ret = -1;
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "Not reconfiguring cache-size");
-                        goto unlock;
-                }
-                table->cache_size = cache_size_new;
+        if ((table->max_file_size <= UINT64_MAX) &&
+            (table->min_file_size > table->max_file_size)) {
+            gf_smsg(this->name, GF_LOG_ERROR, 0, IO_CACHE_MSG_DEFAULTING_TO_OLD,
+                    "minimum-size=%" PRIu64, table->min_file_size,
+                    "maximum-size=%" PRIu64, table->max_file_size, NULL);
+            goto unlock;
+        }
 
-                ret = 0;
+        GF_OPTION_RECONF("cache-size", cache_size_new, options, size_uint64,
+                         unlock);
+        if (!check_cache_size_ok(this, cache_size_new)) {
+            ret = -1;
+            gf_smsg(this->name, GF_LOG_ERROR, 0,
+                    IO_CACHE_MSG_NOT_RECONFIG_CACHE_SIZE, NULL);
+            goto unlock;
         }
+        table->cache_size = cache_size_new;
+
+        ret = 0;
+    }
 unlock:
-        ioc_table_unlock (table);
+    ioc_table_unlock(table);
 out:
-        return ret;
+    return ret;
 }
 
-
 /*
  * init -
  * @this:
  *
  */
 int32_t
-init (xlator_t *this)
+init(xlator_t *this)
 {
-        ioc_table_t     *table             = NULL;
-        dict_t          *xl_options        = NULL;
-        uint32_t         index             = 0;
-        int32_t          ret               = -1;
-        glusterfs_ctx_t *ctx               = NULL;
-        data_t          *data              = 0;
-        uint32_t         num_pages         = 0;
-
-        xl_options = this->options;
-
-        if (!this->children || this->children->next) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "FATAL: io-cache not configured with exactly "
-                        "one child");
-                goto out;
-        }
-
-        if (!this->parents) {
-                gf_log (this->name, GF_LOG_WARNING,
-                        "dangling volume. check volfile ");
-        }
-
-        table = (void *) GF_CALLOC (1, sizeof (*table), gf_ioc_mt_ioc_table_t);
-        if (table == NULL) {
-                gf_log (this->name, GF_LOG_ERROR, "out of memory");
-                goto out;
-        }
-
-        table->xl = this;
-        table->page_size = this->ctx->page_size;
-
-        GF_OPTION_INIT ("cache-size", table->cache_size, size_uint64, out);
-
-        GF_OPTION_INIT ("cache-timeout", table->cache_timeout, int32, out);
-
-        GF_OPTION_INIT ("min-file-size", table->min_file_size, size_uint64, out);
-
-        GF_OPTION_INIT ("max-file-size", table->max_file_size, size_uint64, out);
-
-        if  (!check_cache_size_ok (this, table->cache_size)) {
-                ret = -1;
-                goto out;
-        }
-
-        INIT_LIST_HEAD (&table->priority_list);
-        table->max_pri = 1;
-        data = dict_get (xl_options, "priority");
-        if (data) {
-                char *option_list = data_to_str (data);
-                gf_log (this->name, GF_LOG_TRACE,
-                        "option path %s", option_list);
-                /* parse the list of pattern:priority */
-                table->max_pri = ioc_get_priority_list (option_list,
-                                                        &table->priority_list);
-
-                if (table->max_pri == -1) {
-                        goto out;
-                }
-        }
-        table->max_pri ++;
-
-        INIT_LIST_HEAD (&table->inodes);
-
-        if ((table->max_file_size <= UINT64_MAX)
-            && (table->min_file_size > table->max_file_size)) {
-                gf_log ("io-cache", GF_LOG_ERROR, "minimum size (%"
-                        PRIu64") of a file that can be cached is "
-                        "greater than maximum size (%"PRIu64")",
-                        table->min_file_size, table->max_file_size);
-                goto out;
-        }
-
-        table->inode_lru = GF_CALLOC (table->max_pri,
-                                      sizeof (struct list_head),
-                                      gf_ioc_mt_list_head);
-        if (table->inode_lru == NULL) {
-                goto out;
-        }
-
-        for (index = 0; index < (table->max_pri); index++)
-                INIT_LIST_HEAD (&table->inode_lru[index]);
-
-        this->local_pool = mem_pool_new (ioc_local_t, 64);
-        if (!this->local_pool) {
-                ret = -1;
-                gf_log (this->name, GF_LOG_ERROR,
-                        "failed to create local_t's memory pool");
-                goto out;
-        }
-
-        pthread_mutex_init (&table->table_lock, NULL);
-        this->private = table;
-
-        num_pages = (table->cache_size / table->page_size)
-                + ((table->cache_size % table->page_size)
-                   ? 1 : 0);
-
-        table->mem_pool = mem_pool_new (rbthash_entry_t, num_pages);
-        if (!table->mem_pool) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "Unable to allocate mem_pool");
-                goto out;
-        }
-
-        ret = 0;
-
-        ctx = this->ctx;
-        ioc_log2_page_size = log_base2 (ctx->page_size);
+    ioc_table_t *table = NULL;
+    dict_t *xl_options = NULL;
+    uint32_t index = 0;
+    int32_t ret = -1;
+    glusterfs_ctx_t *ctx = NULL;
+    data_t *data = 0;
+    uint32_t num_pages = 0;
+
+    xl_options = this->options;
+
+    if (!this->children || this->children->next) {
+        gf_smsg(this->name, GF_LOG_ERROR, 0,
+                IO_CACHE_MSG_XLATOR_CHILD_MISCONFIGURED, NULL);
+        goto out;
+    }
+
+    if (!this->parents) {
+        gf_smsg(this->name, GF_LOG_WARNING, 0, IO_CACHE_MSG_VOL_MISCONFIGURED,
+                NULL);
+    }
+
+    table = (void *)GF_CALLOC(1, sizeof(*table), gf_ioc_mt_ioc_table_t);
+    if (table == NULL) {
+        gf_smsg(this->name, GF_LOG_ERROR, ENOMEM, IO_CACHE_MSG_NO_MEMORY, NULL);
+        goto out;
+    }
+
+    table->xl = this;
+    table->page_size = this->ctx->page_size;
+
+    GF_OPTION_INIT("pass-through", this->pass_through, bool, out);
+
+    GF_OPTION_INIT("cache-size", table->cache_size, size_uint64, out);
+
+    GF_OPTION_INIT("cache-timeout", table->cache_timeout, int32, out);
+
+    GF_OPTION_INIT("min-file-size", table->min_file_size, size_uint64, out);
+
+    GF_OPTION_INIT("max-file-size", table->max_file_size, size_uint64, out);
+
+    if (!check_cache_size_ok(this, table->cache_size)) {
+        ret = -1;
+        goto out;
+    }
+
+    INIT_LIST_HEAD(&table->priority_list);
+    table->max_pri = 1;
+    data = dict_get(xl_options, "priority");
+    if (data) {
+        char *option_list = data_to_str(data);
+        gf_msg_trace(this->name, 0, "option path %s", option_list);
+        /* parse the list of pattern:priority */
+        table->max_pri = ioc_get_priority_list(option_list,
+                                               &table->priority_list);
+
+        if (table->max_pri == -1) {
+            goto out;
+        }
+    }
+    table->max_pri++;
+
+    INIT_LIST_HEAD(&table->inodes);
+
+    if ((table->max_file_size <= UINT64_MAX) &&
+        (table->min_file_size > table->max_file_size)) {
+        gf_smsg("io-cache", GF_LOG_ERROR, 0, IO_CACHE_MSG_INVALID_ARGUMENT,
+                "minimum-size=%" PRIu64, table->min_file_size,
+                "maximum-size=%" PRIu64, table->max_file_size, NULL);
+        goto out;
+    }
+
+    table->inode_lru = GF_CALLOC(table->max_pri, sizeof(struct list_head),
+                                 gf_ioc_mt_list_head);
+    if (table->inode_lru == NULL) {
+        goto out;
+    }
+
+    for (index = 0; index < (table->max_pri); index++)
+        INIT_LIST_HEAD(&table->inode_lru[index]);
+
+    this->local_pool = mem_pool_new(ioc_local_t, 64);
+    if (!this->local_pool) {
+        ret = -1;
+        gf_smsg(this->name, GF_LOG_ERROR, ENOMEM,
+                IO_CACHE_MSG_CREATE_MEM_POOL_FAILED, NULL);
+        goto out;
+    }
+
+    pthread_mutex_init(&table->table_lock, NULL);
+    this->private = table;
+
+    num_pages = (table->cache_size / table->page_size) +
+                ((table->cache_size % table->page_size) ? 1 : 0);
+
+    table->mem_pool = mem_pool_new(rbthash_entry_t, num_pages);
+    if (!table->mem_pool) {
+        gf_smsg(this->name, GF_LOG_ERROR, ENOMEM,
+                IO_CACHE_MSG_ALLOC_MEM_POOL_FAILED, NULL);
+        goto out;
+    }
+
+    ret = 0;
+
+    ctx = this->ctx;
+    ioc_log2_page_size = log_base2(ctx->page_size);
 
 out:
-        if (ret == -1) {
-                if (table != NULL) {
-                        GF_FREE (table->inode_lru);
-                        GF_FREE (table);
-                }
+    if (ret == -1) {
+        if (table != NULL) {
+            GF_FREE(table->inode_lru);
+            GF_FREE(table);
         }
+    }
 
-        return ret;
+    return ret;
 }
 
 void
-ioc_page_waitq_dump (ioc_page_t *page, char *prefix)
+ioc_page_waitq_dump(ioc_page_t *page, char *prefix)
 {
-        ioc_waitq_t  *trav                     = NULL;
-        call_frame_t *frame                    = NULL;
-        int32_t       i                        = 0;
-        char          key[GF_DUMP_MAX_BUF_LEN] = {0, };
-
-        trav = page->waitq;
-
-        while (trav) {
-                frame = trav->data;
-                sprintf (key, "waitq.frame[%d]", i++);
-                gf_proc_dump_write (key, "%"PRId64, frame->root->unique);
-
-                trav = trav->next;
-        }
+    ioc_waitq_t *trav = NULL;
+    call_frame_t *frame = NULL;
+    int32_t i = 0;
+    char key[GF_DUMP_MAX_BUF_LEN] = {
+        0,
+    };
+
+    trav = page->waitq;
+
+    while (trav) {
+        frame = trav->data;
+        sprintf(key, "waitq.frame[%d]", i++);
+        gf_proc_dump_write(key, "%" PRId64, frame->root->unique);
+
+        trav = trav->next;
+    }
 }
 
 void
-__ioc_inode_waitq_dump (ioc_inode_t *ioc_inode, char *prefix)
+__ioc_inode_waitq_dump(ioc_inode_t *ioc_inode, char *prefix)
 {
-        ioc_waitq_t *trav                     = NULL;
-        ioc_page_t  *page                     = NULL;
-        int32_t      i                        = 0;
-        char         key[GF_DUMP_MAX_BUF_LEN] = {0, };
+    ioc_waitq_t *trav = NULL;
+    ioc_page_t *page = NULL;
+    int32_t i = 0;
+    char key[GF_DUMP_MAX_BUF_LEN] = {
+        0,
+    };
 
-        trav = ioc_inode->waitq;
+    trav = ioc_inode->waitq;
 
-        while (trav) {
-                page = trav->data;
+    while (trav) {
+        page = trav->data;
 
-                sprintf (key, "cache-validation-waitq.page[%d].offset", i++);
-                gf_proc_dump_write (key, "%"PRId64, page->offset);
+        sprintf(key, "cache-validation-waitq.page[%d].offset", i++);
+        gf_proc_dump_write(key, "%" PRId64, page->offset);
 
-                trav = trav->next;
-        }
+        trav = trav->next;
+    }
 }
 
 void
-__ioc_page_dump (ioc_page_t *page, char *prefix)
+__ioc_page_dump(ioc_page_t *page, char *prefix)
 {
+    int ret = -1;
 
-        int    ret = -1;
-
-        if (!page)
-                return;
-        /* ioc_page_lock can be used to hold the mutex. But in statedump
-         * its better to use trylock to avoid deadlocks.
-         */
-        ret = pthread_mutex_trylock (&page->page_lock);
-        if (ret)
-                goto out;
-        {
-                gf_proc_dump_write ("offset", "%"PRId64, page->offset);
-                gf_proc_dump_write ("size", "%"PRId64, page->size);
-                gf_proc_dump_write ("dirty", "%s", page->dirty ? "yes" : "no");
-                gf_proc_dump_write ("ready", "%s", page->ready ? "yes" : "no");
-                ioc_page_waitq_dump (page, prefix);
-        }
-        pthread_mutex_unlock (&page->page_lock);
+    if (!page)
+        return;
+    /* ioc_page_lock can be used to hold the mutex. But in statedump
+     * its better to use trylock to avoid deadlocks.
+     */
+    ret = pthread_mutex_trylock(&page->page_lock);
+    if (ret)
+        goto out;
+    {
+        gf_proc_dump_write("offset", "%" PRId64, page->offset);
+        gf_proc_dump_write("size", "%" GF_PRI_SIZET, page->size);
+        gf_proc_dump_write("dirty", "%s", page->dirty ? "yes" : "no");
+        gf_proc_dump_write("ready", "%s", page->ready ? "yes" : "no");
+        ioc_page_waitq_dump(page, prefix);
+    }
+    pthread_mutex_unlock(&page->page_lock);
 
 out:
-        if (ret && page)
-                gf_proc_dump_write ("Unable to dump the page information",
-                                    "(Lock acquisition failed) %p", page);
+    if (ret && page)
+        gf_proc_dump_write("Unable to dump the page information",
+                           "(Lock acquisition failed) %p", page);
 
-        return;
+    return;
 }
 
 void
-__ioc_cache_dump (ioc_inode_t *ioc_inode, char *prefix)
+__ioc_cache_dump(ioc_inode_t *ioc_inode, char *prefix)
 {
-        off_t        offset                   = 0;
-        ioc_table_t *table                    = NULL;
-        ioc_page_t  *page                     = NULL;
-        int          i                        = 0;
-        char         key[GF_DUMP_MAX_BUF_LEN] = {0, };
-        char         timestr[256]             = {0, };
-
-        if ((ioc_inode == NULL) || (prefix == NULL)) {
-                goto out;
-        }
-
-        table = ioc_inode->table;
-
-        if (ioc_inode->cache.tv.tv_sec) {
-                gf_time_fmt (timestr, sizeof timestr,
-                             ioc_inode->cache.tv.tv_sec, gf_timefmt_FT);
-                snprintf (timestr + strlen (timestr), sizeof timestr - strlen (timestr),
-                          ".%"GF_PRI_SUSECONDS, ioc_inode->cache.tv.tv_usec);
-
-                gf_proc_dump_write ("last-cache-validation-time", "%s",
-                                    timestr);
-        }
-
-        for (offset = 0; offset < ioc_inode->ia_size;
-             offset += table->page_size) {
-                page = __ioc_page_get (ioc_inode, offset);
-                if (page == NULL) {
-                        continue;
-                }
-
-                sprintf (key, "inode.cache.page[%d]", i++);
-                __ioc_page_dump (page, key);
-        }
+    off_t offset = 0;
+    ioc_table_t *table = NULL;
+    ioc_page_t *page = NULL;
+    int i = 0;
+    char key[GF_DUMP_MAX_BUF_LEN] = {
+        0,
+    };
+    char timestr[GF_TIMESTR_SIZE] = {
+        0,
+    };
+
+    if ((ioc_inode == NULL) || (prefix == NULL)) {
+        goto out;
+    }
+
+    table = ioc_inode->table;
+
+    if (ioc_inode->cache.last_revalidate) {
+        gf_time_fmt(timestr, sizeof timestr, ioc_inode->cache.last_revalidate,
+                    gf_timefmt_FT);
+
+        gf_proc_dump_write("last-cache-validation-time", "%s", timestr);
+    }
+
+    for (offset = 0; offset < ioc_inode->ia_size; offset += table->page_size) {
+        page = __ioc_page_get(ioc_inode, offset);
+        if (page == NULL) {
+            continue;
+        }
+
+        sprintf(key, "inode.cache.page[%d]", i++);
+        __ioc_page_dump(page, key);
+    }
 out:
-        return;
+    return;
 }
 
-
 int
-ioc_inode_dump (xlator_t *this, inode_t *inode)
+ioc_inode_dump(xlator_t *this, inode_t *inode)
 {
-
-        char         *path                            = NULL;
-        int           ret                             = -1;
-        char          key_prefix[GF_DUMP_MAX_BUF_LEN] = {0, };
-        uint64_t      tmp_ioc_inode                   = 0;
-        ioc_inode_t  *ioc_inode                       = NULL;
-        gf_boolean_t  section_added                   = _gf_false;
-        char          uuid_str[64]                    = {0,};
-
-        if (this == NULL || inode == NULL)
-                goto out;
-
-        gf_proc_dump_build_key (key_prefix, "io-cache", "inode");
-
-        inode_ctx_get (inode, this, &tmp_ioc_inode);
-        ioc_inode = (ioc_inode_t *)(long)tmp_ioc_inode;
-        if (ioc_inode == NULL)
-                goto out;
-
-        /* Similar to ioc_page_dump function its better to use
-         * pthread_mutex_trylock and not to use gf_log in statedump
-         * to avoid deadlocks.
-         */
-        ret = pthread_mutex_trylock (&ioc_inode->inode_lock);
-        if (ret)
-                goto out;
-
-        {
-                if (gf_uuid_is_null (ioc_inode->inode->gfid))
-                        goto unlock;
-
-                gf_proc_dump_add_section (key_prefix);
-                section_added = _gf_true;
-
-                __inode_path (ioc_inode->inode, NULL, &path);
-
-                gf_proc_dump_write ("inode.weight", "%d", ioc_inode->weight);
-
-                if (path) {
-                        gf_proc_dump_write ("path", "%s", path);
-                        GF_FREE (path);
-                }
-
-                gf_proc_dump_write ("uuid", "%s", uuid_utoa_r
-                                    (ioc_inode->inode->gfid, uuid_str));
-                __ioc_cache_dump (ioc_inode, key_prefix);
-                __ioc_inode_waitq_dump (ioc_inode, key_prefix);
-        }
+    char *path = NULL;
+    int ret = -1;
+    char key_prefix[GF_DUMP_MAX_BUF_LEN] = {
+        0,
+    };
+    uint64_t tmp_ioc_inode = 0;
+    ioc_inode_t *ioc_inode = NULL;
+    gf_boolean_t section_added = _gf_false;
+    char uuid_str[64] = {
+        0,
+    };
+
+    if (this == NULL || inode == NULL)
+        goto out;
+
+    gf_proc_dump_build_key(key_prefix, "io-cache", "inode");
+
+    inode_ctx_get(inode, this, &tmp_ioc_inode);
+    ioc_inode = (ioc_inode_t *)(long)tmp_ioc_inode;
+    if (ioc_inode == NULL)
+        goto out;
+
+    /* Similar to ioc_page_dump function its better to use
+     * pthread_mutex_trylock and not to use gf_log in statedump
+     * to avoid deadlocks.
+     */
+    ret = pthread_mutex_trylock(&ioc_inode->inode_lock);
+    if (ret)
+        goto out;
+
+    {
+        if (gf_uuid_is_null(ioc_inode->inode->gfid))
+            goto unlock;
+
+        gf_proc_dump_add_section("%s", key_prefix);
+        section_added = _gf_true;
+
+        __inode_path(ioc_inode->inode, NULL, &path);
+
+        gf_proc_dump_write("inode.weight", "%d", ioc_inode->weight);
+
+        if (path) {
+            gf_proc_dump_write("path", "%s", path);
+            GF_FREE(path);
+        }
+
+        gf_proc_dump_write("uuid", "%s",
+                           uuid_utoa_r(ioc_inode->inode->gfid, uuid_str));
+        __ioc_cache_dump(ioc_inode, key_prefix);
+        __ioc_inode_waitq_dump(ioc_inode, key_prefix);
+    }
 unlock:
-        pthread_mutex_unlock (&ioc_inode->inode_lock);
+    pthread_mutex_unlock(&ioc_inode->inode_lock);
 
 out:
-        if (ret && ioc_inode) {
-                if (section_added == _gf_false)
-                        gf_proc_dump_add_section (key_prefix);
-                gf_proc_dump_write ("Unable to print the status of ioc_inode",
-                                    "(Lock acquisition failed) %s",
-                                    uuid_utoa (inode->gfid));
-        }
-        return ret;
+    if (ret && ioc_inode) {
+        if (section_added == _gf_false)
+            gf_proc_dump_add_section("%s", key_prefix);
+        gf_proc_dump_write("Unable to print the status of ioc_inode",
+                           "(Lock acquisition failed) %s",
+                           uuid_utoa(inode->gfid));
+    }
+    return ret;
 }
 
 int
-ioc_priv_dump (xlator_t *this)
+ioc_priv_dump(xlator_t *this)
 {
-        ioc_table_t *priv                            = NULL;
-        char         key_prefix[GF_DUMP_MAX_BUF_LEN] = {0, };
-        int          ret                             = -1;
-        gf_boolean_t add_section                     = _gf_false;
-
-        if (!this || !this->private)
-                goto out;
-
-        priv = this->private;
-
-        gf_proc_dump_build_key (key_prefix, "io-cache", "priv");
-        gf_proc_dump_add_section (key_prefix);
-        add_section = _gf_true;
-
-        ret = pthread_mutex_trylock (&priv->table_lock);
-        if (ret)
-                goto out;
-        {
-                gf_proc_dump_write ("page_size", "%ld", priv->page_size);
-                gf_proc_dump_write ("cache_size", "%ld", priv->cache_size);
-                gf_proc_dump_write ("cache_used", "%ld", priv->cache_used);
-                gf_proc_dump_write ("inode_count", "%u", priv->inode_count);
-                gf_proc_dump_write ("cache_timeout", "%u", priv->cache_timeout);
-                gf_proc_dump_write ("min-file-size", "%u", priv->min_file_size);
-                gf_proc_dump_write ("max-file-size", "%u", priv->max_file_size);
-        }
-        pthread_mutex_unlock (&priv->table_lock);
+    ioc_table_t *priv = NULL;
+    char key_prefix[GF_DUMP_MAX_BUF_LEN] = {
+        0,
+    };
+    int ret = -1;
+    gf_boolean_t add_section = _gf_false;
+
+    if (!this || !this->private)
+        goto out;
+
+    priv = this->private;
+
+    gf_proc_dump_build_key(key_prefix, "io-cache", "priv");
+    gf_proc_dump_add_section("%s", key_prefix);
+    add_section = _gf_true;
+
+    ret = pthread_mutex_trylock(&priv->table_lock);
+    if (ret)
+        goto out;
+    {
+        gf_proc_dump_write("page_size", "%" PRIu64, priv->page_size);
+        gf_proc_dump_write("cache_size", "%" PRIu64, priv->cache_size);
+        gf_proc_dump_write("cache_used", "%" PRIu64, priv->cache_used);
+        gf_proc_dump_write("inode_count", "%u", priv->inode_count);
+        gf_proc_dump_write("cache_timeout", "%u", priv->cache_timeout);
+        gf_proc_dump_write("min-file-size", "%" PRIu64, priv->min_file_size);
+        gf_proc_dump_write("max-file-size", "%" PRIu64, priv->max_file_size);
+    }
+    pthread_mutex_unlock(&priv->table_lock);
 out:
-        if (ret && priv) {
-                if (!add_section) {
-                        gf_proc_dump_build_key (key_prefix, "xlator."
-                                                "performance.io-cache", "priv");
-                        gf_proc_dump_add_section (key_prefix);
-                }
-                gf_proc_dump_write ("Unable to dump the state of private "
-                                    "structure of io-cache xlator", "(Lock "
-                                    "acquisition failed) %s", this->name);
-        }
-
-        return 0;
+    if (ret && priv) {
+        if (!add_section) {
+            gf_proc_dump_build_key(key_prefix,
+                                   "xlator."
+                                   "performance.io-cache",
+                                   "priv");
+            gf_proc_dump_add_section("%s", key_prefix);
+        }
+        gf_proc_dump_write(
+            "Unable to dump the state of private "
+            "structure of io-cache xlator",
+            "(Lock "
+            "acquisition failed) %s",
+            this->name);
+    }
+
+    return 0;
 }
 
 /*
@@ -2058,110 +2088,144 @@ out:
  *
  */
 void
-fini (xlator_t *this)
+fini(xlator_t *this)
 {
-        ioc_table_t         *table = NULL;
-        struct ioc_priority *curr  = NULL, *tmp = NULL;
+    ioc_table_t *table = NULL;
+    struct ioc_priority *curr = NULL, *tmp = NULL;
 
-        table = this->private;
+    table = this->private;
 
-        if (table == NULL)
-                return;
-
-        this->private = NULL;
-
-        if (table->mem_pool != NULL) {
-                mem_pool_destroy (table->mem_pool);
-                table->mem_pool = NULL;
-        }
-
-        list_for_each_entry_safe (curr, tmp, &table->priority_list, list) {
-                list_del_init (&curr->list);
-                GF_FREE (curr->pattern);
-                GF_FREE (curr);
-        }
-
-        /* inode_lru and inodes list can be empty in case fini() is
-         * called soon after init()? Hence commenting the below asserts.
-         */
-        /*for (i = 0; i < table->max_pri; i++) {
-                GF_ASSERT (list_empty (&table->inode_lru[i]));
-        }
-
-        GF_ASSERT (list_empty (&table->inodes));
-        */
-        pthread_mutex_destroy (&table->table_lock);
-        GF_FREE (table);
-
-        this->private = NULL;
+    if (table == NULL)
         return;
+
+    this->private = NULL;
+
+    if (table->mem_pool != NULL) {
+        mem_pool_destroy(table->mem_pool);
+        table->mem_pool = NULL;
+    }
+
+    list_for_each_entry_safe(curr, tmp, &table->priority_list, list)
+    {
+        list_del_init(&curr->list);
+        GF_FREE(curr->pattern);
+        GF_FREE(curr);
+    }
+
+    /* inode_lru and inodes list can be empty in case fini() is
+     * called soon after init()? Hence commenting the below asserts.
+     */
+    /*for (i = 0; i < table->max_pri; i++) {
+            GF_ASSERT (list_empty (&table->inode_lru[i]));
+    }
+
+    GF_ASSERT (list_empty (&table->inodes));
+    */
+    pthread_mutex_destroy(&table->table_lock);
+    GF_FREE(table);
+
+    this->private = NULL;
+    return;
 }
 
 struct xlator_fops fops = {
-        .open        = ioc_open,
-        .create      = ioc_create,
-        .readv       = ioc_readv,
-        .writev      = ioc_writev,
-        .truncate    = ioc_truncate,
-        .ftruncate   = ioc_ftruncate,
-        .lookup      = ioc_lookup,
-        .lk          = ioc_lk,
-        .setattr     = ioc_setattr,
-        .mknod       = ioc_mknod,
-
-        .readdirp    = ioc_readdirp,
-	.discard     = ioc_discard,
-        .zerofill    = ioc_zerofill,
+    .open = ioc_open,
+    .create = ioc_create,
+    .readv = ioc_readv,
+    .writev = ioc_writev,
+    .truncate = ioc_truncate,
+    .ftruncate = ioc_ftruncate,
+    .lookup = ioc_lookup,
+    .lk = ioc_lk,
+    .setattr = ioc_setattr,
+    .mknod = ioc_mknod,
+
+    .readdirp = ioc_readdirp,
+    .discard = ioc_discard,
+    .zerofill = ioc_zerofill,
 };
 
-
 struct xlator_dumpops dumpops = {
-        .priv        = ioc_priv_dump,
-        .inodectx    = ioc_inode_dump,
+    .priv = ioc_priv_dump,
+    .inodectx = ioc_inode_dump,
 };
 
 struct xlator_cbks cbks = {
-        .forget      = ioc_forget,
-        .release     = ioc_release,
-	.invalidate  = ioc_invalidate,
+    .forget = ioc_forget,
+    .release = ioc_release,
+    .invalidate = ioc_invalidate,
 };
 
 struct volume_options options[] = {
-        { .key  = {"priority"},
-          .type = GF_OPTION_TYPE_PRIORITY_LIST,
-          .default_value = "",
-          .description = "Assigns priority to filenames with specific "
-          "patterns so that when a page needs to be ejected "
-          "out of the cache, the page of a file whose "
-          "priority is the lowest will be ejected earlier"
-        },
-        { .key  = {"cache-timeout", "force-revalidate-timeout"},
-          .type = GF_OPTION_TYPE_INT,
-          .min  = 0,
-          .max  = 60,
-          .default_value = "1",
-          .description = "The cached data for a file will be retained till "
-          "'cache-refresh-timeout' seconds, after which data "
-          "re-validation is performed."
-        },
-        { .key  = {"cache-size"},
-          .type = GF_OPTION_TYPE_SIZET,
-          .min  = 4 * GF_UNIT_MB,
-          .max  = 32 * GF_UNIT_GB,
-          .default_value = "32MB",
-          .description = "Size of the read cache."
-        },
-        { .key  = {"min-file-size"},
-          .type = GF_OPTION_TYPE_SIZET,
-          .default_value = "0",
-          .description = "Minimum file size which would be cached by the "
-          "io-cache translator."
-        },
-        { .key  = {"max-file-size"},
-          .type = GF_OPTION_TYPE_SIZET,
-          .default_value = "0",
-          .description = "Maximum file size which would be cached by the "
-          "io-cache translator."
-        },
-        { .key = {NULL} },
+    {
+        .key = {"io-cache"},
+        .type = GF_OPTION_TYPE_BOOL,
+        .default_value = "off",
+        .description = "enable/disable io-cache",
+        .op_version = {GD_OP_VERSION_6_0},
+        .flags = OPT_FLAG_SETTABLE,
+    },
+    {.key = {"priority"},
+     .type = GF_OPTION_TYPE_PRIORITY_LIST,
+     .default_value = "",
+     .description = "Assigns priority to filenames with specific "
+                    "patterns so that when a page needs to be ejected "
+                    "out of the cache, the page of a file whose "
+                    "priority is the lowest will be ejected earlier",
+     .op_version = {1},
+     .flags = OPT_FLAG_CLIENT_OPT | OPT_FLAG_SETTABLE | OPT_FLAG_DOC},
+    {.key = {"cache-timeout", "force-revalidate-timeout"},
+     .type = GF_OPTION_TYPE_INT,
+     .min = 0,
+     .max = 60,
+     .default_value = "1",
+     .description = "The cached data for a file will be retained for "
+                    "'cache-refresh-timeout' seconds, after which data "
+                    "re-validation is performed.",
+     .op_version = {1},
+     .flags = OPT_FLAG_CLIENT_OPT | OPT_FLAG_SETTABLE | OPT_FLAG_DOC},
+    {.key = {"cache-size"},
+     .type = GF_OPTION_TYPE_SIZET,
+     .min = 4 * GF_UNIT_MB,
+     .max = INFINITY,
+     .default_value = "32MB",
+     .description = "Size of the read cache.",
+     .op_version = {1},
+     .flags = OPT_FLAG_CLIENT_OPT | OPT_FLAG_SETTABLE | OPT_FLAG_DOC},
+    {.key = {"min-file-size"},
+     .type = GF_OPTION_TYPE_SIZET,
+     .default_value = "0",
+     .description = "Minimum file size which would be cached by the "
+                    "io-cache translator.",
+     .op_version = {1},
+     .flags = OPT_FLAG_CLIENT_OPT | OPT_FLAG_SETTABLE | OPT_FLAG_DOC},
+    {.key = {"max-file-size"},
+     .type = GF_OPTION_TYPE_SIZET,
+     .default_value = "0",
+     .description = "Maximum file size which would be cached by the "
+                    "io-cache translator.",
+     .op_version = {1},
+     .flags = OPT_FLAG_CLIENT_OPT | OPT_FLAG_SETTABLE | OPT_FLAG_DOC},
+    {.key = {"pass-through"},
+     .type = GF_OPTION_TYPE_BOOL,
+     .default_value = "false",
+     .op_version = {GD_OP_VERSION_4_1_0},
+     .flags = OPT_FLAG_SETTABLE | OPT_FLAG_DOC | OPT_FLAG_CLIENT_OPT,
+     .tags = {"io-cache"},
+     .description = "Enable/Disable io cache translator"},
+    {.key = {NULL}},
+};
+
+xlator_api_t xlator_api = {
+    .init = init,
+    .fini = fini,
+    .reconfigure = reconfigure,
+    .mem_acct_init = mem_acct_init,
+    .op_version = {1}, /* Present from the initial version */
+    .dumpops = &dumpops,
+    .fops = &fops,
+    .cbks = &cbks,
+    .options = options,
+    .identifier = "io-cache",
+    .category = GF_MAINTAINED,
 };
diff --git a/xlators/performance/io-cache/src/io-cache.h b/xlators/performance/io-cache/src/io-cache.h
index 46d758a6608..14923c75edc 100644
--- a/xlators/performance/io-cache/src/io-cache.h
+++ b/xlators/performance/io-cache/src/io-cache.h
@@ -11,27 +11,19 @@
 #ifndef __IO_CACHE_H
 #define __IO_CACHE_H
 
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
-
 #include <sys/types.h>
-#include "compat-errno.h"
-
-#include "glusterfs.h"
-#include "logging.h"
-#include "dict.h"
-#include "xlator.h"
-#include "common-utils.h"
-#include "call-stub.h"
-#include "rbthash.h"
-#include "hashfn.h"
+#include <glusterfs/compat-errno.h>
+
+#include <glusterfs/glusterfs.h>
+#include <glusterfs/dict.h>
+#include <glusterfs/call-stub.h>
+#include <glusterfs/rbthash.h>
 #include <sys/time.h>
 #include <fnmatch.h>
+#include "io-cache-messages.h"
 
-#define IOC_PAGE_SIZE    (1024 * 128)   /* 128KB */
-#define IOC_CACHE_SIZE   (32 * 1024 * 1024)
+#define IOC_PAGE_SIZE (1024 * 128) /* 128KB */
+#define IOC_CACHE_SIZE (32 * 1024 * 1024)
 #define IOC_PAGE_TABLE_BUCKET_COUNT 1
 
 struct ioc_table;
@@ -40,9 +32,9 @@ struct ioc_page;
 struct ioc_inode;
 
 struct ioc_priority {
-        struct list_head list;
-        char             *pattern;
-        uint32_t         priority;
+    struct list_head list;
+    char *pattern;
+    uint32_t priority;
 };
 
 /*
@@ -53,10 +45,10 @@ struct ioc_priority {
  * @data: pointer to the frame which is waiting
  */
 struct ioc_waitq {
-        struct ioc_waitq *next;
-        void             *data;
-        off_t            pending_offset;
-        size_t           pending_size;
+    struct ioc_waitq *next;
+    void *data;
+    off_t pending_offset;
+    size_t pending_size;
 };
 
 /*
@@ -64,39 +56,41 @@ struct ioc_waitq {
  *
  */
 struct ioc_fill {
-        struct list_head list;  /* list of ioc_fill structures of a frame */
-        off_t            offset;
-        size_t           size;
-        struct iovec     *vector;
-        int32_t          count;
-        struct iobref    *iobref;
+    struct list_head list; /* list of ioc_fill structures of a frame */
+    off_t offset;
+    size_t size;
+    struct iovec *vector;
+    int32_t count;
+    struct iobref *iobref;
 };
 
 struct ioc_local {
-        mode_t           mode;
-        int32_t          flags;
-        loc_t            file_loc;
-        off_t            offset;
-        size_t           size;
-        int32_t          op_ret;
-        int32_t          op_errno;
-        struct list_head fill_list;      /* list of ioc_fill structures */
-        off_t            pending_offset; /*
-                                          * offset from this frame should
-                                          * continue
-                                          */
-        size_t           pending_size;   /*
-                                          * size of data this frame is waiting
-                                          * on
-                                          */
-        struct ioc_inode *inode;
-        int32_t          wait_count;
-        pthread_mutex_t  local_lock;
-        struct ioc_waitq *waitq;
-        void             *stub;
-        fd_t             *fd;
-        int32_t          need_xattr;
-        dict_t           *xattr_req;
+    mode_t mode;
+    int32_t flags;
+    loc_t file_loc;
+    off_t offset;
+    size_t size;
+    int32_t op_ret;
+    int32_t op_errno;
+    struct list_head fill_list; /* list of ioc_fill structures */
+    off_t pending_offset;       /*
+                                 * offset from this frame should
+                                 * continue
+                                 */
+    size_t pending_size;        /*
+                                 * size of data this frame is waiting
+                                 * on
+                                 */
+    struct ioc_inode *inode;
+    int32_t wait_count;
+    pthread_mutex_t local_lock;
+    struct ioc_waitq *waitq;
+    void *stub;
+    fd_t *fd;
+    struct iovec *vector;
+    struct iobref *iobref;
+    int32_t need_xattr;
+    dict_t *xattr_req;
 };
 
 /*
@@ -104,71 +98,69 @@ struct ioc_local {
  *
  */
 struct ioc_page {
-        struct list_head    page_lru;
-        struct ioc_inode    *inode;   /* inode this page belongs to */
-        struct ioc_priority *priority;
-        char                dirty;
-        char                ready;
-        struct iovec        *vector;
-        int32_t             count;
-        off_t               offset;
-        size_t              size;
-        struct ioc_waitq    *waitq;
-        struct iobref       *iobref;
-        pthread_mutex_t     page_lock;
-        int32_t             op_errno;
-        char                stale;
+    struct list_head page_lru;
+    struct ioc_inode *inode; /* inode this page belongs to */
+    struct ioc_priority *priority;
+    char dirty;
+    char ready;
+    struct iovec *vector;
+    int32_t count;
+    off_t offset;
+    size_t size;
+    struct ioc_waitq *waitq;
+    struct iobref *iobref;
+    pthread_mutex_t page_lock;
+    int32_t op_errno;
+    char stale;
 };
 
 struct ioc_cache {
-        rbthash_table_t  *page_table;
-        struct list_head  page_lru;
-        time_t            mtime;       /*
-                                        * seconds component of file mtime
-                                        */
-        time_t            mtime_nsec;  /*
-                                        * nanosecond component of file mtime
-                                        */
-        struct timeval    tv;          /*
-                                        * time-stamp at last re-validate
-                                        */
+    rbthash_table_t *page_table;
+    struct list_head page_lru;
+    time_t mtime;           /*
+                             * seconds component of file mtime
+                             */
+    time_t mtime_nsec;      /*
+                             * nanosecond component of file mtime
+                             */
+    time_t last_revalidate; /* timestamp at last re-validate */
 };
 
 struct ioc_inode {
-        struct ioc_table      *table;
-        off_t                  ia_size;
-        struct ioc_cache       cache;
-        struct list_head       inode_list; /*
-                                            * list of inodes, maintained by
-                                            * io-cache translator
-                                            */
-        struct list_head       inode_lru;
-        struct ioc_waitq      *waitq;
-        pthread_mutex_t        inode_lock;
-        uint32_t               weight;      /*
-                                             * weight of the inode, increases
-                                             * on each read
-                                             */
-        inode_t               *inode;
+    struct ioc_table *table;
+    off_t ia_size;
+    struct ioc_cache cache;
+    struct list_head inode_list; /*
+                                  * list of inodes, maintained by
+                                  * io-cache translator
+                                  */
+    struct list_head inode_lru;
+    struct ioc_waitq *waitq;
+    pthread_mutex_t inode_lock;
+    uint32_t weight; /*
+                      * weight of the inode, increases
+                      * on each read
+                      */
+    inode_t *inode;
 };
 
 struct ioc_table {
-        uint64_t         page_size;
-        uint64_t         cache_size;
-        uint64_t         cache_used;
-        uint64_t         min_file_size;
-        uint64_t         max_file_size;
-        struct list_head inodes; /* list of inodes cached */
-        struct list_head active;
-        struct list_head *inode_lru;
-        struct list_head priority_list;
-        int32_t          readv_count;
-        pthread_mutex_t  table_lock;
-        xlator_t         *xl;
-        uint32_t         inode_count;
-        int32_t          cache_timeout;
-        int32_t          max_pri;
-        struct mem_pool  *mem_pool;
+    uint64_t page_size;
+    uint64_t cache_size;
+    uint64_t cache_used;
+    uint64_t min_file_size;
+    uint64_t max_file_size;
+    struct list_head inodes; /* list of inodes cached */
+    struct list_head active;
+    struct list_head *inode_lru;
+    struct list_head priority_list;
+    int32_t readv_count;
+    pthread_mutex_t table_lock;
+    xlator_t *xl;
+    uint32_t inode_count;
+    int32_t cache_timeout;
+    int32_t max_pri;
+    struct mem_pool *mem_pool;
 };
 
 typedef struct ioc_table ioc_table_t;
@@ -179,154 +171,136 @@ typedef struct ioc_waitq ioc_waitq_t;
 typedef struct ioc_fill ioc_fill_t;
 
 void *
-str_to_ptr (char *string);
+str_to_ptr(char *string);
 
 char *
-ptr_to_str (void *ptr);
+ptr_to_str(void *ptr);
 
 int32_t
-ioc_readv_disabled_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                        int32_t op_ret, int32_t op_errno, struct iovec *vector,
-                        int32_t count, struct iatt *stbuf,
-                        struct iobref *iobref, dict_t *xdata);
+ioc_readv_disabled_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                       int32_t op_ret, int32_t op_errno, struct iovec *vector,
+                       int32_t count, struct iatt *stbuf, struct iobref *iobref,
+                       dict_t *xdata);
 
 ioc_page_t *
-__ioc_page_get (ioc_inode_t *ioc_inode, off_t offset);
+__ioc_page_get(ioc_inode_t *ioc_inode, off_t offset);
 
 ioc_page_t *
-__ioc_page_create (ioc_inode_t *ioc_inode, off_t offset);
+__ioc_page_create(ioc_inode_t *ioc_inode, off_t offset);
 
 void
-ioc_page_fault (ioc_inode_t *ioc_inode, call_frame_t *frame, fd_t *fd,
-                off_t offset);
+ioc_page_fault(ioc_inode_t *ioc_inode, call_frame_t *frame, fd_t *fd,
+               off_t offset);
 void
-__ioc_wait_on_page (ioc_page_t *page, call_frame_t *frame, off_t offset,
-                  size_t size);
+__ioc_wait_on_page(ioc_page_t *page, call_frame_t *frame, off_t offset,
+                   size_t size);
 
 ioc_waitq_t *
-__ioc_page_wakeup (ioc_page_t *page, int32_t op_errno);
+__ioc_page_wakeup(ioc_page_t *page, int32_t op_errno);
 
 void
-ioc_page_flush (ioc_page_t *page);
+ioc_page_flush(ioc_page_t *page);
 
 ioc_waitq_t *
-__ioc_page_error (ioc_page_t *page, int32_t op_ret, int32_t op_errno);
+__ioc_page_error(ioc_page_t *page, int32_t op_ret, int32_t op_errno);
 
 void
-ioc_frame_return (call_frame_t *frame);
+ioc_frame_return(call_frame_t *frame);
 
 void
-ioc_waitq_return (ioc_waitq_t *waitq);
+ioc_waitq_return(ioc_waitq_t *waitq);
 
 int32_t
-ioc_frame_fill (ioc_page_t *page, call_frame_t *frame, off_t offset,
-                size_t size, int32_t op_errno);
-
-#define ioc_inode_lock(ioc_inode)                                       \
-        do {                                                            \
-                gf_log (ioc_inode->table->xl->name, GF_LOG_TRACE,       \
-                        "locked inode(%p)", ioc_inode);                 \
-                pthread_mutex_lock (&ioc_inode->inode_lock);            \
-        } while (0)
-
-
-#define ioc_inode_unlock(ioc_inode)                                     \
-        do {                                                            \
-                gf_log (ioc_inode->table->xl->name, GF_LOG_TRACE,       \
-                        "unlocked inode(%p)", ioc_inode);               \
-                pthread_mutex_unlock (&ioc_inode->inode_lock);          \
-        } while (0)
-
-
-#define ioc_table_lock(table)                                   \
-        do {                                                    \
-                gf_log (table->xl->name, GF_LOG_TRACE,          \
-                        "locked table(%p)", table);             \
-                pthread_mutex_lock (&table->table_lock);        \
-        } while (0)
-
-
-#define ioc_table_unlock(table)                                 \
-        do {                                                    \
-                gf_log (table->xl->name, GF_LOG_TRACE,          \
-                        "unlocked table(%p)", table);           \
-                pthread_mutex_unlock (&table->table_lock);      \
-        } while (0)
-
-
-#define ioc_local_lock(local)                                           \
-        do {                                                            \
-                gf_log (local->inode->table->xl->name, GF_LOG_TRACE,    \
-                        "locked local(%p)", local);                     \
-                pthread_mutex_lock (&local->local_lock);                \
-        } while (0)
-
-
-#define ioc_local_unlock(local)                                         \
-        do {                                                            \
-                gf_log (local->inode->table->xl->name, GF_LOG_TRACE,    \
-                        "unlocked local(%p)", local);                   \
-                pthread_mutex_unlock (&local->local_lock);              \
-        } while (0)
-
-
-#define ioc_page_lock(page)                                             \
-        do {                                                            \
-                gf_log (page->inode->table->xl->name, GF_LOG_TRACE,     \
-                        "locked page(%p)", page);                       \
-                pthread_mutex_lock (&page->page_lock);                  \
-        } while (0)
-
-
-#define ioc_page_unlock(page)                                           \
-        do {                                                            \
-                gf_log (page->inode->table->xl->name, GF_LOG_TRACE,     \
-                        "unlocked page(%p)", page);                     \
-                pthread_mutex_unlock (&page->page_lock);                \
-        } while (0)
-
-
-static inline uint64_t
-time_elapsed (struct timeval *now,
-              struct timeval *then)
-{
-        uint64_t sec = now->tv_sec - then->tv_sec;
-
-        if (sec)
-                return sec;
-
-        return 0;
-}
+ioc_frame_fill(ioc_page_t *page, call_frame_t *frame, off_t offset, size_t size,
+               int32_t op_errno);
+
+#define ioc_inode_lock(ioc_inode)                                              \
+    do {                                                                       \
+        gf_msg_trace(ioc_inode->table->xl->name, 0, "locked inode(%p)",        \
+                     ioc_inode);                                               \
+        pthread_mutex_lock(&ioc_inode->inode_lock);                            \
+    } while (0)
+
+#define ioc_inode_unlock(ioc_inode)                                            \
+    do {                                                                       \
+        gf_msg_trace(ioc_inode->table->xl->name, 0, "unlocked inode(%p)",      \
+                     ioc_inode);                                               \
+        pthread_mutex_unlock(&ioc_inode->inode_lock);                          \
+    } while (0)
+
+#define ioc_table_lock(table)                                                  \
+    do {                                                                       \
+        gf_msg_trace(table->xl->name, 0, "locked table(%p)", table);           \
+        pthread_mutex_lock(&table->table_lock);                                \
+    } while (0)
+
+#define ioc_table_unlock(table)                                                \
+    do {                                                                       \
+        gf_msg_trace(table->xl->name, 0, "unlocked table(%p)", table);         \
+        pthread_mutex_unlock(&table->table_lock);                              \
+    } while (0)
+
+#define ioc_local_lock(local)                                                  \
+    do {                                                                       \
+        gf_msg_trace(local->inode->table->xl->name, 0, "locked local(%p)",     \
+                     local);                                                   \
+        pthread_mutex_lock(&local->local_lock);                                \
+    } while (0)
+
+#define ioc_local_unlock(local)                                                \
+    do {                                                                       \
+        gf_msg_trace(local->inode->table->xl->name, 0, "unlocked local(%p)",   \
+                     local);                                                   \
+        pthread_mutex_unlock(&local->local_lock);                              \
+    } while (0)
+
+#define ioc_page_lock(page)                                                    \
+    do {                                                                       \
+        gf_msg_trace(page->inode->table->xl->name, 0, "locked page(%p)",       \
+                     page);                                                    \
+        pthread_mutex_lock(&page->page_lock);                                  \
+    } while (0)
+
+#define ioc_page_unlock(page)                                                  \
+    do {                                                                       \
+        gf_msg_trace(page->inode->table->xl->name, 0, "unlocked page(%p)",     \
+                     page);                                                    \
+        pthread_mutex_unlock(&page->page_lock);                                \
+    } while (0)
 
 ioc_inode_t *
-ioc_inode_search (ioc_table_t *table, inode_t *inode);
+ioc_inode_search(ioc_table_t *table, inode_t *inode);
 
 void
-ioc_inode_destroy (ioc_inode_t *ioc_inode);
+ioc_inode_destroy(ioc_inode_t *ioc_inode);
+
+int32_t
+ioc_inode_update(xlator_t *this, inode_t *inode, char *path,
+                 struct iatt *iabuf);
 
 ioc_inode_t *
-ioc_inode_update (ioc_table_t *table, inode_t *inode, uint32_t weight);
+ioc_inode_create(ioc_table_t *table, inode_t *inode, uint32_t weight);
 
 int64_t
-__ioc_page_destroy (ioc_page_t *page);
+__ioc_page_destroy(ioc_page_t *page);
 
 int64_t
-__ioc_inode_flush (ioc_inode_t *ioc_inode);
+__ioc_inode_flush(ioc_inode_t *ioc_inode);
 
 void
-ioc_inode_flush (ioc_inode_t *ioc_inode);
+ioc_inode_flush(ioc_inode_t *ioc_inode);
 
 void
-ioc_inode_wakeup (call_frame_t *frame, ioc_inode_t *ioc_inode,
-                  struct iatt *stbuf);
+ioc_inode_wakeup(call_frame_t *frame, ioc_inode_t *ioc_inode,
+                 struct iatt *stbuf);
 
 int8_t
-ioc_cache_still_valid (ioc_inode_t *ioc_inode, struct iatt *stbuf);
+ioc_cache_still_valid(ioc_inode_t *ioc_inode, struct iatt *stbuf);
 
 int32_t
-ioc_prune (ioc_table_t *table);
+ioc_prune(ioc_table_t *table);
 
 int32_t
-ioc_need_prune (ioc_table_t *table);
+ioc_need_prune(ioc_table_t *table);
 
 #endif /* __IO_CACHE_H */
diff --git a/xlators/performance/io-cache/src/ioc-inode.c b/xlators/performance/io-cache/src/ioc-inode.c
index 86a54bb14ca..97767d85285 100644
--- a/xlators/performance/io-cache/src/ioc-inode.c
+++ b/xlators/performance/io-cache/src/ioc-inode.c
@@ -8,11 +8,6 @@
   cases as published by the Free Software Foundation.
 */
 
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
-
 #include "io-cache.h"
 #include "ioc-mem-types.h"
 
@@ -24,145 +19,140 @@ extern int ioc_log2_page_size;
  *
  */
 void *
-str_to_ptr (char *string)
+str_to_ptr(char *string)
 {
-        void *ptr = NULL;
+    void *ptr = NULL;
 
-        GF_VALIDATE_OR_GOTO ("io-cache", string, out);
+    GF_VALIDATE_OR_GOTO("io-cache", string, out);
 
-        ptr = (void *)strtoul (string, NULL, 16);
+    ptr = (void *)strtoul(string, NULL, 16);
 
 out:
-        return ptr;
+    return ptr;
 }
 
-
 /*
  * ptr_to_str - convert a pointer to string
  * @ptr: pointer
  *
  */
 char *
-ptr_to_str (void *ptr)
+ptr_to_str(void *ptr)
 {
-        int   ret = 0;
-        char *str = NULL;
+    int ret = 0;
+    char *str = NULL;
 
-        GF_VALIDATE_OR_GOTO ("io-cache", ptr, out);
+    GF_VALIDATE_OR_GOTO("io-cache", ptr, out);
 
-        ret = gf_asprintf (&str, "%p", ptr);
-        if (-1 == ret) {
-                gf_log ("io-cache", GF_LOG_WARNING,
-                        "asprintf failed while converting ptr to str");
-                str = NULL;
-                goto out;
-        }
+    ret = gf_asprintf(&str, "%p", ptr);
+    if (-1 == ret) {
+        gf_smsg("io-cache", GF_LOG_WARNING, 0,
+                IO_CACHE_MSG_STR_COVERSION_FAILED, NULL);
+        str = NULL;
+        goto out;
+    }
 
 out:
-        return str;
+    return str;
 }
 
-
 void
-ioc_inode_wakeup (call_frame_t *frame, ioc_inode_t *ioc_inode,
-                  struct iatt *stbuf)
+ioc_inode_wakeup(call_frame_t *frame, ioc_inode_t *ioc_inode,
+                 struct iatt *stbuf)
 {
-        ioc_waitq_t *waiter            = NULL, *waited = NULL;
-        ioc_waitq_t *page_waitq        = NULL;
-        int8_t       cache_still_valid = 1;
-        ioc_local_t *local             = NULL;
-        int8_t       need_fault        = 0;
-        ioc_page_t  *waiter_page       = NULL;
-
-        GF_VALIDATE_OR_GOTO ("io-cache", frame, out);
-
-        local = frame->local;
-        GF_VALIDATE_OR_GOTO (frame->this->name, local, out);
-
-        if (ioc_inode == NULL) {
-                local->op_ret = -1;
-                local->op_errno = EINVAL;
-                gf_log (frame->this->name, GF_LOG_WARNING, "ioc_inode is NULL");
-                goto out;
-        }
-
-        ioc_inode_lock (ioc_inode);
-        {
-                waiter = ioc_inode->waitq;
-                ioc_inode->waitq = NULL;
-        }
-        ioc_inode_unlock (ioc_inode);
-
-        if (stbuf)
-                cache_still_valid = ioc_cache_still_valid (ioc_inode, stbuf);
-        else
-                cache_still_valid = 0;
-
+    ioc_waitq_t *waiter = NULL, *waited = NULL;
+    ioc_waitq_t *page_waitq = NULL;
+    int8_t cache_still_valid = 1;
+    ioc_local_t *local = NULL;
+    int8_t need_fault = 0;
+    ioc_page_t *waiter_page = NULL;
+
+    GF_VALIDATE_OR_GOTO("io-cache", frame, out);
+
+    local = frame->local;
+    GF_VALIDATE_OR_GOTO(frame->this->name, local, out);
+
+    if (ioc_inode == NULL) {
+        local->op_ret = -1;
+        local->op_errno = EINVAL;
+        gf_smsg(frame->this->name, GF_LOG_WARNING, 0, IO_CACHE_MSG_INODE_NULL,
+                NULL);
+        goto out;
+    }
+
+    if (stbuf)
+        cache_still_valid = ioc_cache_still_valid(ioc_inode, stbuf);
+    else
+        cache_still_valid = 0;
+
+    ioc_inode_lock(ioc_inode);
+    {
+        waiter = ioc_inode->waitq;
         if (!waiter) {
-                gf_log (frame->this->name, GF_LOG_WARNING,
-                        "cache validate called without any "
-                        "page waiting to be validated");
+            gf_smsg(frame->this->name, GF_LOG_WARNING, 0,
+                    IO_CACHE_MSG_PAGE_WAIT_VALIDATE, NULL);
+
+            ioc_inode_unlock(ioc_inode);
+            goto out;
         }
 
         while (waiter) {
-                waiter_page = waiter->data;
-                page_waitq = NULL;
-
-                if (waiter_page) {
-                        if (cache_still_valid) {
-                                /* cache valid, wake up page */
-                                ioc_inode_lock (ioc_inode);
-                                {
-                                        page_waitq =
-                                                __ioc_page_wakeup (waiter_page,
-                                                                   waiter_page->op_errno);
-                                }
-                                ioc_inode_unlock (ioc_inode);
-                                if (page_waitq)
-                                        ioc_waitq_return (page_waitq);
-                        } else {
-                                /* cache invalid, generate page fault and set
-                                 * page->ready = 0, to avoid double faults
-                                 */
-                                ioc_inode_lock (ioc_inode);
-                                {
-                                        if (waiter_page->ready) {
-                                                waiter_page->ready = 0;
-                                                need_fault = 1;
-                                        } else {
-                                                gf_log (frame->this->name,
-                                                        GF_LOG_TRACE,
-                                                        "validate frame(%p) is "
-                                                        "waiting for in-transit"
-                                                        " page = %p", frame,
-                                                        waiter_page);
-                                        }
-                                }
-                                ioc_inode_unlock (ioc_inode);
-
-                                if (need_fault) {
-                                        need_fault = 0;
-                                        ioc_page_fault (ioc_inode, frame,
-                                                        local->fd,
-                                                        waiter_page->offset);
-                                }
-                        }
+            waiter_page = waiter->data;
+            ioc_inode->waitq = waiter->next;
+            page_waitq = NULL;
+
+            if (waiter_page) {
+                if (cache_still_valid) {
+                    /* cache valid, wake up page */
+                    page_waitq = __ioc_page_wakeup(waiter_page,
+                                                   waiter_page->op_errno);
+                    if (page_waitq) {
+                        ioc_inode_unlock(ioc_inode);
+                        ioc_waitq_return(page_waitq);
+                        ioc_inode_lock(ioc_inode);
+                    }
+                } else {
+                    /* cache invalid, generate page fault and set
+                     * page->ready = 0, to avoid double faults
+                     */
+                    if (waiter_page->ready) {
+                        waiter_page->ready = 0;
+                        need_fault = 1;
+                    } else {
+                        gf_msg_trace(frame->this->name, 0,
+                                     "validate "
+                                     "frame(%p) is "
+                                     "waiting for "
+                                     "in-transit"
+                                     " page = %p",
+                                     frame, waiter_page);
+                    }
+
+                    if (need_fault) {
+                        need_fault = 0;
+                        ioc_inode_unlock(ioc_inode);
+                        ioc_page_fault(ioc_inode, frame, local->fd,
+                                       waiter_page->offset);
+                        ioc_inode_lock(ioc_inode);
+                    }
                 }
+            }
 
-                waited = waiter;
-                waiter = waiter->next;
+            waited = waiter;
+            waiter = ioc_inode->waitq;
 
-                waited->data = NULL;
-                GF_FREE (waited);
+            waited->data = NULL;
+            GF_FREE(waited);
         }
+    }
+    ioc_inode_unlock(ioc_inode);
 
 out:
-        return;
+    return;
 }
 
-
 /*
- * ioc_inode_update - create a new ioc_inode_t structure and add it to
+ * ioc_inode_create - create a new ioc_inode_t structure and add it to
  *                    the table table. fill in the fields which are derived
  *                    from inode_t corresponding to the file
  *
@@ -172,40 +162,37 @@ out:
  * not for external reference
  */
 ioc_inode_t *
-ioc_inode_update (ioc_table_t *table, inode_t *inode, uint32_t weight)
+ioc_inode_create(ioc_table_t *table, inode_t *inode, uint32_t weight)
 {
-        ioc_inode_t     *ioc_inode   = NULL;
+    ioc_inode_t *ioc_inode = NULL;
 
-        GF_VALIDATE_OR_GOTO ("io-cache", table, out);
+    GF_VALIDATE_OR_GOTO("io-cache", table, out);
 
-        ioc_inode = GF_CALLOC (1, sizeof (ioc_inode_t), gf_ioc_mt_ioc_inode_t);
-        if (ioc_inode == NULL) {
-                goto out;
-        }
+    ioc_inode = GF_CALLOC(1, sizeof(ioc_inode_t), gf_ioc_mt_ioc_inode_t);
+    if (ioc_inode == NULL) {
+        goto out;
+    }
 
-        ioc_inode->inode = inode;
-        ioc_inode->table = table;
-        INIT_LIST_HEAD (&ioc_inode->cache.page_lru);
-        pthread_mutex_init (&ioc_inode->inode_lock, NULL);
-        ioc_inode->weight = weight;
-
-        ioc_table_lock (table);
-        {
-                table->inode_count++;
-                list_add (&ioc_inode->inode_list, &table->inodes);
-                list_add_tail (&ioc_inode->inode_lru,
-                               &table->inode_lru[weight]);
-        }
-        ioc_table_unlock (table);
+    ioc_inode->inode = inode;
+    ioc_inode->table = table;
+    INIT_LIST_HEAD(&ioc_inode->cache.page_lru);
+    pthread_mutex_init(&ioc_inode->inode_lock, NULL);
+    ioc_inode->weight = weight;
+
+    ioc_table_lock(table);
+    {
+        table->inode_count++;
+        list_add(&ioc_inode->inode_list, &table->inodes);
+        list_add_tail(&ioc_inode->inode_lru, &table->inode_lru[weight]);
+    }
+    ioc_table_unlock(table);
 
-        gf_log (table->xl->name, GF_LOG_TRACE,
-                "adding to inode_lru[%d]", weight);
+    gf_msg_trace(table->xl->name, 0, "adding to inode_lru[%d]", weight);
 
 out:
-        return ioc_inode;
+    return ioc_inode;
 }
 
-
 /*
  * ioc_inode_destroy - destroy an ioc_inode_t object.
  *
@@ -214,27 +201,27 @@ out:
  * to be called only from ioc_forget.
  */
 void
-ioc_inode_destroy (ioc_inode_t *ioc_inode)
+ioc_inode_destroy(ioc_inode_t *ioc_inode)
 {
-        ioc_table_t *table = NULL;
+    ioc_table_t *table = NULL;
 
-        GF_VALIDATE_OR_GOTO ("io-cache", ioc_inode, out);
+    GF_VALIDATE_OR_GOTO("io-cache", ioc_inode, out);
 
-        table = ioc_inode->table;
+    table = ioc_inode->table;
 
-        ioc_table_lock (table);
-        {
-                table->inode_count--;
-                list_del (&ioc_inode->inode_list);
-                list_del (&ioc_inode->inode_lru);
-        }
-        ioc_table_unlock (table);
+    ioc_table_lock(table);
+    {
+        table->inode_count--;
+        list_del(&ioc_inode->inode_list);
+        list_del(&ioc_inode->inode_lru);
+    }
+    ioc_table_unlock(table);
 
-        ioc_inode_flush (ioc_inode);
-        rbthash_table_destroy (ioc_inode->cache.page_table);
+    ioc_inode_flush(ioc_inode);
+    rbthash_table_destroy(ioc_inode->cache.page_table);
 
-        pthread_mutex_destroy (&ioc_inode->inode_lock);
-        GF_FREE (ioc_inode);
+    pthread_mutex_destroy(&ioc_inode->inode_lock);
+    GF_FREE(ioc_inode);
 out:
-        return;
+    return;
 }
diff --git a/xlators/performance/io-cache/src/ioc-mem-types.h b/xlators/performance/io-cache/src/ioc-mem-types.h
index 9b68f9fce5f..20c9a12021e 100644
--- a/xlators/performance/io-cache/src/ioc-mem-types.h
+++ b/xlators/performance/io-cache/src/ioc-mem-types.h
@@ -11,19 +11,19 @@
 #ifndef __IOC_MT_H__
 #define __IOC_MT_H__
 
-#include "mem-types.h"
+#include <glusterfs/mem-types.h>
 
 enum gf_ioc_mem_types_ {
-        gf_ioc_mt_iovec  = gf_common_mt_end + 1,
-        gf_ioc_mt_ioc_table_t,
-        gf_ioc_mt_char,
-        gf_ioc_mt_ioc_waitq_t,
-        gf_ioc_mt_ioc_priority,
-        gf_ioc_mt_list_head,
-        gf_ioc_mt_call_pool_t,
-        gf_ioc_mt_ioc_inode_t,
-        gf_ioc_mt_ioc_fill_t,
-        gf_ioc_mt_ioc_newpage_t,
-        gf_ioc_mt_end
+    gf_ioc_mt_iovec = gf_common_mt_end + 1,
+    gf_ioc_mt_ioc_table_t,
+    gf_ioc_mt_char,
+    gf_ioc_mt_ioc_waitq_t,
+    gf_ioc_mt_ioc_priority,
+    gf_ioc_mt_list_head,
+    gf_ioc_mt_call_pool_t,
+    gf_ioc_mt_ioc_inode_t,
+    gf_ioc_mt_ioc_fill_t,
+    gf_ioc_mt_ioc_newpage_t,
+    gf_ioc_mt_end
 };
 #endif
diff --git a/xlators/performance/io-cache/src/page.c b/xlators/performance/io-cache/src/page.c
index 416cd5fe44e..84b1ae6cb20 100644
--- a/xlators/performance/io-cache/src/page.c
+++ b/xlators/performance/io-cache/src/page.c
@@ -8,81 +8,73 @@
   cases as published by the Free Software Foundation.
 */
 
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
-
-#include "glusterfs.h"
-#include "logging.h"
-#include "dict.h"
-#include "xlator.h"
+#include <glusterfs/glusterfs.h>
+#include <glusterfs/logging.h>
+#include <glusterfs/dict.h>
+#include <glusterfs/xlator.h>
 #include "io-cache.h"
 #include "ioc-mem-types.h"
 #include <assert.h>
 #include <sys/time.h>
-
+#include "io-cache-messages.h"
 char
-ioc_empty (struct ioc_cache *cache)
+ioc_empty(struct ioc_cache *cache)
 {
-        char is_empty = -1;
+    char is_empty = -1;
 
-        GF_VALIDATE_OR_GOTO ("io-cache", cache, out);
+    GF_VALIDATE_OR_GOTO("io-cache", cache, out);
 
-        is_empty = list_empty (&cache->page_lru);
+    is_empty = list_empty(&cache->page_lru);
 
 out:
-        return is_empty;
+    return is_empty;
 }
 
-
 ioc_page_t *
-__ioc_page_get (ioc_inode_t *ioc_inode, off_t offset)
+__ioc_page_get(ioc_inode_t *ioc_inode, off_t offset)
 {
-        ioc_page_t   *page           = NULL;
-        ioc_table_t  *table          = NULL;
-        off_t         rounded_offset = 0;
+    ioc_page_t *page = NULL;
+    ioc_table_t *table = NULL;
+    off_t rounded_offset = 0;
 
-        GF_VALIDATE_OR_GOTO ("io-cache", ioc_inode, out);
+    GF_VALIDATE_OR_GOTO("io-cache", ioc_inode, out);
 
-        table = ioc_inode->table;
-        GF_VALIDATE_OR_GOTO ("io-cache", ioc_inode, out);
+    table = ioc_inode->table;
+    GF_VALIDATE_OR_GOTO("io-cache", ioc_inode, out);
 
-        rounded_offset = floor (offset, table->page_size);
+    rounded_offset = gf_floor(offset, table->page_size);
 
-        page = rbthash_get (ioc_inode->cache.page_table, &rounded_offset,
-                            sizeof (rounded_offset));
+    page = rbthash_get(ioc_inode->cache.page_table, &rounded_offset,
+                       sizeof(rounded_offset));
 
-        if (page != NULL) {
-                /* push the page to the end of the lru list */
-                list_move_tail (&page->page_lru, &ioc_inode->cache.page_lru);
-        }
+    if (page != NULL) {
+        /* push the page to the end of the lru list */
+        list_move_tail(&page->page_lru, &ioc_inode->cache.page_lru);
+    }
 
 out:
-        return page;
+    return page;
 }
 
-
 ioc_page_t *
-ioc_page_get (ioc_inode_t *ioc_inode, off_t offset)
+ioc_page_get(ioc_inode_t *ioc_inode, off_t offset)
 {
-        ioc_page_t *page = NULL;
+    ioc_page_t *page = NULL;
 
-        if (ioc_inode == NULL) {
-                goto out;
-        }
+    if (ioc_inode == NULL) {
+        goto out;
+    }
 
-        ioc_inode_lock (ioc_inode);
-        {
-                page = __ioc_page_get (ioc_inode, offset);
-        }
-        ioc_inode_unlock (ioc_inode);
+    ioc_inode_lock(ioc_inode);
+    {
+        page = __ioc_page_get(ioc_inode, offset);
+    }
+    ioc_inode_unlock(ioc_inode);
 
 out:
-        return page;
+    return page;
 }
 
-
 /*
  * __ioc_page_destroy -
  *
@@ -90,105 +82,108 @@ out:
  *
  */
 int64_t
-__ioc_page_destroy (ioc_page_t *page)
+__ioc_page_destroy(ioc_page_t *page)
 {
-        int64_t  page_size = 0;
-
-        GF_VALIDATE_OR_GOTO ("io-cache", page, out);
-
-        if (page->iobref)
-                page_size = iobref_size (page->iobref);
-
-        if (page->waitq) {
-                /* frames waiting on this page, do not destroy this page */
-                page_size = -1;
-                page->stale = 1;
-        } else {
-                rbthash_remove (page->inode->cache.page_table, &page->offset,
-                                sizeof (page->offset));
-                list_del (&page->page_lru);
-
-                gf_log (page->inode->table->xl->name, GF_LOG_TRACE,
-                        "destroying page = %p, offset = %"PRId64" "
-                        "&& inode = %p",
-                        page, page->offset, page->inode);
-
-                if (page->vector){
-                        iobref_unref (page->iobref);
-                        GF_FREE (page->vector);
-                        page->vector = NULL;
-                }
-
-                page->inode = NULL;
+    int64_t page_size = 0;
+
+    GF_VALIDATE_OR_GOTO("io-cache", page, out);
+
+    if (page->iobref)
+        page_size = iobref_size(page->iobref);
+
+    if (page->waitq) {
+        /* frames waiting on this page, do not destroy this page */
+        page_size = -1;
+        page->stale = 1;
+    } else {
+        rbthash_remove(page->inode->cache.page_table, &page->offset,
+                       sizeof(page->offset));
+        list_del(&page->page_lru);
+
+        gf_msg_trace(page->inode->table->xl->name, 0,
+                     "destroying page = %p, offset = %" PRId64
+                     " "
+                     "&& inode = %p",
+                     page, page->offset, page->inode);
+
+        if (page->vector) {
+            iobref_unref(page->iobref);
+            GF_FREE(page->vector);
+            page->vector = NULL;
         }
 
-        if (page_size != -1) {
-                pthread_mutex_destroy (&page->page_lock);
-                GF_FREE (page);
-        }
+        page->inode = NULL;
+    }
+
+    if (page_size != -1) {
+        pthread_mutex_destroy(&page->page_lock);
+        GF_FREE(page);
+    }
 
 out:
-        return page_size;
+    return page_size;
 }
 
-
 int64_t
-ioc_page_destroy (ioc_page_t *page)
+ioc_page_destroy(ioc_page_t *page)
 {
-        int64_t ret = 0;
-        struct ioc_inode *inode = NULL;
+    int64_t ret = 0;
+    struct ioc_inode *inode = NULL;
 
-        if (page == NULL) {
-                goto out;
-        }
+    if (page == NULL) {
+        goto out;
+    }
 
-        ioc_inode_lock (page->inode);
-        {
-                inode = page->inode;
-                ret = __ioc_page_destroy (page);
-        }
-        ioc_inode_unlock (inode);
+    ioc_inode_lock(page->inode);
+    {
+        inode = page->inode;
+        ret = __ioc_page_destroy(page);
+    }
+    ioc_inode_unlock(inode);
 
 out:
-        return ret;
+    return ret;
 }
 
 int32_t
-__ioc_inode_prune (ioc_inode_t *curr, uint64_t *size_pruned,
-                   uint64_t size_to_prune, uint32_t index)
+__ioc_inode_prune(ioc_inode_t *curr, uint64_t *size_pruned,
+                  uint64_t size_to_prune, uint32_t index)
 {
-        ioc_page_t  *page  = NULL, *next = NULL;
-        int32_t      ret   = 0;
-        ioc_table_t *table = NULL;
+    ioc_page_t *page = NULL, *next = NULL;
+    int32_t ret = 0;
+    ioc_table_t *table = NULL;
 
-        if (curr == NULL) {
-                goto out;
-        }
+    if (curr == NULL) {
+        goto out;
+    }
 
-        table = curr->table;
+    table = curr->table;
 
-        list_for_each_entry_safe (page, next, &curr->cache.page_lru, page_lru) {
-                *size_pruned += page->size;
-                ret = __ioc_page_destroy (page);
+    list_for_each_entry_safe(page, next, &curr->cache.page_lru, page_lru)
+    {
+        *size_pruned += page->size;
+        ret = __ioc_page_destroy(page);
 
-                if (ret != -1)
-                        table->cache_used -= ret;
+        if (ret != -1)
+            table->cache_used -= ret;
 
-                gf_log (table->xl->name, GF_LOG_TRACE,
-                        "index = %d && table->cache_used = %"PRIu64" && table->"
-                        "cache_size = %"PRIu64, index, table->cache_used,
-                        table->cache_size);
+        gf_msg_trace(table->xl->name, 0,
+                     "index = %d && "
+                     "table->cache_used = %" PRIu64
+                     " && table->"
+                     "cache_size = %" PRIu64,
+                     index, table->cache_used, table->cache_size);
 
-                if ((*size_pruned) >= size_to_prune)
-                        break;
-        }
+        if ((*size_pruned) >= size_to_prune)
+            break;
+    }
 
-        if (ioc_empty (&curr->cache)) {
-                list_del_init (&curr->inode_lru);
-        }
+    if (ioc_empty(&curr->cache)) {
+        list_del_init(&curr->inode_lru);
+    }
 
 out:
-        return 0;
+    return 0;
 }
 /*
  * ioc_prune - prune the cache. we have a limit to the number of pages we
@@ -198,46 +193,44 @@ out:
  *
  */
 int32_t
-ioc_prune (ioc_table_t *table)
+ioc_prune(ioc_table_t *table)
 {
-        ioc_inode_t *curr          = NULL, *next_ioc_inode = NULL;
-        int32_t      index         = 0;
-        uint64_t     size_to_prune = 0;
-        uint64_t     size_pruned   = 0;
+    ioc_inode_t *curr = NULL, *next_ioc_inode = NULL;
+    int32_t index = 0;
+    uint64_t size_to_prune = 0;
+    uint64_t size_pruned = 0;
+
+    GF_VALIDATE_OR_GOTO("io-cache", table, out);
+
+    ioc_table_lock(table);
+    {
+        size_to_prune = table->cache_used - table->cache_size;
+        /* take out the least recently used inode */
+        for (index = 0; index < table->max_pri; index++) {
+            list_for_each_entry_safe(curr, next_ioc_inode,
+                                     &table->inode_lru[index], inode_lru)
+            {
+                /* prune page-by-page for this inode, till
+                 * we reach the equilibrium */
+                ioc_inode_lock(curr);
+                {
+                    __ioc_inode_prune(curr, &size_pruned, size_to_prune, index);
+                }
+                ioc_inode_unlock(curr);
 
-        GF_VALIDATE_OR_GOTO ("io-cache", table, out);
+                if (size_pruned >= size_to_prune)
+                    break;
+            } /* list_for_each_entry_safe (curr...) */
 
-        ioc_table_lock (table);
-        {
-                size_to_prune = table->cache_used - table->cache_size;
-                /* take out the least recently used inode */
-                for (index=0; index < table->max_pri; index++) {
-                        list_for_each_entry_safe (curr, next_ioc_inode,
-                                                  &table->inode_lru[index],
-                                                  inode_lru) {
-                                /* prune page-by-page for this inode, till
-                                 * we reach the equilibrium */
-                                ioc_inode_lock (curr);
-                                {
-                                        __ioc_inode_prune (curr, &size_pruned,
-                                                           size_to_prune,
-                                                           index);
-                                }
-                                ioc_inode_unlock (curr);
-
-                                if (size_pruned >= size_to_prune)
-                                        break;
-                        } /* list_for_each_entry_safe (curr...) */
-
-                        if (size_pruned >= size_to_prune)
-                                break;
-                } /* for(index=0;...) */
-
-        } /* ioc_inode_table locked region end */
-        ioc_table_unlock (table);
+            if (size_pruned >= size_to_prune)
+                break;
+        } /* for(index=0;...) */
+
+    } /* ioc_inode_table locked region end */
+    ioc_table_unlock(table);
 
 out:
-        return 0;
+    return 0;
 }
 
 /*
@@ -248,47 +241,46 @@ out:
  *
  */
 ioc_page_t *
-__ioc_page_create (ioc_inode_t *ioc_inode, off_t offset)
+__ioc_page_create(ioc_inode_t *ioc_inode, off_t offset)
 {
-        ioc_table_t *table          = NULL;
-        ioc_page_t  *page           = NULL;
-        off_t        rounded_offset = 0;
-        ioc_page_t  *newpage        = NULL;
+    ioc_table_t *table = NULL;
+    ioc_page_t *page = NULL;
+    off_t rounded_offset = 0;
+    ioc_page_t *newpage = NULL;
 
-        GF_VALIDATE_OR_GOTO ("io-cache", ioc_inode, out);
+    GF_VALIDATE_OR_GOTO("io-cache", ioc_inode, out);
 
-        table = ioc_inode->table;
-        GF_VALIDATE_OR_GOTO ("io-cache", table, out);
+    table = ioc_inode->table;
+    GF_VALIDATE_OR_GOTO("io-cache", table, out);
 
-        rounded_offset = floor (offset, table->page_size);
+    rounded_offset = gf_floor(offset, table->page_size);
 
-        newpage = GF_CALLOC (1, sizeof (*newpage), gf_ioc_mt_ioc_newpage_t);
-        if (newpage == NULL) {
-                goto out;
-        }
+    newpage = GF_CALLOC(1, sizeof(*newpage), gf_ioc_mt_ioc_newpage_t);
+    if (newpage == NULL) {
+        goto out;
+    }
 
-        if (!ioc_inode) {
-                GF_FREE (newpage);
-                newpage = NULL;
-                goto out;
-        }
+    if (!ioc_inode) {
+        GF_FREE(newpage);
+        newpage = NULL;
+        goto out;
+    }
 
-        newpage->offset = rounded_offset;
-        newpage->inode = ioc_inode;
-        pthread_mutex_init (&newpage->page_lock, NULL);
+    newpage->offset = rounded_offset;
+    newpage->inode = ioc_inode;
+    pthread_mutex_init(&newpage->page_lock, NULL);
 
-        rbthash_insert (ioc_inode->cache.page_table, newpage, &rounded_offset,
-                        sizeof (rounded_offset));
+    rbthash_insert(ioc_inode->cache.page_table, newpage, &rounded_offset,
+                   sizeof(rounded_offset));
 
-        list_add_tail (&newpage->page_lru, &ioc_inode->cache.page_lru);
+    list_add_tail(&newpage->page_lru, &ioc_inode->cache.page_lru);
 
-        page = newpage;
+    page = newpage;
 
-        gf_log ("io-cache", GF_LOG_TRACE,
-                "returning new page %p", page);
+    gf_msg_trace("io-cache", 0, "returning new page %p", page);
 
 out:
-        return page;
+    return page;
 }
 
 /*
@@ -301,55 +293,55 @@ out:
  *
  */
 void
-__ioc_wait_on_page (ioc_page_t *page, call_frame_t *frame, off_t offset,
-                    size_t size)
+__ioc_wait_on_page(ioc_page_t *page, call_frame_t *frame, off_t offset,
+                   size_t size)
 {
-        ioc_waitq_t *waitq = NULL;
-        ioc_local_t *local = NULL;
-
-        GF_VALIDATE_OR_GOTO ("io-cache", frame, out);
-        local = frame->local;
-
-        GF_VALIDATE_OR_GOTO (frame->this->name, local, out);
-
-        if (page == NULL) {
-                local->op_ret = -1;
-                local->op_errno = ENOMEM;
-                gf_log (frame->this->name, GF_LOG_WARNING,
-                        "asked to wait on a NULL page");
-                goto out;
-        }
-
-        waitq = GF_CALLOC (1, sizeof (*waitq), gf_ioc_mt_ioc_waitq_t);
-        if (waitq == NULL) {
-                local->op_ret = -1;
-                local->op_errno = ENOMEM;
-                goto out;
-        }
-
-        gf_log (frame->this->name, GF_LOG_TRACE,
-                "frame(%p) waiting on page = %p, offset=%"PRId64", "
-                "size=%"GF_PRI_SIZET"",
-                frame, page, offset, size);
-
-        waitq->data = frame;
-        waitq->next = page->waitq;
-        waitq->pending_offset = offset;
-        waitq->pending_size = size;
-        page->waitq = waitq;
-        /* one frame can wait only once on a given page,
-         * local->wait_count is number of pages a frame is waiting on */
-        ioc_local_lock (local);
-        {
-                local->wait_count++;
-        }
-        ioc_local_unlock (local);
+    ioc_waitq_t *waitq = NULL;
+    ioc_local_t *local = NULL;
+
+    GF_VALIDATE_OR_GOTO("io-cache", frame, out);
+    local = frame->local;
+
+    GF_VALIDATE_OR_GOTO(frame->this->name, local, out);
+
+    if (page == NULL) {
+        local->op_ret = -1;
+        local->op_errno = ENOMEM;
+        gf_smsg(frame->this->name, GF_LOG_WARNING, 0,
+                IO_CACHE_MSG_NULL_PAGE_WAIT, NULL);
+        goto out;
+    }
+
+    waitq = GF_CALLOC(1, sizeof(*waitq), gf_ioc_mt_ioc_waitq_t);
+    if (waitq == NULL) {
+        local->op_ret = -1;
+        local->op_errno = ENOMEM;
+        goto out;
+    }
+
+    gf_msg_trace(frame->this->name, 0,
+                 "frame(%p) waiting on page = %p, offset=%" PRId64
+                 ", "
+                 "size=%" GF_PRI_SIZET "",
+                 frame, page, offset, size);
+
+    waitq->data = frame;
+    waitq->next = page->waitq;
+    waitq->pending_offset = offset;
+    waitq->pending_size = size;
+    page->waitq = waitq;
+    /* one frame can wait only once on a given page,
+     * local->wait_count is number of pages a frame is waiting on */
+    ioc_local_lock(local);
+    {
+        local->wait_count++;
+    }
+    ioc_local_unlock(local);
 
 out:
-        return;
+    return;
 }
 
-
 /*
  * ioc_cache_still_valid - see if cached pages ioc_inode are still valid
  * against given stbuf
@@ -360,11 +352,11 @@ out:
  * assumes ioc_inode is locked
  */
 int8_t
-ioc_cache_still_valid (ioc_inode_t *ioc_inode, struct iatt *stbuf)
+ioc_cache_still_valid(ioc_inode_t *ioc_inode, struct iatt *stbuf)
 {
-        int8_t cache_still_valid = 1;
+    int8_t cache_still_valid = 1;
 
-        GF_VALIDATE_OR_GOTO ("io-cache", ioc_inode, out);
+    GF_VALIDATE_OR_GOTO("io-cache", ioc_inode, out);
 
 #if 0
         if (!stbuf || (stbuf->ia_mtime != ioc_inode->cache.mtime) ||
@@ -372,9 +364,9 @@ ioc_cache_still_valid (ioc_inode_t *ioc_inode, struct iatt *stbuf)
                 cache_still_valid = 0;
 
 #else
-        if (!stbuf || (stbuf->ia_mtime != ioc_inode->cache.mtime)
-            || (stbuf->ia_mtime_nsec != ioc_inode->cache.mtime_nsec))
-                cache_still_valid = 0;
+    if (!stbuf || (stbuf->ia_mtime != ioc_inode->cache.mtime) ||
+        (stbuf->ia_mtime_nsec != ioc_inode->cache.mtime_nsec))
+        cache_still_valid = 0;
 
 #endif
 
@@ -387,183 +379,173 @@ ioc_cache_still_valid (ioc_inode_t *ioc_inode, struct iatt *stbuf)
 #endif
 
 out:
-        return cache_still_valid;
+    return cache_still_valid;
 }
 
-
 void
-ioc_waitq_return (ioc_waitq_t *waitq)
+ioc_waitq_return(ioc_waitq_t *waitq)
 {
-        ioc_waitq_t  *trav   = NULL;
-        ioc_waitq_t  *next   = NULL;
-        call_frame_t *frame  = NULL;
+    ioc_waitq_t *trav = NULL;
+    ioc_waitq_t *next = NULL;
+    call_frame_t *frame = NULL;
 
-        for (trav = waitq; trav; trav = next) {
-                next = trav->next;
+    for (trav = waitq; trav; trav = next) {
+        next = trav->next;
 
-                frame = trav->data;
-                ioc_frame_return (frame);
-                GF_FREE (trav);
-        }
+        frame = trav->data;
+        ioc_frame_return(frame);
+        GF_FREE(trav);
+    }
 }
 
-
 int
-ioc_fault_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-               int32_t op_ret, int32_t op_errno, struct iovec *vector,
-               int32_t count, struct iatt *stbuf, struct iobref *iobref,
-               dict_t *xdata)
+ioc_fault_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int32_t op_ret,
+              int32_t op_errno, struct iovec *vector, int32_t count,
+              struct iatt *stbuf, struct iobref *iobref, dict_t *xdata)
 {
-        ioc_local_t *local            = NULL;
-        off_t        offset           = 0;
-        ioc_inode_t *ioc_inode        = NULL;
-        ioc_table_t *table            = NULL;
-        ioc_page_t  *page             = NULL;
-        int32_t      destroy_size     = 0;
-        size_t       page_size        = 0;
-        ioc_waitq_t *waitq            = NULL;
-        size_t       iobref_page_size = 0;
-        char         zero_filled      = 0;
-
-        GF_ASSERT (frame);
-
-        local = frame->local;
-        GF_ASSERT (local);
-
-        offset = local->pending_offset;
-        ioc_inode = local->inode;
-        GF_ASSERT (ioc_inode);
+    ioc_local_t *local = NULL;
+    off_t offset = 0;
+    ioc_inode_t *ioc_inode = NULL;
+    ioc_table_t *table = NULL;
+    ioc_page_t *page = NULL;
+    int32_t destroy_size = 0;
+    size_t page_size = 0;
+    ioc_waitq_t *waitq = NULL;
+    size_t iobref_page_size = 0;
+    char zero_filled = 0;
+
+    GF_ASSERT(frame);
+
+    local = frame->local;
+    GF_ASSERT(local);
+
+    offset = local->pending_offset;
+    ioc_inode = local->inode;
+    GF_ASSERT(ioc_inode);
+
+    table = ioc_inode->table;
+    GF_ASSERT(table);
+
+    zero_filled = ((op_ret >= 0) && (stbuf->ia_mtime == 0));
+
+    ioc_inode_lock(ioc_inode);
+    {
+        if (op_ret == -1 ||
+            !(zero_filled || ioc_cache_still_valid(ioc_inode, stbuf))) {
+            gf_msg_trace(ioc_inode->table->xl->name, 0,
+                         "cache for inode(%p) is invalid. flushing "
+                         "all pages",
+                         ioc_inode);
+            destroy_size = __ioc_inode_flush(ioc_inode);
+        }
 
-        table = ioc_inode->table;
-        GF_ASSERT (table);
+        if ((op_ret >= 0) && !zero_filled) {
+            ioc_inode->cache.mtime = stbuf->ia_mtime;
+            ioc_inode->cache.mtime_nsec = stbuf->ia_mtime_nsec;
+        }
 
-        zero_filled = ((op_ret >=0) && (stbuf->ia_mtime == 0));
+        ioc_inode->cache.last_revalidate = gf_time();
 
-        ioc_inode_lock (ioc_inode);
-        {
-                if (op_ret == -1 || !(zero_filled ||
-                                      ioc_cache_still_valid(ioc_inode,
-                                                            stbuf))) {
-                        gf_log (ioc_inode->table->xl->name, GF_LOG_TRACE,
-                                "cache for inode(%p) is invalid. flushing "
-                                "all pages", ioc_inode);
-                        destroy_size = __ioc_inode_flush (ioc_inode);
+        if (op_ret < 0) {
+            /* error, readv returned -1 */
+            page = __ioc_page_get(ioc_inode, offset);
+            if (page)
+                waitq = __ioc_page_error(page, op_ret, op_errno);
+        } else {
+            gf_msg_trace(ioc_inode->table->xl->name, 0, "op_ret = %d", op_ret);
+            page = __ioc_page_get(ioc_inode, offset);
+            if (!page) {
+                /* page was flushed */
+                /* some serious bug ? */
+                gf_smsg(frame->this->name, GF_LOG_WARNING, 0,
+                        IO_CACHE_MSG_WASTED_COPY, "offset=%" PRId64, offset,
+                        "page-size=%" PRId64, table->page_size, "ioc_inode=%p",
+                        ioc_inode, NULL);
+            } else {
+                if (page->vector) {
+                    iobref_unref(page->iobref);
+                    GF_FREE(page->vector);
+                    page->vector = NULL;
+                    page->iobref = NULL;
                 }
 
-                if ((op_ret >= 0) && !zero_filled) {
-                        ioc_inode->cache.mtime = stbuf->ia_mtime;
-                        ioc_inode->cache.mtime_nsec = stbuf->ia_mtime_nsec;
+                /* keep a copy of the page for our cache */
+                page->vector = iov_dup(vector, count);
+                if (page->vector == NULL) {
+                    page = __ioc_page_get(ioc_inode, offset);
+                    if (page != NULL)
+                        waitq = __ioc_page_error(page, -1, ENOMEM);
+                    goto unlock;
                 }
 
-                gettimeofday (&ioc_inode->cache.tv, NULL);
-
-                if (op_ret < 0) {
-                        /* error, readv returned -1 */
-                        page = __ioc_page_get (ioc_inode, offset);
-                        if (page)
-                                waitq = __ioc_page_error (page, op_ret,
-                                                          op_errno);
+                page->count = count;
+                if (iobref) {
+                    page->iobref = iobref_ref(iobref);
                 } else {
-                        gf_log (ioc_inode->table->xl->name, GF_LOG_TRACE,
-                                "op_ret = %d", op_ret);
-                        page = __ioc_page_get (ioc_inode, offset);
-                        if (!page) {
-                                /* page was flushed */
-                                /* some serious bug ? */
-                                gf_log (frame->this->name, GF_LOG_WARNING,
-                                        "wasted copy: %"PRId64"[+%"PRId64"] "
-                                        "ioc_inode=%p", offset,
-                                        table->page_size, ioc_inode);
-                        } else {
-                                if (page->vector) {
-                                        iobref_unref (page->iobref);
-                                        GF_FREE (page->vector);
-                                        page->vector = NULL;
-                                        page->iobref = NULL;
-                                }
-
-                                /* keep a copy of the page for our cache */
-                                page->vector = iov_dup (vector, count);
-                                if (page->vector == NULL) {
-                                        page = __ioc_page_get (ioc_inode,
-                                                               offset);
-                                        if (page != NULL)
-                                                waitq = __ioc_page_error (page,
-                                                                          -1,
-                                                                          ENOMEM);
-                                        goto unlock;
-                                }
-
-                                page->count = count;
-                                if (iobref) {
-                                        page->iobref = iobref_ref (iobref);
-                                } else {
-                                        /* TODO: we have got a response to
-                                         * our request and no data */
-                                        gf_log (frame->this->name,
-                                                GF_LOG_CRITICAL,
-                                                "frame>root>rsp_refs is null");
-                                } /* if(frame->root->rsp_refs) */
-
-                                /* page->size should indicate exactly how
-                                 * much the readv call to the child
-                                 * translator returned. earlier op_ret
-                                 * from child translator was used, which
-                                 * gave rise to a bug where reads from
-                                 * io-cached volume were resulting in 0
-                                 * byte replies */
-                                page_size = iov_length(vector, count);
-                                page->size = page_size;
-                                page->op_errno = op_errno;
-
-                                iobref_page_size = iobref_size (page->iobref);
-
-                                if (page->waitq) {
-                                        /* wake up all the frames waiting on
-                                         * this page, including
-                                         * the frame which triggered fault */
-                                        waitq = __ioc_page_wakeup (page,
-                                                                   op_errno);
-                                } /* if(page->waitq) */
-                        } /* if(!page)...else */
-                } /* if(op_ret < 0)...else */
-        } /* ioc_inode locked region end */
+                    /* TODO: we have got a response to
+                     * our request and no data */
+                    gf_smsg(frame->this->name, GF_LOG_CRITICAL, ENOMEM,
+                            IO_CACHE_MSG_FRAME_NULL, NULL);
+                } /* if(frame->root->rsp_refs) */
+
+                /* page->size should indicate exactly how
+                 * much the readv call to the child
+                 * translator returned. earlier op_ret
+                 * from child translator was used, which
+                 * gave rise to a bug where reads from
+                 * io-cached volume were resulting in 0
+                 * byte replies */
+                page_size = iov_length(vector, count);
+                page->size = page_size;
+                page->op_errno = op_errno;
+
+                iobref_page_size = iobref_size(page->iobref);
+
+                if (page->waitq) {
+                    /* wake up all the frames waiting on
+                     * this page, including
+                     * the frame which triggered fault */
+                    waitq = __ioc_page_wakeup(page, op_errno);
+                } /* if(page->waitq) */
+            }     /* if(!page)...else */
+        }         /* if(op_ret < 0)...else */
+    }             /* ioc_inode locked region end */
 unlock:
-        ioc_inode_unlock (ioc_inode);
+    ioc_inode_unlock(ioc_inode);
 
-        ioc_waitq_return (waitq);
+    ioc_waitq_return(waitq);
 
-        if (iobref_page_size) {
-                ioc_table_lock (table);
-                {
-                        table->cache_used += iobref_page_size;
-                }
-                ioc_table_unlock (table);
+    if (iobref_page_size) {
+        ioc_table_lock(table);
+        {
+            table->cache_used += iobref_page_size;
         }
+        ioc_table_unlock(table);
+    }
 
-        if (destroy_size) {
-                ioc_table_lock (table);
-                {
-                        table->cache_used -= destroy_size;
-                }
-                ioc_table_unlock (table);
+    if (destroy_size) {
+        ioc_table_lock(table);
+        {
+            table->cache_used -= destroy_size;
         }
+        ioc_table_unlock(table);
+    }
 
-        if (ioc_need_prune (ioc_inode->table)) {
-                ioc_prune (ioc_inode->table);
-        }
+    if (ioc_need_prune(ioc_inode->table)) {
+        ioc_prune(ioc_inode->table);
+    }
 
-        gf_log (frame->this->name, GF_LOG_TRACE, "fault frame %p returned",
-                frame);
-        pthread_mutex_destroy (&local->local_lock);
+    gf_msg_trace(frame->this->name, 0, "fault frame %p returned", frame);
+    pthread_mutex_destroy(&local->local_lock);
 
-        fd_unref (local->fd);
+    fd_unref(local->fd);
+    if (local->xattr_req)
+        dict_unref(local->xattr_req);
 
-        STACK_DESTROY (frame->root);
-        return 0;
+    STACK_DESTROY(frame->root);
+    return 0;
 }
 
-
 /*
  * ioc_page_fault -
  *
@@ -574,219 +556,212 @@ unlock:
  *
  */
 void
-ioc_page_fault (ioc_inode_t *ioc_inode, call_frame_t *frame, fd_t *fd,
-                off_t offset)
+ioc_page_fault(ioc_inode_t *ioc_inode, call_frame_t *frame, fd_t *fd,
+               off_t offset)
 {
-        ioc_table_t  *table       = NULL;
-        call_frame_t *fault_frame = NULL;
-        ioc_local_t  *fault_local = NULL;
-        int32_t       op_ret      = -1, op_errno = -1;
-        ioc_waitq_t  *waitq       = NULL;
-        ioc_page_t   *page        = NULL;
-
-        GF_ASSERT (ioc_inode);
-        if (frame == NULL) {
-                op_ret = -1;
-                op_errno = EINVAL;
-                gf_log ("io-cache", GF_LOG_WARNING,
-                        "page fault on a NULL frame");
-                goto err;
-        }
-
-        table = ioc_inode->table;
-        fault_frame = copy_frame (frame);
-        if (fault_frame == NULL) {
-                op_ret = -1;
-                op_errno = ENOMEM;
-                goto err;
-        }
-
-        fault_local = mem_get0 (THIS->local_pool);
-        if (fault_local == NULL) {
-                op_ret = -1;
-                op_errno = ENOMEM;
-                STACK_DESTROY (fault_frame->root);
-                goto err;
-        }
-
-        /* NOTE: copy_frame() means, the frame the fop whose fd_ref we
-         * are using till now won't be valid till we get reply from server.
-         * we unref this fd, in fault_cbk */
-        fault_local->fd = fd_ref (fd);
-
-        fault_frame->local = fault_local;
-        pthread_mutex_init (&fault_local->local_lock, NULL);
-
-        INIT_LIST_HEAD (&fault_local->fill_list);
-        fault_local->pending_offset = offset;
-        fault_local->pending_size = table->page_size;
-        fault_local->inode = ioc_inode;
-
-        gf_log (frame->this->name, GF_LOG_TRACE,
-                "stack winding page fault for offset = %"PRId64" with "
-                "frame %p", offset, fault_frame);
-
-        STACK_WIND (fault_frame, ioc_fault_cbk, FIRST_CHILD(fault_frame->this),
-                    FIRST_CHILD(fault_frame->this)->fops->readv, fd,
-                    table->page_size, offset, 0, NULL);
-        return;
+    ioc_table_t *table = NULL;
+    call_frame_t *fault_frame = NULL;
+    ioc_local_t *fault_local = NULL;
+    ioc_local_t *local = NULL;
+    int32_t op_ret = -1, op_errno = -1;
+    ioc_waitq_t *waitq = NULL;
+    ioc_page_t *page = NULL;
+
+    GF_ASSERT(ioc_inode);
+    if (frame == NULL) {
+        op_ret = -1;
+        op_errno = EINVAL;
+        gf_smsg("io-cache", GF_LOG_WARNING, EINVAL, IO_CACHE_MSG_PAGE_FAULT,
+                NULL);
+        goto err;
+    }
+
+    table = ioc_inode->table;
+    fault_frame = copy_frame(frame);
+    if (fault_frame == NULL) {
+        op_ret = -1;
+        op_errno = ENOMEM;
+        goto err;
+    }
+
+    local = frame->local;
+    fault_local = mem_get0(THIS->local_pool);
+    if (fault_local == NULL) {
+        op_ret = -1;
+        op_errno = ENOMEM;
+        STACK_DESTROY(fault_frame->root);
+        goto err;
+    }
+
+    /* NOTE: copy_frame() means, the frame the fop whose fd_ref we
+     * are using till now won't be valid till we get reply from server.
+     * we unref this fd, in fault_cbk */
+    fault_local->fd = fd_ref(fd);
+
+    fault_frame->local = fault_local;
+    pthread_mutex_init(&fault_local->local_lock, NULL);
+
+    INIT_LIST_HEAD(&fault_local->fill_list);
+    fault_local->pending_offset = offset;
+    fault_local->pending_size = table->page_size;
+    fault_local->inode = ioc_inode;
+
+    if (local && local->xattr_req)
+        fault_local->xattr_req = dict_ref(local->xattr_req);
+
+    gf_msg_trace(frame->this->name, 0,
+                 "stack winding page fault for offset = %" PRId64
+                 " with "
+                 "frame %p",
+                 offset, fault_frame);
+
+    STACK_WIND(fault_frame, ioc_fault_cbk, FIRST_CHILD(fault_frame->this),
+               FIRST_CHILD(fault_frame->this)->fops->readv, fd,
+               table->page_size, offset, 0, fault_local->xattr_req);
+    return;
 
 err:
-        ioc_inode_lock (ioc_inode);
-        {
-                page = __ioc_page_get (ioc_inode, offset);
-                if (page != NULL) {
-                        waitq = __ioc_page_error (page, op_ret, op_errno);
-                }
+    ioc_inode_lock(ioc_inode);
+    {
+        page = __ioc_page_get(ioc_inode, offset);
+        if (page != NULL) {
+            waitq = __ioc_page_error(page, op_ret, op_errno);
         }
-        ioc_inode_unlock (ioc_inode);
+    }
+    ioc_inode_unlock(ioc_inode);
 
-        if (waitq != NULL) {
-                ioc_waitq_return (waitq);
-        }
+    if (waitq != NULL) {
+        ioc_waitq_return(waitq);
+    }
 }
 
-
 int32_t
-__ioc_frame_fill (ioc_page_t *page, call_frame_t *frame, off_t offset,
-                  size_t size, int32_t op_errno)
+__ioc_frame_fill(ioc_page_t *page, call_frame_t *frame, off_t offset,
+                 size_t size, int32_t op_errno)
 {
-        ioc_local_t *local      = NULL;
-        ioc_fill_t  *fill       = NULL;
-        off_t        src_offset = 0;
-        off_t        dst_offset = 0;
-        ssize_t      copy_size  = 0;
-        ioc_inode_t *ioc_inode  = NULL;
-        ioc_fill_t  *new        = NULL;
-        int8_t       found      = 0;
-        int32_t      ret        = -1;
-
-        GF_VALIDATE_OR_GOTO ("io-cache", frame, out);
-
-        local = frame->local;
-        GF_VALIDATE_OR_GOTO (frame->this->name, local, out);
-
-        if (page == NULL) {
-                gf_log (frame->this->name, GF_LOG_WARNING,
-                        "NULL page has been provided to serve read request");
-                local->op_ret = -1;
-                local->op_errno = EINVAL;
-                goto out;
+    ioc_local_t *local = NULL;
+    ioc_fill_t *fill = NULL;
+    off_t src_offset = 0;
+    off_t dst_offset = 0;
+    ssize_t copy_size = 0;
+    ioc_inode_t *ioc_inode = NULL;
+    ioc_fill_t *new = NULL;
+    int8_t found = 0;
+    int32_t ret = -1;
+
+    GF_VALIDATE_OR_GOTO("io-cache", frame, out);
+
+    local = frame->local;
+    GF_VALIDATE_OR_GOTO(frame->this->name, local, out);
+
+    if (page == NULL) {
+        gf_smsg(frame->this->name, GF_LOG_WARNING, 0,
+                IO_CACHE_MSG_SERVE_READ_REQUEST, NULL);
+        local->op_ret = -1;
+        local->op_errno = EINVAL;
+        goto out;
+    }
+
+    ioc_inode = page->inode;
+
+    gf_msg_trace(frame->this->name, 0,
+                 "frame (%p) offset = %" PRId64 " && size = %" GF_PRI_SIZET
+                 " "
+                 "&& page->size = %" GF_PRI_SIZET " && wait_count = %d",
+                 frame, offset, size, page->size, local->wait_count);
+
+    /* immediately move this page to the end of the page_lru list */
+    list_move_tail(&page->page_lru, &ioc_inode->cache.page_lru);
+    /* fill local->pending_size bytes from local->pending_offset */
+    if (local->op_ret != -1) {
+        local->op_errno = op_errno;
+
+        if (page->size == 0) {
+            goto done;
         }
 
-        ioc_inode = page->inode;
-
-        gf_log (frame->this->name, GF_LOG_TRACE,
-                "frame (%p) offset = %"PRId64" && size = %"GF_PRI_SIZET" "
-                "&& page->size = %"GF_PRI_SIZET" && wait_count = %d",
-                frame, offset, size, page->size, local->wait_count);
-
-        /* immediately move this page to the end of the page_lru list */
-        list_move_tail (&page->page_lru, &ioc_inode->cache.page_lru);
-        /* fill local->pending_size bytes from local->pending_offset */
-        if (local->op_ret != -1) {
-                local->op_errno = op_errno;
-
-                if (page->size == 0) {
-                        goto done;
-                }
+        if (offset > page->offset)
+            /* offset is offset in file, convert it to offset in
+             * page */
+            src_offset = offset - page->offset;
+        /*FIXME: since offset is the offset within page is the
+         * else case valid? */
+        else
+            /* local->pending_offset is in previous page. do not
+             * fill until we have filled all previous pages */
+            dst_offset = page->offset - offset;
+
+        /* we have to copy from offset to either end of this page
+         * or till the requested size */
+        copy_size = min(page->size - src_offset, size - dst_offset);
+
+        if (copy_size < 0) {
+            /* if page contains fewer bytes and the required offset
+               is beyond the page size in the page */
+            copy_size = src_offset = 0;
+        }
 
-                if (offset > page->offset)
-                        /* offset is offset in file, convert it to offset in
-                         * page */
-                        src_offset = offset - page->offset;
-                /*FIXME: since offset is the offset within page is the
-                 * else case valid? */
-                else
-                        /* local->pending_offset is in previous page. do not
-                         * fill until we have filled all previous pages */
-                        dst_offset = page->offset - offset;
-
-                /* we have to copy from offset to either end of this page
-                 * or till the requested size */
-                copy_size = min (page->size - src_offset,
-                                 size - dst_offset);
-
-                if (copy_size < 0) {
-                        /* if page contains fewer bytes and the required offset
-                           is beyond the page size in the page */
-                        copy_size = src_offset = 0;
-                }
+        gf_msg_trace(page->inode->table->xl->name, 0,
+                     "copy_size = %" GF_PRI_SIZET
+                     " && src_offset = "
+                     "%" PRId64 " && dst_offset = %" PRId64 "",
+                     copy_size, src_offset, dst_offset);
 
-                gf_log (page->inode->table->xl->name, GF_LOG_TRACE,
-                        "copy_size = %"GF_PRI_SIZET" && src_offset = "
-                        "%"PRId64" && dst_offset = %"PRId64"",
-                        copy_size, src_offset, dst_offset);
+        {
+            new = GF_CALLOC(1, sizeof(*new), gf_ioc_mt_ioc_fill_t);
+            if (new == NULL) {
+                local->op_ret = -1;
+                local->op_errno = ENOMEM;
+                goto out;
+            }
+
+            new->offset = page->offset;
+            new->size = copy_size;
+            new->iobref = iobref_ref(page->iobref);
+            new->count = iov_subset(page->vector, page->count, src_offset,
+                                    copy_size, &new->vector, 0);
+            if (new->count < 0) {
+                local->op_ret = -1;
+                local->op_errno = ENOMEM;
 
+                iobref_unref(new->iobref);
+                GF_FREE(new);
+                goto out;
+            }
+
+            /* add the ioc_fill to fill_list for this frame */
+            if (list_empty(&local->fill_list)) {
+                /* if list is empty, then this is the first
+                 * time we are filling frame, add the
+                 * ioc_fill_t to the end of list */
+                list_add_tail(&new->list, &local->fill_list);
+            } else {
+                found = 0;
+                /* list is not empty, we need to look for
+                 * where this offset fits in list */
+                list_for_each_entry(fill, &local->fill_list, list)
                 {
-                        new = GF_CALLOC (1, sizeof (*new),
-                                         gf_ioc_mt_ioc_fill_t);
-                        if (new == NULL) {
-                                local->op_ret = -1;
-                                local->op_errno = ENOMEM;
-                                goto out;
-                        }
-
-                        new->offset = page->offset;
-                        new->size = copy_size;
-                        new->iobref = iobref_ref (page->iobref);
-                        new->count = iov_subset (page->vector, page->count,
-                                                 src_offset,
-                                                 src_offset + copy_size,
-                                                 NULL);
-
-                        new->vector = GF_CALLOC (new->count,
-                                                 sizeof (struct iovec),
-                                                 gf_ioc_mt_iovec);
-                        if (new->vector == NULL) {
-                                local->op_ret = -1;
-                                local->op_errno = ENOMEM;
-
-                                iobref_unref (new->iobref);
-                                GF_FREE (new);
-                                goto out;
-                        }
-
-                        new->count = iov_subset (page->vector, page->count,
-                                                 src_offset,
-                                                 src_offset + copy_size,
-                                                 new->vector);
-
-                        /* add the ioc_fill to fill_list for this frame */
-                        if (list_empty (&local->fill_list)) {
-                                /* if list is empty, then this is the first
-                                 * time we are filling frame, add the
-                                 * ioc_fill_t to the end of list */
-                                list_add_tail (&new->list, &local->fill_list);
-                        } else {
-                                found = 0;
-                                /* list is not empty, we need to look for
-                                 * where this offset fits in list */
-                                list_for_each_entry (fill, &local->fill_list,
-                                                     list) {
-                                        if (fill->offset > new->offset) {
-                                                found = 1;
-                                                break;
-                                        }
-                                }
-
-                                if (found) {
-                                        list_add_tail (&new->list,
-                                                       &fill->list);
-                                } else {
-                                        list_add_tail (&new->list,
-                                                       &local->fill_list);
-                                }
-                        }
+                    if (fill->offset > new->offset) {
+                        found = 1;
+                        break;
+                    }
                 }
 
-                local->op_ret += copy_size;
+                if (found) {
+                    list_add_tail(&new->list, &fill->list);
+                } else {
+                    list_add_tail(&new->list, &local->fill_list);
+                }
+            }
         }
 
+        local->op_ret += copy_size;
+    }
+
 done:
-        ret = 0;
+    ret = 0;
 out:
-        return ret;
+    return ret;
 }
 
 /*
@@ -799,106 +774,109 @@ out:
  *
  */
 static void
-ioc_frame_unwind (call_frame_t *frame)
+ioc_frame_unwind(call_frame_t *frame)
 {
-        ioc_local_t   *local  = NULL;
-        ioc_fill_t    *fill   = NULL, *next = NULL;
-        int32_t        count  = 0;
-        struct iovec  *vector = NULL;
-        int32_t        copied = 0;
-        struct iobref *iobref = NULL;
-        struct iatt    stbuf  = {0,};
-        int32_t        op_ret = 0, op_errno = 0;
-
-        GF_ASSERT (frame);
-
-        local = frame->local;
-        if (local == NULL) {
-                gf_log (frame->this->name, GF_LOG_WARNING,
-                        "local is NULL");
-                op_ret = -1;
-                op_errno = ENOMEM;
-                goto unwind;
-        }
-
-        if (local->op_ret < 0) {
-                op_ret = local->op_ret;
-                op_errno = local->op_errno;
-                goto unwind;
-        }
-
-        //  ioc_local_lock (local);
-        iobref = iobref_new ();
-        if (iobref == NULL) {
+    ioc_local_t *local = NULL;
+    ioc_fill_t *fill = NULL, *next = NULL;
+    int32_t count = 0;
+    struct iovec *vector = NULL;
+    int32_t copied = 0;
+    struct iobref *iobref = NULL;
+    struct iatt stbuf = {
+        0,
+    };
+    int32_t op_ret = 0, op_errno = 0;
+
+    GF_ASSERT(frame);
+
+    local = frame->local;
+    if (local == NULL) {
+        gf_smsg(frame->this->name, GF_LOG_WARNING, ENOMEM,
+                IO_CACHE_MSG_LOCAL_NULL, NULL);
+        op_ret = -1;
+        op_errno = ENOMEM;
+        goto unwind;
+    }
+
+    if (local->op_ret < 0) {
+        op_ret = local->op_ret;
+        op_errno = local->op_errno;
+        goto unwind;
+    }
+
+    //  ioc_local_lock (local);
+    iobref = iobref_new();
+    if (iobref == NULL) {
+        op_ret = -1;
+        op_errno = ENOMEM;
+    }
+
+    if (list_empty(&local->fill_list)) {
+        gf_msg_trace(frame->this->name, 0,
+                     "frame(%p) has 0 entries in local->fill_list "
+                     "(offset = %" PRId64 " && size = %" GF_PRI_SIZET ")",
+                     frame, local->offset, local->size);
+    }
+
+    list_for_each_entry(fill, &local->fill_list, list) { count += fill->count; }
+
+    vector = GF_CALLOC(count, sizeof(*vector), gf_ioc_mt_iovec);
+    if (vector == NULL) {
+        op_ret = -1;
+        op_errno = ENOMEM;
+    }
+
+    list_for_each_entry_safe(fill, next, &local->fill_list, list)
+    {
+        /* # TODO: check why this if clause is needed at all. */
+        if ((vector != NULL) && (iobref != NULL)) {
+            memcpy(((char *)vector) + copied, fill->vector,
+                   fill->count * sizeof(*vector));
+
+            copied += (fill->count * sizeof(*vector));
+
+            if (iobref_merge(iobref, fill->iobref)) {
                 op_ret = -1;
                 op_errno = ENOMEM;
+            }
         }
 
-        if (list_empty (&local->fill_list)) {
-                gf_log (frame->this->name, GF_LOG_TRACE,
-                        "frame(%p) has 0 entries in local->fill_list "
-                        "(offset = %"PRId64" && size = %"GF_PRI_SIZET")",
-                        frame, local->offset, local->size);
-        }
-
-        list_for_each_entry (fill, &local->fill_list, list) {
-                count += fill->count;
-        }
+        list_del(&fill->list);
+        iobref_unref(fill->iobref);
+        GF_FREE(fill->vector);
+        GF_FREE(fill);
+    }
 
-        vector = GF_CALLOC (count, sizeof (*vector), gf_ioc_mt_iovec);
-        if (vector == NULL) {
-                op_ret = -1;
-                op_errno = ENOMEM;
-        }
-
-        list_for_each_entry_safe (fill, next, &local->fill_list, list) {
-                if ((vector != NULL) &&  (iobref != NULL)) {
-                        memcpy (((char *)vector) + copied,
-                                fill->vector,
-                                fill->count * sizeof (*vector));
-
-                        copied += (fill->count * sizeof (*vector));
-
-                        if (iobref_merge (iobref, fill->iobref)) {
-				op_ret = -1;
-				op_errno = ENOMEM;
-			}
-                }
-
-                list_del (&fill->list);
-                iobref_unref (fill->iobref);
-                GF_FREE (fill->vector);
-                GF_FREE (fill);
-        }
-
-        if (op_ret != -1) {
-                op_ret = iov_length (vector, count);
-        }
+    if (op_ret != -1) {
+        op_ret = iov_length(vector, count);
+    }
 
 unwind:
-        gf_log (frame->this->name, GF_LOG_TRACE,
-                "frame(%p) unwinding with op_ret=%d", frame, op_ret);
-
-        //  ioc_local_unlock (local);
-
-        frame->local = NULL;
-        STACK_UNWIND_STRICT (readv, frame, op_ret, op_errno, vector,
-                             count, &stbuf, iobref, NULL);
-
-        if (iobref != NULL) {
-                iobref_unref (iobref);
-        }
-
-        if (vector != NULL) {
-                GF_FREE (vector);
-                vector = NULL;
-        }
-
-        pthread_mutex_destroy (&local->local_lock);
-        if (local)
-                mem_put (local);
-
-        return;
+    gf_msg_trace(frame->this->name, 0, "frame(%p) unwinding with op_ret=%d",
+                 frame, op_ret);
+
+    //  ioc_local_unlock (local);
+
+    frame->local = NULL;
+    STACK_UNWIND_STRICT(readv, frame, op_ret, op_errno, vector, count, &stbuf,
+                        iobref, NULL);
+
+    if (iobref != NULL) {
+        iobref_unref(iobref);
+    }
+
+    if (vector != NULL) {
+        GF_FREE(vector);
+        vector = NULL;
+    }
+
+    if (local) {
+        if (local->xattr_req)
+            dict_unref(local->xattr_req);
+        pthread_mutex_destroy(&local->local_lock);
+        mem_put(local);
+    }
+    return;
 }
 
 /*
@@ -908,27 +886,27 @@ unwind:
  * to be called only when a frame is waiting on an in-transit page
  */
 void
-ioc_frame_return (call_frame_t *frame)
+ioc_frame_return(call_frame_t *frame)
 {
-        ioc_local_t *local      = NULL;
-        int32_t      wait_count = 0;
+    ioc_local_t *local = NULL;
+    int32_t wait_count = 0;
 
-        GF_ASSERT (frame);
+    GF_ASSERT(frame);
 
-        local = frame->local;
-        GF_ASSERT (local->wait_count > 0);
+    local = frame->local;
+    GF_ASSERT(local->wait_count > 0);
 
-        ioc_local_lock (local);
-        {
-                wait_count = --local->wait_count;
-        }
-        ioc_local_unlock (local);
+    ioc_local_lock(local);
+    {
+        wait_count = --local->wait_count;
+    }
+    ioc_local_unlock(local);
 
-        if (!wait_count) {
-                ioc_frame_unwind (frame);
-        }
+    if (!wait_count) {
+        ioc_frame_unwind(frame);
+    }
 
-        return;
+    return;
 }
 
 /*
@@ -938,41 +916,39 @@ ioc_frame_return (call_frame_t *frame)
  * to be called only when a frame is waiting on an in-transit page
  */
 ioc_waitq_t *
-__ioc_page_wakeup (ioc_page_t *page, int32_t op_errno)
+__ioc_page_wakeup(ioc_page_t *page, int32_t op_errno)
 {
-        ioc_waitq_t  *waitq = NULL, *trav = NULL;
-        call_frame_t *frame = NULL;
-        int32_t       ret   = -1;
+    ioc_waitq_t *waitq = NULL, *trav = NULL;
+    call_frame_t *frame = NULL;
+    int32_t ret = -1;
 
-        GF_VALIDATE_OR_GOTO ("io-cache", page, out);
+    GF_VALIDATE_OR_GOTO("io-cache", page, out);
 
-        waitq = page->waitq;
-        page->waitq = NULL;
+    waitq = page->waitq;
+    page->waitq = NULL;
 
-        page->ready = 1;
+    page->ready = 1;
 
-        gf_log (page->inode->table->xl->name, GF_LOG_TRACE,
-                "page is %p && waitq = %p", page, waitq);
+    gf_msg_trace(page->inode->table->xl->name, 0, "page is %p && waitq = %p",
+                 page, waitq);
 
-        for (trav = waitq; trav; trav = trav->next) {
-                frame = trav->data;
-                ret = __ioc_frame_fill (page, frame, trav->pending_offset,
-                                        trav->pending_size, op_errno);
-                if (ret == -1) {
-                        break;
-                }
+    for (trav = waitq; trav; trav = trav->next) {
+        frame = trav->data;
+        ret = __ioc_frame_fill(page, frame, trav->pending_offset,
+                               trav->pending_size, op_errno);
+        if (ret == -1) {
+            break;
         }
+    }
 
-        if (page->stale) {
-                __ioc_page_destroy (page);
-        }
+    if (page->stale) {
+        __ioc_page_destroy(page);
+    }
 
 out:
-        return waitq;
+    return waitq;
 }
 
-
-
 /*
  * ioc_page_error -
  * @page:
@@ -981,46 +957,45 @@ out:
  *
  */
 ioc_waitq_t *
-__ioc_page_error (ioc_page_t *page, int32_t op_ret, int32_t op_errno)
+__ioc_page_error(ioc_page_t *page, int32_t op_ret, int32_t op_errno)
 {
-        ioc_waitq_t  *waitq = NULL, *trav = NULL;
-        call_frame_t *frame = NULL;
-        int64_t       ret   = 0;
-        ioc_table_t  *table = NULL;
-        ioc_local_t  *local = NULL;
-
-        GF_VALIDATE_OR_GOTO ("io-cache", page, out);
+    ioc_waitq_t *waitq = NULL, *trav = NULL;
+    call_frame_t *frame = NULL;
+    int64_t ret = 0;
+    ioc_table_t *table = NULL;
+    ioc_local_t *local = NULL;
 
-        waitq = page->waitq;
-        page->waitq = NULL;
+    GF_VALIDATE_OR_GOTO("io-cache", page, out);
 
-        gf_log (page->inode->table->xl->name, GF_LOG_DEBUG,
-                "page error for page = %p & waitq = %p", page, waitq);
+    waitq = page->waitq;
+    page->waitq = NULL;
 
-        for (trav = waitq; trav; trav = trav->next) {
+    gf_msg_debug(page->inode->table->xl->name, 0,
+                 "page error for page = %p & waitq = %p", page, waitq);
 
-                frame = trav->data;
+    for (trav = waitq; trav; trav = trav->next) {
+        frame = trav->data;
 
-                local = frame->local;
-                ioc_local_lock (local);
-                {
-                        if (local->op_ret != -1) {
-                                local->op_ret = op_ret;
-                                local->op_errno = op_errno;
-                        }
-                }
-                ioc_local_unlock (local);
+        local = frame->local;
+        ioc_local_lock(local);
+        {
+            if (local->op_ret != -1) {
+                local->op_ret = op_ret;
+                local->op_errno = op_errno;
+            }
         }
+        ioc_local_unlock(local);
+    }
 
-        table = page->inode->table;
-        ret = __ioc_page_destroy (page);
+    table = page->inode->table;
+    ret = __ioc_page_destroy(page);
 
-        if (ret != -1) {
-                table->cache_used -= ret;
-        }
+    if (ret != -1) {
+        table->cache_used -= ret;
+    }
 
 out:
-        return waitq;
+    return waitq;
 }
 
 /*
@@ -1031,22 +1006,22 @@ out:
  *
  */
 ioc_waitq_t *
-ioc_page_error (ioc_page_t *page, int32_t op_ret, int32_t op_errno)
+ioc_page_error(ioc_page_t *page, int32_t op_ret, int32_t op_errno)
 {
-        ioc_waitq_t  *waitq = NULL;
-        struct ioc_inode *inode = NULL;
+    ioc_waitq_t *waitq = NULL;
+    struct ioc_inode *inode = NULL;
 
-        if (page == NULL) {
-                goto out;
-        }
+    if (page == NULL) {
+        goto out;
+    }
 
-        ioc_inode_lock (page->inode);
-        {
-                inode = page->inode;
-                waitq = __ioc_page_error (page, op_ret, op_errno);
-        }
-        ioc_inode_unlock (inode);
+    ioc_inode_lock(page->inode);
+    {
+        inode = page->inode;
+        waitq = __ioc_page_error(page, op_ret, op_errno);
+    }
+    ioc_inode_unlock(inode);
 
 out:
-        return waitq;
+    return waitq;
 }
diff --git a/xlators/performance/io-threads/src/Makefile.am b/xlators/performance/io-threads/src/Makefile.am
index d63042e7c05..7570cf41ed2 100644
--- a/xlators/performance/io-threads/src/Makefile.am
+++ b/xlators/performance/io-threads/src/Makefile.am
@@ -1,14 +1,15 @@
 xlator_LTLIBRARIES = io-threads.la
 xlatordir = $(libdir)/glusterfs/$(PACKAGE_VERSION)/xlator/performance
 
-io_threads_la_LDFLAGS = -module -avoid-version 
+io_threads_la_LDFLAGS = -module $(GF_XLATOR_DEFAULT_LDFLAGS)
 
 io_threads_la_SOURCES = io-threads.c
 io_threads_la_LIBADD = $(top_builddir)/libglusterfs/src/libglusterfs.la
 
-noinst_HEADERS = io-threads.h iot-mem-types.h
+noinst_HEADERS = io-threads.h iot-mem-types.h io-threads-messages.h
 
-AM_CPPFLAGS = $(GF_CPPFLAGS) -I$(top_srcdir)/libglusterfs/src
+AM_CPPFLAGS = $(GF_CPPFLAGS) -I$(top_srcdir)/libglusterfs/src \
+	-I$(top_srcdir)/rpc/xdr/src -I$(top_builddir)/rpc/xdr/src
 
 AM_CFLAGS = -Wall $(GF_CFLAGS)
 
diff --git a/xlators/performance/io-threads/src/io-threads-messages.h b/xlators/performance/io-threads/src/io-threads-messages.h
new file mode 100644
index 00000000000..6229c353f96
--- /dev/null
+++ b/xlators/performance/io-threads/src/io-threads-messages.h
@@ -0,0 +1,41 @@
+/*Copyright (c) 2015 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
+*/
+
+#ifndef _IO_THREADS_MESSAGES_H_
+#define _IO_THREADS_MESSAGES_H_
+
+#include <glusterfs/glfs-message-id.h>
+
+/* To add new message IDs, append new identifiers at the end of the list.
+ *
+ * Never remove a message ID. If it's not used anymore, you can rename it or
+ * leave it as it is, but not delete it. This is to prevent reutilization of
+ * IDs by other messages.
+ *
+ * The component name must match one of the entries defined in
+ * glfs-message-id.h.
+ */
+
+GLFS_MSGID(IO_THREADS, IO_THREADS_MSG_INIT_FAILED,
+           IO_THREADS_MSG_XLATOR_CHILD_MISCONFIGURED, IO_THREADS_MSG_NO_MEMORY,
+           IO_THREADS_MSG_VOL_MISCONFIGURED, IO_THREADS_MSG_SIZE_NOT_SET,
+           IO_THREADS_MSG_OUT_OF_MEMORY, IO_THREADS_MSG_PTHREAD_INIT_FAILED,
+           IO_THREADS_MSG_WORKER_THREAD_INIT_FAILED);
+
+#define IO_THREADS_MSG_INIT_FAILED_STR "Thread attribute initialization failed"
+#define IO_THREADS_MSG_SIZE_NOT_SET_STR "Using default thread stack size"
+#define IO_THREADS_MSG_NO_MEMORY_STR "Memory accounting init failed"
+#define IO_THREADS_MSG_XLATOR_CHILD_MISCONFIGURED_STR                          \
+    "FATAL: iot not configured with exactly one child"
+#define IO_THREADS_MSG_VOL_MISCONFIGURED_STR "dangling volume. check volfile"
+#define IO_THREADS_MSG_OUT_OF_MEMORY_STR "out of memory"
+#define IO_THREADS_MSG_PTHREAD_INIT_FAILED_STR "init failed"
+#define IO_THREADS_MSG_WORKER_THREAD_INIT_FAILED_STR                           \
+    "cannot initialize worker threads, exiting init"
+#endif /* _IO_THREADS_MESSAGES_H_ */
diff --git a/xlators/performance/io-threads/src/io-threads.c b/xlators/performance/io-threads/src/io-threads.c
index 148e55ab297..3d24cc97f4b 100644
--- a/xlators/performance/io-threads/src/io-threads.c
+++ b/xlators/performance/io-threads/src/io-threads.c
@@ -8,281 +8,300 @@
   cases as published by the Free Software Foundation.
 */
 
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
-
-#include "call-stub.h"
-#include "defaults.h"
-#include "glusterfs.h"
-#include "logging.h"
-#include "dict.h"
-#include "xlator.h"
+#include <glusterfs/call-stub.h>
+#include <glusterfs/defaults.h>
+#include <glusterfs/glusterfs.h>
+#include <glusterfs/logging.h>
+#include <glusterfs/dict.h>
+#include <glusterfs/xlator.h>
 #include "io-threads.h"
+#include <signal.h>
 #include <stdlib.h>
 #include <sys/time.h>
 #include <time.h>
-#include "locking.h"
+#include <glusterfs/locking.h>
+#include "io-threads-messages.h"
+#include <glusterfs/timespec.h>
 
-void *iot_worker (void *arg);
-int iot_workers_scale (iot_conf_t *conf);
-int __iot_workers_scale (iot_conf_t *conf);
+void *
+iot_worker(void *arg);
+int
+iot_workers_scale(iot_conf_t *conf);
+int
+__iot_workers_scale(iot_conf_t *conf);
 struct volume_options options[];
 
-#define IOT_FOP(name, frame, this, args ...)                                   \
-        do {                                                                   \
-                call_stub_t     *__stub     = NULL;                            \
-                int              __ret      = -1;                              \
+#define IOT_FOP(name, frame, this, args...)                                    \
+    do {                                                                       \
+        call_stub_t *__stub = NULL;                                            \
+        int __ret = -1;                                                        \
                                                                                \
-                __stub = fop_##name##_stub(frame, default_##name##_resume, args);  \
-                if (!__stub) {                                                 \
-                        __ret = -ENOMEM;                                       \
-                        goto out;                                              \
-                }                                                              \
+        __stub = fop_##name##_stub(frame, default_##name##_resume, args);      \
+        if (!__stub) {                                                         \
+            __ret = -ENOMEM;                                                   \
+            goto out;                                                          \
+        }                                                                      \
                                                                                \
-                __ret = iot_schedule (frame, this, __stub);                    \
+        __ret = iot_schedule(frame, this, __stub);                             \
                                                                                \
-        out:                                                                   \
-                if (__ret < 0) {                                               \
-                        default_##name##_failure_cbk (frame, -__ret);          \
-                        if (__stub != NULL) {                                  \
-                                call_stub_destroy (__stub);                    \
-                        }                                                      \
-                }                                                              \
-        } while (0)
+    out:                                                                       \
+        if (__ret < 0) {                                                       \
+            default_##name##_failure_cbk(frame, -__ret);                       \
+            if (__stub != NULL) {                                              \
+                call_stub_destroy(__stub);                                     \
+            }                                                                  \
+        }                                                                      \
+    } while (0)
+
+iot_client_ctx_t *
+iot_get_ctx(xlator_t *this, client_t *client)
+{
+    iot_client_ctx_t *ctx = NULL;
+    iot_client_ctx_t *setted_ctx = NULL;
+    int i;
+
+    if (client_ctx_get(client, this, (void **)&ctx) != 0) {
+        ctx = GF_MALLOC(GF_FOP_PRI_MAX * sizeof(*ctx), gf_iot_mt_client_ctx_t);
+        if (ctx) {
+            for (i = 0; i < GF_FOP_PRI_MAX; ++i) {
+                INIT_LIST_HEAD(&ctx[i].clients);
+                INIT_LIST_HEAD(&ctx[i].reqs);
+            }
+            setted_ctx = client_ctx_set(client, this, ctx);
+            if (ctx != setted_ctx) {
+                GF_FREE(ctx);
+                ctx = setted_ctx;
+            }
+        }
+    }
+
+    return ctx;
+}
 
 call_stub_t *
-__iot_dequeue (iot_conf_t *conf, int *pri, struct timespec *sleep)
-{
-        call_stub_t  *stub = NULL;
-        int           i = 0;
-	struct timeval curtv = {0,}, difftv = {0,};
-
-        *pri = -1;
-	sleep->tv_sec = 0;
-	sleep->tv_nsec = 0;
-        for (i = 0; i < IOT_PRI_MAX; i++) {
-                if (list_empty (&conf->reqs[i]) ||
-                   (conf->ac_iot_count[i] >= conf->ac_iot_limit[i]))
-                        continue;
-
-		if (i == IOT_PRI_LEAST) {
-			pthread_mutex_lock(&conf->throttle.lock);
-			if (!conf->throttle.sample_time.tv_sec) {
-				/* initialize */
-				gettimeofday(&conf->throttle.sample_time, NULL);
-			} else {
-				/*
-				 * Maintain a running count of least priority
-				 * operations that are handled over a particular
-				 * time interval. The count is provided via
-				 * state dump and is used as a measure against
-				 * least priority op throttling.
-				 */
-				gettimeofday(&curtv, NULL);
-				timersub(&curtv, &conf->throttle.sample_time,
-					 &difftv);
-				if (difftv.tv_sec >= IOT_LEAST_THROTTLE_DELAY) {
-					conf->throttle.cached_rate =
-						conf->throttle.sample_cnt;
-					conf->throttle.sample_cnt = 0;
-					conf->throttle.sample_time = curtv;
-				}
-
-				/*
-				 * If we're over the configured rate limit,
-				 * provide an absolute time to the caller that
-				 * represents the soonest we're allowed to
-				 * return another least priority request.
-				 */
-				if (conf->throttle.rate_limit &&
-				    conf->throttle.sample_cnt >=
-						conf->throttle.rate_limit) {
-					struct timeval delay;
-					delay.tv_sec = IOT_LEAST_THROTTLE_DELAY;
-					delay.tv_usec = 0;
-
-					timeradd(&conf->throttle.sample_time,
-						 &delay, &curtv);
-					TIMEVAL_TO_TIMESPEC(&curtv, sleep);
-
-					pthread_mutex_unlock(
-						&conf->throttle.lock);
-					break;
-				}
-			}
-			conf->throttle.sample_cnt++;
-			pthread_mutex_unlock(&conf->throttle.lock);
-		}
-
-                stub = list_entry (conf->reqs[i].next, call_stub_t, list);
-                conf->ac_iot_count[i]++;
-                *pri = i;
-                break;
+__iot_dequeue(iot_conf_t *conf, int *pri)
+{
+    call_stub_t *stub = NULL;
+    int i = 0;
+    iot_client_ctx_t *ctx;
+
+    *pri = -1;
+    for (i = 0; i < GF_FOP_PRI_MAX; i++) {
+        if (conf->ac_iot_count[i] >= conf->ac_iot_limit[i]) {
+            continue;
         }
 
-        if (!stub)
-                return NULL;
+        if (list_empty(&conf->clients[i])) {
+            continue;
+        }
 
-        conf->queue_size--;
-        conf->queue_sizes[*pri]--;
-        list_del_init (&stub->list);
+        /* Get the first per-client queue for this priority. */
+        ctx = list_first_entry(&conf->clients[i], iot_client_ctx_t, clients);
+        if (!ctx) {
+            continue;
+        }
 
-        return stub;
-}
+        if (list_empty(&ctx->reqs)) {
+            continue;
+        }
 
+        /* Get the first request on that queue. */
+        stub = list_first_entry(&ctx->reqs, call_stub_t, list);
+        list_del_init(&stub->list);
+        if (list_empty(&ctx->reqs)) {
+            list_del_init(&ctx->clients);
+        } else {
+            list_rotate_left(&conf->clients[i]);
+        }
 
-void
-__iot_enqueue (iot_conf_t *conf, call_stub_t *stub, int pri)
-{
-        if (pri < 0 || pri >= IOT_PRI_MAX)
-                pri = IOT_PRI_MAX-1;
+        conf->ac_iot_count[i]++;
+        conf->queue_marked[i] = _gf_false;
+        *pri = i;
+        break;
+    }
 
-        list_add_tail (&stub->list, &conf->reqs[pri]);
+    if (!stub)
+        return NULL;
 
-        conf->queue_size++;
-        conf->queue_sizes[pri]++;
+    conf->queue_size--;
+    conf->queue_sizes[*pri]--;
 
-        return;
+    return stub;
 }
 
+void
+__iot_enqueue(iot_conf_t *conf, call_stub_t *stub, int pri)
+{
+    client_t *client = stub->frame->root->client;
+    iot_client_ctx_t *ctx;
+
+    if (pri < 0 || pri >= GF_FOP_PRI_MAX)
+        pri = GF_FOP_PRI_MAX - 1;
+
+    if (client) {
+        ctx = iot_get_ctx(THIS, client);
+        if (ctx) {
+            ctx = &ctx[pri];
+        }
+    } else {
+        ctx = NULL;
+    }
+    if (!ctx) {
+        ctx = &conf->no_client[pri];
+    }
+
+    if (list_empty(&ctx->reqs)) {
+        list_add_tail(&ctx->clients, &conf->clients[pri]);
+    }
+    list_add_tail(&stub->list, &ctx->reqs);
+
+    conf->queue_size++;
+    GF_ATOMIC_INC(conf->stub_cnt);
+    conf->queue_sizes[pri]++;
+}
 
 void *
-iot_worker (void *data)
-{
-        iot_conf_t       *conf = NULL;
-        xlator_t         *this = NULL;
-        call_stub_t      *stub = NULL;
-        struct timespec   sleep_till = {0, };
-        int               ret = 0;
-        int               pri = -1;
-        char              timeout = 0;
-        char              bye = 0;
-	struct timespec	  sleep = {0,};
-
-        conf = data;
-        this = conf->this;
-        THIS = this;
-
-        for (;;) {
-                sleep_till.tv_sec = time (NULL) + conf->idle_time;
-
-                pthread_mutex_lock (&conf->mutex);
-                {
-                        if (pri != -1) {
-                                conf->ac_iot_count[pri]--;
-                                pri = -1;
-                        }
-                        while (conf->queue_size == 0) {
-                                conf->sleep_count++;
-
-                                ret = pthread_cond_timedwait (&conf->cond,
-                                                              &conf->mutex,
-                                                              &sleep_till);
-                                conf->sleep_count--;
-
-                                if (ret == ETIMEDOUT) {
-                                        timeout = 1;
-                                        break;
-                                }
-                        }
-
-                        if (timeout) {
-                                if (conf->curr_count > IOT_MIN_THREADS) {
-                                        conf->curr_count--;
-                                        bye = 1;
-                                        gf_log (conf->this->name, GF_LOG_DEBUG,
-                                                "timeout, terminated. conf->curr_count=%d",
-                                                conf->curr_count);
-                                } else {
-                                        timeout = 0;
-                                }
-                        }
-
-                        stub = __iot_dequeue (conf, &pri, &sleep);
-			if (!stub && (sleep.tv_sec || sleep.tv_nsec)) {
-				pthread_cond_timedwait(&conf->cond,
-						       &conf->mutex, &sleep);
-				pthread_mutex_unlock(&conf->mutex);
-				continue;
-			}
+iot_worker(void *data)
+{
+    iot_conf_t *conf = NULL;
+    xlator_t *this = NULL;
+    call_stub_t *stub = NULL;
+    struct timespec sleep_till = {
+        0,
+    };
+    int ret = 0;
+    int pri = -1;
+    gf_boolean_t bye = _gf_false;
+
+    conf = data;
+    this = conf->this;
+    THIS = this;
+
+    for (;;) {
+        pthread_mutex_lock(&conf->mutex);
+        {
+            if (pri != -1) {
+                conf->ac_iot_count[pri]--;
+                pri = -1;
+            }
+            while (conf->queue_size == 0) {
+                if (conf->down) {
+                    bye = _gf_true; /*Avoid sleep*/
+                    break;
                 }
-                pthread_mutex_unlock (&conf->mutex);
 
-                if (stub) /* guard against spurious wakeups */
-                        call_resume (stub);
+                clock_gettime(CLOCK_REALTIME_COARSE, &sleep_till);
+                sleep_till.tv_sec += conf->idle_time;
 
-                if (bye)
-                        break;
-        }
+                conf->sleep_count++;
+                ret = pthread_cond_timedwait(&conf->cond, &conf->mutex,
+                                             &sleep_till);
+                conf->sleep_count--;
 
-        if (pri != -1) {
-                pthread_mutex_lock (&conf->mutex);
-                {
-                        conf->ac_iot_count[pri]--;
+                if (conf->down || ret == ETIMEDOUT) {
+                    bye = _gf_true;
+                    break;
+                }
+            }
+
+            if (bye) {
+                if (conf->down || conf->curr_count > IOT_MIN_THREADS) {
+                    conf->curr_count--;
+                    if (conf->curr_count == 0)
+                        pthread_cond_broadcast(&conf->cond);
+                    gf_msg_debug(conf->this->name, 0,
+                                 "terminated. "
+                                 "conf->curr_count=%d",
+                                 conf->curr_count);
+                } else {
+                    bye = _gf_false;
                 }
-                pthread_mutex_unlock (&conf->mutex);
+            }
+
+            if (!bye)
+                stub = __iot_dequeue(conf, &pri);
         }
-        return NULL;
-}
+        pthread_mutex_unlock(&conf->mutex);
+
+        if (stub) { /* guard against spurious wakeups */
+            if (stub->poison) {
+                gf_log(this->name, GF_LOG_INFO, "Dropping poisoned request %p.",
+                       stub);
+                call_stub_destroy(stub);
+            } else {
+                call_resume(stub);
+            }
+            GF_ATOMIC_DEC(conf->stub_cnt);
+        }
+        stub = NULL;
 
+        if (bye)
+            break;
+    }
+
+    return NULL;
+}
 
 int
-do_iot_schedule (iot_conf_t *conf, call_stub_t *stub, int pri)
+do_iot_schedule(iot_conf_t *conf, call_stub_t *stub, int pri)
 {
-        int   ret = 0;
+    int ret = 0;
 
-        pthread_mutex_lock (&conf->mutex);
-        {
-                __iot_enqueue (conf, stub, pri);
+    pthread_mutex_lock(&conf->mutex);
+    {
+        __iot_enqueue(conf, stub, pri);
 
-                pthread_cond_signal (&conf->cond);
+        pthread_cond_signal(&conf->cond);
 
-                ret = __iot_workers_scale (conf);
-        }
-        pthread_mutex_unlock (&conf->mutex);
+        ret = __iot_workers_scale(conf);
+    }
+    pthread_mutex_unlock(&conf->mutex);
 
-        return ret;
+    return ret;
 }
 
-char*
-iot_get_pri_meaning (iot_pri_t pri)
-{
-        char    *name = NULL;
-        switch (pri) {
-        case IOT_PRI_HI:
-                name = "fast";
-                break;
-        case IOT_PRI_NORMAL:
-                name = "normal";
-                break;
-        case IOT_PRI_LO:
-                name = "slow";
-                break;
-        case IOT_PRI_LEAST:
-                name = "least priority";
-                break;
-        case IOT_PRI_MAX:
-                name = "invalid";
-                break;
-        }
-        return name;
+char *
+iot_get_pri_meaning(gf_fop_pri_t pri)
+{
+    char *name = NULL;
+    switch (pri) {
+        case GF_FOP_PRI_HI:
+            name = "fast";
+            break;
+        case GF_FOP_PRI_NORMAL:
+            name = "normal";
+            break;
+        case GF_FOP_PRI_LO:
+            name = "slow";
+            break;
+        case GF_FOP_PRI_LEAST:
+            name = "least";
+            break;
+        case GF_FOP_PRI_MAX:
+            name = "invalid";
+            break;
+        case GF_FOP_PRI_UNSPEC:
+            name = "unspecified";
+            break;
+    }
+    return name;
 }
 
 int
-iot_schedule (call_frame_t *frame, xlator_t *this, call_stub_t *stub)
+iot_schedule(call_frame_t *frame, xlator_t *this, call_stub_t *stub)
 {
-        int             ret = -1;
-        iot_pri_t       pri = IOT_PRI_MAX - 1;
-        iot_conf_t      *conf = this->private;
-
-        if ((frame->root->pid < GF_CLIENT_PID_MAX) && conf->least_priority) {
-                pri = IOT_PRI_LEAST;
-                goto out;
-        }
-
-        switch (stub->fop) {
+    int ret = -1;
+    gf_fop_pri_t pri = GF_FOP_PRI_MAX - 1;
+    iot_conf_t *conf = this->private;
+
+    if ((frame->root->pid < GF_CLIENT_PID_MAX) &&
+        (frame->root->pid != GF_CLIENT_PID_NO_ROOT_SQUASH) &&
+        conf->least_priority) {
+        pri = GF_FOP_PRI_LEAST;
+        goto out;
+    }
+
+    switch (stub->fop) {
         case GF_FOP_OPEN:
         case GF_FOP_STAT:
         case GF_FOP_FSTAT:
@@ -293,8 +312,12 @@ iot_schedule (call_frame_t *frame, xlator_t *this, call_stub_t *stub)
         case GF_FOP_STATFS:
         case GF_FOP_READDIR:
         case GF_FOP_READDIRP:
-                pri = IOT_PRI_HI;
-                break;
+        case GF_FOP_GETACTIVELK:
+        case GF_FOP_SETACTIVELK:
+        case GF_FOP_ICREATE:
+        case GF_FOP_NAMELINK:
+            pri = GF_FOP_PRI_HI;
+            break;
 
         case GF_FOP_CREATE:
         case GF_FOP_FLUSH:
@@ -303,6 +326,7 @@ iot_schedule (call_frame_t *frame, xlator_t *this, call_stub_t *stub)
         case GF_FOP_FINODELK:
         case GF_FOP_ENTRYLK:
         case GF_FOP_FENTRYLK:
+        case GF_FOP_LEASE:
         case GF_FOP_UNLINK:
         case GF_FOP_SETATTR:
         case GF_FOP_FSETATTR:
@@ -318,8 +342,9 @@ iot_schedule (call_frame_t *frame, xlator_t *this, call_stub_t *stub)
         case GF_FOP_FSETXATTR:
         case GF_FOP_REMOVEXATTR:
         case GF_FOP_FREMOVEXATTR:
-                pri = IOT_PRI_NORMAL;
-                break;
+        case GF_FOP_PUT:
+            pri = GF_FOP_PRI_NORMAL;
+            break;
 
         case GF_FOP_READ:
         case GF_FOP_WRITE:
@@ -330,816 +355,1236 @@ iot_schedule (call_frame_t *frame, xlator_t *this, call_stub_t *stub)
         case GF_FOP_XATTROP:
         case GF_FOP_FXATTROP:
         case GF_FOP_RCHECKSUM:
-	case GF_FOP_FALLOCATE:
-	case GF_FOP_DISCARD:
+        case GF_FOP_FALLOCATE:
+        case GF_FOP_DISCARD:
         case GF_FOP_ZEROFILL:
-                pri = IOT_PRI_LO;
-                break;
+        case GF_FOP_SEEK:
+            pri = GF_FOP_PRI_LO;
+            break;
 
         case GF_FOP_FORGET:
         case GF_FOP_RELEASE:
         case GF_FOP_RELEASEDIR:
         case GF_FOP_GETSPEC:
-                break;
+            break;
         case GF_FOP_IPC:
         default:
-                return -EINVAL;
-        }
+            return -EINVAL;
+    }
 out:
-        gf_log (this->name, GF_LOG_DEBUG, "%s scheduled as %s fop",
-                gf_fop_list[stub->fop], iot_get_pri_meaning (pri));
-        ret = do_iot_schedule (this->private, stub, pri);
-        return ret;
+    gf_msg_debug(this->name, 0, "%s scheduled as %s priority fop",
+                 gf_fop_list[stub->fop], iot_get_pri_meaning(pri));
+    if (this->private)
+        ret = do_iot_schedule(this->private, stub, pri);
+    return ret;
 }
 
 int
-iot_lookup (call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xdata)
+iot_lookup(call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xdata)
 {
-        IOT_FOP (lookup, frame, this, loc, xdata);
-        return 0;
+    IOT_FOP(lookup, frame, this, loc, xdata);
+    return 0;
 }
 
-
 int
-iot_setattr (call_frame_t *frame, xlator_t *this, loc_t *loc,
-             struct iatt *stbuf, int32_t valid, dict_t *xdata)
+iot_setattr(call_frame_t *frame, xlator_t *this, loc_t *loc, struct iatt *stbuf,
+            int32_t valid, dict_t *xdata)
 {
-        IOT_FOP (setattr, frame, this, loc, stbuf, valid, xdata);
-        return 0;
+    IOT_FOP(setattr, frame, this, loc, stbuf, valid, xdata);
+    return 0;
 }
 
-
 int
-iot_fsetattr (call_frame_t *frame, xlator_t *this, fd_t *fd,
-              struct iatt *stbuf, int32_t valid, dict_t *xdata)
+iot_fsetattr(call_frame_t *frame, xlator_t *this, fd_t *fd, struct iatt *stbuf,
+             int32_t valid, dict_t *xdata)
 {
-        IOT_FOP (fsetattr, frame, this, fd, stbuf, valid, xdata);
-        return 0;
+    IOT_FOP(fsetattr, frame, this, fd, stbuf, valid, xdata);
+    return 0;
 }
 
-
 int
-iot_access (call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t mask,
-            dict_t *xdata)
+iot_access(call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t mask,
+           dict_t *xdata)
 {
-        IOT_FOP (access, frame, this, loc, mask, xdata);
-        return 0;
+    IOT_FOP(access, frame, this, loc, mask, xdata);
+    return 0;
 }
 
-
 int
-iot_readlink (call_frame_t *frame, xlator_t *this, loc_t *loc, size_t size, dict_t *xdata)
+iot_readlink(call_frame_t *frame, xlator_t *this, loc_t *loc, size_t size,
+             dict_t *xdata)
 {
-        IOT_FOP (readlink, frame, this, loc, size, xdata);
-        return 0;
+    IOT_FOP(readlink, frame, this, loc, size, xdata);
+    return 0;
 }
 
-
 int
-iot_mknod (call_frame_t *frame, xlator_t *this, loc_t *loc, mode_t mode,
-           dev_t rdev, mode_t umask, dict_t *xdata)
+iot_mknod(call_frame_t *frame, xlator_t *this, loc_t *loc, mode_t mode,
+          dev_t rdev, mode_t umask, dict_t *xdata)
 {
-        IOT_FOP (mknod, frame, this, loc, mode, rdev, umask, xdata);
-        return 0;
+    IOT_FOP(mknod, frame, this, loc, mode, rdev, umask, xdata);
+    return 0;
 }
 
-
 int
-iot_mkdir (call_frame_t *frame, xlator_t *this, loc_t *loc, mode_t mode,
-           mode_t umask, dict_t *xdata)
+iot_mkdir(call_frame_t *frame, xlator_t *this, loc_t *loc, mode_t mode,
+          mode_t umask, dict_t *xdata)
 {
-        IOT_FOP (mkdir, frame, this, loc, mode, umask, xdata);
-        return 0;
+    IOT_FOP(mkdir, frame, this, loc, mode, umask, xdata);
+    return 0;
 }
 
-
 int
-iot_rmdir (call_frame_t *frame, xlator_t *this, loc_t *loc, int flags, dict_t *xdata)
+iot_rmdir(call_frame_t *frame, xlator_t *this, loc_t *loc, int flags,
+          dict_t *xdata)
 {
-        IOT_FOP (rmdir, frame, this, loc, flags, xdata);
-        return 0;
+    IOT_FOP(rmdir, frame, this, loc, flags, xdata);
+    return 0;
 }
 
-
 int
-iot_symlink (call_frame_t *frame, xlator_t *this, const char *linkname,
-             loc_t *loc, mode_t umask, dict_t *xdata)
+iot_symlink(call_frame_t *frame, xlator_t *this, const char *linkname,
+            loc_t *loc, mode_t umask, dict_t *xdata)
 {
-        IOT_FOP (symlink, frame, this, linkname, loc, umask, xdata);
-        return 0;
+    IOT_FOP(symlink, frame, this, linkname, loc, umask, xdata);
+    return 0;
 }
 
-
 int
-iot_rename (call_frame_t *frame, xlator_t *this, loc_t *oldloc, loc_t *newloc,
-            dict_t *xdata)
+iot_rename(call_frame_t *frame, xlator_t *this, loc_t *oldloc, loc_t *newloc,
+           dict_t *xdata)
 {
-        IOT_FOP (rename, frame, this, oldloc, newloc, xdata);
-        return 0;
+    IOT_FOP(rename, frame, this, oldloc, newloc, xdata);
+    return 0;
 }
 
-
 int
-iot_open (call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t flags,
-          fd_t *fd, dict_t *xdata)
+iot_open(call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t flags,
+         fd_t *fd, dict_t *xdata)
 {
-        IOT_FOP (open, frame, this, loc, flags, fd, xdata);
-        return 0;
+    IOT_FOP(open, frame, this, loc, flags, fd, xdata);
+    return 0;
 }
 
-
 int
-iot_create (call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t flags,
-            mode_t mode, mode_t umask, fd_t *fd, dict_t *xdata)
+iot_create(call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t flags,
+           mode_t mode, mode_t umask, fd_t *fd, dict_t *xdata)
 {
-        IOT_FOP (create, frame, this, loc, flags, mode, umask, fd, xdata);
-        return 0;
+    IOT_FOP(create, frame, this, loc, flags, mode, umask, fd, xdata);
+    return 0;
 }
 
-
 int
-iot_readv (call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
-           off_t offset, uint32_t flags, dict_t *xdata)
+iot_put(call_frame_t *frame, xlator_t *this, loc_t *loc, mode_t mode,
+        mode_t umask, uint32_t flags, struct iovec *vector, int32_t count,
+        off_t offset, struct iobref *iobref, dict_t *xattr, dict_t *xdata)
 {
-        IOT_FOP (readv, frame, this, fd, size, offset, flags, xdata);
-        return 0;
+    IOT_FOP(put, frame, this, loc, mode, umask, flags, vector, count, offset,
+            iobref, xattr, xdata);
+    return 0;
 }
 
+int
+iot_readv(call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
+          off_t offset, uint32_t flags, dict_t *xdata)
+{
+    IOT_FOP(readv, frame, this, fd, size, offset, flags, xdata);
+    return 0;
+}
 
 int
-iot_flush (call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *xdata)
+iot_flush(call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *xdata)
 {
-        IOT_FOP (flush, frame, this, fd, xdata);
-        return 0;
+    IOT_FOP(flush, frame, this, fd, xdata);
+    return 0;
 }
 
+int
+iot_fsync(call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t datasync,
+          dict_t *xdata)
+{
+    IOT_FOP(fsync, frame, this, fd, datasync, xdata);
+    return 0;
+}
 
 int
-iot_fsync (call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t datasync,
+iot_writev(call_frame_t *frame, xlator_t *this, fd_t *fd, struct iovec *vector,
+           int32_t count, off_t offset, uint32_t flags, struct iobref *iobref,
            dict_t *xdata)
 {
-        IOT_FOP (fsync, frame, this, fd, datasync, xdata);
-        return 0;
+    IOT_FOP(writev, frame, this, fd, vector, count, offset, flags, iobref,
+            xdata);
+    return 0;
 }
 
-
 int
-iot_writev (call_frame_t *frame, xlator_t *this, fd_t *fd,
-            struct iovec *vector, int32_t count, off_t offset,
-            uint32_t flags, struct iobref *iobref, dict_t *xdata)
+iot_lk(call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t cmd,
+       struct gf_flock *flock, dict_t *xdata)
 {
-        IOT_FOP (writev, frame, this, fd, vector, count, offset,
-                 flags, iobref, xdata);
-        return 0;
+    IOT_FOP(lk, frame, this, fd, cmd, flock, xdata);
+    return 0;
 }
 
-
 int
-iot_lk (call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t cmd,
-	struct gf_flock *flock, dict_t *xdata)
+iot_stat(call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xdata)
 {
-        IOT_FOP (lk, frame, this, fd, cmd, flock, xdata);
-        return 0;
+    IOT_FOP(stat, frame, this, loc, xdata);
+    return 0;
 }
 
-
 int
-iot_stat (call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xdata)
+iot_fstat(call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *xdata)
 {
-        IOT_FOP (stat, frame, this, loc, xdata);
-        return 0;
+    IOT_FOP(fstat, frame, this, fd, xdata);
+    return 0;
 }
 
-
 int
-iot_fstat (call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *xdata)
+iot_truncate(call_frame_t *frame, xlator_t *this, loc_t *loc, off_t offset,
+             dict_t *xdata)
 {
-        IOT_FOP (fstat, frame, this, fd, xdata);
-        return 0;
+    IOT_FOP(truncate, frame, this, loc, offset, xdata);
+    return 0;
 }
 
-
 int
-iot_truncate (call_frame_t *frame, xlator_t *this, loc_t *loc, off_t offset,
+iot_ftruncate(call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
               dict_t *xdata)
 {
-        IOT_FOP (truncate, frame, this, loc, offset, xdata);
-        return 0;
+    IOT_FOP(ftruncate, frame, this, fd, offset, xdata);
+    return 0;
 }
 
-
 int
-iot_ftruncate (call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
-               dict_t *xdata)
+iot_unlink(call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t xflag,
+           dict_t *xdata)
 {
-        IOT_FOP (ftruncate, frame, this, fd, offset, xdata);
-        return 0;
+    IOT_FOP(unlink, frame, this, loc, xflag, xdata);
+    return 0;
 }
 
-
-
 int
-iot_unlink (call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t xflag,
-            dict_t *xdata)
+iot_link(call_frame_t *frame, xlator_t *this, loc_t *oldloc, loc_t *newloc,
+         dict_t *xdata)
 {
-        IOT_FOP (unlink, frame, this, loc, xflag, xdata);
-        return 0;
+    IOT_FOP(link, frame, this, oldloc, newloc, xdata);
+    return 0;
 }
 
-
 int
-iot_link (call_frame_t *frame, xlator_t *this, loc_t *oldloc, loc_t *newloc,
-          dict_t *xdata)
+iot_opendir(call_frame_t *frame, xlator_t *this, loc_t *loc, fd_t *fd,
+            dict_t *xdata)
 {
-        IOT_FOP (link, frame, this, oldloc, newloc, xdata);
-        return 0;
+    IOT_FOP(opendir, frame, this, loc, fd, xdata);
+    return 0;
 }
 
-
 int
-iot_opendir (call_frame_t *frame, xlator_t *this, loc_t *loc, fd_t *fd,
+iot_fsyncdir(call_frame_t *frame, xlator_t *this, fd_t *fd, int datasync,
              dict_t *xdata)
 {
-        IOT_FOP (opendir, frame, this, loc, fd, xdata);
-        return 0;
+    IOT_FOP(fsyncdir, frame, this, fd, datasync, xdata);
+    return 0;
 }
 
-
 int
-iot_fsyncdir (call_frame_t *frame, xlator_t *this, fd_t *fd, int datasync,
-              dict_t *xdata)
+iot_statfs(call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xdata)
 {
-        IOT_FOP (fsyncdir, frame, this, fd, datasync, xdata);
-        return 0;
+    IOT_FOP(statfs, frame, this, loc, xdata);
+    return 0;
 }
 
-
 int
-iot_statfs (call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xdata)
+iot_setxattr(call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *dict,
+             int32_t flags, dict_t *xdata)
 {
-        IOT_FOP (statfs, frame, this, loc, xdata);
-        return 0;
+    IOT_FOP(setxattr, frame, this, loc, dict, flags, xdata);
+    return 0;
 }
 
-
 int
-iot_setxattr (call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *dict,
-              int32_t flags, dict_t *xdata)
+iot_getxattr(call_frame_t *frame, xlator_t *this, loc_t *loc, const char *name,
+             dict_t *xdata)
 {
-        IOT_FOP (setxattr, frame, this, loc, dict, flags, xdata);
+    iot_conf_t *conf = NULL;
+    dict_t *depths = NULL;
+    int i = 0;
+    int32_t op_ret = 0;
+    int32_t op_errno = 0;
+
+    conf = this->private;
+
+    if (name && strcmp(name, IO_THREADS_QUEUE_SIZE_KEY) == 0) {
+        /*
+         * We explicitly do not want a reference count
+         * for this dict in this translator
+         */
+        depths = dict_new();
+        if (!depths) {
+            op_ret = -1;
+            op_errno = ENOMEM;
+            goto unwind_special_getxattr;
+        }
+
+        for (i = 0; i < GF_FOP_PRI_MAX; i++) {
+            if (dict_set_int32(depths, (char *)fop_pri_to_string(i),
+                               conf->queue_sizes[i]) != 0) {
+                dict_unref(depths);
+                depths = NULL;
+                goto unwind_special_getxattr;
+            }
+        }
+
+    unwind_special_getxattr:
+        STACK_UNWIND_STRICT(getxattr, frame, op_ret, op_errno, depths, xdata);
+        if (depths)
+            dict_unref(depths);
         return 0;
-}
+    }
 
+    IOT_FOP(getxattr, frame, this, loc, name, xdata);
+    return 0;
+}
 
 int
-iot_getxattr (call_frame_t *frame, xlator_t *this, loc_t *loc,
-              const char *name, dict_t *xdata)
+iot_fgetxattr(call_frame_t *frame, xlator_t *this, fd_t *fd, const char *name,
+              dict_t *xdata)
 {
-        IOT_FOP (getxattr, frame, this, loc, name, xdata);
-        return 0;
+    IOT_FOP(fgetxattr, frame, this, fd, name, xdata);
+    return 0;
 }
 
-
 int
-iot_fgetxattr (call_frame_t *frame, xlator_t *this, fd_t *fd,
-               const char *name, dict_t *xdata)
+iot_fsetxattr(call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *dict,
+              int32_t flags, dict_t *xdata)
 {
-        IOT_FOP (fgetxattr, frame, this, fd, name, xdata);
-        return 0;
+    IOT_FOP(fsetxattr, frame, this, fd, dict, flags, xdata);
+    return 0;
 }
 
-
 int
-iot_fsetxattr (call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *dict,
-               int32_t flags, dict_t *xdata)
+iot_removexattr(call_frame_t *frame, xlator_t *this, loc_t *loc,
+                const char *name, dict_t *xdata)
 {
-        IOT_FOP (fsetxattr, frame, this, fd, dict, flags, xdata);
-        return 0;
+    IOT_FOP(removexattr, frame, this, loc, name, xdata);
+    return 0;
 }
 
-
 int
-iot_removexattr (call_frame_t *frame, xlator_t *this, loc_t *loc,
+iot_fremovexattr(call_frame_t *frame, xlator_t *this, fd_t *fd,
                  const char *name, dict_t *xdata)
 {
-        IOT_FOP (removexattr, frame, this, loc, name, xdata);
-        return 0;
+    IOT_FOP(fremovexattr, frame, this, fd, name, xdata);
+    return 0;
 }
 
 int
-iot_fremovexattr (call_frame_t *frame, xlator_t *this, fd_t *fd,
-                 const char *name, dict_t *xdata)
+iot_readdirp(call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
+             off_t offset, dict_t *xdata)
 {
-        IOT_FOP (fremovexattr, frame, this, fd, name, xdata);
-        return 0;
+    IOT_FOP(readdirp, frame, this, fd, size, offset, xdata);
+    return 0;
 }
 
-
 int
-iot_readdirp (call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
-              off_t offset, dict_t *xdata)
+iot_readdir(call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
+            off_t offset, dict_t *xdata)
 {
-        IOT_FOP (readdirp, frame, this, fd, size, offset, xdata);
-        return 0;
+    IOT_FOP(readdir, frame, this, fd, size, offset, xdata);
+    return 0;
 }
 
-
 int
-iot_readdir (call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
-             off_t offset, dict_t *xdata)
+iot_inodelk(call_frame_t *frame, xlator_t *this, const char *volume, loc_t *loc,
+            int32_t cmd, struct gf_flock *lock, dict_t *xdata)
 {
-        IOT_FOP (readdir, frame, this, fd, size, offset, xdata);
-        return 0;
+    IOT_FOP(inodelk, frame, this, volume, loc, cmd, lock, xdata);
+    return 0;
 }
 
 int
-iot_inodelk (call_frame_t *frame, xlator_t *this,
-             const char *volume, loc_t *loc, int32_t cmd, struct gf_flock *lock,
-             dict_t *xdata)
+iot_finodelk(call_frame_t *frame, xlator_t *this, const char *volume, fd_t *fd,
+             int32_t cmd, struct gf_flock *lock, dict_t *xdata)
 {
-        IOT_FOP (inodelk, frame, this, volume, loc, cmd, lock, xdata);
-        return 0;
+    IOT_FOP(finodelk, frame, this, volume, fd, cmd, lock, xdata);
+    return 0;
 }
 
 int
-iot_finodelk (call_frame_t *frame, xlator_t *this,
-              const char *volume, fd_t *fd, int32_t cmd, struct gf_flock *lock,
-              dict_t *xdata)
+iot_entrylk(call_frame_t *frame, xlator_t *this, const char *volume, loc_t *loc,
+            const char *basename, entrylk_cmd cmd, entrylk_type type,
+            dict_t *xdata)
 {
-        IOT_FOP (finodelk, frame, this, volume, fd, cmd, lock, xdata);
-        return 0;
+    IOT_FOP(entrylk, frame, this, volume, loc, basename, cmd, type, xdata);
+    return 0;
 }
 
 int
-iot_entrylk (call_frame_t *frame, xlator_t *this,
-             const char *volume, loc_t *loc, const char *basename,
-             entrylk_cmd cmd, entrylk_type type, dict_t *xdata)
+iot_fentrylk(call_frame_t *frame, xlator_t *this, const char *volume, fd_t *fd,
+             const char *basename, entrylk_cmd cmd, entrylk_type type,
+             dict_t *xdata)
 {
-        IOT_FOP (entrylk, frame, this, volume, loc, basename, cmd, type, xdata);
-        return 0;
+    IOT_FOP(fentrylk, frame, this, volume, fd, basename, cmd, type, xdata);
+    return 0;
 }
 
 int
-iot_fentrylk (call_frame_t *frame, xlator_t *this,
-              const char *volume, fd_t *fd, const char *basename,
-              entrylk_cmd cmd, entrylk_type type, dict_t *xdata)
+iot_xattrop(call_frame_t *frame, xlator_t *this, loc_t *loc,
+            gf_xattrop_flags_t optype, dict_t *xattr, dict_t *xdata)
 {
-        IOT_FOP (fentrylk, frame, this, volume, fd, basename, cmd, type, xdata);
-        return 0;
+    IOT_FOP(xattrop, frame, this, loc, optype, xattr, xdata);
+    return 0;
 }
 
-
 int
-iot_xattrop (call_frame_t *frame, xlator_t *this, loc_t *loc,
+iot_fxattrop(call_frame_t *frame, xlator_t *this, fd_t *fd,
              gf_xattrop_flags_t optype, dict_t *xattr, dict_t *xdata)
 {
-        IOT_FOP (xattrop, frame, this, loc, optype, xattr, xdata);
-        return 0;
+    IOT_FOP(fxattrop, frame, this, fd, optype, xattr, xdata);
+    return 0;
 }
 
+int32_t
+iot_rchecksum(call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
+              int32_t len, dict_t *xdata)
+{
+    IOT_FOP(rchecksum, frame, this, fd, offset, len, xdata);
+    return 0;
+}
 
 int
-iot_fxattrop (call_frame_t *frame, xlator_t *this, fd_t *fd,
-              gf_xattrop_flags_t optype, dict_t *xattr, dict_t *xdata)
+iot_fallocate(call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t mode,
+              off_t offset, size_t len, dict_t *xdata)
 {
-        IOT_FOP (fxattrop, frame, this, fd, optype, xattr, xdata);
-        return 0;
+    IOT_FOP(fallocate, frame, this, fd, mode, offset, len, xdata);
+    return 0;
 }
 
+int
+iot_discard(call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
+            size_t len, dict_t *xdata)
+{
+    IOT_FOP(discard, frame, this, fd, offset, len, xdata);
+    return 0;
+}
 
-int32_t
-iot_rchecksum (call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
-               int32_t len, dict_t *xdata)
+int
+iot_zerofill(call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
+             off_t len, dict_t *xdata)
 {
-        IOT_FOP (rchecksum, frame, this, fd, offset, len, xdata);
-        return 0;
+    IOT_FOP(zerofill, frame, this, fd, offset, len, xdata);
+    return 0;
 }
 
 int
-iot_fallocate(call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t mode,
-	      off_t offset, size_t len, dict_t *xdata)
+iot_seek(call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
+         gf_seek_what_t what, dict_t *xdata)
 {
-        IOT_FOP (fallocate, frame, this, fd, mode, offset, len, xdata);
-        return 0;
+    IOT_FOP(seek, frame, this, fd, offset, what, xdata);
+    return 0;
 }
 
 int
-iot_discard(call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
-	    size_t len, dict_t *xdata)
+iot_lease(call_frame_t *frame, xlator_t *this, loc_t *loc,
+          struct gf_lease *lease, dict_t *xdata)
 {
-        IOT_FOP (discard, frame, this, fd, offset, len, xdata);
-        return 0;
+    IOT_FOP(lease, frame, this, loc, lease, xdata);
+    return 0;
 }
 
 int
-iot_zerofill(call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
-            off_t len, dict_t *xdata)
+iot_getactivelk(call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xdata)
 {
-        IOT_FOP (zerofill, frame, this, fd, offset, len, xdata);
-        return 0;
+    IOT_FOP(getactivelk, frame, this, loc, xdata);
+    return 0;
 }
 
+int
+iot_setactivelk(call_frame_t *frame, xlator_t *this, loc_t *loc,
+                lock_migration_info_t *locklist, dict_t *xdata)
+{
+    IOT_FOP(setactivelk, frame, this, loc, locklist, xdata);
+    return 0;
+}
 
 int
-__iot_workers_scale (iot_conf_t *conf)
+__iot_workers_scale(iot_conf_t *conf)
 {
-        int       scale = 0;
-        int       diff = 0;
-        pthread_t thread;
-        int       ret = 0;
-        int       i = 0;
+    int scale = 0;
+    int diff = 0;
+    pthread_t thread;
+    int ret = 0;
+    int i = 0;
+
+    for (i = 0; i < GF_FOP_PRI_MAX; i++)
+        scale += min(conf->queue_sizes[i], conf->ac_iot_limit[i]);
+
+    if (scale < IOT_MIN_THREADS)
+        scale = IOT_MIN_THREADS;
+
+    if (scale > conf->max_count)
+        scale = conf->max_count;
+
+    if (conf->curr_count < scale) {
+        diff = scale - conf->curr_count;
+    }
+
+    while (diff) {
+        diff--;
+
+        ret = gf_thread_create(&thread, &conf->w_attr, iot_worker, conf,
+                               "iotwr%03hx", conf->curr_count & 0x3ff);
+        if (ret == 0) {
+            pthread_detach(thread);
+            conf->curr_count++;
+            gf_msg_debug(conf->this->name, 0,
+                         "scaled threads to %d (queue_size=%d/%d)",
+                         conf->curr_count, conf->queue_size, scale);
+        } else {
+            break;
+        }
+    }
 
-        for (i = 0; i < IOT_PRI_MAX; i++)
-                scale += min (conf->queue_sizes[i], conf->ac_iot_limit[i]);
+    return diff;
+}
 
-        if (scale < IOT_MIN_THREADS)
-                scale = IOT_MIN_THREADS;
+int
+iot_workers_scale(iot_conf_t *conf)
+{
+    int ret = -1;
 
-        if (scale > conf->max_count)
-                scale = conf->max_count;
+    if (conf == NULL) {
+        ret = -EINVAL;
+        goto out;
+    }
 
-        if (conf->curr_count < scale) {
-                diff = scale - conf->curr_count;
-        }
+    pthread_mutex_lock(&conf->mutex);
+    {
+        ret = __iot_workers_scale(conf);
+    }
+    pthread_mutex_unlock(&conf->mutex);
 
-        while (diff) {
-                diff --;
+out:
+    return ret;
+}
 
-                ret = gf_thread_create (&thread, &conf->w_attr, iot_worker, conf);
-                if (ret == 0) {
-                        conf->curr_count++;
-                        gf_log (conf->this->name, GF_LOG_DEBUG,
-                                "scaled threads to %d (queue_size=%d/%d)",
-                                conf->curr_count, conf->queue_size, scale);
-                } else {
-                        break;
-                }
+int
+set_stack_size(iot_conf_t *conf)
+{
+    int err = 0;
+    size_t stacksize = IOT_THREAD_STACK_SIZE;
+    xlator_t *this = NULL;
+
+    this = THIS;
+
+    err = pthread_attr_init(&conf->w_attr);
+    if (err != 0) {
+        gf_smsg(this->name, GF_LOG_ERROR, err, IO_THREADS_MSG_INIT_FAILED,
+                NULL);
+        return err;
+    }
+
+    err = pthread_attr_setstacksize(&conf->w_attr, stacksize);
+    if (err == EINVAL) {
+        err = pthread_attr_getstacksize(&conf->w_attr, &stacksize);
+        if (!err) {
+            gf_smsg(this->name, GF_LOG_WARNING, 0, IO_THREADS_MSG_SIZE_NOT_SET,
+                    "size=%zd", stacksize, NULL);
+        } else {
+            gf_smsg(this->name, GF_LOG_WARNING, 0, IO_THREADS_MSG_SIZE_NOT_SET,
+                    NULL);
+            err = 0;
         }
+    }
 
-        return diff;
+    conf->stack_size = stacksize;
+    return err;
 }
 
+int32_t
+mem_acct_init(xlator_t *this)
+{
+    int ret = -1;
+
+    if (!this)
+        return ret;
+
+    ret = xlator_mem_acct_init(this, gf_iot_mt_end + 1);
+
+    if (ret != 0) {
+        gf_smsg(this->name, GF_LOG_ERROR, ENOMEM, IO_THREADS_MSG_NO_MEMORY,
+                NULL);
+        return ret;
+    }
+
+    return ret;
+}
 
 int
-iot_workers_scale (iot_conf_t *conf)
+iot_priv_dump(xlator_t *this)
 {
-        int     ret = -1;
+    iot_conf_t *conf = NULL;
+    char key_prefix[GF_DUMP_MAX_BUF_LEN];
+    char key[GF_DUMP_MAX_BUF_LEN];
+    int i = 0;
 
-        if (conf == NULL) {
-                ret = -EINVAL;
-                goto out;
-        }
+    if (!this)
+        return 0;
 
-        pthread_mutex_lock (&conf->mutex);
-        {
-                ret = __iot_workers_scale (conf);
-        }
-        pthread_mutex_unlock (&conf->mutex);
+    conf = this->private;
+    if (!conf)
+        return 0;
 
-out:
-        return ret;
+    snprintf(key_prefix, GF_DUMP_MAX_BUF_LEN, "%s.%s", this->type, this->name);
+
+    gf_proc_dump_add_section("%s", key_prefix);
+
+    gf_proc_dump_write("maximum_threads_count", "%d", conf->max_count);
+    gf_proc_dump_write("current_threads_count", "%d", conf->curr_count);
+    gf_proc_dump_write("sleep_count", "%d", conf->sleep_count);
+    gf_proc_dump_write("idle_time", "%d", conf->idle_time);
+    gf_proc_dump_write("stack_size", "%zd", conf->stack_size);
+    gf_proc_dump_write("max_high_priority_threads", "%d",
+                       conf->ac_iot_limit[GF_FOP_PRI_HI]);
+    gf_proc_dump_write("max_normal_priority_threads", "%d",
+                       conf->ac_iot_limit[GF_FOP_PRI_NORMAL]);
+    gf_proc_dump_write("max_low_priority_threads", "%d",
+                       conf->ac_iot_limit[GF_FOP_PRI_LO]);
+    gf_proc_dump_write("max_least_priority_threads", "%d",
+                       conf->ac_iot_limit[GF_FOP_PRI_LEAST]);
+    gf_proc_dump_write("current_high_priority_threads", "%d",
+                       conf->ac_iot_count[GF_FOP_PRI_HI]);
+    gf_proc_dump_write("current_normal_priority_threads", "%d",
+                       conf->ac_iot_count[GF_FOP_PRI_NORMAL]);
+    gf_proc_dump_write("current_low_priority_threads", "%d",
+                       conf->ac_iot_count[GF_FOP_PRI_LO]);
+    gf_proc_dump_write("current_least_priority_threads", "%d",
+                       conf->ac_iot_count[GF_FOP_PRI_LEAST]);
+    for (i = 0; i < GF_FOP_PRI_MAX; i++) {
+        if (!conf->queue_sizes[i])
+            continue;
+        snprintf(key, sizeof(key), "%s_priority_queue_length",
+                 iot_get_pri_meaning(i));
+        gf_proc_dump_write(key, "%d", conf->queue_sizes[i]);
+    }
+
+    return 0;
 }
 
+/*
+ * We use a decay model to keep track and make sure we're not spawning new
+ * threads too often.  Each increment adds a large value to a counter, and that
+ * counter keeps ticking back down to zero over a fairly long period.  For
+ * example, let's use ONE_WEEK=604800 seconds, and we want to detect when we
+ * have N=3 increments during that time.  Thus, our threshold is
+ * (N-1)*ONE_WEEK.  To see how it works, look at three examples.
+ *
+ *   (a) Two events close together, then one more almost a week later.  The
+ *   first two events push our counter to 2*ONE_WEEK plus a bit.  At the third
+ *   event, we decay down to ONE_WEEK plus a bit and then add ONE_WEEK for the
+ *   new event, exceeding our threshold.
+ *
+ *   (b) One event, then two more almost a week later.  At the time of the
+ *   second and third events, the counter is already non-zero, so when we add
+ *   2*ONE_WEEK we exceed again.
+ *
+ *   (c) Three events, spaced three days apart.  At the time of the second
+ *   event, we decay down to approxitely ONE_WEEK*4/7 and then add another
+ *   ONE_WEEK.  At the third event, we decay again down to ONE_WEEK*8/7 and add
+ *   another ONE_WEEK, so boom.
+ *
+ * Note that in all three cases if that last event came a day later our counter
+ * would have decayed a bit more and we would *not* exceed our threshold.  It's
+ * not exactly the same as a precise "three in one week" limit, but it's very
+ * close and it allows the same kind of tweaking while requiring only constant
+ * space - no arrays of variable length N to allocate or maintain.  All we need
+ * (for each queue) is the value plus the time of the last update.
+ */
+
+typedef struct {
+    time_t update_time;
+    uint32_t value;
+} threshold_t;
+/*
+ * Variables so that I can hack these for testing.
+ * TBD: make these tunable?
+ */
+static uint32_t THRESH_SECONDS = 604800;
+static uint32_t THRESH_EVENTS = 3;
+static uint32_t THRESH_LIMIT = 1209600; /* SECONDS * (EVENTS-1) */
+
+static void
+iot_apply_event(xlator_t *this, threshold_t *thresh)
+{
+    time_t delta, now = gf_time();
+
+    /* Refresh for manual testing/debugging.  It's cheap. */
+    THRESH_LIMIT = THRESH_SECONDS * (THRESH_EVENTS - 1);
+
+    if (thresh->value && thresh->update_time) {
+        delta = now - thresh->update_time;
+        /* Be careful about underflow. */
+        if (thresh->value <= delta) {
+            thresh->value = 0;
+        } else {
+            thresh->value -= delta;
+        }
+    }
+
+    thresh->value += THRESH_SECONDS;
+    if (thresh->value >= THRESH_LIMIT) {
+        gf_log(this->name, GF_LOG_EMERG, "watchdog firing too often");
+        /*
+         * The default action for SIGTRAP is to dump core, but the fact
+         * that it's distinct from other signals we use means that
+         * there are other possibilities as well (e.g. drop into gdb or
+         * invoke a special handler).
+         */
+        kill(getpid(), SIGTRAP);
+    }
+
+    thresh->update_time = now;
+}
 
-void
-set_stack_size (iot_conf_t *conf)
-{
-        int     err = 0;
-        size_t  stacksize = IOT_THREAD_STACK_SIZE;
-        xlator_t *this = NULL;
-
-        this = THIS;
-
-        pthread_attr_init (&conf->w_attr);
-        err = pthread_attr_setstacksize (&conf->w_attr, stacksize);
-        if (err == EINVAL) {
-                err = pthread_attr_getstacksize (&conf->w_attr, &stacksize);
-                if (!err)
-                        gf_log (this->name, GF_LOG_WARNING,
-                                "Using default thread stack size %zd",
-                                stacksize);
-                else
-                        gf_log (this->name, GF_LOG_WARNING,
-                                "Using default thread stack size");
+static void *
+iot_watchdog(void *arg)
+{
+    xlator_t *this = arg;
+    iot_conf_t *priv = this->private;
+    int i;
+    int bad_times[GF_FOP_PRI_MAX] = {
+        0,
+    };
+    threshold_t thresholds[GF_FOP_PRI_MAX] = {{
+        0,
+    }};
+
+    for (;;) {
+        sleep(max(priv->watchdog_secs / 5, 1));
+        pthread_setcancelstate(PTHREAD_CANCEL_DISABLE, NULL);
+        pthread_mutex_lock(&priv->mutex);
+        for (i = 0; i < GF_FOP_PRI_MAX; ++i) {
+            if (priv->queue_marked[i]) {
+                if (++bad_times[i] >= 5) {
+                    gf_log(this->name, GF_LOG_WARNING, "queue %d stalled", i);
+                    iot_apply_event(this, &thresholds[i]);
+                    /*
+                     * We might not get here if the event
+                     * put us over our threshold.
+                     */
+                    ++(priv->ac_iot_limit[i]);
+                    bad_times[i] = 0;
+                }
+            } else {
+                bad_times[i] = 0;
+            }
+            priv->queue_marked[i] = (priv->queue_sizes[i] > 0);
         }
+        pthread_mutex_unlock(&priv->mutex);
+        pthread_setcancelstate(PTHREAD_CANCEL_ENABLE, NULL);
+    }
 
-        conf->stack_size = stacksize;
+    /* NOTREACHED */
+    return NULL;
 }
 
+static void
+start_iot_watchdog(xlator_t *this)
+{
+    iot_conf_t *priv = this->private;
+    int ret;
 
-int32_t
-mem_acct_init (xlator_t *this)
+    if (priv->watchdog_running) {
+        return;
+    }
+
+    ret = pthread_create(&priv->watchdog_thread, NULL, iot_watchdog, this);
+    if (ret == 0) {
+        priv->watchdog_running = _gf_true;
+    } else {
+        gf_log(this->name, GF_LOG_WARNING,
+               "pthread_create(iot_watchdog) failed");
+    }
+}
+
+static void
+stop_iot_watchdog(xlator_t *this)
 {
-        int     ret = -1;
+    iot_conf_t *priv = this->private;
 
-        if (!this)
-                return ret;
+    if (!priv->watchdog_running) {
+        return;
+    }
 
-        ret = xlator_mem_acct_init (this, gf_iot_mt_end + 1);
+    if (pthread_cancel(priv->watchdog_thread) != 0) {
+        gf_log(this->name, GF_LOG_WARNING,
+               "pthread_cancel(iot_watchdog) failed");
+    }
 
-        if (ret != 0) {
-                gf_log (this->name, GF_LOG_ERROR, "Memory accounting init"
-                                "failed");
-                return ret;
-        }
+    if (pthread_join(priv->watchdog_thread, NULL) != 0) {
+        gf_log(this->name, GF_LOG_WARNING, "pthread_join(iot_watchdog) failed");
+    }
 
-        return ret;
+    /* Failure probably means it's already dead. */
+    priv->watchdog_running = _gf_false;
 }
 
 int
-iot_priv_dump (xlator_t *this)
+reconfigure(xlator_t *this, dict_t *options)
 {
-        iot_conf_t     *conf   =   NULL;
-        char           key_prefix[GF_DUMP_MAX_BUF_LEN];
+    iot_conf_t *conf = NULL;
+    int ret = -1;
 
-        if (!this)
-                return 0;
+    conf = this->private;
+    if (!conf)
+        goto out;
 
-        conf = this->private;
-        if (!conf)
-                return 0;
+    GF_OPTION_RECONF("thread-count", conf->max_count, options, int32, out);
 
-        snprintf (key_prefix, GF_DUMP_MAX_BUF_LEN, "%s.%s", this->type,
-                  this->name);
+    GF_OPTION_RECONF("high-prio-threads", conf->ac_iot_limit[GF_FOP_PRI_HI],
+                     options, int32, out);
 
-        gf_proc_dump_add_section(key_prefix);
+    GF_OPTION_RECONF("normal-prio-threads",
+                     conf->ac_iot_limit[GF_FOP_PRI_NORMAL], options, int32,
+                     out);
 
-        gf_proc_dump_write("maximum_threads_count", "%d", conf->max_count);
-        gf_proc_dump_write("current_threads_count", "%d", conf->curr_count);
-        gf_proc_dump_write("sleep_count", "%d", conf->sleep_count);
-        gf_proc_dump_write("idle_time", "%d", conf->idle_time);
-        gf_proc_dump_write("stack_size", "%zd", conf->stack_size);
-        gf_proc_dump_write("high_priority_threads", "%d",
-                           conf->ac_iot_limit[IOT_PRI_HI]);
-        gf_proc_dump_write("normal_priority_threads", "%d",
-                           conf->ac_iot_limit[IOT_PRI_NORMAL]);
-        gf_proc_dump_write("low_priority_threads", "%d",
-                           conf->ac_iot_limit[IOT_PRI_LO]);
-        gf_proc_dump_write("least_priority_threads", "%d",
-                           conf->ac_iot_limit[IOT_PRI_LEAST]);
+    GF_OPTION_RECONF("low-prio-threads", conf->ac_iot_limit[GF_FOP_PRI_LO],
+                     options, int32, out);
 
-	gf_proc_dump_write("cached least rate", "%u",
-			   conf->throttle.cached_rate);
-	gf_proc_dump_write("least rate limit", "%u", conf->throttle.rate_limit);
+    GF_OPTION_RECONF("least-prio-threads", conf->ac_iot_limit[GF_FOP_PRI_LEAST],
+                     options, int32, out);
 
-        return 0;
+    GF_OPTION_RECONF("enable-least-priority", conf->least_priority, options,
+                     bool, out);
+
+    GF_OPTION_RECONF("cleanup-disconnected-reqs",
+                     conf->cleanup_disconnected_reqs, options, bool, out);
+
+    GF_OPTION_RECONF("watchdog-secs", conf->watchdog_secs, options, int32, out);
+
+    GF_OPTION_RECONF("pass-through", this->pass_through, options, bool, out);
+
+    if (conf->watchdog_secs > 0) {
+        start_iot_watchdog(this);
+    } else {
+        stop_iot_watchdog(this);
+    }
+
+    ret = 0;
+out:
+    return ret;
 }
 
 int
-reconfigure (xlator_t *this, dict_t *options)
+init(xlator_t *this)
 {
-	iot_conf_t      *conf = NULL;
-	int		 ret = -1;
+    iot_conf_t *conf = NULL;
+    int ret = -1;
+    int i = 0;
 
-        conf = this->private;
-        if (!conf)
-                goto out;
+    if (!this->children || this->children->next) {
+        gf_smsg("io-threads", GF_LOG_ERROR, 0,
+                IO_THREADS_MSG_XLATOR_CHILD_MISCONFIGURED, NULL);
+        goto out;
+    }
 
-        GF_OPTION_RECONF ("thread-count", conf->max_count, options, int32, out);
+    if (!this->parents) {
+        gf_smsg(this->name, GF_LOG_WARNING, 0, IO_THREADS_MSG_VOL_MISCONFIGURED,
+                NULL);
+    }
 
-        GF_OPTION_RECONF ("high-prio-threads",
-                          conf->ac_iot_limit[IOT_PRI_HI], options, int32, out);
+    conf = (void *)GF_CALLOC(1, sizeof(*conf), gf_iot_mt_iot_conf_t);
+    if (conf == NULL) {
+        gf_smsg(this->name, GF_LOG_ERROR, ENOMEM, IO_THREADS_MSG_OUT_OF_MEMORY,
+                NULL);
+        goto out;
+    }
 
-        GF_OPTION_RECONF ("normal-prio-threads",
-                          conf->ac_iot_limit[IOT_PRI_NORMAL], options, int32,
-                          out);
+    if ((ret = pthread_cond_init(&conf->cond, NULL)) != 0) {
+        gf_smsg(this->name, GF_LOG_ERROR, 0, IO_THREADS_MSG_PTHREAD_INIT_FAILED,
+                "pthread_cond_init ret=%d", ret, NULL);
+        goto out;
+    }
+    conf->cond_inited = _gf_true;
 
-        GF_OPTION_RECONF ("low-prio-threads",
-                          conf->ac_iot_limit[IOT_PRI_LO], options, int32, out);
+    if ((ret = pthread_mutex_init(&conf->mutex, NULL)) != 0) {
+        gf_smsg(this->name, GF_LOG_ERROR, 0, IO_THREADS_MSG_PTHREAD_INIT_FAILED,
+                "pthread_mutex_init ret=%d", ret, NULL);
+        goto out;
+    }
+    conf->mutex_inited = _gf_true;
 
-        GF_OPTION_RECONF ("least-prio-threads",
-                          conf->ac_iot_limit[IOT_PRI_LEAST], options, int32,
-                          out);
-        GF_OPTION_RECONF ("enable-least-priority", conf->least_priority,
-                          options, bool, out);
+    ret = set_stack_size(conf);
 
-	GF_OPTION_RECONF("least-rate-limit", conf->throttle.rate_limit, options,
-			 int32, out);
+    if (ret != 0)
+        goto out;
 
-	ret = 0;
-out:
-	return ret;
-}
+    ret = -1;
 
+    GF_OPTION_INIT("thread-count", conf->max_count, int32, out);
 
-int
-init (xlator_t *this)
-{
-        iot_conf_t *conf = NULL;
-        int         ret  = -1;
-        int         i    = 0;
-
-	if (!this->children || this->children->next) {
-		gf_log ("io-threads", GF_LOG_ERROR,
-			"FATAL: iot not configured with exactly one child");
-                goto out;
-	}
-
-	if (!this->parents) {
-		gf_log (this->name, GF_LOG_WARNING,
-			"dangling volume. check volfile ");
-	}
-
-	conf = (void *) GF_CALLOC (1, sizeof (*conf),
-                                   gf_iot_mt_iot_conf_t);
-        if (conf == NULL) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "out of memory");
-                goto out;
-        }
+    GF_OPTION_INIT("high-prio-threads", conf->ac_iot_limit[GF_FOP_PRI_HI],
+                   int32, out);
 
-        if ((ret = pthread_cond_init(&conf->cond, NULL)) != 0) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "pthread_cond_init failed (%d)", ret);
-                goto out;
-        }
+    GF_OPTION_INIT("normal-prio-threads", conf->ac_iot_limit[GF_FOP_PRI_NORMAL],
+                   int32, out);
 
-        if ((ret = pthread_mutex_init(&conf->mutex, NULL)) != 0) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "pthread_mutex_init failed (%d)", ret);
-                goto out;
-        }
+    GF_OPTION_INIT("low-prio-threads", conf->ac_iot_limit[GF_FOP_PRI_LO], int32,
+                   out);
 
-        set_stack_size (conf);
+    GF_OPTION_INIT("least-prio-threads", conf->ac_iot_limit[GF_FOP_PRI_LEAST],
+                   int32, out);
 
-        GF_OPTION_INIT ("thread-count", conf->max_count, int32, out);
+    GF_OPTION_INIT("idle-time", conf->idle_time, int32, out);
 
-        GF_OPTION_INIT ("high-prio-threads",
-                        conf->ac_iot_limit[IOT_PRI_HI], int32, out);
+    GF_OPTION_INIT("enable-least-priority", conf->least_priority, bool, out);
 
-        GF_OPTION_INIT ("normal-prio-threads",
-                        conf->ac_iot_limit[IOT_PRI_NORMAL], int32, out);
+    GF_OPTION_INIT("cleanup-disconnected-reqs", conf->cleanup_disconnected_reqs,
+                   bool, out);
 
-        GF_OPTION_INIT ("low-prio-threads",
-                        conf->ac_iot_limit[IOT_PRI_LO], int32, out);
+    GF_OPTION_INIT("pass-through", this->pass_through, bool, out);
 
-        GF_OPTION_INIT ("least-prio-threads",
-                        conf->ac_iot_limit[IOT_PRI_LEAST], int32, out);
+    conf->this = this;
+    GF_ATOMIC_INIT(conf->stub_cnt, 0);
 
-        GF_OPTION_INIT ("idle-time", conf->idle_time, int32, out);
-        GF_OPTION_INIT ("enable-least-priority", conf->least_priority,
-                        bool, out);
+    for (i = 0; i < GF_FOP_PRI_MAX; i++) {
+        INIT_LIST_HEAD(&conf->clients[i]);
+        INIT_LIST_HEAD(&conf->no_client[i].clients);
+        INIT_LIST_HEAD(&conf->no_client[i].reqs);
+    }
 
-	GF_OPTION_INIT("least-rate-limit", conf->throttle.rate_limit, int32,
-		       out);
-        if ((ret = pthread_mutex_init(&conf->throttle.lock, NULL)) != 0) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "pthread_mutex_init failed (%d)", ret);
-                goto out;
-        }
-
-        conf->this = this;
+    if (!this->pass_through) {
+        ret = iot_workers_scale(conf);
 
-        for (i = 0; i < IOT_PRI_MAX; i++) {
-                INIT_LIST_HEAD (&conf->reqs[i]);
+        if (ret == -1) {
+            gf_smsg(this->name, GF_LOG_ERROR, 0,
+                    IO_THREADS_MSG_WORKER_THREAD_INIT_FAILED, NULL);
+            goto out;
         }
+    }
 
-	ret = iot_workers_scale (conf);
+    this->private = conf;
 
-        if (ret == -1) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "cannot initialize worker threads, exiting init");
-                goto out;
-        }
+    conf->watchdog_secs = 0;
+    GF_OPTION_INIT("watchdog-secs", conf->watchdog_secs, int32, out);
+    if (conf->watchdog_secs > 0) {
+        start_iot_watchdog(this);
+    }
 
-	this->private = conf;
-        ret = 0;
+    ret = 0;
 out:
-        if (ret)
-                GF_FREE (conf);
+    if (ret)
+        GF_FREE(conf);
 
-	return ret;
+    return ret;
 }
 
+static void
+iot_exit_threads(iot_conf_t *conf)
+{
+    pthread_mutex_lock(&conf->mutex);
+    {
+        conf->down = _gf_true;
+        /*Let all the threads know that xl is going down*/
+        pthread_cond_broadcast(&conf->cond);
+        while (conf->curr_count) /*Wait for threads to exit*/
+            pthread_cond_wait(&conf->cond, &conf->mutex);
+    }
+    pthread_mutex_unlock(&conf->mutex);
+}
+
+int
+notify(xlator_t *this, int32_t event, void *data, ...)
+{
+    iot_conf_t *conf = this->private;
+    xlator_t *victim = data;
+    uint64_t stub_cnt = 0;
+    struct timespec sleep_till = {
+        0,
+    };
+
+    if (GF_EVENT_PARENT_DOWN == event) {
+        if (victim->cleanup_starting) {
+            /* Wait for draining stub from queue before notify PARENT_DOWN */
+            stub_cnt = GF_ATOMIC_GET(conf->stub_cnt);
+            if (stub_cnt) {
+                timespec_now_realtime(&sleep_till);
+                sleep_till.tv_sec += 1;
+                pthread_mutex_lock(&conf->mutex);
+                {
+                    while (stub_cnt) {
+                        (void)pthread_cond_timedwait(&conf->cond, &conf->mutex,
+                                                     &sleep_till);
+                        stub_cnt = GF_ATOMIC_GET(conf->stub_cnt);
+                    }
+                }
+                pthread_mutex_unlock(&conf->mutex);
+            }
+
+            gf_log(this->name, GF_LOG_INFO,
+                   "Notify GF_EVENT_PARENT_DOWN for brick %s", victim->name);
+        } else {
+            iot_exit_threads(conf);
+        }
+    }
+
+    if (GF_EVENT_CHILD_DOWN == event) {
+        if (victim->cleanup_starting) {
+            iot_exit_threads(conf);
+            gf_log(this->name, GF_LOG_INFO,
+                   "Notify GF_EVENT_CHILD_DOWN for brick %s", victim->name);
+        }
+    }
+
+    default_notify(this, event, data);
+
+    return 0;
+}
 
 void
-fini (xlator_t *this)
+fini(xlator_t *this)
 {
-	iot_conf_t *conf = this->private;
+    iot_conf_t *conf = this->private;
+
+    if (!conf)
+        return;
+
+    if (conf->mutex_inited && conf->cond_inited)
+        iot_exit_threads(conf);
+
+    if (conf->cond_inited)
+        pthread_cond_destroy(&conf->cond);
+
+    if (conf->mutex_inited)
+        pthread_mutex_destroy(&conf->mutex);
+
+    stop_iot_watchdog(this);
 
-	GF_FREE (conf);
+    GF_FREE(conf);
 
-	this->private = NULL;
-	return;
+    this->private = NULL;
+    return;
+}
+
+int
+iot_client_destroy(xlator_t *this, client_t *client)
+{
+    void *tmp = NULL;
+
+    if (client_ctx_del(client, this, &tmp) == 0) {
+        GF_FREE(tmp);
+    }
+
+    return 0;
+}
+
+static int
+iot_disconnect_cbk(xlator_t *this, client_t *client)
+{
+    int i;
+    call_stub_t *curr;
+    call_stub_t *next;
+    iot_conf_t *conf = this->private;
+    iot_client_ctx_t *ctx;
+
+    if (!conf || !conf->cleanup_disconnected_reqs) {
+        goto out;
+    }
+
+    pthread_mutex_lock(&conf->mutex);
+    for (i = 0; i < GF_FOP_PRI_MAX; i++) {
+        ctx = &conf->no_client[i];
+        list_for_each_entry_safe(curr, next, &ctx->reqs, list)
+        {
+            if (curr->frame->root->client != client) {
+                continue;
+            }
+            gf_log(this->name, GF_LOG_INFO,
+                   "poisoning %s fop at %p for client %s",
+                   gf_fop_list[curr->fop], curr, client->client_uid);
+            curr->poison = _gf_true;
+        }
+    }
+    pthread_mutex_unlock(&conf->mutex);
+
+out:
+    return 0;
 }
 
 struct xlator_dumpops dumpops = {
-        .priv    = iot_priv_dump,
+    .priv = iot_priv_dump,
 };
 
 struct xlator_fops fops = {
-	.open        = iot_open,
-	.create      = iot_create,
-	.readv       = iot_readv,
-	.writev      = iot_writev,
-	.flush       = iot_flush,
-	.fsync       = iot_fsync,
-	.lk          = iot_lk,
-	.stat        = iot_stat,
-	.fstat       = iot_fstat,
-	.truncate    = iot_truncate,
-	.ftruncate   = iot_ftruncate,
-	.unlink      = iot_unlink,
-        .lookup      = iot_lookup,
-        .setattr     = iot_setattr,
-        .fsetattr    = iot_fsetattr,
-        .access      = iot_access,
-        .readlink    = iot_readlink,
-        .mknod       = iot_mknod,
-        .mkdir       = iot_mkdir,
-        .rmdir       = iot_rmdir,
-        .symlink     = iot_symlink,
-        .rename      = iot_rename,
-        .link        = iot_link,
-        .opendir     = iot_opendir,
-        .fsyncdir    = iot_fsyncdir,
-        .statfs      = iot_statfs,
-        .setxattr    = iot_setxattr,
-        .getxattr    = iot_getxattr,
-        .fgetxattr   = iot_fgetxattr,
-        .fsetxattr   = iot_fsetxattr,
-        .removexattr = iot_removexattr,
-        .fremovexattr = iot_fremovexattr,
-        .readdir     = iot_readdir,
-        .readdirp    = iot_readdirp,
-        .inodelk     = iot_inodelk,
-        .finodelk    = iot_finodelk,
-        .entrylk     = iot_entrylk,
-        .fentrylk    = iot_fentrylk,
-        .xattrop     = iot_xattrop,
-	.fxattrop    = iot_fxattrop,
-        .rchecksum   = iot_rchecksum,
-	.fallocate   = iot_fallocate,
-	.discard     = iot_discard,
-        .zerofill    = iot_zerofill,
+    .open = iot_open,
+    .create = iot_create,
+    .readv = iot_readv,
+    .writev = iot_writev,
+    .flush = iot_flush,
+    .fsync = iot_fsync,
+    .lk = iot_lk,
+    .stat = iot_stat,
+    .fstat = iot_fstat,
+    .truncate = iot_truncate,
+    .ftruncate = iot_ftruncate,
+    .unlink = iot_unlink,
+    .lookup = iot_lookup,
+    .setattr = iot_setattr,
+    .fsetattr = iot_fsetattr,
+    .access = iot_access,
+    .readlink = iot_readlink,
+    .mknod = iot_mknod,
+    .mkdir = iot_mkdir,
+    .rmdir = iot_rmdir,
+    .symlink = iot_symlink,
+    .rename = iot_rename,
+    .link = iot_link,
+    .opendir = iot_opendir,
+    .fsyncdir = iot_fsyncdir,
+    .statfs = iot_statfs,
+    .setxattr = iot_setxattr,
+    .getxattr = iot_getxattr,
+    .fgetxattr = iot_fgetxattr,
+    .fsetxattr = iot_fsetxattr,
+    .removexattr = iot_removexattr,
+    .fremovexattr = iot_fremovexattr,
+    .readdir = iot_readdir,
+    .readdirp = iot_readdirp,
+    .inodelk = iot_inodelk,
+    .finodelk = iot_finodelk,
+    .entrylk = iot_entrylk,
+    .fentrylk = iot_fentrylk,
+    .xattrop = iot_xattrop,
+    .fxattrop = iot_fxattrop,
+    .rchecksum = iot_rchecksum,
+    .fallocate = iot_fallocate,
+    .discard = iot_discard,
+    .zerofill = iot_zerofill,
+    .seek = iot_seek,
+    .lease = iot_lease,
+    .getactivelk = iot_getactivelk,
+    .setactivelk = iot_setactivelk,
+    .put = iot_put,
 };
 
-struct xlator_cbks cbks;
+struct xlator_cbks cbks = {
+    .client_destroy = iot_client_destroy,
+    .client_disconnect = iot_disconnect_cbk,
+};
 
 struct volume_options options[] = {
-	{ .key  = {"thread-count"},
-	  .type = GF_OPTION_TYPE_INT,
-	  .min  = IOT_MIN_THREADS,
-	  .max  = IOT_MAX_THREADS,
-          .default_value = "16",
-          .description = "Number of threads in IO threads translator which "
-                         "perform concurrent IO operations"
-
-	},
-	{ .key  = {"high-prio-threads"},
-	  .type = GF_OPTION_TYPE_INT,
-	  .min  = IOT_MIN_THREADS,
-	  .max  = IOT_MAX_THREADS,
-          .default_value = "16",
-          .description = "Max number of threads in IO threads translator which "
-                         "perform high priority IO operations at a given time"
-
-	},
-	{ .key  = {"normal-prio-threads"},
-	  .type = GF_OPTION_TYPE_INT,
-	  .min  = IOT_MIN_THREADS,
-	  .max  = IOT_MAX_THREADS,
-          .default_value = "16",
-          .description = "Max number of threads in IO threads translator which "
-                         "perform normal priority IO operations at a given time"
-
-	},
-	{ .key  = {"low-prio-threads"},
-	  .type = GF_OPTION_TYPE_INT,
-	  .min  = IOT_MIN_THREADS,
-	  .max  = IOT_MAX_THREADS,
-          .default_value = "16",
-          .description = "Max number of threads in IO threads translator which "
-                         "perform low priority IO operations at a given time"
-
-	},
-	{ .key  = {"least-prio-threads"},
-	  .type = GF_OPTION_TYPE_INT,
-	  .min  = IOT_MIN_THREADS,
-	  .max  = IOT_MAX_THREADS,
-          .default_value = "1",
-          .description = "Max number of threads in IO threads translator which "
-                         "perform least priority IO operations at a given time"
-	},
-        { .key  = {"enable-least-priority"},
-          .type = GF_OPTION_TYPE_BOOL,
-          .default_value = "on",
-          .description = "Enable/Disable least priority"
-        },
-        {.key   = {"idle-time"},
-         .type  = GF_OPTION_TYPE_INT,
-         .min   = 1,
-         .max   = 0x7fffffff,
-         .default_value = "120",
-        },
-	{.key	= {"least-rate-limit"},
-	 .type	= GF_OPTION_TYPE_INT,
-	 .min	= 0,
-         .max	= INT_MAX,
-	 .default_value = "0",
-	 .description = "Max number of least priority operations to handle "
-			"per-second"
-	},
-	{ .key  = {NULL},
-        },
+    {.key = {"thread-count"},
+     .type = GF_OPTION_TYPE_INT,
+     .min = IOT_MIN_THREADS,
+     .max = IOT_MAX_THREADS,
+     .default_value = "16",
+     .op_version = {1},
+     .flags = OPT_FLAG_SETTABLE | OPT_FLAG_DOC,
+     .tags = {"io-threads"},
+     /*.option = "thread-count"*/
+     .description = "Number of threads in IO threads translator which "
+                    "perform concurrent IO operations"
+
+    },
+    {.key = {"high-prio-threads"},
+     .type = GF_OPTION_TYPE_INT,
+     .min = IOT_MIN_THREADS,
+     .max = IOT_MAX_THREADS,
+     .default_value = "16",
+     .op_version = {1},
+     .flags = OPT_FLAG_SETTABLE | OPT_FLAG_DOC,
+     .tags = {"io-threads"},
+     .description = "Max number of threads in IO threads translator which "
+                    "perform high priority IO operations at a given time"
+
+    },
+    {.key = {"normal-prio-threads"},
+     .type = GF_OPTION_TYPE_INT,
+     .min = IOT_MIN_THREADS,
+     .max = IOT_MAX_THREADS,
+     .default_value = "16",
+     .op_version = {1},
+     .flags = OPT_FLAG_SETTABLE | OPT_FLAG_DOC,
+     .tags = {"io-threads"},
+     .description = "Max number of threads in IO threads translator which "
+                    "perform normal priority IO operations at a given time"
+
+    },
+    {.key = {"low-prio-threads"},
+     .type = GF_OPTION_TYPE_INT,
+     .min = IOT_MIN_THREADS,
+     .max = IOT_MAX_THREADS,
+     .default_value = "16",
+     .op_version = {1},
+     .flags = OPT_FLAG_SETTABLE | OPT_FLAG_DOC,
+     .tags = {"io-threads"},
+     .description = "Max number of threads in IO threads translator which "
+                    "perform low priority IO operations at a given time"
+
+    },
+    {.key = {"least-prio-threads"},
+     .type = GF_OPTION_TYPE_INT,
+     .min = IOT_MIN_THREADS,
+     .max = IOT_MAX_THREADS,
+     .default_value = "1",
+     .op_version = {1},
+     .flags = OPT_FLAG_SETTABLE | OPT_FLAG_DOC,
+     .tags = {"io-threads"},
+     .description = "Max number of threads in IO threads translator which "
+                    "perform least priority IO operations at a given time"},
+    {.key = {"enable-least-priority"},
+     .type = GF_OPTION_TYPE_BOOL,
+     .default_value = SITE_H_ENABLE_LEAST_PRIORITY,
+     .op_version = {1},
+     .flags = OPT_FLAG_SETTABLE | OPT_FLAG_DOC,
+     .tags = {"io-threads"},
+     .description = "Enable/Disable least priority"},
+    {
+        .key = {"idle-time"},
+        .type = GF_OPTION_TYPE_INT,
+        .min = 1,
+        .max = 0x7fffffff,
+        .default_value = "120",
+    },
+    {.key = {"watchdog-secs"},
+     .type = GF_OPTION_TYPE_INT,
+     .min = 0,
+     .default_value = 0,
+     .op_version = {GD_OP_VERSION_4_1_0},
+     .flags = OPT_FLAG_SETTABLE | OPT_FLAG_DOC,
+     .tags = {"io-threads"},
+     .description = "Number of seconds a queue must be stalled before "
+                    "starting an 'emergency' thread."},
+    {.key = {"cleanup-disconnected-reqs"},
+     .type = GF_OPTION_TYPE_BOOL,
+     .default_value = "off",
+     .op_version = {GD_OP_VERSION_4_1_0},
+     .flags = OPT_FLAG_SETTABLE | OPT_FLAG_DOC | OPT_FLAG_CLIENT_OPT,
+     .tags = {"io-threads"},
+     .description = "'Poison' queued requests when a client disconnects"},
+    {.key = {"pass-through"},
+     .type = GF_OPTION_TYPE_BOOL,
+     .default_value = "false",
+     .op_version = {GD_OP_VERSION_4_1_0},
+     .flags = OPT_FLAG_SETTABLE | OPT_FLAG_DOC | OPT_FLAG_CLIENT_OPT,
+     .tags = {"io-threads"},
+     .description = "Enable/Disable io threads translator"},
+    {
+        .key = {NULL},
+    },
+};
+
+xlator_api_t xlator_api = {
+    .init = init,
+    .fini = fini,
+    .notify = notify,
+    .reconfigure = reconfigure,
+    .mem_acct_init = mem_acct_init,
+    .op_version = {1}, /* Present from the initial version */
+    .dumpops = &dumpops,
+    .fops = &fops,
+    .cbks = &cbks,
+    .options = options,
+    .identifier = "io-threads",
+    .category = GF_MAINTAINED,
 };
diff --git a/xlators/performance/io-threads/src/io-threads.h b/xlators/performance/io-threads/src/io-threads.h
index 1a9dee9ae2c..f54d2f4912d 100644
--- a/xlators/performance/io-threads/src/io-threads.h
+++ b/xlators/performance/io-threads/src/io-threads.h
@@ -11,80 +11,74 @@
 #ifndef __IOT_H
 #define __IOT_H
 
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
-
-
-#include "compat-errno.h"
-#include "glusterfs.h"
-#include "logging.h"
-#include "dict.h"
-#include "xlator.h"
-#include "common-utils.h"
-#include "list.h"
+#include <glusterfs/compat-errno.h>
+#include <glusterfs/glusterfs.h>
+#include <glusterfs/logging.h>
+#include <glusterfs/dict.h>
+#include <glusterfs/xlator.h>
+#include <glusterfs/common-utils.h>
+#include <glusterfs/list.h>
 #include <stdlib.h>
-#include "locking.h"
+#include <glusterfs/locking.h>
 #include "iot-mem-types.h"
 #include <semaphore.h>
-#include "statedump.h"
-
+#include <glusterfs/statedump.h>
 
 struct iot_conf;
 
-#define MAX_IDLE_SKEW                   4       /* In secs */
-#define skew_sec_idle_time(sec)         ((sec) + (random () % MAX_IDLE_SKEW))
-#define IOT_DEFAULT_IDLE                120     /* In secs. */
-
-#define IOT_MIN_THREADS         1
-#define IOT_DEFAULT_THREADS     16
-#define IOT_MAX_THREADS         64
-
+#define MAX_IDLE_SKEW 4 /* In secs */
+#define skew_sec_idle_time(sec) ((sec) + (random() % MAX_IDLE_SKEW))
+#define IOT_DEFAULT_IDLE 120 /* In secs. */
 
-#define IOT_THREAD_STACK_SIZE   ((size_t)(1024*1024))
+#define IOT_MIN_THREADS 1
+#define IOT_DEFAULT_THREADS 16
+#define IOT_MAX_THREADS 64
 
+#define IOT_THREAD_STACK_SIZE ((size_t)(256 * 1024))
 
-typedef enum {
-        IOT_PRI_HI = 0, /* low latency */
-        IOT_PRI_NORMAL, /* normal */
-        IOT_PRI_LO,     /* bulk */
-        IOT_PRI_LEAST,  /* least */
-        IOT_PRI_MAX,
-} iot_pri_t;
-
-#define IOT_LEAST_THROTTLE_DELAY 1	/* sample interval in seconds */
-struct iot_least_throttle {
-	struct timeval	sample_time;	/* timestamp of current sample */
-	uint32_t	sample_cnt;	/* sample count for active interval */
-	uint32_t	cached_rate;	/* the most recently measured rate */
-	int32_t		rate_limit;	/* user-specified rate limit */
-	pthread_mutex_t	lock;
-};
+typedef struct {
+    struct list_head clients;
+    struct list_head reqs;
+} iot_client_ctx_t;
 
 struct iot_conf {
-        pthread_mutex_t      mutex;
-        pthread_cond_t       cond;
-
-        int32_t              max_count;   /* configured maximum */
-        int32_t              curr_count;  /* actual number of threads running */
-        int32_t              sleep_count;
-
-        int32_t              idle_time;   /* in seconds */
-
-        struct list_head     reqs[IOT_PRI_MAX];
-
-        int32_t              ac_iot_limit[IOT_PRI_MAX];
-        int32_t              ac_iot_count[IOT_PRI_MAX];
-        int                  queue_sizes[IOT_PRI_MAX];
-        int                  queue_size;
-        pthread_attr_t       w_attr;
-        gf_boolean_t         least_priority; /*Enable/Disable least-priority */
-
-        xlator_t            *this;
-        size_t              stack_size;
-
-	struct iot_least_throttle throttle;
+    pthread_mutex_t mutex;
+    pthread_cond_t cond;
+
+    int32_t max_count;  /* configured maximum */
+    int32_t curr_count; /* actual number of threads running */
+    int32_t sleep_count;
+
+    int32_t idle_time; /* in seconds */
+
+    struct list_head clients[GF_FOP_PRI_MAX];
+    /*
+     * It turns out that there are several ways a frame can get to us
+     * without having an associated client (server_first_lookup was the
+     * first one I hit).  Instead of trying to update all such callers,
+     * we use this to queue them.
+     */
+    iot_client_ctx_t no_client[GF_FOP_PRI_MAX];
+
+    int32_t ac_iot_limit[GF_FOP_PRI_MAX];
+    int32_t ac_iot_count[GF_FOP_PRI_MAX];
+    int queue_sizes[GF_FOP_PRI_MAX];
+    int32_t queue_size;
+    gf_atomic_t stub_cnt;
+    pthread_attr_t w_attr;
+    gf_boolean_t least_priority; /*Enable/Disable least-priority */
+
+    xlator_t *this;
+    size_t stack_size;
+    gf_boolean_t down; /*PARENT_DOWN event is notified*/
+    gf_boolean_t mutex_inited;
+    gf_boolean_t cond_inited;
+
+    int32_t watchdog_secs;
+    gf_boolean_t watchdog_running;
+    pthread_t watchdog_thread;
+    gf_boolean_t queue_marked[GF_FOP_PRI_MAX];
+    gf_boolean_t cleanup_disconnected_reqs;
 };
 
 typedef struct iot_conf iot_conf_t;
diff --git a/xlators/performance/io-threads/src/iot-mem-types.h b/xlators/performance/io-threads/src/iot-mem-types.h
index 4fa8302d1f4..29565f34dd4 100644
--- a/xlators/performance/io-threads/src/iot-mem-types.h
+++ b/xlators/performance/io-threads/src/iot-mem-types.h
@@ -8,15 +8,14 @@
   cases as published by the Free Software Foundation.
 */
 
-
 #ifndef __IOT_MEM_TYPES_H__
 #define __IOT_MEM_TYPES_H__
 
-#include "mem-types.h"
+#include <glusterfs/mem-types.h>
 
 enum gf_iot_mem_types_ {
-        gf_iot_mt_iot_conf_t  = gf_common_mt_end + 1,
-        gf_iot_mt_end
+    gf_iot_mt_iot_conf_t = gf_common_mt_end + 1,
+    gf_iot_mt_client_ctx_t,
+    gf_iot_mt_end
 };
 #endif
-
diff --git a/xlators/performance/md-cache/src/Makefile.am b/xlators/performance/md-cache/src/Makefile.am
index 6f3d68be1c4..447ff0f30f0 100644
--- a/xlators/performance/md-cache/src/Makefile.am
+++ b/xlators/performance/md-cache/src/Makefile.am
@@ -1,14 +1,15 @@
 xlator_LTLIBRARIES = md-cache.la
 xlatordir = $(libdir)/glusterfs/$(PACKAGE_VERSION)/xlator/performance
 
-md_cache_la_LDFLAGS = -module -avoid-version
+md_cache_la_LDFLAGS = -module $(GF_XLATOR_DEFAULT_LDFLAGS)
 
 md_cache_la_SOURCES = md-cache.c
 md_cache_la_LIBADD = $(top_builddir)/libglusterfs/src/libglusterfs.la
 
-noinst_HEADERS = md-cache-mem-types.h
+noinst_HEADERS = md-cache-mem-types.h md-cache-messages.h
 
 AM_CPPFLAGS = $(GF_CPPFLAGS) -I$(top_srcdir)/libglusterfs/src \
+	-I$(top_srcdir)/rpc/xdr/src -I$(top_builddir)/rpc/xdr/src \
 	-I$(CONTRIBDIR)/rbtree
 
 AM_CFLAGS = -Wall $(GF_CFLAGS)
diff --git a/xlators/performance/md-cache/src/md-cache-mem-types.h b/xlators/performance/md-cache/src/md-cache-mem-types.h
index 6634cf962a5..47a07005717 100644
--- a/xlators/performance/md-cache/src/md-cache-mem-types.h
+++ b/xlators/performance/md-cache/src/md-cache-mem-types.h
@@ -8,17 +8,16 @@
   cases as published by the Free Software Foundation.
 */
 
-
 #ifndef __MDC_MEM_TYPES_H__
 #define __MDC_MEM_TYPES_H__
 
-#include "mem-types.h"
+#include <glusterfs/mem-types.h>
 
 enum gf_mdc_mem_types_ {
-        gf_mdc_mt_mdc_local_t   = gf_common_mt_end + 1,
-	gf_mdc_mt_md_cache_t,
-	gf_mdc_mt_mdc_conf_t,
-        gf_mdc_mt_end
+    gf_mdc_mt_mdc_local_t = gf_common_mt_end + 1,
+    gf_mdc_mt_md_cache_t,
+    gf_mdc_mt_mdc_conf_t,
+    gf_mdc_mt_mdc_ipc,
+    gf_mdc_mt_end
 };
 #endif
-
diff --git a/xlators/performance/md-cache/src/md-cache-messages.h b/xlators/performance/md-cache/src/md-cache-messages.h
new file mode 100644
index 00000000000..f367bad1991
--- /dev/null
+++ b/xlators/performance/md-cache/src/md-cache-messages.h
@@ -0,0 +1,29 @@
+/*Copyright (c) 2015 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
+*/
+
+#ifndef _MD_CACHE_MESSAGES_H_
+#define _MD_CACHE_MESSAGES_H_
+
+#include <glusterfs/glfs-message-id.h>
+
+/* To add new message IDs, append new identifiers at the end of the list.
+ *
+ * Never remove a message ID. If it's not used anymore, you can rename it or
+ * leave it as it is, but not delete it. This is to prevent reutilization of
+ * IDs by other messages.
+ *
+ * The component name must match one of the entries defined in
+ * glfs-message-id.h.
+ */
+
+GLFS_MSGID(MD_CACHE, MD_CACHE_MSG_NO_MEMORY, MD_CACHE_MSG_DISCARD_UPDATE,
+           MD_CACHE_MSG_CACHE_UPDATE, MD_CACHE_MSG_IPC_UPCALL_FAILED,
+           MD_CACHE_MSG_NO_XATTR_CACHE);
+
+#endif /* _MD_CACHE_MESSAGES_H_ */
diff --git a/xlators/performance/md-cache/src/md-cache.c b/xlators/performance/md-cache/src/md-cache.c
index 51c07339fdf..a405be51f02 100644
--- a/xlators/performance/md-cache/src/md-cache.c
+++ b/xlators/performance/md-cache/src/md-cache.c
@@ -8,2332 +8,4013 @@
   cases as published by the Free Software Foundation.
 */
 
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
-
-#include "glusterfs.h"
-#include "logging.h"
-#include "dict.h"
-#include "xlator.h"
+#include <glusterfs/glusterfs.h>
+#include <glusterfs/defaults.h>
+#include <glusterfs/logging.h>
+#include <glusterfs/dict.h>
+#include <glusterfs/xlator.h>
+#include <glusterfs/syncop.h>
 #include "md-cache-mem-types.h"
-#include "compat-errno.h"
-#include "glusterfs-acl.h"
+#include <glusterfs/compat-errno.h>
+#include <glusterfs/glusterfs-acl.h>
+#include <glusterfs/defaults.h>
+#include <glusterfs/upcall-utils.h>
 #include <assert.h>
 #include <sys/time.h>
-
+#include "md-cache-messages.h"
+#include <glusterfs/statedump.h>
+#include <glusterfs/atomic.h>
 
 /* TODO:
    - cache symlink() link names and nuke symlink-cache
    - send proper postbuf in setattr_cbk even when op_ret = -1
 */
 
-
-struct mdc_conf {
-	int  timeout;
-	gf_boolean_t cache_posix_acl;
-	gf_boolean_t cache_selinux;
-	gf_boolean_t force_readdirp;
+struct mdc_statfs_cache {
+    pthread_mutex_t lock;
+    time_t last_refreshed; /* (time_t)-1 if not yet initialized. */
+    struct statvfs buf;
 };
 
+struct mdc_statistics {
+    gf_atomic_t stat_hit; /* No. of times lookup/stat was served from
+                             mdc */
 
-static struct mdc_key {
-	const char *name;
-	int         load;
-	int         check;
-} mdc_keys[] = {
-	{
-		.name = POSIX_ACL_ACCESS_XATTR,
-		.load = 0,
-		.check = 1,
-	},
-	{
-		.name = POSIX_ACL_DEFAULT_XATTR,
-		.load = 0,
-		.check = 1,
-	},
-	{
-		.name = GF_POSIX_ACL_ACCESS,
-		.load = 0,
-		.check = 1,
-	},
-	{
-		.name = GF_POSIX_ACL_DEFAULT,
-		.load = 0,
-		.check = 1,
-	},
-	{
-		.name = GF_SELINUX_XATTR_KEY,
-		.load = 0,
-		.check = 1,
-	},
-	{
-		.name = "security.capability",
-		.load = 0,
-		.check = 1,
-	},
-	{
-		.name = "gfid-req",
-		.load = 0,
-		.check = 1,
-	},
-        {
-                .name = NULL,
-                .load = 0,
-                .check = 0,
-        }
-};
-
+    gf_atomic_t stat_miss; /* No. of times valid stat wasn't present in
+                              mdc */
 
-static uint64_t
-gfid_to_ino (uuid_t gfid)
-{
-	uint64_t  ino = 0;
-	int       i = 0, j = 0;
+    gf_atomic_t xattr_hit; /* No. of times getxattr was served from mdc,
+                              Note: this doesn't count the xattr served
+                              from lookup */
 
-        for (i = 15; i > (15 - 8); i--) {
-                ino += (uint64_t)(gfid[i]) << j;
-                j += 8;
-        }
+    gf_atomic_t xattr_miss;      /* No. of times xattr req was WIND from mdc */
+    gf_atomic_t negative_lookup; /* No. of negative lookups */
+    gf_atomic_t nameless_lookup; /* No. of negative lookups that were sent
+                                    to bricks */
 
-	return ino;
-}
+    gf_atomic_t stat_invals;  /* No. of invalidates received from upcall */
+    gf_atomic_t xattr_invals; /* No. of invalidates received from upcall */
+    gf_atomic_t need_lookup;  /* No. of lookups issued, because other
+                                 xlators requested for explicit lookup */
+};
 
+struct mdc_conf {
+    uint32_t timeout;
+    gf_boolean_t cache_posix_acl;
+    gf_boolean_t cache_glusterfs_acl;
+    gf_boolean_t cache_selinux;
+    gf_boolean_t cache_capability;
+    gf_boolean_t cache_ima;
+    gf_boolean_t force_readdirp;
+    gf_boolean_t cache_swift_metadata;
+    gf_boolean_t cache_samba_metadata;
+    gf_boolean_t mdc_invalidation;
+    gf_boolean_t global_invalidation;
+
+    time_t last_child_down;
+    gf_lock_t lock;
+    struct mdc_statistics mdc_counter;
+    gf_boolean_t cache_statfs;
+    struct mdc_statfs_cache statfs_cache;
+    char *mdc_xattr_str;
+    gf_atomic_int32_t generation;
+};
 
 struct mdc_local;
 typedef struct mdc_local mdc_local_t;
 
-#define MDC_STACK_UNWIND(fop, frame, params ...) do {           \
-                mdc_local_t *__local = NULL;                    \
-                xlator_t    *__xl    = NULL;                    \
-                if (frame) {                                    \
-                        __xl         = frame->this;             \
-                        __local      = frame->local;            \
-                        frame->local = NULL;                    \
-                }                                               \
-                STACK_UNWIND_STRICT (fop, frame, params);       \
-                mdc_local_wipe (__xl, __local);                 \
-        } while (0)
-
+#define MDC_STACK_UNWIND(fop, frame, params...)                                \
+    do {                                                                       \
+        mdc_local_t *__local = NULL;                                           \
+        xlator_t *__xl = NULL;                                                 \
+        if (frame) {                                                           \
+            __xl = frame->this;                                                \
+            __local = frame->local;                                            \
+            frame->local = NULL;                                               \
+        }                                                                      \
+        STACK_UNWIND_STRICT(fop, frame, params);                               \
+        mdc_local_wipe(__xl, __local);                                         \
+    } while (0)
 
 struct md_cache {
-        ia_prot_t     md_prot;
-        uint32_t      md_nlink;
-        uint32_t      md_uid;
-        uint32_t      md_gid;
-        uint32_t      md_atime;
-        uint32_t      md_atime_nsec;
-        uint32_t      md_mtime;
-        uint32_t      md_mtime_nsec;
-        uint32_t      md_ctime;
-        uint32_t      md_ctime_nsec;
-        uint64_t      md_rdev;
-        uint64_t      md_size;
-        uint64_t      md_blocks;
-        dict_t       *xattr;
-        char         *linkname;
-	time_t        ia_time;
-	time_t        xa_time;
-        gf_lock_t     lock;
+    ia_prot_t md_prot;
+    uint32_t md_nlink;
+    uint32_t md_uid;
+    uint32_t md_gid;
+    uint32_t md_atime_nsec;
+    uint32_t md_mtime_nsec;
+    uint32_t md_ctime_nsec;
+    int64_t md_atime;
+    int64_t md_mtime;
+    int64_t md_ctime;
+    uint64_t md_rdev;
+    uint64_t md_size;
+    uint64_t md_blocks;
+    uint64_t generation;
+    dict_t *xattr;
+    char *linkname;
+    time_t ia_time;
+    time_t xa_time;
+    gf_boolean_t need_lookup;
+    gf_boolean_t valid;
+    gf_boolean_t gen_rollover;
+    gf_boolean_t invalidation_rollover;
+    gf_lock_t lock;
 };
 
-
 struct mdc_local {
-        loc_t   loc;
-        loc_t   loc2;
-        fd_t   *fd;
-        char   *linkname;
-	char   *key;
-        dict_t *xattr;
+    loc_t loc;
+    loc_t loc2;
+    fd_t *fd;
+    char *linkname;
+    char *key;
+    dict_t *xattr;
+    uint64_t incident_time;
+    bool update_cache;
 };
 
-
 int
-__mdc_inode_ctx_get (xlator_t *this, inode_t *inode, struct md_cache **mdc_p)
+__mdc_inode_ctx_get(xlator_t *this, inode_t *inode, struct md_cache **mdc_p)
 {
-        int              ret = 0;
-        struct md_cache *mdc = NULL;
-        uint64_t         mdc_int = 0;
+    int ret = 0;
+    struct md_cache *mdc = NULL;
+    uint64_t mdc_int = 0;
 
-	ret = __inode_ctx_get (inode, this, &mdc_int);
-	mdc = (void *) (long) (mdc_int);
-	if (ret == 0 && mdc_p)
-		*mdc_p = mdc;
+    ret = __inode_ctx_get(inode, this, &mdc_int);
+    mdc = (void *)(long)(mdc_int);
+    if (ret == 0 && mdc_p)
+        *mdc_p = mdc;
 
-	return ret;
+    return ret;
 }
 
-
 int
-mdc_inode_ctx_get (xlator_t *this, inode_t *inode, struct md_cache **mdc_p)
+mdc_inode_ctx_get(xlator_t *this, inode_t *inode, struct md_cache **mdc_p)
 {
-	int   ret;
+    int ret = -1;
+
+    if (!inode)
+        goto out;
 
-	LOCK(&inode->lock);
-	{
-		ret = __mdc_inode_ctx_get (this, inode, mdc_p);
-	}
-	UNLOCK(&inode->lock);
+    LOCK(&inode->lock);
+    {
+        ret = __mdc_inode_ctx_get(this, inode, mdc_p);
+    }
+    UNLOCK(&inode->lock);
 
-	return ret;
+out:
+    return ret;
 }
 
+uint64_t
+__mdc_inc_generation(xlator_t *this, struct md_cache *mdc)
+{
+    uint64_t gen = 0, rollover;
+    struct mdc_conf *conf = NULL;
 
-int
-__mdc_inode_ctx_set (xlator_t *this, inode_t *inode, struct md_cache *mdc)
+    conf = this->private;
+
+    gen = GF_ATOMIC_INC(conf->generation);
+    if (gen == 0) {
+        mdc->gen_rollover = !mdc->gen_rollover;
+        gen = GF_ATOMIC_INC(conf->generation);
+        mdc->ia_time = 0;
+        mdc->generation = 0;
+    }
+
+    rollover = mdc->gen_rollover;
+    gen |= (rollover << 32);
+    return gen;
+}
+
+uint64_t
+mdc_inc_generation(xlator_t *this, inode_t *inode)
 {
-        int              ret = 0;
-        uint64_t         mdc_int = 0;
+    struct mdc_conf *conf = NULL;
+    uint64_t gen = 0;
+    struct md_cache *mdc = NULL;
 
-	mdc_int = (long) mdc;
-	ret = __inode_ctx_set (inode, this, &mdc_int);
+    conf = this->private;
+
+    mdc_inode_ctx_get(this, inode, &mdc);
+
+    if (mdc) {
+        LOCK(&mdc->lock);
+        {
+            gen = __mdc_inc_generation(this, mdc);
+        }
+        UNLOCK(&mdc->lock);
+    } else {
+        gen = GF_ATOMIC_INC(conf->generation);
+        if (gen == 0) {
+            gen = GF_ATOMIC_INC(conf->generation);
+        }
+    }
 
-	return ret;
+    return gen;
 }
 
+uint64_t
+mdc_get_generation(xlator_t *this, inode_t *inode)
+{
+    struct mdc_conf *conf = NULL;
+    uint64_t gen = 0;
+    struct md_cache *mdc = NULL;
+
+    conf = this->private;
+
+    mdc_inode_ctx_get(this, inode, &mdc);
+
+    if (mdc) {
+        LOCK(&mdc->lock);
+        {
+            gen = mdc->generation;
+        }
+        UNLOCK(&mdc->lock);
+    } else
+        gen = GF_ATOMIC_GET(conf->generation);
+
+    return gen;
+}
 
 int
-mdc_inode_ctx_set (xlator_t *this, inode_t *inode, struct md_cache *mdc)
+__mdc_inode_ctx_set(xlator_t *this, inode_t *inode, struct md_cache *mdc)
 {
-	int   ret;
+    int ret = 0;
+    uint64_t mdc_int = 0;
 
-	LOCK(&inode->lock);
-	{
-		ret = __mdc_inode_ctx_set (this, inode, mdc);
-	}
-	UNLOCK(&inode->lock);
+    mdc_int = (long)mdc;
+    ret = __inode_ctx_set(inode, this, &mdc_int);
 
-	return ret;
+    return ret;
 }
 
+int
+mdc_inode_ctx_set(xlator_t *this, inode_t *inode, struct md_cache *mdc)
+{
+    int ret;
+
+    LOCK(&inode->lock);
+    {
+        ret = __mdc_inode_ctx_set(this, inode, mdc);
+    }
+    UNLOCK(&inode->lock);
+
+    return ret;
+}
 
 mdc_local_t *
-mdc_local_get (call_frame_t *frame)
+mdc_local_get(call_frame_t *frame, inode_t *inode)
 {
-        mdc_local_t *local = NULL;
+    mdc_local_t *local = NULL;
 
-        local = frame->local;
-        if (local)
-                goto out;
+    local = frame->local;
+    if (local)
+        goto out;
 
-        local = GF_CALLOC (sizeof (*local), 1, gf_mdc_mt_mdc_local_t);
-        if (!local)
-                goto out;
+    local = GF_CALLOC(sizeof(*local), 1, gf_mdc_mt_mdc_local_t);
+    if (!local)
+        goto out;
 
-        frame->local = local;
+    local->incident_time = mdc_get_generation(frame->this, inode);
+    frame->local = local;
 out:
-        return local;
+    return local;
 }
 
-
 void
-mdc_local_wipe (xlator_t *this, mdc_local_t *local)
+mdc_local_wipe(xlator_t *this, mdc_local_t *local)
 {
-        if (!local)
-                return;
+    if (!local)
+        return;
 
-        loc_wipe (&local->loc);
+    loc_wipe(&local->loc);
 
-        loc_wipe (&local->loc2);
+    loc_wipe(&local->loc2);
 
-        if (local->fd)
-                fd_unref (local->fd);
+    if (local->fd)
+        fd_unref(local->fd);
 
-        GF_FREE (local->linkname);
+    GF_FREE(local->linkname);
 
-        GF_FREE (local->key);
+    GF_FREE(local->key);
 
-        if (local->xattr)
-                dict_unref (local->xattr);
+    if (local->xattr)
+        dict_unref(local->xattr);
 
-        GF_FREE (local);
-        return;
+    GF_FREE(local);
+    return;
 }
 
-
 int
-mdc_inode_wipe (xlator_t *this, inode_t *inode)
+mdc_inode_wipe(xlator_t *this, inode_t *inode)
 {
-        int              ret = 0;
-        uint64_t         mdc_int = 0;
-        struct md_cache *mdc = NULL;
+    int ret = 0;
+    uint64_t mdc_int = 0;
+    struct md_cache *mdc = NULL;
 
-        ret = inode_ctx_del (inode, this, &mdc_int);
-        if (ret != 0)
-                goto out;
+    ret = inode_ctx_del(inode, this, &mdc_int);
+    if (ret != 0)
+        goto out;
 
-        mdc = (void *) (long) mdc_int;
+    mdc = (void *)(long)mdc_int;
 
-        if (mdc->xattr)
-                dict_unref (mdc->xattr);
+    if (mdc->xattr)
+        dict_unref(mdc->xattr);
 
-        GF_FREE (mdc->linkname);
+    GF_FREE(mdc->linkname);
 
-        GF_FREE (mdc);
+    GF_FREE(mdc);
 
-        ret = 0;
+    ret = 0;
 out:
-        return ret;
+    return ret;
 }
 
-
 struct md_cache *
-mdc_inode_prep (xlator_t *this, inode_t *inode)
-{
-        int              ret = 0;
-        struct md_cache *mdc = NULL;
-
-        LOCK (&inode->lock);
-        {
-		ret = __mdc_inode_ctx_get (this, inode, &mdc);
-                if (ret == 0)
-                        goto unlock;
-
-                mdc = GF_CALLOC (sizeof (*mdc), 1, gf_mdc_mt_md_cache_t);
-                if (!mdc) {
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "out of memory :(");
-                        goto unlock;
-                }
+mdc_inode_prep(xlator_t *this, inode_t *inode)
+{
+    int ret = 0;
+    struct md_cache *mdc = NULL;
+
+    LOCK(&inode->lock);
+    {
+        ret = __mdc_inode_ctx_get(this, inode, &mdc);
+        if (ret == 0)
+            goto unlock;
+
+        mdc = GF_CALLOC(sizeof(*mdc), 1, gf_mdc_mt_md_cache_t);
+        if (!mdc) {
+            gf_msg(this->name, GF_LOG_ERROR, ENOMEM, MD_CACHE_MSG_NO_MEMORY,
+                   "out of memory");
+            goto unlock;
+        }
 
-                LOCK_INIT (&mdc->lock);
+        LOCK_INIT(&mdc->lock);
 
-                ret = __mdc_inode_ctx_set (this, inode, mdc);
-                if (ret) {
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "out of memory :(");
-                        GF_FREE (mdc);
-                        mdc = NULL;
-                }
+        ret = __mdc_inode_ctx_set(this, inode, mdc);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, ENOMEM, MD_CACHE_MSG_NO_MEMORY,
+                   "out of memory");
+            GF_FREE(mdc);
+            mdc = NULL;
         }
+    }
 unlock:
-        UNLOCK (&inode->lock);
+    UNLOCK(&inode->lock);
 
-        return mdc;
+    return mdc;
 }
 
-
+/* Cache is valid if:
+ * - It is not cached before any brick was down. Brick down case is handled by
+ *   invalidating all the cache when any brick went down.
+ * - The cache time is not expired
+ */
 static gf_boolean_t
-is_md_cache_iatt_valid (xlator_t *this, struct md_cache *mdc)
-{
-	struct mdc_conf *conf = NULL;
-	time_t           now = 0;
-        gf_boolean_t     ret = _gf_true;
-	conf = this->private;
+__is_cache_valid(xlator_t *this, time_t mdc_time)
+{
+    gf_boolean_t ret = _gf_true;
+    struct mdc_conf *conf = NULL;
+    uint32_t timeout = 0;
+    time_t last_child_down = 0;
+
+    conf = this->private;
+
+    /* conf->lock here is not taken deliberately, so that the multi
+     * threaded IO doesn't contend on a global lock. While updating
+     * the variable, the lock is taken, so that at least the writes are
+     * intact. The read of last_child_down may return junk, but that
+     * is for a very short period of time.
+     */
+    last_child_down = conf->last_child_down;
+    timeout = conf->timeout;
+
+    if ((mdc_time == 0) ||
+        ((last_child_down != 0) && (mdc_time < last_child_down))) {
+        ret = _gf_false;
+        goto out;
+    }
+
+    if (gf_time() >= (mdc_time + timeout)) {
+        ret = _gf_false;
+    }
 
-	time (&now);
+out:
+    return ret;
+}
 
-        LOCK (&mdc->lock);
-        {
-                if (now >= (mdc->ia_time + conf->timeout))
-                        ret = _gf_false;
+static gf_boolean_t
+is_md_cache_iatt_valid(xlator_t *this, struct md_cache *mdc)
+{
+    gf_boolean_t ret = _gf_true;
+
+    LOCK(&mdc->lock);
+    {
+        if (mdc->valid == _gf_false) {
+            ret = mdc->valid;
+        } else {
+            ret = __is_cache_valid(this, mdc->ia_time);
+            if (ret == _gf_false) {
+                mdc->ia_time = 0;
+                mdc->generation = 0;
+            }
         }
-        UNLOCK (&mdc->lock);
+    }
+    UNLOCK(&mdc->lock);
 
-	return ret;
+    return ret;
 }
 
-
 static gf_boolean_t
-is_md_cache_xatt_valid (xlator_t *this, struct md_cache *mdc)
+is_md_cache_xatt_valid(xlator_t *this, struct md_cache *mdc)
 {
-	struct mdc_conf *conf = NULL;
-	time_t           now = 0;
-        gf_boolean_t     ret = _gf_true;
-
-	conf = this->private;
-
-	time (&now);
+    gf_boolean_t ret = _gf_true;
 
-        LOCK (&mdc->lock);
-        {
-                if (now >= (mdc->xa_time + conf->timeout))
-                        ret = _gf_false;
-        }
-        UNLOCK (&mdc->lock);
+    LOCK(&mdc->lock);
+    {
+        ret = __is_cache_valid(this, mdc->xa_time);
+        if (ret == _gf_false)
+            mdc->xa_time = 0;
+    }
+    UNLOCK(&mdc->lock);
 
-	return ret;
+    return ret;
 }
 
-
 void
-mdc_from_iatt (struct md_cache *mdc, struct iatt *iatt)
-{
-        mdc->md_prot       = iatt->ia_prot;
-        mdc->md_nlink      = iatt->ia_nlink;
-        mdc->md_uid        = iatt->ia_uid;
-        mdc->md_gid        = iatt->ia_gid;
-        mdc->md_atime      = iatt->ia_atime;
-        mdc->md_atime_nsec = iatt->ia_atime_nsec;
-        mdc->md_mtime      = iatt->ia_mtime;
-        mdc->md_mtime_nsec = iatt->ia_mtime_nsec;
-        mdc->md_ctime      = iatt->ia_ctime;
-        mdc->md_ctime_nsec = iatt->ia_ctime_nsec;
-        mdc->md_rdev       = iatt->ia_rdev;
-        mdc->md_size       = iatt->ia_size;
-        mdc->md_blocks     = iatt->ia_blocks;
+mdc_from_iatt(struct md_cache *mdc, struct iatt *iatt)
+{
+    mdc->md_prot = iatt->ia_prot;
+    mdc->md_nlink = iatt->ia_nlink;
+    mdc->md_uid = iatt->ia_uid;
+    mdc->md_gid = iatt->ia_gid;
+    mdc->md_atime = iatt->ia_atime;
+    mdc->md_atime_nsec = iatt->ia_atime_nsec;
+    mdc->md_mtime = iatt->ia_mtime;
+    mdc->md_mtime_nsec = iatt->ia_mtime_nsec;
+    mdc->md_ctime = iatt->ia_ctime;
+    mdc->md_ctime_nsec = iatt->ia_ctime_nsec;
+    mdc->md_rdev = iatt->ia_rdev;
+    mdc->md_size = iatt->ia_size;
+    mdc->md_blocks = iatt->ia_blocks;
 }
 
-
 void
-mdc_to_iatt (struct md_cache *mdc, struct iatt *iatt)
+mdc_to_iatt(struct md_cache *mdc, struct iatt *iatt)
 {
-        iatt->ia_prot       = mdc->md_prot;
-        iatt->ia_nlink      = mdc->md_nlink;
-        iatt->ia_uid        = mdc->md_uid;
-        iatt->ia_gid        = mdc->md_gid;
-        iatt->ia_atime      = mdc->md_atime;
-        iatt->ia_atime_nsec = mdc->md_atime_nsec;
-        iatt->ia_mtime      = mdc->md_mtime;
-        iatt->ia_mtime_nsec = mdc->md_mtime_nsec;
-        iatt->ia_ctime      = mdc->md_ctime;
-        iatt->ia_ctime_nsec = mdc->md_ctime_nsec;
-        iatt->ia_rdev       = mdc->md_rdev;
-        iatt->ia_size       = mdc->md_size;
-        iatt->ia_blocks     = mdc->md_blocks;
+    iatt->ia_prot = mdc->md_prot;
+    iatt->ia_nlink = mdc->md_nlink;
+    iatt->ia_uid = mdc->md_uid;
+    iatt->ia_gid = mdc->md_gid;
+    iatt->ia_atime = mdc->md_atime;
+    iatt->ia_atime_nsec = mdc->md_atime_nsec;
+    iatt->ia_mtime = mdc->md_mtime;
+    iatt->ia_mtime_nsec = mdc->md_mtime_nsec;
+    iatt->ia_ctime = mdc->md_ctime;
+    iatt->ia_ctime_nsec = mdc->md_ctime_nsec;
+    iatt->ia_rdev = mdc->md_rdev;
+    iatt->ia_size = mdc->md_size;
+    iatt->ia_blocks = mdc->md_blocks;
 }
 
-
 int
 mdc_inode_iatt_set_validate(xlator_t *this, inode_t *inode, struct iatt *prebuf,
-			    struct iatt *iatt)
-{
-        int              ret = -1;
-        struct md_cache *mdc = NULL;
+                            struct iatt *iatt, gf_boolean_t update_time,
+                            uint64_t incident_time)
+{
+    int ret = 0;
+    struct md_cache *mdc = NULL;
+    uint32_t rollover = 0;
+    uint64_t gen = 0;
+    gf_boolean_t update_xa_time = _gf_false;
+    struct mdc_conf *conf = this->private;
+
+    mdc = mdc_inode_prep(this, inode);
+    if (!mdc) {
+        ret = -1;
+        goto out;
+    }
+
+    rollover = incident_time >> 32;
+    incident_time = (incident_time & 0xffffffff);
+
+    LOCK(&mdc->lock);
+    {
+        if (!iatt || !iatt->ia_ctime) {
+            gf_msg_callingfn("md-cache", GF_LOG_TRACE, 0, 0,
+                             "invalidating iatt(NULL)"
+                             "(%s)",
+                             uuid_utoa(inode->gfid));
+            mdc->ia_time = 0;
+            mdc->valid = 0;
+
+            gen = __mdc_inc_generation(this, mdc);
+            mdc->generation = (gen & 0xffffffff);
+            goto unlock;
+        }
 
-        mdc = mdc_inode_prep (this, inode);
-        if (!mdc)
-                goto out;
+        /* There could be a race in invalidation, where the
+         * invalidations in order A, B reaches md-cache in the order
+         * B, A. Hence, make sure the invalidation A is discarded if
+         * it comes after B. ctime of a file is always in ascending
+         * order unlike atime and mtime(which can be changed by user
+         * to any date), also ctime gets updates when atime/mtime
+         * changes, hence check for ctime only.
+         */
+        if (mdc->md_ctime > iatt->ia_ctime) {
+            gf_msg_callingfn(this->name, GF_LOG_DEBUG, EINVAL,
+                             MD_CACHE_MSG_DISCARD_UPDATE,
+                             "discarding the iatt validate "
+                             "request (%s)",
+                             uuid_utoa(inode->gfid));
+            ret = -1;
+            goto unlock;
+        }
+        if ((mdc->md_ctime == iatt->ia_ctime) &&
+            (mdc->md_ctime_nsec > iatt->ia_ctime_nsec)) {
+            gf_msg_callingfn(this->name, GF_LOG_DEBUG, EINVAL,
+                             MD_CACHE_MSG_DISCARD_UPDATE,
+                             "discarding the iatt validate "
+                             "request(ctime_nsec) (%s)",
+                             uuid_utoa(inode->gfid));
+            ret = -1;
+            goto unlock;
+        }
 
-        LOCK (&mdc->lock);
-        {
-                if (!iatt || !iatt->ia_ctime) {
-                        mdc->ia_time = 0;
-                        goto unlock;
+        /*
+         * Invalidate the inode if the mtime or ctime has changed
+         * and the prebuf doesn't match the value we have cached.
+         * TODO: writev returns with a NULL iatt due to
+         * performance/write-behind, causing invalidation on writes.
+         */
+        if ((iatt->ia_mtime != mdc->md_mtime) ||
+            (iatt->ia_mtime_nsec != mdc->md_mtime_nsec) ||
+            (iatt->ia_ctime != mdc->md_ctime) ||
+            (iatt->ia_ctime_nsec != mdc->md_ctime_nsec)) {
+            if (conf->global_invalidation &&
+                (!prebuf || (prebuf->ia_mtime != mdc->md_mtime) ||
+                 (prebuf->ia_mtime_nsec != mdc->md_mtime_nsec) ||
+                 (prebuf->ia_ctime != mdc->md_ctime) ||
+                 (prebuf->ia_ctime_nsec != mdc->md_ctime_nsec))) {
+                if (IA_ISREG(inode->ia_type)) {
+                    gf_msg("md-cache", GF_LOG_TRACE, 0,
+                           MD_CACHE_MSG_DISCARD_UPDATE,
+                           "prebuf doesn't match the value we have cached,"
+                           " invalidate the inode(%s)",
+                           uuid_utoa(inode->gfid));
+
+                    inode_invalidate(inode);
                 }
+            } else {
+                update_xa_time = _gf_true;
+            }
+        }
 
-		/*
-		 * Invalidate the inode if the mtime or ctime has changed
-		 * and the prebuf doesn't match the value we have cached.
-		 * TODO: writev returns with a NULL iatt due to
-		 * performance/write-behind, causing invalidation on writes.
-		 */
-		if (IA_ISREG(inode->ia_type) &&
-		    ((iatt->ia_mtime != mdc->md_mtime) ||
-		    (iatt->ia_mtime_nsec != mdc->md_mtime_nsec) ||
-		    (iatt->ia_ctime != mdc->md_ctime) ||
-		    (iatt->ia_ctime_nsec != mdc->md_ctime_nsec)))
-			if (!prebuf || (prebuf->ia_ctime != mdc->md_ctime) ||
-			    (prebuf->ia_ctime_nsec != mdc->md_ctime_nsec) ||
-			    (prebuf->ia_mtime != mdc->md_mtime) ||
-			    (prebuf->ia_mtime_nsec != mdc->md_mtime_nsec))
-				inode_invalidate(inode);
-
-                mdc_from_iatt (mdc, iatt);
-
-                time (&mdc->ia_time);
+        if ((mdc->gen_rollover == rollover) &&
+            (incident_time >= mdc->generation)) {
+            mdc_from_iatt(mdc, iatt);
+            mdc->valid = _gf_true;
+            if (update_time) {
+                mdc->ia_time = gf_time();
+                if (mdc->xa_time && update_xa_time)
+                    mdc->xa_time = mdc->ia_time;
+            }
+
+            gf_msg_callingfn(
+                "md-cache", GF_LOG_TRACE, 0, MD_CACHE_MSG_CACHE_UPDATE,
+                "Updated iatt(%s)"
+                " time:%lld generation=%lld",
+                uuid_utoa(iatt->ia_gfid), (unsigned long long)mdc->ia_time,
+                (unsigned long long)mdc->generation);
+        } else {
+            gf_msg_callingfn("md-cache", GF_LOG_TRACE, 0, 0,
+                             "not updating cache (%s)"
+                             "mdc-rollover=%u rollover=%u "
+                             "mdc-generation=%llu "
+                             "mdc-ia_time=%llu incident_time=%llu ",
+                             uuid_utoa(iatt->ia_gfid), mdc->gen_rollover,
+                             rollover, (unsigned long long)mdc->generation,
+                             (unsigned long long)mdc->ia_time,
+                             (unsigned long long)incident_time);
         }
+    }
 unlock:
-        UNLOCK (&mdc->lock);
-        ret = 0;
+    UNLOCK(&mdc->lock);
+
 out:
-        return ret;
+    return ret;
 }
 
-int mdc_inode_iatt_set(xlator_t *this, inode_t *inode, struct iatt *iatt)
+int
+mdc_inode_iatt_set(xlator_t *this, inode_t *inode, struct iatt *iatt,
+                   uint64_t incident_time)
 {
-	return mdc_inode_iatt_set_validate(this, inode, NULL, iatt);
+    return mdc_inode_iatt_set_validate(this, inode, NULL, iatt, _gf_true,
+                                       incident_time);
 }
 
 int
-mdc_inode_iatt_get (xlator_t *this, inode_t *inode, struct iatt *iatt)
+mdc_inode_iatt_get(xlator_t *this, inode_t *inode, struct iatt *iatt)
 {
-        int              ret = -1;
-        struct md_cache *mdc = NULL;
+    int ret = -1;
+    struct md_cache *mdc = NULL;
 
-        if (mdc_inode_ctx_get (this, inode, &mdc) != 0)
-                goto out;
+    if (mdc_inode_ctx_get(this, inode, &mdc) != 0) {
+        gf_msg_trace("md-cache", 0, "mdc_inode_ctx_get failed (%s)",
+                     uuid_utoa(inode->gfid));
+        goto out;
+    }
 
-	if (!is_md_cache_iatt_valid (this, mdc))
-		goto out;
+    if (!is_md_cache_iatt_valid(this, mdc)) {
+        gf_msg_trace("md-cache", 0, "iatt cache not valid for (%s)",
+                     uuid_utoa(inode->gfid));
+        goto out;
+    }
 
-        LOCK (&mdc->lock);
-        {
-                mdc_to_iatt (mdc, iatt);
-        }
-        UNLOCK (&mdc->lock);
+    LOCK(&mdc->lock);
+    {
+        mdc_to_iatt(mdc, iatt);
+    }
+    UNLOCK(&mdc->lock);
 
-        gf_uuid_copy (iatt->ia_gfid, inode->gfid);
-        iatt->ia_ino    = gfid_to_ino (inode->gfid);
-        iatt->ia_dev    = 42;
-        iatt->ia_type   = inode->ia_type;
+    gf_uuid_copy(iatt->ia_gfid, inode->gfid);
+    iatt->ia_ino = gfid_to_ino(inode->gfid);
+    iatt->ia_dev = 42;
+    iatt->ia_type = inode->ia_type;
 
-        ret = 0;
+    ret = 0;
 out:
-        return ret;
+    return ret;
 }
 
 struct updatedict {
-	dict_t *dict;
-	int ret;
+    dict_t *dict;
+    int ret;
 };
 
 static int
+is_mdc_key_satisfied(xlator_t *this, const char *key)
+{
+    int ret = 0;
+    char *pattern = NULL;
+    struct mdc_conf *conf = this->private;
+    char *mdc_xattr_str = NULL;
+    char *tmp = NULL;
+    char *tmp1 = NULL;
+
+    if (!key)
+        goto out;
+
+    /* conf->mdc_xattr_str, is never freed and is hence safely used outside
+     * of lock*/
+    tmp1 = conf->mdc_xattr_str;
+    if (!tmp1)
+        goto out;
+
+    mdc_xattr_str = gf_strdup(tmp1);
+    if (!mdc_xattr_str)
+        goto out;
+
+    pattern = strtok_r(mdc_xattr_str, ",", &tmp);
+    while (pattern) {
+        gf_strTrim(&pattern);
+        if (fnmatch(pattern, key, 0) == 0) {
+            ret = 1;
+            break;
+        } else {
+            gf_msg_trace("md-cache", 0,
+                         "xattr key %s doesn't satisfy "
+                         "caching requirements",
+                         key);
+        }
+        pattern = strtok_r(NULL, ",", &tmp);
+    }
+    GF_FREE(mdc_xattr_str);
+out:
+    return ret;
+}
+
+static int
 updatefn(dict_t *dict, char *key, data_t *value, void *data)
 {
-	struct updatedict *u = data;
-	const char *mdc_key;
-	int i = 0;
-
-	for (mdc_key = mdc_keys[i].name; (mdc_key = mdc_keys[i].name); i++) {
-		if (!mdc_keys[i].check)
-			continue;
-		if (strcmp(mdc_key, key))
-			continue;
-
-		if (!u->dict) {
-			u->dict = dict_new();
-			if (!u->dict) {
-				u->ret = -1;
-				return -1;
-			}
-		}
-
-                /* posix xlator as part of listxattr will send both names
-                 * and values of the xattrs in the dict. But as per man page
-                 * listxattr is mainly supposed to send names of the all the
-                 * xattrs. gfapi, as of now will put all the keys it obtained
-                 * in the dict (sent by posix) into a buffer provided by the
-                 * caller (thus the values of those xattrs are lost). If some
-                 * xlator makes gfapi based calls (ex: snapview-server), then
-                 * it has to unwind the calls by putting those names it got
-                 * in the buffer again into the dict. But now it would not be
-                 * having the values for those xattrs. So it might just put
-                 * a 0 byte value ("") into the dict for each xattr and unwind
-                 * the call. So the xlators which cache the xattrs (as of now
-                 * md-cache caches the acl and selinux related xattrs), should
-                 * not update their cache if the value of a xattr is a 0 byte
-                 * data (i.e. "").
-                 */
-                if (!strcmp (value->data, ""))
-                        continue;
-
-		if (dict_set(u->dict, key, value) < 0) {
-			u->ret = -1;
-			return -1;
-		}
-
-		break;
-	}
-        return 0;
+    struct updatedict *u = data;
+
+    if (is_mdc_key_satisfied(THIS, key)) {
+        if (!u->dict) {
+            u->dict = dict_new();
+            if (!u->dict) {
+                u->ret = -1;
+                return -1;
+            }
+        }
+
+        if (dict_set(u->dict, key, value) < 0) {
+            u->ret = -1;
+            return -1;
+        }
+    }
+    return 0;
 }
 
 static int
 mdc_dict_update(dict_t **tgt, dict_t *src)
 {
-	struct updatedict u = {
-		.dict = *tgt,
-		.ret = 0,
-	};
+    struct updatedict u = {
+        .dict = *tgt,
+        .ret = 0,
+    };
 
-	dict_foreach(src, updatefn, &u);
+    dict_foreach(src, updatefn, &u);
 
-	if (*tgt)
-		return u.ret;
+    if (*tgt)
+        return u.ret;
 
-	if ((u.ret < 0) && u.dict) {
-		dict_unref(u.dict);
-		return u.ret;
-	}
+    if ((u.ret < 0) && u.dict) {
+        dict_unref(u.dict);
+        return u.ret;
+    }
 
-	*tgt = u.dict;
+    *tgt = u.dict;
 
-	return u.ret;
+    return u.ret;
 }
 
 int
-mdc_inode_xatt_set (xlator_t *this, inode_t *inode, dict_t *dict)
+mdc_inode_xatt_set(xlator_t *this, inode_t *inode, dict_t *dict)
 {
-        int              ret = -1;
-        struct md_cache *mdc = NULL;
-	dict_t		*newdict = NULL;
+    int ret = -1;
+    struct md_cache *mdc = NULL;
+    dict_t *newdict = NULL;
 
-        mdc = mdc_inode_prep (this, inode);
-        if (!mdc)
-                goto out;
+    mdc = mdc_inode_prep(this, inode);
+    if (!mdc)
+        goto out;
 
-        if (!dict)
-                goto out;
+    if (!dict) {
+        gf_msg_trace("md-cache", 0,
+                     "mdc_inode_xatt_set failed (%s) "
+                     "dict NULL",
+                     uuid_utoa(inode->gfid));
+        goto out;
+    }
 
-        LOCK (&mdc->lock);
-        {
-                if (mdc->xattr) {
-                        dict_unref (mdc->xattr);
-			mdc->xattr = NULL;
-		}
+    LOCK(&mdc->lock);
+    {
+        if (mdc->xattr) {
+            gf_msg_trace("md-cache", 0,
+                         "deleting the old xattr "
+                         "cache (%s)",
+                         uuid_utoa(inode->gfid));
+            dict_unref(mdc->xattr);
+            mdc->xattr = NULL;
+        }
 
-		ret = mdc_dict_update(&newdict, dict);
-		if (ret < 0) {
-			UNLOCK(&mdc->lock);
-			goto out;
-		}
+        ret = mdc_dict_update(&newdict, dict);
+        if (ret < 0) {
+            UNLOCK(&mdc->lock);
+            goto out;
+        }
 
-		if (newdict)
-			mdc->xattr = newdict;
+        if (newdict)
+            mdc->xattr = newdict;
 
-                time (&mdc->xa_time);
-        }
-        UNLOCK (&mdc->lock);
-        ret = 0;
+        mdc->xa_time = gf_time();
+        gf_msg_trace("md-cache", 0, "xatt cache set for (%s) time:%lld",
+                     uuid_utoa(inode->gfid), (long long)mdc->xa_time);
+    }
+    UNLOCK(&mdc->lock);
+    ret = 0;
 out:
-        return ret;
+    return ret;
 }
 
-
 int
-mdc_inode_xatt_update (xlator_t *this, inode_t *inode, dict_t *dict)
+mdc_inode_xatt_update(xlator_t *this, inode_t *inode, dict_t *dict)
 {
-        int              ret = -1;
-        struct md_cache *mdc = NULL;
+    int ret = -1;
+    struct md_cache *mdc = NULL;
 
-        mdc = mdc_inode_prep (this, inode);
-        if (!mdc)
-                goto out;
+    mdc = mdc_inode_prep(this, inode);
+    if (!mdc)
+        goto out;
 
-        if (!dict)
-                goto out;
+    if (!dict)
+        goto out;
 
-        LOCK (&mdc->lock);
-        {
-		ret = mdc_dict_update(&mdc->xattr, dict);
-		if (ret < 0) {
-			UNLOCK(&mdc->lock);
-			goto out;
-		}
-
-                time (&mdc->xa_time);
+    LOCK(&mdc->lock);
+    {
+        ret = mdc_dict_update(&mdc->xattr, dict);
+        if (ret < 0) {
+            UNLOCK(&mdc->lock);
+            goto out;
         }
-        UNLOCK (&mdc->lock);
+    }
+    UNLOCK(&mdc->lock);
 
-        ret = 0;
+    ret = 0;
 out:
-        return ret;
+    return ret;
 }
 
-
 int
-mdc_inode_xatt_unset (xlator_t *this, inode_t *inode, char *name)
+mdc_inode_xatt_unset(xlator_t *this, inode_t *inode, char *name)
 {
-        int              ret = -1;
-        struct md_cache *mdc = NULL;
+    int ret = -1;
+    struct md_cache *mdc = NULL;
 
-        mdc = mdc_inode_prep (this, inode);
-        if (!mdc)
-                goto out;
+    mdc = mdc_inode_prep(this, inode);
+    if (!mdc)
+        goto out;
 
-        if (!name || !mdc->xattr)
-                goto out;
+    if (!name || !mdc->xattr)
+        goto out;
 
-        LOCK (&mdc->lock);
-        {
-		dict_del (mdc->xattr, name);
-        }
-        UNLOCK (&mdc->lock);
+    LOCK(&mdc->lock);
+    {
+        dict_del(mdc->xattr, name);
+    }
+    UNLOCK(&mdc->lock);
 
-        ret = 0;
+    ret = 0;
 out:
-        return ret;
+    return ret;
 }
 
-
 int
-mdc_inode_xatt_get (xlator_t *this, inode_t *inode, dict_t **dict)
+mdc_inode_xatt_get(xlator_t *this, inode_t *inode, dict_t **dict)
 {
-        int              ret = -1;
-        struct md_cache *mdc = NULL;
+    int ret = -1;
+    struct md_cache *mdc = NULL;
 
-        if (mdc_inode_ctx_get (this, inode, &mdc) != 0)
-                goto out;
+    if (mdc_inode_ctx_get(this, inode, &mdc) != 0) {
+        gf_msg_trace("md-cache", 0, "mdc_inode_ctx_get failed (%s)",
+                     uuid_utoa(inode->gfid));
+        goto out;
+    }
 
-	if (!is_md_cache_xatt_valid (this, mdc))
-		goto out;
+    if (!is_md_cache_xatt_valid(this, mdc)) {
+        gf_msg_trace("md-cache", 0, "xattr cache not valid for (%s)",
+                     uuid_utoa(inode->gfid));
+        goto out;
+    }
 
-        LOCK (&mdc->lock);
-        {
-                ret = 0;
-		/* Missing xattr only means no keys were there, i.e
-		   a negative cache for the "loaded" keys
-		*/
-                if (!mdc->xattr)
-                        goto unlock;
-
-                if (dict)
-                        *dict = dict_ref (mdc->xattr);
+    LOCK(&mdc->lock);
+    {
+        ret = 0;
+        /* Missing xattr only means no keys were there, i.e
+           a negative cache for the "loaded" keys
+        */
+        if (!mdc->xattr) {
+            gf_msg_trace("md-cache", 0, "xattr not present (%s)",
+                         uuid_utoa(inode->gfid));
+            goto unlock;
         }
+
+        if (dict)
+            *dict = dict_ref(mdc->xattr);
+    }
 unlock:
-        UNLOCK (&mdc->lock);
+    UNLOCK(&mdc->lock);
 
 out:
-        return ret;
+    return ret;
 }
 
-
-int
-mdc_inode_iatt_invalidate (xlator_t *this, inode_t *inode)
+gf_boolean_t
+mdc_inode_reset_need_lookup(xlator_t *this, inode_t *inode)
 {
-        int              ret = -1;
-        struct md_cache *mdc = NULL;
+    struct md_cache *mdc = NULL;
+    gf_boolean_t need = _gf_false;
 
-        if (mdc_inode_ctx_get (this, inode, &mdc) != 0)
-                goto out;
+    if (mdc_inode_ctx_get(this, inode, &mdc) != 0)
+        goto out;
 
-        LOCK (&mdc->lock);
-        {
-		mdc->ia_time = 0;
-        }
-        UNLOCK (&mdc->lock);
+    LOCK(&mdc->lock);
+    {
+        need = mdc->need_lookup;
+        mdc->need_lookup = _gf_false;
+    }
+    UNLOCK(&mdc->lock);
 
 out:
-        return ret;
+    return need;
 }
 
-
-int
-mdc_inode_xatt_invalidate (xlator_t *this, inode_t *inode)
+void
+mdc_inode_set_need_lookup(xlator_t *this, inode_t *inode, gf_boolean_t need)
 {
-        int              ret = -1;
-        struct md_cache *mdc = NULL;
+    struct md_cache *mdc = NULL;
 
-        if (mdc_inode_ctx_get (this, inode, &mdc) != 0)
-                goto out;
+    if (mdc_inode_ctx_get(this, inode, &mdc) != 0)
+        goto out;
 
-        LOCK (&mdc->lock);
-        {
-		mdc->xa_time = 0;
-        }
-        UNLOCK (&mdc->lock);
+    LOCK(&mdc->lock);
+    {
+        mdc->need_lookup = need;
+    }
+    UNLOCK(&mdc->lock);
 
 out:
-        return ret;
+    return;
 }
 
-
 void
-mdc_load_reqs (xlator_t *this, dict_t *dict)
+mdc_inode_iatt_invalidate(xlator_t *this, inode_t *inode)
 {
-	const char *mdc_key = NULL;
-	int  i = 0;
-	int  ret = 0;
+    struct md_cache *mdc = NULL;
+    uint32_t gen = 0;
+
+    if (mdc_inode_ctx_get(this, inode, &mdc) != 0)
+        goto out;
+
+    gen = mdc_inc_generation(this, inode) & 0xffffffff;
+
+    LOCK(&mdc->lock);
+    {
+        mdc->ia_time = 0;
+        mdc->valid = _gf_false;
+        mdc->generation = gen;
+    }
+    UNLOCK(&mdc->lock);
 
-	for (mdc_key = mdc_keys[i].name; (mdc_key = mdc_keys[i].name); i++) {
-		if (!mdc_keys[i].load)
-			continue;
-		ret = dict_set_int8 (dict, (char *)mdc_key, 0);
-		if (ret)
-			return;
-	}
+out:
+    return;
 }
 
+int
+mdc_inode_xatt_invalidate(xlator_t *this, inode_t *inode)
+{
+    int ret = -1;
+    struct md_cache *mdc = NULL;
 
-struct checkpair {
-	int  ret;
-	dict_t *rsp;
-};
+    if (mdc_inode_ctx_get(this, inode, &mdc) != 0)
+        goto out;
 
+    LOCK(&mdc->lock);
+    {
+        mdc->xa_time = 0;
+    }
+    UNLOCK(&mdc->lock);
+
+out:
+    return ret;
+}
 
 static int
-is_mdc_key_satisfied (const char *key)
-{
-	const char *mdc_key = NULL;
-	int  i = 0;
+mdc_update_gfid_stat(xlator_t *this, struct iatt *iatt)
+{
+    int ret = 0;
+    inode_table_t *itable = NULL;
+    inode_t *inode = NULL;
+
+    itable = ((xlator_t *)this->graph->top)->itable;
+    inode = inode_find(itable, iatt->ia_gfid);
+    if (!inode) {
+        ret = -1;
+        goto out;
+    }
+    ret = mdc_inode_iatt_set_validate(this, inode, NULL, iatt, _gf_true,
+                                      mdc_inc_generation(this, inode));
+out:
+    return ret;
+}
+
+static bool
+mdc_load_reqs(xlator_t *this, dict_t *dict)
+{
+    struct mdc_conf *conf = this->private;
+    char *pattern = NULL;
+    char *mdc_xattr_str = NULL;
+    char *tmp = NULL;
+    char *tmp1 = NULL;
+    int ret = 0;
+    bool loaded = false;
+
+    tmp1 = conf->mdc_xattr_str;
+    if (!tmp1)
+        goto out;
+
+    mdc_xattr_str = gf_strdup(tmp1);
+    if (!mdc_xattr_str)
+        goto out;
+
+    pattern = strtok_r(mdc_xattr_str, ",", &tmp);
+    while (pattern) {
+        gf_strTrim(&pattern);
+        ret = dict_set_int8(dict, pattern, 0);
+        if (ret) {
+            conf->mdc_xattr_str = NULL;
+            gf_msg("md-cache", GF_LOG_ERROR, 0, MD_CACHE_MSG_NO_XATTR_CACHE,
+                   "Disabled cache for xattrs, dict_set failed");
+            goto out;
+        }
+        pattern = strtok_r(NULL, ",", &tmp);
+    }
 
-	if (!key)
-		return 0;
+    loaded = true;
 
-	for (mdc_key = mdc_keys[i].name; (mdc_key = mdc_keys[i].name); i++) {
-		if (!mdc_keys[i].load)
-			continue;
-		if (strcmp (mdc_key, key) == 0)
-			return 1;
-	}
+out:
+    GF_FREE(mdc_xattr_str);
 
-	return 0;
+    return loaded;
 }
 
+struct checkpair {
+    int ret;
+    dict_t *rsp;
+};
 
 static int
-checkfn (dict_t *this, char *key, data_t *value, void *data)
+checkfn(dict_t *this, char *key, data_t *value, void *data)
 {
-        struct checkpair *pair = data;
+    struct checkpair *pair = data;
 
-	if (!is_mdc_key_satisfied (key))
-		pair->ret = 0;
+    if (!is_mdc_key_satisfied(THIS, key))
+        pair->ret = 0;
 
-        return 0;
+    return 0;
 }
 
-
 int
-mdc_xattr_satisfied (xlator_t *this, dict_t *req, dict_t *rsp)
+mdc_xattr_satisfied(xlator_t *this, dict_t *req, dict_t *rsp)
 {
-        struct checkpair pair = {
-                .ret = 1,
-                .rsp = rsp,
-        };
+    struct checkpair pair = {
+        .ret = 1,
+        .rsp = rsp,
+    };
 
-        dict_foreach (req, checkfn, &pair);
+    dict_foreach(req, checkfn, &pair);
 
-        return pair.ret;
+    return pair.ret;
 }
 
+static void
+mdc_cache_statfs(xlator_t *this, struct statvfs *buf)
+{
+    struct mdc_conf *conf = this->private;
+
+    pthread_mutex_lock(&conf->statfs_cache.lock);
+    {
+        memcpy(&conf->statfs_cache.buf, buf, sizeof(struct statvfs));
+        conf->statfs_cache.last_refreshed = gf_time();
+    }
+    pthread_mutex_unlock(&conf->statfs_cache.lock);
+}
 
 int
-mdc_lookup_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                int32_t op_ret,	int32_t op_errno, inode_t *inode,
-                struct iatt *stbuf, dict_t *dict, struct iatt *postparent)
+mdc_load_statfs_info_from_cache(xlator_t *this, struct statvfs **buf)
 {
-        mdc_local_t *local = NULL;
+    struct mdc_conf *conf = this->private;
+    uint32_t cache_age = 0;
+    int ret = 0;
 
-        local = frame->local;
+    if (!buf || !conf) {
+        ret = -1;
+        goto err;
+    }
 
-        if (op_ret != 0)
-                goto out;
+    *buf = NULL;
 
-        if (!local)
-                goto out;
+    pthread_mutex_lock(&conf->statfs_cache.lock);
+    {
+        /* Skip if the cache is not initialized. */
+        if (conf->statfs_cache.last_refreshed == (time_t)-1) {
+            ret = -1;
+            goto unlock;
+        }
 
-        if (local->loc.parent) {
-                mdc_inode_iatt_set (this, local->loc.parent, postparent);
+        cache_age = (gf_time() - conf->statfs_cache.last_refreshed);
+
+        gf_log(this->name, GF_LOG_DEBUG, "STATFS cache age = %u secs",
+               cache_age);
+        if (cache_age > conf->timeout) {
+            /* Expire the cache. */
+            gf_log(this->name, GF_LOG_DEBUG,
+                   "Cache age %u secs exceeded timeout %u secs", cache_age,
+                   conf->timeout);
+            ret = -1;
+            goto unlock;
         }
 
-        if (local->loc.inode) {
-                mdc_inode_iatt_set (this, local->loc.inode, stbuf);
-                mdc_inode_xatt_set (this, local->loc.inode, dict);
+        *buf = &conf->statfs_cache.buf;
+    }
+unlock:
+    pthread_mutex_unlock(&conf->statfs_cache.lock);
+err:
+    return ret;
+}
+
+static dict_t *
+mdc_prepare_request(xlator_t *this, mdc_local_t *local, dict_t *xdata)
+{
+    if (xdata != NULL) {
+        dict_ref(xdata);
+    }
+
+    if (local == NULL) {
+        return xdata;
+    }
+
+    if (xdata == NULL) {
+        xdata = dict_new();
+        if (xdata == NULL) {
+            local->update_cache = false;
+
+            return NULL;
         }
+    }
+
+    local->update_cache = mdc_load_reqs(this, xdata);
+
+    return xdata;
+}
+
+int
+mdc_statfs_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+               int32_t op_ret, int32_t op_errno, struct statvfs *buf,
+               dict_t *xdata)
+{
+    struct mdc_conf *conf = this->private;
+    mdc_local_t *local = NULL;
+
+    local = frame->local;
+    if (!local)
+        goto out;
+
+    if (op_ret != 0) {
+        if ((op_errno == ENOENT) || (op_errno == ESTALE)) {
+            mdc_inode_iatt_invalidate(this, local->loc.inode);
+        }
+
+        goto out;
+    }
+
+    if (conf && conf->cache_statfs) {
+        mdc_cache_statfs(this, buf);
+    }
+
 out:
-        MDC_STACK_UNWIND (lookup, frame, op_ret, op_errno, inode, stbuf,
-                          dict, postparent);
-        return 0;
+    MDC_STACK_UNWIND(statfs, frame, op_ret, op_errno, buf, xdata);
+
+    return 0;
 }
 
+int
+mdc_statfs(call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xdata)
+{
+    int ret = 0, op_ret = 0, op_errno = 0;
+    struct statvfs *buf = NULL;
+    mdc_local_t *local = NULL;
+    struct mdc_conf *conf = this->private;
+
+    local = mdc_local_get(frame, loc->inode);
+    if (!local) {
+        op_ret = -1;
+        op_errno = ENOMEM;
+        goto out;
+    }
+
+    loc_copy(&local->loc, loc);
+
+    if (!conf) {
+        goto uncached;
+    }
+
+    if (!conf->cache_statfs) {
+        goto uncached;
+    }
+
+    ret = mdc_load_statfs_info_from_cache(this, &buf);
+    if (ret == 0 && buf) {
+        op_ret = 0;
+        op_errno = 0;
+        goto out;
+    }
+
+uncached:
+    STACK_WIND(frame, mdc_statfs_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->statfs, loc, xdata);
+    return 0;
+
+out:
+    MDC_STACK_UNWIND(statfs, frame, op_ret, op_errno, buf, xdata);
+    return 0;
+}
 
 int
-mdc_lookup (call_frame_t *frame, xlator_t *this, loc_t *loc,
-            dict_t *xdata)
+mdc_lookup_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+               int32_t op_ret, int32_t op_errno, inode_t *inode,
+               struct iatt *stbuf, dict_t *dict, struct iatt *postparent)
 {
-        int          ret = 0;
-        struct iatt  stbuf = {0, };
-        struct iatt  postparent = {0, };
-        dict_t      *xattr_rsp = NULL;
-        dict_t      *xattr_alloc = NULL;
-        mdc_local_t *local = NULL;
+    mdc_local_t *local = NULL;
+    struct mdc_conf *conf = this->private;
 
+    local = frame->local;
 
-        local = mdc_local_get (frame);
-        if (!local)
-                goto uncached;
+    if (!local)
+        goto out;
 
-        loc_copy (&local->loc, loc);
+    if (op_ret != 0) {
+        if (op_errno == ENOENT)
+            GF_ATOMIC_INC(conf->mdc_counter.negative_lookup);
 
-	if (!loc->name)
-		/* A nameless discovery is dangerous to serve from cache. We
-		   perform nameless lookup with the intention of
-		   re-establishing an inode "properly"
-		*/
-		goto uncached;
+        if (op_errno == ESTALE) {
+            /* if op_errno is ENOENT, fuse-bridge will unlink the
+             * dentry
+             */
+            if (local->loc.parent)
+                mdc_inode_iatt_invalidate(this, local->loc.parent);
+            else
+                mdc_inode_iatt_invalidate(this, local->loc.inode);
+        }
 
-        ret = mdc_inode_iatt_get (this, loc->inode, &stbuf);
-        if (ret != 0)
-                goto uncached;
+        goto out;
+    }
 
-        if (xdata) {
-                ret = mdc_inode_xatt_get (this, loc->inode, &xattr_rsp);
-                if (ret != 0)
-                        goto uncached;
+    if (local->loc.parent) {
+        mdc_inode_iatt_set(this, local->loc.parent, postparent,
+                           local->incident_time);
+    }
 
-                if (!mdc_xattr_satisfied (this, xdata, xattr_rsp))
-                        goto uncached;
+    if (local->loc.inode) {
+        mdc_inode_iatt_set(this, local->loc.inode, stbuf, local->incident_time);
+        if (local->update_cache) {
+            mdc_inode_xatt_set(this, local->loc.inode, dict);
+        }
+    }
+out:
+    MDC_STACK_UNWIND(lookup, frame, op_ret, op_errno, inode, stbuf, dict,
+                     postparent);
+    return 0;
+}
+
+int
+mdc_lookup(call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xdata)
+{
+    int ret = 0;
+    struct iatt stbuf = {
+        0,
+    };
+    struct iatt postparent = {
+        0,
+    };
+    dict_t *xattr_rsp = NULL;
+    mdc_local_t *local = NULL;
+    struct mdc_conf *conf = this->private;
+
+    local = mdc_local_get(frame, loc->inode);
+    if (!local) {
+        GF_ATOMIC_INC(conf->mdc_counter.stat_miss);
+        goto uncached;
+    }
+
+    loc_copy(&local->loc, loc);
+
+    if (!inode_is_linked(loc->inode)) {
+        GF_ATOMIC_INC(conf->mdc_counter.stat_miss);
+        goto uncached;
+    }
+
+    if (mdc_inode_reset_need_lookup(this, loc->inode)) {
+        GF_ATOMIC_INC(conf->mdc_counter.need_lookup);
+        goto uncached;
+    }
+
+    ret = mdc_inode_iatt_get(this, loc->inode, &stbuf);
+    if (ret != 0) {
+        GF_ATOMIC_INC(conf->mdc_counter.stat_miss);
+        goto uncached;
+    }
+
+    if (xdata) {
+        ret = mdc_inode_xatt_get(this, loc->inode, &xattr_rsp);
+        if (ret != 0) {
+            GF_ATOMIC_INC(conf->mdc_counter.xattr_miss);
+            goto uncached;
         }
 
-        MDC_STACK_UNWIND (lookup, frame, 0, 0, loc->inode, &stbuf,
-                          xattr_rsp, &postparent);
+        if (!mdc_xattr_satisfied(this, xdata, xattr_rsp)) {
+            GF_ATOMIC_INC(conf->mdc_counter.xattr_miss);
+            goto uncached;
+        }
+    }
 
-        if (xattr_rsp)
-                dict_unref (xattr_rsp);
+    GF_ATOMIC_INC(conf->mdc_counter.stat_hit);
+    MDC_STACK_UNWIND(lookup, frame, 0, 0, loc->inode, &stbuf, xattr_rsp,
+                     &postparent);
 
-        return 0;
+    if (xattr_rsp)
+        dict_unref(xattr_rsp);
+
+    return 0;
 
 uncached:
-	if (!xdata)
-		xdata = xattr_alloc = dict_new ();
-	if (xdata)
-		mdc_load_reqs (this, xdata);
-
-        STACK_WIND (frame, mdc_lookup_cbk, FIRST_CHILD (this),
-                    FIRST_CHILD (this)->fops->lookup, loc, xdata);
-
-        if (xattr_rsp)
-                dict_unref (xattr_rsp);
-	if (xattr_alloc)
-		dict_unref (xattr_alloc);
-        return 0;
-}
+    xdata = mdc_prepare_request(this, local, xdata);
+
+    STACK_WIND(frame, mdc_lookup_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->lookup, loc, xdata);
 
+    if (xattr_rsp)
+        dict_unref(xattr_rsp);
+
+    if (xdata != NULL) {
+        dict_unref(xdata);
+    }
+
+    return 0;
+}
 
 int
-mdc_stat_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-              int32_t op_ret, int32_t op_errno, struct iatt *buf, dict_t *xdata)
+mdc_stat_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int32_t op_ret,
+             int32_t op_errno, struct iatt *buf, dict_t *xdata)
 {
-        mdc_local_t  *local = NULL;
+    mdc_local_t *local = NULL;
+
+    local = frame->local;
+    if (!local)
+        goto out;
 
-        if (op_ret != 0)
-                goto out;
+    if (op_ret != 0) {
+        if ((op_errno == ESTALE) || (op_errno == ENOENT)) {
+            mdc_inode_iatt_invalidate(this, local->loc.inode);
+        }
 
-        local = frame->local;
-        if (!local)
-                goto out;
+        goto out;
+    }
 
-        mdc_inode_iatt_set (this, local->loc.inode, buf);
+    mdc_inode_iatt_set(this, local->loc.inode, buf, local->incident_time);
+    if (local->update_cache) {
+        mdc_inode_xatt_set(this, local->loc.inode, xdata);
+    }
 
 out:
-        MDC_STACK_UNWIND (stat, frame, op_ret, op_errno, buf, xdata);
+    MDC_STACK_UNWIND(stat, frame, op_ret, op_errno, buf, xdata);
 
-        return 0;
+    return 0;
 }
 
-
 int
-mdc_stat (call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xdata)
+mdc_stat(call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xdata)
 {
-        int           ret;
-        struct iatt   stbuf;
-        mdc_local_t  *local = NULL;
+    int ret;
+    struct iatt stbuf;
+    mdc_local_t *local = NULL;
+    struct mdc_conf *conf = this->private;
 
-        local = mdc_local_get (frame);
-        if (!local)
-                goto uncached;
+    local = mdc_local_get(frame, loc->inode);
+    if (!local)
+        goto uncached;
 
-        loc_copy (&local->loc, loc);
+    loc_copy(&local->loc, loc);
 
-        ret = mdc_inode_iatt_get (this, loc->inode, &stbuf);
-        if (ret != 0)
-                goto uncached;
+    if (!inode_is_linked(loc->inode)) {
+        GF_ATOMIC_INC(conf->mdc_counter.stat_miss);
+        goto uncached;
+    }
 
-        MDC_STACK_UNWIND (stat, frame, 0, 0, &stbuf, xdata);
+    ret = mdc_inode_iatt_get(this, loc->inode, &stbuf);
+    if (ret != 0)
+        goto uncached;
 
-        return 0;
+    GF_ATOMIC_INC(conf->mdc_counter.stat_hit);
+    MDC_STACK_UNWIND(stat, frame, 0, 0, &stbuf, xdata);
+
+    return 0;
 
 uncached:
-        STACK_WIND (frame, mdc_stat_cbk,
-                    FIRST_CHILD(this), FIRST_CHILD(this)->fops->stat,
-                    loc, xdata);
-        return 0;
-}
+    xdata = mdc_prepare_request(this, local, xdata);
 
+    GF_ATOMIC_INC(conf->mdc_counter.stat_miss);
+    STACK_WIND(frame, mdc_stat_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->stat, loc, xdata);
+
+    if (xdata != NULL) {
+        dict_unref(xdata);
+    }
+
+    return 0;
+}
 
 int
-mdc_fstat_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-               int32_t op_ret, int32_t op_errno, struct iatt *buf,
-               dict_t *xdata)
+mdc_fstat_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int32_t op_ret,
+              int32_t op_errno, struct iatt *buf, dict_t *xdata)
 {
-        mdc_local_t  *local = NULL;
+    mdc_local_t *local = NULL;
 
-        if (op_ret != 0)
-                goto out;
+    local = frame->local;
+    if (!local)
+        goto out;
 
-        local = frame->local;
-        if (!local)
-                goto out;
+    if (op_ret != 0) {
+        if ((op_errno == ENOENT) || (op_errno == ESTALE)) {
+            mdc_inode_iatt_invalidate(this, local->fd->inode);
+        }
+
+        goto out;
+    }
 
-        mdc_inode_iatt_set (this, local->fd->inode, buf);
+    mdc_inode_iatt_set(this, local->fd->inode, buf, local->incident_time);
+    if (local->update_cache) {
+        mdc_inode_xatt_set(this, local->fd->inode, xdata);
+    }
 
 out:
-        MDC_STACK_UNWIND (fstat, frame, op_ret, op_errno, buf, xdata);
+    MDC_STACK_UNWIND(fstat, frame, op_ret, op_errno, buf, xdata);
 
-        return 0;
+    return 0;
 }
 
-
 int
-mdc_fstat (call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *xdata)
+mdc_fstat(call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *xdata)
 {
-        int           ret;
-        struct iatt   stbuf;
-        mdc_local_t  *local = NULL;
+    int ret;
+    struct iatt stbuf;
+    mdc_local_t *local = NULL;
+    struct mdc_conf *conf = this->private;
 
-        local = mdc_local_get (frame);
-        if (!local)
-                goto uncached;
+    local = mdc_local_get(frame, fd->inode);
+    if (!local)
+        goto uncached;
 
-        local->fd = fd_ref (fd);
+    local->fd = __fd_ref(fd);
 
-        ret = mdc_inode_iatt_get (this, fd->inode, &stbuf);
-        if (ret != 0)
-                goto uncached;
+    ret = mdc_inode_iatt_get(this, fd->inode, &stbuf);
+    if (ret != 0)
+        goto uncached;
 
-        MDC_STACK_UNWIND (fstat, frame, 0, 0, &stbuf, xdata);
+    GF_ATOMIC_INC(conf->mdc_counter.stat_hit);
+    MDC_STACK_UNWIND(fstat, frame, 0, 0, &stbuf, xdata);
 
-        return 0;
+    return 0;
 
 uncached:
-        STACK_WIND (frame, mdc_fstat_cbk,
-                    FIRST_CHILD(this), FIRST_CHILD(this)->fops->fstat,
-                    fd, xdata);
-        return 0;
-}
+    xdata = mdc_prepare_request(this, local, xdata);
+
+    GF_ATOMIC_INC(conf->mdc_counter.stat_miss);
+    STACK_WIND(frame, mdc_fstat_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->fstat, fd, xdata);
+
+    if (xdata != NULL) {
+        dict_unref(xdata);
+    }
 
+    return 0;
+}
 
 int
-mdc_truncate_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                  int32_t op_ret, int32_t op_errno,
-                  struct iatt *prebuf, struct iatt *postbuf, dict_t *xdata)
+mdc_truncate_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                 int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
+                 struct iatt *postbuf, dict_t *xdata)
 {
-        mdc_local_t  *local = NULL;
+    mdc_local_t *local = NULL;
+
+    local = frame->local;
 
-        local = frame->local;
+    if (!local)
+        goto out;
 
-        if (op_ret != 0)
-                goto out;
+    if (op_ret != 0) {
+        if ((op_errno == ESTALE) || (op_errno == ENOENT))
+            mdc_inode_iatt_invalidate(this, local->loc.inode);
 
-        if (!local)
-                goto out;
+        goto out;
+    }
 
-        mdc_inode_iatt_set_validate(this, local->loc.inode, prebuf, postbuf);
+    mdc_inode_iatt_set_validate(this, local->loc.inode, prebuf, postbuf,
+                                _gf_true, local->incident_time);
 
 out:
-        MDC_STACK_UNWIND (truncate, frame, op_ret, op_errno, prebuf, postbuf,
-                          xdata);
+    MDC_STACK_UNWIND(truncate, frame, op_ret, op_errno, prebuf, postbuf, xdata);
 
-        return 0;
+    return 0;
 }
 
-
 int
-mdc_truncate (call_frame_t *frame, xlator_t *this, loc_t *loc,
-              off_t offset, dict_t *xdata)
+mdc_truncate(call_frame_t *frame, xlator_t *this, loc_t *loc, off_t offset,
+             dict_t *xdata)
 {
-        mdc_local_t  *local = NULL;
-
-        local = mdc_local_get (frame);
+    mdc_local_t *local = NULL;
 
-        local->loc.inode = inode_ref (loc->inode);
+    local = mdc_local_get(frame, loc->inode);
+    if (local != NULL) {
+        local->loc.inode = inode_ref(loc->inode);
+    }
 
-        STACK_WIND (frame, mdc_truncate_cbk,
-                    FIRST_CHILD(this), FIRST_CHILD(this)->fops->truncate,
-                    loc, offset, xdata);
-        return 0;
+    STACK_WIND(frame, mdc_truncate_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->truncate, loc, offset, xdata);
+    return 0;
 }
 
-
 int
-mdc_ftruncate_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                   int32_t op_ret, int32_t op_errno,
-                   struct iatt *prebuf, struct iatt *postbuf, dict_t *xdata)
+mdc_ftruncate_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                  int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
+                  struct iatt *postbuf, dict_t *xdata)
 {
-        mdc_local_t  *local = NULL;
+    mdc_local_t *local = NULL;
+
+    local = frame->local;
 
-        local = frame->local;
+    if (!local)
+        goto out;
 
-        if (op_ret != 0)
-                goto out;
+    if (op_ret != 0) {
+        if ((op_errno == ENOENT) || (op_errno == ESTALE))
+            mdc_inode_iatt_invalidate(this, local->fd->inode);
 
-        if (!local)
-                goto out;
+        goto out;
+    }
 
-        mdc_inode_iatt_set_validate(this, local->fd->inode, prebuf, postbuf);
+    mdc_inode_iatt_set_validate(this, local->fd->inode, prebuf, postbuf,
+                                _gf_true, local->incident_time);
 
 out:
-        MDC_STACK_UNWIND (ftruncate, frame, op_ret, op_errno, prebuf, postbuf,
-                          xdata);
+    MDC_STACK_UNWIND(ftruncate, frame, op_ret, op_errno, prebuf, postbuf,
+                     xdata);
 
-        return 0;
+    return 0;
 }
 
-
 int
-mdc_ftruncate (call_frame_t *frame, xlator_t *this, fd_t *fd,
-               off_t offset, dict_t *xdata)
+mdc_ftruncate(call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
+              dict_t *xdata)
 {
-        mdc_local_t  *local = NULL;
-
-        local = mdc_local_get (frame);
+    mdc_local_t *local = NULL;
 
-        local->fd = fd_ref (fd);
+    local = mdc_local_get(frame, fd->inode);
+    if (local != NULL) {
+        local->fd = __fd_ref(fd);
+    }
 
-        STACK_WIND (frame, mdc_ftruncate_cbk,
-                    FIRST_CHILD(this), FIRST_CHILD(this)->fops->ftruncate,
-                    fd, offset, xdata);
-        return 0;
+    STACK_WIND(frame, mdc_ftruncate_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->ftruncate, fd, offset, xdata);
+    return 0;
 }
 
-
 int
-mdc_mknod_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-               int32_t op_ret, int32_t op_errno, inode_t *inode,
-               struct iatt *buf, struct iatt *preparent,
-               struct iatt *postparent, dict_t *xdata)
+mdc_mknod_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int32_t op_ret,
+              int32_t op_errno, inode_t *inode, struct iatt *buf,
+              struct iatt *preparent, struct iatt *postparent, dict_t *xdata)
 {
-        mdc_local_t *local = NULL;
-
-        local = frame->local;
+    mdc_local_t *local = NULL;
 
-        if (op_ret != 0)
-                goto out;
+    local = frame->local;
 
-        if (!local)
-                goto out;
+    if (!local)
+        goto out;
 
-        if (local->loc.parent) {
-                mdc_inode_iatt_set (this, local->loc.parent, postparent);
+    if (op_ret != 0) {
+        if ((op_errno == ESTALE) || (op_errno == ENOENT)) {
+            mdc_inode_iatt_invalidate(this, local->loc.parent);
         }
 
-        if (local->loc.inode) {
-                mdc_inode_iatt_set (this, local->loc.inode, buf);
-                mdc_inode_xatt_set (this, local->loc.inode, local->xattr);
-        }
+        goto out;
+    }
+
+    if (local->loc.parent) {
+        mdc_inode_iatt_set(this, local->loc.parent, postparent,
+                           local->incident_time);
+    }
+
+    if (local->loc.inode) {
+        mdc_inode_iatt_set(this, local->loc.inode, buf, local->incident_time);
+    }
 out:
-        MDC_STACK_UNWIND (mknod, frame, op_ret, op_errno, inode, buf,
-                          preparent, postparent, xdata);
-        return 0;
+    MDC_STACK_UNWIND(mknod, frame, op_ret, op_errno, inode, buf, preparent,
+                     postparent, xdata);
+    return 0;
 }
 
+int
+mdc_mknod(call_frame_t *frame, xlator_t *this, loc_t *loc, mode_t mode,
+          dev_t rdev, mode_t umask, dict_t *xdata)
+{
+    mdc_local_t *local = NULL;
+
+    local = mdc_local_get(frame, loc->inode);
+    if (local != NULL) {
+        loc_copy(&local->loc, loc);
+        local->xattr = dict_ref(xdata);
+    }
+
+    STACK_WIND(frame, mdc_mknod_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->mknod, loc, mode, rdev, umask, xdata);
+    return 0;
+}
 
 int
-mdc_mknod (call_frame_t *frame, xlator_t *this, loc_t *loc,
-           mode_t mode, dev_t rdev, mode_t umask, dict_t *xdata)
+mdc_mkdir_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int32_t op_ret,
+              int32_t op_errno, inode_t *inode, struct iatt *buf,
+              struct iatt *preparent, struct iatt *postparent, dict_t *xdata)
 {
-        mdc_local_t  *local = NULL;
+    mdc_local_t *local = NULL;
 
-        local = mdc_local_get (frame);
+    local = frame->local;
 
-        loc_copy (&local->loc, loc);
-        local->xattr = dict_ref (xdata);
+    if (!local)
+        goto out;
 
-        STACK_WIND (frame, mdc_mknod_cbk,
-                    FIRST_CHILD(this), FIRST_CHILD(this)->fops->mknod,
-                    loc, mode, rdev, umask, xdata);
-        return 0;
+    if (op_ret != 0) {
+        if ((op_errno == ESTALE) || (op_errno == ENOENT)) {
+            mdc_inode_iatt_invalidate(this, local->loc.parent);
+        }
+
+        goto out;
+    }
+
+    if (local->loc.parent) {
+        mdc_inode_iatt_set(this, local->loc.parent, postparent,
+                           local->incident_time);
+    }
+
+    if (local->loc.inode) {
+        mdc_inode_iatt_set(this, local->loc.inode, buf, local->incident_time);
+    }
+out:
+    MDC_STACK_UNWIND(mkdir, frame, op_ret, op_errno, inode, buf, preparent,
+                     postparent, xdata);
+    return 0;
 }
 
+int
+mdc_mkdir(call_frame_t *frame, xlator_t *this, loc_t *loc, mode_t mode,
+          mode_t umask, dict_t *xdata)
+{
+    mdc_local_t *local = NULL;
+
+    local = mdc_local_get(frame, loc->inode);
+    if (local != NULL) {
+        loc_copy(&local->loc, loc);
+        local->xattr = dict_ref(xdata);
+    }
+
+    STACK_WIND(frame, mdc_mkdir_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->mkdir, loc, mode, umask, xdata);
+    return 0;
+}
 
 int
-mdc_mkdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-               int32_t op_ret, int32_t op_errno, inode_t *inode,
-               struct iatt *buf, struct iatt *preparent,
+mdc_unlink_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+               int32_t op_ret, int32_t op_errno, struct iatt *preparent,
                struct iatt *postparent, dict_t *xdata)
 {
-        mdc_local_t *local = NULL;
+    mdc_local_t *local = NULL;
 
-        local = frame->local;
+    local = frame->local;
 
-        if (op_ret != 0)
-                goto out;
+    if (!local)
+        goto out;
 
-        if (!local)
-                goto out;
+    if (op_ret != 0) {
+        /* if errno is ESTALE, parent is not present, which implies even
+         * child is not present. Also, man 2 unlink states unlink can
+         * return ENOENT if a component in pathname does not
+         * exist or is a dangling symbolic link. So, invalidate both
+         * parent and child for both errno
+         */
 
-        if (local->loc.parent) {
-                mdc_inode_iatt_set (this, local->loc.parent, postparent);
+        if ((op_errno == ENOENT) || (op_errno == ESTALE)) {
+            mdc_inode_iatt_invalidate(this, local->loc.inode);
+            mdc_inode_iatt_invalidate(this, local->loc.parent);
         }
 
-        if (local->loc.inode) {
-                mdc_inode_iatt_set (this, local->loc.inode, buf);
-                mdc_inode_xatt_set (this, local->loc.inode, local->xattr);
-        }
+        goto out;
+    }
+
+    if (local->loc.parent) {
+        mdc_inode_iatt_set(this, local->loc.parent, postparent,
+                           local->incident_time);
+    }
+
+    if (local->loc.inode) {
+        mdc_inode_iatt_set(this, local->loc.inode, NULL, local->incident_time);
+    }
+
 out:
-        MDC_STACK_UNWIND (mkdir, frame, op_ret, op_errno, inode, buf,
-                          preparent, postparent, xdata);
-        return 0;
+    MDC_STACK_UNWIND(unlink, frame, op_ret, op_errno, preparent, postparent,
+                     xdata);
+    return 0;
 }
 
-
 int
-mdc_mkdir (call_frame_t *frame, xlator_t *this, loc_t *loc,
-           mode_t mode, mode_t umask, dict_t *xdata)
+mdc_unlink(call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t xflag,
+           dict_t *xdata)
 {
-        mdc_local_t  *local = NULL;
+    mdc_local_t *local = NULL;
 
-        local = mdc_local_get (frame);
+    local = mdc_local_get(frame, loc->inode);
+    if (local != NULL) {
+        loc_copy(&local->loc, loc);
+    }
 
-        loc_copy (&local->loc, loc);
-        local->xattr = dict_ref (xdata);
-
-        STACK_WIND (frame, mdc_mkdir_cbk,
-                    FIRST_CHILD(this), FIRST_CHILD(this)->fops->mkdir,
-                    loc, mode, umask, xdata);
-        return 0;
+    STACK_WIND(frame, mdc_unlink_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->unlink, loc, xflag, xdata);
+    return 0;
 }
 
-
 int
-mdc_unlink_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                int32_t op_ret, int32_t op_errno,
-                struct iatt *preparent, struct iatt *postparent, dict_t *xdata)
+mdc_rmdir_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int32_t op_ret,
+              int32_t op_errno, struct iatt *preparent, struct iatt *postparent,
+              dict_t *xdata)
 {
-        mdc_local_t *local = NULL;
+    mdc_local_t *local = NULL;
 
-        local = frame->local;
+    local = frame->local;
 
-        if (op_ret != 0)
-                goto out;
+    if (!local)
+        goto out;
 
-        if (!local)
-                goto out;
+    if (op_ret != 0) {
+        /* if errno is ESTALE, parent is not present, which implies even
+         * child is not present. Also, man 2 rmdir states rmdir can
+         * return ENOENT if a directory component in pathname does not
+         * exist or is a dangling symbolic link. So, invalidate both
+         * parent and child for both errno
+         */
 
-        if (local->loc.parent) {
-                mdc_inode_iatt_set (this, local->loc.parent, postparent);
+        if ((op_errno == ESTALE) || (op_errno == ENOENT)) {
+            mdc_inode_iatt_invalidate(this, local->loc.inode);
+            mdc_inode_iatt_invalidate(this, local->loc.parent);
         }
 
-        if (local->loc.inode) {
-                mdc_inode_iatt_set (this, local->loc.inode, NULL);
-        }
+        goto out;
+    }
+
+    if (local->loc.parent) {
+        mdc_inode_iatt_set(this, local->loc.parent, postparent,
+                           local->incident_time);
+    }
 
 out:
-        MDC_STACK_UNWIND (unlink, frame, op_ret, op_errno,
-                          preparent, postparent, xdata);
-        return 0;
+    MDC_STACK_UNWIND(rmdir, frame, op_ret, op_errno, preparent, postparent,
+                     xdata);
+    return 0;
 }
 
-
 int
-mdc_unlink (call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t xflag,
-            dict_t *xdata)
+mdc_rmdir(call_frame_t *frame, xlator_t *this, loc_t *loc, int flag,
+          dict_t *xdata)
 {
-        mdc_local_t  *local = NULL;
+    mdc_local_t *local = NULL;
 
-        local = mdc_local_get (frame);
+    local = mdc_local_get(frame, loc->inode);
+    if (local != NULL) {
+        loc_copy(&local->loc, loc);
+    }
 
-        loc_copy (&local->loc, loc);
-
-        STACK_WIND (frame, mdc_unlink_cbk,
-                    FIRST_CHILD(this), FIRST_CHILD(this)->fops->unlink,
-                    loc, xflag, xdata);
-        return 0;
+    STACK_WIND(frame, mdc_rmdir_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->rmdir, loc, flag, xdata);
+    return 0;
 }
 
-
 int
-mdc_rmdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                int32_t op_ret, int32_t op_errno,
-                struct iatt *preparent, struct iatt *postparent, dict_t *xdata)
+mdc_symlink_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                int32_t op_ret, int32_t op_errno, inode_t *inode,
+                struct iatt *buf, struct iatt *preparent,
+                struct iatt *postparent, dict_t *xdata)
 {
-        mdc_local_t *local = NULL;
+    mdc_local_t *local = NULL;
 
-        local = frame->local;
+    local = frame->local;
 
-        if (op_ret != 0)
-                goto out;
+    if (!local)
+        goto out;
 
-        if (!local)
-                goto out;
-
-        if (local->loc.parent) {
-                mdc_inode_iatt_set (this, local->loc.parent, postparent);
+    if (op_ret != 0) {
+        if ((op_errno == ESTALE) || (op_errno == ENOENT)) {
+            mdc_inode_iatt_invalidate(this, local->loc.parent);
         }
 
+        goto out;
+    }
+
+    if (local->loc.parent) {
+        mdc_inode_iatt_set(this, local->loc.parent, postparent,
+                           local->incident_time);
+    }
+
+    if (local->loc.inode) {
+        mdc_inode_iatt_set(this, local->loc.inode, buf, local->incident_time);
+    }
 out:
-        MDC_STACK_UNWIND (rmdir, frame, op_ret, op_errno,
-                          preparent, postparent, xdata);
-        return 0;
+    MDC_STACK_UNWIND(symlink, frame, op_ret, op_errno, inode, buf, preparent,
+                     postparent, xdata);
+    return 0;
 }
 
-
 int
-mdc_rmdir (call_frame_t *frame, xlator_t *this, loc_t *loc, int flag,
-           dict_t *xdata)
+mdc_symlink(call_frame_t *frame, xlator_t *this, const char *linkname,
+            loc_t *loc, mode_t umask, dict_t *xdata)
 {
-        mdc_local_t  *local = NULL;
+    mdc_local_t *local = NULL;
+    char *name;
 
-        local = mdc_local_get (frame);
+    name = gf_strdup(linkname);
+    if (name == NULL) {
+        goto wind;
+    }
+    local = mdc_local_get(frame, loc->inode);
+    if (local == NULL) {
+        GF_FREE(name);
+        goto wind;
+    }
 
-        loc_copy (&local->loc, loc);
+    loc_copy(&local->loc, loc);
+    local->linkname = name;
 
-        STACK_WIND (frame, mdc_rmdir_cbk,
-                    FIRST_CHILD(this), FIRST_CHILD(this)->fops->rmdir,
-                    loc, flag, xdata);
-        return 0;
+wind:
+    STACK_WIND(frame, mdc_symlink_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->symlink, linkname, loc, umask, xdata);
+    return 0;
 }
 
-
 int
-mdc_symlink_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                 int32_t op_ret, int32_t op_errno, inode_t *inode,
-                 struct iatt *buf, struct iatt *preparent,
-                 struct iatt *postparent, dict_t *xdata)
+mdc_rename_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+               int32_t op_ret, int32_t op_errno, struct iatt *buf,
+               struct iatt *preoldparent, struct iatt *postoldparent,
+               struct iatt *prenewparent, struct iatt *postnewparent,
+               dict_t *xdata)
 {
-        mdc_local_t *local = NULL;
+    mdc_local_t *local = NULL;
 
-        local = frame->local;
+    local = frame->local;
+    if (!local)
+        goto out;
 
-        if (op_ret != 0)
-                goto out;
+    if (op_ret != 0) {
+        if ((op_errno == ESTALE) || (op_errno == ENOENT)) {
+            mdc_inode_iatt_invalidate(this, local->loc.inode);
+            mdc_inode_iatt_invalidate(this, local->loc2.parent);
+        }
 
-        if (!local)
-                goto out;
+        goto out;
+    }
 
-        if (local->loc.parent) {
-                mdc_inode_iatt_set (this, local->loc.parent, postparent);
-        }
+    if (local->loc.parent) {
+        mdc_inode_iatt_set(this, local->loc.parent, postoldparent,
+                           local->incident_time);
+    }
 
-        if (local->loc.inode) {
-                mdc_inode_iatt_set (this, local->loc.inode, buf);
-        }
+    if (local->loc.inode) {
+        /* TODO: fix dht_rename() not to return linkfile
+           attributes before setting attributes here
+        */
+
+        mdc_inode_iatt_set(this, local->loc.inode, NULL, local->incident_time);
+    }
+
+    if (local->loc2.parent) {
+        mdc_inode_iatt_set(this, local->loc2.parent, postnewparent,
+                           local->incident_time);
+    }
 out:
-        MDC_STACK_UNWIND (symlink, frame, op_ret, op_errno, inode, buf,
-                          preparent, postparent, xdata);
-        return 0;
+    MDC_STACK_UNWIND(rename, frame, op_ret, op_errno, buf, preoldparent,
+                     postoldparent, prenewparent, postnewparent, xdata);
+    return 0;
 }
 
-
 int
-mdc_symlink (call_frame_t *frame, xlator_t *this, const char *linkname,
-             loc_t *loc, mode_t umask, dict_t *xdata)
+mdc_rename(call_frame_t *frame, xlator_t *this, loc_t *oldloc, loc_t *newloc,
+           dict_t *xdata)
 {
-        mdc_local_t  *local = NULL;
-
-        local = mdc_local_get (frame);
+    mdc_local_t *local = NULL;
 
-        loc_copy (&local->loc, loc);
+    local = mdc_local_get(frame, oldloc->inode);
+    if (local != NULL) {
+        loc_copy(&local->loc, oldloc);
+        loc_copy(&local->loc2, newloc);
+    }
 
-        local->linkname = gf_strdup (linkname);
-
-        STACK_WIND (frame, mdc_symlink_cbk,
-                    FIRST_CHILD(this), FIRST_CHILD(this)->fops->symlink,
-                    linkname, loc, umask, xdata);
-        return 0;
+    STACK_WIND(frame, mdc_rename_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->rename, oldloc, newloc, xdata);
+    return 0;
 }
 
-
 int
-mdc_rename_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                int32_t op_ret, int32_t op_errno, struct iatt *buf,
-                struct iatt *preoldparent, struct iatt *postoldparent,
-                struct iatt *prenewparent, struct iatt *postnewparent,
-                dict_t *xdata)
+mdc_link_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int32_t op_ret,
+             int32_t op_errno, inode_t *inode, struct iatt *buf,
+             struct iatt *preparent, struct iatt *postparent, dict_t *xdata)
 {
-        mdc_local_t *local = NULL;
+    mdc_local_t *local = NULL;
 
-        local = frame->local;
+    local = frame->local;
 
-        if (op_ret != 0)
-                goto out;
+    if (!local)
+        goto out;
 
-        if (!local)
-                goto out;
-
-        if (local->loc.parent) {
-                mdc_inode_iatt_set (this, local->loc.parent, postoldparent);
+    if (op_ret != 0) {
+        if ((op_errno == ENOENT) || (op_errno == ESTALE)) {
+            mdc_inode_iatt_invalidate(this, local->loc.inode);
+            mdc_inode_iatt_invalidate(this, local->loc2.parent);
         }
 
-        if (local->loc.inode) {
-		/* TODO: fix dht_rename() not to return linkfile
-		   attributes before setting attributes here
-		*/
+        goto out;
+    }
 
-		mdc_inode_iatt_set (this, local->loc.inode, NULL);
-        }
+    if (local->loc.inode) {
+        mdc_inode_iatt_set(this, local->loc.inode, buf, local->incident_time);
+    }
 
-        if (local->loc2.parent) {
-                mdc_inode_iatt_set (this, local->loc2.parent, postnewparent);
-        }
+    if (local->loc2.parent) {
+        mdc_inode_iatt_set(this, local->loc2.parent, postparent,
+                           local->incident_time);
+    }
 out:
-        MDC_STACK_UNWIND (rename, frame, op_ret, op_errno, buf,
-                          preoldparent, postoldparent, prenewparent,
-                          postnewparent, xdata);
-        return 0;
+    MDC_STACK_UNWIND(link, frame, op_ret, op_errno, inode, buf, preparent,
+                     postparent, xdata);
+    return 0;
 }
 
-
 int
-mdc_rename (call_frame_t *frame, xlator_t *this,
-            loc_t *oldloc, loc_t *newloc, dict_t *xdata)
+mdc_link(call_frame_t *frame, xlator_t *this, loc_t *oldloc, loc_t *newloc,
+         dict_t *xdata)
 {
-        mdc_local_t  *local = NULL;
+    mdc_local_t *local = NULL;
 
-        local = mdc_local_get (frame);
+    local = mdc_local_get(frame, oldloc->inode);
+    if (local != NULL) {
+        loc_copy(&local->loc, oldloc);
+        loc_copy(&local->loc2, newloc);
+    }
 
-        loc_copy (&local->loc, oldloc);
-        loc_copy (&local->loc2, newloc);
-
-        STACK_WIND (frame, mdc_rename_cbk,
-                    FIRST_CHILD(this), FIRST_CHILD(this)->fops->rename,
-                    oldloc, newloc, xdata);
-        return 0;
+    STACK_WIND(frame, mdc_link_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->link, oldloc, newloc, xdata);
+    return 0;
 }
 
-
 int
-mdc_link_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-              int32_t op_ret, int32_t op_errno, inode_t *inode, struct iatt *buf,
-              struct iatt *preparent, struct iatt *postparent, dict_t *xdata)
+mdc_create_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+               int32_t op_ret, int32_t op_errno, fd_t *fd, inode_t *inode,
+               struct iatt *buf, struct iatt *preparent,
+               struct iatt *postparent, dict_t *xdata)
 {
-        mdc_local_t *local = NULL;
+    mdc_local_t *local = NULL;
 
-        local = frame->local;
+    local = frame->local;
 
-        if (op_ret != 0)
-                goto out;
+    if (!local)
+        goto out;
 
-        if (!local)
-                goto out;
-
-        if (local->loc.inode) {
-                mdc_inode_iatt_set (this, local->loc.inode, buf);
+    if (op_ret != 0) {
+        if ((op_errno == ESTALE) || (op_errno == ENOENT)) {
+            mdc_inode_iatt_invalidate(this, local->loc.parent);
         }
 
-        if (local->loc2.parent) {
-                mdc_inode_iatt_set (this, local->loc2.parent, postparent);
-        }
+        goto out;
+    }
+
+    if (local->loc.parent) {
+        mdc_inode_iatt_set(this, local->loc.parent, postparent,
+                           local->incident_time);
+    }
+
+    if (local->loc.inode) {
+        mdc_inode_iatt_set(this, inode, buf, local->incident_time);
+    }
 out:
-        MDC_STACK_UNWIND (link, frame, op_ret, op_errno, inode, buf,
-                          preparent, postparent, xdata);
-        return 0;
+    MDC_STACK_UNWIND(create, frame, op_ret, op_errno, fd, inode, buf, preparent,
+                     postparent, xdata);
+    return 0;
 }
 
-
 int
-mdc_link (call_frame_t *frame, xlator_t *this,
-          loc_t *oldloc, loc_t *newloc, dict_t *xdata)
+mdc_create(call_frame_t *frame, xlator_t *this, loc_t *loc, int flags,
+           mode_t mode, mode_t umask, fd_t *fd, dict_t *xdata)
 {
-        mdc_local_t  *local = NULL;
-
-        local = mdc_local_get (frame);
+    mdc_local_t *local = NULL;
 
-        loc_copy (&local->loc, oldloc);
-        loc_copy (&local->loc2, newloc);
+    local = mdc_local_get(frame, loc->inode);
+    if (local != NULL) {
+        loc_copy(&local->loc, loc);
+        local->xattr = dict_ref(xdata);
+    }
 
-        STACK_WIND (frame, mdc_link_cbk,
-                    FIRST_CHILD(this), FIRST_CHILD(this)->fops->link,
-                    oldloc, newloc, xdata);
-        return 0;
+    STACK_WIND(frame, mdc_create_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->create, loc, flags, mode, umask, fd,
+               xdata);
+    return 0;
 }
 
-
-int
-mdc_create_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                int32_t op_ret, int32_t op_errno, fd_t *fd, inode_t *inode,
-                struct iatt *buf, struct iatt *preparent,
-                struct iatt *postparent, dict_t *xdata)
+static int
+mdc_open_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int32_t op_ret,
+             int32_t op_errno, fd_t *fd, dict_t *xdata)
 {
-        mdc_local_t *local = NULL;
+    mdc_local_t *local = NULL;
 
-        local = frame->local;
+    local = frame->local;
 
-        if (op_ret != 0)
-                goto out;
+    if (!local)
+        goto out;
 
-        if (!local)
-                goto out;
+    if (op_ret != 0) {
+        if ((op_errno == ESTALE) || (op_errno == ENOENT))
+            mdc_inode_iatt_invalidate(this, local->loc.inode);
+        goto out;
+    }
 
-        if (local->loc.parent) {
-                mdc_inode_iatt_set (this, local->loc.parent, postparent);
-        }
+    if (local->fd->flags & O_TRUNC) {
+        /* O_TRUNC modifies file size. Hence invalidate the
+         * cache entry to fetch latest attributes. */
+        mdc_inode_iatt_invalidate(this, local->fd->inode);
+    }
 
-        if (local->loc.inode) {
-                mdc_inode_iatt_set (this, inode, buf);
-                mdc_inode_xatt_set (this, local->loc.inode, local->xattr);
-        }
 out:
-        MDC_STACK_UNWIND (create, frame, op_ret, op_errno, fd, inode, buf,
-                          preparent, postparent, xdata);
-        return 0;
+    MDC_STACK_UNWIND(open, frame, op_ret, op_errno, fd, xdata);
+    return 0;
 }
 
-
-int
-mdc_create (call_frame_t *frame, xlator_t *this, loc_t *loc, int flags,
-            mode_t mode, mode_t umask, fd_t *fd, dict_t *xdata)
+static int
+mdc_open(call_frame_t *frame, xlator_t *this, loc_t *loc, int flags, fd_t *fd,
+         dict_t *xdata)
 {
-        mdc_local_t  *local = NULL;
+    mdc_local_t *local = NULL;
 
-        local = mdc_local_get (frame);
+    if (!fd || !IA_ISREG(fd->inode->ia_type) || !(fd->flags & O_TRUNC)) {
+        goto out;
+    }
 
-        loc_copy (&local->loc, loc);
-        local->xattr = dict_ref (xdata);
+    local = mdc_local_get(frame, loc->inode);
+    if (local != NULL) {
+        local->fd = __fd_ref(fd);
+    }
 
-        STACK_WIND (frame, mdc_create_cbk,
-                    FIRST_CHILD(this), FIRST_CHILD(this)->fops->create,
-                    loc, flags, mode, umask, fd, xdata);
-        return 0;
+out:
+    STACK_WIND(frame, mdc_open_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->open, loc, flags, fd, xdata);
+    return 0;
 }
 
-
 int
-mdc_readv_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-               int32_t op_ret, int32_t op_errno,
-               struct iovec *vector, int32_t count,
-               struct iatt *stbuf, struct iobref *iobref, dict_t *xdata)
+mdc_readv_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int32_t op_ret,
+              int32_t op_errno, struct iovec *vector, int32_t count,
+              struct iatt *stbuf, struct iobref *iobref, dict_t *xdata)
 {
-        mdc_local_t  *local = NULL;
-
-        local = frame->local;
+    mdc_local_t *local = NULL;
 
-        if (op_ret < 0)
-                goto out;
+    local = frame->local;
+    if (!local)
+        goto out;
 
-        if (!local)
-                goto out;
+    if (op_ret < 0) {
+        if ((op_errno == ENOENT) || (op_errno == ESTALE))
+            mdc_inode_iatt_invalidate(this, local->fd->inode);
+        goto out;
+    }
 
-        mdc_inode_iatt_set (this, local->fd->inode, stbuf);
+    mdc_inode_iatt_set(this, local->fd->inode, stbuf, local->incident_time);
 
 out:
-        MDC_STACK_UNWIND (readv, frame, op_ret, op_errno, vector, count,
-                          stbuf, iobref, xdata);
+    MDC_STACK_UNWIND(readv, frame, op_ret, op_errno, vector, count, stbuf,
+                     iobref, xdata);
 
-        return 0;
+    return 0;
 }
 
-
 int
-mdc_readv (call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
-           off_t offset, uint32_t flags, dict_t *xdata)
+mdc_readv(call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
+          off_t offset, uint32_t flags, dict_t *xdata)
 {
-        mdc_local_t  *local = NULL;
+    mdc_local_t *local = NULL;
 
-        local = mdc_local_get (frame);
+    local = mdc_local_get(frame, fd->inode);
+    if (local != NULL) {
+        local->fd = __fd_ref(fd);
+    }
 
-        local->fd = fd_ref (fd);
-
-        STACK_WIND (frame, mdc_readv_cbk,
-                    FIRST_CHILD(this), FIRST_CHILD(this)->fops->readv,
-                    fd, size, offset, flags, xdata);
-        return 0;
+    STACK_WIND(frame, mdc_readv_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->readv, fd, size, offset, flags, xdata);
+    return 0;
 }
 
-
 int
-mdc_writev_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-               int32_t op_ret, int32_t op_errno,
-               struct iatt *prebuf, struct iatt *postbuf, dict_t *xdata)
+mdc_writev_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+               int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
+               struct iatt *postbuf, dict_t *xdata)
 {
-        mdc_local_t  *local = NULL;
+    mdc_local_t *local = NULL;
 
-        local = frame->local;
+    local = frame->local;
+    if (!local)
+        goto out;
 
-        if (op_ret == -1)
-                goto out;
+    if (op_ret == -1) {
+        if ((op_errno == ENOENT) || (op_errno == ESTALE))
+            mdc_inode_iatt_invalidate(this, local->fd->inode);
+        goto out;
+    }
 
-        if (!local)
-                goto out;
-
-        mdc_inode_iatt_set_validate(this, local->fd->inode, prebuf, postbuf);
+    mdc_inode_iatt_set_validate(this, local->fd->inode, prebuf, postbuf,
+                                _gf_true, local->incident_time);
 
 out:
-        MDC_STACK_UNWIND (writev, frame, op_ret, op_errno, prebuf, postbuf,
-                          xdata);
+    MDC_STACK_UNWIND(writev, frame, op_ret, op_errno, prebuf, postbuf, xdata);
 
-        return 0;
+    return 0;
 }
 
-
 int
-mdc_writev (call_frame_t *frame, xlator_t *this, fd_t *fd, struct iovec *vector,
-            int count, off_t offset, uint32_t flags, struct iobref *iobref,
-            dict_t *xdata)
+mdc_writev(call_frame_t *frame, xlator_t *this, fd_t *fd, struct iovec *vector,
+           int count, off_t offset, uint32_t flags, struct iobref *iobref,
+           dict_t *xdata)
 {
-        mdc_local_t  *local = NULL;
+    mdc_local_t *local = NULL;
 
-        local = mdc_local_get (frame);
+    local = mdc_local_get(frame, fd->inode);
+    if (local != NULL) {
+        local->fd = __fd_ref(fd);
+    }
 
-        local->fd = fd_ref (fd);
-
-        STACK_WIND (frame, mdc_writev_cbk,
-                    FIRST_CHILD(this), FIRST_CHILD(this)->fops->writev,
-                    fd, vector, count, offset, flags, iobref, xdata);
-        return 0;
+    STACK_WIND(frame, mdc_writev_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->writev, fd, vector, count, offset,
+               flags, iobref, xdata);
+    return 0;
 }
 
-
 int
-mdc_setattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                 int32_t op_ret, int32_t op_errno,
-                 struct iatt *prebuf, struct iatt *postbuf, dict_t *xdata)
+mdc_setattr_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
+                struct iatt *postbuf, dict_t *xdata)
 {
-        mdc_local_t  *local = NULL;
-
-        local = frame->local;
+    mdc_local_t *local = NULL;
 
-        if (op_ret != 0) {
-		mdc_inode_iatt_set (this, local->loc.inode, NULL);
-                goto out;
-	}
+    local = frame->local;
+    if (!local)
+        goto out;
 
-        if (!local)
-                goto out;
+    if (op_ret != 0) {
+        mdc_inode_iatt_set(this, local->loc.inode, NULL, local->incident_time);
+        goto out;
+    }
 
-	mdc_inode_iatt_set_validate(this, local->loc.inode, prebuf, postbuf);
+    mdc_inode_iatt_set_validate(this, local->loc.inode, prebuf, postbuf,
+                                _gf_true, local->incident_time);
+    mdc_inode_xatt_update(this, local->loc.inode, xdata);
 
 out:
-        MDC_STACK_UNWIND (setattr, frame, op_ret, op_errno, prebuf, postbuf,
-                          xdata);
+    MDC_STACK_UNWIND(setattr, frame, op_ret, op_errno, prebuf, postbuf, xdata);
 
-        return 0;
+    return 0;
 }
 
-
 int
-mdc_setattr (call_frame_t *frame, xlator_t *this, loc_t *loc,
-             struct iatt *stbuf, int valid, dict_t *xdata)
+mdc_setattr(call_frame_t *frame, xlator_t *this, loc_t *loc, struct iatt *stbuf,
+            int valid, dict_t *xdata)
 {
-        mdc_local_t  *local = NULL;
+    mdc_local_t *local = NULL;
+    dict_t *xattr_alloc = NULL;
+    int ret = 0;
+    struct mdc_conf *conf = this->private;
 
-        local = mdc_local_get (frame);
+    local = mdc_local_get(frame, loc->inode);
+    if (local == NULL) {
+        goto wind;
+    }
 
-        loc_copy (&local->loc, loc);
+    loc_copy(&local->loc, loc);
 
-        STACK_WIND (frame, mdc_setattr_cbk,
-                    FIRST_CHILD(this), FIRST_CHILD(this)->fops->setattr,
-                    loc, stbuf, valid, xdata);
-        return 0;
-}
+    if ((valid & GF_SET_ATTR_MODE) && conf->cache_glusterfs_acl) {
+        if (!xdata)
+            xdata = xattr_alloc = dict_new();
+        if (xdata) {
+            ret = dict_set_int8(xdata, GF_POSIX_ACL_ACCESS, 0);
+            if (!ret)
+                ret = dict_set_int8(xdata, GF_POSIX_ACL_DEFAULT, 0);
+            if (ret)
+                mdc_inode_xatt_invalidate(this, local->loc.inode);
+        }
+    }
+
+    if ((valid & GF_SET_ATTR_MODE) && conf->cache_posix_acl) {
+        if (!xdata)
+            xdata = xattr_alloc = dict_new();
+        if (xdata) {
+            ret = dict_set_int8(xdata, POSIX_ACL_ACCESS_XATTR, 0);
+            if (!ret)
+                ret = dict_set_int8(xdata, POSIX_ACL_DEFAULT_XATTR, 0);
+            if (ret)
+                mdc_inode_xatt_invalidate(this, local->loc.inode);
+        }
+    }
 
+wind:
+    STACK_WIND(frame, mdc_setattr_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->setattr, loc, stbuf, valid, xdata);
+
+    if (xattr_alloc)
+        dict_unref(xattr_alloc);
+    return 0;
+}
 
 int
-mdc_fsetattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                  int32_t op_ret, int32_t op_errno,
-                  struct iatt *prebuf, struct iatt *postbuf, dict_t *xdata)
+mdc_fsetattr_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                 int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
+                 struct iatt *postbuf, dict_t *xdata)
 {
-        mdc_local_t  *local = NULL;
-
-        local = frame->local;
+    mdc_local_t *local = NULL;
 
-        if (op_ret != 0)
-                goto out;
+    local = frame->local;
+    if (!local)
+        goto out;
 
-        if (!local)
-                goto out;
+    if (op_ret != 0) {
+        if ((op_errno == ESTALE) || (op_errno == ENOENT))
+            mdc_inode_iatt_invalidate(this, local->fd->inode);
+        goto out;
+    }
 
-        mdc_inode_iatt_set_validate(this, local->fd->inode, prebuf, postbuf);
+    mdc_inode_iatt_set_validate(this, local->fd->inode, prebuf, postbuf,
+                                _gf_true, local->incident_time);
+    mdc_inode_xatt_update(this, local->fd->inode, xdata);
 
 out:
-        MDC_STACK_UNWIND (fsetattr, frame, op_ret, op_errno, prebuf, postbuf,
-                          xdata);
+    MDC_STACK_UNWIND(fsetattr, frame, op_ret, op_errno, prebuf, postbuf, xdata);
 
-        return 0;
+    return 0;
 }
 
-
 int
-mdc_fsetattr (call_frame_t *frame, xlator_t *this, fd_t *fd,
-              struct iatt *stbuf, int valid, dict_t *xdata)
+mdc_fsetattr(call_frame_t *frame, xlator_t *this, fd_t *fd, struct iatt *stbuf,
+             int valid, dict_t *xdata)
 {
-        mdc_local_t  *local = NULL;
+    mdc_local_t *local = NULL;
+    dict_t *xattr_alloc = NULL;
+    int ret = 0;
+    struct mdc_conf *conf = this->private;
 
-        local = mdc_local_get (frame);
+    local = mdc_local_get(frame, fd->inode);
+    if (local == NULL) {
+        goto wind;
+    }
 
-        local->fd = fd_ref (fd);
+    local->fd = __fd_ref(fd);
 
-        STACK_WIND (frame, mdc_fsetattr_cbk,
-                    FIRST_CHILD(this), FIRST_CHILD(this)->fops->fsetattr,
-                    fd, stbuf, valid, xdata);
-        return 0;
-}
+    if ((valid & GF_SET_ATTR_MODE) && conf->cache_glusterfs_acl) {
+        if (!xdata)
+            xdata = xattr_alloc = dict_new();
+        if (xdata) {
+            ret = dict_set_int8(xdata, GF_POSIX_ACL_ACCESS, 0);
+            if (!ret)
+                ret = dict_set_int8(xdata, GF_POSIX_ACL_DEFAULT, 0);
+            if (ret)
+                mdc_inode_xatt_invalidate(this, local->fd->inode);
+        }
+    }
+
+    if ((valid & GF_SET_ATTR_MODE) && conf->cache_posix_acl) {
+        if (!xdata)
+            xdata = xattr_alloc = dict_new();
+        if (xdata) {
+            ret = dict_set_int8(xdata, POSIX_ACL_ACCESS_XATTR, 0);
+            if (!ret)
+                ret = dict_set_int8(xdata, POSIX_ACL_DEFAULT_XATTR, 0);
+            if (ret)
+                mdc_inode_xatt_invalidate(this, local->fd->inode);
+        }
+    }
 
+wind:
+    STACK_WIND(frame, mdc_fsetattr_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->fsetattr, fd, stbuf, valid, xdata);
+
+    if (xattr_alloc)
+        dict_unref(xattr_alloc);
+    return 0;
+}
 
 int
-mdc_fsync_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-               int32_t op_ret, int32_t op_errno,
-               struct iatt *prebuf, struct iatt *postbuf, dict_t *xdata)
+mdc_fsync_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int32_t op_ret,
+              int32_t op_errno, struct iatt *prebuf, struct iatt *postbuf,
+              dict_t *xdata)
 {
-        mdc_local_t  *local = NULL;
+    mdc_local_t *local = NULL;
 
-        local = frame->local;
+    local = frame->local;
+    if (!local)
+        goto out;
 
-        if (op_ret != 0)
-                goto out;
+    if (op_ret != 0) {
+        if ((op_errno == ENOENT) || (op_errno == ESTALE))
+            mdc_inode_iatt_invalidate(this, local->fd->inode);
+        goto out;
+    }
 
-        if (!local)
-                goto out;
-
-        mdc_inode_iatt_set_validate(this, local->fd->inode, prebuf, postbuf);
+    mdc_inode_iatt_set_validate(this, local->fd->inode, prebuf, postbuf,
+                                _gf_true, local->incident_time);
 
 out:
-        MDC_STACK_UNWIND (fsync, frame, op_ret, op_errno, prebuf, postbuf,
-                          xdata);
+    MDC_STACK_UNWIND(fsync, frame, op_ret, op_errno, prebuf, postbuf, xdata);
 
-        return 0;
+    return 0;
 }
 
-
 int
-mdc_fsync (call_frame_t *frame, xlator_t *this, fd_t *fd, int datasync,
-           dict_t *xdata)
+mdc_fsync(call_frame_t *frame, xlator_t *this, fd_t *fd, int datasync,
+          dict_t *xdata)
 {
-        mdc_local_t  *local = NULL;
-
-        local = mdc_local_get (frame);
+    mdc_local_t *local = NULL;
 
-        local->fd = fd_ref (fd);
+    local = mdc_local_get(frame, fd->inode);
+    if (local != NULL) {
+        local->fd = __fd_ref(fd);
+    }
 
-        STACK_WIND (frame, mdc_fsync_cbk,
-                    FIRST_CHILD(this), FIRST_CHILD(this)->fops->fsync,
-                    fd, datasync, xdata);
-        return 0;
+    STACK_WIND(frame, mdc_fsync_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->fsync, fd, datasync, xdata);
+    return 0;
 }
 
-
 int
-mdc_setxattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                  int32_t op_ret, int32_t op_errno, dict_t *xdata)
+mdc_setxattr_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                 int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
-        mdc_local_t  *local = NULL;
+    mdc_local_t *local = NULL;
+    struct iatt prestat = {
+        0,
+    };
+    struct iatt poststat = {
+        0,
+    };
+    int ret = 0;
 
-        local = frame->local;
+    local = frame->local;
+    if (!local)
+        goto out;
 
-        if (op_ret != 0)
-                goto out;
+    if (op_ret != 0) {
+        if ((op_errno == ENOENT) || (op_errno == ESTALE))
+            mdc_inode_iatt_invalidate(this, local->loc.inode);
+        goto out;
+    }
 
-        if (!local)
-                goto out;
+    mdc_inode_xatt_update(this, local->loc.inode, local->xattr);
 
-        mdc_inode_xatt_update (this, local->loc.inode, local->xattr);
+    ret = dict_get_iatt(xdata, GF_PRESTAT, &prestat);
+    if (ret >= 0) {
+        ret = dict_get_iatt(xdata, GF_POSTSTAT, &poststat);
+        mdc_inode_iatt_set_validate(this, local->loc.inode, &prestat, &poststat,
+                                    _gf_true, local->incident_time);
+    }
 
-	mdc_inode_iatt_invalidate (this, local->loc.inode);
+    if (ret < 0)
+        mdc_inode_iatt_invalidate(this, local->loc.inode);
 
 out:
-        MDC_STACK_UNWIND (setxattr, frame, op_ret, op_errno, xdata);
+    MDC_STACK_UNWIND(setxattr, frame, op_ret, op_errno, xdata);
 
-        return 0;
+    return 0;
 }
 
-
 int
-mdc_setxattr (call_frame_t *frame, xlator_t *this, loc_t *loc,
-              dict_t *xattr, int flags, dict_t *xdata)
+mdc_setxattr(call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xattr,
+             int flags, dict_t *xdata)
 {
-        mdc_local_t  *local = NULL;
+    mdc_local_t *local = NULL;
 
-        local = mdc_local_get (frame);
+    local = mdc_local_get(frame, loc->inode);
+    if (local != NULL) {
+        loc_copy(&local->loc, loc);
+        local->xattr = dict_ref(xattr);
+    }
 
-        loc_copy (&local->loc, loc);
-        local->xattr = dict_ref (xattr);
+    STACK_WIND(frame, mdc_setxattr_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->setxattr, loc, xattr, flags, xdata);
 
-        STACK_WIND (frame, mdc_setxattr_cbk,
-                    FIRST_CHILD(this), FIRST_CHILD(this)->fops->setxattr,
-                    loc, xattr, flags, xdata);
-        return 0;
+    return 0;
 }
 
-
 int
-mdc_fsetxattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-		   int32_t op_ret, int32_t op_errno, dict_t *xdata)
+mdc_fsetxattr_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                  int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
-        mdc_local_t  *local = NULL;
-
-        local = frame->local;
-
-        if (op_ret != 0)
-                goto out;
-
-        if (!local)
-                goto out;
+    mdc_local_t *local = NULL;
+    struct iatt prestat = {
+        0,
+    };
+    struct iatt poststat = {
+        0,
+    };
+    int ret = 0;
+
+    local = frame->local;
+    if (!local)
+        goto out;
+
+    if (op_ret != 0) {
+        if ((op_errno == ESTALE) || (op_errno == ENOENT))
+            mdc_inode_iatt_invalidate(this, local->fd->inode);
+        goto out;
+    }
+
+    mdc_inode_xatt_update(this, local->fd->inode, local->xattr);
+
+    ret = dict_get_iatt(xdata, GF_PRESTAT, &prestat);
+    if (ret >= 0) {
+        ret = dict_get_iatt(xdata, GF_POSTSTAT, &poststat);
+        mdc_inode_iatt_set_validate(this, local->fd->inode, &prestat, &poststat,
+                                    _gf_true, local->incident_time);
+    }
+
+    if (ret < 0)
+        mdc_inode_iatt_invalidate(this, local->fd->inode);
 
-        mdc_inode_xatt_update (this, local->fd->inode, local->xattr);
-
-	mdc_inode_iatt_invalidate (this, local->fd->inode);
 out:
-        MDC_STACK_UNWIND (fsetxattr, frame, op_ret, op_errno, xdata);
+    MDC_STACK_UNWIND(fsetxattr, frame, op_ret, op_errno, xdata);
 
-        return 0;
+    return 0;
 }
 
-
 int
-mdc_fsetxattr (call_frame_t *frame, xlator_t *this, fd_t *fd,
-	       dict_t *xattr, int flags, dict_t *xdata)
+mdc_fsetxattr(call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *xattr,
+              int flags, dict_t *xdata)
 {
-        mdc_local_t  *local = NULL;
+    mdc_local_t *local = NULL;
 
-        local = mdc_local_get (frame);
+    local = mdc_local_get(frame, fd->inode);
+    if (local != NULL) {
+        local->fd = __fd_ref(fd);
+        local->xattr = dict_ref(xattr);
+    }
 
-	local->fd = fd_ref (fd);
-        local->xattr = dict_ref (xattr);
+    STACK_WIND(frame, mdc_fsetxattr_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->fsetxattr, fd, xattr, flags, xdata);
 
-        STACK_WIND (frame, mdc_fsetxattr_cbk,
-                    FIRST_CHILD(this), FIRST_CHILD(this)->fops->fsetxattr,
-                    fd, xattr, flags, xdata);
-        return 0;
+    return 0;
 }
 
 int
-mdc_getxattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-		  int32_t op_ret, int32_t op_errno, dict_t *xattr,
-                  dict_t *xdata)
+mdc_getxattr_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                 int32_t op_ret, int32_t op_errno, dict_t *xattr, dict_t *xdata)
 {
-        mdc_local_t  *local = NULL;
+    mdc_local_t *local = NULL;
 
-        if (op_ret < 0)
-                goto out;
+    local = frame->local;
+    if (!local)
+        goto out;
 
-        local = frame->local;
-        if (!local)
-                goto out;
+    if (op_ret < 0) {
+        if ((op_errno == ENOENT) || (op_errno == ESTALE))
+            mdc_inode_iatt_invalidate(this, local->loc.inode);
+        goto out;
+    }
 
-        mdc_inode_xatt_update (this, local->loc.inode, xattr);
+    if (dict_get(xattr, "glusterfs.skip-cache")) {
+        gf_msg(this->name, GF_LOG_DEBUG, 0, 0,
+               "Skipping xattr update due to empty value");
+        goto out;
+    }
+
+    if (local->update_cache) {
+        mdc_inode_xatt_set(this, local->loc.inode, xdata);
+    }
 
 out:
-        MDC_STACK_UNWIND (getxattr, frame, op_ret, op_errno, xattr, xdata);
+    MDC_STACK_UNWIND(getxattr, frame, op_ret, op_errno, xattr, xdata);
 
-        return 0;
+    return 0;
 }
 
-
 int
-mdc_getxattr (call_frame_t *frame, xlator_t *this, loc_t *loc, const char *key,
-              dict_t *xdata)
+mdc_getxattr(call_frame_t *frame, xlator_t *this, loc_t *loc, const char *key,
+             dict_t *xdata)
 {
-        int           ret;
-	int           op_errno = ENODATA;
-        mdc_local_t  *local = NULL;
-	dict_t       *xattr = NULL;
+    int ret;
+    int op_errno = ENODATA;
+    mdc_local_t *local = NULL;
+    dict_t *xattr = NULL;
+    struct mdc_conf *conf = this->private;
+    gf_boolean_t key_satisfied = _gf_false;
 
-        local = mdc_local_get (frame);
-        if (!local)
-                goto uncached;
+    local = mdc_local_get(frame, loc->inode);
+    if (!local) {
+        goto uncached;
+    }
 
-        loc_copy (&local->loc, loc);
+    loc_copy(&local->loc, loc);
 
-	if (!is_mdc_key_satisfied (key))
-		goto uncached;
+    if (!is_mdc_key_satisfied(this, key)) {
+        goto uncached;
+    }
+    key_satisfied = _gf_true;
 
-	ret = mdc_inode_xatt_get (this, loc->inode, &xattr);
-	if (ret != 0)
-		goto uncached;
+    ret = mdc_inode_xatt_get(this, loc->inode, &xattr);
+    if (ret != 0)
+        goto uncached;
 
-	if (!xattr || !dict_get (xattr, (char *)key)) {
-		ret = -1;
-		op_errno = ENODATA;
-	}
+    if (!xattr || !dict_get(xattr, (char *)key)) {
+        ret = -1;
+        op_errno = ENODATA;
+    }
 
-        MDC_STACK_UNWIND (getxattr, frame, ret, op_errno, xattr, xdata);
+    GF_ATOMIC_INC(conf->mdc_counter.xattr_hit);
+    MDC_STACK_UNWIND(getxattr, frame, ret, op_errno, xattr, xdata);
 
-        return 0;
+    if (xattr)
+        dict_unref(xattr);
+
+    return 0;
 
 uncached:
-        STACK_WIND (frame, mdc_getxattr_cbk,
-                    FIRST_CHILD(this), FIRST_CHILD(this)->fops->getxattr,
-                    loc, key, xdata);
-        return 0;
-}
+    if (key_satisfied) {
+        xdata = mdc_prepare_request(this, local, xdata);
+    }
+
+    GF_ATOMIC_INC(conf->mdc_counter.xattr_miss);
+    STACK_WIND(frame, mdc_getxattr_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->getxattr, loc, key, xdata);
+
+    if (key_satisfied && (xdata != NULL)) {
+        dict_unref(xdata);
+    }
 
+    return 0;
+}
 
 int
-mdc_fgetxattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-		   int32_t op_ret, int32_t op_errno, dict_t *xattr,
-                   dict_t *xdata)
+mdc_fgetxattr_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                  int32_t op_ret, int32_t op_errno, dict_t *xattr,
+                  dict_t *xdata)
 {
-        mdc_local_t  *local = NULL;
+    mdc_local_t *local = NULL;
+
+    local = frame->local;
+    if (!local)
+        goto out;
 
-        if (op_ret < 0)
-                goto out;
+    if (op_ret < 0) {
+        if ((op_errno == ENOENT) || (op_errno == ESTALE))
+            mdc_inode_iatt_invalidate(this, local->fd->inode);
+        goto out;
+    }
 
-        local = frame->local;
-        if (!local)
-                goto out;
+    if (dict_get(xattr, "glusterfs.skip-cache")) {
+        gf_msg(this->name, GF_LOG_DEBUG, 0, 0,
+               "Skipping xattr update due to empty value");
+        goto out;
+    }
 
-        mdc_inode_xatt_update (this, local->fd->inode, xattr);
+    if (local->update_cache) {
+        mdc_inode_xatt_set(this, local->fd->inode, xdata);
+    }
 
 out:
-        MDC_STACK_UNWIND (fgetxattr, frame, op_ret, op_errno, xattr, xdata);
+    MDC_STACK_UNWIND(fgetxattr, frame, op_ret, op_errno, xattr, xdata);
 
-        return 0;
+    return 0;
 }
 
-
 int
-mdc_fgetxattr (call_frame_t *frame, xlator_t *this, fd_t *fd, const char *key,
-               dict_t *xdata)
+mdc_fgetxattr(call_frame_t *frame, xlator_t *this, fd_t *fd, const char *key,
+              dict_t *xdata)
 {
-        int           ret;
-        mdc_local_t  *local = NULL;
-	dict_t       *xattr = NULL;
-	int           op_errno = ENODATA;
+    int ret;
+    mdc_local_t *local = NULL;
+    dict_t *xattr = NULL;
+    int op_errno = ENODATA;
+    struct mdc_conf *conf = this->private;
+    gf_boolean_t key_satisfied = _gf_true;
 
-        local = mdc_local_get (frame);
-        if (!local)
-                goto uncached;
+    local = mdc_local_get(frame, fd->inode);
+    if (!local)
+        goto uncached;
 
-        local->fd = fd_ref (fd);
+    local->fd = __fd_ref(fd);
 
-	if (!is_mdc_key_satisfied (key))
-		goto uncached;
+    if (!is_mdc_key_satisfied(this, key)) {
+        key_satisfied = _gf_false;
+        goto uncached;
+    }
 
-	ret = mdc_inode_xatt_get (this, fd->inode, &xattr);
-	if (ret != 0)
-		goto uncached;
+    ret = mdc_inode_xatt_get(this, fd->inode, &xattr);
+    if (ret != 0)
+        goto uncached;
 
-	if (!xattr || !dict_get (xattr, (char *)key)) {
-		ret = -1;
-		op_errno = ENODATA;
-	}
+    if (!xattr || !dict_get(xattr, (char *)key)) {
+        ret = -1;
+        op_errno = ENODATA;
+    }
 
-        MDC_STACK_UNWIND (fgetxattr, frame, ret, op_errno, xattr, xdata);
+    GF_ATOMIC_INC(conf->mdc_counter.xattr_hit);
+    MDC_STACK_UNWIND(fgetxattr, frame, ret, op_errno, xattr, xdata);
 
-        return 0;
+    if (xattr)
+        dict_unref(xattr);
+
+    return 0;
 
 uncached:
-        STACK_WIND (frame, mdc_fgetxattr_cbk,
-                    FIRST_CHILD(this), FIRST_CHILD(this)->fops->fgetxattr,
-                    fd, key, xdata);
-        return 0;
+    if (key_satisfied) {
+        xdata = mdc_prepare_request(this, local, xdata);
+    }
+
+    GF_ATOMIC_INC(conf->mdc_counter.xattr_miss);
+    STACK_WIND(frame, mdc_fgetxattr_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->fgetxattr, fd, key, xdata);
+
+    if (key_satisfied && (xdata != NULL)) {
+        dict_unref(xdata);
+    }
+
+    return 0;
+}
+
+int
+mdc_removexattr_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                    int32_t op_ret, int32_t op_errno, dict_t *xdata)
+{
+    mdc_local_t *local = NULL;
+    struct iatt prestat = {
+        0,
+    };
+    struct iatt poststat = {
+        0,
+    };
+    int ret = 0;
+
+    local = frame->local;
+    if (!local)
+        goto out;
+
+    if (op_ret != 0) {
+        if ((op_errno == ENOENT) || (op_errno == ESTALE))
+            mdc_inode_iatt_invalidate(this, local->loc.inode);
+        goto out;
+    }
+
+    if (local->key)
+        mdc_inode_xatt_unset(this, local->loc.inode, local->key);
+    else
+        mdc_inode_xatt_invalidate(this, local->loc.inode);
+
+    ret = dict_get_iatt(xdata, GF_PRESTAT, &prestat);
+    if (ret >= 0) {
+        ret = dict_get_iatt(xdata, GF_POSTSTAT, &poststat);
+        mdc_inode_iatt_set_validate(this, local->loc.inode, &prestat, &poststat,
+                                    _gf_true, local->incident_time);
+    }
+
+    if (ret < 0)
+        mdc_inode_iatt_invalidate(this, local->loc.inode);
+out:
+    MDC_STACK_UNWIND(removexattr, frame, op_ret, op_errno, xdata);
+
+    return 0;
 }
 
 int
-mdc_removexattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-		     int32_t op_ret, int32_t op_errno, dict_t *xdata)
+mdc_removexattr(call_frame_t *frame, xlator_t *this, loc_t *loc,
+                const char *name, dict_t *xdata)
 {
-        mdc_local_t  *local = NULL;
+    mdc_local_t *local = NULL;
+    int op_errno = ENODATA;
+    int ret = 0;
+    dict_t *xattr = NULL;
+    struct mdc_conf *conf = this->private;
+    char *name2;
 
-        local = frame->local;
+    name2 = gf_strdup(name);
+    if (name2 == NULL) {
+        goto uncached;
+    }
 
-        if (op_ret != 0)
-                goto out;
+    local = mdc_local_get(frame, loc->inode);
+    if (local == NULL) {
+        GF_FREE(name2);
+        goto uncached;
+    }
 
-        if (!local)
-                goto out;
+    loc_copy(&local->loc, loc);
+    local->key = name2;
 
-	if (local->key)
-		mdc_inode_xatt_unset (this, local->loc.inode, local->key);
-	else
-		mdc_inode_xatt_invalidate (this, local->loc.inode);
+    if (!is_mdc_key_satisfied(this, name))
+        goto uncached;
 
-	mdc_inode_iatt_invalidate (this, local->loc.inode);
-out:
-        MDC_STACK_UNWIND (removexattr, frame, op_ret, op_errno, xdata);
+    ret = mdc_inode_xatt_get(this, loc->inode, &xattr);
+    if (ret != 0)
+        goto uncached;
 
-        return 0;
-}
+    GF_ATOMIC_INC(conf->mdc_counter.xattr_hit);
 
+    if (!xattr || !dict_get(xattr, (char *)name)) {
+        ret = -1;
+        op_errno = ENODATA;
 
-int
-mdc_removexattr (call_frame_t *frame, xlator_t *this, loc_t *loc,
-		 const char *name, dict_t *xdata)
-{
-        mdc_local_t  *local = NULL;
+        MDC_STACK_UNWIND(removexattr, frame, ret, op_errno, xdata);
+    } else {
+        STACK_WIND(frame, mdc_removexattr_cbk, FIRST_CHILD(this),
+                   FIRST_CHILD(this)->fops->removexattr, loc, name, xdata);
+    }
 
-        local = mdc_local_get (frame);
+    if (xattr)
+        dict_unref(xattr);
 
-	loc_copy (&local->loc, loc);
+    return 0;
 
-	local->key = gf_strdup (name);
+uncached:
+    GF_ATOMIC_INC(conf->mdc_counter.xattr_miss);
+    STACK_WIND(frame, mdc_removexattr_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->removexattr, loc, name, xdata);
+    return 0;
+}
+
+int
+mdc_fremovexattr_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                     int32_t op_ret, int32_t op_errno, dict_t *xdata)
+{
+    mdc_local_t *local = NULL;
+    struct iatt prestat = {
+        0,
+    };
+    struct iatt poststat = {
+        0,
+    };
+    int ret = 0;
+
+    local = frame->local;
+    if (!local)
+        goto out;
+
+    if (op_ret != 0) {
+        if ((op_errno == ENOENT) || (op_errno == ESTALE))
+            mdc_inode_iatt_invalidate(this, local->fd->inode);
+        goto out;
+    }
+
+    if (local->key)
+        mdc_inode_xatt_unset(this, local->fd->inode, local->key);
+    else
+        mdc_inode_xatt_invalidate(this, local->fd->inode);
+
+    ret = dict_get_iatt(xdata, GF_PRESTAT, &prestat);
+    if (ret >= 0) {
+        ret = dict_get_iatt(xdata, GF_POSTSTAT, &poststat);
+        mdc_inode_iatt_set_validate(this, local->fd->inode, &prestat, &poststat,
+                                    _gf_true, local->incident_time);
+    }
+
+    if (ret < 0)
+        mdc_inode_iatt_invalidate(this, local->fd->inode);
 
-        STACK_WIND (frame, mdc_removexattr_cbk,
-                    FIRST_CHILD(this), FIRST_CHILD(this)->fops->removexattr,
-                    loc, name, xdata);
-        return 0;
-}
+out:
+    MDC_STACK_UNWIND(fremovexattr, frame, op_ret, op_errno, xdata);
 
+    return 0;
+}
 
 int
-mdc_fremovexattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-		      int32_t op_ret, int32_t op_errno, dict_t *xdata)
+mdc_fremovexattr(call_frame_t *frame, xlator_t *this, fd_t *fd,
+                 const char *name, dict_t *xdata)
 {
-        mdc_local_t  *local = NULL;
+    mdc_local_t *local = NULL;
+    int op_errno = ENODATA;
+    int ret = 0;
+    dict_t *xattr = NULL;
+    struct mdc_conf *conf = this->private;
+    char *name2;
 
-        local = frame->local;
+    name2 = gf_strdup(name);
+    if (name2 == NULL) {
+        goto uncached;
+    }
 
-        if (op_ret != 0)
-                goto out;
+    local = mdc_local_get(frame, fd->inode);
+    if (local == NULL) {
+        GF_FREE(name2);
+        goto uncached;
+    }
 
-        if (!local)
-                goto out;
+    local->fd = __fd_ref(fd);
+    local->key = name2;
 
-	if (local->key)
-		mdc_inode_xatt_unset (this, local->fd->inode, local->key);
-	else
-		mdc_inode_xatt_invalidate (this, local->fd->inode);
+    if (!is_mdc_key_satisfied(this, name))
+        goto uncached;
 
-	mdc_inode_iatt_invalidate (this, local->fd->inode);
-out:
-        MDC_STACK_UNWIND (fremovexattr, frame, op_ret, op_errno, xdata);
+    ret = mdc_inode_xatt_get(this, fd->inode, &xattr);
+    if (ret != 0)
+        goto uncached;
 
-        return 0;
+    GF_ATOMIC_INC(conf->mdc_counter.xattr_hit);
+
+    if (!xattr || !dict_get(xattr, (char *)name)) {
+        ret = -1;
+        op_errno = ENODATA;
+
+        MDC_STACK_UNWIND(fremovexattr, frame, ret, op_errno, xdata);
+    } else {
+        STACK_WIND(frame, mdc_fremovexattr_cbk, FIRST_CHILD(this),
+                   FIRST_CHILD(this)->fops->fremovexattr, fd, name, xdata);
+    }
+
+    if (xattr)
+        dict_unref(xattr);
+
+    return 0;
+
+uncached:
+    GF_ATOMIC_INC(conf->mdc_counter.xattr_miss);
+    STACK_WIND(frame, mdc_fremovexattr_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->fremovexattr, fd, name, xdata);
+    return 0;
 }
 
+int32_t
+mdc_opendir_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                int32_t op_ret, int32_t op_errno, fd_t *fd, dict_t *xdata)
+{
+    mdc_local_t *local = NULL;
+
+    local = frame->local;
+    if (!local)
+        goto out;
+
+    if (op_ret == 0)
+        goto out;
+
+    if ((op_errno == ESTALE) || (op_errno == ENOENT))
+        mdc_inode_iatt_invalidate(this, local->loc.inode);
+
+out:
+    MDC_STACK_UNWIND(opendir, frame, op_ret, op_errno, fd, xdata);
+    return 0;
+}
 
 int
-mdc_fremovexattr (call_frame_t *frame, xlator_t *this, fd_t *fd,
-		  const char *name, dict_t *xdata)
+mdc_opendir(call_frame_t *frame, xlator_t *this, loc_t *loc, fd_t *fd,
+            dict_t *xdata)
 {
-        mdc_local_t  *local = NULL;
+    mdc_local_t *local = NULL;
 
-        local = mdc_local_get (frame);
+    local = mdc_local_get(frame, loc->inode);
+    if (local != NULL) {
+        loc_copy(&local->loc, loc);
+    }
 
-	local->fd = fd_ref (fd);
+    /* Tell readdir-ahead to include these keys in xdata when it
+     * internally issues readdirp() in it's opendir_cbk */
+    xdata = mdc_prepare_request(this, local, xdata);
 
-	local->key = gf_strdup (name);
+    STACK_WIND(frame, mdc_opendir_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->opendir, loc, fd, xdata);
 
-        STACK_WIND (frame, mdc_fremovexattr_cbk,
-                    FIRST_CHILD(this), FIRST_CHILD(this)->fops->fremovexattr,
-                    fd, name, xdata);
-        return 0;
-}
+    if (xdata != NULL) {
+        dict_unref(xdata);
+    }
 
+    return 0;
+}
 
 int
-mdc_readdirp_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-		  int op_ret, int op_errno, gf_dirent_t *entries, dict_t *xdata)
+mdc_readdirp_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int op_ret,
+                 int op_errno, gf_dirent_t *entries, dict_t *xdata)
 {
-        gf_dirent_t *entry      = NULL;
+    gf_dirent_t *entry = NULL;
+    mdc_local_t *local = NULL;
+
+    local = frame->local;
+    if (!local)
+        goto unwind;
 
-	if (op_ret <= 0)
-		goto unwind;
+    if (op_ret <= 0) {
+        if ((op_ret == -1) && ((op_errno == ENOENT) || (op_errno == ESTALE)))
+            mdc_inode_iatt_invalidate(this, local->fd->inode);
+        goto unwind;
+    }
 
-        list_for_each_entry (entry, &entries->list, list) {
-                if (!entry->inode)
-			continue;
-                mdc_inode_iatt_set (this, entry->inode, &entry->d_stat);
-                mdc_inode_xatt_set (this, entry->inode, entry->dict);
+    list_for_each_entry(entry, &entries->list, list)
+    {
+        if (!entry->inode)
+            continue;
+        mdc_inode_iatt_set(this, entry->inode, &entry->d_stat,
+                           local->incident_time);
+        if (local->update_cache) {
+            mdc_inode_xatt_set(this, entry->inode, entry->dict);
         }
+    }
 
 unwind:
-	STACK_UNWIND_STRICT (readdirp, frame, op_ret, op_errno, entries, xdata);
-	return 0;
+    MDC_STACK_UNWIND(readdirp, frame, op_ret, op_errno, entries, xdata);
+    return 0;
 }
 
-
 int
-mdc_readdirp (call_frame_t *frame, xlator_t *this, fd_t *fd,
-	      size_t size, off_t offset, dict_t *xdata)
+mdc_readdirp(call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
+             off_t offset, dict_t *xdata)
 {
-	dict_t *xattr_alloc = NULL;
+    mdc_local_t *local = NULL;
+
+    local = mdc_local_get(frame, fd->inode);
+    if (!local)
+        goto out;
+
+    local->fd = __fd_ref(fd);
+
+    xdata = mdc_prepare_request(this, local, xdata);
+
+    STACK_WIND(frame, mdc_readdirp_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->readdirp, fd, size, offset, xdata);
 
-	if (!xdata)
-		xdata = xattr_alloc = dict_new ();
-	if (xdata)
-		mdc_load_reqs (this, xdata);
+    if (xdata != NULL) {
+        dict_unref(xdata);
+    }
 
-	STACK_WIND (frame, mdc_readdirp_cbk,
-		    FIRST_CHILD (this), FIRST_CHILD (this)->fops->readdirp,
-		    fd, size, offset, xdata);
-	if (xattr_alloc)
-		dict_unref (xattr_alloc);
-	return 0;
+    return 0;
+out:
+    MDC_STACK_UNWIND(readdirp, frame, -1, ENOMEM, NULL, NULL);
+    return 0;
 }
 
 int
 mdc_readdir_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int op_ret,
-		int op_errno, gf_dirent_t *entries, dict_t *xdata)
+                int op_errno, gf_dirent_t *entries, dict_t *xdata)
 {
-	STACK_UNWIND_STRICT (readdir, frame, op_ret, op_errno, entries, xdata);
-	return 0;
+    mdc_local_t *local = NULL;
+
+    local = frame->local;
+    if (!local)
+        goto out;
+
+    if (op_ret == 0)
+        goto out;
+
+    if ((op_errno == ESTALE) || (op_errno == ENOENT))
+        mdc_inode_iatt_invalidate(this, local->fd->inode);
+out:
+    MDC_STACK_UNWIND(readdir, frame, op_ret, op_errno, entries, xdata);
+    return 0;
 }
 
 int
-mdc_readdir (call_frame_t *frame, xlator_t *this, fd_t *fd,
-	     size_t size, off_t offset, dict_t *xdata)
+mdc_readdir(call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
+            off_t offset, dict_t *xdata)
 {
-        int need_unref = 0;
-	struct mdc_conf *conf = this->private;
+    mdc_local_t *local = NULL;
+    struct mdc_conf *conf = this->private;
 
-	if (!conf->force_readdirp) {
-		STACK_WIND(frame, mdc_readdir_cbk, FIRST_CHILD(this),
-			   FIRST_CHILD(this)->fops->readdir, fd, size, offset,
-			   xdata);
-		return 0;
-	}
+    local = mdc_local_get(frame, fd->inode);
+    if (!local)
+        goto unwind;
 
-	if (!xdata) {
-                xdata = dict_new ();
-                need_unref = 1;
-        }
+    local->fd = __fd_ref(fd);
+
+    if (!conf->force_readdirp) {
+        STACK_WIND(frame, mdc_readdir_cbk, FIRST_CHILD(this),
+                   FIRST_CHILD(this)->fops->readdir, fd, size, offset, xdata);
+        return 0;
+    }
 
-        if (xdata)
-		mdc_load_reqs (this, xdata);
+    xdata = mdc_prepare_request(this, local, xdata);
 
-	STACK_WIND(frame, mdc_readdirp_cbk, FIRST_CHILD(this),
-		   FIRST_CHILD(this)->fops->readdirp, fd, size, offset,
-		   xdata);
+    STACK_WIND(frame, mdc_readdirp_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->readdirp, fd, size, offset, xdata);
 
-        if (need_unref && xdata)
-                dict_unref (xdata);
+    if (xdata != NULL) {
+        dict_unref(xdata);
+    }
 
-	return 0;
+    return 0;
+unwind:
+    MDC_STACK_UNWIND(readdir, frame, -1, ENOMEM, NULL, NULL);
+    return 0;
 }
 
 int
 mdc_fallocate_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
-                  int32_t op_ret, int32_t op_errno,
-                  struct iatt *prebuf, struct iatt *postbuf, dict_t *xdata)
+                  int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
+                  struct iatt *postbuf, dict_t *xdata)
 {
-        mdc_local_t  *local = NULL;
-
-        local = frame->local;
+    mdc_local_t *local = NULL;
 
-        if (op_ret != 0)
-                goto out;
+    local = frame->local;
+    if (!local)
+        goto out;
 
-        if (!local)
-                goto out;
+    if (op_ret != 0) {
+        if ((op_errno == ENOENT) || (op_errno == ESTALE))
+            mdc_inode_iatt_invalidate(this, local->fd->inode);
+        goto out;
+    }
 
-        mdc_inode_iatt_set_validate(this, local->fd->inode, prebuf, postbuf);
+    mdc_inode_iatt_set_validate(this, local->fd->inode, prebuf, postbuf,
+                                _gf_true, local->incident_time);
 
 out:
-        MDC_STACK_UNWIND (fallocate, frame, op_ret, op_errno, prebuf, postbuf,
-                          xdata);
+    MDC_STACK_UNWIND(fallocate, frame, op_ret, op_errno, prebuf, postbuf,
+                     xdata);
 
-        return 0;
+    return 0;
 }
 
-int mdc_fallocate(call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t mode,
-		  off_t offset, size_t len, dict_t *xdata)
+int
+mdc_fallocate(call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t mode,
+              off_t offset, size_t len, dict_t *xdata)
 {
-	mdc_local_t *local;
+    mdc_local_t *local;
 
-	local = mdc_local_get(frame);
-	local->fd = fd_ref(fd);
+    local = mdc_local_get(frame, fd->inode);
+    if (local != NULL) {
+        local->fd = __fd_ref(fd);
+    }
 
-	STACK_WIND(frame, mdc_fallocate_cbk, FIRST_CHILD(this),
-		   FIRST_CHILD(this)->fops->fallocate, fd, mode, offset, len,
-		   xdata);
+    STACK_WIND(frame, mdc_fallocate_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->fallocate, fd, mode, offset, len,
+               xdata);
 
-	return 0;
+    return 0;
 }
 
 int
 mdc_discard_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
-                int32_t op_ret, int32_t op_errno,
-                struct iatt *prebuf, struct iatt *postbuf, dict_t *xdata)
+                int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
+                struct iatt *postbuf, dict_t *xdata)
 {
-        mdc_local_t  *local = NULL;
+    mdc_local_t *local = NULL;
 
-        local = frame->local;
+    local = frame->local;
+    if (!local)
+        goto out;
 
-        if (op_ret != 0)
-                goto out;
+    if (op_ret != 0) {
+        if ((op_errno == ENOENT) || (op_errno == ESTALE))
+            mdc_inode_iatt_invalidate(this, local->fd->inode);
+        goto out;
+    }
 
-        if (!local)
-                goto out;
-
-        mdc_inode_iatt_set_validate(this, local->fd->inode, prebuf, postbuf);
+    mdc_inode_iatt_set_validate(this, local->fd->inode, prebuf, postbuf,
+                                _gf_true, local->incident_time);
 
 out:
-        MDC_STACK_UNWIND(discard, frame, op_ret, op_errno, prebuf, postbuf,
-                         xdata);
+    MDC_STACK_UNWIND(discard, frame, op_ret, op_errno, prebuf, postbuf, xdata);
 
-        return 0;
+    return 0;
 }
 
-int mdc_discard(call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
-		size_t len, dict_t *xdata)
+int
+mdc_discard(call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
+            size_t len, dict_t *xdata)
 {
-	mdc_local_t *local;
+    mdc_local_t *local;
 
-	local = mdc_local_get(frame);
-	local->fd = fd_ref(fd);
+    local = mdc_local_get(frame, fd->inode);
+    if (local != NULL) {
+        local->fd = __fd_ref(fd);
+    }
 
-	STACK_WIND(frame, mdc_discard_cbk, FIRST_CHILD(this),
-		   FIRST_CHILD(this)->fops->discard, fd, offset, len,
-		   xdata);
+    STACK_WIND(frame, mdc_discard_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->discard, fd, offset, len, xdata);
 
-	return 0;
+    return 0;
 }
 
 int
 mdc_zerofill_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
-                int32_t op_ret, int32_t op_errno,
-                struct iatt *prebuf, struct iatt *postbuf, dict_t *xdata)
+                 int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
+                 struct iatt *postbuf, dict_t *xdata)
+{
+    mdc_local_t *local = NULL;
+
+    local = frame->local;
+    if (!local)
+        goto out;
+
+    if (op_ret != 0) {
+        if ((op_errno == ENOENT) || (op_errno == ESTALE))
+            mdc_inode_iatt_invalidate(this, local->fd->inode);
+        goto out;
+    }
+
+    mdc_inode_iatt_set_validate(this, local->fd->inode, prebuf, postbuf,
+                                _gf_true, local->incident_time);
+
+out:
+    MDC_STACK_UNWIND(zerofill, frame, op_ret, op_errno, prebuf, postbuf, xdata);
+
+    return 0;
+}
+
+int
+mdc_zerofill(call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
+             off_t len, dict_t *xdata)
 {
-        mdc_local_t  *local = NULL;
+    mdc_local_t *local;
+
+    local = mdc_local_get(frame, fd->inode);
+    if (local != NULL) {
+        local->fd = __fd_ref(fd);
+    }
+
+    STACK_WIND(frame, mdc_zerofill_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->zerofill, fd, offset, len, xdata);
+
+    return 0;
+}
 
-        local = frame->local;
+int32_t
+mdc_readlink_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                 int32_t op_ret, int32_t op_errno, const char *path,
+                 struct iatt *buf, dict_t *xdata)
+{
+    mdc_local_t *local = NULL;
 
-        if (op_ret != 0)
-                goto out;
+    local = frame->local;
+    if (!local)
+        goto out;
 
-        if (!local)
-                goto out;
+    if (op_ret == 0)
+        goto out;
 
-        mdc_inode_iatt_set_validate(this, local->fd->inode, prebuf, postbuf);
+    if ((op_errno == ENOENT) || (op_errno == ESTALE))
+        mdc_inode_iatt_invalidate(this, local->loc.inode);
 
 out:
-        MDC_STACK_UNWIND(zerofill, frame, op_ret, op_errno, prebuf, postbuf,
-                         xdata);
+    MDC_STACK_UNWIND(readlink, frame, op_ret, op_errno, path, buf, xdata);
+    return 0;
+}
 
-        return 0;
+int32_t
+mdc_readlink(call_frame_t *frame, xlator_t *this, loc_t *loc, size_t size,
+             dict_t *xdata)
+{
+    mdc_local_t *local = NULL;
+
+    local = mdc_local_get(frame, loc->inode);
+    if (!local)
+        goto unwind;
+
+    loc_copy(&local->loc, loc);
+
+    STACK_WIND(frame, mdc_readlink_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->readlink, loc, size, xdata);
+    return 0;
+
+unwind:
+    MDC_STACK_UNWIND(readlink, frame, -1, ENOMEM, NULL, NULL, NULL);
+    return 0;
 }
 
-int mdc_zerofill(call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
-                off_t len, dict_t *xdata)
+int32_t
+mdc_fsyncdir_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                 int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
-        mdc_local_t *local;
+    mdc_local_t *local = NULL;
 
-        local = mdc_local_get(frame);
-        local->fd = fd_ref(fd);
+    local = frame->local;
+    if (!local)
+        goto out;
 
-        STACK_WIND(frame, mdc_zerofill_cbk, FIRST_CHILD(this),
-                   FIRST_CHILD(this)->fops->zerofill, fd, offset, len,
-                   xdata);
+    if (op_ret == 0)
+        goto out;
 
-        return 0;
+    if ((op_errno == ESTALE) || (op_errno == ENOENT))
+        mdc_inode_iatt_invalidate(this, local->fd->inode);
+
+out:
+    MDC_STACK_UNWIND(fsyncdir, frame, op_ret, op_errno, xdata);
+    return 0;
 }
 
+int32_t
+mdc_fsyncdir(call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t flags,
+             dict_t *xdata)
+{
+    mdc_local_t *local = NULL;
 
-int
-mdc_forget (xlator_t *this, inode_t *inode)
+    local = mdc_local_get(frame, fd->inode);
+    if (!local)
+        goto unwind;
+
+    local->fd = __fd_ref(fd);
+
+    STACK_WIND(frame, mdc_fsyncdir_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->fsyncdir, fd, flags, xdata);
+    return 0;
+
+unwind:
+    MDC_STACK_UNWIND(fsyncdir, frame, -1, ENOMEM, NULL);
+    return 0;
+}
+
+int32_t
+mdc_access_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+               int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
-        mdc_inode_wipe (this, inode);
+    mdc_local_t *local = NULL;
 
-        return 0;
+    local = frame->local;
+    if (!local)
+        goto out;
+
+    if (op_ret == 0)
+        goto out;
+
+    if ((op_errno == ESTALE) || (op_errno == ENOENT))
+        mdc_inode_iatt_invalidate(this, local->loc.inode);
+
+out:
+    MDC_STACK_UNWIND(access, frame, op_ret, op_errno, xdata);
+    return 0;
 }
 
+int32_t
+mdc_access(call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t mask,
+           dict_t *xdata)
+{
+    mdc_local_t *local = NULL;
+
+    local = mdc_local_get(frame, loc->inode);
+    if (!local)
+        goto unwind;
+
+    loc_copy(&local->loc, loc);
+
+    STACK_WIND(frame, mdc_access_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->access, loc, mask, xdata);
+    return 0;
+
+unwind:
+    MDC_STACK_UNWIND(access, frame, -1, ENOMEM, NULL);
+    return 0;
+}
+
+int
+mdc_priv_dump(xlator_t *this)
+{
+    struct mdc_conf *conf = NULL;
+    char key_prefix[GF_DUMP_MAX_BUF_LEN];
+
+    conf = this->private;
+
+    snprintf(key_prefix, GF_DUMP_MAX_BUF_LEN, "%s.%s", this->type, this->name);
+    gf_proc_dump_add_section("%s", key_prefix);
+
+    gf_proc_dump_write("stat_hit_count", "%" PRId64,
+                       GF_ATOMIC_GET(conf->mdc_counter.stat_hit));
+    gf_proc_dump_write("stat_miss_count", "%" PRId64,
+                       GF_ATOMIC_GET(conf->mdc_counter.stat_miss));
+    gf_proc_dump_write("xattr_hit_count", "%" PRId64,
+                       GF_ATOMIC_GET(conf->mdc_counter.xattr_hit));
+    gf_proc_dump_write("xattr_miss_count", "%" PRId64,
+                       GF_ATOMIC_GET(conf->mdc_counter.xattr_miss));
+    gf_proc_dump_write("nameless_lookup_count", "%" PRId64,
+                       GF_ATOMIC_GET(conf->mdc_counter.nameless_lookup));
+    gf_proc_dump_write("negative_lookup_count", "%" PRId64,
+                       GF_ATOMIC_GET(conf->mdc_counter.negative_lookup));
+    gf_proc_dump_write("stat_invalidations_received", "%" PRId64,
+                       GF_ATOMIC_GET(conf->mdc_counter.stat_invals));
+    gf_proc_dump_write("xattr_invalidations_received", "%" PRId64,
+                       GF_ATOMIC_GET(conf->mdc_counter.xattr_invals));
+
+    return 0;
+}
+
+static int32_t
+mdc_dump_metrics(xlator_t *this, int fd)
+{
+    struct mdc_conf *conf = NULL;
+
+    conf = this->private;
+    if (!conf)
+        goto out;
+
+    dprintf(fd, "%s.stat_cache_hit_count %" PRId64 "\n", this->name,
+            GF_ATOMIC_GET(conf->mdc_counter.stat_hit));
+    dprintf(fd, "%s.stat_cache_miss_count %" PRId64 "\n", this->name,
+            GF_ATOMIC_GET(conf->mdc_counter.stat_miss));
+    dprintf(fd, "%s.xattr_cache_hit_count %" PRId64 "\n", this->name,
+            GF_ATOMIC_GET(conf->mdc_counter.xattr_hit));
+    dprintf(fd, "%s.xattr_cache_miss_count %" PRId64 "\n", this->name,
+            GF_ATOMIC_GET(conf->mdc_counter.xattr_miss));
+    dprintf(fd, "%s.nameless_lookup_count %" PRId64 "\n", this->name,
+            GF_ATOMIC_GET(conf->mdc_counter.nameless_lookup));
+    dprintf(fd, "%s.negative_lookup_count %" PRId64 "\n", this->name,
+            GF_ATOMIC_GET(conf->mdc_counter.negative_lookup));
+    dprintf(fd, "%s.stat_cache_invalidations_received %" PRId64 "\n",
+            this->name, GF_ATOMIC_GET(conf->mdc_counter.stat_invals));
+    dprintf(fd, "%s.xattr_cache_invalidations_received %" PRId64 "\n",
+            this->name, GF_ATOMIC_GET(conf->mdc_counter.xattr_invals));
+out:
+    return 0;
+}
+
+int
+mdc_forget(xlator_t *this, inode_t *inode)
+{
+    mdc_inode_wipe(this, inode);
+
+    return 0;
+}
 
 int
-is_strpfx (const char *str1, const char *str2)
+is_strpfx(const char *str1, const char *str2)
 {
-	/* is one of the string a prefix of the other? */
-	int i;
+    /* is one of the string a prefix of the other? */
+    int i;
 
-	for (i = 0; str1[i] == str2[i]; i++) {
-		if (!str1[i] || !str2[i])
-			break;
-	}
+    for (i = 0; str1[i] == str2[i]; i++) {
+        if (!str1[i] || !str2[i])
+            break;
+    }
 
-	return !(str1[i] && str2[i]);
+    return !(str1[i] && str2[i]);
 }
 
+static int
+mdc_key_unload_all(struct mdc_conf *conf)
+{
+    conf->mdc_xattr_str = NULL;
+
+    return 0;
+}
 
 int
-mdc_key_load_set (struct mdc_key *keys, char *pattern, gf_boolean_t val)
+mdc_xattr_list_populate(struct mdc_conf *conf, char *tmp_str)
+{
+    char *mdc_xattr_str = NULL;
+    size_t max_size = 0;
+    int ret = 0;
+
+    max_size = SLEN(
+                   "security.capability,security.selinux,security."
+                   "ima," POSIX_ACL_ACCESS_XATTR "," POSIX_ACL_DEFAULT_XATTR
+                   "," GF_POSIX_ACL_ACCESS "," GF_POSIX_ACL_DEFAULT
+                   ","
+                   "user.swift.metadata,user.DOSATTRIB,user.DosStream.*"
+                   ",user.org.netatalk.Metadata,security.NTACL,"
+                   "user.org.netatalk.ResourceFork") +
+               strlen(tmp_str) + 5; /*Some buffer bytes*/
+
+    mdc_xattr_str = GF_MALLOC(max_size, gf_common_mt_char);
+    GF_CHECK_ALLOC(mdc_xattr_str, ret, out);
+    mdc_xattr_str[0] = '\0';
+
+    if (conf->cache_capability)
+        strcat(mdc_xattr_str, "security.capability,");
+
+    if (conf->cache_selinux)
+        strcat(mdc_xattr_str, "security.selinux,");
+
+    if (conf->cache_ima)
+        strcat(mdc_xattr_str, "security.ima,");
+
+    if (conf->cache_posix_acl)
+        strcat(mdc_xattr_str,
+               POSIX_ACL_ACCESS_XATTR "," POSIX_ACL_DEFAULT_XATTR ",");
+
+    if (conf->cache_glusterfs_acl)
+        strcat(mdc_xattr_str, GF_POSIX_ACL_ACCESS "," GF_POSIX_ACL_DEFAULT ",");
+
+    if (conf->cache_swift_metadata)
+        strcat(mdc_xattr_str, "user.swift.metadata,");
+
+    if (conf->cache_samba_metadata)
+        strcat(mdc_xattr_str,
+               "user.DOSATTRIB,user.DosStream.*,"
+               "user.org.netatalk.Metadata,user.org.netatalk."
+               "ResourceFork,security.NTACL,");
+
+    strcat(mdc_xattr_str, tmp_str);
+
+    LOCK(&conf->lock);
+    {
+        /* This is not freed, else is_mdc_key_satisfied, which is
+         * called by every fop has to take lock, and will lead to
+         * lock contention
+         */
+        conf->mdc_xattr_str = mdc_xattr_str;
+    }
+    UNLOCK(&conf->lock);
+
+out:
+    return ret;
+}
+
+struct set {
+    inode_t *inode;
+    xlator_t *this;
+};
+
+static int
+mdc_inval_xatt(dict_t *d, char *k, data_t *v, void *tmp)
 {
-	struct mdc_key *key = NULL;
+    struct set *tmp1 = NULL;
+    int ret = 0;
+
+    tmp1 = (struct set *)tmp;
+    ret = mdc_inode_xatt_unset(tmp1->this, tmp1->inode, k);
+    return ret;
+}
+
+static int
+mdc_invalidate(xlator_t *this, void *data)
+{
+    struct gf_upcall *up_data = NULL;
+    struct gf_upcall_cache_invalidation *up_ci = NULL;
+    inode_t *inode = NULL;
+    int ret = 0;
+    struct set tmp = {
+        0,
+    };
+    inode_table_t *itable = NULL;
+    struct mdc_conf *conf = this->private;
+    uint64_t gen = 0;
+
+    up_data = (struct gf_upcall *)data;
+
+    if (up_data->event_type != GF_UPCALL_CACHE_INVALIDATION)
+        goto out;
+
+    up_ci = (struct gf_upcall_cache_invalidation *)up_data->data;
+
+    itable = ((xlator_t *)this->graph->top)->itable;
+    inode = inode_find(itable, up_data->gfid);
+    if (!inode) {
+        ret = -1;
+        goto out;
+    }
+
+    if (up_ci->flags & UP_PARENT_DENTRY_FLAGS) {
+        mdc_update_gfid_stat(this, &up_ci->p_stat);
+        if (up_ci->flags & UP_RENAME_FLAGS)
+            mdc_update_gfid_stat(this, &up_ci->oldp_stat);
+    }
+
+    if (up_ci->flags & UP_EXPLICIT_LOOKUP) {
+        mdc_inode_set_need_lookup(this, inode, _gf_true);
+        goto out;
+    }
+
+    if (up_ci->flags &
+        (UP_NLINK | UP_RENAME_FLAGS | UP_FORGET | UP_INVAL_ATTR)) {
+        mdc_inode_iatt_invalidate(this, inode);
+        mdc_inode_xatt_invalidate(this, inode);
+        GF_ATOMIC_INC(conf->mdc_counter.stat_invals);
+        goto out;
+    }
+
+    if (up_ci->flags & IATT_UPDATE_FLAGS) {
+        gen = mdc_inc_generation(this, inode);
+        ret = mdc_inode_iatt_set_validate(this, inode, NULL, &up_ci->stat,
+                                          _gf_false, gen);
+        /* one of the scenarios where ret < 0 is when this invalidate
+         * is older than the current stat, in that case do not
+         * update the xattrs as well
+         */
+        if (ret < 0)
+            goto out;
+        GF_ATOMIC_INC(conf->mdc_counter.stat_invals);
+    }
+
+    if (up_ci->flags & UP_XATTR) {
+        if (up_ci->dict)
+            ret = mdc_inode_xatt_update(this, inode, up_ci->dict);
+        else
+            ret = mdc_inode_xatt_invalidate(this, inode);
+
+        GF_ATOMIC_INC(conf->mdc_counter.xattr_invals);
+    } else if (up_ci->flags & UP_XATTR_RM) {
+        tmp.inode = inode;
+        tmp.this = this;
+        ret = dict_foreach(up_ci->dict, mdc_inval_xatt, &tmp);
+
+        GF_ATOMIC_INC(conf->mdc_counter.xattr_invals);
+    }
 
-	for (key = keys; key->name; key++) {
-		if (is_strpfx (key->name, pattern))
-			key->load = val;
-	}
+out:
+    if (inode)
+        inode_unref(inode);
 
-	return 0;
+    return ret;
 }
 
+struct mdc_ipc {
+    xlator_t *this;
+    dict_t *xattr;
+};
+
+static int
+mdc_send_xattrs_cbk(int ret, call_frame_t *frame, void *data)
+{
+    struct mdc_ipc *tmp = data;
+
+    if (ret < 0) {
+        mdc_key_unload_all(THIS->private);
+        gf_msg("md-cache", GF_LOG_INFO, 0, MD_CACHE_MSG_NO_XATTR_CACHE,
+               "Disabled cache for all xattrs, as registering for "
+               "xattr cache invalidation failed");
+    }
+    STACK_DESTROY(frame->root);
+    dict_unref(tmp->xattr);
+    GF_FREE(tmp);
+
+    return 0;
+}
+
+static int
+mdc_send_xattrs(void *data)
+{
+    int ret = 0;
+    struct mdc_ipc *tmp = data;
+
+    ret = syncop_ipc(FIRST_CHILD(tmp->this), GF_IPC_TARGET_UPCALL, tmp->xattr,
+                     NULL);
+    DECODE_SYNCOP_ERR(ret);
+    if (ret < 0) {
+        gf_msg(tmp->this->name, GF_LOG_WARNING, errno,
+               MD_CACHE_MSG_IPC_UPCALL_FAILED,
+               "Registering the list "
+               "of xattrs that needs invalidaton, with upcall, failed");
+    }
+
+    return ret;
+}
+
+static int
+mdc_register_xattr_inval(xlator_t *this)
+{
+    dict_t *xattr = NULL;
+    int ret = 0;
+    struct mdc_conf *conf = NULL;
+    call_frame_t *frame = NULL;
+    struct mdc_ipc *data = NULL;
+
+    conf = this->private;
+
+    LOCK(&conf->lock);
+    {
+        if (!conf->mdc_invalidation) {
+            UNLOCK(&conf->lock);
+            goto out;
+        }
+    }
+    UNLOCK(&conf->lock);
+
+    xattr = dict_new();
+    if (!xattr) {
+        gf_msg(this->name, GF_LOG_WARNING, ENOMEM, MD_CACHE_MSG_NO_MEMORY,
+               "dict_new failed");
+        ret = -1;
+        goto out;
+    }
+
+    if (!mdc_load_reqs(this, xattr)) {
+        gf_msg(this->name, GF_LOG_WARNING, ENOMEM, MD_CACHE_MSG_NO_MEMORY,
+               "failed to populate cache entries");
+        ret = -1;
+        goto out;
+    }
+
+    frame = create_frame(this, this->ctx->pool);
+    if (!frame) {
+        gf_msg(this->name, GF_LOG_ERROR, ENOMEM, MD_CACHE_MSG_NO_MEMORY,
+               "failed to create the frame");
+        ret = -1;
+        goto out;
+    }
+
+    data = GF_CALLOC(1, sizeof(struct mdc_ipc), gf_mdc_mt_mdc_ipc);
+    if (!data) {
+        gf_msg(this->name, GF_LOG_ERROR, ENOMEM, MD_CACHE_MSG_NO_MEMORY,
+               "failed to allocate memory");
+        ret = -1;
+        goto out;
+    }
+
+    data->this = this;
+    data->xattr = xattr;
+    ret = synctask_new(this->ctx->env, mdc_send_xattrs, mdc_send_xattrs_cbk,
+                       frame, data);
+    if (ret < 0) {
+        gf_msg(this->name, GF_LOG_WARNING, errno,
+               MD_CACHE_MSG_IPC_UPCALL_FAILED,
+               "Registering the list "
+               "of xattrs that needs invalidaton, with upcall, failed");
+    }
+
+out:
+    if (ret < 0) {
+        mdc_key_unload_all(conf);
+        if (xattr)
+            dict_unref(xattr);
+        if (frame)
+            STACK_DESTROY(frame->root);
+        GF_FREE(data);
+        gf_msg(this->name, GF_LOG_INFO, 0, MD_CACHE_MSG_NO_XATTR_CACHE,
+               "Disabled cache for all xattrs, as registering for "
+               "xattr cache invalidation failed");
+    }
+
+    return ret;
+}
 
 int
-reconfigure (xlator_t *this, dict_t *options)
+mdc_reconfigure(xlator_t *this, dict_t *options)
 {
-	struct mdc_conf *conf = NULL;
+    struct mdc_conf *conf = NULL;
+    int timeout = 0, ret = 0;
+    char *tmp_str = NULL;
+
+    conf = this->private;
+
+    GF_OPTION_RECONF("md-cache-timeout", timeout, options, int32, out);
+
+    GF_OPTION_RECONF("cache-selinux", conf->cache_selinux, options, bool, out);
+
+    GF_OPTION_RECONF("cache-capability-xattrs", conf->cache_capability, options,
+                     bool, out);
+
+    GF_OPTION_RECONF("cache-ima-xattrs", conf->cache_ima, options, bool, out);
 
-	conf = this->private;
+    GF_OPTION_RECONF("cache-posix-acl", conf->cache_posix_acl, options, bool,
+                     out);
 
-	GF_OPTION_RECONF ("md-cache-timeout", conf->timeout, options, int32, out);
+    GF_OPTION_RECONF("cache-glusterfs-acl", conf->cache_glusterfs_acl, options,
+                     bool, out);
 
-	GF_OPTION_RECONF ("cache-selinux", conf->cache_selinux, options, bool, out);
-	mdc_key_load_set (mdc_keys, "security.", conf->cache_selinux);
+    GF_OPTION_RECONF("cache-swift-metadata", conf->cache_swift_metadata,
+                     options, bool, out);
 
-	GF_OPTION_RECONF ("cache-posix-acl", conf->cache_posix_acl, options, bool, out);
-	mdc_key_load_set (mdc_keys, "system.posix_acl_", conf->cache_posix_acl);
-	mdc_key_load_set (mdc_keys, "glusterfs.posix_acl.", conf->cache_posix_acl);
+    GF_OPTION_RECONF("cache-samba-metadata", conf->cache_samba_metadata,
+                     options, bool, out);
 
-	GF_OPTION_RECONF("force-readdirp", conf->force_readdirp, options, bool, out);
+    GF_OPTION_RECONF("force-readdirp", conf->force_readdirp, options, bool,
+                     out);
 
+    GF_OPTION_RECONF("cache-invalidation", conf->mdc_invalidation, options,
+                     bool, out);
+
+    GF_OPTION_RECONF("global-cache-invalidation", conf->global_invalidation,
+                     options, bool, out);
+
+    GF_OPTION_RECONF("pass-through", this->pass_through, options, bool, out);
+
+    GF_OPTION_RECONF("md-cache-statfs", conf->cache_statfs, options, bool, out);
+
+    GF_OPTION_RECONF("xattr-cache-list", tmp_str, options, str, out);
+
+    ret = mdc_xattr_list_populate(conf, tmp_str);
+    if (ret < 0)
+        goto out;
+
+    /* If timeout is greater than 60s (default before the patch that added
+     * cache invalidation support was added) then, cache invalidation
+     * feature for md-cache needs to be enabled, if not set timeout to the
+     * previous max which is 60s
+     */
+    if ((timeout > 60) && (!conf->mdc_invalidation)) {
+        conf->timeout = 60;
+        goto out;
+    }
+    conf->timeout = timeout;
+
+    ret = mdc_register_xattr_inval(this);
 out:
-	return 0;
+    return ret;
 }
 
 int32_t
-mem_acct_init (xlator_t *this)
+mdc_mem_acct_init(xlator_t *this)
 {
-        int     ret = -1;
-
-        ret = xlator_mem_acct_init (this, gf_mdc_mt_end + 1);
-        return ret;
+    return xlator_mem_acct_init(this, gf_mdc_mt_end + 1);
 }
 
 int
-init (xlator_t *this)
+mdc_init(xlator_t *this)
 {
-	struct mdc_conf *conf = NULL;
+    struct mdc_conf *conf = NULL;
+    uint32_t timeout = 0;
+    char *tmp_str = NULL;
+
+    conf = GF_CALLOC(sizeof(*conf), 1, gf_mdc_mt_mdc_conf_t);
+    if (!conf) {
+        gf_msg(this->name, GF_LOG_ERROR, ENOMEM, MD_CACHE_MSG_NO_MEMORY,
+               "out of memory");
+        return -1;
+    }
+
+    LOCK_INIT(&conf->lock);
+
+    GF_OPTION_INIT("md-cache-timeout", timeout, uint32, out);
+
+    GF_OPTION_INIT("cache-selinux", conf->cache_selinux, bool, out);
+
+    GF_OPTION_INIT("cache-capability-xattrs", conf->cache_capability, bool,
+                   out);
+
+    GF_OPTION_INIT("cache-ima-xattrs", conf->cache_ima, bool, out);
+
+    GF_OPTION_INIT("cache-posix-acl", conf->cache_posix_acl, bool, out);
+
+    GF_OPTION_INIT("cache-glusterfs-acl", conf->cache_glusterfs_acl, bool, out);
+
+    GF_OPTION_INIT("cache-swift-metadata", conf->cache_swift_metadata, bool,
+                   out);
+
+    GF_OPTION_INIT("cache-samba-metadata", conf->cache_samba_metadata, bool,
+                   out);
+
+    GF_OPTION_INIT("force-readdirp", conf->force_readdirp, bool, out);
 
-	conf = GF_CALLOC (sizeof (*conf), 1, gf_mdc_mt_mdc_conf_t);
-	if (!conf) {
-		gf_log (this->name, GF_LOG_ERROR,
-			"out of memory");
-		return -1;
-	}
+    GF_OPTION_INIT("cache-invalidation", conf->mdc_invalidation, bool, out);
 
-        GF_OPTION_INIT ("md-cache-timeout", conf->timeout, int32, out);
+    GF_OPTION_INIT("global-cache-invalidation", conf->global_invalidation, bool,
+                   out);
 
-	GF_OPTION_INIT ("cache-selinux", conf->cache_selinux, bool, out);
-	mdc_key_load_set (mdc_keys, "security.", conf->cache_selinux);
+    GF_OPTION_INIT("pass-through", this->pass_through, bool, out);
 
-	GF_OPTION_INIT ("cache-posix-acl", conf->cache_posix_acl, bool, out);
-	mdc_key_load_set (mdc_keys, "system.posix_acl_", conf->cache_posix_acl);
-	mdc_key_load_set (mdc_keys, "glusterfs.posix_acl.", conf->cache_posix_acl);
+    pthread_mutex_init(&conf->statfs_cache.lock, NULL);
+    GF_OPTION_INIT("md-cache-statfs", conf->cache_statfs, bool, out);
+
+    GF_OPTION_INIT("xattr-cache-list", tmp_str, str, out);
+    mdc_xattr_list_populate(conf, tmp_str);
+
+    conf->last_child_down = gf_time();
+    conf->statfs_cache.last_refreshed = (time_t)-1;
+
+    /* initialize gf_atomic_t counters */
+    GF_ATOMIC_INIT(conf->mdc_counter.stat_hit, 0);
+    GF_ATOMIC_INIT(conf->mdc_counter.stat_miss, 0);
+    GF_ATOMIC_INIT(conf->mdc_counter.xattr_hit, 0);
+    GF_ATOMIC_INIT(conf->mdc_counter.xattr_miss, 0);
+    GF_ATOMIC_INIT(conf->mdc_counter.negative_lookup, 0);
+    GF_ATOMIC_INIT(conf->mdc_counter.nameless_lookup, 0);
+    GF_ATOMIC_INIT(conf->mdc_counter.stat_invals, 0);
+    GF_ATOMIC_INIT(conf->mdc_counter.xattr_invals, 0);
+    GF_ATOMIC_INIT(conf->mdc_counter.need_lookup, 0);
+    GF_ATOMIC_INIT(conf->generation, 0);
+
+    /* If timeout is greater than 60s (default before the patch that added
+     * cache invalidation support was added) then, cache invalidation
+     * feature for md-cache needs to be enabled, if not set timeout to the
+     * previous max which is 60s
+     */
+    if ((timeout > 60) && (!conf->mdc_invalidation)) {
+        conf->timeout = 60;
+        goto out;
+    }
+    conf->timeout = timeout;
 
-	GF_OPTION_INIT("force-readdirp", conf->force_readdirp, bool, out);
 out:
-	this->private = conf;
+    this->private = conf;
 
-        return 0;
+    return 0;
 }
 
-
 void
-fini (xlator_t *this)
+mdc_update_child_down_time(xlator_t *this, time_t now)
 {
-        return;
+    struct mdc_conf *conf = NULL;
+
+    conf = this->private;
+
+    LOCK(&conf->lock);
+    {
+        conf->last_child_down = now;
+    }
+    UNLOCK(&conf->lock);
 }
 
+int
+mdc_notify(xlator_t *this, int event, void *data, ...)
+{
+    int ret = 0;
+    struct mdc_conf *conf = NULL;
+
+    conf = this->private;
+    switch (event) {
+        case GF_EVENT_CHILD_DOWN:
+        case GF_EVENT_SOME_DESCENDENT_DOWN:
+            mdc_update_child_down_time(this, gf_time());
+            break;
+        case GF_EVENT_UPCALL:
+            if (conf->mdc_invalidation)
+                ret = mdc_invalidate(this, data);
+            break;
+        case GF_EVENT_CHILD_UP:
+        case GF_EVENT_SOME_DESCENDENT_UP:
+            ret = mdc_register_xattr_inval(this);
+            break;
+        default:
+            break;
+    }
+
+    if (default_notify(this, event, data) != 0)
+        ret = -1;
+
+    return ret;
+}
 
-struct xlator_fops fops = {
-        .lookup      = mdc_lookup,
-        .stat        = mdc_stat,
-        .fstat       = mdc_fstat,
-        .truncate    = mdc_truncate,
-        .ftruncate   = mdc_ftruncate,
-        .mknod       = mdc_mknod,
-        .mkdir       = mdc_mkdir,
-        .unlink      = mdc_unlink,
-        .rmdir       = mdc_rmdir,
-        .symlink     = mdc_symlink,
-        .rename      = mdc_rename,
-        .link        = mdc_link,
-        .create      = mdc_create,
-        .readv       = mdc_readv,
-        .writev      = mdc_writev,
-        .setattr     = mdc_setattr,
-        .fsetattr    = mdc_fsetattr,
-        .fsync       = mdc_fsync,
-        .setxattr    = mdc_setxattr,
-        .fsetxattr   = mdc_fsetxattr,
-        .getxattr    = mdc_getxattr,
-        .fgetxattr   = mdc_fgetxattr,
-	.removexattr = mdc_removexattr,
-	.fremovexattr= mdc_fremovexattr,
-	.readdirp    = mdc_readdirp,
-	.readdir     = mdc_readdir,
-	.fallocate   = mdc_fallocate,
-	.discard     = mdc_discard,
-        .zerofill    = mdc_zerofill,
+void
+mdc_fini(xlator_t *this)
+{
+    GF_FREE(this->private);
+}
+
+struct xlator_fops mdc_fops = {
+    .lookup = mdc_lookup,
+    .stat = mdc_stat,
+    .fstat = mdc_fstat,
+    .truncate = mdc_truncate,
+    .ftruncate = mdc_ftruncate,
+    .mknod = mdc_mknod,
+    .mkdir = mdc_mkdir,
+    .unlink = mdc_unlink,
+    .rmdir = mdc_rmdir,
+    .symlink = mdc_symlink,
+    .rename = mdc_rename,
+    .link = mdc_link,
+    .create = mdc_create,
+    .open = mdc_open,
+    .readv = mdc_readv,
+    .writev = mdc_writev,
+    .setattr = mdc_setattr,
+    .fsetattr = mdc_fsetattr,
+    .fsync = mdc_fsync,
+    .setxattr = mdc_setxattr,
+    .fsetxattr = mdc_fsetxattr,
+    .getxattr = mdc_getxattr,
+    .fgetxattr = mdc_fgetxattr,
+    .removexattr = mdc_removexattr,
+    .fremovexattr = mdc_fremovexattr,
+    .opendir = mdc_opendir,
+    .readdirp = mdc_readdirp,
+    .readdir = mdc_readdir,
+    .fallocate = mdc_fallocate,
+    .discard = mdc_discard,
+    .zerofill = mdc_zerofill,
+    .statfs = mdc_statfs,
+    .readlink = mdc_readlink,
+    .fsyncdir = mdc_fsyncdir,
+    .access = mdc_access,
+};
+
+struct xlator_cbks mdc_cbks = {
+    .forget = mdc_forget,
 };
 
+struct xlator_dumpops mdc_dumpops = {
+    .priv = mdc_priv_dump,
+};
 
-struct xlator_cbks cbks = {
-        .forget      = mdc_forget,
+struct volume_options mdc_options[] = {
+    {
+        .key = {"md-cache"},
+        .type = GF_OPTION_TYPE_BOOL,
+        .default_value = "off",
+        .description = "enable/disable md-cache",
+        .op_version = {GD_OP_VERSION_6_0},
+        .flags = OPT_FLAG_SETTABLE,
+    },
+    {
+        .key = {"cache-selinux"},
+        .type = GF_OPTION_TYPE_BOOL,
+        .default_value = "false",
+        .op_version = {2},
+        .flags = OPT_FLAG_SETTABLE | OPT_FLAG_CLIENT_OPT | OPT_FLAG_DOC,
+        .description = "Cache selinux xattr(security.selinux) on client side",
+    },
+    {
+        .key = {"cache-capability-xattrs"},
+        .type = GF_OPTION_TYPE_BOOL,
+        .default_value = "true",
+        .op_version = {GD_OP_VERSION_3_10_0},
+        .flags = OPT_FLAG_SETTABLE | OPT_FLAG_CLIENT_OPT | OPT_FLAG_DOC,
+        .description = "Cache capability xattr(security.capability) on "
+                       "client side",
+    },
+    {
+        .key = {"cache-ima-xattrs"},
+        .type = GF_OPTION_TYPE_BOOL,
+        .default_value = "true",
+        .op_version = {GD_OP_VERSION_3_10_0},
+        .flags = OPT_FLAG_SETTABLE | OPT_FLAG_CLIENT_OPT | OPT_FLAG_DOC,
+        .description = "Cache Linux integrity subsystem xattr(security.ima) "
+                       "on client side",
+    },
+    {
+        .key = {"cache-swift-metadata"},
+        .type = GF_OPTION_TYPE_BOOL,
+        .default_value = "false",
+        .op_version = {GD_OP_VERSION_3_7_10},
+        .flags = OPT_FLAG_SETTABLE | OPT_FLAG_CLIENT_OPT | OPT_FLAG_DOC,
+        .description = "Cache swift metadata (user.swift.metadata xattr)",
+    },
+    {
+        .key = {"cache-samba-metadata"},
+        .type = GF_OPTION_TYPE_BOOL,
+        .default_value = "false",
+        .op_version = {GD_OP_VERSION_3_9_0},
+        .flags = OPT_FLAG_SETTABLE | OPT_FLAG_CLIENT_OPT | OPT_FLAG_DOC,
+        .description = "Cache samba metadata (user.DOSATTRIB, security.NTACL,"
+                       " org.netatalk.Metadata, org.netatalk.ResourceFork, "
+                       "and user.DosStream. xattrs)",
+    },
+    {
+        .key = {"cache-posix-acl"},
+        .type = GF_OPTION_TYPE_BOOL,
+        .default_value = "false",
+        .op_version = {2},
+        .flags = OPT_FLAG_SETTABLE | OPT_FLAG_CLIENT_OPT | OPT_FLAG_DOC,
+        .description = "Cache posix ACL xattrs (system.posix_acl_access, "
+                       "system.posix_acl_default) on client side",
+    },
+    {
+        .key = {"cache-glusterfs-acl"},
+        .type = GF_OPTION_TYPE_BOOL,
+        .default_value = "false",
+        .op_version = {GD_OP_VERSION_6_0},
+        .flags = OPT_FLAG_SETTABLE | OPT_FLAG_CLIENT_OPT | OPT_FLAG_DOC,
+        .description = "Cache virtual glusterfs ACL xattrs "
+                       "(glusterfs.posix.acl, glusterfs.posix.default_acl) "
+                       "on client side",
+    },
+    {
+        .key = {"md-cache-timeout"},
+        .type = GF_OPTION_TYPE_INT,
+        .min = 0,
+        .max = 600,
+        .default_value = SITE_H_MD_CACHE_TIMEOUT,
+        .op_version = {2},
+        .flags = OPT_FLAG_SETTABLE | OPT_FLAG_CLIENT_OPT | OPT_FLAG_DOC,
+        .description = "Time period after which cache has to be refreshed",
+    },
+    {
+        .key = {"force-readdirp"},
+        .type = GF_OPTION_TYPE_BOOL,
+        .default_value = "true",
+        .op_version = {2},
+        .flags = OPT_FLAG_SETTABLE | OPT_FLAG_CLIENT_OPT | OPT_FLAG_DOC,
+        .description = "Convert all readdir requests to readdirplus to "
+                       "collect stat info on each entry.",
+    },
+    {
+        .key = {"cache-invalidation"},
+        .type = GF_OPTION_TYPE_BOOL,
+        .default_value = "false",
+        .op_version = {GD_OP_VERSION_3_9_0},
+        .flags = OPT_FLAG_SETTABLE | OPT_FLAG_CLIENT_OPT | OPT_FLAG_DOC,
+        .description = "When \"on\", invalidates/updates the metadata cache,"
+                       " on receiving the cache-invalidation notifications",
+    },
+    {
+        .key = {"global-cache-invalidation"},
+        .type = GF_OPTION_TYPE_BOOL,
+        .default_value = "true",
+        .op_version = {GD_OP_VERSION_6_0},
+        .flags = OPT_FLAG_SETTABLE | OPT_FLAG_CLIENT_OPT | OPT_FLAG_DOC,
+        .description =
+            "When \"on\", purges all read caches in kernel and glusterfs stack "
+            "whenever a stat change is detected. Stat changes can be detected "
+            "while processing responses to file operations (fop) or through "
+            "upcall notifications. Since purging caches can be an expensive "
+            "operation, it's advised to have this option \"on\" only when a "
+            "file "
+            "can be accessed from multiple different Glusterfs mounts and "
+            "caches across these different mounts are required to be coherent. "
+            "If a file is not accessed across different mounts "
+            "(simple example is having only one mount for a volume), its "
+            "advised to keep "
+            "this option \"off\" as all file modifications go through caches "
+            "keeping them "
+            "coherent. This option overrides value of "
+            "performance.cache-invalidation.",
+    },
+    {
+        .key = {"md-cache-statfs"},
+        .type = GF_OPTION_TYPE_BOOL,
+        .default_value = "off",
+        .op_version = {GD_OP_VERSION_4_0_0},
+        .flags = OPT_FLAG_SETTABLE | OPT_FLAG_CLIENT_OPT | OPT_FLAG_DOC,
+        .description = "Cache statfs information of filesystem on the client",
+    },
+    {
+        .key = {"xattr-cache-list"},
+        .type = GF_OPTION_TYPE_STR,
+        .default_value = "",
+        .op_version = {GD_OP_VERSION_4_0_0},
+        .flags = OPT_FLAG_SETTABLE | OPT_FLAG_CLIENT_OPT | OPT_FLAG_DOC,
+        .description = "A comma separated list of xattrs that shall be "
+                       "cached by md-cache. The only wildcard allowed is '*'",
+    },
+    {.key = {"pass-through"},
+     .type = GF_OPTION_TYPE_BOOL,
+     .default_value = "false",
+     .op_version = {GD_OP_VERSION_4_1_0},
+     .flags = OPT_FLAG_SETTABLE | OPT_FLAG_DOC | OPT_FLAG_CLIENT_OPT,
+     .tags = {"md-cache"},
+     .description = "Enable/Disable md cache translator"},
+    {.key = {NULL}},
 };
 
-struct volume_options options[] = {
-	{ .key = {"cache-selinux"},
-	  .type = GF_OPTION_TYPE_BOOL,
-	  .default_value = "false",
-	},
-	{ .key = {"cache-posix-acl"},
-	  .type = GF_OPTION_TYPE_BOOL,
-	  .default_value = "false",
-	},
-        { .key = {"md-cache-timeout"},
-          .type = GF_OPTION_TYPE_INT,
-          .min = 0,
-          .max = 60,
-          .default_value = "1",
-          .description = "Time period after which cache has to be refreshed",
-        },
-	{ .key = {"force-readdirp"},
-	  .type = GF_OPTION_TYPE_BOOL,
-	  .default_value = "true",
-	  .description = "Convert all readdir requests to readdirplus to "
-			 "collect stat info on each entry.",
-	},
-    { .key = {NULL} },
+xlator_api_t xlator_api = {
+    .init = mdc_init,
+    .fini = mdc_fini,
+    .notify = mdc_notify,
+    .reconfigure = mdc_reconfigure,
+    .mem_acct_init = mdc_mem_acct_init,
+    .dump_metrics = mdc_dump_metrics,
+    .op_version = {1}, /* Present from the initial version */
+    .dumpops = &mdc_dumpops,
+    .fops = &mdc_fops,
+    .cbks = &mdc_cbks,
+    .options = mdc_options,
+    .identifier = "md-cache",
+    .category = GF_MAINTAINED,
 };
diff --git a/xlators/performance/nl-cache/Makefile.am b/xlators/performance/nl-cache/Makefile.am
new file mode 100644
index 00000000000..a985f42a877
--- /dev/null
+++ b/xlators/performance/nl-cache/Makefile.am
@@ -0,0 +1,3 @@
+SUBDIRS = src
+
+CLEANFILES =
diff --git a/xlators/performance/nl-cache/src/Makefile.am b/xlators/performance/nl-cache/src/Makefile.am
new file mode 100644
index 00000000000..c44ce871627
--- /dev/null
+++ b/xlators/performance/nl-cache/src/Makefile.am
@@ -0,0 +1,12 @@
+xlator_LTLIBRARIES = nl-cache.la
+xlatordir = $(libdir)/glusterfs/$(PACKAGE_VERSION)/xlator/performance
+nl_cache_la_LDFLAGS = -module $(GF_XLATOR_DEFAULT_LDFLAGS)
+nl_cache_la_SOURCES = nl-cache.c nl-cache-helper.c
+nl_cache_la_LIBADD = $(top_builddir)/libglusterfs/src/libglusterfs.la
+noinst_HEADERS = nl-cache.h nl-cache-mem-types.h nl-cache-messages.h
+AM_CPPFLAGS = $(GF_CPPFLAGS) -I$(top_srcdir)/libglusterfs/src \
+        -I$(top_srcdir)/rpc/xdr/src -I$(top_builddir)/rpc/xdr/src \
+        -I$(CONTRIBDIR)/timer-wheel
+
+AM_CFLAGS = -Wall -fno-strict-aliasing $(GF_CFLAGS)
+CLEANFILES =
diff --git a/xlators/performance/nl-cache/src/nl-cache-helper.c b/xlators/performance/nl-cache/src/nl-cache-helper.c
new file mode 100644
index 00000000000..29b99b5b8ea
--- /dev/null
+++ b/xlators/performance/nl-cache/src/nl-cache-helper.c
@@ -0,0 +1,1201 @@
+/*
+ *   Copyright (c) 2017 Red Hat, Inc. <http://www.redhat.com>
+ *   This file is part of GlusterFS.
+ *
+ *   This file is licensed to you under your choice of the GNU Lesser
+ *   General Public License, version 3 or any later version (LGPLv3 or
+ *   later), or the GNU General Public License, version 2 (GPLv2), in all
+ *   cases as published by the Free Software Foundation.
+ */
+
+#include "nl-cache.h"
+#include "timer-wheel.h"
+#include <glusterfs/statedump.h>
+
+/* Caching guidelines:
+ * This xlator serves negative lookup(ENOENT lookups) from the cache,
+ * there by making create faster.
+ *   What is cached?
+ *      Negative lookup cache is stored for each directory, and has 2 entries:
+ *      - Negative entries: Populated only when lookup/stat returns ENOENT.
+ *        Fuse mostly sends only one lookup before create, hence negative entry
+ *        cache is almost useless. But for SMB access, multiple lookups/stats
+ *        are sent before creating the file. Hence the negative entry cache.
+ *        It can exist even when the positive entry cache is invalid. It also
+ *        has the entries that were deleted from this directory.
+ *        Freed on receiving upcall(with dentry change flag) or on expiring
+ *        timeout of the cache.
+ *
+ *      - Positive entries: Populated as a part of readdirp, and as a part of
+ *        mkdir followed by creates inside that directory. Lookups and other
+ *        fops do not populate the positive entry (as it can grow long and is
+ *        of no value add)
+ *        Freed on receiving upcall(with dentry change flag) or on expiring
+ *        timeout of the cache.
+ *
+ *   Data structures to store cache?
+ *      The cache of any directory is stored in the inode_ctx of the directory.
+ *      Negative entries are stored as list of strings.
+ *             Search - O(n)
+ *             Add    - O(1)
+ *             Delete - O(n) - as it has to be searched before deleting
+ *      Positive entries are stored as a list, each list node has a pointer
+ *          to the inode of the positive entry or the name of the entry.
+ *          Since the client side inode table already will have inodes for
+ *          positive entries, we just take a ref of that inode and store as
+ *          positive entry cache. In cases like hardlinks and readdirp where
+ *          inode is NULL, we store the names.
+ *          Name Search - O(n)
+ *          Inode Search - O(1) - Actually complexity of inode_find()
+ *          Name/inode Add - O(1)
+ *          Name Delete - O(n)
+ *          Inode Delete - O(1)
+ *
+ * Locking order:
+ *
+ * TODO:
+ * - Fill Positive entries on readdir/p, after which in lookup_cbk check if the
+ *   name is in PE and replace it with inode.
+ * - fini, PARENET_DOWN, disable caching
+ * - Virtual setxattr to dump the inode_ctx, to ease debugging
+ * - Handle dht_nuke xattr: clear all cache
+ * - Special handling for .meta and .trashcan?
+ */
+
+int
+__nlc_inode_ctx_timer_start(xlator_t *this, inode_t *inode, nlc_ctx_t *nlc_ctx);
+int
+__nlc_add_to_lru(xlator_t *this, inode_t *inode, nlc_ctx_t *nlc_ctx);
+void
+nlc_remove_from_lru(xlator_t *this, inode_t *inode);
+void
+__nlc_inode_ctx_timer_delete(xlator_t *this, nlc_ctx_t *nlc_ctx);
+gf_boolean_t
+__nlc_search_ne(nlc_ctx_t *nlc_ctx, const char *name);
+void
+__nlc_free_pe(xlator_t *this, nlc_ctx_t *nlc_ctx, nlc_pe_t *pe);
+void
+__nlc_free_ne(xlator_t *this, nlc_ctx_t *nlc_ctx, nlc_ne_t *ne);
+
+static int32_t
+nlc_get_cache_timeout(xlator_t *this)
+{
+    nlc_conf_t *conf = NULL;
+
+    conf = this->private;
+
+    /* Cache timeout is generally not meant to be changed often,
+     * once set, hence not within locks */
+    return conf->cache_timeout;
+}
+
+static gf_boolean_t
+__nlc_is_cache_valid(xlator_t *this, nlc_ctx_t *nlc_ctx)
+{
+    nlc_conf_t *conf = NULL;
+    time_t last_val_time;
+    gf_boolean_t ret = _gf_false;
+
+    GF_VALIDATE_OR_GOTO(this->name, nlc_ctx, out);
+
+    conf = this->private;
+
+    LOCK(&conf->lock);
+    {
+        last_val_time = conf->last_child_down;
+    }
+    UNLOCK(&conf->lock);
+
+    if ((last_val_time <= nlc_ctx->cache_time) && (nlc_ctx->cache_time != 0))
+        ret = _gf_true;
+out:
+    return ret;
+}
+
+void
+nlc_update_child_down_time(xlator_t *this, time_t now)
+{
+    nlc_conf_t *conf = NULL;
+
+    conf = this->private;
+
+    LOCK(&conf->lock);
+    {
+        conf->last_child_down = now;
+    }
+    UNLOCK(&conf->lock);
+
+    return;
+}
+
+void
+nlc_disable_cache(xlator_t *this)
+{
+    nlc_conf_t *conf = NULL;
+
+    conf = this->private;
+
+    LOCK(&conf->lock);
+    {
+        conf->disable_cache = _gf_true;
+    }
+    UNLOCK(&conf->lock);
+
+    return;
+}
+
+static int
+__nlc_inode_ctx_get(xlator_t *this, inode_t *inode, nlc_ctx_t **nlc_ctx_p)
+{
+    int ret = 0;
+    nlc_ctx_t *nlc_ctx = NULL;
+    uint64_t nlc_ctx_int = 0;
+    uint64_t nlc_pe_int = 0;
+
+    ret = __inode_ctx_get2(inode, this, &nlc_ctx_int, &nlc_pe_int);
+    if (ret == 0 && nlc_ctx_p) {
+        nlc_ctx = (void *)(long)(nlc_ctx_int);
+        *nlc_ctx_p = nlc_ctx;
+    }
+    return ret;
+}
+
+static int
+nlc_inode_ctx_set(xlator_t *this, inode_t *inode, nlc_ctx_t *nlc_ctx,
+                  nlc_pe_t *nlc_pe_p)
+{
+    uint64_t ctx1, ctx2;
+    int ret = -1;
+
+    ctx1 = (uint64_t)(uintptr_t)nlc_ctx;
+    ctx2 = (uint64_t)(uintptr_t)nlc_pe_p;
+
+    /* The caller may choose to set one of the ctxs, hence check
+     * if the ctx1/2 is non zero and then send the address. If we
+     * blindly send the address of both the ctxs, it may reset the
+     * ctx the caller had sent NULL(intended as leave untouched) for.*/
+    LOCK(&inode->lock);
+    {
+        ret = __inode_ctx_set2(inode, this, ctx1 ? &ctx1 : 0, ctx2 ? &ctx2 : 0);
+    }
+    UNLOCK(&inode->lock);
+    return ret;
+}
+
+static void
+nlc_inode_ctx_get(xlator_t *this, inode_t *inode, nlc_ctx_t **nlc_ctx_p)
+{
+    int ret = 0;
+
+    LOCK(&inode->lock);
+    {
+        ret = __nlc_inode_ctx_get(this, inode, nlc_ctx_p);
+        if (ret < 0)
+            gf_msg_debug(this->name, 0,
+                         "inode ctx get failed for "
+                         "inode:%p",
+                         inode);
+    }
+    UNLOCK(&inode->lock);
+
+    return;
+}
+
+static void
+__nlc_inode_clear_entries(xlator_t *this, nlc_ctx_t *nlc_ctx)
+{
+    nlc_pe_t *pe = NULL;
+    nlc_pe_t *tmp = NULL;
+    nlc_ne_t *ne = NULL;
+    nlc_ne_t *tmp1 = NULL;
+
+    if (!nlc_ctx)
+        goto out;
+
+    if (IS_PE_VALID(nlc_ctx->state))
+        list_for_each_entry_safe(pe, tmp, &nlc_ctx->pe, list)
+        {
+            __nlc_free_pe(this, nlc_ctx, pe);
+        }
+
+    if (IS_NE_VALID(nlc_ctx->state))
+        list_for_each_entry_safe(ne, tmp1, &nlc_ctx->ne, list)
+        {
+            __nlc_free_ne(this, nlc_ctx, ne);
+        }
+
+    nlc_ctx->cache_time = 0;
+    nlc_ctx->state = 0;
+    GF_ASSERT(nlc_ctx->cache_size == sizeof(*nlc_ctx));
+    GF_ASSERT(nlc_ctx->refd_inodes == 0);
+out:
+    return;
+}
+
+static void
+nlc_init_invalid_ctx(xlator_t *this, inode_t *inode, nlc_ctx_t *nlc_ctx)
+{
+    nlc_conf_t *conf = NULL;
+    int ret = -1;
+
+    conf = this->private;
+    if (!nlc_ctx)
+        goto out;
+
+    LOCK(&nlc_ctx->lock);
+    {
+        if (__nlc_is_cache_valid(this, nlc_ctx))
+            goto unlock;
+
+        /* The cache/nlc_ctx can be invalid for 2 reasons:
+         * - Because of a child-down/timer expiry, cache is
+         *   invalid but the nlc_ctx is not yet cleaned up.
+         * - nlc_ctx is cleaned up, because of invalidations
+         *   or lru prune etc.*/
+
+        /* If the cache is present but invalid, clear the cache and
+         * reset the timer. */
+        __nlc_inode_clear_entries(this, nlc_ctx);
+
+        /* If timer is present, then it is already part of lru as well
+         * Hence reset the timer and return.*/
+        if (nlc_ctx->timer) {
+            gf_tw_mod_timer_pending(conf->timer_wheel, nlc_ctx->timer,
+                                    conf->cache_timeout);
+            nlc_ctx->cache_time = gf_time();
+            goto unlock;
+        }
+
+        /* If timer was NULL, the nlc_ctx is already cleanedup,
+         * and we need to start timer and add to lru, so that it is
+         * ready to cache entries a fresh */
+        ret = __nlc_inode_ctx_timer_start(this, inode, nlc_ctx);
+        if (ret < 0)
+            goto unlock;
+
+        ret = __nlc_add_to_lru(this, inode, nlc_ctx);
+        if (ret < 0) {
+            __nlc_inode_ctx_timer_delete(this, nlc_ctx);
+            goto unlock;
+        }
+    }
+unlock:
+    UNLOCK(&nlc_ctx->lock);
+out:
+    return;
+}
+
+static nlc_ctx_t *
+nlc_inode_ctx_get_set(xlator_t *this, inode_t *inode, nlc_ctx_t **nlc_ctx_p)
+{
+    uint64_t ctx;
+    int ret = 0;
+    nlc_ctx_t *nlc_ctx = NULL;
+    nlc_conf_t *conf = NULL;
+
+    conf = this->private;
+
+    LOCK(&inode->lock);
+    {
+        ret = __nlc_inode_ctx_get(this, inode, &nlc_ctx);
+        if (nlc_ctx)
+            goto unlock;
+
+        nlc_ctx = GF_CALLOC(sizeof(*nlc_ctx), 1, gf_nlc_mt_nlc_ctx_t);
+        if (!nlc_ctx)
+            goto unlock;
+
+        LOCK_INIT(&nlc_ctx->lock);
+        INIT_LIST_HEAD(&nlc_ctx->pe);
+        INIT_LIST_HEAD(&nlc_ctx->ne);
+
+        ret = __nlc_inode_ctx_timer_start(this, inode, nlc_ctx);
+        if (ret < 0)
+            goto unlock;
+
+        ret = __nlc_add_to_lru(this, inode, nlc_ctx);
+        if (ret < 0) {
+            __nlc_inode_ctx_timer_delete(this, nlc_ctx);
+            goto unlock;
+        }
+
+        ctx = (uint64_t)(uintptr_t)nlc_ctx;
+        ret = __inode_ctx_set2(inode, this, &ctx, NULL);
+        if (ret) {
+            gf_msg(this->name, GF_LOG_ERROR, ENOMEM, NLC_MSG_NO_MEMORY,
+                   "inode ctx set failed");
+            __nlc_inode_ctx_timer_delete(this, nlc_ctx);
+            nlc_remove_from_lru(this, inode);
+            goto unlock;
+        }
+
+        /*TODO: also sizeof (gf_tw_timer_list) + nlc_timer_data_t ?*/
+        nlc_ctx->cache_size = sizeof(*nlc_ctx);
+        GF_ATOMIC_ADD(conf->current_cache_size, nlc_ctx->cache_size);
+    }
+unlock:
+    UNLOCK(&inode->lock);
+
+    if (ret == 0 && nlc_ctx_p) {
+        *nlc_ctx_p = nlc_ctx;
+        nlc_init_invalid_ctx(this, inode, nlc_ctx);
+    }
+
+    if (ret < 0 && nlc_ctx) {
+        LOCK_DESTROY(&nlc_ctx->lock);
+        GF_FREE(nlc_ctx);
+        nlc_ctx = NULL;
+        goto out;
+    }
+
+out:
+    return nlc_ctx;
+}
+
+nlc_local_t *
+nlc_local_init(call_frame_t *frame, xlator_t *this, glusterfs_fop_t fop,
+               loc_t *loc, loc_t *loc2)
+{
+    nlc_local_t *local = NULL;
+
+    local = GF_CALLOC(sizeof(*local), 1, gf_nlc_mt_nlc_local_t);
+    if (!local)
+        goto out;
+
+    if (loc)
+        loc_copy(&local->loc, loc);
+    if (loc2)
+        loc_copy(&local->loc2, loc2);
+
+    local->fop = fop;
+    frame->local = local;
+out:
+    return local;
+}
+
+void
+nlc_local_wipe(xlator_t *this, nlc_local_t *local)
+{
+    if (!local)
+        goto out;
+
+    loc_wipe(&local->loc);
+
+    loc_wipe(&local->loc2);
+
+    GF_FREE(local);
+out:
+    return;
+}
+
+static void
+__nlc_set_dir_state(nlc_ctx_t *nlc_ctx, uint64_t new_state)
+{
+    nlc_ctx->state |= new_state;
+
+    return;
+}
+
+void
+nlc_set_dir_state(xlator_t *this, inode_t *inode, uint64_t state)
+{
+    nlc_ctx_t *nlc_ctx = NULL;
+
+    if (inode->ia_type != IA_IFDIR) {
+        gf_msg_callingfn(this->name, GF_LOG_ERROR, EINVAL, NLC_MSG_EINVAL,
+                         "inode is not of type dir");
+        goto out;
+    }
+
+    nlc_inode_ctx_get_set(this, inode, &nlc_ctx);
+    if (!nlc_ctx)
+        goto out;
+
+    LOCK(&nlc_ctx->lock);
+    {
+        __nlc_set_dir_state(nlc_ctx, state);
+    }
+    UNLOCK(&nlc_ctx->lock);
+out:
+    return;
+}
+
+static void
+nlc_cache_timeout_handler(struct gf_tw_timer_list *timer, void *data,
+                          unsigned long calltime)
+{
+    nlc_timer_data_t *tmp = data;
+    nlc_ctx_t *nlc_ctx = NULL;
+
+    nlc_inode_ctx_get(tmp->this, tmp->inode, &nlc_ctx);
+    if (!nlc_ctx)
+        goto out;
+
+    /* Taking nlc_ctx->lock will lead to deadlock, hence updating
+     * the cache is invalid outside of lock, instead of clear_cache.
+     * Since cache_time is assigned outside of lock, the value can
+     * be invalid for short time, this may result in false negative
+     * which is better than deadlock */
+    nlc_ctx->cache_time = 0;
+out:
+    return;
+}
+
+void
+__nlc_inode_ctx_timer_delete(xlator_t *this, nlc_ctx_t *nlc_ctx)
+{
+    nlc_conf_t *conf = NULL;
+
+    conf = this->private;
+
+    if (nlc_ctx->timer)
+        gf_tw_del_timer(conf->timer_wheel, nlc_ctx->timer);
+
+    if (nlc_ctx->timer_data) {
+        inode_unref(nlc_ctx->timer_data->inode);
+        GF_FREE(nlc_ctx->timer_data);
+        nlc_ctx->timer_data = NULL;
+    }
+
+    GF_FREE(nlc_ctx->timer);
+    nlc_ctx->timer = NULL;
+
+    return;
+}
+
+int
+__nlc_inode_ctx_timer_start(xlator_t *this, inode_t *inode, nlc_ctx_t *nlc_ctx)
+{
+    struct gf_tw_timer_list *timer = NULL;
+    nlc_timer_data_t *tmp = NULL;
+    nlc_conf_t *conf = NULL;
+    int ret = -1;
+
+    conf = this->private;
+
+    /* We are taking inode_table->lock within inode->lock
+     * as the only other caller which takes inode->lock within
+     * inode_table->lock and cause deadlock is inode_table_destroy.
+     * Hopefully, there can be no fop when inode_table_destroy is
+     * being called. */
+    tmp = GF_CALLOC(1, sizeof(*tmp), gf_nlc_mt_nlc_timer_data_t);
+    if (!tmp)
+        goto out;
+    tmp->inode = inode_ref(inode);
+    tmp->this = this;
+
+    timer = GF_CALLOC(1, sizeof(*timer), gf_common_mt_tw_timer_list);
+    if (!timer)
+        goto out;
+
+    INIT_LIST_HEAD(&timer->entry);
+    timer->expires = nlc_get_cache_timeout(this);
+    timer->function = nlc_cache_timeout_handler;
+    timer->data = tmp;
+    nlc_ctx->timer = timer;
+    nlc_ctx->timer_data = tmp;
+    gf_tw_add_timer(conf->timer_wheel, timer);
+
+    nlc_ctx->cache_time = gf_time();
+    gf_msg_trace(this->name, 0,
+                 "Registering timer:%p, inode:%p, "
+                 "gfid:%s",
+                 timer, inode, uuid_utoa(inode->gfid));
+
+    ret = 0;
+
+out:
+    if (ret < 0) {
+        if (tmp && tmp->inode)
+            inode_unref(tmp->inode);
+        GF_FREE(tmp);
+        GF_FREE(timer);
+    }
+
+    return ret;
+}
+
+int
+__nlc_add_to_lru(xlator_t *this, inode_t *inode, nlc_ctx_t *nlc_ctx)
+{
+    nlc_lru_node_t *lru_ino = NULL;
+    uint64_t nlc_pe_int = 0;
+    nlc_conf_t *conf = NULL;
+    int ret = -1;
+
+    conf = this->private;
+
+    lru_ino = GF_CALLOC(1, sizeof(*lru_ino), gf_nlc_mt_nlc_lru_node);
+    if (!lru_ino)
+        goto out;
+
+    INIT_LIST_HEAD(&lru_ino->list);
+    lru_ino->inode = inode_ref(inode);
+    LOCK(&conf->lock);
+    {
+        list_add_tail(&lru_ino->list, &conf->lru);
+    }
+    UNLOCK(&conf->lock);
+
+    nlc_ctx->refd_inodes = 0;
+    ret = __inode_ctx_get2(inode, this, NULL, &nlc_pe_int);
+    if (nlc_pe_int == 0)
+        GF_ATOMIC_ADD(conf->refd_inodes, 1);
+
+    ret = 0;
+
+out:
+    return ret;
+}
+
+void
+nlc_remove_from_lru(xlator_t *this, inode_t *inode)
+{
+    nlc_lru_node_t *lru_node = NULL;
+    nlc_lru_node_t *tmp = NULL;
+    nlc_lru_node_t *tmp1 = NULL;
+    nlc_conf_t *conf = NULL;
+
+    conf = this->private;
+
+    LOCK(&conf->lock);
+    {
+        list_for_each_entry_safe(lru_node, tmp, &conf->lru, list)
+        {
+            if (inode == lru_node->inode) {
+                list_del(&lru_node->list);
+                tmp1 = lru_node;
+                break;
+            }
+        }
+    }
+    UNLOCK(&conf->lock);
+
+    if (tmp1) {
+        inode_unref(tmp1->inode);
+        GF_FREE(tmp1);
+    }
+
+    return;
+}
+
+void
+nlc_lru_prune(xlator_t *this, inode_t *inode)
+{
+    nlc_lru_node_t *lru_node = NULL;
+    nlc_lru_node_t *prune_node = NULL;
+    nlc_lru_node_t *tmp = NULL;
+    nlc_conf_t *conf = NULL;
+
+    conf = this->private;
+
+    LOCK(&conf->lock);
+    {
+        if ((GF_ATOMIC_GET(conf->refd_inodes) < conf->inode_limit) &&
+            (GF_ATOMIC_GET(conf->current_cache_size) < conf->cache_size))
+            goto unlock;
+
+        list_for_each_entry_safe(lru_node, tmp, &conf->lru, list)
+        {
+            list_del(&lru_node->list);
+            prune_node = lru_node;
+            goto unlock;
+        }
+    }
+unlock:
+    UNLOCK(&conf->lock);
+
+    if (prune_node) {
+        nlc_inode_clear_cache(this, prune_node->inode, NLC_LRU_PRUNE);
+        inode_unref(prune_node->inode);
+        GF_FREE(prune_node);
+    }
+    return;
+}
+
+void
+nlc_clear_all_cache(xlator_t *this)
+{
+    nlc_conf_t *conf = NULL;
+    struct list_head clear_list;
+    nlc_lru_node_t *prune_node = NULL;
+    nlc_lru_node_t *tmp = NULL;
+
+    conf = this->private;
+
+    INIT_LIST_HEAD(&clear_list);
+
+    LOCK(&conf->lock);
+    {
+        list_replace_init(&conf->lru, &clear_list);
+    }
+    UNLOCK(&conf->lock);
+
+    list_for_each_entry_safe(prune_node, tmp, &clear_list, list)
+    {
+        list_del(&prune_node->list);
+        nlc_inode_clear_cache(this, prune_node->inode, NLC_LRU_PRUNE);
+        inode_unref(prune_node->inode);
+        GF_FREE(prune_node);
+    }
+
+    return;
+}
+
+void
+__nlc_free_pe(xlator_t *this, nlc_ctx_t *nlc_ctx, nlc_pe_t *pe)
+{
+    uint64_t pe_int = 0;
+    nlc_conf_t *conf = NULL;
+    uint64_t nlc_ctx_int = 0;
+
+    conf = this->private;
+
+    if (pe->inode) {
+        inode_ctx_reset1(pe->inode, this, &pe_int);
+        inode_ctx_get2(pe->inode, this, &nlc_ctx_int, NULL);
+        inode_unref(pe->inode);
+    }
+    list_del(&pe->list);
+
+    nlc_ctx->cache_size -= sizeof(*pe) + sizeof(pe->name);
+    GF_ATOMIC_SUB(conf->current_cache_size, (sizeof(*pe) + sizeof(pe->name)));
+
+    nlc_ctx->refd_inodes -= 1;
+    if (nlc_ctx_int == 0)
+        GF_ATOMIC_SUB(conf->refd_inodes, 1);
+
+    GF_FREE(pe->name);
+    GF_FREE(pe);
+
+    return;
+}
+
+void
+__nlc_free_ne(xlator_t *this, nlc_ctx_t *nlc_ctx, nlc_ne_t *ne)
+{
+    nlc_conf_t *conf = NULL;
+
+    conf = this->private;
+
+    list_del(&ne->list);
+    GF_FREE(ne->name);
+    GF_FREE(ne);
+
+    nlc_ctx->cache_size -= sizeof(*ne) + sizeof(ne->name);
+    GF_ATOMIC_SUB(conf->current_cache_size, (sizeof(*ne) + sizeof(ne->name)));
+
+    return;
+}
+
+void
+nlc_inode_clear_cache(xlator_t *this, inode_t *inode, int reason)
+{
+    nlc_ctx_t *nlc_ctx = NULL;
+
+    nlc_inode_ctx_get(this, inode, &nlc_ctx);
+    if (!nlc_ctx)
+        goto out;
+
+    LOCK(&nlc_ctx->lock);
+    {
+        __nlc_inode_ctx_timer_delete(this, nlc_ctx);
+
+        __nlc_inode_clear_entries(this, nlc_ctx);
+    }
+    UNLOCK(&nlc_ctx->lock);
+
+    if (reason != NLC_LRU_PRUNE)
+        nlc_remove_from_lru(this, inode);
+
+out:
+    return;
+}
+
+static void
+__nlc_del_pe(xlator_t *this, nlc_ctx_t *nlc_ctx, inode_t *entry_ino,
+             const char *name, gf_boolean_t multilink)
+{
+    nlc_pe_t *pe = NULL;
+    nlc_pe_t *tmp = NULL;
+    gf_boolean_t found = _gf_false;
+    uint64_t pe_int = 0;
+
+    if (!IS_PE_VALID(nlc_ctx->state))
+        goto out;
+
+    if (!entry_ino)
+        goto name_search;
+
+    /* If there are hardlinks first search names, followed by inodes */
+    if (multilink) {
+        list_for_each_entry_safe(pe, tmp, &nlc_ctx->pe, list)
+        {
+            if (pe->name && (strcmp(pe->name, name) == 0)) {
+                found = _gf_true;
+                goto out;
+            }
+        }
+        inode_ctx_reset1(entry_ino, this, &pe_int);
+        if (pe_int) {
+            pe = (void *)(long)(pe_int);
+            found = _gf_true;
+            goto out;
+        }
+        goto out;
+    }
+
+    inode_ctx_reset1(entry_ino, this, &pe_int);
+    if (pe_int) {
+        pe = (void *)(long)(pe_int);
+        found = _gf_true;
+        goto out;
+    }
+
+name_search:
+    list_for_each_entry_safe(pe, tmp, &nlc_ctx->pe, list)
+    {
+        if (pe->name && (strcmp(pe->name, name) == 0)) {
+            found = _gf_true;
+            break;
+            /* TODO: can there be duplicates? */
+        }
+    }
+
+out:
+    if (found)
+        __nlc_free_pe(this, nlc_ctx, pe);
+
+    return;
+}
+
+static void
+__nlc_del_ne(xlator_t *this, nlc_ctx_t *nlc_ctx, const char *name)
+{
+    nlc_ne_t *ne = NULL;
+    nlc_ne_t *tmp = NULL;
+
+    if (!IS_NE_VALID(nlc_ctx->state))
+        goto out;
+
+    list_for_each_entry_safe(ne, tmp, &nlc_ctx->ne, list)
+    {
+        if (strcmp(ne->name, name) == 0) {
+            __nlc_free_ne(this, nlc_ctx, ne);
+            break;
+        }
+    }
+out:
+    return;
+}
+
+static void
+__nlc_add_pe(xlator_t *this, nlc_ctx_t *nlc_ctx, inode_t *entry_ino,
+             const char *name)
+{
+    nlc_pe_t *pe = NULL;
+    int ret = -1;
+    nlc_conf_t *conf = NULL;
+    uint64_t nlc_ctx_int = 0;
+
+    conf = this->private;
+
+    /* TODO: There can be no duplicate entries, as it is added only
+    during create. In case there arises duplicate entries, search PE
+    found = __nlc_search (entries, name, _gf_false);
+    can use bit vector to have simple search than sequential search */
+
+    pe = GF_CALLOC(sizeof(*pe), 1, gf_nlc_mt_nlc_pe_t);
+    if (!pe)
+        goto out;
+
+    if (entry_ino) {
+        pe->inode = inode_ref(entry_ino);
+        nlc_inode_ctx_set(this, entry_ino, NULL, pe);
+    } else if (name) {
+        pe->name = gf_strdup(name);
+        if (!pe->name)
+            goto out;
+    }
+
+    list_add(&pe->list, &nlc_ctx->pe);
+
+    nlc_ctx->cache_size += sizeof(*pe) + sizeof(pe->name);
+    GF_ATOMIC_ADD(conf->current_cache_size, (sizeof(*pe) + sizeof(pe->name)));
+
+    nlc_ctx->refd_inodes += 1;
+    inode_ctx_get2(entry_ino, this, &nlc_ctx_int, NULL);
+    if (nlc_ctx_int == 0)
+        GF_ATOMIC_ADD(conf->refd_inodes, 1);
+
+    ret = 0;
+out:
+    if (ret)
+        GF_FREE(pe);
+
+    return;
+}
+
+static void
+__nlc_add_ne(xlator_t *this, nlc_ctx_t *nlc_ctx, const char *name)
+{
+    nlc_ne_t *ne = NULL;
+    int ret = -1;
+    nlc_conf_t *conf = NULL;
+
+    conf = this->private;
+
+    /* TODO: search ne before adding to get rid of duplicate entries
+    found = __nlc_search (entries, name, _gf_false);
+    can use bit vector to have faster search than sequential search */
+
+    ne = GF_CALLOC(sizeof(*ne), 1, gf_nlc_mt_nlc_ne_t);
+    if (!ne)
+        goto out;
+
+    ne->name = gf_strdup(name);
+    if (!ne->name)
+        goto out;
+
+    list_add(&ne->list, &nlc_ctx->ne);
+
+    nlc_ctx->cache_size += sizeof(*ne) + sizeof(ne->name);
+    GF_ATOMIC_ADD(conf->current_cache_size, (sizeof(*ne) + sizeof(ne->name)));
+    ret = 0;
+out:
+    if (ret)
+        GF_FREE(ne);
+
+    return;
+}
+
+void
+nlc_dir_add_ne(xlator_t *this, inode_t *inode, const char *name)
+{
+    nlc_ctx_t *nlc_ctx = NULL;
+
+    if (inode->ia_type != IA_IFDIR) {
+        gf_msg_callingfn(this->name, GF_LOG_ERROR, EINVAL, NLC_MSG_EINVAL,
+                         "inode is not of type dir");
+        goto out;
+    }
+
+    nlc_inode_ctx_get_set(this, inode, &nlc_ctx);
+    if (!nlc_ctx)
+        goto out;
+
+    LOCK(&nlc_ctx->lock);
+    {
+        /* There is one possibility where we need to search before
+         * adding NE: when there are two parallel lookups on a non
+         * existent file */
+        if (!__nlc_search_ne(nlc_ctx, name)) {
+            __nlc_add_ne(this, nlc_ctx, name);
+            __nlc_set_dir_state(nlc_ctx, NLC_NE_VALID);
+        }
+    }
+    UNLOCK(&nlc_ctx->lock);
+out:
+    return;
+}
+
+void
+nlc_dir_remove_pe(xlator_t *this, inode_t *parent, inode_t *entry_ino,
+                  const char *name, gf_boolean_t multilink)
+{
+    nlc_ctx_t *nlc_ctx = NULL;
+
+    if (parent->ia_type != IA_IFDIR) {
+        gf_msg_callingfn(this->name, GF_LOG_ERROR, EINVAL, NLC_MSG_EINVAL,
+                         "inode is not of type dir");
+        goto out;
+    }
+
+    nlc_inode_ctx_get(this, parent, &nlc_ctx);
+    if (!nlc_ctx)
+        goto out;
+
+    LOCK(&nlc_ctx->lock);
+    {
+        if (!__nlc_is_cache_valid(this, nlc_ctx))
+            goto unlock;
+
+        __nlc_del_pe(this, nlc_ctx, entry_ino, name, multilink);
+        __nlc_add_ne(this, nlc_ctx, name);
+        __nlc_set_dir_state(nlc_ctx, NLC_NE_VALID);
+    }
+unlock:
+    UNLOCK(&nlc_ctx->lock);
+out:
+    return;
+}
+
+void
+nlc_dir_add_pe(xlator_t *this, inode_t *inode, inode_t *entry_ino,
+               const char *name)
+{
+    nlc_ctx_t *nlc_ctx = NULL;
+
+    if (inode->ia_type != IA_IFDIR) {
+        gf_msg_callingfn(this->name, GF_LOG_ERROR, EINVAL, NLC_MSG_EINVAL,
+                         "inode is not of type dir");
+        goto out;
+    }
+
+    nlc_inode_ctx_get_set(this, inode, &nlc_ctx);
+    if (!nlc_ctx)
+        goto out;
+
+    LOCK(&nlc_ctx->lock);
+    {
+        __nlc_del_ne(this, nlc_ctx, name);
+        __nlc_add_pe(this, nlc_ctx, entry_ino, name);
+        if (!IS_PE_VALID(nlc_ctx->state))
+            __nlc_set_dir_state(nlc_ctx, NLC_PE_PARTIAL);
+    }
+    UNLOCK(&nlc_ctx->lock);
+out:
+    return;
+}
+
+gf_boolean_t
+__nlc_search_ne(nlc_ctx_t *nlc_ctx, const char *name)
+{
+    gf_boolean_t found = _gf_false;
+    nlc_ne_t *ne = NULL;
+    nlc_ne_t *tmp = NULL;
+
+    if (!IS_NE_VALID(nlc_ctx->state))
+        goto out;
+
+    list_for_each_entry_safe(ne, tmp, &nlc_ctx->ne, list)
+    {
+        if (strcmp(ne->name, name) == 0) {
+            found = _gf_true;
+            break;
+        }
+    }
+out:
+    return found;
+}
+
+static gf_boolean_t
+__nlc_search_pe(nlc_ctx_t *nlc_ctx, const char *name)
+{
+    gf_boolean_t found = _gf_false;
+    nlc_pe_t *pe = NULL;
+    nlc_pe_t *tmp = NULL;
+
+    if (!IS_PE_VALID(nlc_ctx->state))
+        goto out;
+
+    list_for_each_entry_safe(pe, tmp, &nlc_ctx->pe, list)
+    {
+        if (pe->name && (strcmp(pe->name, name) == 0)) {
+            found = _gf_true;
+            break;
+        }
+    }
+out:
+    return found;
+}
+
+static char *
+__nlc_get_pe(nlc_ctx_t *nlc_ctx, const char *name,
+             gf_boolean_t case_insensitive)
+{
+    char *found = NULL;
+    nlc_pe_t *pe = NULL;
+    nlc_pe_t *tmp = NULL;
+
+    if (!IS_PE_VALID(nlc_ctx->state))
+        goto out;
+
+    if (case_insensitive) {
+        list_for_each_entry_safe(pe, tmp, &nlc_ctx->pe, list)
+        {
+            if (pe->name && (strcasecmp(pe->name, name) == 0)) {
+                found = pe->name;
+                break;
+            }
+        }
+    } else {
+        list_for_each_entry_safe(pe, tmp, &nlc_ctx->pe, list)
+        {
+            if (pe->name && (strcmp(pe->name, name) == 0)) {
+                found = pe->name;
+                break;
+            }
+        }
+    }
+out:
+    return found;
+}
+
+gf_boolean_t
+nlc_is_negative_lookup(xlator_t *this, loc_t *loc)
+{
+    nlc_ctx_t *nlc_ctx = NULL;
+    inode_t *inode = NULL;
+    gf_boolean_t neg_entry = _gf_false;
+
+    inode = loc->parent;
+    GF_VALIDATE_OR_GOTO(this->name, inode, out);
+
+    if (inode->ia_type != IA_IFDIR) {
+        gf_msg_callingfn(this->name, GF_LOG_ERROR, EINVAL, NLC_MSG_EINVAL,
+                         "inode is not of type dir");
+        goto out;
+    }
+
+    nlc_inode_ctx_get(this, inode, &nlc_ctx);
+    if (!nlc_ctx)
+        goto out;
+
+    LOCK(&nlc_ctx->lock);
+    {
+        if (!__nlc_is_cache_valid(this, nlc_ctx))
+            goto unlock;
+
+        if (__nlc_search_ne(nlc_ctx, loc->name)) {
+            neg_entry = _gf_true;
+            goto unlock;
+        }
+        if ((nlc_ctx->state & NLC_PE_FULL) &&
+            !__nlc_search_pe(nlc_ctx, loc->name)) {
+            neg_entry = _gf_true;
+            goto unlock;
+        }
+    }
+unlock:
+    UNLOCK(&nlc_ctx->lock);
+
+out:
+    return neg_entry;
+}
+
+gf_boolean_t
+nlc_get_real_file_name(xlator_t *this, loc_t *loc, const char *fname,
+                       int32_t *op_ret, int32_t *op_errno, dict_t *dict)
+{
+    nlc_ctx_t *nlc_ctx = NULL;
+    inode_t *inode = NULL;
+    gf_boolean_t hit = _gf_false;
+    char *found_file = NULL;
+    int ret = 0;
+
+    GF_VALIDATE_OR_GOTO(this->name, loc, out);
+    GF_VALIDATE_OR_GOTO(this->name, fname, out);
+    GF_VALIDATE_OR_GOTO(this->name, op_ret, out);
+    GF_VALIDATE_OR_GOTO(this->name, op_errno, out);
+    GF_VALIDATE_OR_GOTO(this->name, dict, out);
+
+    inode = loc->inode;
+    GF_VALIDATE_OR_GOTO(this->name, inode, out);
+
+    if (inode->ia_type != IA_IFDIR) {
+        gf_msg_callingfn(this->name, GF_LOG_ERROR, EINVAL, NLC_MSG_EINVAL,
+                         "inode is not of type dir");
+        goto out;
+    }
+
+    nlc_inode_ctx_get(this, inode, &nlc_ctx);
+    if (!nlc_ctx)
+        goto out;
+
+    LOCK(&nlc_ctx->lock);
+    {
+        if (!__nlc_is_cache_valid(this, nlc_ctx))
+            goto unlock;
+
+        found_file = __nlc_get_pe(nlc_ctx, fname, _gf_true);
+        if (found_file) {
+            ret = dict_set_dynstr(dict, GF_XATTR_GET_REAL_FILENAME_KEY,
+                                  gf_strdup(found_file));
+            if (ret < 0)
+                goto unlock;
+            *op_ret = strlen(found_file) + 1;
+            hit = _gf_true;
+            goto unlock;
+        }
+        if (!found_file && (nlc_ctx->state & NLC_PE_FULL)) {
+            *op_ret = -1;
+            *op_errno = ENOENT;
+            hit = _gf_true;
+            goto unlock;
+        }
+    }
+unlock:
+    UNLOCK(&nlc_ctx->lock);
+
+out:
+    return hit;
+}
+
+void
+nlc_dump_inodectx(xlator_t *this, inode_t *inode)
+{
+    int32_t ret = -1;
+    char *path = NULL;
+    char key_prefix[GF_DUMP_MAX_BUF_LEN] = {
+        0,
+    };
+    char uuid_str[64] = {
+        0,
+    };
+    nlc_ctx_t *nlc_ctx = NULL;
+    nlc_pe_t *pe = NULL;
+    nlc_pe_t *tmp = NULL;
+    nlc_ne_t *ne = NULL;
+    nlc_ne_t *tmp1 = NULL;
+
+    nlc_inode_ctx_get(this, inode, &nlc_ctx);
+
+    if (!nlc_ctx)
+        goto out;
+
+    ret = TRY_LOCK(&nlc_ctx->lock);
+    if (!ret) {
+        gf_proc_dump_build_key(key_prefix, "xlator.performance.nl-cache",
+                               "nlc_inode");
+        gf_proc_dump_add_section("%s", key_prefix);
+
+        __inode_path(inode, NULL, &path);
+        if (path != NULL) {
+            gf_proc_dump_write("path", "%s", path);
+            GF_FREE(path);
+        }
+
+        uuid_utoa_r(inode->gfid, uuid_str);
+
+        gf_proc_dump_write("inode", "%p", inode);
+        gf_proc_dump_write("gfid", "%s", uuid_str);
+
+        gf_proc_dump_write("state", "%" PRIu64, nlc_ctx->state);
+        gf_proc_dump_write("timer", "%p", nlc_ctx->timer);
+        gf_proc_dump_write("cache-time", "%ld", nlc_ctx->cache_time);
+        gf_proc_dump_write("cache-size", "%zu", nlc_ctx->cache_size);
+        gf_proc_dump_write("refd-inodes", "%" PRIu64, nlc_ctx->refd_inodes);
+
+        if (IS_PE_VALID(nlc_ctx->state))
+            list_for_each_entry_safe(pe, tmp, &nlc_ctx->pe, list)
+            {
+                gf_proc_dump_write("pe", "%p, %p, %s", pe, pe->inode, pe->name);
+            }
+
+        if (IS_NE_VALID(nlc_ctx->state))
+            list_for_each_entry_safe(ne, tmp1, &nlc_ctx->ne, list)
+            {
+                gf_proc_dump_write("ne", "%s", ne->name);
+            }
+
+        UNLOCK(&nlc_ctx->lock);
+    }
+
+    if (ret && nlc_ctx)
+        gf_proc_dump_write("Unable to dump the inode information",
+                           "(Lock acquisition failed) %p (gfid: %s)", nlc_ctx,
+                           uuid_str);
+out:
+    return;
+}
diff --git a/xlators/performance/nl-cache/src/nl-cache-mem-types.h b/xlators/performance/nl-cache/src/nl-cache-mem-types.h
new file mode 100644
index 00000000000..93a17b3fd5a
--- /dev/null
+++ b/xlators/performance/nl-cache/src/nl-cache-mem-types.h
@@ -0,0 +1,27 @@
+/*
+ *   Copyright (c) 2016 Red Hat, Inc. <http://www.redhat.com>
+ *   This file is part of GlusterFS.
+ *
+ *   This file is licensed to you under your choice of the GNU Lesser
+ *   General Public License, version 3 or any later version (LGPLv3 or
+ *   later), or the GNU General Public License, version 2 (GPLv2), in all
+ *   cases as published by the Free Software Foundation.
+ */
+
+#ifndef __NL_CACHE_MEM_TYPES_H__
+#define __NL_CACHE_MEM_TYPES_H__
+
+#include <glusterfs/mem-types.h>
+
+enum gf_nlc_mem_types_ {
+    gf_nlc_mt_nlc_conf_t = gf_common_mt_end + 1,
+    gf_nlc_mt_nlc_ctx_t,
+    gf_nlc_mt_nlc_local_t,
+    gf_nlc_mt_nlc_pe_t,
+    gf_nlc_mt_nlc_ne_t,
+    gf_nlc_mt_nlc_timer_data_t,
+    gf_nlc_mt_nlc_lru_node,
+    gf_nlc_mt_end
+};
+
+#endif /* __NL_CACHE_MEM_TYPES_H__ */
diff --git a/xlators/performance/nl-cache/src/nl-cache-messages.h b/xlators/performance/nl-cache/src/nl-cache-messages.h
new file mode 100644
index 00000000000..222d709e133
--- /dev/null
+++ b/xlators/performance/nl-cache/src/nl-cache-messages.h
@@ -0,0 +1,29 @@
+/*
+ *   Copyright (c) 2016 Red Hat, Inc. <http://www.redhat.com>
+ *   This file is part of GlusterFS.
+ *
+ *   This file is licensed to you under your choice of the GNU Lesser
+ *   General Public License, version 3 or any later version (LGPLv3 or
+ *   later), or the GNU General Public License, version 2 (GPLv2), in all
+ *   cases as published by the Free Software Foundation.
+ */
+
+#ifndef __NL_CACHE_MESSAGES_H__
+#define __NL_CACHE_MESSAGES_H__
+
+#include <glusterfs/glfs-message-id.h>
+
+/* To add new message IDs, append new identifiers at the end of the list.
+ *
+ * Never remove a message ID. If it's not used anymore, you can rename it or
+ * leave it as it is, but not delete it. This is to prevent reutilization of
+ * IDs by other messages.
+ *
+ * The component name must match one of the entries defined in
+ * glfs-message-id.h.
+ */
+
+GLFS_MSGID(NLC, NLC_MSG_NO_MEMORY, NLC_MSG_EINVAL, NLC_MSG_NO_TIMER_WHEEL,
+           NLC_MSG_DICT_FAILURE);
+
+#endif /* __NL_CACHE_MESSAGES_H__ */
diff --git a/xlators/performance/nl-cache/src/nl-cache.c b/xlators/performance/nl-cache/src/nl-cache.c
new file mode 100644
index 00000000000..33a7c471663
--- /dev/null
+++ b/xlators/performance/nl-cache/src/nl-cache.c
@@ -0,0 +1,840 @@
+/*
+ *   Copyright (c) 2016 Red Hat, Inc. <http://www.redhat.com>
+ *   This file is part of GlusterFS.
+ *
+ *   This file is licensed to you under your choice of the GNU Lesser
+ *   General Public License, version 3 or any later version (LGPLv3 or
+ *   later), or the GNU General Public License, version 2 (GPLv2), in all
+ *   cases as published by the Free Software Foundation.
+ */
+
+#include "nl-cache.h"
+#include <glusterfs/statedump.h>
+#include <glusterfs/upcall-utils.h>
+
+static void
+nlc_dentry_op(call_frame_t *frame, xlator_t *this, gf_boolean_t multilink)
+{
+    nlc_local_t *local = frame->local;
+
+    GF_VALIDATE_OR_GOTO(this->name, local, out);
+
+    switch (local->fop) {
+        case GF_FOP_MKDIR:
+            nlc_set_dir_state(this, local->loc.inode, NLC_PE_FULL);
+            /*fall-through*/
+        case GF_FOP_MKNOD:
+        case GF_FOP_CREATE:
+        case GF_FOP_SYMLINK:
+            nlc_dir_add_pe(this, local->loc.parent, local->loc.inode,
+                           local->loc.name);
+            break;
+        case GF_FOP_LINK:
+            nlc_dir_add_pe(this, local->loc2.parent, NULL, local->loc2.name);
+            break;
+        case GF_FOP_RMDIR:
+            nlc_inode_clear_cache(this, local->loc.inode, _gf_false);
+            /*fall-through*/
+        case GF_FOP_UNLINK:
+            nlc_dir_remove_pe(this, local->loc.parent, local->loc.inode,
+                              local->loc.name, multilink);
+            break;
+        case GF_FOP_RENAME:
+            /* TBD: Should these be atomic ?  In case of rename, the
+             * newloc->inode can be NULL, and hence use oldloc->inode */
+            nlc_dir_remove_pe(this, local->loc2.parent, local->loc2.inode,
+                              local->loc2.name, _gf_false);
+
+            /*TODO: Remove old dentry from destination before adding this pe*/
+            nlc_dir_add_pe(this, local->loc.parent, local->loc2.inode,
+                           local->loc.name);
+
+        default:
+            return;
+    }
+
+    nlc_lru_prune(this, NULL);
+out:
+    return;
+}
+
+#define NLC_FOP(_name, _op, loc1, loc2, frame, this, args...)                  \
+    do {                                                                       \
+        nlc_local_t *__local = NULL;                                           \
+        nlc_conf_t *conf = NULL;                                               \
+                                                                               \
+        conf = this->private;                                                  \
+                                                                               \
+        if (!IS_PEC_ENABLED(conf))                                             \
+            goto disabled;                                                     \
+                                                                               \
+        __local = nlc_local_init(frame, this, _op, loc1, loc2);                \
+        GF_VALIDATE_OR_GOTO(this->name, __local, err);                         \
+                                                                               \
+        STACK_WIND(frame, nlc_##_name##_cbk, FIRST_CHILD(this),                \
+                   FIRST_CHILD(this)->fops->_name, args);                      \
+        break;                                                                 \
+    disabled:                                                                  \
+        default_##_name##_resume(frame, this, args);                           \
+        break;                                                                 \
+    err:                                                                       \
+        default_##_name##_failure_cbk(frame, ENOMEM);                          \
+        break;                                                                 \
+    } while (0)
+
+#define NLC_FOP_CBK(_name, multilink, frame, cookie, this, op_ret, op_errno,   \
+                    args...)                                                   \
+    do {                                                                       \
+        nlc_conf_t *conf = NULL;                                               \
+                                                                               \
+        if (op_ret != 0)                                                       \
+            goto out;                                                          \
+                                                                               \
+        conf = this->private;                                                  \
+                                                                               \
+        if (op_ret < 0 || !IS_PEC_ENABLED(conf))                               \
+            goto out;                                                          \
+        nlc_dentry_op(frame, this, multilink);                                 \
+    out:                                                                       \
+        NLC_STACK_UNWIND(_name, frame, op_ret, op_errno, args);                \
+    } while (0)
+
+static int32_t
+nlc_rename_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+               int32_t op_ret, int32_t op_errno, struct iatt *buf,
+               struct iatt *preoldparent, struct iatt *postoldparent,
+               struct iatt *prenewparent, struct iatt *postnewparent,
+               dict_t *xdata)
+{
+    NLC_FOP_CBK(rename, _gf_false, frame, cookie, this, op_ret, op_errno, buf,
+                preoldparent, postoldparent, prenewparent, postnewparent,
+                xdata);
+    return 0;
+}
+
+static int32_t
+nlc_rename(call_frame_t *frame, xlator_t *this, loc_t *oldloc, loc_t *newloc,
+           dict_t *xdata)
+{
+    NLC_FOP(rename, GF_FOP_RENAME, newloc, oldloc, frame, this, oldloc, newloc,
+            xdata);
+    return 0;
+}
+
+static int32_t
+nlc_mknod_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int32_t op_ret,
+              int32_t op_errno, inode_t *inode, struct iatt *buf,
+              struct iatt *preparent, struct iatt *postparent, dict_t *xdata)
+{
+    NLC_FOP_CBK(mknod, _gf_false, frame, cookie, this, op_ret, op_errno, inode,
+                buf, preparent, postparent, xdata);
+    return 0;
+}
+
+static int32_t
+nlc_mknod(call_frame_t *frame, xlator_t *this, loc_t *loc, mode_t mode,
+          dev_t rdev, mode_t umask, dict_t *xdata)
+{
+    NLC_FOP(mknod, GF_FOP_MKNOD, loc, NULL, frame, this, loc, mode, rdev, umask,
+            xdata);
+    return 0;
+}
+
+static int32_t
+nlc_create_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+               int32_t op_ret, int32_t op_errno, fd_t *fd, inode_t *inode,
+               struct iatt *buf, struct iatt *preparent,
+               struct iatt *postparent, dict_t *xdata)
+{
+    NLC_FOP_CBK(create, _gf_false, frame, cookie, this, op_ret, op_errno, fd,
+                inode, buf, preparent, postparent, xdata);
+    return 0;
+}
+
+static int32_t
+nlc_create(call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t flags,
+           mode_t mode, mode_t umask, fd_t *fd, dict_t *xdata)
+{
+    NLC_FOP(create, GF_FOP_CREATE, loc, NULL, frame, this, loc, flags, mode,
+            umask, fd, xdata);
+    return 0;
+}
+
+static int32_t
+nlc_mkdir_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int32_t op_ret,
+              int32_t op_errno, inode_t *inode, struct iatt *buf,
+              struct iatt *preparent, struct iatt *postparent, dict_t *xdata)
+{
+    NLC_FOP_CBK(mkdir, _gf_false, frame, cookie, this, op_ret, op_errno, inode,
+                buf, preparent, postparent, xdata);
+    return 0;
+}
+
+static int32_t
+nlc_mkdir(call_frame_t *frame, xlator_t *this, loc_t *loc, mode_t mode,
+          mode_t umask, dict_t *xdata)
+{
+    NLC_FOP(mkdir, GF_FOP_MKDIR, loc, NULL, frame, this, loc, mode, umask,
+            xdata);
+    return 0;
+}
+
+static int32_t
+nlc_lookup_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+               int32_t op_ret, int32_t op_errno, inode_t *inode,
+               struct iatt *buf, dict_t *xdata, struct iatt *postparent)
+{
+    nlc_local_t *local = NULL;
+    nlc_conf_t *conf = NULL;
+
+    local = frame->local;
+    conf = this->private;
+
+    if (!local)
+        goto out;
+
+    /* Donot add to pe, this may lead to duplicate entry and
+     * requires search before adding if list of strings */
+    if (op_ret < 0 && op_errno == ENOENT) {
+        nlc_dir_add_ne(this, local->loc.parent, local->loc.name);
+        GF_ATOMIC_INC(conf->nlc_counter.nlc_miss);
+    }
+
+out:
+    NLC_STACK_UNWIND(lookup, frame, op_ret, op_errno, inode, buf, xdata,
+                     postparent);
+    return 0;
+}
+
+static int32_t
+nlc_lookup(call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xdata)
+{
+    nlc_local_t *local = NULL;
+    nlc_conf_t *conf = NULL;
+    inode_t *inode = NULL;
+
+    if (loc_is_nameless(loc))
+        goto wind;
+
+    local = nlc_local_init(frame, this, GF_FOP_LOOKUP, loc, NULL);
+    if (!local)
+        goto err;
+
+    conf = this->private;
+
+    inode = inode_grep(loc->inode->table, loc->parent, loc->name);
+    if (inode) {
+        inode_unref(inode);
+        goto wind;
+    }
+
+    if (nlc_is_negative_lookup(this, loc)) {
+        GF_ATOMIC_INC(conf->nlc_counter.nlc_hit);
+        gf_msg_trace(this->name, 0,
+                     "Serving negative lookup from "
+                     "cache:%s",
+                     loc->name);
+        goto unwind;
+    }
+
+wind:
+    STACK_WIND(frame, nlc_lookup_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->lookup, loc, xdata);
+    return 0;
+unwind:
+    NLC_STACK_UNWIND(lookup, frame, -1, ENOENT, NULL, NULL, NULL, NULL);
+    return 0;
+err:
+    NLC_STACK_UNWIND(lookup, frame, -1, ENOMEM, NULL, NULL, NULL, NULL);
+    return 0;
+}
+
+static int32_t
+nlc_rmdir_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int32_t op_ret,
+              int32_t op_errno, struct iatt *preparent, struct iatt *postparent,
+              dict_t *xdata)
+{
+    NLC_FOP_CBK(rmdir, _gf_false, frame, cookie, this, op_ret, op_errno,
+                preparent, postparent, xdata);
+    return 0;
+}
+
+static int32_t
+nlc_rmdir(call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t flags,
+          dict_t *xdata)
+{
+    NLC_FOP(rmdir, GF_FOP_RMDIR, loc, NULL, frame, this, loc, flags, xdata);
+    return 0;
+}
+
+static int32_t
+nlc_getxattr_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                 int32_t op_ret, int32_t op_errno, dict_t *dict, dict_t *xdata)
+{
+    nlc_conf_t *conf = NULL;
+
+    conf = this->private;
+    GF_VALIDATE_OR_GOTO(this->name, conf, out);
+
+    if (!IS_PEC_ENABLED(conf))
+        goto out;
+
+    if (op_ret < 0 && op_errno == ENOENT) {
+        GF_ATOMIC_INC(conf->nlc_counter.getrealfilename_miss);
+    }
+
+out:
+    NLC_STACK_UNWIND(getxattr, frame, op_ret, op_errno, dict, xdata);
+    return 0;
+}
+
+static int32_t
+nlc_getxattr(call_frame_t *frame, xlator_t *this, loc_t *loc, const char *key,
+             dict_t *xdata)
+{
+    int32_t op_ret = -1;
+    int32_t op_errno = 0;
+    dict_t *dict = NULL;
+    nlc_local_t *local = NULL;
+    gf_boolean_t hit = _gf_false;
+    const char *fname = NULL;
+    nlc_conf_t *conf = NULL;
+
+    conf = this->private;
+
+    if (!IS_PEC_ENABLED(conf))
+        goto wind;
+
+    if (!key || (strncmp(key, GF_XATTR_GET_REAL_FILENAME_KEY,
+                         SLEN(GF_XATTR_GET_REAL_FILENAME_KEY)) != 0))
+        goto wind;
+
+    local = nlc_local_init(frame, this, GF_FOP_GETXATTR, loc, NULL);
+    if (!local)
+        goto err;
+
+    if (loc->inode && key) {
+        dict = dict_new();
+        if (!dict)
+            goto err;
+
+        fname = key + SLEN(GF_XATTR_GET_REAL_FILENAME_KEY);
+        hit = nlc_get_real_file_name(this, loc, fname, &op_ret, &op_errno,
+                                     dict);
+        if (hit)
+            goto unwind;
+        else
+            dict_unref(dict);
+    }
+
+    STACK_WIND(frame, nlc_getxattr_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->getxattr, loc, key, xdata);
+    return 0;
+wind:
+    STACK_WIND(frame, default_getxattr_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->getxattr, loc, key, xdata);
+    return 0;
+unwind:
+    GF_ATOMIC_INC(conf->nlc_counter.getrealfilename_hit);
+    NLC_STACK_UNWIND(getxattr, frame, op_ret, op_errno, dict, NULL);
+    dict_unref(dict);
+    return 0;
+err:
+    NLC_STACK_UNWIND(getxattr, frame, -1, ENOMEM, NULL, NULL);
+    return 0;
+}
+
+static int32_t
+nlc_symlink_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                int32_t op_ret, int32_t op_errno, inode_t *inode,
+                struct iatt *buf, struct iatt *preparent,
+                struct iatt *postparent, dict_t *xdata)
+{
+    NLC_FOP_CBK(symlink, _gf_false, frame, cookie, this, op_ret, op_errno,
+                inode, buf, preparent, postparent, xdata);
+    return 0;
+}
+
+static int32_t
+nlc_symlink(call_frame_t *frame, xlator_t *this, const char *linkpath,
+            loc_t *loc, mode_t umask, dict_t *xdata)
+{
+    NLC_FOP(symlink, GF_FOP_SYMLINK, loc, NULL, frame, this, linkpath, loc,
+            umask, xdata);
+    return 0;
+}
+
+static int32_t
+nlc_link_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int32_t op_ret,
+             int32_t op_errno, inode_t *inode, struct iatt *buf,
+             struct iatt *preparent, struct iatt *postparent, dict_t *xdata)
+{
+    NLC_FOP_CBK(link, _gf_false, frame, cookie, this, op_ret, op_errno, inode,
+                buf, preparent, postparent, xdata);
+    return 0;
+}
+
+static int32_t
+nlc_link(call_frame_t *frame, xlator_t *this, loc_t *oldloc, loc_t *newloc,
+         dict_t *xdata)
+{
+    NLC_FOP(link, GF_FOP_LINK, oldloc, newloc, frame, this, oldloc, newloc,
+            xdata);
+    return 0;
+}
+
+static int32_t
+nlc_unlink_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+               int32_t op_ret, int32_t op_errno, struct iatt *preparent,
+               struct iatt *postparent, dict_t *xdata)
+{
+    uint32_t link_count = 0;
+    gf_boolean_t multilink = _gf_false;
+
+    if (xdata && !dict_get_uint32(xdata, GET_LINK_COUNT, &link_count)) {
+        if (link_count > 1)
+            multilink = _gf_true;
+    } else {
+        /* Don't touch cache if we don't know enough */
+        gf_msg(this->name, GF_LOG_WARNING, 0, NLC_MSG_DICT_FAILURE,
+               "Failed to get GET_LINK_COUNT from dict");
+        NLC_STACK_UNWIND(unlink, frame, op_ret, op_errno, preparent, postparent,
+                         xdata);
+        return 0;
+    }
+
+    NLC_FOP_CBK(unlink, multilink, frame, cookie, this, op_ret, op_errno,
+                preparent, postparent, xdata);
+    return 0;
+}
+
+static int32_t
+nlc_unlink(call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t flags,
+           dict_t *xdata)
+{
+    nlc_conf_t *conf = NULL;
+    gf_boolean_t new_dict = _gf_false;
+
+    conf = this->private;
+
+    if (!IS_PEC_ENABLED(conf))
+        goto do_fop;
+
+    if (!xdata) {
+        xdata = dict_new();
+        if (xdata)
+            new_dict = _gf_true;
+    }
+
+    if (xdata && dict_set_uint32(xdata, GET_LINK_COUNT, 0)) {
+        gf_msg(this->name, GF_LOG_WARNING, 0, NLC_MSG_DICT_FAILURE,
+               "Failed to set GET_LINK_COUNT in dict");
+        goto err;
+    }
+
+do_fop:
+    NLC_FOP(unlink, GF_FOP_UNLINK, loc, NULL, frame, this, loc, flags, xdata);
+
+    if (new_dict)
+        dict_unref(xdata);
+    return 0;
+}
+
+static int32_t
+nlc_invalidate(xlator_t *this, void *data)
+{
+    struct gf_upcall *up_data = NULL;
+    struct gf_upcall_cache_invalidation *up_ci = NULL;
+    inode_t *inode = NULL;
+    inode_t *parent1 = NULL;
+    inode_t *parent2 = NULL;
+    int ret = 0;
+    inode_table_t *itable = NULL;
+    nlc_conf_t *conf = NULL;
+
+    up_data = (struct gf_upcall *)data;
+
+    if (up_data->event_type != GF_UPCALL_CACHE_INVALIDATION)
+        goto out;
+
+    conf = this->private;
+    if (!conf)
+        goto out;
+
+    up_ci = (struct gf_upcall_cache_invalidation *)up_data->data;
+
+    /*TODO: Add he inodes found as a member in gf_upcall_cache_invalidation
+     * so that it prevents subsequent xlators from doing inode_find again
+     */
+    itable = ((xlator_t *)this->graph->top)->itable;
+    inode = inode_find(itable, up_data->gfid);
+    if (!inode) {
+        ret = -1;
+        goto out;
+    }
+
+    if ((!((up_ci->flags & UP_TIMES) && inode->ia_type == IA_IFDIR)) &&
+        (!(up_ci->flags & UP_PARENT_DENTRY_FLAGS))) {
+        goto out;
+    }
+
+    if (!gf_uuid_is_null(up_ci->p_stat.ia_gfid)) {
+        parent1 = inode_find(itable, up_ci->p_stat.ia_gfid);
+        if (!parent1) {
+            ret = -1;
+            goto out;
+        }
+    }
+
+    if (!gf_uuid_is_null(up_ci->oldp_stat.ia_gfid)) {
+        parent2 = inode_find(itable, up_ci->oldp_stat.ia_gfid);
+        if (!parent2) {
+            ret = -1;
+            goto out;
+        }
+    }
+
+    /* TODO: get enough data in upcall so that we do not invalidate but
+     * update */
+    if (inode && inode->ia_type == IA_IFDIR)
+        nlc_inode_clear_cache(this, inode, NLC_NONE);
+    if (parent1)
+        nlc_inode_clear_cache(this, parent1, NLC_NONE);
+    if (parent2)
+        nlc_inode_clear_cache(this, parent2, NLC_NONE);
+
+    GF_ATOMIC_INC(conf->nlc_counter.nlc_invals);
+
+out:
+    if (inode)
+        inode_unref(inode);
+    if (parent1)
+        inode_unref(parent1);
+    if (parent2)
+        inode_unref(parent2);
+
+    return ret;
+}
+
+int
+nlc_notify(xlator_t *this, int event, void *data, ...)
+{
+    int ret = 0;
+
+    switch (event) {
+        case GF_EVENT_CHILD_DOWN:
+        case GF_EVENT_SOME_DESCENDENT_DOWN:
+        case GF_EVENT_CHILD_UP:
+        case GF_EVENT_SOME_DESCENDENT_UP:
+            nlc_update_child_down_time(this, gf_time());
+            /* TODO: nlc_clear_all_cache (this); else
+             lru prune will lazily clear it*/
+            break;
+        case GF_EVENT_UPCALL:
+            ret = nlc_invalidate(this, data);
+            break;
+        case GF_EVENT_PARENT_DOWN:
+            nlc_disable_cache(this);
+            nlc_clear_all_cache(this);
+        default:
+            break;
+    }
+
+    if (default_notify(this, event, data) != 0)
+        ret = -1;
+
+    return ret;
+}
+
+static int32_t
+nlc_forget(xlator_t *this, inode_t *inode)
+{
+    uint64_t pe_int = 0;
+    uint64_t nlc_ctx_int = 0;
+    nlc_ctx_t *nlc_ctx = NULL;
+    nlc_conf_t *conf = NULL;
+
+    conf = this->private;
+
+    inode_ctx_reset1(inode, this, &pe_int);
+    GF_ASSERT(pe_int == 0);
+
+    nlc_inode_clear_cache(this, inode, NLC_NONE);
+    inode_ctx_reset0(inode, this, &nlc_ctx_int);
+    nlc_ctx = (void *)(long)nlc_ctx_int;
+    if (nlc_ctx) {
+        GF_FREE(nlc_ctx);
+        GF_ATOMIC_SUB(conf->current_cache_size, sizeof(*nlc_ctx));
+    }
+
+    return 0;
+}
+
+static int32_t
+nlc_inodectx(xlator_t *this, inode_t *inode)
+{
+    nlc_dump_inodectx(this, inode);
+    return 0;
+}
+
+static int32_t
+nlc_priv_dump(xlator_t *this)
+{
+    nlc_conf_t *conf = NULL;
+    char key_prefix[GF_DUMP_MAX_BUF_LEN];
+
+    conf = this->private;
+
+    snprintf(key_prefix, GF_DUMP_MAX_BUF_LEN, "%s.%s", this->type, this->name);
+    gf_proc_dump_add_section("%s", key_prefix);
+
+    gf_proc_dump_write("negative_lookup_hit_count", "%" PRId64,
+                       GF_ATOMIC_GET(conf->nlc_counter.nlc_hit));
+    gf_proc_dump_write("negative_lookup_miss_count", "%" PRId64,
+                       GF_ATOMIC_GET(conf->nlc_counter.nlc_miss));
+    gf_proc_dump_write("get_real_filename_hit_count", "%" PRId64,
+                       GF_ATOMIC_GET(conf->nlc_counter.getrealfilename_hit));
+    gf_proc_dump_write("get_real_filename_miss_count", "%" PRId64,
+                       GF_ATOMIC_GET(conf->nlc_counter.getrealfilename_miss));
+    gf_proc_dump_write("nameless_lookup_count", "%" PRId64,
+                       GF_ATOMIC_GET(conf->nlc_counter.nameless_lookup));
+    gf_proc_dump_write("inodes_with_positive_dentry_cache", "%" PRId64,
+                       GF_ATOMIC_GET(conf->nlc_counter.pe_inode_cnt));
+    gf_proc_dump_write("inodes_with_negative_dentry_cache", "%" PRId64,
+                       GF_ATOMIC_GET(conf->nlc_counter.ne_inode_cnt));
+    gf_proc_dump_write("dentry_invalidations_received", "%" PRId64,
+                       GF_ATOMIC_GET(conf->nlc_counter.nlc_invals));
+    gf_proc_dump_write("cache_limit", "%" PRIu64, conf->cache_size);
+    gf_proc_dump_write("consumed_cache_size", "%" PRId64,
+                       GF_ATOMIC_GET(conf->current_cache_size));
+    gf_proc_dump_write("inode_limit", "%" PRIu64, conf->inode_limit);
+    gf_proc_dump_write("consumed_inodes", "%" PRId64,
+                       GF_ATOMIC_GET(conf->refd_inodes));
+
+    return 0;
+}
+
+static int32_t
+nlc_dump_metrics(xlator_t *this, int fd)
+{
+    nlc_conf_t *conf = NULL;
+
+    conf = this->private;
+
+    dprintf(fd, "%s.negative_lookup_hit_count %" PRId64 "\n", this->name,
+            GF_ATOMIC_GET(conf->nlc_counter.nlc_hit));
+    dprintf(fd, "%s.negative_lookup_miss_count %" PRId64 "\n", this->name,
+            GF_ATOMIC_GET(conf->nlc_counter.nlc_miss));
+    dprintf(fd, "%s.get_real_filename_hit_count %" PRId64 "\n", this->name,
+            GF_ATOMIC_GET(conf->nlc_counter.getrealfilename_hit));
+    dprintf(fd, "%s.get_real_filename_miss_count %" PRId64 "\n", this->name,
+            GF_ATOMIC_GET(conf->nlc_counter.getrealfilename_miss));
+    dprintf(fd, "%s.nameless_lookup_count %" PRId64 "\n", this->name,
+            GF_ATOMIC_GET(conf->nlc_counter.nameless_lookup));
+    dprintf(fd, "%s.inodes_with_positive_dentry_cache %" PRId64 "\n",
+            this->name, GF_ATOMIC_GET(conf->nlc_counter.pe_inode_cnt));
+    dprintf(fd, "%s.inodes_with_negative_dentry_cache %" PRId64 "\n",
+            this->name, GF_ATOMIC_GET(conf->nlc_counter.ne_inode_cnt));
+    dprintf(fd, "%s.dentry_invalidations_received %" PRId64 "\n", this->name,
+            GF_ATOMIC_GET(conf->nlc_counter.nlc_invals));
+    dprintf(fd, "%s.cache_limit %" PRIu64 "\n", this->name, conf->cache_size);
+    dprintf(fd, "%s.consumed_cache_size %" PRId64 "\n", this->name,
+            GF_ATOMIC_GET(conf->current_cache_size));
+    dprintf(fd, "%s.inode_limit %" PRIu64 "\n", this->name, conf->inode_limit);
+    dprintf(fd, "%s.consumed_inodes %" PRId64 "\n", this->name,
+            GF_ATOMIC_GET(conf->refd_inodes));
+
+    return 0;
+}
+
+void
+nlc_fini(xlator_t *this)
+{
+    nlc_conf_t *conf = NULL;
+
+    conf = this->private;
+    GF_FREE(conf);
+
+    glusterfs_ctx_tw_put(this->ctx);
+
+    return;
+}
+
+int32_t
+nlc_mem_acct_init(xlator_t *this)
+{
+    int ret = -1;
+
+    ret = xlator_mem_acct_init(this, gf_nlc_mt_end + 1);
+    return ret;
+}
+
+int32_t
+nlc_reconfigure(xlator_t *this, dict_t *options)
+{
+    nlc_conf_t *conf = NULL;
+
+    conf = this->private;
+
+    GF_OPTION_RECONF("nl-cache-timeout", conf->cache_timeout, options, int32,
+                     out);
+    GF_OPTION_RECONF("nl-cache-positive-entry", conf->positive_entry_cache,
+                     options, bool, out);
+    GF_OPTION_RECONF("nl-cache-limit", conf->cache_size, options, size_uint64,
+                     out);
+    GF_OPTION_RECONF("pass-through", this->pass_through, options, bool, out);
+
+out:
+    return 0;
+}
+
+int32_t
+nlc_init(xlator_t *this)
+{
+    nlc_conf_t *conf = NULL;
+    int ret = -1;
+    inode_table_t *itable = NULL;
+
+    conf = GF_CALLOC(sizeof(*conf), 1, gf_nlc_mt_nlc_conf_t);
+    if (!conf)
+        goto out;
+
+    GF_OPTION_INIT("nl-cache-timeout", conf->cache_timeout, int32, out);
+    GF_OPTION_INIT("nl-cache-positive-entry", conf->positive_entry_cache, bool,
+                   out);
+    GF_OPTION_INIT("nl-cache-limit", conf->cache_size, size_uint64, out);
+    GF_OPTION_INIT("pass-through", this->pass_through, bool, out);
+
+    /* Since the positive entries are stored as list of refs on
+     * existing inodes, we should not overflow the inode lru_limit.
+     * Hence keep the limit of inodes that are refed by this xlator,
+     * to 80% of inode_table->lru_limit. In fuse where the limit is
+     * infinite, take 131072 as lru limit (as in gfapi). */
+    itable = ((xlator_t *)this->graph->top)->itable;
+    if (itable && itable->lru_limit)
+        conf->inode_limit = itable->lru_limit * 80 / 100;
+    else
+        conf->inode_limit = 131072 * 80 / 100;
+
+    LOCK_INIT(&conf->lock);
+    GF_ATOMIC_INIT(conf->current_cache_size, 0);
+    GF_ATOMIC_INIT(conf->refd_inodes, 0);
+    GF_ATOMIC_INIT(conf->nlc_counter.nlc_hit, 0);
+    GF_ATOMIC_INIT(conf->nlc_counter.nlc_miss, 0);
+    GF_ATOMIC_INIT(conf->nlc_counter.nameless_lookup, 0);
+    GF_ATOMIC_INIT(conf->nlc_counter.getrealfilename_hit, 0);
+    GF_ATOMIC_INIT(conf->nlc_counter.getrealfilename_miss, 0);
+    GF_ATOMIC_INIT(conf->nlc_counter.pe_inode_cnt, 0);
+    GF_ATOMIC_INIT(conf->nlc_counter.ne_inode_cnt, 0);
+    GF_ATOMIC_INIT(conf->nlc_counter.nlc_invals, 0);
+
+    INIT_LIST_HEAD(&conf->lru);
+    conf->last_child_down = gf_time();
+
+    conf->timer_wheel = glusterfs_ctx_tw_get(this->ctx);
+    if (!conf->timer_wheel) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, NLC_MSG_NO_TIMER_WHEEL,
+               "Initing the global timer wheel failed");
+        goto out;
+    }
+
+    this->private = conf;
+
+    ret = 0;
+out:
+    if (ret < 0)
+        GF_FREE(conf);
+
+    return ret;
+}
+
+struct xlator_fops nlc_fops = {
+    .rename = nlc_rename,
+    .mknod = nlc_mknod,
+    .create = nlc_create,
+    .mkdir = nlc_mkdir,
+    .lookup = nlc_lookup,
+    .rmdir = nlc_rmdir,
+    .getxattr = nlc_getxattr,
+    .symlink = nlc_symlink,
+    .link = nlc_link,
+    .unlink = nlc_unlink,
+    /* TODO:
+    .readdir              = nlc_readdir,
+    .readdirp             = nlc_readdirp,
+    .seek                 = nlc_seek,
+    .opendir              = nlc_opendir, */
+};
+
+struct xlator_cbks nlc_cbks = {
+    .forget = nlc_forget,
+};
+
+struct xlator_dumpops nlc_dumpops = {
+    .inodectx = nlc_inodectx,
+    .priv = nlc_priv_dump,
+};
+
+struct volume_options nlc_options[] = {
+    {
+        .key = {"nl-cache"},
+        .type = GF_OPTION_TYPE_BOOL,
+        .default_value = "off",
+        .description = "enable/disable nl-cache",
+        .op_version = {GD_OP_VERSION_6_0},
+        .flags = OPT_FLAG_SETTABLE,
+    },
+    {
+        .key = {"nl-cache-positive-entry"},
+        .type = GF_OPTION_TYPE_BOOL,
+        .default_value = "false",
+        .op_version = {GD_OP_VERSION_3_11_0},
+        .flags = OPT_FLAG_SETTABLE | OPT_FLAG_CLIENT_OPT | OPT_FLAG_DOC,
+        .description = "Cache the name of the files/directories that was"
+                       " looked up and are present in a directory",
+    },
+    {
+        .key = {"nl-cache-limit"},
+        .type = GF_OPTION_TYPE_SIZET,
+        .min = 0,
+        .default_value = "131072",
+        .op_version = {GD_OP_VERSION_3_11_0},
+        .flags = OPT_FLAG_SETTABLE | OPT_FLAG_CLIENT_OPT | OPT_FLAG_DOC,
+        .description = "the value over which caching will be disabled for"
+                       "a while and the cache is cleared based on LRU",
+    },
+    {
+        .key = {"nl-cache-timeout"},
+        .type = GF_OPTION_TYPE_TIME,
+        .min = 0,
+        .default_value = "60",
+        .op_version = {GD_OP_VERSION_3_11_0},
+        .flags = OPT_FLAG_SETTABLE | OPT_FLAG_CLIENT_OPT | OPT_FLAG_DOC,
+        .description = "Time period after which cache has to be refreshed",
+    },
+    {.key = {"pass-through"},
+     .type = GF_OPTION_TYPE_BOOL,
+     .default_value = "false",
+     .op_version = {GD_OP_VERSION_4_1_0},
+     .flags = OPT_FLAG_SETTABLE | OPT_FLAG_DOC | OPT_FLAG_CLIENT_OPT,
+     .tags = {"nl-cache"},
+     .description = "Enable/Disable nl cache translator"},
+
+    {.key = {NULL}},
+};
+
+xlator_api_t xlator_api = {
+    .init = nlc_init,
+    .fini = nlc_fini,
+    .notify = nlc_notify,
+    .reconfigure = nlc_reconfigure,
+    .mem_acct_init = nlc_mem_acct_init,
+    .dump_metrics = nlc_dump_metrics,
+    .op_version = {1}, /* Present from the initial version */
+    .dumpops = &nlc_dumpops,
+    .fops = &nlc_fops,
+    .cbks = &nlc_cbks,
+    .options = nlc_options,
+    .identifier = "nl-cache",
+    .category = GF_TECH_PREVIEW,
+};
diff --git a/xlators/performance/nl-cache/src/nl-cache.h b/xlators/performance/nl-cache/src/nl-cache.h
new file mode 100644
index 00000000000..85fcc176342
--- /dev/null
+++ b/xlators/performance/nl-cache/src/nl-cache.h
@@ -0,0 +1,175 @@
+/*
+ *   Copyright (c) 2016 Red Hat, Inc. <http://www.redhat.com>
+ *   This file is part of GlusterFS.
+ *
+ *   This file is licensed to you under your choice of the GNU Lesser
+ *   General Public License, version 3 or any later version (LGPLv3 or
+ *   later), or the GNU General Public License, version 2 (GPLv2), in all
+ *   cases as published by the Free Software Foundation.
+ */
+
+#ifndef __NL_CACHE_H__
+#define __NL_CACHE_H__
+
+#include "nl-cache-mem-types.h"
+#include "nl-cache-messages.h"
+#include <glusterfs/glusterfs.h>
+#include <glusterfs/xlator.h>
+#include <glusterfs/defaults.h>
+#include <glusterfs/atomic.h>
+
+#define NLC_INVALID 0x0000
+#define NLC_PE_FULL 0x0001
+#define NLC_PE_PARTIAL 0x0002
+#define NLC_NE_VALID 0x0004
+
+#define IS_PE_VALID(state)                                                     \
+    ((state != NLC_INVALID) && (state & (NLC_PE_FULL | NLC_PE_PARTIAL)))
+#define IS_NE_VALID(state) ((state != NLC_INVALID) && (state & NLC_NE_VALID))
+
+#define IS_PEC_ENABLED(conf) (conf->positive_entry_cache)
+#define IS_CACHE_ENABLED(conf) ((!conf->cache_disabled))
+
+#define NLC_STACK_UNWIND(fop, frame, params...)                                \
+    do {                                                                       \
+        nlc_local_t *__local = NULL;                                           \
+        xlator_t *__xl = NULL;                                                 \
+        if (frame) {                                                           \
+            __xl = frame->this;                                                \
+            __local = frame->local;                                            \
+            frame->local = NULL;                                               \
+        }                                                                      \
+        STACK_UNWIND_STRICT(fop, frame, params);                               \
+        nlc_local_wipe(__xl, __local);                                         \
+    } while (0)
+
+enum nlc_cache_clear_reason {
+    NLC_NONE = 0,
+    NLC_LRU_PRUNE,
+};
+
+struct nlc_ne {
+    struct list_head list;
+    char *name;
+};
+typedef struct nlc_ne nlc_ne_t;
+
+struct nlc_pe {
+    struct list_head list;
+    inode_t *inode;
+    char *name;
+};
+typedef struct nlc_pe nlc_pe_t;
+
+struct nlc_timer_data {
+    inode_t *inode;
+    xlator_t *this;
+};
+typedef struct nlc_timer_data nlc_timer_data_t;
+
+struct nlc_lru_node {
+    inode_t *inode;
+    struct list_head list;
+};
+typedef struct nlc_lru_node nlc_lru_node_t;
+
+struct nlc_ctx {
+    struct list_head pe; /* list of positive entries */
+    struct list_head ne; /* list of negative entries */
+    uint64_t state;
+    time_t cache_time;
+    struct gf_tw_timer_list *timer;
+    nlc_timer_data_t *timer_data;
+    size_t cache_size;
+    uint64_t refd_inodes;
+    gf_lock_t lock;
+};
+typedef struct nlc_ctx nlc_ctx_t;
+
+struct nlc_local {
+    loc_t loc;
+    loc_t loc2;
+    inode_t *inode;
+    inode_t *parent;
+    fd_t *fd;
+    char *linkname;
+    glusterfs_fop_t fop;
+};
+typedef struct nlc_local nlc_local_t;
+
+struct nlc_statistics {
+    gf_atomic_t nlc_hit;  /* No. of times lookup/stat was served from this xl */
+    gf_atomic_t nlc_miss; /* No. of times negative lookups were sent to disk */
+    /* More granular counters */
+    gf_atomic_t nameless_lookup;
+    gf_atomic_t getrealfilename_hit;
+    gf_atomic_t getrealfilename_miss;
+    gf_atomic_t pe_inode_cnt;
+    gf_atomic_t ne_inode_cnt;
+    gf_atomic_t nlc_invals; /* No. of invalidates received from upcall*/
+};
+
+struct nlc_conf {
+    int32_t cache_timeout;
+    gf_boolean_t positive_entry_cache;
+    gf_boolean_t negative_entry_cache;
+    gf_boolean_t disable_cache;
+    uint64_t cache_size;
+    gf_atomic_t current_cache_size;
+    uint64_t inode_limit;
+    gf_atomic_t refd_inodes;
+    struct tvec_base *timer_wheel;
+    time_t last_child_down;
+    struct list_head lru;
+    gf_lock_t lock;
+    struct nlc_statistics nlc_counter;
+};
+typedef struct nlc_conf nlc_conf_t;
+
+gf_boolean_t
+nlc_get_real_file_name(xlator_t *this, loc_t *loc, const char *fname,
+                       int32_t *op_ret, int32_t *op_errno, dict_t *dict);
+
+gf_boolean_t
+nlc_is_negative_lookup(xlator_t *this, loc_t *loc);
+
+void
+nlc_set_dir_state(xlator_t *this, inode_t *inode, uint64_t state);
+
+void
+nlc_dir_add_pe(xlator_t *this, inode_t *inode, inode_t *entry_ino,
+               const char *name);
+
+void
+nlc_dir_remove_pe(xlator_t *this, inode_t *inode, inode_t *entry_ino,
+                  const char *name, gf_boolean_t multilink);
+
+void
+nlc_dir_add_ne(xlator_t *this, inode_t *inode, const char *name);
+
+void
+nlc_local_wipe(xlator_t *this, nlc_local_t *local);
+
+nlc_local_t *
+nlc_local_init(call_frame_t *frame, xlator_t *this, glusterfs_fop_t fop,
+               loc_t *loc, loc_t *loc2);
+
+void
+nlc_update_child_down_time(xlator_t *this, time_t now);
+
+void
+nlc_inode_clear_cache(xlator_t *this, inode_t *inode, int reason);
+
+void
+nlc_dump_inodectx(xlator_t *this, inode_t *inode);
+
+void
+nlc_clear_all_cache(xlator_t *this);
+
+void
+nlc_disable_cache(xlator_t *this);
+
+void
+nlc_lru_prune(xlator_t *this, inode_t *inode);
+
+#endif /* __NL_CACHE_H__ */
diff --git a/xlators/performance/open-behind/src/Makefile.am b/xlators/performance/open-behind/src/Makefile.am
index 12528570783..41930dcd67d 100644
--- a/xlators/performance/open-behind/src/Makefile.am
+++ b/xlators/performance/open-behind/src/Makefile.am
@@ -1,14 +1,15 @@
 xlator_LTLIBRARIES = open-behind.la
 xlatordir = $(libdir)/glusterfs/$(PACKAGE_VERSION)/xlator/performance
 
-open_behind_la_LDFLAGS = -module -avoid-version
+open_behind_la_LDFLAGS = -module $(GF_XLATOR_DEFAULT_LDFLAGS)
 
 open_behind_la_SOURCES = open-behind.c
 open_behind_la_LIBADD = $(top_builddir)/libglusterfs/src/libglusterfs.la
 
-noinst_HEADERS = open-behind-mem-types.h
+noinst_HEADERS = open-behind-mem-types.h open-behind-messages.h
 
-AM_CPPFLAGS = $(GF_CPPFLAGS) -I$(top_srcdir)/libglusterfs/src
+AM_CPPFLAGS = $(GF_CPPFLAGS) -I$(top_srcdir)/libglusterfs/src \
+	-I$(top_srcdir)/rpc/xdr/src -I$(top_builddir)/rpc/xdr/src
 
 AM_CFLAGS = -Wall $(GF_CFLAGS)
 
diff --git a/xlators/performance/open-behind/src/open-behind-mem-types.h b/xlators/performance/open-behind/src/open-behind-mem-types.h
index 1e94296f424..6c1ab2e19d2 100644
--- a/xlators/performance/open-behind/src/open-behind-mem-types.h
+++ b/xlators/performance/open-behind/src/open-behind-mem-types.h
@@ -11,11 +11,12 @@
 #ifndef __OB_MEM_TYPES_H__
 #define __OB_MEM_TYPES_H__
 
-#include "mem-types.h"
+#include <glusterfs/mem-types.h>
 
 enum gf_ob_mem_types_ {
-        gf_ob_mt_fd_t   = gf_common_mt_end + 1,
-	gf_ob_mt_conf_t,
-        gf_ob_mt_end
+    gf_ob_mt_fd_t = gf_common_mt_end + 1,
+    gf_ob_mt_conf_t,
+    gf_ob_mt_inode_t,
+    gf_ob_mt_end
 };
 #endif
diff --git a/xlators/performance/open-behind/src/open-behind-messages.h b/xlators/performance/open-behind/src/open-behind-messages.h
new file mode 100644
index 00000000000..0e789177684
--- /dev/null
+++ b/xlators/performance/open-behind/src/open-behind-messages.h
@@ -0,0 +1,32 @@
+/*Copyright (c) 2015 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
+*/
+
+#ifndef _OPEN_BEHIND_MESSAGES_H_
+#define _OPEN_BEHIND_MESSAGES_H_
+
+#include <glusterfs/glfs-message-id.h>
+
+/* To add new message IDs, append new identifiers at the end of the list.
+ *
+ * Never remove a message ID. If it's not used anymore, you can rename it or
+ * leave it as it is, but not delete it. This is to prevent reutilization of
+ * IDs by other messages.
+ *
+ * The component name must match one of the entries defined in
+ * glfs-message-id.h.
+ */
+
+GLFS_MSGID(OPEN_BEHIND, OPEN_BEHIND_MSG_XLATOR_CHILD_MISCONFIGURED,
+           OPEN_BEHIND_MSG_VOL_MISCONFIGURED, OPEN_BEHIND_MSG_NO_MEMORY,
+           OPEN_BEHIND_MSG_FAILED, OPEN_BEHIND_MSG_BAD_STATE);
+
+#define OPEN_BEHIND_MSG_FAILED_STR "Failed to submit fop"
+#define OPEN_BEHIND_MSG_BAD_STATE_STR "Unexpected state"
+
+#endif /* _OPEN_BEHIND_MESSAGES_H_ */
diff --git a/xlators/performance/open-behind/src/open-behind.c b/xlators/performance/open-behind/src/open-behind.c
index 742e4df3fdf..600c3b62ffe 100644
--- a/xlators/performance/open-behind/src/open-behind.c
+++ b/xlators/performance/open-behind/src/open-behind.c
@@ -9,1012 +9,1093 @@
 */
 
 #include "open-behind-mem-types.h"
-#include "xlator.h"
-#include "statedump.h"
-#include "call-stub.h"
-#include "defaults.h"
+#include <glusterfs/xlator.h>
+#include <glusterfs/statedump.h>
+#include <glusterfs/call-stub.h>
+#include <glusterfs/defaults.h>
+#include "open-behind-messages.h"
+#include <glusterfs/glusterfs-acl.h>
+
+/* Note: The initial design of open-behind was made to cover the simple case
+ *       of open, read, close for small files. This pattern combined with
+ *       quick-read can do the whole operation without a single request to the
+ *       bricks (except the initial lookup).
+ *
+ *       The way to do this has been improved, but the logic remains the same.
+ *       Basically, this means that any operation sent to the fd or the inode
+ *       that it's not a read, causes the open request to be sent to the
+ *       bricks, and all future operations will be executed synchronously,
+ *       including opens (it's reset once all fd's are closed).
+ */
 
 typedef struct ob_conf {
-	gf_boolean_t  use_anonymous_fd; /* use anonymous FDs wherever safe
-					   e.g - fstat() readv()
-
-					   whereas for fops like writev(), lk(),
-					   the fd is important for side effects
-					   like mandatory locks
-					*/
-	gf_boolean_t  lazy_open; /* delay backend open as much as possible */
-        gf_boolean_t  read_after_open; /* instead of sending readvs on
-                                               anonymous fds, open the file
-                                               first and then send readv i.e
-                                               similar to what writev does
-                                            */
+    gf_boolean_t use_anonymous_fd; /* use anonymous FDs wherever safe
+                                      e.g - fstat() readv()
+
+                                      whereas for fops like writev(), lk(),
+                                      the fd is important for side effects
+                                      like mandatory locks
+                                   */
+    gf_boolean_t lazy_open;        /* delay backend open as much as possible */
+    gf_boolean_t read_after_open;  /* instead of sending readvs on
+                                           anonymous fds, open the file
+                                           first and then send readv i.e
+                                           similar to what writev does
+                                        */
 } ob_conf_t;
 
-
-typedef struct ob_fd {
-	call_frame_t     *open_frame;
-	loc_t             loc;
-	dict_t           *xdata;
-	int               flags;
-	int               op_errno;
-	struct list_head  list;
-} ob_fd_t;
-
-
-ob_fd_t *
-__ob_fd_ctx_get (xlator_t *this, fd_t *fd)
+/* A negative state represents an errno value negated. In this case the
+ * current operation cannot be processed. */
+typedef enum _ob_state {
+    /* There are no opens on the inode or the first open is already
+     * completed. The current operation can be sent directly. */
+    OB_STATE_READY = 0,
+
+    /* There's an open pending and it has been triggered. The current
+     * operation should be "stubbified" and processed with
+     * ob_stub_dispatch(). */
+    OB_STATE_OPEN_TRIGGERED,
+
+    /* There's an open pending but it has not been triggered. The current
+     * operation can be processed directly but using an anonymous fd. */
+    OB_STATE_OPEN_PENDING,
+
+    /* The current operation is the first open on the inode. */
+    OB_STATE_FIRST_OPEN
+} ob_state_t;
+
+typedef struct ob_inode {
+    /* List of stubs pending on the first open. Once the first open is
+     * complete, all these stubs will be resubmitted, and dependencies
+     * will be checked again. */
+    struct list_head resume_fops;
+
+    /* The inode this object references. */
+    inode_t *inode;
+
+    /* The fd from the first open sent to this inode. It will be set
+     * from the moment the open is processed until the open if fully
+     * executed or closed before actually opened. It's NULL in all
+     * other cases. */
+    fd_t *first_fd;
+
+    /* The stub from the first open operation. When open fop starts
+     * being processed, it's assigned the OB_OPEN_PREPARING value
+     * until the actual stub is created. This is necessary to avoid
+     * creating the stub inside a locked region. Once the stub is
+     * successfully created, it's assigned here. This value is set
+     * to NULL once the stub is resumed. */
+    call_stub_t *first_open;
+
+    /* The total number of currently open fd's on this inode. */
+    int32_t open_count;
+
+    /* This flag is set as soon as we know that the open will be
+     * sent to the bricks, even before the stub is ready. */
+    bool triggered;
+} ob_inode_t;
+
+/* Dummy pointer used temporarily while the actual open stub is being created */
+#define OB_OPEN_PREPARING ((call_stub_t *)-1)
+
+#define OB_POST_COMMON(_fop, _xl, _frame, _fd, _args...)                       \
+    case OB_STATE_FIRST_OPEN:                                                  \
+        gf_smsg((_xl)->name, GF_LOG_ERROR, EINVAL, OPEN_BEHIND_MSG_BAD_STATE,  \
+                "fop=%s", #_fop, "state=%d", __ob_state, NULL);                \
+        default_##_fop##_failure_cbk(_frame, EINVAL);                          \
+        break;                                                                 \
+    case OB_STATE_READY:                                                       \
+        default_##_fop(_frame, _xl, ##_args);                                  \
+        break;                                                                 \
+    case OB_STATE_OPEN_TRIGGERED: {                                            \
+        call_stub_t *__ob_stub = fop_##_fop##_stub(_frame, ob_##_fop,          \
+                                                   ##_args);                   \
+        if (__ob_stub != NULL) {                                               \
+            ob_stub_dispatch(_xl, __ob_inode, _fd, __ob_stub);                 \
+            break;                                                             \
+        }                                                                      \
+        __ob_state = -ENOMEM;                                                  \
+    }                                                                          \
+    default:                                                                   \
+        gf_smsg((_xl)->name, GF_LOG_ERROR, -__ob_state,                        \
+                OPEN_BEHIND_MSG_FAILED, "fop=%s", #_fop, NULL);                \
+        default_##_fop##_failure_cbk(_frame, -__ob_state)
+
+#define OB_POST_FD(_fop, _xl, _frame, _fd, _trigger, _args...)                 \
+    do {                                                                       \
+        ob_inode_t *__ob_inode;                                                \
+        fd_t *__first_fd;                                                      \
+        ob_state_t __ob_state = ob_open_and_resume_fd(                         \
+            _xl, _fd, 0, true, _trigger, &__ob_inode, &__first_fd);            \
+        switch (__ob_state) {                                                  \
+            case OB_STATE_OPEN_PENDING:                                        \
+                if (!(_trigger)) {                                             \
+                    fd_t *__ob_fd = fd_anonymous_with_flags((_fd)->inode,      \
+                                                            (_fd)->flags);     \
+                    if (__ob_fd != NULL) {                                     \
+                        default_##_fop(_frame, _xl, ##_args);                  \
+                        fd_unref(__ob_fd);                                     \
+                        break;                                                 \
+                    }                                                          \
+                    __ob_state = -ENOMEM;                                      \
+                }                                                              \
+                OB_POST_COMMON(_fop, _xl, _frame, __first_fd, ##_args);        \
+        }                                                                      \
+    } while (0)
+
+#define OB_POST_FLUSH(_xl, _frame, _fd, _args...)                              \
+    do {                                                                       \
+        ob_inode_t *__ob_inode;                                                \
+        fd_t *__first_fd;                                                      \
+        ob_state_t __ob_state = ob_open_and_resume_fd(                         \
+            _xl, _fd, 0, true, false, &__ob_inode, &__first_fd);               \
+        switch (__ob_state) {                                                  \
+            case OB_STATE_OPEN_PENDING:                                        \
+                default_flush_cbk(_frame, NULL, _xl, 0, 0, NULL);              \
+                break;                                                         \
+                OB_POST_COMMON(flush, _xl, _frame, __first_fd, ##_args);       \
+        }                                                                      \
+    } while (0)
+
+#define OB_POST_INODE(_fop, _xl, _frame, _inode, _trigger, _args...)           \
+    do {                                                                       \
+        ob_inode_t *__ob_inode;                                                \
+        fd_t *__first_fd;                                                      \
+        ob_state_t __ob_state = ob_open_and_resume_inode(                      \
+            _xl, _inode, NULL, 0, true, _trigger, &__ob_inode, &__first_fd);   \
+        switch (__ob_state) {                                                  \
+            case OB_STATE_OPEN_PENDING:                                        \
+                OB_POST_COMMON(_fop, _xl, _frame, __first_fd, ##_args);        \
+        }                                                                      \
+    } while (0)
+
+static ob_inode_t *
+ob_inode_get_locked(xlator_t *this, inode_t *inode)
 {
-	uint64_t    value = 0;
-	int         ret = -1;
-	ob_fd_t    *ob_fd = NULL;
-
-	ret = __fd_ctx_get (fd, this, &value);
-	if (ret)
-		return NULL;
-
-	ob_fd = (void *) ((long) value);
+    ob_inode_t *ob_inode = NULL;
+    uint64_t value = 0;
+
+    if ((__inode_ctx_get(inode, this, &value) == 0) && (value != 0)) {
+        return (ob_inode_t *)(uintptr_t)value;
+    }
+
+    ob_inode = GF_CALLOC(1, sizeof(*ob_inode), gf_ob_mt_inode_t);
+    if (ob_inode != NULL) {
+        ob_inode->inode = inode;
+        INIT_LIST_HEAD(&ob_inode->resume_fops);
+
+        value = (uint64_t)(uintptr_t)ob_inode;
+        if (__inode_ctx_set(inode, this, &value) < 0) {
+            GF_FREE(ob_inode);
+            ob_inode = NULL;
+        }
+    }
 
-	return ob_fd;
+    return ob_inode;
 }
 
-
-ob_fd_t *
-ob_fd_ctx_get (xlator_t *this, fd_t *fd)
+static ob_state_t
+ob_open_and_resume_inode(xlator_t *xl, inode_t *inode, fd_t *fd,
+                         int32_t open_count, bool synchronous, bool trigger,
+                         ob_inode_t **pob_inode, fd_t **pfd)
 {
-	ob_fd_t  *ob_fd = NULL;
+    ob_conf_t *conf;
+    ob_inode_t *ob_inode;
+    call_stub_t *open_stub;
 
-	LOCK (&fd->lock);
-	{
-		ob_fd = __ob_fd_ctx_get (this, fd);
-	}
-	UNLOCK (&fd->lock);
+    if (inode == NULL) {
+        return OB_STATE_READY;
+    }
 
-	return ob_fd;
-}
+    conf = xl->private;
 
+    *pfd = NULL;
 
-int
-__ob_fd_ctx_set (xlator_t *this, fd_t *fd, ob_fd_t *ob_fd)
-{
-	uint64_t    value = 0;
-	int         ret = -1;
+    LOCK(&inode->lock);
+    {
+        ob_inode = ob_inode_get_locked(xl, inode);
+        if (ob_inode == NULL) {
+            UNLOCK(&inode->lock);
 
-	value = (long) ((void *) ob_fd);
+            return -ENOMEM;
+        }
+        *pob_inode = ob_inode;
+
+        ob_inode->open_count += open_count;
+
+        /* If first_fd is not NULL, it means that there's a previous open not
+         * yet completed. */
+        if (ob_inode->first_fd != NULL) {
+            *pfd = ob_inode->first_fd;
+            /* If the current request doesn't trigger the open and it hasn't
+             * been triggered yet, we can continue without issuing the open
+             * only if the current request belongs to the same fd as the
+             * first one. */
+            if (!trigger && !ob_inode->triggered &&
+                (ob_inode->first_fd == fd)) {
+                UNLOCK(&inode->lock);
+
+                return OB_STATE_OPEN_PENDING;
+            }
+
+            /* We need to issue the open. It could have already been triggered
+             * before. In this case open_stub will be NULL. Or the initial open
+             * may not be completely ready yet. In this case open_stub will be
+             * OB_OPEN_PREPARING. */
+            open_stub = ob_inode->first_open;
+            ob_inode->first_open = NULL;
+            ob_inode->triggered = true;
+
+            UNLOCK(&inode->lock);
+
+            if ((open_stub != NULL) && (open_stub != OB_OPEN_PREPARING)) {
+                call_resume(open_stub);
+            }
+
+            return OB_STATE_OPEN_TRIGGERED;
+        }
 
-	ret = __fd_ctx_set (fd, this, value);
+        /* There's no pending open. Only opens can be non synchronous, so all
+         * regular fops will be processed directly. For non synchronous opens,
+         * we'll still process them normally (i.e. synchornous) if there are
+         * more file descriptors open. */
+        if (synchronous || (ob_inode->open_count > open_count)) {
+            UNLOCK(&inode->lock);
 
-	return ret;
-}
+            return OB_STATE_READY;
+        }
 
+        *pfd = fd;
 
-int
-ob_fd_ctx_set (xlator_t *this, fd_t *fd, ob_fd_t *ob_fd)
-{
-	int ret = -1;
+        /* This is the first open. We keep a reference on the fd and set
+         * first_open stub to OB_OPEN_PREPARING until the actual stub can
+         * be assigned (we don't create the stub here to avoid doing memory
+         * allocations inside the mutex). */
+        ob_inode->first_fd = __fd_ref(fd);
+        ob_inode->first_open = OB_OPEN_PREPARING;
 
-	LOCK (&fd->lock);
-	{
-		ret = __ob_fd_ctx_set (this, fd, ob_fd);
-	}
-	UNLOCK (&fd->lock);
+        /* If lazy_open is not set, we'll need to immediately send the open,
+         * so we set triggered right now. */
+        ob_inode->triggered = !conf->lazy_open;
+    }
+    UNLOCK(&inode->lock);
 
-	return ret;
+    return OB_STATE_FIRST_OPEN;
 }
 
-
-ob_fd_t *
-ob_fd_new (void)
+static ob_state_t
+ob_open_and_resume_fd(xlator_t *xl, fd_t *fd, int32_t open_count,
+                      bool synchronous, bool trigger, ob_inode_t **pob_inode,
+                      fd_t **pfd)
 {
-	ob_fd_t  *ob_fd = NULL;
+    uint64_t err;
 
-	ob_fd = GF_CALLOC (1, sizeof (*ob_fd), gf_ob_mt_fd_t);
+    if ((fd_ctx_get(fd, xl, &err) == 0) && (err != 0)) {
+        return (ob_state_t)-err;
+    }
 
-	INIT_LIST_HEAD (&ob_fd->list);
-
-	return ob_fd;
+    return ob_open_and_resume_inode(xl, fd->inode, fd, open_count, synchronous,
+                                    trigger, pob_inode, pfd);
 }
 
-
-void
-ob_fd_free (ob_fd_t *ob_fd)
+static ob_state_t
+ob_open_behind(xlator_t *xl, fd_t *fd, int32_t flags, ob_inode_t **pob_inode,
+               fd_t **pfd)
 {
-	loc_wipe (&ob_fd->loc);
+    bool synchronous;
 
-	if (ob_fd->xdata)
-		dict_unref (ob_fd->xdata);
+    /* TODO: If O_CREAT, O_APPEND, O_WRONLY or O_DIRECT are specified, shouldn't
+     *       we also execute this open synchronously ? */
+    synchronous = (flags & O_TRUNC) != 0;
 
-	if (ob_fd->open_frame)
-		STACK_DESTROY (ob_fd->open_frame->root);
-
-	GF_FREE (ob_fd);
+    return ob_open_and_resume_fd(xl, fd, 1, synchronous, true, pob_inode, pfd);
 }
 
-
-int
-ob_wake_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-	     int op_ret, int op_errno, fd_t *fd_ret, dict_t *xdata)
+static int32_t
+ob_stub_dispatch(xlator_t *xl, ob_inode_t *ob_inode, fd_t *fd,
+                 call_stub_t *stub)
 {
-	fd_t              *fd = NULL;
-	struct list_head   list;
-	ob_fd_t           *ob_fd = NULL;
-	call_stub_t       *stub = NULL, *tmp = NULL;
-
-	fd = frame->local;
-	frame->local = NULL;
-
-	INIT_LIST_HEAD (&list);
-
-	LOCK (&fd->lock);
-	{
-		ob_fd = __ob_fd_ctx_get (this, fd);
-
-		list_splice_init (&ob_fd->list, &list);
-
-		if (op_ret < 0) {
-			/* mark fd BAD for ever */
-			ob_fd->op_errno = op_errno;
-		} else {
-			__fd_ctx_del (fd, this, NULL);
-			ob_fd_free (ob_fd);
-		}
-	}
-	UNLOCK (&fd->lock);
-
-	list_for_each_entry_safe (stub, tmp, &list, list) {
-		list_del_init (&stub->list);
-
-		if (op_ret < 0)
-			call_unwind_error (stub, -1, op_errno);
-		else
-			call_resume (stub);
-	}
-
-	fd_unref (fd);
+    LOCK(&ob_inode->inode->lock);
+    {
+        /* We only queue a stub if the open has not been completed or
+         * cancelled. */
+        if (ob_inode->first_fd == fd) {
+            list_add_tail(&stub->list, &ob_inode->resume_fops);
+            stub = NULL;
+        }
+    }
+    UNLOCK(&ob_inode->inode->lock);
 
-	STACK_DESTROY (frame->root);
+    if (stub != NULL) {
+        call_resume(stub);
+    }
 
-	return 0;
+    return 0;
 }
 
-
-int
-ob_fd_wake (xlator_t *this, fd_t *fd)
+static void
+ob_open_destroy(call_stub_t *stub, fd_t *fd)
 {
-	call_frame_t *frame = NULL;
-	ob_fd_t      *ob_fd = NULL;
-
-	LOCK (&fd->lock);
-	{
-		ob_fd = __ob_fd_ctx_get (this, fd);
-		if (!ob_fd)
-			goto unlock;
-
-		frame = ob_fd->open_frame;
-		ob_fd->open_frame = NULL;
-	}
-unlock:
-	UNLOCK (&fd->lock);
-
-	if (frame) {
-		frame->local = fd_ref (fd);
-
-		STACK_WIND (frame, ob_wake_cbk, FIRST_CHILD (this),
-			    FIRST_CHILD (this)->fops->open,
-			    &ob_fd->loc, ob_fd->flags, fd, ob_fd->xdata);
-	}
-
-	return 0;
+    stub->frame->local = NULL;
+    STACK_DESTROY(stub->frame->root);
+    call_stub_destroy(stub);
+    fd_unref(fd);
 }
 
-
-int
-open_and_resume (xlator_t *this, fd_t *fd, call_stub_t *stub)
+static int32_t
+ob_open_dispatch(xlator_t *xl, ob_inode_t *ob_inode, fd_t *fd,
+                 call_stub_t *stub)
 {
-	ob_fd_t  *ob_fd = NULL;
-	int       op_errno = 0;
-
-	if (!fd)
-		goto nofd;
-
-	LOCK (&fd->lock);
-	{
-		ob_fd = __ob_fd_ctx_get (this, fd);
-		if (!ob_fd)
-			goto unlock;
-
-		if (ob_fd->op_errno) {
-			op_errno = ob_fd->op_errno;
-			goto unlock;
-		}
-
-		list_add_tail (&stub->list, &ob_fd->list);
-	}
-unlock:
-	UNLOCK (&fd->lock);
-
-nofd:
-	if (op_errno)
-		call_unwind_error (stub, -1, op_errno);
-	else if (ob_fd)
-		ob_fd_wake (this, fd);
-	else
-		call_resume (stub);
+    bool closed;
+
+    LOCK(&ob_inode->inode->lock);
+    {
+        closed = ob_inode->first_fd != fd;
+        if (!closed) {
+            if (ob_inode->triggered) {
+                ob_inode->first_open = NULL;
+            } else {
+                ob_inode->first_open = stub;
+                stub = NULL;
+            }
+        }
+    }
+    UNLOCK(&ob_inode->inode->lock);
+
+    if (stub != NULL) {
+        if (closed) {
+            ob_open_destroy(stub, fd);
+        } else {
+            call_resume(stub);
+        }
+    }
 
-	return 0;
+    return 0;
 }
 
-
-int
-ob_open_behind (call_frame_t *frame, xlator_t *this, loc_t *loc, int flags,
-		fd_t *fd, dict_t *xdata)
+static void
+ob_resume_pending(struct list_head *list)
 {
-	ob_fd_t    *ob_fd = NULL;
-	int         ret = -1;
-	ob_conf_t  *conf = NULL;
-
+    call_stub_t *stub;
 
-	conf = this->private;
+    while (!list_empty(list)) {
+        stub = list_first_entry(list, call_stub_t, list);
+        list_del_init(&stub->list);
 
-	if (flags & O_TRUNC) {
-		STACK_WIND (frame, default_open_cbk,
-			    FIRST_CHILD (this), FIRST_CHILD (this)->fops->open,
-			    loc, flags, fd, xdata);
-		return 0;
-	}
-
-	ob_fd = ob_fd_new ();
-	if (!ob_fd)
-		goto enomem;
-
-	ob_fd->open_frame = copy_frame (frame);
-	if (!ob_fd->open_frame)
-		goto enomem;
-	ret = loc_copy (&ob_fd->loc, loc);
-	if (ret)
-		goto enomem;
-
-	ob_fd->flags = flags;
-	if (xdata)
-		ob_fd->xdata = dict_ref (xdata);
-
-	ret = ob_fd_ctx_set (this, fd, ob_fd);
-	if (ret)
-		goto enomem;
-
-	fd_ref (fd);
-
-	STACK_UNWIND_STRICT (open, frame, 0, 0, fd, xdata);
-
-	if (!conf->lazy_open)
-		ob_fd_wake (this, fd);
-
-	fd_unref (fd);
-
-	return 0;
-enomem:
-	if (ob_fd) {
-		if (ob_fd->open_frame)
-			STACK_DESTROY (ob_fd->open_frame->root);
-		loc_wipe (&ob_fd->loc);
-		if (ob_fd->xdata)
-			dict_unref (ob_fd->xdata);
-		GF_FREE (ob_fd);
-	}
-
-	return -1;
-}
-
-
-int
-ob_open (call_frame_t *frame, xlator_t *this, loc_t *loc, int flags,
-	 fd_t *fd, dict_t *xdata)
-{
-	fd_t         *old_fd = NULL;
-	int           ret = -1;
-	int           op_errno = 0;
-	call_stub_t  *stub = NULL;
-
-	old_fd = fd_lookup (fd->inode, 0);
-	if (old_fd) {
-		/* open-behind only when this is the first FD */
-		stub = fop_open_stub (frame, default_open_resume,
-				      loc, flags, fd, xdata);
-		if (!stub) {
-			op_errno = ENOMEM;
-			fd_unref (old_fd);
-			goto err;
-		}
-
-		open_and_resume (this, old_fd, stub);
-
-		fd_unref (old_fd);
-
-		return 0;
-	}
-
-	ret = ob_open_behind (frame, this, loc, flags, fd, xdata);
-	if (ret) {
-		op_errno = ENOMEM;
-		goto err;
-	}
-
-	return 0;
-err:
-	gf_log (this->name, GF_LOG_ERROR, "%s: %s", loc->path,
-		strerror (op_errno));
-
-	STACK_UNWIND_STRICT (open, frame, -1, op_errno, 0, 0);
-
-	return 0;
+        call_resume(stub);
+    }
 }
 
-
-fd_t *
-ob_get_wind_fd (xlator_t *this, fd_t *fd)
+static void
+ob_open_completed(xlator_t *xl, ob_inode_t *ob_inode, fd_t *fd, int32_t op_ret,
+                  int32_t op_errno)
 {
-	ob_conf_t  *conf = NULL;
-	ob_fd_t    *ob_fd = NULL;
-
-	conf = this->private;
-
-	ob_fd = ob_fd_ctx_get (this, fd);
+    struct list_head list;
+
+    INIT_LIST_HEAD(&list);
+
+    if (op_ret < 0) {
+        fd_ctx_set(fd, xl, op_errno <= 0 ? EIO : op_errno);
+    }
+
+    LOCK(&ob_inode->inode->lock);
+    {
+        /* Only update the fields if the file has not been closed before
+         * getting here. */
+        if (ob_inode->first_fd == fd) {
+            list_splice_init(&ob_inode->resume_fops, &list);
+            ob_inode->first_fd = NULL;
+            ob_inode->first_open = NULL;
+            ob_inode->triggered = false;
+        }
+    }
+    UNLOCK(&ob_inode->inode->lock);
 
-	if (ob_fd && conf->use_anonymous_fd)
-		return fd_anonymous (fd->inode);
+    ob_resume_pending(&list);
 
-	return fd_ref (fd);
+    fd_unref(fd);
 }
 
-
-int
-ob_readv (call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
-	  off_t offset, uint32_t flags, dict_t *xdata)
+static int32_t
+ob_open_cbk(call_frame_t *frame, void *cookie, xlator_t *xl, int32_t op_ret,
+            int32_t op_errno, fd_t *fd, dict_t *xdata)
 {
-	call_stub_t  *stub = NULL;
-	fd_t         *wind_fd = NULL;
-        ob_conf_t    *conf = NULL;
+    ob_inode_t *ob_inode;
 
-        conf = this->private;
+    ob_inode = frame->local;
+    frame->local = NULL;
 
-        if (!conf->read_after_open)
-                wind_fd = ob_get_wind_fd (this, fd);
-        else
-                wind_fd = fd_ref (fd);
+    ob_open_completed(xl, ob_inode, cookie, op_ret, op_errno);
 
-	stub = fop_readv_stub (frame, default_readv_resume, wind_fd,
-			       size, offset, flags, xdata);
-	fd_unref (wind_fd);
+    STACK_DESTROY(frame->root);
 
-	if (!stub)
-		goto err;
-
-	open_and_resume (this, wind_fd, stub);
-
-	return 0;
-err:
-	STACK_UNWIND_STRICT (readv, frame, -1, ENOMEM, 0, 0, 0, 0, 0);
-
-	return 0;
+    return 0;
 }
 
-
-int
-ob_writev (call_frame_t *frame, xlator_t *this, fd_t *fd, struct iovec *iov,
-	   int count, off_t offset, uint32_t flags, struct iobref *iobref,
-	   dict_t *xdata)
+static int32_t
+ob_open_resume(call_frame_t *frame, xlator_t *this, loc_t *loc, int flags,
+               fd_t *fd, dict_t *xdata)
 {
-	call_stub_t  *stub = NULL;
-
-	stub = fop_writev_stub (frame, default_writev_resume, fd, iov, count,
-				offset, flags, iobref, xdata);
-	if (!stub)
-		goto err;
-
-	open_and_resume (this, fd, stub);
-
-	return 0;
-err:
-	STACK_UNWIND_STRICT (writev, frame, -1, ENOMEM, 0, 0, 0);
+    STACK_WIND_COOKIE(frame, ob_open_cbk, fd, FIRST_CHILD(this),
+                      FIRST_CHILD(this)->fops->open, loc, flags, fd, xdata);
 
-	return 0;
+    return 0;
 }
 
-
-int
-ob_fstat (call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *xdata)
+static int32_t
+ob_open(call_frame_t *frame, xlator_t *this, loc_t *loc, int flags, fd_t *fd,
+        dict_t *xdata)
 {
-	call_stub_t  *stub = NULL;
-	fd_t         *wind_fd = NULL;
+    ob_inode_t *ob_inode;
+    call_frame_t *open_frame;
+    call_stub_t *stub;
+    fd_t *first_fd;
+    ob_state_t state;
+
+    state = ob_open_behind(this, fd, flags, &ob_inode, &first_fd);
+    if (state == OB_STATE_READY) {
+        /* There's no pending open, but there are other file descriptors opened
+         * or the current flags require a synchronous open. */
+        return default_open(frame, this, loc, flags, fd, xdata);
+    }
+
+    if (state == OB_STATE_OPEN_TRIGGERED) {
+        /* The first open is in progress (either because it was already issued
+         * or because this request triggered it). We try to create a new stub
+         * to retry the operation once the initial open completes. */
+        stub = fop_open_stub(frame, ob_open, loc, flags, fd, xdata);
+        if (stub != NULL) {
+            return ob_stub_dispatch(this, ob_inode, first_fd, stub);
+        }
 
-	wind_fd = ob_get_wind_fd (this, fd);
+        state = -ENOMEM;
+    }
+
+    if (state == OB_STATE_FIRST_OPEN) {
+        /* We try to create a stub for the new open. A new frame needs to be
+         * used because the current one may be destroyed soon after sending
+         * the open's reply. */
+        open_frame = copy_frame(frame);
+        if (open_frame != NULL) {
+            stub = fop_open_stub(open_frame, ob_open_resume, loc, flags, fd,
+                                 xdata);
+            if (stub != NULL) {
+                open_frame->local = ob_inode;
+
+                /* TODO: Previous version passed xdata back to the caller, but
+                 *       probably this doesn't make sense since it won't contain
+                 *       any requested data. I think it would be better to pass
+                 *       NULL for xdata. */
+                default_open_cbk(frame, NULL, this, 0, 0, fd, xdata);
+
+                return ob_open_dispatch(this, ob_inode, first_fd, stub);
+            }
+
+            STACK_DESTROY(open_frame->root);
+        }
 
-	stub = fop_fstat_stub (frame, default_fstat_resume, wind_fd, xdata);
+        /* In case of error, simulate a regular completion but with an error
+         * code. */
+        ob_open_completed(this, ob_inode, first_fd, -1, ENOMEM);
 
-	fd_unref (wind_fd);
+        state = -ENOMEM;
+    }
 
-	if (!stub)
-		goto err;
+    /* In case of failure we need to decrement the number of open files because
+     * ob_fdclose() won't be called. */
 
-	open_and_resume (this, wind_fd, stub);
+    LOCK(&fd->inode->lock);
+    {
+        ob_inode->open_count--;
+    }
+    UNLOCK(&fd->inode->lock);
 
-	return 0;
-err:
-	STACK_UNWIND_STRICT (fstat, frame, -1, ENOMEM, 0, 0);
+    gf_smsg(this->name, GF_LOG_ERROR, -state, OPEN_BEHIND_MSG_FAILED, "fop=%s",
+            "open", "path=%s", loc->path, NULL);
 
-	return 0;
+    return default_open_failure_cbk(frame, -state);
 }
 
-
-int
-ob_flush (call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *xdata)
+static int32_t
+ob_create(call_frame_t *frame, xlator_t *this, loc_t *loc, int flags,
+          mode_t mode, mode_t umask, fd_t *fd, dict_t *xdata)
 {
-	call_stub_t   *stub = NULL;
-	ob_fd_t       *ob_fd = NULL;
-	gf_boolean_t   unwind = _gf_false;
-
-	LOCK (&fd->lock);
-	{
-		ob_fd = __ob_fd_ctx_get (this, fd);
-		if (ob_fd && ob_fd->open_frame)
-			/* if open() was never wound to backend,
-			   no need to wind flush() either.
-			*/
-			unwind = _gf_true;
-	}
-	UNLOCK (&fd->lock);
-
-	if (unwind)
-		goto unwind;
+    ob_inode_t *ob_inode;
+    call_stub_t *stub;
+    fd_t *first_fd;
+    ob_state_t state;
+
+    /* Create requests are never delayed. We always send them synchronously. */
+    state = ob_open_and_resume_fd(this, fd, 1, true, true, &ob_inode,
+                                  &first_fd);
+    if (state == OB_STATE_READY) {
+        /* There's no pending open, but there are other file descriptors opened
+         * so we simply forward the request synchronously. */
+        return default_create(frame, this, loc, flags, mode, umask, fd, xdata);
+    }
+
+    if (state == OB_STATE_OPEN_TRIGGERED) {
+        /* The first open is in progress (either because it was already issued
+         * or because this request triggered it). We try to create a new stub
+         * to retry the operation once the initial open completes. */
+        stub = fop_create_stub(frame, ob_create, loc, flags, mode, umask, fd,
+                               xdata);
+        if (stub != NULL) {
+            return ob_stub_dispatch(this, ob_inode, first_fd, stub);
+        }
 
-	stub = fop_flush_stub (frame, default_flush_resume, fd, xdata);
-	if (!stub)
-		goto err;
+        state = -ENOMEM;
+    }
 
-	open_and_resume (this, fd, stub);
+    /* Since we forced a synchronous request, OB_STATE_FIRST_OPEN will never
+     * be returned by ob_open_and_resume_fd(). If we are here it can only be
+     * because there has been a problem. */
 
-	return 0;
-err:
-	STACK_UNWIND_STRICT (flush, frame, -1, ENOMEM, 0);
+    /* In case of failure we need to decrement the number of open files because
+     * ob_fdclose() won't be called. */
 
-	return 0;
+    LOCK(&fd->inode->lock);
+    {
+        ob_inode->open_count--;
+    }
+    UNLOCK(&fd->inode->lock);
 
-unwind:
-	STACK_UNWIND_STRICT (flush, frame, 0, 0, 0);
+    gf_smsg(this->name, GF_LOG_ERROR, -state, OPEN_BEHIND_MSG_FAILED, "fop=%s",
+            "create", "path=%s", loc->path, NULL);
 
-	return 0;
+    return default_create_failure_cbk(frame, -state);
 }
 
-
-int
-ob_fsync (call_frame_t *frame, xlator_t *this, fd_t *fd, int flag,
-	  dict_t *xdata)
+static int32_t
+ob_readv(call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
+         off_t offset, uint32_t flags, dict_t *xdata)
 {
-	call_stub_t  *stub = NULL;
-
-	stub = fop_fsync_stub (frame, default_fsync_resume, fd, flag, xdata);
-	if (!stub)
-		goto err;
-
-	open_and_resume (this, fd, stub);
+    ob_conf_t *conf = this->private;
+    bool trigger = conf->read_after_open || !conf->use_anonymous_fd;
 
-	return 0;
-err:
-	STACK_UNWIND_STRICT (fsync, frame, -1, ENOMEM, 0, 0, 0);
+    OB_POST_FD(readv, this, frame, fd, trigger, fd, size, offset, flags, xdata);
 
-	return 0;
+    return 0;
 }
 
-
-int
-ob_lk (call_frame_t *frame, xlator_t *this, fd_t *fd, int cmd,
-       struct gf_flock *flock, dict_t *xdata)
+static int32_t
+ob_writev(call_frame_t *frame, xlator_t *this, fd_t *fd, struct iovec *iov,
+          int count, off_t offset, uint32_t flags, struct iobref *iobref,
+          dict_t *xdata)
 {
-	call_stub_t  *stub = NULL;
+    OB_POST_FD(writev, this, frame, fd, true, fd, iov, count, offset, flags,
+               iobref, xdata);
 
-	stub = fop_lk_stub (frame, default_lk_resume, fd, cmd, flock, xdata);
-	if (!stub)
-		goto err;
-
-	open_and_resume (this, fd, stub);
-
-	return 0;
-err:
-	STACK_UNWIND_STRICT (lk, frame, -1, ENOMEM, 0, 0);
-
-	return 0;
+    return 0;
 }
 
-int
-ob_ftruncate (call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
-	      dict_t *xdata)
+static int32_t
+ob_fstat(call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *xdata)
 {
-	call_stub_t  *stub = NULL;
-
-	stub = fop_ftruncate_stub (frame, default_ftruncate_resume, fd, offset,
-				   xdata);
-	if (!stub)
-		goto err;
+    ob_conf_t *conf = this->private;
+    bool trigger = !conf->use_anonymous_fd;
 
-	open_and_resume (this, fd, stub);
+    OB_POST_FD(fstat, this, frame, fd, trigger, fd, xdata);
 
-	return 0;
-err:
-	STACK_UNWIND_STRICT (ftruncate, frame, -1, ENOMEM, 0, 0, 0);
-
-	return 0;
+    return 0;
 }
 
-
-int
-ob_fsetxattr (call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *xattr,
-	      int flags, dict_t *xdata)
+static int32_t
+ob_seek(call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
+        gf_seek_what_t what, dict_t *xdata)
 {
-	call_stub_t  *stub = NULL;
+    ob_conf_t *conf = this->private;
+    bool trigger = !conf->use_anonymous_fd;
 
-	stub = fop_fsetxattr_stub (frame, default_fsetxattr_resume, fd, xattr,
-				   flags, xdata);
-	if (!stub)
-		goto err;
-
-	open_and_resume (this, fd, stub);
-
-	return 0;
-err:
-	STACK_UNWIND_STRICT (fsetxattr, frame, -1, ENOMEM, 0);
+    OB_POST_FD(seek, this, frame, fd, trigger, fd, offset, what, xdata);
 
-	return 0;
+    return 0;
 }
 
-
-int
-ob_fgetxattr (call_frame_t *frame, xlator_t *this, fd_t *fd, const char *name,
-	      dict_t *xdata)
+static int32_t
+ob_flush(call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *xdata)
 {
-	call_stub_t  *stub = NULL;
-
-	stub = fop_fgetxattr_stub (frame, default_fgetxattr_resume, fd, name,
-				   xdata);
-	if (!stub)
-		goto err;
-
-	open_and_resume (this, fd, stub);
+    OB_POST_FLUSH(this, frame, fd, fd, xdata);
 
-	return 0;
-err:
-	STACK_UNWIND_STRICT (fgetxattr, frame, -1, ENOMEM, 0, 0);
-
-	return 0;
+    return 0;
 }
 
-
-int
-ob_fremovexattr (call_frame_t *frame, xlator_t *this, fd_t *fd,
-		 const char *name, dict_t *xdata)
+static int32_t
+ob_fsync(call_frame_t *frame, xlator_t *this, fd_t *fd, int flag, dict_t *xdata)
 {
-	call_stub_t  *stub = NULL;
-
-	stub = fop_fremovexattr_stub (frame, default_fremovexattr_resume, fd,
-				      name, xdata);
-	if (!stub)
-		goto err;
+    OB_POST_FD(fsync, this, frame, fd, true, fd, flag, xdata);
 
-	open_and_resume (this, fd, stub);
+    return 0;
+}
 
-	return 0;
-err:
-	STACK_UNWIND_STRICT (fremovexattr, frame, -1, ENOMEM, 0);
+static int32_t
+ob_lk(call_frame_t *frame, xlator_t *this, fd_t *fd, int cmd,
+      struct gf_flock *flock, dict_t *xdata)
+{
+    OB_POST_FD(lk, this, frame, fd, true, fd, cmd, flock, xdata);
 
-	return 0;
+    return 0;
 }
 
-
-int
-ob_finodelk (call_frame_t *frame, xlator_t *this, const char *volume, fd_t *fd,
-	     int cmd, struct gf_flock *flock, dict_t *xdata)
+static int32_t
+ob_ftruncate(call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
+             dict_t *xdata)
 {
-	call_stub_t  *stub = NULL;
-
-	stub = fop_finodelk_stub (frame, default_finodelk_resume, volume, fd,
-				  cmd, flock, xdata);
-	if (!stub)
-		goto err;
+    OB_POST_FD(ftruncate, this, frame, fd, true, fd, offset, xdata);
 
-	open_and_resume (this, fd, stub);
+    return 0;
+}
 
-	return 0;
-err:
-	STACK_UNWIND_STRICT (finodelk, frame, -1, ENOMEM, 0);
+static int32_t
+ob_fsetxattr(call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *xattr,
+             int flags, dict_t *xdata)
+{
+    OB_POST_FD(fsetxattr, this, frame, fd, true, fd, xattr, flags, xdata);
 
-	return 0;
+    return 0;
 }
 
-
-int
-ob_fentrylk (call_frame_t *frame, xlator_t *this, const char *volume, fd_t *fd,
-	     const char *basename, entrylk_cmd cmd, entrylk_type type,
-	     dict_t *xdata)
+static int32_t
+ob_fgetxattr(call_frame_t *frame, xlator_t *this, fd_t *fd, const char *name,
+             dict_t *xdata)
 {
-	call_stub_t  *stub = NULL;
+    OB_POST_FD(fgetxattr, this, frame, fd, true, fd, name, xdata);
 
-	stub = fop_fentrylk_stub (frame, default_fentrylk_resume, volume, fd,
-				  basename, cmd, type, xdata);
-	if (!stub)
-		goto err;
-
-	open_and_resume (this, fd, stub);
+    return 0;
+}
 
-	return 0;
-err:
-	STACK_UNWIND_STRICT (fentrylk, frame, -1, ENOMEM, 0);
+static int32_t
+ob_fremovexattr(call_frame_t *frame, xlator_t *this, fd_t *fd, const char *name,
+                dict_t *xdata)
+{
+    OB_POST_FD(fremovexattr, this, frame, fd, true, fd, name, xdata);
 
-	return 0;
+    return 0;
 }
 
-
-int
-ob_fxattrop (call_frame_t *frame, xlator_t *this, fd_t *fd,
-	     gf_xattrop_flags_t optype, dict_t *xattr, dict_t *xdata)
+static int32_t
+ob_finodelk(call_frame_t *frame, xlator_t *this, const char *volume, fd_t *fd,
+            int cmd, struct gf_flock *flock, dict_t *xdata)
 {
-	call_stub_t  *stub = NULL;
+    OB_POST_FD(finodelk, this, frame, fd, true, volume, fd, cmd, flock, xdata);
 
-	stub = fop_fxattrop_stub (frame, default_fxattrop_resume, fd, optype,
-				  xattr, xdata);
-	if (!stub)
-		goto err;
-
-	open_and_resume (this, fd, stub);
+    return 0;
+}
 
-	return 0;
-err:
-	STACK_UNWIND_STRICT (fxattrop, frame, -1, ENOMEM, 0, 0);
+static int32_t
+ob_fentrylk(call_frame_t *frame, xlator_t *this, const char *volume, fd_t *fd,
+            const char *basename, entrylk_cmd cmd, entrylk_type type,
+            dict_t *xdata)
+{
+    OB_POST_FD(fentrylk, this, frame, fd, true, volume, fd, basename, cmd, type,
+               xdata);
 
-	return 0;
+    return 0;
 }
 
-
-int
-ob_fsetattr (call_frame_t *frame, xlator_t *this, fd_t *fd,
-	     struct iatt *iatt, int valid, dict_t *xdata)
+static int32_t
+ob_fxattrop(call_frame_t *frame, xlator_t *this, fd_t *fd,
+            gf_xattrop_flags_t optype, dict_t *xattr, dict_t *xdata)
 {
-	call_stub_t  *stub = NULL;
+    OB_POST_FD(fxattrop, this, frame, fd, true, fd, optype, xattr, xdata);
 
-	stub = fop_fsetattr_stub (frame, default_fsetattr_resume, fd,
-				  iatt, valid, xdata);
-	if (!stub)
-		goto err;
-
-	open_and_resume (this, fd, stub);
+    return 0;
+}
 
-	return 0;
-err:
-	STACK_UNWIND_STRICT (fsetattr, frame, -1, ENOMEM, 0, 0, 0);
+static int32_t
+ob_fsetattr(call_frame_t *frame, xlator_t *this, fd_t *fd, struct iatt *iatt,
+            int valid, dict_t *xdata)
+{
+    OB_POST_FD(fsetattr, this, frame, fd, true, fd, iatt, valid, xdata);
 
-	return 0;
+    return 0;
 }
 
-int
+static int32_t
 ob_fallocate(call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t mode,
-	     off_t offset, size_t len, dict_t *xdata)
+             off_t offset, size_t len, dict_t *xdata)
 {
-	call_stub_t *stub;
-
-	stub = fop_fallocate_stub(frame, default_fallocate_resume, fd, mode,
-				  offset, len, xdata);
-	if (!stub)
-		goto err;
+    OB_POST_FD(fallocate, this, frame, fd, true, fd, mode, offset, len, xdata);
 
-	open_and_resume(this, fd, stub);
-
-	return 0;
-err:
-	STACK_UNWIND_STRICT(fallocate, frame, -1, ENOMEM, NULL, NULL, NULL);
-	return 0;
+    return 0;
 }
 
-int
+static int32_t
 ob_discard(call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
-	   size_t len, dict_t *xdata)
+           size_t len, dict_t *xdata)
 {
-	call_stub_t *stub;
-
-	stub = fop_discard_stub(frame, default_discard_resume, fd, offset, len,
-				xdata);
-	if (!stub)
-		goto err;
+    OB_POST_FD(discard, this, frame, fd, true, fd, offset, len, xdata);
 
-	open_and_resume(this, fd, stub);
-
-	return 0;
-err:
-	STACK_UNWIND_STRICT(discard, frame, -1, ENOMEM, NULL, NULL, NULL);
-	return 0;
+    return 0;
 }
 
-int
+static int32_t
 ob_zerofill(call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
-           off_t len, dict_t *xdata)
+            off_t len, dict_t *xdata)
 {
-        call_stub_t *stub;
+    OB_POST_FD(zerofill, this, frame, fd, true, fd, offset, len, xdata);
 
-        stub = fop_zerofill_stub(frame, default_zerofill_resume, fd,
-                                 offset, len, xdata);
-        if (!stub)
-                goto err;
-
-        open_and_resume(this, fd, stub);
-
-        return 0;
-err:
-        STACK_UNWIND_STRICT(zerofill, frame, -1, ENOMEM, NULL, NULL, NULL);
-        return 0;
+    return 0;
 }
 
-
-int
-ob_unlink (call_frame_t *frame, xlator_t *this, loc_t *loc, int xflags,
-	   dict_t *xdata)
+static int32_t
+ob_unlink(call_frame_t *frame, xlator_t *this, loc_t *loc, int xflags,
+          dict_t *xdata)
 {
-	fd_t         *fd = NULL;
-	call_stub_t  *stub = NULL;
+    OB_POST_INODE(unlink, this, frame, loc->inode, true, loc, xflags, xdata);
 
-	stub = fop_unlink_stub (frame, default_unlink_resume, loc,
-				xflags, xdata);
-	if (!stub)
-		goto err;
+    return 0;
+}
 
-	fd = fd_lookup (loc->inode, 0);
+static int32_t
+ob_rename(call_frame_t *frame, xlator_t *this, loc_t *src, loc_t *dst,
+          dict_t *xdata)
+{
+    OB_POST_INODE(rename, this, frame, dst->inode, true, src, dst, xdata);
 
-	open_and_resume (this, fd, stub);
-        if (fd)
-                fd_unref (fd);
+    return 0;
+}
 
-	return 0;
-err:
-	STACK_UNWIND_STRICT (unlink, frame, -1, ENOMEM, 0, 0, 0);
+static int32_t
+ob_setattr(call_frame_t *frame, xlator_t *this, loc_t *loc, struct iatt *stbuf,
+           int32_t valid, dict_t *xdata)
+{
+    OB_POST_INODE(setattr, this, frame, loc->inode, true, loc, stbuf, valid,
+                  xdata);
 
-	return 0;
+    return 0;
 }
 
-
-int
-ob_rename (call_frame_t *frame, xlator_t *this, loc_t *src, loc_t *dst,
-	   dict_t *xdata)
+static int32_t
+ob_setxattr(call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *dict,
+            int32_t flags, dict_t *xdata)
 {
-	fd_t         *fd = NULL;
-	call_stub_t  *stub = NULL;
+    if (dict_get(dict, POSIX_ACL_DEFAULT_XATTR) ||
+        dict_get(dict, POSIX_ACL_ACCESS_XATTR) ||
+        dict_get(dict, GF_SELINUX_XATTR_KEY)) {
+        return default_setxattr(frame, this, loc, dict, flags, xdata);
+    }
 
-	stub = fop_rename_stub (frame, default_rename_resume, src, dst, xdata);
-	if (!stub)
-		goto err;
+    OB_POST_INODE(setxattr, this, frame, loc->inode, true, loc, dict, flags,
+                  xdata);
 
-	if (dst->inode)
-		fd = fd_lookup (dst->inode, 0);
+    return 0;
+}
 
-	open_and_resume (this, fd, stub);
-        if (fd)
-                fd_unref (fd);
+static void
+ob_fdclose(xlator_t *this, fd_t *fd)
+{
+    struct list_head list;
+    ob_inode_t *ob_inode;
+    call_stub_t *stub;
+
+    INIT_LIST_HEAD(&list);
+    stub = NULL;
+
+    LOCK(&fd->inode->lock);
+    {
+        ob_inode = ob_inode_get_locked(this, fd->inode);
+        if (ob_inode != NULL) {
+            ob_inode->open_count--;
+
+            /* If this fd is the same as ob_inode->first_fd, it means that
+             * the initial open has not fully completed. We'll try to cancel
+             * it. */
+            if (ob_inode->first_fd == fd) {
+                if (ob_inode->first_open == OB_OPEN_PREPARING) {
+                    /* In this case ob_open_dispatch() has not been called yet.
+                     * We clear first_fd and first_open to allow that function
+                     * to know that the open is not really needed. This also
+                     * allows other requests to work as expected if they
+                     * arrive before the dispatch function is called. If there
+                     * are pending fops, we can directly process them here.
+                     * (note that there shouldn't be any fd related fops, but
+                     * if there are, it's fine if they fail). */
+                    ob_inode->first_fd = NULL;
+                    ob_inode->first_open = NULL;
+                    ob_inode->triggered = false;
+                    list_splice_init(&ob_inode->resume_fops, &list);
+                } else if (!ob_inode->triggered) {
+                    /* If the open has already been dispatched, we can only
+                     * cancel it if it has not been triggered. Otherwise we
+                     * simply wait until it completes. While it's not triggered,
+                     * first_open must be a valid stub and there can't be any
+                     * pending fops. */
+                    GF_ASSERT((ob_inode->first_open != NULL) &&
+                              list_empty(&ob_inode->resume_fops));
+
+                    ob_inode->first_fd = NULL;
+                    stub = ob_inode->first_open;
+                    ob_inode->first_open = NULL;
+                }
+            }
+        }
+    }
+    UNLOCK(&fd->inode->lock);
 
-	return 0;
-err:
-	STACK_UNWIND_STRICT (rename, frame, -1, ENOMEM, 0, 0, 0, 0, 0, 0);
+    if (stub != NULL) {
+        ob_open_destroy(stub, fd);
+    }
 
-	return 0;
+    ob_resume_pending(&list);
 }
 
-
 int
-ob_release (xlator_t *this, fd_t *fd)
+ob_forget(xlator_t *this, inode_t *inode)
 {
-	ob_fd_t *ob_fd = NULL;
+    ob_inode_t *ob_inode;
+    uint64_t value = 0;
 
-	ob_fd = ob_fd_ctx_get (this, fd);
+    if ((inode_ctx_del(inode, this, &value) == 0) && (value != 0)) {
+        ob_inode = (ob_inode_t *)(uintptr_t)value;
+        GF_FREE(ob_inode);
+    }
 
-	ob_fd_free (ob_fd);
-
-	return 0;
+    return 0;
 }
 
-
 int
-ob_priv_dump (xlator_t *this)
+ob_priv_dump(xlator_t *this)
 {
-        ob_conf_t        *conf       = NULL;
-        char              key_prefix[GF_DUMP_MAX_BUF_LEN];
+    ob_conf_t *conf = NULL;
+    char key_prefix[GF_DUMP_MAX_BUF_LEN];
 
-        conf = this->private;
+    conf = this->private;
 
-        if (!conf)
-                return -1;
+    if (!conf)
+        return -1;
 
-        gf_proc_dump_build_key (key_prefix, "xlator.performance.open-behind",
-                                "priv");
+    gf_proc_dump_build_key(key_prefix, "xlator.performance.open-behind",
+                           "priv");
 
-        gf_proc_dump_add_section (key_prefix);
+    gf_proc_dump_add_section("%s", key_prefix);
 
-        gf_proc_dump_write ("use_anonymous_fd", "%d", conf->use_anonymous_fd);
+    gf_proc_dump_write("use_anonymous_fd", "%d", conf->use_anonymous_fd);
 
-        gf_proc_dump_write ("lazy_open", "%d", conf->lazy_open);
+    gf_proc_dump_write("lazy_open", "%d", conf->lazy_open);
 
-        return 0;
+    return 0;
 }
 
-
 int
-ob_fdctx_dump (xlator_t *this, fd_t *fd)
+ob_fdctx_dump(xlator_t *this, fd_t *fd)
 {
-	ob_fd_t   *ob_fd = NULL;
-        char       key_prefix[GF_DUMP_MAX_BUF_LEN] = {0, };
-	int        ret = 0;
-
-	ret = TRY_LOCK (&fd->lock);
-	if (ret)
-		return 0;
-
-	ob_fd = __ob_fd_ctx_get (this, fd);
-	if (!ob_fd) {
-		UNLOCK (&fd->lock);
-		return 0;
-	}
-
-        gf_proc_dump_build_key (key_prefix, "xlator.performance.open-behind",
-                                "file");
-        gf_proc_dump_add_section (key_prefix);
-
-        gf_proc_dump_write ("fd", "%p", fd);
-
-        gf_proc_dump_write ("open_frame", "%p", ob_fd->open_frame);
+    char key_prefix[GF_DUMP_MAX_BUF_LEN] = {
+        0,
+    };
+    uint64_t value = 0;
+    int ret = 0, error = 0;
+
+    ret = TRY_LOCK(&fd->lock);
+    if (ret)
+        return 0;
 
-        gf_proc_dump_write ("open_frame.root.unique", "%p",
-			    ob_fd->open_frame->root->unique);
+    if ((__fd_ctx_get(fd, this, &value) == 0) && (value != 0)) {
+        error = (int32_t)value;
+    }
 
-	gf_proc_dump_write ("loc.path", "%s", ob_fd->loc.path);
+    gf_proc_dump_build_key(key_prefix, "xlator.performance.open-behind",
+                           "file");
+    gf_proc_dump_add_section("%s", key_prefix);
 
-	gf_proc_dump_write ("loc.ino", "%s", uuid_utoa (ob_fd->loc.gfid));
+    gf_proc_dump_write("fd", "%p", fd);
 
-        gf_proc_dump_write ("flags", "%p", ob_fd->open_frame);
+    gf_proc_dump_write("error", "%d", error);
 
-	UNLOCK (&fd->lock);
+    UNLOCK(&fd->lock);
 
-	return 0;
+    return 0;
 }
 
-
 int
-mem_acct_init (xlator_t *this)
+mem_acct_init(xlator_t *this)
 {
-        int     ret = -1;
+    int ret = -1;
 
-        ret = xlator_mem_acct_init (this, gf_ob_mt_end + 1);
+    ret = xlator_mem_acct_init(this, gf_ob_mt_end + 1);
 
-        if (ret)
-                gf_log (this->name, GF_LOG_ERROR, "Memory accounting failed");
+    if (ret)
+        gf_msg(this->name, GF_LOG_ERROR, ENOMEM, OPEN_BEHIND_MSG_NO_MEMORY,
+               "Memory accounting failed");
 
-        return ret;
+    return ret;
 }
 
-
 int
-reconfigure (xlator_t *this, dict_t *options)
+reconfigure(xlator_t *this, dict_t *options)
 {
-        ob_conf_t  *conf = NULL;
-	int         ret = -1;
+    ob_conf_t *conf = NULL;
+    int ret = -1;
 
-        conf = this->private;
+    conf = this->private;
 
-        GF_OPTION_RECONF ("use-anonymous-fd", conf->use_anonymous_fd, options,
-			  bool, out);
+    GF_OPTION_RECONF("use-anonymous-fd", conf->use_anonymous_fd, options, bool,
+                     out);
 
-        GF_OPTION_RECONF ("lazy-open", conf->lazy_open, options, bool, out);
-        GF_OPTION_RECONF ("read-after-open", conf->read_after_open, options,
-                          bool, out);
+    GF_OPTION_RECONF("lazy-open", conf->lazy_open, options, bool, out);
 
-        ret = 0;
+    GF_OPTION_RECONF("read-after-open", conf->read_after_open, options, bool,
+                     out);
+
+    GF_OPTION_RECONF("pass-through", this->pass_through, options, bool, out);
+    ret = 0;
 out:
-        return ret;
+    return ret;
 }
 
-
 int
-init (xlator_t *this)
+init(xlator_t *this)
 {
-        ob_conf_t    *conf = NULL;
+    ob_conf_t *conf = NULL;
+
+    if (!this->children || this->children->next) {
+        gf_msg(this->name, GF_LOG_ERROR, 0,
+               OPEN_BEHIND_MSG_XLATOR_CHILD_MISCONFIGURED,
+               "FATAL: volume (%s) not configured with exactly one "
+               "child",
+               this->name);
+        return -1;
+    }
 
-        if (!this->children || this->children->next) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "FATAL: volume (%s) not configured with exactly one "
-                        "child", this->name);
-                return -1;
-        }
+    if (!this->parents)
+        gf_msg(this->name, GF_LOG_WARNING, 0, OPEN_BEHIND_MSG_VOL_MISCONFIGURED,
+               "dangling volume. check volfile ");
+
+    conf = GF_CALLOC(1, sizeof(*conf), gf_ob_mt_conf_t);
+    if (!conf)
+        goto err;
+
+    GF_OPTION_INIT("use-anonymous-fd", conf->use_anonymous_fd, bool, err);
 
-        if (!this->parents)
-                gf_log (this->name, GF_LOG_WARNING,
-                        "dangling volume. check volfile ");
+    GF_OPTION_INIT("lazy-open", conf->lazy_open, bool, err);
 
-        conf = GF_CALLOC (1, sizeof (*conf), gf_ob_mt_conf_t);
-        if (!conf)
-                goto err;
+    GF_OPTION_INIT("read-after-open", conf->read_after_open, bool, err);
 
-        GF_OPTION_INIT ("use-anonymous-fd", conf->use_anonymous_fd, bool, err);
+    GF_OPTION_INIT("pass-through", this->pass_through, bool, err);
 
-        GF_OPTION_INIT ("lazy-open", conf->lazy_open, bool, err);
-        GF_OPTION_INIT ("read-after-open", conf->read_after_open, bool, err);
-        this->private = conf;
+    this->private = conf;
 
-	return 0;
+    return 0;
 err:
-	if (conf)
-                GF_FREE (conf);
+    if (conf)
+        GF_FREE(conf);
 
-        return -1;
+    return -1;
 }
 
-
 void
-fini (xlator_t *this)
+fini(xlator_t *this)
 {
-        ob_conf_t *conf = NULL;
+    ob_conf_t *conf = NULL;
 
-        conf = this->private;
+    conf = this->private;
 
-        GF_FREE (conf);
+    GF_FREE(conf);
 
-	return;
+    return;
 }
 
-
 struct xlator_fops fops = {
-        .open        = ob_open,
-        .readv       = ob_readv,
-        .writev      = ob_writev,
-	.flush       = ob_flush,
-	.fsync       = ob_fsync,
-	.fstat       = ob_fstat,
-	.ftruncate   = ob_ftruncate,
-	.fsetxattr   = ob_fsetxattr,
-	.fgetxattr   = ob_fgetxattr,
-	.fremovexattr = ob_fremovexattr,
-	.finodelk    = ob_finodelk,
-	.fentrylk    = ob_fentrylk,
-	.fxattrop    = ob_fxattrop,
-	.fsetattr    = ob_fsetattr,
-	.fallocate   = ob_fallocate,
-	.discard     = ob_discard,
-        .zerofill    = ob_zerofill,
-	.unlink      = ob_unlink,
-	.rename      = ob_rename,
-	.lk          = ob_lk,
+    .open = ob_open,
+    .create = ob_create,
+    .readv = ob_readv,
+    .writev = ob_writev,
+    .flush = ob_flush,
+    .fsync = ob_fsync,
+    .fstat = ob_fstat,
+    .seek = ob_seek,
+    .ftruncate = ob_ftruncate,
+    .fsetxattr = ob_fsetxattr,
+    .setxattr = ob_setxattr,
+    .fgetxattr = ob_fgetxattr,
+    .fremovexattr = ob_fremovexattr,
+    .finodelk = ob_finodelk,
+    .fentrylk = ob_fentrylk,
+    .fxattrop = ob_fxattrop,
+    .fsetattr = ob_fsetattr,
+    .setattr = ob_setattr,
+    .fallocate = ob_fallocate,
+    .discard = ob_discard,
+    .zerofill = ob_zerofill,
+    .unlink = ob_unlink,
+    .rename = ob_rename,
+    .lk = ob_lk,
 };
 
 struct xlator_cbks cbks = {
-        .release  = ob_release,
+    .fdclose = ob_fdclose,
+    .forget = ob_forget,
 };
 
 struct xlator_dumpops dumpops = {
-	.priv    = ob_priv_dump,
-	.fdctx   = ob_fdctx_dump,
+    .priv = ob_priv_dump,
+    .fdctx = ob_fdctx_dump,
 };
 
-
 struct volume_options options[] = {
-        { .key  = {"use-anonymous-fd"},
-          .type = GF_OPTION_TYPE_BOOL,
-          .default_value = "yes",
-          .description = "For read operations, use anonymous FD when "
-          "original FD is open-behind and not yet opened in the backend.",
-        },
-        { .key  = {"lazy-open"},
-          .type = GF_OPTION_TYPE_BOOL,
-          .default_value = "yes",
-          .description = "Perform open in the backend only when a necessary "
-          "FOP arrives (e.g writev on the FD, unlink of the file). When option "
-          "is disabled, perform backend open right after unwinding open().",
-        },
-        { .key  = {"read-after-open"},
-          .type = GF_OPTION_TYPE_BOOL,
-          .default_value = "no",
-          .description = "read is sent only after actual open happens and real "
-          "fd is obtained, instead of doing on anonymous fd (similar to write)",
-        },
-        { .key  = {NULL} }
+    {
+        .key = {"open-behind"},
+        .type = GF_OPTION_TYPE_BOOL,
+        .default_value = "off",
+        .description = "enable/disable open-behind",
+        .op_version = {GD_OP_VERSION_6_0},
+        .flags = OPT_FLAG_SETTABLE,
+    },
+    {
+        .key = {"use-anonymous-fd"},
+        .type = GF_OPTION_TYPE_BOOL,
+        .default_value = "no",
+        .description =
+            "For read operations, use anonymous FD when "
+            "original FD is open-behind and not yet opened in the backend.",
+    },
+    {
+        .key = {"lazy-open"},
+        .type = GF_OPTION_TYPE_BOOL,
+        .default_value = "yes",
+        .description =
+            "Perform open in the backend only when a necessary "
+            "FOP arrives (e.g writev on the FD, unlink of the file). When "
+            "option "
+            "is disabled, perform backend open right after unwinding open().",
+        .op_version = {3},
+        .flags = OPT_FLAG_SETTABLE | OPT_FLAG_CLIENT_OPT,
+        .tags = {},
+        /* option_validation_fn validate_fn; */
+    },
+    {
+        .key = {"read-after-open"},
+        .type = GF_OPTION_TYPE_BOOL,
+        .default_value = "yes",
+        .description = "read is sent only after actual open happens and real "
+                       "fd is obtained, instead of doing on anonymous fd "
+                       "(similar to write)",
+        .op_version = {3},
+        .flags = OPT_FLAG_SETTABLE | OPT_FLAG_CLIENT_OPT,
+        .tags = {},
+        /* option_validation_fn validate_fn; */
+    },
+    {.key = {"pass-through"},
+     .type = GF_OPTION_TYPE_BOOL,
+     .default_value = "false",
+     .op_version = {GD_OP_VERSION_4_1_0},
+     .flags = OPT_FLAG_SETTABLE | OPT_FLAG_DOC | OPT_FLAG_CLIENT_OPT,
+     .tags = {"open-behind"},
+     .description = "Enable/Disable open behind translator"},
+    {.key = {NULL}}
+
+};
 
+xlator_api_t xlator_api = {
+    .init = init,
+    .fini = fini,
+    .reconfigure = reconfigure,
+    .mem_acct_init = mem_acct_init,
+    .op_version = {1}, /* Present from the initial version */
+    .dumpops = &dumpops,
+    .fops = &fops,
+    .cbks = &cbks,
+    .options = options,
+    .identifier = "open-behind",
+    .category = GF_MAINTAINED,
 };
diff --git a/xlators/performance/quick-read/src/Makefile.am b/xlators/performance/quick-read/src/Makefile.am
index 4906f408abc..8eb6cece738 100644
--- a/xlators/performance/quick-read/src/Makefile.am
+++ b/xlators/performance/quick-read/src/Makefile.am
@@ -1,14 +1,15 @@
 xlator_LTLIBRARIES = quick-read.la
 xlatordir = $(libdir)/glusterfs/$(PACKAGE_VERSION)/xlator/performance
 
-quick_read_la_LDFLAGS = -module -avoid-version 
+quick_read_la_LDFLAGS = -module $(GF_XLATOR_DEFAULT_LDFLAGS)
 
 quick_read_la_SOURCES = quick-read.c
 quick_read_la_LIBADD = $(top_builddir)/libglusterfs/src/libglusterfs.la
 
-noinst_HEADERS = quick-read.h quick-read-mem-types.h
+noinst_HEADERS = quick-read.h quick-read-mem-types.h quick-read-messages.h
 
-AM_CPPFLAGS = $(GF_CPPFLAGS) -I$(top_srcdir)/libglusterfs/src
+AM_CPPFLAGS = $(GF_CPPFLAGS) -I$(top_srcdir)/libglusterfs/src \
+	-I$(top_srcdir)/rpc/xdr/src -I$(top_builddir)/rpc/xdr/src
 
 AM_CFLAGS = -Wall $(GF_CFLAGS)
 
diff --git a/xlators/performance/quick-read/src/quick-read-mem-types.h b/xlators/performance/quick-read/src/quick-read-mem-types.h
index 78547f64116..e4aef8549ff 100644
--- a/xlators/performance/quick-read/src/quick-read-mem-types.h
+++ b/xlators/performance/quick-read/src/quick-read-mem-types.h
@@ -11,17 +11,13 @@
 #ifndef __QR_MEM_TYPES_H__
 #define __QR_MEM_TYPES_H__
 
-#include "mem-types.h"
+#include <glusterfs/mem-types.h>
 
 enum gf_qr_mem_types_ {
-        gf_qr_mt_qr_inode_t   = gf_common_mt_end + 1,
-	gf_qr_mt_content_t,
-        gf_qr_mt_qr_fd_ctx_t,
-        gf_qr_mt_iovec,
-        gf_qr_mt_qr_conf_t,
-        gf_qr_mt_qr_priority_t,
-        gf_qr_mt_qr_private_t,
-        gf_qr_mt_qr_unlink_ctx_t,
-        gf_qr_mt_end
+    gf_qr_mt_qr_inode_t = gf_common_mt_end + 1,
+    gf_qr_mt_content_t,
+    gf_qr_mt_qr_priority_t,
+    gf_qr_mt_qr_private_t,
+    gf_qr_mt_end
 };
 #endif
diff --git a/xlators/performance/quick-read/src/quick-read-messages.h b/xlators/performance/quick-read/src/quick-read-messages.h
new file mode 100644
index 00000000000..da9724a3c9c
--- /dev/null
+++ b/xlators/performance/quick-read/src/quick-read-messages.h
@@ -0,0 +1,31 @@
+/*Copyright (c) 2015 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
+*/
+
+#ifndef _QUICK_READ_MESSAGES_H_
+#define _QUICK_READ_MESSAGES_H_
+
+#include <glusterfs/glfs-message-id.h>
+
+/* To add new message IDs, append new identifiers at the end of the list.
+ *
+ * Never remove a message ID. If it's not used anymore, you can rename it or
+ * leave it as it is, but not delete it. This is to prevent reutilization of
+ * IDs by other messages.
+ *
+ * The component name must match one of the entries defined in
+ * glfs-message-id.h.
+ */
+
+GLFS_MSGID(QUICK_READ, QUICK_READ_MSG_ENFORCEMENT_FAILED,
+           QUICK_READ_MSG_INVALID_ARGUMENT,
+           QUICK_READ_MSG_XLATOR_CHILD_MISCONFIGURED, QUICK_READ_MSG_NO_MEMORY,
+           QUICK_READ_MSG_VOL_MISCONFIGURED, QUICK_READ_MSG_DICT_SET_FAILED,
+           QUICK_READ_MSG_INVALID_CONFIG, QUICK_READ_MSG_LRU_NOT_EMPTY);
+
+#endif /* _QUICK_READ_MESSAGES_H_ */
diff --git a/xlators/performance/quick-read/src/quick-read.c b/xlators/performance/quick-read/src/quick-read.c
index aa44c5d388b..7fe4b3c3a4b 100644
--- a/xlators/performance/quick-read/src/quick-read.c
+++ b/xlators/performance/quick-read/src/quick-read.c
@@ -8,1142 +8,1637 @@
   cases as published by the Free Software Foundation.
 */
 
+#include <math.h>
 #include "quick-read.h"
-#include "statedump.h"
+#include <glusterfs/statedump.h>
+#include "quick-read-messages.h"
+#include <glusterfs/upcall-utils.h>
+#include <glusterfs/atomic.h>
 
-qr_inode_t *qr_inode_ctx_get (xlator_t *this, inode_t *inode);
-void __qr_inode_prune (qr_inode_table_t *table, qr_inode_t *qr_inode);
+typedef struct qr_local {
+    inode_t *inode;
+    uint64_t incident_gen;
+    fd_t *fd;
+} qr_local_t;
 
+qr_inode_t *
+qr_inode_ctx_get(xlator_t *this, inode_t *inode);
 
-int
-__qr_inode_ctx_set (xlator_t *this, inode_t *inode, qr_inode_t *qr_inode)
+void
+__qr_inode_prune_data(xlator_t *this, qr_inode_table_t *table,
+                      qr_inode_t *qr_inode);
+
+void
+qr_local_wipe(qr_local_t *local)
 {
-	uint64_t    value = 0;
-	int         ret = -1;
+    if (!local)
+        goto out;
 
-	value = (long) qr_inode;
+    if (local->inode)
+        inode_unref(local->inode);
 
-	ret = __inode_ctx_set (inode, this, &value);
+    if (local->fd)
+        fd_unref(local->fd);
 
-	return ret;
+    GF_FREE(local);
+out:
+    return;
 }
 
+uint64_t
+__qr_get_generation(xlator_t *this, qr_inode_t *qr_inode)
+{
+    uint64_t gen = 0, rollover;
+    qr_private_t *priv = NULL;
+    qr_inode_table_t *table = NULL;
+
+    priv = this->private;
+    table = &priv->table;
+
+    gen = GF_ATOMIC_INC(priv->generation);
+    if (gen == 0) {
+        qr_inode->gen_rollover = !qr_inode->gen_rollover;
+        gen = GF_ATOMIC_INC(priv->generation);
+        __qr_inode_prune_data(this, table, qr_inode);
+        qr_inode->gen = qr_inode->invalidation_time = gen - 1;
+    }
+
+    rollover = qr_inode->gen_rollover;
+    gen |= (rollover << 32);
+    return gen;
+}
 
-qr_inode_t *
-__qr_inode_ctx_get (xlator_t *this, inode_t *inode)
+uint64_t
+qr_get_generation(xlator_t *this, inode_t *inode)
 {
-	qr_inode_t *qr_inode = NULL;
-	uint64_t    value = 0;
-	int         ret = -1;
+    qr_inode_t *qr_inode = NULL;
+    uint64_t gen = 0;
+    qr_inode_table_t *table = NULL;
+    qr_private_t *priv = NULL;
+
+    priv = this->private;
+    table = &priv->table;
 
-	ret = __inode_ctx_get (inode, this, &value);
-	if (ret)
-		return NULL;
+    qr_inode = qr_inode_ctx_get(this, inode);
 
-	qr_inode = (void *) ((long) value);
+    if (qr_inode) {
+        LOCK(&table->lock);
+        {
+            gen = __qr_get_generation(this, qr_inode);
+        }
+        UNLOCK(&table->lock);
+    } else {
+        gen = GF_ATOMIC_INC(priv->generation);
+        if (gen == 0) {
+            gen = GF_ATOMIC_INC(priv->generation);
+        }
+    }
 
-	return qr_inode;
+    return gen;
 }
 
+qr_local_t *
+qr_local_get(xlator_t *this, inode_t *inode)
+{
+    qr_local_t *local = NULL;
+
+    local = GF_CALLOC(1, sizeof(*local), gf_common_mt_char);
+    if (!local)
+        goto out;
+
+    local->incident_gen = qr_get_generation(this, inode);
+out:
+    return local;
+}
+
+#define QR_STACK_UNWIND(fop, frame, params...)                                 \
+    do {                                                                       \
+        qr_local_t *__local = NULL;                                            \
+        if (frame) {                                                           \
+            __local = frame->local;                                            \
+            frame->local = NULL;                                               \
+        }                                                                      \
+        STACK_UNWIND_STRICT(fop, frame, params);                               \
+        qr_local_wipe(__local);                                                \
+    } while (0)
+
+void
+__qr_inode_prune(xlator_t *this, qr_inode_table_t *table, qr_inode_t *qr_inode,
+                 uint64_t gen);
+
+int
+__qr_inode_ctx_set(xlator_t *this, inode_t *inode, qr_inode_t *qr_inode)
+{
+    uint64_t value = 0;
+    int ret = -1;
+
+    value = (long)qr_inode;
+
+    ret = __inode_ctx_set(inode, this, &value);
+
+    return ret;
+}
 
 qr_inode_t *
-qr_inode_ctx_get (xlator_t *this, inode_t *inode)
+__qr_inode_ctx_get(xlator_t *this, inode_t *inode)
 {
-	qr_inode_t *qr_inode = NULL;
+    qr_inode_t *qr_inode = NULL;
+    uint64_t value = 0;
+    int ret = -1;
+
+    ret = __inode_ctx_get(inode, this, &value);
+    if (ret)
+        return NULL;
 
-	LOCK (&inode->lock);
-	{
-		qr_inode = __qr_inode_ctx_get (this, inode);
-	}
-	UNLOCK (&inode->lock);
+    qr_inode = (void *)((long)value);
 
-	return qr_inode;
+    return qr_inode;
 }
 
+qr_inode_t *
+qr_inode_ctx_get(xlator_t *this, inode_t *inode)
+{
+    qr_inode_t *qr_inode = NULL;
+
+    if (inode == NULL)
+        goto out;
+
+    LOCK(&inode->lock);
+    {
+        qr_inode = __qr_inode_ctx_get(this, inode);
+    }
+    UNLOCK(&inode->lock);
+
+out:
+    return qr_inode;
+}
 
 qr_inode_t *
-qr_inode_new (xlator_t *this, inode_t *inode)
+qr_inode_new(xlator_t *this, inode_t *inode)
 {
-        qr_inode_t    *qr_inode = NULL;
+    qr_inode_t *qr_inode = NULL;
 
-        qr_inode = GF_CALLOC (1, sizeof (*qr_inode), gf_qr_mt_qr_inode_t);
-        if (!qr_inode)
-                return NULL;
+    qr_inode = GF_CALLOC(1, sizeof(*qr_inode), gf_qr_mt_qr_inode_t);
+    if (!qr_inode)
+        return NULL;
 
-        INIT_LIST_HEAD (&qr_inode->lru);
+    INIT_LIST_HEAD(&qr_inode->lru);
 
-        qr_inode->priority = 0; /* initial priority */
+    qr_inode->priority = 0; /* initial priority */
 
-        return qr_inode;
+    return qr_inode;
 }
 
-
 qr_inode_t *
-qr_inode_ctx_get_or_new (xlator_t *this, inode_t *inode)
+qr_inode_ctx_get_or_new(xlator_t *this, inode_t *inode)
 {
-	qr_inode_t   *qr_inode = NULL;
-	int           ret = -1;
-	qr_private_t *priv = NULL;
-
-	priv = this->private;
-
-	LOCK (&inode->lock);
-	{
-		qr_inode = __qr_inode_ctx_get (this, inode);
-		if (qr_inode)
-			goto unlock;
-
-		qr_inode = qr_inode_new (this, inode);
-		if (!qr_inode)
-			goto unlock;
-
-		ret = __qr_inode_ctx_set (this, inode, qr_inode);
-		if (ret) {
-			__qr_inode_prune (&priv->table, qr_inode);
-			GF_FREE (qr_inode);
-                        qr_inode = NULL;
-		}
-	}
+    qr_inode_t *qr_inode = NULL;
+    int ret = -1;
+    qr_private_t *priv = NULL;
+
+    priv = this->private;
+
+    LOCK(&inode->lock);
+    {
+        qr_inode = __qr_inode_ctx_get(this, inode);
+        if (qr_inode)
+            goto unlock;
+
+        qr_inode = qr_inode_new(this, inode);
+        if (!qr_inode)
+            goto unlock;
+
+        ret = __qr_inode_ctx_set(this, inode, qr_inode);
+        if (ret) {
+            __qr_inode_prune(this, &priv->table, qr_inode, 0);
+            GF_FREE(qr_inode);
+            qr_inode = NULL;
+        }
+    }
 unlock:
-	UNLOCK (&inode->lock);
+    UNLOCK(&inode->lock);
 
-	return qr_inode;
+    return qr_inode;
 }
 
-
 uint32_t
-qr_get_priority (qr_conf_t *conf, const char *path)
+qr_get_priority(qr_conf_t *conf, const char *path)
 {
-        uint32_t            priority = 0;
-        struct qr_priority *curr     = NULL;
+    uint32_t priority = 0;
+    struct qr_priority *curr = NULL;
 
-        list_for_each_entry (curr, &conf->priority_list, list) {
-                if (fnmatch (curr->pattern, path, FNM_NOESCAPE) == 0)
-                        priority = curr->priority;
-        }
+    list_for_each_entry(curr, &conf->priority_list, list)
+    {
+        if (fnmatch(curr->pattern, path, FNM_NOESCAPE) == 0)
+            priority = curr->priority;
+    }
 
-        return priority;
+    return priority;
 }
 
-
 void
-__qr_inode_register (qr_inode_table_t *table, qr_inode_t *qr_inode)
+__qr_inode_register(xlator_t *this, qr_inode_table_t *table,
+                    qr_inode_t *qr_inode)
 {
-	if (!qr_inode->data)
-		return;
+    qr_private_t *priv = NULL;
 
-	if (list_empty (&qr_inode->lru))
-		/* first time addition of this qr_inode into table */
-		table->cache_used += qr_inode->size;
-	else
-		list_del_init (&qr_inode->lru);
+    if (!qr_inode->data)
+        return;
 
-	list_add_tail (&qr_inode->lru, &table->lru[qr_inode->priority]);
-}
+    priv = this->private;
+    if (!priv)
+        return;
+
+    if (list_empty(&qr_inode->lru))
+        /* first time addition of this qr_inode into table */
+        table->cache_used += qr_inode->size;
+    else
+        list_del_init(&qr_inode->lru);
 
+    list_add_tail(&qr_inode->lru, &table->lru[qr_inode->priority]);
+
+    GF_ATOMIC_INC(priv->qr_counter.files_cached);
+
+    return;
+}
 
 void
-qr_inode_set_priority (xlator_t *this, inode_t *inode, const char *path)
+qr_inode_set_priority(xlator_t *this, inode_t *inode, const char *path)
 {
-	uint32_t          priority = 0;
-	qr_inode_table_t *table = NULL;
-	qr_inode_t       *qr_inode = NULL;
-        qr_private_t     *priv = NULL;
-	qr_conf_t        *conf = NULL;
-
-	qr_inode = qr_inode_ctx_get (this, inode);
-	if (!qr_inode)
-		return;
-
-	priv = this->private;
-	table = &priv->table;
-	conf = &priv->conf;
-
-	if (path)
-		priority = qr_get_priority (conf, path);
-	else
-		/* retain existing priority, just bump LRU */
-		priority = qr_inode->priority;
-
-	LOCK (&table->lock);
-	{
-		qr_inode->priority = priority;
-
-		__qr_inode_register (table, qr_inode);
-	}
-	UNLOCK (&table->lock);
-}
+    uint32_t priority = 0;
+    qr_inode_table_t *table = NULL;
+    qr_inode_t *qr_inode = NULL;
+    qr_private_t *priv = NULL;
+    qr_conf_t *conf = NULL;
+
+    qr_inode = qr_inode_ctx_get(this, inode);
+    if (!qr_inode)
+        return;
 
+    priv = this->private;
+    table = &priv->table;
+    conf = &priv->conf;
+
+    if (path)
+        priority = qr_get_priority(conf, path);
+    else
+        /* retain existing priority, just bump LRU */
+        priority = qr_inode->priority;
+
+    LOCK(&table->lock);
+    {
+        qr_inode->priority = priority;
+
+        __qr_inode_register(this, table, qr_inode);
+    }
+    UNLOCK(&table->lock);
+}
 
-/* To be called with priv->table.lock held */
 void
-__qr_inode_prune (qr_inode_table_t *table, qr_inode_t *qr_inode)
+__qr_inode_prune_data(xlator_t *this, qr_inode_table_t *table,
+                      qr_inode_t *qr_inode)
 {
-	GF_FREE (qr_inode->data);
-	qr_inode->data = NULL;
+    qr_private_t *priv = NULL;
 
-	if (!list_empty (&qr_inode->lru)) {
-		table->cache_used -= qr_inode->size;
-		qr_inode->size = 0;
+    priv = this->private;
 
-		list_del_init (&qr_inode->lru);
-	}
+    GF_FREE(qr_inode->data);
+    qr_inode->data = NULL;
 
-	memset (&qr_inode->buf, 0, sizeof (qr_inode->buf));
-}
+    if (!list_empty(&qr_inode->lru)) {
+        table->cache_used -= qr_inode->size;
+        qr_inode->size = 0;
 
+        list_del_init(&qr_inode->lru);
 
+        GF_ATOMIC_DEC(priv->qr_counter.files_cached);
+    }
+
+    memset(&qr_inode->buf, 0, sizeof(qr_inode->buf));
+}
+
+/* To be called with priv->table.lock held */
 void
-qr_inode_prune (xlator_t *this, inode_t *inode)
+__qr_inode_prune(xlator_t *this, qr_inode_table_t *table, qr_inode_t *qr_inode,
+                 uint64_t gen)
 {
-        qr_private_t     *priv          = NULL;
-        qr_inode_table_t *table         = NULL;
-	qr_inode_t       *qr_inode      = NULL;
-
-	qr_inode = qr_inode_ctx_get (this, inode);
-	if (!qr_inode)
-		return;
-
-	priv = this->private;
-	table = &priv->table;
-
-	LOCK (&table->lock);
-	{
-		__qr_inode_prune (table, qr_inode);
-	}
-	UNLOCK (&table->lock);
+    __qr_inode_prune_data(this, table, qr_inode);
+    if (gen)
+        qr_inode->gen = gen;
+    qr_inode->invalidation_time = __qr_get_generation(this, qr_inode);
 }
 
+void
+qr_inode_prune(xlator_t *this, inode_t *inode, uint64_t gen)
+{
+    qr_private_t *priv = NULL;
+    qr_inode_table_t *table = NULL;
+    qr_inode_t *qr_inode = NULL;
+
+    qr_inode = qr_inode_ctx_get(this, inode);
+    if (!qr_inode)
+        return;
+
+    priv = this->private;
+    table = &priv->table;
+
+    LOCK(&table->lock);
+    {
+        __qr_inode_prune(this, table, qr_inode, gen);
+    }
+    UNLOCK(&table->lock);
+}
 
 /* To be called with priv->table.lock held */
 void
-__qr_cache_prune (qr_inode_table_t *table, qr_conf_t *conf)
+__qr_cache_prune(xlator_t *this, qr_inode_table_t *table, qr_conf_t *conf)
 {
-        qr_inode_t        *curr = NULL;
-	qr_inode_t        *next = NULL;
-        int                index = 0;
-	size_t             size_pruned = 0;
-
-        for (index = 0; index < conf->max_pri; index++) {
-                list_for_each_entry_safe (curr, next, &table->lru[index], lru) {
+    qr_inode_t *curr = NULL;
+    qr_inode_t *next = NULL;
+    int index = 0;
+    size_t size_pruned = 0;
 
-                        size_pruned += curr->size;
+    for (index = 0; index < conf->max_pri; index++) {
+        list_for_each_entry_safe(curr, next, &table->lru[index], lru)
+        {
+            size_pruned += curr->size;
 
-                        __qr_inode_prune (table, curr);
+            __qr_inode_prune(this, table, curr, 0);
 
-                        if (table->cache_used < conf->cache_size)
-				return;
-                }
+            if (table->cache_used < conf->cache_size)
+                return;
         }
+    }
 
-        return;
+    return;
 }
 
-
 void
-qr_cache_prune (xlator_t *this)
+qr_cache_prune(xlator_t *this)
 {
-        qr_private_t      *priv = NULL;
-        qr_conf_t         *conf = NULL;
-        qr_inode_table_t  *table = NULL;
-
-        priv = this->private;
-        table = &priv->table;
-        conf = &priv->conf;
-
-	LOCK (&table->lock);
-	{
-		if (table->cache_used > conf->cache_size)
-			__qr_cache_prune (table, conf);
-	}
-	UNLOCK (&table->lock);
+    qr_private_t *priv = NULL;
+    qr_conf_t *conf = NULL;
+    qr_inode_table_t *table = NULL;
+
+    priv = this->private;
+    table = &priv->table;
+    conf = &priv->conf;
+
+    LOCK(&table->lock);
+    {
+        if (table->cache_used > conf->cache_size)
+            __qr_cache_prune(this, table, conf);
+    }
+    UNLOCK(&table->lock);
 }
 
-
 void *
-qr_content_extract (dict_t *xdata)
+qr_content_extract(dict_t *xdata)
 {
-	data_t  *data = NULL;
-	void    *content = NULL;
+    data_t *data = NULL;
+    void *content = NULL;
+    int ret = 0;
 
-	data = dict_get (xdata, GF_CONTENT_KEY);
-	if (!data)
-		return NULL;
+    ret = dict_get_with_ref(xdata, GF_CONTENT_KEY, &data);
+    if (ret < 0 || !data)
+        return NULL;
 
-	content = GF_CALLOC (1, data->len, gf_qr_mt_content_t);
-	if (!content)
-		return NULL;
+    content = GF_MALLOC(data->len, gf_qr_mt_content_t);
+    if (!content)
+        goto out;
 
-	memcpy (content, data->data, data->len);
+    memcpy(content, data->data, data->len);
 
-	return content;
+out:
+    data_unref(data);
+    return content;
 }
 
-
 void
-qr_content_update (xlator_t *this, qr_inode_t *qr_inode, void *data,
-		   struct iatt *buf)
+qr_content_update(xlator_t *this, qr_inode_t *qr_inode, void *data,
+                  struct iatt *buf, uint64_t gen)
 {
-        qr_private_t      *priv = NULL;
-        qr_inode_table_t  *table = NULL;
+    qr_private_t *priv = NULL;
+    qr_inode_table_t *table = NULL;
+    uint32_t rollover = 0;
 
-        priv = this->private;
-        table = &priv->table;
+    rollover = gen >> 32;
+    gen = gen & 0xffffffff;
 
-	LOCK (&table->lock);
-	{
-		__qr_inode_prune (table, qr_inode);
+    priv = this->private;
+    table = &priv->table;
 
-		qr_inode->data = data;
-		qr_inode->size = buf->ia_size;
+    LOCK(&table->lock);
+    {
+        if ((rollover != qr_inode->gen_rollover) ||
+            (gen && qr_inode->gen && (qr_inode->gen >= gen)))
+            goto unlock;
 
-		qr_inode->ia_mtime = buf->ia_mtime;
-		qr_inode->ia_mtime_nsec = buf->ia_mtime_nsec;
+        if ((qr_inode->data == NULL) && (qr_inode->invalidation_time >= gen))
+            goto unlock;
 
-		qr_inode->buf = *buf;
+        __qr_inode_prune(this, table, qr_inode, gen);
 
-		gettimeofday (&qr_inode->last_refresh, NULL);
+        qr_inode->data = data;
+        data = NULL;
+        qr_inode->size = buf->ia_size;
 
-		__qr_inode_register (table, qr_inode);
-	}
-	UNLOCK (&table->lock);
+        qr_inode->ia_mtime = buf->ia_mtime;
+        qr_inode->ia_mtime_nsec = buf->ia_mtime_nsec;
+        qr_inode->ia_ctime = buf->ia_ctime;
+        qr_inode->ia_ctime_nsec = buf->ia_ctime_nsec;
 
-	qr_cache_prune (this);
-}
+        qr_inode->buf = *buf;
+        qr_inode->last_refresh = gf_time();
+
+        __qr_inode_register(this, table, qr_inode);
+    }
+unlock:
+    UNLOCK(&table->lock);
+
+    if (data)
+        GF_FREE(data);
 
+    qr_cache_prune(this);
+}
 
 gf_boolean_t
-qr_size_fits (qr_conf_t *conf, struct iatt *buf)
+qr_size_fits(qr_conf_t *conf, struct iatt *buf)
 {
-	return (buf->ia_size <= conf->max_file_size);
+    return (buf->ia_size <= conf->max_file_size);
 }
 
+gf_boolean_t
+qr_mtime_equal(qr_inode_t *qr_inode, struct iatt *buf)
+{
+    return (qr_inode->ia_mtime == buf->ia_mtime &&
+            qr_inode->ia_mtime_nsec == buf->ia_mtime_nsec);
+}
 
 gf_boolean_t
-qr_mtime_equal (qr_inode_t *qr_inode, struct iatt *buf)
+qr_ctime_equal(qr_inode_t *qr_inode, struct iatt *buf)
 {
-	return (qr_inode->ia_mtime == buf->ia_mtime &&
-		qr_inode->ia_mtime_nsec == buf->ia_mtime_nsec);
+    return (qr_inode->ia_ctime == buf->ia_ctime &&
+            qr_inode->ia_ctime_nsec == buf->ia_ctime_nsec);
 }
 
+gf_boolean_t
+qr_time_equal(qr_conf_t *conf, qr_inode_t *qr_inode, struct iatt *buf)
+{
+    if (conf->ctime_invalidation)
+        return qr_ctime_equal(qr_inode, buf);
+    else
+        return qr_mtime_equal(qr_inode, buf);
+}
 
 void
-__qr_content_refresh (xlator_t *this, qr_inode_t *qr_inode, struct iatt *buf)
+__qr_content_refresh(xlator_t *this, qr_inode_t *qr_inode, struct iatt *buf,
+                     uint64_t gen)
 {
-        qr_private_t      *priv = NULL;
-        qr_inode_table_t  *table = NULL;
-	qr_conf_t         *conf = NULL;
-
-        priv = this->private;
-        table = &priv->table;
-	conf = &priv->conf;
-
-	if (qr_size_fits (conf, buf) && qr_mtime_equal (qr_inode, buf)) {
-		qr_inode->buf = *buf;
-
-		gettimeofday (&qr_inode->last_refresh, NULL);
-
-		__qr_inode_register (table, qr_inode);
-	} else {
-		__qr_inode_prune (table, qr_inode);
-	}
-
-	return;
+    qr_private_t *priv = NULL;
+    qr_inode_table_t *table = NULL;
+    qr_conf_t *conf = NULL;
+    uint32_t rollover = 0;
+
+    rollover = gen >> 32;
+    gen = gen & 0xffffffff;
+
+    priv = this->private;
+    table = &priv->table;
+    conf = &priv->conf;
+
+    /* allow for rollover of frame->root->unique */
+    if ((rollover != qr_inode->gen_rollover) ||
+        (gen && qr_inode->gen && (qr_inode->gen >= gen)))
+        goto done;
+
+    if ((qr_inode->data == NULL) && (qr_inode->invalidation_time >= gen))
+        goto done;
+
+    qr_inode->gen = gen;
+
+    if (qr_size_fits(conf, buf) && qr_time_equal(conf, qr_inode, buf)) {
+        qr_inode->buf = *buf;
+        qr_inode->last_refresh = gf_time();
+        __qr_inode_register(this, table, qr_inode);
+    } else {
+        __qr_inode_prune(this, table, qr_inode, gen);
+    }
+
+done:
+    return;
 }
 
-
 void
-qr_content_refresh (xlator_t *this, qr_inode_t *qr_inode, struct iatt *buf)
+qr_content_refresh(xlator_t *this, qr_inode_t *qr_inode, struct iatt *buf,
+                   uint64_t gen)
 {
-        qr_private_t      *priv = NULL;
-        qr_inode_table_t  *table = NULL;
+    qr_private_t *priv = NULL;
+    qr_inode_table_t *table = NULL;
 
-        priv = this->private;
-        table = &priv->table;
+    priv = this->private;
+    table = &priv->table;
 
-	LOCK (&table->lock);
-	{
-		__qr_content_refresh (this, qr_inode, buf);
-	}
-	UNLOCK (&table->lock);
+    LOCK(&table->lock);
+    {
+        __qr_content_refresh(this, qr_inode, buf, gen);
+    }
+    UNLOCK(&table->lock);
 }
 
-
 gf_boolean_t
-__qr_cache_is_fresh (xlator_t *this, qr_inode_t *qr_inode)
+__qr_cache_is_fresh(xlator_t *this, qr_inode_t *qr_inode)
 {
-	qr_conf_t        *conf = NULL;
-	qr_private_t     *priv = NULL;
-	struct timeval    now;
-	struct timeval    diff;
-
-	priv = this->private;
-	conf = &priv->conf;
+    qr_conf_t *conf = NULL;
+    qr_private_t *priv = NULL;
 
-	gettimeofday (&now, NULL);
+    priv = this->private;
+    conf = &priv->conf;
 
-	timersub (&now, &qr_inode->last_refresh, &diff);
+    if (qr_inode->last_refresh < priv->last_child_down)
+        return _gf_false;
 
-	if (diff.tv_sec >= conf->cache_timeout)
-		return _gf_false;
+    if (gf_time() - qr_inode->last_refresh >= conf->cache_timeout)
+        return _gf_false;
 
-	return _gf_true;
+    return _gf_true;
 }
 
-
 int
-qr_lookup_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-               int32_t op_ret, int32_t op_errno, inode_t *inode_ret,
-               struct iatt *buf, dict_t *xdata, struct iatt *postparent)
+qr_lookup_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int32_t op_ret,
+              int32_t op_errno, inode_t *inode_ret, struct iatt *buf,
+              dict_t *xdata, struct iatt *postparent)
 {
-        void             *content  = NULL;
-        qr_inode_t       *qr_inode = NULL;
-	inode_t          *inode    = NULL;
-
-	inode = frame->local;
-	frame->local = NULL;
-
-        if (op_ret == -1) {
-		qr_inode_prune (this, inode);
-                goto out;
-	}
-
-	if (dict_get (xdata, "sh-failed")) {
-		qr_inode_prune (this, inode);
-		goto out;
-	}
-
-	content = qr_content_extract (xdata);
-
-	if (content) {
-		/* new content came along, always replace old content */
-		qr_inode = qr_inode_ctx_get_or_new (this, inode);
-		if (!qr_inode) {
-			/* no harm done */
-			GF_FREE (content);
-			goto out;
-		}
-		qr_content_update (this, qr_inode, content, buf);
-	} else {
-		/* purge old content if necessary */
-		qr_inode = qr_inode_ctx_get (this, inode);
-		if (!qr_inode)
-			/* usual path for large files */
-			goto out;
-
-		qr_content_refresh (this, qr_inode, buf);
-	}
-out:
-	if (inode)
-		inode_unref (inode);
+    void *content = NULL;
+    qr_inode_t *qr_inode = NULL;
+    inode_t *inode = NULL;
+    qr_local_t *local = NULL;
+
+    local = frame->local;
+    inode = local->inode;
+
+    if (op_ret == -1) {
+        qr_inode_prune(this, inode, local->incident_gen);
+        goto out;
+    }
+
+    if (dict_get(xdata, GLUSTERFS_BAD_INODE)) {
+        qr_inode_prune(this, inode, local->incident_gen);
+        goto out;
+    }
+
+    if (dict_get(xdata, "sh-failed")) {
+        qr_inode_prune(this, inode, local->incident_gen);
+        goto out;
+    }
+
+    content = qr_content_extract(xdata);
+
+    if (content) {
+        /* new content came along, always replace old content */
+        qr_inode = qr_inode_ctx_get_or_new(this, inode);
+        if (!qr_inode) {
+            /* no harm done */
+            GF_FREE(content);
+            goto out;
+        }
 
-        STACK_UNWIND_STRICT (lookup, frame, op_ret, op_errno, inode_ret,
-			     buf, xdata, postparent);
-        return 0;
-}
+        qr_content_update(this, qr_inode, content, buf, local->incident_gen);
+    } else {
+        /* purge old content if necessary */
+        qr_inode = qr_inode_ctx_get(this, inode);
+        if (!qr_inode)
+            /* usual path for large files */
+            goto out;
 
+        qr_content_refresh(this, qr_inode, buf, local->incident_gen);
+    }
+out:
+    QR_STACK_UNWIND(lookup, frame, op_ret, op_errno, inode_ret, buf, xdata,
+                    postparent);
+    return 0;
+}
 
 int
-qr_lookup (call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xdata)
+qr_lookup(call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xdata)
 {
-        qr_private_t     *priv           = NULL;
-        qr_conf_t        *conf           = NULL;
-        qr_inode_t       *qr_inode       = NULL;
-	int               ret            = -1;
-	dict_t           *new_xdata      = NULL;
-
-        priv = this->private;
-        conf = &priv->conf;
-
-	qr_inode = qr_inode_ctx_get (this, loc->inode);
-	if (qr_inode && qr_inode->data)
-		/* cached. only validate in qr_lookup_cbk */
-		goto wind;
-
-	if (!xdata)
-		xdata = new_xdata = dict_new ();
-
-	if (!xdata)
-		goto wind;
-
-	ret = 0;
-	if (conf->max_file_size)
-		ret = dict_set (xdata, GF_CONTENT_KEY,
-				data_from_uint64 (conf->max_file_size));
-	if (ret)
-		gf_log (this->name, GF_LOG_WARNING,
-			"cannot set key in request dict (%s)",
-			loc->path);
+    qr_private_t *priv = NULL;
+    qr_conf_t *conf = NULL;
+    qr_inode_t *qr_inode = NULL;
+    int ret = -1;
+    dict_t *new_xdata = NULL;
+    qr_local_t *local = NULL;
+
+    priv = this->private;
+    conf = &priv->conf;
+    local = qr_local_get(this, loc->inode);
+    local->inode = inode_ref(loc->inode);
+    frame->local = local;
+
+    qr_inode = qr_inode_ctx_get(this, loc->inode);
+    if (qr_inode && qr_inode->data)
+        /* cached. only validate in qr_lookup_cbk */
+        goto wind;
+
+    if (!xdata)
+        xdata = new_xdata = dict_new();
+
+    if (!xdata)
+        goto wind;
+
+    ret = 0;
+    if (conf->max_file_size)
+        ret = dict_set(xdata, GF_CONTENT_KEY,
+                       data_from_uint64(conf->max_file_size));
+    if (ret)
+        gf_msg(this->name, GF_LOG_WARNING, 0, QUICK_READ_MSG_DICT_SET_FAILED,
+               "cannot set key in request dict (%s)", loc->path);
 wind:
-	frame->local = inode_ref (loc->inode);
-
-        STACK_WIND (frame, qr_lookup_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->lookup, loc, xdata);
+    STACK_WIND(frame, qr_lookup_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->lookup, loc, xdata);
 
-	if (new_xdata)
-		dict_unref (new_xdata);
+    if (new_xdata)
+        dict_unref(new_xdata);
 
-        return 0;
+    return 0;
 }
 
-
 int
-qr_readdirp_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-		 int op_ret, int op_errno, gf_dirent_t *entries, dict_t *xdata)
+qr_readdirp_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int op_ret,
+                int op_errno, gf_dirent_t *entries, dict_t *xdata)
 {
-        gf_dirent_t *entry      = NULL;
-	qr_inode_t  *qr_inode   = NULL;
+    gf_dirent_t *entry = NULL;
+    qr_inode_t *qr_inode = NULL;
+    qr_local_t *local = NULL;
 
-	if (op_ret <= 0)
-		goto unwind;
+    local = frame->local;
 
-        list_for_each_entry (entry, &entries->list, list) {
-                if (!entry->inode)
-			continue;
+    if (op_ret <= 0)
+        goto unwind;
 
-		qr_inode = qr_inode_ctx_get (this, entry->inode);
-		if (!qr_inode)
-			/* no harm */
-			continue;
+    list_for_each_entry(entry, &entries->list, list)
+    {
+        if (!entry->inode)
+            continue;
 
-		qr_content_refresh (this, qr_inode, &entry->d_stat);
-        }
+        qr_inode = qr_inode_ctx_get(this, entry->inode);
+        if (!qr_inode)
+            /* no harm */
+            continue;
+
+        qr_content_refresh(this, qr_inode, &entry->d_stat, local->incident_gen);
+    }
 
 unwind:
-	STACK_UNWIND_STRICT (readdirp, frame, op_ret, op_errno, entries, xdata);
-	return 0;
+    QR_STACK_UNWIND(readdirp, frame, op_ret, op_errno, entries, xdata);
+    return 0;
 }
 
-
 int
-qr_readdirp (call_frame_t *frame, xlator_t *this, fd_t *fd,
-	     size_t size, off_t offset, dict_t *xdata)
+qr_readdirp(call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
+            off_t offset, dict_t *xdata)
 {
-	STACK_WIND (frame, qr_readdirp_cbk,
-		    FIRST_CHILD (this), FIRST_CHILD (this)->fops->readdirp,
-		    fd, size, offset, xdata);
-	return 0;
-}
+    qr_local_t *local = NULL;
 
+    local = qr_local_get(this, NULL);
+    frame->local = local;
+
+    STACK_WIND(frame, qr_readdirp_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->readdirp, fd, size, offset, xdata);
+    return 0;
+}
 
 int
-qr_readv_cached (call_frame_t *frame, qr_inode_t *qr_inode, size_t size,
-		 off_t offset, uint32_t flags, dict_t *xdata)
+qr_readv_cached(call_frame_t *frame, qr_inode_t *qr_inode, size_t size,
+                off_t offset, uint32_t flags, dict_t *xdata)
 {
-	xlator_t         *this = NULL;
-	qr_private_t     *priv = NULL;
-	qr_inode_table_t *table = NULL;
-	int               op_ret = -1;
-	struct iobuf     *iobuf = NULL;
-	struct iobref    *iobref = NULL;
-	struct iovec      iov = {0, };
-	struct iatt       buf = {0, };
-
-	this = frame->this;
-	priv = this->private;
-	table = &priv->table;
+    xlator_t *this = NULL;
+    qr_private_t *priv = NULL;
+    qr_inode_table_t *table = NULL;
+    int op_ret = -1;
+    struct iobuf *iobuf = NULL;
+    struct iobref *iobref = NULL;
+    struct iovec iov = {
+        0,
+    };
+    struct iatt buf = {
+        0,
+    };
+
+    this = frame->this;
+    priv = this->private;
+    table = &priv->table;
+
+    LOCK(&table->lock);
+    {
+        if (!qr_inode->data)
+            goto unlock;
+
+        if (offset >= qr_inode->size)
+            goto unlock;
+
+        if (!__qr_cache_is_fresh(this, qr_inode))
+            goto unlock;
+
+        op_ret = min(size, (qr_inode->size - offset));
+
+        iobuf = iobuf_get2(this->ctx->iobuf_pool, op_ret);
+        if (!iobuf) {
+            op_ret = -1;
+            goto unlock;
+        }
 
-	LOCK (&table->lock);
-	{
-		if (!qr_inode->data)
-			goto unlock;
+        iobref = iobref_new();
+        if (!iobref) {
+            op_ret = -1;
+            goto unlock;
+        }
 
-		if (offset >= qr_inode->size)
-			goto unlock;
+        iobref_add(iobref, iobuf);
 
-		if (!__qr_cache_is_fresh (this, qr_inode))
-			goto unlock;
+        memcpy(iobuf->ptr, qr_inode->data + offset, op_ret);
 
-		op_ret = min (size, (qr_inode->size - offset));
+        buf = qr_inode->buf;
 
-		iobuf = iobuf_get2 (this->ctx->iobuf_pool, op_ret);
-		if (!iobuf) {
-			op_ret = -1;
-			goto unlock;
-		}
+        /* bump LRU */
+        __qr_inode_register(frame->this, table, qr_inode);
+    }
+unlock:
+    UNLOCK(&table->lock);
 
-		iobref = iobref_new ();
-		if (!iobref) {
-			op_ret = -1;
-			goto unlock;
-		}
+    if (op_ret >= 0) {
+        iov.iov_base = iobuf->ptr;
+        iov.iov_len = op_ret;
 
-		iobref_add (iobref, iobuf);
+        GF_ATOMIC_INC(priv->qr_counter.cache_hit);
+        STACK_UNWIND_STRICT(readv, frame, op_ret, 0, &iov, 1, &buf, iobref,
+                            xdata);
+    } else {
+        GF_ATOMIC_INC(priv->qr_counter.cache_miss);
+    }
 
-		memcpy (iobuf->ptr, qr_inode->data + offset, op_ret);
+    if (iobuf)
+        iobuf_unref(iobuf);
 
-		buf = qr_inode->buf;
+    if (iobref)
+        iobref_unref(iobref);
 
-		/* bump LRU */
-		__qr_inode_register (table, qr_inode);
-	}
-unlock:
-	UNLOCK (&table->lock);
+    return op_ret;
+}
 
-	if (op_ret >= 0) {
-		iov.iov_base = iobuf->ptr;
-		iov.iov_len = op_ret;
+int
+qr_readv(call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
+         off_t offset, uint32_t flags, dict_t *xdata)
+{
+    qr_inode_t *qr_inode = NULL;
 
-		STACK_UNWIND_STRICT (readv, frame, op_ret, 0, &iov, 1,
-				     &buf, iobref, xdata);
-	}
-	iobuf_unref (iobuf);
+    qr_inode = qr_inode_ctx_get(this, fd->inode);
+    if (!qr_inode)
+        goto wind;
 
-	iobref_unref (iobref);
+    if (qr_readv_cached(frame, qr_inode, size, offset, flags, xdata) < 0)
+        goto wind;
 
-	return op_ret;
+    return 0;
+wind:
+    STACK_WIND(frame, default_readv_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->readv, fd, size, offset, flags, xdata);
+    return 0;
 }
 
+int32_t
+qr_writev_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int32_t op_ret,
+              int32_t op_errno, struct iatt *prebuf, struct iatt *postbuf,
+              dict_t *xdata)
+{
+    qr_local_t *local = NULL;
+
+    local = frame->local;
+
+    qr_inode_prune(this, local->fd->inode, local->incident_gen);
+
+    QR_STACK_UNWIND(writev, frame, op_ret, op_errno, prebuf, postbuf, xdata);
+    return 0;
+}
 
 int
-qr_readv (call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
-          off_t offset, uint32_t flags, dict_t *xdata)
+qr_writev(call_frame_t *frame, xlator_t *this, fd_t *fd, struct iovec *iov,
+          int count, off_t offset, uint32_t flags, struct iobref *iobref,
+          dict_t *xdata)
 {
-	qr_inode_t *qr_inode = NULL;
+    qr_local_t *local = NULL;
 
-	qr_inode = qr_inode_ctx_get (this, fd->inode);
-	if (!qr_inode)
-		goto wind;
+    local = qr_local_get(this, fd->inode);
+    local->fd = fd_ref(fd);
 
-	if (qr_readv_cached (frame, qr_inode, size, offset, flags, xdata) <= 0)
-		goto wind;
+    frame->local = local;
 
-	return 0;
-wind:
-	STACK_WIND (frame, default_readv_cbk,
-		    FIRST_CHILD (this), FIRST_CHILD (this)->fops->readv,
-		    fd, size, offset, flags, xdata);
-	return 0;
+    STACK_WIND(frame, qr_writev_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->writev, fd, iov, count, offset, flags,
+               iobref, xdata);
+    return 0;
 }
 
+int32_t
+qr_truncate_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
+                struct iatt *postbuf, dict_t *xdata)
+{
+    qr_local_t *local = NULL;
+
+    local = frame->local;
+    qr_inode_prune(this, local->inode, local->incident_gen);
+
+    QR_STACK_UNWIND(truncate, frame, op_ret, op_errno, prebuf, postbuf, xdata);
+    return 0;
+}
 
 int
-qr_writev (call_frame_t *frame, xlator_t *this, fd_t *fd, struct iovec *iov,
-	   int count, off_t offset, uint32_t flags, struct iobref *iobref,
-	   dict_t *xdata)
+qr_truncate(call_frame_t *frame, xlator_t *this, loc_t *loc, off_t offset,
+            dict_t *xdata)
 {
-	qr_inode_prune (this, fd->inode);
+    qr_local_t *local = NULL;
 
-	STACK_WIND (frame, default_writev_cbk,
-		    FIRST_CHILD (this), FIRST_CHILD (this)->fops->writev,
-		    fd, iov, count, offset, flags, iobref, xdata);
-	return 0;
+    local = qr_local_get(this, loc->inode);
+    local->inode = inode_ref(loc->inode);
+    frame->local = local;
+
+    STACK_WIND(frame, qr_truncate_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->truncate, loc, offset, xdata);
+    return 0;
 }
 
+int32_t
+qr_ftruncate_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                 int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
+                 struct iatt *postbuf, dict_t *xdata)
+{
+    qr_local_t *local = NULL;
+
+    local = frame->local;
+    qr_inode_prune(this, local->fd->inode, local->incident_gen);
+
+    QR_STACK_UNWIND(ftruncate, frame, op_ret, op_errno, prebuf, postbuf, xdata);
+    return 0;
+}
 
 int
-qr_truncate (call_frame_t *frame, xlator_t *this, loc_t *loc, off_t offset,
-	     dict_t *xdata)
+qr_ftruncate(call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
+             dict_t *xdata)
 {
-	qr_inode_prune (this, loc->inode);
+    qr_local_t *local = NULL;
 
-	STACK_WIND (frame, default_truncate_cbk,
-		    FIRST_CHILD (this), FIRST_CHILD (this)->fops->truncate,
-		    loc, offset, xdata);
-	return 0;
+    local = qr_local_get(this, fd->inode);
+    local->fd = fd_ref(fd);
+    frame->local = local;
+
+    STACK_WIND(frame, qr_ftruncate_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->ftruncate, fd, offset, xdata);
+    return 0;
 }
 
+int32_t
+qr_fallocate_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                 int32_t op_ret, int32_t op_errno, struct iatt *pre,
+                 struct iatt *post, dict_t *xdata)
+{
+    qr_local_t *local = NULL;
 
-int
-qr_ftruncate (call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
-	      dict_t *xdata)
+    local = frame->local;
+    qr_inode_prune(this, local->fd->inode, local->incident_gen);
+
+    QR_STACK_UNWIND(fallocate, frame, op_ret, op_errno, pre, post, xdata);
+    return 0;
+}
+
+static int
+qr_fallocate(call_frame_t *frame, xlator_t *this, fd_t *fd, int keep_size,
+             off_t offset, size_t len, dict_t *xdata)
 {
-	qr_inode_prune (this, fd->inode);
+    qr_local_t *local = NULL;
+
+    local = qr_local_get(this, fd->inode);
+    local->fd = fd_ref(fd);
+    frame->local = local;
 
-	STACK_WIND (frame, default_ftruncate_cbk,
-		    FIRST_CHILD (this), FIRST_CHILD (this)->fops->ftruncate,
-		    fd, offset, xdata);
-	return 0;
+    STACK_WIND(frame, qr_fallocate_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->fallocate, fd, keep_size, offset, len,
+               xdata);
+    return 0;
 }
 
+int32_t
+qr_discard_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+               int32_t op_ret, int32_t op_errno, struct iatt *pre,
+               struct iatt *post, dict_t *xdata)
+{
+    qr_local_t *local = NULL;
+
+    local = frame->local;
+    qr_inode_prune(this, local->fd->inode, local->incident_gen);
 
-int
-qr_open (call_frame_t *frame, xlator_t *this, loc_t *loc, int flags,
-	 fd_t *fd, dict_t *xdata)
+    QR_STACK_UNWIND(discard, frame, op_ret, op_errno, pre, post, xdata);
+    return 0;
+}
+
+static int
+qr_discard(call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
+           size_t len, dict_t *xdata)
 {
-	qr_inode_set_priority (this, fd->inode, loc->path);
+    qr_local_t *local = NULL;
+
+    local = qr_local_get(this, fd->inode);
+    local->fd = fd_ref(fd);
+    frame->local = local;
 
-	STACK_WIND (frame, default_open_cbk,
-		    FIRST_CHILD (this), FIRST_CHILD (this)->fops->open,
-		    loc, flags, fd, xdata);
-	return 0;
+    STACK_WIND(frame, qr_discard_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->discard, fd, offset, len, xdata);
+    return 0;
 }
 
-int
-qr_forget (xlator_t *this, inode_t *inode)
+int32_t
+qr_zerofill_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                int32_t op_ret, int32_t op_errno, struct iatt *pre,
+                struct iatt *post, dict_t *xdata)
 {
-        qr_inode_t   *qr_inode = NULL;
+    qr_local_t *local = NULL;
 
-	qr_inode = qr_inode_ctx_get (this, inode);
+    local = frame->local;
+    qr_inode_prune(this, local->fd->inode, local->incident_gen);
 
-	if (!qr_inode)
-		return 0;
+    QR_STACK_UNWIND(zerofill, frame, op_ret, op_errno, pre, post, xdata);
+    return 0;
+}
 
-	qr_inode_prune (this, inode);
+static int
+qr_zerofill(call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
+            off_t len, dict_t *xdata)
+{
+    qr_local_t *local = NULL;
 
-	GF_FREE (qr_inode);
+    local = qr_local_get(this, fd->inode);
+    local->fd = fd_ref(fd);
+    frame->local = local;
 
-	return 0;
+    STACK_WIND(frame, qr_zerofill_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->zerofill, fd, offset, len, xdata);
+    return 0;
 }
 
+int
+qr_open(call_frame_t *frame, xlator_t *this, loc_t *loc, int flags, fd_t *fd,
+        dict_t *xdata)
+{
+    qr_inode_set_priority(this, fd->inode, loc->path);
 
-int32_t
-qr_inodectx_dump (xlator_t *this, inode_t *inode)
+    STACK_WIND(frame, default_open_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->open, loc, flags, fd, xdata);
+    return 0;
+}
+
+int
+qr_forget(xlator_t *this, inode_t *inode)
 {
-        qr_inode_t *qr_inode = NULL;
-        int32_t     ret      = -1;
-        char        key_prefix[GF_DUMP_MAX_BUF_LEN] = {0, };
-        char        buf[256]                        = {0, };
+    qr_inode_t *qr_inode = NULL;
 
-        qr_inode = qr_inode_ctx_get (this, inode);
-        if (!qr_inode)
-                goto out;
+    qr_inode = qr_inode_ctx_get(this, inode);
 
-        gf_proc_dump_build_key (key_prefix, "xlator.performance.quick-read",
-                                "inodectx");
-        gf_proc_dump_add_section (key_prefix);
+    if (!qr_inode)
+        return 0;
 
-        gf_proc_dump_write ("entire-file-cached", "%s", qr_inode->data ? "yes" : "no");
+    qr_inode_prune(this, inode, qr_get_generation(this, inode));
 
-        if (qr_inode->last_refresh.tv_sec) {
-                gf_time_fmt (buf, sizeof buf, qr_inode->last_refresh.tv_sec,
-                             gf_timefmt_FT);
-                snprintf (buf + strlen (buf), sizeof buf - strlen (buf),
-                          ".%"GF_PRI_SUSECONDS, qr_inode->last_refresh.tv_usec);
+    GF_FREE(qr_inode);
 
-                gf_proc_dump_write ("last-cache-validation-time", "%s", buf);
-        }
+    return 0;
+}
 
-        ret = 0;
+int32_t
+qr_inodectx_dump(xlator_t *this, inode_t *inode)
+{
+    qr_inode_t *qr_inode = NULL;
+    int32_t ret = -1;
+    char key_prefix[GF_DUMP_MAX_BUF_LEN] = {
+        0,
+    };
+    char buf[GF_TIMESTR_SIZE] = {
+        0,
+    };
+
+    qr_inode = qr_inode_ctx_get(this, inode);
+    if (!qr_inode)
+        goto out;
+
+    gf_proc_dump_build_key(key_prefix, "xlator.performance.quick-read",
+                           "inodectx");
+    gf_proc_dump_add_section("%s", key_prefix);
+
+    gf_proc_dump_write("entire-file-cached", "%s",
+                       qr_inode->data ? "yes" : "no");
+
+    if (qr_inode->last_refresh) {
+        gf_time_fmt(buf, sizeof buf, qr_inode->last_refresh, gf_timefmt_FT);
+        gf_proc_dump_write("last-cache-validation-time", "%s", buf);
+    }
+
+    ret = 0;
 out:
-        return ret;
+    return ret;
 }
 
-
 int
-qr_priv_dump (xlator_t *this)
+qr_priv_dump(xlator_t *this)
 {
-        qr_conf_t        *conf       = NULL;
-        qr_private_t     *priv       = NULL;
-        qr_inode_table_t *table      = NULL;
-        uint32_t          file_count = 0;
-        uint32_t          i          = 0;
-        qr_inode_t       *curr       = NULL;
-        uint64_t          total_size = 0;
-        char              key_prefix[GF_DUMP_MAX_BUF_LEN];
-
-        if (!this) {
-                return -1;
-        }
+    qr_conf_t *conf = NULL;
+    qr_private_t *priv = NULL;
+    qr_inode_table_t *table = NULL;
+    uint32_t file_count = 0;
+    uint32_t i = 0;
+    qr_inode_t *curr = NULL;
+    uint64_t total_size = 0;
+    char key_prefix[GF_DUMP_MAX_BUF_LEN];
 
-        priv = this->private;
-        conf = &priv->conf;
+    if (!this) {
+        return -1;
+    }
 
-        if (!conf)
-                return -1;
+    priv = this->private;
+    conf = &priv->conf;
+    if (!conf)
+        return -1;
 
-        table = &priv->table;
+    table = &priv->table;
 
-        gf_proc_dump_build_key (key_prefix, "xlator.performance.quick-read",
-                                "priv");
+    gf_proc_dump_build_key(key_prefix, "xlator.performance.quick-read", "priv");
 
-        gf_proc_dump_add_section (key_prefix);
+    gf_proc_dump_add_section("%s", key_prefix);
 
-        gf_proc_dump_write ("max_file_size", "%d", conf->max_file_size);
-        gf_proc_dump_write ("cache_timeout", "%d", conf->cache_timeout);
+    gf_proc_dump_write("max_file_size", "%" PRIu64, conf->max_file_size);
+    gf_proc_dump_write("cache_timeout", "%d", conf->cache_timeout);
 
-        if (!table) {
-                goto out;
-        } else {
-                for (i = 0; i < conf->max_pri; i++) {
-                        list_for_each_entry (curr, &table->lru[i], lru) {
-                                file_count++;
-                                total_size += curr->size;
-                        }
-                }
+    if (!table) {
+        goto out;
+    } else {
+        for (i = 0; i < conf->max_pri; i++) {
+            list_for_each_entry(curr, &table->lru[i], lru)
+            {
+                file_count++;
+                total_size += curr->size;
+            }
         }
+    }
 
-        gf_proc_dump_write ("total_files_cached", "%d", file_count);
-        gf_proc_dump_write ("total_cache_used", "%d", total_size);
+    gf_proc_dump_write("total_files_cached", "%d", file_count);
+    gf_proc_dump_write("total_cache_used", "%" PRIu64, total_size);
+    gf_proc_dump_write("cache-hit", "%" GF_PRI_ATOMIC,
+                       GF_ATOMIC_GET(priv->qr_counter.cache_hit));
+    gf_proc_dump_write("cache-miss", "%" GF_PRI_ATOMIC,
+                       GF_ATOMIC_GET(priv->qr_counter.cache_miss));
+    gf_proc_dump_write("cache-invalidations", "%" GF_PRI_ATOMIC,
+                       GF_ATOMIC_GET(priv->qr_counter.file_data_invals));
 
 out:
-        return 0;
+    return 0;
 }
 
+static int32_t
+qr_dump_metrics(xlator_t *this, int fd)
+{
+    qr_private_t *priv = NULL;
+    qr_inode_table_t *table = NULL;
+
+    priv = this->private;
+    table = &priv->table;
+
+    dprintf(fd, "%s.total_files_cached %" PRId64 "\n", this->name,
+            GF_ATOMIC_GET(priv->qr_counter.files_cached));
+    dprintf(fd, "%s.total_cache_used %" PRId64 "\n", this->name,
+            table->cache_used);
+    dprintf(fd, "%s.cache-hit %" PRId64 "\n", this->name,
+            GF_ATOMIC_GET(priv->qr_counter.cache_hit));
+    dprintf(fd, "%s.cache-miss %" PRId64 "\n", this->name,
+            GF_ATOMIC_GET(priv->qr_counter.cache_miss));
+    dprintf(fd, "%s.cache-invalidations %" PRId64 "\n", this->name,
+            GF_ATOMIC_GET(priv->qr_counter.file_data_invals));
+
+    return 0;
+}
 
 int32_t
-mem_acct_init (xlator_t *this)
+qr_mem_acct_init(xlator_t *this)
 {
-        int     ret = -1;
+    int ret = -1;
 
-        if (!this)
-                return ret;
-
-        ret = xlator_mem_acct_init (this, gf_qr_mt_end + 1);
+    if (!this)
+        return ret;
 
-        if (ret != 0) {
-                gf_log (this->name, GF_LOG_ERROR, "Memory accounting init"
-                        "failed");
-                return ret;
-        }
+    ret = xlator_mem_acct_init(this, gf_qr_mt_end + 1);
 
+    if (ret != 0) {
+        gf_msg(this->name, GF_LOG_ERROR, ENOMEM, QUICK_READ_MSG_NO_MEMORY,
+               "Memory accounting init failed");
         return ret;
-}
+    }
 
+    return ret;
+}
 
 static gf_boolean_t
-check_cache_size_ok (xlator_t *this, int64_t cache_size)
+check_cache_size_ok(xlator_t *this, int64_t cache_size)
 {
-        int                     ret = _gf_true;
-        uint64_t                total_mem = 0;
-        uint64_t                max_cache_size = 0;
-        volume_option_t         *opt = NULL;
-
-        GF_ASSERT (this);
-        opt = xlator_volume_option_get (this, "cache-size");
-        if (!opt) {
-                ret = _gf_false;
-                gf_log (this->name, GF_LOG_ERROR,
-                        "could not get cache-size option");
-                goto out;
-        }
-
-        total_mem = get_mem_size ();
-        if (-1 == total_mem)
-                max_cache_size = opt->max;
-        else
-                max_cache_size = total_mem;
-
-        gf_log (this->name, GF_LOG_DEBUG, "Max cache size is %"PRIu64,
-                max_cache_size);
-        if (cache_size > max_cache_size) {
-                ret = _gf_false;
-                gf_log (this->name, GF_LOG_ERROR, "Cache size %"PRIu64
-                        " is greater than the max size of %"PRIu64,
-                        cache_size, max_cache_size);
-                goto out;
-        }
+    int ret = _gf_true;
+    uint64_t total_mem = 0;
+    uint64_t max_cache_size = 0;
+    volume_option_t *opt = NULL;
+
+    GF_ASSERT(this);
+    opt = xlator_volume_option_get(this, "cache-size");
+    if (!opt) {
+        ret = _gf_false;
+        gf_msg(this->name, GF_LOG_ERROR, EINVAL,
+               QUICK_READ_MSG_INVALID_ARGUMENT,
+               "could not get cache-size option");
+        goto out;
+    }
+
+    total_mem = get_mem_size();
+    if (-1 == total_mem)
+        max_cache_size = opt->max;
+    else
+        max_cache_size = total_mem;
+
+    gf_msg_debug(this->name, 0, "Max cache size is %" PRIu64, max_cache_size);
+    if (cache_size > max_cache_size) {
+        ret = _gf_false;
+        gf_msg(this->name, GF_LOG_ERROR, 0, QUICK_READ_MSG_INVALID_ARGUMENT,
+               "Cache size %" PRIu64
+               " is greater than the max size of %" PRIu64,
+               cache_size, max_cache_size);
+        goto out;
+    }
 out:
-        return ret;
+    return ret;
 }
 
 int
-reconfigure (xlator_t *this, dict_t *options)
+qr_reconfigure(xlator_t *this, dict_t *options)
 {
-        int32_t       ret            = -1;
-        qr_private_t *priv           = NULL;
-        qr_conf_t    *conf           = NULL;
-        uint64_t       cache_size_new = 0;
+    int32_t ret = -1;
+    qr_private_t *priv = NULL;
+    qr_conf_t *conf = NULL;
+    uint64_t cache_size_new = 0;
 
-        GF_VALIDATE_OR_GOTO ("quick-read", this, out);
-        GF_VALIDATE_OR_GOTO (this->name, this->private, out);
-        GF_VALIDATE_OR_GOTO (this->name, options, out);
+    GF_VALIDATE_OR_GOTO("quick-read", this, out);
+    GF_VALIDATE_OR_GOTO(this->name, this->private, out);
+    GF_VALIDATE_OR_GOTO(this->name, options, out);
 
-        priv = this->private;
+    priv = this->private;
 
-        conf = &priv->conf;
-        if (!conf) {
-                goto out;
-        }
+    conf = &priv->conf;
+    if (!conf) {
+        goto out;
+    }
 
-        GF_OPTION_RECONF ("cache-timeout", conf->cache_timeout, options, int32,
-                          out);
+    GF_OPTION_RECONF("cache-timeout", conf->cache_timeout, options, int32, out);
 
-        GF_OPTION_RECONF ("cache-size", cache_size_new, options, size_uint64, out);
-        if (!check_cache_size_ok (this, cache_size_new)) {
-                ret = -1;
-                gf_log (this->name, GF_LOG_ERROR,
-                        "Not reconfiguring cache-size");
-                goto out;
-        }
-        conf->cache_size = cache_size_new;
+    GF_OPTION_RECONF("quick-read-cache-invalidation", conf->qr_invalidation,
+                     options, bool, out);
+
+    GF_OPTION_RECONF("ctime-invalidation", conf->ctime_invalidation, options,
+                     bool, out);
 
-        ret = 0;
+    GF_OPTION_RECONF("cache-size", cache_size_new, options, size_uint64, out);
+    if (!check_cache_size_ok(this, cache_size_new)) {
+        ret = -1;
+        gf_msg(this->name, GF_LOG_ERROR, EINVAL, QUICK_READ_MSG_INVALID_CONFIG,
+               "Not reconfiguring cache-size");
+        goto out;
+    }
+    conf->cache_size = cache_size_new;
+
+    ret = 0;
 out:
-        return ret;
+    return ret;
 }
 
-
 int32_t
-qr_get_priority_list (const char *opt_str, struct list_head *first)
+qr_get_priority_list(const char *opt_str, struct list_head *first)
 {
-        int32_t              max_pri      = 1;
-        char                *tmp_str      = NULL;
-        char                *tmp_str1     = NULL;
-        char                *tmp_str2     = NULL;
-        char                *dup_str      = NULL;
-        char                *priority_str = NULL;
-        char                *pattern      = NULL;
-        char                *priority     = NULL;
-        char                *string       = NULL;
-        struct qr_priority  *curr         = NULL, *tmp = NULL;
-
-        GF_VALIDATE_OR_GOTO ("quick-read", opt_str, out);
-        GF_VALIDATE_OR_GOTO ("quick-read", first, out);
-
-        string = gf_strdup (opt_str);
-        if (string == NULL) {
-                max_pri = -1;
-                goto out;
+    int32_t max_pri = 1;
+    char *tmp_str = NULL;
+    char *tmp_str1 = NULL;
+    char *tmp_str2 = NULL;
+    char *dup_str = NULL;
+    char *priority_str = NULL;
+    char *pattern = NULL;
+    char *priority = NULL;
+    char *string = NULL;
+    struct qr_priority *curr = NULL, *tmp = NULL;
+
+    GF_VALIDATE_OR_GOTO("quick-read", opt_str, out);
+    GF_VALIDATE_OR_GOTO("quick-read", first, out);
+
+    string = gf_strdup(opt_str);
+    if (string == NULL) {
+        max_pri = -1;
+        goto out;
+    }
+
+    /* Get the pattern for cache priority.
+     * "option priority *.jpg:1,abc*:2" etc
+     */
+    /* TODO: inode_lru in table is statically hard-coded to 5,
+     * should be changed to run-time configuration
+     */
+    priority_str = strtok_r(string, ",", &tmp_str);
+    while (priority_str) {
+        curr = GF_CALLOC(1, sizeof(*curr), gf_qr_mt_qr_priority_t);
+        if (curr == NULL) {
+            max_pri = -1;
+            goto out;
         }
 
-        /* Get the pattern for cache priority.
-         * "option priority *.jpg:1,abc*:2" etc
-         */
-        /* TODO: inode_lru in table is statically hard-coded to 5,
-         * should be changed to run-time configuration
-         */
-        priority_str = strtok_r (string, ",", &tmp_str);
-        while (priority_str) {
-                curr = GF_CALLOC (1, sizeof (*curr), gf_qr_mt_qr_priority_t);
-                if (curr == NULL) {
-                        max_pri = -1;
-                        goto out;
-                }
-
-                list_add_tail (&curr->list, first);
-
-                dup_str = gf_strdup (priority_str);
-                if (dup_str == NULL) {
-                        max_pri = -1;
-                        goto out;
-                }
-
-                pattern = strtok_r (dup_str, ":", &tmp_str1);
-                if (!pattern) {
-                        max_pri = -1;
-                        goto out;
-                }
-
-                priority = strtok_r (NULL, ":", &tmp_str1);
-                if (!priority) {
-                        max_pri = -1;
-                        goto out;
-                }
-
-                gf_log ("quick-read", GF_LOG_TRACE,
-                        "quick-read priority : pattern %s : priority %s",
-                        pattern,
-                        priority);
-
-                curr->pattern = gf_strdup (pattern);
-                if (curr->pattern == NULL) {
-                        max_pri = -1;
-                        goto out;
-                }
-
-                curr->priority = strtol (priority, &tmp_str2, 0);
-                if (tmp_str2 && (*tmp_str2)) {
-                        max_pri = -1;
-                        goto out;
-                } else {
-                        max_pri = max (max_pri, curr->priority);
-                }
-
-                GF_FREE (dup_str);
-                dup_str = NULL;
-
-                priority_str = strtok_r (NULL, ",", &tmp_str);
-        }
-out:
-        GF_FREE (string);
+        list_add_tail(&curr->list, first);
 
-        GF_FREE (dup_str);
-
-        if (max_pri == -1) {
-                list_for_each_entry_safe (curr, tmp, first, list) {
-                        list_del_init (&curr->list);
-                        GF_FREE (curr->pattern);
-                        GF_FREE (curr);
-                }
+        dup_str = gf_strdup(priority_str);
+        if (dup_str == NULL) {
+            max_pri = -1;
+            goto out;
         }
 
-        return max_pri;
-}
-
+        pattern = strtok_r(dup_str, ":", &tmp_str1);
+        if (!pattern) {
+            max_pri = -1;
+            goto out;
+        }
 
-int32_t
-init (xlator_t *this)
-{
-        int32_t       ret  = -1, i = 0;
-        qr_private_t *priv = NULL;
-        qr_conf_t    *conf = NULL;
-
-        if (!this->children || this->children->next) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "FATAL: volume (%s) not configured with exactly one "
-                        "child", this->name);
-                return -1;
+        priority = strtok_r(NULL, ":", &tmp_str1);
+        if (!priority) {
+            max_pri = -1;
+            goto out;
         }
 
-        if (!this->parents) {
-                gf_log (this->name, GF_LOG_WARNING,
-                        "dangling volume. check volfile ");
+        gf_msg_trace("quick-read", 0,
+                     "quick-read priority : pattern %s : priority %s", pattern,
+                     priority);
+
+        curr->pattern = gf_strdup(pattern);
+        if (curr->pattern == NULL) {
+            max_pri = -1;
+            goto out;
         }
 
-        priv = GF_CALLOC (1, sizeof (*priv), gf_qr_mt_qr_private_t);
-        if (priv == NULL) {
-                ret = -1;
-                goto out;
+        curr->priority = strtol(priority, &tmp_str2, 0);
+        if (tmp_str2 && (*tmp_str2)) {
+            max_pri = -1;
+            goto out;
+        } else {
+            max_pri = max(max_pri, curr->priority);
         }
 
-        LOCK_INIT (&priv->table.lock);
-        conf = &priv->conf;
+        GF_FREE(dup_str);
+        dup_str = NULL;
 
-        GF_OPTION_INIT ("max-file-size", conf->max_file_size, size_uint64, out);
+        priority_str = strtok_r(NULL, ",", &tmp_str);
+    }
+out:
+    GF_FREE(string);
 
-        GF_OPTION_INIT ("cache-timeout", conf->cache_timeout, int32, out);
+    GF_FREE(dup_str);
 
-        GF_OPTION_INIT ("cache-size", conf->cache_size, size_uint64, out);
-        if (!check_cache_size_ok (this, conf->cache_size)) {
-                ret = -1;
-                goto out;
+    if (max_pri == -1) {
+        list_for_each_entry_safe(curr, tmp, first, list)
+        {
+            list_del_init(&curr->list);
+            GF_FREE(curr->pattern);
+            GF_FREE(curr);
         }
+    }
 
-        INIT_LIST_HEAD (&conf->priority_list);
-        conf->max_pri = 1;
-        if (dict_get (this->options, "priority")) {
-                char *option_list = data_to_str (dict_get (this->options,
-                                                           "priority"));
-                gf_log (this->name, GF_LOG_TRACE,
-                        "option path %s", option_list);
-                /* parse the list of pattern:priority */
-                conf->max_pri = qr_get_priority_list (option_list,
-                                                      &conf->priority_list);
-
-                if (conf->max_pri == -1) {
-                        goto out;
-                }
-                conf->max_pri ++;
-        }
+    return max_pri;
+}
 
-        priv->table.lru = GF_CALLOC (conf->max_pri, sizeof (*priv->table.lru),
-                                     gf_common_mt_list_head);
-        if (priv->table.lru == NULL) {
-                ret = -1;
-                goto out;
+int32_t
+qr_init(xlator_t *this)
+{
+    int32_t ret = -1, i = 0;
+    qr_private_t *priv = NULL;
+    qr_conf_t *conf = NULL;
+
+    if (!this->children || this->children->next) {
+        gf_msg(this->name, GF_LOG_ERROR, 0,
+               QUICK_READ_MSG_XLATOR_CHILD_MISCONFIGURED,
+               "FATAL: volume (%s) not configured with exactly one "
+               "child",
+               this->name);
+        return -1;
+    }
+
+    if (!this->parents) {
+        gf_msg(this->name, GF_LOG_WARNING, 0, QUICK_READ_MSG_VOL_MISCONFIGURED,
+               "dangling volume. check volfile ");
+    }
+
+    priv = GF_CALLOC(1, sizeof(*priv), gf_qr_mt_qr_private_t);
+    if (priv == NULL) {
+        ret = -1;
+        goto out;
+    }
+
+    LOCK_INIT(&priv->table.lock);
+    conf = &priv->conf;
+
+    GF_OPTION_INIT("max-file-size", conf->max_file_size, size_uint64, out);
+
+    GF_OPTION_INIT("cache-timeout", conf->cache_timeout, int32, out);
+
+    GF_OPTION_INIT("quick-read-cache-invalidation", conf->qr_invalidation, bool,
+                   out);
+
+    GF_OPTION_INIT("cache-size", conf->cache_size, size_uint64, out);
+    if (!check_cache_size_ok(this, conf->cache_size)) {
+        ret = -1;
+        goto out;
+    }
+
+    GF_OPTION_INIT("ctime-invalidation", conf->ctime_invalidation, bool, out);
+
+    INIT_LIST_HEAD(&conf->priority_list);
+    conf->max_pri = 1;
+    if (dict_get(this->options, "priority")) {
+        char *option_list = data_to_str(dict_get(this->options, "priority"));
+        gf_msg_trace(this->name, 0, "option path %s", option_list);
+        /* parse the list of pattern:priority */
+        conf->max_pri = qr_get_priority_list(option_list, &conf->priority_list);
+
+        if (conf->max_pri == -1) {
+            goto out;
         }
+        conf->max_pri++;
+    }
 
-        for (i = 0; i < conf->max_pri; i++) {
-                INIT_LIST_HEAD (&priv->table.lru[i]);
-        }
+    priv->table.lru = GF_CALLOC(conf->max_pri, sizeof(*priv->table.lru),
+                                gf_common_mt_list_head);
+    if (priv->table.lru == NULL) {
+        ret = -1;
+        goto out;
+    }
 
-        ret = 0;
+    for (i = 0; i < conf->max_pri; i++) {
+        INIT_LIST_HEAD(&priv->table.lru[i]);
+    }
 
-        this->private = priv;
+    ret = 0;
+
+    priv->last_child_down = gf_time();
+    GF_ATOMIC_INIT(priv->generation, 0);
+    this->private = priv;
 out:
-        if ((ret == -1) && priv) {
-                GF_FREE (priv);
-        }
+    if ((ret == -1) && priv) {
+        GF_FREE(priv);
+    }
 
-        return ret;
+    return ret;
 }
 
-
 void
-qr_inode_table_destroy (qr_private_t *priv)
+qr_inode_table_destroy(qr_private_t *priv)
 {
-        int        i    = 0;
-        qr_conf_t *conf = NULL;
-
-        conf = &priv->conf;
-
-        for (i = 0; i < conf->max_pri; i++) {
-                /* There is a known leak of inodes, hence until
-                 * that is fixed, log the assert as warning.
-                GF_ASSERT (list_empty (&priv->table.lru[i]));*/
-                if (!list_empty (&priv->table.lru[i])) {
-                        gf_log ("quick-read", GF_LOG_INFO,
-                               "quick read inode table lru not empty");
-                }
+    int i = 0;
+    qr_conf_t *conf = NULL;
+
+    conf = &priv->conf;
+
+    for (i = 0; i < conf->max_pri; i++) {
+        /* There is a known leak of inodes, hence until
+         * that is fixed, log the assert as warning.
+        GF_ASSERT (list_empty (&priv->table.lru[i]));*/
+        if (!list_empty(&priv->table.lru[i])) {
+            gf_msg("quick-read", GF_LOG_INFO, 0, QUICK_READ_MSG_LRU_NOT_EMPTY,
+                   "quick read inode table lru not empty");
         }
+    }
 
-        LOCK_DESTROY (&priv->table.lock);
+    LOCK_DESTROY(&priv->table.lock);
 
-        return;
+    return;
 }
 
+void
+qr_conf_destroy(qr_conf_t *conf)
+{
+    struct qr_priority *curr = NULL, *tmp = NULL;
+
+    list_for_each_entry_safe(curr, tmp, &conf->priority_list, list)
+    {
+        list_del(&curr->list);
+        GF_FREE(curr->pattern);
+        GF_FREE(curr);
+    }
+
+    return;
+}
 
 void
-qr_conf_destroy (qr_conf_t *conf)
+qr_update_child_down_time(xlator_t *this, time_t now)
 {
-        struct qr_priority *curr = NULL, *tmp = NULL;
+    qr_private_t *priv = NULL;
 
-        list_for_each_entry_safe (curr, tmp, &conf->priority_list, list) {
-                list_del (&curr->list);
-                GF_FREE (curr->pattern);
-                GF_FREE (curr);
+    priv = this->private;
+
+    LOCK(&priv->lock);
+    {
+        priv->last_child_down = now;
+    }
+    UNLOCK(&priv->lock);
+}
+
+static int
+qr_invalidate(xlator_t *this, void *data)
+{
+    struct gf_upcall *up_data = NULL;
+    struct gf_upcall_cache_invalidation *up_ci = NULL;
+    inode_t *inode = NULL;
+    int ret = 0;
+    inode_table_t *itable = NULL;
+    qr_private_t *priv = NULL;
+
+    up_data = (struct gf_upcall *)data;
+
+    if (up_data->event_type != GF_UPCALL_CACHE_INVALIDATION)
+        goto out;
+
+    priv = this->private;
+    up_ci = (struct gf_upcall_cache_invalidation *)up_data->data;
+
+    if (up_ci && (up_ci->flags & UP_WRITE_FLAGS)) {
+        GF_ATOMIC_INC(priv->qr_counter.file_data_invals);
+        itable = ((xlator_t *)this->graph->top)->itable;
+        inode = inode_find(itable, up_data->gfid);
+        if (!inode) {
+            ret = -1;
+            goto out;
         }
+        qr_inode_prune(this, inode, qr_get_generation(this, inode));
+    }
 
-        return;
+out:
+    if (inode)
+        inode_unref(inode);
+
+    return ret;
 }
 
+int
+qr_notify(xlator_t *this, int event, void *data, ...)
+{
+    int ret = 0;
+    qr_private_t *priv = NULL;
+    qr_conf_t *conf = NULL;
+
+    priv = this->private;
+    conf = &priv->conf;
+
+    switch (event) {
+        case GF_EVENT_CHILD_DOWN:
+        case GF_EVENT_SOME_DESCENDENT_DOWN:
+            qr_update_child_down_time(this, gf_time());
+            break;
+        case GF_EVENT_UPCALL:
+            if (conf->qr_invalidation)
+                ret = qr_invalidate(this, data);
+            break;
+        default:
+            break;
+    }
+
+    if (default_notify(this, event, data) != 0)
+        ret = -1;
+
+    return ret;
+}
 
 void
-fini (xlator_t *this)
+qr_fini(xlator_t *this)
 {
-        qr_private_t *priv = NULL;
+    qr_private_t *priv = NULL;
 
-        if (this == NULL) {
-                goto out;
-        }
+    if (this == NULL) {
+        goto out;
+    }
 
-        priv = this->private;
-        if (priv == NULL) {
-                goto out;
-        }
+    priv = this->private;
+    if (priv == NULL) {
+        goto out;
+    }
 
-        qr_inode_table_destroy (priv);
-        qr_conf_destroy (&priv->conf);
+    qr_inode_table_destroy(priv);
+    qr_conf_destroy(&priv->conf);
 
-        this->private = NULL;
+    this->private = NULL;
 
-        GF_FREE (priv);
+    GF_FREE(priv);
 out:
-        return;
+    return;
 }
 
-struct xlator_fops fops = {
-        .lookup      = qr_lookup,
-	.readdirp    = qr_readdirp,
-        .open        = qr_open,
-        .readv       = qr_readv,
-	.writev      = qr_writev,
-	.truncate    = qr_truncate,
-	.ftruncate   = qr_ftruncate
-};
-
-struct xlator_cbks cbks = {
-        .forget  = qr_forget,
+struct xlator_fops qr_fops = {.lookup = qr_lookup,
+                              .readdirp = qr_readdirp,
+                              .open = qr_open,
+                              .readv = qr_readv,
+                              .writev = qr_writev,
+                              .truncate = qr_truncate,
+                              .ftruncate = qr_ftruncate,
+                              .fallocate = qr_fallocate,
+                              .discard = qr_discard,
+                              .zerofill = qr_zerofill};
+
+struct xlator_cbks qr_cbks = {
+    .forget = qr_forget,
 };
 
-struct xlator_dumpops dumpops = {
-        .priv      =  qr_priv_dump,
-        .inodectx  =  qr_inodectx_dump,
+struct xlator_dumpops qr_dumpops = {
+    .priv = qr_priv_dump,
+    .inodectx = qr_inodectx_dump,
 };
 
-struct volume_options options[] = {
-        { .key  = {"priority"},
-          .type = GF_OPTION_TYPE_ANY
-        },
-        { .key  = {"cache-size"},
-          .type = GF_OPTION_TYPE_SIZET,
-          .min  = 0,
-          .max  = 32 * GF_UNIT_GB,
-          .default_value = "128MB",
-          .description = "Size of the read cache."
-        },
-        { .key  = {"cache-timeout"},
-          .type = GF_OPTION_TYPE_INT,
-          .min = 1,
-          .max = 60,
-          .default_value = "1",
-        },
-        { .key  = {"max-file-size"},
-          .type = GF_OPTION_TYPE_SIZET,
-          .min  = 0,
-          .max  = 1 * GF_UNIT_KB * 1000,
-          .default_value = "64KB",
-        },
-        { .key  = {NULL} }
+struct volume_options qr_options[] = {
+    {
+        .key = {"quick-read"},
+        .type = GF_OPTION_TYPE_BOOL,
+        .default_value = "off",
+        .description = "enable/disable quick-read",
+        .op_version = {GD_OP_VERSION_6_0},
+        .flags = OPT_FLAG_SETTABLE,
+    },
+    {.key = {"priority"}, .type = GF_OPTION_TYPE_ANY},
+    {.key = {"cache-size"},
+     .type = GF_OPTION_TYPE_SIZET,
+     .min = 0,
+     .max = INFINITY,
+     .default_value = "128MB",
+     .op_version = {1},
+     .flags = OPT_FLAG_CLIENT_OPT | OPT_FLAG_SETTABLE | OPT_FLAG_DOC,
+     .description = "Size of small file read cache."},
+    {
+        .key = {"cache-timeout"},
+        .type = GF_OPTION_TYPE_INT,
+        .default_value = "1",
+        .op_version = {1},
+        .flags = OPT_FLAG_CLIENT_OPT | OPT_FLAG_SETTABLE | OPT_FLAG_DOC,
+    },
+    {
+        .key = {"max-file-size"},
+        .type = GF_OPTION_TYPE_SIZET,
+        .min = 0,
+        .max = 1 * GF_UNIT_KB * 1000,
+        .default_value = "64KB",
+        .op_version = {1},
+        .flags = OPT_FLAG_CLIENT_OPT | OPT_FLAG_SETTABLE | OPT_FLAG_DOC,
+    },
+    {
+        .key = {"quick-read-cache-invalidation"},
+        .type = GF_OPTION_TYPE_BOOL,
+        .default_value = "false",
+        .op_version = {GD_OP_VERSION_4_0_0},
+        .flags = OPT_FLAG_CLIENT_OPT | OPT_FLAG_SETTABLE | OPT_FLAG_DOC,
+        .description = "When \"on\", invalidates/updates the metadata cache,"
+                       " on receiving the cache-invalidation notifications",
+    },
+    {
+        .key = {"ctime-invalidation"},
+        .type = GF_OPTION_TYPE_BOOL,
+        .default_value = "false",
+        .op_version = {GD_OP_VERSION_5_0},
+        .flags = OPT_FLAG_CLIENT_OPT | OPT_FLAG_SETTABLE | OPT_FLAG_DOC,
+        .description = "Quick-read by default uses mtime to identify changes "
+                       "to file data. However there are applications like "
+                       "rsync which explicitly set mtime making it unreliable "
+                       "for the purpose of identifying change in file content "
+                       ". Since ctime also changes when content of a file "
+                       " changes and it cannot be set explicitly, it becomes "
+                       " suitable for identifying staleness of cached data. "
+                       "This option makes quick-read to prefer ctime over "
+                       "mtime to validate its cache. However, using ctime "
+                       "can result in false positives as ctime changes with "
+                       "just attribute changes like permission without "
+                       "changes to file data. So, use this only when mtime "
+                       "is not reliable",
+    },
+    {.key = {NULL}}};
+
+xlator_api_t xlator_api = {
+    .init = qr_init,
+    .fini = qr_fini,
+    .notify = qr_notify,
+    .reconfigure = qr_reconfigure,
+    .mem_acct_init = qr_mem_acct_init,
+    .dump_metrics = qr_dump_metrics,
+    .op_version = {1}, /* Present from the initial version */
+    .dumpops = &qr_dumpops,
+    .fops = &qr_fops,
+    .cbks = &qr_cbks,
+    .options = qr_options,
+    .identifier = "quick-read",
+    .category = GF_MAINTAINED,
 };
diff --git a/xlators/performance/quick-read/src/quick-read.h b/xlators/performance/quick-read/src/quick-read.h
index 6f0a0541731..20fcc70b3a7 100644
--- a/xlators/performance/quick-read/src/quick-read.h
+++ b/xlators/performance/quick-read/src/quick-read.h
@@ -11,21 +11,16 @@
 #ifndef __QUICK_READ_H
 #define __QUICK_READ_H
 
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
-
-#include "glusterfs.h"
-#include "logging.h"
-#include "dict.h"
-#include "xlator.h"
-#include "list.h"
-#include "compat.h"
-#include "compat-errno.h"
-#include "common-utils.h"
-#include "call-stub.h"
-#include "defaults.h"
+#include <glusterfs/glusterfs.h>
+#include <glusterfs/logging.h>
+#include <glusterfs/dict.h>
+#include <glusterfs/xlator.h>
+#include <glusterfs/list.h>
+#include <glusterfs/compat.h>
+#include <glusterfs/compat-errno.h>
+#include <glusterfs/common-utils.h>
+#include <glusterfs/call-stub.h>
+#include <glusterfs/defaults.h>
 #include <libgen.h>
 #include <sys/time.h>
 #include <sys/types.h>
@@ -34,48 +29,63 @@
 #include <fnmatch.h>
 #include "quick-read-mem-types.h"
 
-
 struct qr_inode {
-	void             *data;
-	size_t            size;
-        int               priority;
-	uint32_t          ia_mtime;
-	uint32_t          ia_mtime_nsec;
-	struct iatt       buf;
-        struct timeval    last_refresh;
-        struct list_head  lru;
+    void *data;
+    size_t size;
+    int priority;
+    uint32_t ia_mtime;
+    uint32_t ia_mtime_nsec;
+    uint32_t ia_ctime;
+    uint32_t ia_ctime_nsec;
+    uint32_t gen_rollover;
+    struct iatt buf;
+    time_t last_refresh;
+    struct list_head lru;
+    uint64_t gen;
+    uint64_t invalidation_time;
 };
 typedef struct qr_inode qr_inode_t;
 
-
 struct qr_priority {
-        char             *pattern;
-        int32_t           priority;
-        struct list_head  list;
+    char *pattern;
+    int32_t priority;
+    struct list_head list;
 };
 typedef struct qr_priority qr_priority_t;
 
 struct qr_conf {
-        uint64_t         max_file_size;
-        int32_t          cache_timeout;
-        uint64_t         cache_size;
-        int              max_pri;
-        struct list_head priority_list;
+    uint64_t max_file_size;
+    int32_t cache_timeout;
+    uint64_t cache_size;
+    int max_pri;
+    gf_boolean_t qr_invalidation;
+    gf_boolean_t ctime_invalidation;
+    struct list_head priority_list;
 };
 typedef struct qr_conf qr_conf_t;
 
 struct qr_inode_table {
-        uint64_t          cache_used;
-        struct list_head *lru;
-        gf_lock_t         lock;
+    uint64_t cache_used;
+    struct list_head *lru;
+    gf_lock_t lock;
 };
 typedef struct qr_inode_table qr_inode_table_t;
 
+struct qr_statistics {
+    gf_atomic_t cache_hit;
+    gf_atomic_t cache_miss;
+    gf_atomic_t file_data_invals; /* No. of invalidates received from upcall */
+    gf_atomic_t files_cached;
+};
+
 struct qr_private {
-        qr_conf_t         conf;
-        qr_inode_table_t  table;
+    qr_conf_t conf;
+    qr_inode_table_t table;
+    time_t last_child_down;
+    gf_lock_t lock;
+    struct qr_statistics qr_counter;
+    gf_atomic_int32_t generation;
 };
 typedef struct qr_private qr_private_t;
 
-
 #endif /* #ifndef __QUICK_READ_H */
diff --git a/xlators/performance/read-ahead/src/Makefile.am b/xlators/performance/read-ahead/src/Makefile.am
index be80ae7ac68..99efca3660c 100644
--- a/xlators/performance/read-ahead/src/Makefile.am
+++ b/xlators/performance/read-ahead/src/Makefile.am
@@ -1,14 +1,15 @@
 xlator_LTLIBRARIES = read-ahead.la
 xlatordir = $(libdir)/glusterfs/$(PACKAGE_VERSION)/xlator/performance
 
-read_ahead_la_LDFLAGS = -module -avoid-version
+read_ahead_la_LDFLAGS = -module $(GF_XLATOR_DEFAULT_LDFLAGS)
 
 read_ahead_la_SOURCES = read-ahead.c page.c
 read_ahead_la_LIBADD = $(top_builddir)/libglusterfs/src/libglusterfs.la
 
-noinst_HEADERS = read-ahead.h read-ahead-mem-types.h
+noinst_HEADERS = read-ahead.h read-ahead-mem-types.h read-ahead-messages.h
 
-AM_CPPFLAGS = $(GF_CPPFLAGS) -I$(top_srcdir)/libglusterfs/src
+AM_CPPFLAGS = $(GF_CPPFLAGS) -I$(top_srcdir)/libglusterfs/src \
+	-I$(top_srcdir)/rpc/xdr/src -I$(top_builddir)/rpc/xdr/src
 
 AM_CFLAGS = -Wall $(GF_CFLAGS)
 
diff --git a/xlators/performance/read-ahead/src/page.c b/xlators/performance/read-ahead/src/page.c
index 6e5b52c5e27..8a58ad8bb7a 100644
--- a/xlators/performance/read-ahead/src/page.c
+++ b/xlators/performance/read-ahead/src/page.c
@@ -8,451 +8,448 @@
   cases as published by the Free Software Foundation.
 */
 
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
-
-#include "glusterfs.h"
-#include "logging.h"
-#include "dict.h"
-#include "xlator.h"
+#include <glusterfs/glusterfs.h>
+#include <glusterfs/logging.h>
+#include <glusterfs/dict.h>
+#include <glusterfs/xlator.h>
 #include "read-ahead.h"
 #include <assert.h>
+#include "read-ahead-messages.h"
 
 ra_page_t *
-ra_page_get (ra_file_t *file, off_t offset)
+ra_page_get(ra_file_t *file, off_t offset)
 {
-        ra_page_t *page           = NULL;
-        off_t      rounded_offset = 0;
+    ra_page_t *page = NULL;
+    off_t rounded_offset = 0;
 
-        GF_VALIDATE_OR_GOTO ("read-ahead", file, out);
+    GF_VALIDATE_OR_GOTO("read-ahead", file, out);
 
-        page = file->pages.next;
-        rounded_offset = floor (offset, file->page_size);
+    page = file->pages.next;
+    rounded_offset = gf_floor(offset, file->page_size);
 
-        while (page != &file->pages && page->offset < rounded_offset)
-                page = page->next;
+    while (page != &file->pages && page->offset < rounded_offset)
+        page = page->next;
 
-        if (page == &file->pages || page->offset != rounded_offset)
-                page = NULL;
+    if (page == &file->pages || page->offset != rounded_offset)
+        page = NULL;
 
 out:
-        return page;
+    return page;
 }
 
-
 ra_page_t *
-ra_page_create (ra_file_t *file, off_t offset)
+ra_page_create(ra_file_t *file, off_t offset)
 {
-        ra_page_t  *page           = NULL;
-        off_t       rounded_offset = 0;
-        ra_page_t  *newpage        = NULL;
+    ra_page_t *page = NULL;
+    off_t rounded_offset = 0;
+    ra_page_t *newpage = NULL;
 
-        GF_VALIDATE_OR_GOTO ("read-ahead", file, out);
+    GF_VALIDATE_OR_GOTO("read-ahead", file, out);
 
-        page           = file->pages.next;
-        rounded_offset = floor (offset, file->page_size);
+    page = file->pages.next;
+    rounded_offset = gf_floor(offset, file->page_size);
 
-        while (page != &file->pages && page->offset < rounded_offset)
-                page = page->next;
+    while (page != &file->pages && page->offset < rounded_offset)
+        page = page->next;
 
-        if (page == &file->pages || page->offset != rounded_offset) {
-                newpage = GF_CALLOC (1, sizeof (*newpage), gf_ra_mt_ra_page_t);
-                if (!newpage) {
-                        goto out;
-                }
+    if (page == &file->pages || page->offset != rounded_offset) {
+        newpage = GF_CALLOC(1, sizeof(*newpage), gf_ra_mt_ra_page_t);
+        if (!newpage) {
+            goto out;
+        }
 
-                newpage->offset = rounded_offset;
-                newpage->prev = page->prev;
-                newpage->next = page;
-                newpage->file = file;
-                page->prev->next = newpage;
-                page->prev = newpage;
+        newpage->offset = rounded_offset;
+        newpage->prev = page->prev;
+        newpage->next = page;
+        newpage->file = file;
+        page->prev->next = newpage;
+        page->prev = newpage;
 
-                page = newpage;
-        }
+        page = newpage;
+    }
 
 out:
-        return page;
+    return page;
 }
 
-
 void
-ra_wait_on_page (ra_page_t *page, call_frame_t *frame)
+ra_wait_on_page(ra_page_t *page, call_frame_t *frame)
 {
-        ra_waitq_t *waitq = NULL;
-        ra_local_t *local = NULL;
+    ra_waitq_t *waitq = NULL;
+    ra_local_t *local = NULL;
 
-        GF_VALIDATE_OR_GOTO ("read-ahead", frame, out);
-        GF_VALIDATE_OR_GOTO (frame->this->name, page, out);
+    GF_VALIDATE_OR_GOTO("read-ahead", frame, out);
+    GF_VALIDATE_OR_GOTO(frame->this->name, page, out);
 
-        local = frame->local;
+    local = frame->local;
 
-        waitq = GF_CALLOC (1, sizeof (*waitq), gf_ra_mt_ra_waitq_t);
-        if (!waitq) {
-                local->op_ret = -1;
-                local->op_errno = ENOMEM;
-                goto out;
-        }
+    waitq = GF_CALLOC(1, sizeof(*waitq), gf_ra_mt_ra_waitq_t);
+    if (!waitq) {
+        local->op_ret = -1;
+        local->op_errno = ENOMEM;
+        goto out;
+    }
 
-        waitq->data = frame;
-        waitq->next = page->waitq;
-        page->waitq = waitq;
+    waitq->data = frame;
+    waitq->next = page->waitq;
+    page->waitq = waitq;
 
-        ra_local_lock (local);
-        {
-                local->wait_count++;
-        }
-        ra_local_unlock (local);
+    ra_local_lock(local);
+    {
+        local->wait_count++;
+    }
+    ra_local_unlock(local);
 
 out:
-        return;
+    return;
 }
 
-
 void
-ra_waitq_return (ra_waitq_t *waitq)
+ra_waitq_return(ra_waitq_t *waitq)
 {
-        ra_waitq_t   *trav  = NULL;
-        ra_waitq_t   *next  = NULL;
-        call_frame_t *frame = NULL;
+    ra_waitq_t *trav = NULL;
+    ra_waitq_t *next = NULL;
+    call_frame_t *frame = NULL;
 
-        for (trav = waitq; trav; trav = next) {
-                next = trav->next;
+    for (trav = waitq; trav; trav = next) {
+        next = trav->next;
 
-                frame = trav->data;
-                ra_frame_return (frame);
-                GF_FREE (trav);
-        }
+        frame = trav->data;
+        ra_frame_return(frame);
+        GF_FREE(trav);
+    }
 
-        return;
+    return;
 }
 
-
 int
-ra_fault_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-              int32_t op_ret, int32_t op_errno, struct iovec *vector,
-              int32_t count, struct iatt *stbuf, struct iobref *iobref,
-              dict_t *xdata)
+ra_fault_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int32_t op_ret,
+             int32_t op_errno, struct iovec *vector, int32_t count,
+             struct iatt *stbuf, struct iobref *iobref, dict_t *xdata)
 {
-        ra_local_t   *local          = NULL;
-        off_t         pending_offset = 0;
-        ra_file_t    *file           = NULL;
-        ra_page_t    *page           = NULL;
-        ra_waitq_t   *waitq          = NULL;
-        fd_t         *fd             = NULL;
-        uint64_t      tmp_file       = 0;
-
-        GF_ASSERT (frame);
-
-        local = frame->local;
-        fd  = local->fd;
-
-        fd_ctx_get (fd, this, &tmp_file);
-
-        file = (ra_file_t *)(long)tmp_file;
-        pending_offset = local->pending_offset;
-
-        if (file == NULL) {
-                gf_log (this->name, GF_LOG_WARNING,
-                        "read-ahead context not set in fd (%p)", fd);
-                op_ret = -1;
-                op_errno = EBADF;
-                goto out;
+    ra_local_t *local = NULL;
+    off_t pending_offset = 0;
+    ra_file_t *file = NULL;
+    ra_page_t *page = NULL;
+    ra_waitq_t *waitq = NULL;
+    fd_t *fd = NULL;
+    uint64_t tmp_file = 0;
+    gf_boolean_t stale = _gf_false;
+
+    GF_ASSERT(frame);
+
+    local = frame->local;
+    fd = local->fd;
+
+    fd_ctx_get(fd, this, &tmp_file);
+
+    file = (ra_file_t *)(long)tmp_file;
+    pending_offset = local->pending_offset;
+
+    if (file == NULL) {
+        gf_msg(this->name, GF_LOG_WARNING, EBADF,
+               READ_AHEAD_MSG_FD_CONTEXT_NOT_SET,
+               "read-ahead context not set in fd (%p)", fd);
+        op_ret = -1;
+        op_errno = EBADF;
+        goto out;
+    }
+
+    ra_file_lock(file);
+    {
+        if (op_ret >= 0)
+            file->stbuf = *stbuf;
+
+        page = ra_page_get(file, pending_offset);
+
+        if (!page) {
+            gf_msg_trace(this->name, 0,
+                         "wasted copy: "
+                         "%" PRId64 "[+%" PRId64 "] file=%p",
+                         pending_offset, file->page_size, file);
+            goto unlock;
         }
 
-        ra_file_lock (file);
-        {
-                if (op_ret >= 0)
-                        file->stbuf = *stbuf;
-
-                page = ra_page_get (file, pending_offset);
-
-                if (!page) {
-                        gf_log (this->name, GF_LOG_TRACE,
-                                "wasted copy: %"PRId64"[+%"PRId64"] file=%p",
-                                pending_offset, file->page_size, file);
-                        goto unlock;
-                }
-
-                /*
-                 * "Dirty" means that the request was a pure read-ahead; it's
-                 * set for requests we issue ourselves, and cleared when user
-                 * requests are issued or put on the waitq.  "Poisoned" means
-                 * that we got a write while a read was still in flight, and we
-                 * couldn't stop it so we marked it instead.  If it's both
-                 * dirty and poisoned by the time we get here, we cancel its
-                 * effect so that a subsequent user read doesn't get data that
-                 * we know is stale (because we made it stale ourselves).  We
-                 * can't use ESTALE because that has special significance.
-                 * ECANCELED has no such special meaning, and is close to what
-                 * we're trying to indicate.
-                 */
-                if (page->dirty && page->poisoned) {
-                        op_ret = -1;
-                        op_errno = ECANCELED;
-                }
-
-                if (op_ret < 0) {
-                        waitq = ra_page_error (page, op_ret, op_errno);
-                        goto unlock;
-                }
-
-                if (page->vector) {
-                        iobref_unref (page->iobref);
-                        GF_FREE (page->vector);
-                }
-
-                page->vector = iov_dup (vector, count);
-                if (page->vector == NULL) {
-                        waitq = ra_page_error (page, -1, ENOMEM);
-                        goto unlock;
-                }
-
-                page->count = count;
-                page->iobref = iobref_ref (iobref);
-                page->ready = 1;
-
-                page->size = iov_length (vector, count);
-
-                waitq = ra_page_wakeup (page);
+        if (page->stale) {
+            page->stale = 0;
+            page->ready = 0;
+            stale = 1;
+            goto unlock;
         }
-unlock:
-        ra_file_unlock (file);
-
-        ra_waitq_return (waitq);
-
-        fd_unref (local->fd);
 
-        mem_put (frame->local);
-        frame->local = NULL;
-
-out:
-        STACK_DESTROY (frame->root);
-        return 0;
-}
+        /*
+         * "Dirty" means that the request was a pure read-ahead; it's
+         * set for requests we issue ourselves, and cleared when user
+         * requests are issued or put on the waitq.  "Poisoned" means
+         * that we got a write while a read was still in flight, and we
+         * couldn't stop it so we marked it instead.  If it's both
+         * dirty and poisoned by the time we get here, we cancel its
+         * effect so that a subsequent user read doesn't get data that
+         * we know is stale (because we made it stale ourselves).  We
+         * can't use ESTALE because that has special significance.
+         * ECANCELED has no such special meaning, and is close to what
+         * we're trying to indicate.
+         */
+        if (page->dirty && page->poisoned) {
+            op_ret = -1;
+            op_errno = ECANCELED;
+        }
 
+        if (op_ret < 0) {
+            waitq = ra_page_error(page, op_ret, op_errno);
+            goto unlock;
+        }
 
-void
-ra_page_fault (ra_file_t *file, call_frame_t *frame, off_t offset)
-{
-        call_frame_t *fault_frame = NULL;
-        ra_local_t   *fault_local = NULL;
-        ra_page_t    *page        = NULL;
-        ra_waitq_t   *waitq       = NULL;
-        int32_t       op_ret      = -1, op_errno = -1;
-
-        GF_VALIDATE_OR_GOTO ("read-ahead", frame, out);
-        GF_VALIDATE_OR_GOTO (frame->this->name, file, out);
-
-        fault_frame = copy_frame (frame);
-        if (fault_frame == NULL) {
-                op_ret = -1;
-                op_errno = ENOMEM;
-                goto err;
+        if (page->vector) {
+            iobref_unref(page->iobref);
+            GF_FREE(page->vector);
         }
 
-        fault_local = mem_get0 (THIS->local_pool);
-        if (fault_local == NULL) {
-                STACK_DESTROY (fault_frame->root);
-                op_ret = -1;
-                op_errno = ENOMEM;
-                goto err;
+        page->vector = iov_dup(vector, count);
+        if (page->vector == NULL) {
+            waitq = ra_page_error(page, -1, ENOMEM);
+            goto unlock;
         }
 
-        fault_frame->local = fault_local;
-        fault_local->pending_offset = offset;
-        fault_local->pending_size = file->page_size;
+        page->count = count;
+        page->iobref = iobref_ref(iobref);
+        page->ready = 1;
 
-        fault_local->fd = fd_ref (file->fd);
+        page->size = iov_length(vector, count);
 
-        STACK_WIND (fault_frame, ra_fault_cbk,
-                    FIRST_CHILD (fault_frame->this),
-                    FIRST_CHILD (fault_frame->this)->fops->readv,
-                    file->fd, file->page_size, offset, 0, NULL);
+        waitq = ra_page_wakeup(page);
+    }
+unlock:
+    ra_file_unlock(file);
 
-        return;
+    if (stale) {
+        STACK_WIND(frame, ra_fault_cbk, FIRST_CHILD(frame->this),
+                   FIRST_CHILD(frame->this)->fops->readv, local->fd,
+                   local->pending_size, local->pending_offset, 0, NULL);
 
-err:
-        ra_file_lock (file);
-        {
-                page = ra_page_get (file, offset);
-                if (page)
-                        waitq = ra_page_error (page, op_ret,
-                                               op_errno);
-        }
-        ra_file_unlock (file);
+        return 0;
+    }
 
-        if (waitq != NULL) {
-                ra_waitq_return (waitq);
-        }
+    ra_waitq_return(waitq);
+
+    fd_unref(local->fd);
+
+    mem_put(frame->local);
+    frame->local = NULL;
 
 out:
-        return;
+    STACK_DESTROY(frame->root);
+    return 0;
 }
 
-
 void
-ra_frame_fill (ra_page_t *page, call_frame_t *frame)
+ra_page_fault(ra_file_t *file, call_frame_t *frame, off_t offset)
 {
-        ra_local_t *local      = NULL;
-        ra_fill_t  *fill       = NULL;
-        off_t       src_offset = 0;
-        off_t       dst_offset = 0;
-        ssize_t     copy_size  = 0;
-        ra_fill_t  *new        = NULL;
-
-        GF_VALIDATE_OR_GOTO ("read-ahead", frame, out);
-        GF_VALIDATE_OR_GOTO (frame->this->name, page, out);
+    call_frame_t *fault_frame = NULL;
+    ra_local_t *fault_local = NULL;
+    ra_page_t *page = NULL;
+    ra_waitq_t *waitq = NULL;
+    int32_t op_ret = -1, op_errno = -1;
+
+    GF_VALIDATE_OR_GOTO("read-ahead", frame, out);
+    GF_VALIDATE_OR_GOTO(frame->this->name, file, out);
+
+    fault_frame = copy_frame(frame);
+    if (fault_frame == NULL) {
+        op_ret = -1;
+        op_errno = ENOMEM;
+        goto err;
+    }
+
+    fault_local = mem_get0(THIS->local_pool);
+    if (fault_local == NULL) {
+        STACK_DESTROY(fault_frame->root);
+        op_ret = -1;
+        op_errno = ENOMEM;
+        goto err;
+    }
+
+    fault_frame->local = fault_local;
+    fault_local->pending_offset = offset;
+    fault_local->pending_size = file->page_size;
+
+    fault_local->fd = fd_ref(file->fd);
+
+    STACK_WIND(fault_frame, ra_fault_cbk, FIRST_CHILD(fault_frame->this),
+               FIRST_CHILD(fault_frame->this)->fops->readv, file->fd,
+               file->page_size, offset, 0, NULL);
+
+    return;
 
-        local = frame->local;
-        fill  = &local->fill;
-
-        if (local->op_ret != -1 && page->size) {
-                if (local->offset > page->offset)
-                        src_offset = local->offset - page->offset;
-                else
-                        dst_offset = page->offset - local->offset;
-
-                copy_size = min (page->size - src_offset,
-                                 local->size - dst_offset);
-
-                if (copy_size < 0) {
-                        /* if page contains fewer bytes and the required offset
-                           is beyond the page size in the page */
-                        copy_size = src_offset = 0;
-                }
-
-                fill = fill->next;
-                while (fill != &local->fill) {
-                        if (fill->offset > page->offset) {
-                                break;
-                        }
-                        fill = fill->next;
-                }
-
-                new = GF_CALLOC (1, sizeof (*new), gf_ra_mt_ra_fill_t);
-                if (new == NULL) {
-                        local->op_ret = -1;
-                        local->op_errno = ENOMEM;
-                        goto out;
-                }
-
-                new->offset = page->offset;
-                new->size = copy_size;
-                new->iobref = iobref_ref (page->iobref);
-                new->count = iov_subset (page->vector, page->count,
-                                         src_offset, src_offset+copy_size,
-                                         NULL);
-                new->vector = GF_CALLOC (new->count, sizeof (struct iovec),
-                                         gf_ra_mt_iovec);
-                if (new->vector == NULL) {
-                        local->op_ret = -1;
-                        local->op_errno = ENOMEM;
-                        GF_FREE (new);
-                        goto out;
-                }
-
-                new->count = iov_subset (page->vector, page->count,
-                                         src_offset, src_offset+copy_size,
-                                         new->vector);
-
-                new->next = fill;
-                new->prev = new->next->prev;
-                new->next->prev = new;
-                new->prev->next = new;
-
-                local->op_ret += copy_size;
-        }
+err:
+    ra_file_lock(file);
+    {
+        page = ra_page_get(file, offset);
+        if (page)
+            waitq = ra_page_error(page, op_ret, op_errno);
+    }
+    ra_file_unlock(file);
+
+    if (waitq != NULL) {
+        ra_waitq_return(waitq);
+    }
 
 out:
-        return;
+    return;
 }
 
-
 void
-ra_frame_unwind (call_frame_t *frame)
+ra_frame_fill(ra_page_t *page, call_frame_t *frame)
 {
-        ra_local_t    *local    = NULL;
-        ra_fill_t     *fill     = NULL;
-        int32_t        count    = 0;
-        struct iovec  *vector   = NULL;
-        int32_t        copied   = 0;
-        struct iobref *iobref   = NULL;
-        ra_fill_t     *next     = NULL;
-        fd_t          *fd       = NULL;
-        ra_file_t     *file     = NULL;
-        uint64_t       tmp_file = 0;
-
-        GF_VALIDATE_OR_GOTO ("read-ahead", frame, out);
-
-        local = frame->local;
-        fill  = local->fill.next;
-
-        iobref  = iobref_new ();
-        if (iobref == NULL) {
-                local->op_ret = -1;
-                local->op_errno = ENOMEM;
+    ra_local_t *local = NULL;
+    ra_fill_t *fill = NULL;
+    off_t src_offset = 0;
+    off_t dst_offset = 0;
+    ssize_t copy_size = 0;
+    ra_fill_t *new = NULL;
+
+    GF_VALIDATE_OR_GOTO("read-ahead", frame, out);
+    GF_VALIDATE_OR_GOTO(frame->this->name, page, out);
+
+    local = frame->local;
+    fill = &local->fill;
+
+    if (local->op_ret != -1 && page->size) {
+        if (local->offset > page->offset)
+            src_offset = local->offset - page->offset;
+        else
+            dst_offset = page->offset - local->offset;
+
+        copy_size = min(page->size - src_offset, local->size - dst_offset);
+
+        if (copy_size < 0) {
+            /* if page contains fewer bytes and the required offset
+               is beyond the page size in the page */
+            copy_size = src_offset = 0;
         }
 
-        frame->local = NULL;
-
+        fill = fill->next;
         while (fill != &local->fill) {
-                count += fill->count;
-                fill = fill->next;
+            if (fill->offset > page->offset) {
+                break;
+            }
+            fill = fill->next;
         }
 
-        vector = GF_CALLOC (count, sizeof (*vector), gf_ra_mt_iovec);
-        if (vector == NULL) {
-                local->op_ret = -1;
-                local->op_errno = ENOMEM;
-                iobref_unref (iobref);
-                iobref = NULL;
+        new = GF_CALLOC(1, sizeof(*new), gf_ra_mt_ra_fill_t);
+        if (new == NULL) {
+            local->op_ret = -1;
+            local->op_errno = ENOMEM;
+            goto out;
+        }
+
+        new->offset = page->offset;
+        new->size = copy_size;
+        new->iobref = iobref_ref(page->iobref);
+        new->count = iov_subset(page->vector, page->count, src_offset,
+                                copy_size, &new->vector, 0);
+        if (new->count < 0) {
+            local->op_ret = -1;
+            local->op_errno = ENOMEM;
+            iobref_unref(new->iobref);
+            GF_FREE(new);
+            goto out;
         }
 
-        fill = local->fill.next;
+        new->next = fill;
+        new->prev = new->next->prev;
+        new->next->prev = new;
+        new->prev->next = new;
 
-        while (fill != &local->fill) {
-                next = fill->next;
+        local->op_ret += copy_size;
+    }
 
-                if ((vector != NULL) && (iobref != NULL)) {
-                        memcpy (((char *)vector) + copied, fill->vector,
-                                fill->count * sizeof (*vector));
+out:
+    return;
+}
 
-                        copied += (fill->count * sizeof (*vector));
-                        if (iobref_merge (iobref, fill->iobref)) {
-				local->op_ret = -1;
-				local->op_errno = ENOMEM;
-				iobref_unref (iobref);
-				iobref = NULL;
-			}
-                }
+void
+ra_frame_unwind(call_frame_t *frame)
+{
+    ra_local_t *local = NULL;
+    ra_fill_t *fill = NULL;
+    int32_t count = 0;
+    struct iovec *vector = NULL;
+    int32_t copied = 0;
+    struct iobref *iobref = NULL;
+    ra_fill_t *next = NULL;
+    fd_t *fd = NULL;
+    ra_file_t *file = NULL;
+    uint64_t tmp_file = 0;
+
+    GF_VALIDATE_OR_GOTO("read-ahead", frame, out);
+
+    local = frame->local;
+    fill = local->fill.next;
+
+    iobref = iobref_new();
+    if (iobref == NULL) {
+        local->op_ret = -1;
+        local->op_errno = ENOMEM;
+    }
+
+    frame->local = NULL;
+
+    while (fill != &local->fill) {
+        count += fill->count;
+        fill = fill->next;
+    }
+
+    vector = GF_CALLOC(count, sizeof(*vector), gf_ra_mt_iovec);
+    if (vector == NULL) {
+        local->op_ret = -1;
+        local->op_errno = ENOMEM;
+        iobref_unref(iobref);
+        iobref = NULL;
+    }
+
+    fill = local->fill.next;
+
+    while (fill != &local->fill) {
+        next = fill->next;
+
+        if ((vector != NULL) && (iobref != NULL)) {
+            memcpy(((char *)vector) + copied, fill->vector,
+                   fill->count * sizeof(*vector));
+
+            copied += (fill->count * sizeof(*vector));
+            if (iobref_merge(iobref, fill->iobref)) {
+                local->op_ret = -1;
+                local->op_errno = ENOMEM;
+                iobref_unref(iobref);
+                iobref = NULL;
+            }
+        }
 
-                fill->next->prev = fill->prev;
-                fill->prev->next = fill->prev;
+        fill->next->prev = fill->prev;
+        fill->prev->next = fill->prev;
 
-                iobref_unref (fill->iobref);
-                GF_FREE (fill->vector);
-                GF_FREE (fill);
+        iobref_unref(fill->iobref);
+        GF_FREE(fill->vector);
+        GF_FREE(fill);
 
-                fill = next;
-        }
+        fill = next;
+    }
 
-        fd = local->fd;
-        fd_ctx_get (fd, frame->this, &tmp_file);
-        file = (ra_file_t *)(long)tmp_file;
+    fd = local->fd;
+    fd_ctx_get(fd, frame->this, &tmp_file);
+    file = (ra_file_t *)(long)tmp_file;
 
-        STACK_UNWIND_STRICT (readv, frame, local->op_ret, local->op_errno,
-                             vector, count, &file->stbuf, iobref, NULL);
+    STACK_UNWIND_STRICT(readv, frame, local->op_ret, local->op_errno, vector,
+                        count, &file->stbuf, iobref, NULL);
 
-        iobref_unref (iobref);
-        pthread_mutex_destroy (&local->local_lock);
-        mem_put (local);
-        GF_FREE (vector);
+    iobref_unref(iobref);
+    pthread_mutex_destroy(&local->local_lock);
+    mem_put(local);
+    GF_FREE(vector);
 
 out:
-        return;
+    return;
 }
 
 /*
@@ -461,27 +458,27 @@ out:
  *
  */
 void
-ra_frame_return (call_frame_t *frame)
+ra_frame_return(call_frame_t *frame)
 {
-        ra_local_t  *local      = NULL;
-        int32_t      wait_count = 0;
+    ra_local_t *local = NULL;
+    int32_t wait_count = 0;
 
-        GF_VALIDATE_OR_GOTO ("read-ahead", frame, out);
+    GF_VALIDATE_OR_GOTO("read-ahead", frame, out);
 
-        local = frame->local;
-        GF_ASSERT (local->wait_count > 0);
+    local = frame->local;
+    GF_ASSERT(local->wait_count > 0);
 
-        ra_local_lock (local);
-        {
-                wait_count = --local->wait_count;
-        }
-        ra_local_unlock (local);
+    ra_local_lock(local);
+    {
+        wait_count = --local->wait_count;
+    }
+    ra_local_unlock(local);
 
-        if (!wait_count)
-                ra_frame_unwind (frame);
+    if (!wait_count)
+        ra_frame_unwind(frame);
 
 out:
-        return;
+    return;
 }
 
 /*
@@ -490,26 +487,26 @@ out:
  *
  */
 ra_waitq_t *
-ra_page_wakeup (ra_page_t *page)
+ra_page_wakeup(ra_page_t *page)
 {
-        ra_waitq_t   *waitq = NULL, *trav = NULL;
-        call_frame_t *frame = NULL;
+    ra_waitq_t *waitq = NULL, *trav = NULL;
+    call_frame_t *frame = NULL;
 
-        GF_VALIDATE_OR_GOTO ("read-ahead", page, out);
+    GF_VALIDATE_OR_GOTO("read-ahead", page, out);
 
-        waitq = page->waitq;
-        page->waitq = NULL;
+    waitq = page->waitq;
+    page->waitq = NULL;
 
-        for (trav = waitq; trav; trav = trav->next) {
-                frame = trav->data;
-                ra_frame_fill (page, frame);
-        }
+    for (trav = waitq; trav; trav = trav->next) {
+        frame = trav->data;
+        ra_frame_fill(page, frame);
+    }
 
-        if (page->stale) {
-                ra_page_purge (page);
-        }
+    if (page->stale) {
+        ra_page_purge(page);
+    }
 out:
-        return waitq;
+    return waitq;
 }
 
 /*
@@ -518,22 +515,22 @@ out:
  *
  */
 void
-ra_page_purge (ra_page_t *page)
+ra_page_purge(ra_page_t *page)
 {
-        GF_VALIDATE_OR_GOTO ("read-ahead", page, out);
+    GF_VALIDATE_OR_GOTO("read-ahead", page, out);
 
-        page->prev->next = page->next;
-        page->next->prev = page->prev;
+    page->prev->next = page->next;
+    page->next->prev = page->prev;
 
-        if (page->iobref) {
-                iobref_unref (page->iobref);
-        }
+    if (page->iobref) {
+        iobref_unref(page->iobref);
+    }
 
-        GF_FREE (page->vector);
-        GF_FREE (page);
+    GF_FREE(page->vector);
+    GF_FREE(page);
 
 out:
-        return;
+    return;
 }
 
 /*
@@ -544,32 +541,32 @@ out:
  *
  */
 ra_waitq_t *
-ra_page_error (ra_page_t *page, int32_t op_ret, int32_t op_errno)
+ra_page_error(ra_page_t *page, int32_t op_ret, int32_t op_errno)
 {
-        ra_waitq_t   *waitq = NULL;
-        ra_waitq_t   *trav  = NULL;
-        call_frame_t *frame = NULL;
-        ra_local_t   *local = NULL;
+    ra_waitq_t *waitq = NULL;
+    ra_waitq_t *trav = NULL;
+    call_frame_t *frame = NULL;
+    ra_local_t *local = NULL;
 
-        GF_VALIDATE_OR_GOTO ("read-ahead", page, out);
+    GF_VALIDATE_OR_GOTO("read-ahead", page, out);
 
-        waitq = page->waitq;
-        page->waitq = NULL;
+    waitq = page->waitq;
+    page->waitq = NULL;
 
-        for (trav = waitq; trav; trav = trav->next) {
-                frame = trav->data;
+    for (trav = waitq; trav; trav = trav->next) {
+        frame = trav->data;
 
-                local = frame->local;
-                if (local->op_ret != -1) {
-                        local->op_ret   = op_ret;
-                        local->op_errno = op_errno;
-                }
+        local = frame->local;
+        if (local->op_ret != -1) {
+            local->op_ret = op_ret;
+            local->op_errno = op_errno;
         }
+    }
 
-        ra_page_purge (page);
+    ra_page_purge(page);
 
 out:
-        return waitq;
+    return waitq;
 }
 
 /*
@@ -578,31 +575,31 @@ out:
  *
  */
 void
-ra_file_destroy (ra_file_t *file)
+ra_file_destroy(ra_file_t *file)
 {
-        ra_conf_t *conf = NULL;
-        ra_page_t *trav = NULL;
+    ra_conf_t *conf = NULL;
+    ra_page_t *trav = NULL;
 
-        GF_VALIDATE_OR_GOTO ("read-ahead", file, out);
+    GF_VALIDATE_OR_GOTO("read-ahead", file, out);
 
-        conf = file->conf;
+    conf = file->conf;
 
-        ra_conf_lock (conf);
-        {
-                file->prev->next = file->next;
-                file->next->prev = file->prev;
-        }
-        ra_conf_unlock (conf);
+    ra_conf_lock(conf);
+    {
+        file->prev->next = file->next;
+        file->next->prev = file->prev;
+    }
+    ra_conf_unlock(conf);
 
+    trav = file->pages.next;
+    while (trav != &file->pages) {
+        ra_page_error(trav, -1, EINVAL);
         trav = file->pages.next;
-        while (trav != &file->pages) {
-                ra_page_error (trav, -1, EINVAL);
-                trav = file->pages.next;
-        }
+    }
 
-        pthread_mutex_destroy (&file->file_lock);
-        GF_FREE (file);
+    pthread_mutex_destroy(&file->file_lock);
+    GF_FREE(file);
 
 out:
-        return;
+    return;
 }
diff --git a/xlators/performance/read-ahead/src/read-ahead-mem-types.h b/xlators/performance/read-ahead/src/read-ahead-mem-types.h
index 219e2928919..f07cfc5bba5 100644
--- a/xlators/performance/read-ahead/src/read-ahead-mem-types.h
+++ b/xlators/performance/read-ahead/src/read-ahead-mem-types.h
@@ -8,19 +8,18 @@
   cases as published by the Free Software Foundation.
 */
 
-
 #ifndef __RA_MEM_TYPES_H__
 #define __RA_MEM_TYPES_H__
 
-#include "mem-types.h"
+#include <glusterfs/mem-types.h>
 
 enum gf_ra_mem_types_ {
-        gf_ra_mt_ra_file_t   = gf_common_mt_end + 1,
-        gf_ra_mt_ra_conf_t,
-        gf_ra_mt_ra_page_t,
-        gf_ra_mt_ra_waitq_t,
-        gf_ra_mt_ra_fill_t,
-        gf_ra_mt_iovec,
-        gf_ra_mt_end
+    gf_ra_mt_ra_file_t = gf_common_mt_end + 1,
+    gf_ra_mt_ra_conf_t,
+    gf_ra_mt_ra_page_t,
+    gf_ra_mt_ra_waitq_t,
+    gf_ra_mt_ra_fill_t,
+    gf_ra_mt_iovec,
+    gf_ra_mt_end
 };
 #endif
diff --git a/xlators/performance/read-ahead/src/read-ahead-messages.h b/xlators/performance/read-ahead/src/read-ahead-messages.h
new file mode 100644
index 00000000000..0302b7a7122
--- /dev/null
+++ b/xlators/performance/read-ahead/src/read-ahead-messages.h
@@ -0,0 +1,31 @@
+/*Copyright (c) 2015 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
+*/
+
+#ifndef _READ_AHEAD_MESSAGES_H_
+#define _READ_AHEAD_MESSAGES_H_
+
+#include <glusterfs/glfs-message-id.h>
+
+/* To add new message IDs, append new identifiers at the end of the list.
+ *
+ * Never remove a message ID. If it's not used anymore, you can rename it or
+ * leave it as it is, but not delete it. This is to prevent reutilization of
+ * IDs by other messages.
+ *
+ * The component name must match one of the entries defined in
+ * glfs-message-id.h.
+ */
+
+GLFS_MSGID(READ_AHEAD, READ_AHEAD_MSG_XLATOR_CHILD_MISCONFIGURED,
+           READ_AHEAD_MSG_VOL_MISCONFIGURED, READ_AHEAD_MSG_NO_MEMORY,
+           READ_AHEAD_MSG_FD_CONTEXT_NOT_SET,
+           READ_AHEAD_MSG_UNDESTROYED_FILE_FOUND,
+           READ_AHEAD_MSG_XLATOR_CONF_NULL);
+
+#endif /* _READ_AHEAD_MESSAGES_H_ */
diff --git a/xlators/performance/read-ahead/src/read-ahead.c b/xlators/performance/read-ahead/src/read-ahead.c
index 7ea9f40a1c8..5246e1317d2 100644
--- a/xlators/performance/read-ahead/src/read-ahead.c
+++ b/xlators/performance/read-ahead/src/read-ahead.c
@@ -15,197 +15,187 @@
   - ensure efficient memory management in case of random seek
 */
 
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
-
-#include "glusterfs.h"
-#include "logging.h"
-#include "dict.h"
-#include "xlator.h"
+#include <glusterfs/glusterfs.h>
+#include <glusterfs/logging.h>
+#include <glusterfs/dict.h>
+#include <glusterfs/xlator.h>
 #include "read-ahead.h"
-#include "statedump.h"
+#include <glusterfs/statedump.h>
 #include <assert.h>
 #include <sys/time.h>
+#include "read-ahead-messages.h"
 
 static void
-read_ahead (call_frame_t *frame, ra_file_t *file);
-
+read_ahead(call_frame_t *frame, ra_file_t *file);
 
 int
-ra_open_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-             int32_t op_ret, int32_t op_errno, fd_t *fd, dict_t *xdata)
+ra_open_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int32_t op_ret,
+            int32_t op_errno, fd_t *fd, dict_t *xdata)
 {
-        ra_conf_t  *conf    = NULL;
-        ra_file_t  *file    = NULL;
-        int         ret     = 0;
-
-        GF_ASSERT (frame);
-        GF_VALIDATE_OR_GOTO (frame->this->name, this, unwind);
-
-        conf  = this->private;
-
-        if (op_ret == -1) {
-                goto unwind;
-        }
-
-        file = GF_CALLOC (1, sizeof (*file), gf_ra_mt_ra_file_t);
-        if (!file) {
-                op_ret = -1;
-                op_errno = ENOMEM;
-                goto unwind;
-        }
-
-        /* If O_DIRECT open, we disable caching on it */
-
-        if ((fd->flags & O_DIRECT) || ((fd->flags & O_ACCMODE) == O_WRONLY))
-                file->disabled = 1;
-
-        file->offset = (unsigned long long) 0;
-        file->conf = conf;
-        file->pages.next = &file->pages;
-        file->pages.prev = &file->pages;
-        file->pages.offset = (unsigned long long) 0;
-        file->pages.file = file;
-
-        ra_conf_lock (conf);
-        {
-                file->next = conf->files.next;
-                conf->files.next = file;
-                file->next->prev = file;
-                file->prev = &conf->files;
-        }
-        ra_conf_unlock (conf);
-
-        file->fd = fd;
-        file->page_count = conf->page_count;
-        file->page_size = conf->page_size;
-        pthread_mutex_init (&file->file_lock, NULL);
-
-        if (!file->disabled) {
-                file->page_count = 1;
-        }
-
-        ret = fd_ctx_set (fd, this, (uint64_t)(long)file);
-        if (ret == -1) {
-                gf_log (frame->this->name, GF_LOG_WARNING,
-                        "cannot set read-ahead context information in fd (%p)",
-                        fd);
-                ra_file_destroy (file);
-                op_ret = -1;
-                op_errno = ENOMEM;
-        }
+    ra_conf_t *conf = NULL;
+    ra_file_t *file = NULL;
+    int ret = 0;
+
+    GF_ASSERT(frame);
+    GF_VALIDATE_OR_GOTO(frame->this->name, this, unwind);
+
+    conf = this->private;
+
+    if (op_ret == -1) {
+        goto unwind;
+    }
+
+    file = GF_CALLOC(1, sizeof(*file), gf_ra_mt_ra_file_t);
+    if (!file) {
+        op_ret = -1;
+        op_errno = ENOMEM;
+        goto unwind;
+    }
+
+    /* If O_DIRECT open, we disable caching on it */
+
+    if ((fd->flags & O_DIRECT) || ((fd->flags & O_ACCMODE) == O_WRONLY))
+        file->disabled = 1;
+
+    file->offset = (unsigned long long)0;
+    file->conf = conf;
+    file->pages.next = &file->pages;
+    file->pages.prev = &file->pages;
+    file->pages.offset = (unsigned long long)0;
+    file->pages.file = file;
+
+    ra_conf_lock(conf);
+    {
+        file->next = conf->files.next;
+        conf->files.next = file;
+        file->next->prev = file;
+        file->prev = &conf->files;
+    }
+    ra_conf_unlock(conf);
+
+    file->fd = fd;
+    file->page_count = conf->page_count;
+    file->page_size = conf->page_size;
+    pthread_mutex_init(&file->file_lock, NULL);
+
+    if (!file->disabled) {
+        file->page_count = 1;
+    }
+
+    ret = fd_ctx_set(fd, this, (uint64_t)(long)file);
+    if (ret == -1) {
+        gf_msg(frame->this->name, GF_LOG_WARNING, 0, READ_AHEAD_MSG_NO_MEMORY,
+               "cannot set read-ahead context"
+               "information in fd (%p)",
+               fd);
+        ra_file_destroy(file);
+        op_ret = -1;
+        op_errno = ENOMEM;
+    }
 
 unwind:
-        frame->local = NULL;
+    frame->local = NULL;
 
-        STACK_UNWIND_STRICT (open, frame, op_ret, op_errno, fd, xdata);
+    STACK_UNWIND_STRICT(open, frame, op_ret, op_errno, fd, xdata);
 
-        return 0;
+    return 0;
 }
 
-
 int
-ra_create_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-               int32_t op_ret, int32_t op_errno, fd_t *fd, inode_t *inode,
-               struct iatt *buf, struct iatt *preparent,
-               struct iatt *postparent, dict_t *xdata)
+ra_create_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int32_t op_ret,
+              int32_t op_errno, fd_t *fd, inode_t *inode, struct iatt *buf,
+              struct iatt *preparent, struct iatt *postparent, dict_t *xdata)
 {
-        ra_conf_t  *conf = NULL;
-        ra_file_t  *file = NULL;
-        int         ret  = 0;
-
-        GF_ASSERT (frame);
-        GF_VALIDATE_OR_GOTO (frame->this->name, this, unwind);
-
-        conf  = this->private;
-
-        if (op_ret == -1) {
-                goto unwind;
-        }
-
-        file = GF_CALLOC (1, sizeof (*file), gf_ra_mt_ra_file_t);
-        if (!file) {
-                op_ret = -1;
-                op_errno = ENOMEM;
-                goto unwind;
-        }
-
-        /* If O_DIRECT open, we disable caching on it */
-
-        if ((fd->flags & O_DIRECT) || ((fd->flags & O_ACCMODE) == O_WRONLY))
-                file->disabled = 1;
-
-        file->offset = (unsigned long long) 0;
-        //file->size = fd->inode->buf.ia_size;
-        file->conf = conf;
-        file->pages.next = &file->pages;
-        file->pages.prev = &file->pages;
-        file->pages.offset = (unsigned long long) 0;
-        file->pages.file = file;
-
-        ra_conf_lock (conf);
-        {
-                file->next = conf->files.next;
-                conf->files.next = file;
-                file->next->prev = file;
-                file->prev = &conf->files;
-        }
-        ra_conf_unlock (conf);
-
-        file->fd = fd;
-        file->page_count = conf->page_count;
-        file->page_size = conf->page_size;
-        pthread_mutex_init (&file->file_lock, NULL);
-
-        ret = fd_ctx_set (fd, this, (uint64_t)(long)file);
-        if (ret == -1) {
-                gf_log (this->name, GF_LOG_WARNING,
-                        "cannot set read ahead context information in fd (%p)",
-                        fd);
-                ra_file_destroy (file);
-                op_ret = -1;
-                op_errno = ENOMEM;
-        }
+    ra_conf_t *conf = NULL;
+    ra_file_t *file = NULL;
+    int ret = 0;
+
+    GF_ASSERT(frame);
+    GF_VALIDATE_OR_GOTO(frame->this->name, this, unwind);
+
+    conf = this->private;
+
+    if (op_ret == -1) {
+        goto unwind;
+    }
+
+    file = GF_CALLOC(1, sizeof(*file), gf_ra_mt_ra_file_t);
+    if (!file) {
+        op_ret = -1;
+        op_errno = ENOMEM;
+        goto unwind;
+    }
+
+    /* If O_DIRECT open, we disable caching on it */
+
+    if ((fd->flags & O_DIRECT) || ((fd->flags & O_ACCMODE) == O_WRONLY))
+        file->disabled = 1;
+
+    file->offset = (unsigned long long)0;
+    // file->size = fd->inode->buf.ia_size;
+    file->conf = conf;
+    file->pages.next = &file->pages;
+    file->pages.prev = &file->pages;
+    file->pages.offset = (unsigned long long)0;
+    file->pages.file = file;
+
+    ra_conf_lock(conf);
+    {
+        file->next = conf->files.next;
+        conf->files.next = file;
+        file->next->prev = file;
+        file->prev = &conf->files;
+    }
+    ra_conf_unlock(conf);
+
+    file->fd = fd;
+    file->page_count = conf->page_count;
+    file->page_size = conf->page_size;
+    pthread_mutex_init(&file->file_lock, NULL);
+
+    ret = fd_ctx_set(fd, this, (uint64_t)(long)file);
+    if (ret == -1) {
+        gf_msg(this->name, GF_LOG_WARNING, 0, READ_AHEAD_MSG_NO_MEMORY,
+               "cannot set read ahead context"
+               "information in fd (%p)",
+               fd);
+        ra_file_destroy(file);
+        op_ret = -1;
+        op_errno = ENOMEM;
+    }
 
 unwind:
-        STACK_UNWIND_STRICT (create, frame, op_ret, op_errno, fd, inode, buf,
-                             preparent, postparent, xdata);
+    STACK_UNWIND_STRICT(create, frame, op_ret, op_errno, fd, inode, buf,
+                        preparent, postparent, xdata);
 
-        return 0;
+    return 0;
 }
 
-
 int
-ra_open (call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t flags,
-         fd_t *fd, dict_t *xdata)
+ra_open(call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t flags,
+        fd_t *fd, dict_t *xdata)
 {
-        GF_ASSERT (frame);
-        GF_ASSERT (this);
+    GF_ASSERT(frame);
+    GF_ASSERT(this);
 
-        STACK_WIND (frame, ra_open_cbk,
-                    FIRST_CHILD (this),
-                    FIRST_CHILD (this)->fops->open,
-                    loc, flags, fd, xdata);
+    STACK_WIND(frame, ra_open_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->open, loc, flags, fd, xdata);
 
-        return 0;
+    return 0;
 }
 
-
 int
-ra_create (call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t flags,
-           mode_t mode, mode_t umask, fd_t *fd, dict_t *xdata)
+ra_create(call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t flags,
+          mode_t mode, mode_t umask, fd_t *fd, dict_t *xdata)
 {
-        GF_ASSERT (frame);
-        GF_ASSERT (this);
+    GF_ASSERT(frame);
+    GF_ASSERT(this);
 
-        STACK_WIND (frame, ra_create_cbk,
-                    FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->create,
-                    loc, flags, mode, umask, fd, xdata);
+    STACK_WIND(frame, ra_create_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->create, loc, flags, mode, umask, fd,
+               xdata);
 
-        return 0;
+    return 0;
 }
 
 /* free cache pages between offset and offset+size,
@@ -213,733 +203,710 @@ ra_create (call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t flags,
 */
 
 static void
-flush_region (call_frame_t *frame, ra_file_t *file, off_t offset, off_t size,
-              int for_write)
+flush_region(call_frame_t *frame, ra_file_t *file, off_t offset, off_t size,
+             int for_write)
 {
-        ra_page_t *trav = NULL;
-        ra_page_t *next = NULL;
-
-        ra_file_lock (file);
-        {
-                trav = file->pages.next;
-                while (trav != &file->pages
-                       && trav->offset < (offset + size)) {
-
-                        next = trav->next;
-                        if (trav->offset >= offset) {
-                                if (!trav->waitq) {
-                                        ra_page_purge (trav);
-                                }
-                                else {
-                                        trav->stale = 1;
-
-                                        if (for_write) {
-                                                trav->poisoned = 1;
-                                        }
-                                }
-                        }
-                        trav = next;
+    ra_page_t *trav = NULL;
+    ra_page_t *next = NULL;
+
+    ra_file_lock(file);
+    {
+        trav = file->pages.next;
+        while (trav != &file->pages && trav->offset < (offset + size)) {
+            next = trav->next;
+            if (trav->offset >= offset) {
+                if (!trav->waitq) {
+                    ra_page_purge(trav);
+                } else {
+                    trav->stale = 1;
+
+                    if (for_write) {
+                        trav->poisoned = 1;
+                    }
                 }
+            }
+            trav = next;
         }
-        ra_file_unlock (file);
+    }
+    ra_file_unlock(file);
 }
 
-
 int
-ra_release (xlator_t *this, fd_t *fd)
+ra_release(xlator_t *this, fd_t *fd)
 {
-        uint64_t tmp_file = 0;
-        int      ret      = 0;
+    uint64_t tmp_file = 0;
+    int ret = 0;
 
-        GF_VALIDATE_OR_GOTO ("read-ahead", this, out);
-        GF_VALIDATE_OR_GOTO (this->name, fd, out);
+    GF_VALIDATE_OR_GOTO("read-ahead", this, out);
+    GF_VALIDATE_OR_GOTO(this->name, fd, out);
 
-        ret = fd_ctx_del (fd, this, &tmp_file);
+    ret = fd_ctx_del(fd, this, &tmp_file);
 
-        if (!ret) {
-                ra_file_destroy ((ra_file_t *)(long)tmp_file);
-        }
+    if (!ret) {
+        ra_file_destroy((ra_file_t *)(long)tmp_file);
+    }
 
 out:
-        return 0;
+    return 0;
 }
 
-
 void
-read_ahead (call_frame_t *frame, ra_file_t *file)
+read_ahead(call_frame_t *frame, ra_file_t *file)
 {
-        off_t      ra_offset   = 0;
-        size_t     ra_size     = 0;
-        off_t      trav_offset = 0;
-        ra_page_t *trav        = NULL;
-        off_t      cap         = 0;
-        char       fault       = 0;
-
-        GF_VALIDATE_OR_GOTO ("read-ahead", frame, out);
-        GF_VALIDATE_OR_GOTO (frame->this->name, file, out);
-
-        if (!file->page_count) {
-                goto out;
+    off_t ra_offset = 0;
+    size_t ra_size = 0;
+    off_t trav_offset = 0;
+    ra_page_t *trav = NULL;
+    off_t cap = 0;
+    char fault = 0;
+
+    GF_VALIDATE_OR_GOTO("read-ahead", frame, out);
+    GF_VALIDATE_OR_GOTO(frame->this->name, file, out);
+
+    if (!file->page_count) {
+        goto out;
+    }
+
+    ra_size = file->page_size * file->page_count;
+    ra_offset = gf_floor(file->offset, file->page_size);
+    cap = file->size ? file->size : file->offset + ra_size;
+
+    while (ra_offset < min(file->offset + ra_size, cap)) {
+        ra_file_lock(file);
+        {
+            trav = ra_page_get(file, ra_offset);
         }
+        ra_file_unlock(file);
 
-        ra_size   = file->page_size * file->page_count;
-        ra_offset = floor (file->offset, file->page_size);
-        cap       = file->size ? file->size : file->offset + ra_size;
+        if (!trav)
+            break;
 
-        while (ra_offset < min (file->offset + ra_size, cap)) {
+        ra_offset += file->page_size;
+    }
 
-                ra_file_lock (file);
-                {
-                        trav = ra_page_get (file, ra_offset);
-                }
-                ra_file_unlock (file);
+    if (trav) {
+        /* comfortable enough */
+        goto out;
+    }
 
-                if (!trav)
-                        break;
+    trav_offset = ra_offset;
 
-                ra_offset += file->page_size;
-        }
+    cap = file->size ? file->size : ra_offset + ra_size;
 
-        if (trav) {
-                /* comfortable enough */
-                goto out;
+    while (trav_offset < min(ra_offset + ra_size, cap)) {
+        fault = 0;
+        ra_file_lock(file);
+        {
+            trav = ra_page_get(file, trav_offset);
+            if (!trav) {
+                fault = 1;
+                trav = ra_page_create(file, trav_offset);
+                if (trav)
+                    trav->dirty = 1;
+            }
         }
+        ra_file_unlock(file);
 
-        trav_offset = ra_offset;
-
-        cap  = file->size ? file->size : ra_offset + ra_size;
-
-        while (trav_offset < min(ra_offset + ra_size, cap)) {
-                fault = 0;
-                ra_file_lock (file);
-                {
-                        trav = ra_page_get (file, trav_offset);
-                        if (!trav) {
-                                fault = 1;
-                                trav = ra_page_create (file, trav_offset);
-                                if (trav)
-                                        trav->dirty = 1;
-                        }
-                }
-                ra_file_unlock (file);
-
-                if (!trav) {
-                        /* OUT OF MEMORY */
-                        break;
-                }
+        if (!trav) {
+            /* OUT OF MEMORY */
+            break;
+        }
 
-                if (fault) {
-                        gf_log (frame->this->name, GF_LOG_TRACE,
-                                "RA at offset=%"PRId64, trav_offset);
-                        ra_page_fault (file, frame, trav_offset);
-                }
-                trav_offset += file->page_size;
+        if (fault) {
+            gf_msg_trace(frame->this->name, 0, "RA at offset=%" PRId64,
+                         trav_offset);
+            ra_page_fault(file, frame, trav_offset);
         }
+        trav_offset += file->page_size;
+    }
 
 out:
-        return;
+    return;
 }
 
-
 int
-ra_need_atime_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                   int32_t op_ret, int32_t op_errno, struct iovec *vector,
-                   int32_t count, struct iatt *stbuf, struct iobref *iobref,
-                   dict_t *xdata)
+ra_need_atime_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                  int32_t op_ret, int32_t op_errno, struct iovec *vector,
+                  int32_t count, struct iatt *stbuf, struct iobref *iobref,
+                  dict_t *xdata)
 {
-        GF_ASSERT (frame);
-        STACK_DESTROY (frame->root);
-        return 0;
+    GF_ASSERT(frame);
+    STACK_DESTROY(frame->root);
+    return 0;
 }
 
-
 static void
-dispatch_requests (call_frame_t *frame, ra_file_t *file)
+dispatch_requests(call_frame_t *frame, ra_file_t *file)
 {
-        ra_local_t   *local             = NULL;
-        ra_conf_t    *conf              = NULL;
-        off_t         rounded_offset    = 0;
-        off_t         rounded_end       = 0;
-        off_t         trav_offset       = 0;
-        ra_page_t    *trav              = NULL;
-        call_frame_t *ra_frame          = NULL;
-        char          need_atime_update = 1;
-        char          fault             = 0;
-
-        GF_VALIDATE_OR_GOTO ("read-ahead", frame, out);
-        GF_VALIDATE_OR_GOTO (frame->this->name, file, out);
-
-        local = frame->local;
-        conf  = file->conf;
-
-        rounded_offset = floor (local->offset, file->page_size);
-        rounded_end    = roof (local->offset + local->size, file->page_size);
-
-        trav_offset = rounded_offset;
-
-        while (trav_offset < rounded_end) {
-                fault = 0;
-
-                ra_file_lock (file);
-                {
-                        trav = ra_page_get (file, trav_offset);
-                        if (!trav) {
-                                trav = ra_page_create (file, trav_offset);
-                                if (!trav) {
-                                        local->op_ret = -1;
-                                        local->op_errno = ENOMEM;
-                                        goto unlock;
-                                }
-                                fault = 1;
-                                need_atime_update = 0;
-                        }
-                        trav->dirty = 0;
-
-                        if (trav->ready) {
-                                gf_log (frame->this->name, GF_LOG_TRACE,
-                                        "HIT at offset=%"PRId64".",
-                                        trav_offset);
-                                ra_frame_fill (trav, frame);
-                        } else {
-                                gf_log (frame->this->name, GF_LOG_TRACE,
-                                        "IN-TRANSIT at offset=%"PRId64".",
-                                        trav_offset);
-                                ra_wait_on_page (trav, frame);
-                                need_atime_update = 0;
-                        }
-                }
-        unlock:
-                ra_file_unlock (file);
+    ra_local_t *local = NULL;
+    ra_conf_t *conf = NULL;
+    off_t rounded_offset = 0;
+    off_t rounded_end = 0;
+    off_t trav_offset = 0;
+    ra_page_t *trav = NULL;
+    call_frame_t *ra_frame = NULL;
+    char need_atime_update = 1;
+    char fault = 0;
 
-                if (local->op_ret == -1) {
-                        goto out;
-                }
+    GF_VALIDATE_OR_GOTO("read-ahead", frame, out);
+    GF_VALIDATE_OR_GOTO(frame->this->name, file, out);
+
+    local = frame->local;
+    conf = file->conf;
+
+    rounded_offset = gf_floor(local->offset, file->page_size);
+    rounded_end = gf_roof(local->offset + local->size, file->page_size);
+
+    trav_offset = rounded_offset;
+
+    while (trav_offset < rounded_end) {
+        fault = 0;
 
-                if (fault) {
-                        gf_log (frame->this->name, GF_LOG_TRACE,
-                                "MISS at offset=%"PRId64".",
-                                trav_offset);
-                        ra_page_fault (file, frame, trav_offset);
+        ra_file_lock(file);
+        {
+            trav = ra_page_get(file, trav_offset);
+            if (!trav) {
+                trav = ra_page_create(file, trav_offset);
+                if (!trav) {
+                    local->op_ret = -1;
+                    local->op_errno = ENOMEM;
+                    goto unlock;
                 }
+                fault = 1;
+                need_atime_update = 0;
+            }
+            trav->dirty = 0;
+
+            if (trav->ready) {
+                gf_msg_trace(frame->this->name, 0, "HIT at offset=%" PRId64 ".",
+                             trav_offset);
+                ra_frame_fill(trav, frame);
+            } else {
+                gf_msg_trace(frame->this->name, 0,
+                             "IN-TRANSIT at "
+                             "offset=%" PRId64 ".",
+                             trav_offset);
+                ra_wait_on_page(trav, frame);
+                need_atime_update = 0;
+            }
+        }
+    unlock:
+        ra_file_unlock(file);
 
-                trav_offset += file->page_size;
+        if (local->op_ret == -1) {
+            goto out;
         }
 
-        if (need_atime_update && conf->force_atime_update) {
-                /* TODO: use untimens() since readv() can confuse underlying
-                   io-cache and others */
-                ra_frame = copy_frame (frame);
-                if (ra_frame == NULL) {
-                        goto out;
-                }
+        if (fault) {
+            gf_msg_trace(frame->this->name, 0, "MISS at offset=%" PRId64 ".",
+                         trav_offset);
+            ra_page_fault(file, frame, trav_offset);
+        }
 
-                STACK_WIND (ra_frame, ra_need_atime_cbk,
-                            FIRST_CHILD (frame->this),
-                            FIRST_CHILD (frame->this)->fops->readv,
-                            file->fd, 1, 1, 0, NULL);
+        trav_offset += file->page_size;
+    }
+
+    if (need_atime_update && conf->force_atime_update) {
+        /* TODO: use untimens() since readv() can confuse underlying
+           io-cache and others */
+        ra_frame = copy_frame(frame);
+        if (ra_frame == NULL) {
+            goto out;
         }
 
+        STACK_WIND(ra_frame, ra_need_atime_cbk, FIRST_CHILD(frame->this),
+                   FIRST_CHILD(frame->this)->fops->readv, file->fd, 1, 1, 0,
+                   NULL);
+    }
+
 out:
-        return ;
+    return;
 }
 
-
 int
-ra_readv_disabled_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                       int32_t op_ret, int32_t op_errno, struct iovec *vector,
-                       int32_t count, struct iatt *stbuf, struct iobref *iobref,
-                       dict_t *xdata)
+ra_readv_disabled_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                      int32_t op_ret, int32_t op_errno, struct iovec *vector,
+                      int32_t count, struct iatt *stbuf, struct iobref *iobref,
+                      dict_t *xdata)
 {
-        GF_ASSERT (frame);
+    GF_ASSERT(frame);
 
-        STACK_UNWIND_STRICT (readv, frame, op_ret, op_errno, vector, count,
-                             stbuf, iobref, xdata);
+    STACK_UNWIND_STRICT(readv, frame, op_ret, op_errno, vector, count, stbuf,
+                        iobref, xdata);
 
-        return 0;
+    return 0;
 }
 
-
 int
-ra_readv (call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
-          off_t offset, uint32_t flags, dict_t *xdata)
+ra_readv(call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
+         off_t offset, uint32_t flags, dict_t *xdata)
 {
-        ra_file_t   *file            = NULL;
-        ra_local_t  *local           = NULL;
-        ra_conf_t   *conf            = NULL;
-        int          op_errno        = EINVAL;
-        char         expected_offset = 1;
-        uint64_t     tmp_file        = 0;
-
-        GF_ASSERT (frame);
-        GF_VALIDATE_OR_GOTO (frame->this->name, this, unwind);
-        GF_VALIDATE_OR_GOTO (frame->this->name, fd, unwind);
-
-        conf = this->private;
-
-        gf_log (this->name, GF_LOG_TRACE,
-                "NEW REQ at offset=%"PRId64" for size=%"GF_PRI_SIZET"",
-                offset, size);
-
-        fd_ctx_get (fd, this, &tmp_file);
-        file = (ra_file_t *)(long)tmp_file;
-
-        if (!file || file->disabled) {
-                goto disabled;
-        }
-
-        if (file->offset != offset) {
-                gf_log (this->name, GF_LOG_TRACE,
-                        "unexpected offset (%"PRId64" != %"PRId64") resetting",
-                        file->offset, offset);
-
-                expected_offset = file->expected = file->page_count = 0;
-        } else {
-                gf_log (this->name, GF_LOG_TRACE,
-                        "expected offset (%"PRId64") when page_count=%d",
-                        offset, file->page_count);
-
-                if (file->expected < (file->page_size * conf->page_count)) {
-                        file->expected += size;
-                        file->page_count = min ((file->expected
-                                                 / file->page_size),
-                                                conf->page_count);
-                }
+    ra_file_t *file = NULL;
+    ra_local_t *local = NULL;
+    ra_conf_t *conf = NULL;
+    int op_errno = EINVAL;
+    char expected_offset = 1;
+    uint64_t tmp_file = 0;
+
+    GF_ASSERT(frame);
+    GF_VALIDATE_OR_GOTO(frame->this->name, this, unwind);
+    GF_VALIDATE_OR_GOTO(frame->this->name, fd, unwind);
+
+    conf = this->private;
+
+    gf_msg_trace(this->name, 0,
+                 "NEW REQ at offset=%" PRId64 " for size=%" GF_PRI_SIZET "",
+                 offset, size);
+
+    fd_ctx_get(fd, this, &tmp_file);
+    file = (ra_file_t *)(long)tmp_file;
+
+    if (!file || file->disabled) {
+        goto disabled;
+    }
+
+    if (file->offset != offset) {
+        gf_msg_trace(this->name, 0,
+                     "unexpected offset (%" PRId64 " != %" PRId64
+                     ") "
+                     "resetting",
+                     file->offset, offset);
+
+        expected_offset = file->expected = file->page_count = 0;
+    } else {
+        gf_msg_trace(this->name, 0,
+                     "expected offset (%" PRId64 ") when page_count=%d", offset,
+                     file->page_count);
+
+        if (file->expected < (file->page_size * conf->page_count)) {
+            file->expected += size;
+            file->page_count = min((file->expected / file->page_size),
+                                   conf->page_count);
         }
+    }
 
-        if (!expected_offset) {
-                flush_region (frame, file, 0, file->pages.prev->offset + 1, 0);
-        }
+    if (!expected_offset) {
+        flush_region(frame, file, 0, file->pages.prev->offset + 1, 0);
+    }
 
-        local = mem_get0 (this->local_pool);
-        if (!local) {
-                op_errno = ENOMEM;
-                goto unwind;
-        }
+    local = mem_get0(this->local_pool);
+    if (!local) {
+        op_errno = ENOMEM;
+        goto unwind;
+    }
 
-        local->fd         = fd;
-        local->offset     = offset;
-        local->size       = size;
-        local->wait_count = 1;
+    local->fd = fd;
+    local->offset = offset;
+    local->size = size;
+    local->wait_count = 1;
 
-        local->fill.next  = &local->fill;
-        local->fill.prev  = &local->fill;
+    local->fill.next = &local->fill;
+    local->fill.prev = &local->fill;
 
-        pthread_mutex_init (&local->local_lock, NULL);
+    pthread_mutex_init(&local->local_lock, NULL);
 
-        frame->local = local;
+    frame->local = local;
 
-        dispatch_requests (frame, file);
+    dispatch_requests(frame, file);
 
-        flush_region (frame, file, 0, floor (offset, file->page_size), 0);
+    flush_region(frame, file, 0, gf_floor(offset, file->page_size), 0);
 
-        read_ahead (frame, file);
+    read_ahead(frame, file);
 
-        ra_frame_return (frame);
+    file->offset = offset + size;
 
-        file->offset = offset + size;
+    ra_frame_return(frame);
 
-        return 0;
+    return 0;
 
 unwind:
-        STACK_UNWIND_STRICT (readv, frame, -1, op_errno, NULL, 0, NULL, NULL,
-                             NULL);
+    STACK_UNWIND_STRICT(readv, frame, -1, op_errno, NULL, 0, NULL, NULL, NULL);
 
-        return 0;
+    return 0;
 
 disabled:
-        STACK_WIND (frame, ra_readv_disabled_cbk,
-                    FIRST_CHILD (frame->this),
-                    FIRST_CHILD (frame->this)->fops->readv,
-                    fd, size, offset, flags, xdata);
-        return 0;
+    STACK_WIND(frame, ra_readv_disabled_cbk, FIRST_CHILD(frame->this),
+               FIRST_CHILD(frame->this)->fops->readv, fd, size, offset, flags,
+               xdata);
+    return 0;
 }
 
-
 int
-ra_flush_cbk (call_frame_t *frame, void *cookie, xlator_t *this, int32_t op_ret,
-              int32_t op_errno, dict_t *xdata)
+ra_flush_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int32_t op_ret,
+             int32_t op_errno, dict_t *xdata)
 {
-        GF_ASSERT (frame);
-        STACK_UNWIND_STRICT (flush, frame, op_ret, op_errno, xdata);
-        return 0;
+    GF_ASSERT(frame);
+    STACK_UNWIND_STRICT(flush, frame, op_ret, op_errno, xdata);
+    return 0;
 }
 
-
-
 int
-ra_fsync_cbk (call_frame_t *frame, void *cookie, xlator_t *this, int32_t op_ret,
-              int32_t op_errno, struct iatt *prebuf, struct iatt *postbuf,
-              dict_t *xdata)
+ra_fsync_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int32_t op_ret,
+             int32_t op_errno, struct iatt *prebuf, struct iatt *postbuf,
+             dict_t *xdata)
 {
-        GF_ASSERT (frame);
-        STACK_UNWIND_STRICT (fsync, frame, op_ret, op_errno, prebuf, postbuf,
-                             xdata);
-        return 0;
+    GF_ASSERT(frame);
+    STACK_UNWIND_STRICT(fsync, frame, op_ret, op_errno, prebuf, postbuf, xdata);
+    return 0;
 }
 
-
 int
-ra_flush (call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *xdata)
+ra_flush(call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *xdata)
 {
-        ra_file_t *file     = NULL;
-        uint64_t   tmp_file = 0;
-        int32_t    op_errno = EINVAL;
-
-        GF_ASSERT (frame);
-        GF_VALIDATE_OR_GOTO (frame->this->name, this, unwind);
-        GF_VALIDATE_OR_GOTO (frame->this->name, fd, unwind);
-
-        fd_ctx_get (fd, this, &tmp_file);
+    int32_t op_errno = EINVAL;
 
-        file = (ra_file_t *)(long)tmp_file;
-        if (file) {
-                flush_region (frame, file, 0, file->pages.prev->offset+1, 0);
-        }
+    GF_ASSERT(frame);
+    GF_VALIDATE_OR_GOTO(frame->this->name, this, unwind);
+    GF_VALIDATE_OR_GOTO(frame->this->name, fd, unwind);
 
-        STACK_WIND (frame, ra_flush_cbk, FIRST_CHILD (this),
-                    FIRST_CHILD (this)->fops->flush, fd, xdata);
-        return 0;
+    STACK_WIND(frame, ra_flush_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->flush, fd, xdata);
+    return 0;
 
 unwind:
-        STACK_UNWIND_STRICT (flush, frame, -1, op_errno, NULL);
-        return 0;
+    STACK_UNWIND_STRICT(flush, frame, -1, op_errno, NULL);
+    return 0;
 }
 
-
 int
-ra_fsync (call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t datasync,
-          dict_t *xdata)
+ra_fsync(call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t datasync,
+         dict_t *xdata)
 {
-        ra_file_t *file     = NULL;
-        uint64_t   tmp_file = 0;
-        int32_t    op_errno = EINVAL;
+    int32_t op_errno = EINVAL;
 
-        GF_ASSERT (frame);
-        GF_VALIDATE_OR_GOTO (frame->this->name, this, unwind);
-        GF_VALIDATE_OR_GOTO (frame->this->name, fd, unwind);
+    GF_ASSERT(frame);
+    GF_VALIDATE_OR_GOTO(frame->this->name, this, unwind);
+    GF_VALIDATE_OR_GOTO(frame->this->name, fd, unwind);
 
-        fd_ctx_get (fd, this, &tmp_file);
-
-        file = (ra_file_t *)(long)tmp_file;
-        if (file) {
-                flush_region (frame, file, 0, file->pages.prev->offset+1, 0);
-        }
-
-        STACK_WIND (frame, ra_fsync_cbk, FIRST_CHILD (this),
-                    FIRST_CHILD (this)->fops->fsync, fd, datasync, xdata);
-        return 0;
+    STACK_WIND(frame, ra_fsync_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->fsync, fd, datasync, xdata);
+    return 0;
 
 unwind:
-        STACK_UNWIND_STRICT (fsync, frame, -1, op_errno, NULL, NULL, NULL);
-        return 0;
+    STACK_UNWIND_STRICT(fsync, frame, -1, op_errno, NULL, NULL, NULL);
+    return 0;
 }
 
-
 int
-ra_writev_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-               int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
-               struct iatt *postbuf, dict_t *xdata)
+ra_writev_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int32_t op_ret,
+              int32_t op_errno, struct iatt *prebuf, struct iatt *postbuf,
+              dict_t *xdata)
 {
-        ra_file_t *file     = NULL;
+    ra_file_t *file = NULL;
 
-        GF_ASSERT (frame);
+    GF_ASSERT(frame);
 
-        file = frame->local;
+    file = frame->local;
 
-        if (file) {
-                flush_region (frame, file, 0, file->pages.prev->offset+1, 1);
-        }
+    if (file) {
+        flush_region(frame, file, 0, file->pages.prev->offset + 1, 1);
+    }
 
-        frame->local = NULL;
-        STACK_UNWIND_STRICT (writev, frame, op_ret, op_errno, prebuf, postbuf,
-                             xdata);
-        return 0;
+    frame->local = NULL;
+    STACK_UNWIND_STRICT(writev, frame, op_ret, op_errno, prebuf, postbuf,
+                        xdata);
+    return 0;
 }
 
-
 int
-ra_writev (call_frame_t *frame, xlator_t *this, fd_t *fd, struct iovec *vector,
-           int32_t count, off_t offset, uint32_t flags, struct iobref *iobref,
-           dict_t *xdata)
+ra_writev(call_frame_t *frame, xlator_t *this, fd_t *fd, struct iovec *vector,
+          int32_t count, off_t offset, uint32_t flags, struct iobref *iobref,
+          dict_t *xdata)
 {
-        ra_file_t *file    = NULL;
-        uint64_t  tmp_file = 0;
-        int32_t   op_errno = EINVAL;
-
-        GF_ASSERT (frame);
-        GF_VALIDATE_OR_GOTO (frame->this->name, this, unwind);
-        GF_VALIDATE_OR_GOTO (frame->this->name, fd, unwind);
-
-        fd_ctx_get (fd, this, &tmp_file);
-        file = (ra_file_t *)(long)tmp_file;
-        if (file) {
-                flush_region (frame, file, 0, file->pages.prev->offset+1, 1);
+    ra_file_t *file = NULL;
+    uint64_t tmp_file = 0;
+    int32_t op_errno = EINVAL;
+    inode_t *inode = NULL;
+    fd_t *iter_fd = NULL;
+
+    GF_ASSERT(frame);
+    GF_VALIDATE_OR_GOTO(frame->this->name, this, unwind);
+    GF_VALIDATE_OR_GOTO(frame->this->name, fd, unwind);
+
+    inode = fd->inode;
+
+    LOCK(&inode->lock);
+    {
+        list_for_each_entry(iter_fd, &inode->fd_list, inode_list)
+        {
+            tmp_file = 0;
+            fd_ctx_get(iter_fd, this, &tmp_file);
+            file = (ra_file_t *)(long)tmp_file;
+
+            if (!file)
+                continue;
+
+            if (iter_fd == fd)
                 frame->local = file;
-                /* reset the read-ahead counters too */
-                file->expected = file->page_count = 0;
+
+            flush_region(frame, file, 0, file->pages.prev->offset + 1, 1);
+
+            /* reset the read-ahead counters too */
+            file->expected = file->page_count = 0;
         }
+    }
+    UNLOCK(&inode->lock);
 
-        STACK_WIND (frame, ra_writev_cbk,
-                    FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->writev,
-                    fd, vector, count, offset, flags, iobref, xdata);
+    STACK_WIND(frame, ra_writev_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->writev, fd, vector, count, offset,
+               flags, iobref, xdata);
 
-        return 0;
+    return 0;
 
 unwind:
-        STACK_UNWIND_STRICT (writev, frame, -1, op_errno, NULL, NULL, NULL);
-        return 0;
+    STACK_UNWIND_STRICT(writev, frame, -1, op_errno, NULL, NULL, NULL);
+    return 0;
 }
 
-
 int
-ra_truncate_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                 int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
-                 struct iatt *postbuf, dict_t *xdata)
+ra_truncate_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
+                struct iatt *postbuf, dict_t *xdata)
 {
-        GF_ASSERT (frame);
+    GF_ASSERT(frame);
 
-        STACK_UNWIND_STRICT (truncate, frame, op_ret, op_errno, prebuf,
-                             postbuf, xdata);
-        return 0;
+    STACK_UNWIND_STRICT(truncate, frame, op_ret, op_errno, prebuf, postbuf,
+                        xdata);
+    return 0;
 }
 
-
 int
-ra_attr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-             int32_t op_ret, int32_t op_errno, struct iatt *buf, dict_t *xdata)
+ra_attr_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int32_t op_ret,
+            int32_t op_errno, struct iatt *buf, dict_t *xdata)
 {
-        GF_ASSERT (frame);
+    GF_ASSERT(frame);
 
-        STACK_UNWIND_STRICT (stat, frame, op_ret, op_errno, buf, xdata);
-        return 0;
+    STACK_UNWIND_STRICT(stat, frame, op_ret, op_errno, buf, xdata);
+    return 0;
 }
 
-
 int
-ra_truncate (call_frame_t *frame, xlator_t *this, loc_t *loc, off_t offset,
-             dict_t *xdata)
+ra_truncate(call_frame_t *frame, xlator_t *this, loc_t *loc, off_t offset,
+            dict_t *xdata)
 {
-        ra_file_t *file     = NULL;
-        fd_t      *iter_fd  = NULL;
-        inode_t   *inode    = NULL;
-        uint64_t   tmp_file = 0;
-        int32_t    op_errno = EINVAL;
+    ra_file_t *file = NULL;
+    fd_t *iter_fd = NULL;
+    inode_t *inode = NULL;
+    uint64_t tmp_file = 0;
+    int32_t op_errno = EINVAL;
 
-        GF_ASSERT (frame);
-        GF_VALIDATE_OR_GOTO (frame->this->name, this, unwind);
-        GF_VALIDATE_OR_GOTO (frame->this->name, loc, unwind);
+    GF_ASSERT(frame);
+    GF_VALIDATE_OR_GOTO(frame->this->name, this, unwind);
+    GF_VALIDATE_OR_GOTO(frame->this->name, loc, unwind);
 
-        inode = loc->inode;
+    inode = loc->inode;
 
-        LOCK (&inode->lock);
+    LOCK(&inode->lock);
+    {
+        list_for_each_entry(iter_fd, &inode->fd_list, inode_list)
         {
-                list_for_each_entry (iter_fd, &inode->fd_list, inode_list) {
-                        fd_ctx_get (iter_fd, this, &tmp_file);
-                        file = (ra_file_t *)(long)tmp_file;
-
-                        if (!file)
-                                continue;
-                        /*
-                         * Truncation invalidates reads just like writing does.
-                         * TBD: this seems to flush more than it should.  The
-                         * only time we should flush at all is when we're
-                         * shortening (not lengthening) the file, and then only
-                         * from new EOF to old EOF.  The same problem exists in
-                         * ra_ftruncate.
-                         */
-                        flush_region (frame, file, 0,
-                                      file->pages.prev->offset + 1, 1);
-                }
+            tmp_file = 0;
+            fd_ctx_get(iter_fd, this, &tmp_file);
+            file = (ra_file_t *)(long)tmp_file;
+
+            if (!file)
+                continue;
+            /*
+             * Truncation invalidates reads just like writing does.
+             * TBD: this seems to flush more than it should.  The
+             * only time we should flush at all is when we're
+             * shortening (not lengthening) the file, and then only
+             * from new EOF to old EOF.  The same problem exists in
+             * ra_ftruncate.
+             */
+            flush_region(frame, file, 0, file->pages.prev->offset + 1, 1);
         }
-        UNLOCK (&inode->lock);
+    }
+    UNLOCK(&inode->lock);
 
-        STACK_WIND (frame, ra_truncate_cbk,
-                    FIRST_CHILD (this),
-                    FIRST_CHILD (this)->fops->truncate,
-                    loc, offset, xdata);
-        return 0;
+    STACK_WIND(frame, ra_truncate_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->truncate, loc, offset, xdata);
+    return 0;
 
 unwind:
-        STACK_UNWIND_STRICT (truncate, frame, -1, op_errno, NULL, NULL, NULL);
-        return 0;
+    STACK_UNWIND_STRICT(truncate, frame, -1, op_errno, NULL, NULL, NULL);
+    return 0;
 }
 
-
 void
-ra_page_dump (struct ra_page *page)
+ra_page_dump(struct ra_page *page)
 {
-        int           i                        = 0;
-        call_frame_t *frame                    = NULL;
-        char          key[GF_DUMP_MAX_BUF_LEN] = {0, };
-        ra_waitq_t   *trav                     = NULL;
+    int i = 0;
+    call_frame_t *frame = NULL;
+    char key[GF_DUMP_MAX_BUF_LEN] = {
+        0,
+    };
+    ra_waitq_t *trav = NULL;
 
-        if (page == NULL) {
-                goto out;
-        }
+    if (page == NULL) {
+        goto out;
+    }
 
-        gf_proc_dump_write ("offset", "%"PRId64, page->offset);
+    gf_proc_dump_write("offset", "%" PRId64, page->offset);
 
-        gf_proc_dump_write ("size", "%"PRId64, page->size);
+    gf_proc_dump_write("size", "%" GF_PRI_SIZET, page->size);
 
-        gf_proc_dump_write ("dirty", "%s", page->dirty ? "yes" : "no");
+    gf_proc_dump_write("dirty", "%s", page->dirty ? "yes" : "no");
 
-        gf_proc_dump_write ("poisoned", "%s", page->poisoned ? "yes" : "no");
+    gf_proc_dump_write("poisoned", "%s", page->poisoned ? "yes" : "no");
 
-        gf_proc_dump_write ("ready", "%s", page->ready ? "yes" : "no");
+    gf_proc_dump_write("ready", "%s", page->ready ? "yes" : "no");
 
-        for (trav = page->waitq; trav; trav = trav->next) {
-		frame = trav->data;
-                sprintf (key, "waiting-frame[%d]", i++);
-                gf_proc_dump_write (key, "%"PRId64, frame->root->unique);
-	}
+    for (trav = page->waitq; trav; trav = trav->next) {
+        frame = trav->data;
+        sprintf(key, "waiting-frame[%d]", i++);
+        gf_proc_dump_write(key, "%" PRId64, frame->root->unique);
+    }
 
 out:
-        return;
+    return;
 }
 
 int32_t
-ra_fdctx_dump (xlator_t *this, fd_t *fd)
+ra_fdctx_dump(xlator_t *this, fd_t *fd)
 {
-	ra_file_t    *file     = NULL;
-        ra_page_t    *page     = NULL;
-        int32_t       ret      = 0, i = 0;
-        uint64_t      tmp_file = 0;
-        char         *path     = NULL;
-        char          key[GF_DUMP_MAX_BUF_LEN]        = {0, };
-        char          key_prefix[GF_DUMP_MAX_BUF_LEN] = {0, };
-
-	fd_ctx_get (fd, this, &tmp_file);
-	file = (ra_file_t *)(long)tmp_file;
-
-        if (file == NULL) {
-                ret = 0;
-                goto out;
-        }
+    ra_file_t *file = NULL;
+    ra_page_t *page = NULL;
+    int32_t ret = 0, i = 0;
+    uint64_t tmp_file = 0;
+    char *path = NULL;
+    char key_prefix[GF_DUMP_MAX_BUF_LEN] = {
+        0,
+    };
+
+    fd_ctx_get(fd, this, &tmp_file);
+    file = (ra_file_t *)(long)tmp_file;
+
+    if (file == NULL) {
+        ret = 0;
+        goto out;
+    }
 
-        gf_proc_dump_build_key (key_prefix,
-                                "xlator.performance.read-ahead",
-                                "file");
+    gf_proc_dump_build_key(key_prefix, "xlator.performance.read-ahead", "file");
 
-        gf_proc_dump_add_section (key_prefix);
+    gf_proc_dump_add_section("%s", key_prefix);
 
-        ret = __inode_path (fd->inode, NULL, &path);
-        if (path != NULL) {
-                gf_proc_dump_write ("path", "%s", path);
-                GF_FREE (path);
-        }
+    ret = __inode_path(fd->inode, NULL, &path);
+    if (path != NULL) {
+        gf_proc_dump_write("path", "%s", path);
+        GF_FREE(path);
+    }
 
-        gf_proc_dump_write ("fd", "%p", fd);
+    gf_proc_dump_write("fd", "%p", fd);
 
-        gf_proc_dump_write ("disabled", "%s", file->disabled ? "yes" : "no");
+    gf_proc_dump_write("disabled", "%s", file->disabled ? "yes" : "no");
 
-        if (file->disabled) {
-                ret = 0;
-                goto out;
-        }
+    if (file->disabled) {
+        ret = 0;
+        goto out;
+    }
 
-        gf_proc_dump_write ("page-size", "%"PRId64, file->page_size);
+    gf_proc_dump_write("page-size", "%" PRId64, file->page_size);
 
-        gf_proc_dump_write ("page-count", "%u", file->page_count);
+    gf_proc_dump_write("page-count", "%u", file->page_count);
 
-        gf_proc_dump_write ("next-expected-offset-for-sequential-reads",
-                            "%"PRId64, file->offset);
+    gf_proc_dump_write("next-expected-offset-for-sequential-reads", "%" PRId64,
+                       file->offset);
 
-        for (page = file->pages.next; page != &file->pages;
-             page = page->next) {
-                sprintf (key, "page[%d]", i);
-                gf_proc_dump_write (key, "%p", page[i++]);
-		ra_page_dump (page);
-        }
+    for (page = file->pages.next; page != &file->pages; page = page->next) {
+        gf_proc_dump_write("page", "%d: %p", i++, (void *)page);
+        ra_page_dump(page);
+    }
 
-        ret = 0;
+    ret = 0;
 out:
-        return ret;
+    return ret;
 }
 
 int
-ra_fstat (call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *xdata)
+ra_fstat(call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *xdata)
 {
-        ra_file_t *file     = NULL;
-        fd_t      *iter_fd  = NULL;
-        inode_t   *inode    = NULL;
-        uint64_t   tmp_file = 0;
-        int32_t    op_errno = EINVAL;
+    ra_file_t *file = NULL;
+    fd_t *iter_fd = NULL;
+    inode_t *inode = NULL;
+    uint64_t tmp_file = 0;
+    int32_t op_errno = EINVAL;
+    ra_conf_t *conf = NULL;
 
-        GF_ASSERT (frame);
-        GF_VALIDATE_OR_GOTO (frame->this->name, this, unwind);
-        GF_VALIDATE_OR_GOTO (frame->this->name, fd, unwind);
+    conf = this->private;
 
-        inode = fd->inode;
+    GF_ASSERT(frame);
+    GF_VALIDATE_OR_GOTO(frame->this->name, this, unwind);
+    GF_VALIDATE_OR_GOTO(frame->this->name, fd, unwind);
 
-        LOCK (&inode->lock);
+    inode = fd->inode;
+
+    if (conf->force_atime_update) {
+        LOCK(&inode->lock);
         {
-                list_for_each_entry (iter_fd, &inode->fd_list, inode_list) {
-                        fd_ctx_get (iter_fd, this, &tmp_file);
-                        file = (ra_file_t *)(long)tmp_file;
-
-                        if (!file)
-                                continue;
-                        flush_region (frame, file, 0,
-                                      file->pages.prev->offset + 1, 0);
-                }
+            list_for_each_entry(iter_fd, &inode->fd_list, inode_list)
+            {
+                tmp_file = 0;
+                fd_ctx_get(iter_fd, this, &tmp_file);
+                file = (ra_file_t *)(long)tmp_file;
+
+                if (!file)
+                    continue;
+                flush_region(frame, file, 0, file->pages.prev->offset + 1, 0);
+            }
         }
-        UNLOCK (&inode->lock);
+        UNLOCK(&inode->lock);
+    }
 
-        STACK_WIND (frame, ra_attr_cbk, FIRST_CHILD (this),
-                    FIRST_CHILD (this)->fops->fstat, fd, xdata);
-        return 0;
+    STACK_WIND(frame, ra_attr_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->fstat, fd, xdata);
+    return 0;
 
 unwind:
-        STACK_UNWIND_STRICT (stat, frame, -1, op_errno, NULL, NULL);
-        return 0;
+    STACK_UNWIND_STRICT(stat, frame, -1, op_errno, NULL, NULL);
+    return 0;
 }
 
-
 int
-ra_ftruncate (call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
-              dict_t *xdata)
+ra_ftruncate(call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
+             dict_t *xdata)
 {
-        ra_file_t *file    = NULL;
-        fd_t      *iter_fd = NULL;
-        inode_t   *inode   = NULL;
-        uint64_t  tmp_file = 0;
-        int32_t   op_errno = EINVAL;
+    ra_file_t *file = NULL;
+    fd_t *iter_fd = NULL;
+    inode_t *inode = NULL;
+    uint64_t tmp_file = 0;
+    int32_t op_errno = EINVAL;
 
-        GF_ASSERT (frame);
-        GF_VALIDATE_OR_GOTO (frame->this->name, this, unwind);
-        GF_VALIDATE_OR_GOTO (frame->this->name, fd, unwind);
+    GF_ASSERT(frame);
+    GF_VALIDATE_OR_GOTO(frame->this->name, this, unwind);
+    GF_VALIDATE_OR_GOTO(frame->this->name, fd, unwind);
 
-        inode = fd->inode;
+    inode = fd->inode;
 
-        LOCK (&inode->lock);
+    LOCK(&inode->lock);
+    {
+        list_for_each_entry(iter_fd, &inode->fd_list, inode_list)
         {
-                list_for_each_entry (iter_fd, &inode->fd_list, inode_list) {
-                        fd_ctx_get (iter_fd, this, &tmp_file);
-                        file = (ra_file_t *)(long)tmp_file;
-                        if (!file)
-                                continue;
-                        /*
-                         * Truncation invalidates reads just like writing does.
-                         * TBD: this seems to flush more than it should.  The
-                         * only time we should flush at all is when we're
-                         * shortening (not lengthening) the file, and then only
-                         * from new EOF to old EOF.  The same problem exists in
-                         * ra_truncate.
-                         */
-                        flush_region (frame, file, 0,
-                                      file->pages.prev->offset + 1, 1);
-                }
+            tmp_file = 0;
+            fd_ctx_get(iter_fd, this, &tmp_file);
+            file = (ra_file_t *)(long)tmp_file;
+            if (!file)
+                continue;
+            /*
+             * Truncation invalidates reads just like writing does.
+             * TBD: this seems to flush more than it should.  The
+             * only time we should flush at all is when we're
+             * shortening (not lengthening) the file, and then only
+             * from new EOF to old EOF.  The same problem exists in
+             * ra_truncate.
+             */
+            flush_region(frame, file, 0, file->pages.prev->offset + 1, 1);
         }
-        UNLOCK (&inode->lock);
+    }
+    UNLOCK(&inode->lock);
 
-        STACK_WIND (frame, ra_truncate_cbk, FIRST_CHILD (this),
-                    FIRST_CHILD (this)->fops->ftruncate, fd, offset, xdata);
-        return 0;
+    STACK_WIND(frame, ra_truncate_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->ftruncate, fd, offset, xdata);
+    return 0;
 
 unwind:
-        STACK_UNWIND_STRICT (truncate, frame, -1, op_errno, NULL, NULL, NULL);
-        return 0;
+    STACK_UNWIND_STRICT(truncate, frame, -1, op_errno, NULL, NULL, NULL);
+    return 0;
 }
 
 int
@@ -947,322 +914,359 @@ ra_discard_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
                int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
                struct iatt *postbuf, dict_t *xdata)
 {
-        GF_ASSERT (frame);
+    GF_ASSERT(frame);
 
-        STACK_UNWIND_STRICT (discard, frame, op_ret, op_errno, prebuf,
-                             postbuf, xdata);
-        return 0;
+    STACK_UNWIND_STRICT(discard, frame, op_ret, op_errno, prebuf, postbuf,
+                        xdata);
+    return 0;
 }
 
 static int
 ra_discard(call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
-	     size_t len, dict_t *xdata)
+           size_t len, dict_t *xdata)
 {
-        ra_file_t *file    = NULL;
-        fd_t      *iter_fd = NULL;
-        inode_t   *inode   = NULL;
-        uint64_t  tmp_file = 0;
-        int32_t   op_errno = EINVAL;
+    ra_file_t *file = NULL;
+    fd_t *iter_fd = NULL;
+    inode_t *inode = NULL;
+    uint64_t tmp_file = 0;
+    int32_t op_errno = EINVAL;
 
-        GF_ASSERT (frame);
-        GF_VALIDATE_OR_GOTO (frame->this->name, this, unwind);
-        GF_VALIDATE_OR_GOTO (frame->this->name, fd, unwind);
+    GF_ASSERT(frame);
+    GF_VALIDATE_OR_GOTO(frame->this->name, this, unwind);
+    GF_VALIDATE_OR_GOTO(frame->this->name, fd, unwind);
 
-        inode = fd->inode;
+    inode = fd->inode;
 
-        LOCK (&inode->lock);
+    LOCK(&inode->lock);
+    {
+        list_for_each_entry(iter_fd, &inode->fd_list, inode_list)
         {
-                list_for_each_entry (iter_fd, &inode->fd_list, inode_list) {
-                        fd_ctx_get (iter_fd, this, &tmp_file);
-                        file = (ra_file_t *)(long)tmp_file;
-                        if (!file)
-                                continue;
+            tmp_file = 0;
+            fd_ctx_get(iter_fd, this, &tmp_file);
+            file = (ra_file_t *)(long)tmp_file;
+            if (!file)
+                continue;
 
-                        flush_region(frame, file, offset, len, 1);
-                }
+            flush_region(frame, file, offset, len, 1);
         }
-        UNLOCK (&inode->lock);
+    }
+    UNLOCK(&inode->lock);
 
-        STACK_WIND (frame, ra_discard_cbk, FIRST_CHILD (this),
-                    FIRST_CHILD (this)->fops->discard, fd, offset, len, xdata);
-        return 0;
+    STACK_WIND(frame, ra_discard_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->discard, fd, offset, len, xdata);
+    return 0;
 
 unwind:
-        STACK_UNWIND_STRICT (discard, frame, -1, op_errno, NULL, NULL, NULL);
-        return 0;
+    STACK_UNWIND_STRICT(discard, frame, -1, op_errno, NULL, NULL, NULL);
+    return 0;
 }
 
 int
 ra_zerofill_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
-               int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
-               struct iatt *postbuf, dict_t *xdata)
+                int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
+                struct iatt *postbuf, dict_t *xdata)
 {
-        GF_ASSERT (frame);
+    GF_ASSERT(frame);
 
-        STACK_UNWIND_STRICT (zerofill, frame, op_ret, op_errno, prebuf,
-                             postbuf, xdata);
-        return 0;
+    STACK_UNWIND_STRICT(zerofill, frame, op_ret, op_errno, prebuf, postbuf,
+                        xdata);
+    return 0;
 }
 
 static int
 ra_zerofill(call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
-             off_t len, dict_t *xdata)
+            off_t len, dict_t *xdata)
 {
-        ra_file_t *file    = NULL;
-        fd_t      *iter_fd = NULL;
-        inode_t   *inode   = NULL;
-        uint64_t  tmp_file = 0;
-        int32_t   op_errno = EINVAL;
+    ra_file_t *file = NULL;
+    fd_t *iter_fd = NULL;
+    inode_t *inode = NULL;
+    uint64_t tmp_file = 0;
+    int32_t op_errno = EINVAL;
 
-        GF_ASSERT (frame);
-        GF_VALIDATE_OR_GOTO (frame->this->name, this, unwind);
-        GF_VALIDATE_OR_GOTO (frame->this->name, fd, unwind);
+    GF_ASSERT(frame);
+    GF_VALIDATE_OR_GOTO(frame->this->name, this, unwind);
+    GF_VALIDATE_OR_GOTO(frame->this->name, fd, unwind);
 
-        inode = fd->inode;
+    inode = fd->inode;
 
-        LOCK (&inode->lock);
+    LOCK(&inode->lock);
+    {
+        list_for_each_entry(iter_fd, &inode->fd_list, inode_list)
         {
-                list_for_each_entry (iter_fd, &inode->fd_list, inode_list) {
-                        fd_ctx_get (iter_fd, this, &tmp_file);
-                        file = (ra_file_t *)(long)tmp_file;
-                        if (!file)
-                                continue;
+            tmp_file = 0;
+            fd_ctx_get(iter_fd, this, &tmp_file);
+            file = (ra_file_t *)(long)tmp_file;
+            if (!file)
+                continue;
 
-                        flush_region(frame, file, offset, len, 1);
-                }
+            flush_region(frame, file, offset, len, 1);
         }
-        UNLOCK (&inode->lock);
+    }
+    UNLOCK(&inode->lock);
 
-        STACK_WIND (frame, ra_zerofill_cbk, FIRST_CHILD (this),
-                    FIRST_CHILD (this)->fops->zerofill, fd,
-                    offset, len, xdata);
-        return 0;
+    STACK_WIND(frame, ra_zerofill_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->zerofill, fd, offset, len, xdata);
+    return 0;
 
 unwind:
-        STACK_UNWIND_STRICT (zerofill, frame, -1, op_errno, NULL, NULL, NULL);
-        return 0;
+    STACK_UNWIND_STRICT(zerofill, frame, -1, op_errno, NULL, NULL, NULL);
+    return 0;
 }
 
 int
-ra_priv_dump (xlator_t *this)
+ra_priv_dump(xlator_t *this)
 {
-        ra_conf_t       *conf                           = NULL;
-        int             ret                             = -1;
-        char            key_prefix[GF_DUMP_MAX_BUF_LEN] = {0, };
-        gf_boolean_t    add_section                     = _gf_false;
-
-        if (!this) {
-                goto out;
-        }
-
-        conf = this->private;
-        if (!conf) {
-                gf_log (this->name, GF_LOG_WARNING, "conf null in xlator");
-                goto out;
-        }
-
-        gf_proc_dump_build_key (key_prefix, "xlator.performance.read-ahead",
-                                "priv");
-
-        gf_proc_dump_add_section (key_prefix);
-        add_section = _gf_true;
-
-        ret = pthread_mutex_trylock (&conf->conf_lock);
-        if (ret)
-                goto out;
-        {
-                gf_proc_dump_write ("page_size", "%d", conf->page_size);
-                gf_proc_dump_write ("page_count", "%d", conf->page_count);
-                gf_proc_dump_write ("force_atime_update", "%d",
-                                    conf->force_atime_update);
-        }
-        pthread_mutex_unlock (&conf->conf_lock);
-
-        ret = 0;
+    ra_conf_t *conf = NULL;
+    int ret = -1;
+    char key_prefix[GF_DUMP_MAX_BUF_LEN] = {
+        0,
+    };
+
+    if (!this) {
+        goto out;
+    }
+
+    conf = this->private;
+    if (!conf) {
+        gf_msg(this->name, GF_LOG_WARNING, 0, READ_AHEAD_MSG_XLATOR_CONF_NULL,
+               "conf null in xlator");
+        goto out;
+    }
+
+    gf_proc_dump_build_key(key_prefix, "xlator.performance.read-ahead", "priv");
+
+    gf_proc_dump_add_section("%s", key_prefix);
+
+    ret = pthread_mutex_trylock(&conf->conf_lock);
+    if (ret)
+        goto out;
+    {
+        gf_proc_dump_write("page_size", "%" PRIu64, conf->page_size);
+        gf_proc_dump_write("page_count", "%d", conf->page_count);
+        gf_proc_dump_write("force_atime_update", "%d",
+                           conf->force_atime_update);
+    }
+    pthread_mutex_unlock(&conf->conf_lock);
+
+    ret = 0;
 out:
-        if (ret && conf) {
-                if (add_section == _gf_false)
-                        gf_proc_dump_add_section (key_prefix);
-
-                gf_proc_dump_write ("Unable to dump priv",
-                                    "(Lock acquisition failed) %s", this->name);
-        }
-        return ret;
+    if (ret && conf) {
+        gf_proc_dump_write("Unable to dump priv",
+                           "(Lock acquisition failed) %s", this->name);
+    }
+    return ret;
 }
 
-
 int32_t
-mem_acct_init (xlator_t *this)
+mem_acct_init(xlator_t *this)
 {
-        int     ret = -1;
+    int ret = -1;
 
-        if (!this) {
-                goto out;
-        }
+    if (!this) {
+        goto out;
+    }
 
-        ret = xlator_mem_acct_init (this, gf_ra_mt_end + 1);
+    ret = xlator_mem_acct_init(this, gf_ra_mt_end + 1);
 
-        if (ret != 0) {
-                gf_log (this->name, GF_LOG_ERROR, "Memory accounting init"
-                        "failed");
-        }
+    if (ret != 0) {
+        gf_msg(this->name, GF_LOG_ERROR, ENOMEM, READ_AHEAD_MSG_NO_MEMORY,
+               "Memory accounting init"
+               "failed");
+    }
 
 out:
-        return ret;
+    return ret;
 }
 
 int
-reconfigure (xlator_t *this, dict_t *options)
+reconfigure(xlator_t *this, dict_t *options)
 {
-        ra_conf_t      *conf = NULL;
-        int             ret = -1;
+    ra_conf_t *conf = NULL;
+    int ret = -1;
 
-        GF_VALIDATE_OR_GOTO ("read-ahead", this, out);
-        GF_VALIDATE_OR_GOTO ("read-ahead", this->private, out);
+    GF_VALIDATE_OR_GOTO("read-ahead", this, out);
+    GF_VALIDATE_OR_GOTO("read-ahead", this->private, out);
 
-        conf = this->private;
+    conf = this->private;
 
-        GF_OPTION_RECONF ("page-count", conf->page_count, options, uint32, out);
+    GF_OPTION_RECONF("page-count", conf->page_count, options, uint32, out);
 
-        GF_OPTION_RECONF ("page-size", conf->page_size, options, size_uint64,
-                          out);
+    GF_OPTION_RECONF("page-size", conf->page_size, options, size_uint64, out);
 
-        ret = 0;
- out:
-        return ret;
+    GF_OPTION_RECONF("pass-through", this->pass_through, options, bool, out);
+
+    ret = 0;
+out:
+    return ret;
 }
 
 int
-init (xlator_t *this)
+init(xlator_t *this)
 {
-        ra_conf_t *conf              = NULL;
-        int32_t    ret               = -1;
+    ra_conf_t *conf = NULL;
+    int32_t ret = -1;
 
-        GF_VALIDATE_OR_GOTO ("read-ahead", this, out);
+    GF_VALIDATE_OR_GOTO("read-ahead", this, out);
 
-        if (!this->children || this->children->next) {
-                gf_log (this->name,  GF_LOG_ERROR,
-                        "FATAL: read-ahead not configured with exactly one"
-                        " child");
-                goto out;
-        }
+    if (!this->children || this->children->next) {
+        gf_msg(this->name, GF_LOG_ERROR, 0,
+               READ_AHEAD_MSG_XLATOR_CHILD_MISCONFIGURED,
+               "FATAL: read-ahead not configured with exactly one"
+               " child");
+        goto out;
+    }
 
-        if (!this->parents) {
-                gf_log (this->name, GF_LOG_WARNING,
-                        "dangling volume. check volfile ");
-        }
+    if (!this->parents) {
+        gf_msg(this->name, GF_LOG_WARNING, 0, READ_AHEAD_MSG_VOL_MISCONFIGURED,
+               "dangling volume. check volfile ");
+    }
 
-        conf = (void *) GF_CALLOC (1, sizeof (*conf), gf_ra_mt_ra_conf_t);
-        if (conf == NULL) {
-                goto out;
-        }
+    conf = (void *)GF_CALLOC(1, sizeof(*conf), gf_ra_mt_ra_conf_t);
+    if (conf == NULL) {
+        goto out;
+    }
 
-        conf->page_size = this->ctx->page_size;
+    conf->page_size = this->ctx->page_size;
 
-        GF_OPTION_INIT ("page-size", conf->page_size, size_uint64, out);
+    GF_OPTION_INIT("page-size", conf->page_size, size_uint64, out);
 
-        GF_OPTION_INIT ("page-count", conf->page_count, uint32, out);
+    GF_OPTION_INIT("page-count", conf->page_count, uint32, out);
 
-        GF_OPTION_INIT ("force-atime-update", conf->force_atime_update, bool, out);
+    GF_OPTION_INIT("force-atime-update", conf->force_atime_update, bool, out);
 
-        conf->files.next = &conf->files;
-        conf->files.prev = &conf->files;
+    GF_OPTION_INIT("pass-through", this->pass_through, bool, out);
 
-        pthread_mutex_init (&conf->conf_lock, NULL);
+    conf->files.next = &conf->files;
+    conf->files.prev = &conf->files;
 
-        this->local_pool = mem_pool_new (ra_local_t, 64);
-        if (!this->local_pool) {
-                ret = -1;
-                gf_log (this->name, GF_LOG_ERROR,
-                        "failed to create local_t's memory pool");
-                goto out;
-        }
+    pthread_mutex_init(&conf->conf_lock, NULL);
 
-        this->private = conf;
-        ret = 0;
+    this->local_pool = mem_pool_new(ra_local_t, 64);
+    if (!this->local_pool) {
+        ret = -1;
+        gf_msg(this->name, GF_LOG_ERROR, ENOMEM, READ_AHEAD_MSG_NO_MEMORY,
+               "failed to create local_t's memory pool");
+        goto out;
+    }
+
+    this->private = conf;
+    ret = 0;
 
 out:
-        if (ret == -1) {
-                GF_FREE (conf);
-        }
+    if (ret == -1) {
+        GF_FREE(conf);
+    }
 
-        return ret;
+    return ret;
 }
 
-
 void
-fini (xlator_t *this)
+fini(xlator_t *this)
 {
-        ra_conf_t *conf = NULL;
+    ra_conf_t *conf = NULL;
 
-        GF_VALIDATE_OR_GOTO ("read-ahead", this, out);
+    GF_VALIDATE_OR_GOTO("read-ahead", this, out);
 
-        conf = this->private;
-        if (conf == NULL) {
-                goto out;
-        }
+    conf = this->private;
+    if (conf == NULL) {
+        goto out;
+    }
 
-        this->private = NULL;
-
-        /* The files structures allocated in open and create are not deleted.
-         * until that is freed, marking the below assert as warning.
-        GF_ASSERT ((conf->files.next == &conf->files)
-                   && (conf->files.prev == &conf->files));
-        */
-        if (!((conf->files.next == &conf->files)
-                   && (conf->files.prev == &conf->files))) {
-                gf_log (this->name, GF_LOG_INFO,
-                       "undestroyed read ahead file structures found");
-        }
+    this->private = NULL;
+
+    /* The files structures allocated in open and create are not deleted.
+     * until that is freed, marking the below assert as warning.
+    GF_ASSERT ((conf->files.next == &conf->files)
+               && (conf->files.prev == &conf->files));
+    */
+    if (!((conf->files.next == &conf->files) &&
+          (conf->files.prev == &conf->files))) {
+        gf_msg(this->name, GF_LOG_INFO, 0,
+               READ_AHEAD_MSG_UNDESTROYED_FILE_FOUND,
+               "undestroyed read ahead file structures found");
+    }
 
-        pthread_mutex_destroy (&conf->conf_lock);
-        GF_FREE (conf);
+    pthread_mutex_destroy(&conf->conf_lock);
+    GF_FREE(conf);
 
 out:
-        return;
+    return;
 }
 
 struct xlator_fops fops = {
-        .open        = ra_open,
-        .create      = ra_create,
-        .readv       = ra_readv,
-        .writev      = ra_writev,
-        .flush       = ra_flush,
-        .fsync       = ra_fsync,
-        .truncate    = ra_truncate,
-        .ftruncate   = ra_ftruncate,
-        .fstat       = ra_fstat,
-	.discard     = ra_discard,
-        .zerofill    = ra_zerofill,
+    .open = ra_open,
+    .create = ra_create,
+    .readv = ra_readv,
+    .writev = ra_writev,
+    .flush = ra_flush,
+    .fsync = ra_fsync,
+    .truncate = ra_truncate,
+    .ftruncate = ra_ftruncate,
+    .fstat = ra_fstat,
+    .discard = ra_discard,
+    .zerofill = ra_zerofill,
 };
 
 struct xlator_cbks cbks = {
-        .release       = ra_release,
+    .release = ra_release,
 };
 
 struct xlator_dumpops dumpops = {
-        .priv      =  ra_priv_dump,
-        .fdctx     =  ra_fdctx_dump,
+    .priv = ra_priv_dump,
+    .fdctx = ra_fdctx_dump,
 };
 
 struct volume_options options[] = {
-        { .key  = {"force-atime-update"},
-          .type = GF_OPTION_TYPE_BOOL,
-          .default_value = "false"
-        },
-        { .key  = {"page-count"},
-          .type = GF_OPTION_TYPE_INT,
-          .min  = 1,
-          .max  = 16,
-          .default_value = "4",
-          .description = "Number of pages that will be pre-fetched"
-        },
-	{ .key = {"page-size"},
-	  .type = GF_OPTION_TYPE_SIZET,
-	  .min = 4096,
-	  .max = 1048576 * 64,
-	  .default_value = "131072",
-	  .description = "Page size with which read-ahead performs server I/O"
-	},
-        { .key = {NULL} },
+    {
+        .key = {"read-ahead"},
+        .type = GF_OPTION_TYPE_BOOL,
+        .default_value = "off",
+        .description = "enable/disable read-ahead",
+        .op_version = {GD_OP_VERSION_6_0},
+        .flags = OPT_FLAG_SETTABLE,
+    },
+    {.key = {"force-atime-update"},
+     .type = GF_OPTION_TYPE_BOOL,
+     .op_version = {1},
+     .tags = {"read-ahead"},
+     .default_value = "false"},
+    {.key = {"page-count"},
+     .type = GF_OPTION_TYPE_INT,
+     .min = 1,
+     .max = 16,
+     .default_value = "4",
+     .op_version = {1},
+     .tags = {"read-ahead"},
+     .description = "Number of pages that will be pre-fetched"},
+    {.key = {"page-size"},
+     .type = GF_OPTION_TYPE_SIZET,
+     .min = 4096,
+     .max = 1048576 * 64,
+     .default_value = "131072",
+     .op_version = {1},
+     .flags = OPT_FLAG_SETTABLE | OPT_FLAG_DOC | OPT_FLAG_CLIENT_OPT,
+     .tags = {"read-ahead"},
+     .description = "Page size with which read-ahead performs server I/O"},
+    {.key = {"pass-through"},
+     .type = GF_OPTION_TYPE_BOOL,
+     .default_value = "false",
+     .op_version = {GD_OP_VERSION_4_1_0},
+     .flags = OPT_FLAG_SETTABLE | OPT_FLAG_DOC | OPT_FLAG_CLIENT_OPT,
+     .tags = {"read-ahead"},
+     .description = "Enable/Disable read ahead translator"},
+    {.key = {NULL}},
+};
+
+xlator_api_t xlator_api = {
+    .init = init,
+    .fini = fini,
+    .reconfigure = reconfigure,
+    .mem_acct_init = mem_acct_init,
+    .op_version = {1}, /* Present from the initial version */
+    .dumpops = &dumpops,
+    .fops = &fops,
+    .cbks = &cbks,
+    .options = options,
+    .identifier = "read-ahead",
+    .category = GF_MAINTAINED,
 };
diff --git a/xlators/performance/read-ahead/src/read-ahead.h b/xlators/performance/read-ahead/src/read-ahead.h
index d1d768c34fb..e9432fb47cc 100644
--- a/xlators/performance/read-ahead/src/read-ahead.h
+++ b/xlators/performance/read-ahead/src/read-ahead.h
@@ -11,17 +11,11 @@
 #ifndef __READ_AHEAD_H
 #define __READ_AHEAD_H
 
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
-
-
-#include "glusterfs.h"
-#include "logging.h"
-#include "dict.h"
-#include "xlator.h"
-#include "common-utils.h"
+#include <glusterfs/glusterfs.h>
+#include <glusterfs/logging.h>
+#include <glusterfs/dict.h>
+#include <glusterfs/xlator.h>
+#include <glusterfs/common-utils.h>
 #include "read-ahead-mem-types.h"
 
 struct ra_conf;
@@ -30,84 +24,77 @@ struct ra_page;
 struct ra_file;
 struct ra_waitq;
 
-
 struct ra_waitq {
-        struct ra_waitq *next;
-        void            *data;
+    struct ra_waitq *next;
+    void *data;
 };
 
-
 struct ra_fill {
-        struct ra_fill *next;
-        struct ra_fill *prev;
-        off_t           offset;
-        size_t          size;
-        struct iovec   *vector;
-        int32_t         count;
-        struct iobref  *iobref;
+    struct ra_fill *next;
+    struct ra_fill *prev;
+    off_t offset;
+    size_t size;
+    struct iovec *vector;
+    int32_t count;
+    struct iobref *iobref;
 };
 
-
 struct ra_local {
-        mode_t            mode;
-        struct ra_fill    fill;
-        off_t             offset;
-        size_t            size;
-        int32_t           op_ret;
-        int32_t           op_errno;
-        off_t             pending_offset;
-        size_t            pending_size;
-        fd_t             *fd;
-        int32_t           wait_count;
-        pthread_mutex_t   local_lock;
+    mode_t mode;
+    struct ra_fill fill;
+    off_t offset;
+    size_t size;
+    int32_t op_ret;
+    int32_t op_errno;
+    off_t pending_offset;
+    size_t pending_size;
+    fd_t *fd;
+    int32_t wait_count;
+    pthread_mutex_t local_lock;
 };
 
-
 struct ra_page {
-        struct ra_page   *next;
-        struct ra_page   *prev;
-        struct ra_file   *file;
-        char              dirty;    /* Internal request, not from user. */
-        char              poisoned; /* Pending read invalidated by write. */
-        char              ready;
-        struct iovec     *vector;
-        int32_t           count;
-        off_t             offset;
-        size_t            size;
-        struct ra_waitq  *waitq;
-        struct iobref    *iobref;
-        char              stale;
+    struct ra_page *next;
+    struct ra_page *prev;
+    struct ra_file *file;
+    char dirty;    /* Internal request, not from user. */
+    char poisoned; /* Pending read invalidated by write. */
+    char ready;
+    struct iovec *vector;
+    int32_t count;
+    off_t offset;
+    size_t size;
+    struct ra_waitq *waitq;
+    struct iobref *iobref;
+    char stale;
 };
 
-
 struct ra_file {
-        struct ra_file    *next;
-        struct ra_file    *prev;
-        struct ra_conf    *conf;
-        fd_t              *fd;
-        int                disabled;
-        size_t             expected;
-        struct ra_page     pages;
-        off_t              offset;
-        size_t             size;
-        int32_t            refcount;
-        pthread_mutex_t    file_lock;
-        struct iatt        stbuf;
-        uint64_t           page_size;
-        uint32_t           page_count;
+    struct ra_file *next;
+    struct ra_file *prev;
+    struct ra_conf *conf;
+    fd_t *fd;
+    int disabled;
+    size_t expected;
+    struct ra_page pages;
+    off_t offset;
+    size_t size;
+    int32_t refcount;
+    pthread_mutex_t file_lock;
+    struct iatt stbuf;
+    uint64_t page_size;
+    uint32_t page_count;
 };
 
-
 struct ra_conf {
-        uint64_t          page_size;
-        uint32_t          page_count;
-        void             *cache_block;
-        struct ra_file    files;
-        gf_boolean_t      force_atime_update;
-        pthread_mutex_t   conf_lock;
+    uint64_t page_size;
+    uint32_t page_count;
+    void *cache_block;
+    struct ra_file files;
+    gf_boolean_t force_atime_update;
+    pthread_mutex_t conf_lock;
 };
 
-
 typedef struct ra_conf ra_conf_t;
 typedef struct ra_local ra_local_t;
 typedef struct ra_page ra_page_t;
@@ -116,77 +103,69 @@ typedef struct ra_waitq ra_waitq_t;
 typedef struct ra_fill ra_fill_t;
 
 ra_page_t *
-ra_page_get (ra_file_t *file,
-             off_t offset);
+ra_page_get(ra_file_t *file, off_t offset);
 
 ra_page_t *
-ra_page_create (ra_file_t *file,
-                off_t offset);
+ra_page_create(ra_file_t *file, off_t offset);
 
 void
-ra_page_fault (ra_file_t *file,
-               call_frame_t *frame,
-               off_t offset);
+ra_page_fault(ra_file_t *file, call_frame_t *frame, off_t offset);
 void
-ra_wait_on_page (ra_page_t *page,
-                 call_frame_t *frame);
+ra_wait_on_page(ra_page_t *page, call_frame_t *frame);
 
 ra_waitq_t *
-ra_page_wakeup (ra_page_t *page);
+ra_page_wakeup(ra_page_t *page);
 
 void
-ra_page_flush (ra_page_t *page);
+ra_page_flush(ra_page_t *page);
 
 ra_waitq_t *
-ra_page_error (ra_page_t *page,
-               int32_t op_ret,
-               int32_t op_errno);
+ra_page_error(ra_page_t *page, int32_t op_ret, int32_t op_errno);
 void
-ra_page_purge (ra_page_t *page);
+ra_page_purge(ra_page_t *page);
 
 void
-ra_frame_return (call_frame_t *frame);
+ra_frame_return(call_frame_t *frame);
 
 void
-ra_frame_fill (ra_page_t *page,
-               call_frame_t *frame);
+ra_frame_fill(ra_page_t *page, call_frame_t *frame);
 
 void
-ra_file_destroy (ra_file_t *file);
+ra_file_destroy(ra_file_t *file);
 
 static inline void
-ra_file_lock (ra_file_t *file)
+ra_file_lock(ra_file_t *file)
 {
-        pthread_mutex_lock (&file->file_lock);
+    pthread_mutex_lock(&file->file_lock);
 }
 
 static inline void
-ra_file_unlock (ra_file_t *file)
+ra_file_unlock(ra_file_t *file)
 {
-        pthread_mutex_unlock (&file->file_lock);
+    pthread_mutex_unlock(&file->file_lock);
 }
 
 static inline void
-ra_conf_lock (ra_conf_t *conf)
+ra_conf_lock(ra_conf_t *conf)
 {
-        pthread_mutex_lock (&conf->conf_lock);
+    pthread_mutex_lock(&conf->conf_lock);
 }
 
 static inline void
-ra_conf_unlock (ra_conf_t *conf)
+ra_conf_unlock(ra_conf_t *conf)
 {
-        pthread_mutex_unlock (&conf->conf_lock);
+    pthread_mutex_unlock(&conf->conf_lock);
 }
 static inline void
-ra_local_lock (ra_local_t *local)
+ra_local_lock(ra_local_t *local)
 {
-        pthread_mutex_lock (&local->local_lock);
+    pthread_mutex_lock(&local->local_lock);
 }
 
 static inline void
-ra_local_unlock (ra_local_t *local)
+ra_local_unlock(ra_local_t *local)
 {
-        pthread_mutex_unlock (&local->local_lock);
+    pthread_mutex_unlock(&local->local_lock);
 }
 
 #endif /* __READ_AHEAD_H */
diff --git a/xlators/performance/readdir-ahead/src/Makefile.am b/xlators/performance/readdir-ahead/src/Makefile.am
index 539d6ede434..3d6b6ae951f 100644
--- a/xlators/performance/readdir-ahead/src/Makefile.am
+++ b/xlators/performance/readdir-ahead/src/Makefile.am
@@ -1,14 +1,17 @@
 xlator_LTLIBRARIES = readdir-ahead.la
 xlatordir = $(libdir)/glusterfs/$(PACKAGE_VERSION)/xlator/performance
 
-readdir_ahead_la_LDFLAGS = -module -avoid-version
+readdir_ahead_la_LDFLAGS = -module $(GF_XLATOR_DEFAULT_LDFLAGS)
 
 readdir_ahead_la_SOURCES = readdir-ahead.c
 readdir_ahead_la_LIBADD = $(top_builddir)/libglusterfs/src/libglusterfs.la
 
-noinst_HEADERS = readdir-ahead.h readdir-ahead-mem-types.h
+noinst_HEADERS = readdir-ahead.h readdir-ahead-mem-types.h \
+	readdir-ahead-messages.h
+
+AM_CPPFLAGS = $(GF_CPPFLAGS) -I$(top_srcdir)/libglusterfs/src \
+	-I$(top_srcdir)/rpc/xdr/src -I$(top_builddir)/rpc/xdr/src
 
-AM_CPPFLAGS = $(GF_CPPFLAGS) -I$(top_srcdir)/libglusterfs/src
 
 AM_CFLAGS = -Wall $(GF_CFLAGS)
 
diff --git a/xlators/performance/readdir-ahead/src/readdir-ahead-mem-types.h b/xlators/performance/readdir-ahead/src/readdir-ahead-mem-types.h
index 39e2c536975..498ffae7f64 100644
--- a/xlators/performance/readdir-ahead/src/readdir-ahead-mem-types.h
+++ b/xlators/performance/readdir-ahead/src/readdir-ahead-mem-types.h
@@ -8,17 +8,17 @@
   cases as published by the Free Software Foundation.
 */
 
-
 #ifndef __RDA_MEM_TYPES_H__
 #define __RDA_MEM_TYPES_H__
 
-#include "mem-types.h"
+#include <glusterfs/mem-types.h>
 
 enum gf_rda_mem_types_ {
-        gf_rda_mt_rda_local   = gf_common_mt_end + 1,
-	gf_rda_mt_rda_fd_ctx,
-	gf_rda_mt_rda_priv,
-        gf_rda_mt_end
+    gf_rda_mt_rda_local = gf_common_mt_end + 1,
+    gf_rda_mt_rda_fd_ctx,
+    gf_rda_mt_rda_priv,
+    gf_rda_mt_inode_ctx_t,
+    gf_rda_mt_end
 };
 
 #endif
diff --git a/xlators/performance/readdir-ahead/src/readdir-ahead-messages.h b/xlators/performance/readdir-ahead/src/readdir-ahead-messages.h
new file mode 100644
index 00000000000..28ec14dd845
--- /dev/null
+++ b/xlators/performance/readdir-ahead/src/readdir-ahead-messages.h
@@ -0,0 +1,30 @@
+/*Copyright (c) 2015 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
+*/
+
+#ifndef _READDIR_AHEAD_MESSAGES_H_
+#define _READDIR_AHEAD_MESSAGES_H_
+
+#include <glusterfs/glfs-message-id.h>
+
+/* To add new message IDs, append new identifiers at the end of the list.
+ *
+ * Never remove a message ID. If it's not used anymore, you can rename it or
+ * leave it as it is, but not delete it. This is to prevent reutilization of
+ * IDs by other messages.
+ *
+ * The component name must match one of the entries defined in
+ * glfs-message-id.h.
+ */
+
+GLFS_MSGID(READDIR_AHEAD, READDIR_AHEAD_MSG_XLATOR_CHILD_MISCONFIGURED,
+           READDIR_AHEAD_MSG_VOL_MISCONFIGURED, READDIR_AHEAD_MSG_NO_MEMORY,
+           READDIR_AHEAD_MSG_DIR_RELEASE_PENDING_STUB,
+           READDIR_AHEAD_MSG_OUT_OF_SEQUENCE, READDIR_AHEAD_MSG_DICT_OP_FAILED);
+
+#endif /* _READDIR_AHEAD_MESSAGES_H_ */
diff --git a/xlators/performance/readdir-ahead/src/readdir-ahead.c b/xlators/performance/readdir-ahead/src/readdir-ahead.c
index 1a567a03ea8..4ba7ee7077a 100644
--- a/xlators/performance/readdir-ahead/src/readdir-ahead.c
+++ b/xlators/performance/readdir-ahead/src/readdir-ahead.c
@@ -23,68 +23,242 @@
  * preloads on the directory.
  */
 
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
-
-#include "glusterfs.h"
-#include "xlator.h"
-#include "call-stub.h"
+#include <math.h>
+#include <glusterfs/glusterfs.h>
+#include <glusterfs/xlator.h>
+#include <glusterfs/call-stub.h>
 #include "readdir-ahead.h"
 #include "readdir-ahead-mem-types.h"
-#include "defaults.h"
+#include <glusterfs/defaults.h>
+#include "readdir-ahead-messages.h"
+static int
+rda_fill_fd(call_frame_t *, xlator_t *, fd_t *);
 
-static int rda_fill_fd(call_frame_t *, xlator_t *, fd_t *);
+static void
+rda_local_wipe(struct rda_local *local)
+{
+    if (local->fd)
+        fd_unref(local->fd);
+    if (local->xattrs)
+        dict_unref(local->xattrs);
+    if (local->inode)
+        inode_unref(local->inode);
+}
 
 /*
  * Get (or create) the fd context for storing prepopulated directory
  * entries.
  */
-static struct
-rda_fd_ctx *get_rda_fd_ctx(fd_t *fd, xlator_t *this)
-{
-	uint64_t val;
-	struct rda_fd_ctx *ctx;
-
-	LOCK(&fd->lock);
-
-	if (__fd_ctx_get(fd, this, &val) < 0) {
-		ctx = GF_CALLOC(1, sizeof(struct rda_fd_ctx),
-				gf_rda_mt_rda_fd_ctx);
-		if (!ctx)
-			goto out;
-
-		LOCK_INIT(&ctx->lock);
-		INIT_LIST_HEAD(&ctx->entries.list);
-		ctx->state = RDA_FD_NEW;
-		/* ctx offset values initialized to 0 */
-
-		if (__fd_ctx_set(fd, this, (uint64_t) ctx) < 0) {
-			GF_FREE(ctx);
-			ctx = NULL;
-			goto out;
-		}
-	} else {
-		ctx = (struct rda_fd_ctx *) val;
-	}
+static struct rda_fd_ctx *
+get_rda_fd_ctx(fd_t *fd, xlator_t *this)
+{
+    uint64_t val;
+    struct rda_fd_ctx *ctx;
+
+    LOCK(&fd->lock);
+
+    if (__fd_ctx_get(fd, this, &val) < 0) {
+        ctx = GF_CALLOC(1, sizeof(struct rda_fd_ctx), gf_rda_mt_rda_fd_ctx);
+        if (!ctx)
+            goto out;
+
+        LOCK_INIT(&ctx->lock);
+        INIT_LIST_HEAD(&ctx->entries.list);
+        ctx->state = RDA_FD_NEW;
+        /* ctx offset values initialized to 0 */
+        ctx->xattrs = NULL;
+
+        if (__fd_ctx_set(fd, this, (uint64_t)(uintptr_t)ctx) < 0) {
+            GF_FREE(ctx);
+            ctx = NULL;
+            goto out;
+        }
+    } else {
+        ctx = (struct rda_fd_ctx *)(uintptr_t)val;
+    }
+out:
+    UNLOCK(&fd->lock);
+    return ctx;
+}
+
+static rda_inode_ctx_t *
+__rda_inode_ctx_get(inode_t *inode, xlator_t *this)
+{
+    int ret = -1;
+    uint64_t ctx_uint = 0;
+    rda_inode_ctx_t *ctx_p = NULL;
+
+    ret = __inode_ctx_get1(inode, this, &ctx_uint);
+    if (ret == 0)
+        return (rda_inode_ctx_t *)(uintptr_t)ctx_uint;
+
+    ctx_p = GF_CALLOC(1, sizeof(*ctx_p), gf_rda_mt_inode_ctx_t);
+    if (!ctx_p)
+        return NULL;
+
+    GF_ATOMIC_INIT(ctx_p->generation, 0);
+
+    ctx_uint = (uint64_t)(uintptr_t)ctx_p;
+    ret = __inode_ctx_set1(inode, this, &ctx_uint);
+    if (ret < 0) {
+        GF_FREE(ctx_p);
+        return NULL;
+    }
+
+    return ctx_p;
+}
+
+static int
+__rda_inode_ctx_update_iatts(inode_t *inode, xlator_t *this,
+                             struct iatt *stbuf_in, struct iatt *stbuf_out,
+                             uint64_t generation)
+{
+    rda_inode_ctx_t *ctx_p = NULL;
+    struct iatt tmp_stat = {
+        0,
+    };
+
+    ctx_p = __rda_inode_ctx_get(inode, this);
+    if (!ctx_p)
+        return -1;
+
+    if ((!stbuf_in) || (stbuf_in->ia_ctime == 0)) {
+        /* A fop modified a file but valid stbuf is not provided.
+         * Can't update iatt to reflect results of fop and hence
+         * invalidate the iatt stored in dentry.
+         *
+         * An example of this case can be response of write request
+         * that is cached in write-behind.
+         */
+        if (stbuf_in)
+            tmp_stat = *stbuf_in;
+        else
+            tmp_stat = ctx_p->statbuf;
+        memset(&ctx_p->statbuf, 0, sizeof(ctx_p->statbuf));
+        gf_uuid_copy(ctx_p->statbuf.ia_gfid, tmp_stat.ia_gfid);
+        ctx_p->statbuf.ia_type = tmp_stat.ia_type;
+        GF_ATOMIC_INC(ctx_p->generation);
+    } else {
+        if (ctx_p->statbuf.ia_ctime) {
+            if (stbuf_in->ia_ctime < ctx_p->statbuf.ia_ctime) {
+                goto out;
+            }
+
+            if ((stbuf_in->ia_ctime == ctx_p->statbuf.ia_ctime) &&
+                (stbuf_in->ia_ctime_nsec < ctx_p->statbuf.ia_ctime_nsec)) {
+                goto out;
+            }
+        } else {
+            if ((generation != -1) &&
+                (generation != GF_ATOMIC_GET(ctx_p->generation)))
+                goto out;
+        }
+
+        ctx_p->statbuf = *stbuf_in;
+    }
+
 out:
-	UNLOCK(&fd->lock);
-	return ctx;
+    if (stbuf_out)
+        *stbuf_out = ctx_p->statbuf;
+
+    return 0;
+}
+
+static int
+rda_inode_ctx_update_iatts(inode_t *inode, xlator_t *this,
+                           struct iatt *stbuf_in, struct iatt *stbuf_out,
+                           uint64_t generation)
+{
+    int ret = -1;
+
+    LOCK(&inode->lock);
+    {
+        ret = __rda_inode_ctx_update_iatts(inode, this, stbuf_in, stbuf_out,
+                                           generation);
+    }
+    UNLOCK(&inode->lock);
+
+    return ret;
 }
 
 /*
  * Reset the tracking state of the context.
  */
 static void
-rda_reset_ctx(struct rda_fd_ctx *ctx)
+rda_reset_ctx(xlator_t *this, struct rda_fd_ctx *ctx)
 {
-	ctx->state = RDA_FD_NEW;
-	ctx->cur_offset = 0;
-	ctx->cur_size = 0;
-	ctx->next_offset = 0;
-        ctx->op_errno = 0;
-	gf_dirent_free(&ctx->entries);
+    struct rda_priv *priv = NULL;
+
+    priv = this->private;
+
+    ctx->state = RDA_FD_NEW;
+    ctx->cur_offset = 0;
+    ctx->next_offset = 0;
+    ctx->op_errno = 0;
+
+    gf_dirent_free(&ctx->entries);
+    GF_ATOMIC_SUB(priv->rda_cache_size, ctx->cur_size);
+    ctx->cur_size = 0;
+
+    if (ctx->xattrs) {
+        dict_unref(ctx->xattrs);
+        ctx->xattrs = NULL;
+    }
+}
+
+static void
+rda_mark_inode_dirty(xlator_t *this, inode_t *inode)
+{
+    inode_t *parent = NULL;
+    fd_t *fd = NULL;
+    uint64_t val = 0;
+    int32_t ret = 0;
+    struct rda_fd_ctx *fd_ctx = NULL;
+    char gfid[GF_UUID_BUF_SIZE] = {0};
+
+    parent = inode_parent(inode, NULL, NULL);
+    if (parent) {
+        LOCK(&parent->lock);
+        {
+            list_for_each_entry(fd, &parent->fd_list, inode_list)
+            {
+                val = 0;
+                fd_ctx_get(fd, this, &val);
+                if (val == 0)
+                    continue;
+
+                fd_ctx = (void *)(uintptr_t)val;
+                uuid_utoa_r(inode->gfid, gfid);
+                if (!GF_ATOMIC_GET(fd_ctx->prefetching))
+                    continue;
+
+                LOCK(&fd_ctx->lock);
+                {
+                    if (GF_ATOMIC_GET(fd_ctx->prefetching)) {
+                        if (fd_ctx->writes_during_prefetch == NULL)
+                            fd_ctx->writes_during_prefetch = dict_new();
+
+                        ret = dict_set_int8(fd_ctx->writes_during_prefetch,
+                                            gfid, 1);
+                        if (ret < 0) {
+                            gf_log(this->name, GF_LOG_WARNING,
+                                   "marking to invalidate stats of %s from an "
+                                   "in progress "
+                                   "prefetching has failed, might result in "
+                                   "stale stat to "
+                                   "application",
+                                   gfid);
+                        }
+                    }
+                }
+                UNLOCK(&fd_ctx->lock);
+            }
+        }
+        UNLOCK(&parent->lock);
+        inode_unref(parent);
+    }
+
+    return;
 }
 
 /*
@@ -95,12 +269,33 @@ rda_reset_ctx(struct rda_fd_ctx *ctx)
 static gf_boolean_t
 rda_can_serve_readdirp(struct rda_fd_ctx *ctx, size_t request_size)
 {
-	if ((ctx->state & RDA_FD_EOD) ||
-	    (ctx->state & RDA_FD_ERROR) ||
-	    (!(ctx->state & RDA_FD_PLUGGED) && (ctx->cur_size > 0)))
-		return _gf_true;
+    if ((ctx->state & RDA_FD_EOD) || (ctx->state & RDA_FD_ERROR) ||
+        (!(ctx->state & RDA_FD_PLUGGED) && (ctx->cur_size > 0)) ||
+        (request_size && ctx->cur_size >= request_size))
+        return _gf_true;
+
+    return _gf_false;
+}
+
+void
+rda_inode_ctx_get_iatt(inode_t *inode, xlator_t *this, struct iatt *attr)
+{
+    rda_inode_ctx_t *ctx_p = NULL;
+
+    if (!inode || !this || !attr)
+        goto out;
+
+    LOCK(&inode->lock);
+    {
+        ctx_p = __rda_inode_ctx_get(inode, this);
+        if (ctx_p) {
+            *attr = ctx_p->statbuf;
+        }
+    }
+    UNLOCK(&inode->lock);
 
-	return _gf_false;
+out:
+    return;
 }
 
 /*
@@ -108,224 +303,337 @@ rda_can_serve_readdirp(struct rda_fd_ctx *ctx, size_t request_size)
  * buffer. ctx must be locked.
  */
 static int32_t
-__rda_serve_readdirp(xlator_t *this, gf_dirent_t *entries, size_t request_size,
-		   struct rda_fd_ctx *ctx)
+__rda_fill_readdirp(xlator_t *this, gf_dirent_t *entries, size_t request_size,
+                    struct rda_fd_ctx *ctx)
 {
-	gf_dirent_t *dirent, *tmp;
-	size_t dirent_size, size = 0;
-	int32_t count = 0;
-	struct rda_priv *priv = this->private;
+    gf_dirent_t *dirent, *tmp;
+    size_t dirent_size, size = 0;
+    int32_t count = 0;
+    struct rda_priv *priv = NULL;
+    struct iatt tmp_stat = {
+        0,
+    };
+
+    priv = this->private;
+
+    list_for_each_entry_safe(dirent, tmp, &ctx->entries.list, list)
+    {
+        dirent_size = gf_dirent_size(dirent->d_name);
+        if (size + dirent_size > request_size)
+            break;
+
+        memset(&tmp_stat, 0, sizeof(tmp_stat));
+
+        if (dirent->inode && (!((strcmp(dirent->d_name, ".") == 0) ||
+                                (strcmp(dirent->d_name, "..") == 0)))) {
+            rda_inode_ctx_get_iatt(dirent->inode, this, &tmp_stat);
+            dirent->d_stat = tmp_stat;
+        }
 
-	list_for_each_entry_safe(dirent, tmp, &ctx->entries.list, list) {
-		dirent_size = gf_dirent_size(dirent->d_name);
-		if (size + dirent_size > request_size)
-			break;
+        size += dirent_size;
+        list_del_init(&dirent->list);
+        ctx->cur_size -= dirent_size;
 
-		size += dirent_size;
-		list_del_init(&dirent->list);
-		ctx->cur_size -= dirent_size;
+        GF_ATOMIC_SUB(priv->rda_cache_size, dirent_size);
 
-		list_add_tail(&dirent->list, &entries->list);
-		ctx->cur_offset = dirent->d_off;
-		count++;
-	}
+        list_add_tail(&dirent->list, &entries->list);
+        ctx->cur_offset = dirent->d_off;
+        count++;
+    }
 
-	if (ctx->cur_size <= priv->rda_low_wmark)
-		ctx->state |= RDA_FD_PLUGGED;
+    if (ctx->cur_size <= priv->rda_low_wmark)
+        ctx->state |= RDA_FD_PLUGGED;
 
-	return count;
+    return count;
 }
 
 static int32_t
-rda_readdirp_stub(call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
-		  off_t offset, dict_t *xdata)
+__rda_serve_readdirp(xlator_t *this, struct rda_fd_ctx *ctx, size_t size,
+                     gf_dirent_t *entries, int *op_errno)
 {
-	gf_dirent_t entries;
-	int32_t ret;
-	struct rda_fd_ctx *ctx;
-        int op_errno = 0;
-
-	ctx = get_rda_fd_ctx(fd, this);
-	INIT_LIST_HEAD(&entries.list);
-	ret = __rda_serve_readdirp(this, &entries, size, ctx);
+    int32_t ret = 0;
 
-	if (!ret && (ctx->state & RDA_FD_ERROR)) {
-		ret = -1;
-		ctx->state &= ~RDA_FD_ERROR;
+    ret = __rda_fill_readdirp(this, entries, size, ctx);
 
-		/*
-		 * the preload has stopped running in the event of an error, so
-		 * pass all future requests along
-		 */
-		ctx->state |= RDA_FD_BYPASS;
-	}
+    if (!ret && (ctx->state & RDA_FD_ERROR)) {
+        ret = -1;
+        ctx->state &= ~RDA_FD_ERROR;
 
         /*
-         * Use the op_errno sent by lower layers as xlators above will check
-         * the op_errno for identifying whether readdir is completed or not.
+         * the preload has stopped running in the event of an error, so
+         * pass all future requests along
          */
-        op_errno = ctx->op_errno;
-
-	STACK_UNWIND_STRICT(readdirp, frame, ret, op_errno, &entries, xdata);
-	gf_dirent_free(&entries);
-
-	return 0;
+        ctx->state |= RDA_FD_BYPASS;
+    }
+    /*
+     * Use the op_errno sent by lower layers as xlators above will check
+     * the op_errno for identifying whether readdir is completed or not.
+     */
+    *op_errno = ctx->op_errno;
+
+    return ret;
 }
 
 static int32_t
 rda_readdirp(call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
-	     off_t off, dict_t *xdata)
-{
-	struct rda_fd_ctx *ctx;
-	call_stub_t *stub;
-	int fill = 0;
-
-	ctx = get_rda_fd_ctx(fd, this);
-	if (!ctx)
-		goto err;
-
-	if (ctx->state & RDA_FD_BYPASS)
-		goto bypass;
-
-	LOCK(&ctx->lock);
-
-	/* recheck now that we have the lock */
-	if (ctx->state & RDA_FD_BYPASS) {
-		UNLOCK(&ctx->lock);
-		goto bypass;
-	}
-
-	/*
-	 * If a new read comes in at offset 0 and the buffer has been
-	 * completed, reset the context and kickstart the filler again.
-	 */
-	if (!off && (ctx->state & RDA_FD_EOD) && (ctx->cur_size == 0)) {
-		rda_reset_ctx(ctx);
-		fill = 1;
-	}
-
-	/*
-	 * If a readdir occurs at an unexpected offset or we already have a
-	 * request pending, admit defeat and just get out of the way.
-	 */
-	if (off != ctx->cur_offset || ctx->stub) {
-		ctx->state |= RDA_FD_BYPASS;
-		UNLOCK(&ctx->lock);
-		goto bypass;
-	}
-
-	stub = fop_readdirp_stub(frame, rda_readdirp_stub, fd, size, off, xdata);
-	if (!stub) {
-		UNLOCK(&ctx->lock);
-		goto err;
-	}
-
-	/*
-	 * If we haven't bypassed the preload, this means we can either serve
-	 * the request out of the preload or the request that enables us to do
-	 * so is in flight...
-	 */
-	if (rda_can_serve_readdirp(ctx, size))
-		call_resume(stub);
-	else
-		ctx->stub = stub;
-
-	UNLOCK(&ctx->lock);
-
-	if (fill)
-		rda_fill_fd(frame, this, fd);
-
-	return 0;
+             off_t off, dict_t *xdata)
+{
+    struct rda_fd_ctx *ctx = NULL;
+    int fill = 0;
+    gf_dirent_t entries;
+    int ret = 0;
+    int op_errno = 0;
+    gf_boolean_t serve = _gf_false;
+
+    ctx = get_rda_fd_ctx(fd, this);
+    if (!ctx)
+        goto err;
+
+    if (ctx->state & RDA_FD_BYPASS)
+        goto bypass;
+
+    INIT_LIST_HEAD(&entries.list);
+    LOCK(&ctx->lock);
+
+    /* recheck now that we have the lock */
+    if (ctx->state & RDA_FD_BYPASS) {
+        UNLOCK(&ctx->lock);
+        goto bypass;
+    }
+
+    /*
+     * If a new read comes in at offset 0 and the buffer has been
+     * completed, reset the context and kickstart the filler again.
+     */
+    if (!off && (ctx->state & RDA_FD_EOD) && (ctx->cur_size == 0)) {
+        rda_reset_ctx(this, ctx);
+        /*
+         * Unref and discard the 'list of xattrs to be fetched'
+         * stored during opendir call. This is done above - inside
+         * rda_reset_ctx().
+         * Now, ref the xdata passed by md-cache in actual readdirp()
+         * call and use that for all subsequent internal readdirp()
+         * requests issued by this xlator.
+         */
+        ctx->xattrs = dict_ref(xdata);
+        fill = 1;
+    }
+
+    /*
+     * If a readdir occurs at an unexpected offset or we already have a
+     * request pending, admit defeat and just get out of the way.
+     */
+    if (off != ctx->cur_offset || ctx->stub) {
+        ctx->state |= RDA_FD_BYPASS;
+        UNLOCK(&ctx->lock);
+        goto bypass;
+    }
+
+    /*
+     * If we haven't bypassed the preload, this means we can either serve
+     * the request out of the preload or the request that enables us to do
+     * so is in flight...
+     */
+    if (rda_can_serve_readdirp(ctx, size)) {
+        ret = __rda_serve_readdirp(this, ctx, size, &entries, &op_errno);
+        serve = _gf_true;
+
+        if (op_errno == ENOENT &&
+            !((ctx->state & RDA_FD_EOD) && (ctx->cur_size == 0)))
+            op_errno = 0;
+    } else {
+        ctx->stub = fop_readdirp_stub(frame, NULL, fd, size, off, xdata);
+        if (!ctx->stub) {
+            UNLOCK(&ctx->lock);
+            goto err;
+        }
+
+        if (!(ctx->state & RDA_FD_RUNNING)) {
+            fill = 1;
+            if (!ctx->xattrs)
+                ctx->xattrs = dict_ref(xdata);
+            ctx->state |= RDA_FD_RUNNING;
+        }
+    }
+
+    UNLOCK(&ctx->lock);
+
+    if (serve) {
+        STACK_UNWIND_STRICT(readdirp, frame, ret, op_errno, &entries, xdata);
+        gf_dirent_free(&entries);
+    }
+
+    if (fill)
+        rda_fill_fd(frame, this, fd);
+
+    return 0;
 
 bypass:
-	STACK_WIND(frame, default_readdirp_cbk, FIRST_CHILD(this),
-		   FIRST_CHILD(this)->fops->readdirp, fd, size, off, xdata);
-	return 0;
+    STACK_WIND(frame, default_readdirp_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->readdirp, fd, size, off, xdata);
+    return 0;
 
 err:
-	STACK_UNWIND_STRICT(readdirp, frame, -1, ENOMEM, NULL, NULL);
-	return 0;
+    STACK_UNWIND_STRICT(readdirp, frame, -1, ENOMEM, NULL, NULL);
+    return 0;
 }
 
 static int32_t
 rda_fill_fd_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
-		 int32_t op_ret, int32_t op_errno, gf_dirent_t *entries,
-		 dict_t *xdata)
-{
-	gf_dirent_t *dirent, *tmp;
-	struct rda_local *local = frame->local;
-	struct rda_fd_ctx *ctx = local->ctx;
-	struct rda_priv *priv = this->private;
-	int fill = 1;
-
-	LOCK(&ctx->lock);
-
-	/* Verify that the preload buffer is still pending on this data. */
-	if (ctx->next_offset != local->offset) {
-		gf_log(this->name, GF_LOG_ERROR,
-			"Out of sequence directory preload.");
-		ctx->state |= (RDA_FD_BYPASS|RDA_FD_ERROR);
-		ctx->op_errno = EUCLEAN;
-
-		goto out;
-	}
-
-	if (entries) {
-		list_for_each_entry_safe(dirent, tmp, &entries->list, list) {
-			list_del_init(&dirent->list);
-			/* must preserve entry order */
-			list_add_tail(&dirent->list, &ctx->entries.list);
-
-			ctx->cur_size += gf_dirent_size(dirent->d_name);
-			ctx->next_offset = dirent->d_off;
-		}
-	}
-
-	if (ctx->cur_size >= priv->rda_high_wmark)
-		ctx->state &= ~RDA_FD_PLUGGED;
-
-	if (!op_ret) {
-		/* we've hit eod */
-		ctx->state &= ~RDA_FD_RUNNING;
-		ctx->state |= RDA_FD_EOD;
-                ctx->op_errno = op_errno;
-	} else if (op_ret == -1) {
-		/* kill the preload and pend the error */
-		ctx->state &= ~RDA_FD_RUNNING;
-		ctx->state |= RDA_FD_ERROR;
-		ctx->op_errno = op_errno;
-	}
-
-	/*
-	 * NOTE: The strict bypass logic in readdirp() means a pending request
-	 * is always based on ctx->cur_offset.
-	 */
-	if (ctx->stub &&
-	    rda_can_serve_readdirp(ctx, ctx->stub->args.size)) {
-		call_resume(ctx->stub);
-		ctx->stub = NULL;
-	}
+                int32_t op_ret, int32_t op_errno, gf_dirent_t *entries,
+                dict_t *xdata)
+{
+    gf_dirent_t *dirent = NULL;
+    gf_dirent_t *tmp = NULL;
+    gf_dirent_t serve_entries;
+    struct rda_local *local = frame->local;
+    struct rda_fd_ctx *ctx = local->ctx;
+    struct rda_priv *priv = this->private;
+    int fill = 1;
+    size_t dirent_size = 0;
+    int ret = 0;
+    gf_boolean_t serve = _gf_false;
+    call_stub_t *stub = NULL;
+    char gfid[GF_UUID_BUF_SIZE] = {
+        0,
+    };
+    uint64_t generation = 0;
+    call_frame_t *fill_frame = NULL;
+
+    INIT_LIST_HEAD(&serve_entries.list);
+    LOCK(&ctx->lock);
+
+    /* Verify that the preload buffer is still pending on this data. */
+    if (ctx->next_offset != local->offset) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, READDIR_AHEAD_MSG_OUT_OF_SEQUENCE,
+               "Out of sequence directory preload.");
+        ctx->state |= (RDA_FD_BYPASS | RDA_FD_ERROR);
+        ctx->op_errno = EUCLEAN;
+
+        goto out;
+    }
+
+    if (entries) {
+        list_for_each_entry_safe(dirent, tmp, &entries->list, list)
+        {
+            list_del_init(&dirent->list);
+
+            /* must preserve entry order */
+            list_add_tail(&dirent->list, &ctx->entries.list);
+            if (dirent->inode) {
+                /* If ctxp->stat is invalidated, don't update it
+                 * with dirent->d_stat as we don't have
+                 * generation number of the inode when readdirp
+                 * request was initiated. So, we pass 0 for
+                 * generation number
+                 */
+
+                generation = -1;
+                if (ctx->writes_during_prefetch) {
+                    memset(gfid, 0, sizeof(gfid));
+                    uuid_utoa_r(dirent->inode->gfid, gfid);
+                    if (dict_get(ctx->writes_during_prefetch, gfid))
+                        generation = 0;
+                }
+
+                if (!((strcmp(dirent->d_name, ".") == 0) ||
+                      (strcmp(dirent->d_name, "..") == 0))) {
+                    rda_inode_ctx_update_iatts(dirent->inode, this,
+                                               &dirent->d_stat, &dirent->d_stat,
+                                               generation);
+                }
+            }
+
+            dirent_size = gf_dirent_size(dirent->d_name);
+
+            ctx->cur_size += dirent_size;
+
+            GF_ATOMIC_ADD(priv->rda_cache_size, dirent_size);
+
+            ctx->next_offset = dirent->d_off;
+        }
+    }
+
+    if (ctx->writes_during_prefetch) {
+        dict_unref(ctx->writes_during_prefetch);
+        ctx->writes_during_prefetch = NULL;
+    }
+
+    GF_ATOMIC_DEC(ctx->prefetching);
+
+    if (ctx->cur_size >= priv->rda_high_wmark)
+        ctx->state &= ~RDA_FD_PLUGGED;
+
+    if (!op_ret || op_errno == ENOENT) {
+        /* we've hit eod */
+        ctx->state &= ~RDA_FD_RUNNING;
+        ctx->state |= RDA_FD_EOD;
+        ctx->op_errno = op_errno;
+    } else if (op_ret == -1) {
+        /* kill the preload and pend the error */
+        ctx->state &= ~RDA_FD_RUNNING;
+        ctx->state |= RDA_FD_ERROR;
+        ctx->op_errno = op_errno;
+    }
+
+    /*
+     * NOTE: The strict bypass logic in readdirp() means a pending request
+     * is always based on ctx->cur_offset.
+     */
+    if (ctx->stub && rda_can_serve_readdirp(ctx, ctx->stub->args.size)) {
+        ret = __rda_serve_readdirp(this, ctx, ctx->stub->args.size,
+                                   &serve_entries, &op_errno);
+        serve = _gf_true;
+        stub = ctx->stub;
+        ctx->stub = NULL;
+    }
 
 out:
-	/*
-	 * If we have been marked for bypass and have no pending stub, clear the
-	 * run state so we stop preloading the context with entries.
-	 */
-	if ((ctx->state & RDA_FD_BYPASS) && !ctx->stub)
-		ctx->state &= ~RDA_FD_RUNNING;
+    /*
+     * If we have been marked for bypass and have no pending stub, clear the
+     * run state so we stop preloading the context with entries.
+     */
+    if (!ctx->stub &&
+        ((ctx->state & RDA_FD_BYPASS) ||
+         GF_ATOMIC_GET(priv->rda_cache_size) > priv->rda_cache_limit))
+        ctx->state &= ~RDA_FD_RUNNING;
+
+    if (!(ctx->state & RDA_FD_RUNNING)) {
+        fill = 0;
+        if (ctx->xattrs) {
+            /*
+             * fill = 0 and hence rda_fill_fd() won't be invoked.
+             * unref for ref taken in rda_fill_fd()
+             */
+            dict_unref(ctx->xattrs);
+            ctx->xattrs = NULL;
+        }
 
-	if (!(ctx->state & RDA_FD_RUNNING)) {
-		fill = 0;
-		STACK_DESTROY(ctx->fill_frame->root);
-		ctx->fill_frame = NULL;
-	}
+        fill_frame = ctx->fill_frame;
+        ctx->fill_frame = NULL;
+    }
 
-	UNLOCK(&ctx->lock);
+    if (op_errno == ENOENT &&
+        !((ctx->state & RDA_FD_EOD) && (ctx->cur_size == 0)))
+        op_errno = 0;
 
-	if (fill)
-		rda_fill_fd(frame, this, local->fd);
+    UNLOCK(&ctx->lock);
+    if (fill_frame) {
+        rda_local_wipe(fill_frame->local);
+        STACK_DESTROY(fill_frame->root);
+    }
 
-	return 0;
+    if (serve) {
+        STACK_UNWIND_STRICT(readdirp, stub->frame, ret, op_errno,
+                            &serve_entries, xdata);
+        gf_dirent_free(&serve_entries);
+        call_stub_destroy(stub);
+    }
+
+    if (fill)
+        rda_fill_fd(frame, this, local->fd);
+
+    return 0;
 }
 
 /*
@@ -334,234 +642,741 @@ out:
 static int
 rda_fill_fd(call_frame_t *frame, xlator_t *this, fd_t *fd)
 {
-	call_frame_t *nframe = NULL;
-	struct rda_local *local = NULL;
-	struct rda_fd_ctx *ctx;
-	off_t offset;
-	struct rda_priv *priv = this->private;
+    call_frame_t *nframe = NULL;
+    struct rda_local *local = NULL;
+    struct rda_local *orig_local = frame->local;
+    struct rda_fd_ctx *ctx;
+    off_t offset;
+    struct rda_priv *priv = this->private;
+
+    ctx = get_rda_fd_ctx(fd, this);
+    if (!ctx)
+        goto err;
+
+    LOCK(&ctx->lock);
+
+    if (ctx->state & RDA_FD_NEW) {
+        ctx->state &= ~RDA_FD_NEW;
+        ctx->state |= RDA_FD_RUNNING;
+        if (priv->rda_low_wmark)
+            ctx->state |= RDA_FD_PLUGGED;
+    }
+
+    offset = ctx->next_offset;
+
+    if (!ctx->fill_frame) {
+        nframe = copy_frame(frame);
+        if (!nframe) {
+            UNLOCK(&ctx->lock);
+            goto err;
+        }
+
+        local = mem_get0(this->local_pool);
+        if (!local) {
+            UNLOCK(&ctx->lock);
+            goto err;
+        }
+
+        local->ctx = ctx;
+        local->fd = fd_ref(fd);
+        nframe->local = local;
 
-	ctx = get_rda_fd_ctx(fd, this);
-	if (!ctx)
-		goto err;
+        ctx->fill_frame = nframe;
 
-	LOCK(&ctx->lock);
+        if (!ctx->xattrs && orig_local && orig_local->xattrs) {
+            /* when this function is invoked by rda_opendir_cbk */
+            ctx->xattrs = dict_ref(orig_local->xattrs);
+        }
+    } else {
+        nframe = ctx->fill_frame;
+        local = nframe->local;
+    }
 
-	if (ctx->state & RDA_FD_NEW) {
-		ctx->state &= ~RDA_FD_NEW;
-		ctx->state |= RDA_FD_RUNNING;
-		if (priv->rda_low_wmark)
-			ctx->state |= RDA_FD_PLUGGED;
-	}
+    local->offset = offset;
+    GF_ATOMIC_INC(ctx->prefetching);
 
-	offset = ctx->next_offset;
+    UNLOCK(&ctx->lock);
 
-	if (!ctx->fill_frame) {
-		nframe = copy_frame(frame);
-		if (!nframe) {
-			UNLOCK(&ctx->lock);
-			goto err;
-		}
+    STACK_WIND(nframe, rda_fill_fd_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->readdirp, fd, priv->rda_req_size,
+               offset, ctx->xattrs);
 
-		local = mem_get0(this->local_pool);
-		if (!local) {
-			UNLOCK(&ctx->lock);
-			goto err;
-		}
+    return 0;
 
-		local->ctx = ctx;
-		local->fd = fd;
-		nframe->local = local;
+err:
+    if (nframe) {
+        rda_local_wipe(nframe->local);
+        FRAME_DESTROY(nframe);
+    }
 
-		ctx->fill_frame = nframe;
-	} else {
-		nframe = ctx->fill_frame;
-		local = nframe->local;
-	}
+    return -1;
+}
 
-	local->offset = offset;
+static int32_t
+rda_opendir_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                int32_t op_ret, int32_t op_errno, fd_t *fd, dict_t *xdata)
+{
+    if (!op_ret)
+        rda_fill_fd(frame, this, fd);
 
-	UNLOCK(&ctx->lock);
+    RDA_STACK_UNWIND(opendir, frame, op_ret, op_errno, fd, xdata);
+    return 0;
+}
 
-	STACK_WIND(nframe, rda_fill_fd_cbk, FIRST_CHILD(this),
-		   FIRST_CHILD(this)->fops->readdirp, fd, priv->rda_req_size,
-		   offset, NULL);
+static int32_t
+rda_opendir(call_frame_t *frame, xlator_t *this, loc_t *loc, fd_t *fd,
+            dict_t *xdata)
+{
+    int op_errno = 0;
+    struct rda_local *local = NULL;
+
+    if (xdata) {
+        local = mem_get0(this->local_pool);
+        if (!local) {
+            op_errno = ENOMEM;
+            goto unwind;
+        }
 
-	return 0;
+        /*
+         * Retrieve list of keys set by md-cache xlator and store it
+         * in local to be consumed in rda_opendir_cbk
+         */
+        local->xattrs = dict_copy_with_ref(xdata, NULL);
+        frame->local = local;
+    }
 
-err:
-	if (nframe)
-		FRAME_DESTROY(nframe);
+    STACK_WIND(frame, rda_opendir_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->opendir, loc, fd, xdata);
+    return 0;
 
-	return -1;
+unwind:
+    STACK_UNWIND_STRICT(opendir, frame, -1, op_errno, fd, xdata);
+    return 0;
 }
 
 static int32_t
-rda_opendir_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
-		    int32_t op_ret, int32_t op_errno, fd_t *fd, dict_t *xdata)
+rda_writev_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+               int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
+               struct iatt *postbuf, dict_t *xdata)
 {
-	if (!op_ret)
-		rda_fill_fd(frame, this, fd);
+    struct rda_local *local = NULL;
+    struct iatt postbuf_out = {
+        0,
+    };
+
+    if (op_ret < 0)
+        goto unwind;
+
+    local = frame->local;
+
+    rda_mark_inode_dirty(this, local->inode);
+
+    rda_inode_ctx_update_iatts(local->inode, this, postbuf, &postbuf_out,
+                               local->generation);
 
-	STACK_UNWIND_STRICT(opendir, frame, op_ret, op_errno, fd, xdata);
-	return 0;
+unwind:
+    RDA_STACK_UNWIND(writev, frame, op_ret, op_errno, prebuf, &postbuf_out,
+                     xdata);
+    return 0;
 }
 
 static int32_t
-rda_opendir(call_frame_t *frame, xlator_t *this, loc_t *loc, fd_t *fd,
-		dict_t *xdata)
+rda_writev(call_frame_t *frame, xlator_t *this, fd_t *fd, struct iovec *vector,
+           int32_t count, off_t off, uint32_t flags, struct iobref *iobref,
+           dict_t *xdata)
+{
+    RDA_COMMON_MODIFICATION_FOP(writev, frame, this, fd->inode, xdata, fd,
+                                vector, count, off, flags, iobref);
+    return 0;
+}
+
+static int32_t
+rda_fallocate_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                  int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
+                  struct iatt *postbuf, dict_t *xdata)
+{
+    struct rda_local *local = NULL;
+    struct iatt postbuf_out = {
+        0,
+    };
+
+    if (op_ret < 0)
+        goto unwind;
+
+    local = frame->local;
+    rda_mark_inode_dirty(this, local->inode);
+    rda_inode_ctx_update_iatts(local->inode, this, postbuf, &postbuf_out,
+                               local->generation);
+
+unwind:
+    RDA_STACK_UNWIND(fallocate, frame, op_ret, op_errno, prebuf, &postbuf_out,
+                     xdata);
+    return 0;
+}
+
+static int32_t
+rda_fallocate(call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t keep_size,
+              off_t offset, size_t len, dict_t *xdata)
+{
+    RDA_COMMON_MODIFICATION_FOP(fallocate, frame, this, fd->inode, xdata, fd,
+                                keep_size, offset, len);
+    return 0;
+}
+
+static int32_t
+rda_zerofill_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                 int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
+                 struct iatt *postbuf, dict_t *xdata)
+{
+    struct rda_local *local = NULL;
+    struct iatt postbuf_out = {
+        0,
+    };
+
+    if (op_ret < 0)
+        goto unwind;
+
+    local = frame->local;
+    rda_mark_inode_dirty(this, local->inode);
+    rda_inode_ctx_update_iatts(local->inode, this, postbuf, &postbuf_out,
+                               local->generation);
+
+unwind:
+    RDA_STACK_UNWIND(zerofill, frame, op_ret, op_errno, prebuf, &postbuf_out,
+                     xdata);
+    return 0;
+}
+
+static int32_t
+rda_zerofill(call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
+             off_t len, dict_t *xdata)
+{
+    RDA_COMMON_MODIFICATION_FOP(zerofill, frame, this, fd->inode, xdata, fd,
+                                offset, len);
+    return 0;
+}
+
+static int32_t
+rda_discard_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
+                struct iatt *postbuf, dict_t *xdata)
+{
+    struct rda_local *local = NULL;
+    struct iatt postbuf_out = {
+        0,
+    };
+
+    if (op_ret < 0)
+        goto unwind;
+
+    local = frame->local;
+    rda_mark_inode_dirty(this, local->inode);
+    rda_inode_ctx_update_iatts(local->inode, this, postbuf, &postbuf_out,
+                               local->generation);
+
+unwind:
+    RDA_STACK_UNWIND(discard, frame, op_ret, op_errno, prebuf, &postbuf_out,
+                     xdata);
+    return 0;
+}
+
+static int32_t
+rda_discard(call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
+            size_t len, dict_t *xdata)
+{
+    RDA_COMMON_MODIFICATION_FOP(discard, frame, this, fd->inode, xdata, fd,
+                                offset, len);
+    return 0;
+}
+
+static int32_t
+rda_ftruncate_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                  int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
+                  struct iatt *postbuf, dict_t *xdata)
+{
+    struct rda_local *local = NULL;
+    struct iatt postbuf_out = {
+        0,
+    };
+
+    if (op_ret < 0)
+        goto unwind;
+
+    local = frame->local;
+    rda_mark_inode_dirty(this, local->inode);
+    rda_inode_ctx_update_iatts(local->inode, this, postbuf, &postbuf_out,
+                               local->generation);
+
+unwind:
+    RDA_STACK_UNWIND(ftruncate, frame, op_ret, op_errno, prebuf, &postbuf_out,
+                     xdata);
+    return 0;
+}
+
+static int32_t
+rda_ftruncate(call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
+              dict_t *xdata)
+{
+    RDA_COMMON_MODIFICATION_FOP(ftruncate, frame, this, fd->inode, xdata, fd,
+                                offset);
+    return 0;
+}
+
+static int32_t
+rda_truncate_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                 int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
+                 struct iatt *postbuf, dict_t *xdata)
+{
+    struct rda_local *local = NULL;
+    struct iatt postbuf_out = {
+        0,
+    };
+
+    if (op_ret < 0)
+        goto unwind;
+
+    local = frame->local;
+    rda_mark_inode_dirty(this, local->inode);
+    rda_inode_ctx_update_iatts(local->inode, this, postbuf, &postbuf_out,
+                               local->generation);
+
+unwind:
+    RDA_STACK_UNWIND(ftruncate, frame, op_ret, op_errno, prebuf, &postbuf_out,
+                     xdata);
+    return 0;
+}
+
+static int32_t
+rda_truncate(call_frame_t *frame, xlator_t *this, loc_t *loc, off_t offset,
+             dict_t *xdata)
+{
+    RDA_COMMON_MODIFICATION_FOP(truncate, frame, this, loc->inode, xdata, loc,
+                                offset);
+    return 0;
+}
+
+static int32_t
+rda_setxattr_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                 int32_t op_ret, int32_t op_errno, dict_t *xdata)
+{
+    struct rda_local *local = NULL;
+
+    if (op_ret < 0)
+        goto unwind;
+
+    local = frame->local;
+    rda_mark_inode_dirty(this, local->inode);
+    rda_inode_ctx_update_iatts(local->inode, this, NULL, NULL,
+                               local->generation);
+unwind:
+    RDA_STACK_UNWIND(setxattr, frame, op_ret, op_errno, xdata);
+    return 0;
+}
+
+static int32_t
+rda_setxattr(call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *dict,
+             int32_t flags, dict_t *xdata)
+{
+    RDA_COMMON_MODIFICATION_FOP(setxattr, frame, this, loc->inode, xdata, loc,
+                                dict, flags);
+    return 0;
+}
+
+static int32_t
+rda_fsetxattr_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                  int32_t op_ret, int32_t op_errno, dict_t *xdata)
+{
+    struct rda_local *local = NULL;
+
+    if (op_ret < 0)
+        goto unwind;
+
+    local = frame->local;
+    rda_mark_inode_dirty(this, local->inode);
+    rda_inode_ctx_update_iatts(local->inode, this, NULL, NULL,
+                               local->generation);
+unwind:
+    RDA_STACK_UNWIND(fsetxattr, frame, op_ret, op_errno, xdata);
+    return 0;
+}
+
+static int32_t
+rda_fsetxattr(call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *dict,
+              int32_t flags, dict_t *xdata)
+{
+    RDA_COMMON_MODIFICATION_FOP(fsetxattr, frame, this, fd->inode, xdata, fd,
+                                dict, flags);
+    return 0;
+}
+
+static int32_t
+rda_setattr_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                int32_t op_ret, int32_t op_errno, struct iatt *statpre,
+                struct iatt *statpost, dict_t *xdata)
+{
+    struct rda_local *local = NULL;
+    struct iatt postbuf_out = {
+        0,
+    };
+
+    if (op_ret < 0)
+        goto unwind;
+
+    local = frame->local;
+    rda_mark_inode_dirty(this, local->inode);
+    rda_inode_ctx_update_iatts(local->inode, this, statpost, &postbuf_out,
+                               local->generation);
+
+unwind:
+    RDA_STACK_UNWIND(setattr, frame, op_ret, op_errno, statpre, &postbuf_out,
+                     xdata);
+    return 0;
+}
+
+static int32_t
+rda_setattr(call_frame_t *frame, xlator_t *this, loc_t *loc, struct iatt *stbuf,
+            int32_t valid, dict_t *xdata)
+{
+    RDA_COMMON_MODIFICATION_FOP(setattr, frame, this, loc->inode, xdata, loc,
+                                stbuf, valid);
+    return 0;
+}
+
+static int32_t
+rda_fsetattr_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                 int32_t op_ret, int32_t op_errno, struct iatt *statpre,
+                 struct iatt *statpost, dict_t *xdata)
+{
+    struct rda_local *local = NULL;
+    struct iatt postbuf_out = {
+        0,
+    };
+
+    if (op_ret < 0)
+        goto unwind;
+
+    local = frame->local;
+    rda_mark_inode_dirty(this, local->inode);
+    rda_inode_ctx_update_iatts(local->inode, this, statpost, &postbuf_out,
+                               local->generation);
+
+unwind:
+    RDA_STACK_UNWIND(fsetattr, frame, op_ret, op_errno, statpre, &postbuf_out,
+                     xdata);
+    return 0;
+}
+
+static int32_t
+rda_fsetattr(call_frame_t *frame, xlator_t *this, fd_t *fd, struct iatt *stbuf,
+             int32_t valid, dict_t *xdata)
+{
+    RDA_COMMON_MODIFICATION_FOP(fsetattr, frame, this, fd->inode, xdata, fd,
+                                stbuf, valid);
+    return 0;
+}
+
+static int32_t
+rda_removexattr_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                    int32_t op_ret, int32_t op_errno, dict_t *xdata)
+{
+    struct rda_local *local = NULL;
+
+    if (op_ret < 0)
+        goto unwind;
+
+    local = frame->local;
+    rda_mark_inode_dirty(this, local->inode);
+    rda_inode_ctx_update_iatts(local->inode, this, NULL, NULL,
+                               local->generation);
+unwind:
+    RDA_STACK_UNWIND(removexattr, frame, op_ret, op_errno, xdata);
+    return 0;
+}
+
+static int32_t
+rda_removexattr(call_frame_t *frame, xlator_t *this, loc_t *loc,
+                const char *name, dict_t *xdata)
 {
-	STACK_WIND(frame, rda_opendir_cbk, FIRST_CHILD(this),
-		   FIRST_CHILD(this)->fops->opendir, loc, fd, xdata);
-	return 0;
+    RDA_COMMON_MODIFICATION_FOP(removexattr, frame, this, loc->inode, xdata,
+                                loc, name);
+    return 0;
+}
+
+static int32_t
+rda_fremovexattr_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                     int32_t op_ret, int32_t op_errno, dict_t *xdata)
+{
+    struct rda_local *local = NULL;
+
+    if (op_ret < 0)
+        goto unwind;
+
+    local = frame->local;
+    rda_mark_inode_dirty(this, local->inode);
+    rda_inode_ctx_update_iatts(local->inode, this, NULL, NULL,
+                               local->generation);
+unwind:
+    RDA_STACK_UNWIND(fremovexattr, frame, op_ret, op_errno, xdata);
+    return 0;
+}
+
+static int32_t
+rda_fremovexattr(call_frame_t *frame, xlator_t *this, fd_t *fd,
+                 const char *name, dict_t *xdata)
+{
+    RDA_COMMON_MODIFICATION_FOP(fremovexattr, frame, this, fd->inode, xdata, fd,
+                                name);
+    return 0;
 }
 
 static int32_t
 rda_releasedir(xlator_t *this, fd_t *fd)
 {
-	uint64_t val;
-	struct rda_fd_ctx *ctx;
+    uint64_t val;
+    struct rda_fd_ctx *ctx;
 
-	if (fd_ctx_del(fd, this, &val) < 0)
-		return -1;
+    if (fd_ctx_del(fd, this, &val) < 0)
+        return -1;
 
-	ctx = (struct rda_fd_ctx *) val;
-	if (!ctx)
-		return 0;
+    ctx = (struct rda_fd_ctx *)(uintptr_t)val;
+    if (!ctx)
+        return 0;
 
-	rda_reset_ctx(ctx);
+    rda_reset_ctx(this, ctx);
 
-	if (ctx->fill_frame)
-		STACK_DESTROY(ctx->fill_frame->root);
+    if (ctx->fill_frame)
+        STACK_DESTROY(ctx->fill_frame->root);
 
-	if (ctx->stub)
-		gf_log(this->name, GF_LOG_ERROR,
-			"released a directory with a pending stub");
+    if (ctx->stub)
+        gf_msg(this->name, GF_LOG_ERROR, 0,
+               READDIR_AHEAD_MSG_DIR_RELEASE_PENDING_STUB,
+               "released a directory with a pending stub");
 
-	GF_FREE(ctx);
-	return 0;
+    GF_FREE(ctx);
+    return 0;
+}
+
+static int
+rda_forget(xlator_t *this, inode_t *inode)
+{
+    uint64_t ctx_uint = 0;
+    rda_inode_ctx_t *ctx = NULL;
+
+    inode_ctx_del1(inode, this, &ctx_uint);
+    if (!ctx_uint)
+        return 0;
+
+    ctx = (rda_inode_ctx_t *)(uintptr_t)ctx_uint;
+
+    GF_FREE(ctx);
+
+    return 0;
 }
 
 int32_t
 mem_acct_init(xlator_t *this)
 {
-	int ret = -1;
+    int ret = -1;
 
-	if (!this)
-		goto out;
+    if (!this)
+        goto out;
 
-	ret = xlator_mem_acct_init(this, gf_rda_mt_end + 1);
+    ret = xlator_mem_acct_init(this, gf_rda_mt_end + 1);
 
-	if (ret != 0)
-		gf_log(this->name, GF_LOG_ERROR, "Memory accounting init"
-		       "failed");
+    if (ret != 0)
+        gf_msg(this->name, GF_LOG_ERROR, ENOMEM, READDIR_AHEAD_MSG_NO_MEMORY,
+               "Memory accounting init"
+               "failed");
 
 out:
-	return ret;
+    return ret;
 }
 
 int
 reconfigure(xlator_t *this, dict_t *options)
 {
-	struct rda_priv *priv = this->private;
-
-	GF_OPTION_RECONF("rda-request-size", priv->rda_req_size, options,
-			 uint32, err);
-	GF_OPTION_RECONF("rda-low-wmark", priv->rda_low_wmark, options, size_uint64,
-			 err);
-	GF_OPTION_RECONF("rda-high-wmark", priv->rda_high_wmark, options, size_uint64,
-			 err);
-
-	return 0;
+    struct rda_priv *priv = this->private;
+
+    GF_OPTION_RECONF("rda-request-size", priv->rda_req_size, options,
+                     size_uint64, err);
+    GF_OPTION_RECONF("rda-low-wmark", priv->rda_low_wmark, options, size_uint64,
+                     err);
+    GF_OPTION_RECONF("rda-high-wmark", priv->rda_high_wmark, options,
+                     size_uint64, err);
+    GF_OPTION_RECONF("rda-cache-limit", priv->rda_cache_limit, options,
+                     size_uint64, err);
+    GF_OPTION_RECONF("parallel-readdir", priv->parallel_readdir, options, bool,
+                     err);
+    GF_OPTION_RECONF("pass-through", this->pass_through, options, bool, err);
+
+    return 0;
 err:
-	return -1;
+    return -1;
 }
 
 int
 init(xlator_t *this)
 {
-	struct rda_priv *priv = NULL;
+    struct rda_priv *priv = NULL;
 
-        GF_VALIDATE_OR_GOTO("readdir-ahead", this, err);
+    GF_VALIDATE_OR_GOTO("readdir-ahead", this, err);
 
-        if (!this->children || this->children->next) {
-                gf_log(this->name,  GF_LOG_ERROR,
-                        "FATAL: readdir-ahead not configured with exactly one"
-                        " child");
-                goto err;
-        }
+    if (!this->children || this->children->next) {
+        gf_msg(this->name, GF_LOG_ERROR, 0,
+               READDIR_AHEAD_MSG_XLATOR_CHILD_MISCONFIGURED,
+               "FATAL: readdir-ahead not configured with exactly one"
+               " child");
+        goto err;
+    }
 
-        if (!this->parents) {
-                gf_log(this->name, GF_LOG_WARNING,
-                        "dangling volume. check volfile ");
-        }
+    if (!this->parents) {
+        gf_msg(this->name, GF_LOG_WARNING, 0,
+               READDIR_AHEAD_MSG_VOL_MISCONFIGURED,
+               "dangling volume. check volfile ");
+    }
+
+    priv = GF_CALLOC(1, sizeof(struct rda_priv), gf_rda_mt_rda_priv);
+    if (!priv)
+        goto err;
+    this->private = priv;
 
-	priv = GF_CALLOC(1, sizeof(struct rda_priv), gf_rda_mt_rda_priv);
-	if (!priv)
-		goto err;
-	this->private = priv;
+    GF_ATOMIC_INIT(priv->rda_cache_size, 0);
 
-	this->local_pool = mem_pool_new(struct rda_local, 32);
-	if (!this->local_pool)
-		goto err;
+    this->local_pool = mem_pool_new(struct rda_local, 32);
+    if (!this->local_pool)
+        goto err;
 
-	GF_OPTION_INIT("rda-request-size", priv->rda_req_size, uint32, err);
-	GF_OPTION_INIT("rda-low-wmark", priv->rda_low_wmark, size_uint64, err);
-	GF_OPTION_INIT("rda-high-wmark", priv->rda_high_wmark, size_uint64, err);
+    GF_OPTION_INIT("rda-request-size", priv->rda_req_size, size_uint64, err);
+    GF_OPTION_INIT("rda-low-wmark", priv->rda_low_wmark, size_uint64, err);
+    GF_OPTION_INIT("rda-high-wmark", priv->rda_high_wmark, size_uint64, err);
+    GF_OPTION_INIT("rda-cache-limit", priv->rda_cache_limit, size_uint64, err);
+    GF_OPTION_INIT("parallel-readdir", priv->parallel_readdir, bool, err);
+    GF_OPTION_INIT("pass-through", this->pass_through, bool, err);
 
-	return 0;
+    return 0;
 
 err:
-	if (this->local_pool)
-		mem_pool_destroy(this->local_pool);
-	if (priv)
-		GF_FREE(priv);
+    if (this->local_pool)
+        mem_pool_destroy(this->local_pool);
+    if (priv)
+        GF_FREE(priv);
 
-        return -1;
+    return -1;
 }
 
-
 void
 fini(xlator_t *this)
 {
-        GF_VALIDATE_OR_GOTO ("readdir-ahead", this, out);
+    GF_VALIDATE_OR_GOTO("readdir-ahead", this, out);
 
-	GF_FREE(this->private);
+    GF_FREE(this->private);
 
 out:
-        return;
+    return;
 }
 
 struct xlator_fops fops = {
-	.opendir	= rda_opendir,
-	.readdirp	= rda_readdirp,
+    .opendir = rda_opendir,
+    .readdirp = rda_readdirp,
+    /* inode write */
+    /* TODO: invalidate a dentry's stats if its pointing to a directory
+     * when entry operations happen in that directory
+     */
+    .writev = rda_writev,
+    .truncate = rda_truncate,
+    .ftruncate = rda_ftruncate,
+    .fallocate = rda_fallocate,
+    .discard = rda_discard,
+    .zerofill = rda_zerofill,
+    /* metadata write */
+    .setxattr = rda_setxattr,
+    .fsetxattr = rda_fsetxattr,
+    .setattr = rda_setattr,
+    .fsetattr = rda_fsetattr,
+    .removexattr = rda_removexattr,
+    .fremovexattr = rda_fremovexattr,
 };
 
 struct xlator_cbks cbks = {
-	.releasedir	= rda_releasedir,
+    .releasedir = rda_releasedir,
+    .forget = rda_forget,
 };
 
 struct volume_options options[] = {
-	{ .key = {"rda-request-size"},
-	  .type = GF_OPTION_TYPE_INT,
-	  .min = 4096,
-	  .max = 131072,
-	  .default_value = "131072",
-	  .description = "readdir-ahead request size",
-	},
-	{ .key = {"rda-low-wmark"},
-	  .type = GF_OPTION_TYPE_SIZET,
-	  .min = 0,
-	  .max = 10 * GF_UNIT_MB,
-	  .default_value = "4096",
-	  .description = "the value under which we plug",
-	},
-	{ .key = {"rda-high-wmark"},
-	  .type = GF_OPTION_TYPE_SIZET,
-	  .min = 0,
-	  .max = 100 * GF_UNIT_MB,
-	  .default_value = "131072",
-	  .description = "the value over which we unplug",
-	},
-        { .key = {NULL} },
+    {
+        .key = {"readdir-ahead"},
+        .type = GF_OPTION_TYPE_BOOL,
+        .default_value = "off",
+        .description = "enable/disable readdir-ahead",
+        .op_version = {GD_OP_VERSION_6_0},
+        .flags = OPT_FLAG_SETTABLE,
+    },
+    {
+        .key = {"rda-request-size"},
+        .type = GF_OPTION_TYPE_SIZET,
+        .min = 4096,
+        .max = 131072,
+        .default_value = "131072",
+        .description = "size of buffer in readdirp calls initiated by "
+                       "readdir-ahead ",
+    },
+    {
+        .key = {"rda-low-wmark"},
+        .type = GF_OPTION_TYPE_SIZET,
+        .min = 0,
+        .max = 10 * GF_UNIT_MB,
+        .default_value = "4096",
+        .description = "the value under which readdir-ahead plugs",
+    },
+    {
+        .key = {"rda-high-wmark"},
+        .type = GF_OPTION_TYPE_SIZET,
+        .min = 0,
+        .max = 100 * GF_UNIT_MB,
+        .default_value = "128KB",
+        .description = "the value over which readdir-ahead unplugs",
+    },
+    {
+        .key = {"rda-cache-limit"},
+        .type = GF_OPTION_TYPE_SIZET,
+        .min = 0,
+        .max = INFINITY,
+        .default_value = "10MB",
+        .description = "maximum size of cache consumed by readdir-ahead "
+                       "xlator. This value is global and total memory "
+                       "consumption by readdir-ahead is capped by this "
+                       "value, irrespective of the number/size of "
+                       "directories cached",
+    },
+    {.key = {"parallel-readdir"},
+     .type = GF_OPTION_TYPE_BOOL,
+     .op_version = {GD_OP_VERSION_3_10_0},
+     .flags = OPT_FLAG_SETTABLE | OPT_FLAG_CLIENT_OPT | OPT_FLAG_DOC,
+     .default_value = "off",
+     .description = "If this option is enabled, the readdir operation "
+                    "is performed in parallel on all the bricks, thus "
+                    "improving the performance of readdir. Note that "
+                    "the performance improvement is higher in large "
+                    "clusters"},
+    {.key = {"pass-through"},
+     .type = GF_OPTION_TYPE_BOOL,
+     .default_value = "false",
+     .op_version = {GD_OP_VERSION_4_1_0},
+     .flags = OPT_FLAG_SETTABLE | OPT_FLAG_DOC | OPT_FLAG_CLIENT_OPT,
+     .tags = {"readdir-ahead"},
+     .description = "Enable/Disable readdir ahead translator"},
+    {.key = {NULL}},
 };
 
+xlator_api_t xlator_api = {
+    .init = init,
+    .fini = fini,
+    .reconfigure = reconfigure,
+    .mem_acct_init = mem_acct_init,
+    .op_version = {1}, /* Present from the initial version */
+    .fops = &fops,
+    .cbks = &cbks,
+    .options = options,
+    .identifier = "readdir-ahead",
+    .category = GF_MAINTAINED,
+};
diff --git a/xlators/performance/readdir-ahead/src/readdir-ahead.h b/xlators/performance/readdir-ahead/src/readdir-ahead.h
index e48786daeca..619c41059ff 100644
--- a/xlators/performance/readdir-ahead/src/readdir-ahead.h
+++ b/xlators/performance/readdir-ahead/src/readdir-ahead.h
@@ -12,35 +12,87 @@
 #define __READDIR_AHEAD_H
 
 /* state flags */
-#define RDA_FD_NEW	(1 << 0)
-#define RDA_FD_RUNNING	(1 << 1)
-#define RDA_FD_EOD	(1 << 2)
-#define RDA_FD_ERROR	(1 << 3)
-#define RDA_FD_BYPASS	(1 << 4)
-#define RDA_FD_PLUGGED	(1 << 5)
+#define RDA_FD_NEW (1 << 0)
+#define RDA_FD_RUNNING (1 << 1)
+#define RDA_FD_EOD (1 << 2)
+#define RDA_FD_ERROR (1 << 3)
+#define RDA_FD_BYPASS (1 << 4)
+#define RDA_FD_PLUGGED (1 << 5)
+
+#define RDA_COMMON_MODIFICATION_FOP(name, frame, this, __inode, __xdata,       \
+                                    args...)                                   \
+    do {                                                                       \
+        struct rda_local *__local = NULL;                                      \
+        rda_inode_ctx_t *ctx_p = NULL;                                         \
+                                                                               \
+        __local = mem_get0(this->local_pool);                                  \
+        __local->inode = inode_ref(__inode);                                   \
+        LOCK(&__inode->lock);                                                  \
+        {                                                                      \
+            ctx_p = __rda_inode_ctx_get(__inode, this);                        \
+        }                                                                      \
+        UNLOCK(&__inode->lock);                                                \
+        __local->generation = GF_ATOMIC_GET(ctx_p->generation);                \
+                                                                               \
+        frame->local = __local;                                                \
+        if (__xdata)                                                           \
+            __local->xattrs = dict_ref(__xdata);                               \
+                                                                               \
+        STACK_WIND(frame, rda_##name##_cbk, FIRST_CHILD(this),                 \
+                   FIRST_CHILD(this)->fops->name, args, __xdata);              \
+    } while (0)
+
+#define RDA_STACK_UNWIND(fop, frame, params...)                                \
+    do {                                                                       \
+        struct rda_local *__local = NULL;                                      \
+        if (frame) {                                                           \
+            __local = frame->local;                                            \
+            frame->local = NULL;                                               \
+        }                                                                      \
+        STACK_UNWIND_STRICT(fop, frame, params);                               \
+        if (__local) {                                                         \
+            rda_local_wipe(__local);                                           \
+            mem_put(__local);                                                  \
+        }                                                                      \
+    } while (0)
 
 struct rda_fd_ctx {
-	off_t cur_offset;	/* current head of the ctx */
-	size_t cur_size;	/* current size of the preload */
-	off_t next_offset;	/* tail of the ctx */
-	uint32_t state;
-	gf_lock_t lock;
-	gf_dirent_t entries;
-	call_frame_t *fill_frame;
-	call_stub_t *stub;
-	int op_errno;
+    off_t cur_offset;  /* current head of the ctx */
+    size_t cur_size;   /* current size of the preload */
+    off_t next_offset; /* tail of the ctx */
+    uint32_t state;
+    gf_lock_t lock;
+    gf_dirent_t entries;
+    call_frame_t *fill_frame;
+    call_stub_t *stub;
+    int op_errno;
+    dict_t *xattrs; /* md-cache keys to be sent in readdirp() */
+    dict_t *writes_during_prefetch;
+    gf_atomic_t prefetching;
 };
 
 struct rda_local {
-	struct rda_fd_ctx *ctx;
-	fd_t *fd;
-	off_t offset;
+    struct rda_fd_ctx *ctx;
+    fd_t *fd;
+    dict_t *xattrs; /* md-cache keys to be sent in readdirp() */
+    inode_t *inode;
+    off_t offset;
+    uint64_t generation;
+    int32_t skip_dir;
 };
 
 struct rda_priv {
-	uint32_t rda_req_size;
-	uint64_t rda_low_wmark;
-	uint64_t rda_high_wmark;
+    uint64_t rda_req_size;
+    uint64_t rda_low_wmark;
+    uint64_t rda_high_wmark;
+    uint64_t rda_cache_limit;
+    gf_atomic_t rda_cache_size;
+    gf_boolean_t parallel_readdir;
 };
 
+typedef struct rda_inode_ctx {
+    struct iatt statbuf;
+    gf_atomic_t generation;
+} rda_inode_ctx_t;
+
 #endif /* __READDIR_AHEAD_H */
diff --git a/xlators/performance/symlink-cache/Makefile.am b/xlators/performance/symlink-cache/Makefile.am
deleted file mode 100644
index d471a3f9243..00000000000
--- a/xlators/performance/symlink-cache/Makefile.am
+++ /dev/null
@@ -1,3 +0,0 @@
-SUBDIRS = src
-
-CLEANFILES = 
diff --git a/xlators/performance/symlink-cache/src/Makefile.am b/xlators/performance/symlink-cache/src/Makefile.am
deleted file mode 100644
index 4091c329325..00000000000
--- a/xlators/performance/symlink-cache/src/Makefile.am
+++ /dev/null
@@ -1,13 +0,0 @@
-xlator_LTLIBRARIES = symlink-cache.la
-xlatordir = $(libdir)/glusterfs/$(PACKAGE_VERSION)/xlator/testing/performance
-
-symlink_cache_la_LDFLAGS = -module -avoid-version 
-
-symlink_cache_la_SOURCES = symlink-cache.c
-symlink_cache_la_LIBADD = $(top_builddir)/libglusterfs/src/libglusterfs.la
-
-AM_CPPFLAGS = $(GF_CPPFLAGS) -I$(top_srcdir)/libglusterfs/src
-
-AM_CFLAGS = -Wall $(GF_CFLAGS)
-
-CLEANFILES = 
diff --git a/xlators/performance/symlink-cache/src/symlink-cache.c b/xlators/performance/symlink-cache/src/symlink-cache.c
deleted file mode 100644
index c65629b6993..00000000000
--- a/xlators/performance/symlink-cache/src/symlink-cache.c
+++ /dev/null
@@ -1,399 +0,0 @@
-/*
-  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
-  This file is part of GlusterFS.
-
-  This file is licensed to you under your choice of the GNU Lesser
-  General Public License, version 3 or any later version (LGPLv3 or
-  later), or the GNU General Public License, version 2 (GPLv2), in all
-  cases as published by the Free Software Foundation.
-*/
-
-
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
-
-#include "glusterfs.h"
-#include "logging.h"
-#include "dict.h"
-#include "xlator.h"
-#include "list.h"
-#include "compat.h"
-#include "compat-errno.h"
-#include "common-utils.h"
-
-struct symlink_cache {
-	time_t ctime;
-	char   *readlink;
-};
-
-
-static int
-symlink_inode_ctx_get (inode_t *inode, xlator_t *this, void **ctx)
-{
-	int ret = 0;
-	uint64_t tmp_ctx = 0;
-	ret = inode_ctx_get (inode, this, &tmp_ctx);
-	if (-1 == ret)
-		gf_log (this->name, GF_LOG_ERROR, "dict get failed");
-	else
-		*ctx = (void *)(long)tmp_ctx;
-
-	return 0;
-}
-
-
-static int
-symlink_inode_ctx_set (inode_t *inode, xlator_t *this, void *ctx)
-{
-	int ret = 0;
-	ret = inode_ctx_put (inode, this, (uint64_t)(long) ctx);
-	if (-1 == ret)
-		gf_log (this->name, GF_LOG_ERROR, "dict set failed");
-
-	return 0;
-}
-
-
-int
-sc_cache_update (xlator_t *this, inode_t *inode, const char *link)
-{
-	struct symlink_cache *sc = NULL;
-
-	symlink_inode_ctx_get (inode, this, VOID(&sc));
-	if (!sc)
-		return 0;
-
-	if (!sc->readlink) {
-		gf_log (this->name, GF_LOG_DEBUG,
-			"updating cache: %s", link);
-
-		sc->readlink = strdup (link);
-	} else {
-		gf_log (this->name, GF_LOG_DEBUG,
-			"not updating existing cache: %s with %s",
-			sc->readlink, link);
-	}
-
-	return 0;
-}
-
-
-int
-sc_cache_set (xlator_t *this, inode_t *inode, struct iatt *buf,
-              const char *link)
-{
-	struct symlink_cache *sc = NULL;
-	int                   ret = -1;
-	int                   need_set = 0;
-
-
-	symlink_inode_ctx_get (inode, this, VOID(&sc));
-	if (!sc) {
-		need_set = 1;
-		sc = CALLOC (1, sizeof (*sc));
-		if (!sc) {
-			gf_log (this->name, GF_LOG_ERROR,
-				"out of memory :(");
-			goto err;
-		}
-	}
-
-	if (sc->readlink) {
-		gf_log (this->name, GF_LOG_DEBUG,
-			"replacing old cache: %s with new cache: %s",
-			sc->readlink, link);
-		FREE (sc->readlink);
-		sc->readlink = NULL;
-	}
-
-	if (link) {
-		sc->readlink = strdup (link);
-		if (!sc->readlink) {
-			gf_log (this->name, GF_LOG_ERROR,
-				"out of memory :(");
-			goto err;
-		}
-	}
-
-	sc->ctime = buf->ia_ctime;
-
-	gf_log (this->name, GF_LOG_DEBUG,
-		"setting symlink cache: %s", link);
-
-	if (need_set) {
-		ret = symlink_inode_ctx_set (inode, this, sc);
-
-		if (ret < 0) {
-			gf_log (this->name, GF_LOG_ERROR,
-				"could not set inode context (%s)",
-				strerror (-ret));
-			goto err;
-		}
-	}
-
-	return 0;
-err:
-
-	if (sc) {
-		FREE (sc->readlink);
-		sc->readlink = NULL;
-		FREE (sc);
-	}
-
-	return -1;
-}
-
-
-int
-sc_cache_flush (xlator_t *this, inode_t *inode)
-{
-	struct symlink_cache *sc = NULL;
-
-	symlink_inode_ctx_get (inode, this, VOID(&sc));
-	if (!sc)
-		return 0;
-
-	if (sc->readlink) {
-		gf_log (this->name, GF_LOG_DEBUG,
-			"flushing cache: %s", sc->readlink);
-
-		FREE (sc->readlink);
-		sc->readlink = NULL;
-	}
-
-	FREE (sc);
-
-	return 0;
-}
-
-
-int
-sc_cache_validate (xlator_t *this, inode_t *inode, struct iatt *buf)
-{
-	struct symlink_cache *sc = NULL;
-	uint64_t tmp_sc = 0;
-
-	if (!IA_ISLNK (buf->ia_type)) {
-		sc_cache_flush (this, inode);
-		return 0;
-	}
-
-	symlink_inode_ctx_get (inode, this, VOID(&sc));
-
-	if (!sc) {
-		sc_cache_set (this, inode, buf, NULL);
-		inode_ctx_get (inode, this, &tmp_sc);
-
-		if (!tmp_sc) {
-			gf_log (this->name, GF_LOG_ERROR,
-				"out of memory :(");
-			return 0;
-		}
-		sc = (struct symlink_cache *)(long)tmp_sc;
-	}
-
-	if (sc->ctime == buf->ia_ctime)
-		return 0;
-
-	/* STALE */
-	if (sc->readlink) {
-		gf_log (this->name, GF_LOG_DEBUG,
-			"flushing cache: %s", sc->readlink);
-
-		FREE (sc->readlink);
-		sc->readlink = NULL;
-	}
-
-	sc->ctime = buf->ia_ctime;
-
-	return 0;
-}
-
-
-
-int
-sc_cache_get (xlator_t *this, inode_t *inode, char **link)
-{
-	struct symlink_cache *sc = NULL;
-
-	symlink_inode_ctx_get (inode, this, VOID(&sc));
-
-	if (!sc)
-		return 0;
-
-	if (link && sc->readlink)
-		*link = strdup (sc->readlink);
-	return 0;
-}
-
-
-int
-sc_readlink_cbk (call_frame_t *frame, void *cookie,
-		 xlator_t *this, int op_ret, int op_errno,
-		 const char *link, struct iatt *sbuf, dict_t *xdata)
-{
-	if (op_ret > 0)
-		sc_cache_update (this, frame->local, link);
-
-	inode_unref (frame->local);
-	frame->local = NULL;
-
-        STACK_UNWIND_STRICT (readlink, frame, op_ret, op_errno, link, sbuf,
-                             xdata);
-        return 0;
-}
-
-
-int
-sc_readlink (call_frame_t *frame, xlator_t *this,
-	     loc_t *loc, size_t size, dict_t *xdata)
-{
-	char *link = NULL;
-        struct iatt buf = {0, };
-
-	sc_cache_get (this, loc->inode, &link);
-
-	if (link) {
-		/* cache hit */
-		gf_log (this->name, GF_LOG_DEBUG,
-			"cache hit %s -> %s",
-			loc->path, link);
-
-                /*
-                  libglusterfsclient, nfs or any other translators
-                  using buf in readlink_cbk should be aware that @buf
-                  is 0 filled
-                */
-		STACK_UNWIND_STRICT (readlink, frame, strlen (link), 0, link,
-                                     &buf, NULL);
-		FREE (link);
-		return 0;
-	}
-
-	frame->local = inode_ref (loc->inode);
-
-        STACK_WIND (frame, sc_readlink_cbk,
-                    FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->readlink,
-                    loc, size, xdata);
-
-	return 0;
-}
-
-
-int
-sc_symlink_cbk (call_frame_t *frame, void *cookie,
-		xlator_t *this, int op_ret, int op_errno,
-                inode_t *inode, struct iatt *buf, struct iatt *preparent,
-                struct iatt *postparent, dict_t *xdata)
-{
-	if (op_ret == 0) {
-		if (frame->local) {
-			sc_cache_set (this, inode, buf, frame->local);
-		}
-	}
-
-        STACK_UNWIND_STRICT (symlink, frame, op_ret, op_errno, inode, buf,
-                             preparent, postparent, xdata);
-        return 0;
-}
-
-
-int
-sc_symlink (call_frame_t *frame, xlator_t *this,
-	    const char *dst, loc_t *src, mode_t umask, dict_t *xdata)
-{
-	frame->local = strdup (dst);
-
-        STACK_WIND (frame, sc_symlink_cbk,
-                    FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->symlink,
-                    dst, src, umask, xdata);
-
-	return 0;
-}
-
-
-int
-sc_lookup_cbk (call_frame_t *frame, void *cookie,
-	       xlator_t *this, int op_ret, int op_errno,
-	       inode_t *inode, struct iatt *buf, dict_t *xdata,
-               struct iatt *postparent)
-{
-	if (op_ret == 0)
-		sc_cache_validate (this, inode, buf);
-	else
-		sc_cache_flush (this, inode);
-
-        STACK_UNWIND_STRICT (lookup, frame, op_ret, op_errno, inode, buf,
-                             xdata, postparent);
-        return 0;
-}
-
-
-int
-sc_lookup (call_frame_t *frame, xlator_t *this,
-	   loc_t *loc, dict_t *xdata)
-{
-        STACK_WIND (frame, sc_lookup_cbk,
-                    FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->lookup,
-                    loc, xdata);
-
-        return 0;
-}
-
-
-int
-sc_forget (xlator_t *this,
-	   inode_t *inode)
-{
-	sc_cache_flush (this, inode);
-
-        return 0;
-}
-
-
-int32_t
-init (xlator_t *this)
-{
-        if (!this->children || this->children->next)
-        {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "FATAL: volume (%s) not configured with exactly one "
-			"child", this->name);
-                return -1;
-        }
-
-	if (!this->parents) {
-		gf_log (this->name, GF_LOG_WARNING,
-			"dangling volume. check volfile ");
-	}
-
-        return 0;
-}
-
-
-void
-fini (xlator_t *this)
-{
-        return;
-}
-
-
-struct xlator_fops fops = {
-	.lookup      = sc_lookup,
-	.symlink     = sc_symlink,
-	.readlink    = sc_readlink,
-};
-
-
-struct xlator_cbks cbks = {
-        .forget  = sc_forget,
-};
-
-struct volume_options options[] = {
-	{ .key = {NULL} },
-};
diff --git a/xlators/performance/write-behind/src/Makefile.am b/xlators/performance/write-behind/src/Makefile.am
index 6c829d8ee36..a6a16fcc080 100644
--- a/xlators/performance/write-behind/src/Makefile.am
+++ b/xlators/performance/write-behind/src/Makefile.am
@@ -1,14 +1,15 @@
 xlator_LTLIBRARIES = write-behind.la
 xlatordir = $(libdir)/glusterfs/$(PACKAGE_VERSION)/xlator/performance
 
-write_behind_la_LDFLAGS = -module -avoid-version 
+write_behind_la_LDFLAGS = -module $(GF_XLATOR_DEFAULT_LDFLAGS)
 
 write_behind_la_SOURCES = write-behind.c
 write_behind_la_LIBADD = $(top_builddir)/libglusterfs/src/libglusterfs.la
 
-noinst_HEADERS = write-behind-mem-types.h
+noinst_HEADERS = write-behind-mem-types.h write-behind-messages.h
 
-AM_CPPFLAGS = $(GF_CPPFLAGS) -I$(top_srcdir)/libglusterfs/src
+AM_CPPFLAGS = $(GF_CPPFLAGS) -I$(top_srcdir)/libglusterfs/src \
+	-I$(top_srcdir)/rpc/xdr/src -I$(top_builddir)/rpc/xdr/src
 
 AM_CFLAGS = -Wall $(GF_CFLAGS)
 
diff --git a/xlators/performance/write-behind/src/write-behind-mem-types.h b/xlators/performance/write-behind/src/write-behind-mem-types.h
index f64f429ce22..a0647299150 100644
--- a/xlators/performance/write-behind/src/write-behind-mem-types.h
+++ b/xlators/performance/write-behind/src/write-behind-mem-types.h
@@ -8,19 +8,17 @@
   cases as published by the Free Software Foundation.
 */
 
-
 #ifndef __WB_MEM_TYPES_H__
 #define __WB_MEM_TYPES_H__
 
-#include "mem-types.h"
+#include <glusterfs/mem-types.h>
 
 enum gf_wb_mem_types_ {
-        gf_wb_mt_wb_file_t   = gf_common_mt_end + 1,
-        gf_wb_mt_wb_request_t,
-        gf_wb_mt_iovec,
-        gf_wb_mt_wb_conf_t,
-        gf_wb_mt_wb_inode_t,
-        gf_wb_mt_end
+    gf_wb_mt_wb_file_t = gf_common_mt_end + 1,
+    gf_wb_mt_wb_request_t,
+    gf_wb_mt_iovec,
+    gf_wb_mt_wb_conf_t,
+    gf_wb_mt_wb_inode_t,
+    gf_wb_mt_end
 };
 #endif
-
diff --git a/xlators/performance/write-behind/src/write-behind-messages.h b/xlators/performance/write-behind/src/write-behind-messages.h
new file mode 100644
index 00000000000..e9ea474879b
--- /dev/null
+++ b/xlators/performance/write-behind/src/write-behind-messages.h
@@ -0,0 +1,31 @@
+/*Copyright (c) 2015 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
+*/
+
+#ifndef _WRITE_BEHIND_MESSAGES_H_
+#define _WRITE_BEHIND_MESSAGES_H_
+
+#include <glusterfs/glfs-message-id.h>
+
+/* To add new message IDs, append new identifiers at the end of the list.
+ *
+ * Never remove a message ID. If it's not used anymore, you can rename it or
+ * leave it as it is, but not delete it. This is to prevent reutilization of
+ * IDs by other messages.
+ *
+ * The component name must match one of the entries defined in
+ * glfs-message-id.h.
+ */
+
+GLFS_MSGID(WRITE_BEHIND, WRITE_BEHIND_MSG_EXCEEDED_MAX_SIZE,
+           WRITE_BEHIND_MSG_INIT_FAILED, WRITE_BEHIND_MSG_INVALID_ARGUMENT,
+           WRITE_BEHIND_MSG_NO_MEMORY, WRITE_BEHIND_MSG_SIZE_NOT_SET,
+           WRITE_BEHIND_MSG_VOL_MISCONFIGURED,
+           WRITE_BEHIND_MSG_RES_UNAVAILABLE);
+
+#endif /* _WRITE_BEHIND_MESSAGES_H_ */
diff --git a/xlators/performance/write-behind/src/write-behind.c b/xlators/performance/write-behind/src/write-behind.c
index 3cb0d449bbc..00cfca016e6 100644
--- a/xlators/performance/write-behind/src/write-behind.c
+++ b/xlators/performance/write-behind/src/write-behind.c
@@ -8,223 +8,258 @@
   cases as published by the Free Software Foundation.
 */
 
-
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
-
-#include "glusterfs.h"
-#include "logging.h"
-#include "dict.h"
-#include "xlator.h"
-#include "list.h"
-#include "compat.h"
-#include "compat-errno.h"
-#include "common-utils.h"
-#include "call-stub.h"
-#include "statedump.h"
-#include "defaults.h"
+#include <glusterfs/glusterfs.h>
+#include <glusterfs/logging.h>
+#include <glusterfs/dict.h>
+#include <glusterfs/xlator.h>
+#include <glusterfs/list.h>
+#include <glusterfs/compat.h>
+#include <glusterfs/compat-errno.h>
+#include <glusterfs/common-utils.h>
+#include <glusterfs/call-stub.h>
+#include <glusterfs/statedump.h>
+#include <glusterfs/defaults.h>
 #include "write-behind-mem-types.h"
+#include "write-behind-messages.h"
 
-#define MAX_VECTOR_COUNT          8
-#define WB_AGGREGATE_SIZE         131072 /* 128 KB */
-#define WB_WINDOW_SIZE            1048576 /* 1MB */
+#define MAX_VECTOR_COUNT 8
+#define WB_AGGREGATE_SIZE 131072 /* 128 KB */
+#define WB_WINDOW_SIZE 1048576   /* 1MB */
 
 typedef struct list_head list_head_t;
 struct wb_conf;
 struct wb_inode;
 
 typedef struct wb_inode {
-        ssize_t      window_conf;
-        ssize_t      window_current;
-	ssize_t      transit; /* size of data stack_wound, and yet
-				 to be fulfilled (wb_fulfill_cbk).
-				 used for trickling_writes
-			      */
-
-        list_head_t  all;    /* All requests, from enqueue() till destroy().
-				Used only for resetting generation
-				number when empty.
-			     */
-        list_head_t  todo;   /* Work to do (i.e, STACK_WIND to server).
-				Once we STACK_WIND, the entry is taken
-				off the list. If it is non-sync write,
-				then we continue to track it via @liability
-				or @temptation depending on the status
-				of its writeback.
-			     */
-        list_head_t  liability;   /* Non-sync writes which are lied
-				     (STACK_UNWIND'ed to caller) but ack
-				     from server not yet complete. This
-				     is the "liability" which we hold, and
-				     must guarantee that dependent operations
-				     which arrive later (which overlap, etc.)
-				     are issued only after their dependencies
-				     in this list are "fulfilled".
-
-				     Server acks for entries in this list
-				     shrinks the window.
-
-				     The sum total of all req->write_size
-				     of entries in this list must be kept less
-				     than the permitted window size.
-				  */
-        list_head_t  temptation;  /* Operations for which we are tempted
-				     to 'lie' (write-behind), but temporarily
-				     holding off (because of insufficient
-				     window capacity, etc.)
-
-				     This is the list to look at to grow
-				     the window (in __wb_pick_unwinds()).
-
-				     Entries typically get chosen from
-				     write-behind from this list, and therefore
-				     get "upgraded" to the "liability" list.
-			     */
-	list_head_t  wip; /* List of write calls in progress, SYNC or non-SYNC
-			     which are currently STACK_WIND'ed towards the server.
-			     This is for guaranteeing that no two overlapping
-			     writes are in progress at the same time. Modules
-			     like eager-lock in AFR depend on this behavior.
-			  */
-	uint64_t     gen;    /* Liability generation number. Represents
-				the current 'state' of liability. Every
-				new addition to the liability list bumps
-				the generation number.
-
-				a newly arrived request is only required
-				to perform causal checks against the entries
-				in the liability list which were present
-				at the time of its addition. the generation
-				number at the time of its addition is stored
-				in the request and used during checks.
-
-				the liability list can grow while the request
-				waits in the todo list waiting for its
-				dependent operations to complete. however
-				it is not of the request's concern to depend
-				itself on those new entries which arrived
-				after it arrived (i.e, those that have a
-				liability generation higher than itself)
-			     */
-	size_t       size; /* Size of the file to catch write after EOF. */
-        gf_lock_t    lock;
-        xlator_t    *this;
-} wb_inode_t;
+    ssize_t window_conf;
+    ssize_t window_current;
+    ssize_t transit; /* size of data stack_wound, and yet
+                        to be fulfilled (wb_fulfill_cbk).
+                        used for trickling_writes
+                     */
+
+    list_head_t all;        /* All requests, from enqueue() till destroy().
+                               Used only for resetting generation
+                               number when empty.
+                            */
+    list_head_t todo;       /* Work to do (i.e, STACK_WIND to server).
+                               Once we STACK_WIND, the entry is taken
+                               off the list. If it is non-sync write,
+                               then we continue to track it via @liability
+                               or @temptation depending on the status
+                               of its writeback.
+                            */
+    list_head_t liability;  /* Non-sync writes which are lied
+                               (STACK_UNWIND'ed to caller) but ack
+                               from server not yet complete. This
+                               is the "liability" which we hold, and
+                               must guarantee that dependent operations
+                               which arrive later (which overlap, etc.)
+                               are issued only after their dependencies
+                               in this list are "fulfilled".
+ 
+                               Server acks for entries in this list
+                               shrinks the window.
+ 
+                               The sum total of all req->write_size
+                               of entries in this list must be kept less
+                               than the permitted window size.
+                            */
+    list_head_t temptation; /* Operations for which we are tempted
+                               to 'lie' (write-behind), but temporarily
+                               holding off (because of insufficient
+                               window capacity, etc.)
+
+                               This is the list to look at to grow
+                               the window (in __wb_pick_unwinds()).
+
+                               Entries typically get chosen from
+                               write-behind from this list, and therefore
+                               get "upgraded" to the "liability" list.
+                       */
+    list_head_t wip;        /* List of write calls in progress, SYNC or non-SYNC
+                               which are currently STACK_WIND'ed towards the server.
+                               This is for guaranteeing that no two overlapping
+                               writes are in progress at the same time. Modules
+                               like eager-lock in AFR depend on this behavior.
+                            */
+    list_head_t invalidate_list; /* list of wb_inodes that were marked for
+                                  * iatt invalidation due to requests in
+                                  * liability queue fulfilled while there
+                                  * was a readdirp session on parent
+                                  * directory. For a directory inode, this
+                                  * list points to list of children.
+                                  */
+    uint64_t gen;                /* Liability generation number. Represents
+                                    the current 'state' of liability. Every
+                                    new addition to the liability list bumps
+                                    the generation number.
+               
+                                    a newly arrived request is only required
+                                    to perform causal checks against the entries
+                                    in the liability list which were present
+                                    at the time of its addition. the generation
+                                    number at the time of its addition is stored
+                                    in the request and used during checks.
+               
+                                    the liability list can grow while the request
+                                    waits in the todo list waiting for its
+                                    dependent operations to complete. however
+                                    it is not of the request's concern to depend
+                                    itself on those new entries which arrived
+                                    after it arrived (i.e, those that have a
+                                    liability generation higher than itself)
+                                 */
+    size_t size; /* Size of the file to catch write after EOF. */
+    gf_lock_t lock;
+    xlator_t *this;
+    inode_t *inode;
+    int dontsync; /* If positive, don't pick lies for
+                   * winding. This is needed to break infinite
+                   * recursion during invocation of
+                   * wb_process_queue from
+                   * wb_fulfill_cbk in case of an
+                   * error during fulfill.
+                   */
+    gf_atomic_int32_t readdirps;
+    gf_atomic_int8_t invalidate;
 
+} wb_inode_t;
 
 typedef struct wb_request {
-        list_head_t           all;
-        list_head_t           todo;
-	list_head_t           lie;  /* either in @liability or @temptation */
-        list_head_t           winds;
-        list_head_t           unwinds;
-        list_head_t           wip;
-
-        call_stub_t          *stub;
-
-        ssize_t               write_size;  /* currently held size
-					      (after collapsing) */
-	size_t                orig_size;   /* size which arrived with the request.
-					      This is the size by which we grow
-					      the window when unwinding the frame.
-					   */
-        size_t                total_size;  /* valid only in @head in wb_fulfill().
-					      This is the size with which we perform
-					      STACK_WIND to server and therefore the
-					      amount by which we shrink the window.
-					   */
-
-	int                   op_ret;
-	int                   op_errno;
-
-        int32_t               refcount;
-        wb_inode_t           *wb_inode;
-        glusterfs_fop_t       fop;
-        gf_lkowner_t          lk_owner;
-	struct iobref        *iobref;
-	uint64_t              gen;  /* inode liability state at the time of
-				       request arrival */
-
-	fd_t                 *fd;
-	struct {
-		size_t        size;          /* 0 size == till infinity */
-		off_t         off;
-		int           append:1;      /* offset is invalid. only one
-						outstanding append at a time */
-		int           tempted:1;     /* true only for non-sync writes */
-		int           lied:1;        /* sin committed */
-		int           fulfilled:1;   /* got server acknowledgement */
-		int           go:1;          /* enough aggregating, good to go */
-	} ordering;
+    list_head_t all;
+    list_head_t todo;
+    list_head_t lie; /* either in @liability or @temptation */
+    list_head_t winds;
+    list_head_t unwinds;
+    list_head_t wip;
+
+    call_stub_t *stub;
+
+    ssize_t write_size; /* currently held size
+                           (after collapsing) */
+    size_t orig_size;   /* size which arrived with the request.
+                           This is the size by which we grow
+                           the window when unwinding the frame.
+                        */
+    size_t total_size;  /* valid only in @head in wb_fulfill().
+                           This is the size with which we perform
+                           STACK_WIND to server and therefore the
+                           amount by which we shrink the window.
+                        */
+
+    int op_ret;
+    int op_errno;
+
+    int32_t refcount;
+    wb_inode_t *wb_inode;
+    glusterfs_fop_t fop;
+    gf_lkowner_t lk_owner;
+    pid_t client_pid;
+    struct iobref *iobref;
+    uint64_t gen; /* inode liability state at the time of
+                     request arrival */
+
+    fd_t *fd;
+    int wind_count; /* number of sync-attempts. Only
+                       for debug purposes */
+    struct {
+        size_t size; /* 0 size == till infinity */
+        off_t off;
+        int append : 1;    /* offset is invalid. only one
+                              outstanding append at a time */
+        int tempted : 1;   /* true only for non-sync writes */
+        int lied : 1;      /* sin committed */
+        int fulfilled : 1; /* got server acknowledgement */
+        int go : 1;        /* enough aggregating, good to go */
+    } ordering;
+
+    /* for debug purposes. A request might outlive the fop it is
+     * representing. So, preserve essential info for logging.
+     */
+    uint64_t unique;
+    uuid_t gfid;
 } wb_request_t;
 
-
 typedef struct wb_conf {
-        uint64_t         aggregate_size;
-        uint64_t         window_size;
-        gf_boolean_t     flush_behind;
-        gf_boolean_t     trickling_writes;
-	gf_boolean_t     strict_write_ordering;
-	gf_boolean_t     strict_O_DIRECT;
+    uint64_t aggregate_size;
+    uint64_t page_size;
+    uint64_t window_size;
+    gf_boolean_t flush_behind;
+    gf_boolean_t trickling_writes;
+    gf_boolean_t strict_write_ordering;
+    gf_boolean_t strict_O_DIRECT;
+    gf_boolean_t resync_after_fsync;
 } wb_conf_t;
 
-
-void
-wb_process_queue (wb_inode_t *wb_inode);
-
-
 wb_inode_t *
-__wb_inode_ctx_get (xlator_t *this, inode_t *inode)
+__wb_inode_ctx_get(xlator_t *this, inode_t *inode)
 {
-        uint64_t    value    = 0;
-        wb_inode_t *wb_inode = NULL;
+    uint64_t value = 0;
+    wb_inode_t *wb_inode = NULL;
+    int ret = 0;
 
-        __inode_ctx_get (inode, this, &value);
-        wb_inode = (wb_inode_t *)(unsigned long) value;
+    ret = __inode_ctx_get(inode, this, &value);
+    if (ret)
+        return NULL;
 
-        return wb_inode;
-}
+    wb_inode = (wb_inode_t *)(unsigned long)value;
 
+    return wb_inode;
+}
 
 wb_inode_t *
-wb_inode_ctx_get (xlator_t *this, inode_t *inode)
+wb_inode_ctx_get(xlator_t *this, inode_t *inode)
 {
-        wb_inode_t *wb_inode = NULL;
+    wb_inode_t *wb_inode = NULL;
 
-        GF_VALIDATE_OR_GOTO ("write-behind", this, out);
-        GF_VALIDATE_OR_GOTO (this->name, inode, out);
+    GF_VALIDATE_OR_GOTO("write-behind", this, out);
+    GF_VALIDATE_OR_GOTO(this->name, inode, out);
 
-        LOCK (&inode->lock);
-        {
-                wb_inode = __wb_inode_ctx_get (this, inode);
-        }
-        UNLOCK (&inode->lock);
+    LOCK(&inode->lock);
+    {
+        wb_inode = __wb_inode_ctx_get(this, inode);
+    }
+    UNLOCK(&inode->lock);
 out:
-        return wb_inode;
+    return wb_inode;
 }
 
-
-gf_boolean_t
-wb_fd_err (fd_t *fd, xlator_t *this, int32_t *op_errno)
+static void
+wb_set_invalidate(wb_inode_t *wb_inode)
 {
-        gf_boolean_t err   = _gf_false;
-        uint64_t     value = 0;
-        int32_t      tmp   = 0;
+    int readdirps = 0;
+    inode_t *parent_inode = NULL;
+    wb_inode_t *wb_parent_inode = NULL;
 
-	if (fd_ctx_get (fd, this, &value) == 0) {
-                if (op_errno) {
-                        tmp = value;
-                        *op_errno = tmp;
-                }
+    parent_inode = inode_parent(wb_inode->inode, NULL, NULL);
+    if (parent_inode)
+        wb_parent_inode = wb_inode_ctx_get(wb_inode->this, parent_inode);
 
-                err = _gf_true;
+    if (wb_parent_inode) {
+        LOCK(&wb_parent_inode->lock);
+        {
+            readdirps = GF_ATOMIC_GET(wb_parent_inode->readdirps);
+            if (readdirps && list_empty(&wb_inode->invalidate_list)) {
+                inode_ref(wb_inode->inode);
+                GF_ATOMIC_INIT(wb_inode->invalidate, 1);
+                list_add(&wb_inode->invalidate_list,
+                         &wb_parent_inode->invalidate_list);
+            }
         }
+        UNLOCK(&wb_parent_inode->lock);
+    } else {
+        GF_ATOMIC_INIT(wb_inode->invalidate, 0);
+    }
 
-        return err;
+    if (parent_inode)
+        inode_unref(parent_inode);
+
+    return;
 }
 
+void
+wb_process_queue(wb_inode_t *wb_inode);
 
 /*
   Below is a succinct explanation of the code deciding whether two regions
@@ -251,2036 +286,2993 @@ wb_fd_err (fd_t *fd, xlator_t *this, int32_t *op_errno)
 */
 
 gf_boolean_t
-wb_requests_overlap (wb_request_t *req1, wb_request_t *req2)
+wb_requests_overlap(wb_request_t *req1, wb_request_t *req2)
 {
-        uint64_t         r1_start = 0;
-	uint64_t         r1_end = 0;
-	uint64_t         r2_start = 0;
-	uint64_t         r2_end = 0;
-        enum _gf_boolean do_overlap = 0;
+    uint64_t r1_start = 0;
+    uint64_t r1_end = 0;
+    uint64_t r2_start = 0;
+    uint64_t r2_end = 0;
+    gf_boolean_t do_overlap = _gf_false;
 
-        r1_start = req1->ordering.off;
-	if (req1->ordering.size)
-		r1_end = r1_start + req1->ordering.size - 1;
-	else
-		r1_end = ULLONG_MAX;
+    r1_start = req1->ordering.off;
+    if (req1->ordering.size)
+        r1_end = r1_start + req1->ordering.size - 1;
+    else
+        r1_end = ULLONG_MAX;
 
-        r2_start = req2->ordering.off;
-	if (req2->ordering.size)
-		r2_end = r2_start + req2->ordering.size - 1;
-	else
-		r2_end = ULLONG_MAX;
+    r2_start = req2->ordering.off;
+    if (req2->ordering.size)
+        r2_end = r2_start + req2->ordering.size - 1;
+    else
+        r2_end = ULLONG_MAX;
 
-        do_overlap = ((r1_end >= r2_start) && (r2_end >= r1_start));
+    do_overlap = ((r1_end >= r2_start) && (r2_end >= r1_start));
 
-        return do_overlap;
+    return do_overlap;
 }
 
-
 gf_boolean_t
-wb_requests_conflict (wb_request_t *lie, wb_request_t *req)
+wb_requests_conflict(wb_request_t *lie, wb_request_t *req)
 {
-	wb_conf_t  *conf = NULL;
+    wb_conf_t *conf = NULL;
 
-	conf = req->wb_inode->this->private;
+    conf = req->wb_inode->this->private;
 
-	if (lie == req)
-		/* request cannot conflict with itself */
-		return _gf_false;
+    if (lie == req)
+        /* request cannot conflict with itself */
+        return _gf_false;
 
-	if (lie->gen >= req->gen)
-		/* this liability entry was behind
-		   us in the todo list */
-		return _gf_false;
+    if (lie->gen >= req->gen)
+        /* this liability entry was behind
+           us in the todo list */
+        return _gf_false;
 
-	if (lie->ordering.append)
-		/* all modifications wait for the completion
-		   of outstanding append */
-		return _gf_true;
+    if (lie->ordering.append)
+        /* all modifications wait for the completion
+           of outstanding append */
+        return _gf_true;
 
-	if (conf->strict_write_ordering)
-		/* We are sure (lie->gen < req->gen) by now. So
-		   skip overlap check if strict write ordering is
-		   requested and always return "conflict" against a
-		   lower generation lie. */
-		return _gf_true;
+    if (conf->strict_write_ordering)
+        /* We are sure (lie->gen < req->gen) by now. So
+           skip overlap check if strict write ordering is
+           requested and always return "conflict" against a
+           lower generation lie. */
+        return _gf_true;
 
-	return wb_requests_overlap (lie, req);
+    return wb_requests_overlap(lie, req);
 }
 
-
-gf_boolean_t
-wb_liability_has_conflict (wb_inode_t *wb_inode, wb_request_t *req)
+wb_request_t *
+wb_liability_has_conflict(wb_inode_t *wb_inode, wb_request_t *req)
 {
-        wb_request_t *each     = NULL;
+    wb_request_t *each = NULL;
 
-        list_for_each_entry (each, &wb_inode->liability, lie) {
-		if (wb_requests_conflict (each, req))
-			return _gf_true;
-        }
+    list_for_each_entry(each, &wb_inode->liability, lie)
+    {
+        if (wb_requests_conflict(each, req) && (!each->ordering.fulfilled))
+            /* A fulfilled request shouldn't block another
+             * request (even a dependent one) from winding.
+             */
+            return each;
+    }
 
-        return _gf_false;
+    return NULL;
 }
 
-
-gf_boolean_t
-wb_wip_has_conflict (wb_inode_t *wb_inode, wb_request_t *req)
+wb_request_t *
+wb_wip_has_conflict(wb_inode_t *wb_inode, wb_request_t *req)
 {
-        wb_request_t *each     = NULL;
+    wb_request_t *each = NULL;
 
-	if (req->stub->fop != GF_FOP_WRITE)
-		/* non-writes fundamentally never conflict with WIP requests */
-		return _gf_false;
+    if (req->stub->fop != GF_FOP_WRITE)
+        /* non-writes fundamentally never conflict with WIP requests */
+        return NULL;
 
-        list_for_each_entry (each, &wb_inode->wip, wip) {
-		if (each == req)
-			/* request never conflicts with itself,
-			   though this condition should never occur.
-			*/
-			continue;
+    list_for_each_entry(each, &wb_inode->wip, wip)
+    {
+        if (each == req)
+            /* request never conflicts with itself,
+               though this condition should never occur.
+            */
+            continue;
 
-		if (wb_requests_overlap (each, req))
-			return _gf_true;
-        }
+        if (wb_requests_overlap(each, req))
+            return each;
+    }
 
-        return _gf_false;
+    return NULL;
 }
 
-
 static int
-__wb_request_unref (wb_request_t *req)
-{
-        int         ret = -1;
-	wb_inode_t *wb_inode = NULL;
-
-	wb_inode = req->wb_inode;
-
-        if (req->refcount <= 0) {
-                gf_log ("wb-request", GF_LOG_WARNING,
-                        "refcount(%d) is <= 0", req->refcount);
-                goto out;
+__wb_request_unref(wb_request_t *req)
+{
+    int ret = -1;
+    wb_inode_t *wb_inode = NULL;
+    char gfid[64] = {
+        0,
+    };
+
+    wb_inode = req->wb_inode;
+
+    if (req->refcount <= 0) {
+        uuid_utoa_r(req->gfid, gfid);
+
+        gf_msg(
+            "wb-request", GF_LOG_WARNING, 0, WRITE_BEHIND_MSG_RES_UNAVAILABLE,
+            "(unique=%" PRIu64 ", fop=%s, gfid=%s, gen=%" PRIu64
+            "): "
+            "refcount(%d) is <= 0 ",
+            req->unique, gf_fop_list[req->fop], gfid, req->gen, req->refcount);
+        goto out;
+    }
+
+    ret = --req->refcount;
+    if (req->refcount == 0) {
+        uuid_utoa_r(req->gfid, gfid);
+
+        gf_log_callingfn(wb_inode->this->name, GF_LOG_DEBUG,
+                         "(unique = %" PRIu64
+                         ", fop=%s, gfid=%s, "
+                         "gen=%" PRIu64
+                         "): destroying request, "
+                         "removing from all queues",
+                         req->unique, gf_fop_list[req->fop], gfid, req->gen);
+
+        list_del_init(&req->todo);
+        list_del_init(&req->lie);
+        list_del_init(&req->wip);
+
+        list_del_init(&req->all);
+        if (list_empty(&wb_inode->all)) {
+            wb_inode->gen = 0;
+            /* in case of accounting errors? */
+            wb_inode->window_current = 0;
         }
 
-        ret = --req->refcount;
-        if (req->refcount == 0) {
-                list_del_init (&req->todo);
-                list_del_init (&req->lie);
-		list_del_init (&req->wip);
-
-		list_del_init (&req->all);
-		if (list_empty (&wb_inode->all)) {
-			wb_inode->gen = 0;
-			/* in case of accounting errors? */
-			wb_inode->window_current = 0;
-		}
+        list_del_init(&req->winds);
+        list_del_init(&req->unwinds);
 
-		list_del_init (&req->winds);
-		list_del_init (&req->unwinds);
-
-                if (req->stub && req->ordering.tempted) {
-                        call_stub_destroy (req->stub);
-			req->stub = NULL;
-                } /* else we would have call_resume()'ed */
+        if (req->stub) {
+            call_stub_destroy(req->stub);
+            req->stub = NULL;
+        }
 
-		if (req->iobref)
-			iobref_unref (req->iobref);
+        if (req->iobref)
+            iobref_unref(req->iobref);
 
-		if (req->fd)
-			fd_unref (req->fd);
+        if (req->fd)
+            fd_unref(req->fd);
 
-                GF_FREE (req);
-        }
+        GF_FREE(req);
+    }
 out:
-        return ret;
+    return ret;
 }
 
-
 static int
-wb_request_unref (wb_request_t *req)
+wb_request_unref(wb_request_t *req)
 {
-        wb_inode_t *wb_inode = NULL;
-        int         ret      = -1;
+    wb_inode_t *wb_inode = NULL;
+    int ret = -1;
 
-        GF_VALIDATE_OR_GOTO ("write-behind", req, out);
+    GF_VALIDATE_OR_GOTO("write-behind", req, out);
 
-        wb_inode = req->wb_inode;
+    wb_inode = req->wb_inode;
 
-        LOCK (&wb_inode->lock);
-        {
-                ret = __wb_request_unref (req);
-        }
-        UNLOCK (&wb_inode->lock);
+    LOCK(&wb_inode->lock);
+    {
+        ret = __wb_request_unref(req);
+    }
+    UNLOCK(&wb_inode->lock);
 
 out:
-        return ret;
+    return ret;
 }
 
-
 static wb_request_t *
-__wb_request_ref (wb_request_t *req)
+__wb_request_ref(wb_request_t *req)
 {
-        GF_VALIDATE_OR_GOTO ("write-behind", req, out);
+    GF_VALIDATE_OR_GOTO("write-behind", req, out);
 
-        if (req->refcount < 0) {
-                gf_log ("wb-request", GF_LOG_WARNING,
-                        "refcount(%d) is < 0", req->refcount);
-                req = NULL;
-                goto out;
-        }
+    if (req->refcount < 0) {
+        gf_msg("wb-request", GF_LOG_WARNING, 0,
+               WRITE_BEHIND_MSG_RES_UNAVAILABLE, "refcount(%d) is < 0",
+               req->refcount);
+        req = NULL;
+        goto out;
+    }
 
-        req->refcount++;
+    req->refcount++;
 
 out:
-        return req;
+    return req;
 }
 
-
 wb_request_t *
-wb_request_ref (wb_request_t *req)
+wb_request_ref(wb_request_t *req)
 {
-        wb_inode_t *wb_inode = NULL;
+    wb_inode_t *wb_inode = NULL;
 
-        GF_VALIDATE_OR_GOTO ("write-behind", req, out);
+    GF_VALIDATE_OR_GOTO("write-behind", req, out);
 
-        wb_inode = req->wb_inode;
-        LOCK (&wb_inode->lock);
-        {
-                req = __wb_request_ref (req);
-        }
-        UNLOCK (&wb_inode->lock);
+    wb_inode = req->wb_inode;
+    LOCK(&wb_inode->lock);
+    {
+        req = __wb_request_ref(req);
+    }
+    UNLOCK(&wb_inode->lock);
 
 out:
-        return req;
+    return req;
 }
 
-
 gf_boolean_t
-wb_enqueue_common (wb_inode_t *wb_inode, call_stub_t *stub, int tempted)
-{
-        wb_request_t *req     = NULL;
-
-        GF_VALIDATE_OR_GOTO ("write-behind", wb_inode, out);
-        GF_VALIDATE_OR_GOTO (wb_inode->this->name, stub, out);
-
-        req = GF_CALLOC (1, sizeof (*req), gf_wb_mt_wb_request_t);
-        if (!req)
-                goto out;
-
-        INIT_LIST_HEAD (&req->all);
-        INIT_LIST_HEAD (&req->todo);
-        INIT_LIST_HEAD (&req->lie);
-        INIT_LIST_HEAD (&req->winds);
-        INIT_LIST_HEAD (&req->unwinds);
-        INIT_LIST_HEAD (&req->wip);
-
-        req->stub = stub;
-        req->wb_inode = wb_inode;
-        req->fop  = stub->fop;
-	req->ordering.tempted = tempted;
-
-        if (stub->fop == GF_FOP_WRITE) {
-                req->write_size = iov_length (stub->args.vector,
-					      stub->args.count);
-
-		/* req->write_size can change as we collapse
-		   small writes. But the window needs to grow
-		   only by how much we acknowledge the app. so
-		   copy the original size in orig_size for the
-		   purpose of accounting.
-		*/
-		req->orig_size = req->write_size;
-
-		/* Let's be optimistic that we can
-		   lie about it
-		*/
-		req->op_ret = req->write_size;
-		req->op_errno = 0;
-
-		if (stub->args.fd->flags & O_APPEND)
-			req->ordering.append = 1;
-        }
-
-        req->lk_owner = stub->frame->root->lk_owner;
-
-	switch (stub->fop) {
-	case GF_FOP_WRITE:
-		LOCK (&wb_inode->lock);
-		{
-			if (wb_inode->size < stub->args.offset) {
-				req->ordering.off = wb_inode->size;
-				req->ordering.size = stub->args.offset
-				                     + req->write_size
-						     - wb_inode->size;
-			} else {
-				req->ordering.off = stub->args.offset;
-				req->ordering.size = req->write_size;
-			}
-
-			if (wb_inode->size < stub->args.offset + req->write_size)
-				wb_inode->size = stub->args.offset
-				                 + req->write_size;
-		}
-		UNLOCK (&wb_inode->lock);
-
-		req->fd = fd_ref (stub->args.fd);
-
-		break;
-	case GF_FOP_READ:
-		req->ordering.off = stub->args.offset;
-		req->ordering.size = stub->args.size;
-
-		req->fd = fd_ref (stub->args.fd);
-
-		break;
-	case GF_FOP_TRUNCATE:
-		req->ordering.off = stub->args.offset;
-		req->ordering.size = 0; /* till infinity */
-		LOCK (&wb_inode->lock);
-		{
-			wb_inode->size = req->ordering.off;
-		}
-		UNLOCK (&wb_inode->lock);
-		break;
-	case GF_FOP_FTRUNCATE:
-		req->ordering.off = stub->args.offset;
-		req->ordering.size = 0; /* till infinity */
-		LOCK (&wb_inode->lock);
-		{
-			wb_inode->size = req->ordering.off;
-		}
-		UNLOCK (&wb_inode->lock);
-
-		req->fd = fd_ref (stub->args.fd);
-
-		break;
-	default:
-		break;
-	}
-
-        LOCK (&wb_inode->lock);
-        {
-                list_add_tail (&req->all, &wb_inode->all);
-
-		req->gen = wb_inode->gen;
-
-                list_add_tail (&req->todo, &wb_inode->todo);
-		__wb_request_ref (req); /* for wind */
+wb_enqueue_common(wb_inode_t *wb_inode, call_stub_t *stub, int tempted)
+{
+    wb_request_t *req = NULL;
+    inode_t *inode = NULL;
+
+    GF_VALIDATE_OR_GOTO("write-behind", wb_inode, out);
+    GF_VALIDATE_OR_GOTO(wb_inode->this->name, stub, out);
+
+    req = GF_CALLOC(1, sizeof(*req), gf_wb_mt_wb_request_t);
+    if (!req)
+        goto out;
+
+    INIT_LIST_HEAD(&req->all);
+    INIT_LIST_HEAD(&req->todo);
+    INIT_LIST_HEAD(&req->lie);
+    INIT_LIST_HEAD(&req->winds);
+    INIT_LIST_HEAD(&req->unwinds);
+    INIT_LIST_HEAD(&req->wip);
+
+    req->stub = stub;
+    req->wb_inode = wb_inode;
+    req->fop = stub->fop;
+    req->ordering.tempted = tempted;
+    req->unique = stub->frame->root->unique;
+
+    inode = ((stub->args.fd != NULL) ? stub->args.fd->inode
+                                     : stub->args.loc.inode);
+
+    if (inode)
+        gf_uuid_copy(req->gfid, inode->gfid);
+
+    if (stub->fop == GF_FOP_WRITE) {
+        req->write_size = iov_length(stub->args.vector, stub->args.count);
+
+        /* req->write_size can change as we collapse
+           small writes. But the window needs to grow
+           only by how much we acknowledge the app. so
+           copy the original size in orig_size for the
+           purpose of accounting.
+        */
+        req->orig_size = req->write_size;
+
+        /* Let's be optimistic that we can
+           lie about it
+        */
+        req->op_ret = req->write_size;
+        req->op_errno = 0;
+
+        if (stub->args.fd && (stub->args.fd->flags & O_APPEND))
+            req->ordering.append = 1;
+    }
+
+    req->lk_owner = stub->frame->root->lk_owner;
+    req->client_pid = stub->frame->root->pid;
+
+    switch (stub->fop) {
+        case GF_FOP_WRITE:
+            LOCK(&wb_inode->lock);
+            {
+                if (wb_inode->size < stub->args.offset) {
+                    req->ordering.off = wb_inode->size;
+                    req->ordering.size = stub->args.offset + req->write_size -
+                                         wb_inode->size;
+                } else {
+                    req->ordering.off = stub->args.offset;
+                    req->ordering.size = req->write_size;
+                }
 
-		if (req->ordering.tempted) {
-			list_add_tail (&req->lie, &wb_inode->temptation);
-			__wb_request_ref (req); /* for unwind */
-		}
+                if (wb_inode->size < stub->args.offset + req->write_size)
+                    wb_inode->size = stub->args.offset + req->write_size;
+            }
+            UNLOCK(&wb_inode->lock);
+
+            req->fd = fd_ref(stub->args.fd);
+
+            break;
+        case GF_FOP_READ:
+            req->ordering.off = stub->args.offset;
+            req->ordering.size = stub->args.size;
+
+            req->fd = fd_ref(stub->args.fd);
+
+            break;
+        case GF_FOP_TRUNCATE:
+            req->ordering.off = stub->args.offset;
+            req->ordering.size = 0; /* till infinity */
+            LOCK(&wb_inode->lock);
+            {
+                wb_inode->size = req->ordering.off;
+            }
+            UNLOCK(&wb_inode->lock);
+            break;
+        case GF_FOP_FTRUNCATE:
+            req->ordering.off = stub->args.offset;
+            req->ordering.size = 0; /* till infinity */
+            LOCK(&wb_inode->lock);
+            {
+                wb_inode->size = req->ordering.off;
+            }
+            UNLOCK(&wb_inode->lock);
+
+            req->fd = fd_ref(stub->args.fd);
+
+            break;
+        default:
+            if (stub && stub->args.fd)
+                req->fd = fd_ref(stub->args.fd);
+
+            break;
+    }
+
+    LOCK(&wb_inode->lock);
+    {
+        list_add_tail(&req->all, &wb_inode->all);
+
+        req->gen = wb_inode->gen;
+
+        list_add_tail(&req->todo, &wb_inode->todo);
+        __wb_request_ref(req); /* for wind */
+
+        if (req->ordering.tempted) {
+            list_add_tail(&req->lie, &wb_inode->temptation);
+            __wb_request_ref(req); /* for unwind */
         }
-        UNLOCK (&wb_inode->lock);
+    }
+    UNLOCK(&wb_inode->lock);
 
 out:
-	if (!req)
-		return _gf_false;
+    if (!req)
+        return _gf_false;
 
-	return _gf_true;
+    return _gf_true;
 }
 
-
 gf_boolean_t
-wb_enqueue (wb_inode_t *wb_inode, call_stub_t *stub)
+wb_enqueue(wb_inode_t *wb_inode, call_stub_t *stub)
 {
-	return wb_enqueue_common (wb_inode, stub, 0);
+    return wb_enqueue_common(wb_inode, stub, 0);
 }
 
-
 gf_boolean_t
-wb_enqueue_tempted (wb_inode_t *wb_inode, call_stub_t *stub)
+wb_enqueue_tempted(wb_inode_t *wb_inode, call_stub_t *stub)
 {
-	return wb_enqueue_common (wb_inode, stub, 1);
+    return wb_enqueue_common(wb_inode, stub, 1);
 }
 
-
 wb_inode_t *
-__wb_inode_create (xlator_t *this, inode_t *inode)
+__wb_inode_create(xlator_t *this, inode_t *inode)
 {
-        wb_inode_t *wb_inode = NULL;
-        wb_conf_t  *conf     = NULL;
+    wb_inode_t *wb_inode = NULL;
+    wb_conf_t *conf = NULL;
+    int ret = 0;
 
-        GF_VALIDATE_OR_GOTO (this->name, inode, out);
+    GF_VALIDATE_OR_GOTO(this->name, inode, out);
 
-        conf = this->private;
+    conf = this->private;
 
-        wb_inode = GF_CALLOC (1, sizeof (*wb_inode), gf_wb_mt_wb_inode_t);
-        if (!wb_inode)
-                goto out;
+    wb_inode = GF_CALLOC(1, sizeof(*wb_inode), gf_wb_mt_wb_inode_t);
+    if (!wb_inode)
+        goto out;
 
-        INIT_LIST_HEAD (&wb_inode->all);
-        INIT_LIST_HEAD (&wb_inode->todo);
-        INIT_LIST_HEAD (&wb_inode->liability);
-        INIT_LIST_HEAD (&wb_inode->temptation);
-        INIT_LIST_HEAD (&wb_inode->wip);
+    INIT_LIST_HEAD(&wb_inode->all);
+    INIT_LIST_HEAD(&wb_inode->todo);
+    INIT_LIST_HEAD(&wb_inode->liability);
+    INIT_LIST_HEAD(&wb_inode->temptation);
+    INIT_LIST_HEAD(&wb_inode->wip);
+    INIT_LIST_HEAD(&wb_inode->invalidate_list);
 
-        wb_inode->this = this;
+    wb_inode->this = this;
 
-        wb_inode->window_conf = conf->window_size;
+    wb_inode->window_conf = conf->window_size;
+    wb_inode->inode = inode;
 
-        LOCK_INIT (&wb_inode->lock);
+    LOCK_INIT(&wb_inode->lock);
+    GF_ATOMIC_INIT(wb_inode->invalidate, 0);
+    GF_ATOMIC_INIT(wb_inode->readdirps, 0);
 
-        __inode_ctx_put (inode, this, (uint64_t)(unsigned long)wb_inode);
+    ret = __inode_ctx_put(inode, this, (uint64_t)(unsigned long)wb_inode);
+    if (ret) {
+        GF_FREE(wb_inode);
+        wb_inode = NULL;
+    }
 
 out:
-        return wb_inode;
+    return wb_inode;
 }
 
-
 wb_inode_t *
-wb_inode_create (xlator_t *this, inode_t *inode)
+wb_inode_create(xlator_t *this, inode_t *inode)
 {
-        wb_inode_t *wb_inode = NULL;
+    wb_inode_t *wb_inode = NULL;
 
-        GF_VALIDATE_OR_GOTO (this->name, inode, out);
+    GF_VALIDATE_OR_GOTO(this->name, inode, out);
 
-        LOCK (&inode->lock);
-        {
-                wb_inode = __wb_inode_ctx_get (this, inode);
-                if (!wb_inode)
-                        wb_inode = __wb_inode_create (this, inode);
-        }
-        UNLOCK (&inode->lock);
+    LOCK(&inode->lock);
+    {
+        wb_inode = __wb_inode_ctx_get(this, inode);
+        if (!wb_inode)
+            wb_inode = __wb_inode_create(this, inode);
+    }
+    UNLOCK(&inode->lock);
 
 out:
-        return wb_inode;
+    return wb_inode;
 }
 
-
 void
-wb_inode_destroy (wb_inode_t *wb_inode)
+wb_inode_destroy(wb_inode_t *wb_inode)
 {
-        GF_VALIDATE_OR_GOTO ("write-behind", wb_inode, out);
+    GF_VALIDATE_OR_GOTO("write-behind", wb_inode, out);
+
+    GF_ASSERT(list_empty(&wb_inode->todo));
+    GF_ASSERT(list_empty(&wb_inode->liability));
+    GF_ASSERT(list_empty(&wb_inode->temptation));
 
-        LOCK_DESTROY (&wb_inode->lock);
-        GF_FREE (wb_inode);
+    LOCK_DESTROY(&wb_inode->lock);
+    GF_FREE(wb_inode);
 out:
-        return;
+    return;
 }
 
+void
+__wb_fulfill_request(wb_request_t *req)
+{
+    wb_inode_t *wb_inode = NULL;
+    char gfid[64] = {
+        0,
+    };
+
+    wb_inode = req->wb_inode;
+
+    req->ordering.fulfilled = 1;
+    wb_inode->window_current -= req->total_size;
+    wb_inode->transit -= req->total_size;
+
+    uuid_utoa_r(req->gfid, gfid);
+
+    gf_log_callingfn(wb_inode->this->name, GF_LOG_DEBUG,
+                     "(unique=%" PRIu64
+                     ", fop=%s, gfid=%s, "
+                     "gen=%" PRIu64
+                     "): request fulfilled. "
+                     "removing the request from liability queue? = %s",
+                     req->unique, gf_fop_list[req->fop], gfid, req->gen,
+                     req->ordering.lied ? "yes" : "no");
+
+    if (req->ordering.lied) {
+        /* 1. If yes, request is in liability queue and hence can be
+              safely removed from list.
+           2. If no, request is in temptation queue and hence should be
+              left in the queue so that wb_pick_unwinds picks it up
+        */
+        list_del_init(&req->lie);
+    } else {
+        /* TODO: fail the req->frame with error if
+           necessary
+        */
+    }
+
+    list_del_init(&req->wip);
+    __wb_request_unref(req);
+}
+
+/* get a flush/fsync waiting on req */
+wb_request_t *
+__wb_request_waiting_on(wb_request_t *req)
+{
+    wb_inode_t *wb_inode = NULL;
+    wb_request_t *trav = NULL;
+
+    wb_inode = req->wb_inode;
+
+    list_for_each_entry(trav, &wb_inode->todo, todo)
+    {
+        if (((trav->stub->fop == GF_FOP_FLUSH) ||
+             (trav->stub->fop == GF_FOP_FSYNC)) &&
+            (trav->gen >= req->gen))
+            return trav;
+    }
+
+    return NULL;
+}
 
 void
-__wb_fulfill_request (wb_request_t *req)
+__wb_add_request_for_retry(wb_request_t *req)
 {
-	wb_inode_t *wb_inode = NULL;
+    wb_inode_t *wb_inode = NULL;
 
-	wb_inode = req->wb_inode;
+    if (!req)
+        goto out;
 
-	req->ordering.fulfilled = 1;
-	wb_inode->window_current -= req->total_size;
-	wb_inode->transit -= req->total_size;
+    wb_inode = req->wb_inode;
 
-	if (!req->ordering.lied) {
-		/* TODO: fail the req->frame with error if
-		   necessary
-		*/
-	}
+    /* response was unwound and no waiter waiting on this request, retry
+       till a flush or fsync (subject to conf->resync_after_fsync).
+    */
+    wb_inode->transit -= req->total_size;
 
-	__wb_request_unref (req);
-}
+    req->total_size = 0;
+
+    list_del_init(&req->winds);
+    list_del_init(&req->todo);
+    list_del_init(&req->wip);
 
+    /* sanitize ordering flags to retry */
+    req->ordering.go = 0;
+
+    /* Add back to todo list to retry */
+    list_add(&req->todo, &wb_inode->todo);
+
+out:
+    return;
+}
 
 void
-wb_head_done (wb_request_t *head)
+__wb_add_head_for_retry(wb_request_t *head)
 {
-	wb_request_t *req = NULL;
-	wb_request_t *tmp     = NULL;
-	wb_inode_t   *wb_inode = NULL;
+    wb_request_t *req = NULL, *tmp = NULL;
 
-	wb_inode = head->wb_inode;
+    if (!head)
+        goto out;
 
-	LOCK (&wb_inode->lock);
-	{
-		list_for_each_entry_safe (req, tmp, &head->winds, winds) {
-			__wb_fulfill_request (req);
-		}
-		__wb_fulfill_request (head);
-	}
-	UNLOCK (&wb_inode->lock);
+    list_for_each_entry_safe_reverse(req, tmp, &head->winds, winds)
+    {
+        __wb_add_request_for_retry(req);
+    }
+
+    __wb_add_request_for_retry(head);
+
+out:
+    return;
 }
 
+void
+wb_add_head_for_retry(wb_request_t *head)
+{
+    if (!head)
+        goto out;
+
+    LOCK(&head->wb_inode->lock);
+    {
+        __wb_add_head_for_retry(head);
+    }
+    UNLOCK(&head->wb_inode->lock);
+
+out:
+    return;
+}
 
 void
-wb_fulfill_err (wb_request_t *head, int op_errno)
+__wb_fulfill_request_err(wb_request_t *req, int32_t op_errno)
 {
-	wb_inode_t *wb_inode;
-	wb_request_t *req;
+    wb_inode_t *wb_inode = NULL;
+    wb_request_t *waiter = NULL;
+    wb_conf_t *conf = NULL;
+
+    wb_inode = req->wb_inode;
+
+    conf = wb_inode->this->private;
+
+    req->op_ret = -1;
+    req->op_errno = op_errno;
 
-	wb_inode = head->wb_inode;
+    if (req->ordering.lied)
+        waiter = __wb_request_waiting_on(req);
+
+    if (!req->ordering.lied || waiter) {
+        if (!req->ordering.lied) {
+            /* response to app is still pending, send failure in
+             * response.
+             */
+        } else {
+            /* response was sent, store the error in a
+             * waiter (either an fsync or flush).
+             */
+            waiter->op_ret = -1;
+            waiter->op_errno = op_errno;
+        }
+
+        if (!req->ordering.lied || (waiter->stub->fop == GF_FOP_FLUSH) ||
+            ((waiter->stub->fop == GF_FOP_FSYNC) &&
+             !conf->resync_after_fsync)) {
+            /* No retry needed, forget the request */
+            __wb_fulfill_request(req);
+            return;
+        }
+    }
 
-	/* for all future requests yet to arrive */
-	fd_ctx_set (head->fd, THIS, op_errno);
+    __wb_add_request_for_retry(req);
 
-	LOCK (&wb_inode->lock);
-	{
-		/* for all requests already arrived */
-		list_for_each_entry (req, &wb_inode->all, all) {
-			if (req->fd != head->fd)
-				continue;
-			req->op_ret = -1;
-			req->op_errno = op_errno;
-		}
-	}
-	UNLOCK (&wb_inode->lock);
+    return;
 }
 
+void
+wb_head_done(wb_request_t *head)
+{
+    wb_request_t *req = NULL;
+    wb_request_t *tmp = NULL;
+    wb_inode_t *wb_inode = NULL;
 
-int
-wb_fulfill_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-		int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
-		struct iatt *postbuf, dict_t *xdata)
+    wb_inode = head->wb_inode;
+
+    LOCK(&wb_inode->lock);
+    {
+        list_for_each_entry_safe(req, tmp, &head->winds, winds)
+        {
+            __wb_fulfill_request(req);
+        }
+
+        __wb_fulfill_request(head);
+    }
+    UNLOCK(&wb_inode->lock);
+}
+
+void
+__wb_fulfill_err(wb_request_t *head, int op_errno)
 {
-        wb_inode_t   *wb_inode   = NULL;
-        wb_request_t *head       = NULL;
+    wb_request_t *req = NULL, *tmp = NULL;
 
-	head = frame->local;
-	frame->local = NULL;
+    if (!head)
+        goto out;
 
-        wb_inode = head->wb_inode;
+    head->wb_inode->dontsync++;
 
-	if (op_ret == -1) {
-		wb_fulfill_err (head, op_errno);
-	} else if (op_ret < head->total_size) {
-		/*
-		 * We've encountered a short write, for whatever reason.
-		 * Set an EIO error for the next fop. This should be
-		 * valid for writev or flush (close).
-		 *
-		 * TODO: Retry the write so we can potentially capture
-		 * a real error condition (i.e., ENOSPC).
-		 */
-		wb_fulfill_err (head, EIO);
-	}
+    list_for_each_entry_safe_reverse(req, tmp, &head->winds, winds)
+    {
+        __wb_fulfill_request_err(req, op_errno);
+    }
 
-	wb_head_done (head);
+    __wb_fulfill_request_err(head, op_errno);
+
+out:
+    return;
+}
 
-        wb_process_queue (wb_inode);
+void
+wb_fulfill_err(wb_request_t *head, int op_errno)
+{
+    wb_inode_t *wb_inode = NULL;
 
-        STACK_DESTROY (frame->root);
+    wb_inode = head->wb_inode;
 
-        return 0;
+    LOCK(&wb_inode->lock);
+    {
+        __wb_fulfill_err(head, op_errno);
+    }
+    UNLOCK(&wb_inode->lock);
 }
 
+void
+__wb_modify_write_request(wb_request_t *req, int synced_size)
+{
+    struct iovec *vector = NULL;
+    int count = 0;
+
+    if (!req || synced_size == 0)
+        goto out;
+
+    req->write_size -= synced_size;
+    req->stub->args.offset += synced_size;
 
-#define WB_IOV_LOAD(vec, cnt, req, head) do {				\
-		memcpy (&vec[cnt], req->stub->args.vector,		\
-			(req->stub->args.count * sizeof(vec[0])));	\
-		cnt += req->stub->args.count;				\
-		head->total_size += req->write_size;			\
-	} while (0)
+    vector = req->stub->args.vector;
+    count = req->stub->args.count;
 
+    req->stub->args.count = iov_skip(vector, count, synced_size);
+
+out:
+    return;
+}
 
 int
-wb_fulfill_head (wb_inode_t *wb_inode, wb_request_t *head)
+__wb_fulfill_short_write(wb_request_t *req, int size, gf_boolean_t *fulfilled)
 {
-	struct iovec  vector[MAX_VECTOR_COUNT];
-	int           count    = 0;
-	wb_request_t *req      = NULL;
-	call_frame_t *frame    = NULL;
-        gf_boolean_t  fderr    = _gf_false;
-        xlator_t     *this     = NULL;
+    int accounted_size = 0;
 
-        this = THIS;
+    if (req == NULL)
+        goto out;
 
-        /* make sure head->total_size is updated before we run into any
-         * errors
-         */
+    if (req->write_size <= size) {
+        accounted_size = req->write_size;
+        __wb_fulfill_request(req);
+        *fulfilled = 1;
+    } else {
+        accounted_size = size;
+        __wb_modify_write_request(req, size);
+        *fulfilled = 0;
+    }
 
-	WB_IOV_LOAD (vector, count, head, head);
+out:
+    return accounted_size;
+}
 
-	list_for_each_entry (req, &head->winds, winds) {
-		WB_IOV_LOAD (vector, count, req, head);
+void
+wb_fulfill_short_write(wb_request_t *head, int size)
+{
+    wb_inode_t *wb_inode = NULL;
+    wb_request_t *req = NULL, *next = NULL;
+    int accounted_size = 0;
+    gf_boolean_t fulfilled = _gf_false;
 
-		if (iobref_merge (head->stub->args.iobref,
-				  req->stub->args.iobref))
-			goto err;
-	}
+    if (!head)
+        goto out;
 
-        if (wb_fd_err (head->fd, this, NULL)) {
-                fderr = _gf_true;
-                goto err;
-        }
+    wb_inode = head->wb_inode;
 
-	frame = create_frame (wb_inode->this, wb_inode->this->ctx->pool);
-	if (!frame)
-		goto err;
+    req = head;
 
-	frame->root->lk_owner = head->lk_owner;
-	frame->local = head;
+    LOCK(&wb_inode->lock);
+    {
+        /* hold a reference to head so that __wb_fulfill_short_write
+         * won't free it. We need head for a cleaner list traversal as
+         * list_for_each_entry_safe doesn't iterate over "head" member.
+         * So, if we pass "next->winds" as head to list_for_each_entry,
+         * "next" is skipped. For a simpler logic we need to traverse
+         * the list in the order. So, we start traversal from
+         * "head->winds" and hence we want head to be alive.
+         */
+        __wb_request_ref(head);
 
-	LOCK (&wb_inode->lock);
-	{
-		wb_inode->transit += head->total_size;
-	}
-	UNLOCK (&wb_inode->lock);
+        next = list_entry(head->winds.next, wb_request_t, winds);
 
-	STACK_WIND (frame, wb_fulfill_cbk, FIRST_CHILD (frame->this),
-		    FIRST_CHILD (frame->this)->fops->writev,
-		    head->fd, vector, count,
-		    head->stub->args.offset,
-		    head->stub->args.flags,
-		    head->stub->args.iobref, NULL);
+        accounted_size = __wb_fulfill_short_write(head, size, &fulfilled);
 
-	return 0;
-err:
-        if (!fderr) {
-                /* frame creation failure */
-                fderr = ENOMEM;
-                wb_fulfill_err (head, fderr);
-        }
+        size -= accounted_size;
 
-	wb_head_done (head);
+        if (size == 0) {
+            if (fulfilled && (next != head))
+                req = next;
 
-	return fderr;
-}
+            goto done;
+        }
 
+        list_for_each_entry_safe(req, next, &head->winds, winds)
+        {
+            accounted_size = __wb_fulfill_short_write(req, size, &fulfilled);
+            size -= accounted_size;
+
+            if (size == 0) {
+                if (fulfilled && (next != head))
+                    req = next;
+                break;
+            }
+        }
+    done:
+        __wb_request_unref(head);
+    }
+    UNLOCK(&wb_inode->lock);
 
-#define NEXT_HEAD(head, req) do {					\
-		if (head)						\
-			ret |= wb_fulfill_head (wb_inode, head);	\
-		head = req;						\
-		expected_offset = req->stub->args.offset +		\
-			req->write_size;				\
-		curr_aggregate = 0;					\
-		vector_count = 0;					\
-	} while (0)
+    wb_add_head_for_retry(req);
+out:
+    return;
+}
 
+int
+wb_fulfill_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+               int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
+               struct iatt *postbuf, dict_t *xdata)
+{
+    wb_inode_t *wb_inode = NULL;
+    wb_request_t *head = NULL;
+
+    head = frame->local;
+    frame->local = NULL;
+
+    wb_inode = head->wb_inode;
+
+    /* There could be a readdirp session in progress. Since wb_fulfill_cbk
+     * can potentially remove a request from liability queue,
+     * wb_readdirp_cbk will miss writes on this inode (as it invalidates
+     * stats only if liability queue is not empty) and hence mark inode
+     * for invalidation of stats in readdirp response. Specifically this
+     * code fixes the following race mentioned in wb_readdirp_cbk:
+     */
+
+    /* <removed comment from wb_readdirp_cbk>
+     * We cannot guarantee integrity of entry->d_stat as there are cached
+     * writes. The stat is most likely stale as it doesn't account the
+     * cached writes. However, checking for non-empty liability list here is
+     * not a fool-proof solution as there can be races like,
+     * 1. readdirp is successful on posix
+     * 2. sync of cached write is successful on posix
+     * 3. write-behind received sync response and removed the request from
+     *    liability queue
+     * 4. readdirp response is processed at write-behind
+     *
+     * In the above scenario, stat for the file is sent back in readdirp
+     * response but it is stale.
+     * </comment> */
+    wb_set_invalidate(wb_inode);
+
+    if (op_ret == -1) {
+        wb_fulfill_err(head, op_errno);
+    } else if (op_ret < head->total_size) {
+        wb_fulfill_short_write(head, op_ret);
+    } else {
+        wb_head_done(head);
+    }
+
+    wb_process_queue(wb_inode);
+
+    STACK_DESTROY(frame->root);
+
+    return 0;
+}
+
+#define WB_IOV_LOAD(vec, cnt, req, head)                                       \
+    do {                                                                       \
+        memcpy(&vec[cnt], req->stub->args.vector,                              \
+               (req->stub->args.count * sizeof(vec[0])));                      \
+        cnt += req->stub->args.count;                                          \
+        head->total_size += req->write_size;                                   \
+    } while (0)
 
 int
-wb_fulfill (wb_inode_t *wb_inode, list_head_t *liabilities)
+wb_fulfill_head(wb_inode_t *wb_inode, wb_request_t *head)
 {
-	wb_request_t  *req     = NULL;
-	wb_request_t  *head    = NULL;
-	wb_request_t  *tmp     = NULL;
-	wb_conf_t     *conf    = NULL;
-	off_t          expected_offset = 0;
-	size_t         curr_aggregate = 0;
-	size_t         vector_count = 0;
-        int            ret          = 0;
+    struct iovec vector[MAX_VECTOR_COUNT];
+    int count = 0;
+    wb_request_t *req = NULL;
+    call_frame_t *frame = NULL;
 
-	conf = wb_inode->this->private;
+    /* make sure head->total_size is updated before we run into any
+     * errors
+     */
 
-	list_for_each_entry_safe (req, tmp, liabilities, winds) {
-		list_del_init (&req->winds);
+    WB_IOV_LOAD(vector, count, head, head);
 
-		if (!head) {
-			NEXT_HEAD (head, req);
-			continue;
-		}
+    list_for_each_entry(req, &head->winds, winds)
+    {
+        WB_IOV_LOAD(vector, count, req, head);
 
-		if (req->fd != head->fd) {
-			NEXT_HEAD (head, req);
-			continue;
-		}
+        if (iobref_merge(head->stub->args.iobref, req->stub->args.iobref))
+            goto err;
+    }
 
-		if (!is_same_lkowner (&req->lk_owner, &head->lk_owner)) {
-			NEXT_HEAD (head, req);
-			continue;
-		}
+    frame = create_frame(wb_inode->this, wb_inode->this->ctx->pool);
+    if (!frame)
+        goto err;
 
-		if (expected_offset != req->stub->args.offset) {
-			NEXT_HEAD (head, req);
-			continue;
-		}
+    frame->root->lk_owner = head->lk_owner;
+    frame->root->pid = head->client_pid;
+    frame->local = head;
 
-		if ((curr_aggregate + req->write_size) > conf->aggregate_size) {
-			NEXT_HEAD (head, req);
-			continue;
-		}
+    LOCK(&wb_inode->lock);
+    {
+        wb_inode->transit += head->total_size;
+    }
+    UNLOCK(&wb_inode->lock);
 
-		if (vector_count + req->stub->args.count >
-		    MAX_VECTOR_COUNT) {
-			NEXT_HEAD (head, req);
-			continue;
-		}
+    STACK_WIND(frame, wb_fulfill_cbk, FIRST_CHILD(frame->this),
+               FIRST_CHILD(frame->this)->fops->writev, head->fd, vector, count,
+               head->stub->args.offset, head->stub->args.flags,
+               head->stub->args.iobref, NULL);
 
-		list_add_tail (&req->winds, &head->winds);
-		curr_aggregate += req->write_size;
-		vector_count += req->stub->args.count;
-	}
-
-	if (head)
-		ret |= wb_fulfill_head (wb_inode, head);
+    return 0;
+err:
+    /* frame creation failure */
+    wb_fulfill_err(head, ENOMEM);
 
-	return ret;
+    return ENOMEM;
 }
 
+#define NEXT_HEAD(head, req)                                                   \
+    do {                                                                       \
+        if (head)                                                              \
+            ret |= wb_fulfill_head(wb_inode, head);                            \
+        head = req;                                                            \
+        expected_offset = req->stub->args.offset + req->write_size;            \
+        curr_aggregate = 0;                                                    \
+        vector_count = 0;                                                      \
+    } while (0)
 
-void
-wb_do_unwinds (wb_inode_t *wb_inode, list_head_t *lies)
-{
-	wb_request_t *req = NULL;
-	wb_request_t *tmp = NULL;
-	call_frame_t *frame = NULL;
-	struct iatt   buf = {0, };
+int
+wb_fulfill(wb_inode_t *wb_inode, list_head_t *liabilities)
+{
+    wb_request_t *req = NULL;
+    wb_request_t *head = NULL;
+    wb_request_t *tmp = NULL;
+    wb_conf_t *conf = NULL;
+    off_t expected_offset = 0;
+    size_t curr_aggregate = 0;
+    size_t vector_count = 0;
+    int ret = 0;
+
+    conf = wb_inode->this->private;
+
+    list_for_each_entry_safe(req, tmp, liabilities, winds)
+    {
+        list_del_init(&req->winds);
+
+        if (!head) {
+            NEXT_HEAD(head, req);
+            continue;
+        }
+
+        if (req->fd != head->fd) {
+            NEXT_HEAD(head, req);
+            continue;
+        }
 
-        list_for_each_entry_safe (req, tmp, lies, unwinds) {
-                frame = req->stub->frame;
+        if (!is_same_lkowner(&req->lk_owner, &head->lk_owner)) {
+            NEXT_HEAD(head, req);
+            continue;
+        }
 
-                STACK_UNWIND_STRICT (writev, frame, req->op_ret, req->op_errno,
-				     &buf, &buf, NULL); /* :O */
-		req->stub->frame = NULL;
+        if (expected_offset != req->stub->args.offset) {
+            NEXT_HEAD(head, req);
+            continue;
+        }
 
-		list_del_init (&req->unwinds);
-                wb_request_unref (req);
+        if ((curr_aggregate + req->write_size) > conf->aggregate_size) {
+            NEXT_HEAD(head, req);
+            continue;
         }
 
-        return;
-}
+        if (vector_count + req->stub->args.count > MAX_VECTOR_COUNT) {
+            NEXT_HEAD(head, req);
+            continue;
+        }
+
+        list_add_tail(&req->winds, &head->winds);
+        curr_aggregate += req->write_size;
+        vector_count += req->stub->args.count;
+    }
 
+    if (head)
+        ret |= wb_fulfill_head(wb_inode, head);
+
+    return ret;
+}
 
 void
-__wb_pick_unwinds (wb_inode_t *wb_inode, list_head_t *lies)
+wb_do_unwinds(wb_inode_t *wb_inode, list_head_t *lies)
 {
-        wb_request_t *req = NULL;
-        wb_request_t *tmp = NULL;
+    wb_request_t *req = NULL;
+    wb_request_t *tmp = NULL;
+    call_frame_t *frame = NULL;
+    struct iatt buf = {
+        0,
+    };
 
-	list_for_each_entry_safe (req, tmp, &wb_inode->temptation, lie) {
-		if (!req->ordering.fulfilled &&
-		    wb_inode->window_current > wb_inode->window_conf)
-			continue;
+    list_for_each_entry_safe(req, tmp, lies, unwinds)
+    {
+        frame = req->stub->frame;
 
-		list_del_init (&req->lie);
-		list_move_tail (&req->unwinds, lies);
+        STACK_UNWIND_STRICT(writev, frame, req->op_ret, req->op_errno, &buf,
+                            &buf, NULL); /* :O */
+        req->stub->frame = NULL;
 
-		wb_inode->window_current += req->orig_size;
+        list_del_init(&req->unwinds);
+        wb_request_unref(req);
+    }
 
-		if (!req->ordering.fulfilled) {
-			/* burden increased */
-			list_add_tail (&req->lie, &wb_inode->liability);
-
-			req->ordering.lied = 1;
+    return;
+}
 
-			wb_inode->gen++;
-		}
-	}
+void
+__wb_pick_unwinds(wb_inode_t *wb_inode, list_head_t *lies)
+{
+    wb_request_t *req = NULL;
+    wb_request_t *tmp = NULL;
+    char gfid[64] = {
+        0,
+    };
+
+    list_for_each_entry_safe(req, tmp, &wb_inode->temptation, lie)
+    {
+        if (!req->ordering.fulfilled &&
+            wb_inode->window_current > wb_inode->window_conf)
+            continue;
+
+        list_del_init(&req->lie);
+        list_move_tail(&req->unwinds, lies);
+
+        wb_inode->window_current += req->orig_size;
+
+        wb_inode->gen++;
+
+        if (!req->ordering.fulfilled) {
+            /* burden increased */
+            list_add_tail(&req->lie, &wb_inode->liability);
+
+            req->ordering.lied = 1;
+
+            uuid_utoa_r(req->gfid, gfid);
+            gf_msg_debug(wb_inode->this->name, 0,
+                         "(unique=%" PRIu64
+                         ", fop=%s, gfid=%s, "
+                         "gen=%" PRIu64
+                         "): added req to liability "
+                         "queue. inode-generation-number=%" PRIu64,
+                         req->stub->frame->root->unique, gf_fop_list[req->fop],
+                         gfid, req->gen, wb_inode->gen);
+        }
+    }
 
-        return;
+    return;
 }
 
-
 int
-__wb_collapse_small_writes (wb_request_t *holder, wb_request_t *req)
-{
-        char          *ptr    = NULL;
-        struct iobuf  *iobuf  = NULL;
-        struct iobref *iobref = NULL;
-        int            ret    = -1;
-        ssize_t        required_size = 0;
-        size_t         holder_len = 0;
-        size_t         req_len = 0;
-
-        if (!holder->iobref) {
-                holder_len = iov_length (holder->stub->args.vector,
-                                         holder->stub->args.count);
-                req_len = iov_length (req->stub->args.vector,
-                                      req->stub->args.count);
-
-                required_size = max ((THIS->ctx->page_size),
-                                     (holder_len + req_len));
-                iobuf = iobuf_get2 (req->wb_inode->this->ctx->iobuf_pool,
-                                    required_size);
-                if (iobuf == NULL) {
-                        goto out;
-                }
+__wb_collapse_small_writes(wb_conf_t *conf, wb_request_t *holder,
+                           wb_request_t *req)
+{
+    char *ptr = NULL;
+    struct iobuf *iobuf = NULL;
+    struct iobref *iobref = NULL;
+    int ret = -1;
+    ssize_t required_size = 0;
+    size_t holder_len = 0;
+    size_t req_len = 0;
+
+    if (!holder->iobref) {
+        holder_len = iov_length(holder->stub->args.vector,
+                                holder->stub->args.count);
+        req_len = iov_length(req->stub->args.vector, req->stub->args.count);
+
+        required_size = max((conf->page_size), (holder_len + req_len));
+        iobuf = iobuf_get2(req->wb_inode->this->ctx->iobuf_pool, required_size);
+        if (iobuf == NULL) {
+            goto out;
+        }
 
-                iobref = iobref_new ();
-                if (iobref == NULL) {
-                        iobuf_unref (iobuf);
-                        goto out;
-                }
+        iobref = iobref_new();
+        if (iobref == NULL) {
+            iobuf_unref(iobuf);
+            goto out;
+        }
 
-                ret = iobref_add (iobref, iobuf);
-                if (ret != 0) {
-                        gf_log (req->wb_inode->this->name, GF_LOG_WARNING,
-                                "cannot add iobuf (%p) into iobref (%p)",
-                                iobuf, iobref);
-                        iobuf_unref (iobuf);
-                        iobref_unref (iobref);
-                        goto out;
-                }
+        ret = iobref_add(iobref, iobuf);
+        if (ret != 0) {
+            gf_msg(req->wb_inode->this->name, GF_LOG_WARNING, -ret,
+                   WRITE_BEHIND_MSG_INVALID_ARGUMENT,
+                   "cannot add iobuf (%p) into iobref (%p)", iobuf, iobref);
+            iobuf_unref(iobuf);
+            iobref_unref(iobref);
+            goto out;
+        }
 
-                iov_unload (iobuf->ptr, holder->stub->args.vector,
-                            holder->stub->args.count);
-                holder->stub->args.vector[0].iov_base = iobuf->ptr;
-		holder->stub->args.count = 1;
+        iov_unload(iobuf->ptr, holder->stub->args.vector,
+                   holder->stub->args.count);
+        holder->stub->args.vector[0].iov_base = iobuf->ptr;
+        holder->stub->args.count = 1;
 
-                iobref_unref (holder->stub->args.iobref);
-                holder->stub->args.iobref = iobref;
+        iobref_unref(holder->stub->args.iobref);
+        holder->stub->args.iobref = iobref;
 
-                iobuf_unref (iobuf);
+        iobuf_unref(iobuf);
 
-                holder->iobref = iobref_ref (iobref);
-        }
+        holder->iobref = iobref_ref(iobref);
+    }
 
-        ptr = holder->stub->args.vector[0].iov_base + holder->write_size;
+    ptr = holder->stub->args.vector[0].iov_base + holder->write_size;
 
-        iov_unload (ptr, req->stub->args.vector,
-                    req->stub->args.count);
+    iov_unload(ptr, req->stub->args.vector, req->stub->args.count);
 
-        holder->stub->args.vector[0].iov_len += req->write_size;
-        holder->write_size += req->write_size;
-        holder->ordering.size += req->write_size;
+    holder->stub->args.vector[0].iov_len += req->write_size;
+    holder->write_size += req->write_size;
+    holder->ordering.size += req->write_size;
 
-        ret = 0;
+    ret = 0;
 out:
-        return ret;
+    return ret;
 }
 
-
 void
-__wb_preprocess_winds (wb_inode_t *wb_inode)
-{
-        off_t         offset_expected = 0;
-        ssize_t       space_left      = 0;
-	wb_request_t *req             = NULL;
-	wb_request_t *tmp             = NULL;
-	wb_request_t *holder          = NULL;
-	wb_conf_t    *conf            = NULL;
-        int           ret             = 0;
-	ssize_t       page_size       = 0;
-
-	/* With asynchronous IO from a VM guest (as a file), there
-	   can be two sequential writes happening in two regions
-	   of the file. But individual (broken down) IO requests
-	   can arrive interleaved.
-
-	   TODO: cycle for each such sequence sifting
-	   through the interleaved ops
-	*/
-
-	page_size = wb_inode->this->ctx->page_size;
-	conf = wb_inode->this->private;
-
-        list_for_each_entry_safe (req, tmp, &wb_inode->todo, todo) {
-		if (!req->ordering.tempted) {
-			if (holder) {
-				if (wb_requests_conflict (holder, req))
-					/* do not hold on write if a
-					   dependent write is in queue */
-					holder->ordering.go = 1;
-			}
-			/* collapse only non-sync writes */
-			continue;
-		} else if (!holder) {
-			/* holder is always a non-sync write */
-			holder = req;
-			continue;
-		}
-
-		offset_expected = holder->stub->args.offset
-			+ holder->write_size;
-
-		if (req->stub->args.offset != offset_expected) {
-			holder->ordering.go = 1;
-			holder = req;
-			continue;
-		}
-
-		if (!is_same_lkowner (&req->lk_owner, &holder->lk_owner)) {
-			holder->ordering.go = 1;
-			holder = req;
-			continue;
-		}
-
-                if (req->fd != holder->fd) {
-                        holder->ordering.go = 1;
-                        holder = req;
-                        continue;
-                }
+__wb_preprocess_winds(wb_inode_t *wb_inode)
+{
+    off_t offset_expected = 0;
+    ssize_t space_left = 0;
+    wb_request_t *req = NULL;
+    wb_request_t *tmp = NULL;
+    wb_request_t *holder = NULL;
+    wb_conf_t *conf = NULL;
+    int ret = 0;
+    ssize_t page_size = 0;
+    char gfid[64] = {
+        0,
+    };
+
+    /* With asynchronous IO from a VM guest (as a file), there
+       can be two sequential writes happening in two regions
+       of the file. But individual (broken down) IO requests
+       can arrive interleaved.
+
+       TODO: cycle for each such sequence sifting
+       through the interleaved ops
+    */
+
+    conf = wb_inode->this->private;
+    page_size = conf->page_size;
+
+    list_for_each_entry_safe(req, tmp, &wb_inode->todo, todo)
+    {
+        if (wb_inode->dontsync && req->ordering.lied) {
+            /* sync has failed. Don't pick lies _again_ for winding
+             * as winding these lies again will trigger an infinite
+             * recursion of wb_process_queue being called from a
+             * failed fulfill. However, pick non-lied requests for
+             * winding so that application won't block indefinitely
+             * waiting for write result.
+             */
+
+            uuid_utoa_r(req->gfid, gfid);
+            gf_msg_debug(wb_inode->this->name, 0,
+                         "(unique=%" PRIu64
+                         ", fop=%s, gfid=%s, "
+                         "gen=%" PRIu64
+                         "): not setting ordering.go"
+                         "as dontsync is set",
+                         req->unique, gf_fop_list[req->fop], gfid, req->gen);
+
+            continue;
+        }
 
-		space_left = page_size - holder->write_size;
+        if (!req->ordering.tempted) {
+            if (holder) {
+                if (wb_requests_conflict(holder, req))
+                    /* do not hold on write if a
+                       dependent write is in queue */
+                    holder->ordering.go = 1;
+            }
+            /* collapse only non-sync writes */
+            continue;
+        } else if (!holder) {
+            /* holder is always a non-sync write */
+            holder = req;
+            continue;
+        }
 
-		if (space_left < req->write_size) {
-			holder->ordering.go = 1;
-			holder = req;
-			continue;
-		}
+        offset_expected = holder->stub->args.offset + holder->write_size;
 
-		ret = __wb_collapse_small_writes (holder, req);
-		if (ret)
-			continue;
+        if (req->stub->args.offset != offset_expected) {
+            holder->ordering.go = 1;
+            holder = req;
+            continue;
+        }
 
-		/* collapsed request is as good as wound
-		   (from its p.o.v)
-		*/
-		list_del_init (&req->todo);
-		__wb_fulfill_request (req);
+        if (!is_same_lkowner(&req->lk_owner, &holder->lk_owner)) {
+            holder->ordering.go = 1;
+            holder = req;
+            continue;
+        }
 
-               /* Only the last @holder in queue which
+        if (req->fd != holder->fd) {
+            holder->ordering.go = 1;
+            holder = req;
+            continue;
+        }
 
-                  - does not have any non-buffered-writes following it
-                  - has not yet filled its capacity
+        space_left = page_size - holder->write_size;
 
-                  does not get its 'go' set, in anticipation of the arrival
-                  of consecutive smaller writes.
-               */
+        if (space_left < req->write_size) {
+            holder->ordering.go = 1;
+            holder = req;
+            continue;
         }
 
-	/* but if trickling writes are enabled, then do not hold back
-	   writes if there are no outstanding requests
-	*/
+        ret = __wb_collapse_small_writes(conf, holder, req);
+        if (ret)
+            continue;
 
-	if (conf->trickling_writes && !wb_inode->transit && holder)
-		holder->ordering.go = 1;
+        /* collapsed request is as good as wound
+           (from its p.o.v)
+        */
+        list_del_init(&req->todo);
+        __wb_fulfill_request(req);
 
-        return;
+        /* Only the last @holder in queue which
+
+           - does not have any non-buffered-writes following it
+           - has not yet filled its capacity
+
+           does not get its 'go' set, in anticipation of the arrival
+           of consecutive smaller writes.
+        */
+    }
+
+    /* but if trickling writes are enabled, then do not hold back
+       writes if there are no outstanding requests
+    */
+
+    if (conf->trickling_writes && !wb_inode->transit && holder)
+        holder->ordering.go = 1;
+
+    if (wb_inode->dontsync > 0)
+        wb_inode->dontsync--;
+
+    return;
 }
 
+int
+__wb_handle_failed_conflict(wb_request_t *req, wb_request_t *conflict,
+                            list_head_t *tasks)
+{
+    wb_conf_t *conf = NULL;
+    char gfid[64] = {
+        0,
+    };
+
+    conf = req->wb_inode->this->private;
+
+    uuid_utoa_r(req->gfid, gfid);
+
+    if ((req->stub->fop != GF_FOP_FLUSH) &&
+        ((req->stub->fop != GF_FOP_FSYNC) || conf->resync_after_fsync)) {
+        if (!req->ordering.lied && list_empty(&conflict->wip)) {
+            /* If request itself is in liability queue,
+             * 1. We cannot unwind as the response has already been
+             *    sent.
+             * 2. We cannot wind till conflict clears up.
+             * 3. So, skip the request for now.
+             * 4. Otherwise, resume (unwind) it with error.
+             */
+            req->op_ret = -1;
+            req->op_errno = conflict->op_errno;
+            if ((req->stub->fop == GF_FOP_TRUNCATE) ||
+                (req->stub->fop == GF_FOP_FTRUNCATE)) {
+                req->stub->frame->local = NULL;
+            }
+
+            list_del_init(&req->todo);
+            list_add_tail(&req->winds, tasks);
+
+            gf_msg_debug(req->wb_inode->this->name, 0,
+                         "(unique=%" PRIu64
+                         ", fop=%s, gfid=%s, "
+                         "gen=%" PRIu64
+                         "): A conflicting write "
+                         "request in liability queue has failed "
+                         "to sync (error = \"%s\"), "
+                         "unwinding this request as a failure",
+                         req->unique, gf_fop_list[req->fop], gfid, req->gen,
+                         strerror(req->op_errno));
+
+            if (req->ordering.tempted) {
+                /* make sure that it won't be unwound in
+                 * wb_do_unwinds too. Otherwise there'll be
+                 * a double wind.
+                 */
+                list_del_init(&req->lie);
+
+                gf_msg_debug(req->wb_inode->this->name, 0,
+                             "(unique=%" PRIu64
+                             ", fop=%s, "
+                             "gfid=%s, gen=%" PRIu64
+                             "): "
+                             "removed from liability queue",
+                             req->unique, gf_fop_list[req->fop], gfid,
+                             req->gen);
+
+                __wb_fulfill_request(req);
+            }
+        }
+    } else {
+        gf_msg_debug(req->wb_inode->this->name, 0,
+                     "(unique=%" PRIu64
+                     ", fop=%s, gfid=%s, "
+                     "gen=%" PRIu64
+                     "): A conflicting write request "
+                     "in liability queue has failed to sync "
+                     "(error = \"%s\"). This is an "
+                     "FSYNC/FLUSH and we need to maintain ordering "
+                     "guarantees with other writes in TODO queue. "
+                     "Hence doing nothing now",
+                     req->unique, gf_fop_list[req->fop], gfid, req->gen,
+                     strerror(conflict->op_errno));
+
+        /* flush and fsync (without conf->resync_after_fsync) act as
+           barriers. We cannot unwind them out of
+           order, when there are earlier generation writes just because
+           there is a conflicting liability with an error. So, wait for
+           our turn till there are no conflicting liabilities.
+
+           This situation can arise when there liabilities spread across
+           multiple generations. For eg., consider two writes with
+           following characterstics:
+
+           1. they belong to different generations gen1, gen2 and
+              (gen1 > gen2).
+           2. they overlap.
+           3. both are liabilities.
+           4. gen1 write was attempted to sync, but the attempt failed.
+           5. there was no attempt to sync gen2 write yet.
+           6. A flush (as part of close) is issued and gets a gen no
+              gen3.
+
+           In the above scenario, if flush is unwound without waiting
+           for gen1 and gen2 writes either to be successfully synced or
+           purged, we end up with these two writes in wb_inode->todo
+           list forever as there will be no attempt to process the queue
+           as flush is the last operation.
+        */
+    }
+
+    return 0;
+}
 
-void
-__wb_pick_winds (wb_inode_t *wb_inode, list_head_t *tasks,
-		 list_head_t *liabilities)
+int
+__wb_pick_winds(wb_inode_t *wb_inode, list_head_t *tasks,
+                list_head_t *liabilities)
 {
-	wb_request_t *req = NULL;
-	wb_request_t *tmp = NULL;
+    wb_request_t *req = NULL;
+    wb_request_t *tmp = NULL;
+    wb_request_t *conflict = NULL;
+    char req_gfid[64] =
+        {
+            0,
+        },
+         conflict_gfid[64] = {
+             0,
+         };
+
+    list_for_each_entry_safe(req, tmp, &wb_inode->todo, todo)
+    {
+        uuid_utoa_r(req->gfid, req_gfid);
+
+        conflict = wb_liability_has_conflict(wb_inode, req);
+        if (conflict) {
+            uuid_utoa_r(conflict->gfid, conflict_gfid);
+
+            gf_msg_debug(wb_inode->this->name, 0,
+                         "Not winding request due to a "
+                         "conflicting write in liability queue. "
+                         "REQ: unique=%" PRIu64
+                         ", fop=%s, "
+                         "gen=%" PRIu64
+                         ", gfid=%s. "
+                         "CONFLICT: unique=%" PRIu64
+                         ", fop=%s, "
+                         "gen=%" PRIu64
+                         ", gfid=%s, "
+                         "conflicts-sync-failed?=%s, "
+                         "conflicts-error=%s",
+                         req->unique, gf_fop_list[req->fop], req->gen, req_gfid,
+                         conflict->unique, gf_fop_list[conflict->fop],
+                         conflict->gen, conflict_gfid,
+                         (conflict->op_ret == 1) ? "yes" : "no",
+                         strerror(conflict->op_errno));
+
+            if (conflict->op_ret == -1) {
+                /* There is a conflicting liability which failed
+                 * to sync in previous attempts, resume the req
+                 * and fail, unless its an fsync/flush.
+                 */
 
-	list_for_each_entry_safe (req, tmp, &wb_inode->todo, todo) {
-		if (wb_liability_has_conflict (wb_inode, req))
-			continue;
+                __wb_handle_failed_conflict(req, conflict, tasks);
+            } else {
+                /* There is a conflicting liability which was
+                 * not attempted to sync even once. Wait till
+                 * at least one attempt to sync is made.
+                 */
+            }
 
-		if (req->ordering.tempted && !req->ordering.go)
-			/* wait some more */
-			continue;
+            continue;
+        }
 
-		if (req->stub->fop == GF_FOP_WRITE) {
-			if (wb_wip_has_conflict (wb_inode, req))
-				continue;
+        if (req->ordering.tempted && !req->ordering.go) {
+            /* wait some more */
+            gf_msg_debug(wb_inode->this->name, 0,
+                         "(unique=%" PRIu64 ", fop=%s, gen=%" PRIu64
+                         ", gfid=%s): ordering.go is not set, "
+                         "hence not winding",
+                         req->unique, gf_fop_list[req->fop], req->gen,
+                         req_gfid);
+            continue;
+        }
 
-			list_add_tail (&req->wip, &wb_inode->wip);
+        if (req->stub->fop == GF_FOP_WRITE) {
+            conflict = wb_wip_has_conflict(wb_inode, req);
+
+            if (conflict) {
+                uuid_utoa_r(conflict->gfid, conflict_gfid);
+
+                gf_msg_debug(wb_inode->this->name, 0,
+                             "Not winding write request as "
+                             "a conflicting write is being "
+                             "synced to backend. "
+                             "REQ: unique=%" PRIu64
+                             " fop=%s,"
+                             " gen=%" PRIu64
+                             ", gfid=%s. "
+                             "CONFLICT: unique=%" PRIu64
+                             " "
+                             "fop=%s, gen=%" PRIu64
+                             ", "
+                             "gfid=%s",
+                             req->unique, gf_fop_list[req->fop], req->gen,
+                             req_gfid, conflict->unique,
+                             gf_fop_list[conflict->fop], conflict->gen,
+                             conflict_gfid);
+                continue;
+            }
+
+            list_add_tail(&req->wip, &wb_inode->wip);
+            req->wind_count++;
+
+            if (!req->ordering.tempted)
+                /* unrefed in wb_writev_cbk */
+                req->stub->frame->local = __wb_request_ref(req);
+        }
 
-			if (!req->ordering.tempted)
-				/* unrefed in wb_writev_cbk */
-				req->stub->frame->local =
-					__wb_request_ref (req);
-		}
+        gf_msg_debug(wb_inode->this->name, 0,
+                     "(unique=%" PRIu64
+                     ", fop=%s, gfid=%s, "
+                     "gen=%" PRIu64
+                     "): picking the request for "
+                     "winding",
+                     req->unique, gf_fop_list[req->fop], req_gfid, req->gen);
 
-		list_del_init (&req->todo);
+        list_del_init(&req->todo);
 
-		if (req->ordering.tempted)
-			list_add_tail (&req->winds, liabilities);
-		else
-			list_add_tail (&req->winds, tasks);
-	}
-}
+        if (req->ordering.tempted) {
+            list_add_tail(&req->winds, liabilities);
+        } else {
+            list_add_tail(&req->winds, tasks);
+        }
+    }
 
+    return 0;
+}
 
 void
-wb_do_winds (wb_inode_t *wb_inode, list_head_t *tasks)
+wb_do_winds(wb_inode_t *wb_inode, list_head_t *tasks)
 {
-	wb_request_t *req = NULL;
-	wb_request_t *tmp = NULL;
+    wb_request_t *req = NULL;
+    wb_request_t *tmp = NULL;
 
-	list_for_each_entry_safe (req, tmp, tasks, winds) {
-		list_del_init (&req->winds);
+    list_for_each_entry_safe(req, tmp, tasks, winds)
+    {
+        list_del_init(&req->winds);
 
-		call_resume (req->stub);
+        if (req->op_ret == -1) {
+            call_unwind_error_keep_stub(req->stub, req->op_ret, req->op_errno);
+        } else {
+            call_resume_keep_stub(req->stub);
+        }
 
-		wb_request_unref (req);
-	}
+        wb_request_unref(req);
+    }
 }
 
-
 void
-wb_process_queue (wb_inode_t *wb_inode)
+wb_process_queue(wb_inode_t *wb_inode)
 {
-        list_head_t tasks  = {0, };
-	list_head_t lies = {0, };
-	list_head_t liabilities = {0, };
-        int         retry       = 0;
+    list_head_t tasks;
+    list_head_t lies;
+    list_head_t liabilities;
+    int wind_failure = 0;
 
-        INIT_LIST_HEAD (&tasks);
-        INIT_LIST_HEAD (&lies);
-        INIT_LIST_HEAD (&liabilities);
+    INIT_LIST_HEAD(&tasks);
+    INIT_LIST_HEAD(&lies);
+    INIT_LIST_HEAD(&liabilities);
 
-        do {
-                LOCK (&wb_inode->lock);
-                {
-                        __wb_preprocess_winds (wb_inode);
+    do {
+        gf_log_callingfn(wb_inode->this->name, GF_LOG_DEBUG,
+                         "processing queues");
 
-                        __wb_pick_winds (wb_inode, &tasks, &liabilities);
+        LOCK(&wb_inode->lock);
+        {
+            __wb_preprocess_winds(wb_inode);
 
-                        __wb_pick_unwinds (wb_inode, &lies);
+            __wb_pick_winds(wb_inode, &tasks, &liabilities);
 
-                }
-                UNLOCK (&wb_inode->lock);
+            __wb_pick_unwinds(wb_inode, &lies);
+        }
+        UNLOCK(&wb_inode->lock);
 
-                wb_do_unwinds (wb_inode, &lies);
+        if (!list_empty(&lies))
+            wb_do_unwinds(wb_inode, &lies);
 
-                wb_do_winds (wb_inode, &tasks);
+        if (!list_empty(&tasks))
+            wb_do_winds(wb_inode, &tasks);
 
-                /* fd might've been marked bad due to previous errors.
-                 * Since, caller of wb_process_queue might be the last fop on
-                 * inode, make sure we keep processing request queue, till there
-                 * are no requests left.
-                 */
-                retry = wb_fulfill (wb_inode, &liabilities);
-        } while (retry);
+        /* If there is an error in wb_fulfill before winding write
+         * requests, we would miss invocation of wb_process_queue
+         * from wb_fulfill_cbk. So, retry processing again.
+         */
+        if (!list_empty(&liabilities))
+            wind_failure = wb_fulfill(wb_inode, &liabilities);
+    } while (wind_failure);
 
-        return;
+    return;
 }
 
-
 void
 wb_set_inode_size(wb_inode_t *wb_inode, struct iatt *postbuf)
 {
-	GF_ASSERT (wb_inode);
-	GF_ASSERT (postbuf);
+    GF_ASSERT(wb_inode);
+    GF_ASSERT(postbuf);
 
-	LOCK (&wb_inode->lock);
-	{
-		wb_inode->size = postbuf->ia_size;
-	}
-	UNLOCK (&wb_inode->lock);
+    LOCK(&wb_inode->lock);
+    {
+        wb_inode->size = postbuf->ia_size;
+    }
+    UNLOCK(&wb_inode->lock);
 }
 
-
 int
-wb_writev_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-	       int32_t op_ret, int32_t op_errno,
-	       struct iatt *prebuf, struct iatt *postbuf, dict_t *xdata)
+wb_writev_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int32_t op_ret,
+              int32_t op_errno, struct iatt *prebuf, struct iatt *postbuf,
+              dict_t *xdata)
 {
-	wb_request_t *req = NULL;
-	wb_inode_t *wb_inode;
+    wb_request_t *req = NULL;
+    wb_inode_t *wb_inode;
 
-	req = frame->local;
-	frame->local = NULL;
-	wb_inode = req->wb_inode;
+    req = frame->local;
+    frame->local = NULL;
+    wb_inode = req->wb_inode;
 
-	wb_request_unref (req);
+    LOCK(&req->wb_inode->lock);
+    {
+        list_del_init(&req->wip);
+    }
+    UNLOCK(&req->wb_inode->lock);
 
-	/* requests could be pending while this was in progress */
-	wb_process_queue(wb_inode);
+    wb_request_unref(req);
 
-	STACK_UNWIND_STRICT (writev, frame, op_ret, op_errno, prebuf, postbuf,
-			     xdata);
-	return 0;
-}
+    /* requests could be pending while this was in progress */
+    wb_process_queue(wb_inode);
 
+    STACK_UNWIND_STRICT(writev, frame, op_ret, op_errno, prebuf, postbuf,
+                        xdata);
+    return 0;
+}
 
 int
-wb_writev_helper (call_frame_t *frame, xlator_t *this, fd_t *fd,
-		  struct iovec *vector, int32_t count, off_t offset,
-		  uint32_t flags, struct iobref *iobref, dict_t *xdata)
+wb_writev_helper(call_frame_t *frame, xlator_t *this, fd_t *fd,
+                 struct iovec *vector, int32_t count, off_t offset,
+                 uint32_t flags, struct iobref *iobref, dict_t *xdata)
 {
-	STACK_WIND (frame, wb_writev_cbk,
-		    FIRST_CHILD (this), FIRST_CHILD (this)->fops->writev,
-		    fd, vector, count, offset, flags, iobref, xdata);
-	return 0;
+    STACK_WIND(frame, wb_writev_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->writev, fd, vector, count, offset,
+               flags, iobref, xdata);
+    return 0;
 }
 
-
 int
-wb_writev (call_frame_t *frame, xlator_t *this, fd_t *fd, struct iovec *vector,
-           int32_t count, off_t offset, uint32_t flags, struct iobref *iobref,
-           dict_t *xdata)
-{
-        wb_inode_t   *wb_inode      = NULL;
-	wb_conf_t    *conf          = NULL;
-        gf_boolean_t  wb_disabled   = 0;
-        call_stub_t  *stub          = NULL;
-        int           ret           = -1;
-        int32_t       op_errno      = EINVAL;
-	int           o_direct      = O_DIRECT;
-
-	conf = this->private;
-
-        if (wb_fd_err (fd, this, &op_errno)) {
-                goto unwind;
-        }
+wb_writev(call_frame_t *frame, xlator_t *this, fd_t *fd, struct iovec *vector,
+          int32_t count, off_t offset, uint32_t flags, struct iobref *iobref,
+          dict_t *xdata)
+{
+    wb_inode_t *wb_inode = NULL;
+    wb_conf_t *conf = NULL;
+    gf_boolean_t wb_disabled = 0;
+    call_stub_t *stub = NULL;
+    int ret = -1;
+    int32_t op_errno = EINVAL;
+    int o_direct = O_DIRECT;
 
-        wb_inode = wb_inode_create (this, fd->inode);
-	if (!wb_inode) {
-		op_errno = ENOMEM;
-		goto unwind;
-	}
-
-	if (!conf->strict_O_DIRECT)
-		o_direct = 0;
-
-	if (fd->flags & (O_SYNC|O_DSYNC|o_direct))
-		wb_disabled = 1;
-
-	if (flags & (O_SYNC|O_DSYNC|o_direct))
-		wb_disabled = 1;
-
-	if (wb_disabled)
-		stub = fop_writev_stub (frame, wb_writev_helper, fd, vector,
-					count, offset, flags, iobref, xdata);
-	else
-		stub = fop_writev_stub (frame, NULL, fd, vector, count, offset,
-					flags, iobref, xdata);
-        if (!stub) {
-                op_errno = ENOMEM;
-                goto unwind;
-        }
+    conf = this->private;
 
-	if (wb_disabled)
-		ret = wb_enqueue (wb_inode, stub);
-	else
-		ret = wb_enqueue_tempted (wb_inode, stub);
+    wb_inode = wb_inode_create(this, fd->inode);
+    if (!wb_inode) {
+        op_errno = ENOMEM;
+        goto unwind;
+    }
 
-	if (!ret) {
-		op_errno = ENOMEM;
-		goto unwind;
-	}
+    if (!conf->strict_O_DIRECT)
+        o_direct = 0;
 
-        wb_process_queue (wb_inode);
+    if (fd->flags & (O_SYNC | O_DSYNC | o_direct))
+        wb_disabled = 1;
 
-        return 0;
+    if (flags & (O_SYNC | O_DSYNC | o_direct))
+        wb_disabled = 1;
+
+    if (wb_disabled)
+        stub = fop_writev_stub(frame, wb_writev_helper, fd, vector, count,
+                               offset, flags, iobref, xdata);
+    else
+        stub = fop_writev_stub(frame, NULL, fd, vector, count, offset, flags,
+                               iobref, xdata);
+    if (!stub) {
+        op_errno = ENOMEM;
+        goto unwind;
+    }
+
+    if (wb_disabled)
+        ret = wb_enqueue(wb_inode, stub);
+    else
+        ret = wb_enqueue_tempted(wb_inode, stub);
+
+    if (!ret) {
+        op_errno = ENOMEM;
+        goto unwind;
+    }
+
+    wb_process_queue(wb_inode);
+
+    return 0;
 
 unwind:
-        STACK_UNWIND_STRICT (writev, frame, -1, op_errno, NULL, NULL, NULL);
+    STACK_UNWIND_STRICT(writev, frame, -1, op_errno, NULL, NULL, NULL);
 
-        if (stub)
-                call_stub_destroy (stub);
+    if (stub)
+        call_stub_destroy(stub);
 
-        return 0;
+    return 0;
 }
 
-
 int
-wb_readv_helper (call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
-                 off_t offset, uint32_t flags, dict_t *xdata)
+wb_readv_helper(call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
+                off_t offset, uint32_t flags, dict_t *xdata)
 {
-        STACK_WIND (frame, default_readv_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->readv, fd, size, offset, flags,
-                    xdata);
-        return 0;
+    STACK_WIND(frame, default_readv_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->readv, fd, size, offset, flags, xdata);
+    return 0;
 }
 
-
 int
-wb_readv (call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
-          off_t offset, uint32_t flags, dict_t *xdata)
+wb_readv(call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
+         off_t offset, uint32_t flags, dict_t *xdata)
 {
-        wb_inode_t   *wb_inode     = NULL;
-        call_stub_t  *stub         = NULL;
+    wb_inode_t *wb_inode = NULL;
+    call_stub_t *stub = NULL;
 
-        wb_inode = wb_inode_ctx_get (this, fd->inode);
-	if (!wb_inode)
-		goto noqueue;
+    wb_inode = wb_inode_ctx_get(this, fd->inode);
+    if (!wb_inode)
+        goto noqueue;
 
-	stub = fop_readv_stub (frame, wb_readv_helper, fd, size,
-			       offset, flags, xdata);
-	if (!stub)
-		goto unwind;
+    stub = fop_readv_stub(frame, wb_readv_helper, fd, size, offset, flags,
+                          xdata);
+    if (!stub)
+        goto unwind;
 
-	if (!wb_enqueue (wb_inode, stub))
-		goto unwind;
+    if (!wb_enqueue(wb_inode, stub))
+        goto unwind;
 
-	wb_process_queue (wb_inode);
+    wb_process_queue(wb_inode);
 
-        return 0;
+    return 0;
 
 unwind:
-        STACK_UNWIND_STRICT (readv, frame, -1, ENOMEM, NULL, 0, NULL, NULL,
-                             NULL);
-        return 0;
+    STACK_UNWIND_STRICT(readv, frame, -1, ENOMEM, NULL, 0, NULL, NULL, NULL);
+
+    if (stub)
+        call_stub_destroy(stub);
+    return 0;
 
 noqueue:
-        STACK_WIND (frame, default_readv_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->readv, fd, size, offset, flags,
-                    xdata);
-        return 0;
+    STACK_WIND(frame, default_readv_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->readv, fd, size, offset, flags, xdata);
+    return 0;
 }
 
-
 int
-wb_flush_bg_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-               int32_t op_ret, int32_t op_errno, dict_t *xdata)
+wb_flush_bg_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
-        STACK_DESTROY (frame->root);
-        return 0;
+    STACK_DESTROY(frame->root);
+    return 0;
 }
 
-
 int
-wb_flush_helper (call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *xdata)
+wb_flush_helper(call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *xdata)
 {
-        wb_conf_t    *conf        = NULL;
-        wb_inode_t   *wb_inode    = NULL;
-        call_frame_t *bg_frame    = NULL;
-	int32_t       op_errno    = 0;
-	int           op_ret      = 0;
+    wb_conf_t *conf = NULL;
+    wb_inode_t *wb_inode = NULL;
+    call_frame_t *bg_frame = NULL;
+    int32_t op_errno = 0;
+    int op_ret = 0;
 
-        conf = this->private;
+    conf = this->private;
 
-	wb_inode = wb_inode_ctx_get (this, fd->inode);
-	if (!wb_inode) {
-		op_ret = -1;
-		op_errno = EINVAL;
-		goto unwind;
-	}
+    wb_inode = wb_inode_ctx_get(this, fd->inode);
+    if (!wb_inode) {
+        op_ret = -1;
+        op_errno = EINVAL;
+        goto unwind;
+    }
 
-	if (wb_fd_err (fd, this, &op_errno)) {
-		op_ret = -1;
-		goto unwind;
-	}
+    if (conf->flush_behind)
+        goto flushbehind;
 
-        if (conf->flush_behind)
-		goto flushbehind;
-
-	STACK_WIND (frame, default_flush_cbk, FIRST_CHILD(this),
-		    FIRST_CHILD(this)->fops->flush, fd, xdata);
-	return 0;
+    STACK_WIND(frame, default_flush_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->flush, fd, xdata);
+    return 0;
 
 flushbehind:
-	bg_frame = copy_frame (frame);
-	if (!bg_frame) {
-		op_ret = -1;
-		op_errno = ENOMEM;
-		goto unwind;
-	}
-
-	STACK_WIND (bg_frame, wb_flush_bg_cbk, FIRST_CHILD(this),
-		    FIRST_CHILD(this)->fops->flush, fd, xdata);
-	/* fall through */
+    bg_frame = copy_frame(frame);
+    if (!bg_frame) {
+        op_ret = -1;
+        op_errno = ENOMEM;
+        goto unwind;
+    }
+
+    STACK_WIND(bg_frame, wb_flush_bg_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->flush, fd, xdata);
+    /* fall through */
 unwind:
-        STACK_UNWIND_STRICT (flush, frame, op_ret, op_errno, NULL);
+    STACK_UNWIND_STRICT(flush, frame, op_ret, op_errno, NULL);
 
-        return 0;
+    return 0;
 }
 
-
 int
-wb_flush (call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *xdata)
+wb_flush(call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *xdata)
 {
-        wb_inode_t   *wb_inode     = NULL;
-        call_stub_t  *stub         = NULL;
+    wb_inode_t *wb_inode = NULL;
+    call_stub_t *stub = NULL;
 
-        wb_inode = wb_inode_ctx_get (this, fd->inode);
-	if (!wb_inode)
-		goto noqueue;
+    wb_inode = wb_inode_ctx_get(this, fd->inode);
+    if (!wb_inode)
+        goto noqueue;
 
-	stub = fop_flush_stub (frame, wb_flush_helper, fd, xdata);
-	if (!stub)
-		goto unwind;
+    stub = fop_flush_stub(frame, wb_flush_helper, fd, xdata);
+    if (!stub)
+        goto unwind;
 
-	if (!wb_enqueue (wb_inode, stub))
-		goto unwind;
+    if (!wb_enqueue(wb_inode, stub))
+        goto unwind;
 
-        wb_process_queue (wb_inode);
+    wb_process_queue(wb_inode);
 
-        return 0;
+    return 0;
 
 unwind:
-        STACK_UNWIND_STRICT (flush, frame, -1, ENOMEM, NULL);
+    STACK_UNWIND_STRICT(flush, frame, -1, ENOMEM, NULL);
 
-        return 0;
+    if (stub)
+        call_stub_destroy(stub);
+
+    return 0;
 
 noqueue:
-        STACK_WIND (frame, default_flush_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->flush, fd, xdata);
-        return 0;
+    STACK_WIND(frame, default_flush_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->flush, fd, xdata);
+    return 0;
 }
 
-
-
 int
-wb_fsync_helper (call_frame_t *frame, xlator_t *this, fd_t *fd,
-                 int32_t datasync, dict_t *xdata)
+wb_fsync_helper(call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t datasync,
+                dict_t *xdata)
 {
-        STACK_WIND (frame, default_fsync_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->fsync, fd, datasync, xdata);
-        return 0;
+    STACK_WIND(frame, default_fsync_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->fsync, fd, datasync, xdata);
+    return 0;
 }
 
-
 int
-wb_fsync (call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t datasync,
-          dict_t *xdata)
+wb_fsync(call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t datasync,
+         dict_t *xdata)
 {
-        wb_inode_t   *wb_inode     = NULL;
-        call_stub_t  *stub         = NULL;
-	int32_t       op_errno     = EINVAL;
+    wb_inode_t *wb_inode = NULL;
+    call_stub_t *stub = NULL;
+    int32_t op_errno = EINVAL;
 
-	if (wb_fd_err (fd, this, &op_errno))
-		goto unwind;
+    wb_inode = wb_inode_ctx_get(this, fd->inode);
+    if (!wb_inode)
+        goto noqueue;
 
-        wb_inode = wb_inode_ctx_get (this, fd->inode);
-	if (!wb_inode)
-		goto noqueue;
+    stub = fop_fsync_stub(frame, wb_fsync_helper, fd, datasync, xdata);
+    if (!stub)
+        goto unwind;
 
-	stub = fop_fsync_stub (frame, wb_fsync_helper, fd, datasync, xdata);
-	if (!stub)
-		goto unwind;
+    if (!wb_enqueue(wb_inode, stub))
+        goto unwind;
 
-	if (!wb_enqueue (wb_inode, stub))
-		goto unwind;
+    wb_process_queue(wb_inode);
 
-	wb_process_queue (wb_inode);
-
-        return 0;
+    return 0;
 
 unwind:
-        STACK_UNWIND_STRICT (fsync, frame, -1, op_errno, NULL, NULL, NULL);
+    STACK_UNWIND_STRICT(fsync, frame, -1, op_errno, NULL, NULL, NULL);
 
-        return 0;
+    if (stub)
+        call_stub_destroy(stub);
+    return 0;
 
 noqueue:
-        STACK_WIND (frame, default_fsync_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->fsync, fd, datasync, xdata);
-        return 0;
+    STACK_WIND(frame, default_fsync_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->fsync, fd, datasync, xdata);
+    return 0;
 }
 
-
 int
-wb_stat_helper (call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xdata)
+wb_stat_helper(call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xdata)
 {
-        STACK_WIND (frame, default_stat_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->stat, loc, xdata);
-        return 0;
+    STACK_WIND(frame, default_stat_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->stat, loc, xdata);
+    return 0;
 }
 
-
 int
-wb_stat (call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xdata)
+wb_stat(call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xdata)
 {
-        wb_inode_t   *wb_inode     = NULL;
-        call_stub_t  *stub         = NULL;
+    wb_inode_t *wb_inode = NULL;
+    call_stub_t *stub = NULL;
 
+    wb_inode = wb_inode_ctx_get(this, loc->inode);
+    if (!wb_inode)
+        goto noqueue;
 
-	wb_inode = wb_inode_ctx_get (this, loc->inode);
-        if (!wb_inode)
-		goto noqueue;
+    stub = fop_stat_stub(frame, wb_stat_helper, loc, xdata);
+    if (!stub)
+        goto unwind;
 
-	stub = fop_stat_stub (frame, wb_stat_helper, loc, xdata);
-	if (!stub)
-		goto unwind;
+    if (!wb_enqueue(wb_inode, stub))
+        goto unwind;
 
-	if (!wb_enqueue (wb_inode, stub))
-		goto unwind;
+    wb_process_queue(wb_inode);
 
-	wb_process_queue (wb_inode);
-
-        return 0;
+    return 0;
 
 unwind:
-        STACK_UNWIND_STRICT (stat, frame, -1, ENOMEM, NULL, NULL);
+    STACK_UNWIND_STRICT(stat, frame, -1, ENOMEM, NULL, NULL);
 
-        if (stub)
-                call_stub_destroy (stub);
-        return 0;
+    if (stub)
+        call_stub_destroy(stub);
+    return 0;
 
 noqueue:
-        STACK_WIND (frame, default_stat_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->stat, loc, xdata);
-	return 0;
+    STACK_WIND(frame, default_stat_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->stat, loc, xdata);
+    return 0;
 }
 
-
 int
-wb_fstat_helper (call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *xdata)
+wb_fstat_helper(call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *xdata)
 {
-        STACK_WIND (frame, default_fstat_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->fstat, fd, xdata);
-        return 0;
+    STACK_WIND(frame, default_fstat_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->fstat, fd, xdata);
+    return 0;
 }
 
-
 int
-wb_fstat (call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *xdata)
+wb_fstat(call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *xdata)
 {
-        wb_inode_t   *wb_inode     = NULL;
-	call_stub_t  *stub         = NULL;
-
+    wb_inode_t *wb_inode = NULL;
+    call_stub_t *stub = NULL;
 
-        wb_inode = wb_inode_ctx_get (this, fd->inode);
-	if (!wb_inode)
-		goto noqueue;
+    wb_inode = wb_inode_ctx_get(this, fd->inode);
+    if (!wb_inode)
+        goto noqueue;
 
-	stub = fop_fstat_stub (frame, wb_fstat_helper, fd, xdata);
-	if (!stub)
-		goto unwind;
+    stub = fop_fstat_stub(frame, wb_fstat_helper, fd, xdata);
+    if (!stub)
+        goto unwind;
 
-	if (!wb_enqueue (wb_inode, stub))
-		goto unwind;
+    if (!wb_enqueue(wb_inode, stub))
+        goto unwind;
 
-	wb_process_queue (wb_inode);
+    wb_process_queue(wb_inode);
 
-        return 0;
+    return 0;
 
 unwind:
-        STACK_UNWIND_STRICT (fstat, frame, -1, ENOMEM, NULL, NULL);
+    STACK_UNWIND_STRICT(fstat, frame, -1, ENOMEM, NULL, NULL);
 
-        if (stub)
-                call_stub_destroy (stub);
-        return 0;
+    if (stub)
+        call_stub_destroy(stub);
+    return 0;
 
 noqueue:
-        STACK_WIND (frame, default_fstat_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->fstat, fd, xdata);
-        return 0;
+    STACK_WIND(frame, default_fstat_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->fstat, fd, xdata);
+    return 0;
 }
 
-
 int32_t
-wb_truncate_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                 int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
-                 struct iatt *postbuf, dict_t *xdata)
+wb_truncate_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
+                struct iatt *postbuf, dict_t *xdata)
 {
-        GF_ASSERT (frame->local);
+    GF_ASSERT(frame->local);
 
-        if (op_ret == 0)
-                wb_set_inode_size (frame->local, postbuf);
+    if (op_ret == 0)
+        wb_set_inode_size(frame->local, postbuf);
 
-        frame->local = NULL;
+    frame->local = NULL;
 
-        STACK_UNWIND_STRICT (truncate, frame, op_ret, op_errno, prebuf,
-                             postbuf, xdata);
-        return 0;
+    STACK_UNWIND_STRICT(truncate, frame, op_ret, op_errno, prebuf, postbuf,
+                        xdata);
+    return 0;
 }
 
-
 int
-wb_truncate_helper (call_frame_t *frame, xlator_t *this, loc_t *loc,
-                    off_t offset, dict_t *xdata)
+wb_truncate_helper(call_frame_t *frame, xlator_t *this, loc_t *loc,
+                   off_t offset, dict_t *xdata)
 {
-        STACK_WIND (frame, wb_truncate_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->truncate, loc, offset, xdata);
-        return 0;
+    STACK_WIND(frame, wb_truncate_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->truncate, loc, offset, xdata);
+    return 0;
 }
 
-
 int
-wb_truncate (call_frame_t *frame, xlator_t *this, loc_t *loc, off_t offset,
-             dict_t *xdata)
+wb_truncate(call_frame_t *frame, xlator_t *this, loc_t *loc, off_t offset,
+            dict_t *xdata)
 {
-        wb_inode_t   *wb_inode     = NULL;
-	call_stub_t  *stub         = NULL;
+    wb_inode_t *wb_inode = NULL;
+    call_stub_t *stub = NULL;
 
-	wb_inode = wb_inode_create (this, loc->inode);
-	if (!wb_inode)
-		goto unwind;
+    wb_inode = wb_inode_create(this, loc->inode);
+    if (!wb_inode)
+        goto unwind;
 
-	frame->local = wb_inode;
+    frame->local = wb_inode;
 
-	stub = fop_truncate_stub (frame, wb_truncate_helper, loc,
-				  offset, xdata);
-	if (!stub)
-		goto unwind;
+    stub = fop_truncate_stub(frame, wb_truncate_helper, loc, offset, xdata);
+    if (!stub)
+        goto unwind;
 
-	if (!wb_enqueue (wb_inode, stub))
-		goto unwind;
+    if (!wb_enqueue(wb_inode, stub))
+        goto unwind;
 
-	wb_process_queue (wb_inode);
+    wb_process_queue(wb_inode);
 
-        return 0;
+    return 0;
 
 unwind:
-        STACK_UNWIND_STRICT (truncate, frame, -1, ENOMEM, NULL, NULL, NULL);
+    STACK_UNWIND_STRICT(truncate, frame, -1, ENOMEM, NULL, NULL, NULL);
 
-        if (stub)
-                call_stub_destroy (stub);
+    if (stub)
+        call_stub_destroy(stub);
 
-        return 0;
+    return 0;
 }
 
-
 int32_t
-wb_ftruncate_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                  int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
-                  struct iatt *postbuf, dict_t *xdata)
+wb_ftruncate_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                 int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
+                 struct iatt *postbuf, dict_t *xdata)
 {
-        GF_ASSERT (frame->local);
+    GF_ASSERT(frame->local);
 
-        if (op_ret == 0)
-                wb_set_inode_size (frame->local, postbuf);
+    if (op_ret == 0)
+        wb_set_inode_size(frame->local, postbuf);
 
-        frame->local = NULL;
+    frame->local = NULL;
 
-        STACK_UNWIND_STRICT (ftruncate, frame, op_ret, op_errno, prebuf,
-                             postbuf, xdata);
-        return 0;
+    STACK_UNWIND_STRICT(ftruncate, frame, op_ret, op_errno, prebuf, postbuf,
+                        xdata);
+    return 0;
 }
 
-
 int
-wb_ftruncate_helper (call_frame_t *frame, xlator_t *this, fd_t *fd,
-                     off_t offset, dict_t *xdata)
+wb_ftruncate_helper(call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
+                    dict_t *xdata)
 {
-        STACK_WIND (frame, wb_ftruncate_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->ftruncate, fd, offset, xdata);
-        return 0;
+    STACK_WIND(frame, wb_ftruncate_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->ftruncate, fd, offset, xdata);
+    return 0;
 }
 
-
 int
-wb_ftruncate (call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
-              dict_t *xdata)
+wb_ftruncate(call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
+             dict_t *xdata)
 {
-        wb_inode_t   *wb_inode     = NULL;
-        call_stub_t  *stub         = NULL;
-        int32_t       op_errno     = 0;
+    wb_inode_t *wb_inode = NULL;
+    call_stub_t *stub = NULL;
+    int32_t op_errno = 0;
 
-        wb_inode = wb_inode_create (this, fd->inode);
-	if (!wb_inode) {
-                op_errno = ENOMEM;
-		goto unwind;
-        }
+    wb_inode = wb_inode_create(this, fd->inode);
+    if (!wb_inode) {
+        op_errno = ENOMEM;
+        goto unwind;
+    }
 
-	if (wb_fd_err (fd, this, &op_errno))
-		goto unwind;
+    frame->local = wb_inode;
 
-	frame->local = wb_inode;
+    stub = fop_ftruncate_stub(frame, wb_ftruncate_helper, fd, offset, xdata);
+    if (!stub) {
+        op_errno = ENOMEM;
+        goto unwind;
+    }
 
-	stub = fop_ftruncate_stub (frame, wb_ftruncate_helper, fd,
-				   offset, xdata);
-	if (!stub) {
-                op_errno = ENOMEM;
-		goto unwind;
-        }
+    if (!wb_enqueue(wb_inode, stub)) {
+        op_errno = ENOMEM;
+        goto unwind;
+    }
 
-	if (!wb_enqueue (wb_inode, stub)) {
-                op_errno = ENOMEM;
-		goto unwind;
-        }
+    wb_process_queue(wb_inode);
 
-	wb_process_queue (wb_inode);
-
-        return 0;
+    return 0;
 
 unwind:
-	frame->local = NULL;
+    frame->local = NULL;
 
-        STACK_UNWIND_STRICT (ftruncate, frame, -1, op_errno, NULL, NULL, NULL);
+    STACK_UNWIND_STRICT(ftruncate, frame, -1, op_errno, NULL, NULL, NULL);
 
-        if (stub)
-                call_stub_destroy (stub);
-        return 0;
+    if (stub)
+        call_stub_destroy(stub);
+    return 0;
 }
 
-
 int
-wb_setattr_helper (call_frame_t *frame, xlator_t *this, loc_t *loc,
-                   struct iatt *stbuf, int32_t valid, dict_t *xdata)
+wb_setattr_helper(call_frame_t *frame, xlator_t *this, loc_t *loc,
+                  struct iatt *stbuf, int32_t valid, dict_t *xdata)
 {
-        STACK_WIND (frame, default_setattr_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->setattr, loc, stbuf, valid, xdata);
-        return 0;
+    STACK_WIND(frame, default_setattr_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->setattr, loc, stbuf, valid, xdata);
+    return 0;
 }
 
-
 int
-wb_setattr (call_frame_t *frame, xlator_t *this, loc_t *loc,
-            struct iatt *stbuf, int32_t valid, dict_t *xdata)
+wb_setattr(call_frame_t *frame, xlator_t *this, loc_t *loc, struct iatt *stbuf,
+           int32_t valid, dict_t *xdata)
 {
-        wb_inode_t   *wb_inode     = NULL;
-        call_stub_t  *stub         = NULL;
+    wb_inode_t *wb_inode = NULL;
+    call_stub_t *stub = NULL;
 
-	wb_inode = wb_inode_ctx_get (this, loc->inode);
-	if (!wb_inode)
-		goto noqueue;
+    wb_inode = wb_inode_ctx_get(this, loc->inode);
+    if (!wb_inode)
+        goto noqueue;
 
-	stub = fop_setattr_stub (frame, wb_setattr_helper, loc, stbuf,
-                                         valid, xdata);
-	if (!stub)
-		goto unwind;
+    stub = fop_setattr_stub(frame, wb_setattr_helper, loc, stbuf, valid, xdata);
+    if (!stub)
+        goto unwind;
 
-	if (!wb_enqueue (wb_inode, stub))
-		goto unwind;
+    if (!wb_enqueue(wb_inode, stub))
+        goto unwind;
 
-	wb_process_queue (wb_inode);
+    wb_process_queue(wb_inode);
 
-        return 0;
+    return 0;
 unwind:
-        STACK_UNWIND_STRICT (setattr, frame, -1, ENOMEM, NULL, NULL, NULL);
+    STACK_UNWIND_STRICT(setattr, frame, -1, ENOMEM, NULL, NULL, NULL);
 
-        if (stub)
-                call_stub_destroy (stub);
-	return 0;
+    if (stub)
+        call_stub_destroy(stub);
+    return 0;
 
 noqueue:
-        STACK_WIND (frame, default_setattr_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->setattr, loc, stbuf, valid, xdata);
-        return 0;
+    STACK_WIND(frame, default_setattr_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->setattr, loc, stbuf, valid, xdata);
+    return 0;
 }
 
-
 int
-wb_fsetattr_helper (call_frame_t *frame, xlator_t *this, fd_t *fd,
-		    struct iatt *stbuf, int32_t valid, dict_t *xdata)
+wb_fsetattr_helper(call_frame_t *frame, xlator_t *this, fd_t *fd,
+                   struct iatt *stbuf, int32_t valid, dict_t *xdata)
 {
-        STACK_WIND (frame, default_fsetattr_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->fsetattr, fd, stbuf, valid, xdata);
-        return 0;
+    STACK_WIND(frame, default_fsetattr_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->fsetattr, fd, stbuf, valid, xdata);
+    return 0;
 }
 
-
 int
-wb_fsetattr (call_frame_t *frame, xlator_t *this, fd_t *fd,
-	     struct iatt *stbuf, int32_t valid, dict_t *xdata)
+wb_fsetattr(call_frame_t *frame, xlator_t *this, fd_t *fd, struct iatt *stbuf,
+            int32_t valid, dict_t *xdata)
 {
-        wb_inode_t   *wb_inode     = NULL;
-        call_stub_t  *stub         = NULL;
+    wb_inode_t *wb_inode = NULL;
+    call_stub_t *stub = NULL;
 
-	wb_inode = wb_inode_ctx_get (this, fd->inode);
-	if (!wb_inode)
-		goto noqueue;
+    wb_inode = wb_inode_ctx_get(this, fd->inode);
+    if (!wb_inode)
+        goto noqueue;
 
-	stub = fop_fsetattr_stub (frame, wb_fsetattr_helper, fd, stbuf,
-				  valid, xdata);
-	if (!stub)
-		goto unwind;
+    stub = fop_fsetattr_stub(frame, wb_fsetattr_helper, fd, stbuf, valid,
+                             xdata);
+    if (!stub)
+        goto unwind;
 
-	if (!wb_enqueue (wb_inode, stub))
-		goto unwind;
+    if (!wb_enqueue(wb_inode, stub))
+        goto unwind;
 
-	wb_process_queue (wb_inode);
+    wb_process_queue(wb_inode);
 
-        return 0;
+    return 0;
 unwind:
-        STACK_UNWIND_STRICT (fsetattr, frame, -1, ENOMEM, NULL, NULL, NULL);
+    STACK_UNWIND_STRICT(fsetattr, frame, -1, ENOMEM, NULL, NULL, NULL);
 
-        if (stub)
-                call_stub_destroy (stub);
-	return 0;
+    if (stub)
+        call_stub_destroy(stub);
+    return 0;
 
 noqueue:
-        STACK_WIND (frame, default_fsetattr_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->fsetattr, fd, stbuf, valid, xdata);
-        return 0;
+    STACK_WIND(frame, default_fsetattr_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->fsetattr, fd, stbuf, valid, xdata);
+    return 0;
 }
 
-
 int32_t
-wb_create (call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t flags,
-           mode_t mode, mode_t umask, fd_t *fd, dict_t *xdata)
+wb_create(call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t flags,
+          mode_t mode, mode_t umask, fd_t *fd, dict_t *xdata)
 {
-        wb_inode_t   *wb_inode     = NULL;
+    wb_inode_t *wb_inode = NULL;
 
-        wb_inode = wb_inode_create (this, fd->inode);
-	if (!wb_inode)
-		goto unwind;
+    wb_inode = wb_inode_create(this, fd->inode);
+    if (!wb_inode)
+        goto unwind;
 
-	if (((flags & O_RDWR) || (flags & O_WRONLY)) && (flags & O_TRUNC))
-		wb_inode->size = 0;
+    if (((flags & O_RDWR) || (flags & O_WRONLY)) && (flags & O_TRUNC))
+        wb_inode->size = 0;
 
-	STACK_WIND_TAIL (frame, FIRST_CHILD(this),
-                         FIRST_CHILD(this)->fops->create, loc, flags, mode,
-                         umask, fd, xdata);
-        return 0;
+    STACK_WIND_TAIL(frame, FIRST_CHILD(this), FIRST_CHILD(this)->fops->create,
+                    loc, flags, mode, umask, fd, xdata);
+    return 0;
 
 unwind:
-	STACK_UNWIND_STRICT (create, frame, -1, ENOMEM, NULL, NULL, NULL, NULL,
-                             NULL, NULL);
-	return 0;
+    STACK_UNWIND_STRICT(create, frame, -1, ENOMEM, NULL, NULL, NULL, NULL, NULL,
+                        NULL);
+    return 0;
 }
 
-
 int32_t
-wb_open (call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t flags,
-         fd_t *fd, dict_t *xdata)
+wb_open(call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t flags,
+        fd_t *fd, dict_t *xdata)
 {
-        wb_inode_t   *wb_inode     = NULL;
+    wb_inode_t *wb_inode = NULL;
 
-        wb_inode = wb_inode_create (this, fd->inode);
-	if (!wb_inode)
-		goto unwind;
+    wb_inode = wb_inode_create(this, fd->inode);
+    if (!wb_inode)
+        goto unwind;
 
-	if (((flags & O_RDWR) || (flags & O_WRONLY)) && (flags & O_TRUNC))
-		wb_inode->size = 0;
+    if (((flags & O_RDWR) || (flags & O_WRONLY)) && (flags & O_TRUNC))
+        wb_inode->size = 0;
 
-	STACK_WIND_TAIL (frame, FIRST_CHILD(this),
-                         FIRST_CHILD(this)->fops->open, loc, flags, fd, xdata);
-        return 0;
+    STACK_WIND_TAIL(frame, FIRST_CHILD(this), FIRST_CHILD(this)->fops->open,
+                    loc, flags, fd, xdata);
+    return 0;
 
 unwind:
-	STACK_UNWIND_STRICT (open, frame, -1, ENOMEM, NULL, NULL);
-	return 0;
+    STACK_UNWIND_STRICT(open, frame, -1, ENOMEM, NULL, NULL);
+    return 0;
 }
 
-
 int32_t
-wb_lookup_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-               int32_t op_ret, int32_t op_errno, inode_t *inode,
-               struct iatt *buf, dict_t *xdata, struct iatt *postparent)
-{
-        if (op_ret == 0) {
-                wb_inode_t *wb_inode = wb_inode_ctx_get (this, inode);
-                if (wb_inode)
-                        wb_set_inode_size (wb_inode, buf);
-        }
+wb_lookup_cbk(call_frame_t *frame, void *cookie, xlator_t *this, int32_t op_ret,
+              int32_t op_errno, inode_t *inode, struct iatt *buf, dict_t *xdata,
+              struct iatt *postparent)
+{
+    if (op_ret == 0) {
+        wb_inode_t *wb_inode = wb_inode_ctx_get(this, inode);
+        if (wb_inode)
+            wb_set_inode_size(wb_inode, buf);
+    }
 
-        STACK_UNWIND_STRICT (lookup, frame, op_ret, op_errno, inode, buf,
-                             xdata, postparent);
-        return 0;
+    STACK_UNWIND_STRICT(lookup, frame, op_ret, op_errno, inode, buf, xdata,
+                        postparent);
+    return 0;
 }
 
+int
+wb_lookup_helper(call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xdata)
+{
+    STACK_WIND(frame, wb_lookup_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->lookup, loc, xdata);
+    return 0;
+}
 
 int32_t
-wb_lookup (call_frame_t *frame, xlator_t *this, loc_t *loc,
-           dict_t *xdata)
+wb_lookup(call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xdata)
 {
-        STACK_WIND (frame, wb_lookup_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->lookup, loc, xdata);
-        return 0;
-}
+    wb_inode_t *wb_inode = NULL;
+    call_stub_t *stub = NULL;
 
+    wb_inode = wb_inode_ctx_get(this, loc->inode);
+    if (!wb_inode)
+        goto noqueue;
 
-int
-wb_forget (xlator_t *this, inode_t *inode)
+    stub = fop_lookup_stub(frame, wb_lookup_helper, loc, xdata);
+    if (!stub)
+        goto unwind;
+
+    if (!wb_enqueue(wb_inode, stub))
+        goto unwind;
+
+    wb_process_queue(wb_inode);
+
+    return 0;
+
+unwind:
+    if (stub)
+        call_stub_destroy(stub);
+
+    STACK_UNWIND_STRICT(lookup, frame, -1, ENOMEM, NULL, NULL, NULL, NULL);
+    return 0;
+
+noqueue:
+    STACK_WIND(frame, wb_lookup_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->lookup, loc, xdata);
+    return 0;
+}
+
+static void
+wb_mark_readdirp_start(xlator_t *this, inode_t *directory)
 {
-        uint64_t    tmp      = 0;
-        wb_inode_t *wb_inode = NULL;
+    wb_inode_t *wb_directory_inode = NULL;
+
+    wb_directory_inode = wb_inode_create(this, directory);
 
-        inode_ctx_del (inode, this, &tmp);
+    if (!wb_directory_inode)
+        return;
 
-        wb_inode = (wb_inode_t *)(long)tmp;
+    LOCK(&wb_directory_inode->lock);
+    {
+        GF_ATOMIC_INC(wb_directory_inode->readdirps);
+    }
+    UNLOCK(&wb_directory_inode->lock);
 
-	if (!wb_inode)
-		return 0;
+    return;
+}
 
-        GF_ASSERT (list_empty (&wb_inode->todo));
-        GF_ASSERT (list_empty (&wb_inode->liability));
-        GF_ASSERT (list_empty (&wb_inode->temptation));
+static void
+wb_mark_readdirp_end(xlator_t *this, inode_t *directory)
+{
+    wb_inode_t *wb_directory_inode = NULL, *wb_inode = NULL, *tmp = NULL;
+    int readdirps = 0;
 
-        GF_FREE (wb_inode);
+    wb_directory_inode = wb_inode_ctx_get(this, directory);
 
-        return 0;
+    if (!wb_directory_inode)
+        return;
+
+    LOCK(&wb_directory_inode->lock);
+    {
+        readdirps = GF_ATOMIC_DEC(wb_directory_inode->readdirps);
+        if (readdirps)
+            goto unlock;
+
+        list_for_each_entry_safe(wb_inode, tmp,
+                                 &wb_directory_inode->invalidate_list,
+                                 invalidate_list)
+        {
+            list_del_init(&wb_inode->invalidate_list);
+            GF_ATOMIC_INIT(wb_inode->invalidate, 0);
+            inode_unref(wb_inode->inode);
+        }
+    }
+unlock:
+    UNLOCK(&wb_directory_inode->lock);
+
+    return;
 }
 
+int32_t
+wb_readdirp_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                int32_t op_ret, int32_t op_errno, gf_dirent_t *entries,
+                dict_t *xdata)
+{
+    wb_inode_t *wb_inode = NULL;
+    gf_dirent_t *entry = NULL;
+    inode_t *inode = NULL;
+    fd_t *fd = NULL;
+
+    fd = frame->local;
+    frame->local = NULL;
 
-int
-wb_release (xlator_t *this, fd_t *fd)
+    if (op_ret <= 0)
+        goto unwind;
+
+    list_for_each_entry(entry, &entries->list, list)
+    {
+        if (!entry->inode || !IA_ISREG(entry->d_stat.ia_type))
+            continue;
+
+        wb_inode = wb_inode_ctx_get(this, entry->inode);
+        if (!wb_inode)
+            continue;
+
+        LOCK(&wb_inode->lock);
+        {
+            if (!list_empty(&wb_inode->liability) ||
+                GF_ATOMIC_GET(wb_inode->invalidate)) {
+                inode = entry->inode;
+
+                entry->inode = NULL;
+                memset(&entry->d_stat, 0, sizeof(entry->d_stat));
+            }
+        }
+        UNLOCK(&wb_inode->lock);
+
+        if (inode) {
+            inode_unref(inode);
+            inode = NULL;
+        }
+    }
+
+unwind:
+    wb_mark_readdirp_end(this, fd->inode);
+
+    frame->local = NULL;
+    STACK_UNWIND_STRICT(readdirp, frame, op_ret, op_errno, entries, xdata);
+    return 0;
+}
+
+int32_t
+wb_readdirp(call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
+            off_t off, dict_t *xdata)
 {
-        uint64_t    tmp      = 0;
+    wb_mark_readdirp_start(this, fd->inode);
 
-        fd_ctx_del (fd, this, &tmp);
+    frame->local = fd;
 
-        return 0;
+    STACK_WIND(frame, wb_readdirp_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->readdirp, fd, size, off, xdata);
+
+    return 0;
 }
 
+int32_t
+wb_link_helper(call_frame_t *frame, xlator_t *this, loc_t *oldloc,
+               loc_t *newloc, dict_t *xdata)
+{
+    STACK_WIND_TAIL(frame, FIRST_CHILD(this), FIRST_CHILD(this)->fops->link,
+                    oldloc, newloc, xdata);
+    return 0;
+}
 
-int
-wb_priv_dump (xlator_t *this)
+int32_t
+wb_link(call_frame_t *frame, xlator_t *this, loc_t *oldloc, loc_t *newloc,
+        dict_t *xdata)
 {
-        wb_conf_t      *conf                            = NULL;
-        char            key_prefix[GF_DUMP_MAX_BUF_LEN] = {0, };
-        int             ret                             = -1;
+    wb_inode_t *wb_inode = NULL;
+    call_stub_t *stub = NULL;
 
-        GF_VALIDATE_OR_GOTO ("write-behind", this, out);
+    wb_inode = wb_inode_ctx_get(this, oldloc->inode);
+    if (!wb_inode)
+        goto noqueue;
 
-        conf = this->private;
-        GF_VALIDATE_OR_GOTO (this->name, conf, out);
+    stub = fop_link_stub(frame, wb_link_helper, oldloc, newloc, xdata);
+    if (!stub)
+        goto unwind;
 
-        gf_proc_dump_build_key (key_prefix, "xlator.performance.write-behind",
-                                "priv");
+    if (!wb_enqueue(wb_inode, stub))
+        goto unwind;
 
-        gf_proc_dump_add_section (key_prefix);
+    wb_process_queue(wb_inode);
 
-        gf_proc_dump_write ("aggregate_size", "%d", conf->aggregate_size);
-        gf_proc_dump_write ("window_size", "%d", conf->window_size);
-        gf_proc_dump_write ("flush_behind", "%d", conf->flush_behind);
-        gf_proc_dump_write ("trickling_writes", "%d", conf->trickling_writes);
+    return 0;
 
-        ret = 0;
-out:
-        return ret;
+unwind:
+    STACK_UNWIND_STRICT(link, frame, -1, ENOMEM, NULL, NULL, NULL, NULL, NULL);
+
+    if (stub)
+        call_stub_destroy(stub);
+
+    return 0;
+
+noqueue:
+    STACK_WIND_TAIL(frame, FIRST_CHILD(this), FIRST_CHILD(this)->fops->link,
+                    oldloc, newloc, xdata);
+    return 0;
 }
 
+int32_t
+wb_fallocate_helper(call_frame_t *frame, xlator_t *this, fd_t *fd,
+                    int32_t keep_size, off_t offset, size_t len, dict_t *xdata)
+{
+    STACK_WIND_TAIL(frame, FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->fallocate, fd, keep_size, offset,
+                    len, xdata);
+    return 0;
+}
 
-void
-__wb_dump_requests (struct list_head *head, char *prefix)
+int32_t
+wb_fallocate(call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t keep_size,
+             off_t offset, size_t len, dict_t *xdata)
 {
-        char          key[GF_DUMP_MAX_BUF_LEN]        = {0, };
-        char          key_prefix[GF_DUMP_MAX_BUF_LEN] = {0, }, flag = 0;
-        wb_request_t *req                             = NULL;
+    wb_inode_t *wb_inode = NULL;
+    call_stub_t *stub = NULL;
+
+    wb_inode = wb_inode_ctx_get(this, fd->inode);
+    if (!wb_inode)
+        goto noqueue;
+
+    stub = fop_fallocate_stub(frame, wb_fallocate_helper, fd, keep_size, offset,
+                              len, xdata);
+    if (!stub)
+        goto unwind;
+
+    if (!wb_enqueue(wb_inode, stub))
+        goto unwind;
+
+    wb_process_queue(wb_inode);
+
+    return 0;
+
+unwind:
+    STACK_UNWIND_STRICT(fallocate, frame, -1, ENOMEM, NULL, NULL, NULL);
+
+    if (stub)
+        call_stub_destroy(stub);
+
+    return 0;
+
+noqueue:
+    STACK_WIND_TAIL(frame, FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->fallocate, fd, keep_size, offset,
+                    len, xdata);
+    return 0;
+}
 
-        list_for_each_entry (req, head, all) {
-                gf_proc_dump_build_key (key_prefix, key,
-                                        (char *)gf_fop_list[req->fop]);
+int32_t
+wb_discard_helper(call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
+                  size_t len, dict_t *xdata)
+{
+    STACK_WIND_TAIL(frame, FIRST_CHILD(this), FIRST_CHILD(this)->fops->discard,
+                    fd, offset, len, xdata);
+    return 0;
+}
 
-                gf_proc_dump_add_section(key_prefix);
+int32_t
+wb_discard(call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
+           size_t len, dict_t *xdata)
+{
+    wb_inode_t *wb_inode = NULL;
+    call_stub_t *stub = NULL;
 
-                gf_proc_dump_write ("request-ptr", "%p", req);
+    wb_inode = wb_inode_ctx_get(this, fd->inode);
+    if (!wb_inode)
+        goto noqueue;
 
-                gf_proc_dump_write ("refcount", "%d", req->refcount);
+    stub = fop_discard_stub(frame, wb_discard_helper, fd, offset, len, xdata);
+    if (!stub)
+        goto unwind;
 
-		if (list_empty (&req->todo))
-			gf_proc_dump_write ("wound", "yes");
-		else
-			gf_proc_dump_write ("wound", "no");
+    if (!wb_enqueue(wb_inode, stub))
+        goto unwind;
 
-                if (req->fop == GF_FOP_WRITE) {
-                        gf_proc_dump_write ("size", "%"GF_PRI_SIZET,
-                                            req->write_size);
+    wb_process_queue(wb_inode);
 
-                        gf_proc_dump_write ("offset", "%"PRId64,
-                                            req->stub->args.offset);
+    return 0;
 
-                        flag = req->ordering.lied;
-                        gf_proc_dump_write ("lied", "%d", flag);
+unwind:
+    STACK_UNWIND_STRICT(discard, frame, -1, ENOMEM, NULL, NULL, NULL);
 
-                        flag = req->ordering.append;
-                        gf_proc_dump_write ("append", "%d", flag);
+    if (stub)
+        call_stub_destroy(stub);
+    return 0;
 
-                        flag = req->ordering.fulfilled;
-                        gf_proc_dump_write ("fulfilled", "%d", flag);
+noqueue:
+    STACK_WIND_TAIL(frame, FIRST_CHILD(this), FIRST_CHILD(this)->fops->discard,
+                    fd, offset, len, xdata);
 
-                        flag = req->ordering.go;
-                        gf_proc_dump_write ("go", "%d", flag);
-                }
-        }
+    return 0;
 }
 
+int32_t
+wb_zerofill_helper(call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
+                   off_t len, dict_t *xdata)
+{
+    STACK_WIND_TAIL(frame, FIRST_CHILD(this), FIRST_CHILD(this)->fops->zerofill,
+                    fd, offset, len, xdata);
+    return 0;
+}
 
-int
-wb_inode_dump (xlator_t *this, inode_t *inode)
+int32_t
+wb_zerofill(call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
+            off_t len, dict_t *xdata)
 {
-        wb_inode_t *wb_inode                       = NULL;
-        int32_t     ret                            = -1;
-        char       *path                           = NULL;
-        char       key_prefix[GF_DUMP_MAX_BUF_LEN] = {0, };
-        char       uuid_str[64] = {0,};
+    wb_inode_t *wb_inode = NULL;
+    call_stub_t *stub = NULL;
 
-        if ((inode == NULL) || (this == NULL)) {
-                ret = 0;
-                goto out;
-        }
+    wb_inode = wb_inode_ctx_get(this, fd->inode);
+    if (!wb_inode)
+        goto noqueue;
 
-        wb_inode = wb_inode_ctx_get (this, inode);
-        if (wb_inode == NULL) {
-                ret = 0;
-                goto out;
-        }
+    stub = fop_zerofill_stub(frame, wb_zerofill_helper, fd, offset, len, xdata);
+    if (!stub)
+        goto unwind;
 
-        gf_proc_dump_build_key (key_prefix, "xlator.performance.write-behind",
-                                "wb_inode");
+    if (!wb_enqueue(wb_inode, stub))
+        goto unwind;
 
-        gf_proc_dump_add_section (key_prefix);
+    wb_process_queue(wb_inode);
 
-        __inode_path (inode, NULL, &path);
-        if (path != NULL) {
-                gf_proc_dump_write ("path", "%s", path);
-                GF_FREE (path);
-        }
+    return 0;
 
-        gf_proc_dump_write ("inode", "%p", inode);
+unwind:
+    STACK_UNWIND_STRICT(zerofill, frame, -1, ENOMEM, NULL, NULL, NULL);
 
-        gf_proc_dump_write ("window_conf", "%"GF_PRI_SIZET,
-                            wb_inode->window_conf);
+    if (stub)
+        call_stub_destroy(stub);
 
-        gf_proc_dump_write ("window_current", "%"GF_PRI_SIZET,
-                            wb_inode->window_current);
+noqueue:
+    STACK_WIND_TAIL(frame, FIRST_CHILD(this), FIRST_CHILD(this)->fops->zerofill,
+                    fd, offset, len, xdata);
+    return 0;
+}
 
+int32_t
+wb_rename(call_frame_t *frame, xlator_t *this, loc_t *oldloc, loc_t *newloc,
+          dict_t *xdata)
+{
+    wb_inode_t *wb_inode = NULL;
+    call_stub_t *stub = NULL;
 
-        ret = TRY_LOCK (&wb_inode->lock);
-        if (!ret)
-        {
-                if (!list_empty (&wb_inode->all)) {
-                        __wb_dump_requests (&wb_inode->all, key_prefix);
-                }
-                UNLOCK (&wb_inode->lock);
-        }
+    wb_inode = wb_inode_ctx_get(this, oldloc->inode);
+    if (!wb_inode)
+        goto noqueue;
 
-        if (ret && wb_inode)
-                gf_proc_dump_write ("Unable to dump the inode information",
-                                    "(Lock acquisition failed) %p (gfid: %s)",
-                                    wb_inode,
-                                    uuid_utoa_r (inode->gfid, uuid_str));
-        ret = 0;
-out:
-        return ret;
-}
+    stub = fop_rename_stub(frame, default_rename_resume, oldloc, newloc, xdata);
+    if (!stub)
+        goto unwind;
+
+    if (!wb_enqueue(wb_inode, stub))
+        goto unwind;
+
+    wb_process_queue(wb_inode);
+
+    return 0;
+
+unwind:
+    if (stub)
+        call_stub_destroy(stub);
 
+    STACK_UNWIND_STRICT(rename, frame, -1, ENOMEM, NULL, NULL, NULL, NULL, NULL,
+                        NULL);
+
+    return 0;
+
+noqueue:
+    STACK_WIND_TAIL(frame, FIRST_CHILD(this), FIRST_CHILD(this)->fops->rename,
+                    oldloc, newloc, xdata);
+    return 0;
+}
 
 int
-mem_acct_init (xlator_t *this)
+wb_forget(xlator_t *this, inode_t *inode)
 {
-        int     ret = -1;
+    uint64_t tmp = 0;
+    wb_inode_t *wb_inode = NULL;
 
-        if (!this) {
-                goto out;
-        }
+    inode_ctx_del(inode, this, &tmp);
 
-        ret = xlator_mem_acct_init (this, gf_wb_mt_end + 1);
+    wb_inode = (wb_inode_t *)(long)tmp;
 
-        if (ret != 0) {
-                gf_log (this->name, GF_LOG_ERROR, "Memory accounting init"
-                        "failed");
-        }
+    if (!wb_inode)
+        return 0;
 
-out:
-        return ret;
+    wb_inode_destroy(wb_inode);
+
+    return 0;
 }
 
+int
+wb_release(xlator_t *this, fd_t *fd)
+{
+    uint64_t tmp = 0;
+
+    (void)fd_ctx_del(fd, this, &tmp);
+
+    return 0;
+}
 
 int
-reconfigure (xlator_t *this, dict_t *options)
+wb_priv_dump(xlator_t *this)
 {
-        wb_conf_t *conf = NULL;
-        int        ret  = -1;
+    wb_conf_t *conf = NULL;
+    char key_prefix[GF_DUMP_MAX_BUF_LEN] = {
+        0,
+    };
+    int ret = -1;
 
-        conf = this->private;
+    GF_VALIDATE_OR_GOTO("write-behind", this, out);
 
-        GF_OPTION_RECONF ("cache-size", conf->window_size, options, size_uint64, out);
+    conf = this->private;
+    GF_VALIDATE_OR_GOTO(this->name, conf, out);
 
-        GF_OPTION_RECONF ("flush-behind", conf->flush_behind, options, bool,
-                          out);
+    gf_proc_dump_build_key(key_prefix, "xlator.performance.write-behind",
+                           "priv");
 
-        GF_OPTION_RECONF ("trickling-writes", conf->trickling_writes, options,
-			  bool, out);
+    gf_proc_dump_add_section("%s", key_prefix);
 
-        GF_OPTION_RECONF ("strict-O_DIRECT", conf->strict_O_DIRECT, options,
-			  bool, out);
+    gf_proc_dump_write("aggregate_size", "%" PRIu64, conf->aggregate_size);
+    gf_proc_dump_write("window_size", "%" PRIu64, conf->window_size);
+    gf_proc_dump_write("flush_behind", "%d", conf->flush_behind);
+    gf_proc_dump_write("trickling_writes", "%d", conf->trickling_writes);
 
-        GF_OPTION_RECONF ("strict-write-ordering", conf->strict_write_ordering,
-			  options, bool, out);
-        ret = 0;
+    ret = 0;
 out:
-        return ret;
+    return ret;
 }
 
-
-int32_t
-init (xlator_t *this)
+void
+__wb_dump_requests(struct list_head *head, char *prefix)
 {
-        wb_conf_t *conf    = NULL;
-        int32_t    ret     = -1;
+    char key[GF_DUMP_MAX_BUF_LEN] = {
+        0,
+    };
+    char key_prefix[GF_DUMP_MAX_BUF_LEN] =
+        {
+            0,
+        },
+         flag = 0;
+    wb_request_t *req = NULL;
 
-        if ((this->children == NULL)
-            || this->children->next) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "FATAL: write-behind (%s) not configured with exactly "
-                        "one child", this->name);
-                goto out;
-        }
+    list_for_each_entry(req, head, all)
+    {
+        gf_proc_dump_build_key(key_prefix, key, "%s",
+                               (char *)gf_fop_list[req->fop]);
 
-        if (this->parents == NULL) {
-                gf_log (this->name, GF_LOG_WARNING,
-                        "dangling volume. check volfilex");
-        }
+        gf_proc_dump_add_section("%s", key_prefix);
 
-        conf = GF_CALLOC (1, sizeof (*conf), gf_wb_mt_wb_conf_t);
-        if (conf == NULL) {
-                goto out;
-        }
+        gf_proc_dump_write("unique", "%" PRIu64, req->unique);
 
-        /* configure 'options aggregate-size <size>' */
-        conf->aggregate_size = WB_AGGREGATE_SIZE;
+        gf_proc_dump_write("refcount", "%d", req->refcount);
 
-        /* configure 'option window-size <size>' */
-        GF_OPTION_INIT ("cache-size", conf->window_size, size_uint64, out);
+        if (list_empty(&req->todo))
+            gf_proc_dump_write("wound", "yes");
+        else
+            gf_proc_dump_write("wound", "no");
 
-        if (!conf->window_size && conf->aggregate_size) {
-                gf_log (this->name, GF_LOG_WARNING,
-                        "setting window-size to be equal to "
-                        "aggregate-size(%"PRIu64")",
-                        conf->aggregate_size);
-                conf->window_size = conf->aggregate_size;
-        }
+        gf_proc_dump_write("generation-number", "%" PRIu64, req->gen);
 
-        if (conf->window_size < conf->aggregate_size) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "aggregate-size(%"PRIu64") cannot be more than "
-                        "window-size(%"PRIu64")", conf->aggregate_size,
-                        conf->window_size);
-                goto out;
-        }
+        gf_proc_dump_write("req->op_ret", "%d", req->op_ret);
+        gf_proc_dump_write("req->op_errno", "%d", req->op_errno);
+        gf_proc_dump_write("sync-attempts", "%d", req->wind_count);
+
+        if (req->fop == GF_FOP_WRITE) {
+            if (list_empty(&req->wip))
+                gf_proc_dump_write("sync-in-progress", "no");
+            else
+                gf_proc_dump_write("sync-in-progress", "yes");
 
-        /* configure 'option flush-behind <on/off>' */
-        GF_OPTION_INIT ("flush-behind", conf->flush_behind, bool, out);
+            gf_proc_dump_write("size", "%" GF_PRI_SIZET, req->write_size);
 
-        GF_OPTION_INIT ("trickling-writes", conf->trickling_writes, bool, out);
+            if (req->stub)
+                gf_proc_dump_write("offset", "%" PRId64,
+                                   req->stub->args.offset);
 
-        GF_OPTION_INIT ("strict-O_DIRECT", conf->strict_O_DIRECT, bool, out);
+            flag = req->ordering.lied;
+            gf_proc_dump_write("lied", "%d", flag);
+
+            flag = req->ordering.append;
+            gf_proc_dump_write("append", "%d", flag);
+
+            flag = req->ordering.fulfilled;
+            gf_proc_dump_write("fulfilled", "%d", flag);
+
+            flag = req->ordering.go;
+            gf_proc_dump_write("go", "%d", flag);
+        }
+    }
+}
 
-        GF_OPTION_INIT ("strict-write-ordering", conf->strict_write_ordering,
-			bool, out);
+int
+wb_inode_dump(xlator_t *this, inode_t *inode)
+{
+    wb_inode_t *wb_inode = NULL;
+    int32_t ret = -1;
+    char *path = NULL;
+    char key_prefix[GF_DUMP_MAX_BUF_LEN] = {
+        0,
+    };
+    char uuid_str[64] = {
+        0,
+    };
+
+    if ((inode == NULL) || (this == NULL)) {
+        ret = 0;
+        goto out;
+    }
 
-        this->private = conf;
+    wb_inode = wb_inode_ctx_get(this, inode);
+    if (wb_inode == NULL) {
         ret = 0;
+        goto out;
+    }
 
-out:
-        if (ret) {
-                GF_FREE (conf);
+    uuid_utoa_r(inode->gfid, uuid_str);
+
+    gf_proc_dump_build_key(key_prefix, "xlator.performance.write-behind",
+                           "wb_inode");
+
+    gf_proc_dump_add_section("%s", key_prefix);
+
+    __inode_path(inode, NULL, &path);
+    if (path != NULL) {
+        gf_proc_dump_write("path", "%s", path);
+        GF_FREE(path);
+    }
+
+    gf_proc_dump_write("inode", "%p", inode);
+
+    gf_proc_dump_write("gfid", "%s", uuid_str);
+
+    gf_proc_dump_write("window_conf", "%" GF_PRI_SIZET, wb_inode->window_conf);
+
+    gf_proc_dump_write("window_current", "%" GF_PRI_SIZET,
+                       wb_inode->window_current);
+
+    gf_proc_dump_write("transit-size", "%" GF_PRI_SIZET, wb_inode->transit);
+
+    gf_proc_dump_write("dontsync", "%d", wb_inode->dontsync);
+
+    ret = TRY_LOCK(&wb_inode->lock);
+    if (!ret) {
+        if (!list_empty(&wb_inode->all)) {
+            __wb_dump_requests(&wb_inode->all, key_prefix);
         }
-        return ret;
+        UNLOCK(&wb_inode->lock);
+    }
+
+    if (ret && wb_inode)
+        gf_proc_dump_write("Unable to dump the inode information",
+                           "(Lock acquisition failed) %p (gfid: %s)", wb_inode,
+                           uuid_str);
+
+    ret = 0;
+out:
+    return ret;
 }
 
+int
+mem_acct_init(xlator_t *this)
+{
+    int ret = -1;
 
-void
-fini (xlator_t *this)
+    if (!this) {
+        goto out;
+    }
+
+    ret = xlator_mem_acct_init(this, gf_wb_mt_end + 1);
+
+    if (ret != 0) {
+        gf_msg(this->name, GF_LOG_ERROR, ENOMEM, WRITE_BEHIND_MSG_NO_MEMORY,
+               "Memory accounting init"
+               "failed");
+    }
+
+out:
+    return ret;
+}
+
+int
+reconfigure(xlator_t *this, dict_t *options)
 {
-        wb_conf_t *conf = NULL;
+    wb_conf_t *conf = NULL;
+    int ret = -1;
 
-        GF_VALIDATE_OR_GOTO ("write-behind", this, out);
+    conf = this->private;
 
-        conf = this->private;
-        if (!conf) {
-                goto out;
-        }
+    GF_OPTION_RECONF("cache-size", conf->window_size, options, size_uint64,
+                     out);
+
+    GF_OPTION_RECONF("flush-behind", conf->flush_behind, options, bool, out);
+
+    GF_OPTION_RECONF("trickling-writes", conf->trickling_writes, options, bool,
+                     out);
+
+    GF_OPTION_RECONF("strict-O_DIRECT", conf->strict_O_DIRECT, options, bool,
+                     out);
 
-        this->private = NULL;
-        GF_FREE (conf);
+    GF_OPTION_RECONF("strict-write-ordering", conf->strict_write_ordering,
+                     options, bool, out);
+    GF_OPTION_RECONF("resync-failed-syncs-after-fsync",
+                     conf->resync_after_fsync, options, bool, out);
 
+    ret = 0;
 out:
-        return;
+    return ret;
 }
 
+int32_t
+init(xlator_t *this)
+{
+    wb_conf_t *conf = NULL;
+    int32_t ret = -1;
+
+    if ((this->children == NULL) || this->children->next) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, WRITE_BEHIND_MSG_INIT_FAILED,
+               "FATAL: write-behind (%s) not configured with exactly "
+               "one child",
+               this->name);
+        goto out;
+    }
+
+    if (this->parents == NULL) {
+        gf_msg(this->name, GF_LOG_WARNING, 0,
+               WRITE_BEHIND_MSG_VOL_MISCONFIGURED,
+               "dangling volume. check volfilex");
+    }
+
+    conf = GF_CALLOC(1, sizeof(*conf), gf_wb_mt_wb_conf_t);
+    if (conf == NULL) {
+        goto out;
+    }
+
+    /* configure 'options aggregate-size <size>' */
+    GF_OPTION_INIT("aggregate-size", conf->aggregate_size, size_uint64, out);
+    conf->page_size = conf->aggregate_size;
+
+    /* configure 'option window-size <size>' */
+    GF_OPTION_INIT("cache-size", conf->window_size, size_uint64, out);
+
+    if (!conf->window_size && conf->aggregate_size) {
+        gf_msg(this->name, GF_LOG_WARNING, 0, WRITE_BEHIND_MSG_SIZE_NOT_SET,
+               "setting window-size to be equal to "
+               "aggregate-size(%" PRIu64 ")",
+               conf->aggregate_size);
+        conf->window_size = conf->aggregate_size;
+    }
+
+    if (conf->window_size < conf->aggregate_size) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, WRITE_BEHIND_MSG_EXCEEDED_MAX_SIZE,
+               "aggregate-size(%" PRIu64
+               ") cannot be more than "
+               "window-size(%" PRIu64 ")",
+               conf->aggregate_size, conf->window_size);
+        goto out;
+    }
+
+    /* configure 'option flush-behind <on/off>' */
+    GF_OPTION_INIT("flush-behind", conf->flush_behind, bool, out);
+
+    GF_OPTION_INIT("trickling-writes", conf->trickling_writes, bool, out);
+
+    GF_OPTION_INIT("strict-O_DIRECT", conf->strict_O_DIRECT, bool, out);
+
+    GF_OPTION_INIT("strict-write-ordering", conf->strict_write_ordering, bool,
+                   out);
+
+    GF_OPTION_INIT("resync-failed-syncs-after-fsync", conf->resync_after_fsync,
+                   bool, out);
+
+    this->private = conf;
+    ret = 0;
 
-struct xlator_fops fops = {
-        .writev      = wb_writev,
-        .readv       = wb_readv,
-        .flush       = wb_flush,
-        .fsync       = wb_fsync,
-        .stat        = wb_stat,
-        .fstat       = wb_fstat,
-        .truncate    = wb_truncate,
-        .ftruncate   = wb_ftruncate,
-        .setattr     = wb_setattr,
-        .fsetattr    = wb_fsetattr,
-};
+out:
+    if (ret) {
+        GF_FREE(conf);
+    }
+    return ret;
+}
 
+void
+fini(xlator_t *this)
+{
+    wb_conf_t *conf = NULL;
 
-struct xlator_cbks cbks = {
-        .forget   = wb_forget,
-        .release  = wb_release
+    GF_VALIDATE_OR_GOTO("write-behind", this, out);
+
+    conf = this->private;
+    if (!conf) {
+        goto out;
+    }
+
+    this->private = NULL;
+    GF_FREE(conf);
+
+out:
+    return;
+}
+
+struct xlator_fops fops = {
+    .writev = wb_writev,
+    .readv = wb_readv,
+    .flush = wb_flush,
+    .fsync = wb_fsync,
+    .stat = wb_stat,
+    .fstat = wb_fstat,
+    .truncate = wb_truncate,
+    .ftruncate = wb_ftruncate,
+    .setattr = wb_setattr,
+    .fsetattr = wb_fsetattr,
+    .lookup = wb_lookup,
+    .readdirp = wb_readdirp,
+    .link = wb_link,
+    .fallocate = wb_fallocate,
+    .discard = wb_discard,
+    .zerofill = wb_zerofill,
+    .rename = wb_rename,
 };
 
+struct xlator_cbks cbks = {.forget = wb_forget, .release = wb_release};
 
 struct xlator_dumpops dumpops = {
-        .priv      =  wb_priv_dump,
-        .inodectx  =  wb_inode_dump,
+    .priv = wb_priv_dump,
+    .inodectx = wb_inode_dump,
 };
 
-
 struct volume_options options[] = {
-        { .key  = {"flush-behind"},
-          .type = GF_OPTION_TYPE_BOOL,
-          .default_value = "on",
-          .description = "If this option is set ON, instructs write-behind "
-                          "translator to perform flush in background, by "
-                          "returning success (or any errors, if any of "
-                          "previous  writes were failed) to application even "
-                          "before flush FOP is sent to backend filesystem. "
-        },
-        { .key  = {"cache-size", "window-size"},
-          .type = GF_OPTION_TYPE_SIZET,
-          .min  = 512 * GF_UNIT_KB,
-          .max  = 1 * GF_UNIT_GB,
-          .default_value = "1MB",
-          .description = "Size of the write-behind buffer for a single file "
-                         "(inode)."
-        },
-        { .key = {"trickling-writes"},
-          .type = GF_OPTION_TYPE_BOOL,
-          .default_value = "on",
-        },
-        { .key = {"strict-O_DIRECT"},
-          .type = GF_OPTION_TYPE_BOOL,
-          .default_value = "off",
-          .description = "This option when set to off, ignores the "
-          "O_DIRECT flag."
-        },
-        { .key = {"strict-write-ordering"},
-          .type = GF_OPTION_TYPE_BOOL,
-          .default_value = "off",
-	  .description = "Do not let later writes overtake earlier writes even "
-	                  "if they do not overlap",
-        },
-        { .key = {NULL} },
+    {
+        .key = {"write-behind"},
+        .type = GF_OPTION_TYPE_BOOL,
+        .default_value = "off",
+        .description = "enable/disable write-behind",
+        .op_version = {GD_OP_VERSION_6_0},
+        .flags = OPT_FLAG_SETTABLE,
+    },
+    {.key = {"flush-behind"},
+     .type = GF_OPTION_TYPE_BOOL,
+     .default_value = "on",
+     .op_version = {1},
+     .flags = OPT_FLAG_SETTABLE | OPT_FLAG_DOC | OPT_FLAG_CLIENT_OPT,
+     .tags = {"write-behind"},
+     .description = "If this option is set ON, instructs write-behind "
+                    "translator to perform flush in background, by "
+                    "returning success (or any errors, if any of "
+                    "previous  writes were failed) to application even "
+                    "before flush FOP is sent to backend filesystem. "},
+    {.key = {"cache-size", "window-size"},
+     .type = GF_OPTION_TYPE_SIZET,
+     .min = 512 * GF_UNIT_KB,
+     .max = 1 * GF_UNIT_GB,
+     .default_value = "1MB",
+     .op_version = {1},
+     .flags = OPT_FLAG_SETTABLE | OPT_FLAG_DOC | OPT_FLAG_CLIENT_OPT,
+     .tags = {"write-behind"},
+     .description = "Size of the write-behind buffer for a single file "
+                    "(inode)."},
+    {
+        .key = {"trickling-writes"},
+        .type = GF_OPTION_TYPE_BOOL,
+        .op_version = {GD_OP_VERSION_3_13_1},
+        .flags = OPT_FLAG_SETTABLE | OPT_FLAG_DOC | OPT_FLAG_CLIENT_OPT,
+        .tags = {"write-behind"},
+        .default_value = "on",
+    },
+    {.key = {"strict-O_DIRECT"},
+     .type = GF_OPTION_TYPE_BOOL,
+     .default_value = "off",
+     .op_version = {2},
+     .flags = OPT_FLAG_SETTABLE | OPT_FLAG_DOC | OPT_FLAG_CLIENT_OPT,
+     .tags = {"write-behind"},
+     .description = "This option when set to off, ignores the "
+                    "O_DIRECT flag."},
+    {
+        .key = {"strict-write-ordering"},
+        .type = GF_OPTION_TYPE_BOOL,
+        .default_value = "off",
+        .op_version = {2},
+        .flags = OPT_FLAG_SETTABLE | OPT_FLAG_DOC | OPT_FLAG_CLIENT_OPT,
+        .tags = {"write-behind"},
+        .description = "Do not let later writes overtake earlier writes even "
+                       "if they do not overlap",
+    },
+    {
+        .key = {"resync-failed-syncs-after-fsync"},
+        .type = GF_OPTION_TYPE_BOOL,
+        .default_value = "off",
+        .op_version = {GD_OP_VERSION_3_7_7},
+        .flags = OPT_FLAG_SETTABLE | OPT_FLAG_DOC | OPT_FLAG_CLIENT_OPT,
+        .tags = {"write-behind"},
+        .description = "If sync of \"cached-writes issued before fsync\" "
+                       "(to backend) fails, this option configures whether "
+                       "to retry syncing them after fsync or forget them. "
+                       "If set to on, cached-writes are retried "
+                       "till a \"flush\" fop (or a successful sync) on sync "
+                       "failures. "
+                       "fsync itself is failed irrespective of the value of "
+                       "this option. ",
+    },
+    {
+        .key = {"aggregate-size"},
+        .type = GF_OPTION_TYPE_SIZET,
+        .default_value = "128KB",
+        .op_version = {GD_OP_VERSION_4_1_0},
+        .flags = OPT_FLAG_SETTABLE | OPT_FLAG_DOC | OPT_FLAG_CLIENT_OPT,
+        .description = "Will aggregate writes until data of specified "
+                       "size is fully filled for a single file provided "
+                       "there are no dependent fops on cached writes. This "
+                       "option just sets the aggregate size. Note that "
+                       "aggregation won't happen if "
+                       "performance.write-behind-trickling-writes"
+                       " is turned on. Hence turn off "
+                       "performance.write-behind.trickling-writes"
+                       " so that writes are aggregated till a max of "
+                       "\"aggregate-size\" bytes",
+    },
+    {.key = {NULL}},
+};
+
+xlator_api_t xlator_api = {
+    .init = init,
+    .fini = fini,
+    .reconfigure = reconfigure,
+    .mem_acct_init = mem_acct_init,
+    .op_version = {1}, /* Present from the initial version */
+    .dumpops = &dumpops,
+    .fops = &fops,
+    .cbks = &cbks,
+    .options = options,
+    .identifier = "write-behind",
+    .category = GF_MAINTAINED,
 };
diff --git a/xlators/cluster/ha/Makefile.am b/xlators/playground/rot-13/Makefile.am
index d471a3f9243..d471a3f9243 100644
--- a/xlators/cluster/ha/Makefile.am
+++ b/xlators/playground/rot-13/Makefile.am
diff --git a/xlators/encryption/rot-13/src/Makefile.am b/xlators/playground/rot-13/src/Makefile.am
index 94e8d18e789..9978661509d 100644
--- a/xlators/encryption/rot-13/src/Makefile.am
+++ b/xlators/playground/rot-13/src/Makefile.am
@@ -1,14 +1,15 @@
 xlator_LTLIBRARIES = rot-13.la
 xlatordir = $(libdir)/glusterfs/$(PACKAGE_VERSION)/xlator/encryption
 
-rot_13_la_LDFLAGS = -module -avoid-version
+rot_13_la_LDFLAGS = -module $(GF_XLATOR_DEFAULT_LDFLAGS)
 
 rot_13_la_SOURCES = rot-13.c
 rot_13_la_LIBADD = $(top_builddir)/libglusterfs/src/libglusterfs.la 
 
 noinst_HEADERS = rot-13.h
 
-AM_CPPFLAGS = $(GF_CPPFLAGS) -I$(top_srcdir)/libglusterfs/src
+AM_CPPFLAGS = $(GF_CPPFLAGS) -I$(top_srcdir)/libglusterfs/src \
+	-I$(top_srcdir)/rpc/xdr/src -I$(top_builddir)/rpc/xdr/src
 
 AM_CFLAGS = -Wall $(GF_CFLAGS)
 
diff --git a/xlators/playground/rot-13/src/rot-13.c b/xlators/playground/rot-13/src/rot-13.c
new file mode 100644
index 00000000000..0f45ee31964
--- /dev/null
+++ b/xlators/playground/rot-13/src/rot-13.c
@@ -0,0 +1,166 @@
+/*
+   Copyright (c) 2006-2012 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
+
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
+#include <ctype.h>
+#include <sys/uio.h>
+
+#include <glusterfs/glusterfs.h>
+#include <glusterfs/xlator.h>
+#include <glusterfs/logging.h>
+
+#include "rot-13.h"
+
+/*
+ * This is a rot13 ``encryption'' xlator. It rot13's data when
+ * writing to disk and rot13's it back when reading it.
+ * This xlator is meant as an example, NOT FOR PRODUCTION
+ * USE ;) (hence no error-checking)
+ */
+
+void
+rot13(char *buf, int len)
+{
+    int i;
+    for (i = 0; i < len; i++) {
+        if (buf[i] >= 'a' && buf[i] <= 'z')
+            buf[i] = 'a' + ((buf[i] - 'a' + 13) % 26);
+        else if (buf[i] >= 'A' && buf[i] <= 'Z')
+            buf[i] = 'A' + ((buf[i] - 'A' + 13) % 26);
+    }
+}
+
+void
+rot13_iovec(struct iovec *vector, int count)
+{
+    int i;
+    for (i = 0; i < count; i++) {
+        rot13(vector[i].iov_base, vector[i].iov_len);
+    }
+}
+
+int32_t
+rot13_readv_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                int32_t op_ret, int32_t op_errno, struct iovec *vector,
+                int32_t count, struct iatt *stbuf, struct iobref *iobref,
+                dict_t *xdata)
+{
+    rot_13_private_t *priv = (rot_13_private_t *)this->private;
+
+    if (priv->decrypt_read)
+        rot13_iovec(vector, count);
+
+    STACK_UNWIND_STRICT(readv, frame, op_ret, op_errno, vector, count, stbuf,
+                        iobref, xdata);
+    return 0;
+}
+
+int32_t
+rot13_readv(call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
+            off_t offset, uint32_t flags, dict_t *xdata)
+{
+    STACK_WIND(frame, rot13_readv_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->readv, fd, size, offset, flags, xdata);
+    return 0;
+}
+
+int32_t
+rot13_writev_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                 int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
+                 struct iatt *postbuf, dict_t *xdata)
+{
+    STACK_UNWIND_STRICT(writev, frame, op_ret, op_errno, prebuf, postbuf,
+                        xdata);
+    return 0;
+}
+
+int32_t
+rot13_writev(call_frame_t *frame, xlator_t *this, fd_t *fd,
+             struct iovec *vector, int32_t count, off_t offset, uint32_t flags,
+             struct iobref *iobref, dict_t *xdata)
+{
+    rot_13_private_t *priv = (rot_13_private_t *)this->private;
+    if (priv->encrypt_write)
+        rot13_iovec(vector, count);
+
+    STACK_WIND(frame, rot13_writev_cbk, FIRST_CHILD(this),
+               FIRST_CHILD(this)->fops->writev, fd, vector, count, offset,
+               flags, iobref, xdata);
+    return 0;
+}
+
+int32_t
+init(xlator_t *this)
+{
+    data_t *data = NULL;
+    rot_13_private_t *priv = NULL;
+
+    if (!this->children || this->children->next) {
+        gf_log("rot13", GF_LOG_ERROR,
+               "FATAL: rot13 should have exactly one child");
+        return -1;
+    }
+
+    if (!this->parents) {
+        gf_log(this->name, GF_LOG_WARNING, "dangling volume. check volfile ");
+    }
+
+    priv = GF_CALLOC(sizeof(rot_13_private_t), 1, 0);
+    if (!priv)
+        return -1;
+
+    priv->decrypt_read = 1;
+    priv->encrypt_write = 1;
+
+    data = dict_get(this->options, "encrypt-write");
+    if (data) {
+        if (gf_string2boolean(data->data, &priv->encrypt_write) == -1) {
+            gf_log(this->name, GF_LOG_ERROR,
+                   "encrypt-write takes only boolean options");
+            GF_FREE(priv);
+            return -1;
+        }
+    }
+
+    data = dict_get(this->options, "decrypt-read");
+    if (data) {
+        if (gf_string2boolean(data->data, &priv->decrypt_read) == -1) {
+            gf_log(this->name, GF_LOG_ERROR,
+                   "decrypt-read takes only boolean options");
+            GF_FREE(priv);
+            return -1;
+        }
+    }
+
+    this->private = priv;
+    gf_log("rot13", GF_LOG_DEBUG, "rot13 xlator loaded");
+    return 0;
+}
+
+void
+fini(xlator_t *this)
+{
+    rot_13_private_t *priv = this->private;
+
+    if (!priv)
+        return;
+    this->private = NULL;
+    GF_FREE(priv);
+
+    return;
+}
+
+struct xlator_fops fops = {.readv = rot13_readv, .writev = rot13_writev};
+
+struct xlator_cbks cbks;
+
+struct volume_options options[] = {
+    {.key = {"encrypt-write"}, .type = GF_OPTION_TYPE_BOOL},
+    {.key = {"decrypt-read"}, .type = GF_OPTION_TYPE_BOOL},
+    {.key = {NULL}},
+};
diff --git a/xlators/encryption/rot-13/src/rot-13.h b/xlators/playground/rot-13/src/rot-13.h
index 3e9fc19c776..edbc99798b4 100644
--- a/xlators/encryption/rot-13/src/rot-13.h
+++ b/xlators/playground/rot-13/src/rot-13.h
@@ -10,14 +10,9 @@
 #ifndef __ROT_13_H__
 #define __ROT_13_H__
 
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
-
 typedef struct {
-	gf_boolean_t encrypt_write;
-	gf_boolean_t decrypt_read;
+    gf_boolean_t encrypt_write;
+    gf_boolean_t decrypt_read;
 } rot_13_private_t;
 
 #endif /* __ROT_13_H__ */
diff --git a/xlators/playground/template/src/Makefile.am b/xlators/playground/template/src/Makefile.am
index 21f1c5f6b75..e76a717a550 100644
--- a/xlators/playground/template/src/Makefile.am
+++ b/xlators/playground/template/src/Makefile.am
@@ -1,14 +1,15 @@
 xlator_LTLIBRARIES = template.la
-xlatordir = $(libdir)/glusterfs/$(PACKAGE_VERSION)/xlator/testing/features
+xlatordir = $(libdir)/glusterfs/$(PACKAGE_VERSION)/xlator/playground
 
-template_la_LDFLAGS = -module -avoid-version
+template_la_LDFLAGS = -module $(GF_XLATOR_DEFAULT_LDFLAGS)
 
 template_la_SOURCES = template.c
 template_la_LIBADD = $(top_builddir)/libglusterfs/src/libglusterfs.la
 
 noinst_HEADERS = template.h
 
-AM_CPPFLAGS = $(GF_CPPFLAGS) -I$(top_srcdir)/libglusterfs/src
+AM_CPPFLAGS = $(GF_CPPFLAGS) -I$(top_srcdir)/libglusterfs/src \
+	-I$(top_srcdir)/rpc/xdr/src -I$(top_builddir)/rpc/xdr/src
 
 AM_CFLAGS = -Wall $(GF_CFLAGS)
 
diff --git a/xlators/playground/template/src/template.c b/xlators/playground/template/src/template.c
index 37a7794a07e..2f25d2363a6 100644
--- a/xlators/playground/template/src/template.c
+++ b/xlators/playground/template/src/template.c
@@ -1,5 +1,5 @@
 /*
-   Copyright (c) 2006-2012 Red Hat, Inc. <http://www.redhat.com>
+   Copyright (c) 2006-2018 Red Hat, Inc. <http://www.redhat.com>
    This file is part of GlusterFS.
 
    This file is licensed to you under your choice of the GNU Lesser
@@ -7,43 +7,180 @@
    later), or the GNU General Public License, version 2 (GPLv2), in all
    cases as published by the Free Software Foundation.
 */
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
-
 #include "template.h"
+#include <glusterfs/statedump.h>
 
-int32_t
-init (xlator_t *this)
+static int32_t
+template_mem_acct_init(xlator_t *this)
 {
+    int ret = -1;
+
+    GF_VALIDATE_OR_GOTO("template", this, out);
 
-        if (!this->children || this->children->next) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "not configured with exactly one child. exiting");
-                return -1;
-        }
+    ret = xlator_mem_acct_init(this, gf_template_mt_end + 1);
 
-        if (!this->parents) {
-                gf_log (this->name, GF_LOG_WARNING,
-                        "dangling volume. check volfile ");
-        }
+    if (ret != 0) {
+        gf_msg(this->name, GF_LOG_ERROR, ENOMEM, TEMPLATE_MSG_NO_MEMORY,
+               "Memory accounting init failed");
+        goto out;
+    }
 
-        return 0;
+    ret = 0;
+out:
+    return ret;
 }
 
-void
-fini (xlator_t *this)
+static int32_t
+template_priv_to_dict(xlator_t *this, dict_t *dict, char *brickname)
 {
-        return;
+    int ret = 0;
+    template_private_t *priv = NULL;
+
+    priv = this->private;
+    ret = dict_set_uint64(dict, "template.dummy", priv->dummy);
+    if (ret)
+        gf_msg_debug(this->name, ENOMEM, "dict_set of dummy key failed");
+
+    return 0;
 }
 
-struct xlator_fops fops = {
+static int32_t
+template_priv(xlator_t *this)
+{
+    template_private_t *priv = NULL;
+
+    priv = this->private;
+    gf_proc_dump_write("template.dummy", "%" PRId32, priv->dummy);
+
+    return 0;
+}
+
+static int32_t
+template_dump_metrics(xlator_t *this, int fd)
+{
+    template_private_t *priv = NULL;
+
+    priv = this->private;
+    /* NOTE: currently this is adding private variable, which can
+       be constant here. But in reality, things which are changing
+       can be added here, so we get to plot them on graph. */
+    dprintf(fd, "%s.private.dummy %d\n", this->name, priv->dummy);
+
+    return 0;
+}
+
+static int32_t
+template_init(xlator_t *this)
+{
+    int ret = -1;
+    template_private_t *priv = NULL;
+
+    if (!this->children || this->children->next) {
+        gf_msg(this->name, GF_LOG_ERROR, EINVAL, TEMPLATE_MSG_NO_GRAPH,
+               "not configured with exactly one child. exiting");
+        goto out;
+    }
+
+    if (!this->parents) {
+        gf_msg(this->name, GF_LOG_ERROR, EINVAL, TEMPLATE_MSG_NO_GRAPH,
+               "dangling volume. check volfile ");
+        goto out;
+    }
+
+    priv = GF_CALLOC(1, sizeof(template_private_t), gf_template_mt_private_t);
+    if (!priv) {
+        gf_msg(this->name, GF_LOG_ERROR, ENOMEM, TEMPLATE_MSG_NO_MEMORY,
+               "priv allocation failed");
+        goto out;
+    }
+
+    GF_OPTION_INIT("dummy", priv->dummy, int32, out);
+
+    this->private = priv;
+    priv = NULL;
+    ret = 0;
+
+out:
+    if (priv)
+        GF_FREE(priv);
+
+    return ret;
+}
+
+static int
+template_reconfigure(xlator_t *this, dict_t *options)
+{
+    int ret = -1;
+    template_private_t *priv = NULL;
+
+    priv = this->private;
+
+    GF_OPTION_RECONF("dummy", priv->dummy, options, int32, out);
+
+    ret = 0;
+out:
+    return ret;
+}
+
+static void
+template_fini(xlator_t *this)
+{
+    template_private_t *priv = NULL;
+
+    priv = this->private;
+    this->private = NULL;
+
+    GF_FREE(priv);
+}
+
+static int
+template_notify(xlator_t *this, int32_t event, void *data, ...)
+{
+    switch (event) {
+        default:
+            default_notify(this, event, data);
+            gf_msg_debug(this->name, 0, "event %d received", event);
+    }
+
+    return 0;
+}
+
+struct xlator_fops template_fops = {};
+
+struct xlator_cbks template_cbks = {};
+
+struct xlator_dumpops template_dumpops = {
+    .priv = template_priv,
+    .priv_to_dict = template_priv_to_dict,
 };
 
-struct xlator_cbks cbks = {
+struct volume_options template_options[] = {
+    {
+        .key = {"dummy"},
+        .type = GF_OPTION_TYPE_INT,
+        .min = 1,
+        .max = 1024,
+        .default_value = "1",
+        .description = "a dummy option to show how to set the option",
+        .op_version = {GD_OP_VERSION_5_0},
+        .flags = OPT_FLAG_SETTABLE | OPT_FLAG_DOC,
+        .level = OPT_STATUS_EXPERIMENTAL,
+        .tags = {"development", "experimental", "template"},
+    },
+    {.key = {NULL}},
 };
 
-struct volume_options options[] = {
-        { .key  = {NULL} },
+xlator_api_t xlator_api = {
+    .init = template_init,
+    .fini = template_fini,
+    .notify = template_notify,
+    .reconfigure = template_reconfigure,
+    .mem_acct_init = template_mem_acct_init,
+    .dump_metrics = template_dump_metrics,
+    .op_version = {GD_OP_VERSION_5_0},
+    .dumpops = &template_dumpops,
+    .fops = &template_fops,
+    .cbks = &template_cbks,
+    .options = template_options,
+    .identifier = "template",
 };
diff --git a/xlators/playground/template/src/template.h b/xlators/playground/template/src/template.h
index d6aced50107..c53dc1c7010 100644
--- a/xlators/playground/template/src/template.h
+++ b/xlators/playground/template/src/template.h
@@ -1,5 +1,5 @@
 /*
-   Copyright (c) 2013 Red Hat, Inc. <http://www.redhat.com>
+   Copyright (c) 2013-2018 Red Hat, Inc. <http://www.redhat.com>
    This file is part of GlusterFS.
 
    This file is licensed to you under your choice of the GNU Lesser
@@ -10,15 +10,34 @@
 #ifndef __TEMPLATE_H__
 #define __TEMPLATE_H__
 
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
+#include <glusterfs/glusterfs.h>
+#include <glusterfs/logging.h>
+#include <glusterfs/dict.h>
+#include <glusterfs/xlator.h>
+#include <glusterfs/defaults.h>
 
-#include "glusterfs.h"
-#include "logging.h"
-#include "dict.h"
-#include "xlator.h"
-#include "defaults.h"
+struct template_private {
+    /* Add all the relevant fields you need here */
+    int32_t dummy;
+};
+
+typedef struct template_private template_private_t;
+
+/* Below section goes to template-mem-types.h */
+#include <glusterfs/mem-types.h>
+
+enum gf_template_mem_types_ {
+    gf_template_mt_private_t = gf_common_mt_end + 1,
+    gf_template_mt_end,
+};
+
+/* This normally goes to another file 'template-messages.h",
+   required for 'gf_msg()'.
+   NOTE: make sure you have added your component (in this case,
+   TEMPLATE) in `libglusterfs/src/glfs-message-id.h`.
+ */
+#include <glusterfs/glfs-message-id.h>
+
+GLFS_MSGID(TEMPLATE, TEMPLATE_MSG_NO_MEMORY, TEMPLATE_MSG_NO_GRAPH);
 
 #endif /* __TEMPLATE_H__ */
diff --git a/xlators/protocol/auth/addr/src/Makefile.am b/xlators/protocol/auth/addr/src/Makefile.am
index 426e7c2fb36..4694d254f12 100644
--- a/xlators/protocol/auth/addr/src/Makefile.am
+++ b/xlators/protocol/auth/addr/src/Makefile.am
@@ -1,14 +1,14 @@
 auth_LTLIBRARIES = addr.la
 authdir = $(libdir)/glusterfs/$(PACKAGE_VERSION)/auth
 
-addr_la_LDFLAGS = -module -avoid-version
+addr_la_LDFLAGS = -module $(GF_XLATOR_LDFLAGS)
 
 addr_la_SOURCES = addr.c
 addr_la_LIBADD = $(top_builddir)/libglusterfs/src/libglusterfs.la
 
-AM_CPPFLAGS = $(GF_CPPFLAGS) \
-	-I$(top_srcdir)/libglusterfs/src \
+AM_CPPFLAGS = $(GF_CPPFLAGS) -I$(top_srcdir)/libglusterfs/src \
 	-I$(top_srcdir)/xlators/protocol/server/src \
+	-I$(top_srcdir)/rpc/xdr/src/ -I$(top_builddir)/rpc/xdr/src/ \
 	-I$(top_srcdir)/rpc/rpc-lib/src/
 
 AM_CFLAGS = -Wall $(GF_CFLAGS)
diff --git a/xlators/protocol/auth/addr/src/addr.c b/xlators/protocol/auth/addr/src/addr.c
index 181d091bd49..bf12c455d7c 100644
--- a/xlators/protocol/auth/addr/src/addr.c
+++ b/xlators/protocol/auth/addr/src/addr.c
@@ -8,223 +8,334 @@
   cases as published by the Free Software Foundation.
 */
 
-
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
-
 #include <fnmatch.h>
 #include <sys/socket.h>
 #include <netdb.h>
 #include "authenticate.h"
-#include "dict.h"
+#include <glusterfs/dict.h>
 #include "rpc-transport.h"
 
-#define ADDR_DELIMITER " ,"
+#define ENTRY_DELIMITER ","
+#define ADDR_DELIMITER "|"
 #define PRIVILEGED_PORT_CEILING 1024
 
 #ifndef AF_INET_SDP
 #define AF_INET_SDP 27
 #endif
 
-auth_result_t
-gf_auth (dict_t *input_params, dict_t *config_params)
+/* An option for subdir validation be like below */
+
+/* 1. '*'
+   2. '192.168.*'
+   3. '
+   4. '!10.10.1*' (Today as per the code, if negate is set on one entry, its
+   never reset)
+   5. '192.168.1.*, 10.1.10.*';168.168.2.* =/dir;* =/another-dir'
+
+*/
+
+int
+compare_addr_and_update(char *option_str, char *peer_addr, char *subvol,
+                        char *delimiter, auth_result_t *result,
+                        auth_result_t status)
 {
-        auth_result_t  result         = AUTH_DONT_CARE;
-        int            ret            = 0;
-        char          *name           = NULL;
-        char          *searchstr      = NULL;
-        peer_info_t   *peer_info      = NULL;
-        data_t        *peer_info_data = NULL;
-        data_t        *allow_addr     = NULL;
-        data_t        *reject_addr    = NULL;
-        char          *addr_str       = NULL;
-        char          *tmp            = NULL;
-        char          *addr_cpy       = NULL;
-        char          *service        = NULL;
-        uint16_t       peer_port      = 0;
-        char           is_inet_sdp    = 0;
-        char           negate         = 0;
-        char           match          = 0;
-        char           peer_addr[UNIX_PATH_MAX];
-        char          *type           = NULL;
-        gf_boolean_t   allow_insecure = _gf_false;
-
-        name = data_to_str (dict_get (input_params, "remote-subvolume"));
-        if (!name) {
-                gf_log ("authenticate/addr", GF_LOG_DEBUG,
-                        "remote-subvolume not specified");
-                goto out;
+    char *addr_str = NULL;
+    char *tmp = NULL;
+    char negate = 0;
+    char match = 0;
+    int length = 0;
+    int ret = 0;
+
+    addr_str = strtok_r(option_str, delimiter, &tmp);
+
+    while (addr_str) {
+        gf_log(subvol, GF_LOG_INFO, "%s = \"%s\", received addr = \"%s\"",
+               (status == AUTH_ACCEPT) ? "allowed" : "rejected", addr_str,
+               peer_addr);
+        if (addr_str[0] == '!') {
+            negate = 1;
+            addr_str++;
         }
 
-        ret = gf_asprintf (&searchstr, "auth.addr.%s.allow", name);
-        if (-1 == ret) {
-                gf_log ("auth/addr", GF_LOG_DEBUG,
-                        "asprintf failed while setting search string");
+        length = strlen(addr_str);
+        if ((addr_str[0] != '*') && valid_host_name(addr_str, length)) {
+            match = gf_is_same_address(addr_str, peer_addr);
+            if (match) {
+                *result = status;
                 goto out;
+            }
+        } else {
+            if (strstr(addr_str, "/")) {
+                match = gf_is_ip_in_net(addr_str, peer_addr);
+                if (negate ? !match : match) {
+                    *result = status;
+                    goto out;
+                }
+            } else {
+                match = fnmatch(addr_str, peer_addr, 0);
+                if (negate ? match : !match) {
+                    *result = status;
+                    goto out;
+                }
+            }
         }
 
-        allow_addr = dict_get (config_params, searchstr);
-        GF_FREE (searchstr);
+        addr_str = strtok_r(NULL, delimiter, &tmp);
+    }
 
-        ret = gf_asprintf (&searchstr, "auth.addr.%s.reject", name);
-        if (-1 == ret) {
-                gf_log ("auth/addr", GF_LOG_ERROR,
-                        "asprintf failed while setting search string");
-                goto out;
-        }
-        reject_addr = dict_get (config_params, searchstr);
-        GF_FREE (searchstr);
-
-        if (!allow_addr) {
-                /* TODO: backword compatibility */
-                ret = gf_asprintf (&searchstr, "auth.ip.%s.allow", name);
-                if (-1 == ret) {
-                        gf_log ("auth/addr", GF_LOG_ERROR,
-                                "asprintf failed while setting search string");
-                        goto out;
-                }
-                allow_addr = dict_get (config_params, searchstr);
-                GF_FREE (searchstr);
-        }
+    ret = -1;
+out:
+    return ret;
+}
 
-        if (!(allow_addr || reject_addr)) {
-                gf_log ("auth/addr",  GF_LOG_DEBUG,
-                        "none of the options auth.addr.%s.allow or "
-                        "auth.addr.%s.reject specified, returning auth_dont_care",
-                        name, name);
-                goto out;
+void
+parse_entries_and_compare(char *option_str, char *peer_addr, char *subvol,
+                          char *subdir, auth_result_t *result,
+                          auth_result_t status)
+{
+    char *entry = NULL;
+    char *entry_cpy = NULL;
+    char *directory = NULL;
+    char *entries = NULL;
+    char *addr_str = NULL;
+    char *addr = NULL;
+    char *tmp = NULL;
+    char *tmpdir = NULL;
+    int ret = 0;
+
+    if (!subdir) {
+        gf_log(subvol, GF_LOG_WARNING,
+               "subdir entry not present, not performing any operation.");
+        goto out;
+    }
+
+    entries = gf_strdup(option_str);
+    if (!entries)
+        goto out;
+
+    if (entries[0] != '/' && !strchr(entries, '(')) {
+        /* Backward compatible option */
+        ret = compare_addr_and_update(entries, peer_addr, subvol, ",", result,
+                                      status);
+        goto out;
+    }
+
+    entry = strtok_r(entries, ENTRY_DELIMITER, &tmp);
+    while (entry) {
+        entry_cpy = gf_strdup(entry);
+        if (!entry_cpy) {
+            goto out;
         }
 
-        peer_info_data = dict_get (input_params, "peer-info");
-        if (!peer_info_data) {
-                gf_log ("auth/addr", GF_LOG_ERROR,
-                        "peer-info not present");
-                goto out;
+        directory = strtok_r(entry_cpy, "(", &tmpdir);
+        if (directory[0] != '/')
+            goto out;
+
+        /* send second portion, after ' =' if directory matches */
+        if (strcmp(subdir, directory))
+            goto next_entry;
+
+        addr_str = strtok_r(NULL, ")", &tmpdir);
+        if (!addr_str)
+            goto out;
+
+        addr = gf_strdup(addr_str);
+        if (!addr)
+            goto out;
+
+        gf_log(subvol, GF_LOG_INFO,
+               "Found an entry for dir %s (%s),"
+               " performing validation",
+               subdir, addr);
+
+        ret = compare_addr_and_update(addr, peer_addr, subvol, ADDR_DELIMITER,
+                                      result, status);
+        if (ret == 0) {
+            break;
         }
 
-        peer_info = data_to_ptr (peer_info_data);
+        GF_FREE(addr);
+        addr = NULL;
 
-        switch (((struct sockaddr *) &peer_info->sockaddr)->sa_family)
-        {
-        case AF_INET_SDP:
-                is_inet_sdp = 1;
-                ((struct sockaddr *) &peer_info->sockaddr)->sa_family = AF_INET;
+    next_entry:
+        entry = strtok_r(NULL, ENTRY_DELIMITER, &tmp);
+        GF_FREE(entry_cpy);
+        entry_cpy = NULL;
+    }
 
+out:
+    GF_FREE(entries);
+    GF_FREE(entry_cpy);
+    GF_FREE(addr);
+}
+
+auth_result_t
+gf_auth(dict_t *input_params, dict_t *config_params)
+{
+    auth_result_t result = AUTH_DONT_CARE;
+    int ret = 0;
+    char *name = NULL;
+    char *searchstr = NULL;
+    peer_info_t *peer_info = NULL;
+    data_t *peer_info_data = NULL;
+    data_t *allow_addr = NULL;
+    data_t *reject_addr = NULL;
+    char *service = NULL;
+    uint16_t peer_port = 0;
+    char peer_addr[UNIX_PATH_MAX] = {
+        0,
+    };
+    char *type = NULL;
+    gf_boolean_t allow_insecure = _gf_false;
+    char *subdir = NULL;
+
+    name = data_to_str(dict_get(input_params, "remote-subvolume"));
+    if (!name) {
+        gf_log("authenticate/addr", GF_LOG_DEBUG,
+               "remote-subvolume not specified");
+        goto out;
+    }
+
+    ret = gf_asprintf(&searchstr, "auth.addr.%s.allow", name);
+    if (-1 == ret) {
+        gf_log("auth/addr", GF_LOG_DEBUG,
+               "asprintf failed while setting search string");
+        goto out;
+    }
+
+    allow_addr = dict_get(config_params, searchstr);
+    GF_FREE(searchstr);
+
+    ret = gf_asprintf(&searchstr, "auth.addr.%s.reject", name);
+    if (-1 == ret) {
+        gf_log("auth/addr", GF_LOG_ERROR,
+               "asprintf failed while setting search string");
+        goto out;
+    }
+    reject_addr = dict_get(config_params, searchstr);
+    GF_FREE(searchstr);
+
+    if (!allow_addr) {
+        /* TODO: backward compatibility */
+        ret = gf_asprintf(&searchstr, "auth.ip.%s.allow", name);
+        if (-1 == ret) {
+            gf_log("auth/addr", GF_LOG_ERROR,
+                   "asprintf failed while setting search string");
+            goto out;
+        }
+        allow_addr = dict_get(config_params, searchstr);
+        GF_FREE(searchstr);
+    }
+
+    if (!(allow_addr || reject_addr)) {
+        gf_log("auth/addr", GF_LOG_DEBUG,
+               "none of the options auth.addr.%s.allow or "
+               "auth.addr.%s.reject specified, returning auth_dont_care",
+               name, name);
+        goto out;
+    }
+
+    peer_info_data = dict_get(input_params, "peer-info");
+    if (!peer_info_data) {
+        gf_log("auth/addr", GF_LOG_ERROR, "peer-info not present");
+        goto out;
+    }
+
+    ret = dict_get_str(input_params, "subdir-mount", &subdir);
+    if (ret) {
+        subdir = "/";
+    }
+
+    peer_info = data_to_ptr(peer_info_data);
+
+    switch (((struct sockaddr *)&peer_info->sockaddr)->sa_family) {
+        case AF_INET_SDP:
         case AF_INET:
         case AF_INET6:
-        {
-                strcpy (peer_addr, peer_info->identifier);
-                service = strrchr (peer_addr, ':');
-                *service = '\0';
-                service ++;
-
-                if (is_inet_sdp) {
-                        ((struct sockaddr *) &peer_info->sockaddr)->sa_family = AF_INET_SDP;
+            strcpy(peer_addr, peer_info->identifier);
+            service = strrchr(peer_addr, ':');
+            *service = '\0';
+            service++;
+
+            ret = dict_get_str(config_params, "rpc-auth-allow-insecure", &type);
+            if (ret == 0) {
+                ret = gf_string2boolean(type, &allow_insecure);
+                if (ret < 0) {
+                    gf_log("auth/addr", GF_LOG_WARNING,
+                           "rpc-auth-allow-insecure option %s "
+                           "is not a valid bool option",
+                           type);
+                    goto out;
                 }
-
-                ret = dict_get_str (config_params, "rpc-auth-allow-insecure",
-                                    &type);
-                if (ret == 0) {
-                        ret = gf_string2boolean (type, &allow_insecure);
-                        if (ret < 0) {
-                                gf_log ("auth/addr", GF_LOG_WARNING,
-                                        "rpc-auth-allow-insecure option %s "
-                                        "is not a valid bool option", type);
-                                goto out;
-                        }
-                }
-
-                peer_port = atoi (service);
-                if (peer_port >= PRIVILEGED_PORT_CEILING && !allow_insecure) {
-                        gf_log ("auth/addr", GF_LOG_ERROR,
-                                "client is bound to port %d which is not privileged",
-                                peer_port);
-                        goto out;
-                }
-                break;
+            }
+
+            peer_port = atoi(service);
+            if (peer_port >= PRIVILEGED_PORT_CEILING && !allow_insecure) {
+                gf_log("auth/addr", GF_LOG_ERROR,
+                       "client is bound to port %d which is not privileged",
+                       peer_port);
+                result = AUTH_REJECT;
+                goto out;
+            }
+            break;
 
         case AF_UNIX:
-                strcpy (peer_addr, peer_info->identifier);
-                break;
+            strcpy(peer_addr, peer_info->identifier);
+            break;
 
         default:
-                gf_log ("authenticate/addr", GF_LOG_ERROR,
-                        "unknown address family %d",
-                        ((struct sockaddr *) &peer_info->sockaddr)->sa_family);
-                goto out;
-        }
-        }
-
-        if (reject_addr) {
-                addr_cpy = gf_strdup (reject_addr->data);
-                if (!addr_cpy)
-                        goto out;
-
-                addr_str = strtok_r (addr_cpy, ADDR_DELIMITER, &tmp);
-
-                while (addr_str) {
-                        gf_log (name,  GF_LOG_DEBUG,
-                                "rejected = \"%s\", received addr = \"%s\"",
-                                addr_str, peer_addr);
-                        if (addr_str[0] == '!') {
-                                negate = 1;
-                                addr_str++;
-                        }
-
-                        match = fnmatch (addr_str, peer_addr, 0);
-                        if (negate ? match : !match) {
-                                result = AUTH_REJECT;
-                                goto out;
-                        }
-                        addr_str = strtok_r (NULL, ADDR_DELIMITER, &tmp);
-                }
-                GF_FREE (addr_cpy);
-                addr_cpy = NULL;
-        }
-
-        if (allow_addr) {
-                addr_cpy = gf_strdup (allow_addr->data);
-                if (!addr_cpy)
-                        goto out;
-
-                addr_str = strtok_r (addr_cpy, ADDR_DELIMITER, &tmp);
-
-                while (addr_str) {
-                        gf_log (name,  GF_LOG_DEBUG,
-                                "allowed = \"%s\", received addr = \"%s\"",
-                                addr_str, peer_addr);
-                        if (addr_str[0] == '!') {
-                                negate = 1;
-                                addr_str++;
-                        }
-
-                        match = fnmatch (addr_str, peer_addr, 0);
-                        if (negate ? match : !match) {
-                                result = AUTH_ACCEPT;
-                                goto out;
-                        }
-                        addr_str = strtok_r (NULL, ADDR_DELIMITER, &tmp);
-                }
-        }
+            gf_log("authenticate/addr", GF_LOG_ERROR,
+                   "unknown address family %d",
+                   ((struct sockaddr *)&peer_info->sockaddr)->sa_family);
+            goto out;
+    }
+
+    if (reject_addr) {
+        parse_entries_and_compare(reject_addr->data, peer_addr, name, subdir,
+                                  &result, AUTH_REJECT);
+        if (result == AUTH_REJECT)
+            goto out;
+    }
+
+    if (allow_addr) {
+        parse_entries_and_compare(allow_addr->data, peer_addr, name, subdir,
+                                  &result, AUTH_ACCEPT);
+    }
 
 out:
-        GF_FREE (addr_cpy);
-
-        return result;
+    return result;
 }
 
 struct volume_options options[] = {
-        { .key   = {"auth.addr.*.allow"},
-          .type  = GF_OPTION_TYPE_INTERNET_ADDRESS_LIST
-        },
-        { .key   = {"auth.addr.*.reject"},
-          .type  = GF_OPTION_TYPE_INTERNET_ADDRESS_LIST
-        },
-        /* Backword compatibility */
-        { .key   = {"auth.ip.*.allow"},
-          .type  = GF_OPTION_TYPE_INTERNET_ADDRESS_LIST
-        },
-        { .key = {NULL} }
-};
+    {
+        .key = {"auth.addr.*.allow"},
+        .type = GF_OPTION_TYPE_INTERNET_ADDRESS_LIST,
+        .default_value = "*",
+        .description = "List of addresses to be allowed to access volume",
+        .op_version = {1},
+        .flags = OPT_FLAG_SETTABLE | OPT_FLAG_DOC,
+        .tags = {},
+        /* option_validation_fn validate_fn; */
+    },
+    {
+        .key = {"auth.addr.*.reject"},
+        .type = GF_OPTION_TYPE_INTERNET_ADDRESS_LIST,
+        .default_value = "*",
+        .description = "List of addresses to be rejected to access volume",
+        .op_version = {1},
+        .flags = OPT_FLAG_SETTABLE | OPT_FLAG_DOC,
+        .tags = {},
+        /* option_validation_fn validate_fn; */
+    },
+    /* Backward compatibility */
+    {
+        .key = {"auth.ip.*.allow"},
+        .type = GF_OPTION_TYPE_INTERNET_ADDRESS_LIST,
+        .default_value = "*",
+        .description = "List of addresses to be allowed to access volume",
+        .op_version = {1},
+        .flags = OPT_FLAG_SETTABLE | OPT_FLAG_DOC,
+        .tags = {},
+        /* option_validation_fn validate_fn; */
+    },
+    {.key = {NULL}}};
diff --git a/xlators/protocol/auth/login/src/Makefile.am b/xlators/protocol/auth/login/src/Makefile.am
index d84db91c4e1..9837437b11e 100644
--- a/xlators/protocol/auth/login/src/Makefile.am
+++ b/xlators/protocol/auth/login/src/Makefile.am
@@ -1,12 +1,13 @@
 auth_LTLIBRARIES = login.la
 authdir = $(libdir)/glusterfs/$(PACKAGE_VERSION)/auth
 
-login_la_LDFLAGS = -module -avoid-version
+login_la_LDFLAGS = -module $(GF_XLATOR_LDFLAGS)
 
 login_la_SOURCES = login.c
 login_la_LIBADD = $(top_builddir)/libglusterfs/src/libglusterfs.la
 
 AM_CPPFLAGS = $(GF_CPPFLAGS) -I$(top_srcdir)/libglusterfs/src \
-	-I$(top_srcdir)/xlators/protocol/server/src
+	-I$(top_srcdir)/xlators/protocol/server/src \
+	-I$(top_srcdir)/rpc/xdr/src -I$(top_builddir)/rpc/xdr/src
 
 AM_CFLAGS = -Wall $(GF_CFLAGS)
diff --git a/xlators/protocol/auth/login/src/login.c b/xlators/protocol/auth/login/src/login.c
index b53c5ccba21..64521267bfe 100644
--- a/xlators/protocol/auth/login/src/login.c
+++ b/xlators/protocol/auth/login/src/login.c
@@ -8,162 +8,203 @@
   cases as published by the Free Software Foundation.
 */
 
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
-
 #include <fnmatch.h>
 #include "authenticate.h"
 
-auth_result_t gf_auth (dict_t *input_params, dict_t *config_params)
+/* Note on strict_auth
+ * - Strict auth kicks in when authentication is using the username, password
+ *   in the volfile to login
+ * - If enabled, auth is rejected if the username and password is not matched
+ *   or is not present
+ * - When using SSL names, this is automatically strict, and allows only those
+ *   names that are present in the allow list, IOW strict auth checking has no
+ *   implication when using SSL names
+ */
+
+auth_result_t
+gf_auth(dict_t *input_params, dict_t *config_params)
 {
-        auth_result_t  result  = AUTH_DONT_CARE;
-        int             ret             = 0;
-        data_t          *allow_user     = NULL;
-        data_t          *username_data  = NULL;
-        data_t          *passwd_data    = NULL;
-        data_t          *password_data  = NULL;
-        char            *username       = NULL;
-        char            *password       = NULL;
-        char            *brick_name     = NULL;
-        char            *searchstr      = NULL;
-        char            *username_str   = NULL;
-        char            *tmp            = NULL;
-        char            *username_cpy   = NULL;
-        gf_boolean_t    using_ssl       = _gf_false;
-
-        username_data = dict_get (input_params, "ssl-name");
-        if (username_data) {
-                gf_log ("auth/login", GF_LOG_INFO,
-                        "connecting user name: %s", username_data->data);
-                using_ssl = _gf_true;
-        }
-        else {
-                username_data = dict_get (input_params, "username");
-                if (!username_data) {
-                        gf_log ("auth/login", GF_LOG_DEBUG,
-                                "username not found, returning DONT-CARE");
-                        goto out;
-                }
-                password_data = dict_get (input_params, "password");
-                if (!password_data) {
-                        gf_log ("auth/login", GF_LOG_WARNING,
-                                "password not found, returning DONT-CARE");
-                        goto out;
-                }
-                password = data_to_str (password_data);
+    auth_result_t result = AUTH_DONT_CARE;
+    int ret = 0;
+    data_t *allow_user = NULL;
+    data_t *username_data = NULL;
+    data_t *passwd_data = NULL;
+    data_t *password_data = NULL;
+    char *username = NULL;
+    char *password = NULL;
+    char *brick_name = NULL;
+    char *searchstr = NULL;
+    char *username_str = NULL;
+    char *tmp = NULL;
+    char *username_cpy = NULL;
+    gf_boolean_t using_ssl = _gf_false;
+    gf_boolean_t strict_auth = _gf_false;
+
+    username_data = dict_get(input_params, "ssl-name");
+    if (username_data) {
+        gf_log("auth/login", GF_LOG_INFO, "connecting user name: %s",
+               username_data->data);
+        using_ssl = _gf_true;
+    } else {
+        ret = dict_get_str_boolean(config_params, "strict-auth-accept",
+                                   _gf_false);
+        if (ret == -1)
+            strict_auth = _gf_false;
+        else
+            strict_auth = ret;
+
+        username_data = dict_get(input_params, "username");
+        if (!username_data) {
+            if (strict_auth) {
+                gf_log("auth/login", GF_LOG_DEBUG,
+                       "username not found, strict auth"
+                       " configured returning REJECT");
+                result = AUTH_REJECT;
+            } else {
+                gf_log("auth/login", GF_LOG_DEBUG,
+                       "username not found, returning"
+                       " DONT-CARE");
+            }
+            goto out;
         }
-        username = data_to_str (username_data);
-
-        brick_name = data_to_str (dict_get (input_params, "remote-subvolume"));
-        if (!brick_name) {
-                gf_log ("auth/login", GF_LOG_ERROR,
-                        "remote-subvolume not specified");
+        password_data = dict_get(input_params, "password");
+        if (!password_data) {
+            if (strict_auth) {
+                gf_log("auth/login", GF_LOG_DEBUG,
+                       "password not found, strict auth"
+                       " configured returning REJECT");
                 result = AUTH_REJECT;
-                goto out;
+            } else {
+                gf_log("auth/login", GF_LOG_WARNING,
+                       "password not found, returning"
+                       " DONT-CARE");
+            }
+            goto out;
         }
-
-        ret = gf_asprintf (&searchstr, "auth.login.%s.%s", brick_name,
-                           using_ssl ? "ssl-allow" : "allow");
-        if (-1 == ret) {
-                gf_log ("auth/login", GF_LOG_WARNING,
-                        "asprintf failed while setting search string, "
-                        "returning DONT-CARE");
-                goto out;
+        password = data_to_str(password_data);
+    }
+    username = data_to_str(username_data);
+
+    brick_name = data_to_str(dict_get(input_params, "remote-subvolume"));
+    if (!brick_name) {
+        gf_log("auth/login", GF_LOG_ERROR, "remote-subvolume not specified");
+        result = AUTH_REJECT;
+        goto out;
+    }
+
+    ret = gf_asprintf(&searchstr, "auth.login.%s.%s", brick_name,
+                      using_ssl ? "ssl-allow" : "allow");
+    if (-1 == ret) {
+        gf_log("auth/login", GF_LOG_ERROR,
+               "asprintf failed while setting search string, "
+               "returning REJECT");
+        result = AUTH_REJECT;
+        goto out;
+    }
+
+    allow_user = dict_get(config_params, searchstr);
+    GF_FREE(searchstr);
+
+    if (allow_user) {
+        gf_log("auth/login", GF_LOG_INFO, "allowed user names: %s",
+               allow_user->data);
+        /*
+         * There's a subtle difference between SSL and non-SSL behavior
+         * if we can't match anything in the "while" loop below.
+         * Intuitively, we should AUTH_REJECT if there's no match.
+         * However, existing code depends on allowing untrusted users
+         * to connect with *no credentials at all* by falling through
+         * the loop.  They're still distinguished from trusted users
+         * who do provide a valid username and password (in fact that's
+         * pretty much the only thing we use non-SSL login auth for),
+         * but they are allowed to connect.  It's wrong, but it's not
+         * worth changing elsewhere.  Therefore, we do the sane thing
+         * only for SSL here.
+         *
+         * For SSL, if there's a list *you must be on it*.  Note that
+         * if there's no list we don't care.  In that case (and the
+         * ssl-allow=* case as well) authorization is effectively
+         * disabled, though authentication and encryption are still
+         * active.
+         *
+         * Read NOTE on strict_auth above.
+         */
+        if (using_ssl || strict_auth) {
+            result = AUTH_REJECT;
         }
-
-        allow_user = dict_get (config_params, searchstr);
-        GF_FREE (searchstr);
-
-        if (allow_user) {
-                gf_log ("auth/login", GF_LOG_INFO,
-                        "allowed user names: %s", allow_user->data);
-                /*
-                 * There's a subtle difference between SSL and non-SSL behavior
-                 * if we can't match anything in the "while" loop below.
-                 * Intuitively, we should AUTH_REJECT if there's no match.
-                 * However, existing code depends on allowing untrusted users
-                 * to connect with *no credentials at all* by falling through
-                 * the loop.  They're still distinguished from trusted users
-                 * who do provide a valid username and password (in fact that's
-                 * pretty much the only thing we use non-SSL login auth for),
-                 * but they are allowed to connect.  It's wrong, but it's not
-                 * worth changing elsewhere.  Therefore, we do the sane thing
-                 * only for SSL here.
-                 *
-                 * For SSL, if there's a list *you must be on it*.  Note that
-                 * if there's no list we don't care.  In that case (and the
-                 * ssl-allow=* case as well) authorization is effectively
-                 * disabled, though authentication and encryption are still
-                 * active.
-                 */
+        username_cpy = gf_strdup(allow_user->data);
+        if (!username_cpy)
+            goto out;
+
+        username_str = strtok_r(username_cpy, " ,", &tmp);
+
+        /*
+         * We have to match a user's *authenticated* name to one in the
+         * list.  If we're using SSL, they're already authenticated.
+         * Otherwise, they need a matching password to complete the
+         * process.
+         */
+        while (username_str) {
+            if (!fnmatch(username_str, username, 0)) {
                 if (using_ssl) {
-                        result = AUTH_REJECT;
+                    result = AUTH_ACCEPT;
+                    break;
+                }
+                ret = gf_asprintf(&searchstr, "auth.login.%s.password",
+                                  username);
+                if (-1 == ret) {
+                    gf_log("auth/login", GF_LOG_WARNING,
+                           "asprintf failed while setting search string");
+                    goto out;
                 }
-                username_cpy = gf_strdup (allow_user->data);
-                if (!username_cpy)
-                        goto out;
-
-                username_str = strtok_r (username_cpy, " ,", &tmp);
-
-                /*
-                 * We have to match a user's *authenticated* name to one in the
-                 * list.  If we're using SSL, they're already authenticated.
-                 * Otherwise, they need a matching password to complete the
-                 * process.
-                 */
-                while (username_str) {
-                        if (!fnmatch (username_str, username, 0)) {
-                                if (using_ssl) {
-                                        result = AUTH_ACCEPT;
-                                        break;
-                                }
-                                ret = gf_asprintf (&searchstr,
-                                                   "auth.login.%s.password",
-                                                   username);
-                                if (-1 == ret) {
-                                        gf_log ("auth/login", GF_LOG_WARNING,
-                                                "asprintf failed while setting search string");
-                                        goto out;
-                                }
-                                passwd_data = dict_get (config_params, searchstr);
-                                GF_FREE (searchstr);
-
-                                if (!passwd_data) {
-                                        gf_log ("auth/login", GF_LOG_ERROR,
-                                                "wrong username/password combination");
-                                        result = AUTH_REJECT;
-                                        goto out;
-                                }
-
-                                result = !((strcmp (data_to_str (passwd_data),
-                                                    password)) ?
-                                           AUTH_ACCEPT :
-                                           AUTH_REJECT);
-                                if (result == AUTH_REJECT)
-                                        gf_log ("auth/login", GF_LOG_ERROR,
-                                                "wrong password for user %s",
-                                                username);
-
-                                break;
-                        }
-                        username_str = strtok_r (NULL, " ,", &tmp);
+                passwd_data = dict_get(config_params, searchstr);
+                GF_FREE(searchstr);
+
+                if (!passwd_data) {
+                    gf_log("auth/login", GF_LOG_ERROR,
+                           "wrong username/password combination");
+                    result = AUTH_REJECT;
+                    goto out;
                 }
+
+                result = !((strcmp(data_to_str(passwd_data), password))
+                               ? AUTH_ACCEPT
+                               : AUTH_REJECT);
+                if (result == AUTH_REJECT)
+                    gf_log("auth/login", GF_LOG_ERROR,
+                           "wrong password for user %s", username);
+
+                break;
+            }
+            username_str = strtok_r(NULL, " ,", &tmp);
         }
+    }
 
 out:
-        GF_FREE (username_cpy);
+    GF_FREE(username_cpy);
 
-        return result;
+    return result;
 }
 
 struct volume_options options[] = {
-        { .key   = {"auth.login.*.allow"},
-          .type  = GF_OPTION_TYPE_ANY
-        },
-        { .key   = {"auth.login.*.password"},
-          .type  = GF_OPTION_TYPE_ANY
-        },
-        { .key = {NULL} }
-};
+    {
+        .key = {"auth.login.*.allow"},
+        .type = GF_OPTION_TYPE_ANY,
+        .default_value = "*",
+        .description = "Username to be allowed access to the volume",
+        .op_version = {1},
+        .flags = OPT_FLAG_SETTABLE | OPT_FLAG_DOC,
+        .tags = {},
+        /* option_validation_fn validate_fn; */
+    },
+    {
+        .key = {"auth.login.*.password"},
+        .type = GF_OPTION_TYPE_ANY,
+        .default_value = "*",
+        .description = "Password for the allowed username",
+        .op_version = {1},
+        .flags = OPT_FLAG_SETTABLE | OPT_FLAG_DOC,
+        .tags = {},
+        /* option_validation_fn validate_fn; */
+    },
+    {.key = {NULL}}};
diff --git a/xlators/protocol/client/src/Makefile.am b/xlators/protocol/client/src/Makefile.am
index cf89d42da30..785a51fc3b4 100644
--- a/xlators/protocol/client/src/Makefile.am
+++ b/xlators/protocol/client/src/Makefile.am
@@ -2,19 +2,20 @@
 xlator_LTLIBRARIES = client.la
 xlatordir = $(libdir)/glusterfs/$(PACKAGE_VERSION)/xlator/protocol
 
-client_la_LDFLAGS = -module -avoid-version
+client_la_LDFLAGS = -module $(GF_XLATOR_DEFAULT_LDFLAGS)
 
 client_la_LIBADD = $(top_builddir)/libglusterfs/src/libglusterfs.la \
 	$(top_builddir)/rpc/rpc-lib/src/libgfrpc.la \
 	$(top_builddir)/rpc/xdr/src/libgfxdr.la
 
 client_la_SOURCES = client.c client-helpers.c client-rpc-fops.c  \
-	client-handshake.c client-callback.c client-lk.c
+	client-handshake.c client-callback.c client-lk.c client-common.c \
+	client-rpc-fops_v2.c
 
-noinst_HEADERS = client.h client-mem-types.h
+noinst_HEADERS = client.h client-mem-types.h client-messages.h client-common.h
 
-AM_CPPFLAGS = $(GF_CPPFLAGS) \
-	-I$(top_srcdir)/libglusterfs/src \
-	-I$(top_srcdir)/rpc/xdr/src -I$(top_srcdir)/rpc/rpc-lib/src/
+AM_CPPFLAGS = $(GF_CPPFLAGS) -I$(top_srcdir)/libglusterfs/src \
+	-I$(top_srcdir)/rpc/xdr/src -I$(top_builddir)/rpc/xdr/src \
+	-I$(top_srcdir)/rpc/rpc-lib/src/
 
 AM_CFLAGS = -Wall $(GF_CFLAGS)
diff --git a/xlators/protocol/client/src/client-callback.c b/xlators/protocol/client/src/client-callback.c
index fdfb3dc313b..d83d9c14899 100644
--- a/xlators/protocol/client/src/client-callback.c
+++ b/xlators/protocol/client/src/client-callback.c
@@ -8,83 +8,305 @@
   cases as published by the Free Software Foundation.
 */
 
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
-
 #include "client.h"
 #include "rpc-clnt.h"
-#include "defaults.h"
+#include <glusterfs/defaults.h>
+#include "client-messages.h"
 
-int
-client_cbk_null (struct rpc_clnt *rpc, void *mydata, void *data)
+static int
+client_cbk_null(struct rpc_clnt *rpc, void *mydata, void *data)
 {
-        gf_log (THIS->name, GF_LOG_WARNING,
-                "this function should not be called");
-        return 0;
+    gf_smsg(THIS->name, GF_LOG_WARNING, 0, PC_MSG_FUNCTION_CALL_ERROR, NULL);
+    return 0;
 }
 
-int
-client_cbk_fetchspec (struct rpc_clnt *rpc, void *mydata, void *data)
+static int
+client_cbk_fetchspec(struct rpc_clnt *rpc, void *mydata, void *data)
 {
-        gf_log (THIS->name, GF_LOG_WARNING,
-                "this function should not be called");
-        return 0;
+    gf_smsg(THIS->name, GF_LOG_WARNING, 0, PC_MSG_FUNCTION_CALL_ERROR, NULL);
+    return 0;
 }
 
-int
-client_cbk_ino_flush (struct rpc_clnt *rpc, void *mydata, void *data)
+static int
+client_cbk_ino_flush(struct rpc_clnt *rpc, void *mydata, void *data)
 {
-        gf_log (THIS->name, GF_LOG_WARNING,
-                "this function should not be called");
-        return 0;
+    gf_smsg(THIS->name, GF_LOG_WARNING, 0, PC_MSG_FUNCTION_CALL_ERROR, NULL);
+    return 0;
 }
 
-int
-client_cbk_upcall (struct rpc_clnt *rpc, void *mydata, void *data)
+static int
+client_cbk_recall_lease(struct rpc_clnt *rpc, void *mydata, void *data)
 {
-        int              ret          = -1;
-        gfs3_upcall_req  up_req;
-        struct gf_upcall upcall_data;
-        struct iovec     *iov         = NULL;
+    int ret = -1;
+    struct iovec *iov = NULL;
+    struct gf_upcall upcall_data = {
+        0,
+    };
+    struct gf_upcall_recall_lease rl_data = {
+        0,
+    };
+    gfs3_recall_lease_req recall_lease = {
+        {
+            0,
+        },
+    };
 
-        gf_log (THIS->name, GF_LOG_TRACE,
-                "Upcall callback is called");
+    GF_VALIDATE_OR_GOTO("client-callback", data, out);
 
-        if (!rpc || !mydata || !data)
-                goto out;
+    iov = (struct iovec *)data;
+    ret = xdr_to_generic(*iov, &recall_lease,
+                         (xdrproc_t)xdr_gfs3_recall_lease_req);
 
-        iov = (struct iovec *)data;
-        ret =  xdr_to_generic (*iov, &up_req,
-                               (xdrproc_t)xdr_gfs3_upcall_req);
+    if (ret < 0) {
+        gf_smsg(THIS->name, GF_LOG_WARNING, -ret, PC_MSG_RECALL_LEASE_FAIL,
+                NULL);
+        goto out;
+    }
 
-        if (ret < 0)
-                goto out;
+    upcall_data.data = &rl_data;
+    ret = gf_proto_recall_lease_to_upcall(&recall_lease, &upcall_data);
+    if (ret < 0)
+        goto out;
 
-        gf_proto_upcall_to_upcall (&up_req, &upcall_data);
+    upcall_data.event_type = GF_UPCALL_RECALL_LEASE;
 
-        gf_log (THIS->name, GF_LOG_TRACE, "Upcall gfid = %s, ret = %d",
-                 (char *)(up_req.gfid), ret);
+    gf_msg_trace(THIS->name, 0, "Upcall gfid = %s, ret = %d", recall_lease.gfid,
+                 ret);
 
-        default_notify (THIS, GF_EVENT_UPCALL, &upcall_data);
+    default_notify(THIS, GF_EVENT_UPCALL, &upcall_data);
 
 out:
-        return 0;
+    if (recall_lease.xdata.xdata_val)
+        free(recall_lease.xdata.xdata_val);
+
+    if (rl_data.dict)
+        dict_unref(rl_data.dict);
+
+    return ret;
 }
 
-rpcclnt_cb_actor_t gluster_cbk_actors[GF_CBK_MAXVALUE] = {
-        [GF_CBK_NULL]      = {"NULL",      GF_CBK_NULL,      client_cbk_null },
-        [GF_CBK_FETCHSPEC] = {"FETCHSPEC", GF_CBK_FETCHSPEC, client_cbk_fetchspec },
-        [GF_CBK_INO_FLUSH] = {"INO_FLUSH", GF_CBK_INO_FLUSH, client_cbk_ino_flush },
-        [GF_CBK_UPCALL]    = {"UPCALL",    GF_CBK_UPCALL,    client_cbk_upcall },
-};
+static int
+client_cbk_cache_invalidation(struct rpc_clnt *rpc, void *mydata, void *data)
+{
+    int ret = -1;
+    struct iovec *iov = NULL;
+    struct gf_upcall upcall_data = {
+        0,
+    };
+    struct gf_upcall_cache_invalidation ca_data = {
+        0,
+    };
+    gfs3_cbk_cache_invalidation_req ca_req = {
+        0,
+    };
+
+    gf_msg_trace(THIS->name, 0, "Upcall callback is called");
+
+    if (!data)
+        goto out;
+
+    iov = (struct iovec *)data;
+    ret = xdr_to_generic(*iov, &ca_req,
+                         (xdrproc_t)xdr_gfs3_cbk_cache_invalidation_req);
+
+    if (ret < 0) {
+        gf_smsg(THIS->name, GF_LOG_WARNING, -ret,
+                PC_MSG_CACHE_INVALIDATION_FAIL, NULL);
+        goto out;
+    }
+
+    upcall_data.data = &ca_data;
+    ret = gf_proto_cache_invalidation_to_upcall(THIS, &ca_req, &upcall_data);
+    if (ret < 0)
+        goto out;
+
+    gf_msg_trace(THIS->name, 0,
+                 "Cache invalidation cbk received for gfid:"
+                 " %s, ret = %d",
+                 ca_req.gfid, ret);
+
+    default_notify(THIS, GF_EVENT_UPCALL, &upcall_data);
+
+out:
+    if (ca_req.gfid)
+        free(ca_req.gfid);
+
+    if (ca_req.xdata.xdata_val)
+        free(ca_req.xdata.xdata_val);
+
+    if (ca_data.dict)
+        dict_unref(ca_data.dict);
+
+    return 0;
+}
+
+static int
+client_cbk_child_up(struct rpc_clnt *rpc, void *mydata, void *data)
+{
+    clnt_conf_t *conf = NULL;
+    xlator_t *this = THIS;
+
+    GF_VALIDATE_OR_GOTO("client", this, out);
+    conf = this->private;
+    GF_VALIDATE_OR_GOTO(this->name, conf, out);
+
+    gf_msg_debug(this->name, 0, "Received CHILD_UP");
+    conf->child_up = _gf_true;
+
+    this->notify(this, GF_EVENT_CHILD_UP, NULL);
+out:
+    return 0;
+}
+
+static int
+client_cbk_child_down(struct rpc_clnt *rpc, void *mydata, void *data)
+{
+    clnt_conf_t *conf = NULL;
+    xlator_t *this = THIS;
+
+    GF_VALIDATE_OR_GOTO("client", this, out);
+    conf = this->private;
+    GF_VALIDATE_OR_GOTO(this->name, conf, out);
+
+    gf_msg_debug(this->name, 0, "Received CHILD_DOWN");
+    conf->child_up = _gf_false;
+
+    this->notify(this, GF_EVENT_CHILD_DOWN, NULL);
+out:
+    return 0;
+}
+
+static int
+client_cbk_inodelk_contention(struct rpc_clnt *rpc, void *mydata, void *data)
+{
+    int ret = -1;
+    struct iovec *iov = NULL;
+    struct gf_upcall upcall_data = {
+        0,
+    };
+    struct gf_upcall_inodelk_contention lc = {
+        {
+            0,
+        },
+    };
+    gfs4_inodelk_contention_req proto_lc = {
+        {
+            0,
+        },
+    };
+
+    GF_VALIDATE_OR_GOTO("client-callback", data, out);
+
+    iov = (struct iovec *)data;
+    ret = xdr_to_generic(*iov, &proto_lc,
+                         (xdrproc_t)xdr_gfs4_inodelk_contention_req);
+
+    if (ret < 0) {
+        gf_smsg(THIS->name, GF_LOG_WARNING, -ret,
+                PC_MSG_INODELK_CONTENTION_FAIL, NULL);
+        goto out;
+    }
+
+    upcall_data.data = &lc;
+    ret = gf_proto_inodelk_contention_to_upcall(&proto_lc, &upcall_data);
+    if (ret < 0)
+        goto out;
+
+    upcall_data.event_type = GF_UPCALL_INODELK_CONTENTION;
 
+    default_notify(THIS, GF_EVENT_UPCALL, &upcall_data);
+
+out:
+    if (proto_lc.domain)
+        free(proto_lc.domain);
+
+    if (proto_lc.xdata.xdata_val)
+        free(proto_lc.xdata.xdata_val);
+
+    if (lc.xdata)
+        dict_unref(lc.xdata);
+
+    return ret;
+}
+
+static int
+client_cbk_entrylk_contention(struct rpc_clnt *rpc, void *mydata, void *data)
+{
+    int ret = -1;
+    struct iovec *iov = NULL;
+    struct gf_upcall upcall_data = {
+        0,
+    };
+    struct gf_upcall_entrylk_contention lc = {
+        0,
+    };
+    gfs4_entrylk_contention_req proto_lc = {
+        {
+            0,
+        },
+    };
+
+    GF_VALIDATE_OR_GOTO("client-callback", data, out);
+
+    iov = (struct iovec *)data;
+    ret = xdr_to_generic(*iov, &proto_lc,
+                         (xdrproc_t)xdr_gfs4_entrylk_contention_req);
+
+    if (ret < 0) {
+        gf_smsg(THIS->name, GF_LOG_WARNING, -ret,
+                PC_MSG_ENTRYLK_CONTENTION_FAIL, NULL);
+        goto out;
+    }
+
+    upcall_data.data = &lc;
+    ret = gf_proto_entrylk_contention_to_upcall(&proto_lc, &upcall_data);
+    if (ret < 0)
+        goto out;
+
+    upcall_data.event_type = GF_UPCALL_ENTRYLK_CONTENTION;
+
+    default_notify(THIS, GF_EVENT_UPCALL, &upcall_data);
+
+out:
+    if (proto_lc.name)
+        free(proto_lc.name);
+
+    if (proto_lc.domain)
+        free(proto_lc.domain);
+
+    if (proto_lc.xdata.xdata_val)
+        free(proto_lc.xdata.xdata_val);
+
+    if (lc.xdata)
+        dict_unref(lc.xdata);
+
+    return ret;
+}
+
+static rpcclnt_cb_actor_t gluster_cbk_actors[GF_CBK_MAXVALUE] = {
+    [GF_CBK_NULL] = {"NULL", client_cbk_null, GF_CBK_NULL},
+    [GF_CBK_FETCHSPEC] = {"FETCHSPEC", client_cbk_fetchspec, GF_CBK_FETCHSPEC},
+    [GF_CBK_INO_FLUSH] = {"INO_FLUSH", client_cbk_ino_flush, GF_CBK_INO_FLUSH},
+    [GF_CBK_CACHE_INVALIDATION] = {"CACHE_INVALIDATION",
+                                   client_cbk_cache_invalidation,
+                                   GF_CBK_CACHE_INVALIDATION},
+    [GF_CBK_CHILD_UP] = {"CHILD_UP", client_cbk_child_up, GF_CBK_CHILD_UP},
+    [GF_CBK_CHILD_DOWN] = {"CHILD_DOWN", client_cbk_child_down,
+                           GF_CBK_CHILD_DOWN},
+    [GF_CBK_RECALL_LEASE] = {"RECALL_LEASE", client_cbk_recall_lease,
+                             GF_CBK_RECALL_LEASE},
+    [GF_CBK_INODELK_CONTENTION] = {"INODELK_CONTENTION",
+                                   client_cbk_inodelk_contention,
+                                   GF_CBK_INODELK_CONTENTION},
+    [GF_CBK_ENTRYLK_CONTENTION] = {"ENTRYLK_CONTENTION",
+                                   client_cbk_entrylk_contention,
+                                   GF_CBK_ENTRYLK_CONTENTION},
+};
 
 struct rpcclnt_cb_program gluster_cbk_prog = {
-        .progname  = "GlusterFS Callback",
-        .prognum   = GLUSTER_CBK_PROGRAM,
-        .progver   = GLUSTER_CBK_VERSION,
-        .actors    = gluster_cbk_actors,
-        .numactors = GF_CBK_MAXVALUE,
+    .progname = "GlusterFS Callback",
+    .prognum = GLUSTER_CBK_PROGRAM,
+    .progver = GLUSTER_CBK_VERSION,
+    .actors = gluster_cbk_actors,
+    .numactors = GF_CBK_MAXVALUE,
 };
diff --git a/xlators/protocol/client/src/client-common.c b/xlators/protocol/client/src/client-common.c
new file mode 100644
index 00000000000..c112820e407
--- /dev/null
+++ b/xlators/protocol/client/src/client-common.c
@@ -0,0 +1,3589 @@
+/*
+  Copyright (c) 2016 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
+*/
+
+#include <glusterfs/dict.h>
+#include <glusterfs/xlator.h>
+#include "rpc-common-xdr.h"
+#include "glusterfs3-xdr.h"
+#include "glusterfs4-xdr.h"
+#include "glusterfs3.h"
+#include "client.h"
+
+/* processing to be done before fops are woudn down */
+int
+client_pre_stat(xlator_t *this, gfs3_stat_req *req, loc_t *loc, dict_t *xdata)
+{
+    int op_errno = ESTALE;
+
+    if (!(loc && loc->inode))
+        goto out;
+
+    if (!gf_uuid_is_null(loc->inode->gfid))
+        memcpy(req->gfid, loc->inode->gfid, 16);
+    else
+        memcpy(req->gfid, loc->gfid, 16);
+
+    GF_ASSERT_AND_GOTO_WITH_ERROR(this->name,
+                                  !gf_uuid_is_null(*((uuid_t *)req->gfid)), out,
+                                  op_errno, EINVAL);
+
+    GF_PROTOCOL_DICT_SERIALIZE(this, xdata, (&req->xdata.xdata_val),
+                               req->xdata.xdata_len, op_errno, out);
+
+    return 0;
+out:
+    return -op_errno;
+}
+
+int
+client_pre_readlink(xlator_t *this, gfs3_readlink_req *req, loc_t *loc,
+                    size_t size, dict_t *xdata)
+{
+    int op_errno = ESTALE;
+
+    if (!(loc && loc->inode))
+        goto out;
+
+    if (!gf_uuid_is_null(loc->inode->gfid))
+        memcpy(req->gfid, loc->inode->gfid, 16);
+    else
+        memcpy(req->gfid, loc->gfid, 16);
+
+    GF_ASSERT_AND_GOTO_WITH_ERROR(this->name,
+                                  !gf_uuid_is_null(*((uuid_t *)req->gfid)), out,
+                                  op_errno, EINVAL);
+    req->size = size;
+    GF_PROTOCOL_DICT_SERIALIZE(this, xdata, (&req->xdata.xdata_val),
+                               req->xdata.xdata_len, op_errno, out);
+    return 0;
+out:
+    return -op_errno;
+}
+
+int
+client_pre_mknod(xlator_t *this, gfs3_mknod_req *req, loc_t *loc, mode_t mode,
+                 dev_t rdev, mode_t umask, dict_t *xdata)
+{
+    int op_errno = ESTALE;
+
+    if (!(loc && loc->parent))
+        goto out;
+
+    if (!gf_uuid_is_null(loc->parent->gfid))
+        memcpy(req->pargfid, loc->parent->gfid, 16);
+    else
+        memcpy(req->pargfid, loc->pargfid, 16);
+
+    GF_ASSERT_AND_GOTO_WITH_ERROR(this->name,
+                                  !gf_uuid_is_null(*((uuid_t *)req->pargfid)),
+                                  out, op_errno, EINVAL);
+    req->bname = (char *)loc->name;
+    req->mode = mode;
+    req->dev = rdev;
+    req->umask = umask;
+
+    GF_PROTOCOL_DICT_SERIALIZE(this, xdata, (&req->xdata.xdata_val),
+                               req->xdata.xdata_len, op_errno, out);
+
+    return 0;
+out:
+    return -op_errno;
+}
+
+int
+client_pre_mkdir(xlator_t *this, gfs3_mkdir_req *req, loc_t *loc, mode_t mode,
+                 mode_t umask, dict_t *xdata)
+{
+    int op_errno = ESTALE;
+
+    if (!(loc && loc->parent))
+        goto out;
+
+    if (!gf_uuid_is_null(loc->parent->gfid))
+        memcpy(req->pargfid, loc->parent->gfid, 16);
+    else
+        memcpy(req->pargfid, loc->pargfid, 16);
+
+    GF_ASSERT_AND_GOTO_WITH_ERROR(this->name,
+                                  !gf_uuid_is_null(*((uuid_t *)req->pargfid)),
+                                  out, op_errno, EINVAL);
+
+    req->bname = (char *)loc->name;
+    req->mode = mode;
+    req->umask = umask;
+
+    GF_PROTOCOL_DICT_SERIALIZE(this, xdata, (&req->xdata.xdata_val),
+                               req->xdata.xdata_len, op_errno, out);
+
+    return 0;
+out:
+    return -op_errno;
+}
+
+int
+client_pre_unlink(xlator_t *this, gfs3_unlink_req *req, loc_t *loc,
+                  int32_t flags, dict_t *xdata)
+{
+    int op_errno = 0;
+
+    if (!(loc && loc->parent))
+        goto out;
+
+    if (!gf_uuid_is_null(loc->parent->gfid))
+        memcpy(req->pargfid, loc->parent->gfid, 16);
+    else
+        memcpy(req->pargfid, loc->pargfid, 16);
+
+    GF_ASSERT_AND_GOTO_WITH_ERROR(this->name,
+                                  !gf_uuid_is_null(*((uuid_t *)req->pargfid)),
+                                  out, op_errno, EINVAL);
+    req->bname = (char *)loc->name;
+    req->xflags = flags;
+
+    GF_PROTOCOL_DICT_SERIALIZE(this, xdata, (&req->xdata.xdata_val),
+                               req->xdata.xdata_len, op_errno, out);
+
+    return 0;
+out:
+    return -op_errno;
+}
+
+int
+client_pre_rmdir(xlator_t *this, gfs3_rmdir_req *req, loc_t *loc, int32_t flags,
+                 dict_t *xdata)
+{
+    int op_errno = ESTALE;
+
+    if (!(loc && loc->parent))
+        goto out;
+
+    if (!gf_uuid_is_null(loc->parent->gfid))
+        memcpy(req->pargfid, loc->parent->gfid, 16);
+    else
+        memcpy(req->pargfid, loc->pargfid, 16);
+
+    GF_ASSERT_AND_GOTO_WITH_ERROR(this->name,
+                                  !gf_uuid_is_null(*((uuid_t *)req->pargfid)),
+                                  out, op_errno, EINVAL);
+    req->bname = (char *)loc->name;
+    req->xflags = flags;
+
+    GF_PROTOCOL_DICT_SERIALIZE(this, xdata, (&req->xdata.xdata_val),
+                               req->xdata.xdata_len, op_errno, out);
+
+    return 0;
+out:
+    return -op_errno;
+}
+
+int
+client_pre_symlink(xlator_t *this, gfs3_symlink_req *req, loc_t *loc,
+                   const char *linkname, mode_t umask, dict_t *xdata)
+{
+    int op_errno = ESTALE;
+
+    if (!(loc && loc->parent))
+        goto out;
+
+    if (!gf_uuid_is_null(loc->parent->gfid))
+        memcpy(req->pargfid, loc->parent->gfid, 16);
+    else
+        memcpy(req->pargfid, loc->pargfid, 16);
+
+    GF_ASSERT_AND_GOTO_WITH_ERROR(this->name,
+                                  !gf_uuid_is_null(*((uuid_t *)req->pargfid)),
+                                  out, op_errno, EINVAL);
+    req->linkname = (char *)linkname;
+    req->bname = (char *)loc->name;
+    req->umask = umask;
+
+    GF_PROTOCOL_DICT_SERIALIZE(this, xdata, (&req->xdata.xdata_val),
+                               req->xdata.xdata_len, op_errno, out);
+    return 0;
+out:
+    return -op_errno;
+}
+
+int
+client_pre_rename(xlator_t *this, gfs3_rename_req *req, loc_t *oldloc,
+                  loc_t *newloc, dict_t *xdata)
+{
+    int op_errno = ESTALE;
+
+    if (!(oldloc && newloc && oldloc->parent && newloc->parent))
+        goto out;
+
+    if (!gf_uuid_is_null(oldloc->parent->gfid))
+        memcpy(req->oldgfid, oldloc->parent->gfid, 16);
+    else
+        memcpy(req->oldgfid, oldloc->pargfid, 16);
+
+    if (!gf_uuid_is_null(newloc->parent->gfid))
+        memcpy(req->newgfid, newloc->parent->gfid, 16);
+    else
+        memcpy(req->newgfid, newloc->pargfid, 16);
+
+    GF_ASSERT_AND_GOTO_WITH_ERROR(this->name,
+                                  !gf_uuid_is_null(*((uuid_t *)req->oldgfid)),
+                                  out, op_errno, EINVAL);
+    GF_ASSERT_AND_GOTO_WITH_ERROR(this->name,
+                                  !gf_uuid_is_null(*((uuid_t *)req->newgfid)),
+                                  out, op_errno, EINVAL);
+    req->oldbname = (char *)oldloc->name;
+    req->newbname = (char *)newloc->name;
+
+    GF_PROTOCOL_DICT_SERIALIZE(this, xdata, (&req->xdata.xdata_val),
+                               req->xdata.xdata_len, op_errno, out);
+
+    return 0;
+out:
+    return -op_errno;
+}
+
+int
+client_pre_link(xlator_t *this, gfs3_link_req *req, loc_t *oldloc,
+                loc_t *newloc, dict_t *xdata)
+{
+    int op_errno = ESTALE;
+
+    if (!(oldloc && oldloc->inode && newloc && newloc->parent))
+        goto out;
+
+    if (!gf_uuid_is_null(oldloc->inode->gfid))
+        memcpy(req->oldgfid, oldloc->inode->gfid, 16);
+    else
+        memcpy(req->oldgfid, oldloc->gfid, 16);
+
+    if (!gf_uuid_is_null(newloc->parent->gfid))
+        memcpy(req->newgfid, newloc->parent->gfid, 16);
+    else
+        memcpy(req->newgfid, newloc->pargfid, 16);
+
+    GF_ASSERT_AND_GOTO_WITH_ERROR(this->name,
+                                  !gf_uuid_is_null(*((uuid_t *)req->oldgfid)),
+                                  out, op_errno, EINVAL);
+    GF_ASSERT_AND_GOTO_WITH_ERROR(this->name,
+                                  !gf_uuid_is_null(*((uuid_t *)req->newgfid)),
+                                  out, op_errno, EINVAL);
+    req->newbname = (char *)newloc->name;
+
+    GF_PROTOCOL_DICT_SERIALIZE(this, xdata, (&req->xdata.xdata_val),
+                               req->xdata.xdata_len, op_errno, out);
+
+    return 0;
+out:
+    return -op_errno;
+}
+
+int
+client_pre_truncate(xlator_t *this, gfs3_truncate_req *req, loc_t *loc,
+                    off_t offset, dict_t *xdata)
+{
+    int op_errno = ESTALE;
+
+    if (!(loc && loc->inode))
+        goto out;
+
+    if (!gf_uuid_is_null(loc->inode->gfid))
+        memcpy(req->gfid, loc->inode->gfid, 16);
+    else
+        memcpy(req->gfid, loc->gfid, 16);
+
+    GF_ASSERT_AND_GOTO_WITH_ERROR(this->name,
+                                  !gf_uuid_is_null(*((uuid_t *)req->gfid)), out,
+                                  op_errno, EINVAL);
+    req->offset = offset;
+
+    GF_PROTOCOL_DICT_SERIALIZE(this, xdata, (&req->xdata.xdata_val),
+                               req->xdata.xdata_len, op_errno, out);
+    return 0;
+out:
+    return -op_errno;
+}
+
+int
+client_pre_open(xlator_t *this, gfs3_open_req *req, loc_t *loc, fd_t *fd,
+                int32_t flags, dict_t *xdata)
+{
+    int op_errno = ESTALE;
+
+    if (!(loc && loc->inode))
+        goto out;
+
+    if (!gf_uuid_is_null(loc->inode->gfid))
+        memcpy(req->gfid, loc->inode->gfid, 16);
+    else
+        memcpy(req->gfid, loc->gfid, 16);
+
+    GF_ASSERT_AND_GOTO_WITH_ERROR(this->name,
+                                  !gf_uuid_is_null(*((uuid_t *)req->gfid)), out,
+                                  op_errno, EINVAL);
+    req->flags = gf_flags_from_flags(flags);
+
+    GF_PROTOCOL_DICT_SERIALIZE(this, xdata, (&req->xdata.xdata_val),
+                               req->xdata.xdata_len, op_errno, out);
+
+    return 0;
+out:
+    return -op_errno;
+}
+
+int
+client_pre_readv(xlator_t *this, gfs3_read_req *req, fd_t *fd, size_t size,
+                 off_t offset, int32_t flags, dict_t *xdata)
+{
+    int64_t remote_fd = -1;
+    int op_errno = ESTALE;
+
+    CLIENT_GET_REMOTE_FD(this, fd, FALLBACK_TO_ANON_FD, remote_fd, op_errno,
+                         out);
+
+    req->size = size;
+    req->offset = offset;
+    req->fd = remote_fd;
+    req->flag = flags;
+
+    memcpy(req->gfid, fd->inode->gfid, 16);
+
+    GF_PROTOCOL_DICT_SERIALIZE(this, xdata, (&req->xdata.xdata_val),
+                               req->xdata.xdata_len, op_errno, out);
+
+    return 0;
+out:
+    return -op_errno;
+}
+
+int
+client_pre_writev(xlator_t *this, gfs3_write_req *req, fd_t *fd, size_t size,
+                  off_t offset, int32_t flags, dict_t **xdata)
+{
+    int64_t remote_fd = -1;
+    int op_errno = ESTALE;
+
+    CLIENT_GET_REMOTE_FD(this, fd, FALLBACK_TO_ANON_FD, remote_fd, op_errno,
+                         out);
+
+    req->size = size;
+    req->offset = offset;
+    req->fd = remote_fd;
+    req->flag = flags;
+
+    memcpy(req->gfid, fd->inode->gfid, 16);
+
+#ifdef GF_TESTING_IO_XDATA
+    if (!*xdata)
+        *xdata = dict_new();
+
+    ret = dict_set_str(*xdata, "testing-the-xdata-key",
+                       "testing-the-xdata-value");
+#endif
+
+    GF_PROTOCOL_DICT_SERIALIZE(this, *xdata, (&req->xdata.xdata_val),
+                               req->xdata.xdata_len, op_errno, out);
+
+    return 0;
+out:
+    return -op_errno;
+}
+
+int
+client_pre_statfs(xlator_t *this, gfs3_statfs_req *req, loc_t *loc,
+                  dict_t *xdata)
+{
+    int op_errno = ESTALE;
+
+    if (!loc)
+        goto out;
+
+    if (loc->inode) {
+        if (!gf_uuid_is_null(loc->inode->gfid))
+            memcpy(req->gfid, loc->inode->gfid, 16);
+        else
+            memcpy(req->gfid, loc->gfid, 16);
+    } else {
+        req->gfid[15] = 1;
+    }
+
+    GF_ASSERT_AND_GOTO_WITH_ERROR(this->name,
+                                  !gf_uuid_is_null(*((uuid_t *)req->gfid)), out,
+                                  op_errno, EINVAL);
+
+    GF_PROTOCOL_DICT_SERIALIZE(this, xdata, (&req->xdata.xdata_val),
+                               req->xdata.xdata_len, op_errno, out);
+
+    return 0;
+out:
+    return -op_errno;
+}
+
+int
+client_pre_flush(xlator_t *this, gfs3_flush_req *req, fd_t *fd, dict_t *xdata)
+{
+    int64_t remote_fd = -1;
+    int op_errno = ESTALE;
+
+    CLIENT_GET_REMOTE_FD(this, fd, DEFAULT_REMOTE_FD, remote_fd, op_errno, out);
+
+    req->fd = remote_fd;
+    memcpy(req->gfid, fd->inode->gfid, 16);
+
+    GF_PROTOCOL_DICT_SERIALIZE(this, xdata, (&req->xdata.xdata_val),
+                               req->xdata.xdata_len, op_errno, out);
+
+    return 0;
+out:
+    return -op_errno;
+}
+
+int
+client_pre_fsync(xlator_t *this, gfs3_fsync_req *req, fd_t *fd, int32_t flags,
+                 dict_t *xdata)
+{
+    int64_t remote_fd = -1;
+    int op_errno = 0;
+
+    CLIENT_GET_REMOTE_FD(this, fd, FALLBACK_TO_ANON_FD, remote_fd, op_errno,
+                         out);
+
+    req->fd = remote_fd;
+    req->data = flags;
+    memcpy(req->gfid, fd->inode->gfid, 16);
+
+    GF_PROTOCOL_DICT_SERIALIZE(this, xdata, (&req->xdata.xdata_val),
+                               req->xdata.xdata_len, op_errno, out);
+
+    return 0;
+out:
+    return -op_errno;
+}
+
+int
+client_pre_setxattr(xlator_t *this, gfs3_setxattr_req *req, loc_t *loc,
+                    dict_t *xattr, int32_t flags, dict_t *xdata)
+{
+    int op_errno = ESTALE;
+
+    if (!(loc && loc->inode))
+        goto out;
+
+    if (!gf_uuid_is_null(loc->inode->gfid))
+        memcpy(req->gfid, loc->inode->gfid, 16);
+    else
+        memcpy(req->gfid, loc->gfid, 16);
+
+    GF_ASSERT_AND_GOTO_WITH_ERROR(this->name,
+                                  !gf_uuid_is_null(*((uuid_t *)req->gfid)), out,
+                                  op_errno, EINVAL);
+    if (xattr) {
+        GF_PROTOCOL_DICT_SERIALIZE(this, xattr, (&req->dict.dict_val),
+                                   req->dict.dict_len, op_errno, out);
+    }
+
+    req->flags = flags;
+
+    GF_PROTOCOL_DICT_SERIALIZE(this, xdata, (&req->xdata.xdata_val),
+                               req->xdata.xdata_len, op_errno, out);
+
+    return 0;
+out:
+    return -op_errno;
+}
+
+int
+client_pre_getxattr(xlator_t *this, gfs3_getxattr_req *req, loc_t *loc,
+                    const char *name, dict_t *xdata)
+{
+    int op_errno = ESTALE;
+
+    if (!loc) {
+        op_errno = EINVAL;
+        goto out;
+    }
+
+    if (loc->inode && !gf_uuid_is_null(loc->inode->gfid))
+        memcpy(req->gfid, loc->inode->gfid, 16);
+    else
+        memcpy(req->gfid, loc->gfid, 16);
+
+    GF_ASSERT_AND_GOTO_WITH_ERROR(this->name,
+                                  !gf_uuid_is_null(*((uuid_t *)req->gfid)), out,
+                                  op_errno, EINVAL);
+    req->namelen = 1; /* Use it as a flag */
+
+    req->name = (char *)name;
+    if (!req->name) {
+        req->name = "";
+        req->namelen = 0;
+    }
+
+    GF_PROTOCOL_DICT_SERIALIZE(this, xdata, (&req->xdata.xdata_val),
+                               req->xdata.xdata_len, op_errno, out);
+
+    return 0;
+out:
+    return -op_errno;
+}
+
+int
+client_pre_removexattr(xlator_t *this, gfs3_removexattr_req *req, loc_t *loc,
+                       const char *name, dict_t *xdata)
+{
+    int op_errno = ESTALE;
+
+    if (!(loc && loc->inode))
+        goto out;
+
+    if (!gf_uuid_is_null(loc->inode->gfid))
+        memcpy(req->gfid, loc->inode->gfid, 16);
+    else
+        memcpy(req->gfid, loc->gfid, 16);
+
+    GF_ASSERT_AND_GOTO_WITH_ERROR(this->name,
+                                  !gf_uuid_is_null(*((uuid_t *)req->gfid)), out,
+                                  op_errno, EINVAL);
+    req->name = (char *)name;
+
+    GF_PROTOCOL_DICT_SERIALIZE(this, xdata, (&req->xdata.xdata_val),
+                               req->xdata.xdata_len, op_errno, out);
+
+    return 0;
+out:
+    return -op_errno;
+}
+
+int
+client_pre_opendir(xlator_t *this, gfs3_opendir_req *req, loc_t *loc, fd_t *fd,
+                   dict_t *xdata)
+{
+    int op_errno = ESTALE;
+
+    if (!(loc && loc->inode))
+        goto out;
+
+    if (!gf_uuid_is_null(loc->inode->gfid))
+        memcpy(req->gfid, loc->inode->gfid, 16);
+    else
+        memcpy(req->gfid, loc->gfid, 16);
+
+    GF_ASSERT_AND_GOTO_WITH_ERROR(this->name,
+                                  !gf_uuid_is_null(*((uuid_t *)req->gfid)), out,
+                                  op_errno, EINVAL);
+
+    GF_PROTOCOL_DICT_SERIALIZE(this, xdata, (&req->xdata.xdata_val),
+                               req->xdata.xdata_len, op_errno, out);
+
+    return 0;
+out:
+    return -op_errno;
+}
+
+int
+client_pre_fsyncdir(xlator_t *this, gfs3_fsyncdir_req *req, fd_t *fd,
+                    int32_t flags, dict_t *xdata)
+{
+    int32_t op_errno = ESTALE;
+    int64_t remote_fd = -1;
+
+    CLIENT_GET_REMOTE_FD(this, fd, DEFAULT_REMOTE_FD, remote_fd, op_errno, out);
+
+    req->fd = remote_fd;
+    req->data = flags;
+    memcpy(req->gfid, fd->inode->gfid, 16);
+
+    GF_PROTOCOL_DICT_SERIALIZE(this, xdata, (&req->xdata.xdata_val),
+                               req->xdata.xdata_len, op_errno, out);
+
+    return 0;
+out:
+    return -op_errno;
+}
+
+int
+client_pre_access(xlator_t *this, gfs3_access_req *req, loc_t *loc,
+                  int32_t mask, dict_t *xdata)
+{
+    int op_errno = ESTALE;
+
+    if (!(loc && loc->inode))
+        goto out;
+
+    if (!gf_uuid_is_null(loc->inode->gfid))
+        memcpy(req->gfid, loc->inode->gfid, 16);
+    else
+        memcpy(req->gfid, loc->gfid, 16);
+
+    GF_ASSERT_AND_GOTO_WITH_ERROR(this->name,
+                                  !gf_uuid_is_null(*((uuid_t *)req->gfid)), out,
+                                  op_errno, EINVAL);
+    req->mask = mask;
+
+    GF_PROTOCOL_DICT_SERIALIZE(this, xdata, (&req->xdata.xdata_val),
+                               req->xdata.xdata_len, op_errno, out);
+    return 0;
+out:
+    return -op_errno;
+}
+
+int
+client_pre_create(xlator_t *this, gfs3_create_req *req, loc_t *loc, fd_t *fd,
+                  mode_t mode, int32_t flags, mode_t umask, dict_t *xdata)
+{
+    int op_errno = ESTALE;
+
+    if (!(loc && loc->parent))
+        goto out;
+
+    if (!gf_uuid_is_null(loc->parent->gfid))
+        memcpy(req->pargfid, loc->parent->gfid, 16);
+    else
+        memcpy(req->pargfid, loc->pargfid, 16);
+
+    GF_ASSERT_AND_GOTO_WITH_ERROR(this->name,
+                                  !gf_uuid_is_null(*((uuid_t *)req->pargfid)),
+                                  out, op_errno, EINVAL);
+    req->bname = (char *)loc->name;
+    req->mode = mode;
+    req->flags = gf_flags_from_flags(flags);
+    req->umask = umask;
+
+    GF_PROTOCOL_DICT_SERIALIZE(this, xdata, (&req->xdata.xdata_val),
+                               req->xdata.xdata_len, op_errno, out);
+
+    return 0;
+out:
+    return -op_errno;
+}
+
+int
+client_pre_ftruncate(xlator_t *this, gfs3_ftruncate_req *req, fd_t *fd,
+                     off_t offset, dict_t *xdata)
+{
+    int64_t remote_fd = -1;
+    int op_errno = EINVAL;
+
+    CLIENT_GET_REMOTE_FD(this, fd, DEFAULT_REMOTE_FD, remote_fd, op_errno, out);
+
+    req->offset = offset;
+    req->fd = remote_fd;
+    memcpy(req->gfid, fd->inode->gfid, 16);
+
+    GF_PROTOCOL_DICT_SERIALIZE(this, xdata, (&req->xdata.xdata_val),
+                               req->xdata.xdata_len, op_errno, out);
+    return 0;
+out:
+    return -op_errno;
+}
+
+int
+client_pre_fstat(xlator_t *this, gfs3_fstat_req *req, fd_t *fd, dict_t *xdata)
+{
+    int64_t remote_fd = -1;
+    int op_errno = ESTALE;
+
+    CLIENT_GET_REMOTE_FD(this, fd, DEFAULT_REMOTE_FD, remote_fd, op_errno, out);
+
+    req->fd = remote_fd;
+    memcpy(req->gfid, fd->inode->gfid, 16);
+
+    GF_PROTOCOL_DICT_SERIALIZE(this, xdata, (&req->xdata.xdata_val),
+                               req->xdata.xdata_len, op_errno, out);
+
+    return 0;
+out:
+    return -op_errno;
+}
+
+int
+client_pre_lk(xlator_t *this, gfs3_lk_req *req, int32_t cmd,
+              struct gf_flock *flock, fd_t *fd, dict_t *xdata)
+{
+    int64_t remote_fd = -1;
+    int op_errno = ESTALE;
+    int32_t gf_cmd = 0;
+    int32_t gf_type = 0;
+    int ret = 0;
+
+    CLIENT_GET_REMOTE_FD(this, fd, DEFAULT_REMOTE_FD, remote_fd, op_errno, out);
+
+    ret = client_cmd_to_gf_cmd(cmd, &gf_cmd);
+    if (ret) {
+        op_errno = EINVAL;
+        gf_smsg(this->name, GF_LOG_WARNING, EINVAL, PC_MSG_UNKNOWN_CMD,
+                "gf_cmd=%d", gf_cmd, NULL);
+        goto out;
+    }
+
+    switch (flock->l_type) {
+        case F_RDLCK:
+            gf_type = GF_LK_F_RDLCK;
+            break;
+        case F_WRLCK:
+            gf_type = GF_LK_F_WRLCK;
+            break;
+        case F_UNLCK:
+            gf_type = GF_LK_F_UNLCK;
+            break;
+    }
+
+    req->fd = remote_fd;
+    req->cmd = gf_cmd;
+    req->type = gf_type;
+    gf_proto_flock_from_flock(&req->flock, flock);
+
+    memcpy(req->gfid, fd->inode->gfid, 16);
+
+    GF_PROTOCOL_DICT_SERIALIZE(this, xdata, (&req->xdata.xdata_val),
+                               req->xdata.xdata_len, op_errno, out);
+
+    return 0;
+out:
+    return -op_errno;
+}
+
+int
+client_pre_lookup(xlator_t *this, gfs3_lookup_req *req, loc_t *loc,
+                  dict_t *xdata)
+{
+    int op_errno = ESTALE;
+
+    if (!(loc && loc->inode))
+        goto out;
+
+    if ((loc->parent) && (!gf_uuid_is_null(loc->parent->gfid)))
+        memcpy(req->pargfid, loc->parent->gfid, 16);
+    else
+        memcpy(req->pargfid, loc->pargfid, 16);
+
+    if ((loc->inode) && (!gf_uuid_is_null(loc->inode->gfid)))
+        memcpy(req->gfid, loc->inode->gfid, 16);
+    else
+        memcpy(req->gfid, loc->gfid, 16);
+
+    if (loc->name)
+        req->bname = (char *)loc->name;
+    else
+        req->bname = "";
+
+    if (xdata) {
+        GF_PROTOCOL_DICT_SERIALIZE(this, xdata, (&req->xdata.xdata_val),
+                                   req->xdata.xdata_len, op_errno, out);
+    }
+    return 0;
+out:
+    return -op_errno;
+}
+
+int
+client_pre_readdir(xlator_t *this, gfs3_readdir_req *req, fd_t *fd, size_t size,
+                   off_t offset, dict_t *xdata)
+{
+    int64_t remote_fd = -1;
+    int op_errno = ESTALE;
+
+    CLIENT_GET_REMOTE_FD(this, fd, DEFAULT_REMOTE_FD, remote_fd, op_errno, out);
+
+    req->size = size;
+    req->offset = offset;
+    req->fd = remote_fd;
+
+    memcpy(req->gfid, fd->inode->gfid, 16);
+    GF_PROTOCOL_DICT_SERIALIZE(this, xdata, (&req->xdata.xdata_val),
+                               req->xdata.xdata_len, op_errno, out);
+
+    return 0;
+out:
+    return -op_errno;
+}
+
+int
+client_pre_inodelk(xlator_t *this, gfs3_inodelk_req *req, loc_t *loc, int cmd,
+                   struct gf_flock *flock, const char *volume, dict_t *xdata)
+{
+    int op_errno = ESTALE;
+    int32_t gf_cmd = 0;
+    int32_t gf_type = 0;
+
+    if (!(loc && loc->inode))
+        goto out;
+
+    if (!gf_uuid_is_null(loc->gfid))
+        memcpy(req->gfid, loc->gfid, 16);
+    else
+        memcpy(req->gfid, loc->inode->gfid, 16);
+
+    GF_ASSERT_AND_GOTO_WITH_ERROR(this->name,
+                                  !gf_uuid_is_null(*((uuid_t *)req->gfid)), out,
+                                  op_errno, EINVAL);
+    if (cmd == F_GETLK || cmd == F_GETLK64)
+        gf_cmd = GF_LK_GETLK;
+    else if (cmd == F_SETLK || cmd == F_SETLK64)
+        gf_cmd = GF_LK_SETLK;
+    else if (cmd == F_SETLKW || cmd == F_SETLKW64)
+        gf_cmd = GF_LK_SETLKW;
+    else {
+        gf_smsg(this->name, GF_LOG_WARNING, EINVAL, PC_MSG_UNKNOWN_CMD,
+                "gf_cmd=%d", gf_cmd, NULL);
+        op_errno = EINVAL;
+        goto out;
+    }
+
+    switch (flock->l_type) {
+        case F_RDLCK:
+            gf_type = GF_LK_F_RDLCK;
+            break;
+        case F_WRLCK:
+            gf_type = GF_LK_F_WRLCK;
+            break;
+        case F_UNLCK:
+            gf_type = GF_LK_F_UNLCK;
+            break;
+    }
+
+    req->volume = (char *)volume;
+    req->cmd = gf_cmd;
+    req->type = gf_type;
+    gf_proto_flock_from_flock(&req->flock, flock);
+
+    GF_PROTOCOL_DICT_SERIALIZE(this, xdata, (&req->xdata.xdata_val),
+                               req->xdata.xdata_len, op_errno, out);
+
+    return 0;
+out:
+    return -op_errno;
+}
+
+int
+client_pre_finodelk(xlator_t *this, gfs3_finodelk_req *req, fd_t *fd, int cmd,
+                    struct gf_flock *flock, const char *volume, dict_t *xdata)
+{
+    int op_errno = ESTALE;
+    int64_t remote_fd = -1;
+    int32_t gf_type = 0;
+    int32_t gf_cmd = 0;
+
+    CLIENT_GET_REMOTE_FD(this, fd, FALLBACK_TO_ANON_FD, remote_fd, op_errno,
+                         out);
+
+    if (cmd == F_GETLK || cmd == F_GETLK64)
+        gf_cmd = GF_LK_GETLK;
+    else if (cmd == F_SETLK || cmd == F_SETLK64)
+        gf_cmd = GF_LK_SETLK;
+    else if (cmd == F_SETLKW || cmd == F_SETLKW64)
+        gf_cmd = GF_LK_SETLKW;
+    else {
+        gf_smsg(this->name, GF_LOG_WARNING, EINVAL, PC_MSG_UNKNOWN_CMD,
+                "gf_cmd=%d", gf_cmd, NULL);
+        goto out;
+    }
+
+    switch (flock->l_type) {
+        case F_RDLCK:
+            gf_type = GF_LK_F_RDLCK;
+            break;
+        case F_WRLCK:
+            gf_type = GF_LK_F_WRLCK;
+            break;
+        case F_UNLCK:
+            gf_type = GF_LK_F_UNLCK;
+            break;
+    }
+
+    req->volume = (char *)volume;
+    req->fd = remote_fd;
+    req->cmd = gf_cmd;
+    req->type = gf_type;
+    gf_proto_flock_from_flock(&req->flock, flock);
+    memcpy(req->gfid, fd->inode->gfid, 16);
+
+    GF_PROTOCOL_DICT_SERIALIZE(this, xdata, (&req->xdata.xdata_val),
+                               req->xdata.xdata_len, op_errno, out);
+    return 0;
+out:
+    return -op_errno;
+}
+
+int
+client_pre_entrylk(xlator_t *this, gfs3_entrylk_req *req, loc_t *loc,
+                   entrylk_cmd cmd_entrylk, entrylk_type type,
+                   const char *volume, const char *basename, dict_t *xdata)
+{
+    int op_errno = ESTALE;
+
+    if (!(loc && loc->inode))
+        goto out;
+
+    if (!gf_uuid_is_null(loc->gfid))
+        memcpy(req->gfid, loc->gfid, 16);
+    else
+        memcpy(req->gfid, loc->inode->gfid, 16);
+
+    GF_ASSERT_AND_GOTO_WITH_ERROR(this->name,
+                                  !gf_uuid_is_null(*((uuid_t *)req->gfid)), out,
+                                  op_errno, EINVAL);
+    req->cmd = cmd_entrylk;
+    req->type = type;
+    req->volume = (char *)volume;
+    req->name = "";
+    if (basename) {
+        req->name = (char *)basename;
+        req->namelen = 1;
+    }
+
+    GF_PROTOCOL_DICT_SERIALIZE(this, xdata, (&req->xdata.xdata_val),
+                               req->xdata.xdata_len, op_errno, out);
+
+    return 0;
+out:
+    return -op_errno;
+}
+
+int
+client_pre_fentrylk(xlator_t *this, gfs3_fentrylk_req *req, fd_t *fd,
+                    entrylk_cmd cmd_entrylk, entrylk_type type,
+                    const char *volume, const char *basename, dict_t *xdata)
+{
+    int64_t remote_fd = -1;
+    int op_errno = ESTALE;
+
+    CLIENT_GET_REMOTE_FD(this, fd, DEFAULT_REMOTE_FD, remote_fd, op_errno, out);
+
+    req->fd = remote_fd;
+    req->cmd = cmd_entrylk;
+    req->type = type;
+    req->volume = (char *)volume;
+    req->name = "";
+    if (basename) {
+        req->name = (char *)basename;
+        req->namelen = 1;
+    }
+    memcpy(req->gfid, fd->inode->gfid, 16);
+
+    GF_PROTOCOL_DICT_SERIALIZE(this, xdata, (&req->xdata.xdata_val),
+                               req->xdata.xdata_len, op_errno, out);
+
+    return 0;
+out:
+    return -op_errno;
+}
+
+int
+client_pre_xattrop(xlator_t *this, gfs3_xattrop_req *req, loc_t *loc,
+                   dict_t *xattr, int32_t flags, dict_t *xdata)
+{
+    int op_errno = ESTALE;
+
+    if (!(loc && loc->inode))
+        goto out;
+
+    if (!gf_uuid_is_null(loc->inode->gfid))
+        memcpy(req->gfid, loc->inode->gfid, 16);
+    else
+        memcpy(req->gfid, loc->gfid, 16);
+
+    GF_ASSERT_AND_GOTO_WITH_ERROR(this->name,
+                                  !gf_uuid_is_null(*((uuid_t *)req->gfid)), out,
+                                  op_errno, EINVAL);
+    if (xattr) {
+        GF_PROTOCOL_DICT_SERIALIZE(this, xattr, (&req->dict.dict_val),
+                                   req->dict.dict_len, op_errno, out);
+    }
+
+    req->flags = flags;
+
+    GF_PROTOCOL_DICT_SERIALIZE(this, xdata, (&req->xdata.xdata_val),
+                               req->xdata.xdata_len, op_errno, out);
+
+    return 0;
+out:
+    return -op_errno;
+}
+
+int
+client_pre_fxattrop(xlator_t *this, gfs3_fxattrop_req *req, fd_t *fd,
+                    dict_t *xattr, int32_t flags, dict_t *xdata)
+{
+    int op_errno = ESTALE;
+    int64_t remote_fd = -1;
+
+    CLIENT_GET_REMOTE_FD(this, fd, FALLBACK_TO_ANON_FD, remote_fd, op_errno,
+                         out);
+
+    req->fd = remote_fd;
+    req->flags = flags;
+    memcpy(req->gfid, fd->inode->gfid, 16);
+
+    if (xattr) {
+        GF_PROTOCOL_DICT_SERIALIZE(this, xattr, (&req->dict.dict_val),
+                                   req->dict.dict_len, op_errno, out);
+    }
+
+    GF_PROTOCOL_DICT_SERIALIZE(this, xdata, (&req->xdata.xdata_val),
+                               req->xdata.xdata_len, op_errno, out);
+
+    return 0;
+out:
+    return -op_errno;
+}
+
+int
+client_pre_fgetxattr(xlator_t *this, gfs3_fgetxattr_req *req, fd_t *fd,
+                     const char *name, dict_t *xdata)
+{
+    int64_t remote_fd = -1;
+    int op_errno = ESTALE;
+
+    CLIENT_GET_REMOTE_FD(this, fd, DEFAULT_REMOTE_FD, remote_fd, op_errno, out);
+
+    req->namelen = 1; /* Use it as a flag */
+    req->fd = remote_fd;
+    req->name = (char *)name;
+    if (!req->name) {
+        req->name = "";
+        req->namelen = 0;
+    }
+    memcpy(req->gfid, fd->inode->gfid, 16);
+
+    GF_PROTOCOL_DICT_SERIALIZE(this, xdata, (&req->xdata.xdata_val),
+                               req->xdata.xdata_len, op_errno, out);
+
+    return 0;
+out:
+    return -op_errno;
+}
+
+int
+client_pre_fsetxattr(xlator_t *this, gfs3_fsetxattr_req *req, fd_t *fd,
+                     int32_t flags, dict_t *xattr, dict_t *xdata)
+{
+    int64_t remote_fd = -1;
+    int op_errno = ESTALE;
+
+    CLIENT_GET_REMOTE_FD(this, fd, DEFAULT_REMOTE_FD, remote_fd, op_errno, out);
+
+    req->fd = remote_fd;
+    req->flags = flags;
+    memcpy(req->gfid, fd->inode->gfid, 16);
+
+    if (xattr) {
+        GF_PROTOCOL_DICT_SERIALIZE(this, xattr, (&req->dict.dict_val),
+                                   req->dict.dict_len, op_errno, out);
+    }
+
+    GF_PROTOCOL_DICT_SERIALIZE(this, xdata, (&req->xdata.xdata_val),
+                               req->xdata.xdata_len, op_errno, out);
+
+    return 0;
+out:
+    return -op_errno;
+}
+
+int
+client_pre_rchecksum(xlator_t *this, gfs3_rchecksum_req *req, fd_t *fd,
+                     int32_t len, off_t offset, dict_t *xdata)
+{
+    int64_t remote_fd = -1;
+    int op_errno = ESTALE;
+
+    CLIENT_GET_REMOTE_FD(this, fd, DEFAULT_REMOTE_FD, remote_fd, op_errno, out);
+
+    req->len = len;
+    req->offset = offset;
+    req->fd = remote_fd;
+
+    GF_PROTOCOL_DICT_SERIALIZE(this, xdata, (&req->xdata.xdata_val),
+                               req->xdata.xdata_len, op_errno, out);
+
+    return 0;
+out:
+    return -op_errno;
+}
+
+int
+client_pre_setattr(xlator_t *this, gfs3_setattr_req *req, loc_t *loc,
+                   int32_t valid, struct iatt *stbuf, dict_t *xdata)
+{
+    int op_errno = ESTALE;
+
+    if (!(loc && loc->inode))
+        return -op_errno;
+
+    if (!gf_uuid_is_null(loc->inode->gfid))
+        memcpy(req->gfid, loc->inode->gfid, 16);
+    else
+        memcpy(req->gfid, loc->gfid, 16);
+
+    GF_ASSERT_AND_GOTO_WITH_ERROR(this->name,
+                                  !gf_uuid_is_null(*((uuid_t *)req->gfid)), out,
+                                  op_errno, EINVAL);
+
+    req->valid = valid;
+    gf_stat_from_iatt(&req->stbuf, stbuf);
+
+    GF_PROTOCOL_DICT_SERIALIZE(this, xdata, (&req->xdata.xdata_val),
+                               req->xdata.xdata_len, op_errno, out);
+
+    return 0;
+out:
+    return -op_errno;
+}
+
+int
+client_pre_fsetattr(xlator_t *this, gfs3_fsetattr_req *req, fd_t *fd,
+                    int32_t valid, struct iatt *stbuf, dict_t *xdata)
+{
+    int op_errno = ESTALE;
+    int64_t remote_fd = -1;
+
+    CLIENT_GET_REMOTE_FD(this, fd, DEFAULT_REMOTE_FD, remote_fd, op_errno, out);
+
+    req->fd = remote_fd;
+    req->valid = valid;
+    gf_stat_from_iatt(&req->stbuf, stbuf);
+
+    GF_PROTOCOL_DICT_SERIALIZE(this, xdata, (&req->xdata.xdata_val),
+                               req->xdata.xdata_len, op_errno, out);
+    return 0;
+out:
+    return -op_errno;
+}
+
+int
+client_pre_readdirp(xlator_t *this, gfs3_readdirp_req *req, fd_t *fd,
+                    size_t size, off_t offset, dict_t *xdata)
+{
+    int op_errno = ESTALE;
+    int64_t remote_fd = -1;
+
+    CLIENT_GET_REMOTE_FD(this, fd, DEFAULT_REMOTE_FD, remote_fd, op_errno, out);
+
+    req->size = size;
+    req->offset = offset;
+    req->fd = remote_fd;
+    memcpy(req->gfid, fd->inode->gfid, 16);
+
+    /* dict itself is 'xdata' here */
+    GF_PROTOCOL_DICT_SERIALIZE(this, xdata, (&req->dict.dict_val),
+                               req->dict.dict_len, op_errno, out);
+
+    return 0;
+out:
+    return -op_errno;
+}
+
+int
+client_pre_fremovexattr(xlator_t *this, gfs3_fremovexattr_req *req, fd_t *fd,
+                        const char *name, dict_t *xdata)
+{
+    int64_t remote_fd = -1;
+    int op_errno = ESTALE;
+
+    if (!(fd && fd->inode))
+        goto out;
+
+    CLIENT_GET_REMOTE_FD(this, fd, DEFAULT_REMOTE_FD, remote_fd, op_errno, out);
+
+    memcpy(req->gfid, fd->inode->gfid, 16);
+    req->name = (char *)name;
+    req->fd = remote_fd;
+
+    GF_PROTOCOL_DICT_SERIALIZE(this, xdata, (&req->xdata.xdata_val),
+                               req->xdata.xdata_len, op_errno, out);
+
+    return 0;
+out:
+    return -op_errno;
+}
+
+int
+client_pre_fallocate(xlator_t *this, gfs3_fallocate_req *req, fd_t *fd,
+                     int32_t flags, off_t offset, size_t size, dict_t *xdata)
+{
+    int op_errno = ESTALE;
+    int64_t remote_fd = -1;
+
+    CLIENT_GET_REMOTE_FD(this, fd, DEFAULT_REMOTE_FD, remote_fd, op_errno, out);
+
+    req->fd = remote_fd;
+    req->flags = flags;
+    req->offset = offset;
+    req->size = size;
+    memcpy(req->gfid, fd->inode->gfid, 16);
+
+    GF_PROTOCOL_DICT_SERIALIZE(this, xdata, (&req->xdata.xdata_val),
+                               req->xdata.xdata_len, op_errno, out);
+    return 0;
+out:
+    return -op_errno;
+}
+
+int
+client_pre_discard(xlator_t *this, gfs3_discard_req *req, fd_t *fd,
+                   off_t offset, size_t size, dict_t *xdata)
+{
+    int op_errno = ESTALE;
+    int64_t remote_fd = -1;
+
+    CLIENT_GET_REMOTE_FD(this, fd, DEFAULT_REMOTE_FD, remote_fd, op_errno, out);
+
+    req->fd = remote_fd;
+    req->offset = offset;
+    req->size = size;
+    memcpy(req->gfid, fd->inode->gfid, 16);
+
+    GF_PROTOCOL_DICT_SERIALIZE(this, xdata, (&req->xdata.xdata_val),
+                               req->xdata.xdata_len, op_errno, out);
+    return 0;
+out:
+    return -op_errno;
+}
+
+int
+client_pre_zerofill(xlator_t *this, gfs3_zerofill_req *req, fd_t *fd,
+                    off_t offset, size_t size, dict_t *xdata)
+{
+    int op_errno = ESTALE;
+    int64_t remote_fd = -1;
+
+    CLIENT_GET_REMOTE_FD(this, fd, DEFAULT_REMOTE_FD, remote_fd, op_errno, out);
+
+    req->fd = remote_fd;
+    req->offset = offset;
+    req->size = size;
+    memcpy(req->gfid, fd->inode->gfid, 16);
+
+    GF_PROTOCOL_DICT_SERIALIZE(this, xdata, (&req->xdata.xdata_val),
+                               req->xdata.xdata_len, op_errno, out);
+    return 0;
+out:
+    return -op_errno;
+}
+
+int
+client_pre_ipc(xlator_t *this, gfs3_ipc_req *req, int32_t cmd, dict_t *xdata)
+{
+    int op_errno = ESTALE;
+
+    req->op = cmd;
+
+    GF_PROTOCOL_DICT_SERIALIZE(this, xdata, (&req->xdata.xdata_val),
+                               req->xdata.xdata_len, op_errno, out);
+    return 0;
+out:
+    return -op_errno;
+}
+
+int
+client_pre_seek(xlator_t *this, gfs3_seek_req *req, fd_t *fd, off_t offset,
+                gf_seek_what_t what, dict_t *xdata)
+{
+    int64_t remote_fd = -1;
+    int op_errno = ESTALE;
+
+    CLIENT_GET_REMOTE_FD(this, fd, DEFAULT_REMOTE_FD, remote_fd, op_errno, out);
+
+    memcpy(req->gfid, fd->inode->gfid, 16);
+    req->fd = remote_fd;
+    req->offset = offset;
+    req->what = what;
+
+    GF_PROTOCOL_DICT_SERIALIZE(this, xdata, (&req->xdata.xdata_val),
+                               req->xdata.xdata_len, op_errno, out);
+
+    return 0;
+out:
+    return -op_errno;
+}
+
+int
+client_pre_lease(xlator_t *this, gfs3_lease_req *req, loc_t *loc,
+                 struct gf_lease *lease, dict_t *xdata)
+{
+    int op_errno = 0;
+
+    if (!(loc && loc->inode))
+        goto out;
+
+    if (!gf_uuid_is_null(loc->inode->gfid))
+        memcpy(req->gfid, loc->inode->gfid, 16);
+    else
+        memcpy(req->gfid, loc->gfid, 16);
+
+    GF_ASSERT_AND_GOTO_WITH_ERROR(this->name,
+                                  !gf_uuid_is_null(*((uuid_t *)req->gfid)), out,
+                                  op_errno, EINVAL);
+
+    gf_proto_lease_from_lease(&req->lease, lease);
+
+    GF_PROTOCOL_DICT_SERIALIZE(this, xdata, (&req->xdata.xdata_val),
+                               req->xdata.xdata_len, op_errno, out);
+out:
+    return -op_errno;
+}
+
+/* processing done after fop responses are obtained */
+int
+client_post_stat(xlator_t *this, gfs3_stat_rsp *rsp, struct iatt *iatt,
+                 dict_t **xdata)
+{
+    int ret = 0;
+
+    if (-1 != rsp->op_ret) {
+        gf_stat_to_iatt(&rsp->stat, iatt);
+    }
+
+    GF_PROTOCOL_DICT_UNSERIALIZE(this, *xdata, (rsp->xdata.xdata_val),
+                                 (rsp->xdata.xdata_len), ret, rsp->op_errno,
+                                 out);
+
+out:
+    return ret;
+}
+
+int
+client_post_readlink(xlator_t *this, gfs3_readlink_rsp *rsp, struct iatt *iatt,
+                     dict_t **xdata)
+{
+    int ret = 0;
+
+    if (-1 != rsp->op_ret) {
+        gf_stat_to_iatt(&rsp->buf, iatt);
+    }
+
+    GF_PROTOCOL_DICT_UNSERIALIZE(this, *xdata, (rsp->xdata.xdata_val),
+                                 (rsp->xdata.xdata_len), ret, rsp->op_errno,
+                                 out);
+
+out:
+
+    return ret;
+}
+
+int
+client_post_mknod(xlator_t *this, gfs3_mknod_rsp *rsp, struct iatt *stbuf,
+                  struct iatt *preparent, struct iatt *postparent,
+                  dict_t **xdata)
+{
+    int ret = 0;
+
+    if (-1 != rsp->op_ret) {
+        gf_stat_to_iatt(&rsp->stat, stbuf);
+        gf_stat_to_iatt(&rsp->preparent, preparent);
+        gf_stat_to_iatt(&rsp->postparent, postparent);
+    }
+    GF_PROTOCOL_DICT_UNSERIALIZE(this, *xdata, (rsp->xdata.xdata_val),
+                                 (rsp->xdata.xdata_len), ret, rsp->op_errno,
+                                 out);
+
+out:
+    return ret;
+}
+
+int
+client_post_mkdir(xlator_t *this, gfs3_mkdir_rsp *rsp, struct iatt *stbuf,
+                  struct iatt *preparent, struct iatt *postparent,
+                  dict_t **xdata)
+{
+    int ret = 0;
+
+    if (-1 != rsp->op_ret) {
+        gf_stat_to_iatt(&rsp->stat, stbuf);
+        gf_stat_to_iatt(&rsp->preparent, preparent);
+        gf_stat_to_iatt(&rsp->postparent, postparent);
+    }
+
+    GF_PROTOCOL_DICT_UNSERIALIZE(this, *xdata, (rsp->xdata.xdata_val),
+                                 (rsp->xdata.xdata_len), ret, rsp->op_errno,
+                                 out);
+
+out:
+    return ret;
+}
+
+int
+client_post_unlink(xlator_t *this, gfs3_unlink_rsp *rsp, struct iatt *preparent,
+                   struct iatt *postparent, dict_t **xdata)
+{
+    int ret = 0;
+
+    if (-1 != rsp->op_ret) {
+        gf_stat_to_iatt(&rsp->preparent, preparent);
+        gf_stat_to_iatt(&rsp->postparent, postparent);
+    }
+
+    GF_PROTOCOL_DICT_UNSERIALIZE(this, *xdata, (rsp->xdata.xdata_val),
+                                 (rsp->xdata.xdata_len), ret, rsp->op_errno,
+                                 out);
+
+    ret = gf_replace_new_iatt_in_dict(*xdata);
+out:
+    return ret;
+}
+
+int
+client_post_rmdir(xlator_t *this, gfs3_rmdir_rsp *rsp, struct iatt *preparent,
+                  struct iatt *postparent, dict_t **xdata)
+{
+    int ret = 0;
+
+    if (-1 != rsp->op_ret) {
+        gf_stat_to_iatt(&rsp->preparent, preparent);
+        gf_stat_to_iatt(&rsp->postparent, postparent);
+    }
+    GF_PROTOCOL_DICT_UNSERIALIZE(this, *xdata, (rsp->xdata.xdata_val),
+                                 (rsp->xdata.xdata_len), ret, rsp->op_errno,
+                                 out);
+
+out:
+    return ret;
+}
+
+int
+client_post_symlink(xlator_t *this, gfs3_symlink_rsp *rsp, struct iatt *stbuf,
+                    struct iatt *preparent, struct iatt *postparent,
+                    dict_t **xdata)
+{
+    int ret = 0;
+
+    if (-1 != rsp->op_ret) {
+        gf_stat_to_iatt(&rsp->stat, stbuf);
+        gf_stat_to_iatt(&rsp->preparent, preparent);
+        gf_stat_to_iatt(&rsp->postparent, postparent);
+    }
+    GF_PROTOCOL_DICT_UNSERIALIZE(this, *xdata, (rsp->xdata.xdata_val),
+                                 (rsp->xdata.xdata_len), ret, rsp->op_errno,
+                                 out);
+out:
+    return ret;
+}
+
+int
+client_post_rename(xlator_t *this, gfs3_rename_rsp *rsp, struct iatt *stbuf,
+                   struct iatt *preoldparent, struct iatt *postoldparent,
+                   struct iatt *prenewparent, struct iatt *postnewparent,
+                   dict_t **xdata)
+{
+    int ret = 0;
+
+    if (-1 != rsp->op_ret) {
+        gf_stat_to_iatt(&rsp->stat, stbuf);
+
+        gf_stat_to_iatt(&rsp->preoldparent, preoldparent);
+        gf_stat_to_iatt(&rsp->postoldparent, postoldparent);
+
+        gf_stat_to_iatt(&rsp->prenewparent, prenewparent);
+        gf_stat_to_iatt(&rsp->postnewparent, postnewparent);
+    }
+
+    GF_PROTOCOL_DICT_UNSERIALIZE(this, *xdata, (rsp->xdata.xdata_val),
+                                 (rsp->xdata.xdata_len), ret, rsp->op_errno,
+                                 out);
+out:
+    return ret;
+}
+
+int
+client_post_link(xlator_t *this, gfs3_link_rsp *rsp, struct iatt *stbuf,
+                 struct iatt *preparent, struct iatt *postparent,
+                 dict_t **xdata)
+{
+    int ret = 0;
+
+    if (-1 != rsp->op_ret) {
+        gf_stat_to_iatt(&rsp->stat, stbuf);
+        gf_stat_to_iatt(&rsp->preparent, preparent);
+        gf_stat_to_iatt(&rsp->postparent, postparent);
+    }
+    GF_PROTOCOL_DICT_UNSERIALIZE(this, *xdata, (rsp->xdata.xdata_val),
+                                 (rsp->xdata.xdata_len), ret, rsp->op_errno,
+                                 out);
+out:
+    return ret;
+}
+
+int
+client_post_truncate(xlator_t *this, gfs3_truncate_rsp *rsp,
+                     struct iatt *prestat, struct iatt *poststat,
+                     dict_t **xdata)
+{
+    int ret = 0;
+
+    if (-1 != rsp->op_ret) {
+        gf_stat_to_iatt(&rsp->prestat, prestat);
+        gf_stat_to_iatt(&rsp->poststat, poststat);
+    }
+    GF_PROTOCOL_DICT_UNSERIALIZE(this, *xdata, (rsp->xdata.xdata_val),
+                                 (rsp->xdata.xdata_len), ret, rsp->op_errno,
+                                 out);
+out:
+    return ret;
+}
+
+int
+client_post_open(xlator_t *this, gfs3_open_rsp *rsp, dict_t **xdata)
+{
+    int ret = 0;
+
+    GF_PROTOCOL_DICT_UNSERIALIZE(this, *xdata, (rsp->xdata.xdata_val),
+                                 (rsp->xdata.xdata_len), ret, rsp->op_errno,
+                                 out);
+out:
+    return ret;
+}
+
+int
+client_post_readv(xlator_t *this, gfs3_read_rsp *rsp, struct iobref **iobref,
+                  struct iobref *rsp_iobref, struct iatt *stat,
+                  struct iovec *vector, struct iovec *rsp_vector, int *rspcount,
+                  dict_t **xdata)
+{
+    int ret = 0;
+
+    if (rsp->op_ret != -1) {
+        *iobref = rsp_iobref;
+        gf_stat_to_iatt(&rsp->stat, stat);
+
+        vector[0].iov_len = rsp->op_ret;
+        if (rsp->op_ret > 0)
+            vector[0].iov_base = rsp_vector->iov_base;
+        *rspcount = 1;
+    }
+    GF_PROTOCOL_DICT_UNSERIALIZE(this, *xdata, (rsp->xdata.xdata_val),
+                                 (rsp->xdata.xdata_len), ret, rsp->op_errno,
+                                 out);
+
+#ifdef GF_TESTING_IO_XDATA
+    dict_dump_to_log(xdata);
+#endif
+out:
+    return ret;
+}
+
+int
+client_post_writev(xlator_t *this, gfs3_write_rsp *rsp, struct iatt *prestat,
+                   struct iatt *poststat, dict_t **xdata)
+{
+    int ret = 0;
+
+    if (-1 != rsp->op_ret) {
+        gf_stat_to_iatt(&rsp->prestat, prestat);
+        gf_stat_to_iatt(&rsp->poststat, poststat);
+    }
+
+    GF_PROTOCOL_DICT_UNSERIALIZE(this, *xdata, (rsp->xdata.xdata_val),
+                                 (rsp->xdata.xdata_len), ret, rsp->op_errno,
+                                 out);
+
+out:
+    return ret;
+}
+
+int
+client_post_statfs(xlator_t *this, gfs3_statfs_rsp *rsp, struct statvfs *statfs,
+                   dict_t **xdata)
+{
+    int ret = 0;
+
+    if (-1 != rsp->op_ret) {
+        gf_statfs_to_statfs(&rsp->statfs, statfs);
+    }
+    GF_PROTOCOL_DICT_UNSERIALIZE(this, *xdata, (rsp->xdata.xdata_val),
+                                 (rsp->xdata.xdata_len), ret, rsp->op_errno,
+                                 out);
+out:
+    return ret;
+}
+
+int
+client_post_flush(xlator_t *this, gf_common_rsp *rsp, dict_t **xdata)
+{
+    int ret = 0;
+
+    GF_PROTOCOL_DICT_UNSERIALIZE(this, *xdata, (rsp->xdata.xdata_val),
+                                 (rsp->xdata.xdata_len), ret, rsp->op_errno,
+                                 out);
+
+out:
+    return ret;
+}
+
+int
+client_post_fsync(xlator_t *this, gfs3_fsync_rsp *rsp, struct iatt *prestat,
+                  struct iatt *poststat, dict_t **xdata)
+{
+    int ret = 0;
+
+    if (-1 != rsp->op_ret) {
+        gf_stat_to_iatt(&rsp->prestat, prestat);
+        gf_stat_to_iatt(&rsp->poststat, poststat);
+    }
+    GF_PROTOCOL_DICT_UNSERIALIZE(this, *xdata, (rsp->xdata.xdata_val),
+                                 (rsp->xdata.xdata_len), ret, rsp->op_errno,
+                                 out);
+out:
+    return ret;
+}
+
+int
+client_post_setxattr(xlator_t *this, gf_common_rsp *rsp, dict_t **xdata)
+{
+    int ret = 0;
+
+    GF_PROTOCOL_DICT_UNSERIALIZE(this, *xdata, (rsp->xdata.xdata_val),
+                                 (rsp->xdata.xdata_len), ret, rsp->op_errno,
+                                 out);
+
+    ret = gf_replace_new_iatt_in_dict(*xdata);
+out:
+    return ret;
+}
+
+int
+client_post_getxattr(xlator_t *this, gfs3_getxattr_rsp *rsp, dict_t **dict,
+                     dict_t **xdata)
+{
+    int op_errno = 0;
+    int ret = 0;
+
+    if (-1 != rsp->op_ret) {
+        GF_PROTOCOL_DICT_UNSERIALIZE(this, *dict, (rsp->dict.dict_val),
+                                     (rsp->dict.dict_len), rsp->op_ret,
+                                     op_errno, out);
+    }
+    GF_PROTOCOL_DICT_UNSERIALIZE(this, *xdata, (rsp->xdata.xdata_val),
+                                 (rsp->xdata.xdata_len), ret, op_errno, out);
+
+out:
+    return -op_errno;
+}
+
+int
+client_post_removexattr(xlator_t *this, gf_common_rsp *rsp, dict_t **xdata)
+{
+    int ret = 0;
+
+    GF_PROTOCOL_DICT_UNSERIALIZE(this, *xdata, (rsp->xdata.xdata_val),
+                                 (rsp->xdata.xdata_len), ret, rsp->op_errno,
+                                 out);
+
+    ret = gf_replace_new_iatt_in_dict(*xdata);
+out:
+    return ret;
+}
+
+int
+client_post_opendir(xlator_t *this, gfs3_opendir_rsp *rsp, dict_t **xdata)
+{
+    int ret = 0;
+
+    GF_PROTOCOL_DICT_UNSERIALIZE(this, *xdata, (rsp->xdata.xdata_val),
+                                 (rsp->xdata.xdata_len), ret, rsp->op_errno,
+                                 out);
+out:
+    return ret;
+}
+
+int
+client_post_fsyncdir(xlator_t *this, gf_common_rsp *rsp, dict_t **xdata)
+{
+    int ret = 0;
+
+    GF_PROTOCOL_DICT_UNSERIALIZE(this, *xdata, (rsp->xdata.xdata_val),
+                                 (rsp->xdata.xdata_len), ret, rsp->op_errno,
+                                 out);
+out:
+    return ret;
+}
+
+int
+client_post_access(xlator_t *this, gf_common_rsp *rsp, dict_t **xdata)
+{
+    int ret = 0;
+
+    GF_PROTOCOL_DICT_UNSERIALIZE(this, *xdata, (rsp->xdata.xdata_val),
+                                 (rsp->xdata.xdata_len), ret, rsp->op_errno,
+                                 out);
+out:
+    return ret;
+}
+
+int
+client_post_create(xlator_t *this, gfs3_create_rsp *rsp, struct iatt *stbuf,
+                   struct iatt *preparent, struct iatt *postparent,
+                   clnt_local_t *local, dict_t **xdata)
+{
+    int ret = 0;
+
+    if (-1 != rsp->op_ret) {
+        gf_stat_to_iatt(&rsp->stat, stbuf);
+
+        gf_stat_to_iatt(&rsp->preparent, preparent);
+        gf_stat_to_iatt(&rsp->postparent, postparent);
+        gf_uuid_copy(local->loc.gfid, stbuf->ia_gfid);
+    }
+    GF_PROTOCOL_DICT_UNSERIALIZE(this, *xdata, (rsp->xdata.xdata_val),
+                                 (rsp->xdata.xdata_len), ret, rsp->op_errno,
+                                 out);
+out:
+    return ret;
+}
+
+int
+client_post_ftruncate(xlator_t *this, gfs3_ftruncate_rsp *rsp,
+                      struct iatt *prestat, struct iatt *poststat,
+                      dict_t **xdata)
+{
+    int ret = 0;
+
+    if (-1 != rsp->op_ret) {
+        gf_stat_to_iatt(&rsp->prestat, prestat);
+        gf_stat_to_iatt(&rsp->poststat, poststat);
+    }
+    GF_PROTOCOL_DICT_UNSERIALIZE(this, *xdata, (rsp->xdata.xdata_val),
+                                 (rsp->xdata.xdata_len), ret, rsp->op_errno,
+                                 out);
+out:
+    return ret;
+}
+
+int
+client_post_fstat(xlator_t *this, gfs3_fstat_rsp *rsp, struct iatt *stat,
+                  dict_t **xdata)
+{
+    int ret = 0;
+
+    if (-1 != rsp->op_ret) {
+        gf_stat_to_iatt(&rsp->stat, stat);
+    }
+    GF_PROTOCOL_DICT_UNSERIALIZE(this, *xdata, (rsp->xdata.xdata_val),
+                                 (rsp->xdata.xdata_len), ret, rsp->op_errno,
+                                 out);
+out:
+    return -ret;
+}
+
+int
+client_post_lk(xlator_t *this, gfs3_lk_rsp *rsp, struct gf_flock *lock,
+               dict_t **xdata)
+{
+    int ret = 0;
+
+    if (rsp->op_ret >= 0) {
+        gf_proto_flock_to_flock(&rsp->flock, lock);
+    }
+    GF_PROTOCOL_DICT_UNSERIALIZE(this, *xdata, (rsp->xdata.xdata_val),
+                                 (rsp->xdata.xdata_len), ret, rsp->op_errno,
+                                 out);
+out:
+    return ret;
+}
+
+int
+client_post_lookup(xlator_t *this, gfs3_lookup_rsp *rsp, struct iatt *stbuf,
+                   struct iatt *postparent, dict_t **xdata)
+{
+    int ret = 0;
+
+    if (-1 != rsp->op_ret) {
+        gf_stat_to_iatt(&rsp->postparent, postparent);
+        gf_stat_to_iatt(&rsp->stat, stbuf);
+    }
+
+    GF_PROTOCOL_DICT_UNSERIALIZE(this, *xdata, (rsp->xdata.xdata_val),
+                                 (rsp->xdata.xdata_len), ret, rsp->op_errno,
+                                 out);
+out:
+    return ret;
+}
+
+int
+client_post_readdir(xlator_t *this, gfs3_readdir_rsp *rsp, gf_dirent_t *entries,
+                    dict_t **xdata)
+{
+    int ret = 0;
+
+    if (rsp->op_ret > 0) {
+        unserialize_rsp_dirent(this, rsp, entries);
+    }
+    GF_PROTOCOL_DICT_UNSERIALIZE(this, *xdata, (rsp->xdata.xdata_val),
+                                 (rsp->xdata.xdata_len), ret, rsp->op_errno,
+                                 out);
+
+out:
+    return ret;
+}
+
+int
+client_post_inodelk(xlator_t *this, gf_common_rsp *rsp, dict_t **xdata)
+{
+    int ret = 0;
+
+    GF_PROTOCOL_DICT_UNSERIALIZE(this, *xdata, (rsp->xdata.xdata_val),
+                                 (rsp->xdata.xdata_len), ret, rsp->op_errno,
+                                 out);
+
+out:
+    return ret;
+}
+
+int
+client_post_finodelk(xlator_t *this, gf_common_rsp *rsp, dict_t **xdata)
+{
+    int ret = 0;
+
+    GF_PROTOCOL_DICT_UNSERIALIZE(this, *xdata, (rsp->xdata.xdata_val),
+                                 (rsp->xdata.xdata_len), ret, rsp->op_errno,
+                                 out);
+
+out:
+    return ret;
+}
+
+int
+client_post_entrylk(xlator_t *this, gf_common_rsp *rsp, dict_t **xdata)
+{
+    int ret = 0;
+
+    GF_PROTOCOL_DICT_UNSERIALIZE(this, *xdata, (rsp->xdata.xdata_val),
+                                 (rsp->xdata.xdata_len), ret, rsp->op_errno,
+                                 out);
+
+out:
+    return ret;
+}
+
+int
+client_post_fentrylk(xlator_t *this, gf_common_rsp *rsp, dict_t **xdata)
+{
+    int ret = 0;
+
+    GF_PROTOCOL_DICT_UNSERIALIZE(this, *xdata, (rsp->xdata.xdata_val),
+                                 (rsp->xdata.xdata_len), ret, rsp->op_errno,
+                                 out);
+
+out:
+    return ret;
+}
+
+int
+client_post_xattrop(xlator_t *this, gfs3_xattrop_rsp *rsp, dict_t **dict,
+                    dict_t **xdata)
+{
+    int op_errno = 0;
+    int ret = 0;
+
+    if (-1 != rsp->op_ret) {
+        GF_PROTOCOL_DICT_UNSERIALIZE(this, *dict, (rsp->dict.dict_val),
+                                     (rsp->dict.dict_len), rsp->op_ret,
+                                     op_errno, out);
+    }
+    GF_PROTOCOL_DICT_UNSERIALIZE(this, *xdata, (rsp->xdata.xdata_val),
+                                 (rsp->xdata.xdata_len), ret, op_errno, out);
+
+out:
+    return -op_errno;
+}
+
+int
+client_post_fxattrop(xlator_t *this, gfs3_fxattrop_rsp *rsp, dict_t **dict,
+                     dict_t **xdata)
+{
+    int op_errno = 0;
+    int ret = 0;
+
+    if (-1 != rsp->op_ret) {
+        GF_PROTOCOL_DICT_UNSERIALIZE(this, *dict, (rsp->dict.dict_val),
+                                     (rsp->dict.dict_len), rsp->op_ret,
+                                     op_errno, out);
+    }
+    GF_PROTOCOL_DICT_UNSERIALIZE(this, *xdata, (rsp->xdata.xdata_val),
+                                 (rsp->xdata.xdata_len), ret, op_errno, out);
+
+out:
+    return -op_errno;
+}
+
+int
+client_post_fgetxattr(xlator_t *this, gfs3_fgetxattr_rsp *rsp, dict_t **dict,
+                      dict_t **xdata)
+{
+    int op_errno = 0;
+    int ret = 0;
+
+    if (-1 != rsp->op_ret) {
+        GF_PROTOCOL_DICT_UNSERIALIZE(this, *dict, (rsp->dict.dict_val),
+                                     (rsp->dict.dict_len), rsp->op_ret,
+                                     op_errno, out);
+    }
+    GF_PROTOCOL_DICT_UNSERIALIZE(this, *xdata, (rsp->xdata.xdata_val),
+                                 (rsp->xdata.xdata_len), ret, op_errno, out);
+
+out:
+    return -op_errno;
+}
+
+int
+client_post_fsetxattr(xlator_t *this, gf_common_rsp *rsp, dict_t **xdata)
+{
+    int ret = 0;
+
+    GF_PROTOCOL_DICT_UNSERIALIZE(this, *xdata, (rsp->xdata.xdata_val),
+                                 (rsp->xdata.xdata_len), ret, rsp->op_errno,
+                                 out);
+
+    ret = gf_replace_new_iatt_in_dict(*xdata);
+out:
+    return ret;
+}
+
+int
+client_post_rchecksum(xlator_t *this, gfs3_rchecksum_rsp *rsp, dict_t **xdata)
+{
+    int ret = 0;
+
+    GF_PROTOCOL_DICT_UNSERIALIZE(this, *xdata, (rsp->xdata.xdata_val),
+                                 (rsp->xdata.xdata_len), ret, rsp->op_errno,
+                                 out);
+out:
+    return ret;
+}
+
+int
+client_post_setattr(xlator_t *this, gfs3_setattr_rsp *rsp, struct iatt *prestat,
+                    struct iatt *poststat, dict_t **xdata)
+{
+    int ret = 0;
+
+    if (-1 != rsp->op_ret) {
+        gf_stat_to_iatt(&rsp->statpre, prestat);
+        gf_stat_to_iatt(&rsp->statpost, poststat);
+    }
+    GF_PROTOCOL_DICT_UNSERIALIZE(this, *xdata, (rsp->xdata.xdata_val),
+                                 (rsp->xdata.xdata_len), ret, rsp->op_errno,
+                                 out);
+out:
+    return ret;
+}
+
+int
+client_post_fsetattr(xlator_t *this, gfs3_fsetattr_rsp *rsp,
+                     struct iatt *prestat, struct iatt *poststat,
+                     dict_t **xdata)
+{
+    int ret = 0;
+
+    if (-1 != rsp->op_ret) {
+        gf_stat_to_iatt(&rsp->statpre, prestat);
+        gf_stat_to_iatt(&rsp->statpost, poststat);
+    }
+    GF_PROTOCOL_DICT_UNSERIALIZE(this, *xdata, (rsp->xdata.xdata_val),
+                                 (rsp->xdata.xdata_len), ret, rsp->op_errno,
+                                 out);
+out:
+    return ret;
+}
+
+int
+client_post_readdirp(xlator_t *this, gfs3_readdirp_rsp *rsp, fd_t *fd,
+                     gf_dirent_t *entries, dict_t **xdata)
+{
+    int ret = 0;
+
+    if (rsp->op_ret > 0) {
+        unserialize_rsp_direntp(this, fd, rsp, entries);
+    }
+
+    GF_PROTOCOL_DICT_UNSERIALIZE(this, *xdata, (rsp->xdata.xdata_val),
+                                 (rsp->xdata.xdata_len), ret, rsp->op_errno,
+                                 out);
+out:
+    return ret;
+}
+
+int
+client_post_fremovexattr(xlator_t *this, gf_common_rsp *rsp, dict_t **xdata)
+{
+    int ret = 0;
+
+    GF_PROTOCOL_DICT_UNSERIALIZE(this, *xdata, (rsp->xdata.xdata_val),
+                                 (rsp->xdata.xdata_len), ret, rsp->op_errno,
+                                 out);
+
+    ret = gf_replace_new_iatt_in_dict(*xdata);
+out:
+    return ret;
+}
+
+int
+client_post_fallocate(xlator_t *this, gfs3_fallocate_rsp *rsp,
+                      struct iatt *prestat, struct iatt *poststat,
+                      dict_t **xdata)
+{
+    int ret = 0;
+
+    if (-1 != rsp->op_ret) {
+        gf_stat_to_iatt(&rsp->statpre, prestat);
+        gf_stat_to_iatt(&rsp->statpost, poststat);
+    }
+    GF_PROTOCOL_DICT_UNSERIALIZE(this, *xdata, (rsp->xdata.xdata_val),
+                                 (rsp->xdata.xdata_len), ret, rsp->op_errno,
+                                 out);
+out:
+    return ret;
+}
+
+int
+client_post_discard(xlator_t *this, gfs3_discard_rsp *rsp, struct iatt *prestat,
+                    struct iatt *poststat, dict_t **xdata)
+{
+    int ret = 0;
+
+    if (-1 != rsp->op_ret) {
+        gf_stat_to_iatt(&rsp->statpre, prestat);
+        gf_stat_to_iatt(&rsp->statpost, poststat);
+    }
+    GF_PROTOCOL_DICT_UNSERIALIZE(this, *xdata, (rsp->xdata.xdata_val),
+                                 (rsp->xdata.xdata_len), ret, rsp->op_errno,
+                                 out);
+out:
+    return ret;
+}
+
+int
+client_post_zerofill(xlator_t *this, gfs3_zerofill_rsp *rsp,
+                     struct iatt *prestat, struct iatt *poststat,
+                     dict_t **xdata)
+{
+    int ret = 0;
+
+    if (-1 != rsp->op_ret) {
+        gf_stat_to_iatt(&rsp->statpre, prestat);
+        gf_stat_to_iatt(&rsp->statpost, poststat);
+    }
+    GF_PROTOCOL_DICT_UNSERIALIZE(this, *xdata, (rsp->xdata.xdata_val),
+                                 (rsp->xdata.xdata_len), ret, rsp->op_errno,
+                                 out);
+out:
+    return ret;
+}
+
+int
+client_post_ipc(xlator_t *this, gfs3_ipc_rsp *rsp, dict_t **xdata)
+{
+    int ret = 0;
+
+    GF_PROTOCOL_DICT_UNSERIALIZE(this, *xdata, (rsp->xdata.xdata_val),
+                                 (rsp->xdata.xdata_len), ret, rsp->op_errno,
+                                 out);
+out:
+    return ret;
+}
+
+int
+client_post_seek(xlator_t *this, gfs3_seek_rsp *rsp, dict_t **xdata)
+{
+    int ret = 0;
+
+    GF_PROTOCOL_DICT_UNSERIALIZE(this, *xdata, (rsp->xdata.xdata_val),
+                                 (rsp->xdata.xdata_len), ret, rsp->op_errno,
+                                 out);
+out:
+    return ret;
+}
+
+int
+client_post_lease(xlator_t *this, gfs3_lease_rsp *rsp, struct gf_lease *lease,
+                  dict_t **xdata)
+{
+    int ret = 0;
+
+    if (rsp->op_ret >= 0) {
+        gf_proto_lease_to_lease(&rsp->lease, lease);
+    }
+
+    GF_PROTOCOL_DICT_UNSERIALIZE(this, *xdata, (rsp->xdata.xdata_val),
+                                 (rsp->xdata.xdata_len), ret, rsp->op_errno,
+                                 out);
+out:
+    return ret;
+}
+
+/* New PRE and POST functions */
+
+int
+client_post_common_iatt(xlator_t *this, gfx_common_iatt_rsp *rsp,
+                        struct iatt *iatt, dict_t **xdata)
+{
+    if (-1 != rsp->op_ret) {
+        gfx_stat_to_iattx(&rsp->stat, iatt);
+    }
+
+    return xdr_to_dict(&rsp->xdata, xdata);
+}
+
+int
+client_post_common_2iatt(xlator_t *this, gfx_common_2iatt_rsp *rsp,
+                         struct iatt *iatt, struct iatt *iatt2, dict_t **xdata)
+{
+    if (-1 != rsp->op_ret) {
+        gfx_stat_to_iattx(&rsp->prestat, iatt);
+        gfx_stat_to_iattx(&rsp->poststat, iatt2);
+    }
+
+    return xdr_to_dict(&rsp->xdata, xdata);
+}
+
+int
+client_post_common_3iatt(xlator_t *this, gfx_common_3iatt_rsp *rsp,
+                         struct iatt *iatt, struct iatt *iatt2,
+                         struct iatt *iatt3, dict_t **xdata)
+{
+    if (-1 != rsp->op_ret) {
+        gfx_stat_to_iattx(&rsp->stat, iatt);
+        gfx_stat_to_iattx(&rsp->preparent, iatt2);
+        gfx_stat_to_iattx(&rsp->postparent, iatt3);
+    }
+
+    return xdr_to_dict(&rsp->xdata, xdata);
+}
+
+int
+client_post_common_dict(xlator_t *this, gfx_common_dict_rsp *rsp, dict_t **dict,
+                        dict_t **xdata)
+{
+    int ret = 0;
+    ret = xdr_to_dict(&rsp->dict, dict);
+    if (ret)
+        gf_msg_debug(this->name, EINVAL,
+                     "while decoding found empty dictionary");
+    xdr_to_dict(&rsp->xdata, xdata);
+
+    return ret;
+}
+
+int
+client_post_readv_v2(xlator_t *this, gfx_read_rsp *rsp, struct iobref **iobref,
+                     struct iobref *rsp_iobref, struct iatt *stat,
+                     struct iovec *vector, struct iovec *rsp_vector,
+                     int *rspcount, dict_t **xdata)
+{
+    int ret = -1;
+
+    if (rsp->op_ret != -1) {
+        *iobref = rsp_iobref;
+        gfx_stat_to_iattx(&rsp->stat, stat);
+
+        vector[0].iov_len = rsp->op_ret;
+        if (rsp->op_ret > 0)
+            vector[0].iov_base = rsp_vector->iov_base;
+        *rspcount = 1;
+    }
+
+    ret = xdr_to_dict(&rsp->xdata, xdata);
+
+#ifdef GF_TESTING_IO_XDATA
+    dict_dump_to_log(xdata);
+#endif
+    return ret;
+}
+
+int
+client_pre_stat_v2(xlator_t *this, gfx_stat_req *req, loc_t *loc, dict_t *xdata)
+{
+    int op_errno = ESTALE;
+
+    if (!(loc && loc->inode))
+        goto out;
+
+    if (!gf_uuid_is_null(loc->inode->gfid))
+        memcpy(req->gfid, loc->inode->gfid, 16);
+    else
+        memcpy(req->gfid, loc->gfid, 16);
+
+    GF_ASSERT_AND_GOTO_WITH_ERROR(this->name,
+                                  !gf_uuid_is_null(*((uuid_t *)req->gfid)), out,
+                                  op_errno, EINVAL);
+
+    dict_to_xdr(xdata, &req->xdata);
+
+    return 0;
+out:
+    return -op_errno;
+}
+
+int
+client_pre_readlink_v2(xlator_t *this, gfx_readlink_req *req, loc_t *loc,
+                       size_t size, dict_t *xdata)
+{
+    int op_errno = ESTALE;
+
+    if (!(loc && loc->inode))
+        goto out;
+
+    if (!gf_uuid_is_null(loc->inode->gfid))
+        memcpy(req->gfid, loc->inode->gfid, 16);
+    else
+        memcpy(req->gfid, loc->gfid, 16);
+
+    GF_ASSERT_AND_GOTO_WITH_ERROR(this->name,
+                                  !gf_uuid_is_null(*((uuid_t *)req->gfid)), out,
+                                  op_errno, EINVAL);
+    req->size = size;
+    dict_to_xdr(xdata, &req->xdata);
+    return 0;
+out:
+    return -op_errno;
+}
+
+int
+client_pre_mknod_v2(xlator_t *this, gfx_mknod_req *req, loc_t *loc, mode_t mode,
+                    dev_t rdev, mode_t umask, dict_t *xdata)
+{
+    int op_errno = ESTALE;
+
+    if (!(loc && loc->parent))
+        goto out;
+
+    if (!gf_uuid_is_null(loc->parent->gfid))
+        memcpy(req->pargfid, loc->parent->gfid, 16);
+    else
+        memcpy(req->pargfid, loc->pargfid, 16);
+
+    GF_ASSERT_AND_GOTO_WITH_ERROR(this->name,
+                                  !gf_uuid_is_null(*((uuid_t *)req->pargfid)),
+                                  out, op_errno, EINVAL);
+    req->bname = (char *)loc->name;
+    req->mode = mode;
+    req->dev = rdev;
+    req->umask = umask;
+
+    dict_to_xdr(xdata, &req->xdata);
+
+    return 0;
+out:
+    return -op_errno;
+}
+
+int
+client_pre_mkdir_v2(xlator_t *this, gfx_mkdir_req *req, loc_t *loc, mode_t mode,
+                    mode_t umask, dict_t *xdata)
+{
+    int op_errno = ESTALE;
+
+    if (!(loc && loc->parent))
+        goto out;
+
+    if (!gf_uuid_is_null(loc->parent->gfid))
+        memcpy(req->pargfid, loc->parent->gfid, 16);
+    else
+        memcpy(req->pargfid, loc->pargfid, 16);
+
+    GF_ASSERT_AND_GOTO_WITH_ERROR(this->name,
+                                  !gf_uuid_is_null(*((uuid_t *)req->pargfid)),
+                                  out, op_errno, EINVAL);
+
+    req->bname = (char *)loc->name;
+    req->mode = mode;
+    req->umask = umask;
+
+    dict_to_xdr(xdata, &req->xdata);
+
+    return 0;
+out:
+    return -op_errno;
+}
+
+int
+client_pre_unlink_v2(xlator_t *this, gfx_unlink_req *req, loc_t *loc,
+                     int32_t flags, dict_t *xdata)
+{
+    int op_errno = 0;
+
+    if (!(loc && loc->parent))
+        goto out;
+
+    if (!gf_uuid_is_null(loc->parent->gfid))
+        memcpy(req->pargfid, loc->parent->gfid, 16);
+    else
+        memcpy(req->pargfid, loc->pargfid, 16);
+
+    GF_ASSERT_AND_GOTO_WITH_ERROR(this->name,
+                                  !gf_uuid_is_null(*((uuid_t *)req->pargfid)),
+                                  out, op_errno, EINVAL);
+    req->bname = (char *)loc->name;
+    req->xflags = flags;
+
+    dict_to_xdr(xdata, &req->xdata);
+
+    return 0;
+out:
+    return -op_errno;
+}
+
+int
+client_pre_rmdir_v2(xlator_t *this, gfx_rmdir_req *req, loc_t *loc,
+                    int32_t flags, dict_t *xdata)
+{
+    int op_errno = ESTALE;
+
+    if (!(loc && loc->parent))
+        goto out;
+
+    if (!gf_uuid_is_null(loc->parent->gfid))
+        memcpy(req->pargfid, loc->parent->gfid, 16);
+    else
+        memcpy(req->pargfid, loc->pargfid, 16);
+
+    GF_ASSERT_AND_GOTO_WITH_ERROR(this->name,
+                                  !gf_uuid_is_null(*((uuid_t *)req->pargfid)),
+                                  out, op_errno, EINVAL);
+    req->bname = (char *)loc->name;
+    req->xflags = flags;
+
+    dict_to_xdr(xdata, &req->xdata);
+
+    return 0;
+out:
+    return -op_errno;
+}
+
+int
+client_pre_symlink_v2(xlator_t *this, gfx_symlink_req *req, loc_t *loc,
+                      const char *linkname, mode_t umask, dict_t *xdata)
+{
+    int op_errno = ESTALE;
+
+    if (!(loc && loc->parent))
+        goto out;
+
+    if (!gf_uuid_is_null(loc->parent->gfid))
+        memcpy(req->pargfid, loc->parent->gfid, 16);
+    else
+        memcpy(req->pargfid, loc->pargfid, 16);
+
+    GF_ASSERT_AND_GOTO_WITH_ERROR(this->name,
+                                  !gf_uuid_is_null(*((uuid_t *)req->pargfid)),
+                                  out, op_errno, EINVAL);
+    req->linkname = (char *)linkname;
+    req->bname = (char *)loc->name;
+    req->umask = umask;
+
+    dict_to_xdr(xdata, &req->xdata);
+    return 0;
+out:
+    return -op_errno;
+}
+
+int
+client_pre_rename_v2(xlator_t *this, gfx_rename_req *req, loc_t *oldloc,
+                     loc_t *newloc, dict_t *xdata)
+{
+    int op_errno = ESTALE;
+
+    if (!(oldloc && newloc && oldloc->parent && newloc->parent))
+        goto out;
+
+    if (!gf_uuid_is_null(oldloc->parent->gfid))
+        memcpy(req->oldgfid, oldloc->parent->gfid, 16);
+    else
+        memcpy(req->oldgfid, oldloc->pargfid, 16);
+
+    if (!gf_uuid_is_null(newloc->parent->gfid))
+        memcpy(req->newgfid, newloc->parent->gfid, 16);
+    else
+        memcpy(req->newgfid, newloc->pargfid, 16);
+
+    GF_ASSERT_AND_GOTO_WITH_ERROR(this->name,
+                                  !gf_uuid_is_null(*((uuid_t *)req->oldgfid)),
+                                  out, op_errno, EINVAL);
+    GF_ASSERT_AND_GOTO_WITH_ERROR(this->name,
+                                  !gf_uuid_is_null(*((uuid_t *)req->newgfid)),
+                                  out, op_errno, EINVAL);
+    req->oldbname = (char *)oldloc->name;
+    req->newbname = (char *)newloc->name;
+
+    dict_to_xdr(xdata, &req->xdata);
+
+    return 0;
+out:
+    return -op_errno;
+}
+
+int
+client_pre_link_v2(xlator_t *this, gfx_link_req *req, loc_t *oldloc,
+                   loc_t *newloc, dict_t *xdata)
+{
+    int op_errno = ESTALE;
+
+    if (!(oldloc && oldloc->inode && newloc && newloc->parent))
+        goto out;
+
+    if (!gf_uuid_is_null(oldloc->inode->gfid))
+        memcpy(req->oldgfid, oldloc->inode->gfid, 16);
+    else
+        memcpy(req->oldgfid, oldloc->gfid, 16);
+
+    if (!gf_uuid_is_null(newloc->parent->gfid))
+        memcpy(req->newgfid, newloc->parent->gfid, 16);
+    else
+        memcpy(req->newgfid, newloc->pargfid, 16);
+
+    GF_ASSERT_AND_GOTO_WITH_ERROR(this->name,
+                                  !gf_uuid_is_null(*((uuid_t *)req->oldgfid)),
+                                  out, op_errno, EINVAL);
+    GF_ASSERT_AND_GOTO_WITH_ERROR(this->name,
+                                  !gf_uuid_is_null(*((uuid_t *)req->newgfid)),
+                                  out, op_errno, EINVAL);
+    req->newbname = (char *)newloc->name;
+
+    dict_to_xdr(xdata, &req->xdata);
+
+    return 0;
+out:
+    return -op_errno;
+}
+
+int
+client_pre_truncate_v2(xlator_t *this, gfx_truncate_req *req, loc_t *loc,
+                       off_t offset, dict_t *xdata)
+{
+    int op_errno = ESTALE;
+
+    if (!(loc && loc->inode))
+        goto out;
+
+    if (!gf_uuid_is_null(loc->inode->gfid))
+        memcpy(req->gfid, loc->inode->gfid, 16);
+    else
+        memcpy(req->gfid, loc->gfid, 16);
+
+    GF_ASSERT_AND_GOTO_WITH_ERROR(this->name,
+                                  !gf_uuid_is_null(*((uuid_t *)req->gfid)), out,
+                                  op_errno, EINVAL);
+    req->offset = offset;
+
+    dict_to_xdr(xdata, &req->xdata);
+    return 0;
+out:
+    return -op_errno;
+}
+
+int
+client_pre_open_v2(xlator_t *this, gfx_open_req *req, loc_t *loc, fd_t *fd,
+                   int32_t flags, dict_t *xdata)
+{
+    int op_errno = ESTALE;
+
+    if (!(loc && loc->inode))
+        goto out;
+
+    if (!gf_uuid_is_null(loc->inode->gfid))
+        memcpy(req->gfid, loc->inode->gfid, 16);
+    else
+        memcpy(req->gfid, loc->gfid, 16);
+
+    GF_ASSERT_AND_GOTO_WITH_ERROR(this->name,
+                                  !gf_uuid_is_null(*((uuid_t *)req->gfid)), out,
+                                  op_errno, EINVAL);
+    req->flags = gf_flags_from_flags(flags);
+
+    dict_to_xdr(xdata, &req->xdata);
+
+    return 0;
+out:
+    return -op_errno;
+}
+
+int
+client_pre_readv_v2(xlator_t *this, gfx_read_req *req, fd_t *fd, size_t size,
+                    off_t offset, int32_t flags, dict_t *xdata)
+{
+    int64_t remote_fd = -1;
+    int op_errno = ESTALE;
+
+    CLIENT_GET_REMOTE_FD(this, fd, FALLBACK_TO_ANON_FD, remote_fd, op_errno,
+                         out);
+
+    req->size = size;
+    req->offset = offset;
+    req->fd = remote_fd;
+    req->flag = flags;
+
+    memcpy(req->gfid, fd->inode->gfid, 16);
+
+    dict_to_xdr(xdata, &req->xdata);
+
+    return 0;
+out:
+    return -op_errno;
+}
+
+int
+client_pre_writev_v2(xlator_t *this, gfx_write_req *req, fd_t *fd, size_t size,
+                     off_t offset, int32_t flags, dict_t **xdata)
+{
+    int64_t remote_fd = -1;
+    int op_errno = ESTALE;
+
+    CLIENT_GET_REMOTE_FD(this, fd, FALLBACK_TO_ANON_FD, remote_fd, op_errno,
+                         out);
+
+    req->size = size;
+    req->offset = offset;
+    req->fd = remote_fd;
+    req->flag = flags;
+
+    memcpy(req->gfid, fd->inode->gfid, 16);
+
+#ifdef GF_TESTING_IO_XDATA
+    if (!*xdata)
+        *xdata = dict_new();
+
+    ret = dict_set_str(*xdata, "testing-the-xdata-key",
+                       "testing-the-xdata-value");
+#endif
+
+    dict_to_xdr(*xdata, &req->xdata);
+
+    return 0;
+out:
+    return -op_errno;
+}
+
+int
+client_pre_copy_file_range_v2(xlator_t *this, gfx_copy_file_range_req *req,
+                              fd_t *fd_in, off64_t off_in, fd_t *fd_out,
+                              off64_t off_out, size_t size, int32_t flags,
+                              dict_t **xdata)
+{
+    int64_t remote_fd_in = -1;
+    int64_t remote_fd_out = -1;
+    int op_errno = ESTALE;
+
+    CLIENT_GET_REMOTE_FD(this, fd_in, FALLBACK_TO_ANON_FD, remote_fd_in,
+                         op_errno, out);
+
+    CLIENT_GET_REMOTE_FD(this, fd_out, FALLBACK_TO_ANON_FD, remote_fd_out,
+                         op_errno, out);
+    req->size = size;
+    req->off_in = off_in;
+    req->off_out = off_out;
+    req->fd_in = remote_fd_in;
+    req->fd_out = remote_fd_out;
+    req->flag = flags;
+
+    memcpy(req->gfid1, fd_in->inode->gfid, 16);
+    memcpy(req->gfid2, fd_out->inode->gfid, 16);
+
+    dict_to_xdr(*xdata, &req->xdata);
+
+    return 0;
+out:
+    return -op_errno;
+}
+
+int
+client_pre_statfs_v2(xlator_t *this, gfx_statfs_req *req, loc_t *loc,
+                     dict_t *xdata)
+{
+    int op_errno = ESTALE;
+
+    if (!loc)
+        goto out;
+
+    if (loc->inode) {
+        if (!gf_uuid_is_null(loc->inode->gfid))
+            memcpy(req->gfid, loc->inode->gfid, 16);
+        else
+            memcpy(req->gfid, loc->gfid, 16);
+    } else {
+        req->gfid[15] = 1;
+    }
+
+    GF_ASSERT_AND_GOTO_WITH_ERROR(this->name,
+                                  !gf_uuid_is_null(*((uuid_t *)req->gfid)), out,
+                                  op_errno, EINVAL);
+
+    dict_to_xdr(xdata, &req->xdata);
+
+    return 0;
+out:
+    return -op_errno;
+}
+
+int
+client_pre_flush_v2(xlator_t *this, gfx_flush_req *req, fd_t *fd, dict_t *xdata)
+{
+    int64_t remote_fd = -1;
+    int op_errno = ESTALE;
+
+    CLIENT_GET_REMOTE_FD(this, fd, DEFAULT_REMOTE_FD, remote_fd, op_errno, out);
+
+    req->fd = remote_fd;
+    memcpy(req->gfid, fd->inode->gfid, 16);
+
+    dict_to_xdr(xdata, &req->xdata);
+
+    return 0;
+out:
+    return -op_errno;
+}
+
+int
+client_pre_fsync_v2(xlator_t *this, gfx_fsync_req *req, fd_t *fd, int32_t flags,
+                    dict_t *xdata)
+{
+    int64_t remote_fd = -1;
+    int op_errno = 0;
+
+    CLIENT_GET_REMOTE_FD(this, fd, FALLBACK_TO_ANON_FD, remote_fd, op_errno,
+                         out);
+
+    req->fd = remote_fd;
+    req->data = flags;
+    memcpy(req->gfid, fd->inode->gfid, 16);
+
+    dict_to_xdr(xdata, &req->xdata);
+
+    return 0;
+out:
+    return -op_errno;
+}
+
+int
+client_pre_setxattr_v2(xlator_t *this, gfx_setxattr_req *req, loc_t *loc,
+                       dict_t *xattr, int32_t flags, dict_t *xdata)
+{
+    int op_errno = ESTALE;
+
+    if (!(loc && loc->inode))
+        goto out;
+
+    if (!gf_uuid_is_null(loc->inode->gfid))
+        memcpy(req->gfid, loc->inode->gfid, 16);
+    else
+        memcpy(req->gfid, loc->gfid, 16);
+
+    GF_ASSERT_AND_GOTO_WITH_ERROR(this->name,
+                                  !gf_uuid_is_null(*((uuid_t *)req->gfid)), out,
+                                  op_errno, EINVAL);
+    if (xattr) {
+        dict_to_xdr(xattr, &req->dict);
+    }
+
+    req->flags = flags;
+
+    dict_to_xdr(xdata, &req->xdata);
+
+    return 0;
+out:
+    return -op_errno;
+}
+
+int
+client_pre_getxattr_v2(xlator_t *this, gfx_getxattr_req *req, loc_t *loc,
+                       const char *name, dict_t *xdata)
+{
+    int op_errno = ESTALE;
+
+    if (!loc) {
+        op_errno = EINVAL;
+        goto out;
+    }
+
+    if (loc->inode && !gf_uuid_is_null(loc->inode->gfid))
+        memcpy(req->gfid, loc->inode->gfid, 16);
+    else
+        memcpy(req->gfid, loc->gfid, 16);
+
+    GF_ASSERT_AND_GOTO_WITH_ERROR(this->name,
+                                  !gf_uuid_is_null(*((uuid_t *)req->gfid)), out,
+                                  op_errno, EINVAL);
+    req->namelen = 1; /* Use it as a flag */
+
+    req->name = (char *)name;
+    if (!req->name) {
+        req->name = "";
+        req->namelen = 0;
+    }
+
+    dict_to_xdr(xdata, &req->xdata);
+
+    return 0;
+out:
+    return -op_errno;
+}
+
+int
+client_pre_removexattr_v2(xlator_t *this, gfx_removexattr_req *req, loc_t *loc,
+                          const char *name, dict_t *xdata)
+{
+    int op_errno = ESTALE;
+
+    if (!(loc && loc->inode))
+        goto out;
+
+    if (!gf_uuid_is_null(loc->inode->gfid))
+        memcpy(req->gfid, loc->inode->gfid, 16);
+    else
+        memcpy(req->gfid, loc->gfid, 16);
+
+    GF_ASSERT_AND_GOTO_WITH_ERROR(this->name,
+                                  !gf_uuid_is_null(*((uuid_t *)req->gfid)), out,
+                                  op_errno, EINVAL);
+    req->name = (char *)name;
+
+    dict_to_xdr(xdata, &req->xdata);
+
+    return 0;
+out:
+    return -op_errno;
+}
+
+int
+client_pre_opendir_v2(xlator_t *this, gfx_opendir_req *req, loc_t *loc,
+                      fd_t *fd, dict_t *xdata)
+{
+    int op_errno = ESTALE;
+
+    if (!(loc && loc->inode))
+        goto out;
+
+    if (!gf_uuid_is_null(loc->inode->gfid))
+        memcpy(req->gfid, loc->inode->gfid, 16);
+    else
+        memcpy(req->gfid, loc->gfid, 16);
+
+    GF_ASSERT_AND_GOTO_WITH_ERROR(this->name,
+                                  !gf_uuid_is_null(*((uuid_t *)req->gfid)), out,
+                                  op_errno, EINVAL);
+
+    dict_to_xdr(xdata, &req->xdata);
+
+    return 0;
+out:
+    return -op_errno;
+}
+
+int
+client_pre_fsyncdir_v2(xlator_t *this, gfx_fsyncdir_req *req, fd_t *fd,
+                       int32_t flags, dict_t *xdata)
+{
+    int32_t op_errno = ESTALE;
+    int64_t remote_fd = -1;
+
+    CLIENT_GET_REMOTE_FD(this, fd, DEFAULT_REMOTE_FD, remote_fd, op_errno, out);
+
+    req->fd = remote_fd;
+    req->data = flags;
+    memcpy(req->gfid, fd->inode->gfid, 16);
+
+    dict_to_xdr(xdata, &req->xdata);
+
+    return 0;
+out:
+    return -op_errno;
+}
+
+int
+client_pre_access_v2(xlator_t *this, gfx_access_req *req, loc_t *loc,
+                     int32_t mask, dict_t *xdata)
+{
+    int op_errno = ESTALE;
+
+    if (!(loc && loc->inode))
+        goto out;
+
+    if (!gf_uuid_is_null(loc->inode->gfid))
+        memcpy(req->gfid, loc->inode->gfid, 16);
+    else
+        memcpy(req->gfid, loc->gfid, 16);
+
+    GF_ASSERT_AND_GOTO_WITH_ERROR(this->name,
+                                  !gf_uuid_is_null(*((uuid_t *)req->gfid)), out,
+                                  op_errno, EINVAL);
+    req->mask = mask;
+
+    dict_to_xdr(xdata, &req->xdata);
+    return 0;
+out:
+    return -op_errno;
+}
+
+int
+client_pre_create_v2(xlator_t *this, gfx_create_req *req, loc_t *loc, fd_t *fd,
+                     mode_t mode, int32_t flags, mode_t umask, dict_t *xdata)
+{
+    int op_errno = ESTALE;
+
+    if (!(loc && loc->parent))
+        goto out;
+
+    if (!gf_uuid_is_null(loc->parent->gfid))
+        memcpy(req->pargfid, loc->parent->gfid, 16);
+    else
+        memcpy(req->pargfid, loc->pargfid, 16);
+
+    GF_ASSERT_AND_GOTO_WITH_ERROR(this->name,
+                                  !gf_uuid_is_null(*((uuid_t *)req->pargfid)),
+                                  out, op_errno, EINVAL);
+    req->bname = (char *)loc->name;
+    req->mode = mode;
+    req->flags = gf_flags_from_flags(flags);
+    req->umask = umask;
+
+    dict_to_xdr(xdata, &req->xdata);
+
+    return 0;
+out:
+    return -op_errno;
+}
+
+int
+client_pre_ftruncate_v2(xlator_t *this, gfx_ftruncate_req *req, fd_t *fd,
+                        off_t offset, dict_t *xdata)
+{
+    int64_t remote_fd = -1;
+    int op_errno = EINVAL;
+
+    CLIENT_GET_REMOTE_FD(this, fd, DEFAULT_REMOTE_FD, remote_fd, op_errno, out);
+
+    req->offset = offset;
+    req->fd = remote_fd;
+    memcpy(req->gfid, fd->inode->gfid, 16);
+
+    dict_to_xdr(xdata, &req->xdata);
+    return 0;
+out:
+    return -op_errno;
+}
+
+int
+client_pre_fstat_v2(xlator_t *this, gfx_fstat_req *req, fd_t *fd, dict_t *xdata)
+{
+    int64_t remote_fd = -1;
+    int op_errno = ESTALE;
+
+    CLIENT_GET_REMOTE_FD(this, fd, DEFAULT_REMOTE_FD, remote_fd, op_errno, out);
+
+    req->fd = remote_fd;
+    memcpy(req->gfid, fd->inode->gfid, 16);
+
+    dict_to_xdr(xdata, &req->xdata);
+
+    return 0;
+out:
+    return -op_errno;
+}
+
+int
+client_pre_lk_v2(xlator_t *this, gfx_lk_req *req, int32_t cmd,
+                 struct gf_flock *flock, fd_t *fd, dict_t *xdata)
+{
+    int64_t remote_fd = -1;
+    int op_errno = ESTALE;
+    int32_t gf_cmd = 0;
+    int32_t gf_type = 0;
+    int ret = 0;
+
+    CLIENT_GET_REMOTE_FD(this, fd, DEFAULT_REMOTE_FD, remote_fd, op_errno, out);
+
+    ret = client_cmd_to_gf_cmd(cmd, &gf_cmd);
+    if (ret) {
+        op_errno = EINVAL;
+        gf_smsg(this->name, GF_LOG_WARNING, EINVAL, PC_MSG_UNKNOWN_CMD,
+                "gf_cmd=%d", gf_cmd, NULL);
+        goto out;
+    }
+
+    switch (flock->l_type) {
+        case F_RDLCK:
+            gf_type = GF_LK_F_RDLCK;
+            break;
+        case F_WRLCK:
+            gf_type = GF_LK_F_WRLCK;
+            break;
+        case F_UNLCK:
+            gf_type = GF_LK_F_UNLCK;
+            break;
+    }
+
+    req->fd = remote_fd;
+    req->cmd = gf_cmd;
+    req->type = gf_type;
+    gf_proto_flock_from_flock(&req->flock, flock);
+
+    memcpy(req->gfid, fd->inode->gfid, 16);
+
+    dict_to_xdr(xdata, &req->xdata);
+
+    return 0;
+out:
+    return -op_errno;
+}
+
+int
+client_pre_lookup_v2(xlator_t *this, gfx_lookup_req *req, loc_t *loc,
+                     dict_t *xdata)
+{
+    int op_errno = ESTALE;
+
+    if (!(loc && loc->inode))
+        goto out;
+
+    if ((loc->parent) && (!gf_uuid_is_null(loc->parent->gfid)))
+        memcpy(req->pargfid, loc->parent->gfid, 16);
+    else
+        memcpy(req->pargfid, loc->pargfid, 16);
+
+    if ((loc->inode) && (!gf_uuid_is_null(loc->inode->gfid)))
+        memcpy(req->gfid, loc->inode->gfid, 16);
+    else
+        memcpy(req->gfid, loc->gfid, 16);
+
+    if (loc->name)
+        req->bname = (char *)loc->name;
+    else
+        req->bname = "";
+
+    if (xdata) {
+        dict_to_xdr(xdata, &req->xdata);
+    }
+    return 0;
+out:
+    return -op_errno;
+}
+
+int
+client_pre_readdir_v2(xlator_t *this, gfx_readdir_req *req, fd_t *fd,
+                      size_t size, off_t offset, dict_t *xdata)
+{
+    int64_t remote_fd = -1;
+    int op_errno = ESTALE;
+
+    CLIENT_GET_REMOTE_FD(this, fd, DEFAULT_REMOTE_FD, remote_fd, op_errno, out);
+
+    req->size = size;
+    req->offset = offset;
+    req->fd = remote_fd;
+
+    memcpy(req->gfid, fd->inode->gfid, 16);
+    dict_to_xdr(xdata, &req->xdata);
+
+    return 0;
+out:
+    return -op_errno;
+}
+
+int
+client_pre_inodelk_v2(xlator_t *this, gfx_inodelk_req *req, loc_t *loc, int cmd,
+                      struct gf_flock *flock, const char *volume, dict_t *xdata)
+{
+    int op_errno = ESTALE;
+    int32_t gf_cmd = 0;
+    int32_t gf_type = 0;
+
+    if (!(loc && loc->inode))
+        goto out;
+
+    if (!gf_uuid_is_null(loc->gfid))
+        memcpy(req->gfid, loc->gfid, 16);
+    else
+        memcpy(req->gfid, loc->inode->gfid, 16);
+
+    GF_ASSERT_AND_GOTO_WITH_ERROR(this->name,
+                                  !gf_uuid_is_null(*((uuid_t *)req->gfid)), out,
+                                  op_errno, EINVAL);
+    if (cmd == F_GETLK || cmd == F_GETLK64)
+        gf_cmd = GF_LK_GETLK;
+    else if (cmd == F_SETLK || cmd == F_SETLK64)
+        gf_cmd = GF_LK_SETLK;
+    else if (cmd == F_SETLKW || cmd == F_SETLKW64)
+        gf_cmd = GF_LK_SETLKW;
+    else {
+        gf_smsg(this->name, GF_LOG_WARNING, EINVAL, PC_MSG_UNKNOWN_CMD,
+                "gf_cmd=%d", gf_cmd, NULL);
+        op_errno = EINVAL;
+        goto out;
+    }
+
+    switch (flock->l_type) {
+        case F_RDLCK:
+            gf_type = GF_LK_F_RDLCK;
+            break;
+        case F_WRLCK:
+            gf_type = GF_LK_F_WRLCK;
+            break;
+        case F_UNLCK:
+            gf_type = GF_LK_F_UNLCK;
+            break;
+    }
+
+    req->volume = (char *)volume;
+    req->cmd = gf_cmd;
+    req->type = gf_type;
+    gf_proto_flock_from_flock(&req->flock, flock);
+
+    dict_to_xdr(xdata, &req->xdata);
+
+    return 0;
+out:
+    return -op_errno;
+}
+
+int
+client_pre_finodelk_v2(xlator_t *this, gfx_finodelk_req *req, fd_t *fd, int cmd,
+                       struct gf_flock *flock, const char *volume,
+                       dict_t *xdata)
+{
+    int op_errno = ESTALE;
+    int64_t remote_fd = -1;
+    int32_t gf_type = 0;
+    int32_t gf_cmd = 0;
+
+    CLIENT_GET_REMOTE_FD(this, fd, FALLBACK_TO_ANON_FD, remote_fd, op_errno,
+                         out);
+
+    if (cmd == F_GETLK || cmd == F_GETLK64)
+        gf_cmd = GF_LK_GETLK;
+    else if (cmd == F_SETLK || cmd == F_SETLK64)
+        gf_cmd = GF_LK_SETLK;
+    else if (cmd == F_SETLKW || cmd == F_SETLKW64)
+        gf_cmd = GF_LK_SETLKW;
+    else {
+        gf_smsg(this->name, GF_LOG_WARNING, EINVAL, PC_MSG_UNKNOWN_CMD,
+                "gf_cmd=%d", gf_cmd, NULL);
+        goto out;
+    }
+
+    switch (flock->l_type) {
+        case F_RDLCK:
+            gf_type = GF_LK_F_RDLCK;
+            break;
+        case F_WRLCK:
+            gf_type = GF_LK_F_WRLCK;
+            break;
+        case F_UNLCK:
+            gf_type = GF_LK_F_UNLCK;
+            break;
+    }
+
+    req->volume = (char *)volume;
+    req->fd = remote_fd;
+    req->cmd = gf_cmd;
+    req->type = gf_type;
+    gf_proto_flock_from_flock(&req->flock, flock);
+    memcpy(req->gfid, fd->inode->gfid, 16);
+
+    dict_to_xdr(xdata, &req->xdata);
+    return 0;
+out:
+    return -op_errno;
+}
+
+int
+client_pre_entrylk_v2(xlator_t *this, gfx_entrylk_req *req, loc_t *loc,
+                      entrylk_cmd cmd_entrylk, entrylk_type type,
+                      const char *volume, const char *basename, dict_t *xdata)
+{
+    int op_errno = ESTALE;
+
+    if (!(loc && loc->inode))
+        goto out;
+
+    if (!gf_uuid_is_null(loc->gfid))
+        memcpy(req->gfid, loc->gfid, 16);
+    else
+        memcpy(req->gfid, loc->inode->gfid, 16);
+
+    GF_ASSERT_AND_GOTO_WITH_ERROR(this->name,
+                                  !gf_uuid_is_null(*((uuid_t *)req->gfid)), out,
+                                  op_errno, EINVAL);
+    req->cmd = cmd_entrylk;
+    req->type = type;
+    req->volume = (char *)volume;
+    req->name = "";
+    if (basename) {
+        req->name = (char *)basename;
+        req->namelen = 1;
+    }
+
+    dict_to_xdr(xdata, &req->xdata);
+
+    return 0;
+out:
+    return -op_errno;
+}
+
+int
+client_pre_fentrylk_v2(xlator_t *this, gfx_fentrylk_req *req, fd_t *fd,
+                       entrylk_cmd cmd_entrylk, entrylk_type type,
+                       const char *volume, const char *basename, dict_t *xdata)
+{
+    int64_t remote_fd = -1;
+    int op_errno = ESTALE;
+
+    CLIENT_GET_REMOTE_FD(this, fd, DEFAULT_REMOTE_FD, remote_fd, op_errno, out);
+
+    req->fd = remote_fd;
+    req->cmd = cmd_entrylk;
+    req->type = type;
+    req->volume = (char *)volume;
+    req->name = "";
+    if (basename) {
+        req->name = (char *)basename;
+        req->namelen = 1;
+    }
+    memcpy(req->gfid, fd->inode->gfid, 16);
+
+    dict_to_xdr(xdata, &req->xdata);
+
+    return 0;
+out:
+    return -op_errno;
+}
+
+int
+client_pre_xattrop_v2(xlator_t *this, gfx_xattrop_req *req, loc_t *loc,
+                      dict_t *xattr, int32_t flags, dict_t *xdata)
+{
+    int op_errno = ESTALE;
+
+    if (!(loc && loc->inode))
+        goto out;
+
+    if (!gf_uuid_is_null(loc->inode->gfid))
+        memcpy(req->gfid, loc->inode->gfid, 16);
+    else
+        memcpy(req->gfid, loc->gfid, 16);
+
+    GF_ASSERT_AND_GOTO_WITH_ERROR(this->name,
+                                  !gf_uuid_is_null(*((uuid_t *)req->gfid)), out,
+                                  op_errno, EINVAL);
+    dict_to_xdr(xattr, &req->dict);
+
+    req->flags = flags;
+
+    dict_to_xdr(xdata, &req->xdata);
+
+    return 0;
+out:
+    return -op_errno;
+}
+
+int
+client_pre_fxattrop_v2(xlator_t *this, gfx_fxattrop_req *req, fd_t *fd,
+                       dict_t *xattr, int32_t flags, dict_t *xdata)
+{
+    int op_errno = ESTALE;
+    int64_t remote_fd = -1;
+
+    CLIENT_GET_REMOTE_FD(this, fd, FALLBACK_TO_ANON_FD, remote_fd, op_errno,
+                         out);
+
+    req->fd = remote_fd;
+    req->flags = flags;
+    memcpy(req->gfid, fd->inode->gfid, 16);
+
+    dict_to_xdr(xattr, &req->dict);
+
+    dict_to_xdr(xdata, &req->xdata);
+
+    return 0;
+out:
+    return -op_errno;
+}
+
+int
+client_pre_fgetxattr_v2(xlator_t *this, gfx_fgetxattr_req *req, fd_t *fd,
+                        const char *name, dict_t *xdata)
+{
+    int64_t remote_fd = -1;
+    int op_errno = ESTALE;
+
+    CLIENT_GET_REMOTE_FD(this, fd, DEFAULT_REMOTE_FD, remote_fd, op_errno, out);
+
+    req->namelen = 1; /* Use it as a flag */
+    req->fd = remote_fd;
+    req->name = (char *)name;
+    if (!req->name) {
+        req->name = "";
+        req->namelen = 0;
+    }
+    memcpy(req->gfid, fd->inode->gfid, 16);
+
+    dict_to_xdr(xdata, &req->xdata);
+
+    return 0;
+out:
+    return -op_errno;
+}
+
+int
+client_pre_fsetxattr_v2(xlator_t *this, gfx_fsetxattr_req *req, fd_t *fd,
+                        int32_t flags, dict_t *xattr, dict_t *xdata)
+{
+    int64_t remote_fd = -1;
+    int op_errno = ESTALE;
+
+    CLIENT_GET_REMOTE_FD(this, fd, DEFAULT_REMOTE_FD, remote_fd, op_errno, out);
+
+    req->fd = remote_fd;
+    req->flags = flags;
+    memcpy(req->gfid, fd->inode->gfid, 16);
+
+    if (xattr) {
+        dict_to_xdr(xattr, &req->dict);
+    }
+
+    dict_to_xdr(xdata, &req->xdata);
+
+    return 0;
+out:
+    return -op_errno;
+}
+
+int
+client_pre_rchecksum_v2(xlator_t *this, gfx_rchecksum_req *req, fd_t *fd,
+                        int32_t len, off_t offset, dict_t *xdata)
+{
+    int64_t remote_fd = -1;
+    int op_errno = ESTALE;
+
+    CLIENT_GET_REMOTE_FD(this, fd, DEFAULT_REMOTE_FD, remote_fd, op_errno, out);
+
+    req->len = len;
+    req->offset = offset;
+    req->fd = remote_fd;
+
+    dict_to_xdr(xdata, &req->xdata);
+
+    return 0;
+out:
+    return -op_errno;
+}
+
+int
+client_pre_setattr_v2(xlator_t *this, gfx_setattr_req *req, loc_t *loc,
+                      int32_t valid, struct iatt *stbuf, dict_t *xdata)
+{
+    int op_errno = ESTALE;
+
+    if (!(loc && loc->inode))
+        return -op_errno;
+
+    if (!gf_uuid_is_null(loc->inode->gfid))
+        memcpy(req->gfid, loc->inode->gfid, 16);
+    else
+        memcpy(req->gfid, loc->gfid, 16);
+
+    GF_ASSERT_AND_GOTO_WITH_ERROR(this->name,
+                                  !gf_uuid_is_null(*((uuid_t *)req->gfid)), out,
+                                  op_errno, EINVAL);
+
+    req->valid = valid;
+    gfx_stat_from_iattx(&req->stbuf, stbuf);
+
+    dict_to_xdr(xdata, &req->xdata);
+
+    return 0;
+out:
+    return -op_errno;
+}
+
+int
+client_pre_fsetattr_v2(xlator_t *this, gfx_fsetattr_req *req, fd_t *fd,
+                       int32_t valid, struct iatt *stbuf, dict_t *xdata)
+{
+    int op_errno = ESTALE;
+    int64_t remote_fd = -1;
+
+    CLIENT_GET_REMOTE_FD(this, fd, DEFAULT_REMOTE_FD, remote_fd, op_errno, out);
+
+    memcpy(req->gfid, fd->inode->gfid, 16);
+    req->fd = remote_fd;
+    req->valid = valid;
+    gfx_stat_from_iattx(&req->stbuf, stbuf);
+
+    dict_to_xdr(xdata, &req->xdata);
+    return 0;
+out:
+    return -op_errno;
+}
+
+int
+client_pre_readdirp_v2(xlator_t *this, gfx_readdirp_req *req, fd_t *fd,
+                       size_t size, off_t offset, dict_t *xdata)
+{
+    int op_errno = ESTALE;
+    int64_t remote_fd = -1;
+
+    CLIENT_GET_REMOTE_FD(this, fd, DEFAULT_REMOTE_FD, remote_fd, op_errno, out);
+
+    req->size = size;
+    req->offset = offset;
+    req->fd = remote_fd;
+    memcpy(req->gfid, fd->inode->gfid, 16);
+
+    /* dict itself is 'xdata' here */
+    dict_to_xdr(xdata, &req->xdata);
+
+    return 0;
+out:
+    return -op_errno;
+}
+
+int
+client_pre_fremovexattr_v2(xlator_t *this, gfx_fremovexattr_req *req, fd_t *fd,
+                           const char *name, dict_t *xdata)
+{
+    int64_t remote_fd = -1;
+    int op_errno = ESTALE;
+
+    if (!(fd && fd->inode))
+        goto out;
+
+    CLIENT_GET_REMOTE_FD(this, fd, DEFAULT_REMOTE_FD, remote_fd, op_errno, out);
+
+    memcpy(req->gfid, fd->inode->gfid, 16);
+    req->name = (char *)name;
+    req->fd = remote_fd;
+
+    dict_to_xdr(xdata, &req->xdata);
+
+    return 0;
+out:
+    return -op_errno;
+}
+
+int
+client_pre_fallocate_v2(xlator_t *this, gfx_fallocate_req *req, fd_t *fd,
+                        int32_t flags, off_t offset, size_t size, dict_t *xdata)
+{
+    int op_errno = ESTALE;
+    int64_t remote_fd = -1;
+
+    CLIENT_GET_REMOTE_FD(this, fd, DEFAULT_REMOTE_FD, remote_fd, op_errno, out);
+
+    req->fd = remote_fd;
+    req->flags = flags;
+    req->offset = offset;
+    req->size = size;
+    memcpy(req->gfid, fd->inode->gfid, 16);
+
+    dict_to_xdr(xdata, &req->xdata);
+    return 0;
+out:
+    return -op_errno;
+}
+
+int
+client_pre_discard_v2(xlator_t *this, gfx_discard_req *req, fd_t *fd,
+                      off_t offset, size_t size, dict_t *xdata)
+{
+    int op_errno = ESTALE;
+    int64_t remote_fd = -1;
+
+    CLIENT_GET_REMOTE_FD(this, fd, DEFAULT_REMOTE_FD, remote_fd, op_errno, out);
+
+    req->fd = remote_fd;
+    req->offset = offset;
+    req->size = size;
+    memcpy(req->gfid, fd->inode->gfid, 16);
+
+    dict_to_xdr(xdata, &req->xdata);
+    return 0;
+out:
+    return -op_errno;
+}
+
+int
+client_pre_zerofill_v2(xlator_t *this, gfx_zerofill_req *req, fd_t *fd,
+                       off_t offset, size_t size, dict_t *xdata)
+{
+    int op_errno = ESTALE;
+    int64_t remote_fd = -1;
+
+    CLIENT_GET_REMOTE_FD(this, fd, DEFAULT_REMOTE_FD, remote_fd, op_errno, out);
+
+    req->fd = remote_fd;
+    req->offset = offset;
+    req->size = size;
+    memcpy(req->gfid, fd->inode->gfid, 16);
+
+    dict_to_xdr(xdata, &req->xdata);
+    return 0;
+out:
+    return -op_errno;
+}
+
+int
+client_pre_ipc_v2(xlator_t *this, gfx_ipc_req *req, int32_t cmd, dict_t *xdata)
+{
+    req->op = cmd;
+
+    dict_to_xdr(xdata, &req->xdata);
+    return 0;
+}
+
+int
+client_pre_seek_v2(xlator_t *this, gfx_seek_req *req, fd_t *fd, off_t offset,
+                   gf_seek_what_t what, dict_t *xdata)
+{
+    int64_t remote_fd = -1;
+    int op_errno = ESTALE;
+
+    CLIENT_GET_REMOTE_FD(this, fd, DEFAULT_REMOTE_FD, remote_fd, op_errno, out);
+
+    memcpy(req->gfid, fd->inode->gfid, 16);
+    req->fd = remote_fd;
+    req->offset = offset;
+    req->what = what;
+
+    dict_to_xdr(xdata, &req->xdata);
+
+    return 0;
+out:
+    return -op_errno;
+}
+
+int
+client_pre_lease_v2(xlator_t *this, gfx_lease_req *req, loc_t *loc,
+                    struct gf_lease *lease, dict_t *xdata)
+{
+    int op_errno = 0;
+
+    if (!(loc && loc->inode))
+        goto out;
+
+    if (!gf_uuid_is_null(loc->inode->gfid))
+        memcpy(req->gfid, loc->inode->gfid, 16);
+    else
+        memcpy(req->gfid, loc->gfid, 16);
+
+    GF_ASSERT_AND_GOTO_WITH_ERROR(this->name,
+                                  !gf_uuid_is_null(*((uuid_t *)req->gfid)), out,
+                                  op_errno, EINVAL);
+
+    gf_proto_lease_from_lease(&req->lease, lease);
+
+    dict_to_xdr(xdata, &req->xdata);
+out:
+    return -op_errno;
+}
+
+int
+client_pre_put_v2(xlator_t *this, gfx_put_req *req, loc_t *loc, mode_t mode,
+                  mode_t umask, int32_t flags, size_t size, off_t offset,
+                  dict_t *xattr, dict_t *xdata)
+{
+    int op_errno = ESTALE;
+
+    if (!(loc && loc->parent))
+        goto out;
+
+    if (!gf_uuid_is_null(loc->parent->gfid))
+        memcpy(req->pargfid, loc->parent->gfid, 16);
+    else
+        memcpy(req->pargfid, loc->pargfid, 16);
+
+    GF_ASSERT_AND_GOTO_WITH_ERROR(this->name,
+                                  !gf_uuid_is_null(*((uuid_t *)req->pargfid)),
+                                  out, op_errno, EINVAL);
+    req->bname = (char *)loc->name;
+    req->mode = mode;
+    req->umask = umask;
+    req->flag = gf_flags_from_flags(flags);
+    req->size = size;
+    req->offset = offset;
+
+    if (xattr)
+        dict_to_xdr(xattr, &req->xattr);
+
+    dict_to_xdr(xdata, &req->xdata);
+
+    return 0;
+out:
+    return -op_errno;
+}
+
+int
+client_post_create_v2(xlator_t *this, gfx_create_rsp *rsp, struct iatt *stbuf,
+                      struct iatt *preparent, struct iatt *postparent,
+                      clnt_local_t *local, dict_t **xdata)
+{
+    if (-1 != rsp->op_ret) {
+        gfx_stat_to_iattx(&rsp->stat, stbuf);
+
+        gfx_stat_to_iattx(&rsp->preparent, preparent);
+        gfx_stat_to_iattx(&rsp->postparent, postparent);
+        gf_uuid_copy(local->loc.gfid, stbuf->ia_gfid);
+    }
+    return xdr_to_dict(&rsp->xdata, xdata);
+}
+
+int
+client_post_lease_v2(xlator_t *this, gfx_lease_rsp *rsp, struct gf_lease *lease,
+                     dict_t **xdata)
+{
+    if (rsp->op_ret >= 0) {
+        gf_proto_lease_to_lease(&rsp->lease, lease);
+    }
+
+    return xdr_to_dict(&rsp->xdata, xdata);
+}
+
+int
+client_post_lk_v2(xlator_t *this, gfx_lk_rsp *rsp, struct gf_flock *lock,
+                  dict_t **xdata)
+{
+    if (rsp->op_ret >= 0) {
+        gf_proto_flock_to_flock(&rsp->flock, lock);
+    }
+    return xdr_to_dict(&rsp->xdata, xdata);
+}
+
+int
+client_post_readdir_v2(xlator_t *this, gfx_readdir_rsp *rsp,
+                       gf_dirent_t *entries, dict_t **xdata)
+{
+    if (rsp->op_ret > 0) {
+        unserialize_rsp_dirent_v2(this, rsp, entries);
+    }
+    return xdr_to_dict(&rsp->xdata, xdata);
+}
+
+int
+client_post_readdirp_v2(xlator_t *this, gfx_readdirp_rsp *rsp, fd_t *fd,
+                        gf_dirent_t *entries, dict_t **xdata)
+{
+    if (rsp->op_ret > 0) {
+        unserialize_rsp_direntp_v2(this, fd, rsp, entries);
+    }
+    return xdr_to_dict(&rsp->xdata, xdata);
+}
+
+int
+client_post_rename_v2(xlator_t *this, gfx_rename_rsp *rsp, struct iatt *stbuf,
+                      struct iatt *preoldparent, struct iatt *postoldparent,
+                      struct iatt *prenewparent, struct iatt *postnewparent,
+                      dict_t **xdata)
+{
+    if (-1 != rsp->op_ret) {
+        gfx_stat_to_iattx(&rsp->stat, stbuf);
+
+        gfx_stat_to_iattx(&rsp->preoldparent, preoldparent);
+        gfx_stat_to_iattx(&rsp->postoldparent, postoldparent);
+
+        gfx_stat_to_iattx(&rsp->prenewparent, prenewparent);
+        gfx_stat_to_iattx(&rsp->postnewparent, postnewparent);
+    }
+
+    return xdr_to_dict(&rsp->xdata, xdata);
+}
diff --git a/xlators/protocol/client/src/client-common.h b/xlators/protocol/client/src/client-common.h
new file mode 100644
index 00000000000..a2043d8742a
--- /dev/null
+++ b/xlators/protocol/client/src/client-common.h
@@ -0,0 +1,630 @@
+/*
+  Copyright (c); 2016 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later);, or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
+*/
+
+#ifndef __CLIENT_COMMON_H__
+#define __CLIENT_COMMON_H__
+
+#include <glusterfs/dict.h>
+#include <glusterfs/xlator.h>
+#include "rpc-common-xdr.h"
+#include "glusterfs3-xdr.h"
+#include "glusterfs4-xdr.h"
+#include "glusterfs3.h"
+#include "client.h"
+
+int
+client_pre_stat(xlator_t *this, gfs3_stat_req *req, loc_t *loc, dict_t *xdata);
+
+int
+client_pre_readlink(xlator_t *this, gfs3_readlink_req *req, loc_t *loc,
+                    size_t size, dict_t *xdata);
+
+int
+client_pre_mknod(xlator_t *this, gfs3_mknod_req *req, loc_t *loc, mode_t mode,
+                 dev_t rdev, mode_t umask, dict_t *xdata);
+
+int
+client_pre_mkdir(xlator_t *this, gfs3_mkdir_req *req, loc_t *loc, mode_t mode,
+                 mode_t umask, dict_t *xdata);
+
+int
+client_pre_unlink(xlator_t *this, gfs3_unlink_req *req, loc_t *loc,
+                  int32_t flags, dict_t *xdata);
+
+int
+client_pre_rmdir(xlator_t *this, gfs3_rmdir_req *req, loc_t *loc, int32_t flags,
+                 dict_t *xdata);
+
+int
+client_pre_symlink(xlator_t *this, gfs3_symlink_req *req, loc_t *loc,
+                   const char *linkname, mode_t umask, dict_t *xdata);
+
+int
+client_pre_rename(xlator_t *this, gfs3_rename_req *req, loc_t *oldloc,
+                  loc_t *newloc, dict_t *xdata);
+
+int
+client_pre_link(xlator_t *this, gfs3_link_req *req, loc_t *oldloc,
+                loc_t *newloc, dict_t *xdata);
+
+int
+client_pre_truncate(xlator_t *this, gfs3_truncate_req *req, loc_t *loc,
+                    off_t offset, dict_t *xdata);
+
+int
+client_pre_open(xlator_t *this, gfs3_open_req *req, loc_t *loc, fd_t *fd,
+                int32_t flags, dict_t *xdata);
+
+int
+client_pre_readv(xlator_t *this, gfs3_read_req *req, fd_t *fd, size_t size,
+                 off_t offset, int32_t flags, dict_t *xdata);
+
+int
+client_pre_writev(xlator_t *this, gfs3_write_req *req, fd_t *fd, size_t size,
+                  off_t offset, int32_t flags, dict_t **xdata);
+
+int
+client_pre_statfs(xlator_t *this, gfs3_statfs_req *req, loc_t *loc,
+                  dict_t *xdata);
+
+int
+client_pre_flush(xlator_t *this, gfs3_flush_req *req, fd_t *fd, dict_t *xdata);
+
+int
+client_pre_fsync(xlator_t *this, gfs3_fsync_req *req, fd_t *fd, int32_t flags,
+                 dict_t *xdata);
+
+int
+client_pre_setxattr(xlator_t *this, gfs3_setxattr_req *req, loc_t *loc,
+                    dict_t *xattr, int32_t flags, dict_t *xdata);
+
+int
+client_pre_getxattr(xlator_t *this, gfs3_getxattr_req *req, loc_t *loc,
+                    const char *name, dict_t *xdata);
+
+int
+client_pre_removexattr(xlator_t *this, gfs3_removexattr_req *req, loc_t *loc,
+                       const char *name, dict_t *xdata);
+
+int
+client_pre_opendir(xlator_t *this, gfs3_opendir_req *req, loc_t *loc, fd_t *fd,
+                   dict_t *xdata);
+
+int
+client_pre_fsyncdir(xlator_t *this, gfs3_fsyncdir_req *req, fd_t *fd,
+                    int32_t flags, dict_t *xdata);
+
+int
+client_pre_access(xlator_t *this, gfs3_access_req *req, loc_t *loc,
+                  int32_t mask, dict_t *xdata);
+
+int
+client_pre_create(xlator_t *this, gfs3_create_req *req, loc_t *loc, fd_t *fd,
+                  mode_t mode, int32_t flags, mode_t umask, dict_t *xdata);
+
+int
+client_pre_ftruncate(xlator_t *this, gfs3_ftruncate_req *req, fd_t *fd,
+                     off_t offset, dict_t *xdata);
+
+int
+client_pre_fstat(xlator_t *this, gfs3_fstat_req *req, fd_t *fd, dict_t *xdata);
+
+int
+client_pre_lk(xlator_t *this, gfs3_lk_req *req, int32_t cmd,
+              struct gf_flock *flock, fd_t *fd, dict_t *xdata);
+
+int
+client_pre_lookup(xlator_t *this, gfs3_lookup_req *req, loc_t *loc,
+                  dict_t *xdata);
+
+int
+client_pre_readdir(xlator_t *this, gfs3_readdir_req *req, fd_t *fd, size_t size,
+                   off_t offset, dict_t *xdata);
+
+int
+client_pre_inodelk(xlator_t *this, gfs3_inodelk_req *req, loc_t *loc, int cmd,
+                   struct gf_flock *flock, const char *volume, dict_t *xdata);
+
+int
+client_pre_finodelk(xlator_t *this, gfs3_finodelk_req *req, fd_t *fd, int cmd,
+                    struct gf_flock *flock, const char *volume, dict_t *xdata);
+
+int
+client_pre_entrylk(xlator_t *this, gfs3_entrylk_req *req, loc_t *loc,
+                   entrylk_cmd cmd_entrylk, entrylk_type type,
+                   const char *volume, const char *basename, dict_t *xdata);
+
+int
+client_pre_fentrylk(xlator_t *this, gfs3_fentrylk_req *req, fd_t *fd,
+                    entrylk_cmd cmd_entrylk, entrylk_type type,
+                    const char *volume, const char *basename, dict_t *xdata);
+
+int
+client_pre_xattrop(xlator_t *this, gfs3_xattrop_req *req, loc_t *loc,
+                   dict_t *xattr, int32_t flags, dict_t *xdata);
+
+int
+client_pre_fxattrop(xlator_t *this, gfs3_fxattrop_req *req, fd_t *fd,
+                    dict_t *xattr, int32_t flags, dict_t *xdata);
+
+int
+client_pre_fgetxattr(xlator_t *this, gfs3_fgetxattr_req *req, fd_t *fd,
+                     const char *name, dict_t *xdata);
+
+int
+client_pre_fsetxattr(xlator_t *this, gfs3_fsetxattr_req *req, fd_t *fd,
+                     int32_t flags, dict_t *xattr, dict_t *xdata);
+int
+client_pre_seek(xlator_t *this, gfs3_seek_req *req, fd_t *fd, off_t offset,
+                gf_seek_what_t what, dict_t *xdata);
+
+int
+client_pre_rchecksum(xlator_t *this, gfs3_rchecksum_req *req, fd_t *fd,
+                     int32_t len, off_t offset, dict_t *xdata);
+
+int
+client_pre_setattr(xlator_t *this, gfs3_setattr_req *req, loc_t *loc,
+                   int32_t valid, struct iatt *stbuf, dict_t *xdata);
+int
+client_pre_fsetattr(xlator_t *this, gfs3_fsetattr_req *req, fd_t *fd,
+                    int32_t valid, struct iatt *stbuf, dict_t *xdata);
+
+int
+client_pre_readdirp(xlator_t *this, gfs3_readdirp_req *req, fd_t *fd,
+                    size_t size, off_t offset, dict_t *xdata);
+
+int
+client_pre_fremovexattr(xlator_t *this, gfs3_fremovexattr_req *req, fd_t *fd,
+                        const char *name, dict_t *xdata);
+
+int
+client_pre_fallocate(xlator_t *this, gfs3_fallocate_req *req, fd_t *fd,
+                     int32_t flags, off_t offset, size_t size, dict_t *xdata);
+int
+client_pre_discard(xlator_t *this, gfs3_discard_req *req, fd_t *fd,
+                   off_t offset, size_t size, dict_t *xdata);
+int
+client_pre_zerofill(xlator_t *this, gfs3_zerofill_req *req, fd_t *fd,
+                    off_t offset, size_t size, dict_t *xdata);
+int
+client_pre_ipc(xlator_t *this, gfs3_ipc_req *req, int32_t cmd, dict_t *xdata);
+
+int
+client_pre_lease(xlator_t *this, gfs3_lease_req *req, loc_t *loc,
+                 struct gf_lease *lease, dict_t *xdata);
+
+int
+client_post_stat(xlator_t *this, gfs3_stat_rsp *rsp, struct iatt *iatt,
+                 dict_t **xdata);
+
+int
+client_post_readlink(xlator_t *this, gfs3_readlink_rsp *rsp, struct iatt *iatt,
+                     dict_t **xdata);
+
+int
+client_post_mknod(xlator_t *this, gfs3_mknod_rsp *rsp, struct iatt *stbuf,
+                  struct iatt *preparent, struct iatt *postparent,
+                  dict_t **xdata);
+
+int
+client_post_mkdir(xlator_t *this, gfs3_mkdir_rsp *rsp, struct iatt *stbuf,
+                  struct iatt *preparent, struct iatt *postparent,
+                  dict_t **xdata);
+
+int
+client_post_unlink(xlator_t *this, gfs3_unlink_rsp *rsp, struct iatt *preparent,
+                   struct iatt *postparent, dict_t **xdata);
+
+int
+client_post_rmdir(xlator_t *this, gfs3_rmdir_rsp *rsp, struct iatt *preparent,
+                  struct iatt *postparent, dict_t **xdata);
+
+int
+client_post_symlink(xlator_t *this, gfs3_symlink_rsp *rsp, struct iatt *stbuf,
+                    struct iatt *preparent, struct iatt *postparent,
+                    dict_t **xdata);
+
+int
+client_post_rename(xlator_t *this, gfs3_rename_rsp *rsp, struct iatt *stbuf,
+                   struct iatt *preoldparent, struct iatt *postoldparent,
+                   struct iatt *prenewparent, struct iatt *postnewparent,
+                   dict_t **xdata);
+int
+client_post_link(xlator_t *this, gfs3_link_rsp *rsp, struct iatt *stbuf,
+                 struct iatt *preparent, struct iatt *postparent,
+                 dict_t **xdata);
+
+int
+client_post_truncate(xlator_t *this, gfs3_truncate_rsp *rsp,
+                     struct iatt *prestat, struct iatt *poststat,
+                     dict_t **xdata);
+
+int
+client_post_open(xlator_t *this, gfs3_open_rsp *rsp, dict_t **xdata);
+
+int
+client_post_readv(xlator_t *this, gfs3_read_rsp *rsp, struct iobref **iobref,
+                  struct iobref *rsp_iobref, struct iatt *stat,
+                  struct iovec *vector, struct iovec *rsp_vector, int *rspcount,
+                  dict_t **xdata);
+
+int
+client_post_writev(xlator_t *this, gfs3_write_rsp *rsp, struct iatt *prestat,
+                   struct iatt *poststat, dict_t **xdata);
+
+int
+client_post_statfs(xlator_t *this, gfs3_statfs_rsp *rsp, struct statvfs *statfs,
+                   dict_t **xdata);
+
+int
+client_post_flush(xlator_t *this, gf_common_rsp *rsp, dict_t **xdata);
+
+int
+client_post_fsync(xlator_t *this, gfs3_fsync_rsp *rsp, struct iatt *prestat,
+                  struct iatt *poststat, dict_t **xdata);
+int
+client_post_setxattr(xlator_t *this, gf_common_rsp *rsp, dict_t **xdata);
+
+int
+client_post_getxattr(xlator_t *this, gfs3_getxattr_rsp *rsp, dict_t **dict,
+                     dict_t **xdata);
+
+int
+client_post_removexattr(xlator_t *this, gf_common_rsp *rsp, dict_t **xdata);
+
+int
+client_post_opendir(xlator_t *this, gfs3_opendir_rsp *rsp, dict_t **xdata);
+
+int
+client_post_fsyncdir(xlator_t *this, gf_common_rsp *rsp, dict_t **xdata);
+
+int
+client_post_access(xlator_t *this, gf_common_rsp *rsp, dict_t **xdata);
+
+int
+client_post_create(xlator_t *this, gfs3_create_rsp *rsp, struct iatt *stbuf,
+                   struct iatt *preparent, struct iatt *postparent,
+                   clnt_local_t *local, dict_t **xdata);
+
+int
+client_post_ftruncate(xlator_t *this, gfs3_ftruncate_rsp *rsp,
+                      struct iatt *prestat, struct iatt *poststat,
+                      dict_t **xdata);
+
+int
+client_post_fstat(xlator_t *this, gfs3_fstat_rsp *rsp, struct iatt *stat,
+                  dict_t **xdata);
+
+int
+client_post_lk(xlator_t *this, gfs3_lk_rsp *rsp, struct gf_flock *lock,
+               dict_t **xdata);
+
+int
+client_post_lookup(xlator_t *this, gfs3_lookup_rsp *rsp, struct iatt *stbuf,
+                   struct iatt *postparent, dict_t **xdata);
+
+int
+client_post_readdir(xlator_t *this, gfs3_readdir_rsp *rsp, gf_dirent_t *entries,
+                    dict_t **xdata);
+
+int
+client_post_inodelk(xlator_t *this, gf_common_rsp *rsp, dict_t **xdata);
+
+int
+client_post_finodelk(xlator_t *this, gf_common_rsp *rsp, dict_t **xdata);
+
+int
+client_post_entrylk(xlator_t *this, gf_common_rsp *rsp, dict_t **xdata);
+
+int
+client_post_fentrylk(xlator_t *this, gf_common_rsp *rsp, dict_t **xdata);
+
+int
+client_post_xattrop(xlator_t *this, gfs3_xattrop_rsp *rsp, dict_t **dict,
+                    dict_t **xdata);
+
+int
+client_post_fxattrop(xlator_t *this, gfs3_fxattrop_rsp *rsp, dict_t **dict,
+                     dict_t **xdata);
+
+int
+client_post_fgetxattr(xlator_t *this, gfs3_fgetxattr_rsp *rsp, dict_t **dict,
+                      dict_t **xdata);
+
+int
+client_post_fsetxattr(xlator_t *this, gf_common_rsp *rsp, dict_t **xdata);
+
+int
+client_post_rchecksum(xlator_t *this, gfs3_rchecksum_rsp *rsp, dict_t **xdata);
+
+int
+client_post_setattr(xlator_t *this, gfs3_setattr_rsp *rsp, struct iatt *prestat,
+                    struct iatt *poststat, dict_t **xdata);
+
+int
+client_post_fsetattr(xlator_t *this, gfs3_fsetattr_rsp *rsp,
+                     struct iatt *prestat, struct iatt *poststat,
+                     dict_t **xdata);
+
+int
+client_post_readdirp(xlator_t *this, gfs3_readdirp_rsp *rsp, fd_t *fd,
+                     gf_dirent_t *entries, dict_t **xdata);
+
+int
+client_post_fremovexattr(xlator_t *this, gf_common_rsp *rsp, dict_t **xdata);
+
+int
+client_post_fallocate(xlator_t *this, gfs3_fallocate_rsp *rsp,
+                      struct iatt *prestat, struct iatt *poststat,
+                      dict_t **xdata);
+
+int
+client_post_discard(xlator_t *this, gfs3_discard_rsp *rsp, struct iatt *prestat,
+                    struct iatt *poststat, dict_t **xdata);
+
+int
+client_post_zerofill(xlator_t *this, gfs3_zerofill_rsp *rsp,
+                     struct iatt *prestat, struct iatt *poststat,
+                     dict_t **xdata);
+
+int
+client_post_ipc(xlator_t *this, gfs3_ipc_rsp *rsp, dict_t **xdata);
+
+int
+client_post_seek(xlator_t *this, gfs3_seek_rsp *rsp, dict_t **xdata);
+
+int
+client_post_lease(xlator_t *this, gfs3_lease_rsp *rsp, struct gf_lease *lease,
+                  dict_t **xdata);
+
+/* New functions for version 4 */
+int
+client_post_common_dict(xlator_t *this, gfx_common_dict_rsp *rsp, dict_t **dict,
+                        dict_t **xdata);
+int
+client_post_common_3iatt(xlator_t *this, gfx_common_3iatt_rsp *rsp,
+                         struct iatt *iatt, struct iatt *iatt2,
+                         struct iatt *iatt3, dict_t **xdata);
+int
+client_post_common_2iatt(xlator_t *this, gfx_common_2iatt_rsp *rsp,
+                         struct iatt *iatt, struct iatt *iatt2, dict_t **xdata);
+int
+client_post_common_iatt(xlator_t *this, gfx_common_iatt_rsp *rsp,
+                        struct iatt *iatt, dict_t **xdata);
+int
+client_post_common_rsp(xlator_t *this, gfx_common_rsp *rsp, dict_t **xdata);
+
+int
+client_pre_stat_v2(xlator_t *this, gfx_stat_req *req, loc_t *loc,
+                   dict_t *xdata);
+
+int
+client_pre_readlink_v2(xlator_t *this, gfx_readlink_req *req, loc_t *loc,
+                       size_t size, dict_t *xdata);
+
+int
+client_pre_mknod_v2(xlator_t *this, gfx_mknod_req *req, loc_t *loc, mode_t mode,
+                    dev_t rdev, mode_t umask, dict_t *xdata);
+
+int
+client_pre_mkdir_v2(xlator_t *this, gfx_mkdir_req *req, loc_t *loc, mode_t mode,
+                    mode_t umask, dict_t *xdata);
+
+int
+client_pre_unlink_v2(xlator_t *this, gfx_unlink_req *req, loc_t *loc,
+                     int32_t flags, dict_t *xdata);
+
+int
+client_pre_rmdir_v2(xlator_t *this, gfx_rmdir_req *req, loc_t *loc,
+                    int32_t flags, dict_t *xdata);
+
+int
+client_pre_symlink_v2(xlator_t *this, gfx_symlink_req *req, loc_t *loc,
+                      const char *linkname, mode_t umask, dict_t *xdata);
+
+int
+client_pre_rename_v2(xlator_t *this, gfx_rename_req *req, loc_t *oldloc,
+                     loc_t *newloc, dict_t *xdata);
+
+int
+client_pre_link_v2(xlator_t *this, gfx_link_req *req, loc_t *oldloc,
+                   loc_t *newloc, dict_t *xdata);
+
+int
+client_pre_truncate_v2(xlator_t *this, gfx_truncate_req *req, loc_t *loc,
+                       off_t offset, dict_t *xdata);
+
+int
+client_pre_open_v2(xlator_t *this, gfx_open_req *req, loc_t *loc, fd_t *fd,
+                   int32_t flags, dict_t *xdata);
+
+int
+client_pre_readv_v2(xlator_t *this, gfx_read_req *req, fd_t *fd, size_t size,
+                    off_t offset, int32_t flags, dict_t *xdata);
+
+int
+client_pre_writev_v2(xlator_t *this, gfx_write_req *req, fd_t *fd, size_t size,
+                     off_t offset, int32_t flags, dict_t **xdata);
+
+int
+client_pre_statfs_v2(xlator_t *this, gfx_statfs_req *req, loc_t *loc,
+                     dict_t *xdata);
+
+int
+client_pre_flush_v2(xlator_t *this, gfx_flush_req *req, fd_t *fd,
+                    dict_t *xdata);
+
+int
+client_pre_fsync_v2(xlator_t *this, gfx_fsync_req *req, fd_t *fd, int32_t flags,
+                    dict_t *xdata);
+
+int
+client_pre_setxattr_v2(xlator_t *this, gfx_setxattr_req *req, loc_t *loc,
+                       dict_t *xattr, int32_t flags, dict_t *xdata);
+
+int
+client_pre_getxattr_v2(xlator_t *this, gfx_getxattr_req *req, loc_t *loc,
+                       const char *name, dict_t *xdata);
+
+int
+client_pre_removexattr_v2(xlator_t *this, gfx_removexattr_req *req, loc_t *loc,
+                          const char *name, dict_t *xdata);
+
+int
+client_pre_opendir_v2(xlator_t *this, gfx_opendir_req *req, loc_t *loc,
+                      fd_t *fd, dict_t *xdata);
+
+int
+client_pre_fsyncdir_v2(xlator_t *this, gfx_fsyncdir_req *req, fd_t *fd,
+                       int32_t flags, dict_t *xdata);
+
+int
+client_pre_access_v2(xlator_t *this, gfx_access_req *req, loc_t *loc,
+                     int32_t mask, dict_t *xdata);
+
+int
+client_pre_create_v2(xlator_t *this, gfx_create_req *req, loc_t *loc, fd_t *fd,
+                     mode_t mode, int32_t flags, mode_t umask, dict_t *xdata);
+
+int
+client_pre_ftruncate_v2(xlator_t *this, gfx_ftruncate_req *req, fd_t *fd,
+                        off_t offset, dict_t *xdata);
+
+int
+client_pre_fstat_v2(xlator_t *this, gfx_fstat_req *req, fd_t *fd,
+                    dict_t *xdata);
+
+int
+client_pre_lk_v2(xlator_t *this, gfx_lk_req *req, int32_t cmd,
+                 struct gf_flock *flock, fd_t *fd, dict_t *xdata);
+
+int
+client_pre_lookup_v2(xlator_t *this, gfx_lookup_req *req, loc_t *loc,
+                     dict_t *xdata);
+
+int
+client_pre_readdir_v2(xlator_t *this, gfx_readdir_req *req, fd_t *fd,
+                      size_t size, off_t offset, dict_t *xdata);
+
+int
+client_pre_inodelk_v2(xlator_t *this, gfx_inodelk_req *req, loc_t *loc, int cmd,
+                      struct gf_flock *flock, const char *volume,
+                      dict_t *xdata);
+
+int
+client_pre_finodelk_v2(xlator_t *this, gfx_finodelk_req *req, fd_t *fd, int cmd,
+                       struct gf_flock *flock, const char *volume,
+                       dict_t *xdata);
+
+int
+client_pre_entrylk_v2(xlator_t *this, gfx_entrylk_req *req, loc_t *loc,
+                      entrylk_cmd cmd_entrylk, entrylk_type type,
+                      const char *volume, const char *basename, dict_t *xdata);
+
+int
+client_pre_fentrylk_v2(xlator_t *this, gfx_fentrylk_req *req, fd_t *fd,
+                       entrylk_cmd cmd_entrylk, entrylk_type type,
+                       const char *volume, const char *basename, dict_t *xdata);
+
+int
+client_pre_xattrop_v2(xlator_t *this, gfx_xattrop_req *req, loc_t *loc,
+                      dict_t *xattr, int32_t flags, dict_t *xdata);
+
+int
+client_pre_fxattrop_v2(xlator_t *this, gfx_fxattrop_req *req, fd_t *fd,
+                       dict_t *xattr, int32_t flags, dict_t *xdata);
+
+int
+client_pre_fgetxattr_v2(xlator_t *this, gfx_fgetxattr_req *req, fd_t *fd,
+                        const char *name, dict_t *xdata);
+
+int
+client_pre_fsetxattr_v2(xlator_t *this, gfx_fsetxattr_req *req, fd_t *fd,
+                        int32_t flags, dict_t *xattr, dict_t *xdata);
+int
+client_pre_seek_v2(xlator_t *this, gfx_seek_req *req, fd_t *fd, off_t offset,
+                   gf_seek_what_t what, dict_t *xdata);
+
+int
+client_pre_rchecksum_v2(xlator_t *this, gfx_rchecksum_req *req, fd_t *fd,
+                        int32_t len, off_t offset, dict_t *xdata);
+
+int
+client_pre_setattr_v2(xlator_t *this, gfx_setattr_req *req, loc_t *loc,
+                      int32_t valid, struct iatt *stbuf, dict_t *xdata);
+int
+client_pre_fsetattr_v2(xlator_t *this, gfx_fsetattr_req *req, fd_t *fd,
+                       int32_t valid, struct iatt *stbuf, dict_t *xdata);
+
+int
+client_pre_readdirp_v2(xlator_t *this, gfx_readdirp_req *req, fd_t *fd,
+                       size_t size, off_t offset, dict_t *xdata);
+
+int
+client_pre_fremovexattr_v2(xlator_t *this, gfx_fremovexattr_req *req, fd_t *fd,
+                           const char *name, dict_t *xdata);
+
+int
+client_pre_fallocate_v2(xlator_t *this, gfx_fallocate_req *req, fd_t *fd,
+                        int32_t flags, off_t offset, size_t size,
+                        dict_t *xdata);
+int
+client_pre_discard_v2(xlator_t *this, gfx_discard_req *req, fd_t *fd,
+                      off_t offset, size_t size, dict_t *xdata);
+int
+client_pre_zerofill_v2(xlator_t *this, gfx_zerofill_req *req, fd_t *fd,
+                       off_t offset, size_t size, dict_t *xdata);
+int
+client_pre_ipc_v2(xlator_t *this, gfx_ipc_req *req, int32_t cmd, dict_t *xdata);
+
+int
+client_pre_lease_v2(xlator_t *this, gfx_lease_req *req, loc_t *loc,
+                    struct gf_lease *lease, dict_t *xdata);
+
+int
+client_pre_put_v2(xlator_t *this, gfx_put_req *req, loc_t *loc, mode_t mode,
+                  mode_t umask, int32_t flags, size_t size, off_t offset,
+                  dict_t *xattr, dict_t *xdata);
+
+int
+client_post_readv_v2(xlator_t *this, gfx_read_rsp *rsp, struct iobref **iobref,
+                     struct iobref *rsp_iobref, struct iatt *stat,
+                     struct iovec *vector, struct iovec *rsp_vector,
+                     int *rspcount, dict_t **xdata);
+
+int
+client_post_create_v2(xlator_t *this, gfx_create_rsp *rsp, struct iatt *stbuf,
+                      struct iatt *preparent, struct iatt *postparent,
+                      clnt_local_t *local, dict_t **xdata);
+int
+client_post_lease_v2(xlator_t *this, gfx_lease_rsp *rsp, struct gf_lease *lease,
+                     dict_t **xdata);
+int
+client_post_lk_v2(xlator_t *this, gfx_lk_rsp *rsp, struct gf_flock *lock,
+                  dict_t **xdata);
+int
+client_post_readdir_v2(xlator_t *this, gfx_readdir_rsp *rsp,
+                       gf_dirent_t *entries, dict_t **xdata);
+int
+client_post_readdirp_v2(xlator_t *this, gfx_readdirp_rsp *rsp, fd_t *fd,
+                        gf_dirent_t *entries, dict_t **xdata);
+int
+client_post_rename_v2(xlator_t *this, gfx_rename_rsp *rsp, struct iatt *stbuf,
+                      struct iatt *preoldparent, struct iatt *postoldparent,
+                      struct iatt *prenewparent, struct iatt *postnewparent,
+                      dict_t **xdata);
+
+int
+client_pre_copy_file_range_v2(xlator_t *this, gfx_copy_file_range_req *req,
+                              fd_t *fd_in, off64_t off_in, fd_t *fd_out,
+                              off64_t off_out, size_t size, int32_t flags,
+                              dict_t **xdata);
+
+#endif /* __CLIENT_COMMON_H__ */
diff --git a/xlators/protocol/client/src/client-handshake.c b/xlators/protocol/client/src/client-handshake.c
index b4c4604eb9f..ea5ef5c1800 100644
--- a/xlators/protocol/client/src/client-handshake.c
+++ b/xlators/protocol/client/src/client-handshake.c
@@ -8,1688 +8,1408 @@
   cases as published by the Free Software Foundation.
 */
 
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
-
-#include "fd-lk.h"
+#include <glusterfs/fd-lk.h>
 #include "client.h"
-#include "xlator.h"
-#include "defaults.h"
-#include "glusterfs.h"
-#include "statedump.h"
-#include "compat-errno.h"
+#include <glusterfs/xlator.h>
+#include <glusterfs/defaults.h>
+#include <glusterfs/glusterfs.h>
+#include <glusterfs/statedump.h>
+#include <glusterfs/compat-errno.h>
 
 #include "glusterfs3.h"
 #include "portmap-xdr.h"
 #include "rpc-common-xdr.h"
+#include "client-messages.h"
+#include "xdr-rpc.h"
 
 #define CLIENT_REOPEN_MAX_ATTEMPTS 1024
 extern rpc_clnt_prog_t clnt3_3_fop_prog;
+extern rpc_clnt_prog_t clnt4_0_fop_prog;
 extern rpc_clnt_prog_t clnt_pmap_prog;
 
-int client_set_lk_version_cbk (struct rpc_req *req, struct iovec *iov,
-                               int count, void *myframe);
-
-int client_set_lk_version (xlator_t *this);
-
-typedef struct client_fd_lk_local {
-        int             ref;
-        gf_boolean_t    error;
-        gf_lock_t       lock;
-        clnt_fd_ctx_t *fdctx;
-}clnt_fd_lk_local_t;
-
-int
-client3_getspec_cbk (struct rpc_req *req, struct iovec *iov, int count,
-                     void *myframe)
+int32_t
+client3_getspec(call_frame_t *frame, xlator_t *this, void *data)
 {
-        gf_getspec_rsp           rsp   = {0,};
-        call_frame_t            *frame = NULL;
-        int                      ret   = 0;
-
-        frame = myframe;
-
-        if (!frame || !frame->this) {
-                gf_log (THIS->name, GF_LOG_ERROR, "frame not found with the request, "
-                        "returning EINVAL");
-                rsp.op_ret   = -1;
-                rsp.op_errno = EINVAL;
-                goto out;
-        }
-        if (-1 == req->rpc_status) {
-                gf_log (frame->this->name, GF_LOG_WARNING,
-                        "received RPC status error, returning ENOTCONN");
-                rsp.op_ret   = -1;
-                rsp.op_errno = ENOTCONN;
-                goto out;
-        }
-
-        ret = xdr_to_generic (*iov, &rsp, (xdrproc_t)xdr_gf_getspec_rsp);
-        if (ret < 0) {
-                gf_log (frame->this->name, GF_LOG_ERROR,
-                        "XDR decoding failed, returning EINVAL");
-                rsp.op_ret   = -1;
-                rsp.op_errno = EINVAL;
-                goto out;
-        }
-
-        if (-1 == rsp.op_ret) {
-                gf_log (frame->this->name, GF_LOG_WARNING,
-                        "failed to get the 'volume file' from server");
-                goto out;
-        }
-
-out:
-        CLIENT_STACK_UNWIND (getspec, frame, rsp.op_ret, rsp.op_errno,
-                             rsp.spec);
-
-        /* Don't use 'GF_FREE', this is allocated by libc */
-        free (rsp.spec);
-
-        return 0;
+    CLIENT_STACK_UNWIND(getspec, frame, -1, ENOSYS, NULL);
+    return 0;
 }
 
-int32_t client3_getspec (call_frame_t *frame, xlator_t *this, void *data)
+static int
+client_notify_parents_child_up(xlator_t *this)
 {
-        clnt_conf_t    *conf     = NULL;
-        clnt_args_t    *args     = NULL;
-        gf_getspec_req  req      = {0,};
-        int             op_errno = ESTALE;
-        int             ret      = 0;
-
-        if (!frame || !this || !data)
-                goto unwind;
-
-        args = data;
-        conf = this->private;
-        req.flags = args->flags;
-        req.key   = (char *)args->name;
+    clnt_conf_t *conf = NULL;
+    int ret = 0;
 
-        ret = client_submit_request (this, &req, frame, conf->handshake,
-                                     GF_HNDSK_GETSPEC, client3_getspec_cbk,
-                                     NULL, NULL, 0, NULL, 0, NULL,
-                                     (xdrproc_t)xdr_gf_getspec_req);
+    GF_VALIDATE_OR_GOTO("client", this, out);
+    conf = this->private;
+    GF_VALIDATE_OR_GOTO(this->name, conf, out);
 
+    if (conf->child_up) {
+        ret = client_notify_dispatch_uniq(this, GF_EVENT_CHILD_UP, NULL);
         if (ret) {
-                gf_log (this->name, GF_LOG_WARNING,
-                        "failed to send the request");
+            gf_smsg(this->name, GF_LOG_INFO, 0, PC_MSG_CHILD_UP_NOTIFY_FAILED,
+                    NULL);
+            goto out;
         }
+    } else {
+        gf_smsg(this->name, GF_LOG_INFO, 0, PC_MSG_CHILD_STATUS, NULL);
+    }
 
-        return 0;
-unwind:
-        CLIENT_STACK_UNWIND (getspec, frame, -1, op_errno, NULL);
-        return 0;
-
+out:
+    return 0;
 }
 
-int
-client_notify_parents_child_up (xlator_t *this)
+void
+client_default_reopen_done(clnt_fd_ctx_t *fdctx, int64_t rfd, xlator_t *this)
 {
-        clnt_conf_t *conf = NULL;
-        int          ret  = 0;
-
-        conf = this->private;
-        ret = client_notify_dispatch (this, GF_EVENT_CHILD_UP, NULL);
-        if (ret)
-                gf_log (this->name, GF_LOG_INFO,
-                        "notify of CHILD_UP failed");
-
-        return 0;
+    gf_log_callingfn(this->name, GF_LOG_WARNING,
+                     "This function should never be called");
 }
 
-int
-clnt_fd_lk_reacquire_failed (xlator_t *this, clnt_fd_ctx_t *fdctx,
-                             clnt_conf_t *conf)
-{
-        int      ret = -1;
-
-        GF_VALIDATE_OR_GOTO ("client", this, out);
-        GF_VALIDATE_OR_GOTO (this->name, conf, out);
-        GF_VALIDATE_OR_GOTO (this->name, fdctx, out);
-
-        pthread_mutex_lock (&conf->lock);
-        {
-                fdctx->remote_fd     = -1;
-                fdctx->lk_heal_state = GF_LK_HEAL_DONE;
-        }
-        pthread_mutex_unlock (&conf->lock);
-
-        ret = 0;
-out:
-        return ret;
-}
-
-int
-client_set_lk_version_cbk (struct rpc_req *req, struct iovec *iov,
-                           int count, void *myframe)
+static void
+client_reopen_done(clnt_fd_ctx_t *fdctx, int64_t rfd, xlator_t *this)
 {
-        int32_t           ret    = -1;
-        call_frame_t     *fr     = NULL;
-        gf_set_lk_ver_rsp rsp    = {0,};
-
-        fr = (call_frame_t *) myframe;
-        GF_VALIDATE_OR_GOTO ("client", fr, out);
-
-        if (req->rpc_status == -1) {
-                gf_log (fr->this->name, GF_LOG_WARNING,
-                        "received RPC status error");
-                goto out;
-        }
-
-        ret = xdr_to_generic (*iov, &rsp, (xdrproc_t)xdr_gf_set_lk_ver_rsp);
-        if (ret < 0)
-                gf_log (fr->this->name, GF_LOG_WARNING,
-                        "xdr decoding failed");
+    clnt_conf_t *conf = this->private;
+    gf_boolean_t destroy = _gf_false;
+
+    pthread_spin_lock(&conf->fd_lock);
+    {
+        fdctx->remote_fd = rfd;
+        fdctx->reopen_attempts = 0;
+        fdctx->reopen_done = client_default_reopen_done;
+        if (!fdctx->released)
+            list_add_tail(&fdctx->sfd_pos, &conf->saved_fds);
         else
-                gf_log (fr->this->name, GF_LOG_INFO,
-                        "Server lk version = %d", rsp.lk_ver);
+            destroy = _gf_true;
+    }
+    pthread_spin_unlock(&conf->fd_lock);
 
-        ret = 0;
-out:
-        if (fr)
-                STACK_DESTROY (fr->root);
-
-        return ret;
+    if (destroy)
+        client_fdctx_destroy(this, fdctx);
 }
 
-//TODO: Check for all released fdctx and destroy them
-int
-client_set_lk_version (xlator_t *this)
+static void
+client_child_up_reopen_done(clnt_fd_ctx_t *fdctx, int64_t rfd, xlator_t *this)
 {
-        int                 ret      = -1;
-        clnt_conf_t        *conf     = NULL;
-        call_frame_t       *frame    = NULL;
-        gf_set_lk_ver_req   req      = {0, };
-        char               *process_uuid = NULL;
-
-        GF_VALIDATE_OR_GOTO ("client", this, err);
-
-        conf = (clnt_conf_t *) this->private;
-
-        req.lk_ver = client_get_lk_ver (conf);
-        ret = dict_get_str (this->options, "process-uuid", &process_uuid);
-        if (!process_uuid) {
-                ret = -1;
-                goto err;
-        }
-        req.uid = gf_strdup (process_uuid);
-        if (!req.uid) {
-                ret = -1;
-                goto err;
-        }
-
-        frame = create_frame (this, this->ctx->pool);
-        if (!frame) {
-                ret = -1;
-                goto out;
-        }
-
-        gf_log (this->name, GF_LOG_DEBUG, "Sending SET_LK_VERSION");
-
-        ret = client_submit_request (this, &req, frame,
-                                     conf->handshake,
-                                     GF_HNDSK_SET_LK_VER,
-                                     client_set_lk_version_cbk,
-                                     NULL, NULL, 0, NULL, 0, NULL,
-                                     (xdrproc_t)xdr_gf_set_lk_ver_req);
-out:
-        GF_FREE (req.uid);
-        return ret;
-err:
-        gf_log (this->name, GF_LOG_WARNING,
-                "Failed to send SET_LK_VERSION to server");
-
-        return ret;
+    clnt_conf_t *conf = this->private;
+    uint64_t fd_count = 0;
+
+    LOCK(&conf->rec_lock);
+    {
+        fd_count = --(conf->reopen_fd_count);
+    }
+    UNLOCK(&conf->rec_lock);
+
+    client_reopen_done(fdctx, rfd, this);
+    if (fd_count == 0) {
+        gf_smsg(this->name, GF_LOG_INFO, 0, PC_MSG_CHILD_UP_NOTIFY, NULL);
+        client_notify_parents_child_up(this);
+    }
 }
 
 int
-client_fd_lk_count (fd_lk_ctx_t *lk_ctx)
+client3_3_reopen_cbk(struct rpc_req *req, struct iovec *iov, int count,
+                     void *myframe)
 {
-        int               count     = 0;
-        fd_lk_ctx_node_t *fd_lk   = NULL;
-
-        GF_VALIDATE_OR_GOTO ("client", lk_ctx, err);
+    int32_t ret = -1;
+    gfs3_open_rsp rsp = {
+        0,
+    };
+    call_frame_t *frame = myframe;
+    xlator_t *this = frame->this;
+    clnt_local_t *local = frame->local;
+    clnt_fd_ctx_t *fdctx = local->fdctx;
+
+    if (-1 == req->rpc_status) {
+        gf_smsg(frame->this->name, GF_LOG_WARNING, ENOTCONN,
+                PC_MSG_RPC_STATUS_ERROR, NULL);
+        rsp.op_ret = -1;
+        rsp.op_errno = ENOTCONN;
+        goto out;
+    }
+
+    ret = xdr_to_generic(*iov, &rsp, (xdrproc_t)xdr_gfs3_open_rsp);
+    if (ret < 0) {
+        gf_smsg(frame->this->name, GF_LOG_ERROR, EINVAL,
+                PC_MSG_XDR_DECODING_FAILED, NULL);
+        rsp.op_ret = -1;
+        rsp.op_errno = EINVAL;
+        goto out;
+    }
+
+    if (rsp.op_ret < 0) {
+        gf_smsg(frame->this->name, GF_LOG_WARNING, rsp.op_errno,
+                PC_MSG_REOPEN_FAILED, "path=%s", local->loc.path);
+    } else {
+        gf_msg_debug(frame->this->name, 0,
+                     "reopen on %s succeeded (remote-fd = %" PRId64 ")",
+                     local->loc.path, rsp.fd);
+    }
+
+    if (rsp.op_ret == -1) {
+        goto out;
+    }
 
-        LOCK (&lk_ctx->lock);
-        {
-                list_for_each_entry (fd_lk, &lk_ctx->lk_list, next)
-                        count++;
-        }
-        UNLOCK (&lk_ctx->lock);
+out:
+    fdctx->reopen_done(fdctx, (rsp.op_ret) ? -1 : rsp.fd, this);
 
-        return count;
-err:
-        return -1;
-}
+    frame->local = NULL;
+    STACK_DESTROY(frame->root);
 
-clnt_fd_lk_local_t *
-clnt_fd_lk_local_ref (xlator_t *this, clnt_fd_lk_local_t *local)
-{
-        GF_VALIDATE_OR_GOTO (this->name, local, out);
+    client_local_wipe(local);
 
-        LOCK (&local->lock);
-        {
-                local->ref++;
-        }
-        UNLOCK (&local->lock);
-out:
-        return local;
+    return 0;
 }
 
 int
-clnt_fd_lk_local_unref (xlator_t *this, clnt_fd_lk_local_t *local)
+client3_3_reopendir_cbk(struct rpc_req *req, struct iovec *iov, int count,
+                        void *myframe)
 {
-        int   ref = -1;
+    int32_t ret = -1;
+    gfs3_open_rsp rsp = {
+        0,
+    };
+    call_frame_t *frame = myframe;
+    clnt_local_t *local = frame->local;
+    clnt_fd_ctx_t *fdctx = local->fdctx;
+
+    if (-1 == req->rpc_status) {
+        gf_smsg(frame->this->name, GF_LOG_WARNING, ENOTCONN,
+                PC_MSG_RPC_STATUS_ERROR, NULL);
+        rsp.op_ret = -1;
+        rsp.op_errno = ENOTCONN;
+        goto out;
+    }
+
+    ret = xdr_to_generic(*iov, &rsp, (xdrproc_t)xdr_gfs3_opendir_rsp);
+    if (ret < 0) {
+        gf_smsg(frame->this->name, GF_LOG_ERROR, EINVAL,
+                PC_MSG_XDR_DECODING_FAILED, NULL);
+        rsp.op_ret = -1;
+        rsp.op_errno = EINVAL;
+        goto out;
+    }
+
+    if (rsp.op_ret < 0) {
+        gf_smsg(frame->this->name, GF_LOG_WARNING, rsp.op_errno,
+                PC_MSG_REOPEN_FAILED, "path=%s", local->loc.path, NULL);
+    } else {
+        gf_smsg(frame->this->name, GF_LOG_INFO, 0, PC_MSG_DIR_OP_SUCCESS,
+                "path=%s", local->loc.path, "fd=%" PRId64, rsp.fd, NULL);
+    }
+
+    if (-1 == rsp.op_ret) {
+        goto out;
+    }
 
-        GF_VALIDATE_OR_GOTO (this->name, local, out);
-
-        LOCK (&local->lock);
-        {
-                ref = --local->ref;
-        }
-        UNLOCK (&local->lock);
-
-        if (ref == 0) {
-                LOCK_DESTROY (&local->lock);
-                GF_FREE (local);
-        }
 out:
-        return ref;
-}
-
-clnt_fd_lk_local_t *
-clnt_fd_lk_local_create (clnt_fd_ctx_t *fdctx)
-{
-        clnt_fd_lk_local_t *local = NULL;
+    fdctx->reopen_done(fdctx, (rsp.op_ret) ? -1 : rsp.fd, frame->this);
 
-        local = GF_CALLOC (1, sizeof (clnt_fd_lk_local_t),
-                           gf_client_mt_clnt_fd_lk_local_t);
-        if (!local)
-                goto out;
-
-        local->ref    = 1;
-        local->error  = _gf_false;
-        local->fdctx = fdctx;
+    frame->local = NULL;
+    STACK_DESTROY(frame->root);
+    client_local_wipe(local);
 
-        LOCK_INIT (&local->lock);
-out:
-        return local;
+    return 0;
 }
 
-void
-clnt_mark_fd_bad (clnt_conf_t *conf, clnt_fd_ctx_t *fdctx)
+static int
+protocol_client_reopendir(clnt_fd_ctx_t *fdctx, xlator_t *this)
 {
-        pthread_mutex_lock (&conf->lock);
+    int ret = -1;
+    gfs3_opendir_req req = {
         {
-                fdctx->remote_fd = -1;
-        }
-        pthread_mutex_unlock (&conf->lock);
-}
-
-int
-clnt_release_reopen_fd_cbk (struct rpc_req *req, struct iovec *iov,
-                            int count, void *myframe)
-{
-        xlator_t       *this   = NULL;
-        call_frame_t   *frame  = NULL;
-        clnt_conf_t    *conf   = NULL;
-        clnt_fd_ctx_t  *fdctx  = NULL;
+            0,
+        },
+    };
+    clnt_local_t *local = NULL;
+    call_frame_t *frame = NULL;
+    clnt_conf_t *conf = NULL;
 
-        frame  = myframe;
-        this   = frame->this;
-        fdctx  = (clnt_fd_ctx_t *) frame->local;
-        conf   = (clnt_conf_t *) this->private;
+    conf = this->private;
 
-        clnt_fd_lk_reacquire_failed (this, fdctx, conf);
+    local = mem_get0(this->local_pool);
+    if (!local) {
+        goto out;
+    }
+    local->fdctx = fdctx;
 
-        fdctx->reopen_done (fdctx, this);
+    gf_uuid_copy(local->loc.gfid, fdctx->gfid);
+    ret = loc_path(&local->loc, NULL);
+    if (ret < 0)
+        goto out;
 
-        frame->local = NULL;
-        STACK_DESTROY (frame->root);
+    frame = create_frame(this, this->ctx->pool);
+    if (!frame) {
+        goto out;
+    }
 
-        return 0;
-}
+    memcpy(req.gfid, fdctx->gfid, 16);
 
-int
-clnt_release_reopen_fd (xlator_t *this, clnt_fd_ctx_t *fdctx)
-{
-        int               ret     = -1;
-        clnt_conf_t      *conf    = NULL;
-        call_frame_t     *frame   = NULL;
-        gfs3_release_req  req     = {{0,},};
+    gf_msg_debug(frame->this->name, 0, "attempting reopen on %s",
+                 local->loc.path);
 
-        conf = (clnt_conf_t *) this->private;
+    frame->local = local;
 
-        frame  = create_frame (this, this->ctx->pool);
-        if (!frame)
-                goto out;
+    ret = client_submit_request(this, &req, frame, conf->fops, GFS3_OP_OPENDIR,
+                                client3_3_reopendir_cbk, NULL,
+                                (xdrproc_t)xdr_gfs3_opendir_req);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_ERROR, 0, PC_MSG_DIR_OP_FAILED, NULL);
+    }
 
-        frame->local = (void *) fdctx;
-        req.fd       = fdctx->remote_fd;
+    return 0;
 
-        ret    = client_submit_request (this, &req, frame, conf->fops,
-                                        GFS3_OP_RELEASE,
-                                        clnt_release_reopen_fd_cbk, NULL,
-                                        NULL, 0, NULL, 0, NULL,
-                                        (xdrproc_t)xdr_gfs3_releasedir_req);
-        return 0;
- out:
-        if (ret) {
-                clnt_fd_lk_reacquire_failed (this, fdctx, conf);
-                fdctx->reopen_done (fdctx, this);
-        }
-        return 0;
-}
-
-int
-clnt_reacquire_lock_error (xlator_t *this, clnt_fd_ctx_t *fdctx,
-                           clnt_conf_t *conf)
-{
-        int32_t   ret       = -1;
-
-        GF_VALIDATE_OR_GOTO ("client", this, out);
-        GF_VALIDATE_OR_GOTO (this->name, fdctx, out);
-        GF_VALIDATE_OR_GOTO (this->name, conf, out);
-
-        clnt_release_reopen_fd (this, fdctx);
-
-        ret = 0;
 out:
-        return ret;
-}
+    if (local)
+        client_local_wipe(local);
 
-gf_boolean_t
-clnt_fd_lk_local_error_status (xlator_t *this,
-                               clnt_fd_lk_local_t *local)
-{
-        gf_boolean_t error = _gf_false;
+    fdctx->reopen_done(fdctx, fdctx->remote_fd, this);
 
-        LOCK (&local->lock);
-        {
-                error = local->error;
-        }
-        UNLOCK (&local->lock);
-
-        return error;
+    return 0;
 }
 
-int
-clnt_fd_lk_local_mark_error (xlator_t *this,
-                             clnt_fd_lk_local_t *local)
+static int
+protocol_client_reopenfile(clnt_fd_ctx_t *fdctx, xlator_t *this)
 {
-        int32_t       ret   = -1;
-        clnt_conf_t  *conf  = NULL;
-        gf_boolean_t  error = _gf_false;
-
-        GF_VALIDATE_OR_GOTO ("client", this, out);
-        GF_VALIDATE_OR_GOTO (this->name, local, out);
-
-        conf = (clnt_conf_t *) this->private;
-
-        LOCK (&local->lock);
+    int ret = -1;
+    gfs3_open_req req = {
         {
-                error        = local->error;
-                local->error = _gf_true;
-        }
-        UNLOCK (&local->lock);
+            0,
+        },
+    };
+    clnt_local_t *local = NULL;
+    call_frame_t *frame = NULL;
+    clnt_conf_t *conf = NULL;
 
-        if (!error)
-                clnt_reacquire_lock_error (this, local->fdctx, conf);
-        ret = 0;
-out:
-        return ret;
-}
+    conf = this->private;
 
-int
-client_reacquire_lock_cbk (struct rpc_req *req, struct iovec *iov,
-                           int count, void *myframe)
-{
-        int32_t             ret        = -1;
-        xlator_t           *this       = NULL;
-        gfs3_lk_rsp         rsp        = {0,};
-        call_frame_t       *frame      = NULL;
-        clnt_conf_t        *conf       = NULL;
-        clnt_fd_ctx_t      *fdctx      = NULL;
-        clnt_fd_lk_local_t *local      = NULL;
-        struct gf_flock     lock       = {0,};
-
-        frame = (call_frame_t *) myframe;
-        this  = frame->this;
-        local = (clnt_fd_lk_local_t *) frame->local;
-        conf  = (clnt_conf_t *) this->private;
-
-        if (req->rpc_status == -1) {
-                gf_log ("client", GF_LOG_WARNING,
-                        "request failed at rpc");
-                goto out;
-        }
+    frame = create_frame(this, this->ctx->pool);
+    if (!frame) {
+        goto out;
+    }
 
-        ret = xdr_to_generic (*iov, &rsp, (xdrproc_t)xdr_gfs3_lk_rsp);
-        if (ret < 0) {
-                gf_log (this->name, GF_LOG_ERROR, "XDR decoding failed");
-                goto out;
-        }
+    local = mem_get0(this->local_pool);
+    if (!local) {
+        goto out;
+    }
 
-        if (rsp.op_ret == -1) {
-                gf_log (this->name, GF_LOG_ERROR, "lock request failed");
-                ret = -1;
-                goto out;
-        }
+    local->fdctx = fdctx;
+    gf_uuid_copy(local->loc.gfid, fdctx->gfid);
+    ret = loc_path(&local->loc, NULL);
+    if (ret < 0)
+        goto out;
 
-        fdctx = local->fdctx;
+    frame->local = local;
 
-        gf_proto_flock_to_flock (&rsp.flock, &lock);
+    memcpy(req.gfid, fdctx->gfid, 16);
+    req.flags = gf_flags_from_flags(fdctx->flags);
+    req.flags = req.flags & (~(O_TRUNC | O_CREAT | O_EXCL));
 
-        gf_log (this->name, GF_LOG_DEBUG, "%s type lock reacquired on file "
-                "with gfid %s from %"PRIu64 " to %"PRIu64,
-                get_lk_type (lock.l_type), uuid_utoa (fdctx->gfid),
-                lock.l_start, lock.l_start + lock.l_len);
+    gf_msg_debug(frame->this->name, 0, "attempting reopen on %s",
+                 local->loc.path);
 
-        if (!clnt_fd_lk_local_error_status (this, local) &&
-            clnt_fd_lk_local_unref (this, local) == 0) {
-                pthread_mutex_lock (&conf->lock);
-                {
-                        fdctx->lk_heal_state = GF_LK_HEAL_DONE;
-                }
-                pthread_mutex_unlock (&conf->lock);
+    ret = client_submit_request(this, &req, frame, conf->fops, GFS3_OP_OPEN,
+                                client3_3_reopen_cbk, NULL,
+                                (xdrproc_t)xdr_gfs3_open_req);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_ERROR, 0, PC_MSG_DIR_OP_FAILED, NULL);
+    }
 
-                fdctx->reopen_done (fdctx, this);
-        }
+    return 0;
 
-        ret = 0;
 out:
-        if (ret < 0) {
-                clnt_fd_lk_local_mark_error (this, local);
-
-                clnt_fd_lk_local_unref (this, local);
-        }
-
+    if (frame) {
         frame->local = NULL;
-        STACK_DESTROY (frame->root);
-
-        return ret;
-}
-
-int
-_client_reacquire_lock (xlator_t *this, clnt_fd_ctx_t *fdctx)
-{
-        int32_t             ret       = -1;
-        int32_t             gf_cmd    = 0;
-        int32_t             gf_type   = 0;
-        gfs3_lk_req         req       = {{0,},};
-        struct gf_flock     flock     = {0,};
-        fd_lk_ctx_t        *lk_ctx    = NULL;
-        clnt_fd_lk_local_t *local     = NULL;
-        fd_lk_ctx_node_t   *fd_lk     = NULL;
-        call_frame_t       *frame     = NULL;
-        clnt_conf_t        *conf      = NULL;
-
-        conf   = (clnt_conf_t *) this->private;
-        lk_ctx = fdctx->lk_ctx;
-
-        local = clnt_fd_lk_local_create (fdctx);
-        if (!local) {
-                gf_log (this->name, GF_LOG_WARNING, "clnt_fd_lk_local_create "
-                        "failed, aborting reacquring of locks on %s.",
-                        uuid_utoa (fdctx->gfid));
-                clnt_reacquire_lock_error (this, fdctx, conf);
-                goto out;
-        }
-
-        list_for_each_entry (fd_lk, &lk_ctx->lk_list, next) {
-                memcpy (&flock, &fd_lk->user_flock,
-                        sizeof (struct gf_flock));
-
-                /* Always send F_SETLK even if the cmd was F_SETLKW */
-                /* to avoid frame being blocked if lock cannot be granted. */
-                ret = client_cmd_to_gf_cmd (F_SETLK, &gf_cmd);
-                if (ret) {
-                        gf_log (this->name, GF_LOG_WARNING,
-                                "client_cmd_to_gf_cmd failed, "
-                                "aborting reacquiring of locks");
-                        break;
-                }
-
-                gf_type   = client_type_to_gf_type (flock.l_type);
-                req.fd    = fdctx->remote_fd;
-                req.cmd   = gf_cmd;
-                req.type  = gf_type;
-                (void) gf_proto_flock_from_flock (&req.flock,
-                                                  &flock);
-
-                memcpy (req.gfid, fdctx->gfid, 16);
-
-                frame = create_frame (this, this->ctx->pool);
-                if (!frame) {
-                        ret = -1;
-                        break;
-                }
-
-                frame->local          = clnt_fd_lk_local_ref (this, local);
-                frame->root->lk_owner = fd_lk->user_flock.l_owner;
-
-                ret = client_submit_request (this, &req, frame,
-                                             conf->fops, GFS3_OP_LK,
-                                             client_reacquire_lock_cbk,
-                                             NULL, NULL, 0, NULL, 0, NULL,
-                                             (xdrproc_t)xdr_gfs3_lk_req);
-                if (ret) {
-                        gf_log (this->name, GF_LOG_WARNING,
-                                "reacquiring locks failed on file with gfid %s",
-                                uuid_utoa (fdctx->gfid));
-                        break;
-                }
-
-                ret   = 0;
-                frame = NULL;
-        }
-
-        if (local)
-                (void) clnt_fd_lk_local_unref (this, local);
-out:
-        return ret;
-}
+        STACK_DESTROY(frame->root);
+    }
 
-int
-client_reacquire_lock (xlator_t *this, clnt_fd_ctx_t *fdctx)
-{
-        int32_t          ret       = -1;
-        fd_lk_ctx_t     *lk_ctx    = NULL;
+    if (local)
+        client_local_wipe(local);
 
-        GF_VALIDATE_OR_GOTO ("client", this, out);
-        GF_VALIDATE_OR_GOTO (this->name, fdctx, out);
+    fdctx->reopen_done(fdctx, fdctx->remote_fd, this);
 
-        if (client_fd_lk_list_empty (fdctx->lk_ctx, _gf_false)) {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "fd lock list is empty");
-                fdctx->reopen_done (fdctx, this);
-        } else {
-                lk_ctx = fdctx->lk_ctx;
-
-                LOCK (&lk_ctx->lock);
-                {
-                        (void) _client_reacquire_lock (this, fdctx);
-                }
-                UNLOCK (&lk_ctx->lock);
-        }
-        ret = 0;
-out:
-        return ret;
-}
-
-void
-client_default_reopen_done (clnt_fd_ctx_t *fdctx, xlator_t *this)
-{
-        gf_log_callingfn (this->name, GF_LOG_WARNING,
-                          "This function should never be called");
+    return 0;
 }
 
-void
-client_reopen_done (clnt_fd_ctx_t *fdctx, xlator_t *this)
-{
-        clnt_conf_t  *conf    = NULL;
-        gf_boolean_t destroy  = _gf_false;
-
-        conf = this->private;
-
-        pthread_mutex_lock (&conf->lock);
-        {
-                fdctx->reopen_attempts = 0;
-                if (!fdctx->released)
-                        list_add_tail (&fdctx->sfd_pos, &conf->saved_fds);
-                else
-                        destroy = _gf_true;
-                fdctx->reopen_done = client_default_reopen_done;
-        }
-        pthread_mutex_unlock (&conf->lock);
-
-        if (destroy)
-                client_fdctx_destroy (this, fdctx);
-}
-
-void
-client_child_up_reopen_done (clnt_fd_ctx_t *fdctx, xlator_t *this)
+static void
+protocol_client_reopen(clnt_fd_ctx_t *fdctx, xlator_t *this)
 {
-        clnt_conf_t  *conf    = NULL;
-        uint64_t     fd_count = 0;
-
-        conf = this->private;
-
-        LOCK (&conf->rec_lock);
-        {
-                fd_count = --(conf->reopen_fd_count);
-        }
-        UNLOCK (&conf->rec_lock);
-
-        client_reopen_done (fdctx, this);
-        if (fd_count == 0) {
-                gf_log (this->name, GF_LOG_INFO,
-                        "last fd open'd/lock-self-heal'd - notifying CHILD-UP");
-                client_set_lk_version (this);
-                client_notify_parents_child_up (this);
-        }
+    if (fdctx->is_dir)
+        protocol_client_reopendir(fdctx, this);
+    else
+        protocol_client_reopenfile(fdctx, this);
 }
 
+/* v4.x +  */
 int
-client3_3_reopen_cbk (struct rpc_req *req, struct iovec *iov, int count,
-                      void           *myframe)
+client4_0_reopen_cbk(struct rpc_req *req, struct iovec *iov, int count,
+                     void *myframe)
 {
-        int32_t        ret                   = -1;
-        gfs3_open_rsp  rsp                   = {0,};
-        gf_boolean_t   attempt_lock_recovery = _gf_false;
-        clnt_local_t  *local                 = NULL;
-        clnt_conf_t   *conf                  = NULL;
-        clnt_fd_ctx_t *fdctx                 = NULL;
-        call_frame_t  *frame                 = NULL;
-        xlator_t      *this                  = NULL;
-
-        frame = myframe;
-        this  = frame->this;
-        conf  = this->private;
-        local = frame->local;
-        fdctx = local->fdctx;
-
-        if (-1 == req->rpc_status) {
-                gf_log (frame->this->name, GF_LOG_WARNING,
-                        "received RPC status error, returning ENOTCONN");
-                rsp.op_ret   = -1;
-                rsp.op_errno = ENOTCONN;
-                goto out;
-        }
-
-        ret = xdr_to_generic (*iov, &rsp, (xdrproc_t)xdr_gfs3_open_rsp);
-        if (ret < 0) {
-                gf_log (frame->this->name, GF_LOG_ERROR, "XDR decoding failed");
-                rsp.op_ret   = -1;
-                rsp.op_errno = EINVAL;
-                goto out;
-        }
-
-        if (rsp.op_ret < 0) {
-                gf_log (frame->this->name, GF_LOG_WARNING,
-                        "reopen on %s failed (%s)",
-                        local->loc.path, strerror (rsp.op_errno));
-        } else {
-                gf_log (frame->this->name, GF_LOG_DEBUG,
-                        "reopen on %s succeeded (remote-fd = %"PRId64")",
-                        local->loc.path, rsp.fd);
-        }
-
-        if (rsp.op_ret == -1) {
-                ret = -1;
-                goto out;
-        }
-
-        pthread_mutex_lock (&conf->lock);
-        {
-                fdctx->remote_fd = rsp.fd;
-                if (!fdctx->released) {
-                        if (conf->lk_heal &&
-                            !client_fd_lk_list_empty (fdctx->lk_ctx,
-                                                      _gf_false)) {
-                                attempt_lock_recovery = _gf_true;
-                                fdctx->lk_heal_state  = GF_LK_HEAL_IN_PROGRESS;
-                        }
-                }
-        }
-        pthread_mutex_unlock (&conf->lock);
-
-        ret = 0;
-
-        if (attempt_lock_recovery) {
-                /* Delay decrementing the reopen fd count until all the
-                   locks corresponding to this fd are acquired.*/
-                gf_log (this->name, GF_LOG_DEBUG, "acquiring locks "
-                        "on %s", local->loc.path);
-                ret = client_reacquire_lock (frame->this, local->fdctx);
-                if (ret) {
-                        clnt_reacquire_lock_error (this, local->fdctx, conf);
-                        gf_log (this->name, GF_LOG_WARNING, "acquiring locks "
-                                "failed on %s", local->loc.path);
-                }
-        }
+    int32_t ret = -1;
+    gfx_open_rsp rsp = {
+        0,
+    };
+    call_frame_t *frame = myframe;
+    xlator_t *this = frame->this;
+    clnt_local_t *local = frame->local;
+    clnt_fd_ctx_t *fdctx = local->fdctx;
+
+    if (-1 == req->rpc_status) {
+        gf_smsg(frame->this->name, GF_LOG_WARNING, ENOTCONN,
+                PC_MSG_RPC_STATUS_ERROR, NULL);
+        rsp.op_ret = -1;
+        rsp.op_errno = ENOTCONN;
+        goto out;
+    }
+
+    ret = xdr_to_generic(*iov, &rsp, (xdrproc_t)xdr_gfx_open_rsp);
+    if (ret < 0) {
+        gf_smsg(frame->this->name, GF_LOG_ERROR, EINVAL,
+                PC_MSG_XDR_DECODING_FAILED, NULL);
+        rsp.op_ret = -1;
+        rsp.op_errno = EINVAL;
+        goto out;
+    }
+
+    if (rsp.op_ret < 0) {
+        gf_smsg(frame->this->name, GF_LOG_WARNING, rsp.op_errno,
+                PC_MSG_REOPEN_FAILED, "path=%s", local->loc.path, NULL);
+    } else {
+        gf_msg_debug(frame->this->name, 0,
+                     "reopen on %s succeeded (remote-fd = %" PRId64 ")",
+                     local->loc.path, rsp.fd);
+    }
+
+    if (rsp.op_ret == -1) {
+        goto out;
+    }
 
 out:
-        if (!attempt_lock_recovery)
-                fdctx->reopen_done (fdctx, this);
+    fdctx->reopen_done(fdctx, (rsp.op_ret) ? -1 : rsp.fd, this);
 
-        frame->local = NULL;
-        STACK_DESTROY (frame->root);
+    frame->local = NULL;
+    STACK_DESTROY(frame->root);
 
-        client_local_wipe (local);
+    client_local_wipe(local);
 
-        return 0;
+    return 0;
 }
 
 int
-client3_3_reopendir_cbk (struct rpc_req *req, struct iovec *iov, int count,
-                         void           *myframe)
+client4_0_reopendir_cbk(struct rpc_req *req, struct iovec *iov, int count,
+                        void *myframe)
 {
-        int32_t        ret   = -1;
-        gfs3_open_rsp  rsp   = {0,};
-        clnt_local_t  *local = NULL;
-        clnt_conf_t   *conf  = NULL;
-        clnt_fd_ctx_t *fdctx = NULL;
-        call_frame_t  *frame = NULL;
-
-        frame = myframe;
-        local = frame->local;
-        fdctx = local->fdctx;
-        conf  = frame->this->private;
-
-
-        if (-1 == req->rpc_status) {
-                gf_log (frame->this->name, GF_LOG_WARNING,
-                        "received RPC status error, returning ENOTCONN");
-                rsp.op_ret   = -1;
-                rsp.op_errno = ENOTCONN;
-                goto out;
-        }
-
-        ret = xdr_to_generic (*iov, &rsp, (xdrproc_t)xdr_gfs3_opendir_rsp);
-        if (ret < 0) {
-                gf_log (frame->this->name, GF_LOG_ERROR, "XDR decoding failed");
-                rsp.op_ret   = -1;
-                rsp.op_errno = EINVAL;
-                goto out;
-        }
-
-        if (rsp.op_ret < 0) {
-                gf_log (frame->this->name, GF_LOG_WARNING,
-                        "reopendir on %s failed (%s)",
-                        local->loc.path, strerror (rsp.op_errno));
-        } else {
-                gf_log (frame->this->name, GF_LOG_INFO,
-                        "reopendir on %s succeeded (fd = %"PRId64")",
-                        local->loc.path, rsp.fd);
-        }
-
-        if (-1 == rsp.op_ret) {
-                ret = -1;
-                goto out;
-        }
-
-        pthread_mutex_lock (&conf->lock);
-        {
-                fdctx->remote_fd = rsp.fd;
-        }
-        pthread_mutex_unlock (&conf->lock);
+    int32_t ret = -1;
+    gfx_open_rsp rsp = {
+        0,
+    };
+    call_frame_t *frame = myframe;
+    clnt_local_t *local = frame->local;
+    clnt_fd_ctx_t *fdctx = local->fdctx;
+
+    if (-1 == req->rpc_status) {
+        gf_smsg(frame->this->name, GF_LOG_WARNING, ENOTCONN,
+                PC_MSG_RPC_STATUS_ERROR, NULL);
+        rsp.op_ret = -1;
+        rsp.op_errno = ENOTCONN;
+        goto out;
+    }
+
+    ret = xdr_to_generic(*iov, &rsp, (xdrproc_t)xdr_gfx_open_rsp);
+    if (ret < 0) {
+        gf_smsg(frame->this->name, GF_LOG_ERROR, EINVAL,
+                PC_MSG_XDR_DECODING_FAILED, NULL);
+        rsp.op_ret = -1;
+        rsp.op_errno = EINVAL;
+        goto out;
+    }
+
+    if (rsp.op_ret < 0) {
+        gf_smsg(frame->this->name, GF_LOG_WARNING, rsp.op_errno,
+                PC_MSG_DIR_OP_FAILED, "dir-path=%s", local->loc.path, NULL);
+    } else {
+        gf_smsg(frame->this->name, GF_LOG_INFO, 0, PC_MSG_DIR_OP_SUCCESS,
+                "path=%s", local->loc.path, "fd=%" PRId64, rsp.fd, NULL);
+    }
+
+    if (-1 == rsp.op_ret) {
+        goto out;
+    }
 
 out:
-        fdctx->reopen_done (fdctx, frame->this);
+    fdctx->reopen_done(fdctx, (rsp.op_ret) ? -1 : rsp.fd, frame->this);
 
-        frame->local = NULL;
-        STACK_DESTROY (frame->root);
-        client_local_wipe (local);
+    frame->local = NULL;
+    STACK_DESTROY(frame->root);
+    client_local_wipe(local);
 
-        return 0;
+    return 0;
 }
 
 static int
-protocol_client_reopendir (clnt_fd_ctx_t *fdctx, xlator_t *this)
+protocol_client_reopendir_v2(clnt_fd_ctx_t *fdctx, xlator_t *this)
 {
-        int               ret   = -1;
-        gfs3_opendir_req  req   = {{0,},};
-        clnt_local_t     *local = NULL;
-        call_frame_t     *frame = NULL;
-        clnt_conf_t      *conf  = NULL;
-
-        conf = this->private;
-
-        local = mem_get0 (this->local_pool);
-        if (!local) {
-                ret = -1;
-                goto out;
-        }
-        local->fdctx    = fdctx;
-
-        gf_uuid_copy (local->loc.gfid, fdctx->gfid);
-        ret = loc_path (&local->loc, NULL);
-        if (ret < 0)
-                goto out;
-
-        frame = create_frame (this, this->ctx->pool);
-        if (!frame) {
-                ret = -1;
-                goto out;
-        }
-
-        memcpy (req.gfid, fdctx->gfid, 16);
-
-        gf_log (frame->this->name, GF_LOG_DEBUG,
-                "attempting reopen on %s", local->loc.path);
-
-        frame->local = local;
-
-        ret = client_submit_request (this, &req, frame, conf->fops,
-                                     GFS3_OP_OPENDIR,
-                                     client3_3_reopendir_cbk, NULL,
-                                     NULL, 0, NULL, 0, NULL,
-                                     (xdrproc_t)xdr_gfs3_opendir_req);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "failed to send the re-opendir request");
-        }
-
-        return 0;
+    int ret = -1;
+    gfx_opendir_req req = {
+        {
+            0,
+        },
+    };
+    call_frame_t *frame = NULL;
+    clnt_conf_t *conf = this->private;
+    clnt_local_t *local = mem_get0(this->local_pool);
+
+    if (!local) {
+        ret = -1;
+        goto out;
+    }
+    local->fdctx = fdctx;
+
+    gf_uuid_copy(local->loc.gfid, fdctx->gfid);
+    ret = loc_path(&local->loc, NULL);
+    if (ret < 0)
+        goto out;
+
+    frame = create_frame(this, this->ctx->pool);
+    if (!frame) {
+        ret = -1;
+        goto out;
+    }
+
+    memcpy(req.gfid, fdctx->gfid, 16);
+
+    gf_msg_debug(frame->this->name, 0, "attempting reopen on %s",
+                 local->loc.path);
+
+    frame->local = local;
+
+    ret = client_submit_request(this, &req, frame, conf->fops, GFS3_OP_OPENDIR,
+                                client4_0_reopendir_cbk, NULL,
+                                (xdrproc_t)xdr_gfx_opendir_req);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_ERROR, 0, PC_MSG_DIR_OP_FAILED, NULL);
+    }
+
+    return 0;
 
 out:
-        if (local)
-                client_local_wipe (local);
-
-        fdctx->reopen_done (fdctx, this);
+    if (local)
+        client_local_wipe(local);
 
-        return 0;
+    fdctx->reopen_done(fdctx, fdctx->remote_fd, this);
 
+    return 0;
 }
 
 static int
-protocol_client_reopenfile (clnt_fd_ctx_t *fdctx, xlator_t *this)
+protocol_client_reopenfile_v2(clnt_fd_ctx_t *fdctx, xlator_t *this)
 {
-        int            ret   = -1;
-        gfs3_open_req  req   = {{0,},};
-        clnt_local_t  *local = NULL;
-        call_frame_t  *frame = NULL;
-        clnt_conf_t   *conf  = NULL;
-
-        conf  = this->private;
-
-        frame = create_frame (this, this->ctx->pool);
-        if (!frame) {
-                ret = -1;
-                goto out;
-        }
-
-        local = mem_get0 (this->local_pool);
-        if (!local) {
-                ret = -1;
-                goto out;
-        }
-
-        local->fdctx    = fdctx;
-        gf_uuid_copy (local->loc.gfid, fdctx->gfid);
-        ret = loc_path (&local->loc, NULL);
-        if (ret < 0)
-                goto out;
-
-        frame->local    = local;
-
-        memcpy (req.gfid, fdctx->gfid, 16);
-        req.flags    = gf_flags_from_flags (fdctx->flags);
-        req.flags    = req.flags & (~(O_TRUNC|O_CREAT|O_EXCL));
-
-        gf_log (frame->this->name, GF_LOG_DEBUG,
-                "attempting reopen on %s", local->loc.path);
-
-        ret = client_submit_request (this, &req, frame, conf->fops,
-                                     GFS3_OP_OPEN, client3_3_reopen_cbk, NULL,
-                                     NULL, 0, NULL, 0, NULL,
-                                     (xdrproc_t)xdr_gfs3_open_req);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "failed to send the re-open request");
-        }
-
-        return 0;
+    int ret = -1;
+    gfx_open_req req = {
+        {
+            0,
+        },
+    };
+    clnt_local_t *local = NULL;
+    clnt_conf_t *conf = this->private;
+    call_frame_t *frame = create_frame(this, this->ctx->pool);
+
+    if (!frame) {
+        ret = -1;
+        goto out;
+    }
+
+    local = mem_get0(this->local_pool);
+    if (!local) {
+        ret = -1;
+        goto out;
+    }
+
+    local->fdctx = fdctx;
+    gf_uuid_copy(local->loc.gfid, fdctx->gfid);
+    ret = loc_path(&local->loc, NULL);
+    if (ret < 0)
+        goto out;
+
+    frame->local = local;
+
+    memcpy(req.gfid, fdctx->gfid, 16);
+    req.flags = gf_flags_from_flags(fdctx->flags);
+    req.flags = req.flags & (~(O_TRUNC | O_CREAT | O_EXCL));
+
+    gf_msg_debug(frame->this->name, 0, "attempting reopen on %s",
+                 local->loc.path);
+
+    ret = client_submit_request(this, &req, frame, conf->fops, GFS3_OP_OPEN,
+                                client4_0_reopen_cbk, NULL,
+                                (xdrproc_t)xdr_gfx_open_req);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_ERROR, 0, PC_MSG_DIR_OP_FAILED, NULL);
+    }
+
+    return 0;
 
 out:
-        if (frame) {
-                frame->local = NULL;
-                STACK_DESTROY (frame->root);
-        }
-
-        if (local)
-                client_local_wipe (local);
+    if (frame) {
+        frame->local = NULL;
+        STACK_DESTROY(frame->root);
+    }
 
-        fdctx->reopen_done (fdctx, this);
+    if (local)
+        client_local_wipe(local);
 
-        return 0;
+    fdctx->reopen_done(fdctx, fdctx->remote_fd, this);
 
+    return 0;
 }
 
 static void
-protocol_client_reopen (clnt_fd_ctx_t *fdctx, xlator_t *this)
+protocol_client_reopen_v2(clnt_fd_ctx_t *fdctx, xlator_t *this)
 {
-        if (fdctx->is_dir)
-                protocol_client_reopendir (fdctx, this);
-        else
-                protocol_client_reopenfile (fdctx, this);
+    if (fdctx->is_dir)
+        protocol_client_reopendir_v2(fdctx, this);
+    else
+        protocol_client_reopenfile_v2(fdctx, this);
 }
 
 gf_boolean_t
-__is_fd_reopen_in_progress (clnt_fd_ctx_t *fdctx)
+__is_fd_reopen_in_progress(clnt_fd_ctx_t *fdctx)
 {
-        if (fdctx->reopen_done == client_default_reopen_done)
-                return _gf_false;
-        return _gf_true;
+    if (fdctx->reopen_done == client_default_reopen_done)
+        return _gf_false;
+    return _gf_true;
 }
 
 void
-client_attempt_reopen (fd_t *fd, xlator_t *this)
+client_attempt_reopen(fd_t *fd, xlator_t *this)
 {
-        clnt_conf_t   *conf  = NULL;
-        clnt_fd_ctx_t *fdctx = NULL;
-        gf_boolean_t  reopen = _gf_false;
-
-        if (!fd || !this)
-                goto out;
-
-        conf = this->private;
-        pthread_mutex_lock (&conf->lock);
-        {
-                fdctx = this_fd_get_ctx (fd, this);
-                if (!fdctx)
-                        goto unlock;
-                if (__is_fd_reopen_in_progress (fdctx))
-                        goto unlock;
-                if (fdctx->remote_fd != -1)
-                        goto unlock;
-
-                if (fdctx->reopen_attempts == CLIENT_REOPEN_MAX_ATTEMPTS) {
-                        reopen = _gf_true;
-                        fdctx->reopen_done = client_reopen_done;
-                        list_del_init (&fdctx->sfd_pos);
-                } else {
-                        fdctx->reopen_attempts++;
-                }
+    if (!fd || !this)
+        goto out;
+
+    clnt_conf_t *conf = this->private;
+    clnt_fd_ctx_t *fdctx = NULL;
+    gf_boolean_t reopen = _gf_false;
+
+    pthread_spin_lock(&conf->fd_lock);
+    {
+        fdctx = this_fd_get_ctx(fd, this);
+        if (!fdctx) {
+            pthread_spin_unlock(&conf->fd_lock);
+            goto out;
+        }
+
+        if (__is_fd_reopen_in_progress(fdctx))
+            goto unlock;
+        if (fdctx->remote_fd != -1)
+            goto unlock;
+
+        if (fdctx->reopen_attempts == CLIENT_REOPEN_MAX_ATTEMPTS) {
+            reopen = _gf_true;
+            fdctx->reopen_done = client_reopen_done;
+            list_del_init(&fdctx->sfd_pos);
+        } else {
+            fdctx->reopen_attempts++;
         }
+    }
 unlock:
-        pthread_mutex_unlock (&conf->lock);
-        if (reopen)
-                protocol_client_reopen (fdctx, this);
+    pthread_spin_unlock(&conf->fd_lock);
+    if (reopen) {
+        if (conf->fops->progver == GLUSTER_FOP_VERSION_v2)
+            protocol_client_reopen_v2(fdctx, this);
+        else
+            protocol_client_reopen(fdctx, this);
+    }
 out:
-        return;
+    return;
 }
 
-int
-client_post_handshake (call_frame_t *frame, xlator_t *this)
+static int
+client_post_handshake(call_frame_t *frame, xlator_t *this)
 {
-        clnt_conf_t            *conf = NULL;
-        clnt_fd_ctx_t          *tmp = NULL;
-        clnt_fd_ctx_t          *fdctx = NULL;
-        struct list_head        reopen_head;
+    clnt_conf_t *conf = NULL;
+    clnt_fd_ctx_t *tmp = NULL;
+    clnt_fd_ctx_t *fdctx = NULL;
+    struct list_head reopen_head;
 
-        int count = 0;
+    int count = 0;
 
-        if (!this || !this->private)
-                goto out;
+    if (!this || !this->private)
+        goto out;
 
-        conf = this->private;
-        INIT_LIST_HEAD (&reopen_head);
+    conf = this->private;
+    INIT_LIST_HEAD(&reopen_head);
 
-        pthread_mutex_lock (&conf->lock);
+    pthread_spin_lock(&conf->fd_lock);
+    {
+        list_for_each_entry_safe(fdctx, tmp, &conf->saved_fds, sfd_pos)
         {
-                list_for_each_entry_safe (fdctx, tmp, &conf->saved_fds,
-                                          sfd_pos) {
-                        if (fdctx->remote_fd != -1)
-                                continue;
-
-                        fdctx->reopen_done = client_child_up_reopen_done;
-                        list_del_init (&fdctx->sfd_pos);
-                        list_add_tail (&fdctx->sfd_pos, &reopen_head);
-                        count++;
-                }
-        }
-        pthread_mutex_unlock (&conf->lock);
-
-        /* Delay notifying CHILD_UP to parents
-           until all locks are recovered */
-        if (count > 0) {
-                gf_log (this->name, GF_LOG_INFO,
-                        "%d fds open - Delaying child_up until they are re-opened",
-                        count);
-                client_save_number_fds (conf, count);
-
-                list_for_each_entry_safe (fdctx, tmp, &reopen_head, sfd_pos) {
-                        list_del_init (&fdctx->sfd_pos);
-
-                        protocol_client_reopen (fdctx, this);
-                }
-        } else {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "No fds to open - notifying all parents child up");
-                client_set_lk_version (this);
-                client_notify_parents_child_up (this);
-        }
+            if (fdctx->remote_fd != -1 ||
+                (!list_empty(&fdctx->lock_list) && conf->strict_locks))
+                continue;
+
+            fdctx->reopen_done = client_child_up_reopen_done;
+            list_del_init(&fdctx->sfd_pos);
+            list_add_tail(&fdctx->sfd_pos, &reopen_head);
+            count++;
+        }
+    }
+    pthread_spin_unlock(&conf->fd_lock);
+
+    /* Delay notifying CHILD_UP to parents
+       until all locks are recovered */
+    if (count > 0) {
+        gf_smsg(this->name, GF_LOG_INFO, 0, PC_MSG_CHILD_UP_NOTIFY_DELAY,
+                "count=%d", count, NULL);
+        client_save_number_fds(conf, count);
+
+        list_for_each_entry_safe(fdctx, tmp, &reopen_head, sfd_pos)
+        {
+            list_del_init(&fdctx->sfd_pos);
+
+            if (conf->fops->progver == GLUSTER_FOP_VERSION_v2)
+                protocol_client_reopen_v2(fdctx, this);
+            else
+                protocol_client_reopen(fdctx, this);
+        }
+    } else {
+        gf_msg_debug(this->name, 0,
+                     "No fds to open - notifying all parents child "
+                     "up");
+        client_notify_parents_child_up(this);
+    }
 out:
-        return 0;
+    return 0;
 }
 
 int
-client_setvolume_cbk (struct rpc_req *req, struct iovec *iov, int count, void *myframe)
+client_setvolume_cbk(struct rpc_req *req, struct iovec *iov, int count,
+                     void *myframe)
 {
-        call_frame_t         *frame         = NULL;
-        clnt_conf_t          *conf          = NULL;
-        xlator_t             *this          = NULL;
-        dict_t               *reply         = NULL;
-        char                 *process_uuid  = NULL;
-        char                 *remote_error  = NULL;
-        char                 *remote_subvol = NULL;
-        gf_setvolume_rsp      rsp           = {0,};
-        int                   ret           = 0;
-        int32_t               op_ret        = 0;
-        int32_t               op_errno      = 0;
-        gf_boolean_t          auth_fail     = _gf_false;
-        uint32_t              lk_ver        = 0;
-
-        frame = myframe;
-        this  = frame->this;
-        conf  = this->private;
-
-        if (-1 == req->rpc_status) {
-                gf_log (frame->this->name, GF_LOG_WARNING,
-                        "received RPC status error");
-                op_ret = -1;
-                goto out;
-        }
-
-        ret = xdr_to_generic (*iov, &rsp, (xdrproc_t)xdr_gf_setvolume_rsp);
+    call_frame_t *frame = myframe;
+    xlator_t *this = frame->this;
+    clnt_conf_t *conf = this->private;
+    dict_t *reply = NULL;
+    char *process_uuid = NULL;
+    char *volume_id = NULL;
+    char *remote_error = NULL;
+    char *remote_subvol = NULL;
+    gf_setvolume_rsp rsp = {
+        0,
+    };
+    int ret = 0;
+    int32_t op_ret = 0;
+    int32_t op_errno = 0;
+    gf_boolean_t auth_fail = _gf_false;
+    glusterfs_ctx_t *ctx = NULL;
+
+    GF_VALIDATE_OR_GOTO(this->name, conf, out);
+    ctx = this->ctx;
+    GF_VALIDATE_OR_GOTO(this->name, ctx, out);
+
+    if (-1 == req->rpc_status) {
+        gf_smsg(frame->this->name, GF_LOG_WARNING, ENOTCONN,
+                PC_MSG_RPC_STATUS_ERROR, NULL);
+        op_ret = -1;
+        goto out;
+    }
+
+    ret = xdr_to_generic(*iov, &rsp, (xdrproc_t)xdr_gf_setvolume_rsp);
+    if (ret < 0) {
+        gf_smsg(this->name, GF_LOG_ERROR, EINVAL, PC_MSG_XDR_DECODING_FAILED,
+                NULL);
+        op_ret = -1;
+        goto out;
+    }
+    op_ret = rsp.op_ret;
+    op_errno = gf_error_to_errno(rsp.op_errno);
+    if (-1 == rsp.op_ret) {
+        gf_smsg(frame->this->name, GF_LOG_WARNING, op_errno,
+                PC_MSG_VOL_SET_FAIL, NULL);
+    }
+
+    reply = dict_new();
+    if (!reply)
+        goto out;
+
+    if (rsp.dict.dict_len) {
+        ret = dict_unserialize(rsp.dict.dict_val, rsp.dict.dict_len, &reply);
         if (ret < 0) {
-                gf_log (this->name, GF_LOG_ERROR, "XDR decoding failed");
+            gf_smsg(frame->this->name, GF_LOG_WARNING, 0,
+                    PC_MSG_DICT_UNSERIALIZE_FAIL, NULL);
+            goto out;
+        }
+    }
+
+    ret = dict_get_str_sizen(reply, "ERROR", &remote_error);
+    if (ret < 0) {
+        gf_smsg(this->name, GF_LOG_WARNING, EINVAL, PC_MSG_DICT_GET_FAILED,
+                "ERROR string", NULL);
+    }
+
+    ret = dict_get_str_sizen(reply, "process-uuid", &process_uuid);
+    if (ret < 0) {
+        gf_smsg(this->name, GF_LOG_WARNING, EINVAL, PC_MSG_DICT_GET_FAILED,
+                "process-uuid", NULL);
+    }
+
+    if (op_ret < 0) {
+        gf_smsg(this->name, GF_LOG_ERROR, op_errno, PC_MSG_SETVOLUME_FAIL,
+                "remote-error=%s", remote_error, NULL);
+
+        errno = op_errno;
+        if (remote_error && (op_errno == EACCES)) {
+            auth_fail = _gf_true;
+            op_ret = 0;
+        }
+        if ((op_errno == ENOENT) && this->ctx->cmd_args.subdir_mount &&
+            (ctx->graph_id <= 1)) {
+            /* A case of subdir not being present at the moment,
+               ride on auth_fail framework to notify the error */
+            /* Make sure this case is handled only in the new
+               graph, so mount may fail in this case. In case
+               of 'add-brick' etc, we need to continue retry */
+            auth_fail = _gf_true;
+            op_ret = 0;
+        }
+        if (op_errno == ESTALE) {
+            ret = client_notify_dispatch(this, GF_EVENT_VOLFILE_MODIFIED, NULL);
+            if (ret)
+                gf_smsg(this->name, GF_LOG_INFO, 0,
+                        PC_MSG_VOLFILE_NOTIFY_FAILED, NULL);
+        }
+        goto out;
+    }
+
+    ret = dict_get_str_sizen(this->options, "remote-subvolume", &remote_subvol);
+    if (ret || !remote_subvol) {
+        gf_smsg(this->name, GF_LOG_WARNING, 0, PC_MSG_FIND_KEY_FAILED,
+                "remote-subvolume", NULL);
+        goto out;
+    }
+
+    ret = dict_get_str_sizen(reply, "volume-id", &volume_id);
+    if (ret < 0) {
+        /* this can happen if the server is of old version, so treat it as
+           just debug message */
+        gf_msg_debug(this->name, EINVAL,
+                     "failed to get 'volume-id' from reply dict");
+    } else if (ctx->master && strncmp("snapd", remote_subvol, 5)) {
+        /* TODO: if it is a fuse mount or a snapshot enabled client, don't
+           bother */
+        /* If any value is set, the first element will be non-0.
+           It would be '0', but not '\0' :-) */
+        if (ctx->volume_id[0]) {
+            if (strcmp(ctx->volume_id, volume_id)) {
+                /* Ideally it shouldn't even come here, as server itself
+                   should fail the handshake in that case */
+                gf_smsg(this->name, GF_LOG_ERROR, EINVAL, PC_MSG_VOL_ID_CHANGED,
+                        "vol-id=%s", volume_id, "ctx->vol-id=%s",
+                        ctx->volume_id, NULL);
                 op_ret = -1;
                 goto out;
-        }
-        op_ret   = rsp.op_ret;
-        op_errno = gf_error_to_errno (rsp.op_errno);
-        if (-1 == rsp.op_ret) {
-                gf_log (frame->this->name, GF_LOG_WARNING,
-                        "failed to set the volume (%s)",
-                        (op_errno)? strerror (op_errno) : "--");
-        }
-
-        reply = dict_new ();
-        if (!reply)
-                goto out;
-
-        if (rsp.dict.dict_len) {
-                ret = dict_unserialize (rsp.dict.dict_val,
-                                        rsp.dict.dict_len, &reply);
-                if (ret < 0) {
-                        gf_log (frame->this->name, GF_LOG_WARNING,
-                                "failed to unserialize buffer to dict");
-                        goto out;
-                }
-        }
-
-        ret = dict_get_str (reply, "ERROR", &remote_error);
-        if (ret < 0) {
-                gf_log (this->name, GF_LOG_WARNING,
-                        "failed to get ERROR string from reply dict");
-        }
-
-        ret = dict_get_str (reply, "process-uuid", &process_uuid);
-        if (ret < 0) {
-                gf_log (this->name, GF_LOG_WARNING,
-                        "failed to get 'process-uuid' from reply dict");
-        }
-
-        if (op_ret < 0) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "SETVOLUME on remote-host failed: %s",
-                        remote_error ? remote_error : strerror (op_errno));
-                errno = op_errno;
-                if (remote_error &&
-                    (strcmp ("Authentication failed", remote_error) == 0)) {
-                        auth_fail = _gf_true;
-                        op_ret = 0;
-                }
-                if (op_errno == ESTALE) {
-                        ret = client_notify_dispatch (this,
-                                                      GF_EVENT_VOLFILE_MODIFIED,
-                                                      NULL);
-                        if (ret)
-                                gf_log (this->name, GF_LOG_INFO,
-                                        "notify of VOLFILE_MODIFIED failed");
-                }
-                goto out;
-        }
-
-        ret = dict_get_str (this->options, "remote-subvolume",
-                            &remote_subvol);
-        if (ret || !remote_subvol) {
-                gf_log (this->name, GF_LOG_WARNING,
-                        "failed to find key 'remote-subvolume' in the options");
-                goto out;
-        }
-
-        ret = dict_get_uint32 (reply, "clnt-lk-version", &lk_ver);
-        if (ret) {
-                gf_log (this->name, GF_LOG_WARNING,
-                        "failed to find key 'clnt-lk-version' in the options");
-                goto out;
-        }
-
-        gf_log (this->name, GF_LOG_DEBUG, "clnt-lk-version = %d, "
-                "server-lk-version = %d", client_get_lk_ver (conf), lk_ver);
-        /* TODO: currently setpeer path is broken */
-        /*
-        if (process_uuid && req->conn &&
-            !strcmp (this->ctx->process_uuid, process_uuid)) {
-                rpc_transport_t      *peer_trans    = NULL;
-                uint64_t              peertrans_int = 0;
-
-                ret = dict_get_uint64 (reply, "transport-ptr",
-                                       &peertrans_int);
-                if (ret)
-                        goto out;
-
-                gf_log (this->name, GF_LOG_WARNING,
-                        "attaching to the local volume '%s'",
-                        remote_subvol);
-
-                peer_trans = (void *) (long) (peertrans_int);
-
-                rpc_transport_setpeer (req->conn->trans, peer_trans);
-        }
-        */
-
-        conf->client_id = glusterfs_leaf_position(this);
-
-        gf_log (this->name, GF_LOG_INFO,
-                "Connected to %s, attached to remote volume '%s'.",
-                conf->rpc->conn.name,
-                remote_subvol);
-
-        rpc_clnt_set_connected (&conf->rpc->conn);
-
-        op_ret = 0;
-        conf->connecting = 0;
-        conf->connected = 1;
-
-        if (lk_ver != client_get_lk_ver (conf)) {
-                gf_log (this->name, GF_LOG_INFO, "Server and Client "
-                        "lk-version numbers are not same, reopening the fds");
-                client_mark_fd_bad (this);
-                client_post_handshake (frame, frame->this);
+            }
         } else {
-                /*TODO: Traverse the saved fd list, and send
-                  release to the server on fd's that were closed
-                  during grace period */
-                gf_log (this->name, GF_LOG_INFO, "Server and Client "
-                        "lk-version numbers are same, no need to "
-                        "reopen the fds");
-                client_notify_parents_child_up (frame->this);
+            strncpy(ctx->volume_id, volume_id, GF_UUID_BUF_SIZE);
         }
+    }
 
+    uint32_t child_up_int;
+    ret = dict_get_uint32(reply, "child_up", &child_up_int);
+    if (ret) {
+        /*
+         * This would happen in cases where the server trying to     *
+         * connect to this client is running an older version. Hence *
+         * setting the child_up to _gf_true in this case.            *
+         */
+        gf_smsg(this->name, GF_LOG_WARNING, 0, PC_MSG_FIND_KEY_FAILED,
+                "child_up", NULL);
+        conf->child_up = _gf_true;
+    } else {
+        conf->child_up = (child_up_int != 0);
+    }
+
+    /* TODO: currently setpeer path is broken */
+    /*
+    if (process_uuid && req->conn &&
+        !strcmp (this->ctx->process_uuid, process_uuid)) {
+            rpc_transport_t      *peer_trans    = NULL;
+            uint64_t              peertrans_int = 0;
+
+            ret = dict_get_uint64 (reply, "transport-ptr",
+                                   &peertrans_int);
+            if (ret)
+                    goto out;
+
+            gf_log (this->name, GF_LOG_WARNING,
+                    "attaching to the local volume '%s'",
+                    remote_subvol);
+
+            peer_trans = (void *) (long) (peertrans_int);
+
+            rpc_transport_setpeer (req->conn->trans, peer_trans);
+    }
+    */
+
+    conf->client_id = glusterfs_leaf_position(this);
+
+    gf_smsg(this->name, GF_LOG_INFO, 0, PC_MSG_REMOTE_VOL_CONNECTED,
+            "conn-name=%s", conf->rpc->conn.name, "remote_subvol=%s",
+            remote_subvol, NULL);
+
+    op_ret = 0;
+    conf->connected = 1;
+
+    client_post_handshake(frame, frame->this);
 out:
-        if (auth_fail) {
-                gf_log (this->name, GF_LOG_INFO, "sending AUTH_FAILED event");
-                ret = client_notify_dispatch (this, GF_EVENT_AUTH_FAILED, NULL);
-                if (ret)
-                        gf_log (this->name, GF_LOG_INFO,
-                                "notify of AUTH_FAILED failed");
-                conf->connecting = 0;
-                conf->connected = 0;
-                ret = -1;
-        }
-        if (-1 == op_ret) {
-                /* Let the connection/re-connection happen in
-                 * background, for now, don't hang here,
-                 * tell the parents that i am all ok..
-                 */
-                gf_log (this->name, GF_LOG_INFO, "sending CHILD_CONNECTING event");
-                ret = client_notify_dispatch (this, GF_EVENT_CHILD_CONNECTING,
-                                              NULL);
-                if (ret)
-                        gf_log (this->name, GF_LOG_INFO,
-                                "notify of CHILD_CONNECTING failed");
-                conf->connecting= 1;
-                ret = 0;
-        }
+    if (auth_fail) {
+        gf_smsg(this->name, GF_LOG_INFO, 0, PC_MSG_AUTH_FAILED, NULL);
+        ret = client_notify_dispatch(this, GF_EVENT_AUTH_FAILED, NULL);
+        if (ret)
+            gf_smsg(this->name, GF_LOG_INFO, 0,
+                    PC_MSG_AUTH_FAILED_NOTIFY_FAILED, NULL);
+        conf->connected = 0;
+        ret = -1;
+    }
+    if (-1 == op_ret) {
+        /* Let the connection/re-connection happen in
+         * background, for now, don't hang here,
+         * tell the parents that i am all ok..
+         */
+        gf_smsg(this->name, GF_LOG_INFO, 0, PC_MSG_CHILD_CONNECTING_EVENT,
+                NULL);
+        ret = client_notify_dispatch(this, GF_EVENT_CHILD_CONNECTING, NULL);
+        if (ret)
+            gf_smsg(this->name, GF_LOG_INFO, 0,
+                    PC_MSG_CHILD_CONNECTING_NOTIFY_FAILED, NULL);
+        /*
+         * The reconnection *won't* happen in the background (see
+         * previous comment) unless we kill the current connection.
+         */
+        rpc_transport_disconnect(conf->rpc->conn.trans, _gf_false);
+        ret = 0;
+    }
 
-        free (rsp.dict.dict_val);
+    free(rsp.dict.dict_val);
 
-        STACK_DESTROY (frame->root);
+    STACK_DESTROY(frame->root);
 
-        if (reply)
-                dict_unref (reply);
+    if (reply)
+        dict_unref(reply);
 
-        return ret;
+    return ret;
 }
 
 int
-client_setvolume (xlator_t *this, struct rpc_clnt *rpc)
+client_setvolume(xlator_t *this, struct rpc_clnt *rpc)
 {
-        int               ret             = 0;
-        gf_setvolume_req  req             = {{0,},};
-        call_frame_t     *fr              = NULL;
-        char             *process_uuid_xl = NULL;
-        clnt_conf_t      *conf            = NULL;
-        dict_t           *options         = NULL;
-        char             counter_str[32]  = {0};
-
-        options = this->options;
-        conf    = this->private;
-
-        if (conf->fops) {
-                ret = dict_set_int32 (options, "fops-version",
-                                      conf->fops->prognum);
-                if (ret < 0) {
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "failed to set version-fops(%d) in handshake msg",
-                                conf->fops->prognum);
-                        goto fail;
-                }
-        }
-
-        if (conf->mgmt) {
-                ret = dict_set_int32 (options, "mgmt-version", conf->mgmt->prognum);
-                if (ret < 0) {
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "failed to set version-mgmt(%d) in handshake msg",
-                                conf->mgmt->prognum);
-                        goto fail;
-                }
-        }
-
-        /* When lock-heal is enabled:
-         * With multiple graphs possible in the same process, we need a
-           field to bring the uniqueness. Graph-ID should be enough to get the
-           job done.
-         * When lock-heal is disabled, connection-id should always be unique so
-         * that server never gets to reuse the previous connection resources
-         * so it cleans up the resources on every disconnect. Otherwise
-         * it may lead to stale resources, i.e. leaked file desciptors,
-         * inode/entry locks
-        */
-        if (!conf->lk_heal) {
-                snprintf (counter_str, sizeof (counter_str),
-                          "-%"PRIu64, conf->setvol_count);
-                conf->setvol_count++;
-        }
-        ret = gf_asprintf (&process_uuid_xl, "%s-%s-%d%s",
-                           this->ctx->process_uuid, this->name,
-                           this->graph->id, counter_str);
-        if (-1 == ret) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "asprintf failed while setting process_uuid");
-                goto fail;
-        }
-
-        ret = dict_set_dynstr (options, "process-uuid", process_uuid_xl);
-        if (ret < 0) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "failed to set process-uuid(%s) in handshake msg",
-                        process_uuid_xl);
-                goto fail;
-        }
-
-        ret = dict_set_str (options, "client-version", PACKAGE_VERSION);
-        if (ret < 0) {
-                gf_log (this->name, GF_LOG_WARNING,
-                        "failed to set client-version(%s) in handshake msg",
-                        PACKAGE_VERSION);
-        }
-
-        if (this->ctx->cmd_args.volfile_server) {
-                if (this->ctx->cmd_args.volfile_id) {
-                        ret = dict_set_str (options, "volfile-key",
-                                            this->ctx->cmd_args.volfile_id);
-                        if (ret)
-                                gf_log (this->name, GF_LOG_ERROR,
-                                        "failed to set 'volfile-key'");
-                }
-                ret = dict_set_uint32 (options, "volfile-checksum",
-                                       this->graph->volfile_checksum);
-                if (ret)
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "failed to set 'volfile-checksum'");
-        }
-
-        ret = dict_set_int16 (options, "clnt-lk-version",
-                              client_get_lk_ver (conf));
+    int ret = 0;
+    gf_setvolume_req req = {
+        {
+            0,
+        },
+    };
+    call_frame_t *fr = NULL;
+    char *process_uuid_xl = NULL;
+    char *remote_subvol = NULL;
+    clnt_conf_t *conf = this->private;
+    dict_t *options = this->options;
+    char counter_str[32] = {0};
+    char hostname[256] = {
+        0,
+    };
+
+    if (conf->fops) {
+        ret = dict_set_int32_sizen(options, "fops-version",
+                                   conf->fops->prognum);
         if (ret < 0) {
-                gf_log (this->name, GF_LOG_WARNING,
-                        "failed to set clnt-lk-version(%"PRIu32") in handshake msg",
-                        client_get_lk_ver (conf));
+            gf_smsg(this->name, GF_LOG_ERROR, 0, PC_MSG_DICT_SET_FAILED,
+                    "version-fops=%d", conf->fops->prognum, NULL);
+            goto fail;
         }
+    }
 
-        ret = dict_serialized_length (options);
+    if (conf->mgmt) {
+        ret = dict_set_int32_sizen(options, "mgmt-version",
+                                   conf->mgmt->prognum);
         if (ret < 0) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "failed to get serialized length of dict");
-                ret = -1;
-                goto fail;
-        }
-        req.dict.dict_len = ret;
-        req.dict.dict_val = GF_CALLOC (1, req.dict.dict_len,
-                                       gf_client_mt_clnt_req_buf_t);
-        ret = dict_serialize (options, req.dict.dict_val);
+            gf_smsg(this->name, GF_LOG_ERROR, 0, PC_MSG_DICT_SET_FAILED,
+                    "version-mgmt=%d", conf->mgmt->prognum, NULL);
+            goto fail;
+        }
+    }
+
+    /*
+     * Connection-id should always be unique so that server never gets to
+     * reuse the previous connection resources so it cleans up the resources
+     * on every disconnect. Otherwise it may lead to stale resources, i.e.
+     * leaked file descriptors, inode/entry locks
+     */
+
+    snprintf(counter_str, sizeof(counter_str), "-%" PRIu64, conf->setvol_count);
+    conf->setvol_count++;
+
+    if (gethostname(hostname, 256) == -1) {
+        gf_smsg(this->name, GF_LOG_ERROR, errno, PC_MSG_GETHOSTNAME_FAILED,
+                NULL);
+
+        goto fail;
+    }
+
+    ret = gf_asprintf(&process_uuid_xl, GLUSTER_PROCESS_UUID_FMT,
+                      this->ctx->process_uuid, this->graph->id, getpid(),
+                      hostname, this->name, counter_str);
+    if (-1 == ret) {
+        gf_smsg(this->name, GF_LOG_ERROR, 0, PC_MSG_PROCESS_UUID_SET_FAIL,
+                NULL);
+        goto fail;
+    }
+
+    ret = dict_set_dynstr_sizen(options, "process-uuid", process_uuid_xl);
+    if (ret < 0) {
+        gf_smsg(this->name, GF_LOG_ERROR, 0, PC_MSG_DICT_SET_FAILED,
+                "process-uuid=%s", process_uuid_xl, NULL);
+        goto fail;
+    }
+
+    if (this->ctx->cmd_args.process_name) {
+        ret = dict_set_str_sizen(options, "process-name",
+                                 this->ctx->cmd_args.process_name);
         if (ret < 0) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "failed to serialize dictionary");
-                goto fail;
-        }
-
-        fr  = create_frame (this, this->ctx->pool);
-        if (!fr)
-                goto fail;
+            gf_smsg(this->name, GF_LOG_INFO, 0, PC_MSG_DICT_SET_FAILED,
+                    "process-name", NULL);
+        }
+    }
+
+    ret = dict_set_str_sizen(options, "client-version", PACKAGE_VERSION);
+    if (ret < 0) {
+        gf_smsg(this->name, GF_LOG_WARNING, 0, PC_MSG_DICT_SET_FAILED,
+                "client-version=%s", PACKAGE_VERSION, NULL);
+    }
+
+    ret = dict_get_str_sizen(this->options, "remote-subvolume", &remote_subvol);
+    if (ret || !remote_subvol) {
+        gf_smsg(this->name, GF_LOG_WARNING, 0, PC_MSG_FIND_KEY_FAILED,
+                "remote-subvolume", NULL);
+        goto fail;
+    }
+
+    /* volume-id to be sent only for regular volume, not snap volume */
+    if (strncmp("snapd", remote_subvol, 5)) {
+        /* If any value is set, the first element will be non-0.
+           It would be '0', but not '\0' :-) */
+        if (!this->ctx->volume_id[0]) {
+            strncpy(this->ctx->volume_id, this->graph->volume_id,
+                    GF_UUID_BUF_SIZE);
+        }
+        if (this->ctx->volume_id[0]) {
+            ret = dict_set_str(options, "volume-id", this->ctx->volume_id);
+            if (ret < 0) {
+                gf_smsg(this->name, GF_LOG_INFO, 0, PC_MSG_DICT_SET_FAILED,
+                        "volume-id", NULL);
+            }
+        }
+    }
+
+    if (this->ctx->cmd_args.volfile_server) {
+        if (this->ctx->cmd_args.volfile_id) {
+            ret = dict_set_str_sizen(options, "volfile-key",
+                                     this->ctx->cmd_args.volfile_id);
+            if (ret)
+                gf_smsg(this->name, GF_LOG_ERROR, 0,
+                        PC_MSG_VOLFILE_KEY_SET_FAILED, NULL);
+        }
+        ret = dict_set_uint32(options, "volfile-checksum",
+                              this->graph->volfile_checksum);
+        if (ret)
+            gf_smsg(this->name, GF_LOG_ERROR, 0, PC_MSG_VOLFILE_CHECKSUM_FAILED,
+                    NULL);
+    }
 
-        ret = client_submit_request (this, &req, fr, conf->handshake,
-                                     GF_HNDSK_SETVOLUME, client_setvolume_cbk,
-                                     NULL, NULL, 0, NULL, 0, NULL,
-                                     (xdrproc_t)xdr_gf_setvolume_req);
+    if (this->ctx->cmd_args.subdir_mount) {
+        ret = dict_set_str_sizen(options, "subdir-mount",
+                                 this->ctx->cmd_args.subdir_mount);
+        if (ret) {
+            gf_log(THIS->name, GF_LOG_ERROR, "Failed to set subdir_mount");
+            /* It makes sense to fail, as per the CLI, we
+               should be doing a subdir_mount */
+            goto fail;
+        }
+    }
+
+    /* Insert a dummy key value pair to avoid failure at server side for
+     * clnt-lk-version with new clients.
+     */
+    ret = dict_set_uint32(options, "clnt-lk-version", 1);
+    if (ret < 0) {
+        gf_smsg(this->name, GF_LOG_WARNING, 0, PC_MSG_DICT_SET_FAILED,
+                "clnt-lk-version(1)", NULL);
+    }
+
+    ret = dict_set_int32_sizen(options, "opversion", GD_OP_VERSION_MAX);
+    if (ret < 0) {
+        gf_smsg(this->name, GF_LOG_ERROR, 0, PC_MSG_DICT_SET_FAILED,
+                "client opversion", NULL);
+    }
+
+    ret = dict_allocate_and_serialize(options, (char **)&req.dict.dict_val,
+                                      &req.dict.dict_len);
+    if (ret != 0) {
+        ret = -1;
+        gf_smsg(this->name, GF_LOG_ERROR, 0, PC_MSG_DICT_SERIALIZE_FAIL, NULL);
+        goto fail;
+    }
+
+    fr = create_frame(this, this->ctx->pool);
+    if (!fr)
+        goto fail;
+
+    ret = client_submit_request(this, &req, fr, conf->handshake,
+                                GF_HNDSK_SETVOLUME, client_setvolume_cbk, NULL,
+                                (xdrproc_t)xdr_gf_setvolume_req);
 
 fail:
-        GF_FREE (req.dict.dict_val);
+    GF_FREE(req.dict.dict_val);
 
-        return ret;
+    return ret;
 }
 
-int
-select_server_supported_programs (xlator_t *this, gf_prog_detail *prog)
+static int
+select_server_supported_programs(xlator_t *this, gf_prog_detail *prog)
 {
-        gf_prog_detail *trav     = NULL;
-        clnt_conf_t    *conf     = NULL;
-        int             ret      = -1;
-
-        if (!this || !prog) {
-                gf_log (THIS->name, GF_LOG_WARNING,
-                        "xlator not found OR RPC program not found");
-                goto out;
+    gf_prog_detail *trav = NULL;
+    clnt_conf_t *conf = NULL;
+    int ret = -1;
+
+    if (!this || !prog) {
+        gf_smsg(THIS->name, GF_LOG_WARNING, 0, PC_MSG_PGM_NOT_FOUND, NULL);
+        goto out;
+    }
+
+    conf = this->private;
+    trav = prog;
+
+    while (trav) {
+        /* Select 'programs' */
+        if ((clnt3_3_fop_prog.prognum == trav->prognum) &&
+            (clnt3_3_fop_prog.progver == trav->progver)) {
+            conf->fops = &clnt3_3_fop_prog;
+            if (conf->rpc)
+                conf->rpc->auth_value = AUTH_GLUSTERFS_v2;
+            ret = 0;
+            /* In normal flow, we don't want to use old protocol type.
+               but if it is for testing, lets use it */
+            if (conf->old_protocol)
+                goto done;
+        }
+
+        if ((clnt4_0_fop_prog.prognum == trav->prognum) &&
+            (clnt4_0_fop_prog.progver == trav->progver)) {
+            conf->fops = &clnt4_0_fop_prog;
+            if (conf->rpc)
+                conf->rpc->auth_value = AUTH_GLUSTERFS_v3;
+            ret = 0;
+            /* this is latest program, lets use this program only */
+            /* if we are testing for old-protocol, lets not break this */
+            if (!conf->old_protocol)
+                goto done;
         }
 
-        conf = this->private;
-        trav = prog;
-
-        while (trav) {
-                /* Select 'programs' */
-                if ((clnt3_3_fop_prog.prognum == trav->prognum) &&
-                    (clnt3_3_fop_prog.progver == trav->progver)) {
-                        conf->fops = &clnt3_3_fop_prog;
-                        gf_log (this->name, GF_LOG_INFO,
-                                "Using Program %s, Num (%"PRId64"), "
-                                "Version (%"PRId64")",
-                                trav->progname, trav->prognum, trav->progver);
-                        ret = 0;
-                }
-                if (ret) {
-                        gf_log (this->name, GF_LOG_TRACE,
-                                "%s (%"PRId64") not supported", trav->progname,
-                                trav->progver);
-                }
-                trav = trav->next;
+        if (ret) {
+            gf_msg_debug(this->name, 0, "%s (%" PRId64 ") not supported",
+                         trav->progname, trav->progver);
         }
+        trav = trav->next;
+    }
+
+done:
+    if (!ret)
+        gf_smsg(this->name, GF_LOG_INFO, 0, PC_MSG_VERSION_INFO,
+                "Program-name=%s", conf->fops->progname, "Num=%d",
+                conf->fops->prognum, "Version=%d", conf->fops->progver, NULL);
 
 out:
-        return ret;
+    return ret;
 }
 
-
 int
-server_has_portmap (xlator_t *this, gf_prog_detail *prog)
+server_has_portmap(xlator_t *this, gf_prog_detail *prog)
 {
-        gf_prog_detail *trav     = NULL;
-        int             ret      = -1;
+    gf_prog_detail *trav = NULL;
+    int ret = -1;
 
-        if (!this || !prog) {
-                gf_log (THIS->name, GF_LOG_WARNING,
-                        "xlator not found OR RPC program not found");
-                goto out;
-        }
+    if (!this || !prog) {
+        gf_smsg(THIS->name, GF_LOG_WARNING, 0, PC_MSG_PGM_NOT_FOUND, NULL);
+        goto out;
+    }
 
-        trav = prog;
+    trav = prog;
 
-        while (trav) {
-                if ((trav->prognum == GLUSTER_PMAP_PROGRAM) &&
-                    (trav->progver == GLUSTER_PMAP_VERSION)) {
-                        gf_log (this->name, GF_LOG_DEBUG,
-                                "detected portmapper on server");
-                        ret = 0;
-                        break;
-                }
-                trav = trav->next;
+    while (trav) {
+        if ((trav->prognum == GLUSTER_PMAP_PROGRAM) &&
+            (trav->progver == GLUSTER_PMAP_VERSION)) {
+            gf_msg_debug(this->name, 0, "detected portmapper on server");
+            ret = 0;
+            break;
         }
+        trav = trav->next;
+    }
 
 out:
-        return ret;
+    return ret;
 }
 
-
 int
-client_query_portmap_cbk (struct rpc_req *req, struct iovec *iov, int count, void *myframe)
+client_query_portmap_cbk(struct rpc_req *req, struct iovec *iov, int count,
+                         void *myframe)
 {
-        struct pmap_port_by_brick_rsp     rsp   = {0,};
-        call_frame_t                     *frame = NULL;
-        clnt_conf_t                      *conf  = NULL;
-        int                               ret   = -1;
-        struct rpc_clnt_config            config = {0, };
-        xlator_t                         *this   = NULL;
-
-        frame = myframe;
-        if (!frame || !frame->this || !frame->this->private) {
-                gf_log (THIS->name, GF_LOG_WARNING,
-                        "frame not found with rpc request");
-                goto out;
-        }
-        this  = frame->this;
-        conf  = frame->this->private;
-
-        if (-1 == req->rpc_status) {
-                gf_log (this->name, GF_LOG_WARNING,
-                        "received RPC status error, try again later");
-                goto out;
-        }
-
-        ret = xdr_to_generic (*iov, &rsp, (xdrproc_t)xdr_pmap_port_by_brick_rsp);
-        if (ret < 0) {
-                gf_log (this->name, GF_LOG_ERROR, "XDR decoding failed");
-                goto out;
-        }
-
-        if (-1 == rsp.op_ret) {
-                ret = -1;
-                gf_log (this->name, ((!conf->portmap_err_logged) ?
-                                     GF_LOG_ERROR : GF_LOG_DEBUG),
-                        "failed to get the port number for remote subvolume. "
-                        "Please run 'gluster volume status' on server to see "
-                        "if brick process is running.");
-                conf->portmap_err_logged = 1;
-                goto out;
+    struct pmap_port_by_brick_rsp rsp = {
+        0,
+    };
+    call_frame_t *frame = NULL;
+    clnt_conf_t *conf = NULL;
+    int ret = -1;
+    struct rpc_clnt_config config = {
+        0,
+    };
+    xlator_t *this = NULL;
+
+    frame = myframe;
+    if (!frame || !frame->this || !frame->this->private) {
+        gf_smsg(THIS->name, GF_LOG_WARNING, EINVAL, PC_MSG_FRAME_NOT_FOUND,
+                NULL);
+        goto out;
+    }
+    this = frame->this;
+    conf = frame->this->private;
+
+    if (-1 == req->rpc_status) {
+        gf_smsg(this->name, GF_LOG_WARNING, ENOTCONN, PC_MSG_RPC_STATUS_ERROR,
+                NULL);
+        goto out;
+    }
+
+    ret = xdr_to_generic(*iov, &rsp, (xdrproc_t)xdr_pmap_port_by_brick_rsp);
+    if (ret < 0) {
+        gf_smsg(this->name, GF_LOG_ERROR, EINVAL, PC_MSG_XDR_DECODING_FAILED,
+                NULL);
+        goto out;
+    }
+
+    if (-1 == rsp.op_ret) {
+        ret = -1;
+        if (!conf->portmap_err_logged) {
+            gf_smsg(this->name, GF_LOG_ERROR, 0, PC_MSG_PORT_NUM_ERROR, NULL);
+        } else {
+            gf_msg_debug(this->name, 0,
+                         "failed to get the port number for "
+                         "remote subvolume. Please run 'gluster "
+                         "volume status' on server to see "
+                         "if brick process is running.");
         }
+        conf->portmap_err_logged = 1;
+        goto out;
+    }
 
-        conf->portmap_err_logged = 0;
-        conf->disconnect_err_logged = 0;
-        config.remote_port = rsp.port;
-        rpc_clnt_reconfig (conf->rpc, &config);
+    conf->portmap_err_logged = 0;
+    conf->disconnect_err_logged = 0;
+    config.remote_port = rsp.port;
+    rpc_clnt_reconfig(conf->rpc, &config);
 
-        conf->skip_notify = 1;
-	conf->quick_reconnect = 1;
+    conf->skip_notify = 1;
+    conf->quick_reconnect = 1;
 
 out:
-        if (frame)
-                STACK_DESTROY (frame->root);
+    if (frame)
+        STACK_DESTROY(frame->root);
 
-        if (conf) {
-                /* Need this to connect the same transport on different port */
-                /* ie, glusterd to glusterfsd */
-                rpc_transport_disconnect (conf->rpc->conn.trans);
-        }
+    if (conf) {
+        /* Need this to connect the same transport on different port */
+        /* ie, glusterd to glusterfsd */
+        rpc_transport_disconnect(conf->rpc->conn.trans, _gf_false);
+    }
 
-        return ret;
+    return ret;
 }
 
-
 int
-client_query_portmap (xlator_t *this, struct rpc_clnt *rpc)
+client_query_portmap(xlator_t *this, struct rpc_clnt *rpc)
 {
-        int                      ret             = -1;
-        pmap_port_by_brick_req   req             = {0,};
-        call_frame_t            *fr              = NULL;
-        clnt_conf_t             *conf            = NULL;
-        dict_t                  *options         = NULL;
-        char                    *remote_subvol   = NULL;
-        char                    *xprt            = NULL;
-        char                     brick_name[PATH_MAX] = {0,};
-
-        options = this->options;
-        conf    = this->private;
-
-        ret = dict_get_str (options, "remote-subvolume", &remote_subvol);
-        if (ret < 0) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "remote-subvolume not set in volfile");
-                goto fail;
-        }
-
-        req.brick = remote_subvol;
-
-        if (!dict_get_str (options, "transport-type", &xprt)) {
-                if (!strcmp (xprt, "rdma")) {
-                        snprintf (brick_name, sizeof(brick_name), "%s.rdma",
-                                  remote_subvol);
-                        req.brick = brick_name;
-                }
-        }
-
-        fr  = create_frame (this, this->ctx->pool);
-        if (!fr) {
-                ret = -1;
-                goto fail;
-        }
-
-        ret = client_submit_request (this, &req, fr, &clnt_pmap_prog,
-                                     GF_PMAP_PORTBYBRICK,
-                                     client_query_portmap_cbk,
-                                     NULL, NULL, 0, NULL, 0, NULL,
-                                     (xdrproc_t)xdr_pmap_port_by_brick_req);
+    int ret = -1;
+    pmap_port_by_brick_req req = {
+        0,
+    };
+    call_frame_t *fr = NULL;
+    dict_t *options = NULL;
+    char *remote_subvol = NULL;
+    char *xprt = NULL;
+    char brick_name[PATH_MAX] = {
+        0,
+    };
+
+    options = this->options;
+
+    ret = dict_get_str_sizen(options, "remote-subvolume", &remote_subvol);
+    if (ret < 0) {
+        gf_smsg(this->name, GF_LOG_ERROR, 0, PC_MSG_REMOTE_SUBVOL_SET_FAIL,
+                NULL);
+        goto fail;
+    }
+
+    req.brick = remote_subvol;
+
+    if (!dict_get_str_sizen(options, "transport-type", &xprt)) {
+        if (!strcmp(xprt, "rdma")) {
+            snprintf(brick_name, sizeof(brick_name), "%s.rdma", remote_subvol);
+            req.brick = brick_name;
+        }
+    }
+
+    fr = create_frame(this, this->ctx->pool);
+    if (!fr) {
+        ret = -1;
+        goto fail;
+    }
+
+    ret = client_submit_request(this, &req, fr, &clnt_pmap_prog,
+                                GF_PMAP_PORTBYBRICK, client_query_portmap_cbk,
+                                NULL, (xdrproc_t)xdr_pmap_port_by_brick_req);
 
 fail:
-        return ret;
+    return ret;
 }
 
-
-int
-client_dump_version_cbk (struct rpc_req *req, struct iovec *iov, int count,
-                         void *myframe)
+static int
+client_dump_version_cbk(struct rpc_req *req, struct iovec *iov, int count,
+                        void *myframe)
 {
-        gf_dump_rsp     rsp   = {0,};
-        gf_prog_detail *trav  = NULL;
-        gf_prog_detail *next  = NULL;
-        call_frame_t   *frame = NULL;
-        clnt_conf_t    *conf  = NULL;
-        int             ret   = 0;
-
-        frame = myframe;
-        conf  = frame->this->private;
-
-        if (-1 == req->rpc_status) {
-                gf_log (frame->this->name, GF_LOG_WARNING,
-                        "received RPC status error");
-                goto out;
-        }
-
-        ret = xdr_to_generic (*iov, &rsp, (xdrproc_t)xdr_gf_dump_rsp);
-        if (ret < 0) {
-                gf_log (frame->this->name, GF_LOG_ERROR, "XDR decoding failed");
-                goto out;
-        }
-        if (-1 == rsp.op_ret) {
-                gf_log (frame->this->name, GF_LOG_WARNING,
-                        "failed to get the 'versions' from server");
-                goto out;
-        }
-
-        if (server_has_portmap (frame->this, rsp.prog) == 0) {
-                ret = client_query_portmap (frame->this, conf->rpc);
-                goto out;
-        }
-
-        /* Check for the proper version string */
-        /* Reply in "Name:Program-Number:Program-Version,..." format */
-        ret = select_server_supported_programs (frame->this, rsp.prog);
-        if (ret) {
-                gf_log (frame->this->name, GF_LOG_ERROR,
-                        "server doesn't support the version");
-                goto out;
-        }
-
-        client_setvolume (frame->this, conf->rpc);
+    gf_dump_rsp rsp = {
+        0,
+    };
+    gf_prog_detail *trav = NULL;
+    gf_prog_detail *next = NULL;
+    call_frame_t *frame = NULL;
+    clnt_conf_t *conf = NULL;
+    int ret = 0;
+
+    frame = myframe;
+    conf = frame->this->private;
+
+    if (-1 == req->rpc_status) {
+        gf_smsg(frame->this->name, GF_LOG_WARNING, ENOTCONN,
+                PC_MSG_RPC_STATUS_ERROR, NULL);
+        goto out;
+    }
+
+    ret = xdr_to_generic(*iov, &rsp, (xdrproc_t)xdr_gf_dump_rsp);
+    if (ret < 0) {
+        gf_smsg(frame->this->name, GF_LOG_ERROR, EINVAL,
+                PC_MSG_XDR_DECODING_FAILED, NULL);
+        goto out;
+    }
+    if (-1 == rsp.op_ret) {
+        gf_smsg(frame->this->name, GF_LOG_WARNING, 0, PC_MSG_VERSION_ERROR,
+                NULL);
+        goto out;
+    }
+
+    if (server_has_portmap(frame->this, rsp.prog) == 0) {
+        ret = client_query_portmap(frame->this, conf->rpc);
+        goto out;
+    }
+
+    /* Check for the proper version string */
+    /* Reply in "Name:Program-Number:Program-Version,..." format */
+    ret = select_server_supported_programs(frame->this, rsp.prog);
+    if (ret) {
+        gf_smsg(frame->this->name, GF_LOG_ERROR, 0, PC_MSG_VERSION_ERROR, NULL);
+        goto out;
+    }
+
+    client_setvolume(frame->this, conf->rpc);
 
 out:
-        /* don't use GF_FREE, buffer was allocated by libc */
-        if (rsp.prog) {
-                trav = rsp.prog;
-                while (trav) {
-                        next = trav->next;
-                        free (trav->progname);
-                        free (trav);
-                        trav = next;
-                }
+    /* don't use GF_FREE, buffer was allocated by libc */
+    if (rsp.prog) {
+        trav = rsp.prog;
+        while (trav) {
+            next = trav->next;
+            free(trav->progname);
+            free(trav);
+            trav = next;
         }
+    }
 
-        STACK_DESTROY (frame->root);
+    STACK_DESTROY(frame->root);
 
-        if (ret != 0)
-                rpc_transport_disconnect (conf->rpc->conn.trans);
+    if (ret != 0)
+        rpc_transport_disconnect(conf->rpc->conn.trans, _gf_false);
 
-        return ret;
+    return ret;
 }
 
 int
-client_handshake (xlator_t *this, struct rpc_clnt *rpc)
+client_handshake(xlator_t *this, struct rpc_clnt *rpc)
 {
-        call_frame_t *frame = NULL;
-        clnt_conf_t  *conf  = NULL;
-        gf_dump_req   req   = {0,};
-        int           ret   = 0;
-
-        conf = this->private;
-        if (!conf->handshake) {
-                gf_log (this->name, GF_LOG_WARNING, "handshake program not found");
-                goto out;
-        }
-
-        frame = create_frame (this, this->ctx->pool);
-        if (!frame)
-                goto out;
-
-        req.gfs_id = 0xbabe;
-        ret = client_submit_request (this, &req, frame, conf->dump,
-                                     GF_DUMP_DUMP, client_dump_version_cbk,
-                                     NULL, NULL, 0, NULL, 0,
-                                     NULL, (xdrproc_t)xdr_gf_dump_req);
+    call_frame_t *frame = NULL;
+    clnt_conf_t *conf = NULL;
+    gf_dump_req req = {
+        0,
+    };
+    int ret = 0;
+
+    conf = this->private;
+    if (!conf->handshake) {
+        gf_smsg(this->name, GF_LOG_WARNING, 0, PC_MSG_HANDSHAKE_PGM_NOT_FOUND,
+                NULL);
+        goto out;
+    }
+
+    frame = create_frame(this, this->ctx->pool);
+    if (!frame)
+        goto out;
+
+    req.gfs_id = 0xbabe;
+    ret = client_submit_request(this, &req, frame, conf->dump, GF_DUMP_DUMP,
+                                client_dump_version_cbk, NULL,
+                                (xdrproc_t)xdr_gf_dump_req);
 
 out:
-        return ret;
+    return ret;
 }
 
 char *clnt_handshake_procs[GF_HNDSK_MAXVALUE] = {
-        [GF_HNDSK_NULL]         = "NULL",
-        [GF_HNDSK_SETVOLUME]    = "SETVOLUME",
-        [GF_HNDSK_GETSPEC]      = "GETSPEC",
-        [GF_HNDSK_PING]         = "PING",
-        [GF_HNDSK_SET_LK_VER]   = "SET_LK_VER"
+    [GF_HNDSK_NULL] = "NULL",
+    [GF_HNDSK_SETVOLUME] = "SETVOLUME",
+    [GF_HNDSK_GETSPEC] = "GETSPEC",
+    [GF_HNDSK_PING] = "PING",
 };
 
 rpc_clnt_prog_t clnt_handshake_prog = {
-        .progname  = "GlusterFS Handshake",
-        .prognum   = GLUSTER_HNDSK_PROGRAM,
-        .progver   = GLUSTER_HNDSK_VERSION,
-        .procnames = clnt_handshake_procs,
+    .progname = "GlusterFS Handshake",
+    .prognum = GLUSTER_HNDSK_PROGRAM,
+    .progver = GLUSTER_HNDSK_VERSION,
+    .procnames = clnt_handshake_procs,
 };
 
 char *clnt_dump_proc[GF_DUMP_MAXVALUE] = {
-        [GF_DUMP_NULL] = "NULL",
-        [GF_DUMP_DUMP] = "DUMP",
+    [GF_DUMP_NULL] = "NULL",
+    [GF_DUMP_DUMP] = "DUMP",
 };
 
 rpc_clnt_prog_t clnt_dump_prog = {
-        .progname  = "GF-DUMP",
-        .prognum   = GLUSTER_DUMP_PROGRAM,
-        .progver   = GLUSTER_DUMP_VERSION,
-        .procnames = clnt_dump_proc,
+    .progname = "GF-DUMP",
+    .prognum = GLUSTER_DUMP_PROGRAM,
+    .progver = GLUSTER_DUMP_VERSION,
+    .procnames = clnt_dump_proc,
 };
 
 char *clnt_pmap_procs[GF_PMAP_MAXVALUE] = {
-        [GF_PMAP_PORTBYBRICK] = "PORTBYBRICK",
+    [GF_PMAP_PORTBYBRICK] = "PORTBYBRICK",
 };
 
 rpc_clnt_prog_t clnt_pmap_prog = {
-        .progname   = "PORTMAP",
-        .prognum    = GLUSTER_PMAP_PROGRAM,
-        .progver    = GLUSTER_PMAP_VERSION,
-        .procnames  = clnt_pmap_procs,
+    .progname = "PORTMAP",
+    .prognum = GLUSTER_PMAP_PROGRAM,
+    .progver = GLUSTER_PMAP_VERSION,
+    .procnames = clnt_pmap_procs,
 };
diff --git a/xlators/protocol/client/src/client-helpers.c b/xlators/protocol/client/src/client-helpers.c
index be5e7b57739..189dfddd021 100644
--- a/xlators/protocol/client/src/client-helpers.c
+++ b/xlators/protocol/client/src/client-helpers.c
@@ -8,353 +8,892 @@
   cases as published by the Free Software Foundation.
 */
 
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
-
 #include "client.h"
-#include "fd.h"
+#include <glusterfs/fd.h>
+#include "client-messages.h"
+#include "client-common.h"
+#include <glusterfs/compat-errno.h>
+#include <glusterfs/common-utils.h>
 
 int
-client_fd_lk_list_empty (fd_lk_ctx_t *lk_ctx, gf_boolean_t try_lock)
+client_fd_lk_list_empty(fd_lk_ctx_t *lk_ctx, gf_boolean_t try_lock)
 {
-        int  ret = 1;
-
-        if (!lk_ctx) {
-                ret = -1;
-                goto out;
-        }
-
-        if (try_lock) {
-                ret = TRY_LOCK (&lk_ctx->lock);
-                if (ret != 0) {
-                        ret = -1;
-                        goto out;
-                }
-        } else {
-                LOCK (&lk_ctx->lock);
+    int ret = 1;
+
+    if (!lk_ctx) {
+        ret = -1;
+        goto out;
+    }
+
+    if (try_lock) {
+        ret = TRY_LOCK(&lk_ctx->lock);
+        if (ret != 0) {
+            ret = -1;
+            goto out;
         }
+    } else {
+        LOCK(&lk_ctx->lock);
+    }
 
-        ret = list_empty (&lk_ctx->lk_list);
-        UNLOCK (&lk_ctx->lock);
+    ret = list_empty(&lk_ctx->lk_list);
+    UNLOCK(&lk_ctx->lock);
 out:
-        return ret;
+    return ret;
 }
 
 clnt_fd_ctx_t *
-this_fd_del_ctx (fd_t *file, xlator_t *this)
+this_fd_del_ctx(fd_t *file, xlator_t *this)
 {
-        int         dict_ret = -1;
-        uint64_t    ctxaddr  = 0;
+    int dict_ret = -1;
+    uint64_t ctxaddr = 0;
 
-        GF_VALIDATE_OR_GOTO ("client", this, out);
-        GF_VALIDATE_OR_GOTO (this->name, file, out);
+    GF_VALIDATE_OR_GOTO("client", this, out);
+    GF_VALIDATE_OR_GOTO(this->name, file, out);
 
-        dict_ret = fd_ctx_del (file, this, &ctxaddr);
+    dict_ret = fd_ctx_del(file, this, &ctxaddr);
 
-        if (dict_ret < 0) {
-                ctxaddr = 0;
-        }
+    if (dict_ret < 0) {
+        ctxaddr = 0;
+    }
 
 out:
-        return (clnt_fd_ctx_t *)(unsigned long)ctxaddr;
+    return (clnt_fd_ctx_t *)(unsigned long)ctxaddr;
 }
 
-
 clnt_fd_ctx_t *
-this_fd_get_ctx (fd_t *file, xlator_t *this)
+this_fd_get_ctx(fd_t *file, xlator_t *this)
 {
-        int         dict_ret = -1;
-        uint64_t    ctxaddr = 0;
+    int dict_ret = -1;
+    uint64_t ctxaddr = 0;
 
-        GF_VALIDATE_OR_GOTO ("client", this, out);
-        GF_VALIDATE_OR_GOTO (this->name, file, out);
+    GF_VALIDATE_OR_GOTO("client", this, out);
+    GF_VALIDATE_OR_GOTO(this->name, file, out);
 
-        dict_ret = fd_ctx_get (file, this, &ctxaddr);
+    dict_ret = fd_ctx_get(file, this, &ctxaddr);
 
-        if (dict_ret < 0) {
-                ctxaddr = 0;
-        }
+    if (dict_ret < 0) {
+        ctxaddr = 0;
+    }
 
 out:
-        return (clnt_fd_ctx_t *)(unsigned long)ctxaddr;
+    return (clnt_fd_ctx_t *)(unsigned long)ctxaddr;
 }
 
-
 void
-this_fd_set_ctx (fd_t *file, xlator_t *this, loc_t *loc, clnt_fd_ctx_t *ctx)
+this_fd_set_ctx(fd_t *file, xlator_t *this, loc_t *loc, clnt_fd_ctx_t *ctx)
 {
-        uint64_t oldaddr = 0;
-        int32_t  ret = -1;
-
-        GF_VALIDATE_OR_GOTO ("client", this, out);
-        GF_VALIDATE_OR_GOTO (this->name, file, out);
-
-        ret = fd_ctx_get (file, this, &oldaddr);
-        if (ret >= 0) {
-                if (loc)
-                        gf_log (this->name, GF_LOG_INFO,
-                                "%s (%s): trying duplicate remote fd set. ",
-                                loc->path, uuid_utoa (loc->inode->gfid));
-                else
-                        gf_log (this->name, GF_LOG_INFO,
-                                "%p: trying duplicate remote fd set. ", file);
+    uint64_t oldaddr = 0;
+    int32_t ret = -1;
+
+    GF_VALIDATE_OR_GOTO("client", this, out);
+    GF_VALIDATE_OR_GOTO(this->name, file, out);
+
+    ret = fd_ctx_get(file, this, &oldaddr);
+    if (ret >= 0) {
+        if (loc)
+            gf_smsg(this->name, GF_LOG_INFO, 0, PC_MSG_FD_DUPLICATE_TRY,
+                    "path=%s", loc->path, "gfid=%s",
+                    uuid_utoa(loc->inode->gfid), NULL);
+        else
+            gf_smsg(this->name, GF_LOG_INFO, 0, PC_MSG_FD_DUPLICATE_TRY,
+                    "file=%p", file, NULL);
+    }
+
+    ret = fd_ctx_set(file, this, (uint64_t)(unsigned long)ctx);
+    if (ret < 0) {
+        if (loc)
+            gf_smsg(this->name, GF_LOG_WARNING, 0, PC_MSG_FD_SET_FAIL,
+                    "path=%s", loc->path, "gfid=%s",
+                    uuid_utoa(loc->inode->gfid), NULL);
+        else
+            gf_smsg(this->name, GF_LOG_WARNING, 0, PC_MSG_FD_SET_FAIL,
+                    "file=%p", file, NULL);
+    }
+out:
+    return;
+}
+
+int
+client_local_wipe(clnt_local_t *local)
+{
+    if (local) {
+        loc_wipe(&local->loc);
+        loc_wipe(&local->loc2);
+
+        if (local->fd) {
+            fd_unref(local->fd);
         }
 
-        ret = fd_ctx_set (file, this, (uint64_t)(unsigned long)ctx);
-        if (ret < 0) {
-                if (loc)
-                        gf_log (this->name, GF_LOG_WARNING,
-                                "%s (%s): failed to set remote fd",
-                                loc->path, uuid_utoa (loc->inode->gfid));
-                else
-                        gf_log (this->name, GF_LOG_WARNING,
-                                "%p: failed to set remote fd", file);
+        if (local->iobref) {
+            iobref_unref(local->iobref);
         }
-out:
-        return;
-}
 
+        GF_FREE(local->name);
+        mem_put(local);
+    }
 
+    return 0;
+}
 int
-client_local_wipe (clnt_local_t *local)
+unserialize_rsp_dirent(xlator_t *this, struct gfs3_readdir_rsp *rsp,
+                       gf_dirent_t *entries)
 {
-        if (local) {
-                loc_wipe (&local->loc);
-                loc_wipe (&local->loc2);
+    struct gfs3_dirlist *trav = NULL;
+    gf_dirent_t *entry = NULL;
+    int entry_len = 0;
+    int ret = -1;
+    clnt_conf_t *conf = NULL;
 
-                if (local->fd) {
-                        fd_unref (local->fd);
-                }
+    conf = this->private;
 
-                if (local->iobref) {
-                        iobref_unref (local->iobref);
-                }
+    trav = rsp->reply;
+    while (trav) {
+        entry_len = gf_dirent_size(trav->name);
+        entry = GF_CALLOC(1, entry_len, gf_common_mt_gf_dirent_t);
+        if (!entry)
+            goto out;
 
-                GF_FREE (local->name);
+        entry->d_ino = trav->d_ino;
+        gf_itransform(this, trav->d_off, &entry->d_off, conf->client_id);
+        entry->d_len = trav->d_len;
+        entry->d_type = trav->d_type;
 
-                mem_put (local);
-        }
+        strcpy(entry->d_name, trav->name);
 
-        return 0;
+        list_add_tail(&entry->list, &entries->list);
+
+        trav = trav->nextentry;
+    }
+
+    ret = 0;
+out:
+    return ret;
 }
 
 int
-unserialize_rsp_dirent (xlator_t *this, struct gfs3_readdir_rsp *rsp,
+unserialize_rsp_direntp(xlator_t *this, fd_t *fd, struct gfs3_readdirp_rsp *rsp,
                         gf_dirent_t *entries)
 {
-        struct gfs3_dirlist  *trav      = NULL;
-	gf_dirent_t          *entry     = NULL;
-        int                   entry_len = 0;
-        int                   ret       = -1;
-        clnt_conf_t          *conf = NULL;
+    struct gfs3_dirplist *trav = NULL;
+    gf_dirent_t *entry = NULL;
+    inode_table_t *itable = NULL;
+    int entry_len = 0;
+    int ret = -1;
+    clnt_conf_t *conf = NULL;
+
+    trav = rsp->reply;
+
+    if (fd)
+        itable = fd->inode->table;
 
-        conf = this->private;
+    conf = this->private;
+    if (!conf)
+        goto out;
 
-        trav = rsp->reply;
-        while (trav) {
-                entry_len = gf_dirent_size (trav->name);
-                entry = GF_CALLOC (1, entry_len, gf_common_mt_gf_dirent_t);
-                if (!entry)
-                        goto out;
+    while (trav) {
+        entry_len = gf_dirent_size(trav->name);
+        entry = GF_CALLOC(1, entry_len, gf_common_mt_gf_dirent_t);
+        if (!entry)
+            goto out;
 
-                entry->d_ino  = trav->d_ino;
-                gf_itransform (this, trav->d_off, &entry->d_off,
-                                      conf->client_id);
-                entry->d_len  = trav->d_len;
-                entry->d_type = trav->d_type;
+        entry->d_ino = trav->d_ino;
+        gf_itransform(this, trav->d_off, &entry->d_off, conf->client_id);
+        entry->d_len = trav->d_len;
+        entry->d_type = trav->d_type;
 
-                strcpy (entry->d_name, trav->name);
+        gf_stat_to_iatt(&trav->stat, &entry->d_stat);
 
-		list_add_tail (&entry->list, &entries->list);
+        strcpy(entry->d_name, trav->name);
 
-                trav = trav->nextentry;
+        if (trav->dict.dict_val) {
+            entry->dict = dict_new();
+            if (!entry->dict)
+                goto out;
+
+            ret = dict_unserialize(trav->dict.dict_val, trav->dict.dict_len,
+                                   &entry->dict);
+            if (ret < 0) {
+                gf_smsg(THIS->name, GF_LOG_WARNING, EINVAL,
+                        PC_MSG_DICT_UNSERIALIZE_FAIL, "xattr", NULL);
+                goto out;
+            }
         }
 
-        ret = 0;
+        entry->inode = inode_find(itable, entry->d_stat.ia_gfid);
+        if (!entry->inode)
+            entry->inode = inode_new(itable);
+
+        list_add_tail(&entry->list, &entries->list);
+
+        trav = trav->nextentry;
+        entry = NULL;
+    }
+
+    ret = 0;
 out:
-        return ret;
+    if (entry)
+        gf_dirent_entry_free(entry);
+    return ret;
+}
+
+int
+clnt_readdirp_rsp_cleanup(gfs3_readdirp_rsp *rsp)
+{
+    gfs3_dirplist *prev = NULL;
+    gfs3_dirplist *trav = NULL;
+
+    trav = rsp->reply;
+    prev = trav;
+    while (trav) {
+        trav = trav->nextentry;
+        /* on client, the rpc lib allocates this */
+        free(prev->dict.dict_val);
+        free(prev->name);
+        free(prev);
+        prev = trav;
+    }
+
+    return 0;
 }
 
 int
-unserialize_rsp_direntp (xlator_t *this, fd_t *fd,
-                         struct gfs3_readdirp_rsp *rsp, gf_dirent_t *entries)
+unserialize_rsp_dirent_v2(xlator_t *this, struct gfx_readdir_rsp *rsp,
+                          gf_dirent_t *entries)
 {
-        struct gfs3_dirplist *trav      = NULL;
-        char                 *buf       = NULL;
-	gf_dirent_t          *entry     = NULL;
-        inode_table_t        *itable    = NULL;
-        int                   entry_len = 0;
-        int                   ret       = -1;
-        clnt_conf_t          *conf      = NULL;
+    struct gfx_dirlist *trav = NULL;
+    gf_dirent_t *entry = NULL;
+    int entry_len = 0;
+    int ret = -1;
+    clnt_conf_t *conf = NULL;
 
-        trav = rsp->reply;
+    conf = this->private;
 
-        if (fd)
-                itable = fd->inode->table;
+    trav = rsp->reply;
+    while (trav) {
+        entry_len = gf_dirent_size(trav->name);
+        entry = GF_CALLOC(1, entry_len, gf_common_mt_gf_dirent_t);
+        if (!entry)
+            goto out;
 
-        conf = this->private;
-        if (!conf)
-                goto out;
+        entry->d_ino = trav->d_ino;
+        gf_itransform(this, trav->d_off, &entry->d_off, conf->client_id);
+        entry->d_len = trav->d_len;
+        entry->d_type = trav->d_type;
 
-        while (trav) {
-                entry_len = gf_dirent_size (trav->name);
-                entry = GF_CALLOC (1, entry_len, gf_common_mt_gf_dirent_t);
-                if (!entry)
-                        goto out;
-
-                entry->d_ino  = trav->d_ino;
-                gf_itransform (this, trav->d_off, &entry->d_off,
-                                      conf->client_id);
-                entry->d_len  = trav->d_len;
-                entry->d_type = trav->d_type;
-
-                gf_stat_to_iatt (&trav->stat, &entry->d_stat);
-
-                strcpy (entry->d_name, trav->name);
-
-                if (trav->dict.dict_val) {
-                        /* Dictionary is sent along with response */
-                        buf = memdup (trav->dict.dict_val, trav->dict.dict_len);
-                        if (!buf)
-                                goto out;
-
-                        entry->dict = dict_new ();
-
-                        ret = dict_unserialize (buf, trav->dict.dict_len,
-                                                &entry->dict);
-                        if (ret < 0) {
-                                gf_log (THIS->name, GF_LOG_WARNING,
-                                        "failed to unserialize xattr dict");
-                                errno = EINVAL;
-                                goto out;
-                        }
-                        entry->dict->extra_free = buf;
-                        buf = NULL;
-                }
-
-                entry->inode = inode_find (itable, entry->d_stat.ia_gfid);
-                if (!entry->inode)
-                        entry->inode = inode_new (itable);
-
-		list_add_tail (&entry->list, &entries->list);
-
-                trav = trav->nextentry;
-        }
+        strcpy(entry->d_name, trav->name);
+
+        list_add_tail(&entry->list, &entries->list);
 
-        ret = 0;
+        trav = trav->nextentry;
+    }
+
+    ret = 0;
 out:
-        return ret;
+    return ret;
 }
 
 int
-clnt_readdirp_rsp_cleanup (gfs3_readdirp_rsp *rsp)
+unserialize_rsp_direntp_v2(xlator_t *this, fd_t *fd,
+                           struct gfx_readdirp_rsp *rsp, gf_dirent_t *entries)
 {
-        gfs3_dirplist *prev = NULL;
-        gfs3_dirplist *trav = NULL;
+    struct gfx_dirplist *trav = NULL;
+    gf_dirent_t *entry = NULL;
+    inode_table_t *itable = NULL;
+    int entry_len = 0;
+    int ret = -1;
+    clnt_conf_t *conf = NULL;
+
+    trav = rsp->reply;
+
+    if (fd)
+        itable = fd->inode->table;
+
+    conf = this->private;
+    if (!conf)
+        goto out;
+
+    while (trav) {
+        entry_len = gf_dirent_size(trav->name);
+        entry = GF_CALLOC(1, entry_len, gf_common_mt_gf_dirent_t);
+        if (!entry)
+            goto out;
+
+        entry->d_ino = trav->d_ino;
+        gf_itransform(this, trav->d_off, &entry->d_off, conf->client_id);
+        entry->d_len = trav->d_len;
+        entry->d_type = trav->d_type;
+
+        gfx_stat_to_iattx(&trav->stat, &entry->d_stat);
+
+        strcpy(entry->d_name, trav->name);
+
+        xdr_to_dict(&trav->dict, &entry->dict);
+
+        entry->inode = inode_find(itable, entry->d_stat.ia_gfid);
+        if (!entry->inode)
+            entry->inode = inode_new(itable);
 
-        trav = rsp->reply;
+        list_add_tail(&entry->list, &entries->list);
+
+        trav = trav->nextentry;
+    }
+
+    ret = 0;
+out:
+    return ret;
+}
+
+int
+clnt_readdirp_rsp_cleanup_v2(gfx_readdirp_rsp *rsp)
+{
+    gfx_dirplist *prev = NULL;
+    gfx_dirplist *trav = NULL;
+
+    trav = rsp->reply;
+    prev = trav;
+    while (trav) {
+        trav = trav->nextentry;
+        free(prev->name);
+        free(prev);
         prev = trav;
-        while (trav) {
-                trav = trav->nextentry;
-                /* on client, the rpc lib allocates this */
-                free (prev->dict.dict_val);
-                free (prev->name);
-                free (prev);
-                prev = trav;
-        }
+    }
 
-        return 0;
+    return 0;
 }
 
 int
-clnt_readdir_rsp_cleanup (gfs3_readdir_rsp *rsp)
+clnt_readdir_rsp_cleanup(gfs3_readdir_rsp *rsp)
 {
-        gfs3_dirlist *prev = NULL;
-        gfs3_dirlist *trav = NULL;
+    gfs3_dirlist *prev = NULL;
+    gfs3_dirlist *trav = NULL;
+
+    trav = rsp->reply;
+    prev = trav;
+    while (trav) {
+        trav = trav->nextentry;
+        /* on client, the rpc lib allocates this */
+        free(prev->name);
+        free(prev);
+        prev = trav;
+    }
 
-        trav = rsp->reply;
+    return 0;
+}
+
+int
+clnt_readdir_rsp_cleanup_v2(gfx_readdir_rsp *rsp)
+{
+    gfx_dirlist *prev = NULL;
+    gfx_dirlist *trav = NULL;
+
+    trav = rsp->reply;
+    prev = trav;
+    while (trav) {
+        trav = trav->nextentry;
+        /* on client, the rpc lib allocates this */
+        free(prev->name);
+        free(prev);
         prev = trav;
-        while (trav) {
-                trav = trav->nextentry;
-                /* on client, the rpc lib allocates this */
-                free (prev->name);
-                free (prev);
-                prev = trav;
-        }
+    }
 
-        return 0;
+    return 0;
 }
 
 int
-client_get_remote_fd (xlator_t *this, fd_t *fd, int flags, int64_t *remote_fd)
+client_get_remote_fd(xlator_t *this, fd_t *fd, int flags, int64_t *remote_fd)
 {
-        clnt_fd_ctx_t *fdctx    = NULL;
-        clnt_conf_t   *conf     = NULL;
-
-        GF_VALIDATE_OR_GOTO (this->name, fd, out);
-        GF_VALIDATE_OR_GOTO (this->name, remote_fd, out);
-
-        conf = this->private;
-        pthread_mutex_lock (&conf->lock);
-        {
-                fdctx = this_fd_get_ctx (fd, this);
-                if (!fdctx)
-                        *remote_fd = GF_ANON_FD_NO;
-                else if (__is_fd_reopen_in_progress (fdctx))
-                        *remote_fd = -1;
-                else
-                        *remote_fd = fdctx->remote_fd;
+    clnt_fd_ctx_t *fdctx = NULL;
+    clnt_conf_t *conf = NULL;
+    gf_boolean_t locks_held = _gf_false;
+
+    GF_VALIDATE_OR_GOTO(this->name, fd, out);
+    GF_VALIDATE_OR_GOTO(this->name, remote_fd, out);
+
+    conf = this->private;
+    pthread_spin_lock(&conf->fd_lock);
+    {
+        fdctx = this_fd_get_ctx(fd, this);
+        if (!fdctx) {
+            if (fd->anonymous) {
+                *remote_fd = GF_ANON_FD_NO;
+            } else {
+                *remote_fd = -1;
+                gf_msg_debug(this->name, EBADF, "not a valid fd for gfid: %s",
+                             uuid_utoa(fd->inode->gfid));
+            }
+        } else {
+            if (__is_fd_reopen_in_progress(fdctx))
+                *remote_fd = -1;
+            else
+                *remote_fd = fdctx->remote_fd;
+
+            locks_held = !list_empty(&fdctx->lock_list);
         }
-        pthread_mutex_unlock (&conf->lock);
+    }
+    pthread_spin_unlock(&conf->fd_lock);
 
-        if ((flags & FALLBACK_TO_ANON_FD) && (*remote_fd == -1))
-                *remote_fd = GF_ANON_FD_NO;
+    if ((flags & FALLBACK_TO_ANON_FD) && (*remote_fd == -1) && (!locks_held))
+        *remote_fd = GF_ANON_FD_NO;
 
-        return 0;
+    return 0;
 out:
-        return -1;
+    return -1;
 }
 
 gf_boolean_t
-client_is_reopen_needed (fd_t *fd, xlator_t *this, int64_t remote_fd)
+client_is_reopen_needed(fd_t *fd, xlator_t *this, int64_t remote_fd)
+{
+    clnt_conf_t *conf = NULL;
+    clnt_fd_ctx_t *fdctx = NULL;
+    gf_boolean_t res = _gf_false;
+
+    conf = this->private;
+    pthread_spin_lock(&conf->fd_lock);
+    {
+        fdctx = this_fd_get_ctx(fd, this);
+        if (fdctx && (fdctx->remote_fd == -1) && (remote_fd == GF_ANON_FD_NO))
+            res = _gf_true;
+    }
+    pthread_spin_unlock(&conf->fd_lock);
+
+    return res;
+}
+
+int
+client_fd_fop_prepare_local(call_frame_t *frame, fd_t *fd, int64_t remote_fd)
+{
+    xlator_t *this = NULL;
+    clnt_local_t *local = NULL;
+    int ret = 0;
+
+    if (!frame || !fd) {
+        ret = -EINVAL;
+        goto out;
+    }
+
+    this = frame->this;
+
+    frame->local = mem_get0(this->local_pool);
+    if (frame->local == NULL) {
+        ret = -ENOMEM;
+        goto out;
+    }
+
+    local = frame->local;
+    local->fd = fd_ref(fd);
+    local->attempt_reopen = client_is_reopen_needed(fd, this, remote_fd);
+
+    return 0;
+out:
+    return ret;
+}
+
+void
+clnt_getactivelk_rsp_cleanup(gfs3_getactivelk_rsp *rsp)
 {
-        clnt_fd_ctx_t   *fdctx = NULL;
+    gfs3_locklist *trav = NULL;
+    gfs3_locklist *next = NULL;
 
-        fdctx = this_fd_get_ctx (fd, this);
-        if (fdctx && (fdctx->remote_fd == -1) &&
-            (remote_fd == GF_ANON_FD_NO))
-                return _gf_true;
-        return _gf_false;
+    trav = rsp->reply;
+
+    while (trav) {
+        next = trav->nextentry;
+        free(trav->client_uid);
+        free(trav);
+        trav = next;
+    }
 }
 
+void
+clnt_getactivelk_rsp_cleanup_v2(gfx_getactivelk_rsp *rsp)
+{
+    gfs3_locklist *trav = NULL;
+    gfs3_locklist *next = NULL;
+
+    trav = rsp->reply;
+
+    while (trav) {
+        next = trav->nextentry;
+        free(trav->client_uid);
+        free(trav);
+        trav = next;
+    }
+}
 int
-client_fd_fop_prepare_local (call_frame_t *frame, fd_t *fd, int64_t remote_fd)
+clnt_unserialize_rsp_locklist(xlator_t *this, struct gfs3_getactivelk_rsp *rsp,
+                              lock_migration_info_t *lmi)
 {
-        xlator_t     *this  = NULL;
-        clnt_conf_t  *conf  = NULL;
-        clnt_local_t *local = NULL;
-        int          ret    = 0;
+    struct gfs3_locklist *trav = NULL;
+    lock_migration_info_t *temp = NULL;
+    int ret = -1;
+    clnt_conf_t *conf = NULL;
+
+    trav = rsp->reply;
+
+    conf = this->private;
+    if (!conf)
+        goto out;
+
+    while (trav) {
+        temp = GF_CALLOC(1, sizeof(*lmi), gf_common_mt_lock_mig);
+        if (temp == NULL) {
+            gf_smsg(this->name, GF_LOG_ERROR, 0, PC_MSG_NO_MEM, NULL);
+            goto out;
+        }
 
-        this = frame->this;
-        conf = this->private;
+        INIT_LIST_HEAD(&temp->list);
 
-        if (!frame || !fd) {
-                ret = -EINVAL;
-                goto out;
+        gf_proto_flock_to_flock(&trav->flock, &temp->flock);
+
+        temp->lk_flags = trav->lk_flags;
+
+        temp->client_uid = gf_strdup(trav->client_uid);
+
+        list_add_tail(&temp->list, &lmi->list);
+
+        trav = trav->nextentry;
+    }
+
+    ret = 0;
+out:
+    return ret;
+}
+int
+clnt_unserialize_rsp_locklist_v2(xlator_t *this,
+                                 struct gfx_getactivelk_rsp *rsp,
+                                 lock_migration_info_t *lmi)
+{
+    struct gfs3_locklist *trav = NULL;
+    lock_migration_info_t *temp = NULL;
+    int ret = -1;
+    clnt_conf_t *conf = NULL;
+
+    trav = rsp->reply;
+
+    conf = this->private;
+    if (!conf)
+        goto out;
+
+    while (trav) {
+        temp = GF_CALLOC(1, sizeof(*lmi), gf_common_mt_lock_mig);
+        if (temp == NULL) {
+            gf_smsg(this->name, GF_LOG_ERROR, 0, PC_MSG_NO_MEM, NULL);
+            goto out;
         }
 
-        frame->local = mem_get0 (this->local_pool);
-        if (frame->local == NULL) {
-                ret = -ENOMEM;
-                goto out;
+        INIT_LIST_HEAD(&temp->list);
+
+        gf_proto_flock_to_flock(&trav->flock, &temp->flock);
+
+        temp->lk_flags = trav->lk_flags;
+
+        temp->client_uid = gf_strdup(trav->client_uid);
+
+        list_add_tail(&temp->list, &lmi->list);
+
+        trav = trav->nextentry;
+    }
+
+    ret = 0;
+out:
+    return ret;
+}
+
+void
+clnt_setactivelk_req_cleanup(gfs3_setactivelk_req *req)
+{
+    gfs3_locklist *trav = NULL;
+    gfs3_locklist *next = NULL;
+
+    trav = req->request;
+
+    while (trav) {
+        next = trav->nextentry;
+        GF_FREE(trav->client_uid);
+        GF_FREE(trav);
+        trav = next;
+    }
+}
+
+void
+clnt_setactivelk_req_cleanup_v2(gfx_setactivelk_req *req)
+{
+    gfs3_locklist *trav = NULL;
+    gfs3_locklist *next = NULL;
+
+    trav = req->request;
+
+    while (trav) {
+        next = trav->nextentry;
+        GF_FREE(trav->client_uid);
+        GF_FREE(trav);
+        trav = next;
+    }
+}
+
+int
+serialize_req_locklist(lock_migration_info_t *locklist,
+                       gfs3_setactivelk_req *req)
+{
+    lock_migration_info_t *tmp = NULL;
+    gfs3_locklist *trav = NULL;
+    gfs3_locklist *prev = NULL;
+    int ret = -1;
+
+    GF_VALIDATE_OR_GOTO("server", locklist, out);
+    GF_VALIDATE_OR_GOTO("server", req, out);
+
+    list_for_each_entry(tmp, &locklist->list, list)
+    {
+        trav = GF_CALLOC(1, sizeof(*trav), gf_client_mt_clnt_lock_request_t);
+        if (!trav)
+            goto out;
+
+        switch (tmp->flock.l_type) {
+            case F_RDLCK:
+                tmp->flock.l_type = GF_LK_F_RDLCK;
+                break;
+            case F_WRLCK:
+                tmp->flock.l_type = GF_LK_F_WRLCK;
+                break;
+            case F_UNLCK:
+                tmp->flock.l_type = GF_LK_F_UNLCK;
+                break;
+
+            default:
+                gf_smsg(THIS->name, GF_LOG_ERROR, 0, PC_MSG_UNKNOWN_LOCK_TYPE,
+                        "type=%" PRId32, tmp->flock.l_type, NULL);
+                break;
+        }
+
+        gf_proto_flock_from_flock(&trav->flock, &tmp->flock);
+
+        trav->lk_flags = tmp->lk_flags;
+
+        trav->client_uid = gf_strdup(tmp->client_uid);
+        if (!trav->client_uid) {
+            gf_smsg(THIS->name, GF_LOG_ERROR, 0, PC_MSG_CLIENT_UID_ALLOC_FAILED,
+                    NULL);
+            ret = -1;
+            goto out;
         }
 
-        local = frame->local;
-        local->fd = fd_ref (fd);
-        local->attempt_reopen = client_is_reopen_needed (fd, this, remote_fd);
-        return 0;
+        if (prev)
+            prev->nextentry = trav;
+        else
+            req->request = trav;
+
+        prev = trav;
+        trav = NULL;
+    }
+
+    ret = 0;
 out:
-        return ret;
+    GF_FREE(trav);
+
+    return ret;
+}
+
+int
+serialize_req_locklist_v2(lock_migration_info_t *locklist,
+                          gfx_setactivelk_req *req)
+{
+    lock_migration_info_t *tmp = NULL;
+    gfs3_locklist *trav = NULL;
+    gfs3_locklist *prev = NULL;
+    int ret = -1;
+
+    GF_VALIDATE_OR_GOTO("server", locklist, out);
+    GF_VALIDATE_OR_GOTO("server", req, out);
+
+    list_for_each_entry(tmp, &locklist->list, list)
+    {
+        trav = GF_CALLOC(1, sizeof(*trav), gf_client_mt_clnt_lock_request_t);
+        if (!trav)
+            goto out;
+
+        switch (tmp->flock.l_type) {
+            case F_RDLCK:
+                tmp->flock.l_type = GF_LK_F_RDLCK;
+                break;
+            case F_WRLCK:
+                tmp->flock.l_type = GF_LK_F_WRLCK;
+                break;
+            case F_UNLCK:
+                tmp->flock.l_type = GF_LK_F_UNLCK;
+                break;
+
+            default:
+                gf_smsg(THIS->name, GF_LOG_ERROR, 0, PC_MSG_UNKNOWN_LOCK_TYPE,
+                        "type=%" PRId32, tmp->flock.l_type, NULL);
+                break;
+        }
+
+        gf_proto_flock_from_flock(&trav->flock, &tmp->flock);
+
+        trav->lk_flags = tmp->lk_flags;
+
+        trav->client_uid = gf_strdup(tmp->client_uid);
+        if (!trav->client_uid) {
+            gf_smsg(THIS->name, GF_LOG_ERROR, 0, PC_MSG_CLIENT_UID_ALLOC_FAILED,
+                    NULL);
+            ret = -1;
+            goto out;
+        }
+
+        if (prev)
+            prev->nextentry = trav;
+        else
+            req->request = trav;
+
+        prev = trav;
+        trav = NULL;
+    }
+
+    ret = 0;
+out:
+    GF_FREE(trav);
+
+    return ret;
+}
+
+extern int
+client3_3_releasedir_cbk(struct rpc_req *req, struct iovec *iov, int count,
+                         void *myframe);
+extern int
+client3_3_release_cbk(struct rpc_req *req, struct iovec *iov, int count,
+                      void *myframe);
+extern int
+client4_0_releasedir_cbk(struct rpc_req *req, struct iovec *iov, int count,
+                         void *myframe);
+extern int
+client4_0_release_cbk(struct rpc_req *req, struct iovec *iov, int count,
+                      void *myframe);
+
+static int
+send_release4_0_over_wire(xlator_t *this, clnt_fd_ctx_t *fdctx,
+                          call_frame_t *fr)
+{
+    clnt_conf_t *conf = NULL;
+    conf = (clnt_conf_t *)this->private;
+    if (fdctx->is_dir) {
+        gfx_releasedir_req req = {
+            {
+                0,
+            },
+        };
+        memcpy(req.gfid, fdctx->gfid, 16);
+        req.fd = fdctx->remote_fd;
+
+        gf_msg_trace(this->name, 0, "sending releasedir on fd");
+        (void)client_submit_request(
+            this, &req, fr, conf->fops, GFS3_OP_RELEASEDIR,
+            client4_0_releasedir_cbk, NULL, (xdrproc_t)xdr_gfx_releasedir_req);
+    } else {
+        gfx_release_req req = {
+            {
+                0,
+            },
+        };
+        memcpy(req.gfid, fdctx->gfid, 16);
+        req.fd = fdctx->remote_fd;
+        gf_msg_trace(this->name, 0, "sending release on fd");
+        (void)client_submit_request(this, &req, fr, conf->fops, GFS3_OP_RELEASE,
+                                    client4_0_release_cbk, NULL,
+                                    (xdrproc_t)xdr_gfx_release_req);
+    }
+
+    return 0;
+}
+
+static int
+send_release3_3_over_wire(xlator_t *this, clnt_fd_ctx_t *fdctx,
+                          call_frame_t *fr)
+{
+    clnt_conf_t *conf = NULL;
+    conf = (clnt_conf_t *)this->private;
+    if (fdctx->is_dir) {
+        gfs3_releasedir_req req = {
+            {
+                0,
+            },
+        };
+        memcpy(req.gfid, fdctx->gfid, 16);
+        req.fd = fdctx->remote_fd;
+        gf_msg_trace(this->name, 0, "sending releasedir on fd");
+        (void)client_submit_request(
+            this, &req, fr, conf->fops, GFS3_OP_RELEASEDIR,
+            client3_3_releasedir_cbk, NULL, (xdrproc_t)xdr_gfs3_releasedir_req);
+    } else {
+        gfs3_release_req req = {
+            {
+                0,
+            },
+        };
+        memcpy(req.gfid, fdctx->gfid, 16);
+        req.fd = fdctx->remote_fd;
+        gf_msg_trace(this->name, 0, "sending release on fd");
+        (void)client_submit_request(this, &req, fr, conf->fops, GFS3_OP_RELEASE,
+                                    client3_3_release_cbk, NULL,
+                                    (xdrproc_t)xdr_gfs3_release_req);
+    }
+
+    return 0;
+}
+
+int
+client_fdctx_destroy(xlator_t *this, clnt_fd_ctx_t *fdctx)
+{
+    clnt_conf_t *conf = NULL;
+    call_frame_t *fr = NULL;
+    int32_t ret = -1;
+    char parent_down = 0;
+    fd_lk_ctx_t *lk_ctx = NULL;
+
+    GF_VALIDATE_OR_GOTO("client", this, out);
+    GF_VALIDATE_OR_GOTO(this->name, fdctx, out);
+
+    conf = (clnt_conf_t *)this->private;
+
+    if (fdctx->remote_fd == -1) {
+        gf_msg_debug(this->name, 0, "not a valid fd");
+        goto out;
+    }
+
+    pthread_mutex_lock(&conf->lock);
+    {
+        parent_down = conf->parent_down;
+    }
+    pthread_mutex_unlock(&conf->lock);
+    lk_ctx = fdctx->lk_ctx;
+    fdctx->lk_ctx = NULL;
+
+    if (lk_ctx)
+        fd_lk_ctx_unref(lk_ctx);
+
+    if (!parent_down)
+        rpc_clnt_ref(conf->rpc);
+    else
+        goto out;
+
+    fr = create_frame(this, this->ctx->pool);
+    if (fr == NULL) {
+        goto out;
+    }
+
+    ret = 0;
+
+    if (conf->fops->progver == GLUSTER_FOP_VERSION)
+        send_release3_3_over_wire(this, fdctx, fr);
+    else
+        send_release4_0_over_wire(this, fdctx, fr);
+
+    rpc_clnt_unref(conf->rpc);
+out:
+    if (fdctx) {
+        fdctx->remote_fd = -1;
+        GF_FREE(fdctx);
+    }
+
+    return ret;
 }
diff --git a/xlators/protocol/client/src/client-lk.c b/xlators/protocol/client/src/client-lk.c
index b3c36a42021..795839734c5 100644
--- a/xlators/protocol/client/src/client-lk.c
+++ b/xlators/protocol/client/src/client-lk.c
@@ -8,566 +8,508 @@
   cases as published by the Free Software Foundation.
 */
 
-#include "common-utils.h"
-#include "xlator.h"
+#include <glusterfs/common-utils.h>
+#include <glusterfs/xlator.h>
 #include "client.h"
-#include "lkowner.h"
+#include <glusterfs/lkowner.h>
+#include "client-messages.h"
 
 static void
-__insert_and_merge (clnt_fd_ctx_t *fdctx, client_posix_lock_t *lock);
+__insert_and_merge(clnt_fd_ctx_t *fdctx, client_posix_lock_t *lock);
 
 static void
-__dump_client_lock (client_posix_lock_t *lock)
+__dump_client_lock(client_posix_lock_t *lock)
 {
-        xlator_t *this = NULL;
-
-        this = THIS;
-
-        gf_log (this->name, GF_LOG_INFO,
-                "{fd=%p}"
-                "{%s lk-owner:%s %"PRId64" - %"PRId64"}"
-                "{start=%"PRId64" end=%"PRId64"}",
-                lock->fd,
-                lock->fl_type == F_WRLCK ? "Write-Lock" : "Read-Lock",
-                lkowner_utoa (&lock->owner),
-                lock->user_flock.l_start,
-                lock->user_flock.l_len,
-                lock->fl_start,
-                lock->fl_end);
+    xlator_t *this = NULL;
+
+    this = THIS;
+
+    gf_smsg(
+        this->name, GF_LOG_INFO, 0, PC_MSG_CLIENT_LOCK_INFO, "fd=%p", lock->fd,
+        "fl_type=%s", lock->fl_type == F_WRLCK ? "Write-Lock" : "Read-Lock",
+        "lk-owner=%s", lkowner_utoa(&lock->owner), "l_start=%" PRId64,
+        lock->user_flock.l_start, "l_len=%" PRId64, lock->user_flock.l_len,
+        "start=%" PRId64, lock->fl_start, "end=%" PRId64, lock->fl_end, NULL);
 }
 
 static int
-dump_client_locks_fd (clnt_fd_ctx_t *fdctx)
+dump_client_locks_fd(clnt_fd_ctx_t *fdctx)
 {
-        client_posix_lock_t *lock = NULL;
-        int count = 0;
+    client_posix_lock_t *lock = NULL;
+    int count = 0;
 
-        pthread_mutex_lock (&fdctx->mutex);
-        {
-                list_for_each_entry (lock, &fdctx->lock_list, list) {
-                        __dump_client_lock (lock);
-                        count++;
-                }
-        }
-        pthread_mutex_unlock (&fdctx->mutex);
-
-        return count;
+    list_for_each_entry(lock, &fdctx->lock_list, list)
+    {
+        __dump_client_lock(lock);
+        count++;
+    }
 
+    return count;
 }
 
 int
-dump_client_locks (inode_t *inode)
+dump_client_locks(inode_t *inode)
 {
-        fd_t             *fd    = NULL;
-        clnt_conf_t    *conf  = NULL;
-        xlator_t         *this  = NULL;
-        clnt_fd_ctx_t  *fdctx = NULL;
+    fd_t *fd = NULL;
+    xlator_t *this = NULL;
+    clnt_fd_ctx_t *fdctx = NULL;
+    clnt_conf_t *conf = NULL;
 
-        int total_count = 0;
-        int locks_fd_count   = 0;
+    int total_count = 0;
+    int locks_fd_count = 0;
 
-        this = THIS;
-        conf = this->private;
+    this = THIS;
+    conf = this->private;
 
-        LOCK (&inode->lock);
+    LOCK(&inode->lock);
+    {
+        list_for_each_entry(fd, &inode->fd_list, inode_list)
         {
-                list_for_each_entry (fd, &inode->fd_list, inode_list) {
-                        locks_fd_count = 0;
-
-                        pthread_mutex_lock (&conf->lock);
-                        {
-                                fdctx = this_fd_get_ctx (fd, this);
-                        }
-                        pthread_mutex_unlock (&conf->lock);
+            locks_fd_count = 0;
 
-                        if (fdctx)
-                                locks_fd_count = dump_client_locks_fd (fdctx);
-
-                        total_count += locks_fd_count;
-                }
+            pthread_spin_lock(&conf->fd_lock);
+            fdctx = this_fd_get_ctx(fd, this);
+            if (fdctx)
+                locks_fd_count = dump_client_locks_fd(fdctx);
+            pthread_spin_unlock(&conf->fd_lock);
 
+            total_count += locks_fd_count;
         }
-        UNLOCK (&inode->lock);
-
-        return total_count;
+    }
+    UNLOCK(&inode->lock);
 
+    return total_count;
 }
 
 static off_t
-__get_lock_length (off_t start, off_t end)
+__get_lock_length(off_t start, off_t end)
 {
-        if (end == LLONG_MAX)
-                return 0;
-        else
-                return (end - start + 1);
+    if (end == LLONG_MAX)
+        return 0;
+    else
+        return (end - start + 1);
 }
 
 /* Add two locks */
 static client_posix_lock_t *
-add_locks (client_posix_lock_t *l1, client_posix_lock_t *l2)
+add_locks(client_posix_lock_t *l1, client_posix_lock_t *l2)
 {
-	client_posix_lock_t *sum = NULL;
+    client_posix_lock_t *sum = NULL;
 
-	sum = GF_CALLOC (1, sizeof (*sum), gf_client_mt_clnt_lock_t);
-	if (!sum)
-		return NULL;
+    sum = GF_CALLOC(1, sizeof(*sum), gf_client_mt_clnt_lock_t);
+    if (!sum)
+        return NULL;
 
-	sum->fl_start = min (l1->fl_start, l2->fl_start);
-	sum->fl_end   = max (l1->fl_end, l2->fl_end);
+    sum->fl_start = min(l1->fl_start, l2->fl_start);
+    sum->fl_end = max(l1->fl_end, l2->fl_end);
 
-        sum->user_flock.l_start = sum->fl_start;
-        sum->user_flock.l_len   = __get_lock_length (sum->fl_start,
-                                                     sum->fl_end);
+    sum->user_flock.l_start = sum->fl_start;
+    sum->user_flock.l_len = __get_lock_length(sum->fl_start, sum->fl_end);
 
-	return sum;
+    return sum;
 }
 
-
 /* Return true if the locks overlap, false otherwise */
 static int
-locks_overlap (client_posix_lock_t *l1, client_posix_lock_t *l2)
+locks_overlap(client_posix_lock_t *l1, client_posix_lock_t *l2)
 {
-	/*
-	   Note:
-	   FUSE always gives us absolute offsets, so no need to worry
-	   about SEEK_CUR or SEEK_END
-	*/
-
-	return ((l1->fl_end >= l2->fl_start) &&
-		(l2->fl_end >= l1->fl_start));
+    /*
+       Note:
+       FUSE always gives us absolute offsets, so no need to worry
+       about SEEK_CUR or SEEK_END
+    */
+
+    return ((l1->fl_end >= l2->fl_start) && (l2->fl_end >= l1->fl_start));
 }
 
 static void
-__delete_client_lock (client_posix_lock_t *lock)
+__delete_client_lock(client_posix_lock_t *lock)
 {
-	list_del_init (&lock->list);
+    list_del_init(&lock->list);
 }
 
 /* Destroy a posix_lock */
 static void
-__destroy_client_lock (client_posix_lock_t *lock)
+__destroy_client_lock(client_posix_lock_t *lock)
 {
-	GF_FREE (lock);
+    GF_FREE(lock);
 }
 
 /* Subtract two locks */
 struct _values {
-	client_posix_lock_t *locks[3];
+    client_posix_lock_t *locks[3];
 };
 
 /* {big} must always be contained inside {small} */
 static struct _values
-subtract_locks (client_posix_lock_t *big, client_posix_lock_t *small)
+subtract_locks(client_posix_lock_t *big, client_posix_lock_t *small)
 {
-	struct _values v = { .locks = {0, 0, 0} };
-
-	if ((big->fl_start == small->fl_start) &&
-	    (big->fl_end   == small->fl_end)) {
-		/* both edges coincide with big */
-		v.locks[0] = GF_CALLOC (1, sizeof (client_posix_lock_t),
-                                        gf_client_mt_clnt_lock_t );
-		GF_ASSERT (v.locks[0]);
-		memcpy (v.locks[0], big, sizeof (client_posix_lock_t));
-		v.locks[0]->fl_type = small->fl_type;
-	}
-	else if ((small->fl_start > big->fl_start) &&
-		 (small->fl_end   < big->fl_end)) {
-		/* both edges lie inside big */
-		v.locks[0] = GF_CALLOC (1, sizeof (client_posix_lock_t),
-                                        gf_client_mt_clnt_lock_t);
-		GF_ASSERT (v.locks[0]);
-		v.locks[1] = GF_CALLOC (1, sizeof (client_posix_lock_t),
-                                     gf_client_mt_clnt_lock_t);
-		GF_ASSERT (v.locks[1]);
-		v.locks[2] = GF_CALLOC (1, sizeof (client_posix_lock_t),
-                                        gf_client_mt_clnt_lock_t);
-		GF_ASSERT (v.locks[2]);
-
-		memcpy (v.locks[0], big, sizeof (client_posix_lock_t));
-		v.locks[0]->fl_end = small->fl_start - 1;
-                v.locks[0]->user_flock.l_len = __get_lock_length (v.locks[0]->fl_start,
-                                                                  v.locks[0]->fl_end);
-
-		memcpy (v.locks[1], small, sizeof (client_posix_lock_t));
-		memcpy (v.locks[2], big, sizeof (client_posix_lock_t));
-		v.locks[2]->fl_start = small->fl_end + 1;
-                v.locks[2]->user_flock.l_start = small->fl_end + 1;
-	}
-	/* one edge coincides with big */
-	else if (small->fl_start == big->fl_start) {
-		v.locks[0] = GF_CALLOC (1, sizeof (client_posix_lock_t),
-                                        gf_client_mt_clnt_lock_t);
-		GF_ASSERT (v.locks[0]);
-		v.locks[1] = GF_CALLOC (1, sizeof (client_posix_lock_t),
-                                        gf_client_mt_clnt_lock_t);
-		GF_ASSERT (v.locks[1]);
-
-		memcpy (v.locks[0], big, sizeof (client_posix_lock_t));
-		v.locks[0]->fl_start = small->fl_end + 1;
-                v.locks[0]->user_flock.l_start = small->fl_end + 1;
-
-		memcpy (v.locks[1], small, sizeof (client_posix_lock_t));
-	}
-	else if (small->fl_end   == big->fl_end) {
-		v.locks[0] = GF_CALLOC (1, sizeof (client_posix_lock_t),
-                                        gf_client_mt_clnt_lock_t);
-		GF_ASSERT (v.locks[0]);
-		v.locks[1] = GF_CALLOC (1, sizeof (client_posix_lock_t),
-                                        gf_client_mt_clnt_lock_t);
-		GF_ASSERT (v.locks[1]);
-
-		memcpy (v.locks[0], big, sizeof (client_posix_lock_t));
-		v.locks[0]->fl_end = small->fl_start - 1;
-                v.locks[0]->user_flock.l_len = __get_lock_length (v.locks[0]->fl_start,
-                                                                  v.locks[0]->fl_end);
-
-		memcpy (v.locks[1], small, sizeof (client_posix_lock_t));
-	}
-        else {
-                /* LOG-TODO : decide what more info is required here*/
-                gf_log ("client-protocol", GF_LOG_CRITICAL,
-                        "Unexpected case in subtract_locks. Please send "
-                        "a bug report to gluster-devel@gluster.org");
-        }
-
-        return v;
+    struct _values v = {.locks = {0, 0, 0}};
+
+    if ((big->fl_start == small->fl_start) && (big->fl_end == small->fl_end)) {
+        /* both edges coincide with big */
+        v.locks[0] = GF_MALLOC(sizeof(client_posix_lock_t),
+                               gf_client_mt_clnt_lock_t);
+        GF_ASSERT(v.locks[0]);
+        memcpy(v.locks[0], big, sizeof(client_posix_lock_t));
+        v.locks[0]->fl_type = small->fl_type;
+    } else if ((small->fl_start > big->fl_start) &&
+               (small->fl_end < big->fl_end)) {
+        /* both edges lie inside big */
+        v.locks[0] = GF_MALLOC(sizeof(client_posix_lock_t),
+                               gf_client_mt_clnt_lock_t);
+        GF_ASSERT(v.locks[0]);
+        memcpy(v.locks[0], big, sizeof(client_posix_lock_t));
+        v.locks[0]->fl_end = small->fl_start - 1;
+        v.locks[0]->user_flock.l_len = __get_lock_length(v.locks[0]->fl_start,
+                                                         v.locks[0]->fl_end);
+        v.locks[1] = GF_MALLOC(sizeof(client_posix_lock_t),
+                               gf_client_mt_clnt_lock_t);
+        GF_ASSERT(v.locks[1]);
+        memcpy(v.locks[1], small, sizeof(client_posix_lock_t));
+        v.locks[2] = GF_MALLOC(sizeof(client_posix_lock_t),
+                               gf_client_mt_clnt_lock_t);
+        GF_ASSERT(v.locks[2]);
+        memcpy(v.locks[2], big, sizeof(client_posix_lock_t));
+        v.locks[2]->fl_start = small->fl_end + 1;
+        v.locks[2]->user_flock.l_start = small->fl_end + 1;
+    }
+    /* one edge coincides with big */
+    else if (small->fl_start == big->fl_start) {
+        v.locks[0] = GF_MALLOC(sizeof(client_posix_lock_t),
+                               gf_client_mt_clnt_lock_t);
+        GF_ASSERT(v.locks[0]);
+        memcpy(v.locks[0], big, sizeof(client_posix_lock_t));
+        v.locks[0]->fl_start = small->fl_end + 1;
+        v.locks[0]->user_flock.l_start = small->fl_end + 1;
+        v.locks[1] = GF_MALLOC(sizeof(client_posix_lock_t),
+                               gf_client_mt_clnt_lock_t);
+        GF_ASSERT(v.locks[1]);
+        memcpy(v.locks[1], small, sizeof(client_posix_lock_t));
+    } else if (small->fl_end == big->fl_end) {
+        v.locks[0] = GF_MALLOC(sizeof(client_posix_lock_t),
+                               gf_client_mt_clnt_lock_t);
+        GF_ASSERT(v.locks[0]);
+        memcpy(v.locks[0], big, sizeof(client_posix_lock_t));
+        v.locks[0]->fl_end = small->fl_start - 1;
+        v.locks[0]->user_flock.l_len = __get_lock_length(v.locks[0]->fl_start,
+                                                         v.locks[0]->fl_end);
+
+        v.locks[1] = GF_MALLOC(sizeof(client_posix_lock_t),
+                               gf_client_mt_clnt_lock_t);
+        GF_ASSERT(v.locks[1]);
+        memcpy(v.locks[1], small, sizeof(client_posix_lock_t));
+    } else {
+        /* LOG-TODO : decide what more info is required here*/
+        gf_smsg("client-protocol", GF_LOG_CRITICAL, 0, PC_MSG_LOCK_ERROR, NULL);
+    }
+
+    return v;
 }
 
 static void
-__delete_unlck_locks (clnt_fd_ctx_t *fdctx)
+__delete_unlck_locks(clnt_fd_ctx_t *fdctx)
 {
-	client_posix_lock_t *l = NULL;
-	client_posix_lock_t *tmp = NULL;
-
-	list_for_each_entry_safe (l, tmp, &fdctx->lock_list, list) {
-		if (l->fl_type == F_UNLCK) {
-			__delete_client_lock (l);
-			__destroy_client_lock (l);
-		}
-	}
+    client_posix_lock_t *l = NULL;
+    client_posix_lock_t *tmp = NULL;
+
+    list_for_each_entry_safe(l, tmp, &fdctx->lock_list, list)
+    {
+        if (l->fl_type == F_UNLCK) {
+            __delete_client_lock(l);
+            __destroy_client_lock(l);
+        }
+    }
 }
 
 static void
-__insert_lock (clnt_fd_ctx_t *fdctx, client_posix_lock_t *lock)
+__insert_lock(clnt_fd_ctx_t *fdctx, client_posix_lock_t *lock)
 {
-	list_add_tail (&lock->list, &fdctx->lock_list);
+    list_add_tail(&lock->list, &fdctx->lock_list);
 
-	return;
+    return;
 }
 
 static void
-__insert_and_merge (clnt_fd_ctx_t *fdctx, client_posix_lock_t *lock)
+__insert_and_merge(clnt_fd_ctx_t *fdctx, client_posix_lock_t *lock)
 {
-        client_posix_lock_t  *conf = NULL;
-        client_posix_lock_t  *t = NULL;
-        client_posix_lock_t  *sum = NULL;
-        int            i = 0;
-        struct _values v = { .locks = {0, 0, 0} };
-
-        list_for_each_entry_safe (conf, t, &fdctx->lock_list, list) {
-                if (!locks_overlap (conf, lock))
-                        continue;
+    client_posix_lock_t *conf = NULL;
+    client_posix_lock_t *t = NULL;
+    client_posix_lock_t *sum = NULL;
+    int i = 0;
+    struct _values v = {.locks = {0, 0, 0}};
 
-                if (is_same_lkowner (&conf->owner, &lock->owner)) {
-                        if (conf->fl_type == lock->fl_type) {
-                                sum = add_locks (lock, conf);
+    list_for_each_entry_safe(conf, t, &fdctx->lock_list, list)
+    {
+        if (!locks_overlap(conf, lock))
+            continue;
 
-                                sum->fd = lock->fd;
+        if (is_same_lkowner(&conf->owner, &lock->owner)) {
+            if (conf->fl_type == lock->fl_type) {
+                sum = add_locks(lock, conf);
 
-                                __delete_client_lock (conf);
-                                __destroy_client_lock (conf);
+                sum->fd = lock->fd;
 
-                                __destroy_client_lock (lock);
-                                __insert_and_merge (fdctx, sum);
+                __delete_client_lock(conf);
+                __destroy_client_lock(conf);
 
-                                return;
-                        } else {
-                                sum = add_locks (lock, conf);
+                __destroy_client_lock(lock);
+                __insert_and_merge(fdctx, sum);
 
-                                sum->fd = conf->fd;
-                                sum->owner = conf->owner;
+                return;
+            } else {
+                sum = add_locks(lock, conf);
 
-                                v = subtract_locks (sum, lock);
+                sum->fd = conf->fd;
+                sum->owner = conf->owner;
 
-                                __delete_client_lock (conf);
-                                __destroy_client_lock (conf);
+                v = subtract_locks(sum, lock);
 
-                                __delete_client_lock (lock);
-                                __destroy_client_lock (lock);
+                __delete_client_lock(conf);
+                __destroy_client_lock(conf);
 
-                                __destroy_client_lock (sum);
+                __delete_client_lock(lock);
+                __destroy_client_lock(lock);
 
-                                for (i = 0; i < 3; i++) {
-                                        if (!v.locks[i])
-                                                continue;
+                __destroy_client_lock(sum);
 
-                                        INIT_LIST_HEAD (&v.locks[i]->list);
-                                        __insert_and_merge (fdctx,
-                                                            v.locks[i]);
-                                }
+                for (i = 0; i < 3; i++) {
+                    if (!v.locks[i])
+                        continue;
 
-                                __delete_unlck_locks (fdctx);
-                                return;
-                        }
+                    INIT_LIST_HEAD(&v.locks[i]->list);
+                    __insert_and_merge(fdctx, v.locks[i]);
                 }
 
-                if (lock->fl_type == F_UNLCK) {
-                        continue;
-                }
+                __delete_unlck_locks(fdctx);
+                return;
+            }
+        }
 
-                if ((conf->fl_type == F_RDLCK) && (lock->fl_type == F_RDLCK)) {
-                        __insert_lock (fdctx, lock);
-                        return;
-                }
+        if (lock->fl_type == F_UNLCK) {
+            continue;
         }
 
-        /* no conflicts, so just insert */
-        if (lock->fl_type != F_UNLCK) {
-                __insert_lock (fdctx, lock);
-        } else {
-                __destroy_client_lock (lock);
+        if ((conf->fl_type == F_RDLCK) && (lock->fl_type == F_RDLCK)) {
+            __insert_lock(fdctx, lock);
+            return;
         }
+    }
+
+    /* no conflicts, so just insert */
+    if (lock->fl_type != F_UNLCK) {
+        __insert_lock(fdctx, lock);
+    } else {
+        __destroy_client_lock(lock);
+    }
 }
 
 static void
-client_setlk (clnt_fd_ctx_t *fdctx, client_posix_lock_t *lock)
+client_setlk(clnt_fd_ctx_t *fdctx, client_posix_lock_t *lock)
 {
-        pthread_mutex_lock (&fdctx->mutex);
-        {
-                __insert_and_merge (fdctx, lock);
-        }
-        pthread_mutex_unlock (&fdctx->mutex);
-
-        return;
+    __insert_and_merge(fdctx, lock);
 }
 
 static void
-destroy_client_lock (client_posix_lock_t *lock)
+destroy_client_lock(client_posix_lock_t *lock)
 {
-        GF_FREE (lock);
+    GF_FREE(lock);
 }
 
 int32_t
-delete_granted_locks_owner (fd_t *fd, gf_lkowner_t *owner)
+delete_granted_locks_owner(fd_t *fd, gf_lkowner_t *owner)
 {
-        clnt_fd_ctx_t       *fdctx = NULL;
-        client_posix_lock_t *lock  = NULL;
-        client_posix_lock_t *tmp   = NULL;
-        xlator_t            *this  = NULL;
-
-        struct list_head delete_list;
-        int ret   = 0;
-        int count = 0;
-
-        INIT_LIST_HEAD (&delete_list);
-        this = THIS;
-        fdctx = this_fd_get_ctx (fd, this);
-        if (!fdctx) {
-                gf_log (this->name, GF_LOG_WARNING,
-                        "fdctx not valid");
-                ret = -1;
-                goto out;
-        }
-
-        pthread_mutex_lock (&fdctx->mutex);
-        {
-                list_for_each_entry_safe (lock, tmp, &fdctx->lock_list, list) {
-                        if (!is_same_lkowner (&lock->owner, owner)) {
-                                list_del_init (&lock->list);
-                                list_add_tail (&lock->list, &delete_list);
-                                count++;
-                        }
-                }
-        }
-        pthread_mutex_unlock (&fdctx->mutex);
-
-        list_for_each_entry_safe (lock, tmp, &delete_list, list) {
-                list_del_init (&lock->list);
-                destroy_client_lock (lock);
+    clnt_fd_ctx_t *fdctx = NULL;
+    client_posix_lock_t *lock = NULL;
+    client_posix_lock_t *tmp = NULL;
+    xlator_t *this = NULL;
+    clnt_conf_t *conf = NULL;
+
+    struct list_head delete_list;
+    int ret = 0;
+    int count = 0;
+
+    INIT_LIST_HEAD(&delete_list);
+    this = THIS;
+    conf = this->private;
+
+    pthread_spin_lock(&conf->fd_lock);
+
+    fdctx = this_fd_get_ctx(fd, this);
+    if (!fdctx) {
+        pthread_spin_unlock(&conf->fd_lock);
+
+        gf_smsg(this->name, GF_LOG_WARNING, EINVAL, PC_MSG_FD_CTX_INVALID,
+                NULL);
+        ret = -1;
+        goto out;
+    }
+
+    list_for_each_entry_safe(lock, tmp, &fdctx->lock_list, list)
+    {
+        if (is_same_lkowner(&lock->owner, owner)) {
+            list_del_init(&lock->list);
+            list_add_tail(&lock->list, &delete_list);
+            count++;
         }
+    }
 
-        /* FIXME: Need to actually print the locks instead of count */
-        gf_log (this->name, GF_LOG_TRACE,
-                "Number of locks cleared=%d", count);
-
-out:
-        return ret;
-}
-
-int32_t
-delete_granted_locks_fd (clnt_fd_ctx_t *fdctx)
-{
-        client_posix_lock_t *lock = NULL;
-        client_posix_lock_t *tmp = NULL;
-        xlator_t            *this = NULL;
-
-        struct list_head delete_list;
-        int ret   = 0;
-        int count = 0;
-
-        INIT_LIST_HEAD (&delete_list);
-        this = THIS;
+    pthread_spin_unlock(&conf->fd_lock);
 
-        pthread_mutex_lock (&fdctx->mutex);
+    if (!list_empty(&delete_list)) {
+        list_for_each_entry_safe(lock, tmp, &delete_list, list)
         {
-                list_splice_init (&fdctx->lock_list, &delete_list);
+            list_del_init(&lock->list);
+            destroy_client_lock(lock);
         }
-        pthread_mutex_unlock (&fdctx->mutex);
+    }
 
-        list_for_each_entry_safe (lock, tmp, &delete_list, list) {
-                list_del_init (&lock->list);
-                count++;
-                destroy_client_lock (lock);
-        }
-
-        /* FIXME: Need to actually print the locks instead of count */
-        gf_log (this->name, GF_LOG_TRACE,
-                "Number of locks cleared=%d", count);
+    /* FIXME: Need to actually print the locks instead of count */
+    gf_msg_trace(this->name, 0, "Number of locks cleared=%d", count);
 
-        return  ret;
+out:
+    return ret;
 }
 
 int32_t
-client_cmd_to_gf_cmd (int32_t cmd, int32_t *gf_cmd)
+client_cmd_to_gf_cmd(int32_t cmd, int32_t *gf_cmd)
 {
-        int ret = 0;
-
-        if (cmd == F_GETLK || cmd == F_GETLK64)
-                *gf_cmd = GF_LK_GETLK;
-        else if (cmd == F_SETLK || cmd == F_SETLK64)
-                *gf_cmd = GF_LK_SETLK;
-        else if (cmd == F_SETLKW || cmd == F_SETLKW64)
-                *gf_cmd = GF_LK_SETLKW;
-        else if (cmd == F_RESLK_LCK)
-                *gf_cmd = GF_LK_RESLK_LCK;
-        else if (cmd == F_RESLK_LCKW)
-                *gf_cmd = GF_LK_RESLK_LCKW;
-        else if (cmd == F_RESLK_UNLCK)
-                *gf_cmd = GF_LK_RESLK_UNLCK;
-        else if (cmd == F_GETLK_FD)
-                *gf_cmd = GF_LK_GETLK_FD;
-        else
-                ret = -1;
-
-        return ret;
-
+    int ret = 0;
+
+    if (cmd == F_GETLK || cmd == F_GETLK64)
+        *gf_cmd = GF_LK_GETLK;
+    else if (cmd == F_SETLK || cmd == F_SETLK64)
+        *gf_cmd = GF_LK_SETLK;
+    else if (cmd == F_SETLKW || cmd == F_SETLKW64)
+        *gf_cmd = GF_LK_SETLKW;
+    else if (cmd == F_RESLK_LCK)
+        *gf_cmd = GF_LK_RESLK_LCK;
+    else if (cmd == F_RESLK_LCKW)
+        *gf_cmd = GF_LK_RESLK_LCKW;
+    else if (cmd == F_RESLK_UNLCK)
+        *gf_cmd = GF_LK_RESLK_UNLCK;
+    else if (cmd == F_GETLK_FD)
+        *gf_cmd = GF_LK_GETLK_FD;
+    else
+        ret = -1;
+
+    return ret;
 }
 
 static client_posix_lock_t *
-new_client_lock (struct gf_flock *flock, gf_lkowner_t *owner,
-                 int32_t cmd, fd_t *fd)
+new_client_lock(struct gf_flock *flock, gf_lkowner_t *owner, int32_t cmd,
+                fd_t *fd)
 {
-        client_posix_lock_t *new_lock = NULL;
+    client_posix_lock_t *new_lock = NULL;
 
-        new_lock = GF_CALLOC (1, sizeof (*new_lock),
-                              gf_client_mt_clnt_lock_t);
-        if (!new_lock) {
-                goto out;
-        }
+    new_lock = GF_CALLOC(1, sizeof(*new_lock), gf_client_mt_clnt_lock_t);
+    if (!new_lock) {
+        goto out;
+    }
 
-        INIT_LIST_HEAD (&new_lock->list);
-        new_lock->fd = fd;
-        memcpy (&new_lock->user_flock, flock, sizeof (struct gf_flock));
+    INIT_LIST_HEAD(&new_lock->list);
+    new_lock->fd = fd;
+    memcpy(&new_lock->user_flock, flock, sizeof(struct gf_flock));
 
-        new_lock->fl_type  = flock->l_type;
-        new_lock->fl_start = flock->l_start;
+    new_lock->fl_type = flock->l_type;
+    new_lock->fl_start = flock->l_start;
 
-	if (flock->l_len == 0)
-		new_lock->fl_end = LLONG_MAX;
-	else
-		new_lock->fl_end = flock->l_start + flock->l_len - 1;
+    if (flock->l_len == 0)
+        new_lock->fl_end = LLONG_MAX;
+    else
+        new_lock->fl_end = flock->l_start + flock->l_len - 1;
 
-        new_lock->owner = *owner;
+    new_lock->owner = *owner;
 
-        new_lock->cmd = cmd; /* Not really useful */
+    new_lock->cmd = cmd; /* Not really useful */
 
 out:
-        return new_lock;
+    return new_lock;
 }
 
 void
-client_save_number_fds (clnt_conf_t *conf, int count)
+client_save_number_fds(clnt_conf_t *conf, int count)
 {
-        LOCK (&conf->rec_lock);
-        {
-                conf->reopen_fd_count = count;
-        }
-        UNLOCK (&conf->rec_lock);
+    LOCK(&conf->rec_lock);
+    {
+        conf->reopen_fd_count = count;
+    }
+    UNLOCK(&conf->rec_lock);
 }
 
 int
-client_add_lock_for_recovery (fd_t *fd, struct gf_flock *flock,
-                              gf_lkowner_t *owner, int32_t cmd)
+client_add_lock_for_recovery(fd_t *fd, struct gf_flock *flock,
+                             gf_lkowner_t *owner, int32_t cmd)
 {
-        clnt_fd_ctx_t       *fdctx = NULL;
-        xlator_t            *this  = NULL;
-        client_posix_lock_t *lock  = NULL;
-        clnt_conf_t         *conf  = NULL;
+    clnt_fd_ctx_t *fdctx = NULL;
+    xlator_t *this = NULL;
+    client_posix_lock_t *lock = NULL;
+    clnt_conf_t *conf = NULL;
 
-        int ret = 0;
+    int ret = 0;
 
-        this = THIS;
-        conf = this->private;
+    this = THIS;
+    conf = this->private;
 
-        pthread_mutex_lock (&conf->lock);
-        {
-                fdctx = this_fd_get_ctx (fd, this);
-        }
-        pthread_mutex_unlock (&conf->lock);
+    pthread_spin_lock(&conf->fd_lock);
 
-        if (!fdctx) {
-                gf_log (this->name, GF_LOG_WARNING,
-                        "failed to get fd context. sending EBADFD");
-                ret = -EBADFD;
-                goto out;
-        }
+    fdctx = this_fd_get_ctx(fd, this);
+    if (!fdctx) {
+        pthread_spin_unlock(&conf->fd_lock);
 
-        lock = new_client_lock (flock, owner, cmd, fd);
-        if (!lock) {
-                ret = -ENOMEM;
-                goto out;
-        }
+        gf_smsg(this->name, GF_LOG_WARNING, 0, PC_MSG_FD_GET_FAIL, NULL);
+        ret = -EBADFD;
+        goto out;
+    }
 
-        client_setlk (fdctx, lock);
+    lock = new_client_lock(flock, owner, cmd, fd);
+    if (!lock) {
+        pthread_spin_unlock(&conf->fd_lock);
 
-out:
-        return ret;
+        ret = -ENOMEM;
+        goto out;
+    }
+
+    client_setlk(fdctx, lock);
 
+    pthread_spin_unlock(&conf->fd_lock);
+
+out:
+    return ret;
 }
 
 int32_t
-client_dump_locks (char *name, inode_t *inode,
-                   dict_t *dict)
+client_dump_locks(char *name, inode_t *inode, dict_t *dict)
 {
-        int     ret = 0;
-        dict_t *new_dict = NULL;
-        char    dict_string[256];
+    int ret = 0;
+    dict_t *new_dict = NULL;
+    char dict_string[256];
 
-        GF_ASSERT (dict);
-        new_dict = dict;
+    GF_ASSERT(dict);
+    new_dict = dict;
 
-        ret = dump_client_locks (inode);
-        snprintf (dict_string, 256, "%d locks dumped in log file", ret);
+    ret = dump_client_locks(inode);
+    snprintf(dict_string, 256, "%d locks dumped in log file", ret);
 
-        ret = dict_set_dynstr(new_dict, CLIENT_DUMP_LOCKS, dict_string);
-        if (ret) {
-                gf_log (THIS->name, GF_LOG_WARNING,
-                        "could not set dict with %s", CLIENT_DUMP_LOCKS);
-                goto out;
-        }
+    ret = dict_set_dynstr(new_dict, CLIENT_DUMP_LOCKS, dict_string);
+    if (ret) {
+        gf_smsg(THIS->name, GF_LOG_WARNING, 0, PC_MSG_DICT_SET_FAIL, "lock=%s",
+                CLIENT_DUMP_LOCKS, NULL);
+        goto out;
+    }
 
 out:
 
-        return ret;
+    return ret;
 }
 
 int32_t
-is_client_dump_locks_cmd (char *name)
+is_client_dump_locks_cmd(char *name)
 {
-        int ret = 0;
+    int ret = 0;
 
-        if (strcmp (name, CLIENT_DUMP_LOCKS) == 0)
-                ret = 1;
+    if (strcmp(name, CLIENT_DUMP_LOCKS) == 0)
+        ret = 1;
 
-        return ret;
+    return ret;
 }
diff --git a/xlators/protocol/client/src/client-mem-types.h b/xlators/protocol/client/src/client-mem-types.h
index f6573da2ddd..f61fa0c1828 100644
--- a/xlators/protocol/client/src/client-mem-types.h
+++ b/xlators/protocol/client/src/client-mem-types.h
@@ -8,18 +8,19 @@
   cases as published by the Free Software Foundation.
 */
 
-
 #ifndef __CLIENT_MEM_TYPES_H__
 #define __CLIENT_MEM_TYPES_H__
 
-#include "mem-types.h"
+#include <glusterfs/mem-types.h>
 
 enum gf_client_mem_types_ {
-        gf_client_mt_clnt_conf_t = gf_common_mt_end + 1,
-        gf_client_mt_clnt_req_buf_t,
-        gf_client_mt_clnt_fdctx_t,
-        gf_client_mt_clnt_lock_t,
-        gf_client_mt_clnt_fd_lk_local_t,
-        gf_client_mt_end,
+    gf_client_mt_clnt_conf_t = gf_common_mt_end + 1,
+    gf_client_mt_clnt_req_buf_t,
+    gf_client_mt_clnt_fdctx_t,
+    gf_client_mt_clnt_lock_t,
+    gf_client_mt_clnt_fd_lk_local_t,
+    gf_client_mt_compound_req_t,
+    gf_client_mt_clnt_lock_request_t,
+    gf_client_mt_end,
 };
 #endif /* __CLIENT_MEM_TYPES_H__ */
diff --git a/xlators/protocol/client/src/client-messages.h b/xlators/protocol/client/src/client-messages.h
new file mode 100644
index 00000000000..25a851d80b9
--- /dev/null
+++ b/xlators/protocol/client/src/client-messages.h
@@ -0,0 +1,174 @@
+/*
+  Copyright (c) 2015 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
+*/
+
+#ifndef _PC_MESSAGES_H__
+#define _PC_MESSAGES_H__
+
+#include <glusterfs/glfs-message-id.h>
+
+/* To add new message IDs, append new identifiers at the end of the list.
+ *
+ * Never remove a message ID. If it's not used anymore, you can rename it or
+ * leave it as it is, but not delete it. This is to prevent reutilization of
+ * IDs by other messages.
+ *
+ * The component name must match one of the entries defined in
+ * glfs-message-id.h.
+ */
+
+GLFS_MSGID(
+    PC, PC_MSG_TIMER_EXPIRED, PC_MSG_DIR_OP_FAILED, PC_MSG_FILE_OP_FAILED,
+    PC_MSG_TIMER_REG, PC_MSG_GRACE_TIMER_CANCELLED, PC_MSG_DICT_SET_FAILED,
+    PC_MSG_DICT_GET_FAILED, PC_MSG_NO_MEMORY, PC_MSG_RPC_CBK_FAILED,
+    PC_MSG_FUNCTION_CALL_ERROR, PC_MSG_RPC_INITED_ALREADY, PC_MSG_RPC_INIT,
+    PC_MSG_RPC_DESTROY, PC_MSG_RPC_INVALID_CALL, PC_MSG_INVALID_ENTRY,
+    PC_MSG_HANDSHAKE_RETURN, PC_MSG_CHILD_UP_NOTIFY_FAILED,
+    PC_MSG_CLIENT_DISCONNECTED, PC_MSG_CHILD_DOWN_NOTIFY_FAILED,
+    PC_MSG_PARENT_UP, PC_MSG_PARENT_DOWN, PC_MSG_RPC_INIT_FAILED,
+    PC_MSG_RPC_NOTIFY_FAILED, PC_MSG_FD_DUPLICATE_TRY, PC_MSG_FD_SET_FAIL,
+    PC_MSG_DICT_UNSERIALIZE_FAIL, PC_MSG_FD_GET_FAIL, PC_MSG_FD_CTX_INVALID,
+    PC_MSG_FOP_SEND_FAILED, PC_MSG_XDR_DECODING_FAILED, PC_MSG_REMOTE_OP_FAILED,
+    PC_MSG_RPC_STATUS_ERROR, PC_MSG_VOL_FILE_NOT_FOUND, PC_MSG_SEND_REQ_FAIL,
+    PC_MSG_LOCK_VERSION_SERVER, PC_MSG_SET_LK_VERSION_ERROR,
+    PC_MSG_LOCK_REQ_FAIL, PC_MSG_CLIENT_REQ_FAIL, PC_MSG_LOCK_ERROR,
+    PC_MSG_LOCK_REACQUIRE, PC_MSG_CHILD_UP_NOTIFY, PC_MSG_CHILD_UP_NOTIFY_DELAY,
+    PC_MSG_VOL_SET_FAIL, PC_MSG_SETVOLUME_FAIL, PC_MSG_VOLFILE_NOTIFY_FAILED,
+    PC_MSG_REMOTE_VOL_CONNECTED, PC_MSG_LOCK_MISMATCH, PC_MSG_LOCK_MATCH,
+    PC_MSG_AUTH_FAILED, PC_MSG_AUTH_FAILED_NOTIFY_FAILED,
+    PC_MSG_CHILD_CONNECTING_EVENT, PC_MSG_CHILD_CONNECTING_NOTIFY_FAILED,
+    PC_MSG_PROCESS_UUID_SET_FAIL, PC_MSG_DICT_ERROR, PC_MSG_DICT_SERIALIZE_FAIL,
+    PC_MSG_PGM_NOT_FOUND, PC_MSG_VERSION_INFO, PC_MSG_PORT_NUM_ERROR,
+    PC_MSG_VERSION_ERROR, PC_MSG_DIR_OP_SUCCESS, PC_MSG_BAD_FD,
+    PC_MSG_CLIENT_LOCK_INFO, PC_MSG_CACHE_INVALIDATION_FAIL,
+    PC_MSG_CHILD_STATUS, PC_MSG_GFID_NULL, PC_MSG_RECALL_LEASE_FAIL,
+    PC_MSG_INODELK_CONTENTION_FAIL, PC_MSG_ENTRYLK_CONTENTION_FAIL,
+    PC_MSG_BIGGER_SIZE, PC_MSG_CLIENT_DUMP_LOCKS_FAILED, PC_MSG_UNKNOWN_CMD,
+    PC_MSG_REOPEN_FAILED, PC_MSG_FIND_KEY_FAILED, PC_MSG_VOL_ID_CHANGED,
+    PC_MSG_GETHOSTNAME_FAILED, PC_MSG_VOLFILE_KEY_SET_FAILED,
+    PC_MSG_VOLFILE_CHECKSUM_FAILED, PC_MSG_FRAME_NOT_FOUND,
+    PC_MSG_REMOTE_SUBVOL_SET_FAIL, PC_MSG_HANDSHAKE_PGM_NOT_FOUND,
+    PC_MSG_MERGE_IOBREF_FAILED, PC_MSG_ADD_IOBUF_FAILED,
+    PC_MSG_RELEASE_DIR_OP_FAILED, PC_MSG_REMOTE_HOST_SET_FAILED,
+    PC_MSG_REMOTE_PORT_SET_FAILED, PC_MSG_REMOTE_HOST_NOT_SET,
+    PC_MSG_NOREMOTE_HOST, PC_MSG_REMOTE_SUBVOL_NOT_GIVEN,
+    PC_MSG_FATAL_CLIENT_PROTOCOL, PC_MSG_VOL_DANGLING,
+    PC_MSG_CREATE_MEM_POOL_FAILED, PC_MSG_PVT_XLATOR_NULL, PC_MSG_XLATOR_NULL,
+    PC_MSG_LEASE_FOP_FAILED, PC_MSG_DICT_SET_FAIL, PC_MSG_NO_MEM,
+    PC_MSG_UNKNOWN_LOCK_TYPE, PC_MSG_CLIENT_UID_ALLOC_FAILED);
+
+#define PC_MSG_REMOTE_OP_FAILED_STR "remote operation failed."
+#define PC_MSG_XDR_DECODING_FAILED_STR "XDR decoding failed"
+#define PC_MSG_FOP_SEND_FAILED_STR "failed to send the fop"
+#define PC_MSG_BIGGER_SIZE_STR "read-size is bigger than iobuf isze"
+#define PC_MSG_CLIENT_DUMP_LOCKS_FAILED_STR "client dump locks failed"
+#define PC_MSG_UNKNOWN_CMD_STR "Unknown cmd"
+#define PC_MSG_CHILD_UP_NOTIFY_FAILED_STR "notify of CHILD_UP failed"
+#define PC_MSG_CHILD_STATUS_STR                                                \
+    "Defering sending CHILD_UP message as the client translators are not yet " \
+    "ready to serve"
+#define PC_MSG_CHILD_UP_NOTIFY_STR "last fd open'd - notifying CHILD_UP"
+#define PC_MSG_RPC_STATUS_ERROR_STR                                            \
+    "received RPC status error, returning ENOTCONN"
+#define PC_MSG_REOPEN_FAILED_STR "reopen failed"
+#define PC_MSG_DIR_OP_SUCCESS_STR "reopen dir succeeded"
+#define PC_MSG_DIR_OP_FAILED_STR "failed to send the re-opendir request"
+#define PC_MSG_CHILD_UP_NOTIFY_DELAY_STR                                       \
+    "fds open - Delaying child_up until they are re-opened"
+#define PC_MSG_VOL_SET_FAIL_STR "failed to set the volume"
+#define PC_MSG_DICT_UNSERIALIZE_FAIL_STR "failed to unserialize buffer to dict"
+#define PC_MSG_DICT_GET_FAILED_STR "failed to get from reply dict"
+#define PC_MSG_SETVOLUME_FAIL_STR "SETVOLUME on remote-host failed"
+#define PC_MSG_VOLFILE_NOTIFY_FAILED_STR "notify of VOLFILE_MODIFIED failed"
+#define PC_MSG_FIND_KEY_FAILED_STR "failed to find key in the options"
+#define PC_MSG_VOL_ID_CHANGED_STR                                              \
+    "volume-id changed, can't connect to server. Needs remount"
+#define PC_MSG_REMOTE_VOL_CONNECTED_STR "Connected, attached to remote volume"
+#define PC_MSG_AUTH_FAILED_STR "sending AUTH_FAILED event"
+#define PC_MSG_AUTH_FAILED_NOTIFY_FAILED_STR "notify of AUTH_FAILED failed"
+#define PC_MSG_CHILD_CONNECTING_EVENT_STR "sending CHILD_CONNECTING event"
+#define PC_MSG_CHILD_CONNECTING_NOTIFY_FAILED_STR                              \
+    "notify of CHILD_CONNECTING failed"
+#define PC_MSG_DICT_SET_FAILED_STR "failed to set in handshake msg"
+#define PC_MSG_GETHOSTNAME_FAILED_STR "gethostname: failed"
+#define PC_MSG_PROCESS_UUID_SET_FAIL_STR                                       \
+    "asprintf failed while setting process_uuid"
+#define PC_MSG_VOLFILE_KEY_SET_FAILED_STR "failed to set volfile-key"
+#define PC_MSG_VOLFILE_CHECKSUM_FAILED_STR "failed to set volfile-checksum"
+#define PC_MSG_DICT_SERIALIZE_FAIL_STR "failed to serialize dictionary"
+#define PC_MSG_PGM_NOT_FOUND_STR "xlator not found OR RPC program not found"
+#define PC_MSG_VERSION_INFO_STR "Using Program"
+#define PC_MSG_FRAME_NOT_FOUND_STR "frame not found with rpc request"
+#define PC_MSG_PORT_NUM_ERROR_STR                                              \
+    "failed to get the port number for remote subvolume. Please run gluster "  \
+    "volume status on server to see if brick process is running"
+#define PC_MSG_REMOTE_SUBVOL_SET_FAIL_STR "remote-subvolume not set in volfile"
+#define PC_MSG_VERSION_ERROR_STR "failed to get the version from server"
+#define PC_MSG_NO_VERSION_SUPPORT_STR "server doesn't support the version"
+#define PC_MSG_HANDSHAKE_PGM_NOT_FOUND_STR "handshake program not found"
+#define PC_MSG_MERGE_IOBREF_FAILED_STR                                         \
+    "cannot merge iobref passed from caller into new_iobref"
+#define PC_MSG_ADD_IOBUF_FAILED_STR "cannot add iobuf into iobref"
+#define PC_MSG_RELEASE_DIR_OP_FAILED_STR "release dir op failed"
+#define PC_MSG_FILE_OP_FAILED_STR "release fop failed"
+#define PC_MSG_REMOTE_HOST_SET_FAILED_STR "failed to set remote-host"
+#define PC_MSG_REMOTE_PORT_SET_FAILED_STR "failed to set remote-port"
+#define PC_MSG_RPC_INIT_STR "client rpc init command"
+#define PC_MSG_RPC_DESTROY_STR "client rpc destroy command"
+#define PC_MSG_HANDSHAKE_RETURN_STR "handshake msg returned"
+#define PC_MSG_CLIENT_DISCONNECTED_STR                                         \
+    "disconnected from client, process will keep trying to connect glusterd "  \
+    "until brick's port is available"
+#define PC_MSG_CHILD_DOWN_NOTIFY_FAILED_STR "CHILD_DOWN notify failed"
+#define PC_MSG_PARENT_UP_STR                                                   \
+    "parent translators are ready, attempting connect on transport"
+#define PC_MSG_PARENT_DOWN_STR                                                 \
+    "current graph is no longer active, destroying rpc_client"
+#define PC_MSG_REMOTE_HOST_NOT_SET_STR                                         \
+    "Remote host is not set. Assuming the volfile server as remote host"
+#define PC_MSG_NOREMOTE_HOST_STR "No remote host to connect"
+#define PC_MSG_REMOTE_SUBVOL_NOT_GIVEN_STR "option 'remote-subvolume' not given"
+#define PC_MSG_NO_MEMORY_STR "Memory accounting init failed"
+#define PC_MSG_RPC_INVALID_CALL_STR                                            \
+    "RPC destroy called on already destroyed connection"
+#define PC_MSG_RPC_INITED_ALREADY_STR "client rpc already init'ed"
+#define PC_MSG_RPC_INIT_FAILED_STR "failed to initialize RPC"
+#define PC_MSG_RPC_NOTIFY_FAILED_STR "failed to register notify"
+#define PC_MSG_RPC_CBK_FAILED_STR "failed to reister callback program"
+#define PC_MSG_FATAL_CLIENT_PROTOCOL_STR                                       \
+    "FATAL: client protocol, translator cannot have any subvolumes"
+#define PC_MSG_VOL_DANGLING_STR "Volume is dangling"
+#define PC_MSG_CREATE_MEM_POOL_FAILED_STR                                      \
+    "failed to create local_t's memory pool"
+#define PC_MSG_XLATOR_NULL_STR "xlator is NULL"
+#define PC_MSG_PVT_XLATOR_NULL_STR "private structure of the xlator is NULL"
+#define PC_MSG_LEASE_FOP_FAILED_STR "Lease fop failed"
+#define PC_MSG_LOCK_ERROR_STR                                                  \
+    "Unexpected case in subtract_locks. Please send a bug report to "          \
+    "gluster-devel@gluster.org"
+#define PC_MSG_FD_CTX_INVALID_STR "fdctx not valid"
+#define PC_MSG_FD_GET_FAIL_STR "failed to get fd context. sending EBADFD"
+#define PC_MSG_DICT_SET_FAIL_STR "could not set dict"
+#define PC_MSG_CLIENT_LOCK_INFO_STR "client lock info"
+#define PC_MSG_BAD_FD_STR "remote_fd is -1. EBADFD"
+#define PC_MSG_FUNCTION_CALL_ERROR_STR "this function should not be called"
+#define PC_MSG_RECALL_LEASE_FAIL_STR "XDR decode of recall lease failed"
+#define PC_MSG_CACHE_INVALIDATION_FAIL_STR                                     \
+    "XDR decode of cache_invalidation failed"
+#define PC_MSG_INODELK_CONTENTION_FAIL_STR                                     \
+    "XDR decode of inodelk contention failed"
+#define PC_MSG_ENTRYLK_CONTENTION_FAIL_STR                                     \
+    "XDR decode of entrylk contention failed"
+#define PC_MSG_FD_DUPLICATE_TRY_STR "trying duplicate remote fd set"
+#define PC_MSG_FD_SET_FAIL_STR "failed to set remote-fd"
+#define PC_MSG_NO_MEM_STR "No memory"
+#define PC_MSG_UNKNOWN_LOCK_TYPE_STR "Unknown lock type"
+#define PC_MSG_CLIENT_UID_ALLOC_FAILED_STR "client-uid could not be allocated"
+
+#endif /* !_PC_MESSAGES_H__ */
diff --git a/xlators/protocol/client/src/client-rpc-fops.c b/xlators/protocol/client/src/client-rpc-fops.c
index 55fd327da8e..f402121c15b 100644
--- a/xlators/protocol/client/src/client-rpc-fops.c
+++ b/xlators/protocol/client/src/client-rpc-fops.c
@@ -8,6258 +8,6072 @@
   cases as published by the Free Software Foundation.
 */
 
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
-
 #include "client.h"
 #include "rpc-common-xdr.h"
 #include "glusterfs3-xdr.h"
 #include "glusterfs3.h"
-#include "compat-errno.h"
+#include <glusterfs/compat-errno.h>
+#include "client-messages.h"
+#include <glusterfs/defaults.h>
+#include "client-common.h"
 
-int32_t client3_getspec (call_frame_t *frame, xlator_t *this, void *data);
+int32_t
+client3_getspec(call_frame_t *frame, xlator_t *this, void *data);
 rpc_clnt_prog_t clnt3_3_fop_prog;
 
-
 int
-client_submit_vec_request (xlator_t  *this, void *req, call_frame_t  *frame,
-                           rpc_clnt_prog_t *prog, int procnum,
-                           fop_cbk_fn_t cbkfn,
-                           struct iovec  *payload, int payloadcnt,
-                           struct iobref *iobref, xdrproc_t xdrproc)
+client_is_setlk(int32_t cmd)
 {
-        int             ret        = 0;
-        clnt_conf_t    *conf       = NULL;
-        struct iovec    iov        = {0, };
-        struct iobuf   *iobuf      = NULL;
-        int             count      = 0;
-        struct iobref  *new_iobref = NULL;
-        ssize_t         xdr_size   = 0;
-        struct rpc_req  rpcreq     = {0, };
-
-        conf = this->private;
-
-        if (req && xdrproc) {
-                xdr_size = xdr_sizeof (xdrproc, req);
-                iobuf = iobuf_get2 (this->ctx->iobuf_pool, xdr_size);
-                if (!iobuf) {
-                        goto unwind;
-                };
-
-                new_iobref = iobref_new ();
-                if (!new_iobref) {
-                        goto unwind;
-                }
-
-                if (iobref != NULL) {
-                        ret = iobref_merge (new_iobref, iobref);
-                        if (ret != 0) {
-                                gf_log (this->name, GF_LOG_WARNING,
-                                        "cannot merge iobref passed from caller "
-                                        "into new_iobref");
-                        }
-                }
-
-                ret = iobref_add (new_iobref, iobuf);
-                if (ret != 0) {
-                        gf_log (this->name, GF_LOG_WARNING,
-                                "cannot add iobuf into iobref");
-                        goto unwind;
-                }
-
-                iov.iov_base = iobuf->ptr;
-                iov.iov_len  = iobuf_size (iobuf);
-
-                /* Create the xdr payload */
-                ret = xdr_serialize_generic (iov, req, xdrproc);
-                if (ret == -1) {
-                        gf_log_callingfn ("", GF_LOG_WARNING,
-                                          "XDR function failed");
-                        goto unwind;
-                }
-
-                iov.iov_len = ret;
-                count = 1;
-        }
-
-        /* Send the msg */
-        ret = rpc_clnt_submit (conf->rpc, prog, procnum, cbkfn, &iov, count,
-                               payload, payloadcnt, new_iobref, frame, NULL, 0,
-                               NULL, 0, NULL);
-        if (ret < 0) {
-                gf_log (this->name, GF_LOG_DEBUG, "rpc_clnt_submit failed");
-        }
-
-        if (new_iobref)
-                iobref_unref (new_iobref);
-
-        if (iobuf)
-                iobuf_unref (iobuf);
-
-        return ret;
-
-unwind:
-        rpcreq.rpc_status = -1;
-        cbkfn (&rpcreq, NULL, 0, frame);
-
-        if (new_iobref)
-                iobref_unref (new_iobref);
+    if ((cmd == F_SETLK) || (cmd == F_SETLK64) || (cmd == F_SETLKW) ||
+        (cmd == F_SETLKW64)) {
+        return 1;
+    }
 
-        if (iobuf)
-                iobuf_unref (iobuf);
-
-        return ret;
+    return 0;
 }
 
 /* CBK */
-
 int
-client3_3_symlink_cbk (struct rpc_req *req, struct iovec *iov, int count,
-                       void *myframe)
+client3_3_symlink_cbk(struct rpc_req *req, struct iovec *iov, int count,
+                      void *myframe)
 {
-        call_frame_t     *frame      = NULL;
-        gfs3_symlink_rsp  rsp        = {0,};
-        struct iatt       stbuf      = {0,};
-        struct iatt       preparent  = {0,};
-        struct iatt       postparent = {0,};
-        int               ret        = 0;
-        clnt_local_t     *local      = NULL;
-        inode_t          *inode      = NULL;
-        xlator_t         *this       = NULL;
-        dict_t           *xdata      = NULL;
-
-        this = THIS;
-
-        frame = myframe;
-
-        local = frame->local;
-        inode = local->loc.inode;
-
-        if (-1 == req->rpc_status) {
-                rsp.op_ret   = -1;
-                rsp.op_errno = ENOTCONN;
-                goto out;
-        }
-        ret = xdr_to_generic (*iov, &rsp, (xdrproc_t)xdr_gfs3_symlink_rsp);
-        if (ret < 0) {
-                gf_log (this->name, GF_LOG_ERROR, "XDR decoding failed");
-                rsp.op_ret   = -1;
-                rsp.op_errno = EINVAL;
-                goto out;
-        }
-
-        if (-1 != rsp.op_ret) {
-                gf_stat_to_iatt (&rsp.stat, &stbuf);
-
-                gf_stat_to_iatt (&rsp.preparent, &preparent);
-                gf_stat_to_iatt (&rsp.postparent, &postparent);
-        }
+    call_frame_t *frame = NULL;
+    gfs3_symlink_rsp rsp = {
+        0,
+    };
+    struct iatt stbuf = {
+        0,
+    };
+    struct iatt preparent = {
+        0,
+    };
+    struct iatt postparent = {
+        0,
+    };
+    int ret = 0;
+    clnt_local_t *local = NULL;
+    inode_t *inode = NULL;
+    xlator_t *this = NULL;
+    dict_t *xdata = NULL;
+
+    this = THIS;
+
+    frame = myframe;
+
+    local = frame->local;
+    inode = local->loc.inode;
+
+    if (-1 == req->rpc_status) {
+        rsp.op_ret = -1;
+        rsp.op_errno = ENOTCONN;
+        goto out;
+    }
+    ret = xdr_to_generic(*iov, &rsp, (xdrproc_t)xdr_gfs3_symlink_rsp);
+    if (ret < 0) {
+        gf_smsg(this->name, GF_LOG_ERROR, EINVAL, PC_MSG_XDR_DECODING_FAILED,
+                NULL);
+        rsp.op_ret = -1;
+        rsp.op_errno = EINVAL;
+        goto out;
+    }
 
-        GF_PROTOCOL_DICT_UNSERIALIZE (this, xdata, (rsp.xdata.xdata_val),
-                                      (rsp.xdata.xdata_len), ret,
-                                      rsp.op_errno, out);
+    ret = client_post_symlink(this, &rsp, &stbuf, &preparent, &postparent,
+                              &xdata);
 
 out:
-        if (rsp.op_ret == -1) {
-                /* no need to print the gfid, because it will be null, since
-                 * symlink operation failed.
-                 */
-                gf_log (this->name, GF_LOG_WARNING,
-                        "remote operation failed: %s. Path: (%s to %s)",
-                        strerror (gf_error_to_errno (rsp.op_errno)),
-                        local->loc.path, local->loc2.path);
+    if (rsp.op_ret == -1) {
+        if (GF_IGNORE_IF_GSYNCD_SAFE_ERROR(frame, rsp.op_errno)) {
+            /* no need to print the gfid, because it will be null,
+             * since symlink operation failed.
+             */
+            gf_smsg(this->name, GF_LOG_WARNING, gf_error_to_errno(rsp.op_errno),
+                    PC_MSG_REMOTE_OP_FAILED, "loc1=%s", local->loc.path,
+                    "loc2=%s", local->loc2.path, NULL);
         }
+    }
 
-        CLIENT_STACK_UNWIND (symlink, frame, rsp.op_ret,
-                             gf_error_to_errno (rsp.op_errno), inode, &stbuf,
-                             &preparent, &postparent, xdata);
+    CLIENT_STACK_UNWIND(symlink, frame, rsp.op_ret,
+                        gf_error_to_errno(rsp.op_errno), inode, &stbuf,
+                        &preparent, &postparent, xdata);
 
-        free (rsp.xdata.xdata_val);
+    free(rsp.xdata.xdata_val);
 
-        if (xdata)
-                dict_unref (xdata);
+    if (xdata)
+        dict_unref(xdata);
 
-        return 0;
+    return 0;
 }
 
-
 int
-client3_3_mknod_cbk (struct rpc_req *req, struct iovec *iov, int count,
-                     void *myframe)
+client3_3_mknod_cbk(struct rpc_req *req, struct iovec *iov, int count,
+                    void *myframe)
 {
-        call_frame_t     *frame      = NULL;
-        gfs3_mknod_rsp    rsp        = {0,};
-        struct iatt       stbuf      = {0,};
-        struct iatt       preparent  = {0,};
-        struct iatt       postparent = {0,};
-        int               ret        = 0;
-        clnt_local_t     *local      = NULL;
-        inode_t          *inode      = NULL;
-        xlator_t         *this       = NULL;
-        dict_t           *xdata      = NULL;
-
-        this = THIS;
-
-        frame = myframe;
-
-        local = frame->local;
-
-        inode = local->loc.inode;
-
-        if (-1 == req->rpc_status) {
-                rsp.op_ret   = -1;
-                rsp.op_errno = ENOTCONN;
-                goto out;
-        }
-        ret = xdr_to_generic (*iov, &rsp, (xdrproc_t)xdr_gfs3_mknod_rsp);
-        if (ret < 0) {
-                gf_log (this->name, GF_LOG_ERROR, "XDR decoding failed");
-                rsp.op_ret   = -1;
-                rsp.op_errno = EINVAL;
-                goto out;
-        }
-
-        if (-1 != rsp.op_ret) {
-                gf_stat_to_iatt (&rsp.stat, &stbuf);
-
-                gf_stat_to_iatt (&rsp.preparent, &preparent);
-                gf_stat_to_iatt (&rsp.postparent, &postparent);
-        }
+    call_frame_t *frame = NULL;
+    gfs3_mknod_rsp rsp = {
+        0,
+    };
+    struct iatt stbuf = {
+        0,
+    };
+    struct iatt preparent = {
+        0,
+    };
+    struct iatt postparent = {
+        0,
+    };
+    int ret = 0;
+    clnt_local_t *local = NULL;
+    inode_t *inode = NULL;
+    xlator_t *this = NULL;
+    dict_t *xdata = NULL;
+
+    this = THIS;
+
+    frame = myframe;
+
+    local = frame->local;
+
+    inode = local->loc.inode;
+
+    if (-1 == req->rpc_status) {
+        rsp.op_ret = -1;
+        rsp.op_errno = ENOTCONN;
+        goto out;
+    }
+    ret = xdr_to_generic(*iov, &rsp, (xdrproc_t)xdr_gfs3_mknod_rsp);
+    if (ret < 0) {
+        gf_smsg(this->name, GF_LOG_ERROR, EINVAL, PC_MSG_XDR_DECODING_FAILED,
+                NULL);
+        rsp.op_ret = -1;
+        rsp.op_errno = EINVAL;
+        goto out;
+    }
 
-        GF_PROTOCOL_DICT_UNSERIALIZE (this, xdata, (rsp.xdata.xdata_val),
-                                      (rsp.xdata.xdata_len), ret,
-                                      rsp.op_errno, out);
+    ret = client_post_mknod(this, &rsp, &stbuf, &preparent, &postparent,
+                            &xdata);
 
 out:
-        if (rsp.op_ret == -1) {
-                gf_log (this->name, GF_LOG_WARNING,
-                        "remote operation failed: %s. Path: %s",
-                        strerror (gf_error_to_errno (rsp.op_errno)),
-                        local->loc.path);
-        }
+    if (rsp.op_ret == -1 &&
+        GF_IGNORE_IF_GSYNCD_SAFE_ERROR(frame, rsp.op_errno)) {
+        gf_smsg(this->name,
+                fop_log_level(GF_FOP_MKNOD, gf_error_to_errno(rsp.op_errno)),
+                gf_error_to_errno(rsp.op_errno), PC_MSG_REMOTE_OP_FAILED,
+                "path=%s", local->loc.path, NULL);
+    }
 
-        CLIENT_STACK_UNWIND (mknod, frame, rsp.op_ret,
-                             gf_error_to_errno (rsp.op_errno), inode,
-                             &stbuf, &preparent, &postparent, xdata);
+    CLIENT_STACK_UNWIND(mknod, frame, rsp.op_ret,
+                        gf_error_to_errno(rsp.op_errno), inode, &stbuf,
+                        &preparent, &postparent, xdata);
 
-        free (rsp.xdata.xdata_val);
+    free(rsp.xdata.xdata_val);
 
-        if (xdata)
-                dict_unref (xdata);
+    if (xdata)
+        dict_unref(xdata);
 
-        return 0;
+    return 0;
 }
 
 int
-client3_3_mkdir_cbk (struct rpc_req *req, struct iovec *iov, int count,
-                     void *myframe)
+client3_3_mkdir_cbk(struct rpc_req *req, struct iovec *iov, int count,
+                    void *myframe)
 {
-        call_frame_t     *frame      = NULL;
-        gfs3_mkdir_rsp    rsp        = {0,};
-        struct iatt       stbuf      = {0,};
-        struct iatt       preparent  = {0,};
-        struct iatt       postparent = {0,};
-        int               ret        = 0;
-        clnt_local_t     *local      = NULL;
-        inode_t          *inode      = NULL;
-        xlator_t         *this       = NULL;
-        dict_t           *xdata      = NULL;
-
-        this = THIS;
-
-        frame = myframe;
-
-        local = frame->local;
-        inode = local->loc.inode;
-
-        if (-1 == req->rpc_status) {
-                rsp.op_ret   = -1;
-                rsp.op_errno = ENOTCONN;
-                goto out;
-        }
-        ret = xdr_to_generic (*iov, &rsp, (xdrproc_t)xdr_gfs3_mkdir_rsp);
-        if (ret < 0) {
-                gf_log (this->name, GF_LOG_ERROR, "XDR decoding failed");
-                rsp.op_ret   = -1;
-                rsp.op_errno = EINVAL;
-                goto out;
-        }
-
-        if (-1 != rsp.op_ret) {
-                gf_stat_to_iatt (&rsp.stat, &stbuf);
-
-                gf_stat_to_iatt (&rsp.preparent, &preparent);
-                gf_stat_to_iatt (&rsp.postparent, &postparent);
-        }
+    call_frame_t *frame = NULL;
+    gfs3_mkdir_rsp rsp = {
+        0,
+    };
+    struct iatt stbuf = {
+        0,
+    };
+    struct iatt preparent = {
+        0,
+    };
+    struct iatt postparent = {
+        0,
+    };
+    int ret = 0;
+    clnt_local_t *local = NULL;
+    inode_t *inode = NULL;
+    xlator_t *this = NULL;
+    dict_t *xdata = NULL;
+
+    this = THIS;
+
+    frame = myframe;
+
+    local = frame->local;
+    inode = local->loc.inode;
+
+    if (-1 == req->rpc_status) {
+        rsp.op_ret = -1;
+        rsp.op_errno = ENOTCONN;
+        goto out;
+    }
+    ret = xdr_to_generic(*iov, &rsp, (xdrproc_t)xdr_gfs3_mkdir_rsp);
+    if (ret < 0) {
+        gf_smsg(this->name, GF_LOG_ERROR, EINVAL, PC_MSG_XDR_DECODING_FAILED,
+                NULL);
+        rsp.op_ret = -1;
+        rsp.op_errno = EINVAL;
+        goto out;
+    }
 
-        GF_PROTOCOL_DICT_UNSERIALIZE (this, xdata, (rsp.xdata.xdata_val),
-                                      (rsp.xdata.xdata_len), ret,
-                                      rsp.op_errno, out);
+    ret = client_post_mkdir(this, &rsp, &stbuf, &preparent, &postparent,
+                            &xdata);
 
 out:
-        if (rsp.op_ret == -1) {
-                gf_log (this->name, GF_LOG_WARNING,
-                        "remote operation failed: %s. Path: %s",
-                        strerror (gf_error_to_errno (rsp.op_errno)),
-                        local->loc.path);
-        }
+    if (rsp.op_ret == -1 &&
+        GF_IGNORE_IF_GSYNCD_SAFE_ERROR(frame, rsp.op_errno)) {
+        gf_smsg(this->name,
+                fop_log_level(GF_FOP_MKDIR, gf_error_to_errno(rsp.op_errno)),
+                gf_error_to_errno(rsp.op_errno), PC_MSG_REMOTE_OP_FAILED,
+                "Path=%s", local->loc.path, NULL);
+    }
 
-        CLIENT_STACK_UNWIND (mkdir, frame, rsp.op_ret,
-                             gf_error_to_errno (rsp.op_errno), inode,
-                             &stbuf, &preparent, &postparent, xdata);
+    CLIENT_STACK_UNWIND(mkdir, frame, rsp.op_ret,
+                        gf_error_to_errno(rsp.op_errno), inode, &stbuf,
+                        &preparent, &postparent, xdata);
 
-        free (rsp.xdata.xdata_val);
+    free(rsp.xdata.xdata_val);
 
-        if (xdata)
-                dict_unref (xdata);
+    if (xdata)
+        dict_unref(xdata);
 
-        return 0;
+    return 0;
 }
 
 int
-_copy_gfid_from_inode_holders (uuid_t gfid, loc_t *loc, fd_t *fd)
+_copy_gfid_from_inode_holders(uuid_t gfid, loc_t *loc, fd_t *fd)
 {
-        int     ret = 0;
-
-        if (fd && fd->inode && !gf_uuid_is_null (fd->inode->gfid)) {
-                gf_uuid_copy (gfid, fd->inode->gfid);
-                goto out;
-        }
-
-        if (!loc) {
-                GF_ASSERT (0);
-                ret = -1;
-                goto out;
-        }
-
-        if (loc->inode && !gf_uuid_is_null (loc->inode->gfid)) {
-                gf_uuid_copy (gfid, loc->inode->gfid);
-        } else if (!gf_uuid_is_null (loc->gfid)) {
-                gf_uuid_copy (gfid, loc->gfid);
-        } else {
-                GF_ASSERT (0);
-                ret = -1;
-        }
+    int ret = 0;
+
+    if (fd && fd->inode && !gf_uuid_is_null(fd->inode->gfid)) {
+        gf_uuid_copy(gfid, fd->inode->gfid);
+        goto out;
+    }
+
+    if (!loc) {
+        GF_ASSERT(0);
+        ret = -1;
+        goto out;
+    }
+
+    if (loc->inode && !gf_uuid_is_null(loc->inode->gfid)) {
+        gf_uuid_copy(gfid, loc->inode->gfid);
+    } else if (!gf_uuid_is_null(loc->gfid)) {
+        gf_uuid_copy(gfid, loc->gfid);
+    } else {
+        GF_ASSERT(0);
+        ret = -1;
+    }
 out:
-        return ret;
+    return ret;
 }
 
 int
-client_add_fd_to_saved_fds (xlator_t *this, fd_t *fd, loc_t *loc, int32_t flags,
-                            int64_t remote_fd, int is_dir)
+client_add_fd_to_saved_fds(xlator_t *this, fd_t *fd, loc_t *loc, int32_t flags,
+                           int64_t remote_fd, int is_dir)
 {
-        int             ret = 0;
-        uuid_t          gfid = {0};
-        clnt_conf_t     *conf       = NULL;
-        clnt_fd_ctx_t   *fdctx      = NULL;
-
-        conf  = this->private;
-        ret = _copy_gfid_from_inode_holders (gfid, loc, fd);
-        if (ret) {
-                ret = -EINVAL;
-                goto out;
-        }
-
-        fdctx = GF_CALLOC (1, sizeof (*fdctx),
-                           gf_client_mt_clnt_fdctx_t);
-        if (!fdctx) {
-                ret = -ENOMEM;
-                goto out;
-        }
-
-        gf_uuid_copy (fdctx->gfid, gfid);
-        fdctx->is_dir        = is_dir;
-        fdctx->remote_fd     = remote_fd;
-        fdctx->flags         = flags;
-        fdctx->lk_ctx        = fd_lk_ctx_ref (fd->lk_ctx);
-        fdctx->lk_heal_state = GF_LK_HEAL_DONE;
-        fdctx->reopen_done   = client_default_reopen_done;
-
-        INIT_LIST_HEAD (&fdctx->sfd_pos);
-        INIT_LIST_HEAD (&fdctx->lock_list);
-
-        this_fd_set_ctx (fd, this, loc, fdctx);
-
-        pthread_mutex_lock (&conf->lock);
-        {
-                list_add_tail (&fdctx->sfd_pos, &conf->saved_fds);
-        }
-        pthread_mutex_unlock (&conf->lock);
+    int ret = 0;
+    uuid_t gfid = {0};
+    clnt_conf_t *conf = NULL;
+    clnt_fd_ctx_t *fdctx = NULL;
+
+    conf = this->private;
+    ret = _copy_gfid_from_inode_holders(gfid, loc, fd);
+    if (ret) {
+        ret = -EINVAL;
+        goto out;
+    }
+
+    fdctx = GF_CALLOC(1, sizeof(*fdctx), gf_client_mt_clnt_fdctx_t);
+    if (!fdctx) {
+        ret = -ENOMEM;
+        goto out;
+    }
+
+    gf_uuid_copy(fdctx->gfid, gfid);
+    fdctx->is_dir = is_dir;
+    fdctx->remote_fd = remote_fd;
+    fdctx->flags = flags;
+    fdctx->lk_ctx = fd_lk_ctx_ref(fd->lk_ctx);
+    fdctx->reopen_done = client_default_reopen_done;
+
+    INIT_LIST_HEAD(&fdctx->sfd_pos);
+    INIT_LIST_HEAD(&fdctx->lock_list);
+
+    pthread_spin_lock(&conf->fd_lock);
+    {
+        this_fd_set_ctx(fd, this, loc, fdctx);
+
+        list_add_tail(&fdctx->sfd_pos, &conf->saved_fds);
+    }
+    pthread_spin_unlock(&conf->fd_lock);
 out:
-        return ret;
+    return ret;
 }
 
 int
-client3_3_open_cbk (struct rpc_req *req, struct iovec *iov, int count,
-                    void *myframe)
+client3_3_open_cbk(struct rpc_req *req, struct iovec *iov, int count,
+                   void *myframe)
 {
-        clnt_local_t  *local = NULL;
-        clnt_conf_t   *conf  = NULL;
-        call_frame_t  *frame = NULL;
-        fd_t          *fd    = NULL;
-        int            ret   = 0;
-        gfs3_open_rsp  rsp   = {0,};
-        xlator_t *this       = NULL;
-        dict_t  *xdata       = NULL;
+    clnt_local_t *local = NULL;
+    call_frame_t *frame = NULL;
+    fd_t *fd = NULL;
+    int ret = 0;
+    gfs3_open_rsp rsp = {
+        0,
+    };
+    xlator_t *this = NULL;
+    dict_t *xdata = NULL;
 
+    this = THIS;
 
-        this = THIS;
+    frame = myframe;
+    local = frame->local;
 
-        frame = myframe;
-        local = frame->local;
+    fd = local->fd;
 
-        conf  = frame->this->private;
-        fd    = local->fd;
-
-        if (-1 == req->rpc_status) {
-                rsp.op_ret   = -1;
-                rsp.op_errno = ENOTCONN;
-                goto out;
-        }
-        ret = xdr_to_generic (*iov, &rsp, (xdrproc_t)xdr_gfs3_open_rsp);
-        if (ret < 0) {
-                gf_log (this->name, GF_LOG_ERROR, "XDR decoding failed");
-                rsp.op_ret   = -1;
-                rsp.op_errno = EINVAL;
-                goto out;
-        }
+    if (-1 == req->rpc_status) {
+        rsp.op_ret = -1;
+        rsp.op_errno = ENOTCONN;
+        goto out;
+    }
+    ret = xdr_to_generic(*iov, &rsp, (xdrproc_t)xdr_gfs3_open_rsp);
+    if (ret < 0) {
+        gf_smsg(this->name, GF_LOG_ERROR, EINVAL, PC_MSG_XDR_DECODING_FAILED,
+                NULL);
+        rsp.op_ret = -1;
+        rsp.op_errno = EINVAL;
+        goto out;
+    }
 
-        if (-1 != rsp.op_ret) {
-                ret = client_add_fd_to_saved_fds (frame->this, fd, &local->loc,
-                                                  local->flags, rsp.fd, 0);
-                if (ret) {
-                        rsp.op_ret = -1;
-                        rsp.op_errno = -ret;
-                        goto out;
-                }
+    if (-1 != rsp.op_ret) {
+        ret = client_add_fd_to_saved_fds(frame->this, fd, &local->loc,
+                                         local->flags, rsp.fd, 0);
+        if (ret) {
+            rsp.op_ret = -1;
+            rsp.op_errno = -ret;
+            goto out;
         }
+    }
 
-        GF_PROTOCOL_DICT_UNSERIALIZE (this, xdata, (rsp.xdata.xdata_val),
-                                      (rsp.xdata.xdata_len), ret,
-                                      rsp.op_errno, out);
-
+    ret = client_post_open(this, &rsp, &xdata);
 out:
-        if (rsp.op_ret == -1) {
-                gf_log (this->name, fop_log_level (GF_FOP_OPEN,
-                                        gf_error_to_errno (rsp.op_errno)),
-                        "remote operation failed: %s. Path: %s (%s)",
-                        strerror (gf_error_to_errno (rsp.op_errno)),
-                        local->loc.path, loc_gfid_utoa (&local->loc));
-        }
+    if (rsp.op_ret == -1) {
+        gf_smsg(this->name,
+                fop_log_level(GF_FOP_OPEN, gf_error_to_errno(rsp.op_errno)),
+                gf_error_to_errno(rsp.op_errno), PC_MSG_REMOTE_OP_FAILED,
+                "Path=%s", local->loc.path, "gfid=%s",
+                loc_gfid_utoa(&local->loc), NULL);
+    }
 
-        CLIENT_STACK_UNWIND (open, frame, rsp.op_ret,
-                             gf_error_to_errno (rsp.op_errno), fd, xdata);
+    CLIENT_STACK_UNWIND(open, frame, rsp.op_ret,
+                        gf_error_to_errno(rsp.op_errno), fd, xdata);
 
-        free (rsp.xdata.xdata_val);
+    free(rsp.xdata.xdata_val);
 
-        if (xdata)
-                dict_unref (xdata);
+    if (xdata)
+        dict_unref(xdata);
 
-        return 0;
+    return 0;
 }
 
-
 int
-client3_3_stat_cbk (struct rpc_req *req, struct iovec *iov, int count,
-                    void *myframe)
+client3_3_stat_cbk(struct rpc_req *req, struct iovec *iov, int count,
+                   void *myframe)
 {
-        gfs3_stat_rsp rsp = {0,};
-        call_frame_t   *frame = NULL;
-        struct iatt  iatt = {0,};
-        int ret = 0;
-        xlator_t *this       = NULL;
-        dict_t  *xdata       = NULL;
-
-
-        this = THIS;
-
-        frame = myframe;
-
-        if (-1 == req->rpc_status) {
-                rsp.op_ret   = -1;
-                rsp.op_errno = ENOTCONN;
-                goto out;
-        }
-        ret = xdr_to_generic (*iov, &rsp, (xdrproc_t)xdr_gfs3_stat_rsp);
-        if (ret < 0) {
-                gf_log (this->name, GF_LOG_ERROR, "XDR decoding failed");
-                rsp.op_ret   = -1;
-                rsp.op_errno = EINVAL;
-                goto out;
-        }
-
-        if (-1 != rsp.op_ret) {
-                gf_stat_to_iatt (&rsp.stat, &iatt);
-        }
-
-        GF_PROTOCOL_DICT_UNSERIALIZE (this, xdata, (rsp.xdata.xdata_val),
-                                      (rsp.xdata.xdata_len), ret,
-                                      rsp.op_errno, out);
+    gfs3_stat_rsp rsp = {
+        0,
+    };
+    call_frame_t *frame = NULL;
+    struct iatt iatt = {
+        0,
+    };
+    int ret = 0;
+    xlator_t *this = NULL;
+    dict_t *xdata = NULL;
+
+    this = THIS;
+
+    frame = myframe;
+
+    if (-1 == req->rpc_status) {
+        rsp.op_ret = -1;
+        rsp.op_errno = ENOTCONN;
+        goto out;
+    }
+    ret = xdr_to_generic(*iov, &rsp, (xdrproc_t)xdr_gfs3_stat_rsp);
+    if (ret < 0) {
+        gf_smsg(this->name, GF_LOG_ERROR, EINVAL, PC_MSG_XDR_DECODING_FAILED,
+                NULL);
+        rsp.op_ret = -1;
+        rsp.op_errno = EINVAL;
+        goto out;
+    }
 
+    ret = client_post_stat(this, &rsp, &iatt, &xdata);
 out:
-        if (rsp.op_ret == -1) {
-                gf_log (this->name, GF_LOG_WARNING, "remote operation failed: %s",
-                        strerror (gf_error_to_errno (rsp.op_errno)));
+    if (rsp.op_ret == -1) {
+        /* stale filehandles are possible during normal operations, no
+         * need to spam the logs with these */
+        if (rsp.op_errno == ESTALE) {
+            gf_msg_debug(this->name, 0, "remote operation failed: %s",
+                         strerror(gf_error_to_errno(rsp.op_errno)));
+        } else {
+            gf_smsg(this->name, GF_LOG_WARNING, gf_error_to_errno(rsp.op_errno),
+                    PC_MSG_REMOTE_OP_FAILED, NULL);
         }
+    }
 
-        CLIENT_STACK_UNWIND (stat, frame, rsp.op_ret,
-                             gf_error_to_errno (rsp.op_errno), &iatt, xdata);
+    CLIENT_STACK_UNWIND(stat, frame, rsp.op_ret,
+                        gf_error_to_errno(rsp.op_errno), &iatt, xdata);
 
-        free (rsp.xdata.xdata_val);
+    free(rsp.xdata.xdata_val);
 
-        if (xdata)
-                dict_unref (xdata);
+    if (xdata)
+        dict_unref(xdata);
 
-        return 0;
+    return 0;
 }
 
 int
-client3_3_readlink_cbk (struct rpc_req *req, struct iovec *iov, int count,
-                        void *myframe)
+client3_3_readlink_cbk(struct rpc_req *req, struct iovec *iov, int count,
+                       void *myframe)
 {
-        gfs3_readlink_rsp rsp = {0,};
-        call_frame_t   *frame = NULL;
-        struct iatt  iatt = {0,};
-        int ret = 0;
-        xlator_t *this       = NULL;
-        dict_t  *xdata       = NULL;
-
-
-        this = THIS;
-
-        frame = myframe;
-
-        if (-1 == req->rpc_status) {
-                rsp.op_ret   = -1;
-                rsp.op_errno = ENOTCONN;
-                goto out;
-        }
-        ret = xdr_to_generic (*iov, &rsp, (xdrproc_t)xdr_gfs3_readlink_rsp);
-        if (ret < 0) {
-                gf_log (this->name, GF_LOG_ERROR, "XDR decoding failed");
-                rsp.op_ret   = -1;
-                rsp.op_errno = EINVAL;
-                goto out;
-        }
-
-        if (-1 != rsp.op_ret) {
-                gf_stat_to_iatt (&rsp.buf, &iatt);
-        }
+    gfs3_readlink_rsp rsp = {
+        0,
+    };
+    call_frame_t *frame = NULL;
+    struct iatt iatt = {
+        0,
+    };
+    int ret = 0;
+    xlator_t *this = NULL;
+    dict_t *xdata = NULL;
+
+    this = THIS;
+
+    frame = myframe;
+
+    if (-1 == req->rpc_status) {
+        rsp.op_ret = -1;
+        rsp.op_errno = ENOTCONN;
+        goto out;
+    }
+    ret = xdr_to_generic(*iov, &rsp, (xdrproc_t)xdr_gfs3_readlink_rsp);
+    if (ret < 0) {
+        gf_smsg(this->name, GF_LOG_ERROR, EINVAL, PC_MSG_XDR_DECODING_FAILED,
+                NULL);
+        rsp.op_ret = -1;
+        rsp.op_errno = EINVAL;
+        goto out;
+    }
 
-        GF_PROTOCOL_DICT_UNSERIALIZE (this, xdata, (rsp.xdata.xdata_val),
-                                      (rsp.xdata.xdata_len), ret,
-                                      rsp.op_errno, out);
+    ret = client_post_readlink(this, &rsp, &iatt, &xdata);
 
 out:
-        if (rsp.op_ret == -1) {
-                gf_log (this->name, (gf_error_to_errno(rsp.op_errno) == ENOENT)?
-                        GF_LOG_DEBUG:GF_LOG_WARNING, "remote operation failed:"
-                        " %s", strerror (gf_error_to_errno (rsp.op_errno)));
+    if (rsp.op_ret == -1) {
+        if (gf_error_to_errno(rsp.op_errno) == ENOENT) {
+            gf_msg_debug(this->name, 0,
+                         "remote operation failed:"
+                         " %s",
+                         strerror(gf_error_to_errno(rsp.op_errno)));
+        } else {
+            gf_smsg(this->name, GF_LOG_WARNING, gf_error_to_errno(rsp.op_errno),
+                    PC_MSG_REMOTE_OP_FAILED, NULL);
         }
+    }
 
-        CLIENT_STACK_UNWIND (readlink, frame, rsp.op_ret,
-                             gf_error_to_errno (rsp.op_errno), rsp.path,
-                             &iatt, xdata);
+    CLIENT_STACK_UNWIND(readlink, frame, rsp.op_ret,
+                        gf_error_to_errno(rsp.op_errno), rsp.path, &iatt,
+                        xdata);
 
-        /* This is allocated by the libc while decoding RPC msg */
-        /* Hence no 'GF_FREE', but just 'free' */
-        free (rsp.path);
+    /* This is allocated by the libc while decoding RPC msg */
+    /* Hence no 'GF_FREE', but just 'free' */
+    free(rsp.path);
 
-        free (rsp.xdata.xdata_val);
+    free(rsp.xdata.xdata_val);
 
-        if (xdata)
-                dict_unref (xdata);
+    if (xdata)
+        dict_unref(xdata);
 
-        return 0;
+    return 0;
 }
 
 int
-client3_3_unlink_cbk (struct rpc_req *req, struct iovec *iov, int count,
-                      void *myframe)
+client3_3_unlink_cbk(struct rpc_req *req, struct iovec *iov, int count,
+                     void *myframe)
 {
-        call_frame_t    *frame      = NULL;
-        gfs3_unlink_rsp  rsp        = {0,};
-        struct iatt      preparent  = {0,};
-        struct iatt      postparent = {0,};
-        int              ret        = 0;
-        xlator_t *this       = NULL;
-        dict_t  *xdata       = NULL;
-
-
-        this = THIS;
-
-        frame = myframe;
-
-        if (-1 == req->rpc_status) {
-                rsp.op_ret   = -1;
-                rsp.op_errno = ENOTCONN;
-                goto out;
-        }
-        ret = xdr_to_generic (*iov, &rsp, (xdrproc_t)xdr_gfs3_unlink_rsp);
-        if (ret < 0) {
-                gf_log (this->name, GF_LOG_ERROR, "XDR decoding failed");
-                rsp.op_ret   = -1;
-                rsp.op_errno = EINVAL;
-                goto out;
-        }
-
-        if (-1 != rsp.op_ret) {
-                gf_stat_to_iatt (&rsp.preparent, &preparent);
-                gf_stat_to_iatt (&rsp.postparent, &postparent);
-        }
+    call_frame_t *frame = NULL;
+    gfs3_unlink_rsp rsp = {
+        0,
+    };
+    struct iatt preparent = {
+        0,
+    };
+    struct iatt postparent = {
+        0,
+    };
+    int ret = 0;
+    xlator_t *this = NULL;
+    dict_t *xdata = NULL;
+
+    this = THIS;
+
+    frame = myframe;
+
+    if (-1 == req->rpc_status) {
+        rsp.op_ret = -1;
+        rsp.op_errno = ENOTCONN;
+        goto out;
+    }
+    ret = xdr_to_generic(*iov, &rsp, (xdrproc_t)xdr_gfs3_unlink_rsp);
+    if (ret < 0) {
+        gf_smsg(this->name, GF_LOG_ERROR, EINVAL, PC_MSG_XDR_DECODING_FAILED,
+                NULL);
+        rsp.op_ret = -1;
+        rsp.op_errno = EINVAL;
+        goto out;
+    }
 
-        GF_PROTOCOL_DICT_UNSERIALIZE (this, xdata, (rsp.xdata.xdata_val),
-                                      (rsp.xdata.xdata_len), ret,
-                                      rsp.op_errno, out);
+    ret = client_post_unlink(this, &rsp, &preparent, &postparent, &xdata);
 
 out:
-        if (rsp.op_ret == -1) {
-                gf_log (this->name,
-                        ((gf_error_to_errno (rsp.op_errno) == ENOENT)
-                        ? GF_LOG_DEBUG : GF_LOG_WARNING),
-                        "remote operation failed: %s",
-                        strerror (gf_error_to_errno (rsp.op_errno)));
+    if (rsp.op_ret == -1) {
+        if (gf_error_to_errno(rsp.op_errno) == ENOENT) {
+            gf_msg_debug(this->name, 0,
+                         "remote operation failed:"
+                         " %s",
+                         strerror(gf_error_to_errno(rsp.op_errno)));
+        } else {
+            gf_smsg(this->name, GF_LOG_WARNING, gf_error_to_errno(rsp.op_errno),
+                    PC_MSG_REMOTE_OP_FAILED, NULL);
         }
+    }
 
-        CLIENT_STACK_UNWIND (unlink, frame, rsp.op_ret,
-                             gf_error_to_errno (rsp.op_errno), &preparent,
-                             &postparent, xdata);
+    CLIENT_STACK_UNWIND(unlink, frame, rsp.op_ret,
+                        gf_error_to_errno(rsp.op_errno), &preparent,
+                        &postparent, xdata);
 
-        free (rsp.xdata.xdata_val);
+    free(rsp.xdata.xdata_val);
 
-        if (xdata)
-                dict_unref (xdata);
+    if (xdata)
+        dict_unref(xdata);
 
-        return 0;
+    return 0;
 }
 
 int
-client3_3_rmdir_cbk (struct rpc_req *req, struct iovec *iov, int count,
-                     void *myframe)
+client3_3_rmdir_cbk(struct rpc_req *req, struct iovec *iov, int count,
+                    void *myframe)
 {
-        gfs3_rmdir_rsp rsp = {0,};
-        call_frame_t   *frame = NULL;
-        struct iatt  preparent  = {0,};
-        struct iatt  postparent = {0,};
-        int ret = 0;
-        xlator_t *this       = NULL;
-        dict_t  *xdata       = NULL;
-
-
-        this = THIS;
-
-        frame = myframe;
-
-        if (-1 == req->rpc_status) {
-                rsp.op_ret   = -1;
-                rsp.op_errno = ENOTCONN;
-                goto out;
-        }
-        ret = xdr_to_generic (*iov, &rsp, (xdrproc_t)xdr_gfs3_rmdir_rsp);
-        if (ret < 0) {
-                gf_log (this->name, GF_LOG_ERROR, "XDR decoding failed");
-                rsp.op_ret   = -1;
-                rsp.op_errno = EINVAL;
-                goto out;
-        }
-
-        if (-1 != rsp.op_ret) {
-                gf_stat_to_iatt (&rsp.preparent, &preparent);
-                gf_stat_to_iatt (&rsp.postparent, &postparent);
-        }
+    gfs3_rmdir_rsp rsp = {
+        0,
+    };
+    call_frame_t *frame = NULL;
+    struct iatt preparent = {
+        0,
+    };
+    struct iatt postparent = {
+        0,
+    };
+    int ret = 0;
+    xlator_t *this = NULL;
+    dict_t *xdata = NULL;
+
+    this = THIS;
+
+    frame = myframe;
+
+    if (-1 == req->rpc_status) {
+        rsp.op_ret = -1;
+        rsp.op_errno = ENOTCONN;
+        goto out;
+    }
+    ret = xdr_to_generic(*iov, &rsp, (xdrproc_t)xdr_gfs3_rmdir_rsp);
+    if (ret < 0) {
+        gf_smsg(this->name, GF_LOG_ERROR, EINVAL, PC_MSG_XDR_DECODING_FAILED,
+                NULL);
+        rsp.op_ret = -1;
+        rsp.op_errno = EINVAL;
+        goto out;
+    }
 
-        GF_PROTOCOL_DICT_UNSERIALIZE (this, xdata, (rsp.xdata.xdata_val),
-                                      (rsp.xdata.xdata_len), ret,
-                                      rsp.op_errno, out);
+    ret = client_post_rmdir(this, &rsp, &preparent, &postparent, &xdata);
 
 out:
-        if (rsp.op_ret == -1) {
-                gf_log (this->name, GF_LOG_WARNING, "remote operation failed: %s",
-                        strerror (gf_error_to_errno (rsp.op_errno)));
+    if (rsp.op_ret == -1) {
+        if (GF_IGNORE_IF_GSYNCD_SAFE_ERROR(frame, rsp.op_errno)) {
+            gf_smsg(this->name, GF_LOG_WARNING, gf_error_to_errno(rsp.op_errno),
+                    PC_MSG_REMOTE_OP_FAILED, NULL);
         }
-        CLIENT_STACK_UNWIND (rmdir, frame, rsp.op_ret,
-                             gf_error_to_errno (rsp.op_errno), &preparent,
-                             &postparent, xdata);
+    }
+    CLIENT_STACK_UNWIND(rmdir, frame, rsp.op_ret,
+                        gf_error_to_errno(rsp.op_errno), &preparent,
+                        &postparent, xdata);
 
-        free (rsp.xdata.xdata_val);
+    free(rsp.xdata.xdata_val);
 
-        if (xdata)
-                dict_unref (xdata);
+    if (xdata)
+        dict_unref(xdata);
 
-        return 0;
+    return 0;
 }
 
-
 int
-client3_3_truncate_cbk (struct rpc_req *req, struct iovec *iov, int count,
-                        void *myframe)
+client3_3_truncate_cbk(struct rpc_req *req, struct iovec *iov, int count,
+                       void *myframe)
 {
-        gfs3_truncate_rsp rsp = {0,};
-        call_frame_t   *frame = NULL;
-        struct iatt  prestat  = {0,};
-        struct iatt  poststat = {0,};
-        int ret = 0;
-        xlator_t *this       = NULL;
-        dict_t  *xdata       = NULL;
-
-
-        this = THIS;
-
-        frame = myframe;
-
-        if (-1 == req->rpc_status) {
-                rsp.op_ret   = -1;
-                rsp.op_errno = ENOTCONN;
-                goto out;
-        }
-        ret = xdr_to_generic (*iov, &rsp, (xdrproc_t)xdr_gfs3_truncate_rsp);
-        if (ret < 0) {
-                gf_log (this->name, GF_LOG_ERROR, "XDR decoding failed");
-                rsp.op_ret   = -1;
-                rsp.op_errno = EINVAL;
-                goto out;
-        }
-
-        if (-1 != rsp.op_ret) {
-                gf_stat_to_iatt (&rsp.prestat, &prestat);
-                gf_stat_to_iatt (&rsp.poststat, &poststat);
-        }
+    gfs3_truncate_rsp rsp = {
+        0,
+    };
+    call_frame_t *frame = NULL;
+    struct iatt prestat = {
+        0,
+    };
+    struct iatt poststat = {
+        0,
+    };
+    int ret = 0;
+    xlator_t *this = NULL;
+    dict_t *xdata = NULL;
+
+    this = THIS;
+
+    frame = myframe;
+
+    if (-1 == req->rpc_status) {
+        rsp.op_ret = -1;
+        rsp.op_errno = ENOTCONN;
+        goto out;
+    }
+    ret = xdr_to_generic(*iov, &rsp, (xdrproc_t)xdr_gfs3_truncate_rsp);
+    if (ret < 0) {
+        gf_smsg(this->name, GF_LOG_ERROR, EINVAL, PC_MSG_XDR_DECODING_FAILED,
+                NULL);
+        rsp.op_ret = -1;
+        rsp.op_errno = EINVAL;
+        goto out;
+    }
 
-        GF_PROTOCOL_DICT_UNSERIALIZE (this, xdata, (rsp.xdata.xdata_val),
-                                      (rsp.xdata.xdata_len), ret,
-                                      rsp.op_errno, out);
+    ret = client_post_truncate(this, &rsp, &prestat, &poststat, &xdata);
 
 out:
-        if (rsp.op_ret == -1) {
-                gf_log (this->name, GF_LOG_WARNING, "remote operation failed: %s",
-                        strerror (gf_error_to_errno (rsp.op_errno)));
-        }
-        CLIENT_STACK_UNWIND (truncate, frame, rsp.op_ret,
-                             gf_error_to_errno (rsp.op_errno), &prestat,
-                             &poststat, xdata);
+    if (rsp.op_ret == -1) {
+        gf_smsg(this->name, GF_LOG_WARNING, gf_error_to_errno(rsp.op_errno),
+                PC_MSG_REMOTE_OP_FAILED, NULL);
+    }
+    CLIENT_STACK_UNWIND(truncate, frame, rsp.op_ret,
+                        gf_error_to_errno(rsp.op_errno), &prestat, &poststat,
+                        xdata);
 
-        free (rsp.xdata.xdata_val);
+    free(rsp.xdata.xdata_val);
 
-        if (xdata)
-                dict_unref (xdata);
+    if (xdata)
+        dict_unref(xdata);
 
-        return 0;
+    return 0;
 }
 
-
 int
-client3_3_statfs_cbk (struct rpc_req *req, struct iovec *iov, int count,
-                      void *myframe)
+client3_3_statfs_cbk(struct rpc_req *req, struct iovec *iov, int count,
+                     void *myframe)
 {
-        gfs3_statfs_rsp rsp = {0,};
-        call_frame_t   *frame = NULL;
-        struct statvfs  statfs = {0,};
-        int ret = 0;
-        xlator_t *this       = NULL;
-        dict_t  *xdata       = NULL;
-
-
-        this = THIS;
-
-        frame = myframe;
-
-        if (-1 == req->rpc_status) {
-                rsp.op_ret   = -1;
-                rsp.op_errno = ENOTCONN;
-                goto out;
-        }
-        ret = xdr_to_generic (*iov, &rsp, (xdrproc_t)xdr_gfs3_statfs_rsp);
-        if (ret < 0) {
-                gf_log (this->name, GF_LOG_ERROR, "XDR decoding failed");
-                rsp.op_ret   = -1;
-                rsp.op_errno = EINVAL;
-                goto out;
-        }
-
-        if (-1 != rsp.op_ret) {
-                gf_statfs_to_statfs (&rsp.statfs, &statfs);
-        }
+    gfs3_statfs_rsp rsp = {
+        0,
+    };
+    call_frame_t *frame = NULL;
+    struct statvfs statfs = {
+        0,
+    };
+    int ret = 0;
+    xlator_t *this = NULL;
+    dict_t *xdata = NULL;
+
+    this = THIS;
+
+    frame = myframe;
+
+    if (-1 == req->rpc_status) {
+        rsp.op_ret = -1;
+        rsp.op_errno = ENOTCONN;
+        goto out;
+    }
+    ret = xdr_to_generic(*iov, &rsp, (xdrproc_t)xdr_gfs3_statfs_rsp);
+    if (ret < 0) {
+        gf_smsg(this->name, GF_LOG_ERROR, EINVAL, PC_MSG_XDR_DECODING_FAILED,
+                NULL);
+        rsp.op_ret = -1;
+        rsp.op_errno = EINVAL;
+        goto out;
+    }
 
-        GF_PROTOCOL_DICT_UNSERIALIZE (this, xdata, (rsp.xdata.xdata_val),
-                                      (rsp.xdata.xdata_len), ret,
-                                      rsp.op_errno, out);
+    ret = client_post_statfs(this, &rsp, &statfs, &xdata);
 
 out:
-        if (rsp.op_ret == -1) {
-                gf_log (this->name, GF_LOG_WARNING, "remote operation failed: %s",
-                        strerror (gf_error_to_errno (rsp.op_errno)));
-        }
-        CLIENT_STACK_UNWIND (statfs, frame, rsp.op_ret,
-                             gf_error_to_errno (rsp.op_errno), &statfs, xdata);
+    if (rsp.op_ret == -1) {
+        gf_smsg(this->name, GF_LOG_WARNING, gf_error_to_errno(rsp.op_errno),
+                PC_MSG_REMOTE_OP_FAILED, NULL);
+    }
+    CLIENT_STACK_UNWIND(statfs, frame, rsp.op_ret,
+                        gf_error_to_errno(rsp.op_errno), &statfs, xdata);
 
-        free (rsp.xdata.xdata_val);
+    free(rsp.xdata.xdata_val);
 
-        if (xdata)
-                dict_unref (xdata);
+    if (xdata)
+        dict_unref(xdata);
 
-        return 0;
+    return 0;
 }
 
-
 int
-client3_3_writev_cbk (struct rpc_req *req, struct iovec *iov, int count,
-                      void *myframe)
+client3_3_writev_cbk(struct rpc_req *req, struct iovec *iov, int count,
+                     void *myframe)
 {
-        gfs3_write_rsp rsp = {0,};
-        call_frame_t   *frame = NULL;
-        struct iatt  prestat  = {0,};
-        struct iatt  poststat = {0,};
-        int ret = 0;
-        xlator_t *this       = NULL;
-        dict_t  *xdata       = NULL;
-        clnt_local_t    *local = NULL;
+    gfs3_write_rsp rsp = {
+        0,
+    };
+    call_frame_t *frame = NULL;
+    struct iatt prestat = {
+        0,
+    };
+    struct iatt poststat = {
+        0,
+    };
+    int ret = 0;
+    xlator_t *this = NULL;
+    dict_t *xdata = NULL;
+    clnt_local_t *local = NULL;
+
+    this = THIS;
+
+    frame = myframe;
+    local = frame->local;
+
+    if (-1 == req->rpc_status) {
+        rsp.op_ret = -1;
+        rsp.op_errno = ENOTCONN;
+        goto out;
+    }
+
+    ret = xdr_to_generic(*iov, &rsp, (xdrproc_t)xdr_gfs3_write_rsp);
+    if (ret < 0) {
+        gf_smsg(this->name, GF_LOG_ERROR, EINVAL, PC_MSG_XDR_DECODING_FAILED,
+                NULL);
+        rsp.op_ret = -1;
+        rsp.op_errno = EINVAL;
+        goto out;
+    }
 
+    ret = client_post_writev(this, &rsp, &prestat, &poststat, &xdata);
+    if (ret < 0)
+        goto out;
+out:
+    if (rsp.op_ret == -1) {
+        gf_smsg(this->name, GF_LOG_WARNING, gf_error_to_errno(rsp.op_errno),
+                PC_MSG_REMOTE_OP_FAILED, NULL);
+    } else if (rsp.op_ret >= 0) {
+        if (local->attempt_reopen)
+            client_attempt_reopen(local->fd, this);
+    }
+    CLIENT_STACK_UNWIND(writev, frame, rsp.op_ret,
+                        gf_error_to_errno(rsp.op_errno), &prestat, &poststat,
+                        xdata);
+
+    free(rsp.xdata.xdata_val);
+
+    if (xdata)
+        dict_unref(xdata);
+
+    return 0;
+}
 
-        this = THIS;
+int
+client3_3_flush_cbk(struct rpc_req *req, struct iovec *iov, int count,
+                    void *myframe)
+{
+    call_frame_t *frame = NULL;
+    clnt_local_t *local = NULL;
+    xlator_t *this = NULL;
+    dict_t *xdata = NULL;
 
-        frame = myframe;
-        local = frame->local;
+    gf_common_rsp rsp = {
+        0,
+    };
+    int ret = 0;
 
-        if (-1 == req->rpc_status) {
-                rsp.op_ret   = -1;
-                rsp.op_errno = ENOTCONN;
-                goto out;
-        }
+    frame = myframe;
+    this = THIS;
+    local = frame->local;
 
-        ret = xdr_to_generic (*iov, &rsp, (xdrproc_t)xdr_gfs3_write_rsp);
-        if (ret < 0) {
-                gf_log (this->name, GF_LOG_ERROR, "XDR decoding failed");
-                rsp.op_ret   = -1;
-                rsp.op_errno = EINVAL;
-                goto out;
-        }
+    if (-1 == req->rpc_status) {
+        rsp.op_ret = -1;
+        rsp.op_errno = ENOTCONN;
+        goto out;
+    }
+    ret = xdr_to_generic(*iov, &rsp, (xdrproc_t)xdr_gf_common_rsp);
+    if (ret < 0) {
+        gf_smsg(this->name, GF_LOG_ERROR, EINVAL, PC_MSG_XDR_DECODING_FAILED,
+                NULL);
+        rsp.op_ret = -1;
+        rsp.op_errno = EINVAL;
+        goto out;
+    }
 
-        if (-1 != rsp.op_ret) {
-                gf_stat_to_iatt (&rsp.prestat, &prestat);
-                gf_stat_to_iatt (&rsp.poststat, &poststat);
-        }
+    if ((rsp.op_ret >= 0 || (rsp.op_errno == ENOTCONN)) &&
+        !fd_is_anonymous(local->fd)) {
+        /* Delete all saved locks of the owner issuing flush */
+        ret = delete_granted_locks_owner(local->fd, &local->owner);
+        gf_msg_trace(this->name, 0, "deleting locks of owner (%s) returned %d",
+                     lkowner_utoa(&local->owner), ret);
+    }
 
-        GF_PROTOCOL_DICT_UNSERIALIZE (this, xdata, (rsp.xdata.xdata_val),
-                                      (rsp.xdata.xdata_len), ret,
-                                      rsp.op_errno, out);
+    ret = client_post_flush(this, &rsp, &xdata);
 
 out:
-        if (rsp.op_ret == -1) {
-                gf_log (this->name, GF_LOG_WARNING, "remote operation failed: %s",
-                        strerror (gf_error_to_errno (rsp.op_errno)));
-        } else if (rsp.op_ret >= 0) {
-                if (local->attempt_reopen)
-                        client_attempt_reopen (local->fd, this);
-        }
-        CLIENT_STACK_UNWIND (writev, frame, rsp.op_ret,
-                             gf_error_to_errno (rsp.op_errno), &prestat,
-                             &poststat, xdata);
+    if (rsp.op_ret == -1) {
+        gf_smsg(this->name,
+                fop_log_level(GF_FOP_FLUSH, gf_error_to_errno(rsp.op_errno)),
+                gf_error_to_errno(rsp.op_errno), PC_MSG_REMOTE_OP_FAILED, NULL);
+    }
+    CLIENT_STACK_UNWIND(flush, frame, rsp.op_ret,
+                        gf_error_to_errno(rsp.op_errno), xdata);
 
-        free (rsp.xdata.xdata_val);
+    free(rsp.xdata.xdata_val);
 
-        if (xdata)
-                dict_unref (xdata);
+    if (xdata)
+        dict_unref(xdata);
 
-        return 0;
+    return 0;
 }
 
 int
-client3_3_flush_cbk (struct rpc_req *req, struct iovec *iov, int count,
-                     void *myframe)
+client3_3_fsync_cbk(struct rpc_req *req, struct iovec *iov, int count,
+                    void *myframe)
 {
-        call_frame_t    *frame      = NULL;
-        clnt_local_t  *local      = NULL;
-        xlator_t *this       = NULL;
-        dict_t  *xdata       = NULL;
-
-        gf_common_rsp    rsp        = {0,};
-        int              ret        = 0;
-
-        frame = myframe;
-        this  = THIS;
-        local = frame->local;
-
-        if (-1 == req->rpc_status) {
-                rsp.op_ret   = -1;
-                rsp.op_errno = ENOTCONN;
-                goto out;
-        }
-        ret = xdr_to_generic (*iov, &rsp, (xdrproc_t)xdr_gf_common_rsp);
-        if (ret < 0) {
-                gf_log (this->name, GF_LOG_ERROR, "XDR decoding failed");
-                rsp.op_ret   = -1;
-                rsp.op_errno = EINVAL;
-                goto out;
-        }
-
-        if (rsp.op_ret >= 0 && !fd_is_anonymous (local->fd)) {
-                /* Delete all saved locks of the owner issuing flush */
-                ret = delete_granted_locks_owner (local->fd, &local->owner);
-                gf_log (this->name, GF_LOG_TRACE,
-                        "deleting locks of owner (%s) returned %d",
-                        lkowner_utoa (&local->owner), ret);
-        }
+    gfs3_fsync_rsp rsp = {
+        0,
+    };
+    call_frame_t *frame = NULL;
+    struct iatt prestat = {
+        0,
+    };
+    struct iatt poststat = {
+        0,
+    };
+    int ret = 0;
+    xlator_t *this = NULL;
+    dict_t *xdata = NULL;
+
+    this = THIS;
+
+    frame = myframe;
+
+    if (-1 == req->rpc_status) {
+        rsp.op_ret = -1;
+        rsp.op_errno = ENOTCONN;
+        goto out;
+    }
+
+    ret = xdr_to_generic(*iov, &rsp, (xdrproc_t)xdr_gfs3_fsync_rsp);
+    if (ret < 0) {
+        gf_smsg(this->name, GF_LOG_ERROR, EINVAL, PC_MSG_XDR_DECODING_FAILED,
+                NULL);
+        rsp.op_ret = -1;
+        rsp.op_errno = EINVAL;
+        goto out;
+    }
 
-        GF_PROTOCOL_DICT_UNSERIALIZE (this, xdata, (rsp.xdata.xdata_val),
-                                      (rsp.xdata.xdata_len), ret,
-                                      rsp.op_errno, out);
+    ret = client_post_fsync(this, &rsp, &prestat, &poststat, &xdata);
+    if (ret < 0)
+        goto out;
 
 out:
-        if (rsp.op_ret == -1) {
-                gf_log (this->name, fop_log_level (GF_FOP_FLUSH,
-                                        gf_error_to_errno (rsp.op_errno)),
-                        "remote operation failed: %s",
-                        strerror (gf_error_to_errno (rsp.op_errno)));
-        }
-        CLIENT_STACK_UNWIND (flush, frame, rsp.op_ret,
-                             gf_error_to_errno (rsp.op_errno), xdata);
+    if (rsp.op_ret == -1) {
+        gf_smsg(this->name, GF_LOG_WARNING, gf_error_to_errno(rsp.op_errno),
+                PC_MSG_REMOTE_OP_FAILED, NULL);
+    }
+    CLIENT_STACK_UNWIND(fsync, frame, rsp.op_ret,
+                        gf_error_to_errno(rsp.op_errno), &prestat, &poststat,
+                        xdata);
 
-        free (rsp.xdata.xdata_val);
+    free(rsp.xdata.xdata_val);
 
-        if (xdata)
-                dict_unref (xdata);
+    if (xdata)
+        dict_unref(xdata);
 
-        return 0;
+    return 0;
 }
 
 int
-client3_3_fsync_cbk (struct rpc_req *req, struct iovec *iov, int count,
-                     void *myframe)
+client3_3_setxattr_cbk(struct rpc_req *req, struct iovec *iov, int count,
+                       void *myframe)
 {
-        gfs3_fsync_rsp rsp = {0,};
-        call_frame_t   *frame = NULL;
-        struct iatt  prestat  = {0,};
-        struct iatt  poststat = {0,};
-        int ret = 0;
-        xlator_t *this       = NULL;
-        dict_t  *xdata       = NULL;
-
-
-        this = THIS;
-
-        frame = myframe;
+    call_frame_t *frame = NULL;
+    gf_common_rsp rsp = {
+        0,
+    };
+    int ret = 0;
+    xlator_t *this = NULL;
+    dict_t *xdata = NULL;
+    int op_errno = EINVAL;
 
-        if (-1 == req->rpc_status) {
-                rsp.op_ret   = -1;
-                rsp.op_errno = ENOTCONN;
-                goto out;
-        }
+    this = THIS;
 
-        ret = xdr_to_generic (*iov, &rsp, (xdrproc_t)xdr_gfs3_fsync_rsp);
-        if (ret < 0) {
-                gf_log (this->name, GF_LOG_ERROR, "XDR decoding failed");
-                rsp.op_ret   = -1;
-                rsp.op_errno = EINVAL;
-                goto out;
-        }
+    frame = myframe;
 
-        if (-1 != rsp.op_ret) {
-                gf_stat_to_iatt (&rsp.prestat, &prestat);
-                gf_stat_to_iatt (&rsp.poststat, &poststat);
-        }
+    if (-1 == req->rpc_status) {
+        rsp.op_ret = -1;
+        rsp.op_errno = ENOTCONN;
+        goto out;
+    }
+
+    ret = xdr_to_generic(*iov, &rsp, (xdrproc_t)xdr_gf_common_rsp);
+    if (ret < 0) {
+        gf_smsg(this->name, GF_LOG_ERROR, EINVAL, PC_MSG_XDR_DECODING_FAILED,
+                NULL);
+        rsp.op_ret = -1;
+        rsp.op_errno = EINVAL;
+        goto out;
+    }
 
-        GF_PROTOCOL_DICT_UNSERIALIZE (this, xdata, (rsp.xdata.xdata_val),
-                                      (rsp.xdata.xdata_len), ret,
-                                      rsp.op_errno, out);
+    ret = client_post_setxattr(this, &rsp, &xdata);
+    if (ret < 0)
+        goto out;
 
 out:
-        if (rsp.op_ret == -1) {
-                gf_log (this->name, GF_LOG_WARNING, "remote operation failed: %s",
-                        strerror (gf_error_to_errno (rsp.op_errno)));
+    op_errno = gf_error_to_errno(rsp.op_errno);
+    if (rsp.op_ret == -1) {
+        if (op_errno == ENOTSUP) {
+            gf_msg_debug(this->name, 0,
+                         "remote operation failed:"
+                         " %s",
+                         strerror(op_errno));
+        } else {
+            gf_smsg(this->name, GF_LOG_WARNING, op_errno,
+                    PC_MSG_REMOTE_OP_FAILED, NULL);
         }
-        CLIENT_STACK_UNWIND (fsync, frame, rsp.op_ret,
-                             gf_error_to_errno (rsp.op_errno), &prestat,
-                             &poststat, xdata);
+    }
 
-        free (rsp.xdata.xdata_val);
+    CLIENT_STACK_UNWIND(setxattr, frame, rsp.op_ret, op_errno, xdata);
 
-        if (xdata)
-                dict_unref (xdata);
+    free(rsp.xdata.xdata_val);
+
+    if (xdata)
+        dict_unref(xdata);
 
-        return 0;
+    return 0;
 }
 
 int
-client3_3_setxattr_cbk (struct rpc_req *req, struct iovec *iov, int count,
-                        void *myframe)
+client3_3_getxattr_cbk(struct rpc_req *req, struct iovec *iov, int count,
+                       void *myframe)
 {
-        call_frame_t  *frame    = NULL;
-        gf_common_rsp  rsp      = {0,};
-        int            ret      = 0;
-        xlator_t      *this     = NULL;
-        dict_t        *xdata    = NULL;
-        int            op_errno = EINVAL;
-
-        this = THIS;
+    call_frame_t *frame = NULL;
+    dict_t *dict = NULL;
+    int op_errno = EINVAL;
+    gfs3_getxattr_rsp rsp = {
+        0,
+    };
+    int ret = 0;
+    clnt_local_t *local = NULL;
+    xlator_t *this = NULL;
+    dict_t *xdata = NULL;
+
+    this = THIS;
+
+    frame = myframe;
+    local = frame->local;
+
+    if (-1 == req->rpc_status) {
+        rsp.op_ret = -1;
+        op_errno = ENOTCONN;
+        goto out;
+    }
+
+    ret = xdr_to_generic(*iov, &rsp, (xdrproc_t)xdr_gfs3_getxattr_rsp);
+    if (ret < 0) {
+        gf_smsg(this->name, GF_LOG_ERROR, EINVAL, PC_MSG_XDR_DECODING_FAILED,
+                NULL);
+        rsp.op_ret = -1;
+        op_errno = EINVAL;
+        goto out;
+    }
 
-        frame = myframe;
+    op_errno = gf_error_to_errno(rsp.op_errno);
+    ret = client_post_getxattr(this, &rsp, &dict, &xdata);
+    if (ret) {
+        op_errno = -ret;
+        goto out;
+    }
 
-        if (-1 == req->rpc_status) {
-                rsp.op_ret   = -1;
-                rsp.op_errno = ENOTCONN;
-                goto out;
+out:
+    if (rsp.op_ret == -1) {
+        if ((op_errno == ENOTSUP) || (op_errno == ENODATA) ||
+            (op_errno == ESTALE) || (op_errno == ENOENT)) {
+            gf_msg_debug(this->name, 0,
+                         "remote operation failed: %s. Path: %s "
+                         "(%s). Key: %s",
+                         strerror(op_errno), local->loc.path,
+                         loc_gfid_utoa(&local->loc),
+                         (local->name) ? local->name : "(null)");
+        } else {
+            gf_smsg(this->name, GF_LOG_WARNING, op_errno,
+                    PC_MSG_REMOTE_OP_FAILED, "Path=%s", local->loc.path,
+                    "gfid=%s", loc_gfid_utoa(&local->loc), "Key=%s",
+                    (local->name) ? local->name : "(null)", NULL);
         }
+    }
 
-        ret = xdr_to_generic (*iov, &rsp, (xdrproc_t)xdr_gf_common_rsp);
-        if (ret < 0) {
-                gf_log (this->name, GF_LOG_ERROR, "XDR decoding failed");
-                rsp.op_ret   = -1;
-                rsp.op_errno = EINVAL;
-                goto out;
-        }
+    CLIENT_STACK_UNWIND(getxattr, frame, rsp.op_ret, op_errno, dict, xdata);
 
-        GF_PROTOCOL_DICT_UNSERIALIZE (this, xdata, (rsp.xdata.xdata_val),
-                                      (rsp.xdata.xdata_len), ret,
-                                      rsp.op_errno, out);
+    /* don't use GF_FREE, this memory was allocated by libc */
+    free(rsp.dict.dict_val);
 
-out:
-        op_errno = gf_error_to_errno (rsp.op_errno);
-        if (rsp.op_ret == -1) {
-                gf_log (this->name, ((op_errno == ENOTSUP) ?
-                                     GF_LOG_DEBUG : GF_LOG_WARNING),
-                        "remote operation failed: %s",
-                        strerror (op_errno));
-        }
-        CLIENT_STACK_UNWIND (setxattr, frame, rsp.op_ret, op_errno, xdata);
+    free(rsp.xdata.xdata_val);
 
-        free (rsp.xdata.xdata_val);
+    if (xdata)
+        dict_unref(xdata);
 
-        if (xdata)
-                dict_unref (xdata);
+    if (dict)
+        dict_unref(dict);
 
-        return 0;
+    return 0;
 }
 
 int
-client3_3_getxattr_cbk (struct rpc_req *req, struct iovec *iov, int count,
+client3_3_fgetxattr_cbk(struct rpc_req *req, struct iovec *iov, int count,
                         void *myframe)
 {
-        call_frame_t      *frame    = NULL;
-        dict_t            *dict     = NULL;
-        int                op_errno = EINVAL;
-        gfs3_getxattr_rsp  rsp      = {0,};
-        int                ret      = 0;
-        clnt_local_t    *local    = NULL;
-        xlator_t *this       = NULL;
-        dict_t  *xdata       = NULL;
-
-
-        this = THIS;
+    call_frame_t *frame = NULL;
+    dict_t *dict = NULL;
+    gfs3_fgetxattr_rsp rsp = {
+        0,
+    };
+    int ret = 0;
+    int op_errno = EINVAL;
+    xlator_t *this = NULL;
+    dict_t *xdata = NULL;
 
-        frame = myframe;
-        local = frame->local;
+    this = THIS;
 
-        if (-1 == req->rpc_status) {
-                rsp.op_ret = -1;
-                op_errno = ENOTCONN;
-                goto out;
-        }
-
-        ret = xdr_to_generic (*iov, &rsp, (xdrproc_t)xdr_gfs3_getxattr_rsp);
-        if (ret < 0) {
-                gf_log (this->name, GF_LOG_ERROR, "XDR decoding failed");
-                rsp.op_ret = -1;
-                op_errno = EINVAL;
-                goto out;
-        }
-
-        op_errno = gf_error_to_errno (rsp.op_errno);
-        if (-1 != rsp.op_ret) {
-                GF_PROTOCOL_DICT_UNSERIALIZE (frame->this, dict,
-                                              (rsp.dict.dict_val),
-                                              (rsp.dict.dict_len), rsp.op_ret,
-                                              op_errno, out);
-        }
-
-        GF_PROTOCOL_DICT_UNSERIALIZE (this, xdata, (rsp.xdata.xdata_val),
-                                      (rsp.xdata.xdata_len), ret,
-                                      op_errno, out);
+    frame = myframe;
 
+    if (-1 == req->rpc_status) {
+        rsp.op_ret = -1;
+        op_errno = ENOTCONN;
+        goto out;
+    }
+    ret = xdr_to_generic(*iov, &rsp, (xdrproc_t)xdr_gfs3_fgetxattr_rsp);
+    if (ret < 0) {
+        gf_smsg(this->name, GF_LOG_ERROR, EINVAL, PC_MSG_XDR_DECODING_FAILED,
+                NULL);
+        rsp.op_ret = -1;
+        op_errno = EINVAL;
+        goto out;
+    }
+
+    op_errno = gf_error_to_errno(rsp.op_errno);
+    ret = client_post_fgetxattr(this, &rsp, &dict, &xdata);
+    if (ret) {
+        op_errno = -ret;
+        goto out;
+    }
 out:
-        if (rsp.op_ret == -1) {
-                gf_log (this->name, (((op_errno == ENOTSUP) ||
-                                      (op_errno == ENODATA) ||
-                                      (op_errno == ESTALE) ||
-                                      (op_errno == ENOENT)) ?
-                                     GF_LOG_DEBUG : GF_LOG_WARNING),
-                        "remote operation failed: %s. Path: %s (%s). Key: %s",
-                        strerror (op_errno),
-                        local->loc.path, loc_gfid_utoa (&local->loc),
-                        (local->name) ? local->name : "(null)");
+    if (rsp.op_ret == -1) {
+        if ((op_errno == ENOTSUP) || (op_errno == ERANGE) ||
+            (op_errno == ENODATA) || (op_errno == ENOENT)) {
+            gf_msg_debug(this->name, 0, "remote operation failed: %s",
+                         strerror(op_errno));
+        } else {
+            gf_smsg(this->name, GF_LOG_WARNING, op_errno,
+                    PC_MSG_REMOTE_OP_FAILED, NULL);
         }
+    }
 
-        CLIENT_STACK_UNWIND (getxattr, frame, rsp.op_ret, op_errno, dict, xdata);
+    CLIENT_STACK_UNWIND(fgetxattr, frame, rsp.op_ret, op_errno, dict, xdata);
 
-        /* don't use GF_FREE, this memory was allocated by libc */
-        free (rsp.dict.dict_val);
+    free(rsp.dict.dict_val);
 
-        free (rsp.xdata.xdata_val);
+    free(rsp.xdata.xdata_val);
 
-        if (xdata)
-                dict_unref (xdata);
+    if (xdata)
+        dict_unref(xdata);
 
-        if (dict)
-                dict_unref (dict);
+    if (dict)
+        dict_unref(dict);
 
-        return 0;
+    return 0;
 }
 
 int
-client3_3_fgetxattr_cbk (struct rpc_req *req, struct iovec *iov, int count,
-                         void *myframe)
+client3_3_removexattr_cbk(struct rpc_req *req, struct iovec *iov, int count,
+                          void *myframe)
 {
-        call_frame_t       *frame    = NULL;
-        dict_t             *dict     = NULL;
-        gfs3_fgetxattr_rsp  rsp      = {0,};
-        int                 ret      = 0;
-        int                 op_errno = EINVAL;
-        clnt_local_t     *local    = NULL;
-        xlator_t *this       = NULL;
-        dict_t  *xdata       = NULL;
+    call_frame_t *frame = NULL;
+    gf_common_rsp rsp = {
+        0,
+    };
+    int ret = 0;
+    xlator_t *this = NULL;
+    dict_t *xdata = NULL;
+    gf_loglevel_t loglevel = GF_LOG_NONE;
 
+    this = THIS;
 
-        this = THIS;
+    frame = myframe;
 
-        frame = myframe;
-        local = frame->local;
-
-        if (-1 == req->rpc_status) {
-                rsp.op_ret = -1;
-                op_errno = ENOTCONN;
-                goto out;
-        }
-        ret = xdr_to_generic (*iov, &rsp, (xdrproc_t)xdr_gfs3_fgetxattr_rsp);
-        if (ret < 0) {
-                gf_log (this->name, GF_LOG_ERROR, "XDR decoding failed");
-                rsp.op_ret = -1;
-                op_errno = EINVAL;
-                goto out;
-        }
-
-        op_errno = gf_error_to_errno (rsp.op_errno);
-        if (-1 != rsp.op_ret) {
-                GF_PROTOCOL_DICT_UNSERIALIZE (frame->this, dict,
-                                              (rsp.dict.dict_val),
-                                              (rsp.dict.dict_len), rsp.op_ret,
-                                              op_errno, out);
-        }
-        GF_PROTOCOL_DICT_UNSERIALIZE (this, xdata, (rsp.xdata.xdata_val),
-                                      (rsp.xdata.xdata_len), ret,
-                                      op_errno, out);
+    if (-1 == req->rpc_status) {
+        rsp.op_ret = -1;
+        rsp.op_errno = ENOTCONN;
+        goto out;
+    }
+
+    ret = xdr_to_generic(*iov, &rsp, (xdrproc_t)xdr_gf_common_rsp);
+    if (ret < 0) {
+        gf_smsg(this->name, GF_LOG_ERROR, EINVAL, PC_MSG_XDR_DECODING_FAILED,
+                NULL);
+        rsp.op_ret = -1;
+        rsp.op_errno = EINVAL;
+        goto out;
+    }
 
+    ret = client_post_removexattr(this, &rsp, &xdata);
 out:
-        if (rsp.op_ret == -1) {
-                gf_log (this->name, (((op_errno == ENOTSUP) ||
-                                      (op_errno == ERANGE)  ||
-                                      (op_errno == ENODATA) ||
-                                      (op_errno == ENOENT)) ?
-                                      GF_LOG_DEBUG : GF_LOG_WARNING),
-                                      "remote operation failed: %s",
-                                      strerror (op_errno));
-        }
-
-        CLIENT_STACK_UNWIND (fgetxattr, frame, rsp.op_ret, op_errno, dict, xdata);
+    if (rsp.op_ret == -1) {
+        /* EPERM/EACCESS is returned some times in case of selinux
+           attributes, or other system attributes which may not be
+           possible to remove from an user process is encountered.
+           we can't treat it as an error */
+        if ((ENODATA == rsp.op_errno) || (ENOATTR == rsp.op_errno) ||
+            (EPERM == rsp.op_errno) || (EACCES == rsp.op_errno))
+            loglevel = GF_LOG_DEBUG;
+        else
+            loglevel = GF_LOG_WARNING;
 
-        free (rsp.dict.dict_val);
+        gf_smsg(this->name, loglevel, gf_error_to_errno(rsp.op_errno),
+                PC_MSG_REMOTE_OP_FAILED, NULL);
+    }
 
-        free (rsp.xdata.xdata_val);
+    CLIENT_STACK_UNWIND(removexattr, frame, rsp.op_ret,
+                        gf_error_to_errno(rsp.op_errno), xdata);
 
-        if (xdata)
-                dict_unref (xdata);
+    free(rsp.xdata.xdata_val);
 
-        if (dict)
-                dict_unref (dict);
+    if (xdata)
+        dict_unref(xdata);
 
-        return 0;
+    return 0;
 }
 
 int
-client3_3_removexattr_cbk (struct rpc_req *req, struct iovec *iov, int count,
+client3_3_fremovexattr_cbk(struct rpc_req *req, struct iovec *iov, int count,
                            void *myframe)
 {
-        call_frame_t    *frame      = NULL;
-        gf_common_rsp    rsp        = {0,};
-        int              ret        = 0;
-        xlator_t        *this       = NULL;
-        dict_t          *xdata      = NULL;
-        gf_loglevel_t    loglevel   = GF_LOG_NONE;
-
-        this = THIS;
-
-        frame = myframe;
+    call_frame_t *frame = NULL;
+    gf_common_rsp rsp = {
+        0,
+    };
+    int ret = 0;
+    xlator_t *this = NULL;
+    dict_t *xdata = NULL;
 
-        if (-1 == req->rpc_status) {
-                rsp.op_ret   = -1;
-                rsp.op_errno = ENOTCONN;
-                goto out;
-        }
+    this = THIS;
 
-        ret = xdr_to_generic (*iov, &rsp, (xdrproc_t)xdr_gf_common_rsp);
-        if (ret < 0) {
-                gf_log (this->name, GF_LOG_ERROR, "XDR decoding failed");
-                rsp.op_ret   = -1;
-                rsp.op_errno = EINVAL;
-                goto out;
-        }
+    frame = myframe;
 
-        GF_PROTOCOL_DICT_UNSERIALIZE (this, xdata, (rsp.xdata.xdata_val),
-                                      (rsp.xdata.xdata_len), ret,
-                                      rsp.op_errno, out);
+    if (-1 == req->rpc_status) {
+        rsp.op_ret = -1;
+        rsp.op_errno = ENOTCONN;
+        goto out;
+    }
+
+    ret = xdr_to_generic(*iov, &rsp, (xdrproc_t)xdr_gf_common_rsp);
+    if (ret < 0) {
+        gf_smsg(this->name, GF_LOG_ERROR, EINVAL, PC_MSG_XDR_DECODING_FAILED,
+                NULL);
+        rsp.op_ret = -1;
+        rsp.op_errno = EINVAL;
+        goto out;
+    }
 
+    ret = client_post_fremovexattr(this, &rsp, &xdata);
 out:
-        if (rsp.op_ret == -1) {
-                if ((ENODATA == rsp.op_errno) || (ENOATTR == rsp.op_errno))
-                        loglevel = GF_LOG_DEBUG;
-                else
-                        loglevel = GF_LOG_WARNING;
-
-                gf_log (this->name, loglevel, "remote operation failed: %s",
-                        strerror (gf_error_to_errno (rsp.op_errno)));
-        }
-
-        CLIENT_STACK_UNWIND (removexattr, frame, rsp.op_ret,
-                             gf_error_to_errno (rsp.op_errno), xdata);
+    if (rsp.op_ret == -1) {
+        gf_smsg(this->name, GF_LOG_WARNING, gf_error_to_errno(rsp.op_errno),
+                PC_MSG_REMOTE_OP_FAILED, NULL);
+    }
+    CLIENT_STACK_UNWIND(fremovexattr, frame, rsp.op_ret,
+                        gf_error_to_errno(rsp.op_errno), xdata);
 
-        free (rsp.xdata.xdata_val);
+    free(rsp.xdata.xdata_val);
 
-        if (xdata)
-                dict_unref (xdata);
+    if (xdata)
+        dict_unref(xdata);
 
-        return 0;
+    return 0;
 }
 
 int
-client3_3_fremovexattr_cbk (struct rpc_req *req, struct iovec *iov, int count,
-                            void *myframe)
+client3_3_fsyncdir_cbk(struct rpc_req *req, struct iovec *iov, int count,
+                       void *myframe)
 {
-        call_frame_t    *frame      = NULL;
-        gf_common_rsp    rsp        = {0,};
-        int              ret        = 0;
-        xlator_t *this       = NULL;
-        dict_t  *xdata       = NULL;
-
+    call_frame_t *frame = NULL;
+    gf_common_rsp rsp = {
+        0,
+    };
+    int ret = 0;
+    xlator_t *this = NULL;
+    dict_t *xdata = NULL;
 
-        this = THIS;
+    this = THIS;
 
-        frame = myframe;
+    frame = myframe;
 
-        if (-1 == req->rpc_status) {
-                rsp.op_ret   = -1;
-                rsp.op_errno = ENOTCONN;
-                goto out;
-        }
-
-        ret = xdr_to_generic (*iov, &rsp, (xdrproc_t)xdr_gf_common_rsp);
-        if (ret < 0) {
-                gf_log (this->name, GF_LOG_ERROR, "XDR decoding failed");
-                rsp.op_ret   = -1;
-                rsp.op_errno = EINVAL;
-                goto out;
-        }
+    if (-1 == req->rpc_status) {
+        rsp.op_ret = -1;
+        rsp.op_errno = ENOTCONN;
+        goto out;
+    }
+    ret = xdr_to_generic(*iov, &rsp, (xdrproc_t)xdr_gf_common_rsp);
+    if (ret < 0) {
+        gf_smsg(this->name, GF_LOG_ERROR, EINVAL, PC_MSG_XDR_DECODING_FAILED,
+                NULL);
+        rsp.op_ret = -1;
+        rsp.op_errno = EINVAL;
+        goto out;
+    }
 
-        GF_PROTOCOL_DICT_UNSERIALIZE (this, xdata, (rsp.xdata.xdata_val),
-                                      (rsp.xdata.xdata_len), ret,
-                                      rsp.op_errno, out);
+    ret = client_post_fsyncdir(this, &rsp, &xdata);
 
 out:
-        if (rsp.op_ret == -1) {
-                gf_log (this->name, GF_LOG_WARNING, "remote operation failed: %s",
-                        strerror (gf_error_to_errno (rsp.op_errno)));
-        }
-        CLIENT_STACK_UNWIND (fremovexattr, frame, rsp.op_ret,
-                             gf_error_to_errno (rsp.op_errno), xdata);
+    if (rsp.op_ret == -1) {
+        gf_smsg(this->name, GF_LOG_WARNING, gf_error_to_errno(rsp.op_errno),
+                PC_MSG_REMOTE_OP_FAILED, NULL);
+    }
+    CLIENT_STACK_UNWIND(fsyncdir, frame, rsp.op_ret,
+                        gf_error_to_errno(rsp.op_errno), xdata);
 
-        free (rsp.xdata.xdata_val);
+    free(rsp.xdata.xdata_val);
 
-        if (xdata)
-                dict_unref (xdata);
+    if (xdata)
+        dict_unref(xdata);
 
-        return 0;
+    return 0;
 }
 
 int
-client3_3_fsyncdir_cbk (struct rpc_req *req, struct iovec *iov, int count,
-                        void *myframe)
+client3_3_access_cbk(struct rpc_req *req, struct iovec *iov, int count,
+                     void *myframe)
 {
-        call_frame_t    *frame      = NULL;
-        gf_common_rsp    rsp        = {0,};
-        int              ret        = 0;
-        xlator_t *this       = NULL;
-        dict_t  *xdata       = NULL;
+    call_frame_t *frame = NULL;
+    gf_common_rsp rsp = {
+        0,
+    };
+    int ret = 0;
+    xlator_t *this = NULL;
+    dict_t *xdata = NULL;
 
+    this = THIS;
 
-        this = THIS;
+    frame = myframe;
 
-        frame = myframe;
-
-        if (-1 == req->rpc_status) {
-                rsp.op_ret   = -1;
-                rsp.op_errno = ENOTCONN;
-                goto out;
-        }
-        ret = xdr_to_generic (*iov, &rsp, (xdrproc_t)xdr_gf_common_rsp);
-        if (ret < 0) {
-                gf_log (this->name, GF_LOG_ERROR, "XDR decoding failed");
-                rsp.op_ret   = -1;
-                rsp.op_errno = EINVAL;
-                goto out;
-        }
+    if (-1 == req->rpc_status) {
+        rsp.op_ret = -1;
+        rsp.op_errno = ENOTCONN;
+        goto out;
+    }
+    ret = xdr_to_generic(*iov, &rsp, (xdrproc_t)xdr_gf_common_rsp);
+    if (ret < 0) {
+        gf_smsg(this->name, GF_LOG_ERROR, EINVAL, PC_MSG_XDR_DECODING_FAILED,
+                NULL);
+        rsp.op_ret = -1;
+        rsp.op_errno = EINVAL;
+        goto out;
+    }
 
-        GF_PROTOCOL_DICT_UNSERIALIZE (this, xdata, (rsp.xdata.xdata_val),
-                                      (rsp.xdata.xdata_len), ret,
-                                      rsp.op_errno, out);
+    ret = client_post_access(this, &rsp, &xdata);
 
 out:
-        if (rsp.op_ret == -1) {
-                gf_log (this->name, GF_LOG_WARNING, "remote operation failed: %s",
-                        strerror (gf_error_to_errno (rsp.op_errno)));
-        }
-        CLIENT_STACK_UNWIND (fsyncdir, frame, rsp.op_ret,
-                             gf_error_to_errno (rsp.op_errno), xdata);
+    if (rsp.op_ret == -1) {
+        gf_smsg(this->name, GF_LOG_WARNING, gf_error_to_errno(rsp.op_errno),
+                PC_MSG_REMOTE_OP_FAILED, NULL);
+    }
+    CLIENT_STACK_UNWIND(access, frame, rsp.op_ret,
+                        gf_error_to_errno(rsp.op_errno), xdata);
 
-        free (rsp.xdata.xdata_val);
+    free(rsp.xdata.xdata_val);
 
-        if (xdata)
-                dict_unref (xdata);
+    if (xdata)
+        dict_unref(xdata);
 
-        return 0;
+    return 0;
 }
 
 int
-client3_3_access_cbk (struct rpc_req *req, struct iovec *iov, int count,
-                      void *myframe)
+client3_3_ftruncate_cbk(struct rpc_req *req, struct iovec *iov, int count,
+                        void *myframe)
 {
-        call_frame_t    *frame      = NULL;
-        gf_common_rsp    rsp        = {0,};
-        int              ret        = 0;
-        xlator_t *this       = NULL;
-        dict_t  *xdata       = NULL;
-
-
-        this = THIS;
-
-        frame = myframe;
-
-        if (-1 == req->rpc_status) {
-                rsp.op_ret   = -1;
-                rsp.op_errno = ENOTCONN;
-                goto out;
-        }
-        ret = xdr_to_generic (*iov, &rsp, (xdrproc_t)xdr_gf_common_rsp);
-        if (ret < 0) {
-                gf_log (this->name, GF_LOG_ERROR, "XDR decoding failed");
-                rsp.op_ret   = -1;
-                rsp.op_errno = EINVAL;
-                goto out;
-        }
+    gfs3_ftruncate_rsp rsp = {
+        0,
+    };
+    call_frame_t *frame = NULL;
+    struct iatt prestat = {
+        0,
+    };
+    struct iatt poststat = {
+        0,
+    };
+    int ret = 0;
+    xlator_t *this = NULL;
+    dict_t *xdata = NULL;
+
+    this = THIS;
+
+    frame = myframe;
+
+    if (-1 == req->rpc_status) {
+        rsp.op_ret = -1;
+        rsp.op_errno = ENOTCONN;
+        goto out;
+    }
+    ret = xdr_to_generic(*iov, &rsp, (xdrproc_t)xdr_gfs3_ftruncate_rsp);
+    if (ret < 0) {
+        gf_smsg(this->name, GF_LOG_ERROR, EINVAL, PC_MSG_XDR_DECODING_FAILED,
+                NULL);
+        rsp.op_ret = -1;
+        rsp.op_errno = EINVAL;
+        goto out;
+    }
 
-        GF_PROTOCOL_DICT_UNSERIALIZE (this, xdata, (rsp.xdata.xdata_val),
-                                      (rsp.xdata.xdata_len), ret,
-                                      rsp.op_errno, out);
+    ret = client_post_ftruncate(this, &rsp, &prestat, &poststat, &xdata);
 
 out:
-        if (rsp.op_ret == -1) {
-                gf_log (this->name, GF_LOG_WARNING, "remote operation failed: %s",
-                        strerror (gf_error_to_errno (rsp.op_errno)));
-        }
-        CLIENT_STACK_UNWIND (access, frame, rsp.op_ret,
-                             gf_error_to_errno (rsp.op_errno), xdata);
+    if (rsp.op_ret == -1) {
+        gf_smsg(this->name, GF_LOG_WARNING, gf_error_to_errno(rsp.op_errno),
+                PC_MSG_REMOTE_OP_FAILED, NULL);
+    }
+    CLIENT_STACK_UNWIND(ftruncate, frame, rsp.op_ret,
+                        gf_error_to_errno(rsp.op_errno), &prestat, &poststat,
+                        xdata);
 
-        free (rsp.xdata.xdata_val);
+    free(rsp.xdata.xdata_val);
 
-        if (xdata)
-                dict_unref (xdata);
+    if (xdata)
+        dict_unref(xdata);
 
-        return 0;
+    return 0;
 }
 
-
 int
-client3_3_ftruncate_cbk (struct rpc_req *req, struct iovec *iov, int count,
-                         void *myframe)
+client3_3_fstat_cbk(struct rpc_req *req, struct iovec *iov, int count,
+                    void *myframe)
 {
-        gfs3_ftruncate_rsp rsp = {0,};
-        call_frame_t   *frame = NULL;
-        struct iatt  prestat  = {0,};
-        struct iatt  poststat = {0,};
-        int ret = 0;
-        xlator_t *this       = NULL;
-        dict_t  *xdata       = NULL;
-
-
-        this = THIS;
-
-        frame = myframe;
-
-        if (-1 == req->rpc_status) {
-                rsp.op_ret   = -1;
-                rsp.op_errno = ENOTCONN;
-                goto out;
-        }
-        ret = xdr_to_generic (*iov, &rsp, (xdrproc_t)xdr_gfs3_ftruncate_rsp);
-        if (ret < 0) {
-                gf_log (this->name, GF_LOG_ERROR, "XDR decoding failed");
-                rsp.op_ret   = -1;
-                rsp.op_errno = EINVAL;
-                goto out;
-        }
-
-        if (-1 != rsp.op_ret) {
-                gf_stat_to_iatt (&rsp.prestat, &prestat);
-                gf_stat_to_iatt (&rsp.poststat, &poststat);
-        }
+    gfs3_fstat_rsp rsp = {
+        0,
+    };
+    call_frame_t *frame = NULL;
+    struct iatt stat = {
+        0,
+    };
+    int ret = 0;
+    xlator_t *this = NULL;
+    dict_t *xdata = NULL;
+
+    this = THIS;
+
+    frame = myframe;
+
+    if (-1 == req->rpc_status) {
+        rsp.op_ret = -1;
+        rsp.op_errno = ENOTCONN;
+        goto out;
+    }
+    ret = xdr_to_generic(*iov, &rsp, (xdrproc_t)xdr_gfs3_fstat_rsp);
+    if (ret < 0) {
+        gf_smsg(this->name, GF_LOG_ERROR, EINVAL, PC_MSG_XDR_DECODING_FAILED,
+                NULL);
+        rsp.op_ret = -1;
+        rsp.op_errno = EINVAL;
+        goto out;
+    }
 
-        GF_PROTOCOL_DICT_UNSERIALIZE (this, xdata, (rsp.xdata.xdata_val),
-                                      (rsp.xdata.xdata_len), ret,
-                                      rsp.op_errno, out);
+    ret = client_post_fstat(this, &rsp, &stat, &xdata);
 
 out:
-        if (rsp.op_ret == -1) {
-                gf_log (this->name, GF_LOG_WARNING, "remote operation failed: %s",
-                        strerror (gf_error_to_errno (rsp.op_errno)));
-        }
-        CLIENT_STACK_UNWIND (ftruncate, frame, rsp.op_ret,
-                             gf_error_to_errno (rsp.op_errno), &prestat,
-                             &poststat, xdata);
+    if (rsp.op_ret == -1) {
+        gf_smsg(this->name, GF_LOG_WARNING, gf_error_to_errno(rsp.op_errno),
+                PC_MSG_REMOTE_OP_FAILED, NULL);
+    }
+    CLIENT_STACK_UNWIND(fstat, frame, rsp.op_ret,
+                        gf_error_to_errno(rsp.op_errno), &stat, xdata);
 
-        free (rsp.xdata.xdata_val);
+    free(rsp.xdata.xdata_val);
 
-        if (xdata)
-                dict_unref (xdata);
+    if (xdata)
+        dict_unref(xdata);
 
-        return 0;
+    return 0;
 }
 
 int
-client3_3_fstat_cbk (struct rpc_req *req, struct iovec *iov, int count,
-                     void *myframe)
+client3_3_inodelk_cbk(struct rpc_req *req, struct iovec *iov, int count,
+                      void *myframe)
 {
-        gfs3_fstat_rsp rsp = {0,};
-        call_frame_t   *frame = NULL;
-        struct iatt  stat  = {0,};
-        int ret = 0;
-        xlator_t *this       = NULL;
-        dict_t  *xdata       = NULL;
-
+    call_frame_t *frame = NULL;
+    gf_common_rsp rsp = {
+        0,
+    };
+    int ret = 0;
+    xlator_t *this = NULL;
+    dict_t *xdata = NULL;
 
-        this = THIS;
+    this = THIS;
 
-        frame = myframe;
+    frame = myframe;
 
-        if (-1 == req->rpc_status) {
-                rsp.op_ret   = -1;
-                rsp.op_errno = ENOTCONN;
-                goto out;
-        }
-        ret = xdr_to_generic (*iov, &rsp, (xdrproc_t)xdr_gfs3_fstat_rsp);
-        if (ret < 0) {
-                gf_log (this->name, GF_LOG_ERROR, "XDR decoding failed");
-                rsp.op_ret   = -1;
-                rsp.op_errno = EINVAL;
-                goto out;
-        }
-
-        if (-1 != rsp.op_ret) {
-                gf_stat_to_iatt (&rsp.stat, &stat);
-        }
-
-        GF_PROTOCOL_DICT_UNSERIALIZE (this, xdata, (rsp.xdata.xdata_val),
-                                      (rsp.xdata.xdata_len), ret,
-                                      rsp.op_errno, out);
+    if (-1 == req->rpc_status) {
+        rsp.op_ret = -1;
+        rsp.op_errno = ENOTCONN;
+        goto out;
+    }
+    ret = xdr_to_generic(*iov, &rsp, (xdrproc_t)xdr_gf_common_rsp);
+    if (ret < 0) {
+        gf_smsg(this->name, GF_LOG_ERROR, EINVAL, PC_MSG_XDR_DECODING_FAILED,
+                NULL);
+        rsp.op_ret = -1;
+        rsp.op_errno = EINVAL;
+        goto out;
+    }
 
+    ret = client_post_inodelk(this, &rsp, &xdata);
 out:
-        if (rsp.op_ret == -1) {
-                gf_log (this->name, GF_LOG_WARNING, "remote operation failed: %s",
-                        strerror (gf_error_to_errno (rsp.op_errno)));
-        }
-        CLIENT_STACK_UNWIND (fstat, frame, rsp.op_ret,
-                             gf_error_to_errno (rsp.op_errno), &stat,  xdata);
+    if (rsp.op_ret == -1) {
+        gf_smsg(this->name,
+                fop_log_level(GF_FOP_INODELK, gf_error_to_errno(rsp.op_errno)),
+                gf_error_to_errno(rsp.op_errno), PC_MSG_REMOTE_OP_FAILED, NULL);
+    }
+    CLIENT_STACK_UNWIND(inodelk, frame, rsp.op_ret,
+                        gf_error_to_errno(rsp.op_errno), xdata);
 
-        free (rsp.xdata.xdata_val);
+    free(rsp.xdata.xdata_val);
 
-        if (xdata)
-                dict_unref (xdata);
+    if (xdata)
+        dict_unref(xdata);
 
-        return 0;
+    return 0;
 }
 
-
 int
-client3_3_inodelk_cbk (struct rpc_req *req, struct iovec *iov, int count,
+client3_3_finodelk_cbk(struct rpc_req *req, struct iovec *iov, int count,
                        void *myframe)
 {
-        call_frame_t    *frame      = NULL;
-        gf_common_rsp    rsp        = {0,};
-        int              ret        = 0;
-        xlator_t *this       = NULL;
-        dict_t  *xdata       = NULL;
-
-
-        this = THIS;
-
-        frame = myframe;
-
-        if (-1 == req->rpc_status) {
-                rsp.op_ret   = -1;
-                rsp.op_errno = ENOTCONN;
-                goto out;
-        }
-        ret = xdr_to_generic (*iov, &rsp, (xdrproc_t)xdr_gf_common_rsp);
-        if (ret < 0) {
-                gf_log (this->name, GF_LOG_ERROR, "XDR decoding failed");
-                rsp.op_ret   = -1;
-                rsp.op_errno = EINVAL;
-                goto out;
-        }
-
-        GF_PROTOCOL_DICT_UNSERIALIZE (this, xdata, (rsp.xdata.xdata_val),
-                                      (rsp.xdata.xdata_len), ret,
-                                      rsp.op_errno, out);
-
-out:
-        if (rsp.op_ret == -1) {
-                gf_log (this->name, fop_log_level (GF_FOP_INODELK,
-                                        gf_error_to_errno (rsp.op_errno)),
-                        "remote operation failed: %s",
-                        strerror (gf_error_to_errno (rsp.op_errno)));
-        }
-        CLIENT_STACK_UNWIND (inodelk, frame, rsp.op_ret,
-                             gf_error_to_errno (rsp.op_errno), xdata);
-
-        free (rsp.xdata.xdata_val);
-
-        if (xdata)
-                dict_unref (xdata);
-
-        return 0;
-}
-
-int
-client3_3_finodelk_cbk (struct rpc_req *req, struct iovec *iov, int count,
-                        void *myframe)
-{
-        call_frame_t  *frame = NULL;
-        gf_common_rsp rsp    = {0,};
-        int           ret    = 0;
-        xlator_t      *this  = NULL;
-        dict_t        *xdata = NULL;
-        clnt_local_t  *local = NULL;
-
-
-        frame = myframe;
-        this = frame->this;
-        local = frame->local;
-
-        if (-1 == req->rpc_status) {
-                rsp.op_ret   = -1;
-                rsp.op_errno = ENOTCONN;
-                goto out;
-        }
-        ret = xdr_to_generic (*iov, &rsp, (xdrproc_t)xdr_gf_common_rsp);
-        if (ret < 0) {
-                gf_log (this->name, GF_LOG_ERROR, "XDR decoding failed");
-                rsp.op_ret   = -1;
-                rsp.op_errno = EINVAL;
-                goto out;
-        }
-
-        GF_PROTOCOL_DICT_UNSERIALIZE (this, xdata, (rsp.xdata.xdata_val),
-                                      (rsp.xdata.xdata_len), ret,
-                                      rsp.op_errno, out);
+    call_frame_t *frame = NULL;
+    gf_common_rsp rsp = {
+        0,
+    };
+    int ret = 0;
+    xlator_t *this = NULL;
+    dict_t *xdata = NULL;
+    clnt_local_t *local = NULL;
+
+    frame = myframe;
+    this = frame->this;
+    local = frame->local;
+
+    if (-1 == req->rpc_status) {
+        rsp.op_ret = -1;
+        rsp.op_errno = ENOTCONN;
+        goto out;
+    }
+    ret = xdr_to_generic(*iov, &rsp, (xdrproc_t)xdr_gf_common_rsp);
+    if (ret < 0) {
+        gf_smsg(this->name, GF_LOG_ERROR, EINVAL, PC_MSG_XDR_DECODING_FAILED,
+                NULL);
+        rsp.op_ret = -1;
+        rsp.op_errno = EINVAL;
+        goto out;
+    }
 
+    ret = client_post_finodelk(this, &rsp, &xdata);
 out:
-        if (rsp.op_ret == -1) {
-                gf_log (this->name, fop_log_level (GF_FOP_FINODELK,
-                                        gf_error_to_errno (rsp.op_errno)),
-                        "remote operation failed: %s",
-                        strerror (gf_error_to_errno (rsp.op_errno)));
-        } else if (rsp.op_ret == 0) {
-                if (local->attempt_reopen)
-                        client_attempt_reopen (local->fd, this);
-        }
-        CLIENT_STACK_UNWIND (finodelk, frame, rsp.op_ret,
-                             gf_error_to_errno (rsp.op_errno), xdata);
-
-        free (rsp.xdata.xdata_val);
-
-        if (xdata)
-                dict_unref (xdata);
-
-        return 0;
+    if (rsp.op_ret == -1) {
+        gf_smsg(this->name,
+                fop_log_level(GF_FOP_FINODELK, gf_error_to_errno(rsp.op_errno)),
+                gf_error_to_errno(rsp.op_errno), PC_MSG_REMOTE_OP_FAILED, NULL);
+    } else if (rsp.op_ret == 0) {
+        if (local->attempt_reopen)
+            client_attempt_reopen(local->fd, this);
+    }
+    CLIENT_STACK_UNWIND(finodelk, frame, rsp.op_ret,
+                        gf_error_to_errno(rsp.op_errno), xdata);
+
+    free(rsp.xdata.xdata_val);
+
+    if (xdata)
+        dict_unref(xdata);
+
+    return 0;
 }
 
 int
-client3_3_entrylk_cbk (struct rpc_req *req, struct iovec *iov, int count,
-                       void *myframe)
+client3_3_entrylk_cbk(struct rpc_req *req, struct iovec *iov, int count,
+                      void *myframe)
 {
-        call_frame_t    *frame      = NULL;
-        gf_common_rsp    rsp        = {0,};
-        int              ret        = 0;
-        xlator_t *this       = NULL;
-        dict_t  *xdata       = NULL;
-
+    call_frame_t *frame = NULL;
+    gf_common_rsp rsp = {
+        0,
+    };
+    int ret = 0;
+    xlator_t *this = NULL;
+    dict_t *xdata = NULL;
 
-        this = THIS;
+    this = THIS;
 
-        frame = myframe;
+    frame = myframe;
 
-        if (-1 == req->rpc_status) {
-                rsp.op_ret   = -1;
-                rsp.op_errno = ENOTCONN;
-                goto out;
-        }
-        ret = xdr_to_generic (*iov, &rsp, (xdrproc_t)xdr_gf_common_rsp);
-        if (ret < 0) {
-                gf_log (this->name, GF_LOG_ERROR, "XDR decoding failed");
-                rsp.op_ret   = -1;
-                rsp.op_errno = EINVAL;
-                goto out;
-        }
-
-        GF_PROTOCOL_DICT_UNSERIALIZE (this, xdata, (rsp.xdata.xdata_val),
-                                      (rsp.xdata.xdata_len), ret,
-                                      rsp.op_errno, out);
+    if (-1 == req->rpc_status) {
+        rsp.op_ret = -1;
+        rsp.op_errno = ENOTCONN;
+        goto out;
+    }
+    ret = xdr_to_generic(*iov, &rsp, (xdrproc_t)xdr_gf_common_rsp);
+    if (ret < 0) {
+        gf_smsg(this->name, GF_LOG_ERROR, EINVAL, PC_MSG_XDR_DECODING_FAILED,
+                NULL);
+        rsp.op_ret = -1;
+        rsp.op_errno = EINVAL;
+        goto out;
+    }
 
+    ret = client_post_entrylk(this, &rsp, &xdata);
 out:
-        if (rsp.op_ret == -1) {
-                gf_log (this->name, fop_log_level (GF_FOP_ENTRYLK,
-                                        gf_error_to_errno (rsp.op_errno)),
-                        "remote operation failed: %s",
-                        strerror (gf_error_to_errno (rsp.op_errno)));
-        }
+    if (rsp.op_ret == -1) {
+        gf_smsg(this->name,
+                fop_log_level(GF_FOP_ENTRYLK, gf_error_to_errno(rsp.op_errno)),
+                gf_error_to_errno(rsp.op_errno), PC_MSG_REMOTE_OP_FAILED, NULL);
+    }
 
-        CLIENT_STACK_UNWIND (entrylk, frame, rsp.op_ret,
-                             gf_error_to_errno (rsp.op_errno), xdata);
+    CLIENT_STACK_UNWIND(entrylk, frame, rsp.op_ret,
+                        gf_error_to_errno(rsp.op_errno), xdata);
 
-        free (rsp.xdata.xdata_val);
+    free(rsp.xdata.xdata_val);
 
-        if (xdata)
-                dict_unref (xdata);
+    if (xdata)
+        dict_unref(xdata);
 
-        return 0;
+    return 0;
 }
 
 int
-client3_3_fentrylk_cbk (struct rpc_req *req, struct iovec *iov, int count,
-                        void *myframe)
+client3_3_fentrylk_cbk(struct rpc_req *req, struct iovec *iov, int count,
+                       void *myframe)
 {
-        call_frame_t    *frame      = NULL;
-        gf_common_rsp    rsp        = {0,};
-        int              ret        = 0;
-        xlator_t *this       = NULL;
-        dict_t  *xdata       = NULL;
-
+    call_frame_t *frame = NULL;
+    gf_common_rsp rsp = {
+        0,
+    };
+    int ret = 0;
+    xlator_t *this = NULL;
+    dict_t *xdata = NULL;
 
-        this = THIS;
+    this = THIS;
 
-        frame = myframe;
+    frame = myframe;
 
-        if (-1 == req->rpc_status) {
-                rsp.op_ret   = -1;
-                rsp.op_errno = ENOTCONN;
-                goto out;
-        }
-        ret = xdr_to_generic (*iov, &rsp, (xdrproc_t)xdr_gf_common_rsp);
-        if (ret < 0) {
-                gf_log (this->name, GF_LOG_ERROR, "XDR decoding failed");
-                rsp.op_ret   = -1;
-                rsp.op_errno = EINVAL;
-                goto out;
-        }
+    if (-1 == req->rpc_status) {
+        rsp.op_ret = -1;
+        rsp.op_errno = ENOTCONN;
+        goto out;
+    }
+    ret = xdr_to_generic(*iov, &rsp, (xdrproc_t)xdr_gf_common_rsp);
+    if (ret < 0) {
+        gf_smsg(this->name, GF_LOG_ERROR, EINVAL, PC_MSG_XDR_DECODING_FAILED,
+                NULL);
+        rsp.op_ret = -1;
+        rsp.op_errno = EINVAL;
+        goto out;
+    }
 
-        GF_PROTOCOL_DICT_UNSERIALIZE (this, xdata, (rsp.xdata.xdata_val),
-                                      (rsp.xdata.xdata_len), ret,
-                                      rsp.op_errno, out);
+    ret = client_post_fentrylk(this, &rsp, &xdata);
 
 out:
-        if ((rsp.op_ret == -1) &&
-            (EAGAIN != gf_error_to_errno (rsp.op_errno))) {
-                gf_log (this->name, GF_LOG_WARNING, "remote operation failed: %s",
-                        strerror (gf_error_to_errno (rsp.op_errno)));
-        }
+    if ((rsp.op_ret == -1) && (EAGAIN != gf_error_to_errno(rsp.op_errno))) {
+        gf_smsg(this->name, GF_LOG_WARNING, gf_error_to_errno(rsp.op_errno),
+                PC_MSG_REMOTE_OP_FAILED, NULL);
+    }
 
-        CLIENT_STACK_UNWIND (fentrylk, frame, rsp.op_ret,
-                             gf_error_to_errno (rsp.op_errno), xdata);
+    CLIENT_STACK_UNWIND(fentrylk, frame, rsp.op_ret,
+                        gf_error_to_errno(rsp.op_errno), xdata);
 
-        free (rsp.xdata.xdata_val);
+    free(rsp.xdata.xdata_val);
 
-        if (xdata)
-                dict_unref (xdata);
+    if (xdata)
+        dict_unref(xdata);
 
-        return 0;
+    return 0;
 }
 
 int
-client3_3_xattrop_cbk (struct rpc_req *req, struct iovec *iov, int count,
-                       void *myframe)
+client3_3_xattrop_cbk(struct rpc_req *req, struct iovec *iov, int count,
+                      void *myframe)
 {
-        call_frame_t     *frame    = NULL;
-        dict_t           *dict     = NULL;
-        gfs3_xattrop_rsp  rsp      = {0,};
-        int               ret      = 0;
-        int               op_errno = EINVAL;
-        clnt_local_t   *local    = NULL;
-        xlator_t *this       = NULL;
-        dict_t  *xdata       = NULL;
-
-
-        this = THIS;
-
-        frame = myframe;
-        local = frame->local;
-
-        if (-1 == req->rpc_status) {
-                rsp.op_ret = -1;
-                op_errno = ENOTCONN;
-                goto out;
-        }
-        ret = xdr_to_generic (*iov, &rsp, (xdrproc_t)xdr_gfs3_xattrop_rsp);
-        if (ret < 0) {
-                gf_log (this->name, GF_LOG_ERROR, "XDR decoding failed");
-                rsp.op_ret = -1;
-                op_errno = EINVAL;
-                goto out;
-        }
-
-        op_errno = rsp.op_errno;
-        if (-1 != rsp.op_ret) {
-                GF_PROTOCOL_DICT_UNSERIALIZE (frame->this, dict,
-                                              (rsp.dict.dict_val),
-                                              (rsp.dict.dict_len), rsp.op_ret,
-                                              op_errno, out);
-        }
-
-        GF_PROTOCOL_DICT_UNSERIALIZE (this, xdata, (rsp.xdata.xdata_val),
-                                      (rsp.xdata.xdata_len), ret,
-                                      op_errno, out);
-
+    call_frame_t *frame = NULL;
+    dict_t *dict = NULL;
+    gfs3_xattrop_rsp rsp = {
+        0,
+    };
+    int ret = 0;
+    int op_errno = EINVAL;
+    clnt_local_t *local = NULL;
+    xlator_t *this = NULL;
+    dict_t *xdata = NULL;
+
+    this = THIS;
+
+    frame = myframe;
+    local = frame->local;
+
+    if (-1 == req->rpc_status) {
+        rsp.op_ret = -1;
+        op_errno = ENOTCONN;
+        goto out;
+    }
+    ret = xdr_to_generic(*iov, &rsp, (xdrproc_t)xdr_gfs3_xattrop_rsp);
+    if (ret < 0) {
+        gf_smsg(this->name, GF_LOG_ERROR, EINVAL, PC_MSG_XDR_DECODING_FAILED,
+                NULL);
+        rsp.op_ret = -1;
+        op_errno = EINVAL;
+        goto out;
+    }
+
+    op_errno = rsp.op_errno;
+    ret = client_post_xattrop(this, &rsp, &dict, &xdata);
+    if (ret) {
+        op_errno = -ret;
+        goto out;
+    }
 out:
-        if (rsp.op_ret == -1) {
-                gf_log (this->name, GF_LOG_WARNING,
-                        "remote operation failed: %s. Path: %s (%s)",
-                        strerror (gf_error_to_errno (rsp.op_errno)),
-                        local->loc.path, loc_gfid_utoa (&local->loc));
-        }
+    if (rsp.op_ret == -1) {
+        gf_smsg(this->name, fop_log_level(GF_FOP_XATTROP, op_errno),
+                gf_error_to_errno(rsp.op_errno), PC_MSG_REMOTE_OP_FAILED,
+                "Path=%s", local->loc.path, "gfid=%s",
+                loc_gfid_utoa(&local->loc), NULL);
+    }
 
-        CLIENT_STACK_UNWIND (xattrop, frame, rsp.op_ret,
-                             gf_error_to_errno (op_errno), dict, xdata);
+    CLIENT_STACK_UNWIND(xattrop, frame, rsp.op_ret, gf_error_to_errno(op_errno),
+                        dict, xdata);
 
-        free (rsp.dict.dict_val);
+    free(rsp.dict.dict_val);
 
-        free (rsp.xdata.xdata_val);
+    free(rsp.xdata.xdata_val);
 
-        if (xdata)
-                dict_unref (xdata);
+    if (xdata)
+        dict_unref(xdata);
 
-        if (dict)
-                dict_unref (dict);
+    if (dict)
+        dict_unref(dict);
 
-        return 0;
+    return 0;
 }
 
 int
-client3_3_fxattrop_cbk (struct rpc_req *req, struct iovec *iov, int count,
-                        void *myframe)
+client3_3_fxattrop_cbk(struct rpc_req *req, struct iovec *iov, int count,
+                       void *myframe)
 {
-        call_frame_t      *frame    = NULL;
-        dict_t            *dict     = NULL;
-        dict_t            *xdata    = NULL;
-        gfs3_fxattrop_rsp  rsp      = {0,};
-        int                ret      = 0;
-        int                op_errno = 0;
-        clnt_local_t    *local    = NULL;
-        xlator_t *this       = NULL;
-
-        this = THIS;
-
-        frame = myframe;
-        local = frame->local;
-
-        if (-1 == req->rpc_status) {
-                rsp.op_ret   = -1;
-                op_errno = ENOTCONN;
-                goto out;
-        }
-
-        ret = xdr_to_generic (*iov, &rsp, (xdrproc_t)xdr_gfs3_fxattrop_rsp);
-        if (ret < 0) {
-                rsp.op_ret = -1;
-                op_errno = EINVAL;
-                gf_log (this->name, GF_LOG_ERROR, "XDR decoding failed");
-                goto out;
-        }
-        op_errno = rsp.op_errno;
-        if (-1 != rsp.op_ret) {
-                GF_PROTOCOL_DICT_UNSERIALIZE (frame->this, dict,
-                                              (rsp.dict.dict_val),
-                                              (rsp.dict.dict_len), rsp.op_ret,
-                                              op_errno, out);
-        }
+    call_frame_t *frame = NULL;
+    dict_t *dict = NULL;
+    dict_t *xdata = NULL;
+    gfs3_fxattrop_rsp rsp = {
+        0,
+    };
+    int ret = 0;
+    int op_errno = 0;
+    clnt_local_t *local = NULL;
+    xlator_t *this = NULL;
+
+    this = THIS;
+
+    frame = myframe;
+    local = frame->local;
+
+    if (-1 == req->rpc_status) {
+        rsp.op_ret = -1;
+        op_errno = ENOTCONN;
+        goto out;
+    }
 
-        GF_PROTOCOL_DICT_UNSERIALIZE (frame->this, xdata,
-                                      (rsp.xdata.xdata_val),
-                                      (rsp.xdata.xdata_len), rsp.op_ret,
-                                      op_errno, out);
+    ret = xdr_to_generic(*iov, &rsp, (xdrproc_t)xdr_gfs3_fxattrop_rsp);
+    if (ret < 0) {
+        rsp.op_ret = -1;
+        op_errno = EINVAL;
+        gf_smsg(this->name, GF_LOG_ERROR, EINVAL, PC_MSG_XDR_DECODING_FAILED,
+                NULL);
+        goto out;
+    }
+    op_errno = rsp.op_errno;
+    ret = client_post_fxattrop(this, &rsp, &dict, &xdata);
+    if (ret) {
+        rsp.op_ret = -1;
+        op_errno = -ret;
+        goto out;
+    }
 out:
 
-        if (rsp.op_ret == -1) {
-                gf_log (this->name, GF_LOG_WARNING,
-                        "remote operation failed: %s",
-                        strerror (gf_error_to_errno (op_errno)));
-        } else if (rsp.op_ret == 0) {
-                if (local->attempt_reopen)
-                        client_attempt_reopen (local->fd, this);
-        }
-        CLIENT_STACK_UNWIND (fxattrop, frame, rsp.op_ret,
-                             gf_error_to_errno (op_errno), dict, xdata);
+    if (rsp.op_ret == -1) {
+        gf_smsg(this->name, GF_LOG_WARNING, gf_error_to_errno(rsp.op_errno),
+                PC_MSG_REMOTE_OP_FAILED, NULL);
+    } else if (rsp.op_ret == 0) {
+        if (local->attempt_reopen)
+            client_attempt_reopen(local->fd, this);
+    }
+    CLIENT_STACK_UNWIND(fxattrop, frame, rsp.op_ret,
+                        gf_error_to_errno(op_errno), dict, xdata);
 
-        free (rsp.dict.dict_val);
+    free(rsp.dict.dict_val);
 
-        free (rsp.xdata.xdata_val);
+    free(rsp.xdata.xdata_val);
 
-        if (xdata)
-                dict_unref (xdata);
+    if (xdata)
+        dict_unref(xdata);
 
-        if (dict)
-                dict_unref (dict);
+    if (dict)
+        dict_unref(dict);
 
-        return 0;
+    return 0;
 }
 
 int
-client3_3_fsetxattr_cbk (struct rpc_req *req, struct iovec *iov, int count,
-                         void *myframe)
+client3_3_fsetxattr_cbk(struct rpc_req *req, struct iovec *iov, int count,
+                        void *myframe)
 {
-        call_frame_t  *frame    = NULL;
-        gf_common_rsp  rsp      = {0,};
-        int            ret      = 0;
-        xlator_t      *this     = NULL;
-        dict_t        *xdata    = NULL;
-        int            op_errno = EINVAL;
+    call_frame_t *frame = NULL;
+    gf_common_rsp rsp = {
+        0,
+    };
+    int ret = 0;
+    xlator_t *this = NULL;
+    dict_t *xdata = NULL;
+    int op_errno = EINVAL;
 
-        this = THIS;
+    this = THIS;
 
-        frame = myframe;
+    frame = myframe;
 
-        if (-1 == req->rpc_status) {
-                rsp.op_ret   = -1;
-                rsp.op_errno = ENOTCONN;
-                goto out;
-        }
-        ret = xdr_to_generic (*iov, &rsp, (xdrproc_t)xdr_gf_common_rsp);
-        if (ret < 0) {
-                gf_log (this->name, GF_LOG_ERROR, "XDR decoding failed");
-                rsp.op_ret   = -1;
-                rsp.op_errno = EINVAL;
-                goto out;
-        }
+    if (-1 == req->rpc_status) {
+        rsp.op_ret = -1;
+        rsp.op_errno = ENOTCONN;
+        goto out;
+    }
+    ret = xdr_to_generic(*iov, &rsp, (xdrproc_t)xdr_gf_common_rsp);
+    if (ret < 0) {
+        gf_smsg(this->name, GF_LOG_ERROR, EINVAL, PC_MSG_XDR_DECODING_FAILED,
+                NULL);
+        rsp.op_ret = -1;
+        rsp.op_errno = EINVAL;
+        goto out;
+    }
 
-        GF_PROTOCOL_DICT_UNSERIALIZE (this, xdata, (rsp.xdata.xdata_val),
-                                      (rsp.xdata.xdata_len), ret,
-                                      rsp.op_errno, out);
+    ret = client_post_setxattr(this, &rsp, &xdata);
 
 out:
-        op_errno = gf_error_to_errno (rsp.op_errno);
-        if (rsp.op_ret == -1) {
-                gf_log (this->name, ((op_errno == ENOTSUP) ?
-                                     GF_LOG_DEBUG : GF_LOG_WARNING),
-                        "remote operation failed: %s",
-                        strerror (op_errno));
+    op_errno = gf_error_to_errno(rsp.op_errno);
+    if (rsp.op_ret == -1) {
+        if (op_errno == ENOTSUP) {
+            gf_msg_debug(this->name, 0,
+                         "remote operation failed:"
+                         " %s",
+                         strerror(op_errno));
+        } else {
+            gf_smsg(this->name, GF_LOG_WARNING, rsp.op_errno,
+                    PC_MSG_REMOTE_OP_FAILED, NULL);
         }
+    }
 
-        CLIENT_STACK_UNWIND (fsetxattr, frame, rsp.op_ret, op_errno, xdata);
+    CLIENT_STACK_UNWIND(fsetxattr, frame, rsp.op_ret, op_errno, xdata);
 
-        free (rsp.xdata.xdata_val);
+    free(rsp.xdata.xdata_val);
 
-        if (xdata)
-                dict_unref (xdata);
+    if (xdata)
+        dict_unref(xdata);
 
-        return 0;
+    return 0;
 }
 
 int
-client3_3_fsetattr_cbk (struct rpc_req *req, struct iovec *iov, int count,
-                        void *myframe)
+client3_3_fsetattr_cbk(struct rpc_req *req, struct iovec *iov, int count,
+                       void *myframe)
 {
-        call_frame_t    *frame      = NULL;
-        gfs3_fsetattr_rsp rsp        = {0,};
-        struct iatt      prestat    = {0,};
-        struct iatt      poststat   = {0,};
-        int              ret        = 0;
-        xlator_t *this       = NULL;
-        dict_t  *xdata       = NULL;
-
-
-        this = THIS;
-
-        frame = myframe;
-
-        if (-1 == req->rpc_status) {
-                rsp.op_ret   = -1;
-                rsp.op_errno = ENOTCONN;
-                goto out;
-        }
-        ret = xdr_to_generic (*iov, &rsp, (xdrproc_t)xdr_gfs3_fsetattr_rsp);
-        if (ret < 0) {
-                gf_log (this->name, GF_LOG_ERROR, "XDR decoding failed");
-                rsp.op_ret   = -1;
-                rsp.op_errno = EINVAL;
-                goto out;
-        }
-
-        if (-1 != rsp.op_ret) {
-                gf_stat_to_iatt (&rsp.statpre, &prestat);
-                gf_stat_to_iatt (&rsp.statpost, &poststat);
-        }
-
-        GF_PROTOCOL_DICT_UNSERIALIZE (this, xdata, (rsp.xdata.xdata_val),
-                                      (rsp.xdata.xdata_len), ret,
-                                      rsp.op_errno, out);
+    call_frame_t *frame = NULL;
+    gfs3_fsetattr_rsp rsp = {
+        0,
+    };
+    struct iatt prestat = {
+        0,
+    };
+    struct iatt poststat = {
+        0,
+    };
+    int ret = 0;
+    xlator_t *this = NULL;
+    dict_t *xdata = NULL;
+
+    this = THIS;
+
+    frame = myframe;
+
+    if (-1 == req->rpc_status) {
+        rsp.op_ret = -1;
+        rsp.op_errno = ENOTCONN;
+        goto out;
+    }
+    ret = xdr_to_generic(*iov, &rsp, (xdrproc_t)xdr_gfs3_fsetattr_rsp);
+    if (ret < 0) {
+        gf_smsg(this->name, GF_LOG_ERROR, EINVAL, PC_MSG_XDR_DECODING_FAILED,
+                NULL);
+        rsp.op_ret = -1;
+        rsp.op_errno = EINVAL;
+        goto out;
+    }
 
+    ret = client_post_fsetattr(this, &rsp, &prestat, &poststat, &xdata);
 out:
-        if (rsp.op_ret == -1) {
-                gf_log (this->name, GF_LOG_WARNING, "remote operation failed: %s",
-                        strerror (gf_error_to_errno (rsp.op_errno)));
-        }
-        CLIENT_STACK_UNWIND (fsetattr, frame, rsp.op_ret,
-                             gf_error_to_errno (rsp.op_errno), &prestat,
-                             &poststat, xdata);
+    if (rsp.op_ret == -1) {
+        gf_smsg(this->name, GF_LOG_WARNING, gf_error_to_errno(rsp.op_errno),
+                PC_MSG_REMOTE_OP_FAILED, NULL);
+    }
+    CLIENT_STACK_UNWIND(fsetattr, frame, rsp.op_ret,
+                        gf_error_to_errno(rsp.op_errno), &prestat, &poststat,
+                        xdata);
 
-        free (rsp.xdata.xdata_val);
+    free(rsp.xdata.xdata_val);
 
-        if (xdata)
-                dict_unref (xdata);
+    if (xdata)
+        dict_unref(xdata);
 
-        return 0;
+    return 0;
 }
 
 int
-client3_3_fallocate_cbk (struct rpc_req *req, struct iovec *iov, int count,
+client3_3_fallocate_cbk(struct rpc_req *req, struct iovec *iov, int count,
                         void *myframe)
 {
-        call_frame_t    *frame      = NULL;
-        gfs3_fallocate_rsp rsp      = {0,};
-        struct iatt      prestat    = {0,};
-        struct iatt      poststat   = {0,};
-        int              ret        = 0;
-        xlator_t *this       = NULL;
-        dict_t  *xdata       = NULL;
-
-
-        this = THIS;
-
-        frame = myframe;
-
-        if (-1 == req->rpc_status) {
-                rsp.op_ret   = -1;
-                rsp.op_errno = ENOTCONN;
-                goto out;
-        }
-        ret = xdr_to_generic (*iov, &rsp, (xdrproc_t)xdr_gfs3_fallocate_rsp);
-        if (ret < 0) {
-                gf_log (this->name, GF_LOG_ERROR, "XDR decoding failed");
-                rsp.op_ret   = -1;
-                rsp.op_errno = EINVAL;
-                goto out;
-        }
+    call_frame_t *frame = NULL;
+    gfs3_fallocate_rsp rsp = {
+        0,
+    };
+    struct iatt prestat = {
+        0,
+    };
+    struct iatt poststat = {
+        0,
+    };
+    int ret = 0;
+    xlator_t *this = NULL;
+    dict_t *xdata = NULL;
+
+    this = THIS;
+
+    frame = myframe;
+
+    if (-1 == req->rpc_status) {
+        rsp.op_ret = -1;
+        rsp.op_errno = ENOTCONN;
+        goto out;
+    }
+    ret = xdr_to_generic(*iov, &rsp, (xdrproc_t)xdr_gfs3_fallocate_rsp);
+    if (ret < 0) {
+        gf_smsg(this->name, GF_LOG_ERROR, EINVAL, PC_MSG_XDR_DECODING_FAILED,
+                NULL);
+        rsp.op_ret = -1;
+        rsp.op_errno = EINVAL;
+        goto out;
+    }
 
-        if (-1 != rsp.op_ret) {
-                gf_stat_to_iatt (&rsp.statpre, &prestat);
-                gf_stat_to_iatt (&rsp.statpost, &poststat);
-        }
+    ret = client_post_fallocate(this, &rsp, &prestat, &poststat, &xdata);
+    if (ret < 0)
+        goto out;
 
-        GF_PROTOCOL_DICT_UNSERIALIZE (this, xdata, (rsp.xdata.xdata_val),
-                                      (rsp.xdata.xdata_len), ret,
-                                      rsp.op_errno, out);
+    GF_PROTOCOL_DICT_UNSERIALIZE(this, xdata, (rsp.xdata.xdata_val),
+                                 (rsp.xdata.xdata_len), ret, rsp.op_errno, out);
 
 out:
-        if (rsp.op_ret == -1) {
-                gf_log (this->name, GF_LOG_WARNING, "remote operation failed: %s",
-                        strerror (gf_error_to_errno (rsp.op_errno)));
-        }
-        CLIENT_STACK_UNWIND (fallocate, frame, rsp.op_ret,
-                             gf_error_to_errno (rsp.op_errno), &prestat,
-                             &poststat, xdata);
+    if (rsp.op_ret == -1) {
+        gf_smsg(this->name, GF_LOG_WARNING, gf_error_to_errno(rsp.op_errno),
+                PC_MSG_REMOTE_OP_FAILED, NULL);
+    }
+    CLIENT_STACK_UNWIND(fallocate, frame, rsp.op_ret,
+                        gf_error_to_errno(rsp.op_errno), &prestat, &poststat,
+                        xdata);
 
-        free (rsp.xdata.xdata_val);
+    free(rsp.xdata.xdata_val);
 
-        if (xdata)
-                dict_unref (xdata);
+    if (xdata)
+        dict_unref(xdata);
 
-        return 0;
+    return 0;
 }
 
 int
 client3_3_discard_cbk(struct rpc_req *req, struct iovec *iov, int count,
-		      void *myframe)
+                      void *myframe)
 {
-        call_frame_t    *frame      = NULL;
-        gfs3_discard_rsp rsp      = {0,};
-        struct iatt      prestat    = {0,};
-        struct iatt      poststat   = {0,};
-        int              ret        = 0;
-        xlator_t *this       = NULL;
-        dict_t  *xdata       = NULL;
-
-        this = THIS;
-
-        frame = myframe;
-
-        if (-1 == req->rpc_status) {
-                rsp.op_ret   = -1;
-                rsp.op_errno = ENOTCONN;
-                goto out;
-        }
-        ret = xdr_to_generic(*iov, &rsp, (xdrproc_t) xdr_gfs3_discard_rsp);
-        if (ret < 0) {
-                gf_log (this->name, GF_LOG_ERROR, "XDR decoding failed");
-                rsp.op_ret   = -1;
-                rsp.op_errno = EINVAL;
-                goto out;
-        }
-
-        if (-1 != rsp.op_ret) {
-                gf_stat_to_iatt (&rsp.statpre, &prestat);
-                gf_stat_to_iatt (&rsp.statpost, &poststat);
-        }
+    call_frame_t *frame = NULL;
+    gfs3_discard_rsp rsp = {
+        0,
+    };
+    struct iatt prestat = {
+        0,
+    };
+    struct iatt poststat = {
+        0,
+    };
+    int ret = 0;
+    xlator_t *this = NULL;
+    dict_t *xdata = NULL;
+
+    this = THIS;
+
+    frame = myframe;
+
+    if (-1 == req->rpc_status) {
+        rsp.op_ret = -1;
+        rsp.op_errno = ENOTCONN;
+        goto out;
+    }
+    ret = xdr_to_generic(*iov, &rsp, (xdrproc_t)xdr_gfs3_discard_rsp);
+    if (ret < 0) {
+        gf_smsg(this->name, GF_LOG_ERROR, EINVAL, PC_MSG_XDR_DECODING_FAILED,
+                NULL);
+        rsp.op_ret = -1;
+        rsp.op_errno = EINVAL;
+        goto out;
+    }
 
-        GF_PROTOCOL_DICT_UNSERIALIZE (this, xdata, (rsp.xdata.xdata_val),
-                                      (rsp.xdata.xdata_len), ret,
-                                      rsp.op_errno, out);
+    ret = client_post_discard(this, &rsp, &prestat, &poststat, &xdata);
 
 out:
-        if (rsp.op_ret == -1) {
-                gf_log (this->name, GF_LOG_WARNING, "remote operation failed: %s",
-                        strerror (gf_error_to_errno (rsp.op_errno)));
-        }
-        CLIENT_STACK_UNWIND (discard, frame, rsp.op_ret,
-                             gf_error_to_errno (rsp.op_errno), &prestat,
-                             &poststat, xdata);
+    if (rsp.op_ret == -1) {
+        gf_smsg(this->name, GF_LOG_WARNING, gf_error_to_errno(rsp.op_errno),
+                PC_MSG_REMOTE_OP_FAILED, NULL);
+    }
+    CLIENT_STACK_UNWIND(discard, frame, rsp.op_ret,
+                        gf_error_to_errno(rsp.op_errno), &prestat, &poststat,
+                        xdata);
 
-        free (rsp.xdata.xdata_val);
+    free(rsp.xdata.xdata_val);
 
-        if (xdata)
-                dict_unref (xdata);
+    if (xdata)
+        dict_unref(xdata);
 
-        return 0;
+    return 0;
 }
 
 int
 client3_3_zerofill_cbk(struct rpc_req *req, struct iovec *iov, int count,
-                      void *myframe)
+                       void *myframe)
 {
-        call_frame_t    *frame         = NULL;
-        gfs3_zerofill_rsp rsp          = {0,};
-        struct iatt      prestat       = {0,};
-        struct iatt      poststat      = {0,};
-        int              ret           = 0;
-        xlator_t *this                 = NULL;
-        dict_t  *xdata                 = NULL;
-
-        this = THIS;
-
-        frame = myframe;
-
-        if (-1 == req->rpc_status) {
-                rsp.op_ret   = -1;
-                rsp.op_errno = ENOTCONN;
-                goto out;
-        }
-        ret = xdr_to_generic(*iov, &rsp, (xdrproc_t) xdr_gfs3_zerofill_rsp);
-        if (ret < 0) {
-                gf_log (this->name, GF_LOG_ERROR, "XDR decoding failed");
-                rsp.op_ret   = -1;
-                rsp.op_errno = EINVAL;
-                goto out;
-        }
-
-        if (-1 != rsp.op_ret) {
-                gf_stat_to_iatt (&rsp.statpre, &prestat);
-                gf_stat_to_iatt (&rsp.statpost, &poststat);
-        }
-
-        GF_PROTOCOL_DICT_UNSERIALIZE (this, xdata, (rsp.xdata.xdata_val),
-                                      (rsp.xdata.xdata_len), ret,
-                                      rsp.op_errno, out);
+    call_frame_t *frame = NULL;
+    gfs3_zerofill_rsp rsp = {
+        0,
+    };
+    struct iatt prestat = {
+        0,
+    };
+    struct iatt poststat = {
+        0,
+    };
+    int ret = 0;
+    xlator_t *this = NULL;
+    dict_t *xdata = NULL;
+
+    this = THIS;
+
+    frame = myframe;
+
+    if (-1 == req->rpc_status) {
+        rsp.op_ret = -1;
+        rsp.op_errno = ENOTCONN;
+        goto out;
+    }
+    ret = xdr_to_generic(*iov, &rsp, (xdrproc_t)xdr_gfs3_zerofill_rsp);
+    if (ret < 0) {
+        gf_smsg(this->name, GF_LOG_ERROR, EINVAL, PC_MSG_XDR_DECODING_FAILED,
+                NULL);
+        rsp.op_ret = -1;
+        rsp.op_errno = EINVAL;
+        goto out;
+    }
 
+    ret = client_post_zerofill(this, &rsp, &prestat, &poststat, &xdata);
 out:
-        if (rsp.op_ret == -1) {
-                gf_log (this->name, GF_LOG_WARNING,
-                        "remote operation failed: %s",
-                        strerror (gf_error_to_errno (rsp.op_errno)));
-        }
-        CLIENT_STACK_UNWIND (zerofill, frame, rsp.op_ret,
-                             gf_error_to_errno (rsp.op_errno), &prestat,
-                             &poststat, xdata);
+    if (rsp.op_ret == -1) {
+        gf_smsg(this->name, GF_LOG_WARNING, gf_error_to_errno(rsp.op_errno),
+                PC_MSG_REMOTE_OP_FAILED, NULL);
+    }
+    CLIENT_STACK_UNWIND(zerofill, frame, rsp.op_ret,
+                        gf_error_to_errno(rsp.op_errno), &prestat, &poststat,
+                        xdata);
 
-        free (rsp.xdata.xdata_val);
+    free(rsp.xdata.xdata_val);
 
-        if (xdata)
-                dict_unref (xdata);
+    if (xdata)
+        dict_unref(xdata);
 
-        return 0;
+    return 0;
 }
 
 int
-client3_3_ipc_cbk (struct rpc_req *req, struct iovec *iov, int count,
-                      void *myframe)
+client3_3_ipc_cbk(struct rpc_req *req, struct iovec *iov, int count,
+                  void *myframe)
 {
-        call_frame_t    *frame         = NULL;
-        gfs3_ipc_rsp     rsp          = {0,};
-        int              ret           = 0;
-        xlator_t *this                 = NULL;
-        dict_t  *xdata                 = NULL;
+    call_frame_t *frame = NULL;
+    gfs3_ipc_rsp rsp = {
+        0,
+    };
+    int ret = 0;
+    xlator_t *this = NULL;
+    dict_t *xdata = NULL;
 
-        this = THIS;
+    this = THIS;
 
-        frame = myframe;
-
-        if (-1 == req->rpc_status) {
-                rsp.op_ret   = -1;
-                rsp.op_errno = ENOTCONN;
-                goto out;
-        }
-        ret = xdr_to_generic(*iov, &rsp, (xdrproc_t) xdr_gfs3_ipc_rsp);
-        if (ret < 0) {
-                gf_log (this->name, GF_LOG_ERROR, "XDR decoding failed");
-                rsp.op_ret   = -1;
-                rsp.op_errno = EINVAL;
-                goto out;
-        }
+    frame = myframe;
 
-        GF_PROTOCOL_DICT_UNSERIALIZE (this, xdata, (rsp.xdata.xdata_val),
-                                      (rsp.xdata.xdata_len), ret,
-                                      rsp.op_errno, out);
+    if (-1 == req->rpc_status) {
+        rsp.op_ret = -1;
+        rsp.op_errno = ENOTCONN;
+        goto out;
+    }
+    ret = xdr_to_generic(*iov, &rsp, (xdrproc_t)xdr_gfs3_ipc_rsp);
+    if (ret < 0) {
+        gf_smsg(this->name, GF_LOG_ERROR, EINVAL, PC_MSG_XDR_DECODING_FAILED,
+                NULL);
+        rsp.op_ret = -1;
+        rsp.op_errno = EINVAL;
+        goto out;
+    }
 
+    ret = client_post_ipc(this, &rsp, &xdata);
 out:
-        if (rsp.op_ret == -1) {
-                gf_log (this->name, GF_LOG_WARNING,
-                        "remote operation failed: %s",
-                        strerror (gf_error_to_errno (rsp.op_errno)));
-        }
-        CLIENT_STACK_UNWIND (ipc, frame,
-                             rsp.op_ret, gf_error_to_errno (rsp.op_errno),
-                             xdata);
+    if (rsp.op_ret == -1) {
+        gf_smsg(this->name, GF_LOG_WARNING, gf_error_to_errno(rsp.op_errno),
+                PC_MSG_REMOTE_OP_FAILED, NULL);
+    }
+    CLIENT_STACK_UNWIND(ipc, frame, rsp.op_ret, gf_error_to_errno(rsp.op_errno),
+                        xdata);
 
-        free (rsp.xdata.xdata_val);
+    free(rsp.xdata.xdata_val);
 
-        if (xdata)
-                dict_unref (xdata);
+    if (xdata)
+        dict_unref(xdata);
 
-        return 0;
+    return 0;
 }
 
 int
-client3_3_setattr_cbk (struct rpc_req *req, struct iovec *iov, int count,
-                       void *myframe)
+client3_3_seek_cbk(struct rpc_req *req, struct iovec *iov, int count,
+                   void *myframe)
 {
-        call_frame_t    *frame      = NULL;
-        gfs3_setattr_rsp rsp        = {0,};
-        struct iatt      prestat    = {0,};
-        struct iatt      poststat   = {0,};
-        int              ret        = 0;
-        xlator_t *this       = NULL;
-        dict_t  *xdata       = NULL;
-
+    call_frame_t *frame = NULL;
+    struct gfs3_seek_rsp rsp = {
+        0,
+    };
+    int ret = 0;
+    xlator_t *this = NULL;
+    dict_t *xdata = NULL;
 
-        this = THIS;
+    this = THIS;
 
-        frame = myframe;
+    frame = myframe;
 
-        if (-1 == req->rpc_status) {
-                rsp.op_ret   = -1;
-                rsp.op_errno = ENOTCONN;
-                goto out;
-        }
-
-        ret = xdr_to_generic (*iov, &rsp, (xdrproc_t)xdr_gfs3_setattr_rsp);
-        if (ret < 0) {
-                gf_log (this->name, GF_LOG_ERROR, "XDR decoding failed");
-                rsp.op_ret   = -1;
-                rsp.op_errno = EINVAL;
-                goto out;
-        }
-
-        if (-1 != rsp.op_ret) {
-                gf_stat_to_iatt (&rsp.statpre, &prestat);
-                gf_stat_to_iatt (&rsp.statpost, &poststat);
-        }
+    if (-1 == req->rpc_status) {
+        rsp.op_ret = -1;
+        rsp.op_errno = ENOTCONN;
+        goto out;
+    }
+    ret = xdr_to_generic(*iov, &rsp, (xdrproc_t)xdr_gfs3_seek_rsp);
+    if (ret < 0) {
+        gf_smsg(this->name, GF_LOG_ERROR, EINVAL, PC_MSG_XDR_DECODING_FAILED,
+                NULL);
+        rsp.op_ret = -1;
+        rsp.op_errno = EINVAL;
+        goto out;
+    }
 
-        GF_PROTOCOL_DICT_UNSERIALIZE (this, xdata, (rsp.xdata.xdata_val),
-                                      (rsp.xdata.xdata_len), ret,
-                                      rsp.op_errno, out);
+    ret = client_post_seek(this, &rsp, &xdata);
 
 out:
-        if (rsp.op_ret == -1) {
-                gf_log (this->name, GF_LOG_WARNING, "remote operation failed: %s",
-                        strerror (gf_error_to_errno (rsp.op_errno)));
-        }
-        CLIENT_STACK_UNWIND (setattr, frame, rsp.op_ret,
-                             gf_error_to_errno (rsp.op_errno), &prestat,
-                             &poststat, xdata);
+    if (rsp.op_ret == -1) {
+        gf_smsg(this->name, GF_LOG_WARNING, gf_error_to_errno(rsp.op_errno),
+                PC_MSG_REMOTE_OP_FAILED, NULL);
+    }
+    CLIENT_STACK_UNWIND(seek, frame, rsp.op_ret,
+                        gf_error_to_errno(rsp.op_errno), rsp.offset, xdata);
 
-        free (rsp.xdata.xdata_val);
+    free(rsp.xdata.xdata_val);
 
-        if (xdata)
-                dict_unref (xdata);
+    if (xdata)
+        dict_unref(xdata);
 
-        return 0;
+    return 0;
 }
 
 int
-client3_3_create_cbk (struct rpc_req *req, struct iovec *iov, int count,
+client3_3_setattr_cbk(struct rpc_req *req, struct iovec *iov, int count,
                       void *myframe)
 {
-        call_frame_t    *frame      = NULL;
-        fd_t            *fd         = NULL;
-        inode_t         *inode      = NULL;
-        struct iatt      stbuf      = {0, };
-        struct iatt      preparent  = {0, };
-        struct iatt      postparent = {0, };
-        int32_t          ret        = -1;
-        clnt_local_t    *local      = NULL;
-        gfs3_create_rsp  rsp        = {0,};
-        xlator_t *this       = NULL;
-        dict_t  *xdata       = NULL;
-
-        this = THIS;
-
-        frame = myframe;
-        local = frame->local;
-        fd    = local->fd;
-        inode = local->loc.inode;
-
-        if (-1 == req->rpc_status) {
-                rsp.op_ret   = -1;
-                rsp.op_errno = ENOTCONN;
-                goto out;
-        }
-
-        ret = xdr_to_generic (*iov, &rsp, (xdrproc_t)xdr_gfs3_create_rsp);
-        if (ret < 0) {
-                gf_log (this->name, GF_LOG_ERROR, "XDR decoding failed");
-                rsp.op_ret   = -1;
-                rsp.op_errno = EINVAL;
-                goto out;
-        }
-
-        if (-1 != rsp.op_ret) {
-                gf_stat_to_iatt (&rsp.stat, &stbuf);
-
-                gf_stat_to_iatt (&rsp.preparent, &preparent);
-                gf_stat_to_iatt (&rsp.postparent, &postparent);
-                gf_uuid_copy (local->loc.gfid, stbuf.ia_gfid);
-                ret = client_add_fd_to_saved_fds (frame->this, fd, &local->loc,
-                                                  local->flags, rsp.fd, 0);
-                if (ret) {
-                        rsp.op_ret = -1;
-                        rsp.op_errno = -ret;
-                        goto out;
-                }
-        }
+    call_frame_t *frame = NULL;
+    gfs3_setattr_rsp rsp = {
+        0,
+    };
+    struct iatt prestat = {
+        0,
+    };
+    struct iatt poststat = {
+        0,
+    };
+    int ret = 0;
+    xlator_t *this = NULL;
+    dict_t *xdata = NULL;
+
+    this = THIS;
+
+    frame = myframe;
+
+    if (-1 == req->rpc_status) {
+        rsp.op_ret = -1;
+        rsp.op_errno = ENOTCONN;
+        goto out;
+    }
+
+    ret = xdr_to_generic(*iov, &rsp, (xdrproc_t)xdr_gfs3_setattr_rsp);
+    if (ret < 0) {
+        gf_smsg(this->name, GF_LOG_ERROR, EINVAL, PC_MSG_XDR_DECODING_FAILED,
+                NULL);
+        rsp.op_ret = -1;
+        rsp.op_errno = EINVAL;
+        goto out;
+    }
 
-        GF_PROTOCOL_DICT_UNSERIALIZE (this, xdata, (rsp.xdata.xdata_val),
-                                      (rsp.xdata.xdata_len), ret,
-                                      rsp.op_errno, out);
+    ret = client_post_setattr(this, &rsp, &prestat, &poststat, &xdata);
 
 out:
-        if (rsp.op_ret == -1) {
-                gf_log (this->name, GF_LOG_WARNING,
-                        "remote operation failed: %s. Path: %s",
-                        strerror (gf_error_to_errno (rsp.op_errno)),
-                        local->loc.path);
-        }
+    if (rsp.op_ret == -1) {
+        gf_smsg(this->name, GF_LOG_WARNING, gf_error_to_errno(rsp.op_errno),
+                PC_MSG_REMOTE_OP_FAILED, NULL);
+    }
+    CLIENT_STACK_UNWIND(setattr, frame, rsp.op_ret,
+                        gf_error_to_errno(rsp.op_errno), &prestat, &poststat,
+                        xdata);
 
-        CLIENT_STACK_UNWIND (create, frame, rsp.op_ret,
-                             gf_error_to_errno (rsp.op_errno), fd, inode,
-                             &stbuf, &preparent, &postparent, xdata);
+    free(rsp.xdata.xdata_val);
 
-        free (rsp.xdata.xdata_val);
+    if (xdata)
+        dict_unref(xdata);
 
-        if (xdata)
-                dict_unref (xdata);
-
-        return 0;
+    return 0;
 }
 
-
 int
-client3_3_rchecksum_cbk (struct rpc_req *req, struct iovec *iov, int count,
-                         void *myframe)
+client3_3_create_cbk(struct rpc_req *req, struct iovec *iov, int count,
+                     void *myframe)
 {
-        call_frame_t *frame = NULL;
-        gfs3_rchecksum_rsp rsp        = {0,};
-        int              ret        = 0;
-        xlator_t *this       = NULL;
-        dict_t  *xdata       = NULL;
-
-
-        this = THIS;
-
-        frame = myframe;
-
-        if (-1 == req->rpc_status) {
-                rsp.op_ret   = -1;
-                rsp.op_errno = ENOTCONN;
-                goto out;
-        }
-
-        ret = xdr_to_generic (*iov, &rsp, (xdrproc_t)xdr_gfs3_rchecksum_rsp);
-        if (ret < 0) {
-                gf_log (this->name, GF_LOG_ERROR, "XDR decoding failed");
-                rsp.op_ret   = -1;
-                rsp.op_errno = EINVAL;
-                goto out;
+    call_frame_t *frame = NULL;
+    fd_t *fd = NULL;
+    inode_t *inode = NULL;
+    struct iatt stbuf = {
+        0,
+    };
+    struct iatt preparent = {
+        0,
+    };
+    struct iatt postparent = {
+        0,
+    };
+    int32_t ret = -1;
+    clnt_local_t *local = NULL;
+    gfs3_create_rsp rsp = {
+        0,
+    };
+    xlator_t *this = NULL;
+    dict_t *xdata = NULL;
+
+    this = THIS;
+
+    frame = myframe;
+    local = frame->local;
+    fd = local->fd;
+    inode = local->loc.inode;
+
+    if (-1 == req->rpc_status) {
+        rsp.op_ret = -1;
+        rsp.op_errno = ENOTCONN;
+        goto out;
+    }
+
+    ret = xdr_to_generic(*iov, &rsp, (xdrproc_t)xdr_gfs3_create_rsp);
+    if (ret < 0) {
+        gf_smsg(this->name, GF_LOG_ERROR, EINVAL, PC_MSG_XDR_DECODING_FAILED,
+                NULL);
+        rsp.op_ret = -1;
+        rsp.op_errno = EINVAL;
+        goto out;
+    }
+
+    if (-1 != rsp.op_ret) {
+        ret = client_post_create(this, &rsp, &stbuf, &preparent, &postparent,
+                                 local, &xdata);
+        if (ret < 0)
+            goto out;
+        ret = client_add_fd_to_saved_fds(frame->this, fd, &local->loc,
+                                         local->flags, rsp.fd, 0);
+        if (ret) {
+            rsp.op_ret = -1;
+            rsp.op_errno = -ret;
+            goto out;
         }
-
-        GF_PROTOCOL_DICT_UNSERIALIZE (this, xdata, (rsp.xdata.xdata_val),
-                                      (rsp.xdata.xdata_len), ret,
-                                      rsp.op_errno, out);
+    }
 
 out:
-        if (rsp.op_ret == -1) {
-                gf_log (this->name, GF_LOG_WARNING, "remote operation failed: %s",
-                        strerror (gf_error_to_errno (rsp.op_errno)));
-        }
-        CLIENT_STACK_UNWIND (rchecksum, frame, rsp.op_ret,
-                             gf_error_to_errno (rsp.op_errno),
-                             rsp.weak_checksum,
-                             (uint8_t *)rsp.strong_checksum.strong_checksum_val,
-                             xdata);
-
-        if (rsp.strong_checksum.strong_checksum_val) {
-                /* This is allocated by the libc while decoding RPC msg */
-                /* Hence no 'GF_FREE', but just 'free' */
-                free (rsp.strong_checksum.strong_checksum_val);
-        }
+    if (rsp.op_ret == -1) {
+        gf_smsg(this->name, GF_LOG_WARNING, gf_error_to_errno(rsp.op_errno),
+                PC_MSG_REMOTE_OP_FAILED, "Path=%s", local->loc.path, NULL);
+    }
 
-        free (rsp.xdata.xdata_val);
+    CLIENT_STACK_UNWIND(create, frame, rsp.op_ret,
+                        gf_error_to_errno(rsp.op_errno), fd, inode, &stbuf,
+                        &preparent, &postparent, xdata);
 
-        if (xdata)
-                dict_unref (xdata);
+    free(rsp.xdata.xdata_val);
 
-        return 0;
+    if (xdata)
+        dict_unref(xdata);
+
+    return 0;
 }
 
 int
-client3_3_lk_cbk (struct rpc_req *req, struct iovec *iov, int count,
-                  void *myframe)
+client3_3_rchecksum_cbk(struct rpc_req *req, struct iovec *iov, int count,
+                        void *myframe)
 {
-        call_frame_t    *frame      = NULL;
-        clnt_local_t  *local      = NULL;
-        struct gf_flock     lock       = {0,};
-        gfs3_lk_rsp      rsp        = {0,};
-        int              ret        = 0;
-        xlator_t *this       = NULL;
-        dict_t  *xdata       = NULL;
-
-        this = THIS;
-
-        frame = myframe;
-        local = frame->local;
-
-        if (-1 == req->rpc_status) {
-                rsp.op_ret   = -1;
-                rsp.op_errno = ENOTCONN;
-                goto out;
-        }
+    call_frame_t *frame = NULL;
+    gfs3_rchecksum_rsp rsp = {
+        0,
+    };
+    int ret = 0;
+    xlator_t *this = NULL;
+    dict_t *xdata = NULL;
 
-        ret = xdr_to_generic (*iov, &rsp, (xdrproc_t)xdr_gfs3_lk_rsp);
-        if (ret < 0) {
-                gf_log (this->name, GF_LOG_ERROR, "XDR decoding failed");
-                rsp.op_ret   = -1;
-                rsp.op_errno = EINVAL;
-                goto out;
-        }
+    this = THIS;
 
-        if (rsp.op_ret >= 0) {
-                gf_proto_flock_to_flock (&rsp.flock, &lock);
-        }
+    frame = myframe;
 
-        /* Save the lock to the client lock cache to be able
-           to recover in the case of server reboot.*/
-        /*
-          temporarily
-        if (local->cmd == F_SETLK || local->cmd == F_SETLKW) {
-                ret = client_add_lock_for_recovery (local->fd, &lock,
-                                                    local->owner, local->cmd);
-                if (ret < 0) {
-                        rsp.op_ret = -1;
-                        rsp.op_errno = -ret;
-                }
-        }
-        */
+    if (-1 == req->rpc_status) {
+        rsp.op_ret = -1;
+        rsp.op_errno = ENOTCONN;
+        goto out;
+    }
+
+    ret = xdr_to_generic(*iov, &rsp, (xdrproc_t)xdr_gfs3_rchecksum_rsp);
+    if (ret < 0) {
+        gf_smsg(this->name, GF_LOG_ERROR, EINVAL, PC_MSG_XDR_DECODING_FAILED,
+                NULL);
+        rsp.op_ret = -1;
+        rsp.op_errno = EINVAL;
+        goto out;
+    }
 
-        GF_PROTOCOL_DICT_UNSERIALIZE (this, xdata, (rsp.xdata.xdata_val),
-                                      (rsp.xdata.xdata_len), ret,
-                                      rsp.op_errno, out);
+    ret = client_post_rchecksum(this, &rsp, &xdata);
 
 out:
-        if ((rsp.op_ret == -1) &&
-            (EAGAIN != gf_error_to_errno (rsp.op_errno))) {
-                gf_log (this->name, GF_LOG_WARNING,
-                        "remote operation failed: %s",
-                        strerror (gf_error_to_errno (rsp.op_errno)));
-        }
-
-        CLIENT_STACK_UNWIND (lk, frame, rsp.op_ret,
-                             gf_error_to_errno (rsp.op_errno), &lock, xdata);
-
-        free (rsp.xdata.xdata_val);
+    if (rsp.op_ret == -1) {
+        gf_smsg(this->name, GF_LOG_WARNING, gf_error_to_errno(rsp.op_errno),
+                PC_MSG_REMOTE_OP_FAILED, NULL);
+    }
+    CLIENT_STACK_UNWIND(rchecksum, frame, rsp.op_ret,
+                        gf_error_to_errno(rsp.op_errno), rsp.weak_checksum,
+                        (uint8_t *)rsp.strong_checksum.strong_checksum_val,
+                        xdata);
+
+    if (rsp.strong_checksum.strong_checksum_val) {
+        /* This is allocated by the libc while decoding RPC msg */
+        /* Hence no 'GF_FREE', but just 'free' */
+        free(rsp.strong_checksum.strong_checksum_val);
+    }
 
-        free (rsp.flock.lk_owner.lk_owner_val);
+    free(rsp.xdata.xdata_val);
 
-        if (xdata)
-                dict_unref (xdata);
+    if (xdata)
+        dict_unref(xdata);
 
-        return 0;
+    return 0;
 }
 
 int
-client3_3_readdir_cbk (struct rpc_req *req, struct iovec *iov, int count,
-                       void *myframe)
+client3_3_lease_cbk(struct rpc_req *req, struct iovec *iov, int count,
+                    void *myframe)
 {
-        call_frame_t     *frame    = NULL;
-        gfs3_readdir_rsp  rsp      = {0,};
-        int32_t           ret      = 0;
-        clnt_local_t     *local    = NULL;
-        gf_dirent_t       entries;
-        xlator_t         *this     = NULL;
-        dict_t           *xdata    = NULL;
-
-        this = THIS;
-
-        frame = myframe;
-        local = frame->local;
-
-        if (-1 == req->rpc_status) {
-                rsp.op_ret   = -1;
-                rsp.op_errno = ENOTCONN;
-                goto out;
-        }
-
-        ret = xdr_to_generic (*iov, &rsp, (xdrproc_t)xdr_gfs3_readdir_rsp);
-        if (ret < 0) {
-                gf_log (this->name, GF_LOG_ERROR, "XDR decoding failed");
-                rsp.op_ret   = -1;
-                rsp.op_errno = EINVAL;
-                goto out;
-        }
-
-        INIT_LIST_HEAD (&entries.list);
-        if (rsp.op_ret > 0) {
-                unserialize_rsp_dirent (this, &rsp, &entries);
-        }
+    call_frame_t *frame = NULL;
+    struct gf_lease lease = {
+        0,
+    };
+    gfs3_lease_rsp rsp = {
+        0,
+    };
+    int ret = 0;
+    xlator_t *this = NULL;
+    dict_t *xdata = NULL;
+
+    this = THIS;
+
+    frame = myframe;
+
+    if (-1 == req->rpc_status) {
+        gf_smsg(this->name, GF_LOG_ERROR, ENOTCONN, PC_MSG_LEASE_FOP_FAILED,
+                NULL);
+        rsp.op_ret = -1;
+        rsp.op_errno = ENOTCONN;
+        goto out;
+    }
+
+    ret = xdr_to_generic(*iov, &rsp, (xdrproc_t)xdr_gfs3_lease_rsp);
+    if (ret < 0) {
+        gf_smsg(this->name, GF_LOG_ERROR, EINVAL, PC_MSG_XDR_DECODING_FAILED,
+                NULL);
+        rsp.op_ret = -1;
+        rsp.op_errno = EINVAL;
+        goto out;
+    }
 
-        GF_PROTOCOL_DICT_UNSERIALIZE (frame->this, xdata,
-                                      (rsp.xdata.xdata_val),
-                                      (rsp.xdata.xdata_len), rsp.op_ret,
-                                      rsp.op_errno, out);
+    ret = client_post_lease(this, &rsp, &lease, &xdata);
 
 out:
-        if (rsp.op_ret == -1) {
-                gf_log (this->name, GF_LOG_WARNING,
-                        "remote operation failed: %s remote_fd = %d",
-                        strerror (gf_error_to_errno (rsp.op_errno)),
-                                  local->cmd);
-        }
-        CLIENT_STACK_UNWIND (readdir, frame, rsp.op_ret,
-                             gf_error_to_errno (rsp.op_errno), &entries, xdata);
-
-        if (rsp.op_ret != -1) {
-                gf_dirent_free (&entries);
-        }
+    if (rsp.op_ret == -1) {
+        gf_smsg(this->name, GF_LOG_WARNING, gf_error_to_errno(rsp.op_errno),
+                PC_MSG_REMOTE_OP_FAILED, NULL);
+    }
 
-        free (rsp.xdata.xdata_val);
+    CLIENT_STACK_UNWIND(lease, frame, rsp.op_ret,
+                        gf_error_to_errno(rsp.op_errno), &lease, xdata);
 
-        if (xdata)
-                dict_unref (xdata);
+    free(rsp.xdata.xdata_val);
 
-        clnt_readdir_rsp_cleanup (&rsp);
+    if (xdata)
+        dict_unref(xdata);
 
-        return 0;
+    return 0;
 }
 
-
 int
-client3_3_readdirp_cbk (struct rpc_req *req, struct iovec *iov, int count,
-                        void *myframe)
+client3_3_lk_cbk(struct rpc_req *req, struct iovec *iov, int count,
+                 void *myframe)
 {
-        call_frame_t      *frame = NULL;
-        gfs3_readdirp_rsp  rsp   = {0,};
-        int32_t            ret   = 0;
-        clnt_local_t      *local = NULL;
-        gf_dirent_t        entries;
-        xlator_t          *this  = NULL;
-        dict_t            *xdata = NULL;
-
-        this = THIS;
-
-        frame = myframe;
-        local = frame->local;
-
-        if (-1 == req->rpc_status) {
-                rsp.op_ret   = -1;
-                rsp.op_errno = ENOTCONN;
-                goto out;
-        }
+    call_frame_t *frame = NULL;
+    struct gf_flock lock = {
+        0,
+    };
+    gfs3_lk_rsp rsp = {
+        0,
+    };
+    int ret = 0;
+    xlator_t *this = NULL;
+    dict_t *xdata = NULL;
+    clnt_local_t *local = NULL;
+
+    this = THIS;
+
+    frame = myframe;
+    local = frame->local;
+
+    if (-1 == req->rpc_status) {
+        rsp.op_ret = -1;
+        rsp.op_errno = ENOTCONN;
+        goto out;
+    }
+
+    ret = xdr_to_generic(*iov, &rsp, (xdrproc_t)xdr_gfs3_lk_rsp);
+    if (ret < 0) {
+        gf_smsg(this->name, GF_LOG_ERROR, EINVAL, PC_MSG_XDR_DECODING_FAILED,
+                NULL);
+        rsp.op_ret = -1;
+        rsp.op_errno = EINVAL;
+        goto out;
+    }
 
-        ret = xdr_to_generic (*iov, &rsp, (xdrproc_t)xdr_gfs3_readdirp_rsp);
-        if (ret < 0) {
-                gf_log (this->name, GF_LOG_ERROR, "XDR decoding failed");
-                rsp.op_ret   = -1;
-                rsp.op_errno = EINVAL;
-                goto out;
-        }
+    if (rsp.op_ret >= 0) {
+        ret = client_post_lk(this, &rsp, &lock, &xdata);
+        if (ret < 0)
+            goto out;
 
-        INIT_LIST_HEAD (&entries.list);
-        if (rsp.op_ret > 0) {
-                unserialize_rsp_direntp (this, local->fd, &rsp, &entries);
-        }
+        /* Save the lock to the client lock cache to be able
+           to recover in the case of server reboot.*/
 
-        GF_PROTOCOL_DICT_UNSERIALIZE (this, xdata, (rsp.xdata.xdata_val),
-                                      (rsp.xdata.xdata_len), ret,
-                                      rsp.op_errno, out);
+        if (client_is_setlk(local->cmd)) {
+            ret = client_add_lock_for_recovery(local->fd, &lock, &local->owner,
+                                               local->cmd);
+            if (ret < 0) {
+                rsp.op_ret = -1;
+                rsp.op_errno = -ret;
+            }
+        }
+    }
 
 out:
-        if (rsp.op_ret == -1) {
-                gf_log (this->name, GF_LOG_WARNING,
-                        "remote operation failed: %s",
-                        strerror (gf_error_to_errno (rsp.op_errno)));
-        }
-        CLIENT_STACK_UNWIND (readdirp, frame, rsp.op_ret,
-                             gf_error_to_errno (rsp.op_errno), &entries, xdata);
+    if ((rsp.op_ret == -1) && (EAGAIN != gf_error_to_errno(rsp.op_errno))) {
+        gf_smsg(this->name, GF_LOG_WARNING, gf_error_to_errno(rsp.op_errno),
+                PC_MSG_REMOTE_OP_FAILED, NULL);
+    }
 
-        if (rsp.op_ret != -1) {
-                gf_dirent_free (&entries);
-        }
-        free (rsp.xdata.xdata_val);
+    CLIENT_STACK_UNWIND(lk, frame, rsp.op_ret, gf_error_to_errno(rsp.op_errno),
+                        &lock, xdata);
 
-        if (xdata)
-                dict_unref (xdata);
+    free(rsp.xdata.xdata_val);
 
-        clnt_readdirp_rsp_cleanup (&rsp);
+    free(rsp.flock.lk_owner.lk_owner_val);
 
-        return 0;
-}
+    if (xdata)
+        dict_unref(xdata);
 
+    return 0;
+}
 
 int
-client3_3_rename_cbk (struct rpc_req *req, struct iovec *iov, int count,
+client3_3_readdir_cbk(struct rpc_req *req, struct iovec *iov, int count,
                       void *myframe)
 {
-        call_frame_t     *frame      = NULL;
-        gfs3_rename_rsp   rsp        = {0,};
-        struct iatt       stbuf      = {0,};
-        struct iatt       preoldparent  = {0,};
-        struct iatt       postoldparent = {0,};
-        struct iatt       prenewparent  = {0,};
-        struct iatt       postnewparent = {0,};
-        int               ret        = 0;
-        xlator_t *this       = NULL;
-        dict_t  *xdata       = NULL;
-
-        this = THIS;
-
-        frame = myframe;
-
-        if (-1 == req->rpc_status) {
-                rsp.op_ret   = -1;
-                rsp.op_errno = ENOTCONN;
-                goto out;
-        }
+    call_frame_t *frame = NULL;
+    gfs3_readdir_rsp rsp = {
+        0,
+    };
+    int32_t ret = 0;
+    clnt_local_t *local = NULL;
+    gf_dirent_t entries;
+    xlator_t *this = NULL;
+    dict_t *xdata = NULL;
 
-        ret = xdr_to_generic (*iov, &rsp, (xdrproc_t)xdr_gfs3_rename_rsp);
-        if (ret < 0) {
-                gf_log (this->name, GF_LOG_ERROR, "XDR decoding failed");
-                rsp.op_ret   = -1;
-                rsp.op_errno = EINVAL;
-                goto out;
-        }
+    this = THIS;
 
-        if (-1 != rsp.op_ret) {
-                gf_stat_to_iatt (&rsp.stat, &stbuf);
+    frame = myframe;
+    local = frame->local;
 
-                gf_stat_to_iatt (&rsp.preoldparent, &preoldparent);
-                gf_stat_to_iatt (&rsp.postoldparent, &postoldparent);
+    INIT_LIST_HEAD(&entries.list);
 
-                gf_stat_to_iatt (&rsp.prenewparent, &prenewparent);
-                gf_stat_to_iatt (&rsp.postnewparent, &postnewparent);
-        }
+    if (-1 == req->rpc_status) {
+        rsp.op_ret = -1;
+        rsp.op_errno = ENOTCONN;
+        goto out;
+    }
+
+    ret = xdr_to_generic(*iov, &rsp, (xdrproc_t)xdr_gfs3_readdir_rsp);
+    if (ret < 0) {
+        gf_smsg(this->name, GF_LOG_ERROR, EINVAL, PC_MSG_XDR_DECODING_FAILED,
+                NULL);
+        rsp.op_ret = -1;
+        rsp.op_errno = EINVAL;
+        goto out;
+    }
 
-        GF_PROTOCOL_DICT_UNSERIALIZE (this, xdata, (rsp.xdata.xdata_val),
-                                      (rsp.xdata.xdata_len), ret,
-                                      rsp.op_errno, out);
+    ret = client_post_readdir(this, &rsp, &entries, &xdata);
 
 out:
-        if (rsp.op_ret == -1) {
-                gf_log (this->name, GF_LOG_WARNING, "remote operation failed: %s",
-                        strerror (gf_error_to_errno (rsp.op_errno)));
-        }
-        CLIENT_STACK_UNWIND (rename, frame, rsp.op_ret,
-                             gf_error_to_errno (rsp.op_errno),
-                             &stbuf, &preoldparent, &postoldparent,
-                             &prenewparent, &postnewparent, xdata);
+    if (rsp.op_ret == -1) {
+        gf_smsg(this->name, GF_LOG_WARNING, gf_error_to_errno(rsp.op_errno),
+                PC_MSG_REMOTE_OP_FAILED, "remote_fd=%d", local->cmd, NULL);
+    }
+    CLIENT_STACK_UNWIND(readdir, frame, rsp.op_ret,
+                        gf_error_to_errno(rsp.op_errno), &entries, xdata);
 
-        free (rsp.xdata.xdata_val);
+    if (rsp.op_ret != -1) {
+        gf_dirent_free(&entries);
+    }
 
-        if (xdata)
-                dict_unref (xdata);
+    free(rsp.xdata.xdata_val);
+
+    if (xdata)
+        dict_unref(xdata);
 
-        return 0;
+    clnt_readdir_rsp_cleanup(&rsp);
+
+    return 0;
 }
 
 int
-client3_3_link_cbk (struct rpc_req *req, struct iovec *iov, int count,
-                    void *myframe)
+client3_3_readdirp_cbk(struct rpc_req *req, struct iovec *iov, int count,
+                       void *myframe)
 {
-        call_frame_t     *frame      = NULL;
-        gfs3_link_rsp     rsp        = {0,};
-        struct iatt       stbuf      = {0,};
-        struct iatt       preparent  = {0,};
-        struct iatt       postparent = {0,};
-        int               ret        = 0;
-        clnt_local_t     *local      = NULL;
-        inode_t          *inode      = NULL;
-        xlator_t *this       = NULL;
-        dict_t  *xdata       = NULL;
+    call_frame_t *frame = NULL;
+    gfs3_readdirp_rsp rsp = {
+        0,
+    };
+    int32_t ret = 0;
+    clnt_local_t *local = NULL;
+    gf_dirent_t entries;
+    xlator_t *this = NULL;
+    dict_t *xdata = NULL;
 
+    this = THIS;
 
-        this = THIS;
+    frame = myframe;
+    local = frame->local;
 
-        frame = myframe;
+    INIT_LIST_HEAD(&entries.list);
 
-        local = frame->local;
-        inode = local->loc.inode;
-
-        if (-1 == req->rpc_status) {
-                rsp.op_ret   = -1;
-                rsp.op_errno = ENOTCONN;
-                goto out;
-        }
-
-        ret = xdr_to_generic (*iov, &rsp, (xdrproc_t)xdr_gfs3_link_rsp);
-        if (ret < 0) {
-                gf_log (this->name, GF_LOG_ERROR, "XDR decoding failed");
-                rsp.op_ret   = -1;
-                rsp.op_errno = EINVAL;
-                goto out;
-        }
-
-        if (-1 != rsp.op_ret) {
-                gf_stat_to_iatt (&rsp.stat, &stbuf);
-
-                gf_stat_to_iatt (&rsp.preparent, &preparent);
-                gf_stat_to_iatt (&rsp.postparent, &postparent);
-        }
-
-        GF_PROTOCOL_DICT_UNSERIALIZE (this, xdata, (rsp.xdata.xdata_val),
-                                      (rsp.xdata.xdata_len), ret,
-                                      rsp.op_errno, out);
+    if (-1 == req->rpc_status) {
+        rsp.op_ret = -1;
+        rsp.op_errno = ENOTCONN;
+        goto out;
+    }
+
+    ret = xdr_to_generic(*iov, &rsp, (xdrproc_t)xdr_gfs3_readdirp_rsp);
+    if (ret < 0) {
+        gf_smsg(this->name, GF_LOG_ERROR, EINVAL, PC_MSG_XDR_DECODING_FAILED,
+                NULL);
+        rsp.op_ret = -1;
+        rsp.op_errno = EINVAL;
+        goto out;
+    }
 
+    ret = client_post_readdirp(this, &rsp, local->fd, &entries, &xdata);
 out:
-        if (rsp.op_ret == -1) {
-                gf_log (this->name, GF_LOG_WARNING,
-                        "remote operation failed: %s (%s -> %s)",
-                        strerror (gf_error_to_errno (rsp.op_errno)),
-                        local->loc.path, local->loc2.path);
-        }
+    if (rsp.op_ret == -1) {
+        gf_smsg(this->name, GF_LOG_WARNING, gf_error_to_errno(rsp.op_errno),
+                PC_MSG_REMOTE_OP_FAILED, NULL);
+    }
+    CLIENT_STACK_UNWIND(readdirp, frame, rsp.op_ret,
+                        gf_error_to_errno(rsp.op_errno), &entries, xdata);
 
-        CLIENT_STACK_UNWIND (link, frame, rsp.op_ret,
-                             gf_error_to_errno (rsp.op_errno), inode,
-                             &stbuf, &preparent, &postparent, xdata);
+    if (rsp.op_ret != -1) {
+        gf_dirent_free(&entries);
+    }
+    free(rsp.xdata.xdata_val);
 
-        free (rsp.xdata.xdata_val);
+    if (xdata)
+        dict_unref(xdata);
 
-        if (xdata)
-                dict_unref (xdata);
+    clnt_readdirp_rsp_cleanup(&rsp);
 
-        return 0;
+    return 0;
 }
 
-
 int
-client3_3_opendir_cbk (struct rpc_req *req, struct iovec *iov, int count,
-                       void *myframe)
+client3_3_rename_cbk(struct rpc_req *req, struct iovec *iov, int count,
+                     void *myframe)
 {
-        clnt_local_t      *local = NULL;
-        clnt_conf_t       *conf = NULL;
-        call_frame_t      *frame = NULL;
-        fd_t              *fd = NULL;
-        int ret = 0;
-        gfs3_opendir_rsp  rsp = {0,};
-        xlator_t *this       = NULL;
-        dict_t  *xdata       = NULL;
-
-        this = THIS;
-
-        frame = myframe;
-        local = frame->local;
-
-        conf  = frame->this->private;
-        fd    = local->fd;
-
-        if (-1 == req->rpc_status) {
-                rsp.op_ret   = -1;
-                rsp.op_errno = ENOTCONN;
-                goto out;
-        }
-
-        ret = xdr_to_generic (*iov, &rsp, (xdrproc_t)xdr_gfs3_opendir_rsp);
-        if (ret < 0) {
-                gf_log (this->name, GF_LOG_ERROR, "XDR decoding failed");
-                rsp.op_ret   = -1;
-                rsp.op_errno = EINVAL;
-                goto out;
-        }
-
-        if (-1 != rsp.op_ret) {
-                ret = client_add_fd_to_saved_fds (frame->this, fd, &local->loc,
-                                                  0, rsp.fd, 1);
-                if (ret) {
-                        rsp.op_ret = -1;
-                        rsp.op_errno = -ret;
-                        goto out;
-                }
-        }
+    call_frame_t *frame = NULL;
+    gfs3_rename_rsp rsp = {
+        0,
+    };
+    struct iatt stbuf = {
+        0,
+    };
+    struct iatt preoldparent = {
+        0,
+    };
+    struct iatt postoldparent = {
+        0,
+    };
+    struct iatt prenewparent = {
+        0,
+    };
+    struct iatt postnewparent = {
+        0,
+    };
+    int ret = 0;
+    xlator_t *this = NULL;
+    dict_t *xdata = NULL;
+
+    this = THIS;
+
+    frame = myframe;
+
+    if (-1 == req->rpc_status) {
+        rsp.op_ret = -1;
+        rsp.op_errno = ENOTCONN;
+        goto out;
+    }
+
+    ret = xdr_to_generic(*iov, &rsp, (xdrproc_t)xdr_gfs3_rename_rsp);
+    if (ret < 0) {
+        gf_smsg(this->name, GF_LOG_ERROR, EINVAL, PC_MSG_XDR_DECODING_FAILED,
+                NULL);
+        rsp.op_ret = -1;
+        rsp.op_errno = EINVAL;
+        goto out;
+    }
 
-        GF_PROTOCOL_DICT_UNSERIALIZE (this, xdata, (rsp.xdata.xdata_val),
-                                      (rsp.xdata.xdata_len), ret,
-                                      rsp.op_errno, out);
+    ret = client_post_rename(this, &rsp, &stbuf, &preoldparent, &postoldparent,
+                             &prenewparent, &postnewparent, &xdata);
 
 out:
-        if (rsp.op_ret == -1) {
-                gf_log (this->name, fop_log_level (GF_FOP_OPENDIR,
-                                        gf_error_to_errno (rsp.op_errno)),
-                        "remote operation failed: %s. Path: %s (%s)",
-                        strerror (gf_error_to_errno (rsp.op_errno)),
-                        local->loc.path, loc_gfid_utoa (&local->loc));
-        }
-        CLIENT_STACK_UNWIND (opendir, frame, rsp.op_ret,
-                             gf_error_to_errno (rsp.op_errno), fd, xdata);
+    if (rsp.op_ret == -1) {
+        gf_smsg(this->name, GF_LOG_WARNING, gf_error_to_errno(rsp.op_errno),
+                PC_MSG_REMOTE_OP_FAILED, NULL);
+    }
+    CLIENT_STACK_UNWIND(rename, frame, rsp.op_ret,
+                        gf_error_to_errno(rsp.op_errno), &stbuf, &preoldparent,
+                        &postoldparent, &prenewparent, &postnewparent, xdata);
 
-        free (rsp.xdata.xdata_val);
+    free(rsp.xdata.xdata_val);
 
-        if (xdata)
-                dict_unref (xdata);
+    if (xdata)
+        dict_unref(xdata);
 
-        return 0;
+    return 0;
 }
 
-
 int
-client3_3_lookup_cbk (struct rpc_req *req, struct iovec *iov, int count,
-                      void *myframe)
+client3_3_link_cbk(struct rpc_req *req, struct iovec *iov, int count,
+                   void *myframe)
 {
-        clnt_local_t    *local      = NULL;
-        call_frame_t    *frame      = NULL;
-        int              ret        = 0;
-        gfs3_lookup_rsp  rsp        = {0,};
-        struct iatt      stbuf      = {0,};
-        struct iatt      postparent = {0,};
-        int              op_errno   = EINVAL;
-        dict_t          *xdata      = NULL;
-        inode_t         *inode      = NULL;
-        xlator_t        *this       = NULL;
-
-        this = THIS;
-
-        frame = myframe;
-        local = frame->local;
-        inode = local->loc.inode;
-
-        if (-1 == req->rpc_status) {
-                rsp.op_ret   = -1;
-                op_errno = ENOTCONN;
-                goto out;
-        }
+    call_frame_t *frame = NULL;
+    gfs3_link_rsp rsp = {
+        0,
+    };
+    struct iatt stbuf = {
+        0,
+    };
+    struct iatt preparent = {
+        0,
+    };
+    struct iatt postparent = {
+        0,
+    };
+    int ret = 0;
+    clnt_local_t *local = NULL;
+    inode_t *inode = NULL;
+    xlator_t *this = NULL;
+    dict_t *xdata = NULL;
+
+    this = THIS;
+
+    frame = myframe;
+
+    local = frame->local;
+    inode = local->loc.inode;
+
+    if (-1 == req->rpc_status) {
+        rsp.op_ret = -1;
+        rsp.op_errno = ENOTCONN;
+        goto out;
+    }
+
+    ret = xdr_to_generic(*iov, &rsp, (xdrproc_t)xdr_gfs3_link_rsp);
+    if (ret < 0) {
+        gf_smsg(this->name, GF_LOG_ERROR, EINVAL, PC_MSG_XDR_DECODING_FAILED,
+                NULL);
+        rsp.op_ret = -1;
+        rsp.op_errno = EINVAL;
+        goto out;
+    }
 
-        ret = xdr_to_generic (*iov, &rsp, (xdrproc_t)xdr_gfs3_lookup_rsp);
-        if (ret < 0) {
-                gf_log (this->name, GF_LOG_ERROR, "XDR decoding failed");
-                rsp.op_ret   = -1;
-                op_errno = EINVAL;
-                goto out;
+    ret = client_post_link(this, &rsp, &stbuf, &preparent, &postparent, &xdata);
+out:
+    if (rsp.op_ret == -1) {
+        if (GF_IGNORE_IF_GSYNCD_SAFE_ERROR(frame, rsp.op_errno)) {
+            gf_smsg(this->name, GF_LOG_WARNING, gf_error_to_errno(rsp.op_errno),
+                    PC_MSG_REMOTE_OP_FAILED, "loc1=%s", local->loc.path,
+                    "loc2=%s", local->loc2.path, NULL);
         }
+    }
 
-        op_errno = gf_error_to_errno (rsp.op_errno);
-        gf_stat_to_iatt (&rsp.postparent, &postparent);
+    CLIENT_STACK_UNWIND(link, frame, rsp.op_ret,
+                        gf_error_to_errno(rsp.op_errno), inode, &stbuf,
+                        &preparent, &postparent, xdata);
 
-        if (rsp.op_ret == -1)
-                goto out;
+    free(rsp.xdata.xdata_val);
 
-        rsp.op_ret = -1;
-        gf_stat_to_iatt (&rsp.stat, &stbuf);
+    if (xdata)
+        dict_unref(xdata);
 
-        GF_PROTOCOL_DICT_UNSERIALIZE (frame->this, xdata, (rsp.xdata.xdata_val),
-                                      (rsp.xdata.xdata_len), rsp.op_ret,
-                                      op_errno, out);
+    return 0;
+}
 
-        if ((!gf_uuid_is_null (inode->gfid))
-            && (gf_uuid_compare (stbuf.ia_gfid, inode->gfid) != 0)) {
-                gf_log (frame->this->name, GF_LOG_DEBUG,
-                        "gfid changed for %s", local->loc.path);
+int
+client3_3_opendir_cbk(struct rpc_req *req, struct iovec *iov, int count,
+                      void *myframe)
+{
+    clnt_local_t *local = NULL;
+    call_frame_t *frame = NULL;
+    fd_t *fd = NULL;
+    int ret = 0;
+    gfs3_opendir_rsp rsp = {
+        0,
+    };
+    xlator_t *this = NULL;
+    dict_t *xdata = NULL;
 
-                rsp.op_ret = -1;
-                op_errno = ESTALE;
-                if (xdata)
-                        ret = dict_set_int32 (xdata, "gfid-changed", 1);
+    this = THIS;
 
-                goto out;
-        }
+    frame = myframe;
+    local = frame->local;
 
-        rsp.op_ret = 0;
+    fd = local->fd;
 
-out:
-        rsp.op_errno = op_errno;
-        if (rsp.op_ret == -1) {
-                /* any error other than ENOENT */
-                if (!(local->loc.name && rsp.op_errno == ENOENT) &&
-		    !(rsp.op_errno == ESTALE))
-                        gf_log (this->name, GF_LOG_WARNING,
-                                "remote operation failed: %s. Path: %s (%s)",
-                                strerror (rsp.op_errno), local->loc.path,
-                                loc_gfid_utoa (&local->loc));
-                else
-                        gf_log (this->name, GF_LOG_TRACE, "not found on remote node");
+    if (-1 == req->rpc_status) {
+        rsp.op_ret = -1;
+        rsp.op_errno = ENOTCONN;
+        goto out;
+    }
+
+    ret = xdr_to_generic(*iov, &rsp, (xdrproc_t)xdr_gfs3_opendir_rsp);
+    if (ret < 0) {
+        gf_smsg(this->name, GF_LOG_ERROR, EINVAL, PC_MSG_XDR_DECODING_FAILED,
+                NULL);
+        rsp.op_ret = -1;
+        rsp.op_errno = EINVAL;
+        goto out;
+    }
 
+    if (-1 != rsp.op_ret) {
+        ret = client_add_fd_to_saved_fds(frame->this, fd, &local->loc, 0,
+                                         rsp.fd, 1);
+        if (ret) {
+            rsp.op_ret = -1;
+            rsp.op_errno = -ret;
+            goto out;
         }
+    }
 
-        CLIENT_STACK_UNWIND (lookup, frame, rsp.op_ret, rsp.op_errno, inode,
-                             &stbuf, xdata, &postparent);
-
-        if (xdata)
-                dict_unref (xdata);
-
-        free (rsp.xdata.xdata_val);
-
-        return 0;
+    ret = client_post_opendir(this, &rsp, &xdata);
+out:
+    if (rsp.op_ret == -1) {
+        gf_smsg(this->name,
+                fop_log_level(GF_FOP_OPENDIR, gf_error_to_errno(rsp.op_errno)),
+                gf_error_to_errno(rsp.op_errno), PC_MSG_REMOTE_OP_FAILED,
+                "Path=%s", local->loc.path, "gfid=%s",
+                loc_gfid_utoa(&local->loc), NULL);
+    }
+    CLIENT_STACK_UNWIND(opendir, frame, rsp.op_ret,
+                        gf_error_to_errno(rsp.op_errno), fd, xdata);
+
+    free(rsp.xdata.xdata_val);
+
+    if (xdata)
+        dict_unref(xdata);
+
+    return 0;
 }
 
 int
-client3_3_readv_cbk (struct rpc_req *req, struct iovec *iov, int count,
+client3_3_lookup_cbk(struct rpc_req *req, struct iovec *iov, int count,
                      void *myframe)
 {
-        call_frame_t   *frame  = NULL;
-        struct iobref  *iobref = NULL;
-        struct iovec    vector[MAX_IOVEC] = {{0}, };
-        struct iatt     stat   = {0,};
-        gfs3_read_rsp   rsp    = {0,};
-        int             ret    = 0, rspcount = 0;
-        clnt_local_t   *local  = NULL;
-        xlator_t *this       = NULL;
-        dict_t  *xdata       = NULL;
-
-        this = THIS;
-
-        memset (vector, 0, sizeof (vector));
-
-        frame = myframe;
-        local = frame->local;
-
-        if (-1 == req->rpc_status) {
-                rsp.op_ret   = -1;
-                rsp.op_errno = ENOTCONN;
-                goto out;
-        }
+    clnt_local_t *local = NULL;
+    call_frame_t *frame = NULL;
+    int ret = 0;
+    gfs3_lookup_rsp rsp = {
+        0,
+    };
+    struct iatt stbuf = {
+        0,
+    };
+    struct iatt postparent = {
+        0,
+    };
+    int op_errno = EINVAL;
+    dict_t *xdata = NULL;
+    inode_t *inode = NULL;
+    xlator_t *this = NULL;
+
+    this = THIS;
+
+    frame = myframe;
+    local = frame->local;
+    inode = local->loc.inode;
+
+    if (-1 == req->rpc_status) {
+        rsp.op_ret = -1;
+        op_errno = ENOTCONN;
+        goto out;
+    }
+
+    ret = xdr_to_generic(*iov, &rsp, (xdrproc_t)xdr_gfs3_lookup_rsp);
+    if (ret < 0) {
+        gf_smsg(this->name, GF_LOG_ERROR, EINVAL, PC_MSG_XDR_DECODING_FAILED,
+                NULL);
+        rsp.op_ret = -1;
+        op_errno = EINVAL;
+        goto out;
+    }
+
+    /* Preserve the op_errno received from the server */
+    op_errno = gf_error_to_errno(rsp.op_errno);
+
+    ret = client_post_lookup(this, &rsp, &stbuf, &postparent, &xdata);
+    if (ret < 0) {
+        /* Don't change the op_errno if the fop failed on server */
+        if (rsp.op_ret == 0)
+            op_errno = rsp.op_errno;
+        rsp.op_ret = -1;
+        goto out;
+    }
 
-        ret = xdr_to_generic (*iov, &rsp, (xdrproc_t)xdr_gfs3_read_rsp);
-        if (ret < 0) {
-                gf_log (this->name, GF_LOG_ERROR, "XDR decoding failed");
-                rsp.op_ret   = -1;
-                rsp.op_errno = EINVAL;
-                goto out;
-        }
+    if (rsp.op_ret < 0)
+        goto out;
 
-        if (rsp.op_ret != -1) {
-                iobref = req->rsp_iobref;
-                gf_stat_to_iatt (&rsp.stat, &stat);
+    if ((!gf_uuid_is_null(inode->gfid)) &&
+        (gf_uuid_compare(stbuf.ia_gfid, inode->gfid) != 0)) {
+        gf_msg_debug(frame->this->name, 0, "gfid changed for %s",
+                     local->loc.path);
 
-                vector[0].iov_len = rsp.op_ret;
-                if (rsp.op_ret > 0)
-                        vector[0].iov_base = req->rsp[1].iov_base;
-                rspcount = 1;
-        }
-        GF_PROTOCOL_DICT_UNSERIALIZE (this, xdata, (rsp.xdata.xdata_val),
-                                      (rsp.xdata.xdata_len), ret,
-                                      rsp.op_errno, out);
+        rsp.op_ret = -1;
+        op_errno = ESTALE;
+        if (xdata)
+            ret = dict_set_int32(xdata, "gfid-changed", 1);
+
+        goto out;
+    }
 
-#ifdef GF_TESTING_IO_XDATA
-        dict_dump_to_log (xdata);
-#endif
+    rsp.op_ret = 0;
 
 out:
-        if (rsp.op_ret == -1) {
-                gf_log (this->name, GF_LOG_WARNING,
-                        "remote operation failed: %s",
-                        strerror (gf_error_to_errno (rsp.op_errno)));
-        } else if (rsp.op_ret >= 0) {
-                if (local->attempt_reopen)
-                        client_attempt_reopen (local->fd, this);
-        }
-        CLIENT_STACK_UNWIND (readv, frame, rsp.op_ret,
-                             gf_error_to_errno (rsp.op_errno), vector, rspcount,
-                             &stat, iobref, xdata);
+    /* Restore the correct op_errno to rsp.op_errno */
+    rsp.op_errno = op_errno;
+    if (rsp.op_ret == -1) {
+        /* any error other than ENOENT */
+        if (!(local->loc.name && rsp.op_errno == ENOENT) &&
+            !(rsp.op_errno == ESTALE))
+            gf_smsg(this->name, GF_LOG_WARNING, rsp.op_errno,
+                    PC_MSG_REMOTE_OP_FAILED, "Path=%s", local->loc.path,
+                    "gfid=%s", loc_gfid_utoa(&local->loc), NULL);
+        else
+            gf_msg_trace(this->name, 0,
+                         "not found on remote "
+                         "node");
+    }
 
-        free (rsp.xdata.xdata_val);
+    CLIENT_STACK_UNWIND(lookup, frame, rsp.op_ret, rsp.op_errno, inode, &stbuf,
+                        xdata, &postparent);
 
-        if (xdata)
-                dict_unref (xdata);
+    if (xdata)
+        dict_unref(xdata);
 
-        return 0;
+    free(rsp.xdata.xdata_val);
+
+    return 0;
 }
 
 int
-client3_3_release_cbk (struct rpc_req *req, struct iovec *iov, int count,
-                       void *myframe)
+client3_3_readv_cbk(struct rpc_req *req, struct iovec *iov, int count,
+                    void *myframe)
 {
-        call_frame_t   *frame = NULL;
+    call_frame_t *frame = NULL;
+    struct iobref *iobref = NULL;
+    struct iovec vector[MAX_IOVEC] = {
+        {0},
+    };
+    struct iatt stat = {
+        0,
+    };
+    gfs3_read_rsp rsp = {
+        0,
+    };
+    int ret = 0, rspcount = 0;
+    clnt_local_t *local = NULL;
+    xlator_t *this = NULL;
+    dict_t *xdata = NULL;
+
+    this = THIS;
+
+    frame = myframe;
+    local = frame->local;
+
+    if (-1 == req->rpc_status) {
+        rsp.op_ret = -1;
+        rsp.op_errno = ENOTCONN;
+        goto out;
+    }
+
+    ret = xdr_to_generic(*iov, &rsp, (xdrproc_t)xdr_gfs3_read_rsp);
+    if (ret < 0) {
+        gf_smsg(this->name, GF_LOG_ERROR, EINVAL, PC_MSG_XDR_DECODING_FAILED,
+                NULL);
+        rsp.op_ret = -1;
+        rsp.op_errno = EINVAL;
+        goto out;
+    }
+
+    memset(vector, 0, sizeof(vector));
 
-        frame = myframe;
-        STACK_DESTROY (frame->root);
-        return 0;
+    ret = client_post_readv(this, &rsp, &iobref, req->rsp_iobref, &stat, vector,
+                            &req->rsp[1], &rspcount, &xdata);
+out:
+    if (rsp.op_ret == -1) {
+        gf_smsg(this->name, GF_LOG_WARNING, gf_error_to_errno(rsp.op_errno),
+                PC_MSG_REMOTE_OP_FAILED, NULL);
+    } else if (rsp.op_ret >= 0) {
+        if (local->attempt_reopen)
+            client_attempt_reopen(local->fd, this);
+    }
+    CLIENT_STACK_UNWIND(readv, frame, rsp.op_ret,
+                        gf_error_to_errno(rsp.op_errno), vector, rspcount,
+                        &stat, iobref, xdata);
+
+    free(rsp.xdata.xdata_val);
+
+    if (xdata)
+        dict_unref(xdata);
+
+    return 0;
 }
+
 int
-client3_3_releasedir_cbk (struct rpc_req *req, struct iovec *iov, int count,
-                          void *myframe)
+client3_3_release_cbk(struct rpc_req *req, struct iovec *iov, int count,
+                      void *myframe)
 {
-        call_frame_t   *frame = NULL;
+    call_frame_t *frame = NULL;
 
-        frame = myframe;
-        STACK_DESTROY (frame->root);
-        return 0;
+    frame = myframe;
+    STACK_DESTROY(frame->root);
+    return 0;
 }
-
 int
-client_fdctx_destroy (xlator_t *this, clnt_fd_ctx_t *fdctx)
+client3_3_releasedir_cbk(struct rpc_req *req, struct iovec *iov, int count,
+                         void *myframe)
 {
-        clnt_conf_t  *conf        = NULL;
-        call_frame_t *fr          = NULL;
-        int32_t       ret         = -1;
-        char          parent_down = 0;
-        fd_lk_ctx_t  *lk_ctx      = NULL;
+    call_frame_t *frame = NULL;
 
-        GF_VALIDATE_OR_GOTO ("client", this, out);
-        GF_VALIDATE_OR_GOTO (this->name, fdctx, out);
+    frame = myframe;
+    STACK_DESTROY(frame->root);
+    return 0;
+}
 
-        conf = (clnt_conf_t *) this->private;
+static int
+client3_3_getactivelk_cbk(struct rpc_req *req, struct iovec *iov, int count,
+                          void *myframe)
+{
+    call_frame_t *frame = NULL;
+    gfs3_getactivelk_rsp rsp = {
+        0,
+    };
+    int32_t ret = 0;
+    lock_migration_info_t locklist;
+    xlator_t *this = NULL;
+    dict_t *xdata = NULL;
 
-        if (fdctx->remote_fd == -1) {
-                gf_log (this->name, GF_LOG_DEBUG, "not a valid fd");
-                goto out;
-        }
+    this = THIS;
 
-        pthread_mutex_lock (&conf->lock);
-        {
-                parent_down   = conf->parent_down;
-                lk_ctx        = fdctx->lk_ctx;
-                fdctx->lk_ctx = NULL;
-        }
-        pthread_mutex_unlock (&conf->lock);
+    frame = myframe;
 
-        if (lk_ctx)
-                fd_lk_ctx_unref (lk_ctx);
+    if (-1 == req->rpc_status) {
+        rsp.op_ret = -1;
+        rsp.op_errno = ENOTCONN;
+        goto out;
+    }
+
+    ret = xdr_to_generic(*iov, &rsp, (xdrproc_t)xdr_gfs3_getactivelk_rsp);
+    if (ret < 0) {
+        gf_smsg(this->name, GF_LOG_ERROR, EINVAL, PC_MSG_XDR_DECODING_FAILED,
+                NULL);
+        rsp.op_ret = -1;
+        rsp.op_errno = EINVAL;
+        goto out;
+    }
 
-        if (!parent_down)
-                rpc_clnt_ref (conf->rpc);
-        else
-                goto out;
+    INIT_LIST_HEAD(&locklist.list);
 
-        fr = create_frame (this, this->ctx->pool);
-        if (fr == NULL) {
-                goto out;
-        }
+    if (rsp.op_ret > 0) {
+        clnt_unserialize_rsp_locklist(this, &rsp, &locklist);
+    }
 
-        ret = 0;
-
-        if (fdctx->is_dir) {
-                gfs3_releasedir_req  req = {{0,},};
-                req.fd = fdctx->remote_fd;
-                gf_log (this->name, GF_LOG_TRACE, "sending releasedir on fd");
-                client_submit_request (this, &req, fr, &clnt3_3_fop_prog,
-                                       GFS3_OP_RELEASEDIR,
-                                       client3_3_releasedir_cbk,
-                                       NULL, NULL, 0, NULL, 0, NULL,
-                                       (xdrproc_t)xdr_gfs3_releasedir_req);
-        } else {
-                gfs3_release_req  req = {{0,},};
-                req.fd = fdctx->remote_fd;
-                gf_log (this->name, GF_LOG_TRACE, "sending release on fd");
-                client_submit_request (this, &req, fr, &clnt3_3_fop_prog,
-                                       GFS3_OP_RELEASE,
-                                       client3_3_release_cbk, NULL,
-                                       NULL, 0, NULL, 0, NULL,
-                                       (xdrproc_t)xdr_gfs3_release_req);
-        }
+    GF_PROTOCOL_DICT_UNSERIALIZE(this, xdata, (rsp.xdata.xdata_val),
+                                 (rsp.xdata.xdata_len), ret, rsp.op_errno, out);
 
-        rpc_clnt_unref (conf->rpc);
 out:
-        if (fdctx) {
-                fdctx->remote_fd = -1;
-                GF_FREE (fdctx);
-        }
+    if (rsp.op_ret == -1) {
+        gf_smsg(this->name, GF_LOG_WARNING, gf_error_to_errno(rsp.op_errno),
+                PC_MSG_REMOTE_OP_FAILED, NULL);
+    }
+
+    CLIENT_STACK_UNWIND(getactivelk, frame, rsp.op_ret,
+                        gf_error_to_errno(rsp.op_errno), &locklist, xdata);
+
+    free(rsp.xdata.xdata_val);
 
-        return ret;
+    if (xdata)
+        dict_unref(xdata);
+
+    clnt_getactivelk_rsp_cleanup(&rsp);
+
+    return 0;
 }
 
-int32_t
-client3_3_releasedir (call_frame_t *frame, xlator_t *this,
-                      void *data)
+static int
+client3_3_setactivelk_cbk(struct rpc_req *req, struct iovec *iov, int count,
+                          void *myframe)
 {
-        clnt_conf_t         *conf        = NULL;
-        clnt_fd_ctx_t       *fdctx       = NULL;
-        clnt_args_t         *args        = NULL;
-        int64_t              remote_fd   = -1;
+    call_frame_t *frame = NULL;
+    gfs3_getactivelk_rsp rsp = {
+        0,
+    };
+    int32_t ret = 0;
+    xlator_t *this = NULL;
+    dict_t *xdata = NULL;
 
-        if (!this || !data)
-                goto out;
+    this = THIS;
 
-        args = data;
-        conf = this->private;
+    frame = myframe;
 
-        pthread_mutex_lock (&conf->lock);
-        {
-                fdctx = this_fd_del_ctx (args->fd, this);
-                if (fdctx != NULL) {
-                        remote_fd = fdctx->remote_fd;
+    if (-1 == req->rpc_status) {
+        rsp.op_ret = -1;
+        rsp.op_errno = ENOTCONN;
+        goto out;
+    }
+
+    ret = xdr_to_generic(*iov, &rsp, (xdrproc_t)xdr_gfs3_setactivelk_rsp);
+    if (ret < 0) {
+        gf_smsg(this->name, GF_LOG_ERROR, EINVAL, PC_MSG_XDR_DECODING_FAILED,
+                NULL);
+        rsp.op_ret = -1;
+        rsp.op_errno = EINVAL;
+        goto out;
+    }
 
-                        /* fdctx->remote_fd == -1 indicates a reopen attempt
-                           in progress. Just mark ->released = 1 and let
-                           reopen_cbk handle releasing
-                        */
+    GF_PROTOCOL_DICT_UNSERIALIZE(this, xdata, (rsp.xdata.xdata_val),
+                                 (rsp.xdata.xdata_len), ret, rsp.op_errno, out);
 
-                        if (remote_fd != -1)
-                                list_del_init (&fdctx->sfd_pos);
+out:
+    if (rsp.op_ret == -1) {
+        gf_smsg(this->name, GF_LOG_WARNING, gf_error_to_errno(rsp.op_errno),
+                PC_MSG_REMOTE_OP_FAILED, NULL);
+    }
 
-                        fdctx->released = 1;
-                }
-        }
-        pthread_mutex_unlock (&conf->lock);
+    CLIENT_STACK_UNWIND(setactivelk, frame, rsp.op_ret,
+                        gf_error_to_errno(rsp.op_errno), xdata);
 
-        if (remote_fd != -1)
-                client_fdctx_destroy (this, fdctx);
-out:
+    free(rsp.xdata.xdata_val);
+
+    if (xdata)
+        dict_unref(xdata);
 
-        return 0;
+    return 0;
 }
 
 int32_t
-client3_3_release (call_frame_t *frame, xlator_t *this,
-                   void *data)
+client3_3_releasedir(call_frame_t *frame, xlator_t *this, void *data)
 {
-        int64_t           remote_fd     = -1;
-        clnt_conf_t      *conf          = NULL;
-        clnt_fd_ctx_t    *fdctx         = NULL;
-        clnt_args_t      *args          = NULL;
-        lk_heal_state_t   lk_heal_state = GF_LK_HEAL_DONE;
-
-        if (!this || !data)
-                goto out;
+    clnt_conf_t *conf = NULL;
+    clnt_fd_ctx_t *fdctx = NULL;
+    clnt_args_t *args = NULL;
+    int64_t remote_fd = -1;
+    gf_boolean_t destroy = _gf_false;
+
+    if (!this || !data)
+        goto out;
+
+    args = data;
+    conf = this->private;
+
+    pthread_spin_lock(&conf->fd_lock);
+    {
+        fdctx = this_fd_del_ctx(args->fd, this);
+        if (fdctx != NULL) {
+            remote_fd = fdctx->remote_fd;
+
+            /* fdctx->remote_fd == -1 indicates a reopen attempt
+               in progress. Just mark ->released = 1 and let
+               reopen_cbk handle releasing
+            */
+
+            if (remote_fd == -1) {
+                fdctx->released = 1;
+            } else {
+                list_del_init(&fdctx->sfd_pos);
+                destroy = _gf_true;
+            }
+        }
+    }
+    pthread_spin_unlock(&conf->fd_lock);
+
+    if (destroy)
+        client_fdctx_destroy(this, fdctx);
 
-        args = data;
-        conf = this->private;
+out:
 
-        pthread_mutex_lock (&conf->lock);
-        {
-                fdctx = this_fd_del_ctx (args->fd, this);
-                if (fdctx != NULL) {
-                        remote_fd     = fdctx->remote_fd;
-                        lk_heal_state = fdctx->lk_heal_state;
-
-                        /* fdctx->remote_fd == -1 indicates a reopen attempt
-                           in progress. Just mark ->released = 1 and let
-                           reopen_cbk handle releasing
-                        */
-
-                        if (remote_fd != -1 &&
-                            lk_heal_state == GF_LK_HEAL_DONE)
-                                list_del_init (&fdctx->sfd_pos);
-
-                        fdctx->released = 1;
-                }
-        }
-        pthread_mutex_unlock (&conf->lock);
+    return 0;
+}
 
-        if (remote_fd != -1 && lk_heal_state == GF_LK_HEAL_DONE)
-                client_fdctx_destroy (this, fdctx);
+int32_t
+client3_3_release(call_frame_t *frame, xlator_t *this, void *data)
+{
+    int64_t remote_fd = -1;
+    clnt_conf_t *conf = NULL;
+    clnt_fd_ctx_t *fdctx = NULL;
+    clnt_args_t *args = NULL;
+    gf_boolean_t destroy = _gf_false;
+
+    if (!this || !data)
+        goto out;
+
+    args = data;
+    conf = this->private;
+
+    pthread_spin_lock(&conf->fd_lock);
+    {
+        fdctx = this_fd_del_ctx(args->fd, this);
+        if (fdctx != NULL) {
+            remote_fd = fdctx->remote_fd;
+
+            /* fdctx->remote_fd == -1 indicates a reopen attempt
+               in progress. Just mark ->released = 1 and let
+               reopen_cbk handle releasing
+            */
+            if (remote_fd == -1) {
+                fdctx->released = 1;
+            } else {
+                list_del_init(&fdctx->sfd_pos);
+                destroy = _gf_true;
+            }
+        }
+    }
+    pthread_spin_unlock(&conf->fd_lock);
+
+    if (destroy)
+        client_fdctx_destroy(this, fdctx);
 out:
-        return 0;
+    return 0;
 }
 
-
 int32_t
-client3_3_lookup (call_frame_t *frame, xlator_t *this,
-                  void *data)
+client3_3_lookup(call_frame_t *frame, xlator_t *this, void *data)
 {
-        clnt_conf_t     *conf              = NULL;
-        clnt_local_t    *local             = NULL;
-        clnt_args_t     *args              = NULL;
-        gfs3_lookup_req  req               = {{0,},};
-        int              ret               = 0;
-        int              op_errno          = ESTALE;
-        data_t          *content           = NULL;
-        struct iovec     vector[MAX_IOVEC] = {{0}, };
-        int              count             = 0;
-        struct iobref   *rsp_iobref        = NULL;
-        struct iobuf    *rsp_iobuf         = NULL;
-        struct iovec    *rsphdr            = NULL;
-
-        if (!frame || !this || !data)
-                goto unwind;
-
-        memset (vector, 0, sizeof (vector));
-
-        conf = this->private;
-        args = data;
-        local = mem_get0 (this->local_pool);
-        if (!local) {
-                op_errno = ENOMEM;
+    clnt_conf_t *conf = NULL;
+    clnt_local_t *local = NULL;
+    clnt_args_t *args = NULL;
+    gfs3_lookup_req req = {
+        {
+            0,
+        },
+    };
+    int ret = 0;
+    int op_errno = ESTALE;
+    data_t *content = NULL;
+    struct iovec vector[MAX_IOVEC] = {
+        {0},
+    };
+    int count = 0;
+    struct iobref *rsp_iobref = NULL;
+    struct iobuf *rsp_iobuf = NULL;
+    struct iovec *rsphdr = NULL;
+    client_payload_t cp;
+
+    if (!frame || !this || !data)
+        goto unwind;
+
+    conf = this->private;
+    args = data;
+    local = mem_get0(this->local_pool);
+    if (!local) {
+        op_errno = ENOMEM;
+        goto unwind;
+    }
+    frame->local = local;
+
+    if (!(args->loc && args->loc->inode))
+        goto unwind;
+
+    loc_copy(&local->loc, args->loc);
+    loc_path(&local->loc, NULL);
+
+    if (args->xdata) {
+        content = dict_get_sizen(args->xdata, GF_CONTENT_KEY);
+        if (content != NULL) {
+            rsp_iobref = iobref_new();
+            if (rsp_iobref == NULL) {
                 goto unwind;
-        }
-        frame->local = local;
+            }
 
-        if (!(args->loc && args->loc->inode))
+            /* TODO: what is the size we should send ? */
+            /* This change very much depends on quick-read
+               changes */
+            rsp_iobuf = iobuf_get(this->ctx->iobuf_pool);
+            if (rsp_iobuf == NULL) {
                 goto unwind;
-
-        loc_copy (&local->loc, args->loc);
-        loc_path (&local->loc, NULL);
-
-        if (args->loc->parent) {
-                if (!gf_uuid_is_null (args->loc->parent->gfid))
-                        memcpy (req.pargfid, args->loc->parent->gfid, 16);
-                else
-                        memcpy (req.pargfid, args->loc->pargfid, 16);
-        } else {
-                if (!gf_uuid_is_null (args->loc->inode->gfid))
-                        memcpy (req.gfid, args->loc->inode->gfid, 16);
-                else
-                        memcpy (req.gfid, args->loc->gfid, 16);
-        }
-
-        if (args->xdata) {
-                content = dict_get (args->xdata, GF_CONTENT_KEY);
-                if (content != NULL) {
-                        rsp_iobref = iobref_new ();
-                        if (rsp_iobref == NULL) {
-                                goto unwind;
-                        }
-
-                        /* TODO: what is the size we should send ? */
-                        /* This change very much depends on quick-read
-                           changes */
-                        rsp_iobuf = iobuf_get (this->ctx->iobuf_pool);
-                        if (rsp_iobuf == NULL) {
-                                goto unwind;
-                        }
-
-                        iobref_add (rsp_iobref, rsp_iobuf);
-                        iobuf_unref (rsp_iobuf);
-                        rsphdr = &vector[0];
-                        rsphdr->iov_base = iobuf_ptr (rsp_iobuf);
-                        rsphdr->iov_len = iobuf_pagesize (rsp_iobuf);
-                        count = 1;
-                        local->iobref = rsp_iobref;
-                        rsp_iobuf = NULL;
-                        rsp_iobref = NULL;
-                }
-
-                GF_PROTOCOL_DICT_SERIALIZE (this, args->xdata,
-                                            (&req.xdata.xdata_val),
-                                            req.xdata.xdata_len,
-                                            op_errno, unwind);
-        }
-
-        if (args->loc->name)
-                req.bname = (char *)args->loc->name;
-        else
-                req.bname = "";
-
-        ret = client_submit_request (this, &req, frame, conf->fops,
-                                     GFS3_OP_LOOKUP, client3_3_lookup_cbk,
-                                     NULL, rsphdr, count,
-                                     NULL, 0, local->iobref,
-                                     (xdrproc_t)xdr_gfs3_lookup_req);
-
-        if (ret) {
-                gf_log (this->name, GF_LOG_WARNING, "failed to send the fop");
-        }
-
-        GF_FREE (req.xdata.xdata_val);
-
-        return 0;
+            }
+
+            iobref_add(rsp_iobref, rsp_iobuf);
+            memset(vector, 0, sizeof(vector));
+            rsphdr = &vector[0];
+            rsphdr->iov_base = iobuf_ptr(rsp_iobuf);
+            rsphdr->iov_len = iobuf_pagesize(rsp_iobuf);
+            count = 1;
+            local->iobref = rsp_iobref;
+            iobuf_unref(rsp_iobuf);
+            rsp_iobuf = NULL;
+            rsp_iobref = NULL;
+        }
+    }
+
+    ret = client_pre_lookup(this, &req, args->loc, args->xdata);
+    if (ret) {
+        op_errno = -ret;
+        goto unwind;
+    }
+
+    memset(&cp, 0, sizeof(client_payload_t));
+    cp.rsphdr = rsphdr;
+    cp.rsphdr_cnt = count;
+    cp.rsp_iobref = local->iobref;
+    ret = client_submit_request(this, &req, frame, conf->fops, GFS3_OP_LOOKUP,
+                                client3_3_lookup_cbk, &cp,
+                                (xdrproc_t)xdr_gfs3_lookup_req);
+
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_WARNING, 0, PC_MSG_FOP_SEND_FAILED, NULL);
+    }
+
+    GF_FREE(req.xdata.xdata_val);
+
+    return 0;
 
 unwind:
-        CLIENT_STACK_UNWIND (lookup, frame, -1, op_errno, NULL, NULL, NULL,
-                             NULL);
-
-        GF_FREE (req.xdata.xdata_val);
+    CLIENT_STACK_UNWIND(lookup, frame, -1, op_errno, NULL, NULL, NULL, NULL);
 
-        if (rsp_iobref)
-                iobref_unref (rsp_iobref);
+    GF_FREE(req.xdata.xdata_val);
 
-        if (rsp_iobuf)
-                iobuf_unref (rsp_iobuf);
+    if (rsp_iobref)
+        iobref_unref(rsp_iobref);
 
-        return 0;
+    return 0;
 }
 
 int32_t
-client3_3_stat (call_frame_t *frame, xlator_t *this,
-                void *data)
+client3_3_stat(call_frame_t *frame, xlator_t *this, void *data)
 {
-        clnt_conf_t   *conf     = NULL;
-        clnt_args_t   *args     = NULL;
-        gfs3_stat_req  req      = {{0,},};
-        int            ret      = 0;
-        int            op_errno = ESTALE;
-
-        if (!frame || !this || !data)
-                goto unwind;
-
-        args = data;
-        if (!(args->loc && args->loc->inode))
-                goto unwind;
-
-        if (!gf_uuid_is_null (args->loc->inode->gfid))
-                memcpy (req.gfid,  args->loc->inode->gfid, 16);
-        else
-                memcpy (req.gfid, args->loc->gfid, 16);
-
-        GF_ASSERT_AND_GOTO_WITH_ERROR (this->name,
-                                       !gf_uuid_is_null (*((uuid_t*)req.gfid)),
-                                       unwind, op_errno, EINVAL);
-        conf = this->private;
-
-        GF_PROTOCOL_DICT_SERIALIZE (this, args->xdata, (&req.xdata.xdata_val),
-                                    req.xdata.xdata_len, op_errno, unwind);
-
-        ret = client_submit_request (this, &req, frame, conf->fops,
-                                     GFS3_OP_STAT, client3_3_stat_cbk, NULL,
-                                     NULL, 0, NULL, 0, NULL,
-                                     (xdrproc_t)xdr_gfs3_stat_req);
-        if (ret) {
-                gf_log (this->name, GF_LOG_WARNING, "failed to send the fop");
-        }
-
-        GF_FREE (req.xdata.xdata_val);
-
-        return 0;
+    clnt_conf_t *conf = NULL;
+    clnt_args_t *args = NULL;
+    gfs3_stat_req req = {
+        {
+            0,
+        },
+    };
+    int ret = 0;
+    int op_errno = ESTALE;
+
+    if (!frame || !this || !data)
+        goto unwind;
+
+    args = data;
+    conf = this->private;
+
+    ret = client_pre_stat(this, &req, args->loc, args->xdata);
+    if (ret) {
+        op_errno = -ret;
+        goto unwind;
+    }
+    ret = client_submit_request(this, &req, frame, conf->fops, GFS3_OP_STAT,
+                                client3_3_stat_cbk, NULL,
+                                (xdrproc_t)xdr_gfs3_stat_req);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_WARNING, 0, PC_MSG_FOP_SEND_FAILED, NULL);
+    }
+
+    GF_FREE(req.xdata.xdata_val);
+
+    return 0;
 unwind:
-        CLIENT_STACK_UNWIND (stat, frame, -1, op_errno, NULL, NULL);
+    CLIENT_STACK_UNWIND(stat, frame, -1, op_errno, NULL, NULL);
 
-        GF_FREE (req.xdata.xdata_val);
+    GF_FREE(req.xdata.xdata_val);
 
-        return 0;
+    return 0;
 }
 
-
 int32_t
-client3_3_truncate (call_frame_t *frame, xlator_t *this,
-                    void *data)
+client3_3_truncate(call_frame_t *frame, xlator_t *this, void *data)
 {
-        clnt_conf_t       *conf     = NULL;
-        clnt_args_t       *args     = NULL;
-        gfs3_truncate_req  req      = {{0,},};
-        int                ret      = 0;
-        int                op_errno = ESTALE;
-
-        if (!frame || !this || !data)
-                goto unwind;
-
-        args = data;
-
-        if (!(args->loc && args->loc->inode))
-                goto unwind;
-
-        if (!gf_uuid_is_null (args->loc->inode->gfid))
-                memcpy (req.gfid,  args->loc->inode->gfid, 16);
-        else
-                memcpy (req.gfid, args->loc->gfid, 16);
-
-        GF_ASSERT_AND_GOTO_WITH_ERROR (this->name,
-                                       !gf_uuid_is_null (*((uuid_t*)req.gfid)),
-                                       unwind, op_errno, EINVAL);
-        req.offset = args->offset;
-
-        conf = this->private;
-
-        GF_PROTOCOL_DICT_SERIALIZE (this, args->xdata, (&req.xdata.xdata_val),
-                                    req.xdata.xdata_len, op_errno, unwind);
-
-        ret = client_submit_request (this, &req, frame, conf->fops,
-                                     GFS3_OP_TRUNCATE,
-                                     client3_3_truncate_cbk, NULL,
-                                     NULL, 0, NULL, 0,
-                                     NULL, (xdrproc_t)xdr_gfs3_truncate_req);
-        if (ret) {
-                gf_log (this->name, GF_LOG_WARNING, "failed to send the fop");
-        }
-
-        GF_FREE (req.xdata.xdata_val);
-
-        return 0;
+    clnt_conf_t *conf = NULL;
+    clnt_args_t *args = NULL;
+    gfs3_truncate_req req = {
+        {
+            0,
+        },
+    };
+    int ret = 0;
+    int op_errno = ESTALE;
+
+    if (!frame || !this || !data)
+        goto unwind;
+
+    args = data;
+    conf = this->private;
+
+    ret = client_pre_truncate(this, &req, args->loc, args->offset, args->xdata);
+    if (ret) {
+        op_errno = -ret;
+        goto unwind;
+    }
+    ret = client_submit_request(this, &req, frame, conf->fops, GFS3_OP_TRUNCATE,
+                                client3_3_truncate_cbk, NULL,
+                                (xdrproc_t)xdr_gfs3_truncate_req);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_WARNING, 0, PC_MSG_FOP_SEND_FAILED, NULL);
+    }
+
+    GF_FREE(req.xdata.xdata_val);
+
+    return 0;
 unwind:
-        CLIENT_STACK_UNWIND (truncate, frame, -1, op_errno, NULL, NULL, NULL);
-        GF_FREE (req.xdata.xdata_val);
+    CLIENT_STACK_UNWIND(truncate, frame, -1, op_errno, NULL, NULL, NULL);
+    GF_FREE(req.xdata.xdata_val);
 
-        return 0;
+    return 0;
 }
 
-
 int32_t
-client3_3_ftruncate (call_frame_t *frame, xlator_t *this,
-                     void *data)
+client3_3_ftruncate(call_frame_t *frame, xlator_t *this, void *data)
 {
-        clnt_args_t        *args     = NULL;
-        int64_t             remote_fd = -1;
-        clnt_conf_t        *conf     = NULL;
-        gfs3_ftruncate_req  req      = {{0,},};
-        int                 op_errno = EINVAL;
-        int                 ret      = 0;
-
-        if (!frame || !this || !data)
-                goto unwind;
-
-        args = data;
-
-        conf = this->private;
-
-        CLIENT_GET_REMOTE_FD (this, args->fd, DEFAULT_REMOTE_FD,
-                              remote_fd, op_errno, unwind);
-
-        req.offset = args->offset;
-        req.fd     = remote_fd;
-        memcpy (req.gfid, args->fd->inode->gfid, 16);
-
-        GF_PROTOCOL_DICT_SERIALIZE (this, args->xdata, (&req.xdata.xdata_val),
-                                    req.xdata.xdata_len, op_errno, unwind);
-
-        ret = client_submit_request (this, &req, frame, conf->fops,
-                                     GFS3_OP_FTRUNCATE,
-                                     client3_3_ftruncate_cbk, NULL,
-                                     NULL, 0, NULL, 0,
-                                     NULL, (xdrproc_t)xdr_gfs3_ftruncate_req);
-        if (ret) {
-                gf_log (this->name, GF_LOG_WARNING, "failed to send the fop");
-        }
-
-        GF_FREE (req.xdata.xdata_val);
-
-        return 0;
+    clnt_args_t *args = NULL;
+    clnt_conf_t *conf = NULL;
+    gfs3_ftruncate_req req = {
+        {
+            0,
+        },
+    };
+    int op_errno = EINVAL;
+    int ret = 0;
+
+    if (!frame || !this || !data)
+        goto unwind;
+
+    args = data;
+
+    conf = this->private;
+
+    ret = client_pre_ftruncate(this, &req, args->fd, args->offset, args->xdata);
+    if (ret) {
+        op_errno = -ret;
+        goto unwind;
+    }
+    ret = client_submit_request(this, &req, frame, conf->fops,
+                                GFS3_OP_FTRUNCATE, client3_3_ftruncate_cbk,
+                                NULL, (xdrproc_t)xdr_gfs3_ftruncate_req);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_WARNING, 0, PC_MSG_FOP_SEND_FAILED, NULL);
+    }
+
+    GF_FREE(req.xdata.xdata_val);
+
+    return 0;
 unwind:
-        CLIENT_STACK_UNWIND (ftruncate, frame, -1, op_errno, NULL, NULL, NULL);
-        GF_FREE (req.xdata.xdata_val);
+    CLIENT_STACK_UNWIND(ftruncate, frame, -1, op_errno, NULL, NULL, NULL);
+    GF_FREE(req.xdata.xdata_val);
 
-        return 0;
+    return 0;
 }
 
-
-
 int32_t
-client3_3_access (call_frame_t *frame, xlator_t *this,
-                  void *data)
+client3_3_access(call_frame_t *frame, xlator_t *this, void *data)
 {
-        clnt_conf_t     *conf     = NULL;
-        clnt_args_t     *args     = NULL;
-        gfs3_access_req  req      = {{0,},};
-        int              ret      = 0;
-        int              op_errno = ESTALE;
-
-        if (!frame || !this || !data)
-                goto unwind;
-
-        args = data;
-
-        if (!(args->loc && args->loc->inode))
-                goto unwind;
-
-        if (!gf_uuid_is_null (args->loc->inode->gfid))
-                memcpy (req.gfid,  args->loc->inode->gfid, 16);
-        else
-                memcpy (req.gfid, args->loc->gfid, 16);
-
-        GF_ASSERT_AND_GOTO_WITH_ERROR (this->name,
-                                       !gf_uuid_is_null (*((uuid_t*)req.gfid)),
-                                       unwind, op_errno, EINVAL);
-        req.mask = args->mask;
-
-        conf = this->private;
-
-        GF_PROTOCOL_DICT_SERIALIZE (this, args->xdata, (&req.xdata.xdata_val),
-                                    req.xdata.xdata_len, op_errno, unwind);
-
-        ret = client_submit_request (this, &req, frame, conf->fops,
-                                     GFS3_OP_ACCESS,
-                                     client3_3_access_cbk, NULL,
-                                     NULL, 0, NULL, 0,
-                                     NULL, (xdrproc_t)xdr_gfs3_access_req);
-        if (ret) {
-                gf_log (this->name, GF_LOG_WARNING, "failed to send the fop");
-        }
-
-        GF_FREE (req.xdata.xdata_val);
-
-        return 0;
+    clnt_conf_t *conf = NULL;
+    clnt_args_t *args = NULL;
+    gfs3_access_req req = {
+        {
+            0,
+        },
+    };
+    int ret = 0;
+    int op_errno = ESTALE;
+
+    if (!frame || !this || !data)
+        goto unwind;
+
+    args = data;
+
+    conf = this->private;
+
+    ret = client_pre_access(this, &req, args->loc, args->mask, args->xdata);
+    if (ret) {
+        op_errno = -ret;
+        goto unwind;
+    }
+    ret = client_submit_request(this, &req, frame, conf->fops, GFS3_OP_ACCESS,
+                                client3_3_access_cbk, NULL,
+                                (xdrproc_t)xdr_gfs3_access_req);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_WARNING, 0, PC_MSG_FOP_SEND_FAILED, NULL);
+    }
+
+    GF_FREE(req.xdata.xdata_val);
+
+    return 0;
 unwind:
-        CLIENT_STACK_UNWIND (access, frame, -1, op_errno, NULL);
-        GF_FREE (req.xdata.xdata_val);
+    CLIENT_STACK_UNWIND(access, frame, -1, op_errno, NULL);
+    GF_FREE(req.xdata.xdata_val);
 
-        return 0;
+    return 0;
 }
 
 int32_t
-client3_3_readlink (call_frame_t *frame, xlator_t *this,
-                    void *data)
+client3_3_readlink(call_frame_t *frame, xlator_t *this, void *data)
 {
-        clnt_conf_t       *conf              = NULL;
-        clnt_args_t       *args              = NULL;
-        gfs3_readlink_req  req               = {{0,},};
-        int                ret               = 0;
-        int                op_errno          = ESTALE;
-        clnt_local_t      *local             = NULL;
-        struct iobuf      *rsp_iobuf         = NULL;
-        struct iobref     *rsp_iobref        = NULL;
-        struct iovec      *rsphdr            = NULL;
-        int                count             = 0;
-        struct iovec       vector[MAX_IOVEC] = {{0}, };
-
-        if (!frame || !this || !data)
-                goto unwind;
-
-        args = data;
-
-        if (!(args->loc && args->loc->inode))
-                goto unwind;
-
-        if (!gf_uuid_is_null (args->loc->inode->gfid))
-                memcpy (req.gfid,  args->loc->inode->gfid, 16);
-        else
-                memcpy (req.gfid, args->loc->gfid, 16);
-
-        GF_ASSERT_AND_GOTO_WITH_ERROR (this->name,
-                                       !gf_uuid_is_null (*((uuid_t*)req.gfid)),
-                                       unwind, op_errno, EINVAL);
-        req.size = args->size;
-        conf = this->private;
-
-        local = mem_get0 (this->local_pool);
-        if (!local) {
-                op_errno = ENOMEM;
-                goto unwind;
-        }
-
-        frame->local = local;
-
-        GF_PROTOCOL_DICT_SERIALIZE (this, args->xdata, (&req.xdata.xdata_val),
-                                    req.xdata.xdata_len, op_errno, unwind);
-
-        rsp_iobref = iobref_new ();
-        if (rsp_iobref == NULL) {
-                goto unwind;
-        }
-
-        rsp_iobuf = iobuf_get (this->ctx->iobuf_pool);
-        if (rsp_iobuf == NULL) {
-                goto unwind;
-        }
-
-        iobref_add (rsp_iobref, rsp_iobuf);
-        iobuf_unref (rsp_iobuf);
-        rsphdr = &vector[0];
-        rsphdr->iov_base = iobuf_ptr (rsp_iobuf);
-        rsphdr->iov_len = iobuf_pagesize (rsp_iobuf);
-        count = 1;
-        local->iobref = rsp_iobref;
-        rsp_iobuf = NULL;
-        rsp_iobref = NULL;
-
-        ret = client_submit_request (this, &req, frame, conf->fops,
-                                     GFS3_OP_READLINK,
-                                     client3_3_readlink_cbk, NULL,
-                                     rsphdr, count, NULL, 0,
-                                     local->iobref,
-                                     (xdrproc_t)xdr_gfs3_readlink_req);
-        if (ret) {
-                gf_log (this->name, GF_LOG_WARNING, "failed to send the fop");
-        }
-
-        GF_FREE (req.xdata.xdata_val);
-
-        return 0;
+    clnt_conf_t *conf = NULL;
+    clnt_args_t *args = NULL;
+    gfs3_readlink_req req = {
+        {
+            0,
+        },
+    };
+    int ret = 0;
+    int op_errno = ESTALE;
+    clnt_local_t *local = NULL;
+    struct iobuf *rsp_iobuf = NULL;
+    struct iobref *rsp_iobref = NULL;
+    struct iovec *rsphdr = NULL;
+    int count = 0;
+    struct iovec vector[MAX_IOVEC] = {
+        {0},
+    };
+    client_payload_t cp;
+
+    if (!frame || !this || !data)
+        goto unwind;
+
+    args = data;
+
+    conf = this->private;
+
+    local = mem_get0(this->local_pool);
+    if (!local) {
+        op_errno = ENOMEM;
+        goto unwind;
+    }
+
+    frame->local = local;
+
+    ret = client_pre_readlink(this, &req, args->loc, args->size, args->xdata);
+    if (ret) {
+        op_errno = -ret;
+        goto unwind;
+    }
+
+    rsp_iobref = iobref_new();
+    if (rsp_iobref == NULL) {
+        goto unwind;
+    }
+
+    rsp_iobuf = iobuf_get(this->ctx->iobuf_pool);
+    if (rsp_iobuf == NULL) {
+        goto unwind;
+    }
+
+    rsphdr = &vector[0];
+    rsphdr->iov_base = iobuf_ptr(rsp_iobuf);
+    rsphdr->iov_len = iobuf_pagesize(rsp_iobuf);
+    count = 1;
+    local->iobref = rsp_iobref;
+    iobref_add(rsp_iobref, rsp_iobuf);
+    iobuf_unref(rsp_iobuf);
+    rsp_iobuf = NULL;
+    rsp_iobref = NULL;
+
+    memset(&cp, 0, sizeof(client_payload_t));
+    cp.rsphdr = rsphdr;
+    cp.rsphdr_cnt = count;
+    cp.rsp_iobref = local->iobref;
+    ret = client_submit_request(this, &req, frame, conf->fops, GFS3_OP_READLINK,
+                                client3_3_readlink_cbk, &cp,
+                                (xdrproc_t)xdr_gfs3_readlink_req);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_WARNING, 0, PC_MSG_FOP_SEND_FAILED, NULL);
+    }
+
+    GF_FREE(req.xdata.xdata_val);
+
+    return 0;
 unwind:
-        if (rsp_iobref != NULL) {
-                iobref_unref (rsp_iobref);
-        }
+    if (rsp_iobref != NULL) {
+        iobref_unref(rsp_iobref);
+    }
 
-        CLIENT_STACK_UNWIND (readlink, frame, -1, op_errno, NULL, NULL, NULL);
-        GF_FREE (req.xdata.xdata_val);
+    CLIENT_STACK_UNWIND(readlink, frame, -1, op_errno, NULL, NULL, NULL);
+    GF_FREE(req.xdata.xdata_val);
 
-        return 0;
+    return 0;
 }
 
-
-
-
 int32_t
-client3_3_unlink (call_frame_t *frame, xlator_t *this,
-                  void *data)
+client3_3_unlink(call_frame_t *frame, xlator_t *this, void *data)
 {
-        clnt_conf_t     *conf     = NULL;
-        clnt_args_t     *args     = NULL;
-        gfs3_unlink_req  req      = {{0,},};
-        int              ret      = 0;
-        int              op_errno = 0;
-
-        if (!frame || !this || !data)
-                goto unwind;
-
-        args = data;
-
-        if (!(args->loc && args->loc->parent))
-                goto unwind;
-
-        if (!gf_uuid_is_null (args->loc->parent->gfid))
-                memcpy (req.pargfid,  args->loc->parent->gfid, 16);
-        else
-                memcpy (req.pargfid, args->loc->pargfid, 16);
-
-        GF_ASSERT_AND_GOTO_WITH_ERROR (this->name,
-                                       !gf_uuid_is_null (*((uuid_t*)req.pargfid)),
-                                       unwind, op_errno, EINVAL);
-        req.bname = (char *)args->loc->name;
-        req.xflags = args->flags;
-        conf = this->private;
-
-        GF_PROTOCOL_DICT_SERIALIZE (this, args->xdata, (&req.xdata.xdata_val),
-                                    req.xdata.xdata_len, op_errno, unwind);
-
-        ret = client_submit_request (this, &req, frame, conf->fops,
-                                     GFS3_OP_UNLINK,
-                                     client3_3_unlink_cbk, NULL,
-                                     NULL, 0, NULL, 0,
-                                     NULL, (xdrproc_t)xdr_gfs3_unlink_req);
-        if (ret) {
-                gf_log (this->name, GF_LOG_WARNING, "failed to send the fop");
-        }
-
-        GF_FREE (req.xdata.xdata_val);
-
-        return 0;
+    clnt_conf_t *conf = NULL;
+    clnt_args_t *args = NULL;
+    gfs3_unlink_req req = {
+        {
+            0,
+        },
+    };
+    int ret = 0;
+    int op_errno = 0;
+
+    if (!frame || !this || !data)
+        goto unwind;
+
+    args = data;
+    conf = this->private;
+
+    ret = client_pre_unlink(this, &req, args->loc, args->flags, args->xdata);
+    if (ret) {
+        op_errno = -ret;
+        goto unwind;
+    }
+    ret = client_submit_request(this, &req, frame, conf->fops, GFS3_OP_UNLINK,
+                                client3_3_unlink_cbk, NULL,
+                                (xdrproc_t)xdr_gfs3_unlink_req);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_WARNING, 0, PC_MSG_FOP_SEND_FAILED, NULL);
+    }
+
+    GF_FREE(req.xdata.xdata_val);
+
+    return 0;
 unwind:
-        CLIENT_STACK_UNWIND (unlink, frame, -1, op_errno, NULL, NULL, NULL);
-        GF_FREE (req.xdata.xdata_val);
+    CLIENT_STACK_UNWIND(unlink, frame, -1, op_errno, NULL, NULL, NULL);
+    GF_FREE(req.xdata.xdata_val);
 
-        return 0;
+    return 0;
 }
 
-
-
 int32_t
-client3_3_rmdir (call_frame_t *frame, xlator_t *this,
-                 void *data)
+client3_3_rmdir(call_frame_t *frame, xlator_t *this, void *data)
 {
-        clnt_conf_t    *conf     = NULL;
-        clnt_args_t    *args     = NULL;
-        gfs3_rmdir_req  req      = {{0,},};
-        int             ret      = 0;
-        int             op_errno = ESTALE;
-
-        if (!frame || !this || !data)
-                goto unwind;
-
-        args = data;
-
-        if (!(args->loc && args->loc->parent))
-                goto unwind;
-
-        if (!gf_uuid_is_null (args->loc->parent->gfid))
-                memcpy (req.pargfid,  args->loc->parent->gfid, 16);
-        else
-                memcpy (req.pargfid, args->loc->pargfid, 16);
-
-        GF_ASSERT_AND_GOTO_WITH_ERROR (this->name,
-                                       !gf_uuid_is_null (*((uuid_t*)req.pargfid)),
-                                       unwind, op_errno, EINVAL);
-        req.bname = (char *)args->loc->name;
-        req.xflags = args->flags;
-        conf = this->private;
-
-        GF_PROTOCOL_DICT_SERIALIZE (this, args->xdata, (&req.xdata.xdata_val),
-                                    req.xdata.xdata_len, op_errno, unwind);
-
-        ret = client_submit_request (this, &req, frame, conf->fops,
-                                     GFS3_OP_RMDIR, client3_3_rmdir_cbk, NULL,
-                                     NULL, 0, NULL, 0,
-                                     NULL, (xdrproc_t)xdr_gfs3_rmdir_req);
-        if (ret) {
-                gf_log (this->name, GF_LOG_WARNING, "failed to send the fop");
-        }
-        GF_FREE (req.xdata.xdata_val);
-
-        return 0;
+    clnt_conf_t *conf = NULL;
+    clnt_args_t *args = NULL;
+    gfs3_rmdir_req req = {
+        {
+            0,
+        },
+    };
+    int ret = 0;
+    int op_errno = ESTALE;
+
+    if (!frame || !this || !data)
+        goto unwind;
+
+    args = data;
+    conf = this->private;
+
+    ret = client_pre_rmdir(this, &req, args->loc, args->flags, args->xdata);
+
+    if (ret) {
+        op_errno = -ret;
+        goto unwind;
+    }
+    ret = client_submit_request(this, &req, frame, conf->fops, GFS3_OP_RMDIR,
+                                client3_3_rmdir_cbk, NULL,
+                                (xdrproc_t)xdr_gfs3_rmdir_req);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_WARNING, 0, PC_MSG_FOP_SEND_FAILED, NULL);
+    }
+    GF_FREE(req.xdata.xdata_val);
+
+    return 0;
 unwind:
-        CLIENT_STACK_UNWIND (rmdir, frame, -1, op_errno, NULL, NULL, NULL);
-        GF_FREE (req.xdata.xdata_val);
+    CLIENT_STACK_UNWIND(rmdir, frame, -1, op_errno, NULL, NULL, NULL);
+    GF_FREE(req.xdata.xdata_val);
 
-        return 0;
+    return 0;
 }
 
-
-
 int32_t
-client3_3_symlink (call_frame_t *frame, xlator_t *this,
-                   void *data)
+client3_3_symlink(call_frame_t *frame, xlator_t *this, void *data)
 {
-        clnt_local_t     *local    = NULL;
-        clnt_conf_t      *conf     = NULL;
-        clnt_args_t      *args     = NULL;
-        gfs3_symlink_req  req      = {{0,},};
-        int               ret      = 0;
-        int               op_errno = ESTALE;
-
-        if (!frame || !this || !data)
-                goto unwind;
-
-        args = data;
-        local = mem_get0 (this->local_pool);
-        if (!local) {
-                op_errno = ENOMEM;
-                goto unwind;
-        }
-
-        frame->local = local;
-
-        if (!(args->loc && args->loc->parent))
-                goto unwind;
-
-        loc_copy (&local->loc, args->loc);
-        loc_path (&local->loc, NULL);
-
-        if (!gf_uuid_is_null (args->loc->parent->gfid))
-                memcpy (req.pargfid,  args->loc->parent->gfid, 16);
-        else
-                memcpy (req.pargfid, args->loc->pargfid, 16);
-
-        GF_ASSERT_AND_GOTO_WITH_ERROR (this->name,
-                                       !gf_uuid_is_null (*((uuid_t*)req.pargfid)),
-                                       unwind, op_errno, EINVAL);
-        req.linkname = (char *)args->linkname;
-        req.bname    = (char *)args->loc->name;
-        req.umask = args->umask;
-        local->loc2.path = gf_strdup (req.linkname);
-
-        conf = this->private;
-
-        GF_PROTOCOL_DICT_SERIALIZE (this, args->xdata, (&req.xdata.xdata_val),
-                                    req.xdata.xdata_len, op_errno, unwind);
-
-        ret = client_submit_request (this, &req, frame, conf->fops,
-                                     GFS3_OP_SYMLINK, client3_3_symlink_cbk,
-                                     NULL,  NULL, 0, NULL,
-                                     0, NULL, (xdrproc_t)xdr_gfs3_symlink_req);
-        if (ret) {
-                gf_log (this->name, GF_LOG_WARNING, "failed to send the fop");
-        }
-
-        GF_FREE (req.xdata.xdata_val);
-
-        return 0;
+    clnt_local_t *local = NULL;
+    clnt_conf_t *conf = NULL;
+    clnt_args_t *args = NULL;
+    gfs3_symlink_req req = {
+        {
+            0,
+        },
+    };
+    int ret = 0;
+    int op_errno = ESTALE;
+
+    if (!frame || !this || !data)
+        goto unwind;
+
+    args = data;
+    conf = this->private;
+
+    local = mem_get0(this->local_pool);
+    if (!local) {
+        op_errno = ENOMEM;
+        goto unwind;
+    }
+
+    frame->local = local;
+
+    if (!(args->loc && args->loc->parent))
+        goto unwind;
+
+    loc_copy(&local->loc, args->loc);
+    loc_path(&local->loc, NULL);
+
+    local->loc2.path = gf_strdup(args->linkname);
+
+    ret = client_pre_symlink(this, &req, args->loc, args->linkname, args->umask,
+                             args->xdata);
+    if (ret) {
+        op_errno = -ret;
+        goto unwind;
+    }
+    ret = client_submit_request(this, &req, frame, conf->fops, GFS3_OP_SYMLINK,
+                                client3_3_symlink_cbk, NULL,
+                                (xdrproc_t)xdr_gfs3_symlink_req);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_WARNING, 0, PC_MSG_FOP_SEND_FAILED, NULL);
+    }
+
+    GF_FREE(req.xdata.xdata_val);
+
+    return 0;
 unwind:
 
-        CLIENT_STACK_UNWIND (symlink, frame, -1, op_errno, NULL, NULL, NULL,
-                             NULL, NULL);
+    CLIENT_STACK_UNWIND(symlink, frame, -1, op_errno, NULL, NULL, NULL, NULL,
+                        NULL);
 
-        GF_FREE (req.xdata.xdata_val);
+    GF_FREE(req.xdata.xdata_val);
 
-        return 0;
+    return 0;
 }
 
-
-
 int32_t
-client3_3_rename (call_frame_t *frame, xlator_t *this,
-                  void *data)
+client3_3_rename(call_frame_t *frame, xlator_t *this, void *data)
 {
-        clnt_conf_t     *conf     = NULL;
-        clnt_args_t     *args     = NULL;
-        gfs3_rename_req  req      = {{0,},};
-        int              ret      = 0;
-        int              op_errno = ESTALE;
-
-        if (!frame || !this || !data)
-                goto unwind;
-
-        args = data;
-
-        if (!(args->oldloc && args->newloc && args->oldloc->parent &&
-              args->newloc->parent))
-                goto unwind;
-
-        if (!gf_uuid_is_null (args->oldloc->parent->gfid))
-                memcpy (req.oldgfid,  args->oldloc->parent->gfid, 16);
-        else
-                memcpy (req.oldgfid, args->oldloc->pargfid, 16);
-
-        if (!gf_uuid_is_null (args->newloc->parent->gfid))
-                memcpy (req.newgfid, args->newloc->parent->gfid, 16);
-        else
-                memcpy (req.newgfid, args->newloc->pargfid, 16);
-
-        GF_ASSERT_AND_GOTO_WITH_ERROR (this->name,
-                                       !gf_uuid_is_null (*((uuid_t*)req.oldgfid)),
-                                       unwind, op_errno, EINVAL);
-        GF_ASSERT_AND_GOTO_WITH_ERROR (this->name,
-                                       !gf_uuid_is_null (*((uuid_t*)req.newgfid)),
-                                       unwind, op_errno, EINVAL);
-        req.oldbname =  (char *)args->oldloc->name;
-        req.newbname = (char *)args->newloc->name;
-        conf = this->private;
-
-        GF_PROTOCOL_DICT_SERIALIZE (this, args->xdata, (&req.xdata.xdata_val),
-                                    req.xdata.xdata_len, op_errno, unwind);
-
-        ret = client_submit_request (this, &req, frame, conf->fops,
-                                     GFS3_OP_RENAME, client3_3_rename_cbk, NULL,
-                                     NULL, 0, NULL, 0,
-                                     NULL, (xdrproc_t)xdr_gfs3_rename_req);
-        if (ret) {
-                gf_log (this->name, GF_LOG_WARNING, "failed to send the fop");
-        }
-
-        GF_FREE (req.xdata.xdata_val);
-
-        return 0;
+    clnt_conf_t *conf = NULL;
+    clnt_args_t *args = NULL;
+    gfs3_rename_req req = {
+        {
+            0,
+        },
+    };
+    int ret = 0;
+    int op_errno = ESTALE;
+
+    if (!frame || !this || !data)
+        goto unwind;
+
+    args = data;
+    conf = this->private;
+
+    ret = client_pre_rename(this, &req, args->oldloc, args->newloc,
+                            args->xdata);
+    if (ret) {
+        op_errno = -ret;
+        goto unwind;
+    }
+    ret = client_submit_request(this, &req, frame, conf->fops, GFS3_OP_RENAME,
+                                client3_3_rename_cbk, NULL,
+                                (xdrproc_t)xdr_gfs3_rename_req);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_WARNING, 0, PC_MSG_FOP_SEND_FAILED, NULL);
+    }
+
+    GF_FREE(req.xdata.xdata_val);
+
+    return 0;
 unwind:
-        CLIENT_STACK_UNWIND (rename, frame, -1, op_errno, NULL, NULL, NULL,
-                             NULL, NULL, NULL);
+    CLIENT_STACK_UNWIND(rename, frame, -1, op_errno, NULL, NULL, NULL, NULL,
+                        NULL, NULL);
 
-        GF_FREE (req.xdata.xdata_val);
+    GF_FREE(req.xdata.xdata_val);
 
-        return 0;
+    return 0;
 }
 
-
-
 int32_t
-client3_3_link (call_frame_t *frame, xlator_t *this,
-                void *data)
+client3_3_link(call_frame_t *frame, xlator_t *this, void *data)
 {
-        clnt_local_t  *local    = NULL;
-        clnt_conf_t   *conf     = NULL;
-        clnt_args_t   *args     = NULL;
-        gfs3_link_req  req      = {{0,},};
-        int            ret      = 0;
-        int            op_errno = ESTALE;
-
-        if (!frame || !this || !data)
-                goto unwind;
-
-        args = data;
-
-        if (!(args->oldloc && args->oldloc->inode && args->newloc &&
-              args->newloc->parent))
-                goto unwind;
-
-        if (!gf_uuid_is_null (args->oldloc->inode->gfid))
-                memcpy (req.oldgfid,  args->oldloc->inode->gfid, 16);
-        else
-                memcpy (req.oldgfid, args->oldloc->gfid, 16);
-
-        if (!gf_uuid_is_null (args->newloc->parent->gfid))
-                memcpy (req.newgfid, args->newloc->parent->gfid, 16);
-        else
-                memcpy (req.newgfid, args->newloc->pargfid, 16);
-
-        GF_ASSERT_AND_GOTO_WITH_ERROR (this->name,
-                                       !gf_uuid_is_null (*((uuid_t*)req.oldgfid)),
-                                       unwind, op_errno, EINVAL);
-        GF_ASSERT_AND_GOTO_WITH_ERROR (this->name,
-                                       !gf_uuid_is_null (*((uuid_t*)req.newgfid)),
-                                       unwind, op_errno, EINVAL);
-        local = mem_get0 (this->local_pool);
-        if (!local) {
-                op_errno = ENOMEM;
-                goto unwind;
-        }
-
-        loc_copy (&local->loc, args->oldloc);
-        loc_path (&local->loc, NULL);
-        loc_copy (&local->loc2, args->newloc);
-        loc_path (&local->loc2, NULL);
-        frame->local = local;
-
-        req.newbname = (char *)args->newloc->name;
-        conf = this->private;
-
-        GF_PROTOCOL_DICT_SERIALIZE (this, args->xdata, (&req.xdata.xdata_val),
-                                    req.xdata.xdata_len, op_errno, unwind);
-
-        ret = client_submit_request (this, &req, frame, conf->fops,
-                                     GFS3_OP_LINK, client3_3_link_cbk, NULL,
-                                     NULL, 0, NULL, 0, NULL,
-                                     (xdrproc_t)xdr_gfs3_link_req);
-        if (ret) {
-                gf_log (this->name, GF_LOG_WARNING, "failed to send the fop");
-        }
-
-        GF_FREE (req.xdata.xdata_val);
-
-        return 0;
+    clnt_local_t *local = NULL;
+    clnt_conf_t *conf = NULL;
+    clnt_args_t *args = NULL;
+    gfs3_link_req req = {
+        {
+            0,
+        },
+    };
+    int ret = 0;
+    int op_errno = ESTALE;
+
+    if (!frame || !this || !data)
+        goto unwind;
+
+    args = data;
+    conf = this->private;
+
+    local = mem_get0(this->local_pool);
+    if (!local) {
+        op_errno = ENOMEM;
+        goto unwind;
+    }
+
+    frame->local = local;
+
+    ret = client_pre_link(this, &req, args->oldloc, args->newloc, args->xdata);
+    if (ret) {
+        op_errno = -ret;
+        goto unwind;
+    }
+
+    loc_copy(&local->loc, args->oldloc);
+    loc_path(&local->loc, NULL);
+    loc_copy(&local->loc2, args->newloc);
+    loc_path(&local->loc2, NULL);
+
+    ret = client_submit_request(this, &req, frame, conf->fops, GFS3_OP_LINK,
+                                client3_3_link_cbk, NULL,
+                                (xdrproc_t)xdr_gfs3_link_req);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_WARNING, 0, PC_MSG_FOP_SEND_FAILED, NULL);
+    }
+
+    GF_FREE(req.xdata.xdata_val);
+
+    return 0;
 unwind:
-        CLIENT_STACK_UNWIND (link, frame, -1, op_errno, NULL, NULL, NULL, NULL, NULL);
-        GF_FREE (req.xdata.xdata_val);
+    CLIENT_STACK_UNWIND(link, frame, -1, op_errno, NULL, NULL, NULL, NULL,
+                        NULL);
+    GF_FREE(req.xdata.xdata_val);
 
-        return 0;
+    return 0;
 }
 
-
-
 int32_t
-client3_3_mknod (call_frame_t *frame, xlator_t *this,
-                 void *data)
+client3_3_mknod(call_frame_t *frame, xlator_t *this, void *data)
 {
-        clnt_local_t   *local    = NULL;
-        clnt_conf_t    *conf     = NULL;
-        clnt_args_t    *args     = NULL;
-        gfs3_mknod_req  req      = {{0,},};
-        int             ret      = 0;
-        int             op_errno = ESTALE;
-
-        if (!frame || !this || !data)
-                goto unwind;
-
-        args = data;
-
-        local = mem_get0 (this->local_pool);
-        if (!local) {
-                op_errno = ENOMEM;
-                goto unwind;
-        }
-        frame->local = local;
-
-        if (!(args->loc && args->loc->parent))
-                goto unwind;
-
-        loc_copy (&local->loc, args->loc);
-        loc_path (&local->loc, NULL);
-
-        if (!gf_uuid_is_null (args->loc->parent->gfid))
-                memcpy (req.pargfid,  args->loc->parent->gfid, 16);
-        else
-                memcpy (req.pargfid, args->loc->pargfid, 16);
-
-        GF_ASSERT_AND_GOTO_WITH_ERROR (this->name,
-                                       !gf_uuid_is_null (*((uuid_t*)req.pargfid)),
-                                       unwind, op_errno, EINVAL);
-        req.bname  = (char *)args->loc->name;
-        req.mode   = args->mode;
-        req.dev    = args->rdev;
-        req.umask = args->umask;
-
-        conf = this->private;
-
-        GF_PROTOCOL_DICT_SERIALIZE (this, args->xdata, (&req.xdata.xdata_val),
-                                    req.xdata.xdata_len, op_errno, unwind);
-
-        ret = client_submit_request (this, &req, frame, conf->fops,
-                                     GFS3_OP_MKNOD, client3_3_mknod_cbk, NULL,
-                                     NULL, 0, NULL, 0,
-                                     NULL, (xdrproc_t)xdr_gfs3_mknod_req);
-        if (ret) {
-                gf_log (this->name, GF_LOG_WARNING, "failed to send the fop");
-        }
-        GF_FREE (req.xdata.xdata_val);
-
-        return 0;
+    clnt_local_t *local = NULL;
+    clnt_conf_t *conf = NULL;
+    clnt_args_t *args = NULL;
+    gfs3_mknod_req req = {
+        {
+            0,
+        },
+    };
+    int ret = 0;
+    int op_errno = ESTALE;
+
+    if (!frame || !this || !data)
+        goto unwind;
+
+    args = data;
+    conf = this->private;
+
+    local = mem_get0(this->local_pool);
+    if (!local) {
+        op_errno = ENOMEM;
+        goto unwind;
+    }
+    frame->local = local;
+
+    loc_copy(&local->loc, args->loc);
+    loc_path(&local->loc, NULL);
+
+    ret = client_pre_mknod(this, &req, args->loc, args->mode, args->rdev,
+                           args->umask, args->xdata);
+    if (ret) {
+        op_errno = -ret;
+        goto unwind;
+    }
+    ret = client_submit_request(this, &req, frame, conf->fops, GFS3_OP_MKNOD,
+                                client3_3_mknod_cbk, NULL,
+                                (xdrproc_t)xdr_gfs3_mknod_req);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_WARNING, 0, PC_MSG_FOP_SEND_FAILED, NULL);
+    }
+    GF_FREE(req.xdata.xdata_val);
+
+    return 0;
 unwind:
-        CLIENT_STACK_UNWIND (mknod, frame, -1, op_errno, NULL, NULL, NULL,
-                             NULL, NULL);
+    CLIENT_STACK_UNWIND(mknod, frame, -1, op_errno, NULL, NULL, NULL, NULL,
+                        NULL);
 
-        GF_FREE (req.xdata.xdata_val);
+    GF_FREE(req.xdata.xdata_val);
 
-        return 0;
+    return 0;
 }
 
-
-
 int32_t
-client3_3_mkdir (call_frame_t *frame, xlator_t *this,
-                 void *data)
+client3_3_mkdir(call_frame_t *frame, xlator_t *this, void *data)
 {
-        clnt_local_t   *local    = NULL;
-        clnt_conf_t    *conf     = NULL;
-        clnt_args_t    *args     = NULL;
-        gfs3_mkdir_req  req      = {{0,},};
-        int             ret      = 0;
-        int             op_errno = ESTALE;
-
-        if (!frame || !this || !data)
-                goto unwind;
-
-        args = data;
-
-        local = mem_get0 (this->local_pool);
-        if (!local) {
-                op_errno = ENOMEM;
-                goto unwind;
-        }
-        frame->local = local;
-
-        if (!(args->loc && args->loc->parent))
-                goto unwind;
-
-        loc_copy (&local->loc, args->loc);
-        loc_path (&local->loc, NULL);
-
-        if (!gf_uuid_is_null (args->loc->parent->gfid))
-                memcpy (req.pargfid,  args->loc->parent->gfid, 16);
-        else
-                memcpy (req.pargfid, args->loc->pargfid, 16);
-
-        GF_ASSERT_AND_GOTO_WITH_ERROR (this->name,
-                                       !gf_uuid_is_null (*((uuid_t*)req.pargfid)),
-                                       unwind, op_errno, EINVAL);
-
-        req.bname = (char *)args->loc->name;
-        req.mode  = args->mode;
-        req.umask = args->umask;
-
-        conf = this->private;
-
-        GF_PROTOCOL_DICT_SERIALIZE (this, args->xdata, (&req.xdata.xdata_val),
-                                    req.xdata.xdata_len, op_errno, unwind);
-
-        ret = client_submit_request (this, &req, frame, conf->fops,
-                                     GFS3_OP_MKDIR, client3_3_mkdir_cbk, NULL,
-                                     NULL, 0, NULL, 0,
-                                     NULL, (xdrproc_t)xdr_gfs3_mkdir_req);
-        if (ret) {
-                gf_log (this->name, GF_LOG_WARNING, "failed to send the fop");
-        }
-        GF_FREE (req.xdata.xdata_val);
-
-        return 0;
+    clnt_local_t *local = NULL;
+    clnt_conf_t *conf = NULL;
+    clnt_args_t *args = NULL;
+    gfs3_mkdir_req req = {
+        {
+            0,
+        },
+    };
+    int ret = 0;
+    int op_errno = ESTALE;
+
+    if (!frame || !this || !data)
+        goto unwind;
+
+    args = data;
+    conf = this->private;
+
+    if (!args->xdata || !dict_get_sizen(args->xdata, "gfid-req")) {
+        op_errno = EPERM;
+        gf_msg_callingfn(this->name, GF_LOG_WARNING, op_errno, PC_MSG_GFID_NULL,
+                         "mkdir: %s is received "
+                         "without gfid-req %p",
+                         args->loc->path, args->xdata);
+        goto unwind;
+    }
+
+    local = mem_get0(this->local_pool);
+    if (!local) {
+        op_errno = ENOMEM;
+        goto unwind;
+    }
+    frame->local = local;
+
+    if (!(args->loc && args->loc->parent))
+        goto unwind;
+
+    loc_copy(&local->loc, args->loc);
+    loc_path(&local->loc, NULL);
+
+    ret = client_pre_mkdir(this, &req, args->loc, args->mode, args->umask,
+                           args->xdata);
+    if (ret) {
+        op_errno = -ret;
+        goto unwind;
+    }
+    ret = client_submit_request(this, &req, frame, conf->fops, GFS3_OP_MKDIR,
+                                client3_3_mkdir_cbk, NULL,
+                                (xdrproc_t)xdr_gfs3_mkdir_req);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_WARNING, 0, PC_MSG_FOP_SEND_FAILED, NULL);
+    }
+    GF_FREE(req.xdata.xdata_val);
+
+    return 0;
 unwind:
-        CLIENT_STACK_UNWIND (mkdir, frame, -1, op_errno, NULL, NULL, NULL,
-                             NULL, NULL);
+    CLIENT_STACK_UNWIND(mkdir, frame, -1, op_errno, NULL, NULL, NULL, NULL,
+                        NULL);
 
-        GF_FREE (req.xdata.xdata_val);
+    GF_FREE(req.xdata.xdata_val);
 
-        return 0;
+    return 0;
 }
 
-
 int32_t
-client3_3_create (call_frame_t *frame, xlator_t *this,
-                  void *data)
+client3_3_create(call_frame_t *frame, xlator_t *this, void *data)
 {
-        clnt_local_t    *local    = NULL;
-        clnt_conf_t     *conf     = NULL;
-        clnt_args_t     *args     = NULL;
-        gfs3_create_req  req      = {{0,},};
-        int              ret      = 0;
-        int              op_errno = ESTALE;
-
-        if (!frame || !this || !data)
-                goto unwind;
-
-        args = data;
-
-        local = mem_get0 (this->local_pool);
-        if (!local) {
-                op_errno = ENOMEM;
-                goto unwind;
-        }
-        frame->local = local;
-
-        if (!(args->loc && args->loc->parent))
-                goto unwind;
-
-        local->fd = fd_ref (args->fd);
-        local->flags = args->flags;
-
-        loc_copy (&local->loc, args->loc);
-        loc_path (&local->loc, NULL);
-
-        if (!gf_uuid_is_null (args->loc->parent->gfid))
-                memcpy (req.pargfid,  args->loc->parent->gfid, 16);
-        else
-                memcpy (req.pargfid, args->loc->pargfid, 16);
-
-        GF_ASSERT_AND_GOTO_WITH_ERROR (this->name,
-                                       !gf_uuid_is_null (*((uuid_t*)req.pargfid)),
-                                       unwind, op_errno, EINVAL);
-        req.bname = (char *)args->loc->name;
-        req.mode  = args->mode;
-        req.flags = gf_flags_from_flags (args->flags);
-        req.umask = args->umask;
-
-        conf = this->private;
-
-        GF_PROTOCOL_DICT_SERIALIZE (this, args->xdata, (&req.xdata.xdata_val),
-                                    req.xdata.xdata_len, op_errno, unwind);
-
-        ret = client_submit_request (this, &req, frame, conf->fops,
-                                     GFS3_OP_CREATE, client3_3_create_cbk, NULL,
-                                     NULL, 0, NULL, 0,
-                                     NULL, (xdrproc_t)xdr_gfs3_create_req);
-        if (ret) {
-                gf_log (this->name, GF_LOG_WARNING, "failed to send the fop");
-        }
-
-        GF_FREE (req.xdata.xdata_val);
-
-        return 0;
+    clnt_local_t *local = NULL;
+    clnt_conf_t *conf = NULL;
+    clnt_args_t *args = NULL;
+    gfs3_create_req req = {
+        {
+            0,
+        },
+    };
+    int ret = 0;
+    int op_errno = ESTALE;
+
+    if (!frame || !this || !data)
+        goto unwind;
+
+    args = data;
+    conf = this->private;
+
+    local = mem_get0(this->local_pool);
+    if (!local) {
+        op_errno = ENOMEM;
+        goto unwind;
+    }
+    frame->local = local;
+
+    local->fd = fd_ref(args->fd);
+    local->flags = args->flags;
+
+    loc_copy(&local->loc, args->loc);
+    loc_path(&local->loc, NULL);
+
+    ret = client_pre_create(this, &req, args->loc, args->fd, args->mode,
+                            args->flags, args->umask, args->xdata);
+    if (ret) {
+        op_errno = -ret;
+        goto unwind;
+    }
+    ret = client_submit_request(this, &req, frame, conf->fops, GFS3_OP_CREATE,
+                                client3_3_create_cbk, NULL,
+                                (xdrproc_t)xdr_gfs3_create_req);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_WARNING, 0, PC_MSG_FOP_SEND_FAILED, NULL);
+    }
+
+    GF_FREE(req.xdata.xdata_val);
+
+    return 0;
 unwind:
-        CLIENT_STACK_UNWIND (create, frame, -1, op_errno, NULL, NULL, NULL,
-                             NULL, NULL, NULL);
+    CLIENT_STACK_UNWIND(create, frame, -1, op_errno, NULL, NULL, NULL, NULL,
+                        NULL, NULL);
 
-        GF_FREE (req.xdata.xdata_val);
+    GF_FREE(req.xdata.xdata_val);
 
-        return 0;
+    return 0;
 }
 
-
-
 int32_t
-client3_3_open (call_frame_t *frame, xlator_t *this,
-                void *data)
+client3_3_open(call_frame_t *frame, xlator_t *this, void *data)
 {
-        clnt_local_t  *local    = NULL;
-        clnt_conf_t   *conf     = NULL;
-        clnt_args_t   *args     = NULL;
-        gfs3_open_req  req      = {{0,},};
-        int            ret      = 0;
-        int            op_errno = ESTALE;
-
-        if (!frame || !this || !data)
-                goto unwind;
+    clnt_local_t *local = NULL;
+    clnt_conf_t *conf = NULL;
+    clnt_args_t *args = NULL;
+    gfs3_open_req req = {
+        {
+            0,
+        },
+    };
+    int ret = 0;
+    int op_errno = ESTALE;
 
-        args = data;
+    if (!frame || !this || !data)
+        goto unwind;
 
-        local = mem_get0 (this->local_pool);
-        if (!local) {
-                op_errno = ENOMEM;
-                goto unwind;
-        }
-        frame->local = local;
+    args = data;
 
-        if (!(args->loc && args->loc->inode))
-                goto unwind;
+    conf = this->private;
 
-        local->fd = fd_ref (args->fd);
-        local->flags = args->flags;
-        loc_copy (&local->loc, args->loc);
-        loc_path (&local->loc, NULL);
+    local = mem_get0(this->local_pool);
+    if (!local) {
+        op_errno = ENOMEM;
+        goto unwind;
+    }
+    frame->local = local;
 
-        if (!gf_uuid_is_null (args->loc->inode->gfid))
-                memcpy (req.gfid,  args->loc->inode->gfid, 16);
-        else
-                memcpy (req.gfid, args->loc->gfid, 16);
+    local->flags = args->flags;
 
-        GF_ASSERT_AND_GOTO_WITH_ERROR (this->name,
-                                       !gf_uuid_is_null (*((uuid_t*)req.gfid)),
-                                       unwind, op_errno, EINVAL);
-        req.flags = gf_flags_from_flags (args->flags);
+    local->fd = fd_ref(args->fd);
+    loc_copy(&local->loc, args->loc);
+    loc_path(&local->loc, NULL);
 
-        conf = this->private;
+    ret = client_pre_open(this, &req, args->loc, args->fd, args->flags,
+                          args->xdata);
 
-        GF_PROTOCOL_DICT_SERIALIZE (this, args->xdata, (&req.xdata.xdata_val),
-                                    req.xdata.xdata_len, op_errno, unwind);
-
-        ret = client_submit_request (this, &req, frame, conf->fops,
-                                     GFS3_OP_OPEN, client3_3_open_cbk, NULL,
-                                     NULL, 0, NULL, 0, NULL,
-                                     (xdrproc_t)xdr_gfs3_open_req);
-        if (ret) {
-                gf_log (this->name, GF_LOG_WARNING, "failed to send the fop");
-        }
+    if (ret) {
+        op_errno = -ret;
+        goto unwind;
+    }
+    ret = client_submit_request(this, &req, frame, conf->fops, GFS3_OP_OPEN,
+                                client3_3_open_cbk, NULL,
+                                (xdrproc_t)xdr_gfs3_open_req);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_WARNING, 0, PC_MSG_FOP_SEND_FAILED, NULL);
+    }
 
-        GF_FREE (req.xdata.xdata_val);
+    GF_FREE(req.xdata.xdata_val);
 
-        return 0;
+    return 0;
 unwind:
-        CLIENT_STACK_UNWIND (open, frame, -1, op_errno, NULL, NULL);
+    CLIENT_STACK_UNWIND(open, frame, -1, op_errno, NULL, NULL);
 
-        GF_FREE (req.xdata.xdata_val);
+    GF_FREE(req.xdata.xdata_val);
 
-        return 0;
+    return 0;
 }
 
-
-
 int32_t
-client3_3_readv (call_frame_t *frame, xlator_t *this,
-                 void *data)
+client3_3_readv(call_frame_t *frame, xlator_t *this, void *data)
 {
-        clnt_args_t    *args       = NULL;
-        int64_t         remote_fd  = -1;
-        clnt_conf_t    *conf       = NULL;
-        clnt_local_t   *local      = NULL;
-        int             op_errno   = ESTALE;
-        gfs3_read_req   req        = {{0,},};
-        int             ret        = 0;
-        struct iovec    rsp_vec    = {0, };
-        struct iobuf   *rsp_iobuf  = NULL;
-        struct iobref  *rsp_iobref = NULL;
-
-        if (!frame || !this || !data)
-                goto unwind;
-
-        args = data;
-        conf = this->private;
-
-        CLIENT_GET_REMOTE_FD (this, args->fd, FALLBACK_TO_ANON_FD,
-                              remote_fd, op_errno, unwind);
-        ret = client_fd_fop_prepare_local (frame, args->fd, remote_fd);
-        if (ret) {
-                op_errno = -ret;
-                goto unwind;
-        }
-        local = frame->local;
-
-        req.size   = args->size;
-        req.offset = args->offset;
-        req.fd     = remote_fd;
-        req.flag   = args->flags;
-
-        memcpy (req.gfid, args->fd->inode->gfid, 16);
-
-        rsp_iobuf = iobuf_get2 (this->ctx->iobuf_pool, args->size);
-        if (rsp_iobuf == NULL) {
-                op_errno = ENOMEM;
-                goto unwind;
-        }
-
-        rsp_iobref = iobref_new ();
-        if (rsp_iobref == NULL) {
-                op_errno = ENOMEM;
-                goto unwind;
-        }
-
-        iobref_add (rsp_iobref, rsp_iobuf);
-        iobuf_unref (rsp_iobuf);
-
-        rsp_vec.iov_base = iobuf_ptr (rsp_iobuf);
-        rsp_vec.iov_len = iobuf_pagesize (rsp_iobuf);
-
-        rsp_iobuf = NULL;
-
-        if (args->size > rsp_vec.iov_len) {
-                gf_log (this->name, GF_LOG_WARNING,
-                        "read-size (%lu) is bigger than iobuf size (%lu)",
-                        (unsigned long)args->size,
-                        (unsigned long)rsp_vec.iov_len);
-                op_errno = EINVAL;
-                goto unwind;
-        }
-
-        local->iobref = rsp_iobref;
-        rsp_iobref = NULL;
-
-        GF_PROTOCOL_DICT_SERIALIZE (this, args->xdata, (&req.xdata.xdata_val),
-                                    req.xdata.xdata_len, op_errno, unwind);
-
-        ret = client_submit_request (this, &req, frame, conf->fops,
-                                     GFS3_OP_READ, client3_3_readv_cbk, NULL,
-                                     NULL, 0, &rsp_vec, 1,
-                                     local->iobref,
-                                     (xdrproc_t)xdr_gfs3_read_req);
-        if (ret) {
-                //unwind is done in the cbk
-                gf_log (this->name, GF_LOG_WARNING, "failed to send the fop");
-        }
-
-        GF_FREE (req.xdata.xdata_val);
-
-        return 0;
+    clnt_args_t *args = NULL;
+    clnt_conf_t *conf = NULL;
+    clnt_local_t *local = NULL;
+    int op_errno = ESTALE;
+    gfs3_read_req req = {
+        {
+            0,
+        },
+    };
+    int ret = 0;
+    struct iovec rsp_vec = {
+        0,
+    };
+    struct iobuf *rsp_iobuf = NULL;
+    struct iobref *rsp_iobref = NULL;
+    client_payload_t cp;
+
+    if (!frame || !this || !data)
+        goto unwind;
+
+    args = data;
+    conf = this->private;
+
+    ret = client_pre_readv(this, &req, args->fd, args->size, args->offset,
+                           args->flags, args->xdata);
+    if (ret) {
+        op_errno = -ret;
+        goto unwind;
+    }
+
+    ret = client_fd_fop_prepare_local(frame, args->fd, req.fd);
+    if (ret) {
+        op_errno = -ret;
+        goto unwind;
+    }
+    local = frame->local;
+
+    rsp_iobuf = iobuf_get2(this->ctx->iobuf_pool, args->size);
+    if (rsp_iobuf == NULL) {
+        op_errno = ENOMEM;
+        goto unwind;
+    }
+
+    rsp_iobref = iobref_new();
+    if (rsp_iobref == NULL) {
+        op_errno = ENOMEM;
+        goto unwind;
+    }
+
+    rsp_vec.iov_base = iobuf_ptr(rsp_iobuf);
+    rsp_vec.iov_len = iobuf_pagesize(rsp_iobuf);
+
+    iobref_add(rsp_iobref, rsp_iobuf);
+    iobuf_unref(rsp_iobuf);
+    rsp_iobuf = NULL;
+
+    if (args->size > rsp_vec.iov_len) {
+        gf_smsg(this->name, GF_LOG_WARNING, ENOMEM, PC_MSG_BIGGER_SIZE,
+                "read-size=%lu", (unsigned long)args->size, "iobuf size=%lu",
+                (unsigned long)rsp_vec.iov_len, NULL);
+        op_errno = EINVAL;
+        goto unwind;
+    }
+
+    local->iobref = rsp_iobref;
+    rsp_iobref = NULL;
+
+    memset(&cp, 0, sizeof(client_payload_t));
+    cp.rsp_payload = &rsp_vec;
+    cp.rsp_payload_cnt = 1;
+    cp.rsp_iobref = local->iobref;
+    ret = client_submit_request(this, &req, frame, conf->fops, GFS3_OP_READ,
+                                client3_3_readv_cbk, &cp,
+                                (xdrproc_t)xdr_gfs3_read_req);
+    if (ret) {
+        // unwind is done in the cbk
+        gf_smsg(this->name, GF_LOG_WARNING, 0, PC_MSG_FOP_SEND_FAILED, NULL);
+    }
+
+    GF_FREE(req.xdata.xdata_val);
+
+    return 0;
 unwind:
-        if (rsp_iobuf)
-                iobuf_unref (rsp_iobuf);
+    if (rsp_iobuf)
+        iobuf_unref(rsp_iobuf);
 
-        if (rsp_iobref)
-                iobref_unref (rsp_iobref);
+    if (rsp_iobref)
+        iobref_unref(rsp_iobref);
 
-        CLIENT_STACK_UNWIND (readv, frame, -1, op_errno, NULL, 0, NULL, NULL, NULL);
-        GF_FREE (req.xdata.xdata_val);
+    CLIENT_STACK_UNWIND(readv, frame, -1, op_errno, NULL, 0, NULL, NULL, NULL);
+    GF_FREE(req.xdata.xdata_val);
 
-        return 0;
+    return 0;
 }
 
-
 int32_t
-client3_3_writev (call_frame_t *frame, xlator_t *this, void *data)
+client3_3_writev(call_frame_t *frame, xlator_t *this, void *data)
 {
-        clnt_args_t    *args     = NULL;
-        int64_t         remote_fd = -1;
-        clnt_conf_t    *conf     = NULL;
-        gfs3_write_req  req      = {{0,},};
-        int             op_errno = ESTALE;
-        int             ret      = 0;
-
-        if (!frame || !this || !data)
-                goto unwind;
-
-        args = data;
-        conf = this->private;
-
-        CLIENT_GET_REMOTE_FD (this, args->fd, FALLBACK_TO_ANON_FD,
-                              remote_fd, op_errno, unwind);
-        ret = client_fd_fop_prepare_local (frame, args->fd, remote_fd);
-        if (ret) {
-                op_errno = -ret;
-                goto unwind;
-        }
-
-        req.size   = args->size;
-        req.offset = args->offset;
-        req.fd     = remote_fd;
-        req.flag   = args->flags;
-
-        memcpy (req.gfid, args->fd->inode->gfid, 16);
-
-#ifdef GF_TESTING_IO_XDATA
-        if (!args->xdata)
-                args->xdata = dict_new ();
-
-        ret = dict_set_str (args->xdata, "testing-the-xdata-key",
-                            "testing-the-xdata-value");
-#endif
-
-        GF_PROTOCOL_DICT_SERIALIZE (this, args->xdata, (&req.xdata.xdata_val),
-                                    req.xdata.xdata_len, op_errno, unwind);
-
-        ret = client_submit_vec_request (this, &req, frame, conf->fops,
-                                         GFS3_OP_WRITE, client3_3_writev_cbk,
-                                         args->vector, args->count,
-                                         args->iobref,
-                                         (xdrproc_t)xdr_gfs3_write_req);
-        if (ret) {
-                /*
-                 * If the lower layers fail to submit a request, they'll also
-                 * do the unwind for us (see rpc_clnt_submit), so don't unwind
-                 * here in such cases.
-                 */
-                gf_log (this->name, GF_LOG_WARNING, "failed to send the fop");
-        }
+    clnt_args_t *args = NULL;
+    clnt_conf_t *conf = NULL;
+    gfs3_write_req req = {
+        {
+            0,
+        },
+    };
+    int op_errno = ESTALE;
+    int ret = 0;
+    client_payload_t cp;
+
+    if (!frame || !this || !data)
+        goto unwind;
+
+    args = data;
+    conf = this->private;
+
+    ret = client_pre_writev(this, &req, args->fd, args->size, args->offset,
+                            args->flags, &args->xdata);
+
+    if (ret) {
+        op_errno = -ret;
+        goto unwind;
+    }
+
+    ret = client_fd_fop_prepare_local(frame, args->fd, req.fd);
+    if (ret) {
+        op_errno = -ret;
+        goto unwind;
+    }
+
+    memset(&cp, 0, sizeof(client_payload_t));
+    cp.iobref = args->iobref;
+    cp.payload = args->vector;
+    cp.payload_cnt = args->count;
+    ret = client_submit_request(this, &req, frame, conf->fops, GFS3_OP_WRITE,
+                                client3_3_writev_cbk, &cp,
+                                (xdrproc_t)xdr_gfs3_write_req);
+    if (ret) {
+        /*
+         * If the lower layers fail to submit a request, they'll also
+         * do the unwind for us (see rpc_clnt_submit), so don't unwind
+         * here in such cases.
+         */
+        gf_smsg(this->name, GF_LOG_WARNING, 0, PC_MSG_FOP_SEND_FAILED, NULL);
+    }
 
-        GF_FREE (req.xdata.xdata_val);
+    GF_FREE(req.xdata.xdata_val);
 
-        return 0;
+    return 0;
 
 unwind:
-        CLIENT_STACK_UNWIND (writev, frame, -1, op_errno, NULL, NULL, NULL);
-        GF_FREE (req.xdata.xdata_val);
+    CLIENT_STACK_UNWIND(writev, frame, -1, op_errno, NULL, NULL, NULL);
+    GF_FREE(req.xdata.xdata_val);
 
-        return 0;
+    return 0;
 }
 
-
 int32_t
-client3_3_flush (call_frame_t *frame, xlator_t *this,
-                 void *data)
+client3_3_flush(call_frame_t *frame, xlator_t *this, void *data)
 {
-        clnt_args_t    *args     = NULL;
-        gfs3_flush_req  req      = {{0,},};
-        int64_t         remote_fd = -1;
-        clnt_conf_t    *conf     = NULL;
-        clnt_local_t   *local    = NULL;
-        int             op_errno = ESTALE;
-        int             ret      = 0;
-
-        if (!frame || !this || !data)
-                goto unwind;
-
-        args = data;
-        conf = this->private;
-
-        CLIENT_GET_REMOTE_FD (this, args->fd, DEFAULT_REMOTE_FD,
-                              remote_fd, op_errno, unwind);
-
-        conf = this->private;
-
-        local = mem_get0 (this->local_pool);
-        if (!local) {
-                op_errno = ENOMEM;
-                goto unwind;
-        }
-
-        local->fd = fd_ref (args->fd);
-        local->owner = frame->root->lk_owner;
-        frame->local = local;
-
-        req.fd = remote_fd;
-        memcpy (req.gfid, args->fd->inode->gfid, 16);
-
-        GF_PROTOCOL_DICT_SERIALIZE (this, args->xdata, (&req.xdata.xdata_val),
-                                    req.xdata.xdata_len, op_errno, unwind);
-
-        ret = client_submit_request (this, &req, frame, conf->fops,
-                                     GFS3_OP_FLUSH, client3_3_flush_cbk, NULL,
-                                     NULL, 0, NULL, 0,
-                                     NULL, (xdrproc_t)xdr_gfs3_flush_req);
-        if (ret) {
-                gf_log (this->name, GF_LOG_WARNING, "failed to send the fop");
-        }
-
-        GF_FREE (req.xdata.xdata_val);
-
-
-        return 0;
+    clnt_args_t *args = NULL;
+    gfs3_flush_req req = {
+        {
+            0,
+        },
+    };
+    clnt_conf_t *conf = NULL;
+    clnt_local_t *local = NULL;
+    int op_errno = ESTALE;
+    int ret = 0;
+
+    if (!frame || !this || !data)
+        goto unwind;
+
+    args = data;
+    conf = this->private;
+
+    local = mem_get0(this->local_pool);
+    if (!local) {
+        op_errno = ENOMEM;
+        goto unwind;
+    }
+
+    frame->local = local;
+
+    local->fd = fd_ref(args->fd);
+    local->owner = frame->root->lk_owner;
+    ret = client_pre_flush(this, &req, args->fd, args->xdata);
+    if (ret) {
+        op_errno = -ret;
+        if (op_errno == EBADF) {
+            ret = delete_granted_locks_owner(local->fd, &local->owner);
+            gf_msg_trace(this->name, 0,
+                         "deleting locks of owner (%s) returned %d",
+                         lkowner_utoa(&local->owner), ret);
+        }
+
+        goto unwind;
+    }
+
+    ret = client_submit_request(this, &req, frame, conf->fops, GFS3_OP_FLUSH,
+                                client3_3_flush_cbk, NULL,
+                                (xdrproc_t)xdr_gfs3_flush_req);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_WARNING, 0, PC_MSG_FOP_SEND_FAILED, NULL);
+    }
+
+    GF_FREE(req.xdata.xdata_val);
+
+    return 0;
 
 unwind:
-        CLIENT_STACK_UNWIND (flush, frame, -1, op_errno, NULL);
-        GF_FREE (req.xdata.xdata_val);
+    CLIENT_STACK_UNWIND(flush, frame, -1, op_errno, NULL);
+    GF_FREE(req.xdata.xdata_val);
 
-        return 0;
+    return 0;
 }
 
-
-
 int32_t
-client3_3_fsync (call_frame_t *frame, xlator_t *this,
-                 void *data)
+client3_3_fsync(call_frame_t *frame, xlator_t *this, void *data)
 {
-        clnt_args_t    *args      = NULL;
-        gfs3_fsync_req  req       = {{0,},};
-        int64_t         remote_fd = -1;
-        clnt_conf_t    *conf      = NULL;
-        int             op_errno  = 0;
-        int             ret       = 0;
-
-        if (!frame || !this || !data)
-                goto unwind;
-
-        args = data;
-        conf = this->private;
-
-        CLIENT_GET_REMOTE_FD (this, args->fd, DEFAULT_REMOTE_FD,
-                              remote_fd, op_errno, unwind);
-
-        req.fd   = remote_fd;
-        req.data = args->flags;
-        memcpy (req.gfid, args->fd->inode->gfid, 16);
-
-        GF_PROTOCOL_DICT_SERIALIZE (this, args->xdata, (&req.xdata.xdata_val),
-                                    req.xdata.xdata_len, op_errno, unwind);
-
-        ret = client_submit_request (this, &req, frame, conf->fops,
-                                     GFS3_OP_FSYNC, client3_3_fsync_cbk, NULL,
-                                     NULL, 0, NULL, 0,
-                                     NULL, (xdrproc_t)xdr_gfs3_fsync_req);
-        if (ret) {
-                gf_log (this->name, GF_LOG_WARNING, "failed to send the fop");
-
-        }
-
-        GF_FREE (req.xdata.xdata_val);
-
-        return 0;
+    clnt_args_t *args = NULL;
+    gfs3_fsync_req req = {
+        {
+            0,
+        },
+    };
+    clnt_conf_t *conf = NULL;
+    int op_errno = 0;
+    int ret = 0;
+
+    if (!frame || !this || !data)
+        goto unwind;
+
+    args = data;
+    conf = this->private;
+
+    ret = client_pre_fsync(this, &req, args->fd, args->flags, args->xdata);
+    if (ret) {
+        op_errno = -ret;
+        goto unwind;
+    }
+    ret = client_submit_request(this, &req, frame, conf->fops, GFS3_OP_FSYNC,
+                                client3_3_fsync_cbk, NULL,
+                                (xdrproc_t)xdr_gfs3_fsync_req);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_WARNING, 0, PC_MSG_FOP_SEND_FAILED, NULL);
+    }
+
+    GF_FREE(req.xdata.xdata_val);
+
+    return 0;
 
 unwind:
-        CLIENT_STACK_UNWIND (fsync, frame, -1, op_errno, NULL, NULL, NULL);
-        GF_FREE (req.xdata.xdata_val);
+    CLIENT_STACK_UNWIND(fsync, frame, -1, op_errno, NULL, NULL, NULL);
+    GF_FREE(req.xdata.xdata_val);
 
-        return 0;
+    return 0;
 }
 
-
-
 int32_t
-client3_3_fstat (call_frame_t *frame, xlator_t *this,
-                 void *data)
+client3_3_fstat(call_frame_t *frame, xlator_t *this, void *data)
 {
-        clnt_args_t    *args     = NULL;
-        gfs3_fstat_req  req      = {{0,},};
-        int64_t         remote_fd = -1;
-        clnt_conf_t    *conf     = NULL;
-        int             op_errno = ESTALE;
-        int           ret        = 0;
-
-        if (!frame || !this || !data)
-                goto unwind;
+    clnt_args_t *args = NULL;
+    gfs3_fstat_req req = {
+        {
+            0,
+        },
+    };
+    clnt_conf_t *conf = NULL;
+    int op_errno = ESTALE;
+    int ret = 0;
 
-        args = data;
-        conf = this->private;
+    if (!frame || !this || !data)
+        goto unwind;
 
-        CLIENT_GET_REMOTE_FD (this, args->fd, DEFAULT_REMOTE_FD,
-                              remote_fd, op_errno, unwind);
+    args = data;
+    conf = this->private;
 
-        req.fd = remote_fd;
-        memcpy (req.gfid, args->fd->inode->gfid, 16);
+    ret = client_pre_fstat(this, &req, args->fd, args->xdata);
 
-        GF_PROTOCOL_DICT_SERIALIZE (this, args->xdata, (&req.xdata.xdata_val),
-                                    req.xdata.xdata_len, op_errno, unwind);
+    if (ret) {
+        op_errno = -ret;
+        goto unwind;
+    }
 
-        ret = client_submit_request (this, &req, frame, conf->fops,
-                                     GFS3_OP_FSTAT, client3_3_fstat_cbk, NULL,
-                                     NULL, 0, NULL, 0,
-                                     NULL, (xdrproc_t)xdr_gfs3_fstat_req);
-        if (ret) {
-                gf_log (this->name, GF_LOG_WARNING, "failed to send the fop");
-        }
+    ret = client_submit_request(this, &req, frame, conf->fops, GFS3_OP_FSTAT,
+                                client3_3_fstat_cbk, NULL,
+                                (xdrproc_t)xdr_gfs3_fstat_req);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_WARNING, 0, PC_MSG_FOP_SEND_FAILED, NULL);
+    }
 
-        GF_FREE (req.xdata.xdata_val);
+    GF_FREE(req.xdata.xdata_val);
 
-        return 0;
+    return 0;
 
 unwind:
-        CLIENT_STACK_UNWIND (fstat, frame, -1, op_errno, NULL, NULL);
-        GF_FREE (req.xdata.xdata_val);
+    CLIENT_STACK_UNWIND(fstat, frame, -1, op_errno, NULL, NULL);
+    GF_FREE(req.xdata.xdata_val);
 
-        return 0;
+    return 0;
 }
 
-
-
 int32_t
-client3_3_opendir (call_frame_t *frame, xlator_t *this,
-                   void *data)
+client3_3_opendir(call_frame_t *frame, xlator_t *this, void *data)
 {
-        clnt_local_t     *local    = NULL;
-        clnt_conf_t      *conf     = NULL;
-        clnt_args_t      *args     = NULL;
-        gfs3_opendir_req  req      = {{0,},};
-        int               ret      = 0;
-        int               op_errno = ESTALE;
-
-        if (!frame || !this || !data)
-                goto unwind;
-
-        args = data;
-
-        local = mem_get0 (this->local_pool);
-        if (!local) {
-                op_errno = ENOMEM;
-                goto unwind;
-        }
-        frame->local = local;
-
-        if (!(args->loc && args->loc->inode))
-                goto unwind;
-
-        local->fd = fd_ref (args->fd);
-        loc_copy (&local->loc, args->loc);
-        loc_path (&local->loc, NULL);
-
-        if (!gf_uuid_is_null (args->loc->inode->gfid))
-                memcpy (req.gfid,  args->loc->inode->gfid, 16);
-        else
-                memcpy (req.gfid, args->loc->gfid, 16);
-
-        GF_ASSERT_AND_GOTO_WITH_ERROR (this->name,
-                                       !gf_uuid_is_null (*((uuid_t*)req.gfid)),
-                                       unwind, op_errno, EINVAL);
-
-        conf = this->private;
-
-        GF_PROTOCOL_DICT_SERIALIZE (this, args->xdata, (&req.xdata.xdata_val),
-                                    req.xdata.xdata_len, op_errno, unwind);
-
-        ret = client_submit_request (this, &req, frame, conf->fops,
-                                     GFS3_OP_OPENDIR, client3_3_opendir_cbk,
-                                     NULL, NULL, 0, NULL, 0, NULL,
-                                     (xdrproc_t)xdr_gfs3_opendir_req);
-        if (ret) {
-                gf_log (this->name, GF_LOG_WARNING, "failed to send the fop");
-        }
-
-        GF_FREE (req.xdata.xdata_val);
-
-        return 0;
+    clnt_local_t *local = NULL;
+    clnt_conf_t *conf = NULL;
+    clnt_args_t *args = NULL;
+    gfs3_opendir_req req = {
+        {
+            0,
+        },
+    };
+    int ret = 0;
+    int op_errno = ESTALE;
+
+    if (!frame || !this || !data)
+        goto unwind;
+
+    args = data;
+    conf = this->private;
+
+    local = mem_get0(this->local_pool);
+    if (!local) {
+        op_errno = ENOMEM;
+        goto unwind;
+    }
+    frame->local = local;
+
+    local->fd = fd_ref(args->fd);
+    loc_copy(&local->loc, args->loc);
+    loc_path(&local->loc, NULL);
+
+    ret = client_pre_opendir(this, &req, args->loc, args->fd, args->xdata);
+    if (ret) {
+        op_errno = -ret;
+        goto unwind;
+    }
+    ret = client_submit_request(this, &req, frame, conf->fops, GFS3_OP_OPENDIR,
+                                client3_3_opendir_cbk, NULL,
+                                (xdrproc_t)xdr_gfs3_opendir_req);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_WARNING, 0, PC_MSG_FOP_SEND_FAILED, NULL);
+    }
+
+    GF_FREE(req.xdata.xdata_val);
+
+    return 0;
 
 unwind:
-        CLIENT_STACK_UNWIND (opendir, frame, -1, op_errno, NULL, NULL);
+    CLIENT_STACK_UNWIND(opendir, frame, -1, op_errno, NULL, NULL);
 
-        GF_FREE (req.xdata.xdata_val);
+    GF_FREE(req.xdata.xdata_val);
 
-        return 0;
+    return 0;
 }
 
-
-
 int32_t
-client3_3_fsyncdir (call_frame_t *frame, xlator_t *this, void *data)
+client3_3_fsyncdir(call_frame_t *frame, xlator_t *this, void *data)
 {
-        clnt_args_t       *args      = NULL;
-        int64_t            remote_fd = -1;
-        clnt_conf_t       *conf      = NULL;
-        gfs3_fsyncdir_req  req       = {{0,},};
-        int                ret       = 0;
-        int32_t            op_errno  = ESTALE;
-
-        if (!frame || !this || !data)
-                goto unwind;
-
-        args = data;
-        conf = this->private;
-
-        CLIENT_GET_REMOTE_FD (this, args->fd, DEFAULT_REMOTE_FD,
-                              remote_fd, op_errno, unwind);
-
-        req.fd   = remote_fd;
-        req.data = args->flags;
-        memcpy (req.gfid, args->fd->inode->gfid, 16);
-
-        conf = this->private;
-
-        GF_PROTOCOL_DICT_SERIALIZE (this, args->xdata, (&req.xdata.xdata_val),
-                                    req.xdata.xdata_len, op_errno, unwind);
-
-        ret = client_submit_request (this, &req, frame, conf->fops,
-                                     GFS3_OP_FSYNCDIR, client3_3_fsyncdir_cbk,
-                                     NULL, NULL, 0,
-                                     NULL, 0, NULL,
-                                     (xdrproc_t)xdr_gfs3_fsyncdir_req);
-        if (ret) {
-                gf_log (this->name, GF_LOG_WARNING, "failed to send the fop");
-        }
-
-        GF_FREE (req.xdata.xdata_val);
-
-        return 0;
+    clnt_args_t *args = NULL;
+    clnt_conf_t *conf = NULL;
+    gfs3_fsyncdir_req req = {
+        {
+            0,
+        },
+    };
+    int ret = 0;
+    int32_t op_errno = ESTALE;
+
+    if (!frame || !this || !data)
+        goto unwind;
+
+    args = data;
+    conf = this->private;
+
+    ret = client_pre_fsyncdir(this, &req, args->fd, args->flags, args->xdata);
+    if (ret) {
+        op_errno = -ret;
+        goto unwind;
+    }
+    ret = client_submit_request(this, &req, frame, conf->fops, GFS3_OP_FSYNCDIR,
+                                client3_3_fsyncdir_cbk, NULL,
+                                (xdrproc_t)xdr_gfs3_fsyncdir_req);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_WARNING, 0, PC_MSG_FOP_SEND_FAILED, NULL);
+    }
+
+    GF_FREE(req.xdata.xdata_val);
+
+    return 0;
 
 unwind:
-        CLIENT_STACK_UNWIND (fsyncdir, frame, -1, op_errno, NULL);
-        GF_FREE (req.xdata.xdata_val);
+    CLIENT_STACK_UNWIND(fsyncdir, frame, -1, op_errno, NULL);
+    GF_FREE(req.xdata.xdata_val);
 
-        return 0;
+    return 0;
 }
 
-
-
 int32_t
-client3_3_statfs (call_frame_t *frame, xlator_t *this,
-                  void *data)
+client3_3_statfs(call_frame_t *frame, xlator_t *this, void *data)
 {
-        clnt_conf_t     *conf     = NULL;
-        clnt_args_t     *args     = NULL;
-        gfs3_statfs_req  req      = {{0,},};
-        int              ret      = 0;
-        int              op_errno = ESTALE;
-
-        if (!frame || !this || !data)
-                goto unwind;
-
-        args = data;
-
-        if (!args->loc)
-                goto unwind;
-
-        if (args->loc->inode) {
-                if (!gf_uuid_is_null (args->loc->inode->gfid))
-                        memcpy (req.gfid,  args->loc->inode->gfid, 16);
-                else
-                        memcpy (req.gfid, args->loc->gfid, 16);
-        } else
-                req.gfid[15] = 1;
+    clnt_conf_t *conf = NULL;
+    clnt_args_t *args = NULL;
+    gfs3_statfs_req req = {
+        {
+            0,
+        },
+    };
+    int ret = 0;
+    int op_errno = ESTALE;
 
-        GF_ASSERT_AND_GOTO_WITH_ERROR (this->name,
-                                       !gf_uuid_is_null (*((uuid_t*)req.gfid)),
-                                       unwind, op_errno, EINVAL);
+    if (!frame || !this || !data)
+        goto unwind;
 
-        conf = this->private;
+    args = data;
 
-        GF_PROTOCOL_DICT_SERIALIZE (this, args->xdata, (&req.xdata.xdata_val),
-                                    req.xdata.xdata_len, op_errno, unwind);
+    conf = this->private;
 
-        ret = client_submit_request (this, &req, frame, conf->fops,
-                                     GFS3_OP_STATFS, client3_3_statfs_cbk, NULL,
-                                     NULL, 0, NULL, 0,
-                                     NULL, (xdrproc_t)xdr_gfs3_statfs_req);
-        if (ret) {
-                gf_log (this->name, GF_LOG_WARNING, "failed to send the fop");
-        }
+    ret = client_pre_statfs(this, &req, args->loc, args->xdata);
+    if (ret) {
+        op_errno = -ret;
+        goto unwind;
+    }
+    ret = client_submit_request(this, &req, frame, conf->fops, GFS3_OP_STATFS,
+                                client3_3_statfs_cbk, NULL,
+                                (xdrproc_t)xdr_gfs3_statfs_req);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_WARNING, 0, PC_MSG_FOP_SEND_FAILED, NULL);
+    }
 
-        GF_FREE (req.xdata.xdata_val);
+    GF_FREE(req.xdata.xdata_val);
 
-        return 0;
+    return 0;
 
 unwind:
-        CLIENT_STACK_UNWIND (statfs, frame, -1, op_errno, NULL, NULL);
-        GF_FREE (req.xdata.xdata_val);
+    CLIENT_STACK_UNWIND(statfs, frame, -1, op_errno, NULL, NULL);
+    GF_FREE(req.xdata.xdata_val);
 
-        return 0;
+    return 0;
 }
 
-
-
 int32_t
-client3_3_setxattr (call_frame_t *frame, xlator_t *this,
-                    void *data)
+client3_3_setxattr(call_frame_t *frame, xlator_t *this, void *data)
 {
-        clnt_conf_t       *conf     = NULL;
-        clnt_args_t       *args     = NULL;
-        gfs3_setxattr_req  req      = {{0,},};
-        int                ret      = 0;
-        int                op_errno = ESTALE;
-
-        if (!frame || !this || !data)
-                goto unwind;
-
-        args = data;
-
-        if (!(args->loc && args->loc->inode))
-                goto unwind;
-
-        if (!gf_uuid_is_null (args->loc->inode->gfid))
-                memcpy (req.gfid,  args->loc->inode->gfid, 16);
-        else
-                memcpy (req.gfid, args->loc->gfid, 16);
-
-        GF_ASSERT_AND_GOTO_WITH_ERROR (this->name,
-                                       !gf_uuid_is_null (*((uuid_t*)req.gfid)),
-                                       unwind, op_errno, EINVAL);
-        if (args->xattr) {
-                GF_PROTOCOL_DICT_SERIALIZE (this, args->xattr,
-                                            (&req.dict.dict_val),
-                                            req.dict.dict_len,
-                                            op_errno, unwind);
-        }
-
-        req.flags = args->flags;
-
-        conf = this->private;
-
-        GF_PROTOCOL_DICT_SERIALIZE (this, args->xdata, (&req.xdata.xdata_val),
-                                    req.xdata.xdata_len, op_errno, unwind);
-
-        ret = client_submit_request (this, &req, frame, conf->fops,
-                                     GFS3_OP_SETXATTR, client3_3_setxattr_cbk,
-                                     NULL, NULL, 0, NULL, 0, NULL,
-                                     (xdrproc_t)xdr_gfs3_setxattr_req);
-        if (ret) {
-                gf_log (this->name, GF_LOG_WARNING, "failed to send the fop");
-        }
-        GF_FREE (req.dict.dict_val);
-
-        GF_FREE (req.xdata.xdata_val);
-
-        return 0;
+    clnt_conf_t *conf = NULL;
+    clnt_args_t *args = NULL;
+    gfs3_setxattr_req req = {
+        {
+            0,
+        },
+    };
+    int ret = 0;
+    int op_errno = ESTALE;
+
+    if (!frame || !this || !data)
+        goto unwind;
+
+    args = data;
+    conf = this->private;
+
+    ret = client_pre_setxattr(this, &req, args->loc, args->xattr, args->flags,
+                              args->xdata);
+    if (ret) {
+        op_errno = -ret;
+        goto unwind;
+    }
+    ret = client_submit_request(this, &req, frame, conf->fops, GFS3_OP_SETXATTR,
+                                client3_3_setxattr_cbk, NULL,
+                                (xdrproc_t)xdr_gfs3_setxattr_req);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_WARNING, 0, PC_MSG_FOP_SEND_FAILED, NULL);
+    }
+    GF_FREE(req.dict.dict_val);
+
+    GF_FREE(req.xdata.xdata_val);
+
+    return 0;
 unwind:
-        CLIENT_STACK_UNWIND (setxattr, frame, -1, op_errno, NULL);
-        GF_FREE (req.dict.dict_val);
+    CLIENT_STACK_UNWIND(setxattr, frame, -1, op_errno, NULL);
+    GF_FREE(req.dict.dict_val);
 
-        GF_FREE (req.xdata.xdata_val);
+    GF_FREE(req.xdata.xdata_val);
 
-        return 0;
+    return 0;
 }
 
-
-
 int32_t
-client3_3_fsetxattr (call_frame_t *frame, xlator_t *this,
-                     void *data)
+client3_3_fsetxattr(call_frame_t *frame, xlator_t *this, void *data)
 {
-        clnt_args_t        *args     = NULL;
-        int64_t             remote_fd = -1;
-        clnt_conf_t        *conf     = NULL;
-        gfs3_fsetxattr_req  req      = {{0,},};
-        int                 op_errno = ESTALE;
-        int                 ret      = 0;
-
-        if (!frame || !this || !data)
-                goto unwind;
-
-        args = data;
-        conf = this->private;
-
-        CLIENT_GET_REMOTE_FD (this, args->fd, DEFAULT_REMOTE_FD,
-                              remote_fd, op_errno, unwind);
-
-        req.fd    = remote_fd;
-        req.flags = args->flags;
-        memcpy (req.gfid, args->fd->inode->gfid, 16);
-
-        if (args->xattr) {
-                GF_PROTOCOL_DICT_SERIALIZE (this, args->xattr,
-                                            (&req.dict.dict_val),
-                                            req.dict.dict_len,
-                                            op_errno, unwind);
-        }
-
-        GF_PROTOCOL_DICT_SERIALIZE (this, args->xdata, (&req.xdata.xdata_val),
-                                    req.xdata.xdata_len, op_errno, unwind);
-
-        ret = client_submit_request (this, &req, frame, conf->fops,
-                                     GFS3_OP_FSETXATTR, client3_3_fsetxattr_cbk,
-                                     NULL, NULL, 0, NULL, 0, NULL,
-                                     (xdrproc_t)xdr_gfs3_fsetxattr_req);
-        if (ret) {
-                gf_log (this->name, GF_LOG_WARNING, "failed to send the fop");
-        }
-
-        GF_FREE (req.dict.dict_val);
-
-        GF_FREE (req.xdata.xdata_val);
-
-        return 0;
+    clnt_args_t *args = NULL;
+    clnt_conf_t *conf = NULL;
+    gfs3_fsetxattr_req req = {
+        {
+            0,
+        },
+    };
+    int op_errno = ESTALE;
+    int ret = 0;
+
+    if (!frame || !this || !data)
+        goto unwind;
+
+    args = data;
+    conf = this->private;
+
+    ret = client_pre_fsetxattr(this, &req, args->fd, args->flags, args->xattr,
+                               args->xdata);
+    if (ret) {
+        op_errno = -ret;
+        goto unwind;
+    }
+    ret = client_submit_request(this, &req, frame, conf->fops,
+                                GFS3_OP_FSETXATTR, client3_3_fsetxattr_cbk,
+                                NULL, (xdrproc_t)xdr_gfs3_fsetxattr_req);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_WARNING, 0, PC_MSG_FOP_SEND_FAILED, NULL);
+    }
+
+    GF_FREE(req.dict.dict_val);
+
+    GF_FREE(req.xdata.xdata_val);
+
+    return 0;
 unwind:
-        CLIENT_STACK_UNWIND (fsetxattr, frame, -1, op_errno, NULL);
-        GF_FREE (req.dict.dict_val);
+    CLIENT_STACK_UNWIND(fsetxattr, frame, -1, op_errno, NULL);
+    GF_FREE(req.dict.dict_val);
 
-        GF_FREE (req.xdata.xdata_val);
+    GF_FREE(req.xdata.xdata_val);
 
-        return 0;
+    return 0;
 }
 
-
-
-
 int32_t
-client3_3_fgetxattr (call_frame_t *frame, xlator_t *this,
-                     void *data)
+client3_3_fgetxattr(call_frame_t *frame, xlator_t *this, void *data)
 {
-        clnt_args_t        *args       = NULL;
-        int64_t             remote_fd  = -1;
-        clnt_conf_t        *conf       = NULL;
-        gfs3_fgetxattr_req  req        = {{0,},};
-        int                 op_errno   = ESTALE;
-        int                 ret        = 0;
-        int                 count      = 0;
-        clnt_local_t       *local      = NULL;
-        struct iobref      *rsp_iobref = NULL;
-        struct iobuf       *rsp_iobuf  = NULL;
-        struct iovec       *rsphdr     = NULL;
-        struct iovec        vector[MAX_IOVEC] = {{0}, };
-
-        if (!frame || !this || !data)
-                goto unwind;
-
-        args = data;
-        conf = this->private;
-
-        CLIENT_GET_REMOTE_FD (this, args->fd, DEFAULT_REMOTE_FD,
-                              remote_fd, op_errno, unwind);
-
-        local = mem_get0 (this->local_pool);
-        if (!local) {
-                op_errno = ENOMEM;
-                goto unwind;
-        }
-        frame->local = local;
-
-        rsp_iobref = iobref_new ();
-        if (rsp_iobref == NULL) {
-                op_errno = ENOMEM;
-                goto unwind;
-        }
-
-        /* TODO: what is the size we should send ? */
-        rsp_iobuf = iobuf_get2 (this->ctx->iobuf_pool, 8 * GF_UNIT_KB);
-        if (rsp_iobuf == NULL) {
-                op_errno = ENOMEM;
-                goto unwind;
-        }
-
-        iobref_add (rsp_iobref, rsp_iobuf);
-        iobuf_unref (rsp_iobuf);
-
-        rsphdr = &vector[0];
-        rsphdr->iov_base = iobuf_ptr (rsp_iobuf);
-        rsphdr->iov_len = iobuf_pagesize (rsp_iobuf);;
-        count = 1;
-        local->iobref = rsp_iobref;
-        rsp_iobuf = NULL;
-        rsp_iobref = NULL;
-
-        req.namelen = 1; /* Use it as a flag */
-        req.fd   = remote_fd;
-        req.name = (char *)args->name;
-        if (!req.name) {
-                req.name = "";
-                req.namelen = 0;
-        }
-        memcpy (req.gfid, args->fd->inode->gfid, 16);
-
-        GF_PROTOCOL_DICT_SERIALIZE (this, args->xdata, (&req.xdata.xdata_val),
-                                    req.xdata.xdata_len, op_errno, unwind);
-
-        ret = client_submit_request (this, &req, frame, conf->fops,
-                                     GFS3_OP_FGETXATTR,
-                                     client3_3_fgetxattr_cbk, NULL,
-                                     rsphdr, count,
-                                     NULL, 0, local->iobref,
-                                     (xdrproc_t)xdr_gfs3_fgetxattr_req);
-        if (ret) {
-                gf_log (this->name, GF_LOG_WARNING, "failed to send the fop");
-        }
-
-        GF_FREE (req.xdata.xdata_val);
-
-        return 0;
+    clnt_args_t *args = NULL;
+    clnt_conf_t *conf = NULL;
+    gfs3_fgetxattr_req req = {
+        {
+            0,
+        },
+    };
+    int op_errno = ESTALE;
+    int ret = 0;
+    int count = 0;
+    clnt_local_t *local = NULL;
+    struct iobref *rsp_iobref = NULL;
+    struct iobuf *rsp_iobuf = NULL;
+    struct iovec *rsphdr = NULL;
+    struct iovec vector[MAX_IOVEC] = {
+        {0},
+    };
+    client_payload_t cp;
+
+    if (!frame || !this || !data)
+        goto unwind;
+
+    args = data;
+    conf = this->private;
+
+    local = mem_get0(this->local_pool);
+    if (!local) {
+        op_errno = ENOMEM;
+        goto unwind;
+    }
+    frame->local = local;
+
+    rsp_iobref = iobref_new();
+    if (rsp_iobref == NULL) {
+        op_errno = ENOMEM;
+        goto unwind;
+    }
+
+    /* TODO: what is the size we should send ? */
+    rsp_iobuf = iobuf_get2(this->ctx->iobuf_pool, 8 * GF_UNIT_KB);
+    if (rsp_iobuf == NULL) {
+        op_errno = ENOMEM;
+        goto unwind;
+    }
+
+    rsphdr = &vector[0];
+    rsphdr->iov_base = iobuf_ptr(rsp_iobuf);
+    rsphdr->iov_len = iobuf_pagesize(rsp_iobuf);
+    ;
+    count = 1;
+    local->iobref = rsp_iobref;
+    iobref_add(rsp_iobref, rsp_iobuf);
+    iobuf_unref(rsp_iobuf);
+    rsp_iobuf = NULL;
+    rsp_iobref = NULL;
+
+    ret = client_pre_fgetxattr(this, &req, args->fd, args->name, args->xdata);
+    if (ret) {
+        op_errno = -ret;
+        goto unwind;
+    }
+
+    memset(&cp, 0, sizeof(client_payload_t));
+    cp.rsphdr = rsphdr;
+    cp.rsphdr_cnt = count;
+    cp.rsp_iobref = local->iobref;
+    ret = client_submit_request(this, &req, frame, conf->fops,
+                                GFS3_OP_FGETXATTR, client3_3_fgetxattr_cbk, &cp,
+                                (xdrproc_t)xdr_gfs3_fgetxattr_req);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_WARNING, 0, PC_MSG_FOP_SEND_FAILED, NULL);
+    }
+
+    GF_FREE(req.xdata.xdata_val);
+
+    return 0;
 unwind:
-        CLIENT_STACK_UNWIND (fgetxattr, frame, -1, op_errno, NULL, NULL);
-
-        if (rsp_iobuf)
-                iobuf_unref (rsp_iobuf);
+    CLIENT_STACK_UNWIND(fgetxattr, frame, -1, op_errno, NULL, NULL);
 
-        if (rsp_iobref)
-                iobref_unref (rsp_iobref);
+    if (rsp_iobref)
+        iobref_unref(rsp_iobref);
 
-        GF_FREE (req.xdata.xdata_val);
+    GF_FREE(req.xdata.xdata_val);
 
-        return 0;
+    return 0;
 }
 
-
-
 int32_t
-client3_3_getxattr (call_frame_t *frame, xlator_t *this,
-                    void *data)
+client3_3_getxattr(call_frame_t *frame, xlator_t *this, void *data)
 {
-        clnt_conf_t       *conf       = NULL;
-        clnt_args_t       *args       = NULL;
-        gfs3_getxattr_req  req        = {{0,},};
-        dict_t            *dict       = NULL;
-        int                ret        = 0;
-        int32_t            op_ret     = -1;
-        int                op_errno   = ESTALE;
-        int                count      = 0;
-        clnt_local_t      *local      = NULL;
-        struct iobref     *rsp_iobref = NULL;
-        struct iobuf      *rsp_iobuf  = NULL;
-        struct iovec      *rsphdr     = NULL;
-        struct iovec       vector[MAX_IOVEC] = {{0}, };
-
-        if (!frame || !this || !data) {
-                op_errno = 0;
-                goto unwind;
-        }
-        args = data;
-
-        if (!args->loc) {
-                op_errno = EINVAL;
-                goto unwind;
-        }
-
-        local = mem_get0 (this->local_pool);
-        if (!local) {
-                op_errno = ENOMEM;
-                goto unwind;
-        }
-
-        loc_copy (&local->loc, args->loc);
-        loc_path (&local->loc, NULL);
-        if (args->name)
-                local->name = gf_strdup (args->name);
-
-        frame->local = local;
-
-        rsp_iobref = iobref_new ();
-        if (rsp_iobref == NULL) {
+    clnt_conf_t *conf = NULL;
+    clnt_args_t *args = NULL;
+    gfs3_getxattr_req req = {
+        {
+            0,
+        },
+    };
+    dict_t *dict = NULL;
+    int ret = 0;
+    int32_t op_ret = -1;
+    int op_errno = ESTALE;
+    int count = 0;
+    clnt_local_t *local = NULL;
+    struct iobref *rsp_iobref = NULL;
+    struct iobuf *rsp_iobuf = NULL;
+    struct iovec *rsphdr = NULL;
+    struct iovec vector[MAX_IOVEC] = {
+        {0},
+    };
+    client_payload_t cp;
+
+    if (!frame || !this || !data) {
+        op_errno = 0;
+        goto unwind;
+    }
+
+    args = data;
+
+    local = mem_get0(this->local_pool);
+    if (!local) {
+        op_errno = ENOMEM;
+        goto unwind;
+    }
+
+    frame->local = local;
+
+    loc_copy(&local->loc, args->loc);
+    loc_path(&local->loc, NULL);
+
+    if (args->name)
+        local->name = gf_strdup(args->name);
+
+    rsp_iobref = iobref_new();
+    if (rsp_iobref == NULL) {
+        op_errno = ENOMEM;
+        goto unwind;
+    }
+
+    /* TODO: what is the size we should send ? */
+    rsp_iobuf = iobuf_get2(this->ctx->iobuf_pool, 8 * GF_UNIT_KB);
+    if (rsp_iobuf == NULL) {
+        op_errno = ENOMEM;
+        goto unwind;
+    }
+
+    rsphdr = &vector[0];
+    rsphdr->iov_base = iobuf_ptr(rsp_iobuf);
+    rsphdr->iov_len = iobuf_pagesize(rsp_iobuf);
+    count = 1;
+    local->iobref = rsp_iobref;
+    iobref_add(rsp_iobref, rsp_iobuf);
+    iobuf_unref(rsp_iobuf);
+    rsp_iobuf = NULL;
+    rsp_iobref = NULL;
+
+    conf = this->private;
+
+    if (args && args->name) {
+        if (is_client_dump_locks_cmd((char *)args->name)) {
+            dict = dict_new();
+
+            if (!dict) {
                 op_errno = ENOMEM;
                 goto unwind;
-        }
+            }
 
-        /* TODO: what is the size we should send ? */
-        rsp_iobuf = iobuf_get2 (this->ctx->iobuf_pool, 8 * GF_UNIT_KB);
-        if (rsp_iobuf == NULL) {
+            ret = client_dump_locks((char *)args->name, args->loc->inode, dict);
+            if (ret) {
+                gf_smsg(this->name, GF_LOG_WARNING, EINVAL,
+                        PC_MSG_CLIENT_DUMP_LOCKS_FAILED, NULL);
                 op_errno = ENOMEM;
                 goto unwind;
-        }
-
-        iobref_add (rsp_iobref, rsp_iobuf);
-        iobuf_unref (rsp_iobuf);
-
-        rsphdr = &vector[0];
-        rsphdr->iov_base = iobuf_ptr (rsp_iobuf);
-        rsphdr->iov_len = iobuf_pagesize (rsp_iobuf);
-        count = 1;
-        local->iobref = rsp_iobref;
-        rsp_iobuf = NULL;
-        rsp_iobref = NULL;
-
-        if (args->loc->inode && !gf_uuid_is_null (args->loc->inode->gfid))
-                memcpy (req.gfid,  args->loc->inode->gfid, 16);
-        else
-                memcpy (req.gfid, args->loc->gfid, 16);
-
-        GF_ASSERT_AND_GOTO_WITH_ERROR (this->name,
-                                       !gf_uuid_is_null (*((uuid_t*)req.gfid)),
-                                       unwind, op_errno, EINVAL);
-        req.namelen = 1; /* Use it as a flag */
-
-        req.name = (char *)args->name;
-        if (!req.name) {
-                req.name = "";
-                req.namelen = 0;
-        }
-
-        conf = this->private;
-
-        if (args && args->name) {
-                if (is_client_dump_locks_cmd ((char *)args->name)) {
-                        dict = dict_new ();
-                        ret = client_dump_locks ((char *)args->name,
-                                                 args->loc->inode,
-                                                 dict);
-                        if (ret) {
-                                gf_log (this->name, GF_LOG_WARNING,
-                                        "Client dump locks failed");
-                                op_errno = EINVAL;
-                        }
-
-                        GF_ASSERT (dict);
-                        op_ret = 0;
-                        op_errno = 0;
-                        goto unwind;
-                }
-        }
-
-        GF_PROTOCOL_DICT_SERIALIZE (this, args->xdata, (&req.xdata.xdata_val),
-                                    req.xdata.xdata_len, op_errno, unwind);
-
-        ret = client_submit_request (this, &req, frame, conf->fops,
-                                     GFS3_OP_GETXATTR,
-                                     client3_3_getxattr_cbk, NULL,
-                                     rsphdr, count,
-                                     NULL, 0, local->iobref,
-                                     (xdrproc_t)xdr_gfs3_getxattr_req);
-        if (ret) {
-                gf_log (this->name, GF_LOG_WARNING, "failed to send the fop");
-        }
-
-        GF_FREE (req.xdata.xdata_val);
-
-        return 0;
+            }
+
+            GF_ASSERT(dict);
+            op_ret = 0;
+            op_errno = 0;
+            goto unwind;
+        }
+    }
+
+    ret = client_pre_getxattr(this, &req, args->loc, args->name, args->xdata);
+    if (ret) {
+        op_errno = -ret;
+        goto unwind;
+    }
+
+    memset(&cp, 0, sizeof(client_payload_t));
+    cp.rsphdr = rsphdr;
+    cp.rsphdr_cnt = count;
+    cp.rsp_iobref = local->iobref;
+    ret = client_submit_request(this, &req, frame, conf->fops, GFS3_OP_GETXATTR,
+                                client3_3_getxattr_cbk, &cp,
+                                (xdrproc_t)xdr_gfs3_getxattr_req);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_WARNING, 0, PC_MSG_FOP_SEND_FAILED, NULL);
+    }
+
+    GF_FREE(req.xdata.xdata_val);
+
+    return 0;
 unwind:
-        if (rsp_iobuf)
-                iobuf_unref (rsp_iobuf);
+    if (rsp_iobref)
+        iobref_unref(rsp_iobref);
 
-        if (rsp_iobref)
-                iobref_unref (rsp_iobref);
+    CLIENT_STACK_UNWIND(getxattr, frame, op_ret, op_errno, dict, NULL);
 
-        CLIENT_STACK_UNWIND (getxattr, frame, op_ret, op_errno, dict, NULL);
+    if (dict) {
+        dict_unref(dict);
+    }
 
-        GF_FREE (req.xdata.xdata_val);
+    GF_FREE(req.xdata.xdata_val);
 
-        return 0;
+    return 0;
 }
 
-
-
 int32_t
-client3_3_xattrop (call_frame_t *frame, xlator_t *this,
-                   void *data)
+client3_3_xattrop(call_frame_t *frame, xlator_t *this, void *data)
 {
-        clnt_conf_t      *conf       = NULL;
-        clnt_args_t      *args       = NULL;
-        gfs3_xattrop_req  req        = {{0,},};
-        int               ret        = 0;
-        int               op_errno   = ESTALE;
-        int               count      = 0;
-        clnt_local_t   *local      = NULL;
-        struct iobref    *rsp_iobref = NULL;
-        struct iobuf     *rsp_iobuf  = NULL;
-        struct iovec     *rsphdr     = NULL;
-        struct iovec      vector[MAX_IOVEC] = {{0}, };
-
-        if (!frame || !this || !data)
-                goto unwind;
-
-        args = data;
-
-        if (!(args->loc && args->loc->inode))
-                goto unwind;
-
-        local = mem_get0 (this->local_pool);
-        if (!local) {
-                op_errno = ENOMEM;
-                goto unwind;
-        }
-        frame->local = local;
-
-        rsp_iobref = iobref_new ();
-        if (rsp_iobref == NULL) {
-                op_errno = ENOMEM;
-                goto unwind;
-        }
-
-        /* TODO: what is the size we should send ? */
-        rsp_iobuf = iobuf_get2 (this->ctx->iobuf_pool, 8 * GF_UNIT_KB);
-        if (rsp_iobuf == NULL) {
-                op_errno = ENOMEM;
-                goto unwind;
-        }
-
-        iobref_add (rsp_iobref, rsp_iobuf);
-        iobuf_unref (rsp_iobuf);
-
-        rsphdr = &vector[0];
-        rsphdr->iov_base = iobuf_ptr (rsp_iobuf);
-        rsphdr->iov_len = iobuf_pagesize (rsp_iobuf);
-        count = 1;
-        local->iobref = rsp_iobref;
-        rsp_iobuf = NULL;
-        rsp_iobref = NULL;
-
-        if (!gf_uuid_is_null (args->loc->inode->gfid))
-                memcpy (req.gfid,  args->loc->inode->gfid, 16);
-        else
-                memcpy (req.gfid, args->loc->gfid, 16);
-
-        loc_copy (&local->loc, args->loc);
-        loc_path (&local->loc, NULL);
-        GF_ASSERT_AND_GOTO_WITH_ERROR (this->name,
-                                       !gf_uuid_is_null (*((uuid_t*)req.gfid)),
-                                       unwind, op_errno, EINVAL);
-        if (args->xattr) {
-                GF_PROTOCOL_DICT_SERIALIZE (this, args->xattr,
-                                            (&req.dict.dict_val),
-                                            req.dict.dict_len,
-                                            op_errno, unwind);
-        }
-
-        req.flags = args->flags;
-
-        conf = this->private;
-
-        GF_PROTOCOL_DICT_SERIALIZE (this, args->xdata, (&req.xdata.xdata_val),
-                                    req.xdata.xdata_len, op_errno, unwind);
-
-        ret = client_submit_request (this, &req, frame, conf->fops,
-                                     GFS3_OP_XATTROP,
-                                     client3_3_xattrop_cbk, NULL,
-                                     rsphdr, count,
-                                     NULL, 0, local->iobref,
-                                     (xdrproc_t)xdr_gfs3_xattrop_req);
-        if (ret) {
-                gf_log (this->name, GF_LOG_WARNING, "failed to send the fop");
-        }
-
-        GF_FREE (req.dict.dict_val);
-
-        GF_FREE (req.xdata.xdata_val);
-
-        return 0;
+    clnt_conf_t *conf = NULL;
+    clnt_args_t *args = NULL;
+    gfs3_xattrop_req req = {
+        {
+            0,
+        },
+    };
+    int ret = 0;
+    int op_errno = ESTALE;
+    int count = 0;
+    clnt_local_t *local = NULL;
+    struct iobref *rsp_iobref = NULL;
+    struct iobuf *rsp_iobuf = NULL;
+    struct iovec *rsphdr = NULL;
+    struct iovec vector[MAX_IOVEC] = {
+        {0},
+    };
+    client_payload_t cp;
+
+    if (!frame || !this || !data)
+        goto unwind;
+
+    args = data;
+
+    if (!(args->loc && args->loc->inode))
+        goto unwind;
+
+    local = mem_get0(this->local_pool);
+    if (!local) {
+        op_errno = ENOMEM;
+        goto unwind;
+    }
+    frame->local = local;
+
+    rsp_iobref = iobref_new();
+    if (rsp_iobref == NULL) {
+        op_errno = ENOMEM;
+        goto unwind;
+    }
+
+    /* TODO: what is the size we should send ? */
+    rsp_iobuf = iobuf_get2(this->ctx->iobuf_pool, 8 * GF_UNIT_KB);
+    if (rsp_iobuf == NULL) {
+        op_errno = ENOMEM;
+        goto unwind;
+    }
+
+    rsphdr = &vector[0];
+    rsphdr->iov_base = iobuf_ptr(rsp_iobuf);
+    rsphdr->iov_len = iobuf_pagesize(rsp_iobuf);
+    count = 1;
+    local->iobref = rsp_iobref;
+    iobref_add(rsp_iobref, rsp_iobuf);
+    iobuf_unref(rsp_iobuf);
+    rsp_iobuf = NULL;
+    rsp_iobref = NULL;
+
+    loc_copy(&local->loc, args->loc);
+    loc_path(&local->loc, NULL);
+    conf = this->private;
+
+    ret = client_pre_xattrop(this, &req, args->loc, args->xattr, args->flags,
+                             args->xdata);
+    if (ret) {
+        op_errno = -ret;
+        goto unwind;
+    }
+
+    memset(&cp, 0, sizeof(client_payload_t));
+    cp.rsphdr = rsphdr;
+    cp.rsphdr_cnt = count;
+    cp.rsp_iobref = local->iobref;
+    ret = client_submit_request(this, &req, frame, conf->fops, GFS3_OP_XATTROP,
+                                client3_3_xattrop_cbk, &cp,
+                                (xdrproc_t)xdr_gfs3_xattrop_req);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_WARNING, 0, PC_MSG_FOP_SEND_FAILED, NULL);
+    }
+
+    GF_FREE(req.dict.dict_val);
+
+    GF_FREE(req.xdata.xdata_val);
+
+    return 0;
 unwind:
-        CLIENT_STACK_UNWIND (xattrop, frame, -1, op_errno, NULL, NULL);
-
-        GF_FREE (req.dict.dict_val);
+    CLIENT_STACK_UNWIND(xattrop, frame, -1, op_errno, NULL, NULL);
 
-        if (rsp_iobuf)
-                iobuf_unref (rsp_iobuf);
+    GF_FREE(req.dict.dict_val);
 
-        if (rsp_iobref)
-                iobref_unref (rsp_iobref);
+    if (rsp_iobref)
+        iobref_unref(rsp_iobref);
 
-        GF_FREE (req.xdata.xdata_val);
+    GF_FREE(req.xdata.xdata_val);
 
-        return 0;
+    return 0;
 }
 
-
-
 int32_t
-client3_3_fxattrop (call_frame_t *frame, xlator_t *this,
-                    void *data)
+client3_3_fxattrop(call_frame_t *frame, xlator_t *this, void *data)
 {
-        clnt_args_t       *args       = NULL;
-        int64_t            remote_fd  = -1;
-        clnt_conf_t       *conf       = NULL;
-        clnt_local_t      *local      = NULL;
-        gfs3_fxattrop_req  req        = {{0,},};
-        int                op_errno   = ESTALE;
-        int                ret        = 0;
-        int                count      = 0;
-        struct iobref     *rsp_iobref = NULL;
-        struct iobuf      *rsp_iobuf  = NULL;
-        struct iovec      *rsphdr     = NULL;
-        struct iovec       vector[MAX_IOVEC] = {{0}, };
-
-        if (!frame || !this || !data)
-                goto unwind;
-
-        args = data;
-        conf = this->private;
-
-        CLIENT_GET_REMOTE_FD (this, args->fd, FALLBACK_TO_ANON_FD,
-                              remote_fd, op_errno, unwind);
-        ret = client_fd_fop_prepare_local (frame, args->fd, remote_fd);
-        if (ret) {
-                op_errno = -ret;
-                goto unwind;
-        }
-
-        local = frame->local;
-
-        req.fd     = remote_fd;
-        req.flags  = args->flags;
-        memcpy (req.gfid, args->fd->inode->gfid, 16);
-
-        rsp_iobref = iobref_new ();
-        if (rsp_iobref == NULL) {
-                op_errno = ENOMEM;
-                goto unwind;
-        }
-
-        /* TODO: what is the size we should send ? */
-        rsp_iobuf = iobuf_get2 (this->ctx->iobuf_pool, 8 * GF_UNIT_KB);
-        if (rsp_iobuf == NULL) {
-                op_errno = ENOMEM;
-                goto unwind;
-        }
-
-        iobref_add (rsp_iobref, rsp_iobuf);
-        iobuf_unref (rsp_iobuf);
-        rsphdr = &vector[0];
-        rsphdr->iov_base = iobuf_ptr (rsp_iobuf);
-        rsphdr->iov_len = iobuf_pagesize (rsp_iobuf);
-        count = 1;
-        local->iobref = rsp_iobref;
-        rsp_iobuf = NULL;
-        rsp_iobref = NULL;
-
-        if (args->xattr) {
-                GF_PROTOCOL_DICT_SERIALIZE (this, args->xattr,
-                                            (&req.dict.dict_val),
-                                            req.dict.dict_len,
-                                            op_errno, unwind);
-        }
-
-        GF_PROTOCOL_DICT_SERIALIZE (this, args->xdata, (&req.xdata.xdata_val),
-                                    req.xdata.xdata_len, op_errno, unwind);
-
-        ret = client_submit_request (this, &req, frame, conf->fops,
-                                     GFS3_OP_FXATTROP,
-                                     client3_3_fxattrop_cbk, NULL,
-                                     rsphdr, count,
-                                     NULL, 0, local->iobref,
-                                     (xdrproc_t)xdr_gfs3_fxattrop_req);
-        if (ret) {
-                gf_log (this->name, GF_LOG_WARNING, "failed to send the fop");
-        }
-
-        GF_FREE (req.dict.dict_val);
-
-        GF_FREE (req.xdata.xdata_val);
-
-        return 0;
+    clnt_args_t *args = NULL;
+    clnt_conf_t *conf = NULL;
+    clnt_local_t *local = NULL;
+    gfs3_fxattrop_req req = {
+        {
+            0,
+        },
+    };
+    int op_errno = ESTALE;
+    int ret = 0;
+    int count = 0;
+    struct iobref *rsp_iobref = NULL;
+    struct iobuf *rsp_iobuf = NULL;
+    struct iovec *rsphdr = NULL;
+    struct iovec vector[MAX_IOVEC] = {
+        {0},
+    };
+    client_payload_t cp;
+
+    if (!frame || !this || !data)
+        goto unwind;
+
+    args = data;
+    conf = this->private;
+
+    ret = client_pre_fxattrop(this, &req, args->fd, args->xattr, args->flags,
+                              args->xdata);
+    if (ret) {
+        op_errno = -ret;
+        goto unwind;
+    }
+    ret = client_fd_fop_prepare_local(frame, args->fd, req.fd);
+    if (ret) {
+        op_errno = -ret;
+        goto unwind;
+    }
+
+    local = frame->local;
+
+    rsp_iobref = iobref_new();
+    if (rsp_iobref == NULL) {
+        op_errno = ENOMEM;
+        goto unwind;
+    }
+
+    /* TODO: what is the size we should send ? */
+    rsp_iobuf = iobuf_get2(this->ctx->iobuf_pool, 8 * GF_UNIT_KB);
+    if (rsp_iobuf == NULL) {
+        op_errno = ENOMEM;
+        goto unwind;
+    }
+
+    rsphdr = &vector[0];
+    rsphdr->iov_base = iobuf_ptr(rsp_iobuf);
+    rsphdr->iov_len = iobuf_pagesize(rsp_iobuf);
+    count = 1;
+    local->iobref = rsp_iobref;
+    iobref_add(rsp_iobref, rsp_iobuf);
+    iobuf_unref(rsp_iobuf);
+    rsp_iobuf = NULL;
+    rsp_iobref = NULL;
+
+    memset(&cp, 0, sizeof(client_payload_t));
+    cp.rsphdr = rsphdr;
+    cp.rsphdr_cnt = count;
+    cp.rsp_iobref = local->iobref;
+    ret = client_submit_request(this, &req, frame, conf->fops, GFS3_OP_FXATTROP,
+                                client3_3_fxattrop_cbk, &cp,
+                                (xdrproc_t)xdr_gfs3_fxattrop_req);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_WARNING, 0, PC_MSG_FOP_SEND_FAILED, NULL);
+    }
+
+    GF_FREE(req.dict.dict_val);
+
+    GF_FREE(req.xdata.xdata_val);
+
+    return 0;
 unwind:
-        CLIENT_STACK_UNWIND (fxattrop, frame, -1, op_errno, NULL, NULL);
-
-        GF_FREE (req.dict.dict_val);
+    CLIENT_STACK_UNWIND(fxattrop, frame, -1, op_errno, NULL, NULL);
 
-        if (rsp_iobref)
-                iobref_unref (rsp_iobref);
+    GF_FREE(req.dict.dict_val);
 
-        if (rsp_iobuf)
-                iobuf_unref (rsp_iobuf);
+    if (rsp_iobref)
+        iobref_unref(rsp_iobref);
 
-        GF_FREE (req.xdata.xdata_val);
+    GF_FREE(req.xdata.xdata_val);
 
-        return 0;
+    return 0;
 }
 
-
 int32_t
-client3_3_removexattr (call_frame_t *frame, xlator_t *this,
-                       void *data)
+client3_3_removexattr(call_frame_t *frame, xlator_t *this, void *data)
 {
-        clnt_conf_t          *conf     = NULL;
-        clnt_args_t          *args     = NULL;
-        gfs3_removexattr_req  req      = {{0,},};
-        int                   ret      = 0;
-        int                   op_errno = ESTALE;
-
-        if (!frame || !this || !data)
-                goto unwind;
-
-        args = data;
-
-        if (!(args->loc && args->loc->inode))
-                goto unwind;
-
-        if (!gf_uuid_is_null (args->loc->inode->gfid))
-                memcpy (req.gfid,  args->loc->inode->gfid, 16);
-        else
-                memcpy (req.gfid, args->loc->gfid, 16);
-
-        GF_ASSERT_AND_GOTO_WITH_ERROR (this->name,
-                                       !gf_uuid_is_null (*((uuid_t*)req.gfid)),
-                                       unwind, op_errno, EINVAL);
-        req.name = (char *)args->name;
-
-        conf = this->private;
-
-        GF_PROTOCOL_DICT_SERIALIZE (this, args->xdata, (&req.xdata.xdata_val),
-                                    req.xdata.xdata_len, op_errno, unwind);
-
-        ret = client_submit_request (this, &req, frame, conf->fops,
-                                     GFS3_OP_REMOVEXATTR,
-                                     client3_3_removexattr_cbk, NULL,
-                                     NULL, 0, NULL, 0, NULL,
-                                     (xdrproc_t)xdr_gfs3_removexattr_req);
-        if (ret) {
-                gf_log (this->name, GF_LOG_WARNING, "failed to send the fop");
-        }
-
-        GF_FREE (req.xdata.xdata_val);
-
-        return 0;
+    clnt_conf_t *conf = NULL;
+    clnt_args_t *args = NULL;
+    gfs3_removexattr_req req = {
+        {
+            0,
+        },
+    };
+    int ret = 0;
+    int op_errno = ESTALE;
+
+    if (!frame || !this || !data)
+        goto unwind;
+
+    args = data;
+    conf = this->private;
+
+    ret = client_pre_removexattr(this, &req, args->loc, args->name,
+                                 args->xdata);
+    if (ret) {
+        op_errno = -ret;
+        goto unwind;
+    }
+    ret = client_submit_request(this, &req, frame, conf->fops,
+                                GFS3_OP_REMOVEXATTR, client3_3_removexattr_cbk,
+                                NULL, (xdrproc_t)xdr_gfs3_removexattr_req);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_WARNING, 0, PC_MSG_FOP_SEND_FAILED, NULL);
+    }
+
+    GF_FREE(req.xdata.xdata_val);
+
+    return 0;
 unwind:
-        CLIENT_STACK_UNWIND (removexattr, frame, -1, op_errno, NULL);
-        GF_FREE (req.xdata.xdata_val);
+    CLIENT_STACK_UNWIND(removexattr, frame, -1, op_errno, NULL);
+    GF_FREE(req.xdata.xdata_val);
 
-        return 0;
+    return 0;
 }
 
 int32_t
-client3_3_fremovexattr (call_frame_t *frame, xlator_t *this,
-                        void *data)
+client3_3_fremovexattr(call_frame_t *frame, xlator_t *this, void *data)
 {
-        clnt_conf_t           *conf     = NULL;
-        clnt_args_t           *args     = NULL;
-        gfs3_fremovexattr_req  req      = {{0,},};
-        int                    ret      = 0;
-        int64_t                remote_fd = -1;
-        int                    op_errno = ESTALE;
-
-        if (!frame || !this || !data)
-                goto unwind;
-
-        args = data;
-
-        if (!(args->fd && args->fd->inode))
-                goto unwind;
-
-        conf = this->private;
-
-        CLIENT_GET_REMOTE_FD (this, args->fd, DEFAULT_REMOTE_FD,
-                              remote_fd, op_errno, unwind);
-
-        memcpy (req.gfid,  args->fd->inode->gfid, 16);
-        req.name = (char *)args->name;
-        req.fd = remote_fd;
-
-        GF_PROTOCOL_DICT_SERIALIZE (this, args->xdata, (&req.xdata.xdata_val),
-                                    req.xdata.xdata_len, op_errno, unwind);
-
-        ret = client_submit_request (this, &req, frame, conf->fops,
-                                     GFS3_OP_FREMOVEXATTR,
-                                     client3_3_fremovexattr_cbk, NULL,
-                                     NULL, 0, NULL, 0, NULL,
-                                     (xdrproc_t)xdr_gfs3_fremovexattr_req);
-        if (ret) {
-                gf_log (this->name, GF_LOG_WARNING, "failed to send the fop");
-        }
-
-        GF_FREE (req.xdata.xdata_val);
-
-        return 0;
+    clnt_conf_t *conf = NULL;
+    clnt_args_t *args = NULL;
+    gfs3_fremovexattr_req req = {
+        {
+            0,
+        },
+    };
+    int ret = 0;
+    int op_errno = ESTALE;
+
+    if (!frame || !this || !data)
+        goto unwind;
+
+    args = data;
+
+    conf = this->private;
+
+    ret = client_pre_fremovexattr(this, &req, args->fd, args->name,
+                                  args->xdata);
+    if (ret) {
+        op_errno = -ret;
+        goto unwind;
+    }
+    ret = client_submit_request(
+        this, &req, frame, conf->fops, GFS3_OP_FREMOVEXATTR,
+        client3_3_fremovexattr_cbk, NULL, (xdrproc_t)xdr_gfs3_fremovexattr_req);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_WARNING, 0, PC_MSG_FOP_SEND_FAILED, NULL);
+    }
+
+    GF_FREE(req.xdata.xdata_val);
+
+    return 0;
 unwind:
-        CLIENT_STACK_UNWIND (fremovexattr, frame, -1, op_errno, NULL);
-        GF_FREE (req.xdata.xdata_val);
+    CLIENT_STACK_UNWIND(fremovexattr, frame, -1, op_errno, NULL);
+    GF_FREE(req.xdata.xdata_val);
 
-        return 0;
+    return 0;
 }
 
 int32_t
-client3_3_lk (call_frame_t *frame, xlator_t *this,
-              void *data)
+client3_3_lease(call_frame_t *frame, xlator_t *this, void *data)
 {
-        clnt_args_t     *args       = NULL;
-        gfs3_lk_req      req        = {{0,},};
-        int32_t          gf_cmd     = 0;
-        int32_t          gf_type    = 0;
-        int64_t          remote_fd  = -1;
-        clnt_local_t    *local      = NULL;
-        clnt_conf_t     *conf       = NULL;
-        int              op_errno   = ESTALE;
-        int              ret        = 0;
-
-        if (!frame || !this || !data)
-                goto unwind;
-
-        args = data;
-        conf = this->private;
-        local = mem_get0 (this->local_pool);
-        if (!local) {
-                op_errno = ENOMEM;
-                goto unwind;
-        }
-        frame->local = local;
-
-        CLIENT_GET_REMOTE_FD (this, args->fd, DEFAULT_REMOTE_FD,
-                              remote_fd, op_errno, unwind);
-
-        ret = client_cmd_to_gf_cmd (args->cmd, &gf_cmd);
-        if (ret) {
-                op_errno = EINVAL;
-                gf_log (this->name, GF_LOG_WARNING,
-                        "Unknown cmd (%d)!", gf_cmd);
-                goto unwind;
-        }
-
-        switch (args->flock->l_type) {
-        case F_RDLCK:
-                gf_type = GF_LK_F_RDLCK;
-                break;
-        case F_WRLCK:
-                gf_type = GF_LK_F_WRLCK;
-                break;
-        case F_UNLCK:
-                gf_type = GF_LK_F_UNLCK;
-                break;
-        }
-
-        local->owner = frame->root->lk_owner;
-        local->cmd   = args->cmd;
-        local->fd    = fd_ref (args->fd);
-
-        req.fd    = remote_fd;
-        req.cmd   = gf_cmd;
-        req.type  = gf_type;
-        gf_proto_flock_from_flock (&req.flock, args->flock);
-
-        memcpy (req.gfid, args->fd->inode->gfid, 16);
-
-        GF_PROTOCOL_DICT_SERIALIZE (this, args->xdata, (&req.xdata.xdata_val),
-                                    req.xdata.xdata_len, op_errno, unwind);
-
-        ret = client_submit_request (this, &req, frame, conf->fops, GFS3_OP_LK,
-                                     client3_3_lk_cbk, NULL,
-                                     NULL, 0, NULL, 0, NULL,
-                                     (xdrproc_t)xdr_gfs3_lk_req);
-        if (ret) {
-                gf_log (this->name, GF_LOG_WARNING, "failed to send the fop");
-        }
-
-        GF_FREE (req.xdata.xdata_val);
-
-        return 0;
+    clnt_args_t *args = NULL;
+    gfs3_lease_req req = {
+        {
+            0,
+        },
+    };
+    clnt_conf_t *conf = NULL;
+    int op_errno = ESTALE;
+    int ret = 0;
+
+    GF_VALIDATE_OR_GOTO("client", this, unwind);
+    GF_VALIDATE_OR_GOTO(this->name, frame, unwind);
+    GF_VALIDATE_OR_GOTO(this->name, data, unwind);
+
+    args = data;
+    conf = this->private;
+
+    ret = client_pre_lease(this, &req, args->loc, args->lease, args->xdata);
+    if (ret < 0) {
+        op_errno = -ret;
+        goto unwind;
+    }
+    ret = client_submit_request(this, &req, frame, conf->fops, GFS3_OP_LEASE,
+                                client3_3_lease_cbk, NULL,
+                                (xdrproc_t)xdr_gfs3_lease_req);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_WARNING, 0, PC_MSG_FOP_SEND_FAILED, NULL);
+    }
+
+    GF_FREE(req.xdata.xdata_val);
+
+    return 0;
 unwind:
-        CLIENT_STACK_UNWIND (lk, frame, -1, op_errno, NULL, NULL);
-        GF_FREE (req.xdata.xdata_val);
+    CLIENT_STACK_UNWIND(lease, frame, -1, op_errno, NULL, NULL);
+    GF_FREE(req.xdata.xdata_val);
 
-        return 0;
+    return 0;
 }
 
-
 int32_t
-client3_3_inodelk (call_frame_t *frame, xlator_t *this,
-                   void *data)
+client3_3_lk(call_frame_t *frame, xlator_t *this, void *data)
 {
-        clnt_conf_t      *conf     = NULL;
-        clnt_args_t      *args    = NULL;
-        gfs3_inodelk_req  req     = {{0,},};
-        int               ret     = 0;
-        int32_t           gf_cmd  = 0;
-        int32_t           gf_type = 0;
-        int               op_errno = ESTALE;
-
-        if (!frame || !this || !data)
-                goto unwind;
-
-        args = data;
-        if (!(args->loc && args->loc->inode))
-                goto unwind;
-
-        if (!gf_uuid_is_null (args->loc->inode->gfid))
-                memcpy (req.gfid,  args->loc->inode->gfid, 16);
-        else
-                memcpy (req.gfid, args->loc->gfid, 16);
-
-        GF_ASSERT_AND_GOTO_WITH_ERROR (this->name,
-                                       !gf_uuid_is_null (*((uuid_t*)req.gfid)),
-                                       unwind, op_errno, EINVAL);
-        if (args->cmd == F_GETLK || args->cmd == F_GETLK64)
-                gf_cmd = GF_LK_GETLK;
-        else if (args->cmd == F_SETLK || args->cmd == F_SETLK64)
-                gf_cmd = GF_LK_SETLK;
-        else if (args->cmd == F_SETLKW || args->cmd == F_SETLKW64)
-                gf_cmd = GF_LK_SETLKW;
-        else {
-                gf_log (this->name, GF_LOG_WARNING,
-                        "Unknown cmd (%d)!", gf_cmd);
-                op_errno = EINVAL;
-                goto unwind;
-        }
-
-        switch (args->flock->l_type) {
-        case F_RDLCK:
-                gf_type = GF_LK_F_RDLCK;
-                break;
-        case F_WRLCK:
-                gf_type = GF_LK_F_WRLCK;
-                break;
-        case F_UNLCK:
-                gf_type = GF_LK_F_UNLCK;
-                break;
-        }
-
-        req.volume = (char *)args->volume;
-        req.cmd    = gf_cmd;
-        req.type   = gf_type;
-        gf_proto_flock_from_flock (&req.flock, args->flock);
-
-        conf = this->private;
-
-        GF_PROTOCOL_DICT_SERIALIZE (this, args->xdata, (&req.xdata.xdata_val),
-                                    req.xdata.xdata_len, op_errno, unwind);
-
-        ret = client_submit_request (this, &req, frame, conf->fops,
-                                     GFS3_OP_INODELK,
-                                     client3_3_inodelk_cbk, NULL,
-                                     NULL, 0, NULL, 0,
-                                     NULL, (xdrproc_t)xdr_gfs3_inodelk_req);
-        if (ret) {
-                gf_log (this->name, GF_LOG_WARNING, "failed to send the fop");
-        }
-
-        GF_FREE (req.xdata.xdata_val);
-
-        return 0;
+    clnt_args_t *args = NULL;
+    gfs3_lk_req req = {
+        {
+            0,
+        },
+    };
+    int32_t gf_cmd = 0;
+    clnt_local_t *local = NULL;
+    clnt_conf_t *conf = NULL;
+    int op_errno = ESTALE;
+    int ret = 0;
+
+    if (!frame || !this || !data)
+        goto unwind;
+
+    args = data;
+    conf = this->private;
+    local = mem_get0(this->local_pool);
+    if (!local) {
+        op_errno = ENOMEM;
+        goto unwind;
+    }
+    frame->local = local;
+
+    ret = client_cmd_to_gf_cmd(args->cmd, &gf_cmd);
+    if (ret) {
+        op_errno = EINVAL;
+        gf_smsg(this->name, GF_LOG_WARNING, EINVAL, PC_MSG_UNKNOWN_CMD,
+                "gf_cmd=%d", gf_cmd, NULL);
+        goto unwind;
+    }
+
+    local->owner = frame->root->lk_owner;
+    local->cmd = args->cmd;
+    local->fd = fd_ref(args->fd);
+
+    ret = client_pre_lk(this, &req, args->cmd, args->flock, args->fd,
+                        args->xdata);
+    if (ret) {
+        op_errno = -ret;
+
+        if ((op_errno == EBADF) && (args->flock->l_type == F_UNLCK) &&
+            client_is_setlk(local->cmd)) {
+            client_add_lock_for_recovery(local->fd, args->flock, &local->owner,
+                                         local->cmd);
+        }
+
+        goto unwind;
+    }
+
+    ret = client_submit_request(this, &req, frame, conf->fops, GFS3_OP_LK,
+                                client3_3_lk_cbk, NULL,
+                                (xdrproc_t)xdr_gfs3_lk_req);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_WARNING, 0, PC_MSG_FOP_SEND_FAILED, NULL);
+    }
+
+    GF_FREE(req.xdata.xdata_val);
+
+    return 0;
 unwind:
-        CLIENT_STACK_UNWIND (inodelk, frame, -1, op_errno, NULL);
-        GF_FREE (req.xdata.xdata_val);
+    CLIENT_STACK_UNWIND(lk, frame, -1, op_errno, NULL, NULL);
+    GF_FREE(req.xdata.xdata_val);
 
-        return 0;
+    return 0;
 }
 
-
-
 int32_t
-client3_3_finodelk (call_frame_t *frame, xlator_t *this,
-                    void *data)
+client3_3_inodelk(call_frame_t *frame, xlator_t *this, void *data)
 {
-        clnt_args_t       *args     = NULL;
-        gfs3_finodelk_req  req      = {{0,},};
-        int32_t            gf_cmd   = 0;
-        int32_t            gf_type  = 0;
-        int64_t            remote_fd = -1;
-        clnt_conf_t       *conf     = NULL;
-        int                op_errno = ESTALE;
-        int                ret      = 0;
-
-        if (!frame || !this || !data)
-                goto unwind;
-
-        args = data;
-        conf = this->private;
-        CLIENT_GET_REMOTE_FD (this, args->fd, FALLBACK_TO_ANON_FD,
-                              remote_fd, op_errno, unwind);
-        ret = client_fd_fop_prepare_local (frame, args->fd, remote_fd);
-        if (ret) {
-                op_errno = -ret;
-                goto unwind;
-        }
-
-        if (args->cmd == F_GETLK || args->cmd == F_GETLK64)
-                gf_cmd = GF_LK_GETLK;
-        else if (args->cmd == F_SETLK || args->cmd == F_SETLK64)
-                gf_cmd = GF_LK_SETLK;
-        else if (args->cmd == F_SETLKW || args->cmd == F_SETLKW64)
-                gf_cmd = GF_LK_SETLKW;
-        else {
-                gf_log (this->name, GF_LOG_WARNING,
-                        "Unknown cmd (%d)!", gf_cmd);
-                goto unwind;
-        }
-
-        switch (args->flock->l_type) {
-        case F_RDLCK:
-                gf_type = GF_LK_F_RDLCK;
-                break;
-        case F_WRLCK:
-                gf_type = GF_LK_F_WRLCK;
-                break;
-        case F_UNLCK:
-                gf_type = GF_LK_F_UNLCK;
-                break;
-        }
-
-        req.volume = (char *)args->volume;
-        req.fd    = remote_fd;
-        req.cmd   = gf_cmd;
-        req.type  = gf_type;
-        gf_proto_flock_from_flock (&req.flock, args->flock);
-        memcpy (req.gfid, args->fd->inode->gfid, 16);
-
-        GF_PROTOCOL_DICT_SERIALIZE (this, args->xdata, (&req.xdata.xdata_val),
-                                    req.xdata.xdata_len, op_errno, unwind);
-
-        ret = client_submit_request (this, &req, frame, conf->fops,
-                                     GFS3_OP_FINODELK,
-                                     client3_3_finodelk_cbk, NULL,
-                                     NULL, 0, NULL, 0,
-                                     NULL, (xdrproc_t)xdr_gfs3_finodelk_req);
-        if (ret) {
-                gf_log (this->name, GF_LOG_WARNING, "failed to send the fop");
-        }
-
-        GF_FREE (req.xdata.xdata_val);
-        return 0;
+    clnt_conf_t *conf = NULL;
+    clnt_args_t *args = NULL;
+    gfs3_inodelk_req req = {
+        {
+            0,
+        },
+    };
+    int ret = 0;
+    int op_errno = ESTALE;
+
+    if (!frame || !this || !data)
+        goto unwind;
+
+    args = data;
+    conf = this->private;
+
+    ret = client_pre_inodelk(this, &req, args->loc, args->cmd, args->flock,
+                             args->volume, args->xdata);
+    if (ret) {
+        op_errno = -ret;
+        goto unwind;
+    }
+    ret = client_submit_request(this, &req, frame, conf->fops, GFS3_OP_INODELK,
+                                client3_3_inodelk_cbk, NULL,
+                                (xdrproc_t)xdr_gfs3_inodelk_req);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_WARNING, 0, PC_MSG_FOP_SEND_FAILED, NULL);
+    }
+
+    GF_FREE(req.xdata.xdata_val);
+
+    return 0;
 unwind:
-        CLIENT_STACK_UNWIND (finodelk, frame, -1, op_errno, NULL);
-        GF_FREE (req.xdata.xdata_val);
+    CLIENT_STACK_UNWIND(inodelk, frame, -1, op_errno, NULL);
+    GF_FREE(req.xdata.xdata_val);
 
-        return 0;
+    return 0;
 }
 
-
 int32_t
-client3_3_entrylk (call_frame_t *frame, xlator_t *this,
-                   void *data)
+client3_3_finodelk(call_frame_t *frame, xlator_t *this, void *data)
 {
-        clnt_conf_t      *conf     = NULL;
-        clnt_args_t      *args     = NULL;
-        gfs3_entrylk_req  req      = {{0,},};
-        int               ret      = 0;
-        int               op_errno = ESTALE;
-
-        if (!frame || !this || !data)
-                goto unwind;
-
-        args = data;
-
-        if (!(args->loc && args->loc->inode))
-                goto unwind;
-
-        if (!gf_uuid_is_null (args->loc->inode->gfid))
-                memcpy (req.gfid,  args->loc->inode->gfid, 16);
-        else
-                memcpy (req.gfid, args->loc->gfid, 16);
-
-        GF_ASSERT_AND_GOTO_WITH_ERROR (this->name,
-                                       !gf_uuid_is_null (*((uuid_t*)req.gfid)),
-                                       unwind, op_errno, EINVAL);
-        req.cmd = args->cmd_entrylk;
-        req.type = args->type;
-        req.volume = (char *)args->volume;
-        req.name = "";
-        if (args->basename) {
-                req.name = (char *)args->basename;
-                req.namelen = 1;
-        }
-
-        conf = this->private;
-
-        GF_PROTOCOL_DICT_SERIALIZE (this, args->xdata, (&req.xdata.xdata_val),
-                                    req.xdata.xdata_len, op_errno, unwind);
-
-        ret = client_submit_request (this, &req, frame, conf->fops,
-                                     GFS3_OP_ENTRYLK,
-                                     client3_3_entrylk_cbk, NULL,
-                                     NULL, 0, NULL, 0,
-                                     NULL, (xdrproc_t)xdr_gfs3_entrylk_req);
-        if (ret) {
-                gf_log (this->name, GF_LOG_WARNING, "failed to send the fop");
-        }
-
-        GF_FREE (req.xdata.xdata_val);
-
-        return 0;
+    clnt_args_t *args = NULL;
+    gfs3_finodelk_req req = {
+        {
+            0,
+        },
+    };
+    clnt_conf_t *conf = NULL;
+    int op_errno = ESTALE;
+    int ret = 0;
+
+    if (!frame || !this || !data)
+        goto unwind;
+
+    args = data;
+    conf = this->private;
+
+    ret = client_pre_finodelk(this, &req, args->fd, args->cmd, args->flock,
+                              args->volume, args->xdata);
+    if (ret) {
+        op_errno = -ret;
+        goto unwind;
+    }
+
+    ret = client_fd_fop_prepare_local(frame, args->fd, req.fd);
+    if (ret) {
+        op_errno = -ret;
+        goto unwind;
+    }
+
+    ret = client_submit_request(this, &req, frame, conf->fops, GFS3_OP_FINODELK,
+                                client3_3_finodelk_cbk, NULL,
+                                (xdrproc_t)xdr_gfs3_finodelk_req);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_WARNING, 0, PC_MSG_FOP_SEND_FAILED, NULL);
+    }
+
+    GF_FREE(req.xdata.xdata_val);
+    return 0;
 unwind:
-        CLIENT_STACK_UNWIND (entrylk, frame, -1, op_errno, NULL);
-        GF_FREE (req.xdata.xdata_val);
+    CLIENT_STACK_UNWIND(finodelk, frame, -1, op_errno, NULL);
+    GF_FREE(req.xdata.xdata_val);
 
-        return 0;
+    return 0;
 }
 
-
-
 int32_t
-client3_3_fentrylk (call_frame_t *frame, xlator_t *this,
-                    void *data)
+client3_3_entrylk(call_frame_t *frame, xlator_t *this, void *data)
 {
-        clnt_args_t       *args     = NULL;
-        gfs3_fentrylk_req  req      = {{0,},};
-        int64_t            remote_fd = -1;
-        clnt_conf_t       *conf     = NULL;
-        int                op_errno = ESTALE;
-        int           ret        = 0;
-
-        if (!frame || !this || !data)
-                goto unwind;
+    clnt_conf_t *conf = NULL;
+    clnt_args_t *args = NULL;
+    gfs3_entrylk_req req = {
+        {
+            0,
+        },
+    };
+    int ret = 0;
+    int op_errno = ESTALE;
 
-        args = data;
-        conf = this->private;
+    if (!frame || !this || !data)
+        goto unwind;
 
-        CLIENT_GET_REMOTE_FD (this, args->fd, DEFAULT_REMOTE_FD,
-                              remote_fd, op_errno, unwind);
+    args = data;
 
-        req.fd  = remote_fd;
-        req.cmd = args->cmd_entrylk;
-        req.type = args->type;
-        req.volume = (char *)args->volume;
-        req.name = "";
-        if (args->basename) {
-                req.name = (char *)args->basename;
-                req.namelen = 1;
-        }
-        memcpy (req.gfid, args->fd->inode->gfid, 16);
+    conf = this->private;
 
-        GF_PROTOCOL_DICT_SERIALIZE (this, args->xdata, (&req.xdata.xdata_val),
-                                    req.xdata.xdata_len, op_errno, unwind);
+    ret = client_pre_entrylk(this, &req, args->loc, args->cmd_entrylk,
+                             args->type, args->volume, args->basename,
+                             args->xdata);
+    if (ret) {
+        op_errno = -ret;
+        goto unwind;
+    }
 
-        ret = client_submit_request (this, &req, frame, conf->fops,
-                                     GFS3_OP_FENTRYLK,
-                                     client3_3_fentrylk_cbk, NULL,
-                                     NULL, 0, NULL, 0,
-                                     NULL, (xdrproc_t)xdr_gfs3_fentrylk_req);
-        if (ret) {
-                gf_log (this->name, GF_LOG_WARNING, "failed to send the fop");
-        }
+    ret = client_submit_request(this, &req, frame, conf->fops, GFS3_OP_ENTRYLK,
+                                client3_3_entrylk_cbk, NULL,
+                                (xdrproc_t)xdr_gfs3_entrylk_req);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_WARNING, 0, PC_MSG_FOP_SEND_FAILED, NULL);
+    }
 
-        GF_FREE (req.xdata.xdata_val);
+    GF_FREE(req.xdata.xdata_val);
 
-        return 0;
+    return 0;
 unwind:
-        CLIENT_STACK_UNWIND (fentrylk, frame, -1, op_errno, NULL);
-        GF_FREE (req.xdata.xdata_val);
+    CLIENT_STACK_UNWIND(entrylk, frame, -1, op_errno, NULL);
+    GF_FREE(req.xdata.xdata_val);
 
-        return 0;
+    return 0;
 }
 
-
 int32_t
-client3_3_rchecksum (call_frame_t *frame, xlator_t *this,
-                     void *data)
+client3_3_fentrylk(call_frame_t *frame, xlator_t *this, void *data)
 {
-        clnt_args_t        *args     = NULL;
-        int64_t             remote_fd = -1;
-        clnt_conf_t        *conf     = NULL;
-        gfs3_rchecksum_req  req      = {0,};
-        int                 op_errno = ESTALE;
-        int                 ret        = 0;
-
-        if (!frame || !this || !data)
-                goto unwind;
-
-        args = data;
-        conf = this->private;
-
-        CLIENT_GET_REMOTE_FD (this, args->fd, DEFAULT_REMOTE_FD,
-                              remote_fd, op_errno, unwind);
-
-        req.len    = args->len;
-        req.offset = args->offset;
-        req.fd     = remote_fd;
-
-        GF_PROTOCOL_DICT_SERIALIZE (this, args->xdata, (&req.xdata.xdata_val),
-                                    req.xdata.xdata_len, op_errno, unwind);
-
-        ret = client_submit_request (this, &req, frame, conf->fops,
-                                     GFS3_OP_RCHECKSUM,
-                                     client3_3_rchecksum_cbk, NULL,
-                                     NULL, 0, NULL,
-                                     0, NULL,
-                                     (xdrproc_t)xdr_gfs3_rchecksum_req);
-        if (ret) {
-                gf_log (this->name, GF_LOG_WARNING, "failed to send the fop");
-        }
-
-        GF_FREE (req.xdata.xdata_val);
-
-        return 0;
+    clnt_args_t *args = NULL;
+    gfs3_fentrylk_req req = {
+        {
+            0,
+        },
+    };
+    clnt_conf_t *conf = NULL;
+    int op_errno = ESTALE;
+    int ret = 0;
+
+    if (!frame || !this || !data)
+        goto unwind;
+
+    args = data;
+    conf = this->private;
+
+    ret = client_pre_fentrylk(this, &req, args->fd, args->cmd_entrylk,
+                              args->type, args->volume, args->basename,
+                              args->xdata);
+    if (ret) {
+        op_errno = -ret;
+        goto unwind;
+    }
+    ret = client_submit_request(this, &req, frame, conf->fops, GFS3_OP_FENTRYLK,
+                                client3_3_fentrylk_cbk, NULL,
+                                (xdrproc_t)xdr_gfs3_fentrylk_req);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_WARNING, 0, PC_MSG_FOP_SEND_FAILED, NULL);
+    }
+
+    GF_FREE(req.xdata.xdata_val);
+
+    return 0;
 unwind:
-        CLIENT_STACK_UNWIND (rchecksum, frame, -1, op_errno, 0, NULL, NULL);
-        GF_FREE (req.xdata.xdata_val);
+    CLIENT_STACK_UNWIND(fentrylk, frame, -1, op_errno, NULL);
+    GF_FREE(req.xdata.xdata_val);
 
-        return 0;
+    return 0;
 }
 
-
-
 int32_t
-client3_3_readdir (call_frame_t *frame, xlator_t *this,
-                   void *data)
+client3_3_rchecksum(call_frame_t *frame, xlator_t *this, void *data)
 {
-        clnt_args_t      *args       = NULL;
-        int64_t           remote_fd  = -1;
-        clnt_conf_t      *conf       = NULL;
-        gfs3_readdir_req  req        = {{0,},};
-        gfs3_readdir_rsp  rsp        = {0, };
-        clnt_local_t     *local      = NULL;
-        int               op_errno   = ESTALE;
-        int               ret        = 0;
-        int               count      = 0;
-        struct iobref    *rsp_iobref = NULL;
-        struct iobuf     *rsp_iobuf  = NULL;
-        struct iovec     *rsphdr     = NULL;
-        struct iovec      vector[MAX_IOVEC] = {{0}, };
-        int               readdir_rsp_size  = 0;
-
-        if (!frame || !this || !data)
-                goto unwind;
-
-        args = data;
-        conf = this->private;
-
-        CLIENT_GET_REMOTE_FD (this, args->fd, DEFAULT_REMOTE_FD,
-                              remote_fd, op_errno, unwind);
+    clnt_args_t *args = NULL;
+    clnt_conf_t *conf = NULL;
+    gfs3_rchecksum_req req = {
+        0,
+    };
+    int op_errno = ESTALE;
+    int ret = 0;
+
+    if (!frame || !this || !data)
+        goto unwind;
+
+    args = data;
+    conf = this->private;
+
+    ret = client_pre_rchecksum(this, &req, args->fd, args->len, args->offset,
+                               args->xdata);
+    if (ret) {
+        op_errno = -ret;
+        goto unwind;
+    }
+    ret = client_submit_request(this, &req, frame, conf->fops,
+                                GFS3_OP_RCHECKSUM, client3_3_rchecksum_cbk,
+                                NULL, (xdrproc_t)xdr_gfs3_rchecksum_req);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_WARNING, 0, PC_MSG_FOP_SEND_FAILED, NULL);
+    }
+
+    GF_FREE(req.xdata.xdata_val);
+
+    return 0;
+unwind:
+    CLIENT_STACK_UNWIND(rchecksum, frame, -1, op_errno, 0, NULL, NULL);
+    GF_FREE(req.xdata.xdata_val);
 
-        readdir_rsp_size = xdr_sizeof ((xdrproc_t) xdr_gfs3_readdir_rsp, &rsp)
-                + args->size;
+    return 0;
+}
 
-        local = mem_get0 (this->local_pool);
-        if (!local) {
-                op_errno = ENOMEM;
-                goto unwind;
+int32_t
+client3_3_readdir(call_frame_t *frame, xlator_t *this, void *data)
+{
+    clnt_args_t *args = NULL;
+    int64_t remote_fd = -1;
+    clnt_conf_t *conf = NULL;
+    gfs3_readdir_req req = {
+        {
+            0,
+        },
+    };
+    gfs3_readdir_rsp rsp = {
+        0,
+    };
+    clnt_local_t *local = NULL;
+    int op_errno = ESTALE;
+    int ret = 0;
+    int count = 0;
+    struct iobref *rsp_iobref = NULL;
+    struct iobuf *rsp_iobuf = NULL;
+    struct iovec *rsphdr = NULL;
+    struct iovec vector[MAX_IOVEC] = {
+        {0},
+    };
+    int readdir_rsp_size = 0;
+    client_payload_t cp;
+
+    if (!frame || !this || !data)
+        goto unwind;
+
+    args = data;
+    conf = this->private;
+
+    readdir_rsp_size = xdr_sizeof((xdrproc_t)xdr_gfs3_readdir_rsp, &rsp) +
+                       args->size;
+
+    local = mem_get0(this->local_pool);
+    if (!local) {
+        op_errno = ENOMEM;
+        goto unwind;
+    }
+    frame->local = local;
+
+    local->cmd = remote_fd;
+
+    if ((readdir_rsp_size + GLUSTERFS_RPC_REPLY_SIZE +
+         GLUSTERFS_RDMA_MAX_HEADER_SIZE) > (GLUSTERFS_RDMA_INLINE_THRESHOLD)) {
+        rsp_iobref = iobref_new();
+        if (rsp_iobref == NULL) {
+            goto unwind;
         }
-        frame->local = local;
-
-        if ((readdir_rsp_size + GLUSTERFS_RPC_REPLY_SIZE + GLUSTERFS_RDMA_MAX_HEADER_SIZE)
-            > (GLUSTERFS_RDMA_INLINE_THRESHOLD)) {
-                rsp_iobref = iobref_new ();
-                if (rsp_iobref == NULL) {
-                        goto unwind;
-                }
-
-                /* TODO: what is the size we should send ? */
-                /* This iobuf will live for only receiving the response,
-                   so not harmful */
-                rsp_iobuf = iobuf_get (this->ctx->iobuf_pool);
-                if (rsp_iobuf == NULL) {
-                        goto unwind;
-                }
-
-                iobref_add (rsp_iobref, rsp_iobuf);
-                iobuf_unref (rsp_iobuf);
-
-                rsphdr = &vector[0];
-                rsphdr->iov_base = iobuf_ptr (rsp_iobuf);
-                rsphdr->iov_len  = iobuf_pagesize (rsp_iobuf);
-                count = 1;
-                local->iobref = rsp_iobref;
-                rsp_iobuf = NULL;
-                rsp_iobref = NULL;
+
+        /* TODO: what is the size we should send ? */
+        /* This iobuf will live for only receiving the response,
+           so not harmful */
+        rsp_iobuf = iobuf_get(this->ctx->iobuf_pool);
+        if (rsp_iobuf == NULL) {
+            goto unwind;
         }
 
-        req.size = args->size;
-        req.offset = args->offset;
-        req.fd = remote_fd;
+        iobref_add(rsp_iobref, rsp_iobuf);
 
-        local->cmd = remote_fd;
+        rsphdr = &vector[0];
+        rsphdr->iov_base = iobuf_ptr(rsp_iobuf);
+        rsphdr->iov_len = iobuf_pagesize(rsp_iobuf);
+        count = 1;
+        local->iobref = rsp_iobref;
+        iobuf_unref(rsp_iobuf);
+        rsp_iobuf = NULL;
+        rsp_iobref = NULL;
+    }
 
-        memcpy (req.gfid, args->fd->inode->gfid, 16);
-        GF_PROTOCOL_DICT_SERIALIZE (this, args->xdata, (&req.xdata.xdata_val),
-                                    req.xdata.xdata_len, op_errno, unwind);
+    ret = client_pre_readdir(this, &req, args->fd, args->size, args->offset,
+                             args->xdata);
+    if (ret) {
+        op_errno = -ret;
+        goto unwind;
+    }
 
-        ret = client_submit_request (this, &req, frame, conf->fops,
-                                     GFS3_OP_READDIR,
-                                     client3_3_readdir_cbk, NULL,
-                                     rsphdr, count,
-                                     NULL, 0, rsp_iobref,
-                                     (xdrproc_t)xdr_gfs3_readdir_req);
+    memset(&cp, 0, sizeof(client_payload_t));
+    cp.rsphdr = rsphdr;
+    cp.rsphdr_cnt = count;
+    cp.rsp_iobref = rsp_iobref;
+    ret = client_submit_request(this, &req, frame, conf->fops, GFS3_OP_READDIR,
+                                client3_3_readdir_cbk, &cp,
+                                (xdrproc_t)xdr_gfs3_readdir_req);
 
-        if (ret) {
-                gf_log (this->name, GF_LOG_WARNING, "failed to send the fop");
-        }
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_WARNING, 0, PC_MSG_FOP_SEND_FAILED, NULL);
+    }
 
-        GF_FREE (req.xdata.xdata_val);
+    GF_FREE(req.xdata.xdata_val);
 
-        return 0;
+    return 0;
 
 unwind:
-        if (rsp_iobref)
-                iobref_unref (rsp_iobref);
-
-        if (rsp_iobuf)
-                iobuf_unref (rsp_iobuf);
+    if (rsp_iobref)
+        iobref_unref(rsp_iobref);
 
-        CLIENT_STACK_UNWIND (readdir, frame, -1, op_errno, NULL, NULL);
-        GF_FREE (req.xdata.xdata_val);
+    CLIENT_STACK_UNWIND(readdir, frame, -1, op_errno, NULL, NULL);
+    GF_FREE(req.xdata.xdata_val);
 
-        return 0;
+    return 0;
 }
 
-
 int32_t
-client3_3_readdirp (call_frame_t *frame, xlator_t *this,
-                    void *data)
+client3_3_readdirp(call_frame_t *frame, xlator_t *this, void *data)
 {
-        clnt_args_t      *args              = NULL;
-        gfs3_readdirp_req req               = {{0,},};
-        gfs3_readdirp_rsp rsp               = {0,};
-        int64_t           remote_fd         = -1;
-        clnt_conf_t      *conf              = NULL;
-        int               op_errno          = ESTALE;
-        int               ret               = 0;
-        int               count             = 0;
-        int               readdirp_rsp_size = 0;
-        struct iobref    *rsp_iobref        = NULL;
-        struct iobuf     *rsp_iobuf         = NULL;
-        struct iovec     *rsphdr            = NULL;
-        struct iovec      vector[MAX_IOVEC] = {{0}, };
-        clnt_local_t     *local             = NULL;
-
-        if (!frame || !this || !data)
-                goto unwind;
-
-        args = data;
-        conf = this->private;
-
-        CLIENT_GET_REMOTE_FD (this, args->fd, DEFAULT_REMOTE_FD,
-                              remote_fd, op_errno, unwind);
-
-        readdirp_rsp_size = xdr_sizeof ((xdrproc_t) xdr_gfs3_readdirp_rsp, &rsp)
-                + args->size;
-
-        local = mem_get0 (this->local_pool);
-        if (!local) {
-                op_errno = ENOMEM;
-                goto unwind;
-        }
-        frame->local = local;
-
-        if ((readdirp_rsp_size + GLUSTERFS_RPC_REPLY_SIZE
-             + GLUSTERFS_RDMA_MAX_HEADER_SIZE)
-            > (GLUSTERFS_RDMA_INLINE_THRESHOLD)) {
-                rsp_iobref = iobref_new ();
-                if (rsp_iobref == NULL) {
-                        goto unwind;
-                }
-
-                /* TODO: what is the size we should send ? */
-                /* This iobuf will live for only receiving the response,
-                   so not harmful */
-                rsp_iobuf = iobuf_get (this->ctx->iobuf_pool);
-                if (rsp_iobuf == NULL) {
-                        goto unwind;
-                }
-
-                iobref_add (rsp_iobref, rsp_iobuf);
-                iobuf_unref (rsp_iobuf);
-
-                rsphdr = &vector[0];
-                rsphdr->iov_base = iobuf_ptr (rsp_iobuf);
-                rsphdr->iov_len  = iobuf_pagesize (rsp_iobuf);
-                count = 1;
-                local->iobref = rsp_iobref;
-                rsp_iobuf = NULL;
-                rsp_iobref = NULL;
+    clnt_args_t *args = NULL;
+    gfs3_readdirp_req req = {
+        {
+            0,
+        },
+    };
+    gfs3_readdirp_rsp rsp = {
+        0,
+    };
+    clnt_conf_t *conf = NULL;
+    int op_errno = ESTALE;
+    int ret = 0;
+    int count = 0;
+    int readdirp_rsp_size = 0;
+    struct iobref *rsp_iobref = NULL;
+    struct iobuf *rsp_iobuf = NULL;
+    struct iovec *rsphdr = NULL;
+    struct iovec vector[MAX_IOVEC] = {
+        {0},
+    };
+    clnt_local_t *local = NULL;
+    client_payload_t cp;
+
+    if (!frame || !this || !data)
+        goto unwind;
+
+    args = data;
+    conf = this->private;
+
+    local = mem_get0(this->local_pool);
+    if (!local) {
+        op_errno = ENOMEM;
+        goto unwind;
+    }
+    frame->local = local;
+
+    ret = client_pre_readdirp(this, &req, args->fd, args->size, args->offset,
+                              args->xdata);
+
+    if (ret) {
+        op_errno = -ret;
+        goto unwind;
+    }
+
+    readdirp_rsp_size = xdr_sizeof((xdrproc_t)xdr_gfs3_readdirp_rsp, &rsp) +
+                        args->size;
+
+    if ((readdirp_rsp_size + GLUSTERFS_RPC_REPLY_SIZE +
+         GLUSTERFS_RDMA_MAX_HEADER_SIZE) > (GLUSTERFS_RDMA_INLINE_THRESHOLD)) {
+        rsp_iobref = iobref_new();
+        if (rsp_iobref == NULL) {
+            goto unwind;
         }
 
-        local->fd = fd_ref (args->fd);
+        /* TODO: what is the size we should send ? */
+        /* This iobuf will live for only receiving the response,
+           so not harmful */
+        rsp_iobuf = iobuf_get(this->ctx->iobuf_pool);
+        if (rsp_iobuf == NULL) {
+            goto unwind;
+        }
 
-        req.size = args->size;
-        req.offset = args->offset;
-        req.fd = remote_fd;
-        memcpy (req.gfid, args->fd->inode->gfid, 16);
+        rsphdr = &vector[0];
+        rsphdr->iov_base = iobuf_ptr(rsp_iobuf);
+        rsphdr->iov_len = iobuf_pagesize(rsp_iobuf);
+        count = 1;
+        local->iobref = rsp_iobref;
+        iobref_add(rsp_iobref, rsp_iobuf);
+        iobuf_unref(rsp_iobuf);
+        rsp_iobuf = NULL;
+        rsp_iobref = NULL;
+    }
 
-        /* dict itself is 'xdata' here */
-        GF_PROTOCOL_DICT_SERIALIZE (this, args->xdata, (&req.dict.dict_val),
-                                    req.dict.dict_len, op_errno, unwind);
+    local->fd = fd_ref(args->fd);
 
-        ret = client_submit_request (this, &req, frame, conf->fops,
-                                     GFS3_OP_READDIRP,
-                                     client3_3_readdirp_cbk, NULL,
-                                     rsphdr, count, NULL,
-                                     0, rsp_iobref,
-                                     (xdrproc_t)xdr_gfs3_readdirp_req);
-        if (ret) {
-                gf_log (this->name, GF_LOG_WARNING, "failed to send the fop");
-        }
+    memset(&cp, 0, sizeof(client_payload_t));
+    cp.rsphdr = rsphdr;
+    cp.rsphdr_cnt = count;
+    cp.rsp_iobref = rsp_iobref;
+    ret = client_submit_request(this, &req, frame, conf->fops, GFS3_OP_READDIRP,
+                                client3_3_readdirp_cbk, &cp,
+                                (xdrproc_t)xdr_gfs3_readdirp_req);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_WARNING, 0, PC_MSG_FOP_SEND_FAILED, NULL);
+    }
 
-        GF_FREE (req.dict.dict_val);
+    GF_FREE(req.dict.dict_val);
 
-        return 0;
+    return 0;
 unwind:
-        if (rsp_iobref)
-                iobref_unref (rsp_iobref);
+    if (rsp_iobref)
+        iobref_unref(rsp_iobref);
 
-        if (rsp_iobuf)
-                iobuf_unref (rsp_iobuf);
+    GF_FREE(req.dict.dict_val);
 
-        GF_FREE (req.dict.dict_val);
-
-        CLIENT_STACK_UNWIND (readdirp, frame, -1, op_errno, NULL, NULL);
-        return 0;
+    CLIENT_STACK_UNWIND(readdirp, frame, -1, op_errno, NULL, NULL);
+    return 0;
 }
 
-
 int32_t
-client3_3_setattr (call_frame_t *frame, xlator_t *this,
-                   void *data)
+client3_3_setattr(call_frame_t *frame, xlator_t *this, void *data)
 {
-        clnt_conf_t      *conf     = NULL;
-        clnt_args_t      *args     = NULL;
-        gfs3_setattr_req  req      = {{0,},};
-        int               ret      = 0;
-        int               op_errno = ESTALE;
-
-        if (!frame || !this || !data)
-                goto unwind;
-
-        args = data;
-
-        if (!(args->loc && args->loc->inode))
-                goto unwind;
+    clnt_conf_t *conf = NULL;
+    clnt_args_t *args = NULL;
+    gfs3_setattr_req req = {
+        {
+            0,
+        },
+    };
+    int ret = 0;
+    int op_errno = ESTALE;
 
-        if (!gf_uuid_is_null (args->loc->inode->gfid))
-                memcpy (req.gfid, args->loc->inode->gfid, 16);
-        else
-                memcpy (req.gfid, args->loc->gfid, 16);
+    if (!frame || !this || !data)
+        goto unwind;
 
-        GF_ASSERT_AND_GOTO_WITH_ERROR (this->name,
-                                       !gf_uuid_is_null (*((uuid_t*)req.gfid)),
-                                       unwind, op_errno, EINVAL);
-        req.valid = args->valid;
-        gf_stat_from_iatt (&req.stbuf, args->stbuf);
+    args = data;
+    conf = this->private;
 
-        conf = this->private;
+    ret = client_pre_setattr(this, &req, args->loc, args->valid, args->stbuf,
+                             args->xdata);
 
-        GF_PROTOCOL_DICT_SERIALIZE (this, args->xdata, (&req.xdata.xdata_val),
-                                    req.xdata.xdata_len, op_errno, unwind);
+    if (ret) {
+        op_errno = -ret;
+        goto unwind;
+    }
 
-        ret = client_submit_request (this, &req, frame, conf->fops,
-                                     GFS3_OP_SETATTR,
-                                     client3_3_setattr_cbk, NULL,
-                                     NULL, 0, NULL, 0,
-                                     NULL, (xdrproc_t)xdr_gfs3_setattr_req);
-        if (ret) {
-                gf_log (this->name, GF_LOG_WARNING, "failed to send the fop");
-        }
+    ret = client_submit_request(this, &req, frame, conf->fops, GFS3_OP_SETATTR,
+                                client3_3_setattr_cbk, NULL,
+                                (xdrproc_t)xdr_gfs3_setattr_req);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_WARNING, 0, PC_MSG_FOP_SEND_FAILED, NULL);
+    }
 
-        GF_FREE (req.xdata.xdata_val);
+    GF_FREE(req.xdata.xdata_val);
 
-        return 0;
+    return 0;
 unwind:
-        CLIENT_STACK_UNWIND (setattr, frame, -1, op_errno, NULL, NULL, NULL);
-        GF_FREE (req.xdata.xdata_val);
+    CLIENT_STACK_UNWIND(setattr, frame, -1, op_errno, NULL, NULL, NULL);
+    GF_FREE(req.xdata.xdata_val);
 
-        return 0;
+    return 0;
 }
 
 int32_t
-client3_3_fsetattr (call_frame_t *frame, xlator_t *this, void *data)
+client3_3_fsetattr(call_frame_t *frame, xlator_t *this, void *data)
 {
-        clnt_args_t       *args     = NULL;
-        int64_t            remote_fd = -1;
-        clnt_conf_t       *conf     = NULL;
-        gfs3_fsetattr_req  req      = {0,};
-        int                op_errno = ESTALE;
-        int                ret        = 0;
-
-        if (!frame || !this || !data)
-                goto unwind;
-
-        args = data;
-        conf = this->private;
+    clnt_args_t *args = NULL;
+    clnt_conf_t *conf = NULL;
+    gfs3_fsetattr_req req = {
+        0,
+    };
+    int op_errno = ESTALE;
+    int ret = 0;
+
+    if (!frame || !this || !data)
+        goto unwind;
+
+    args = data;
+    conf = this->private;
+
+    ret = client_pre_fsetattr(this, &req, args->fd, args->valid, args->stbuf,
+                              args->xdata);
+
+    if (ret) {
+        op_errno = -ret;
+        goto unwind;
+    }
+    ret = client_submit_request(this, &req, frame, conf->fops, GFS3_OP_FSETATTR,
+                                client3_3_fsetattr_cbk, NULL,
+                                (xdrproc_t)xdr_gfs3_fsetattr_req);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_WARNING, 0, PC_MSG_FOP_SEND_FAILED, NULL);
+    }
+
+    GF_FREE(req.xdata.xdata_val);
+
+    return 0;
+unwind:
+    CLIENT_STACK_UNWIND(fsetattr, frame, -1, op_errno, NULL, NULL, NULL);
+    GF_FREE(req.xdata.xdata_val);
 
-        CLIENT_GET_REMOTE_FD (this, args->fd, DEFAULT_REMOTE_FD,
-                              remote_fd, op_errno, unwind);
+    return 0;
+}
 
-        req.fd = remote_fd;
-        req.valid = args->valid;
-        gf_stat_from_iatt (&req.stbuf, args->stbuf);
+int32_t
+client3_3_fallocate(call_frame_t *frame, xlator_t *this, void *data)
+{
+    clnt_args_t *args = NULL;
+    clnt_conf_t *conf = NULL;
+    gfs3_fallocate_req req = {
+        {0},
+    };
+    int op_errno = ESTALE;
+    int ret = 0;
+
+    if (!frame || !this || !data)
+        goto unwind;
+
+    args = data;
+    conf = this->private;
+
+    ret = client_pre_fallocate(this, &req, args->fd, args->flags, args->offset,
+                               args->size, args->xdata);
+    if (ret) {
+        op_errno = -ret;
+        goto unwind;
+    }
+
+    ret = client_submit_request(this, &req, frame, conf->fops,
+                                GFS3_OP_FALLOCATE, client3_3_fallocate_cbk,
+                                NULL, (xdrproc_t)xdr_gfs3_fallocate_req);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_WARNING, 0, PC_MSG_FOP_SEND_FAILED, NULL);
+    }
+
+    GF_FREE(req.xdata.xdata_val);
+
+    return 0;
+unwind:
+    CLIENT_STACK_UNWIND(fallocate, frame, -1, op_errno, NULL, NULL, NULL);
+    GF_FREE(req.xdata.xdata_val);
 
-        GF_PROTOCOL_DICT_SERIALIZE (this, args->xdata, (&req.xdata.xdata_val),
-                                    req.xdata.xdata_len, op_errno, unwind);
+    return 0;
+}
 
-        ret = client_submit_request (this, &req, frame, conf->fops,
-                                     GFS3_OP_FSETATTR,
-                                     client3_3_fsetattr_cbk, NULL,
-                                     NULL, 0, NULL, 0,
-                                     NULL, (xdrproc_t)xdr_gfs3_fsetattr_req);
-        if (ret) {
-                gf_log (this->name, GF_LOG_WARNING, "failed to send the fop");
-        }
+int32_t
+client3_3_discard(call_frame_t *frame, xlator_t *this, void *data)
+{
+    clnt_args_t *args = NULL;
+    clnt_conf_t *conf = NULL;
+    gfs3_discard_req req = {
+        {0},
+    };
+    int op_errno = ESTALE;
+    int ret = 0;
+
+    if (!frame || !this || !data)
+        goto unwind;
+
+    args = data;
+    conf = this->private;
+
+    ret = client_pre_discard(this, &req, args->fd, args->offset, args->size,
+                             args->xdata);
+    if (ret) {
+        op_errno = -ret;
+        goto unwind;
+    }
+
+    ret = client_submit_request(this, &req, frame, conf->fops, GFS3_OP_DISCARD,
+                                client3_3_discard_cbk, NULL,
+                                (xdrproc_t)xdr_gfs3_discard_req);
+    if (ret)
+        gf_smsg(this->name, GF_LOG_WARNING, 0, PC_MSG_FOP_SEND_FAILED, NULL);
+
+    GF_FREE(req.xdata.xdata_val);
+
+    return 0;
+unwind:
+    CLIENT_STACK_UNWIND(discard, frame, -1, op_errno, NULL, NULL, NULL);
+    GF_FREE(req.xdata.xdata_val);
 
-        GF_FREE (req.xdata.xdata_val);
+    return 0;
+}
 
-        return 0;
+int32_t
+client3_3_zerofill(call_frame_t *frame, xlator_t *this, void *data)
+{
+    clnt_args_t *args = NULL;
+    clnt_conf_t *conf = NULL;
+    gfs3_zerofill_req req = {
+        {0},
+    };
+    int op_errno = ESTALE;
+    int ret = 0;
+
+    GF_ASSERT(frame);
+
+    if (!this || !data)
+        goto unwind;
+
+    args = data;
+    conf = this->private;
+
+    ret = client_pre_zerofill(this, &req, args->fd, args->offset, args->size,
+                              args->xdata);
+    if (ret) {
+        op_errno = -ret;
+        goto unwind;
+    }
+
+    ret = client_submit_request(this, &req, frame, conf->fops, GFS3_OP_ZEROFILL,
+                                client3_3_zerofill_cbk, NULL,
+                                (xdrproc_t)xdr_gfs3_zerofill_req);
+    if (ret)
+        gf_smsg(this->name, GF_LOG_WARNING, 0, PC_MSG_FOP_SEND_FAILED, NULL);
+
+    GF_FREE(req.xdata.xdata_val);
+
+    return 0;
 unwind:
-        CLIENT_STACK_UNWIND (fsetattr, frame, -1, op_errno, NULL, NULL, NULL);
-        GF_FREE (req.xdata.xdata_val);
+    CLIENT_STACK_UNWIND(zerofill, frame, -1, op_errno, NULL, NULL, NULL);
+    GF_FREE(req.xdata.xdata_val);
 
-        return 0;
+    return 0;
 }
 
 int32_t
-client3_3_fallocate(call_frame_t *frame, xlator_t *this, void *data)
+client3_3_ipc(call_frame_t *frame, xlator_t *this, void *data)
 {
-        clnt_args_t       *args     = NULL;
-        int64_t            remote_fd = -1;
-        clnt_conf_t       *conf     = NULL;
-	gfs3_fallocate_req req	    = {{0},};
-        int                op_errno = ESTALE;
-        int                ret        = 0;
-
-        if (!frame || !this || !data)
-                goto unwind;
+    clnt_args_t *args = NULL;
+    clnt_conf_t *conf = NULL;
+    gfs3_ipc_req req = {
+        0,
+    };
+    int op_errno = ESTALE;
+    int ret = 0;
 
-        args = data;
-        conf = this->private;
+    GF_ASSERT(frame);
 
-        CLIENT_GET_REMOTE_FD (this, args->fd, DEFAULT_REMOTE_FD,
-                              remote_fd, op_errno, unwind);
+    if (!this || !data)
+        goto unwind;
 
-        req.fd = remote_fd;
-	req.flags = args->flags;
-	req.offset = args->offset;
-	req.size = args->size;
-	memcpy(req.gfid, args->fd->inode->gfid, 16);
+    args = data;
+    conf = this->private;
 
-        GF_PROTOCOL_DICT_SERIALIZE (this, args->xdata, (&req.xdata.xdata_val),
-                                    req.xdata.xdata_len, op_errno, unwind);
+    ret = client_pre_ipc(this, &req, args->cmd, args->xdata);
 
-        ret = client_submit_request (this, &req, frame, conf->fops,
-                                     GFS3_OP_FALLOCATE,
-                                     client3_3_fallocate_cbk, NULL,
-                                     NULL, 0, NULL, 0,
-                                     NULL, (xdrproc_t)xdr_gfs3_fallocate_req);
-        if (ret) {
-                gf_log (this->name, GF_LOG_WARNING, "failed to send the fop");
-        }
+    if (ret) {
+        op_errno = -ret;
+        goto unwind;
+    }
 
-        GF_FREE (req.xdata.xdata_val);
+    ret = client_submit_request(this, &req, frame, conf->fops, GFS3_OP_IPC,
+                                client3_3_ipc_cbk, NULL,
+                                (xdrproc_t)xdr_gfs3_ipc_req);
+    if (ret)
+        gf_smsg(this->name, GF_LOG_WARNING, 0, PC_MSG_FOP_SEND_FAILED, NULL);
 
-        return 0;
+    GF_FREE(req.xdata.xdata_val);
+
+    return 0;
 unwind:
-        CLIENT_STACK_UNWIND (fallocate, frame, -1, op_errno, NULL, NULL, NULL);
-        GF_FREE (req.xdata.xdata_val);
+    CLIENT_STACK_UNWIND(ipc, frame, -1, op_errno, NULL);
+    GF_FREE(req.xdata.xdata_val);
 
-        return 0;
+    return 0;
 }
 
 int32_t
-client3_3_discard(call_frame_t *frame, xlator_t *this, void *data)
+client3_3_seek(call_frame_t *frame, xlator_t *this, void *data)
 {
-        clnt_args_t       *args     = NULL;
-        int64_t            remote_fd = -1;
-        clnt_conf_t       *conf     = NULL;
-	gfs3_discard_req   req	    = {{0},};
-        int                op_errno = ESTALE;
-        int                ret        = 0;
-
-        if (!frame || !this || !data)
-                goto unwind;
+    clnt_args_t *args = NULL;
+    clnt_conf_t *conf = NULL;
+    struct gfs3_seek_req req = {
+        {
+            0,
+        },
+    };
+    int op_errno = ESTALE;
+    int ret = 0;
 
-        args = data;
-        conf = this->private;
+    GF_ASSERT(frame);
 
-        CLIENT_GET_REMOTE_FD (this, args->fd, DEFAULT_REMOTE_FD,
-                              remote_fd, op_errno, unwind);
+    if (!this || !data)
+        goto unwind;
 
-        req.fd = remote_fd;
-	req.offset = args->offset;
-	req.size = args->size;
-	memcpy(req.gfid, args->fd->inode->gfid, 16);
+    args = data;
+    conf = this->private;
 
-        GF_PROTOCOL_DICT_SERIALIZE (this, args->xdata, (&req.xdata.xdata_val),
-                                    req.xdata.xdata_len, op_errno, unwind);
+    ret = client_pre_seek(this, &req, args->fd, args->offset, args->what,
+                          args->xdata);
+    if (ret) {
+        op_errno = -ret;
+        goto unwind;
+    }
 
-        ret = client_submit_request(this, &req, frame, conf->fops,
-                                    GFS3_OP_DISCARD, client3_3_discard_cbk,
-				    NULL, NULL, 0, NULL, 0, NULL,
-                                    (xdrproc_t) xdr_gfs3_discard_req);
-        if (ret)
-                gf_log (this->name, GF_LOG_WARNING, "failed to send the fop");
+    ret = client_submit_request(this, &req, frame, conf->fops, GFS3_OP_SEEK,
+                                client3_3_seek_cbk, NULL,
+                                (xdrproc_t)xdr_gfs3_seek_req);
+    if (ret)
+        gf_smsg(this->name, GF_LOG_WARNING, 0, PC_MSG_FOP_SEND_FAILED, NULL);
 
-        GF_FREE (req.xdata.xdata_val);
+    GF_FREE(req.xdata.xdata_val);
 
-        return 0;
+    return 0;
 unwind:
-        CLIENT_STACK_UNWIND(discard, frame, -1, op_errno, NULL, NULL, NULL);
-        GF_FREE (req.xdata.xdata_val);
+    CLIENT_STACK_UNWIND(ipc, frame, -1, op_errno, NULL);
+    GF_FREE(req.xdata.xdata_val);
 
-        return 0;
+    return 0;
 }
 
-int32_t
-client3_3_zerofill(call_frame_t *frame, xlator_t *this, void *data)
+static int32_t
+client3_3_getactivelk(call_frame_t *frame, xlator_t *this, void *data)
 {
-        clnt_args_t       *args        = NULL;
-        int64_t            remote_fd   = -1;
-        clnt_conf_t       *conf        = NULL;
-        gfs3_zerofill_req   req        = {{0},};
-        int                op_errno    = ESTALE;
-        int                ret         = 0;
-
-        GF_ASSERT (frame);
+    clnt_conf_t *conf = NULL;
+    clnt_args_t *args = NULL;
+    gfs3_getactivelk_req req = {
+        {
+            0,
+        },
+    };
+    int ret = 0;
+    int op_errno = ESTALE;
+
+    if (!frame || !this || !data)
+        goto unwind;
+
+    args = data;
+    if (!(args->loc && args->loc->inode))
+        goto unwind;
+
+    if (!gf_uuid_is_null(args->loc->inode->gfid))
+        memcpy(req.gfid, args->loc->inode->gfid, 16);
+    else
+        memcpy(req.gfid, args->loc->gfid, 16);
+
+    GF_ASSERT_AND_GOTO_WITH_ERROR(this->name,
+                                  !gf_uuid_is_null(*((uuid_t *)req.gfid)),
+                                  unwind, op_errno, EINVAL);
+    conf = this->private;
+
+    GF_PROTOCOL_DICT_SERIALIZE(this, args->xdata, (&req.xdata.xdata_val),
+                               req.xdata.xdata_len, op_errno, unwind);
+
+    ret = client_submit_request(this, &req, frame, conf->fops,
+                                GFS3_OP_GETACTIVELK, client3_3_getactivelk_cbk,
+                                NULL, (xdrproc_t)xdr_gfs3_getactivelk_req);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_WARNING, 0, PC_MSG_FOP_SEND_FAILED, NULL);
+    }
+
+    GF_FREE(req.xdata.xdata_val);
+
+    return 0;
+unwind:
+    CLIENT_STACK_UNWIND(getactivelk, frame, -1, op_errno, NULL, NULL);
 
-        if (!this || !data)
-                goto unwind;
+    GF_FREE(req.xdata.xdata_val);
 
-        args = data;
-        conf = this->private;
+    return 0;
+}
 
-        CLIENT_GET_REMOTE_FD (this, args->fd, DEFAULT_REMOTE_FD,
-                              remote_fd, op_errno, unwind);
+static int32_t
+client3_3_setactivelk(call_frame_t *frame, xlator_t *this, void *data)
+{
+    clnt_conf_t *conf = NULL;
+    clnt_args_t *args = NULL;
+    gfs3_setactivelk_req req = {
+        {
+            0,
+        },
+    };
+    int ret = 0;
+    int op_errno = ESTALE;
 
-        req.fd = remote_fd;
-        req.offset = args->offset;
-        req.size = args->size;
-        memcpy(req.gfid, args->fd->inode->gfid, 16);
+    if (!frame || !this || !data)
+        goto unwind;
 
-        GF_PROTOCOL_DICT_SERIALIZE (this, args->xdata, (&req.xdata.xdata_val),
-                                    req.xdata.xdata_len, op_errno, unwind);
+    args = data;
+    if (!(args->loc && args->loc->inode && args->locklist))
+        goto unwind;
 
-        ret = client_submit_request(this, &req, frame, conf->fops,
-                                    GFS3_OP_ZEROFILL, client3_3_zerofill_cbk,
-                                    NULL, NULL, 0, NULL, 0, NULL,
-                                    (xdrproc_t) xdr_gfs3_zerofill_req);
-        if (ret)
-                gf_log (this->name, GF_LOG_WARNING, "failed to send the fop");
+    if (!gf_uuid_is_null(args->loc->inode->gfid))
+        memcpy(req.gfid, args->loc->inode->gfid, 16);
+    else
+        memcpy(req.gfid, args->loc->gfid, 16);
 
-        GF_FREE (req.xdata.xdata_val);
+    GF_ASSERT_AND_GOTO_WITH_ERROR(this->name,
+                                  !gf_uuid_is_null(*((uuid_t *)req.gfid)),
+                                  unwind, op_errno, EINVAL);
+    conf = this->private;
 
-        return 0;
-unwind:
-        CLIENT_STACK_UNWIND(zerofill, frame, -1, op_errno, NULL, NULL, NULL);
-        GF_FREE (req.xdata.xdata_val);
+    GF_PROTOCOL_DICT_SERIALIZE(this, args->xdata, (&req.xdata.xdata_val),
+                               req.xdata.xdata_len, op_errno, unwind);
 
-        return 0;
-}
+    ret = serialize_req_locklist(args->locklist, &req);
 
-int32_t
-client3_3_ipc (call_frame_t *frame, xlator_t *this, void *data)
-{
-        clnt_args_t       *args        = NULL;
-        clnt_conf_t       *conf        = NULL;
-        gfs3_ipc_req       req        = {0,};
-        int                op_errno    = ESTALE;
-        int                ret         = 0;
+    if (ret)
+        goto unwind;
 
-        GF_ASSERT (frame);
+    ret = client_submit_request(this, &req, frame, conf->fops,
+                                GFS3_OP_SETACTIVELK, client3_3_setactivelk_cbk,
+                                NULL, (xdrproc_t)xdr_gfs3_setactivelk_req);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_WARNING, 0, PC_MSG_FOP_SEND_FAILED, NULL);
+    }
 
-        if (!this || !data)
-                goto unwind;
+    clnt_setactivelk_req_cleanup(&req);
 
-        args = data;
-        conf = this->private;
+    GF_FREE(req.xdata.xdata_val);
 
-        req.op = args->cmd;
+    return 0;
 
-        GF_PROTOCOL_DICT_SERIALIZE (this, args->xdata, (&req.xdata.xdata_val),
-                                    req.xdata.xdata_len, op_errno, unwind);
+unwind:
 
-        ret = client_submit_request(this, &req, frame, conf->fops,
-                                    GFS3_OP_IPC, client3_3_ipc_cbk,
-                                    NULL, NULL, 0, NULL, 0, NULL,
-                                    (xdrproc_t) xdr_gfs3_ipc_req);
-        if (ret)
-                gf_log (this->name, GF_LOG_WARNING, "failed to send the fop");
+    CLIENT_STACK_UNWIND(setactivelk, frame, -1, op_errno, NULL);
 
-        GF_FREE (req.xdata.xdata_val);
+    GF_FREE(req.xdata.xdata_val);
 
-        return 0;
-unwind:
-        CLIENT_STACK_UNWIND(ipc, frame, -1, op_errno, NULL);
-        GF_FREE (req.xdata.xdata_val);
+    clnt_setactivelk_req_cleanup(&req);
 
-        return 0;
+    return 0;
 }
 
 /* Table Specific to FOPS */
-
-
 rpc_clnt_procedure_t clnt3_3_fop_actors[GF_FOP_MAXVALUE] = {
-        [GF_FOP_NULL]         = { "NULL",         NULL},
-        [GF_FOP_STAT]         = { "STAT",         client3_3_stat },
-        [GF_FOP_READLINK]     = { "READLINK",     client3_3_readlink },
-        [GF_FOP_MKNOD]        = { "MKNOD",        client3_3_mknod },
-        [GF_FOP_MKDIR]        = { "MKDIR",        client3_3_mkdir },
-        [GF_FOP_UNLINK]       = { "UNLINK",       client3_3_unlink },
-        [GF_FOP_RMDIR]        = { "RMDIR",        client3_3_rmdir },
-        [GF_FOP_SYMLINK]      = { "SYMLINK",      client3_3_symlink },
-        [GF_FOP_RENAME]       = { "RENAME",       client3_3_rename },
-        [GF_FOP_LINK]         = { "LINK",         client3_3_link },
-        [GF_FOP_TRUNCATE]     = { "TRUNCATE",     client3_3_truncate },
-        [GF_FOP_OPEN]         = { "OPEN",         client3_3_open },
-        [GF_FOP_READ]         = { "READ",         client3_3_readv },
-        [GF_FOP_WRITE]        = { "WRITE",        client3_3_writev },
-        [GF_FOP_STATFS]       = { "STATFS",       client3_3_statfs },
-        [GF_FOP_FLUSH]        = { "FLUSH",        client3_3_flush },
-        [GF_FOP_FSYNC]        = { "FSYNC",        client3_3_fsync },
-        [GF_FOP_SETXATTR]     = { "SETXATTR",     client3_3_setxattr },
-        [GF_FOP_GETXATTR]     = { "GETXATTR",     client3_3_getxattr },
-        [GF_FOP_REMOVEXATTR]  = { "REMOVEXATTR",  client3_3_removexattr },
-        [GF_FOP_OPENDIR]      = { "OPENDIR",      client3_3_opendir },
-        [GF_FOP_FSYNCDIR]     = { "FSYNCDIR",     client3_3_fsyncdir },
-        [GF_FOP_ACCESS]       = { "ACCESS",       client3_3_access },
-        [GF_FOP_CREATE]       = { "CREATE",       client3_3_create },
-        [GF_FOP_FTRUNCATE]    = { "FTRUNCATE",    client3_3_ftruncate },
-        [GF_FOP_FSTAT]        = { "FSTAT",        client3_3_fstat },
-        [GF_FOP_LK]           = { "LK",           client3_3_lk },
-        [GF_FOP_LOOKUP]       = { "LOOKUP",       client3_3_lookup },
-        [GF_FOP_READDIR]      = { "READDIR",      client3_3_readdir },
-        [GF_FOP_INODELK]      = { "INODELK",      client3_3_inodelk },
-        [GF_FOP_FINODELK]     = { "FINODELK",     client3_3_finodelk },
-        [GF_FOP_ENTRYLK]      = { "ENTRYLK",      client3_3_entrylk },
-        [GF_FOP_FENTRYLK]     = { "FENTRYLK",     client3_3_fentrylk },
-        [GF_FOP_XATTROP]      = { "XATTROP",      client3_3_xattrop },
-        [GF_FOP_FXATTROP]     = { "FXATTROP",     client3_3_fxattrop },
-        [GF_FOP_FGETXATTR]    = { "FGETXATTR",    client3_3_fgetxattr },
-        [GF_FOP_FSETXATTR]    = { "FSETXATTR",    client3_3_fsetxattr },
-        [GF_FOP_RCHECKSUM]    = { "RCHECKSUM",    client3_3_rchecksum },
-        [GF_FOP_SETATTR]      = { "SETATTR",      client3_3_setattr },
-        [GF_FOP_FSETATTR]     = { "FSETATTR",     client3_3_fsetattr },
-        [GF_FOP_READDIRP]     = { "READDIRP",     client3_3_readdirp },
-	[GF_FOP_FALLOCATE]    = { "FALLOCATE",	  client3_3_fallocate },
-	[GF_FOP_DISCARD]      = { "DISCARD",	  client3_3_discard },
-        [GF_FOP_ZEROFILL]     = { "ZEROFILL",     client3_3_zerofill},
-        [GF_FOP_RELEASE]      = { "RELEASE",      client3_3_release },
-        [GF_FOP_RELEASEDIR]   = { "RELEASEDIR",   client3_3_releasedir },
-        [GF_FOP_GETSPEC]      = { "GETSPEC",      client3_getspec },
-        [GF_FOP_FREMOVEXATTR] = { "FREMOVEXATTR", client3_3_fremovexattr },
-        [GF_FOP_IPC]          = { "IPC",          client3_3_ipc },
+    [GF_FOP_NULL] = {"NULL", NULL},
+    [GF_FOP_STAT] = {"STAT", client3_3_stat},
+    [GF_FOP_READLINK] = {"READLINK", client3_3_readlink},
+    [GF_FOP_MKNOD] = {"MKNOD", client3_3_mknod},
+    [GF_FOP_MKDIR] = {"MKDIR", client3_3_mkdir},
+    [GF_FOP_UNLINK] = {"UNLINK", client3_3_unlink},
+    [GF_FOP_RMDIR] = {"RMDIR", client3_3_rmdir},
+    [GF_FOP_SYMLINK] = {"SYMLINK", client3_3_symlink},
+    [GF_FOP_RENAME] = {"RENAME", client3_3_rename},
+    [GF_FOP_LINK] = {"LINK", client3_3_link},
+    [GF_FOP_TRUNCATE] = {"TRUNCATE", client3_3_truncate},
+    [GF_FOP_OPEN] = {"OPEN", client3_3_open},
+    [GF_FOP_READ] = {"READ", client3_3_readv},
+    [GF_FOP_WRITE] = {"WRITE", client3_3_writev},
+    [GF_FOP_STATFS] = {"STATFS", client3_3_statfs},
+    [GF_FOP_FLUSH] = {"FLUSH", client3_3_flush},
+    [GF_FOP_FSYNC] = {"FSYNC", client3_3_fsync},
+    [GF_FOP_SETXATTR] = {"SETXATTR", client3_3_setxattr},
+    [GF_FOP_GETXATTR] = {"GETXATTR", client3_3_getxattr},
+    [GF_FOP_REMOVEXATTR] = {"REMOVEXATTR", client3_3_removexattr},
+    [GF_FOP_OPENDIR] = {"OPENDIR", client3_3_opendir},
+    [GF_FOP_FSYNCDIR] = {"FSYNCDIR", client3_3_fsyncdir},
+    [GF_FOP_ACCESS] = {"ACCESS", client3_3_access},
+    [GF_FOP_CREATE] = {"CREATE", client3_3_create},
+    [GF_FOP_FTRUNCATE] = {"FTRUNCATE", client3_3_ftruncate},
+    [GF_FOP_FSTAT] = {"FSTAT", client3_3_fstat},
+    [GF_FOP_LK] = {"LK", client3_3_lk},
+    [GF_FOP_LOOKUP] = {"LOOKUP", client3_3_lookup},
+    [GF_FOP_READDIR] = {"READDIR", client3_3_readdir},
+    [GF_FOP_INODELK] = {"INODELK", client3_3_inodelk},
+    [GF_FOP_FINODELK] = {"FINODELK", client3_3_finodelk},
+    [GF_FOP_ENTRYLK] = {"ENTRYLK", client3_3_entrylk},
+    [GF_FOP_FENTRYLK] = {"FENTRYLK", client3_3_fentrylk},
+    [GF_FOP_XATTROP] = {"XATTROP", client3_3_xattrop},
+    [GF_FOP_FXATTROP] = {"FXATTROP", client3_3_fxattrop},
+    [GF_FOP_FGETXATTR] = {"FGETXATTR", client3_3_fgetxattr},
+    [GF_FOP_FSETXATTR] = {"FSETXATTR", client3_3_fsetxattr},
+    [GF_FOP_RCHECKSUM] = {"RCHECKSUM", client3_3_rchecksum},
+    [GF_FOP_SETATTR] = {"SETATTR", client3_3_setattr},
+    [GF_FOP_FSETATTR] = {"FSETATTR", client3_3_fsetattr},
+    [GF_FOP_READDIRP] = {"READDIRP", client3_3_readdirp},
+    [GF_FOP_FALLOCATE] = {"FALLOCATE", client3_3_fallocate},
+    [GF_FOP_DISCARD] = {"DISCARD", client3_3_discard},
+    [GF_FOP_ZEROFILL] = {"ZEROFILL", client3_3_zerofill},
+    [GF_FOP_RELEASE] = {"RELEASE", client3_3_release},
+    [GF_FOP_RELEASEDIR] = {"RELEASEDIR", client3_3_releasedir},
+    [GF_FOP_GETSPEC] = {"GETSPEC", client3_getspec},
+    [GF_FOP_FREMOVEXATTR] = {"FREMOVEXATTR", client3_3_fremovexattr},
+    [GF_FOP_IPC] = {"IPC", client3_3_ipc},
+    [GF_FOP_SEEK] = {"SEEK", client3_3_seek},
+    [GF_FOP_LEASE] = {"LEASE", client3_3_lease},
+    [GF_FOP_GETACTIVELK] = {"GETACTIVELK", client3_3_getactivelk},
+    [GF_FOP_SETACTIVELK] = {"SETACTIVELK", client3_3_setactivelk},
+    [GF_FOP_COMPOUND] = {"COMPOUND", NULL},
 };
 
 /* Used From RPC-CLNT library to log proper name of procedure based on number */
 char *clnt3_3_fop_names[GFS3_OP_MAXVALUE] = {
-        [GFS3_OP_NULL]        = "NULL",
-        [GFS3_OP_STAT]        = "STAT",
-        [GFS3_OP_READLINK]    = "READLINK",
-        [GFS3_OP_MKNOD]       = "MKNOD",
-        [GFS3_OP_MKDIR]       = "MKDIR",
-        [GFS3_OP_UNLINK]      = "UNLINK",
-        [GFS3_OP_RMDIR]       = "RMDIR",
-        [GFS3_OP_SYMLINK]     = "SYMLINK",
-        [GFS3_OP_RENAME]      = "RENAME",
-        [GFS3_OP_LINK]        = "LINK",
-        [GFS3_OP_TRUNCATE]    = "TRUNCATE",
-        [GFS3_OP_OPEN]        = "OPEN",
-        [GFS3_OP_READ]        = "READ",
-        [GFS3_OP_WRITE]       = "WRITE",
-        [GFS3_OP_STATFS]      = "STATFS",
-        [GFS3_OP_FLUSH]       = "FLUSH",
-        [GFS3_OP_FSYNC]       = "FSYNC",
-        [GFS3_OP_SETXATTR]    = "SETXATTR",
-        [GFS3_OP_GETXATTR]    = "GETXATTR",
-        [GFS3_OP_REMOVEXATTR] = "REMOVEXATTR",
-        [GFS3_OP_OPENDIR]     = "OPENDIR",
-        [GFS3_OP_FSYNCDIR]    = "FSYNCDIR",
-        [GFS3_OP_ACCESS]      = "ACCESS",
-        [GFS3_OP_CREATE]      = "CREATE",
-        [GFS3_OP_FTRUNCATE]   = "FTRUNCATE",
-        [GFS3_OP_FSTAT]       = "FSTAT",
-        [GFS3_OP_LK]          = "LK",
-        [GFS3_OP_LOOKUP]      = "LOOKUP",
-        [GFS3_OP_READDIR]     = "READDIR",
-        [GFS3_OP_INODELK]     = "INODELK",
-        [GFS3_OP_FINODELK]    = "FINODELK",
-        [GFS3_OP_ENTRYLK]     = "ENTRYLK",
-        [GFS3_OP_FENTRYLK]    = "FENTRYLK",
-        [GFS3_OP_XATTROP]     = "XATTROP",
-        [GFS3_OP_FXATTROP]    = "FXATTROP",
-        [GFS3_OP_FGETXATTR]   = "FGETXATTR",
-        [GFS3_OP_FSETXATTR]   = "FSETXATTR",
-        [GFS3_OP_RCHECKSUM]   = "RCHECKSUM",
-        [GFS3_OP_SETATTR]     = "SETATTR",
-        [GFS3_OP_FSETATTR]    = "FSETATTR",
-        [GFS3_OP_READDIRP]    = "READDIRP",
-        [GFS3_OP_RELEASE]     = "RELEASE",
-        [GFS3_OP_RELEASEDIR]  = "RELEASEDIR",
-        [GFS3_OP_FREMOVEXATTR] = "FREMOVEXATTR",
-	[GFS3_OP_FALLOCATE]   = "FALLOCATE",
-	[GFS3_OP_DISCARD]     = "DISCARD",
-        [GFS3_OP_ZEROFILL]    = "ZEROFILL",
-        [GFS3_OP_IPC]         = "IPC",
-
+    [GFS3_OP_NULL] = "NULL",
+    [GFS3_OP_STAT] = "STAT",
+    [GFS3_OP_READLINK] = "READLINK",
+    [GFS3_OP_MKNOD] = "MKNOD",
+    [GFS3_OP_MKDIR] = "MKDIR",
+    [GFS3_OP_UNLINK] = "UNLINK",
+    [GFS3_OP_RMDIR] = "RMDIR",
+    [GFS3_OP_SYMLINK] = "SYMLINK",
+    [GFS3_OP_RENAME] = "RENAME",
+    [GFS3_OP_LINK] = "LINK",
+    [GFS3_OP_TRUNCATE] = "TRUNCATE",
+    [GFS3_OP_OPEN] = "OPEN",
+    [GFS3_OP_READ] = "READ",
+    [GFS3_OP_WRITE] = "WRITE",
+    [GFS3_OP_STATFS] = "STATFS",
+    [GFS3_OP_FLUSH] = "FLUSH",
+    [GFS3_OP_FSYNC] = "FSYNC",
+    [GFS3_OP_SETXATTR] = "SETXATTR",
+    [GFS3_OP_GETXATTR] = "GETXATTR",
+    [GFS3_OP_REMOVEXATTR] = "REMOVEXATTR",
+    [GFS3_OP_OPENDIR] = "OPENDIR",
+    [GFS3_OP_FSYNCDIR] = "FSYNCDIR",
+    [GFS3_OP_ACCESS] = "ACCESS",
+    [GFS3_OP_CREATE] = "CREATE",
+    [GFS3_OP_FTRUNCATE] = "FTRUNCATE",
+    [GFS3_OP_FSTAT] = "FSTAT",
+    [GFS3_OP_LK] = "LK",
+    [GFS3_OP_LOOKUP] = "LOOKUP",
+    [GFS3_OP_READDIR] = "READDIR",
+    [GFS3_OP_INODELK] = "INODELK",
+    [GFS3_OP_FINODELK] = "FINODELK",
+    [GFS3_OP_ENTRYLK] = "ENTRYLK",
+    [GFS3_OP_FENTRYLK] = "FENTRYLK",
+    [GFS3_OP_XATTROP] = "XATTROP",
+    [GFS3_OP_FXATTROP] = "FXATTROP",
+    [GFS3_OP_FGETXATTR] = "FGETXATTR",
+    [GFS3_OP_FSETXATTR] = "FSETXATTR",
+    [GFS3_OP_RCHECKSUM] = "RCHECKSUM",
+    [GFS3_OP_SETATTR] = "SETATTR",
+    [GFS3_OP_FSETATTR] = "FSETATTR",
+    [GFS3_OP_READDIRP] = "READDIRP",
+    [GFS3_OP_RELEASE] = "RELEASE",
+    [GFS3_OP_RELEASEDIR] = "RELEASEDIR",
+    [GFS3_OP_FREMOVEXATTR] = "FREMOVEXATTR",
+    [GFS3_OP_FALLOCATE] = "FALLOCATE",
+    [GFS3_OP_DISCARD] = "DISCARD",
+    [GFS3_OP_ZEROFILL] = "ZEROFILL",
+    [GFS3_OP_IPC] = "IPC",
+    [GFS3_OP_SEEK] = "SEEK",
+    [GFS3_OP_LEASE] = "LEASE",
+    [GFS3_OP_GETACTIVELK] = "GETACTIVELK",
+    [GFS3_OP_SETACTIVELK] = "SETACTIVELK",
+    [GFS3_OP_COMPOUND] = "COMPOUND",
 };
 
 rpc_clnt_prog_t clnt3_3_fop_prog = {
-        .progname  = "GlusterFS 3.3",
-        .prognum   = GLUSTER_FOP_PROGRAM,
-        .progver   = GLUSTER_FOP_VERSION,
-        .numproc   = GLUSTER_FOP_PROCCNT,
-        .proctable = clnt3_3_fop_actors,
-        .procnames = clnt3_3_fop_names,
+    .progname = "GlusterFS 3.3",
+    .prognum = GLUSTER_FOP_PROGRAM,
+    .progver = GLUSTER_FOP_VERSION,
+    .numproc = GLUSTER_FOP_PROCCNT,
+    .proctable = clnt3_3_fop_actors,
+    .procnames = clnt3_3_fop_names,
 };
diff --git a/xlators/protocol/client/src/client-rpc-fops_v2.c b/xlators/protocol/client/src/client-rpc-fops_v2.c
new file mode 100644
index 00000000000..0d80d4e8efb
--- /dev/null
+++ b/xlators/protocol/client/src/client-rpc-fops_v2.c
@@ -0,0 +1,6177 @@
+/*
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
+*/
+
+#include "client.h"
+#include "rpc-common-xdr.h"
+#include "glusterfs4-xdr.h"
+#include "glusterfs3.h"
+#include <glusterfs/compat-errno.h>
+#include "client-messages.h"
+#include <glusterfs/defaults.h>
+#include "client-common.h"
+
+extern int32_t
+client3_getspec(call_frame_t *frame, xlator_t *this, void *data);
+extern int32_t
+client3_3_getxattr(call_frame_t *frame, xlator_t *this, void *data);
+
+int
+client4_0_symlink_cbk(struct rpc_req *req, struct iovec *iov, int count,
+                      void *myframe)
+{
+    gfx_common_3iatt_rsp rsp = {
+        0,
+    };
+    call_frame_t *frame = NULL;
+    struct iatt stbuf = {
+        0,
+    };
+    struct iatt preparent = {
+        0,
+    };
+    struct iatt postparent = {
+        0,
+    };
+    int ret = 0;
+    clnt_local_t *local = NULL;
+    inode_t *inode = NULL;
+    xlator_t *this = NULL;
+    dict_t *xdata = NULL;
+
+    this = THIS;
+
+    frame = myframe;
+
+    local = frame->local;
+    inode = local->loc.inode;
+
+    if (-1 == req->rpc_status) {
+        rsp.op_ret = -1;
+        rsp.op_errno = ENOTCONN;
+        goto out;
+    }
+    ret = xdr_to_generic(*iov, &rsp, (xdrproc_t)xdr_gfx_common_3iatt_rsp);
+    if (ret < 0) {
+        gf_smsg(this->name, GF_LOG_ERROR, EINVAL, PC_MSG_XDR_DECODING_FAILED,
+                NULL);
+        rsp.op_ret = -1;
+        rsp.op_errno = EINVAL;
+        goto out;
+    }
+
+    ret = client_post_common_3iatt(this, &rsp, &stbuf, &preparent, &postparent,
+                                   &xdata);
+
+out:
+    if (rsp.op_ret == -1) {
+        if (GF_IGNORE_IF_GSYNCD_SAFE_ERROR(frame, rsp.op_errno)) {
+            /* no need to print the gfid, because it will be null,
+             * since symlink operation failed.
+             */
+            gf_smsg(this->name, GF_LOG_WARNING, gf_error_to_errno(rsp.op_errno),
+                    PC_MSG_REMOTE_OP_FAILED, "source=%s", local->loc.path,
+                    "target=%s", local->loc2.path, NULL);
+        }
+    }
+
+    CLIENT_STACK_UNWIND(symlink, frame, rsp.op_ret,
+                        gf_error_to_errno(rsp.op_errno), inode, &stbuf,
+                        &preparent, &postparent, xdata);
+
+    if (xdata)
+        dict_unref(xdata);
+
+    return 0;
+}
+
+int
+client4_0_mknod_cbk(struct rpc_req *req, struct iovec *iov, int count,
+                    void *myframe)
+{
+    gfx_common_3iatt_rsp rsp = {
+        0,
+    };
+    call_frame_t *frame = NULL;
+    struct iatt stbuf = {
+        0,
+    };
+    struct iatt preparent = {
+        0,
+    };
+    struct iatt postparent = {
+        0,
+    };
+    int ret = 0;
+    clnt_local_t *local = NULL;
+    inode_t *inode = NULL;
+    xlator_t *this = NULL;
+    dict_t *xdata = NULL;
+
+    this = THIS;
+
+    frame = myframe;
+
+    local = frame->local;
+
+    inode = local->loc.inode;
+
+    if (-1 == req->rpc_status) {
+        rsp.op_ret = -1;
+        rsp.op_errno = ENOTCONN;
+        goto out;
+    }
+    ret = xdr_to_generic(*iov, &rsp, (xdrproc_t)xdr_gfx_common_3iatt_rsp);
+    if (ret < 0) {
+        gf_smsg(this->name, GF_LOG_ERROR, EINVAL, PC_MSG_XDR_DECODING_FAILED,
+                NULL);
+        rsp.op_ret = -1;
+        rsp.op_errno = EINVAL;
+        goto out;
+    }
+
+    ret = client_post_common_3iatt(this, &rsp, &stbuf, &preparent, &postparent,
+                                   &xdata);
+
+out:
+    if (rsp.op_ret == -1 &&
+        GF_IGNORE_IF_GSYNCD_SAFE_ERROR(frame, rsp.op_errno)) {
+        gf_smsg(this->name,
+                fop_log_level(GF_FOP_MKNOD, gf_error_to_errno(rsp.op_errno)),
+                gf_error_to_errno(rsp.op_errno), PC_MSG_REMOTE_OP_FAILED,
+                "path=%s", local->loc.path, NULL);
+    }
+
+    CLIENT_STACK_UNWIND(mknod, frame, rsp.op_ret,
+                        gf_error_to_errno(rsp.op_errno), inode, &stbuf,
+                        &preparent, &postparent, xdata);
+
+    if (xdata)
+        dict_unref(xdata);
+
+    return 0;
+}
+
+int
+client4_0_mkdir_cbk(struct rpc_req *req, struct iovec *iov, int count,
+                    void *myframe)
+{
+    gfx_common_3iatt_rsp rsp = {
+        0,
+    };
+    call_frame_t *frame = NULL;
+    struct iatt stbuf = {
+        0,
+    };
+    struct iatt preparent = {
+        0,
+    };
+    struct iatt postparent = {
+        0,
+    };
+    int ret = 0;
+    clnt_local_t *local = NULL;
+    inode_t *inode = NULL;
+    xlator_t *this = NULL;
+    dict_t *xdata = NULL;
+
+    this = THIS;
+
+    frame = myframe;
+
+    local = frame->local;
+    inode = local->loc.inode;
+
+    if (-1 == req->rpc_status) {
+        rsp.op_ret = -1;
+        rsp.op_errno = ENOTCONN;
+        goto out;
+    }
+    ret = xdr_to_generic(*iov, &rsp, (xdrproc_t)xdr_gfx_common_3iatt_rsp);
+    if (ret < 0) {
+        gf_smsg(this->name, GF_LOG_ERROR, EINVAL, PC_MSG_XDR_DECODING_FAILED,
+                NULL);
+        rsp.op_ret = -1;
+        rsp.op_errno = EINVAL;
+        goto out;
+    }
+
+    ret = client_post_common_3iatt(this, &rsp, &stbuf, &preparent, &postparent,
+                                   &xdata);
+
+out:
+    if (rsp.op_ret == -1 &&
+        GF_IGNORE_IF_GSYNCD_SAFE_ERROR(frame, rsp.op_errno)) {
+        gf_smsg(this->name,
+                fop_log_level(GF_FOP_MKDIR, gf_error_to_errno(rsp.op_errno)),
+                gf_error_to_errno(rsp.op_errno), PC_MSG_REMOTE_OP_FAILED,
+                "path=%s", local->loc.path, NULL);
+    }
+
+    CLIENT_STACK_UNWIND(mkdir, frame, rsp.op_ret,
+                        gf_error_to_errno(rsp.op_errno), inode, &stbuf,
+                        &preparent, &postparent, xdata);
+
+    if (xdata)
+        dict_unref(xdata);
+
+    return 0;
+}
+
+int
+client4_0_open_cbk(struct rpc_req *req, struct iovec *iov, int count,
+                   void *myframe)
+{
+    clnt_local_t *local = NULL;
+    call_frame_t *frame = NULL;
+    fd_t *fd = NULL;
+    int ret = 0;
+    gfx_open_rsp rsp = {
+        0,
+    };
+    xlator_t *this = NULL;
+    dict_t *xdata = NULL;
+
+    this = THIS;
+
+    frame = myframe;
+    local = frame->local;
+
+    fd = local->fd;
+
+    if (-1 == req->rpc_status) {
+        rsp.op_ret = -1;
+        rsp.op_errno = ENOTCONN;
+        goto out;
+    }
+    ret = xdr_to_generic(*iov, &rsp, (xdrproc_t)xdr_gfx_open_rsp);
+    if (ret < 0) {
+        gf_smsg(this->name, GF_LOG_ERROR, EINVAL, PC_MSG_XDR_DECODING_FAILED,
+                NULL);
+        rsp.op_ret = -1;
+        rsp.op_errno = EINVAL;
+        goto out;
+    }
+
+    if (-1 != rsp.op_ret) {
+        ret = client_add_fd_to_saved_fds(frame->this, fd, &local->loc,
+                                         local->flags, rsp.fd, 0);
+        if (ret) {
+            rsp.op_ret = -1;
+            rsp.op_errno = -ret;
+            goto out;
+        }
+    }
+
+    ret = xdr_to_dict(&rsp.xdata, &xdata);
+out:
+    if (rsp.op_ret == -1) {
+        gf_smsg(this->name,
+                fop_log_level(GF_FOP_OPEN, gf_error_to_errno(rsp.op_errno)),
+                gf_error_to_errno(rsp.op_errno), PC_MSG_REMOTE_OP_FAILED,
+                "path=%s", local->loc.path, "gfid=%s",
+                loc_gfid_utoa(&local->loc), NULL);
+    }
+
+    CLIENT_STACK_UNWIND(open, frame, rsp.op_ret,
+                        gf_error_to_errno(rsp.op_errno), fd, xdata);
+
+    if (xdata)
+        dict_unref(xdata);
+
+    return 0;
+}
+
+int
+client4_0_stat_cbk(struct rpc_req *req, struct iovec *iov, int count,
+                   void *myframe)
+{
+    gfx_common_iatt_rsp rsp = {
+        0,
+    };
+    call_frame_t *frame = NULL;
+    struct iatt iatt = {
+        0,
+    };
+    int ret = 0;
+    xlator_t *this = NULL;
+    dict_t *xdata = NULL;
+
+    this = THIS;
+
+    frame = myframe;
+
+    if (-1 == req->rpc_status) {
+        rsp.op_ret = -1;
+        rsp.op_errno = ENOTCONN;
+        goto out;
+    }
+    ret = xdr_to_generic(*iov, &rsp, (xdrproc_t)xdr_gfx_common_iatt_rsp);
+    if (ret < 0) {
+        gf_smsg(this->name, GF_LOG_ERROR, EINVAL, PC_MSG_XDR_DECODING_FAILED,
+                NULL);
+        rsp.op_ret = -1;
+        rsp.op_errno = EINVAL;
+        goto out;
+    }
+
+    ret = client_post_common_iatt(this, &rsp, &iatt, &xdata);
+out:
+    if (rsp.op_ret == -1) {
+        /* stale filehandles are possible during normal operations, no
+         * need to spam the logs with these */
+        if (rsp.op_errno == ESTALE) {
+            gf_msg_debug(this->name, 0, "remote operation failed: %s",
+                         strerror(gf_error_to_errno(rsp.op_errno)));
+        } else {
+            gf_smsg(this->name, GF_LOG_WARNING, gf_error_to_errno(rsp.op_errno),
+                    PC_MSG_REMOTE_OP_FAILED, NULL);
+        }
+    }
+
+    CLIENT_STACK_UNWIND(stat, frame, rsp.op_ret,
+                        gf_error_to_errno(rsp.op_errno), &iatt, xdata);
+
+    if (xdata)
+        dict_unref(xdata);
+
+    return 0;
+}
+
+int
+client4_0_readlink_cbk(struct rpc_req *req, struct iovec *iov, int count,
+                       void *myframe)
+{
+    gfx_readlink_rsp rsp = {
+        0,
+    };
+    call_frame_t *frame = NULL;
+    struct iatt iatt = {
+        0,
+    };
+    int ret = 0;
+    xlator_t *this = NULL;
+    dict_t *xdata = NULL;
+
+    this = THIS;
+
+    frame = myframe;
+
+    if (-1 == req->rpc_status) {
+        rsp.op_ret = -1;
+        rsp.op_errno = ENOTCONN;
+        goto out;
+    }
+    ret = xdr_to_generic(*iov, &rsp, (xdrproc_t)xdr_gfx_readlink_rsp);
+    if (ret < 0) {
+        gf_smsg(this->name, GF_LOG_ERROR, EINVAL, PC_MSG_XDR_DECODING_FAILED,
+                NULL);
+        rsp.op_ret = -1;
+        rsp.op_errno = EINVAL;
+        goto out;
+    }
+    gfx_stat_to_iattx(&rsp.buf, &iatt);
+
+    ret = xdr_to_dict(&rsp.xdata, &xdata);
+out:
+    if (rsp.op_ret == -1) {
+        if (gf_error_to_errno(rsp.op_errno) == ENOENT) {
+            gf_msg_debug(this->name, 0,
+                         "remote operation failed:"
+                         " %s",
+                         strerror(gf_error_to_errno(rsp.op_errno)));
+        } else {
+            gf_smsg(this->name, GF_LOG_WARNING, gf_error_to_errno(rsp.op_errno),
+                    PC_MSG_REMOTE_OP_FAILED, NULL);
+        }
+    }
+
+    CLIENT_STACK_UNWIND(readlink, frame, rsp.op_ret,
+                        gf_error_to_errno(rsp.op_errno), rsp.path, &iatt,
+                        xdata);
+
+    /* This is allocated by the libc while decoding RPC msg */
+    /* Hence no 'GF_FREE', but just 'free' */
+    free(rsp.path);
+
+    if (xdata)
+        dict_unref(xdata);
+
+    return 0;
+}
+
+int
+client4_0_unlink_cbk(struct rpc_req *req, struct iovec *iov, int count,
+                     void *myframe)
+{
+    call_frame_t *frame = NULL;
+    gfx_common_2iatt_rsp rsp = {
+        0,
+    };
+    struct iatt preparent = {
+        0,
+    };
+    struct iatt postparent = {
+        0,
+    };
+    int ret = 0;
+    xlator_t *this = NULL;
+    dict_t *xdata = NULL;
+
+    this = THIS;
+
+    frame = myframe;
+
+    if (-1 == req->rpc_status) {
+        rsp.op_ret = -1;
+        rsp.op_errno = ENOTCONN;
+        goto out;
+    }
+    ret = xdr_to_generic(*iov, &rsp, (xdrproc_t)xdr_gfx_common_2iatt_rsp);
+    if (ret < 0) {
+        gf_smsg(this->name, GF_LOG_ERROR, EINVAL, PC_MSG_XDR_DECODING_FAILED,
+                NULL);
+        rsp.op_ret = -1;
+        rsp.op_errno = EINVAL;
+        goto out;
+    }
+
+    ret = client_post_common_2iatt(this, &rsp, &preparent, &postparent, &xdata);
+
+out:
+    if (rsp.op_ret == -1) {
+        if (gf_error_to_errno(rsp.op_errno) == ENOENT) {
+            gf_msg_debug(this->name, 0,
+                         "remote operation failed:"
+                         " %s",
+                         strerror(gf_error_to_errno(rsp.op_errno)));
+        } else {
+            gf_smsg(this->name, GF_LOG_WARNING, gf_error_to_errno(rsp.op_errno),
+                    PC_MSG_REMOTE_OP_FAILED, NULL);
+        }
+    }
+
+    CLIENT_STACK_UNWIND(unlink, frame, rsp.op_ret,
+                        gf_error_to_errno(rsp.op_errno), &preparent,
+                        &postparent, xdata);
+
+    if (xdata)
+        dict_unref(xdata);
+
+    return 0;
+}
+
+int
+client4_0_rmdir_cbk(struct rpc_req *req, struct iovec *iov, int count,
+                    void *myframe)
+{
+    gfx_common_2iatt_rsp rsp = {
+        0,
+    };
+    call_frame_t *frame = NULL;
+    struct iatt preparent = {
+        0,
+    };
+    struct iatt postparent = {
+        0,
+    };
+    int ret = 0;
+    xlator_t *this = NULL;
+    dict_t *xdata = NULL;
+
+    this = THIS;
+
+    frame = myframe;
+
+    if (-1 == req->rpc_status) {
+        rsp.op_ret = -1;
+        rsp.op_errno = ENOTCONN;
+        goto out;
+    }
+    ret = xdr_to_generic(*iov, &rsp, (xdrproc_t)xdr_gfx_common_2iatt_rsp);
+    if (ret < 0) {
+        gf_smsg(this->name, GF_LOG_ERROR, EINVAL, PC_MSG_XDR_DECODING_FAILED,
+                NULL);
+        rsp.op_ret = -1;
+        rsp.op_errno = EINVAL;
+        goto out;
+    }
+
+    ret = client_post_common_2iatt(this, &rsp, &preparent, &postparent, &xdata);
+
+out:
+    if (rsp.op_ret == -1) {
+        if (GF_IGNORE_IF_GSYNCD_SAFE_ERROR(frame, rsp.op_errno)) {
+            gf_smsg(this->name, GF_LOG_WARNING, gf_error_to_errno(rsp.op_errno),
+                    PC_MSG_REMOTE_OP_FAILED, NULL);
+        }
+    }
+    CLIENT_STACK_UNWIND(rmdir, frame, rsp.op_ret,
+                        gf_error_to_errno(rsp.op_errno), &preparent,
+                        &postparent, xdata);
+
+    if (xdata)
+        dict_unref(xdata);
+
+    return 0;
+}
+
+int
+client4_0_truncate_cbk(struct rpc_req *req, struct iovec *iov, int count,
+                       void *myframe)
+{
+    gfx_common_2iatt_rsp rsp = {
+        0,
+    };
+    call_frame_t *frame = NULL;
+    struct iatt prestat = {
+        0,
+    };
+    struct iatt poststat = {
+        0,
+    };
+    int ret = 0;
+    xlator_t *this = NULL;
+    dict_t *xdata = NULL;
+
+    this = THIS;
+
+    frame = myframe;
+
+    if (-1 == req->rpc_status) {
+        rsp.op_ret = -1;
+        rsp.op_errno = ENOTCONN;
+        goto out;
+    }
+    ret = xdr_to_generic(*iov, &rsp, (xdrproc_t)xdr_gfx_common_2iatt_rsp);
+    if (ret < 0) {
+        gf_smsg(this->name, GF_LOG_ERROR, EINVAL, PC_MSG_XDR_DECODING_FAILED,
+                NULL);
+        rsp.op_ret = -1;
+        rsp.op_errno = EINVAL;
+        goto out;
+    }
+
+    ret = client_post_common_2iatt(this, &rsp, &prestat, &poststat, &xdata);
+
+out:
+    if (rsp.op_ret == -1) {
+        gf_smsg(this->name, GF_LOG_WARNING, gf_error_to_errno(rsp.op_errno),
+                PC_MSG_REMOTE_OP_FAILED, NULL);
+    }
+    CLIENT_STACK_UNWIND(truncate, frame, rsp.op_ret,
+                        gf_error_to_errno(rsp.op_errno), &prestat, &poststat,
+                        xdata);
+
+    if (xdata)
+        dict_unref(xdata);
+
+    return 0;
+}
+
+int
+client4_0_statfs_cbk(struct rpc_req *req, struct iovec *iov, int count,
+                     void *myframe)
+{
+    gfx_statfs_rsp rsp = {
+        0,
+    };
+    call_frame_t *frame = NULL;
+    struct statvfs statfs = {
+        0,
+    };
+    int ret = 0;
+    xlator_t *this = NULL;
+    dict_t *xdata = NULL;
+
+    this = THIS;
+
+    frame = myframe;
+
+    if (-1 == req->rpc_status) {
+        rsp.op_ret = -1;
+        rsp.op_errno = ENOTCONN;
+        goto out;
+    }
+    ret = xdr_to_generic(*iov, &rsp, (xdrproc_t)xdr_gfx_statfs_rsp);
+    if (ret < 0) {
+        gf_smsg(this->name, GF_LOG_ERROR, EINVAL, PC_MSG_XDR_DECODING_FAILED,
+                NULL);
+        rsp.op_ret = -1;
+        rsp.op_errno = EINVAL;
+        goto out;
+    }
+
+    if (-1 != rsp.op_ret) {
+        gf_statfs_to_statfs(&rsp.statfs, &statfs);
+    }
+    ret = xdr_to_dict(&rsp.xdata, &xdata);
+out:
+    if (rsp.op_ret == -1) {
+        gf_smsg(this->name, GF_LOG_WARNING, gf_error_to_errno(rsp.op_errno),
+                PC_MSG_REMOTE_OP_FAILED, NULL);
+    }
+    CLIENT_STACK_UNWIND(statfs, frame, rsp.op_ret,
+                        gf_error_to_errno(rsp.op_errno), &statfs, xdata);
+
+    if (xdata)
+        dict_unref(xdata);
+
+    return 0;
+}
+
+int
+client4_0_writev_cbk(struct rpc_req *req, struct iovec *iov, int count,
+                     void *myframe)
+{
+    gfx_common_2iatt_rsp rsp = {
+        0,
+    };
+    call_frame_t *frame = NULL;
+    struct iatt prestat = {
+        0,
+    };
+    struct iatt poststat = {
+        0,
+    };
+    int ret = 0;
+    xlator_t *this = NULL;
+    dict_t *xdata = NULL;
+    clnt_local_t *local = NULL;
+
+    this = THIS;
+
+    frame = myframe;
+    local = frame->local;
+
+    if (-1 == req->rpc_status) {
+        rsp.op_ret = -1;
+        rsp.op_errno = ENOTCONN;
+        goto out;
+    }
+
+    ret = xdr_to_generic(*iov, &rsp, (xdrproc_t)xdr_gfx_common_2iatt_rsp);
+    if (ret < 0) {
+        gf_smsg(this->name, GF_LOG_ERROR, EINVAL, PC_MSG_XDR_DECODING_FAILED,
+                NULL);
+        rsp.op_ret = -1;
+        rsp.op_errno = EINVAL;
+        goto out;
+    }
+
+    ret = client_post_common_2iatt(this, &rsp, &prestat, &poststat, &xdata);
+    if (ret < 0)
+        goto out;
+out:
+    if (rsp.op_ret == -1) {
+        gf_smsg(this->name, GF_LOG_WARNING, gf_error_to_errno(rsp.op_errno),
+                PC_MSG_REMOTE_OP_FAILED, NULL);
+    } else if (rsp.op_ret >= 0) {
+        if (local->attempt_reopen)
+            client_attempt_reopen(local->fd, this);
+    }
+    CLIENT_STACK_UNWIND(writev, frame, rsp.op_ret,
+                        gf_error_to_errno(rsp.op_errno), &prestat, &poststat,
+                        xdata);
+
+    if (xdata)
+        dict_unref(xdata);
+
+    return 0;
+}
+
+int
+client4_0_flush_cbk(struct rpc_req *req, struct iovec *iov, int count,
+                    void *myframe)
+{
+    call_frame_t *frame = NULL;
+    clnt_local_t *local = NULL;
+    xlator_t *this = NULL;
+    dict_t *xdata = NULL;
+    gfx_common_rsp rsp = {
+        0,
+    };
+    int ret = 0;
+
+    frame = myframe;
+    this = THIS;
+    local = frame->local;
+
+    if (-1 == req->rpc_status) {
+        rsp.op_ret = -1;
+        rsp.op_errno = ENOTCONN;
+        goto out;
+    }
+    ret = xdr_to_generic(*iov, &rsp, (xdrproc_t)xdr_gfx_common_rsp);
+    if (ret < 0) {
+        gf_smsg(this->name, GF_LOG_ERROR, EINVAL, PC_MSG_XDR_DECODING_FAILED,
+                NULL);
+        rsp.op_ret = -1;
+        rsp.op_errno = EINVAL;
+        goto out;
+    }
+
+    if ((rsp.op_ret >= 0 || (rsp.op_errno == ENOTCONN)) &&
+        !fd_is_anonymous(local->fd)) {
+        /* Delete all saved locks of the owner issuing flush */
+        ret = delete_granted_locks_owner(local->fd, &local->owner);
+        gf_msg_trace(this->name, 0, "deleting locks of owner (%s) returned %d",
+                     lkowner_utoa(&local->owner), ret);
+    }
+
+    xdr_to_dict(&rsp.xdata, &xdata);
+
+out:
+    if (rsp.op_ret == -1) {
+        gf_smsg(this->name,
+                fop_log_level(GF_FOP_FLUSH, gf_error_to_errno(rsp.op_errno)),
+                gf_error_to_errno(rsp.op_errno), PC_MSG_REMOTE_OP_FAILED, NULL);
+    }
+    CLIENT_STACK_UNWIND(flush, frame, rsp.op_ret,
+                        gf_error_to_errno(rsp.op_errno), xdata);
+
+    if (xdata)
+        dict_unref(xdata);
+
+    return 0;
+}
+
+int
+client4_0_fsync_cbk(struct rpc_req *req, struct iovec *iov, int count,
+                    void *myframe)
+{
+    gfx_common_2iatt_rsp rsp = {
+        0,
+    };
+    call_frame_t *frame = NULL;
+    struct iatt prestat = {
+        0,
+    };
+    struct iatt poststat = {
+        0,
+    };
+    int ret = 0;
+    xlator_t *this = NULL;
+    dict_t *xdata = NULL;
+
+    this = THIS;
+
+    frame = myframe;
+
+    if (-1 == req->rpc_status) {
+        rsp.op_ret = -1;
+        rsp.op_errno = ENOTCONN;
+        goto out;
+    }
+
+    ret = xdr_to_generic(*iov, &rsp, (xdrproc_t)xdr_gfx_common_2iatt_rsp);
+    if (ret < 0) {
+        gf_smsg(this->name, GF_LOG_ERROR, EINVAL, PC_MSG_XDR_DECODING_FAILED,
+                NULL);
+        rsp.op_ret = -1;
+        rsp.op_errno = EINVAL;
+        goto out;
+    }
+
+    ret = client_post_common_2iatt(this, &rsp, &prestat, &poststat, &xdata);
+    if (ret < 0)
+        goto out;
+
+out:
+    if (rsp.op_ret == -1) {
+        gf_smsg(this->name, GF_LOG_WARNING, gf_error_to_errno(rsp.op_errno),
+                PC_MSG_REMOTE_OP_FAILED, NULL);
+    }
+    CLIENT_STACK_UNWIND(fsync, frame, rsp.op_ret,
+                        gf_error_to_errno(rsp.op_errno), &prestat, &poststat,
+                        xdata);
+    if (xdata)
+        dict_unref(xdata);
+
+    return 0;
+}
+
+int
+client4_0_setxattr_cbk(struct rpc_req *req, struct iovec *iov, int count,
+                       void *myframe)
+{
+    call_frame_t *frame = NULL;
+    gfx_common_rsp rsp = {
+        0,
+    };
+    int ret = 0;
+    xlator_t *this = NULL;
+    dict_t *xdata = NULL;
+    int op_errno = EINVAL;
+
+    this = THIS;
+
+    frame = myframe;
+
+    if (-1 == req->rpc_status) {
+        rsp.op_ret = -1;
+        rsp.op_errno = ENOTCONN;
+        goto out;
+    }
+
+    ret = xdr_to_generic(*iov, &rsp, (xdrproc_t)xdr_gfx_common_rsp);
+    if (ret < 0) {
+        gf_smsg(this->name, GF_LOG_ERROR, EINVAL, PC_MSG_XDR_DECODING_FAILED,
+                NULL);
+        rsp.op_ret = -1;
+        rsp.op_errno = EINVAL;
+        goto out;
+    }
+
+    xdr_to_dict(&rsp.xdata, &xdata);
+out:
+    op_errno = gf_error_to_errno(rsp.op_errno);
+    if (rsp.op_ret == -1) {
+        if (op_errno == ENOTSUP) {
+            gf_msg_debug(this->name, 0,
+                         "remote operation failed:"
+                         " %s",
+                         strerror(op_errno));
+        } else {
+            gf_smsg(this->name, GF_LOG_WARNING, op_errno,
+                    PC_MSG_REMOTE_OP_FAILED, NULL);
+        }
+    }
+
+    CLIENT_STACK_UNWIND(setxattr, frame, rsp.op_ret, op_errno, xdata);
+
+    if (xdata)
+        dict_unref(xdata);
+
+    return 0;
+}
+
+int
+client4_0_getxattr_cbk(struct rpc_req *req, struct iovec *iov, int count,
+                       void *myframe)
+{
+    gfx_common_dict_rsp rsp = {
+        0,
+    };
+    call_frame_t *frame = NULL;
+    dict_t *dict = NULL;
+    int op_errno = EINVAL;
+    int ret = 0;
+    clnt_local_t *local = NULL;
+    xlator_t *this = NULL;
+    dict_t *xdata = NULL;
+
+    this = THIS;
+
+    frame = myframe;
+    local = frame->local;
+
+    if (-1 == req->rpc_status) {
+        rsp.op_ret = -1;
+        op_errno = ENOTCONN;
+        goto out;
+    }
+
+    ret = xdr_to_generic(*iov, &rsp, (xdrproc_t)xdr_gfx_common_dict_rsp);
+    if (ret < 0) {
+        gf_smsg(this->name, GF_LOG_ERROR, EINVAL, PC_MSG_XDR_DECODING_FAILED,
+                NULL);
+        rsp.op_ret = -1;
+        op_errno = EINVAL;
+        goto out;
+    }
+
+    op_errno = gf_error_to_errno(rsp.op_errno);
+    ret = client_post_common_dict(this, &rsp, &dict, &xdata);
+    if (ret) {
+        op_errno = -ret;
+        goto out;
+    }
+
+out:
+    if (rsp.op_ret == -1) {
+        if ((op_errno == ENOTSUP) || (op_errno == ENODATA) ||
+            (op_errno == ESTALE) || (op_errno == ENOENT)) {
+            gf_msg_debug(this->name, 0,
+                         "remote operation failed: %s. Path: %s "
+                         "(%s). Key: %s",
+                         strerror(op_errno), local->loc.path,
+                         loc_gfid_utoa(&local->loc),
+                         (local->name) ? local->name : "(null)");
+        } else {
+            gf_smsg(this->name, GF_LOG_WARNING, op_errno,
+                    PC_MSG_REMOTE_OP_FAILED, "path=%s", local->loc.path,
+                    "gfid=%s", loc_gfid_utoa(&local->loc), "key=%s",
+                    (local->name) ? local->name : "(null)", NULL);
+        }
+    } else {
+        /* This is required as many places, `if (ret)` is checked
+           for syncop_getxattr() */
+        gf_msg_debug(this->name, 0, "resetting op_ret to 0 from %d",
+                     rsp.op_ret);
+        rsp.op_ret = 0;
+    }
+
+    CLIENT_STACK_UNWIND(getxattr, frame, rsp.op_ret, op_errno, dict, xdata);
+
+    if (xdata)
+        dict_unref(xdata);
+
+    if (dict)
+        dict_unref(dict);
+
+    return 0;
+}
+
+int
+client4_0_fgetxattr_cbk(struct rpc_req *req, struct iovec *iov, int count,
+                        void *myframe)
+{
+    gfx_common_dict_rsp rsp = {
+        0,
+    };
+    call_frame_t *frame = NULL;
+    dict_t *dict = NULL;
+    int ret = 0;
+    int op_errno = EINVAL;
+    xlator_t *this = NULL;
+    dict_t *xdata = NULL;
+
+    this = THIS;
+
+    frame = myframe;
+
+    if (-1 == req->rpc_status) {
+        rsp.op_ret = -1;
+        op_errno = ENOTCONN;
+        goto out;
+    }
+    ret = xdr_to_generic(*iov, &rsp, (xdrproc_t)xdr_gfx_common_dict_rsp);
+    if (ret < 0) {
+        gf_smsg(this->name, GF_LOG_ERROR, EINVAL, PC_MSG_XDR_DECODING_FAILED,
+                NULL);
+        rsp.op_ret = -1;
+        op_errno = EINVAL;
+        goto out;
+    }
+
+    op_errno = gf_error_to_errno(rsp.op_errno);
+    ret = client_post_common_dict(this, &rsp, &dict, &xdata);
+    if (ret) {
+        op_errno = -ret;
+        goto out;
+    }
+out:
+    if (rsp.op_ret == -1) {
+        if ((op_errno == ENOTSUP) || (op_errno == ERANGE) ||
+            (op_errno == ENODATA) || (op_errno == ENOENT)) {
+            gf_msg_debug(this->name, 0, "remote operation failed: %s",
+                         strerror(op_errno));
+        } else {
+            gf_smsg(this->name, GF_LOG_WARNING, op_errno,
+                    PC_MSG_REMOTE_OP_FAILED, NULL);
+        }
+    } else {
+        /* This is required as many places, `if (ret)` is checked
+           for syncop_fgetxattr() */
+        gf_msg_debug(this->name, 0, "resetting op_ret to 0 from %d",
+                     rsp.op_ret);
+        rsp.op_ret = 0;
+    }
+
+    CLIENT_STACK_UNWIND(fgetxattr, frame, rsp.op_ret, op_errno, dict, xdata);
+
+    if (xdata)
+        dict_unref(xdata);
+
+    if (dict)
+        dict_unref(dict);
+
+    return 0;
+}
+
+int
+client4_0_removexattr_cbk(struct rpc_req *req, struct iovec *iov, int count,
+                          void *myframe)
+{
+    call_frame_t *frame = NULL;
+    gfx_common_rsp rsp = {
+        0,
+    };
+    int ret = 0;
+    xlator_t *this = NULL;
+    dict_t *xdata = NULL;
+    gf_loglevel_t loglevel = GF_LOG_NONE;
+
+    this = THIS;
+
+    frame = myframe;
+
+    if (-1 == req->rpc_status) {
+        rsp.op_ret = -1;
+        rsp.op_errno = ENOTCONN;
+        goto out;
+    }
+
+    ret = xdr_to_generic(*iov, &rsp, (xdrproc_t)xdr_gfx_common_rsp);
+    if (ret < 0) {
+        gf_smsg(this->name, GF_LOG_ERROR, EINVAL, PC_MSG_XDR_DECODING_FAILED,
+                NULL);
+        rsp.op_ret = -1;
+        rsp.op_errno = EINVAL;
+        goto out;
+    }
+
+    xdr_to_dict(&rsp.xdata, &xdata);
+out:
+    if (rsp.op_ret == -1) {
+        /* EPERM/EACCESS is returned some times in case of selinux
+           attributes, or other system attributes which may not be
+           possible to remove from an user process is encountered.
+           we can't treat it as an error */
+        if ((ENODATA == rsp.op_errno) || (ENOATTR == rsp.op_errno) ||
+            (EPERM == rsp.op_errno) || (EACCES == rsp.op_errno))
+            loglevel = GF_LOG_DEBUG;
+        else
+            loglevel = GF_LOG_WARNING;
+
+        gf_smsg(this->name, loglevel, gf_error_to_errno(rsp.op_errno),
+                PC_MSG_REMOTE_OP_FAILED, NULL);
+    }
+
+    CLIENT_STACK_UNWIND(removexattr, frame, rsp.op_ret,
+                        gf_error_to_errno(rsp.op_errno), xdata);
+
+    if (xdata)
+        dict_unref(xdata);
+
+    return 0;
+}
+
+int
+client4_0_fremovexattr_cbk(struct rpc_req *req, struct iovec *iov, int count,
+                           void *myframe)
+{
+    call_frame_t *frame = NULL;
+    gfx_common_rsp rsp = {
+        0,
+    };
+    int ret = 0;
+    xlator_t *this = NULL;
+    dict_t *xdata = NULL;
+
+    this = THIS;
+
+    frame = myframe;
+
+    if (-1 == req->rpc_status) {
+        rsp.op_ret = -1;
+        rsp.op_errno = ENOTCONN;
+        goto out;
+    }
+
+    ret = xdr_to_generic(*iov, &rsp, (xdrproc_t)xdr_gfx_common_rsp);
+    if (ret < 0) {
+        gf_smsg(this->name, GF_LOG_ERROR, EINVAL, PC_MSG_XDR_DECODING_FAILED,
+                NULL);
+        rsp.op_ret = -1;
+        rsp.op_errno = EINVAL;
+        goto out;
+    }
+
+    xdr_to_dict(&rsp.xdata, &xdata);
+out:
+    if (rsp.op_ret == -1) {
+        gf_smsg(this->name, GF_LOG_WARNING, gf_error_to_errno(rsp.op_errno),
+                PC_MSG_REMOTE_OP_FAILED, NULL);
+    }
+    CLIENT_STACK_UNWIND(fremovexattr, frame, rsp.op_ret,
+                        gf_error_to_errno(rsp.op_errno), xdata);
+
+    if (xdata)
+        dict_unref(xdata);
+
+    return 0;
+}
+
+int
+client4_0_fsyncdir_cbk(struct rpc_req *req, struct iovec *iov, int count,
+                       void *myframe)
+{
+    call_frame_t *frame = NULL;
+    gfx_common_rsp rsp = {
+        0,
+    };
+    int ret = 0;
+    xlator_t *this = NULL;
+    dict_t *xdata = NULL;
+
+    this = THIS;
+
+    frame = myframe;
+
+    if (-1 == req->rpc_status) {
+        rsp.op_ret = -1;
+        rsp.op_errno = ENOTCONN;
+        goto out;
+    }
+    ret = xdr_to_generic(*iov, &rsp, (xdrproc_t)xdr_gfx_common_rsp);
+    if (ret < 0) {
+        gf_smsg(this->name, GF_LOG_ERROR, EINVAL, PC_MSG_XDR_DECODING_FAILED,
+                NULL);
+        rsp.op_ret = -1;
+        rsp.op_errno = EINVAL;
+        goto out;
+    }
+
+    xdr_to_dict(&rsp.xdata, &xdata);
+
+out:
+    if (rsp.op_ret == -1) {
+        gf_smsg(this->name, GF_LOG_WARNING, gf_error_to_errno(rsp.op_errno),
+                PC_MSG_REMOTE_OP_FAILED, NULL);
+    }
+    CLIENT_STACK_UNWIND(fsyncdir, frame, rsp.op_ret,
+                        gf_error_to_errno(rsp.op_errno), xdata);
+
+    if (xdata)
+        dict_unref(xdata);
+
+    return 0;
+}
+
+int
+client4_0_access_cbk(struct rpc_req *req, struct iovec *iov, int count,
+                     void *myframe)
+{
+    call_frame_t *frame = NULL;
+    gfx_common_rsp rsp = {
+        0,
+    };
+    int ret = 0;
+    xlator_t *this = NULL;
+    dict_t *xdata = NULL;
+
+    this = THIS;
+
+    frame = myframe;
+
+    if (-1 == req->rpc_status) {
+        rsp.op_ret = -1;
+        rsp.op_errno = ENOTCONN;
+        goto out;
+    }
+    ret = xdr_to_generic(*iov, &rsp, (xdrproc_t)xdr_gfx_common_rsp);
+    if (ret < 0) {
+        gf_smsg(this->name, GF_LOG_ERROR, EINVAL, PC_MSG_XDR_DECODING_FAILED,
+                NULL);
+        rsp.op_ret = -1;
+        rsp.op_errno = EINVAL;
+        goto out;
+    }
+
+    xdr_to_dict(&rsp.xdata, &xdata);
+
+out:
+    if (rsp.op_ret == -1) {
+        gf_smsg(this->name, GF_LOG_WARNING, gf_error_to_errno(rsp.op_errno),
+                PC_MSG_REMOTE_OP_FAILED, NULL);
+    }
+    CLIENT_STACK_UNWIND(access, frame, rsp.op_ret,
+                        gf_error_to_errno(rsp.op_errno), xdata);
+
+    if (xdata)
+        dict_unref(xdata);
+
+    return 0;
+}
+
+int
+client4_0_ftruncate_cbk(struct rpc_req *req, struct iovec *iov, int count,
+                        void *myframe)
+{
+    gfx_common_2iatt_rsp rsp = {
+        0,
+    };
+    call_frame_t *frame = NULL;
+    struct iatt prestat = {
+        0,
+    };
+    struct iatt poststat = {
+        0,
+    };
+    int ret = 0;
+    xlator_t *this = NULL;
+    dict_t *xdata = NULL;
+
+    this = THIS;
+
+    frame = myframe;
+
+    if (-1 == req->rpc_status) {
+        rsp.op_ret = -1;
+        rsp.op_errno = ENOTCONN;
+        goto out;
+    }
+    ret = xdr_to_generic(*iov, &rsp, (xdrproc_t)xdr_gfx_common_2iatt_rsp);
+    if (ret < 0) {
+        gf_smsg(this->name, GF_LOG_ERROR, EINVAL, PC_MSG_XDR_DECODING_FAILED,
+                NULL);
+        rsp.op_ret = -1;
+        rsp.op_errno = EINVAL;
+        goto out;
+    }
+
+    ret = client_post_common_2iatt(this, &rsp, &prestat, &poststat, &xdata);
+
+out:
+    if (rsp.op_ret == -1) {
+        gf_smsg(this->name, GF_LOG_WARNING, gf_error_to_errno(rsp.op_errno),
+                PC_MSG_REMOTE_OP_FAILED, NULL);
+    }
+    CLIENT_STACK_UNWIND(ftruncate, frame, rsp.op_ret,
+                        gf_error_to_errno(rsp.op_errno), &prestat, &poststat,
+                        xdata);
+
+    if (xdata)
+        dict_unref(xdata);
+
+    return 0;
+}
+
+int
+client4_0_fstat_cbk(struct rpc_req *req, struct iovec *iov, int count,
+                    void *myframe)
+{
+    gfx_common_iatt_rsp rsp = {
+        0,
+    };
+    call_frame_t *frame = NULL;
+    struct iatt stat = {
+        0,
+    };
+    int ret = 0;
+    xlator_t *this = NULL;
+    dict_t *xdata = NULL;
+
+    this = THIS;
+
+    frame = myframe;
+
+    if (-1 == req->rpc_status) {
+        rsp.op_ret = -1;
+        rsp.op_errno = ENOTCONN;
+        goto out;
+    }
+    ret = xdr_to_generic(*iov, &rsp, (xdrproc_t)xdr_gfx_common_iatt_rsp);
+    if (ret < 0) {
+        gf_smsg(this->name, GF_LOG_ERROR, EINVAL, PC_MSG_XDR_DECODING_FAILED,
+                NULL);
+        rsp.op_ret = -1;
+        rsp.op_errno = EINVAL;
+        goto out;
+    }
+
+    ret = client_post_common_iatt(this, &rsp, &stat, &xdata);
+
+out:
+    if (rsp.op_ret == -1) {
+        gf_smsg(this->name, GF_LOG_WARNING, gf_error_to_errno(rsp.op_errno),
+                PC_MSG_REMOTE_OP_FAILED, NULL);
+    }
+    CLIENT_STACK_UNWIND(fstat, frame, rsp.op_ret,
+                        gf_error_to_errno(rsp.op_errno), &stat, xdata);
+
+    if (xdata)
+        dict_unref(xdata);
+
+    return 0;
+}
+
+int
+client4_0_inodelk_cbk(struct rpc_req *req, struct iovec *iov, int count,
+                      void *myframe)
+{
+    call_frame_t *frame = NULL;
+    gfx_common_rsp rsp = {
+        0,
+    };
+    int ret = 0;
+    xlator_t *this = NULL;
+    dict_t *xdata = NULL;
+
+    this = THIS;
+
+    frame = myframe;
+
+    if (-1 == req->rpc_status) {
+        rsp.op_ret = -1;
+        rsp.op_errno = ENOTCONN;
+        goto out;
+    }
+    ret = xdr_to_generic(*iov, &rsp, (xdrproc_t)xdr_gfx_common_rsp);
+    if (ret < 0) {
+        gf_smsg(this->name, GF_LOG_ERROR, EINVAL, PC_MSG_XDR_DECODING_FAILED,
+                NULL);
+        rsp.op_ret = -1;
+        rsp.op_errno = EINVAL;
+        goto out;
+    }
+
+    xdr_to_dict(&rsp.xdata, &xdata);
+out:
+    if (rsp.op_ret == -1) {
+        gf_smsg(this->name,
+                fop_log_level(GF_FOP_INODELK, gf_error_to_errno(rsp.op_errno)),
+                gf_error_to_errno(rsp.op_errno), PC_MSG_REMOTE_OP_FAILED, NULL);
+    }
+    CLIENT_STACK_UNWIND(inodelk, frame, rsp.op_ret,
+                        gf_error_to_errno(rsp.op_errno), xdata);
+
+    if (xdata)
+        dict_unref(xdata);
+
+    return 0;
+}
+
+int
+client4_0_finodelk_cbk(struct rpc_req *req, struct iovec *iov, int count,
+                       void *myframe)
+{
+    call_frame_t *frame = NULL;
+    gfx_common_rsp rsp = {
+        0,
+    };
+    int ret = 0;
+    xlator_t *this = NULL;
+    dict_t *xdata = NULL;
+    clnt_local_t *local = NULL;
+
+    frame = myframe;
+    this = frame->this;
+    local = frame->local;
+
+    if (-1 == req->rpc_status) {
+        rsp.op_ret = -1;
+        rsp.op_errno = ENOTCONN;
+        goto out;
+    }
+    ret = xdr_to_generic(*iov, &rsp, (xdrproc_t)xdr_gfx_common_rsp);
+    if (ret < 0) {
+        gf_smsg(this->name, GF_LOG_ERROR, EINVAL, PC_MSG_XDR_DECODING_FAILED,
+                NULL);
+        rsp.op_ret = -1;
+        rsp.op_errno = EINVAL;
+        goto out;
+    }
+
+    xdr_to_dict(&rsp.xdata, &xdata);
+out:
+    if (rsp.op_ret == -1) {
+        gf_smsg(this->name,
+                fop_log_level(GF_FOP_FINODELK, gf_error_to_errno(rsp.op_errno)),
+                gf_error_to_errno(rsp.op_errno), PC_MSG_REMOTE_OP_FAILED, NULL);
+    } else if (rsp.op_ret == 0) {
+        if (local->attempt_reopen)
+            client_attempt_reopen(local->fd, this);
+    }
+    CLIENT_STACK_UNWIND(finodelk, frame, rsp.op_ret,
+                        gf_error_to_errno(rsp.op_errno), xdata);
+
+    if (xdata)
+        dict_unref(xdata);
+
+    return 0;
+}
+
+int
+client4_0_entrylk_cbk(struct rpc_req *req, struct iovec *iov, int count,
+                      void *myframe)
+{
+    call_frame_t *frame = NULL;
+    gfx_common_rsp rsp = {
+        0,
+    };
+    int ret = 0;
+    xlator_t *this = NULL;
+    dict_t *xdata = NULL;
+
+    this = THIS;
+
+    frame = myframe;
+
+    if (-1 == req->rpc_status) {
+        rsp.op_ret = -1;
+        rsp.op_errno = ENOTCONN;
+        goto out;
+    }
+    ret = xdr_to_generic(*iov, &rsp, (xdrproc_t)xdr_gfx_common_rsp);
+    if (ret < 0) {
+        gf_smsg(this->name, GF_LOG_ERROR, EINVAL, PC_MSG_XDR_DECODING_FAILED,
+                NULL);
+        rsp.op_ret = -1;
+        rsp.op_errno = EINVAL;
+        goto out;
+    }
+
+    xdr_to_dict(&rsp.xdata, &xdata);
+out:
+    if (rsp.op_ret == -1) {
+        gf_smsg(this->name,
+                fop_log_level(GF_FOP_ENTRYLK, gf_error_to_errno(rsp.op_errno)),
+                gf_error_to_errno(rsp.op_errno), PC_MSG_REMOTE_OP_FAILED, NULL);
+    }
+
+    CLIENT_STACK_UNWIND(entrylk, frame, rsp.op_ret,
+                        gf_error_to_errno(rsp.op_errno), xdata);
+
+    if (xdata)
+        dict_unref(xdata);
+
+    return 0;
+}
+
+int
+client4_0_fentrylk_cbk(struct rpc_req *req, struct iovec *iov, int count,
+                       void *myframe)
+{
+    call_frame_t *frame = NULL;
+    gfx_common_rsp rsp = {
+        0,
+    };
+    int ret = 0;
+    xlator_t *this = NULL;
+    dict_t *xdata = NULL;
+
+    this = THIS;
+
+    frame = myframe;
+
+    if (-1 == req->rpc_status) {
+        rsp.op_ret = -1;
+        rsp.op_errno = ENOTCONN;
+        goto out;
+    }
+    ret = xdr_to_generic(*iov, &rsp, (xdrproc_t)xdr_gfx_common_rsp);
+    if (ret < 0) {
+        gf_smsg(this->name, GF_LOG_ERROR, EINVAL, PC_MSG_XDR_DECODING_FAILED,
+                NULL);
+        rsp.op_ret = -1;
+        rsp.op_errno = EINVAL;
+        goto out;
+    }
+
+    xdr_to_dict(&rsp.xdata, &xdata);
+
+out:
+    if ((rsp.op_ret == -1) && (EAGAIN != gf_error_to_errno(rsp.op_errno))) {
+        gf_smsg(this->name, GF_LOG_WARNING, gf_error_to_errno(rsp.op_errno),
+                PC_MSG_REMOTE_OP_FAILED, NULL);
+    }
+
+    CLIENT_STACK_UNWIND(fentrylk, frame, rsp.op_ret,
+                        gf_error_to_errno(rsp.op_errno), xdata);
+
+    if (xdata)
+        dict_unref(xdata);
+
+    return 0;
+}
+
+int
+client4_0_xattrop_cbk(struct rpc_req *req, struct iovec *iov, int count,
+                      void *myframe)
+{
+    call_frame_t *frame = NULL;
+    dict_t *dict = NULL;
+    gfx_common_dict_rsp rsp = {
+        0,
+    };
+    int ret = 0;
+    int op_errno = EINVAL;
+    clnt_local_t *local = NULL;
+    xlator_t *this = NULL;
+    dict_t *xdata = NULL;
+
+    this = THIS;
+
+    frame = myframe;
+    local = frame->local;
+
+    if (-1 == req->rpc_status) {
+        rsp.op_ret = -1;
+        op_errno = ENOTCONN;
+        goto out;
+    }
+    ret = xdr_to_generic(*iov, &rsp, (xdrproc_t)xdr_gfx_common_dict_rsp);
+    if (ret < 0) {
+        gf_smsg(this->name, GF_LOG_ERROR, EINVAL, PC_MSG_XDR_DECODING_FAILED,
+                NULL);
+        rsp.op_ret = -1;
+        op_errno = EINVAL;
+        goto out;
+    }
+
+    op_errno = rsp.op_errno;
+    ret = client_post_common_dict(this, &rsp, &dict, &xdata);
+    if (ret) {
+        op_errno = -ret;
+        goto out;
+    }
+out:
+    if (rsp.op_ret == -1) {
+        gf_smsg(this->name, fop_log_level(GF_FOP_XATTROP, op_errno),
+                gf_error_to_errno(rsp.op_errno), PC_MSG_REMOTE_OP_FAILED,
+                "Path=%s", local->loc.path, "gfid=%s",
+                loc_gfid_utoa(&local->loc), NULL);
+    } else {
+        /* This is required as many places, `if (ret)` is checked
+           for syncop_xattrop() */
+        gf_msg_debug(this->name, 0, "resetting op_ret to 0 from %d",
+                     rsp.op_ret);
+        rsp.op_ret = 0;
+    }
+
+    CLIENT_STACK_UNWIND(xattrop, frame, rsp.op_ret, gf_error_to_errno(op_errno),
+                        dict, xdata);
+
+    if (xdata)
+        dict_unref(xdata);
+
+    if (dict)
+        dict_unref(dict);
+
+    return 0;
+}
+
+int
+client4_0_fxattrop_cbk(struct rpc_req *req, struct iovec *iov, int count,
+                       void *myframe)
+{
+    call_frame_t *frame = NULL;
+    dict_t *dict = NULL;
+    dict_t *xdata = NULL;
+    gfx_common_dict_rsp rsp = {
+        0,
+    };
+    int ret = 0;
+    int op_errno = 0;
+    clnt_local_t *local = NULL;
+    xlator_t *this = NULL;
+
+    this = THIS;
+
+    frame = myframe;
+    local = frame->local;
+
+    if (-1 == req->rpc_status) {
+        rsp.op_ret = -1;
+        op_errno = ENOTCONN;
+        goto out;
+    }
+
+    ret = xdr_to_generic(*iov, &rsp, (xdrproc_t)xdr_gfx_common_dict_rsp);
+    if (ret < 0) {
+        rsp.op_ret = -1;
+        op_errno = EINVAL;
+        gf_smsg(this->name, GF_LOG_ERROR, EINVAL, PC_MSG_XDR_DECODING_FAILED,
+                NULL);
+        goto out;
+    }
+    op_errno = rsp.op_errno;
+    ret = client_post_common_dict(this, &rsp, &dict, &xdata);
+    if (ret) {
+        rsp.op_ret = -1;
+        op_errno = -ret;
+        goto out;
+    }
+out:
+
+    if (rsp.op_ret == -1) {
+        gf_smsg(this->name, GF_LOG_WARNING, gf_error_to_errno(rsp.op_errno),
+                PC_MSG_REMOTE_OP_FAILED, NULL);
+    } else {
+        /* This is required as many places, `if (ret)` is checked
+           for syncop_fxattrop() */
+        gf_msg_debug(this->name, 0, "resetting op_ret to 0 from %d",
+                     rsp.op_ret);
+        rsp.op_ret = 0;
+
+        if (local->attempt_reopen)
+            client_attempt_reopen(local->fd, this);
+    }
+
+    CLIENT_STACK_UNWIND(fxattrop, frame, rsp.op_ret,
+                        gf_error_to_errno(op_errno), dict, xdata);
+    if (xdata)
+        dict_unref(xdata);
+
+    if (dict)
+        dict_unref(dict);
+
+    return 0;
+}
+
+int
+client4_0_fsetxattr_cbk(struct rpc_req *req, struct iovec *iov, int count,
+                        void *myframe)
+{
+    call_frame_t *frame = NULL;
+    gfx_common_rsp rsp = {
+        0,
+    };
+    int ret = 0;
+    xlator_t *this = NULL;
+    dict_t *xdata = NULL;
+    int op_errno = EINVAL;
+
+    this = THIS;
+
+    frame = myframe;
+
+    if (-1 == req->rpc_status) {
+        rsp.op_ret = -1;
+        rsp.op_errno = ENOTCONN;
+        goto out;
+    }
+    ret = xdr_to_generic(*iov, &rsp, (xdrproc_t)xdr_gfx_common_rsp);
+    if (ret < 0) {
+        gf_smsg(this->name, GF_LOG_ERROR, EINVAL, PC_MSG_XDR_DECODING_FAILED,
+                NULL);
+        rsp.op_ret = -1;
+        rsp.op_errno = EINVAL;
+        goto out;
+    }
+
+    xdr_to_dict(&rsp.xdata, &xdata);
+
+out:
+    op_errno = gf_error_to_errno(rsp.op_errno);
+    if (rsp.op_ret == -1) {
+        if (op_errno == ENOTSUP) {
+            gf_msg_debug(this->name, 0,
+                         "remote operation failed:"
+                         " %s",
+                         strerror(op_errno));
+        } else {
+            gf_smsg(this->name, GF_LOG_WARNING, rsp.op_errno,
+                    PC_MSG_REMOTE_OP_FAILED, NULL);
+        }
+    }
+
+    CLIENT_STACK_UNWIND(fsetxattr, frame, rsp.op_ret, op_errno, xdata);
+
+    if (xdata)
+        dict_unref(xdata);
+
+    return 0;
+}
+
+int
+client4_0_fallocate_cbk(struct rpc_req *req, struct iovec *iov, int count,
+                        void *myframe)
+{
+    call_frame_t *frame = NULL;
+    gfx_common_2iatt_rsp rsp = {
+        0,
+    };
+    struct iatt prestat = {
+        0,
+    };
+    struct iatt poststat = {
+        0,
+    };
+    int ret = 0;
+    xlator_t *this = NULL;
+    dict_t *xdata = NULL;
+
+    this = THIS;
+
+    frame = myframe;
+
+    if (-1 == req->rpc_status) {
+        rsp.op_ret = -1;
+        rsp.op_errno = ENOTCONN;
+        goto out;
+    }
+    ret = xdr_to_generic(*iov, &rsp, (xdrproc_t)xdr_gfx_common_2iatt_rsp);
+    if (ret < 0) {
+        gf_smsg(this->name, GF_LOG_ERROR, EINVAL, PC_MSG_XDR_DECODING_FAILED,
+                NULL);
+        rsp.op_ret = -1;
+        rsp.op_errno = EINVAL;
+        goto out;
+    }
+
+    ret = client_post_common_2iatt(this, &rsp, &prestat, &poststat, &xdata);
+    if (ret < 0)
+        goto out;
+
+out:
+    if (rsp.op_ret == -1) {
+        gf_smsg(this->name, GF_LOG_WARNING, gf_error_to_errno(rsp.op_errno),
+                PC_MSG_REMOTE_OP_FAILED, NULL);
+    }
+    CLIENT_STACK_UNWIND(fallocate, frame, rsp.op_ret,
+                        gf_error_to_errno(rsp.op_errno), &prestat, &poststat,
+                        xdata);
+
+    if (xdata)
+        dict_unref(xdata);
+
+    return 0;
+}
+
+int
+client4_0_discard_cbk(struct rpc_req *req, struct iovec *iov, int count,
+                      void *myframe)
+{
+    gfx_common_2iatt_rsp rsp = {
+        0,
+    };
+    call_frame_t *frame = NULL;
+    struct iatt prestat = {
+        0,
+    };
+    struct iatt poststat = {
+        0,
+    };
+    int ret = 0;
+    xlator_t *this = NULL;
+    dict_t *xdata = NULL;
+
+    this = THIS;
+
+    frame = myframe;
+
+    if (-1 == req->rpc_status) {
+        rsp.op_ret = -1;
+        rsp.op_errno = ENOTCONN;
+        goto out;
+    }
+    ret = xdr_to_generic(*iov, &rsp, (xdrproc_t)xdr_gfx_common_2iatt_rsp);
+    if (ret < 0) {
+        gf_smsg(this->name, GF_LOG_ERROR, EINVAL, PC_MSG_XDR_DECODING_FAILED,
+                NULL);
+        rsp.op_ret = -1;
+        rsp.op_errno = EINVAL;
+        goto out;
+    }
+
+    ret = client_post_common_2iatt(this, &rsp, &prestat, &poststat, &xdata);
+
+out:
+    if (rsp.op_ret == -1) {
+        gf_smsg(this->name, GF_LOG_WARNING, gf_error_to_errno(rsp.op_errno),
+                PC_MSG_REMOTE_OP_FAILED, NULL);
+    }
+    CLIENT_STACK_UNWIND(discard, frame, rsp.op_ret,
+                        gf_error_to_errno(rsp.op_errno), &prestat, &poststat,
+                        xdata);
+    if (xdata)
+        dict_unref(xdata);
+
+    return 0;
+}
+
+int
+client4_0_zerofill_cbk(struct rpc_req *req, struct iovec *iov, int count,
+                       void *myframe)
+{
+    call_frame_t *frame = NULL;
+    gfx_common_2iatt_rsp rsp = {
+        0,
+    };
+    struct iatt prestat = {
+        0,
+    };
+    struct iatt poststat = {
+        0,
+    };
+    int ret = 0;
+    xlator_t *this = NULL;
+    dict_t *xdata = NULL;
+
+    this = THIS;
+
+    frame = myframe;
+
+    if (-1 == req->rpc_status) {
+        rsp.op_ret = -1;
+        rsp.op_errno = ENOTCONN;
+        goto out;
+    }
+    ret = xdr_to_generic(*iov, &rsp, (xdrproc_t)xdr_gfx_common_2iatt_rsp);
+    if (ret < 0) {
+        gf_smsg(this->name, GF_LOG_ERROR, EINVAL, PC_MSG_XDR_DECODING_FAILED,
+                NULL);
+        rsp.op_ret = -1;
+        rsp.op_errno = EINVAL;
+        goto out;
+    }
+
+    ret = client_post_common_2iatt(this, &rsp, &prestat, &poststat, &xdata);
+out:
+    if (rsp.op_ret == -1) {
+        gf_smsg(this->name, GF_LOG_WARNING, gf_error_to_errno(rsp.op_errno),
+                PC_MSG_REMOTE_OP_FAILED, NULL);
+    }
+    CLIENT_STACK_UNWIND(zerofill, frame, rsp.op_ret,
+                        gf_error_to_errno(rsp.op_errno), &prestat, &poststat,
+                        xdata);
+
+    if (xdata)
+        dict_unref(xdata);
+
+    return 0;
+}
+
+int
+client4_0_ipc_cbk(struct rpc_req *req, struct iovec *iov, int count,
+                  void *myframe)
+{
+    call_frame_t *frame = NULL;
+    gfx_common_rsp rsp = {
+        0,
+    };
+    int ret = 0;
+    xlator_t *this = NULL;
+    dict_t *xdata = NULL;
+
+    this = THIS;
+
+    frame = myframe;
+
+    if (-1 == req->rpc_status) {
+        rsp.op_ret = -1;
+        rsp.op_errno = ENOTCONN;
+        goto out;
+    }
+    ret = xdr_to_generic(*iov, &rsp, (xdrproc_t)xdr_gfx_common_rsp);
+    if (ret < 0) {
+        gf_smsg(this->name, GF_LOG_ERROR, EINVAL, PC_MSG_XDR_DECODING_FAILED,
+                NULL);
+        rsp.op_ret = -1;
+        rsp.op_errno = EINVAL;
+        goto out;
+    }
+
+    xdr_to_dict(&rsp.xdata, &xdata);
+out:
+    if (rsp.op_ret == -1) {
+        gf_smsg(this->name, GF_LOG_WARNING, gf_error_to_errno(rsp.op_errno),
+                PC_MSG_REMOTE_OP_FAILED, NULL);
+    }
+    CLIENT_STACK_UNWIND(ipc, frame, rsp.op_ret, gf_error_to_errno(rsp.op_errno),
+                        xdata);
+
+    if (xdata)
+        dict_unref(xdata);
+
+    return 0;
+}
+
+int
+client4_0_seek_cbk(struct rpc_req *req, struct iovec *iov, int count,
+                   void *myframe)
+{
+    call_frame_t *frame = NULL;
+    struct gfx_seek_rsp rsp = {
+        0,
+    };
+    int ret = 0;
+    xlator_t *this = NULL;
+    dict_t *xdata = NULL;
+
+    this = THIS;
+
+    frame = myframe;
+
+    if (-1 == req->rpc_status) {
+        rsp.op_ret = -1;
+        rsp.op_errno = ENOTCONN;
+        goto out;
+    }
+    ret = xdr_to_generic(*iov, &rsp, (xdrproc_t)xdr_gfx_seek_rsp);
+    if (ret < 0) {
+        gf_smsg(this->name, GF_LOG_ERROR, EINVAL, PC_MSG_XDR_DECODING_FAILED,
+                NULL);
+        rsp.op_ret = -1;
+        rsp.op_errno = EINVAL;
+        goto out;
+    }
+
+    xdr_to_dict(&rsp.xdata, &xdata);
+out:
+    if (rsp.op_ret == -1) {
+        gf_smsg(this->name, GF_LOG_WARNING, gf_error_to_errno(rsp.op_errno),
+                PC_MSG_REMOTE_OP_FAILED, NULL);
+    }
+    CLIENT_STACK_UNWIND(seek, frame, rsp.op_ret,
+                        gf_error_to_errno(rsp.op_errno), rsp.offset, xdata);
+
+    if (xdata)
+        dict_unref(xdata);
+
+    return 0;
+}
+
+int
+client4_0_setattr_cbk(struct rpc_req *req, struct iovec *iov, int count,
+                      void *myframe)
+{
+    gfx_common_2iatt_rsp rsp = {
+        0,
+    };
+    call_frame_t *frame = NULL;
+    struct iatt prestat = {
+        0,
+    };
+    struct iatt poststat = {
+        0,
+    };
+    int ret = 0;
+    xlator_t *this = NULL;
+    dict_t *xdata = NULL;
+
+    this = THIS;
+
+    frame = myframe;
+
+    if (-1 == req->rpc_status) {
+        rsp.op_ret = -1;
+        rsp.op_errno = ENOTCONN;
+        goto out;
+    }
+
+    ret = xdr_to_generic(*iov, &rsp, (xdrproc_t)xdr_gfx_common_2iatt_rsp);
+    if (ret < 0) {
+        gf_smsg(this->name, GF_LOG_ERROR, EINVAL, PC_MSG_XDR_DECODING_FAILED,
+                NULL);
+        rsp.op_ret = -1;
+        rsp.op_errno = EINVAL;
+        goto out;
+    }
+
+    ret = client_post_common_2iatt(this, &rsp, &prestat, &poststat, &xdata);
+
+out:
+    if (rsp.op_ret == -1) {
+        gf_smsg(this->name, GF_LOG_WARNING, gf_error_to_errno(rsp.op_errno),
+                PC_MSG_REMOTE_OP_FAILED, NULL);
+    }
+    CLIENT_STACK_UNWIND(setattr, frame, rsp.op_ret,
+                        gf_error_to_errno(rsp.op_errno), &prestat, &poststat,
+                        xdata);
+
+    if (xdata)
+        dict_unref(xdata);
+
+    return 0;
+}
+
+int
+client4_0_fsetattr_cbk(struct rpc_req *req, struct iovec *iov, int count,
+                       void *myframe)
+{
+    gfx_common_2iatt_rsp rsp = {
+        0,
+    };
+    call_frame_t *frame = NULL;
+    struct iatt prestat = {
+        0,
+    };
+    struct iatt poststat = {
+        0,
+    };
+    int ret = 0;
+    xlator_t *this = NULL;
+    dict_t *xdata = NULL;
+
+    this = THIS;
+
+    frame = myframe;
+
+    if (-1 == req->rpc_status) {
+        rsp.op_ret = -1;
+        rsp.op_errno = ENOTCONN;
+        goto out;
+    }
+
+    ret = xdr_to_generic(*iov, &rsp, (xdrproc_t)xdr_gfx_common_2iatt_rsp);
+    if (ret < 0) {
+        gf_smsg(this->name, GF_LOG_ERROR, EINVAL, PC_MSG_XDR_DECODING_FAILED,
+                NULL);
+        rsp.op_ret = -1;
+        rsp.op_errno = EINVAL;
+        goto out;
+    }
+
+    ret = client_post_common_2iatt(this, &rsp, &prestat, &poststat, &xdata);
+
+out:
+    if (rsp.op_ret == -1) {
+        gf_smsg(this->name, GF_LOG_WARNING, gf_error_to_errno(rsp.op_errno),
+                PC_MSG_REMOTE_OP_FAILED, NULL);
+    }
+    CLIENT_STACK_UNWIND(fsetattr, frame, rsp.op_ret,
+                        gf_error_to_errno(rsp.op_errno), &prestat, &poststat,
+                        xdata);
+
+    if (xdata)
+        dict_unref(xdata);
+
+    return 0;
+}
+
+int
+client4_0_create_cbk(struct rpc_req *req, struct iovec *iov, int count,
+                     void *myframe)
+{
+    call_frame_t *frame = NULL;
+    fd_t *fd = NULL;
+    inode_t *inode = NULL;
+    struct iatt stbuf = {
+        0,
+    };
+    struct iatt preparent = {
+        0,
+    };
+    struct iatt postparent = {
+        0,
+    };
+    int32_t ret = -1;
+    clnt_local_t *local = NULL;
+    gfx_create_rsp rsp = {
+        0,
+    };
+    xlator_t *this = NULL;
+    dict_t *xdata = NULL;
+
+    this = THIS;
+
+    frame = myframe;
+    local = frame->local;
+    fd = local->fd;
+    inode = local->loc.inode;
+
+    if (-1 == req->rpc_status) {
+        rsp.op_ret = -1;
+        rsp.op_errno = ENOTCONN;
+        goto out;
+    }
+
+    ret = xdr_to_generic(*iov, &rsp, (xdrproc_t)xdr_gfx_create_rsp);
+    if (ret < 0) {
+        gf_smsg(this->name, GF_LOG_ERROR, EINVAL, PC_MSG_XDR_DECODING_FAILED,
+                NULL);
+        rsp.op_ret = -1;
+        rsp.op_errno = EINVAL;
+        goto out;
+    }
+
+    ret = client_post_create_v2(this, &rsp, &stbuf, &preparent, &postparent,
+                                local, &xdata);
+    if (ret < 0)
+        goto out;
+
+    if (-1 != rsp.op_ret) {
+        ret = client_add_fd_to_saved_fds(frame->this, fd, &local->loc,
+                                         local->flags, rsp.fd, 0);
+        if (ret) {
+            rsp.op_ret = -1;
+            rsp.op_errno = -ret;
+            goto out;
+        }
+    }
+
+out:
+    if (rsp.op_ret == -1) {
+        gf_smsg(this->name, GF_LOG_WARNING, gf_error_to_errno(rsp.op_errno),
+                PC_MSG_REMOTE_OP_FAILED, "path=%s", local->loc.path, NULL);
+    }
+
+    CLIENT_STACK_UNWIND(create, frame, rsp.op_ret,
+                        gf_error_to_errno(rsp.op_errno), fd, inode, &stbuf,
+                        &preparent, &postparent, xdata);
+
+    if (xdata)
+        dict_unref(xdata);
+
+    return 0;
+}
+
+int
+client4_0_lease_cbk(struct rpc_req *req, struct iovec *iov, int count,
+                    void *myframe)
+{
+    call_frame_t *frame = NULL;
+    struct gf_lease lease = {
+        0,
+    };
+    gfx_lease_rsp rsp = {
+        0,
+    };
+    int ret = 0;
+    xlator_t *this = NULL;
+    dict_t *xdata = NULL;
+
+    this = THIS;
+
+    frame = myframe;
+
+    if (-1 == req->rpc_status) {
+        gf_smsg(this->name, GF_LOG_ERROR, ENOTCONN, PC_MSG_REMOTE_OP_FAILED,
+                NULL);
+        rsp.op_ret = -1;
+        rsp.op_errno = ENOTCONN;
+        goto out;
+    }
+
+    ret = xdr_to_generic(*iov, &rsp, (xdrproc_t)xdr_gfx_lease_rsp);
+    if (ret < 0) {
+        gf_smsg(this->name, GF_LOG_ERROR, EINVAL, PC_MSG_XDR_DECODING_FAILED,
+                NULL);
+        rsp.op_ret = -1;
+        rsp.op_errno = EINVAL;
+        goto out;
+    }
+
+    ret = client_post_lease_v2(this, &rsp, &lease, &xdata);
+
+out:
+    if (rsp.op_ret == -1) {
+        gf_smsg(this->name, GF_LOG_WARNING, gf_error_to_errno(rsp.op_errno),
+                PC_MSG_REMOTE_OP_FAILED, NULL);
+    }
+
+    CLIENT_STACK_UNWIND(lease, frame, rsp.op_ret,
+                        gf_error_to_errno(rsp.op_errno), &lease, xdata);
+
+    if (xdata)
+        dict_unref(xdata);
+
+    return 0;
+}
+
+int
+client4_0_lk_cbk(struct rpc_req *req, struct iovec *iov, int count,
+                 void *myframe)
+{
+    call_frame_t *frame = NULL;
+    struct gf_flock lock = {
+        0,
+    };
+    gfx_lk_rsp rsp = {
+        0,
+    };
+    int ret = 0;
+    xlator_t *this = NULL;
+    dict_t *xdata = NULL;
+    clnt_local_t *local = NULL;
+
+    this = THIS;
+
+    frame = myframe;
+    local = frame->local;
+
+    if (-1 == req->rpc_status) {
+        rsp.op_ret = -1;
+        rsp.op_errno = ENOTCONN;
+        goto out;
+    }
+
+    ret = xdr_to_generic(*iov, &rsp, (xdrproc_t)xdr_gfx_lk_rsp);
+    if (ret < 0) {
+        gf_smsg(this->name, GF_LOG_ERROR, EINVAL, PC_MSG_XDR_DECODING_FAILED,
+                NULL);
+        rsp.op_ret = -1;
+        rsp.op_errno = EINVAL;
+        goto out;
+    }
+
+    if (rsp.op_ret >= 0) {
+        ret = client_post_lk_v2(this, &rsp, &lock, &xdata);
+        if (ret < 0)
+            goto out;
+
+        /* Save the lock to the client lock cache to be able
+           to recover in the case of server reboot.*/
+
+        if (client_is_setlk(local->cmd)) {
+            ret = client_add_lock_for_recovery(local->fd, &lock, &local->owner,
+                                               local->cmd);
+            if (ret < 0) {
+                rsp.op_ret = -1;
+                rsp.op_errno = -ret;
+            }
+        }
+    }
+
+out:
+    if ((rsp.op_ret == -1) && (EAGAIN != gf_error_to_errno(rsp.op_errno))) {
+        gf_smsg(this->name, GF_LOG_WARNING, gf_error_to_errno(rsp.op_errno),
+                PC_MSG_REMOTE_OP_FAILED, NULL);
+    }
+
+    CLIENT_STACK_UNWIND(lk, frame, rsp.op_ret, gf_error_to_errno(rsp.op_errno),
+                        &lock, xdata);
+
+    free(rsp.flock.lk_owner.lk_owner_val);
+
+    if (xdata)
+        dict_unref(xdata);
+
+    return 0;
+}
+
+int
+client4_0_readdir_cbk(struct rpc_req *req, struct iovec *iov, int count,
+                      void *myframe)
+{
+    call_frame_t *frame = NULL;
+    gfx_readdir_rsp rsp = {
+        0,
+    };
+    int32_t ret = 0;
+    clnt_local_t *local = NULL;
+    gf_dirent_t entries;
+    xlator_t *this = NULL;
+    dict_t *xdata = NULL;
+
+    this = THIS;
+
+    frame = myframe;
+    local = frame->local;
+
+    INIT_LIST_HEAD(&entries.list);
+
+    if (-1 == req->rpc_status) {
+        rsp.op_ret = -1;
+        rsp.op_errno = ENOTCONN;
+        goto out;
+    }
+
+    ret = xdr_to_generic(*iov, &rsp, (xdrproc_t)xdr_gfx_readdir_rsp);
+    if (ret < 0) {
+        gf_smsg(this->name, GF_LOG_ERROR, EINVAL, PC_MSG_XDR_DECODING_FAILED,
+                NULL);
+        rsp.op_ret = -1;
+        rsp.op_errno = EINVAL;
+        goto out;
+    }
+
+    ret = client_post_readdir_v2(this, &rsp, &entries, &xdata);
+
+out:
+    if (rsp.op_ret == -1) {
+        gf_smsg(this->name, GF_LOG_WARNING, gf_error_to_errno(rsp.op_errno),
+                PC_MSG_REMOTE_OP_FAILED, "remote_fd=%d", local->cmd, NULL);
+    }
+    CLIENT_STACK_UNWIND(readdir, frame, rsp.op_ret,
+                        gf_error_to_errno(rsp.op_errno), &entries, xdata);
+
+    if (rsp.op_ret != -1) {
+        gf_dirent_free(&entries);
+    }
+
+    if (xdata)
+        dict_unref(xdata);
+
+    clnt_readdir_rsp_cleanup_v2(&rsp);
+
+    return 0;
+}
+
+int
+client4_0_readdirp_cbk(struct rpc_req *req, struct iovec *iov, int count,
+                       void *myframe)
+{
+    call_frame_t *frame = NULL;
+    gfx_readdirp_rsp rsp = {
+        0,
+    };
+    int32_t ret = 0;
+    clnt_local_t *local = NULL;
+    gf_dirent_t entries;
+    xlator_t *this = NULL;
+    dict_t *xdata = NULL;
+
+    this = THIS;
+
+    frame = myframe;
+    local = frame->local;
+
+    INIT_LIST_HEAD(&entries.list);
+
+    if (-1 == req->rpc_status) {
+        rsp.op_ret = -1;
+        rsp.op_errno = ENOTCONN;
+        goto out;
+    }
+
+    ret = xdr_to_generic(*iov, &rsp, (xdrproc_t)xdr_gfx_readdirp_rsp);
+    if (ret < 0) {
+        gf_smsg(this->name, GF_LOG_ERROR, EINVAL, PC_MSG_XDR_DECODING_FAILED,
+                NULL);
+        rsp.op_ret = -1;
+        rsp.op_errno = EINVAL;
+        goto out;
+    }
+
+    ret = client_post_readdirp_v2(this, &rsp, local->fd, &entries, &xdata);
+out:
+    if (rsp.op_ret == -1) {
+        gf_smsg(this->name, GF_LOG_WARNING, gf_error_to_errno(rsp.op_errno),
+                PC_MSG_REMOTE_OP_FAILED, NULL);
+    }
+    CLIENT_STACK_UNWIND(readdirp, frame, rsp.op_ret,
+                        gf_error_to_errno(rsp.op_errno), &entries, xdata);
+
+    if (rsp.op_ret != -1) {
+        gf_dirent_free(&entries);
+    }
+
+    if (xdata)
+        dict_unref(xdata);
+
+    clnt_readdirp_rsp_cleanup_v2(&rsp);
+
+    return 0;
+}
+
+int
+client4_0_rename_cbk(struct rpc_req *req, struct iovec *iov, int count,
+                     void *myframe)
+{
+    call_frame_t *frame = NULL;
+    gfx_rename_rsp rsp = {
+        0,
+    };
+    struct iatt stbuf = {
+        0,
+    };
+    struct iatt preoldparent = {
+        0,
+    };
+    struct iatt postoldparent = {
+        0,
+    };
+    struct iatt prenewparent = {
+        0,
+    };
+    struct iatt postnewparent = {
+        0,
+    };
+    int ret = 0;
+    xlator_t *this = NULL;
+    dict_t *xdata = NULL;
+
+    this = THIS;
+
+    frame = myframe;
+
+    if (-1 == req->rpc_status) {
+        rsp.op_ret = -1;
+        rsp.op_errno = ENOTCONN;
+        goto out;
+    }
+
+    ret = xdr_to_generic(*iov, &rsp, (xdrproc_t)xdr_gfx_rename_rsp);
+    if (ret < 0) {
+        gf_smsg(this->name, GF_LOG_ERROR, EINVAL, PC_MSG_XDR_DECODING_FAILED,
+                NULL);
+        rsp.op_ret = -1;
+        rsp.op_errno = EINVAL;
+        goto out;
+    }
+
+    client_post_rename_v2(this, &rsp, &stbuf, &preoldparent, &postoldparent,
+                          &prenewparent, &postnewparent, &xdata);
+out:
+    if (rsp.op_ret == -1) {
+        gf_smsg(this->name, GF_LOG_WARNING, gf_error_to_errno(rsp.op_errno),
+                PC_MSG_REMOTE_OP_FAILED, NULL);
+    }
+    CLIENT_STACK_UNWIND(rename, frame, rsp.op_ret,
+                        gf_error_to_errno(rsp.op_errno), &stbuf, &preoldparent,
+                        &postoldparent, &prenewparent, &postnewparent, xdata);
+
+    if (xdata)
+        dict_unref(xdata);
+
+    return 0;
+}
+
+int
+client4_0_link_cbk(struct rpc_req *req, struct iovec *iov, int count,
+                   void *myframe)
+{
+    gfx_common_3iatt_rsp rsp = {
+        0,
+    };
+    call_frame_t *frame = NULL;
+    struct iatt stbuf = {
+        0,
+    };
+    struct iatt preparent = {
+        0,
+    };
+    struct iatt postparent = {
+        0,
+    };
+    int ret = 0;
+    clnt_local_t *local = NULL;
+    inode_t *inode = NULL;
+    xlator_t *this = NULL;
+    dict_t *xdata = NULL;
+
+    this = THIS;
+
+    frame = myframe;
+
+    local = frame->local;
+    inode = local->loc.inode;
+
+    if (-1 == req->rpc_status) {
+        rsp.op_ret = -1;
+        rsp.op_errno = ENOTCONN;
+        goto out;
+    }
+
+    ret = xdr_to_generic(*iov, &rsp, (xdrproc_t)xdr_gfx_common_3iatt_rsp);
+    if (ret < 0) {
+        gf_smsg(this->name, GF_LOG_ERROR, EINVAL, PC_MSG_XDR_DECODING_FAILED,
+                NULL);
+        rsp.op_ret = -1;
+        rsp.op_errno = EINVAL;
+        goto out;
+    }
+
+    ret = client_post_common_3iatt(this, &rsp, &stbuf, &preparent, &postparent,
+                                   &xdata);
+out:
+    if (rsp.op_ret == -1) {
+        if (GF_IGNORE_IF_GSYNCD_SAFE_ERROR(frame, rsp.op_errno)) {
+            gf_smsg(this->name, GF_LOG_WARNING, gf_error_to_errno(rsp.op_errno),
+                    PC_MSG_REMOTE_OP_FAILED, "source=%s", local->loc.path,
+                    "target=%s", local->loc2.path, NULL);
+        }
+    }
+
+    CLIENT_STACK_UNWIND(link, frame, rsp.op_ret,
+                        gf_error_to_errno(rsp.op_errno), inode, &stbuf,
+                        &preparent, &postparent, xdata);
+
+    if (xdata)
+        dict_unref(xdata);
+
+    return 0;
+}
+
+int
+client4_0_opendir_cbk(struct rpc_req *req, struct iovec *iov, int count,
+                      void *myframe)
+{
+    clnt_local_t *local = NULL;
+    call_frame_t *frame = NULL;
+    fd_t *fd = NULL;
+    int ret = 0;
+    gfx_open_rsp rsp = {
+        0,
+    };
+    xlator_t *this = NULL;
+    dict_t *xdata = NULL;
+
+    this = THIS;
+
+    frame = myframe;
+    local = frame->local;
+
+    fd = local->fd;
+
+    if (-1 == req->rpc_status) {
+        rsp.op_ret = -1;
+        rsp.op_errno = ENOTCONN;
+        goto out;
+    }
+
+    /* open and opendir are two operations dealing with same thing,
+       but separated by fop number only */
+    ret = xdr_to_generic(*iov, &rsp, (xdrproc_t)xdr_gfx_open_rsp);
+    if (ret < 0) {
+        gf_smsg(this->name, GF_LOG_ERROR, EINVAL, PC_MSG_XDR_DECODING_FAILED,
+                NULL);
+        rsp.op_ret = -1;
+        rsp.op_errno = EINVAL;
+        goto out;
+    }
+
+    if (-1 != rsp.op_ret) {
+        ret = client_add_fd_to_saved_fds(frame->this, fd, &local->loc, 0,
+                                         rsp.fd, 1);
+        if (ret) {
+            rsp.op_ret = -1;
+            rsp.op_errno = -ret;
+            goto out;
+        }
+    }
+
+    ret = xdr_to_dict(&rsp.xdata, &xdata);
+out:
+    if (rsp.op_ret == -1) {
+        gf_smsg(this->name,
+                fop_log_level(GF_FOP_OPENDIR, gf_error_to_errno(rsp.op_errno)),
+                gf_error_to_errno(rsp.op_errno), PC_MSG_REMOTE_OP_FAILED,
+                "path=%s", local->loc.path, "gfid=%s",
+                loc_gfid_utoa(&local->loc), NULL);
+    }
+    CLIENT_STACK_UNWIND(opendir, frame, rsp.op_ret,
+                        gf_error_to_errno(rsp.op_errno), fd, xdata);
+
+    if (xdata)
+        dict_unref(xdata);
+
+    return 0;
+}
+
+int
+client4_0_lookup_cbk(struct rpc_req *req, struct iovec *iov, int count,
+                     void *myframe)
+{
+    gfx_common_2iatt_rsp rsp = {
+        0,
+    };
+    clnt_local_t *local = NULL;
+    call_frame_t *frame = NULL;
+    int ret = 0;
+    struct iatt stbuf = {
+        0,
+    };
+    struct iatt postparent = {
+        0,
+    };
+    int op_errno = EINVAL;
+    dict_t *xdata = NULL;
+    inode_t *inode = NULL;
+    xlator_t *this = NULL;
+
+    this = THIS;
+
+    frame = myframe;
+    local = frame->local;
+    inode = local->loc.inode;
+
+    if (-1 == req->rpc_status) {
+        rsp.op_ret = -1;
+        op_errno = ENOTCONN;
+        goto out;
+    }
+
+    ret = xdr_to_generic(*iov, &rsp, (xdrproc_t)xdr_gfx_common_2iatt_rsp);
+    if (ret < 0) {
+        gf_smsg(this->name, GF_LOG_ERROR, EINVAL, PC_MSG_XDR_DECODING_FAILED,
+                NULL);
+        rsp.op_ret = -1;
+        op_errno = EINVAL;
+        goto out;
+    }
+
+    /* Preserve the op_errno received from the server */
+    op_errno = gf_error_to_errno(rsp.op_errno);
+
+    ret = client_post_common_2iatt(this, &rsp, &stbuf, &postparent, &xdata);
+    if (ret < 0) {
+        /* Don't change the op_errno if the fop failed on server */
+        if (rsp.op_ret == 0)
+            op_errno = rsp.op_errno;
+        rsp.op_ret = -1;
+        goto out;
+    }
+
+    if (rsp.op_ret < 0)
+        goto out;
+
+    if ((!gf_uuid_is_null(inode->gfid)) &&
+        (gf_uuid_compare(stbuf.ia_gfid, inode->gfid) != 0)) {
+        gf_msg_debug(frame->this->name, 0, "gfid changed for %s",
+                     local->loc.path);
+
+        rsp.op_ret = -1;
+        op_errno = ESTALE;
+        if (xdata)
+            ret = dict_set_int32_sizen(xdata, "gfid-changed", 1);
+
+        goto out;
+    }
+
+    rsp.op_ret = 0;
+
+out:
+    /* Restore the correct op_errno to rsp.op_errno */
+    rsp.op_errno = op_errno;
+    if (rsp.op_ret == -1) {
+        /* any error other than ENOENT */
+        if (!(local->loc.name && rsp.op_errno == ENOENT) &&
+            !(rsp.op_errno == ESTALE))
+            gf_smsg(this->name, GF_LOG_WARNING, rsp.op_errno,
+                    PC_MSG_REMOTE_OP_FAILED, "path=%s", local->loc.path,
+                    "gfid=%s", loc_gfid_utoa(&local->loc), NULL);
+        else
+            gf_msg_trace(this->name, 0,
+                         "not found on remote "
+                         "node");
+    }
+
+    CLIENT_STACK_UNWIND(lookup, frame, rsp.op_ret, rsp.op_errno, inode, &stbuf,
+                        xdata, &postparent);
+
+    if (xdata)
+        dict_unref(xdata);
+
+    return 0;
+}
+
+int
+client4_0_readv_cbk(struct rpc_req *req, struct iovec *iov, int count,
+                    void *myframe)
+{
+    call_frame_t *frame = NULL;
+    struct iobref *iobref = NULL;
+    struct iovec vector[MAX_IOVEC] = {
+        {0},
+    };
+    struct iatt stat = {
+        0,
+    };
+    gfx_read_rsp rsp = {
+        0,
+    };
+    int ret = 0, rspcount = 0;
+    clnt_local_t *local = NULL;
+    xlator_t *this = NULL;
+    dict_t *xdata = NULL;
+
+    this = THIS;
+
+    frame = myframe;
+    local = frame->local;
+
+    if (-1 == req->rpc_status) {
+        rsp.op_ret = -1;
+        rsp.op_errno = ENOTCONN;
+        goto out;
+    }
+
+    ret = xdr_to_generic(*iov, &rsp, (xdrproc_t)xdr_gfx_read_rsp);
+    if (ret < 0) {
+        gf_smsg(this->name, GF_LOG_ERROR, EINVAL, PC_MSG_XDR_DECODING_FAILED,
+                NULL);
+        rsp.op_ret = -1;
+        rsp.op_errno = EINVAL;
+        goto out;
+    }
+
+    memset(vector, 0, sizeof(vector));
+
+    ret = client_post_readv_v2(this, &rsp, &iobref, req->rsp_iobref, &stat,
+                               vector, &req->rsp[1], &rspcount, &xdata);
+out:
+    if (rsp.op_ret == -1) {
+        gf_smsg(this->name, GF_LOG_WARNING, gf_error_to_errno(rsp.op_errno),
+                PC_MSG_REMOTE_OP_FAILED, NULL);
+    } else if (rsp.op_ret >= 0) {
+        if (local->attempt_reopen)
+            client_attempt_reopen(local->fd, this);
+    }
+    CLIENT_STACK_UNWIND(readv, frame, rsp.op_ret,
+                        gf_error_to_errno(rsp.op_errno), vector, rspcount,
+                        &stat, iobref, xdata);
+
+    if (xdata)
+        dict_unref(xdata);
+
+    return 0;
+}
+
+int
+client4_0_release_cbk(struct rpc_req *req, struct iovec *iov, int count,
+                      void *myframe)
+{
+    call_frame_t *frame = NULL;
+
+    frame = myframe;
+    STACK_DESTROY(frame->root);
+    return 0;
+}
+int
+client4_0_releasedir_cbk(struct rpc_req *req, struct iovec *iov, int count,
+                         void *myframe)
+{
+    call_frame_t *frame = NULL;
+
+    frame = myframe;
+    STACK_DESTROY(frame->root);
+    return 0;
+}
+
+int
+client4_0_getactivelk_cbk(struct rpc_req *req, struct iovec *iov, int count,
+                          void *myframe)
+{
+    call_frame_t *frame = NULL;
+    gfx_getactivelk_rsp rsp = {
+        0,
+    };
+    int32_t ret = 0;
+    lock_migration_info_t locklist;
+    xlator_t *this = NULL;
+    dict_t *xdata = NULL;
+
+    this = THIS;
+
+    frame = myframe;
+
+    if (-1 == req->rpc_status) {
+        rsp.op_ret = -1;
+        rsp.op_errno = ENOTCONN;
+        goto out;
+    }
+
+    ret = xdr_to_generic(*iov, &rsp, (xdrproc_t)xdr_gfx_getactivelk_rsp);
+    if (ret < 0) {
+        gf_smsg(this->name, GF_LOG_ERROR, EINVAL, PC_MSG_XDR_DECODING_FAILED,
+                NULL);
+        rsp.op_ret = -1;
+        rsp.op_errno = EINVAL;
+        goto out;
+    }
+
+    INIT_LIST_HEAD(&locklist.list);
+
+    if (rsp.op_ret > 0) {
+        clnt_unserialize_rsp_locklist_v2(this, &rsp, &locklist);
+    }
+
+    xdr_to_dict(&rsp.xdata, &xdata);
+
+out:
+    if (rsp.op_ret == -1) {
+        gf_smsg(this->name, GF_LOG_WARNING, gf_error_to_errno(rsp.op_errno),
+                PC_MSG_REMOTE_OP_FAILED, NULL);
+    }
+
+    CLIENT_STACK_UNWIND(getactivelk, frame, rsp.op_ret,
+                        gf_error_to_errno(rsp.op_errno), &locklist, xdata);
+    if (xdata)
+        dict_unref(xdata);
+
+    clnt_getactivelk_rsp_cleanup_v2(&rsp);
+
+    return 0;
+}
+
+int
+client4_0_setactivelk_cbk(struct rpc_req *req, struct iovec *iov, int count,
+                          void *myframe)
+{
+    call_frame_t *frame = NULL;
+    gfx_common_rsp rsp = {
+        0,
+    };
+    int32_t ret = 0;
+    xlator_t *this = NULL;
+    dict_t *xdata = NULL;
+
+    this = THIS;
+
+    frame = myframe;
+
+    if (-1 == req->rpc_status) {
+        rsp.op_ret = -1;
+        rsp.op_errno = ENOTCONN;
+        goto out;
+    }
+
+    ret = xdr_to_generic(*iov, &rsp, (xdrproc_t)xdr_gfx_common_rsp);
+    if (ret < 0) {
+        gf_smsg(this->name, GF_LOG_ERROR, EINVAL, PC_MSG_XDR_DECODING_FAILED,
+                NULL);
+        rsp.op_ret = -1;
+        rsp.op_errno = EINVAL;
+        goto out;
+    }
+
+    xdr_to_dict(&rsp.xdata, &xdata);
+out:
+    if (rsp.op_ret == -1) {
+        gf_smsg(this->name, GF_LOG_WARNING, gf_error_to_errno(rsp.op_errno),
+                PC_MSG_REMOTE_OP_FAILED, NULL);
+    }
+
+    CLIENT_STACK_UNWIND(setactivelk, frame, rsp.op_ret,
+                        gf_error_to_errno(rsp.op_errno), xdata);
+
+    if (xdata)
+        dict_unref(xdata);
+
+    return 0;
+}
+
+int
+client4_0_copy_file_range_cbk(struct rpc_req *req, struct iovec *iov, int count,
+                              void *myframe)
+{
+    gfx_common_3iatt_rsp rsp = {
+        0,
+    };
+    call_frame_t *frame = NULL;
+    struct iatt stbuf = {
+        0,
+    };
+    struct iatt prestat = {
+        0,
+    };
+    struct iatt poststat = {
+        0,
+    };
+    int ret = 0;
+    xlator_t *this = NULL;
+    dict_t *xdata = NULL;
+    clnt_local_t *local = NULL;
+
+    this = THIS;
+
+    frame = myframe;
+    local = frame->local;
+
+    if (-1 == req->rpc_status) {
+        rsp.op_ret = -1;
+        rsp.op_errno = ENOTCONN;
+        goto out;
+    }
+
+    ret = xdr_to_generic(*iov, &rsp, (xdrproc_t)xdr_gfx_common_3iatt_rsp);
+    if (ret < 0) {
+        gf_smsg(this->name, GF_LOG_ERROR, EINVAL, PC_MSG_XDR_DECODING_FAILED,
+                NULL);
+        rsp.op_ret = -1;
+        rsp.op_errno = EINVAL;
+        goto out;
+    }
+
+    ret = client_post_common_3iatt(this, &rsp, &stbuf, &prestat, &poststat,
+                                   &xdata);
+    if (ret < 0)
+        goto out;
+out:
+    if (rsp.op_ret == -1) {
+        gf_smsg(this->name, GF_LOG_WARNING, gf_error_to_errno(rsp.op_errno),
+                PC_MSG_REMOTE_OP_FAILED, NULL);
+    } else if (rsp.op_ret >= 0) {
+        if (local->attempt_reopen)
+            client_attempt_reopen(local->fd, this);
+        if (local->attempt_reopen_out)
+            client_attempt_reopen(local->fd_out, this);
+    }
+    CLIENT_STACK_UNWIND(copy_file_range, frame, rsp.op_ret,
+                        gf_error_to_errno(rsp.op_errno), &stbuf, &prestat,
+                        &poststat, xdata);
+
+    if (xdata)
+        dict_unref(xdata);
+
+    return 0;
+}
+
+int32_t
+client4_0_releasedir(call_frame_t *frame, xlator_t *this, void *data)
+{
+    clnt_conf_t *conf = NULL;
+    clnt_fd_ctx_t *fdctx = NULL;
+    clnt_args_t *args = NULL;
+    int64_t remote_fd = -1;
+    gf_boolean_t destroy = _gf_false;
+
+    if (!this || !data)
+        goto out;
+
+    args = data;
+    conf = this->private;
+
+    pthread_spin_lock(&conf->fd_lock);
+    {
+        fdctx = this_fd_del_ctx(args->fd, this);
+        if (fdctx != NULL) {
+            remote_fd = fdctx->remote_fd;
+
+            /* fdctx->remote_fd == -1 indicates a reopen attempt
+               in progress. Just mark ->released = 1 and let
+               reopen_cbk handle releasing
+            */
+
+            if (remote_fd == -1) {
+                fdctx->released = 1;
+            } else {
+                list_del_init(&fdctx->sfd_pos);
+                destroy = _gf_true;
+            }
+        }
+    }
+    pthread_spin_unlock(&conf->fd_lock);
+
+    if (destroy)
+        client_fdctx_destroy(this, fdctx);
+
+out:
+
+    return 0;
+}
+
+int32_t
+client4_0_release(call_frame_t *frame, xlator_t *this, void *data)
+{
+    int64_t remote_fd = -1;
+    clnt_conf_t *conf = NULL;
+    clnt_fd_ctx_t *fdctx = NULL;
+    clnt_args_t *args = NULL;
+    gf_boolean_t destroy = _gf_false;
+
+    if (!this || !data)
+        goto out;
+
+    args = data;
+    conf = this->private;
+
+    pthread_spin_lock(&conf->fd_lock);
+    {
+        fdctx = this_fd_del_ctx(args->fd, this);
+        if (fdctx != NULL) {
+            remote_fd = fdctx->remote_fd;
+
+            /* fdctx->remote_fd == -1 indicates a reopen attempt
+               in progress. Just mark ->released = 1 and let
+               reopen_cbk handle releasing
+            */
+            if (remote_fd == -1) {
+                fdctx->released = 1;
+            } else {
+                list_del_init(&fdctx->sfd_pos);
+                destroy = _gf_true;
+            }
+        }
+    }
+    pthread_spin_unlock(&conf->fd_lock);
+
+    if (destroy)
+        client_fdctx_destroy(this, fdctx);
+out:
+    return 0;
+}
+
+int32_t
+client4_0_lookup(call_frame_t *frame, xlator_t *this, void *data)
+{
+    clnt_conf_t *conf = NULL;
+    clnt_local_t *local = NULL;
+    clnt_args_t *args = NULL;
+    gfx_lookup_req req = {
+        {
+            0,
+        },
+    };
+    int ret = 0;
+    int op_errno = ESTALE;
+    data_t *content = NULL;
+    struct iovec vector[MAX_IOVEC] = {
+        {0},
+    };
+    int count = 0;
+    struct iobref *rsp_iobref = NULL;
+    struct iobuf *rsp_iobuf = NULL;
+    struct iovec *rsphdr = NULL;
+    client_payload_t cp;
+
+    if (!frame || !this || !data)
+        goto unwind;
+
+    conf = this->private;
+    args = data;
+    local = mem_get0(this->local_pool);
+    if (!local) {
+        op_errno = ENOMEM;
+        goto unwind;
+    }
+    frame->local = local;
+
+    if (!(args->loc && args->loc->inode))
+        goto unwind;
+
+    loc_copy(&local->loc, args->loc);
+    loc_path(&local->loc, NULL);
+
+    if (args->xdata) {
+        content = dict_get_sizen(args->xdata, GF_CONTENT_KEY);
+        if (content != NULL) {
+            rsp_iobref = iobref_new();
+            if (rsp_iobref == NULL) {
+                goto unwind;
+            }
+
+            /* TODO: what is the size we should send ? */
+            /* This change very much depends on quick-read
+               changes */
+            rsp_iobuf = iobuf_get(this->ctx->iobuf_pool);
+            if (rsp_iobuf == NULL) {
+                goto unwind;
+            }
+
+            iobref_add(rsp_iobref, rsp_iobuf);
+            memset(vector, 0, sizeof(vector));
+            rsphdr = &vector[0];
+            rsphdr->iov_base = iobuf_ptr(rsp_iobuf);
+            rsphdr->iov_len = iobuf_pagesize(rsp_iobuf);
+            count = 1;
+            local->iobref = rsp_iobref;
+            iobuf_unref(rsp_iobuf);
+            rsp_iobuf = NULL;
+            rsp_iobref = NULL;
+        }
+    }
+
+    ret = client_pre_lookup_v2(this, &req, args->loc, args->xdata);
+    if (ret) {
+        op_errno = -ret;
+        goto unwind;
+    }
+
+    memset(&cp, 0, sizeof(client_payload_t));
+
+    cp.rsphdr = rsphdr;
+    cp.rsphdr_cnt = count;
+    cp.rsp_iobref = local->iobref;
+    ret = client_submit_request(this, &req, frame, conf->fops, GFS3_OP_LOOKUP,
+                                client4_0_lookup_cbk, &cp,
+                                (xdrproc_t)xdr_gfx_lookup_req);
+
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_WARNING, 0, PC_MSG_FOP_SEND_FAILED, NULL);
+    }
+
+    GF_FREE(req.xdata.pairs.pairs_val);
+
+    return 0;
+
+unwind:
+    CLIENT_STACK_UNWIND(lookup, frame, -1, op_errno, NULL, NULL, NULL, NULL);
+
+    GF_FREE(req.xdata.pairs.pairs_val);
+
+    if (rsp_iobref)
+        iobref_unref(rsp_iobref);
+
+    return 0;
+}
+
+int32_t
+client4_0_stat(call_frame_t *frame, xlator_t *this, void *data)
+{
+    clnt_conf_t *conf = NULL;
+    clnt_args_t *args = NULL;
+    gfx_stat_req req = {
+        {
+            0,
+        },
+    };
+    int ret = 0;
+    int op_errno = ESTALE;
+
+    if (!frame || !this || !data)
+        goto unwind;
+
+    args = data;
+    conf = this->private;
+
+    ret = client_pre_stat_v2(this, &req, args->loc, args->xdata);
+    if (ret) {
+        op_errno = -ret;
+        goto unwind;
+    }
+    ret = client_submit_request(this, &req, frame, conf->fops, GFS3_OP_STAT,
+                                client4_0_stat_cbk, NULL,
+                                (xdrproc_t)xdr_gfx_stat_req);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_WARNING, 0, PC_MSG_FOP_SEND_FAILED, NULL);
+    }
+
+    GF_FREE(req.xdata.pairs.pairs_val);
+
+    return 0;
+unwind:
+    CLIENT_STACK_UNWIND(stat, frame, -1, op_errno, NULL, NULL);
+
+    GF_FREE(req.xdata.pairs.pairs_val);
+
+    return 0;
+}
+
+int32_t
+client4_0_truncate(call_frame_t *frame, xlator_t *this, void *data)
+{
+    clnt_conf_t *conf = NULL;
+    clnt_args_t *args = NULL;
+    gfx_truncate_req req = {
+        {
+            0,
+        },
+    };
+    int ret = 0;
+    int op_errno = ESTALE;
+
+    if (!frame || !this || !data)
+        goto unwind;
+
+    args = data;
+    conf = this->private;
+
+    ret = client_pre_truncate_v2(this, &req, args->loc, args->offset,
+                                 args->xdata);
+    if (ret) {
+        op_errno = -ret;
+        goto unwind;
+    }
+    ret = client_submit_request(this, &req, frame, conf->fops, GFS3_OP_TRUNCATE,
+                                client4_0_truncate_cbk, NULL,
+                                (xdrproc_t)xdr_gfx_truncate_req);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_WARNING, 0, PC_MSG_FOP_SEND_FAILED, NULL);
+    }
+
+    GF_FREE(req.xdata.pairs.pairs_val);
+
+    return 0;
+unwind:
+    CLIENT_STACK_UNWIND(truncate, frame, -1, op_errno, NULL, NULL, NULL);
+    GF_FREE(req.xdata.pairs.pairs_val);
+
+    return 0;
+}
+
+int32_t
+client4_0_ftruncate(call_frame_t *frame, xlator_t *this, void *data)
+{
+    clnt_args_t *args = NULL;
+    clnt_conf_t *conf = NULL;
+    gfx_ftruncate_req req = {
+        {
+            0,
+        },
+    };
+    int op_errno = EINVAL;
+    int ret = 0;
+
+    if (!frame || !this || !data)
+        goto unwind;
+
+    args = data;
+
+    conf = this->private;
+
+    ret = client_pre_ftruncate_v2(this, &req, args->fd, args->offset,
+                                  args->xdata);
+    if (ret) {
+        op_errno = -ret;
+        goto unwind;
+    }
+    ret = client_submit_request(this, &req, frame, conf->fops,
+                                GFS3_OP_FTRUNCATE, client4_0_ftruncate_cbk,
+                                NULL, (xdrproc_t)xdr_gfx_ftruncate_req);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_WARNING, 0, PC_MSG_FOP_SEND_FAILED, NULL);
+    }
+
+    GF_FREE(req.xdata.pairs.pairs_val);
+
+    return 0;
+unwind:
+    CLIENT_STACK_UNWIND(ftruncate, frame, -1, op_errno, NULL, NULL, NULL);
+    GF_FREE(req.xdata.pairs.pairs_val);
+
+    return 0;
+}
+
+int32_t
+client4_0_access(call_frame_t *frame, xlator_t *this, void *data)
+{
+    clnt_conf_t *conf = NULL;
+    clnt_args_t *args = NULL;
+    gfx_access_req req = {
+        {
+            0,
+        },
+    };
+    int ret = 0;
+    int op_errno = ESTALE;
+
+    if (!frame || !this || !data)
+        goto unwind;
+
+    args = data;
+
+    conf = this->private;
+
+    ret = client_pre_access_v2(this, &req, args->loc, args->mask, args->xdata);
+    if (ret) {
+        op_errno = -ret;
+        goto unwind;
+    }
+    ret = client_submit_request(this, &req, frame, conf->fops, GFS3_OP_ACCESS,
+                                client4_0_access_cbk, NULL,
+                                (xdrproc_t)xdr_gfx_access_req);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_WARNING, 0, PC_MSG_FOP_SEND_FAILED, NULL);
+    }
+
+    GF_FREE(req.xdata.pairs.pairs_val);
+
+    return 0;
+unwind:
+    CLIENT_STACK_UNWIND(access, frame, -1, op_errno, NULL);
+    GF_FREE(req.xdata.pairs.pairs_val);
+
+    return 0;
+}
+
+int32_t
+client4_0_readlink(call_frame_t *frame, xlator_t *this, void *data)
+{
+    clnt_conf_t *conf = NULL;
+    clnt_args_t *args = NULL;
+    gfx_readlink_req req = {
+        {
+            0,
+        },
+    };
+    int ret = 0;
+    int op_errno = ESTALE;
+    clnt_local_t *local = NULL;
+
+    if (!frame || !this || !data)
+        goto unwind;
+
+    args = data;
+
+    conf = this->private;
+
+    local = mem_get0(this->local_pool);
+    if (!local) {
+        op_errno = ENOMEM;
+        goto unwind;
+    }
+
+    frame->local = local;
+
+    ret = client_pre_readlink_v2(this, &req, args->loc, args->size,
+                                 args->xdata);
+    if (ret) {
+        op_errno = -ret;
+        goto unwind;
+    }
+
+    ret = client_submit_request(this, &req, frame, conf->fops, GFS3_OP_READLINK,
+                                client4_0_readlink_cbk, NULL,
+                                (xdrproc_t)xdr_gfx_readlink_req);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_WARNING, 0, PC_MSG_FOP_SEND_FAILED, NULL);
+    }
+
+    GF_FREE(req.xdata.pairs.pairs_val);
+
+    return 0;
+unwind:
+
+    CLIENT_STACK_UNWIND(readlink, frame, -1, op_errno, NULL, NULL, NULL);
+    GF_FREE(req.xdata.pairs.pairs_val);
+
+    return 0;
+}
+
+int32_t
+client4_0_unlink(call_frame_t *frame, xlator_t *this, void *data)
+{
+    clnt_conf_t *conf = NULL;
+    clnt_args_t *args = NULL;
+    gfx_unlink_req req = {
+        {
+            0,
+        },
+    };
+    int ret = 0;
+    int op_errno = 0;
+
+    if (!frame || !this || !data)
+        goto unwind;
+
+    args = data;
+    conf = this->private;
+
+    ret = client_pre_unlink_v2(this, &req, args->loc, args->flags, args->xdata);
+    if (ret) {
+        op_errno = -ret;
+        goto unwind;
+    }
+    ret = client_submit_request(this, &req, frame, conf->fops, GFS3_OP_UNLINK,
+                                client4_0_unlink_cbk, NULL,
+                                (xdrproc_t)xdr_gfx_unlink_req);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_WARNING, 0, PC_MSG_FOP_SEND_FAILED, NULL);
+    }
+
+    GF_FREE(req.xdata.pairs.pairs_val);
+
+    return 0;
+unwind:
+    CLIENT_STACK_UNWIND(unlink, frame, -1, op_errno, NULL, NULL, NULL);
+    GF_FREE(req.xdata.pairs.pairs_val);
+
+    return 0;
+}
+
+int32_t
+client4_0_rmdir(call_frame_t *frame, xlator_t *this, void *data)
+{
+    clnt_conf_t *conf = NULL;
+    clnt_args_t *args = NULL;
+    gfx_rmdir_req req = {
+        {
+            0,
+        },
+    };
+    int ret = 0;
+    int op_errno = ESTALE;
+
+    if (!frame || !this || !data)
+        goto unwind;
+
+    args = data;
+    conf = this->private;
+
+    ret = client_pre_rmdir_v2(this, &req, args->loc, args->flags, args->xdata);
+
+    if (ret) {
+        op_errno = -ret;
+        goto unwind;
+    }
+    ret = client_submit_request(this, &req, frame, conf->fops, GFS3_OP_RMDIR,
+                                client4_0_rmdir_cbk, NULL,
+                                (xdrproc_t)xdr_gfx_rmdir_req);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_WARNING, 0, PC_MSG_FOP_SEND_FAILED, NULL);
+    }
+    GF_FREE(req.xdata.pairs.pairs_val);
+
+    return 0;
+unwind:
+    CLIENT_STACK_UNWIND(rmdir, frame, -1, op_errno, NULL, NULL, NULL);
+    GF_FREE(req.xdata.pairs.pairs_val);
+
+    return 0;
+}
+
+int32_t
+client4_0_symlink(call_frame_t *frame, xlator_t *this, void *data)
+{
+    clnt_local_t *local = NULL;
+    clnt_conf_t *conf = NULL;
+    clnt_args_t *args = NULL;
+    gfx_symlink_req req = {
+        {
+            0,
+        },
+    };
+    int ret = 0;
+    int op_errno = ESTALE;
+
+    if (!frame || !this || !data)
+        goto unwind;
+
+    args = data;
+    conf = this->private;
+
+    local = mem_get0(this->local_pool);
+    if (!local) {
+        op_errno = ENOMEM;
+        goto unwind;
+    }
+
+    frame->local = local;
+
+    if (!(args->loc && args->loc->parent))
+        goto unwind;
+
+    loc_copy(&local->loc, args->loc);
+    loc_path(&local->loc, NULL);
+
+    local->loc2.path = gf_strdup(args->linkname);
+
+    ret = client_pre_symlink_v2(this, &req, args->loc, args->linkname,
+                                args->umask, args->xdata);
+    if (ret) {
+        op_errno = -ret;
+        goto unwind;
+    }
+    ret = client_submit_request(this, &req, frame, conf->fops, GFS3_OP_SYMLINK,
+                                client4_0_symlink_cbk, NULL,
+                                (xdrproc_t)xdr_gfx_symlink_req);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_WARNING, 0, PC_MSG_FOP_SEND_FAILED, NULL);
+    }
+
+    GF_FREE(req.xdata.pairs.pairs_val);
+
+    return 0;
+unwind:
+
+    CLIENT_STACK_UNWIND(symlink, frame, -1, op_errno, NULL, NULL, NULL, NULL,
+                        NULL);
+
+    GF_FREE(req.xdata.pairs.pairs_val);
+
+    return 0;
+}
+
+int32_t
+client4_0_rename(call_frame_t *frame, xlator_t *this, void *data)
+{
+    clnt_conf_t *conf = NULL;
+    clnt_args_t *args = NULL;
+    gfx_rename_req req = {
+        {
+            0,
+        },
+    };
+    int ret = 0;
+    int op_errno = ESTALE;
+
+    if (!frame || !this || !data)
+        goto unwind;
+
+    args = data;
+    conf = this->private;
+
+    ret = client_pre_rename_v2(this, &req, args->oldloc, args->newloc,
+                               args->xdata);
+    if (ret) {
+        op_errno = -ret;
+        goto unwind;
+    }
+    ret = client_submit_request(this, &req, frame, conf->fops, GFS3_OP_RENAME,
+                                client4_0_rename_cbk, NULL,
+                                (xdrproc_t)xdr_gfx_rename_req);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_WARNING, 0, PC_MSG_FOP_SEND_FAILED, NULL);
+    }
+
+    GF_FREE(req.xdata.pairs.pairs_val);
+
+    return 0;
+unwind:
+    CLIENT_STACK_UNWIND(rename, frame, -1, op_errno, NULL, NULL, NULL, NULL,
+                        NULL, NULL);
+
+    GF_FREE(req.xdata.pairs.pairs_val);
+
+    return 0;
+}
+
+int32_t
+client4_0_link(call_frame_t *frame, xlator_t *this, void *data)
+{
+    clnt_local_t *local = NULL;
+    clnt_conf_t *conf = NULL;
+    clnt_args_t *args = NULL;
+    gfx_link_req req = {
+        {
+            0,
+        },
+    };
+    int ret = 0;
+    int op_errno = ESTALE;
+
+    if (!frame || !this || !data)
+        goto unwind;
+
+    args = data;
+    conf = this->private;
+
+    local = mem_get0(this->local_pool);
+    if (!local) {
+        op_errno = ENOMEM;
+        goto unwind;
+    }
+
+    frame->local = local;
+
+    ret = client_pre_link_v2(this, &req, args->oldloc, args->newloc,
+                             args->xdata);
+    if (ret) {
+        op_errno = -ret;
+        goto unwind;
+    }
+
+    loc_copy(&local->loc, args->oldloc);
+    loc_path(&local->loc, NULL);
+    loc_copy(&local->loc2, args->newloc);
+    loc_path(&local->loc2, NULL);
+
+    ret = client_submit_request(this, &req, frame, conf->fops, GFS3_OP_LINK,
+                                client4_0_link_cbk, NULL,
+                                (xdrproc_t)xdr_gfx_link_req);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_WARNING, 0, PC_MSG_FOP_SEND_FAILED, NULL);
+    }
+
+    GF_FREE(req.xdata.pairs.pairs_val);
+
+    return 0;
+unwind:
+    CLIENT_STACK_UNWIND(link, frame, -1, op_errno, NULL, NULL, NULL, NULL,
+                        NULL);
+    GF_FREE(req.xdata.pairs.pairs_val);
+
+    return 0;
+}
+
+int32_t
+client4_0_mknod(call_frame_t *frame, xlator_t *this, void *data)
+{
+    clnt_local_t *local = NULL;
+    clnt_conf_t *conf = NULL;
+    clnt_args_t *args = NULL;
+    gfx_mknod_req req = {
+        {
+            0,
+        },
+    };
+    int ret = 0;
+    int op_errno = ESTALE;
+
+    if (!frame || !this || !data)
+        goto unwind;
+
+    args = data;
+    conf = this->private;
+
+    local = mem_get0(this->local_pool);
+    if (!local) {
+        op_errno = ENOMEM;
+        goto unwind;
+    }
+    frame->local = local;
+
+    loc_copy(&local->loc, args->loc);
+    loc_path(&local->loc, NULL);
+
+    ret = client_pre_mknod_v2(this, &req, args->loc, args->mode, args->rdev,
+                              args->umask, args->xdata);
+    if (ret) {
+        op_errno = -ret;
+        goto unwind;
+    }
+    ret = client_submit_request(this, &req, frame, conf->fops, GFS3_OP_MKNOD,
+                                client4_0_mknod_cbk, NULL,
+                                (xdrproc_t)xdr_gfx_mknod_req);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_WARNING, 0, PC_MSG_FOP_SEND_FAILED, NULL);
+    }
+    GF_FREE(req.xdata.pairs.pairs_val);
+
+    return 0;
+unwind:
+    CLIENT_STACK_UNWIND(mknod, frame, -1, op_errno, NULL, NULL, NULL, NULL,
+                        NULL);
+
+    GF_FREE(req.xdata.pairs.pairs_val);
+
+    return 0;
+}
+
+int32_t
+client4_0_mkdir(call_frame_t *frame, xlator_t *this, void *data)
+{
+    clnt_local_t *local = NULL;
+    clnt_conf_t *conf = NULL;
+    clnt_args_t *args = NULL;
+    gfx_mkdir_req req = {
+        {
+            0,
+        },
+    };
+    int ret = 0;
+    int op_errno = ESTALE;
+
+    if (!frame || !this || !data)
+        goto unwind;
+
+    args = data;
+    conf = this->private;
+
+    if (!args->xdata || !dict_get_sizen(args->xdata, "gfid-req")) {
+        op_errno = EPERM;
+        gf_msg_callingfn(this->name, GF_LOG_WARNING, op_errno, PC_MSG_GFID_NULL,
+                         "mkdir: %s is received "
+                         "without gfid-req %p",
+                         args->loc->path, args->xdata);
+        goto unwind;
+    }
+
+    local = mem_get0(this->local_pool);
+    if (!local) {
+        op_errno = ENOMEM;
+        goto unwind;
+    }
+    frame->local = local;
+
+    if (!(args->loc && args->loc->parent))
+        goto unwind;
+
+    loc_copy(&local->loc, args->loc);
+    loc_path(&local->loc, NULL);
+
+    ret = client_pre_mkdir_v2(this, &req, args->loc, args->mode, args->umask,
+                              args->xdata);
+    if (ret) {
+        op_errno = -ret;
+        goto unwind;
+    }
+    ret = client_submit_request(this, &req, frame, conf->fops, GFS3_OP_MKDIR,
+                                client4_0_mkdir_cbk, NULL,
+                                (xdrproc_t)xdr_gfx_mkdir_req);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_WARNING, 0, PC_MSG_FOP_SEND_FAILED, NULL);
+    }
+    GF_FREE(req.xdata.pairs.pairs_val);
+
+    return 0;
+unwind:
+    CLIENT_STACK_UNWIND(mkdir, frame, -1, op_errno, NULL, NULL, NULL, NULL,
+                        NULL);
+
+    GF_FREE(req.xdata.pairs.pairs_val);
+
+    return 0;
+}
+
+int32_t
+client4_0_create(call_frame_t *frame, xlator_t *this, void *data)
+{
+    clnt_local_t *local = NULL;
+    clnt_conf_t *conf = NULL;
+    clnt_args_t *args = NULL;
+    gfx_create_req req = {
+        {
+            0,
+        },
+    };
+    int ret = 0;
+    int op_errno = ESTALE;
+
+    if (!frame || !this || !data)
+        goto unwind;
+
+    args = data;
+    conf = this->private;
+
+    local = mem_get0(this->local_pool);
+    if (!local) {
+        op_errno = ENOMEM;
+        goto unwind;
+    }
+    frame->local = local;
+
+    local->fd = fd_ref(args->fd);
+    local->flags = args->flags;
+
+    loc_copy(&local->loc, args->loc);
+    loc_path(&local->loc, NULL);
+
+    ret = client_pre_create_v2(this, &req, args->loc, args->fd, args->mode,
+                               args->flags, args->umask, args->xdata);
+    if (ret) {
+        op_errno = -ret;
+        goto unwind;
+    }
+    ret = client_submit_request(this, &req, frame, conf->fops, GFS3_OP_CREATE,
+                                client4_0_create_cbk, NULL,
+                                (xdrproc_t)xdr_gfx_create_req);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_WARNING, 0, PC_MSG_FOP_SEND_FAILED, NULL);
+    }
+
+    GF_FREE(req.xdata.pairs.pairs_val);
+
+    return 0;
+unwind:
+    CLIENT_STACK_UNWIND(create, frame, -1, op_errno, NULL, NULL, NULL, NULL,
+                        NULL, NULL);
+
+    GF_FREE(req.xdata.pairs.pairs_val);
+
+    return 0;
+}
+
+int32_t
+client4_0_open(call_frame_t *frame, xlator_t *this, void *data)
+{
+    clnt_local_t *local = NULL;
+    clnt_conf_t *conf = NULL;
+    clnt_args_t *args = NULL;
+    gfx_open_req req = {
+        {
+            0,
+        },
+    };
+    int ret = 0;
+    int op_errno = ESTALE;
+
+    if (!frame || !this || !data)
+        goto unwind;
+
+    args = data;
+
+    conf = this->private;
+
+    local = mem_get0(this->local_pool);
+    if (!local) {
+        op_errno = ENOMEM;
+        goto unwind;
+    }
+    frame->local = local;
+
+    local->flags = args->flags;
+
+    local->fd = fd_ref(args->fd);
+    loc_copy(&local->loc, args->loc);
+    loc_path(&local->loc, NULL);
+
+    ret = client_pre_open_v2(this, &req, args->loc, args->fd, args->flags,
+                             args->xdata);
+
+    if (ret) {
+        op_errno = -ret;
+        goto unwind;
+    }
+    ret = client_submit_request(this, &req, frame, conf->fops, GFS3_OP_OPEN,
+                                client4_0_open_cbk, NULL,
+                                (xdrproc_t)xdr_gfx_open_req);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_WARNING, 0, PC_MSG_FOP_SEND_FAILED, NULL);
+    }
+
+    GF_FREE(req.xdata.pairs.pairs_val);
+
+    return 0;
+unwind:
+    CLIENT_STACK_UNWIND(open, frame, -1, op_errno, NULL, NULL);
+
+    GF_FREE(req.xdata.pairs.pairs_val);
+
+    return 0;
+}
+
+int32_t
+client4_0_readv(call_frame_t *frame, xlator_t *this, void *data)
+{
+    clnt_args_t *args = NULL;
+    clnt_conf_t *conf = NULL;
+    clnt_local_t *local = NULL;
+    int op_errno = ESTALE;
+    gfx_read_req req = {
+        {
+            0,
+        },
+    };
+    int ret = 0;
+    struct iovec rsp_vec = {
+        0,
+    };
+    struct iobuf *rsp_iobuf = NULL;
+    struct iobref *rsp_iobref = NULL;
+    client_payload_t cp;
+
+    if (!frame || !this || !data)
+        goto unwind;
+
+    args = data;
+    conf = this->private;
+
+    ret = client_pre_readv_v2(this, &req, args->fd, args->size, args->offset,
+                              args->flags, args->xdata);
+    if (ret) {
+        op_errno = -ret;
+        goto unwind;
+    }
+
+    ret = client_fd_fop_prepare_local(frame, args->fd, req.fd);
+    if (ret) {
+        op_errno = -ret;
+        goto unwind;
+    }
+    local = frame->local;
+
+    rsp_iobuf = iobuf_get2(this->ctx->iobuf_pool, args->size);
+    if (rsp_iobuf == NULL) {
+        op_errno = ENOMEM;
+        goto unwind;
+    }
+
+    rsp_iobref = iobref_new();
+    if (rsp_iobref == NULL) {
+        op_errno = ENOMEM;
+        goto unwind;
+    }
+
+    iobref_add(rsp_iobref, rsp_iobuf);
+    rsp_vec.iov_base = iobuf_ptr(rsp_iobuf);
+    rsp_vec.iov_len = iobuf_pagesize(rsp_iobuf);
+    local->iobref = rsp_iobref;
+    iobuf_unref(rsp_iobuf);
+    rsp_iobref = NULL;
+    rsp_iobuf = NULL;
+
+    if (args->size > rsp_vec.iov_len) {
+        gf_smsg(this->name, GF_LOG_WARNING, ENOMEM, PC_MSG_BIGGER_SIZE,
+                "read-size=%lu", (unsigned long)args->size, "iobuf-size=%lu",
+                (unsigned long)rsp_vec.iov_len, NULL);
+        op_errno = EINVAL;
+        goto unwind;
+    }
+
+    memset(&cp, 0, sizeof(client_payload_t));
+
+    cp.rsp_payload = &rsp_vec;
+    cp.rsp_payload_cnt = 1;
+    cp.rsp_iobref = local->iobref;
+    ret = client_submit_request(this, &req, frame, conf->fops, GFS3_OP_READ,
+                                client4_0_readv_cbk, &cp,
+                                (xdrproc_t)xdr_gfx_read_req);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_WARNING, 0, PC_MSG_FOP_SEND_FAILED, NULL);
+    }
+
+    GF_FREE(req.xdata.pairs.pairs_val);
+
+    return 0;
+unwind:
+    if (rsp_iobuf)
+        iobuf_unref(rsp_iobuf);
+
+    CLIENT_STACK_UNWIND(readv, frame, -1, op_errno, NULL, 0, NULL, NULL, NULL);
+    GF_FREE(req.xdata.pairs.pairs_val);
+
+    return 0;
+}
+
+int32_t
+client4_0_writev(call_frame_t *frame, xlator_t *this, void *data)
+{
+    clnt_args_t *args = NULL;
+    clnt_conf_t *conf = NULL;
+    gfx_write_req req = {
+        {
+            0,
+        },
+    };
+    int op_errno = ESTALE;
+    int ret = 0;
+    client_payload_t cp;
+
+    if (!frame || !this || !data)
+        goto unwind;
+
+    args = data;
+    conf = this->private;
+
+    ret = client_pre_writev_v2(this, &req, args->fd, args->size, args->offset,
+                               args->flags, &args->xdata);
+
+    if (ret) {
+        op_errno = -ret;
+        goto unwind;
+    }
+
+    ret = client_fd_fop_prepare_local(frame, args->fd, req.fd);
+    if (ret) {
+        op_errno = -ret;
+        goto unwind;
+    }
+
+    memset(&cp, 0, sizeof(client_payload_t));
+
+    cp.iobref = args->iobref;
+    cp.payload = args->vector;
+    cp.payload_cnt = args->count;
+    ret = client_submit_request(this, &req, frame, conf->fops, GFS3_OP_WRITE,
+                                client4_0_writev_cbk, &cp,
+                                (xdrproc_t)xdr_gfx_write_req);
+    if (ret) {
+        /*
+         * If the lower layers fail to submit a request, they'll also
+         * do the unwind for us (see rpc_clnt_submit), so don't unwind
+         * here in such cases.
+         */
+        gf_smsg(this->name, GF_LOG_WARNING, 0, PC_MSG_FOP_SEND_FAILED, NULL);
+    }
+
+    GF_FREE(req.xdata.pairs.pairs_val);
+
+    return 0;
+
+unwind:
+    CLIENT_STACK_UNWIND(writev, frame, -1, op_errno, NULL, NULL, NULL);
+    GF_FREE(req.xdata.pairs.pairs_val);
+
+    return 0;
+}
+
+int32_t
+client4_0_flush(call_frame_t *frame, xlator_t *this, void *data)
+{
+    clnt_args_t *args = NULL;
+    gfx_flush_req req = {
+        {
+            0,
+        },
+    };
+    clnt_conf_t *conf = NULL;
+    clnt_local_t *local = NULL;
+    int op_errno = ESTALE;
+    int ret = 0;
+
+    if (!frame || !this || !data)
+        goto unwind;
+
+    args = data;
+    conf = this->private;
+
+    local = mem_get0(this->local_pool);
+    if (!local) {
+        op_errno = ENOMEM;
+        goto unwind;
+    }
+
+    frame->local = local;
+
+    local->fd = fd_ref(args->fd);
+    local->owner = frame->root->lk_owner;
+    ret = client_pre_flush_v2(this, &req, args->fd, args->xdata);
+    if (ret) {
+        op_errno = -ret;
+        if (op_errno == EBADF) {
+            ret = delete_granted_locks_owner(local->fd, &local->owner);
+            gf_msg_trace(this->name, 0,
+                         "deleting locks of owner (%s) returned %d",
+                         lkowner_utoa(&local->owner), ret);
+        }
+
+        goto unwind;
+    }
+    ret = client_submit_request(this, &req, frame, conf->fops, GFS3_OP_FLUSH,
+                                client4_0_flush_cbk, NULL,
+                                (xdrproc_t)xdr_gfx_flush_req);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_WARNING, 0, PC_MSG_FOP_SEND_FAILED, NULL);
+    }
+
+    GF_FREE(req.xdata.pairs.pairs_val);
+
+    return 0;
+
+unwind:
+    CLIENT_STACK_UNWIND(flush, frame, -1, op_errno, NULL);
+    GF_FREE(req.xdata.pairs.pairs_val);
+
+    return 0;
+}
+
+int32_t
+client4_0_fsync(call_frame_t *frame, xlator_t *this, void *data)
+{
+    clnt_args_t *args = NULL;
+    gfx_fsync_req req = {
+        {
+            0,
+        },
+    };
+    clnt_conf_t *conf = NULL;
+    int op_errno = 0;
+    int ret = 0;
+
+    if (!frame || !this || !data)
+        goto unwind;
+
+    args = data;
+    conf = this->private;
+
+    ret = client_pre_fsync_v2(this, &req, args->fd, args->flags, args->xdata);
+    if (ret) {
+        op_errno = -ret;
+        goto unwind;
+    }
+    ret = client_submit_request(this, &req, frame, conf->fops, GFS3_OP_FSYNC,
+                                client4_0_fsync_cbk, NULL,
+                                (xdrproc_t)xdr_gfx_fsync_req);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_WARNING, 0, PC_MSG_FOP_SEND_FAILED, NULL);
+    }
+
+    GF_FREE(req.xdata.pairs.pairs_val);
+
+    return 0;
+
+unwind:
+    CLIENT_STACK_UNWIND(fsync, frame, -1, op_errno, NULL, NULL, NULL);
+    GF_FREE(req.xdata.pairs.pairs_val);
+
+    return 0;
+}
+
+int32_t
+client4_0_fstat(call_frame_t *frame, xlator_t *this, void *data)
+{
+    clnt_args_t *args = NULL;
+    gfx_fstat_req req = {
+        {
+            0,
+        },
+    };
+    clnt_conf_t *conf = NULL;
+    int op_errno = ESTALE;
+    int ret = 0;
+
+    if (!frame || !this || !data)
+        goto unwind;
+
+    args = data;
+    conf = this->private;
+
+    ret = client_pre_fstat_v2(this, &req, args->fd, args->xdata);
+
+    if (ret) {
+        op_errno = -ret;
+        goto unwind;
+    }
+
+    ret = client_submit_request(this, &req, frame, conf->fops, GFS3_OP_FSTAT,
+                                client4_0_fstat_cbk, NULL,
+                                (xdrproc_t)xdr_gfx_fstat_req);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_WARNING, 0, PC_MSG_FOP_SEND_FAILED, NULL);
+    }
+
+    GF_FREE(req.xdata.pairs.pairs_val);
+
+    return 0;
+
+unwind:
+    CLIENT_STACK_UNWIND(fstat, frame, -1, op_errno, NULL, NULL);
+    GF_FREE(req.xdata.pairs.pairs_val);
+
+    return 0;
+}
+
+int32_t
+client4_0_opendir(call_frame_t *frame, xlator_t *this, void *data)
+{
+    clnt_local_t *local = NULL;
+    clnt_conf_t *conf = NULL;
+    clnt_args_t *args = NULL;
+    gfx_opendir_req req = {
+        {
+            0,
+        },
+    };
+    int ret = 0;
+    int op_errno = ESTALE;
+
+    if (!frame || !this || !data)
+        goto unwind;
+
+    args = data;
+    conf = this->private;
+
+    local = mem_get0(this->local_pool);
+    if (!local) {
+        op_errno = ENOMEM;
+        goto unwind;
+    }
+    frame->local = local;
+
+    local->fd = fd_ref(args->fd);
+    loc_copy(&local->loc, args->loc);
+    loc_path(&local->loc, NULL);
+
+    ret = client_pre_opendir_v2(this, &req, args->loc, args->fd, args->xdata);
+    if (ret) {
+        op_errno = -ret;
+        goto unwind;
+    }
+    ret = client_submit_request(this, &req, frame, conf->fops, GFS3_OP_OPENDIR,
+                                client4_0_opendir_cbk, NULL,
+                                (xdrproc_t)xdr_gfx_opendir_req);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_WARNING, 0, PC_MSG_FOP_SEND_FAILED, NULL);
+    }
+
+    GF_FREE(req.xdata.pairs.pairs_val);
+
+    return 0;
+
+unwind:
+    CLIENT_STACK_UNWIND(opendir, frame, -1, op_errno, NULL, NULL);
+
+    GF_FREE(req.xdata.pairs.pairs_val);
+
+    return 0;
+}
+
+int32_t
+client4_0_fsyncdir(call_frame_t *frame, xlator_t *this, void *data)
+{
+    clnt_args_t *args = NULL;
+    clnt_conf_t *conf = NULL;
+    gfx_fsyncdir_req req = {
+        {
+            0,
+        },
+    };
+    int ret = 0;
+    int32_t op_errno = ESTALE;
+
+    if (!frame || !this || !data)
+        goto unwind;
+
+    args = data;
+    conf = this->private;
+
+    ret = client_pre_fsyncdir_v2(this, &req, args->fd, args->flags,
+                                 args->xdata);
+    if (ret) {
+        op_errno = -ret;
+        goto unwind;
+    }
+    ret = client_submit_request(this, &req, frame, conf->fops, GFS3_OP_FSYNCDIR,
+                                client4_0_fsyncdir_cbk, NULL,
+                                (xdrproc_t)xdr_gfx_fsyncdir_req);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_WARNING, 0, PC_MSG_FOP_SEND_FAILED, NULL);
+    }
+
+    GF_FREE(req.xdata.pairs.pairs_val);
+
+    return 0;
+
+unwind:
+    CLIENT_STACK_UNWIND(fsyncdir, frame, -1, op_errno, NULL);
+    GF_FREE(req.xdata.pairs.pairs_val);
+
+    return 0;
+}
+
+int32_t
+client4_0_statfs(call_frame_t *frame, xlator_t *this, void *data)
+{
+    clnt_conf_t *conf = NULL;
+    clnt_args_t *args = NULL;
+    gfx_statfs_req req = {
+        {
+            0,
+        },
+    };
+    int ret = 0;
+    int op_errno = ESTALE;
+
+    if (!frame || !this || !data)
+        goto unwind;
+
+    args = data;
+
+    conf = this->private;
+
+    ret = client_pre_statfs_v2(this, &req, args->loc, args->xdata);
+    if (ret) {
+        op_errno = -ret;
+        goto unwind;
+    }
+    ret = client_submit_request(this, &req, frame, conf->fops, GFS3_OP_STATFS,
+                                client4_0_statfs_cbk, NULL,
+                                (xdrproc_t)xdr_gfx_statfs_req);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_WARNING, 0, PC_MSG_FOP_SEND_FAILED, NULL);
+    }
+
+    GF_FREE(req.xdata.pairs.pairs_val);
+
+    return 0;
+
+unwind:
+    CLIENT_STACK_UNWIND(statfs, frame, -1, op_errno, NULL, NULL);
+    GF_FREE(req.xdata.pairs.pairs_val);
+
+    return 0;
+}
+
+int32_t
+client4_0_setxattr(call_frame_t *frame, xlator_t *this, void *data)
+{
+    clnt_conf_t *conf = NULL;
+    clnt_args_t *args = NULL;
+    gfx_setxattr_req req = {
+        {
+            0,
+        },
+    };
+    int ret = 0;
+    int op_errno = ESTALE;
+
+    if (!frame || !this || !data)
+        goto unwind;
+
+    args = data;
+    conf = this->private;
+
+    ret = client_pre_setxattr_v2(this, &req, args->loc, args->xattr,
+                                 args->flags, args->xdata);
+    if (ret) {
+        op_errno = -ret;
+        goto unwind;
+    }
+    ret = client_submit_request(this, &req, frame, conf->fops, GFS3_OP_SETXATTR,
+                                client4_0_setxattr_cbk, NULL,
+                                (xdrproc_t)xdr_gfx_setxattr_req);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_WARNING, 0, PC_MSG_FOP_SEND_FAILED, NULL);
+    }
+    GF_FREE(req.dict.pairs.pairs_val);
+
+    GF_FREE(req.xdata.pairs.pairs_val);
+
+    return 0;
+unwind:
+    CLIENT_STACK_UNWIND(setxattr, frame, -1, op_errno, NULL);
+    GF_FREE(req.dict.pairs.pairs_val);
+
+    GF_FREE(req.xdata.pairs.pairs_val);
+
+    return 0;
+}
+
+int32_t
+client4_0_fsetxattr(call_frame_t *frame, xlator_t *this, void *data)
+{
+    clnt_args_t *args = NULL;
+    clnt_conf_t *conf = NULL;
+    gfx_fsetxattr_req req = {
+        {
+            0,
+        },
+    };
+    int op_errno = ESTALE;
+    int ret = 0;
+
+    if (!frame || !this || !data)
+        goto unwind;
+
+    args = data;
+    conf = this->private;
+
+    ret = client_pre_fsetxattr_v2(this, &req, args->fd, args->flags,
+                                  args->xattr, args->xdata);
+    if (ret) {
+        op_errno = -ret;
+        goto unwind;
+    }
+    ret = client_submit_request(this, &req, frame, conf->fops,
+                                GFS3_OP_FSETXATTR, client4_0_fsetxattr_cbk,
+                                NULL, (xdrproc_t)xdr_gfx_fsetxattr_req);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_WARNING, 0, PC_MSG_FOP_SEND_FAILED, NULL);
+    }
+
+    GF_FREE(req.dict.pairs.pairs_val);
+    GF_FREE(req.xdata.pairs.pairs_val);
+
+    return 0;
+unwind:
+    CLIENT_STACK_UNWIND(fsetxattr, frame, -1, op_errno, NULL);
+    GF_FREE(req.dict.pairs.pairs_val);
+    GF_FREE(req.xdata.pairs.pairs_val);
+
+    return 0;
+}
+
+int32_t
+client4_0_fgetxattr(call_frame_t *frame, xlator_t *this, void *data)
+{
+    clnt_args_t *args = NULL;
+    clnt_conf_t *conf = NULL;
+    gfx_fgetxattr_req req = {
+        {
+            0,
+        },
+    };
+    int op_errno = ESTALE;
+    int ret = 0;
+    clnt_local_t *local = NULL;
+
+    if (!frame || !this || !data)
+        goto unwind;
+
+    args = data;
+    conf = this->private;
+
+    local = mem_get0(this->local_pool);
+    if (!local) {
+        op_errno = ENOMEM;
+        goto unwind;
+    }
+    frame->local = local;
+
+    ret = client_pre_fgetxattr_v2(this, &req, args->fd, args->name,
+                                  args->xdata);
+    if (ret) {
+        op_errno = -ret;
+        goto unwind;
+    }
+    ret = client_submit_request(this, &req, frame, conf->fops,
+                                GFS3_OP_FGETXATTR, client4_0_fgetxattr_cbk,
+                                NULL, (xdrproc_t)xdr_gfx_fgetxattr_req);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_WARNING, 0, PC_MSG_FOP_SEND_FAILED, NULL);
+    }
+
+    GF_FREE(req.xdata.pairs.pairs_val);
+
+    return 0;
+unwind:
+    CLIENT_STACK_UNWIND(fgetxattr, frame, -1, op_errno, NULL, NULL);
+
+    GF_FREE(req.xdata.pairs.pairs_val);
+
+    return 0;
+}
+
+int32_t
+client4_0_getxattr(call_frame_t *frame, xlator_t *this, void *data)
+{
+    clnt_conf_t *conf = NULL;
+    clnt_args_t *args = NULL;
+    gfx_getxattr_req req = {
+        {
+            0,
+        },
+    };
+    dict_t *dict = NULL;
+    int ret = 0;
+    int32_t op_ret = -1;
+    int op_errno = ESTALE;
+    clnt_local_t *local = NULL;
+
+    if (!frame || !this || !data) {
+        op_errno = 0;
+        goto unwind;
+    }
+    args = data;
+
+    local = mem_get0(this->local_pool);
+    if (!local) {
+        op_errno = ENOMEM;
+        goto unwind;
+    }
+
+    frame->local = local;
+
+    loc_copy(&local->loc, args->loc);
+    loc_path(&local->loc, NULL);
+
+    if (args->name)
+        local->name = gf_strdup(args->name);
+
+    conf = this->private;
+
+    if (args && args->name) {
+        if (is_client_dump_locks_cmd((char *)args->name)) {
+            dict = dict_new();
+
+            if (!dict) {
+                op_errno = ENOMEM;
+                goto unwind;
+            }
+
+            ret = client_dump_locks((char *)args->name, args->loc->inode, dict);
+            if (ret) {
+                gf_smsg(this->name, GF_LOG_WARNING, EINVAL,
+                        PC_MSG_CLIENT_DUMP_LOCKS_FAILED, NULL);
+                op_errno = ENOMEM;
+                goto unwind;
+            }
+
+            GF_ASSERT(dict);
+            op_ret = 0;
+            op_errno = 0;
+            goto unwind;
+        }
+    }
+
+    ret = client_pre_getxattr_v2(this, &req, args->loc, args->name,
+                                 args->xdata);
+    if (ret) {
+        op_errno = -ret;
+        goto unwind;
+    }
+    ret = client_submit_request(this, &req, frame, conf->fops, GFS3_OP_GETXATTR,
+                                client4_0_getxattr_cbk, NULL,
+                                (xdrproc_t)xdr_gfx_getxattr_req);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_WARNING, 0, PC_MSG_FOP_SEND_FAILED, NULL);
+    }
+
+    GF_FREE(req.xdata.pairs.pairs_val);
+
+    return 0;
+unwind:
+    CLIENT_STACK_UNWIND(getxattr, frame, op_ret, op_errno, dict, NULL);
+
+    if (dict) {
+        dict_unref(dict);
+    }
+
+    GF_FREE(req.xdata.pairs.pairs_val);
+
+    return 0;
+}
+
+int32_t
+client4_0_xattrop(call_frame_t *frame, xlator_t *this, void *data)
+{
+    clnt_conf_t *conf = NULL;
+    clnt_args_t *args = NULL;
+    gfx_xattrop_req req = {
+        {
+            0,
+        },
+    };
+    int ret = 0;
+    int op_errno = ESTALE;
+    clnt_local_t *local = NULL;
+
+    if (!frame || !this || !data)
+        goto unwind;
+
+    args = data;
+
+    if (!(args->loc && args->loc->inode))
+        goto unwind;
+
+    local = mem_get0(this->local_pool);
+    if (!local) {
+        op_errno = ENOMEM;
+        goto unwind;
+    }
+    frame->local = local;
+
+    loc_copy(&local->loc, args->loc);
+    loc_path(&local->loc, NULL);
+    conf = this->private;
+
+    ret = client_pre_xattrop_v2(this, &req, args->loc, args->xattr, args->flags,
+                                args->xdata);
+    if (ret) {
+        op_errno = -ret;
+        goto unwind;
+    }
+    ret = client_submit_request(this, &req, frame, conf->fops, GFS3_OP_XATTROP,
+                                client4_0_xattrop_cbk, NULL,
+                                (xdrproc_t)xdr_gfx_xattrop_req);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_WARNING, 0, PC_MSG_FOP_SEND_FAILED, NULL);
+    }
+
+    GF_FREE(req.dict.pairs.pairs_val);
+    GF_FREE(req.xdata.pairs.pairs_val);
+
+    return 0;
+unwind:
+    CLIENT_STACK_UNWIND(xattrop, frame, -1, op_errno, NULL, NULL);
+
+    GF_FREE(req.dict.pairs.pairs_val);
+    GF_FREE(req.xdata.pairs.pairs_val);
+
+    return 0;
+}
+
+int32_t
+client4_0_fxattrop(call_frame_t *frame, xlator_t *this, void *data)
+{
+    clnt_args_t *args = NULL;
+    clnt_conf_t *conf = NULL;
+    gfx_fxattrop_req req = {
+        {
+            0,
+        },
+    };
+    int op_errno = ESTALE;
+    int ret = 0;
+
+    if (!frame || !this || !data)
+        goto unwind;
+
+    args = data;
+    conf = this->private;
+
+    ret = client_pre_fxattrop_v2(this, &req, args->fd, args->xattr, args->flags,
+                                 args->xdata);
+    if (ret) {
+        op_errno = -ret;
+        goto unwind;
+    }
+    ret = client_fd_fop_prepare_local(frame, args->fd, req.fd);
+    if (ret) {
+        op_errno = -ret;
+        goto unwind;
+    }
+
+    ret = client_submit_request(this, &req, frame, conf->fops, GFS3_OP_FXATTROP,
+                                client4_0_fxattrop_cbk, NULL,
+                                (xdrproc_t)xdr_gfx_fxattrop_req);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_WARNING, 0, PC_MSG_FOP_SEND_FAILED, NULL);
+    }
+
+    GF_FREE(req.dict.pairs.pairs_val);
+    GF_FREE(req.xdata.pairs.pairs_val);
+
+    return 0;
+unwind:
+    CLIENT_STACK_UNWIND(fxattrop, frame, -1, op_errno, NULL, NULL);
+
+    GF_FREE(req.dict.pairs.pairs_val);
+    GF_FREE(req.xdata.pairs.pairs_val);
+
+    return 0;
+}
+
+int32_t
+client4_0_removexattr(call_frame_t *frame, xlator_t *this, void *data)
+{
+    clnt_conf_t *conf = NULL;
+    clnt_args_t *args = NULL;
+    gfx_removexattr_req req = {
+        {
+            0,
+        },
+    };
+    int ret = 0;
+    int op_errno = ESTALE;
+
+    if (!frame || !this || !data)
+        goto unwind;
+
+    args = data;
+    conf = this->private;
+
+    ret = client_pre_removexattr_v2(this, &req, args->loc, args->name,
+                                    args->xdata);
+    if (ret) {
+        op_errno = -ret;
+        goto unwind;
+    }
+    ret = client_submit_request(this, &req, frame, conf->fops,
+                                GFS3_OP_REMOVEXATTR, client4_0_removexattr_cbk,
+                                NULL, (xdrproc_t)xdr_gfx_removexattr_req);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_WARNING, 0, PC_MSG_FOP_SEND_FAILED, NULL);
+    }
+
+    GF_FREE(req.xdata.pairs.pairs_val);
+
+    return 0;
+unwind:
+    CLIENT_STACK_UNWIND(removexattr, frame, -1, op_errno, NULL);
+    GF_FREE(req.xdata.pairs.pairs_val);
+
+    return 0;
+}
+
+int32_t
+client4_0_fremovexattr(call_frame_t *frame, xlator_t *this, void *data)
+{
+    clnt_conf_t *conf = NULL;
+    clnt_args_t *args = NULL;
+    gfx_fremovexattr_req req = {
+        {
+            0,
+        },
+    };
+    int ret = 0;
+    int op_errno = ESTALE;
+
+    if (!frame || !this || !data)
+        goto unwind;
+
+    args = data;
+
+    conf = this->private;
+
+    ret = client_pre_fremovexattr_v2(this, &req, args->fd, args->name,
+                                     args->xdata);
+    if (ret) {
+        op_errno = -ret;
+        goto unwind;
+    }
+    ret = client_submit_request(
+        this, &req, frame, conf->fops, GFS3_OP_FREMOVEXATTR,
+        client4_0_fremovexattr_cbk, NULL, (xdrproc_t)xdr_gfx_fremovexattr_req);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_WARNING, 0, PC_MSG_FOP_SEND_FAILED, NULL);
+    }
+
+    GF_FREE(req.xdata.pairs.pairs_val);
+
+    return 0;
+unwind:
+    CLIENT_STACK_UNWIND(fremovexattr, frame, -1, op_errno, NULL);
+    GF_FREE(req.xdata.pairs.pairs_val);
+
+    return 0;
+}
+
+int32_t
+client4_0_lease(call_frame_t *frame, xlator_t *this, void *data)
+{
+    clnt_args_t *args = NULL;
+    gfx_lease_req req = {
+        {
+            0,
+        },
+    };
+    clnt_conf_t *conf = NULL;
+    int op_errno = ESTALE;
+    int ret = 0;
+
+    GF_VALIDATE_OR_GOTO("client", this, unwind);
+    GF_VALIDATE_OR_GOTO(this->name, frame, unwind);
+    GF_VALIDATE_OR_GOTO(this->name, data, unwind);
+
+    args = data;
+    conf = this->private;
+
+    ret = client_pre_lease_v2(this, &req, args->loc, args->lease, args->xdata);
+    if (ret < 0) {
+        op_errno = -ret;
+        goto unwind;
+    }
+    ret = client_submit_request(this, &req, frame, conf->fops, GFS3_OP_LEASE,
+                                client4_0_lease_cbk, NULL,
+                                (xdrproc_t)xdr_gfx_lease_req);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_WARNING, 0, PC_MSG_FOP_SEND_FAILED, NULL);
+    }
+
+    GF_FREE(req.xdata.pairs.pairs_val);
+
+    return 0;
+unwind:
+    CLIENT_STACK_UNWIND(lease, frame, -1, op_errno, NULL, NULL);
+    GF_FREE(req.xdata.pairs.pairs_val);
+
+    return 0;
+}
+
+int32_t
+client4_0_lk(call_frame_t *frame, xlator_t *this, void *data)
+{
+    clnt_args_t *args = NULL;
+    gfx_lk_req req = {
+        {
+            0,
+        },
+    };
+    int32_t gf_cmd = 0;
+    clnt_local_t *local = NULL;
+    clnt_conf_t *conf = NULL;
+    int op_errno = ESTALE;
+    int ret = 0;
+
+    if (!frame || !this || !data)
+        goto unwind;
+
+    args = data;
+    conf = this->private;
+    local = mem_get0(this->local_pool);
+    if (!local) {
+        op_errno = ENOMEM;
+        goto unwind;
+    }
+    frame->local = local;
+
+    ret = client_cmd_to_gf_cmd(args->cmd, &gf_cmd);
+    if (ret) {
+        op_errno = EINVAL;
+        gf_smsg(this->name, GF_LOG_WARNING, EINVAL, PC_MSG_UNKNOWN_CMD,
+                "gf_cmd=%d", gf_cmd, NULL);
+        goto unwind;
+    }
+
+    local->owner = frame->root->lk_owner;
+    local->cmd = args->cmd;
+    local->fd = fd_ref(args->fd);
+
+    ret = client_pre_lk_v2(this, &req, args->cmd, args->flock, args->fd,
+                           args->xdata);
+    if (ret) {
+        op_errno = -ret;
+
+        if ((op_errno == EBADF) && (args->flock->l_type == F_UNLCK) &&
+            client_is_setlk(local->cmd)) {
+            client_add_lock_for_recovery(local->fd, args->flock, &local->owner,
+                                         local->cmd);
+        }
+
+        goto unwind;
+    }
+
+    ret = client_submit_request(this, &req, frame, conf->fops, GFS3_OP_LK,
+                                client4_0_lk_cbk, NULL,
+                                (xdrproc_t)xdr_gfx_lk_req);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_WARNING, 0, PC_MSG_FOP_SEND_FAILED, NULL);
+    }
+
+    GF_FREE(req.xdata.pairs.pairs_val);
+
+    return 0;
+unwind:
+    CLIENT_STACK_UNWIND(lk, frame, -1, op_errno, NULL, NULL);
+    GF_FREE(req.xdata.pairs.pairs_val);
+
+    return 0;
+}
+
+int32_t
+client4_0_inodelk(call_frame_t *frame, xlator_t *this, void *data)
+{
+    clnt_conf_t *conf = NULL;
+    clnt_args_t *args = NULL;
+    gfx_inodelk_req req = {
+        {
+            0,
+        },
+    };
+    int ret = 0;
+    int op_errno = ESTALE;
+
+    if (!frame || !this || !data)
+        goto unwind;
+
+    args = data;
+    conf = this->private;
+
+    ret = client_pre_inodelk_v2(this, &req, args->loc, args->cmd, args->flock,
+                                args->volume, args->xdata);
+    if (ret) {
+        op_errno = -ret;
+        goto unwind;
+    }
+    ret = client_submit_request(this, &req, frame, conf->fops, GFS3_OP_INODELK,
+                                client4_0_inodelk_cbk, NULL,
+                                (xdrproc_t)xdr_gfx_inodelk_req);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_WARNING, 0, PC_MSG_FOP_SEND_FAILED, NULL);
+    }
+
+    GF_FREE(req.xdata.pairs.pairs_val);
+
+    return 0;
+unwind:
+    CLIENT_STACK_UNWIND(inodelk, frame, -1, op_errno, NULL);
+    GF_FREE(req.xdata.pairs.pairs_val);
+
+    return 0;
+}
+
+int32_t
+client4_0_finodelk(call_frame_t *frame, xlator_t *this, void *data)
+{
+    clnt_args_t *args = NULL;
+    gfx_finodelk_req req = {
+        {
+            0,
+        },
+    };
+    clnt_conf_t *conf = NULL;
+    int op_errno = ESTALE;
+    int ret = 0;
+
+    if (!frame || !this || !data)
+        goto unwind;
+
+    args = data;
+    conf = this->private;
+
+    ret = client_pre_finodelk_v2(this, &req, args->fd, args->cmd, args->flock,
+                                 args->volume, args->xdata);
+    if (ret) {
+        op_errno = -ret;
+        goto unwind;
+    }
+
+    ret = client_fd_fop_prepare_local(frame, args->fd, req.fd);
+    if (ret) {
+        op_errno = -ret;
+        goto unwind;
+    }
+
+    ret = client_submit_request(this, &req, frame, conf->fops, GFS3_OP_FINODELK,
+                                client4_0_finodelk_cbk, NULL,
+                                (xdrproc_t)xdr_gfx_finodelk_req);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_WARNING, 0, PC_MSG_FOP_SEND_FAILED, NULL);
+    }
+
+    GF_FREE(req.xdata.pairs.pairs_val);
+    return 0;
+unwind:
+    CLIENT_STACK_UNWIND(finodelk, frame, -1, op_errno, NULL);
+    GF_FREE(req.xdata.pairs.pairs_val);
+
+    return 0;
+}
+
+int32_t
+client4_0_entrylk(call_frame_t *frame, xlator_t *this, void *data)
+{
+    clnt_conf_t *conf = NULL;
+    clnt_args_t *args = NULL;
+    gfx_entrylk_req req = {
+        {
+            0,
+        },
+    };
+    int ret = 0;
+    int op_errno = ESTALE;
+
+    if (!frame || !this || !data)
+        goto unwind;
+
+    args = data;
+
+    conf = this->private;
+
+    ret = client_pre_entrylk_v2(this, &req, args->loc, args->cmd_entrylk,
+                                args->type, args->volume, args->basename,
+                                args->xdata);
+    if (ret) {
+        op_errno = -ret;
+        goto unwind;
+    }
+
+    ret = client_submit_request(this, &req, frame, conf->fops, GFS3_OP_ENTRYLK,
+                                client4_0_entrylk_cbk, NULL,
+                                (xdrproc_t)xdr_gfx_entrylk_req);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_WARNING, 0, PC_MSG_FOP_SEND_FAILED, NULL);
+    }
+
+    GF_FREE(req.xdata.pairs.pairs_val);
+
+    return 0;
+unwind:
+    CLIENT_STACK_UNWIND(entrylk, frame, -1, op_errno, NULL);
+    GF_FREE(req.xdata.pairs.pairs_val);
+
+    return 0;
+}
+
+int32_t
+client4_0_fentrylk(call_frame_t *frame, xlator_t *this, void *data)
+{
+    clnt_args_t *args = NULL;
+    gfx_fentrylk_req req = {
+        {
+            0,
+        },
+    };
+    clnt_conf_t *conf = NULL;
+    int op_errno = ESTALE;
+    int ret = 0;
+
+    if (!frame || !this || !data)
+        goto unwind;
+
+    args = data;
+    conf = this->private;
+
+    ret = client_pre_fentrylk_v2(this, &req, args->fd, args->cmd_entrylk,
+                                 args->type, args->volume, args->basename,
+                                 args->xdata);
+    if (ret) {
+        op_errno = -ret;
+        goto unwind;
+    }
+    ret = client_submit_request(this, &req, frame, conf->fops, GFS3_OP_FENTRYLK,
+                                client4_0_fentrylk_cbk, NULL,
+                                (xdrproc_t)xdr_gfx_fentrylk_req);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_WARNING, 0, PC_MSG_FOP_SEND_FAILED, NULL);
+    }
+
+    GF_FREE(req.xdata.pairs.pairs_val);
+
+    return 0;
+unwind:
+    CLIENT_STACK_UNWIND(fentrylk, frame, -1, op_errno, NULL);
+    GF_FREE(req.xdata.pairs.pairs_val);
+
+    return 0;
+}
+
+int32_t
+client4_0_readdir(call_frame_t *frame, xlator_t *this, void *data)
+{
+    clnt_args_t *args = NULL;
+    int64_t remote_fd = -1;
+    clnt_conf_t *conf = NULL;
+    gfx_readdir_req req = {
+        {
+            0,
+        },
+    };
+    gfx_readdir_rsp rsp = {
+        0,
+    };
+    clnt_local_t *local = NULL;
+    int op_errno = ESTALE;
+    int ret = 0;
+    int count = 0;
+    struct iobref *rsp_iobref = NULL;
+    struct iobuf *rsp_iobuf = NULL;
+    struct iovec *rsphdr = NULL;
+    struct iovec vector[MAX_IOVEC] = {
+        {0},
+    };
+    int readdir_rsp_size = 0;
+    client_payload_t cp;
+
+    if (!frame || !this || !data)
+        goto unwind;
+
+    args = data;
+    conf = this->private;
+
+    readdir_rsp_size = xdr_sizeof((xdrproc_t)xdr_gfx_readdir_rsp, &rsp) +
+                       args->size;
+
+    local = mem_get0(this->local_pool);
+    if (!local) {
+        op_errno = ENOMEM;
+        goto unwind;
+    }
+    frame->local = local;
+
+    local->cmd = remote_fd;
+
+    if ((readdir_rsp_size + GLUSTERFS_RPC_REPLY_SIZE +
+         GLUSTERFS_RDMA_MAX_HEADER_SIZE) > (GLUSTERFS_RDMA_INLINE_THRESHOLD)) {
+        rsp_iobref = iobref_new();
+        if (rsp_iobref == NULL) {
+            goto unwind;
+        }
+
+        /* TODO: what is the size we should send ? */
+        /* This iobuf will live for only receiving the response,
+           so not harmful */
+        rsp_iobuf = iobuf_get(this->ctx->iobuf_pool);
+        if (rsp_iobuf == NULL) {
+            goto unwind;
+        }
+
+        iobref_add(rsp_iobref, rsp_iobuf);
+
+        rsphdr = &vector[0];
+        rsphdr->iov_base = iobuf_ptr(rsp_iobuf);
+        rsphdr->iov_len = iobuf_pagesize(rsp_iobuf);
+        count = 1;
+        local->iobref = rsp_iobref;
+        iobuf_unref(rsp_iobuf);
+        rsp_iobuf = NULL;
+        rsp_iobref = NULL;
+    }
+
+    ret = client_pre_readdir_v2(this, &req, args->fd, args->size, args->offset,
+                                args->xdata);
+    if (ret) {
+        op_errno = -ret;
+        goto unwind;
+    }
+
+    memset(&cp, 0, sizeof(client_payload_t));
+
+    cp.rsphdr = rsphdr;
+    cp.rsphdr_cnt = count;
+    cp.rsp_iobref = rsp_iobref;
+    ret = client_submit_request(this, &req, frame, conf->fops, GFS3_OP_READDIR,
+                                client4_0_readdir_cbk, &cp,
+                                (xdrproc_t)xdr_gfx_readdir_req);
+
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_WARNING, 0, PC_MSG_FOP_SEND_FAILED, NULL);
+    }
+
+    GF_FREE(req.xdata.pairs.pairs_val);
+
+    return 0;
+
+unwind:
+    if (rsp_iobref)
+        iobref_unref(rsp_iobref);
+
+    CLIENT_STACK_UNWIND(readdir, frame, -1, op_errno, NULL, NULL);
+    GF_FREE(req.xdata.pairs.pairs_val);
+
+    return 0;
+}
+
+int32_t
+client4_0_readdirp(call_frame_t *frame, xlator_t *this, void *data)
+{
+    clnt_args_t *args = NULL;
+    gfx_readdirp_req req = {
+        {
+            0,
+        },
+    };
+    gfx_readdirp_rsp rsp = {
+        0,
+    };
+    clnt_conf_t *conf = NULL;
+    int op_errno = ESTALE;
+    int ret = 0;
+    int count = 0;
+    int readdirp_rsp_size = 0;
+    struct iobref *rsp_iobref = NULL;
+    struct iobuf *rsp_iobuf = NULL;
+    struct iovec *rsphdr = NULL;
+    struct iovec vector[MAX_IOVEC] = {
+        {0},
+    };
+    clnt_local_t *local = NULL;
+    client_payload_t cp;
+
+    if (!frame || !this || !data)
+        goto unwind;
+
+    args = data;
+    conf = this->private;
+
+    local = mem_get0(this->local_pool);
+    if (!local) {
+        op_errno = ENOMEM;
+        goto unwind;
+    }
+    frame->local = local;
+
+    ret = client_pre_readdirp_v2(this, &req, args->fd, args->size, args->offset,
+                                 args->xdata);
+
+    if (ret) {
+        op_errno = -ret;
+        goto unwind;
+    }
+
+    readdirp_rsp_size = xdr_sizeof((xdrproc_t)xdr_gfx_readdirp_rsp, &rsp) +
+                        args->size;
+
+    if ((readdirp_rsp_size + GLUSTERFS_RPC_REPLY_SIZE +
+         GLUSTERFS_RDMA_MAX_HEADER_SIZE) > (GLUSTERFS_RDMA_INLINE_THRESHOLD)) {
+        rsp_iobref = iobref_new();
+        if (rsp_iobref == NULL) {
+            goto unwind;
+        }
+
+        /* TODO: what is the size we should send ? */
+        /* This iobuf will live for only receiving the response,
+           so not harmful */
+        rsp_iobuf = iobuf_get(this->ctx->iobuf_pool);
+        if (rsp_iobuf == NULL) {
+            goto unwind;
+        }
+
+        rsphdr = &vector[0];
+        rsphdr->iov_base = iobuf_ptr(rsp_iobuf);
+        rsphdr->iov_len = iobuf_pagesize(rsp_iobuf);
+        count = 1;
+        local->iobref = rsp_iobref;
+        iobref_add(rsp_iobref, rsp_iobuf);
+        iobuf_unref(rsp_iobuf);
+        rsp_iobuf = NULL;
+        rsp_iobref = NULL;
+    }
+
+    local->fd = fd_ref(args->fd);
+
+    memset(&cp, 0, sizeof(client_payload_t));
+
+    cp.rsphdr = rsphdr;
+    cp.rsphdr_cnt = count;
+    cp.rsp_iobref = rsp_iobref;
+    ret = client_submit_request(this, &req, frame, conf->fops, GFS3_OP_READDIRP,
+                                client4_0_readdirp_cbk, &cp,
+                                (xdrproc_t)xdr_gfx_readdirp_req);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_WARNING, 0, PC_MSG_FOP_SEND_FAILED, NULL);
+    }
+
+    GF_FREE(req.xdata.pairs.pairs_val);
+
+    return 0;
+unwind:
+    if (rsp_iobref)
+        iobref_unref(rsp_iobref);
+
+    GF_FREE(req.xdata.pairs.pairs_val);
+
+    CLIENT_STACK_UNWIND(readdirp, frame, -1, op_errno, NULL, NULL);
+    return 0;
+}
+
+int32_t
+client4_0_setattr(call_frame_t *frame, xlator_t *this, void *data)
+{
+    clnt_conf_t *conf = NULL;
+    clnt_args_t *args = NULL;
+    gfx_setattr_req req = {
+        {
+            0,
+        },
+    };
+    int ret = 0;
+    int op_errno = ESTALE;
+
+    if (!frame || !this || !data)
+        goto unwind;
+
+    args = data;
+    conf = this->private;
+
+    ret = client_pre_setattr_v2(this, &req, args->loc, args->valid, args->stbuf,
+                                args->xdata);
+
+    if (ret) {
+        op_errno = -ret;
+        goto unwind;
+    }
+
+    ret = client_submit_request(this, &req, frame, conf->fops, GFS3_OP_SETATTR,
+                                client4_0_setattr_cbk, NULL,
+                                (xdrproc_t)xdr_gfx_setattr_req);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_WARNING, 0, PC_MSG_FOP_SEND_FAILED, NULL);
+    }
+
+    GF_FREE(req.xdata.pairs.pairs_val);
+
+    return 0;
+unwind:
+    CLIENT_STACK_UNWIND(setattr, frame, -1, op_errno, NULL, NULL, NULL);
+    GF_FREE(req.xdata.pairs.pairs_val);
+
+    return 0;
+}
+
+int32_t
+client4_0_fallocate(call_frame_t *frame, xlator_t *this, void *data)
+{
+    clnt_args_t *args = NULL;
+    clnt_conf_t *conf = NULL;
+    gfx_fallocate_req req = {
+        {0},
+    };
+    int op_errno = ESTALE;
+    int ret = 0;
+
+    if (!frame || !this || !data)
+        goto unwind;
+
+    args = data;
+    conf = this->private;
+
+    ret = client_pre_fallocate_v2(this, &req, args->fd, args->flags,
+                                  args->offset, args->size, args->xdata);
+    if (ret) {
+        op_errno = -ret;
+        goto unwind;
+    }
+
+    ret = client_submit_request(this, &req, frame, conf->fops,
+                                GFS3_OP_FALLOCATE, client4_0_fallocate_cbk,
+                                NULL, (xdrproc_t)xdr_gfx_fallocate_req);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_WARNING, 0, PC_MSG_FOP_SEND_FAILED, NULL);
+    }
+
+    GF_FREE(req.xdata.pairs.pairs_val);
+
+    return 0;
+unwind:
+    CLIENT_STACK_UNWIND(fallocate, frame, -1, op_errno, NULL, NULL, NULL);
+    GF_FREE(req.xdata.pairs.pairs_val);
+
+    return 0;
+}
+
+int32_t
+client4_0_discard(call_frame_t *frame, xlator_t *this, void *data)
+{
+    clnt_args_t *args = NULL;
+    clnt_conf_t *conf = NULL;
+    gfx_discard_req req = {
+        {0},
+    };
+    int op_errno = ESTALE;
+    int ret = 0;
+
+    if (!frame || !this || !data)
+        goto unwind;
+
+    args = data;
+    conf = this->private;
+
+    ret = client_pre_discard_v2(this, &req, args->fd, args->offset, args->size,
+                                args->xdata);
+    if (ret) {
+        op_errno = -ret;
+        goto unwind;
+    }
+
+    ret = client_submit_request(this, &req, frame, conf->fops, GFS3_OP_DISCARD,
+                                client4_0_discard_cbk, NULL,
+                                (xdrproc_t)xdr_gfx_discard_req);
+    if (ret)
+        gf_smsg(this->name, GF_LOG_WARNING, 0, PC_MSG_FOP_SEND_FAILED, NULL);
+
+    GF_FREE(req.xdata.pairs.pairs_val);
+
+    return 0;
+unwind:
+    CLIENT_STACK_UNWIND(discard, frame, -1, op_errno, NULL, NULL, NULL);
+    GF_FREE(req.xdata.pairs.pairs_val);
+
+    return 0;
+}
+
+int32_t
+client4_0_zerofill(call_frame_t *frame, xlator_t *this, void *data)
+{
+    clnt_args_t *args = NULL;
+    clnt_conf_t *conf = NULL;
+    gfx_zerofill_req req = {
+        {0},
+    };
+    int op_errno = ESTALE;
+    int ret = 0;
+
+    GF_ASSERT(frame);
+
+    if (!this || !data)
+        goto unwind;
+
+    args = data;
+    conf = this->private;
+
+    ret = client_pre_zerofill_v2(this, &req, args->fd, args->offset, args->size,
+                                 args->xdata);
+    if (ret) {
+        op_errno = -ret;
+        goto unwind;
+    }
+
+    ret = client_submit_request(this, &req, frame, conf->fops, GFS3_OP_ZEROFILL,
+                                client4_0_zerofill_cbk, NULL,
+                                (xdrproc_t)xdr_gfx_zerofill_req);
+    if (ret)
+        gf_smsg(this->name, GF_LOG_WARNING, 0, PC_MSG_FOP_SEND_FAILED, NULL);
+
+    GF_FREE(req.xdata.pairs.pairs_val);
+
+    return 0;
+unwind:
+    CLIENT_STACK_UNWIND(zerofill, frame, -1, op_errno, NULL, NULL, NULL);
+    GF_FREE(req.xdata.pairs.pairs_val);
+
+    return 0;
+}
+
+int32_t
+client4_0_ipc(call_frame_t *frame, xlator_t *this, void *data)
+{
+    clnt_args_t *args = NULL;
+    clnt_conf_t *conf = NULL;
+    gfx_ipc_req req = {
+        0,
+    };
+    int op_errno = ESTALE;
+    int ret = 0;
+
+    GF_ASSERT(frame);
+
+    if (!this || !data)
+        goto unwind;
+
+    args = data;
+    conf = this->private;
+
+    ret = client_pre_ipc_v2(this, &req, args->cmd, args->xdata);
+
+    if (ret) {
+        op_errno = -ret;
+        goto unwind;
+    }
+
+    ret = client_submit_request(this, &req, frame, conf->fops, GFS3_OP_IPC,
+                                client4_0_ipc_cbk, NULL,
+                                (xdrproc_t)xdr_gfx_ipc_req);
+    if (ret)
+        gf_smsg(this->name, GF_LOG_WARNING, 0, PC_MSG_FOP_SEND_FAILED, NULL);
+
+    GF_FREE(req.xdata.pairs.pairs_val);
+
+    return 0;
+unwind:
+    CLIENT_STACK_UNWIND(ipc, frame, -1, op_errno, NULL);
+    GF_FREE(req.xdata.pairs.pairs_val);
+
+    return 0;
+}
+
+int32_t
+client4_0_seek(call_frame_t *frame, xlator_t *this, void *data)
+{
+    clnt_args_t *args = NULL;
+    clnt_conf_t *conf = NULL;
+    struct gfx_seek_req req = {
+        {
+            0,
+        },
+    };
+    int op_errno = ESTALE;
+    int ret = 0;
+
+    GF_ASSERT(frame);
+
+    if (!this || !data)
+        goto unwind;
+
+    args = data;
+    conf = this->private;
+
+    ret = client_pre_seek_v2(this, &req, args->fd, args->offset, args->what,
+                             args->xdata);
+    if (ret) {
+        op_errno = -ret;
+        goto unwind;
+    }
+
+    ret = client_submit_request(this, &req, frame, conf->fops, GFS3_OP_SEEK,
+                                client4_0_seek_cbk, NULL,
+                                (xdrproc_t)xdr_gfx_seek_req);
+    if (ret)
+        gf_smsg(this->name, GF_LOG_WARNING, 0, PC_MSG_FOP_SEND_FAILED, NULL);
+
+    GF_FREE(req.xdata.pairs.pairs_val);
+
+    return 0;
+unwind:
+    CLIENT_STACK_UNWIND(ipc, frame, -1, op_errno, NULL);
+    GF_FREE(req.xdata.pairs.pairs_val);
+
+    return 0;
+}
+
+int32_t
+client4_0_getactivelk(call_frame_t *frame, xlator_t *this, void *data)
+{
+    clnt_conf_t *conf = NULL;
+    clnt_args_t *args = NULL;
+    gfx_getactivelk_req req = {
+        {
+            0,
+        },
+    };
+    int ret = 0;
+    int op_errno = ESTALE;
+
+    if (!frame || !this || !data)
+        goto unwind;
+
+    args = data;
+    if (!(args->loc && args->loc->inode))
+        goto unwind;
+
+    if (!gf_uuid_is_null(args->loc->inode->gfid))
+        memcpy(req.gfid, args->loc->inode->gfid, 16);
+    else
+        memcpy(req.gfid, args->loc->gfid, 16);
+
+    GF_ASSERT_AND_GOTO_WITH_ERROR(this->name,
+                                  !gf_uuid_is_null(*((uuid_t *)req.gfid)),
+                                  unwind, op_errno, EINVAL);
+    conf = this->private;
+
+    dict_to_xdr(args->xdata, &req.xdata);
+
+    ret = client_submit_request(this, &req, frame, conf->fops,
+                                GFS3_OP_GETACTIVELK, client4_0_getactivelk_cbk,
+                                NULL, (xdrproc_t)xdr_gfx_getactivelk_req);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_WARNING, 0, PC_MSG_FOP_SEND_FAILED, NULL);
+    }
+
+    GF_FREE(req.xdata.pairs.pairs_val);
+
+    return 0;
+unwind:
+    CLIENT_STACK_UNWIND(getactivelk, frame, -1, op_errno, NULL, NULL);
+
+    GF_FREE(req.xdata.pairs.pairs_val);
+
+    return 0;
+}
+
+int32_t
+client4_0_setactivelk(call_frame_t *frame, xlator_t *this, void *data)
+{
+    clnt_conf_t *conf = NULL;
+    clnt_args_t *args = NULL;
+    gfx_setactivelk_req req = {
+        {
+            0,
+        },
+    };
+    int ret = 0;
+    int op_errno = ESTALE;
+
+    if (!frame || !this || !data)
+        goto unwind;
+
+    args = data;
+    if (!(args->loc && args->loc->inode && args->locklist))
+        goto unwind;
+
+    if (!gf_uuid_is_null(args->loc->inode->gfid))
+        memcpy(req.gfid, args->loc->inode->gfid, 16);
+    else
+        memcpy(req.gfid, args->loc->gfid, 16);
+
+    GF_ASSERT_AND_GOTO_WITH_ERROR(this->name,
+                                  !gf_uuid_is_null(*((uuid_t *)req.gfid)),
+                                  unwind, op_errno, EINVAL);
+    conf = this->private;
+
+    dict_to_xdr(args->xdata, &req.xdata);
+    ret = serialize_req_locklist_v2(args->locklist, &req);
+
+    if (ret)
+        goto unwind;
+
+    ret = client_submit_request(this, &req, frame, conf->fops,
+                                GFS3_OP_SETACTIVELK, client4_0_setactivelk_cbk,
+                                NULL, (xdrproc_t)xdr_gfx_setactivelk_req);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_WARNING, 0, PC_MSG_FOP_SEND_FAILED, NULL);
+    }
+
+    clnt_setactivelk_req_cleanup_v2(&req);
+
+    GF_FREE(req.xdata.pairs.pairs_val);
+
+    return 0;
+
+unwind:
+
+    CLIENT_STACK_UNWIND(setactivelk, frame, -1, op_errno, NULL);
+
+    GF_FREE(req.xdata.pairs.pairs_val);
+
+    clnt_setactivelk_req_cleanup_v2(&req);
+
+    return 0;
+}
+
+int
+client4_rchecksum_cbk(struct rpc_req *req, struct iovec *iov, int count,
+                      void *myframe)
+{
+    call_frame_t *frame = NULL;
+    gfx_rchecksum_rsp rsp = {
+        0,
+    };
+    int ret = 0;
+    xlator_t *this = NULL;
+    dict_t *xdata = NULL;
+
+    this = THIS;
+
+    frame = myframe;
+
+    if (-1 == req->rpc_status) {
+        rsp.op_ret = -1;
+        rsp.op_errno = ENOTCONN;
+        goto out;
+    }
+
+    ret = xdr_to_generic(*iov, &rsp, (xdrproc_t)xdr_gfx_rchecksum_rsp);
+    if (ret < 0) {
+        gf_smsg(this->name, GF_LOG_ERROR, EINVAL, PC_MSG_XDR_DECODING_FAILED,
+                NULL);
+        rsp.op_ret = -1;
+        rsp.op_errno = EINVAL;
+        goto out;
+    }
+
+    xdr_to_dict(&rsp.xdata, &xdata);
+out:
+    if (rsp.op_ret == -1) {
+        gf_smsg(this->name, GF_LOG_WARNING, gf_error_to_errno(rsp.op_errno),
+                PC_MSG_REMOTE_OP_FAILED, NULL);
+    }
+
+    CLIENT_STACK_UNWIND(rchecksum, frame, rsp.op_ret,
+                        gf_error_to_errno(rsp.op_errno), rsp.weak_checksum,
+                        (uint8_t *)rsp.strong_checksum.strong_checksum_val,
+                        xdata);
+
+    if (rsp.strong_checksum.strong_checksum_val) {
+        /* This is allocated by the libc while decoding RPC msg */
+        /* Hence no 'GF_FREE', but just 'free' */
+        free(rsp.strong_checksum.strong_checksum_val);
+    }
+
+    if (xdata)
+        dict_unref(xdata);
+
+    return 0;
+}
+
+int32_t
+client4_namelink_cbk(struct rpc_req *req, struct iovec *iov, int count,
+                     void *myframe)
+{
+    int32_t ret = 0;
+    struct iatt prebuf = {
+        0,
+    };
+    struct iatt postbuf = {
+        0,
+    };
+    dict_t *xdata = NULL;
+    call_frame_t *frame = NULL;
+    gfx_common_2iatt_rsp rsp = {
+        0,
+    };
+
+    frame = myframe;
+
+    if (req->rpc_status == -1) {
+        rsp.op_ret = -1;
+        rsp.op_errno = ENOTCONN;
+        goto out;
+    }
+
+    ret = xdr_to_generic(*iov, &rsp, (xdrproc_t)xdr_gfx_common_2iatt_rsp);
+    if (ret < 0) {
+        rsp.op_ret = -1;
+        rsp.op_errno = EINVAL;
+        goto out;
+    }
+
+    if (rsp.op_ret != -1) {
+        gfx_stat_to_iattx(&rsp.prestat, &prebuf);
+        gfx_stat_to_iattx(&rsp.poststat, &postbuf);
+    }
+
+    xdr_to_dict(&rsp.xdata, &xdata);
+out:
+    CLIENT_STACK_UNWIND(namelink, frame, rsp.op_ret,
+                        gf_error_to_errno(rsp.op_errno), &prebuf, &postbuf,
+                        xdata);
+    if (xdata)
+        dict_unref(xdata);
+    return 0;
+}
+
+int32_t
+client4_icreate_cbk(struct rpc_req *req, struct iovec *iov, int count,
+                    void *myframe)
+{
+    int32_t ret = 0;
+    inode_t *inode = NULL;
+    clnt_local_t *local = NULL;
+    struct iatt stbuf = {
+        0,
+    };
+    dict_t *xdata = NULL;
+    call_frame_t *frame = NULL;
+    gfx_common_iatt_rsp rsp = {
+        0,
+    };
+
+    frame = myframe;
+    local = frame->local;
+
+    inode = local->loc.inode;
+
+    if (req->rpc_status == -1) {
+        rsp.op_ret = -1;
+        rsp.op_errno = ENOTCONN;
+        goto out;
+    }
+
+    ret = xdr_to_generic(*iov, &rsp, (xdrproc_t)xdr_gfx_common_iatt_rsp);
+    if (ret < 0) {
+        rsp.op_ret = -1;
+        rsp.op_errno = EINVAL;
+        goto out;
+    }
+
+    if (rsp.op_ret != -1)
+        gfx_stat_to_iattx(&rsp.stat, &stbuf);
+
+    xdr_to_dict(&rsp.xdata, &xdata);
+out:
+    CLIENT_STACK_UNWIND(icreate, frame, rsp.op_ret,
+                        gf_error_to_errno(rsp.op_errno), inode, &stbuf, xdata);
+    if (xdata)
+        dict_unref(xdata);
+    return 0;
+}
+
+int
+client4_0_put_cbk(struct rpc_req *req, struct iovec *iov, int count,
+                  void *myframe)
+{
+    gfx_common_3iatt_rsp rsp = {
+        0,
+    };
+    call_frame_t *frame = NULL;
+    int ret = 0;
+    xlator_t *this = NULL;
+    dict_t *xdata = NULL;
+    clnt_local_t *local = NULL;
+    struct iatt stbuf = {
+        0,
+    };
+    struct iatt preparent = {
+        0,
+    };
+    struct iatt postparent = {
+        0,
+    };
+    inode_t *inode = NULL;
+
+    this = THIS;
+
+    frame = myframe;
+    local = frame->local;
+    inode = local->loc.inode;
+
+    if (-1 == req->rpc_status) {
+        rsp.op_ret = -1;
+        rsp.op_errno = ENOTCONN;
+        goto out;
+    }
+
+    ret = xdr_to_generic(*iov, &rsp, (xdrproc_t)xdr_gfx_common_3iatt_rsp);
+    if (ret < 0) {
+        gf_smsg(this->name, GF_LOG_ERROR, EINVAL, PC_MSG_XDR_DECODING_FAILED,
+                NULL);
+        rsp.op_ret = -1;
+        rsp.op_errno = EINVAL;
+        goto out;
+    }
+
+    if (-1 != rsp.op_ret) {
+        ret = client_post_common_3iatt(this, &rsp, &stbuf, &preparent,
+                                       &postparent, &xdata);
+        if (ret < 0)
+            goto out;
+    }
+out:
+    if (rsp.op_ret == -1) {
+        gf_smsg(this->name, GF_LOG_WARNING, gf_error_to_errno(rsp.op_errno),
+                PC_MSG_REMOTE_OP_FAILED, NULL);
+    }
+
+    CLIENT_STACK_UNWIND(put, frame, rsp.op_ret, gf_error_to_errno(rsp.op_errno),
+                        inode, &stbuf, &preparent, &postparent, xdata);
+
+    if (xdata)
+        dict_unref(xdata);
+
+    return 0;
+}
+
+int32_t
+client4_0_namelink(call_frame_t *frame, xlator_t *this, void *data)
+{
+    int32_t ret = 0;
+    int32_t op_errno = EINVAL;
+    clnt_conf_t *conf = NULL;
+    clnt_args_t *args = NULL;
+    gfx_namelink_req req = {
+        {
+            0,
+        },
+    };
+
+    GF_ASSERT(frame);
+
+    args = data;
+    conf = this->private;
+
+    if (!(args->loc && args->loc->parent))
+        goto unwind;
+
+    if (!gf_uuid_is_null(args->loc->parent->gfid))
+        memcpy(req.pargfid, args->loc->parent->gfid, sizeof(uuid_t));
+    else
+        memcpy(req.pargfid, args->loc->pargfid, sizeof(uuid_t));
+
+    GF_ASSERT_AND_GOTO_WITH_ERROR(this->name,
+                                  !gf_uuid_is_null(*((uuid_t *)req.pargfid)),
+                                  unwind, op_errno, EINVAL);
+
+    req.bname = (char *)args->loc->name;
+
+    dict_to_xdr(args->xdata, &req.xdata);
+    ret = client_submit_request(this, &req, frame, conf->fops, GFS3_OP_NAMELINK,
+                                client4_namelink_cbk, NULL,
+                                (xdrproc_t)xdr_gfx_namelink_req);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_WARNING, 0, PC_MSG_FOP_SEND_FAILED, NULL);
+    }
+
+    GF_FREE(req.xdata.pairs.pairs_val);
+    return 0;
+
+unwind:
+    CLIENT_STACK_UNWIND(namelink, frame, -1, op_errno, NULL, NULL, NULL);
+    return 0;
+}
+
+int32_t
+client4_0_icreate(call_frame_t *frame, xlator_t *this, void *data)
+{
+    int32_t ret = 0;
+    int32_t op_errno = EINVAL;
+    clnt_conf_t *conf = NULL;
+    clnt_args_t *args = NULL;
+    clnt_local_t *local = NULL;
+    gfx_icreate_req req = {
+        {
+            0,
+        },
+    };
+
+    GF_ASSERT(frame);
+
+    args = data;
+    conf = this->private;
+
+    if (!(args->loc && args->loc->inode))
+        goto unwind;
+
+    local = mem_get0(this->local_pool);
+    if (!local) {
+        op_errno = ENOMEM;
+        goto unwind;
+    }
+    frame->local = local;
+
+    loc_copy(&local->loc, args->loc);
+
+    req.mode = args->mode;
+    memcpy(req.gfid, args->loc->gfid, sizeof(uuid_t));
+
+    op_errno = ESTALE;
+    dict_to_xdr(args->xdata, &req.xdata);
+    ret = client_submit_request(this, &req, frame, conf->fops, GFS3_OP_ICREATE,
+                                client4_icreate_cbk, NULL,
+                                (xdrproc_t)xdr_gfx_icreate_req);
+    if (ret)
+        goto free_reqdata;
+    GF_FREE(req.xdata.pairs.pairs_val);
+    return 0;
+
+free_reqdata:
+    GF_FREE(req.xdata.pairs.pairs_val);
+unwind:
+    CLIENT_STACK_UNWIND(icreate, frame, -1, op_errno, NULL, NULL, NULL);
+    return 0;
+}
+
+int32_t
+client4_0_put(call_frame_t *frame, xlator_t *this, void *data)
+{
+    clnt_args_t *args = NULL;
+    clnt_conf_t *conf = NULL;
+    gfx_put_req req = {
+        {
+            0,
+        },
+    };
+    int op_errno = ESTALE;
+    int ret = 0;
+    clnt_local_t *local = NULL;
+    client_payload_t cp;
+
+    if (!frame || !this || !data)
+        goto unwind;
+
+    args = data;
+    conf = this->private;
+
+    local = mem_get0(this->local_pool);
+    if (!local) {
+        op_errno = ENOMEM;
+        goto unwind;
+    }
+    frame->local = local;
+
+    loc_copy(&local->loc, args->loc);
+    loc_path(&local->loc, NULL);
+
+    ret = client_pre_put_v2(this, &req, args->loc, args->mode, args->umask,
+                            args->flags, args->size, args->offset, args->xattr,
+                            args->xdata);
+
+    if (ret) {
+        op_errno = -ret;
+        goto unwind;
+    }
+
+    memset(&cp, 0, sizeof(client_payload_t));
+
+    cp.iobref = args->iobref;
+    cp.payload = args->vector;
+    cp.payload_cnt = args->count;
+    ret = client_submit_request(this, &req, frame, conf->fops, GFS3_OP_PUT,
+                                client4_0_put_cbk, &cp,
+                                (xdrproc_t)xdr_gfx_put_req);
+    if (ret) {
+        /*
+         * If the lower layers fail to submit a request, they'll also
+         * do the unwind for us (see rpc_clnt_submit), so don't unwind
+         * here in such cases.
+         */
+        gf_smsg(this->name, GF_LOG_WARNING, 0, PC_MSG_FOP_SEND_FAILED, NULL);
+    }
+
+    return 0;
+
+unwind:
+    CLIENT_STACK_UNWIND(put, frame, -1, op_errno, NULL, NULL, NULL, NULL, NULL);
+    return 0;
+}
+
+int32_t
+client4_0_copy_file_range(call_frame_t *frame, xlator_t *this, void *data)
+{
+    clnt_args_t *args = NULL;
+    clnt_conf_t *conf = NULL;
+    clnt_local_t *local = NULL;
+    gfx_copy_file_range_req req = {
+        {
+            0,
+        },
+    };
+    int op_errno = ESTALE;
+    int ret = 0;
+
+    if (!frame || !this || !data)
+        goto unwind;
+
+    args = data;
+    conf = this->private;
+
+    ret = client_pre_copy_file_range_v2(this, &req, args->fd, args->off_in,
+                                        args->fd_out, args->off_out, args->size,
+                                        args->flags, &args->xdata);
+
+    if (ret) {
+        op_errno = -ret;
+        goto unwind;
+    }
+
+    ret = client_fd_fop_prepare_local(frame, args->fd, req.fd_in);
+    if (ret) {
+        op_errno = -ret;
+        goto unwind;
+    }
+
+    /*
+     * Since frame->local is allocated in above function call
+     * itself, better to use it (with the assumption that it
+     * has been allocated) directly instead of again calling
+     * client_fd_fop_prepare_local or modifying it, as doing
+     * so requires changes in other places as well.
+     */
+
+    local = frame->local;
+    local->fd_out = fd_ref(args->fd_out);
+    local->attempt_reopen_out = client_is_reopen_needed(args->fd_out, this,
+                                                        req.fd_out);
+
+    ret = client_submit_request(this, &req, frame, conf->fops,
+                                GFS3_OP_COPY_FILE_RANGE,
+                                client4_0_copy_file_range_cbk, NULL,
+                                (xdrproc_t)xdr_gfx_copy_file_range_req);
+    if (ret) {
+        /*
+         * If the lower layers fail to submit a request, they'll also
+         * do the unwind for us (see rpc_clnt_submit), so don't unwind
+         * here in such cases.
+         */
+        gf_smsg(this->name, GF_LOG_WARNING, 0, PC_MSG_FOP_SEND_FAILED, NULL);
+    }
+
+    GF_FREE(req.xdata.pairs.pairs_val);
+
+    return 0;
+
+unwind:
+    CLIENT_STACK_UNWIND(copy_file_range, frame, -1, op_errno, NULL, NULL, NULL,
+                        NULL);
+    GF_FREE(req.xdata.pairs.pairs_val);
+
+    return 0;
+}
+
+int32_t
+client4_0_fsetattr(call_frame_t *frame, xlator_t *this, void *data)
+{
+    clnt_args_t *args = NULL;
+    clnt_conf_t *conf = NULL;
+    gfx_fsetattr_req req = {
+        {0},
+    };
+    int op_errno = ESTALE;
+    int ret = 0;
+
+    if (!frame || !this || !data)
+        goto unwind;
+
+    args = data;
+    conf = this->private;
+
+    ret = client_pre_fsetattr_v2(this, &req, args->fd, args->valid, args->stbuf,
+                                 args->xdata);
+    if (ret) {
+        op_errno = -ret;
+        goto unwind;
+    }
+    ret = client_submit_request(this, &req, frame, conf->fops, GFS3_OP_FSETATTR,
+                                client4_0_fsetattr_cbk, NULL,
+                                (xdrproc_t)xdr_gfx_fsetattr_req);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_WARNING, 0, PC_MSG_FOP_SEND_FAILED, NULL);
+    }
+
+    GF_FREE(req.xdata.pairs.pairs_val);
+
+    return 0;
+unwind:
+    CLIENT_STACK_UNWIND(fsetattr, frame, -1, op_errno, NULL, NULL, NULL);
+    GF_FREE(req.xdata.pairs.pairs_val);
+
+    return 0;
+}
+
+int32_t
+client4_0_rchecksum(call_frame_t *frame, xlator_t *this, void *data)
+{
+    clnt_args_t *args = NULL;
+    int64_t remote_fd = -1;
+    clnt_conf_t *conf = NULL;
+    gfx_rchecksum_req req = {
+        {0},
+    };
+    int op_errno = ESTALE;
+    int ret = 0;
+
+    if (!frame || !this || !data)
+        goto unwind;
+
+    args = data;
+    conf = this->private;
+
+    CLIENT_GET_REMOTE_FD(this, args->fd, DEFAULT_REMOTE_FD, remote_fd, op_errno,
+                         unwind);
+
+    req.len = args->len;
+    req.offset = args->offset;
+    req.fd = remote_fd;
+    memcpy(req.gfid, args->fd->inode->gfid, 16);
+
+    dict_to_xdr(args->xdata, &req.xdata);
+
+    ret = client_submit_request(this, &req, frame, conf->fops,
+                                GFS3_OP_RCHECKSUM, client4_rchecksum_cbk, NULL,
+                                (xdrproc_t)xdr_gfx_rchecksum_req);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_WARNING, 0, PC_MSG_FOP_SEND_FAILED, NULL);
+    }
+
+    GF_FREE(req.xdata.pairs.pairs_val);
+
+    return 0;
+unwind:
+    CLIENT_STACK_UNWIND(rchecksum, frame, -1, op_errno, 0, NULL, NULL);
+    GF_FREE(req.xdata.pairs.pairs_val);
+
+    return 0;
+}
+
+/* Used From RPC-CLNT library to log proper name of procedure based on number */
+char *clnt4_0_fop_names[GFS3_OP_MAXVALUE] = {
+    [GFS3_OP_NULL] = "NULL",
+    [GFS3_OP_STAT] = "STAT",
+    [GFS3_OP_READLINK] = "READLINK",
+    [GFS3_OP_MKNOD] = "MKNOD",
+    [GFS3_OP_MKDIR] = "MKDIR",
+    [GFS3_OP_UNLINK] = "UNLINK",
+    [GFS3_OP_RMDIR] = "RMDIR",
+    [GFS3_OP_SYMLINK] = "SYMLINK",
+    [GFS3_OP_RENAME] = "RENAME",
+    [GFS3_OP_LINK] = "LINK",
+    [GFS3_OP_TRUNCATE] = "TRUNCATE",
+    [GFS3_OP_OPEN] = "OPEN",
+    [GFS3_OP_READ] = "READ",
+    [GFS3_OP_WRITE] = "WRITE",
+    [GFS3_OP_STATFS] = "STATFS",
+    [GFS3_OP_FLUSH] = "FLUSH",
+    [GFS3_OP_FSYNC] = "FSYNC",
+    [GFS3_OP_SETXATTR] = "SETXATTR",
+    [GFS3_OP_GETXATTR] = "GETXATTR",
+    [GFS3_OP_REMOVEXATTR] = "REMOVEXATTR",
+    [GFS3_OP_OPENDIR] = "OPENDIR",
+    [GFS3_OP_FSYNCDIR] = "FSYNCDIR",
+    [GFS3_OP_ACCESS] = "ACCESS",
+    [GFS3_OP_CREATE] = "CREATE",
+    [GFS3_OP_FTRUNCATE] = "FTRUNCATE",
+    [GFS3_OP_FSTAT] = "FSTAT",
+    [GFS3_OP_LK] = "LK",
+    [GFS3_OP_LOOKUP] = "LOOKUP",
+    [GFS3_OP_READDIR] = "READDIR",
+    [GFS3_OP_INODELK] = "INODELK",
+    [GFS3_OP_FINODELK] = "FINODELK",
+    [GFS3_OP_ENTRYLK] = "ENTRYLK",
+    [GFS3_OP_FENTRYLK] = "FENTRYLK",
+    [GFS3_OP_XATTROP] = "XATTROP",
+    [GFS3_OP_FXATTROP] = "FXATTROP",
+    [GFS3_OP_FGETXATTR] = "FGETXATTR",
+    [GFS3_OP_FSETXATTR] = "FSETXATTR",
+    [GFS3_OP_RCHECKSUM] = "RCHECKSUM",
+    [GFS3_OP_SETATTR] = "SETATTR",
+    [GFS3_OP_FSETATTR] = "FSETATTR",
+    [GFS3_OP_READDIRP] = "READDIRP",
+    [GFS3_OP_RELEASE] = "RELEASE",
+    [GFS3_OP_RELEASEDIR] = "RELEASEDIR",
+    [GFS3_OP_FREMOVEXATTR] = "FREMOVEXATTR",
+    [GFS3_OP_FALLOCATE] = "FALLOCATE",
+    [GFS3_OP_DISCARD] = "DISCARD",
+    [GFS3_OP_ZEROFILL] = "ZEROFILL",
+    [GFS3_OP_IPC] = "IPC",
+    [GFS3_OP_SEEK] = "SEEK",
+    [GFS3_OP_LEASE] = "LEASE",
+    [GFS3_OP_GETACTIVELK] = "GETACTIVELK",
+    [GFS3_OP_SETACTIVELK] = "SETACTIVELK",
+    [GFS3_OP_COMPOUND] = "COMPOUND",
+    [GFS3_OP_ICREATE] = "ICREATE",
+    [GFS3_OP_NAMELINK] = "NAMELINK",
+};
+
+rpc_clnt_procedure_t clnt4_0_fop_actors[GF_FOP_MAXVALUE] = {
+    [GF_FOP_NULL] = {"NULL", NULL},
+    [GF_FOP_STAT] = {"STAT", client4_0_stat},
+    [GF_FOP_READLINK] = {"READLINK", client4_0_readlink},
+    [GF_FOP_MKNOD] = {"MKNOD", client4_0_mknod},
+    [GF_FOP_MKDIR] = {"MKDIR", client4_0_mkdir},
+    [GF_FOP_UNLINK] = {"UNLINK", client4_0_unlink},
+    [GF_FOP_RMDIR] = {"RMDIR", client4_0_rmdir},
+    [GF_FOP_SYMLINK] = {"SYMLINK", client4_0_symlink},
+    [GF_FOP_RENAME] = {"RENAME", client4_0_rename},
+    [GF_FOP_LINK] = {"LINK", client4_0_link},
+    [GF_FOP_TRUNCATE] = {"TRUNCATE", client4_0_truncate},
+    [GF_FOP_OPEN] = {"OPEN", client4_0_open},
+    [GF_FOP_READ] = {"READ", client4_0_readv},
+    [GF_FOP_WRITE] = {"WRITE", client4_0_writev},
+    [GF_FOP_STATFS] = {"STATFS", client4_0_statfs},
+    [GF_FOP_FLUSH] = {"FLUSH", client4_0_flush},
+    [GF_FOP_FSYNC] = {"FSYNC", client4_0_fsync},
+    [GF_FOP_GETXATTR] = {"GETXATTR", client4_0_getxattr},
+    [GF_FOP_SETXATTR] = {"SETXATTR", client4_0_setxattr},
+    [GF_FOP_REMOVEXATTR] = {"REMOVEXATTR", client4_0_removexattr},
+    [GF_FOP_OPENDIR] = {"OPENDIR", client4_0_opendir},
+    [GF_FOP_FSYNCDIR] = {"FSYNCDIR", client4_0_fsyncdir},
+    [GF_FOP_ACCESS] = {"ACCESS", client4_0_access},
+    [GF_FOP_CREATE] = {"CREATE", client4_0_create},
+    [GF_FOP_FTRUNCATE] = {"FTRUNCATE", client4_0_ftruncate},
+    [GF_FOP_FSTAT] = {"FSTAT", client4_0_fstat},
+    [GF_FOP_LK] = {"LK", client4_0_lk},
+    [GF_FOP_LOOKUP] = {"LOOKUP", client4_0_lookup},
+    [GF_FOP_READDIR] = {"READDIR", client4_0_readdir},
+    [GF_FOP_INODELK] = {"INODELK", client4_0_inodelk},
+    [GF_FOP_FINODELK] = {"FINODELK", client4_0_finodelk},
+    [GF_FOP_ENTRYLK] = {"ENTRYLK", client4_0_entrylk},
+    [GF_FOP_FENTRYLK] = {"FENTRYLK", client4_0_fentrylk},
+    [GF_FOP_XATTROP] = {"XATTROP", client4_0_xattrop},
+    [GF_FOP_FXATTROP] = {"FXATTROP", client4_0_fxattrop},
+    [GF_FOP_FGETXATTR] = {"FGETXATTR", client4_0_fgetxattr},
+    [GF_FOP_FSETXATTR] = {"FSETXATTR", client4_0_fsetxattr},
+    [GF_FOP_RCHECKSUM] = {"RCHECKSUM", client4_0_rchecksum},
+    [GF_FOP_SETATTR] = {"SETATTR", client4_0_setattr},
+    [GF_FOP_FSETATTR] = {"FSETATTR", client4_0_fsetattr},
+    [GF_FOP_READDIRP] = {"READDIRP", client4_0_readdirp},
+    [GF_FOP_FALLOCATE] = {"FALLOCATE", client4_0_fallocate},
+    [GF_FOP_DISCARD] = {"DISCARD", client4_0_discard},
+    [GF_FOP_ZEROFILL] = {"ZEROFILL", client4_0_zerofill},
+    [GF_FOP_RELEASE] = {"RELEASE", client4_0_release},
+    [GF_FOP_RELEASEDIR] = {"RELEASEDIR", client4_0_releasedir},
+    [GF_FOP_GETSPEC] = {"GETSPEC", client3_getspec},
+    [GF_FOP_FREMOVEXATTR] = {"FREMOVEXATTR", client4_0_fremovexattr},
+    [GF_FOP_IPC] = {"IPC", client4_0_ipc},
+    [GF_FOP_SEEK] = {"SEEK", client4_0_seek},
+    [GF_FOP_LEASE] = {"LEASE", client4_0_lease},
+    [GF_FOP_GETACTIVELK] = {"GETACTIVELK", client4_0_getactivelk},
+    [GF_FOP_SETACTIVELK] = {"SETACTIVELK", client4_0_setactivelk},
+    [GF_FOP_COMPOUND] = {"COMPOUND", NULL},
+    [GF_FOP_ICREATE] = {"ICREATE", client4_0_icreate},
+    [GF_FOP_NAMELINK] = {"NAMELINK", client4_0_namelink},
+    [GF_FOP_COPY_FILE_RANGE] = {"COPY-FILE-RANGE", client4_0_copy_file_range},
+};
+
+rpc_clnt_prog_t clnt4_0_fop_prog = {
+    .progname = "GlusterFS 4.x v1",
+    .prognum = GLUSTER_FOP_PROGRAM,
+    .progver = GLUSTER_FOP_VERSION_v2,
+    .numproc = GLUSTER_FOP_PROCCNT,
+    .proctable = clnt4_0_fop_actors,
+    .procnames = clnt4_0_fop_names,
+};
diff --git a/xlators/protocol/client/src/client.c b/xlators/protocol/client/src/client.c
index 5745de096e0..0f31fea9511 100644
--- a/xlators/protocol/client/src/client.c
+++ b/xlators/protocol/client/src/client.c
@@ -8,2844 +8,2990 @@
   cases as published by the Free Software Foundation.
 */
 
-
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
-
 #include "client.h"
-#include "xlator.h"
-#include "defaults.h"
-#include "glusterfs.h"
-#include "statedump.h"
-#include "compat-errno.h"
-#include "event.h"
+#include <glusterfs/xlator.h>
+#include <glusterfs/defaults.h>
+#include <glusterfs/glusterfs.h>
+#include <glusterfs/statedump.h>
+#include <glusterfs/compat-errno.h>
+#include <glusterfs/gf-event.h>
 
 #include "xdr-rpc.h"
 #include "glusterfs3.h"
-#include "gf-dirent.h"
+#include "client-messages.h"
 
 extern rpc_clnt_prog_t clnt_handshake_prog;
 extern rpc_clnt_prog_t clnt_dump_prog;
 extern struct rpcclnt_cb_program gluster_cbk_prog;
 
-int client_handshake (xlator_t *this, struct rpc_clnt *rpc);
-int client_init_rpc (xlator_t *this);
-int client_destroy_rpc (xlator_t *this);
-int client_mark_fd_bad (xlator_t *this);
-
+int
+client_handshake(xlator_t *this, struct rpc_clnt *rpc);
 static int
-client_fini_complete (xlator_t *this)
-{
-        GF_VALIDATE_OR_GOTO (this->name, this->private, out);
+client_destroy_rpc(xlator_t *this);
 
-        clnt_conf_t *conf = this->private;
+static void
+client_filter_o_direct(clnt_conf_t *conf, int32_t *flags)
+{
+    if (conf->filter_o_direct)
+        *flags = (*flags & ~O_DIRECT);
+}
 
-        if (!conf->destroy)
-                return 0;
+static int
+client_fini_complete(xlator_t *this)
+{
+    GF_VALIDATE_OR_GOTO(this->name, this->private, out);
 
-        this->private = NULL;
+    clnt_conf_t *conf = this->private;
+    if (!conf->destroy)
+        return 0;
 
-        pthread_mutex_destroy (&conf->lock);
-        GF_FREE (conf);
+    pthread_mutex_lock(&conf->lock);
+    {
+        conf->fini_completed = _gf_true;
+        pthread_cond_broadcast(&conf->fini_complete_cond);
+    }
+    pthread_mutex_unlock(&conf->lock);
 
 out:
-        return 0;
+    return 0;
 }
 
 static int
-client_notify_dispatch_uniq (xlator_t *this, int32_t event, void *data, ...)
+client_is_last_child_down(xlator_t *this, int32_t event, struct rpc_clnt *rpc)
 {
-        clnt_conf_t     *conf = this->private;
-
-        if (conf->last_sent_event == event)
-                return 0;
+    rpc_clnt_connection_t *conn = NULL;
+    clnt_conf_t *conf = NULL;
+    int ret = 0;
 
-        return client_notify_dispatch (this, event, data);
-}
-
-int
-client_notify_dispatch (xlator_t *this, int32_t event, void *data, ...)
-{
-       int              ret  = -1;
-       glusterfs_ctx_t *ctx  = this->ctx;
-       clnt_conf_t     *conf = this->private;
-
-       pthread_mutex_lock (&ctx->notify_lock);
-       {
-                while (ctx->notifying)
-                        pthread_cond_wait (&ctx->notify_cond,
-                                           &ctx->notify_lock);
-               ctx->notifying = 1;
-       }
-       pthread_mutex_unlock (&ctx->notify_lock);
-
-       /* We assume that all translators in the graph handle notification
-        * events in sequence.
-        * */
-       ret = default_notify (this, event, data);
-
-       /* NB (Even) with MT-epoll and EPOLLET|EPOLLONESHOT we are guaranteed
-        * that there would be atmost one poller thread executing this
-        * notification function. This allows us to update last_sent_event
-        * without explicit synchronization. See epoll(7).
-        */
-       conf->last_sent_event = event;
-
-       pthread_mutex_lock (&ctx->notify_lock);
-       {
-               ctx->notifying = 0;
-               pthread_cond_signal (&ctx->notify_cond);
-       }
-       pthread_mutex_unlock (&ctx->notify_lock);
-
-       return ret;
-}
+    if (!this || !rpc)
+        goto out;
 
-int32_t
-client_type_to_gf_type (short l_type)
-{
-        int32_t  gf_type = GF_LK_EOL;
-
-        switch (l_type) {
-        case F_RDLCK:
-                gf_type = GF_LK_F_RDLCK;
-                break;
-        case F_WRLCK:
-                gf_type = GF_LK_F_WRLCK;
-                break;
-        case F_UNLCK:
-                gf_type = GF_LK_F_UNLCK;
-                break;
-        }
+    conf = this->private;
+    if (!conf)
+        goto out;
 
-        return gf_type;
-}
+    if (!conf->parent_down)
+        goto out;
 
-uint32_t
-client_get_lk_ver (clnt_conf_t *conf)
-{
-        uint32_t  lk_ver = 0;
+    if (event != GF_EVENT_CHILD_DOWN)
+        goto out;
 
-        GF_VALIDATE_OR_GOTO ("client", conf, out);
-
-        pthread_mutex_lock (&conf->lock);
-        {
-                lk_ver = conf->lk_version;
+    conn = &rpc->conn;
+    pthread_mutex_lock(&conn->lock);
+    {
+        if (!conn->reconnect && rpc->disabled) {
+            ret = 1;
         }
-        pthread_mutex_unlock (&conf->lock);
+    }
+    pthread_mutex_unlock(&conn->lock);
 out:
-        return lk_ver;
+    return ret;
 }
 
-void
-client_grace_timeout (void *data)
-{
-        int               ver  = 0;
-        xlator_t         *this = NULL;
-        struct clnt_conf *conf = NULL;
-        struct rpc_clnt  *rpc  = NULL;
+int
+client_notify_dispatch_uniq(xlator_t *this, int32_t event, void *data, ...)
+{
+    clnt_conf_t *conf = this->private;
+    glusterfs_ctx_t *ctx = this->ctx;
+    glusterfs_graph_t *graph = this->graph;
+
+    pthread_mutex_lock(&ctx->notify_lock);
+    {
+        while (ctx->notifying)
+            pthread_cond_wait(&ctx->notify_cond, &ctx->notify_lock);
+
+        if (client_is_last_child_down(this, event, data) && graph) {
+            pthread_mutex_lock(&graph->mutex);
+            {
+                graph->parent_down++;
+                if (graph->parent_down == graph_total_client_xlator(graph)) {
+                    graph->used = 0;
+                    pthread_cond_broadcast(&graph->child_down_cond);
+                }
+            }
+            pthread_mutex_unlock(&graph->mutex);
+        }
+    }
+    pthread_mutex_unlock(&ctx->notify_lock);
 
-        GF_VALIDATE_OR_GOTO ("client", data, out);
+    if (conf->last_sent_event == event)
+        return 0;
 
-        this = THIS;
+    return client_notify_dispatch(this, event, data);
 
-        rpc = (struct rpc_clnt *) data;
+    /* Please avoid any code that access xlator object here
+     * Because for a child down event, once we do the signal
+     * we will start cleanup.
+     */
+}
 
-        conf = (struct clnt_conf *) this->private;
+int
+client_notify_dispatch(xlator_t *this, int32_t event, void *data, ...)
+{
+    int ret = -1;
+    glusterfs_ctx_t *ctx = this->ctx;
 
-        pthread_mutex_lock (&conf->lock);
-        {
-                ver = ++conf->lk_version;
-                /* ver == 0 is a special value used by server
-                   to notify client that this is a fresh connect.*/
-                if (ver == 0)
-                        ver = ++conf->lk_version;
-
-                gf_timer_call_cancel (this->ctx, conf->grace_timer);
-                conf->grace_timer = NULL;
-        }
-        pthread_mutex_unlock (&conf->lock);
+    clnt_conf_t *conf = this->private;
 
-        gf_log (this->name, GF_LOG_WARNING,
-                "client grace timer expired, updating "
-                "the lk-version to %d", ver);
+    pthread_mutex_lock(&ctx->notify_lock);
+    {
+        while (ctx->notifying)
+            pthread_cond_wait(&ctx->notify_cond, &ctx->notify_lock);
+        ctx->notifying = 1;
+    }
+    pthread_mutex_unlock(&ctx->notify_lock);
 
-        client_mark_fd_bad (this);
-out:
-        return;
-}
+    /* We assume that all translators in the graph handle notification
+     * events in sequence.
+     * */
 
-int32_t
-client_register_grace_timer (xlator_t *this, clnt_conf_t *conf)
-{
-        int32_t  ret = -1;
+    ret = default_notify(this, event, data);
 
-        GF_VALIDATE_OR_GOTO ("client", this, out);
-        GF_VALIDATE_OR_GOTO (this->name, conf, out);
+    /* NB (Even) with MT-epoll and EPOLLET|EPOLLONESHOT we are guaranteed
+     * that there would be atmost one poller thread executing this
+     * notification function. This allows us to update last_sent_event
+     * without explicit synchronization. See epoll(7).
+     */
+    conf->last_sent_event = event;
 
-        pthread_mutex_lock (&conf->lock);
-        {
-                if (conf->grace_timer || !conf->grace_timer_needed) {
-                        gf_log (this->name, GF_LOG_TRACE,
-				"Client grace timer is already set "
-				"or a grace-timer has already time out, "
-				"not registering a new timer");
-                } else {
-                        gf_log (this->name, GF_LOG_INFO,
-                                "Registering a grace timer");
-
-                        conf->grace_timer_needed = _gf_false;
-
-                        conf->grace_timer =
-                                gf_timer_call_after (this->ctx,
-                                                     conf->grace_ts,
-                                                     client_grace_timeout,
-                                                     conf->rpc);
-                }
-        }
-        pthread_mutex_unlock (&conf->lock);
+    pthread_mutex_lock(&ctx->notify_lock);
+    {
+        ctx->notifying = 0;
+        pthread_cond_signal(&ctx->notify_cond);
+    }
+    pthread_mutex_unlock(&ctx->notify_lock);
 
-        ret = 0;
-out:
-        return ret;
+    /* Please avoid any code that access xlator object here
+     * Because for a child down event, once we do the signal
+     * we will start cleanup.
+     */
+
+    return ret;
 }
 
 int
-client_submit_request (xlator_t *this, void *req, call_frame_t *frame,
-                       rpc_clnt_prog_t *prog, int procnum, fop_cbk_fn_t cbkfn,
-                       struct iobref *iobref,  struct iovec *rsphdr,
-                       int rsphdr_count, struct iovec *rsp_payload,
-                       int rsp_payload_count, struct iobref *rsp_iobref,
-                       xdrproc_t xdrproc)
-{
-        int             ret        = -1;
-        clnt_conf_t    *conf       = NULL;
-        struct iovec    iov        = {0, };
-        struct iobuf   *iobuf      = NULL;
-        int             count      = 0;
-        struct iobref  *new_iobref = NULL;
-        ssize_t         xdr_size   = 0;
-        struct rpc_req  rpcreq     = {0, };
-        uint64_t        ngroups    = 0;
-        uint64_t        gid        = 0;
-
-        GF_VALIDATE_OR_GOTO ("client", this, out);
-        GF_VALIDATE_OR_GOTO (this->name, prog, out);
-        GF_VALIDATE_OR_GOTO (this->name, frame, out);
-
-        conf = this->private;
-
-        /* If 'setvolume' is not successful, we should not send frames to
-           server, mean time we should be able to send 'DUMP' and 'SETVOLUME'
-           call itself even if its not connected */
-        if (!(conf->connected ||
-              ((prog->prognum == GLUSTER_DUMP_PROGRAM) ||
-               (prog->prognum == GLUSTER_PMAP_PROGRAM) ||
-               ((prog->prognum == GLUSTER_HNDSK_PROGRAM) &&
-                (procnum == GF_HNDSK_SETVOLUME))))) {
-                /* This particular error captured/logged in
-                   functions calling this */
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "connection in disconnected state");
-                goto out;
-       }
-
-        if (req && xdrproc) {
-                xdr_size = xdr_sizeof (xdrproc, req);
-                iobuf = iobuf_get2 (this->ctx->iobuf_pool, xdr_size);
-                if (!iobuf) {
-                        goto out;
-                };
-
-                new_iobref = iobref_new ();
-                if (!new_iobref) {
-                        goto out;
-                }
-
-                if (iobref != NULL) {
-                        ret = iobref_merge (new_iobref, iobref);
-                        if (ret != 0) {
-                                gf_log (this->name, GF_LOG_WARNING,
-                                        "cannot merge iobref passed from caller "
-                                        "into new_iobref");
-                        }
-                }
-
-                ret = iobref_add (new_iobref, iobuf);
-                if (ret != 0) {
-                        gf_log (this->name, GF_LOG_WARNING,
-                                "cannot add iobuf into iobref");
-                        goto out;
-                }
-
-                iov.iov_base = iobuf->ptr;
-                iov.iov_len  = iobuf_size (iobuf);
-
-                /* Create the xdr payload */
-                ret = xdr_serialize_generic (iov, req, xdrproc);
-                if (ret == -1) {
-                        /* callingfn so that, we can get to know which xdr
-                           function was called */
-                        gf_log_callingfn (this->name, GF_LOG_WARNING,
-                                          "XDR payload creation failed");
-                        goto out;
-                }
-                iov.iov_len = ret;
-                count = 1;
+client_submit_request(xlator_t *this, void *req, call_frame_t *frame,
+                      rpc_clnt_prog_t *prog, int procnum, fop_cbk_fn_t cbkfn,
+                      client_payload_t *cp, xdrproc_t xdrproc)
+{
+    int ret = -1;
+    clnt_conf_t *conf = NULL;
+    struct iovec iov = {
+        0,
+    };
+    struct iobuf *iobuf = NULL;
+    int count = 0;
+    struct iobref *new_iobref = NULL;
+    ssize_t xdr_size = 0;
+    struct rpc_req rpcreq = {
+        0,
+    };
+
+    GF_VALIDATE_OR_GOTO("client", this, out);
+    GF_VALIDATE_OR_GOTO(this->name, prog, out);
+    GF_VALIDATE_OR_GOTO(this->name, frame, out);
+
+    conf = this->private;
+
+    /* If 'setvolume' is not successful, we should not send frames to
+       server, mean time we should be able to send 'DUMP' and 'SETVOLUME'
+       call itself even if its not connected */
+    if (!(conf->connected || ((prog->prognum == GLUSTER_DUMP_PROGRAM) ||
+                              (prog->prognum == GLUSTER_PMAP_PROGRAM) ||
+                              ((prog->prognum == GLUSTER_HNDSK_PROGRAM) &&
+                               (procnum == GF_HNDSK_SETVOLUME))))) {
+        /* This particular error captured/logged in
+           functions calling this */
+        gf_msg_debug(this->name, 0, "connection in disconnected state");
+        goto out;
+    }
+
+    if (req && xdrproc) {
+        xdr_size = xdr_sizeof(xdrproc, req);
+        iobuf = iobuf_get2(this->ctx->iobuf_pool, xdr_size);
+        if (!iobuf) {
+            goto out;
         }
 
-        /* do not send all groups if they are resolved server-side */
-        if (!conf->send_gids) {
-                /* copy some values for restoring later */
-                ngroups = frame->root->ngrps;
-                frame->root->ngrps = 1;
-                if (ngroups <= SMALL_GROUP_COUNT) {
-                        gid = frame->root->groups_small[0];
-                        frame->root->groups_small[0] = frame->root->gid;
-                        frame->root->groups = frame->root->groups_small;
-                }
+        new_iobref = iobref_new();
+        if (!new_iobref) {
+            goto out;
         }
 
-        /* Send the msg */
-        ret = rpc_clnt_submit (conf->rpc, prog, procnum, cbkfn, &iov, count,
-                               NULL, 0, new_iobref, frame, rsphdr, rsphdr_count,
-                               rsp_payload, rsp_payload_count, rsp_iobref);
+        if (cp && cp->iobref != NULL) {
+            ret = iobref_merge(new_iobref, cp->iobref);
+            if (ret != 0) {
+                gf_smsg(this->name, GF_LOG_WARNING, ENOMEM,
+                        PC_MSG_MERGE_IOBREF_FAILED, NULL);
+            }
+        }
 
-        if (ret < 0) {
-                gf_log (this->name, GF_LOG_DEBUG, "rpc_clnt_submit failed");
+        ret = iobref_add(new_iobref, iobuf);
+        if (ret != 0) {
+            gf_smsg(this->name, GF_LOG_WARNING, ENOMEM, PC_MSG_ADD_IOBUF_FAILED,
+                    NULL);
+            goto out;
         }
 
-        if (!conf->send_gids) {
-                /* restore previous values */
-                frame->root->ngrps = ngroups;
-                if (ngroups <= SMALL_GROUP_COUNT)
-                        frame->root->groups_small[0] = gid;
+        iov.iov_base = iobuf->ptr;
+        iov.iov_len = iobuf_size(iobuf);
+
+        /* Create the xdr payload */
+        ret = xdr_serialize_generic(iov, req, xdrproc);
+        if (ret == -1) {
+            /* callingfn so that, we can get to know which xdr
+               function was called */
+            gf_log_callingfn(this->name, GF_LOG_WARNING,
+                             "XDR payload creation failed");
+            goto out;
+        }
+        iov.iov_len = ret;
+        count = 1;
+    }
+
+    /* do not send all groups if they are resolved server-side */
+    if (!conf->send_gids) {
+        if (frame->root->ngrps <= SMALL_GROUP_COUNT) {
+            frame->root->groups_small[0] = frame->root->gid;
+            frame->root->groups = frame->root->groups_small;
         }
+        frame->root->ngrps = 1;
+    }
 
-        ret = 0;
+    /* Send the msg */
+    if (cp) {
+        ret = rpc_clnt_submit(conf->rpc, prog, procnum, cbkfn, &iov, count,
+                              cp->payload, cp->payload_cnt, new_iobref, frame,
+                              cp->rsphdr, cp->rsphdr_cnt, cp->rsp_payload,
+                              cp->rsp_payload_cnt, cp->rsp_iobref);
+    } else {
+        ret = rpc_clnt_submit(conf->rpc, prog, procnum, cbkfn, &iov, count,
+                              NULL, 0, new_iobref, frame, NULL, 0, NULL, 0,
+                              NULL);
+    }
 
-        if (new_iobref)
-                iobref_unref (new_iobref);
+    if (ret < 0) {
+        gf_msg_debug(this->name, 0, "rpc_clnt_submit failed");
+    }
 
-        if (iobuf)
-                iobuf_unref (iobuf);
+    ret = 0;
 
-        return ret;
+    if (new_iobref)
+        iobref_unref(new_iobref);
+
+    if (iobuf)
+        iobuf_unref(iobuf);
+
+    return ret;
 
 out:
-        rpcreq.rpc_status = -1;
+    rpcreq.rpc_status = -1;
 
-        cbkfn (&rpcreq, NULL, 0, frame);
+    cbkfn(&rpcreq, NULL, 0, frame);
 
-        if (new_iobref)
-                iobref_unref (new_iobref);
+    if (new_iobref)
+        iobref_unref(new_iobref);
 
-        if (iobuf)
-                iobuf_unref (iobuf);
+    if (iobuf)
+        iobuf_unref(iobuf);
 
-        return 0;
+    return ret;
 }
 
-
-int32_t
-client_forget (xlator_t *this, inode_t *inode)
+static int32_t
+client_forget(xlator_t *this, inode_t *inode)
 {
-        /* Nothing here */
-	return 0;
+    /* Nothing here */
+    return 0;
 }
 
-int32_t
-client_releasedir (xlator_t *this, fd_t *fd)
+static int32_t
+client_releasedir(xlator_t *this, fd_t *fd)
 {
-        int          ret  = -1;
-        clnt_conf_t *conf = NULL;
-        rpc_clnt_procedure_t *proc = NULL;
-        clnt_args_t  args = {0,};
+    int ret = -1;
+    clnt_conf_t *conf = NULL;
+    rpc_clnt_procedure_t *proc = NULL;
+    clnt_args_t args = {
+        0,
+    };
 
-        conf = this->private;
-        if (!conf || !conf->fops)
-                goto out;
+    conf = this->private;
+    if (!conf || !conf->fops)
+        goto out;
 
+    proc = &conf->fops->proctable[GF_FOP_RELEASEDIR];
+    if (proc->fn) {
         args.fd = fd;
-
-        proc = &conf->fops->proctable[GF_FOP_RELEASEDIR];
-
-        if (proc->fn) {
-                ret = proc->fn (NULL, this, &args);
-        }
+        ret = proc->fn(NULL, this, &args);
+    }
 out:
-        if (ret)
-                gf_log (this->name, GF_LOG_WARNING,
-                        "releasedir fop failed");
-	return 0;
+    if (ret)
+        gf_smsg(this->name, GF_LOG_WARNING, 0, PC_MSG_RELEASE_DIR_OP_FAILED,
+                NULL);
+    return 0;
 }
 
-int32_t
-client_release (xlator_t *this, fd_t *fd)
+static int32_t
+client_release(xlator_t *this, fd_t *fd)
 {
-        int          ret  = -1;
-        clnt_conf_t *conf = NULL;
-        rpc_clnt_procedure_t *proc = NULL;
-        clnt_args_t  args = {0,};
+    int ret = -1;
+    clnt_conf_t *conf = NULL;
+    rpc_clnt_procedure_t *proc = NULL;
+    clnt_args_t args = {
+        0,
+    };
 
-        conf = this->private;
-        if (!conf || !conf->fops)
-                goto out;
+    conf = this->private;
+    if (!conf || !conf->fops)
+        goto out;
 
+    proc = &conf->fops->proctable[GF_FOP_RELEASE];
+    if (proc->fn) {
         args.fd = fd;
-        proc = &conf->fops->proctable[GF_FOP_RELEASE];
-
-        if (proc->fn) {
-                ret = proc->fn (NULL, this, &args);
-        }
+        ret = proc->fn(NULL, this, &args);
+    }
 out:
-        if (ret)
-                gf_log (this->name, GF_LOG_WARNING,
-                        "release fop failed");
-	return 0;
+    if (ret)
+        gf_smsg(this->name, GF_LOG_WARNING, 0, PC_MSG_FILE_OP_FAILED, NULL);
+    return 0;
 }
 
-
-int32_t
-client_lookup (call_frame_t *frame, xlator_t *this, loc_t *loc,
-               dict_t *xdata)
+static int32_t
+client_lookup(call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xdata)
 {
-        int          ret  = -1;
-        clnt_conf_t *conf = NULL;
-        rpc_clnt_procedure_t *proc = NULL;
-        clnt_args_t  args = {0,};
+    int ret = -1;
+    clnt_conf_t *conf = NULL;
+    rpc_clnt_procedure_t *proc = NULL;
+    clnt_args_t args = {
+        0,
+    };
 
-        conf = this->private;
-        if (!conf || !conf->fops)
-                goto out;
+    conf = this->private;
+    if (!conf || !conf->fops)
+        goto out;
 
+    proc = &conf->fops->proctable[GF_FOP_LOOKUP];
+    if (proc->fn) {
         args.loc = loc;
         args.xdata = xdata;
-
-        proc = &conf->fops->proctable[GF_FOP_LOOKUP];
-
-        if (proc->fn)
-                ret = proc->fn (frame, this, &args);
+        ret = proc->fn(frame, this, &args);
+    }
 out:
-        /* think of avoiding a missing frame */
-        if (ret)
-                STACK_UNWIND_STRICT (lookup, frame, -1, ENOTCONN,
-                                     NULL, NULL, NULL, NULL);
+    /* think of avoiding a missing frame */
+    if (ret)
+        STACK_UNWIND_STRICT(lookup, frame, -1, ENOTCONN, NULL, NULL, NULL,
+                            NULL);
 
-	return 0;
+    return 0;
 }
 
-
-int32_t
-client_stat (call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xdata)
+static int32_t
+client_stat(call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xdata)
 {
-        int          ret  = -1;
-        clnt_conf_t *conf = NULL;
-        rpc_clnt_procedure_t *proc = NULL;
-        clnt_args_t  args = {0,};
+    int ret = -1;
+    clnt_conf_t *conf = NULL;
+    rpc_clnt_procedure_t *proc = NULL;
+    clnt_args_t args = {
+        0,
+    };
 
-        conf = this->private;
-        if (!conf || !conf->fops)
-                goto out;
+    conf = this->private;
+    if (!conf || !conf->fops)
+        goto out;
 
+    proc = &conf->fops->proctable[GF_FOP_STAT];
+    if (proc->fn) {
         args.loc = loc;
         args.xdata = xdata;
-
-        proc = &conf->fops->proctable[GF_FOP_STAT];
-
-        if (proc->fn)
-                ret = proc->fn (frame, this, &args);
+        ret = proc->fn(frame, this, &args);
+    }
 out:
-        if (ret)
-                STACK_UNWIND_STRICT (stat, frame, -1, ENOTCONN, NULL, NULL);
+    if (ret)
+        STACK_UNWIND_STRICT(stat, frame, -1, ENOTCONN, NULL, NULL);
 
-	return 0;
+    return 0;
 }
 
-
-int32_t
-client_truncate (call_frame_t *frame, xlator_t *this, loc_t *loc,
-                 off_t offset, dict_t *xdata)
+static int32_t
+client_truncate(call_frame_t *frame, xlator_t *this, loc_t *loc, off_t offset,
+                dict_t *xdata)
 {
-        int          ret  = -1;
-        clnt_conf_t *conf = NULL;
-        rpc_clnt_procedure_t *proc = NULL;
-        clnt_args_t  args = {0,};
+    int ret = -1;
+    clnt_conf_t *conf = NULL;
+    rpc_clnt_procedure_t *proc = NULL;
+    clnt_args_t args = {
+        0,
+    };
 
-        conf = this->private;
-        if (!conf || !conf->fops)
-                goto out;
+    conf = this->private;
+    if (!conf || !conf->fops)
+        goto out;
 
-        args.loc    = loc;
+    proc = &conf->fops->proctable[GF_FOP_TRUNCATE];
+    if (proc->fn) {
+        args.loc = loc;
         args.offset = offset;
         args.xdata = xdata;
-
-        proc = &conf->fops->proctable[GF_FOP_TRUNCATE];
-
-        if (proc->fn)
-                ret = proc->fn (frame, this, &args);
+        ret = proc->fn(frame, this, &args);
+    }
 out:
-        if (ret)
-                STACK_UNWIND_STRICT (truncate, frame, -1, ENOTCONN, NULL, NULL, NULL);
+    if (ret)
+        STACK_UNWIND_STRICT(truncate, frame, -1, ENOTCONN, NULL, NULL, NULL);
 
-
-	return 0;
+    return 0;
 }
 
-
-int32_t
-client_ftruncate (call_frame_t *frame, xlator_t *this, fd_t *fd,
-                  off_t offset, dict_t *xdata)
+static int32_t
+client_ftruncate(call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
+                 dict_t *xdata)
 {
-        int          ret  = -1;
-        clnt_conf_t *conf = NULL;
-        rpc_clnt_procedure_t *proc = NULL;
-        clnt_args_t  args = {0,};
+    int ret = -1;
+    clnt_conf_t *conf = NULL;
+    rpc_clnt_procedure_t *proc = NULL;
+    clnt_args_t args = {
+        0,
+    };
 
-        conf = this->private;
-        if (!conf || !conf->fops)
-                goto out;
+    conf = this->private;
+    if (!conf || !conf->fops)
+        goto out;
 
-        args.fd     = fd;
+    proc = &conf->fops->proctable[GF_FOP_FTRUNCATE];
+    if (proc->fn) {
+        args.fd = fd;
         args.offset = offset;
         args.xdata = xdata;
-
-        proc = &conf->fops->proctable[GF_FOP_FTRUNCATE];
-
-        if (proc->fn)
-                ret = proc->fn (frame, this, &args);
+        ret = proc->fn(frame, this, &args);
+    }
 out:
-        if (ret)
-                STACK_UNWIND_STRICT (ftruncate, frame, -1, ENOTCONN, NULL, NULL, NULL);
+    if (ret)
+        STACK_UNWIND_STRICT(ftruncate, frame, -1, ENOTCONN, NULL, NULL, NULL);
 
-	return 0;
+    return 0;
 }
 
-
-
-int32_t
-client_access (call_frame_t *frame, xlator_t *this, loc_t *loc,
-               int32_t mask, dict_t *xdata)
+static int32_t
+client_access(call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t mask,
+              dict_t *xdata)
 {
-        int          ret  = -1;
-        clnt_conf_t *conf = NULL;
-        rpc_clnt_procedure_t *proc = NULL;
-        clnt_args_t  args = {0,};
+    int ret = -1;
+    clnt_conf_t *conf = NULL;
+    rpc_clnt_procedure_t *proc = NULL;
+    clnt_args_t args = {
+        0,
+    };
 
-        conf = this->private;
-        if (!conf || !conf->fops)
-                goto out;
+    conf = this->private;
+    if (!conf || !conf->fops)
+        goto out;
 
-        args.loc  = loc;
+    proc = &conf->fops->proctable[GF_FOP_ACCESS];
+    if (proc->fn) {
+        args.loc = loc;
         args.mask = mask;
         args.xdata = xdata;
-
-        proc = &conf->fops->proctable[GF_FOP_ACCESS];
-
-        if (proc->fn)
-                ret = proc->fn (frame, this, &args);
+        ret = proc->fn(frame, this, &args);
+    }
 out:
-        if (ret)
-                STACK_UNWIND_STRICT (access, frame, -1, ENOTCONN, NULL);
+    if (ret)
+        STACK_UNWIND_STRICT(access, frame, -1, ENOTCONN, NULL);
 
-	return 0;
+    return 0;
 }
 
-
-
-
-int32_t
-client_readlink (call_frame_t *frame, xlator_t *this, loc_t *loc,
-                 size_t size, dict_t *xdata)
+static int32_t
+client_readlink(call_frame_t *frame, xlator_t *this, loc_t *loc, size_t size,
+                dict_t *xdata)
 {
-        int          ret  = -1;
-        clnt_conf_t *conf = NULL;
-        rpc_clnt_procedure_t *proc = NULL;
-        clnt_args_t  args = {0,};
+    int ret = -1;
+    clnt_conf_t *conf = NULL;
+    rpc_clnt_procedure_t *proc = NULL;
+    clnt_args_t args = {
+        0,
+    };
 
-        conf = this->private;
-        if (!conf || !conf->fops)
-                goto out;
+    conf = this->private;
+    if (!conf || !conf->fops)
+        goto out;
 
-        args.loc  = loc;
+    proc = &conf->fops->proctable[GF_FOP_READLINK];
+    if (proc->fn) {
+        args.loc = loc;
         args.size = size;
         args.xdata = xdata;
-
-        proc = &conf->fops->proctable[GF_FOP_READLINK];
-
-        if (proc->fn)
-                ret = proc->fn (frame, this, &args);
+        ret = proc->fn(frame, this, &args);
+    }
 out:
-        if (ret)
-                STACK_UNWIND_STRICT (readlink, frame, -1, ENOTCONN, NULL, NULL, NULL);
+    if (ret)
+        STACK_UNWIND_STRICT(readlink, frame, -1, ENOTCONN, NULL, NULL, NULL);
 
-	return 0;
+    return 0;
 }
 
-
-int
-client_mknod (call_frame_t *frame, xlator_t *this, loc_t *loc, mode_t mode,
-              dev_t rdev, mode_t umask, dict_t *xdata)
-{
-        int          ret  = -1;
-        clnt_conf_t *conf = NULL;
-        rpc_clnt_procedure_t *proc = NULL;
-        clnt_args_t  args = {0,};
-
-        conf = this->private;
-        if (!conf || !conf->fops)
-                goto out;
-
-        args.loc  = loc;
+static int
+client_mknod(call_frame_t *frame, xlator_t *this, loc_t *loc, mode_t mode,
+             dev_t rdev, mode_t umask, dict_t *xdata)
+{
+    int ret = -1;
+    clnt_conf_t *conf = NULL;
+    rpc_clnt_procedure_t *proc = NULL;
+    clnt_args_t args = {
+        0,
+    };
+
+    conf = this->private;
+    if (!conf || !conf->fops)
+        goto out;
+
+    proc = &conf->fops->proctable[GF_FOP_MKNOD];
+    if (proc->fn) {
+        args.loc = loc;
         args.mode = mode;
         args.rdev = rdev;
         args.umask = umask;
         args.xdata = xdata;
-
-        proc = &conf->fops->proctable[GF_FOP_MKNOD];
-
-        if (proc->fn)
-                ret = proc->fn (frame, this, &args);
+        ret = proc->fn(frame, this, &args);
+    }
 out:
-        if (ret)
-                STACK_UNWIND_STRICT (mknod, frame, -1, ENOTCONN,
-                                     NULL, NULL, NULL, NULL, NULL);
+    if (ret)
+        STACK_UNWIND_STRICT(mknod, frame, -1, ENOTCONN, NULL, NULL, NULL, NULL,
+                            NULL);
 
-	return 0;
+    return 0;
 }
 
-
-int
-client_mkdir (call_frame_t *frame, xlator_t *this, loc_t *loc,
-              mode_t mode, mode_t umask, dict_t *xdata)
-{
-        int          ret  = -1;
-        clnt_conf_t *conf = NULL;
-        rpc_clnt_procedure_t *proc = NULL;
-        clnt_args_t  args = {0,};
-
-        conf = this->private;
-        if (!conf || !conf->fops)
-                goto out;
-
-        args.loc  = loc;
+static int
+client_mkdir(call_frame_t *frame, xlator_t *this, loc_t *loc, mode_t mode,
+             mode_t umask, dict_t *xdata)
+{
+    int ret = -1;
+    clnt_conf_t *conf = NULL;
+    rpc_clnt_procedure_t *proc = NULL;
+    clnt_args_t args = {
+        0,
+    };
+
+    conf = this->private;
+    if (!conf || !conf->fops)
+        goto out;
+
+    proc = &conf->fops->proctable[GF_FOP_MKDIR];
+    if (proc->fn) {
+        args.loc = loc;
         args.mode = mode;
         args.umask = umask;
         args.xdata = xdata;
-
-        proc = &conf->fops->proctable[GF_FOP_MKDIR];
-
-        if (proc->fn)
-                ret = proc->fn (frame, this, &args);
+        ret = proc->fn(frame, this, &args);
+    }
 out:
-        if (ret)
-                STACK_UNWIND_STRICT (mkdir, frame, -1, ENOTCONN,
-                                     NULL, NULL, NULL, NULL, NULL);
+    if (ret)
+        STACK_UNWIND_STRICT(mkdir, frame, -1, ENOTCONN, NULL, NULL, NULL, NULL,
+                            NULL);
 
-	return 0;
+    return 0;
 }
 
-
-
-int32_t
-client_unlink (call_frame_t *frame, xlator_t *this, loc_t *loc,
-               int xflag, dict_t *xdata)
+static int32_t
+client_unlink(call_frame_t *frame, xlator_t *this, loc_t *loc, int xflag,
+              dict_t *xdata)
 {
-        int          ret  = -1;
-        clnt_conf_t *conf = NULL;
-        rpc_clnt_procedure_t *proc = NULL;
-        clnt_args_t  args = {0,};
+    int ret = -1;
+    clnt_conf_t *conf = NULL;
+    rpc_clnt_procedure_t *proc = NULL;
+    clnt_args_t args = {
+        0,
+    };
 
-        conf = this->private;
-        if (!conf || !conf->fops)
-                goto out;
+    conf = this->private;
+    if (!conf || !conf->fops)
+        goto out;
 
+    proc = &conf->fops->proctable[GF_FOP_UNLINK];
+    if (proc->fn) {
         args.loc = loc;
         args.xdata = xdata;
         args.flags = xflag;
-
-        proc = &conf->fops->proctable[GF_FOP_UNLINK];
-
-        if (proc->fn)
-                ret = proc->fn (frame, this, &args);
+        ret = proc->fn(frame, this, &args);
+    }
 out:
-        if (ret)
-                STACK_UNWIND_STRICT (unlink, frame, -1, ENOTCONN,
-                                     NULL, NULL, NULL);
+    if (ret)
+        STACK_UNWIND_STRICT(unlink, frame, -1, ENOTCONN, NULL, NULL, NULL);
 
-	return 0;
+    return 0;
 }
 
-int32_t
-client_rmdir (call_frame_t *frame, xlator_t *this, loc_t *loc, int flags,
-              dict_t *xdata)
+static int32_t
+client_rmdir(call_frame_t *frame, xlator_t *this, loc_t *loc, int flags,
+             dict_t *xdata)
 {
-        int          ret  = -1;
-        clnt_conf_t *conf = NULL;
-        rpc_clnt_procedure_t *proc = NULL;
-        clnt_args_t  args = {0,};
+    int ret = -1;
+    clnt_conf_t *conf = NULL;
+    rpc_clnt_procedure_t *proc = NULL;
+    clnt_args_t args = {
+        0,
+    };
 
-        conf = this->private;
-        if (!conf || !conf->fops)
-                goto out;
+    conf = this->private;
+    if (!conf || !conf->fops)
+        goto out;
 
+    proc = &conf->fops->proctable[GF_FOP_RMDIR];
+    if (proc->fn) {
         args.loc = loc;
         args.flags = flags;
         args.xdata = xdata;
-
-        proc = &conf->fops->proctable[GF_FOP_RMDIR];
-
-        if (proc->fn)
-                ret = proc->fn (frame, this, &args);
+        ret = proc->fn(frame, this, &args);
+    }
 out:
-        /* think of avoiding a missing frame */
-        if (ret)
-                STACK_UNWIND_STRICT (rmdir, frame, -1, ENOTCONN,
-                                     NULL, NULL, NULL);
+    /* think of avoiding a missing frame */
+    if (ret)
+        STACK_UNWIND_STRICT(rmdir, frame, -1, ENOTCONN, NULL, NULL, NULL);
 
-	return 0;
+    return 0;
 }
 
-
-int
-client_symlink (call_frame_t *frame, xlator_t *this, const char *linkpath,
-                loc_t *loc, mode_t umask, dict_t *xdata)
-{
-        int          ret  = -1;
-        clnt_conf_t *conf = NULL;
-        rpc_clnt_procedure_t *proc = NULL;
-        clnt_args_t  args = {0,};
-
-        conf = this->private;
-        if (!conf || !conf->fops)
-                goto out;
-
+static int
+client_symlink(call_frame_t *frame, xlator_t *this, const char *linkpath,
+               loc_t *loc, mode_t umask, dict_t *xdata)
+{
+    int ret = -1;
+    clnt_conf_t *conf = NULL;
+    rpc_clnt_procedure_t *proc = NULL;
+    clnt_args_t args = {
+        0,
+    };
+
+    conf = this->private;
+    if (!conf || !conf->fops)
+        goto out;
+
+    proc = &conf->fops->proctable[GF_FOP_SYMLINK];
+    if (proc->fn) {
         args.linkname = linkpath;
-        args.loc      = loc;
-        args.umask    = umask;
-        args.xdata    = xdata;
-
-        proc = &conf->fops->proctable[GF_FOP_SYMLINK];
-
-        if (proc->fn)
-                ret = proc->fn (frame, this, &args);
+        args.loc = loc;
+        args.umask = umask;
+        args.xdata = xdata;
+        ret = proc->fn(frame, this, &args);
+    }
 out:
-        if (ret)
-                STACK_UNWIND_STRICT (symlink, frame, -1, ENOTCONN,
-                                     NULL, NULL, NULL, NULL, NULL);
+    if (ret)
+        STACK_UNWIND_STRICT(symlink, frame, -1, ENOTCONN, NULL, NULL, NULL,
+                            NULL, NULL);
 
-	return 0;
+    return 0;
 }
 
-
-
-int32_t
-client_rename (call_frame_t *frame, xlator_t *this, loc_t *oldloc,
-               loc_t *newloc, dict_t *xdata)
+static int32_t
+client_rename(call_frame_t *frame, xlator_t *this, loc_t *oldloc, loc_t *newloc,
+              dict_t *xdata)
 {
-        int          ret  = -1;
-        clnt_conf_t *conf = NULL;
-        rpc_clnt_procedure_t *proc = NULL;
-        clnt_args_t  args = {0,};
+    int ret = -1;
+    clnt_conf_t *conf = NULL;
+    rpc_clnt_procedure_t *proc = NULL;
+    clnt_args_t args = {
+        0,
+    };
 
-        conf = this->private;
-        if (!conf || !conf->fops)
-                goto out;
+    conf = this->private;
+    if (!conf || !conf->fops)
+        goto out;
 
+    proc = &conf->fops->proctable[GF_FOP_RENAME];
+    if (proc->fn) {
         args.oldloc = oldloc;
         args.newloc = newloc;
-        args.xdata  = xdata;
-
-        proc = &conf->fops->proctable[GF_FOP_RENAME];
-
-        if (proc->fn)
-                ret = proc->fn (frame, this, &args);
+        args.xdata = xdata;
+        ret = proc->fn(frame, this, &args);
+    }
 out:
-        if (ret)
-                STACK_UNWIND_STRICT (rename, frame, -1, ENOTCONN,
-                                     NULL, NULL, NULL, NULL, NULL, NULL);
+    if (ret)
+        STACK_UNWIND_STRICT(rename, frame, -1, ENOTCONN, NULL, NULL, NULL, NULL,
+                            NULL, NULL);
 
-	return 0;
+    return 0;
 }
 
-
-
-int32_t
-client_link (call_frame_t *frame, xlator_t *this, loc_t *oldloc,
-             loc_t *newloc, dict_t *xdata)
+static int32_t
+client_link(call_frame_t *frame, xlator_t *this, loc_t *oldloc, loc_t *newloc,
+            dict_t *xdata)
 {
-        int          ret  = -1;
-        clnt_conf_t *conf = NULL;
-        rpc_clnt_procedure_t *proc = NULL;
-        clnt_args_t  args = {0,};
+    int ret = -1;
+    clnt_conf_t *conf = NULL;
+    rpc_clnt_procedure_t *proc = NULL;
+    clnt_args_t args = {
+        0,
+    };
 
-        conf = this->private;
-        if (!conf || !conf->fops)
-                goto out;
+    conf = this->private;
+    if (!conf || !conf->fops)
+        goto out;
 
+    proc = &conf->fops->proctable[GF_FOP_LINK];
+    if (proc->fn) {
         args.oldloc = oldloc;
         args.newloc = newloc;
         args.xdata = xdata;
-
-        proc = &conf->fops->proctable[GF_FOP_LINK];
-
-        if (proc->fn)
-                ret = proc->fn (frame, this, &args);
+        ret = proc->fn(frame, this, &args);
+    }
 out:
-        if (ret)
-                STACK_UNWIND_STRICT (link, frame, -1, ENOTCONN,
-                                     NULL, NULL, NULL, NULL, NULL);
+    if (ret)
+        STACK_UNWIND_STRICT(link, frame, -1, ENOTCONN, NULL, NULL, NULL, NULL,
+                            NULL);
 
-	return 0;
+    return 0;
 }
 
-
-
-int32_t
-client_create (call_frame_t *frame, xlator_t *this, loc_t *loc,  int32_t flags,
-               mode_t mode, mode_t umask, fd_t *fd, dict_t *xdata)
+static int32_t
+client_create(call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t flags,
+              mode_t mode, mode_t umask, fd_t *fd, dict_t *xdata)
 {
-        int          ret  = -1;
-        clnt_conf_t *conf = NULL;
-        rpc_clnt_procedure_t *proc = NULL;
-        clnt_args_t  args = {0,};
+    int ret = -1;
+    clnt_conf_t *conf = NULL;
+    rpc_clnt_procedure_t *proc = NULL;
+    clnt_args_t args = {
+        0,
+    };
 
-        conf = this->private;
-        if (!conf || !conf->fops)
-                goto out;
+    conf = this->private;
+    if (!conf || !conf->fops)
+        goto out;
 
+    proc = &conf->fops->proctable[GF_FOP_CREATE];
+    if (proc->fn) {
         args.loc = loc;
         args.mode = mode;
         args.fd = fd;
         args.umask = umask;
         args.xdata = xdata;
-
-        if (!conf->filter_o_direct)
-                args.flags = flags;
-        else
-                args.flags = (flags & ~O_DIRECT);
-
-        proc = &conf->fops->proctable[GF_FOP_CREATE];
-
-        if (proc->fn)
-                ret = proc->fn (frame, this, &args);
+        args.flags = flags;
+        client_filter_o_direct(conf, &args.flags);
+        ret = proc->fn(frame, this, &args);
+    }
 out:
-        if (ret)
-                STACK_UNWIND_STRICT (create, frame, -1, ENOTCONN,
-                                     NULL, NULL, NULL, NULL, NULL, NULL);
+    if (ret)
+        STACK_UNWIND_STRICT(create, frame, -1, ENOTCONN, NULL, NULL, NULL, NULL,
+                            NULL, NULL);
 
-	return 0;
+    return 0;
 }
 
-
-
-int32_t
-client_open (call_frame_t *frame, xlator_t *this, loc_t *loc,
-             int32_t flags, fd_t *fd, dict_t *xdata)
+static int32_t
+client_open(call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t flags,
+            fd_t *fd, dict_t *xdata)
 {
-        int          ret  = -1;
-        clnt_conf_t *conf = NULL;
-        rpc_clnt_procedure_t *proc = NULL;
-        clnt_args_t  args = {0,};
+    int ret = -1;
+    clnt_conf_t *conf = NULL;
+    rpc_clnt_procedure_t *proc = NULL;
+    clnt_args_t args = {
+        0,
+    };
 
-        conf = this->private;
-        if (!conf || !conf->fops)
-                goto out;
+    conf = this->private;
+    if (!conf || !conf->fops)
+        goto out;
 
+    proc = &conf->fops->proctable[GF_FOP_OPEN];
+    if (proc->fn) {
         args.loc = loc;
         args.fd = fd;
         args.xdata = xdata;
-
-        if (!conf->filter_o_direct)
-                args.flags = flags;
-        else
-                args.flags = (flags & ~O_DIRECT);
-
-        proc = &conf->fops->proctable[GF_FOP_OPEN];
-
-        if (proc->fn)
-                ret = proc->fn (frame, this, &args);
-
+        args.flags = flags;
+        client_filter_o_direct(conf, &args.flags);
+        ret = proc->fn(frame, this, &args);
+    }
 out:
-        if (ret)
-                STACK_UNWIND_STRICT (open, frame, -1, ENOTCONN, NULL, NULL);
+    if (ret)
+        STACK_UNWIND_STRICT(open, frame, -1, ENOTCONN, NULL, NULL);
 
-	return 0;
+    return 0;
 }
 
-
-
-int32_t
-client_readv (call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
-              off_t offset, uint32_t flags, dict_t *xdata)
+static int32_t
+client_readv(call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
+             off_t offset, uint32_t flags, dict_t *xdata)
 {
-        int          ret  = -1;
-        clnt_conf_t *conf = NULL;
-        rpc_clnt_procedure_t *proc = NULL;
-        clnt_args_t  args = {0,};
+    int ret = -1;
+    clnt_conf_t *conf = NULL;
+    rpc_clnt_procedure_t *proc = NULL;
+    clnt_args_t args = {
+        0,
+    };
 
-        conf = this->private;
-        if (!conf || !conf->fops)
-                goto out;
+    conf = this->private;
+    if (!conf || !conf->fops)
+        goto out;
 
-        args.fd     = fd;
-        args.size   = size;
+    proc = &conf->fops->proctable[GF_FOP_READ];
+    if (proc->fn) {
+        args.fd = fd;
+        args.size = size;
         args.offset = offset;
-        args.flags  = flags;
+        args.flags = flags;
         args.xdata = xdata;
+        client_filter_o_direct(conf, &args.flags);
 
-        proc = &conf->fops->proctable[GF_FOP_READ];
-
-        if (proc->fn)
-                ret = proc->fn (frame, this, &args);
-
+        ret = proc->fn(frame, this, &args);
+    }
 out:
-        if (ret)
-                STACK_UNWIND_STRICT (readv, frame, -1, ENOTCONN,
-                                     NULL, 0, NULL, NULL, NULL);
+    if (ret)
+        STACK_UNWIND_STRICT(readv, frame, -1, ENOTCONN, NULL, 0, NULL, NULL,
+                            NULL);
 
-	return 0;
+    return 0;
 }
 
-
-
-
-int32_t
-client_writev (call_frame_t *frame, xlator_t *this, fd_t *fd,
-               struct iovec *vector, int32_t count, off_t off,
-               uint32_t flags, struct iobref *iobref, dict_t *xdata)
+static int32_t
+client_writev(call_frame_t *frame, xlator_t *this, fd_t *fd,
+              struct iovec *vector, int32_t count, off_t off, uint32_t flags,
+              struct iobref *iobref, dict_t *xdata)
 {
-        int          ret  = -1;
-        clnt_conf_t *conf = NULL;
-        rpc_clnt_procedure_t *proc = NULL;
-        clnt_args_t  args = {0,};
+    int ret = -1;
+    clnt_conf_t *conf = NULL;
+    rpc_clnt_procedure_t *proc = NULL;
+    clnt_args_t args = {
+        0,
+    };
 
-        conf = this->private;
-        if (!conf || !conf->fops)
-                goto out;
+    conf = this->private;
+    if (!conf || !conf->fops)
+        goto out;
 
-        args.fd     = fd;
+    proc = &conf->fops->proctable[GF_FOP_WRITE];
+    if (proc->fn) {
+        args.fd = fd;
         args.vector = vector;
-        args.count  = count;
+        args.count = count;
         args.offset = off;
-        args.size   = iov_length (vector, count);
-        args.flags  = flags;
+        args.size = iov_length(vector, count);
+        args.flags = flags;
         args.iobref = iobref;
         args.xdata = xdata;
-
-        proc = &conf->fops->proctable[GF_FOP_WRITE];
-
-        if (proc->fn)
-                ret = proc->fn (frame, this, &args);
+        client_filter_o_direct(conf, &args.flags);
+        ret = proc->fn(frame, this, &args);
+    }
 out:
-        if (ret)
-                STACK_UNWIND_STRICT (writev, frame, -1, ENOTCONN, NULL, NULL, NULL);
+    if (ret)
+        STACK_UNWIND_STRICT(writev, frame, -1, ENOTCONN, NULL, NULL, NULL);
 
-	return 0;
+    return 0;
 }
 
-
-int32_t
-client_flush (call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *xdata)
+static int32_t
+client_flush(call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *xdata)
 {
-        int          ret  = -1;
-        clnt_conf_t *conf = NULL;
-        rpc_clnt_procedure_t *proc = NULL;
-        clnt_args_t  args = {0,};
+    int ret = -1;
+    clnt_conf_t *conf = NULL;
+    rpc_clnt_procedure_t *proc = NULL;
+    clnt_args_t args = {
+        0,
+    };
 
-        conf = this->private;
-        if (!conf || !conf->fops)
-                goto out;
+    conf = this->private;
+    if (!conf || !conf->fops)
+        goto out;
 
+    proc = &conf->fops->proctable[GF_FOP_FLUSH];
+    if (proc->fn) {
         args.fd = fd;
         args.xdata = xdata;
-
-        proc = &conf->fops->proctable[GF_FOP_FLUSH];
-
-        if (proc->fn)
-                ret = proc->fn (frame, this, &args);
+        ret = proc->fn(frame, this, &args);
+    }
 out:
-        if (ret)
-                STACK_UNWIND_STRICT (flush, frame, -1, ENOTCONN, NULL);
+    if (ret)
+        STACK_UNWIND_STRICT(flush, frame, -1, ENOTCONN, NULL);
 
-	return 0;
+    return 0;
 }
 
-
-
-int32_t
-client_fsync (call_frame_t *frame, xlator_t *this, fd_t *fd,
-              int32_t flags, dict_t *xdata)
+static int32_t
+client_fsync(call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t flags,
+             dict_t *xdata)
 {
-        int          ret  = -1;
-        clnt_conf_t *conf = NULL;
-        rpc_clnt_procedure_t *proc = NULL;
-        clnt_args_t  args = {0,};
+    int ret = -1;
+    clnt_conf_t *conf = NULL;
+    rpc_clnt_procedure_t *proc = NULL;
+    clnt_args_t args = {
+        0,
+    };
 
-        conf = this->private;
-        if (!conf || !conf->fops)
-                goto out;
+    conf = this->private;
+    if (!conf || !conf->fops)
+        goto out;
 
-        args.fd    = fd;
+    proc = &conf->fops->proctable[GF_FOP_FSYNC];
+    if (proc->fn) {
+        args.fd = fd;
         args.flags = flags;
         args.xdata = xdata;
-
-        proc = &conf->fops->proctable[GF_FOP_FSYNC];
-
-        if (proc->fn)
-                ret = proc->fn (frame, this, &args);
+        ret = proc->fn(frame, this, &args);
+    }
 out:
-        if (ret)
-                STACK_UNWIND_STRICT (fsync, frame, -1, ENOTCONN, NULL, NULL, NULL);
+    if (ret)
+        STACK_UNWIND_STRICT(fsync, frame, -1, ENOTCONN, NULL, NULL, NULL);
 
-	return 0;
+    return 0;
 }
 
-
-
-int32_t
-client_fstat (call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *xdata)
+static int32_t
+client_fstat(call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *xdata)
 {
-        int          ret  = -1;
-        clnt_conf_t *conf = NULL;
-        rpc_clnt_procedure_t *proc = NULL;
-        clnt_args_t  args = {0,};
+    int ret = -1;
+    clnt_conf_t *conf = NULL;
+    rpc_clnt_procedure_t *proc = NULL;
+    clnt_args_t args = {
+        0,
+    };
 
-        conf = this->private;
-        if (!conf || !conf->fops)
-                goto out;
+    conf = this->private;
+    if (!conf || !conf->fops)
+        goto out;
 
+    proc = &conf->fops->proctable[GF_FOP_FSTAT];
+    if (proc->fn) {
         args.fd = fd;
         args.xdata = xdata;
-
-        proc = &conf->fops->proctable[GF_FOP_FSTAT];
-
-        if (proc->fn)
-                ret = proc->fn (frame, this, &args);
+        ret = proc->fn(frame, this, &args);
+    }
 out:
-        if (ret)
-                STACK_UNWIND_STRICT (fstat, frame, -1, ENOTCONN, NULL, NULL);
+    if (ret)
+        STACK_UNWIND_STRICT(fstat, frame, -1, ENOTCONN, NULL, NULL);
 
-	return 0;
+    return 0;
 }
 
-
-
-int32_t
-client_opendir (call_frame_t *frame, xlator_t *this, loc_t *loc, fd_t *fd,
-                dict_t *xdata)
+static int32_t
+client_opendir(call_frame_t *frame, xlator_t *this, loc_t *loc, fd_t *fd,
+               dict_t *xdata)
 {
-        int          ret  = -1;
-        clnt_conf_t *conf = NULL;
-        rpc_clnt_procedure_t *proc = NULL;
-        clnt_args_t  args = {0,};
+    int ret = -1;
+    clnt_conf_t *conf = NULL;
+    rpc_clnt_procedure_t *proc = NULL;
+    clnt_args_t args = {
+        0,
+    };
 
-        conf = this->private;
-        if (!conf || !conf->fops)
-                goto out;
+    conf = this->private;
+    if (!conf || !conf->fops)
+        goto out;
 
+    proc = &conf->fops->proctable[GF_FOP_OPENDIR];
+    if (proc->fn) {
         args.loc = loc;
-        args.fd  = fd;
+        args.fd = fd;
         args.xdata = xdata;
-
-        proc = &conf->fops->proctable[GF_FOP_OPENDIR];
-
-        if (proc->fn)
-                ret = proc->fn (frame, this, &args);
+        ret = proc->fn(frame, this, &args);
+    }
 out:
-        if (ret)
-                STACK_UNWIND_STRICT (opendir, frame, -1, ENOTCONN, NULL, NULL);
+    if (ret)
+        STACK_UNWIND_STRICT(opendir, frame, -1, ENOTCONN, NULL, NULL);
 
-	return 0;
+    return 0;
 }
 
-
-
 int32_t
-client_fsyncdir (call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t flags, dict_t *xdata)
+client_fsyncdir(call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t flags,
+                dict_t *xdata)
 {
-        int          ret  = -1;
-        clnt_conf_t *conf = NULL;
-        rpc_clnt_procedure_t *proc = NULL;
-        clnt_args_t  args = {0,};
+    int ret = -1;
+    clnt_conf_t *conf = NULL;
+    rpc_clnt_procedure_t *proc = NULL;
+    clnt_args_t args = {
+        0,
+    };
 
-        conf = this->private;
-        if (!conf || !conf->fops)
-                goto out;
+    conf = this->private;
+    if (!conf || !conf->fops)
+        goto out;
 
-        args.fd    = fd;
+    proc = &conf->fops->proctable[GF_FOP_FSYNCDIR];
+    if (proc->fn) {
+        args.fd = fd;
         args.flags = flags;
         args.xdata = xdata;
-
-        proc = &conf->fops->proctable[GF_FOP_FSYNCDIR];
-
-        if (proc->fn)
-                ret = proc->fn (frame, this, &args);
+        ret = proc->fn(frame, this, &args);
+    }
 out:
-        if (ret)
-                STACK_UNWIND_STRICT (fsyncdir, frame, -1, ENOTCONN, NULL);
+    if (ret)
+        STACK_UNWIND_STRICT(fsyncdir, frame, -1, ENOTCONN, NULL);
 
-	return 0;
+    return 0;
 }
 
-
-
-int32_t
-client_statfs (call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xdata)
+static int32_t
+client_statfs(call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xdata)
 {
-        int          ret  = -1;
-        clnt_conf_t *conf = NULL;
-        rpc_clnt_procedure_t *proc = NULL;
-        clnt_args_t  args = {0,};
+    int ret = -1;
+    clnt_conf_t *conf = NULL;
+    rpc_clnt_procedure_t *proc = NULL;
+    clnt_args_t args = {
+        0,
+    };
 
-        conf = this->private;
-        if (!conf || !conf->fops)
-                goto out;
+    conf = this->private;
+    if (!conf || !conf->fops)
+        goto out;
 
+    proc = &conf->fops->proctable[GF_FOP_STATFS];
+    if (proc->fn) {
         args.loc = loc;
         args.xdata = xdata;
+        ret = proc->fn(frame, this, &args);
+    }
+out:
+    if (ret)
+        STACK_UNWIND_STRICT(statfs, frame, -1, ENOTCONN, NULL, NULL);
 
-        proc = &conf->fops->proctable[GF_FOP_STATFS];
+    return 0;
+}
 
-        if (proc->fn)
-                ret = proc->fn (frame, this, &args);
+static int32_t
+client_copy_file_range(call_frame_t *frame, xlator_t *this, fd_t *fd_in,
+                       off_t off_in, fd_t *fd_out, off_t off_out, size_t len,
+                       uint32_t flags, dict_t *xdata)
+{
+    int ret = -1;
+    clnt_conf_t *conf = NULL;
+    rpc_clnt_procedure_t *proc = NULL;
+    clnt_args_t args = {
+        0,
+    };
+
+    conf = this->private;
+    if (!conf || !conf->fops)
+        goto out;
+
+    proc = &conf->fops->proctable[GF_FOP_COPY_FILE_RANGE];
+    if (proc->fn) {
+        args.fd = fd_in;
+        args.fd_out = fd_out;
+        args.offset = off_in;
+        args.off_out = off_out;
+        args.size = len;
+        args.flags = flags;
+        args.xdata = xdata;
+        ret = proc->fn(frame, this, &args);
+    }
 out:
-        if (ret)
-                STACK_UNWIND_STRICT (statfs, frame, -1, ENOTCONN, NULL, NULL);
+    if (ret)
+        STACK_UNWIND_STRICT(copy_file_range, frame, -1, ENOTCONN, NULL, NULL,
+                            NULL, NULL);
 
-	return 0;
+    return 0;
 }
 
 static gf_boolean_t
-is_client_rpc_init_command (dict_t *dict, xlator_t *this,
-                            char **value)
+is_client_rpc_init_command(dict_t *dict, xlator_t *this, char **value)
 {
-        gf_boolean_t ret      = _gf_false;
-        int          dict_ret = -1;
+    gf_boolean_t ret = _gf_false;
 
-        if (!strstr (this->name, "replace-brick")) {
-                gf_log (this->name, GF_LOG_TRACE, "name is !replace-brick");
-                goto out;
-        }
-        dict_ret = dict_get_str (dict, CLIENT_CMD_CONNECT, value);
-        if (dict_ret) {
-                gf_log (this->name, GF_LOG_TRACE, "key %s not present",
-                        CLIENT_CMD_CONNECT);
-                goto out;
-        }
+    int dict_ret = dict_get_str_sizen(dict, CLIENT_CMD_CONNECT, value);
+    if (dict_ret) {
+        gf_msg_trace(this->name, 0, "key %s not present", CLIENT_CMD_CONNECT);
+        goto out;
+    }
 
-        ret = _gf_true;
+    ret = _gf_true;
 
 out:
-        return ret;
-
+    return ret;
 }
 
 static gf_boolean_t
-is_client_rpc_destroy_command (dict_t *dict, xlator_t *this)
+is_client_rpc_destroy_command(dict_t *dict, xlator_t *this)
 {
-        gf_boolean_t ret      = _gf_false;
-        int          dict_ret = -1;
-        char        *dummy    = NULL;
+    gf_boolean_t ret = _gf_false;
+    int dict_ret = -1;
+    char *dummy = NULL;
 
-        if (strncmp (this->name, "replace-brick", 13)) {
-                gf_log (this->name, GF_LOG_TRACE, "name is !replace-brick");
-                goto out;
-        }
+    if (strncmp(this->name, "replace-brick", 13)) {
+        gf_msg_trace(this->name, 0, "name is !replace-brick");
+        goto out;
+    }
 
-        dict_ret = dict_get_str (dict, CLIENT_CMD_DISCONNECT, &dummy);
-        if (dict_ret) {
-                gf_log (this->name, GF_LOG_TRACE, "key %s not present",
-                        CLIENT_CMD_DISCONNECT);
-                goto out;
-        }
+    dict_ret = dict_get_str_sizen(dict, CLIENT_CMD_DISCONNECT, &dummy);
+    if (dict_ret) {
+        gf_msg_trace(this->name, 0, "key %s not present",
+                     CLIENT_CMD_DISCONNECT);
+        goto out;
+    }
 
-        ret = _gf_true;
+    ret = _gf_true;
 
 out:
-        return ret;
-
+    return ret;
 }
 
-static gf_boolean_t
-client_set_remote_options (char *value, xlator_t *this)
-{
-        char         *dup_value       = NULL;
-        char         *host            = NULL;
-        char         *subvol          = NULL;
-        char         *host_dup        = NULL;
-        char         *subvol_dup      = NULL;
-        char         *remote_port_str = NULL;
-        char         *tmp             = NULL;
-        int           remote_port     = 0;
-        gf_boolean_t  ret             = _gf_false;
-
-        dup_value = gf_strdup (value);
-        host = strtok_r (dup_value, ":", &tmp);
-        subvol = strtok_r (NULL, ":", &tmp);
-        remote_port_str = strtok_r (NULL, ":", &tmp);
-
-        if (!subvol) {
-                gf_log (this->name, GF_LOG_WARNING,
-                        "proper value not passed as subvolume");
-                goto out;
-        }
-
-        host_dup = gf_strdup (host);
+static int
+client_set_remote_options(char *value, xlator_t *this)
+{
+    char *dup_value = NULL;
+    char *host = NULL;
+    char *subvol = NULL;
+    char *host_dup = NULL;
+    char *subvol_dup = NULL;
+    char *remote_port_str = NULL;
+    char *tmp = NULL;
+    int remote_port = 0;
+    int ret = -1;
+
+    dup_value = gf_strdup(value);
+    if (dup_value == NULL) {
+        goto out;
+    }
+    host = strtok_r(dup_value, ":", &tmp);
+    subvol = strtok_r(NULL, ":", &tmp);
+    remote_port_str = strtok_r(NULL, ":", &tmp);
+
+    if (host) {
+        host_dup = gf_strdup(host);
         if (!host_dup) {
-                goto out;
+            goto out;
         }
-
-        ret = dict_set_dynstr (this->options, "remote-host", host_dup);
+        ret = dict_set_dynstr_sizen(this->options, "remote-host", host_dup);
         if (ret) {
-                gf_log (this->name, GF_LOG_WARNING,
-                        "failed to set remote-host with %s", host);
-                goto out;
+            gf_smsg(this->name, GF_LOG_WARNING, 0,
+                    PC_MSG_REMOTE_HOST_SET_FAILED, "host=%s", host, NULL);
+            GF_FREE(host_dup);
+            goto out;
         }
+    }
 
-        subvol_dup = gf_strdup (subvol);
+    if (subvol) {
+        subvol_dup = gf_strdup(subvol);
         if (!subvol_dup) {
-                goto out;
+            goto out;
         }
 
-        ret = dict_set_dynstr (this->options, "remote-subvolume", subvol_dup);
+        ret = dict_set_dynstr_sizen(this->options, "remote-subvolume",
+                                    subvol_dup);
         if (ret) {
-                gf_log (this->name, GF_LOG_WARNING,
-                        "failed to set remote-host with %s", host);
-                goto out;
+            gf_smsg(this->name, GF_LOG_WARNING, 0,
+                    PC_MSG_REMOTE_HOST_SET_FAILED, "host=%s", host, NULL);
+            GF_FREE(subvol_dup);
+            goto out;
         }
+    }
 
-        remote_port = atoi (remote_port_str);
-        GF_ASSERT (remote_port);
+    if (remote_port_str) {
+        remote_port = atoi(remote_port_str);
 
-        ret = dict_set_int32 (this->options, "remote-port",
-                              remote_port);
+        ret = dict_set_int32_sizen(this->options, "remote-port", remote_port);
         if (ret) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "failed to set remote-port to %d", remote_port);
-                goto out;
+            gf_smsg(this->name, GF_LOG_ERROR, 0, PC_MSG_REMOTE_PORT_SET_FAILED,
+                    "remote-port=%d", remote_port, NULL);
+            goto out;
         }
+    }
 
-        ret = _gf_true;
+    ret = 0;
 out:
-        GF_FREE (dup_value);
+    GF_FREE(dup_value);
 
-        return ret;
+    return ret;
 }
 
-
-int32_t
-client_setxattr (call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *dict,
-                 int32_t flags, dict_t *xdata)
+static int32_t
+client_setxattr(call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *dict,
+                int32_t flags, dict_t *xdata)
 {
-        int                   ret         = -1;
-        int                   op_ret      = -1;
-        int                   op_errno    = ENOTCONN;
-        int                   need_unwind = 0;
-        clnt_conf_t          *conf        = NULL;
-        rpc_clnt_procedure_t *proc        = NULL;
-        clnt_args_t           args        = {0,};
-        char                 *value       = NULL;
-
-
-        if (is_client_rpc_init_command (dict, this, &value) == _gf_true) {
-                GF_ASSERT (value);
-                gf_log (this->name, GF_LOG_INFO, "client rpc init command");
-                ret = client_set_remote_options (value, this);
-                if (ret) {
-                        (void) client_destroy_rpc (this);
-                        ret = client_init_rpc (this);
-                }
+    int ret = -1;
+    int op_ret = -1;
+    int op_errno = ENOTCONN;
+    int need_unwind = 0;
+    clnt_conf_t *conf = NULL;
+    rpc_clnt_procedure_t *proc = NULL;
+    clnt_args_t args = {
+        0,
+    };
+    char *value = NULL;
 
-                if (!ret) {
-                        op_ret      = 0;
-                        op_errno    = 0;
-                }
-                need_unwind = 1;
-                goto out;
-        }
-
-        if (is_client_rpc_destroy_command (dict, this) == _gf_true) {
-                gf_log (this->name, GF_LOG_INFO, "client rpc destroy command");
-                ret = client_destroy_rpc (this);
-                if (ret) {
-                        op_ret      = 0;
-                        op_errno    = 0;
-                }
-                need_unwind = 1;
-                goto out;
+    if (is_client_rpc_init_command(dict, this, &value) == _gf_true) {
+        GF_ASSERT(value);
+        gf_smsg(this->name, GF_LOG_INFO, 0, PC_MSG_RPC_INIT, NULL);
+        ret = client_set_remote_options(value, this);
+        if (!ret) {
+            op_ret = 0;
+            op_errno = 0;
         }
+        need_unwind = 1;
+        goto out;
+    }
 
-        conf = this->private;
-        if (!conf || !conf->fops) {
-                op_errno = ENOTCONN;
-                need_unwind = 1;
-                goto out;
+    if (is_client_rpc_destroy_command(dict, this) == _gf_true) {
+        gf_smsg(this->name, GF_LOG_INFO, 0, PC_MSG_RPC_DESTROY, NULL);
+        ret = client_destroy_rpc(this);
+        if (ret) {
+            op_ret = 0;
+            op_errno = 0;
         }
-
-        args.loc   = loc;
+        need_unwind = 1;
+        goto out;
+    }
+
+    conf = this->private;
+    if (!conf || !conf->fops) {
+        op_errno = ENOTCONN;
+        need_unwind = 1;
+        goto out;
+    }
+
+    proc = &conf->fops->proctable[GF_FOP_SETXATTR];
+    if (proc->fn) {
+        args.loc = loc;
         args.xattr = dict;
         args.flags = flags;
         args.xdata = xdata;
-
-        proc = &conf->fops->proctable[GF_FOP_SETXATTR];
-
-        if (proc->fn) {
-                ret = proc->fn (frame, this, &args);
-                if (ret) {
-                        need_unwind = 1;
-                }
+        ret = proc->fn(frame, this, &args);
+        if (ret) {
+            need_unwind = 1;
         }
+    }
 out:
-        if (need_unwind)
-                STACK_UNWIND_STRICT (setxattr, frame, op_ret, op_errno, NULL);
+    if (need_unwind)
+        STACK_UNWIND_STRICT(setxattr, frame, op_ret, op_errno, NULL);
 
-	return 0;
+    return 0;
 }
 
-
-
-int32_t
-client_fsetxattr (call_frame_t *frame, xlator_t *this, fd_t *fd,
-                  dict_t *dict, int32_t flags, dict_t *xdata)
+static int32_t
+client_fsetxattr(call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *dict,
+                 int32_t flags, dict_t *xdata)
 {
-        int          ret  = -1;
-        clnt_conf_t *conf = NULL;
-        rpc_clnt_procedure_t *proc = NULL;
-        clnt_args_t  args = {0,};
+    int ret = -1;
+    clnt_conf_t *conf = NULL;
+    rpc_clnt_procedure_t *proc = NULL;
+    clnt_args_t args = {
+        0,
+    };
 
-        conf = this->private;
-        if (!conf || !conf->fops)
-                goto out;
+    conf = this->private;
+    if (!conf || !conf->fops)
+        goto out;
 
-        args.fd    = fd;
+    proc = &conf->fops->proctable[GF_FOP_FSETXATTR];
+    if (proc->fn) {
+        args.fd = fd;
         args.xattr = dict;
         args.flags = flags;
         args.xdata = xdata;
-
-        proc = &conf->fops->proctable[GF_FOP_FSETXATTR];
-
-        if (proc->fn)
-                ret = proc->fn (frame, this, &args);
+        ret = proc->fn(frame, this, &args);
+    }
 out:
-        if (ret)
-                STACK_UNWIND_STRICT (fsetxattr, frame, -1, ENOTCONN, NULL);
+    if (ret)
+        STACK_UNWIND_STRICT(fsetxattr, frame, -1, ENOTCONN, NULL);
 
-	return 0;
+    return 0;
 }
 
-
-
-
-int32_t
-client_fgetxattr (call_frame_t *frame, xlator_t *this, fd_t *fd,
-                  const char *name, dict_t *xdata)
+static int32_t
+client_fgetxattr(call_frame_t *frame, xlator_t *this, fd_t *fd,
+                 const char *name, dict_t *xdata)
 {
-        int          ret  = -1;
-        clnt_conf_t *conf = NULL;
-        rpc_clnt_procedure_t *proc = NULL;
-        clnt_args_t  args = {0,};
+    int ret = -1;
+    clnt_conf_t *conf = NULL;
+    rpc_clnt_procedure_t *proc = NULL;
+    clnt_args_t args = {
+        0,
+    };
 
-        conf = this->private;
-        if (!conf || !conf->fops)
-                goto out;
+    conf = this->private;
+    if (!conf || !conf->fops)
+        goto out;
 
+    proc = &conf->fops->proctable[GF_FOP_FGETXATTR];
+    if (proc->fn) {
         args.fd = fd;
         args.name = name;
         args.xdata = xdata;
-
-        proc = &conf->fops->proctable[GF_FOP_FGETXATTR];
-
-        if (proc->fn)
-                ret = proc->fn (frame, this, &args);
+        ret = proc->fn(frame, this, &args);
+    }
 out:
-        if (ret)
-                STACK_UNWIND_STRICT (fgetxattr, frame, -1, ENOTCONN, NULL, NULL);
+    if (ret)
+        STACK_UNWIND_STRICT(fgetxattr, frame, -1, ENOTCONN, NULL, NULL);
 
-	return 0;
+    return 0;
 }
 
-
-
-int32_t
-client_getxattr (call_frame_t *frame, xlator_t *this, loc_t *loc,
-                 const char *name, dict_t *xdata)
+static int32_t
+client_getxattr(call_frame_t *frame, xlator_t *this, loc_t *loc,
+                const char *name, dict_t *xdata)
 {
-        int          ret  = -1;
-        clnt_conf_t *conf = NULL;
-        rpc_clnt_procedure_t *proc = NULL;
-        clnt_args_t  args = {0,};
+    int ret = -1;
+    clnt_conf_t *conf = NULL;
+    rpc_clnt_procedure_t *proc = NULL;
+    clnt_args_t args = {
+        0,
+    };
 
-        conf = this->private;
-        if (!conf || !conf->fops)
-                goto out;
+    conf = this->private;
+    if (!conf || !conf->fops)
+        goto out;
 
+    proc = &conf->fops->proctable[GF_FOP_GETXATTR];
+    if (proc->fn) {
         args.name = name;
-        args.loc  = loc;
+        args.loc = loc;
         args.xdata = xdata;
-
-        proc = &conf->fops->proctable[GF_FOP_GETXATTR];
-
-        if (proc->fn)
-                ret = proc->fn (frame, this, &args);
+        ret = proc->fn(frame, this, &args);
+    }
 out:
-        if (ret)
-                STACK_UNWIND_STRICT (getxattr, frame, -1, ENOTCONN, NULL, NULL);
+    if (ret)
+        STACK_UNWIND_STRICT(getxattr, frame, -1, ENOTCONN, NULL, NULL);
 
-	return 0;
+    return 0;
 }
 
-
-
-int32_t
-client_xattrop (call_frame_t *frame, xlator_t *this, loc_t *loc,
-                gf_xattrop_flags_t flags, dict_t *dict, dict_t *xdata)
+static int32_t
+client_xattrop(call_frame_t *frame, xlator_t *this, loc_t *loc,
+               gf_xattrop_flags_t flags, dict_t *dict, dict_t *xdata)
 {
-        int          ret  = -1;
-        clnt_conf_t *conf = NULL;
-        rpc_clnt_procedure_t *proc = NULL;
-        clnt_args_t  args = {0,};
+    int ret = -1;
+    clnt_conf_t *conf = NULL;
+    rpc_clnt_procedure_t *proc = NULL;
+    clnt_args_t args = {
+        0,
+    };
 
-        conf = this->private;
-        if (!conf || !conf->fops)
-                goto out;
+    conf = this->private;
+    if (!conf || !conf->fops)
+        goto out;
 
+    proc = &conf->fops->proctable[GF_FOP_XATTROP];
+    if (proc->fn) {
         args.loc = loc;
         args.flags = flags;
         args.xattr = dict;
         args.xdata = xdata;
-
-        proc = &conf->fops->proctable[GF_FOP_XATTROP];
-
-        if (proc->fn)
-                ret = proc->fn (frame, this, &args);
+        ret = proc->fn(frame, this, &args);
+    }
 out:
-        if (ret)
-                STACK_UNWIND_STRICT (xattrop, frame, -1, ENOTCONN, NULL, NULL);
+    if (ret)
+        STACK_UNWIND_STRICT(xattrop, frame, -1, ENOTCONN, NULL, NULL);
 
-	return 0;
+    return 0;
 }
 
-
-
-int32_t
-client_fxattrop (call_frame_t *frame, xlator_t *this, fd_t *fd,
-                 gf_xattrop_flags_t flags, dict_t *dict, dict_t *xdata)
+static int32_t
+client_fxattrop(call_frame_t *frame, xlator_t *this, fd_t *fd,
+                gf_xattrop_flags_t flags, dict_t *dict, dict_t *xdata)
 {
-        int          ret  = -1;
-        clnt_conf_t *conf = NULL;
-        rpc_clnt_procedure_t *proc = NULL;
-        clnt_args_t  args = {0,};
+    int ret = -1;
+    clnt_conf_t *conf = NULL;
+    rpc_clnt_procedure_t *proc = NULL;
+    clnt_args_t args = {
+        0,
+    };
 
-        conf = this->private;
-        if (!conf || !conf->fops)
-                goto out;
+    conf = this->private;
+    if (!conf || !conf->fops)
+        goto out;
 
+    proc = &conf->fops->proctable[GF_FOP_FXATTROP];
+    if (proc->fn) {
         args.fd = fd;
         args.flags = flags;
         args.xattr = dict;
         args.xdata = xdata;
-
-        proc = &conf->fops->proctable[GF_FOP_FXATTROP];
-
-        if (proc->fn)
-                ret = proc->fn (frame, this, &args);
+        ret = proc->fn(frame, this, &args);
+    }
 out:
-        if (ret)
-                STACK_UNWIND_STRICT (fxattrop, frame, -1, ENOTCONN, NULL, NULL);
+    if (ret)
+        STACK_UNWIND_STRICT(fxattrop, frame, -1, ENOTCONN, NULL, NULL);
 
-	return 0;
+    return 0;
 }
 
-
-
-int32_t
-client_removexattr (call_frame_t *frame, xlator_t *this, loc_t *loc,
-                    const char *name, dict_t *xdata)
+static int32_t
+client_removexattr(call_frame_t *frame, xlator_t *this, loc_t *loc,
+                   const char *name, dict_t *xdata)
 {
-        int          ret  = -1;
-        clnt_conf_t *conf = NULL;
-        rpc_clnt_procedure_t *proc = NULL;
-        clnt_args_t  args = {0,};
+    int ret = -1;
+    clnt_conf_t *conf = NULL;
+    rpc_clnt_procedure_t *proc = NULL;
+    clnt_args_t args = {
+        0,
+    };
 
-        conf = this->private;
-        if (!conf || !conf->fops)
-                goto out;
+    conf = this->private;
+    if (!conf || !conf->fops)
+        goto out;
 
+    proc = &conf->fops->proctable[GF_FOP_REMOVEXATTR];
+    if (proc->fn) {
         args.name = name;
-        args.loc  = loc;
+        args.loc = loc;
         args.xdata = xdata;
-
-        proc = &conf->fops->proctable[GF_FOP_REMOVEXATTR];
-
-        if (proc->fn)
-                ret = proc->fn (frame, this, &args);
+        ret = proc->fn(frame, this, &args);
+    }
 out:
-        if (ret)
-                STACK_UNWIND_STRICT (removexattr, frame, -1, ENOTCONN, NULL);
+    if (ret)
+        STACK_UNWIND_STRICT(removexattr, frame, -1, ENOTCONN, NULL);
 
-	return 0;
+    return 0;
 }
 
-int32_t
-client_fremovexattr (call_frame_t *frame, xlator_t *this, fd_t *fd,
-                     const char *name, dict_t *xdata)
+static int32_t
+client_fremovexattr(call_frame_t *frame, xlator_t *this, fd_t *fd,
+                    const char *name, dict_t *xdata)
 {
-        int          ret  = -1;
-        clnt_conf_t *conf = NULL;
-        rpc_clnt_procedure_t *proc = NULL;
-        clnt_args_t  args = {0,};
+    int ret = -1;
+    clnt_conf_t *conf = NULL;
+    rpc_clnt_procedure_t *proc = NULL;
+    clnt_args_t args = {
+        0,
+    };
 
-        conf = this->private;
-        if (!conf || !conf->fops)
-                goto out;
+    conf = this->private;
+    if (!conf || !conf->fops)
+        goto out;
 
+    proc = &conf->fops->proctable[GF_FOP_FREMOVEXATTR];
+    if (proc->fn) {
         args.name = name;
-        args.fd  = fd;
+        args.fd = fd;
         args.xdata = xdata;
+        ret = proc->fn(frame, this, &args);
+    }
+out:
+    if (ret)
+        STACK_UNWIND_STRICT(fremovexattr, frame, -1, ENOTCONN, NULL);
+
+    return 0;
+}
 
-        proc = &conf->fops->proctable[GF_FOP_FREMOVEXATTR];
+static int32_t
+client_lease(call_frame_t *frame, xlator_t *this, loc_t *loc,
+             struct gf_lease *lease, dict_t *xdata)
+{
+    int ret = -1;
+    clnt_conf_t *conf = NULL;
+    rpc_clnt_procedure_t *proc = NULL;
+    clnt_args_t args = {
+        0,
+    };
+
+    conf = this->private;
+    if (!conf || !conf->fops)
+        goto out;
 
-        if (proc->fn)
-                ret = proc->fn (frame, this, &args);
+    proc = &conf->fops->proctable[GF_FOP_LEASE];
+    if (proc->fn) {
+        args.loc = loc;
+        args.lease = lease;
+        args.xdata = xdata;
+        ret = proc->fn(frame, this, &args);
+    }
 out:
-        if (ret)
-                STACK_UNWIND_STRICT (fremovexattr, frame, -1, ENOTCONN, NULL);
+    if (ret)
+        STACK_UNWIND_STRICT(lk, frame, -1, ENOTCONN, NULL, NULL);
 
-	return 0;
+    return 0;
 }
 
-int32_t
-client_lk (call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t cmd,
-           struct gf_flock *lock, dict_t *xdata)
+static int32_t
+client_lk(call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t cmd,
+          struct gf_flock *lock, dict_t *xdata)
 {
-        int          ret  = -1;
-        clnt_conf_t *conf = NULL;
-        rpc_clnt_procedure_t *proc = NULL;
-        clnt_args_t  args = {0,};
+    int ret = -1;
+    clnt_conf_t *conf = NULL;
+    rpc_clnt_procedure_t *proc = NULL;
+    clnt_args_t args = {
+        0,
+    };
 
-        conf = this->private;
-        if (!conf || !conf->fops)
-                goto out;
+    conf = this->private;
+    if (!conf || !conf->fops)
+        goto out;
 
-        args.fd    = fd;
-        args.cmd   = cmd;
+    proc = &conf->fops->proctable[GF_FOP_LK];
+    if (proc->fn) {
+        args.fd = fd;
+        args.cmd = cmd;
         args.flock = lock;
         args.xdata = xdata;
-
-        proc = &conf->fops->proctable[GF_FOP_LK];
-
-        if (proc->fn)
-                ret = proc->fn (frame, this, &args);
+        ret = proc->fn(frame, this, &args);
+    }
 out:
-        if (ret)
-                STACK_UNWIND_STRICT (lk, frame, -1, ENOTCONN, NULL, NULL);
+    if (ret)
+        STACK_UNWIND_STRICT(lk, frame, -1, ENOTCONN, NULL, NULL);
 
-	return 0;
+    return 0;
 }
 
-
-int32_t
-client_inodelk (call_frame_t *frame, xlator_t *this, const char *volume,
-                loc_t *loc, int32_t cmd, struct gf_flock *lock, dict_t *xdata)
+static int32_t
+client_inodelk(call_frame_t *frame, xlator_t *this, const char *volume,
+               loc_t *loc, int32_t cmd, struct gf_flock *lock, dict_t *xdata)
 {
-        int          ret  = -1;
-        clnt_conf_t *conf = NULL;
-        rpc_clnt_procedure_t *proc = NULL;
-        clnt_args_t  args = {0,};
+    int ret = -1;
+    clnt_conf_t *conf = NULL;
+    rpc_clnt_procedure_t *proc = NULL;
+    clnt_args_t args = {
+        0,
+    };
 
-        conf = this->private;
-        if (!conf || !conf->fops)
-                goto out;
+    conf = this->private;
+    if (!conf || !conf->fops)
+        goto out;
 
-        args.loc    = loc;
-        args.cmd    = cmd;
-        args.flock  = lock;
+    proc = &conf->fops->proctable[GF_FOP_INODELK];
+    if (proc->fn) {
+        args.loc = loc;
+        args.cmd = cmd;
+        args.flock = lock;
         args.volume = volume;
         args.xdata = xdata;
-
-        proc = &conf->fops->proctable[GF_FOP_INODELK];
-
-        if (proc->fn)
-                ret = proc->fn (frame, this, &args);
+        ret = proc->fn(frame, this, &args);
+    }
 out:
-        if (ret)
-                STACK_UNWIND_STRICT (inodelk, frame, -1, ENOTCONN, NULL);
+    if (ret)
+        STACK_UNWIND_STRICT(inodelk, frame, -1, ENOTCONN, NULL);
 
-	return 0;
+    return 0;
 }
 
-
-
-int32_t
-client_finodelk (call_frame_t *frame, xlator_t *this, const char *volume,
-                 fd_t *fd, int32_t cmd, struct gf_flock *lock, dict_t *xdata)
+static int32_t
+client_finodelk(call_frame_t *frame, xlator_t *this, const char *volume,
+                fd_t *fd, int32_t cmd, struct gf_flock *lock, dict_t *xdata)
 {
-        int          ret  = -1;
-        clnt_conf_t *conf = NULL;
-        rpc_clnt_procedure_t *proc = NULL;
-        clnt_args_t  args = {0,};
+    int ret = -1;
+    clnt_conf_t *conf = NULL;
+    rpc_clnt_procedure_t *proc = NULL;
+    clnt_args_t args = {
+        0,
+    };
 
-        conf = this->private;
-        if (!conf || !conf->fops)
-                goto out;
+    conf = this->private;
+    if (!conf || !conf->fops)
+        goto out;
 
-        args.fd     = fd;
-        args.cmd    = cmd;
-        args.flock  = lock;
+    proc = &conf->fops->proctable[GF_FOP_FINODELK];
+    if (proc->fn) {
+        args.fd = fd;
+        args.cmd = cmd;
+        args.flock = lock;
         args.volume = volume;
         args.xdata = xdata;
-
-        proc = &conf->fops->proctable[GF_FOP_FINODELK];
-
-        if (proc->fn)
-                ret = proc->fn (frame, this, &args);
+        ret = proc->fn(frame, this, &args);
+    }
 out:
-        if (ret)
-                STACK_UNWIND_STRICT (finodelk, frame, -1, ENOTCONN, NULL);
+    if (ret)
+        STACK_UNWIND_STRICT(finodelk, frame, -1, ENOTCONN, NULL);
 
-	return 0;
+    return 0;
 }
 
-
-int32_t
-client_entrylk (call_frame_t *frame, xlator_t *this, const char *volume,
-                loc_t *loc, const char *basename, entrylk_cmd cmd,
-                entrylk_type type, dict_t *xdata)
+static int32_t
+client_entrylk(call_frame_t *frame, xlator_t *this, const char *volume,
+               loc_t *loc, const char *basename, entrylk_cmd cmd,
+               entrylk_type type, dict_t *xdata)
 {
-        int          ret  = -1;
-        clnt_conf_t *conf = NULL;
-        rpc_clnt_procedure_t *proc = NULL;
-        clnt_args_t  args = {0,};
+    int ret = -1;
+    clnt_conf_t *conf = NULL;
+    rpc_clnt_procedure_t *proc = NULL;
+    clnt_args_t args = {
+        0,
+    };
 
-        conf = this->private;
-        if (!conf || !conf->fops)
-                goto out;
+    conf = this->private;
+    if (!conf || !conf->fops)
+        goto out;
 
-        args.loc          = loc;
-        args.basename     = basename;
-        args.type         = type;
-        args.volume       = volume;
-        args.cmd_entrylk  = cmd;
+    proc = &conf->fops->proctable[GF_FOP_ENTRYLK];
+    if (proc->fn) {
+        args.loc = loc;
+        args.basename = basename;
+        args.type = type;
+        args.volume = volume;
+        args.cmd_entrylk = cmd;
         args.xdata = xdata;
-
-        proc = &conf->fops->proctable[GF_FOP_ENTRYLK];
-
-        if (proc->fn)
-                ret = proc->fn (frame, this, &args);
+        ret = proc->fn(frame, this, &args);
+    }
 out:
-        if (ret)
-                STACK_UNWIND_STRICT (entrylk, frame, -1, ENOTCONN, NULL);
+    if (ret)
+        STACK_UNWIND_STRICT(entrylk, frame, -1, ENOTCONN, NULL);
 
-	return 0;
+    return 0;
 }
 
-
-
-int32_t
-client_fentrylk (call_frame_t *frame, xlator_t *this, const char *volume,
-                 fd_t *fd, const char *basename, entrylk_cmd cmd,
-                 entrylk_type type, dict_t *xdata)
+static int32_t
+client_fentrylk(call_frame_t *frame, xlator_t *this, const char *volume,
+                fd_t *fd, const char *basename, entrylk_cmd cmd,
+                entrylk_type type, dict_t *xdata)
 {
-        int          ret  = -1;
-        clnt_conf_t *conf = NULL;
-        rpc_clnt_procedure_t *proc = NULL;
-        clnt_args_t  args = {0,};
+    int ret = -1;
+    clnt_conf_t *conf = NULL;
+    rpc_clnt_procedure_t *proc = NULL;
+    clnt_args_t args = {
+        0,
+    };
 
-        conf = this->private;
-        if (!conf || !conf->fops)
-                goto out;
+    conf = this->private;
+    if (!conf || !conf->fops)
+        goto out;
 
-        args.fd           = fd;
-        args.basename     = basename;
-        args.type         = type;
-        args.volume       = volume;
-        args.cmd_entrylk  = cmd;
+    proc = &conf->fops->proctable[GF_FOP_FENTRYLK];
+    if (proc->fn) {
+        args.fd = fd;
+        args.basename = basename;
+        args.type = type;
+        args.volume = volume;
+        args.cmd_entrylk = cmd;
         args.xdata = xdata;
-
-        proc = &conf->fops->proctable[GF_FOP_FENTRYLK];
-
-        if (proc->fn)
-                ret = proc->fn (frame, this, &args);
+        ret = proc->fn(frame, this, &args);
+    }
 out:
-        if (ret)
-                STACK_UNWIND_STRICT (fentrylk, frame, -1, ENOTCONN, NULL);
+    if (ret)
+        STACK_UNWIND_STRICT(fentrylk, frame, -1, ENOTCONN, NULL);
 
-	return 0;
+    return 0;
 }
 
-
-int32_t
-client_rchecksum (call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
-                  int32_t len, dict_t *xdata)
+static int32_t
+client_rchecksum(call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
+                 int32_t len, dict_t *xdata)
 {
-        int          ret  = -1;
-        clnt_conf_t *conf = NULL;
-        rpc_clnt_procedure_t *proc = NULL;
-        clnt_args_t  args = {0,};
+    int ret = -1;
+    clnt_conf_t *conf = NULL;
+    rpc_clnt_procedure_t *proc = NULL;
+    clnt_args_t args = {
+        0,
+    };
 
-        conf = this->private;
-        if (!conf || !conf->fops)
-                goto out;
+    conf = this->private;
+    if (!conf || !conf->fops)
+        goto out;
 
+    proc = &conf->fops->proctable[GF_FOP_RCHECKSUM];
+    if (proc->fn) {
         args.fd = fd;
         args.offset = offset;
         args.len = len;
         args.xdata = xdata;
-
-        proc = &conf->fops->proctable[GF_FOP_RCHECKSUM];
-
-        if (proc->fn)
-                ret = proc->fn (frame, this, &args);
+        ret = proc->fn(frame, this, &args);
+    }
 out:
-        if (ret)
-                STACK_UNWIND_STRICT (rchecksum, frame, -1, ENOTCONN, 0, NULL, NULL);
+    if (ret)
+        STACK_UNWIND_STRICT(rchecksum, frame, -1, ENOTCONN, 0, NULL, NULL);
 
-	return 0;
+    return 0;
 }
 
 int32_t
-client_readdir (call_frame_t *frame, xlator_t *this, fd_t *fd,
-                size_t size, off_t off, dict_t *xdata)
-{
-        int          ret  = -1;
-        clnt_conf_t *conf = NULL;
-        rpc_clnt_procedure_t *proc = NULL;
-        clnt_args_t  args = {0,};
-
-        conf = this->private;
-        if (!conf || !conf->fops)
-                goto out;
-
+client_readdir(call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
+               off_t off, dict_t *xdata)
+{
+    int ret = -1;
+    clnt_conf_t *conf = NULL;
+    rpc_clnt_procedure_t *proc = NULL;
+    clnt_args_t args = {
+        0,
+    };
+
+    conf = this->private;
+    if (!conf || !conf->fops)
+        goto out;
+
+    proc = &conf->fops->proctable[GF_FOP_READDIR];
+    if (proc->fn) {
         if (off != 0)
-                off = gf_dirent_orig_offset(this, off);
+            off = gf_dirent_orig_offset(this, off);
 
         args.fd = fd;
         args.size = size;
         args.offset = off;
         args.xdata = xdata;
-
-        proc = &conf->fops->proctable[GF_FOP_READDIR];
-
-        if (proc->fn)
-                ret = proc->fn (frame, this, &args);
+        ret = proc->fn(frame, this, &args);
+    }
 out:
-        if (ret)
-                STACK_UNWIND_STRICT (readdir, frame, -1, ENOTCONN, NULL, NULL);
+    if (ret)
+        STACK_UNWIND_STRICT(readdir, frame, -1, ENOTCONN, NULL, NULL);
 
-	return 0;
+    return 0;
 }
 
-
-int32_t
-client_readdirp (call_frame_t *frame, xlator_t *this, fd_t *fd,
-                 size_t size, off_t off, dict_t *dict)
+static int32_t
+client_readdirp(call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
+                off_t off, dict_t *dict)
 {
-        int          ret  = -1;
-        clnt_conf_t *conf = NULL;
-        rpc_clnt_procedure_t *proc = NULL;
-        clnt_args_t  args = {0,};
+    int ret = -1;
+    clnt_conf_t *conf = NULL;
+    rpc_clnt_procedure_t *proc = NULL;
+    clnt_args_t args = {
+        0,
+    };
 
-        conf = this->private;
-        if (!conf || !conf->fops)
-                goto out;
+    conf = this->private;
+    if (!conf || !conf->fops)
+        goto out;
 
+    proc = &conf->fops->proctable[GF_FOP_READDIRP];
+    if (proc->fn) {
         if (off != 0)
-                off = gf_dirent_orig_offset(this, off);
+            off = gf_dirent_orig_offset(this, off);
 
         args.fd = fd;
         args.size = size;
         args.offset = off;
         args.xdata = dict;
-
-        proc = &conf->fops->proctable[GF_FOP_READDIRP];
-
-        if (proc->fn)
-                ret = proc->fn (frame, this, &args);
+        ret = proc->fn(frame, this, &args);
+    }
 out:
-        if (ret)
-                STACK_UNWIND_STRICT (readdirp, frame, -1, ENOTCONN, NULL, NULL);
+    if (ret)
+        STACK_UNWIND_STRICT(readdirp, frame, -1, ENOTCONN, NULL, NULL);
 
-	return 0;
+    return 0;
 }
 
-
-int32_t
-client_setattr (call_frame_t *frame, xlator_t *this, loc_t *loc,
-                struct iatt *stbuf, int32_t valid, dict_t *xdata)
+static int32_t
+client_setattr(call_frame_t *frame, xlator_t *this, loc_t *loc,
+               struct iatt *stbuf, int32_t valid, dict_t *xdata)
 {
-        int          ret  = -1;
-        clnt_conf_t *conf = NULL;
-        rpc_clnt_procedure_t *proc = NULL;
-        clnt_args_t  args = {0,};
+    int ret = -1;
+    clnt_conf_t *conf = NULL;
+    rpc_clnt_procedure_t *proc = NULL;
+    clnt_args_t args = {
+        0,
+    };
 
-        conf = this->private;
-        if (!conf || !conf->fops)
-                goto out;
+    conf = this->private;
+    if (!conf || !conf->fops)
+        goto out;
 
+    proc = &conf->fops->proctable[GF_FOP_SETATTR];
+    if (proc->fn) {
         args.loc = loc;
         args.stbuf = stbuf;
         args.valid = valid;
         args.xdata = xdata;
-
-        proc = &conf->fops->proctable[GF_FOP_SETATTR];
-
-        if (proc->fn)
-                ret = proc->fn (frame, this, &args);
+        ret = proc->fn(frame, this, &args);
+    }
 out:
-        if (ret)
-                STACK_UNWIND_STRICT (setattr, frame, -1, ENOTCONN, NULL, NULL, NULL);
+    if (ret)
+        STACK_UNWIND_STRICT(setattr, frame, -1, ENOTCONN, NULL, NULL, NULL);
 
-	return 0;
+    return 0;
 }
 
-int32_t
-client_fsetattr (call_frame_t *frame, xlator_t *this, fd_t *fd,
-                 struct iatt *stbuf, int32_t valid, dict_t *xdata)
+static int32_t
+client_fsetattr(call_frame_t *frame, xlator_t *this, fd_t *fd,
+                struct iatt *stbuf, int32_t valid, dict_t *xdata)
 {
-        int          ret  = -1;
-        clnt_conf_t *conf = NULL;
-        rpc_clnt_procedure_t *proc = NULL;
-        clnt_args_t  args = {0,};
+    int ret = -1;
+    clnt_conf_t *conf = NULL;
+    rpc_clnt_procedure_t *proc = NULL;
+    clnt_args_t args = {
+        0,
+    };
 
-        conf = this->private;
-        if (!conf || !conf->fops)
-                goto out;
+    conf = this->private;
+    if (!conf || !conf->fops)
+        goto out;
 
+    proc = &conf->fops->proctable[GF_FOP_FSETATTR];
+    if (proc->fn) {
         args.fd = fd;
         args.stbuf = stbuf;
         args.valid = valid;
         args.xdata = xdata;
-
-        proc = &conf->fops->proctable[GF_FOP_FSETATTR];
-
-        if (proc->fn)
-                ret = proc->fn (frame, this, &args);
+        ret = proc->fn(frame, this, &args);
+    }
 out:
-        if (ret)
-                STACK_UNWIND_STRICT (fsetattr, frame, -1, ENOTCONN, NULL, NULL, NULL);
+    if (ret)
+        STACK_UNWIND_STRICT(fsetattr, frame, -1, ENOTCONN, NULL, NULL, NULL);
 
-	return 0;
+    return 0;
 }
 
-int32_t
+static int32_t
 client_fallocate(call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t mode,
-		 off_t offset, size_t len, dict_t *xdata)
+                 off_t offset, size_t len, dict_t *xdata)
 {
-        int          ret  = -1;
-        clnt_conf_t *conf = NULL;
-        rpc_clnt_procedure_t *proc = NULL;
-        clnt_args_t  args = {0,};
+    int ret = -1;
+    clnt_conf_t *conf = NULL;
+    rpc_clnt_procedure_t *proc = NULL;
+    clnt_args_t args = {
+        0,
+    };
 
-        conf = this->private;
-        if (!conf || !conf->fops)
-                goto out;
+    conf = this->private;
+    if (!conf || !conf->fops)
+        goto out;
 
+    proc = &conf->fops->proctable[GF_FOP_FALLOCATE];
+    if (proc->fn) {
         args.fd = fd;
-	args.flags = mode;
-	args.offset = offset;
-	args.size = len;
+        args.flags = mode;
+        args.offset = offset;
+        args.size = len;
         args.xdata = xdata;
-
-        proc = &conf->fops->proctable[GF_FOP_FALLOCATE];
-
-        if (proc->fn)
-                ret = proc->fn (frame, this, &args);
+        ret = proc->fn(frame, this, &args);
+    }
 out:
-        if (ret)
-                STACK_UNWIND_STRICT (fallocate, frame, -1, ENOTCONN, NULL, NULL, NULL);
+    if (ret)
+        STACK_UNWIND_STRICT(fallocate, frame, -1, ENOTCONN, NULL, NULL, NULL);
 
-	return 0;
+    return 0;
 }
 
-int32_t
+static int32_t
 client_discard(call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
-	       size_t len, dict_t *xdata)
+               size_t len, dict_t *xdata)
 {
-        int          ret  = -1;
-        clnt_conf_t *conf = NULL;
-        rpc_clnt_procedure_t *proc = NULL;
-        clnt_args_t  args = {0,};
+    int ret = -1;
+    clnt_conf_t *conf = NULL;
+    rpc_clnt_procedure_t *proc = NULL;
+    clnt_args_t args = {
+        0,
+    };
 
-        conf = this->private;
-        if (!conf || !conf->fops)
-                goto out;
+    conf = this->private;
+    if (!conf || !conf->fops)
+        goto out;
 
+    proc = &conf->fops->proctable[GF_FOP_DISCARD];
+    if (proc->fn) {
         args.fd = fd;
-	args.offset = offset;
-	args.size = len;
+        args.offset = offset;
+        args.size = len;
         args.xdata = xdata;
-
-        proc = &conf->fops->proctable[GF_FOP_DISCARD];
-
-        if (proc->fn)
-                ret = proc->fn (frame, this, &args);
+        ret = proc->fn(frame, this, &args);
+    }
 out:
-        if (ret)
-                STACK_UNWIND_STRICT(discard, frame, -1, ENOTCONN, NULL, NULL, NULL);
+    if (ret)
+        STACK_UNWIND_STRICT(discard, frame, -1, ENOTCONN, NULL, NULL, NULL);
 
-	return 0;
+    return 0;
 }
 
-int32_t
+static int32_t
 client_zerofill(call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
-               off_t len, dict_t *xdata)
+                off_t len, dict_t *xdata)
 {
-        int          ret              = -1;
-        clnt_conf_t *conf             = NULL;
-        rpc_clnt_procedure_t *proc    = NULL;
-        clnt_args_t  args             = {0,};
+    int ret = -1;
+    clnt_conf_t *conf = NULL;
+    rpc_clnt_procedure_t *proc = NULL;
+    clnt_args_t args = {
+        0,
+    };
 
-        conf = this->private;
-        if (!conf || !conf->fops)
-                goto out;
+    conf = this->private;
+    if (!conf || !conf->fops)
+        goto out;
 
+    proc = &conf->fops->proctable[GF_FOP_ZEROFILL];
+    if (proc->fn) {
         args.fd = fd;
         args.offset = offset;
         args.size = len;
         args.xdata = xdata;
+        ret = proc->fn(frame, this, &args);
+    }
+out:
+    if (ret)
+        STACK_UNWIND_STRICT(zerofill, frame, -1, ENOTCONN, NULL, NULL, NULL);
+
+    return 0;
+}
+
+static int32_t
+client_ipc(call_frame_t *frame, xlator_t *this, int32_t op, dict_t *xdata)
+{
+    int ret = -1;
+    clnt_conf_t *conf = NULL;
+    rpc_clnt_procedure_t *proc = NULL;
+    clnt_args_t args = {
+        0,
+    };
 
-        proc = &conf->fops->proctable[GF_FOP_ZEROFILL];
+    conf = this->private;
+    if (!conf || !conf->fops)
+        goto out;
 
-        if (proc->fn)
-                ret = proc->fn (frame, this, &args);
+    proc = &conf->fops->proctable[GF_FOP_IPC];
+    if (proc->fn) {
+        args.cmd = op;
+        args.xdata = xdata;
+        ret = proc->fn(frame, this, &args);
+    }
 out:
-        if (ret)
-                STACK_UNWIND_STRICT(zerofill, frame, -1, ENOTCONN,
-                                    NULL, NULL, NULL);
+    if (ret)
+        STACK_UNWIND_STRICT(ipc, frame, -1, ENOTCONN, NULL);
 
-        return 0;
+    return 0;
 }
 
-
-int32_t
-client_ipc (call_frame_t *frame, xlator_t *this, int32_t op, dict_t *xdata)
+static int32_t
+client_seek(call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
+            gf_seek_what_t what, dict_t *xdata)
 {
-        int          ret              = -1;
-        clnt_conf_t *conf             = NULL;
-        rpc_clnt_procedure_t *proc    = NULL;
-        clnt_args_t  args             = {0,};
+    int ret = -1;
+    clnt_conf_t *conf = NULL;
+    rpc_clnt_procedure_t *proc = NULL;
+    clnt_args_t args = {
+        0,
+    };
 
-        conf = this->private;
-        if (!conf || !conf->fops)
-                goto out;
+    conf = this->private;
+    if (!conf || !conf->fops)
+        goto out;
 
-        args.cmd = op;
+    proc = &conf->fops->proctable[GF_FOP_SEEK];
+    if (proc->fn) {
+        args.fd = fd;
+        args.offset = offset;
+        args.what = what;
         args.xdata = xdata;
+        ret = proc->fn(frame, this, &args);
+    }
+out:
+    if (ret)
+        STACK_UNWIND_STRICT(seek, frame, -1, ENOTCONN, 0, NULL);
 
-        proc = &conf->fops->proctable[GF_FOP_IPC];
-        if (!proc) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "rpc procedure not found for %s",
-                        gf_fop_list[GF_FOP_IPC]);
-                goto out;
-        }
-        if (proc->fn)
-                ret = proc->fn (frame, this, &args);
+    return 0;
+}
+
+static int32_t
+client_getactivelk(call_frame_t *frame, xlator_t *this, loc_t *loc,
+                   dict_t *xdata)
+{
+    int ret = -1;
+    clnt_conf_t *conf = NULL;
+    rpc_clnt_procedure_t *proc = NULL;
+    clnt_args_t args = {
+        0,
+    };
+
+    conf = this->private;
+    if (!conf || !conf->fops)
+        goto out;
+
+    proc = &conf->fops->proctable[GF_FOP_GETACTIVELK];
+    if (proc->fn) {
+        args.loc = loc;
+        args.xdata = xdata;
+        ret = proc->fn(frame, this, &args);
+    }
 out:
-        if (ret)
-                STACK_UNWIND_STRICT(ipc, frame, -1, ENOTCONN, NULL);
+    if (ret)
+        STACK_UNWIND_STRICT(getactivelk, frame, -1, ENOTCONN, NULL, NULL);
 
-        return 0;
+    return 0;
 }
 
+static int32_t
+client_setactivelk(call_frame_t *frame, xlator_t *this, loc_t *loc,
+                   lock_migration_info_t *locklist, dict_t *xdata)
+{
+    int ret = -1;
+    clnt_conf_t *conf = NULL;
+    rpc_clnt_procedure_t *proc = NULL;
+    clnt_args_t args = {
+        0,
+    };
 
-int32_t
-client_getspec (call_frame_t *frame, xlator_t *this, const char *key,
-                int32_t flags)
+    conf = this->private;
+    if (!conf || !conf->fops)
+        goto out;
+
+    proc = &conf->fops->proctable[GF_FOP_SETACTIVELK];
+    if (proc->fn) {
+        args.loc = loc;
+        args.xdata = xdata;
+        args.locklist = locklist;
+        ret = proc->fn(frame, this, &args);
+    }
+out:
+    if (ret)
+        STACK_UNWIND_STRICT(setactivelk, frame, -1, ENOTCONN, NULL);
+
+    return 0;
+}
+
+static int32_t
+client_getspec(call_frame_t *frame, xlator_t *this, const char *key,
+               int32_t flags)
 {
-        int          ret  = -1;
-        clnt_conf_t *conf = NULL;
-        rpc_clnt_procedure_t *proc = NULL;
-        clnt_args_t  args = {0,};
+    int ret = -1;
+    clnt_conf_t *conf = NULL;
+    rpc_clnt_procedure_t *proc = NULL;
+    clnt_args_t args = {
+        0,
+    };
 
-        conf = this->private;
-        if (!conf || !conf->fops || !conf->handshake)
-                goto out;
+    conf = this->private;
+    if (!conf || !conf->fops || !conf->handshake)
+        goto out;
 
+    /* For all other xlators, getspec is an fop, hence its in fops table */
+    proc = &conf->fops->proctable[GF_FOP_GETSPEC];
+    if (proc->fn) {
         args.name = key;
         args.flags = flags;
-
-        /* For all other xlators, getspec is an fop, hence its in fops table */
-        proc = &conf->fops->proctable[GF_FOP_GETSPEC];
-
-        if (proc->fn) {
-                /* But at protocol level, this is handshake */
-                ret = proc->fn (frame, this, &args);
-        }
+        /* But at protocol level, this is handshake */
+        ret = proc->fn(frame, this, &args);
+    }
 out:
-        if (ret)
-                STACK_UNWIND_STRICT (getspec, frame, -1, EINVAL, NULL);
+    if (ret)
+        STACK_UNWIND_STRICT(getspec, frame, -1, EINVAL, NULL);
 
-	return 0;
+    return 0;
 }
 
-
-int
-client_mark_fd_bad (xlator_t *this)
+static int32_t
+client_compound(call_frame_t *frame, xlator_t *this, void *data, dict_t *xdata)
 {
-        clnt_conf_t            *conf = NULL;
-        clnt_fd_ctx_t          *tmp = NULL, *fdctx = NULL;
+    int ret = -1;
+    clnt_conf_t *conf = NULL;
+    compound_args_t *args = data;
+    rpc_clnt_procedure_t *proc = NULL;
 
-        conf = this->private;
+    conf = this->private;
+    if (!conf || !conf->fops)
+        goto out;
 
-        pthread_mutex_lock (&conf->lock);
-        {
-                list_for_each_entry_safe (fdctx, tmp, &conf->saved_fds,
-                                          sfd_pos) {
-                        fdctx->remote_fd = -1;
-                }
-        }
-        pthread_mutex_unlock (&conf->lock);
+    proc = &conf->fops->proctable[GF_FOP_COMPOUND];
+    if (proc->fn) {
+        args->xdata = xdata;
+        ret = proc->fn(frame, this, args);
+    }
+out:
+    if (ret)
+        STACK_UNWIND_STRICT(compound, frame, -1, ENOTCONN, NULL, NULL);
 
-        return 0;
+    return 0;
 }
 
+int32_t
+client_namelink(call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xdata)
+{
+    int32_t ret = -1;
+    clnt_conf_t *conf = NULL;
+    clnt_args_t args = {
+        0,
+    };
+    rpc_clnt_procedure_t *proc = NULL;
 
-int
-client_rpc_notify (struct rpc_clnt *rpc, void *mydata, rpc_clnt_event_t event,
-                   void *data)
-{
-        xlator_t *this = NULL;
-        char *handshake = NULL;
-        clnt_conf_t  *conf = NULL;
-        int ret = 0;
-
-        this = mydata;
-        if (!this || !this->private) {
-                gf_log ("client", GF_LOG_ERROR,
-                        (this != NULL) ?
-                        "private structure of the xlator is NULL":
-                        "xlator is NULL");
-                goto out;
-        }
+    conf = this->private;
+    if (!conf || !conf->fops || !conf->handshake)
+        goto out;
 
-        conf = this->private;
+    proc = &conf->fops->proctable[GF_FOP_NAMELINK];
+    if (proc->fn) {
+        args.loc = loc;
+        args.xdata = xdata;
+        ret = proc->fn(frame, this, &args);
+    }
+out:
+    if (ret)
+        STACK_UNWIND_STRICT(namelink, frame, -1, EINVAL, NULL, NULL, NULL);
+    return 0;
+}
 
-        switch (event) {
-        case RPC_CLNT_CONNECT:
-        {
-                conf->connected = 1;
-                // connect happened, send 'get_supported_versions' mop
-                ret = dict_get_str (this->options, "disable-handshake",
-                                    &handshake);
-
-                gf_log (this->name, GF_LOG_DEBUG, "got RPC_CLNT_CONNECT");
-
-                if ((ret < 0) || (strcasecmp (handshake, "on"))) {
-                        ret = client_handshake (this, rpc);
-                        if (ret)
-                                gf_log (this->name, GF_LOG_WARNING,
-                                        "handshake msg returned %d", ret);
-                } else {
-                        //conf->rpc->connected = 1;
-                        ret = client_notify_dispatch_uniq (this,
-                                                           GF_EVENT_CHILD_UP,
-                                                           NULL);
-                        if (ret)
-                                gf_log (this->name, GF_LOG_INFO,
-                                        "CHILD_UP notify failed");
-                }
+static int32_t
+client_icreate(call_frame_t *frame, xlator_t *this, loc_t *loc, mode_t mode,
+               dict_t *xdata)
+{
+    int32_t ret = -1;
+    clnt_conf_t *conf = NULL;
+    clnt_args_t args = {
+        0,
+    };
+    rpc_clnt_procedure_t *proc = NULL;
 
-                /* Cancel grace timer if set */
-                pthread_mutex_lock (&conf->lock);
-                {
-                        conf->grace_timer_needed = _gf_true;
+    conf = this->private;
+    if (!conf || !conf->fops || !conf->handshake)
+        goto out;
 
-                        if (conf->grace_timer) {
-                                gf_log (this->name, GF_LOG_WARNING,
-                                        "Cancelling the grace timer");
+    proc = &conf->fops->proctable[GF_FOP_ICREATE];
+    if (proc->fn) {
+        args.loc = loc;
+        args.mode = mode;
+        args.xdata = xdata;
+        ret = proc->fn(frame, this, &args);
+    }
+out:
+    if (ret)
+        STACK_UNWIND_STRICT(icreate, frame, -1, EINVAL, NULL, NULL, NULL);
+    return 0;
+}
 
-                                gf_timer_call_cancel (this->ctx,
-                                                      conf->grace_timer);
+static int32_t
+client_put(call_frame_t *frame, xlator_t *this, loc_t *loc, mode_t mode,
+           mode_t umask, uint32_t flags, struct iovec *vector, int32_t count,
+           off_t off, struct iobref *iobref, dict_t *xattr, dict_t *xdata)
+{
+    int ret = -1;
+    clnt_conf_t *conf = NULL;
+    rpc_clnt_procedure_t *proc = NULL;
+    clnt_args_t args = {
+        0,
+    };
 
-                                conf->grace_timer      = NULL;
-                        }
-                }
-                pthread_mutex_unlock (&conf->lock);
+    conf = this->private;
+    if (!conf || !conf->fops)
+        goto out;
 
-                break;
-        }
-        case RPC_CLNT_DISCONNECT:
-                if (!conf->lk_heal)
-                        client_mark_fd_bad (this);
-                else
-                        client_register_grace_timer (this, conf);
-
-                if (!conf->skip_notify) {
-                        if (conf->connected) {
-                               gf_log (this->name,
-                                        ((!conf->disconnect_err_logged)
-                                        ? GF_LOG_INFO : GF_LOG_DEBUG),
-                                        "disconnected from %s. Client process "
-                                        "will keep trying to connect to "
-                                        "glusterd until brick's port is "
-                                        "available",
-                                  conf->rpc->conn.name);
-
-                                if (conf->portmap_err_logged)
-                                        conf->disconnect_err_logged = 1;
-                        }
-
-                        /* If the CHILD_DOWN event goes to parent xlator
-                           multiple times, the logic of parent xlator notify
-                           may get screwed up.. (eg. CHILD_MODIFIED event in
-                           replicate), hence make sure events which are passed
-                           to parent are genuine */
-                        ret = client_notify_dispatch_uniq (this,
-                                                           GF_EVENT_CHILD_DOWN,
-                                                           NULL);
-                        if (ret)
-                                gf_log (this->name, GF_LOG_INFO,
-                                        "CHILD_DOWN notify failed");
-
-                } else {
-                        if (conf->connected)
-                                gf_log (this->name, GF_LOG_DEBUG,
-                                        "disconnected (skipped notify)");
-                }
+    proc = &conf->fops->proctable[GF_FOP_PUT];
+    if (proc->fn) {
+        args.loc = loc;
+        args.mode = mode;
+        args.umask = umask;
+        args.flags = flags;
+        args.vector = vector;
+        args.count = count;
+        args.offset = off;
+        args.size = iov_length(vector, count);
+        args.iobref = iobref;
+        args.xattr = xattr;
+        args.xdata = xdata;
 
-                conf->connected = 0;
-                conf->skip_notify = 0;
+        client_filter_o_direct(conf, &args.flags);
+        ret = proc->fn(frame, this, &args);
+    }
+out:
+    if (ret)
+        STACK_UNWIND_STRICT(put, frame, -1, ENOTCONN, NULL, NULL, NULL, NULL,
+                            NULL);
 
-                if (conf->quick_reconnect) {
-                        conf->quick_reconnect = 0;
-                        rpc_clnt_start (rpc);
+    return 0;
+}
 
-                } else {
-                        rpc->conn.config.remote_port = 0;
+static void
+client_mark_fd_bad(xlator_t *this)
+{
+    clnt_conf_t *conf = this->private;
+    clnt_fd_ctx_t *tmp = NULL, *fdctx = NULL;
 
-                }
+    pthread_spin_lock(&conf->fd_lock);
+    {
+        list_for_each_entry_safe(fdctx, tmp, &conf->saved_fds, sfd_pos)
+        {
+            fdctx->remote_fd = -1;
+        }
+    }
+    pthread_spin_unlock(&conf->fd_lock);
+}
 
-                break;
+int
+client_rpc_notify(struct rpc_clnt *rpc, void *mydata, rpc_clnt_event_t event,
+                  void *data)
+{
+    xlator_t *this = NULL;
+    clnt_conf_t *conf = NULL;
+    gf_boolean_t is_parent_down = _gf_false;
+    int ret = 0;
+
+    this = mydata;
+    if (!this || !this->private) {
+        gf_smsg("client", GF_LOG_ERROR, EINVAL, PC_MSG_XLATOR_NULL,
+                (this != NULL) ? "private structue" : "", NULL);
+        goto out;
+    }
+
+    conf = this->private;
+
+    switch (event) {
+        case RPC_CLNT_PING: {
+            if (conf->connection_to_brick) {
+                ret = default_notify(this, GF_EVENT_CHILD_PING, data);
+                if (ret)
+                    gf_log(this->name, GF_LOG_INFO, "CHILD_PING notify failed");
+                conf->last_sent_event = GF_EVENT_CHILD_PING;
+            }
+            break;
+        }
+        case RPC_CLNT_CONNECT: {
+            conf->can_log_disconnect = 1;
+            // connect happened, send 'get_supported_versions' mop
+
+            gf_msg_debug(this->name, 0, "got RPC_CLNT_CONNECT");
 
+            ret = client_handshake(this, rpc);
+            if (ret)
+                gf_smsg(this->name, GF_LOG_WARNING, 0, PC_MSG_HANDSHAKE_RETURN,
+                        "ret=%d", ret, NULL);
+            break;
+        }
+        case RPC_CLNT_DISCONNECT:
+            gf_msg_debug(this->name, 0, "got RPC_CLNT_DISCONNECT");
+
+            client_mark_fd_bad(this);
+
+            if (!conf->skip_notify) {
+                if (conf->can_log_disconnect) {
+                    if (!conf->disconnect_err_logged) {
+                        gf_smsg(this->name, GF_LOG_INFO, 0,
+                                PC_MSG_CLIENT_DISCONNECTED, "conn-name=%s",
+                                conf->rpc->conn.name, NULL);
+                    } else {
+                        gf_msg_debug(this->name, 0,
+                                     "disconnected from %s. "
+                                     "Client process will keep"
+                                     " trying to connect to "
+                                     "glusterd until brick's "
+                                     "port is available",
+                                     conf->rpc->conn.name);
+                    }
+                    if (conf->portmap_err_logged)
+                        conf->disconnect_err_logged = 1;
+                }
+                /*
+                 * Once we complete the child down notification,
+                 * There is a chance that the graph might get freed,
+                 * So it is not safe to access any xlator contens
+                 * So here we are checking whether the parent is down
+                 * or not.
+                 */
+                pthread_mutex_lock(&conf->lock);
+                {
+                    is_parent_down = conf->parent_down;
+                }
+                pthread_mutex_unlock(&conf->lock);
+
+                /* If the CHILD_DOWN event goes to parent xlator
+                   multiple times, the logic of parent xlator notify
+                   may get screwed up.. (eg. CHILD_MODIFIED event in
+                   replicate), hence make sure events which are passed
+                   to parent are genuine */
+                ret = client_notify_dispatch_uniq(this, GF_EVENT_CHILD_DOWN,
+                                                  rpc);
+                if (is_parent_down) {
+                    /* If parent is down, then there should not be any
+                     * operation after a child down.
+                     */
+                    goto out;
+                }
+                if (ret)
+                    gf_smsg(this->name, GF_LOG_INFO, 0,
+                            PC_MSG_CHILD_DOWN_NOTIFY_FAILED, NULL);
+
+            } else {
+                if (conf->can_log_disconnect)
+                    gf_msg_debug(this->name, 0,
+                                 "disconnected (skipped notify)");
+            }
+
+            conf->connected = 0;
+            conf->can_log_disconnect = 0;
+            conf->skip_notify = 0;
+
+            if (conf->quick_reconnect) {
+                conf->connection_to_brick = _gf_true;
+                conf->quick_reconnect = 0;
+                rpc_clnt_cleanup_and_start(rpc);
+
+            } else {
+                rpc->conn.config.remote_port = 0;
+                conf->connection_to_brick = _gf_false;
+            }
+            break;
         case RPC_CLNT_DESTROY:
-                ret = client_fini_complete (this);
-                break;
+            ret = client_fini_complete(this);
+            break;
 
         default:
-                gf_log (this->name, GF_LOG_TRACE,
-                        "got some other RPC event %d", event);
+            gf_msg_trace(this->name, 0, "got some other RPC event %d", event);
 
-                break;
-        }
+            break;
+    }
 
 out:
-        return 0;
+    return 0;
 }
 
-
 int
-notify (xlator_t *this, int32_t event, void *data, ...)
+notify(xlator_t *this, int32_t event, void *data, ...)
 {
-        clnt_conf_t     *conf  = NULL;
+    clnt_conf_t *conf = NULL;
+    glusterfs_graph_t *graph = this->graph;
+    int ret = -1;
 
-        conf = this->private;
-        if (!conf)
-                return 0;
+    conf = this->private;
+    if (!conf)
+        return 0;
 
-        switch (event) {
-        case GF_EVENT_PARENT_UP:
-        {
-                gf_log (this->name, GF_LOG_INFO,
-                        "parent translators are ready, attempting connect "
-                        "on transport");
+    switch (event) {
+        case GF_EVENT_PARENT_UP: {
+            gf_smsg(this->name, GF_LOG_INFO, 0, PC_MSG_PARENT_UP, NULL);
 
-                rpc_clnt_start (conf->rpc);
-                break;
+            rpc_clnt_start(conf->rpc);
+            break;
         }
 
         case GF_EVENT_PARENT_DOWN:
-                gf_log (this->name, GF_LOG_INFO,
-                        "current graph is no longer active, destroying "
-                        "rpc_client ");
+            gf_smsg(this->name, GF_LOG_INFO, 0, PC_MSG_PARENT_DOWN, NULL);
+
+            pthread_mutex_lock(&conf->lock);
+            {
+                conf->parent_down = 1;
+            }
+            pthread_mutex_unlock(&conf->lock);
 
-                pthread_mutex_lock (&conf->lock);
+            ret = rpc_clnt_disable(conf->rpc);
+            if (ret == -1 && graph) {
+                pthread_mutex_lock(&graph->mutex);
                 {
-                        conf->parent_down = 1;
+                    graph->parent_down++;
+                    if (graph->parent_down ==
+                        graph_total_client_xlator(graph)) {
+                        graph->used = 0;
+                        pthread_cond_broadcast(&graph->child_down_cond);
+                    }
                 }
-                pthread_mutex_unlock (&conf->lock);
-
-                rpc_clnt_disable (conf->rpc);
-                break;
+                pthread_mutex_unlock(&graph->mutex);
+            }
+            break;
 
         default:
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "got %d, calling default_notify ()", event);
+            gf_msg_debug(this->name, 0, "got %d, calling default_notify ()",
+                         event);
 
-                default_notify (this, event, data);
-                conf->last_sent_event = event;
-                break;
-        }
+            default_notify(this, event, data);
+            conf->last_sent_event = event;
+            break;
+    }
 
-        return 0;
+    return 0;
 }
 
-int
-client_check_remote_host (xlator_t *this, dict_t *options)
+static int
+client_check_remote_host(xlator_t *this, dict_t *options)
 {
-        char           *remote_host     = NULL;
-        int             ret             = -1;
+    char *remote_host = NULL;
+    int ret = -1;
 
-        ret = dict_get_str (options, "remote-host", &remote_host);
-        if (ret < 0) {
-                gf_log (this->name, GF_LOG_INFO, "Remote host is not set. "
-                        "Assuming the volfile server as remote host.");
+    ret = dict_get_str_sizen(options, "remote-host", &remote_host);
+    if (ret < 0) {
+        gf_smsg(this->name, GF_LOG_INFO, EINVAL, PC_MSG_REMOTE_HOST_NOT_SET,
+                NULL);
 
-                if (!this->ctx->cmd_args.volfile_server) {
-                        gf_log (this->name, GF_LOG_ERROR,
-                                                 "No remote host to connect.");
-                        goto out;
-                }
+        if (!this->ctx->cmd_args.volfile_server) {
+            gf_smsg(this->name, GF_LOG_ERROR, EINVAL, PC_MSG_NOREMOTE_HOST,
+                    NULL);
+            goto out;
+        }
 
-                ret = dict_set_str (options, "remote-host",
-                                    this->ctx->cmd_args.volfile_server);
-                if (ret == -1) {
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "Failed to set the remote host");
-                        goto out;
-                }
+        ret = dict_set_str_sizen(options, "remote-host",
+                                 this->ctx->cmd_args.volfile_server);
+        if (ret == -1) {
+            gf_smsg(this->name, GF_LOG_ERROR, 0, PC_MSG_REMOTE_HOST_SET_FAILED,
+                    NULL);
+            goto out;
         }
+    }
 
-        ret = 0;
+    ret = 0;
 out:
-        return ret;
+    return ret;
 }
 
-int
-build_client_config (xlator_t *this, clnt_conf_t *conf)
+static int
+build_client_config(xlator_t *this, clnt_conf_t *conf)
 {
-        int                     ret = -1;
+    int ret = -1;
 
-       if (!conf)
-                goto out;
+    GF_OPTION_INIT("frame-timeout", conf->rpc_conf.rpc_timeout, int32, out);
 
-        GF_OPTION_INIT ("frame-timeout", conf->rpc_conf.rpc_timeout,
-                        int32, out);
+    GF_OPTION_INIT("remote-port", conf->rpc_conf.remote_port, int32, out);
 
-        GF_OPTION_INIT ("remote-port", conf->rpc_conf.remote_port,
-                        int32, out);
+    GF_OPTION_INIT("ping-timeout", conf->opt.ping_timeout, int32, out);
 
-        GF_OPTION_INIT ("ping-timeout", conf->opt.ping_timeout,
-                        int32, out);
+    GF_OPTION_INIT("remote-subvolume", conf->opt.remote_subvolume, path, out);
+    if (!conf->opt.remote_subvolume)
+        gf_smsg(this->name, GF_LOG_WARNING, EINVAL,
+                PC_MSG_REMOTE_SUBVOL_NOT_GIVEN, NULL);
 
-        GF_OPTION_INIT ("remote-subvolume", conf->opt.remote_subvolume,
-                        path, out);
-        if (!conf->opt.remote_subvolume)
-                gf_log (this->name, GF_LOG_WARNING,
-                        "option 'remote-subvolume' not given");
+    GF_OPTION_INIT("filter-O_DIRECT", conf->filter_o_direct, bool, out);
 
-        GF_OPTION_INIT ("filter-O_DIRECT", conf->filter_o_direct,
-                        bool, out);
+    GF_OPTION_INIT("send-gids", conf->send_gids, bool, out);
 
-        GF_OPTION_INIT ("send-gids", conf->send_gids, bool, out);
+    GF_OPTION_INIT("testing.old-protocol", conf->old_protocol, bool, out);
+    GF_OPTION_INIT("strict-locks", conf->strict_locks, bool, out);
 
-        conf->client_id = glusterfs_leaf_position(this);
+    conf->client_id = glusterfs_leaf_position(this);
 
-        ret = client_check_remote_host (this, this->options);
-        if (ret)
-                goto out;
+    ret = client_check_remote_host(this, this->options);
+    if (ret)
+        goto out;
 
-        ret = 0;
+    ret = 0;
 out:
-        return ret;
+    return ret;
 }
 
-
 int32_t
-mem_acct_init (xlator_t *this)
+mem_acct_init(xlator_t *this)
 {
-        int     ret = -1;
-
-        if (!this)
-                return ret;
-
-        ret = xlator_mem_acct_init (this, gf_client_mt_end + 1);
-
-        if (ret != 0) {
-                gf_log (this->name, GF_LOG_ERROR, "Memory accounting init"
-                        "failed");
-                return ret;
-        }
+    int ret = -1;
 
+    if (!this)
         return ret;
-}
-
-int
-client_destroy_rpc (xlator_t *this)
-{
-        int          ret  = -1;
-        clnt_conf_t *conf = NULL;
-
-        conf = this->private;
-        if (!conf)
-                goto out;
-
-        if (conf->rpc) {
-                /* cleanup the saved-frames before last unref */
-                rpc_clnt_connection_cleanup (&conf->rpc->conn);
-
-                conf->rpc = rpc_clnt_unref (conf->rpc);
-                ret = 0;
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "Client rpc conn destroyed");
-                goto out;
-        }
 
-        gf_log (this->name, GF_LOG_WARNING,
-                "RPC destroy called on already destroyed "
-                "connection");
+    ret = xlator_mem_acct_init(this, gf_client_mt_end + 1);
 
-out:
+    if (ret != 0) {
+        gf_smsg(this->name, GF_LOG_ERROR, ENOMEM, PC_MSG_NO_MEMORY, NULL);
         return ret;
+    }
+
+    return ret;
 }
 
-int
-client_init_rpc (xlator_t *this)
+static int
+client_destroy_rpc(xlator_t *this)
 {
-        int          ret  = -1;
-        clnt_conf_t *conf = NULL;
-
-        conf = this->private;
-
-        if (conf->rpc) {
-                gf_log (this->name, GF_LOG_WARNING,
-                        "client rpc already init'ed");
-                ret = -1;
-                goto out;
-        }
-
-        conf->rpc = rpc_clnt_new (this->options, this->ctx, this->name, 0);
-        if (!conf->rpc) {
-                gf_log (this->name, GF_LOG_ERROR, "failed to initialize RPC");
-                goto out;
-        }
-
-        ret = rpc_clnt_register_notify (conf->rpc, client_rpc_notify, this);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR, "failed to register notify");
-                goto out;
-        }
+    int ret = -1;
+    clnt_conf_t *conf = NULL;
 
-        conf->handshake = &clnt_handshake_prog;
-        conf->dump      = &clnt_dump_prog;
+    conf = this->private;
+    if (!conf)
+        goto out;
 
-        ret = rpcclnt_cbk_program_register (conf->rpc, &gluster_cbk_prog,
-                                            this);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "failed to register callback program");
-                goto out;
-        }
+    if (conf->rpc) {
+        /* cleanup the saved-frames before last unref */
+        rpc_clnt_connection_cleanup(&conf->rpc->conn);
 
+        conf->rpc = rpc_clnt_unref(conf->rpc);
         ret = 0;
+        gf_msg_debug(this->name, 0, "Client rpc conn destroyed");
+        goto out;
+    }
+
+    gf_smsg(this->name, GF_LOG_WARNING, 0, PC_MSG_RPC_INVALID_CALL, NULL);
 
-        gf_log (this->name, GF_LOG_DEBUG, "client init successful");
 out:
-        return ret;
+    return ret;
 }
 
-
-int
-client_init_grace_timer (xlator_t *this, dict_t *options,
-                         clnt_conf_t *conf)
+static int
+client_init_rpc(xlator_t *this)
 {
-        char      timestr[64]    = {0,};
-        char     *lk_heal        = NULL;
-        int32_t   ret            = -1;
-        int32_t   grace_timeout  = -1;
+    int ret = -1;
+    clnt_conf_t *conf = NULL;
 
-        GF_VALIDATE_OR_GOTO ("client", this, out);
-        GF_VALIDATE_OR_GOTO (this->name, options, out);
-        GF_VALIDATE_OR_GOTO (this->name, conf, out);
+    conf = this->private;
 
-        conf->lk_heal = _gf_false;
+    if (conf->rpc) {
+        gf_smsg(this->name, GF_LOG_WARNING, 0, PC_MSG_RPC_INITED_ALREADY, NULL);
+        ret = -1;
+        goto out;
+    }
 
-        ret = dict_get_str (options, "lk-heal", &lk_heal);
-        if (!ret)
-                gf_string2boolean (lk_heal, &conf->lk_heal);
+    conf->rpc = rpc_clnt_new(this->options, this, this->name, 0);
+    if (!conf->rpc) {
+        gf_smsg(this->name, GF_LOG_ERROR, 0, PC_MSG_RPC_INIT_FAILED, NULL);
+        goto out;
+    }
 
-        gf_log (this->name, GF_LOG_DEBUG, "lk-heal = %s",
-                (conf->lk_heal) ? "on" : "off");
+    ret = rpc_clnt_register_notify(conf->rpc, client_rpc_notify, this);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_ERROR, 0, PC_MSG_RPC_NOTIFY_FAILED, NULL);
+        goto out;
+    }
 
-        ret = dict_get_int32 (options, "grace-timeout", &grace_timeout);
-        if (!ret)
-                conf->grace_ts.tv_sec = grace_timeout;
-        else
-                conf->grace_ts.tv_sec = 10;
+    conf->handshake = &clnt_handshake_prog;
+    conf->dump = &clnt_dump_prog;
 
-        conf->grace_ts.tv_nsec  = 0;
+    ret = rpcclnt_cbk_program_register(conf->rpc, &gluster_cbk_prog, this);
+    if (ret) {
+        gf_smsg(this->name, GF_LOG_ERROR, 0, PC_MSG_RPC_CBK_FAILED, NULL);
+        goto out;
+    }
 
-        gf_time_fmt (timestr, sizeof timestr, conf->grace_ts.tv_sec,
-                     gf_timefmt_s);
-        gf_log (this->name, GF_LOG_DEBUG, "Client grace timeout value = %s",
-                timestr);
+    ret = 0;
 
-        ret = 0;
+    gf_msg_debug(this->name, 0, "client init successful");
 out:
-        return ret;
+    return ret;
 }
 
-int
-client_check_event_threads (xlator_t *this, clnt_conf_t *conf, int32_t old,
-                            int32_t new)
+static int
+client_check_event_threads(xlator_t *this, clnt_conf_t *conf, int32_t old,
+                           int32_t new)
 {
-        if (old == new)
-                return 0;
+    if (old == new)
+        return 0;
 
-        conf->event_threads = new;
-        return event_reconfigure_threads (this->ctx->event_pool,
-                                          conf->event_threads);
+    conf->event_threads = new;
+    return gf_event_reconfigure_threads(this->ctx->event_pool,
+                                        conf->event_threads);
 }
 
 int
-reconfigure (xlator_t *this, dict_t *options)
-{
-	clnt_conf_t *conf              = NULL;
-	int          ret               = -1;
-        int          subvol_ret        = 0;
-        char        *old_remote_subvol = NULL;
-        char        *new_remote_subvol = NULL;
-        char        *old_remote_host   = NULL;
-        char        *new_remote_host   = NULL;
-        int32_t      new_nthread       = 0;
-
-	conf = this->private;
-
-        GF_OPTION_RECONF ("frame-timeout", conf->rpc_conf.rpc_timeout,
-                          options, int32, out);
-
-        GF_OPTION_RECONF ("ping-timeout", conf->opt.ping_timeout,
-                          options, int32, out);
-
-        GF_OPTION_RECONF ("event-threads", new_nthread, options,
-                          int32, out);
-        ret = client_check_event_threads (this, conf, conf->event_threads,
-                                          new_nthread);
-        if (ret)
-                goto out;
-
-        ret = client_check_remote_host (this, options);
-        if (ret)
-                goto out;
-
-        subvol_ret = dict_get_str (this->options, "remote-host",
-                                   &old_remote_host);
-
+reconfigure(xlator_t *this, dict_t *options)
+{
+    clnt_conf_t *conf = NULL;
+    int ret = -1;
+    int subvol_ret = 0;
+    char *old_remote_subvol = NULL;
+    char *new_remote_subvol = NULL;
+    char *old_remote_host = NULL;
+    char *new_remote_host = NULL;
+    int32_t new_nthread = 0;
+    struct rpc_clnt_config rpc_config = {
+        0,
+    };
+
+    conf = this->private;
+
+    GF_OPTION_RECONF("frame-timeout", conf->rpc_conf.rpc_timeout, options,
+                     int32, out);
+
+    GF_OPTION_RECONF("ping-timeout", rpc_config.ping_timeout, options, int32,
+                     out);
+
+    GF_OPTION_RECONF("event-threads", new_nthread, options, int32, out);
+    ret = client_check_event_threads(this, conf, conf->event_threads,
+                                     new_nthread);
+    if (ret)
+        goto out;
+
+    ret = client_check_remote_host(this, options);
+    if (ret)
+        goto out;
+
+    subvol_ret = dict_get_str_sizen(this->options, "remote-host",
+                                    &old_remote_host);
+
+    if (subvol_ret == 0) {
+        subvol_ret = dict_get_str_sizen(options, "remote-host",
+                                        &new_remote_host);
         if (subvol_ret == 0) {
-                subvol_ret = dict_get_str (options, "remote-host",
-                                           &new_remote_host);
-                if (subvol_ret == 0) {
-                        if (strcmp (old_remote_host, new_remote_host)) {
-                                ret = 1;
-                                goto out;
-                        }
-                }
+            if (strcmp(old_remote_host, new_remote_host)) {
+                ret = 1;
+                goto out;
+            }
         }
+    }
 
-        subvol_ret = dict_get_str (this->options, "remote-subvolume",
-                                   &old_remote_subvol);
+    subvol_ret = dict_get_str_sizen(this->options, "remote-subvolume",
+                                    &old_remote_subvol);
 
+    if (subvol_ret == 0) {
+        subvol_ret = dict_get_str_sizen(options, "remote-subvolume",
+                                        &new_remote_subvol);
         if (subvol_ret == 0) {
-                subvol_ret = dict_get_str (options, "remote-subvolume",
-                                           &new_remote_subvol);
-                if (subvol_ret == 0) {
-                        if (strcmp (old_remote_subvol, new_remote_subvol)) {
-                                ret = 1;
-                                goto out;
-                        }
-                }
+            if (strcmp(old_remote_subvol, new_remote_subvol)) {
+                ret = 1;
+                goto out;
+            }
         }
+    }
 
-        GF_OPTION_RECONF ("filter-O_DIRECT", conf->filter_o_direct,
-                          options, bool, out);
+    /* Reconfiguring client xlator's @rpc with new frame-timeout
+     * and ping-timeout */
+    rpc_clnt_reconfig(conf->rpc, &rpc_config);
 
-        GF_OPTION_RECONF ("send-gids", conf->send_gids, options, bool, out);
+    GF_OPTION_RECONF("filter-O_DIRECT", conf->filter_o_direct, options, bool,
+                     out);
 
-        ret = client_init_grace_timer (this, options, conf);
-        if (ret)
-                goto out;
+    GF_OPTION_RECONF("send-gids", conf->send_gids, options, bool, out);
+    GF_OPTION_RECONF("strict-locks", conf->strict_locks, options, bool, out);
 
-        ret = 0;
+    ret = 0;
 out:
-	return ret;
-
+    return ret;
 }
 
-
 int
-init (xlator_t *this)
+init(xlator_t *this)
 {
-        int          ret = -1;
-        clnt_conf_t *conf = NULL;
-
-        if (this->children) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "FATAL: client protocol translator cannot have any "
-                        "subvolumes");
-                goto out;
-        }
+    int ret = -1;
+    clnt_conf_t *conf = NULL;
 
-        if (!this->parents) {
-                gf_log (this->name, GF_LOG_WARNING,
-                        "Volume is dangling. ");
-        }
+    if (this->children) {
+        gf_smsg(this->name, GF_LOG_ERROR, EINVAL, PC_MSG_FATAL_CLIENT_PROTOCOL,
+                NULL);
+        goto out;
+    }
 
-        conf = GF_CALLOC (1, sizeof (*conf), gf_client_mt_clnt_conf_t);
-        if (!conf)
-                goto out;
+    if (!this->parents) {
+        gf_smsg(this->name, GF_LOG_WARNING, EINVAL, PC_MSG_VOL_DANGLING, NULL);
+    }
 
-        pthread_mutex_init (&conf->lock, NULL);
-        INIT_LIST_HEAD (&conf->saved_fds);
+    conf = GF_CALLOC(1, sizeof(*conf), gf_client_mt_clnt_conf_t);
+    if (!conf)
+        goto out;
 
-        /* Initialize parameters for lock self healing*/
-        conf->lk_version         = 1;
-        conf->grace_timer        = NULL;
-        conf->grace_timer_needed = _gf_true;
+    pthread_mutex_init(&conf->lock, NULL);
+    pthread_cond_init(&conf->fini_complete_cond, NULL);
+    pthread_spin_init(&conf->fd_lock, 0);
+    INIT_LIST_HEAD(&conf->saved_fds);
 
-        /* Set event threads to the configured default */
-        GF_OPTION_INIT("event-threads", conf->event_threads, int32, out);
-        ret = client_check_event_threads (this, conf, STARTING_EVENT_THREADS,
-                                          conf->event_threads);
-        if (ret)
-                goto out;
+    conf->child_up = _gf_false;
 
-        ret = client_init_grace_timer (this, this->options, conf);
-        if (ret)
-                goto out;
+    /* Set event threads to the configured default */
+    GF_OPTION_INIT("event-threads", conf->event_threads, int32, out);
+    ret = client_check_event_threads(this, conf, STARTING_EVENT_THREADS,
+                                     conf->event_threads);
+    if (ret)
+        goto out;
 
-        LOCK_INIT (&conf->rec_lock);
+    LOCK_INIT(&conf->rec_lock);
 
-        conf->last_sent_event = -1; /* To start with we don't have any events */
+    conf->last_sent_event = -1; /* To start with we don't have any events */
 
-        this->private = conf;
+    this->private = conf;
 
-        /* If it returns -1, then its a failure, if it returns +1 we need
-           have to understand that 'this' is subvolume of a xlator which,
-           will set the remote host and remote subvolume in a setxattr
-           call.
-        */
+    /* If it returns -1, then its a failure, if it returns +1 we need
+       have to understand that 'this' is subvolume of a xlator which,
+       will set the remote host and remote subvolume in a setxattr
+       call.
+    */
 
-        ret = build_client_config (this, conf);
-        if (ret == -1)
-                goto out;
+    ret = build_client_config(this, conf);
+    if (ret == -1)
+        goto out;
 
-        if (ret) {
-                ret = 0;
-                goto out;
-        }
+    if (ret) {
+        ret = 0;
+        goto out;
+    }
 
-        this->local_pool = mem_pool_new (clnt_local_t, 64);
-        if (!this->local_pool) {
-                ret = -1;
-                gf_log (this->name, GF_LOG_ERROR,
-                        "failed to create local_t's memory pool");
-                goto out;
-        }
+    this->local_pool = mem_pool_new(clnt_local_t, 64);
+    if (!this->local_pool) {
+        ret = -1;
+        gf_smsg(this->name, GF_LOG_ERROR, ENOMEM, PC_MSG_CREATE_MEM_POOL_FAILED,
+                NULL);
+        goto out;
+    }
 
-        ret = client_init_rpc (this);
+    ret = client_init_rpc(this);
 out:
-        if (ret)
-                this->fini (this);
+    if (ret)
+        this->fini(this);
 
-        return ret;
+    return ret;
 }
 
 void
-fini (xlator_t *this)
+fini(xlator_t *this)
 {
-        clnt_conf_t *conf = NULL;
+    clnt_conf_t *conf = NULL;
 
-        conf = this->private;
-        if (!conf)
-                return;
-
-        conf->destroy = 1;
-        if (conf->rpc) {
-                /* cleanup the saved-frames before last unref */
-                rpc_clnt_connection_cleanup (&conf->rpc->conn);
-                rpc_clnt_unref (conf->rpc);
-        }
+    conf = this->private;
+    if (!conf)
+        return;
 
-        /* Saved Fds */
-        /* TODO: */
+    conf->fini_completed = _gf_false;
+    conf->destroy = 1;
+    if (conf->rpc) {
+        /* cleanup the saved-frames before last unref */
+        rpc_clnt_connection_cleanup(&conf->rpc->conn);
+        rpc_clnt_unref(conf->rpc);
+    }
 
-        return;
-}
+    pthread_mutex_lock(&conf->lock);
+    {
+        while (!conf->fini_completed)
+            pthread_cond_wait(&conf->fini_complete_cond, &conf->lock);
+    }
+    pthread_mutex_unlock(&conf->lock);
 
-static void
-client_fd_lk_ctx_dump (xlator_t *this, fd_lk_ctx_t *lk_ctx, int nth_fd)
-{
-        gf_boolean_t            use_try_lock             = _gf_true;
-        int                     ret                      = -1;
-        int                     lock_no                  = 0;
-        fd_lk_ctx_t             *lk_ctx_ref              = NULL;
-        fd_lk_ctx_node_t        *plock                   = NULL;
-        char                    key[GF_DUMP_MAX_BUF_LEN] = {0,};
-
-        lk_ctx_ref = fd_lk_ctx_try_ref (lk_ctx);
-        if (!lk_ctx_ref)
-                return;
-
-        ret = client_fd_lk_list_empty (lk_ctx_ref, (use_try_lock = _gf_true));
-        if (ret != 0)
-                return;
-
-        ret = TRY_LOCK (&lk_ctx_ref->lock);
-        if (ret)
-                return;
-
-        gf_proc_dump_write ("------","------");
-
-        lock_no = 0;
-        list_for_each_entry (plock, &lk_ctx_ref->lk_list, next) {
-                snprintf (key, sizeof (key), "granted-posix-lock[%d]",
-                          lock_no++);
-                gf_proc_dump_write (key, "owner = %s, cmd = %s "
-                                    "fl_type = %s, fl_start = %"
-                                    PRId64", fl_end = %"PRId64
-                                    ", user_flock: l_type = %s, "
-                                    "l_start = %"PRId64", l_len = %"PRId64,
-                                    lkowner_utoa (&plock->user_flock.l_owner),
-                                    get_lk_cmd (plock->cmd),
-                                    get_lk_type (plock->fl_type),
-                                    plock->fl_start, plock->fl_end,
-                                    get_lk_type (plock->user_flock.l_type),
-                                    plock->user_flock.l_start,
-                                    plock->user_flock.l_len);
-        }
-        gf_proc_dump_write ("------","------");
+    pthread_spin_destroy(&conf->fd_lock);
+    pthread_mutex_destroy(&conf->lock);
+    pthread_cond_destroy(&conf->fini_complete_cond);
+    GF_FREE(conf);
 
-        UNLOCK (&lk_ctx_ref->lock);
-        fd_lk_ctx_unref (lk_ctx_ref);
+    /* Saved Fds */
+    /* TODO: */
 
+    return;
 }
 
-int
-client_priv_dump (xlator_t *this)
-{
-        clnt_conf_t             *conf = NULL;
-        int                     ret   = -1;
-        clnt_fd_ctx_t           *tmp = NULL;
-        int                     i = 0;
-        char                    key[GF_DUMP_MAX_BUF_LEN];
-        char                    key_prefix[GF_DUMP_MAX_BUF_LEN];
-        rpc_clnt_connection_t  *conn = NULL;
-
-        if (!this)
-                return -1;
+static void
+client_fd_lk_ctx_dump(xlator_t *this, fd_lk_ctx_t *lk_ctx, int nth_fd)
+{
+    gf_boolean_t use_try_lock = _gf_true;
+    int ret = -1;
+    int lock_no = 0;
+    fd_lk_ctx_t *lk_ctx_ref = NULL;
+    fd_lk_ctx_node_t *plock = NULL;
+    char key[GF_DUMP_MAX_BUF_LEN] = {
+        0,
+    };
+
+    lk_ctx_ref = fd_lk_ctx_ref(lk_ctx);
+    if (!lk_ctx_ref)
+        return;
 
-        conf = this->private;
-        if (!conf)
-                return -1;
+    ret = client_fd_lk_list_empty(lk_ctx_ref, (use_try_lock = _gf_true));
+    if (ret != 0)
+        return;
 
-        ret = pthread_mutex_trylock(&conf->lock);
-        if (ret)
-                return -1;
+    gf_proc_dump_write("------", "------");
 
-        gf_proc_dump_build_key(key_prefix, "xlator.protocol.client",
-                               "%s.priv", this->name);
+    lock_no = 0;
 
-        gf_proc_dump_add_section(key_prefix);
+    ret = TRY_LOCK(&lk_ctx_ref->lock);
+    if (ret)
+        return;
 
-        list_for_each_entry(tmp, &conf->saved_fds, sfd_pos) {
-                sprintf (key, "fd.%d.remote_fd", i);
-                gf_proc_dump_write(key, "%d", tmp->remote_fd);
-                client_fd_lk_ctx_dump (this, tmp->lk_ctx, i);
-                i++;
-        }
+    list_for_each_entry(plock, &lk_ctx_ref->lk_list, next)
+    {
+        snprintf(key, sizeof(key), "granted-posix-lock[%d]", lock_no++);
+        gf_proc_dump_write(
+            key,
+            "owner = %s, cmd = %s "
+            "fl_type = %s, fl_start = %" PRId64 ", fl_end = %" PRId64
+            ", user_flock: l_type = %s, "
+            "l_start = %" PRId64 ", l_len = %" PRId64,
+            lkowner_utoa(&plock->user_flock.l_owner), get_lk_cmd(plock->cmd),
+            get_lk_type(plock->fl_type), plock->fl_start, plock->fl_end,
+            get_lk_type(plock->user_flock.l_type), plock->user_flock.l_start,
+            plock->user_flock.l_len);
+    }
+    UNLOCK(&lk_ctx_ref->lock);
 
-        gf_proc_dump_write("connecting", "%d", conf->connecting);
-
-        gf_proc_dump_write ("connected", "%d", conf->connected);
-
-        if (conf->rpc) {
-                conn = &conf->rpc->conn;
-                gf_proc_dump_write("total_bytes_read", "%"PRIu64,
-                                   conn->trans->total_bytes_read);
-                gf_proc_dump_write("ping_timeout", "%"PRIu32,
-                                   conn->ping_timeout);
-                gf_proc_dump_write("total_bytes_written", "%"PRIu64,
-                                   conn->trans->total_bytes_write);
-                gf_proc_dump_write("ping_msgs_sent", "%"PRIu64,
-                                    conn->pingcnt);
-                gf_proc_dump_write("msgs_sent", "%"PRIu64,
-                                    conn->msgcnt);
-        }
-        pthread_mutex_unlock(&conf->lock);
+    gf_proc_dump_write("------", "------");
 
-        return 0;
+    fd_lk_ctx_unref(lk_ctx_ref);
+}
 
+static int
+client_priv_dump(xlator_t *this)
+{
+    clnt_conf_t *conf = NULL;
+    int ret = -1;
+    clnt_fd_ctx_t *tmp = NULL;
+    int i = 0;
+    char key[GF_DUMP_MAX_BUF_LEN];
+    char key_prefix[GF_DUMP_MAX_BUF_LEN];
+    rpc_clnt_connection_t *conn = NULL;
+
+    if (!this)
+        return -1;
+
+    conf = this->private;
+    if (!conf)
+        return -1;
+
+    gf_proc_dump_build_key(key_prefix, "xlator.protocol.client", "%s.priv",
+                           this->name);
+
+    gf_proc_dump_add_section("%s", key_prefix);
+
+    ret = pthread_mutex_trylock(&conf->lock);
+    if (ret)
+        return -1;
+
+    pthread_spin_lock(&conf->fd_lock);
+    list_for_each_entry(tmp, &conf->saved_fds, sfd_pos)
+    {
+        sprintf(key, "fd.%d.remote_fd", i);
+        gf_proc_dump_write(key, "%" PRId64, tmp->remote_fd);
+        client_fd_lk_ctx_dump(this, tmp->lk_ctx, i);
+        i++;
+    }
+    pthread_spin_unlock(&conf->fd_lock);
+
+    gf_proc_dump_write("connected", "%d", conf->connected);
+
+    if (conf->rpc) {
+        conn = &conf->rpc->conn;
+        gf_proc_dump_write("total_bytes_read", "%" PRIu64,
+                           conn->trans->total_bytes_read);
+        gf_proc_dump_write("ping_timeout", "%" PRIu32, conn->ping_timeout);
+        gf_proc_dump_write("total_bytes_written", "%" PRIu64,
+                           conn->trans->total_bytes_write);
+        gf_proc_dump_write("ping_msgs_sent", "%" PRIu64, conn->pingcnt);
+        gf_proc_dump_write("msgs_sent", "%" PRIu64, conn->msgcnt);
+    }
+    pthread_mutex_unlock(&conf->lock);
+
+    return 0;
 }
 
 int32_t
-client_inodectx_dump (xlator_t *this, inode_t *inode)
+client_inodectx_dump(xlator_t *this, inode_t *inode)
 {
-        if (!inode)
-                return -1;
+    if (!inode)
+        return -1;
 
-        if (!this)
-                return -1;
+    if (!this)
+        return -1;
 
-        /*TODO*/
+    /*TODO*/
 
-        return 0;
+    return 0;
 }
 
-
-
-
-struct xlator_cbks cbks = {
-        .forget     = client_forget,
-        .release    = client_release,
-        .releasedir = client_releasedir
-};
+struct xlator_cbks cbks = {.forget = client_forget,
+                           .release = client_release,
+                           .releasedir = client_releasedir};
 
 struct xlator_fops fops = {
-        .stat        = client_stat,
-        .readlink    = client_readlink,
-        .mknod       = client_mknod,
-        .mkdir       = client_mkdir,
-        .unlink      = client_unlink,
-        .rmdir       = client_rmdir,
-        .symlink     = client_symlink,
-        .rename      = client_rename,
-        .link        = client_link,
-        .truncate    = client_truncate,
-        .open        = client_open,
-        .readv       = client_readv,
-        .writev      = client_writev,
-        .statfs      = client_statfs,
-        .flush       = client_flush,
-        .fsync       = client_fsync,
-        .setxattr    = client_setxattr,
-        .getxattr    = client_getxattr,
-        .fsetxattr   = client_fsetxattr,
-        .fgetxattr   = client_fgetxattr,
-        .removexattr = client_removexattr,
-        .fremovexattr = client_fremovexattr,
-        .opendir     = client_opendir,
-        .readdir     = client_readdir,
-        .readdirp    = client_readdirp,
-        .fsyncdir    = client_fsyncdir,
-        .access      = client_access,
-        .ftruncate   = client_ftruncate,
-        .fstat       = client_fstat,
-        .create      = client_create,
-        .lk          = client_lk,
-        .inodelk     = client_inodelk,
-        .finodelk    = client_finodelk,
-        .entrylk     = client_entrylk,
-        .fentrylk    = client_fentrylk,
-        .lookup      = client_lookup,
-        .rchecksum   = client_rchecksum,
-        .xattrop     = client_xattrop,
-        .fxattrop    = client_fxattrop,
-        .setattr     = client_setattr,
-        .fsetattr    = client_fsetattr,
-	.fallocate   = client_fallocate,
-	.discard     = client_discard,
-        .zerofill    = client_zerofill,
-        .getspec     = client_getspec,
-        .ipc         = client_ipc,
+    .stat = client_stat,
+    .readlink = client_readlink,
+    .mknod = client_mknod,
+    .mkdir = client_mkdir,
+    .unlink = client_unlink,
+    .rmdir = client_rmdir,
+    .symlink = client_symlink,
+    .rename = client_rename,
+    .link = client_link,
+    .truncate = client_truncate,
+    .open = client_open,
+    .readv = client_readv,
+    .writev = client_writev,
+    .statfs = client_statfs,
+    .flush = client_flush,
+    .fsync = client_fsync,
+    .setxattr = client_setxattr,
+    .getxattr = client_getxattr,
+    .fsetxattr = client_fsetxattr,
+    .fgetxattr = client_fgetxattr,
+    .removexattr = client_removexattr,
+    .fremovexattr = client_fremovexattr,
+    .opendir = client_opendir,
+    .readdir = client_readdir,
+    .readdirp = client_readdirp,
+    .fsyncdir = client_fsyncdir,
+    .access = client_access,
+    .ftruncate = client_ftruncate,
+    .fstat = client_fstat,
+    .create = client_create,
+    .lk = client_lk,
+    .inodelk = client_inodelk,
+    .finodelk = client_finodelk,
+    .entrylk = client_entrylk,
+    .fentrylk = client_fentrylk,
+    .lookup = client_lookup,
+    .rchecksum = client_rchecksum,
+    .xattrop = client_xattrop,
+    .fxattrop = client_fxattrop,
+    .setattr = client_setattr,
+    .fsetattr = client_fsetattr,
+    .fallocate = client_fallocate,
+    .discard = client_discard,
+    .zerofill = client_zerofill,
+    .getspec = client_getspec,
+    .ipc = client_ipc,
+    .seek = client_seek,
+    .lease = client_lease,
+    .compound = client_compound,
+    .getactivelk = client_getactivelk,
+    .setactivelk = client_setactivelk,
+    .icreate = client_icreate,
+    .namelink = client_namelink,
+    .put = client_put,
+    .copy_file_range = client_copy_file_range,
 };
 
-
 struct xlator_dumpops dumpops = {
-        .priv      =  client_priv_dump,
-        .inodectx  =  client_inodectx_dump,
+    .priv = client_priv_dump,
+    .inodectx = client_inodectx_dump,
 };
 
-
 struct volume_options options[] = {
-        { .key   = {"username"},
-          .type  = GF_OPTION_TYPE_ANY
-        },
-        { .key   = {"password"},
-          .type  = GF_OPTION_TYPE_ANY
-        },
-        { .key   = {"transport-type"},
-          .value = {"tcp", "socket", "ib-verbs", "unix", "ib-sdp",
-                    "tcp/client", "ib-verbs/client", "rdma"},
-          .type  = GF_OPTION_TYPE_STR
-        },
-        { .key   = {"remote-host"},
-          .type  = GF_OPTION_TYPE_INTERNET_ADDRESS
-        },
-        { .key   = {"remote-port"},
-          .type  = GF_OPTION_TYPE_INT,
-        },
-        { .key   = {"remote-subvolume"},
-          .type  = GF_OPTION_TYPE_ANY
-        },
-        { .key   = {"frame-timeout",
-                    "rpc-timeout" },
-          .type  = GF_OPTION_TYPE_TIME,
-          .min   = 0,
-          .max   = 86400,
-          .default_value = "1800",
-          .description = "Time frame after which the (file) operation would be "
-                         "declared as dead, if the server does not respond for "
-                         "a particular (file) operation."
-        },
-        { .key   = {"ping-timeout"},
-          .type  = GF_OPTION_TYPE_TIME,
-          .min   = 0,
-          .max   = 1013,
-          .default_value = "42",
-          .description = "Time duration for which the client waits to "
-                         "check if the server is responsive."
-        },
-        { .key   = {"client-bind-insecure"},
-          .type  = GF_OPTION_TYPE_BOOL
-        },
-        { .key   = {"lk-heal"},
-          .type  = GF_OPTION_TYPE_BOOL,
-          .default_value = "off",
-          .description = "When the connection to client is lost, server "
-                         "cleans up all the locks held by the client. After "
-                         "the connection is restored, the client reacquires "
-                         "(heals) the fcntl locks released by the server."
-        },
-        { .key   = {"grace-timeout"},
-          .type  = GF_OPTION_TYPE_INT,
-          .min   = 10,
-          .max   = 1800,
-          .default_value = "10",
-          .description = "Specifies the duration for the lock state to be "
-                         "maintained on the client after a network "
-                         "disconnection. Range 10-1800 seconds."
-        },
-        {.key  = {"tcp-window-size"},
-         .type = GF_OPTION_TYPE_SIZET,
-         .min  = GF_MIN_SOCKET_WINDOW_SIZE,
-         .max  = GF_MAX_SOCKET_WINDOW_SIZE,
-         .description = "Specifies the window size for tcp socket."
-        },
-        { .key   = {"filter-O_DIRECT"},
-          .type  = GF_OPTION_TYPE_BOOL,
-          .default_value = "disable",
-          .description = "If enabled, in open() and creat() calls, O_DIRECT "
-          "flag will be filtered at the client protocol level so server will "
-          "still continue to cache the file. This works similar to NFS's "
-          "behavior of O_DIRECT",
-        },
-        { .key   = {"send-gids"},
-          .type  = GF_OPTION_TYPE_BOOL,
-          .default_value = "on",
-        },
-        { .key   = {"event-threads"},
-          .type  = GF_OPTION_TYPE_INT,
-          .min   = 1,
-          .max   = 32,
-          .default_value = "2",
-          .description = "Specifies the number of event threads to execute "
-                         "in parallel. Larger values would help process"
-                         " responses faster, depending on available processing"
-                         " power. Range 1-32 threads."
-        },
-        { .key   = {NULL} },
+    {.key = {"username"}, .type = GF_OPTION_TYPE_ANY},
+    {.key = {"password"}, .type = GF_OPTION_TYPE_ANY},
+    {
+        .key = {"transport-type"},
+        .value = {"tcp", "socket", "ib-verbs", "unix", "ib-sdp", "tcp/client",
+                  "ib-verbs/client", "rdma"},
+        .type = GF_OPTION_TYPE_STR,
+        .default_value = "tcp",
+    },
+    {.key = {"remote-host"},
+     .type = GF_OPTION_TYPE_INTERNET_ADDRESS,
+     .default_value = "{{ brick.hostname }}"},
+    {
+        .key = {"remote-port"},
+        .type = GF_OPTION_TYPE_INT,
+    },
+    {.key = {"remote-subvolume"},
+     .type = GF_OPTION_TYPE_ANY,
+     .default_value = "{{ brick.path }}"},
+    {.key = {"frame-timeout", "rpc-timeout"},
+     .type = GF_OPTION_TYPE_TIME,
+     .min = 0,
+     .max = 86400,
+     .default_value = "1800",
+     .description = "Time frame after which the (file) operation would be "
+                    "declared as dead, if the server does not respond for "
+                    "a particular (file) operation.",
+     .op_version = {1},
+     .flags = OPT_FLAG_SETTABLE | OPT_FLAG_DOC},
+    {.key = {"ping-timeout"},
+     .type = GF_OPTION_TYPE_TIME,
+     .min = 0,
+     .max = 1013,
+     .default_value = TOSTRING(GF_NETWORK_TIMEOUT),
+     .description = "Time duration for which the client waits to "
+                    "check if the server is responsive.",
+     .op_version = {1},
+     .flags = OPT_FLAG_SETTABLE | OPT_FLAG_DOC},
+    {.key = {"client-bind-insecure"}, .type = GF_OPTION_TYPE_BOOL},
+    {.key = {"tcp-window-size"},
+     .type = GF_OPTION_TYPE_SIZET,
+     .min = GF_MIN_SOCKET_WINDOW_SIZE,
+     .max = GF_MAX_SOCKET_WINDOW_SIZE,
+     .description = "Specifies the window size for tcp socket.",
+     .op_version = {1},
+     .flags = OPT_FLAG_SETTABLE | OPT_FLAG_DOC},
+    {.key = {"filter-O_DIRECT"},
+     .type = GF_OPTION_TYPE_BOOL,
+     .default_value = "disable",
+     .description =
+         "If enabled, in open/creat/readv/writev fops, "
+         "O_DIRECT flag will be filtered at the client protocol level so "
+         "server will still continue to cache the file. This works similar to "
+         "NFS's behavior of O_DIRECT. Anon-fds can choose to readv/writev "
+         "using O_DIRECT",
+     .op_version = {2},
+     .flags = OPT_FLAG_SETTABLE | OPT_FLAG_DOC},
+    {.key = {"send-gids"},
+     .type = GF_OPTION_TYPE_BOOL,
+     .default_value = "on",
+     .op_version = {GD_OP_VERSION_3_6_0},
+     .flags = OPT_FLAG_SETTABLE},
+    {.key = {"event-threads"},
+     .type = GF_OPTION_TYPE_INT,
+     .min = 1,
+     .max = 32,
+     .default_value = "2",
+     .description = "Specifies the number of event threads to execute "
+                    "in parallel. Larger values would help process"
+                    " responses faster, depending on available processing"
+                    " power. Range 1-32 threads.",
+     .op_version = {GD_OP_VERSION_3_7_0},
+     .flags = OPT_FLAG_SETTABLE | OPT_FLAG_DOC},
+
+    /* This option is required for running code-coverage tests with
+       old protocol */
+    {
+        .key = {"testing.old-protocol"},
+        .type = GF_OPTION_TYPE_BOOL,
+        .default_value = "off",
+        .op_version = {GD_OP_VERSION_7_0},
+        .flags = OPT_FLAG_SETTABLE,
+    },
+    {.key = {"strict-locks"},
+     .type = GF_OPTION_TYPE_BOOL,
+     .default_value = "off",
+     .op_version = {GD_OP_VERSION_7_0},
+     .flags = OPT_FLAG_SETTABLE,
+     .description = "When set, doesn't reopen saved fds after reconnect "
+                    "if POSIX locks are held on them. Hence subsequent "
+                    "operations on these fds will fail. This is "
+                    "necessary for stricter lock complaince as bricks "
+                    "cleanup any granted locks when a client "
+                    "disconnects."},
+    {.key = {NULL}},
 };
 
+xlator_api_t xlator_api = {
+    .init = init,
+    .fini = fini,
+    .notify = notify,
+    .reconfigure = reconfigure,
+    .mem_acct_init = mem_acct_init,
+    .op_version = {1}, /* Present from the initial version */
+    .dumpops = &dumpops,
+    .fops = &fops,
+    .cbks = &cbks,
+    .options = options,
+    .identifier = "client",
+    .category = GF_MAINTAINED,
+};
diff --git a/xlators/protocol/client/src/client.h b/xlators/protocol/client/src/client.h
index 40200b6afc4..ab799c41755 100644
--- a/xlators/protocol/client/src/client.h
+++ b/xlators/protocol/client/src/client.h
@@ -15,258 +15,380 @@
 #include <stdint.h>
 
 #include "rpc-clnt.h"
-#include "list.h"
-#include "inode.h"
+#include <glusterfs/list.h>
+#include <glusterfs/inode.h>
 #include "client-mem-types.h"
 #include "protocol-common.h"
 #include "glusterfs3.h"
-#include "fd-lk.h"
+#include "glusterfs3-xdr.h"
+#include <glusterfs/fd-lk.h>
+#include <glusterfs/defaults.h>
+#include <glusterfs/default-args.h>
+#include "client-messages.h"
 
 /* FIXME: Needs to be defined in a common file */
-#define CLIENT_CMD_CONNECT    "trusted.glusterfs.client-connect"
-#define CLIENT_CMD_DISCONNECT "trusted.glusterfs.client-disconnect"
-#define CLIENT_DUMP_LOCKS     "trusted.glusterfs.clientlk-dump"
-#define GF_MAX_SOCKET_WINDOW_SIZE  (1 * GF_UNIT_MB)
-#define GF_MIN_SOCKET_WINDOW_SIZE  (0)
+#define CLIENT_DUMP_LOCKS "trusted.glusterfs.clientlk-dump"
+#define GF_MAX_SOCKET_WINDOW_SIZE (1 * GF_UNIT_MB)
+#define GF_MIN_SOCKET_WINDOW_SIZE (0)
 
 typedef enum {
-        GF_LK_HEAL_IN_PROGRESS,
-        GF_LK_HEAL_DONE,
-} lk_heal_state_t;
-
-typedef enum {
-        DEFAULT_REMOTE_FD = 0,
-        FALLBACK_TO_ANON_FD = 1
+    DEFAULT_REMOTE_FD = 0,
+    FALLBACK_TO_ANON_FD = 1
 } clnt_remote_fd_flags_t;
 
-#define CLIENT_GET_REMOTE_FD(xl, fd, flags, remote_fd, op_errno, label) \
-        do {                                                            \
-                int     _ret    = 0;                                    \
-                _ret = client_get_remote_fd (xl, fd, flags, &remote_fd);\
-                if (_ret < 0) {                                         \
-                        op_errno = errno;                               \
-                        goto label;                                     \
-                }                                                       \
-                if (remote_fd == -1) {                                  \
-                        gf_log (xl->name, GF_LOG_WARNING, " (%s) "      \
-                                "remote_fd is -1. EBADFD",              \
-                                uuid_utoa (fd->inode->gfid));           \
-                        op_errno = EBADFD;                              \
-                        goto label;                                     \
-                }                                                       \
-        } while (0)
-
-#define CLIENT_STACK_UNWIND(op, frame, params ...) do {             \
-                if (!frame)                                         \
-                        break;                                      \
-                clnt_local_t *__local = frame->local;               \
-                frame->local = NULL;                                \
-                STACK_UNWIND_STRICT (op, frame, params);            \
-                client_local_wipe (__local);                        \
-        } while (0)
-
+#define CLIENT_POST_FOP(fop, this_rsp_u, this_args_cbk, params...)             \
+    do {                                                                       \
+        gf_common_rsp *_this_rsp = &CPD_RSP_FIELD(this_rsp_u, fop);            \
+                                                                               \
+        int _op_ret = _this_rsp->op_ret;                                       \
+        int _op_errno = gf_error_to_errno(_this_rsp->op_errno);                \
+        args_##fop##_cbk_store(this_args_cbk, _op_ret, _op_errno, params);     \
+    } while (0)
+
+#define CLIENT_POST_FOP_TYPE(fop, this_rsp_u, this_args_cbk, params...)        \
+    do {                                                                       \
+        gfs3_##fop##_rsp *_this_rsp = &CPD_RSP_FIELD(this_rsp_u, fop);         \
+                                                                               \
+        int _op_ret = _this_rsp->op_ret;                                       \
+        int _op_errno = gf_error_to_errno(_this_rsp->op_errno);                \
+        args_##fop##_cbk_store(this_args_cbk, _op_ret, _op_errno, params);     \
+    } while (0)
+
+#define CLIENT_GET_REMOTE_FD(xl, fd, flags, remote_fd, op_errno, label)        \
+    do {                                                                       \
+        int _ret = 0;                                                          \
+        _ret = client_get_remote_fd(xl, fd, flags, &remote_fd);                \
+        if (_ret < 0) {                                                        \
+            op_errno = errno;                                                  \
+            goto label;                                                        \
+        }                                                                      \
+        if (remote_fd == -1) {                                                 \
+            gf_smsg(xl->name, GF_LOG_WARNING, EBADFD, PC_MSG_BAD_FD,           \
+                    "gfid=%s", uuid_utoa(fd->inode->gfid), NULL);              \
+            op_errno = EBADFD;                                                 \
+            goto label;                                                        \
+        }                                                                      \
+    } while (0)
+
+#define CLIENT_STACK_UNWIND(op, frame, params...)                              \
+    do {                                                                       \
+        if (!frame)                                                            \
+            break;                                                             \
+        clnt_local_t *__local = frame->local;                                  \
+        frame->local = NULL;                                                   \
+        STACK_UNWIND_STRICT(op, frame, params);                                \
+        client_local_wipe(__local);                                            \
+    } while (0)
 
 struct clnt_options {
-        char *remote_subvolume;
-        int   ping_timeout;
+    char *remote_subvolume;
+    int ping_timeout;
 };
 
 typedef struct clnt_conf {
-        struct rpc_clnt       *rpc;
-        struct clnt_options    opt;
-        struct rpc_clnt_config rpc_conf;
-	struct list_head       saved_fds;
-        pthread_mutex_t        lock;
-        int                    connecting;
-        int                    connected;
-
-        rpc_clnt_prog_t       *fops;
-        rpc_clnt_prog_t       *mgmt;
-        rpc_clnt_prog_t       *handshake;
-        rpc_clnt_prog_t       *dump;
-
-        int                    client_id;
-        uint64_t               reopen_fd_count; /* Count of fds reopened after a
-                                                   connection is established */
-        gf_lock_t              rec_lock;
-        int                    skip_notify;
-
-        int                    last_sent_event; /* Flag used to make sure we are
-                                                   not repeating the same event
-                                                   which was sent earlier */
-        char                   portmap_err_logged; /* flag used to prevent
-                                                      excessive logging */
-        char                   disconnect_err_logged; /* flag used to prevent
-                                                         excessive disconnect
-                                                         logging */
-        gf_boolean_t           lk_heal;
-        uint16_t               lk_version; /* this variable is used to distinguish
-                                              client-server transaction while
-                                              performing lock healing */
-        struct timespec        grace_ts;
-        gf_timer_t            *grace_timer;
-        gf_boolean_t           grace_timer_needed; /* The state of this flag will
-                                                      be used to decide whether
-                                                      a new grace-timer must be
-                                                      registered or not. False
-                                                      means dont register, true
-                                                      means register */
-        char                   parent_down;
-	gf_boolean_t           quick_reconnect; /* When reconnecting after
-						   portmap query, do not let
-						   the reconnection happen after
-						   the usual 3-second wait
-						*/
-        gf_boolean_t           filter_o_direct; /* if set, filter O_DIRECT from
-                                                   the flags list of open() */
-        /* set volume is the op which results in creating/re-using
-         * the conn-id and is called once per connection, this remembers
-         * how manytimes set_volume is called
-         */
-        uint64_t               setvol_count;
-
-        gf_boolean_t           send_gids; /* let the server resolve gids */
-
-        int                     event_threads; /* # of event threads
-                                                * configured */
-
-        gf_boolean_t           destroy; /* if enabled implies fini was called
-                                         * on @this xlator instance */
+    struct rpc_clnt *rpc;
+    struct clnt_options opt;
+    struct rpc_clnt_config rpc_conf;
+    struct list_head saved_fds;
+    pthread_spinlock_t fd_lock; /* protects saved_fds list
+                                 * and all fdctx */
+    pthread_mutex_t lock;
+    int connected;
+
+    rpc_clnt_prog_t *fops;
+    rpc_clnt_prog_t *mgmt;
+    rpc_clnt_prog_t *handshake;
+    rpc_clnt_prog_t *dump;
+
+    int client_id;
+    uint64_t reopen_fd_count; /* Count of fds reopened after a
+                                 connection is established */
+    gf_lock_t rec_lock;
+    int skip_notify;
+
+    int last_sent_event;        /* Flag used to make sure we are
+                                   not repeating the same event
+                                   which was sent earlier */
+    char portmap_err_logged;    /* flag used to prevent
+                                   excessive logging */
+    char disconnect_err_logged; /* flag used to prevent
+                                   excessive disconnect
+                                   logging */
+    char parent_down;
+    gf_boolean_t quick_reconnect; /* When reconnecting after
+                                     portmap query, do not let
+                                     the reconnection happen after
+                                     the usual 3-second wait
+                                  */
+    gf_boolean_t filter_o_direct; /* if set, filter O_DIRECT from
+                                     the flags list of open() */
+    /* set volume is the op which results in creating/re-using
+     * the conn-id and is called once per connection, this remembers
+     * how manytimes set_volume is called
+     */
+    uint64_t setvol_count;
+
+    gf_boolean_t send_gids; /* let the server resolve gids */
+
+    int event_threads; /* # of event threads
+                        * configured */
+
+    gf_boolean_t destroy; /* if enabled implies fini was called
+                           * on @this xlator instance */
+
+    gf_boolean_t child_up; /* Set to true, when child is up, and
+                            * false, when child is down */
+
+    gf_boolean_t can_log_disconnect; /* socket level connection is
+                                      * up, disconnects can be
+                                      * logged
+                                      */
+
+    gf_boolean_t old_protocol;         /* used only for old-protocol testing */
+    pthread_cond_t fini_complete_cond; /* Used to wait till we finsh the fini
+                                          compltely, ie client_fini_complete
+                                          to return*/
+    gf_boolean_t fini_completed;
+    gf_boolean_t strict_locks; /* When set, doesn't reopen saved fds after
+                                  reconnect if POSIX locks are held on them.
+                                  Hence subsequent operations on these fds will
+                                  fail. This is necessary for stricter lock
+                                  complaince as bricks cleanup any granted
+                                  locks when a client disconnects.
+                               */
+
+    gf_boolean_t connection_to_brick; /*True from attempt to connect to brick
+                                        till disconnection to brick*/
 } clnt_conf_t;
 
 typedef struct _client_fd_ctx {
-        struct list_head  sfd_pos;      /*  Stores the reference to this
-                                            fd's position in the saved_fds list.
-                                        */
-        int64_t           remote_fd;
-        char              is_dir;
-        char              released;
-        int32_t           flags;
-        fd_lk_ctx_t      *lk_ctx;
-        pthread_mutex_t   mutex;
-        lk_heal_state_t   lk_heal_state;
-        uuid_t            gfid;
-        void (*reopen_done) (struct _client_fd_ctx*, xlator_t *);
-        struct list_head  lock_list;     /* List of all granted locks on this fd */
-        int32_t           reopen_attempts;
+    struct list_head sfd_pos; /*  Stores the reference to this
+                                  fd's position in the saved_fds list.
+                              */
+    int64_t remote_fd;
+    char is_dir;
+    char released;
+    int32_t flags;
+    fd_lk_ctx_t *lk_ctx;
+    uuid_t gfid;
+    void (*reopen_done)(struct _client_fd_ctx *, int64_t rfd, xlator_t *);
+    struct list_head lock_list; /* List of all granted locks on this fd */
+    int32_t reopen_attempts;
 } clnt_fd_ctx_t;
 
 typedef struct _client_posix_lock {
-        fd_t              *fd;            /* The fd on which the lk operation was made */
-
-        struct gf_flock    user_flock;    /* the flock supplied by the user */
-        off_t              fl_start;
-        off_t              fl_end;
-        short              fl_type;
-        int32_t            cmd;           /* the cmd for the lock call */
-        gf_lkowner_t       owner; /* lock owner from fuse */
-        struct list_head   list;          /* reference used to add to the fdctx list of locks */
+    fd_t *fd; /* The fd on which the lk operation was made */
+
+    struct gf_flock user_flock; /* the flock supplied by the user */
+    off_t fl_start;
+    off_t fl_end;
+    short fl_type;
+    int32_t cmd;        /* the cmd for the lock call */
+    gf_lkowner_t owner; /* lock owner from fuse */
+    struct list_head
+        list; /* reference used to add to the fdctx list of locks */
 } client_posix_lock_t;
 
 typedef struct client_local {
-        loc_t                loc;
-        loc_t                loc2;
-        fd_t                *fd;
-        clnt_fd_ctx_t       *fdctx;
-        uint32_t             flags;
-        struct iobref       *iobref;
-
-        client_posix_lock_t *client_lock;
-        gf_lkowner_t         owner;
-        int32_t              cmd;
-        struct list_head     lock_list;
-        pthread_mutex_t      mutex;
-        char                *name;
-        gf_boolean_t         attempt_reopen;
+    loc_t loc;
+    loc_t loc2;
+    fd_t *fd;
+    fd_t *fd_out; /* used in copy_file_range */
+    clnt_fd_ctx_t *fdctx;
+    uint32_t flags;
+    struct iobref *iobref;
+
+    client_posix_lock_t *client_lock;
+    gf_lkowner_t owner;
+    int32_t cmd;
+    struct list_head lock_list;
+    pthread_mutex_t mutex;
+    char *name;
+    gf_boolean_t attempt_reopen;
+    /*
+     * The below boolean variable is used
+     * only for copy_file_range fop
+     */
+    gf_boolean_t attempt_reopen_out;
 } clnt_local_t;
 
 typedef struct client_args {
-        loc_t              *loc;
-        fd_t               *fd;
-        const char         *linkname;
-        struct iobref      *iobref;
-        struct iovec       *vector;
-        dict_t             *xattr;
-        struct iatt        *stbuf;
-        loc_t              *oldloc;
-        loc_t              *newloc;
-        const char         *name;
-        struct gf_flock    *flock;
-        const char         *volume;
-        const char         *basename;
-        off_t               offset;
-        int32_t             mask;
-        int32_t             cmd;
-        size_t              size;
-        mode_t              mode;
-        dev_t               rdev;
-        int32_t             flags;
-        int32_t             count;
-        int32_t             datasync;
-        entrylk_cmd         cmd_entrylk;
-        entrylk_type        type;
-        gf_xattrop_flags_t  optype;
-        int32_t             valid;
-        int32_t             len;
-
-        mode_t              umask;
-        dict_t             *xdata;
+    loc_t *loc;
+    /*
+     * This is the source fd for copy_file_range and
+     * the default fd for any other fd based fop which
+     * requires only one fd (i.e. opetates on one fd)
+     */
+    fd_t *fd;
+    fd_t *fd_out; /* this is the destination fd for copy_file_range */
+    const char *linkname;
+    struct iobref *iobref;
+    struct iovec *vector;
+    dict_t *xattr;
+    struct iatt *stbuf;
+    loc_t *oldloc;
+    loc_t *newloc;
+    const char *name;
+    struct gf_flock *flock;
+    const char *volume;
+    const char *basename;
+
+    off_t offset;
+    /*
+     * According to the man page of copy_file_range,
+     * the offsets for source and destination file
+     * are of type loff_t. But the type loff_t is
+     * linux specific and is actual a typedef of
+     * off64_t.
+     */
+    off64_t off_in;  /* used in copy_file_range for source fd */
+    off64_t off_out; /* used in copy_file_range for dst fd */
+    int32_t mask;
+    int32_t cmd;
+    size_t size;
+    mode_t mode;
+    dev_t rdev;
+    int32_t flags;
+    int32_t count;
+    int32_t datasync;
+    entrylk_cmd cmd_entrylk;
+    entrylk_type type;
+    gf_xattrop_flags_t optype;
+    int32_t valid;
+    int32_t len;
+    gf_seek_what_t what;
+    struct gf_lease *lease;
+
+    mode_t umask;
+    dict_t *xdata;
+    lock_migration_info_t *locklist;
 } clnt_args_t;
 
-typedef ssize_t (*gfs_serialize_t) (struct iovec outmsg, void *args);
-
-clnt_fd_ctx_t *this_fd_get_ctx (fd_t *file, xlator_t *this);
-clnt_fd_ctx_t *this_fd_del_ctx (fd_t *file, xlator_t *this);
-void this_fd_set_ctx (fd_t *file, xlator_t *this, loc_t *loc,
-                      clnt_fd_ctx_t *ctx);
-
-int client_local_wipe (clnt_local_t *local);
-int client_submit_request (xlator_t *this, void *req,
-                           call_frame_t *frame, rpc_clnt_prog_t *prog,
-                           int procnum, fop_cbk_fn_t cbk,
-                           struct iobref *iobref,
-                           struct iovec *rsphdr, int rsphdr_count,
-                           struct iovec *rsp_payload, int rsp_count,
-                           struct iobref *rsp_iobref, xdrproc_t xdrproc);
-
-int unserialize_rsp_dirent (xlator_t *this, struct gfs3_readdir_rsp *rsp,
-                            gf_dirent_t *entries);
-int unserialize_rsp_direntp (xlator_t *this, fd_t *fd,
-                             struct gfs3_readdirp_rsp *rsp, gf_dirent_t *entries);
-
-int clnt_readdir_rsp_cleanup (gfs3_readdir_rsp *rsp);
-int clnt_readdirp_rsp_cleanup (gfs3_readdirp_rsp *rsp);
-int client_attempt_lock_recovery (xlator_t *this, clnt_fd_ctx_t *fdctx);
-int32_t delete_granted_locks_owner (fd_t *fd, gf_lkowner_t *owner);
-int client_add_lock_for_recovery (fd_t *fd, struct gf_flock *flock,
-                                  gf_lkowner_t *owner, int32_t cmd);
-int32_t delete_granted_locks_fd (clnt_fd_ctx_t *fdctx);
-int32_t client_cmd_to_gf_cmd (int32_t cmd, int32_t *gf_cmd);
-void client_save_number_fds (clnt_conf_t *conf, int count);
-int dump_client_locks (inode_t *inode);
-int client_notify_parents_child_up (xlator_t *this);
-int32_t is_client_dump_locks_cmd (char *name);
-int32_t client_dump_locks (char *name, inode_t *inode,
-                           dict_t *dict);
-int client_fdctx_destroy (xlator_t *this, clnt_fd_ctx_t *fdctx);
-
-uint32_t client_get_lk_ver (clnt_conf_t *conf);
-
-int32_t client_type_to_gf_type (short l_type);
-
-int client_mark_fd_bad (xlator_t *this);
-
-int client_set_lk_version (xlator_t *this);
-
-int client_fd_lk_list_empty (fd_lk_ctx_t *lk_ctx, gf_boolean_t use_try_lock);
-void client_default_reopen_done (clnt_fd_ctx_t *fdctx, xlator_t *this);
-void client_attempt_reopen (fd_t *fd, xlator_t *this);
-int client_get_remote_fd (xlator_t *this, fd_t *fd, int flags,
-                          int64_t *remote_fd);
-int client_fd_fop_prepare_local (call_frame_t *frame, fd_t *fd,
-                                 int64_t remote_fd);
+typedef struct client_payload {
+    struct iobref *iobref;
+    struct iovec *payload;
+    struct iovec *rsphdr;
+    struct iovec *rsp_payload;
+    struct iobref *rsp_iobref;
+    int payload_cnt;
+    int rsphdr_cnt;
+    int rsp_payload_cnt;
+} client_payload_t;
+
+typedef ssize_t (*gfs_serialize_t)(struct iovec outmsg, void *args);
+
+clnt_fd_ctx_t *
+this_fd_get_ctx(fd_t *file, xlator_t *this);
+clnt_fd_ctx_t *
+this_fd_del_ctx(fd_t *file, xlator_t *this);
+void
+this_fd_set_ctx(fd_t *file, xlator_t *this, loc_t *loc, clnt_fd_ctx_t *ctx);
+
+int
+client_local_wipe(clnt_local_t *local);
+int
+client_submit_request(xlator_t *this, void *req, call_frame_t *frame,
+                      rpc_clnt_prog_t *prog, int procnum, fop_cbk_fn_t cbk,
+                      client_payload_t *cp, xdrproc_t xdrproc);
+
+int
+unserialize_rsp_dirent(xlator_t *this, struct gfs3_readdir_rsp *rsp,
+                       gf_dirent_t *entries);
+int
+unserialize_rsp_direntp(xlator_t *this, fd_t *fd, struct gfs3_readdirp_rsp *rsp,
+                        gf_dirent_t *entries);
+
+int
+clnt_readdir_rsp_cleanup(gfs3_readdir_rsp *rsp);
+int
+clnt_readdirp_rsp_cleanup(gfs3_readdirp_rsp *rsp);
+int
+client_attempt_lock_recovery(xlator_t *this, clnt_fd_ctx_t *fdctx);
+int32_t
+delete_granted_locks_owner(fd_t *fd, gf_lkowner_t *owner);
+int32_t
+delete_granted_locks_fd(clnt_fd_ctx_t *fdctx);
+int32_t
+client_cmd_to_gf_cmd(int32_t cmd, int32_t *gf_cmd);
+void
+client_save_number_fds(clnt_conf_t *conf, int count);
+int
+dump_client_locks(inode_t *inode);
+int32_t
+is_client_dump_locks_cmd(char *name);
+int32_t
+client_dump_locks(char *name, inode_t *inode, dict_t *dict);
+int
+client_fdctx_destroy(xlator_t *this, clnt_fd_ctx_t *fdctx);
+
+int
+client_fd_lk_list_empty(fd_lk_ctx_t *lk_ctx, gf_boolean_t use_try_lock);
+void
+client_default_reopen_done(clnt_fd_ctx_t *fdctx, int64_t rfd, xlator_t *this);
+void
+client_attempt_reopen(fd_t *fd, xlator_t *this);
+int
+client_get_remote_fd(xlator_t *this, fd_t *fd, int flags, int64_t *remote_fd);
+int
+client_fd_fop_prepare_local(call_frame_t *frame, fd_t *fd, int64_t remote_fd);
 gf_boolean_t
-__is_fd_reopen_in_progress (clnt_fd_ctx_t *fdctx);
+__is_fd_reopen_in_progress(clnt_fd_ctx_t *fdctx);
+int
+client_notify_dispatch(xlator_t *this, int32_t event, void *data, ...);
 int
-client_notify_dispatch (xlator_t *this, int32_t event, void *data, ...);
+client_notify_dispatch_uniq(xlator_t *this, int32_t event, void *data, ...);
+
+gf_boolean_t
+client_is_reopen_needed(fd_t *fd, xlator_t *this, int64_t remote_fd);
+
+int
+client_add_fd_to_saved_fds(xlator_t *this, fd_t *fd, loc_t *loc, int32_t flags,
+                           int64_t remote_fd, int is_dir);
+int
+clnt_unserialize_rsp_locklist(xlator_t *this, struct gfs3_getactivelk_rsp *rsp,
+                              lock_migration_info_t *lmi);
+void
+clnt_getactivelk_rsp_cleanup(gfs3_getactivelk_rsp *rsp);
+
+void
+clnt_setactivelk_req_cleanup(gfs3_setactivelk_req *req);
+
+int
+serialize_req_locklist(lock_migration_info_t *locklist,
+                       gfs3_setactivelk_req *req);
+
+void
+clnt_getactivelk_rsp_cleanup_v2(gfx_getactivelk_rsp *rsp);
+
+void
+clnt_setactivelk_req_cleanup_v2(gfx_setactivelk_req *req);
+
+int
+serialize_req_locklist_v2(lock_migration_info_t *locklist,
+                          gfx_setactivelk_req *req);
+
+int
+clnt_unserialize_rsp_locklist_v2(xlator_t *this,
+                                 struct gfx_getactivelk_rsp *rsp,
+                                 lock_migration_info_t *lmi);
+
+int
+unserialize_rsp_dirent_v2(xlator_t *this, struct gfx_readdir_rsp *rsp,
+                          gf_dirent_t *entries);
+int
+unserialize_rsp_direntp_v2(xlator_t *this, fd_t *fd,
+                           struct gfx_readdirp_rsp *rsp, gf_dirent_t *entries);
+
+int
+clnt_readdir_rsp_cleanup_v2(gfx_readdir_rsp *rsp);
+int
+clnt_readdirp_rsp_cleanup_v2(gfx_readdirp_rsp *rsp);
+
+int
+client_add_lock_for_recovery(fd_t *fd, struct gf_flock *flock,
+                             gf_lkowner_t *owner, int32_t cmd);
+
+int
+client_is_setlk(int32_t cmd);
+
 #endif /* !_CLIENT_H */
diff --git a/xlators/protocol/server/src/Makefile.am b/xlators/protocol/server/src/Makefile.am
index 6a18bf02561..5e875c8df0b 100644
--- a/xlators/protocol/server/src/Makefile.am
+++ b/xlators/protocol/server/src/Makefile.am
@@ -1,24 +1,31 @@
+if WITH_SERVER
 xlator_LTLIBRARIES = server.la
+endif
+
 xlatordir = $(libdir)/glusterfs/$(PACKAGE_VERSION)/xlator/protocol
 
-server_la_LDFLAGS = -module -avoid-version
+server_la_LDFLAGS = -module $(GF_XLATOR_DEFAULT_LDFLAGS)
 
 server_la_LIBADD = $(top_builddir)/libglusterfs/src/libglusterfs.la \
 	$(top_builddir)/rpc/rpc-lib/src/libgfrpc.la \
-	$(top_builddir)/rpc/xdr/src/libgfxdr.la
+	$(top_builddir)/rpc/xdr/src/libgfxdr.la $(LIB_DL)
 
 server_la_SOURCES = server.c server-resolve.c server-helpers.c  \
-	server-rpc-fops.c server-handshake.c authenticate.c
+	server-rpc-fops.c server-handshake.c authenticate.c  \
+	server-common.c server-rpc-fops_v2.c
+
+server_la_HEADERS = server.h server-helpers.h server-mem-types.h \
+	authenticate.h server-messages.h server-common.h
 
-noinst_HEADERS = server.h server-helpers.h server-mem-types.h authenticate.h
+server_ladir = $(includedir)/glusterfs/server
 
-AM_CPPFLAGS = $(GF_CPPFLAGS) \
-	-I$(top_srcdir)/libglusterfs/src \
+AM_CPPFLAGS = $(GF_CPPFLAGS) -I$(top_srcdir)/libglusterfs/src \
 	-DCONFDIR=\"$(sysconfdir)/glusterfs\" \
 	-DLIBDIR=\"$(libdir)/glusterfs/$(PACKAGE_VERSION)/auth\" \
 	-I$(top_srcdir)/xlators/protocol/lib/src \
 	-I$(top_srcdir)/rpc/rpc-lib/src \
-	-I$(top_srcdir)/rpc/xdr/src
+	-I$(top_srcdir)/rpc/xdr/src -I$(top_builddir)/rpc/xdr/src \
+        -I$(top_srcdir)/glusterfsd/src
 
 AM_CFLAGS = -Wall $(GF_CFLAGS) \
             -DDATADIR=\"$(localstatedir)\"
diff --git a/xlators/protocol/server/src/authenticate.c b/xlators/protocol/server/src/authenticate.c
index d8d138a84bc..c1229f9ebf3 100644
--- a/xlators/protocol/server/src/authenticate.c
+++ b/xlators/protocol/server/src/authenticate.c
@@ -8,13 +8,6 @@
   cases as published by the Free Software Foundation.
 */
 
-
-
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
-
 #ifndef _GNU_SOURCE
 #define _GNU_SOURCE
 #endif
@@ -23,231 +16,217 @@
 #include <dlfcn.h>
 #include <errno.h>
 #include "authenticate.h"
+#include "server-messages.h"
 
 static int
-init (dict_t *this, char *key, data_t *value, void *data)
+init(dict_t *this, char *key, data_t *value, void *data)
 {
-        void          *handle       = NULL;
-        char          *auth_file    = NULL;
-        auth_handle_t *auth_handle  = NULL;
-        auth_fn_t      authenticate = NULL;
-        int           *error        = NULL;
-        int            ret          = 0;
-
-        /* It gets over written */
-        error = data;
-
-        if (!strncasecmp (key, "ip", strlen ("ip"))) {
-                gf_log ("authenticate", GF_LOG_ERROR,
-                        "AUTHENTICATION MODULE \"IP\" HAS BEEN REPLACED "
-                        "BY \"ADDR\"");
-                dict_set (this, key, data_from_dynptr (NULL, 0));
-                /* TODO: 1.3.x backword compatibility */
-                // *error = -1;
-                // return;
-                key = "addr";
-        }
-
-        ret = gf_asprintf (&auth_file, "%s/%s.so", LIBDIR, key);
-        if (-1 == ret) {
-                dict_set (this, key, data_from_dynptr (NULL, 0));
-                *error = -1;
-                return -1;
-        }
-
-        handle = dlopen (auth_file, RTLD_LAZY);
-        if (!handle) {
-                gf_log ("authenticate", GF_LOG_ERROR, "dlopen(%s): %s\n",
-                        auth_file, dlerror ());
-                dict_set (this, key, data_from_dynptr (NULL, 0));
-                GF_FREE (auth_file);
-                *error = -1;
-                return -1;
-        }
-        GF_FREE (auth_file);
-
-        authenticate = dlsym (handle, "gf_auth");
-        if (!authenticate) {
-                gf_log ("authenticate", GF_LOG_ERROR,
-                        "dlsym(gf_auth) on %s\n", dlerror ());
-                dict_set (this, key, data_from_dynptr (NULL, 0));
-                dlclose (handle);
-                *error = -1;
-                return -1;
-        }
-
-        auth_handle = GF_CALLOC (1, sizeof (*auth_handle),
-                                 gf_common_mt_auth_handle_t);
-        if (!auth_handle) {
-                dict_set (this, key, data_from_dynptr (NULL, 0));
-                *error = -1;
-                dlclose (handle);
-                return -1;
-        }
-        auth_handle->vol_opt = GF_CALLOC (1, sizeof (volume_opt_list_t),
-                                          gf_common_mt_volume_opt_list_t);
-        if (!auth_handle->vol_opt) {
-                dict_set (this, key, data_from_dynptr (NULL, 0));
-                *error = -1;
-                GF_FREE (auth_handle);
-                dlclose (handle);
-                return -1;
-        }
-        auth_handle->vol_opt->given_opt = dlsym (handle, "options");
-        if (auth_handle->vol_opt->given_opt == NULL) {
-                gf_log ("authenticate", GF_LOG_DEBUG,
-                        "volume option validation not specified");
-        }
-
-        auth_handle->authenticate = authenticate;
-        auth_handle->handle = handle;
-
-        dict_set (this, key,
-                  data_from_dynptr (auth_handle, sizeof (*auth_handle)));
-        return 0;
+    void *handle = NULL;
+    char *auth_file = NULL;
+    auth_handle_t *auth_handle = NULL;
+    auth_fn_t authenticate = NULL;
+    int *error = NULL;
+    int ret = 0;
+
+    /* It gets over written */
+    error = data;
+
+    if (!strncasecmp(key, "ip", SLEN("ip"))) {
+        gf_msg("authenticate", GF_LOG_ERROR, 0, PS_MSG_AUTHENTICATE_ERROR,
+               "AUTHENTICATION MODULE "
+               "\"IP\" HAS BEEN REPLACED BY \"ADDR\"");
+        dict_set(this, key, data_from_dynptr(NULL, 0));
+        /* TODO: 1.3.x backward compatibility */
+        // *error = -1;
+        // return;
+        key = "addr";
+    }
+
+    ret = gf_asprintf(&auth_file, "%s/%s.so", LIBDIR, key);
+    if (-1 == ret) {
+        dict_set(this, key, data_from_dynptr(NULL, 0));
+        *error = -1;
+        return -1;
+    }
+
+    handle = dlopen(auth_file, RTLD_LAZY);
+    if (!handle) {
+        gf_msg("authenticate", GF_LOG_ERROR, 0, PS_MSG_AUTHENTICATE_ERROR,
+               "dlopen(%s): %s\n", auth_file, dlerror());
+        dict_set(this, key, data_from_dynptr(NULL, 0));
+        GF_FREE(auth_file);
+        *error = -1;
+        return -1;
+    }
+    GF_FREE(auth_file);
+
+    authenticate = dlsym(handle, "gf_auth");
+    if (!authenticate) {
+        gf_msg("authenticate", GF_LOG_ERROR, 0, PS_MSG_AUTHENTICATE_ERROR,
+               "dlsym(gf_auth) on %s\n", dlerror());
+        dict_set(this, key, data_from_dynptr(NULL, 0));
+        dlclose(handle);
+        *error = -1;
+        return -1;
+    }
+
+    auth_handle = GF_CALLOC(1, sizeof(*auth_handle),
+                            gf_common_mt_auth_handle_t);
+    if (!auth_handle) {
+        dict_set(this, key, data_from_dynptr(NULL, 0));
+        *error = -1;
+        dlclose(handle);
+        return -1;
+    }
+    auth_handle->vol_opt = GF_CALLOC(1, sizeof(volume_opt_list_t),
+                                     gf_common_mt_volume_opt_list_t);
+    if (!auth_handle->vol_opt) {
+        dict_set(this, key, data_from_dynptr(NULL, 0));
+        *error = -1;
+        GF_FREE(auth_handle);
+        dlclose(handle);
+        return -1;
+    }
+    auth_handle->vol_opt->given_opt = dlsym(handle, "options");
+    if (auth_handle->vol_opt->given_opt == NULL) {
+        gf_msg_debug("authenticate", 0,
+                     "volume option validation "
+                     "not specified");
+    }
+
+    auth_handle->authenticate = authenticate;
+    auth_handle->handle = handle;
+
+    dict_set(this, key, data_from_dynptr(auth_handle, sizeof(*auth_handle)));
+    return 0;
 }
 
 static int
-fini (dict_t *this, char *key, data_t *value, void *data)
+fini(dict_t *this, char *key, data_t *value, void *data)
 {
-        auth_handle_t *handle = data_to_ptr (value);
-        if (handle) {
-                dlclose (handle->handle);
-        }
-        return 0;
+    auth_handle_t *handle = data_to_ptr(value);
+    if (handle) {
+        dlclose(handle->handle);
+    }
+    return 0;
 }
 
 static int
-_gf_auth_option_validate (dict_t *d, char *k, data_t *v, void *tmp)
+_gf_auth_option_validate(dict_t *d, char *k, data_t *v, void *tmp)
 {
-        auth_handle_t *handle = NULL;
-        xlator_t      *xl = NULL;
-        int ret = 0;
+    auth_handle_t *handle = NULL;
+    xlator_t *xl = NULL;
+    int ret = 0;
 
-        xl = tmp;
+    xl = tmp;
 
-        handle = data_to_ptr (v);
-        if (!handle)
-                return 0;
+    handle = data_to_ptr(v);
+    if (!handle)
+        return 0;
 
-        list_add_tail (&(handle->vol_opt->list), &(xl->volume_options));
+    list_add_tail(&(handle->vol_opt->list), &(xl->volume_options));
 
-        ret = xlator_options_validate_list (xl, xl->options,
-                                            handle->vol_opt, NULL);
-        if (ret) {
-                gf_log ("authenticate", GF_LOG_ERROR,
-                        "volume option validation failed");
-                return -1;
-        }
-        return 0;
+    ret = xlator_options_validate_list(xl, xl->options, handle->vol_opt, NULL);
+    if (ret) {
+        gf_msg("authenticate", GF_LOG_ERROR, 0, PS_MSG_VOL_VALIDATE_FAILED,
+               "volume option validation "
+               "failed");
+        return -1;
+    }
+    return 0;
 }
 
 int32_t
-gf_auth_init (xlator_t *xl, dict_t *auth_modules)
+gf_auth_init(xlator_t *xl, dict_t *auth_modules)
 {
-        int ret = 0;
+    int ret = 0;
 
-        dict_foreach (auth_modules, init, &ret);
-        if (ret)
-                goto out;
+    dict_foreach(auth_modules, init, &ret);
+    if (ret)
+        goto out;
 
-        ret = dict_foreach (auth_modules, _gf_auth_option_validate, xl);
+    ret = dict_foreach(auth_modules, _gf_auth_option_validate, xl);
 
 out:
-        if (ret) {
-                gf_log (xl->name, GF_LOG_ERROR, "authentication init failed");
-                dict_foreach (auth_modules, fini, &ret);
-                ret = -1;
-        }
-        return ret;
+    if (ret) {
+        gf_msg(xl->name, GF_LOG_ERROR, 0, PS_MSG_AUTH_INIT_FAILED,
+               "authentication init failed");
+        dict_foreach(auth_modules, fini, &ret);
+        ret = -1;
+    }
+    return ret;
 }
 
-static dict_t *__input_params;
-static dict_t *__config_params;
+typedef struct {
+    dict_t *iparams;
+    dict_t *cparams;
+    int64_t result;
+} gf_auth_args_t;
 
-int
-map (dict_t *this, char *key, data_t *value, void *data)
+static int
+gf_auth_one_method(dict_t *this, char *key, data_t *value, void *data)
 {
-        dict_t *res = data;
-        auth_fn_t authenticate;
-        auth_handle_t *handle = NULL;
-
-        if (value && (handle = data_to_ptr (value)) &&
-            (authenticate = handle->authenticate)) {
-                dict_set (res, key,
-                          int_to_data (authenticate (__input_params,
-                                                     __config_params)));
-        } else {
-                dict_set (res, key, int_to_data (AUTH_DONT_CARE));
-        }
+    gf_auth_args_t *args = data;
+    auth_handle_t *handle = NULL;
+
+    if (!value) {
         return 0;
-}
+    }
 
-int
-reduce (dict_t *this, char *key, data_t *value, void *data)
-{
-        int64_t val = 0;
-        int64_t *res = data;
-        if (!data)
-                return 0;
-
-        val = data_to_int64 (value);
-        switch (val)
-        {
-        case AUTH_ACCEPT:
-                if (AUTH_DONT_CARE == *res)
-                        *res = AUTH_ACCEPT;
-                break;
+    handle = data_to_ptr(value);
+    if (!handle || !handle->authenticate) {
+        return 0;
+    }
 
+    switch (handle->authenticate(args->iparams, args->cparams)) {
+        case AUTH_ACCEPT:
+            if (args->result != AUTH_REJECT) {
+                args->result = AUTH_ACCEPT;
+            }
+            /* FALLTHROUGH */
+        default:
+            return 0;
         case AUTH_REJECT:
-                *res = AUTH_REJECT;
-                break;
-
-        case AUTH_DONT_CARE:
-                break;
-        }
-        return 0;
+            args->result = AUTH_REJECT;
+            return -1;
+    }
 }
 
-
 auth_result_t
-gf_authenticate (dict_t *input_params,
-                 dict_t *config_params,
-                 dict_t *auth_modules)
+gf_authenticate(dict_t *input_params, dict_t *config_params,
+                dict_t *auth_modules)
 {
-        char *name = NULL;
-        dict_t *results = NULL;
-        int64_t result = AUTH_DONT_CARE;
-        data_t *peerinfo_data = NULL;
+    char *name = NULL;
+    data_t *peerinfo_data = NULL;
+    gf_auth_args_t args;
 
-        results = get_new_dict ();
-        __input_params = input_params;
-        __config_params = config_params;
+    args.iparams = input_params;
+    args.cparams = config_params;
+    args.result = AUTH_DONT_CARE;
 
-        dict_foreach (auth_modules, map, results);
+    dict_foreach(auth_modules, gf_auth_one_method, &args);
 
-        dict_foreach (results, reduce, &result);
-        if (AUTH_DONT_CARE == result) {
-                peerinfo_data = dict_get (input_params, "peer-info-name");
+    if (AUTH_DONT_CARE == args.result) {
+        peerinfo_data = dict_get(input_params, "peer-info-name");
 
-                if (peerinfo_data) {
-                        name = peerinfo_data->data;
-                }
-
-                gf_log ("auth", GF_LOG_ERROR,
-                        "no authentication module is interested in "
-                        "accepting remote-client %s", name);
-                result = AUTH_REJECT;
+        if (peerinfo_data) {
+            name = peerinfo_data->data;
         }
 
-        dict_destroy (results);
-        return result;
+        gf_msg("auth", GF_LOG_ERROR, 0, PS_MSG_REMOTE_CLIENT_REFUSED,
+               "no authentication module is interested in "
+               "accepting remote-client %s",
+               name);
+        args.result = AUTH_REJECT;
+    }
+
+    return args.result;
 }
 
 void
-gf_auth_fini (dict_t *auth_modules)
+gf_auth_fini(dict_t *auth_modules)
 {
-        int32_t dummy;
+    int32_t dummy;
 
-        dict_foreach (auth_modules, fini, &dummy);
+    dict_foreach(auth_modules, fini, &dummy);
 }
diff --git a/xlators/protocol/server/src/authenticate.h b/xlators/protocol/server/src/authenticate.h
index d4d43e49873..6888cf696e6 100644
--- a/xlators/protocol/server/src/authenticate.h
+++ b/xlators/protocol/server/src/authenticate.h
@@ -11,41 +11,32 @@
 #ifndef _AUTHENTICATE_H
 #define _AUTHENTICATE_H
 
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
-
 #ifndef _GNU_SOURCE
 #define _GNU_SOURCE
 #endif
 
 #include <stdio.h>
 #include <fnmatch.h>
-#include "dict.h"
-#include "compat.h"
-#include "list.h"
-#include "xlator.h"
+#include <glusterfs/dict.h>
+#include <glusterfs/compat.h>
+#include <glusterfs/list.h>
+#include <glusterfs/xlator.h>
 
-typedef enum {
-        AUTH_ACCEPT,
-        AUTH_REJECT,
-        AUTH_DONT_CARE
-} auth_result_t;
+typedef enum { AUTH_ACCEPT, AUTH_REJECT, AUTH_DONT_CARE } auth_result_t;
 
-typedef auth_result_t (*auth_fn_t) (dict_t *input_params,
-                                    dict_t *config_params);
+typedef auth_result_t (*auth_fn_t)(dict_t *input_params, dict_t *config_params);
 
 typedef struct {
-        void              *handle;
-        auth_fn_t          authenticate;
-        volume_opt_list_t *vol_opt;
+    void *handle;
+    auth_fn_t authenticate;
+    volume_opt_list_t *vol_opt;
 } auth_handle_t;
 
-auth_result_t gf_authenticate (dict_t *input_params,
-                               dict_t *config_params,
-                               dict_t *auth_modules);
-int32_t gf_auth_init (xlator_t *xl, dict_t *auth_modules);
-void gf_auth_fini (dict_t *auth_modules);
+int32_t
+gf_auth_init(xlator_t *xl, dict_t *auth_modules);
+void
+gf_auth_fini(dict_t *auth_modules);
+auth_result_t
+gf_authenticate(dict_t *, dict_t *, dict_t *);
 
 #endif /* _AUTHENTICATE_H */
diff --git a/xlators/protocol/server/src/server-common.c b/xlators/protocol/server/src/server-common.c
new file mode 100644
index 00000000000..cd79cf4d930
--- /dev/null
+++ b/xlators/protocol/server/src/server-common.c
@@ -0,0 +1,842 @@
+#include "server.h"
+#include <glusterfs/defaults.h>
+#include "rpc-common-xdr.h"
+#include "glusterfs3-xdr.h"
+#include "glusterfs3.h"
+#include <glusterfs/compat-errno.h>
+#include "server-messages.h"
+#include "server-helpers.h"
+#include <glusterfs/defaults.h>
+#include <glusterfs/fd.h>
+#include "xdr-nfs3.h"
+
+void
+server_post_stat(server_state_t *state, gfs3_stat_rsp *rsp, struct iatt *stbuf)
+{
+    if (state->client->subdir_mount &&
+        !gf_uuid_compare(stbuf->ia_gfid, state->client->subdir_gfid)) {
+        /* This is very important as when we send iatt of
+           root-inode, fuse/client expect the gfid to be 1,
+           along with inode number. As for subdirectory mount,
+           we use inode table which is shared by everyone, but
+           make sure we send fops only from subdir and below,
+           we have to alter inode gfid and send it to client */
+        static uuid_t gfid = {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1};
+
+        stbuf->ia_ino = 1;
+        gf_uuid_copy(stbuf->ia_gfid, gfid);
+    }
+
+    gf_stat_from_iatt(&rsp->stat, stbuf);
+}
+
+void
+server_post_readlink(gfs3_readlink_rsp *rsp, struct iatt *stbuf,
+                     const char *buf)
+{
+    gf_stat_from_iatt(&rsp->buf, stbuf);
+    rsp->path = (char *)buf;
+}
+
+void
+server_post_mknod(server_state_t *state, gfs3_mknod_rsp *rsp,
+                  struct iatt *stbuf, struct iatt *preparent,
+                  struct iatt *postparent, inode_t *inode)
+{
+    inode_t *link_inode = NULL;
+
+    gf_stat_from_iatt(&rsp->stat, stbuf);
+    gf_stat_from_iatt(&rsp->preparent, preparent);
+    gf_stat_from_iatt(&rsp->postparent, postparent);
+
+    link_inode = inode_link(inode, state->loc.parent, state->loc.name, stbuf);
+    inode_lookup(link_inode);
+    inode_unref(link_inode);
+}
+
+void
+server_post_mkdir(server_state_t *state, gfs3_mkdir_rsp *rsp, inode_t *inode,
+                  struct iatt *stbuf, struct iatt *preparent,
+                  struct iatt *postparent, dict_t *xdata)
+{
+    inode_t *link_inode = NULL;
+
+    gf_stat_from_iatt(&rsp->stat, stbuf);
+    gf_stat_from_iatt(&rsp->preparent, preparent);
+    gf_stat_from_iatt(&rsp->postparent, postparent);
+
+    link_inode = inode_link(inode, state->loc.parent, state->loc.name, stbuf);
+    inode_lookup(link_inode);
+    inode_unref(link_inode);
+}
+
+void
+server_post_unlink(server_state_t *state, gfs3_unlink_rsp *rsp,
+                   struct iatt *preparent, struct iatt *postparent)
+{
+    inode_unlink(state->loc.inode, state->loc.parent, state->loc.name);
+
+    forget_inode_if_no_dentry(state->loc.inode);
+
+    gf_stat_from_iatt(&rsp->preparent, preparent);
+    gf_stat_from_iatt(&rsp->postparent, postparent);
+}
+
+void
+server_post_rmdir(server_state_t *state, gfs3_rmdir_rsp *rsp,
+                  struct iatt *preparent, struct iatt *postparent)
+{
+    inode_unlink(state->loc.inode, state->loc.parent, state->loc.name);
+    /* parent should not be found for directories after
+     * inode_unlink, since directories cannot have
+     * hardlinks.
+     */
+    forget_inode_if_no_dentry(state->loc.inode);
+
+    gf_stat_from_iatt(&rsp->preparent, preparent);
+    gf_stat_from_iatt(&rsp->postparent, postparent);
+}
+
+void
+server_post_symlink(server_state_t *state, gfs3_symlink_rsp *rsp,
+                    inode_t *inode, struct iatt *stbuf, struct iatt *preparent,
+                    struct iatt *postparent, dict_t *xdata)
+{
+    inode_t *link_inode = NULL;
+
+    gf_stat_from_iatt(&rsp->stat, stbuf);
+    gf_stat_from_iatt(&rsp->preparent, preparent);
+    gf_stat_from_iatt(&rsp->postparent, postparent);
+
+    link_inode = inode_link(inode, state->loc.parent, state->loc.name, stbuf);
+    inode_lookup(link_inode);
+    inode_unref(link_inode);
+}
+
+void
+server_post_link(server_state_t *state, gfs3_link_rsp *rsp, inode_t *inode,
+                 struct iatt *stbuf, struct iatt *preparent,
+                 struct iatt *postparent, dict_t *xdata)
+{
+    inode_t *link_inode = NULL;
+
+    gf_stat_from_iatt(&rsp->stat, stbuf);
+    gf_stat_from_iatt(&rsp->preparent, preparent);
+    gf_stat_from_iatt(&rsp->postparent, postparent);
+
+    link_inode = inode_link(inode, state->loc2.parent, state->loc2.name, stbuf);
+    inode_lookup(link_inode);
+    inode_unref(link_inode);
+}
+
+void
+server_post_truncate(gfs3_truncate_rsp *rsp, struct iatt *prebuf,
+                     struct iatt *postbuf)
+{
+    gf_stat_from_iatt(&rsp->prestat, prebuf);
+    gf_stat_from_iatt(&rsp->poststat, postbuf);
+}
+
+void
+server_post_writev(gfs3_write_rsp *rsp, struct iatt *prebuf,
+                   struct iatt *postbuf)
+{
+    gf_stat_from_iatt(&rsp->prestat, prebuf);
+    gf_stat_from_iatt(&rsp->poststat, postbuf);
+}
+
+void
+server_post_statfs(gfs3_statfs_rsp *rsp, struct statvfs *stbuf)
+{
+    gf_statfs_from_statfs(&rsp->statfs, stbuf);
+}
+
+void
+server_post_fsync(gfs3_fsync_rsp *rsp, struct iatt *prebuf,
+                  struct iatt *postbuf)
+{
+    gf_stat_from_iatt(&rsp->prestat, prebuf);
+    gf_stat_from_iatt(&rsp->poststat, postbuf);
+}
+
+void
+server_post_ftruncate(gfs3_ftruncate_rsp *rsp, struct iatt *prebuf,
+                      struct iatt *postbuf)
+{
+    gf_stat_from_iatt(&rsp->prestat, prebuf);
+    gf_stat_from_iatt(&rsp->poststat, postbuf);
+}
+
+void
+server_post_fstat(server_state_t *state, gfs3_fstat_rsp *rsp,
+                  struct iatt *stbuf)
+{
+    if (state->client->subdir_mount &&
+        !gf_uuid_compare(stbuf->ia_gfid, state->client->subdir_gfid)) {
+        /* This is very important as when we send iatt of
+           root-inode, fuse/client expect the gfid to be 1,
+           along with inode number. As for subdirectory mount,
+           we use inode table which is shared by everyone, but
+           make sure we send fops only from subdir and below,
+           we have to alter inode gfid and send it to client */
+        static uuid_t gfid = {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1};
+
+        stbuf->ia_ino = 1;
+        gf_uuid_copy(stbuf->ia_gfid, gfid);
+    }
+
+    gf_stat_from_iatt(&rsp->stat, stbuf);
+}
+
+void
+server_post_lk(xlator_t *this, gfs3_lk_rsp *rsp, struct gf_flock *lock)
+{
+    switch (lock->l_type) {
+        case F_RDLCK:
+            lock->l_type = GF_LK_F_RDLCK;
+            break;
+        case F_WRLCK:
+            lock->l_type = GF_LK_F_WRLCK;
+            break;
+        case F_UNLCK:
+            lock->l_type = GF_LK_F_UNLCK;
+            break;
+        default:
+            gf_msg(this->name, GF_LOG_ERROR, 0, PS_MSG_LOCK_ERROR,
+                   "Unknown lock type: %" PRId32 "!", lock->l_type);
+            break;
+    }
+
+    gf_proto_flock_from_flock(&rsp->flock, lock);
+}
+
+int
+server_post_readdir(gfs3_readdir_rsp *rsp, gf_dirent_t *entries)
+{
+    int ret = 0;
+
+    ret = serialize_rsp_dirent(entries, rsp);
+
+    return ret;
+}
+void
+server_post_zerofill(gfs3_zerofill_rsp *rsp, struct iatt *statpre,
+                     struct iatt *statpost)
+{
+    gf_stat_from_iatt(&rsp->statpre, statpre);
+    gf_stat_from_iatt(&rsp->statpost, statpost);
+}
+
+void
+server_post_discard(gfs3_discard_rsp *rsp, struct iatt *statpre,
+                    struct iatt *statpost)
+{
+    gf_stat_from_iatt(&rsp->statpre, statpre);
+    gf_stat_from_iatt(&rsp->statpost, statpost);
+}
+
+void
+server_post_fallocate(gfs3_fallocate_rsp *rsp, struct iatt *statpre,
+                      struct iatt *statpost)
+{
+    gf_stat_from_iatt(&rsp->statpre, statpre);
+    gf_stat_from_iatt(&rsp->statpost, statpost);
+}
+
+void
+server_post_seek(gfs3_seek_rsp *rsp, off_t offset)
+{
+    rsp->offset = offset;
+}
+
+int
+server_post_readdirp(gfs3_readdirp_rsp *rsp, gf_dirent_t *entries)
+{
+    int ret = 0;
+
+    ret = serialize_rsp_direntp(entries, rsp);
+
+    return ret;
+}
+
+void
+server_post_fsetattr(gfs3_fsetattr_rsp *rsp, struct iatt *statpre,
+                     struct iatt *statpost)
+{
+    gf_stat_from_iatt(&rsp->statpre, statpre);
+    gf_stat_from_iatt(&rsp->statpost, statpost);
+}
+
+void
+server_post_setattr(gfs3_setattr_rsp *rsp, struct iatt *statpre,
+                    struct iatt *statpost)
+{
+    gf_stat_from_iatt(&rsp->statpre, statpre);
+    gf_stat_from_iatt(&rsp->statpost, statpost);
+}
+
+void
+server_post_rchecksum(gfs3_rchecksum_rsp *rsp, uint32_t weak_checksum,
+                      uint8_t *strong_checksum)
+{
+    rsp->weak_checksum = weak_checksum;
+
+    rsp->strong_checksum.strong_checksum_val = (char *)strong_checksum;
+    rsp->strong_checksum.strong_checksum_len = MD5_DIGEST_LENGTH;
+}
+
+void
+server_post_rename(call_frame_t *frame, server_state_t *state,
+                   gfs3_rename_rsp *rsp, struct iatt *stbuf,
+                   struct iatt *preoldparent, struct iatt *postoldparent,
+                   struct iatt *prenewparent, struct iatt *postnewparent)
+{
+    inode_t *tmp_inode = NULL;
+
+    stbuf->ia_type = state->loc.inode->ia_type;
+
+    /* TODO: log gfid of the inodes */
+    gf_msg_trace(frame->root->client->bound_xl->name, 0,
+                 "%" PRId64
+                 ": "
+                 "RENAME_CBK %s ==> %s",
+                 frame->root->unique, state->loc.name, state->loc2.name);
+
+    /* Before renaming the inode, we have to get the inode for the
+     * destination entry (i.e. inode with state->loc2.parent as
+     * parent and state->loc2.name as name). If it exists, then
+     * unlink that inode, and send forget on that inode if the
+     * unlinked entry is the last entry. In case of fuse client
+     * the fuse kernel module itself sends the forget on the
+     * unlinked inode.
+     */
+    tmp_inode = inode_grep(state->loc.inode->table, state->loc2.parent,
+                           state->loc2.name);
+    if (tmp_inode) {
+        inode_unlink(tmp_inode, state->loc2.parent, state->loc2.name);
+        forget_inode_if_no_dentry(tmp_inode);
+        inode_unref(tmp_inode);
+    }
+
+    inode_rename(state->itable, state->loc.parent, state->loc.name,
+                 state->loc2.parent, state->loc2.name, state->loc.inode, stbuf);
+    gf_stat_from_iatt(&rsp->stat, stbuf);
+
+    gf_stat_from_iatt(&rsp->preoldparent, preoldparent);
+    gf_stat_from_iatt(&rsp->postoldparent, postoldparent);
+
+    gf_stat_from_iatt(&rsp->prenewparent, prenewparent);
+    gf_stat_from_iatt(&rsp->postnewparent, postnewparent);
+}
+
+int
+server_post_open(call_frame_t *frame, xlator_t *this, gfs3_open_rsp *rsp,
+                 fd_t *fd)
+{
+    server_ctx_t *serv_ctx = NULL;
+    uint64_t fd_no = 0;
+
+    serv_ctx = server_ctx_get(frame->root->client, this);
+    if (serv_ctx == NULL) {
+        gf_msg(this->name, GF_LOG_INFO, 0, PS_MSG_SERVER_CTX_GET_FAILED,
+               "server_ctx_get() "
+               "failed");
+        return -1;
+    }
+
+    fd_bind(fd);
+    fd_ref(fd);
+    fd_no = gf_fd_unused_get(serv_ctx->fdtable, fd);
+    rsp->fd = fd_no;
+
+    return 0;
+}
+
+void
+server_post_readv(gfs3_read_rsp *rsp, struct iatt *stbuf, int op_ret)
+{
+    gf_stat_from_iatt(&rsp->stat, stbuf);
+    rsp->size = op_ret;
+}
+
+int
+server_post_opendir(call_frame_t *frame, xlator_t *this, gfs3_opendir_rsp *rsp,
+                    fd_t *fd)
+{
+    server_ctx_t *serv_ctx = NULL;
+    uint64_t fd_no = 0;
+
+    serv_ctx = server_ctx_get(frame->root->client, this);
+    if (serv_ctx == NULL) {
+        gf_msg(this->name, GF_LOG_INFO, 0, PS_MSG_SERVER_CTX_GET_FAILED,
+               "server_ctx_get() "
+               "failed");
+        return -1;
+    }
+
+    fd_bind(fd);
+    fd_ref(fd);
+    fd_no = gf_fd_unused_get(serv_ctx->fdtable, fd);
+    rsp->fd = fd_no;
+
+    return 0;
+}
+
+int
+server_post_create(call_frame_t *frame, gfs3_create_rsp *rsp,
+                   server_state_t *state, xlator_t *this, fd_t *fd,
+                   inode_t *inode, struct iatt *stbuf, struct iatt *preparent,
+                   struct iatt *postparent)
+{
+    server_ctx_t *serv_ctx = NULL;
+    inode_t *link_inode = NULL;
+    uint64_t fd_no = 0;
+    int op_errno = 0;
+
+    link_inode = inode_link(inode, state->loc.parent, state->loc.name, stbuf);
+
+    if (!link_inode) {
+        op_errno = ENOENT;
+        goto out;
+    }
+
+    if (link_inode != inode) {
+        /*
+          VERY racy code (if used anywhere else)
+          -- don't do this without understanding
+        */
+
+        inode_ctx_merge(fd, fd->inode, link_inode);
+        inode_unref(fd->inode);
+        fd->inode = inode_ref(link_inode);
+    }
+
+    inode_lookup(link_inode);
+    inode_unref(link_inode);
+
+    serv_ctx = server_ctx_get(frame->root->client, this);
+    if (serv_ctx == NULL) {
+        gf_msg(this->name, GF_LOG_INFO, 0, PS_MSG_SERVER_CTX_GET_FAILED,
+               "server_ctx_get() "
+               "failed");
+        goto out;
+    }
+
+    fd_bind(fd);
+    fd_ref(fd);
+    fd_no = gf_fd_unused_get(serv_ctx->fdtable, fd);
+
+    if ((fd_no > UINT64_MAX) || (fd == 0)) {
+        op_errno = errno;
+    }
+
+    rsp->fd = fd_no;
+    gf_stat_from_iatt(&rsp->stat, stbuf);
+    gf_stat_from_iatt(&rsp->preparent, preparent);
+    gf_stat_from_iatt(&rsp->postparent, postparent);
+
+    return 0;
+out:
+    return -op_errno;
+}
+
+/*TODO: Handle revalidate path */
+void
+server_post_lookup(gfs3_lookup_rsp *rsp, call_frame_t *frame,
+                   server_state_t *state, inode_t *inode, struct iatt *stbuf,
+                   struct iatt *postparent)
+{
+    inode_t *root_inode = NULL;
+    inode_t *link_inode = NULL;
+    static uuid_t rootgfid = {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1};
+
+    root_inode = frame->root->client->bound_xl->itable->root;
+
+    if (!__is_root_gfid(inode->gfid)) {
+        link_inode = inode_link(inode, state->loc.parent, state->loc.name,
+                                stbuf);
+        if (link_inode) {
+            inode_lookup(link_inode);
+            inode_unref(link_inode);
+        }
+    }
+
+    if ((inode == root_inode) || (state->client->subdir_mount &&
+                                  (inode == state->client->subdir_inode))) {
+        /* we just looked up root ("/") OR
+           subdir mount directory, which is root ('/') in client */
+        /* This is very important as when we send iatt of
+           root-inode, fuse/client expect the gfid to be 1,
+           along with inode number. As for subdirectory mount,
+           we use inode table which is shared by everyone, but
+           make sure we send fops only from subdir and below,
+           we have to alter inode gfid and send it to client */
+        stbuf->ia_ino = 1;
+        gf_uuid_copy(stbuf->ia_gfid, rootgfid);
+        if (inode->ia_type == 0)
+            inode->ia_type = stbuf->ia_type;
+    }
+
+    gf_stat_from_iatt(&rsp->stat, stbuf);
+}
+
+void
+server_post_lease(gfs3_lease_rsp *rsp, struct gf_lease *lease)
+{
+    gf_proto_lease_from_lease(&rsp->lease, lease);
+}
+
+/* Version 4 helpers */
+
+void
+server4_post_readlink(gfx_readlink_rsp *rsp, struct iatt *stbuf,
+                      const char *buf)
+{
+    gfx_stat_from_iattx(&rsp->buf, stbuf);
+    rsp->path = (char *)buf;
+}
+
+void
+server4_post_common_3iatt(server_state_t *state, gfx_common_3iatt_rsp *rsp,
+                          inode_t *inode, struct iatt *stbuf,
+                          struct iatt *preparent, struct iatt *postparent)
+{
+    inode_t *link_inode = NULL;
+
+    gfx_stat_from_iattx(&rsp->stat, stbuf);
+    if (state->client->subdir_mount &&
+        !gf_uuid_compare(preparent->ia_gfid, state->client->subdir_gfid)) {
+        /* This is very important as when we send iatt of
+           root-inode, fuse/client expect the gfid to be 1,
+           along with inode number. As for subdirectory mount,
+           we use inode table which is shared by everyone, but
+           make sure we send fops only from subdir and below,
+           we have to alter inode gfid and send it to client */
+        static uuid_t gfid = {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1};
+
+        preparent->ia_ino = 1;
+        postparent->ia_ino = 1;
+        gf_uuid_copy(preparent->ia_gfid, gfid);
+        gf_uuid_copy(postparent->ia_gfid, gfid);
+    }
+
+    gfx_stat_from_iattx(&rsp->preparent, preparent);
+    gfx_stat_from_iattx(&rsp->postparent, postparent);
+
+    link_inode = inode_link(inode, state->loc.parent, state->loc.name, stbuf);
+    inode_lookup(link_inode);
+    inode_unref(link_inode);
+}
+
+void
+server4_post_common_3iatt_noinode(gfx_common_3iatt_rsp *rsp, struct iatt *stbuf,
+                                  struct iatt *prebuf_dst,
+                                  struct iatt *postbuf_dst)
+{
+    gfx_stat_from_iattx(&rsp->stat, stbuf);
+    gfx_stat_from_iattx(&rsp->preparent, prebuf_dst);
+    gfx_stat_from_iattx(&rsp->postparent, postbuf_dst);
+}
+
+void
+server4_post_common_2iatt(gfx_common_2iatt_rsp *rsp, struct iatt *prebuf,
+                          struct iatt *postbuf)
+{
+    gfx_stat_from_iattx(&rsp->prestat, prebuf);
+    gfx_stat_from_iattx(&rsp->poststat, postbuf);
+}
+
+void
+server4_post_entry_remove(server_state_t *state, gfx_common_2iatt_rsp *rsp,
+                          struct iatt *prebuf, struct iatt *postbuf)
+{
+    inode_unlink(state->loc.inode, state->loc.parent, state->loc.name);
+    /* parent should not be found for directories after
+     * inode_unlink, since directories cannot have
+     * hardlinks.
+     */
+    forget_inode_if_no_dentry(state->loc.inode);
+
+    gfx_stat_from_iattx(&rsp->prestat, prebuf);
+    gfx_stat_from_iattx(&rsp->poststat, postbuf);
+}
+
+void
+server4_post_statfs(gfx_statfs_rsp *rsp, struct statvfs *stbuf)
+{
+    gf_statfs_from_statfs(&rsp->statfs, stbuf);
+}
+
+void
+server4_post_common_iatt(server_state_t *state, gfx_common_iatt_rsp *rsp,
+                         struct iatt *stbuf)
+{
+    if (state->client->subdir_mount &&
+        !gf_uuid_compare(stbuf->ia_gfid, state->client->subdir_gfid)) {
+        /* This is very important as when we send iatt of
+           root-inode, fuse/client expect the gfid to be 1,
+           along with inode number. As for subdirectory mount,
+           we use inode table which is shared by everyone, but
+           make sure we send fops only from subdir and below,
+           we have to alter inode gfid and send it to client */
+        static uuid_t gfid = {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1};
+
+        stbuf->ia_ino = 1;
+        gf_uuid_copy(stbuf->ia_gfid, gfid);
+    }
+
+    gfx_stat_from_iattx(&rsp->stat, stbuf);
+}
+
+void
+server4_post_lk(xlator_t *this, gfx_lk_rsp *rsp, struct gf_flock *lock)
+{
+    switch (lock->l_type) {
+        case F_RDLCK:
+            lock->l_type = GF_LK_F_RDLCK;
+            break;
+        case F_WRLCK:
+            lock->l_type = GF_LK_F_WRLCK;
+            break;
+        case F_UNLCK:
+            lock->l_type = GF_LK_F_UNLCK;
+            break;
+        default:
+            gf_msg(this->name, GF_LOG_ERROR, 0, PS_MSG_LOCK_ERROR,
+                   "Unknown lock type: %" PRId32 "!", lock->l_type);
+            break;
+    }
+
+    gf_proto_flock_from_flock(&rsp->flock, lock);
+}
+
+int
+server4_post_readdir(gfx_readdir_rsp *rsp, gf_dirent_t *entries)
+{
+    int ret = 0;
+
+    ret = serialize_rsp_dirent_v2(entries, rsp);
+
+    return ret;
+}
+
+void
+server4_post_seek(gfx_seek_rsp *rsp, off_t offset)
+{
+    rsp->offset = offset;
+}
+
+int
+server4_post_readdirp(gfx_readdirp_rsp *rsp, gf_dirent_t *entries)
+{
+    int ret = 0;
+
+    ret = serialize_rsp_direntp_v2(entries, rsp);
+
+    return ret;
+}
+
+void
+server4_post_rchecksum(gfx_rchecksum_rsp *rsp, uint32_t weak_checksum,
+                       uint8_t *strong_checksum)
+{
+    rsp->weak_checksum = weak_checksum;
+    /* When the length encoding changes, update the change
+       in posix code also. */
+    rsp->strong_checksum.strong_checksum_val = (char *)strong_checksum;
+    rsp->strong_checksum.strong_checksum_len = SHA256_DIGEST_LENGTH;
+    rsp->flags = 1; /* Indicates SHA256 TYPE */
+}
+
+void
+server4_post_rename(call_frame_t *frame, server_state_t *state,
+                    gfx_rename_rsp *rsp, struct iatt *stbuf,
+                    struct iatt *preoldparent, struct iatt *postoldparent,
+                    struct iatt *prenewparent, struct iatt *postnewparent)
+{
+    inode_t *tmp_inode = NULL;
+
+    stbuf->ia_type = state->loc.inode->ia_type;
+
+    /* TODO: log gfid of the inodes */
+    gf_msg_trace(frame->root->client->bound_xl->name, 0,
+                 "%" PRId64
+                 ": "
+                 "RENAME_CBK %s ==> %s",
+                 frame->root->unique, state->loc.name, state->loc2.name);
+
+    /* Before renaming the inode, we have to get the inode for the
+     * destination entry (i.e. inode with state->loc2.parent as
+     * parent and state->loc2.name as name). If it exists, then
+     * unlink that inode, and send forget on that inode if the
+     * unlinked entry is the last entry. In case of fuse client
+     * the fuse kernel module itself sends the forget on the
+     * unlinked inode.
+     */
+    tmp_inode = inode_grep(state->loc.inode->table, state->loc2.parent,
+                           state->loc2.name);
+    if (tmp_inode) {
+        inode_unlink(tmp_inode, state->loc2.parent, state->loc2.name);
+        forget_inode_if_no_dentry(tmp_inode);
+        inode_unref(tmp_inode);
+    }
+
+    inode_rename(state->itable, state->loc.parent, state->loc.name,
+                 state->loc2.parent, state->loc2.name, state->loc.inode, stbuf);
+    gfx_stat_from_iattx(&rsp->stat, stbuf);
+
+    gfx_stat_from_iattx(&rsp->preoldparent, preoldparent);
+    gfx_stat_from_iattx(&rsp->postoldparent, postoldparent);
+
+    gfx_stat_from_iattx(&rsp->prenewparent, prenewparent);
+    gfx_stat_from_iattx(&rsp->postnewparent, postnewparent);
+}
+
+int
+server4_post_open(call_frame_t *frame, xlator_t *this, gfx_open_rsp *rsp,
+                  fd_t *fd)
+{
+    server_ctx_t *serv_ctx = NULL;
+    uint64_t fd_no = 0;
+
+    serv_ctx = server_ctx_get(frame->root->client, this);
+    if (serv_ctx == NULL) {
+        gf_msg(this->name, GF_LOG_INFO, 0, PS_MSG_SERVER_CTX_GET_FAILED,
+               "server_ctx_get() "
+               "failed");
+        return -1;
+    }
+
+    fd_bind(fd);
+    fd_ref(fd);
+    fd_no = gf_fd_unused_get(serv_ctx->fdtable, fd);
+    rsp->fd = fd_no;
+
+    return 0;
+}
+
+void
+server4_post_readv(gfx_read_rsp *rsp, struct iatt *stbuf, int op_ret)
+{
+    gfx_stat_from_iattx(&rsp->stat, stbuf);
+    rsp->size = op_ret;
+}
+
+int
+server4_post_create(call_frame_t *frame, gfx_create_rsp *rsp,
+                    server_state_t *state, xlator_t *this, fd_t *fd,
+                    inode_t *inode, struct iatt *stbuf, struct iatt *preparent,
+                    struct iatt *postparent)
+{
+    server_ctx_t *serv_ctx = NULL;
+    inode_t *link_inode = NULL;
+    uint64_t fd_no = 0;
+    int op_errno = 0;
+
+    link_inode = inode_link(inode, state->loc.parent, state->loc.name, stbuf);
+
+    if (!link_inode) {
+        op_errno = ENOENT;
+        goto out;
+    }
+
+    if (link_inode != inode) {
+        /*
+          VERY racy code (if used anywhere else)
+          -- don't do this without understanding
+        */
+
+        inode_ctx_merge(fd, fd->inode, link_inode);
+        inode_unref(fd->inode);
+        fd->inode = inode_ref(link_inode);
+    }
+
+    inode_lookup(link_inode);
+    inode_unref(link_inode);
+
+    serv_ctx = server_ctx_get(frame->root->client, this);
+    if (serv_ctx == NULL) {
+        gf_msg(this->name, GF_LOG_INFO, 0, PS_MSG_SERVER_CTX_GET_FAILED,
+               "server_ctx_get() "
+               "failed");
+        goto out;
+    }
+
+    fd_bind(fd);
+    fd_ref(fd);
+    fd_no = gf_fd_unused_get(serv_ctx->fdtable, fd);
+
+    if ((fd_no > UINT64_MAX) || (fd == 0)) {
+        op_errno = errno;
+    }
+
+    rsp->fd = fd_no;
+    gfx_stat_from_iattx(&rsp->stat, stbuf);
+    gfx_stat_from_iattx(&rsp->preparent, preparent);
+    gfx_stat_from_iattx(&rsp->postparent, postparent);
+
+    return 0;
+out:
+    return -op_errno;
+}
+
+/*TODO: Handle revalidate path */
+void
+server4_post_lookup(gfx_common_2iatt_rsp *rsp, call_frame_t *frame,
+                    server_state_t *state, inode_t *inode, struct iatt *stbuf)
+{
+    inode_t *root_inode = NULL;
+    inode_t *link_inode = NULL;
+    static uuid_t rootgfid = {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1};
+
+    root_inode = frame->root->client->bound_xl->itable->root;
+
+    if (!__is_root_gfid(inode->gfid)) {
+        link_inode = inode_link(inode, state->loc.parent, state->loc.name,
+                                stbuf);
+        if (link_inode) {
+            inode_lookup(link_inode);
+            inode_unref(link_inode);
+        }
+    }
+
+    if ((inode == root_inode) || (state->client->subdir_mount &&
+                                  (inode == state->client->subdir_inode))) {
+        /* we just looked up root ("/") OR
+           subdir mount directory, which is root ('/') in client */
+        /* This is very important as when we send iatt of
+           root-inode, fuse/client expect the gfid to be 1,
+           along with inode number. As for subdirectory mount,
+           we use inode table which is shared by everyone, but
+           make sure we send fops only from subdir and below,
+           we have to alter inode gfid and send it to client */
+        stbuf->ia_ino = 1;
+        gf_uuid_copy(stbuf->ia_gfid, rootgfid);
+        if (inode->ia_type == 0)
+            inode->ia_type = stbuf->ia_type;
+    }
+
+    gfx_stat_from_iattx(&rsp->prestat, stbuf);
+}
+
+void
+server4_post_lease(gfx_lease_rsp *rsp, struct gf_lease *lease)
+{
+    gf_proto_lease_from_lease(&rsp->lease, lease);
+}
+
+void
+server4_post_link(server_state_t *state, gfx_common_3iatt_rsp *rsp,
+                  inode_t *inode, struct iatt *stbuf, struct iatt *preparent,
+                  struct iatt *postparent)
+{
+    inode_t *link_inode = NULL;
+
+    gfx_stat_from_iattx(&rsp->stat, stbuf);
+    gfx_stat_from_iattx(&rsp->preparent, preparent);
+    gfx_stat_from_iattx(&rsp->postparent, postparent);
+
+    link_inode = inode_link(inode, state->loc2.parent, state->loc2.name, stbuf);
+    inode_lookup(link_inode);
+    inode_unref(link_inode);
+}
diff --git a/xlators/protocol/server/src/server-common.h b/xlators/protocol/server/src/server-common.h
new file mode 100644
index 00000000000..6200415e304
--- /dev/null
+++ b/xlators/protocol/server/src/server-common.h
@@ -0,0 +1,199 @@
+#include "server.h"
+#include <glusterfs/defaults.h>
+#include "rpc-common-xdr.h"
+#include "glusterfs3-xdr.h"
+#include "glusterfs3.h"
+#include <glusterfs/compat-errno.h>
+#include "server-messages.h"
+#include <glusterfs/defaults.h>
+
+#include "xdr-nfs3.h"
+void
+server_post_stat(server_state_t *state, gfs3_stat_rsp *rsp, struct iatt *stbuf);
+
+void
+server_post_readlink(gfs3_readlink_rsp *rsp, struct iatt *stbuf,
+                     const char *buf);
+
+void
+server_post_mknod(server_state_t *state, gfs3_mknod_rsp *rsp,
+                  struct iatt *stbuf, struct iatt *preparent,
+                  struct iatt *postparent, inode_t *inode);
+void
+server_post_mkdir(server_state_t *state, gfs3_mkdir_rsp *rsp, inode_t *inode,
+                  struct iatt *stbuf, struct iatt *preparent,
+                  struct iatt *postparent, dict_t *xdata);
+
+void
+server_post_unlink(server_state_t *state, gfs3_unlink_rsp *rsp,
+                   struct iatt *preparent, struct iatt *postparent);
+void
+server_post_rmdir(server_state_t *state, gfs3_rmdir_rsp *rsp,
+                  struct iatt *preparent, struct iatt *postparent);
+
+void
+server_post_symlink(server_state_t *state, gfs3_symlink_rsp *rsp,
+                    inode_t *inode, struct iatt *stbuf, struct iatt *preparent,
+                    struct iatt *postparent, dict_t *xdata);
+void
+server_post_link(server_state_t *state, gfs3_link_rsp *rsp, inode_t *inode,
+                 struct iatt *stbuf, struct iatt *preparent,
+                 struct iatt *postparent, dict_t *xdata);
+void
+server_post_truncate(gfs3_truncate_rsp *rsp, struct iatt *prebuf,
+                     struct iatt *postbuf);
+
+void
+server_post_writev(gfs3_write_rsp *rsp, struct iatt *prebuf,
+                   struct iatt *postbuf);
+void
+server_post_statfs(gfs3_statfs_rsp *rsp, struct statvfs *stbuf);
+
+void
+server_post_fsync(gfs3_fsync_rsp *rsp, struct iatt *prebuf,
+                  struct iatt *postbuf);
+
+void
+server_post_ftruncate(gfs3_ftruncate_rsp *rsp, struct iatt *prebuf,
+                      struct iatt *postbuf);
+
+void
+server_post_fstat(server_state_t *state, gfs3_fstat_rsp *rsp,
+                  struct iatt *stbuf);
+
+void
+server_post_lk(xlator_t *this, gfs3_lk_rsp *rsp, struct gf_flock *lock);
+
+int
+server_post_readdir(gfs3_readdir_rsp *rsp, gf_dirent_t *entries);
+
+void
+server_post_zerofill(gfs3_zerofill_rsp *rsp, struct iatt *statpre,
+                     struct iatt *statpost);
+
+void
+server_post_discard(gfs3_discard_rsp *rsp, struct iatt *statpre,
+                    struct iatt *statpost);
+
+void
+server_post_fallocate(gfs3_fallocate_rsp *rsp, struct iatt *statpre,
+                      struct iatt *statpost);
+
+void
+server_post_seek(gfs3_seek_rsp *rsp, off_t offset);
+
+int
+server_post_readdirp(gfs3_readdirp_rsp *rsp, gf_dirent_t *entries);
+
+void
+server_post_fsetattr(gfs3_fsetattr_rsp *rsp, struct iatt *statpre,
+                     struct iatt *statpost);
+
+void
+server_post_setattr(gfs3_setattr_rsp *rsp, struct iatt *statpre,
+                    struct iatt *statpost);
+
+void
+server_post_rchecksum(gfs3_rchecksum_rsp *rsp, uint32_t weak_checksum,
+                      uint8_t *strong_checksum);
+
+void
+server_post_rename(call_frame_t *frame, server_state_t *state,
+                   gfs3_rename_rsp *rsp, struct iatt *stbuf,
+                   struct iatt *preoldparent, struct iatt *postoldparent,
+                   struct iatt *prenewparent, struct iatt *postnewparent);
+
+int
+server_post_open(call_frame_t *frame, xlator_t *this, gfs3_open_rsp *rsp,
+                 fd_t *fd);
+void
+server_post_readv(gfs3_read_rsp *rsp, struct iatt *stbuf, int op_ret);
+
+int
+server_post_opendir(call_frame_t *frame, xlator_t *this, gfs3_opendir_rsp *rsp,
+                    fd_t *fd);
+
+int
+server_post_create(call_frame_t *frame, gfs3_create_rsp *rsp,
+                   server_state_t *state, xlator_t *this, fd_t *fd,
+                   inode_t *inode, struct iatt *stbuf, struct iatt *preparent,
+                   struct iatt *postparent);
+
+void
+server_post_lookup(gfs3_lookup_rsp *rsp, call_frame_t *frame,
+                   server_state_t *state, inode_t *inode, struct iatt *stbuf,
+                   struct iatt *postparent);
+
+void
+server_post_lease(gfs3_lease_rsp *rsp, struct gf_lease *lease);
+
+void
+server4_post_readlink(gfx_readlink_rsp *rsp, struct iatt *stbuf,
+                      const char *buf);
+
+void
+server4_post_statfs(gfx_statfs_rsp *rsp, struct statvfs *stbuf);
+
+void
+server4_post_lk(xlator_t *this, gfx_lk_rsp *rsp, struct gf_flock *lock);
+
+int
+server4_post_readdir(gfx_readdir_rsp *rsp, gf_dirent_t *entries);
+
+void
+server4_post_seek(gfx_seek_rsp *rsp, off_t offset);
+
+int
+server4_post_readdirp(gfx_readdirp_rsp *rsp, gf_dirent_t *entries);
+
+void
+server4_post_rchecksum(gfx_rchecksum_rsp *rsp, uint32_t weak_checksum,
+                       uint8_t *strong_checksum);
+
+void
+server4_post_rename(call_frame_t *frame, server_state_t *state,
+                    gfx_rename_rsp *rsp, struct iatt *stbuf,
+                    struct iatt *preoldparent, struct iatt *postoldparent,
+                    struct iatt *prenewparent, struct iatt *postnewparent);
+
+int
+server4_post_open(call_frame_t *frame, xlator_t *this, gfx_open_rsp *rsp,
+                  fd_t *fd);
+void
+server4_post_readv(gfx_read_rsp *rsp, struct iatt *stbuf, int op_ret);
+
+int
+server4_post_create(call_frame_t *frame, gfx_create_rsp *rsp,
+                    server_state_t *state, xlator_t *this, fd_t *fd,
+                    inode_t *inode, struct iatt *stbuf, struct iatt *preparent,
+                    struct iatt *postparent);
+
+void
+server4_post_common_2iatt(gfx_common_2iatt_rsp *rsp, struct iatt *stbuf1,
+                          struct iatt *stbuf2);
+
+void
+server4_post_entry_remove(server_state_t *state, gfx_common_2iatt_rsp *rsp,
+                          struct iatt *stbuf1, struct iatt *stbuf2);
+
+void
+server4_post_common_3iatt(server_state_t *state, gfx_common_3iatt_rsp *rsp,
+                          inode_t *inode, struct iatt *stbuf, struct iatt *pre,
+                          struct iatt *post);
+void
+server4_post_common_iatt(server_state_t *state, gfx_common_iatt_rsp *rsp,
+                         struct iatt *stbuf);
+void
+server4_post_lease(gfx_lease_rsp *rsp, struct gf_lease *lease);
+void
+server4_post_lookup(gfx_common_2iatt_rsp *rsp, call_frame_t *frame,
+                    server_state_t *state, inode_t *inode, struct iatt *stbuf);
+void
+server4_post_link(server_state_t *state, gfx_common_3iatt_rsp *rsp,
+                  inode_t *inode, struct iatt *stbuf, struct iatt *pre,
+                  struct iatt *post);
+
+void
+server4_post_common_3iatt_noinode(gfx_common_3iatt_rsp *rsp, struct iatt *stbuf,
+                                  struct iatt *prebuf_dst,
+                                  struct iatt *postbuf_dst);
diff --git a/xlators/protocol/server/src/server-handshake.c b/xlators/protocol/server/src/server-handshake.c
index 9396499cc3d..85c87c1ab8b 100644
--- a/xlators/protocol/server/src/server-handshake.c
+++ b/xlators/protocol/server/src/server-handshake.c
@@ -8,785 +8,789 @@
   cases as published by the Free Software Foundation.
 */
 
-
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
-
 #include "server.h"
 #include "server-helpers.h"
 #include "rpc-common-xdr.h"
 #include "glusterfs3-xdr.h"
-#include "compat-errno.h"
+#include <glusterfs/compat-errno.h>
 #include "glusterfs3.h"
 #include "authenticate.h"
+#include "server-messages.h"
+#include <glusterfs/syscall.h>
+#include <glusterfs/events.h>
+#include <glusterfs/syncop.h>
 
 struct __get_xl_struct {
-        const char *name;
-        xlator_t *reply;
+    const char *name;
+    xlator_t *reply;
 };
 int
-gf_compare_client_version (rpcsvc_request_t *req, int fop_prognum,
-                           int mgmt_prognum)
-{
-        int ret = -1;
-        /* TODO: think.. */
-        if (glusterfs3_3_fop_prog.prognum == fop_prognum)
-                ret = 0;
-
-        return ret;
-}
-
-void __check_and_set (xlator_t *each, void *data)
-{
-        if (!strcmp (each->name,
-                     ((struct __get_xl_struct *) data)->name))
-                ((struct __get_xl_struct *) data)->reply = each;
-}
-
-static xlator_t *
-get_xlator_by_name (xlator_t *some_xl, const char *name)
+gf_compare_client_version(rpcsvc_request_t *req, int fop_prognum,
+                          int mgmt_prognum)
 {
-        struct __get_xl_struct get = {
-                .name = name,
-                .reply = NULL
-        };
+    int ret = -1;
+    /* TODO: think.. */
+    if (glusterfs3_3_fop_prog.prognum == fop_prognum)
+        ret = 0;
 
-        xlator_foreach (some_xl, __check_and_set, &get);
-
-        return get.reply;
+    return ret;
 }
 
-
 int
-_volfile_update_checksum (xlator_t *this, char *key, uint32_t checksum)
+server_getspec(rpcsvc_request_t *req)
 {
-        server_conf_t       *conf         = NULL;
-        struct _volfile_ctx *temp_volfile = NULL;
-
-        conf         = this->private;
-        temp_volfile = conf->volfile;
-
-        while (temp_volfile) {
-                if ((NULL == key) && (NULL == temp_volfile->key))
-                        break;
-                if ((NULL == key) || (NULL == temp_volfile->key)) {
-                        temp_volfile = temp_volfile->next;
-                        continue;
-                }
-                if (strcmp (temp_volfile->key, key) == 0)
-                        break;
-                temp_volfile = temp_volfile->next;
-        }
-
-        if (!temp_volfile) {
-                temp_volfile = GF_CALLOC (1, sizeof (struct _volfile_ctx),
-                                          gf_server_mt_volfile_ctx_t);
-                if (!temp_volfile)
-                        goto out;
-                temp_volfile->next  = conf->volfile;
-                temp_volfile->key   = (key)? gf_strdup (key): NULL;
-                temp_volfile->checksum = checksum;
-
-                conf->volfile = temp_volfile;
-                goto out;
-        }
+    int32_t ret = 0;
+    int32_t op_errno = ENOENT;
+    gf_getspec_req args = {
+        0,
+    };
+    gf_getspec_rsp rsp = {
+        0,
+    };
+
+    ret = xdr_to_generic(req->msg[0], &args, (xdrproc_t)xdr_gf_getspec_req);
+    if (ret < 0) {
+        // failed to decode msg;
+        req->rpc_err = GARBAGE_ARGS;
+        op_errno = EINVAL;
+        goto fail;
+    }
+
+    op_errno = ENOSYS;
+fail:
+    rsp.spec = "<this method is not in use, use glusterd for getspec>";
+    rsp.op_errno = gf_errno_to_error(op_errno);
+    rsp.op_ret = -1;
 
-        if (temp_volfile->checksum != checksum) {
-                gf_log (this->name, GF_LOG_INFO,
-                        "the volume file was modified between a prior access "
-                        "and now. This may lead to inconsistency between "
-                        "clients, you are advised to remount client");
-                temp_volfile->checksum  = checksum;
-        }
+    server_submit_reply(NULL, req, &rsp, NULL, 0, NULL,
+                        (xdrproc_t)xdr_gf_getspec_rsp);
 
-out:
-        return 0;
+    return 0;
 }
 
-
-static size_t
-getspec_build_volfile_path (xlator_t *this, const char *key, char *path,
-                            size_t path_len)
+static void
+server_first_lookup_done(rpcsvc_request_t *req, gf_setvolume_rsp *rsp)
 {
-        char            *filename      = NULL;
-        server_conf_t   *conf          = NULL;
-        int              ret           = -1;
-        int              free_filename = 0;
-        char             data_key[256] = {0,};
-
-        conf = this->private;
-
-        /* Inform users that this option is changed now */
-        ret = dict_get_str (this->options, "client-volume-filename",
-                            &filename);
-        if (ret == 0) {
-                gf_log (this->name, GF_LOG_WARNING,
-                        "option 'client-volume-filename' is changed to "
-                        "'volume-filename.<key>' which now takes 'key' as an "
-                        "option to choose/fetch different files from server. "
-                        "Refer documentation or contact developers for more "
-                        "info. Currently defaulting to given file '%s'",
-                        filename);
-        }
-
-        if (key && !filename) {
-                sprintf (data_key, "volume-filename.%s", key);
-                ret = dict_get_str (this->options, data_key, &filename);
-                if (ret < 0) {
-                        /* Make sure that key doesn't contain "../" in path */
-                        if ((gf_strstr (key, "/", "..")) == -1) {
-                                gf_log (this->name, GF_LOG_ERROR,
-                                        "%s: invalid key", key);
-                                goto out;
-                        }
-                }
-        }
-
-        if (!filename) {
-                ret = dict_get_str (this->options,
-                                    "volume-filename.default", &filename);
-                if (ret < 0) {
-                        gf_log (this->name, GF_LOG_DEBUG,
-                                "no default volume filename given, "
-                                "defaulting to %s", DEFAULT_VOLUME_FILE_PATH);
-                }
-        }
+    server_submit_reply(NULL, req, rsp, NULL, 0, NULL,
+                        (xdrproc_t)xdr_gf_setvolume_rsp);
 
-        if (!filename && key) {
-                ret = gf_asprintf (&filename, "%s/%s.vol", conf->conf_dir, key);
-                if (-1 == ret)
-                        goto out;
-
-                free_filename = 1;
-        }
-        if (!filename)
-                filename = DEFAULT_VOLUME_FILE_PATH;
-
-        ret = -1;
-
-        if ((filename) && (path_len > strlen (filename))) {
-                strcpy (path, filename);
-                ret = strlen (filename);
-        }
-
-out:
-        if (free_filename)
-                GF_FREE (filename);
-
-        return ret;
+    GF_FREE(rsp->dict.dict_val);
+    GF_FREE(rsp);
 }
 
-int
-_validate_volfile_checksum (xlator_t *this, char *key,
-                            uint32_t checksum)
+static inode_t *
+do_path_lookup(xlator_t *xl, dict_t *dict, inode_t *parinode, char *basename)
 {
-        char                 filename[PATH_MAX] = {0,};
-        server_conf_t       *conf         = NULL;
-        struct _volfile_ctx *temp_volfile = NULL;
-        int                  ret          = 0;
-        int                  fd           = 0;
-        uint32_t             local_checksum = 0;
-
-        conf         = this->private;
-        temp_volfile = conf->volfile;
-
-        if (!checksum)
-                goto out;
-
-        if (!temp_volfile) {
-                ret = getspec_build_volfile_path (this, key, filename,
-                                                  sizeof (filename));
-                if (ret <= 0)
-                        goto out;
-                fd = open (filename, O_RDONLY);
-                if (-1 == fd) {
-                        ret = 0;
-                        gf_log (this->name, GF_LOG_INFO,
-                                "failed to open volume file (%s) : %s",
-                                filename, strerror (errno));
-                        goto out;
-                }
-                get_checksum_for_file (fd, &local_checksum);
-                _volfile_update_checksum (this, key, local_checksum);
-                close (fd);
-        }
-
-        temp_volfile = conf->volfile;
-        while (temp_volfile) {
-                if ((NULL == key) && (NULL == temp_volfile->key))
-                        break;
-                if ((NULL == key) || (NULL == temp_volfile->key)) {
-                        temp_volfile = temp_volfile->next;
-                        continue;
-                }
-                if (strcmp (temp_volfile->key, key) == 0)
-                        break;
-                temp_volfile = temp_volfile->next;
-        }
-
-        if (!temp_volfile)
-                goto out;
-
-        if ((temp_volfile->checksum) &&
-            (checksum != temp_volfile->checksum))
-                ret = -1;
+    int ret = 0;
+    loc_t loc = {
+        0,
+    };
+    uuid_t gfid = {
+        0,
+    };
+    struct iatt iatt = {
+        0,
+    };
+    inode_t *inode = NULL;
+
+    loc.parent = parinode;
+    loc_touchup(&loc, basename);
+    loc.inode = inode_new(xl->itable);
+
+    gf_uuid_generate(gfid);
+    ret = dict_set_gfuuid(dict, "gfid-req", gfid, true);
+    if (ret) {
+        gf_log(xl->name, GF_LOG_ERROR, "failed to set 'gfid-req' for subdir");
+        goto out;
+    }
+
+    ret = syncop_lookup(xl, &loc, &iatt, NULL, dict, NULL);
+    if (ret < 0) {
+        gf_log(xl->name, GF_LOG_ERROR, "first lookup on subdir (%s) failed: %s",
+               basename, strerror(errno));
+    }
+
+    /* Inode linking is required so that the
+       resolution happens all fine for future fops */
+    inode = inode_link(loc.inode, loc.parent, loc.name, &iatt);
+
+    /* Extra ref so the pointer is valid till client is valid */
+    /* FIXME: not a priority, but this can lead to some inode
+       leaks if subdir is more than 1 level depth. Leak is only
+       per subdir entry, and not dependent on number of
+       connections, so it should be fine for now */
+    inode_ref(inode);
 
 out:
-        return ret;
+    return inode;
 }
 
-
 int
-server_getspec (rpcsvc_request_t *req)
+server_first_lookup(xlator_t *this, client_t *client, dict_t *reply)
 {
-        int32_t              ret                    = -1;
-        int32_t              op_errno               = ENOENT;
-        int32_t              spec_fd                = -1;
-        size_t               file_len               = 0;
-        char                 filename[PATH_MAX]  = {0,};
-        struct stat          stbuf                  = {0,};
-        uint32_t             checksum               = 0;
-        char                *key                    = NULL;
-        server_conf_t       *conf                   = NULL;
-        xlator_t            *this                   = NULL;
-        gf_getspec_req       args                   = {0,};
-        gf_getspec_rsp       rsp                    = {0,};
-
-        this = req->svc->mydata;
-        conf = this->private;
-        ret = xdr_to_generic (req->msg[0], &args,
-                              (xdrproc_t)xdr_gf_getspec_req);
-        if (ret < 0) {
-                //failed to decode msg;
-                req->rpc_err = GARBAGE_ARGS;
-                op_errno = EINVAL;
+    loc_t loc = {
+        0,
+    };
+    struct iatt iatt = {
+        0,
+    };
+    dict_t *dict = NULL;
+    int ret = 0;
+    xlator_t *xl = client->bound_xl;
+    char *msg = NULL;
+    inode_t *inode = NULL;
+    char *bname = NULL;
+    char *str = NULL;
+    char *tmp = NULL;
+    char *saveptr = NULL;
+
+    loc.path = "/";
+    loc.name = "";
+    loc.inode = xl->itable->root;
+    loc.parent = NULL;
+    gf_uuid_copy(loc.gfid, loc.inode->gfid);
+
+    ret = syncop_lookup(xl, &loc, &iatt, NULL, NULL, NULL);
+    if (ret < 0)
+        gf_log(xl->name, GF_LOG_ERROR, "lookup on root failed: %s",
+               strerror(errno));
+    /* Ignore error from lookup, don't set
+     * failure in rsp->op_ret. lookup on a snapview-server
+     * can fail with ESTALE
+     */
+    /* TODO-SUBDIR-MOUNT: validate above comment with respect to subdir lookup
+     */
+
+    if (client->subdir_mount) {
+        str = tmp = gf_strdup(client->subdir_mount);
+        dict = dict_new();
+        inode = xl->itable->root;
+        bname = strtok_r(str, "/", &saveptr);
+        while (bname != NULL) {
+            inode = do_path_lookup(xl, dict, inode, bname);
+            if (inode == NULL) {
+                gf_log(this->name, GF_LOG_ERROR,
+                       "first lookup on subdir (%s) failed: %s",
+                       client->subdir_mount, strerror(errno));
+                ret = -1;
                 goto fail;
+            }
+            bname = strtok_r(NULL, "/", &saveptr);
         }
 
-        ret = getspec_build_volfile_path (this, args.key,
-                                          filename, sizeof (filename));
-        if (ret > 0) {
-                /* to allocate the proper buffer to hold the file data */
-                ret = stat (filename, &stbuf);
-                if (ret < 0){
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "Unable to stat %s (%s)",
-                                filename, strerror (errno));
-                        op_errno = errno;
-                        goto fail;
-                }
-
-                spec_fd = open (filename, O_RDONLY);
-                if (spec_fd < 0) {
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "Unable to open %s (%s)",
-                                filename, strerror (errno));
-                        op_errno = errno;
-                        goto fail;
-                }
-                ret = file_len = stbuf.st_size;
-
-                if (conf->verify_volfile) {
-                        get_checksum_for_file (spec_fd, &checksum);
-                        _volfile_update_checksum (this, key, checksum);
-                }
-        } else {
-                op_errno = ENOENT;
-        }
+        /* Can be used in server_resolve() */
+        gf_uuid_copy(client->subdir_gfid, inode->gfid);
+        client->subdir_inode = inode;
+    }
 
-        if (file_len) {
-                rsp.spec = GF_CALLOC (file_len, sizeof (char),
-                                      gf_server_mt_rsp_buf_t);
-                if (!rsp.spec) {
-                        ret = -1;
-                        op_errno = ENOMEM;
-                        goto fail;
-                }
-                ret = read (spec_fd, rsp.spec, file_len);
-        }
+    ret = 0;
+    goto out;
 
-        /* convert to XDR */
-        op_errno = errno;
 fail:
-        if (!rsp.spec)
-                rsp.spec = "";
-        rsp.op_errno = gf_errno_to_error (op_errno);
-        rsp.op_ret   = ret;
+    /* we should say to client, it is not possible
+       to connect */
+    ret = gf_asprintf(&msg, "subdirectory for mount \"%s\" is not found",
+                      client->subdir_mount);
+    if (-1 == ret) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, PS_MSG_ASPRINTF_FAILED,
+               "asprintf failed while setting error msg");
+    }
+    ret = dict_set_dynstr(reply, "ERROR", msg);
+    if (ret < 0)
+        gf_msg_debug(this->name, 0,
+                     "failed to set error "
+                     "msg");
+
+    ret = -1;
+out:
+    if (dict)
+        dict_unref(dict);
 
-        if (spec_fd != -1)
-                close (spec_fd);
+    inode_unref(loc.inode);
 
-        server_submit_reply (NULL, req, &rsp, NULL, 0, NULL,
-                             (xdrproc_t)xdr_gf_getspec_rsp);
+    if (tmp)
+        GF_FREE(tmp);
 
-        return 0;
+    return ret;
 }
 
-
 int
-server_setvolume (rpcsvc_request_t *req)
+server_setvolume(rpcsvc_request_t *req)
 {
-        gf_setvolume_req     args          = {{0,},};
-        gf_setvolume_rsp     rsp           = {0,};
-        client_t            *client        = NULL;
-        server_ctx_t        *serv_ctx      = NULL;
-        server_conf_t       *conf          = NULL;
-        peer_info_t         *peerinfo      = NULL;
-        dict_t              *reply         = NULL;
-        dict_t              *config_params = NULL;
-        dict_t              *params        = NULL;
-        char                *name          = NULL;
-        char                *client_uid    = NULL;
-        char                *clnt_version  = NULL;
-        xlator_t            *xl            = NULL;
-        char                *msg           = NULL;
-        char                *volfile_key   = NULL;
-        xlator_t            *this          = NULL;
-        uint32_t             checksum      = 0;
-        int32_t              ret           = -1;
-        int32_t              op_ret        = -1;
-        int32_t              op_errno      = EINVAL;
-        int32_t              fop_version   = 0;
-        int32_t              mgmt_version  = 0;
-        uint32_t             lk_version    = 0;
-        char                *buf           = NULL;
-        gf_boolean_t        cancelled      = _gf_false;
-
-        params = dict_new ();
-        reply  = dict_new ();
-        ret = xdr_to_generic (req->msg[0], &args,
-                              (xdrproc_t)xdr_gf_setvolume_req);
-        if (ret < 0) {
-                //failed to decode msg;
-                req->rpc_err = GARBAGE_ARGS;
-                goto fail;
-        }
-
-        this = req->svc->mydata;
-
-        config_params = dict_copy_with_ref (this->options, NULL);
-        conf          = this->private;
-
-        buf = memdup (args.dict.dict_val, args.dict.dict_len);
-        if (buf == NULL) {
-                op_ret = -1;
-                op_errno = ENOMEM;
-                goto fail;
-        }
-
-        ret = dict_unserialize (buf, args.dict.dict_len, &params);
-        if (ret < 0) {
-                ret = dict_set_str (reply, "ERROR",
-                                    "Internal error: failed to unserialize "
-                                    "request dictionary");
-                if (ret < 0)
-                        gf_log (this->name, GF_LOG_DEBUG,
-                                "failed to set error msg \"%s\"",
-                                "Internal error: failed to unserialize "
-                                "request dictionary");
-
-                op_ret = -1;
-                op_errno = EINVAL;
-                goto fail;
-        }
-
-        params->extra_free = buf;
-        buf = NULL;
-
-        ret = dict_get_str (params, "process-uuid", &client_uid);
-        if (ret < 0) {
-                ret = dict_set_str (reply, "ERROR",
-                                    "UUID not specified");
-                if (ret < 0)
-                        gf_log (this->name, GF_LOG_DEBUG,
-                                "failed to set error msg");
-
-                op_ret = -1;
-                op_errno = EINVAL;
-                goto fail;
-        }
-
-        /*lk_verion :: [1..2^31-1]*/
-        ret = dict_get_uint32 (params, "clnt-lk-version", &lk_version);
-        if (ret < 0) {
-                ret = dict_set_str (reply, "ERROR",
-                                    "lock state version not supplied");
-                if (ret < 0)
-                        gf_log (this->name, GF_LOG_DEBUG,
-                                "failed to set error msg");
-
-                op_ret = -1;
-                op_errno = EINVAL;
-                goto fail;
-        }
-
-        client = gf_client_get (this, &req->cred, client_uid);
-        if (client == NULL) {
-                op_ret = -1;
-                op_errno = ENOMEM;
-                goto fail;
-        }
-
-        gf_log (this->name, GF_LOG_DEBUG, "Connected to %s", client->client_uid);
-        cancelled = server_cancel_grace_timer (this, client);
-        if (cancelled)//Do gf_client_put on behalf of grace-timer-handler.
-                gf_client_put (client, NULL);
-
-        serv_ctx = server_ctx_get (client, client->this);
-        if (serv_ctx == NULL) {
-                gf_log (this->name, GF_LOG_INFO, "server_ctx_get() failed");
-                goto fail;
-        }
-
-        if (serv_ctx->lk_version != 0 &&
-            serv_ctx->lk_version != lk_version) {
-                (void) server_connection_cleanup (this, client,
-                                                  INTERNAL_LOCKS | POSIX_LOCKS);
-        }
-
-        if (req->trans->xl_private != client)
-                req->trans->xl_private = client;
-
-        auth_set_username_passwd (params, config_params, client);
-        if (req->trans->ssl_name) {
-                if (dict_set_str(params,"ssl-name",req->trans->ssl_name) != 0) {
-                        gf_log (this->name, GF_LOG_WARNING,
-                                "failed to set ssl_name %s", req->trans->ssl_name);
-                        /* Not fatal, auth will just fail. */
-                }
-        }
-
-        ret = dict_get_int32 (params, "fops-version", &fop_version);
-        if (ret < 0) {
-                ret = dict_set_str (reply, "ERROR",
-                                    "No FOP version number specified");
-                if (ret < 0)
-                        gf_log (this->name, GF_LOG_DEBUG,
-                                "failed to set error msg");
-        }
-
-        ret = dict_get_int32 (params, "mgmt-version", &mgmt_version);
-        if (ret < 0) {
-                ret = dict_set_str (reply, "ERROR",
-                                    "No MGMT version number specified");
-                if (ret < 0)
-                        gf_log (this->name, GF_LOG_DEBUG,
-                                "failed to set error msg");
-        }
-
-        ret = gf_compare_client_version (req, fop_version, mgmt_version);
-        if (ret != 0) {
-                ret = gf_asprintf (&msg, "version mismatch: client(%d)"
-                                   " - client-mgmt(%d)",
-                                   fop_version, mgmt_version);
-                /* get_supported_version (req)); */
-                if (-1 == ret) {
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "asprintf failed while setting up error msg");
-                        goto fail;
-                }
-                ret = dict_set_dynstr (reply, "ERROR", msg);
-                if (ret < 0)
-                        gf_log (this->name, GF_LOG_DEBUG,
-                                "failed to set error msg");
-
-                op_ret = -1;
-                op_errno = EINVAL;
-                goto fail;
-        }
-
-        ret = dict_get_str (params, "remote-subvolume", &name);
-        if (ret < 0) {
-                ret = dict_set_str (reply, "ERROR",
-                                    "No remote-subvolume option specified");
-                if (ret < 0)
-                        gf_log (this->name, GF_LOG_DEBUG,
-                                "failed to set error msg");
-
-                op_ret = -1;
-                op_errno = EINVAL;
-                goto fail;
-        }
-
-        xl = get_xlator_by_name (this, name);
-        if (xl == NULL) {
-                ret = gf_asprintf (&msg, "remote-subvolume \"%s\" is not found",
-                                   name);
-                if (-1 == ret) {
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "asprintf failed while setting error msg");
-                        goto fail;
-                }
-                ret = dict_set_dynstr (reply, "ERROR", msg);
-                if (ret < 0)
-                        gf_log (this->name, GF_LOG_DEBUG,
-                                "failed to set error msg");
-
-                op_ret = -1;
-                op_errno = ENOENT;
-                goto fail;
-        }
-
-        if (conf->verify_volfile) {
-                ret = dict_get_uint32 (params, "volfile-checksum", &checksum);
-                if (ret == 0) {
-                        ret = dict_get_str (params, "volfile-key",
-                                            &volfile_key);
-                        if (ret)
-                                gf_log (this->name, GF_LOG_DEBUG,
-                                        "failed to set 'volfile-key'");
-
-                        ret = _validate_volfile_checksum (this, volfile_key,
-                                                          checksum);
-                        if (-1 == ret) {
-                                ret = dict_set_str (reply, "ERROR",
-                                                    "volume-file checksum "
-                                                    "varies from earlier "
-                                                    "access");
-                                if (ret < 0)
-                                        gf_log (this->name, GF_LOG_DEBUG,
-                                                "failed to set error msg");
-
-                                op_ret   = -1;
-                                op_errno = ESTALE;
-                                goto fail;
-                        }
-                }
-        }
-
-
-        peerinfo = &req->trans->peerinfo;
-        if (peerinfo) {
-                ret = dict_set_static_ptr (params, "peer-info", peerinfo);
-                if (ret < 0)
-                        gf_log (this->name, GF_LOG_DEBUG,
-                                "failed to set peer-info");
-        }
-        if (conf->auth_modules == NULL) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "Authentication module not initialized");
-        }
-
-        ret = dict_get_str (params, "client-version", &clnt_version);
-        if (ret)
-                gf_log (this->name, GF_LOG_INFO, "client-version not set, "
-                        "may be of older version");
-
-        ret = gf_authenticate (params, config_params,
-                               conf->auth_modules);
-
-        if (ret == AUTH_ACCEPT) {
-
-                gf_log (this->name, GF_LOG_INFO,
-                        "accepted client from %s (version: %s)",
-                        client->client_uid,
-                        (clnt_version) ? clnt_version : "old");
-                op_ret = 0;
-                client->bound_xl = xl;
-                ret = dict_set_str (reply, "ERROR", "Success");
-                if (ret < 0)
-                        gf_log (this->name, GF_LOG_DEBUG,
-                                "failed to set error msg");
-        } else {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "Cannot authenticate client from %s %s",
-                        client->client_uid,
-                        (clnt_version) ? clnt_version : "old");
-
-                op_ret = -1;
-                op_errno = EACCES;
-                ret = dict_set_str (reply, "ERROR", "Authentication failed");
-                if (ret < 0)
-                        gf_log (this->name, GF_LOG_DEBUG,
-                                "failed to set error msg");
-                goto fail;
-        }
-
-        if (client->bound_xl == NULL) {
-                ret = dict_set_str (reply, "ERROR",
-                                    "Check volfile and handshake "
-                                    "options in protocol/client");
-                if (ret < 0)
-                        gf_log (this->name, GF_LOG_DEBUG,
-                                "failed to set error msg");
-
-                op_ret = -1;
-                op_errno = EACCES;
-                goto fail;
-        }
-
-        if ((client->bound_xl != NULL) &&
-            (ret >= 0)                   &&
-            (client->bound_xl->itable == NULL)) {
-                /* create inode table for this bound_xl, if one doesn't
-                   already exist */
-
-                gf_log (this->name, GF_LOG_TRACE,
-                        "creating inode table with lru_limit=%"PRId32", "
-                        "xlator=%s", conf->inode_lru_limit,
-                        client->bound_xl->name);
-
-                /* TODO: what is this ? */
-                client->bound_xl->itable =
-                        inode_table_new (conf->inode_lru_limit,
-                                         client->bound_xl);
-        }
-
-        ret = dict_set_str (reply, "process-uuid",
-                            this->ctx->process_uuid);
-        if (ret)
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "failed to set 'process-uuid'");
-
-        ret = dict_set_uint32 (reply, "clnt-lk-version", serv_ctx->lk_version);
+    gf_setvolume_req args = {
+        {
+            0,
+        },
+    };
+    gf_setvolume_rsp *rsp = NULL;
+    client_t *client = NULL;
+    server_ctx_t *serv_ctx = NULL;
+    server_conf_t *conf = NULL;
+    peer_info_t *peerinfo = NULL;
+    dict_t *reply = NULL;
+    dict_t *config_params = NULL;
+    dict_t *params = NULL;
+    char *name = NULL;
+    char *volume_id = NULL;
+    char *client_uid = NULL;
+    char *clnt_version = NULL;
+    xlator_t *xl = NULL;
+    char *msg = NULL;
+    xlator_t *this = NULL;
+    int32_t ret = -1;
+    int32_t op_ret = -1;
+    int32_t op_errno = EINVAL;
+    uint32_t opversion = 0;
+    rpc_transport_t *xprt = NULL;
+    int32_t fop_version = 0;
+    int32_t mgmt_version = 0;
+    glusterfs_ctx_t *ctx = NULL;
+    struct _child_status *tmp = NULL;
+    char *subdir_mount = NULL;
+    char *client_name = NULL;
+    gf_boolean_t cleanup_starting = _gf_false;
+    gf_boolean_t xlator_in_graph = _gf_true;
+
+    params = dict_new();
+    reply = dict_new();
+    ret = xdr_to_generic(req->msg[0], &args, (xdrproc_t)xdr_gf_setvolume_req);
+    if (ret < 0) {
+        // failed to decode msg;
+        req->rpc_err = GARBAGE_ARGS;
+        goto fail;
+    }
+    ctx = THIS->ctx;
+
+    this = req->svc->xl;
+    /* this is to ensure config_params is populated with the first brick
+     * details at first place if brick multiplexing is enabled
+     */
+    config_params = dict_copy_with_ref(this->options, NULL);
+
+    ret = dict_unserialize(args.dict.dict_val, args.dict.dict_len, &params);
+    if (ret < 0) {
+        ret = dict_set_sizen_str_sizen(reply, "ERROR",
+                                       "Internal error: failed to unserialize "
+                                       "request dictionary");
+        if (ret < 0)
+            gf_msg_debug(this->name, 0,
+                         "failed to set error "
+                         "msg \"%s\"",
+                         "Internal error: failed "
+                         "to unserialize request dictionary");
+
+        op_ret = -1;
+        op_errno = EINVAL;
+        goto fail;
+    }
+
+    ret = dict_get_str(params, "remote-subvolume", &name);
+    if (ret < 0) {
+        ret = dict_set_str(reply, "ERROR",
+                           "No remote-subvolume option specified");
+        if (ret < 0)
+            gf_msg_debug(this->name, 0,
+                         "failed to set error "
+                         "msg");
+
+        op_ret = -1;
+        op_errno = EINVAL;
+        goto fail;
+    }
+
+    LOCK(&ctx->volfile_lock);
+    {
+        xl = get_xlator_by_name(this, name);
+        if (!xl) {
+            xlator_in_graph = _gf_false;
+            xl = this;
+        }
+    }
+    UNLOCK(&ctx->volfile_lock);
+    if (xl == NULL) {
+        ret = gf_asprintf(&msg, "remote-subvolume \"%s\" is not found", name);
+        if (-1 == ret) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, PS_MSG_ASPRINTF_FAILED,
+                   "asprintf failed while setting error msg");
+            goto fail;
+        }
+        ret = dict_set_dynstr(reply, "ERROR", msg);
+        if (ret < 0)
+            gf_msg_debug(this->name, 0,
+                         "failed to set error "
+                         "msg");
+
+        op_ret = -1;
+        op_errno = ENOENT;
+        goto fail;
+    }
+
+    config_params = dict_copy_with_ref(xl->options, config_params);
+    conf = this->private;
+
+    if (conf->parent_up == _gf_false) {
+        /* PARENT_UP indicates that all xlators in graph are inited
+         * successfully
+         */
+        op_ret = -1;
+        op_errno = EAGAIN;
+
+        ret = dict_set_str(reply, "ERROR",
+                           "xlator graph in server is not initialised "
+                           "yet. Try again later");
+        if (ret < 0)
+            gf_msg_debug(this->name, 0,
+                         "failed to set error: "
+                         "xlator graph in server is not "
+                         "initialised yet. Try again later");
+        goto fail;
+    }
+
+    pthread_mutex_lock(&conf->mutex);
+    list_for_each_entry(tmp, &conf->child_status->status_list, status_list)
+    {
+        if (strcmp(tmp->name, name) == 0)
+            break;
+    }
+
+    if (!tmp->name) {
+        gf_msg(this->name, GF_LOG_INFO, 0, PS_MSG_CHILD_STATUS_FAILED,
+               "No xlator %s is found in child status list", name);
+    } else {
+        ret = dict_set_int32(reply, "child_up", tmp->child_up);
+        if (ret < 0)
+            gf_msg(this->name, GF_LOG_ERROR, 0, PS_MSG_DICT_GET_FAILED,
+                   "Failed to set 'child_up' for xlator %s "
+                   "in the reply dict",
+                   tmp->name);
+        if (!tmp->child_up) {
+            ret = dict_set_str(reply, "ERROR",
+                               "Not received child_up for this xlator");
+            if (ret < 0)
+                gf_msg_debug(this->name, 0, "failed to set error msg");
+
+            gf_msg(this->name, GF_LOG_ERROR, 0, PS_MSG_CHILD_STATUS_FAILED,
+                   "Not received child_up for this xlator %s", name);
+            op_ret = -1;
+            op_errno = EAGAIN;
+            pthread_mutex_unlock(&conf->mutex);
+            goto fail;
+        }
+    }
+    pthread_mutex_unlock(&conf->mutex);
+
+    ret = dict_get_str(params, "process-uuid", &client_uid);
+    if (ret < 0) {
+        ret = dict_set_str(reply, "ERROR", "UUID not specified");
+        if (ret < 0)
+            gf_msg_debug(this->name, 0,
+                         "failed to set error "
+                         "msg");
+
+        op_ret = -1;
+        op_errno = EINVAL;
+        goto fail;
+    }
+
+    ret = dict_get_str(params, "subdir-mount", &subdir_mount);
+    if (ret < 0) {
+        /* Not a problem at all as the key is optional */
+    }
+    ret = dict_get_str(params, "process-name", &client_name);
+    if (ret < 0) {
+        client_name = "unknown";
+    }
+
+    /* If any value is set, the first element will be non-0.
+       It would be '0', but not '\0' :-) */
+    if (xl->graph->volume_id[0]) {
+        ret = dict_get_str_sizen(params, "volume-id", &volume_id);
+        if (!ret && strcmp(xl->graph->volume_id, volume_id)) {
+            ret = dict_set_str(reply, "ERROR",
+                               "Volume-ID different, possible case "
+                               "of same brick re-used in another volume");
+            if (ret < 0)
+                gf_msg_debug(this->name, 0, "failed to set error msg");
+
+            op_ret = -1;
+            op_errno = EINVAL;
+            goto fail;
+        }
+        ret = dict_set_str(reply, "volume-id", tmp->volume_id);
         if (ret)
-                gf_log (this->name, GF_LOG_WARNING,
-                        "failed to set 'clnt-lk-version'");
+            gf_msg_debug(this->name, 0, "failed to set 'volume-id'");
+    }
+    client = gf_client_get(this, &req->cred, client_uid, subdir_mount);
+    if (client == NULL) {
+        op_ret = -1;
+        op_errno = ENOMEM;
+        goto fail;
+    }
+
+    client->client_name = gf_strdup(client_name);
+
+    gf_msg_debug(this->name, 0, "Connected to %s", client->client_uid);
+
+    serv_ctx = server_ctx_get(client, client->this);
+    if (serv_ctx == NULL) {
+        gf_msg(this->name, GF_LOG_INFO, 0, PS_MSG_SERVER_CTX_GET_FAILED,
+               "server_ctx_get() "
+               "failed");
+        goto fail;
+    }
+
+    pthread_mutex_lock(&conf->mutex);
+    if (xl->cleanup_starting) {
+        cleanup_starting = _gf_true;
+    } else if (req->trans->xl_private != client) {
+        req->trans->xl_private = client;
+    }
+    pthread_mutex_unlock(&conf->mutex);
+
+    if (cleanup_starting) {
+        op_ret = -1;
+        op_errno = EAGAIN;
+
+        ret = dict_set_str(reply, "ERROR",
+                           "cleanup flag is set for xlator. "
+                           " Try again later");
+        if (ret < 0)
+            gf_msg_debug(this->name, 0,
+                         "failed to set error: "
+                         "cleanup flag is set for xlator. "
+                         "Try again later");
+        goto fail;
+    }
+
+    auth_set_username_passwd(params, config_params, client);
+    if (req->trans->ssl_name) {
+        if (dict_set_str(params, "ssl-name", req->trans->ssl_name) != 0) {
+            gf_msg(this->name, GF_LOG_WARNING, 0, PS_MSG_SSL_NAME_SET_FAILED,
+                   "failed to set "
+                   "ssl_name %s",
+                   req->trans->ssl_name);
+            /* Not fatal, auth will just fail. */
+        }
+    }
+
+    ret = dict_get_int32(params, "fops-version", &fop_version);
+    if (ret < 0) {
+        ret = dict_set_str(reply, "ERROR", "No FOP version number specified");
+        if (ret < 0)
+            gf_msg_debug(this->name, 0,
+                         "failed to set error "
+                         "msg");
+    }
+
+    ret = dict_get_int32(params, "mgmt-version", &mgmt_version);
+    if (ret < 0) {
+        ret = dict_set_str(reply, "ERROR", "No MGMT version number specified");
+        if (ret < 0)
+            gf_msg_debug(this->name, 0,
+                         "failed to set error "
+                         "msg");
+    }
+
+    ret = gf_compare_client_version(req, fop_version, mgmt_version);
+    if (ret != 0) {
+        ret = gf_asprintf(&msg,
+                          "version mismatch: client(%d)"
+                          " - client-mgmt(%d)",
+                          fop_version, mgmt_version);
+        /* get_supported_version (req)); */
+        if (-1 == ret) {
+            gf_msg(this->name, GF_LOG_ERROR, 0, PS_MSG_ASPRINTF_FAILED,
+                   "asprintf failed while"
+                   "setting up error msg");
+            goto fail;
+        }
+        ret = dict_set_dynstr(reply, "ERROR", msg);
+        if (ret < 0)
+            gf_msg_debug(this->name, 0,
+                         "failed to set error "
+                         "msg");
+
+        op_ret = -1;
+        op_errno = EINVAL;
+        goto fail;
+    }
+
+    peerinfo = &req->trans->peerinfo;
+    if (peerinfo) {
+        ret = dict_set_static_ptr(params, "peer-info", peerinfo);
+        if (ret < 0)
+            gf_msg_debug(this->name, 0,
+                         "failed to set "
+                         "peer-info");
+    }
+
+    ret = dict_get_uint32(params, "opversion", &opversion);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_INFO, 0, PS_MSG_CLIENT_OPVERSION_GET_FAILED,
+               "Failed to get client opversion");
+    }
+    client->opversion = opversion;
+    /* Assign op-version value to the client */
+    pthread_mutex_lock(&conf->mutex);
+    list_for_each_entry(xprt, &conf->xprt_list, list)
+    {
+        if (strcmp(peerinfo->identifier, xprt->peerinfo.identifier))
+            continue;
+        xprt->peerinfo.max_op_version = opversion;
+    }
+    pthread_mutex_unlock(&conf->mutex);
+
+    if (conf->auth_modules == NULL) {
+        gf_msg(this->name, GF_LOG_ERROR, 0, PS_MSG_AUTH_INIT_FAILED,
+               "Authentication module not initialized");
+    }
+
+    ret = dict_get_str(params, "client-version", &clnt_version);
+    if (ret)
+        gf_msg(this->name, GF_LOG_INFO, 0, PS_MSG_CLIENT_VERSION_NOT_SET,
+               "client-version not set, may be of older version");
+
+    ret = gf_authenticate(params, config_params, conf->auth_modules);
+
+    if (ret == AUTH_ACCEPT) {
+        /* Store options received from client side */
+        req->trans->clnt_options = dict_ref(params);
+
+        gf_msg(this->name, GF_LOG_INFO, 0, PS_MSG_CLIENT_ACCEPTED,
+               "accepted client from %s (version: %s) with subvol %s",
+               client->client_uid, (clnt_version) ? clnt_version : "old", name);
+
+        gf_event(EVENT_CLIENT_CONNECT,
+                 "client_uid=%s;"
+                 "client_identifier=%s;server_identifier=%s;"
+                 "brick_path=%s;subdir_mount=%s",
+                 client->client_uid, req->trans->peerinfo.identifier,
+                 req->trans->myinfo.identifier, name, subdir_mount);
 
-        ret = dict_set_uint64 (reply, "transport-ptr",
-                               ((uint64_t) (long) req->trans));
-        if (ret)
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "failed to set 'transport-ptr'");
+        op_ret = 0;
+        client->bound_xl = xl;
+
+        /* Don't be confused by the below line (like how ERROR can
+           be Success), key checked on client is 'ERROR' and hence
+           we send 'Success' in this key */
+        ret = dict_set_str(reply, "ERROR", "Success");
+        if (ret < 0)
+            gf_msg_debug(this->name, 0,
+                         "failed to set error "
+                         "msg");
+    } else {
+        op_ret = -1;
+        if (!xlator_in_graph) {
+            gf_msg(this->name, GF_LOG_ERROR, ENOENT, PS_MSG_AUTHENTICATE_ERROR,
+                   "Cannot authenticate client"
+                   " from %s %s because brick is not attached in graph",
+                   client->client_uid, (clnt_version) ? clnt_version : "old");
+
+            op_errno = ENOENT;
+            ret = dict_set_str(reply, "ERROR", "Brick not found");
+        } else {
+            gf_event(EVENT_CLIENT_AUTH_REJECT,
+                     "client_uid=%s;"
+                     "client_identifier=%s;server_identifier=%s;"
+                     "brick_path=%s",
+                     client->client_uid, req->trans->peerinfo.identifier,
+                     req->trans->myinfo.identifier, name);
+            gf_msg(this->name, GF_LOG_ERROR, EACCES, PS_MSG_AUTHENTICATE_ERROR,
+                   "Cannot authenticate client"
+                   " from %s %s",
+                   client->client_uid, (clnt_version) ? clnt_version : "old");
+
+            op_errno = EACCES;
+            ret = dict_set_str(reply, "ERROR", "Authentication failed");
+        }
+        if (ret < 0)
+            gf_msg_debug(this->name, 0,
+                         "failed to set error "
+                         "msg");
+        goto fail;
+    }
+
+    if (client->bound_xl == NULL) {
+        ret = dict_set_str(reply, "ERROR",
+                           "Check volfile and handshake "
+                           "options in protocol/client");
+        if (ret < 0)
+            gf_msg_debug(this->name, 0,
+                         "failed to set error "
+                         "msg");
+
+        op_ret = -1;
+        op_errno = EACCES;
+        goto fail;
+    }
+
+    LOCK(&conf->itable_lock);
+    {
+        if (client->bound_xl->itable == NULL) {
+            /* create inode table for this bound_xl, if one doesn't
+               already exist */
+
+            gf_msg_trace(this->name, 0,
+                         "creating inode table with"
+                         " lru_limit=%" PRId32 ", xlator=%s",
+                         conf->inode_lru_limit, client->bound_xl->name);
+
+            /* TODO: what is this ? */
+            client->bound_xl->itable = inode_table_new(conf->inode_lru_limit,
+                                                       client->bound_xl);
+        }
+    }
+    UNLOCK(&conf->itable_lock);
+
+    ret = dict_set_str(reply, "process-uuid", this->ctx->process_uuid);
+    if (ret)
+        gf_msg_debug(this->name, 0, "failed to set 'process-uuid'");
+
+    /* Insert a dummy key value pair to avoid failure at client side for
+     * clnt-lk-version with older clients.
+     */
+    ret = dict_set_uint32(reply, "clnt-lk-version", 0);
+    if (ret) {
+        gf_msg(this->name, GF_LOG_WARNING, 0, PS_MSG_CLIENT_LK_VERSION_ERROR,
+               "failed to set "
+               "'clnt-lk-version'");
+    }
+
+    ret = dict_set_uint64(reply, "transport-ptr", ((uint64_t)(long)req->trans));
+    if (ret)
+        gf_msg_debug(this->name, 0, "failed to set 'transport-ptr'");
 
 fail:
-        rsp.dict.dict_len = dict_serialized_length (reply);
-        if (rsp.dict.dict_len > UINT_MAX) {
-                gf_log ("server-handshake", GF_LOG_DEBUG,
-                        "failed to get serialized length of reply dict");
-                op_ret   = -1;
-                op_errno = EINVAL;
-                rsp.dict.dict_len = 0;
-        }
-
-        if (rsp.dict.dict_len) {
-                rsp.dict.dict_val = GF_CALLOC (1, rsp.dict.dict_len,
-                                               gf_server_mt_rsp_buf_t);
-                if (rsp.dict.dict_val) {
-                        ret = dict_serialize (reply, rsp.dict.dict_val);
-                        if (ret < 0) {
-                                gf_log ("server-handshake", GF_LOG_DEBUG,
-                                        "failed to serialize reply dict");
-                                op_ret = -1;
-                                op_errno = -ret;
-                        }
-                }
+    /* It is important to validate the lookup on '/' as part of handshake,
+       because if lookup itself can't succeed, we should communicate this
+       to client. Very important in case of subdirectory mounts, where if
+       client is trying to mount a non-existing directory */
+    if (op_ret >= 0 && client->bound_xl->itable) {
+        if (client->bound_xl->cleanup_starting) {
+            op_ret = -1;
+            op_errno = EAGAIN;
+            ret = dict_set_str(reply, "ERROR",
+                               "cleanup flag is set for xlator "
+                               "before call first_lookup Try again later");
+            /* quisce coverity about UNUSED_VALUE ret */
+            (void)(ret);
+        } else {
+            op_ret = server_first_lookup(this, client, reply);
+            if (op_ret == -1)
+                op_errno = ENOENT;
         }
-        rsp.op_ret   = op_ret;
-        rsp.op_errno = gf_errno_to_error (op_errno);
+    }
 
-        /* if bound_xl is NULL or something fails, then put the connection
-         * back. Otherwise the connection would have been added to the
-         * list of connections the server is maintaining and might segfault
-         * during statedump when bound_xl of the connection is accessed.
-         */
-        if (op_ret && !xl) {
-                /* We would have set the xl_private of the transport to the
-                 * @conn. But if we have put the connection i.e shutting down
-                 * the connection, then we should set xl_private to NULL as it
-                 * would be pointing to a freed memory and would segfault when
-                 * accessed upon getting DISCONNECT.
-                 */
-                gf_client_put (client, NULL);
-                req->trans->xl_private = NULL;
-        }
-        server_submit_reply (NULL, req, &rsp, NULL, 0, NULL,
-                             (xdrproc_t)xdr_gf_setvolume_rsp);
+    rsp = GF_CALLOC(1, sizeof(gf_setvolume_rsp), gf_server_mt_setvolume_rsp_t);
+    GF_ASSERT(rsp);
 
+    rsp->op_ret = 0;
 
-        free (args.dict.dict_val);
+    ret = dict_allocate_and_serialize(reply, (char **)&rsp->dict.dict_val,
+                                      &rsp->dict.dict_len);
+    if (ret != 0) {
+        ret = -1;
+        gf_msg_debug("server-handshake", 0, "failed to serialize reply dict");
+        op_ret = -1;
+        op_errno = -ret;
+    }
+
+    rsp->op_ret = op_ret;
+    rsp->op_errno = gf_errno_to_error(op_errno);
+
+    /* if bound_xl is NULL or something fails, then put the connection
+     * back. Otherwise the connection would have been added to the
+     * list of connections the server is maintaining and might segfault
+     * during statedump when bound_xl of the connection is accessed.
+     */
+    if (op_ret && !xl && (client != NULL)) {
+        /* We would have set the xl_private of the transport to the
+         * @conn. But if we have put the connection i.e shutting down
+         * the connection, then we should set xl_private to NULL as it
+         * would be pointing to a freed memory and would segfault when
+         * accessed upon getting DISCONNECT.
+         */
+        gf_client_put(client, NULL);
+        req->trans->xl_private = NULL;
+    }
 
-        GF_FREE (rsp.dict.dict_val);
+    /* Send the response properly */
+    server_first_lookup_done(req, rsp);
 
-        dict_unref (params);
-        dict_unref (reply);
-        dict_unref (config_params);
+    free(args.dict.dict_val);
 
-        GF_FREE (buf);
+    dict_unref(params);
+    dict_unref(reply);
+    if (config_params) {
+        /*
+         * This might be null if we couldn't even find the translator
+         * (brick) to copy it from.
+         */
+        dict_unref(config_params);
+    }
 
-        return 0;
+    return 0;
 }
 
-
 int
-server_ping (rpcsvc_request_t *req)
+server_ping(rpcsvc_request_t *req)
 {
-        gf_common_rsp rsp = {0,};
+    gf_common_rsp rsp = {
+        0,
+    };
 
-        /* Accepted */
-        rsp.op_ret = 0;
+    /* Accepted */
+    rsp.op_ret = 0;
 
-        server_submit_reply (NULL, req, &rsp, NULL, 0, NULL,
-                             (xdrproc_t)xdr_gf_common_rsp);
+    server_submit_reply(NULL, req, &rsp, NULL, 0, NULL,
+                        (xdrproc_t)xdr_gf_common_rsp);
 
-        return 0;
+    return 0;
 }
 
 int
-server_set_lk_version (rpcsvc_request_t *req)
+server_set_lk_version(rpcsvc_request_t *req)
 {
-        int                 op_ret   = -1;
-        int                 op_errno = EINVAL;
-        gf_set_lk_ver_req   args     = {0,};
-        gf_set_lk_ver_rsp   rsp      = {0,};
-        client_t           *client   = NULL;
-        server_ctx_t       *serv_ctx = NULL;
-        xlator_t           *this     = NULL;
-
-        this = req->svc->mydata;
-        //TODO: Decide on an appropriate errno for the error-path
-        //below
-        if (!this)
-                goto fail;
-
-        op_ret = xdr_to_generic (req->msg[0], &args,
-                              (xdrproc_t)xdr_gf_set_lk_ver_req);
-        if (op_ret < 0) {
-                //failed to decode msg;
-                req->rpc_err = GARBAGE_ARGS;
-                goto fail;
-        }
-
-        client = gf_client_get (this, &req->cred, args.uid);
-        serv_ctx = server_ctx_get (client, client->this);
-        if (serv_ctx == NULL) {
-                gf_log (this->name, GF_LOG_INFO, "server_ctx_get() failed");
-                goto fail;
-        }
-
-        serv_ctx->lk_version = args.lk_ver;
-        rsp.lk_ver   = args.lk_ver;
-
-        op_ret = 0;
+    int ret = -1;
+    gf_set_lk_ver_req args = {
+        0,
+    };
+    gf_set_lk_ver_rsp rsp = {
+        0,
+    };
+
+    ret = xdr_to_generic(req->msg[0], &args, (xdrproc_t)xdr_gf_set_lk_ver_req);
+    if (ret < 0) {
+        /* failed to decode msg */
+        req->rpc_err = GARBAGE_ARGS;
+        goto fail;
+    }
+
+    rsp.lk_ver = args.lk_ver;
 fail:
-        if (client)
-                gf_client_put (client, NULL);
+    server_submit_reply(NULL, req, &rsp, NULL, 0, NULL,
+                        (xdrproc_t)xdr_gf_set_lk_ver_rsp);
 
-        rsp.op_ret   = op_ret;
-        rsp.op_errno = op_errno;
-        server_submit_reply (NULL, req, &rsp, NULL, 0, NULL,
-                             (xdrproc_t)xdr_gf_set_lk_ver_rsp);
+    free(args.uid);
 
-        free (args.uid);
-
-        return 0;
+    return 0;
 }
 
-rpcsvc_actor_t gluster_handshake_actors[GF_HNDSK_MAXVALUE] = {
-        [GF_HNDSK_NULL]       = {"NULL",       GF_HNDSK_NULL,       server_null,           NULL, 0, DRC_NA},
-        [GF_HNDSK_SETVOLUME]  = {"SETVOLUME",  GF_HNDSK_SETVOLUME,  server_setvolume,      NULL, 0, DRC_NA},
-        [GF_HNDSK_GETSPEC]    = {"GETSPEC",    GF_HNDSK_GETSPEC,    server_getspec,        NULL, 0, DRC_NA},
-        [GF_HNDSK_PING]       = {"PING",       GF_HNDSK_PING,       server_ping,           NULL, 0, DRC_NA},
-        [GF_HNDSK_SET_LK_VER] = {"SET_LK_VER", GF_HNDSK_SET_LK_VER, server_set_lk_version, NULL, 0, DRC_NA},
+static rpcsvc_actor_t gluster_handshake_actors[GF_HNDSK_MAXVALUE] = {
+    [GF_HNDSK_NULL] = {"NULL", server_null, NULL, GF_HNDSK_NULL, DRC_NA, 0},
+    [GF_HNDSK_SETVOLUME] = {"SETVOLUME", server_setvolume, NULL,
+                            GF_HNDSK_SETVOLUME, DRC_NA, 0},
+    [GF_HNDSK_GETSPEC] = {"GETSPEC", server_getspec, NULL, GF_HNDSK_GETSPEC,
+                          DRC_NA, 0},
+    [GF_HNDSK_PING] = {"PING", server_ping, NULL, GF_HNDSK_PING, DRC_NA, 0},
+    [GF_HNDSK_SET_LK_VER] = {"SET_LK_VER", server_set_lk_version, NULL,
+                             GF_HNDSK_SET_LK_VER, DRC_NA, 0},
 };
 
-
 struct rpcsvc_program gluster_handshake_prog = {
-        .progname  = "GlusterFS Handshake",
-        .prognum   = GLUSTER_HNDSK_PROGRAM,
-        .progver   = GLUSTER_HNDSK_VERSION,
-        .actors    = gluster_handshake_actors,
-        .numactors = GF_HNDSK_MAXVALUE,
+    .progname = "GlusterFS Handshake",
+    .prognum = GLUSTER_HNDSK_PROGRAM,
+    .progver = GLUSTER_HNDSK_VERSION,
+    .actors = gluster_handshake_actors,
+    .numactors = GF_HNDSK_MAXVALUE,
 };
diff --git a/xlators/protocol/server/src/server-helpers.c b/xlators/protocol/server/src/server-helpers.c
index 545be6e9580..6e644912a46 100644
--- a/xlators/protocol/server/src/server-helpers.c
+++ b/xlators/protocol/server/src/server-helpers.c
@@ -8,1202 +8,1500 @@
   cases as published by the Free Software Foundation.
 */
 
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
-
 #include "server.h"
 #include "server-helpers.h"
-#include "gidcache.h"
+#include <glusterfs/gidcache.h>
+#include "server-messages.h"
+#include <glusterfs/syscall.h>
+#include <glusterfs/defaults.h>
+#include <glusterfs/default-args.h>
+#include "server-common.h"
 
 #include <fnmatch.h>
 #include <pwd.h>
-#include <grp.h>
 
 /* based on nfs_fix_aux_groups() */
 int
-gid_resolve (server_conf_t *conf, call_stack_t *root)
+gid_resolve(server_conf_t *conf, call_stack_t *root)
 {
-        int               ret = 0;
-        struct passwd     mypw;
-        char              mystrs[1024];
-        struct passwd    *result;
-        gid_t             mygroups[GF_MAX_AUX_GROUPS];
-        gid_list_t        gl;
-        const gid_list_t *agl;
-        int               ngroups, i;
-
-        agl = gid_cache_lookup (&conf->gid_cache, root->uid, 0, 0);
-        if (agl) {
-                root->ngrps = agl->gl_count;
-                goto fill_groups;
+    int ret = 0;
+    struct passwd mypw;
+    char mystrs[1024];
+    struct passwd *result;
+    gid_t *mygroups = NULL;
+    gid_list_t gl;
+    int ngroups;
+    const gid_list_t *agl;
+
+    agl = gid_cache_lookup(&conf->gid_cache, root->uid, 0, 0);
+    if (agl) {
+        root->ngrps = agl->gl_count;
+
+        if (root->ngrps > 0) {
+            ret = call_stack_alloc_groups(root, agl->gl_count);
+            if (ret == 0) {
+                memcpy(root->groups, agl->gl_list,
+                       sizeof(gid_t) * agl->gl_count);
+            }
         }
 
-        ret = getpwuid_r (root->uid, &mypw, mystrs, sizeof(mystrs), &result);
-        if (ret != 0) {
-                gf_log("gid-cache", GF_LOG_ERROR, "getpwuid_r(%u) failed",
-                       root->uid);
-                return -1;
-        }
+        gid_cache_release(&conf->gid_cache, agl);
 
-        if (!result) {
-                gf_log ("gid-cache", GF_LOG_ERROR, "getpwuid_r(%u) found "
-                        "nothing", root->uid);
-                return -1;
-        }
+        return ret;
+    }
 
-        gf_log ("gid-cache", GF_LOG_TRACE, "mapped %u => %s", root->uid,
-                result->pw_name);
+    ret = getpwuid_r(root->uid, &mypw, mystrs, sizeof(mystrs), &result);
+    if (ret != 0) {
+        gf_smsg("gid-cache", GF_LOG_ERROR, errno, PS_MSG_GET_UID_FAILED,
+                "uid=%u", root->uid, NULL);
+        return -1;
+    }
 
-        ngroups = GF_MAX_AUX_GROUPS;
-        ret = getgrouplist (result->pw_name, root->gid, mygroups, &ngroups);
-        if (ret == -1) {
-                gf_log ("gid-cache", GF_LOG_ERROR, "could not map %s to group "
-                        "list (%d gids)", result->pw_name, root->ngrps);
-                return -1;
-        }
-        root->ngrps = (uint16_t) ngroups;
-
-fill_groups:
-        if (agl) {
-                /* the gl is not complete, we only use gl.gl_list later on */
-                gl.gl_list = agl->gl_list;
-        } else {
-                /* setup a full gid_list_t to add it to the gid_cache */
-                gl.gl_id = root->uid;
-                gl.gl_uid = root->uid;
-                gl.gl_gid = root->gid;
-                gl.gl_count = root->ngrps;
-
-                gl.gl_list = GF_MALLOC (root->ngrps * sizeof(gid_t),
-                                        gf_common_mt_groups_t);
-                if (gl.gl_list)
-                        memcpy (gl.gl_list, mygroups,
-                                sizeof(gid_t) * root->ngrps);
-                else
-                        return -1;
-        }
+    if (!result) {
+        gf_smsg("gid-cache", GF_LOG_ERROR, 0, PS_MSG_UID_NOT_FOUND, "uid=%u",
+                root->uid, NULL);
+        return -1;
+    }
 
-        if (root->ngrps == 0) {
-                ret = 0;
-                goto out;
-        }
+    gf_msg_trace("gid-cache", 0, "mapped %u => %s", root->uid, result->pw_name);
 
-        if (call_stack_alloc_groups (root, root->ngrps) != 0) {
-                ret = -1;
-                goto out;
-        }
+    ngroups = gf_getgrouplist(result->pw_name, root->gid, &mygroups);
+    if (ngroups == -1) {
+        gf_smsg("gid-cache", GF_LOG_ERROR, 0, PS_MSG_MAPPING_ERROR,
+                "pw_name=%s", result->pw_name, "root->ngtps=%d", root->ngrps,
+                NULL);
+        return -1;
+    }
+    root->ngrps = (uint16_t)ngroups;
+
+    /* setup a full gid_list_t to add it to the gid_cache */
+    gl.gl_id = root->uid;
+    gl.gl_uid = root->uid;
+    gl.gl_gid = root->gid;
+    gl.gl_count = root->ngrps;
+
+    gl.gl_list = GF_MALLOC(root->ngrps * sizeof(gid_t), gf_common_mt_groups_t);
+    if (gl.gl_list)
+        memcpy(gl.gl_list, mygroups, sizeof(gid_t) * root->ngrps);
+    else {
+        GF_FREE(mygroups);
+        return -1;
+    }
 
-        /* finally fill the groups from the */
-        for (i = 0; i < root->ngrps; ++i)
-                root->groups[i] = gl.gl_list[i];
+    if (root->ngrps > 0) {
+        call_stack_set_groups(root, root->ngrps, &mygroups);
+    }
 
-out:
-        if (agl) {
-                gid_cache_release (&conf->gid_cache, agl);
-        } else {
-                if (gid_cache_add (&conf->gid_cache, &gl) != 1)
-                        GF_FREE (gl.gl_list);
-        }
+    if (gid_cache_add(&conf->gid_cache, &gl) != 1)
+        GF_FREE(gl.gl_list);
 
-        return ret;
+    return ret;
 }
 
 int
-server_resolve_groups (call_frame_t *frame, rpcsvc_request_t *req)
+server_resolve_groups(call_frame_t *frame, rpcsvc_request_t *req)
 {
-        xlator_t      *this = NULL;
-        server_conf_t *conf = NULL;
+    xlator_t *this = NULL;
+    server_conf_t *conf = NULL;
 
-        GF_VALIDATE_OR_GOTO ("server", frame, out);
-        GF_VALIDATE_OR_GOTO ("server", req, out);
+    GF_VALIDATE_OR_GOTO("server", frame, out);
+    GF_VALIDATE_OR_GOTO("server", req, out);
 
-        this = req->trans->xl;
-        conf = this->private;
+    this = req->trans->xl;
+    conf = this->private;
 
-        return gid_resolve (conf, frame->root);
+    return gid_resolve(conf, frame->root);
 out:
-        return -1;
+    return -1;
 }
 
 int
-server_decode_groups (call_frame_t *frame, rpcsvc_request_t *req)
+server_decode_groups(call_frame_t *frame, rpcsvc_request_t *req)
 {
-        int     i = 0;
+    int i = 0;
 
-        GF_VALIDATE_OR_GOTO ("server", frame, out);
-        GF_VALIDATE_OR_GOTO ("server", req, out);
+    GF_VALIDATE_OR_GOTO("server", frame, out);
+    GF_VALIDATE_OR_GOTO("server", req, out);
 
-        if (call_stack_alloc_groups (frame->root, req->auxgidcount) != 0)
-                return -1;
+    if (call_stack_alloc_groups(frame->root, req->auxgidcount) != 0)
+        return -1;
 
-        frame->root->ngrps = req->auxgidcount;
-        if (frame->root->ngrps == 0)
-                return 0;
+    frame->root->ngrps = req->auxgidcount;
+    if (frame->root->ngrps == 0)
+        return 0;
 
-	/* ngrps cannot be bigger than USHRT_MAX(65535) */
-        if (frame->root->ngrps > GF_MAX_AUX_GROUPS)
-                return -1;
+    /* ngrps cannot be bigger than USHRT_MAX(65535) */
+    if (frame->root->ngrps > GF_MAX_AUX_GROUPS)
+        return -1;
 
-        for (; i < frame->root->ngrps; ++i)
-                frame->root->groups[i] = req->auxgids[i];
+    for (; i < frame->root->ngrps; ++i)
+        frame->root->groups[i] = req->auxgids[i];
 out:
-        return 0;
+    return 0;
 }
 
-
 void
-server_loc_wipe (loc_t *loc)
+server_loc_wipe(loc_t *loc)
 {
-        if (loc->parent) {
-                inode_unref (loc->parent);
-                loc->parent = NULL;
-        }
+    if (loc->parent) {
+        inode_unref(loc->parent);
+        loc->parent = NULL;
+    }
 
-        if (loc->inode) {
-                inode_unref (loc->inode);
-                loc->inode = NULL;
-        }
+    if (loc->inode) {
+        inode_unref(loc->inode);
+        loc->inode = NULL;
+    }
 
-        GF_FREE ((void *)loc->path);
+    GF_FREE((void *)loc->path);
 }
 
-
 void
-server_resolve_wipe (server_resolve_t *resolve)
+server_resolve_wipe(server_resolve_t *resolve)
 {
-        GF_FREE ((void *)resolve->path);
+    GF_FREE((void *)resolve->path);
 
-        GF_FREE ((void *)resolve->bname);
+    GF_FREE((void *)resolve->bname);
 
-        loc_wipe (&resolve->resolve_loc);
+    loc_wipe(&resolve->resolve_loc);
 }
 
-
 void
-free_state (server_state_t *state)
+free_state(server_state_t *state)
 {
-        if (state->xprt) {
-                rpc_transport_unref (state->xprt);
-                state->xprt = NULL;
-        }
-        if (state->fd) {
-                fd_unref (state->fd);
-                state->fd = NULL;
-        }
-
-        if (state->params) {
-                dict_unref (state->params);
-                state->params = NULL;
-        }
-
-        if (state->iobref) {
-                iobref_unref (state->iobref);
-                state->iobref = NULL;
-        }
-
-        if (state->iobuf) {
-                iobuf_unref (state->iobuf);
-                state->iobuf = NULL;
-        }
-
-        if (state->dict) {
-                dict_unref (state->dict);
-                state->dict = NULL;
-        }
-
-        if (state->xdata) {
-                dict_unref (state->xdata);
-                state->xdata = NULL;
-        }
-
-        GF_FREE ((void *)state->volume);
-
-        GF_FREE ((void *)state->name);
-
-        server_loc_wipe (&state->loc);
-        server_loc_wipe (&state->loc2);
-
-        server_resolve_wipe (&state->resolve);
-        server_resolve_wipe (&state->resolve2);
-
-        GF_FREE (state);
+    if (state->fd) {
+        fd_unref(state->fd);
+        state->fd = NULL;
+    }
+
+    if (state->params) {
+        dict_unref(state->params);
+        state->params = NULL;
+    }
+
+    if (state->iobref) {
+        iobref_unref(state->iobref);
+        state->iobref = NULL;
+    }
+
+    if (state->iobuf) {
+        iobuf_unref(state->iobuf);
+        state->iobuf = NULL;
+    }
+
+    if (state->dict) {
+        dict_unref(state->dict);
+        state->dict = NULL;
+    }
+
+    if (state->xdata) {
+        dict_unref(state->xdata);
+        state->xdata = NULL;
+    }
+
+    GF_FREE((void *)state->volume);
+
+    GF_FREE((void *)state->name);
+
+    server_loc_wipe(&state->loc);
+    server_loc_wipe(&state->loc2);
+
+    server_resolve_wipe(&state->resolve);
+    server_resolve_wipe(&state->resolve2);
+
+    /* Call rpc_trnasport_unref to avoid crashes at last after free
+       all resources because of server_rpc_notify (for transport destroy)
+       call's xlator_mem_cleanup if all xprt are destroyed that internally
+       call's inode_table_destroy.
+    */
+    if (state->xprt) {
+        rpc_transport_unref(state->xprt);
+        state->xprt = NULL;
+    }
+
+    GF_FREE(state);
 }
 
-
 static int
-server_connection_cleanup_flush_cbk (call_frame_t *frame, void *cookie,
-                                     xlator_t *this, int32_t op_ret,
-                                     int32_t op_errno, dict_t *xdata)
+server_connection_cleanup_flush_cbk(call_frame_t *frame, void *cookie,
+                                    xlator_t *this, int32_t op_ret,
+                                    int32_t op_errno, dict_t *xdata)
 {
-        int32_t    ret    = -1;
-        fd_t      *fd     = NULL;
-        client_t  *client = NULL;
-
-        GF_VALIDATE_OR_GOTO ("server", this, out);
-        GF_VALIDATE_OR_GOTO ("server", frame, out);
-
-        fd = frame->local;
-        client = frame->root->client;
-
-        fd_unref (fd);
-        frame->local = NULL;
+    int32_t ret = -1;
+    fd_t *fd = NULL;
+    client_t *client = NULL;
+    uint64_t fd_cnt = 0;
+    xlator_t *victim = NULL;
+    server_conf_t *conf = NULL;
+    xlator_t *serv_xl = NULL;
+    rpc_transport_t *xprt = NULL;
+    rpc_transport_t *xp_next = NULL;
+    int32_t detach = (long)cookie;
+    gf_boolean_t xprt_found = _gf_false;
+
+    GF_VALIDATE_OR_GOTO("server", this, out);
+    GF_VALIDATE_OR_GOTO("server", frame, out);
+
+    fd = frame->local;
+    client = frame->root->client;
+    serv_xl = frame->this;
+    conf = serv_xl->private;
+
+    fd_unref(fd);
+    frame->local = NULL;
+
+    if (client)
+        victim = client->bound_xl;
+
+    if (victim) {
+        fd_cnt = GF_ATOMIC_DEC(client->fd_cnt);
+        if (!fd_cnt && conf && detach) {
+            pthread_mutex_lock(&conf->mutex);
+            {
+                list_for_each_entry_safe(xprt, xp_next, &conf->xprt_list, list)
+                {
+                    if (!xprt->xl_private)
+                        continue;
+                    if (xprt->xl_private == client) {
+                        xprt_found = _gf_true;
+                        break;
+                    }
+                }
+            }
+            pthread_mutex_unlock(&conf->mutex);
+            if (xprt_found) {
+                rpc_transport_unref(xprt);
+            }
+        }
+    }
 
-        gf_client_unref (client);
-        STACK_DESTROY (frame->root);
+    gf_client_unref(client);
+    STACK_DESTROY(frame->root);
 
-        ret = 0;
+    ret = 0;
 out:
-        return ret;
+    return ret;
 }
 
-
 static int
-do_fd_cleanup (xlator_t *this, client_t* client, fdentry_t *fdentries, int fd_count)
+do_fd_cleanup(xlator_t *this, client_t *client, fdentry_t *fdentries,
+              int fd_count, int32_t detach)
 {
-        fd_t               *fd = NULL;
-        int                 i = 0, ret = -1;
-        call_frame_t       *tmp_frame = NULL;
-        xlator_t           *bound_xl = NULL;
-        char               *path     = NULL;
+    fd_t *fd = NULL;
+    int i = 0, ret = -1;
+    call_frame_t *tmp_frame = NULL;
+    xlator_t *bound_xl = NULL;
+    char *path = NULL;
 
-        GF_VALIDATE_OR_GOTO ("server", this, out);
-        GF_VALIDATE_OR_GOTO ("server", fdentries, out);
+    GF_VALIDATE_OR_GOTO("server", this, out);
+    GF_VALIDATE_OR_GOTO("server", fdentries, out);
 
-        bound_xl = client->bound_xl;
-        for (i = 0;i < fd_count; i++) {
-                fd = fdentries[i].fd;
+    bound_xl = client->bound_xl;
 
-                if (fd != NULL) {
-                        tmp_frame = create_frame (this, this->ctx->pool);
-                        if (tmp_frame == NULL) {
-                                goto out;
-                        }
-
-                        GF_ASSERT (fd->inode);
-
-                        ret = inode_path (fd->inode, NULL, &path);
-
-                        if (ret > 0) {
-                                gf_log (this->name, GF_LOG_INFO,
-                                        "fd cleanup on %s", path);
-                                GF_FREE (path);
-                        }  else {
-
-                                gf_log (this->name, GF_LOG_INFO,
-                                        "fd cleanup on inode with gfid %s",
-                                        uuid_utoa (fd->inode->gfid));
-                        }
-
-                        tmp_frame->local = fd;
-                        tmp_frame->root->pid = 0;
-                        gf_client_ref (client);
-                        tmp_frame->root->client = client;
-                        memset (&tmp_frame->root->lk_owner, 0,
-                                sizeof (gf_lkowner_t));
-
-                        STACK_WIND (tmp_frame,
-                                    server_connection_cleanup_flush_cbk,
-                                    bound_xl, bound_xl->fops->flush, fd, NULL);
-                }
+    for (i = 0; i < fd_count; i++) {
+        fd = fdentries[i].fd;
+
+        if (fd != NULL) {
+            tmp_frame = create_frame(this, this->ctx->pool);
+            if (tmp_frame == NULL) {
+                goto out;
+            }
+
+            tmp_frame->root->type = GF_OP_TYPE_FOP;
+            GF_ASSERT(fd->inode);
+
+            ret = inode_path(fd->inode, NULL, &path);
+
+            if (ret > 0) {
+                gf_smsg(this->name, GF_LOG_INFO, 0, PS_MSG_FD_CLEANUP,
+                        "path=%s", path, NULL);
+                GF_FREE(path);
+            } else {
+                gf_smsg(this->name, GF_LOG_INFO, 0, PS_MSG_FD_CLEANUP,
+                        "inode-gfid=%s", uuid_utoa(fd->inode->gfid), NULL);
+            }
+
+            tmp_frame->local = fd;
+            tmp_frame->root->pid = 0;
+            gf_client_ref(client);
+            tmp_frame->root->client = client;
+            memset(&tmp_frame->root->lk_owner, 0, sizeof(gf_lkowner_t));
+
+            STACK_WIND_COOKIE(tmp_frame, server_connection_cleanup_flush_cbk,
+                              (void *)(long)detach, bound_xl,
+                              bound_xl->fops->flush, fd, NULL);
         }
+    }
 
-        GF_FREE (fdentries);
-        ret = 0;
+    GF_FREE(fdentries);
+    ret = 0;
 
 out:
-        return ret;
+    return ret;
 }
 
-
 int
-server_connection_cleanup (xlator_t *this, client_t *client,
-                           int32_t flags)
+server_connection_cleanup(xlator_t *this, client_t *client, int32_t flags,
+                          gf_boolean_t *fd_exist)
 {
-        server_ctx_t        *serv_ctx  = NULL;
-        fdentry_t           *fdentries = NULL;
-        uint32_t             fd_count  = 0;
-        int                  cd_ret    = 0;
-        int                  ret       = 0;
-
-        GF_VALIDATE_OR_GOTO (this->name, this, out);
-        GF_VALIDATE_OR_GOTO (this->name, client, out);
-        GF_VALIDATE_OR_GOTO (this->name, flags, out);
-
-        serv_ctx = server_ctx_get (client, client->this);
-
-        if (serv_ctx == NULL) {
-                gf_log (this->name, GF_LOG_INFO, "server_ctx_get() failed");
-                goto out;
-        }
-
-        LOCK (&serv_ctx->fdtable_lock);
-        {
-                if (serv_ctx->fdtable && (flags & POSIX_LOCKS))
-                        fdentries = gf_fd_fdtable_get_all_fds (serv_ctx->fdtable,
-                                                               &fd_count);
+    server_ctx_t *serv_ctx = NULL;
+    fdentry_t *fdentries = NULL;
+    uint32_t fd_count = 0;
+    int cd_ret = 0;
+    int ret = 0;
+    xlator_t *bound_xl = NULL;
+    int i = 0;
+    fd_t *fd = NULL;
+    uint64_t fd_cnt = 0;
+    int32_t detach = 0;
+
+    GF_VALIDATE_OR_GOTO("server", this, out);
+    GF_VALIDATE_OR_GOTO(this->name, client, out);
+    GF_VALIDATE_OR_GOTO(this->name, flags, out);
+
+    serv_ctx = server_ctx_get(client, client->this);
+
+    if (serv_ctx == NULL) {
+        gf_smsg(this->name, GF_LOG_INFO, 0, PS_MSG_SERVER_CTX_GET_FAILED, NULL);
+        goto out;
+    }
+
+    LOCK(&serv_ctx->fdtable_lock);
+    {
+        if (serv_ctx->fdtable && (flags & POSIX_LOCKS))
+            fdentries = gf_fd_fdtable_get_all_fds(serv_ctx->fdtable, &fd_count);
+    }
+    UNLOCK(&serv_ctx->fdtable_lock);
+
+    if (client->bound_xl == NULL)
+        goto out;
+
+    if (flags & INTERNAL_LOCKS) {
+        cd_ret = gf_client_disconnect(client);
+    }
+
+    if (fdentries != NULL) {
+        /* Loop to configure fd_count on victim brick */
+        bound_xl = client->bound_xl;
+        if (bound_xl) {
+            for (i = 0; i < fd_count; i++) {
+                fd = fdentries[i].fd;
+                if (!fd)
+                    continue;
+                fd_cnt++;
+            }
+            if (fd_cnt) {
+                if (fd_exist)
+                    (*fd_exist) = _gf_true;
+                GF_ATOMIC_ADD(client->fd_cnt, fd_cnt);
+            }
         }
-        UNLOCK (&serv_ctx->fdtable_lock);
 
-        if (client->bound_xl == NULL)
-                goto out;
+        /* If fd_exist is not NULL it means function is invoke
+           by server_rpc_notify at the time of getting DISCONNECT
+           notification
+        */
+        if (fd_exist)
+            detach = 1;
 
-        if (flags & INTERNAL_LOCKS) {
-                cd_ret = gf_client_disconnect (client);
-        }
-
-        if (fdentries != NULL)
-                ret = do_fd_cleanup (this, client, fdentries, fd_count);
-        else
-                gf_log (this->name, GF_LOG_INFO, "no fdentries to clean");
+        gf_msg_debug(this->name, 0,
+                     "Performing cleanup on %d "
+                     "fdentries",
+                     fd_count);
+        ret = do_fd_cleanup(this, client, fdentries, fd_count, detach);
+    } else
+        gf_smsg(this->name, GF_LOG_INFO, 0, PS_MSG_FDENTRY_NULL, NULL);
 
-        if (cd_ret || ret)
-                ret = -1;
+    if (cd_ret || ret)
+        ret = -1;
 
 out:
-        return ret;
+    return ret;
 }
 
-
 static call_frame_t *
-server_alloc_frame (rpcsvc_request_t *req)
+server_alloc_frame(rpcsvc_request_t *req)
 {
-        call_frame_t    *frame  = NULL;
-        server_state_t  *state  = NULL;
-        client_t        *client = NULL;
+    call_frame_t *frame = NULL;
+    server_state_t *state = NULL;
+    client_t *client = NULL;
 
-        GF_VALIDATE_OR_GOTO ("server", req, out);
-        GF_VALIDATE_OR_GOTO ("server", req->trans, out);
-        GF_VALIDATE_OR_GOTO ("server", req->svc, out);
-        GF_VALIDATE_OR_GOTO ("server", req->svc->ctx, out);
+    GF_VALIDATE_OR_GOTO("server", req, out);
+    GF_VALIDATE_OR_GOTO("server", req->trans, out);
+    GF_VALIDATE_OR_GOTO("server", req->svc, out);
+    GF_VALIDATE_OR_GOTO("server", req->svc->ctx, out);
 
-        client = req->trans->xl_private;
-        GF_VALIDATE_OR_GOTO ("server", client, out);
+    client = req->trans->xl_private;
+    GF_VALIDATE_OR_GOTO("server", client, out);
 
-        frame = create_frame (client->this, req->svc->ctx->pool);
-        if (!frame)
-                goto out;
+    frame = create_frame(client->this, req->svc->ctx->pool);
+    if (!frame)
+        goto out;
 
-        state = GF_CALLOC (1, sizeof (*state), gf_server_mt_state_t);
-        if (!state)
-                goto out;
+    frame->root->type = GF_OP_TYPE_FOP;
+    state = GF_CALLOC(1, sizeof(*state), gf_server_mt_state_t);
+    if (!state)
+        goto out;
 
-        if (client->bound_xl)
-                state->itable = client->bound_xl->itable;
+    if (client->bound_xl)
+        state->itable = client->bound_xl->itable;
 
-        state->xprt  = rpc_transport_ref (req->trans);
-        state->resolve.fd_no = -1;
-        state->resolve2.fd_no = -1;
+    state->xprt = rpc_transport_ref(req->trans);
+    state->resolve.fd_no = -1;
+    state->resolve2.fd_no = -1;
 
-        frame->root->client  = client;
-        frame->root->state = state;        /* which socket */
-        frame->root->unique = 0;           /* which call */
+    frame->root->client = client;
+    frame->root->state = state; /* which socket */
 
-        frame->this = client->this;
+    frame->this = client->this;
 out:
-        return frame;
+    return frame;
 }
 
-
 call_frame_t *
-get_frame_from_request (rpcsvc_request_t *req)
+get_frame_from_request(rpcsvc_request_t *req)
 {
-        call_frame_t  *frame = NULL;
-        client_t      *client = NULL;
-        client_t      *tmp_client = NULL;
-        xlator_t  *this = NULL;
-        server_conf_t *priv = NULL;
-        clienttable_t *clienttable = NULL;
-        unsigned int   i           = 0;
-
-        GF_VALIDATE_OR_GOTO ("server", req, out);
-
-        client = req->trans->xl_private;
-
-        frame = server_alloc_frame (req);
-        if (!frame)
-                goto out;
-
-        frame->root->op       = req->procnum;
-
-        frame->root->unique   = req->xid;
-
-        client = req->trans->xl_private;
-        this = req->trans->xl;
-        priv = this->private;
-        clienttable = this->ctx->clienttable;
-
-        for (i = 0; i < clienttable->max_clients; i++) {
-                tmp_client = clienttable->cliententries[i].client;
-                if (client == tmp_client) {
-                        /* for non trusted clients username and password
-                           would not have been set. So for non trusted clients
-                           (i.e clients not from the same machine as the brick,
-                           and clients from outside the storage pool)
-                           do the root-squashing.
-                           TODO: If any client within the storage pool (i.e
-                           mounting within a machine from the pool but using
-                           other machine's ip/hostname from the same pool)
-                           is present treat it as a trusted client
-                        */
-                        if (!client->auth.username && req->pid != NFS_PID)
-                                RPC_AUTH_ROOT_SQUASH (req);
-
-                        /* Problem: If we just check whether the client is
-                           trusted client and do not do root squashing for
-                           them, then for smb clients and UFO clients root
-                           squashing will never happen as they use the fuse
-                           mounts done within the trusted pool (i.e they are
-                           trusted clients).
-                           Solution: To fix it, do root squashing for trusted
-                           clients also. If one wants to have a client within
-                           the storage pool for which root-squashing does not
-                           happen, then the client has to be mounted with
-                           --no-root-squash option. But for defrag client and
-                           gsyncd client do not do root-squashing.
-                        */
-                        if (client->auth.username &&
-                            req->pid != GF_CLIENT_PID_NO_ROOT_SQUASH &&
-                            req->pid != GF_CLIENT_PID_GSYNCD &&
-                            req->pid != GF_CLIENT_PID_DEFRAG &&
-                            req->pid != GF_CLIENT_PID_AFR_SELF_HEALD &&
-                            req->pid != GF_CLIENT_PID_QUOTA_MOUNT)
-                                RPC_AUTH_ROOT_SQUASH (req);
-
-                        /* For nfs clients the server processes will be running
-                           within the trusted storage pool machines. So if we
-                           do not do root-squashing for nfs servers, thinking
-                           that its a trusted client, then root-squashing wont
-                           work for nfs clients.
-                        */
-                        if (req->pid == NFS_PID)
-                                RPC_AUTH_ROOT_SQUASH (req);
-                }
+    call_frame_t *frame = NULL;
+    client_t *client = NULL;
+    client_t *tmp_client = NULL;
+    xlator_t *this = NULL;
+    server_conf_t *priv = NULL;
+    clienttable_t *clienttable = NULL;
+    unsigned int i = 0;
+    rpc_transport_t *trans = NULL;
+    server_state_t *state = NULL;
+
+    GF_VALIDATE_OR_GOTO("server", req, out);
+
+    frame = server_alloc_frame(req);
+    if (!frame)
+        goto out;
+
+    frame->root->op = req->procnum;
+
+    client = req->trans->xl_private;
+    this = req->trans->xl;
+    priv = this->private;
+    clienttable = this->ctx->clienttable;
+
+    for (i = 0; i < clienttable->max_clients; i++) {
+        tmp_client = clienttable->cliententries[i].client;
+        if (client == tmp_client) {
+            /* for non trusted clients username and password
+               would not have been set. So for non trusted clients
+               (i.e clients not from the same machine as the brick,
+               and clients from outside the storage pool)
+               do the root-squashing and all-squashing.
+               TODO: If any client within the storage pool (i.e
+               mounting within a machine from the pool but using
+               other machine's ip/hostname from the same pool)
+               is present treat it as a trusted client
+            */
+            if (!client->auth.username && req->pid != NFS_PID) {
+                RPC_AUTH_ROOT_SQUASH(req);
+                RPC_AUTH_ALL_SQUASH(req);
+            }
+
+            /* Problem: If we just check whether the client is
+               trusted client and do not do root squashing and
+               all squashing for them, then for smb clients and
+               UFO clients root squashing and all squashing will
+               never happen as they use the fuse mounts done within
+               the trusted pool (i.e they are trusted clients).
+               Solution: To fix it, do root squashing and all squashing
+               for trusted clients also. If one wants to have a client
+               within the storage pool for which root-squashing does
+               not happen, then the client has to be mounted with
+               --no-root-squash option. But for defrag client and
+               gsyncd client do not do root-squashing and all-squashing.
+            */
+            if (client->auth.username &&
+                req->pid != GF_CLIENT_PID_NO_ROOT_SQUASH &&
+                req->pid != GF_CLIENT_PID_GSYNCD &&
+                req->pid != GF_CLIENT_PID_DEFRAG &&
+                req->pid != GF_CLIENT_PID_SELF_HEALD &&
+                req->pid != GF_CLIENT_PID_QUOTA_MOUNT) {
+                RPC_AUTH_ROOT_SQUASH(req);
+                RPC_AUTH_ALL_SQUASH(req);
+            }
+
+            /* For nfs clients the server processes will be running
+               within the trusted storage pool machines. So if we
+               do not do root-squashing and all-squashing for nfs
+               servers, thinking that its a trusted client, then
+               root-squashing and all-squashing won't work for nfs
+               clients.
+            */
+            if (req->pid == NFS_PID) {
+                RPC_AUTH_ROOT_SQUASH(req);
+                RPC_AUTH_ALL_SQUASH(req);
+            }
         }
-
-        frame->root->uid      = req->uid;
-        frame->root->gid      = req->gid;
-        frame->root->pid      = req->pid;
-        gf_client_ref (client);
-        frame->root->client   = client;
-        frame->root->lk_owner = req->lk_owner;
-
-        if (priv->server_manage_gids)
-            server_resolve_groups (frame, req);
-        else
-            server_decode_groups (frame, req);
-
-        frame->local = req;
+    }
+
+    /* Add a ref for this fop */
+    if (client)
+        gf_client_ref(client);
+
+    frame->root->uid = req->uid;
+    frame->root->gid = req->gid;
+    frame->root->pid = req->pid;
+    frame->root->client = client;
+    frame->root->lk_owner = req->lk_owner;
+
+    if (priv->server_manage_gids)
+        server_resolve_groups(frame, req);
+    else
+        server_decode_groups(frame, req);
+    trans = req->trans;
+    if (trans) {
+        memcpy(&frame->root->identifier, trans->peerinfo.identifier,
+               sizeof(trans->peerinfo.identifier));
+    }
+
+    /* more fields, for the clients which are 3.x series this will be 0 */
+    frame->root->flags = req->flags;
+    frame->root->ctime = req->ctime;
+
+    frame->local = req;
+
+    state = CALL_STATE(frame);
+    state->client = client;
 out:
-        return frame;
+    return frame;
 }
 
-
 int
-server_build_config (xlator_t *this, server_conf_t *conf)
+server_build_config(xlator_t *this, server_conf_t *conf)
 {
-        data_t     *data = NULL;
-        int         ret = -1;
-        struct stat buf = {0,};
-
-        GF_VALIDATE_OR_GOTO ("server", this, out);
-        GF_VALIDATE_OR_GOTO ("server", conf, out);
-
-        ret = dict_get_int32 (this->options, "inode-lru-limit",
-                              &conf->inode_lru_limit);
-        if (ret < 0) {
-                conf->inode_lru_limit = 16384;
+    data_t *data = NULL;
+    int ret = -1;
+    struct stat buf = {
+        0,
+    };
+
+    GF_VALIDATE_OR_GOTO("server", this, out);
+    GF_VALIDATE_OR_GOTO("server", conf, out);
+
+    ret = dict_get_int32(this->options, "inode-lru-limit",
+                         &conf->inode_lru_limit);
+    if (ret < 0) {
+        conf->inode_lru_limit = 16384;
+    }
+
+    conf->verify_volfile = 1;
+    data = dict_get(this->options, "verify-volfile-checksum");
+    if (data) {
+        ret = gf_string2boolean(data->data, &conf->verify_volfile);
+        if (ret != 0) {
+            gf_smsg(this->name, GF_LOG_WARNING, EINVAL, PS_MSG_WRONG_VALUE,
+                    NULL);
         }
+    }
 
-        conf->verify_volfile = 1;
-        data = dict_get (this->options, "verify-volfile-checksum");
-        if (data) {
-                ret = gf_string2boolean(data->data, &conf->verify_volfile);
-                if (ret != 0) {
-                        gf_log (this->name, GF_LOG_WARNING,
-                                "wrong value for 'verify-volfile-checksum', "
-                                "Neglecting option");
-                }
+    data = dict_get(this->options, "trace");
+    if (data) {
+        ret = gf_string2boolean(data->data, &conf->trace);
+        if (ret != 0) {
+            gf_smsg(this->name, GF_LOG_WARNING, EINVAL, PS_MSG_INVALID_ENTRY,
+                    NULL);
         }
-
-        data = dict_get (this->options, "trace");
-        if (data) {
-                ret = gf_string2boolean (data->data, &conf->trace);
-                if (ret != 0) {
-                        gf_log (this->name, GF_LOG_WARNING,
-                                "'trace' takes on only boolean values. "
-                                "Neglecting option");
-                }
+    }
+
+    /* TODO: build_rpc_config (); */
+    ret = dict_get_int32(this->options, "limits.transaction-size",
+                         &conf->rpc_conf.max_block_size);
+    if (ret < 0) {
+        gf_msg_trace(this->name, 0,
+                     "defaulting limits.transaction-"
+                     "size to %d",
+                     DEFAULT_BLOCK_SIZE);
+        conf->rpc_conf.max_block_size = DEFAULT_BLOCK_SIZE;
+    }
+
+    data = dict_get(this->options, "config-directory");
+    if (data) {
+        /* Check whether the specified directory exists,
+           or directory specified is non standard */
+        ret = sys_stat(data->data, &buf);
+        if ((ret != 0) || !S_ISDIR(buf.st_mode)) {
+            gf_smsg(this->name, GF_LOG_ERROR, 0, PS_MSG_DIR_NOT_FOUND,
+                    "data=%s", data->data, NULL);
+            ret = -1;
+            goto out;
         }
-
-        /* TODO: build_rpc_config (); */
-        ret = dict_get_int32 (this->options, "limits.transaction-size",
-                              &conf->rpc_conf.max_block_size);
-        if (ret < 0) {
-                gf_log (this->name, GF_LOG_TRACE,
-                        "defaulting limits.transaction-size to %d",
-                        DEFAULT_BLOCK_SIZE);
-                conf->rpc_conf.max_block_size = DEFAULT_BLOCK_SIZE;
+        /* Make sure that conf-dir doesn't contain ".." in path */
+        if ((gf_strstr(data->data, "/", "..")) == -1) {
+            ret = -1;
+            gf_smsg(this->name, GF_LOG_ERROR, 0, PS_MSG_CONF_DIR_INVALID,
+                    "data=%s", data->data, NULL);
+            goto out;
         }
 
-        data = dict_get (this->options, "config-directory");
-        if (data) {
-                /* Check whether the specified directory exists,
-                   or directory specified is non standard */
-                ret = stat (data->data, &buf);
-                if ((ret != 0) || !S_ISDIR (buf.st_mode)) {
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "Directory '%s' doesn't exist, exiting.",
-                                data->data);
-                        ret = -1;
-                        goto out;
-                }
-                /* Make sure that conf-dir doesn't contain ".." in path */
-                if ((gf_strstr (data->data, "/", "..")) == -1) {
-                        ret = -1;
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "%s: invalid conf_dir", data->data);
-                        goto out;
-                }
-
-                conf->conf_dir = gf_strdup (data->data);
-        }
-        ret = 0;
+        conf->conf_dir = gf_strdup(data->data);
+    }
+    ret = 0;
 out:
-        return ret;
+    return ret;
 }
 
-
 void
-print_caller (char *str, int size, call_frame_t *frame)
+print_caller(char *str, int size, call_frame_t *frame)
 {
-        server_state_t  *state = NULL;
+    server_state_t *state = NULL;
 
-        GF_VALIDATE_OR_GOTO ("server", str, out);
-        GF_VALIDATE_OR_GOTO ("server", frame, out);
+    GF_VALIDATE_OR_GOTO("server", str, out);
+    GF_VALIDATE_OR_GOTO("server", frame, out);
 
-        state = CALL_STATE (frame);
+    state = CALL_STATE(frame);
 
-        snprintf (str, size,
-                  " Callid=%"PRId64", Client=%s",
-                  frame->root->unique,
-                  state->xprt->peerinfo.identifier);
+    snprintf(str, size, " Callid=%" PRId64 ", Client=%s", frame->root->unique,
+             state->xprt->peerinfo.identifier);
 
 out:
-        return;
+    return;
 }
 
-
 void
-server_print_resolve (char *str, int size, server_resolve_t *resolve)
+server_print_resolve(char *str, int size, server_resolve_t *resolve)
 {
-        int filled = 0;
+    int filled = 0;
 
-        GF_VALIDATE_OR_GOTO ("server", str, out);
-
-        if (!resolve) {
-                snprintf (str, size, "<nul>");
-                return;
-        }
+    GF_VALIDATE_OR_GOTO("server", str, out);
 
-        filled += snprintf (str + filled, size - filled,
-                            " Resolve={");
-        if (resolve->fd_no != -1)
-                filled += snprintf (str + filled, size - filled,
-                                    "fd=%"PRId64",", (uint64_t) resolve->fd_no);
-        if (resolve->bname)
-                filled += snprintf (str + filled, size - filled,
-                                    "bname=%s,", resolve->bname);
-        if (resolve->path)
-                filled += snprintf (str + filled, size - filled,
-                                    "path=%s", resolve->path);
-
-        snprintf (str + filled, size - filled, "}");
-out:
+    if (!resolve) {
+        snprintf(str, size, "<nul>");
         return;
+    }
+
+    filled += snprintf(str + filled, size - filled, " Resolve={");
+    if (resolve->fd_no != -1)
+        filled += snprintf(str + filled, size - filled, "fd=%" PRId64 ",",
+                           (uint64_t)resolve->fd_no);
+    if (resolve->bname)
+        filled += snprintf(str + filled, size - filled, "bname=%s,",
+                           resolve->bname);
+    if (resolve->path)
+        filled += snprintf(str + filled, size - filled, "path=%s",
+                           resolve->path);
+
+    snprintf(str + filled, size - filled, "}");
+out:
+    return;
 }
 
-
 void
-server_print_loc (char *str, int size, loc_t *loc)
+server_print_loc(char *str, int size, loc_t *loc)
 {
-        int filled = 0;
+    int filled = 0;
 
-        GF_VALIDATE_OR_GOTO ("server", str, out);
+    GF_VALIDATE_OR_GOTO("server", str, out);
 
-        if (!loc) {
-                snprintf (str, size, "<nul>");
-                return;
-        }
+    if (!loc) {
+        snprintf(str, size, "<nul>");
+        return;
+    }
 
-        filled += snprintf (str + filled, size - filled,
-                            " Loc={");
+    filled += snprintf(str + filled, size - filled, " Loc={");
 
-        if (loc->path)
-                filled += snprintf (str + filled, size - filled,
-                                    "path=%s,", loc->path);
-        if (loc->inode)
-                filled += snprintf (str + filled, size - filled,
-                                    "inode=%p,", loc->inode);
-        if (loc->parent)
-                filled += snprintf (str + filled, size - filled,
-                                    "parent=%p", loc->parent);
+    if (loc->path)
+        filled += snprintf(str + filled, size - filled, "path=%s,", loc->path);
+    if (loc->inode)
+        filled += snprintf(str + filled, size - filled, "inode=%p,",
+                           loc->inode);
+    if (loc->parent)
+        filled += snprintf(str + filled, size - filled, "parent=%p",
+                           loc->parent);
 
-        snprintf (str + filled, size - filled, "}");
+    snprintf(str + filled, size - filled, "}");
 out:
-        return;
+    return;
 }
 
-
 void
-server_print_params (char *str, int size, server_state_t *state)
+server_print_params(char *str, int size, server_state_t *state)
 {
-        int filled = 0;
-
-        GF_VALIDATE_OR_GOTO ("server", str, out);
-
-        filled += snprintf (str + filled, size - filled,
-                            " Params={");
-
-        if (state->fd)
-                filled += snprintf (str + filled, size - filled,
-                                    "fd=%p,", state->fd);
-        if (state->valid)
-                filled += snprintf (str + filled, size - filled,
-                                    "valid=%d,", state->valid);
-        if (state->flags)
-                filled += snprintf (str + filled, size - filled,
-                                    "flags=%d,", state->flags);
-        if (state->wbflags)
-                filled += snprintf (str + filled, size - filled,
-                                    "wbflags=%d,", state->wbflags);
-        if (state->size)
-                filled += snprintf (str + filled, size - filled,
-                                    "size=%zu,", state->size);
-        if (state->offset)
-                filled += snprintf (str + filled, size - filled,
-                                    "offset=%"PRId64",", state->offset);
-        if (state->cmd)
-                filled += snprintf (str + filled, size - filled,
-                                    "cmd=%d,", state->cmd);
-        if (state->type)
-                filled += snprintf (str + filled, size - filled,
-                                    "type=%d,", state->type);
-        if (state->name)
-                filled += snprintf (str + filled, size - filled,
-                                    "name=%s,", state->name);
-        if (state->mask)
-                filled += snprintf (str + filled, size - filled,
-                                    "mask=%d,", state->mask);
-        if (state->volume)
-                filled += snprintf (str + filled, size - filled,
-                                    "volume=%s,", state->volume);
+    int filled = 0;
+
+    GF_VALIDATE_OR_GOTO("server", str, out);
+
+    filled += snprintf(str + filled, size - filled, " Params={");
+
+    if (state->fd)
+        filled += snprintf(str + filled, size - filled, "fd=%p,", state->fd);
+    if (state->valid)
+        filled += snprintf(str + filled, size - filled, "valid=%d,",
+                           state->valid);
+    if (state->flags)
+        filled += snprintf(str + filled, size - filled, "flags=%d,",
+                           state->flags);
+    if (state->wbflags)
+        filled += snprintf(str + filled, size - filled, "wbflags=%d,",
+                           state->wbflags);
+    if (state->size)
+        filled += snprintf(str + filled, size - filled, "size=%zu,",
+                           state->size);
+    if (state->offset)
+        filled += snprintf(str + filled, size - filled, "offset=%" PRId64 ",",
+                           state->offset);
+    if (state->cmd)
+        filled += snprintf(str + filled, size - filled, "cmd=%d,", state->cmd);
+    if (state->type)
+        filled += snprintf(str + filled, size - filled, "type=%d,",
+                           state->type);
+    if (state->name)
+        filled += snprintf(str + filled, size - filled, "name=%s,",
+                           state->name);
+    if (state->mask)
+        filled += snprintf(str + filled, size - filled, "mask=%d,",
+                           state->mask);
+    if (state->volume)
+        filled += snprintf(str + filled, size - filled, "volume=%s,",
+                           state->volume);
 
 /* FIXME
         snprintf (str + filled, size - filled,
                   "bound_xl=%s}", state->client->bound_xl->name);
 */
 out:
-        return;
+    return;
 }
 
-
 int
-server_resolve_is_empty (server_resolve_t *resolve)
+server_resolve_is_empty(server_resolve_t *resolve)
 {
-        if (resolve->fd_no != -1)
-                return 0;
+    if (resolve->fd_no != -1)
+        return 0;
 
-        if (resolve->path != 0)
-                return 0;
+    if (resolve->path != 0)
+        return 0;
 
-        if (resolve->bname != 0)
-                return 0;
+    if (resolve->bname != 0)
+        return 0;
 
-        return 1;
+    return 1;
 }
 
-
 void
-server_print_reply (call_frame_t *frame, int op_ret, int op_errno)
+server_print_reply(call_frame_t *frame, int op_ret, int op_errno)
 {
-        server_conf_t   *conf = NULL;
-        server_state_t  *state = NULL;
-        xlator_t        *this = NULL;
-        char             caller[512];
-        char             fdstr[32];
-        char            *op = "UNKNOWN";
+    server_conf_t *conf = NULL;
+    server_state_t *state = NULL;
+    xlator_t *this = NULL;
+    char caller[512];
+    char fdstr[32];
+    char *op = "UNKNOWN";
 
-        GF_VALIDATE_OR_GOTO ("server", frame, out);
+    GF_VALIDATE_OR_GOTO("server", frame, out);
 
-        this = frame->this;
-        conf = this->private;
+    this = frame->this;
+    conf = this->private;
 
-        GF_VALIDATE_OR_GOTO ("server", conf, out);
-        GF_VALIDATE_OR_GOTO ("server", conf->trace, out);
+    GF_VALIDATE_OR_GOTO("server", conf, out);
+    GF_VALIDATE_OR_GOTO("server", conf->trace, out);
 
-        state = CALL_STATE (frame);
+    state = CALL_STATE(frame);
 
-        print_caller (caller, 256, frame);
+    print_caller(caller, 256, frame);
 
-        switch (frame->root->type) {
+    switch (frame->root->type) {
         case GF_OP_TYPE_FOP:
-                op = (char *)gf_fop_list[frame->root->op];
-                break;
+            op = (char *)gf_fop_list[frame->root->op];
+            break;
         default:
-                op = "";
-        }
+            op = "";
+    }
 
-        fdstr[0] = '\0';
-        if (state->fd)
-                snprintf (fdstr, 32, " fd=%p", state->fd);
+    fdstr[0] = '\0';
+    if (state->fd)
+        snprintf(fdstr, 32, " fd=%p", state->fd);
 
-        gf_log (this->name, GF_LOG_INFO,
-                "%s%s => (%d, %d)%s",
-                op, caller, op_ret, op_errno, fdstr);
+    gf_smsg(this->name, GF_LOG_INFO, op_errno, PS_MSG_SERVER_MSG, "op=%s", op,
+            "caller=%s", caller, "op_ret=%d", op_ret, "op_errno=%d", op_errno,
+            "fdstr=%s", fdstr, NULL);
 out:
-        return;
+    return;
 }
 
-
 void
-server_print_request (call_frame_t *frame)
+server_print_request(call_frame_t *frame)
 {
-        server_conf_t   *conf  = NULL;
-        xlator_t        *this  = NULL;
-        server_state_t  *state = NULL;
-        char            *op    = "UNKNOWN";
-        char             resolve_vars[256];
-        char             resolve2_vars[256];
-        char             loc_vars[256];
-        char             loc2_vars[256];
-        char             other_vars[512];
-        char             caller[512];
+    server_conf_t *conf = NULL;
+    xlator_t *this = NULL;
+    server_state_t *state = NULL;
+    char *op = "UNKNOWN";
+    char resolve_vars[256];
+    char resolve2_vars[256];
+    char loc_vars[256];
+    char loc2_vars[256];
+    char other_vars[512];
+    char caller[512];
 
-        GF_VALIDATE_OR_GOTO ("server", frame, out);
+    GF_VALIDATE_OR_GOTO("server", frame, out);
 
-        this = frame->this;
-        conf = this->private;
+    this = frame->this;
+    conf = this->private;
 
-        GF_VALIDATE_OR_GOTO ("server", conf, out);
+    GF_VALIDATE_OR_GOTO("server", conf, out);
 
-        if (!conf->trace)
-                goto out;
+    if (!conf->trace)
+        goto out;
 
-        state = CALL_STATE (frame);
+    state = CALL_STATE(frame);
 
-        memset (resolve_vars, '\0', 256);
-        memset (resolve2_vars, '\0', 256);
-        memset (loc_vars, '\0', 256);
-        memset (loc2_vars, '\0', 256);
-        memset (other_vars, '\0', 256);
+    memset(resolve_vars, '\0', 256);
+    memset(resolve2_vars, '\0', 256);
+    memset(loc_vars, '\0', 256);
+    memset(loc2_vars, '\0', 256);
+    memset(other_vars, '\0', 256);
 
-        print_caller (caller, 256, frame);
+    print_caller(caller, 256, frame);
 
-        if (!server_resolve_is_empty (&state->resolve)) {
-                server_print_resolve (resolve_vars, 256, &state->resolve);
-                server_print_loc (loc_vars, 256, &state->loc);
-        }
+    if (!server_resolve_is_empty(&state->resolve)) {
+        server_print_resolve(resolve_vars, 256, &state->resolve);
+        server_print_loc(loc_vars, 256, &state->loc);
+    }
 
-        if (!server_resolve_is_empty (&state->resolve2)) {
-                server_print_resolve (resolve2_vars, 256, &state->resolve2);
-                server_print_loc (loc2_vars, 256, &state->loc2);
-        }
+    if (!server_resolve_is_empty(&state->resolve2)) {
+        server_print_resolve(resolve2_vars, 256, &state->resolve2);
+        server_print_loc(loc2_vars, 256, &state->loc2);
+    }
 
-        server_print_params (other_vars, 512, state);
+    server_print_params(other_vars, 512, state);
 
-        switch (frame->root->type) {
+    switch (frame->root->type) {
         case GF_OP_TYPE_FOP:
-                op = (char *)gf_fop_list[frame->root->op];
-                break;
+            op = (char *)gf_fop_list[frame->root->op];
+            break;
         default:
-                op = "";
-                break;
-        }
-
-        gf_log (this->name, GF_LOG_INFO,
-                "%s%s%s%s%s%s%s",
-                op, caller,
-                resolve_vars, loc_vars, resolve2_vars, loc2_vars, other_vars);
+            op = "";
+            break;
+    }
+
+    gf_smsg(this->name, GF_LOG_INFO, 0, PS_MSG_SERVER_MSG, "op=%s", op,
+            "caller=%s", caller, "resolve_vars=%s", resolve_vars, "loc_vars=%s",
+            loc_vars, "resolve2_vars=%s", resolve2_vars, "loc2_vars=%s",
+            loc2_vars, "other_vars=%s", other_vars, NULL);
 out:
-        return;
+    return;
 }
 
-
 int
-serialize_rsp_direntp (gf_dirent_t *entries, gfs3_readdirp_rsp *rsp)
+serialize_rsp_direntp(gf_dirent_t *entries, gfs3_readdirp_rsp *rsp)
 {
-        gf_dirent_t         *entry = NULL;
-        gfs3_dirplist       *trav  = NULL;
-        gfs3_dirplist       *prev  = NULL;
-        int                  ret   = -1;
-
-        GF_VALIDATE_OR_GOTO ("server", entries, out);
-        GF_VALIDATE_OR_GOTO ("server", rsp, out);
-
-        list_for_each_entry (entry, &entries->list, list) {
-                trav = GF_CALLOC (1, sizeof (*trav), gf_server_mt_dirent_rsp_t);
-                if (!trav)
-                        goto out;
-
-                trav->d_ino  = entry->d_ino;
-                trav->d_off  = entry->d_off;
-                trav->d_len  = entry->d_len;
-                trav->d_type = entry->d_type;
-                trav->name   = entry->d_name;
-
-                gf_stat_from_iatt (&trav->stat, &entry->d_stat);
-
-                /* if 'dict' is present, pack it */
-                if (entry->dict) {
-                        trav->dict.dict_len = dict_serialized_length (entry->dict);
-                        if (trav->dict.dict_len > UINT_MAX) {
-                                gf_log (THIS->name, GF_LOG_ERROR,
-                                        "failed to get serialized length "
-                                        "of reply dict");
-                                errno = EINVAL;
-                                trav->dict.dict_len = 0;
-                                goto out;
-                        }
-
-                        trav->dict.dict_val = GF_CALLOC (1, trav->dict.dict_len,
-                                                         gf_server_mt_rsp_buf_t);
-                        if (!trav->dict.dict_val) {
-                                errno = ENOMEM;
-                                trav->dict.dict_len = 0;
-                                goto out;
-                        }
-
-                        ret = dict_serialize (entry->dict, trav->dict.dict_val);
-                        if (ret < 0) {
-                                gf_log (THIS->name, GF_LOG_ERROR,
-                                        "failed to serialize reply dict");
-                                errno = -ret;
-                                trav->dict.dict_len = 0;
-                                goto out;
-                        }
-                }
+    gf_dirent_t *entry = NULL;
+    gfs3_dirplist *trav = NULL;
+    gfs3_dirplist *prev = NULL;
+    int ret = -1;
+
+    GF_VALIDATE_OR_GOTO("server", entries, out);
+    GF_VALIDATE_OR_GOTO("server", rsp, out);
+
+    list_for_each_entry(entry, &entries->list, list)
+    {
+        trav = GF_CALLOC(1, sizeof(*trav), gf_server_mt_dirent_rsp_t);
+        if (!trav)
+            goto out;
+
+        trav->d_ino = entry->d_ino;
+        trav->d_off = entry->d_off;
+        trav->d_len = entry->d_len;
+        trav->d_type = entry->d_type;
+        trav->name = entry->d_name;
+
+        gf_stat_from_iatt(&trav->stat, &entry->d_stat);
+
+        /* if 'dict' is present, pack it */
+        if (entry->dict) {
+            ret = dict_allocate_and_serialize(entry->dict,
+                                              (char **)&trav->dict.dict_val,
+                                              &trav->dict.dict_len);
+            if (ret != 0) {
+                gf_smsg(THIS->name, GF_LOG_ERROR, 0, PS_MSG_DICT_SERIALIZE_FAIL,
+                        NULL);
+                errno = -ret;
+                trav->dict.dict_len = 0;
+                goto out;
+            }
+        }
 
-                if (prev)
-                        prev->nextentry = trav;
-                else
-                        rsp->reply = trav;
+        if (prev)
+            prev->nextentry = trav;
+        else
+            rsp->reply = trav;
 
-                prev = trav;
-                trav = NULL;
-        }
+        prev = trav;
+        trav = NULL;
+    }
 
-        ret = 0;
+    ret = 0;
 out:
-        GF_FREE (trav);
+    GF_FREE(trav);
 
-        return ret;
+    return ret;
 }
 
-
 int
-serialize_rsp_dirent (gf_dirent_t *entries, gfs3_readdir_rsp *rsp)
+serialize_rsp_direntp_v2(gf_dirent_t *entries, gfx_readdirp_rsp *rsp)
 {
-        gf_dirent_t   *entry = NULL;
-        gfs3_dirlist  *trav  = NULL;
-        gfs3_dirlist  *prev  = NULL;
-        int           ret    = -1;
-
-        GF_VALIDATE_OR_GOTO ("server", entries, out);
-        GF_VALIDATE_OR_GOTO ("server", rsp, out);
-
-        list_for_each_entry (entry, &entries->list, list) {
-                trav = GF_CALLOC (1, sizeof (*trav), gf_server_mt_dirent_rsp_t);
-                if (!trav)
-                        goto out;
-                trav->d_ino  = entry->d_ino;
-                trav->d_off  = entry->d_off;
-                trav->d_len  = entry->d_len;
-                trav->d_type = entry->d_type;
-                trav->name   = entry->d_name;
-                if (prev)
-                        prev->nextentry = trav;
-                else
-                        rsp->reply = trav;
-
-                prev = trav;
-        }
+    gf_dirent_t *entry = NULL;
+    gfx_dirplist *trav = NULL;
+    gfx_dirplist *prev = NULL;
+    int ret = -1;
+
+    GF_VALIDATE_OR_GOTO("server", entries, out);
+    GF_VALIDATE_OR_GOTO("server", rsp, out);
+
+    list_for_each_entry(entry, &entries->list, list)
+    {
+        trav = GF_CALLOC(1, sizeof(*trav), gf_server_mt_dirent_rsp_t);
+        if (!trav)
+            goto out;
+
+        trav->d_ino = entry->d_ino;
+        trav->d_off = entry->d_off;
+        trav->d_len = entry->d_len;
+        trav->d_type = entry->d_type;
+        trav->name = entry->d_name;
+
+        gfx_stat_from_iattx(&trav->stat, &entry->d_stat);
+        dict_to_xdr(entry->dict, &trav->dict);
+
+        if (prev)
+            prev->nextentry = trav;
+        else
+            rsp->reply = trav;
 
-        ret = 0;
+        prev = trav;
+        trav = NULL;
+    }
+
+    ret = 0;
 out:
-        return ret;
-}
+    GF_FREE(trav);
 
+    return ret;
+}
 
 int
-readdir_rsp_cleanup (gfs3_readdir_rsp *rsp)
+serialize_rsp_dirent(gf_dirent_t *entries, gfs3_readdir_rsp *rsp)
 {
-        gfs3_dirlist  *prev = NULL;
-        gfs3_dirlist  *trav = NULL;
+    gf_dirent_t *entry = NULL;
+    gfs3_dirlist *trav = NULL;
+    gfs3_dirlist *prev = NULL;
+    int ret = -1;
+
+    GF_VALIDATE_OR_GOTO("server", rsp, out);
+    GF_VALIDATE_OR_GOTO("server", entries, out);
+
+    list_for_each_entry(entry, &entries->list, list)
+    {
+        trav = GF_CALLOC(1, sizeof(*trav), gf_server_mt_dirent_rsp_t);
+        if (!trav)
+            goto out;
+        trav->d_ino = entry->d_ino;
+        trav->d_off = entry->d_off;
+        trav->d_len = entry->d_len;
+        trav->d_type = entry->d_type;
+        trav->name = entry->d_name;
+        if (prev)
+            prev->nextentry = trav;
+        else
+            rsp->reply = trav;
 
-        trav = rsp->reply;
         prev = trav;
-        while (trav) {
-                trav = trav->nextentry;
-                GF_FREE (prev);
-                prev = trav;
-        }
+    }
 
-        return 0;
+    ret = 0;
+out:
+    return ret;
 }
 
+int
+serialize_rsp_dirent_v2(gf_dirent_t *entries, gfx_readdir_rsp *rsp)
+{
+    gf_dirent_t *entry = NULL;
+    gfx_dirlist *trav = NULL;
+    gfx_dirlist *prev = NULL;
+    int ret = -1;
+
+    GF_VALIDATE_OR_GOTO("server", rsp, out);
+    GF_VALIDATE_OR_GOTO("server", entries, out);
+
+    list_for_each_entry(entry, &entries->list, list)
+    {
+        trav = GF_CALLOC(1, sizeof(*trav), gf_server_mt_dirent_rsp_t);
+        if (!trav)
+            goto out;
+        trav->d_ino = entry->d_ino;
+        trav->d_off = entry->d_off;
+        trav->d_len = entry->d_len;
+        trav->d_type = entry->d_type;
+        trav->name = entry->d_name;
+        if (prev)
+            prev->nextentry = trav;
+        else
+            rsp->reply = trav;
+
+        prev = trav;
+    }
+
+    ret = 0;
+out:
+    return ret;
+}
 
 int
-readdirp_rsp_cleanup (gfs3_readdirp_rsp *rsp)
+readdir_rsp_cleanup(gfs3_readdir_rsp *rsp)
 {
-        gfs3_dirplist *prev = NULL;
-        gfs3_dirplist *trav = NULL;
+    gfs3_dirlist *prev = NULL;
+    gfs3_dirlist *trav = NULL;
+
+    trav = rsp->reply;
+    prev = trav;
+    while (trav) {
+        trav = trav->nextentry;
+        GF_FREE(prev);
+        prev = trav;
+    }
 
-        trav = rsp->reply;
+    return 0;
+}
+
+int
+readdirp_rsp_cleanup(gfs3_readdirp_rsp *rsp)
+{
+    gfs3_dirplist *prev = NULL;
+    gfs3_dirplist *trav = NULL;
+
+    trav = rsp->reply;
+    prev = trav;
+    while (trav) {
+        trav = trav->nextentry;
+        GF_FREE(prev->dict.dict_val);
+        GF_FREE(prev);
         prev = trav;
-        while (trav) {
-                trav = trav->nextentry;
-                GF_FREE (prev->dict.dict_val);
-                GF_FREE (prev);
-                prev = trav;
-        }
+    }
 
-        return 0;
+    return 0;
 }
 
+int
+readdir_rsp_cleanup_v2(gfx_readdir_rsp *rsp)
+{
+    gfx_dirlist *prev = NULL;
+    gfx_dirlist *trav = NULL;
+
+    trav = rsp->reply;
+    prev = trav;
+    while (trav) {
+        trav = trav->nextentry;
+        GF_FREE(prev);
+        prev = trav;
+    }
+
+    return 0;
+}
 
 int
-gf_server_check_getxattr_cmd (call_frame_t *frame, const char *key)
+readdirp_rsp_cleanup_v2(gfx_readdirp_rsp *rsp)
 {
+    gfx_dirplist *prev = NULL;
+    gfx_dirplist *trav = NULL;
+
+    trav = rsp->reply;
+    prev = trav;
+    while (trav) {
+        trav = trav->nextentry;
+        GF_FREE(prev->dict.pairs.pairs_val);
+        GF_FREE(prev);
+        prev = trav;
+    }
 
-        server_conf_t    *conf = NULL;
-        rpc_transport_t  *xprt = NULL;
+    return 0;
+}
 
-        conf = frame->this->private;
-        if (!conf)
-                return 0;
+static int
+common_rsp_locklist(lock_migration_info_t *locklist, gfs3_locklist **reply)
+{
+    lock_migration_info_t *tmp = NULL;
+    gfs3_locklist *trav = NULL;
+    gfs3_locklist *prev = NULL;
+    int ret = -1;
+
+    GF_VALIDATE_OR_GOTO("server", locklist, out);
+
+    list_for_each_entry(tmp, &locklist->list, list)
+    {
+        trav = GF_CALLOC(1, sizeof(*trav), gf_server_mt_lock_mig_t);
+        if (!trav)
+            goto out;
+
+        switch (tmp->flock.l_type) {
+            case F_RDLCK:
+                tmp->flock.l_type = GF_LK_F_RDLCK;
+                break;
+            case F_WRLCK:
+                tmp->flock.l_type = GF_LK_F_WRLCK;
+                break;
+            case F_UNLCK:
+                tmp->flock.l_type = GF_LK_F_UNLCK;
+                break;
 
-        if (fnmatch ("*list*mount*point*", key, 0) == 0) {
-                /* list all the client protocol connecting to this process */
-                pthread_mutex_lock (&conf->mutex);
-                {
-                        list_for_each_entry (xprt, &conf->xprt_list, list) {
-                                gf_log ("mount-point-list", GF_LOG_INFO,
-                                        "%s", xprt->peerinfo.identifier);
-                        }
-                }
-                pthread_mutex_unlock (&conf->mutex);
+            default:
+                gf_smsg(THIS->name, GF_LOG_ERROR, 0, PS_MSG_LOCK_ERROR,
+                        "lock_type=%" PRId32, tmp->flock.l_type, NULL);
+                break;
         }
 
-        /* Add more options/keys here */
+        gf_proto_flock_from_flock(&trav->flock, &tmp->flock);
 
-        return 0;
+        trav->lk_flags = tmp->lk_flags;
+
+        trav->client_uid = tmp->client_uid;
+
+        if (prev)
+            prev->nextentry = trav;
+        else
+            *reply = trav;
+
+        prev = trav;
+        trav = NULL;
+    }
+
+    ret = 0;
+out:
+    GF_FREE(trav);
+    return ret;
 }
 
+int
+serialize_rsp_locklist(lock_migration_info_t *locklist,
+                       gfs3_getactivelk_rsp *rsp)
+{
+    int ret = 0;
 
+    GF_VALIDATE_OR_GOTO("server", rsp, out);
+    ret = common_rsp_locklist(locklist, &rsp->reply);
+out:
+    return ret;
+}
 int
-gf_server_check_setxattr_cmd (call_frame_t *frame, dict_t *dict)
+serialize_rsp_locklist_v2(lock_migration_info_t *locklist,
+                          gfx_getactivelk_rsp *rsp)
 {
+    int ret = 0;
 
-        server_conf_t    *conf        = NULL;
-        rpc_transport_t  *xprt        = NULL;
-        uint64_t          total_read  = 0;
-        uint64_t          total_write = 0;
+    GF_VALIDATE_OR_GOTO("server", rsp, out);
+    ret = common_rsp_locklist(locklist, &rsp->reply);
+out:
+    return ret;
+}
 
-        conf = frame->this->private;
-        if (!conf || !dict)
-                return 0;
+int
+getactivelkinfo_rsp_cleanup(gfs3_getactivelk_rsp *rsp)
+{
+    gfs3_locklist *prev = NULL;
+    gfs3_locklist *trav = NULL;
 
-        if (dict_foreach_fnmatch (dict, "*io*stat*dump",
-                                  dict_null_foreach_fn, NULL ) > 0) {
-                list_for_each_entry (xprt, &conf->xprt_list, list) {
-                        total_read  += xprt->total_bytes_read;
-                        total_write += xprt->total_bytes_write;
-                }
-                gf_log ("stats", GF_LOG_INFO,
-                        "total-read %"PRIu64", total-write %"PRIu64,
-                        total_read, total_write);
-        }
+    trav = rsp->reply;
+    prev = trav;
 
-        return 0;
+    while (trav) {
+        trav = trav->nextentry;
+        GF_FREE(prev);
+        prev = trav;
+    }
+
+    return 0;
 }
+int
+getactivelkinfo_rsp_cleanup_v2(gfx_getactivelk_rsp *rsp)
+{
+    gfs3_locklist *prev = NULL;
+    gfs3_locklist *trav = NULL;
 
+    trav = rsp->reply;
+    prev = trav;
 
-gf_boolean_t
-server_cancel_grace_timer (xlator_t *this, client_t *client)
+    while (trav) {
+        trav = trav->nextentry;
+        GF_FREE(prev);
+        prev = trav;
+    }
+
+    return 0;
+}
+
+int
+gf_server_check_getxattr_cmd(call_frame_t *frame, const char *key)
 {
-        server_ctx_t  *serv_ctx  = NULL;
-        gf_timer_t    *timer     = NULL;
-        gf_boolean_t   cancelled = _gf_false;
-
-        if (!this || !client) {
-                gf_log (THIS->name, GF_LOG_ERROR,
-                        "Invalid arguments to cancel connection timer");
-                return cancelled;
-        }
+    server_conf_t *conf = NULL;
+    rpc_transport_t *xprt = NULL;
 
-        serv_ctx = server_ctx_get (client, client->this);
+    conf = frame->this->private;
+    if (!conf)
+        return 0;
 
-        if (serv_ctx == NULL) {
-                gf_log (this->name, GF_LOG_INFO, "server_ctx_get() failed");
-                goto out;
+    if (fnmatch("*list*mount*point*", key, 0) == 0) {
+        /* list all the client protocol connecting to this process */
+        pthread_mutex_lock(&conf->mutex);
+        {
+            list_for_each_entry(xprt, &conf->xprt_list, list)
+            {
+                gf_smsg("mount-point-list", GF_LOG_INFO, 0,
+                        PS_MSG_MOUNT_PT_FAIL, "identifier=%s",
+                        xprt->peerinfo.identifier, NULL);
+            }
         }
+        pthread_mutex_unlock(&conf->mutex);
+    }
 
-        LOCK (&serv_ctx->fdtable_lock);
+    /* Add more options/keys here */
+
+    return 0;
+}
+
+int
+gf_server_check_setxattr_cmd(call_frame_t *frame, dict_t *dict)
+{
+    server_conf_t *conf = NULL;
+    rpc_transport_t *xprt = NULL;
+    uint64_t total_read = 0;
+    uint64_t total_write = 0;
+
+    conf = frame->this->private;
+    if (!conf || !dict)
+        return 0;
+
+    if (dict_foreach_fnmatch(dict, "*io*stat*dump", dict_null_foreach_fn,
+                             NULL) > 0) {
+        list_for_each_entry(xprt, &conf->xprt_list, list)
         {
-                if (serv_ctx->grace_timer) {
-                        timer = serv_ctx->grace_timer;
-                        serv_ctx->grace_timer = NULL;
-                }
+            total_read += xprt->total_bytes_read;
+            total_write += xprt->total_bytes_write;
         }
-        UNLOCK (&serv_ctx->fdtable_lock);
+        gf_smsg("stats", GF_LOG_INFO, 0, PS_MSG_RW_STAT, "total-read=%" PRIu64,
+                total_read, "total-write=%" PRIu64, total_write, NULL);
+    }
 
-        if (timer) {
-                gf_timer_call_cancel (this->ctx, timer);
-                cancelled = _gf_true;
-        }
-out:
-        return cancelled;
+    return 0;
 }
 
-server_ctx_t*
-server_ctx_get (client_t *client, xlator_t *xlator)
+server_ctx_t *
+server_ctx_get(client_t *client, xlator_t *xlator)
 {
-        void *tmp = NULL;
-        server_ctx_t *ctx = NULL;
+    void *tmp = NULL;
+    server_ctx_t *ctx = NULL;
+    server_ctx_t *setted_ctx = NULL;
 
-        client_ctx_get (client, xlator, &tmp);
+    client_ctx_get(client, xlator, &tmp);
 
-        ctx = tmp;
+    ctx = tmp;
 
-        if (ctx != NULL)
-                goto out;
+    if (ctx != NULL)
+        goto out;
 
-        ctx = GF_CALLOC (1, sizeof (server_ctx_t), gf_server_mt_server_conf_t);
+    ctx = GF_CALLOC(1, sizeof(server_ctx_t), gf_server_mt_server_conf_t);
 
-        if (ctx == NULL)
-                goto out;
+    if (ctx == NULL)
+        goto out;
 
-     /* ctx->lk_version = 0; redundant */
-        ctx->fdtable = gf_fd_fdtable_alloc ();
+    ctx->fdtable = gf_fd_fdtable_alloc();
 
-        if (ctx->fdtable == NULL) {
-                GF_FREE (ctx);
-                ctx = NULL;
-                goto out;
-        }
+    if (ctx->fdtable == NULL) {
+        GF_FREE(ctx);
+        ctx = NULL;
+        goto out;
+    }
 
-        LOCK_INIT (&ctx->fdtable_lock);
+    LOCK_INIT(&ctx->fdtable_lock);
 
-        if (client_ctx_set (client, xlator, ctx) != 0) {
-              LOCK_DESTROY (&ctx->fdtable_lock);
-              GF_FREE (ctx->fdtable);
-              GF_FREE (ctx);
-              ctx = NULL;
-        }
+    setted_ctx = client_ctx_set(client, xlator, ctx);
+    if (ctx != setted_ctx) {
+        LOCK_DESTROY(&ctx->fdtable_lock);
+        GF_FREE(ctx->fdtable);
+        GF_FREE(ctx);
+        ctx = setted_ctx;
+    }
 
 out:
-        return ctx;
+    return ctx;
 }